From f4cfc77a577e0dfd152878830294c8eb3fe47861 Mon Sep 17 00:00:00 2001 From: Jayryn <77808206+Jayryn@users.noreply.github.com> Date: Tue, 28 May 2024 09:14:30 +0200 Subject: [PATCH 0001/2091] Check if database symlink already exists test -L $object [object exists and is a symbolic link (same as -h)] It is not recommended to use -h [True if file exists and is a symbolic link. This operator is retained for compatibility with previous versions of this program. Do not rely on its existence; use -L instead.] --- debian/postinst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/debian/postinst b/debian/postinst index f82cbfff..223d322b 100644 --- a/debian/postinst +++ b/debian/postinst @@ -25,8 +25,10 @@ case "$1" in # and permissions for the config. mkdir -v -p "$CONDUWUIT_DATABASE_PATH" - # symlink the previous location for compatibility - ln -s -v "$CONDUWUIT_DATABASE_PATH" "/var/lib/matrix-conduit" + # symlink the previous location for compatibility if it does not exist yet. + if ! test -L "/var/lib/matrix-conduit" ; then + ln -s -v "$CONDUWUIT_DATABASE_PATH" "/var/lib/matrix-conduit" + fi chown -v conduwuit:conduwuit -R "$CONDUWUIT_DATABASE_PATH" chown -v conduwuit:conduwuit -R "$CONDUWUIT_CONFIG_PATH" From dd1d8fa76013ba7a923b657053a2681cf884cc43 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 29 May 2024 17:21:59 +0000 Subject: [PATCH 0002/2091] upgrade to hyper-util 0.1.5 Signed-off-by: Jason Volk --- Cargo.lock | 48 +++++++++++++++++++++++++----------------------- Cargo.toml | 9 ++++++++- 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16315310..29b03a71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -97,9 +97,9 @@ checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" [[package]] name = "async-compression" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c90a406b4495d129f00461241616194cb8a032c8d1c53c657f0961d5f8e0498" +checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" dependencies = [ "brotli", "flate2", @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" dependencies = [ "addr2line", "cc", @@ -514,9 +514,9 @@ dependencies = [ [[package]] name = "clang-sys" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a483f3cbf7cec2e153d424d0e92329d816becc6421389bd494375c6065921b9b" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", @@ -740,6 +740,8 @@ dependencies = [ "conduit_service", "http 1.1.0", "http-body-util", + "hyper 1.3.1", + "hyper-util", "log", "regex", "ruma", @@ -1244,9 +1246,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glob" @@ -1623,9 +1625,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d8d52be92d09acc2e01dddb7fde3ad983fc6489c7db4837e605bc3fca4cb63e" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", "futures-channel", @@ -1927,9 +1929,9 @@ checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "loole" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6725f0feab07fcf90f6de5417c06d7fef976fa6e5912fa9e21cb5e4dc6ae5da" +checksum = "ad95468e4700cb37d8d1f198050db18cebe55e4b4c8aa9180a715deedb2f8965" [[package]] name = "lru-cache" @@ -2177,9 +2179,9 @@ dependencies = [ [[package]] name = "object" -version = "0.32.2" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" dependencies = [ "memchr", ] @@ -3741,9 +3743,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", @@ -3770,9 +3772,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", @@ -3889,7 +3891,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.8", + "winnow 0.6.9", ] [[package]] @@ -4564,9 +4566,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c52e9c97a68071b23e836c9380edae937f17b9c4667bd021973efc689f618d" +checksum = "86c949fede1d13936a99f14fafd3e76fd642b556dd2ce96287fbe2e0151bfac6" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index bcfbda32..83f81817 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -216,7 +216,14 @@ features = [ ] [workspace.dependencies.hyper-util] -version = "0.1.4" +version = "0.1.5" +features = [ + "client", + "server-auto", + "server-graceful", + "service", + "tokio", +] # to support multiple variations of setting a config option [workspace.dependencies.either] From faa2b95c84794f993fde68588c2ff6d59d66fdcf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 25 May 2024 21:24:17 +0000 Subject: [PATCH 0003/2091] add unwrap_infallible tool Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 5d4e5cba..582263a9 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -279,3 +279,15 @@ pub fn available_parallelism() -> usize { .expect("Unable to query for available parallelism.") .get() } + +/// Boilerplate for wraps which are typed to never error. +/// +/// * +#[must_use] +#[inline(always)] +pub fn unwrap_infallible(result: Result) -> T { + match result { + Ok(val) => val, + Err(err) => match err {}, + } +} From 0baa57f5d95c081136625596ace67d2e93e81410 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 29 May 2024 16:59:20 +0000 Subject: [PATCH 0004/2091] add back unix socket listener. Signed-off-by: Jason Volk --- src/router/Cargo.toml | 2 + src/router/run.rs | 12 ++-- src/router/serve.rs | 133 +++++++++++++++++++++++++++++++----------- 3 files changed, 108 insertions(+), 39 deletions(-) diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index dae9d14c..4b197d0a 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -66,6 +66,8 @@ bytes.workspace = true clap.workspace = true http-body-util.workspace = true http.workspace = true +hyper.workspace = true +hyper-util.workspace = true regex.workspace = true ruma.workspace = true sentry.optional = true diff --git a/src/router/run.rs b/src/router/run.rs index 2603c04a..e6149fa2 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration}; use axum_server::Handle as ServerHandle; use tokio::{ signal, - sync::oneshot::{self, Sender}, + sync::broadcast::{self, Sender}, }; use tracing::{debug, info, warn}; @@ -40,14 +40,16 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { .insert(handle.clone()); server.interrupt.store(false, Ordering::Release); - let (tx, rx) = oneshot::channel::<()>(); - let sigs = server.runtime().spawn(sighandle(server.clone(), tx)); + let (tx, _) = broadcast::channel::<()>(1); + let sigs = server + .runtime() + .spawn(sighandle(server.clone(), tx.clone())); // Prepare to serve http clients let res; // Serve clients if cfg!(unix) && config.unix_socket_path.is_some() { - res = serve::unix_socket(&server, app, rx).await; + res = serve::unix_socket(&server, app, tx.subscribe()).await; } else if config.tls.is_some() { res = serve::tls(&server, app, handle.clone(), addrs).await; } else { @@ -66,7 +68,7 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { _ = services().admin.handle.lock().await.take(); debug_info!("Finished"); - Ok(res?) + res } /// Async initializations diff --git a/src/router/serve.rs b/src/router/serve.rs index 37ed9902..bddd9a2d 100644 --- a/src/router/serve.rs +++ b/src/router/serve.rs @@ -1,27 +1,31 @@ -#[cfg(unix)] -use std::fs::Permissions; // only for UNIX sockets stuff and *nix container checks -#[cfg(unix)] -use std::os::unix::fs::PermissionsExt as _; use std::{ - io, net::SocketAddr, + path::Path, sync::{atomic::Ordering, Arc}, }; -use axum::Router; +use axum::{extract::Request, routing::IntoMakeService, Router}; use axum_server::{bind, bind_rustls, tls_rustls::RustlsConfig, Handle as ServerHandle}; #[cfg(feature = "axum_dual_protocol")] use axum_server_dual_protocol::ServerExt; -use conduit::{debug_info, Server}; +use conduit::{debug_error, debug_info, utils, Error, Result, Server}; +use hyper::{body::Incoming, service::service_fn}; +use hyper_util::{ + rt::{TokioExecutor, TokioIo}, + server, +}; use tokio::{ - sync::oneshot::{self}, + fs, + sync::broadcast::{self}, task::JoinSet, }; +use tower::{Service, ServiceExt}; use tracing::{debug, info, warn}; +use utils::unwrap_infallible; pub(crate) async fn plain( - server: &Arc, app: axum::routing::IntoMakeService, handle: ServerHandle, addrs: Vec, -) -> io::Result<()> { + server: &Arc, app: IntoMakeService, handle: ServerHandle, addrs: Vec, +) -> Result<()> { let mut join_set = JoinSet::new(); for addr in &addrs { join_set.spawn_on(bind(*addr).handle(handle.clone()).serve(app.clone()), server.runtime()); @@ -48,8 +52,8 @@ pub(crate) async fn plain( } pub(crate) async fn tls( - server: &Arc, app: axum::routing::IntoMakeService, handle: ServerHandle, addrs: Vec, -) -> io::Result<()> { + server: &Arc, app: IntoMakeService, handle: ServerHandle, addrs: Vec, +) -> Result<()> { let config = &server.config; let tls = config.tls.as_ref().expect("TLS configuration"); @@ -107,31 +111,92 @@ pub(crate) async fn tls( } #[cfg(unix)] -#[allow(unused_variables)] pub(crate) async fn unix_socket( - server: &Arc, app: axum::routing::IntoMakeService, rx: oneshot::Receiver<()>, -) -> io::Result<()> { - let config = &server.config; - let path = config.unix_socket_path.as_ref().unwrap(); - - if path.exists() { - warn!( - "UNIX socket path {:#?} already exists (unclean shutdown?), attempting to remove it.", - path.display() - ); - tokio::fs::remove_file(&path).await?; + server: &Arc, app: IntoMakeService, mut shutdown: broadcast::Receiver<()>, +) -> Result<()> { + let mut tasks = JoinSet::<()>::new(); + let executor = TokioExecutor::new(); + let builder = server::conn::auto::Builder::new(executor); + let listener = unix_socket_init(server).await?; + loop { + let app = app.clone(); + let builder = builder.clone(); + tokio::select! { + _sig = shutdown.recv() => break, + accept = listener.accept() => match accept { + Ok(conn) => unix_socket_accept(server, &listener, &mut tasks, app, builder, conn).await, + Err(err) => debug_error!(?listener, "accept error: {err}"), + }, + } } - tokio::fs::create_dir_all(path.parent().unwrap()).await?; - - let socket_perms = config.unix_socket_perms.to_string(); - let octal_perms = u32::from_str_radix(&socket_perms, 8).unwrap(); - tokio::fs::set_permissions(&path, Permissions::from_mode(octal_perms)) - .await - .unwrap(); - - let bind = tokio::net::UnixListener::bind(path)?; - info!("Listening at {:?}", path); + drop(listener); + tasks.shutdown().await; Ok(()) } + +#[cfg(unix)] +async fn unix_socket_accept( + server: &Arc, listener: &tokio::net::UnixListener, tasks: &mut JoinSet<()>, + mut app: IntoMakeService, builder: server::conn::auto::Builder, + conn: (tokio::net::UnixStream, tokio::net::unix::SocketAddr), +) { + let (socket, remote) = conn; + let socket = TokioIo::new(socket); + debug!(?listener, ?socket, ?remote, "accepted"); + + let called = unwrap_infallible(app.call(()).await); + let handler = service_fn(move |req: Request| called.clone().oneshot(req)); + + let task = async move { + builder + .serve_connection(socket, handler) + .await + .map_err(|e| debug_error!(?remote, "connection error: {e}")) + .expect("connection error"); + }; + + _ = tasks.spawn_on(task, server.runtime()); + while tasks.try_join_next().is_some() {} +} + +#[cfg(unix)] +async fn unix_socket_init(server: &Arc) -> Result { + use std::os::unix::fs::PermissionsExt; + + let config = &server.config; + let path = config + .unix_socket_path + .as_ref() + .expect("failed to extract configured unix socket path"); + + if path.exists() { + warn!("Removing existing UNIX socket {:#?} (unclean shutdown?)...", path.display()); + fs::remove_file(&path) + .await + .map_err(|e| warn!("Failed to remove existing UNIX socket: {e}")) + .unwrap(); + } + + let dir = path.parent().unwrap_or_else(|| Path::new("/")); + if let Err(e) = fs::create_dir_all(dir).await { + return Err(Error::Err(format!("Failed to create {dir:?} for socket {path:?}: {e}"))); + } + + let listener = tokio::net::UnixListener::bind(path); + if let Err(e) = listener { + return Err(Error::Err(format!("Failed to bind listener {path:?}: {e}"))); + } + + let socket_perms = config.unix_socket_perms.to_string(); + let octal_perms = u32::from_str_radix(&socket_perms, 8).expect("failed to convert octal permissions"); + let perms = std::fs::Permissions::from_mode(octal_perms); + if let Err(e) = fs::set_permissions(&path, perms).await { + return Err(Error::Err(format!("Failed to set socket {path:?} permissions: {e}"))); + } + + info!("Listening at {:?}", path); + + Ok(listener.unwrap()) +} From 2e45cb281a8728eceaf4605b0d8ce16b6b30613a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 30 May 2024 22:38:39 +0000 Subject: [PATCH 0005/2091] split router::serve units. Signed-off-by: Jason Volk --- src/router/mod.rs | 10 +- src/router/run.rs | 12 +-- src/router/serve.rs | 202 -------------------------------------- src/router/serve/mod.rs | 26 +++++ src/router/serve/plain.rs | 38 +++++++ src/router/serve/tls.rs | 68 +++++++++++++ src/router/serve/unix.rs | 107 ++++++++++++++++++++ 7 files changed, 245 insertions(+), 218 deletions(-) delete mode 100644 src/router/serve.rs create mode 100644 src/router/serve/mod.rs create mode 100644 src/router/serve/plain.rs create mode 100644 src/router/serve/tls.rs create mode 100644 src/router/serve/unix.rs diff --git a/src/router/mod.rs b/src/router/mod.rs index 6467d5ee..9bc27356 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -1,8 +1,8 @@ -pub(crate) mod layers; -pub(crate) mod request; -pub(crate) mod router; -pub(crate) mod run; -pub(crate) mod serve; +mod layers; +mod request; +mod router; +mod run; +mod serve; extern crate conduit_core as conduit; diff --git a/src/router/run.rs b/src/router/run.rs index e6149fa2..d20f8669 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -24,9 +24,7 @@ use crate::{layers, serve}; #[tracing::instrument(skip_all)] #[allow(clippy::let_underscore_must_use)] // various of these are intended pub(crate) async fn run(server: Arc) -> Result<(), Error> { - let config = &server.config; let app = layers::build(&server)?; - let addrs = config.get_bind_addrs(); // Install the admin room callback here for now _ = services().admin.handle.lock().await.insert(admin::handle); @@ -45,16 +43,8 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { .runtime() .spawn(sighandle(server.clone(), tx.clone())); - // Prepare to serve http clients - let res; // Serve clients - if cfg!(unix) && config.unix_socket_path.is_some() { - res = serve::unix_socket(&server, app, tx.subscribe()).await; - } else if config.tls.is_some() { - res = serve::tls(&server, app, handle.clone(), addrs).await; - } else { - res = serve::plain(&server, app, handle.clone(), addrs).await; - } + let res = serve::serve(&server, app, handle, tx.subscribe()).await; // Join the signal handler before we leave. sigs.abort(); diff --git a/src/router/serve.rs b/src/router/serve.rs deleted file mode 100644 index bddd9a2d..00000000 --- a/src/router/serve.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::{ - net::SocketAddr, - path::Path, - sync::{atomic::Ordering, Arc}, -}; - -use axum::{extract::Request, routing::IntoMakeService, Router}; -use axum_server::{bind, bind_rustls, tls_rustls::RustlsConfig, Handle as ServerHandle}; -#[cfg(feature = "axum_dual_protocol")] -use axum_server_dual_protocol::ServerExt; -use conduit::{debug_error, debug_info, utils, Error, Result, Server}; -use hyper::{body::Incoming, service::service_fn}; -use hyper_util::{ - rt::{TokioExecutor, TokioIo}, - server, -}; -use tokio::{ - fs, - sync::broadcast::{self}, - task::JoinSet, -}; -use tower::{Service, ServiceExt}; -use tracing::{debug, info, warn}; -use utils::unwrap_infallible; - -pub(crate) async fn plain( - server: &Arc, app: IntoMakeService, handle: ServerHandle, addrs: Vec, -) -> Result<()> { - let mut join_set = JoinSet::new(); - for addr in &addrs { - join_set.spawn_on(bind(*addr).handle(handle.clone()).serve(app.clone()), server.runtime()); - } - - info!("Listening on {addrs:?}"); - while join_set.join_next().await.is_some() {} - - let spawn_active = server.requests_spawn_active.load(Ordering::Relaxed); - let handle_active = server.requests_handle_active.load(Ordering::Relaxed); - debug_info!( - spawn_finished = server.requests_spawn_finished.load(Ordering::Relaxed), - handle_finished = server.requests_handle_finished.load(Ordering::Relaxed), - panics = server.requests_panic.load(Ordering::Relaxed), - spawn_active, - handle_active, - "Stopped listening on {addrs:?}", - ); - - debug_assert!(spawn_active == 0, "active request tasks are not joined"); - debug_assert!(handle_active == 0, "active request handles still pending"); - - Ok(()) -} - -pub(crate) async fn tls( - server: &Arc, app: IntoMakeService, handle: ServerHandle, addrs: Vec, -) -> Result<()> { - let config = &server.config; - let tls = config.tls.as_ref().expect("TLS configuration"); - - debug!( - "Using direct TLS. Certificate path {} and certificate private key path {}", - &tls.certs, &tls.key - ); - info!( - "Note: It is strongly recommended that you use a reverse proxy instead of running conduwuit directly with TLS." - ); - let conf = RustlsConfig::from_pem_file(&tls.certs, &tls.key).await?; - - if cfg!(feature = "axum_dual_protocol") { - info!( - "conduwuit was built with axum_dual_protocol feature to listen on both HTTP and HTTPS. This will only \ - take effect if `dual_protocol` is enabled in `[global.tls]`" - ); - } - - let mut join_set = JoinSet::new(); - if cfg!(feature = "axum_dual_protocol") && tls.dual_protocol { - #[cfg(feature = "axum_dual_protocol")] - for addr in &addrs { - join_set.spawn_on( - axum_server_dual_protocol::bind_dual_protocol(*addr, conf.clone()) - .set_upgrade(false) - .handle(handle.clone()) - .serve(app.clone()), - server.runtime(), - ); - } - } else { - for addr in &addrs { - join_set.spawn_on( - bind_rustls(*addr, conf.clone()) - .handle(handle.clone()) - .serve(app.clone()), - server.runtime(), - ); - } - } - - if cfg!(feature = "axum_dual_protocol") && tls.dual_protocol { - warn!( - "Listening on {:?} with TLS certificate {} and supporting plain text (HTTP) connections too (insecure!)", - addrs, &tls.certs - ); - } else { - info!("Listening on {:?} with TLS certificate {}", addrs, &tls.certs); - } - - while join_set.join_next().await.is_some() {} - - Ok(()) -} - -#[cfg(unix)] -pub(crate) async fn unix_socket( - server: &Arc, app: IntoMakeService, mut shutdown: broadcast::Receiver<()>, -) -> Result<()> { - let mut tasks = JoinSet::<()>::new(); - let executor = TokioExecutor::new(); - let builder = server::conn::auto::Builder::new(executor); - let listener = unix_socket_init(server).await?; - loop { - let app = app.clone(); - let builder = builder.clone(); - tokio::select! { - _sig = shutdown.recv() => break, - accept = listener.accept() => match accept { - Ok(conn) => unix_socket_accept(server, &listener, &mut tasks, app, builder, conn).await, - Err(err) => debug_error!(?listener, "accept error: {err}"), - }, - } - } - - drop(listener); - tasks.shutdown().await; - - Ok(()) -} - -#[cfg(unix)] -async fn unix_socket_accept( - server: &Arc, listener: &tokio::net::UnixListener, tasks: &mut JoinSet<()>, - mut app: IntoMakeService, builder: server::conn::auto::Builder, - conn: (tokio::net::UnixStream, tokio::net::unix::SocketAddr), -) { - let (socket, remote) = conn; - let socket = TokioIo::new(socket); - debug!(?listener, ?socket, ?remote, "accepted"); - - let called = unwrap_infallible(app.call(()).await); - let handler = service_fn(move |req: Request| called.clone().oneshot(req)); - - let task = async move { - builder - .serve_connection(socket, handler) - .await - .map_err(|e| debug_error!(?remote, "connection error: {e}")) - .expect("connection error"); - }; - - _ = tasks.spawn_on(task, server.runtime()); - while tasks.try_join_next().is_some() {} -} - -#[cfg(unix)] -async fn unix_socket_init(server: &Arc) -> Result { - use std::os::unix::fs::PermissionsExt; - - let config = &server.config; - let path = config - .unix_socket_path - .as_ref() - .expect("failed to extract configured unix socket path"); - - if path.exists() { - warn!("Removing existing UNIX socket {:#?} (unclean shutdown?)...", path.display()); - fs::remove_file(&path) - .await - .map_err(|e| warn!("Failed to remove existing UNIX socket: {e}")) - .unwrap(); - } - - let dir = path.parent().unwrap_or_else(|| Path::new("/")); - if let Err(e) = fs::create_dir_all(dir).await { - return Err(Error::Err(format!("Failed to create {dir:?} for socket {path:?}: {e}"))); - } - - let listener = tokio::net::UnixListener::bind(path); - if let Err(e) = listener { - return Err(Error::Err(format!("Failed to bind listener {path:?}: {e}"))); - } - - let socket_perms = config.unix_socket_perms.to_string(); - let octal_perms = u32::from_str_radix(&socket_perms, 8).expect("failed to convert octal permissions"); - let perms = std::fs::Permissions::from_mode(octal_perms); - if let Err(e) = fs::set_permissions(&path, perms).await { - return Err(Error::Err(format!("Failed to set socket {path:?} permissions: {e}"))); - } - - info!("Listening at {:?}", path); - - Ok(listener.unwrap()) -} diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs new file mode 100644 index 00000000..f9d1ad6e --- /dev/null +++ b/src/router/serve/mod.rs @@ -0,0 +1,26 @@ +mod plain; +mod tls; +mod unix; + +use std::sync::Arc; + +use axum::{routing::IntoMakeService, Router}; +use axum_server::Handle as ServerHandle; +use conduit::{Error, Result, Server}; +use tokio::sync::broadcast; + +/// Serve clients +pub(super) async fn serve( + server: &Arc, app: IntoMakeService, handle: ServerHandle, shutdown: broadcast::Receiver<()>, +) -> Result<(), Error> { + let config = &server.config; + let addrs = config.get_bind_addrs(); + + if cfg!(unix) && config.unix_socket_path.is_some() { + unix::serve(server, app, shutdown).await + } else if config.tls.is_some() { + tls::serve(server, app, handle, addrs).await + } else { + plain::serve(server, app, handle, addrs).await + } +} diff --git a/src/router/serve/plain.rs b/src/router/serve/plain.rs new file mode 100644 index 00000000..339f8940 --- /dev/null +++ b/src/router/serve/plain.rs @@ -0,0 +1,38 @@ +use std::{ + net::SocketAddr, + sync::{atomic::Ordering, Arc}, +}; + +use axum::{routing::IntoMakeService, Router}; +use axum_server::{bind, Handle as ServerHandle}; +use conduit::{debug_info, Result, Server}; +use tokio::task::JoinSet; +use tracing::info; + +pub(super) async fn serve( + server: &Arc, app: IntoMakeService, handle: ServerHandle, addrs: Vec, +) -> Result<()> { + let mut join_set = JoinSet::new(); + for addr in &addrs { + join_set.spawn_on(bind(*addr).handle(handle.clone()).serve(app.clone()), server.runtime()); + } + + info!("Listening on {addrs:?}"); + while join_set.join_next().await.is_some() {} + + let spawn_active = server.requests_spawn_active.load(Ordering::Relaxed); + let handle_active = server.requests_handle_active.load(Ordering::Relaxed); + debug_info!( + spawn_finished = server.requests_spawn_finished.load(Ordering::Relaxed), + handle_finished = server.requests_handle_finished.load(Ordering::Relaxed), + panics = server.requests_panic.load(Ordering::Relaxed), + spawn_active, + handle_active, + "Stopped listening on {addrs:?}", + ); + + debug_assert!(spawn_active == 0, "active request tasks are not joined"); + debug_assert!(handle_active == 0, "active request handles still pending"); + + Ok(()) +} diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs new file mode 100644 index 00000000..e4edeb32 --- /dev/null +++ b/src/router/serve/tls.rs @@ -0,0 +1,68 @@ +use std::{net::SocketAddr, sync::Arc}; + +use axum::{routing::IntoMakeService, Router}; +use axum_server::{bind_rustls, tls_rustls::RustlsConfig, Handle as ServerHandle}; +#[cfg(feature = "axum_dual_protocol")] +use axum_server_dual_protocol::ServerExt; +use conduit::{Result, Server}; +use tokio::task::JoinSet; +use tracing::{debug, info, warn}; + +pub(super) async fn serve( + server: &Arc, app: IntoMakeService, handle: ServerHandle, addrs: Vec, +) -> Result<()> { + let config = &server.config; + let tls = config.tls.as_ref().expect("TLS configuration"); + + debug!( + "Using direct TLS. Certificate path {} and certificate private key path {}", + &tls.certs, &tls.key + ); + info!( + "Note: It is strongly recommended that you use a reverse proxy instead of running conduwuit directly with TLS." + ); + let conf = RustlsConfig::from_pem_file(&tls.certs, &tls.key).await?; + + if cfg!(feature = "axum_dual_protocol") { + info!( + "conduwuit was built with axum_dual_protocol feature to listen on both HTTP and HTTPS. This will only \ + take effect if `dual_protocol` is enabled in `[global.tls]`" + ); + } + + let mut join_set = JoinSet::new(); + if cfg!(feature = "axum_dual_protocol") && tls.dual_protocol { + #[cfg(feature = "axum_dual_protocol")] + for addr in &addrs { + join_set.spawn_on( + axum_server_dual_protocol::bind_dual_protocol(*addr, conf.clone()) + .set_upgrade(false) + .handle(handle.clone()) + .serve(app.clone()), + server.runtime(), + ); + } + } else { + for addr in &addrs { + join_set.spawn_on( + bind_rustls(*addr, conf.clone()) + .handle(handle.clone()) + .serve(app.clone()), + server.runtime(), + ); + } + } + + if cfg!(feature = "axum_dual_protocol") && tls.dual_protocol { + warn!( + "Listening on {:?} with TLS certificate {} and supporting plain text (HTTP) connections too (insecure!)", + addrs, &tls.certs + ); + } else { + info!("Listening on {:?} with TLS certificate {}", addrs, &tls.certs); + } + + while join_set.join_next().await.is_some() {} + + Ok(()) +} diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs new file mode 100644 index 00000000..2f0cf9cf --- /dev/null +++ b/src/router/serve/unix.rs @@ -0,0 +1,107 @@ +#![cfg(unix)] + +use std::{path::Path, sync::Arc}; + +use axum::{extract::Request, routing::IntoMakeService, Router}; +use conduit::{debug_error, utils, Error, Result, Server}; +use hyper::{body::Incoming, service::service_fn}; +use hyper_util::{ + rt::{TokioExecutor, TokioIo}, + server, +}; +use tokio::{ + fs, + sync::broadcast::{self}, + task::JoinSet, +}; +use tower::{Service, ServiceExt}; +use tracing::{debug, info, warn}; +use utils::unwrap_infallible; + +pub(super) async fn serve( + server: &Arc, app: IntoMakeService, mut shutdown: broadcast::Receiver<()>, +) -> Result<()> { + let mut tasks = JoinSet::<()>::new(); + let executor = TokioExecutor::new(); + let builder = server::conn::auto::Builder::new(executor); + let listener = init(server).await?; + loop { + let app = app.clone(); + let builder = builder.clone(); + tokio::select! { + _sig = shutdown.recv() => break, + conn = listener.accept() => match conn { + Ok(conn) => accept(server, &listener, &mut tasks, app, builder, conn).await, + Err(err) => debug_error!(?listener, "accept error: {err}"), + }, + } + } + + drop(listener); + tasks.shutdown().await; + + Ok(()) +} + +async fn accept( + server: &Arc, listener: &tokio::net::UnixListener, tasks: &mut JoinSet<()>, + mut app: IntoMakeService, builder: server::conn::auto::Builder, + conn: (tokio::net::UnixStream, tokio::net::unix::SocketAddr), +) { + let (socket, remote) = conn; + let socket = TokioIo::new(socket); + debug!(?listener, ?socket, ?remote, "accepted"); + + let called = unwrap_infallible(app.call(()).await); + let handler = service_fn(move |req: Request| called.clone().oneshot(req)); + + let task = async move { + builder + .serve_connection(socket, handler) + .await + .map_err(|e| debug_error!(?remote, "connection error: {e}")) + .expect("connection error"); + }; + + _ = tasks.spawn_on(task, server.runtime()); + while tasks.try_join_next().is_some() {} +} + +async fn init(server: &Arc) -> Result { + use std::os::unix::fs::PermissionsExt; + + let config = &server.config; + let path = config + .unix_socket_path + .as_ref() + .expect("failed to extract configured unix socket path"); + + if path.exists() { + warn!("Removing existing UNIX socket {:#?} (unclean shutdown?)...", path.display()); + fs::remove_file(&path) + .await + .map_err(|e| warn!("Failed to remove existing UNIX socket: {e}")) + .unwrap(); + } + + let dir = path.parent().unwrap_or_else(|| Path::new("/")); + if let Err(e) = fs::create_dir_all(dir).await { + return Err(Error::Err(format!("Failed to create {dir:?} for socket {path:?}: {e}"))); + } + + let listener = tokio::net::UnixListener::bind(path); + if let Err(e) = listener { + return Err(Error::Err(format!("Failed to bind listener {path:?}: {e}"))); + } + + let socket_perms = config.unix_socket_perms.to_string(); + let octal_perms = u32::from_str_radix(&socket_perms, 8).expect("failed to convert octal permissions"); + let perms = std::fs::Permissions::from_mode(octal_perms); + if let Err(e) = fs::set_permissions(&path, perms).await { + return Err(Error::Err(format!("Failed to set socket {path:?} permissions: {e}"))); + } + + info!("Listening at {:?}", path); + + Ok(listener.unwrap()) +} From 6d1144bb697f42cd513dffff49cb913a795ac183 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 May 2024 00:43:30 +0000 Subject: [PATCH 0006/2091] move unix socket unlink from services to router Signed-off-by: Jason Volk --- src/router/serve/unix.rs | 29 ++++++++++++++++++++++------- src/service/globals/mod.rs | 2 -- src/service/services.rs | 14 ++------------ 3 files changed, 24 insertions(+), 21 deletions(-) diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 2f0cf9cf..ae768d24 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -11,6 +11,7 @@ use hyper_util::{ }; use tokio::{ fs, + net::{unix::SocketAddr, UnixListener, UnixStream}, sync::broadcast::{self}, task::JoinSet, }; @@ -37,16 +38,14 @@ pub(super) async fn serve( } } - drop(listener); - tasks.shutdown().await; + fini(listener, tasks).await; Ok(()) } async fn accept( - server: &Arc, listener: &tokio::net::UnixListener, tasks: &mut JoinSet<()>, - mut app: IntoMakeService, builder: server::conn::auto::Builder, - conn: (tokio::net::UnixStream, tokio::net::unix::SocketAddr), + server: &Arc, listener: &UnixListener, tasks: &mut JoinSet<()>, mut app: IntoMakeService, + builder: server::conn::auto::Builder, conn: (UnixStream, SocketAddr), ) { let (socket, remote) = conn; let socket = TokioIo::new(socket); @@ -67,7 +66,7 @@ async fn accept( while tasks.try_join_next().is_some() {} } -async fn init(server: &Arc) -> Result { +async fn init(server: &Arc) -> Result { use std::os::unix::fs::PermissionsExt; let config = &server.config; @@ -89,7 +88,7 @@ async fn init(server: &Arc) -> Result { return Err(Error::Err(format!("Failed to create {dir:?} for socket {path:?}: {e}"))); } - let listener = tokio::net::UnixListener::bind(path); + let listener = UnixListener::bind(path); if let Err(e) = listener { return Err(Error::Err(format!("Failed to bind listener {path:?}: {e}"))); } @@ -105,3 +104,19 @@ async fn init(server: &Arc) -> Result { Ok(listener.unwrap()) } + +async fn fini(listener: UnixListener, mut tasks: JoinSet<()>) { + let local = listener.local_addr(); + + drop(listener); + tasks.shutdown().await; + + if let Ok(local) = local { + if let Some(path) = local.as_pathname() { + debug!(?path, "Removing unix socket file."); + if let Err(e) = fs::remove_file(path).await { + warn!(?path, "Failed to remove UNIX socket file: {e}"); + } + } + } +} diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 8f035fcd..723979da 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -395,8 +395,6 @@ impl Service { pub fn well_known_server(&self) -> &Option { &self.config.well_known.server } - pub fn unix_socket_path(&self) -> &Option { &self.config.unix_socket_path } - pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool { for cidr in &self.cidr_range_denylist { if cidr.includes(ip) { diff --git a/src/service/services.rs b/src/service/services.rs index a2a9ca0b..a9356562 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -6,11 +6,8 @@ use std::{ use conduit::{debug_info, Result, Server}; use database::KeyValueDatabase; use lru_cache::LruCache; -use tokio::{ - fs, - sync::{broadcast, Mutex, RwLock}, -}; -use tracing::{debug, info, trace, warn}; +use tokio::sync::{broadcast, Mutex, RwLock}; +use tracing::{debug, info, trace}; use crate::{ account_data, admin, appservice, globals, key_backups, media, presence, pusher, rooms, sending, transaction_ids, @@ -321,13 +318,6 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} info!("Shutting down services"); self.interrupt().await; - debug!("Removing unix socket file."); - if let Some(path) = self.globals.unix_socket_path().as_ref() { - if let Err(e) = fs::remove_file(path).await { - warn!("Failed to remove UNIX socket file: {e}"); - } - } - debug!("Waiting for update worker..."); if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { updates_handle.abort(); From c3a0d2830914bb87d57e5721f40a2ecc1147a1cf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 May 2024 03:00:55 +0000 Subject: [PATCH 0007/2091] add tracing span; mute connection errors. Signed-off-by: Jason Volk --- src/router/serve/unix.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index ae768d24..a526ba29 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -3,7 +3,7 @@ use std::{path::Path, sync::Arc}; use axum::{extract::Request, routing::IntoMakeService, Router}; -use conduit::{debug_error, utils, Error, Result, Server}; +use conduit::{debug_error, trace, utils, Error, Result, Server}; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, @@ -19,6 +19,7 @@ use tower::{Service, ServiceExt}; use tracing::{debug, info, warn}; use utils::unwrap_infallible; +#[tracing::instrument(skip_all)] pub(super) async fn serve( server: &Arc, app: IntoMakeService, mut shutdown: broadcast::Receiver<()>, ) -> Result<()> { @@ -43,23 +44,21 @@ pub(super) async fn serve( Ok(()) } +#[allow(clippy::let_underscore_must_use)] async fn accept( server: &Arc, listener: &UnixListener, tasks: &mut JoinSet<()>, mut app: IntoMakeService, builder: server::conn::auto::Builder, conn: (UnixStream, SocketAddr), ) { let (socket, remote) = conn; let socket = TokioIo::new(socket); - debug!(?listener, ?socket, ?remote, "accepted"); + trace!(?listener, ?socket, ?remote, "accepted"); let called = unwrap_infallible(app.call(()).await); let handler = service_fn(move |req: Request| called.clone().oneshot(req)); let task = async move { - builder - .serve_connection(socket, handler) - .await - .map_err(|e| debug_error!(?remote, "connection error: {e}")) - .expect("connection error"); + // bug on darwin causes all results to be errors. do not unwrap this + _ = builder.serve_connection(socket, handler).await; }; _ = tasks.spawn_on(task, server.runtime()); From eed8a2a801194b14364f074dc37e6ad11db29ba3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 1 Jun 2024 10:19:26 +0000 Subject: [PATCH 0008/2091] add and sort incomplete package metadata Signed-off-by: Jason Volk --- Cargo.toml | 14 ++++++++------ deps/rust-rocksdb/Cargo.toml | 8 +++++++- src/admin/Cargo.toml | 8 +++++++- src/api/Cargo.toml | 8 +++++++- src/core/Cargo.toml | 8 +++++++- src/database/Cargo.toml | 8 +++++++- src/main/Cargo.toml | 14 ++++++++------ src/router/Cargo.toml | 8 +++++++- src/service/Cargo.toml | 8 +++++++- 9 files changed, 65 insertions(+), 19 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 83f81817..de6878f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,19 +6,21 @@ members = ["src/*"] default-members = ["src/*"] [workspace.package] -description = "a very cool fork of Conduit, a Matrix homeserver written in Rust" -license = "Apache-2.0" authors = [ "strawberry ", "timokoesters ", ] -version = "0.4.1" +categories = ["network-programming"] +description = "a very cool fork of Conduit, a Matrix homeserver written in Rust" edition = "2021" -# See also `rust-toolchain.toml` -rust-version = "1.77.0" homepage = "https://conduwuit.puppyirl.gay/" -repository = "https://github.com/girlbossceo/conduwuit" +keywords = ["chat", "matrix", "server"] +license = "Apache-2.0" +# See also `rust-toolchain.toml` readme = "README.md" +repository = "https://github.com/girlbossceo/conduwuit" +rust-version = "1.77.0" +version = "0.4.1" [workspace.metadata.crane] name = "conduit" diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 1a39281d..4dce69d1 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -1,7 +1,13 @@ [package] name = "rust-rocksdb-uwu" -version = "0.0.1" +categories.workspace = true +description = "dylib wrapper for rust-rocksdb" edition = "2021" +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version = "0.0.1" [features] default = ["snappy", "lz4", "zstd", "zlib", "bzip2"] diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 49ec4267..244cf7d2 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -1,7 +1,13 @@ [package] name = "conduit_admin" -version.workspace = true +categories.workspace = true +description.workspace = true edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true [lib] path = "mod.rs" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 890d66af..315fa76c 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -1,7 +1,13 @@ [package] name = "conduit_api" -version.workspace = true +categories.workspace = true +description.workspace = true edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true [lib] path = "mod.rs" diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 6b700f1f..988a810b 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -1,7 +1,13 @@ [package] name = "conduit_core" -version.workspace = true +categories.workspace = true +description.workspace = true edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true [lib] path = "mod.rs" diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 1855cbff..ea013298 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -1,7 +1,13 @@ [package] name = "conduit_database" -version.workspace = true +categories.workspace = true +description.workspace = true edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true [lib] path = "mod.rs" diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index cecdeb58..22c6b2ac 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -2,15 +2,17 @@ # TODO: when can we rename to conduwuit? name = "conduit" default-run = "conduit" -description.workspace = true -license.workspace = true authors.workspace = true -homepage.workspace = true -repository.workspace = true -readme.workspace = true -version.workspace = true +categories.workspace = true +description.workspace = true edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true rust-version.workspace = true +version.workspace = true metadata.crane.workspace = true [package.metadata.deb] diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 4b197d0a..11028112 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -1,7 +1,13 @@ [package] name = "conduit_router" -version.workspace = true +categories.workspace = true +description.workspace = true edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true [lib] path = "mod.rs" diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 580706ff..f2990a46 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -1,7 +1,13 @@ [package] name = "conduit_service" -version.workspace = true +categories.workspace = true +description.workspace = true edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true [lib] path = "mod.rs" From 14039d9df4b57e485924e996b57b212274fc7ead Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 1 Jun 2024 10:26:05 +0000 Subject: [PATCH 0009/2091] cleanup unused extern crates. Signed-off-by: Jason Volk --- Cargo.lock | 25 ------------------------- src/core/Cargo.toml | 8 -------- src/database/Cargo.toml | 13 ------------- src/main/Cargo.toml | 4 ---- src/service/Cargo.toml | 32 -------------------------------- 5 files changed, 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 29b03a71..375440ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,12 +572,8 @@ name = "conduit" version = "0.4.1" dependencies = [ "clap", - "conduit_admin", - "conduit_api", "conduit_core", - "conduit_database", "conduit_router", - "conduit_service", "console-subscriber", "hardened_malloc-rs", "log", @@ -655,16 +651,11 @@ dependencies = [ name = "conduit_core" version = "0.4.1" dependencies = [ - "async-trait", "axum 0.7.5", "axum-server", - "base64 0.22.1", "bytes", - "clap", - "cyborgtime", "either", "figment", - "futures-util", "hardened_malloc-rs", "http 1.1.0", "http-body-util", @@ -674,7 +665,6 @@ dependencies = [ "itertools 0.13.0", "libloading", "log", - "lru-cache", "nix", "parking_lot", "rand", @@ -688,8 +678,6 @@ dependencies = [ "serde", "serde_json", "serde_regex", - "serde_yaml", - "sha-1", "thiserror", "thread_local", "tikv-jemalloc-ctl", @@ -708,7 +696,6 @@ version = "0.4.1" dependencies = [ "chrono", "conduit_core", - "futures-util", "log", "lru-cache", "parking_lot", @@ -716,9 +703,6 @@ dependencies = [ "rusqlite", "rust-rocksdb-uwu", "thread_local", - "tikv-jemalloc-ctl", - "tikv-jemalloc-sys", - "tikv-jemallocator", "tokio", "tracing", "zstd", @@ -764,13 +748,11 @@ dependencies = [ "async-trait", "base64 0.22.1", "bytes", - "clap", "conduit_core", "conduit_database", "cyborgtime", "futures-util", "hickory-resolver", - "hmac", "http 1.1.0", "image", "ipaddress", @@ -786,21 +768,14 @@ dependencies = [ "ruma", "ruma-identifiers-validation", "rusqlite", - "rust-rocksdb-uwu", "serde", "serde_json", "serde_yaml", - "sha-1", "sha2", "thread_local", - "tikv-jemalloc-ctl", - "tikv-jemalloc-sys", - "tikv-jemallocator", "tokio", "tracing", - "tracing-subscriber", "url", - "webpage", "zstd", ] diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 988a810b..a0c8a060 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -71,16 +71,11 @@ perf_measurements = [] sentry_telemetry = [] [dependencies] -async-trait.workspace = true axum-server.workspace = true axum.workspace = true -base64.workspace = true bytes.workspace = true -clap.workspace = true -cyborgtime.workspace = true either.workspace = true figment.workspace = true -futures-util.workspace = true http-body-util.workspace = true http.workspace = true image.workspace = true @@ -89,7 +84,6 @@ ipaddress.workspace = true itertools.workspace = true libloading.workspace = true log.workspace = true -lru-cache.workspace = true parking_lot.optional = true parking_lot.workspace = true rand.workspace = true @@ -105,8 +99,6 @@ sanitize-filename.workspace = true serde_json.workspace = true serde_regex.workspace = true serde.workspace = true -serde_yaml.workspace = true -sha-1.workspace = true thiserror.workspace = true thread_local.optional = true thread_local.workspace = true diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index ea013298..e23d2d7b 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -41,14 +41,8 @@ rocksdb = [ "dep:rust-rocksdb", ] jemalloc = [ - "dep:tikv-jemalloc-sys", - "dep:tikv-jemalloc-ctl", - "dep:tikv-jemallocator", "rust-rocksdb/jemalloc", ] -jemalloc_prof = [ - "tikv-jemalloc-sys/profiling", -] io_uring = [ "rust-rocksdb/io-uring", ] @@ -59,7 +53,6 @@ zstd_compression = [ [dependencies] chrono.workspace = true conduit-core.workspace = true -futures-util.workspace = true log.workspace = true lru-cache.workspace = true parking_lot.optional = true @@ -71,12 +64,6 @@ rust-rocksdb.optional = true rust-rocksdb.workspace = true thread_local.optional = true thread_local.workspace = true -tikv-jemallocator.optional = true -tikv-jemallocator.workspace = true -tikv-jemalloc-ctl.optional = true -tikv-jemalloc-ctl.workspace = true -tikv-jemalloc-sys.optional = true -tikv-jemalloc-sys.workspace = true tokio.workspace = true tracing.workspace = true zstd.optional = true diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 22c6b2ac..06c8c697 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -69,10 +69,6 @@ perf_measurements = [ [dependencies] conduit-router.workspace = true -conduit-admin.workspace = true -conduit-api.workspace = true -conduit-service.workspace = true -conduit-database.workspace = true conduit-core.workspace = true tokio.workspace = true diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index f2990a46..e4189125 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -18,11 +18,7 @@ crate-type = [ [features] default = [ - "rocksdb", - "io_uring", - "jemalloc", "gzip_compression", - "zstd_compression", "brotli_compression", "release_max_log_level", ] @@ -39,21 +35,6 @@ sqlite = [ "dep:parking_lot", "dep:thread_local", ] -rocksdb = [ - "dep:rust-rocksdb", -] -jemalloc = [ - "dep:tikv-jemalloc-sys", - "dep:tikv-jemalloc-ctl", - "dep:tikv-jemallocator", - "rust-rocksdb/jemalloc", -] -io_uring = [ - "rust-rocksdb/io-uring", -] -zstd_compression = [ - "rust-rocksdb/zstd", -] gzip_compression = [ "reqwest/gzip", ] @@ -69,13 +50,11 @@ argon2.workspace = true async-trait.workspace = true base64.workspace = true bytes.workspace = true -clap.workspace = true conduit-core.workspace = true conduit-database.workspace = true cyborgtime.workspace = true futures-util.workspace = true hickory-resolver.workspace = true -hmac.workspace = true http.workspace = true image.workspace = true ipaddress.workspace = true @@ -93,27 +72,16 @@ ruma-identifiers-validation.workspace = true ruma.workspace = true rusqlite.optional = true rusqlite.workspace = true -rust-rocksdb.optional = true -rust-rocksdb.workspace = true serde_json.workspace = true serde.workspace = true serde_yaml.workspace = true -sha-1.workspace = true sha2.optional = true sha2.workspace = true thread_local.optional = true thread_local.workspace = true -tikv-jemallocator.optional = true -tikv-jemallocator.workspace = true -tikv-jemalloc-ctl.optional = true -tikv-jemalloc-ctl.workspace = true -tikv-jemalloc-sys.optional = true -tikv-jemalloc-sys.workspace = true tokio.workspace = true -tracing-subscriber.workspace = true tracing.workspace = true url.workspace = true -webpage.workspace = true zstd.optional = true zstd.workspace = true From 9e51525c25fa1d06e3e5fd30389c0edd7f83dc8c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:15:02 +0000 Subject: [PATCH 0010/2091] Fix uninlined-format-args Signed-off-by: Jason Volk --- src/admin/appservice/appservice_command.rs | 2 +- src/admin/debug/debug_commands.rs | 3 +- src/admin/fsck/fsck_commands.rs | 5 ++-- src/admin/query/account_data.rs | 14 +++------ src/admin/query/appservice.rs | 14 +++------ src/admin/query/globals.rs | 35 +++++++--------------- src/admin/query/presence.rs | 14 +++------ src/admin/query/room_alias.rs | 21 ++++--------- src/admin/query/sending.rs | 28 +++++------------ src/admin/query/users.rs | 7 ++--- src/admin/room/room_alias_commands.rs | 22 +++++++------- src/admin/room/room_moderation_commands.rs | 11 +++---- src/core/error.rs | 2 +- src/core/utils/content_disposition.rs | 4 +-- 14 files changed, 59 insertions(+), 123 deletions(-) diff --git a/src/admin/appservice/appservice_command.rs b/src/admin/appservice/appservice_command.rs index 409ef83b..d15cb7a6 100644 --- a/src/admin/appservice/appservice_command.rs +++ b/src/admin/appservice/appservice_command.rs @@ -47,7 +47,7 @@ pub(crate) async fn show(_body: Vec<&str>, appservice_identifier: String) -> Res { Some(config) => { let config_str = serde_yaml::to_string(&config).expect("config should've been validated on register"); - let output = format!("Config for {}:\n\n```yaml\n{}\n```", appservice_identifier, config_str,); + let output = format!("Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```",); let output_html = format!( "Config for {}:\n\n
{}
", escape_html(&appservice_identifier), diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index a658b84d..74313089 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -294,8 +294,7 @@ pub(crate) async fn ping(_body: Vec<&str>, server: Box) -> Result { diff --git a/src/admin/fsck/fsck_commands.rs b/src/admin/fsck/fsck_commands.rs index dec407d8..2eab94de 100644 --- a/src/admin/fsck/fsck_commands.rs +++ b/src/admin/fsck/fsck_commands.rs @@ -17,9 +17,8 @@ pub(crate) async fn check_all_users(_body: Vec<&str>) -> Result ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, AccountData::Get { @@ -39,11 +36,8 @@ pub(crate) async fn account_data(subcommand: AccountData) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, } diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 88c74d5e..d108ca8d 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -17,11 +17,8 @@ pub(crate) async fn appservice(subcommand: Appservice) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, Appservice::All => { @@ -30,11 +27,8 @@ pub(crate) async fn appservice(subcommand: Appservice) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, } diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 37a549ad..e5249118 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -12,11 +12,8 @@ pub(crate) async fn globals(subcommand: Globals) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, Globals::CurrentCount => { @@ -25,11 +22,8 @@ pub(crate) async fn globals(subcommand: Globals) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, Globals::LastCheckForUpdatesId => { @@ -38,11 +32,8 @@ pub(crate) async fn globals(subcommand: Globals) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, Globals::LoadKeypair => { @@ -51,11 +42,8 @@ pub(crate) async fn globals(subcommand: Globals) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, Globals::SigningKeysFor { @@ -66,11 +54,8 @@ pub(crate) async fn globals(subcommand: Globals) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, } diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index b77febfa..f69dd19a 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -14,11 +14,8 @@ pub(crate) async fn presence(subcommand: Presence) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, Presence::PresenceSince { @@ -31,11 +28,8 @@ pub(crate) async fn presence(subcommand: Presence) -> Result = results.collect(); Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{:?}```", presence_since), - format!( - "

Query completed in {query_time:?}:

\n
{:?}\n
", - presence_since - ), + format!("Query completed in {query_time:?}:\n\n```\n{presence_since:?}```"), + format!("

Query completed in {query_time:?}:

\n
{presence_since:?}\n
"), )) }, } diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index eb2bac8a..62d05899 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -14,11 +14,8 @@ pub(crate) async fn room_alias(subcommand: RoomAlias) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, RoomAlias::LocalAliasesForRoom { @@ -31,11 +28,8 @@ pub(crate) async fn room_alias(subcommand: RoomAlias) -> Result = results.collect(); Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{:?}```", aliases), - format!( - "

Query completed in {query_time:?}:

\n
{:?}\n
", - aliases - ), + format!("Query completed in {query_time:?}:\n\n```\n{aliases:?}```"), + format!("

Query completed in {query_time:?}:

\n
{aliases:?}\n
"), )) }, RoomAlias::AllLocalAliases => { @@ -46,11 +40,8 @@ pub(crate) async fn room_alias(subcommand: RoomAlias) -> Result = results.collect(); Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{:?}```", aliases), - format!( - "

Query completed in {query_time:?}:

\n
{:?}\n
", - aliases - ), + format!("Query completed in {query_time:?}:\n\n```\n{aliases:?}```"), + format!("

Query completed in {query_time:?}:

\n
{aliases:?}\n
"), )) }, } diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 19d73cf6..5a21ec87 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -14,11 +14,8 @@ pub(crate) async fn sending(subcommand: Sending) -> Result> = results.collect(); Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{:?}```", active_requests), - format!( - "

Query completed in {query_time:?}:

\n
{:?}\n
", - active_requests - ), + format!("Query completed in {query_time:?}:\n\n```\n{active_requests:?}```"), + format!("

Query completed in {query_time:?}:

\n
{active_requests:?}\n
"), )) }, Sending::QueuedRequests { @@ -96,11 +93,8 @@ pub(crate) async fn sending(subcommand: Sending) -> Result>>(); Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{:?}```", queued_requests), - format!( - "

Query completed in {query_time:?}:

\n
{:?}\n
", - queued_requests - ), + format!("Query completed in {query_time:?}:\n\n```\n{queued_requests:?}```"), + format!("

Query completed in {query_time:?}:

\n
{queued_requests:?}\n
"), )) }, Sending::ActiveRequestsFor { @@ -178,11 +172,8 @@ pub(crate) async fn sending(subcommand: Sending) -> Result>>(); Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{:?}```", active_requests), - format!( - "

Query completed in {query_time:?}:

\n
{:?}\n
", - active_requests - ), + format!("Query completed in {query_time:?}:\n\n```\n{active_requests:?}```"), + format!("

Query completed in {query_time:?}:

\n
{active_requests:?}\n
"), )) }, Sending::GetLatestEduCount { @@ -193,11 +184,8 @@ pub(crate) async fn sending(subcommand: Sending) -> ResultQuery completed in {query_time:?}:

\n
{:?}\n
", - results - ), + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

Query completed in {query_time:?}:

\n
{results:?}\n
"), )) }, } diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index 2b624ed3..ef99d880 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -14,11 +14,8 @@ pub(crate) async fn users(subcommand: Users) -> Result let users = results.collect::>(); Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{:?}```", users), - format!( - "

Query completed in {query_time:?}:

\n
{:?}\n
", - users - ), + format!("Query completed in {query_time:?}:\n\n```\n{users:?}```"), + format!("

Query completed in {query_time:?}:

\n
{users:?}\n
"), )) }, } diff --git a/src/admin/room/room_alias_commands.rs b/src/admin/room/room_alias_commands.rs index b767251a..33c165b8 100644 --- a/src/admin/room/room_alias_commands.rs +++ b/src/admin/room/room_alias_commands.rs @@ -20,7 +20,7 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu let room_alias_str = format!("#{}:{}", room_alias_localpart, services().globals.server_name()); let room_alias = match RoomAliasId::parse_box(room_alias_str) { Ok(alias) => alias, - Err(err) => return Ok(RoomMessageEventContent::text_plain(format!("Failed to parse alias: {}", err))), + Err(err) => return Ok(RoomMessageEventContent::text_plain(format!("Failed to parse alias: {err}"))), }; match command { RoomAliasCommand::Set { @@ -30,14 +30,12 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu } => match (force, services().rooms.alias.resolve_local_alias(&room_alias)) { (true, Ok(Some(id))) => match services().rooms.alias.set_alias(&room_alias, &room_id) { Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( - "Successfully overwrote alias (formerly {})", - id + "Successfully overwrote alias (formerly {id})" ))), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {}", err))), + Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), }, (false, Ok(Some(id))) => Ok(RoomMessageEventContent::text_plain(format!( - "Refusing to overwrite in use alias for {}, use -f or --force to overwrite", - id + "Refusing to overwrite in use alias for {id}, use -f or --force to overwrite" ))), (_, Ok(None)) => match services().rooms.alias.set_alias(&room_alias, &room_id) { Ok(()) => Ok(RoomMessageEventContent::text_plain("Successfully set alias")), @@ -49,18 +47,18 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu .. } => match services().rooms.alias.resolve_local_alias(&room_alias) { Ok(Some(id)) => match services().rooms.alias.remove_alias(&room_alias) { - Ok(()) => Ok(RoomMessageEventContent::text_plain(format!("Removed alias from {}", id))), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {}", err))), + Ok(()) => Ok(RoomMessageEventContent::text_plain(format!("Removed alias from {id}"))), + Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), }, Ok(None) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {}", err))), + Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {err}"))), }, RoomAliasCommand::Which { .. } => match services().rooms.alias.resolve_local_alias(&room_alias) { - Ok(Some(id)) => Ok(RoomMessageEventContent::text_plain(format!("Alias resolves to {}", id))), + Ok(Some(id)) => Ok(RoomMessageEventContent::text_plain(format!("Alias resolves to {id}"))), Ok(None) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {}", err))), + Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {err}"))), }, RoomAliasCommand::List { .. @@ -93,7 +91,7 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu let html = format!("Aliases for {room_id}:\n
    {html_list}
"); Ok(RoomMessageEventContent::text_html(plain, html)) }, - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to list aliases: {}", err))), + Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to list aliases: {err}"))), } } else { let aliases = services() diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 14ea090d..88b05fd0 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -481,7 +481,7 @@ pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> // as the room name if we dont have it TODO: do same if we have a room alias for // this let plain_list = room_ids.iter().fold(String::new(), |mut output, room_id| { - writeln!(output, "- `{}`", room_id).unwrap(); + writeln!(output, "- `{room_id}`").unwrap(); output }); @@ -490,16 +490,13 @@ pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> output }); - let plain = format!("Rooms:\n{}", plain_list); - let html = format!("Rooms:\n
    {}
", html_list); + let plain = format!("Rooms:\n{plain_list}"); + let html = format!("Rooms:\n
    {html_list}
"); Ok(RoomMessageEventContent::text_html(plain, html)) }, Err(e) => { error!("Failed to list banned rooms: {}", e); - Ok(RoomMessageEventContent::text_plain(format!( - "Unable to list room aliases: {}", - e - ))) + Ok(RoomMessageEventContent::text_plain(format!("Unable to list room aliases: {e}"))) }, } }, diff --git a/src/core/error.rs b/src/core/error.rs index 40ad8ecc..7049571a 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -135,7 +135,7 @@ impl From for Error { } impl fmt::Debug for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self) } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } } #[derive(Clone)] diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index 2a2a6d18..e1d40bec 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -127,9 +127,9 @@ mod tests { }; // cargo test -- --nocapture - println!("{}", SAMPLE); + println!("{SAMPLE}"); println!("{}", sanitize_filename::sanitize_with_options(SAMPLE, options.clone())); - println!("{:?}", SAMPLE); + println!("{SAMPLE:?}"); println!("{:?}", sanitize_filename::sanitize_with_options(SAMPLE, options.clone())); assert_eq!(SANITISED, sanitize_filename::sanitize_with_options(SAMPLE, options.clone())); From b3fc8516ed392282e93d3395c4f335944ac2badd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:16:29 +0000 Subject: [PATCH 0011/2091] Fix unnested-or-patterns Signed-off-by: Jason Volk --- src/api/client_server/membership.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/api/client_server/membership.rs b/src/api/client_server/membership.rs index 7ccea350..34af9cdf 100644 --- a/src/api/client_server/membership.rs +++ b/src/api/client_server/membership.rs @@ -995,10 +995,7 @@ pub async fn join_room_by_id_helper( let restriction_rooms = match join_rules_event_content { Some(RoomJoinRulesEventContent { - join_rule: JoinRule::Restricted(restricted), - }) - | Some(RoomJoinRulesEventContent { - join_rule: JoinRule::KnockRestricted(restricted), + join_rule: JoinRule::Restricted(restricted) | JoinRule::KnockRestricted(restricted), }) => restricted .allow .into_iter() From 02081b66c47218b6ab3d537217727be949a46d4e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:18:57 +0000 Subject: [PATCH 0012/2091] Fix some unnecessary-unwraps w/ addl cleanup/simplification. Signed-off-by: Jason Volk --- src/api/client_server/sync.rs | 2 +- src/service/globals/data.rs | 18 ++++-------------- src/service/globals/mod.rs | 26 ++++++++------------------ src/service/rooms/timeline/mod.rs | 2 +- src/service/sending/mod.rs | 10 +++++----- src/service/sending/sender.rs | 6 +++--- 6 files changed, 22 insertions(+), 42 deletions(-) diff --git a/src/api/client_server/sync.rs b/src/api/client_server/sync.rs index 5925fc53..fb087489 100644 --- a/src/api/client_server/sync.rs +++ b/src/api/client_server/sync.rs @@ -142,7 +142,7 @@ pub(crate) async fn sync_events_route( .collect::>(); // Coalesce database writes for the remainder of this scope. - let _cork = services().globals.cork_and_flush()?; + let _cork = services().globals.db.cork_and_flush(); for room_id in all_joined_rooms { let room_id = room_id?; diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index f1fa621d..5b26598b 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -25,15 +25,9 @@ pub trait Data: Send + Sync { async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()>; fn cleanup(&self) -> Result<()>; - /// TODO: use this? - #[allow(dead_code)] - fn flush(&self) -> Result<()>; - fn cork(&self) -> Result; - fn cork_and_flush(&self) -> Result; + fn cork(&self) -> Cork; + fn cork_and_flush(&self) -> Cork; - /// TODO: use this? - #[allow(dead_code)] - fn cork_and_sync(&self) -> Result; fn memory_usage(&self) -> String; fn clear_caches(&self, amount: u32); fn load_keypair(&self) -> Result; @@ -177,13 +171,9 @@ impl Data for KeyValueDatabase { fn cleanup(&self) -> Result<()> { self.db.cleanup() } - fn flush(&self) -> Result<()> { self.db.flush() } + fn cork(&self) -> Cork { Cork::new(&self.db, false, false) } - fn cork(&self) -> Result { Ok(Cork::new(&self.db, false, false)) } - - fn cork_and_flush(&self) -> Result { Ok(Cork::new(&self.db, true, false)) } - - fn cork_and_sync(&self) -> Result { Ok(Cork::new(&self.db, true, true)) } + fn cork_and_flush(&self) -> Cork { Cork::new(&self.db, true, false) } fn memory_usage(&self) -> String { let auth_chain_cache = self.auth_chain_cache.lock().unwrap().len(); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 723979da..2621ae24 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -1,3 +1,10 @@ +mod client; +mod data; +pub(super) mod emerg_access; +pub(super) mod migrations; +mod resolver; +pub(super) mod updates; + use std::{ collections::{BTreeMap, HashMap}, fs, @@ -29,14 +36,7 @@ use tokio::{ use tracing::{error, trace}; use url::Url; -use crate::{database::Cork, services, Config, Result}; - -mod client; -mod data; -pub(crate) mod emerg_access; -pub(crate) mod migrations; -mod resolver; -pub(crate) mod updates; +use crate::{services, Config, Result}; type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries @@ -194,16 +194,6 @@ impl Service { self.db.watch(user_id, device_id).await } - pub fn cleanup(&self) -> Result<()> { self.db.cleanup() } - - /// TODO: use this? - #[allow(dead_code)] - pub fn flush(&self) -> Result<()> { self.db.flush() } - - pub fn cork(&self) -> Result { self.db.cork() } - - pub fn cork_and_flush(&self) -> Result { self.db.cork_and_flush() } - pub fn server_name(&self) -> &ServerName { self.config.server_name.as_ref() } pub fn max_request_size(&self) -> u32 { self.config.max_request_size } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 4d91375f..04f03242 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -195,7 +195,7 @@ impl Service { state_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex ) -> Result> { // Coalesce database writes for the remainder of this scope. - let _cork = services().globals.cork_and_flush()?; + let _cork = services().globals.db.cork_and_flush(); let shortroomid = services() .rooms diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index a9f64f7b..654ec523 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -81,7 +81,7 @@ impl Service { pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { let dest = Destination::Push(user.to_owned(), pushkey); let event = SendingEvent::Pdu(pdu_id.to_owned()); - let _cork = services().globals.cork()?; + let _cork = services().globals.db.cork(); let keys = self.db.queue_requests(&[(&dest, event.clone())])?; self.dispatch(Msg { dest, @@ -94,7 +94,7 @@ impl Service { pub fn send_pdu_appservice(&self, appservice_id: String, pdu_id: Vec) -> Result<()> { let dest = Destination::Appservice(appservice_id); let event = SendingEvent::Pdu(pdu_id); - let _cork = services().globals.cork()?; + let _cork = services().globals.db.cork(); let keys = self.db.queue_requests(&[(&dest, event.clone())])?; self.dispatch(Msg { dest, @@ -121,7 +121,7 @@ impl Service { .into_iter() .map(|server| (Destination::Normal(server), SendingEvent::Pdu(pdu_id.to_owned()))) .collect::>(); - let _cork = services().globals.cork()?; + let _cork = services().globals.db.cork(); let keys = self.db.queue_requests( &requests .iter() @@ -143,7 +143,7 @@ impl Service { pub fn send_edu_server(&self, server: &ServerName, serialized: Vec) -> Result<()> { let dest = Destination::Normal(server.to_owned()); let event = SendingEvent::Edu(serialized); - let _cork = services().globals.cork()?; + let _cork = services().globals.db.cork(); let keys = self.db.queue_requests(&[(&dest, event.clone())])?; self.dispatch(Msg { dest, @@ -170,7 +170,7 @@ impl Service { .into_iter() .map(|server| (Destination::Normal(server), SendingEvent::Edu(serialized.clone()))) .collect::>(); - let _cork = services().globals.cork()?; + let _cork = services().globals.db.cork(); let keys = self.db.queue_requests( &requests .iter() diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 8bb93105..3d1d89da 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -100,7 +100,7 @@ impl Service { fn handle_response_ok( &self, dest: &Destination, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, ) { - let _cork = services().globals.cork(); + let _cork = services().globals.db.cork(); self.db .delete_all_active_requests_for(dest) .expect("all active requests deleted"); @@ -173,7 +173,7 @@ impl Service { return Ok(None); } - let _cork = services().globals.cork(); + let _cork = services().globals.db.cork(); let mut events = Vec::new(); // Must retry any previous transaction for this remote. @@ -187,7 +187,7 @@ impl Service { } // Compose the next transaction - let _cork = services().globals.cork(); + let _cork = services().globals.db.cork(); if !new_events.is_empty() { self.db.mark_as_active(&new_events)?; for (e, _) in new_events { From 2ab427fe99f09883ead44922999af44de19e5a25 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:21:08 +0000 Subject: [PATCH 0013/2091] Fix default-trait-access --- src/core/alloc/default.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/alloc/default.rs b/src/core/alloc/default.rs index 6e4128bf..4e2f8d7e 100644 --- a/src/core/alloc/default.rs +++ b/src/core/alloc/default.rs @@ -2,8 +2,8 @@ /// Always returns the empty string #[must_use] -pub fn memory_stats() -> String { Default::default() } +pub fn memory_stats() -> String { String::default() } /// Always returns the empty string #[must_use] -pub fn memory_usage() -> String { Default::default() } +pub fn memory_usage() -> String { String::default() } From 68cbf191540db71568f9272ff88a8d493ee647ff Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:22:48 +0000 Subject: [PATCH 0014/2091] Fix items-after-statements Signed-off-by: Jason Volk --- src/api/client_server/sync.rs | 3 ++- src/database/rocksdb/kvtree.rs | 6 +++--- src/database/rocksdb/mod.rs | 3 ++- src/database/rocksdb/opts.rs | 3 ++- src/service/pusher/mod.rs | 7 +++---- src/service/sending/appservice.rs | 5 ++--- 6 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/api/client_server/sync.rs b/src/api/client_server/sync.rs index fb087489..933f7b58 100644 --- a/src/api/client_server/sync.rs +++ b/src/api/client_server/sync.rs @@ -494,6 +494,8 @@ async fn handle_left_room( async fn process_presence_updates( presence_updates: &mut HashMap, since: u64, syncing_user: &OwnedUserId, ) -> Result<()> { + use crate::service::presence::Presence; + // Take presence updates for (user_id, _, presence_bytes) in services().presence.presence_since(since) { if !services() @@ -504,7 +506,6 @@ async fn process_presence_updates( continue; } - use crate::service::presence::Presence; let presence_event = Presence::from_json_bytes_to_event(&presence_bytes, &user_id)?; match presence_updates.entry(user_id) { Entry::Vacant(slot) => { diff --git a/src/database/rocksdb/kvtree.rs b/src/database/rocksdb/kvtree.rs index 02a0f3bf..00d01f8e 100644 --- a/src/database/rocksdb/kvtree.rs +++ b/src/database/rocksdb/kvtree.rs @@ -23,13 +23,13 @@ impl KvTree for RocksDbEngineTree<'_> { } fn multi_get(&self, keys: &[&[u8]]) -> Result>>> { - let mut readoptions = rust_rocksdb::ReadOptions::default(); - readoptions.set_total_order_seek(true); - // Optimization can be `true` if key vector is pre-sorted **by the column // comparator**. const SORTED: bool = false; + let mut readoptions = rust_rocksdb::ReadOptions::default(); + readoptions.set_total_order_seek(true); + let mut ret: Vec>> = Vec::with_capacity(keys.len()); for res in self .db diff --git a/src/database/rocksdb/mod.rs b/src/database/rocksdb/mod.rs index 5198e6b7..28ce5072 100644 --- a/src/database/rocksdb/mod.rs +++ b/src/database/rocksdb/mod.rs @@ -261,8 +261,9 @@ impl KeyValueDatabaseEngine for Arc { impl Drop for Engine { fn drop(&mut self) { - debug!("Waiting for background tasks to finish..."); const BLOCKING: bool = true; + + debug!("Waiting for background tasks to finish..."); self.rocks.cancel_all_background_work(BLOCKING); debug!("Shutting down background threads"); diff --git a/src/database/rocksdb/opts.rs b/src/database/rocksdb/opts.rs index afed79a5..06c90a77 100644 --- a/src/database/rocksdb/opts.rs +++ b/src/database/rocksdb/opts.rs @@ -17,13 +17,14 @@ use super::{ /// columns, therefor columns should only be opened after passing this result /// through cf_options(). pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_cache: &Cache) -> Options { + const MIN_PARALLELISM: usize = 2; + let mut opts = Options::default(); // Logging set_logging_defaults(&mut opts, config); // Processing - const MIN_PARALLELISM: usize = 2; let threads = if config.rocksdb_parallelism_threads == 0 { cmp::max(MIN_PARALLELISM, utils::available_parallelism()) } else { diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 261d69dd..636ba4dd 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -49,12 +49,11 @@ impl Service { where T: OutgoingRequest + Debug, { - let dest = dest.replace(services().globals.notification_push_path(), ""); - - trace!("Push gateway destination: {dest}"); - const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_0]; + let dest = dest.replace(services().globals.notification_push_path(), ""); + trace!("Push gateway destination: {dest}"); + let http_request = request .try_into_http_request::(&dest, SendAccessToken::IfRequired(""), &VERSIONS) .map_err(|e| { diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 4764f889..4df5b934 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -14,6 +14,8 @@ pub(crate) async fn send_request(registration: Registration, request: T) -> R where T: OutgoingRequest + Debug, { + const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_0]; + let Some(dest) = registration.url else { return Ok(None); }; @@ -21,9 +23,6 @@ where trace!("Appservice URL \"{dest}\", Appservice ID: {}", registration.id); let hs_token = registration.hs_token.as_str(); - - const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_0]; - let mut http_request = request .try_into_http_request::(&dest, SendAccessToken::IfRequired(hs_token), &VERSIONS) .map_err(|e| { From e5eccb3a0c43892587ea1a3e32b51b851eafd65f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:23:21 +0000 Subject: [PATCH 0015/2091] Fix unreadable-literal Signed-off-by: Jason Volk --- src/core/config/check.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 99ca3cfd..a631c4f8 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -91,7 +91,7 @@ pub fn check(config: &Config) -> Result<(), Error> { return Err(Error::bad_config("Registration token was specified but is empty (\"\")")); } - if config.max_request_size < 5120000 { + if config.max_request_size < 5_120_000 { return Err(Error::bad_config("Max request size is less than 5MB. Please increase it.")); } From 0f3d43153b6e85082f641bafdfb5fd900a443c00 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:25:22 +0000 Subject: [PATCH 0016/2091] Fix unused-self Signed-off-by: Jason Volk --- src/core/config/mod.rs | 4 ++-- src/service/rooms/event_handler/mod.rs | 24 ++++++++++++------------ src/service/rooms/typing/mod.rs | 8 ++++---- src/service/sending/sender.rs | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index b4e2886a..665cbe20 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -411,7 +411,7 @@ impl Config { }; // don't start if we're listening on both UNIX sockets and TCP at same time - if config.is_dual_listening(&raw_config) { + if Self::is_dual_listening(&raw_config) { return Err(Error::bad_config("dual listening on UNIX and TCP sockets not allowed.")); }; @@ -455,7 +455,7 @@ impl Config { /// Checks the presence of the `address` and `unix_socket_path` keys in the /// raw_config, exiting the process if both keys were detected. - fn is_dual_listening(&self, raw_config: &Figment) -> bool { + fn is_dual_listening(raw_config: &Figment) -> bool { let check_address = raw_config.find_value("address"); let check_unix_socket = raw_config.find_value("unix_socket_path"); diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 15e3d8d3..754365c3 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -118,7 +118,7 @@ impl Service { .ok_or_else(|| Error::bad_database("Failed to find create event in db."))?; // Procure the room version - let room_version_id = self.get_room_version_id(&create_event)?; + let room_version_id = Self::get_room_version_id(&create_event)?; let first_pdu_in_room = services() .rooms @@ -130,7 +130,7 @@ impl Service { .handle_outlier_pdu(origin, &create_event, event_id, room_id, value, false, pub_key_map) .await?; - self.check_room_id(room_id, &incoming_pdu)?; + Self::check_room_id(room_id, &incoming_pdu)?; // 8. if not timeline event: stop if !is_timeline_event { @@ -308,7 +308,7 @@ impl Service { // 2. Check signatures, otherwise drop // 3. check content hash, redact if doesn't match - let room_version_id = self.get_room_version_id(create_event)?; + let room_version_id = Self::get_room_version_id(create_event)?; let guard = pub_key_map.read().await; let mut val = match ruma::signatures::verify_event(&guard, &value, &room_version_id) { @@ -347,7 +347,7 @@ impl Service { ) .map_err(|_| Error::bad_database("Event is not a valid PDU."))?; - self.check_room_id(room_id, &incoming_pdu)?; + Self::check_room_id(room_id, &incoming_pdu)?; if !auth_events_known { // 4. fetch any missing auth events doing all checks listed here starting at 1. @@ -382,7 +382,7 @@ impl Service { continue; }; - self.check_room_id(room_id, &auth_event)?; + Self::check_room_id(room_id, &auth_event)?; match auth_events.entry(( auth_event.kind.to_string().into(), @@ -417,7 +417,7 @@ impl Service { } if !state_res::event_auth::auth_check( - &self.to_room_version(&room_version_id), + &Self::to_room_version(&room_version_id), &incoming_pdu, None::, // TODO: third party invite |k, s| auth_events.get(&(k.to_string().into(), s.to_owned())), @@ -460,7 +460,7 @@ impl Service { debug!("Upgrading to timeline pdu"); let timer = tokio::time::Instant::now(); - let room_version_id = self.get_room_version_id(create_event)?; + let room_version_id = Self::get_room_version_id(create_event)?; // 10. Fetch missing state and auth chain events by calling /state_ids at // backwards extremities doing all the checks in this list starting at 1. @@ -488,7 +488,7 @@ impl Service { } let state_at_incoming_event = state_at_incoming_event.expect("we always set this to some above"); - let room_version = self.to_room_version(&room_version_id); + let room_version = Self::to_room_version(&room_version_id); debug!("Performing auth check"); // 11. Check the auth of the event passes based on the state of the event @@ -1221,7 +1221,7 @@ impl Service { .await .pop() { - self.check_room_id(room_id, &pdu)?; + Self::check_room_id(room_id, &pdu)?; if amount > services().globals.max_fetch_prev_events() { // Max limit reached @@ -1329,7 +1329,7 @@ impl Service { } } - fn check_room_id(&self, room_id: &RoomId, pdu: &PduEvent) -> Result<()> { + fn check_room_id(room_id: &RoomId, pdu: &PduEvent) -> Result<()> { if pdu.room_id != room_id { warn!("Found event from room {} in room {}", pdu.room_id, room_id); return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event has wrong room id")); @@ -1337,7 +1337,7 @@ impl Service { Ok(()) } - fn get_room_version_id(&self, create_event: &PduEvent) -> Result { + fn get_room_version_id(create_event: &PduEvent) -> Result { let create_event_content: RoomCreateEventContent = serde_json::from_str(create_event.content.get()).map_err(|e| { error!("Invalid create event: {}", e); @@ -1347,7 +1347,7 @@ impl Service { Ok(create_event_content.room_version) } - fn to_room_version(&self, room_version_id: &RoomVersionId) -> RoomVersion { + fn to_room_version(room_version_id: &RoomVersionId) -> RoomVersion { RoomVersion::new(room_version_id).expect("room version is supported") } } diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 37876f3f..0b89dfdd 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -44,7 +44,7 @@ impl Service { // update federation if user_is_local(user_id) { - self.federation_send(room_id, user_id, true)?; + Self::federation_send(room_id, user_id, true)?; } Ok(()) @@ -70,7 +70,7 @@ impl Service { // update federation if user_is_local(user_id) { - self.federation_send(room_id, user_id, false)?; + Self::federation_send(room_id, user_id, false)?; } Ok(()) @@ -126,7 +126,7 @@ impl Service { // update federation for user in removable { if user_is_local(&user) { - self.federation_send(room_id, &user, false)?; + Self::federation_send(room_id, &user, false)?; } } } @@ -163,7 +163,7 @@ impl Service { }) } - fn federation_send(&self, room_id: &RoomId, user_id: &UserId, typing: bool) -> Result<()> { + fn federation_send(room_id: &RoomId, user_id: &UserId, typing: bool) -> Result<()> { debug_assert!(user_is_local(user_id), "tried to broadcast typing status of remote user",); if !services().globals.config.allow_outgoing_typing { return Ok(()); diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 3d1d89da..9e107df8 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -80,12 +80,12 @@ impl Service { ) { match response { Ok(dest) => self.handle_response_ok(&dest, futures, statuses), - Err((dest, e)) => self.handle_response_err(dest, futures, statuses, &e), + Err((dest, e)) => Self::handle_response_err(dest, futures, statuses, &e), }; } fn handle_response_err( - &self, dest: Destination, _futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, e: &Error, + dest: Destination, _futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, e: &Error, ) { debug!(dest = ?dest, "{e:?}"); statuses.entry(dest).and_modify(|e| { From 4521e93d04e25a9c218a1840c7d18f1b6e2bfe6e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:27:03 +0000 Subject: [PATCH 0017/2091] Fix stable-sort-primitive Signed-off-by: Jason Volk --- src/service/rooms/auth_chain/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 511b762e..d8e38c85 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -121,7 +121,7 @@ impl Service { ); } - full_auth_chain.sort(); + full_auth_chain.sort_unstable(); full_auth_chain.dedup(); debug!( chain_length = ?full_auth_chain.len(), From 050841a871a80670b8e52ab4b7f15cc0c4e68cb5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:29:45 +0000 Subject: [PATCH 0018/2091] Fix inefficient-to-string Signed-off-by: Jason Volk --- src/router/layers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index a6e30e98..33d0cb20 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -160,7 +160,7 @@ fn catch_panic(err: Box) -> http::Response() { s.clone() } else if let Some(s) = err.downcast_ref::<&str>() { - s.to_string() + (*s).to_owned() } else { "Unknown internal server error occurred.".to_owned() }; From 38ab1083e31f94d6932a5f3671539785ff1ac5e0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 00:30:04 +0000 Subject: [PATCH 0019/2091] Fix ptr-cast-constness Signed-off-by: Jason Volk --- src/core/alloc/je.rs | 5 ++++- src/router/run.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 4092d815..ebf3575f 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -30,7 +30,10 @@ pub fn memory_stats() -> String { let mut str = String::new(); let opaque = std::ptr::from_mut(&mut str).cast::(); - let opts_p: *const c_char = std::ffi::CString::new(opts_s).expect("cstring").into_raw() as *const c_char; + let opts_p: *const c_char = std::ffi::CString::new(opts_s) + .expect("cstring") + .into_raw() + .cast_const(); // SAFETY: calls malloc_stats_print() with our string instance which must remain // in this frame. https://docs.rs/tikv-jemalloc-sys/latest/tikv_jemalloc_sys/fn.malloc_stats_print.html diff --git a/src/router/run.rs b/src/router/run.rs index d20f8669..e6238853 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -97,7 +97,7 @@ pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { .take() .unwrap(); - let s = std::ptr::from_ref(s) as *mut Services; + let s: *mut Services = std::ptr::from_ref(s).cast_mut(); //SAFETY: Services was instantiated in start() and leaked into the SERVICES // global perusing as 'static for the duration of run_server(). Now we reclaim // it to drop it before unloading the module. If this is not done there will be From 53fe2362fcf04d0478d6f223b58da7a8e5b9569c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 01:43:25 +0000 Subject: [PATCH 0020/2091] Fix path-buf-push-overwrite Signed-off-by: Jason Volk --- src/service/media/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 5521011d..0697577f 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -559,9 +559,7 @@ mod tests { .db .create_file_metadata(None, mxc, width, height, Some(content_disposition), Some(content_type)) .unwrap(); - let mut r = PathBuf::new(); - r.push("/tmp"); - r.push("media"); + let mut r = PathBuf::from("/tmp/media"); // r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD)); // use the sha256 hash of the key as the file name instead of the key itself // this is because the base64 encoded key can be longer than 255 characters. From ee52d2f751b6a4394b11af2de123f9bfbe592f79 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 1 Jun 2024 09:20:37 +0000 Subject: [PATCH 0021/2091] refactor lints into categories. lints are now more strict. rust: * ALL lints which rustc defaults to "allow" have been set to "warn". * NEW "warn" lints which produce a warning as of this commit have been explicitly identified and commented with a TODO for later review. clippy: * ALL categories (sans restriction) now fully enabled to "warn". * redundant lints set to "warn" from categories now at "warn" are removed. * previous "allow" sadness moved into respective categories. * new warnings produced as of this commit have been explicitly identified: - nursery lints set to "allow" marked with TODO for later review. - pedantic lints set to "allow" Signed-off-by: Jason Volk --- Cargo.toml | 254 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 145 insertions(+), 109 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index de6878f0..cabd47a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -644,145 +644,181 @@ opt-level = 'z' [profile.test] incremental = false +############################################################################### +# +# Linting +# + [workspace.lints.rust] -missing_abi = "warn" -noop_method_call = "warn" -pointer_structural_match = "warn" -explicit_outlives_requirements = "warn" -unused_extern_crates = "warn" -unused_import_braces = "warn" -unused_lifetimes = "warn" -unused_qualifications = "warn" -unused_macro_rules = "warn" -dead_code = "warn" -elided_lifetimes_in_paths = "warn" -macro_use_extern_crate = "warn" -single_use_lifetimes = "warn" -unsafe_op_in_unsafe_fn = "warn" -unreachable_pub = "warn" +absolute-paths-not-starting-with-crate = "warn" +#box-pointers = "warn" # TODO +deprecated-in-future = "warn" +elided-lifetimes-in-paths = "warn" +explicit-outlives-requirements = "warn" +ffi-unwind-calls = "warn" +keyword-idents = "warn" +macro-use-extern-crate = "warn" +meta-variable-misuse = "warn" +missing-abi = "warn" +#missing-copy-implementations = "warn" # TODO +#missing-debug-implementations = "warn" # TODO +non-ascii-idents = "warn" +rust-2021-incompatible-closure-captures = "warn" +rust-2021-incompatible-or-patterns = "warn" +rust-2021-prefixes-incompatible-syntax = "warn" +rust-2021-prelude-collisions = "warn" +single-use-lifetimes = "warn" +#trivial-casts = "warn" # TODO +trivial-numeric-casts = "warn" +unit-bindings = "warn" +#unnameable-types = "warn" # TODO +unreachable-pub = "warn" +unsafe-op-in-unsafe-fn = "warn" +unstable-features = "warn" +unused-extern-crates = "warn" +unused-import-braces = "warn" +unused-lifetimes = "warn" +unused-macro-rules = "warn" +unused-qualifications = "warn" +#unused-results = "warn" # TODO -# this seems to suggest broken code and is not working correctly -unused_braces = "allow" - -# cfgs cannot be limited to features or cargo build --all-features panics for unsuspecting users. +## some sadness +let_underscore_drop = "allow" +missing_docs = "allow" # cfgs cannot be limited to expected cfgs or their de facto non-transitive/opt-in use-case e.g. # tokio_unstable will warn. unexpected_cfgs = "allow" +# this seems to suggest broken code and is not working correctly +unused_braces = "allow" +# buggy, but worth checking on occasionally +unused_crate_dependencies = "allow" +unsafe_code = "allow" +variant_size_differences = "allow" -# some sadness -missing_docs = "allow" +####################################### +# +# Clippy lints +# [workspace.lints.clippy] -# pedantic = "warn" -suspicious = "warn" # assume deny in practice -perf = "warn" # assume deny in practice +################### +cargo = "warn" -redundant_clone = "warn" -cloned_instead_of_copied = "warn" -expl_impl_clone_on_copy = "warn" -unnecessary_cast = "warn" -cast_lossless = "warn" -ptr_as_ptr = "warn" -mut_mut = "warn" -char_lit_as_u8 = "warn" -dbg_macro = "warn" -empty_structs_with_brackets = "warn" -get_unwrap = "warn" -negative_feature_names = "warn" -pub_without_shorthand = "warn" -rc_buffer = "warn" -rc_mutex = "warn" -redundant_feature_names = "warn" -redundant_type_annotations = "warn" -rest_pat_in_fully_bound_structs = "warn" -str_to_string = "warn" -string_to_string = "warn" -tests_outside_test_module = "warn" -undocumented_unsafe_blocks = "warn" -unneeded_field_pattern = "warn" -unseparated_literal_suffix = "warn" -wildcard_dependencies = "warn" -or_fun_call = "warn" -unnecessary_lazy_evaluations = "warn" +## some sadness +multiple_crate_versions = { level = "allow", priority = 1 } + +################### +complexity = "warn" + +################### +correctness = "warn" + +################### +nursery = "warn" + +### some sadness +branches_sharing_code = { level = "allow", priority = 1 } # TODO +cognitive_complexity = { level = "allow", priority = 1 } # TODO +derive_partial_eq_without_eq = { level = "allow", priority = 1 } # TODO +equatable_if_let = { level = "allow", priority = 1 } # TODO +future_not_send = { level = "allow", priority = 1 } # TODO +missing_const_for_fn = { level = "allow", priority = 1 } # TODO +needless_collect = { level = "allow", priority = 1 } # TODO +needless_pass_by_ref_mut = { level = "allow", priority = 1 } # TODO +option_if_let_else = { level = "allow", priority = 1 } # TODO +redundant_pub_crate = { level = "allow", priority = 1 } # TODO +significant_drop_in_scrutinee = { level = "allow", priority = 1 } # TODO +significant_drop_tightening = { level = "allow", priority = 1 } # TODO +suboptimal_flops = { level = "allow", priority = 1 } # TODO +use_self = { level = "allow", priority = 1 } # TODO +useless_let_if_seq = { level = "allow", priority = 1 } # TODO + +################### +pedantic = "warn" + +## some sadness +cast_possible_truncation = "allow" +cast_precision_loss = "allow" +cast_sign_loss = "allow" +doc_markdown = "allow" +error_impl_error = "allow" +expect_used = "allow" +if_not_else = "allow" +if_then_some_else_none = "allow" +implicit_return = "allow" +inline_always = "allow" +map_err_ignore = "allow" +missing_docs_in_private_items = "allow" +missing_errors_doc = "allow" +missing_panics_doc = "allow" +mod_module_files = "allow" +module_name_repetitions = "allow" +multiple_inherent_impl = "allow" +no_effect_underscore_binding = "allow" +ref_patterns = "allow" +same_name_method = "allow" +similar_names = { level = "allow", priority = 1 } +single_call_fn = "allow" +string_add = "allow" +string_slice = "allow" +struct_field_names = { level = "allow", priority = 1 } +unnecessary_wraps = { level = "allow", priority = 1 } +unused_async = { level = "allow", priority = 1 } +unwrap_used = "allow" + +################### +perf = "warn" + +################### +#restriction = "warn" + +#arithmetic_side_effects = "warn" # TODO +#as_conversions = "warn" # TODO assertions_on_result_states = "warn" +dbg_macro = "warn" default_union_representation = "warn" deref_by_slicing = "warn" empty_drop = "warn" +empty_structs_with_brackets = "warn" exit = "warn" filetype_is_file = "warn" float_cmp_const = "warn" +fn_to_numeric_cast_any = "warn" format_push_string = "warn" +get_unwrap = "warn" impl_trait_in_params = "warn" +let_underscore_must_use = "warn" +let_underscore_untyped = "warn" lossy_float_literal = "warn" mem_forget = "warn" missing_assert_message = "warn" mutex_atomic = "warn" +pub_without_shorthand = "warn" +rc_buffer = "warn" +rc_mutex = "warn" +redundant_type_annotations = "warn" +rest_pat_in_fully_bound_structs = "warn" semicolon_outside_block = "warn" -fn_to_numeric_cast = "warn" -fn_to_numeric_cast_with_truncation = "warn" +str_to_string = "warn" string_lit_chars_any = "warn" +string_to_string = "warn" suspicious_xor_used_as_pow = "warn" +tests_outside_test_module = "warn" try_err = "warn" +undocumented_unsafe_blocks = "warn" unnecessary_safety_comment = "warn" unnecessary_safety_doc = "warn" unnecessary_self_imports = "warn" +unneeded_field_pattern = "warn" +unseparated_literal_suffix = "warn" verbose_file_reads = "warn" -cast_possible_wrap = "warn" -redundant_closure_for_method_calls = "warn" -large_futures = "warn" -semicolon_if_nothing_returned = "warn" -match_bool = "warn" -struct_excessive_bools = "warn" -must_use_candidate = "warn" -collapsible_else_if = "warn" -inconsistent_struct_constructor = "warn" -manual_string_new = "warn" -zero_sized_map_values = "warn" -unnecessary_box_returns = "warn" -map_unwrap_or = "warn" -implicit_clone = "warn" -match_wildcard_for_single_variants = "warn" -match_same_arms = "warn" -ignored_unit_patterns = "warn" -redundant_else = "warn" -explicit_into_iter_loop = "warn" -used_underscore_binding = "warn" -needless_pass_by_value = "warn" -too_many_lines = "warn" -let_underscore_untyped = "warn" -single_match = "warn" -single_match_else = "warn" -explicit_deref_methods = "warn" -explicit_iter_loop = "warn" -manual_let_else = "warn" -trivially_copy_pass_by_ref = "warn" -wildcard_imports = "warn" -checked_conversions = "warn" -let_underscore_must_use = "warn" -#integer_arithmetic = "warn" -#as_conversions = "warn" -# some sadness -missing_errors_doc = "allow" -missing_panics_doc = "allow" -module_name_repetitions = "allow" -if_not_else = "allow" -doc_markdown = "allow" -cast_possible_truncation = "allow" -cast_precision_loss = "allow" -cast_sign_loss = "allow" -same_name_method = "allow" -mod_module_files = "allow" -unwrap_used = "allow" -expect_used = "allow" -if_then_some_else_none = "allow" +################### +style = "warn" + +################### +suspicious = "warn" + +## some sadness let_underscore_future = "allow" -map_err_ignore = "allow" -missing_docs_in_private_items = "allow" -multiple_inherent_impl = "allow" -error_impl_error = "allow" -string_add = "allow" -string_slice = "allow" -ref_patterns = "allow" -unnecessary_wraps = "allow" From 9df5265c00c5083a3567c8be176ca149adf2c833 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 May 2024 00:14:22 +0000 Subject: [PATCH 0022/2091] split sending resolver into unit. Signed-off-by: Jason Volk --- src/admin/debug/debug_commands.rs | 2 +- src/service/sending/mod.rs | 13 +- src/service/sending/resolve.rs | 460 +++++++++++++++++++++++++++++ src/service/sending/send.rs | 465 +----------------------------- 4 files changed, 474 insertions(+), 466 deletions(-) create mode 100644 src/service/sending/resolve.rs diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 74313089..873a23e7 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -5,7 +5,7 @@ use ruma::{ api::client::error::ErrorKind, events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, RoomId, RoomVersionId, ServerName, }; -use service::{rooms::event_handler::parse_incoming_pdu, sending::send::resolve_actual_dest, services, PduEvent}; +use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve::resolve_actual_dest, services, PduEvent}; use tokio::sync::RwLock; use tracing::{debug, info, warn}; use tracing_subscriber::EnvFilter; diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 654ec523..4ad40f67 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -1,6 +1,13 @@ +mod appservice; +mod data; +pub mod resolve; +mod send; +mod sender; + use std::{fmt::Debug, sync::Arc}; use data::Data; +pub use resolve::FedDest; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, @@ -10,12 +17,6 @@ use tracing::{error, warn}; use crate::{server_is_ours, services, Config, Error, Result}; -mod appservice; -mod data; -pub mod send; -pub mod sender; -pub use send::FedDest; - pub struct Service { pub db: Arc, diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs new file mode 100644 index 00000000..d452a555 --- /dev/null +++ b/src/service/sending/resolve.rs @@ -0,0 +1,460 @@ +use std::{ + fmt::Debug, + net::{IpAddr, SocketAddr}, +}; + +use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; +use ipaddress::IPAddress; +use ruma::ServerName; +use tracing::{debug, error, trace}; + +use crate::{debug_error, debug_info, debug_warn, services, Error, Result}; + +/// Wraps either an literal IP address plus port, or a hostname plus complement +/// (colon-plus-port if it was specified). +/// +/// Note: A `FedDest::Named` might contain an IP address in string form if there +/// was no port specified to construct a `SocketAddr` with. +/// +/// # Examples: +/// ```rust +/// # use conduit_service::sending::FedDest; +/// # fn main() -> Result<(), std::net::AddrParseError> { +/// FedDest::Literal("198.51.100.3:8448".parse()?); +/// FedDest::Literal("[2001:db8::4:5]:443".parse()?); +/// FedDest::Named("matrix.example.org".to_owned(), String::new()); +/// FedDest::Named("matrix.example.org".to_owned(), ":8448".to_owned()); +/// FedDest::Named("198.51.100.5".to_owned(), String::new()); +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FedDest { + Literal(SocketAddr), + Named(String, String), +} + +pub(crate) struct ActualDest { + pub(crate) dest: FedDest, + pub(crate) host: String, + pub(crate) string: String, + pub(crate) cached: bool, +} + +#[tracing::instrument(skip_all, name = "resolve")] +pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result { + let cached; + let cached_result = services() + .globals + .actual_destinations() + .read() + .await + .get(server_name) + .cloned(); + + let (dest, host) = if let Some(result) = cached_result { + cached = true; + result + } else { + cached = false; + validate_dest(server_name)?; + resolve_actual_dest(server_name, false).await? + }; + + let string = dest.clone().into_https_string(); + Ok(ActualDest { + dest, + host, + string, + cached, + }) +} + +/// Returns: `actual_destination`, host header +/// Implemented according to the specification at +/// Numbers in comments below refer to bullet points in linked section of +/// specification +pub async fn resolve_actual_dest(dest: &ServerName, no_cache_dest: bool) -> Result<(FedDest, String)> { + trace!("Finding actual destination for {dest}"); + let dest_str = dest.as_str().to_owned(); + let mut hostname = dest_str.clone(); + + #[allow(clippy::single_match_else)] + let actual_dest = match get_ip_with_port(&dest_str) { + Some(host_port) => { + debug!("1: IP literal with provided or default port"); + host_port + }, + None => { + if let Some(pos) = dest_str.find(':') { + debug!("2: Hostname with included port"); + let (host, port) = dest_str.split_at(pos); + if !no_cache_dest { + query_and_cache_override(host, host, port.parse::().unwrap_or(8448)).await?; + } + + FedDest::Named(host.to_owned(), port.to_owned()) + } else { + trace!("Requesting well known for {dest}"); + if let Some(delegated_hostname) = request_well_known(dest.as_str()).await? { + debug!("3: A .well-known file is available"); + hostname = add_port_to_hostname(&delegated_hostname).into_uri_string(); + match get_ip_with_port(&delegated_hostname) { + Some(host_and_port) => { + debug!("3.1: IP literal in .well-known file"); + host_and_port + }, + None => { + if let Some(pos) = delegated_hostname.find(':') { + debug!("3.2: Hostname with port in .well-known file"); + let (host, port) = delegated_hostname.split_at(pos); + if !no_cache_dest { + query_and_cache_override(host, host, port.parse::().unwrap_or(8448)).await?; + } + + FedDest::Named(host.to_owned(), port.to_owned()) + } else { + trace!("Delegated hostname has no port in this branch"); + if let Some(hostname_override) = query_srv_record(&delegated_hostname).await? { + debug!("3.3: SRV lookup successful"); + let force_port = hostname_override.port(); + if !no_cache_dest { + query_and_cache_override( + &delegated_hostname, + &hostname_override.hostname(), + force_port.unwrap_or(8448), + ) + .await?; + } + + if let Some(port) = force_port { + FedDest::Named(delegated_hostname, format!(":{port}")) + } else { + add_port_to_hostname(&delegated_hostname) + } + } else { + debug!("3.4: No SRV records, just use the hostname from .well-known"); + if !no_cache_dest { + query_and_cache_override(&delegated_hostname, &delegated_hostname, 8448) + .await?; + } + + add_port_to_hostname(&delegated_hostname) + } + } + }, + } + } else { + trace!("4: No .well-known or an error occured"); + if let Some(hostname_override) = query_srv_record(&dest_str).await? { + debug!("4: No .well-known; SRV record found"); + let force_port = hostname_override.port(); + + if !no_cache_dest { + query_and_cache_override( + &hostname, + &hostname_override.hostname(), + force_port.unwrap_or(8448), + ) + .await?; + } + + if let Some(port) = force_port { + FedDest::Named(hostname.clone(), format!(":{port}")) + } else { + add_port_to_hostname(&hostname) + } + } else { + debug!("4: No .well-known; 5: No SRV record found"); + if !no_cache_dest { + query_and_cache_override(&dest_str, &dest_str, 8448).await?; + } + + add_port_to_hostname(&dest_str) + } + } + } + }, + }; + + // Can't use get_ip_with_port here because we don't want to add a port + // to an IP address if it wasn't specified + let hostname = if let Ok(addr) = hostname.parse::() { + FedDest::Literal(addr) + } else if let Ok(addr) = hostname.parse::() { + FedDest::Named(addr.to_string(), ":8448".to_owned()) + } else if let Some(pos) = hostname.find(':') { + let (host, port) = hostname.split_at(pos); + FedDest::Named(host.to_owned(), port.to_owned()) + } else { + FedDest::Named(hostname, ":8448".to_owned()) + }; + + debug!("Actual destination: {actual_dest:?} hostname: {hostname:?}"); + Ok((actual_dest, hostname.into_uri_string())) +} + +#[tracing::instrument(skip_all, name = "well-known")] +async fn request_well_known(dest: &str) -> Result> { + if !services() + .globals + .resolver + .overrides + .read() + .unwrap() + .contains_key(dest) + { + query_and_cache_override(dest, dest, 8448).await?; + } + + let response = services() + .globals + .client + .well_known + .get(&format!("https://{dest}/.well-known/matrix/server")) + .send() + .await; + + trace!("response: {:?}", response); + if let Err(e) = &response { + debug!("error: {e:?}"); + return Ok(None); + } + + let response = response?; + if !response.status().is_success() { + debug!("response not 2XX"); + return Ok(None); + } + + let text = response.text().await?; + trace!("response text: {:?}", text); + if text.len() >= 12288 { + debug_warn!("response contains junk"); + return Ok(None); + } + + let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); + + let m_server = body + .get("m.server") + .unwrap_or(&serde_json::Value::Null) + .as_str() + .unwrap_or_default(); + + if ruma_identifiers_validation::server_name::validate(m_server).is_err() { + debug_error!("response content missing or invalid"); + return Ok(None); + } + + debug_info!("{:?} found at {:?}", dest, m_server); + Ok(Some(m_server.to_owned())) +} + +#[tracing::instrument(skip_all, name = "ip")] +async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { + match services() + .globals + .dns_resolver() + .lookup_ip(hostname.to_owned()) + .await + { + Err(e) => handle_resolve_error(&e), + Ok(override_ip) => { + if hostname != overname { + debug_info!("{:?} overriden by {:?}", overname, hostname); + } + services() + .globals + .resolver + .overrides + .write() + .unwrap() + .insert(overname.to_owned(), (override_ip.iter().collect(), port)); + + Ok(()) + }, + } +} + +#[tracing::instrument(skip_all, name = "srv")] +async fn query_srv_record(hostname: &'_ str) -> Result> { + fn handle_successful_srv(srv: &SrvLookup) -> Option { + srv.iter().next().map(|result| { + FedDest::Named( + result.target().to_string().trim_end_matches('.').to_owned(), + format!(":{}", result.port()), + ) + }) + } + + async fn lookup_srv(hostname: &str) -> Result { + debug!("querying SRV for {:?}", hostname); + let hostname = hostname.trim_end_matches('.'); + services() + .globals + .dns_resolver() + .srv_lookup(hostname.to_owned()) + .await + } + + let hostnames = [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")]; + + for hostname in hostnames { + match lookup_srv(&hostname).await { + Ok(result) => return Ok(handle_successful_srv(&result)), + Err(e) => handle_resolve_error(&e)?, + } + } + + Ok(None) +} + +#[allow(clippy::single_match_else)] +fn handle_resolve_error(e: &ResolveError) -> Result<()> { + use hickory_resolver::error::ResolveErrorKind; + + match *e.kind() { + ResolveErrorKind::NoRecordsFound { + .. + } => { + // Raise to debug_warn if we can find out the result wasn't from cache + debug!("{e}"); + Ok(()) + }, + _ => { + error!("DNS {e}"); + Err(Error::Err(e.to_string())) + }, + } +} + +fn validate_dest(dest: &ServerName) -> Result<()> { + if dest == services().globals.server_name() { + return Err(Error::bad_config("Won't send federation request to ourselves")); + } + + if dest.is_ip_literal() || IPAddress::is_valid(dest.host()) { + validate_dest_ip_literal(dest)?; + } + + Ok(()) +} + +fn validate_dest_ip_literal(dest: &ServerName) -> Result<()> { + trace!("Destination is an IP literal, checking against IP range denylist.",); + debug_assert!( + dest.is_ip_literal() || !IPAddress::is_valid(dest.host()), + "Destination is not an IP literal." + ); + let ip = IPAddress::parse(dest.host()).map_err(|e| { + debug_error!("Failed to parse IP literal from string: {}", e); + Error::BadServerResponse("Invalid IP address") + })?; + + validate_ip(&ip)?; + + Ok(()) +} + +pub(crate) fn validate_ip(ip: &IPAddress) -> Result<()> { + if !services().globals.valid_cidr_range(ip) { + return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); + } + + Ok(()) +} + +fn get_ip_with_port(dest_str: &str) -> Option { + if let Ok(dest) = dest_str.parse::() { + Some(FedDest::Literal(dest)) + } else if let Ok(ip_addr) = dest_str.parse::() { + Some(FedDest::Literal(SocketAddr::new(ip_addr, 8448))) + } else { + None + } +} + +fn add_port_to_hostname(dest_str: &str) -> FedDest { + let (host, port) = match dest_str.find(':') { + None => (dest_str, ":8448"), + Some(pos) => dest_str.split_at(pos), + }; + + FedDest::Named(host.to_owned(), port.to_owned()) +} + +impl FedDest { + fn into_https_string(self) -> String { + match self { + Self::Literal(addr) => format!("https://{addr}"), + Self::Named(host, port) => format!("https://{host}{port}"), + } + } + + fn into_uri_string(self) -> String { + match self { + Self::Literal(addr) => addr.to_string(), + Self::Named(host, port) => format!("{host}{port}"), + } + } + + fn hostname(&self) -> String { + match &self { + Self::Literal(addr) => addr.ip().to_string(), + Self::Named(host, _) => host.clone(), + } + } + + fn port(&self) -> Option { + match &self { + Self::Literal(addr) => Some(addr.port()), + Self::Named(_, port) => port[1..].parse().ok(), + } + } +} + +#[cfg(test)] +mod tests { + use super::{add_port_to_hostname, get_ip_with_port, FedDest}; + + #[test] + fn ips_get_default_ports() { + assert_eq!( + get_ip_with_port("1.1.1.1"), + Some(FedDest::Literal("1.1.1.1:8448".parse().unwrap())) + ); + assert_eq!( + get_ip_with_port("dead:beef::"), + Some(FedDest::Literal("[dead:beef::]:8448".parse().unwrap())) + ); + } + + #[test] + fn ips_keep_custom_ports() { + assert_eq!( + get_ip_with_port("1.1.1.1:1234"), + Some(FedDest::Literal("1.1.1.1:1234".parse().unwrap())) + ); + assert_eq!( + get_ip_with_port("[dead::beef]:8933"), + Some(FedDest::Literal("[dead::beef]:8933".parse().unwrap())) + ); + } + + #[test] + fn hostnames_get_default_ports() { + assert_eq!( + add_port_to_hostname("example.com"), + FedDest::Named(String::from("example.com"), String::from(":8448")) + ); + } + + #[test] + fn hostnames_keep_custom_ports() { + assert_eq!( + add_port_to_hostname("example.com:1337"), + FedDest::Named(String::from("example.com"), String::from(":1337")) + ); + } +} diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index a835e438..e8432d12 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,10 +1,5 @@ -use std::{ - fmt::Debug, - mem, - net::{IpAddr, SocketAddr}, -}; +use std::{fmt::Debug, mem}; -use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; use reqwest::{Client, Method, Request, Response, Url}; @@ -15,40 +10,10 @@ use ruma::{ }, OwnedServerName, ServerName, }; -use tracing::{debug, error, trace}; +use tracing::{debug, trace}; -use crate::{debug_error, debug_info, debug_warn, services, Error, Result}; - -/// Wraps either an literal IP address plus port, or a hostname plus complement -/// (colon-plus-port if it was specified). -/// -/// Note: A `FedDest::Named` might contain an IP address in string form if there -/// was no port specified to construct a `SocketAddr` with. -/// -/// # Examples: -/// ```rust -/// # use conduit_service::sending::FedDest; -/// # fn main() -> Result<(), std::net::AddrParseError> { -/// FedDest::Literal("198.51.100.3:8448".parse()?); -/// FedDest::Literal("[2001:db8::4:5]:443".parse()?); -/// FedDest::Named("matrix.example.org".to_owned(), String::new()); -/// FedDest::Named("matrix.example.org".to_owned(), ":8448".to_owned()); -/// FedDest::Named("198.51.100.5".to_owned(), String::new()); -/// # Ok(()) -/// # } -/// ``` -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum FedDest { - Literal(SocketAddr), - Named(String, String), -} - -struct ActualDest { - dest: FedDest, - host: String, - string: String, - cached: bool, -} +use super::{resolve, resolve::ActualDest}; +use crate::{debug_error, debug_warn, services, Error, Result}; #[tracing::instrument(skip_all, name = "send")] pub async fn send(client: &Client, dest: &ServerName, req: T) -> Result @@ -59,7 +24,7 @@ where return Err(Error::bad_config("Federation is disabled.")); } - let actual = get_actual_dest(dest).await?; + let actual = resolve::get_actual_dest(dest).await?; let request = prepare::(dest, &actual, req).await?; execute::(client, dest, &actual, request).await } @@ -177,294 +142,6 @@ where Err(e.into()) } -#[tracing::instrument(skip_all, name = "resolve")] -async fn get_actual_dest(server_name: &ServerName) -> Result { - let cached; - let cached_result = services() - .globals - .actual_destinations() - .read() - .await - .get(server_name) - .cloned(); - - let (dest, host) = if let Some(result) = cached_result { - cached = true; - result - } else { - cached = false; - validate_dest(server_name)?; - resolve_actual_dest(server_name, false).await? - }; - - let string = dest.clone().into_https_string(); - Ok(ActualDest { - dest, - host, - string, - cached, - }) -} - -/// Returns: `actual_destination`, host header -/// Implemented according to the specification at -/// Numbers in comments below refer to bullet points in linked section of -/// specification -pub async fn resolve_actual_dest(dest: &ServerName, no_cache_dest: bool) -> Result<(FedDest, String)> { - trace!("Finding actual destination for {dest}"); - let dest_str = dest.as_str().to_owned(); - let mut hostname = dest_str.clone(); - - #[allow(clippy::single_match_else)] - let actual_dest = match get_ip_with_port(&dest_str) { - Some(host_port) => { - debug!("1: IP literal with provided or default port"); - host_port - }, - None => { - if let Some(pos) = dest_str.find(':') { - debug!("2: Hostname with included port"); - let (host, port) = dest_str.split_at(pos); - if !no_cache_dest { - query_and_cache_override(host, host, port.parse::().unwrap_or(8448)).await?; - } - - FedDest::Named(host.to_owned(), port.to_owned()) - } else { - trace!("Requesting well known for {dest}"); - if let Some(delegated_hostname) = request_well_known(dest.as_str()).await? { - debug!("3: A .well-known file is available"); - hostname = add_port_to_hostname(&delegated_hostname).into_uri_string(); - match get_ip_with_port(&delegated_hostname) { - Some(host_and_port) => { - debug!("3.1: IP literal in .well-known file"); - host_and_port - }, - None => { - if let Some(pos) = delegated_hostname.find(':') { - debug!("3.2: Hostname with port in .well-known file"); - let (host, port) = delegated_hostname.split_at(pos); - if !no_cache_dest { - query_and_cache_override(host, host, port.parse::().unwrap_or(8448)).await?; - } - - FedDest::Named(host.to_owned(), port.to_owned()) - } else { - trace!("Delegated hostname has no port in this branch"); - if let Some(hostname_override) = query_srv_record(&delegated_hostname).await? { - debug!("3.3: SRV lookup successful"); - let force_port = hostname_override.port(); - if !no_cache_dest { - query_and_cache_override( - &delegated_hostname, - &hostname_override.hostname(), - force_port.unwrap_or(8448), - ) - .await?; - } - - if let Some(port) = force_port { - FedDest::Named(delegated_hostname, format!(":{port}")) - } else { - add_port_to_hostname(&delegated_hostname) - } - } else { - debug!("3.4: No SRV records, just use the hostname from .well-known"); - if !no_cache_dest { - query_and_cache_override(&delegated_hostname, &delegated_hostname, 8448) - .await?; - } - - add_port_to_hostname(&delegated_hostname) - } - } - }, - } - } else { - trace!("4: No .well-known or an error occured"); - if let Some(hostname_override) = query_srv_record(&dest_str).await? { - debug!("4: No .well-known; SRV record found"); - let force_port = hostname_override.port(); - - if !no_cache_dest { - query_and_cache_override( - &hostname, - &hostname_override.hostname(), - force_port.unwrap_or(8448), - ) - .await?; - } - - if let Some(port) = force_port { - FedDest::Named(hostname.clone(), format!(":{port}")) - } else { - add_port_to_hostname(&hostname) - } - } else { - debug!("4: No .well-known; 5: No SRV record found"); - if !no_cache_dest { - query_and_cache_override(&dest_str, &dest_str, 8448).await?; - } - - add_port_to_hostname(&dest_str) - } - } - } - }, - }; - - // Can't use get_ip_with_port here because we don't want to add a port - // to an IP address if it wasn't specified - let hostname = if let Ok(addr) = hostname.parse::() { - FedDest::Literal(addr) - } else if let Ok(addr) = hostname.parse::() { - FedDest::Named(addr.to_string(), ":8448".to_owned()) - } else if let Some(pos) = hostname.find(':') { - let (host, port) = hostname.split_at(pos); - FedDest::Named(host.to_owned(), port.to_owned()) - } else { - FedDest::Named(hostname, ":8448".to_owned()) - }; - - debug!("Actual destination: {actual_dest:?} hostname: {hostname:?}"); - Ok((actual_dest, hostname.into_uri_string())) -} - -#[tracing::instrument(skip_all, name = "well-known")] -async fn request_well_known(dest: &str) -> Result> { - if !services() - .globals - .resolver - .overrides - .read() - .unwrap() - .contains_key(dest) - { - query_and_cache_override(dest, dest, 8448).await?; - } - - let response = services() - .globals - .client - .well_known - .get(&format!("https://{dest}/.well-known/matrix/server")) - .send() - .await; - - trace!("response: {:?}", response); - if let Err(e) = &response { - debug!("error: {e:?}"); - return Ok(None); - } - - let response = response?; - if !response.status().is_success() { - debug!("response not 2XX"); - return Ok(None); - } - - let text = response.text().await?; - trace!("response text: {:?}", text); - if text.len() >= 12288 { - debug_warn!("response contains junk"); - return Ok(None); - } - - let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); - - let m_server = body - .get("m.server") - .unwrap_or(&serde_json::Value::Null) - .as_str() - .unwrap_or_default(); - - if ruma_identifiers_validation::server_name::validate(m_server).is_err() { - debug_error!("response content missing or invalid"); - return Ok(None); - } - - debug_info!("{:?} found at {:?}", dest, m_server); - Ok(Some(m_server.to_owned())) -} - -#[tracing::instrument(skip_all, name = "ip")] -async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { - match services() - .globals - .dns_resolver() - .lookup_ip(hostname.to_owned()) - .await - { - Err(e) => handle_resolve_error(&e), - Ok(override_ip) => { - if hostname != overname { - debug_info!("{:?} overriden by {:?}", overname, hostname); - } - services() - .globals - .resolver - .overrides - .write() - .unwrap() - .insert(overname.to_owned(), (override_ip.iter().collect(), port)); - - Ok(()) - }, - } -} - -#[tracing::instrument(skip_all, name = "srv")] -async fn query_srv_record(hostname: &'_ str) -> Result> { - fn handle_successful_srv(srv: &SrvLookup) -> Option { - srv.iter().next().map(|result| { - FedDest::Named( - result.target().to_string().trim_end_matches('.').to_owned(), - format!(":{}", result.port()), - ) - }) - } - - async fn lookup_srv(hostname: &str) -> Result { - debug!("querying SRV for {:?}", hostname); - let hostname = hostname.trim_end_matches('.'); - services() - .globals - .dns_resolver() - .srv_lookup(hostname.to_owned()) - .await - } - - let hostnames = [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")]; - - for hostname in hostnames { - match lookup_srv(&hostname).await { - Ok(result) => return Ok(handle_successful_srv(&result)), - Err(e) => handle_resolve_error(&e)?, - } - } - - Ok(None) -} - -#[allow(clippy::single_match_else)] -fn handle_resolve_error(e: &ResolveError) -> Result<()> { - use hickory_resolver::error::ResolveErrorKind; - - match *e.kind() { - ResolveErrorKind::NoRecordsFound { - .. - } => { - // Raise to debug_warn if we can find out the result wasn't from cache - debug!("{e}"); - Ok(()) - }, - _ => { - error!("DNS {e}"); - Err(Error::Err(e.to_string())) - }, - } -} - fn sign_request(dest: &ServerName, http_request: &mut http::Request>) where T: OutgoingRequest + Debug, @@ -533,139 +210,9 @@ fn validate_url(url: &Url) -> Result<()> { if let Some(url_host) = url.host_str() { if let Ok(ip) = IPAddress::parse(url_host) { trace!("Checking request URL IP {ip:?}"); - validate_ip(&ip)?; + resolve::validate_ip(&ip)?; } } Ok(()) } - -fn validate_dest(dest: &ServerName) -> Result<()> { - if dest == services().globals.server_name() { - return Err(Error::bad_config("Won't send federation request to ourselves")); - } - - if dest.is_ip_literal() || IPAddress::is_valid(dest.host()) { - validate_dest_ip_literal(dest)?; - } - - Ok(()) -} - -fn validate_dest_ip_literal(dest: &ServerName) -> Result<()> { - trace!("Destination is an IP literal, checking against IP range denylist.",); - debug_assert!( - dest.is_ip_literal() || !IPAddress::is_valid(dest.host()), - "Destination is not an IP literal." - ); - let ip = IPAddress::parse(dest.host()).map_err(|e| { - debug_error!("Failed to parse IP literal from string: {}", e); - Error::BadServerResponse("Invalid IP address") - })?; - - validate_ip(&ip)?; - - Ok(()) -} - -fn validate_ip(ip: &IPAddress) -> Result<()> { - if !services().globals.valid_cidr_range(ip) { - return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); - } - - Ok(()) -} - -fn get_ip_with_port(dest_str: &str) -> Option { - if let Ok(dest) = dest_str.parse::() { - Some(FedDest::Literal(dest)) - } else if let Ok(ip_addr) = dest_str.parse::() { - Some(FedDest::Literal(SocketAddr::new(ip_addr, 8448))) - } else { - None - } -} - -fn add_port_to_hostname(dest_str: &str) -> FedDest { - let (host, port) = match dest_str.find(':') { - None => (dest_str, ":8448"), - Some(pos) => dest_str.split_at(pos), - }; - - FedDest::Named(host.to_owned(), port.to_owned()) -} - -impl FedDest { - fn into_https_string(self) -> String { - match self { - Self::Literal(addr) => format!("https://{addr}"), - Self::Named(host, port) => format!("https://{host}{port}"), - } - } - - fn into_uri_string(self) -> String { - match self { - Self::Literal(addr) => addr.to_string(), - Self::Named(host, port) => format!("{host}{port}"), - } - } - - fn hostname(&self) -> String { - match &self { - Self::Literal(addr) => addr.ip().to_string(), - Self::Named(host, _) => host.clone(), - } - } - - fn port(&self) -> Option { - match &self { - Self::Literal(addr) => Some(addr.port()), - Self::Named(_, port) => port[1..].parse().ok(), - } - } -} - -#[cfg(test)] -mod tests { - use super::{add_port_to_hostname, get_ip_with_port, FedDest}; - - #[test] - fn ips_get_default_ports() { - assert_eq!( - get_ip_with_port("1.1.1.1"), - Some(FedDest::Literal("1.1.1.1:8448".parse().unwrap())) - ); - assert_eq!( - get_ip_with_port("dead:beef::"), - Some(FedDest::Literal("[dead:beef::]:8448".parse().unwrap())) - ); - } - - #[test] - fn ips_keep_custom_ports() { - assert_eq!( - get_ip_with_port("1.1.1.1:1234"), - Some(FedDest::Literal("1.1.1.1:1234".parse().unwrap())) - ); - assert_eq!( - get_ip_with_port("[dead::beef]:8933"), - Some(FedDest::Literal("[dead::beef]:8933".parse().unwrap())) - ); - } - - #[test] - fn hostnames_get_default_ports() { - assert_eq!( - add_port_to_hostname("example.com"), - FedDest::Named(String::from("example.com"), String::from(":8448")) - ); - } - - #[test] - fn hostnames_keep_custom_ports() { - assert_eq!( - add_port_to_hostname("example.com:1337"), - FedDest::Named(String::from("example.com"), String::from(":1337")) - ); - } -} From ba48758b89dba451551fd243ac2abd1276e01df4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 May 2024 01:21:15 +0000 Subject: [PATCH 0023/2091] impl fmt::Display for FedDest Signed-off-by: Jason Volk --- src/admin/debug/debug_commands.rs | 2 +- src/service/sending/resolve.rs | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 873a23e7..27f259a6 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -448,7 +448,7 @@ pub(crate) async fn resolve_true_destination( let (actual_dest, hostname_uri) = resolve_actual_dest(&server_name, no_cache).await?; Ok(RoomMessageEventContent::text_plain(format!( - "Actual destination: {actual_dest:?} | Hostname URI: {hostname_uri}" + "Actual destination: {actual_dest} | Hostname URI: {hostname_uri}" ))) } diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index d452a555..243180ed 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -1,4 +1,5 @@ use std::{ + fmt, fmt::Debug, net::{IpAddr, SocketAddr}, }; @@ -414,6 +415,15 @@ impl FedDest { } } +impl fmt::Display for FedDest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + FedDest::Named(host, port) => write!(f, "{host}{port}"), + FedDest::Literal(addr) => write!(f, "{addr}"), + } + } +} + #[cfg(test)] mod tests { use super::{add_port_to_hostname, get_ip_with_port, FedDest}; From f1d1366129ffc64e0aad1400664fed3cab15a98f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 May 2024 05:08:00 +0000 Subject: [PATCH 0024/2091] split resolve_actual_dest Signed-off-by: Jason Volk --- src/admin/debug/debug_commands.rs | 2 +- src/service/sending/resolve.rs | 211 +++++++++++++++--------------- 2 files changed, 108 insertions(+), 105 deletions(-) diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 27f259a6..6a5cee35 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -445,7 +445,7 @@ pub(crate) async fn resolve_true_destination( )); } - let (actual_dest, hostname_uri) = resolve_actual_dest(&server_name, no_cache).await?; + let (actual_dest, hostname_uri) = resolve_actual_dest(&server_name, !no_cache).await?; Ok(RoomMessageEventContent::text_plain(format!( "Actual destination: {actual_dest} | Hostname URI: {hostname_uri}" diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 243180ed..b79f18c8 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -59,7 +59,7 @@ pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result Result /// Numbers in comments below refer to bullet points in linked section of /// specification -pub async fn resolve_actual_dest(dest: &ServerName, no_cache_dest: bool) -> Result<(FedDest, String)> { +pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result<(FedDest, String)> { trace!("Finding actual destination for {dest}"); - let dest_str = dest.as_str().to_owned(); - let mut hostname = dest_str.clone(); - - #[allow(clippy::single_match_else)] - let actual_dest = match get_ip_with_port(&dest_str) { - Some(host_port) => { - debug!("1: IP literal with provided or default port"); - host_port - }, + let mut host = dest.as_str().to_owned(); + let actual_dest = match get_ip_with_port(dest.as_str()) { + Some(host_port) => actual_dest_1(host_port)?, None => { - if let Some(pos) = dest_str.find(':') { - debug!("2: Hostname with included port"); - let (host, port) = dest_str.split_at(pos); - if !no_cache_dest { - query_and_cache_override(host, host, port.parse::().unwrap_or(8448)).await?; - } - - FedDest::Named(host.to_owned(), port.to_owned()) + if let Some(pos) = dest.as_str().find(':') { + actual_dest_2(dest, cache, pos).await? + } else if let Some(delegated) = request_well_known(dest.as_str()).await? { + actual_dest_3(&mut host, cache, delegated).await? + } else if let Some(overrider) = query_srv_record(dest.as_str()).await? { + actual_dest_4(&host, cache, overrider).await? } else { - trace!("Requesting well known for {dest}"); - if let Some(delegated_hostname) = request_well_known(dest.as_str()).await? { - debug!("3: A .well-known file is available"); - hostname = add_port_to_hostname(&delegated_hostname).into_uri_string(); - match get_ip_with_port(&delegated_hostname) { - Some(host_and_port) => { - debug!("3.1: IP literal in .well-known file"); - host_and_port - }, - None => { - if let Some(pos) = delegated_hostname.find(':') { - debug!("3.2: Hostname with port in .well-known file"); - let (host, port) = delegated_hostname.split_at(pos); - if !no_cache_dest { - query_and_cache_override(host, host, port.parse::().unwrap_or(8448)).await?; - } - - FedDest::Named(host.to_owned(), port.to_owned()) - } else { - trace!("Delegated hostname has no port in this branch"); - if let Some(hostname_override) = query_srv_record(&delegated_hostname).await? { - debug!("3.3: SRV lookup successful"); - let force_port = hostname_override.port(); - if !no_cache_dest { - query_and_cache_override( - &delegated_hostname, - &hostname_override.hostname(), - force_port.unwrap_or(8448), - ) - .await?; - } - - if let Some(port) = force_port { - FedDest::Named(delegated_hostname, format!(":{port}")) - } else { - add_port_to_hostname(&delegated_hostname) - } - } else { - debug!("3.4: No SRV records, just use the hostname from .well-known"); - if !no_cache_dest { - query_and_cache_override(&delegated_hostname, &delegated_hostname, 8448) - .await?; - } - - add_port_to_hostname(&delegated_hostname) - } - } - }, - } - } else { - trace!("4: No .well-known or an error occured"); - if let Some(hostname_override) = query_srv_record(&dest_str).await? { - debug!("4: No .well-known; SRV record found"); - let force_port = hostname_override.port(); - - if !no_cache_dest { - query_and_cache_override( - &hostname, - &hostname_override.hostname(), - force_port.unwrap_or(8448), - ) - .await?; - } - - if let Some(port) = force_port { - FedDest::Named(hostname.clone(), format!(":{port}")) - } else { - add_port_to_hostname(&hostname) - } - } else { - debug!("4: No .well-known; 5: No SRV record found"); - if !no_cache_dest { - query_and_cache_override(&dest_str, &dest_str, 8448).await?; - } - - add_port_to_hostname(&dest_str) - } - } + actual_dest_5(dest, cache).await? } }, }; // Can't use get_ip_with_port here because we don't want to add a port // to an IP address if it wasn't specified - let hostname = if let Ok(addr) = hostname.parse::() { + let host = if let Ok(addr) = host.parse::() { FedDest::Literal(addr) - } else if let Ok(addr) = hostname.parse::() { + } else if let Ok(addr) = host.parse::() { FedDest::Named(addr.to_string(), ":8448".to_owned()) - } else if let Some(pos) = hostname.find(':') { - let (host, port) = hostname.split_at(pos); + } else if let Some(pos) = host.find(':') { + let (host, port) = host.split_at(pos); FedDest::Named(host.to_owned(), port.to_owned()) } else { - FedDest::Named(hostname, ":8448".to_owned()) + FedDest::Named(host, ":8448".to_owned()) }; - debug!("Actual destination: {actual_dest:?} hostname: {hostname:?}"); - Ok((actual_dest, hostname.into_uri_string())) + debug!("Actual destination: {actual_dest:?} hostname: {host:?}"); + Ok((actual_dest, host.into_uri_string())) +} + +fn actual_dest_1(host_port: FedDest) -> Result { + debug!("1: IP literal with provided or default port"); + Ok(host_port) +} + +async fn actual_dest_2(dest: &ServerName, cache: bool, pos: usize) -> Result { + debug!("2: Hostname with included port"); + let (host, port) = dest.as_str().split_at(pos); + conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache).await?; + Ok(FedDest::Named(host.to_owned(), port.to_owned())) +} + +async fn actual_dest_3(host: &mut String, cache: bool, delegated: String) -> Result { + debug!("3: A .well-known file is available"); + *host = add_port_to_hostname(&delegated).into_uri_string(); + match get_ip_with_port(&delegated) { + Some(host_and_port) => actual_dest_3_1(host_and_port), + None => { + if let Some(pos) = delegated.find(':') { + actual_dest_3_2(cache, delegated, pos).await + } else { + trace!("Delegated hostname has no port in this branch"); + if let Some(overrider) = query_srv_record(&delegated).await? { + actual_dest_3_3(cache, delegated, overrider).await + } else { + actual_dest_3_4(cache, delegated).await + } + } + }, + } +} + +fn actual_dest_3_1(host_and_port: FedDest) -> Result { + debug!("3.1: IP literal in .well-known file"); + Ok(host_and_port) +} + +async fn actual_dest_3_2(cache: bool, delegated: String, pos: usize) -> Result { + debug!("3.2: Hostname with port in .well-known file"); + let (host, port) = delegated.split_at(pos); + conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache).await?; + Ok(FedDest::Named(host.to_owned(), port.to_owned())) +} + +async fn actual_dest_3_3(cache: bool, delegated: String, overrider: FedDest) -> Result { + debug!("3.3: SRV lookup successful"); + let force_port = overrider.port(); + conditional_query_and_cache_override(&delegated, &overrider.hostname(), force_port.unwrap_or(8448), cache).await?; + if let Some(port) = force_port { + Ok(FedDest::Named(delegated, format!(":{port}"))) + } else { + Ok(add_port_to_hostname(&delegated)) + } +} + +async fn actual_dest_3_4(cache: bool, delegated: String) -> Result { + debug!("3.4: No SRV records, just use the hostname from .well-known"); + conditional_query_and_cache_override(&delegated, &delegated, 8448, cache).await?; + Ok(add_port_to_hostname(&delegated)) +} + +async fn actual_dest_4(host: &str, cache: bool, overrider: FedDest) -> Result { + debug!("4: No .well-known; SRV record found"); + let force_port = overrider.port(); + conditional_query_and_cache_override(host, &overrider.hostname(), force_port.unwrap_or(8448), cache).await?; + if let Some(port) = force_port { + Ok(FedDest::Named(host.to_owned(), format!(":{port}"))) + } else { + Ok(add_port_to_hostname(host)) + } +} + +async fn actual_dest_5(dest: &ServerName, cache: bool) -> Result { + debug!("5: No SRV record found"); + conditional_query_and_cache_override(dest.as_str(), dest.as_str(), 8448, cache).await?; + Ok(add_port_to_hostname(dest.as_str())) } #[tracing::instrument(skip_all, name = "well-known")] async fn request_well_known(dest: &str) -> Result> { + trace!("Requesting well known for {dest}"); if !services() .globals .resolver @@ -252,6 +246,15 @@ async fn request_well_known(dest: &str) -> Result> { Ok(Some(m_server.to_owned())) } +#[inline] +async fn conditional_query_and_cache_override(overname: &str, hostname: &str, port: u16, cache: bool) -> Result<()> { + if cache { + query_and_cache_override(overname, hostname, port).await + } else { + Ok(()) + } +} + #[tracing::instrument(skip_all, name = "ip")] async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { match services() From 5fe5ab279c99a5cf0b99d452f16bb1427591d7f0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 28 May 2024 04:52:58 +0000 Subject: [PATCH 0025/2091] split RouterExt impl related into ruma_wrapper unit. slightly restrict client_server mod index. Signed-off-by: Jason Volk --- src/api/client_server/mod.rs | 140 ++++++++++++++++----------------- src/api/mod.rs | 4 +- src/api/router.rs | 75 ++---------------- src/api/ruma_wrapper/mod.rs | 3 + src/api/ruma_wrapper/router.rs | 69 ++++++++++++++++ 5 files changed, 149 insertions(+), 142 deletions(-) create mode 100644 src/api/ruma_wrapper/router.rs diff --git a/src/api/client_server/mod.rs b/src/api/client_server/mod.rs index 171e9bbe..ced2bde5 100644 --- a/src/api/client_server/mod.rs +++ b/src/api/client_server/mod.rs @@ -1,76 +1,76 @@ -pub(crate) mod account; -pub(crate) mod alias; -pub(crate) mod backup; -pub(crate) mod capabilities; -pub(crate) mod config; -pub(crate) mod context; -pub(crate) mod device; -pub(crate) mod directory; -pub(crate) mod filter; -pub(crate) mod keys; -pub(crate) mod media; -pub(crate) mod membership; -pub(crate) mod message; -pub(crate) mod presence; -pub(crate) mod profile; -pub(crate) mod push; -pub(crate) mod read_marker; -pub(crate) mod redact; -pub(crate) mod relations; -pub(crate) mod report; -pub(crate) mod room; -pub(crate) mod search; -pub(crate) mod session; -pub(crate) mod space; -pub(crate) mod state; -pub(crate) mod sync; -pub(crate) mod tag; -pub(crate) mod thirdparty; -pub(crate) mod threads; -pub(crate) mod to_device; -pub(crate) mod typing; -pub(crate) mod unstable; -pub(crate) mod unversioned; -pub(crate) mod user_directory; -pub(crate) mod voip; +pub(super) mod account; +pub(super) mod alias; +pub(super) mod backup; +pub(super) mod capabilities; +pub(super) mod config; +pub(super) mod context; +pub(super) mod device; +pub(super) mod directory; +pub(super) mod filter; +pub(super) mod keys; +pub(super) mod media; +pub(super) mod membership; +pub(super) mod message; +pub(super) mod presence; +pub(super) mod profile; +pub(super) mod push; +pub(super) mod read_marker; +pub(super) mod redact; +pub(super) mod relations; +pub(super) mod report; +pub(super) mod room; +pub(super) mod search; +pub(super) mod session; +pub(super) mod space; +pub(super) mod state; +pub(super) mod sync; +pub(super) mod tag; +pub(super) mod thirdparty; +pub(super) mod threads; +pub(super) mod to_device; +pub(super) mod typing; +pub(super) mod unstable; +pub(super) mod unversioned; +pub(super) mod user_directory; +pub(super) mod voip; -pub(crate) use account::*; +pub(super) use account::*; pub use alias::get_alias_helper; -pub(crate) use alias::*; -pub(crate) use backup::*; -pub(crate) use capabilities::*; -pub(crate) use config::*; -pub(crate) use context::*; -pub(crate) use device::*; -pub(crate) use directory::*; -pub(crate) use filter::*; -pub(crate) use keys::*; -pub(crate) use media::*; -pub(crate) use membership::*; +pub(super) use alias::*; +pub(super) use backup::*; +pub(super) use capabilities::*; +pub(super) use config::*; +pub(super) use context::*; +pub(super) use device::*; +pub(super) use directory::*; +pub(super) use filter::*; +pub(super) use keys::*; +pub(super) use media::*; +pub(super) use membership::*; pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room}; -pub(crate) use message::*; -pub(crate) use presence::*; -pub(crate) use profile::*; -pub(crate) use push::*; -pub(crate) use read_marker::*; -pub(crate) use redact::*; -pub(crate) use relations::*; -pub(crate) use report::*; -pub(crate) use room::*; -pub(crate) use search::*; -pub(crate) use session::*; -pub(crate) use space::*; -pub(crate) use state::*; -pub(crate) use sync::*; -pub(crate) use tag::*; -pub(crate) use thirdparty::*; -pub(crate) use threads::*; -pub(crate) use to_device::*; -pub(crate) use typing::*; -pub(crate) use unstable::*; -pub(crate) use unversioned::*; -pub(crate) use user_directory::*; -pub(crate) use voip::*; +pub(super) use message::*; +pub(super) use presence::*; +pub(super) use profile::*; +pub(super) use push::*; +pub(super) use read_marker::*; +pub(super) use redact::*; +pub(super) use relations::*; +pub(super) use report::*; +pub(super) use room::*; +pub(super) use search::*; +pub(super) use session::*; +pub(super) use space::*; +pub(super) use state::*; +pub(super) use sync::*; +pub(super) use tag::*; +pub(super) use thirdparty::*; +pub(super) use threads::*; +pub(super) use to_device::*; +pub(super) use typing::*; +pub(super) use unstable::*; +pub(super) use unversioned::*; +pub(super) use user_directory::*; +pub(super) use voip::*; /// generated device ID length const DEVICE_ID_LENGTH: usize = 10; diff --git a/src/api/mod.rs b/src/api/mod.rs index 956bcdf7..b835f536 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,8 +7,8 @@ extern crate conduit_core as conduit; extern crate conduit_service as service; pub use client_server::membership::{join_room_by_id_helper, leave_all_rooms}; -pub(crate) use conduit::{debug_error, debug_info, debug_warn, error::RumaResponse, utils, Error, Result}; -pub(crate) use ruma_wrapper::Ruma; +pub(crate) use conduit::{debug_error, debug_info, debug_warn, utils, Error, Result}; +pub(crate) use ruma_wrapper::{Ruma, RumaResponse}; pub(crate) use service::{pdu::PduEvent, services, user_is_local}; conduit::mod_ctor! {} diff --git a/src/api/router.rs b/src/api/router.rs index 6081a089..90d69873 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -1,15 +1,13 @@ -use std::future::Future; - use axum::{ response::IntoResponse, - routing::{any, get, on, post, MethodFilter}, + routing::{any, get, post}, Router, }; -use conduit::{Error, Result, Server}; -use http::{Method, Uri}; -use ruma::api::{client::error::ErrorKind, IncomingRequest}; +use conduit::{Error, Server}; +use http::Uri; +use ruma::api::client::error::ErrorKind; -use crate::{client_server, server_server, Ruma, RumaResponse}; +use crate::{client_server, ruma_wrapper::RouterExt, server_server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; @@ -234,66 +232,3 @@ async fn initial_sync(_uri: Uri) -> impl IntoResponse { } async fn federation_disabled() -> impl IntoResponse { Error::bad_config("Federation is disabled.") } - -trait RouterExt { - fn ruma_route(self, handler: H) -> Self - where - H: RumaHandler, - T: 'static; -} - -impl RouterExt for Router { - #[inline(always)] - fn ruma_route(self, handler: H) -> Self - where - H: RumaHandler, - T: 'static, - { - handler.add_routes(self) - } -} - -trait RumaHandler { - fn add_routes(&self, router: Router) -> Router; - - fn add_route(&self, router: Router, path: &str) -> Router; -} - -impl RumaHandler> for F -where - Req: IncomingRequest + Send + 'static, - F: FnOnce(Ruma) -> Fut + Clone + Send + Sync + 'static, - Fut: Future> + Send, - E: IntoResponse, -{ - #[inline(always)] - fn add_routes(&self, router: Router) -> Router { - Req::METADATA - .history - .all_paths() - .fold(router, |router, path| self.add_route(router, path)) - } - - #[inline(always)] - fn add_route(&self, router: Router, path: &str) -> Router { - let handle = self.clone(); - let method = method_to_filter(Req::METADATA.method); - let action = |req| async { handle(req).await.map(RumaResponse) }; - router.route(path, on(method, action)) - } -} - -#[inline] -fn method_to_filter(method: Method) -> MethodFilter { - match method { - Method::DELETE => MethodFilter::DELETE, - Method::GET => MethodFilter::GET, - Method::HEAD => MethodFilter::HEAD, - Method::OPTIONS => MethodFilter::OPTIONS, - Method::PATCH => MethodFilter::PATCH, - Method::POST => MethodFilter::POST, - Method::PUT => MethodFilter::PUT, - Method::TRACE => MethodFilter::TRACE, - m => panic!("Unsupported HTTP method: {m:?}"), - } -} diff --git a/src/api/ruma_wrapper/mod.rs b/src/api/ruma_wrapper/mod.rs index 1e12995f..a130ddd9 100644 --- a/src/api/ruma_wrapper/mod.rs +++ b/src/api/ruma_wrapper/mod.rs @@ -1,11 +1,14 @@ mod auth; mod request; +mod router; mod xmatrix; use std::ops::Deref; +pub(super) use conduit::error::RumaResponse; use ruma::{CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId}; +pub(super) use self::router::RouterExt; use crate::service::appservice::RegistrationInfo; /// Extractor for Ruma request structs diff --git a/src/api/ruma_wrapper/router.rs b/src/api/ruma_wrapper/router.rs new file mode 100644 index 00000000..f769d2bb --- /dev/null +++ b/src/api/ruma_wrapper/router.rs @@ -0,0 +1,69 @@ +use std::future::Future; + +use axum::{ + response::IntoResponse, + routing::{on, MethodFilter}, + Router, +}; +use conduit::Result; +use http::Method; +use ruma::api::IncomingRequest; + +use super::{Ruma, RumaResponse}; + +pub(in super::super) trait RouterExt { + fn ruma_route(self, handler: H) -> Self + where + H: RumaHandler; +} + +impl RouterExt for Router { + fn ruma_route(self, handler: H) -> Self + where + H: RumaHandler, + { + handler.add_routes(self) + } +} + +pub(in super::super) trait RumaHandler { + fn add_routes(&self, router: Router) -> Router; + + fn add_route(&self, router: Router, path: &str) -> Router; +} + +impl RumaHandler> for F +where + Req: IncomingRequest + Send + 'static, + F: FnOnce(Ruma) -> Fut + Clone + Send + Sync + 'static, + Fut: Future> + Send, + E: IntoResponse, +{ + fn add_routes(&self, router: Router) -> Router { + Req::METADATA + .history + .all_paths() + .fold(router, |router, path| self.add_route(router, path)) + } + + fn add_route(&self, router: Router, path: &str) -> Router { + let handle = self.clone(); + let method = method_to_filter(&Req::METADATA.method); + let action = |req| async { handle(req).await.map(RumaResponse) }; + router.route(path, on(method, action)) + } +} + +const fn method_to_filter(method: &Method) -> MethodFilter { + match *method { + Method::DELETE => MethodFilter::DELETE, + Method::GET => MethodFilter::GET, + Method::HEAD => MethodFilter::HEAD, + Method::OPTIONS => MethodFilter::OPTIONS, + Method::PATCH => MethodFilter::PATCH, + Method::POST => MethodFilter::POST, + Method::PUT => MethodFilter::PUT, + Method::TRACE => MethodFilter::TRACE, + _ => panic!("Unsupported HTTP method"), + } +} From 90d9a997a5096532f7fbaa279ec19fe43cce492b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 27 May 2024 20:05:33 +0000 Subject: [PATCH 0026/2091] split / cleanup core utils. Signed-off-by: Jason Volk --- src/api/client_server/unversioned.rs | 6 +- src/api/server_server.rs | 2 +- src/core/debug.rs | 3 + src/core/mod.rs | 2 +- src/core/utils/debug.rs | 40 ++++++ src/core/utils/html.rs | 37 ++++++ src/core/utils/json.rs | 39 ++++++ src/core/utils/mod.rs | 191 ++------------------------- src/core/utils/sys.rs | 36 +++++ src/core/version.rs | 28 ++++ src/main/clap.rs | 3 +- src/main/server.rs | 5 +- src/service/globals/client.rs | 4 +- 13 files changed, 205 insertions(+), 191 deletions(-) create mode 100644 src/core/utils/debug.rs create mode 100644 src/core/utils/html.rs create mode 100644 src/core/utils/json.rs create mode 100644 src/core/utils/sys.rs create mode 100644 src/core/version.rs diff --git a/src/api/client_server/unversioned.rs b/src/api/client_server/unversioned.rs index 69a477a6..56ab9a90 100644 --- a/src/api/client_server/unversioned.rs +++ b/src/api/client_server/unversioned.rs @@ -10,7 +10,7 @@ use ruma::api::client::{ error::ErrorKind, }; -use crate::{services, utils::conduwuit_version, Error, Result, Ruma}; +use crate::{services, Error, Result, Ruma}; /// # `GET /_matrix/client/versions` /// @@ -145,7 +145,7 @@ pub(crate) async fn syncv3_client_server_json() -> Result { Ok(Json(serde_json::json!({ "server": server_url, - "version": conduwuit_version(), + "version": conduit::version::conduwuit(), }))) } @@ -156,7 +156,7 @@ pub(crate) async fn syncv3_client_server_json() -> Result { pub(crate) async fn conduwuit_server_version() -> Result { Ok(Json(serde_json::json!({ "name": "conduwuit", - "version": conduwuit_version(), + "version": conduit::version::conduwuit(), }))) } diff --git a/src/api/server_server.rs b/src/api/server_server.rs index f0d2ef01..72429752 100644 --- a/src/api/server_server.rs +++ b/src/api/server_server.rs @@ -74,7 +74,7 @@ pub(crate) async fn get_server_version_route( Ok(get_server_version::v1::Response { server: Some(get_server_version::v1::Server { name: Some("Conduwuit".to_owned()), - version: Some(utils::conduwuit_version()), + version: Some(conduit::version::conduwuit()), }), }) } diff --git a/src/core/debug.rs b/src/core/debug.rs index 6db9a4dd..207f08fa 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -2,6 +2,9 @@ use std::{panic, panic::PanicInfo}; +/// Export all of the ancillary tools from here as well. +pub use crate::utils::debug::*; + /// Log event at given level in debug-mode (when debug-assertions are enabled). /// In release-mode it becomes DEBUG level, and possibly subject to elision. /// diff --git a/src/core/mod.rs b/src/core/mod.rs index 5911e027..baebbb8f 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -7,12 +7,12 @@ pub mod mods; pub mod pducount; pub mod server; pub mod utils; +pub mod version; pub use config::Config; pub use error::{Error, Result, RumaResponse}; pub use pducount::PduCount; pub use server::Server; -pub use utils::conduwuit_version; #[cfg(not(conduit_mods))] pub mod mods { diff --git a/src/core/utils/debug.rs b/src/core/utils/debug.rs new file mode 100644 index 00000000..e4151f39 --- /dev/null +++ b/src/core/utils/debug.rs @@ -0,0 +1,40 @@ +use std::fmt; + +/// Debug-formats the given slice, but only up to the first `max_len` elements. +/// Any further elements are replaced by an ellipsis. +/// +/// See also [`slice_truncated()`], +pub struct TruncatedSlice<'a, T> { + inner: &'a [T], + max_len: usize, +} + +impl fmt::Debug for TruncatedSlice<'_, T> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.inner.len() <= self.max_len { + write!(f, "{:?}", self.inner) + } else { + f.debug_list() + .entries(&self.inner[..self.max_len]) + .entry(&"...") + .finish() + } + } +} + +/// See [`TruncatedSlice`]. Useful for `#[instrument]`: +/// +/// ``` +/// use conduit_core::utils::debug::slice_truncated; +/// +/// #[tracing::instrument(fields(foos = slice_truncated(foos, 42)))] +/// fn bar(foos: &[&str]); +/// ``` +pub fn slice_truncated( + slice: &[T], max_len: usize, +) -> tracing::field::DebugValue> { + tracing::field::debug(TruncatedSlice { + inner: slice, + max_len, + }) +} diff --git a/src/core/utils/html.rs b/src/core/utils/html.rs new file mode 100644 index 00000000..3b44a31b --- /dev/null +++ b/src/core/utils/html.rs @@ -0,0 +1,37 @@ +use std::fmt; + +/// Wrapper struct which will emit the HTML-escaped version of the contained +/// string when passed to a format string. +pub struct Escape<'a>(pub &'a str); + +/// Copied from librustdoc: +/// * +impl fmt::Display for Escape<'_> { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { + // Because the internet is always right, turns out there's not that many + // characters to escape: http://stackoverflow.com/questions/7381974 + let Escape(s) = *self; + let pile_o_bits = s; + let mut last = 0; + for (i, ch) in s.char_indices() { + let s = match ch { + '>' => ">", + '<' => "<", + '&' => "&", + '\'' => "'", + '"' => """, + _ => continue, + }; + fmt.write_str(&pile_o_bits[last..i])?; + fmt.write_str(s)?; + // NOTE: we only expect single byte characters here - which is fine as long as + // we only match single byte characters + last = i + 1; + } + + if last < s.len() { + fmt.write_str(&pile_o_bits[last..])?; + } + Ok(()) + } +} diff --git a/src/core/utils/json.rs b/src/core/utils/json.rs new file mode 100644 index 00000000..a9adad54 --- /dev/null +++ b/src/core/utils/json.rs @@ -0,0 +1,39 @@ +use std::{fmt, str::FromStr}; + +use ruma::{canonical_json::try_from_json_map, CanonicalJsonError, CanonicalJsonObject}; + +use crate::Result; + +/// Fallible conversion from any value that implements `Serialize` to a +/// `CanonicalJsonObject`. +/// +/// `value` must serialize to an `serde_json::Value::Object`. +pub fn to_canonical_object(value: T) -> Result { + use serde::ser::Error; + + match serde_json::to_value(value).map_err(CanonicalJsonError::SerDe)? { + serde_json::Value::Object(map) => try_from_json_map(map), + _ => Err(CanonicalJsonError::SerDe(serde_json::Error::custom("Value must be an object"))), + } +} + +pub fn deserialize_from_str<'de, D: serde::de::Deserializer<'de>, T: FromStr, E: fmt::Display>( + deserializer: D, +) -> Result { + struct Visitor, E>(std::marker::PhantomData); + impl, Err: fmt::Display> serde::de::Visitor<'_> for Visitor { + type Value = T; + + fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(formatter, "a parsable string") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + v.parse().map_err(serde::de::Error::custom) + } + } + deserializer.deserialize_str(Visitor(std::marker::PhantomData)) +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 582263a9..e85080f1 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -1,21 +1,26 @@ +pub mod content_disposition; +pub mod debug; +pub mod defer; +pub mod html; +pub mod json; +pub mod sys; + use std::{ cmp, cmp::Ordering, - fmt, - str::FromStr, time::{SystemTime, UNIX_EPOCH}, }; +pub use debug::slice_truncated as debug_slice_truncated; +pub use html::Escape as HtmlEscape; +pub use json::{deserialize_from_str, to_canonical_object}; use rand::prelude::*; use ring::digest; -use ruma::{canonical_json::try_from_json_map, CanonicalJsonError, CanonicalJsonObject, OwnedUserId}; -use tracing::debug; +use ruma::OwnedUserId; +pub use sys::available_parallelism; use crate::{Error, Result}; -pub mod content_disposition; -pub mod defer; - pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), max) } #[must_use] @@ -108,178 +113,6 @@ pub fn common_elements( })) } -/// Fallible conversion from any value that implements `Serialize` to a -/// `CanonicalJsonObject`. -/// -/// `value` must serialize to an `serde_json::Value::Object`. -pub fn to_canonical_object(value: T) -> Result { - use serde::ser::Error; - - match serde_json::to_value(value).map_err(CanonicalJsonError::SerDe)? { - serde_json::Value::Object(map) => try_from_json_map(map), - _ => Err(CanonicalJsonError::SerDe(serde_json::Error::custom("Value must be an object"))), - } -} - -pub fn deserialize_from_str<'de, D: serde::de::Deserializer<'de>, T: FromStr, E: fmt::Display>( - deserializer: D, -) -> Result { - struct Visitor, E>(std::marker::PhantomData); - impl, Err: fmt::Display> serde::de::Visitor<'_> for Visitor { - type Value = T; - - fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(formatter, "a parsable string") - } - - fn visit_str(self, v: &str) -> Result - where - E: serde::de::Error, - { - v.parse().map_err(serde::de::Error::custom) - } - } - deserializer.deserialize_str(Visitor(std::marker::PhantomData)) -} - -// Copied from librustdoc: -// https://github.com/rust-lang/rust/blob/cbaeec14f90b59a91a6b0f17fc046c66fa811892/src/librustdoc/html/escape.rs - -/// Wrapper struct which will emit the HTML-escaped version of the contained -/// string when passed to a format string. -pub struct HtmlEscape<'a>(pub &'a str); - -impl fmt::Display for HtmlEscape<'_> { - fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { - // Because the internet is always right, turns out there's not that many - // characters to escape: http://stackoverflow.com/questions/7381974 - let HtmlEscape(s) = *self; - let pile_o_bits = s; - let mut last = 0; - for (i, ch) in s.char_indices() { - let s = match ch { - '>' => ">", - '<' => "<", - '&' => "&", - '\'' => "'", - '"' => """, - _ => continue, - }; - fmt.write_str(&pile_o_bits[last..i])?; - fmt.write_str(s)?; - // NOTE: we only expect single byte characters here - which is fine as long as - // we only match single byte characters - last = i + 1; - } - - if last < s.len() { - fmt.write_str(&pile_o_bits[last..])?; - } - Ok(()) - } -} - -/// one true function for returning the conduwuit version with the necessary -/// CONDUWUIT_VERSION_EXTRA env variables used if specified -/// -/// Set the environment variable `CONDUWUIT_VERSION_EXTRA` to any UTF-8 string -/// to include it in parenthesis after the SemVer version. A common value are -/// git commit hashes. -#[must_use] -pub fn conduwuit_version() -> String { - match option_env!("CONDUWUIT_VERSION_EXTRA") { - Some(extra) => { - if extra.is_empty() { - env!("CARGO_PKG_VERSION").to_owned() - } else { - format!("{} ({})", env!("CARGO_PKG_VERSION"), extra) - } - }, - None => match option_env!("CONDUIT_VERSION_EXTRA") { - Some(extra) => { - if extra.is_empty() { - env!("CARGO_PKG_VERSION").to_owned() - } else { - format!("{} ({})", env!("CARGO_PKG_VERSION"), extra) - } - }, - None => env!("CARGO_PKG_VERSION").to_owned(), - }, - } -} - -/// Debug-formats the given slice, but only up to the first `max_len` elements. -/// Any further elements are replaced by an ellipsis. -/// -/// See also [`debug_slice_truncated()`], -pub struct TruncatedDebugSlice<'a, T> { - inner: &'a [T], - max_len: usize, -} - -impl fmt::Debug for TruncatedDebugSlice<'_, T> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self.inner.len() <= self.max_len { - write!(f, "{:?}", self.inner) - } else { - f.debug_list() - .entries(&self.inner[..self.max_len]) - .entry(&"...") - .finish() - } - } -} - -/// See [`TruncatedDebugSlice`]. Useful for `#[instrument]`: -/// -/// ``` -/// use conduit_core::utils::debug_slice_truncated; -/// -/// #[tracing::instrument(fields(foos = debug_slice_truncated(foos, 42)))] -/// fn bar(foos: &[&str]); -/// ``` -pub fn debug_slice_truncated( - slice: &[T], max_len: usize, -) -> tracing::field::DebugValue> { - tracing::field::debug(TruncatedDebugSlice { - inner: slice, - max_len, - }) -} - -/// This is needed for opening lots of file descriptors, which tends to -/// happen more often when using RocksDB and making lots of federation -/// connections at startup. The soft limit is usually 1024, and the hard -/// limit is usually 512000; I've personally seen it hit >2000. -/// -/// * -/// * -#[cfg(unix)] -pub fn maximize_fd_limit() -> Result<(), nix::errno::Errno> { - use nix::sys::resource::{getrlimit, setrlimit, Resource::RLIMIT_NOFILE as NOFILE}; - - let (soft_limit, hard_limit) = getrlimit(NOFILE)?; - if soft_limit < hard_limit { - setrlimit(NOFILE, hard_limit, hard_limit)?; - assert_eq!((hard_limit, hard_limit), getrlimit(NOFILE)?, "getrlimit != setrlimit"); - debug!(to = hard_limit, from = soft_limit, "Raised RLIMIT_NOFILE",); - } - - Ok(()) -} - -/// Get the number of threads which could execute in parallel based on the -/// hardware and administrative constraints of this system. This value should be -/// used to hint the size of thread-pools and divide-and-conquer algorithms. -/// -/// * -#[must_use] -pub fn available_parallelism() -> usize { - std::thread::available_parallelism() - .expect("Unable to query for available parallelism.") - .get() -} - /// Boilerplate for wraps which are typed to never error. /// /// * diff --git a/src/core/utils/sys.rs b/src/core/utils/sys.rs new file mode 100644 index 00000000..825ec903 --- /dev/null +++ b/src/core/utils/sys.rs @@ -0,0 +1,36 @@ +use tracing::debug; + +use crate::Result; + +/// This is needed for opening lots of file descriptors, which tends to +/// happen more often when using RocksDB and making lots of federation +/// connections at startup. The soft limit is usually 1024, and the hard +/// limit is usually 512000; I've personally seen it hit >2000. +/// +/// * +/// * +#[cfg(unix)] +pub fn maximize_fd_limit() -> Result<(), nix::errno::Errno> { + use nix::sys::resource::{getrlimit, setrlimit, Resource::RLIMIT_NOFILE as NOFILE}; + + let (soft_limit, hard_limit) = getrlimit(NOFILE)?; + if soft_limit < hard_limit { + setrlimit(NOFILE, hard_limit, hard_limit)?; + assert_eq!((hard_limit, hard_limit), getrlimit(NOFILE)?, "getrlimit != setrlimit"); + debug!(to = hard_limit, from = soft_limit, "Raised RLIMIT_NOFILE",); + } + + Ok(()) +} + +/// Get the number of threads which could execute in parallel based on the +/// hardware and administrative constraints of this system. This value should be +/// used to hint the size of thread-pools and divide-and-conquer algorithms. +/// +/// * +#[must_use] +pub fn available_parallelism() -> usize { + std::thread::available_parallelism() + .expect("Unable to query for available parallelism.") + .get() +} diff --git a/src/core/version.rs b/src/core/version.rs new file mode 100644 index 00000000..f65bac99 --- /dev/null +++ b/src/core/version.rs @@ -0,0 +1,28 @@ +/// one true function for returning the conduwuit version with the necessary +/// CONDUWUIT_VERSION_EXTRA env variables used if specified +/// +/// Set the environment variable `CONDUWUIT_VERSION_EXTRA` to any UTF-8 string +/// to include it in parenthesis after the SemVer version. A common value are +/// git commit hashes. +#[must_use] +pub fn conduwuit() -> String { + match option_env!("CONDUWUIT_VERSION_EXTRA") { + Some(extra) => { + if extra.is_empty() { + env!("CARGO_PKG_VERSION").to_owned() + } else { + format!("{} ({})", env!("CARGO_PKG_VERSION"), extra) + } + }, + None => match option_env!("CONDUIT_VERSION_EXTRA") { + Some(extra) => { + if extra.is_empty() { + env!("CARGO_PKG_VERSION").to_owned() + } else { + format!("{} ({})", env!("CARGO_PKG_VERSION"), extra) + } + }, + None => env!("CARGO_PKG_VERSION").to_owned(), + }, + } +} diff --git a/src/main/clap.rs b/src/main/clap.rs index 81a6da72..a2fb588e 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -3,11 +3,10 @@ use std::path::PathBuf; use clap::Parser; -use conduit_core::utils::conduwuit_version; /// Commandline arguments #[derive(Parser, Debug)] -#[clap(version = conduwuit_version(), about, long_about = None)] +#[clap(version = conduit::version::conduwuit(), about, long_about = None)] pub(crate) struct Args { #[arg(short, long)] /// Optional argument to the path of a conduwuit config TOML file diff --git a/src/main/server.rs b/src/main/server.rs index c3f6a928..2395469b 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,11 +1,10 @@ use std::sync::Arc; use conduit::{ - conduwuit_version, config::Config, info, log::{LogLevelReloadHandles, ReloadHandle}, - utils::maximize_fd_limit, + utils::sys::maximize_fd_limit, Error, Result, }; use tokio::runtime; @@ -43,7 +42,7 @@ impl Server { database_path = ?config.database_path, log_levels = %config.log, "{}", - conduwuit_version(), + conduit::version::conduwuit(), ); Ok(Arc::new(Server { diff --git a/src/service/globals/client.rs b/src/service/globals/client.rs index 82747ae7..33f6d85f 100644 --- a/src/service/globals/client.rs +++ b/src/service/globals/client.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use reqwest::redirect; -use crate::{service::globals::resolver, utils::conduwuit_version, Config, Result}; +use crate::{service::globals::resolver, Config, Result}; pub struct Client { pub default: reqwest::Client, @@ -87,7 +87,7 @@ impl Client { } fn base(config: &Config) -> Result { - let version = conduwuit_version(); + let version = conduit::version::conduwuit(); let user_agent = format!("Conduwuit/{version}"); From 1108235c632476a746a19ef39822e926f7d4b94f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 25 May 2024 20:16:28 +0000 Subject: [PATCH 0027/2091] misc simplifications and cleanup Signed-off-by: Jason Volk --- src/main/main.rs | 17 ++++++++--------- src/main/server.rs | 6 ++---- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/main/main.rs b/src/main/main.rs index a96945a7..ba1cb2d3 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -25,21 +25,20 @@ fn main() -> Result<(), Error> { .build() .expect("built runtime"); - let handle = runtime.handle(); - let server: Arc = Server::build(args, Some(handle))?; - runtime.block_on(async { async_main(server.clone()).await })?; + let server: Arc = Server::build(args, Some(runtime.handle()))?; + runtime.block_on(async_main(&server))?; // explicit drop here to trace thread and tls dtors drop(runtime); - debug_info!("Exit"); + Ok(()) } /// Operate the server normally in release-mode static builds. This will start, /// run and stop the server within the asynchronous runtime. #[cfg(not(conduit_mods))] -async fn async_main(server: Arc) -> Result<(), Error> { +async fn async_main(server: &Arc) -> Result<(), Error> { extern crate conduit_router as router; use tracing::error; @@ -66,22 +65,22 @@ async fn async_main(server: Arc) -> Result<(), Error> { /// and hot-reload portions of the server as-needed before returning for an /// actual shutdown. This is not available in release-mode or static builds. #[cfg(conduit_mods)] -async fn async_main(server: Arc) -> Result<(), Error> { +async fn async_main(server: &Arc) -> Result<(), Error> { let mut starts = true; let mut reloads = true; while reloads { - if let Err(error) = mods::open(&server).await { + if let Err(error) = mods::open(server).await { error!("Loading router: {error}"); return Err(error); } - let result = mods::run(&server, starts).await; + let result = mods::run(server, starts).await; if let Ok(result) = result { (starts, reloads) = result; } let force = !reloads || result.is_err(); - if let Err(error) = mods::close(&server, force).await { + if let Err(error) = mods::close(server, force).await { error!("Unloading router: {error}"); return Err(error); } diff --git a/src/main/server.rs b/src/main/server.rs index 2395469b..d960bfac 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -139,9 +139,7 @@ fn init_tracing(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard) { let (flame_layer, flame_guard) = match tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) { Ok(ok) => ok, - Err(e) => { - panic!("failed to initialize tracing-flame: {e}"); - }, + Err(e) => panic!("failed to initialize tracing-flame: {e}"), }; let flame_layer = flame_layer .with_empty_samples(false) @@ -175,7 +173,7 @@ fn init_tracing(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard) { #[cfg_attr(not(feature = "perf_measurements"), allow(clippy::let_unit_value))] let flame_guard = (); - tracing::subscriber::set_global_default(subscriber).unwrap(); + tracing::subscriber::set_global_default(subscriber).expect("failed to set global tracing subscriber"); #[cfg(all(feature = "tokio_console", feature = "release_max_log_level", tokio_unstable))] tracing::error!( From 89ab687f16e519499cde88efd5e4da375d403653 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 1 Jun 2024 08:03:20 +0000 Subject: [PATCH 0028/2091] move signal handling out to main Signed-off-by: Jason Volk --- src/core/server.rs | 6 ++++- src/main/main.rs | 37 +++++++++++++++++++++++++-- src/router/run.rs | 63 +++++++++++----------------------------------- 3 files changed, 55 insertions(+), 51 deletions(-) diff --git a/src/core/server.rs b/src/core/server.rs index 4bfff340..9e4497ae 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -6,7 +6,7 @@ use std::{ time::SystemTime, }; -use tokio::runtime; +use tokio::{runtime, sync::broadcast}; use crate::{config::Config, log::LogLevelReloadHandles}; @@ -22,6 +22,9 @@ pub struct Server { /// command to initiate shutdown. pub shutdown: Mutex>, + /// Reload/shutdown signal + pub signal: broadcast::Sender<&'static str>, + /// Reload/shutdown desired indicator; when false, shutdown is desired. This /// is an observable used on shutdown and modifying is not recommended. pub reload: AtomicBool, @@ -51,6 +54,7 @@ impl Server { config, started: SystemTime::now(), shutdown: Mutex::new(None), + signal: broadcast::channel::<&'static str>(1).0, reload: AtomicBool::new(false), interrupt: AtomicBool::new(false), runtime, diff --git a/src/main/main.rs b/src/main/main.rs index ba1cb2d3..73e6e690 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -6,9 +6,10 @@ extern crate conduit_core as conduit; use std::{cmp, sync::Arc, time::Duration}; -use conduit::{debug_info, error, utils::available_parallelism, Error, Result}; +use conduit::{debug_error, debug_info, error, utils::available_parallelism, warn, Error, Result}; use server::Server; -use tokio::runtime; +use tokio::{runtime, signal}; +use tracing::debug; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; @@ -26,6 +27,7 @@ fn main() -> Result<(), Error> { .expect("built runtime"); let server: Arc = Server::build(args, Some(runtime.handle()))?; + runtime.spawn(signal(server.clone())); runtime.block_on(async_main(&server))?; // explicit drop here to trace thread and tls dtors @@ -94,3 +96,34 @@ async fn async_main(server: &Arc) -> Result<(), Error> { debug_info!("Exit runtime"); Ok(()) } + +#[tracing::instrument(skip_all)] +async fn signal(server: Arc) { + let (mut term, mut quit); + #[cfg(unix)] + { + use signal::unix; + quit = unix::signal(unix::SignalKind::quit()).expect("SIGQUIT handler"); + term = unix::signal(unix::SignalKind::terminate()).expect("SIGTERM handler"); + }; + + loop { + debug!("Installed signal handlers"); + let sig: &'static str; + #[cfg(unix)] + tokio::select! { + _ = term.recv() => { sig = "SIGTERM"; }, + _ = quit.recv() => { sig = "Ctrl+\\"; }, + _ = signal::ctrl_c() => { sig = "Ctrl+C"; }, + } + #[cfg(not(unix))] + tokio::select! { + _ = signal::ctrl_c() => { sig = "Ctrl+C"; }, + } + + warn!("Received signal {}", sig); + if let Err(e) = server.server.signal.send(sig) { + debug_error!("signal channel: {e}"); + } + } +} diff --git a/src/router/run.rs b/src/router/run.rs index e6238853..8afb2dc3 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -1,11 +1,8 @@ use std::{sync::Arc, time::Duration}; use axum_server::Handle as ServerHandle; -use tokio::{ - signal, - sync::broadcast::{self, Sender}, -}; -use tracing::{debug, info, warn}; +use tokio::sync::broadcast::{self, Sender}; +use tracing::{debug, error, info}; extern crate conduit_admin as admin; extern crate conduit_core as conduit; @@ -39,9 +36,7 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { server.interrupt.store(false, Ordering::Release); let (tx, _) = broadcast::channel::<()>(1); - let sigs = server - .runtime() - .spawn(sighandle(server.clone(), tx.clone())); + let sigs = server.runtime().spawn(signal(server.clone(), tx.clone())); // Serve clients let res = serve::serve(&server, app, handle, tx.subscribe()).await; @@ -115,51 +110,25 @@ pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { } #[tracing::instrument(skip_all)] -async fn sighandle(server: Arc, tx: Sender<()>) -> Result<(), Error> { - let ctrl_c = async { - signal::ctrl_c() - .await - .expect("failed to install Ctrl+C handler"); +async fn signal(server: Arc, tx: Sender<()>) { + let sig: &'static str = server + .signal + .subscribe() + .recv() + .await + .expect("channel error"); + debug!("Received signal {}", sig); + if sig == "Ctrl+C" { let reload = cfg!(unix) && cfg!(debug_assertions); server.reload.store(reload, Ordering::Release); - }; - - #[cfg(unix)] - let ctrl_bs = async { - signal::unix::signal(signal::unix::SignalKind::quit()) - .expect("failed to install Ctrl+\\ handler") - .recv() - .await; - }; - - #[cfg(unix)] - let terminate = async { - signal::unix::signal(signal::unix::SignalKind::terminate()) - .expect("failed to install SIGTERM handler") - .recv() - .await; - }; - - debug!("Installed signal handlers"); - let sig: &str; - #[cfg(unix)] - tokio::select! { - () = ctrl_c => { sig = "Ctrl+C"; }, - () = ctrl_bs => { sig = "Ctrl+\\"; }, - () = terminate => { sig = "SIGTERM"; }, } - #[cfg(not(unix))] - tokio::select! { - _ = ctrl_c => { sig = "Ctrl+C"; }, - } - - warn!("Received {}", sig); server.interrupt.store(true, Ordering::Release); services().globals.rotate.fire(); - tx.send(()) - .expect("failed sending shutdown transaction to oneshot channel"); + if let Err(e) = tx.send(()) { + error!("failed sending shutdown transaction to channel: {e}"); + } if let Some(handle) = server.shutdown.lock().expect("locked").as_ref() { let pending = server.requests_spawn_active.load(Ordering::Relaxed); @@ -172,6 +141,4 @@ async fn sighandle(server: Arc, tx: Sender<()>) -> Result<(), Error> { handle.shutdown(); } } - - Ok(()) } From 102bd1b4a6a0ab3c955fda06c86aed214d583a51 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 03:40:31 +0000 Subject: [PATCH 0029/2091] use debug_warn for parse_incoming_pdu err results. Signed-off-by: Jason Volk --- src/api/server_server.rs | 2 +- src/service/rooms/event_handler/parse_incoming_pdu.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/server_server.rs b/src/api/server_server.rs index 72429752..d5edd03b 100644 --- a/src/api/server_server.rs +++ b/src/api/server_server.rs @@ -244,7 +244,7 @@ pub(crate) async fn send_transaction_message_route( parsed_pdus.push(match parse_incoming_pdu(pdu) { Ok(t) => t, Err(e) => { - warn!("Could not parse PDU: {e}"); + debug_warn!("Could not parse PDU: {e}"); continue; }, }); diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 133ab66e..d9d9f063 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -6,7 +6,7 @@ use crate::{service::pdu::gen_event_id_canonical_json, services, Error, Result}; pub fn parse_incoming_pdu(pdu: &RawJsonValue) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { - warn!("Error parsing incoming event {:?}: {:?}", pdu, e); + warn!("Error parsing incoming event {pdu:?}: {e:?}"); Error::BadServerResponse("Invalid PDU in server response") })?; From 7d2f510cc3d390c8596d0f1614bbe0ba4dc5a875 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 May 2024 05:39:51 +0000 Subject: [PATCH 0030/2091] single-source for defaulty log filter string Signed-off-by: Jason Volk --- src/core/config/mod.rs | 3 ++- src/main/server.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 665cbe20..bb13923b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -975,7 +975,8 @@ fn default_tracing_flame_output_path() -> String { "./tracing.folded".to_owned() fn default_trusted_servers() -> Vec { vec![OwnedServerName::try_from("matrix.org").unwrap()] } -fn default_log() -> String { +#[must_use] +pub fn default_log() -> String { // do debug logging by default for debug builds if cfg!(debug_assertions) { "debug".to_owned() diff --git a/src/main/server.rs b/src/main/server.rs index d960bfac..8e61d502 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use conduit::{ + config, config::Config, info, log::{LogLevelReloadHandles, ReloadHandle}, @@ -103,7 +104,7 @@ fn init_tracing(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard) { Ok(s) => s, Err(e) => { eprintln!("It looks like your config is invalid. The following error occured while parsing it: {e}"); - EnvFilter::try_new("warn").unwrap() + EnvFilter::try_new(config::default_log()).expect("failed to set default EnvFilter") }, }; From c2586737ae552ef3fa1c8d9911c3a7b822ef387a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 1 Jun 2024 01:36:37 +0000 Subject: [PATCH 0031/2091] accept receipts prior to events Signed-off-by: Jason Volk --- src/api/mod.rs | 2 +- src/api/server_server.rs | 22 +++------------------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index b835f536..84207d71 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,7 +7,7 @@ extern crate conduit_core as conduit; extern crate conduit_service as service; pub use client_server::membership::{join_room_by_id_helper, leave_all_rooms}; -pub(crate) use conduit::{debug_error, debug_info, debug_warn, utils, Error, Result}; +pub(crate) use conduit::{debug_info, debug_warn, utils, Error, Result}; pub(crate) use ruma_wrapper::{Ruma, RumaResponse}; pub(crate) use service::{pdu::PduEvent, services, user_is_local}; diff --git a/src/api/server_server.rs b/src/api/server_server.rs index d5edd03b..1fe1ed1b 100644 --- a/src/api/server_server.rs +++ b/src/api/server_server.rs @@ -54,7 +54,6 @@ use tracing::{debug, error, trace, warn}; use crate::{ client_server::{self, claim_keys_helper, get_keys_helper}, - debug_error, service::{ pdu::{gen_event_id_canonical_json, PduBuilder}, rooms::event_handler::parse_incoming_pdu, @@ -369,22 +368,9 @@ pub(crate) async fn send_transaction_message_route( } if services().rooms.state_cache.is_joined(&user_id, &room_id)? { - if let Some((event_id, _)) = user_updates - .event_ids - .iter() - .filter_map(|id| { - services() - .rooms - .timeline - .get_pdu_count(id) - .ok() - .flatten() - .map(|r| (id, r)) - }) - .max_by_key(|(_, count)| *count) - { + for event_id in &user_updates.event_ids { let mut user_receipts = BTreeMap::new(); - user_receipts.insert(user_id.clone(), user_updates.data); + user_receipts.insert(user_id.clone(), user_updates.data.clone()); let mut receipts = BTreeMap::new(); receipts.insert(ReceiptType::Read, user_receipts); @@ -396,13 +382,11 @@ pub(crate) async fn send_transaction_message_route( content: ReceiptEventContent(receipt_content), room_id: room_id.clone(), }; + services() .rooms .read_receipt .readreceipt_update(&user_id, &room_id, event)?; - } else { - // TODO fetch missing events - debug_error!("No known event ids in read receipt: {:?}", user_updates); } } else { debug_warn!(%user_id, %room_id, "received read receipt EDU for user not in room"); From 887496d040be61fe9a78169b4fa89febbec08812 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 1 Jun 2024 12:10:25 +0000 Subject: [PATCH 0032/2091] consolidate default cargo features Signed-off-by: Jason Volk --- Cargo.lock | 7 +-- src/admin/Cargo.toml | 8 --- src/api/Cargo.toml | 7 --- src/core/Cargo.toml | 11 ---- src/database/Cargo.toml | 8 --- src/main/Cargo.toml | 117 ++++++++++++++++++++++++++++++++++------ src/router/Cargo.toml | 9 ---- src/service/Cargo.toml | 17 ------ 8 files changed, 104 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 375440ef..b2975e33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -572,8 +572,12 @@ name = "conduit" version = "0.4.1" dependencies = [ "clap", + "conduit_admin", + "conduit_api", "conduit_core", + "conduit_database", "conduit_router", + "conduit_service", "console-subscriber", "hardened_malloc-rs", "log", @@ -761,18 +765,15 @@ dependencies = [ "log", "loole", "lru-cache", - "parking_lot", "rand", "regex", "reqwest", "ruma", "ruma-identifiers-validation", - "rusqlite", "serde", "serde_json", "serde_yaml", "sha2", - "thread_local", "tokio", "tracing", "url", diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 244cf7d2..42327430 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -17,14 +17,6 @@ crate-type = [ ] [features] -default = [ - "rocksdb", - "io_uring", - "jemalloc", - "zstd_compression", - "release_max_log_level", -] - dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 315fa76c..a80a254e 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -17,13 +17,6 @@ crate-type = [ ] [features] -default = [ - "element_hacks", - "gzip_compression", - "brotli_compression", - "release_max_log_level", -] - element_hacks = [] dev_release_log_level = [] release_max_log_level = [ diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index a0c8a060..dd17946f 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -17,17 +17,6 @@ crate-type = [ ] [features] -default = [ - "rocksdb", - "io_uring", - "jemalloc", - "gzip_compression", - "zstd_compression", - "brotli_compression", - "sentry_telemetry", - "release_max_log_level", -] - dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index e23d2d7b..113414cb 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -17,14 +17,6 @@ crate-type = [ ] [features] -default = [ - "rocksdb", - "io_uring", - "jemalloc", - "zstd_compression", - "release_max_log_level", -] - dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 06c8c697..94c62656 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -37,27 +37,57 @@ assets = [ [features] default = [ - "sentry_telemetry", + "brotli_compression", + "element_hacks", + "gzip_compression", + "io_uring", + "jemalloc", "release_max_log_level", + "rocksdb", + "sentry_telemetry", + "systemd", + "zstd_compression", ] -# increases performance, reduces build times, and reduces binary size by not compiling or -# genreating code for log level filters that users will generally not use (debug and trace) -release_max_log_level = [ - "tracing/max_level_trace", - "tracing/release_max_level_info", - "log/max_level_trace", - "log/release_max_level_info", +axum_dual_protocol = [ + "conduit-router/axum_dual_protocol", ] -sentry_telemetry = [ - "dep:sentry", - "dep:sentry-tracing", - "dep:sentry-tower", +brotli_compression = [ + "conduit-api/brotli_compression", + "conduit-core/brotli_compression", + "conduit-router/brotli_compression", + "conduit-service/brotli_compression", ] -# enable the tokio_console server ncompatible with release_max_log_level -tokio_console = [ - "dep:console-subscriber", - "tokio/tracing", +dev_release_log_level = [ + "conduit-api/dev_release_log_level", + "conduit-core/dev_release_log_level", + "conduit-database/dev_release_log_level", + "conduit-router/dev_release_log_level", + "conduit-service/dev_release_log_level", +] +element_hacks = [ + "conduit-api/element_hacks", +] +gzip_compression = [ + "conduit-api/gzip_compression", + "conduit-router/gzip_compression", + "conduit-service/gzip_compression", +] +hardened_malloc = [ + "conduit-core/hardened_malloc", +] +io_uring = [ + "conduit-admin/io_uring", + "conduit-core/io_uring", + "conduit-database/io_uring", +] +jemalloc = [ + "conduit-admin/jemalloc", + "conduit-core/jemalloc", + "conduit-database/jemalloc", +] +jemalloc_prof = [ + "conduit-core/jemalloc_prof", ] perf_measurements = [ "dep:opentelemetry", @@ -65,11 +95,64 @@ perf_measurements = [ "dep:tracing-opentelemetry", "dep:opentelemetry_sdk", "dep:opentelemetry-jaeger", + "conduit-core/perf_measurements", + "conduit-core/sentry_telemetry", +] +# increases performance, reduces build times, and reduces binary size by not compiling or +# genreating code for log level filters that users will generally not use (debug and trace) +release_max_log_level = [ + "tracing/max_level_trace", + "tracing/release_max_level_info", + "log/max_level_trace", + "log/release_max_level_info", + "conduit-admin/release_max_log_level", + "conduit-api/release_max_log_level", + "conduit-core/release_max_log_level", + "conduit-database/release_max_log_level", + "conduit-router/release_max_log_level", + "conduit-service/release_max_log_level", +] +rocksdb = [ + "conduit-admin/rocksdb", + "conduit-core/rocksdb", + "conduit-database/rocksdb", +] +sentry_telemetry = [ + "dep:sentry", + "dep:sentry-tracing", + "dep:sentry-tower", + "conduit-core/sentry_telemetry", + "conduit-router/sentry_telemetry", +] +sha256_media = [ + "conduit-service/sha256_media", +] +sqlite = [ + "conduit-core/sqlite", + "conduit-database/sqlite", +] +systemd = [ + "conduit-router/systemd", +] +# enable the tokio_console server ncompatible with release_max_log_level +tokio_console = [ + "dep:console-subscriber", + "tokio/tracing", +] +zstd_compression = [ + "conduit-admin/zstd_compression", + "conduit-core/zstd_compression", + "conduit-database/zstd_compression", + "conduit-router/zstd_compression", ] [dependencies] -conduit-router.workspace = true +conduit-admin.workspace = true +conduit-api.workspace = true conduit-core.workspace = true +conduit-database.workspace = true +conduit-router.workspace = true +conduit-service.workspace = true tokio.workspace = true log.workspace = true diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 11028112..6084aa07 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -17,15 +17,6 @@ crate-type = [ ] [features] -default = [ - "systemd", - "sentry_telemetry", - "gzip_compression", - "zstd_compression", - "brotli_compression", - "release_max_log_level", -] - dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index e4189125..f6f682a9 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -17,12 +17,6 @@ crate-type = [ ] [features] -default = [ - "gzip_compression", - "brotli_compression", - "release_max_log_level", -] - dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", @@ -30,11 +24,6 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] -sqlite = [ - "dep:rusqlite", - "dep:parking_lot", - "dep:thread_local", -] gzip_compression = [ "reqwest/gzip", ] @@ -63,22 +52,16 @@ jsonwebtoken.workspace = true log.workspace = true loole.workspace = true lru-cache.workspace = true -parking_lot.optional = true -parking_lot.workspace = true rand.workspace = true regex.workspace = true reqwest.workspace = true ruma-identifiers-validation.workspace = true ruma.workspace = true -rusqlite.optional = true -rusqlite.workspace = true serde_json.workspace = true serde.workspace = true serde_yaml.workspace = true sha2.optional = true sha2.workspace = true -thread_local.optional = true -thread_local.workspace = true tokio.workspace = true tracing.workspace = true url.workspace = true From 95ca9d00a2c8d61f429b477247b73655af0831a9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 2 Jun 2024 21:08:52 +0000 Subject: [PATCH 0033/2091] nix: get all features from main crate --- nix/pkgs/main/default.nix | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 86595691..f91a9cdd 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -23,8 +23,6 @@ let # We perform default-feature unification in nix, because some of the dependencies # on the nix side depend on feature values. -workspaceMembers = builtins.map (member: "${inputs.self}/src/${member}") - (builtins.attrNames (builtins.readDir "${inputs.self}/src")); crateFeatures = path: let manifest = lib.importTOML "${path}/Cargo.toml"; in lib.remove "default" (lib.attrNames manifest.features) ++ @@ -34,10 +32,8 @@ crateFeatures = path: manifest.dependencies); crateDefaultFeatures = path: (lib.importTOML "${path}/Cargo.toml").features.default; -allDefaultFeatures = lib.unique - (lib.flatten (builtins.map crateDefaultFeatures workspaceMembers)); -allFeatures = lib.unique - (lib.flatten (builtins.map crateFeatures workspaceMembers)); +allDefaultFeatures = crateDefaultFeatures "${inputs.self}/src/main"; +allFeatures = crateFeatures "${inputs.self}/src/main"; features' = lib.unique (features ++ lib.optionals default_features allDefaultFeatures ++ From de79b66cea4937d68f57f0e0490e5c04a9fc6151 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 2 Jun 2024 18:00:38 -0400 Subject: [PATCH 0034/2091] misc docs changes Signed-off-by: strawberry --- docs/appservices.md | 22 ++++++---------------- docs/deploying/generic.md | 4 +++- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/docs/appservices.md b/docs/appservices.md index 84385ab9..8d646921 100644 --- a/docs/appservices.md +++ b/docs/appservices.md @@ -12,9 +12,9 @@ and later starting it. At some point the appservice guide should ask you to add a registration yaml file to the homeserver. In Synapse you would do this by adding the path to the -homeserver.yaml, but in Conduit you can do this from within Matrix: +homeserver.yaml, but in conduwuit you can do this from within Matrix: -First, go into the #admins room of your homeserver. The first person that +First, go into the `#admins` room of your homeserver. The first person that registered on the homeserver automatically joins it. Then send a message into the room like this: @@ -31,13 +31,13 @@ the room like this: ``` You can confirm it worked by sending a message like this: -`@conduit:your.server.name: appservices list` +`!admin appservices list` The `@conduit` bot should answer with `Appservices (1): your-bridge` -Then you are done. Conduit will send messages to the appservices and the +Then you are done. conduwuit will send messages to the appservices and the appservice can send requests to the homeserver. You don't need to restart -Conduit, but if it doesn't work, restarting while the appservice is running +conduwuit, but if it doesn't work, restarting while the appservice is running could help. ## Appservice-specific instructions @@ -46,16 +46,6 @@ could help. To remove an appservice go to your admin room and execute -`@conduit:your.server.name: appservices unregister ` +`!admin appservices unregister ` where `` one of the output of `appservices list`. - -### Tested appservices - -These appservices have been tested and work with Conduit without any extra steps: - -- [matrix-appservice-discord](https://github.com/Half-Shot/matrix-appservice-discord) -- [mautrix-hangouts](https://github.com/mautrix/hangouts/) -- [mautrix-telegram](https://github.com/mautrix/telegram/) -- [mautrix-signal](https://github.com/mautrix/signal/) from version `0.2.2` forward. -- [heisenbridge](https://github.com/hifi/heisenbridge/) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index e23c75d4..dc47f51b 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -49,7 +49,7 @@ The systemd unit for conduwuit can be found [here](../configuration.md#example-s Now we need to create the conduwuit's config file in `/etc/conduwuit/conduwuit.toml`. The example config can be found at [conduwuit-example.toml](../configuration.md).**Please take a moment to read it. You need to change at least the server name.** -RocksDB (`rocksdb`) is the only supported database backend. SQLite only exists for historical reasons and is not recommended. Any performance issues, storage issues, database issues, etc will not be assisted if using SQLite and you will be asked to migrate to RocksDB first. +RocksDB is the only supported database backend. SQLite only exists for historical reasons, is not recommended, and will be removed soon (likely in v0.5.0). Any performance issues, storage issues, database issues, etc will not be assisted if using SQLite and you will be asked to migrate to RocksDB first. ## Setting the correct file permissions @@ -74,6 +74,8 @@ sudo chmod 700 /var/lib/conduwuit/ Refer to the documentation or various guides online of your chosen reverse proxy software. A [Caddy](https://caddyserver.com/) example will be provided as this is the recommended reverse proxy for new users and is very trivial to use (handles TLS, reverse proxy headers, etc transparently with proper defaults). +Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization header, making federation non-functional. If using Apache, you need to use `nocanon` to prevent this. + ### Caddy Create `/etc/caddy/conf.d/conduwuit_caddyfile` and enter this (substitute for your server name). From f09e0dc1379055c0a2fe6e112a022e723fb447c7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 2 Jun 2024 19:17:23 -0400 Subject: [PATCH 0035/2091] add conduwuit community code of conduct this is a code of conduct that the moderation team made up, specific to conduwuit's community spaces such as the matrix rooms. the matrix foundation and contributor's covenant still apply, but having conduwuit-specific additional guidelines help Signed-off-by: strawberry --- docs/SUMMARY.md | 1 + docs/conduwuit_coc.md | 46 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 docs/conduwuit_coc.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index bafe88dc..6f3bd570 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -17,3 +17,4 @@ - [Contributing](contributing.md) - [Testing](development/testing.md) - [Hot Reloading ("Live" Development)](development/hot_reload.md) +- [conduwuit Community Code of Conduct](conduwuit_coc.md) diff --git a/docs/conduwuit_coc.md b/docs/conduwuit_coc.md new file mode 100644 index 00000000..858884c5 --- /dev/null +++ b/docs/conduwuit_coc.md @@ -0,0 +1,46 @@ +This page serves as a referenceable link to conduwuit's Code of Conduct for the Matrix rooms and any further community spaces related to conduwuit. + +# Conduwuit Community Code of Conduct + +Welcome to the Conduwuit community! We’re excited to have you here. This space is dedicated to fostering a positive, supportive, and inclusive environment for everyone. Here are our guidelines to help maintain the welcoming atmosphere that sets Conduwuit apart. + +For the foundational rules, please refer to the [Matrix.org Code of Conduct](https://matrix.org/legal/code-of-conduct/) and the [Contributor's Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md). Below are additional guidelines specific to the Conduwuit community. + +## Our Values and Guidelines + +1. **Respect and Inclusivity**: We are committed to maintaining a community where everyone feels safe and respected. Discrimination, harassment, or hate speech of any kind will not be tolerated. Recognise that each community member experiences the world differently based on their past experiences, background, and identity. Share your own experiences and be open to learning about others' diverse perspectives. + +2. **Positivity and Constructiveness**: Engage in constructive discussions and support each other. If you are feeling angry, negative, or aggressive, take a break until you can participate in a positive and constructive manner. Process intense feelings with a friend or in a private setting before engaging in community conversations to help maintain a supportive and focused environment. + +3. **Clarity and Understanding**: We understand that our community includes neurodivergent individuals and those who may not appreciate sarcasm or subtlety. Communicate clearly and kindly, avoiding sarcasm and ensuring your messages are easily understood by all. Additionally, avoid putting the burden of education on marginalized groups by doing your own research before asking for explanations. + +4. **Be Open to Inclusivity**: Actively engage in conversations about making our community more inclusive. Report discriminatory behaviour to the moderators and be open to constructive critiques that aim to improve our community. Understand that discussing discrimination and negative experiences can be emotionally taxing, so focus on the message rather than critiquing the tone used. + +5. **Commit to Inclusivity**: Building an inclusive community requires time, energy, and resources. Recognise that addressing discrimination and bias is an ongoing process that necessitates commitment and action from all community members. + +## Community Spaces + +This Code of Conduct applies to the entire [Conduwuit Space](https://matrix.to/#/#conduwuit-space:puppygock.gay) and its rooms: + +### [Conduwuit](https://matrix.to/#/#conduwuit:puppygock.gay) + +This room is for support and discussions about Conduwuit. Ask questions, share insights, and help each other out. + +### [Conduwuit Offtopic](https://matrix.to/#/#conduwuit-offtopic:girlboss.ceo) + +For off-topic community conversations about any subject. While this room allows for a wide range of topics, the same CoC applies. Keep discussions respectful and inclusive, and avoid divisive subjects like country/world politics. General topics, such as world events, are welcome as long as they follow the CoC. + +### [Conduwuit Development](https://matrix.to/#/#conduwuit-dev:puppygock.gay) + +This room is dedicated to discussing active development of Conduwuit. Posting requires an elevated power level, which can be requested in one of the other rooms. Use this space to collaborate and innovate. + +## Enforcement + +We have a zero-tolerance policy for violations of this Code of Conduct. If someone’s behavior makes you uncomfortable, please report it to the moderators. Actions we may take include: + +1. **Warning**: A warning given directly in the room or via a private message from the moderators, identifying the violation and requesting corrective action. +2. **Kick or Ban**: Egregious behavior may result in an immediate kick or ban to protect other community members. Bans are considered permanent and will only be reversed in exceptional circumstances after proven good behaviour. + +Together, let’s build a community where everyone feels valued and respected. + +- The Conduwuit Moderation Team From 1a4736d40bcd5aa26d86e3d66cd64ec1bdcd055e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Jun 2024 22:36:00 +0000 Subject: [PATCH 0036/2091] support configuring multiple bind hosts; default to dual localhost. Signed-off-by: Jason Volk --- src/core/config/check.rs | 69 +++++++++++++++++++--------------------- src/core/config/mod.rs | 52 +++++++++++++++++++----------- 2 files changed, 67 insertions(+), 54 deletions(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index a631c4f8..403aa27c 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -27,46 +27,43 @@ pub fn check(config: &Config) -> Result<(), Error> { )); } - if config.address.is_loopback() && cfg!(unix) { - debug!( - "Found loopback listening address {}, running checks if we're in a container.", - config.address - ); + config.get_bind_addrs().iter().for_each(|addr| { + if addr.ip().is_loopback() && cfg!(unix) { + debug!("Found loopback listening address {addr}, running checks if we're in a container.",); - #[cfg(unix)] - if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() - /* Host */ - { - error!( - "You are detected using OpenVZ with a loopback/localhost listening address of {}. If you are using \ - OpenVZ for containers and you use NAT-based networking to communicate with the host and guest, this \ - will NOT work. Please change this to \"0.0.0.0\". If this is expected, you can ignore.", - config.address - ); - } + #[cfg(unix)] + if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() + /* Host */ + { + error!( + "You are detected using OpenVZ with a loopback/localhost listening address of {addr}. If you are \ + using OpenVZ for containers and you use NAT-based networking to communicate with the host and \ + guest, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, you can \ + ignore.", + ); + } - #[cfg(unix)] - if Path::new("/.dockerenv").exists() { - error!( - "You are detected using Docker with a loopback/localhost listening address of {}. If you are using a \ - reverse proxy on the host and require communication to conduwuit in the Docker container via \ - NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, \ - you can ignore.", - config.address - ); - } + #[cfg(unix)] + if Path::new("/.dockerenv").exists() { + error!( + "You are detected using Docker with a loopback/localhost listening address of {addr}. If you are \ + using a reverse proxy on the host and require communication to conduwuit in the Docker container \ + via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". If this is \ + expected, you can ignore.", + ); + } - #[cfg(unix)] - if Path::new("/run/.containerenv").exists() { - error!( - "You are detected using Podman with a loopback/localhost listening address of {}. If you are using a \ - reverse proxy on the host and require communication to conduwuit in the Podman container via \ - NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, \ - you can ignore.", - config.address - ); + #[cfg(unix)] + if Path::new("/run/.containerenv").exists() { + error!( + "You are detected using Podman with a loopback/localhost listening address of {addr}. If you are \ + using a reverse proxy on the host and require communication to conduwuit in the Podman container \ + via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". If this is \ + expected, you can ignore.", + ); + } } - } + }); // rocksdb does not allow max_log_files to be 0 if config.rocksdb_max_log_files == 0 && cfg!(feature = "rocksdb") { diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index bb13923b..67b83975 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1,7 +1,7 @@ use std::{ collections::BTreeMap, fmt::{self, Write as _}, - net::{IpAddr, Ipv6Addr, SocketAddr}, + net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, path::PathBuf, }; @@ -36,13 +36,20 @@ struct ListeningPort { ports: Either>, } +#[derive(Deserialize, Clone, Debug)] +#[serde(transparent)] +struct ListeningAddr { + #[serde(with = "either::serde_untagged")] + addrs: Either>, +} + /// all the config options for conduwuit #[derive(Clone, Debug, Deserialize)] #[allow(clippy::struct_excessive_bools)] pub struct Config { /// [`IpAddr`] conduwuit will listen on (can be IPv4 or IPv6) #[serde(default = "default_address")] - pub address: IpAddr, + address: ListeningAddr, /// default TCP port(s) conduwuit will listen on #[serde(default = "default_port")] port: ListeningPort, @@ -471,22 +478,27 @@ impl Config { #[must_use] pub fn get_bind_addrs(&self) -> Vec { - match &self.port.ports { - Left(port) => { - // Left is only 1 value, so make a vec with 1 value only - let port_vec = [port]; + let mut addrs = Vec::new(); + for host in &self.get_bind_hosts() { + for port in &self.get_bind_ports() { + addrs.push(SocketAddr::new(*host, *port)); + } + } - port_vec - .iter() - .copied() - .map(|port| SocketAddr::from((self.address, *port))) - .collect::>() - }, - Right(ports) => ports - .iter() - .copied() - .map(|port| SocketAddr::from((self.address, port))) - .collect::>(), + addrs + } + + fn get_bind_hosts(&self) -> Vec { + match &self.address.addrs { + Left(addr) => vec![*addr], + Right(addrs) => addrs.clone(), + } + } + + fn get_bind_ports(&self) -> Vec { + match &self.port.ports { + Left(port) => vec![*port], + Right(ports) => ports.clone(), } } @@ -875,7 +887,11 @@ impl fmt::Display for Config { fn true_fn() -> bool { true } -fn default_address() -> IpAddr { Ipv6Addr::LOCALHOST.into() } +fn default_address() -> ListeningAddr { + ListeningAddr { + addrs: Right(vec![Ipv4Addr::LOCALHOST.into(), Ipv6Addr::LOCALHOST.into()]), + } +} fn default_port() -> ListeningPort { ListeningPort { From aef25ea1f752b60d1e8db9ed54b94089aed02101 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 28 May 2024 02:28:45 -0400 Subject: [PATCH 0037/2091] enable tracing feature for axum Signed-off-by: strawberry --- Cargo.lock | 2 ++ Cargo.toml | 1 + 2 files changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index b2975e33..a6384b35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -224,6 +224,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -261,6 +262,7 @@ dependencies = [ "sync_wrapper 0.1.2", "tower-layer", "tower-service", + "tracing", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cabd47a2..cc6d5960 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ features = [ "json", "matched-path", "tokio", + "tracing", ] [workspace.dependencies.axum-extra] From 4aead5de7a8890a0bebd4794c75a7e3e385012b3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 28 May 2024 12:51:22 -0400 Subject: [PATCH 0038/2091] reflax a couple restrictions on custom room IDs and aliases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🙃 Signed-off-by: strawberry --- src/api/client_server/room.rs | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/src/api/client_server/room.rs b/src/api/client_server/room.rs index 4593b2e4..cb8930ec 100644 --- a/src/api/client_server/room.rs +++ b/src/api/client_server/room.rs @@ -900,20 +900,6 @@ async fn room_alias_check( ErrorKind::InvalidParam, "Room alias contained spaces which is not a valid room alias.", )); - } else if room_alias_name.len() > 255 { - // there is nothing spec-wise saying to check the limit of this, - // however absurdly long room aliases are guaranteed to be unreadable or done - // maliciously. there is no reason a room alias should even exceed 100 - // characters as is. generally in spec, 255 is matrix's fav number - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Room alias is excessively long, clients may not be able to handle this. Please shorten it.", - )); - } else if room_alias_name.contains('"') { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Room alias contained `\"` which is not allowed.", - )); } // check if room alias is forbidden @@ -979,8 +965,6 @@ fn custom_room_id_check(custom_room_id: &String) -> Result { ErrorKind::InvalidParam, "Custom room ID contained spaces which is not valid.", )); - } else if custom_room_id.len() > 255 { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Custom room ID is too long.")); } let full_room_id = format!("!{}:{}", custom_room_id, services().globals.config.server_name); From 884cbab13557ddd34c2a37f7cbb88522095332f8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 28 May 2024 12:53:39 -0400 Subject: [PATCH 0039/2091] ci: comment out hardened_malloc clippy check for now i need to either fix static linking, or just make it dynamic always (but then kinda useless idk) https://gitlab.com/conduwuit/conduwuit/-/jobs/6953419673#L3155 Signed-off-by: strawberry --- engage.toml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/engage.toml b/engage.toml index 1ce015be..9a2ae731 100644 --- a/engage.toml +++ b/engage.toml @@ -115,18 +115,18 @@ cargo clippy \ -D warnings """ -[[task]] -name = "clippy/hardened_malloc" -group = "lints" -script = """ -cargo clippy \ - --workspace \ - --features hardened_malloc \ - --all-targets \ - --color=always \ - -- \ - -D warnings -""" +#[[task]] +#name = "clippy/hardened_malloc" +#group = "lints" +#script = """ +#cargo clippy \ +# --workspace \ +# --features hardened_malloc \ +# --all-targets \ +# --color=always \ +# -- \ +# -D warnings +#""" [[task]] name = "lychee" From 68f42f5a2fc0a5e84c59aa7a72dbe7855ea2de27 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 28 May 2024 15:07:30 -0400 Subject: [PATCH 0040/2091] fed: relax read receipt EDU check so in theory: guest users, peaking over federation, and world readable rooms should be allowed to send read receipts even if they're not joined. relaxing this check to only allow the read receipt if the server has at least 1 member in the room makes some of this still work Signed-off-by: strawberry --- src/api/server_server.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/api/server_server.rs b/src/api/server_server.rs index 1fe1ed1b..7eb4d7cf 100644 --- a/src/api/server_server.rs +++ b/src/api/server_server.rs @@ -367,7 +367,13 @@ pub(crate) async fn send_transaction_message_route( continue; } - if services().rooms.state_cache.is_joined(&user_id, &room_id)? { + if services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(Result::ok) + .any(|member| member.server_name() == user_id.server_name()) + { for event_id in &user_updates.event_ids { let mut user_receipts = BTreeMap::new(); user_receipts.insert(user_id.clone(), user_updates.data.clone()); @@ -389,7 +395,7 @@ pub(crate) async fn send_transaction_message_route( .readreceipt_update(&user_id, &room_id, event)?; } } else { - debug_warn!(%user_id, %room_id, "received read receipt EDU for user not in room"); + debug_warn!(%user_id, %room_id, %origin, "received read receipt EDU from server who does not have a single member from their server in the room"); continue; } } @@ -411,7 +417,7 @@ pub(crate) async fn send_transaction_message_route( .acl_check(typing.user_id.server_name(), &typing.room_id) .is_err() { - debug_warn!(%typing.user_id, %typing.room_id, "received typing EDU for ACL'd user's server"); + debug_warn!(%typing.user_id, %typing.room_id, %origin, "received typing EDU for ACL'd user's server"); continue; } @@ -441,7 +447,7 @@ pub(crate) async fn send_transaction_message_route( .await?; } } else { - debug_warn!(%typing.user_id, %typing.room_id, "received typing EDU for user not in room"); + debug_warn!(%typing.user_id, %typing.room_id, %origin, "received typing EDU for user not in room"); continue; } }, From f0533e07ef1cf2257e5250be988226630cfcf8f1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 28 May 2024 15:59:08 -0400 Subject: [PATCH 0041/2091] fed: remove unnecessary mutables, use with_capacity in couple more places Signed-off-by: strawberry --- src/api/server_server.rs | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/api/server_server.rs b/src/api/server_server.rs index 7eb4d7cf..277c971d 100644 --- a/src/api/server_server.rs +++ b/src/api/server_server.rs @@ -88,15 +88,15 @@ pub(crate) async fn get_server_version_route( // Response type for this endpoint is Json because we need to calculate a // signature for the response pub(crate) async fn get_server_keys_route() -> Result { - let mut verify_keys: BTreeMap = BTreeMap::new(); - verify_keys.insert( + let verify_keys: BTreeMap = BTreeMap::from([( format!("ed25519:{}", services().globals.keypair().version()) .try_into() .expect("found invalid server signing keys in DB"), VerifyKey { key: Base64::new(services().globals.keypair().public_key().to_vec()), }, - ); + )]); + let mut response = serde_json::from_slice( get_server_keys::v2::Response { server_key: Raw::new(&ServerSigningKeys { @@ -375,14 +375,11 @@ pub(crate) async fn send_transaction_message_route( .any(|member| member.server_name() == user_id.server_name()) { for event_id in &user_updates.event_ids { - let mut user_receipts = BTreeMap::new(); - user_receipts.insert(user_id.clone(), user_updates.data.clone()); + let user_receipts = BTreeMap::from([(user_id.clone(), user_updates.data.clone())]); - let mut receipts = BTreeMap::new(); - receipts.insert(ReceiptType::Read, user_receipts); + let receipts = BTreeMap::from([(ReceiptType::Read, user_receipts)]); - let mut receipt_content = BTreeMap::new(); - receipt_content.insert(event_id.to_owned(), receipts); + let receipt_content = BTreeMap::from([(event_id.to_owned(), receipts)]); let event = ReceiptEvent { content: ReceiptEventContent(receipt_content), @@ -633,13 +630,17 @@ pub(crate) async fn get_backfill_route(body: Ruma) -> .max() .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event not found."))?; - let limit = body.limit.min(uint!(100)); + let limit = body + .limit + .min(uint!(100)) + .try_into() + .expect("UInt could not be converted to usize"); let all_events = services() .rooms .timeline .pdus_until(user_id!("@doesntmatter:conduit.rs"), &body.room_id, until)? - .take(limit.try_into().unwrap()); + .take(limit); let events = all_events .filter_map(Result::ok) @@ -685,11 +686,17 @@ pub(crate) async fn get_missing_events_route( .event_handler .acl_check(origin, &body.room_id)?; - let mut queued_events = body.latest_events.clone(); - let mut events = Vec::new(); + let limit = body + .limit + .try_into() + .expect("UInt could not be converted to usize"); - let mut i = 0; - while i < queued_events.len() && events.len() < u64::from(body.limit) as usize { + let mut queued_events = body.latest_events.clone(); + // the vec will never have more entries the limit + let mut events = Vec::with_capacity(limit); + + let mut i: usize = 0; + while i < queued_events.len() && events.len() < limit { if let Some(pdu) = services().rooms.timeline.get_pdu_json(&queued_events[i])? { let room_id_str = pdu .get("room_id") From 1ac72ab914f959c7beb438459b532819b831b273 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 28 May 2024 16:55:08 -0400 Subject: [PATCH 0042/2091] init a few state hashmaps using with_capacity Signed-off-by: strawberry --- src/service/rooms/event_handler/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 754365c3..e523ce79 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -375,7 +375,7 @@ impl Service { // auth events debug!("Checking based on auth events"); // Build map of auth events - let mut auth_events = HashMap::new(); + let mut auth_events = HashMap::with_capacity(incoming_pdu.auth_events.len()); for id in &incoming_pdu.auth_events { let Some(auth_event) = services().rooms.timeline.get_pdu(id)? else { warn!("Could not find auth event {}", id); @@ -814,7 +814,7 @@ impl Service { &self, incoming_pdu: &Arc, room_id: &RoomId, room_version_id: &RoomVersionId, ) -> Result>>> { debug!("Calculating state at event using state res"); - let mut extremity_sstatehashes = HashMap::new(); + let mut extremity_sstatehashes = HashMap::with_capacity(incoming_pdu.prev_events.len()); let mut okay = true; for prev_eventid in &incoming_pdu.prev_events { @@ -949,7 +949,7 @@ impl Service { .fetch_and_handle_outliers(origin, &collect, create_event, room_id, room_version_id, pub_key_map) .await; - let mut state: HashMap<_, Arc> = HashMap::new(); + let mut state: HashMap<_, Arc> = HashMap::with_capacity(state_vec.len()); for (pdu, _) in state_vec { let state_key = pdu .state_key @@ -1196,7 +1196,7 @@ impl Service { Vec>, HashMap, (Arc, BTreeMap)>, )> { - let mut graph: HashMap, _> = HashMap::new(); + let mut graph: HashMap, _> = HashMap::with_capacity(initial_set.len()); let mut eventid_info = HashMap::new(); let mut todo_outlier_stack: Vec> = initial_set; @@ -1242,7 +1242,7 @@ impl Service { .flatten() }) { if pdu.origin_server_ts > first_pdu_in_room.origin_server_ts { - amount += 1; + amount = amount.saturating_add(1); for prev_prev in &pdu.prev_events { if !graph.contains_key(prev_prev) { todo_outlier_stack.push(prev_prev.clone()); From 9b096cc67b9a95ca2b5fdee008127f5feb98fd81 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 28 May 2024 18:21:25 -0400 Subject: [PATCH 0043/2091] fix: check if you've left the room before forgetting it Signed-off-by: strawberry --- src/api/client_server/membership.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/api/client_server/membership.rs b/src/api/client_server/membership.rs index 34af9cdf..3b8920f8 100644 --- a/src/api/client_server/membership.rs +++ b/src/api/client_server/membership.rs @@ -509,6 +509,17 @@ pub(crate) async fn unban_user_route(body: Ruma) -> Res pub(crate) async fn forget_room_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + if services() + .rooms + .state_cache + .is_joined(sender_user, &body.room_id)? + { + return Err(Error::BadRequest( + ErrorKind::Unknown, + "You must leave the room before forgetting it", + )); + } + services() .rooms .state_cache From f11103b43b0a66a1214613039ea0f14297bd4607 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 30 May 2024 23:03:59 -0400 Subject: [PATCH 0044/2091] media: check detected content-type against MSC2702 only return `inline` if the detected content-type is an allowed inline content-type as defined by MSC2702 Signed-off-by: strawberry --- src/core/utils/content_disposition.rs | 55 ++++++++++++++++++--------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index e1d40bec..8991c259 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -1,5 +1,3 @@ -use infer::MatcherType; - use crate::debug_info; const ATTACHMENT: &str = "attachment"; @@ -7,34 +5,53 @@ const INLINE: &str = "inline"; const APPLICATION_OCTET_STREAM: &str = "application/octet-stream"; const IMAGE_SVG_XML: &str = "image/svg+xml"; +/// as defined by MSC2702 +const ALLOWED_INLINE_CONTENT_TYPES: [&str; 26] = [ + "text/css", + "text/plain", + "text/csv", + "application/json", + "application/ld+json", + "image/jpeg", + "image/gif", + "image/png", + "image/apng", + "image/webp", + "image/avif", + "video/mp4", + "video/webm", + "video/ogg", + "video/quicktime", + "audio/mp4", + "audio/webm", + "audio/aac", + "audio/mpeg", + "audio/ogg", + "audio/wave", + "audio/wav", + "audio/x-wav", + "audio/x-pn-wav", + "audio/flac", + "audio/x-flac", +]; + /// Returns a Content-Disposition of `attachment` or `inline`, depending on the /// *parsed* contents of the file uploaded via format magic keys using `infer` /// crate (basically libmagic without needing libmagic). -/// -/// This forbids trusting what the client or remote server says the file is from -/// their `Content-Type` and we try to detect it ourselves. Also returns -/// `attachment` if the Content-Type does not match what we detected. -/// -/// TODO: add a "strict" function for comparing the Content-Type with what we -/// detected: `file_type.mime_type() != content_type` #[must_use] #[tracing::instrument(skip(buf))] pub fn content_disposition_type(buf: &[u8], content_type: &Option) -> &'static str { let Some(file_type) = infer::get(buf) else { + debug_info!("Failed to infer the file's contents, assuming attachment for Content-Disposition"); return ATTACHMENT; }; - debug_info!("MIME type: {}", file_type.mime_type()); + debug_info!("detected MIME type: {}", file_type.mime_type()); - match file_type.matcher_type() { - MatcherType::Image | MatcherType::Audio | MatcherType::Text | MatcherType::Video => { - if file_type.mime_type().contains("xml") { - ATTACHMENT - } else { - INLINE - } - }, - _ => ATTACHMENT, + if ALLOWED_INLINE_CONTENT_TYPES.contains(&file_type.mime_type()) { + INLINE + } else { + ATTACHMENT } } From b1886583d94bd1e4b89121cc305e20e4b9095d21 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 30 May 2024 23:14:35 -0400 Subject: [PATCH 0045/2091] csp: fix typo, add base-uri none Signed-off-by: strawberry --- src/router/layers.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index 33d0cb20..2b1eaca6 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -66,7 +66,7 @@ pub(crate) fn build(server: &Arc) -> io::Result Date: Thu, 30 May 2024 23:33:52 -0400 Subject: [PATCH 0046/2091] media: return application/octet-stream if no content-type was provided Signed-off-by: strawberry --- src/core/utils/content_disposition.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index 8991c259..1eafcc4a 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -62,13 +62,13 @@ pub fn content_disposition_type(buf: &[u8], content_type: &Option) -> &' #[must_use] #[tracing::instrument(skip(buf))] pub fn make_content_type(buf: &[u8], content_type: &Option) -> &'static str { - let Some(file_type) = infer::get(buf) else { - debug_info!("Failed to infer the file's contents"); + let Some(claimed_content_type) = content_type else { return APPLICATION_OCTET_STREAM; }; - let Some(claimed_content_type) = content_type else { - return file_type.mime_type(); + let Some(file_type) = infer::get(buf) else { + debug_info!("Failed to infer the file's contents"); + return APPLICATION_OCTET_STREAM; }; if claimed_content_type.contains("svg") && file_type.mime_type().contains("xml") { From ffdf47d1ea34a571431b85baa00f5f77a1c5e73c Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 30 May 2024 23:36:48 -0400 Subject: [PATCH 0047/2091] add `latest_pdu_in_room` timeline function Signed-off-by: strawberry --- src/service/rooms/timeline/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 04f03242..6c534b4b 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -82,6 +82,14 @@ impl Service { .transpose() } + #[tracing::instrument(skip(self))] + pub fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result>> { + self.all_pdus(user_id!("@placeholder:conduwuit.placeholder"), room_id)? + .last() + .map(|o| o.map(|(_, p)| Arc::new(p))) + .transpose() + } + #[tracing::instrument(skip(self))] pub fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { self.db.last_timeline_count(sender_user, room_id) From 9f359e05509a39612f8e79624ddb711d19df21b7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 30 May 2024 23:37:34 -0400 Subject: [PATCH 0048/2091] make `resolve_state` public to resolve new forced compressed room state Signed-off-by: strawberry --- src/service/rooms/event_handler/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index e523ce79..2243cb0a 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -671,7 +671,7 @@ impl Service { Ok(pdu_id) } - async fn resolve_state( + pub async fn resolve_state( &self, room_id: &RoomId, room_version_id: &RoomVersionId, incoming_state: HashMap>, ) -> Result>> { debug!("Loading current room state ids"); From 45e3fdba69696b6ac1d772bcc50cac96a7d0e6ab Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 30 May 2024 23:42:29 -0400 Subject: [PATCH 0049/2091] admin room: add `get-latest-pdu` and `get-first-pdu` commands Signed-off-by: strawberry --- src/admin/debug/debug_commands.rs | 42 +++++++++++++++++++++++++++++++ src/admin/debug/mod.rs | 24 +++++++++++++++++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 6a5cee35..4522b678 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -430,6 +430,48 @@ pub(crate) async fn verify_json(body: Vec<&str>) -> Result, room_id: Box) -> Result { + if !services() + .rooms + .state_cache + .server_in_room(&services().globals.config.server_name, &room_id)? + { + return Ok(RoomMessageEventContent::text_plain( + "We are not participating in the room / we don't know about the room ID.", + )); + } + + let first_pdu = services() + .rooms + .timeline + .first_pdu_in_room(&room_id)? + .ok_or_else(|| Error::bad_database("Failed to find the first PDU in database"))?; + + Ok(RoomMessageEventContent::text_plain(format!("{first_pdu:?}"))) +} + +#[tracing::instrument(skip(_body))] +pub(crate) async fn latest_pdu_in_room(_body: Vec<&str>, room_id: Box) -> Result { + if !services() + .rooms + .state_cache + .server_in_room(&services().globals.config.server_name, &room_id)? + { + return Ok(RoomMessageEventContent::text_plain( + "We are not participating in the room / we don't know about the room ID.", + )); + } + + let latest_pdu = services() + .rooms + .timeline + .latest_pdu_in_room(&room_id)? + .ok_or_else(|| Error::bad_database("Failed to find the latest PDU in database"))?; + + Ok(RoomMessageEventContent::text_plain(format!("{latest_pdu:?}"))) +} + pub(crate) async fn resolve_true_destination( _body: Vec<&str>, server_name: Box, no_cache: bool, ) -> Result { diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 260e16d5..614a4ae5 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -1,4 +1,5 @@ use clap::Subcommand; +use debug_commands::{first_pdu_in_room, latest_pdu_in_room}; use ruma::{events::room::message::RoomMessageEventContent, EventId, RoomId, ServerName}; use self::debug_commands::{ @@ -45,7 +46,8 @@ pub(crate) enum DebugCommand { server: Box, }, - /// Same as `get-remote-pdu` but accepts a codeblock newline delimited list + /// - Same as `get-remote-pdu` but accepts a codeblock newline delimited + /// list /// of PDUs and a single server to fetch from GetRemotePduList { /// Argument for us to attempt to fetch all the events from the @@ -107,6 +109,20 @@ pub(crate) enum DebugCommand { /// the command. VerifyJson, + /// - Prints the very first PDU in the specified room (typically + /// m.room.create) + FirstPduInRoom { + /// The room ID + room_id: Box, + }, + + /// - Prints the latest ("last") PDU in the specified room (typically a + /// message) + LatestPduInRoom { + /// The room ID + room_id: Box, + }, + /// - Runs a server name through conduwuit's true destination resolution /// process /// @@ -148,6 +164,12 @@ pub(crate) async fn process(command: DebugCommand, body: Vec<&str>) -> Result change_log_level(body, filter, reset).await?, DebugCommand::SignJson => sign_json(body).await?, DebugCommand::VerifyJson => verify_json(body).await?, + DebugCommand::FirstPduInRoom { + room_id, + } => first_pdu_in_room(body, room_id).await?, + DebugCommand::LatestPduInRoom { + room_id, + } => latest_pdu_in_room(body, room_id).await?, DebugCommand::GetRemotePduList { server, force, From 173ff26eb6051bb866a252fdbd216d2afccf4d0a Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 30 May 2024 23:44:57 -0400 Subject: [PATCH 0050/2091] disable URL previews by default upon admin room creation Signed-off-by: strawberry --- src/service/admin.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/service/admin.rs b/src/service/admin.rs index b44c9d0c..f38c75c4 100644 --- a/src/service/admin.rs +++ b/src/service/admin.rs @@ -14,6 +14,7 @@ use ruma::{ message::RoomMessageEventContent, name::RoomNameEventContent, power_levels::RoomPowerLevelsEventContent, + preview_url::RoomPreviewUrlsEventContent, topic::RoomTopicEventContent, }, TimelineEventType, @@ -387,6 +388,27 @@ impl Service { services().rooms.alias.set_alias(&alias, &room_id)?; + // 7. (ad-hoc) Disable room previews for everyone by default + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomPreviewUrls, + content: to_raw_value(&RoomPreviewUrlsEventContent { + disabled: true, + }) + .expect("event is valid we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + &server_user, + &room_id, + &state_lock, + ) + .await?; + Ok(()) } From 9a5f1dac575c9ca076cdeda691a3405ea43e0911 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 31 May 2024 00:19:26 -0400 Subject: [PATCH 0051/2091] drop unnecessarily verbose get_alias_helper logging Signed-off-by: strawberry --- src/api/client_server/alias.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/client_server/alias.rs b/src/api/client_server/alias.rs index 8c3c0e4c..bee9fed6 100644 --- a/src/api/client_server/alias.rs +++ b/src/api/client_server/alias.rs @@ -119,7 +119,7 @@ pub async fn get_alias_helper( ) .await; - debug_info!("room alias server_name get_alias_helper response: {response:?}"); + debug!("room alias server_name get_alias_helper response: {response:?}"); if let Err(ref e) = response { debug_info!( @@ -140,7 +140,7 @@ pub async fn get_alias_helper( }, ) .await; - debug_info!("Got response from server {server} for room aliases: {response:?}"); + debug!("Got response from server {server} for room aliases: {response:?}"); if let Ok(ref response) = response { if !response.servers.is_empty() { @@ -162,7 +162,7 @@ pub async fn get_alias_helper( pre_servers.push(room_alias.server_name().into()); let servers = room_available_servers(&room_id, &room_alias, &Some(pre_servers)); - debug_warn!( + debug!( "room alias servers from federation response for room ID {room_id} and room alias {room_alias}: \ {servers:?}" ); @@ -213,7 +213,7 @@ pub async fn get_alias_helper( let servers = room_available_servers(&room_id, &room_alias, &None); - debug_warn!("room alias servers for room ID {room_id} and room alias {room_alias}"); + debug!("room alias servers for room ID {room_id} and room alias {room_alias}"); Ok(get_alias::v3::Response::new(room_id, servers)) } From 023fb41c4969ea48a85b3ae12b78d9bacb1bfcd0 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 1 Jun 2024 00:08:26 -0400 Subject: [PATCH 0052/2091] load .env file before initializing flake to use `DIRENV_DEVSHELL` in .env Signed-off-by: strawberry --- .envrc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.envrc b/.envrc index 56451c1a..952ec2f8 100644 --- a/.envrc +++ b/.envrc @@ -1,7 +1,7 @@ #!/usr/bin/env bash +dotenv_if_exists + use flake ".#${DIRENV_DEVSHELL:-default}" PATH_add bin - -dotenv_if_exists From 3c4e325036b6b3f568b2de94adc3abe5e3b955f1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 1 Jun 2024 00:12:28 -0400 Subject: [PATCH 0053/2091] nix(bin): cache `default` devshell on top of `all-features` devshell Signed-off-by: strawberry --- bin/nix-build-and-cache | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/nix-build-and-cache b/bin/nix-build-and-cache index f74a64e9..e466bc83 100755 --- a/bin/nix-build-and-cache +++ b/bin/nix-build-and-cache @@ -70,6 +70,7 @@ ci() { --inputs-from "$toplevel" # Keep sorted + "$toplevel#devShells.x86_64-linux.default" "$toplevel#devShells.x86_64-linux.all-features" attic#default nixpkgs#direnv From b63937af0b603b35c850c87c5f824447231b5c1a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 1 Jun 2024 00:17:03 -0400 Subject: [PATCH 0054/2091] ci(engage): add cargo test default, use all-features devshell for cargo test --all-features Signed-off-by: strawberry --- engage.toml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/engage.toml b/engage.toml index 9a2ae731..011244df 100644 --- a/engage.toml +++ b/engage.toml @@ -134,13 +134,27 @@ group = "lints" script = "lychee --verbose --offline docs *.md" [[task]] -name = "cargo" +name = "cargo/all" +group = "tests" +script = """ +env DIRENV_DEVSHELL=all-features \ + direnv exec . \ + cargo test \ + --workspace \ + --all-targets \ + --all-features \ + --color=always \ + -- \ + --color=always +""" + +[[task]] +name = "cargo/default" group = "tests" script = """ cargo test \ --workspace \ --all-targets \ - --all-features \ --color=always \ -- \ --color=always From f6fa2a4f652325fcd7713469708c95165bb58ccb Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 1 Jun 2024 00:22:50 -0400 Subject: [PATCH 0055/2091] use `swap_remove` instead of `remove` in a few places `swap_remove` is faster if we don't care about the order (O(1)) Signed-off-by: strawberry --- src/api/client_server/alias.rs | 4 ++-- src/api/server_server.rs | 2 +- src/service/rooms/timeline/mod.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/client_server/alias.rs b/src/api/client_server/alias.rs index bee9fed6..fbb1d8cc 100644 --- a/src/api/client_server/alias.rs +++ b/src/api/client_server/alias.rs @@ -247,13 +247,13 @@ fn room_available_servers( .iter() .position(|server_name| server_is_ours(server_name)) { - servers.remove(server_index); + servers.swap_remove(server_index); servers.insert(0, services().globals.server_name().to_owned()); } else if let Some(alias_server_index) = servers .iter() .position(|server| server == room_alias.server_name()) { - servers.remove(alias_server_index); + servers.swap_remove(alias_server_index); servers.insert(0, room_alias.server_name().into()); } diff --git a/src/api/server_server.rs b/src/api/server_server.rs index 277c971d..f966bdbf 100644 --- a/src/api/server_server.rs +++ b/src/api/server_server.rs @@ -1922,7 +1922,7 @@ pub(crate) async fn get_room_information_route( .iter() .position(|server| server == services().globals.server_name()) { - servers.remove(server_index); + servers.swap_remove(server_index); servers.insert(0, services().globals.server_name().to_owned()); } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 6c534b4b..b45b2a3b 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1078,7 +1078,7 @@ impl Service { .into_iter() .position(|server_name| server_is_ours(&server_name)) { - servers.remove(server_index); + servers.swap_remove(server_index); } servers.sort_unstable(); From 79fb8091dcaac4abaa5db64af5679fd81ffe30de Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 1 Jun 2024 00:28:14 -0400 Subject: [PATCH 0056/2091] ci(engage): use all-features direnv devshell for clippy/all Signed-off-by: strawberry --- engage.toml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/engage.toml b/engage.toml index 011244df..3deeada9 100644 --- a/engage.toml +++ b/engage.toml @@ -93,13 +93,15 @@ cargo clippy \ name = "clippy/all" group = "lints" script = """ -cargo clippy \ - --workspace \ - --all-targets \ - --all-features \ - --color=always \ - -- \ - -D warnings +env DIRENV_DEVSHELL=all-features \ + direnv exec . \ + cargo clippy \ + --workspace \ + --all-targets \ + --all-features \ + --color=always \ + -- \ + -D warnings """ [[task]] From 4ea7af57804bb1085550686450a7a46fc9cb6628 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 1 Jun 2024 00:33:41 -0400 Subject: [PATCH 0057/2091] ci(engage): use all-features devshell for cargo doc / rustdoc --all-features Signed-off-by: strawberry --- engage.toml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/engage.toml b/engage.toml index 3deeada9..39da78a8 100644 --- a/engage.toml +++ b/engage.toml @@ -69,12 +69,15 @@ script = "cargo fmt --check -- --color=always" name = "cargo-doc" group = "lints" script = """ -RUSTDOCFLAGS="-D warnings" cargo doc \ - --workspace \ - --all-features \ - --no-deps \ - --document-private-items \ - --color always +env DIRENV_DEVSHELL=all-features \ + RUSTDOCFLAGS="-D warnings" \ + direnv exec . \ + cargo doc \ + --workspace \ + --all-features \ + --no-deps \ + --document-private-items \ + --color always """ [[task]] From 83220b43a2174ba12db80126300292960abd57e4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 1 Jun 2024 00:49:56 -0400 Subject: [PATCH 0058/2091] use saturating_add and vec with_capacity in even more places Signed-off-by: strawberry --- src/admin/media/media_commands.rs | 4 ++-- src/admin/room/room_moderation_commands.rs | 4 ++-- src/service/appservice/mod.rs | 4 ++-- src/service/globals/mod.rs | 2 +- src/service/media/mod.rs | 6 +++--- src/service/rooms/auth_chain/mod.rs | 20 ++++++++++---------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/admin/media/media_commands.rs b/src/admin/media/media_commands.rs index 8e87b736..e69fbfec 100644 --- a/src/admin/media/media_commands.rs +++ b/src/admin/media/media_commands.rs @@ -23,7 +23,7 @@ pub(crate) async fn delete( debug!("Got event ID to delete media from: {event_id}"); let mut mxc_urls = vec![]; - let mut mxc_deletion_count = 0; + let mut mxc_deletion_count: usize = 0; // parsing the PDU for any MXC URLs begins here if let Some(event_json) = services().rooms.timeline.get_pdu_json(&event_id)? { @@ -123,7 +123,7 @@ pub(crate) async fn delete( for mxc_url in mxc_urls { services().media.delete(mxc_url).await?; - mxc_deletion_count += 1; + mxc_deletion_count = mxc_deletion_count.saturating_add(1); } return Ok(RoomMessageEventContent::text_plain(format!( diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 88b05fd0..46ae0184 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -185,7 +185,7 @@ pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> .try_into() .expect("#admins:server_name is a valid alias name"); - let mut room_ban_count = 0; + let mut room_ban_count: usize = 0; let mut room_ids: Vec = Vec::new(); for &room in &rooms_s { @@ -297,7 +297,7 @@ pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> for room_id in room_ids { if services().rooms.metadata.ban_room(&room_id, true).is_ok() { debug!("Banned {room_id} successfully"); - room_ban_count += 1; + room_ban_count = room_ban_count.saturating_add(1); } debug!("Making all users leave the room {}", &room_id); diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 1d3bc98a..8970dba8 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -64,8 +64,8 @@ impl TryFrom> for NamespaceRegex { type Error = regex::Error; fn try_from(value: Vec) -> Result { - let mut exclusive = vec![]; - let mut non_exclusive = vec![]; + let mut exclusive = Vec::with_capacity(value.len()); + let mut non_exclusive = Vec::with_capacity(value.len()); for namespace in value { if namespace.exclusive { diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 2621ae24..b30d1566 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -305,7 +305,7 @@ impl Service { pub fn block_non_admin_invites(&self) -> bool { self.config.block_non_admin_invites } pub fn supported_room_versions(&self) -> Vec { - let mut room_versions: Vec = vec![]; + let mut room_versions: Vec = Vec::with_capacity(self.stable_room_versions.len()); room_versions.extend(self.stable_room_versions.clone()); if self.allow_unstable_room_versions() { room_versions.extend(self.unstable_room_versions.clone()); diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 0697577f..c7326e1a 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -182,7 +182,7 @@ impl Service { /// Deletes all remote only media files in the given at or after /// time/duration. Returns a u32 with the amount of media files deleted. - pub async fn delete_all_remote_media_at_after_time(&self, time: String) -> Result { + pub async fn delete_all_remote_media_at_after_time(&self, time: String) -> Result { let all_keys = self.db.get_all_media_keys(); let user_duration: SystemTime = match cyborgtime::parse_duration(&time) { @@ -278,12 +278,12 @@ impl Service { debug!("Deleting media now in the past \"{:?}\".", user_duration); - let mut deletion_count = 0; + let mut deletion_count: usize = 0; for mxc in remote_mxcs { debug!("Deleting MXC {mxc} from database and filesystem"); self.delete(mxc).await?; - deletion_count += 1; + deletion_count = deletion_count.saturating_add(1); } Ok(deletion_count) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index d8e38c85..fce98521 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -54,9 +54,9 @@ impl Service { "start", ); - let mut hits = 0; - let mut misses = 0; - let mut full_auth_chain = Vec::new(); + let mut hits: usize = 0; + let mut misses: usize = 0; + let mut full_auth_chain = Vec::with_capacity(buckets.len()); for chunk in buckets { if chunk.is_empty() { continue; @@ -70,13 +70,13 @@ impl Service { { trace!("Found cache entry for whole chunk"); full_auth_chain.extend(cached.iter().copied()); - hits += 1; + hits = hits.saturating_add(1); continue; } - let mut hits2 = 0; - let mut misses2 = 0; - let mut chunk_cache = Vec::new(); + let mut hits2: usize = 0; + let mut misses2: usize = 0; + let mut chunk_cache = Vec::with_capacity(chunk.len()); for (sevent_id, event_id) in chunk { if let Some(cached) = services() .rooms @@ -85,7 +85,7 @@ impl Service { { trace!(?event_id, "Found cache entry for event"); chunk_cache.extend(cached.iter().copied()); - hits2 += 1; + hits2 = hits2.saturating_add(1); } else { let auth_chain = self.get_auth_chain_inner(room_id, event_id)?; services() @@ -93,7 +93,7 @@ impl Service { .auth_chain .cache_auth_chain(vec![sevent_id], &auth_chain)?; chunk_cache.extend(auth_chain.iter()); - misses2 += 1; + misses2 = misses2.saturating_add(1); debug!( event_id = ?event_id, chain_length = ?auth_chain.len(), @@ -111,7 +111,7 @@ impl Service { .auth_chain .cache_auth_chain_vec(chunk_key, &chunk_cache)?; full_auth_chain.extend(chunk_cache.iter()); - misses += 1; + misses = misses.saturating_add(1); debug!( chunk_cache_length = ?chunk_cache.len(), hits = ?hits2, From b29a8791de8fc7c1468e3f99183675696a2d9754 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 2 Jun 2024 19:29:08 -0400 Subject: [PATCH 0059/2091] admincmd: leave all rooms if deactivating all users with `--force` Signed-off-by: strawberry --- src/admin/user/mod.rs | 4 ++-- src/admin/user/user_commands.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 771b866b..031dd97a 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -38,7 +38,7 @@ pub(crate) enum UserCommand { /// Recommended to use in conjunction with list-local-users. /// /// Users will not be removed from joined rooms by default. - /// Can be overridden with --leave-rooms flag. + /// Can be overridden with --leave-rooms OR the --force flag. /// Removing a mass amount of users from a room may cause a significant /// amount of leave events. The time to leave rooms may depend significantly /// on joined rooms and servers. @@ -50,7 +50,7 @@ pub(crate) enum UserCommand { /// Remove users from their joined rooms leave_rooms: bool, #[arg(short, long)] - /// Also deactivate admin accounts + /// Also deactivate admin accounts and will assume leave all rooms too force: bool, }, diff --git a/src/admin/user/user_commands.rs b/src/admin/user/user_commands.rs index 7422a758..d7fc321d 100644 --- a/src/admin/user/user_commands.rs +++ b/src/admin/user/user_commands.rs @@ -238,7 +238,7 @@ pub(crate) async fn deactivate_all(body: Vec<&str>, leave_rooms: bool, force: bo } } - let mut deactivation_count: u16 = 0; + let mut deactivation_count: usize = 0; let mut admins = Vec::new(); if !force { @@ -279,7 +279,7 @@ pub(crate) async fn deactivate_all(body: Vec<&str>, leave_rooms: bool, force: bo } } - if leave_rooms { + if leave_rooms || force { for &user_id in &user_ids { leave_all_rooms(user_id).await; } @@ -291,7 +291,7 @@ pub(crate) async fn deactivate_all(body: Vec<&str>, leave_rooms: bool, force: bo ))) } else { Ok(RoomMessageEventContent::text_plain(format!( - "Deactivated {} accounts.\nSkipped admin accounts: {:?}. Use --force to deactivate admin accounts", + "Deactivated {} accounts.\nSkipped admin accounts: {}. Use --force to deactivate admin accounts", deactivation_count, admins.join(", ") ))) From 84290bd668e77d485a9512fbb8a77073d9be9a46 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 3 Jun 2024 00:07:38 -0400 Subject: [PATCH 0060/2091] update deps, remove unnecessary zstd crate, pin rust-rocksdb rev Signed-off-by: strawberry --- Cargo.lock | 66 ++++++++++++++++++------------------ Cargo.toml | 21 +++++------- deps/rust-rocksdb/Cargo.toml | 3 +- src/core/Cargo.toml | 2 -- src/database/Cargo.toml | 2 -- src/service/Cargo.toml | 2 -- 6 files changed, 44 insertions(+), 52 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6384b35..c941056d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -501,9 +501,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" @@ -693,7 +693,6 @@ dependencies = [ "tracing", "tracing-subscriber", "url", - "zstd", ] [[package]] @@ -711,7 +710,6 @@ dependencies = [ "thread_local", "tokio", "tracing", - "zstd", ] [[package]] @@ -779,7 +777,6 @@ dependencies = [ "tokio", "tracing", "url", - "zstd", ] [[package]] @@ -1691,9 +1688,9 @@ dependencies = [ [[package]] name = "infer" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb33622da908807a06f9513c19b3c1ad50fab3e4137d82a78107d502075aa199" +checksum = "bc150e5ce2330295b8616ce0e3f53250e53af31759a9dbedad1621ba29151847" [[package]] name = "inlinable_string" @@ -2036,9 +2033,9 @@ checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" [[package]] name = "nix" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags 2.5.0", "cfg-if", @@ -2506,9 +2503,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.84" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec96c6a92621310b51366f1e28d05ef11489516e93be030060e5fc12024a49d6" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] @@ -2941,7 +2938,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.22.0+9.2.1" -source = "git+https://github.com/zaidoon1/rust-rocksdb?branch=master#368922d31b3fdb1c163542c1bfebd386ab46fc74" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=e9e1cb5ba92a44ea225fe8d13b31aa23621b9035#e9e1cb5ba92a44ea225fe8d13b31aa23621b9035" dependencies = [ "bindgen", "bzip2-sys", @@ -2958,7 +2955,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.26.0" -source = "git+https://github.com/zaidoon1/rust-rocksdb?branch=master#368922d31b3fdb1c163542c1bfebd386ab46fc74" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=e9e1cb5ba92a44ea225fe8d13b31aa23621b9035#e9e1cb5ba92a44ea225fe8d13b31aa23621b9035" dependencies = [ "libc", "rust-librocksdb-sys", @@ -3153,9 +3150,9 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "sentry" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00421ed8fa0c995f07cde48ba6c89e80f2b312f74ff637326f392fbfd23abe02" +checksum = "3b73ab9d5b35ed718611e89db8c886647821cfce79723b9e2e4e7cb3fd9cdd49" dependencies = [ "httpdate", "reqwest", @@ -3175,9 +3172,9 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a79194074f34b0cbe5dd33896e5928bbc6ab63a889bd9df2264af5acb186921e" +checksum = "8ce52a2633e0332821389f865762b94852e71b359cd8e95cbf91f38c154302cb" dependencies = [ "backtrace", "once_cell", @@ -3187,9 +3184,9 @@ dependencies = [ [[package]] name = "sentry-contexts" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba8870c5dba2bfd9db25c75574a11429f6b95957b0a78ac02e2970dd7a5249a" +checksum = "10565e6eb013737bf7555250f8b4b8248a571e1de4de7230037f5dd5ea51bc63" dependencies = [ "hostname 0.4.0", "libc", @@ -3201,12 +3198,15 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46a75011ea1c0d5c46e9e57df03ce81f5c7f0a9e199086334a1f9c0a541e0826" +checksum = "a2c5e7a0430cfa86b7cb6e7bc01a56331937b2ba08ab703c2e7331139a99f121" dependencies = [ + "crc32fast", + "itertools 0.13.0", "once_cell", "rand", + "regex", "sentry-types", "serde", "serde_json", @@ -3214,9 +3214,9 @@ dependencies = [ [[package]] name = "sentry-debug-images" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ec2a486336559414ab66548da610da5e9626863c3c4ffca07d88f7dc71c8de8" +checksum = "2be3e693a65dba8578790de0c6adc0a833815979590ed867336aff8fbcf9cbf7" dependencies = [ "findshlibs", "once_cell", @@ -3225,9 +3225,9 @@ dependencies = [ [[package]] name = "sentry-log" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e74b7261245ff17a8c48e8f3e1e96fb6b84146870121af880d53aef6a5b4f784" +checksum = "184b15d2ee727a35a0731ab6c860491a8737149abbe06eee9fbe43fac24b2f07" dependencies = [ "log", "sentry-core", @@ -3235,9 +3235,9 @@ dependencies = [ [[package]] name = "sentry-panic" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eaa3ecfa3c8750c78dcfd4637cfa2598b95b52897ed184b4dc77fcf7d95060d" +checksum = "6f4040c0ad5670b90fc60759bbb31e3fc0d6517448d0c7440255a6f7362c50e7" dependencies = [ "sentry-backtrace", "sentry-core", @@ -3245,9 +3245,9 @@ dependencies = [ [[package]] name = "sentry-tower" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df141464944fdf8e2a6f2184eb1d973a20456466f788346b6e3a51791cdaa370" +checksum = "d03daa33dfa3adc8b99da9afe10b1775a8c689e3242bf616badec7455916c525" dependencies = [ "http 1.1.0", "pin-project", @@ -3259,9 +3259,9 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f715932bf369a61b7256687c6f0554141b7ce097287e30e3f7ed6e9de82498fe" +checksum = "151ff7c03687737ba11969ccdc6f45eb7775ed7a95880810297c6cc8c3574e75" dependencies = [ "sentry-backtrace", "sentry-core", @@ -3271,9 +3271,9 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.32.3" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4519c900ce734f7a0eb7aba0869dfb225a7af8820634a7dd51449e3b093cfb7c" +checksum = "10d5ad9e33b9f6f598387a6a23aa23c7f13e4e7740a34cd4d9c8db851bfdae05" dependencies = [ "debugid", "hex", diff --git a/Cargo.toml b/Cargo.toml index cc6d5960..ee9c0100 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,7 +29,7 @@ name = "conduit" version = "0.5.0" [workspace.dependencies.infer] -version = "0.15" +version = "0.16" default-features = false [workspace.dependencies.jsonwebtoken] @@ -114,7 +114,7 @@ features = [ ] [workspace.dependencies.serde] -version = "1.0.201" +version = "1.0.203" features = ["rc"] [workspace.dependencies.serde_json] @@ -184,7 +184,7 @@ version = "0.3.30" default-features = false [workspace.dependencies.tokio] -version = "1.37.0" +version = "1.38.0" features = [ "fs", "net", @@ -261,7 +261,7 @@ version = "2.1.1" # used to replace the channels of the tokio runtime [workspace.dependencies.loole] -version = "0.3.0" +version = "0.3.1" [workspace.dependencies.async-trait] version = "0.1.80" @@ -313,9 +313,6 @@ features = [ "bzip2", ] -[workspace.dependencies.zstd] -version = "0.13.1" - # to listen on both HTTP and HTTPS if listening on TLS dierctly from conduwuit for complement or sytest [workspace.dependencies.axum-server-dual-protocol] version = "0.6" @@ -344,7 +341,7 @@ features = ["rt-tokio"] # optional sentry metrics for crash/panic reporting [workspace.dependencies.sentry] -version = "0.32.3" +version = "0.33.0" default-features = false features = [ "backtrace", @@ -360,9 +357,9 @@ features = [ ] [workspace.dependencies.sentry-tracing] -version = "0.32.3" +version = "0.33.0" [workspace.dependencies.sentry-tower] -version = "0.32.3" +version = "0.33.0" # jemalloc usage [workspace.dependencies.tikv-jemalloc-sys] @@ -386,7 +383,7 @@ features = ["bundled"] # used only by rusqlite [workspace.dependencies.parking_lot] -version = "0.12.2" +version = "0.12.3" # used only by rusqlite [workspace.dependencies.thread_local] @@ -400,7 +397,7 @@ default-features = false version = "0.2" [workspace.dependencies.nix] -version = "0.28.0" +version = "0.29.0" features = ["resource"] [workspace.dependencies.sd-notify] diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 4dce69d1..ff549845 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,8 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/zaidoon1/rust-rocksdb" -branch = "master" +rev = "e9e1cb5ba92a44ea225fe8d13b31aa23621b9035" +#branch = "master" default-features = false [lib] diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index dd17946f..67aba76f 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -101,8 +101,6 @@ tokio.workspace = true tracing-subscriber.workspace = true tracing.workspace = true url.workspace = true -zstd.optional = true -zstd.workspace = true [target.'cfg(unix)'.dependencies] nix.workspace = true diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 113414cb..18e073ed 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -58,8 +58,6 @@ thread_local.optional = true thread_local.workspace = true tokio.workspace = true tracing.workspace = true -zstd.optional = true -zstd.workspace = true [lints] workspace = true diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index f6f682a9..2cdf6240 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -65,8 +65,6 @@ sha2.workspace = true tokio.workspace = true tracing.workspace = true url.workspace = true -zstd.optional = true -zstd.workspace = true [lints] workspace = true From ff7dfec74c1078266a7995b952557cf353edd335 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 3 Jun 2024 00:52:33 -0400 Subject: [PATCH 0061/2091] slightly cleanup update check Signed-off-by: strawberry --- src/service/globals/updates.rs | 28 +++++++++++++--------------- src/service/services.rs | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/service/globals/updates.rs b/src/service/globals/updates.rs index 9413c54a..c6ac9fff 100644 --- a/src/service/globals/updates.rs +++ b/src/service/globals/updates.rs @@ -3,13 +3,16 @@ use std::time::Duration; use ruma::events::room::message::RoomMessageEventContent; use serde::Deserialize; use tokio::{task::JoinHandle, time::interval}; -use tracing::{debug, error, warn}; +use tracing::{error, warn}; use crate::{ conduit::{Error, Result}, services, }; +const CHECK_FOR_UPDATES_URL: &str = "https://pupbrain.dev/check-for-updates/stable"; +const CHECK_FOR_UPDATES_INTERVAL: u64 = 7200; // 2 hours + #[derive(Deserialize)] struct CheckForUpdatesResponseEntry { id: u64, @@ -22,39 +25,34 @@ struct CheckForUpdatesResponse { } #[tracing::instrument] -pub async fn start_check_for_updates_task() -> Result> { - let timer_interval = Duration::from_secs(7200); // 2 hours +pub fn start_check_for_updates_task() -> JoinHandle<()> { + let timer_interval = Duration::from_secs(CHECK_FOR_UPDATES_INTERVAL); - Ok(services().server.runtime().spawn(async move { + services().server.runtime().spawn(async move { let mut i = interval(timer_interval); loop { - tokio::select! { - _ = i.tick() => { - debug!(target: "start_check_for_updates_task", "Timer ticked"); - }, - } + i.tick().await; if let Err(e) = try_handle_updates().await { warn!(%e, "Failed to check for updates"); } } - })) + }) } +#[tracing::instrument(skip_all)] async fn try_handle_updates() -> Result<()> { let response = services() .globals .client .default - .get("https://pupbrain.dev/check-for-updates/stable") + .get(CHECK_FOR_UPDATES_URL) .send() .await?; - let response = serde_json::from_str::(&response.text().await?).map_err(|e| { - error!("Bad check for updates response: {e}"); - Error::BadServerResponse("Bad version check response") - })?; + let response = serde_json::from_str::(&response.text().await?) + .map_err(|e| Error::Err(format!("Bad check for updates response: {e}")))?; let mut last_update_id = services().globals.last_check_for_updates_id()?; for update in response.updates { diff --git a/src/service/services.rs b/src/service/services.rs index a9356562..05894753 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -289,7 +289,7 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} } if self.globals.allow_check_for_updates() { - let handle = globals::updates::start_check_for_updates_task().await?; + let handle = globals::updates::start_check_for_updates_task(); #[allow(clippy::let_underscore_must_use)] // needed for shutdown { From 2e83e56a078f5fcef7b995d875c3548313c6edf2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 3 Jun 2024 01:17:00 -0400 Subject: [PATCH 0062/2091] remove deleted config options and update `address` example option Signed-off-by: strawberry --- conduwuit-example.toml | 19 +++++-------------- src/core/config/mod.rs | 12 ------------ 2 files changed, 5 insertions(+), 26 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 910ebe8f..1c6f56d4 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -77,11 +77,16 @@ database_backend = "rocksdb" # forwarded to the conduwuit instance running on this port # Docker users: Don't change this, you'll need to map an external port to this. # To listen on multiple ports, specify a vector e.g. [8080, 8448] +# +# default if unspecified is 8008 port = 6167 # default address (IPv4 or IPv6) conduwuit will listen on. Generally you want this to be # localhost (127.0.0.1 / ::1). If you are using Docker or a container NAT networking setup, you # likely need this to be 0.0.0.0. +# To listen multiple addresses, specify a vector e.g. ["127.0.0.1", "::1"] +# +# default if unspecified is both IPv4 and IPv6 localhost: ["127.0.0.1", "::1"] address = "127.0.0.1" # Max request size for file uploads @@ -375,15 +380,6 @@ allow_profile_lookup_federation_requests = true # Defaults to 256.0 #db_cache_capacity_mb = 256.0 -# Interval in seconds when conduwuit will run database cleanup operations. -# -# For SQLite: this will flush the WAL by executing `PRAGMA wal_checkpoint(RESTART)` (https://www.sqlite.org/pragma.html#pragma_wal_checkpoint) -# For RocksDB: this will run `flush_opt` to flush database memtables to SST files on disk (https://docs.rs/rocksdb/latest/rocksdb/struct.DBCommon.html#method.flush_opt) -# These operations always run on shutdown. -# -# Defaults to 30 minutes (1800 seconds) to avoid IO amplification from too frequent cleanups -#cleanup_second_interval = 1800 - ### RocksDB options @@ -492,11 +488,6 @@ allow_profile_lookup_federation_requests = true # Defaults to 1 (TolerateCorruptedTailRecords) #rocksdb_recovery_mode = 1 -# Controls whether memory buffers are written to storage at the fixed interval set by `cleanup_period_interval` -# even when they are not full. Setting this will increase load on the storage backplane and is never advised -# under normal circumstances. -#rocksdb_periodic_cleanup = false - ### Domain Name Resolution and Caching diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 67b83975..6c2c9655 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -94,9 +94,6 @@ pub struct Config { #[serde(default = "default_roomid_spacehierarchy_cache_capacity")] pub roomid_spacehierarchy_cache_capacity: u32, - #[serde(default = "default_cleanup_second_interval")] - pub cleanup_second_interval: u32, - #[serde(default = "default_dns_cache_entries")] pub dns_cache_entries: u32, #[serde(default = "default_dns_min_ttl")] @@ -249,8 +246,6 @@ pub struct Config { #[serde(default)] pub rocksdb_read_only: bool, #[serde(default)] - pub rocksdb_periodic_cleanup: bool, - #[serde(default)] pub rocksdb_compaction_prio_idle: bool, #[serde(default = "true_fn")] pub rocksdb_compaction_ioprio_idle: bool, @@ -541,7 +536,6 @@ impl fmt::Display for Config { "Roomid space hierarchy cache capacity", &self.roomid_spacehierarchy_cache_capacity.to_string(), ), - ("Cleanup interval in seconds", &self.cleanup_second_interval.to_string()), ("DNS cache entry limit", &self.dns_cache_entries.to_string()), ("DNS minimum TTL", &self.dns_min_ttl.to_string()), ("DNS minimum NXDOMAIN TTL", &self.dns_min_ttl_nxdomain.to_string()), @@ -750,8 +744,6 @@ impl fmt::Display for Config { #[cfg(feature = "rocksdb")] ("RocksDB Read-only Mode", &self.rocksdb_read_only.to_string()), #[cfg(feature = "rocksdb")] - ("RocksDB Periodic Cleanup", &self.rocksdb_periodic_cleanup.to_string()), - #[cfg(feature = "rocksdb")] ( "RocksDB Compaction Idle Priority", &self.rocksdb_compaction_prio_idle.to_string(), @@ -929,10 +921,6 @@ fn default_stateinfo_cache_capacity() -> u32 { 100 } fn default_roomid_spacehierarchy_cache_capacity() -> u32 { 100 } -fn default_cleanup_second_interval() -> u32 { - 1800 // every 30 minutes -} - fn default_dns_cache_entries() -> u32 { 32768 } fn default_dns_min_ttl() -> u64 { 60 * 180 } From 919735b4ce8b417e60b408ae4cce2f7638ac0be6 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 3 Jun 2024 01:58:26 -0400 Subject: [PATCH 0063/2091] remove usages of `&String` and `&Owned[..]` Signed-off-by: strawberry --- src/admin/handler.rs | 4 ++-- src/admin/utils.rs | 6 +++--- src/api/client_server/alias.rs | 6 +++--- src/api/client_server/room.rs | 4 ++-- src/api/client_server/sync.rs | 2 +- src/service/admin.rs | 4 ++-- src/service/sending/sender.rs | 4 ++-- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 7b7396ef..1a2ff38e 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -8,7 +8,7 @@ use ruma::{ room::message::{Relation::Reply, RoomMessageEventContent}, TimelineEventType, }, - OwnedRoomId, OwnedUserId, ServerName, UserId, + OwnedRoomId, OwnedUserId, RoomId, ServerName, UserId, }; use serde_json::value::to_raw_value; use tokio::sync::MutexGuard; @@ -125,7 +125,7 @@ async fn handle_event(event: AdminRoomEvent, admin_room: OwnedRoomId, server_use } async fn handle_response_error( - e: &Error, admin_room: &OwnedRoomId, server_user: &UserId, state_lock: &MutexGuard<'_, ()>, + e: &Error, admin_room: &RoomId, server_user: &UserId, state_lock: &MutexGuard<'_, ()>, ) -> Result<()> { error!("Failed to build and append admin room response PDU: \"{e}\""); let error_room_message = RoomMessageEventContent::text_plain(format!( diff --git a/src/admin/utils.rs b/src/admin/utils.rs index 7031b848..6e65f2b2 100644 --- a/src/admin/utils.rs +++ b/src/admin/utils.rs @@ -1,5 +1,5 @@ pub(crate) use conduit::utils::HtmlEscape; -use ruma::OwnedRoomId; +use ruma::{OwnedRoomId, RoomId}; use crate::services; @@ -9,9 +9,9 @@ pub(crate) fn escape_html(s: &str) -> String { .replace('>', ">") } -pub(crate) fn get_room_info(id: &OwnedRoomId) -> (OwnedRoomId, u64, String) { +pub(crate) fn get_room_info(id: &RoomId) -> (OwnedRoomId, u64, String) { ( - id.clone(), + id.into(), services() .rooms .state_cache diff --git a/src/api/client_server/alias.rs b/src/api/client_server/alias.rs index fbb1d8cc..e70105f3 100644 --- a/src/api/client_server/alias.rs +++ b/src/api/client_server/alias.rs @@ -8,7 +8,7 @@ use ruma::{ }, federation, }, - OwnedRoomAliasId, OwnedRoomId, OwnedServerName, + OwnedRoomAliasId, OwnedServerName, RoomAliasId, RoomId, }; use tracing::debug; @@ -219,7 +219,7 @@ pub async fn get_alias_helper( } fn room_available_servers( - room_id: &OwnedRoomId, room_alias: &OwnedRoomAliasId, pre_servers: &Option>, + room_id: &RoomId, room_alias: &RoomAliasId, pre_servers: &Option>, ) -> Vec { // find active servers in room state cache to suggest let mut servers: Vec = services() @@ -260,7 +260,7 @@ fn room_available_servers( servers } -async fn alias_checks(room_alias: &OwnedRoomAliasId, appservice_info: &Option) -> Result<()> { +async fn alias_checks(room_alias: &RoomAliasId, appservice_info: &Option) -> Result<()> { if !server_is_ours(room_alias.server_name()) { return Err(Error::BadRequest(ErrorKind::InvalidParam, "Alias is from another server.")); } diff --git a/src/api/client_server/room.rs b/src/api/client_server/room.rs index cb8930ec..2c27f569 100644 --- a/src/api/client_server/room.rs +++ b/src/api/client_server/room.rs @@ -886,7 +886,7 @@ fn default_power_levels_content( /// if a room is being created with a room alias, run our checks async fn room_alias_check( - room_alias_name: &String, appservice_info: &Option, + room_alias_name: &str, appservice_info: &Option, ) -> Result { // Basic checks on the room alias validity if room_alias_name.contains(':') { @@ -944,7 +944,7 @@ async fn room_alias_check( } /// if a room is being created with a custom room ID, run our checks against it -fn custom_room_id_check(custom_room_id: &String) -> Result { +fn custom_room_id_check(custom_room_id: &str) -> Result { // apply forbidden room alias checks to custom room IDs too if services() .globals diff --git a/src/api/client_server/sync.rs b/src/api/client_server/sync.rs index 933f7b58..8a83cbe7 100644 --- a/src/api/client_server/sync.rs +++ b/src/api/client_server/sync.rs @@ -492,7 +492,7 @@ async fn handle_left_room( } async fn process_presence_updates( - presence_updates: &mut HashMap, since: u64, syncing_user: &OwnedUserId, + presence_updates: &mut HashMap, since: u64, syncing_user: &UserId, ) -> Result<()> { use crate::service::presence::Presence; diff --git a/src/service/admin.rs b/src/service/admin.rs index f38c75c4..19923d0a 100644 --- a/src/service/admin.rs +++ b/src/service/admin.rs @@ -111,9 +111,9 @@ impl Service { .await; } - async fn receive(&self, event: AdminRoomEvent, room: &OwnedRoomId, user: &UserId) -> Result<(), Error> { + async fn receive(&self, event: AdminRoomEvent, room: &RoomId, user: &UserId) -> Result<(), Error> { if let Some(handle) = self.handle.lock().await.as_ref() { - handle(event, room.clone(), user.into()).await + handle(event, room.into(), user.into()).await } else { Err(Error::Err("Admin module is not loaded.".into())) } diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 9e107df8..c4de3e60 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -406,7 +406,7 @@ async fn send_events(dest: Destination, events: Vec) -> SendingRes } #[tracing::instrument(skip(dest, events))] -async fn send_events_dest_appservice(dest: &Destination, id: &String, events: Vec) -> SendingResult { +async fn send_events_dest_appservice(dest: &Destination, id: &str, events: Vec) -> SendingResult { let mut pdu_jsons = Vec::new(); for event in &events { @@ -469,7 +469,7 @@ async fn send_events_dest_appservice(dest: &Destination, id: &String, events: Ve #[tracing::instrument(skip(dest, events))] async fn send_events_dest_push( - dest: &Destination, userid: &OwnedUserId, pushkey: &String, events: Vec, + dest: &Destination, userid: &OwnedUserId, pushkey: &str, events: Vec, ) -> SendingResult { let mut pdus = Vec::new(); From 282c2feca8e5a63c30799607902a4c1935a1415a Mon Sep 17 00:00:00 2001 From: Tom Black Date: Tue, 4 Jun 2024 17:13:21 +0100 Subject: [PATCH 0064/2091] Clarify purpose and temporary muting --- docs/conduwuit_coc.md | 71 +++++++++++++++++++++++++++++++------------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/docs/conduwuit_coc.md b/docs/conduwuit_coc.md index 858884c5..3a5b8777 100644 --- a/docs/conduwuit_coc.md +++ b/docs/conduwuit_coc.md @@ -1,45 +1,76 @@ -This page serves as a referenceable link to conduwuit's Code of Conduct for the Matrix rooms and any further community spaces related to conduwuit. - # Conduwuit Community Code of Conduct -Welcome to the Conduwuit community! We’re excited to have you here. This space is dedicated to fostering a positive, supportive, and inclusive environment for everyone. Here are our guidelines to help maintain the welcoming atmosphere that sets Conduwuit apart. +Welcome to the conduwuit community! We’re excited to have you here. Conduwuit is a hard-fork of the Conduit homeserver, +aimed at making Matrix more accessible and inclusive for everyone. -For the foundational rules, please refer to the [Matrix.org Code of Conduct](https://matrix.org/legal/code-of-conduct/) and the [Contributor's Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md). Below are additional guidelines specific to the Conduwuit community. +This space is dedicated to fostering a positive, supportive, and inclusive environment for everyone. This Code of +Conduct applies to all conduwuit spaces, including any further community rooms that reference this CoC. Here are our +guidelines to help maintain the welcoming atmosphere that sets conduwuit apart. + +For the foundational rules, please refer to the [Matrix.org Code of Conduct](https://matrix.org/legal/code-of-conduct/) +and the [Contributor's Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md). Below are +additional guidelines specific to the conduwuit community. ## Our Values and Guidelines -1. **Respect and Inclusivity**: We are committed to maintaining a community where everyone feels safe and respected. Discrimination, harassment, or hate speech of any kind will not be tolerated. Recognise that each community member experiences the world differently based on their past experiences, background, and identity. Share your own experiences and be open to learning about others' diverse perspectives. +1. **Respect and Inclusivity**: We are committed to maintaining a community where everyone feels safe and respected. + Discrimination, harassment, or hate speech of any kind will not be tolerated. Recognise that each community member + experiences the world differently based on their past experiences, background, and identity. Share your own + experiences and be open to learning about others' diverse perspectives. -2. **Positivity and Constructiveness**: Engage in constructive discussions and support each other. If you are feeling angry, negative, or aggressive, take a break until you can participate in a positive and constructive manner. Process intense feelings with a friend or in a private setting before engaging in community conversations to help maintain a supportive and focused environment. +2. **Positivity and Constructiveness**: Engage in constructive discussions and support each other. If you feel angry, + negative, or aggressive, take a break until you can participate in a positive and constructive manner. Process + intense feelings with a friend or in a private setting before engaging in community conversations to help maintain + a supportive and focused environment. -3. **Clarity and Understanding**: We understand that our community includes neurodivergent individuals and those who may not appreciate sarcasm or subtlety. Communicate clearly and kindly, avoiding sarcasm and ensuring your messages are easily understood by all. Additionally, avoid putting the burden of education on marginalized groups by doing your own research before asking for explanations. +3. **Clarity and Understanding**: Our community includes neurodivergent individuals and those who may not appreciate + sarcasm or subtlety. Communicate clearly and kindly, avoiding sarcasm and ensuring your messages are easily + understood by all. Additionally, avoid putting the burden of education on marginalized groups by doing your own + research before asking for explanations. -4. **Be Open to Inclusivity**: Actively engage in conversations about making our community more inclusive. Report discriminatory behaviour to the moderators and be open to constructive critiques that aim to improve our community. Understand that discussing discrimination and negative experiences can be emotionally taxing, so focus on the message rather than critiquing the tone used. +4. **Be Open to Inclusivity**: Actively engage in conversations about making our community more inclusive. Report + discriminatory behavior to the moderators and be open to constructive feedback that aims to improve our community. + Understand that discussing discrimination and negative experiences can be emotionally taxing, so focus on the + message rather than critiquing the tone used. -5. **Commit to Inclusivity**: Building an inclusive community requires time, energy, and resources. Recognise that addressing discrimination and bias is an ongoing process that necessitates commitment and action from all community members. +5. **Commit to Inclusivity**: Building an inclusive community requires time, energy, and resources. Recognise that + addressing discrimination and bias is an ongoing process that necessitates commitment and action from all community + members. -## Community Spaces +## Matrix Community -This Code of Conduct applies to the entire [Conduwuit Space](https://matrix.to/#/#conduwuit-space:puppygock.gay) and its rooms: +This Code of Conduct applies to the entire [Conduwuit Matrix Space](https://matrix.to/#/#conduwuit-space:puppygock.gay) +and its rooms, including: -### [Conduwuit](https://matrix.to/#/#conduwuit:puppygock.gay) +### [#conduwuit:puppygock.gay](https://matrix.to/#/#conduwuit:puppygock.gay) -This room is for support and discussions about Conduwuit. Ask questions, share insights, and help each other out. +This room is for support and discussions about conduwuit. Ask questions, share insights, and help each other out. -### [Conduwuit Offtopic](https://matrix.to/#/#conduwuit-offtopic:girlboss.ceo) +### [#conduwuit-offtopic:girlboss.ceo](https://matrix.to/#/#conduwuit-offtopic:girlboss.ceo) -For off-topic community conversations about any subject. While this room allows for a wide range of topics, the same CoC applies. Keep discussions respectful and inclusive, and avoid divisive subjects like country/world politics. General topics, such as world events, are welcome as long as they follow the CoC. +For off-topic community conversations about any subject. While this room allows for a wide range of topics, the same +CoC applies. Keep discussions respectful and inclusive, and avoid divisive subjects like country/world politics. +General topics, such as world events, are welcome as long as they follow the CoC. -### [Conduwuit Development](https://matrix.to/#/#conduwuit-dev:puppygock.gay) +### [#conduwuit-dev:puppygock.gay](https://matrix.to/#/#conduwuit-dev:puppygock.gay) -This room is dedicated to discussing active development of Conduwuit. Posting requires an elevated power level, which can be requested in one of the other rooms. Use this space to collaborate and innovate. +This room is dedicated to discussing active development of conduwuit. Posting requires an elevated power level, which +can be requested in one of the other rooms. Use this space to collaborate and innovate. ## Enforcement -We have a zero-tolerance policy for violations of this Code of Conduct. If someone’s behavior makes you uncomfortable, please report it to the moderators. Actions we may take include: +We have a zero-tolerance policy for violations of this Code of Conduct. If someone’s behavior makes you uncomfortable, +please report it to the moderators. Actions we may take include: -1. **Warning**: A warning given directly in the room or via a private message from the moderators, identifying the violation and requesting corrective action. -2. **Kick or Ban**: Egregious behavior may result in an immediate kick or ban to protect other community members. Bans are considered permanent and will only be reversed in exceptional circumstances after proven good behaviour. +1. **Warning**: A warning given directly in the room or via a private message from the moderators, identifying + the violation and requesting corrective action. +2. **Temporary Mute**: Temporary restriction from participating in discussions for a specified period to allow for + reflection and cooling off. +3. **Kick or Ban**: Egregious behavior may result in an immediate kick or ban to protect other community members. + Bans are considered permanent and will only be reversed in exceptional circumstances after proven good behavior. + +Please highlight issues directly in rooms when possible, but if you don't feel comfortable doing that, then please send +a DM to one of the moderators directly. Together, let’s build a community where everyone feels valued and respected. From 732e8b82aa74fcfacbe78afbd62946e5d6bdd791 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 4 Jun 2024 23:51:02 +0000 Subject: [PATCH 0065/2091] Abstract password hashing into util. Signed-off-by: Jason Volk --- Cargo.lock | 3 +- Cargo.toml | 4 ++ src/api/Cargo.toml | 1 - src/api/client_server/session.rs | 13 +----- src/core/Cargo.toml | 1 + src/core/utils/hash.rs | 72 +++++++++++++++++++++++++++++++ src/core/utils/mod.rs | 1 + src/main/server.rs | 7 ++- src/service/Cargo.toml | 1 - src/service/globals/migrations.rs | 17 ++------ src/service/globals/mod.rs | 10 ----- src/service/uiaa/mod.rs | 13 +----- src/service/users/data.rs | 13 +----- 13 files changed, 92 insertions(+), 64 deletions(-) create mode 100644 src/core/utils/hash.rs diff --git a/Cargo.lock b/Cargo.lock index c941056d..78badeb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -624,7 +624,6 @@ dependencies = [ name = "conduit_api" version = "0.4.1" dependencies = [ - "argon2", "axum 0.7.5", "axum-extra", "base64 0.22.1", @@ -657,6 +656,7 @@ dependencies = [ name = "conduit_core" version = "0.4.1" dependencies = [ + "argon2", "axum 0.7.5", "axum-server", "bytes", @@ -748,7 +748,6 @@ dependencies = [ name = "conduit_service" version = "0.4.1" dependencies = [ - "argon2", "async-trait", "base64 0.22.1", "bytes", diff --git a/Cargo.toml b/Cargo.toml index ee9c0100..22df188d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -815,6 +815,10 @@ verbose_file_reads = "warn" ################### style = "warn" +## some sadness +# trivial assertions are quite alright +assertions_on_constants = "allow" + ################### suspicious = "warn" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index a80a254e..5ac15dd5 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -33,7 +33,6 @@ brotli_compression = [ ] [dependencies] -argon2.workspace = true axum-extra.workspace = true axum.workspace = true base64.workspace = true diff --git a/src/api/client_server/session.rs b/src/api/client_server/session.rs index 28df625f..6df46549 100644 --- a/src/api/client_server/session.rs +++ b/src/api/client_server/session.rs @@ -1,4 +1,3 @@ -use argon2::{PasswordHash, PasswordVerifier}; use ruma::{ api::client::{ error::ErrorKind, @@ -21,7 +20,7 @@ use serde::Deserialize; use tracing::{debug, info, warn}; use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; -use crate::{services, utils, Error, Result, Ruma}; +use crate::{services, utils, utils::hash, Error, Result, Ruma}; #[derive(Debug, Deserialize)] struct Claims { @@ -87,15 +86,7 @@ pub(crate) async fn login_route(body: Ruma) -> Result>> = Mutex::new(None); + +#[allow(clippy::let_underscore_must_use)] +pub fn init() { + // 19456 Kib blocks, iterations = 2, parallelism = 1 + // * + debug_assert!(M_COST == 19_456, "M_COST default changed"); + debug_assert!(T_COST == 2, "T_COST default changed"); + debug_assert!(P_COST == 1, "P_COST default changed"); + + let algorithm = Algorithm::Argon2id; + let version = Version::default(); + let out_len: Option = None; + let params = Params::new(M_COST, T_COST, P_COST, out_len).expect("valid parameters"); + let state = Argon2::new(algorithm, version, params); + _ = STATE.lock().expect("hashing state locked").insert(state); +} + +pub fn password(password: &str) -> Result { + let salt = SaltString::generate(rand::thread_rng()); + STATE + .lock() + .expect("hashing state locked") + .as_ref() + .expect("hashing state initialized") + .hash_password(password.as_bytes(), &salt) + .map(|it| it.to_string()) +} + +pub fn verify_password(password: &str, password_hash: &str) -> Result<(), password_hash::Error> { + let password_hash = PasswordHash::new(password_hash)?; + STATE + .lock() + .expect("hashing state locked") + .as_ref() + .expect("hashing state initialized") + .verify_password(password.as_bytes(), &password_hash) +} + +#[cfg(test)] +mod tests { + #[test] + fn password_hash_and_verify() { + use crate::utils::hash; + hash::init(); + let preimage = "temp123"; + let digest = hash::password(preimage).expect("digest"); + hash::verify_password(preimage, &digest).expect("verified"); + } + + #[test] + #[should_panic(expected = "unverified")] + fn password_hash_and_verify_fail() { + use crate::utils::hash; + hash::init(); + let preimage = "temp123"; + let fakeimage = "temp321"; + let digest = hash::password(preimage).expect("digest"); + hash::verify_password(fakeimage, &digest).expect("unverified"); + } +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index e85080f1..f3a45e47 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -1,6 +1,7 @@ pub mod content_disposition; pub mod debug; pub mod defer; +pub mod hash; pub mod html; pub mod json; pub mod sys; diff --git a/src/main/server.rs b/src/main/server.rs index 8e61d502..cd5b1cdb 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -5,7 +5,7 @@ use conduit::{ config::Config, info, log::{LogLevelReloadHandles, ReloadHandle}, - utils::sys::maximize_fd_limit, + utils::{hash, sys}, Error, Result, }; use tokio::runtime; @@ -31,13 +31,16 @@ pub(crate) struct Server { impl Server { pub(crate) fn build(args: Args, runtime: Option<&runtime::Handle>) -> Result, Error> { let config = Config::new(args.config)?; + #[cfg(feature = "sentry_telemetry")] let sentry_guard = init_sentry(&config); let (tracing_reload_handle, tracing_flame_guard) = init_tracing(&config); config.check()?; #[cfg(unix)] - maximize_fd_limit().expect("Unable to increase maximum soft and hard file descriptor limit"); + sys::maximize_fd_limit().expect("Unable to increase maximum soft and hard file descriptor limit"); + hash::init(); + info!( server_name = %config.server_name, database_path = ?config.database_path, diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 2cdf6240..3bfd0e51 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -35,7 +35,6 @@ sha256_media = [ ] [dependencies] -argon2.workspace = true async-trait.workspace = true base64.workspace = true bytes.workspace = true diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 0e04caeb..75990df0 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -6,10 +6,8 @@ use std::{ sync::Arc, }; -use argon2::{password_hash::SaltString, PasswordHasher, PasswordVerifier}; use database::KeyValueDatabase; use itertools::Itertools; -use rand::thread_rng; use ruma::{ events::{push_rules::PushRulesEvent, GlobalAccountDataEventType}, push::Ruleset, @@ -67,18 +65,9 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result if services().globals.database_version()? < 2 { // We accidentally inserted hashed versions of "" into the db instead of just "" for (userid, password) in db.userid_password.iter() { - let salt = SaltString::generate(thread_rng()); - let empty_pass = services() - .globals - .argon - .hash_password(b"", &salt) - .expect("our own password to be properly hashed"); - let empty_hashed_password = services() - .globals - .argon - .verify_password(&password, &empty_pass) - .is_ok(); - + let empty_pass = utils::hash::password("").expect("our own password to be properly hashed"); + let password = std::str::from_utf8(&password).expect("password is valid utf-8"); + let empty_hashed_password = utils::hash::verify_password(password, &empty_pass).is_ok(); if empty_hashed_password { db.userid_password.insert(&userid, b"")?; } diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index b30d1566..7543741f 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -14,7 +14,6 @@ use std::{ time::Instant, }; -use argon2::Argon2; use base64::{engine::general_purpose, Engine as _}; use data::Data; use hickory_resolver::TokioAsyncResolver; @@ -61,7 +60,6 @@ pub struct Service { pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, pub rotate: RotationHandler, - pub argon: Argon2<'static>, } /// Handles "rotation" of long-polling requests. "Rotation" in this context is @@ -125,13 +123,6 @@ impl Service { // Experimental, partially supported room versions let unstable_room_versions = vec![RoomVersionId::V2, RoomVersionId::V3, RoomVersionId::V4, RoomVersionId::V5]; - // 19456 Kib blocks, iterations = 2, parallelism = 1 for more info https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html#argon2id - let argon = Argon2::new( - argon2::Algorithm::Argon2id, - argon2::Version::default(), - argon2::Params::new(19456, 2, 1, None).expect("valid parameters"), - ); - let mut cidr_range_denylist = Vec::new(); for cidr in config.ip_range_denylist.clone() { let cidr = IPAddress::parse(cidr).expect("valid cidr range"); @@ -159,7 +150,6 @@ impl Service { updates_handle: Mutex::new(None), stateres_mutex: Arc::new(Mutex::new(())), rotate: RotationHandler::new(), - argon, }; fs::create_dir_all(s.get_media_folder())?; diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 63293867..11f848ad 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -2,8 +2,7 @@ mod data; use std::sync::Arc; -use argon2::{PasswordHash, PasswordVerifier}; -use conduit::{utils, Error, Result}; +use conduit::{utils, utils::hash, Error, Result}; use data::Data; use ruma::{ api::client::{ @@ -70,15 +69,7 @@ impl Service { // Check if password is correct if let Some(hash) = services().users.password_hash(&user_id)? { - let hash_matches = services() - .globals - .argon - .verify_password( - password.as_bytes(), - &PasswordHash::new(&hash).expect("valid hash in database"), - ) - .is_ok(); - + let hash_matches = hash::verify_password(password, &hash).is_ok(); if !hash_matches { uiaainfo.auth_error = Some(ruma::api::client::error::StandardErrorBody { kind: ErrorKind::forbidden(), diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 1254a988..f2355755 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -1,6 +1,5 @@ use std::{collections::BTreeMap, mem::size_of}; -use argon2::{password_hash::SaltString, PasswordHasher}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, @@ -227,7 +226,7 @@ impl Data for KeyValueDatabase { /// Hash and set the user's password to the Argon2 hash fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { if let Some(password) = password { - if let Ok(hash) = calculate_password_hash(password) { + if let Ok(hash) = utils::hash::password(password) { self.userid_password .insert(user_id.as_bytes(), hash.as_bytes())?; Ok(()) @@ -1021,13 +1020,3 @@ fn get_username_with_valid_password(username: &[u8], password: &[u8]) -> Option< } } } - -/// Calculate a new hash for the given password -fn calculate_password_hash(password: &str) -> Result { - let salt = SaltString::generate(rand::thread_rng()); - services() - .globals - .argon - .hash_password(password.as_bytes(), &salt) - .map(|it| it.to_string()) -} From c9fbbdce1c592cb7d01510b7c0479f23135a4a6c Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 3 Jun 2024 18:07:39 -0400 Subject: [PATCH 0066/2091] csp: remove unusual directives, slight security improvement Signed-off-by: strawberry --- src/router/layers.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index 2b1eaca6..ea872984 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -21,6 +21,10 @@ use tracing::Level; use crate::{request, router}; +const CONDUWUIT_CSP: &str = + "sandbox; default-src 'none'; font-src 'none'; script-src 'none'; frame-ancestors 'none'; base-uri 'none';"; +const CONDUWUIT_PERMISSIONS_POLICY: &str = "interest-cohort=(),browsing-topics=()"; + pub(crate) fn build(server: &Arc) -> io::Result> { let layers = ServiceBuilder::new(); @@ -60,14 +64,11 @@ pub(crate) fn build(server: &Arc) -> io::Result Date: Mon, 3 Jun 2024 23:24:26 -0400 Subject: [PATCH 0067/2091] csp: set `form-action 'none'` Signed-off-by: strawberry --- src/router/layers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index ea872984..38d4ca97 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -21,8 +21,8 @@ use tracing::Level; use crate::{request, router}; -const CONDUWUIT_CSP: &str = - "sandbox; default-src 'none'; font-src 'none'; script-src 'none'; frame-ancestors 'none'; base-uri 'none';"; +const CONDUWUIT_CSP: &str = "sandbox; default-src 'none'; font-src 'none'; script-src 'none'; frame-ancestors 'none'; \ + form-action 'none'; base-uri 'none';"; const CONDUWUIT_PERMISSIONS_POLICY: &str = "interest-cohort=(),browsing-topics=()"; pub(crate) fn build(server: &Arc) -> io::Result> { From 19926ba00dd640dc7d28ba32ee1734da41deff8c Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 3 Jun 2024 18:13:48 -0400 Subject: [PATCH 0068/2091] sort `ALLOWED_INLINE_CONTENT_TYPES` Signed-off-by: strawberry --- src/core/utils/content_disposition.rs | 39 ++++++++++++++------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index 1eafcc4a..be182548 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -7,32 +7,33 @@ const IMAGE_SVG_XML: &str = "image/svg+xml"; /// as defined by MSC2702 const ALLOWED_INLINE_CONTENT_TYPES: [&str; 26] = [ - "text/css", - "text/plain", - "text/csv", + // keep sorted "application/json", "application/ld+json", - "image/jpeg", - "image/gif", - "image/png", - "image/apng", - "image/webp", - "image/avif", - "video/mp4", - "video/webm", - "video/ogg", - "video/quicktime", - "audio/mp4", - "audio/webm", "audio/aac", + "audio/flac", + "audio/mp4", "audio/mpeg", "audio/ogg", - "audio/wave", "audio/wav", - "audio/x-wav", - "audio/x-pn-wav", - "audio/flac", + "audio/wave", + "audio/webm", "audio/x-flac", + "audio/x-pn-wav", + "audio/x-wav", + "image/apng", + "image/avif", + "image/gif", + "image/jpeg", + "image/png", + "image/webp", + "text/css", + "text/csv", + "text/plain", + "video/mp4", + "video/ogg", + "video/quicktime", + "video/webm", ]; /// Returns a Content-Disposition of `attachment` or `inline`, depending on the From df8ba04e317becc42d132644b849fd340877c5a9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 3 Jun 2024 18:52:02 -0400 Subject: [PATCH 0069/2091] media: trust client Content-Type again Signed-off-by: strawberry --- src/api/client_server/media.rs | 32 +++++++-------------------- src/core/utils/content_disposition.rs | 25 --------------------- 2 files changed, 8 insertions(+), 49 deletions(-) diff --git a/src/api/client_server/media.rs b/src/api/client_server/media.rs index 7b492913..84a51f33 100644 --- a/src/api/client_server/media.rs +++ b/src/api/client_server/media.rs @@ -22,9 +22,7 @@ use crate::{ services, utils::{ self, - content_disposition::{ - content_disposition_type, make_content_disposition, make_content_type, sanitise_filename, - }, + content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename}, }, Error, Result, Ruma, RumaResponse, }; @@ -131,8 +129,6 @@ pub(crate) async fn create_content_route( utils::random_string(MXC_LENGTH) ); - let content_type = Some(make_content_type(&body.file, &body.content_type).to_owned()); - services() .media .create( @@ -143,12 +139,12 @@ pub(crate) async fn create_content_route( .map(|filename| { format!( "{}; filename={}", - content_disposition_type(&body.file, &content_type), + content_disposition_type(&body.file, &body.content_type), sanitise_filename(filename.to_owned()) ) }) .as_deref(), - content_type.as_deref(), + body.content_type.as_deref(), &body.file, ) .await?; @@ -193,7 +189,6 @@ pub(crate) async fn get_content_route(body: Ruma) -> R }) = services().media.get(mxc.clone()).await? { let content_disposition = Some(make_content_disposition(&file, &content_type, content_disposition, None)); - let content_type = Some(make_content_type(&file, &content_type).to_owned()); Ok(get_content::v3::Response { file, @@ -222,11 +217,10 @@ pub(crate) async fn get_content_route(body: Ruma) -> R response.content_disposition, None, )); - let content_type = Some(make_content_type(&response.file, &response.content_type).to_owned()); Ok(get_content::v3::Response { file: response.file, - content_type, + content_type: response.content_type, content_disposition, cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()), @@ -279,7 +273,6 @@ pub(crate) async fn get_content_as_filename_route( content_disposition, Some(body.filename.clone()), )); - let content_type = Some(make_content_type(&file, &content_type).to_owned()); Ok(get_content_as_filename::v3::Response { file, @@ -305,13 +298,10 @@ pub(crate) async fn get_content_as_filename_route( remote_content_response.content_disposition, None, )); - let content_type = Some( - make_content_type(&remote_content_response.file, &remote_content_response.content_type).to_owned(), - ); Ok(get_content_as_filename::v3::Response { content_disposition, - content_type, + content_type: remote_content_response.content_type, file: remote_content_response.file, cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), @@ -376,7 +366,6 @@ pub(crate) async fn get_content_thumbnail_route( .await? { let content_disposition = Some(make_content_disposition(&file, &content_type, content_disposition, None)); - let content_type = Some(make_content_type(&file, &content_type).to_owned()); Ok(get_content_thumbnail::v3::Response { file, @@ -434,13 +423,10 @@ pub(crate) async fn get_content_thumbnail_route( get_thumbnail_response.content_disposition, None, )); - let content_type = Some( - make_content_type(&get_thumbnail_response.file, &get_thumbnail_response.content_type).to_owned(), - ); Ok(get_content_thumbnail::v3::Response { file: get_thumbnail_response.file, - content_type, + content_type: get_thumbnail_response.content_type, cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()), content_disposition, @@ -509,22 +495,20 @@ async fn get_remote_content( None, )); - let content_type = Some(make_content_type(&content_response.file, &content_response.content_type).to_owned()); - services() .media .create( None, mxc.to_owned(), content_disposition.as_deref(), - content_type.as_deref(), + content_response.content_type.as_deref(), &content_response.file, ) .await?; Ok(get_content::v3::Response { file: content_response.file, - content_type, + content_type: content_response.content_type, content_disposition, cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()), diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index be182548..90a85c25 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -2,8 +2,6 @@ use crate::debug_info; const ATTACHMENT: &str = "attachment"; const INLINE: &str = "inline"; -const APPLICATION_OCTET_STREAM: &str = "application/octet-stream"; -const IMAGE_SVG_XML: &str = "image/svg+xml"; /// as defined by MSC2702 const ALLOWED_INLINE_CONTENT_TYPES: [&str; 26] = [ @@ -56,29 +54,6 @@ pub fn content_disposition_type(buf: &[u8], content_type: &Option) -> &' } } -/// overrides the Content-Type with what we detected -/// -/// SVG is special-cased due to the MIME type being classified as `text/xml` but -/// browsers need `image/svg+xml` -#[must_use] -#[tracing::instrument(skip(buf))] -pub fn make_content_type(buf: &[u8], content_type: &Option) -> &'static str { - let Some(claimed_content_type) = content_type else { - return APPLICATION_OCTET_STREAM; - }; - - let Some(file_type) = infer::get(buf) else { - debug_info!("Failed to infer the file's contents"); - return APPLICATION_OCTET_STREAM; - }; - - if claimed_content_type.contains("svg") && file_type.mime_type().contains("xml") { - return IMAGE_SVG_XML; - } - - file_type.mime_type() -} - /// sanitises the file name for the Content-Disposition using /// `sanitize_filename` crate #[tracing::instrument] From b781771a9b42b5a3260a2298b6ec59f96cc23269 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 3 Jun 2024 23:14:31 -0400 Subject: [PATCH 0070/2091] media: drop Content-Type detection support Signed-off-by: strawberry --- Cargo.lock | 7 ------- Cargo.toml | 4 ---- src/api/client_server/media.rs | 11 +++-------- src/core/Cargo.toml | 1 - src/core/utils/content_disposition.rs | 26 +++++++++++--------------- 5 files changed, 14 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 78badeb9..b2b00be5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -666,7 +666,6 @@ dependencies = [ "http 1.1.0", "http-body-util", "image", - "infer", "ipaddress", "itertools 0.13.0", "libloading", @@ -1685,12 +1684,6 @@ dependencies = [ "serde", ] -[[package]] -name = "infer" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc150e5ce2330295b8616ce0e3f53250e53af31759a9dbedad1621ba29151847" - [[package]] name = "inlinable_string" version = "0.1.15" diff --git a/Cargo.toml b/Cargo.toml index 22df188d..254b6513 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,10 +28,6 @@ name = "conduit" [workspace.dependencies.sanitize-filename] version = "0.5.0" -[workspace.dependencies.infer] -version = "0.16" -default-features = false - [workspace.dependencies.jsonwebtoken] version = "9.3.0" diff --git a/src/api/client_server/media.rs b/src/api/client_server/media.rs index 84a51f33..f457edfa 100644 --- a/src/api/client_server/media.rs +++ b/src/api/client_server/media.rs @@ -139,7 +139,7 @@ pub(crate) async fn create_content_route( .map(|filename| { format!( "{}; filename={}", - content_disposition_type(&body.file, &body.content_type), + content_disposition_type(&body.content_type), sanitise_filename(filename.to_owned()) ) }) @@ -188,7 +188,7 @@ pub(crate) async fn get_content_route(body: Ruma) -> R content_disposition, }) = services().media.get(mxc.clone()).await? { - let content_disposition = Some(make_content_disposition(&file, &content_type, content_disposition, None)); + let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); Ok(get_content::v3::Response { file, @@ -212,7 +212,6 @@ pub(crate) async fn get_content_route(body: Ruma) -> R })?; let content_disposition = Some(make_content_disposition( - &response.file, &response.content_type, response.content_disposition, None, @@ -268,7 +267,6 @@ pub(crate) async fn get_content_as_filename_route( }) = services().media.get(mxc.clone()).await? { let content_disposition = Some(make_content_disposition( - &file, &content_type, content_disposition, Some(body.filename.clone()), @@ -293,7 +291,6 @@ pub(crate) async fn get_content_as_filename_route( { Ok(remote_content_response) => { let content_disposition = Some(make_content_disposition( - &remote_content_response.file, &remote_content_response.content_type, remote_content_response.content_disposition, None, @@ -365,7 +362,7 @@ pub(crate) async fn get_content_thumbnail_route( ) .await? { - let content_disposition = Some(make_content_disposition(&file, &content_type, content_disposition, None)); + let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); Ok(get_content_thumbnail::v3::Response { file, @@ -418,7 +415,6 @@ pub(crate) async fn get_content_thumbnail_route( .await?; let content_disposition = Some(make_content_disposition( - &get_thumbnail_response.file, &get_thumbnail_response.content_type, get_thumbnail_response.content_disposition, None, @@ -489,7 +485,6 @@ async fn get_remote_content( .await?; let content_disposition = Some(make_content_disposition( - &content_response.file, &content_response.content_type, content_response.content_disposition, None, diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index e725966f..f2ccf43e 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -69,7 +69,6 @@ figment.workspace = true http-body-util.workspace = true http.workspace = true image.workspace = true -infer.workspace = true ipaddress.workspace = true itertools.workspace = true libloading.workspace = true diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index 90a85c25..c9045b32 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -1,5 +1,3 @@ -use crate::debug_info; - const ATTACHMENT: &str = "attachment"; const INLINE: &str = "inline"; @@ -35,19 +33,17 @@ const ALLOWED_INLINE_CONTENT_TYPES: [&str; 26] = [ ]; /// Returns a Content-Disposition of `attachment` or `inline`, depending on the -/// *parsed* contents of the file uploaded via format magic keys using `infer` -/// crate (basically libmagic without needing libmagic). +/// Content-Type against MSC2702 list of safe inline Content-Types +/// (`ALLOWED_INLINE_CONTENT_TYPES`) #[must_use] -#[tracing::instrument(skip(buf))] -pub fn content_disposition_type(buf: &[u8], content_type: &Option) -> &'static str { - let Some(file_type) = infer::get(buf) else { - debug_info!("Failed to infer the file's contents, assuming attachment for Content-Disposition"); +#[tracing::instrument] +pub fn content_disposition_type(content_type: &Option) -> &'static str { + let Some(content_type) = content_type else { + // no Content-Type was given to us, assume attachment return ATTACHMENT; }; - debug_info!("detected MIME type: {}", file_type.mime_type()); - - if ALLOWED_INLINE_CONTENT_TYPES.contains(&file_type.mime_type()) { + if ALLOWED_INLINE_CONTENT_TYPES.contains(&content_type.to_lowercase().as_str()) { INLINE } else { ATTACHMENT @@ -74,9 +70,9 @@ pub fn sanitise_filename(filename: String) -> String { /// `Content-Disposition: attachment/inline; filename=filename.ext` /// /// else: `Content-Disposition: attachment/inline` -#[tracing::instrument(skip(file))] +#[tracing::instrument] pub fn make_content_disposition( - file: &[u8], content_type: &Option, content_disposition: Option, req_filename: Option, + content_type: &Option, content_disposition: Option, req_filename: Option, ) -> String { let filename: String; @@ -98,10 +94,10 @@ pub fn make_content_disposition( if !filename.is_empty() { // Content-Disposition: attachment/inline; filename=filename.ext - format!("{}; filename={}", content_disposition_type(file, content_type), filename) + format!("{}; filename={}", content_disposition_type(content_type), filename) } else { // Content-Disposition: attachment/inline - String::from(content_disposition_type(file, content_type)) + String::from(content_disposition_type(content_type)) } } From bf10ff65a412b3ac1fc1220d44ae351854e0f5d6 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 5 Jun 2024 03:24:24 -0400 Subject: [PATCH 0071/2091] media: ignore Content-Type params, use binary_search Signed-off-by: strawberry --- src/core/utils/content_disposition.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index c9045b32..1c2b066d 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -1,3 +1,5 @@ +use crate::debug_info; + const ATTACHMENT: &str = "attachment"; const INLINE: &str = "inline"; @@ -36,14 +38,26 @@ const ALLOWED_INLINE_CONTENT_TYPES: [&str; 26] = [ /// Content-Type against MSC2702 list of safe inline Content-Types /// (`ALLOWED_INLINE_CONTENT_TYPES`) #[must_use] -#[tracing::instrument] pub fn content_disposition_type(content_type: &Option) -> &'static str { let Some(content_type) = content_type else { - // no Content-Type was given to us, assume attachment + debug_info!("No Content-Type was given, assuming attachment for Content-Disposition"); return ATTACHMENT; }; - if ALLOWED_INLINE_CONTENT_TYPES.contains(&content_type.to_lowercase().as_str()) { + // is_sorted is unstable + /* debug_assert!(ALLOWED_INLINE_CONTENT_TYPES.is_sorted(), + * "ALLOWED_INLINE_CONTENT_TYPES is not sorted"); */ + + let content_type = content_type + .split(';') + .next() + .unwrap_or(content_type) + .to_ascii_lowercase(); + + if ALLOWED_INLINE_CONTENT_TYPES + .binary_search(&content_type.as_str()) + .is_ok() + { INLINE } else { ATTACHMENT @@ -70,7 +84,6 @@ pub fn sanitise_filename(filename: String) -> String { /// `Content-Disposition: attachment/inline; filename=filename.ext` /// /// else: `Content-Disposition: attachment/inline` -#[tracing::instrument] pub fn make_content_disposition( content_type: &Option, content_disposition: Option, req_filename: Option, ) -> String { From c1227340b3d8491b269974e6ee961a0cfb5dad67 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 5 Jun 2024 18:17:46 -0400 Subject: [PATCH 0072/2091] update complement results Signed-off-by: strawberry --- tests/test_results/complement/test_results.jsonl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index ec2d2bdb..51cd57d4 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -147,10 +147,10 @@ {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments"} -{"Action":"fail","Test":"TestMediaWithoutFileName"} -{"Action":"fail","Test":"TestMediaWithoutFileName/parallel"} -{"Action":"fail","Test":"TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_locally"} -{"Action":"fail","Test":"TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_over_federation"} +{"Action":"pass","Test":"TestMediaWithoutFileName"} +{"Action":"pass","Test":"TestMediaWithoutFileName/parallel"} +{"Action":"pass","Test":"TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_locally"} +{"Action":"pass","Test":"TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_over_federation"} {"Action":"pass","Test":"TestMediaWithoutFileName/parallel/Can_upload_without_a_file_name"} {"Action":"fail","Test":"TestNetworkPartitionOrdering"} {"Action":"fail","Test":"TestOutboundFederationIgnoresMissingEventWithBadJSONForRoomVersion6"} From c738c119f85bcdacc52666132746a8073ad6bac8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 4 Jun 2024 01:13:43 -0400 Subject: [PATCH 0073/2091] delete unnecessary `real_users_cache`, fix overwriting push_target iter, add proper function for getting local active users in room this `real_users_cache` cache seems weird, and i have no idea what prompted its creation upstream. perhaps they did this because sqlite was very slow and their rocksdb setup is very poor, so a "solution" was to stick member counts in memory. slow iterators, scanning, etc do not apply to conduwuit where our rocksdb is extremely tuned, and i seriously doubt something like this would have any real world net-positive performance impact. also for some reason, there is suspicious logic where we overwrite the entire push target collection. both of these things could be a potential cause for receiving notifications in rooms we've left. Signed-off-by: strawberry --- src/database/kvdatabase.rs | 4 +-- src/service/globals/data.rs | 9 +---- src/service/rooms/state_cache/data.rs | 49 ++++++++------------------- src/service/rooms/state_cache/mod.rs | 14 ++++---- src/service/rooms/timeline/mod.rs | 8 ++--- src/service/sending/sender.rs | 4 +-- 6 files changed, 30 insertions(+), 58 deletions(-) diff --git a/src/database/kvdatabase.rs b/src/database/kvdatabase.rs index ddbb8e92..906712dc 100644 --- a/src/database/kvdatabase.rs +++ b/src/database/kvdatabase.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashMap}, path::Path, sync::{Arc, Mutex, RwLock}, }; @@ -150,7 +150,6 @@ pub struct KeyValueDatabase { pub senderkey_pusher: Arc, pub auth_chain_cache: Mutex, Arc<[u64]>>>, - pub our_real_users_cache: RwLock>>>, pub appservice_in_room_cache: RwLock>>, pub lasttimelinecount_cache: Mutex>, } @@ -265,7 +264,6 @@ impl KeyValueDatabase { auth_chain_cache: Mutex::new(LruCache::new( (f64::from(config.auth_chain_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, )), - our_real_users_cache: RwLock::new(HashMap::new()), appservice_in_room_cache: RwLock::new(HashMap::new()), lasttimelinecount_cache: Mutex::new(HashMap::new()), }) diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 5b26598b..354ff2f7 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -177,19 +177,16 @@ impl Data for KeyValueDatabase { fn memory_usage(&self) -> String { let auth_chain_cache = self.auth_chain_cache.lock().unwrap().len(); - let our_real_users_cache = self.our_real_users_cache.read().unwrap().len(); let appservice_in_room_cache = self.appservice_in_room_cache.read().unwrap().len(); let lasttimelinecount_cache = self.lasttimelinecount_cache.lock().unwrap().len(); let max_auth_chain_cache = self.auth_chain_cache.lock().unwrap().capacity(); - let max_our_real_users_cache = self.our_real_users_cache.read().unwrap().capacity(); let max_appservice_in_room_cache = self.appservice_in_room_cache.read().unwrap().capacity(); let max_lasttimelinecount_cache = self.lasttimelinecount_cache.lock().unwrap().capacity(); format!( "\ auth_chain_cache: {auth_chain_cache} / {max_auth_chain_cache} -our_real_users_cache: {our_real_users_cache} / {max_our_real_users_cache} appservice_in_room_cache: {appservice_in_room_cache} / {max_appservice_in_room_cache} lasttimelinecount_cache: {lasttimelinecount_cache} / {max_lasttimelinecount_cache}\n\n {}", @@ -203,14 +200,10 @@ lasttimelinecount_cache: {lasttimelinecount_cache} / {max_lasttimelinecount_cach *c = LruCache::new(c.capacity()); } if amount > 2 { - let c = &mut *self.our_real_users_cache.write().unwrap(); - *c = HashMap::new(); - } - if amount > 3 { let c = &mut *self.appservice_in_room_cache.write().unwrap(); *c = HashMap::new(); } - if amount > 4 { + if amount > 3 { let c = &mut *self.lasttimelinecount_cache.lock().unwrap(); *c = HashMap::new(); } diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 08e93d89..5694b22c 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, sync::Arc}; +use std::collections::HashSet; use itertools::Itertools; use ruma::{ @@ -29,8 +29,6 @@ pub trait Data: Send + Sync { fn update_joined_count(&self, room_id: &RoomId) -> Result<()>; - fn get_our_real_users(&self, room_id: &RoomId) -> Result>>; - fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result; /// Makes a user forget a room. @@ -48,6 +46,10 @@ pub trait Data: Send + Sync { /// Returns an iterator over all joined members of a room. fn room_members<'a>(&'a self, room_id: &RoomId) -> Box> + 'a>; + /// Returns a vec of all the users joined in a room who are active + /// (not guests, not deactivated users) + fn active_local_users_in_room(&self, room_id: &RoomId) -> Vec; + fn room_joined_count(&self, room_id: &RoomId) -> Result>; fn room_invited_count(&self, room_id: &RoomId) -> Result>; @@ -225,13 +227,9 @@ impl Data for KeyValueDatabase { let mut joinedcount = 0_u64; let mut invitedcount = 0_u64; let mut joined_servers = HashSet::new(); - let mut real_users = HashSet::new(); for joined in self.room_members(room_id).filter_map(Result::ok) { joined_servers.insert(joined.server_name().to_owned()); - if user_is_local(&joined) && !services().users.is_deactivated(&joined).unwrap_or(true) { - real_users.insert(joined); - } joinedcount = joinedcount.saturating_add(1); } @@ -245,11 +243,6 @@ impl Data for KeyValueDatabase { self.roomid_invitedcount .insert(room_id.as_bytes(), &invitedcount.to_be_bytes())?; - self.our_real_users_cache - .write() - .unwrap() - .insert(room_id.to_owned(), Arc::new(real_users)); - for old_joined_server in self.room_servers(room_id).filter_map(Result::ok) { if !joined_servers.remove(&old_joined_server) { // Server not in room anymore @@ -288,28 +281,6 @@ impl Data for KeyValueDatabase { Ok(()) } - #[tracing::instrument(skip(self, room_id))] - fn get_our_real_users(&self, room_id: &RoomId) -> Result>> { - let maybe = self - .our_real_users_cache - .read() - .unwrap() - .get(room_id) - .cloned(); - if let Some(users) = maybe { - Ok(users) - } else { - self.update_joined_count(room_id)?; - Ok(Arc::clone( - self.our_real_users_cache - .read() - .unwrap() - .get(room_id) - .unwrap(), - )) - } - } - #[tracing::instrument(skip(self, room_id, appservice))] fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { let maybe = self @@ -429,6 +400,16 @@ impl Data for KeyValueDatabase { })) } + /// Returns a vec of all our local users joined in a room who are active + /// (not guests / not deactivated users) + #[tracing::instrument(skip(self))] + fn active_local_users_in_room(&self, room_id: &RoomId) -> Vec { + self.room_members(room_id) + .filter_map(Result::ok) + .filter(|user| user_is_local(user) && !services().users.is_deactivated(user).unwrap_or(true)) + .collect_vec() + } + /// Returns the number of users which are currently in a room #[tracing::instrument(skip(self))] fn room_joined_count(&self, room_id: &RoomId) -> Result> { diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index c9ac278c..ab59c0ad 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, sync::Arc}; +use std::sync::Arc; use data::Data; use itertools::Itertools; @@ -212,11 +212,6 @@ impl Service { #[tracing::instrument(skip(self, room_id))] pub fn update_joined_count(&self, room_id: &RoomId) -> Result<()> { self.db.update_joined_count(room_id) } - #[tracing::instrument(skip(self, room_id))] - pub fn get_our_real_users(&self, room_id: &RoomId) -> Result>> { - self.db.get_our_real_users(room_id) - } - #[tracing::instrument(skip(self, room_id, appservice))] pub fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { self.db.appservice_in_room(room_id, appservice) @@ -278,6 +273,13 @@ impl Service { #[tracing::instrument(skip(self))] pub fn room_joined_count(&self, room_id: &RoomId) -> Result> { self.db.room_joined_count(room_id) } + #[tracing::instrument(skip(self))] + /// Returns a vec of all the users joined in a room who are active + /// (not guests, not deactivated users) + pub fn active_local_users_in_room(&self, room_id: &RoomId) -> Vec { + self.db.active_local_users_in_room(room_id) + } + #[tracing::instrument(skip(self))] pub fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.db.room_invited_count(room_id) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index b45b2a3b..a9482853 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -309,21 +309,19 @@ impl Service { let mut push_target = services() .rooms .state_cache - .get_our_real_users(&pdu.room_id)?; + .active_local_users_in_room(&pdu.room_id); if pdu.kind == TimelineEventType::RoomMember { if let Some(state_key) = &pdu.state_key { let target_user_id = UserId::parse(state_key.clone()).expect("This state_key was previously validated"); if !push_target.contains(&target_user_id) { - let mut target = push_target.as_ref().clone(); - target.insert(target_user_id); - push_target = Arc::new(target); + push_target.push(target_user_id); } } } - for user in push_target.iter() { + for user in &push_target { // Don't notify the user of their own events if user == &pdu.sender { continue; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index c4de3e60..9192fdca 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -485,7 +485,7 @@ async fn send_events_dest_push( .ok_or_else(|| { ( dest.clone(), - Error::bad_database("[Push] Event in servernamevent_datas not found in db."), + Error::bad_database("[Push] Event in servernameevent_data not found in db."), ) })?, ); @@ -567,7 +567,7 @@ async fn send_events_dest_normal( ); ( dest.clone(), - Error::bad_database("[Normal] Event in servernamevent_datas not found in db."), + Error::bad_database("[Normal] Event in servernameevent_data not found in db."), ) })?, ); From 0857fe7907e3d70575e8e2bf4fd43759e831adf4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 5 Jun 2024 01:52:53 -0400 Subject: [PATCH 0074/2091] abstract+add more "users in room" accessors, check membership state on `active_local_joined_users_in_room` `roomuserid_joined` cf seems unreliable, so in the mean time we need to check membership state (or maybe this is a more reliable check anyways) Signed-off-by: strawberry --- src/service/rooms/state_cache/data.rs | 66 +++++++++++++++++++++------ src/service/rooms/state_cache/mod.rs | 23 ++++++++-- src/service/rooms/timeline/mod.rs | 4 +- 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 5694b22c..6f491d89 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use itertools::Itertools; use ruma::{ - events::{AnyStrippedStateEvent, AnySyncStateEvent}, + events::{room::member::MembershipState, AnyStrippedStateEvent, AnySyncStateEvent}, serde::Raw, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; @@ -43,12 +43,23 @@ pub trait Data: Send + Sync { /// know). fn server_rooms<'a>(&'a self, server: &ServerName) -> Box> + 'a>; - /// Returns an iterator over all joined members of a room. + /// Returns an iterator of all joined members of a room. fn room_members<'a>(&'a self, room_id: &RoomId) -> Box> + 'a>; - /// Returns a vec of all the users joined in a room who are active - /// (not guests, not deactivated users) - fn active_local_users_in_room(&self, room_id: &RoomId) -> Vec; + /// Returns an iterator of all our local users + /// in the room, even if they're deactivated/guests + fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a>; + + /// Returns an iterator of all our local users in a room who are active (not + /// deactivated, not guest) + fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a>; + + /// Returns an iterator of all our local users joined in a room who are + /// active (not deactivated, not guest) and have a joined membership state + /// in the room + fn active_local_joined_users_in_room<'a>( + &'a self, room_id: &'a RoomId, + ) -> Box + 'a>; fn room_joined_count(&self, room_id: &RoomId) -> Result>; @@ -381,7 +392,7 @@ impl Data for KeyValueDatabase { })) } - /// Returns an iterator over all joined members of a room. + /// Returns an iterator of all joined members of a room. #[tracing::instrument(skip(self))] fn room_members<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { let mut prefix = room_id.as_bytes().to_vec(); @@ -400,14 +411,43 @@ impl Data for KeyValueDatabase { })) } - /// Returns a vec of all our local users joined in a room who are active - /// (not guests / not deactivated users) + /// Returns an iterator of all our local users in the room, even if they're + /// deactivated/guests + fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a> { + Box::new( + self.room_members(room_id) + .filter_map(Result::ok) + .filter(|user| user_is_local(user)), + ) + } + + /// Returns an iterator of all our local users in a room who are active (not + /// deactivated, not guest) #[tracing::instrument(skip(self))] - fn active_local_users_in_room(&self, room_id: &RoomId) -> Vec { - self.room_members(room_id) - .filter_map(Result::ok) - .filter(|user| user_is_local(user) && !services().users.is_deactivated(user).unwrap_or(true)) - .collect_vec() + fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a> { + Box::new( + self.local_users_in_room(room_id) + .filter(|user| !services().users.is_deactivated(user).unwrap_or(true)), + ) + } + + /// Returns an iterator of all our local users joined in a room who are + /// active (not deactivated, not guest) and have a joined membership state + /// in the room + /// + /// TODO: why is `roomuserid_joined` not reliable? + #[tracing::instrument(skip(self))] + fn active_local_joined_users_in_room<'a>( + &'a self, room_id: &'a RoomId, + ) -> Box + 'a> { + Box::new(self.active_local_users_in_room(room_id).filter(|user_id| { + services() + .rooms + .state_accessor + .get_member(room_id, user_id) + .unwrap_or(None) + .map_or(false, |membership| membership.membership == MembershipState::Join) + })) } /// Returns the number of users which are currently in a room diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index ab59c0ad..3f47ff4c 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -274,12 +274,29 @@ impl Service { pub fn room_joined_count(&self, room_id: &RoomId) -> Result> { self.db.room_joined_count(room_id) } #[tracing::instrument(skip(self))] - /// Returns a vec of all the users joined in a room who are active - /// (not guests, not deactivated users) - pub fn active_local_users_in_room(&self, room_id: &RoomId) -> Vec { + /// Returns an iterator of all our local users in the room, even if they're + /// deactivated/guests + pub fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> impl Iterator + 'a { + self.db.local_users_in_room(room_id) + } + + #[tracing::instrument(skip(self))] + /// Returns an iterator of all our local users in a room who are active (not + /// deactivated, not guest) + pub fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> impl Iterator + 'a { self.db.active_local_users_in_room(room_id) } + #[tracing::instrument(skip(self))] + /// Returns an iterator of all our local users joined in a room who are + /// active (not deactivated, not guest) and have a joined membership state + /// in the room + pub fn active_local_joined_users_in_room<'a>( + &'a self, room_id: &'a RoomId, + ) -> impl Iterator + 'a { + self.db.active_local_joined_users_in_room(room_id) + } + #[tracing::instrument(skip(self))] pub fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.db.room_invited_count(room_id) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index a9482853..34a60507 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -6,6 +6,7 @@ use std::{ }; use data::Data; +use itertools::Itertools; use rand::prelude::SliceRandom; use ruma::{ api::{client::error::ErrorKind, federation}, @@ -309,7 +310,8 @@ impl Service { let mut push_target = services() .rooms .state_cache - .active_local_users_in_room(&pdu.room_id); + .active_local_joined_users_in_room(&pdu.room_id) + .collect_vec(); if pdu.kind == TimelineEventType::RoomMember { if let Some(state_key) = &pdu.state_key { From 38238c309f541ec3850598ea9d3fc7a1fad672bc Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 5 Jun 2024 02:09:31 -0400 Subject: [PATCH 0075/2091] appservices: remove unnecessary `services()` call for `self` Signed-off-by: strawberry --- src/service/appservice/mod.rs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 8970dba8..49faa6f6 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -140,9 +140,7 @@ impl Service { /// Registers an appservice and returns the ID to the caller pub async fn register_appservice(&self, yaml: Registration) -> Result { //TODO: Check for collisions between exclusive appservice namespaces - services() - .appservice - .registration_info + self.registration_info .write() .await .insert(yaml.id.clone(), yaml.clone().try_into()?); @@ -157,9 +155,7 @@ impl Service { /// * `service_name` - the name you send to register the service previously pub async fn unregister_appservice(&self, service_name: &str) -> Result<()> { // removes the appservice registration info - services() - .appservice - .registration_info + self.registration_info .write() .await .remove(service_name) From 3af153f5ae64c0669ff0aa0b7ac7c13df3a14dcd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 5 Jun 2024 04:32:58 +0000 Subject: [PATCH 0076/2091] split s2s into units Signed-off-by: Jason Volk --- src/api/mod.rs | 2 +- src/api/router.rs | 57 +- src/api/server/backfill.rs | 69 + src/api/server/event.rs | 52 + src/api/server/event_auth.rs | 59 + src/api/server/get_missing_events.rs | 86 ++ src/api/server/hierarchy.rs | 21 + src/api/server/invite.rs | 177 +++ src/api/server/key.rs | 76 + src/api/server/make_join.rs | 241 +++ src/api/server/make_leave.rs | 105 ++ src/api/server/mod.rs | 39 + src/api/server/publicrooms.rs | 74 + src/api/server/query.rs | 102 ++ src/api/server/send.rs | 379 +++++ src/api/server/send_join.rs | 300 ++++ src/api/server/send_leave.rs | 186 +++ src/api/server/state.rs | 71 + src/api/server/state_ids.rs | 53 + src/api/server/user.rs | 108 ++ src/api/server/version.rs | 17 + src/api/server/well_known.rs | 17 + src/api/server_server.rs | 2061 -------------------------- 23 files changed, 2260 insertions(+), 2092 deletions(-) create mode 100644 src/api/server/backfill.rs create mode 100644 src/api/server/event.rs create mode 100644 src/api/server/event_auth.rs create mode 100644 src/api/server/get_missing_events.rs create mode 100644 src/api/server/hierarchy.rs create mode 100644 src/api/server/invite.rs create mode 100644 src/api/server/key.rs create mode 100644 src/api/server/make_join.rs create mode 100644 src/api/server/make_leave.rs create mode 100644 src/api/server/mod.rs create mode 100644 src/api/server/publicrooms.rs create mode 100644 src/api/server/query.rs create mode 100644 src/api/server/send.rs create mode 100644 src/api/server/send_join.rs create mode 100644 src/api/server/send_leave.rs create mode 100644 src/api/server/state.rs create mode 100644 src/api/server/state_ids.rs create mode 100644 src/api/server/user.rs create mode 100644 src/api/server/version.rs create mode 100644 src/api/server/well_known.rs delete mode 100644 src/api/server_server.rs diff --git a/src/api/mod.rs b/src/api/mod.rs index 84207d71..a42ba551 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,7 +1,7 @@ pub mod client_server; pub mod router; mod ruma_wrapper; -pub mod server_server; +pub mod server; extern crate conduit_core as conduit; extern crate conduit_service as service; diff --git a/src/api/router.rs b/src/api/router.rs index 90d69873..ad592fc0 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -7,7 +7,7 @@ use conduit::{Error, Server}; use http::Uri; use ruma::api::client::error::ErrorKind; -use crate::{client_server, ruma_wrapper::RouterExt, server_server}; +use crate::{client_server, ruma_wrapper::RouterExt, server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; @@ -188,35 +188,32 @@ pub fn build(router: Router, server: &Server) -> Router { if config.allow_federation { router - .ruma_route(server_server::get_server_version_route) - .route("/_matrix/key/v2/server", get(server_server::get_server_keys_route)) - .route( - "/_matrix/key/v2/server/:key_id", - get(server_server::get_server_keys_deprecated_route), - ) - .ruma_route(server_server::get_public_rooms_route) - .ruma_route(server_server::get_public_rooms_filtered_route) - .ruma_route(server_server::send_transaction_message_route) - .ruma_route(server_server::get_event_route) - .ruma_route(server_server::get_backfill_route) - .ruma_route(server_server::get_missing_events_route) - .ruma_route(server_server::get_event_authorization_route) - .ruma_route(server_server::get_room_state_route) - .ruma_route(server_server::get_room_state_ids_route) - .ruma_route(server_server::create_leave_event_template_route) - .ruma_route(server_server::create_leave_event_v1_route) - .ruma_route(server_server::create_leave_event_v2_route) - .ruma_route(server_server::create_join_event_template_route) - .ruma_route(server_server::create_join_event_v1_route) - .ruma_route(server_server::create_join_event_v2_route) - .ruma_route(server_server::create_invite_route) - .ruma_route(server_server::get_devices_route) - .ruma_route(server_server::get_room_information_route) - .ruma_route(server_server::get_profile_information_route) - .ruma_route(server_server::get_keys_route) - .ruma_route(server_server::claim_keys_route) - .ruma_route(server_server::get_hierarchy_route) - .ruma_route(server_server::well_known_server) + .ruma_route(server::get_server_version_route) + .route("/_matrix/key/v2/server", get(server::get_server_keys_route)) + .route("/_matrix/key/v2/server/:key_id", get(server::get_server_keys_deprecated_route)) + .ruma_route(server::get_public_rooms_route) + .ruma_route(server::get_public_rooms_filtered_route) + .ruma_route(server::send_transaction_message_route) + .ruma_route(server::get_event_route) + .ruma_route(server::get_backfill_route) + .ruma_route(server::get_missing_events_route) + .ruma_route(server::get_event_authorization_route) + .ruma_route(server::get_room_state_route) + .ruma_route(server::get_room_state_ids_route) + .ruma_route(server::create_leave_event_template_route) + .ruma_route(server::create_leave_event_v1_route) + .ruma_route(server::create_leave_event_v2_route) + .ruma_route(server::create_join_event_template_route) + .ruma_route(server::create_join_event_v1_route) + .ruma_route(server::create_join_event_v2_route) + .ruma_route(server::create_invite_route) + .ruma_route(server::get_devices_route) + .ruma_route(server::get_room_information_route) + .ruma_route(server::get_profile_information_route) + .ruma_route(server::get_keys_route) + .ruma_route(server::claim_keys_route) + .ruma_route(server::get_hierarchy_route) + .ruma_route(server::well_known_server) .route("/_conduwuit/local_user_count", get(client_server::conduwuit_local_user_count)) } else { router diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs new file mode 100644 index 00000000..e8da77f2 --- /dev/null +++ b/src/api/server/backfill.rs @@ -0,0 +1,69 @@ +use ruma::{ + api::{client::error::ErrorKind, federation::backfill::get_backfill}, + uint, user_id, MilliSecondsSinceUnixEpoch, +}; + +use crate::{services, Error, PduEvent, Result, Ruma}; + +/// # `GET /_matrix/federation/v1/backfill/` +/// +/// Retrieves events from before the sender joined the room, if the room's +/// history visibility allows. +pub(crate) async fn get_backfill_route(body: Ruma) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + if !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + } + + services() + .rooms + .event_handler + .acl_check(origin, &body.room_id)?; + + let until = body + .v + .iter() + .map(|event_id| services().rooms.timeline.get_pdu_count(event_id)) + .filter_map(|r| r.ok().flatten()) + .max() + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event not found."))?; + + let limit = body + .limit + .min(uint!(100)) + .try_into() + .expect("UInt could not be converted to usize"); + + let all_events = services() + .rooms + .timeline + .pdus_until(user_id!("@doesntmatter:conduit.rs"), &body.room_id, until)? + .take(limit); + + let events = all_events + .filter_map(Result::ok) + .filter(|(_, e)| { + matches!( + services() + .rooms + .state_accessor + .server_can_see_event(origin, &e.room_id, &e.event_id,), + Ok(true), + ) + }) + .map(|(_, pdu)| services().rooms.timeline.get_pdu_json(&pdu.event_id)) + .filter_map(|r| r.ok().flatten()) + .map(PduEvent::convert_to_outgoing_federation_event) + .collect(); + + Ok(get_backfill::v1::Response { + origin: services().globals.server_name().to_owned(), + origin_server_ts: MilliSecondsSinceUnixEpoch::now(), + pdus: events, + }) +} diff --git a/src/api/server/event.rs b/src/api/server/event.rs new file mode 100644 index 00000000..f24530f3 --- /dev/null +++ b/src/api/server/event.rs @@ -0,0 +1,52 @@ +use ruma::{ + api::{client::error::ErrorKind, federation::event::get_event}, + MilliSecondsSinceUnixEpoch, RoomId, +}; + +use crate::{services, Error, PduEvent, Result, Ruma}; + +/// # `GET /_matrix/federation/v1/event/{eventId}` +/// +/// Retrieves a single event from the server. +/// +/// - Only works if a user of this server is currently invited or joined the +/// room +pub(crate) async fn get_event_route(body: Ruma) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + let event = services() + .rooms + .timeline + .get_pdu_json(&body.event_id)? + .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Event not found."))?; + + let room_id_str = event + .get("room_id") + .and_then(|val| val.as_str()) + .ok_or_else(|| Error::bad_database("Invalid event in database."))?; + + let room_id = + <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; + + if !services() + .rooms + .state_cache + .server_in_room(origin, room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + } + + if !services() + .rooms + .state_accessor + .server_can_see_event(origin, room_id, &body.event_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not allowed to see event.")); + } + + Ok(get_event::v1::Response { + origin: services().globals.server_name().to_owned(), + origin_server_ts: MilliSecondsSinceUnixEpoch::now(), + pdu: PduEvent::convert_to_outgoing_federation_event(event), + }) +} diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs new file mode 100644 index 00000000..91770336 --- /dev/null +++ b/src/api/server/event_auth.rs @@ -0,0 +1,59 @@ +use std::sync::Arc; + +use ruma::{ + api::{client::error::ErrorKind, federation::authorization::get_event_authorization}, + RoomId, +}; + +use crate::{services, Error, PduEvent, Result, Ruma}; + +/// # `GET /_matrix/federation/v1/event_auth/{roomId}/{eventId}` +/// +/// Retrieves the auth chain for a given event. +/// +/// - This does not include the event itself +pub(crate) async fn get_event_authorization_route( + body: Ruma, +) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + if !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + } + + services() + .rooms + .event_handler + .acl_check(origin, &body.room_id)?; + + let event = services() + .rooms + .timeline + .get_pdu_json(&body.event_id)? + .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Event not found."))?; + + let room_id_str = event + .get("room_id") + .and_then(|val| val.as_str()) + .ok_or_else(|| Error::bad_database("Invalid event in database."))?; + + let room_id = + <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; + + let auth_chain_ids = services() + .rooms + .auth_chain + .event_ids_iter(room_id, vec![Arc::from(&*body.event_id)]) + .await?; + + Ok(get_event_authorization::v1::Response { + auth_chain: auth_chain_ids + .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok()?) + .map(PduEvent::convert_to_outgoing_federation_event) + .collect(), + }) +} diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs new file mode 100644 index 00000000..1179afe7 --- /dev/null +++ b/src/api/server/get_missing_events.rs @@ -0,0 +1,86 @@ +use ruma::{ + api::{client::error::ErrorKind, federation::event::get_missing_events}, + OwnedEventId, RoomId, +}; + +use crate::{services, Error, PduEvent, Result, Ruma}; + +/// # `POST /_matrix/federation/v1/get_missing_events/{roomId}` +/// +/// Retrieves events that the sender is missing. +pub(crate) async fn get_missing_events_route( + body: Ruma, +) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + if !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room")); + } + + services() + .rooms + .event_handler + .acl_check(origin, &body.room_id)?; + + let limit = body + .limit + .try_into() + .expect("UInt could not be converted to usize"); + + let mut queued_events = body.latest_events.clone(); + // the vec will never have more entries the limit + let mut events = Vec::with_capacity(limit); + + let mut i: usize = 0; + while i < queued_events.len() && events.len() < limit { + if let Some(pdu) = services().rooms.timeline.get_pdu_json(&queued_events[i])? { + let room_id_str = pdu + .get("room_id") + .and_then(|val| val.as_str()) + .ok_or_else(|| Error::bad_database("Invalid event in database."))?; + + let event_room_id = <&RoomId>::try_from(room_id_str) + .map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; + + if event_room_id != body.room_id { + return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event from wrong room.")); + } + + if body.earliest_events.contains(&queued_events[i]) { + i = i.saturating_add(1); + continue; + } + + if !services() + .rooms + .state_accessor + .server_can_see_event(origin, &body.room_id, &queued_events[i])? + { + i = i.saturating_add(1); + continue; + } + + queued_events.extend_from_slice( + &serde_json::from_value::>( + serde_json::to_value( + pdu.get("prev_events") + .cloned() + .ok_or_else(|| Error::bad_database("Event in db has no prev_events property."))?, + ) + .expect("canonical json is valid json value"), + ) + .map_err(|_| Error::bad_database("Invalid prev_events in event in database."))?, + ); + events.push(PduEvent::convert_to_outgoing_federation_event(pdu)); + } + i = i.saturating_add(1); + } + + Ok(get_missing_events::v1::Response { + events, + }) +} diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs new file mode 100644 index 00000000..84b30e80 --- /dev/null +++ b/src/api/server/hierarchy.rs @@ -0,0 +1,21 @@ +use ruma::api::{client::error::ErrorKind, federation::space::get_hierarchy}; + +use crate::{services, Error, Result, Ruma}; + +/// # `GET /_matrix/federation/v1/hierarchy/{roomId}` +/// +/// Gets the space tree in a depth-first manner to locate child rooms of a given +/// space. +pub(crate) async fn get_hierarchy_route(body: Ruma) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + if services().rooms.metadata.exists(&body.room_id)? { + services() + .rooms + .spaces + .get_federation_hierarchy(&body.room_id, origin, body.suggested_only) + .await + } else { + Err(Error::BadRequest(ErrorKind::NotFound, "Room does not exist.")) + } +} diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs new file mode 100644 index 00000000..9ebc60ee --- /dev/null +++ b/src/api/server/invite.rs @@ -0,0 +1,177 @@ +use ruma::{ + api::{client::error::ErrorKind, federation::membership::create_invite}, + events::room::member::{MembershipState, RoomMemberEventContent}, + serde::JsonObject, + CanonicalJsonValue, EventId, OwnedUserId, +}; +use tracing::warn; + +use crate::{ + service::server_is_ours, + services, + utils::{self}, + Error, PduEvent, Result, Ruma, +}; + +/// # `PUT /_matrix/federation/v2/invite/{roomId}/{eventId}` +/// +/// Invites a remote user to a room. +pub(crate) async fn create_invite_route(body: Ruma) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + // ACL check origin + services() + .rooms + .event_handler + .acl_check(origin, &body.room_id)?; + + if !services() + .globals + .supported_room_versions() + .contains(&body.room_version) + { + return Err(Error::BadRequest( + ErrorKind::IncompatibleRoomVersion { + room_version: body.room_version.clone(), + }, + "Server does not support this room version.", + )); + } + + if let Some(server) = body.room_id.server_name() { + if services() + .globals + .config + .forbidden_remote_server_names + .contains(&server.to_owned()) + { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server is banned on this homeserver.", + )); + } + } + + if services() + .globals + .config + .forbidden_remote_server_names + .contains(origin) + { + warn!( + "Received federated/remote invite from banned server {origin} for room ID {}. Rejecting.", + body.room_id + ); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server is banned on this homeserver.", + )); + } + + if let Some(via) = &body.via { + if via.is_empty() { + return Err(Error::BadRequest(ErrorKind::InvalidParam, "via field must not be empty.")); + } + } + + let mut signed_event = utils::to_canonical_object(&body.event) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invite event is invalid."))?; + + let invited_user: OwnedUserId = serde_json::from_value( + signed_event + .get("state_key") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event has no state_key property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "state_key is not a user ID."))?; + + if !server_is_ours(invited_user.server_name()) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "User does not belong to this homeserver.", + )); + } + + // Make sure we're not ACL'ed from their room. + services() + .rooms + .event_handler + .acl_check(invited_user.server_name(), &body.room_id)?; + + ruma::signatures::hash_and_sign_event( + services().globals.server_name().as_str(), + services().globals.keypair(), + &mut signed_event, + &body.room_version, + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; + + // Generate event id + let event_id = EventId::parse(format!( + "${}", + ruma::signatures::reference_hash(&signed_event, &body.room_version) + .expect("ruma can calculate reference hashes") + )) + .expect("ruma's reference hashes are valid event ids"); + + // Add event_id back + signed_event.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.to_string())); + + let sender: OwnedUserId = serde_json::from_value( + signed_event + .get("sender") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event had no sender property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "sender is not a user ID."))?; + + if services().rooms.metadata.is_banned(&body.room_id)? && !services().users.is_admin(&invited_user)? { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "This room is banned on this homeserver.", + )); + } + + if services().globals.block_non_admin_invites() && !services().users.is_admin(&invited_user)? { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "This server does not allow room invites.", + )); + } + + let mut invite_state = body.invite_room_state.clone(); + + let mut event: JsonObject = serde_json::from_str(body.event.get()) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event bytes."))?; + + event.insert("event_id".to_owned(), "$placeholder".into()); + + let pdu: PduEvent = serde_json::from_value(event.into()) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event."))?; + + invite_state.push(pdu.to_stripped_state_event()); + + // If we are active in the room, the remote server will notify us about the join + // via /send + if !services() + .rooms + .state_cache + .server_in_room(services().globals.server_name(), &body.room_id)? + { + services().rooms.state_cache.update_membership( + &body.room_id, + &invited_user, + RoomMemberEventContent::new(MembershipState::Invite), + &sender, + Some(invite_state), + body.via.clone(), + true, + )?; + } + + Ok(create_invite::v2::Response { + event: PduEvent::convert_to_outgoing_federation_event(signed_event), + }) +} diff --git a/src/api/server/key.rs b/src/api/server/key.rs new file mode 100644 index 00000000..ddc8e8ef --- /dev/null +++ b/src/api/server/key.rs @@ -0,0 +1,76 @@ +use std::{ + collections::BTreeMap, + time::{Duration, SystemTime}, +}; + +use axum::{response::IntoResponse, Json}; +use ruma::{ + api::{ + federation::discovery::{get_server_keys, ServerSigningKeys, VerifyKey}, + OutgoingResponse, + }, + serde::{Base64, Raw}, + MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, +}; + +use crate::{services, Result}; + +/// # `GET /_matrix/key/v2/server` +/// +/// Gets the public signing keys of this server. +/// +/// - Matrix does not support invalidating public keys, so the key returned by +/// this will be valid +/// forever. +// Response type for this endpoint is Json because we need to calculate a +// signature for the response +pub(crate) async fn get_server_keys_route() -> Result { + let verify_keys: BTreeMap = BTreeMap::from([( + format!("ed25519:{}", services().globals.keypair().version()) + .try_into() + .expect("found invalid server signing keys in DB"), + VerifyKey { + key: Base64::new(services().globals.keypair().public_key().to_vec()), + }, + )]); + + let mut response = serde_json::from_slice( + get_server_keys::v2::Response { + server_key: Raw::new(&ServerSigningKeys { + server_name: services().globals.server_name().to_owned(), + verify_keys, + old_verify_keys: BTreeMap::new(), + signatures: BTreeMap::new(), + valid_until_ts: MilliSecondsSinceUnixEpoch::from_system_time( + SystemTime::now() + .checked_add(Duration::from_secs(86400 * 7)) + .expect("valid_until_ts should not get this high"), + ) + .expect("time is valid"), + }) + .expect("static conversion, no errors"), + } + .try_into_http_response::>() + .unwrap() + .body(), + ) + .unwrap(); + + ruma::signatures::sign_json( + services().globals.server_name().as_str(), + services().globals.keypair(), + &mut response, + ) + .unwrap(); + + Ok(Json(response)) +} + +/// # `GET /_matrix/key/v2/server/{keyId}` +/// +/// Gets the public signing keys of this server. +/// +/// - Matrix does not support invalidating public keys, so the key returned by +/// this will be valid +/// forever. +pub(crate) async fn get_server_keys_deprecated_route() -> impl IntoResponse { get_server_keys_route().await } diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs new file mode 100644 index 00000000..0a6b1992 --- /dev/null +++ b/src/api/server/make_join.rs @@ -0,0 +1,241 @@ +use std::sync::Arc; + +use ruma::{ + api::{client::error::ErrorKind, federation::membership::prepare_join_event}, + events::{ + room::{ + join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + }, + StateEventType, TimelineEventType, + }, + RoomVersionId, +}; +use serde_json::value::to_raw_value; +use tracing::warn; + +use crate::{ + service::{pdu::PduBuilder, user_is_local}, + services, Error, Result, Ruma, +}; + +/// # `GET /_matrix/federation/v1/make_join/{roomId}/{userId}` +/// +/// Creates a join template. +pub(crate) async fn create_join_event_template_route( + body: Ruma, +) -> Result { + if !services().rooms.metadata.exists(&body.room_id)? { + return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + } + + let origin = body.origin.as_ref().expect("server is authenticated"); + if body.user_id.server_name() != origin { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to join on behalf of another server/user", + )); + } + + // ACL check origin server + services() + .rooms + .event_handler + .acl_check(origin, &body.room_id)?; + + if services() + .globals + .config + .forbidden_remote_server_names + .contains(origin) + { + warn!( + "Server {origin} for remote user {} tried joining room ID {} which has a server name that is globally \ + forbidden. Rejecting.", + &body.user_id, &body.room_id, + ); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server is banned on this homeserver.", + )); + } + + if let Some(server) = body.room_id.server_name() { + if services() + .globals + .config + .forbidden_remote_server_names + .contains(&server.to_owned()) + { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server is banned on this homeserver.", + )); + } + } + + let mutex_state = Arc::clone( + services() + .globals + .roomid_mutex_state + .write() + .await + .entry(body.room_id.clone()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + let join_rules_event = + services() + .rooms + .state_accessor + .room_state_get(&body.room_id, &StateEventType::RoomJoinRules, "")?; + + let join_rules_event_content: Option = join_rules_event + .as_ref() + .map(|join_rules_event| { + serde_json::from_str(join_rules_event.content.get()) + .map_err(|_| Error::bad_database("Invalid join rules event in db.")) + }) + .transpose()?; + + let join_authorized_via_users_server = if let Some(join_rules_event_content) = join_rules_event_content { + if let JoinRule::Restricted(r) | JoinRule::KnockRestricted(r) = join_rules_event_content.join_rule { + if r.allow + .iter() + .filter_map(|rule| { + if let AllowRule::RoomMembership(membership) = rule { + Some(membership) + } else { + None + } + }) + .any(|m| { + services() + .rooms + .state_cache + .is_joined(&body.user_id, &m.room_id) + .unwrap_or(false) + }) { + if services() + .rooms + .state_cache + .is_left(&body.user_id, &body.room_id) + .unwrap_or(true) + { + let members: Vec<_> = services() + .rooms + .state_cache + .room_members(&body.room_id) + .filter_map(Result::ok) + .filter(|user| user_is_local(user)) + .collect(); + + let mut auth_user = None; + + for user in members { + if services() + .rooms + .state_accessor + .user_can_invite(&body.room_id, &user, &body.user_id, &state_lock) + .await + .unwrap_or(false) + { + auth_user = Some(user); + break; + } + } + if auth_user.is_some() { + auth_user + } else { + return Err(Error::BadRequest( + ErrorKind::UnableToGrantJoin, + "No user on this server is able to assist in joining.", + )); + } + } else { + // If the user has any state other than leave, either: + // - the auth_check will deny them (ban, knock - (until/unless MSC4123 is + // merged)) + // - they are able to join via other methods (invite) + // - they are already in the room (join) + None + } + } else { + return Err(Error::BadRequest( + ErrorKind::UnableToAuthorizeJoin, + "User is not known to be in any required room.", + )); + } + } else { + None + } + } else { + None + }; + + let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; + if !body.ver.contains(&room_version_id) { + return Err(Error::BadRequest( + ErrorKind::IncompatibleRoomVersion { + room_version: room_version_id, + }, + "Room version not supported.", + )); + } + + let content = to_raw_value(&RoomMemberEventContent { + avatar_url: None, + blurhash: None, + displayname: None, + is_direct: None, + membership: MembershipState::Join, + third_party_invite: None, + reason: None, + join_authorized_via_users_server, + }) + .expect("member event is valid value"); + + let (_pdu, mut pdu_json) = services().rooms.timeline.create_hash_and_sign_event( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content, + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &body.user_id, + &body.room_id, + &state_lock, + )?; + + drop(state_lock); + + // room v3 and above removed the "event_id" field from remote PDU format + match room_version_id { + RoomVersionId::V1 | RoomVersionId::V2 => {}, + RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + | RoomVersionId::V7 + | RoomVersionId::V8 + | RoomVersionId::V9 + | RoomVersionId::V10 + | RoomVersionId::V11 => { + pdu_json.remove("event_id"); + }, + _ => { + warn!("Unexpected or unsupported room version {room_version_id}"); + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + }; + + Ok(prepare_join_event::v1::Response { + room_version: Some(room_version_id), + event: to_raw_value(&pdu_json).expect("CanonicalJson can be serialized to JSON"), + }) +} diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs new file mode 100644 index 00000000..72c931c5 --- /dev/null +++ b/src/api/server/make_leave.rs @@ -0,0 +1,105 @@ +use std::sync::Arc; + +use ruma::{ + api::{client::error::ErrorKind, federation::membership::prepare_leave_event}, + events::{ + room::member::{MembershipState, RoomMemberEventContent}, + TimelineEventType, + }, + RoomVersionId, +}; +use serde_json::value::to_raw_value; + +use crate::{service::pdu::PduBuilder, services, Error, Result, Ruma}; + +/// # `PUT /_matrix/federation/v1/make_leave/{roomId}/{eventId}` +/// +/// Creates a leave template. +pub(crate) async fn create_leave_event_template_route( + body: Ruma, +) -> Result { + if !services().rooms.metadata.exists(&body.room_id)? { + return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + } + + let origin = body.origin.as_ref().expect("server is authenticated"); + if body.user_id.server_name() != origin { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to leave on behalf of another server/user", + )); + } + + // ACL check origin + services() + .rooms + .event_handler + .acl_check(origin, &body.room_id)?; + + let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; + + let mutex_state = Arc::clone( + services() + .globals + .roomid_mutex_state + .write() + .await + .entry(body.room_id.clone()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + let content = to_raw_value(&RoomMemberEventContent { + avatar_url: None, + blurhash: None, + displayname: None, + is_direct: None, + membership: MembershipState::Leave, + third_party_invite: None, + reason: None, + join_authorized_via_users_server: None, + }) + .expect("member event is valid value"); + + let (_pdu, mut pdu_json) = services().rooms.timeline.create_hash_and_sign_event( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content, + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + }, + &body.user_id, + &body.room_id, + &state_lock, + )?; + + drop(state_lock); + + // room v3 and above removed the "event_id" field from remote PDU format + match room_version_id { + RoomVersionId::V1 | RoomVersionId::V2 => {}, + RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + | RoomVersionId::V7 + | RoomVersionId::V8 + | RoomVersionId::V9 + | RoomVersionId::V10 + | RoomVersionId::V11 => { + pdu_json.remove("event_id"); + }, + _ => { + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + }; + + Ok(prepare_leave_event::v1::Response { + room_version: Some(room_version_id), + event: to_raw_value(&pdu_json).expect("CanonicalJson can be serialized to JSON"), + }) +} diff --git a/src/api/server/mod.rs b/src/api/server/mod.rs new file mode 100644 index 00000000..dce9c6ff --- /dev/null +++ b/src/api/server/mod.rs @@ -0,0 +1,39 @@ +pub(super) mod backfill; +pub(super) mod event; +pub(super) mod event_auth; +pub(super) mod get_missing_events; +pub(super) mod hierarchy; +pub(super) mod invite; +pub(super) mod key; +pub(super) mod make_join; +pub(super) mod make_leave; +pub(super) mod publicrooms; +pub(super) mod query; +pub(super) mod send; +pub(super) mod send_join; +pub(super) mod send_leave; +pub(super) mod state; +pub(super) mod state_ids; +pub(super) mod user; +pub(super) mod version; +pub(super) mod well_known; + +pub(super) use backfill::*; +pub(super) use event::*; +pub(super) use event_auth::*; +pub(super) use get_missing_events::*; +pub(super) use hierarchy::*; +pub(super) use invite::*; +pub(super) use key::*; +pub(super) use make_join::*; +pub(super) use make_leave::*; +pub(super) use publicrooms::*; +pub(super) use query::*; +pub(super) use send::*; +pub(super) use send_join::*; +pub(super) use send_leave::*; +pub(super) use state::*; +pub(super) use state_ids::*; +pub(super) use user::*; +pub(super) use version::*; +pub(super) use well_known::*; diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs new file mode 100644 index 00000000..2a1a2ab9 --- /dev/null +++ b/src/api/server/publicrooms.rs @@ -0,0 +1,74 @@ +use ruma::{ + api::{ + client::error::ErrorKind, + federation::directory::{get_public_rooms, get_public_rooms_filtered}, + }, + directory::Filter, +}; + +use crate::{ + client_server::{self}, + services, Error, Result, Ruma, +}; + +/// # `POST /_matrix/federation/v1/publicRooms` +/// +/// Lists the public rooms on this server. +pub(crate) async fn get_public_rooms_filtered_route( + body: Ruma, +) -> Result { + if !services() + .globals + .allow_public_room_directory_over_federation() + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Room directory is not public")); + } + + let response = client_server::get_public_rooms_filtered_helper( + None, + body.limit, + body.since.as_deref(), + &body.filter, + &body.room_network, + ) + .await + .map_err(|_| Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list."))?; + + Ok(get_public_rooms_filtered::v1::Response { + chunk: response.chunk, + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, + }) +} + +/// # `GET /_matrix/federation/v1/publicRooms` +/// +/// Lists the public rooms on this server. +pub(crate) async fn get_public_rooms_route( + body: Ruma, +) -> Result { + if !services() + .globals + .allow_public_room_directory_over_federation() + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Room directory is not public")); + } + + let response = client_server::get_public_rooms_filtered_helper( + None, + body.limit, + body.since.as_deref(), + &Filter::default(), + &body.room_network, + ) + .await + .map_err(|_| Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list."))?; + + Ok(get_public_rooms::v1::Response { + chunk: response.chunk, + prev_batch: response.prev_batch, + next_batch: response.next_batch, + total_room_count_estimate: response.total_room_count_estimate, + }) +} diff --git a/src/api/server/query.rs b/src/api/server/query.rs new file mode 100644 index 00000000..89baedb3 --- /dev/null +++ b/src/api/server/query.rs @@ -0,0 +1,102 @@ +use get_profile_information::v1::ProfileField; +use rand::seq::SliceRandom; +use ruma::{ + api::{ + client::error::ErrorKind, + federation::query::{get_profile_information, get_room_information}, + }, + OwnedServerName, +}; + +use crate::{service::server_is_ours, services, Error, Result, Ruma}; + +/// # `GET /_matrix/federation/v1/query/directory` +/// +/// Resolve a room alias to a room id. +pub(crate) async fn get_room_information_route( + body: Ruma, +) -> Result { + let room_id = services() + .rooms + .alias + .resolve_local_alias(&body.room_alias)? + .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Room alias not found."))?; + + let mut servers: Vec = services() + .rooms + .state_cache + .room_servers(&room_id) + .filter_map(Result::ok) + .collect(); + + servers.sort_unstable(); + servers.dedup(); + + servers.shuffle(&mut rand::thread_rng()); + + // insert our server as the very first choice if in list + if let Some(server_index) = servers + .iter() + .position(|server| server == services().globals.server_name()) + { + servers.swap_remove(server_index); + servers.insert(0, services().globals.server_name().to_owned()); + } + + Ok(get_room_information::v1::Response { + room_id, + servers, + }) +} + +/// # `GET /_matrix/federation/v1/query/profile` +/// +/// +/// Gets information on a profile. +pub(crate) async fn get_profile_information_route( + body: Ruma, +) -> Result { + if !services() + .globals + .allow_profile_lookup_federation_requests() + { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Profile lookup over federation is not allowed on this homeserver.", + )); + } + + if !server_is_ours(body.user_id.server_name()) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "User does not belong to this server.", + )); + } + + let mut displayname = None; + let mut avatar_url = None; + let mut blurhash = None; + + match &body.field { + Some(ProfileField::DisplayName) => { + displayname = services().users.displayname(&body.user_id)?; + }, + Some(ProfileField::AvatarUrl) => { + avatar_url = services().users.avatar_url(&body.user_id)?; + blurhash = services().users.blurhash(&body.user_id)?; + }, + // TODO: what to do with custom + Some(_) => {}, + None => { + displayname = services().users.displayname(&body.user_id)?; + avatar_url = services().users.avatar_url(&body.user_id)?; + blurhash = services().users.blurhash(&body.user_id)?; + }, + } + + Ok(get_profile_information::v1::Response { + displayname, + avatar_url, + blurhash, + }) +} diff --git a/src/api/server/send.rs b/src/api/server/send.rs new file mode 100644 index 00000000..592eb4f9 --- /dev/null +++ b/src/api/server/send.rs @@ -0,0 +1,379 @@ +use std::{collections::BTreeMap, sync::Arc, time::Instant}; + +use conduit::debug_warn; +use ruma::{ + api::{ + client::error::ErrorKind, + federation::transactions::{ + edu::{DeviceListUpdateContent, DirectDeviceContent, Edu, SigningKeyUpdateContent}, + send_transaction_message, + }, + }, + events::receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType}, + to_device::DeviceIdOrAllDevices, +}; +use tokio::sync::RwLock; +use tracing::{debug, error, trace, warn}; + +use crate::{ + service::rooms::event_handler::parse_incoming_pdu, + services, + utils::{self}, + Error, Result, Ruma, +}; + +/// # `PUT /_matrix/federation/v1/send/{txnId}` +/// +/// Push EDUs and PDUs to this server. +pub(crate) async fn send_transaction_message_route( + body: Ruma, +) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + if *origin != body.body.origin { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Not allowed to send transactions on behalf of other servers", + )); + } + + if body.pdus.len() > 50_usize { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Not allowed to send more than 50 PDUs in one transaction", + )); + } + + if body.edus.len() > 100_usize { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Not allowed to send more than 100 EDUs in one transaction", + )); + } + + // This is all the auth_events that have been recursively fetched so they don't + // have to be deserialized over and over again. + // TODO: make this persist across requests but not in a DB Tree (in globals?) + // TODO: This could potentially also be some sort of trie (suffix tree) like + // structure so that once an auth event is known it would know (using indexes + // maybe) all of the auth events that it references. + // let mut auth_cache = EventMap::new(); + + let txn_start_time = Instant::now(); + let mut parsed_pdus = Vec::with_capacity(body.pdus.len()); + for pdu in &body.pdus { + parsed_pdus.push(match parse_incoming_pdu(pdu) { + Ok(t) => t, + Err(e) => { + debug_warn!("Could not parse PDU: {e}"); + continue; + }, + }); + + // We do not add the event_id field to the pdu here because of signature + // and hashes checks + } + + trace!( + pdus = ?parsed_pdus.len(), + edus = ?body.edus.len(), + elapsed = ?txn_start_time.elapsed(), + id = ?body.transaction_id, + origin =?body.origin, + "Starting txn", + ); + + // We go through all the signatures we see on the PDUs and fetch the + // corresponding signing keys + let pub_key_map = RwLock::new(BTreeMap::new()); + if !parsed_pdus.is_empty() { + services() + .rooms + .event_handler + .fetch_required_signing_keys(parsed_pdus.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) + .await + .unwrap_or_else(|e| { + warn!("Could not fetch all signatures for PDUs from {origin}: {:?}", e); + }); + + debug!( + elapsed = ?txn_start_time.elapsed(), + "Fetched signing keys" + ); + } + + let mut resolved_map = BTreeMap::new(); + for (event_id, value, room_id) in parsed_pdus { + let pdu_start_time = Instant::now(); + let mutex = Arc::clone( + services() + .globals + .roomid_mutex_federation + .write() + .await + .entry(room_id.clone()) + .or_default(), + ); + let mutex_lock = mutex.lock().await; + resolved_map.insert( + event_id.clone(), + services() + .rooms + .event_handler + .handle_incoming_pdu(origin, &room_id, &event_id, value, true, &pub_key_map) + .await + .map(|_| ()), + ); + drop(mutex_lock); + + debug!( + pdu_elapsed = ?pdu_start_time.elapsed(), + txn_elapsed = ?txn_start_time.elapsed(), + "Finished PDU {event_id}", + ); + } + + for pdu in &resolved_map { + if let Err(e) = pdu.1 { + if matches!(e, Error::BadRequest(ErrorKind::NotFound, _)) { + warn!("Incoming PDU failed {:?}", pdu); + } + } + } + + for edu in body + .edus + .iter() + .filter_map(|edu| serde_json::from_str::(edu.json().get()).ok()) + { + match edu { + Edu::Presence(presence) => { + if !services().globals.allow_incoming_presence() { + continue; + } + + for update in presence.push { + if update.user_id.server_name() != origin { + debug_warn!(%update.user_id, %origin, "received presence EDU for user not belonging to origin"); + continue; + } + + services().presence.set_presence( + &update.user_id, + &update.presence, + Some(update.currently_active), + Some(update.last_active_ago), + update.status_msg.clone(), + )?; + } + }, + Edu::Receipt(receipt) => { + if !services().globals.allow_incoming_read_receipts() { + continue; + } + + for (room_id, room_updates) in receipt.receipts { + if services() + .rooms + .event_handler + .acl_check(origin, &room_id) + .is_err() + { + debug_warn!(%origin, %room_id, "received read receipt EDU from ACL'd server"); + continue; + } + + for (user_id, user_updates) in room_updates.read { + if user_id.server_name() != origin { + debug_warn!(%user_id, %origin, "received read receipt EDU for user not belonging to origin"); + continue; + } + + if services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(Result::ok) + .any(|member| member.server_name() == user_id.server_name()) + { + for event_id in &user_updates.event_ids { + let user_receipts = BTreeMap::from([(user_id.clone(), user_updates.data.clone())]); + + let receipts = BTreeMap::from([(ReceiptType::Read, user_receipts)]); + + let receipt_content = BTreeMap::from([(event_id.to_owned(), receipts)]); + + let event = ReceiptEvent { + content: ReceiptEventContent(receipt_content), + room_id: room_id.clone(), + }; + + services() + .rooms + .read_receipt + .readreceipt_update(&user_id, &room_id, event)?; + } + } else { + debug_warn!(%user_id, %room_id, %origin, "received read receipt EDU from server who does not have a single member from their server in the room"); + continue; + } + } + } + }, + Edu::Typing(typing) => { + if !services().globals.config.allow_incoming_typing { + continue; + } + + if typing.user_id.server_name() != origin { + debug_warn!(%typing.user_id, %origin, "received typing EDU for user not belonging to origin"); + continue; + } + + if services() + .rooms + .event_handler + .acl_check(typing.user_id.server_name(), &typing.room_id) + .is_err() + { + debug_warn!(%typing.user_id, %typing.room_id, %origin, "received typing EDU for ACL'd user's server"); + continue; + } + + if services() + .rooms + .state_cache + .is_joined(&typing.user_id, &typing.room_id)? + { + if typing.typing { + let timeout = utils::millis_since_unix_epoch().saturating_add( + services() + .globals + .config + .typing_federation_timeout_s + .saturating_mul(1000), + ); + services() + .rooms + .typing + .typing_add(&typing.user_id, &typing.room_id, timeout) + .await?; + } else { + services() + .rooms + .typing + .typing_remove(&typing.user_id, &typing.room_id) + .await?; + } + } else { + debug_warn!(%typing.user_id, %typing.room_id, %origin, "received typing EDU for user not in room"); + continue; + } + }, + Edu::DeviceListUpdate(DeviceListUpdateContent { + user_id, + .. + }) => { + if user_id.server_name() != origin { + debug_warn!(%user_id, %origin, "received device list update EDU for user not belonging to origin"); + continue; + } + + services().users.mark_device_key_update(&user_id)?; + }, + Edu::DirectToDevice(DirectDeviceContent { + sender, + ev_type, + message_id, + messages, + }) => { + if sender.server_name() != origin { + debug_warn!(%sender, %origin, "received direct to device EDU for user not belonging to origin"); + continue; + } + + // Check if this is a new transaction id + if services() + .transaction_ids + .existing_txnid(&sender, None, &message_id)? + .is_some() + { + continue; + } + + for (target_user_id, map) in &messages { + for (target_device_id_maybe, event) in map { + match target_device_id_maybe { + DeviceIdOrAllDevices::DeviceId(target_device_id) => { + services().users.add_to_device_event( + &sender, + target_user_id, + target_device_id, + &ev_type.to_string(), + event.deserialize_as().map_err(|e| { + error!("To-Device event is invalid: {event:?} {e}"); + Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") + })?, + )?; + }, + + DeviceIdOrAllDevices::AllDevices => { + for target_device_id in services().users.all_device_ids(target_user_id) { + services().users.add_to_device_event( + &sender, + target_user_id, + &target_device_id?, + &ev_type.to_string(), + event.deserialize_as().map_err(|_| { + Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") + })?, + )?; + } + }, + } + } + } + + // Save transaction id with empty data + services() + .transaction_ids + .add_txnid(&sender, None, &message_id, &[])?; + }, + Edu::SigningKeyUpdate(SigningKeyUpdateContent { + user_id, + master_key, + self_signing_key, + }) => { + if user_id.server_name() != origin { + debug_warn!(%user_id, %origin, "received signing key update EDU from server that does not belong to user's server"); + continue; + } + + if let Some(master_key) = master_key { + services() + .users + .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true)?; + } + }, + Edu::_Custom(custom) => { + debug_warn!(?custom, "received custom/unknown EDU"); + }, + } + } + + debug!( + pdus = ?body.pdus.len(), + edus = ?body.edus.len(), + elapsed = ?txn_start_time.elapsed(), + id = ?body.transaction_id, + origin =?body.origin, + "Finished txn", + ); + + Ok(send_transaction_message::v1::Response { + pdus: resolved_map + .into_iter() + .map(|(e, r)| (e, r.map_err(|e| e.sanitized_error()))) + .collect(), + }) +} diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs new file mode 100644 index 00000000..18d075b4 --- /dev/null +++ b/src/api/server/send_join.rs @@ -0,0 +1,300 @@ +#![allow(deprecated)] + +use std::{collections::BTreeMap, sync::Arc}; + +use ruma::{ + api::{client::error::ErrorKind, federation::membership::create_join_event}, + events::{room::member::MembershipState, StateEventType}, + CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, +}; +use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use tokio::sync::RwLock; +use tracing::warn; + +use crate::{service::pdu::gen_event_id_canonical_json, services, Error, PduEvent, Result, Ruma}; + +/// helper method for /send_join v1 and v2 +async fn create_join_event( + origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, +) -> Result { + if !services().rooms.metadata.exists(room_id)? { + return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + } + + // ACL check origin server + services().rooms.event_handler.acl_check(origin, room_id)?; + + // We need to return the state prior to joining, let's keep a reference to that + // here + let shortstatehash = services() + .rooms + .state + .get_room_shortstatehash(room_id)? + .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Event state not found."))?; + + let pub_key_map = RwLock::new(BTreeMap::new()); + // let mut auth_cache = EventMap::new(); + + // We do not add the event_id field to the pdu here because of signature and + // hashes checks + let room_version_id = services().rooms.state.get_room_version(room_id)?; + + let Ok((event_id, mut value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { + // Event could not be converted to canonical json + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Could not convert event to canonical json.", + )); + }; + + let event_type: StateEventType = serde_json::from_value( + value + .get("type") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event has invalid event type."))?; + + if event_type != StateEventType::RoomMember { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to send non-membership state event to join endpoint.", + )); + } + + let content = value + .get("content") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property."))? + .as_object() + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid."))?; + + let membership: MembershipState = serde_json::from_value( + content + .get("membership") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event content missing membership property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Event has an invalid membership state."))?; + + if membership != MembershipState::Join { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to send a non-join membership event to join endpoint.", + )); + } + + // ACL check sender server name + let sender: OwnedUserId = serde_json::from_value( + value + .get("sender") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "sender is not a valid user ID."))?; + + services() + .rooms + .event_handler + .acl_check(sender.server_name(), room_id)?; + + // check if origin server is trying to send for another server + if sender.server_name() != origin { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to join on behalf of another server.", + )); + } + + let state_key: OwnedUserId = serde_json::from_value( + value + .get("state_key") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID."))?; + + if state_key != sender { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "State key does not match sender user", + )); + } + + ruma::signatures::hash_and_sign_event( + services().globals.server_name().as_str(), + services().globals.keypair(), + &mut value, + &room_version_id, + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; + + let origin: OwnedServerName = serde_json::from_value( + serde_json::to_value( + value + .get("origin") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?, + ) + .expect("CanonicalJson is valid json value"), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; + + services() + .rooms + .event_handler + .fetch_required_signing_keys([&value], &pub_key_map) + .await?; + + let mutex = Arc::clone( + services() + .globals + .roomid_mutex_federation + .write() + .await + .entry(room_id.to_owned()) + .or_default(), + ); + let mutex_lock = mutex.lock().await; + let pdu_id: Vec = services() + .rooms + .event_handler + .handle_incoming_pdu(&origin, room_id, &event_id, value.clone(), true, &pub_key_map) + .await? + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; + drop(mutex_lock); + + let state_ids = services() + .rooms + .state_accessor + .state_full_ids(shortstatehash) + .await?; + let auth_chain_ids = services() + .rooms + .auth_chain + .event_ids_iter(room_id, state_ids.values().cloned().collect()) + .await?; + + services().sending.send_pdu_room(room_id, &pdu_id)?; + + Ok(create_join_event::v1::RoomState { + auth_chain: auth_chain_ids + .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok().flatten()) + .map(PduEvent::convert_to_outgoing_federation_event) + .collect(), + state: state_ids + .iter() + .filter_map(|(_, id)| services().rooms.timeline.get_pdu_json(id).ok().flatten()) + .map(PduEvent::convert_to_outgoing_federation_event) + .collect(), + // Event field is required if the room version supports restricted join rules. + event: Some( + to_raw_value(&CanonicalJsonValue::Object(value.clone())) + .expect("To raw json should not fail since only change was adding signature"), + ), + }) +} + +/// # `PUT /_matrix/federation/v1/send_join/{roomId}/{eventId}` +/// +/// Submits a signed join event. +pub(crate) async fn create_join_event_v1_route( + body: Ruma, +) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + if services() + .globals + .config + .forbidden_remote_server_names + .contains(origin) + { + warn!( + "Server {origin} tried joining room ID {} who has a server name that is globally forbidden. Rejecting.", + &body.room_id, + ); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server is banned on this homeserver.", + )); + } + + if let Some(server) = body.room_id.server_name() { + if services() + .globals + .config + .forbidden_remote_server_names + .contains(&server.to_owned()) + { + warn!( + "Server {origin} tried joining room ID {} which has a server name that is globally forbidden. \ + Rejecting.", + &body.room_id, + ); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server is banned on this homeserver.", + )); + } + } + + let room_state = create_join_event(origin, &body.room_id, &body.pdu).await?; + + Ok(create_join_event::v1::Response { + room_state, + }) +} + +/// # `PUT /_matrix/federation/v2/send_join/{roomId}/{eventId}` +/// +/// Submits a signed join event. +pub(crate) async fn create_join_event_v2_route( + body: Ruma, +) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + if services() + .globals + .config + .forbidden_remote_server_names + .contains(origin) + { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server is banned on this homeserver.", + )); + } + + if let Some(server) = body.room_id.server_name() { + if services() + .globals + .config + .forbidden_remote_server_names + .contains(&server.to_owned()) + { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server is banned on this homeserver.", + )); + } + } + + let create_join_event::v1::RoomState { + auth_chain, + state, + event, + } = create_join_event(origin, &body.room_id, &body.pdu).await?; + let room_state = create_join_event::v2::RoomState { + members_omitted: false, + auth_chain, + state, + event, + servers_in_room: None, + }; + + Ok(create_join_event::v2::Response { + room_state, + }) +} diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs new file mode 100644 index 00000000..ec843588 --- /dev/null +++ b/src/api/server/send_leave.rs @@ -0,0 +1,186 @@ +#![allow(deprecated)] + +use std::{collections::BTreeMap, sync::Arc}; + +use ruma::{ + api::{client::error::ErrorKind, federation::membership::create_leave_event}, + events::{room::member::MembershipState, StateEventType}, + OwnedServerName, OwnedUserId, RoomId, ServerName, +}; +use serde_json::value::RawValue as RawJsonValue; +use tokio::sync::RwLock; + +use crate::{ + service::{pdu::gen_event_id_canonical_json, server_is_ours}, + services, Error, Result, Ruma, +}; + +/// # `PUT /_matrix/federation/v1/send_leave/{roomId}/{eventId}` +/// +/// Submits a signed leave event. +pub(crate) async fn create_leave_event_v1_route( + body: Ruma, +) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + create_leave_event(origin, &body.room_id, &body.pdu).await?; + + Ok(create_leave_event::v1::Response::new()) +} + +/// # `PUT /_matrix/federation/v2/send_leave/{roomId}/{eventId}` +/// +/// Submits a signed leave event. +pub(crate) async fn create_leave_event_v2_route( + body: Ruma, +) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + create_leave_event(origin, &body.room_id, &body.pdu).await?; + + Ok(create_leave_event::v2::Response::new()) +} + +async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue) -> Result<()> { + if !services().rooms.metadata.exists(room_id)? { + return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + } + + // ACL check origin + services().rooms.event_handler.acl_check(origin, room_id)?; + + let pub_key_map = RwLock::new(BTreeMap::new()); + + // We do not add the event_id field to the pdu here because of signature and + // hashes checks + let room_version_id = services().rooms.state.get_room_version(room_id)?; + let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { + // Event could not be converted to canonical json + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Could not convert event to canonical json.", + )); + }; + + let content = value + .get("content") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property."))? + .as_object() + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event content not an object."))?; + + let membership: MembershipState = serde_json::from_value( + content + .get("membership") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event membership is missing."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Event membership state is not valid."))?; + + if membership != MembershipState::Leave { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to send a non-leave membership event to leave endpoint.", + )); + } + + let event_type: StateEventType = serde_json::from_value( + value + .get("type") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Event does not have a valid state event type."))?; + + if event_type != StateEventType::RoomMember { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to send non-membership state event to leave endpoint.", + )); + } + + // ACL check sender server name + let sender: OwnedUserId = serde_json::from_value( + value + .get("sender") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "User ID in sender is invalid."))?; + + services() + .rooms + .event_handler + .acl_check(sender.server_name(), room_id)?; + + if sender.server_name() != origin { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to leave on behalf of another server.", + )); + } + + let state_key: OwnedUserId = serde_json::from_value( + value + .get("state_key") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID"))?; + + if state_key != sender { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "state_key does not match sender user.", + )); + } + + let origin: OwnedServerName = serde_json::from_value( + serde_json::to_value( + value + .get("origin") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?, + ) + .expect("CanonicalJson is valid json value"), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; + + services() + .rooms + .event_handler + .fetch_required_signing_keys([&value], &pub_key_map) + .await?; + + let mutex = Arc::clone( + services() + .globals + .roomid_mutex_federation + .write() + .await + .entry(room_id.to_owned()) + .or_default(), + ); + let mutex_lock = mutex.lock().await; + let pdu_id: Vec = services() + .rooms + .event_handler + .handle_incoming_pdu(&origin, room_id, &event_id, value, true, &pub_key_map) + .await? + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; + + drop(mutex_lock); + + let servers = services() + .rooms + .state_cache + .room_servers(room_id) + .filter_map(Result::ok) + .filter(|server| !server_is_ours(server)); + + services().sending.send_pdu_servers(servers, &pdu_id)?; + + Ok(()) +} diff --git a/src/api/server/state.rs b/src/api/server/state.rs new file mode 100644 index 00000000..10796644 --- /dev/null +++ b/src/api/server/state.rs @@ -0,0 +1,71 @@ +use std::sync::Arc; + +use ruma::api::{client::error::ErrorKind, federation::event::get_room_state}; + +use crate::{services, Error, PduEvent, Result, Ruma}; + +/// # `GET /_matrix/federation/v1/state/{roomId}` +/// +/// Retrieves the current state of the room. +pub(crate) async fn get_room_state_route( + body: Ruma, +) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + if !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + } + + services() + .rooms + .event_handler + .acl_check(origin, &body.room_id)?; + + let shortstatehash = services() + .rooms + .state_accessor + .pdu_shortstatehash(&body.event_id)? + .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Pdu state not found."))?; + + let pdus = services() + .rooms + .state_accessor + .state_full_ids(shortstatehash) + .await? + .into_values() + .map(|id| { + PduEvent::convert_to_outgoing_federation_event( + services() + .rooms + .timeline + .get_pdu_json(&id) + .unwrap() + .unwrap(), + ) + }) + .collect(); + + let auth_chain_ids = services() + .rooms + .auth_chain + .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) + .await?; + + Ok(get_room_state::v1::Response { + auth_chain: auth_chain_ids + .filter_map(|id| { + services() + .rooms + .timeline + .get_pdu_json(&id) + .ok()? + .map(PduEvent::convert_to_outgoing_federation_event) + }) + .collect(), + pdus, + }) +} diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs new file mode 100644 index 00000000..b56f7cfa --- /dev/null +++ b/src/api/server/state_ids.rs @@ -0,0 +1,53 @@ +use std::sync::Arc; + +use ruma::api::{client::error::ErrorKind, federation::event::get_room_state_ids}; + +use crate::{services, Error, Result, Ruma}; + +/// # `GET /_matrix/federation/v1/state_ids/{roomId}` +/// +/// Retrieves the current state of the room. +pub(crate) async fn get_room_state_ids_route( + body: Ruma, +) -> Result { + let origin = body.origin.as_ref().expect("server is authenticated"); + + if !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + } + + services() + .rooms + .event_handler + .acl_check(origin, &body.room_id)?; + + let shortstatehash = services() + .rooms + .state_accessor + .pdu_shortstatehash(&body.event_id)? + .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Pdu state not found."))?; + + let pdu_ids = services() + .rooms + .state_accessor + .state_full_ids(shortstatehash) + .await? + .into_values() + .map(|id| (*id).to_owned()) + .collect(); + + let auth_chain_ids = services() + .rooms + .auth_chain + .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) + .await?; + + Ok(get_room_state_ids::v1::Response { + auth_chain_ids: auth_chain_ids.map(|id| (*id).to_owned()).collect(), + pdu_ids, + }) +} diff --git a/src/api/server/user.rs b/src/api/server/user.rs new file mode 100644 index 00000000..0176de75 --- /dev/null +++ b/src/api/server/user.rs @@ -0,0 +1,108 @@ +use ruma::api::{ + client::error::ErrorKind, + federation::{ + device::get_devices::{self, v1::UserDevice}, + keys::{claim_keys, get_keys}, + }, +}; + +use crate::{ + client_server::{claim_keys_helper, get_keys_helper}, + service::user_is_local, + services, Error, Result, Ruma, +}; + +/// # `GET /_matrix/federation/v1/user/devices/{userId}` +/// +/// Gets information on all devices of the user. +pub(crate) async fn get_devices_route(body: Ruma) -> Result { + if !user_is_local(&body.user_id) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Tried to access user from other server.", + )); + } + + let origin = body.origin.as_ref().expect("server is authenticated"); + + Ok(get_devices::v1::Response { + user_id: body.user_id.clone(), + stream_id: services() + .users + .get_devicelist_version(&body.user_id)? + .unwrap_or(0) + .try_into() + .expect("version will not grow that large"), + devices: services() + .users + .all_devices_metadata(&body.user_id) + .filter_map(Result::ok) + .filter_map(|metadata| { + let device_id_string = metadata.device_id.as_str().to_owned(); + let device_display_name = if services().globals.allow_device_name_federation() { + metadata.display_name + } else { + Some(device_id_string) + }; + Some(UserDevice { + keys: services() + .users + .get_device_keys(&body.user_id, &metadata.device_id) + .ok()??, + device_id: metadata.device_id, + device_display_name, + }) + }) + .collect(), + master_key: services() + .users + .get_master_key(None, &body.user_id, &|u| u.server_name() == origin)?, + self_signing_key: services() + .users + .get_self_signing_key(None, &body.user_id, &|u| u.server_name() == origin)?, + }) +} + +/// # `POST /_matrix/federation/v1/user/keys/query` +/// +/// Gets devices and identity keys for the given users. +pub(crate) async fn get_keys_route(body: Ruma) -> Result { + if body.device_keys.iter().any(|(u, _)| !user_is_local(u)) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "User does not belong to this server.", + )); + } + + let result = get_keys_helper( + None, + &body.device_keys, + |u| Some(u.server_name()) == body.origin.as_deref(), + services().globals.allow_device_name_federation(), + ) + .await?; + + Ok(get_keys::v1::Response { + device_keys: result.device_keys, + master_keys: result.master_keys, + self_signing_keys: result.self_signing_keys, + }) +} + +/// # `POST /_matrix/federation/v1/user/keys/claim` +/// +/// Claims one-time keys. +pub(crate) async fn claim_keys_route(body: Ruma) -> Result { + if body.one_time_keys.iter().any(|(u, _)| !user_is_local(u)) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Tried to access user from other server.", + )); + } + + let result = claim_keys_helper(&body.one_time_keys).await?; + + Ok(claim_keys::v1::Response { + one_time_keys: result.one_time_keys, + }) +} diff --git a/src/api/server/version.rs b/src/api/server/version.rs new file mode 100644 index 00000000..557b10dc --- /dev/null +++ b/src/api/server/version.rs @@ -0,0 +1,17 @@ +use ruma::api::federation::discovery::get_server_version; + +use crate::{Result, Ruma}; + +/// # `GET /_matrix/federation/v1/version` +/// +/// Get version information on this server. +pub(crate) async fn get_server_version_route( + _body: Ruma, +) -> Result { + Ok(get_server_version::v1::Response { + server: Some(get_server_version::v1::Server { + name: Some("Conduwuit".to_owned()), + version: Some(conduit::version::conduwuit()), + }), + }) +} diff --git a/src/api/server/well_known.rs b/src/api/server/well_known.rs new file mode 100644 index 00000000..35862662 --- /dev/null +++ b/src/api/server/well_known.rs @@ -0,0 +1,17 @@ +use ruma::api::{client::error::ErrorKind, federation::discovery::discover_homeserver}; + +use crate::{services, Error, Result, Ruma}; + +/// # `GET /.well-known/matrix/server` +/// +/// Returns the .well-known URL if it is configured, otherwise returns 404. +pub(crate) async fn well_known_server( + _body: Ruma, +) -> Result { + Ok(discover_homeserver::Response { + server: match services().globals.well_known_server() { + Some(server_name) => server_name.to_owned(), + None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), + }, + }) +} diff --git a/src/api/server_server.rs b/src/api/server_server.rs deleted file mode 100644 index f966bdbf..00000000 --- a/src/api/server_server.rs +++ /dev/null @@ -1,2061 +0,0 @@ -#![allow(deprecated)] -// Conduit implements the older APIs - -use std::{ - collections::BTreeMap, - sync::Arc, - time::{Duration, Instant, SystemTime}, -}; - -use axum::{response::IntoResponse, Json}; -use conduit::debug_warn; -use get_profile_information::v1::ProfileField; -use rand::seq::SliceRandom; -use ruma::{ - api::{ - client::error::ErrorKind, - federation::{ - authorization::get_event_authorization, - backfill::get_backfill, - device::get_devices::{self, v1::UserDevice}, - directory::{get_public_rooms, get_public_rooms_filtered}, - discovery::{discover_homeserver, get_server_keys, get_server_version, ServerSigningKeys, VerifyKey}, - event::{get_event, get_missing_events, get_room_state, get_room_state_ids}, - keys::{claim_keys, get_keys}, - membership::{ - create_invite, create_join_event, create_leave_event, prepare_join_event, prepare_leave_event, - }, - query::{get_profile_information, get_room_information}, - space::get_hierarchy, - transactions::{ - edu::{DeviceListUpdateContent, DirectDeviceContent, Edu, SigningKeyUpdateContent}, - send_transaction_message, - }, - }, - OutgoingResponse, - }, - directory::Filter, - events::{ - receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType}, - room::{ - join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent}, - member::{MembershipState, RoomMemberEventContent}, - }, - StateEventType, TimelineEventType, - }, - serde::{Base64, JsonObject, Raw}, - to_device::DeviceIdOrAllDevices, - uint, user_id, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedServerName, - OwnedServerSigningKeyId, OwnedUserId, RoomId, RoomVersionId, ServerName, -}; -use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::sync::RwLock; -use tracing::{debug, error, trace, warn}; - -use crate::{ - client_server::{self, claim_keys_helper, get_keys_helper}, - service::{ - pdu::{gen_event_id_canonical_json, PduBuilder}, - rooms::event_handler::parse_incoming_pdu, - server_is_ours, user_is_local, - }, - services, - utils::{self}, - Error, PduEvent, Result, Ruma, -}; - -/// # `GET /_matrix/federation/v1/version` -/// -/// Get version information on this server. -pub(crate) async fn get_server_version_route( - _body: Ruma, -) -> Result { - Ok(get_server_version::v1::Response { - server: Some(get_server_version::v1::Server { - name: Some("Conduwuit".to_owned()), - version: Some(conduit::version::conduwuit()), - }), - }) -} - -/// # `GET /_matrix/key/v2/server` -/// -/// Gets the public signing keys of this server. -/// -/// - Matrix does not support invalidating public keys, so the key returned by -/// this will be valid -/// forever. -// Response type for this endpoint is Json because we need to calculate a -// signature for the response -pub(crate) async fn get_server_keys_route() -> Result { - let verify_keys: BTreeMap = BTreeMap::from([( - format!("ed25519:{}", services().globals.keypair().version()) - .try_into() - .expect("found invalid server signing keys in DB"), - VerifyKey { - key: Base64::new(services().globals.keypair().public_key().to_vec()), - }, - )]); - - let mut response = serde_json::from_slice( - get_server_keys::v2::Response { - server_key: Raw::new(&ServerSigningKeys { - server_name: services().globals.server_name().to_owned(), - verify_keys, - old_verify_keys: BTreeMap::new(), - signatures: BTreeMap::new(), - valid_until_ts: MilliSecondsSinceUnixEpoch::from_system_time( - SystemTime::now() - .checked_add(Duration::from_secs(86400 * 7)) - .expect("valid_until_ts should not get this high"), - ) - .expect("time is valid"), - }) - .expect("static conversion, no errors"), - } - .try_into_http_response::>() - .unwrap() - .body(), - ) - .unwrap(); - - ruma::signatures::sign_json( - services().globals.server_name().as_str(), - services().globals.keypair(), - &mut response, - ) - .unwrap(); - - Ok(Json(response)) -} - -/// # `GET /_matrix/key/v2/server/{keyId}` -/// -/// Gets the public signing keys of this server. -/// -/// - Matrix does not support invalidating public keys, so the key returned by -/// this will be valid -/// forever. -pub(crate) async fn get_server_keys_deprecated_route() -> impl IntoResponse { get_server_keys_route().await } - -/// # `POST /_matrix/federation/v1/publicRooms` -/// -/// Lists the public rooms on this server. -pub(crate) async fn get_public_rooms_filtered_route( - body: Ruma, -) -> Result { - if !services() - .globals - .allow_public_room_directory_over_federation() - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Room directory is not public")); - } - - let response = client_server::get_public_rooms_filtered_helper( - None, - body.limit, - body.since.as_deref(), - &body.filter, - &body.room_network, - ) - .await - .map_err(|_| Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list."))?; - - Ok(get_public_rooms_filtered::v1::Response { - chunk: response.chunk, - prev_batch: response.prev_batch, - next_batch: response.next_batch, - total_room_count_estimate: response.total_room_count_estimate, - }) -} - -/// # `GET /_matrix/federation/v1/publicRooms` -/// -/// Lists the public rooms on this server. -pub(crate) async fn get_public_rooms_route( - body: Ruma, -) -> Result { - if !services() - .globals - .allow_public_room_directory_over_federation() - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Room directory is not public")); - } - - let response = client_server::get_public_rooms_filtered_helper( - None, - body.limit, - body.since.as_deref(), - &Filter::default(), - &body.room_network, - ) - .await - .map_err(|_| Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list."))?; - - Ok(get_public_rooms::v1::Response { - chunk: response.chunk, - prev_batch: response.prev_batch, - next_batch: response.next_batch, - total_room_count_estimate: response.total_room_count_estimate, - }) -} - -/// # `PUT /_matrix/federation/v1/send/{txnId}` -/// -/// Push EDUs and PDUs to this server. -pub(crate) async fn send_transaction_message_route( - body: Ruma, -) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if *origin != body.body.origin { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Not allowed to send transactions on behalf of other servers", - )); - } - - if body.pdus.len() > 50_usize { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Not allowed to send more than 50 PDUs in one transaction", - )); - } - - if body.edus.len() > 100_usize { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Not allowed to send more than 100 EDUs in one transaction", - )); - } - - // This is all the auth_events that have been recursively fetched so they don't - // have to be deserialized over and over again. - // TODO: make this persist across requests but not in a DB Tree (in globals?) - // TODO: This could potentially also be some sort of trie (suffix tree) like - // structure so that once an auth event is known it would know (using indexes - // maybe) all of the auth events that it references. - // let mut auth_cache = EventMap::new(); - - let txn_start_time = Instant::now(); - let mut parsed_pdus = Vec::with_capacity(body.pdus.len()); - for pdu in &body.pdus { - parsed_pdus.push(match parse_incoming_pdu(pdu) { - Ok(t) => t, - Err(e) => { - debug_warn!("Could not parse PDU: {e}"); - continue; - }, - }); - - // We do not add the event_id field to the pdu here because of signature - // and hashes checks - } - - trace!( - pdus = ?parsed_pdus.len(), - edus = ?body.edus.len(), - elapsed = ?txn_start_time.elapsed(), - id = ?body.transaction_id, - origin =?body.origin, - "Starting txn", - ); - - // We go through all the signatures we see on the PDUs and fetch the - // corresponding signing keys - let pub_key_map = RwLock::new(BTreeMap::new()); - if !parsed_pdus.is_empty() { - services() - .rooms - .event_handler - .fetch_required_signing_keys(parsed_pdus.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) - .await - .unwrap_or_else(|e| { - warn!("Could not fetch all signatures for PDUs from {origin}: {:?}", e); - }); - - debug!( - elapsed = ?txn_start_time.elapsed(), - "Fetched signing keys" - ); - } - - let mut resolved_map = BTreeMap::new(); - for (event_id, value, room_id) in parsed_pdus { - let pdu_start_time = Instant::now(); - let mutex = Arc::clone( - services() - .globals - .roomid_mutex_federation - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let mutex_lock = mutex.lock().await; - resolved_map.insert( - event_id.clone(), - services() - .rooms - .event_handler - .handle_incoming_pdu(origin, &room_id, &event_id, value, true, &pub_key_map) - .await - .map(|_| ()), - ); - drop(mutex_lock); - - debug!( - pdu_elapsed = ?pdu_start_time.elapsed(), - txn_elapsed = ?txn_start_time.elapsed(), - "Finished PDU {event_id}", - ); - } - - for pdu in &resolved_map { - if let Err(e) = pdu.1 { - if matches!(e, Error::BadRequest(ErrorKind::NotFound, _)) { - warn!("Incoming PDU failed {:?}", pdu); - } - } - } - - for edu in body - .edus - .iter() - .filter_map(|edu| serde_json::from_str::(edu.json().get()).ok()) - { - match edu { - Edu::Presence(presence) => { - if !services().globals.allow_incoming_presence() { - continue; - } - - for update in presence.push { - if update.user_id.server_name() != origin { - debug_warn!(%update.user_id, %origin, "received presence EDU for user not belonging to origin"); - continue; - } - - services().presence.set_presence( - &update.user_id, - &update.presence, - Some(update.currently_active), - Some(update.last_active_ago), - update.status_msg.clone(), - )?; - } - }, - Edu::Receipt(receipt) => { - if !services().globals.allow_incoming_read_receipts() { - continue; - } - - for (room_id, room_updates) in receipt.receipts { - if services() - .rooms - .event_handler - .acl_check(origin, &room_id) - .is_err() - { - debug_warn!(%origin, %room_id, "received read receipt EDU from ACL'd server"); - continue; - } - - for (user_id, user_updates) in room_updates.read { - if user_id.server_name() != origin { - debug_warn!(%user_id, %origin, "received read receipt EDU for user not belonging to origin"); - continue; - } - - if services() - .rooms - .state_cache - .room_members(&room_id) - .filter_map(Result::ok) - .any(|member| member.server_name() == user_id.server_name()) - { - for event_id in &user_updates.event_ids { - let user_receipts = BTreeMap::from([(user_id.clone(), user_updates.data.clone())]); - - let receipts = BTreeMap::from([(ReceiptType::Read, user_receipts)]); - - let receipt_content = BTreeMap::from([(event_id.to_owned(), receipts)]); - - let event = ReceiptEvent { - content: ReceiptEventContent(receipt_content), - room_id: room_id.clone(), - }; - - services() - .rooms - .read_receipt - .readreceipt_update(&user_id, &room_id, event)?; - } - } else { - debug_warn!(%user_id, %room_id, %origin, "received read receipt EDU from server who does not have a single member from their server in the room"); - continue; - } - } - } - }, - Edu::Typing(typing) => { - if !services().globals.config.allow_incoming_typing { - continue; - } - - if typing.user_id.server_name() != origin { - debug_warn!(%typing.user_id, %origin, "received typing EDU for user not belonging to origin"); - continue; - } - - if services() - .rooms - .event_handler - .acl_check(typing.user_id.server_name(), &typing.room_id) - .is_err() - { - debug_warn!(%typing.user_id, %typing.room_id, %origin, "received typing EDU for ACL'd user's server"); - continue; - } - - if services() - .rooms - .state_cache - .is_joined(&typing.user_id, &typing.room_id)? - { - if typing.typing { - let timeout = utils::millis_since_unix_epoch().saturating_add( - services() - .globals - .config - .typing_federation_timeout_s - .saturating_mul(1000), - ); - services() - .rooms - .typing - .typing_add(&typing.user_id, &typing.room_id, timeout) - .await?; - } else { - services() - .rooms - .typing - .typing_remove(&typing.user_id, &typing.room_id) - .await?; - } - } else { - debug_warn!(%typing.user_id, %typing.room_id, %origin, "received typing EDU for user not in room"); - continue; - } - }, - Edu::DeviceListUpdate(DeviceListUpdateContent { - user_id, - .. - }) => { - if user_id.server_name() != origin { - debug_warn!(%user_id, %origin, "received device list update EDU for user not belonging to origin"); - continue; - } - - services().users.mark_device_key_update(&user_id)?; - }, - Edu::DirectToDevice(DirectDeviceContent { - sender, - ev_type, - message_id, - messages, - }) => { - if sender.server_name() != origin { - debug_warn!(%sender, %origin, "received direct to device EDU for user not belonging to origin"); - continue; - } - - // Check if this is a new transaction id - if services() - .transaction_ids - .existing_txnid(&sender, None, &message_id)? - .is_some() - { - continue; - } - - for (target_user_id, map) in &messages { - for (target_device_id_maybe, event) in map { - match target_device_id_maybe { - DeviceIdOrAllDevices::DeviceId(target_device_id) => { - services().users.add_to_device_event( - &sender, - target_user_id, - target_device_id, - &ev_type.to_string(), - event.deserialize_as().map_err(|e| { - error!("To-Device event is invalid: {event:?} {e}"); - Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") - })?, - )?; - }, - - DeviceIdOrAllDevices::AllDevices => { - for target_device_id in services().users.all_device_ids(target_user_id) { - services().users.add_to_device_event( - &sender, - target_user_id, - &target_device_id?, - &ev_type.to_string(), - event.deserialize_as().map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") - })?, - )?; - } - }, - } - } - } - - // Save transaction id with empty data - services() - .transaction_ids - .add_txnid(&sender, None, &message_id, &[])?; - }, - Edu::SigningKeyUpdate(SigningKeyUpdateContent { - user_id, - master_key, - self_signing_key, - }) => { - if user_id.server_name() != origin { - debug_warn!(%user_id, %origin, "received signing key update EDU from server that does not belong to user's server"); - continue; - } - - if let Some(master_key) = master_key { - services() - .users - .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true)?; - } - }, - Edu::_Custom(custom) => { - debug_warn!(?custom, "received custom/unknown EDU"); - }, - } - } - - debug!( - pdus = ?body.pdus.len(), - edus = ?body.edus.len(), - elapsed = ?txn_start_time.elapsed(), - id = ?body.transaction_id, - origin =?body.origin, - "Finished txn", - ); - - Ok(send_transaction_message::v1::Response { - pdus: resolved_map - .into_iter() - .map(|(e, r)| (e, r.map_err(|e| e.sanitized_error()))) - .collect(), - }) -} - -/// # `GET /_matrix/federation/v1/event/{eventId}` -/// -/// Retrieves a single event from the server. -/// -/// - Only works if a user of this server is currently invited or joined the -/// room -pub(crate) async fn get_event_route(body: Ruma) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - let event = services() - .rooms - .timeline - .get_pdu_json(&body.event_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Event not found."))?; - - let room_id_str = event - .get("room_id") - .and_then(|val| val.as_str()) - .ok_or_else(|| Error::bad_database("Invalid event in database."))?; - - let room_id = - <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; - - if !services() - .rooms - .state_cache - .server_in_room(origin, room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); - } - - if !services() - .rooms - .state_accessor - .server_can_see_event(origin, room_id, &body.event_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not allowed to see event.")); - } - - Ok(get_event::v1::Response { - origin: services().globals.server_name().to_owned(), - origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - pdu: PduEvent::convert_to_outgoing_federation_event(event), - }) -} - -/// # `GET /_matrix/federation/v1/backfill/` -/// -/// Retrieves events from before the sender joined the room, if the room's -/// history visibility allows. -pub(crate) async fn get_backfill_route(body: Ruma) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); - } - - services() - .rooms - .event_handler - .acl_check(origin, &body.room_id)?; - - let until = body - .v - .iter() - .map(|event_id| services().rooms.timeline.get_pdu_count(event_id)) - .filter_map(|r| r.ok().flatten()) - .max() - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event not found."))?; - - let limit = body - .limit - .min(uint!(100)) - .try_into() - .expect("UInt could not be converted to usize"); - - let all_events = services() - .rooms - .timeline - .pdus_until(user_id!("@doesntmatter:conduit.rs"), &body.room_id, until)? - .take(limit); - - let events = all_events - .filter_map(Result::ok) - .filter(|(_, e)| { - matches!( - services() - .rooms - .state_accessor - .server_can_see_event(origin, &e.room_id, &e.event_id,), - Ok(true), - ) - }) - .map(|(_, pdu)| services().rooms.timeline.get_pdu_json(&pdu.event_id)) - .filter_map(|r| r.ok().flatten()) - .map(PduEvent::convert_to_outgoing_federation_event) - .collect(); - - Ok(get_backfill::v1::Response { - origin: services().globals.server_name().to_owned(), - origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - pdus: events, - }) -} - -/// # `POST /_matrix/federation/v1/get_missing_events/{roomId}` -/// -/// Retrieves events that the sender is missing. -pub(crate) async fn get_missing_events_route( - body: Ruma, -) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room")); - } - - services() - .rooms - .event_handler - .acl_check(origin, &body.room_id)?; - - let limit = body - .limit - .try_into() - .expect("UInt could not be converted to usize"); - - let mut queued_events = body.latest_events.clone(); - // the vec will never have more entries the limit - let mut events = Vec::with_capacity(limit); - - let mut i: usize = 0; - while i < queued_events.len() && events.len() < limit { - if let Some(pdu) = services().rooms.timeline.get_pdu_json(&queued_events[i])? { - let room_id_str = pdu - .get("room_id") - .and_then(|val| val.as_str()) - .ok_or_else(|| Error::bad_database("Invalid event in database."))?; - - let event_room_id = <&RoomId>::try_from(room_id_str) - .map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; - - if event_room_id != body.room_id { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event from wrong room.")); - } - - if body.earliest_events.contains(&queued_events[i]) { - i = i.saturating_add(1); - continue; - } - - if !services() - .rooms - .state_accessor - .server_can_see_event(origin, &body.room_id, &queued_events[i])? - { - i = i.saturating_add(1); - continue; - } - - queued_events.extend_from_slice( - &serde_json::from_value::>( - serde_json::to_value( - pdu.get("prev_events") - .cloned() - .ok_or_else(|| Error::bad_database("Event in db has no prev_events property."))?, - ) - .expect("canonical json is valid json value"), - ) - .map_err(|_| Error::bad_database("Invalid prev_events in event in database."))?, - ); - events.push(PduEvent::convert_to_outgoing_federation_event(pdu)); - } - i = i.saturating_add(1); - } - - Ok(get_missing_events::v1::Response { - events, - }) -} - -/// # `GET /_matrix/federation/v1/event_auth/{roomId}/{eventId}` -/// -/// Retrieves the auth chain for a given event. -/// -/// - This does not include the event itself -pub(crate) async fn get_event_authorization_route( - body: Ruma, -) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); - } - - services() - .rooms - .event_handler - .acl_check(origin, &body.room_id)?; - - let event = services() - .rooms - .timeline - .get_pdu_json(&body.event_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Event not found."))?; - - let room_id_str = event - .get("room_id") - .and_then(|val| val.as_str()) - .ok_or_else(|| Error::bad_database("Invalid event in database."))?; - - let room_id = - <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; - - let auth_chain_ids = services() - .rooms - .auth_chain - .event_ids_iter(room_id, vec![Arc::from(&*body.event_id)]) - .await?; - - Ok(get_event_authorization::v1::Response { - auth_chain: auth_chain_ids - .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok()?) - .map(PduEvent::convert_to_outgoing_federation_event) - .collect(), - }) -} - -/// # `GET /_matrix/federation/v1/state/{roomId}` -/// -/// Retrieves the current state of the room. -pub(crate) async fn get_room_state_route( - body: Ruma, -) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); - } - - services() - .rooms - .event_handler - .acl_check(origin, &body.room_id)?; - - let shortstatehash = services() - .rooms - .state_accessor - .pdu_shortstatehash(&body.event_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Pdu state not found."))?; - - let pdus = services() - .rooms - .state_accessor - .state_full_ids(shortstatehash) - .await? - .into_values() - .map(|id| { - PduEvent::convert_to_outgoing_federation_event( - services() - .rooms - .timeline - .get_pdu_json(&id) - .unwrap() - .unwrap(), - ) - }) - .collect(); - - let auth_chain_ids = services() - .rooms - .auth_chain - .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) - .await?; - - Ok(get_room_state::v1::Response { - auth_chain: auth_chain_ids - .filter_map(|id| { - services() - .rooms - .timeline - .get_pdu_json(&id) - .ok()? - .map(PduEvent::convert_to_outgoing_federation_event) - }) - .collect(), - pdus, - }) -} - -/// # `GET /_matrix/federation/v1/state_ids/{roomId}` -/// -/// Retrieves the current state of the room. -pub(crate) async fn get_room_state_ids_route( - body: Ruma, -) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); - } - - services() - .rooms - .event_handler - .acl_check(origin, &body.room_id)?; - - let shortstatehash = services() - .rooms - .state_accessor - .pdu_shortstatehash(&body.event_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Pdu state not found."))?; - - let pdu_ids = services() - .rooms - .state_accessor - .state_full_ids(shortstatehash) - .await? - .into_values() - .map(|id| (*id).to_owned()) - .collect(); - - let auth_chain_ids = services() - .rooms - .auth_chain - .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) - .await?; - - Ok(get_room_state_ids::v1::Response { - auth_chain_ids: auth_chain_ids.map(|id| (*id).to_owned()).collect(), - pdu_ids, - }) -} - -/// # `GET /_matrix/federation/v1/make_join/{roomId}/{userId}` -/// -/// Creates a join template. -pub(crate) async fn create_join_event_template_route( - body: Ruma, -) -> Result { - if !services().rooms.metadata.exists(&body.room_id)? { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); - } - - let origin = body.origin.as_ref().expect("server is authenticated"); - if body.user_id.server_name() != origin { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to join on behalf of another server/user", - )); - } - - // ACL check origin server - services() - .rooms - .event_handler - .acl_check(origin, &body.room_id)?; - - if services() - .globals - .config - .forbidden_remote_server_names - .contains(origin) - { - warn!( - "Server {origin} for remote user {} tried joining room ID {} which has a server name that is globally \ - forbidden. Rejecting.", - &body.user_id, &body.room_id, - ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); - } - - if let Some(server) = body.room_id.server_name() { - if services() - .globals - .config - .forbidden_remote_server_names - .contains(&server.to_owned()) - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); - } - } - - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - - let join_rules_event = - services() - .rooms - .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomJoinRules, "")?; - - let join_rules_event_content: Option = join_rules_event - .as_ref() - .map(|join_rules_event| { - serde_json::from_str(join_rules_event.content.get()) - .map_err(|_| Error::bad_database("Invalid join rules event in db.")) - }) - .transpose()?; - - let join_authorized_via_users_server = if let Some(join_rules_event_content) = join_rules_event_content { - if let JoinRule::Restricted(r) | JoinRule::KnockRestricted(r) = join_rules_event_content.join_rule { - if r.allow - .iter() - .filter_map(|rule| { - if let AllowRule::RoomMembership(membership) = rule { - Some(membership) - } else { - None - } - }) - .any(|m| { - services() - .rooms - .state_cache - .is_joined(&body.user_id, &m.room_id) - .unwrap_or(false) - }) { - if services() - .rooms - .state_cache - .is_left(&body.user_id, &body.room_id) - .unwrap_or(true) - { - let members: Vec<_> = services() - .rooms - .state_cache - .room_members(&body.room_id) - .filter_map(Result::ok) - .filter(|user| user_is_local(user)) - .collect(); - - let mut auth_user = None; - - for user in members { - if services() - .rooms - .state_accessor - .user_can_invite(&body.room_id, &user, &body.user_id, &state_lock) - .await - .unwrap_or(false) - { - auth_user = Some(user); - break; - } - } - if auth_user.is_some() { - auth_user - } else { - return Err(Error::BadRequest( - ErrorKind::UnableToGrantJoin, - "No user on this server is able to assist in joining.", - )); - } - } else { - // If the user has any state other than leave, either: - // - the auth_check will deny them (ban, knock - (until/unless MSC4123 is - // merged)) - // - they are able to join via other methods (invite) - // - they are already in the room (join) - None - } - } else { - return Err(Error::BadRequest( - ErrorKind::UnableToAuthorizeJoin, - "User is not known to be in any required room.", - )); - } - } else { - None - } - } else { - None - }; - - let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; - if !body.ver.contains(&room_version_id) { - return Err(Error::BadRequest( - ErrorKind::IncompatibleRoomVersion { - room_version: room_version_id, - }, - "Room version not supported.", - )); - } - - let content = to_raw_value(&RoomMemberEventContent { - avatar_url: None, - blurhash: None, - displayname: None, - is_direct: None, - membership: MembershipState::Join, - third_party_invite: None, - reason: None, - join_authorized_via_users_server, - }) - .expect("member event is valid value"); - - let (_pdu, mut pdu_json) = services().rooms.timeline.create_hash_and_sign_event( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content, - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &body.user_id, - &body.room_id, - &state_lock, - )?; - - drop(state_lock); - - // room v3 and above removed the "event_id" field from remote PDU format - match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 - | RoomVersionId::V11 => { - pdu_json.remove("event_id"); - }, - _ => { - warn!("Unexpected or unsupported room version {room_version_id}"); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, - }; - - Ok(prepare_join_event::v1::Response { - room_version: Some(room_version_id), - event: to_raw_value(&pdu_json).expect("CanonicalJson can be serialized to JSON"), - }) -} - -/// helper method for /send_join v1 and v2 -async fn create_join_event( - origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, -) -> Result { - if !services().rooms.metadata.exists(room_id)? { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); - } - - // ACL check origin server - services().rooms.event_handler.acl_check(origin, room_id)?; - - // We need to return the state prior to joining, let's keep a reference to that - // here - let shortstatehash = services() - .rooms - .state - .get_room_shortstatehash(room_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Event state not found."))?; - - let pub_key_map = RwLock::new(BTreeMap::new()); - // let mut auth_cache = EventMap::new(); - - // We do not add the event_id field to the pdu here because of signature and - // hashes checks - let room_version_id = services().rooms.state.get_room_version(room_id)?; - - let Ok((event_id, mut value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { - // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); - }; - - let event_type: StateEventType = serde_json::from_value( - value - .get("type") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event has invalid event type."))?; - - if event_type != StateEventType::RoomMember { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to send non-membership state event to join endpoint.", - )); - } - - let content = value - .get("content") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property."))? - .as_object() - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid."))?; - - let membership: MembershipState = serde_json::from_value( - content - .get("membership") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event content missing membership property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Event has an invalid membership state."))?; - - if membership != MembershipState::Join { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to send a non-join membership event to join endpoint.", - )); - } - - // ACL check sender server name - let sender: OwnedUserId = serde_json::from_value( - value - .get("sender") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "sender is not a valid user ID."))?; - - services() - .rooms - .event_handler - .acl_check(sender.server_name(), room_id)?; - - // check if origin server is trying to send for another server - if sender.server_name() != origin { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to join on behalf of another server.", - )); - } - - let state_key: OwnedUserId = serde_json::from_value( - value - .get("state_key") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID."))?; - - if state_key != sender { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "State key does not match sender user", - )); - } - - ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), - &mut value, - &room_version_id, - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; - - let origin: OwnedServerName = serde_json::from_value( - serde_json::to_value( - value - .get("origin") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?, - ) - .expect("CanonicalJson is valid json value"), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; - - services() - .rooms - .event_handler - .fetch_required_signing_keys([&value], &pub_key_map) - .await?; - - let mutex = Arc::clone( - services() - .globals - .roomid_mutex_federation - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let mutex_lock = mutex.lock().await; - let pdu_id: Vec = services() - .rooms - .event_handler - .handle_incoming_pdu(&origin, room_id, &event_id, value.clone(), true, &pub_key_map) - .await? - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; - drop(mutex_lock); - - let state_ids = services() - .rooms - .state_accessor - .state_full_ids(shortstatehash) - .await?; - let auth_chain_ids = services() - .rooms - .auth_chain - .event_ids_iter(room_id, state_ids.values().cloned().collect()) - .await?; - - services().sending.send_pdu_room(room_id, &pdu_id)?; - - Ok(create_join_event::v1::RoomState { - auth_chain: auth_chain_ids - .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok().flatten()) - .map(PduEvent::convert_to_outgoing_federation_event) - .collect(), - state: state_ids - .iter() - .filter_map(|(_, id)| services().rooms.timeline.get_pdu_json(id).ok().flatten()) - .map(PduEvent::convert_to_outgoing_federation_event) - .collect(), - // Event field is required if the room version supports restricted join rules. - event: Some( - to_raw_value(&CanonicalJsonValue::Object(value.clone())) - .expect("To raw json should not fail since only change was adding signature"), - ), - }) -} - -/// # `PUT /_matrix/federation/v1/send_join/{roomId}/{eventId}` -/// -/// Submits a signed join event. -pub(crate) async fn create_join_event_v1_route( - body: Ruma, -) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if services() - .globals - .config - .forbidden_remote_server_names - .contains(origin) - { - warn!( - "Server {origin} tried joining room ID {} who has a server name that is globally forbidden. Rejecting.", - &body.room_id, - ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); - } - - if let Some(server) = body.room_id.server_name() { - if services() - .globals - .config - .forbidden_remote_server_names - .contains(&server.to_owned()) - { - warn!( - "Server {origin} tried joining room ID {} which has a server name that is globally forbidden. \ - Rejecting.", - &body.room_id, - ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); - } - } - - let room_state = create_join_event(origin, &body.room_id, &body.pdu).await?; - - Ok(create_join_event::v1::Response { - room_state, - }) -} - -/// # `PUT /_matrix/federation/v2/send_join/{roomId}/{eventId}` -/// -/// Submits a signed join event. -pub(crate) async fn create_join_event_v2_route( - body: Ruma, -) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if services() - .globals - .config - .forbidden_remote_server_names - .contains(origin) - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); - } - - if let Some(server) = body.room_id.server_name() { - if services() - .globals - .config - .forbidden_remote_server_names - .contains(&server.to_owned()) - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); - } - } - - let create_join_event::v1::RoomState { - auth_chain, - state, - event, - } = create_join_event(origin, &body.room_id, &body.pdu).await?; - let room_state = create_join_event::v2::RoomState { - members_omitted: false, - auth_chain, - state, - event, - servers_in_room: None, - }; - - Ok(create_join_event::v2::Response { - room_state, - }) -} - -/// # `PUT /_matrix/federation/v1/make_leave/{roomId}/{eventId}` -/// -/// Creates a leave template. -pub(crate) async fn create_leave_event_template_route( - body: Ruma, -) -> Result { - if !services().rooms.metadata.exists(&body.room_id)? { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); - } - - let origin = body.origin.as_ref().expect("server is authenticated"); - if body.user_id.server_name() != origin { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to leave on behalf of another server/user", - )); - } - - // ACL check origin - services() - .rooms - .event_handler - .acl_check(origin, &body.room_id)?; - - let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; - - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - - let content = to_raw_value(&RoomMemberEventContent { - avatar_url: None, - blurhash: None, - displayname: None, - is_direct: None, - membership: MembershipState::Leave, - third_party_invite: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("member event is valid value"); - - let (_pdu, mut pdu_json) = services().rooms.timeline.create_hash_and_sign_event( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content, - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - }, - &body.user_id, - &body.room_id, - &state_lock, - )?; - - drop(state_lock); - - // room v3 and above removed the "event_id" field from remote PDU format - match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 - | RoomVersionId::V11 => { - pdu_json.remove("event_id"); - }, - _ => { - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, - }; - - Ok(prepare_leave_event::v1::Response { - room_version: Some(room_version_id), - event: to_raw_value(&pdu_json).expect("CanonicalJson can be serialized to JSON"), - }) -} - -async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue) -> Result<()> { - if !services().rooms.metadata.exists(room_id)? { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); - } - - // ACL check origin - services().rooms.event_handler.acl_check(origin, room_id)?; - - let pub_key_map = RwLock::new(BTreeMap::new()); - - // We do not add the event_id field to the pdu here because of signature and - // hashes checks - let room_version_id = services().rooms.state.get_room_version(room_id)?; - let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { - // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); - }; - - let content = value - .get("content") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property."))? - .as_object() - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event content not an object."))?; - - let membership: MembershipState = serde_json::from_value( - content - .get("membership") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event membership is missing."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Event membership state is not valid."))?; - - if membership != MembershipState::Leave { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to send a non-leave membership event to leave endpoint.", - )); - } - - let event_type: StateEventType = serde_json::from_value( - value - .get("type") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Event does not have a valid state event type."))?; - - if event_type != StateEventType::RoomMember { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to send non-membership state event to leave endpoint.", - )); - } - - // ACL check sender server name - let sender: OwnedUserId = serde_json::from_value( - value - .get("sender") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "User ID in sender is invalid."))?; - - services() - .rooms - .event_handler - .acl_check(sender.server_name(), room_id)?; - - if sender.server_name() != origin { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to leave on behalf of another server.", - )); - } - - let state_key: OwnedUserId = serde_json::from_value( - value - .get("state_key") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID"))?; - - if state_key != sender { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "state_key does not match sender user.", - )); - } - - let origin: OwnedServerName = serde_json::from_value( - serde_json::to_value( - value - .get("origin") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?, - ) - .expect("CanonicalJson is valid json value"), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; - - services() - .rooms - .event_handler - .fetch_required_signing_keys([&value], &pub_key_map) - .await?; - - let mutex = Arc::clone( - services() - .globals - .roomid_mutex_federation - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let mutex_lock = mutex.lock().await; - let pdu_id: Vec = services() - .rooms - .event_handler - .handle_incoming_pdu(&origin, room_id, &event_id, value, true, &pub_key_map) - .await? - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; - - drop(mutex_lock); - - let servers = services() - .rooms - .state_cache - .room_servers(room_id) - .filter_map(Result::ok) - .filter(|server| !server_is_ours(server)); - - services().sending.send_pdu_servers(servers, &pdu_id)?; - - Ok(()) -} - -/// # `PUT /_matrix/federation/v1/send_leave/{roomId}/{eventId}` -/// -/// Submits a signed leave event. -pub(crate) async fn create_leave_event_v1_route( - body: Ruma, -) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - create_leave_event(origin, &body.room_id, &body.pdu).await?; - - Ok(create_leave_event::v1::Response::new()) -} - -/// # `PUT /_matrix/federation/v2/send_leave/{roomId}/{eventId}` -/// -/// Submits a signed leave event. -pub(crate) async fn create_leave_event_v2_route( - body: Ruma, -) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - create_leave_event(origin, &body.room_id, &body.pdu).await?; - - Ok(create_leave_event::v2::Response::new()) -} - -/// # `PUT /_matrix/federation/v2/invite/{roomId}/{eventId}` -/// -/// Invites a remote user to a room. -pub(crate) async fn create_invite_route(body: Ruma) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - // ACL check origin - services() - .rooms - .event_handler - .acl_check(origin, &body.room_id)?; - - if !services() - .globals - .supported_room_versions() - .contains(&body.room_version) - { - return Err(Error::BadRequest( - ErrorKind::IncompatibleRoomVersion { - room_version: body.room_version.clone(), - }, - "Server does not support this room version.", - )); - } - - if let Some(server) = body.room_id.server_name() { - if services() - .globals - .config - .forbidden_remote_server_names - .contains(&server.to_owned()) - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); - } - } - - if services() - .globals - .config - .forbidden_remote_server_names - .contains(origin) - { - warn!( - "Received federated/remote invite from banned server {origin} for room ID {}. Rejecting.", - body.room_id - ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); - } - - if let Some(via) = &body.via { - if via.is_empty() { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "via field must not be empty.")); - } - } - - let mut signed_event = utils::to_canonical_object(&body.event) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invite event is invalid."))?; - - let invited_user: OwnedUserId = serde_json::from_value( - signed_event - .get("state_key") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event has no state_key property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "state_key is not a user ID."))?; - - if !server_is_ours(invited_user.server_name()) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "User does not belong to this homeserver.", - )); - } - - // Make sure we're not ACL'ed from their room. - services() - .rooms - .event_handler - .acl_check(invited_user.server_name(), &body.room_id)?; - - ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), - &mut signed_event, - &body.room_version, - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; - - // Generate event id - let event_id = EventId::parse(format!( - "${}", - ruma::signatures::reference_hash(&signed_event, &body.room_version) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - // Add event_id back - signed_event.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.to_string())); - - let sender: OwnedUserId = serde_json::from_value( - signed_event - .get("sender") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event had no sender property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "sender is not a user ID."))?; - - if services().rooms.metadata.is_banned(&body.room_id)? && !services().users.is_admin(&invited_user)? { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "This room is banned on this homeserver.", - )); - } - - if services().globals.block_non_admin_invites() && !services().users.is_admin(&invited_user)? { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "This server does not allow room invites.", - )); - } - - let mut invite_state = body.invite_room_state.clone(); - - let mut event: JsonObject = serde_json::from_str(body.event.get()) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event bytes."))?; - - event.insert("event_id".to_owned(), "$placeholder".into()); - - let pdu: PduEvent = serde_json::from_value(event.into()) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event."))?; - - invite_state.push(pdu.to_stripped_state_event()); - - // If we are active in the room, the remote server will notify us about the join - // via /send - if !services() - .rooms - .state_cache - .server_in_room(services().globals.server_name(), &body.room_id)? - { - services().rooms.state_cache.update_membership( - &body.room_id, - &invited_user, - RoomMemberEventContent::new(MembershipState::Invite), - &sender, - Some(invite_state), - body.via.clone(), - true, - )?; - } - - Ok(create_invite::v2::Response { - event: PduEvent::convert_to_outgoing_federation_event(signed_event), - }) -} - -/// # `GET /_matrix/federation/v1/user/devices/{userId}` -/// -/// Gets information on all devices of the user. -pub(crate) async fn get_devices_route(body: Ruma) -> Result { - if !user_is_local(&body.user_id) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Tried to access user from other server.", - )); - } - - let origin = body.origin.as_ref().expect("server is authenticated"); - - Ok(get_devices::v1::Response { - user_id: body.user_id.clone(), - stream_id: services() - .users - .get_devicelist_version(&body.user_id)? - .unwrap_or(0) - .try_into() - .expect("version will not grow that large"), - devices: services() - .users - .all_devices_metadata(&body.user_id) - .filter_map(Result::ok) - .filter_map(|metadata| { - let device_id_string = metadata.device_id.as_str().to_owned(); - let device_display_name = if services().globals.allow_device_name_federation() { - metadata.display_name - } else { - Some(device_id_string) - }; - Some(UserDevice { - keys: services() - .users - .get_device_keys(&body.user_id, &metadata.device_id) - .ok()??, - device_id: metadata.device_id, - device_display_name, - }) - }) - .collect(), - master_key: services() - .users - .get_master_key(None, &body.user_id, &|u| u.server_name() == origin)?, - self_signing_key: services() - .users - .get_self_signing_key(None, &body.user_id, &|u| u.server_name() == origin)?, - }) -} - -/// # `GET /_matrix/federation/v1/query/directory` -/// -/// Resolve a room alias to a room id. -pub(crate) async fn get_room_information_route( - body: Ruma, -) -> Result { - let room_id = services() - .rooms - .alias - .resolve_local_alias(&body.room_alias)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Room alias not found."))?; - - let mut servers: Vec = services() - .rooms - .state_cache - .room_servers(&room_id) - .filter_map(Result::ok) - .collect(); - - servers.sort_unstable(); - servers.dedup(); - - servers.shuffle(&mut rand::thread_rng()); - - // insert our server as the very first choice if in list - if let Some(server_index) = servers - .iter() - .position(|server| server == services().globals.server_name()) - { - servers.swap_remove(server_index); - servers.insert(0, services().globals.server_name().to_owned()); - } - - Ok(get_room_information::v1::Response { - room_id, - servers, - }) -} - -/// # `GET /_matrix/federation/v1/query/profile` -/// -/// -/// Gets information on a profile. -pub(crate) async fn get_profile_information_route( - body: Ruma, -) -> Result { - if !services() - .globals - .allow_profile_lookup_federation_requests() - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Profile lookup over federation is not allowed on this homeserver.", - )); - } - - if !server_is_ours(body.user_id.server_name()) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "User does not belong to this server.", - )); - } - - let mut displayname = None; - let mut avatar_url = None; - let mut blurhash = None; - - match &body.field { - Some(ProfileField::DisplayName) => { - displayname = services().users.displayname(&body.user_id)?; - }, - Some(ProfileField::AvatarUrl) => { - avatar_url = services().users.avatar_url(&body.user_id)?; - blurhash = services().users.blurhash(&body.user_id)?; - }, - // TODO: what to do with custom - Some(_) => {}, - None => { - displayname = services().users.displayname(&body.user_id)?; - avatar_url = services().users.avatar_url(&body.user_id)?; - blurhash = services().users.blurhash(&body.user_id)?; - }, - } - - Ok(get_profile_information::v1::Response { - displayname, - avatar_url, - blurhash, - }) -} - -/// # `POST /_matrix/federation/v1/user/keys/query` -/// -/// Gets devices and identity keys for the given users. -pub(crate) async fn get_keys_route(body: Ruma) -> Result { - if body.device_keys.iter().any(|(u, _)| !user_is_local(u)) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "User does not belong to this server.", - )); - } - - let result = get_keys_helper( - None, - &body.device_keys, - |u| Some(u.server_name()) == body.origin.as_deref(), - services().globals.allow_device_name_federation(), - ) - .await?; - - Ok(get_keys::v1::Response { - device_keys: result.device_keys, - master_keys: result.master_keys, - self_signing_keys: result.self_signing_keys, - }) -} - -/// # `POST /_matrix/federation/v1/user/keys/claim` -/// -/// Claims one-time keys. -pub(crate) async fn claim_keys_route(body: Ruma) -> Result { - if body.one_time_keys.iter().any(|(u, _)| !user_is_local(u)) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Tried to access user from other server.", - )); - } - - let result = claim_keys_helper(&body.one_time_keys).await?; - - Ok(claim_keys::v1::Response { - one_time_keys: result.one_time_keys, - }) -} - -/// # `GET /.well-known/matrix/server` -/// -/// Returns the .well-known URL if it is configured, otherwise returns 404. -pub(crate) async fn well_known_server( - _body: Ruma, -) -> Result { - Ok(discover_homeserver::Response { - server: match services().globals.well_known_server() { - Some(server_name) => server_name.to_owned(), - None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), - }, - }) -} - -/// # `GET /_matrix/federation/v1/hierarchy/{roomId}` -/// -/// Gets the space tree in a depth-first manner to locate child rooms of a given -/// space. -pub(crate) async fn get_hierarchy_route(body: Ruma) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if services().rooms.metadata.exists(&body.room_id)? { - services() - .rooms - .spaces - .get_federation_hierarchy(&body.room_id, origin, body.suggested_only) - .await - } else { - Err(Error::BadRequest(ErrorKind::NotFound, "Room does not exist.")) - } -} From 8428f43c782da2e58375ac2963a8b2c45ec31a90 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 6 Jun 2024 03:43:21 -0400 Subject: [PATCH 0077/2091] add legacy element hack for UIAA using invalid `user` field see: - https://github.com/girlbossceo/ruwuma/commit/e9302a955614429ca969eb4c7f561fc87a4f6f15 - https://github.com/element-hq/element-android/issues/8043 - https://github.com/element-hq/element-ios/issues/7405 Signed-off-by: strawberry --- Cargo.lock | 24 ++++++++++++------------ src/main/Cargo.toml | 1 + src/service/Cargo.toml | 1 + src/service/uiaa/mod.rs | 15 ++++++++++++++- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2b00be5..23457c4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2721,7 +2721,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "assign", "js_int", @@ -2741,7 +2741,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "js_int", "ruma-common", @@ -2753,7 +2753,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "as_variant", "assign", @@ -2776,7 +2776,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "as_variant", "base64 0.22.1", @@ -2806,7 +2806,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "as_variant", "indexmap 2.2.6", @@ -2828,7 +2828,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "js_int", "ruma-common", @@ -2840,7 +2840,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "js_int", "thiserror", @@ -2849,7 +2849,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "js_int", "ruma-common", @@ -2859,7 +2859,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "once_cell", "proc-macro-crate", @@ -2874,7 +2874,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "js_int", "ruma-common", @@ -2886,7 +2886,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -2902,7 +2902,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#042444dc1d30c40a97e5acd7cebcda4ed1f76ed6" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 94c62656..373ea878 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -67,6 +67,7 @@ dev_release_log_level = [ ] element_hacks = [ "conduit-api/element_hacks", + "conduit-service/element_hacks", ] gzip_compression = [ "conduit-api/gzip_compression", diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 3bfd0e51..63e3e95f 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -17,6 +17,7 @@ crate-type = [ ] [features] +element_hacks = [] dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 11f848ad..bec6c15b 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -58,9 +58,22 @@ impl Service { AuthData::Password(Password { identifier, password, + #[cfg(feature = "element_hacks")] + user, .. }) => { - let UserIdentifier::UserIdOrLocalpart(username) = identifier else { + #[cfg(feature = "element_hacks")] + let username = if let Some(UserIdentifier::UserIdOrLocalpart(username)) = identifier { + username + } else if let Some(username) = user { + username + } else { + return Err(Error::BadRequest(ErrorKind::Unrecognized, "Identifier type not recognized.")); + }; + + #[cfg(not(feature = "element_hacks"))] + let Some(UserIdentifier::UserIdOrLocalpart(username)) = identifier + else { return Err(Error::BadRequest(ErrorKind::Unrecognized, "Identifier type not recognized.")); }; From f32380772ff539e0a7af84e1462891209b4d6dba Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Jun 2024 18:21:02 +0000 Subject: [PATCH 0078/2091] rename api::client_server to api::client Signed-off-by: Jason Volk --- src/admin/room/room_moderation_commands.rs | 2 +- src/admin/user/user_commands.rs | 2 +- src/api/{client_server => client}/account.rs | 5 +- src/api/{client_server => client}/alias.rs | 0 src/api/{client_server => client}/backup.rs | 0 .../{client_server => client}/capabilities.rs | 0 src/api/{client_server => client}/config.rs | 0 src/api/{client_server => client}/context.rs | 0 src/api/{client_server => client}/device.rs | 0 .../{client_server => client}/directory.rs | 0 src/api/{client_server => client}/filter.rs | 0 src/api/{client_server => client}/keys.rs | 0 src/api/{client_server => client}/media.rs | 0 .../{client_server => client}/membership.rs | 0 src/api/{client_server => client}/message.rs | 0 src/api/{client_server => client}/mod.rs | 0 src/api/{client_server => client}/presence.rs | 0 src/api/{client_server => client}/profile.rs | 0 src/api/{client_server => client}/push.rs | 0 .../{client_server => client}/read_marker.rs | 0 src/api/{client_server => client}/redact.rs | 0 .../{client_server => client}/relations.rs | 0 src/api/{client_server => client}/report.rs | 0 src/api/{client_server => client}/room.rs | 2 +- src/api/{client_server => client}/search.rs | 0 src/api/{client_server => client}/session.rs | 0 src/api/{client_server => client}/space.rs | 0 src/api/{client_server => client}/state.rs | 0 src/api/{client_server => client}/sync.rs | 0 src/api/{client_server => client}/tag.rs | 0 .../{client_server => client}/thirdparty.rs | 0 src/api/{client_server => client}/threads.rs | 0 .../{client_server => client}/to_device.rs | 0 src/api/{client_server => client}/typing.rs | 0 src/api/{client_server => client}/unstable.rs | 0 .../{client_server => client}/unversioned.rs | 0 .../user_directory.rs | 0 src/api/{client_server => client}/voip.rs | 0 src/api/mod.rs | 3 +- src/api/router.rs | 278 +++++++++--------- src/api/server/publicrooms.rs | 9 +- src/api/server/user.rs | 2 +- 42 files changed, 149 insertions(+), 154 deletions(-) rename src/api/{client_server => client}/account.rs (99%) rename src/api/{client_server => client}/alias.rs (100%) rename src/api/{client_server => client}/backup.rs (100%) rename src/api/{client_server => client}/capabilities.rs (100%) rename src/api/{client_server => client}/config.rs (100%) rename src/api/{client_server => client}/context.rs (100%) rename src/api/{client_server => client}/device.rs (100%) rename src/api/{client_server => client}/directory.rs (100%) rename src/api/{client_server => client}/filter.rs (100%) rename src/api/{client_server => client}/keys.rs (100%) rename src/api/{client_server => client}/media.rs (100%) rename src/api/{client_server => client}/membership.rs (100%) rename src/api/{client_server => client}/message.rs (100%) rename src/api/{client_server => client}/mod.rs (100%) rename src/api/{client_server => client}/presence.rs (100%) rename src/api/{client_server => client}/profile.rs (100%) rename src/api/{client_server => client}/push.rs (100%) rename src/api/{client_server => client}/read_marker.rs (100%) rename src/api/{client_server => client}/redact.rs (100%) rename src/api/{client_server => client}/relations.rs (100%) rename src/api/{client_server => client}/report.rs (100%) rename src/api/{client_server => client}/room.rs (99%) rename src/api/{client_server => client}/search.rs (100%) rename src/api/{client_server => client}/session.rs (100%) rename src/api/{client_server => client}/space.rs (100%) rename src/api/{client_server => client}/state.rs (100%) rename src/api/{client_server => client}/sync.rs (100%) rename src/api/{client_server => client}/tag.rs (100%) rename src/api/{client_server => client}/thirdparty.rs (100%) rename src/api/{client_server => client}/threads.rs (100%) rename src/api/{client_server => client}/to_device.rs (100%) rename src/api/{client_server => client}/typing.rs (100%) rename src/api/{client_server => client}/unstable.rs (100%) rename src/api/{client_server => client}/unversioned.rs (100%) rename src/api/{client_server => client}/user_directory.rs (100%) rename src/api/{client_server => client}/voip.rs (100%) diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 46ae0184..f5176147 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -1,6 +1,6 @@ use std::fmt::Write; -use api::client_server::{get_alias_helper, leave_room}; +use api::client::{get_alias_helper, leave_room}; use ruma::{ events::room::message::RoomMessageEventContent, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, RoomOrAliasId, }; diff --git a/src/admin/user/user_commands.rs b/src/admin/user/user_commands.rs index d7fc321d..24f30931 100644 --- a/src/admin/user/user_commands.rs +++ b/src/admin/user/user_commands.rs @@ -1,6 +1,6 @@ use std::{fmt::Write as _, sync::Arc}; -use api::client_server::{join_room_by_id_helper, leave_all_rooms}; +use api::client::{join_room_by_id_helper, leave_all_rooms}; use conduit::utils; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, UserId}; use tracing::{error, info, warn}; diff --git a/src/api/client_server/account.rs b/src/api/client/account.rs similarity index 99% rename from src/api/client_server/account.rs rename to src/api/client/account.rs index fe0c333b..a5ff5d82 100644 --- a/src/api/client_server/account.rs +++ b/src/api/client/account.rs @@ -18,9 +18,8 @@ use ruma::{ }; use tracing::{error, info, warn}; -use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; +use super::{join_room_by_id_helper, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; use crate::{ - client_server::{self, join_room_by_id_helper}, service::user_is_local, services, utils::{self}, @@ -539,7 +538,7 @@ pub(crate) async fn deactivate_route(body: Ruma) -> Res } // Make the user leave all rooms before deactivation - client_server::leave_all_rooms(sender_user).await; + super::leave_all_rooms(sender_user).await; // Remove devices and mark account as deactivated services().users.deactivate_account(sender_user)?; diff --git a/src/api/client_server/alias.rs b/src/api/client/alias.rs similarity index 100% rename from src/api/client_server/alias.rs rename to src/api/client/alias.rs diff --git a/src/api/client_server/backup.rs b/src/api/client/backup.rs similarity index 100% rename from src/api/client_server/backup.rs rename to src/api/client/backup.rs diff --git a/src/api/client_server/capabilities.rs b/src/api/client/capabilities.rs similarity index 100% rename from src/api/client_server/capabilities.rs rename to src/api/client/capabilities.rs diff --git a/src/api/client_server/config.rs b/src/api/client/config.rs similarity index 100% rename from src/api/client_server/config.rs rename to src/api/client/config.rs diff --git a/src/api/client_server/context.rs b/src/api/client/context.rs similarity index 100% rename from src/api/client_server/context.rs rename to src/api/client/context.rs diff --git a/src/api/client_server/device.rs b/src/api/client/device.rs similarity index 100% rename from src/api/client_server/device.rs rename to src/api/client/device.rs diff --git a/src/api/client_server/directory.rs b/src/api/client/directory.rs similarity index 100% rename from src/api/client_server/directory.rs rename to src/api/client/directory.rs diff --git a/src/api/client_server/filter.rs b/src/api/client/filter.rs similarity index 100% rename from src/api/client_server/filter.rs rename to src/api/client/filter.rs diff --git a/src/api/client_server/keys.rs b/src/api/client/keys.rs similarity index 100% rename from src/api/client_server/keys.rs rename to src/api/client/keys.rs diff --git a/src/api/client_server/media.rs b/src/api/client/media.rs similarity index 100% rename from src/api/client_server/media.rs rename to src/api/client/media.rs diff --git a/src/api/client_server/membership.rs b/src/api/client/membership.rs similarity index 100% rename from src/api/client_server/membership.rs rename to src/api/client/membership.rs diff --git a/src/api/client_server/message.rs b/src/api/client/message.rs similarity index 100% rename from src/api/client_server/message.rs rename to src/api/client/message.rs diff --git a/src/api/client_server/mod.rs b/src/api/client/mod.rs similarity index 100% rename from src/api/client_server/mod.rs rename to src/api/client/mod.rs diff --git a/src/api/client_server/presence.rs b/src/api/client/presence.rs similarity index 100% rename from src/api/client_server/presence.rs rename to src/api/client/presence.rs diff --git a/src/api/client_server/profile.rs b/src/api/client/profile.rs similarity index 100% rename from src/api/client_server/profile.rs rename to src/api/client/profile.rs diff --git a/src/api/client_server/push.rs b/src/api/client/push.rs similarity index 100% rename from src/api/client_server/push.rs rename to src/api/client/push.rs diff --git a/src/api/client_server/read_marker.rs b/src/api/client/read_marker.rs similarity index 100% rename from src/api/client_server/read_marker.rs rename to src/api/client/read_marker.rs diff --git a/src/api/client_server/redact.rs b/src/api/client/redact.rs similarity index 100% rename from src/api/client_server/redact.rs rename to src/api/client/redact.rs diff --git a/src/api/client_server/relations.rs b/src/api/client/relations.rs similarity index 100% rename from src/api/client_server/relations.rs rename to src/api/client/relations.rs diff --git a/src/api/client_server/report.rs b/src/api/client/report.rs similarity index 100% rename from src/api/client_server/report.rs rename to src/api/client/report.rs diff --git a/src/api/client_server/room.rs b/src/api/client/room.rs similarity index 99% rename from src/api/client_server/room.rs rename to src/api/client/room.rs index 2c27f569..0cea6ef7 100644 --- a/src/api/client_server/room.rs +++ b/src/api/client/room.rs @@ -28,8 +28,8 @@ use ruma::{ use serde_json::{json, value::to_raw_value}; use tracing::{error, info, warn}; +use super::invite_helper; use crate::{ - client_server::invite_helper, service::{appservice::RegistrationInfo, pdu::PduBuilder}, services, Error, Result, Ruma, }; diff --git a/src/api/client_server/search.rs b/src/api/client/search.rs similarity index 100% rename from src/api/client_server/search.rs rename to src/api/client/search.rs diff --git a/src/api/client_server/session.rs b/src/api/client/session.rs similarity index 100% rename from src/api/client_server/session.rs rename to src/api/client/session.rs diff --git a/src/api/client_server/space.rs b/src/api/client/space.rs similarity index 100% rename from src/api/client_server/space.rs rename to src/api/client/space.rs diff --git a/src/api/client_server/state.rs b/src/api/client/state.rs similarity index 100% rename from src/api/client_server/state.rs rename to src/api/client/state.rs diff --git a/src/api/client_server/sync.rs b/src/api/client/sync.rs similarity index 100% rename from src/api/client_server/sync.rs rename to src/api/client/sync.rs diff --git a/src/api/client_server/tag.rs b/src/api/client/tag.rs similarity index 100% rename from src/api/client_server/tag.rs rename to src/api/client/tag.rs diff --git a/src/api/client_server/thirdparty.rs b/src/api/client/thirdparty.rs similarity index 100% rename from src/api/client_server/thirdparty.rs rename to src/api/client/thirdparty.rs diff --git a/src/api/client_server/threads.rs b/src/api/client/threads.rs similarity index 100% rename from src/api/client_server/threads.rs rename to src/api/client/threads.rs diff --git a/src/api/client_server/to_device.rs b/src/api/client/to_device.rs similarity index 100% rename from src/api/client_server/to_device.rs rename to src/api/client/to_device.rs diff --git a/src/api/client_server/typing.rs b/src/api/client/typing.rs similarity index 100% rename from src/api/client_server/typing.rs rename to src/api/client/typing.rs diff --git a/src/api/client_server/unstable.rs b/src/api/client/unstable.rs similarity index 100% rename from src/api/client_server/unstable.rs rename to src/api/client/unstable.rs diff --git a/src/api/client_server/unversioned.rs b/src/api/client/unversioned.rs similarity index 100% rename from src/api/client_server/unversioned.rs rename to src/api/client/unversioned.rs diff --git a/src/api/client_server/user_directory.rs b/src/api/client/user_directory.rs similarity index 100% rename from src/api/client_server/user_directory.rs rename to src/api/client/user_directory.rs diff --git a/src/api/client_server/voip.rs b/src/api/client/voip.rs similarity index 100% rename from src/api/client_server/voip.rs rename to src/api/client/voip.rs diff --git a/src/api/mod.rs b/src/api/mod.rs index a42ba551..2fef4dbf 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,4 +1,4 @@ -pub mod client_server; +pub mod client; pub mod router; mod ruma_wrapper; pub mod server; @@ -6,7 +6,6 @@ pub mod server; extern crate conduit_core as conduit; extern crate conduit_service as service; -pub use client_server::membership::{join_room_by_id_helper, leave_all_rooms}; pub(crate) use conduit::{debug_info, debug_warn, utils, Error, Result}; pub(crate) use ruma_wrapper::{Ruma, RumaResponse}; pub(crate) use service::{pdu::PduEvent, services, user_is_local}; diff --git a/src/api/router.rs b/src/api/router.rs index ad592fc0..3157f10d 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -7,184 +7,184 @@ use conduit::{Error, Server}; use http::Uri; use ruma::api::client::error::ErrorKind; -use crate::{client_server, ruma_wrapper::RouterExt, server}; +use crate::{client, ruma_wrapper::RouterExt, server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; let router = router - .ruma_route(client_server::get_supported_versions_route) - .ruma_route(client_server::get_register_available_route) - .ruma_route(client_server::register_route) - .ruma_route(client_server::get_login_types_route) - .ruma_route(client_server::login_route) - .ruma_route(client_server::whoami_route) - .ruma_route(client_server::logout_route) - .ruma_route(client_server::logout_all_route) - .ruma_route(client_server::change_password_route) - .ruma_route(client_server::deactivate_route) - .ruma_route(client_server::third_party_route) - .ruma_route(client_server::request_3pid_management_token_via_email_route) - .ruma_route(client_server::request_3pid_management_token_via_msisdn_route) - .ruma_route(client_server::check_registration_token_validity) - .ruma_route(client_server::get_capabilities_route) - .ruma_route(client_server::get_pushrules_all_route) - .ruma_route(client_server::set_pushrule_route) - .ruma_route(client_server::get_pushrule_route) - .ruma_route(client_server::set_pushrule_enabled_route) - .ruma_route(client_server::get_pushrule_enabled_route) - .ruma_route(client_server::get_pushrule_actions_route) - .ruma_route(client_server::set_pushrule_actions_route) - .ruma_route(client_server::delete_pushrule_route) - .ruma_route(client_server::get_room_event_route) - .ruma_route(client_server::get_room_aliases_route) - .ruma_route(client_server::get_filter_route) - .ruma_route(client_server::create_filter_route) - .ruma_route(client_server::set_global_account_data_route) - .ruma_route(client_server::set_room_account_data_route) - .ruma_route(client_server::get_global_account_data_route) - .ruma_route(client_server::get_room_account_data_route) - .ruma_route(client_server::set_displayname_route) - .ruma_route(client_server::get_displayname_route) - .ruma_route(client_server::set_avatar_url_route) - .ruma_route(client_server::get_avatar_url_route) - .ruma_route(client_server::get_profile_route) - .ruma_route(client_server::set_presence_route) - .ruma_route(client_server::get_presence_route) - .ruma_route(client_server::upload_keys_route) - .ruma_route(client_server::get_keys_route) - .ruma_route(client_server::claim_keys_route) - .ruma_route(client_server::create_backup_version_route) - .ruma_route(client_server::update_backup_version_route) - .ruma_route(client_server::delete_backup_version_route) - .ruma_route(client_server::get_latest_backup_info_route) - .ruma_route(client_server::get_backup_info_route) - .ruma_route(client_server::add_backup_keys_route) - .ruma_route(client_server::add_backup_keys_for_room_route) - .ruma_route(client_server::add_backup_keys_for_session_route) - .ruma_route(client_server::delete_backup_keys_for_room_route) - .ruma_route(client_server::delete_backup_keys_for_session_route) - .ruma_route(client_server::delete_backup_keys_route) - .ruma_route(client_server::get_backup_keys_for_room_route) - .ruma_route(client_server::get_backup_keys_for_session_route) - .ruma_route(client_server::get_backup_keys_route) - .ruma_route(client_server::set_read_marker_route) - .ruma_route(client_server::create_receipt_route) - .ruma_route(client_server::create_typing_event_route) - .ruma_route(client_server::create_room_route) - .ruma_route(client_server::redact_event_route) - .ruma_route(client_server::report_event_route) - .ruma_route(client_server::create_alias_route) - .ruma_route(client_server::delete_alias_route) - .ruma_route(client_server::get_alias_route) - .ruma_route(client_server::join_room_by_id_route) - .ruma_route(client_server::join_room_by_id_or_alias_route) - .ruma_route(client_server::joined_members_route) - .ruma_route(client_server::leave_room_route) - .ruma_route(client_server::forget_room_route) - .ruma_route(client_server::joined_rooms_route) - .ruma_route(client_server::kick_user_route) - .ruma_route(client_server::ban_user_route) - .ruma_route(client_server::unban_user_route) - .ruma_route(client_server::invite_user_route) - .ruma_route(client_server::set_room_visibility_route) - .ruma_route(client_server::get_room_visibility_route) - .ruma_route(client_server::get_public_rooms_route) - .ruma_route(client_server::get_public_rooms_filtered_route) - .ruma_route(client_server::search_users_route) - .ruma_route(client_server::get_member_events_route) - .ruma_route(client_server::get_protocols_route) - .ruma_route(client_server::send_message_event_route) - .ruma_route(client_server::send_state_event_for_key_route) - .ruma_route(client_server::get_state_events_route) - .ruma_route(client_server::get_state_events_for_key_route) + .ruma_route(client::get_supported_versions_route) + .ruma_route(client::get_register_available_route) + .ruma_route(client::register_route) + .ruma_route(client::get_login_types_route) + .ruma_route(client::login_route) + .ruma_route(client::whoami_route) + .ruma_route(client::logout_route) + .ruma_route(client::logout_all_route) + .ruma_route(client::change_password_route) + .ruma_route(client::deactivate_route) + .ruma_route(client::third_party_route) + .ruma_route(client::request_3pid_management_token_via_email_route) + .ruma_route(client::request_3pid_management_token_via_msisdn_route) + .ruma_route(client::check_registration_token_validity) + .ruma_route(client::get_capabilities_route) + .ruma_route(client::get_pushrules_all_route) + .ruma_route(client::set_pushrule_route) + .ruma_route(client::get_pushrule_route) + .ruma_route(client::set_pushrule_enabled_route) + .ruma_route(client::get_pushrule_enabled_route) + .ruma_route(client::get_pushrule_actions_route) + .ruma_route(client::set_pushrule_actions_route) + .ruma_route(client::delete_pushrule_route) + .ruma_route(client::get_room_event_route) + .ruma_route(client::get_room_aliases_route) + .ruma_route(client::get_filter_route) + .ruma_route(client::create_filter_route) + .ruma_route(client::set_global_account_data_route) + .ruma_route(client::set_room_account_data_route) + .ruma_route(client::get_global_account_data_route) + .ruma_route(client::get_room_account_data_route) + .ruma_route(client::set_displayname_route) + .ruma_route(client::get_displayname_route) + .ruma_route(client::set_avatar_url_route) + .ruma_route(client::get_avatar_url_route) + .ruma_route(client::get_profile_route) + .ruma_route(client::set_presence_route) + .ruma_route(client::get_presence_route) + .ruma_route(client::upload_keys_route) + .ruma_route(client::get_keys_route) + .ruma_route(client::claim_keys_route) + .ruma_route(client::create_backup_version_route) + .ruma_route(client::update_backup_version_route) + .ruma_route(client::delete_backup_version_route) + .ruma_route(client::get_latest_backup_info_route) + .ruma_route(client::get_backup_info_route) + .ruma_route(client::add_backup_keys_route) + .ruma_route(client::add_backup_keys_for_room_route) + .ruma_route(client::add_backup_keys_for_session_route) + .ruma_route(client::delete_backup_keys_for_room_route) + .ruma_route(client::delete_backup_keys_for_session_route) + .ruma_route(client::delete_backup_keys_route) + .ruma_route(client::get_backup_keys_for_room_route) + .ruma_route(client::get_backup_keys_for_session_route) + .ruma_route(client::get_backup_keys_route) + .ruma_route(client::set_read_marker_route) + .ruma_route(client::create_receipt_route) + .ruma_route(client::create_typing_event_route) + .ruma_route(client::create_room_route) + .ruma_route(client::redact_event_route) + .ruma_route(client::report_event_route) + .ruma_route(client::create_alias_route) + .ruma_route(client::delete_alias_route) + .ruma_route(client::get_alias_route) + .ruma_route(client::join_room_by_id_route) + .ruma_route(client::join_room_by_id_or_alias_route) + .ruma_route(client::joined_members_route) + .ruma_route(client::leave_room_route) + .ruma_route(client::forget_room_route) + .ruma_route(client::joined_rooms_route) + .ruma_route(client::kick_user_route) + .ruma_route(client::ban_user_route) + .ruma_route(client::unban_user_route) + .ruma_route(client::invite_user_route) + .ruma_route(client::set_room_visibility_route) + .ruma_route(client::get_room_visibility_route) + .ruma_route(client::get_public_rooms_route) + .ruma_route(client::get_public_rooms_filtered_route) + .ruma_route(client::search_users_route) + .ruma_route(client::get_member_events_route) + .ruma_route(client::get_protocols_route) + .ruma_route(client::send_message_event_route) + .ruma_route(client::send_state_event_for_key_route) + .ruma_route(client::get_state_events_route) + .ruma_route(client::get_state_events_for_key_route) // Ruma doesn't have support for multiple paths for a single endpoint yet, and these routes // share one Ruma request / response type pair with {get,send}_state_event_for_key_route .route( "/_matrix/client/r0/rooms/:room_id/state/:event_type", - get(client_server::get_state_events_for_empty_key_route) - .put(client_server::send_state_event_for_empty_key_route), + get(client::get_state_events_for_empty_key_route) + .put(client::send_state_event_for_empty_key_route), ) .route( "/_matrix/client/v3/rooms/:room_id/state/:event_type", - get(client_server::get_state_events_for_empty_key_route) - .put(client_server::send_state_event_for_empty_key_route), + get(client::get_state_events_for_empty_key_route) + .put(client::send_state_event_for_empty_key_route), ) // These two endpoints allow trailing slashes .route( "/_matrix/client/r0/rooms/:room_id/state/:event_type/", - get(client_server::get_state_events_for_empty_key_route) - .put(client_server::send_state_event_for_empty_key_route), + get(client::get_state_events_for_empty_key_route) + .put(client::send_state_event_for_empty_key_route), ) .route( "/_matrix/client/v3/rooms/:room_id/state/:event_type/", - get(client_server::get_state_events_for_empty_key_route) - .put(client_server::send_state_event_for_empty_key_route), + get(client::get_state_events_for_empty_key_route) + .put(client::send_state_event_for_empty_key_route), ) - .ruma_route(client_server::sync_events_route) - .ruma_route(client_server::sync_events_v4_route) - .ruma_route(client_server::get_context_route) - .ruma_route(client_server::get_message_events_route) - .ruma_route(client_server::search_events_route) - .ruma_route(client_server::turn_server_route) - .ruma_route(client_server::send_event_to_device_route) - .ruma_route(client_server::get_media_config_route) - .ruma_route(client_server::get_media_preview_route) - .ruma_route(client_server::create_content_route) + .ruma_route(client::sync_events_route) + .ruma_route(client::sync_events_v4_route) + .ruma_route(client::get_context_route) + .ruma_route(client::get_message_events_route) + .ruma_route(client::search_events_route) + .ruma_route(client::turn_server_route) + .ruma_route(client::send_event_to_device_route) + .ruma_route(client::get_media_config_route) + .ruma_route(client::get_media_preview_route) + .ruma_route(client::create_content_route) // legacy v1 media routes .route( "/_matrix/media/v1/preview_url", - get(client_server::get_media_preview_v1_route) + get(client::get_media_preview_v1_route) ) .route( "/_matrix/media/v1/config", - get(client_server::get_media_config_v1_route) + get(client::get_media_config_v1_route) ) .route( "/_matrix/media/v1/upload", - post(client_server::create_content_v1_route) + post(client::create_content_v1_route) ) .route( "/_matrix/media/v1/download/:server_name/:media_id", - get(client_server::get_content_v1_route) + get(client::get_content_v1_route) ) .route( "/_matrix/media/v1/download/:server_name/:media_id/:file_name", - get(client_server::get_content_as_filename_v1_route) + get(client::get_content_as_filename_v1_route) ) .route( "/_matrix/media/v1/thumbnail/:server_name/:media_id", - get(client_server::get_content_thumbnail_v1_route) + get(client::get_content_thumbnail_v1_route) ) - .ruma_route(client_server::get_content_route) - .ruma_route(client_server::get_content_as_filename_route) - .ruma_route(client_server::get_content_thumbnail_route) - .ruma_route(client_server::get_devices_route) - .ruma_route(client_server::get_device_route) - .ruma_route(client_server::update_device_route) - .ruma_route(client_server::delete_device_route) - .ruma_route(client_server::delete_devices_route) - .ruma_route(client_server::get_tags_route) - .ruma_route(client_server::update_tag_route) - .ruma_route(client_server::delete_tag_route) - .ruma_route(client_server::upload_signing_keys_route) - .ruma_route(client_server::upload_signatures_route) - .ruma_route(client_server::get_key_changes_route) - .ruma_route(client_server::get_pushers_route) - .ruma_route(client_server::set_pushers_route) - // .ruma_route(client_server::third_party_route) - .ruma_route(client_server::upgrade_room_route) - .ruma_route(client_server::get_threads_route) - .ruma_route(client_server::get_relating_events_with_rel_type_and_event_type_route) - .ruma_route(client_server::get_relating_events_with_rel_type_route) - .ruma_route(client_server::get_relating_events_route) - .ruma_route(client_server::get_hierarchy_route) - .ruma_route(client_server::get_mutual_rooms_route) - .ruma_route(client_server::well_known_support) - .ruma_route(client_server::well_known_client) - .route("/_conduwuit/server_version", get(client_server::conduwuit_server_version)) + .ruma_route(client::get_content_route) + .ruma_route(client::get_content_as_filename_route) + .ruma_route(client::get_content_thumbnail_route) + .ruma_route(client::get_devices_route) + .ruma_route(client::get_device_route) + .ruma_route(client::update_device_route) + .ruma_route(client::delete_device_route) + .ruma_route(client::delete_devices_route) + .ruma_route(client::get_tags_route) + .ruma_route(client::update_tag_route) + .ruma_route(client::delete_tag_route) + .ruma_route(client::upload_signing_keys_route) + .ruma_route(client::upload_signatures_route) + .ruma_route(client::get_key_changes_route) + .ruma_route(client::get_pushers_route) + .ruma_route(client::set_pushers_route) + // .ruma_route(client::third_party_route) + .ruma_route(client::upgrade_room_route) + .ruma_route(client::get_threads_route) + .ruma_route(client::get_relating_events_with_rel_type_and_event_type_route) + .ruma_route(client::get_relating_events_with_rel_type_route) + .ruma_route(client::get_relating_events_route) + .ruma_route(client::get_hierarchy_route) + .ruma_route(client::get_mutual_rooms_route) + .ruma_route(client::well_known_support) + .ruma_route(client::well_known_client) + .route("/_conduwuit/server_version", get(client::conduwuit_server_version)) .route("/_matrix/client/r0/rooms/:room_id/initialSync", get(initial_sync)) .route("/_matrix/client/v3/rooms/:room_id/initialSync", get(initial_sync)) - .route("/client/server.json", get(client_server::syncv3_client_server_json)); + .route("/client/server.json", get(client::syncv3_client_server_json)); if config.allow_federation { router @@ -214,7 +214,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(server::claim_keys_route) .ruma_route(server::get_hierarchy_route) .ruma_route(server::well_known_server) - .route("/_conduwuit/local_user_count", get(client_server::conduwuit_local_user_count)) + .route("/_conduwuit/local_user_count", get(client::conduwuit_local_user_count)) } else { router .route("/_matrix/federation/*path", any(federation_disabled)) diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index 2a1a2ab9..5a91f018 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -6,10 +6,7 @@ use ruma::{ directory::Filter, }; -use crate::{ - client_server::{self}, - services, Error, Result, Ruma, -}; +use crate::{services, Error, Result, Ruma}; /// # `POST /_matrix/federation/v1/publicRooms` /// @@ -24,7 +21,7 @@ pub(crate) async fn get_public_rooms_filtered_route( return Err(Error::BadRequest(ErrorKind::forbidden(), "Room directory is not public")); } - let response = client_server::get_public_rooms_filtered_helper( + let response = crate::client::get_public_rooms_filtered_helper( None, body.limit, body.since.as_deref(), @@ -55,7 +52,7 @@ pub(crate) async fn get_public_rooms_route( return Err(Error::BadRequest(ErrorKind::forbidden(), "Room directory is not public")); } - let response = client_server::get_public_rooms_filtered_helper( + let response = crate::client::get_public_rooms_filtered_helper( None, body.limit, body.since.as_deref(), diff --git a/src/api/server/user.rs b/src/api/server/user.rs index 0176de75..5bfc070c 100644 --- a/src/api/server/user.rs +++ b/src/api/server/user.rs @@ -7,7 +7,7 @@ use ruma::api::{ }; use crate::{ - client_server::{claim_keys_helper, get_keys_helper}, + client::{claim_keys_helper, get_keys_helper}, service::user_is_local, services, Error, Result, Ruma, }; From b2e56777af3b1dc892dd4a6a932f2c28ced90ad9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 27 May 2024 22:40:51 +0000 Subject: [PATCH 0079/2091] fix missing toolchain in cargo smoketest run invocation. Signed-off-by: Jason Volk --- tests/cargo_smoke.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cargo_smoke.sh b/tests/cargo_smoke.sh index 6971f560..e0eb3ad3 100755 --- a/tests/cargo_smoke.sh +++ b/tests/cargo_smoke.sh @@ -20,7 +20,7 @@ conduwuit () { UWU_OPTS=$@ rm -rf /tmp/uwu_smoketest.db echo -e "[global]\nserver_name = \"localhost\"\ndatabase_path = \"/tmp/uwu_smoketest.db\"" > /tmp/uwu_smoketest.toml - cargo run $UWU_OPTS -- -c /tmp/uwu_smoketest.toml & + cargo $UWU_OPTS -- -c /tmp/uwu_smoketest.toml & sleep 15s kill -QUIT %1 wait %1 @@ -37,7 +37,7 @@ element () { run cargo "$TOOLCHAIN" test $ELEMENT_OPTS --all-targets run cargo "$TOOLCHAIN" bench $ELEMENT_OPTS --all-targets run cargo "$TOOLCHAIN" run $ELEMENT_OPTS --bin conduit -- -V - run conduwuit $ELEMENT_OPTS --bin conduit + run conduwuit "$TOOLCHAIN" run $ELEMENT_OPTS --bin conduit fi } From dd49b3c3a1ede82b0324d5755831354a6463a4f6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 3 Jun 2024 02:23:27 +0000 Subject: [PATCH 0080/2091] fix/simplify emergency access initialization Signed-off-by: Jason Volk --- src/service/globals/emerg_access.rs | 47 +++++++++++------------------ src/service/services.rs | 6 ++-- 2 files changed, 19 insertions(+), 34 deletions(-) diff --git a/src/service/globals/emerg_access.rs b/src/service/globals/emerg_access.rs index bda664d6..fdf803cf 100644 --- a/src/service/globals/emerg_access.rs +++ b/src/service/globals/emerg_access.rs @@ -1,9 +1,6 @@ use conduit::Result; use ruma::{ - events::{ - push_rules::PushRulesEventContent, room::message::RoomMessageEventContent, GlobalAccountDataEvent, - GlobalAccountDataEventType, - }, + events::{push_rules::PushRulesEventContent, GlobalAccountDataEvent, GlobalAccountDataEventType}, push::Ruleset, UserId, }; @@ -11,28 +8,11 @@ use tracing::{error, warn}; use crate::services; -pub(crate) async fn init_emergency_access() { - // Set emergency access for the conduit user - match set_emergency_access() { - Ok(pwd_set) => { - if pwd_set { - warn!( - "The Conduit account emergency password is set! Please unset it as soon as you finish admin \ - account recovery!" - ); - services() - .admin - .send_message(RoomMessageEventContent::text_plain( - "The Conduit account emergency password is set! Please unset it as soon as you finish admin \ - account recovery!", - )) - .await; - } - }, - Err(e) => { - error!("Could not set the configured emergency password for the conduit user: {}", e); - }, - }; +/// Set emergency access for the conduit user +pub(crate) fn init_emergency_access() { + if let Err(e) = set_emergency_access() { + error!("Could not set the configured emergency password for the conduit user: {e}"); + } } /// Sets the emergency password and push rules for the @conduit account in case @@ -45,9 +25,9 @@ fn set_emergency_access() -> Result { .users .set_password(&conduit_user, services().globals.emergency_password().as_deref())?; - let (ruleset, res) = match services().globals.emergency_password() { - Some(_) => (Ruleset::server_default(&conduit_user), Ok(true)), - None => (Ruleset::new(), Ok(false)), + let (ruleset, pwd_set) = match services().globals.emergency_password() { + Some(_) => (Ruleset::server_default(&conduit_user), true), + None => (Ruleset::new(), false), }; services().account_data.update( @@ -62,5 +42,12 @@ fn set_emergency_access() -> Result { .expect("to json value always works"), )?; - res + if pwd_set { + warn!( + "The Conduit account emergency password is set! Please unset it as soon as you finish admin account \ + recovery!" + ); + } + + Ok(pwd_set) } diff --git a/src/service/services.rs b/src/service/services.rs index 05894753..4883d574 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -276,14 +276,12 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} pub async fn start(&self) -> Result<()> { debug_info!("Starting services"); + globals::migrations::migrations(&self.db, &self.globals.config).await?; + globals::emerg_access::init_emergency_access(); self.admin.start_handler().await; - - globals::emerg_access::init_emergency_access().await; - self.sending.start_handler().await; - if self.globals.config.allow_local_presence { self.presence.start_handler().await; } From 9bb52cb3ec2149656bbca78b69596b1a1307d481 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 3 Jun 2024 01:32:57 +0000 Subject: [PATCH 0081/2091] add missing dev_release_log_level feature projection Signed-off-by: Jason Volk --- src/main/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 373ea878..ed5a12a6 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -59,6 +59,7 @@ brotli_compression = [ "conduit-service/brotli_compression", ] dev_release_log_level = [ + "conduit-admin/dev_release_log_level", "conduit-api/dev_release_log_level", "conduit-core/dev_release_log_level", "conduit-database/dev_release_log_level", From 76a4d8aa4c20d1f038474e0bac89f284cf091b3c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 3 Jun 2024 06:54:58 +0000 Subject: [PATCH 0082/2091] additional clippy configuration these are documentary values seeking review, not final standards set for the project. Signed-off-by: Jason Volk --- Cargo.toml | 5 ++--- clippy.toml | 8 +++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 254b6513..e78382c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -645,7 +645,7 @@ incremental = false [workspace.lints.rust] absolute-paths-not-starting-with-crate = "warn" -#box-pointers = "warn" # TODO +#box-pointers = "warn" deprecated-in-future = "warn" elided-lifetimes-in-paths = "warn" explicit-outlives-requirements = "warn" @@ -711,9 +711,8 @@ correctness = "warn" ################### nursery = "warn" -### some sadness +## some sadness branches_sharing_code = { level = "allow", priority = 1 } # TODO -cognitive_complexity = { level = "allow", priority = 1 } # TODO derive_partial_eq_without_eq = { level = "allow", priority = 1 } # TODO equatable_if_let = { level = "allow", priority = 1 } # TODO future_not_send = { level = "allow", priority = 1 } # TODO diff --git a/clippy.toml b/clippy.toml index afa92de4..0a04ecd2 100644 --- a/clippy.toml +++ b/clippy.toml @@ -1 +1,7 @@ -too-many-lines-threshold = 700 +array-size-threshold = 4096 +cognitive-complexity-threshold = 94 # TODO reduce me ALARA +excessive-nesting-threshold = 11 # TODO reduce me to 4 or 5 +future-size-threshold = 7745 # TODO reduce me ALARA +stack-size-threshold = 178030 # reduce me ALARA +too-many-lines-threshold = 700 # TODO reduce me to <= 100 +type-complexity-threshold = 250 # reduce me to ~200 From 0e3d192ad277fcd0749045631f9172bdaf4ecc6d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 3 Jun 2024 10:44:46 +0000 Subject: [PATCH 0083/2091] fix trivial-casts Signed-off-by: Jason Volk --- Cargo.toml | 2 +- src/service/globals/resolver.rs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e78382c3..3a12069b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -662,7 +662,7 @@ rust-2021-incompatible-or-patterns = "warn" rust-2021-prefixes-incompatible-syntax = "warn" rust-2021-prelude-collisions = "warn" single-use-lifetimes = "warn" -#trivial-casts = "warn" # TODO +trivial-casts = "warn" trivial-numeric-casts = "warn" unit-bindings = "warn" #unnameable-types = "warn" # TODO diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index d109d389..e90201ed 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -129,6 +129,8 @@ fn resolve_to_reqwest(resolver: Arc, name: Name) -> Resolvin .into_iter() .map(|ip| SocketAddr::new(ip, 0)); - Ok(Box::new(results) as Addrs) + let results: Addrs = Box::new(results); + + Ok(results) }) } From 73718a1208e9dcbce335e2709d674b583c53fd35 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 4 Jun 2024 18:30:51 +0000 Subject: [PATCH 0084/2091] elminate generic argument in ruma_wrapper::auth Signed-off-by: Jason Volk --- src/api/ruma_wrapper/auth.rs | 8 ++------ src/api/ruma_wrapper/request.rs | 3 ++- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/api/ruma_wrapper/auth.rs b/src/api/ruma_wrapper/auth.rs index 282248c1..43737d20 100644 --- a/src/api/ruma_wrapper/auth.rs +++ b/src/api/ruma_wrapper/auth.rs @@ -4,7 +4,7 @@ use axum::RequestPartsExt; use axum_extra::{headers::Authorization, typed_header::TypedHeaderRejectionReason, TypedHeader}; use http::uri::PathAndQuery; use ruma::{ - api::{client::error::ErrorKind, AuthScheme, IncomingRequest}, + api::{client::error::ErrorKind, AuthScheme}, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; use tracing::warn; @@ -26,11 +26,7 @@ pub(super) struct Auth { pub(super) appservice_info: Option, } -pub(super) async fn auth(request: &mut Request) -> Result -where - T: IncomingRequest, -{ - let metadata = T::METADATA; +pub(super) async fn auth(request: &mut Request, metadata: &ruma::api::Metadata) -> Result { let token = match &request.auth { Some(TypedHeader(Authorization(bearer))) => Some(bearer.token()), None => request.query.access_token.as_deref(), diff --git a/src/api/ruma_wrapper/request.rs b/src/api/ruma_wrapper/request.rs index 4a60bbd2..b74fbd3e 100644 --- a/src/api/ruma_wrapper/request.rs +++ b/src/api/ruma_wrapper/request.rs @@ -45,8 +45,9 @@ where type Rejection = Error; async fn from_request(request: hyper::Request, _state: &S) -> Result { + let meta = T::METADATA; let mut request: Request = extract(request).await?; - let auth: Auth = auth::auth::(&mut request).await?; + let auth: Auth = auth::auth(&mut request, &meta).await?; let body = make_body::(&mut request, &auth)?; Ok(Ruma { body, From 427aa4645c6b8ba200a081cf6723c9848182e916 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 4 Jun 2024 20:16:36 +0000 Subject: [PATCH 0085/2091] cleanup/reduce some tracing spans. Signed-off-by: Jason Volk --- src/service/sending/sender.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 9192fdca..39b7b600 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -159,7 +159,7 @@ impl Service { } } - #[tracing::instrument(skip(self, dest, new_events, statuses))] + #[tracing::instrument(skip_all)] fn select_events( &self, dest: &Destination, @@ -206,7 +206,7 @@ impl Service { Ok(Some(events)) } - #[tracing::instrument(skip(self, dest, statuses))] + #[tracing::instrument(skip_all)] fn select_events_current(&self, dest: Destination, statuses: &mut CurTransactionStatus) -> Result<(bool, bool)> { let (mut allow, mut retry) = (true, false); statuses @@ -234,7 +234,7 @@ impl Service { Ok((allow, retry)) } - #[tracing::instrument(skip(self, server_name))] + #[tracing::instrument(skip_all)] fn select_edus(&self, server_name: &ServerName) -> Result<(Vec>, u64)> { // u64: count of last edu let since = self.db.get_latest_educount(server_name)?; @@ -285,7 +285,6 @@ impl Service { } /// Look for presence -#[tracing::instrument(skip(server_name, since, max_edu_count, events))] fn select_edus_presence( server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, ) -> Result { @@ -330,7 +329,6 @@ fn select_edus_presence( } /// Look for read receipts in this room -#[tracing::instrument(skip(room_id, since, max_edu_count, events))] fn select_edus_receipts( room_id: &RoomId, since: u64, max_edu_count: &mut u64, events: &mut Vec>, ) -> Result { From e4aa20ebebd1740b4f2b0dee7969785b95d08a65 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 5 Jun 2024 05:41:29 +0000 Subject: [PATCH 0086/2091] move services ctor/dtor detail into service Signed-off-by: Jason Volk --- src/router/run.rs | 28 ++++------------------------ src/service/mod.rs | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/router/run.rs b/src/router/run.rs index 8afb2dc3..4d06442f 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -6,14 +6,12 @@ use tracing::{debug, error, info}; extern crate conduit_admin as admin; extern crate conduit_core as conduit; -extern crate conduit_database as database; extern crate conduit_service as service; use std::sync::atomic::Ordering; use conduit::{debug_info, trace, Error, Result, Server}; -use database::KeyValueDatabase; -use service::{services, Services}; +use service::services; use crate::{layers, serve}; @@ -58,15 +56,10 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { /// Async initializations #[tracing::instrument(skip_all)] -#[allow(clippy::let_underscore_must_use)] pub(crate) async fn start(server: Arc) -> Result<(), Error> { debug!("Starting..."); - let d = Arc::new(KeyValueDatabase::load_or_create(&server).await?); - let s = Box::new(Services::build(server, d.clone()).await?); - _ = service::SERVICES - .write() - .expect("write locked") - .insert(Box::leak(s)); + + service::init(&server).await?; services().start().await?; #[cfg(feature = "systemd")] @@ -84,23 +77,10 @@ pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { // Wait for all completions before dropping or we'll lose them to the module // unload and explode. services().shutdown().await; - // Deactivate services(). Any further use will panic the caller. - let s = service::SERVICES - .write() - .expect("write locked") - .take() - .unwrap(); + service::fini(); - let s: *mut Services = std::ptr::from_ref(s).cast_mut(); - //SAFETY: Services was instantiated in start() and leaked into the SERVICES - // global perusing as 'static for the duration of run_server(). Now we reclaim - // it to drop it before unloading the module. If this is not done there will be - // multiple instances after module reload. - let s = unsafe { Box::from_raw(s) }; debug!("Cleaning up..."); - // Drop it so we encounter any trouble before the infolog message - drop(s); #[cfg(feature = "systemd")] sd_notify::notify(true, &[sd_notify::NotifyState::Stopping]).expect("failed to notify systemd of stopping state"); diff --git a/src/service/mod.rs b/src/service/mod.rs index 028ed959..2178316a 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -17,9 +17,9 @@ pub mod users; extern crate conduit_core as conduit; extern crate conduit_database as database; -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; -pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, PduCount, Result}; +pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, PduCount, Result, Server}; pub(crate) use database::KeyValueDatabase; pub use globals::{server_is_ours, user_is_local}; pub use pdu::PduEvent; @@ -30,7 +30,35 @@ pub(crate) use crate as service; conduit::mod_ctor! {} conduit::mod_dtor! {} -pub static SERVICES: RwLock> = RwLock::new(None); +static SERVICES: RwLock> = RwLock::new(None); + +#[allow(clippy::let_underscore_must_use)] +pub async fn init(server: &Arc) -> Result<()> { + let d = Arc::new(KeyValueDatabase::load_or_create(server).await?); + let s = Box::new(Services::build(server.clone(), d.clone()).await?); + _ = SERVICES.write().expect("write locked").insert(Box::leak(s)); + + Ok(()) +} + +pub fn fini() { + // Deactivate services(). Any further use will panic the caller. + let s = SERVICES + .write() + .expect("write locked") + .take() + .expect("services initialized"); + + let s: *mut Services = std::ptr::from_ref(s).cast_mut(); + //SAFETY: Services was instantiated in init() and leaked into the SERVICES + // global perusing as 'static for the duration of service. Now we reclaim + // it to drop it before unloading the module. If this is not done there wil + // be multiple instances after module reload. + let s = unsafe { Box::from_raw(s) }; + + // Drop it so we encounter any trouble before the infolog message + drop(s); +} #[must_use] pub fn services() -> &'static Services { From 0e74ade7d799d5c09836b15e57642b6d91c76f28 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 5 Jun 2024 19:59:50 +0000 Subject: [PATCH 0087/2091] isolate axum shutdown in router; minor run-cycle/signalling tweaks Signed-off-by: Jason Volk --- Cargo.lock | 1 - src/core/Cargo.toml | 1 - src/core/server.rs | 30 +++++++++++----------------- src/main/main.rs | 43 ++++++++++++++++++++++------------------ src/main/mods.rs | 2 +- src/router/request.rs | 4 ++-- src/router/run.rs | 44 ++++++++++++++++------------------------- src/service/services.rs | 4 ++-- 8 files changed, 57 insertions(+), 72 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 23457c4e..2f5d12b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -658,7 +658,6 @@ version = "0.4.1" dependencies = [ "argon2", "axum 0.7.5", - "axum-server", "bytes", "either", "figment", diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index f2ccf43e..c90a03b3 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -61,7 +61,6 @@ sentry_telemetry = [] [dependencies] argon2.workspace = true -axum-server.workspace = true axum.workspace = true bytes.workspace = true either.workspace = true diff --git a/src/core/server.rs b/src/core/server.rs index 9e4497ae..6cf477cb 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -1,8 +1,5 @@ use std::{ - sync::{ - atomic::{AtomicBool, AtomicU32}, - Mutex, - }, + sync::atomic::{AtomicBool, AtomicU32}, time::SystemTime, }; @@ -18,24 +15,20 @@ pub struct Server { /// Timestamp server was started; used for uptime. pub started: SystemTime, - /// Reload/shutdown signal channel. Called from the signal handler or admin - /// command to initiate shutdown. - pub shutdown: Mutex>, - - /// Reload/shutdown signal - pub signal: broadcast::Sender<&'static str>, + /// Reload/shutdown pending indicator; server is shutting down. This is an + /// observable used on shutdown and should not be modified. + pub stopping: AtomicBool, /// Reload/shutdown desired indicator; when false, shutdown is desired. This /// is an observable used on shutdown and modifying is not recommended. - pub reload: AtomicBool, - - /// Reload/shutdown pending indicator; server is shutting down. This is an - /// observable used on shutdown and should not be modified. - pub interrupt: AtomicBool, + pub reloading: AtomicBool, /// Handle to the runtime pub runtime: Option, + /// Reload/shutdown signal + pub signal: broadcast::Sender<&'static str>, + /// Log level reload handles. pub tracing_reload_handle: LogLevelReloadHandles, @@ -53,11 +46,10 @@ impl Server { Self { config, started: SystemTime::now(), - shutdown: Mutex::new(None), - signal: broadcast::channel::<&'static str>(1).0, - reload: AtomicBool::new(false), - interrupt: AtomicBool::new(false), + stopping: AtomicBool::new(false), + reloading: AtomicBool::new(false), runtime, + signal: broadcast::channel::<&'static str>(1).0, tracing_reload_handle, requests_spawn_active: AtomicU32::new(0), requests_spawn_finished: AtomicU32::new(0), diff --git a/src/main/main.rs b/src/main/main.rs index 73e6e690..7933538d 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -6,10 +6,9 @@ extern crate conduit_core as conduit; use std::{cmp, sync::Arc, time::Duration}; -use conduit::{debug_error, debug_info, error, utils::available_parallelism, warn, Error, Result}; +use conduit::{debug_error, debug_info, error, trace, utils::available_parallelism, warn, Error, Result}; use server::Server; use tokio::{runtime, signal}; -use tracing::debug; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; @@ -97,33 +96,39 @@ async fn async_main(server: &Arc) -> Result<(), Error> { Ok(()) } +#[cfg(unix)] #[tracing::instrument(skip_all)] async fn signal(server: Arc) { - let (mut term, mut quit); - #[cfg(unix)] - { - use signal::unix; - quit = unix::signal(unix::SignalKind::quit()).expect("SIGQUIT handler"); - term = unix::signal(unix::SignalKind::terminate()).expect("SIGTERM handler"); - }; - + use signal::unix; + let mut quit = unix::signal(unix::SignalKind::quit()).expect("SIGQUIT handler"); + let mut term = unix::signal(unix::SignalKind::terminate()).expect("SIGTERM handler"); loop { - debug!("Installed signal handlers"); + trace!("Installed signal handlers"); let sig: &'static str; - #[cfg(unix)] tokio::select! { + _ = signal::ctrl_c() => { sig = "SIGINT"; }, + _ = quit.recv() => { sig = "SIGQUIT"; }, _ = term.recv() => { sig = "SIGTERM"; }, - _ = quit.recv() => { sig = "Ctrl+\\"; }, - _ = signal::ctrl_c() => { sig = "Ctrl+C"; }, - } - #[cfg(not(unix))] - tokio::select! { - _ = signal::ctrl_c() => { sig = "Ctrl+C"; }, } - warn!("Received signal {}", sig); + warn!("Received {sig}"); if let Err(e) = server.server.signal.send(sig) { debug_error!("signal channel: {e}"); } } } + +#[cfg(not(unix))] +#[tracing::instrument(skip_all)] +async fn signal(server: Arc) { + loop { + tokio::select! { + _ = signal::ctrl_c() => { + warn!("Received Ctrl+C"); + if let Err(e) = server.server.signal.send("SIGINT") { + debug_error!("signal channel: {e}"); + } + }, + } + } +} diff --git a/src/main/mods.rs b/src/main/mods.rs index 90a25f97..125cd54a 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -44,7 +44,7 @@ pub(crate) async fn run(server: &Arc, starts: bool) -> Result<(bool, boo error!("Running server: {error}"); return Err(error); } - let reloads = server.server.reload.swap(false, Ordering::AcqRel); + let reloads = server.server.reloading.swap(false, Ordering::AcqRel); let stops = !reloads || stale(server).await? <= restart_thresh(); let starts = reloads && stops; if stops { diff --git a/src/router/request.rs b/src/router/request.rs index 25bdd98e..0693e671 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -13,7 +13,7 @@ use tracing::{debug, error, trace}; pub(crate) async fn spawn( State(server): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { - if server.interrupt.load(Ordering::Relaxed) { + if server.stopping.load(Ordering::Relaxed) { debug_warn!("unavailable pending shutdown"); return Err(StatusCode::SERVICE_UNAVAILABLE); } @@ -35,7 +35,7 @@ pub(crate) async fn spawn( pub(crate) async fn handle( State(server): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { - if server.interrupt.load(Ordering::Relaxed) { + if server.stopping.load(Ordering::Relaxed) { debug_warn!( method = %req.method(), uri = %req.uri(), diff --git a/src/router/run.rs b/src/router/run.rs index 4d06442f..85e058cf 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -25,16 +25,12 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { _ = services().admin.handle.lock().await.insert(admin::handle); // Setup shutdown/signal handling + server.stopping.store(false, Ordering::Release); let handle = ServerHandle::new(); - _ = server - .shutdown - .lock() - .expect("locked") - .insert(handle.clone()); - - server.interrupt.store(false, Ordering::Release); let (tx, _) = broadcast::channel::<()>(1); - let sigs = server.runtime().spawn(signal(server.clone(), tx.clone())); + let sigs = server + .runtime() + .spawn(signal(server.clone(), tx.clone(), handle.clone())); // Serve clients let res = serve::serve(&server, app, handle, tx.subscribe()).await; @@ -43,10 +39,6 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { sigs.abort(); _ = sigs.await; - // Reset the axum handle instance; this should be reusable and might be - // reload-survivable but better to be safe than sorry. - _ = server.shutdown.lock().expect("locked").take(); - // Remove the admin room callback _ = services().admin.handle.lock().await.take(); @@ -76,7 +68,7 @@ pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { // Wait for all completions before dropping or we'll lose them to the module // unload and explode. - services().shutdown().await; + services().stop().await; // Deactivate services(). Any further use will panic the caller. service::fini(); @@ -90,7 +82,7 @@ pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { } #[tracing::instrument(skip_all)] -async fn signal(server: Arc, tx: Sender<()>) { +async fn signal(server: Arc, tx: Sender<()>, handle: axum_server::Handle) { let sig: &'static str = server .signal .subscribe() @@ -99,26 +91,24 @@ async fn signal(server: Arc, tx: Sender<()>) { .expect("channel error"); debug!("Received signal {}", sig); - if sig == "Ctrl+C" { + if sig == "SIGINT" { let reload = cfg!(unix) && cfg!(debug_assertions); - server.reload.store(reload, Ordering::Release); + server.reloading.store(reload, Ordering::Release); } - server.interrupt.store(true, Ordering::Release); + server.stopping.store(true, Ordering::Release); services().globals.rotate.fire(); if let Err(e) = tx.send(()) { error!("failed sending shutdown transaction to channel: {e}"); } - if let Some(handle) = server.shutdown.lock().expect("locked").as_ref() { - let pending = server.requests_spawn_active.load(Ordering::Relaxed); - if pending > 0 { - let timeout = Duration::from_secs(36); - trace!(pending, ?timeout, "Notifying for graceful shutdown"); - handle.graceful_shutdown(Some(timeout)); - } else { - debug!(pending, "Notifying for immediate shutdown"); - handle.shutdown(); - } + let pending = server.requests_spawn_active.load(Ordering::Relaxed); + if pending > 0 { + let timeout = Duration::from_secs(36); + trace!(pending, ?timeout, "Notifying for graceful shutdown"); + handle.graceful_shutdown(Some(timeout)); + } else { + debug!(pending, "Notifying for immediate shutdown"); + handle.shutdown(); } } diff --git a/src/service/services.rs b/src/service/services.rs index 4883d574..a62fa7e1 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -301,7 +301,7 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} pub async fn interrupt(&self) { trace!("Interrupting services..."); - self.server.interrupt.store(true, atomic::Ordering::Release); + self.server.stopping.store(true, atomic::Ordering::Release); self.globals.rotate.fire(); self.sending.interrupt(); @@ -312,7 +312,7 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} } #[tracing::instrument(skip_all)] - pub async fn shutdown(&self) { + pub async fn stop(&self) { info!("Shutting down services"); self.interrupt().await; From 6e59135a7d776d7aa124e564aef8b31d4d164248 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 5 Jun 2024 21:57:10 +0000 Subject: [PATCH 0088/2091] eliminate RotationHandler Signed-off-by: Jason Volk --- src/core/server.rs | 5 ++++- src/router/run.rs | 1 - src/service/globals/data.rs | 8 +++++++- src/service/globals/mod.rs | 34 +--------------------------------- src/service/services.rs | 1 - 5 files changed, 12 insertions(+), 37 deletions(-) diff --git a/src/core/server.rs b/src/core/server.rs index 6cf477cb..7be6fa58 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -1,5 +1,5 @@ use std::{ - sync::atomic::{AtomicBool, AtomicU32}, + sync::atomic::{AtomicBool, AtomicU32, Ordering}, time::SystemTime, }; @@ -65,4 +65,7 @@ impl Server { .as_ref() .expect("runtime handle available in Server") } + + #[inline] + pub fn running(&self) -> bool { !self.stopping.load(Ordering::Acquire) } } diff --git a/src/router/run.rs b/src/router/run.rs index 85e058cf..80a7d1ee 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -97,7 +97,6 @@ async fn signal(server: Arc, tx: Sender<()>, handle: axum_server::Handle } server.stopping.store(true, Ordering::Release); - services().globals.rotate.fire(); if let Err(e) = tx.send(()) { error!("failed sending shutdown transaction to channel: {e}"); } diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 354ff2f7..b49fc4e9 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -159,7 +159,13 @@ impl Data for KeyValueDatabase { // One time keys futures.push(self.userid_lastonetimekeyupdate.watch_prefix(&userid_bytes)); - futures.push(Box::pin(services().globals.rotate.watch())); + futures.push(Box::pin(async move { + let _result = services().server.signal.subscribe().recv().await; + })); + + if !services().server.running() { + return Ok(()); + } // Wait until one of them finds something trace!(futures = futures.len(), "watch started"); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 7543741f..e806a6b0 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -8,7 +8,6 @@ pub(super) mod updates; use std::{ collections::{BTreeMap, HashMap}, fs, - future::Future, path::PathBuf, sync::Arc, time::Instant, @@ -29,7 +28,7 @@ use ruma::{ ServerName, UserId, }; use tokio::{ - sync::{broadcast, Mutex, RwLock}, + sync::{Mutex, RwLock}, task::JoinHandle, }; use tracing::{error, trace}; @@ -59,36 +58,6 @@ pub struct Service { pub roomid_federationhandletime: RwLock>, pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, - pub rotate: RotationHandler, -} - -/// Handles "rotation" of long-polling requests. "Rotation" in this context is -/// similar to "rotation" of log files and the like. -/// -/// This is utilized to have sync workers return early and release read locks on -/// the database. -pub struct RotationHandler(broadcast::Sender<()>, ()); - -impl RotationHandler { - fn new() -> Self { - let (s, _r) = broadcast::channel(1); - Self(s, ()) - } - - pub fn watch(&self) -> impl Future { - let mut r = self.0.subscribe(); - #[allow(clippy::let_underscore_must_use)] - async move { - _ = r.recv().await; - } - } - - #[allow(clippy::let_underscore_must_use)] - pub fn fire(&self) { _ = self.0.send(()); } -} - -impl Default for RotationHandler { - fn default() -> Self { Self::new() } } impl Service { @@ -149,7 +118,6 @@ impl Service { roomid_federationhandletime: RwLock::new(HashMap::new()), updates_handle: Mutex::new(None), stateres_mutex: Arc::new(Mutex::new(())), - rotate: RotationHandler::new(), }; fs::create_dir_all(s.get_media_folder())?; diff --git a/src/service/services.rs b/src/service/services.rs index a62fa7e1..d9805010 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -303,7 +303,6 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} trace!("Interrupting services..."); self.server.stopping.store(true, atomic::Ordering::Release); - self.globals.rotate.fire(); self.sending.interrupt(); self.presence.interrupt(); self.admin.interrupt(); From 4a68e28c710b922ddccc897c163f5700d9bf46ae Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Jun 2024 17:44:57 +0000 Subject: [PATCH 0089/2091] use debug_warn for presence spam Signed-off-by: Jason Volk --- src/service/presence/data.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index ad8e55f8..bf730398 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -1,4 +1,4 @@ -use conduit::debug_info; +use conduit::debug_warn; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; use crate::{ @@ -67,7 +67,7 @@ impl Data for KeyValueDatabase { // tighten for state flicker? if !state_changed && last_active_ts <= last_last_active_ts { - debug_info!( + debug_warn!( "presence spam {:?} last_active_ts:{:?} <= {:?}", user_id, last_active_ts, From f871d8fd4ee375b62431d550d18d934e7f8952ac Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Jun 2024 18:04:29 +0000 Subject: [PATCH 0090/2091] move impl FromRequest for Ruma up one level; some cleanup Signed-off-by: Jason Volk --- src/api/ruma_wrapper/auth.rs | 54 ++++---- .../ruma_wrapper/{router.rs => handler.rs} | 0 src/api/ruma_wrapper/mod.rs | 118 ++++++++++++++++-- src/api/ruma_wrapper/request.rs | 113 +---------------- 4 files changed, 142 insertions(+), 143 deletions(-) rename src/api/ruma_wrapper/{router.rs => handler.rs} (100%) diff --git a/src/api/ruma_wrapper/auth.rs b/src/api/ruma_wrapper/auth.rs index 43737d20..35fd443d 100644 --- a/src/api/ruma_wrapper/auth.rs +++ b/src/api/ruma_wrapper/auth.rs @@ -1,10 +1,14 @@ use std::collections::BTreeMap; use axum::RequestPartsExt; -use axum_extra::{headers::Authorization, typed_header::TypedHeaderRejectionReason, TypedHeader}; +use axum_extra::{ + headers::{authorization::Bearer, Authorization}, + typed_header::TypedHeaderRejectionReason, + TypedHeader, +}; use http::uri::PathAndQuery; use ruma::{ - api::{client::error::ErrorKind, AuthScheme}, + api::{client::error::ErrorKind, AuthScheme, Metadata}, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; use tracing::warn; @@ -20,14 +24,17 @@ enum Token { } pub(super) struct Auth { + pub(super) origin: Option, pub(super) sender_user: Option, pub(super) sender_device: Option, - pub(super) origin: Option, pub(super) appservice_info: Option, } -pub(super) async fn auth(request: &mut Request, metadata: &ruma::api::Metadata) -> Result { - let token = match &request.auth { +pub(super) async fn auth( + request: &mut Request, json_body: &Option, metadata: &Metadata, +) -> Result { + let bearer: Option>> = request.parts.extract().await?; + let token = match &bearer { Some(TypedHeader(Authorization(bearer))) => Some(bearer.token()), None => request.query.access_token.as_deref(), }; @@ -78,9 +85,9 @@ pub(super) async fn auth(request: &mut Request, metadata: &ruma::api::Metadata) (AuthScheme::AccessToken, Token::Appservice(info)) => Ok(auth_appservice(request, info)?), (AuthScheme::None | AuthScheme::AccessTokenOptional | AuthScheme::AppserviceToken, Token::Appservice(info)) => { Ok(Auth { + origin: None, sender_user: None, sender_device: None, - origin: None, appservice_info: Some(*info), }) }, @@ -91,12 +98,12 @@ pub(super) async fn auth(request: &mut Request, metadata: &ruma::api::Metadata) AuthScheme::AccessToken | AuthScheme::AccessTokenOptional | AuthScheme::None, Token::User((user_id, device_id)), ) => Ok(Auth { + origin: None, sender_user: Some(user_id), sender_device: Some(device_id), - origin: None, appservice_info: None, }), - (AuthScheme::ServerSignatures, Token::None) => Ok(auth_server(request).await?), + (AuthScheme::ServerSignatures, Token::None) => Ok(auth_server(request, json_body).await?), (AuthScheme::None | AuthScheme::AppserviceToken | AuthScheme::AccessTokenOptional, Token::None) => Ok(Auth { sender_user: None, sender_device: None, @@ -114,7 +121,7 @@ pub(super) async fn auth(request: &mut Request, metadata: &ruma::api::Metadata) } } -fn auth_appservice(request: &mut Request, info: Box) -> Result { +fn auth_appservice(request: &Request, info: Box) -> Result { let user_id = request .query .user_id @@ -139,14 +146,14 @@ fn auth_appservice(request: &mut Request, info: Box) -> Result } Ok(Auth { + origin: None, sender_user: Some(user_id), sender_device: None, - origin: None, appservice_info: Some(*info), }) } -async fn auth_server(request: &mut Request) -> Result { +async fn auth_server(request: &mut Request, json_body: &Option) -> Result { if !services().globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); } @@ -167,15 +174,11 @@ async fn auth_server(request: &mut Request) -> Result { Error::BadRequest(ErrorKind::forbidden(), msg) })?; - let origin_signatures = BTreeMap::from_iter([(x_matrix.key.clone(), CanonicalJsonValue::String(x_matrix.sig))]); - - let signatures = BTreeMap::from_iter([( - x_matrix.origin.as_str().to_owned(), - CanonicalJsonValue::Object(origin_signatures), - )]); + let origin = &x_matrix.origin; + let signatures = BTreeMap::from_iter([(x_matrix.key.clone(), CanonicalJsonValue::String(x_matrix.sig))]); + let signatures = BTreeMap::from_iter([(origin.as_str().to_owned(), CanonicalJsonValue::Object(signatures))]); let server_destination = services().globals.server_name().as_str().to_owned(); - if let Some(destination) = x_matrix.destination.as_ref() { if destination != &server_destination { return Err(Error::BadRequest(ErrorKind::forbidden(), "Invalid authorization.")); @@ -197,22 +200,19 @@ async fn auth_server(request: &mut Request) -> Result { CanonicalJsonValue::String(request.parts.method.to_string()), ), ("uri".to_owned(), signature_uri), - ( - "origin".to_owned(), - CanonicalJsonValue::String(x_matrix.origin.as_str().to_owned()), - ), + ("origin".to_owned(), CanonicalJsonValue::String(origin.as_str().to_owned())), ("destination".to_owned(), CanonicalJsonValue::String(server_destination)), ("signatures".to_owned(), CanonicalJsonValue::Object(signatures)), ]); - if let Some(json_body) = &request.json { + if let Some(json_body) = json_body { request_map.insert("content".to_owned(), json_body.clone()); }; let keys_result = services() .rooms .event_handler - .fetch_signing_keys_for_server(&x_matrix.origin, vec![x_matrix.key.clone()]) + .fetch_signing_keys_for_server(origin, vec![x_matrix.key.clone()]) .await; let keys = keys_result.map_err(|e| { @@ -220,17 +220,17 @@ async fn auth_server(request: &mut Request) -> Result { Error::BadRequest(ErrorKind::forbidden(), "Failed to fetch signing keys.") })?; - let pub_key_map = BTreeMap::from_iter([(x_matrix.origin.as_str().to_owned(), keys)]); + let pub_key_map = BTreeMap::from_iter([(origin.as_str().to_owned(), keys)]); match ruma::signatures::verify_json(&pub_key_map, &request_map) { Ok(()) => Ok(Auth { + origin: Some(origin.clone()), sender_user: None, sender_device: None, - origin: Some(x_matrix.origin), appservice_info: None, }), Err(e) => { - warn!("Failed to verify json request from {}: {e}\n{request_map:?}", x_matrix.origin); + warn!("Failed to verify json request from {origin}: {e}\n{request_map:?}"); if request.parts.uri.to_string().contains('@') { warn!( diff --git a/src/api/ruma_wrapper/router.rs b/src/api/ruma_wrapper/handler.rs similarity index 100% rename from src/api/ruma_wrapper/router.rs rename to src/api/ruma_wrapper/handler.rs diff --git a/src/api/ruma_wrapper/mod.rs b/src/api/ruma_wrapper/mod.rs index a130ddd9..fb633b91 100644 --- a/src/api/ruma_wrapper/mod.rs +++ b/src/api/ruma_wrapper/mod.rs @@ -1,26 +1,69 @@ mod auth; +mod handler; mod request; -mod router; mod xmatrix; -use std::ops::Deref; +use std::{mem, ops::Deref}; +use axum::{async_trait, body::Body, extract::FromRequest}; +use bytes::{BufMut, BytesMut}; pub(super) use conduit::error::RumaResponse; -use ruma::{CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId}; +use conduit::{debug, debug_warn, trace, warn}; +use ruma::{ + api::{client::error::ErrorKind, IncomingRequest}, + CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, +}; -pub(super) use self::router::RouterExt; -use crate::service::appservice::RegistrationInfo; +pub(super) use self::handler::RouterExt; +use self::{auth::Auth, request::Request}; +use crate::{service::appservice::RegistrationInfo, services, Error, Result}; /// Extractor for Ruma request structs pub(crate) struct Ruma { /// Request struct body pub(crate) body: T, - pub(crate) sender_user: Option, - pub(crate) sender_device: Option, - /// X-Matrix origin/server + + /// Federation server authentication: X-Matrix origin + /// None when not a federation server. pub(crate) origin: Option, - pub(crate) json_body: Option, // This is None when body is not a valid string + + /// Local user authentication: user_id. + /// None when not an authenticated local user. + pub(crate) sender_user: Option, + + /// Local user authentication: device_id. + /// None when not an authenticated local user or no device. + pub(crate) sender_device: Option, + + /// Appservice authentication; registration info. + /// None when not an appservice. pub(crate) appservice_info: Option, + + /// Parsed JSON content. + /// None when body is not a valid string + pub(crate) json_body: Option, +} + +#[async_trait] +impl FromRequest for Ruma +where + T: IncomingRequest, +{ + type Rejection = Error; + + async fn from_request(request: hyper::Request, _: &S) -> Result { + let mut request = request::from(request).await?; + let mut json_body = serde_json::from_slice::(&request.body).ok(); + let auth = auth::auth(&mut request, &json_body, &T::METADATA).await?; + Ok(Ruma { + body: make_body::(&mut request, &mut json_body, &auth)?, + origin: auth.origin, + sender_user: auth.sender_user, + sender_device: auth.sender_device, + appservice_info: auth.appservice_info, + json_body, + }) + } } impl Deref for Ruma { @@ -28,3 +71,60 @@ impl Deref for Ruma { fn deref(&self) -> &Self::Target { &self.body } } + +fn make_body(request: &mut Request, json_body: &mut Option, auth: &Auth) -> Result +where + T: IncomingRequest, +{ + let body = if let Some(CanonicalJsonValue::Object(json_body)) = json_body { + let user_id = auth.sender_user.clone().unwrap_or_else(|| { + UserId::parse_with_server_name("", services().globals.server_name()).expect("we know this is valid") + }); + + let uiaa_request = json_body + .get("auth") + .and_then(|auth| auth.as_object()) + .and_then(|auth| auth.get("session")) + .and_then(|session| session.as_str()) + .and_then(|session| { + services().uiaa.get_uiaa_request( + &user_id, + &auth.sender_device.clone().unwrap_or_else(|| "".into()), + session, + ) + }); + + if let Some(CanonicalJsonValue::Object(initial_request)) = uiaa_request { + for (key, value) in initial_request { + json_body.entry(key).or_insert(value); + } + } + + let mut buf = BytesMut::new().writer(); + serde_json::to_writer(&mut buf, &json_body).expect("value serialization can't fail"); + buf.into_inner().freeze() + } else { + mem::take(&mut request.body) + }; + + let mut http_request = hyper::Request::builder() + .uri(request.parts.uri.clone()) + .method(request.parts.method.clone()); + *http_request.headers_mut().unwrap() = request.parts.headers.clone(); + let http_request = http_request.body(body).unwrap(); + debug!( + "{:?} {:?} {:?}", + http_request.method(), + http_request.uri(), + http_request.headers() + ); + + trace!("{:?} {:?} {:?}", http_request.method(), http_request.uri(), json_body); + let body = T::try_from_http_request(http_request, &request.path).map_err(|e| { + warn!("try_from_http_request failed: {e:?}",); + debug_warn!("JSON body: {:?}", json_body); + Error::BadRequest(ErrorKind::BadJson, "Failed to deserialize request.") + })?; + + Ok(body) +} diff --git a/src/api/ruma_wrapper/request.rs b/src/api/ruma_wrapper/request.rs index b74fbd3e..59639eaa 100644 --- a/src/api/ruma_wrapper/request.rs +++ b/src/api/ruma_wrapper/request.rs @@ -1,25 +1,11 @@ -use std::{mem, str}; +use std::str; -use axum::{ - async_trait, - extract::{FromRequest, Path}, - RequestExt, RequestPartsExt, -}; -use axum_extra::{ - headers::{authorization::Bearer, Authorization}, - TypedHeader, -}; -use bytes::{BufMut, Bytes, BytesMut}; -use conduit::debug_warn; +use axum::{extract::Path, RequestExt, RequestPartsExt}; +use bytes::Bytes; use http::request::Parts; -use ruma::{ - api::{client::error::ErrorKind, IncomingRequest}, - CanonicalJsonValue, UserId, -}; +use ruma::api::client::error::ErrorKind; use serde::Deserialize; -use tracing::{debug, trace, warn}; -use super::{auth, auth::Auth, Ruma}; use crate::{services, Error, Result}; #[derive(Deserialize)] @@ -29,100 +15,17 @@ pub(super) struct QueryParams { } pub(super) struct Request { - pub(super) auth: Option>>, pub(super) path: Path>, pub(super) query: QueryParams, - pub(super) json: Option, pub(super) body: Bytes, pub(super) parts: Parts, } -#[async_trait] -impl FromRequest for Ruma -where - T: IncomingRequest, -{ - type Rejection = Error; - - async fn from_request(request: hyper::Request, _state: &S) -> Result { - let meta = T::METADATA; - let mut request: Request = extract(request).await?; - let auth: Auth = auth::auth(&mut request, &meta).await?; - let body = make_body::(&mut request, &auth)?; - Ok(Ruma { - body, - sender_user: auth.sender_user, - sender_device: auth.sender_device, - origin: auth.origin, - json_body: request.json, - appservice_info: auth.appservice_info, - }) - } -} - -fn make_body(request: &mut Request, auth: &Auth) -> Result -where - T: IncomingRequest, -{ - let body = if let Some(CanonicalJsonValue::Object(json_body)) = &mut request.json { - let user_id = auth.sender_user.clone().unwrap_or_else(|| { - UserId::parse_with_server_name("", services().globals.server_name()).expect("we know this is valid") - }); - - let uiaa_request = json_body - .get("auth") - .and_then(|auth| auth.as_object()) - .and_then(|auth| auth.get("session")) - .and_then(|session| session.as_str()) - .and_then(|session| { - services().uiaa.get_uiaa_request( - &user_id, - &auth.sender_device.clone().unwrap_or_else(|| "".into()), - session, - ) - }); - - if let Some(CanonicalJsonValue::Object(initial_request)) = uiaa_request { - for (key, value) in initial_request { - json_body.entry(key).or_insert(value); - } - } - - let mut buf = BytesMut::new().writer(); - serde_json::to_writer(&mut buf, &request.json).expect("value serialization can't fail"); - buf.into_inner().freeze() - } else { - mem::take(&mut request.body) - }; - - let mut http_request = hyper::Request::builder() - .uri(request.parts.uri.clone()) - .method(request.parts.method.clone()); - *http_request.headers_mut().unwrap() = request.parts.headers.clone(); - let http_request = http_request.body(body).unwrap(); - debug!( - "{:?} {:?} {:?}", - http_request.method(), - http_request.uri(), - http_request.headers() - ); - - trace!("{:?} {:?} {:?}", http_request.method(), http_request.uri(), request.json); - let body = T::try_from_http_request(http_request, &request.path).map_err(|e| { - warn!("try_from_http_request failed: {e:?}",); - debug_warn!("JSON body: {:?}", request.json); - Error::BadRequest(ErrorKind::BadJson, "Failed to deserialize request.") - })?; - - Ok(body) -} - -async fn extract(request: hyper::Request) -> Result { +pub(super) async fn from(request: hyper::Request) -> Result { let limited = request.with_limited_body(); let (mut parts, body) = limited.into_parts(); - let auth = parts.extract().await?; - let path = parts.extract().await?; + let path: Path> = parts.extract().await?; let query = serde_html_form::from_str(parts.uri.query().unwrap_or_default()) .map_err(|_| Error::BadRequest(ErrorKind::Unknown, "Failed to read query parameters"))?; @@ -137,13 +40,9 @@ async fn extract(request: hyper::Request) -> Result { .await .map_err(|_| Error::BadRequest(ErrorKind::TooLarge, "Request body too large"))?; - let json = serde_json::from_slice::(&body).ok(); - Ok(Request { - auth, path, query, - json, body, parts, }) From aebae11c8289def05428ac82b6e71f1c143063de Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 7 Jun 2024 00:53:30 +0000 Subject: [PATCH 0091/2091] reintroduce the variadic macro for ruma handler. Signed-off-by: Jason Volk --- src/api/ruma_wrapper/handler.rs | 48 ++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/api/ruma_wrapper/handler.rs b/src/api/ruma_wrapper/handler.rs index f769d2bb..73361989 100644 --- a/src/api/ruma_wrapper/handler.rs +++ b/src/api/ruma_wrapper/handler.rs @@ -1,6 +1,7 @@ use std::future::Future; use axum::{ + extract::FromRequestParts, response::IntoResponse, routing::{on, MethodFilter}, Router, @@ -32,27 +33,38 @@ pub(in super::super) trait RumaHandler { fn add_route(&self, router: Router, path: &str) -> Router; } -impl RumaHandler> for F -where - Req: IncomingRequest + Send + 'static, - F: FnOnce(Ruma) -> Fut + Clone + Send + Sync + 'static, - Fut: Future> + Send, - E: IntoResponse, -{ - fn add_routes(&self, router: Router) -> Router { - Req::METADATA - .history - .all_paths() - .fold(router, |router, path| self.add_route(router, path)) - } +macro_rules! ruma_handler { + ( $($tx:ident),* $(,)? ) => { + #[allow(non_snake_case)] + impl RumaHandler<($($tx,)* Ruma,)> for Fun + where + Req: IncomingRequest + Send + 'static, + Ret: IntoResponse, + Fut: Future> + Send, + Fun: FnOnce($($tx,)* Ruma) -> Fut + Clone + Send + Sync + 'static, + $( $tx: FromRequestParts<()> + Send + 'static, )* + { + fn add_routes(&self, router: Router) -> Router { + Req::METADATA + .history + .all_paths() + .fold(router, |router, path| self.add_route(router, path)) + } - fn add_route(&self, router: Router, path: &str) -> Router { - let handle = self.clone(); - let method = method_to_filter(&Req::METADATA.method); - let action = |req| async { handle(req).await.map(RumaResponse) }; - router.route(path, on(method, action)) + fn add_route(&self, router: Router, path: &str) -> Router { + let handle = self.clone(); + let method = method_to_filter(&Req::METADATA.method); + let action = |$($tx,)* req| async { handle($($tx,)* req).await.map(RumaResponse) }; + router.route(path, on(method, action)) + } + } } } +ruma_handler!(); +ruma_handler!(T1); +ruma_handler!(T1, T2); +ruma_handler!(T1, T2, T3); +ruma_handler!(T1, T2, T3, T4); const fn method_to_filter(method: &Method) -> MethodFilter { match *method { From c2267d4c0359a07f0a06851344d86807870678d4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 5 Jun 2024 07:30:30 +0000 Subject: [PATCH 0092/2091] add services state to router Signed-off-by: Jason Volk --- src/router/router.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/router/router.rs b/src/router/router.rs index 4f45df54..c0a382b6 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -2,13 +2,18 @@ use std::sync::Arc; use axum::{response::IntoResponse, routing::get, Router}; use conduit::{Error, Server}; +use conduit_service as service; use http::Uri; use ruma::api::client::error::ErrorKind; extern crate conduit_api as api; pub(crate) fn build(server: &Arc) -> Router { - let router = Router::new().fallback(not_found).route("/", get(it_works)); + let state = service::services(); + let router = Router::new() + .route("/", get(it_works)) + .fallback(not_found) + .with_state(state); api::router::build(router, server) } From 0bade5317fe38ff87acc69d2e556727610153c35 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Jun 2024 22:31:52 +0000 Subject: [PATCH 0093/2091] add connection info to router Signed-off-by: Jason Volk --- src/router/layers.rs | 7 ++----- src/router/serve/mod.rs | 4 ++-- src/router/serve/plain.rs | 5 +++-- src/router/serve/tls.rs | 5 +++-- src/router/serve/unix.rs | 24 +++++++++++++++++------- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index 38d4ca97..c09f7e87 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -25,7 +25,7 @@ const CONDUWUIT_CSP: &str = "sandbox; default-src 'none'; font-src 'none'; scrip form-action 'none'; base-uri 'none';"; const CONDUWUIT_PERMISSIONS_POLICY: &str = "interest-cohort=(),browsing-topics=()"; -pub(crate) fn build(server: &Arc) -> io::Result> { +pub(crate) fn build(server: &Arc) -> io::Result { let layers = ServiceBuilder::new(); #[cfg(feature = "sentry_telemetry")] @@ -74,10 +74,7 @@ pub(crate) fn build(server: &Arc) -> io::Result, app: IntoMakeService, handle: ServerHandle, shutdown: broadcast::Receiver<()>, + server: &Arc, app: Router, handle: ServerHandle, shutdown: broadcast::Receiver<()>, ) -> Result<(), Error> { let config = &server.config; let addrs = config.get_bind_addrs(); diff --git a/src/router/serve/plain.rs b/src/router/serve/plain.rs index 339f8940..b79d342d 100644 --- a/src/router/serve/plain.rs +++ b/src/router/serve/plain.rs @@ -3,15 +3,16 @@ use std::{ sync::{atomic::Ordering, Arc}, }; -use axum::{routing::IntoMakeService, Router}; +use axum::Router; use axum_server::{bind, Handle as ServerHandle}; use conduit::{debug_info, Result, Server}; use tokio::task::JoinSet; use tracing::info; pub(super) async fn serve( - server: &Arc, app: IntoMakeService, handle: ServerHandle, addrs: Vec, + server: &Arc, app: Router, handle: ServerHandle, addrs: Vec, ) -> Result<()> { + let app = app.into_make_service_with_connect_info::(); let mut join_set = JoinSet::new(); for addr in &addrs { join_set.spawn_on(bind(*addr).handle(handle.clone()).serve(app.clone()), server.runtime()); diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index e4edeb32..6f58ce82 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -1,6 +1,6 @@ use std::{net::SocketAddr, sync::Arc}; -use axum::{routing::IntoMakeService, Router}; +use axum::Router; use axum_server::{bind_rustls, tls_rustls::RustlsConfig, Handle as ServerHandle}; #[cfg(feature = "axum_dual_protocol")] use axum_server_dual_protocol::ServerExt; @@ -9,7 +9,7 @@ use tokio::task::JoinSet; use tracing::{debug, info, warn}; pub(super) async fn serve( - server: &Arc, app: IntoMakeService, handle: ServerHandle, addrs: Vec, + server: &Arc, app: Router, handle: ServerHandle, addrs: Vec, ) -> Result<()> { let config = &server.config; let tls = config.tls.as_ref().expect("TLS configuration"); @@ -31,6 +31,7 @@ pub(super) async fn serve( } let mut join_set = JoinSet::new(); + let app = app.into_make_service_with_connect_info::(); if cfg!(feature = "axum_dual_protocol") && tls.dual_protocol { #[cfg(feature = "axum_dual_protocol")] for addr in &addrs { diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index a526ba29..6c406d28 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -1,8 +1,15 @@ #![cfg(unix)] -use std::{path::Path, sync::Arc}; +use std::{ + net::{self, IpAddr, Ipv4Addr}, + path::Path, + sync::Arc, +}; -use axum::{extract::Request, routing::IntoMakeService, Router}; +use axum::{ + extract::{connect_info::IntoMakeServiceWithConnectInfo, Request}, + Router, +}; use conduit::{debug_error, trace, utils, Error, Result, Server}; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ @@ -19,12 +26,15 @@ use tower::{Service, ServiceExt}; use tracing::{debug, info, warn}; use utils::unwrap_infallible; +type MakeService = IntoMakeServiceWithConnectInfo; + +static NULL_ADDR: net::SocketAddr = net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); + #[tracing::instrument(skip_all)] -pub(super) async fn serve( - server: &Arc, app: IntoMakeService, mut shutdown: broadcast::Receiver<()>, -) -> Result<()> { +pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broadcast::Receiver<()>) -> Result<()> { let mut tasks = JoinSet::<()>::new(); let executor = TokioExecutor::new(); + let app = app.into_make_service_with_connect_info::(); let builder = server::conn::auto::Builder::new(executor); let listener = init(server).await?; loop { @@ -46,14 +56,14 @@ pub(super) async fn serve( #[allow(clippy::let_underscore_must_use)] async fn accept( - server: &Arc, listener: &UnixListener, tasks: &mut JoinSet<()>, mut app: IntoMakeService, + server: &Arc, listener: &UnixListener, tasks: &mut JoinSet<()>, mut app: MakeService, builder: server::conn::auto::Builder, conn: (UnixStream, SocketAddr), ) { let (socket, remote) = conn; let socket = TokioIo::new(socket); trace!(?listener, ?socket, ?remote, "accepted"); - let called = unwrap_infallible(app.call(()).await); + let called = unwrap_infallible(app.call(NULL_ADDR).await); let handler = service_fn(move |req: Request| called.clone().oneshot(req)); let task = async move { From 3ada847570bd71a8cc48e2bc435676ed44216f9b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 7 Jun 2024 01:01:30 +0000 Subject: [PATCH 0094/2091] extract client ip from connection state Signed-off-by: Jason Volk --- Cargo.lock | 29 +++++++++++++++++++++++++++++ Cargo.toml | 3 +++ src/api/Cargo.toml | 1 + src/router/Cargo.toml | 1 + src/router/layers.rs | 2 ++ 5 files changed, 36 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 2f5d12b8..dea29a0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -227,6 +227,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-client-ip" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72188bed20deb981f3a4a9fe674e5980fd9e9c2bd880baa94715ad5d60d64c67" +dependencies = [ + "axum 0.7.5", + "forwarded-header-value", + "serde", +] + [[package]] name = "axum-core" version = "0.3.4" @@ -625,6 +636,7 @@ name = "conduit_api" version = "0.4.1" dependencies = [ "axum 0.7.5", + "axum-client-ip", "axum-extra", "base64 0.22.1", "bytes", @@ -715,6 +727,7 @@ name = "conduit_router" version = "0.4.1" dependencies = [ "axum 0.7.5", + "axum-client-ip", "axum-server", "axum-server-dual-protocol", "bytes", @@ -1100,6 +1113,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty", + "thiserror", +] + [[package]] name = "futf" version = "0.1.5" @@ -2044,6 +2067,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + [[package]] name = "nu-ansi-term" version = "0.46.0" diff --git a/Cargo.toml b/Cargo.toml index 3a12069b..a9c35d8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,6 +83,9 @@ features = ["typed-header"] version = "0.6.0" features = ["tls-rustls"] +[workspace.dependencies.axum-client-ip] +version = "0.6.0" + [workspace.dependencies.tower] version = "0.4.13" features = ["util"] diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 5ac15dd5..fb486a71 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -33,6 +33,7 @@ brotli_compression = [ ] [dependencies] +axum-client-ip.workspace = true axum-extra.workspace = true axum.workspace = true base64.workspace = true diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 6084aa07..f613826b 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -46,6 +46,7 @@ axum_dual_protocol = [ ] [dependencies] +axum-client-ip.workspace = true axum-server-dual-protocol.optional = true axum-server-dual-protocol.workspace = true axum-server.workspace = true diff --git a/src/router/layers.rs b/src/router/layers.rs index c09f7e87..b358a3e6 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -4,6 +4,7 @@ use axum::{ extract::{DefaultBodyLimit, MatchedPath}, Router, }; +use axum_client_ip::SecureClientIpSource; use conduit::Server; use http::{ header::{self, HeaderName}, @@ -46,6 +47,7 @@ pub(crate) fn build(server: &Arc) -> io::Result { .on_response(DefaultOnResponse::new().level(Level::DEBUG)), ) .layer(axum::middleware::from_fn_with_state(Arc::clone(server), request::handle)) + .layer(SecureClientIpSource::ConnectInfo.into_extension()) .layer(SetResponseHeaderLayer::if_not_present( HeaderName::from_static("origin-agent-cluster"), // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin-Agent-Cluster HeaderValue::from_static("?1"), From 1f19356693edffbb57bcb196c3a26f6fd5a8dd8f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 04:40:35 +0000 Subject: [PATCH 0095/2091] chore(deps): update aquasecurity/trivy-action action to v0.22.0 --- .github/workflows/trivy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 510bbfc8..631e010f 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@v4 - name: Run Trivy code and vulnerability scanner on repo - uses: aquasecurity/trivy-action@0.21.0 + uses: aquasecurity/trivy-action@0.22.0 with: scan-type: repo format: sarif @@ -34,7 +34,7 @@ jobs: severity: CRITICAL,HIGH,MEDIUM,LOW - name: Run Trivy code and vulnerability scanner on filesystem - uses: aquasecurity/trivy-action@0.21.0 + uses: aquasecurity/trivy-action@0.22.0 with: scan-type: fs format: sarif From 82a3b737747a3e75d4334f5c2bcd6c57841b20db Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 04:40:31 +0000 Subject: [PATCH 0096/2091] chore(deps): update rust crate clap to v4.5.6 --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dea29a0a..46fb1683 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -538,9 +538,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.4" +version = "4.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +checksum = "a9689a29b593160de5bc4aacab7b5d54fb52231de70122626c178e6a368994c7" dependencies = [ "clap_builder", "clap_derive", @@ -548,9 +548,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.2" +version = "4.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +checksum = "2e5387378c84f6faa26890ebf9f0a92989f8873d4d380467bcd0d8d8620424df" dependencies = [ "anstyle", "clap_lex", @@ -558,9 +558,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.4" +version = "4.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "528131438037fd55894f62d6e9f068b8f45ac57ffa77517819645d10aed04f64" +checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" dependencies = [ "heck 0.5.0", "proc-macro2", From 8d32fb14450ffa49e8a05e4c0f7b37c538792a10 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 7 Jun 2024 04:41:50 +0000 Subject: [PATCH 0097/2091] chore(deps): update sentry-rust monorepo to 0.34.0 --- Cargo.lock | 57 +++++++++++++++++++++++------------------------------- Cargo.toml | 6 +++--- 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46fb1683..7010084d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2717,7 +2717,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 0.26.1", + "webpki-roots", "winreg 0.52.0", ] @@ -3170,13 +3170,13 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "sentry" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b73ab9d5b35ed718611e89db8c886647821cfce79723b9e2e4e7cb3fd9cdd49" +checksum = "5484316556650182f03b43d4c746ce0e3e48074a21e2f51244b648b6542e1066" dependencies = [ "httpdate", "reqwest", - "rustls 0.21.12", + "rustls 0.22.4", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -3187,14 +3187,14 @@ dependencies = [ "sentry-tracing", "tokio", "ureq", - "webpki-roots 0.25.4", + "webpki-roots", ] [[package]] name = "sentry-backtrace" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce52a2633e0332821389f865762b94852e71b359cd8e95cbf91f38c154302cb" +checksum = "40aa225bb41e2ec9d7c90886834367f560efc1af028f1c5478a6cce6a59c463a" dependencies = [ "backtrace", "once_cell", @@ -3204,9 +3204,9 @@ dependencies = [ [[package]] name = "sentry-contexts" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10565e6eb013737bf7555250f8b4b8248a571e1de4de7230037f5dd5ea51bc63" +checksum = "1a8dd746da3d16cb8c39751619cefd4fcdbd6df9610f3310fd646b55f6e39910" dependencies = [ "hostname 0.4.0", "libc", @@ -3218,15 +3218,12 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2c5e7a0430cfa86b7cb6e7bc01a56331937b2ba08ab703c2e7331139a99f121" +checksum = "161283cfe8e99c8f6f236a402b9ccf726b201f365988b5bb637ebca0abbd4a30" dependencies = [ - "crc32fast", - "itertools 0.13.0", "once_cell", "rand", - "regex", "sentry-types", "serde", "serde_json", @@ -3234,9 +3231,9 @@ dependencies = [ [[package]] name = "sentry-debug-images" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2be3e693a65dba8578790de0c6adc0a833815979590ed867336aff8fbcf9cbf7" +checksum = "8fc6b25e945fcaa5e97c43faee0267eebda9f18d4b09a251775d8fef1086238a" dependencies = [ "findshlibs", "once_cell", @@ -3245,9 +3242,9 @@ dependencies = [ [[package]] name = "sentry-log" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "184b15d2ee727a35a0731ab6c860491a8737149abbe06eee9fbe43fac24b2f07" +checksum = "75bbcc61886955045a1dd4bdb173412a80bb2571be3c5bfcf7eb8f55a442bbf5" dependencies = [ "log", "sentry-core", @@ -3255,9 +3252,9 @@ dependencies = [ [[package]] name = "sentry-panic" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f4040c0ad5670b90fc60759bbb31e3fc0d6517448d0c7440255a6f7362c50e7" +checksum = "bc74f229c7186dd971a9491ffcbe7883544aa064d1589bd30b83fb856cd22d63" dependencies = [ "sentry-backtrace", "sentry-core", @@ -3265,9 +3262,9 @@ dependencies = [ [[package]] name = "sentry-tower" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03daa33dfa3adc8b99da9afe10b1775a8c689e3242bf616badec7455916c525" +checksum = "6c90802b38c899a2c9e557dff25ad186362eddf755d5f5244001b172dd03bead" dependencies = [ "http 1.1.0", "pin-project", @@ -3279,9 +3276,9 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "151ff7c03687737ba11969ccdc6f45eb7775ed7a95880810297c6cc8c3574e75" +checksum = "cd3c5faf2103cd01eeda779ea439b68c4ee15adcdb16600836e97feafab362ec" dependencies = [ "sentry-backtrace", "sentry-core", @@ -3291,9 +3288,9 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.33.0" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d5ad9e33b9f6f598387a6a23aa23c7f13e4e7740a34cd4d9c8db851bfdae05" +checksum = "5d68cdf6bc41b8ff3ae2a9c4671e97426dcdd154cc1d4b6b72813f285d6b163f" dependencies = [ "debugid", "hex", @@ -4162,7 +4159,7 @@ dependencies = [ "rustls-pki-types", "rustls-webpki 0.102.4", "url", - "webpki-roots 0.26.1", + "webpki-roots", ] [[package]] @@ -4340,12 +4337,6 @@ dependencies = [ "url", ] -[[package]] -name = "webpki-roots" -version = "0.25.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" - [[package]] name = "webpki-roots" version = "0.26.1" diff --git a/Cargo.toml b/Cargo.toml index a9c35d8f..be6186e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -340,7 +340,7 @@ features = ["rt-tokio"] # optional sentry metrics for crash/panic reporting [workspace.dependencies.sentry] -version = "0.33.0" +version = "0.34.0" default-features = false features = [ "backtrace", @@ -356,9 +356,9 @@ features = [ ] [workspace.dependencies.sentry-tracing] -version = "0.33.0" +version = "0.34.0" [workspace.dependencies.sentry-tower] -version = "0.33.0" +version = "0.34.0" # jemalloc usage [workspace.dependencies.tikv-jemalloc-sys] From 176d95c2a8e55c31de479cf0857028cc3daf6927 Mon Sep 17 00:00:00 2001 From: Lux Aliaga Date: Fri, 7 Jun 2024 12:53:44 -0400 Subject: [PATCH 0098/2091] admin: media: Force flag on past media removal When enabled, if a file is deemed unremovable, it skips past it and continues deleting all other files that fit the criteria. Additionally, fix age comparison under the same command. Signed-off-by: Lux Aliaga --- src/admin/media/media_commands.rs | 6 ++++-- src/admin/media/mod.rs | 6 +++++- src/service/media/mod.rs | 12 +++++++++--- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/admin/media/media_commands.rs b/src/admin/media/media_commands.rs index e69fbfec..956f5743 100644 --- a/src/admin/media/media_commands.rs +++ b/src/admin/media/media_commands.rs @@ -164,10 +164,12 @@ pub(crate) async fn delete_list(body: Vec<&str>) -> Result, duration: String) -> Result { +pub(crate) async fn delete_past_remote_media( + _body: Vec<&str>, duration: String, force: bool, +) -> Result { let deleted_count = services() .media - .delete_all_remote_media_at_after_time(duration) + .delete_all_remote_media_at_after_time(duration, force) .await?; Ok(RoomMessageEventContent::text_plain(format!( diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 4e21b750..b5957370 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -32,6 +32,9 @@ pub(crate) enum MediaCommand { /// - The duration (at or after), e.g. "5m" to delete all media in the /// past 5 minutes duration: String, + /// Continues deleting remote media if an undeletable object is found + #[arg(short, long)] + force: bool, }, } @@ -44,6 +47,7 @@ pub(crate) async fn process(command: MediaCommand, body: Vec<&str>) -> Result delete_list(body).await?, MediaCommand::DeletePastRemoteMedia { duration, - } => delete_past_remote_media(body, duration).await?, + force, + } => delete_past_remote_media(body, duration, force).await?, }) } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index c7326e1a..d23e082c 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -182,7 +182,7 @@ impl Service { /// Deletes all remote only media files in the given at or after /// time/duration. Returns a u32 with the amount of media files deleted. - pub async fn delete_all_remote_media_at_after_time(&self, time: String) -> Result { + pub async fn delete_all_remote_media_at_after_time(&self, time: String, force: bool) -> Result { let all_keys = self.db.get_all_media_keys(); let user_duration: SystemTime = match cyborgtime::parse_duration(&time) { @@ -258,11 +258,17 @@ impl Service { debug!("btime is unsupported, using mtime instead"); file_metadata.modified()? }, - Err(err) => return Err(err.into()), + Err(err) => { + if force { + error!("Could not delete MXC path {:?}: {:?}. Skipping...", path, err); + continue; + } + return Err(err.into()); + }, }; debug!("File created at: {:?}", file_created_at); - if file_created_at >= user_duration { + if file_created_at <= user_duration { debug!("File is within user duration, pushing to list of file paths and keys to delete."); remote_mxcs.push(mxc.to_string()); } From 40e4019f7fd77f0802f61ecefea8db51acaf190c Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 00:05:37 -0400 Subject: [PATCH 0099/2091] add missing TURN example config options Signed-off-by: strawberry --- conduwuit-example.toml | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 1c6f56d4..04bf26cd 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -702,6 +702,44 @@ allow_profile_lookup_federation_requests = true #typing_client_timeout_max_s = 45 +### TURN / VoIP + +# vector list of TURN URIs/servers to use +# +# No default +#turn_uris = ["turn:example.turn.uri?transport=udp", "turn:example.turn.uri?transport=tcp"] + +# TURN secret to use for generating the HMAC-SHA1 hash apart of username and password generation +# +# this is more secure, but if needed you can use traditional username/password below. +# +# no default +#turn_secret = "" + +# TURN username to provide the client +# +# no default +#turn_username = "" + +# TURN password to provide the client +# +# no default +#turn_password = "" + +# TURN TTL +# +# Default is 86400 seconds +#turn_ttl = 86400 + +# allow guests/unauthenticated users to access TURN credentials +# +# this is the equivalent of Synapse's `turn_allow_guests` config option. this allows +# any unauthenticated user to call `/_matrix/client/v3/voip/turnServer`. +# +# defaults to false +#turn_allow_guests = false + + # Other options not in [global]: # # From 1cc7cf54a762ce7b19793c7ba977e0874020bb61 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 5 Jun 2024 18:42:56 -0400 Subject: [PATCH 0100/2091] add config option to allow guests to access TURN server `turn_allow_guests` Signed-off-by: strawberry --- src/api/client/voip.rs | 17 +++++++++++++---- src/api/ruma_wrapper/auth.rs | 17 +++++++++++++++-- src/core/config/mod.rs | 2 ++ 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index c5ea96f9..9608dc88 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -1,12 +1,15 @@ use std::time::{Duration, SystemTime}; use base64::{engine::general_purpose, Engine as _}; +use conduit::utils; use hmac::{Hmac, Mac}; -use ruma::{api::client::voip::get_turn_server_info, SecondsSinceUnixEpoch}; +use ruma::{api::client::voip::get_turn_server_info, SecondsSinceUnixEpoch, UserId}; use sha1::Sha1; use crate::{services, Result, Ruma}; +const RANDOM_USER_ID_LENGTH: usize = 10; + type HmacSha1 = Hmac; /// # `GET /_matrix/client/r0/voip/turnServer` @@ -15,8 +18,6 @@ type HmacSha1 = Hmac; pub(crate) async fn turn_server_route( body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let turn_secret = services().globals.turn_secret().clone(); let (username, password) = if !turn_secret.is_empty() { @@ -27,7 +28,15 @@ pub(crate) async fn turn_server_route( ) .expect("time is valid"); - let username: String = format!("{}:{}", expiry.get(), sender_user); + let user = body.sender_user.unwrap_or_else(|| { + UserId::parse_with_server_name( + utils::random_string(RANDOM_USER_ID_LENGTH).to_lowercase(), + &services().globals.config.server_name, + ) + .unwrap() + }); + + let username: String = format!("{}:{}", expiry.get(), user); let mut mac = HmacSha1::new_from_slice(turn_secret.as_bytes()).expect("HMAC can take key of any size"); mac.update(username.as_bytes()); diff --git a/src/api/ruma_wrapper/auth.rs b/src/api/ruma_wrapper/auth.rs index 35fd443d..f6dcc01f 100644 --- a/src/api/ruma_wrapper/auth.rs +++ b/src/api/ruma_wrapper/auth.rs @@ -91,8 +91,21 @@ pub(super) async fn auth( appservice_info: Some(*info), }) }, - (AuthScheme::AccessToken, Token::None) => { - Err(Error::BadRequest(ErrorKind::MissingToken, "Missing access token.")) + (AuthScheme::AccessToken, Token::None) => match request.parts.uri.path() { + // TODO: can we check this better? + "/_matrix/client/v3/voip/turnServer" | "/_matrix/client/r0/voip/turnServer" => { + if services().globals.config.turn_allow_guests { + Ok(Auth { + origin: None, + sender_user: None, + sender_device: None, + appservice_info: None, + }) + } else { + Err(Error::BadRequest(ErrorKind::MissingToken, "Missing access token.")) + } + }, + _ => Err(Error::BadRequest(ErrorKind::MissingToken, "Missing access token.")), }, ( AuthScheme::AccessToken | AuthScheme::AccessTokenOptional | AuthScheme::None, diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 6c2c9655..ae9902e1 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -165,6 +165,8 @@ pub struct Config { #[serde(default)] pub allow_public_room_directory_without_auth: bool, #[serde(default)] + pub turn_allow_guests: bool, + #[serde(default)] pub lockdown_public_room_directory: bool, #[serde(default)] pub allow_device_name_federation: bool, From bfbb29dded3937aae309e6659a3e6820a925604c Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 01:16:18 -0400 Subject: [PATCH 0101/2091] add `is_world_readable` state_accessor func, use `self` instead of `services()` Signed-off-by: strawberry --- src/api/client/directory.rs | 17 +----------- src/service/rooms/spaces/mod.rs | 23 +--------------- src/service/rooms/state_accessor/mod.rs | 35 ++++++++++++++++--------- 3 files changed, 25 insertions(+), 50 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 7cfb3392..0e5b8afa 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -14,7 +14,6 @@ use ruma::{ canonical_alias::RoomCanonicalAliasEventContent, create::RoomCreateEventContent, guest_access::{GuestAccess, RoomGuestAccessEventContent}, - history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, join_rules::{JoinRule, RoomJoinRulesEventContent}, topic::RoomTopicEventContent, }, @@ -261,21 +260,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( }) }) .unwrap_or(None), - world_readable: services() - .rooms - .state_accessor - .room_state_get(&room_id, &StateEventType::RoomHistoryVisibility, "")? - .map_or(Ok(false), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomHistoryVisibilityEventContent| { - c.history_visibility == HistoryVisibility::WorldReadable - }) - .map_err(|e| { - error!( - "Invalid room history visibility event in database for room {room_id}, assuming is \"shared\": {e}", - ); - Error::bad_database("Invalid room history visibility event in database.") - })}).unwrap_or(false), + world_readable: services().rooms.state_accessor.is_world_readable(&room_id)?, guest_can_join: services() .rooms .state_accessor diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 219b8c39..72ebaf4a 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -18,7 +18,6 @@ use ruma::{ canonical_alias::RoomCanonicalAliasEventContent, create::RoomCreateEventContent, guest_access::{GuestAccess, RoomGuestAccessEventContent}, - history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent, RoomMembership}, topic::RoomTopicEventContent, }, @@ -591,7 +590,7 @@ impl Service { }) }) .unwrap_or(None), - world_readable: world_readable(room_id)?, + world_readable: services().rooms.state_accessor.is_world_readable(room_id)?, guest_can_join: guest_can_join(room_id)?, avatar_url: services() .rooms @@ -858,26 +857,6 @@ fn guest_can_join(room_id: &RoomId) -> Result { }) } -/// Checks if guests are able to view room content without joining -fn world_readable(room_id: &RoomId) -> Result { - Ok(services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomHistoryVisibility, "")? - .map_or(Ok(false), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomHistoryVisibilityEventContent| c.history_visibility == HistoryVisibility::WorldReadable) - .map_err(|e| { - error!( - "Invalid room history visibility event in database for room {room_id}, assuming is \ - \"shared\": {e} " - ); - Error::bad_database("Invalid room history visibility event in database.") - }) - }) - .unwrap_or(false)) -} - /// Returns the join rule for a given room fn get_join_rule(current_room: &RoomId) -> Result<(SpaceRoomJoinRule, Vec), Error> { Ok(services() diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index a05ba719..0681c298 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -259,20 +259,14 @@ impl Service { } pub fn get_name(&self, room_id: &RoomId) -> Result> { - services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomName, "")? + self.room_state_get(room_id, &StateEventType::RoomName, "")? .map_or(Ok(None), |s| { Ok(serde_json::from_str(s.content.get()).map_or_else(|_| None, |c: RoomNameEventContent| Some(c.name))) }) } pub fn get_avatar(&self, room_id: &RoomId) -> Result> { - services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomAvatar, "")? + self.room_state_get(room_id, &StateEventType::RoomAvatar, "")? .map_or(Ok(ruma::JsOption::Undefined), |s| { serde_json::from_str(s.content.get()) .map_err(|_| Error::bad_database("Invalid room avatar event in database.")) @@ -280,10 +274,7 @@ impl Service { } pub fn get_member(&self, room_id: &RoomId, user_id: &UserId) -> Result> { - services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())? + self.room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())? .map_or(Ok(None), |s| { serde_json::from_str(s.content.get()) .map_err(|_| Error::bad_database("Invalid room member event in database.")) @@ -310,4 +301,24 @@ impl Service { .create_hash_and_sign_event(new_event, sender, room_id, state_lock) .is_ok()) } + + /// Checks if guests are able to view room content without joining + pub fn is_world_readable(&self, room_id: &RoomId) -> Result { + Ok(self + .room_state_get(room_id, &StateEventType::RoomHistoryVisibility, "")? + .map_or(Ok(false), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomHistoryVisibilityEventContent| { + c.history_visibility == HistoryVisibility::WorldReadable + }) + .map_err(|e| { + error!( + "Invalid room history visibility event in database for room {room_id}, assuming not world \ + readable: {e} " + ); + Error::bad_database("Invalid room history visibility event in database.") + }) + }) + .unwrap_or(false)) + } } From 81487e3f07fcf8349a8ee0e7c0b54c3e5e860242 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 01:21:01 -0400 Subject: [PATCH 0102/2091] fix(fed): dont reject `/get_missing_events` on world readable rooms Signed-off-by: strawberry --- src/api/server/get_missing_events.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index 1179afe7..1c9a6a38 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -13,19 +13,23 @@ pub(crate) async fn get_missing_events_route( ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room")); - } - services() .rooms .event_handler .acl_check(origin, &body.room_id)?; + if !services() + .rooms + .state_accessor + .is_world_readable(&body.room_id)? + && !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room")); + } + let limit = body .limit .try_into() From 8103bd7310c8ebae44bb4c6f3587dfb77853e3bd Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 01:45:00 -0400 Subject: [PATCH 0103/2091] fix(fed): dont reject `/backfill/` on world readable rooms Signed-off-by: strawberry --- src/api/server/backfill.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index e8da77f2..e3ff82e4 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -12,19 +12,23 @@ use crate::{services, Error, PduEvent, Result, Ruma}; pub(crate) async fn get_backfill_route(body: Ruma) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); - } - services() .rooms .event_handler .acl_check(origin, &body.room_id)?; + if !services() + .rooms + .state_accessor + .is_world_readable(&body.room_id)? + && !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + } + let until = body .v .iter() From 197a02bf8debe5aea749af86f6143761054665a8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 01:46:02 -0400 Subject: [PATCH 0104/2091] fix(fed): dont reject `/event/` on world readable rooms Signed-off-by: strawberry --- src/api/server/event.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/api/server/event.rs b/src/api/server/event.rs index f24530f3..29f538b4 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -28,10 +28,11 @@ pub(crate) async fn get_event_route(body: Ruma) -> Resul let room_id = <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; - if !services() - .rooms - .state_cache - .server_in_room(origin, room_id)? + if !services().rooms.state_accessor.is_world_readable(room_id)? + && !services() + .rooms + .state_cache + .server_in_room(origin, room_id)? { return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); } From d10bc67c9d7ca6975c25a2d29c9b4e54e3416e15 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 01:47:06 -0400 Subject: [PATCH 0105/2091] fix(fed): dont reject `/event_auth/` on world readable rooms Signed-off-by: strawberry --- src/api/server/event_auth.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 91770336..1ddf2ce3 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -17,19 +17,23 @@ pub(crate) async fn get_event_authorization_route( ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); - } - services() .rooms .event_handler .acl_check(origin, &body.room_id)?; + if !services() + .rooms + .state_accessor + .is_world_readable(&body.room_id)? + && !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + } + let event = services() .rooms .timeline From 10dfbf6420fa0d5d5422cb39e14839cd314623ea Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 01:47:35 -0400 Subject: [PATCH 0106/2091] fix(fed): dont reject `/state/` on world readable rooms Signed-off-by: strawberry --- src/api/server/state.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/api/server/state.rs b/src/api/server/state.rs index 10796644..c858f6fd 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -6,25 +6,29 @@ use crate::{services, Error, PduEvent, Result, Ruma}; /// # `GET /_matrix/federation/v1/state/{roomId}` /// -/// Retrieves the current state of the room. +/// Retrieves a snapshot of a room's state at a given event. pub(crate) async fn get_room_state_route( body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); - } - services() .rooms .event_handler .acl_check(origin, &body.room_id)?; + if !services() + .rooms + .state_accessor + .is_world_readable(&body.room_id)? + && !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + } + let shortstatehash = services() .rooms .state_accessor From a04ff7d4af7287026143909ae56633497d5dd893 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 01:48:05 -0400 Subject: [PATCH 0107/2091] fix(fed): dont reject `/state_ids/` on world readable rooms Signed-off-by: strawberry --- src/api/server/state_ids.rs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index b56f7cfa..9fe695dc 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -6,25 +6,30 @@ use crate::{services, Error, Result, Ruma}; /// # `GET /_matrix/federation/v1/state_ids/{roomId}` /// -/// Retrieves the current state of the room. +/// Retrieves a snapshot of a room's state at a given event, in the form of +/// event IDs. pub(crate) async fn get_room_state_ids_route( body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - if !services() - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); - } - services() .rooms .event_handler .acl_check(origin, &body.room_id)?; + if !services() + .rooms + .state_accessor + .is_world_readable(&body.room_id)? + && !services() + .rooms + .state_cache + .server_in_room(origin, &body.room_id)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + } + let shortstatehash = services() .rooms .state_accessor From 97ddb2ce87f12539b3ff5e6b32438e1196733188 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 02:02:10 -0400 Subject: [PATCH 0108/2091] bump conduwuit to version 0.4.2 Signed-off-by: strawberry --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7010084d..2c370222 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -582,7 +582,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" -version = "0.4.1" +version = "0.4.2" dependencies = [ "clap", "conduit_admin", @@ -610,7 +610,7 @@ dependencies = [ [[package]] name = "conduit_admin" -version = "0.4.1" +version = "0.4.2" dependencies = [ "clap", "conduit_api", @@ -633,7 +633,7 @@ dependencies = [ [[package]] name = "conduit_api" -version = "0.4.1" +version = "0.4.2" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -666,7 +666,7 @@ dependencies = [ [[package]] name = "conduit_core" -version = "0.4.1" +version = "0.4.2" dependencies = [ "argon2", "axum 0.7.5", @@ -707,7 +707,7 @@ dependencies = [ [[package]] name = "conduit_database" -version = "0.4.1" +version = "0.4.2" dependencies = [ "chrono", "conduit_core", @@ -724,7 +724,7 @@ dependencies = [ [[package]] name = "conduit_router" -version = "0.4.1" +version = "0.4.2" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -757,7 +757,7 @@ dependencies = [ [[package]] name = "conduit_service" -version = "0.4.1" +version = "0.4.2" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index be6186e0..a36e1fd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" rust-version = "1.77.0" -version = "0.4.1" +version = "0.4.2" [workspace.metadata.crane] name = "conduit" From 82ac6b01b2da84f21b72a86c898ea3e6f90699fc Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 03:40:58 -0400 Subject: [PATCH 0109/2091] service(auth_chain): use `self` instead of `services()` Signed-off-by: strawberry --- src/service/rooms/auth_chain/mod.rs | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index fce98521..84e423e3 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -63,11 +63,7 @@ impl Service { } let chunk_key: Vec = chunk.iter().map(|(short, _)| short).copied().collect(); - if let Some(cached) = services() - .rooms - .auth_chain - .get_cached_eventid_authchain(&chunk_key)? - { + if let Some(cached) = self.get_cached_eventid_authchain(&chunk_key)? { trace!("Found cache entry for whole chunk"); full_auth_chain.extend(cached.iter().copied()); hits = hits.saturating_add(1); @@ -78,20 +74,13 @@ impl Service { let mut misses2: usize = 0; let mut chunk_cache = Vec::with_capacity(chunk.len()); for (sevent_id, event_id) in chunk { - if let Some(cached) = services() - .rooms - .auth_chain - .get_cached_eventid_authchain(&[sevent_id])? - { + if let Some(cached) = self.get_cached_eventid_authchain(&[sevent_id])? { trace!(?event_id, "Found cache entry for event"); chunk_cache.extend(cached.iter().copied()); hits2 = hits2.saturating_add(1); } else { let auth_chain = self.get_auth_chain_inner(room_id, event_id)?; - services() - .rooms - .auth_chain - .cache_auth_chain(vec![sevent_id], &auth_chain)?; + self.cache_auth_chain(vec![sevent_id], &auth_chain)?; chunk_cache.extend(auth_chain.iter()); misses2 = misses2.saturating_add(1); debug!( @@ -106,10 +95,7 @@ impl Service { chunk_cache.sort_unstable(); chunk_cache.dedup(); - services() - .rooms - .auth_chain - .cache_auth_chain_vec(chunk_key, &chunk_cache)?; + self.cache_auth_chain_vec(chunk_key, &chunk_cache)?; full_auth_chain.extend(chunk_cache.iter()); misses = misses.saturating_add(1); debug!( From a061644b2dde4fc5964d101c525030aeb195ee9d Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 03:42:33 -0400 Subject: [PATCH 0110/2091] service(event_handler): use `self` instead of `services()` Signed-off-by: strawberry --- src/service/rooms/event_handler/mod.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 2243cb0a..d1413cb5 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -93,7 +93,7 @@ impl Service { } // 1.3.1 Check room ACL on origin field/server - services().rooms.event_handler.acl_check(origin, room_id)?; + self.acl_check(origin, room_id)?; // 1.3.2 Check room ACL on sender's server name let sender: OwnedUserId = serde_json::from_value( @@ -105,10 +105,7 @@ impl Service { ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "User ID in sender is invalid"))?; - services() - .rooms - .event_handler - .acl_check(sender.server_name(), room_id)?; + self.acl_check(sender.server_name(), room_id)?; // Fetch create event let create_event = services() @@ -199,9 +196,7 @@ impl Service { .await .insert(room_id.to_owned(), (event_id.to_owned(), start_time)); - let r = services() - .rooms - .event_handler + let r = self .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, &create_event, origin, room_id, pub_key_map) .await; From 3ed561cb318b2d9daec18f108826d31780e82135 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 03:44:59 -0400 Subject: [PATCH 0111/2091] service(globals): use `self` instead of `services()` Signed-off-by: strawberry --- src/service/globals/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index e806a6b0..9327ecaa 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -290,7 +290,7 @@ impl Service { let mut keys = self.db.signing_keys_for(origin)?; if origin == self.server_name() { keys.insert( - format!("ed25519:{}", services().globals.keypair().version()) + format!("ed25519:{}", self.keypair().version()) .try_into() .expect("found invalid server signing keys in DB"), VerifyKey { From 0ebabba97191c391fec256571e6ebddd087ecf1e Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 03:49:01 -0400 Subject: [PATCH 0112/2091] services(pdu_metadata): use `self` instead of `services()` Signed-off-by: strawberry --- src/service/rooms/pdu_metadata/mod.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 25da29dd..c4f45848 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -73,11 +73,7 @@ impl Service { match dir { Direction::Forward => { - let relations_until = - &services() - .rooms - .pdu_metadata - .relations_until(sender_user, room_id, target, from, depth)?; + let relations_until = &self.relations_until(sender_user, room_id, target, from, depth)?; let events_after: Vec<_> = relations_until // TODO: should be relations_after .iter() .filter(|(_, pdu)| { @@ -123,11 +119,7 @@ impl Service { }) }, Direction::Backward => { - let relations_until = - &services() - .rooms - .pdu_metadata - .relations_until(sender_user, room_id, target, from, depth)?; + let relations_until = &self.relations_until(sender_user, room_id, target, from, depth)?; let events_before: Vec<_> = relations_until .iter() .filter(|(_, pdu)| { From 2230bc7339755aefd600cbec51202a663f378b47 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 03:50:16 -0400 Subject: [PATCH 0113/2091] services(state): use `self` instead of `services()` Signed-off-by: strawberry --- src/service/rooms/state/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 5a05b162..8a32d9ca 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -355,7 +355,7 @@ impl Service { &self, room_id: &RoomId, kind: &TimelineEventType, sender: &UserId, state_key: Option<&str>, content: &serde_json::value::RawValue, ) -> Result>> { - let Some(shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? else { + let Some(shortstatehash) = self.get_room_shortstatehash(room_id)? else { return Ok(HashMap::new()); }; From 14ec41c2118baad5c66c70527cb058df191764a8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 03:51:23 -0400 Subject: [PATCH 0114/2091] services(state_cache): use `self` instead of `services()` Signed-off-by: strawberry --- src/service/rooms/state_cache/mod.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 3f47ff4c..8f21a352 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -408,9 +408,7 @@ impl Service { })? .flatten(); - let mut servers: Vec = services() - .rooms - .state_cache + let mut servers: Vec = self .room_members(room_id) .filter_map(Result::ok) .counts_by(|user| user.server_name().to_owned()) From c0c7f23a0579a2e736c7874012fa3d5b2095f824 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 03:52:37 -0400 Subject: [PATCH 0115/2091] services(timeline): use `self` instead of `services()` Signed-off-by: strawberry --- src/service/rooms/timeline/mod.rs | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 34a60507..b8462b87 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -494,11 +494,7 @@ impl Service { } if let Ok(content) = serde_json::from_str::(pdu.content.get()) { - if let Some(related_pducount) = services() - .rooms - .timeline - .get_pdu_count(&content.relates_to.event_id)? - { + if let Some(related_pducount) = self.get_pdu_count(&content.relates_to.event_id)? { services() .rooms .pdu_metadata @@ -513,11 +509,7 @@ impl Service { } => { // We need to do it again here, because replies don't have // event_id as a top level field - if let Some(related_pducount) = services() - .rooms - .timeline - .get_pdu_count(&in_reply_to.event_id)? - { + if let Some(related_pducount) = self.get_pdu_count(&in_reply_to.event_id)? { services() .rooms .pdu_metadata @@ -646,7 +638,7 @@ impl Service { // Our depth is the maximum depth of prev_events + 1 let depth = prev_events .iter() - .filter_map(|event_id| Some(services().rooms.timeline.get_pdu(event_id).ok()??.depth)) + .filter_map(|event_id| Some(self.get_pdu(event_id).ok()??.depth)) .max() .unwrap_or_else(|| uint!(0)) + uint!(1); @@ -945,9 +937,7 @@ impl Service { return Ok(None); } - let pdu_id = services() - .rooms - .timeline + let pdu_id = self .append_pdu(pdu, pdu_json, new_room_leaves, state_lock) .await?; @@ -1138,7 +1128,7 @@ impl Service { let mutex_lock = mutex.lock().await; // Skip the PDU if we already have it as a timeline event - if let Some(pdu_id) = services().rooms.timeline.get_pdu_id(&event_id)? { + if let Some(pdu_id) = self.get_pdu_id(&event_id)? { info!("We already know {event_id} at {pdu_id:?}"); return Ok(()); } From ab5db37851fd1b65349af5e31b618f255a191fd6 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Jun 2024 16:57:48 -0400 Subject: [PATCH 0116/2091] fix bad double split insert on `roomuserid_joined` cf Signed-off-by: strawberry --- src/service/rooms/state_cache/data.rs | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 6f491d89..71a2c034 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -121,10 +121,8 @@ impl Data for KeyValueDatabase { fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { let roomid = room_id.as_bytes().to_vec(); - let mut roomid_prefix = room_id.as_bytes().to_vec(); - roomid_prefix.push(0xFF); - let mut roomuser_id = roomid_prefix.clone(); + let mut roomuser_id = roomid.clone(); roomuser_id.push(0xFF); roomuser_id.extend_from_slice(user_id.as_bytes()); @@ -139,13 +137,10 @@ impl Data for KeyValueDatabase { self.userroomid_leftstate.remove(&userroom_id)?; self.roomuserid_leftcount.remove(&roomuser_id)?; - if self - .roomuserid_joined - .scan_prefix(roomid_prefix.clone()) - .count() == 0 + if self.roomuserid_joined.scan_prefix(roomid.clone()).count() == 0 && self .roomuserid_invitecount - .scan_prefix(roomid_prefix) + .scan_prefix(roomid.clone()) .count() == 0 { self.roomid_inviteviaservers.remove(&roomid)?; @@ -198,10 +193,9 @@ impl Data for KeyValueDatabase { fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { let roomid = room_id.as_bytes().to_vec(); - let mut roomid_prefix = room_id.as_bytes().to_vec(); - roomid_prefix.push(0xFF); - let mut roomuser_id = roomid_prefix.clone(); + let mut roomuser_id = roomid.clone(); + roomuser_id.push(0xFF); roomuser_id.extend_from_slice(user_id.as_bytes()); let mut userroom_id = user_id.as_bytes().to_vec(); @@ -219,13 +213,10 @@ impl Data for KeyValueDatabase { self.userroomid_invitestate.remove(&userroom_id)?; self.roomuserid_invitecount.remove(&roomuser_id)?; - if self - .roomuserid_joined - .scan_prefix(roomid_prefix.clone()) - .count() == 0 + if self.roomuserid_joined.scan_prefix(roomid.clone()).count() == 0 && self .roomuserid_invitecount - .scan_prefix(roomid_prefix) + .scan_prefix(roomid.clone()) .count() == 0 { self.roomid_inviteviaservers.remove(&roomid)?; @@ -434,8 +425,6 @@ impl Data for KeyValueDatabase { /// Returns an iterator of all our local users joined in a room who are /// active (not deactivated, not guest) and have a joined membership state /// in the room - /// - /// TODO: why is `roomuserid_joined` not reliable? #[tracing::instrument(skip(self))] fn active_local_joined_users_in_room<'a>( &'a self, room_id: &'a RoomId, From 7f5b59afbbc38fd6fb294f068946ef289234c0b9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Jun 2024 16:58:24 -0400 Subject: [PATCH 0117/2091] add conduwuit-specific db migration fixing double split db entries for a few months now we accidentally had double 0xFF splits being inserted into `roomuserid_joined` cf when membership counts and such are being updated. this is a conduwuit-specific db migration and does NOT break conduit compatibility. `fix_bad_double_separator_in_state_cache` Signed-off-by: strawberry --- src/service/globals/migrations.rs | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 75990df0..3127c3ec 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -6,6 +6,7 @@ use std::{ sync::Arc, }; +use conduit::{debug_info, debug_warn}; use database::KeyValueDatabase; use itertools::Itertools; use ruma::{ @@ -547,6 +548,37 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result } } + if db + .global + .get(b"fix_bad_double_separator_in_state_cache")? + .is_none() + { + warn!("Fixing bad double separator in state_cache roomuserid_joined"); + let mut iter_count: usize = 0; + for (mut key, value) in db.roomuserid_joined.iter() { + iter_count = iter_count.saturating_add(1); + debug_info!(%iter_count); + let first_sep_index = key.iter().position(|&i| i == 0xFF).unwrap(); + + if key + .iter() + .get(first_sep_index..=first_sep_index + 1) + .copied() + .collect_vec() == vec![0xFF, 0xFF] + { + debug_warn!("Found bad key: {key:?}"); + db.roomuserid_joined.remove(&key)?; + + key.remove(first_sep_index); + debug_warn!("Fixed key: {key:?}"); + db.roomuserid_joined.insert(&key, &value)?; + } + } + + db.global + .insert(b"fix_bad_double_separator_in_state_cache", &[])?; + } + assert_eq!( services().globals.database_version().unwrap(), latest_database_version, @@ -614,6 +646,9 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result .globals .bump_database_version(latest_database_version)?; + db.global + .insert(b"fix_bad_double_separator_in_state_cache", &[])?; + // Create the admin room and server user on first run services().admin.create_admin_room().await?; From 0524e6ed52fbc0f95d378c231460ae9544440bd2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Jun 2024 17:32:04 -0400 Subject: [PATCH 0118/2091] remove unnecessary `active_local_joined_users_in_room` state_cache accessor the underlying bug has been fixed Signed-off-by: strawberry --- src/service/rooms/state_cache/data.rs | 34 ++++----------------------- src/service/rooms/state_cache/mod.rs | 14 ++--------- src/service/rooms/timeline/mod.rs | 2 +- 3 files changed, 8 insertions(+), 42 deletions(-) diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 71a2c034..1124b457 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -2,7 +2,7 @@ use std::collections::HashSet; use itertools::Itertools; use ruma::{ - events::{room::member::MembershipState, AnyStrippedStateEvent, AnySyncStateEvent}, + events::{AnyStrippedStateEvent, AnySyncStateEvent}, serde::Raw, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; @@ -50,17 +50,10 @@ pub trait Data: Send + Sync { /// in the room, even if they're deactivated/guests fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a>; - /// Returns an iterator of all our local users in a room who are active (not - /// deactivated, not guest) + /// Returns an iterator of all our local joined users in a room who are + /// active (not deactivated, not guest) fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a>; - /// Returns an iterator of all our local users joined in a room who are - /// active (not deactivated, not guest) and have a joined membership state - /// in the room - fn active_local_joined_users_in_room<'a>( - &'a self, room_id: &'a RoomId, - ) -> Box + 'a>; - fn room_joined_count(&self, room_id: &RoomId) -> Result>; fn room_invited_count(&self, room_id: &RoomId) -> Result>; @@ -412,8 +405,8 @@ impl Data for KeyValueDatabase { ) } - /// Returns an iterator of all our local users in a room who are active (not - /// deactivated, not guest) + /// Returns an iterator of all our local joined users in a room who are + /// active (not deactivated, not guest) #[tracing::instrument(skip(self))] fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a> { Box::new( @@ -422,23 +415,6 @@ impl Data for KeyValueDatabase { ) } - /// Returns an iterator of all our local users joined in a room who are - /// active (not deactivated, not guest) and have a joined membership state - /// in the room - #[tracing::instrument(skip(self))] - fn active_local_joined_users_in_room<'a>( - &'a self, room_id: &'a RoomId, - ) -> Box + 'a> { - Box::new(self.active_local_users_in_room(room_id).filter(|user_id| { - services() - .rooms - .state_accessor - .get_member(room_id, user_id) - .unwrap_or(None) - .map_or(false, |membership| membership.membership == MembershipState::Join) - })) - } - /// Returns the number of users which are currently in a room #[tracing::instrument(skip(self))] fn room_joined_count(&self, room_id: &RoomId) -> Result> { diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 8f21a352..4fdfa3c0 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -281,22 +281,12 @@ impl Service { } #[tracing::instrument(skip(self))] - /// Returns an iterator of all our local users in a room who are active (not - /// deactivated, not guest) + /// Returns an iterator of all our local joined users in a room who are + /// active (not deactivated, not guest) pub fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> impl Iterator + 'a { self.db.active_local_users_in_room(room_id) } - #[tracing::instrument(skip(self))] - /// Returns an iterator of all our local users joined in a room who are - /// active (not deactivated, not guest) and have a joined membership state - /// in the room - pub fn active_local_joined_users_in_room<'a>( - &'a self, room_id: &'a RoomId, - ) -> impl Iterator + 'a { - self.db.active_local_joined_users_in_room(room_id) - } - #[tracing::instrument(skip(self))] pub fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.db.room_invited_count(room_id) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index b8462b87..affe35ab 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -310,7 +310,7 @@ impl Service { let mut push_target = services() .rooms .state_cache - .active_local_joined_users_in_room(&pdu.room_id) + .active_local_users_in_room(&pdu.room_id) .collect_vec(); if pdu.kind == TimelineEventType::RoomMember { From ccf9f95cc98557e5896ee81e681210f0a67939c4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 9 Jun 2024 01:47:03 -0400 Subject: [PATCH 0119/2091] retroactively fix bad data in `roomuserid_joined`, remove pointless prefix scans Signed-off-by: strawberry --- src/service/globals/migrations.rs | 100 +++++++++++++++++++++++++- src/service/rooms/state_cache/data.rs | 21 +----- src/service/rooms/state_cache/mod.rs | 16 +++++ 3 files changed, 117 insertions(+), 20 deletions(-) diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 3127c3ec..aa172b5c 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -10,13 +10,13 @@ use conduit::{debug_info, debug_warn}; use database::KeyValueDatabase; use itertools::Itertools; use ruma::{ - events::{push_rules::PushRulesEvent, GlobalAccountDataEventType}, + events::{push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType}, push::Ruleset, EventId, OwnedRoomId, RoomId, UserId, }; use tracing::{debug, error, info, warn}; -use crate::{services, utils, Config, Error, Result}; +use crate::{globals::data::Data, services, utils, Config, Error, Result}; pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result<()> { // Matrix resource ownership is based on the server name; changing it @@ -555,6 +555,9 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result { warn!("Fixing bad double separator in state_cache roomuserid_joined"); let mut iter_count: usize = 0; + + let _cork = db.cork(); + for (mut key, value) in db.roomuserid_joined.iter() { iter_count = iter_count.saturating_add(1); debug_info!(%iter_count); @@ -575,10 +578,101 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result } } + db.cleanup()?; + + warn!("Finished fixing"); + db.global .insert(b"fix_bad_double_separator_in_state_cache", &[])?; } + if db + .global + .get(b"retroactively_fix_bad_data_from_roomuserid_joined")? + .is_none() + { + warn!("Retroactively fixing bad data from broken roomuserid_joined"); + + let room_ids = services() + .rooms + .metadata + .iter_ids() + .filter_map(Result::ok) + .collect_vec(); + + let _cork = db.cork(); + + for room_id in room_ids.clone() { + debug_info!("Fixing room {room_id}"); + + let users_in_room = services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(Result::ok) + .collect_vec(); + + let joined_members = users_in_room + .iter() + .filter(|user_id| { + services() + .rooms + .state_accessor + .get_member(&room_id, user_id) + .unwrap_or(None) + .map_or(false, |membership| membership.membership == MembershipState::Join) + }) + .collect_vec(); + + let non_joined_members = users_in_room + .iter() + .filter(|user_id| { + services() + .rooms + .state_accessor + .get_member(&room_id, user_id) + .unwrap_or(None) + .map_or(false, |membership| { + membership.membership == MembershipState::Leave + || membership.membership == MembershipState::Ban + }) + }) + .collect_vec(); + + for user_id in joined_members { + debug_info!("User is joined, marking as joined"); + services() + .rooms + .state_cache + .mark_as_joined(user_id, &room_id)?; + } + + for user_id in non_joined_members { + debug_info!("User is left or banned, marking as left"); + services() + .rooms + .state_cache + .mark_as_left(user_id, &room_id)?; + } + } + + for room_id in room_ids { + debug_info!( + "Updating joined count for room {room_id} to fix servers in room after correcting membership \ + states" + ); + + services().rooms.state_cache.update_joined_count(&room_id)?; + } + + db.cleanup()?; + + warn!("Finished fixing"); + + db.global + .insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; + } + assert_eq!( services().globals.database_version().unwrap(), latest_database_version, @@ -648,6 +742,8 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result db.global .insert(b"fix_bad_double_separator_in_state_cache", &[])?; + db.global + .insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; // Create the admin room and server user on first run services().admin.create_admin_room().await?; diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 1124b457..1c0571c9 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -130,14 +130,7 @@ impl Data for KeyValueDatabase { self.userroomid_leftstate.remove(&userroom_id)?; self.roomuserid_leftcount.remove(&roomuser_id)?; - if self.roomuserid_joined.scan_prefix(roomid.clone()).count() == 0 - && self - .roomuserid_invitecount - .scan_prefix(roomid.clone()) - .count() == 0 - { - self.roomid_inviteviaservers.remove(&roomid)?; - } + self.roomid_inviteviaservers.remove(&roomid)?; Ok(()) } @@ -206,14 +199,7 @@ impl Data for KeyValueDatabase { self.userroomid_invitestate.remove(&userroom_id)?; self.roomuserid_invitecount.remove(&roomuser_id)?; - if self.roomuserid_joined.scan_prefix(roomid.clone()).count() == 0 - && self - .roomuserid_invitecount - .scan_prefix(roomid.clone()) - .count() == 0 - { - self.roomid_inviteviaservers.remove(&roomid)?; - } + self.roomid_inviteviaservers.remove(&roomid)?; Ok(()) } @@ -654,8 +640,7 @@ impl Data for KeyValueDatabase { #[tracing::instrument(skip(self))] fn servers_invite_via(&self, room_id: &RoomId) -> Result>> { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xFF); + let key = room_id.as_bytes().to_vec(); self.roomid_inviteviaservers .get(&key)? diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 4fdfa3c0..1cfd56db 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -217,6 +217,22 @@ impl Service { self.db.appservice_in_room(room_id, appservice) } + /// Direct DB function to directly mark a user as left. It is not + /// recommended to use this directly. You most likely should use + /// `update_membership` instead + #[tracing::instrument(skip(self))] + pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { + self.db.mark_as_left(user_id, room_id) + } + + /// Direct DB function to directly mark a user as joined. It is not + /// recommended to use this directly. You most likely should use + /// `update_membership` instead + #[tracing::instrument(skip(self))] + pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { + self.db.mark_as_joined(user_id, room_id) + } + /// Makes a user forget a room. #[tracing::instrument(skip(self))] pub fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { self.db.forget(room_id, user_id) } From 9a3c52aa75900ce71084574fdd730a5d39d579a4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 9 Jun 2024 12:05:30 -0400 Subject: [PATCH 0120/2091] update complement results they seem to be failures out of our control (?) Signed-off-by: strawberry --- tests/test_results/complement/test_results.jsonl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 51cd57d4..502a4530 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -55,7 +55,7 @@ {"Action":"fail","Test":"TestFederationKeyUploadQuery/Can_claim_remote_one_time_key_using_POST"} {"Action":"fail","Test":"TestFederationKeyUploadQuery/Can_query_remote_device_keys_using_POST"} {"Action":"pass","Test":"TestFederationRedactSendsWithoutEvent"} -{"Action":"pass","Test":"TestFederationRejectInvite"} +{"Action":"fail","Test":"TestFederationRejectInvite"} {"Action":"fail","Test":"TestGetMissingEventsGapFilling"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility"} @@ -199,7 +199,7 @@ {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/good_connectivity"} {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/interrupted_connectivity"} {"Action":"fail","Test":"TestToDeviceMessagesOverFederation/stopped_server"} -{"Action":"pass","Test":"TestUnbanViaInvite"} +{"Action":"fail","Test":"TestUnbanViaInvite"} {"Action":"fail","Test":"TestUnknownEndpoints"} {"Action":"pass","Test":"TestUnknownEndpoints/Client-server_endpoints"} {"Action":"fail","Test":"TestUnknownEndpoints/Key_endpoints"} From 6e50b07bf56315b002d6ef2f98221ebbc87c171b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 03:57:18 +0000 Subject: [PATCH 0121/2091] Fix large future Signed-off-by: Jason Volk --- src/admin/handler.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 1a2ff38e..b09bdb8d 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -81,7 +81,8 @@ async fn handle_event(event: AdminRoomEvent, admin_room: OwnedRoomId, server_use let (mut message_content, reply) = match event { AdminRoomEvent::SendMessage(content) => (content, None), AdminRoomEvent::ProcessMessage(room_message, reply_id) => { - (process_admin_message(room_message).await, Some(reply_id)) + // This future is ~8 KiB so it's better to start it off the stack. + (Box::pin(process_admin_message(room_message)).await, Some(reply_id)) }, }; From fc1b8326e63fae30bc5d8b02590cfbba4f8a320b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 04:22:47 +0000 Subject: [PATCH 0122/2091] split join_room_by_id_helper into local and remote prior stack frame allocated 180 KiB Signed-off-by: Jason Volk --- clippy.toml | 2 +- src/api/client/membership.rs | 946 ++++++++++++++++++----------------- 2 files changed, 478 insertions(+), 470 deletions(-) diff --git a/clippy.toml b/clippy.toml index 0a04ecd2..1743ee5c 100644 --- a/clippy.toml +++ b/clippy.toml @@ -2,6 +2,6 @@ array-size-threshold = 4096 cognitive-complexity-threshold = 94 # TODO reduce me ALARA excessive-nesting-threshold = 11 # TODO reduce me to 4 or 5 future-size-threshold = 7745 # TODO reduce me ALARA -stack-size-threshold = 178030 # reduce me ALARA +stack-size-threshold = 173577 # reduce me ALARA too-many-lines-threshold = 700 # TODO reduce me to <= 100 type-complexity-threshold = 250 # reduce me to ~200 diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 3b8920f8..b50ef2cb 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -30,7 +30,7 @@ use ruma::{ OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::sync::RwLock; +use tokio::sync::{MutexGuard, RwLock}; use tracing::{debug, error, info, trace, warn}; use super::get_alias_helper; @@ -627,7 +627,7 @@ pub(crate) async fn joined_members_route( pub async fn join_room_by_id_helper( sender_user: Option<&UserId>, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, + third_party_signed: Option<&ThirdPartySigned>, ) -> Result { let sender_user = sender_user.expect("user is authenticated"); @@ -655,27 +655,468 @@ pub async fn join_room_by_id_helper( .state_cache .server_in_room(services().globals.server_name(), room_id)? { - info!("Joining {room_id} over federation."); + join_room_by_id_helper_remote(sender_user, room_id, reason, servers, third_party_signed, state_lock).await + } else { + join_room_by_id_helper_local(sender_user, room_id, reason, servers, third_party_signed, state_lock).await + } +} +async fn join_room_by_id_helper_remote( + sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], + _third_party_signed: Option<&ThirdPartySigned>, state_lock: MutexGuard<'_, ()>, +) -> Result { + info!("Joining {room_id} over federation."); + + let (make_join_response, remote_server) = make_join_request(sender_user, room_id, servers).await?; + + info!("make_join finished"); + + let room_version_id = match make_join_response.room_version { + Some(room_version) + if services() + .globals + .supported_room_versions() + .contains(&room_version) => + { + room_version + }, + _ => return Err(Error::BadServerResponse("Room version is not supported")), + }; + + let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) + .map_err(|_| Error::BadServerResponse("Invalid make_join event json received from server."))?; + + let join_authorized_via_users_server = join_event_stub + .get("content") + .map(|s| { + s.as_object()? + .get("join_authorised_via_users_server")? + .as_str() + }) + .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()); + + // TODO: Is origin needed? + join_event_stub.insert( + "origin".to_owned(), + CanonicalJsonValue::String(services().globals.server_name().as_str().to_owned()), + ); + join_event_stub.insert( + "origin_server_ts".to_owned(), + CanonicalJsonValue::Integer( + utils::millis_since_unix_epoch() + .try_into() + .expect("Timestamp is valid js_int value"), + ), + ); + join_event_stub.insert( + "content".to_owned(), + to_canonical_value(RoomMemberEventContent { + membership: MembershipState::Join, + displayname: services().users.displayname(sender_user)?, + avatar_url: services().users.avatar_url(sender_user)?, + is_direct: None, + third_party_invite: None, + blurhash: services().users.blurhash(sender_user)?, + reason, + join_authorized_via_users_server: join_authorized_via_users_server.clone(), + }) + .expect("event is valid, we just created it"), + ); + + // We keep the "event_id" in the pdu only in v1 or + // v2 rooms + match room_version_id { + RoomVersionId::V1 | RoomVersionId::V2 => {}, + _ => { + join_event_stub.remove("event_id"); + }, + }; + + // In order to create a compatible ref hash (EventID) the `hashes` field needs + // to be present + ruma::signatures::hash_and_sign_event( + services().globals.server_name().as_str(), + services().globals.keypair(), + &mut join_event_stub, + &room_version_id, + ) + .expect("event is valid, we just created it"); + + // Generate event id + let event_id = format!( + "${}", + ruma::signatures::reference_hash(&join_event_stub, &room_version_id) + .expect("ruma can calculate reference hashes") + ); + let event_id = <&EventId>::try_from(event_id.as_str()).expect("ruma's reference hashes are valid event ids"); + + // Add event_id back + join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); + + // It has enough fields to be called a proper event now + let mut join_event = join_event_stub; + + info!("Asking {remote_server} for send_join in room {room_id}"); + let send_join_response = services() + .sending + .send_federation_request( + &remote_server, + federation::membership::create_join_event::v2::Request { + room_id: room_id.to_owned(), + event_id: event_id.to_owned(), + pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), + omit_members: false, + }, + ) + .await?; + + info!("send_join finished"); + + if join_authorized_via_users_server.is_some() { + match &room_version_id { + RoomVersionId::V1 + | RoomVersionId::V2 + | RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + | RoomVersionId::V7 => { + warn!( + "Found `join_authorised_via_users_server` but room {} is version {}. Ignoring.", + room_id, &room_version_id + ); + }, + // only room versions 8 and above using `join_authorized_via_users_server` (restricted joins) need to + // validate and send signatures + RoomVersionId::V8 | RoomVersionId::V9 | RoomVersionId::V10 | RoomVersionId::V11 => { + if let Some(signed_raw) = &send_join_response.room_state.event { + info!( + "There is a signed event. This room is probably using restricted joins. Adding signature to \ + our event" + ); + let Ok((signed_event_id, signed_value)) = gen_event_id_canonical_json(signed_raw, &room_version_id) + else { + // Event could not be converted to canonical json + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Could not convert event to canonical json.", + )); + }; + + if signed_event_id != event_id { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Server sent event with wrong event id", + )); + } + + match signed_value["signatures"] + .as_object() + .ok_or(Error::BadRequest( + ErrorKind::InvalidParam, + "Server sent invalid signatures type", + )) + .and_then(|e| { + e.get(remote_server.as_str()) + .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Server did not send its signature")) + }) { + Ok(signature) => { + join_event + .get_mut("signatures") + .expect("we created a valid pdu") + .as_object_mut() + .expect("we created a valid pdu") + .insert(remote_server.to_string(), signature.clone()); + }, + Err(e) => { + warn!( + "Server {remote_server} sent invalid signature in sendjoin signatures for event \ + {signed_value:?}: {e:?}", + ); + }, + } + } + }, + _ => { + warn!( + "Unexpected or unsupported room version {} for room {}", + &room_version_id, room_id + ); + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + } + } + + services().rooms.short.get_or_create_shortroomid(room_id)?; + + info!("Parsing join event"); + let parsed_join_pdu = PduEvent::from_id_val(event_id, join_event.clone()) + .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; + + let mut state = HashMap::new(); + let pub_key_map = RwLock::new(BTreeMap::new()); + + info!("Fetching join signing keys"); + services() + .rooms + .event_handler + .fetch_join_signing_keys(&send_join_response, &room_version_id, &pub_key_map) + .await?; + + info!("Going through send_join response room_state"); + for result in send_join_response + .room_state + .state + .iter() + .map(|pdu| validate_and_add_event_id(pdu, &room_version_id, &pub_key_map)) + { + let Ok((event_id, value)) = result.await else { + continue; + }; + + let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| { + warn!("Invalid PDU in send_join response: {} {:?}", e, value); + Error::BadServerResponse("Invalid PDU in send_join response.") + })?; + + services() + .rooms + .outlier + .add_pdu_outlier(&event_id, &value)?; + if let Some(state_key) = &pdu.state_key { + let shortstatekey = services() + .rooms + .short + .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key)?; + state.insert(shortstatekey, pdu.event_id.clone()); + } + } + + info!("Going through send_join response auth_chain"); + for result in send_join_response + .room_state + .auth_chain + .iter() + .map(|pdu| validate_and_add_event_id(pdu, &room_version_id, &pub_key_map)) + { + let Ok((event_id, value)) = result.await else { + continue; + }; + + services() + .rooms + .outlier + .add_pdu_outlier(&event_id, &value)?; + } + + debug!("Running send_join auth check"); + + let auth_check = state_res::event_auth::auth_check( + &state_res::RoomVersion::new(&room_version_id).expect("room version is supported"), + &parsed_join_pdu, + None::, // TODO: third party invite + |k, s| { + services() + .rooms + .timeline + .get_pdu( + state.get( + &services() + .rooms + .short + .get_or_create_shortstatekey(&k.to_string().into(), s) + .ok()?, + )?, + ) + .ok()? + }, + ) + .map_err(|e| { + warn!("Auth check failed: {e}"); + Error::BadRequest(ErrorKind::forbidden(), "Auth check failed") + })?; + + if !auth_check { + return Err(Error::BadRequest(ErrorKind::forbidden(), "Auth check failed")); + } + + info!("Saving state from send_join"); + let (statehash_before_join, new, removed) = services().rooms.state_compressor.save_state( + room_id, + Arc::new( + state + .into_iter() + .map(|(k, id)| { + services() + .rooms + .state_compressor + .compress_state_event(k, &id) + }) + .collect::>()?, + ), + )?; + + services() + .rooms + .state + .force_state(room_id, statehash_before_join, new, removed, &state_lock) + .await?; + + info!("Updating joined counts for new room"); + services().rooms.state_cache.update_joined_count(room_id)?; + + // We append to state before appending the pdu, so we don't have a moment in + // time with the pdu without it's state. This is okay because append_pdu can't + // fail. + let statehash_after_join = services().rooms.state.append_to_state(&parsed_join_pdu)?; + + info!("Appending new room join event"); + services() + .rooms + .timeline + .append_pdu( + &parsed_join_pdu, + join_event, + vec![(*parsed_join_pdu.event_id).to_owned()], + &state_lock, + ) + .await?; + + info!("Setting final room state for new room"); + // We set the room state after inserting the pdu, so that we never have a moment + // in time where events in the current room state do not exist + services() + .rooms + .state + .set_room_state(room_id, statehash_after_join, &state_lock)?; + + Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) +} + +async fn join_room_by_id_helper_local( + sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], + _third_party_signed: Option<&ThirdPartySigned>, state_lock: MutexGuard<'_, ()>, +) -> Result { + info!("We can join locally"); + + let join_rules_event = + services() + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomJoinRules, "")?; + + let join_rules_event_content: Option = join_rules_event + .as_ref() + .map(|join_rules_event| { + serde_json::from_str(join_rules_event.content.get()).map_err(|e| { + warn!("Invalid join rules event: {}", e); + Error::bad_database("Invalid join rules event in db.") + }) + }) + .transpose()?; + + let restriction_rooms = match join_rules_event_content { + Some(RoomJoinRulesEventContent { + join_rule: JoinRule::Restricted(restricted) | JoinRule::KnockRestricted(restricted), + }) => restricted + .allow + .into_iter() + .filter_map(|a| match a { + AllowRule::RoomMembership(r) => Some(r.room_id), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + + let local_members = services() + .rooms + .state_cache + .room_members(room_id) + .filter_map(Result::ok) + .filter(|user| user_is_local(user)) + .collect::>(); + + let mut authorized_user: Option = None; + + if restriction_rooms.iter().any(|restriction_room_id| { + services() + .rooms + .state_cache + .is_joined(sender_user, restriction_room_id) + .unwrap_or(false) + }) { + for user in local_members { + if services() + .rooms + .state_accessor + .user_can_invite(room_id, &user, sender_user, &state_lock) + .await + .unwrap_or(false) + { + authorized_user = Some(user); + break; + } + } + } + + let event = RoomMemberEventContent { + membership: MembershipState::Join, + displayname: services().users.displayname(sender_user)?, + avatar_url: services().users.avatar_url(sender_user)?, + is_direct: None, + third_party_invite: None, + blurhash: services().users.blurhash(sender_user)?, + reason: reason.clone(), + join_authorized_via_users_server: authorized_user, + }; + + // Try normal join first + let error = match services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&event).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_user.to_string()), + redacts: None, + }, + sender_user, + room_id, + &state_lock, + ) + .await + { + Ok(_event_id) => return Ok(join_room_by_id::v3::Response::new(room_id.to_owned())), + Err(e) => e, + }; + + if !restriction_rooms.is_empty() + && servers + .iter() + .any(|server_name| !server_is_ours(server_name)) + { + info!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); let (make_join_response, remote_server) = make_join_request(sender_user, room_id, servers).await?; - info!("make_join finished"); - let room_version_id = match make_join_response.room_version { - Some(room_version) + Some(room_version_id) if services() .globals .supported_room_versions() - .contains(&room_version) => + .contains(&room_version_id) => { - room_version + room_version_id }, _ => return Err(Error::BadServerResponse("Room version is not supported")), }; - let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) .map_err(|_| Error::BadServerResponse("Invalid make_join event json received from server."))?; - let join_authorized_via_users_server = join_event_stub .get("content") .map(|s| { @@ -684,7 +1125,6 @@ pub async fn join_room_by_id_helper( .as_str() }) .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()); - // TODO: Is origin needed? join_event_stub.insert( "origin".to_owned(), @@ -708,7 +1148,7 @@ pub async fn join_room_by_id_helper( third_party_invite: None, blurhash: services().users.blurhash(sender_user)?, reason, - join_authorized_via_users_server: join_authorized_via_users_server.clone(), + join_authorized_via_users_server, }) .expect("event is valid, we just created it"), ); @@ -744,9 +1184,8 @@ pub async fn join_room_by_id_helper( join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); // It has enough fields to be called a proper event now - let mut join_event = join_event_stub; + let join_event = join_event_stub; - info!("Asking {remote_server} for send_join in room {room_id}"); let send_join_response = services() .sending .send_federation_request( @@ -760,470 +1199,39 @@ pub async fn join_room_by_id_helper( ) .await?; - info!("send_join finished"); + if let Some(signed_raw) = send_join_response.room_state.event { + let Ok((signed_event_id, signed_value)) = gen_event_id_canonical_json(&signed_raw, &room_version_id) else { + // Event could not be converted to canonical json + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Could not convert event to canonical json.", + )); + }; - if join_authorized_via_users_server.is_some() { - match &room_version_id { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 => { - warn!( - "Found `join_authorised_via_users_server` but room {} is version {}. Ignoring.", - room_id, &room_version_id - ); - }, - // only room versions 8 and above using `join_authorized_via_users_server` (restricted joins) need to - // validate and send signatures - RoomVersionId::V8 | RoomVersionId::V9 | RoomVersionId::V10 | RoomVersionId::V11 => { - if let Some(signed_raw) = &send_join_response.room_state.event { - info!( - "There is a signed event. This room is probably using restricted joins. Adding signature \ - to our event" - ); - let Ok((signed_event_id, signed_value)) = - gen_event_id_canonical_json(signed_raw, &room_version_id) - else { - // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); - }; - - if signed_event_id != event_id { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Server sent event with wrong event id", - )); - } - - match signed_value["signatures"] - .as_object() - .ok_or(Error::BadRequest( - ErrorKind::InvalidParam, - "Server sent invalid signatures type", - )) - .and_then(|e| { - e.get(remote_server.as_str()).ok_or(Error::BadRequest( - ErrorKind::InvalidParam, - "Server did not send its signature", - )) - }) { - Ok(signature) => { - join_event - .get_mut("signatures") - .expect("we created a valid pdu") - .as_object_mut() - .expect("we created a valid pdu") - .insert(remote_server.to_string(), signature.clone()); - }, - Err(e) => { - warn!( - "Server {remote_server} sent invalid signature in sendjoin signatures for event \ - {signed_value:?}: {e:?}", - ); - }, - } - } - }, - _ => { - warn!( - "Unexpected or unsupported room version {} for room {}", - &room_version_id, room_id - ); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, + if signed_event_id != event_id { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Server sent event with wrong event id", + )); } - } - - services().rooms.short.get_or_create_shortroomid(room_id)?; - - info!("Parsing join event"); - let parsed_join_pdu = PduEvent::from_id_val(event_id, join_event.clone()) - .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; - - let mut state = HashMap::new(); - let pub_key_map = RwLock::new(BTreeMap::new()); - - info!("Fetching join signing keys"); - services() - .rooms - .event_handler - .fetch_join_signing_keys(&send_join_response, &room_version_id, &pub_key_map) - .await?; - - info!("Going through send_join response room_state"); - for result in send_join_response - .room_state - .state - .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version_id, &pub_key_map)) - { - let Ok((event_id, value)) = result.await else { - continue; - }; - - let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| { - warn!("Invalid PDU in send_join response: {} {:?}", e, value); - Error::BadServerResponse("Invalid PDU in send_join response.") - })?; + drop(state_lock); + let pub_key_map = RwLock::new(BTreeMap::new()); services() .rooms - .outlier - .add_pdu_outlier(&event_id, &value)?; - if let Some(state_key) = &pdu.state_key { - let shortstatekey = services() - .rooms - .short - .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key)?; - state.insert(shortstatekey, pdu.event_id.clone()); - } - } - - info!("Going through send_join response auth_chain"); - for result in send_join_response - .room_state - .auth_chain - .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version_id, &pub_key_map)) - { - let Ok((event_id, value)) = result.await else { - continue; - }; - - services() - .rooms - .outlier - .add_pdu_outlier(&event_id, &value)?; - } - - debug!("Running send_join auth check"); - - let auth_check = state_res::event_auth::auth_check( - &state_res::RoomVersion::new(&room_version_id).expect("room version is supported"), - &parsed_join_pdu, - None::, // TODO: third party invite - |k, s| { - services() - .rooms - .timeline - .get_pdu( - state.get( - &services() - .rooms - .short - .get_or_create_shortstatekey(&k.to_string().into(), s) - .ok()?, - )?, - ) - .ok()? - }, - ) - .map_err(|e| { - warn!("Auth check failed: {e}"); - Error::BadRequest(ErrorKind::forbidden(), "Auth check failed") - })?; - - if !auth_check { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Auth check failed")); - } - - info!("Saving state from send_join"); - let (statehash_before_join, new, removed) = services().rooms.state_compressor.save_state( - room_id, - Arc::new( - state - .into_iter() - .map(|(k, id)| { - services() - .rooms - .state_compressor - .compress_state_event(k, &id) - }) - .collect::>()?, - ), - )?; - - services() - .rooms - .state - .force_state(room_id, statehash_before_join, new, removed, &state_lock) - .await?; - - info!("Updating joined counts for new room"); - services().rooms.state_cache.update_joined_count(room_id)?; - - // We append to state before appending the pdu, so we don't have a moment in - // time with the pdu without it's state. This is okay because append_pdu can't - // fail. - let statehash_after_join = services().rooms.state.append_to_state(&parsed_join_pdu)?; - - info!("Appending new room join event"); - services() - .rooms - .timeline - .append_pdu( - &parsed_join_pdu, - join_event, - vec![(*parsed_join_pdu.event_id).to_owned()], - &state_lock, - ) - .await?; - - info!("Setting final room state for new room"); - // We set the room state after inserting the pdu, so that we never have a moment - // in time where events in the current room state do not exist - services() - .rooms - .state - .set_room_state(room_id, statehash_after_join, &state_lock)?; - } else { - info!("We can join locally"); - - let join_rules_event = - services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomJoinRules, "")?; - - let join_rules_event_content: Option = join_rules_event - .as_ref() - .map(|join_rules_event| { - serde_json::from_str(join_rules_event.content.get()).map_err(|e| { - warn!("Invalid join rules event: {}", e); - Error::bad_database("Invalid join rules event in db.") - }) - }) - .transpose()?; - - let restriction_rooms = match join_rules_event_content { - Some(RoomJoinRulesEventContent { - join_rule: JoinRule::Restricted(restricted) | JoinRule::KnockRestricted(restricted), - }) => restricted - .allow - .into_iter() - .filter_map(|a| match a { - AllowRule::RoomMembership(r) => Some(r.room_id), - _ => None, - }) - .collect(), - _ => Vec::new(), - }; - - let local_members = services() - .rooms - .state_cache - .room_members(room_id) - .filter_map(Result::ok) - .filter(|user| user_is_local(user)) - .collect::>(); - - let mut authorized_user: Option = None; - - if restriction_rooms.iter().any(|restriction_room_id| { - services() - .rooms - .state_cache - .is_joined(sender_user, restriction_room_id) - .unwrap_or(false) - }) { - for user in local_members { - if services() - .rooms - .state_accessor - .user_can_invite(room_id, &user, sender_user, &state_lock) - .await - .unwrap_or(false) - { - authorized_user = Some(user); - break; - } - } - } - - let event = RoomMemberEventContent { - membership: MembershipState::Join, - displayname: services().users.displayname(sender_user)?, - avatar_url: services().users.avatar_url(sender_user)?, - is_direct: None, - third_party_invite: None, - blurhash: services().users.blurhash(sender_user)?, - reason: reason.clone(), - join_authorized_via_users_server: authorized_user, - }; - - // Try normal join first - let error = match services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_user.to_string()), - redacts: None, - }, - sender_user, - room_id, - &state_lock, - ) - .await - { - Ok(_event_id) => return Ok(join_room_by_id::v3::Response::new(room_id.to_owned())), - Err(e) => e, - }; - - if !restriction_rooms.is_empty() - && servers - .iter() - .any(|server_name| !server_is_ours(server_name)) - { - info!( - "We couldn't do the join locally, maybe federation can help to satisfy the restricted join \ - requirements" - ); - let (make_join_response, remote_server) = make_join_request(sender_user, room_id, servers).await?; - - let room_version_id = match make_join_response.room_version { - Some(room_version_id) - if services() - .globals - .supported_room_versions() - .contains(&room_version_id) => - { - room_version_id - }, - _ => return Err(Error::BadServerResponse("Room version is not supported")), - }; - let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) - .map_err(|_| Error::BadServerResponse("Invalid make_join event json received from server."))?; - let join_authorized_via_users_server = join_event_stub - .get("content") - .map(|s| { - s.as_object()? - .get("join_authorised_via_users_server")? - .as_str() - }) - .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()); - // TODO: Is origin needed? - join_event_stub.insert( - "origin".to_owned(), - CanonicalJsonValue::String(services().globals.server_name().as_str().to_owned()), - ); - join_event_stub.insert( - "origin_server_ts".to_owned(), - CanonicalJsonValue::Integer( - utils::millis_since_unix_epoch() - .try_into() - .expect("Timestamp is valid js_int value"), - ), - ); - join_event_stub.insert( - "content".to_owned(), - to_canonical_value(RoomMemberEventContent { - membership: MembershipState::Join, - displayname: services().users.displayname(sender_user)?, - avatar_url: services().users.avatar_url(sender_user)?, - is_direct: None, - third_party_invite: None, - blurhash: services().users.blurhash(sender_user)?, - reason, - join_authorized_via_users_server, - }) - .expect("event is valid, we just created it"), - ); - - // We keep the "event_id" in the pdu only in v1 or - // v2 rooms - match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => { - join_event_stub.remove("event_id"); - }, - }; - - // In order to create a compatible ref hash (EventID) the `hashes` field needs - // to be present - ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), - &mut join_event_stub, - &room_version_id, - ) - .expect("event is valid, we just created it"); - - // Generate event id - let event_id = format!( - "${}", - ruma::signatures::reference_hash(&join_event_stub, &room_version_id) - .expect("ruma can calculate reference hashes") - ); - let event_id = - <&EventId>::try_from(event_id.as_str()).expect("ruma's reference hashes are valid event ids"); - - // Add event_id back - join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); - - // It has enough fields to be called a proper event now - let join_event = join_event_stub; - - let send_join_response = services() - .sending - .send_federation_request( - &remote_server, - federation::membership::create_join_event::v2::Request { - room_id: room_id.to_owned(), - event_id: event_id.to_owned(), - pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), - omit_members: false, - }, - ) + .event_handler + .fetch_required_signing_keys([&signed_value], &pub_key_map) + .await?; + services() + .rooms + .event_handler + .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true, &pub_key_map) .await?; - - if let Some(signed_raw) = send_join_response.room_state.event { - let Ok((signed_event_id, signed_value)) = gen_event_id_canonical_json(&signed_raw, &room_version_id) - else { - // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); - }; - - if signed_event_id != event_id { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Server sent event with wrong event id", - )); - } - - drop(state_lock); - let pub_key_map = RwLock::new(BTreeMap::new()); - services() - .rooms - .event_handler - .fetch_required_signing_keys([&signed_value], &pub_key_map) - .await?; - services() - .rooms - .event_handler - .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true, &pub_key_map) - .await?; - } else { - return Err(error); - } } else { return Err(error); } + } else { + return Err(error); } Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) From db2c9f28b60977addb9b4605bd1229b7e5c56c16 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 04:40:19 +0000 Subject: [PATCH 0123/2091] split admin room moderation commands prior stack frame allocated 170 KiB Signed-off-by: Jason Volk --- clippy.toml | 2 +- src/admin/room/room_moderation_commands.rs | 856 +++++++++++---------- 2 files changed, 430 insertions(+), 428 deletions(-) diff --git a/clippy.toml b/clippy.toml index 1743ee5c..c942b93c 100644 --- a/clippy.toml +++ b/clippy.toml @@ -2,6 +2,6 @@ array-size-threshold = 4096 cognitive-complexity-threshold = 94 # TODO reduce me ALARA excessive-nesting-threshold = 11 # TODO reduce me to 4 or 5 future-size-threshold = 7745 # TODO reduce me ALARA -stack-size-threshold = 173577 # reduce me ALARA +stack-size-threshold = 144000 # reduce me ALARA too-many-lines-threshold = 700 # TODO reduce me to <= 100 type-complexity-threshold = 250 # reduce me to ~200 diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index f5176147..6b759c9b 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -18,91 +18,103 @@ pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> force, room, disable_federation, - } => { - debug!("Got room alias or ID: {}", room); + } => ban_room(body, force, room, disable_federation).await, + RoomModerationCommand::BanListOfRooms { + force, + disable_federation, + } => ban_list_of_rooms(body, force, disable_federation).await, + RoomModerationCommand::UnbanRoom { + room, + enable_federation, + } => unban_room(body, room, enable_federation).await, + RoomModerationCommand::ListBannedRooms => list_banned_rooms(body).await, + } +} - let admin_room_alias: Box = format!("#admins:{}", services().globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid alias name"); +async fn ban_room( + _body: Vec<&str>, force: bool, room: Box, disable_federation: bool, +) -> Result { + debug!("Got room alias or ID: {}", room); - if let Some(admin_room_id) = Service::get_admin_room().await? { - if room.to_string().eq(&admin_room_id) || room.to_string().eq(&admin_room_alias) { - return Ok(RoomMessageEventContent::text_plain("Not allowed to ban the admin room.")); - } + let admin_room_alias: Box = format!("#admins:{}", services().globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid alias name"); + + if let Some(admin_room_id) = Service::get_admin_room().await? { + if room.to_string().eq(&admin_room_id) || room.to_string().eq(&admin_room_alias) { + return Ok(RoomMessageEventContent::text_plain("Not allowed to ban the admin room.")); + } + } + + let room_id = if room.is_room_id() { + let room_id = match RoomId::parse(&room) { + Ok(room_id) => room_id, + Err(e) => { + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to parse room ID {room}. Please note that this requires a full room ID \ + (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" + ))) + }, + }; + + debug!("Room specified is a room ID, banning room ID"); + + services().rooms.metadata.ban_room(&room_id, true)?; + + room_id + } else if room.is_room_alias_id() { + let room_alias = match RoomAliasId::parse(&room) { + Ok(room_alias) => room_alias, + Err(e) => { + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to parse room ID {room}. Please note that this requires a full room ID \ + (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" + ))) + }, + }; + + debug!( + "Room specified is not a room ID, attempting to resolve room alias to a room ID locally, if not using \ + get_alias_helper to fetch room ID remotely" + ); + + let room_id = if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { + room_id + } else { + debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); + + match get_alias_helper(room_alias, None).await { + Ok(response) => { + debug!("Got federation response fetching room ID for room {room}: {:?}", response); + response.room_id + }, + Err(e) => { + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to resolve room alias {room} to a room ID: {e}" + ))); + }, } + }; - let room_id = if room.is_room_id() { - let room_id = match RoomId::parse(&room) { - Ok(room_id) => room_id, - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse room ID {room}. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" - ))) - }, - }; + services().rooms.metadata.ban_room(&room_id, true)?; - debug!("Room specified is a room ID, banning room ID"); + room_id + } else { + return Ok(RoomMessageEventContent::text_plain( + "Room specified is not a room ID or room alias. Please note that this requires a full room ID \ + (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`)", + )); + }; - services().rooms.metadata.ban_room(&room_id, true)?; - - room_id - } else if room.is_room_alias_id() { - let room_alias = match RoomAliasId::parse(&room) { - Ok(room_alias) => room_alias, - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse room ID {room}. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" - ))) - }, - }; - - debug!( - "Room specified is not a room ID, attempting to resolve room alias to a room ID locally, if not \ - using get_alias_helper to fetch room ID remotely" - ); - - let room_id = if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { - room_id - } else { - debug!( - "We don't have this room alias to a room ID locally, attempting to fetch room ID over \ - federation" - ); - - match get_alias_helper(room_alias, None).await { - Ok(response) => { - debug!("Got federation response fetching room ID for room {room}: {:?}", response); - response.room_id - }, - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to resolve room alias {room} to a room ID: {e}" - ))); - }, - } - }; - - services().rooms.metadata.ban_room(&room_id, true)?; - - room_id - } else { - return Ok(RoomMessageEventContent::text_plain( - "Room specified is not a room ID or room alias. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`)", - )); - }; - - debug!("Making all users leave the room {}", &room); - if force { - for local_user in services() - .rooms - .state_cache - .room_members(&room_id) - .filter_map(|user| { - user.ok().filter(|local_user| { - user_is_local(local_user) + debug!("Making all users leave the room {}", &room); + if force { + for local_user in services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(|user| { + user.ok().filter(|local_user| { + user_is_local(local_user) // additional wrapped check here is to avoid adding remote users // who are in the admin room to the list of local users (would fail auth check) && (user_is_local(local_user) @@ -110,18 +122,221 @@ pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> .users .is_admin(local_user) .unwrap_or(true)) // since this is a force - // operation, assume user - // is an admin if somehow - // this fails + // operation, assume user + // is an admin if somehow + // this fails + }) + }) + .collect::>() + { + debug!( + "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", + &local_user, &room_id + ); + + if let Err(e) = leave_room(&local_user, &room_id, None).await { + warn!(%e, "Failed to leave room"); + } + } + } else { + for local_user in services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(|user| { + user.ok().filter(|local_user| { + local_user.server_name() == services().globals.server_name() + // additional wrapped check here is to avoid adding remote users + // who are in the admin room to the list of local users (would fail auth check) + && (local_user.server_name() + == services().globals.server_name() + && !services() + .users + .is_admin(local_user) + .unwrap_or(false)) + }) + }) + .collect::>() + { + debug!("Attempting leave for user {} in room {}", &local_user, &room_id); + if let Err(e) = leave_room(&local_user, &room_id, None).await { + error!( + "Error attempting to make local user {} leave room {} during room banning: {}", + &local_user, &room_id, e + ); + return Ok(RoomMessageEventContent::text_plain(format!( + "Error attempting to make local user {} leave room {} during room banning (room is still banned \ + but not removing any more users): {}\nIf you would like to ignore errors, use --force", + &local_user, &room_id, e + ))); + } + } + } + + if disable_federation { + services().rooms.metadata.disable_room(&room_id, true)?; + return Ok(RoomMessageEventContent::text_plain( + "Room banned, removed all our local users, and disabled incoming federation with room.", + )); + } + + Ok(RoomMessageEventContent::text_plain( + "Room banned and removed all our local users, use `!admin federation disable-room` to stop receiving new \ + inbound federation events as well if needed.", + )) +} + +async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: bool) -> Result { + if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { + let rooms_s = body.clone().drain(1..body.len() - 1).collect::>(); + + let admin_room_alias: Box = format!("#admins:{}", services().globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid alias name"); + + let mut room_ban_count: usize = 0; + let mut room_ids: Vec = Vec::new(); + + for &room in &rooms_s { + match <&RoomOrAliasId>::try_from(room) { + Ok(room_alias_or_id) => { + if let Some(admin_room_id) = Service::get_admin_room().await? { + if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(&admin_room_alias) { + info!("User specified admin room in bulk ban list, ignoring"); + continue; + } + } + + if room_alias_or_id.is_room_id() { + let room_id = match RoomId::parse(room_alias_or_id) { + Ok(room_id) => room_id, + Err(e) => { + if force { + // ignore rooms we failed to parse if we're force banning + warn!( + "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ + logging here: {e}" + ); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!( + "{room} is not a valid room ID or room alias, please fix the list and try again: \ + {e}" + ))); + }, + }; + + room_ids.push(room_id); + } + + if room_alias_or_id.is_room_alias_id() { + match RoomAliasId::parse(room_alias_or_id) { + Ok(room_alias) => { + let room_id = + if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { + room_id + } else { + debug!( + "We don't have this room alias to a room ID locally, attempting to fetch \ + room ID over federation" + ); + + match get_alias_helper(room_alias, None).await { + Ok(response) => { + debug!( + "Got federation response fetching room ID for room {room}: {:?}", + response + ); + response.room_id + }, + Err(e) => { + // don't fail if force blocking + if force { + warn!("Failed to resolve room alias {room} to a room ID: {e}"); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to resolve room alias {room} to a room ID: {e}" + ))); + }, + } + }; + + room_ids.push(room_id); + }, + Err(e) => { + if force { + // ignore rooms we failed to parse if we're force deleting + error!( + "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ + logging here: {e}" + ); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!( + "{room} is not a valid room ID or room alias, please fix the list and try again: \ + {e}" + ))); + }, + } + } + }, + Err(e) => { + if force { + // ignore rooms we failed to parse if we're force deleting + error!( + "Error parsing room \"{room}\" during bulk room banning, ignoring error and logging here: \ + {e}" + ); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!( + "{room} is not a valid room ID or room alias, please fix the list and try again: {e}" + ))); + }, + } + } + + for room_id in room_ids { + if services().rooms.metadata.ban_room(&room_id, true).is_ok() { + debug!("Banned {room_id} successfully"); + room_ban_count = room_ban_count.saturating_add(1); + } + + debug!("Making all users leave the room {}", &room_id); + if force { + for local_user in services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(|user| { + user.ok().filter(|local_user| { + local_user.server_name() == services().globals.server_name() + // additional wrapped check here is to avoid adding remote users + // who are in the admin room to the list of local users (would fail auth check) + && (local_user.server_name() + == services().globals.server_name() + && services() + .users + .is_admin(local_user) + .unwrap_or(true)) // since this is a + // force operation, + // assume user is + // an admin if + // somehow this + // fails }) }) .collect::>() { debug!( "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", - &local_user, &room_id + &local_user, room_id ); - if let Err(e) = leave_room(&local_user, &room_id, None).await { warn!(%e, "Failed to leave room"); } @@ -134,215 +349,6 @@ pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> .filter_map(|user| { user.ok().filter(|local_user| { local_user.server_name() == services().globals.server_name() - // additional wrapped check here is to avoid adding remote users - // who are in the admin room to the list of local users (would fail auth check) - && (local_user.server_name() - == services().globals.server_name() - && !services() - .users - .is_admin(local_user) - .unwrap_or(false)) - }) - }) - .collect::>() - { - debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(&local_user, &room_id, None).await { - error!( - "Error attempting to make local user {} leave room {} during room banning: {}", - &local_user, &room_id, e - ); - return Ok(RoomMessageEventContent::text_plain(format!( - "Error attempting to make local user {} leave room {} during room banning (room is still \ - banned but not removing any more users): {}\nIf you would like to ignore errors, use \ - --force", - &local_user, &room_id, e - ))); - } - } - } - - if disable_federation { - services().rooms.metadata.disable_room(&room_id, true)?; - return Ok(RoomMessageEventContent::text_plain( - "Room banned, removed all our local users, and disabled incoming federation with room.", - )); - } - - Ok(RoomMessageEventContent::text_plain( - "Room banned and removed all our local users, use `!admin federation disable-room` to stop receiving \ - new inbound federation events as well if needed.", - )) - }, - RoomModerationCommand::BanListOfRooms { - force, - disable_federation, - } => { - if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { - let rooms_s = body.clone().drain(1..body.len() - 1).collect::>(); - - let admin_room_alias: Box = format!("#admins:{}", services().globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid alias name"); - - let mut room_ban_count: usize = 0; - let mut room_ids: Vec = Vec::new(); - - for &room in &rooms_s { - match <&RoomOrAliasId>::try_from(room) { - Ok(room_alias_or_id) => { - if let Some(admin_room_id) = Service::get_admin_room().await? { - if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(&admin_room_alias) { - info!("User specified admin room in bulk ban list, ignoring"); - continue; - } - } - - if room_alias_or_id.is_room_id() { - let room_id = match RoomId::parse(room_alias_or_id) { - Ok(room_id) => room_id, - Err(e) => { - if force { - // ignore rooms we failed to parse if we're force banning - warn!( - "Error parsing room \"{room}\" during bulk room banning, ignoring \ - error and logging here: {e}" - ); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try \ - again: {e}" - ))); - }, - }; - - room_ids.push(room_id); - } - - if room_alias_or_id.is_room_alias_id() { - match RoomAliasId::parse(room_alias_or_id) { - Ok(room_alias) => { - let room_id = if let Some(room_id) = - services().rooms.alias.resolve_local_alias(&room_alias)? - { - room_id - } else { - debug!( - "We don't have this room alias to a room ID locally, attempting to \ - fetch room ID over federation" - ); - - match get_alias_helper(room_alias, None).await { - Ok(response) => { - debug!( - "Got federation response fetching room ID for room {room}: \ - {:?}", - response - ); - response.room_id - }, - Err(e) => { - // don't fail if force blocking - if force { - warn!("Failed to resolve room alias {room} to a room ID: {e}"); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to resolve room alias {room} to a room ID: {e}" - ))); - }, - } - }; - - room_ids.push(room_id); - }, - Err(e) => { - if force { - // ignore rooms we failed to parse if we're force deleting - error!( - "Error parsing room \"{room}\" during bulk room banning, ignoring \ - error and logging here: {e}" - ); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try \ - again: {e}" - ))); - }, - } - } - }, - Err(e) => { - if force { - // ignore rooms we failed to parse if we're force deleting - error!( - "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ - logging here: {e}" - ); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try again: {e}" - ))); - }, - } - } - - for room_id in room_ids { - if services().rooms.metadata.ban_room(&room_id, true).is_ok() { - debug!("Banned {room_id} successfully"); - room_ban_count = room_ban_count.saturating_add(1); - } - - debug!("Making all users leave the room {}", &room_id); - if force { - for local_user in services() - .rooms - .state_cache - .room_members(&room_id) - .filter_map(|user| { - user.ok().filter(|local_user| { - local_user.server_name() == services().globals.server_name() - // additional wrapped check here is to avoid adding remote users - // who are in the admin room to the list of local users (would fail auth check) - && (local_user.server_name() - == services().globals.server_name() - && services() - .users - .is_admin(local_user) - .unwrap_or(true)) // since this is a - // force operation, - // assume user is - // an admin if - // somehow this - // fails - }) - }) - .collect::>() - { - debug!( - "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting \ - admins too)", - &local_user, room_id - ); - if let Err(e) = leave_room(&local_user, &room_id, None).await { - warn!(%e, "Failed to leave room"); - } - } - } else { - for local_user in services() - .rooms - .state_cache - .room_members(&room_id) - .filter_map(|user| { - user.ok().filter(|local_user| { - local_user.server_name() == services().globals.server_name() // additional wrapped check here is to avoid adding remote users // who are in the admin room to the list of local users (would fail auth check) && (local_user.server_name() @@ -351,154 +357,150 @@ pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> .users .is_admin(local_user) .unwrap_or(false)) - }) - }) - .collect::>() - { - debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(&local_user, &room_id, None).await { - error!( - "Error attempting to make local user {} leave room {} during bulk room banning: {}", - &local_user, &room_id, e - ); - return Ok(RoomMessageEventContent::text_plain(format!( - "Error attempting to make local user {} leave room {} during room banning (room \ - is still banned but not removing any more users and not banning any more rooms): \ - {}\nIf you would like to ignore errors, use --force", - &local_user, &room_id, e - ))); - } - } - } - - if disable_federation { - services().rooms.metadata.disable_room(&room_id, true)?; + }) + }) + .collect::>() + { + debug!("Attempting leave for user {} in room {}", &local_user, &room_id); + if let Err(e) = leave_room(&local_user, &room_id, None).await { + error!( + "Error attempting to make local user {} leave room {} during bulk room banning: {}", + &local_user, &room_id, e + ); + return Ok(RoomMessageEventContent::text_plain(format!( + "Error attempting to make local user {} leave room {} during room banning (room is still \ + banned but not removing any more users and not banning any more rooms): {}\nIf you would \ + like to ignore errors, use --force", + &local_user, &room_id, e + ))); } } + } - if disable_federation { - return Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and disabled \ - incoming federation with the room." - ))); - } + if disable_federation { + services().rooms.metadata.disable_room(&room_id, true)?; + } + } + + if disable_federation { + return Ok(RoomMessageEventContent::text_plain(format!( + "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and disabled \ + incoming federation with the room." + ))); + } + return Ok(RoomMessageEventContent::text_plain(format!( + "Finished bulk room ban, banned {room_ban_count} total rooms and evicted all users." + ))); + } + + Ok(RoomMessageEventContent::text_plain( + "Expected code block in command body. Add --help for details.", + )) +} + +async fn unban_room( + _body: Vec<&str>, room: Box, enable_federation: bool, +) -> Result { + let room_id = if room.is_room_id() { + let room_id = match RoomId::parse(&room) { + Ok(room_id) => room_id, + Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk room ban, banned {room_ban_count} total rooms and evicted all users." - ))); - } + "Failed to parse room ID {room}. Please note that this requires a full room ID \ + (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" + ))) + }, + }; - Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )) - }, - RoomModerationCommand::UnbanRoom { - room, - enable_federation, - } => { - let room_id = if room.is_room_id() { - let room_id = match RoomId::parse(&room) { - Ok(room_id) => room_id, - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse room ID {room}. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" - ))) - }, - }; + debug!("Room specified is a room ID, unbanning room ID"); - debug!("Room specified is a room ID, unbanning room ID"); + services().rooms.metadata.ban_room(&room_id, false)?; - services().rooms.metadata.ban_room(&room_id, false)?; + room_id + } else if room.is_room_alias_id() { + let room_alias = match RoomAliasId::parse(&room) { + Ok(room_alias) => room_alias, + Err(e) => { + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to parse room ID {room}. Please note that this requires a full room ID \ + (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" + ))) + }, + }; - room_id - } else if room.is_room_alias_id() { - let room_alias = match RoomAliasId::parse(&room) { - Ok(room_alias) => room_alias, - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse room ID {room}. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" - ))) - }, - }; + debug!( + "Room specified is not a room ID, attempting to resolve room alias to a room ID locally, if not using \ + get_alias_helper to fetch room ID remotely" + ); - debug!( - "Room specified is not a room ID, attempting to resolve room alias to a room ID locally, if not \ - using get_alias_helper to fetch room ID remotely" - ); + let room_id = if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { + room_id + } else { + debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); - let room_id = if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { - room_id - } else { - debug!( - "We don't have this room alias to a room ID locally, attempting to fetch room ID over \ - federation" - ); - - match get_alias_helper(room_alias, None).await { - Ok(response) => { - debug!("Got federation response fetching room ID for room {room}: {:?}", response); - response.room_id - }, - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to resolve room alias {room} to a room ID: {e}" - ))); - }, - } - }; - - services().rooms.metadata.ban_room(&room_id, false)?; - - room_id - } else { - return Ok(RoomMessageEventContent::text_plain( - "Room specified is not a room ID or room alias. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`)", - )); - }; - - if enable_federation { - services().rooms.metadata.disable_room(&room_id, false)?; - return Ok(RoomMessageEventContent::text_plain("Room unbanned.")); - } - - Ok(RoomMessageEventContent::text_plain( - "Room unbanned, you may need to re-enable federation with the room using enable-room if this is a \ - remote room to make it fully functional.", - )) - }, - RoomModerationCommand::ListBannedRooms => { - let rooms = services() - .rooms - .metadata - .list_banned_rooms() - .collect::, _>>(); - - match rooms { - Ok(room_ids) => { - // TODO: add room name from our state cache if available, default to the room ID - // as the room name if we dont have it TODO: do same if we have a room alias for - // this - let plain_list = room_ids.iter().fold(String::new(), |mut output, room_id| { - writeln!(output, "- `{room_id}`").unwrap(); - output - }); - - let html_list = room_ids.iter().fold(String::new(), |mut output, room_id| { - writeln!(output, "
  • {}
    • ", escape_html(room_id.as_ref())).unwrap(); - output - }); - - let plain = format!("Rooms:\n{plain_list}"); - let html = format!("Rooms:\n
      {html_list}
    "); - Ok(RoomMessageEventContent::text_html(plain, html)) + match get_alias_helper(room_alias, None).await { + Ok(response) => { + debug!("Got federation response fetching room ID for room {room}: {:?}", response); + response.room_id }, Err(e) => { - error!("Failed to list banned rooms: {}", e); - Ok(RoomMessageEventContent::text_plain(format!("Unable to list room aliases: {e}"))) + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to resolve room alias {room} to a room ID: {e}" + ))); }, } + }; + + services().rooms.metadata.ban_room(&room_id, false)?; + + room_id + } else { + return Ok(RoomMessageEventContent::text_plain( + "Room specified is not a room ID or room alias. Please note that this requires a full room ID \ + (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`)", + )); + }; + + if enable_federation { + services().rooms.metadata.disable_room(&room_id, false)?; + return Ok(RoomMessageEventContent::text_plain("Room unbanned.")); + } + + Ok(RoomMessageEventContent::text_plain( + "Room unbanned, you may need to re-enable federation with the room using enable-room if this is a remote room \ + to make it fully functional.", + )) +} + +async fn list_banned_rooms(_body: Vec<&str>) -> Result { + let rooms = services() + .rooms + .metadata + .list_banned_rooms() + .collect::, _>>(); + + match rooms { + Ok(room_ids) => { + // TODO: add room name from our state cache if available, default to the room ID + // as the room name if we dont have it TODO: do same if we have a room alias for + // this + let plain_list = room_ids.iter().fold(String::new(), |mut output, room_id| { + writeln!(output, "- `{room_id}`").unwrap(); + output + }); + + let html_list = room_ids.iter().fold(String::new(), |mut output, room_id| { + writeln!(output, "
  • {}
    • ", escape_html(room_id.as_ref())).unwrap(); + output + }); + + let plain = format!("Rooms:\n{plain_list}"); + let html = format!("Rooms:\n
      {html_list}
    "); + Ok(RoomMessageEventContent::text_html(plain, html)) + }, + Err(e) => { + error!("Failed to list banned rooms: {}", e); + Ok(RoomMessageEventContent::text_plain(format!("Unable to list room aliases: {e}"))) }, } } From b65f05ce19f7f581a5770d82b82b2ee25a9e9971 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 29 May 2024 14:57:53 +0000 Subject: [PATCH 0124/2091] simplify lifetime parameters Signed-off-by: Jason Volk --- src/service/rooms/state_cache/data.rs | 4 ++-- src/service/rooms/state_cache/mod.rs | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 1c0571c9..acf390d9 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -72,7 +72,7 @@ pub trait Data: Send + Sync { fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result>; /// Returns an iterator over all rooms this user joined. - fn rooms_joined<'a>(&'a self, user_id: &UserId) -> Box> + 'a>; + fn rooms_joined(&self, user_id: &UserId) -> Box> + '_>; /// Returns an iterator over all rooms a user was invited to. fn rooms_invited<'a>(&'a self, user_id: &UserId) -> StrippedStateEventIter<'a>; @@ -494,7 +494,7 @@ impl Data for KeyValueDatabase { /// Returns an iterator over all rooms this user joined. #[tracing::instrument(skip(self))] - fn rooms_joined<'a>(&'a self, user_id: &UserId) -> Box> + 'a> { + fn rooms_joined(&self, user_id: &UserId) -> Box> + '_> { Box::new( self.userroomid_joined .scan_prefix(user_id.as_bytes().to_vec()) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 1cfd56db..2b3f5fb7 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -239,7 +239,7 @@ impl Service { /// Returns an iterator of all servers participating in this room. #[tracing::instrument(skip(self))] - pub fn room_servers<'a>(&'a self, room_id: &RoomId) -> impl Iterator> + 'a { + pub fn room_servers(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_servers(room_id) } @@ -251,7 +251,7 @@ impl Service { /// Returns an iterator of all rooms a server participates in (as far as we /// know). #[tracing::instrument(skip(self))] - pub fn server_rooms<'a>(&'a self, server: &ServerName) -> impl Iterator> + 'a { + pub fn server_rooms(&self, server: &ServerName) -> impl Iterator> + '_ { self.db.server_rooms(server) } @@ -282,7 +282,7 @@ impl Service { /// Returns an iterator over all joined members of a room. #[tracing::instrument(skip(self))] - pub fn room_members<'a>(&'a self, room_id: &RoomId) -> impl Iterator> + 'a { + pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_members(room_id) } @@ -308,13 +308,13 @@ impl Service { /// Returns an iterator over all User IDs who ever joined a room. #[tracing::instrument(skip(self))] - pub fn room_useroncejoined<'a>(&'a self, room_id: &RoomId) -> impl Iterator> + 'a { + pub fn room_useroncejoined(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_useroncejoined(room_id) } /// Returns an iterator over all invited members of a room. #[tracing::instrument(skip(self))] - pub fn room_members_invited<'a>(&'a self, room_id: &RoomId) -> impl Iterator> + 'a { + pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_members_invited(room_id) } @@ -330,15 +330,15 @@ impl Service { /// Returns an iterator over all rooms this user joined. #[tracing::instrument(skip(self))] - pub fn rooms_joined<'a>(&'a self, user_id: &UserId) -> impl Iterator> + 'a { + pub fn rooms_joined(&self, user_id: &UserId) -> impl Iterator> + '_ { self.db.rooms_joined(user_id) } /// Returns an iterator over all rooms a user was invited to. #[tracing::instrument(skip(self))] - pub fn rooms_invited<'a>( - &'a self, user_id: &UserId, - ) -> impl Iterator>)>> + 'a { + pub fn rooms_invited( + &self, user_id: &UserId, + ) -> impl Iterator>)>> + '_ { self.db.rooms_invited(user_id) } @@ -354,9 +354,9 @@ impl Service { /// Returns an iterator over all rooms a user left. #[tracing::instrument(skip(self))] - pub fn rooms_left<'a>( - &'a self, user_id: &UserId, - ) -> impl Iterator>)>> + 'a { + pub fn rooms_left( + &self, user_id: &UserId, + ) -> impl Iterator>)>> + '_ { self.db.rooms_left(user_id) } From 67f4285504c20206c85da9a62e6c0fda521a2c3a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 05:15:27 +0000 Subject: [PATCH 0125/2091] Fix branches sharing code Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/database/rocksdb/mod.rs | 9 +++------ src/service/admin.rs | 6 ++---- src/service/rooms/threads/mod.rs | 3 +-- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a36e1fd1..7c263042 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -715,7 +715,6 @@ correctness = "warn" nursery = "warn" ## some sadness -branches_sharing_code = { level = "allow", priority = 1 } # TODO derive_partial_eq_without_eq = { level = "allow", priority = 1 } # TODO equatable_if_let = { level = "allow", priority = 1 } # TODO future_not_send = { level = "allow", priority = 1 } # TODO diff --git a/src/database/rocksdb/mod.rs b/src/database/rocksdb/mod.rs index 28ce5072..f0c3882f 100644 --- a/src/database/rocksdb/mod.rs +++ b/src/database/rocksdb/mod.rs @@ -182,7 +182,7 @@ impl KeyValueDatabaseEngine for Arc { let options = BackupEngineOptions::new(path.unwrap())?; let mut engine = BackupEngine::open(&options, &self.env)?; - let ret = if self.config.database_backups_to_keep > 0 { + if self.config.database_backups_to_keep > 0 { if let Err(e) = engine.create_new_backup_flush(&self.rocks, true) { return Err(Box::new(e)); } @@ -193,10 +193,7 @@ impl KeyValueDatabaseEngine for Arc { "Created database backup #{} using {} bytes in {} files", info.backup_id, info.size, info.num_files, ); - Ok(()) - } else { - Ok(()) - }; + } if self.config.database_backups_to_keep >= 0 { let keep = u32::try_from(self.config.database_backups_to_keep)?; @@ -205,7 +202,7 @@ impl KeyValueDatabaseEngine for Arc { } } - ret + Ok(()) } fn backup_list(&self) -> Result { diff --git a/src/service/admin.rs b/src/service/admin.rs index 19923d0a..63d6d5c6 100644 --- a/src/service/admin.rs +++ b/src/service/admin.rs @@ -529,10 +529,8 @@ impl Service { &room_id, &state_lock, ).await?; - - Ok(()) - } else { - Ok(()) } + + Ok(()) } } diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index a7d5c434..6c48e842 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -91,11 +91,10 @@ impl Service { let mut users = Vec::new(); if let Some(userids) = self.db.get_participants(root_id)? { users.extend_from_slice(&userids); - users.push(pdu.sender.clone()); } else { users.push(root_pdu.sender); - users.push(pdu.sender.clone()); } + users.push(pdu.sender.clone()); self.db.update_participants(root_id, &users) } From b525031a254f2b95e0ed20fc4c78de02e138328f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 05:20:31 +0000 Subject: [PATCH 0126/2091] Fix derive partial eq without eq Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/service/rooms/spaces/mod.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7c263042..a6cce3f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -715,7 +715,6 @@ correctness = "warn" nursery = "warn" ## some sadness -derive_partial_eq_without_eq = { level = "allow", priority = 1 } # TODO equatable_if_let = { level = "allow", priority = 1 } # TODO future_not_send = { level = "allow", priority = 1 } # TODO missing_const_for_fn = { level = "allow", priority = 1 } # TODO diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 72ebaf4a..fec3fdcc 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -233,7 +233,7 @@ impl Arena { } // Note: perhaps use some better form of token rather than just room count -#[derive(Debug, PartialEq)] +#[derive(Debug, Eq, PartialEq)] pub struct PagnationToken { pub skip: UInt, pub limit: UInt, From 89d7d4832439c06344322d0aa601a7802c3ede25 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 05:25:51 +0000 Subject: [PATCH 0127/2091] Fix equatable if let Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/api/client/membership.rs | 2 +- src/api/client/report.rs | 4 ++-- src/service/rooms/spaces/mod.rs | 17 ++++++++++------- 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a6cce3f7..f52b9c8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -715,7 +715,6 @@ correctness = "warn" nursery = "warn" ## some sadness -equatable_if_let = { level = "allow", priority = 1 } # TODO future_not_send = { level = "allow", priority = 1 } # TODO missing_const_for_fn = { level = "allow", priority = 1 } # TODO needless_collect = { level = "allow", priority = 1 } # TODO diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index b50ef2cb..f59da7b7 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -631,7 +631,7 @@ pub async fn join_room_by_id_helper( ) -> Result { let sender_user = sender_user.expect("user is authenticated"); - if let Ok(true) = services().rooms.state_cache.is_joined(sender_user, room_id) { + if matches!(services().rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { info!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { room_id: room_id.into(), diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 3e6ba880..dae6086f 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -104,14 +104,14 @@ fn is_report_valid( )); } - if let Some(true) = score.map(|s| s > int!(0) || s < int!(-100)) { + if score.map(|s| s > int!(0) || s < int!(-100)) == Some(true) { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Invalid score, must be within 0 to -100", )); }; - if let Some(true) = reason.clone().map(|s| s.len() >= 750) { + if reason.clone().map(|s| s.len() >= 750) == Some(true) { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Reason too long, should be 750 characters or fewer", diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index fec3fdcc..18b9258e 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -108,7 +108,7 @@ impl Arena { // whole space tree. // // You should only ever encounter a traversed node when going up through parents - while let Some(true) = self.traversed(current) { + while self.traversed(current) == Some(true) { if let Some(next) = self.next_sibling(current) { current = next; } else if let Some(parent) = self.parent(current) { @@ -821,12 +821,15 @@ fn is_accessable_child_recurse( SpaceRoomJoinRule::Restricted => { for room in allowed_room_ids { if let Ok((join_rule, allowed_room_ids)) = get_join_rule(room) { - if let Ok(true) = is_accessable_child_recurse( - room, - &join_rule, - identifier, - &allowed_room_ids, - recurse_num + 1, + if matches!( + is_accessable_child_recurse( + room, + &join_rule, + identifier, + &allowed_room_ids, + recurse_num + 1, + ), + Ok(true) ) { return Ok(true); } From 7688d678709c8da6e6b5466b7f658bb51fcc0db5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 06:50:44 +0000 Subject: [PATCH 0128/2091] Fix needless pass by ref mut Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/service/sending/sender.rs | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f52b9c8f..3633bc1d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -718,7 +718,6 @@ nursery = "warn" future_not_send = { level = "allow", priority = 1 } # TODO missing_const_for_fn = { level = "allow", priority = 1 } # TODO needless_collect = { level = "allow", priority = 1 } # TODO -needless_pass_by_ref_mut = { level = "allow", priority = 1 } # TODO option_if_let_else = { level = "allow", priority = 1 } # TODO redundant_pub_crate = { level = "allow", priority = 1 } # TODO significant_drop_in_scrutinee = { level = "allow", priority = 1 } # TODO diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 39b7b600..7efe3c72 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -60,12 +60,12 @@ impl Service { let mut futures: SendingFutures<'_> = FuturesUnordered::new(); let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); - self.initial_transactions(&mut futures, &mut statuses); + self.initial_transactions(&futures, &mut statuses); loop { debug_assert!(!receiver.is_closed(), "channel error"); tokio::select! { request = receiver.recv_async() => match request { - Ok(request) => self.handle_request(request, &mut futures, &mut statuses), + Ok(request) => self.handle_request(request, &futures, &mut statuses), Err(_) => return Ok(()), }, Some(response) = futures.next() => { @@ -98,7 +98,7 @@ impl Service { } fn handle_response_ok( - &self, dest: &Destination, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, + &self, dest: &Destination, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, ) { let _cork = services().globals.db.cork(); self.db @@ -125,7 +125,7 @@ impl Service { } } - fn handle_request(&self, msg: Msg, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus) { + fn handle_request(&self, msg: Msg, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus) { let iv = vec![(msg.event, msg.queue_id)]; if let Ok(Some(events)) = self.select_events(&msg.dest, iv, statuses) { if !events.is_empty() { @@ -136,7 +136,7 @@ impl Service { } } - fn initial_transactions(&self, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus) { + fn initial_transactions(&self, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus) { let keep = usize::try_from(self.startup_netburst_keep).unwrap_or(usize::MAX); let mut txns = HashMap::>::new(); for (key, dest, event) in self.db.active_requests().filter_map(Result::ok) { From a8de5d1e60e5bfdb8a1db35172dcec58df4520d8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 06:10:21 +0000 Subject: [PATCH 0129/2091] Fix futures not Send Signed-off-by: Jason Volk --- Cargo.lock | 24 +++++++++---------- Cargo.toml | 1 - src/api/client/keys.rs | 2 +- src/router/mod.rs | 6 ++--- src/service/globals/migrations.rs | 7 +++++- src/service/pusher/mod.rs | 2 +- .../rooms/event_handler/signing_keys.rs | 2 +- src/service/sending/appservice.rs | 2 +- src/service/sending/mod.rs | 4 ++-- src/service/sending/send.rs | 12 +++++----- 10 files changed, 33 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c370222..ade7465f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2749,7 +2749,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "assign", "js_int", @@ -2769,7 +2769,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "js_int", "ruma-common", @@ -2781,7 +2781,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "as_variant", "assign", @@ -2804,7 +2804,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "as_variant", "base64 0.22.1", @@ -2834,7 +2834,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "as_variant", "indexmap 2.2.6", @@ -2856,7 +2856,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "js_int", "ruma-common", @@ -2868,7 +2868,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "js_int", "thiserror", @@ -2877,7 +2877,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "js_int", "ruma-common", @@ -2887,7 +2887,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "once_cell", "proc-macro-crate", @@ -2902,7 +2902,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "js_int", "ruma-common", @@ -2914,7 +2914,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -2930,7 +2930,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#e9302a955614429ca969eb4c7f561fc87a4f6f15" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 3633bc1d..4c9ba3f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -715,7 +715,6 @@ correctness = "warn" nursery = "warn" ## some sadness -future_not_send = { level = "allow", priority = 1 } # TODO missing_const_for_fn = { level = "allow", priority = 1 } # TODO needless_collect = { level = "allow", priority = 1 } # TODO option_if_let_else = { level = "allow", priority = 1 } # TODO diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index b021bea1..0cfb0138 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -248,7 +248,7 @@ pub(crate) async fn get_key_changes_route( }) } -pub(crate) async fn get_keys_helper bool>( +pub(crate) async fn get_keys_helper bool + Send>( sender_user: Option<&UserId>, device_keys_input: &BTreeMap>, allowed_signatures: F, include_display_names: bool, ) -> Result { diff --git a/src/router/mod.rs b/src/router/mod.rs index 9bc27356..e9bae3c5 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -14,16 +14,16 @@ conduit::mod_ctor! {} conduit::mod_dtor! {} #[no_mangle] -pub extern "Rust" fn start(server: &Arc) -> Pin>>> { +pub extern "Rust" fn start(server: &Arc) -> Pin> + Send>> { Box::pin(run::start(server.clone())) } #[no_mangle] -pub extern "Rust" fn stop(server: &Arc) -> Pin>>> { +pub extern "Rust" fn stop(server: &Arc) -> Pin> + Send>> { Box::pin(run::stop(server.clone())) } #[no_mangle] -pub extern "Rust" fn run(server: &Arc) -> Pin>>> { +pub extern "Rust" fn run(server: &Arc) -> Pin> + Send>> { Box::pin(run::run(server.clone())) } diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index aa172b5c..0d0978a7 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -528,15 +528,20 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result #[cfg(feature = "sha256_media")] { + use std::path::PathBuf; if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") { warn!("sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names"); // Move old media files to new names + let mut changes = Vec::<(PathBuf, PathBuf)>::new(); for (key, _) in db.mediaid_file.iter() { let old_path = services().globals.get_media_file(&key); debug!("Old file path: {old_path:?}"); let path = services().globals.get_media_file_new(&key); debug!("New file path: {path:?}"); - // move the file to the new location + changes.push((old_path, path)); + } + // move the file to the new location + for (old_path, path) in changes { if old_path.exists() { tokio::fs::rename(&old_path, &path).await?; } diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 636ba4dd..5736ef1d 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -47,7 +47,7 @@ impl Service { #[tracing::instrument(skip(self, dest, request))] pub async fn send_request(&self, dest: &str, request: T) -> Result where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_0]; diff --git a/src/service/rooms/event_handler/signing_keys.rs b/src/service/rooms/event_handler/signing_keys.rs index 91ba0aa4..d22af9bd 100644 --- a/src/service/rooms/event_handler/signing_keys.rs +++ b/src/service/rooms/event_handler/signing_keys.rs @@ -28,7 +28,7 @@ impl super::Service { &'a self, events: E, pub_key_map: &RwLock>>, ) -> Result<()> where - E: IntoIterator>, + E: IntoIterator> + Send, { let mut server_key_ids = HashMap::new(); for event in events { diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 4df5b934..721424e1 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -12,7 +12,7 @@ use crate::{debug_error, services, utils, Error, Result}; /// registration file pub(crate) async fn send_request(registration: Registration, request: T) -> Result> where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_0]; diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 4ad40f67..ab813489 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -219,7 +219,7 @@ impl Service { #[tracing::instrument(skip(self, request), name = "request")] pub async fn send_federation_request(&self, dest: &ServerName, request: T) -> Result where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { let client = &services().globals.client.federation; send::send(client, dest, request).await @@ -233,7 +233,7 @@ impl Service { &self, registration: Registration, request: T, ) -> Result> where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { appservice::send_request(registration, request).await } diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index e8432d12..f600f196 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -18,7 +18,7 @@ use crate::{debug_error, debug_warn, services, Error, Result}; #[tracing::instrument(skip_all, name = "send")] pub async fn send(client: &Client, dest: &ServerName, req: T) -> Result where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { if !services().globals.allow_federation() { return Err(Error::bad_config("Federation is disabled.")); @@ -33,7 +33,7 @@ async fn execute( client: &Client, dest: &ServerName, actual: &ActualDest, request: Request, ) -> Result where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { let method = request.method().clone(); let url = request.url().clone(); @@ -50,7 +50,7 @@ where async fn prepare(dest: &ServerName, actual: &ActualDest, req: T) -> Result where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_5]; @@ -72,7 +72,7 @@ async fn handle_response( dest: &ServerName, actual: &ActualDest, method: &Method, url: &Url, mut response: Response, ) -> Result where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { trace!("Received response from {} for {} with {}", actual.string, url, response.url()); let status = response.status(); @@ -121,7 +121,7 @@ fn handle_error( _dest: &ServerName, actual: &ActualDest, method: &Method, url: &Url, mut e: reqwest::Error, ) -> Result where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { if e.is_timeout() || e.is_connect() { e = e.without_url(); @@ -144,7 +144,7 @@ where fn sign_request(dest: &ServerName, http_request: &mut http::Request>) where - T: OutgoingRequest + Debug, + T: OutgoingRequest + Debug + Send, { let mut req_map = serde_json::Map::new(); if !http_request.body().is_empty() { From c3c91e9d8043690f7babcfc00173440f0360cd27 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 10:13:49 +0000 Subject: [PATCH 0130/2091] Fix suboptimal flops Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/database/sqlite/mod.rs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4c9ba3f5..dbf3cf65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -721,7 +721,6 @@ option_if_let_else = { level = "allow", priority = 1 } # TODO redundant_pub_crate = { level = "allow", priority = 1 } # TODO significant_drop_in_scrutinee = { level = "allow", priority = 1 } # TODO significant_drop_tightening = { level = "allow", priority = 1 } # TODO -suboptimal_flops = { level = "allow", priority = 1 } # TODO use_self = { level = "allow", priority = 1 } # TODO useless_let_if_seq = { level = "allow", priority = 1 } # TODO diff --git a/src/database/sqlite/mod.rs b/src/database/sqlite/mod.rs index 61d78a9a..745c6aa7 100644 --- a/src/database/sqlite/mod.rs +++ b/src/database/sqlite/mod.rs @@ -109,7 +109,7 @@ impl KeyValueDatabaseEngine for Arc { clippy::cast_sign_loss )] let cache_size_per_thread = ((config.db_cache_capacity_mb * 1024.0) - / ((conduit::utils::available_parallelism() as f64 * 2.0) + 1.0)) as u32; + / (conduit::utils::available_parallelism() as f64).mul_add(2.0, 1.0)) as u32; let writer = Mutex::new(Engine::prepare_conn(&path, cache_size_per_thread)?); From eae41fc411188a7edc649506b736ef46ffed1d74 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 10:23:06 +0000 Subject: [PATCH 0131/2091] Fix use-self Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/api/ruma_wrapper/mod.rs | 2 +- src/core/config/mod.rs | 2 +- src/core/config/proxy.rs | 26 +++++++++++++------------- src/core/log.rs | 6 +++--- src/core/pducount.rs | 12 ++++++------ src/database/cork.rs | 2 +- src/database/kvdatabase.rs | 2 +- src/database/rocksdb/mod.rs | 4 ++-- src/database/sqlite/mod.rs | 4 ++-- src/main/server.rs | 4 ++-- src/service/appservice/mod.rs | 6 +++--- src/service/globals/client.rs | 4 ++-- src/service/globals/resolver.rs | 2 +- src/service/pdu.rs | 2 +- src/service/rooms/spaces/mod.rs | 6 +++--- src/service/sending/mod.rs | 6 +++--- src/service/sending/resolve.rs | 4 ++-- 18 files changed, 47 insertions(+), 48 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dbf3cf65..74c8a0c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -721,7 +721,6 @@ option_if_let_else = { level = "allow", priority = 1 } # TODO redundant_pub_crate = { level = "allow", priority = 1 } # TODO significant_drop_in_scrutinee = { level = "allow", priority = 1 } # TODO significant_drop_tightening = { level = "allow", priority = 1 } # TODO -use_self = { level = "allow", priority = 1 } # TODO useless_let_if_seq = { level = "allow", priority = 1 } # TODO ################### diff --git a/src/api/ruma_wrapper/mod.rs b/src/api/ruma_wrapper/mod.rs index fb633b91..09c07cef 100644 --- a/src/api/ruma_wrapper/mod.rs +++ b/src/api/ruma_wrapper/mod.rs @@ -55,7 +55,7 @@ where let mut request = request::from(request).await?; let mut json_body = serde_json::from_slice::(&request.body).ok(); let auth = auth::auth(&mut request, &json_body, &T::METADATA).await?; - Ok(Ruma { + Ok(Self { body: make_body::(&mut request, &mut json_body, &auth)?, origin: auth.origin, sender_user: auth.sender_user, diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index ae9902e1..dadb000b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -409,7 +409,7 @@ impl Config { .merge(Env::prefixed("CONDUWUIT_").global().split("__")) }; - let config = match raw_config.extract::() { + let config = match raw_config.extract::() { Err(e) => return Err(Error::BadConfig(format!("{e}"))), Ok(config) => config, }; diff --git a/src/core/config/proxy.rs b/src/core/config/proxy.rs index f41a92f6..a2d64fa6 100644 --- a/src/core/config/proxy.rs +++ b/src/core/config/proxy.rs @@ -42,11 +42,11 @@ pub enum ProxyConfig { impl ProxyConfig { pub fn to_proxy(&self) -> Result> { Ok(match self.clone() { - ProxyConfig::None => None, - ProxyConfig::Global { + Self::None => None, + Self::Global { url, } => Some(Proxy::all(url)?), - ProxyConfig::ByDomain(proxies) => Some(Proxy::custom(move |url| { + Self::ByDomain(proxies) => Some(Proxy::custom(move |url| { proxies.iter().find_map(|proxy| proxy.for_url(url)).cloned() // first matching // proxy })), @@ -108,18 +108,18 @@ enum WildCardedDomain { impl WildCardedDomain { fn matches(&self, domain: &str) -> bool { match self { - WildCardedDomain::WildCard => true, - WildCardedDomain::WildCarded(d) => domain.ends_with(d), - WildCardedDomain::Exact(d) => domain == d, + Self::WildCard => true, + Self::WildCarded(d) => domain.ends_with(d), + Self::Exact(d) => domain == d, } } fn more_specific_than(&self, other: &Self) -> bool { match (self, other) { - (WildCardedDomain::WildCard, WildCardedDomain::WildCard) => false, - (_, WildCardedDomain::WildCard) => true, - (WildCardedDomain::Exact(a), WildCardedDomain::WildCarded(_)) => other.matches(a), - (WildCardedDomain::WildCarded(a), WildCardedDomain::WildCarded(b)) => a != b && a.ends_with(b), + (Self::WildCard, Self::WildCard) => false, + (_, Self::WildCard) => true, + (Self::Exact(a), Self::WildCarded(_)) => other.matches(a), + (Self::WildCarded(a), Self::WildCarded(b)) => a != b && a.ends_with(b), _ => false, } } @@ -130,11 +130,11 @@ impl std::str::FromStr for WildCardedDomain { fn from_str(s: &str) -> Result { // maybe do some domain validation? Ok(if s.starts_with("*.") { - WildCardedDomain::WildCarded(s[1..].to_owned()) + Self::WildCarded(s[1..].to_owned()) } else if s == "*" { - WildCardedDomain::WildCarded(String::new()) + Self::WildCarded(String::new()) } else { - WildCardedDomain::Exact(s.to_owned()) + Self::Exact(s.to_owned()) }) } } diff --git a/src/core/log.rs b/src/core/log.rs index d31ca194..eae25eb0 100644 --- a/src/core/log.rs +++ b/src/core/log.rs @@ -21,7 +21,7 @@ pub trait ReloadHandle { } impl ReloadHandle for reload::Handle { - fn reload(&self, new_value: L) -> Result<(), reload::Error> { reload::Handle::reload(self, new_value) } + fn reload(&self, new_value: L) -> Result<(), reload::Error> { Self::reload(self, new_value) } } struct LogLevelReloadHandlesInner { @@ -37,8 +37,8 @@ pub struct LogLevelReloadHandles { impl LogLevelReloadHandles { #[must_use] - pub fn new(handles: Vec + Send + Sync>>) -> LogLevelReloadHandles { - LogLevelReloadHandles { + pub fn new(handles: Vec + Send + Sync>>) -> Self { + Self { inner: Arc::new(LogLevelReloadHandlesInner { handles, }), diff --git a/src/core/pducount.rs b/src/core/pducount.rs index 8adb4ca5..094988b6 100644 --- a/src/core/pducount.rs +++ b/src/core/pducount.rs @@ -29,8 +29,8 @@ impl PduCount { #[must_use] pub fn stringify(&self) -> String { match self { - PduCount::Backfilled(x) => format!("-{x}"), - PduCount::Normal(x) => x.to_string(), + Self::Backfilled(x) => format!("-{x}"), + Self::Normal(x) => x.to_string(), } } } @@ -42,10 +42,10 @@ impl PartialOrd for PduCount { impl Ord for PduCount { fn cmp(&self, other: &Self) -> Ordering { match (self, other) { - (PduCount::Normal(s), PduCount::Normal(o)) => s.cmp(o), - (PduCount::Backfilled(s), PduCount::Backfilled(o)) => o.cmp(s), - (PduCount::Normal(_), PduCount::Backfilled(_)) => Ordering::Greater, - (PduCount::Backfilled(_), PduCount::Normal(_)) => Ordering::Less, + (Self::Normal(s), Self::Normal(o)) => s.cmp(o), + (Self::Backfilled(s), Self::Backfilled(o)) => o.cmp(s), + (Self::Normal(_), Self::Backfilled(_)) => Ordering::Greater, + (Self::Backfilled(_), Self::Normal(_)) => Ordering::Less, } } } diff --git a/src/database/cork.rs b/src/database/cork.rs index 752260a6..dcabd707 100644 --- a/src/database/cork.rs +++ b/src/database/cork.rs @@ -11,7 +11,7 @@ pub struct Cork { impl Cork { pub fn new(db: &Arc, flush: bool, sync: bool) -> Self { db.cork().unwrap(); - Cork { + Self { db: db.clone(), flush, sync, diff --git a/src/database/kvdatabase.rs b/src/database/kvdatabase.rs index 906712dc..ce3bf30f 100644 --- a/src/database/kvdatabase.rs +++ b/src/database/kvdatabase.rs @@ -157,7 +157,7 @@ pub struct KeyValueDatabase { impl KeyValueDatabase { /// Load an existing database or create a new one. #[allow(clippy::too_many_lines)] - pub async fn load_or_create(server: &Arc) -> Result { + pub async fn load_or_create(server: &Arc) -> Result { let config = &server.config; check_db_setup(config)?; let builder = build(config)?; diff --git a/src/database/rocksdb/mod.rs b/src/database/rocksdb/mod.rs index f0c3882f..c9493cab 100644 --- a/src/database/rocksdb/mod.rs +++ b/src/database/rocksdb/mod.rs @@ -84,7 +84,7 @@ impl KeyValueDatabaseEngine for Arc { db.latest_sequence_number(), load_time.elapsed() ); - Ok(Arc::new(Engine { + Ok(Self::new(Engine { config: config.clone(), row_cache, col_cache, @@ -110,7 +110,7 @@ impl KeyValueDatabaseEngine for Arc { Ok(Arc::new(RocksDbEngineTree { name, - db: Arc::clone(self), + db: Self::clone(self), watchers: Watchers::default(), })) } diff --git a/src/database/sqlite/mod.rs b/src/database/sqlite/mod.rs index 745c6aa7..562d2a10 100644 --- a/src/database/sqlite/mod.rs +++ b/src/database/sqlite/mod.rs @@ -113,7 +113,7 @@ impl KeyValueDatabaseEngine for Arc { let writer = Mutex::new(Engine::prepare_conn(&path, cache_size_per_thread)?); - let arc = Arc::new(Engine { + let arc = Self::new(Engine { writer, read_conn_tls: ThreadLocal::new(), read_iterator_conn_tls: ThreadLocal::new(), @@ -131,7 +131,7 @@ impl KeyValueDatabaseEngine for Arc { )?; Ok(Arc::new(SqliteTable { - engine: Arc::clone(self), + engine: Self::clone(self), name: name.to_owned(), watchers: Watchers::default(), })) diff --git a/src/main/server.rs b/src/main/server.rs index cd5b1cdb..b1bccfe9 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -29,7 +29,7 @@ pub(crate) struct Server { } impl Server { - pub(crate) fn build(args: Args, runtime: Option<&runtime::Handle>) -> Result, Error> { + pub(crate) fn build(args: Args, runtime: Option<&runtime::Handle>) -> Result, Error> { let config = Config::new(args.config)?; #[cfg(feature = "sentry_telemetry")] @@ -49,7 +49,7 @@ impl Server { conduit::version::conduwuit(), ); - Ok(Arc::new(Server { + Ok(Arc::new(Self { server: Arc::new(conduit::Server::new(config, runtime.cloned(), tracing_reload_handle)), _tracing_flame_guard: tracing_flame_guard, diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 49faa6f6..1f29c415 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -75,7 +75,7 @@ impl TryFrom> for NamespaceRegex { } } - Ok(NamespaceRegex { + Ok(Self { exclusive: if exclusive.is_empty() { None } else { @@ -102,8 +102,8 @@ pub struct RegistrationInfo { impl TryFrom for RegistrationInfo { type Error = regex::Error; - fn try_from(value: Registration) -> Result { - Ok(RegistrationInfo { + fn try_from(value: Registration) -> Result { + Ok(Self { users: value.namespaces.users.clone().try_into()?, aliases: value.namespaces.aliases.clone().try_into()?, rooms: value.namespaces.rooms.clone().try_into()?, diff --git a/src/service/globals/client.rs b/src/service/globals/client.rs index 33f6d85f..5e1b129d 100644 --- a/src/service/globals/client.rs +++ b/src/service/globals/client.rs @@ -15,8 +15,8 @@ pub struct Client { } impl Client { - pub fn new(config: &Config, resolver: &Arc) -> Client { - Client { + pub fn new(config: &Config, resolver: &Arc) -> Self { + Self { default: Self::base(config) .unwrap() .dns_resolver(resolver.clone()) diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index e90201ed..39e4cfb0 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -83,7 +83,7 @@ impl Resolver { let resolver = Arc::new(TokioAsyncResolver::tokio(conf, opts)); let overrides = Arc::new(StdRwLock::new(TlsNameMap::new())); - Resolver { + Self { destinations: Arc::new(RwLock::new(WellKnownMap::new())), overrides: overrides.clone(), resolver: resolver.clone(), diff --git a/src/service/pdu.rs b/src/service/pdu.rs index f608200b..6274d635 100644 --- a/src/service/pdu.rs +++ b/src/service/pdu.rs @@ -56,7 +56,7 @@ pub struct PduEvent { impl PduEvent { #[tracing::instrument(skip(self))] - pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &PduEvent) -> crate::Result<()> { + pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> crate::Result<()> { self.unsigned = None; let mut content = serde_json::from_str(self.content.get()) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 18b9258e..6fcaccb9 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -211,7 +211,7 @@ impl Arena { fn new(root: OwnedRoomId, max_depth: usize) -> Self { let zero_depth = max_depth == 0; - Arena { + Self { nodes: vec![Node { parent: None, next_sibling: None, @@ -248,7 +248,7 @@ impl FromStr for PagnationToken { let mut values = value.split('_'); let mut pag_tok = || { - Some(PagnationToken { + Some(Self { skip: UInt::from_str(values.next()?).ok()?, limit: UInt::from_str(values.next()?).ok()?, max_depth: UInt::from_str(values.next()?).ok()?, @@ -316,7 +316,7 @@ impl From for SpaceHierarchyRoomsChunk { .. } = value.summary; - SpaceHierarchyRoomsChunk { + Self { canonical_alias, name, num_joined_members, diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index ab813489..225d37c7 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -259,19 +259,19 @@ impl Destination { #[tracing::instrument(skip(self))] pub fn get_prefix(&self) -> Vec { let mut prefix = match self { - Destination::Appservice(server) => { + Self::Appservice(server) => { let mut p = b"+".to_vec(); p.extend_from_slice(server.as_bytes()); p }, - Destination::Push(user, pushkey) => { + Self::Push(user, pushkey) => { let mut p = b"$".to_vec(); p.extend_from_slice(user.as_bytes()); p.push(0xFF); p.extend_from_slice(pushkey.as_bytes()); p }, - Destination::Normal(server) => { + Self::Normal(server) => { let mut p = Vec::new(); p.extend_from_slice(server.as_bytes()); p diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index b79f18c8..1d043d8b 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -421,8 +421,8 @@ impl FedDest { impl fmt::Display for FedDest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - FedDest::Named(host, port) => write!(f, "{host}{port}"), - FedDest::Literal(addr) => write!(f, "{addr}"), + Self::Named(host, port) => write!(f, "{host}{port}"), + Self::Literal(addr) => write!(f, "{addr}"), } } } From f52acd9cdf2c217ae45677840ba1e821bbee6cb4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 10:29:53 +0000 Subject: [PATCH 0132/2091] Fix idiomatic let if Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/api/client/account.rs | 9 ++++----- src/api/client/sync.rs | 9 ++++----- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 74c8a0c7..ce074a79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -721,7 +721,6 @@ option_if_let_else = { level = "allow", priority = 1 } # TODO redundant_pub_crate = { level = "allow", priority = 1 } # TODO significant_drop_in_scrutinee = { level = "allow", priority = 1 } # TODO significant_drop_tightening = { level = "allow", priority = 1 } # TODO -useless_let_if_seq = { level = "allow", priority = 1 } # TODO ################### pedantic = "warn" diff --git a/src/api/client/account.rs b/src/api/client/account.rs index a5ff5d82..72830f9d 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -173,8 +173,7 @@ pub(crate) async fn register_route(body: Ruma) -> Result< // UIAA let mut uiaainfo; - let skip_auth; - if services().globals.config.registration_token.is_some() { + let skip_auth = if services().globals.config.registration_token.is_some() { // Registration token required uiaainfo = UiaaInfo { flows: vec![AuthFlow { @@ -185,7 +184,7 @@ pub(crate) async fn register_route(body: Ruma) -> Result< session: None, auth_error: None, }; - skip_auth = body.appservice_info.is_some(); + body.appservice_info.is_some() } else { // No registration token necessary, but clients must still go through the flow uiaainfo = UiaaInfo { @@ -197,8 +196,8 @@ pub(crate) async fn register_route(body: Ruma) -> Result< session: None, auth_error: None, }; - skip_auth = body.appservice_info.is_some() || is_guest; - } + body.appservice_info.is_some() || is_guest + }; if !skip_auth { if let Some(auth) = &body.auth { diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 8a83cbe7..b441eb6c 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1042,8 +1042,7 @@ fn load_timeline( sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: u64, ) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> { let timeline_pdus; - let limited; - if services() + let limited = if services() .rooms .timeline .last_timeline_count(sender_user, room_id)? @@ -1073,11 +1072,11 @@ fn load_timeline( // They /sync response doesn't always return all messages, so we say the output // is limited unless there are events in non_timeline_pdus - limited = non_timeline_pdus.next().is_some(); + non_timeline_pdus.next().is_some() } else { timeline_pdus = Vec::new(); - limited = false; - } + false + }; Ok((timeline_pdus, limited)) } From f0557e3303724f862f38a8a2bad882caadce614e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Jun 2024 20:50:40 +0000 Subject: [PATCH 0133/2091] split migrations function Signed-off-by: Jason Volk --- src/service/globals/migrations.rs | 1219 +++++++++++++++-------------- 1 file changed, 641 insertions(+), 578 deletions(-) diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 0d0978a7..d8936de7 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -16,7 +16,7 @@ use ruma::{ }; use tracing::{debug, error, info, warn}; -use crate::{globals::data::Data, services, utils, Config, Error, Result}; +use crate::{services, utils, Config, Error, Result}; pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result<()> { // Matrix resource ownership is based on the server name; changing it @@ -44,513 +44,62 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result if services().users.count()? > 0 { // MIGRATIONS if services().globals.database_version()? < 1 { - for (roomserverid, _) in db.roomserverids.iter() { - let mut parts = roomserverid.split(|&b| b == 0xFF); - let room_id = parts.next().expect("split always returns one element"); - let Some(servername) = parts.next() else { - error!("Migration: Invalid roomserverid in db."); - continue; - }; - let mut serverroomid = servername.to_vec(); - serverroomid.push(0xFF); - serverroomid.extend_from_slice(room_id); - - db.serverroomids.insert(&serverroomid, &[])?; - } - - services().globals.bump_database_version(1)?; - - warn!("Migration: 0 -> 1 finished"); + db_lt_1(db, config).await?; } if services().globals.database_version()? < 2 { - // We accidentally inserted hashed versions of "" into the db instead of just "" - for (userid, password) in db.userid_password.iter() { - let empty_pass = utils::hash::password("").expect("our own password to be properly hashed"); - let password = std::str::from_utf8(&password).expect("password is valid utf-8"); - let empty_hashed_password = utils::hash::verify_password(password, &empty_pass).is_ok(); - if empty_hashed_password { - db.userid_password.insert(&userid, b"")?; - } - } - - services().globals.bump_database_version(2)?; - - warn!("Migration: 1 -> 2 finished"); + db_lt_2(db, config).await?; } if services().globals.database_version()? < 3 { - // Move media to filesystem - for (key, content) in db.mediaid_file.iter() { - if content.is_empty() { - continue; - } - - #[allow(deprecated)] - let path = services().globals.get_media_file(&key); - let mut file = fs::File::create(path)?; - file.write_all(&content)?; - db.mediaid_file.insert(&key, &[])?; - } - - services().globals.bump_database_version(3)?; - - warn!("Migration: 2 -> 3 finished"); + db_lt_3(db, config).await?; } if services().globals.database_version()? < 4 { - // Add federated users to services() as deactivated - for our_user in services().users.iter() { - let our_user = our_user?; - if services().users.is_deactivated(&our_user)? { - continue; - } - for room in services().rooms.state_cache.rooms_joined(&our_user) { - for user in services().rooms.state_cache.room_members(&room?) { - let user = user?; - if user.server_name() != config.server_name { - info!(?user, "Migration: creating user"); - services().users.create(&user, None)?; - } - } - } - } - - services().globals.bump_database_version(4)?; - - warn!("Migration: 3 -> 4 finished"); + db_lt_4(db, config).await?; } if services().globals.database_version()? < 5 { - // Upgrade user data store - for (roomuserdataid, _) in db.roomuserdataid_accountdata.iter() { - let mut parts = roomuserdataid.split(|&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let user_id = parts.next().unwrap(); - let event_type = roomuserdataid.rsplit(|&b| b == 0xFF).next().unwrap(); - - let mut key = room_id.to_vec(); - key.push(0xFF); - key.extend_from_slice(user_id); - key.push(0xFF); - key.extend_from_slice(event_type); - - db.roomusertype_roomuserdataid - .insert(&key, &roomuserdataid)?; - } - - services().globals.bump_database_version(5)?; - - warn!("Migration: 4 -> 5 finished"); + db_lt_5(db, config).await?; } if services().globals.database_version()? < 6 { - // Set room member count - for (roomid, _) in db.roomid_shortstatehash.iter() { - let string = utils::string_from_bytes(&roomid).unwrap(); - let room_id = <&RoomId>::try_from(string.as_str()).unwrap(); - services().rooms.state_cache.update_joined_count(room_id)?; - } - - services().globals.bump_database_version(6)?; - - warn!("Migration: 5 -> 6 finished"); + db_lt_6(db, config).await?; } if services().globals.database_version()? < 7 { - // Upgrade state store - let mut last_roomstates: HashMap = HashMap::new(); - let mut current_sstatehash: Option = None; - let mut current_room = None; - let mut current_state = HashSet::new(); - - let handle_state = |current_sstatehash: u64, - current_room: &RoomId, - current_state: HashSet<_>, - last_roomstates: &mut HashMap<_, _>| { - let last_roomsstatehash = last_roomstates.get(current_room); - - let states_parents = last_roomsstatehash.map_or_else( - || Ok(Vec::new()), - |&last_roomsstatehash| { - services() - .rooms - .state_compressor - .load_shortstatehash_info(last_roomsstatehash) - }, - )?; - - let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew = current_state - .difference(&parent_stateinfo.1) - .copied() - .collect::>(); - - let statediffremoved = parent_stateinfo - .1 - .difference(¤t_state) - .copied() - .collect::>(); - - (statediffnew, statediffremoved) - } else { - (current_state, HashSet::new()) - }; - - services().rooms.state_compressor.save_state_from_diff( - current_sstatehash, - Arc::new(statediffnew), - Arc::new(statediffremoved), - 2, // every state change is 2 event changes on average - states_parents, - )?; - - /* - let mut tmp = services().rooms.load_shortstatehash_info(¤t_sstatehash)?; - let state = tmp.pop().unwrap(); - println!( - "{}\t{}{:?}: {:?} + {:?} - {:?}", - current_room, - " ".repeat(tmp.len()), - utils::u64_from_bytes(¤t_sstatehash).unwrap(), - tmp.last().map(|b| utils::u64_from_bytes(&b.0).unwrap()), - state - .2 - .iter() - .map(|b| utils::u64_from_bytes(&b[size_of::()..]).unwrap()) - .collect::>(), - state - .3 - .iter() - .map(|b| utils::u64_from_bytes(&b[size_of::()..]).unwrap()) - .collect::>() - ); - */ - - Ok::<_, Error>(()) - }; - - for (k, seventid) in db.db.open_tree("stateid_shorteventid")?.iter() { - let sstatehash = utils::u64_from_bytes(&k[0..size_of::()]).expect("number of bytes is correct"); - let sstatekey = k[size_of::()..].to_vec(); - if Some(sstatehash) != current_sstatehash { - if let Some(current_sstatehash) = current_sstatehash { - handle_state( - current_sstatehash, - current_room.as_deref().unwrap(), - current_state, - &mut last_roomstates, - )?; - last_roomstates.insert(current_room.clone().unwrap(), current_sstatehash); - } - current_state = HashSet::new(); - current_sstatehash = Some(sstatehash); - - let event_id = db.shorteventid_eventid.get(&seventid).unwrap().unwrap(); - let string = utils::string_from_bytes(&event_id).unwrap(); - let event_id = <&EventId>::try_from(string.as_str()).unwrap(); - let pdu = services() - .rooms - .timeline - .get_pdu(event_id) - .unwrap() - .unwrap(); - - if Some(&pdu.room_id) != current_room.as_ref() { - current_room = Some(pdu.room_id.clone()); - } - } - - let mut val = sstatekey; - val.extend_from_slice(&seventid); - current_state.insert(val.try_into().expect("size is correct")); - } - - if let Some(current_sstatehash) = current_sstatehash { - handle_state( - current_sstatehash, - current_room.as_deref().unwrap(), - current_state, - &mut last_roomstates, - )?; - } - - services().globals.bump_database_version(7)?; - - warn!("Migration: 6 -> 7 finished"); + db_lt_7(db, config).await?; } if services().globals.database_version()? < 8 { - // Generate short room ids for all rooms - for (room_id, _) in db.roomid_shortstatehash.iter() { - let shortroomid = services().globals.next_count()?.to_be_bytes(); - db.roomid_shortroomid.insert(&room_id, &shortroomid)?; - info!("Migration: 8"); - } - // Update pduids db layout - let mut batch = db.pduid_pdu.iter().filter_map(|(key, v)| { - if !key.starts_with(b"!") { - return None; - } - let mut parts = key.splitn(2, |&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let count = parts.next().unwrap(); - - let short_room_id = db - .roomid_shortroomid - .get(room_id) - .unwrap() - .expect("shortroomid should exist"); - - let mut new_key = short_room_id; - new_key.extend_from_slice(count); - - Some((new_key, v)) - }); - - db.pduid_pdu.insert_batch(&mut batch)?; - - let mut batch2 = db.eventid_pduid.iter().filter_map(|(k, value)| { - if !value.starts_with(b"!") { - return None; - } - let mut parts = value.splitn(2, |&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let count = parts.next().unwrap(); - - let short_room_id = db - .roomid_shortroomid - .get(room_id) - .unwrap() - .expect("shortroomid should exist"); - - let mut new_value = short_room_id; - new_value.extend_from_slice(count); - - Some((k, new_value)) - }); - - db.eventid_pduid.insert_batch(&mut batch2)?; - - services().globals.bump_database_version(8)?; - - warn!("Migration: 7 -> 8 finished"); + db_lt_8(db, config).await?; } if services().globals.database_version()? < 9 { - // Update tokenids db layout - let mut iter = db - .tokenids - .iter() - .filter_map(|(key, _)| { - if !key.starts_with(b"!") { - return None; - } - let mut parts = key.splitn(4, |&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let word = parts.next().unwrap(); - let _pdu_id_room = parts.next().unwrap(); - let pdu_id_count = parts.next().unwrap(); - - let short_room_id = db - .roomid_shortroomid - .get(room_id) - .unwrap() - .expect("shortroomid should exist"); - let mut new_key = short_room_id; - new_key.extend_from_slice(word); - new_key.push(0xFF); - new_key.extend_from_slice(pdu_id_count); - Some((new_key, Vec::new())) - }) - .peekable(); - - while iter.peek().is_some() { - db.tokenids.insert_batch(&mut iter.by_ref().take(1000))?; - debug!("Inserted smaller batch"); - } - - info!("Deleting starts"); - - let batch2: Vec<_> = db - .tokenids - .iter() - .filter_map(|(key, _)| { - if key.starts_with(b"!") { - Some(key) - } else { - None - } - }) - .collect(); - - for key in batch2 { - db.tokenids.remove(&key)?; - } - - services().globals.bump_database_version(9)?; - - warn!("Migration: 8 -> 9 finished"); + db_lt_9(db, config).await?; } if services().globals.database_version()? < 10 { - // Add other direction for shortstatekeys - for (statekey, shortstatekey) in db.statekey_shortstatekey.iter() { - db.shortstatekey_statekey - .insert(&shortstatekey, &statekey)?; - } - - // Force E2EE device list updates so we can send them over federation - for user_id in services().users.iter().filter_map(Result::ok) { - services().users.mark_device_key_update(&user_id)?; - } - - services().globals.bump_database_version(10)?; - - warn!("Migration: 9 -> 10 finished"); + db_lt_10(db, config).await?; } if services().globals.database_version()? < 11 { - db.db - .open_tree("userdevicesessionid_uiaarequest")? - .clear()?; - services().globals.bump_database_version(11)?; - - warn!("Migration: 10 -> 11 finished"); + db_lt_11(db, config).await?; } if services().globals.database_version()? < 12 { - for username in services().users.list_local_users()? { - let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { - Ok(u) => u, - Err(e) => { - warn!("Invalid username {username}: {e}"); - continue; - }, - }; - - let raw_rules_list = services() - .account_data - .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) - .unwrap() - .expect("Username is invalid"); - - let mut account_data = serde_json::from_str::(raw_rules_list.get()).unwrap(); - let rules_list = &mut account_data.content.global; - - //content rule - { - let content_rule_transformation = [".m.rules.contains_user_name", ".m.rule.contains_user_name"]; - - let rule = rules_list.content.get(content_rule_transformation[0]); - if rule.is_some() { - let mut rule = rule.unwrap().clone(); - content_rule_transformation[1].clone_into(&mut rule.rule_id); - rules_list - .content - .shift_remove(content_rule_transformation[0]); - rules_list.content.insert(rule); - } - } - - //underride rules - { - let underride_rule_transformation = [ - [".m.rules.call", ".m.rule.call"], - [".m.rules.room_one_to_one", ".m.rule.room_one_to_one"], - [".m.rules.encrypted_room_one_to_one", ".m.rule.encrypted_room_one_to_one"], - [".m.rules.message", ".m.rule.message"], - [".m.rules.encrypted", ".m.rule.encrypted"], - ]; - - for transformation in underride_rule_transformation { - let rule = rules_list.underride.get(transformation[0]); - if let Some(rule) = rule { - let mut rule = rule.clone(); - transformation[1].clone_into(&mut rule.rule_id); - rules_list.underride.shift_remove(transformation[0]); - rules_list.underride.insert(rule); - } - } - } - - services().account_data.update( - None, - &user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - )?; - } - - services().globals.bump_database_version(12)?; - - warn!("Migration: 11 -> 12 finished"); + db_lt_12(db, config).await?; } // This migration can be reused as-is anytime the server-default rules are // updated. if services().globals.database_version()? < 13 { - for username in services().users.list_local_users()? { - let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { - Ok(u) => u, - Err(e) => { - warn!("Invalid username {username}: {e}"); - continue; - }, - }; - - let raw_rules_list = services() - .account_data - .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) - .unwrap() - .expect("Username is invalid"); - - let mut account_data = serde_json::from_str::(raw_rules_list.get()).unwrap(); - - let user_default_rules = Ruleset::server_default(&user); - account_data - .content - .global - .update_with_server_default(user_default_rules); - - services().account_data.update( - None, - &user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - )?; - } - - services().globals.bump_database_version(13)?; - - warn!("Migration: 12 -> 13 finished"); + db_lt_13(db, config).await?; } #[cfg(feature = "sha256_media")] - { - use std::path::PathBuf; - if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") { - warn!("sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names"); - // Move old media files to new names - let mut changes = Vec::<(PathBuf, PathBuf)>::new(); - for (key, _) in db.mediaid_file.iter() { - let old_path = services().globals.get_media_file(&key); - debug!("Old file path: {old_path:?}"); - let path = services().globals.get_media_file_new(&key); - debug!("New file path: {path:?}"); - changes.push((old_path, path)); - } - // move the file to the new location - for (old_path, path) in changes { - if old_path.exists() { - tokio::fs::rename(&old_path, &path).await?; - } - } - - services().globals.bump_database_version(14)?; - - warn!("Migration: 13 -> 14 finished"); - } + if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") { + feat_sha256_media(db, config).await?; } if db @@ -558,37 +107,7 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result .get(b"fix_bad_double_separator_in_state_cache")? .is_none() { - warn!("Fixing bad double separator in state_cache roomuserid_joined"); - let mut iter_count: usize = 0; - - let _cork = db.cork(); - - for (mut key, value) in db.roomuserid_joined.iter() { - iter_count = iter_count.saturating_add(1); - debug_info!(%iter_count); - let first_sep_index = key.iter().position(|&i| i == 0xFF).unwrap(); - - if key - .iter() - .get(first_sep_index..=first_sep_index + 1) - .copied() - .collect_vec() == vec![0xFF, 0xFF] - { - debug_warn!("Found bad key: {key:?}"); - db.roomuserid_joined.remove(&key)?; - - key.remove(first_sep_index); - debug_warn!("Fixed key: {key:?}"); - db.roomuserid_joined.insert(&key, &value)?; - } - } - - db.cleanup()?; - - warn!("Finished fixing"); - - db.global - .insert(b"fix_bad_double_separator_in_state_cache", &[])?; + fix_bad_double_separator_in_state_cache(db, config).await?; } if db @@ -596,86 +115,7 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result .get(b"retroactively_fix_bad_data_from_roomuserid_joined")? .is_none() { - warn!("Retroactively fixing bad data from broken roomuserid_joined"); - - let room_ids = services() - .rooms - .metadata - .iter_ids() - .filter_map(Result::ok) - .collect_vec(); - - let _cork = db.cork(); - - for room_id in room_ids.clone() { - debug_info!("Fixing room {room_id}"); - - let users_in_room = services() - .rooms - .state_cache - .room_members(&room_id) - .filter_map(Result::ok) - .collect_vec(); - - let joined_members = users_in_room - .iter() - .filter(|user_id| { - services() - .rooms - .state_accessor - .get_member(&room_id, user_id) - .unwrap_or(None) - .map_or(false, |membership| membership.membership == MembershipState::Join) - }) - .collect_vec(); - - let non_joined_members = users_in_room - .iter() - .filter(|user_id| { - services() - .rooms - .state_accessor - .get_member(&room_id, user_id) - .unwrap_or(None) - .map_or(false, |membership| { - membership.membership == MembershipState::Leave - || membership.membership == MembershipState::Ban - }) - }) - .collect_vec(); - - for user_id in joined_members { - debug_info!("User is joined, marking as joined"); - services() - .rooms - .state_cache - .mark_as_joined(user_id, &room_id)?; - } - - for user_id in non_joined_members { - debug_info!("User is left or banned, marking as left"); - services() - .rooms - .state_cache - .mark_as_left(user_id, &room_id)?; - } - } - - for room_id in room_ids { - debug_info!( - "Updating joined count for room {room_id} to fix servers in room after correcting membership \ - states" - ); - - services().rooms.state_cache.update_joined_count(&room_id)?; - } - - db.cleanup()?; - - warn!("Finished fixing"); - - db.global - .insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; + retroactively_fix_bad_data_from_roomuserid_joined(db, config).await?; } assert_eq!( @@ -761,3 +201,626 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result Ok(()) } + +async fn db_lt_1(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + for (roomserverid, _) in db.roomserverids.iter() { + let mut parts = roomserverid.split(|&b| b == 0xFF); + let room_id = parts.next().expect("split always returns one element"); + let Some(servername) = parts.next() else { + error!("Migration: Invalid roomserverid in db."); + continue; + }; + let mut serverroomid = servername.to_vec(); + serverroomid.push(0xFF); + serverroomid.extend_from_slice(room_id); + + db.serverroomids.insert(&serverroomid, &[])?; + } + + services().globals.bump_database_version(1)?; + info!("Migration: 0 -> 1 finished"); + Ok(()) +} + +async fn db_lt_2(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + // We accidentally inserted hashed versions of "" into the db instead of just "" + for (userid, password) in db.userid_password.iter() { + let empty_pass = utils::hash::password("").expect("our own password to be properly hashed"); + let password = std::str::from_utf8(&password).expect("password is valid utf-8"); + let empty_hashed_password = utils::hash::verify_password(password, &empty_pass).is_ok(); + if empty_hashed_password { + db.userid_password.insert(&userid, b"")?; + } + } + + services().globals.bump_database_version(2)?; + info!("Migration: 1 -> 2 finished"); + Ok(()) +} + +async fn db_lt_3(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + // Move media to filesystem + for (key, content) in db.mediaid_file.iter() { + if content.is_empty() { + continue; + } + + #[allow(deprecated)] + let path = services().globals.get_media_file(&key); + let mut file = fs::File::create(path)?; + file.write_all(&content)?; + db.mediaid_file.insert(&key, &[])?; + } + + services().globals.bump_database_version(3)?; + info!("Migration: 2 -> 3 finished"); + Ok(()) +} + +async fn db_lt_4(_db: &KeyValueDatabase, config: &Config) -> Result<()> { + // Add federated users to services() as deactivated + for our_user in services().users.iter() { + let our_user = our_user?; + if services().users.is_deactivated(&our_user)? { + continue; + } + for room in services().rooms.state_cache.rooms_joined(&our_user) { + for user in services().rooms.state_cache.room_members(&room?) { + let user = user?; + if user.server_name() != config.server_name { + info!(?user, "Migration: creating user"); + services().users.create(&user, None)?; + } + } + } + } + + services().globals.bump_database_version(4)?; + info!("Migration: 3 -> 4 finished"); + Ok(()) +} + +async fn db_lt_5(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + // Upgrade user data store + for (roomuserdataid, _) in db.roomuserdataid_accountdata.iter() { + let mut parts = roomuserdataid.split(|&b| b == 0xFF); + let room_id = parts.next().unwrap(); + let user_id = parts.next().unwrap(); + let event_type = roomuserdataid.rsplit(|&b| b == 0xFF).next().unwrap(); + + let mut key = room_id.to_vec(); + key.push(0xFF); + key.extend_from_slice(user_id); + key.push(0xFF); + key.extend_from_slice(event_type); + + db.roomusertype_roomuserdataid + .insert(&key, &roomuserdataid)?; + } + + services().globals.bump_database_version(5)?; + info!("Migration: 4 -> 5 finished"); + Ok(()) +} + +async fn db_lt_6(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + // Set room member count + for (roomid, _) in db.roomid_shortstatehash.iter() { + let string = utils::string_from_bytes(&roomid).unwrap(); + let room_id = <&RoomId>::try_from(string.as_str()).unwrap(); + services().rooms.state_cache.update_joined_count(room_id)?; + } + + services().globals.bump_database_version(6)?; + info!("Migration: 5 -> 6 finished"); + Ok(()) +} + +async fn db_lt_7(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + // Upgrade state store + let mut last_roomstates: HashMap = HashMap::new(); + let mut current_sstatehash: Option = None; + let mut current_room = None; + let mut current_state = HashSet::new(); + + let handle_state = |current_sstatehash: u64, + current_room: &RoomId, + current_state: HashSet<_>, + last_roomstates: &mut HashMap<_, _>| { + let last_roomsstatehash = last_roomstates.get(current_room); + + let states_parents = last_roomsstatehash.map_or_else( + || Ok(Vec::new()), + |&last_roomsstatehash| { + services() + .rooms + .state_compressor + .load_shortstatehash_info(last_roomsstatehash) + }, + )?; + + let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { + let statediffnew = current_state + .difference(&parent_stateinfo.1) + .copied() + .collect::>(); + + let statediffremoved = parent_stateinfo + .1 + .difference(¤t_state) + .copied() + .collect::>(); + + (statediffnew, statediffremoved) + } else { + (current_state, HashSet::new()) + }; + + services().rooms.state_compressor.save_state_from_diff( + current_sstatehash, + Arc::new(statediffnew), + Arc::new(statediffremoved), + 2, // every state change is 2 event changes on average + states_parents, + )?; + + /* + let mut tmp = services().rooms.load_shortstatehash_info(¤t_sstatehash)?; + let state = tmp.pop().unwrap(); + println!( + "{}\t{}{:?}: {:?} + {:?} - {:?}", + current_room, + " ".repeat(tmp.len()), + utils::u64_from_bytes(¤t_sstatehash).unwrap(), + tmp.last().map(|b| utils::u64_from_bytes(&b.0).unwrap()), + state + .2 + .iter() + .map(|b| utils::u64_from_bytes(&b[size_of::()..]).unwrap()) + .collect::>(), + state + .3 + .iter() + .map(|b| utils::u64_from_bytes(&b[size_of::()..]).unwrap()) + .collect::>() + ); + */ + + Ok::<_, Error>(()) + }; + + for (k, seventid) in db.db.open_tree("stateid_shorteventid")?.iter() { + let sstatehash = utils::u64_from_bytes(&k[0..size_of::()]).expect("number of bytes is correct"); + let sstatekey = k[size_of::()..].to_vec(); + if Some(sstatehash) != current_sstatehash { + if let Some(current_sstatehash) = current_sstatehash { + handle_state( + current_sstatehash, + current_room.as_deref().unwrap(), + current_state, + &mut last_roomstates, + )?; + last_roomstates.insert(current_room.clone().unwrap(), current_sstatehash); + } + current_state = HashSet::new(); + current_sstatehash = Some(sstatehash); + + let event_id = db.shorteventid_eventid.get(&seventid).unwrap().unwrap(); + let string = utils::string_from_bytes(&event_id).unwrap(); + let event_id = <&EventId>::try_from(string.as_str()).unwrap(); + let pdu = services() + .rooms + .timeline + .get_pdu(event_id) + .unwrap() + .unwrap(); + + if Some(&pdu.room_id) != current_room.as_ref() { + current_room = Some(pdu.room_id.clone()); + } + } + + let mut val = sstatekey; + val.extend_from_slice(&seventid); + current_state.insert(val.try_into().expect("size is correct")); + } + + if let Some(current_sstatehash) = current_sstatehash { + handle_state( + current_sstatehash, + current_room.as_deref().unwrap(), + current_state, + &mut last_roomstates, + )?; + } + + services().globals.bump_database_version(7)?; + info!("Migration: 6 -> 7 finished"); + Ok(()) +} + +async fn db_lt_8(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + // Generate short room ids for all rooms + for (room_id, _) in db.roomid_shortstatehash.iter() { + let shortroomid = services().globals.next_count()?.to_be_bytes(); + db.roomid_shortroomid.insert(&room_id, &shortroomid)?; + info!("Migration: 8"); + } + // Update pduids db layout + let mut batch = db.pduid_pdu.iter().filter_map(|(key, v)| { + if !key.starts_with(b"!") { + return None; + } + let mut parts = key.splitn(2, |&b| b == 0xFF); + let room_id = parts.next().unwrap(); + let count = parts.next().unwrap(); + + let short_room_id = db + .roomid_shortroomid + .get(room_id) + .unwrap() + .expect("shortroomid should exist"); + + let mut new_key = short_room_id; + new_key.extend_from_slice(count); + + Some((new_key, v)) + }); + + db.pduid_pdu.insert_batch(&mut batch)?; + + let mut batch2 = db.eventid_pduid.iter().filter_map(|(k, value)| { + if !value.starts_with(b"!") { + return None; + } + let mut parts = value.splitn(2, |&b| b == 0xFF); + let room_id = parts.next().unwrap(); + let count = parts.next().unwrap(); + + let short_room_id = db + .roomid_shortroomid + .get(room_id) + .unwrap() + .expect("shortroomid should exist"); + + let mut new_value = short_room_id; + new_value.extend_from_slice(count); + + Some((k, new_value)) + }); + + db.eventid_pduid.insert_batch(&mut batch2)?; + + services().globals.bump_database_version(8)?; + info!("Migration: 7 -> 8 finished"); + Ok(()) +} + +async fn db_lt_9(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + // Update tokenids db layout + let mut iter = db + .tokenids + .iter() + .filter_map(|(key, _)| { + if !key.starts_with(b"!") { + return None; + } + let mut parts = key.splitn(4, |&b| b == 0xFF); + let room_id = parts.next().unwrap(); + let word = parts.next().unwrap(); + let _pdu_id_room = parts.next().unwrap(); + let pdu_id_count = parts.next().unwrap(); + + let short_room_id = db + .roomid_shortroomid + .get(room_id) + .unwrap() + .expect("shortroomid should exist"); + let mut new_key = short_room_id; + new_key.extend_from_slice(word); + new_key.push(0xFF); + new_key.extend_from_slice(pdu_id_count); + Some((new_key, Vec::new())) + }) + .peekable(); + + while iter.peek().is_some() { + db.tokenids.insert_batch(&mut iter.by_ref().take(1000))?; + debug!("Inserted smaller batch"); + } + + info!("Deleting starts"); + + let batch2: Vec<_> = db + .tokenids + .iter() + .filter_map(|(key, _)| { + if key.starts_with(b"!") { + Some(key) + } else { + None + } + }) + .collect(); + + for key in batch2 { + db.tokenids.remove(&key)?; + } + + services().globals.bump_database_version(9)?; + info!("Migration: 8 -> 9 finished"); + Ok(()) +} + +async fn db_lt_10(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + // Add other direction for shortstatekeys + for (statekey, shortstatekey) in db.statekey_shortstatekey.iter() { + db.shortstatekey_statekey + .insert(&shortstatekey, &statekey)?; + } + + // Force E2EE device list updates so we can send them over federation + for user_id in services().users.iter().filter_map(Result::ok) { + services().users.mark_device_key_update(&user_id)?; + } + + services().globals.bump_database_version(10)?; + info!("Migration: 9 -> 10 finished"); + Ok(()) +} + +async fn db_lt_11(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + db.db + .open_tree("userdevicesessionid_uiaarequest")? + .clear()?; + + services().globals.bump_database_version(11)?; + info!("Migration: 10 -> 11 finished"); + Ok(()) +} + +async fn db_lt_12(_db: &KeyValueDatabase, config: &Config) -> Result<()> { + for username in services().users.list_local_users()? { + let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { + Ok(u) => u, + Err(e) => { + warn!("Invalid username {username}: {e}"); + continue; + }, + }; + + let raw_rules_list = services() + .account_data + .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) + .unwrap() + .expect("Username is invalid"); + + let mut account_data = serde_json::from_str::(raw_rules_list.get()).unwrap(); + let rules_list = &mut account_data.content.global; + + //content rule + { + let content_rule_transformation = [".m.rules.contains_user_name", ".m.rule.contains_user_name"]; + + let rule = rules_list.content.get(content_rule_transformation[0]); + if rule.is_some() { + let mut rule = rule.unwrap().clone(); + content_rule_transformation[1].clone_into(&mut rule.rule_id); + rules_list + .content + .shift_remove(content_rule_transformation[0]); + rules_list.content.insert(rule); + } + } + + //underride rules + { + let underride_rule_transformation = [ + [".m.rules.call", ".m.rule.call"], + [".m.rules.room_one_to_one", ".m.rule.room_one_to_one"], + [".m.rules.encrypted_room_one_to_one", ".m.rule.encrypted_room_one_to_one"], + [".m.rules.message", ".m.rule.message"], + [".m.rules.encrypted", ".m.rule.encrypted"], + ]; + + for transformation in underride_rule_transformation { + let rule = rules_list.underride.get(transformation[0]); + if let Some(rule) = rule { + let mut rule = rule.clone(); + transformation[1].clone_into(&mut rule.rule_id); + rules_list.underride.shift_remove(transformation[0]); + rules_list.underride.insert(rule); + } + } + } + + services().account_data.update( + None, + &user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(account_data).expect("to json value always works"), + )?; + } + + services().globals.bump_database_version(12)?; + info!("Migration: 11 -> 12 finished"); + Ok(()) +} + +async fn db_lt_13(_db: &KeyValueDatabase, config: &Config) -> Result<()> { + for username in services().users.list_local_users()? { + let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { + Ok(u) => u, + Err(e) => { + warn!("Invalid username {username}: {e}"); + continue; + }, + }; + + let raw_rules_list = services() + .account_data + .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) + .unwrap() + .expect("Username is invalid"); + + let mut account_data = serde_json::from_str::(raw_rules_list.get()).unwrap(); + + let user_default_rules = Ruleset::server_default(&user); + account_data + .content + .global + .update_with_server_default(user_default_rules); + + services().account_data.update( + None, + &user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(account_data).expect("to json value always works"), + )?; + } + + services().globals.bump_database_version(13)?; + info!("Migration: 12 -> 13 finished"); + Ok(()) +} + +#[cfg(feature = "sha256_media")] +async fn feat_sha256_media(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + use std::path::PathBuf; + warn!("sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names"); + // Move old media files to new names + let mut changes = Vec::<(PathBuf, PathBuf)>::new(); + for (key, _) in db.mediaid_file.iter() { + let old_path = services().globals.get_media_file(&key); + debug!("Old file path: {old_path:?}"); + let path = services().globals.get_media_file_new(&key); + debug!("New file path: {path:?}"); + changes.push((old_path, path)); + } + // move the file to the new location + for (old_path, path) in changes { + if old_path.exists() { + tokio::fs::rename(&old_path, &path).await?; + } + } + + services().globals.bump_database_version(14)?; + info!("Migration: 13 -> 14 finished"); + Ok(()) +} + +async fn fix_bad_double_separator_in_state_cache(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + warn!("Fixing bad double separator in state_cache roomuserid_joined"); + let mut iter_count: usize = 0; + + let _cork = db.db.cork(); + + for (mut key, value) in db.roomuserid_joined.iter() { + iter_count = iter_count.saturating_add(1); + debug_info!(%iter_count); + let first_sep_index = key.iter().position(|&i| i == 0xFF).unwrap(); + + if key + .iter() + .get(first_sep_index..=first_sep_index + 1) + .copied() + .collect_vec() + == vec![0xFF, 0xFF] + { + debug_warn!("Found bad key: {key:?}"); + db.roomuserid_joined.remove(&key)?; + + key.remove(first_sep_index); + debug_warn!("Fixed key: {key:?}"); + db.roomuserid_joined.insert(&key, &value)?; + } + } + + db.db.cleanup()?; + db.global + .insert(b"fix_bad_double_separator_in_state_cache", &[])?; + + info!("Finished fixing"); + Ok(()) +} + +async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + warn!("Retroactively fixing bad data from broken roomuserid_joined"); + + let room_ids = services() + .rooms + .metadata + .iter_ids() + .filter_map(Result::ok) + .collect_vec(); + + let _cork = db.db.cork(); + + for room_id in room_ids.clone() { + debug_info!("Fixing room {room_id}"); + + let users_in_room = services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(Result::ok) + .collect_vec(); + + let joined_members = users_in_room + .iter() + .filter(|user_id| { + services() + .rooms + .state_accessor + .get_member(&room_id, user_id) + .unwrap_or(None) + .map_or(false, |membership| membership.membership == MembershipState::Join) + }) + .collect_vec(); + + let non_joined_members = users_in_room + .iter() + .filter(|user_id| { + services() + .rooms + .state_accessor + .get_member(&room_id, user_id) + .unwrap_or(None) + .map_or(false, |membership| { + membership.membership == MembershipState::Leave || membership.membership == MembershipState::Ban + }) + }) + .collect_vec(); + + for user_id in joined_members { + debug_info!("User is joined, marking as joined"); + services() + .rooms + .state_cache + .mark_as_joined(user_id, &room_id)?; + } + + for user_id in non_joined_members { + debug_info!("User is left or banned, marking as left"); + services() + .rooms + .state_cache + .mark_as_left(user_id, &room_id)?; + } + } + + for room_id in room_ids { + debug_info!( + "Updating joined count for room {room_id} to fix servers in room after correcting membership states" + ); + + services().rooms.state_cache.update_joined_count(&room_id)?; + } + + db.db.cleanup()?; + db.global + .insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; + + info!("Finished fixing"); + Ok(()) +} From cb03654dc1a3e627ed889b4c5bbdceea64bb935f Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 14:53:26 -0400 Subject: [PATCH 0134/2091] fix broken federated room invites/joins Signed-off-by: strawberry --- src/api/client/membership.rs | 122 ++++++++++++++------------ src/service/rooms/state_cache/data.rs | 52 ++++++----- src/service/rooms/state_cache/mod.rs | 2 +- 3 files changed, 97 insertions(+), 79 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index f59da7b7..2d04be2f 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -152,21 +152,23 @@ pub(crate) async fn join_room_by_id_route( let mut servers = services() .rooms .state_cache - .servers_invite_via(&body.room_id)? - .unwrap_or( - services() - .rooms - .state_cache - .invite_state(sender_user, &body.room_id)? - .unwrap_or_default() - .iter() - .filter_map(|event| serde_json::from_str(event.json().get()).ok()) - .filter_map(|event: serde_json::Value| event.get("sender").cloned()) - .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) - .filter_map(|sender| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()) - .collect::>(), - ); + .servers_invite_via(&body.room_id) + .filter_map(Result::ok) + .collect::>(); + + servers.extend( + services() + .rooms + .state_cache + .invite_state(sender_user, &body.room_id)? + .unwrap_or_default() + .iter() + .filter_map(|event| serde_json::from_str(event.json().get()).ok()) + .filter_map(|event: serde_json::Value| event.get("sender").cloned()) + .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) + .filter_map(|sender| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), + ); if let Some(server) = body.room_id.server_name() { servers.push(server.into()); @@ -206,21 +208,22 @@ pub(crate) async fn join_room_by_id_or_alias_route( services() .rooms .state_cache - .servers_invite_via(&room_id)? - .unwrap_or( - services() - .rooms - .state_cache - .invite_state(sender_user, &room_id)? - .unwrap_or_default() - .iter() - .filter_map(|event| serde_json::from_str(event.json().get()).ok()) - .filter_map(|event: serde_json::Value| event.get("sender").cloned()) - .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) - .filter_map(|sender| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()) - .collect(), - ), + .servers_invite_via(&room_id) + .filter_map(Result::ok), + ); + + servers.extend( + services() + .rooms + .state_cache + .invite_state(sender_user, &room_id)? + .unwrap_or_default() + .iter() + .filter_map(|event| serde_json::from_str(event.json().get()).ok()) + .filter_map(|event: serde_json::Value| event.get("sender").cloned()) + .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) + .filter_map(|sender| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), ); if let Some(server) = room_id.server_name() { @@ -240,21 +243,22 @@ pub(crate) async fn join_room_by_id_or_alias_route( services() .rooms .state_cache - .servers_invite_via(&response.room_id)? - .unwrap_or( - services() - .rooms - .state_cache - .invite_state(sender_user, &response.room_id)? - .unwrap_or_default() - .iter() - .filter_map(|event| serde_json::from_str(event.json().get()).ok()) - .filter_map(|event: serde_json::Value| event.get("sender").cloned()) - .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) - .filter_map(|sender| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()) - .collect(), - ), + .servers_invite_via(&response.room_id) + .filter_map(Result::ok), + ); + + servers.extend( + services() + .rooms + .state_cache + .invite_state(sender_user, &response.room_id)? + .unwrap_or_default() + .iter() + .filter_map(|event| serde_json::from_str(event.json().get()).ok()) + .filter_map(|event: serde_json::Value| event.get("sender").cloned()) + .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) + .filter_map(|sender| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), ); (servers, response.room_id) @@ -1680,21 +1684,23 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { .invite_state(user_id, room_id)? .ok_or(Error::BadRequest(ErrorKind::BadState, "User is not invited."))?; - let servers: HashSet = services() + let mut servers: HashSet = services() .rooms .state_cache - .servers_invite_via(room_id)? - .map_or( - invite_state - .iter() - .filter_map(|event| serde_json::from_str(event.json().get()).ok()) - .filter_map(|event: serde_json::Value| event.get("sender").cloned()) - .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) - .filter_map(|sender| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()) - .collect::>(), - HashSet::from_iter, - ); + .servers_invite_via(room_id) + .filter_map(Result::ok) + .collect(); + + servers.extend( + invite_state + .iter() + .filter_map(|event| serde_json::from_str(event.json().get()).ok()) + .filter_map(|event: serde_json::Value| event.get("sender").cloned()) + .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) + .filter_map(|sender| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()) + .collect::>(), + ); debug!("servers in remote_leave_room: {servers:?}"); diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index acf390d9..d380c1ab 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -6,7 +6,6 @@ use ruma::{ serde::Raw, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; -use tracing::error; use crate::{ appservice::RegistrationInfo, @@ -94,7 +93,7 @@ pub trait Data: Send + Sync { /// Gets the servers to either accept or decline invites via for a given /// room. - fn servers_invite_via(&self, room_id: &RoomId) -> Result>>; + fn servers_invite_via<'a>(&'a self, room_id: &RoomId) -> Box> + 'a>; /// Add the given servers the list to accept or decline invites via for a /// given room. @@ -159,7 +158,10 @@ impl Data for KeyValueDatabase { self.roomuserid_leftcount.remove(&roomuser_id)?; if let Some(servers) = invite_via { - let mut prev_servers = self.servers_invite_via(room_id)?.unwrap_or(Vec::new()); + let mut prev_servers = self + .servers_invite_via(room_id) + .filter_map(Result::ok) + .collect_vec(); #[allow(clippy::redundant_clone)] // this is a necessary clone? prev_servers.append(servers.clone().as_mut()); let servers = prev_servers.iter().rev().unique().rev().collect_vec(); @@ -639,30 +641,40 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn servers_invite_via(&self, room_id: &RoomId) -> Result>> { + fn servers_invite_via<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { let key = room_id.as_bytes().to_vec(); - self.roomid_inviteviaservers - .get(&key)? - .map(|servers| { - let state = serde_json::from_slice(&servers).map_err(|e| { - error!("Invalid state in userroomid_leftstate: {e}"); - Error::bad_database("Invalid state in userroomid_leftstate.") - })?; - - Ok(state) - }) - .transpose() + Box::new( + self.roomid_inviteviaservers + .scan_prefix(key) + .map(|(_, servers)| { + ServerName::parse( + utils::string_from_bytes( + servers + .rsplit(|&b| b == 0xFF) + .next() + .expect("rsplit always returns an element"), + ) + .map_err(|_| { + Error::bad_database("Server name in roomid_inviteviaservers is invalid unicode.") + })?, + ) + .map_err(|_| Error::bad_database("Server name in roomid_inviteviaservers is invalid.")) + }), + ) } #[tracing::instrument(skip(self))] fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) -> Result<()> { - let mut prev_servers = self.servers_invite_via(room_id)?.unwrap_or(Vec::new()); - prev_servers.append(servers.to_owned().as_mut()); + let mut prev_servers = self + .servers_invite_via(room_id) + .filter_map(Result::ok) + .collect_vec(); + prev_servers.extend(servers.to_owned()); + prev_servers.sort_unstable(); + prev_servers.dedup(); - let servers = prev_servers.iter().rev().unique().rev().collect_vec(); - - let servers = servers + let servers = prev_servers .iter() .map(|server| server.as_bytes()) .collect_vec() diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 2b3f5fb7..c9d1bbd7 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -377,7 +377,7 @@ impl Service { pub fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.is_left(user_id, room_id) } #[tracing::instrument(skip(self))] - pub fn servers_invite_via(&self, room_id: &RoomId) -> Result>> { + pub fn servers_invite_via(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.servers_invite_via(room_id) } From 5bfb62e979a85cd4c3921e72741e2e012c1a97aa Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 15:38:20 -0400 Subject: [PATCH 0135/2091] Revert "update complement results" This reverts commit 9a3c52aa75900ce71084574fdd730a5d39d579a4. --- tests/test_results/complement/test_results.jsonl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 502a4530..51cd57d4 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -55,7 +55,7 @@ {"Action":"fail","Test":"TestFederationKeyUploadQuery/Can_claim_remote_one_time_key_using_POST"} {"Action":"fail","Test":"TestFederationKeyUploadQuery/Can_query_remote_device_keys_using_POST"} {"Action":"pass","Test":"TestFederationRedactSendsWithoutEvent"} -{"Action":"fail","Test":"TestFederationRejectInvite"} +{"Action":"pass","Test":"TestFederationRejectInvite"} {"Action":"fail","Test":"TestGetMissingEventsGapFilling"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility"} @@ -199,7 +199,7 @@ {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/good_connectivity"} {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/interrupted_connectivity"} {"Action":"fail","Test":"TestToDeviceMessagesOverFederation/stopped_server"} -{"Action":"fail","Test":"TestUnbanViaInvite"} +{"Action":"pass","Test":"TestUnbanViaInvite"} {"Action":"fail","Test":"TestUnknownEndpoints"} {"Action":"pass","Test":"TestUnknownEndpoints/Client-server_endpoints"} {"Action":"fail","Test":"TestUnknownEndpoints/Key_endpoints"} From 4741a76896102bce1b743bc0ed74dda4926e115b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Sun, 9 Jun 2024 12:58:28 +0000 Subject: [PATCH 0136/2091] chore(deps): update rust crate regex to v1.10.5 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ade7465f..b7d5aaa1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2631,9 +2631,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", From 130aae87589ebcc886620dba9830b8dca5c83d30 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:06:40 +0000 Subject: [PATCH 0137/2091] chore(deps): update rust crate url to v2.5.1 --- Cargo.lock | 269 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 262 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b7d5aaa1..d9b11af4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -983,6 +983,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ed25519" version = "2.2.3" @@ -1638,6 +1649,124 @@ dependencies = [ "tracing", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "idna" version = "0.4.0" @@ -1650,12 +1779,14 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", ] [[package]] @@ -1900,6 +2031,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -3532,6 +3669,12 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "string_cache" version = "0.8.7" @@ -3596,6 +3739,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tendril" version = "0.4.3" @@ -3721,6 +3875,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -4164,12 +4328,12 @@ dependencies = [ [[package]] name = "url" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.0", "percent-encoding", "serde", ] @@ -4186,6 +4350,18 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.8.0" @@ -4582,6 +4758,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xml5ever" version = "0.18.0" @@ -4599,6 +4787,30 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.34" @@ -4619,12 +4831,55 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zstd" version = "0.13.1" From d7b8af627c066b468a6cd085970b1cd0e6129ff2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 14:06:50 +0000 Subject: [PATCH 0138/2091] chore(deps): update rust crate console-subscriber to 0.3 --- Cargo.lock | 13 +++++++------ Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9b11af4..7a9da024 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -791,9 +791,9 @@ dependencies = [ [[package]] name = "console-api" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787" +checksum = "a257c22cd7e487dd4a13d413beabc512c5052f0bc048db0da6a84c3d8a6142fd" dependencies = [ "futures-core", "prost", @@ -804,9 +804,9 @@ dependencies = [ [[package]] name = "console-subscriber" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e" +checksum = "31c4cc54bae66f7d9188996404abdf7fdfa23034ef8e43478c8810828abad758" dependencies = [ "console-api", "crossbeam-channel", @@ -814,6 +814,7 @@ dependencies = [ "futures-task", "hdrhistogram", "humantime", + "prost", "prost-types", "serde", "serde_json", @@ -4055,9 +4056,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" dependencies = [ "async-stream", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index ce074a79..67166ede 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -393,7 +393,7 @@ version = "0.3.1" default-features = false [workspace.dependencies.console-subscriber] -version = "0.2" +version = "0.3" [workspace.dependencies.nix] version = "0.29.0" From 1b2c8236fbdd66684cddc5be02c5018f50e25e36 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 15:40:21 +0000 Subject: [PATCH 0139/2091] chore(deps): update rust crate clap to v4.5.7 --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a9da024..c47d5771 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -538,9 +538,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.6" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9689a29b593160de5bc4aacab7b5d54fb52231de70122626c178e6a368994c7" +checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" dependencies = [ "clap_builder", "clap_derive", @@ -548,9 +548,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.6" +version = "4.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5387378c84f6faa26890ebf9f0a92989f8873d4d380467bcd0d8d8620424df" +checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" dependencies = [ "anstyle", "clap_lex", From b9d38fd3ba1e686f9c52b178f47ff5ee4f108e15 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 19:49:19 +0000 Subject: [PATCH 0140/2091] chore(deps): update rust crate http-body-util to v0.1.2 --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c47d5771..17b10c94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1527,12 +1527,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", - "futures-core", + "futures-util", "http 1.1.0", "http-body 1.0.0", "pin-project-lite", From ac020783951630ef9217c9dfa1fc3e67197f1e5f Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 16:43:04 -0400 Subject: [PATCH 0141/2091] bump cargo.lock, add tracing feature to axum-extra Signed-off-by: strawberry --- Cargo.lock | 61 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 2 +- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 17b10c94..7605b94e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -178,7 +178,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "itoa", "matchit", "memchr", @@ -296,6 +296,7 @@ dependencies = [ "tower", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -486,9 +487,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f" +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" dependencies = [ "jobserver", "libc", @@ -570,9 +571,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" [[package]] name = "color_quant" @@ -1540,9 +1541,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "9f3935c160d00ac752e09787e6e6bfc26494c2183cc922f1bc678a60d4733bc2" [[package]] name = "httpdate" @@ -1558,9 +1559,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", @@ -1624,7 +1625,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.28", + "hyper 0.14.29", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -2775,8 +2776,8 @@ checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.6", - "regex-syntax 0.8.3", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", ] [[package]] @@ -2790,13 +2791,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.3", + "regex-syntax 0.8.4", ] [[package]] @@ -2807,9 +2808,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" @@ -4011,14 +4012,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.13" +version = "0.8.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e43f8cc456c9704c851ae29c67e17ef65d2c30017c17a9765b89c382dc8bba" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.13", + "toml_edit 0.22.14", ] [[package]] @@ -4043,15 +4044,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.13" +version = "0.22.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c127785850e8c20836d49732ae6abfa47616e60bf9d9f57c43c250361a9db96c" +checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" dependencies = [ "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.9", + "winnow 0.6.13", ] [[package]] @@ -4068,7 +4069,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.28", + "hyper 0.14.29", "hyper-timeout", "percent-encoding", "pin-project", @@ -4516,9 +4517,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009" +checksum = "3c452ad30530b54a4d8e71952716a212b08efd0f3562baa66c29a618b07da7c3" dependencies = [ "rustls-pki-types", ] @@ -4732,9 +4733,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.9" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86c949fede1d13936a99f14fafd3e76fd642b556dd2ce96287fbe2e0151bfac6" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" dependencies = [ "memchr", ] @@ -4773,9 +4774,9 @@ checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] name = "xml5ever" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c376f76ed09df711203e20c3ef5ce556f0166fa03d39590016c0fd625437fad" +checksum = "9bbb26405d8e919bc1547a5aa9abc95cbfa438f04844f5fdd9dc7596b748bf69" dependencies = [ "log", "mac", diff --git a/Cargo.toml b/Cargo.toml index 67166ede..1fd5af7a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,7 @@ features = [ [workspace.dependencies.axum-extra] version = "0.9.3" default-features = false -features = ["typed-header"] +features = ["typed-header", "tracing"] [workspace.dependencies.axum-server] version = "0.6.0" From 6b843ec4dd1b46c07c00512627a1deec7d6c74fb Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 16:49:17 -0400 Subject: [PATCH 0142/2091] switch to patch/fork of axum-server to fix shutdown hangs Signed-off-by: strawberry --- Cargo.lock | 3 +-- Cargo.toml | 5 +++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7605b94e..bf8c74a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,8 +302,7 @@ dependencies = [ [[package]] name = "axum-server" version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad46c3ec4e12f4a4b6835e173ba21c25e484c9d02b49770bf006ce5367c036" +source = "git+https://github.com/lexe-app/axum-server?branch=phlip9/fix-graceful-shutdown#8e3368d899079818934e61cc9c839abcbbcada8a" dependencies = [ "arc-swap", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 1fd5af7a..58e84a22 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -427,6 +427,11 @@ branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" git = "https://github.com/girlbossceo/tracing" branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" +# fixes hyper graceful shutdowns [https://github.com/programatik29/axum-server/issues/114] +[patch.crates-io.axum-server] +git = "https://github.com/lexe-app/axum-server" +branch = "phlip9/fix-graceful-shutdown" + # # Our crates # From adf0bfd8942eda3c14ee9c97888ae66af063690f Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 16:57:44 -0400 Subject: [PATCH 0143/2091] bump ruma Signed-off-by: strawberry --- Cargo.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf8c74a9..5e238598 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2887,7 +2887,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "assign", "js_int", @@ -2907,7 +2907,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "ruma-common", @@ -2919,7 +2919,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "as_variant", "assign", @@ -2942,7 +2942,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "as_variant", "base64 0.22.1", @@ -2972,7 +2972,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "as_variant", "indexmap 2.2.6", @@ -2994,7 +2994,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "ruma-common", @@ -3006,7 +3006,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "thiserror", @@ -3015,7 +3015,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "ruma-common", @@ -3025,7 +3025,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "once_cell", "proc-macro-crate", @@ -3040,7 +3040,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "ruma-common", @@ -3052,7 +3052,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3068,7 +3068,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#f8f6db89d8efa2b4fddbfcaf502759b80cd2d6cb" +source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "itertools 0.12.1", "js_int", From 8210e8c42ec561eacf2c616a967ceb0953494147 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 17:20:59 -0400 Subject: [PATCH 0144/2091] slightly adjust sliding sync code for ruma bump Signed-off-by: strawberry --- src/api/client/sync.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index b441eb6c..5d881b45 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -611,7 +611,7 @@ async fn load_joined_room( .unwrap_or(0); // Recalculate heroes (first 5 members) - let mut heroes = Vec::new(); + let mut heroes: Vec = Vec::with_capacity(5); if joined_member_count.saturating_add(invited_member_count) <= 5 { // Go through all PDUs and for each member event, check if the user is still @@ -636,7 +636,7 @@ async fn load_joined_room( && (services().rooms.state_cache.is_joined(&user_id, room_id)? || services().rooms.state_cache.is_invited(&user_id, room_id)?) { - Ok::<_, Error>(Some(state_key.clone())) + Ok::<_, Error>(Some(user_id)) } else { Ok(None) } @@ -644,12 +644,11 @@ async fn load_joined_room( Ok(None) } }) - // Filter out buggy users .filter_map(Result::ok) // Filter for possible heroes .flatten() { - if heroes.contains(&hero) || hero == sender_user.as_str() { + if heroes.contains(&hero) || hero == sender_user { continue; } @@ -1584,6 +1583,7 @@ pub(crate) async fn sync_events_v4_route( ), num_live: None, // Count events in timeline greater than global sync counter timestamp: None, + heroes: None, }, ); } From 1287a86c05aadf57c847e94fddf71b011dd6c1bc Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 18:35:29 -0400 Subject: [PATCH 0145/2091] client-api: export `validate_and_add_event_id` Signed-off-by: strawberry --- src/api/client/membership.rs | 2 +- src/api/client/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 2d04be2f..ad63d896 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1314,7 +1314,7 @@ async fn make_join_request( make_join_response_and_server } -async fn validate_and_add_event_id( +pub async fn validate_and_add_event_id( pdu: &RawJsonValue, room_version: &RoomVersionId, pub_key_map: &RwLock>>, ) -> Result<(OwnedEventId, CanonicalJsonObject)> { let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index ced2bde5..6a6f16a7 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -47,7 +47,7 @@ pub(super) use filter::*; pub(super) use keys::*; pub(super) use media::*; pub(super) use membership::*; -pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room}; +pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room, validate_and_add_event_id}; pub(super) use message::*; pub(super) use presence::*; pub(super) use profile::*; From 03ba9bde29549190ea26c85a31335b2266cb08ee Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 18:35:48 -0400 Subject: [PATCH 0146/2091] admin cmd to force download and use a server's room state Signed-off-by: strawberry --- src/admin/debug/debug_commands.rs | 153 +++++++++++++++++++++++++++++- src/admin/debug/mod.rs | 27 +++++- 2 files changed, 176 insertions(+), 4 deletions(-) diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 4522b678..56563748 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -1,9 +1,15 @@ -use std::{collections::BTreeMap, sync::Arc, time::Instant}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, + time::Instant, +}; +use api::client::validate_and_add_event_id; use conduit::{utils::HtmlEscape, Error, Result}; use ruma::{ - api::client::error::ErrorKind, events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, - RoomId, RoomVersionId, ServerName, + api::{client::error::ErrorKind, federation::event::get_room_state}, + events::room::message::RoomMessageEventContent, + CanonicalJsonObject, EventId, RoomId, RoomVersionId, ServerName, }; use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve::resolve_actual_dest, services, PduEvent}; use tokio::sync::RwLock; @@ -472,6 +478,147 @@ pub(crate) async fn latest_pdu_in_room(_body: Vec<&str>, room_id: Box) - Ok(RoomMessageEventContent::text_plain(format!("{latest_pdu:?}"))) } +#[tracing::instrument(skip(_body))] +pub(crate) async fn force_set_room_state_from_server( + _body: Vec<&str>, server_name: Box, room_id: Box, +) -> Result { + if !services() + .rooms + .state_cache + .server_in_room(&services().globals.config.server_name, &room_id)? + { + return Ok(RoomMessageEventContent::text_plain( + "We are not participating in the room / we don't know about the room ID.", + )); + } + + let first_pdu = services() + .rooms + .timeline + .latest_pdu_in_room(&room_id)? + .ok_or_else(|| Error::bad_database("Failed to find the latest PDU in database"))?; + + let room_version = services().rooms.state.get_room_version(&room_id)?; + + let mut state: HashMap> = HashMap::new(); + let pub_key_map = RwLock::new(BTreeMap::new()); + + let remote_state_response = services() + .sending + .send_federation_request( + &server_name, + get_room_state::v1::Request { + room_id: room_id.clone().into(), + event_id: first_pdu.event_id.clone().into(), + }, + ) + .await?; + + let mut events = Vec::with_capacity(remote_state_response.pdus.len()); + + for pdu in remote_state_response.pdus.clone() { + events.push(match parse_incoming_pdu(&pdu) { + Ok(t) => t, + Err(e) => { + warn!("Could not parse PDU, ignoring: {e}"); + continue; + }, + }); + } + + info!("Fetching required signing keys for all the state events we got"); + services() + .rooms + .event_handler + .fetch_required_signing_keys(events.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) + .await?; + + info!("Going through room_state response PDUs"); + for result in remote_state_response + .pdus + .iter() + .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map)) + { + let Ok((event_id, value)) = result.await else { + continue; + }; + + let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| { + warn!("Invalid PDU in fetching remote room state PDUs response: {} {:?}", e, value); + Error::BadServerResponse("Invalid PDU in send_join response.") + })?; + + services() + .rooms + .outlier + .add_pdu_outlier(&event_id, &value)?; + if let Some(state_key) = &pdu.state_key { + let shortstatekey = services() + .rooms + .short + .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key)?; + state.insert(shortstatekey, pdu.event_id.clone()); + } + } + + info!("Going through auth_chain response"); + for result in remote_state_response + .auth_chain + .iter() + .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map)) + { + let Ok((event_id, value)) = result.await else { + continue; + }; + + services() + .rooms + .outlier + .add_pdu_outlier(&event_id, &value)?; + } + + let new_room_state = services() + .rooms + .event_handler + .resolve_state(room_id.clone().as_ref(), &room_version, state) + .await?; + + info!("Forcing new room state"); + let (short_state_hash, new, removed) = services() + .rooms + .state_compressor + .save_state(room_id.clone().as_ref(), new_room_state)?; + + let mutex_state = Arc::clone( + services() + .globals + .roomid_mutex_state + .write() + .await + .entry(room_id.clone().into()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + services() + .rooms + .state + .force_state(room_id.clone().as_ref(), short_state_hash, new, removed, &state_lock) + .await?; + + info!( + "Updating joined counts for room just in case (e.g. we may have found a difference in the room's \ + m.room.member state" + ); + services().rooms.state_cache.update_joined_count(&room_id)?; + + drop(state_lock); + + Ok(RoomMessageEventContent::text_plain( + "Successfully forced the room state from the requested remote server.", + )) +} + pub(crate) async fn resolve_true_destination( _body: Vec<&str>, server_name: Box, no_cache: bool, ) -> Result { diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 614a4ae5..afcc49b0 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use debug_commands::{first_pdu_in_room, latest_pdu_in_room}; +use debug_commands::{first_pdu_in_room, force_set_room_state_from_server, latest_pdu_in_room}; use ruma::{events::room::message::RoomMessageEventContent, EventId, RoomId, ServerName}; use self::debug_commands::{ @@ -123,6 +123,27 @@ pub(crate) enum DebugCommand { room_id: Box, }, + /// - Forcefully replaces the room state of our local copy of the specified + /// room, with the copy (auth chain and room state events) the specified + /// remote server says. + /// + /// A common desire for room deletion is to simply "reset" our copy of the + /// room. While this admin command is not a replacement for that, if you + /// know you have split/broken room state and you know another server in the + /// room that has the best/working room state, this command can let you use + /// their room state. Such example is your server saying users are in a + /// room, but other servers are saying they're not in the room in question. + /// + /// This command will get the latest PDU in the room we know about, and + /// request the room state at that point in time via + /// `/_matrix/federation/v1/state/{roomId}`. + ForceSetRoomStateFromServer { + /// The impacted room ID + room_id: Box, + /// The server we will use to query the room state for + server_name: Box, + }, + /// - Runs a server name through conduwuit's true destination resolution /// process /// @@ -174,6 +195,10 @@ pub(crate) async fn process(command: DebugCommand, body: Vec<&str>) -> Result get_remote_pdu_list(body, server, force).await?, + DebugCommand::ForceSetRoomStateFromServer { + room_id, + server_name, + } => force_set_room_state_from_server(body, server_name, room_id).await?, DebugCommand::ResolveTrueDestination { server_name, no_cache, From d2063013b492e1288f53cb76bae8c7bc1c5285cf Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Jun 2024 19:47:07 -0400 Subject: [PATCH 0147/2091] adminroom: add various state_cache db accessors Signed-off-by: strawberry --- src/admin/query/mod.rs | 74 +++++++++ src/admin/query/room_state_cache.rs | 233 ++++++++++++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 src/admin/query/room_state_cache.rs diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index bebbb771..7fe6a2a2 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -3,10 +3,12 @@ pub(crate) mod appservice; pub(crate) mod globals; pub(crate) mod presence; pub(crate) mod room_alias; +pub(crate) mod room_state_cache; pub(crate) mod sending; pub(crate) mod users; use clap::Subcommand; +use room_state_cache::room_state_cache; use ruma::{ events::{room::message::RoomMessageEventContent, RoomAccountDataEventType}, RoomAliasId, RoomId, ServerName, UserId, @@ -38,6 +40,10 @@ pub(crate) enum QueryCommand { #[command(subcommand)] RoomAlias(RoomAlias), + /// - rooms/state_cache iterators and getters + #[command(subcommand)] + RoomStateCache(RoomStateCache), + /// - globals.rs iterators and getters #[command(subcommand)] Globals(Globals), @@ -127,6 +133,73 @@ pub(crate) enum RoomAlias { AllLocalAliases, } +#[cfg_attr(test, derive(Debug))] +#[derive(Subcommand)] +pub(crate) enum RoomStateCache { + RoomServers { + room_id: Box, + }, + + ServerRooms { + server: Box, + }, + + RoomMembers { + room_id: Box, + }, + + LocalUsersInRoom { + room_id: Box, + }, + + ActiveLocalUsersInRoom { + room_id: Box, + }, + + RoomJoinedCount { + room_id: Box, + }, + + RoomInvitedCount { + room_id: Box, + }, + + RoomUserOnceJoined { + room_id: Box, + }, + + RoomMembersInvited { + room_id: Box, + }, + + GetInviteCount { + room_id: Box, + user_id: Box, + }, + + GetLeftCount { + room_id: Box, + user_id: Box, + }, + + RoomsJoined { + user_id: Box, + }, + + RoomsLeft { + user_id: Box, + }, + + RoomsInvited { + user_id: Box, + }, + + InviteState { + user_id: Box, + room_id: Box, + }, +} + #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] /// All the getters and iterators from src/database/key_value/globals.rs @@ -216,6 +289,7 @@ pub(crate) async fn process(command: QueryCommand, _body: Vec<&str>) -> Result appservice(command).await?, QueryCommand::Presence(command) => presence(command).await?, QueryCommand::RoomAlias(command) => room_alias(command).await?, + QueryCommand::RoomStateCache(command) => room_state_cache(command).await?, QueryCommand::Globals(command) => globals(command).await?, QueryCommand::Sending(command) => sending(command).await?, QueryCommand::Users(command) => users(command).await?, diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs new file mode 100644 index 00000000..5b8ff8bb --- /dev/null +++ b/src/admin/query/room_state_cache.rs @@ -0,0 +1,233 @@ +use ruma::events::room::message::RoomMessageEventContent; + +use super::RoomStateCache; +use crate::{services, Result}; + +pub(crate) async fn room_state_cache(subcommand: RoomStateCache) -> Result { + match subcommand { + RoomStateCache::RoomServers { + room_id, + } => { + let timer = tokio::time::Instant::now(); + let results: Result> = services() + .rooms + .state_cache + .room_servers(&room_id) + .collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::ServerRooms { + server, + } => { + let timer = tokio::time::Instant::now(); + let results: Result> = services().rooms.state_cache.server_rooms(&server).collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::RoomMembers { + room_id, + } => { + let timer = tokio::time::Instant::now(); + let results: Result> = services() + .rooms + .state_cache + .room_members(&room_id) + .collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::LocalUsersInRoom { + room_id, + } => { + let timer = tokio::time::Instant::now(); + let results: Vec<_> = services() + .rooms + .state_cache + .local_users_in_room(&room_id) + .collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::ActiveLocalUsersInRoom { + room_id, + } => { + let timer = tokio::time::Instant::now(); + let results: Vec<_> = services() + .rooms + .state_cache + .active_local_users_in_room(&room_id) + .collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::RoomJoinedCount { + room_id, + } => { + let timer = tokio::time::Instant::now(); + let results = services().rooms.state_cache.room_joined_count(&room_id); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::RoomInvitedCount { + room_id, + } => { + let timer = tokio::time::Instant::now(); + let results = services().rooms.state_cache.room_invited_count(&room_id); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::RoomUserOnceJoined { + room_id, + } => { + let timer = tokio::time::Instant::now(); + let results: Result> = services() + .rooms + .state_cache + .room_useroncejoined(&room_id) + .collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::RoomMembersInvited { + room_id, + } => { + let timer = tokio::time::Instant::now(); + let results: Result> = services() + .rooms + .state_cache + .room_members_invited(&room_id) + .collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::GetInviteCount { + room_id, + user_id, + } => { + let timer = tokio::time::Instant::now(); + let results = services() + .rooms + .state_cache + .get_invite_count(&room_id, &user_id); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::GetLeftCount { + room_id, + user_id, + } => { + let timer = tokio::time::Instant::now(); + let results = services() + .rooms + .state_cache + .get_left_count(&room_id, &user_id); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::RoomsJoined { + user_id, + } => { + let timer = tokio::time::Instant::now(); + let results: Result> = services() + .rooms + .state_cache + .rooms_joined(&user_id) + .collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::RoomsInvited { + user_id, + } => { + let timer = tokio::time::Instant::now(); + let results: Result> = services() + .rooms + .state_cache + .rooms_invited(&user_id) + .collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::RoomsLeft { + user_id, + } => { + let timer = tokio::time::Instant::now(); + let results: Result> = services().rooms.state_cache.rooms_left(&user_id).collect(); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + RoomStateCache::InviteState { + user_id, + room_id, + } => { + let timer = tokio::time::Instant::now(); + let results = services() + .rooms + .state_cache + .invite_state(&user_id, &room_id); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), + )) + }, + } +} From 016270b33b9cdd852447d095e42ee247a234ae9d Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Jun 2024 13:49:39 -0400 Subject: [PATCH 0148/2091] adminroom: add server_in_room state cache accessor Signed-off-by: strawberry --- src/admin/query/mod.rs | 5 +++++ src/admin/query/room_state_cache.rs | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index 7fe6a2a2..2e5b0bb8 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -136,6 +136,11 @@ pub(crate) enum RoomAlias { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] pub(crate) enum RoomStateCache { + ServerInRoom { + server: Box, + room_id: Box, + }, + RoomServers { room_id: Box, }, diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index 5b8ff8bb..c062497e 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -5,6 +5,22 @@ use crate::{services, Result}; pub(crate) async fn room_state_cache(subcommand: RoomStateCache) -> Result { match subcommand { + RoomStateCache::ServerInRoom { + server, + room_id, + } => { + let timer = tokio::time::Instant::now(); + let result = services() + .rooms + .state_cache + .server_in_room(&server, &room_id); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::text_html( + format!("Query completed in {query_time:?}:\n\n```\n{result:?}```"), + format!("

    Query completed in {query_time:?}:

    \n
    {result:?}\n
    "), + )) + }, RoomStateCache::RoomServers { room_id, } => { From 4b4c0952a24e84f5416c9c3fed14f37c5a446b3e Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Jun 2024 20:45:17 -0400 Subject: [PATCH 0149/2091] refactor `guest_can_join` into 1 single function Signed-off-by: strawberry --- src/api/client/directory.rs | 8 +------- src/service/rooms/spaces/mod.rs | 16 +--------------- src/service/rooms/state_accessor/mod.rs | 15 ++++++++++++--- 3 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 0e5b8afa..aee4341f 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -13,7 +13,6 @@ use ruma::{ avatar::RoomAvatarEventContent, canonical_alias::RoomCanonicalAliasEventContent, create::RoomCreateEventContent, - guest_access::{GuestAccess, RoomGuestAccessEventContent}, join_rules::{JoinRule, RoomJoinRulesEventContent}, topic::RoomTopicEventContent, }, @@ -264,12 +263,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( guest_can_join: services() .rooms .state_accessor - .room_state_get(&room_id, &StateEventType::RoomGuestAccess, "")? - .map_or(Ok(false), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomGuestAccessEventContent| c.guest_access == GuestAccess::CanJoin) - .map_err(|_| Error::bad_database("Invalid room guest access event in database.")) - })?, + .guest_can_join(&room_id)?, avatar_url: services() .rooms .state_accessor diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 6fcaccb9..9a5dc9d0 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -17,7 +17,6 @@ use ruma::{ avatar::RoomAvatarEventContent, canonical_alias::RoomCanonicalAliasEventContent, create::RoomCreateEventContent, - guest_access::{GuestAccess, RoomGuestAccessEventContent}, join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent, RoomMembership}, topic::RoomTopicEventContent, }, @@ -591,7 +590,7 @@ impl Service { }) .unwrap_or(None), world_readable: services().rooms.state_accessor.is_world_readable(room_id)?, - guest_can_join: guest_can_join(room_id)?, + guest_can_join: services().rooms.state_accessor.guest_can_join(room_id)?, avatar_url: services() .rooms .state_accessor @@ -847,19 +846,6 @@ fn is_accessable_child_recurse( } } -/// Checks if guests are able to join a given room -fn guest_can_join(room_id: &RoomId) -> Result { - services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomGuestAccess, "")? - .map_or(Ok(false), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomGuestAccessEventContent| c.guest_access == GuestAccess::CanJoin) - .map_err(|_| Error::bad_database("Invalid room guest access event in database.")) - }) -} - /// Returns the join rule for a given room fn get_join_rule(current_room: &RoomId) -> Result<(SpaceRoomJoinRule, Vec), Error> { Ok(services() diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 0681c298..32ed66ee 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -10,6 +10,7 @@ use ruma::{ events::{ room::{ avatar::RoomAvatarEventContent, + guest_access::{GuestAccess, RoomGuestAccessEventContent}, history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, member::{MembershipState, RoomMemberEventContent}, name::RoomNameEventContent, @@ -304,8 +305,7 @@ impl Service { /// Checks if guests are able to view room content without joining pub fn is_world_readable(&self, room_id: &RoomId) -> Result { - Ok(self - .room_state_get(room_id, &StateEventType::RoomHistoryVisibility, "")? + self.room_state_get(room_id, &StateEventType::RoomHistoryVisibility, "")? .map_or(Ok(false), |s| { serde_json::from_str(s.content.get()) .map(|c: RoomHistoryVisibilityEventContent| { @@ -319,6 +319,15 @@ impl Service { Error::bad_database("Invalid room history visibility event in database.") }) }) - .unwrap_or(false)) + } + + /// Checks if guests are able to join a given room + pub fn guest_can_join(&self, room_id: &RoomId) -> Result { + self.room_state_get(room_id, &StateEventType::RoomGuestAccess, "")? + .map_or(Ok(false), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomGuestAccessEventContent| c.guest_access == GuestAccess::CanJoin) + .map_err(|_| Error::bad_database("Invalid room guest access event in database.")) + }) } } From 88d038ffecc1d341db622ecfd2f66507e2bf736c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Jun 2024 20:52:41 -0400 Subject: [PATCH 0150/2091] refactor getting room's canonical alias into 1 function Signed-off-by: strawberry --- src/api/client/directory.rs | 8 +------- src/service/rooms/spaces/mod.rs | 8 +------- src/service/rooms/state_accessor/mod.rs | 13 ++++++++++++- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index aee4341f..9ce4ffd7 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -11,7 +11,6 @@ use ruma::{ events::{ room::{ avatar::RoomAvatarEventContent, - canonical_alias::RoomCanonicalAliasEventContent, create::RoomCreateEventContent, join_rules::{JoinRule, RoomJoinRulesEventContent}, topic::RoomTopicEventContent, @@ -229,12 +228,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( canonical_alias: services() .rooms .state_accessor - .room_state_get(&room_id, &StateEventType::RoomCanonicalAlias, "")? - .map_or(Ok(None), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomCanonicalAliasEventContent| c.alias) - .map_err(|_| Error::bad_database("Invalid canonical alias event in database.")) - })?, + .get_canonical_alias(&room_id)?, name: services().rooms.state_accessor.get_name(&room_id)?, num_joined_members: services() .rooms diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 9a5dc9d0..c96ab7a0 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -15,7 +15,6 @@ use ruma::{ events::{ room::{ avatar::RoomAvatarEventContent, - canonical_alias::RoomCanonicalAliasEventContent, create::RoomCreateEventContent, join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent, RoomMembership}, topic::RoomTopicEventContent, @@ -558,12 +557,7 @@ impl Service { canonical_alias: services() .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomCanonicalAlias, "")? - .map_or(Ok(None), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomCanonicalAliasEventContent| c.alias) - .map_err(|_| Error::bad_database("Invalid canonical alias event in database.")) - })?, + .get_canonical_alias(room_id)?, name: services().rooms.state_accessor.get_name(room_id)?, num_joined_members: services() .rooms diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 32ed66ee..62c8c224 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -10,6 +10,7 @@ use ruma::{ events::{ room::{ avatar::RoomAvatarEventContent, + canonical_alias::RoomCanonicalAliasEventContent, guest_access::{GuestAccess, RoomGuestAccessEventContent}, history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, member::{MembershipState, RoomMemberEventContent}, @@ -17,7 +18,7 @@ use ruma::{ }, StateEventType, }, - EventId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, + EventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; use serde_json::value::to_raw_value; use tokio::sync::MutexGuard; @@ -330,4 +331,14 @@ impl Service { .map_err(|_| Error::bad_database("Invalid room guest access event in database.")) }) } + + /// Gets the primary alias from canonical alias event + pub fn get_canonical_alias(&self, room_id: &RoomId) -> Result, Error> { + self.room_state_get(room_id, &StateEventType::RoomCanonicalAlias, "")? + .map_or(Ok(None), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomCanonicalAliasEventContent| c.alias) + .map_err(|_| Error::bad_database("Invalid canonical alias event in database.")) + }) + } } From fcdf1463ef4e3c8da70b88d6e8bf36bc8e6e10cf Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Jun 2024 20:56:51 -0400 Subject: [PATCH 0151/2091] refactor `get_room_topic` into 1 single function Signed-off-by: strawberry --- src/api/client/directory.rs | 11 +---------- src/service/rooms/spaces/mod.rs | 11 +---------- src/service/rooms/state_accessor/mod.rs | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 9ce4ffd7..ef475a68 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -13,7 +13,6 @@ use ruma::{ avatar::RoomAvatarEventContent, create::RoomCreateEventContent, join_rules::{JoinRule, RoomJoinRulesEventContent}, - topic::RoomTopicEventContent, }, StateEventType, }, @@ -243,15 +242,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( topic: services() .rooms .state_accessor - .room_state_get(&room_id, &StateEventType::RoomTopic, "")? - .map_or(Ok(None), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomTopicEventContent| Some(c.topic)) - .map_err(|e| { - error!("Invalid room topic event in database for room {room_id}: {e}"); - Error::bad_database("Invalid room topic event in database.") - }) - }) + .get_room_topic(&room_id) .unwrap_or(None), world_readable: services().rooms.state_accessor.is_world_readable(&room_id)?, guest_can_join: services() diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index c96ab7a0..ffe1935d 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -17,7 +17,6 @@ use ruma::{ avatar::RoomAvatarEventContent, create::RoomCreateEventContent, join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent, RoomMembership}, - topic::RoomTopicEventContent, }, space::child::{HierarchySpaceChildEvent, SpaceChildEventContent}, StateEventType, @@ -573,15 +572,7 @@ impl Service { topic: services() .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomTopic, "")? - .map_or(Ok(None), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomTopicEventContent| Some(c.topic)) - .map_err(|_| { - error!("Invalid room topic event in database for room {}", room_id); - Error::bad_database("Invalid room topic event in database.") - }) - }) + .get_room_topic(room_id) .unwrap_or(None), world_readable: services().rooms.state_accessor.is_world_readable(room_id)?, guest_can_join: services().rooms.state_accessor.guest_can_join(room_id)?, diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 62c8c224..d3dc92ef 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -15,6 +15,7 @@ use ruma::{ history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, member::{MembershipState, RoomMemberEventContent}, name::RoomNameEventContent, + topic::RoomTopicEventContent, }, StateEventType, }, @@ -341,4 +342,17 @@ impl Service { .map_err(|_| Error::bad_database("Invalid canonical alias event in database.")) }) } + + /// Gets the room topic + pub fn get_room_topic(&self, room_id: &RoomId) -> Result, Error> { + self.room_state_get(room_id, &StateEventType::RoomTopic, "")? + .map_or(Ok(None), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomTopicEventContent| Some(c.topic)) + .map_err(|e| { + error!("Invalid room topic event in database for room {room_id}: {e}"); + Error::bad_database("Invalid room topic event in database.") + }) + }) + } } From 9bb90213e1c747d544d13f49fbcbe2a98191590c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 9 Jun 2024 21:13:53 -0400 Subject: [PATCH 0152/2091] adminroom: add user ID parsing utils Signed-off-by: strawberry --- src/admin/utils.rs | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/admin/utils.rs b/src/admin/utils.rs index 6e65f2b2..4c30e5a8 100644 --- a/src/admin/utils.rs +++ b/src/admin/utils.rs @@ -1,7 +1,9 @@ pub(crate) use conduit::utils::HtmlEscape; -use ruma::{OwnedRoomId, RoomId}; +use conduit_core::Error; +use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; +use service::user_is_local; -use crate::services; +use crate::{services, Result}; pub(crate) fn escape_html(s: &str) -> String { s.replace('&', "&") @@ -28,3 +30,35 @@ pub(crate) fn get_room_info(id: &RoomId) -> (OwnedRoomId, u64, String) { .unwrap_or_else(|| id.to_string()), ) } + +/// Parses user ID +pub(crate) fn parse_user_id(user_id: &str) -> Result { + UserId::parse_with_server_name(user_id.to_lowercase(), services().globals.server_name()) + .map_err(|e| Error::Err(format!("The supplied username is not a valid username: {e}"))) +} + +/// Parses user ID as our local user +pub(crate) fn parse_local_user_id(user_id: &str) -> Result { + let user_id = parse_user_id(user_id)?; + + if !user_is_local(&user_id) { + return Err(Error::Err(String::from("User does not belong to our server."))); + } + + Ok(user_id) +} + +/// Parses user ID that is an active (not guest or deactivated) local user +pub(crate) fn parse_active_local_user_id(user_id: &str) -> Result { + let user_id = parse_local_user_id(user_id)?; + + if !services().users.exists(&user_id)? { + return Err(Error::Err(String::from("User does not exist on this server."))); + } + + if services().users.is_deactivated(&user_id)? { + return Err(Error::Err(String::from("User is deactivated."))); + } + + Ok(user_id) +} From b4f0a8a8b5e96561a3017ea1c38bfaf0ee8d0ad9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 02:26:53 -0400 Subject: [PATCH 0153/2091] adminroom: clean up and optimise user commands `deactivate-all` was terrible and incredibly inefficient Signed-off-by: strawberry --- src/admin/user/user_commands.rs | 304 +++++++++++++------------------- 1 file changed, 126 insertions(+), 178 deletions(-) diff --git a/src/admin/user/user_commands.rs b/src/admin/user/user_commands.rs index 24f30931..4869af9a 100644 --- a/src/admin/user/user_commands.rs +++ b/src/admin/user/user_commands.rs @@ -1,20 +1,29 @@ -use std::{fmt::Write as _, sync::Arc}; +use std::fmt::Write as _; use api::client::{join_room_by_id_helper, leave_all_rooms}; use conduit::utils; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, UserId}; +use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, OwnedUserId, RoomId, UserId}; use tracing::{error, info, warn}; -use crate::{escape_html, get_room_info, services, user_is_local, Result}; +use crate::{ + escape_html, get_room_info, services, + utils::{parse_active_local_user_id, parse_local_user_id}, + Result, +}; const AUTO_GEN_PASSWORD_LENGTH: usize = 25; pub(crate) async fn list(_body: Vec<&str>) -> Result { match services().users.list_local_users() { Ok(users) => { - let mut msg = format!("Found {} local user account(s):\n", users.len()); - msg += &users.join("\n"); - Ok(RoomMessageEventContent::text_plain(&msg)) + let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); + plain_msg += &users.join("\n"); + plain_msg += "\n```"; + + let mut html_msg = format!("

    Found {} local user account(s):

    ", users.len());
+			html_msg += &users.join("\n");
+			html_msg += "\n
    "; + Ok(RoomMessageEventContent::text_html(&plain_msg, &html_msg)) }, Err(e) => Ok(RoomMessageEventContent::text_plain(e.to_string())), } @@ -23,34 +32,15 @@ pub(crate) async fn list(_body: Vec<&str>) -> Result { pub(crate) async fn create( _body: Vec<&str>, username: String, password: Option, ) -> Result { - let password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); - // Validate user id - let user_id = - match UserId::parse_with_server_name(username.as_str().to_lowercase(), services().globals.server_name()) { - Ok(id) => id, - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "The supplied username is not a valid username: {e}" - ))) - }, - }; - - if !user_is_local(&user_id) { - return Ok(RoomMessageEventContent::text_plain(format!( - "User {user_id} does not belong to our server." - ))); - } - - if user_id.is_historical() { - return Ok(RoomMessageEventContent::text_plain(format!( - "Userid {user_id} is not allowed due to historical" - ))); - } + let user_id = parse_local_user_id(&username)?; if services().users.exists(&user_id)? { return Ok(RoomMessageEventContent::text_plain(format!("Userid {user_id} already exists"))); } + + let password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); + // Create user services().users.create(&user_id, Some(password.as_str()))?; @@ -134,76 +124,43 @@ pub(crate) async fn deactivate( _body: Vec<&str>, leave_rooms: bool, user_id: String, ) -> Result { // Validate user id - let user_id = - match UserId::parse_with_server_name(user_id.as_str().to_lowercase(), services().globals.server_name()) { - Ok(id) => Arc::::from(id), - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "The supplied username is not a valid username: {e}" - ))) - }, - }; + let user_id = parse_local_user_id(&user_id)?; - // check if user belongs to our server - if user_id.server_name() != services().globals.server_name() { - return Ok(RoomMessageEventContent::text_plain(format!( - "User {user_id} does not belong to our server." - ))); - } - - // don't deactivate the conduit service account + // don't deactivate the server service account if user_id == UserId::parse_with_server_name("conduit", services().globals.server_name()).expect("conduit user exists") { return Ok(RoomMessageEventContent::text_plain( - "Not allowed to deactivate the Conduit service account.", + "Not allowed to deactivate the server service account.", )); } - if services().users.exists(&user_id)? { - RoomMessageEventContent::text_plain(format!("Making {user_id} leave all rooms before deactivation...")); + services().users.deactivate_account(&user_id)?; - services().users.deactivate_account(&user_id)?; - - if leave_rooms { - leave_all_rooms(&user_id).await; - } - - Ok(RoomMessageEventContent::text_plain(format!( - "User {user_id} has been deactivated" - ))) - } else { - Ok(RoomMessageEventContent::text_plain(format!( - "User {user_id} doesn't exist on this server" - ))) + if leave_rooms { + services() + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "Making {user_id} leave all rooms after deactivation..." + ))) + .await; + leave_all_rooms(&user_id).await; } + + Ok(RoomMessageEventContent::text_plain(format!( + "User {user_id} has been deactivated" + ))) } pub(crate) async fn reset_password(_body: Vec<&str>, username: String) -> Result { - // Validate user id - let user_id = - match UserId::parse_with_server_name(username.as_str().to_lowercase(), services().globals.server_name()) { - Ok(id) => Arc::::from(id), - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "The supplied username is not a valid username: {e}" - ))) - }, - }; + let user_id = parse_local_user_id(&username)?; - // check if user belongs to our server - if user_id.server_name() != services().globals.server_name() { - return Ok(RoomMessageEventContent::text_plain(format!( - "User {user_id} does not belong to our server." - ))); - } - - // Check if the specified user is valid - if !services().users.exists(&user_id)? - || user_id - == UserId::parse_with_server_name("conduit", services().globals.server_name()).expect("conduit user exists") + if user_id + == UserId::parse_with_server_name("conduit", services().globals.server_name()).expect("conduit user exists") { - return Ok(RoomMessageEventContent::text_plain("The specified user does not exist!")); + return Ok(RoomMessageEventContent::text_plain( + "Not allowed to set the password for the server account. Please use the emergency password config option.", + )); } let new_password = utils::random_string(AUTO_GEN_PASSWORD_LENGTH); @@ -222,106 +179,95 @@ pub(crate) async fn reset_password(_body: Vec<&str>, username: String) -> Result } pub(crate) async fn deactivate_all(body: Vec<&str>, leave_rooms: bool, force: bool) -> Result { - if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { - let usernames = body.clone().drain(1..body.len() - 1).collect::>(); - - let mut user_ids: Vec<&UserId> = Vec::new(); - - for &username in &usernames { - match <&UserId>::try_from(username) { - Ok(user_id) => user_ids.push(user_id), - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "{username} is not a valid username: {e}" - ))) - }, - } - } - - let mut deactivation_count: usize = 0; - let mut admins = Vec::new(); - - if !force { - user_ids.retain(|&user_id| match services().users.is_admin(user_id) { - Ok(is_admin) => { - if is_admin { - admins.push(user_id.localpart()); - false - } else { - true - } - }, - Err(_) => false, - }); - } - - for &user_id in &user_ids { - // check if user belongs to our server and skips over non-local users - if user_id.server_name() != services().globals.server_name() { - continue; - } - - // don't deactivate the conduit service account - if user_id - == UserId::parse_with_server_name("conduit", services().globals.server_name()) - .expect("conduit user exists") - { - continue; - } - - // user does not exist on our server - if !services().users.exists(user_id)? { - continue; - } - - if services().users.deactivate_account(user_id).is_ok() { - deactivation_count = deactivation_count.saturating_add(1); - } - } - - if leave_rooms || force { - for &user_id in &user_ids { - leave_all_rooms(user_id).await; - } - } - - if admins.is_empty() { - Ok(RoomMessageEventContent::text_plain(format!( - "Deactivated {deactivation_count} accounts." - ))) - } else { - Ok(RoomMessageEventContent::text_plain(format!( - "Deactivated {} accounts.\nSkipped admin accounts: {}. Use --force to deactivate admin accounts", - deactivation_count, - admins.join(", ") - ))) - } - } else { - Ok(RoomMessageEventContent::text_plain( + if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { + return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", - )) + )); + } + + let usernames = body.clone().drain(1..body.len() - 1).collect::>(); + + let mut user_ids: Vec = Vec::with_capacity(usernames.len()); + let mut admins = Vec::new(); + + for username in usernames { + match parse_active_local_user_id(username) { + Ok(user_id) => { + if services().users.is_admin(&user_id)? && !force { + services() + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "{username} is an admin and --force is not set, skipping over" + ))) + .await; + admins.push(username); + continue; + } + + // don't deactivate the server service account + if user_id + == UserId::parse_with_server_name("conduit", services().globals.server_name()) + .expect("server user exists") + { + services() + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "{username} is the server service account, skipping over" + ))) + .await; + continue; + } + + user_ids.push(user_id); + }, + Err(e) => { + services() + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "{username} is not a valid username, skipping over: {e}" + ))) + .await; + continue; + }, + } + } + + let mut deactivation_count: usize = 0; + + for user_id in user_ids { + match services().users.deactivate_account(&user_id) { + Ok(()) => { + deactivation_count = deactivation_count.saturating_add(1); + if leave_rooms || force { + info!("Forcing user {user_id} to leave all rooms apart of deactivate-all"); + leave_all_rooms(&user_id).await; + } + }, + Err(e) => { + services() + .admin + .send_message(RoomMessageEventContent::text_plain(format!("Failed deactivating user: {e}"))) + .await; + }, + } + } + + if admins.is_empty() { + Ok(RoomMessageEventContent::text_plain(format!( + "Deactivated {deactivation_count} accounts." + ))) + } else { + Ok(RoomMessageEventContent::text_plain(format!( + "Deactivated {deactivation_count} accounts.\nSkipped admin accounts: {}. Use --force to deactivate admin \ + accounts", + admins.join(", ") + ))) } } pub(crate) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Result { // Validate user id - let user_id = - match UserId::parse_with_server_name(user_id.as_str().to_lowercase(), services().globals.server_name()) { - Ok(id) => Arc::::from(id), - Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "The supplied username is not a valid username: {e}" - ))) - }, - }; - - if !user_is_local(&user_id) { - return Ok(RoomMessageEventContent::text_plain("User does not belong to our server.")); - } - - if !services().users.exists(&user_id)? { - return Ok(RoomMessageEventContent::text_plain("User does not exist on this server.")); - } + let user_id = parse_local_user_id(&user_id)?; let mut rooms: Vec<(OwnedRoomId, u64, String)> = services() .rooms @@ -347,6 +293,7 @@ pub(crate) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Resu .collect::>() .join("\n") ); + let output_html = format!( "\n\t\t\n{}
    Rooms {user_id} Joined \ ({})
    idmembersname
    ", @@ -365,5 +312,6 @@ pub(crate) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Resu output }) ); + Ok(RoomMessageEventContent::text_html(output_plain, output_html)) } From c834e86e67204ecbc723505eb63b9ed0dc09f84d Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 02:48:04 -0400 Subject: [PATCH 0154/2091] adminroom: user cmds to put/get/delete room account data primarily useful for inserting `m.server_notice` user account data onto the admin room Signed-off-by: strawberry --- src/admin/user/mod.rs | 43 +++++++++++++- src/admin/user/user_commands.rs | 102 +++++++++++++++++++++++++++++++- 2 files changed, 142 insertions(+), 3 deletions(-) diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 031dd97a..e715daf0 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -1,7 +1,8 @@ pub(crate) mod user_commands; use clap::Subcommand; -use ruma::events::room::message::RoomMessageEventContent; +use ruma::{events::room::message::RoomMessageEventContent, RoomId}; +use user_commands::{delete_room_tag, get_room_tags, put_room_tag}; use self::user_commands::{create, deactivate, deactivate_all, list, list_joined_rooms, reset_password}; use crate::Result; @@ -62,6 +63,32 @@ pub(crate) enum UserCommand { ListJoinedRooms { user_id: String, }, + + /// - Puts a room tag for the specified user and room ID. + /// + /// This is primarily useful if you'd like to set your admin room + /// to the special "System Alerts" section in Element as a way to + /// permanently see your admin room without it being buried away in your + /// favourites or rooms. To do this, you would pass your user, your admin + /// room's internal ID, and the tag name `m.server_notice`. + PutRoomTag { + user_id: String, + room_id: Box, + tag: String, + }, + + /// - Deletes the room tag for the specified user and room ID + DeleteRoomTag { + user_id: String, + room_id: Box, + tag: String, + }, + + /// - Gets all the room tags for the specified user and room ID + GetRoomTags { + user_id: String, + room_id: Box, + }, } pub(crate) async fn process(command: UserCommand, body: Vec<&str>) -> Result { @@ -85,5 +112,19 @@ pub(crate) async fn process(command: UserCommand, body: Vec<&str>) -> Result list_joined_rooms(body, user_id).await?, + UserCommand::PutRoomTag { + user_id, + room_id, + tag, + } => put_room_tag(body, user_id, room_id, tag).await?, + UserCommand::DeleteRoomTag { + user_id, + room_id, + tag, + } => delete_room_tag(body, user_id, room_id, tag).await?, + UserCommand::GetRoomTags { + user_id, + room_id, + } => get_room_tags(body, user_id, room_id).await?, }) } diff --git a/src/admin/user/user_commands.rs b/src/admin/user/user_commands.rs index 4869af9a..9bd963dd 100644 --- a/src/admin/user/user_commands.rs +++ b/src/admin/user/user_commands.rs @@ -1,8 +1,15 @@ -use std::fmt::Write as _; +use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{join_room_by_id_helper, leave_all_rooms}; use conduit::utils; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, OwnedUserId, RoomId, UserId}; +use ruma::{ + events::{ + room::message::RoomMessageEventContent, + tag::{TagEvent, TagEventContent, TagInfo}, + RoomAccountDataEventType, + }, + OwnedRoomId, OwnedUserId, RoomId, UserId, +}; use tracing::{error, info, warn}; use crate::{ @@ -315,3 +322,94 @@ pub(crate) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Resu Ok(RoomMessageEventContent::text_html(output_plain, output_html)) } + +pub(crate) async fn put_room_tag( + _body: Vec<&str>, user_id: String, room_id: Box, tag: String, +) -> Result { + let user_id = parse_active_local_user_id(&user_id)?; + + let event = services() + .account_data + .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?; + + let mut tags_event = event.map_or_else( + || TagEvent { + content: TagEventContent { + tags: BTreeMap::new(), + }, + }, + |e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"), + ); + + tags_event + .content + .tags + .insert(tag.clone().into(), TagInfo::new()); + + services().account_data.update( + Some(&room_id), + &user_id, + RoomAccountDataEventType::Tag, + &serde_json::to_value(tags_event).expect("to json value always works"), + )?; + + Ok(RoomMessageEventContent::text_plain(format!( + "Successfully updated room account data for {user_id} and room {room_id} with tag {tag}" + ))) +} + +pub(crate) async fn delete_room_tag( + _body: Vec<&str>, user_id: String, room_id: Box, tag: String, +) -> Result { + let user_id = parse_active_local_user_id(&user_id)?; + + let event = services() + .account_data + .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?; + + let mut tags_event = event.map_or_else( + || TagEvent { + content: TagEventContent { + tags: BTreeMap::new(), + }, + }, + |e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"), + ); + + tags_event.content.tags.remove(&tag.clone().into()); + + services().account_data.update( + Some(&room_id), + &user_id, + RoomAccountDataEventType::Tag, + &serde_json::to_value(tags_event).expect("to json value always works"), + )?; + + Ok(RoomMessageEventContent::text_plain(format!( + "Successfully updated room account data for {user_id} and room {room_id}, deleting room tag {tag}" + ))) +} + +pub(crate) async fn get_room_tags( + _body: Vec<&str>, user_id: String, room_id: Box, +) -> Result { + let user_id = parse_active_local_user_id(&user_id)?; + + let event = services() + .account_data + .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?; + + let tags_event = event.map_or_else( + || TagEvent { + content: TagEventContent { + tags: BTreeMap::new(), + }, + }, + |e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"), + ); + + Ok(RoomMessageEventContent::text_html( + format!("
    \n{:?}\n
    ", tags_event.content.tags), + format!("```\n{:?}\n```", tags_event.content.tags), + )) +} From 14a3471fcb2bfb990e385059f30eadbed23790de Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 02:49:15 -0400 Subject: [PATCH 0155/2091] adminroom: fix function typo Signed-off-by: strawberry --- src/admin/federation/federation_commands.rs | 2 +- src/admin/federation/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/admin/federation/federation_commands.rs b/src/admin/federation/federation_commands.rs index 3405b6fa..7f6b7004 100644 --- a/src/admin/federation/federation_commands.rs +++ b/src/admin/federation/federation_commands.rs @@ -14,7 +14,7 @@ pub(crate) async fn enable_room(_body: Vec<&str>, room_id: Box) -> Resul Ok(RoomMessageEventContent::text_plain("Room enabled.")) } -pub(crate) async fn incoming_federeation(_body: Vec<&str>) -> Result { +pub(crate) async fn incoming_federation(_body: Vec<&str>) -> Result { let map = services().globals.roomid_federationhandletime.read().await; let mut msg = format!("Handling {} incoming pdus:\n", map.len()); diff --git a/src/admin/federation/mod.rs b/src/admin/federation/mod.rs index b3da33f2..81469feb 100644 --- a/src/admin/federation/mod.rs +++ b/src/admin/federation/mod.rs @@ -2,7 +2,7 @@ use clap::Subcommand; use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId}; use self::federation_commands::{ - disable_room, enable_room, fetch_support_well_known, incoming_federeation, remote_user_in_rooms, + disable_room, enable_room, fetch_support_well_known, incoming_federation, remote_user_in_rooms, }; use crate::Result; @@ -51,7 +51,7 @@ pub(crate) async fn process(command: FederationCommand, body: Vec<&str>) -> Resu FederationCommand::EnableRoom { room_id, } => enable_room(body, room_id).await?, - FederationCommand::IncomingFederation => incoming_federeation(body).await?, + FederationCommand::IncomingFederation => incoming_federation(body).await?, FederationCommand::FetchSupportWellKnown { server_name, } => fetch_support_well_known(body, server_name).await?, From 0e7c3cb3389c04d10b6dcb9c7112e601a16b491f Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 02:57:11 -0400 Subject: [PATCH 0156/2091] adminroom: simplify codeblock checks Signed-off-by: strawberry --- src/admin/appservice/appservice_command.rs | 38 +-- src/admin/debug/debug_commands.rs | 179 ++++++----- src/admin/media/media_commands.rs | 44 +-- src/admin/room/room_moderation_commands.rs | 344 ++++++++++----------- 4 files changed, 306 insertions(+), 299 deletions(-) diff --git a/src/admin/appservice/appservice_command.rs b/src/admin/appservice/appservice_command.rs index d15cb7a6..6f36ebf7 100644 --- a/src/admin/appservice/appservice_command.rs +++ b/src/admin/appservice/appservice_command.rs @@ -3,26 +3,26 @@ use ruma::{api::appservice::Registration, events::room::message::RoomMessageEven use crate::{escape_html, services, Result}; pub(crate) async fn register(body: Vec<&str>) -> Result { - if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { - let appservice_config = body[1..body.len().checked_sub(1).unwrap()].join("\n"); - let parsed_config = serde_yaml::from_str::(&appservice_config); - match parsed_config { - Ok(yaml) => match services().appservice.register_appservice(yaml).await { - Ok(id) => Ok(RoomMessageEventContent::text_plain(format!( - "Appservice registered with ID: {id}." - ))), - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Failed to register appservice: {e}" - ))), - }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Could not parse appservice config: {e}" - ))), - } - } else { - Ok(RoomMessageEventContent::text_plain( + if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { + return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", - )) + )); + } + + let appservice_config = body[1..body.len().checked_sub(1).unwrap()].join("\n"); + let parsed_config = serde_yaml::from_str::(&appservice_config); + match parsed_config { + Ok(yaml) => match services().appservice.register_appservice(yaml).await { + Ok(id) => Ok(RoomMessageEventContent::text_plain(format!( + "Appservice registered with ID: {id}." + ))), + Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + "Failed to register appservice: {e}" + ))), + }, + Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + "Could not parse appservice config: {e}" + ))), } } diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 56563748..bd15ba50 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -43,28 +43,30 @@ pub(crate) async fn get_auth_chain(_body: Vec<&str>, event_id: Box) -> } pub(crate) async fn parse_pdu(body: Vec<&str>) -> Result { - if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { - let string = body[1..body.len() - 1].join("\n"); - match serde_json::from_str(&string) { - Ok(value) => match ruma::signatures::reference_hash(&value, &RoomVersionId::V6) { - Ok(hash) => { - let event_id = EventId::parse(format!("${hash}")); + if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { + return Ok(RoomMessageEventContent::text_plain( + "Expected code block in command body. Add --help for details.", + )); + } - match serde_json::from_value::(serde_json::to_value(value).expect("value is json")) { - Ok(pdu) => Ok(RoomMessageEventContent::text_plain(format!("EventId: {event_id:?}\n{pdu:#?}"))), - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "EventId: {event_id:?}\nCould not parse event: {e}" - ))), - } - }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Could not parse PDU JSON: {e:?}"))), + let string = body[1..body.len() - 1].join("\n"); + match serde_json::from_str(&string) { + Ok(value) => match ruma::signatures::reference_hash(&value, &RoomVersionId::V6) { + Ok(hash) => { + let event_id = EventId::parse(format!("${hash}")); + + match serde_json::from_value::(serde_json::to_value(value).expect("value is json")) { + Ok(pdu) => Ok(RoomMessageEventContent::text_plain(format!("EventId: {event_id:?}\n{pdu:#?}"))), + Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + "EventId: {event_id:?}\nCould not parse event: {e}" + ))), + } }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Invalid json in command body: {e}" - ))), - } - } else { - Ok(RoomMessageEventContent::text_plain("Expected code block in command body.")) + Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Could not parse PDU JSON: {e:?}"))), + }, + Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + "Invalid json in command body: {e}" + ))), } } @@ -117,33 +119,40 @@ pub(crate) async fn get_remote_pdu_list( if server == services().globals.server_name() { return Ok(RoomMessageEventContent::text_plain( - "Not allowed to send federation requests to ourselves. Please use `get-pdu` for fetching local PDUs.", + "Not allowed to send federation requests to ourselves. Please use `get-pdu` for fetching local PDUs from \ + the database.", )); } - if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { - let list = body - .clone() - .drain(1..body.len().checked_sub(1).unwrap()) - .filter_map(|pdu| EventId::parse(pdu).ok()) - .collect::>(); - - for pdu in list { - if force { - if let Err(e) = get_remote_pdu(Vec::new(), Box::from(pdu), server.clone()).await { - warn!(%e, "Failed to get remote PDU, ignoring error"); - } - } else { - get_remote_pdu(Vec::new(), Box::from(pdu), server.clone()).await?; - } - } - - return Ok(RoomMessageEventContent::text_plain("Fetched list of remote PDUs.")); + if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { + return Ok(RoomMessageEventContent::text_plain( + "Expected code block in command body. Add --help for details.", + )); } - Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )) + let list = body + .clone() + .drain(1..body.len().checked_sub(1).unwrap()) + .filter_map(|pdu| EventId::parse(pdu).ok()) + .collect::>(); + + for pdu in list { + if force { + if let Err(e) = get_remote_pdu(Vec::new(), Box::from(pdu), server.clone()).await { + services() + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "Failed to get remote PDU, ignoring error: {e}" + ))) + .await; + warn!(%e, "Failed to get remote PDU, ignoring error"); + } + } else { + get_remote_pdu(Vec::new(), Box::from(pdu), server.clone()).await?; + } + } + + Ok(RoomMessageEventContent::text_plain("Fetched list of remote PDUs.")) } pub(crate) async fn get_remote_pdu( @@ -384,55 +393,55 @@ pub(crate) async fn change_log_level( } pub(crate) async fn sign_json(body: Vec<&str>) -> Result { - if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { - let string = body[1..body.len().checked_sub(1).unwrap()].join("\n"); - match serde_json::from_str(&string) { - Ok(mut value) => { - ruma::signatures::sign_json( - services().globals.server_name().as_str(), - services().globals.keypair(), - &mut value, - ) - .expect("our request json is what ruma expects"); - let json_text = serde_json::to_string_pretty(&value).expect("canonical json is valid json"); - Ok(RoomMessageEventContent::text_plain(json_text)) - }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), - } - } else { - Ok(RoomMessageEventContent::text_plain( + if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { + return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", - )) + )); + } + + let string = body[1..body.len().checked_sub(1).unwrap()].join("\n"); + match serde_json::from_str(&string) { + Ok(mut value) => { + ruma::signatures::sign_json( + services().globals.server_name().as_str(), + services().globals.keypair(), + &mut value, + ) + .expect("our request json is what ruma expects"); + let json_text = serde_json::to_string_pretty(&value).expect("canonical json is valid json"); + Ok(RoomMessageEventContent::text_plain(json_text)) + }, + Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), } } pub(crate) async fn verify_json(body: Vec<&str>) -> Result { - if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { - let string = body[1..body.len().checked_sub(1).unwrap()].join("\n"); - match serde_json::from_str(&string) { - Ok(value) => { - let pub_key_map = RwLock::new(BTreeMap::new()); - - services() - .rooms - .event_handler - .fetch_required_signing_keys([&value], &pub_key_map) - .await?; - - let pub_key_map = pub_key_map.read().await; - match ruma::signatures::verify_json(&pub_key_map, &value) { - Ok(()) => Ok(RoomMessageEventContent::text_plain("Signature correct")), - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Signature verification failed: {e}" - ))), - } - }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), - } - } else { - Ok(RoomMessageEventContent::text_plain( + if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { + return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", - )) + )); + } + + let string = body[1..body.len().checked_sub(1).unwrap()].join("\n"); + match serde_json::from_str(&string) { + Ok(value) => { + let pub_key_map = RwLock::new(BTreeMap::new()); + + services() + .rooms + .event_handler + .fetch_required_signing_keys([&value], &pub_key_map) + .await?; + + let pub_key_map = pub_key_map.read().await; + match ruma::signatures::verify_json(&pub_key_map, &value) { + Ok(()) => Ok(RoomMessageEventContent::text_plain("Signature correct")), + Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + "Signature verification failed: {e}" + ))), + } + }, + Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), } } diff --git a/src/admin/media/media_commands.rs b/src/admin/media/media_commands.rs index 956f5743..a78ee387 100644 --- a/src/admin/media/media_commands.rs +++ b/src/admin/media/media_commands.rs @@ -138,30 +138,30 @@ pub(crate) async fn delete( } pub(crate) async fn delete_list(body: Vec<&str>) -> Result { - if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { - let mxc_list = body - .clone() - .drain(1..body.len().checked_sub(1).unwrap()) - .collect::>(); - - let mut mxc_deletion_count: usize = 0; - - for mxc in mxc_list { - debug!("Deleting MXC {mxc} in bulk"); - services().media.delete(mxc.to_owned()).await?; - mxc_deletion_count = mxc_deletion_count - .checked_add(1) - .expect("mxc_deletion_count should not get this high"); - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database and the filesystem.", - ))); + if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { + return Ok(RoomMessageEventContent::text_plain( + "Expected code block in command body. Add --help for details.", + )); } - Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )) + let mxc_list = body + .clone() + .drain(1..body.len().checked_sub(1).unwrap()) + .collect::>(); + + let mut mxc_deletion_count: usize = 0; + + for mxc in mxc_list { + debug!("Deleting MXC {mxc} in bulk"); + services().media.delete(mxc.to_owned()).await?; + mxc_deletion_count = mxc_deletion_count + .checked_add(1) + .expect("mxc_deletion_count should not get this high"); + } + + Ok(RoomMessageEventContent::text_plain(format!( + "Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database and the filesystem.", + ))) } pub(crate) async fn delete_past_remote_media( diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 6b759c9b..f6d3206a 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -187,135 +187,137 @@ async fn ban_room( } async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: bool) -> Result { - if body.len() > 2 && body[0].trim().starts_with("```") && body.last().unwrap().trim() == "```" { - let rooms_s = body.clone().drain(1..body.len() - 1).collect::>(); + if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { + return Ok(RoomMessageEventContent::text_plain( + "Expected code block in command body. Add --help for details.", + )); + } - let admin_room_alias: Box = format!("#admins:{}", services().globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid alias name"); + let rooms_s = body.clone().drain(1..body.len() - 1).collect::>(); - let mut room_ban_count: usize = 0; - let mut room_ids: Vec = Vec::new(); + let admin_room_alias: Box = format!("#admins:{}", services().globals.server_name()) + .try_into() + .expect("#admins:server_name is a valid alias name"); - for &room in &rooms_s { - match <&RoomOrAliasId>::try_from(room) { - Ok(room_alias_or_id) => { - if let Some(admin_room_id) = Service::get_admin_room().await? { - if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(&admin_room_alias) { - info!("User specified admin room in bulk ban list, ignoring"); - continue; - } - } + let mut room_ban_count: usize = 0; + let mut room_ids: Vec = Vec::new(); - if room_alias_or_id.is_room_id() { - let room_id = match RoomId::parse(room_alias_or_id) { - Ok(room_id) => room_id, - Err(e) => { - if force { - // ignore rooms we failed to parse if we're force banning - warn!( - "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ - logging here: {e}" - ); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try again: \ - {e}" - ))); - }, - }; - - room_ids.push(room_id); - } - - if room_alias_or_id.is_room_alias_id() { - match RoomAliasId::parse(room_alias_or_id) { - Ok(room_alias) => { - let room_id = - if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { - room_id - } else { - debug!( - "We don't have this room alias to a room ID locally, attempting to fetch \ - room ID over federation" - ); - - match get_alias_helper(room_alias, None).await { - Ok(response) => { - debug!( - "Got federation response fetching room ID for room {room}: {:?}", - response - ); - response.room_id - }, - Err(e) => { - // don't fail if force blocking - if force { - warn!("Failed to resolve room alias {room} to a room ID: {e}"); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to resolve room alias {room} to a room ID: {e}" - ))); - }, - } - }; - - room_ids.push(room_id); - }, - Err(e) => { - if force { - // ignore rooms we failed to parse if we're force deleting - error!( - "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ - logging here: {e}" - ); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try again: \ - {e}" - ))); - }, - } - } - }, - Err(e) => { - if force { - // ignore rooms we failed to parse if we're force deleting - error!( - "Error parsing room \"{room}\" during bulk room banning, ignoring error and logging here: \ - {e}" - ); + for &room in &rooms_s { + match <&RoomOrAliasId>::try_from(room) { + Ok(room_alias_or_id) => { + if let Some(admin_room_id) = Service::get_admin_room().await? { + if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(&admin_room_alias) { + info!("User specified admin room in bulk ban list, ignoring"); continue; } + } - return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try again: {e}" - ))); - }, - } + if room_alias_or_id.is_room_id() { + let room_id = match RoomId::parse(room_alias_or_id) { + Ok(room_id) => room_id, + Err(e) => { + if force { + // ignore rooms we failed to parse if we're force banning + warn!( + "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ + logging here: {e}" + ); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!( + "{room} is not a valid room ID or room alias, please fix the list and try again: {e}" + ))); + }, + }; + + room_ids.push(room_id); + } + + if room_alias_or_id.is_room_alias_id() { + match RoomAliasId::parse(room_alias_or_id) { + Ok(room_alias) => { + let room_id = + if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { + room_id + } else { + debug!( + "We don't have this room alias to a room ID locally, attempting to fetch room \ + ID over federation" + ); + + match get_alias_helper(room_alias, None).await { + Ok(response) => { + debug!( + "Got federation response fetching room ID for room {room}: {:?}", + response + ); + response.room_id + }, + Err(e) => { + // don't fail if force blocking + if force { + warn!("Failed to resolve room alias {room} to a room ID: {e}"); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to resolve room alias {room} to a room ID: {e}" + ))); + }, + } + }; + + room_ids.push(room_id); + }, + Err(e) => { + if force { + // ignore rooms we failed to parse if we're force deleting + error!( + "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ + logging here: {e}" + ); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!( + "{room} is not a valid room ID or room alias, please fix the list and try again: {e}" + ))); + }, + } + } + }, + Err(e) => { + if force { + // ignore rooms we failed to parse if we're force deleting + error!( + "Error parsing room \"{room}\" during bulk room banning, ignoring error and logging here: {e}" + ); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!( + "{room} is not a valid room ID or room alias, please fix the list and try again: {e}" + ))); + }, + } + } + + for room_id in room_ids { + if services().rooms.metadata.ban_room(&room_id, true).is_ok() { + debug!("Banned {room_id} successfully"); + room_ban_count = room_ban_count.saturating_add(1); } - for room_id in room_ids { - if services().rooms.metadata.ban_room(&room_id, true).is_ok() { - debug!("Banned {room_id} successfully"); - room_ban_count = room_ban_count.saturating_add(1); - } - - debug!("Making all users leave the room {}", &room_id); - if force { - for local_user in services() - .rooms - .state_cache - .room_members(&room_id) - .filter_map(|user| { - user.ok().filter(|local_user| { - local_user.server_name() == services().globals.server_name() + debug!("Making all users leave the room {}", &room_id); + if force { + for local_user in services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(|user| { + user.ok().filter(|local_user| { + local_user.server_name() == services().globals.server_name() // additional wrapped check here is to avoid adding remote users // who are in the admin room to the list of local users (would fail auth check) && (local_user.server_name() @@ -324,31 +326,31 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo .users .is_admin(local_user) .unwrap_or(true)) // since this is a - // force operation, - // assume user is - // an admin if - // somehow this - // fails - }) + // force operation, + // assume user is + // an admin if + // somehow this + // fails }) - .collect::>() - { - debug!( - "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", - &local_user, room_id - ); - if let Err(e) = leave_room(&local_user, &room_id, None).await { - warn!(%e, "Failed to leave room"); - } + }) + .collect::>() + { + debug!( + "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", + &local_user, room_id + ); + if let Err(e) = leave_room(&local_user, &room_id, None).await { + warn!(%e, "Failed to leave room"); } - } else { - for local_user in services() - .rooms - .state_cache - .room_members(&room_id) - .filter_map(|user| { - user.ok().filter(|local_user| { - local_user.server_name() == services().globals.server_name() + } + } else { + for local_user in services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(|user| { + user.ok().filter(|local_user| { + local_user.server_name() == services().globals.server_name() // additional wrapped check here is to avoid adding remote users // who are in the admin room to the list of local users (would fail auth check) && (local_user.server_name() @@ -357,45 +359,41 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo .users .is_admin(local_user) .unwrap_or(false)) - }) }) - .collect::>() - { - debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(&local_user, &room_id, None).await { - error!( - "Error attempting to make local user {} leave room {} during bulk room banning: {}", - &local_user, &room_id, e - ); - return Ok(RoomMessageEventContent::text_plain(format!( - "Error attempting to make local user {} leave room {} during room banning (room is still \ - banned but not removing any more users and not banning any more rooms): {}\nIf you would \ - like to ignore errors, use --force", - &local_user, &room_id, e - ))); - } + }) + .collect::>() + { + debug!("Attempting leave for user {} in room {}", &local_user, &room_id); + if let Err(e) = leave_room(&local_user, &room_id, None).await { + error!( + "Error attempting to make local user {} leave room {} during bulk room banning: {}", + &local_user, &room_id, e + ); + return Ok(RoomMessageEventContent::text_plain(format!( + "Error attempting to make local user {} leave room {} during room banning (room is still \ + banned but not removing any more users and not banning any more rooms): {}\nIf you would \ + like to ignore errors, use --force", + &local_user, &room_id, e + ))); } } - - if disable_federation { - services().rooms.metadata.disable_room(&room_id, true)?; - } } if disable_federation { - return Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and disabled \ - incoming federation with the room." - ))); + services().rooms.metadata.disable_room(&room_id, true)?; } - return Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk room ban, banned {room_ban_count} total rooms and evicted all users." - ))); } - Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )) + if disable_federation { + Ok(RoomMessageEventContent::text_plain(format!( + "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and disabled incoming \ + federation with the room." + ))) + } else { + Ok(RoomMessageEventContent::text_plain(format!( + "Finished bulk room ban, banned {room_ban_count} total rooms and evicted all users." + ))) + } } async fn unban_room( From 74b29ce06738a1e728649c5e54e5cc7fc7506b33 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 03:27:15 -0400 Subject: [PATCH 0157/2091] adminroom: improved room list outputs, add counts Signed-off-by: strawberry --- src/admin/federation/federation_commands.rs | 10 ++-- src/admin/room/room_moderation_commands.rs | 62 ++++++++++++++------- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/src/admin/federation/federation_commands.rs b/src/admin/federation/federation_commands.rs index 7f6b7004..b52a62f2 100644 --- a/src/admin/federation/federation_commands.rs +++ b/src/admin/federation/federation_commands.rs @@ -101,7 +101,8 @@ pub(crate) async fn remote_user_in_rooms(_body: Vec<&str>, user_id: Box) rooms.reverse(); let output_plain = format!( - "Rooms {user_id} shares with us:\n{}", + "Rooms {user_id} shares with us ({}):\n{}", + rooms.len(), rooms .iter() .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) @@ -109,15 +110,16 @@ pub(crate) async fn remote_user_in_rooms(_body: Vec<&str>, user_id: Box) .join("\n") ); let output_html = format!( - "\n\t\t\n{}
    Rooms {user_id} shares with \ - us
    idmembersname
    ", + "\n\t\t\n{}
    Rooms {user_id} shares with us \ + ({})
    idmembersname
    ", + rooms.len(), rooms .iter() .fold(String::new(), |mut output, (id, members, name)| { writeln!( output, "{}\t{}\t{}", - escape_html(id.as_ref()), + id, members, escape_html(name) ) diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index f6d3206a..9886756b 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -6,11 +6,8 @@ use ruma::{ }; use tracing::{debug, error, info, warn}; -use super::{ - super::{escape_html, Service}, - RoomModerationCommand, -}; -use crate::{services, user_is_local, Result}; +use super::{super::Service, RoomModerationCommand}; +use crate::{escape_html, get_room_info, services, user_is_local, Result}; pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> Result { match command { @@ -479,26 +476,51 @@ async fn list_banned_rooms(_body: Vec<&str>) -> Result match rooms { Ok(room_ids) => { - // TODO: add room name from our state cache if available, default to the room ID - // as the room name if we dont have it TODO: do same if we have a room alias for - // this - let plain_list = room_ids.iter().fold(String::new(), |mut output, room_id| { - writeln!(output, "- `{room_id}`").unwrap(); - output - }); + if room_ids.is_empty() { + return Ok(RoomMessageEventContent::text_plain("No rooms are banned.")); + } - let html_list = room_ids.iter().fold(String::new(), |mut output, room_id| { - writeln!(output, "
  • {}
    • ", escape_html(room_id.as_ref())).unwrap(); - output - }); + let mut rooms = room_ids + .into_iter() + .map(|room_id| get_room_info(&room_id)) + .collect::>(); + rooms.sort_by_key(|r| r.1); + rooms.reverse(); - let plain = format!("Rooms:\n{plain_list}"); - let html = format!("Rooms:\n
      {html_list}
    "); - Ok(RoomMessageEventContent::text_html(plain, html)) + let output_plain = format!( + "Rooms Banned ({}):\n```\n{}```", + rooms.len(), + rooms + .iter() + .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) + .collect::>() + .join("\n") + ); + + let output_html = format!( + "\n\t\t\n{}
    Rooms Banned ({}) \ +
    idmembersname
    ", + rooms.len(), + rooms + .iter() + .fold(String::new(), |mut output, (id, members, name)| { + writeln!( + output, + "{}\t{}\t{}", + id, + members, + escape_html(name.as_ref()) + ) + .expect("should be able to write to string buffer"); + output + }) + ); + + Ok(RoomMessageEventContent::text_html(output_plain, output_html)) }, Err(e) => { error!("Failed to list banned rooms: {}", e); - Ok(RoomMessageEventContent::text_plain(format!("Unable to list room aliases: {e}"))) + Ok(RoomMessageEventContent::text_plain(format!("Unable to list banned rooms: {e}"))) }, } } From f1d90e5df60064a342ca2532704c1720b1ba6bbd Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 21:04:51 -0400 Subject: [PATCH 0158/2091] log client/remote IP address on various routes tracing calls this uses InsecureClientIp as this is purely for informational and logging purposes Signed-off-by: strawberry --- src/api/client/account.rs | 39 ++++++++++++++++++++++------------- src/api/client/directory.rs | 18 +++++++++------- src/api/client/membership.rs | 12 ++++++++--- src/api/server/invite.rs | 6 +++++- src/api/server/publicrooms.rs | 7 +++++-- src/api/server/send.rs | 4 +++- src/core/utils/mod.rs | 3 +-- src/router/layers.rs | 1 - 8 files changed, 59 insertions(+), 31 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 72830f9d..dc384c4c 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -1,5 +1,6 @@ use std::fmt::Write; +use axum_client_ip::InsecureClientIp; use conduit::debug_info; use register::RegistrationKind; use ruma::{ @@ -39,8 +40,9 @@ const RANDOM_USER_ID_LENGTH: usize = 10; /// /// Note: This will not reserve the username, so the username might become /// invalid when trying to register +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn get_register_available_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { // Validate user id let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), services().globals.server_name()) @@ -87,7 +89,10 @@ pub(crate) async fn get_register_available_route( /// - If `inhibit_login` is false: Creates a device and returns device id and /// access_token #[allow(clippy::doc_markdown)] -pub(crate) async fn register_route(body: Ruma) -> Result { +#[tracing::instrument(skip_all, fields(%client_ip))] +pub(crate) async fn register_route( + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, +) -> Result { if !services().globals.allow_registration() && body.appservice_info.is_none() { info!( "Registration disabled and request not from known appservice, rejecting registration attempt for username \ @@ -104,8 +109,8 @@ pub(crate) async fn register_route(body: Ruma) -> Result< || (services().globals.allow_registration() && services().globals.config.registration_token.is_some())) { info!( - "Guest registration disabled / registration enabled with token configured, rejecting guest registration, \ - initial device name: {:?}", + "Guest registration disabled / registration enabled with token configured, rejecting guest registration \ + attempt, initial device name: {:?}", body.initial_device_display_name ); return Err(Error::BadRequest( @@ -297,14 +302,14 @@ pub(crate) async fn register_route(body: Ruma) -> Result< services() .admin .send_message(RoomMessageEventContent::notice_plain(format!( - "New user \"{user_id}\" registered on this server." + "New user \"{user_id}\" registered on this server from IP {client_ip}." ))) .await; } // log in conduit admin channel if a guest registered if body.appservice_info.is_none() && is_guest && services().globals.log_guest_registrations() { - info!("New guest user \"{user_id}\" registered on this server."); + info!("New guest user \"{user_id}\" registered on this server from IP."); if let Some(device_display_name) = &body.initial_device_display_name { if body @@ -316,14 +321,15 @@ pub(crate) async fn register_route(body: Ruma) -> Result< .admin .send_message(RoomMessageEventContent::notice_plain(format!( "Guest user \"{user_id}\" with device display name `{device_display_name}` registered on this \ - server." + server from IP {client_ip}." ))) .await; } else { services() .admin .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with no device display name registered on this server.", + "Guest user \"{user_id}\" with no device display name registered on this server from IP \ + {client_ip}.", ))) .await; } @@ -331,7 +337,8 @@ pub(crate) async fn register_route(body: Ruma) -> Result< services() .admin .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with no device display name registered on this server.", + "Guest user \"{user_id}\" with no device display name registered on this server from IP \ + {client_ip}.", ))) .await; } @@ -352,7 +359,7 @@ pub(crate) async fn register_route(body: Ruma) -> Result< .make_user_admin(&user_id, displayname) .await?; - warn!("Granting {} admin privileges as the first user", user_id); + warn!("Granting {user_id} admin privileges as the first user"); } } } @@ -416,8 +423,9 @@ pub(crate) async fn register_route(body: Ruma) -> Result< /// last seen ts) /// - Forgets to-device events /// - Triggers device list updates +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn change_password_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -466,7 +474,7 @@ pub(crate) async fn change_password_route( } } - info!("User {} changed their password.", sender_user); + info!("User {sender_user} changed their password."); services() .admin .send_message(RoomMessageEventContent::notice_plain(format!( @@ -504,7 +512,10 @@ pub(crate) async fn whoami_route(body: Ruma) -> Result) -> Result { +#[tracing::instrument(skip_all, fields(%client_ip))] +pub(crate) async fn deactivate_route( + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -542,7 +553,7 @@ pub(crate) async fn deactivate_route(body: Ruma) -> Res // Remove devices and mark account as deactivated services().users.deactivate_account(sender_user)?; - info!("User {} deactivated their account.", sender_user); + info!("User {sender_user} deactivated their account."); services() .admin .send_message(RoomMessageEventContent::notice_plain(format!( diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index ef475a68..20f0ccca 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,3 +1,4 @@ +use axum_client_ip::InsecureClientIp; use ruma::{ api::{ client::{ @@ -27,8 +28,9 @@ use crate::{service::server_is_ours, services, Error, Result, Ruma}; /// Lists the public rooms on this server. /// /// - Rooms are ordered by the number of joined members +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn get_public_rooms_filtered_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { if let Some(server) = &body.server { if services() @@ -52,8 +54,8 @@ pub(crate) async fn get_public_rooms_filtered_route( ) .await .map_err(|e| { - warn!("Failed to return our /publicRooms: {e}"); - Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list.") + warn!(?body.server, "Failed to return /publicRooms: {e}"); + Error::BadRequest(ErrorKind::Unknown, "Failed to return the requested server's public room list.") })?; Ok(response) @@ -64,8 +66,9 @@ pub(crate) async fn get_public_rooms_filtered_route( /// Lists the public rooms on this server. /// /// - Rooms are ordered by the number of joined members +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn get_public_rooms_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { if let Some(server) = &body.server { if services() @@ -89,8 +92,8 @@ pub(crate) async fn get_public_rooms_route( ) .await .map_err(|e| { - warn!("Failed to return our /publicRooms: {e}"); - Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list.") + warn!(?body.server, "Failed to return /publicRooms: {e}"); + Error::BadRequest(ErrorKind::Unknown, "Failed to return the requested server's public room list.") })?; Ok(get_public_rooms::v3::Response { @@ -106,8 +109,9 @@ pub(crate) async fn get_public_rooms_route( /// Sets the visibility of a given room in the room directory. /// /// - TODO: Access control checks +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn set_room_visibility_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index ad63d896..79ea50bd 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -5,6 +5,7 @@ use std::{ time::{Duration, Instant}, }; +use axum_client_ip::InsecureClientIp; use ruma::{ api::{ client::{ @@ -141,8 +142,9 @@ async fn banned_room_check(user_id: &UserId, room_id: Option<&RoomId>, server_na /// rules locally /// - If the server does not know about the room: asks other servers over /// federation +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn join_room_by_id_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -193,8 +195,9 @@ pub(crate) async fn join_room_by_id_route( /// - If the server does not know about the room: use the server name query /// param if specified. if not specified, asks other servers over federation /// via room alias server name and room ID server name +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn join_room_by_id_or_alias_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_deref().expect("user is authenticated"); let body = body.body; @@ -295,7 +298,10 @@ pub(crate) async fn leave_room_route(body: Ruma) -> Res /// # `POST /_matrix/client/r0/rooms/{roomId}/invite` /// /// Tries to send an invite event into the room. -pub(crate) async fn invite_user_route(body: Ruma) -> Result { +#[tracing::instrument(skip_all, fields(%client_ip))] +pub(crate) async fn invite_user_route( + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); if !services().users.is_admin(sender_user)? && services().globals.block_non_admin_invites() { diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 9ebc60ee..1b256b4a 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -1,3 +1,4 @@ +use axum_client_ip::InsecureClientIp; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_invite}, events::room::member::{MembershipState, RoomMemberEventContent}, @@ -16,7 +17,10 @@ use crate::{ /// # `PUT /_matrix/federation/v2/invite/{roomId}/{eventId}` /// /// Invites a remote user to a room. -pub(crate) async fn create_invite_route(body: Ruma) -> Result { +#[tracing::instrument(skip_all, fields(%client_ip))] +pub(crate) async fn create_invite_route( + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, +) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); // ACL check origin diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index 5a91f018..6b5010e5 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -1,3 +1,4 @@ +use axum_client_ip::InsecureClientIp; use ruma::{ api::{ client::error::ErrorKind, @@ -11,8 +12,9 @@ use crate::{services, Error, Result, Ruma}; /// # `POST /_matrix/federation/v1/publicRooms` /// /// Lists the public rooms on this server. +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn get_public_rooms_filtered_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { if !services() .globals @@ -42,8 +44,9 @@ pub(crate) async fn get_public_rooms_filtered_route( /// # `GET /_matrix/federation/v1/publicRooms` /// /// Lists the public rooms on this server. +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn get_public_rooms_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { if !services() .globals diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 592eb4f9..e2413a4a 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, sync::Arc, time::Instant}; +use axum_client_ip::InsecureClientIp; use conduit::debug_warn; use ruma::{ api::{ @@ -25,8 +26,9 @@ use crate::{ /// # `PUT /_matrix/federation/v1/send/{txnId}` /// /// Push EDUs and PDUs to this server. +#[tracing::instrument(skip_all, fields(%client_ip))] pub(crate) async fn send_transaction_message_route( - body: Ruma, + InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index f3a45e47..9ffbbbd0 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -7,8 +7,7 @@ pub mod json; pub mod sys; use std::{ - cmp, - cmp::Ordering, + cmp::{self, Ordering}, time::{SystemTime, UNIX_EPOCH}, }; diff --git a/src/router/layers.rs b/src/router/layers.rs index b358a3e6..3c7a5387 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -37,7 +37,6 @@ pub(crate) fn build(server: &Arc) -> io::Result { let layers = layers .sensitive_headers([header::AUTHORIZATION]) - .sensitive_request_headers([HeaderName::from_static("x-forwarded-for")].into()) .layer(axum::middleware::from_fn_with_state(Arc::clone(server), request::spawn)) .layer( TraceLayer::new_for_http() From 65fbb80145bb003dd9ac2f390e1e01933aa9691d Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 21:14:18 -0400 Subject: [PATCH 0159/2091] adminroom: leave all rooms by default on manual deactivations Signed-off-by: strawberry --- src/admin/user/mod.rs | 22 ++++++++++++---------- src/admin/user/user_commands.rs | 10 ++++++---- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index e715daf0..e11429de 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -26,11 +26,11 @@ pub(crate) enum UserCommand { /// - Deactivate a user /// - /// User will not be removed from all rooms by default. - /// Use --leave-rooms to force the user to leave all rooms + /// User will be removed from all rooms by default. + /// Use --no-leave-rooms to not leave all rooms by default. Deactivate { #[arg(short, long)] - leave_rooms: bool, + no_leave_rooms: bool, user_id: String, }, @@ -38,8 +38,10 @@ pub(crate) enum UserCommand { /// /// Recommended to use in conjunction with list-local-users. /// - /// Users will not be removed from joined rooms by default. - /// Can be overridden with --leave-rooms OR the --force flag. + /// Users will be removed from joined rooms by default. + /// + /// Can be overridden with --no-leave-rooms. + /// /// Removing a mass amount of users from a room may cause a significant /// amount of leave events. The time to leave rooms may depend significantly /// on joined rooms and servers. @@ -49,7 +51,7 @@ pub(crate) enum UserCommand { DeactivateAll { #[arg(short, long)] /// Remove users from their joined rooms - leave_rooms: bool, + no_leave_rooms: bool, #[arg(short, long)] /// Also deactivate admin accounts and will assume leave all rooms too force: bool, @@ -99,16 +101,16 @@ pub(crate) async fn process(command: UserCommand, body: Vec<&str>) -> Result create(body, username, password).await?, UserCommand::Deactivate { - leave_rooms, + no_leave_rooms, user_id, - } => deactivate(body, leave_rooms, user_id).await?, + } => deactivate(body, no_leave_rooms, user_id).await?, UserCommand::ResetPassword { username, } => reset_password(body, username).await?, UserCommand::DeactivateAll { - leave_rooms, + no_leave_rooms, force, - } => deactivate_all(body, leave_rooms, force).await?, + } => deactivate_all(body, no_leave_rooms, force).await?, UserCommand::ListJoinedRooms { user_id, } => list_joined_rooms(body, user_id).await?, diff --git a/src/admin/user/user_commands.rs b/src/admin/user/user_commands.rs index 9bd963dd..8b90894e 100644 --- a/src/admin/user/user_commands.rs +++ b/src/admin/user/user_commands.rs @@ -128,7 +128,7 @@ pub(crate) async fn create( } pub(crate) async fn deactivate( - _body: Vec<&str>, leave_rooms: bool, user_id: String, + _body: Vec<&str>, no_leave_rooms: bool, user_id: String, ) -> Result { // Validate user id let user_id = parse_local_user_id(&user_id)?; @@ -144,7 +144,7 @@ pub(crate) async fn deactivate( services().users.deactivate_account(&user_id)?; - if leave_rooms { + if !no_leave_rooms { services() .admin .send_message(RoomMessageEventContent::text_plain(format!( @@ -185,7 +185,9 @@ pub(crate) async fn reset_password(_body: Vec<&str>, username: String) -> Result } } -pub(crate) async fn deactivate_all(body: Vec<&str>, leave_rooms: bool, force: bool) -> Result { +pub(crate) async fn deactivate_all( + body: Vec<&str>, no_leave_rooms: bool, force: bool, +) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -245,7 +247,7 @@ pub(crate) async fn deactivate_all(body: Vec<&str>, leave_rooms: bool, force: bo match services().users.deactivate_account(&user_id) { Ok(()) => { deactivation_count = deactivation_count.saturating_add(1); - if leave_rooms || force { + if !no_leave_rooms { info!("Forcing user {user_id} to leave all rooms apart of deactivate-all"); leave_all_rooms(&user_id).await; } From 305dfc3b426e71c69612a2f12dcd255c4d0e8cde Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 22:54:55 -0400 Subject: [PATCH 0160/2091] adminroom: cmds to view room topic and room members this will be extended more Signed-off-by: strawberry --- src/admin/room/mod.rs | 24 +++++++ src/admin/room/room_info_commands.rs | 93 ++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 src/admin/room/room_info_commands.rs diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index ccc8c8be..ced96c95 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -7,6 +7,7 @@ use crate::Result; pub(crate) mod room_alias_commands; pub(crate) mod room_commands; pub(crate) mod room_directory_commands; +pub(crate) mod room_info_commands; pub(crate) mod room_moderation_commands; #[cfg_attr(test, derive(Debug))] @@ -17,6 +18,10 @@ pub(crate) enum RoomCommand { page: Option, }, + #[command(subcommand)] + /// - View information about a room we know about + Info(RoomInfoCommand), + #[command(subcommand)] /// - Manage moderation of remote or local rooms Moderation(RoomModerationCommand), @@ -30,6 +35,23 @@ pub(crate) enum RoomCommand { Directory(RoomDirectoryCommand), } +#[cfg_attr(test, derive(Debug))] +#[derive(Subcommand)] +pub(crate) enum RoomInfoCommand { + /// - List joined members in a room + ListJoinedMembers { + room_id: Box, + }, + + /// - Displays room topic + /// + /// Room topics can be huge, so this is in its + /// own separate command + ViewRoomTopic { + room_id: Box, + }, +} + #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] pub(crate) enum RoomAliasCommand { @@ -147,6 +169,8 @@ pub(crate) enum RoomModerationCommand { pub(crate) async fn process(command: RoomCommand, body: Vec<&str>) -> Result { Ok(match command { + RoomCommand::Info(command) => room_info_commands::process(command, body).await?, + RoomCommand::Alias(command) => room_alias_commands::process(command, body).await?, RoomCommand::Directory(command) => room_directory_commands::process(command, body).await?, diff --git a/src/admin/room/room_info_commands.rs b/src/admin/room/room_info_commands.rs new file mode 100644 index 00000000..0bcbc7ae --- /dev/null +++ b/src/admin/room/room_info_commands.rs @@ -0,0 +1,93 @@ +use std::fmt::Write; + +use ruma::{events::room::message::RoomMessageEventContent, RoomId}; +use service::services; + +use super::RoomInfoCommand; +use crate::{escape_html, Result}; + +pub(crate) async fn process(command: RoomInfoCommand, body: Vec<&str>) -> Result { + match command { + RoomInfoCommand::ListJoinedMembers { + room_id, + } => list_joined_members(body, room_id).await, + RoomInfoCommand::ViewRoomTopic { + room_id, + } => view_room_topic(body, room_id).await, + } +} + +async fn list_joined_members(_body: Vec<&str>, room_id: Box) -> Result { + let room_name = services() + .rooms + .state_accessor + .get_name(&room_id) + .ok() + .flatten() + .unwrap_or_else(|| room_id.to_string()); + + let members = services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(Result::ok); + + let member_info = members + .into_iter() + .map(|user_id| { + ( + user_id.clone(), + services() + .users + .displayname(&user_id) + .unwrap_or(None) + .unwrap_or_else(|| user_id.to_string()), + ) + }) + .collect::>(); + + let output_plain = format!( + "{} Members in Room \"{}\":\n```\n{}\n```", + member_info.len(), + room_name, + member_info + .iter() + .map(|(mxid, displayname)| format!("{mxid} | {displayname}")) + .collect::>() + .join("\n") + ); + + let output_html = format!( + "\n\t\n{}
    {} Members in Room \"{}\"
    MXIDDisplay \ + Name
    ", + member_info.len(), + room_name, + member_info + .iter() + .fold(String::new(), |mut output, (mxid, displayname)| { + writeln!( + output, + "{}\t{}", + mxid, + escape_html(displayname.as_ref()) + ) + .expect("should be able to write to string buffer"); + output + }) + ); + + Ok(RoomMessageEventContent::text_html(output_plain, output_html)) +} + +async fn view_room_topic(_body: Vec<&str>, room_id: Box) -> Result { + let Some(room_topic) = services().rooms.state_accessor.get_room_topic(&room_id)? else { + return Ok(RoomMessageEventContent::text_plain("Room does not have a room topic set.")); + }; + + let output_html = format!("

    Room topic:

    \n
    \n{}
    ", escape_html(&room_topic)); + + Ok(RoomMessageEventContent::text_html( + format!("Room topic:\n\n```{room_topic}\n```"), + output_html, + )) +} From 2e31bcc213d944af2201e3ee014819b2fc06c6ec Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 11 Jun 2024 01:18:49 -0400 Subject: [PATCH 0161/2091] use our fork of axum-server instead Signed-off-by: strawberry --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e238598..f8b1c5ef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,7 +302,7 @@ dependencies = [ [[package]] name = "axum-server" version = "0.6.0" -source = "git+https://github.com/lexe-app/axum-server?branch=phlip9/fix-graceful-shutdown#8e3368d899079818934e61cc9c839abcbbcada8a" +source = "git+https://github.com/girlbossceo/axum-server?branch=phlip9/fix-graceful-shutdown#8e3368d899079818934e61cc9c839abcbbcada8a" dependencies = [ "arc-swap", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 58e84a22..63528ef4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -429,7 +429,7 @@ branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" # fixes hyper graceful shutdowns [https://github.com/programatik29/axum-server/issues/114] [patch.crates-io.axum-server] -git = "https://github.com/lexe-app/axum-server" +git = "https://github.com/girlbossceo/axum-server" branch = "phlip9/fix-graceful-shutdown" # From 91519959ed0aa6f78088b03777740356c9163772 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 10 Jun 2024 23:55:41 -0400 Subject: [PATCH 0162/2091] mark account as deactivated before leaving rooms Signed-off-by: strawberry --- src/api/client/account.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index dc384c4c..db9498b7 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -547,12 +547,12 @@ pub(crate) async fn deactivate_route( return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } - // Make the user leave all rooms before deactivation - super::leave_all_rooms(sender_user).await; - // Remove devices and mark account as deactivated services().users.deactivate_account(sender_user)?; + // Make the user leave all rooms before deactivation + super::leave_all_rooms(sender_user).await; + info!("User {sender_user} deactivated their account."); services() .admin From 25d44cad312f27e353cbcbff21c604084f34acf5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 11 Jun 2024 00:42:54 -0400 Subject: [PATCH 0163/2091] slightly dedupe pfp/displayname updates, update blurhash in PDUs too Signed-off-by: strawberry --- src/api/client/mod.rs | 1 + src/api/client/profile.rs | 255 ++++++++++++++++++++------------------ src/service/users/data.rs | 2 +- src/service/users/mod.rs | 2 +- 4 files changed, 137 insertions(+), 123 deletions(-) diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index 6a6f16a7..1461be94 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -51,6 +51,7 @@ pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room, valida pub(super) use message::*; pub(super) use presence::*; pub(super) use profile::*; +pub use profile::{update_all_rooms, update_avatar_url, update_displayname}; pub(super) use push::*; pub(super) use read_marker::*; pub(super) use redact::*; diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 96cbe7fd..8a53b335 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -10,6 +10,7 @@ use ruma::{ }, events::{room::member::RoomMemberEventContent, StateEventType, TimelineEventType}, presence::PresenceState, + OwnedMxcUri, OwnedRoomId, OwnedUserId, }; use serde_json::value::to_raw_value; use tracing::warn; @@ -28,70 +29,14 @@ pub(crate) async fn set_displayname_route( body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - - services() - .users - .set_displayname(sender_user, body.displayname.clone()) - .await?; - - // Send a new membership event and presence update into all joined rooms - let all_rooms_joined: Vec<_> = services() + let all_joined_rooms: Vec = services() .rooms .state_cache .rooms_joined(sender_user) .filter_map(Result::ok) - .map(|room_id| { - Ok::<_, Error>(( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - displayname: body.displayname.clone(), - join_authorized_via_users_server: None, - ..serde_json::from_str( - services() - .rooms - .state_accessor - .room_state_get(&room_id, &StateEventType::RoomMember, sender_user.as_str())? - .ok_or_else(|| { - Error::bad_database("Tried to send displayname update for user not in the room.") - })? - .content - .get(), - ) - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_user.to_string()), - redacts: None, - }, - room_id, - )) - }) - .filter_map(Result::ok) .collect(); - for (pdu_builder, room_id) in all_rooms_joined { - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - - if let Err(e) = services() - .rooms - .timeline - .build_and_append_pdu(pdu_builder, sender_user, &room_id, &state_lock) - .await - { - warn!(%e, "Failed to update/send new display name in room"); - } - } + update_displayname(sender_user.clone(), body.displayname.clone(), all_joined_rooms).await?; if services().globals.allow_local_presence() { // Presence update @@ -168,75 +113,20 @@ pub(crate) async fn set_avatar_url_route( body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - - services() - .users - .set_avatar_url(sender_user, body.avatar_url.clone()) - .await?; - - services() - .users - .set_blurhash(sender_user, body.blurhash.clone()) - .await?; - - // Send a new membership event and presence update into all joined rooms - let all_joined_rooms: Vec<_> = services() + let all_joined_rooms: Vec = services() .rooms .state_cache .rooms_joined(sender_user) .filter_map(Result::ok) - .map(|room_id| { - Ok::<_, Error>(( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - avatar_url: body.avatar_url.clone(), - join_authorized_via_users_server: None, - ..serde_json::from_str( - services() - .rooms - .state_accessor - .room_state_get(&room_id, &StateEventType::RoomMember, sender_user.as_str())? - .ok_or_else(|| { - Error::bad_database("Tried to send displayname update for user not in the room.") - })? - .content - .get(), - ) - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_user.to_string()), - redacts: None, - }, - room_id, - )) - }) - .filter_map(Result::ok) .collect(); - for (pdu_builder, room_id) in all_joined_rooms { - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - - if let Err(e) = services() - .rooms - .timeline - .build_and_append_pdu(pdu_builder, sender_user, &room_id, &state_lock) - .await - { - warn!(%e, "Failed to set/update room with new avatar URL / pfp"); - } - } + update_avatar_url( + sender_user.clone(), + body.avatar_url.clone(), + body.blurhash.clone(), + all_joined_rooms, + ) + .await?; if services().globals.allow_local_presence() { // Presence update @@ -363,3 +253,126 @@ pub(crate) async fn get_profile_route(body: Ruma) -> R displayname: services().users.displayname(&body.user_id)?, }) } + +pub async fn update_displayname( + user_id: OwnedUserId, displayname: Option, all_joined_rooms: Vec, +) -> Result<()> { + services() + .users + .set_displayname(&user_id, displayname.clone()) + .await?; + + // Send a new join membership event into all joined rooms + let all_joined_rooms: Vec<_> = all_joined_rooms + .iter() + .map(|room_id| { + Ok::<_, Error>(( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + displayname: displayname.clone(), + join_authorized_via_users_server: None, + ..serde_json::from_str( + services() + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())? + .ok_or_else(|| { + Error::bad_database("Tried to send display name update for user not in the room.") + })? + .content + .get(), + ) + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + room_id, + )) + }) + .filter_map(Result::ok) + .collect(); + + update_all_rooms(all_joined_rooms, user_id).await; + + Ok(()) +} + +pub async fn update_avatar_url( + user_id: OwnedUserId, avatar_url: Option, blurhash: Option, all_joined_rooms: Vec, +) -> Result<()> { + services() + .users + .set_avatar_url(&user_id, avatar_url.clone()) + .await?; + services() + .users + .set_blurhash(&user_id, blurhash.clone()) + .await?; + + // Send a new join membership event into all joined rooms + let all_joined_rooms: Vec<_> = all_joined_rooms + .iter() + .map(|room_id| { + Ok::<_, Error>(( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + avatar_url: avatar_url.clone(), + blurhash: blurhash.clone(), + join_authorized_via_users_server: None, + ..serde_json::from_str( + services() + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())? + .ok_or_else(|| { + Error::bad_database("Tried to send avatar URL update for user not in the room.") + })? + .content + .get(), + ) + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + room_id, + )) + }) + .filter_map(Result::ok) + .collect(); + + update_all_rooms(all_joined_rooms, user_id).await; + + Ok(()) +} + +pub async fn update_all_rooms(all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: OwnedUserId) { + for (pdu_builder, room_id) in all_joined_rooms { + let mutex_state = Arc::clone( + services() + .globals + .roomid_mutex_state + .write() + .await + .entry(room_id.clone()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + if let Err(e) = services() + .rooms + .timeline + .build_and_append_pdu(pdu_builder, &user_id, room_id, &state_lock) + .await + { + warn!(%user_id, %room_id, %e, "Failed to update/send new profile join membership update in room"); + } + } +} diff --git a/src/service/users/data.rs b/src/service/users/data.rs index f2355755..5d3eadd8 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -307,7 +307,7 @@ impl Data for KeyValueDatabase { .transpose() } - /// Sets a new avatar_url or removes it if avatar_url is None. + /// Sets a new blurhash or removes it if blurhash is None. fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()> { if let Some(blurhash) = blurhash { self.userid_blurhash diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index ec17e796..b326078b 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -299,7 +299,7 @@ impl Service { /// Get the blurhash of a user. pub fn blurhash(&self, user_id: &UserId) -> Result> { self.db.blurhash(user_id) } - /// Sets a new avatar_url or removes it if avatar_url is None. + /// Sets a new blurhash or removes it if blurhash is None. pub async fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()> { self.db.set_blurhash(user_id, blurhash) } From ea03a50e2123d4a9cb8a9e421840681a71dfb037 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 11 Jun 2024 01:15:55 -0400 Subject: [PATCH 0164/2091] remove pfps/avatars and display names upon deactivation Signed-off-by: strawberry --- src/admin/user/user_commands.rs | 19 +++++++++- src/api/client/account.rs | 12 ++++++- src/api/client/membership.rs | 64 ++++++++++++++++++++++----------- 3 files changed, 72 insertions(+), 23 deletions(-) diff --git a/src/admin/user/user_commands.rs b/src/admin/user/user_commands.rs index 8b90894e..aba32417 100644 --- a/src/admin/user/user_commands.rs +++ b/src/admin/user/user_commands.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, fmt::Write as _}; -use api::client::{join_room_by_id_helper, leave_all_rooms}; +use api::client::{join_room_by_id_helper, leave_all_rooms, update_avatar_url, update_displayname}; use conduit::utils; use ruma::{ events::{ @@ -151,6 +151,15 @@ pub(crate) async fn deactivate( "Making {user_id} leave all rooms after deactivation..." ))) .await; + + let all_joined_rooms: Vec = services() + .rooms + .state_cache + .rooms_joined(&user_id) + .filter_map(Result::ok) + .collect(); + update_displayname(user_id.clone(), None, all_joined_rooms.clone()).await?; + update_avatar_url(user_id.clone(), None, None, all_joined_rooms).await?; leave_all_rooms(&user_id).await; } @@ -249,6 +258,14 @@ pub(crate) async fn deactivate_all( deactivation_count = deactivation_count.saturating_add(1); if !no_leave_rooms { info!("Forcing user {user_id} to leave all rooms apart of deactivate-all"); + let all_joined_rooms: Vec = services() + .rooms + .state_cache + .rooms_joined(&user_id) + .filter_map(Result::ok) + .collect(); + update_displayname(user_id.clone(), None, all_joined_rooms.clone()).await?; + update_avatar_url(user_id.clone(), None, None, all_joined_rooms).await?; leave_all_rooms(&user_id).await; } }, diff --git a/src/api/client/account.rs b/src/api/client/account.rs index db9498b7..5521bdff 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -15,7 +15,7 @@ use ruma::{ uiaa::{AuthFlow, AuthType, UiaaInfo}, }, events::{room::message::RoomMessageEventContent, GlobalAccountDataEventType}, - push, UserId, + push, OwnedRoomId, UserId, }; use tracing::{error, info, warn}; @@ -550,6 +550,16 @@ pub(crate) async fn deactivate_route( // Remove devices and mark account as deactivated services().users.deactivate_account(sender_user)?; + // Remove profile pictures and display name + let all_joined_rooms: Vec = services() + .rooms + .state_cache + .rooms_joined(sender_user) + .filter_map(Result::ok) + .collect(); + super::update_displayname(sender_user.clone(), None, all_joined_rooms.clone()).await?; + super::update_avatar_url(sender_user.clone(), None, None, all_joined_rooms).await?; + // Make the user leave all rooms before deactivation super::leave_all_rooms(sender_user).await; diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 79ea50bd..f781808c 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1,6 +1,7 @@ use std::{ cmp, collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, + net::IpAddr, sync::Arc, time::{Duration, Instant}, }; @@ -36,13 +37,12 @@ use tracing::{debug, error, info, trace, warn}; use super::get_alias_helper; use crate::{ + client::{update_avatar_url, update_displayname}, service::{ pdu::{gen_event_id_canonical_json, PduBuilder}, server_is_ours, user_is_local, }, - services, - utils::{self}, - Error, PduEvent, Result, Ruma, + services, utils, Error, PduEvent, Result, Ruma, }; /// Checks if the room is banned in any way possible and the sender user is not @@ -51,7 +51,9 @@ use crate::{ /// Performs automatic deactivation if `auto_deactivate_banned_room_attempts` is /// enabled #[tracing::instrument] -async fn banned_room_check(user_id: &UserId, room_id: Option<&RoomId>, server_name: Option<&ServerName>) -> Result<()> { +async fn banned_room_check( + user_id: &UserId, room_id: Option<&RoomId>, server_name: Option<&ServerName>, client_ip: IpAddr, +) -> Result<()> { if !services().users.is_admin(user_id)? { if let Some(room_id) = room_id { if services().rooms.metadata.is_banned(room_id)? @@ -63,7 +65,7 @@ async fn banned_room_check(user_id: &UserId, room_id: Option<&RoomId>, server_na { warn!( "User {user_id} who is not an admin attempted to send an invite for or attempted to join a banned \ - room or banned room server name: {room_id}." + room or banned room server name: {room_id}" ); if services() @@ -75,15 +77,25 @@ async fn banned_room_check(user_id: &UserId, room_id: Option<&RoomId>, server_na services() .admin .send_message(RoomMessageEventContent::text_plain(format!( - "Automatically deactivating user {user_id} due to attempted banned room join" + "Automatically deactivating user {user_id} due to attempted banned room join from IP \ + {client_ip}" ))) .await; - // ignore errors - leave_all_rooms(user_id).await; if let Err(e) = services().users.deactivate_account(user_id) { - warn!(%e, "Failed to deactivate account"); + warn!(%user_id, %e, "Failed to deactivate account"); } + + let all_joined_rooms: Vec = services() + .rooms + .state_cache + .rooms_joined(user_id) + .filter_map(Result::ok) + .collect(); + + update_displayname(user_id.into(), None, all_joined_rooms.clone()).await?; + update_avatar_url(user_id.into(), None, None, all_joined_rooms).await?; + leave_all_rooms(user_id).await; } return Err(Error::BadRequest( @@ -112,15 +124,25 @@ async fn banned_room_check(user_id: &UserId, room_id: Option<&RoomId>, server_na services() .admin .send_message(RoomMessageEventContent::text_plain(format!( - "Automatically deactivating user {user_id} due to attempted banned room join" + "Automatically deactivating user {user_id} due to attempted banned room join from IP \ + {client_ip}" ))) .await; - // ignore errors - leave_all_rooms(user_id).await; if let Err(e) = services().users.deactivate_account(user_id) { - warn!(%e, "Failed to deactivate account"); + warn!(%user_id, %e, "Failed to deactivate account"); } + + let all_joined_rooms: Vec = services() + .rooms + .state_cache + .rooms_joined(user_id) + .filter_map(Result::ok) + .collect(); + + update_displayname(user_id.into(), None, all_joined_rooms.clone()).await?; + update_avatar_url(user_id.into(), None, None, all_joined_rooms).await?; + leave_all_rooms(user_id).await; } return Err(Error::BadRequest( @@ -148,7 +170,7 @@ pub(crate) async fn join_room_by_id_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - banned_room_check(sender_user, Some(&body.room_id), body.room_id.server_name()).await?; + banned_room_check(sender_user, Some(&body.room_id), body.room_id.server_name(), client_ip).await?; // There is no body.server_name for /roomId/join let mut servers = services() @@ -204,7 +226,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { Ok(room_id) => { - banned_room_check(sender_user, Some(&room_id), room_id.server_name()).await?; + banned_room_check(sender_user, Some(&room_id), room_id.server_name(), client_ip).await?; let mut servers = body.server_name.clone(); servers.extend( @@ -238,7 +260,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( Err(room_alias) => { let response = get_alias_helper(room_alias.clone(), Some(body.server_name.clone())).await?; - banned_room_check(sender_user, Some(&response.room_id), Some(room_alias.server_name())).await?; + banned_room_check(sender_user, Some(&response.room_id), Some(room_alias.server_name()), client_ip).await?; let mut servers = body.server_name; servers.extend(response.servers); @@ -315,7 +337,7 @@ pub(crate) async fn invite_user_route( )); } - banned_room_check(sender_user, Some(&body.room_id), body.room_id.server_name()).await?; + banned_room_check(sender_user, Some(&body.room_id), body.room_id.server_name(), client_ip).await?; if let invite_user::v3::InvitationRecipient::UserId { user_id, @@ -1578,11 +1600,11 @@ pub async fn leave_all_rooms(user_id: &UserId) { }; // ignore errors - if let Err(e) = services().rooms.state_cache.forget(&room_id, user_id) { - warn!(%e, "Failed to forget room"); - } if let Err(e) = leave_room(user_id, &room_id, None).await { - warn!(%e, "Failed to leave room"); + warn!(%room_id, %user_id, %e, "Failed to leave room"); + } + if let Err(e) = services().rooms.state_cache.forget(&room_id, user_id) { + warn!(%room_id, %user_id, %e, "Failed to forget room"); } } } From 01a77f8a716bc93b0ddacc7ba8d7d294089fbc69 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 11 Jun 2024 12:37:18 -0400 Subject: [PATCH 0165/2091] add `replaces_state` and `prev_sender` in `unsigned` for membership changes Signed-off-by: strawberry --- src/service/rooms/timeline/mod.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index affe35ab..df23da8f 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -236,11 +236,19 @@ impl Service { "prev_content".to_owned(), CanonicalJsonValue::Object( utils::to_canonical_object(prev_state.content.clone()).map_err(|e| { - error!("Failed to convert prev_state to canonical JSON: {}", e); + error!("Failed to convert prev_state to canonical JSON: {e}"); Error::bad_database("Failed to convert prev_state to canonical JSON.") })?, ), ); + unsigned.insert( + String::from("prev_sender"), + CanonicalJsonValue::String(prev_state.sender.clone().to_string()), + ); + unsigned.insert( + String::from("replaces_state"), + CanonicalJsonValue::String(prev_state.event_id.clone().to_string()), + ); } } } else { @@ -660,6 +668,10 @@ impl Service { "prev_sender".to_owned(), serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"), ); + unsigned.insert( + "replaces_state".to_owned(), + serde_json::to_value(&prev_pdu.event_id).expect("EventId is valid json"), + ); } } From d5677b6ae7aae52372b398894f17139318149279 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 11 Jun 2024 18:35:48 -0400 Subject: [PATCH 0166/2091] bump cargo.lock due to yanked httparse Signed-off-by: strawberry --- Cargo.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8b1c5ef..64768b17 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1540,9 +1540,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.2" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f3935c160d00ac752e09787e6e6bfc26494c2183cc922f1bc678a60d4733bc2" +checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" [[package]] name = "httpdate" @@ -2071,9 +2071,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +checksum = "e9764018d143cc854c9f17f0b907de70f14393b1f502da6375dce70f00514eb3" dependencies = [ "cc", "libc", From 73da353e5283c76f9c4dba210eee3a4f60cf8ec1 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 12 Jun 2024 00:14:36 -0400 Subject: [PATCH 0167/2091] fix missing next_batch for search The previous code would fail to return next_batch if any of the events in the window were not visible to the user. It would also return an unnecessary next_batch when no more results are available if the total number of results is exactly `skip + limit`. This bug will become much more severe with a full filtering implementation, because we will be more likely to trigger it by filtering out events in a search call. Currently, it is only possible to trigger with rooms that have history visibility set to "invited" or "joined". Signed-off-by: strawberry --- src/api/client/search.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/api/client/search.rs b/src/api/client/search.rs index 2441f555..de98ec4e 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -166,11 +166,8 @@ pub(crate) async fn search_events_route(body: Ruma) .take(limit) .collect(); - let next_batch = if results.len() < limit { - None - } else { - Some(next_batch.to_string()) - }; + let more_unloaded_results = searches.iter_mut().any(|s| s.peek().is_some()); + let next_batch = more_unloaded_results.then(|| next_batch.to_string()); Ok(search_events::v3::Response::new(ResultCategories { room_events: ResultRoomEvents { From 81cd677b4e984d22ada504fa4121e0da3080bcce Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 12 Jun 2024 00:17:03 -0400 Subject: [PATCH 0168/2091] fix dropped events in search The previous code would drop some events entirely if any events between `skip` and `skip + limit` were not visible to the user. This would cause the set of events skipped by the `skip(skip)` method to extend past `skip` in the raw result set, because `skip(skip)` was being called *after* filtering out invisible events. This bug will become much more severe with a full filtering implementation, because it will be more likely for events to be filtered out. Currently, it is only possible to trigger with rooms that have history visibility set to "invited" or "joined". Signed-off-by: strawberry --- src/api/client/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/search.rs b/src/api/client/search.rs index de98ec4e..b9bf9b62 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -133,6 +133,7 @@ pub(crate) async fn search_events_route(body: Ruma) let results: Vec<_> = results .iter() + .skip(skip) .filter_map(|result| { services() .rooms @@ -162,7 +163,6 @@ pub(crate) async fn search_events_route(body: Ruma) }) }) .filter_map(Result::ok) - .skip(skip) .take(limit) .collect(); From 20a54aacd633363057371e6b5a4b2f954e51468c Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 12 Jun 2024 00:22:49 -0400 Subject: [PATCH 0169/2091] factor search tokenization out into a function This ensures that the tokenization algorithm will remain in sync between querying, indexing, and deindexing. The existing code had slightly different behavior for querying, because it did not discard words with >50 bytes. This was inconsequential, because >50 byte tokens are never present in the index. Signed-off-by: strawberry --- src/service/rooms/search/data.rs | 36 +++++++++++++++++--------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/service/rooms/search/data.rs b/src/service/rooms/search/data.rs index 091f190e..0e6d251a 100644 --- a/src/service/rooms/search/data.rs +++ b/src/service/rooms/search/data.rs @@ -10,20 +10,26 @@ pub trait Data: Send + Sync { fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a>; } +/// Splits a string into tokens used as keys in the search inverted index +/// +/// This may be used to tokenize both message bodies (for indexing) or search +/// queries (for querying). +fn tokenize(body: &str) -> impl Iterator + '_ { + body.split_terminator(|c: char| !c.is_alphanumeric()) + .filter(|s| !s.is_empty()) + .filter(|word| word.len() <= 50) + .map(str::to_lowercase) +} + impl Data for KeyValueDatabase { fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { - let mut batch = message_body - .split_terminator(|c: char| !c.is_alphanumeric()) - .filter(|s| !s.is_empty()) - .filter(|word| word.len() <= 50) - .map(str::to_lowercase) - .map(|word| { - let mut key = shortroomid.to_be_bytes().to_vec(); - key.extend_from_slice(word.as_bytes()); - key.push(0xFF); - key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here - (key, Vec::new()) - }); + let mut batch = tokenize(message_body).map(|word| { + let mut key = shortroomid.to_be_bytes().to_vec(); + key.extend_from_slice(word.as_bytes()); + key.push(0xFF); + key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here + (key, Vec::new()) + }); self.tokenids.insert_batch(&mut batch) } @@ -37,11 +43,7 @@ impl Data for KeyValueDatabase { .to_be_bytes() .to_vec(); - let words: Vec<_> = search_string - .split_terminator(|c: char| !c.is_alphanumeric()) - .filter(|s| !s.is_empty()) - .map(str::to_lowercase) - .collect(); + let words: Vec<_> = tokenize(search_string).collect(); let iterators = words.clone().into_iter().map(move |word| { let mut prefix2 = prefix.clone(); From eb73d8c669abbdb9ad104de9da08bb69895a9ab6 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 12 Jun 2024 00:33:12 -0400 Subject: [PATCH 0170/2091] fix: de-index pdus when redacted bit of code dedupe as well Co-authored-by: strawberry Signed-off-by: strawberry --- src/service/rooms/search/data.rs | 18 +++++++++++++++++ src/service/rooms/search/mod.rs | 5 +++++ src/service/rooms/timeline/mod.rs | 33 +++++++++++++++++++------------ 3 files changed, 43 insertions(+), 13 deletions(-) diff --git a/src/service/rooms/search/data.rs b/src/service/rooms/search/data.rs index 0e6d251a..7fa49989 100644 --- a/src/service/rooms/search/data.rs +++ b/src/service/rooms/search/data.rs @@ -7,6 +7,8 @@ type SearchPdusResult<'a> = Result> + 'a pub trait Data: Send + Sync { fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()>; + fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()>; + fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a>; } @@ -34,6 +36,22 @@ impl Data for KeyValueDatabase { self.tokenids.insert_batch(&mut batch) } + fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { + let batch = tokenize(message_body).map(|word| { + let mut key = shortroomid.to_be_bytes().to_vec(); + key.extend_from_slice(word.as_bytes()); + key.push(0xFF); + key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here + key + }); + + for token in batch { + self.tokenids.remove(&token)?; + } + + Ok(()) + } + fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a> { let prefix = services() .rooms diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 80ac45ae..9e7f14c7 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -17,6 +17,11 @@ impl Service { self.db.index_pdu(shortroomid, pdu_id, message_body) } + #[tracing::instrument(skip(self))] + pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { + self.db.deindex_pdu(shortroomid, pdu_id, message_body) + } + #[tracing::instrument(skip(self))] pub fn search_pdus<'a>( &'a self, room_id: &RoomId, search_string: &str, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index df23da8f..c7d045e9 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -68,6 +68,11 @@ struct ExtractRelatesToEventId { relates_to: ExtractEventId, } +#[derive(Deserialize)] +struct ExtractBody { + body: Option, +} + pub struct Service { pub db: Arc, @@ -397,7 +402,7 @@ impl Service { | RoomVersionId::V9 | RoomVersionId::V10 => { if let Some(redact_id) = &pdu.redacts { - self.redact_pdu(redact_id, pdu)?; + self.redact_pdu(redact_id, pdu, shortroomid)?; } }, RoomVersionId::V11 => { @@ -407,7 +412,7 @@ impl Service { Error::bad_database("Invalid content in redaction pdu.") })?; if let Some(redact_id) = &content.redacts { - self.redact_pdu(redact_id, pdu)?; + self.redact_pdu(redact_id, pdu, shortroomid)?; } }, _ => { @@ -463,11 +468,6 @@ impl Service { } }, TimelineEventType::RoomMessage => { - #[derive(Deserialize)] - struct ExtractBody { - body: Option, - } - let content = serde_json::from_str::(pdu.content.get()) .map_err(|_| Error::bad_database("Invalid content in pdu."))?; @@ -984,14 +984,26 @@ impl Service { /// Replace a PDU with the redacted form. #[tracing::instrument(skip(self, reason))] - pub fn redact_pdu(&self, event_id: &EventId, reason: &PduEvent) -> Result<()> { + pub fn redact_pdu(&self, event_id: &EventId, reason: &PduEvent, shortroomid: u64) -> Result<()> { // TODO: Don't reserialize, keep original json if let Some(pdu_id) = self.get_pdu_id(event_id)? { let mut pdu = self .get_pdu_from_id(&pdu_id)? .ok_or_else(|| Error::bad_database("PDU ID points to invalid PDU."))?; + + if let Ok(content) = serde_json::from_str::(pdu.content.get()) { + if let Some(body) = content.body { + services() + .rooms + .search + .deindex_pdu(shortroomid, &pdu_id, &body)?; + } + } + let room_version_id = services().rooms.state.get_room_version(&pdu.room_id)?; + pdu.redact(room_version_id, reason)?; + self.replace_pdu( &pdu_id, &utils::to_canonical_object(&pdu).map_err(|e| { @@ -1188,11 +1200,6 @@ impl Service { drop(insert_lock); if pdu.kind == TimelineEventType::RoomMessage { - #[derive(Deserialize)] - struct ExtractBody { - body: Option, - } - let content = serde_json::from_str::(pdu.content.get()) .map_err(|_| Error::bad_database("Invalid content in pdu."))?; From 0688a96c379f5ce6dad6e63f23076a0294cb5028 Mon Sep 17 00:00:00 2001 From: Benjamin Lee Date: Wed, 12 Jun 2024 00:36:38 -0400 Subject: [PATCH 0171/2091] drop redacted events from search results Signed-off-by: strawberry --- src/api/client/search.rs | 11 ++++++----- src/service/pdu.rs | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/api/client/search.rs b/src/api/client/search.rs index b9bf9b62..e88558fd 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -141,11 +141,12 @@ pub(crate) async fn search_events_route(body: Ruma) .get_pdu_from_id(result) .ok()? .filter(|pdu| { - services() - .rooms - .state_accessor - .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) - .unwrap_or(false) + !pdu.is_redacted() + && services() + .rooms + .state_accessor + .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) + .unwrap_or(false) }) .map(|pdu| pdu.to_room_event()) }) diff --git a/src/service/pdu.rs b/src/service/pdu.rs index 6274d635..135ae1fd 100644 --- a/src/service/pdu.rs +++ b/src/service/pdu.rs @@ -21,6 +21,11 @@ use tracing::warn; use crate::{services, Error}; +#[derive(Deserialize)] +struct ExtractRedactedBecause { + redacted_because: Option, +} + /// Content hashes of a PDU. #[derive(Clone, Debug, Deserialize, Serialize)] pub struct EventHash { @@ -76,6 +81,19 @@ impl PduEvent { Ok(()) } + #[must_use] + pub fn is_redacted(&self) -> bool { + let Some(unsigned) = &self.unsigned else { + return false; + }; + + let Ok(unsigned) = ExtractRedactedBecause::deserialize(&**unsigned) else { + return false; + }; + + unsigned.redacted_because.is_some() + } + pub fn remove_transaction_id(&mut self) -> crate::Result<()> { if let Some(unsigned) = &self.unsigned { let mut unsigned: BTreeMap> = serde_json::from_str(unsigned.get()) From 26d103d31486a932e24b4095eed546c12f142bf6 Mon Sep 17 00:00:00 2001 From: Matthias Ahouansou Date: Wed, 12 Jun 2024 00:40:45 -0400 Subject: [PATCH 0172/2091] fix: only process admin commands if server user is in the room Should prevent some edge cases with a misconfigured admin room Signed-off-by: strawberry --- src/service/rooms/timeline/mod.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index c7d045e9..82f5f1bf 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -489,7 +489,13 @@ impl Service { // the administrator can execute commands as conduit let from_conduit = pdu.sender == server_user && services().globals.emergency_password().is_none(); if let Some(admin_room) = service::admin::Service::get_admin_room().await? { - if to_conduit && !from_conduit && admin_room == pdu.room_id { + if to_conduit + && !from_conduit && admin_room == pdu.room_id + && services() + .rooms + .state_cache + .is_joined(&UserId::parse(server_user).unwrap(), &admin_room)? + { services() .admin .process_message(body, pdu.event_id.clone()) From f712c0cefb28681cd4fbd27b59761bcb24b19989 Mon Sep 17 00:00:00 2001 From: Matthias Ahouansou Date: Wed, 12 Jun 2024 01:42:39 -0400 Subject: [PATCH 0173/2091] fix: restrict who can remove aliases Previously, anyone could remove any local alias, meaning that someone could re-route a popular alias elsewhere Now, only the creator of the alias, users who can set canonical aliases for the room, server admins and the server user can delete aliases added some additional changes/fixes to adapt to our codebase Co-authored-by: strawberry Signed-off-by: strawberry --- src/admin/room/room_alias_commands.rs | 24 +++++++-- src/api/client/alias.rs | 29 ++++------- src/api/client/room.rs | 7 ++- src/database/kvdatabase.rs | 2 + src/service/admin.rs | 19 +++++-- src/service/rooms/alias/data.rs | 34 +++++++++++-- src/service/rooms/alias/mod.rs | 72 +++++++++++++++++++++++++-- 7 files changed, 151 insertions(+), 36 deletions(-) diff --git a/src/admin/room/room_alias_commands.rs b/src/admin/room/room_alias_commands.rs index 33c165b8..7312b0f6 100644 --- a/src/admin/room/room_alias_commands.rs +++ b/src/admin/room/room_alias_commands.rs @@ -1,11 +1,14 @@ use std::fmt::Write; -use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId}; +use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, UserId}; use super::RoomAliasCommand; use crate::{escape_html, services, Result}; pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Result { + let server_user = UserId::parse_with_server_name(String::from("conduit"), services().globals.server_name()) + .expect("server's username is valid"); + match command { RoomAliasCommand::Set { ref room_alias_localpart, @@ -28,7 +31,11 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu room_id, .. } => match (force, services().rooms.alias.resolve_local_alias(&room_alias)) { - (true, Ok(Some(id))) => match services().rooms.alias.set_alias(&room_alias, &room_id) { + (true, Ok(Some(id))) => match services() + .rooms + .alias + .set_alias(&room_alias, &room_id, &server_user) + { Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( "Successfully overwrote alias (formerly {id})" ))), @@ -37,7 +44,11 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu (false, Ok(Some(id))) => Ok(RoomMessageEventContent::text_plain(format!( "Refusing to overwrite in use alias for {id}, use -f or --force to overwrite" ))), - (_, Ok(None)) => match services().rooms.alias.set_alias(&room_alias, &room_id) { + (_, Ok(None)) => match services() + .rooms + .alias + .set_alias(&room_alias, &room_id, &server_user) + { Ok(()) => Ok(RoomMessageEventContent::text_plain("Successfully set alias")), Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), }, @@ -46,7 +57,12 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu RoomAliasCommand::Remove { .. } => match services().rooms.alias.resolve_local_alias(&room_alias) { - Ok(Some(id)) => match services().rooms.alias.remove_alias(&room_alias) { + Ok(Some(id)) => match services() + .rooms + .alias + .remove_alias(&room_alias, &server_user) + .await + { Ok(()) => Ok(RoomMessageEventContent::text_plain(format!("Removed alias from {id}"))), Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), }, diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index e70105f3..5752ea9b 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -22,6 +22,8 @@ use crate::{ /// /// Creates a new room alias on this server. pub(crate) async fn create_alias_route(body: Ruma) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + alias_checks(&body.room_alias, &body.appservice_info).await?; // this isn't apart of alias_checks or delete alias route because we should @@ -43,17 +45,10 @@ pub(crate) async fn create_alias_route(body: Ruma) -> return Err(Error::Conflict("Alias already exists.")); } - if services() + services() .rooms .alias - .set_alias(&body.room_alias, &body.room_id) - .is_err() - { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid room alias. Alias must be in the form of '#localpart:server_name'", - )); - }; + .set_alias(&body.room_alias, &body.room_id, sender_user)?; Ok(create_alias::v3::Response::new()) } @@ -62,10 +57,12 @@ pub(crate) async fn create_alias_route(body: Ruma) -> /// /// Deletes a room alias from this server. /// -/// - TODO: additional access control checks /// - TODO: Update canonical alias event pub(crate) async fn delete_alias_route(body: Ruma) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + alias_checks(&body.room_alias, &body.appservice_info).await?; + if services() .rooms .alias @@ -75,17 +72,11 @@ pub(crate) async fn delete_alias_route(body: Ruma) -> return Err(Error::BadRequest(ErrorKind::NotFound, "Alias does not exist.")); } - if services() + services() .rooms .alias - .remove_alias(&body.room_alias) - .is_err() - { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid room alias. Alias must be in the form of '#localpart:server_name'", - )); - }; + .remove_alias(&body.room_alias, sender_user) + .await?; // TODO: update alt_aliases? diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 0cea6ef7..14071d39 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -467,7 +467,10 @@ pub(crate) async fn create_room_route(body: Ruma) -> R // Homeserver specific stuff if let Some(alias) = alias { - services().rooms.alias.set_alias(&alias, &room_id)?; + services() + .rooms + .alias + .set_alias(&alias, &room_id, sender_user)?; } if body.visibility == room::Visibility::Public { @@ -787,7 +790,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> services() .rooms .alias - .set_alias(&alias, &replacement_room)?; + .set_alias(&alias, &replacement_room, sender_user)?; } // Get the old room power levels diff --git a/src/database/kvdatabase.rs b/src/database/kvdatabase.rs index ce3bf30f..ef34fec4 100644 --- a/src/database/kvdatabase.rs +++ b/src/database/kvdatabase.rs @@ -57,6 +57,7 @@ pub struct KeyValueDatabase { pub eventid_pduid: Arc, pub roomid_pduleaves: Arc, pub alias_roomid: Arc, + pub alias_userid: Arc, // UserId = AliasId (User who created the alias) pub aliasid_alias: Arc, // AliasId = RoomId + Count pub publicroomids: Arc, @@ -193,6 +194,7 @@ impl KeyValueDatabase { roomid_pduleaves: builder.open_tree("roomid_pduleaves")?, alias_roomid: builder.open_tree("alias_roomid")?, + alias_userid: builder.open_tree("alias_userid")?, aliasid_alias: builder.open_tree("aliasid_alias")?, publicroomids: builder.open_tree("publicroomids")?, diff --git a/src/service/admin.rs b/src/service/admin.rs index 63d6d5c6..37a07a58 100644 --- a/src/service/admin.rs +++ b/src/service/admin.rs @@ -72,8 +72,9 @@ impl Service { let Ok(Some(admin_room)) = Self::get_admin_room().await else { return Ok(()); }; - let server_name = services().globals.server_name(); - let server_user = UserId::parse(format!("@conduit:{server_name}")).expect("server's username is valid"); + + let server_user = UserId::parse_with_server_name(String::from("conduit"), services().globals.server_name()) + .expect("server's username is valid"); loop { debug_assert!(!receiver.is_closed(), "channel closed"); @@ -386,7 +387,10 @@ impl Service { ) .await?; - services().rooms.alias.set_alias(&alias, &room_id)?; + services() + .rooms + .alias + .set_alias(&alias, &room_id, &server_user)?; // 7. (ad-hoc) Disable room previews for everyone by default services() @@ -533,4 +537,13 @@ impl Service { Ok(()) } + + /// Checks whether a given user is an admin of this server + pub async fn user_is_admin(&self, user_id: &UserId) -> Result { + let Ok(Some(admin_room)) = Self::get_admin_room().await else { + return Ok(false); + }; + + services().rooms.state_cache.is_joined(user_id, &admin_room) + } } diff --git a/src/service/rooms/alias/data.rs b/src/service/rooms/alias/data.rs index 536f8228..e0f4335e 100644 --- a/src/service/rooms/alias/data.rs +++ b/src/service/rooms/alias/data.rs @@ -1,10 +1,10 @@ -use ruma::{api::client::error::ErrorKind, OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId}; +use ruma::{api::client::error::ErrorKind, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, UserId}; use crate::{services, utils, Error, KeyValueDatabase, Result}; pub trait Data: Send + Sync { /// Creates or updates the alias to the given room id. - fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId) -> Result<()>; + fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()>; /// Forgets about an alias. Returns an error if the alias did not exist. fn remove_alias(&self, alias: &RoomAliasId) -> Result<()>; @@ -12,6 +12,9 @@ pub trait Data: Send + Sync { /// Looks up the roomid for the given alias. fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result>; + /// Finds the user who assigned the given alias to a room + fn who_created_alias(&self, alias: &RoomAliasId) -> Result>; + /// Returns all local aliases that point to the given room fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, @@ -22,13 +25,19 @@ pub trait Data: Send + Sync { } impl Data for KeyValueDatabase { - fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId) -> Result<()> { + fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { + // Comes first as we don't want a stuck alias + self.alias_userid + .insert(alias.alias().as_bytes(), user_id.as_bytes())?; + self.alias_roomid .insert(alias.alias().as_bytes(), room_id.as_bytes())?; + let mut aliasid = room_id.as_bytes().to_vec(); aliasid.push(0xFF); aliasid.extend_from_slice(&services().globals.next_count()?.to_be_bytes()); self.aliasid_alias.insert(&aliasid, alias.as_bytes())?; + Ok(()) } @@ -40,10 +49,14 @@ impl Data for KeyValueDatabase { for (key, _) in self.aliasid_alias.scan_prefix(prefix) { self.aliasid_alias.remove(&key)?; } + self.alias_roomid.remove(alias.alias().as_bytes())?; + + self.alias_userid.remove(alias.alias().as_bytes())?; } else { - return Err(Error::BadRequest(ErrorKind::NotFound, "Alias does not exist.")); + return Err(Error::BadRequest(ErrorKind::NotFound, "Alias does not exist or is invalid.")); } + Ok(()) } @@ -60,6 +73,19 @@ impl Data for KeyValueDatabase { .transpose() } + fn who_created_alias(&self, alias: &RoomAliasId) -> Result> { + self.alias_userid + .get(alias.alias().as_bytes())? + .map(|bytes| { + UserId::parse( + utils::string_from_bytes(&bytes) + .map_err(|_| Error::bad_database("User ID in alias_userid is invalid unicode."))?, + ) + .map_err(|_| Error::bad_database("User ID in alias_roomid is invalid.")) + }) + .transpose() + } + fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index f8d10b45..311456fe 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -3,9 +3,16 @@ mod data; use std::sync::Arc; use data::Data; -use ruma::{OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId}; +use ruma::{ + api::client::error::ErrorKind, + events::{ + room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, + StateEventType, + }, + OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId, UserId, +}; -use crate::Result; +use crate::{services, Error, Result}; pub struct Service { pub db: Arc, @@ -13,10 +20,21 @@ pub struct Service { impl Service { #[tracing::instrument(skip(self))] - pub fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId) -> Result<()> { self.db.set_alias(alias, room_id) } + pub fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { + self.db.set_alias(alias, room_id, user_id) + } #[tracing::instrument(skip(self))] - pub fn remove_alias(&self, alias: &RoomAliasId) -> Result<()> { self.db.remove_alias(alias) } + pub async fn remove_alias(&self, alias: &RoomAliasId, user_id: &UserId) -> Result<()> { + if self.user_can_remove_alias(alias, user_id).await? { + self.db.remove_alias(alias) + } else { + Err(Error::BadRequest( + ErrorKind::forbidden(), + "User is not permitted to remove this alias.", + )) + } + } #[tracing::instrument(skip(self))] pub fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { @@ -34,4 +52,50 @@ impl Service { pub fn all_local_aliases<'a>(&'a self) -> Box> + 'a> { self.db.all_local_aliases() } + + async fn user_can_remove_alias(&self, alias: &RoomAliasId, user_id: &UserId) -> Result { + let Some(room_id) = self.resolve_local_alias(alias)? else { + return Err(Error::BadRequest(ErrorKind::NotFound, "Alias not found.")); + }; + + let server_user = + UserId::parse_with_server_name(String::from("conduit"), services().globals.server_name()).unwrap(); + + // The creator of an alias can remove it + if self + .db + .who_created_alias(alias)? + .is_some_and(|user| user == user_id) + // Server admins can remove any local alias + || services().admin.user_is_admin(user_id).await? + // Always allow the server service account to remove the alias, since there may not be an admin room + || server_user == user_id + { + Ok(true) + // Checking whether the user is able to change canonical aliases of the + // room + } else if let Some(event) = + services() + .rooms + .state_accessor + .room_state_get(&room_id, &StateEventType::RoomPowerLevels, "")? + { + serde_json::from_str(event.content.get()) + .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) + .map(|content: RoomPowerLevelsEventContent| { + RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomCanonicalAlias) + }) + // If there is no power levels event, only the room creator can change + // canonical aliases + } else if let Some(event) = + services() + .rooms + .state_accessor + .room_state_get(&room_id, &StateEventType::RoomCreate, "")? + { + Ok(event.sender == user_id) + } else { + Err(Error::bad_database("Room has no m.room.create event")) + } + } } From 8fff7ea706605eea89953052971a4375652c7606 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 12 Jun 2024 02:10:59 -0400 Subject: [PATCH 0174/2091] cleanup+refactor admin room alias and server account accessing to globals Signed-off-by: strawberry --- src/admin/room/room_alias_commands.rs | 11 ++-- src/admin/room/room_moderation_commands.rs | 16 +++--- src/admin/user/user_commands.rs | 15 ++---- src/api/client/account.rs | 2 +- src/api/client/state.rs | 4 +- src/service/admin.rs | 61 +++++++++------------- src/service/globals/emerg_access.rs | 10 ++-- src/service/globals/migrations.rs | 5 +- src/service/globals/mod.rs | 10 +++- src/service/rooms/alias/mod.rs | 3 +- src/service/rooms/timeline/mod.rs | 16 +++--- src/service/users/mod.rs | 25 ++++----- 12 files changed, 77 insertions(+), 101 deletions(-) diff --git a/src/admin/room/room_alias_commands.rs b/src/admin/room/room_alias_commands.rs index 7312b0f6..bde13e53 100644 --- a/src/admin/room/room_alias_commands.rs +++ b/src/admin/room/room_alias_commands.rs @@ -1,13 +1,12 @@ use std::fmt::Write; -use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, UserId}; +use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId}; use super::RoomAliasCommand; use crate::{escape_html, services, Result}; pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Result { - let server_user = UserId::parse_with_server_name(String::from("conduit"), services().globals.server_name()) - .expect("server's username is valid"); + let server_user = &services().globals.server_user; match command { RoomAliasCommand::Set { @@ -34,7 +33,7 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu (true, Ok(Some(id))) => match services() .rooms .alias - .set_alias(&room_alias, &room_id, &server_user) + .set_alias(&room_alias, &room_id, server_user) { Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( "Successfully overwrote alias (formerly {id})" @@ -47,7 +46,7 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu (_, Ok(None)) => match services() .rooms .alias - .set_alias(&room_alias, &room_id, &server_user) + .set_alias(&room_alias, &room_id, server_user) { Ok(()) => Ok(RoomMessageEventContent::text_plain("Successfully set alias")), Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), @@ -60,7 +59,7 @@ pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu Ok(Some(id)) => match services() .rooms .alias - .remove_alias(&room_alias, &server_user) + .remove_alias(&room_alias, server_user) .await { Ok(()) => Ok(RoomMessageEventContent::text_plain(format!("Removed alias from {id}"))), diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 9886756b..5989ff73 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -33,12 +33,10 @@ async fn ban_room( ) -> Result { debug!("Got room alias or ID: {}", room); - let admin_room_alias: Box = format!("#admins:{}", services().globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid alias name"); + let admin_room_alias = &services().globals.admin_alias; - if let Some(admin_room_id) = Service::get_admin_room().await? { - if room.to_string().eq(&admin_room_id) || room.to_string().eq(&admin_room_alias) { + if let Some(admin_room_id) = Service::get_admin_room()? { + if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) { return Ok(RoomMessageEventContent::text_plain("Not allowed to ban the admin room.")); } } @@ -192,9 +190,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo let rooms_s = body.clone().drain(1..body.len() - 1).collect::>(); - let admin_room_alias: Box = format!("#admins:{}", services().globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid alias name"); + let admin_room_alias = &services().globals.admin_alias; let mut room_ban_count: usize = 0; let mut room_ids: Vec = Vec::new(); @@ -202,8 +198,8 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo for &room in &rooms_s { match <&RoomOrAliasId>::try_from(room) { Ok(room_alias_or_id) => { - if let Some(admin_room_id) = Service::get_admin_room().await? { - if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(&admin_room_alias) { + if let Some(admin_room_id) = Service::get_admin_room()? { + if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias) { info!("User specified admin room in bulk ban list, ignoring"); continue; } diff --git a/src/admin/user/user_commands.rs b/src/admin/user/user_commands.rs index aba32417..cada5022 100644 --- a/src/admin/user/user_commands.rs +++ b/src/admin/user/user_commands.rs @@ -8,7 +8,7 @@ use ruma::{ tag::{TagEvent, TagEventContent, TagInfo}, RoomAccountDataEventType, }, - OwnedRoomId, OwnedUserId, RoomId, UserId, + OwnedRoomId, OwnedUserId, RoomId, }; use tracing::{error, info, warn}; @@ -134,9 +134,7 @@ pub(crate) async fn deactivate( let user_id = parse_local_user_id(&user_id)?; // don't deactivate the server service account - if user_id - == UserId::parse_with_server_name("conduit", services().globals.server_name()).expect("conduit user exists") - { + if user_id == services().globals.server_user { return Ok(RoomMessageEventContent::text_plain( "Not allowed to deactivate the server service account.", )); @@ -171,9 +169,7 @@ pub(crate) async fn deactivate( pub(crate) async fn reset_password(_body: Vec<&str>, username: String) -> Result { let user_id = parse_local_user_id(&username)?; - if user_id - == UserId::parse_with_server_name("conduit", services().globals.server_name()).expect("conduit user exists") - { + if user_id == services().globals.server_user { return Ok(RoomMessageEventContent::text_plain( "Not allowed to set the password for the server account. Please use the emergency password config option.", )); @@ -223,10 +219,7 @@ pub(crate) async fn deactivate_all( } // don't deactivate the server service account - if user_id - == UserId::parse_with_server_name("conduit", services().globals.server_name()) - .expect("server user exists") - { + if user_id == services().globals.server_user { services() .admin .send_message(RoomMessageEventContent::text_plain(format!( diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 5521bdff..d7868e21 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -347,7 +347,7 @@ pub(crate) async fn register_route( // If this is the first real user, grant them admin privileges except for guest // users Note: the server user, @conduit:servername, is generated first if !is_guest { - if let Some(admin_room) = service::admin::Service::get_admin_room().await? { + if let Some(admin_room) = service::admin::Service::get_admin_room()? { if services() .rooms .state_cache diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 7445cc40..32e0fb68 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -216,7 +216,7 @@ async fn allowed_to_send_state_event( }, // admin room is a sensitive room, it should not ever be made public StateEventType::RoomJoinRules => { - if let Some(admin_room_id) = service::admin::Service::get_admin_room().await? { + if let Some(admin_room_id) = service::admin::Service::get_admin_room()? { if admin_room_id == room_id { if let Ok(join_rule) = serde_json::from_str::(json.json().get()) { if join_rule.join_rule == JoinRule::Public { @@ -231,7 +231,7 @@ async fn allowed_to_send_state_event( }, // admin room is a sensitive room, it should not ever be made world readable StateEventType::RoomHistoryVisibility => { - if let Some(admin_room_id) = service::admin::Service::get_admin_room().await? { + if let Some(admin_room_id) = service::admin::Service::get_admin_room()? { if admin_room_id == room_id { if let Ok(visibility_content) = serde_json::from_str::(json.json().get()) diff --git a/src/service/admin.rs b/src/service/admin.rs index 37a07a58..6c2eb8df 100644 --- a/src/service/admin.rs +++ b/src/service/admin.rs @@ -19,7 +19,7 @@ use ruma::{ }, TimelineEventType, }, - EventId, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, RoomVersionId, UserId, + EventId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; use tokio::{sync::Mutex, task::JoinHandle}; @@ -69,18 +69,17 @@ impl Service { async fn handler(self: &Arc) -> Result<()> { let receiver = self.receiver.lock().await; - let Ok(Some(admin_room)) = Self::get_admin_room().await else { + let Ok(Some(admin_room)) = Self::get_admin_room() else { return Ok(()); }; - let server_user = UserId::parse_with_server_name(String::from("conduit"), services().globals.server_name()) - .expect("server's username is valid"); + let server_user = &services().globals.server_user; loop { debug_assert!(!receiver.is_closed(), "channel closed"); tokio::select! { event = receiver.recv_async() => match event { - Ok(event) => self.receive(event, &admin_room, &server_user).await?, + Ok(event) => self.receive(event, &admin_room, server_user).await?, Err(_e) => return Ok(()), } } @@ -130,15 +129,11 @@ impl Service { /// /// Errors are propagated from the database, and will have None if there is /// no admin room - pub async fn get_admin_room() -> Result> { - let admin_room_alias: Box = format!("#admins:{}", services().globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid alias name"); - + pub fn get_admin_room() -> Result> { services() .rooms .alias - .resolve_local_alias(&admin_room_alias) + .resolve_local_alias(&services().globals.admin_alias) } /// Create the admin room. @@ -162,10 +157,9 @@ impl Service { let state_lock = mutex_state.lock().await; // Create a user for the server - let server_user = UserId::parse_with_server_name("conduit", services().globals.server_name()) - .expect("@conduit:server_name is valid"); + let server_user = &services().globals.server_user; - services().users.create(&server_user, None)?; + services().users.create(server_user, None)?; let room_version = services().globals.default_room_version(); let mut content = match room_version { @@ -205,7 +199,7 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -233,7 +227,7 @@ impl Service { state_key: Some(server_user.to_string()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -258,7 +252,7 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -277,7 +271,7 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -296,7 +290,7 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -315,7 +309,7 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -335,7 +329,7 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -355,16 +349,14 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) .await?; // 6. Room alias - let alias: OwnedRoomAliasId = format!("#admins:{}", services().globals.server_name()) - .try_into() - .expect("#admins:server_name is a valid alias name"); + let alias = &services().globals.admin_alias; services() .rooms @@ -381,7 +373,7 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -390,7 +382,7 @@ impl Service { services() .rooms .alias - .set_alias(&alias, &room_id, &server_user)?; + .set_alias(alias, &room_id, server_user)?; // 7. (ad-hoc) Disable room previews for everyone by default services() @@ -407,7 +399,7 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -420,7 +412,7 @@ impl Service { /// /// In conduit, this is equivalent to granting admin privileges. pub async fn make_user_admin(&self, user_id: &UserId, displayname: String) -> Result<()> { - if let Some(room_id) = Self::get_admin_room().await? { + if let Some(room_id) = Self::get_admin_room()? { let mutex_state = Arc::clone( services() .globals @@ -433,8 +425,7 @@ impl Service { let state_lock = mutex_state.lock().await; // Use the server user to grant the new admin's power level - let server_user = UserId::parse_with_server_name("conduit", services().globals.server_name()) - .expect("@conduit:server_name is valid"); + let server_user = &services().globals.server_user; // Invite and join the real user services() @@ -458,7 +449,7 @@ impl Service { state_key: Some(user_id.to_string()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -510,7 +501,7 @@ impl Service { state_key: Some(String::new()), redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ) @@ -529,7 +520,7 @@ impl Service { state_key: None, redacts: None, }, - &server_user, + server_user, &room_id, &state_lock, ).await?; @@ -540,7 +531,7 @@ impl Service { /// Checks whether a given user is an admin of this server pub async fn user_is_admin(&self, user_id: &UserId) -> Result { - let Ok(Some(admin_room)) = Self::get_admin_room().await else { + let Ok(Some(admin_room)) = Self::get_admin_room() else { return Ok(false); }; diff --git a/src/service/globals/emerg_access.rs b/src/service/globals/emerg_access.rs index fdf803cf..001c3e51 100644 --- a/src/service/globals/emerg_access.rs +++ b/src/service/globals/emerg_access.rs @@ -2,7 +2,6 @@ use conduit::Result; use ruma::{ events::{push_rules::PushRulesEventContent, GlobalAccountDataEvent, GlobalAccountDataEventType}, push::Ruleset, - UserId, }; use tracing::{error, warn}; @@ -18,21 +17,20 @@ pub(crate) fn init_emergency_access() { /// Sets the emergency password and push rules for the @conduit account in case /// emergency password is set fn set_emergency_access() -> Result { - let conduit_user = UserId::parse_with_server_name("conduit", services().globals.server_name()) - .expect("@conduit:server_name is a valid UserId"); + let conduit_user = &services().globals.server_user; services() .users - .set_password(&conduit_user, services().globals.emergency_password().as_deref())?; + .set_password(conduit_user, services().globals.emergency_password().as_deref())?; let (ruleset, pwd_set) = match services().globals.emergency_password() { - Some(_) => (Ruleset::server_default(&conduit_user), true), + Some(_) => (Ruleset::server_default(conduit_user), true), None => (Ruleset::new(), false), }; services().account_data.update( None, - &conduit_user, + conduit_user, GlobalAccountDataEventType::PushRules.to_string().into(), &serde_json::to_value(&GlobalAccountDataEvent { content: PushRulesEventContent { diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index d8936de7..83b3be99 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -22,10 +22,9 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result // Matrix resource ownership is based on the server name; changing it // requires recreating the database from scratch. if services().users.count()? > 0 { - let conduit_user = - UserId::parse_with_server_name("conduit", &config.server_name).expect("@conduit:server_name is valid"); + let conduit_user = &services().globals.server_user; - if !services().users.exists(&conduit_user)? { + if !services().users.exists(conduit_user)? { error!("The {} server user does not exist, and the database is not new.", conduit_user); return Err(Error::bad_database( "Cannot reuse an existing database after changing the server name, please delete the old one first.", diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 9327ecaa..317a0d56 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -24,8 +24,8 @@ use ruma::{ federation::discovery::{ServerSigningKeys, VerifyKey}, }, serde::Base64, - DeviceId, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomVersionId, - ServerName, UserId, + DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, + RoomAliasId, RoomVersionId, ServerName, UserId, }; use tokio::{ sync::{Mutex, RwLock}, @@ -58,6 +58,8 @@ pub struct Service { pub roomid_federationhandletime: RwLock>, pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, + pub server_user: OwnedUserId, + pub admin_alias: OwnedRoomAliasId, } impl Service { @@ -118,6 +120,10 @@ impl Service { roomid_federationhandletime: RwLock::new(HashMap::new()), updates_handle: Mutex::new(None), stateres_mutex: Arc::new(Mutex::new(())), + admin_alias: RoomAliasId::parse(format!("#admins:{}", &config.server_name)) + .expect("#admins:server_name is valid alias name"), + server_user: UserId::parse_with_server_name(String::from("conduit"), &config.server_name) + .expect("@conduit:server_name is valid"), }; fs::create_dir_all(s.get_media_folder())?; diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 311456fe..18d513a9 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -58,8 +58,7 @@ impl Service { return Err(Error::BadRequest(ErrorKind::NotFound, "Alias not found.")); }; - let server_user = - UserId::parse_with_server_name(String::from("conduit"), services().globals.server_name()).unwrap(); + let server_user = &services().globals.server_user; // The creator of an alias can remove it if self diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 82f5f1bf..1addd567 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -477,24 +477,24 @@ impl Service { .search .index_pdu(shortroomid, &pdu_id, &body)?; - let server_user = format!("@conduit:{}", services().globals.server_name()); + let server_user = &services().globals.server_user; let to_conduit = body.starts_with(&format!("{server_user}: ")) || body.starts_with(&format!("{server_user} ")) || body.starts_with("!admin") || body == format!("{server_user}:") - || body == server_user; + || body == *server_user; // This will evaluate to false if the emergency password is set up so that // the administrator can execute commands as conduit - let from_conduit = pdu.sender == server_user && services().globals.emergency_password().is_none(); - if let Some(admin_room) = service::admin::Service::get_admin_room().await? { + let from_conduit = pdu.sender == *server_user && services().globals.emergency_password().is_none(); + if let Some(admin_room) = service::admin::Service::get_admin_room()? { if to_conduit && !from_conduit && admin_room == pdu.room_id && services() .rooms .state_cache - .is_joined(&UserId::parse(server_user).unwrap(), &admin_room)? + .is_joined(server_user, &admin_room)? { services() .admin @@ -795,7 +795,7 @@ impl Service { state_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex ) -> Result> { let (pdu, pdu_json) = self.create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock)?; - if let Some(admin_room) = service::admin::Service::get_admin_room().await? { + if let Some(admin_room) = service::admin::Service::get_admin_room()? { if admin_room == room_id { match pdu.event_type() { TimelineEventType::RoomEncryption => { @@ -810,8 +810,8 @@ impl Service { .state_key() .filter(|v| v.starts_with('@')) .unwrap_or(sender.as_str()); - let server_name = services().globals.server_name(); - let server_user = format!("@conduit:{server_name}"); + let server_user = &services().globals.server_user.to_string(); + let content = serde_json::from_str::(pdu.content.get()) .map_err(|_| Error::bad_database("Invalid content in pdu."))?; diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index b326078b..c3de86a0 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -9,7 +9,6 @@ use data::Data; use ruma::{ api::client::{ device::Device, - error::ErrorKind, filter::FilterDefinition, sync::sync_events::{ self, @@ -20,10 +19,10 @@ use ruma::{ events::AnyToDeviceEvent, serde::Raw, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, OwnedDeviceId, OwnedDeviceKeyId, OwnedMxcUri, OwnedRoomId, OwnedUserId, - RoomAliasId, UInt, UserId, + UInt, UserId, }; -use crate::{services, Error, Result}; +use crate::{service, services, Error, Result}; pub struct SlidingSyncCache { lists: BTreeMap, @@ -234,18 +233,14 @@ impl Service { /// Check if a user is an admin pub fn is_admin(&self, user_id: &UserId) -> Result { - let admin_room_alias_id = RoomAliasId::parse(format!("#admins:{}", services().globals.server_name())) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?; - let admin_room_id = services() - .rooms - .alias - .resolve_local_alias(&admin_room_alias_id)? - .unwrap(); - - services() - .rooms - .state_cache - .is_joined(user_id, &admin_room_id) + if let Some(admin_room_id) = service::admin::Service::get_admin_room()? { + services() + .rooms + .state_cache + .is_joined(user_id, &admin_room_id) + } else { + Ok(false) + } } /// Create a new user account on this homeserver. From 556e78214a1fd70ebd65a3905cae580e0760c202 Mon Sep 17 00:00:00 2001 From: Matthias Ahouansou Date: Wed, 12 Jun 2024 02:13:27 -0400 Subject: [PATCH 0175/2091] fix: only allow the server user to set the admin alias Should make it safer to move the alias if the admin room broke on a public server. Signed-off-by: strawberry --- src/service/rooms/alias/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 18d513a9..80f125dc 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -21,7 +21,14 @@ pub struct Service { impl Service { #[tracing::instrument(skip(self))] pub fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { - self.db.set_alias(alias, room_id, user_id) + if alias == services().globals.admin_alias && user_id != services().globals.server_user { + Err(Error::BadRequest( + ErrorKind::forbidden(), + "Only the server user can set this alias", + )) + } else { + self.db.set_alias(alias, room_id, user_id) + } } #[tracing::instrument(skip(self))] From d0069cc100ea0789295d7d00fdf9a7910822c187 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 12 Jun 2024 02:28:52 -0400 Subject: [PATCH 0176/2091] log out any sessions when the server emergency password is unset Signed-off-by: strawberry --- src/service/globals/emerg_access.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/service/globals/emerg_access.rs b/src/service/globals/emerg_access.rs index 001c3e51..50c5f8c3 100644 --- a/src/service/globals/emerg_access.rs +++ b/src/service/globals/emerg_access.rs @@ -42,9 +42,12 @@ fn set_emergency_access() -> Result { if pwd_set { warn!( - "The Conduit account emergency password is set! Please unset it as soon as you finish admin account \ - recovery!" + "The server account emergency password is set! Please unset it as soon as you finish admin account \ + recovery! You will be logged out of the server service account when you finish." ); + } else { + // logs out any users still in the server service account and removes sessions + services().users.deactivate_account(conduit_user)?; } Ok(pwd_set) From 2ab3231ea6fb1333c6cd0424c51ff579deab839c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 8 Jun 2024 02:17:12 +0000 Subject: [PATCH 0177/2091] add tracing_subscriber::filter::ParseError to amalgam Signed-off-by: Jason Volk --- src/core/error.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/error.rs b/src/core/error.rs index 7049571a..133ffb1c 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -81,6 +81,8 @@ pub enum Error { #[error("{0} in {1}")] InconsistentRoomState(&'static str, ruma::OwnedRoomId), #[error("{0}")] + TracingFilter(#[from] tracing_subscriber::filter::ParseError), + #[error("{0}")] AdminCommand(&'static str), #[error("{0}")] Err(String), From 39110ebc3cef00027da4806886e4fc1ef210e188 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 11 Jun 2024 00:16:02 +0000 Subject: [PATCH 0178/2091] add fmt::Error as core Error source Signed-off-by: Jason Volk --- src/core/error.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/error.rs b/src/core/error.rs index 133ffb1c..3aead9d5 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -85,6 +85,8 @@ pub enum Error { #[error("{0}")] AdminCommand(&'static str), #[error("{0}")] + Fmt(#[from] fmt::Error), + #[error("{0}")] Err(String), } From 029e1c630a597a0ba67ea2906c7f297294e55981 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 8 Jun 2024 01:13:20 +0000 Subject: [PATCH 0179/2091] split core log into directory Signed-off-by: Jason Volk --- src/core/log/mod.rs | 33 ++++++++++++++++++++++++++++++ src/core/{log.rs => log/reload.rs} | 26 +---------------------- 2 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 src/core/log/mod.rs rename src/core/{log.rs => log/reload.rs} (79%) diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs new file mode 100644 index 00000000..aac25ec2 --- /dev/null +++ b/src/core/log/mod.rs @@ -0,0 +1,33 @@ +mod reload; + +pub use reload::ReloadHandle; +pub use reload::LogLevelReloadHandles; + +// Wraps for logging macros. Use these macros rather than extern tracing:: or log:: crates in +// project code. ::log and ::tracing can still be used if necessary but discouraged. Remember +// debug_ log macros are also exported to the crate namespace like these. + +#[macro_export] +macro_rules! error { + ( $($x:tt)+ ) => { tracing::error!( $($x)+ ); } +} + +#[macro_export] +macro_rules! warn { + ( $($x:tt)+ ) => { tracing::warn!( $($x)+ ); } +} + +#[macro_export] +macro_rules! info { + ( $($x:tt)+ ) => { tracing::info!( $($x)+ ); } +} + +#[macro_export] +macro_rules! debug { + ( $($x:tt)+ ) => { tracing::debug!( $($x)+ ); } +} + +#[macro_export] +macro_rules! trace { + ( $($x:tt)+ ) => { tracing::trace!( $($x)+ ); } +} diff --git a/src/core/log.rs b/src/core/log/reload.rs similarity index 79% rename from src/core/log.rs rename to src/core/log/reload.rs index eae25eb0..7646254e 100644 --- a/src/core/log.rs +++ b/src/core/log/reload.rs @@ -49,31 +49,7 @@ impl LogLevelReloadHandles { for handle in &self.inner.handles { handle.reload(new_value.clone())?; } + Ok(()) } } - -#[macro_export] -macro_rules! error { - ( $($x:tt)+ ) => { tracing::error!( $($x)+ ); } -} - -#[macro_export] -macro_rules! warn { - ( $($x:tt)+ ) => { tracing::warn!( $($x)+ ); } -} - -#[macro_export] -macro_rules! info { - ( $($x:tt)+ ) => { tracing::info!( $($x)+ ); } -} - -#[macro_export] -macro_rules! debug { - ( $($x:tt)+ ) => { tracing::debug!( $($x)+ ); } -} - -#[macro_export] -macro_rules! trace { - ( $($x:tt)+ ) => { tracing::trace!( $($x)+ ); } -} From 877c04de52f4223c6e1e21d8ba60cb1117a4b2b9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 11 Jun 2024 00:14:53 +0000 Subject: [PATCH 0180/2091] add color utils to log suite Signed-off-by: Jason Volk --- src/core/log/color.rs | 27 +++++++++++++++++++++++++++ src/core/log/mod.rs | 1 + 2 files changed, 28 insertions(+) create mode 100644 src/core/log/color.rs diff --git a/src/core/log/color.rs b/src/core/log/color.rs new file mode 100644 index 00000000..9baa99c2 --- /dev/null +++ b/src/core/log/color.rs @@ -0,0 +1,27 @@ +use super::Level; + +/// @returns (Foreground, Background) +#[inline] +#[must_use] +pub fn html(level: &Level) -> (&'static str, &'static str) { + match *level { + Level::TRACE => ("#000000", "#A0A0A0"), + Level::DEBUG => ("#000000", "#FFFFFF"), + Level::ERROR => ("#000000", "#FF0000"), + Level::WARN => ("#000000", "#FFFF00"), + Level::INFO => ("#FFFFFF", "#008E00"), + } +} + +/// @returns (Foreground) +#[inline] +#[must_use] +pub fn code_tag(level: &Level) -> &'static str { + match *level { + Level::TRACE => "#888888", + Level::DEBUG => "#C8C8C8", + Level::ERROR => "#FF0000", + Level::WARN => "#FFFF00", + Level::INFO => "#00FF00", + } +} diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index aac25ec2..c4f04707 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -1,3 +1,4 @@ +pub mod color; mod reload; pub use reload::ReloadHandle; From 1bb4021b906d880b0a235120a720f8b3effae973 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 11 Jun 2024 00:15:29 +0000 Subject: [PATCH 0181/2091] add format util to log suite Signed-off-by: Jason Volk --- src/core/log/fmt.rs | 18 ++++++++++++++++++ src/core/log/mod.rs | 1 + 2 files changed, 19 insertions(+) create mode 100644 src/core/log/fmt.rs diff --git a/src/core/log/fmt.rs b/src/core/log/fmt.rs new file mode 100644 index 00000000..d21f3dbc --- /dev/null +++ b/src/core/log/fmt.rs @@ -0,0 +1,18 @@ +use std::fmt::Write; + +use super::{color, Level}; +use crate::Result; + +pub fn html(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()> +where + S: Write, +{ + let color = color::code_tag(level); + let level = level.as_str().to_uppercase(); + write!( + out, + "{level:>5} {span:<12} {msg}
    " + )?; + + Ok(()) +} diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index c4f04707..eae82b50 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -1,4 +1,5 @@ pub mod color; +pub mod fmt; mod reload; pub use reload::ReloadHandle; From aa34021b274cb3f87b4c319d9ad670fd809eb4e1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 11 Jun 2024 01:26:31 +0000 Subject: [PATCH 0182/2091] tracing capture interface Signed-off-by: Jason Volk --- Cargo.lock | 1 + Cargo.toml | 2 + src/admin/debug/debug_commands.rs | 12 +---- src/core/Cargo.toml | 1 + src/core/log/capture/data.rs | 31 ++++++++++++ src/core/log/capture/guard.rs | 12 +++++ src/core/log/capture/layer.rs | 82 +++++++++++++++++++++++++++++++ src/core/log/capture/mod.rs | 50 +++++++++++++++++++ src/core/log/capture/state.rs | 35 +++++++++++++ src/core/log/capture/util.rs | 19 +++++++ src/core/log/fmt.rs | 2 +- src/core/log/mod.rs | 16 ++++-- src/core/log/server.rs | 14 ++++++ src/core/server.rs | 10 ++-- src/main/server.rs | 24 ++++++--- 15 files changed, 284 insertions(+), 27 deletions(-) create mode 100644 src/core/log/capture/data.rs create mode 100644 src/core/log/capture/guard.rs create mode 100644 src/core/log/capture/layer.rs create mode 100644 src/core/log/capture/mod.rs create mode 100644 src/core/log/capture/state.rs create mode 100644 src/core/log/capture/util.rs create mode 100644 src/core/log/server.rs diff --git a/Cargo.lock b/Cargo.lock index 64768b17..5b2b9b05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -701,6 +701,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tracing", + "tracing-core", "tracing-subscriber", "url", ] diff --git a/Cargo.toml b/Cargo.toml index 63528ef4..217e2434 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,6 +159,8 @@ default-features = false [workspace.dependencies.tracing-subscriber] version = "0.3.18" features = ["env-filter"] +[workspace.dependencies.tracing-core] +version = "0.1.32" # for URL previews [workspace.dependencies.webpage] diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index bd15ba50..21a1751c 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -344,11 +344,7 @@ pub(crate) async fn change_log_level( }, }; - match services() - .server - .tracing_reload_handle - .reload(&old_filter_layer) - { + match services().server.log.reload.reload(&old_filter_layer) { Ok(()) => { return Ok(RoomMessageEventContent::text_plain(format!( "Successfully changed log level back to config value {}", @@ -373,11 +369,7 @@ pub(crate) async fn change_log_level( }, }; - match services() - .server - .tracing_reload_handle - .reload(&new_filter_layer) - { + match services().server.log.reload.reload(&new_filter_layer) { Ok(()) => { return Ok(RoomMessageEventContent::text_plain("Successfully changed log level")); }, diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index c90a03b3..4a694d90 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -97,6 +97,7 @@ tikv-jemalloc-ctl.workspace = true tikv-jemalloc-sys.optional = true tikv-jemalloc-sys.workspace = true tokio.workspace = true +tracing-core.workspace = true tracing-subscriber.workspace = true tracing.workspace = true url.workspace = true diff --git a/src/core/log/capture/data.rs b/src/core/log/capture/data.rs new file mode 100644 index 00000000..ea104f57 --- /dev/null +++ b/src/core/log/capture/data.rs @@ -0,0 +1,31 @@ +use tracing::Level; +use tracing_core::{span::Current, Event}; + +use super::layer::Value; + +pub struct Data<'a> { + pub event: &'a Event<'a>, + pub current: &'a Current, + pub values: Option<&'a mut [Value]>, +} + +impl Data<'_> { + #[must_use] + pub fn level(&self) -> Level { *self.event.metadata().level() } + + #[must_use] + pub fn mod_name(&self) -> &str { self.event.metadata().module_path().unwrap_or_default() } + + #[must_use] + pub fn span_name(&self) -> &str { self.current.metadata().map_or("", |s| s.name()) } + + #[must_use] + pub fn message(&self) -> &str { + self.values + .as_ref() + .expect("values are not composed for a filter") + .iter() + .find(|(k, _)| *k == "message") + .map_or("", |(_, v)| v.as_str()) + } +} diff --git a/src/core/log/capture/guard.rs b/src/core/log/capture/guard.rs new file mode 100644 index 00000000..a126fedc --- /dev/null +++ b/src/core/log/capture/guard.rs @@ -0,0 +1,12 @@ +use std::sync::Arc; + +use super::Capture; + +/// Capture instance scope guard. +pub struct Guard { + pub(super) capture: Arc, +} + +impl Drop for Guard { + fn drop(&mut self) { self.capture.stop(); } +} diff --git a/src/core/log/capture/layer.rs b/src/core/log/capture/layer.rs new file mode 100644 index 00000000..48b125da --- /dev/null +++ b/src/core/log/capture/layer.rs @@ -0,0 +1,82 @@ +use std::{fmt, sync::Arc}; + +use tracing::field::{Field, Visit}; +use tracing_core::{Event, Subscriber}; +use tracing_subscriber::{layer::Context, registry::LookupSpan}; + +use super::{Capture, Data, State}; + +pub type Value = (&'static str, String); + +pub struct Layer { + state: Arc, +} + +struct Visitor { + values: Vec, +} + +impl Layer { + pub fn new(state: &Arc) -> Self { + Self { + state: state.clone(), + } + } +} + +impl fmt::Debug for Layer { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.debug_struct("capture::Layer").finish() + } +} + +impl tracing_subscriber::Layer for Layer +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + fn on_event(&self, event: &Event<'_>, ctx: Context<'_, S>) { + self.state + .active + .read() + .expect("shared lock") + .iter() + .filter(|capture| filter(capture, event, &ctx)) + .for_each(|capture| handle(capture, event, &ctx)); + } +} + +fn handle(capture: &Capture, event: &Event<'_>, ctx: &Context<'_, S>) +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + let mut visitor = Visitor { + values: Vec::new(), + }; + event.record(&mut visitor); + + let mut closure = capture.closure.lock().expect("exclusive lock"); + closure(Data { + event, + current: &ctx.current_span(), + values: Some(&mut visitor.values), + }); +} + +fn filter(capture: &Capture, event: &Event<'_>, ctx: &Context<'_, S>) -> bool +where + S: Subscriber + for<'a> LookupSpan<'a>, +{ + capture.filter.as_ref().map_or(true, |filter| { + filter(Data { + event, + current: &ctx.current_span(), + values: None, + }) + }) +} + +impl Visit for Visitor { + fn record_debug(&mut self, f: &Field, v: &dyn fmt::Debug) { self.values.push((f.name(), format!("{v:?}"))); } + + fn record_str(&mut self, f: &Field, v: &str) { self.values.push((f.name(), v.to_owned())); } +} diff --git a/src/core/log/capture/mod.rs b/src/core/log/capture/mod.rs new file mode 100644 index 00000000..f0495ad9 --- /dev/null +++ b/src/core/log/capture/mod.rs @@ -0,0 +1,50 @@ +pub mod data; +mod guard; +pub mod layer; +pub mod state; +pub mod util; + +use std::sync::{Arc, Mutex}; + +pub use data::Data; +use guard::Guard; +pub use layer::{Layer, Value}; +pub use state::State; +pub use util::to_html; + +pub type Filter = dyn Fn(Data<'_>) -> bool + Send + Sync + 'static; +pub type Closure = dyn FnMut(Data<'_>) + Send + Sync + 'static; + +/// Capture instance state. +pub struct Capture { + state: Arc, + filter: Option>, + closure: Mutex>, +} + +impl Capture { + /// Construct a new capture instance. Capture does not start until the Guard + /// is in scope. + #[must_use] + pub fn new(state: &Arc, filter: Option, closure: C) -> Arc + where + F: Fn(Data<'_>) -> bool + Send + Sync + 'static, + C: FnMut(Data<'_>) + Send + Sync + 'static, + { + Arc::new(Self { + state: state.clone(), + filter: filter.map(|p| -> Box { Box::new(p) }), + closure: Mutex::new(Box::new(closure)), + }) + } + + #[must_use] + pub fn start(self: &Arc) -> Guard { + self.state.add(self); + Guard { + capture: self.clone(), + } + } + + pub fn stop(self: &Arc) { self.state.del(self); } +} diff --git a/src/core/log/capture/state.rs b/src/core/log/capture/state.rs new file mode 100644 index 00000000..f2c401cd --- /dev/null +++ b/src/core/log/capture/state.rs @@ -0,0 +1,35 @@ +use std::sync::{Arc, RwLock}; + +use super::Capture; + +/// Capture layer state. +pub struct State { + pub(super) active: RwLock>>, +} + +impl Default for State { + fn default() -> Self { Self::new() } +} + +impl State { + #[must_use] + pub fn new() -> Self { + Self { + active: RwLock::new(Vec::new()), + } + } + + pub(super) fn add(&self, capture: &Arc) { + self.active + .write() + .expect("locked for writing") + .push(capture.clone()); + } + + pub(super) fn del(&self, capture: &Arc) { + let mut vec = self.active.write().expect("locked for writing"); + if let Some(pos) = vec.iter().position(|v| Arc::ptr_eq(v, capture)) { + vec.swap_remove(pos); + } + } +} diff --git a/src/core/log/capture/util.rs b/src/core/log/capture/util.rs new file mode 100644 index 00000000..54a19b6a --- /dev/null +++ b/src/core/log/capture/util.rs @@ -0,0 +1,19 @@ +use std::sync::{Arc, Mutex}; + +use super::{super::fmt, Closure}; + +pub fn to_html(out: &Arc>) -> Box +where + S: std::fmt::Write + Send + 'static, +{ + let out = out.clone(); + Box::new(move |data| { + fmt::html( + &mut *out.lock().expect("locked"), + &data.level(), + data.span_name(), + data.message(), + ) + .expect("log line appended"); + }) +} diff --git a/src/core/log/fmt.rs b/src/core/log/fmt.rs index d21f3dbc..a2e3a6f7 100644 --- a/src/core/log/fmt.rs +++ b/src/core/log/fmt.rs @@ -11,7 +11,7 @@ where let level = level.as_str().to_uppercase(); write!( out, - "{level:>5} {span:<12} {msg}
    " + "{level:>5} {span:^12} {msg}
    " )?; Ok(()) diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index eae82b50..556bf2f8 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -1,13 +1,19 @@ +pub mod capture; pub mod color; pub mod fmt; mod reload; +mod server; -pub use reload::ReloadHandle; -pub use reload::LogLevelReloadHandles; +pub use capture::Capture; +pub use reload::{LogLevelReloadHandles, ReloadHandle}; +pub use server::Server; +pub use tracing::Level; +pub use tracing_core::{Event, Metadata}; -// Wraps for logging macros. Use these macros rather than extern tracing:: or log:: crates in -// project code. ::log and ::tracing can still be used if necessary but discouraged. Remember -// debug_ log macros are also exported to the crate namespace like these. +// Wraps for logging macros. Use these macros rather than extern tracing:: or +// log:: crates in project code. ::log and ::tracing can still be used if +// necessary but discouraged. Remember debug_ log macros are also exported to +// the crate namespace like these. #[macro_export] macro_rules! error { diff --git a/src/core/log/server.rs b/src/core/log/server.rs new file mode 100644 index 00000000..644cecd1 --- /dev/null +++ b/src/core/log/server.rs @@ -0,0 +1,14 @@ +use std::sync::Arc; + +use super::{capture, reload::LogLevelReloadHandles}; + +/// Logging subsystem. This is a singleton member of super::Server which holds +/// all logging and tracing related state rather than shoving it all in +/// super::Server directly. +pub struct Server { + /// General log level reload handles. + pub reload: LogLevelReloadHandles, + + /// Tracing capture state for ephemeral/oneshot uses. + pub capture: Arc, +} diff --git a/src/core/server.rs b/src/core/server.rs index 7be6fa58..95eb7e01 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -5,7 +5,7 @@ use std::{ use tokio::{runtime, sync::broadcast}; -use crate::{config::Config, log::LogLevelReloadHandles}; +use crate::{config::Config, log}; /// Server runtime state; public portion pub struct Server { @@ -29,8 +29,8 @@ pub struct Server { /// Reload/shutdown signal pub signal: broadcast::Sender<&'static str>, - /// Log level reload handles. - pub tracing_reload_handle: LogLevelReloadHandles, + /// Logging subsystem state + pub log: log::Server, /// TODO: move stats pub requests_spawn_active: AtomicU32, @@ -42,7 +42,7 @@ pub struct Server { impl Server { #[must_use] - pub fn new(config: Config, runtime: Option, tracing_reload_handle: LogLevelReloadHandles) -> Self { + pub fn new(config: Config, runtime: Option, log: log::Server) -> Self { Self { config, started: SystemTime::now(), @@ -50,7 +50,7 @@ impl Server { reloading: AtomicBool::new(false), runtime, signal: broadcast::channel::<&'static str>(1).0, - tracing_reload_handle, + log, requests_spawn_active: AtomicU32::new(0), requests_spawn_finished: AtomicU32::new(0), requests_handle_active: AtomicU32::new(0), diff --git a/src/main/server.rs b/src/main/server.rs index b1bccfe9..ac93da17 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -4,7 +4,7 @@ use conduit::{ config, config::Config, info, - log::{LogLevelReloadHandles, ReloadHandle}, + log::{self, capture, LogLevelReloadHandles, ReloadHandle}, utils::{hash, sys}, Error, Result, }; @@ -34,7 +34,7 @@ impl Server { #[cfg(feature = "sentry_telemetry")] let sentry_guard = init_sentry(&config); - let (tracing_reload_handle, tracing_flame_guard) = init_tracing(&config); + let (tracing_reload_handle, tracing_flame_guard, capture) = init_tracing(&config); config.check()?; #[cfg(unix)] @@ -50,7 +50,14 @@ impl Server { ); Ok(Arc::new(Self { - server: Arc::new(conduit::Server::new(config, runtime.cloned(), tracing_reload_handle)), + server: Arc::new(conduit::Server::new( + config, + runtime.cloned(), + log::Server { + reload: tracing_reload_handle, + capture, + }, + )), _tracing_flame_guard: tracing_flame_guard, @@ -100,7 +107,7 @@ type TracingFlameGuard = (); // clippy thinks the filter_layer clones are redundant if the next usage is // behind a disabled feature. #[allow(clippy::redundant_clone)] -fn init_tracing(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard) { +fn init_tracing(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard, Arc) { let registry = Registry::default(); let fmt_layer = tracing_subscriber::fmt::Layer::new(); let filter_layer = match EnvFilter::try_new(&config.log) { @@ -122,7 +129,12 @@ fn init_tracing(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard) { let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); reload_handles.push(Box::new(fmt_reload_handle)); - let subscriber = subscriber.with(fmt_layer.with_filter(fmt_reload_filter)); + + let cap_state = Arc::new(capture::State::new()); + let cap_layer = capture::Layer::new(&cap_state); + let subscriber = subscriber + .with(fmt_layer.with_filter(fmt_reload_filter)) + .with(cap_layer); #[cfg(feature = "sentry_telemetry")] let subscriber = { @@ -185,5 +197,5 @@ fn init_tracing(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard) { needs access to trace-level events. 'release_max_log_level' must be disabled to use tokio-console." ); - (LogLevelReloadHandles::new(reload_handles), flame_guard) + (LogLevelReloadHandles::new(reload_handles), flame_guard, cap_state) } From c914a4fd91f2e64284e4f3c2323c21c281b5b19b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 11 Jun 2024 01:27:58 +0000 Subject: [PATCH 0183/2091] capture logs for resolve-true-destination admin cmd Signed-off-by: Jason Volk --- src/admin/debug/debug_commands.rs | 34 +++++++++++++++++++++++-------- src/service/sending/resolve.rs | 1 + 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 21a1751c..287c78dd 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -1,11 +1,15 @@ use std::{ collections::{BTreeMap, HashMap}, - sync::Arc, + sync::{Arc, Mutex}, time::Instant, }; -use api::client::validate_and_add_event_id; -use conduit::{utils::HtmlEscape, Error, Result}; +use conduit::{ + debug, info, log, + log::{capture, Capture}, + utils::HtmlEscape, + warn, Error, Result, +}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, events::room::message::RoomMessageEventContent, @@ -13,9 +17,10 @@ use ruma::{ }; use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve::resolve_actual_dest, services, PduEvent}; use tokio::sync::RwLock; -use tracing::{debug, info, warn}; use tracing_subscriber::EnvFilter; +use crate::api::client::validate_and_add_event_id; + pub(crate) async fn get_auth_chain(_body: Vec<&str>, event_id: Box) -> Result { let event_id = Arc::::from(event_id); if let Some(event) = services().rooms.timeline.get_pdu_json(&event_id)? { @@ -635,11 +640,24 @@ pub(crate) async fn resolve_true_destination( )); } - let (actual_dest, hostname_uri) = resolve_actual_dest(&server_name, !no_cache).await?; + let filter: &capture::Filter = &|data| { + data.level() <= log::Level::DEBUG + && data.mod_name().starts_with("conduit") + && matches!(data.span_name(), "actual" | "well-known" | "srv") + }; - Ok(RoomMessageEventContent::text_plain(format!( - "Actual destination: {actual_dest} | Hostname URI: {hostname_uri}" - ))) + let state = &services().server.log.capture; + let logs = Arc::new(Mutex::new(String::new())); + let capture = Capture::new(state, Some(filter), capture::to_html(&logs)); + let (actual_dest, hostname_uri); + { + let _capture_scope = capture.start(); + (actual_dest, hostname_uri) = resolve_actual_dest(&server_name, !no_cache).await?; + }; + + let plain = format!("Actual destination: {actual_dest} | Hostname URI: {hostname_uri}"); + let html = format!("{}
    {plain}", logs.lock().expect("locked")); + Ok(RoomMessageEventContent::text_html(plain, html)) } #[must_use] diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 1d043d8b..23080bed 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -75,6 +75,7 @@ pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result /// Numbers in comments below refer to bullet points in linked section of /// specification +#[tracing::instrument(skip_all, name = "actual")] pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result<(FedDest, String)> { trace!("Finding actual destination for {dest}"); let mut host = dest.as_str().to_owned(); From 1f9225e4d128ca43e6992931e369214bdfa220f7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 12 Jun 2024 18:52:24 +0000 Subject: [PATCH 0184/2091] add markdown log format for capture Signed-off-by: Jason Volk --- Cargo.lock | 28 +++++++++++++++++++++ Cargo.toml | 1 + src/admin/debug/debug_commands.rs | 10 +++++--- src/core/log/capture/mod.rs | 2 +- src/core/log/capture/util.rs | 42 ++++++++++++++++++++++--------- src/core/log/fmt.rs | 29 +++++++++++++++++++++ 6 files changed, 95 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b2b9b05..283cdb84 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2714,6 +2714,24 @@ dependencies = [ "prost", ] +[[package]] +name = "pulldown-cmark" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" +dependencies = [ + "bitflags 2.5.0", + "memchr", + "pulldown-cmark-escape", + "unicase", +] + +[[package]] +name = "pulldown-cmark-escape" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" + [[package]] name = "quick-error" version = "1.2.3" @@ -2980,6 +2998,7 @@ dependencies = [ "js_int", "js_option", "percent-encoding", + "pulldown-cmark", "regex", "ruma-common", "ruma-identifiers-validation", @@ -4279,6 +4298,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "unicase" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" +dependencies = [ + "version_check", +] + [[package]] name = "unicode-bidi" version = "0.3.15" diff --git a/Cargo.toml b/Cargo.toml index 217e2434..1fd8bdbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -280,6 +280,7 @@ features = [ "appservice-api-c", "client-api", "federation-api", + "markdown", "push-gateway-api-c", "state-res", "unstable-exhaustive-types", diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 287c78dd..0316abc2 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -648,16 +648,18 @@ pub(crate) async fn resolve_true_destination( let state = &services().server.log.capture; let logs = Arc::new(Mutex::new(String::new())); - let capture = Capture::new(state, Some(filter), capture::to_html(&logs)); + let capture = Capture::new(state, Some(filter), capture::fmt_markdown(logs.clone())); let (actual_dest, hostname_uri); { let _capture_scope = capture.start(); (actual_dest, hostname_uri) = resolve_actual_dest(&server_name, !no_cache).await?; }; - let plain = format!("Actual destination: {actual_dest} | Hostname URI: {hostname_uri}"); - let html = format!("{}
    {plain}", logs.lock().expect("locked")); - Ok(RoomMessageEventContent::text_html(plain, html)) + let msg = format!( + "{}\nDestination: {actual_dest}\nHostname URI: {hostname_uri}", + logs.lock().expect("locked") + ); + Ok(RoomMessageEventContent::text_markdown(msg)) } #[must_use] diff --git a/src/core/log/capture/mod.rs b/src/core/log/capture/mod.rs index f0495ad9..c0fcd31e 100644 --- a/src/core/log/capture/mod.rs +++ b/src/core/log/capture/mod.rs @@ -10,7 +10,7 @@ pub use data::Data; use guard::Guard; pub use layer::{Layer, Value}; pub use state::State; -pub use util::to_html; +pub use util::*; pub type Filter = dyn Fn(Data<'_>) -> bool + Send + Sync + 'static; pub type Closure = dyn FnMut(Data<'_>) + Send + Sync + 'static; diff --git a/src/core/log/capture/util.rs b/src/core/log/capture/util.rs index 54a19b6a..8bad4ba0 100644 --- a/src/core/log/capture/util.rs +++ b/src/core/log/capture/util.rs @@ -1,19 +1,37 @@ use std::sync::{Arc, Mutex}; -use super::{super::fmt, Closure}; +use super::{ + super::{fmt, Level}, + Closure, Data, +}; +use crate::Result; -pub fn to_html(out: &Arc>) -> Box +pub fn fmt_html(out: Arc>) -> Box where S: std::fmt::Write + Send + 'static, { - let out = out.clone(); - Box::new(move |data| { - fmt::html( - &mut *out.lock().expect("locked"), - &data.level(), - data.span_name(), - data.message(), - ) - .expect("log line appended"); - }) + fmt(fmt::html, out) +} + +pub fn fmt_markdown(out: Arc>) -> Box +where + S: std::fmt::Write + Send + 'static, +{ + fmt(fmt::markdown, out) +} + +pub fn fmt(fun: F, out: Arc>) -> Box +where + F: Fn(&mut S, &Level, &str, &str) -> Result<()> + Send + Sync + Copy + 'static, + S: std::fmt::Write + Send + 'static, +{ + Box::new(move |data| call(fun, &mut *out.lock().expect("locked"), &data)) +} + +fn call(fun: F, out: &mut S, data: &Data<'_>) +where + F: Fn(&mut S, &Level, &str, &str) -> Result<()>, + S: std::fmt::Write, +{ + fun(out, &data.level(), data.span_name(), data.message()).expect("log line appended"); } diff --git a/src/core/log/fmt.rs b/src/core/log/fmt.rs index a2e3a6f7..dbaef890 100644 --- a/src/core/log/fmt.rs +++ b/src/core/log/fmt.rs @@ -16,3 +16,32 @@ where Ok(()) } + +pub fn markdown(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()> +where + S: Write, +{ + let level = level.as_str().to_uppercase(); + writeln!(out, "`{level:>5}` `{span:^12}` `{msg}`")?; + + Ok(()) +} + +pub fn markdown_table(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()> +where + S: Write, +{ + let level = level.as_str().to_uppercase(); + writeln!(out, "| `{level:>5}` | `{span:^12}` | `{msg} |")?; + + Ok(()) +} + +pub fn markdown_table_head(out: &mut S) -> Result<()> +where + S: Write, +{ + write!(out, "| level | span | message |\n|------:|:----:|:--------|\n")?; + + Ok(()) +} From e76e604771b0315064cf1d5dc56283f852c4ea78 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 11 Jun 2024 22:37:16 +0000 Subject: [PATCH 0185/2091] add MxidError to our Error Signed-off-by: Jason Volk --- src/core/error.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/error.rs b/src/core/error.rs index 3aead9d5..5847fb38 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -87,6 +87,8 @@ pub enum Error { #[error("{0}")] Fmt(#[from] fmt::Error), #[error("{0}")] + Mxid(#[from] ruma::IdParseError), + #[error("{0}")] Err(String), } From 5df744343743ff453fcfd1f33cad886308ea3a96 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 13 Jun 2024 02:15:51 +0000 Subject: [PATCH 0186/2091] add log suppression tool Signed-off-by: Jason Volk --- src/core/log/mod.rs | 2 ++ src/core/log/suppress.rs | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 src/core/log/suppress.rs diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 556bf2f8..c8330903 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -3,10 +3,12 @@ pub mod color; pub mod fmt; mod reload; mod server; +mod suppress; pub use capture::Capture; pub use reload::{LogLevelReloadHandles, ReloadHandle}; pub use server::Server; +pub use suppress::Suppress; pub use tracing::Level; pub use tracing_core::{Event, Metadata}; diff --git a/src/core/log/suppress.rs b/src/core/log/suppress.rs new file mode 100644 index 00000000..6e883086 --- /dev/null +++ b/src/core/log/suppress.rs @@ -0,0 +1,38 @@ +use std::sync::Arc; + +use super::EnvFilter; +use crate::Server; + +pub struct Suppress { + server: Arc, + restore: EnvFilter, +} + +impl Suppress { + pub fn new(server: &Arc) -> Self { + let config = &server.config.log; + Self::from_filters(server, EnvFilter::try_new(config).unwrap_or_default(), &EnvFilter::default()) + } + + fn from_filters(server: &Arc, restore: EnvFilter, suppress: &EnvFilter) -> Self { + server + .log + .reload + .reload(suppress) + .expect("log filter reloaded"); + Self { + server: server.clone(), + restore, + } + } +} + +impl Drop for Suppress { + fn drop(&mut self) { + self.server + .log + .reload + .reload(&self.restore) + .expect("log filter reloaded"); + } +} From 571ab6ac2bf730fd1ea53d567e33bf7ad15e7531 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 10 Jun 2024 06:02:17 +0000 Subject: [PATCH 0187/2091] admin terminal console Signed-off-by: Jason Volk --- Cargo.lock | 316 +++++++++++++++-- Cargo.toml | 7 + src/admin/handler.rs | 115 ++----- src/api/client/account.rs | 5 +- src/core/log/mod.rs | 1 + src/main/Cargo.toml | 3 + src/main/main.rs | 25 +- src/router/run.rs | 25 +- src/service/Cargo.toml | 22 +- src/service/admin.rs | 540 ------------------------------ src/service/admin/console.rs | 121 +++++++ src/service/admin/create.rs | 297 ++++++++++++++++ src/service/admin/grant.rs | 139 ++++++++ src/service/admin/mod.rs | 247 ++++++++++++++ src/service/globals/data.rs | 4 +- src/service/globals/migrations.rs | 2 +- src/service/rooms/timeline/mod.rs | 2 +- 17 files changed, 1190 insertions(+), 681 deletions(-) delete mode 100644 src/service/admin.rs create mode 100644 src/service/admin/console.rs create mode 100644 src/service/admin/create.rs create mode 100644 src/service/admin/grant.rs create mode 100644 src/service/admin/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 283cdb84..07148055 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,7 +130,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -141,7 +141,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -389,7 +389,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn", + "syn 2.0.66", ] [[package]] @@ -510,6 +510,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "cfg_aliases" version = "0.2.1" @@ -565,7 +571,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -574,6 +580,15 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +[[package]] +name = "clipboard-win" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79f4473f5144e20d9aceaf2972478f06ddf687831eafeeb434fbaf0acc4144ad" +dependencies = [ + "error-code", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -681,7 +696,7 @@ dependencies = [ "itertools 0.13.0", "libloading", "log", - "nix", + "nix 0.29.0", "parking_lot", "rand", "regex", @@ -781,10 +796,12 @@ dependencies = [ "reqwest", "ruma", "ruma-identifiers-validation", + "rustyline", "serde", "serde_json", "serde_yaml", "sha2", + "termimad", "tokio", "tracing", "url", @@ -840,6 +857,15 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6051f239ecec86fde3410901ab7860d458d160371533842974fc61f96d15879b" +[[package]] +name = "coolor" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e93977247fb916abeee1ff8c6594c9b421fd9c26c9b720a3944acb2a7de27b" +dependencies = [ + "crossterm", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -874,6 +900,45 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crokey" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48209802ec5862bb034cb16719eec24d1c759e62921be7d3c899d0d85f3344b" +dependencies = [ + "crokey-proc_macros", + "crossterm", + "once_cell", + "serde", + "strict", +] + +[[package]] +name = "crokey-proc_macros" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397d3c009d8df93c4b063ddaa44a81ee7098feb056f99b00896c36e2cee9a9f7" +dependencies = [ + "crossterm", + "proc-macro2", + "quote", + "strict", + "syn 1.0.109", +] + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + [[package]] name = "crossbeam-channel" version = "0.5.13" @@ -883,12 +948,65 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +[[package]] +name = "crossterm" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +dependencies = [ + "bitflags 2.5.0", + "crossterm_winapi", + "libc", + "mio", + "parking_lot", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crypto-common" version = "0.1.6" @@ -924,7 +1042,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -993,7 +1111,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -1039,7 +1157,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -1048,6 +1166,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "error-code" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0474425d51df81997e2f90a21591180b38eccf27292d755f3e30750225c175b" + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -1187,7 +1311,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -1480,7 +1604,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -1766,7 +1890,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -1978,6 +2102,29 @@ dependencies = [ "typewit", ] +[[package]] +name = "lazy-regex" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d12be4595afdf58bd19e4a9f4e24187da2a66700786ff660a418e9059937a4c" +dependencies = [ + "lazy-regex-proc_macros", + "once_cell", + "regex", +] + +[[package]] +name = "lazy-regex-proc_macros" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44bcd58e6c97a7fcbaffcdc95728b393b8d98933bfadad49ed4097845b57ef0b" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "syn 2.0.66", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -2151,6 +2298,15 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimad" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c5d708226d186590a7b6d4a9780e2bdda5f689e0d58cd17012a298efd745d2" +dependencies = [ + "once_cell", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -2174,6 +2330,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.48.0", ] @@ -2184,6 +2341,18 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "cfg_aliases 0.1.1", + "libc", +] + [[package]] name = "nix" version = "0.29.0" @@ -2192,7 +2361,7 @@ checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags 2.5.0", "cfg-if", - "cfg_aliases", + "cfg_aliases 0.2.1", "libc", ] @@ -2490,7 +2659,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -2583,7 +2752,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -2677,7 +2846,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", "version_check", "yansi", ] @@ -2702,7 +2871,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -3053,7 +3222,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn", + "syn 2.0.66", "toml", ] @@ -3250,6 +3419,25 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +[[package]] +name = "rustyline" +version = "14.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "clipboard-win", + "libc", + "log", + "memchr", + "nix 0.28.0", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "windows-sys 0.52.0", +] + [[package]] name = "ryu" version = "1.0.18" @@ -3478,7 +3666,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -3607,6 +3795,27 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.2" @@ -3696,6 +3905,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strict" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f42444fea5b87a39db4218d9422087e66a85d0e7a0963a439b07bcdf91804006" + [[package]] name = "string_cache" version = "0.8.7" @@ -3737,6 +3952,17 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.66" @@ -3768,7 +3994,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -3782,6 +4008,22 @@ dependencies = [ "utf-8", ] +[[package]] +name = "termimad" +version = "0.29.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aab6c8572830b10362f27e242c7c5e749f062ec310b76a0d0b56670eca81f28e" +dependencies = [ + "coolor", + "crokey", + "crossbeam", + "lazy-regex", + "minimad", + "serde", + "thiserror", + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.61" @@ -3799,7 +4041,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -3958,7 +4200,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -4174,7 +4416,7 @@ source = "git+https://github.com/girlbossceo/tracing?branch=tracing-subscriber/e dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -4328,6 +4570,18 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode-width" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -4392,6 +4646,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.8.0" @@ -4456,7 +4716,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.66", "wasm-bindgen-shared", ] @@ -4490,7 +4750,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4837,7 +5097,7 @@ checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", "synstructure", ] @@ -4858,7 +5118,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] @@ -4878,7 +5138,7 @@ checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", "synstructure", ] @@ -4907,7 +5167,7 @@ checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.66", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 1fd8bdbc..801aa5a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -414,6 +414,13 @@ features = [ "light", ] +[workspace.dependencies.rustyline] +version = "14.0.0" +default-features = false + +[workspace.dependencies.termimad] +version = "0.29.2" + # # Patches # diff --git a/src/admin/handler.rs b/src/admin/handler.rs index b09bdb8d..b44f08f4 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,24 +1,19 @@ -use std::sync::Arc; - use clap::Parser; +use conduit::trace; use regex::Regex; use ruma::{ events::{ relation::InReplyTo, room::message::{Relation::Reply, RoomMessageEventContent}, - TimelineEventType, }, - OwnedRoomId, OwnedUserId, RoomId, ServerName, UserId, + ServerName, }; -use serde_json::value::to_raw_value; -use tokio::sync::MutexGuard; -use tracing::error; extern crate conduit_service as service; -use conduit::{Error, Result}; -pub(crate) use service::admin::{AdminRoomEvent, Service}; -use service::{admin::HandlerResult, pdu::PduBuilder}; +use conduit::Result; +use service::admin::HandlerResult; +pub(crate) use service::admin::{AdminEvent, Service}; use self::{fsck::FsckCommand, tester::TesterCommands}; use crate::{ @@ -30,7 +25,7 @@ pub(crate) const PAGE_SIZE: usize = 100; #[cfg_attr(test, derive(Debug))] #[derive(Parser)] -#[command(name = "@conduit:server.name:", version = env!("CARGO_PKG_VERSION"))] +#[command(name = "admin", version = env!("CARGO_PKG_VERSION"))] pub(crate) enum AdminCommand { #[command(subcommand)] /// - Commands for managing appservices @@ -73,85 +68,29 @@ pub(crate) enum AdminCommand { } #[must_use] -pub fn handle(event: AdminRoomEvent, room: OwnedRoomId, user: OwnedUserId) -> HandlerResult { - Box::pin(handle_event(event, room, user)) -} +pub fn handle(event: AdminEvent) -> HandlerResult { Box::pin(handle_event(event)) } -async fn handle_event(event: AdminRoomEvent, admin_room: OwnedRoomId, server_user: OwnedUserId) -> Result<()> { - let (mut message_content, reply) = match event { - AdminRoomEvent::SendMessage(content) => (content, None), - AdminRoomEvent::ProcessMessage(room_message, reply_id) => { - // This future is ~8 KiB so it's better to start it off the stack. - (Box::pin(process_admin_message(room_message)).await, Some(reply_id)) +#[tracing::instrument(skip_all, name = "admin")] +async fn handle_event(event: AdminEvent) -> Result { Ok(AdminEvent::Reply(process_event(event).await)) } + +async fn process_event(event: AdminEvent) -> Option { + let (mut message_content, reply_id) = match event { + AdminEvent::Command(room_message, reply_id) => (Box::pin(process_admin_message(room_message)).await, reply_id), + AdminEvent::Notice(content) => (content, None), + AdminEvent::Reply(_) => return None, + }; + + message_content.relates_to = reply_id.map(|reply_id| Reply { + in_reply_to: InReplyTo { + event_id: reply_id.into(), }, - }; + }); - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(admin_room.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - - if let Some(reply) = reply { - message_content.relates_to = Some(Reply { - in_reply_to: InReplyTo { - event_id: reply.into(), - }, - }); - } - - let response_pdu = PduBuilder { - event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&message_content).expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - }; - - if let Err(e) = services() - .rooms - .timeline - .build_and_append_pdu(response_pdu, &server_user, &admin_room, &state_lock) - .await - { - handle_response_error(&e, &admin_room, &server_user, &state_lock).await?; - } - - Ok(()) -} - -async fn handle_response_error( - e: &Error, admin_room: &RoomId, server_user: &UserId, state_lock: &MutexGuard<'_, ()>, -) -> Result<()> { - error!("Failed to build and append admin room response PDU: \"{e}\""); - let error_room_message = RoomMessageEventContent::text_plain(format!( - "Failed to build and append admin room PDU: \"{e}\"\n\nThe original admin command may have finished \ - successfully, but we could not return the output." - )); - - let response_pdu = PduBuilder { - event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&error_room_message).expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - }; - - services() - .rooms - .timeline - .build_and_append_pdu(response_pdu, server_user, admin_room, state_lock) - .await?; - - Ok(()) + Some(message_content) } // Parse and process a message from the admin room +#[tracing::instrument(name = "process")] async fn process_admin_message(room_message: String) -> RoomMessageEventContent { let mut lines = room_message.lines().filter(|l| !l.trim().is_empty()); let command_line = lines.next().expect("each string has at least one line"); @@ -181,9 +120,13 @@ async fn process_admin_message(room_message: String) -> RoomMessageEventContent // Parse chat messages from the admin room into an AdminCommand object fn parse_admin_command(command_line: &str) -> Result { - // Note: argv[0] is `@conduit:servername:`, which is treated as the main command let mut argv = command_line.split_whitespace().collect::>(); + // First indice has to be "admin" but for console convenience we add it here + if !argv.is_empty() && !argv[0].ends_with("admin") { + argv.insert(0, "admin"); + } + // Replace `help command` with `command --help` // Clap has a help subcommand, but it omits the long help description. if argv.len() > 1 && argv[1] == "help" { @@ -213,9 +156,11 @@ fn parse_admin_command(command_line: &str) -> Result { argv[3] = &command_with_dashes_argv3; } + trace!(?command_line, ?argv, "parse"); AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) } +#[tracing::instrument(skip_all, name = "command")] async fn process_admin_command(command: AdminCommand, body: Vec<&str>) -> Result { let reply_message_content = match command { AdminCommand::Appservices(command) => appservice::process(command, body).await?, diff --git a/src/api/client/account.rs b/src/api/client/account.rs index d7868e21..42049b56 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -354,10 +354,7 @@ pub(crate) async fn register_route( .room_joined_count(&admin_room)? == Some(1) { - services() - .admin - .make_user_admin(&user_id, displayname) - .await?; + service::admin::make_user_admin(&user_id, displayname).await?; warn!("Granting {user_id} admin privileges as the first user"); } diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index c8330903..97da6b40 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -11,6 +11,7 @@ pub use server::Server; pub use suppress::Suppress; pub use tracing::Level; pub use tracing_core::{Event, Metadata}; +pub use tracing_subscriber::EnvFilter; // Wraps for logging macros. Use these macros rather than extern tracing:: or // log:: crates in project code. ::log and ::tracing can still be used if diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index ed5a12a6..bc3e3951 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -58,6 +58,9 @@ brotli_compression = [ "conduit-router/brotli_compression", "conduit-service/brotli_compression", ] +console = [ + "conduit-service/console", +] dev_release_log_level = [ "conduit-admin/dev_release_log_level", "conduit-api/dev_release_log_level", diff --git a/src/main/main.rs b/src/main/main.rs index 7933538d..6e1bfe38 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -4,7 +4,11 @@ mod server; extern crate conduit_core as conduit; -use std::{cmp, sync::Arc, time::Duration}; +use std::{ + cmp, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; use conduit::{debug_error, debug_info, error, trace, utils::available_parallelism, warn, Error, Result}; use server::Server; @@ -100,8 +104,13 @@ async fn async_main(server: &Arc) -> Result<(), Error> { #[tracing::instrument(skip_all)] async fn signal(server: Arc) { use signal::unix; - let mut quit = unix::signal(unix::SignalKind::quit()).expect("SIGQUIT handler"); - let mut term = unix::signal(unix::SignalKind::terminate()).expect("SIGTERM handler"); + use unix::SignalKind; + + const CONSOLE: bool = cfg!(feature = "console"); + const RELOADING: bool = cfg!(all(unix, debug_assertions, not(CONSOLE))); + + let mut quit = unix::signal(SignalKind::quit()).expect("SIGQUIT handler"); + let mut term = unix::signal(SignalKind::terminate()).expect("SIGTERM handler"); loop { trace!("Installed signal handlers"); let sig: &'static str; @@ -111,6 +120,16 @@ async fn signal(server: Arc) { _ = term.recv() => { sig = "SIGTERM"; }, } + // Indicate the SIGINT is requesting a hot-reload. + if RELOADING && sig == "SIGINT" { + server.server.reloading.store(true, Ordering::Release); + } + + // Indicate the signal is requesting a shutdown + if matches!(sig, "SIGQUIT" | "SIGTERM") || (!CONSOLE && sig == "SIGINT") { + server.server.stopping.store(true, Ordering::Release); + } + warn!("Received {sig}"); if let Err(e) = server.server.signal.send(sig) { debug_error!("signal channel: {e}"); diff --git a/src/router/run.rs b/src/router/run.rs index 80a7d1ee..e9876ef3 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -83,20 +83,23 @@ pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { #[tracing::instrument(skip_all)] async fn signal(server: Arc, tx: Sender<()>, handle: axum_server::Handle) { - let sig: &'static str = server - .signal - .subscribe() - .recv() - .await - .expect("channel error"); + loop { + let sig: &'static str = server + .signal + .subscribe() + .recv() + .await + .expect("channel error"); - debug!("Received signal {}", sig); - if sig == "SIGINT" { - let reload = cfg!(unix) && cfg!(debug_assertions); - server.reloading.store(reload, Ordering::Release); + if !server.running() { + handle_shutdown(&server, &tx, &handle, sig).await; + break; + } } +} - server.stopping.store(true, Ordering::Release); +async fn handle_shutdown(server: &Arc, tx: &Sender<()>, handle: &axum_server::Handle, sig: &str) { + debug!("Received signal {}", sig); if let Err(e) = tx.send(()) { error!("failed sending shutdown transaction to channel: {e}"); } diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 63e3e95f..8a254ef8 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -17,20 +17,24 @@ crate-type = [ ] [features] -element_hacks = [] +brotli_compression = [ + "reqwest/brotli", +] +console = [ + "dep:rustyline", + "dep:termimad", +] dev_release_log_level = [] +element_hacks = [] +gzip_compression = [ + "reqwest/gzip", +] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", "log/max_level_trace", "log/release_max_level_info", ] -gzip_compression = [ - "reqwest/gzip", -] -brotli_compression = [ - "reqwest/brotli", -] sha256_media = [ "dep:sha2", ] @@ -57,11 +61,15 @@ regex.workspace = true reqwest.workspace = true ruma-identifiers-validation.workspace = true ruma.workspace = true +rustyline.workspace = true +rustyline.optional = true serde_json.workspace = true serde.workspace = true serde_yaml.workspace = true sha2.optional = true sha2.workspace = true +termimad.workspace = true +termimad.optional = true tokio.workspace = true tracing.workspace = true url.workspace = true diff --git a/src/service/admin.rs b/src/service/admin.rs deleted file mode 100644 index 6c2eb8df..00000000 --- a/src/service/admin.rs +++ /dev/null @@ -1,540 +0,0 @@ -use std::{collections::BTreeMap, future::Future, pin::Pin, sync::Arc}; - -use conduit::{Error, Result}; -use ruma::{ - api::client::error::ErrorKind, - events::{ - room::{ - canonical_alias::RoomCanonicalAliasEventContent, - create::RoomCreateEventContent, - guest_access::{GuestAccess, RoomGuestAccessEventContent}, - history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, - join_rules::{JoinRule, RoomJoinRulesEventContent}, - member::{MembershipState, RoomMemberEventContent}, - message::RoomMessageEventContent, - name::RoomNameEventContent, - power_levels::RoomPowerLevelsEventContent, - preview_url::RoomPreviewUrlsEventContent, - topic::RoomTopicEventContent, - }, - TimelineEventType, - }, - EventId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId, UserId, -}; -use serde_json::value::to_raw_value; -use tokio::{sync::Mutex, task::JoinHandle}; -use tracing::{error, warn}; - -use crate::{pdu::PduBuilder, services}; - -pub type HandlerResult = Pin> + Send>>; -pub type Handler = fn(AdminRoomEvent, OwnedRoomId, OwnedUserId) -> HandlerResult; - -pub struct Service { - sender: loole::Sender, - receiver: Mutex>, - handler_join: Mutex>>, - pub handle: Mutex>, -} - -#[derive(Debug)] -pub enum AdminRoomEvent { - ProcessMessage(String, Arc), - SendMessage(RoomMessageEventContent), -} - -impl Service { - #[must_use] - pub fn build() -> Arc { - let (sender, receiver) = loole::unbounded(); - Arc::new(Self { - sender, - receiver: Mutex::new(receiver), - handler_join: Mutex::new(None), - handle: Mutex::new(None), - }) - } - - pub async fn start_handler(self: &Arc) { - let self_ = Arc::clone(self); - let handle = services().server.runtime().spawn(async move { - self_ - .handler() - .await - .expect("Failed to initialize admin room handler"); - }); - - _ = self.handler_join.lock().await.insert(handle); - } - - async fn handler(self: &Arc) -> Result<()> { - let receiver = self.receiver.lock().await; - let Ok(Some(admin_room)) = Self::get_admin_room() else { - return Ok(()); - }; - - let server_user = &services().globals.server_user; - - loop { - debug_assert!(!receiver.is_closed(), "channel closed"); - tokio::select! { - event = receiver.recv_async() => match event { - Ok(event) => self.receive(event, &admin_room, server_user).await?, - Err(_e) => return Ok(()), - } - } - } - } - - pub async fn close(&self) { - self.interrupt(); - if let Some(handler_join) = self.handler_join.lock().await.take() { - if let Err(e) = handler_join.await { - error!("Failed to shutdown: {e:?}"); - } - } - } - - pub fn interrupt(&self) { - if !self.sender.is_closed() { - self.sender.close(); - } - } - - pub async fn send_message(&self, message_content: RoomMessageEventContent) { - self.send(AdminRoomEvent::SendMessage(message_content)) - .await; - } - - pub async fn process_message(&self, room_message: String, event_id: Arc) { - self.send(AdminRoomEvent::ProcessMessage(room_message, event_id)) - .await; - } - - async fn receive(&self, event: AdminRoomEvent, room: &RoomId, user: &UserId) -> Result<(), Error> { - if let Some(handle) = self.handle.lock().await.as_ref() { - handle(event, room.into(), user.into()).await - } else { - Err(Error::Err("Admin module is not loaded.".into())) - } - } - - async fn send(&self, message: AdminRoomEvent) { - debug_assert!(!self.sender.is_full(), "channel full"); - debug_assert!(!self.sender.is_closed(), "channel closed"); - self.sender.send(message).expect("message sent"); - } - - /// Gets the room ID of the admin room - /// - /// Errors are propagated from the database, and will have None if there is - /// no admin room - pub fn get_admin_room() -> Result> { - services() - .rooms - .alias - .resolve_local_alias(&services().globals.admin_alias) - } - - /// Create the admin room. - /// - /// Users in this room are considered admins by conduit, and the room can be - /// used to issue admin commands by talking to the server user inside it. - pub async fn create_admin_room(&self) -> Result<()> { - let room_id = RoomId::new(services().globals.server_name()); - - services().rooms.short.get_or_create_shortroomid(&room_id)?; - - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - - // Create a user for the server - let server_user = &services().globals.server_user; - - services().users.create(server_user, None)?; - - let room_version = services().globals.default_room_version(); - let mut content = match room_version { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => RoomCreateEventContent::new_v1(server_user.clone()), - RoomVersionId::V11 => RoomCreateEventContent::new_v11(), - _ => { - warn!("Unexpected or unsupported room version {}", room_version); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, - }; - - content.federate = true; - content.predecessor = None; - content.room_version = room_version; - - // 1. The room create event - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomCreate, - content: to_raw_value(&content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - // 2. Make conduit bot join - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Join, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(server_user.to_string()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - // 3. Power levels - let mut users = BTreeMap::new(); - users.insert(server_user.clone(), 100.into()); - - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&RoomPowerLevelsEventContent { - users, - ..Default::default() - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - // 4.1 Join Rules - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomJoinRules, - content: to_raw_value(&RoomJoinRulesEventContent::new(JoinRule::Invite)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - // 4.2 History Visibility - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomHistoryVisibility, - content: to_raw_value(&RoomHistoryVisibilityEventContent::new(HistoryVisibility::Shared)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - // 4.3 Guest Access - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomGuestAccess, - content: to_raw_value(&RoomGuestAccessEventContent::new(GuestAccess::Forbidden)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - // 5. Events implied by name and topic - let room_name = format!("{} Admin Room", services().globals.server_name()); - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomName, - content: to_raw_value(&RoomNameEventContent::new(room_name)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomTopic, - content: to_raw_value(&RoomTopicEventContent { - topic: format!("Manage {}", services().globals.server_name()), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - // 6. Room alias - let alias = &services().globals.admin_alias; - - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomCanonicalAlias, - content: to_raw_value(&RoomCanonicalAliasEventContent { - alias: Some(alias.clone()), - alt_aliases: Vec::new(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - services() - .rooms - .alias - .set_alias(alias, &room_id, server_user)?; - - // 7. (ad-hoc) Disable room previews for everyone by default - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomPreviewUrls, - content: to_raw_value(&RoomPreviewUrlsEventContent { - disabled: true, - }) - .expect("event is valid we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - Ok(()) - } - - /// Invite the user to the conduit admin room. - /// - /// In conduit, this is equivalent to granting admin privileges. - pub async fn make_user_admin(&self, user_id: &UserId, displayname: String) -> Result<()> { - if let Some(room_id) = Self::get_admin_room()? { - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - - // Use the server user to grant the new admin's power level - let server_user = &services().globals.server_user; - - // Invite and join the real user - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Invite, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Join, - displayname: Some(displayname), - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - }, - user_id, - &room_id, - &state_lock, - ) - .await?; - - // Set power level - let mut users = BTreeMap::new(); - users.insert(server_user.clone(), 100.into()); - users.insert(user_id.to_owned(), 100.into()); - - services() - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&RoomPowerLevelsEventContent { - users, - ..Default::default() - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - - // Send welcome message - services().rooms.timeline.build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&RoomMessageEventContent::text_html( - format!("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `@conduit:{}: --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`", services().globals.server_name()), - format!("

    Thank you for trying out conduwuit!

    \n

    conduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

    \n

    Helpful links:

    \n
    \n

    Git and Documentation: https://github.com/girlbossceo/conduwuit
    Report issues: https://github.com/girlbossceo/conduwuit/issues

    \n
    \n

    For a list of available commands, send the following message in this room: @conduit:{}: --help

    \n

    Here are some rooms you can join (by typing the command):

    \n

    conduwuit room (Ask questions and get notified on updates):
    /join #conduwuit:puppygock.gay

    \n", services().globals.server_name()), - )) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - }, - server_user, - &room_id, - &state_lock, - ).await?; - } - - Ok(()) - } - - /// Checks whether a given user is an admin of this server - pub async fn user_is_admin(&self, user_id: &UserId) -> Result { - let Ok(Some(admin_room)) = Self::get_admin_room() else { - return Ok(false); - }; - - services().rooms.state_cache.is_joined(user_id, &admin_room) - } -} diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs new file mode 100644 index 00000000..66faf2c1 --- /dev/null +++ b/src/service/admin/console.rs @@ -0,0 +1,121 @@ +#![cfg(feature = "console")] +use std::sync::Arc; + +use conduit::{error, log, trace}; +use ruma::events::room::message::RoomMessageEventContent; +use rustyline::{error::ReadlineError, history, Editor}; +use termimad::MadSkin; +use tokio::{sync::Mutex, task::JoinHandle}; + +use crate::services; + +pub struct Console { + join: Mutex>>, + input: Mutex>, + output: MadSkin, +} + +impl Console { + #[must_use] + pub fn new() -> Arc { + use rustyline::config::{Behavior, BellStyle}; + use termimad::{crossterm::style::Color, Alignment, CompoundStyle, LineStyle}; + + let config = rustyline::Config::builder() + .enable_signals(false) + .behavior(Behavior::PreferTerm) + .bell_style(BellStyle::Visible) + .auto_add_history(true) + .max_history_size(100) + .expect("valid history size") + .indent_size(4) + .tab_stop(4) + .build(); + + let history = history::MemHistory::with_config(config); + let input = Editor::with_history(config, history).expect("builder configuration succeeded"); + + let mut output = MadSkin::default_dark(); + + let code_style = CompoundStyle::with_fgbg(Color::AnsiValue(40), Color::AnsiValue(234)); + output.inline_code = code_style.clone(); + output.code_block = LineStyle { + left_margin: 0, + right_margin: 0, + align: Alignment::Left, + compound_style: code_style, + }; + + Arc::new(Self { + join: None.into(), + input: Mutex::new(input), + output, + }) + } + + #[allow(clippy::let_underscore_must_use)] + pub async fn start(self: &Arc) { + let mut join = self.join.lock().await; + if join.is_none() { + let self_ = Arc::clone(self); + _ = join.insert(services().server.runtime().spawn(self_.worker())); + } + } + + pub fn interrupt(self: &Arc) { Self::handle_interrupt(); } + + #[allow(clippy::let_underscore_must_use)] + pub async fn close(self: &Arc) { + if let Some(join) = self.join.lock().await.take() { + _ = join.await; + } + } + + #[tracing::instrument(skip_all, name = "console")] + async fn worker(self: Arc) { + loop { + let mut input = self.input.lock().await; + + let suppression = log::Suppress::new(&services().server); + let line = tokio::task::block_in_place(|| input.readline("uwu> ")); + drop(suppression); + + match line { + Ok(string) => self.handle(string).await, + Err(e) => match e { + ReadlineError::Eof => break, + ReadlineError::Interrupted => Self::handle_interrupt(), + ReadlineError::WindowResized => Self::handle_winch(), + _ => error!("console: {e:?}"), + }, + } + } + + self.join.lock().await.take(); + } + + async fn handle(&self, line: String) { + if line.is_empty() { + return; + } + + match services().admin.command_in_place(line, None).await { + Ok(Some(content)) => self.output(content).await, + Err(e) => error!("processing command: {e}"), + _ => (), + } + } + + async fn output(&self, output_content: RoomMessageEventContent) { + let output = self.output.term_text(output_content.body()); + println!("{output}"); + } + + fn handle_interrupt() { + trace!("interrupted"); + } + + fn handle_winch() { + trace!("winch"); + } +} diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs new file mode 100644 index 00000000..c65bb6fa --- /dev/null +++ b/src/service/admin/create.rs @@ -0,0 +1,297 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use conduit::{Error, Result}; +use ruma::{ + api::client::error::ErrorKind, + events::{ + room::{ + canonical_alias::RoomCanonicalAliasEventContent, + create::RoomCreateEventContent, + guest_access::{GuestAccess, RoomGuestAccessEventContent}, + history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + name::RoomNameEventContent, + power_levels::RoomPowerLevelsEventContent, + preview_url::RoomPreviewUrlsEventContent, + topic::RoomTopicEventContent, + }, + TimelineEventType, + }, + RoomId, RoomVersionId, +}; +use serde_json::value::to_raw_value; +use tracing::warn; + +use crate::{pdu::PduBuilder, services}; + +/// Create the admin room. +/// +/// Users in this room are considered admins by conduit, and the room can be +/// used to issue admin commands by talking to the server user inside it. +pub async fn create_admin_room() -> Result<()> { + let room_id = RoomId::new(services().globals.server_name()); + + services().rooms.short.get_or_create_shortroomid(&room_id)?; + + let mutex_state = Arc::clone( + services() + .globals + .roomid_mutex_state + .write() + .await + .entry(room_id.clone()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + // Create a user for the server + let server_user = &services().globals.server_user; + services().users.create(server_user, None)?; + + let room_version = services().globals.default_room_version(); + let mut content = match room_version { + RoomVersionId::V1 + | RoomVersionId::V2 + | RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + | RoomVersionId::V7 + | RoomVersionId::V8 + | RoomVersionId::V9 + | RoomVersionId::V10 => RoomCreateEventContent::new_v1(server_user.clone()), + RoomVersionId::V11 => RoomCreateEventContent::new_v11(), + _ => { + warn!("Unexpected or unsupported room version {}", room_version); + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + }; + + content.federate = true; + content.predecessor = None; + content.room_version = room_version; + + // 1. The room create event + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomCreate, + content: to_raw_value(&content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + // 2. Make conduit bot join + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + join_authorized_via_users_server: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(server_user.to_string()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + // 3. Power levels + let mut users = BTreeMap::new(); + users.insert(server_user.clone(), 100.into()); + + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomPowerLevels, + content: to_raw_value(&RoomPowerLevelsEventContent { + users, + ..Default::default() + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + // 4.1 Join Rules + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomJoinRules, + content: to_raw_value(&RoomJoinRulesEventContent::new(JoinRule::Invite)) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + // 4.2 History Visibility + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomHistoryVisibility, + content: to_raw_value(&RoomHistoryVisibilityEventContent::new(HistoryVisibility::Shared)) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + // 4.3 Guest Access + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomGuestAccess, + content: to_raw_value(&RoomGuestAccessEventContent::new(GuestAccess::Forbidden)) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + // 5. Events implied by name and topic + let room_name = format!("{} Admin Room", services().globals.server_name()); + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomName, + content: to_raw_value(&RoomNameEventContent::new(room_name)) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomTopic, + content: to_raw_value(&RoomTopicEventContent { + topic: format!("Manage {}", services().globals.server_name()), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + // 6. Room alias + let alias = &services().globals.admin_alias; + + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomCanonicalAlias, + content: to_raw_value(&RoomCanonicalAliasEventContent { + alias: Some(alias.clone()), + alt_aliases: Vec::new(), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + services() + .rooms + .alias + .set_alias(alias, &room_id, server_user)?; + + // 7. (ad-hoc) Disable room previews for everyone by default + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomPreviewUrls, + content: to_raw_value(&RoomPreviewUrlsEventContent { + disabled: true, + }) + .expect("event is valid we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + Ok(()) +} diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs new file mode 100644 index 00000000..601bc486 --- /dev/null +++ b/src/service/admin/grant.rs @@ -0,0 +1,139 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use conduit::Result; +use ruma::{ + events::{ + room::{ + member::{MembershipState, RoomMemberEventContent}, + message::RoomMessageEventContent, + power_levels::RoomPowerLevelsEventContent, + }, + TimelineEventType, + }, + UserId, +}; +use serde_json::value::to_raw_value; + +use super::Service; +use crate::{pdu::PduBuilder, services}; + +/// Invite the user to the conduit admin room. +/// +/// In conduit, this is equivalent to granting admin privileges. +pub async fn make_user_admin(user_id: &UserId, displayname: String) -> Result<()> { + if let Some(room_id) = Service::get_admin_room()? { + let mutex_state = Arc::clone( + services() + .globals + .roomid_mutex_state + .write() + .await + .entry(room_id.clone()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + // Use the server user to grant the new admin's power level + let server_user = &services().globals.server_user; + + // Invite and join the real user + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Invite, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + join_authorized_via_users_server: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, + displayname: Some(displayname), + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + join_authorized_via_users_server: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + }, + user_id, + &room_id, + &state_lock, + ) + .await?; + + // Set power level + let mut users = BTreeMap::new(); + users.insert(server_user.clone(), 100.into()); + users.insert(user_id.to_owned(), 100.into()); + + services() + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomPowerLevels, + content: to_raw_value(&RoomPowerLevelsEventContent { + users, + ..Default::default() + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + + // Send welcome message + services().rooms.timeline.build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomMessage, + content: to_raw_value(&RoomMessageEventContent::text_html( + format!("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `@conduit:{}: --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`", services().globals.server_name()), + format!("

    Thank you for trying out conduwuit!

    \n

    conduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

    \n

    Helpful links:

    \n
    \n

    Git and Documentation: https://github.com/girlbossceo/conduwuit
    Report issues: https://github.com/girlbossceo/conduwuit/issues

    \n
    \n

    For a list of available commands, send the following message in this room: @conduit:{}: --help

    \n

    Here are some rooms you can join (by typing the command):

    \n

    conduwuit room (Ask questions and get notified on updates):
    /join #conduwuit:puppygock.gay

    \n", services().globals.server_name()), + )) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }, + server_user, + &room_id, + &state_lock, + ).await?; + } + + Ok(()) +} diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs new file mode 100644 index 00000000..d93b5a03 --- /dev/null +++ b/src/service/admin/mod.rs @@ -0,0 +1,247 @@ +pub mod console; +mod create; +mod grant; + +use std::{future::Future, pin::Pin, sync::Arc}; + +use conduit::{Error, Result}; +pub use create::create_admin_room; +pub use grant::make_user_admin; +use ruma::{ + events::{room::message::RoomMessageEventContent, TimelineEventType}, + EventId, OwnedRoomId, RoomId, UserId, +}; +use serde_json::value::to_raw_value; +use tokio::{ + sync::{Mutex, MutexGuard}, + task::JoinHandle, +}; +use tracing::error; + +use crate::{pdu::PduBuilder, services}; + +pub type HandlerResult = Pin> + Send>>; +pub type Handler = fn(AdminEvent) -> HandlerResult; + +pub struct Service { + sender: loole::Sender, + receiver: Mutex>, + handler_join: Mutex>>, + pub handle: Mutex>, + #[cfg(feature = "console")] + pub console: Arc, +} + +#[derive(Debug)] +pub enum AdminEvent { + Command(String, Option>), + Reply(Option), + Notice(RoomMessageEventContent), +} + +impl Service { + #[must_use] + pub fn build() -> Arc { + let (sender, receiver) = loole::unbounded(); + Arc::new(Self { + sender, + receiver: Mutex::new(receiver), + handler_join: Mutex::new(None), + handle: Mutex::new(None), + #[cfg(feature = "console")] + console: console::Console::new(), + }) + } + + pub fn interrupt(&self) { + #[cfg(feature = "console")] + self.console.interrupt(); + + if !self.sender.is_closed() { + self.sender.close(); + } + } + + pub async fn close(&self) { + self.interrupt(); + + #[cfg(feature = "console")] + self.console.close().await; + + if let Some(handler_join) = self.handler_join.lock().await.take() { + if let Err(e) = handler_join.await { + error!("Failed to shutdown: {e:?}"); + } + } + } + + pub async fn start_handler(self: &Arc) { + let self_ = Arc::clone(self); + let handle = services().server.runtime().spawn(async move { + self_ + .handler() + .await + .expect("Failed to initialize admin room handler"); + }); + + _ = self.handler_join.lock().await.insert(handle); + } + + async fn handler(self: &Arc) -> Result<()> { + let receiver = self.receiver.lock().await; + let mut signals = services().server.signal.subscribe(); + loop { + debug_assert!(!receiver.is_closed(), "channel closed"); + tokio::select! { + event = receiver.recv_async() => match event { + Ok(event) => self.receive(event).await, + Err(_) => return Ok(()), + }, + sig = signals.recv() => match sig { + Ok(sig) => self.handle_signal(sig).await, + Err(_) => continue, + }, + } + } + } + + pub async fn send_text(&self, body: &str) { + self.send_message(RoomMessageEventContent::text_plain(body)) + .await; + } + + pub async fn send_message(&self, message_content: RoomMessageEventContent) { + self.send(AdminEvent::Notice(message_content)).await; + } + + pub async fn command(&self, command: String, event_id: Option>) { + self.send(AdminEvent::Command(command, event_id)).await; + } + + pub async fn command_in_place( + &self, command: String, event_id: Option>, + ) -> Result> { + match self.handle(AdminEvent::Command(command, event_id)).await? { + AdminEvent::Reply(content) => Ok(content), + _ => Ok(None), + } + } + + async fn send(&self, message: AdminEvent) { + debug_assert!(!self.sender.is_full(), "channel full"); + debug_assert!(!self.sender.is_closed(), "channel closed"); + self.sender.send(message).expect("message sent"); + } + + async fn receive(&self, event: AdminEvent) { + if let Ok(AdminEvent::Reply(content)) = self.handle(event).await { + handle_response(content).await; + } + } + + async fn handle(&self, event: AdminEvent) -> Result { + if let Some(handle) = self.handle.lock().await.as_ref() { + handle(event).await + } else { + Err(Error::Err("Admin module is not loaded.".into())) + } + } + + async fn handle_signal(&self, #[allow(unused_variables)] sig: &'static str) { + #[cfg(feature = "console")] + if sig == "SIGINT" && services().server.running() { + self.console.start().await; + } + } + + /// Checks whether a given user is an admin of this server + pub async fn user_is_admin(&self, user_id: &UserId) -> Result { + let Ok(Some(admin_room)) = Self::get_admin_room() else { + return Ok(false); + }; + + services().rooms.state_cache.is_joined(user_id, &admin_room) + } + + /// Gets the room ID of the admin room + /// + /// Errors are propagated from the database, and will have None if there is + /// no admin room + pub fn get_admin_room() -> Result> { + services() + .rooms + .alias + .resolve_local_alias(&services().globals.admin_alias) + } +} + +async fn handle_response(content: Option) { + if let Some(content) = content { + if let Err(e) = respond_to_room(content).await { + error!("{e}"); + } + } +} + +async fn respond_to_room(output_content: RoomMessageEventContent) -> Result<()> { + let Ok(Some(admin_room)) = Service::get_admin_room() else { + return Ok(()); + }; + + let mutex_state = Arc::clone( + services() + .globals + .roomid_mutex_state + .write() + .await + .entry(admin_room.clone()) + .or_default(), + ); + let state_lock = mutex_state.lock().await; + + let response_pdu = PduBuilder { + event_type: TimelineEventType::RoomMessage, + content: to_raw_value(&output_content).expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }; + + let server_user = &services().globals.server_user; + if let Err(e) = services() + .rooms + .timeline + .build_and_append_pdu(response_pdu, server_user, &admin_room, &state_lock) + .await + { + handle_response_error(&e, &admin_room, server_user, &state_lock).await?; + } + + Ok(()) +} + +async fn handle_response_error( + e: &Error, admin_room: &RoomId, server_user: &UserId, state_lock: &MutexGuard<'_, ()>, +) -> Result<()> { + error!("Failed to build and append admin room response PDU: \"{e}\""); + let error_room_message = RoomMessageEventContent::text_plain(format!( + "Failed to build and append admin room PDU: \"{e}\"\n\nThe original admin command may have finished \ + successfully, but we could not return the output." + )); + + let response_pdu = PduBuilder { + event_type: TimelineEventType::RoomMessage, + content: to_raw_value(&error_room_message).expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }; + + services() + .rooms + .timeline + .build_and_append_pdu(response_pdu, server_user, admin_room, state_lock) + .await?; + + Ok(()) +} diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index b49fc4e9..9851bc0a 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -160,7 +160,9 @@ impl Data for KeyValueDatabase { futures.push(self.userid_lastonetimekeyupdate.watch_prefix(&userid_bytes)); futures.push(Box::pin(async move { - let _result = services().server.signal.subscribe().recv().await; + while services().server.running() { + let _result = services().server.signal.subscribe().recv().await; + } })); if !services().server.running() { diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 83b3be99..bc9eaaa8 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -190,7 +190,7 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result .insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; // Create the admin room and server user on first run - services().admin.create_admin_room().await?; + crate::admin::create_admin_room().await?; warn!( "Created new {} database with version {}", diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 1addd567..31a687a0 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -498,7 +498,7 @@ impl Service { { services() .admin - .process_message(body, pdu.event_id.clone()) + .command(body, Some(pdu.event_id.clone())) .await; } } From d4775f076309aa91fb1972706bc6244de14ff03e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 13 Jun 2024 22:22:21 +0000 Subject: [PATCH 0188/2091] add server-side command escape w/ public echo for admins Signed-off-by: Jason Volk --- conduwuit-example.toml | 5 ++ src/admin/handler.rs | 13 ++- src/core/config/mod.rs | 3 + src/service/admin/mod.rs | 134 ++++++++++++++++++++++++------ src/service/rooms/timeline/mod.rs | 34 ++------ 5 files changed, 134 insertions(+), 55 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 04bf26cd..97652eee 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -198,6 +198,11 @@ registration_token = "change this token for something specific to your server" # defaults to false # block_non_admin_invites = false +# Allows admins to enter commands in rooms other than #admins by prefixing with \!admin. The reply +# will be publicly visible to the room, originating from the sender. +# defaults to true +#admin_escape_commands = true + # List of forbidden username patterns/strings. Values in this list are matched as *contains*. # This is checked upon username availability check, registration, and startup as warnings if any local users in your database # have a forbidden username. diff --git a/src/admin/handler.rs b/src/admin/handler.rs index b44f08f4..e283def5 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -90,9 +90,8 @@ async fn process_event(event: AdminEvent) -> Option { } // Parse and process a message from the admin room -#[tracing::instrument(name = "process")] -async fn process_admin_message(room_message: String) -> RoomMessageEventContent { - let mut lines = room_message.lines().filter(|l| !l.trim().is_empty()); +async fn process_admin_message(msg: String) -> RoomMessageEventContent { + let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); let command_line = lines.next().expect("each string has at least one line"); let body = lines.collect::>(); @@ -122,8 +121,14 @@ async fn process_admin_message(room_message: String) -> RoomMessageEventContent fn parse_admin_command(command_line: &str) -> Result { let mut argv = command_line.split_whitespace().collect::>(); + // Remove any escapes that came with a server-side escape command + if !argv.is_empty() && argv[0].ends_with("admin") { + argv[0] = argv[0].trim_start_matches('\\'); + } + // First indice has to be "admin" but for console convenience we add it here - if !argv.is_empty() && !argv[0].ends_with("admin") { + let server_user = services().globals.server_user.as_str(); + if !argv.is_empty() && !argv[0].ends_with("admin") && !argv[0].starts_with(server_user) { argv.insert(0, "admin"); } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index dadb000b..58696f88 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -338,6 +338,8 @@ pub struct Config { #[serde(default)] pub block_non_admin_invites: bool, + #[serde(default = "true_fn")] + pub admin_escape_commands: bool, #[serde(default)] pub sentry: bool, @@ -610,6 +612,7 @@ impl fmt::Display for Config { "Block non-admin room invites (local and remote, admins can still send and receive invites)", &self.block_non_admin_invites.to_string(), ), + ("Enable admin escape commands", &self.admin_escape_commands.to_string()), ("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()), ("Allow incoming federated typing", &self.allow_incoming_typing.to_string()), ( diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index d93b5a03..aa4c2c21 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -8,7 +8,10 @@ use conduit::{Error, Result}; pub use create::create_admin_room; pub use grant::make_user_admin; use ruma::{ - events::{room::message::RoomMessageEventContent, TimelineEventType}, + events::{ + room::message::{Relation, RoomMessageEventContent}, + TimelineEventType, + }, EventId, OwnedRoomId, RoomId, UserId, }; use serde_json::value::to_raw_value; @@ -18,7 +21,7 @@ use tokio::{ }; use tracing::error; -use crate::{pdu::PduBuilder, services}; +use crate::{pdu::PduBuilder, services, PduEvent}; pub type HandlerResult = Pin> + Send>>; pub type Handler = fn(AdminEvent) -> HandlerResult; @@ -156,11 +159,11 @@ impl Service { /// Checks whether a given user is an admin of this server pub async fn user_is_admin(&self, user_id: &UserId) -> Result { - let Ok(Some(admin_room)) = Self::get_admin_room() else { - return Ok(false); - }; - - services().rooms.state_cache.is_joined(user_id, &admin_room) + if let Ok(Some(admin_room)) = Self::get_admin_room() { + services().rooms.state_cache.is_joined(user_id, &admin_room) + } else { + Ok(false) + } } /// Gets the room ID of the admin room @@ -168,25 +171,52 @@ impl Service { /// Errors are propagated from the database, and will have None if there is /// no admin room pub fn get_admin_room() -> Result> { - services() + if let Some(room_id) = services() .rooms .alias - .resolve_local_alias(&services().globals.admin_alias) + .resolve_local_alias(&services().globals.admin_alias)? + { + if services() + .rooms + .state_cache + .is_joined(&services().globals.server_user, &room_id)? + { + return Ok(Some(room_id)); + } + } + + Ok(None) } } async fn handle_response(content: Option) { - if let Some(content) = content { - if let Err(e) = respond_to_room(content).await { - error!("{e}"); + if let Some(content) = content.as_ref() { + if let Some(Relation::Reply { + in_reply_to, + }) = content.relates_to.as_ref() + { + if let Ok(Some(pdu)) = services().rooms.timeline.get_pdu(&in_reply_to.event_id) { + let response_sender = if is_admin_room(&pdu.room_id) { + &services().globals.server_user + } else { + &pdu.sender + }; + + respond_to_room(content, &pdu.room_id, response_sender).await; + } } } } -async fn respond_to_room(output_content: RoomMessageEventContent) -> Result<()> { - let Ok(Some(admin_room)) = Service::get_admin_room() else { - return Ok(()); - }; +async fn respond_to_room(content: &RoomMessageEventContent, room_id: &RoomId, user_id: &UserId) { + assert!( + services() + .admin + .user_is_admin(user_id) + .await + .expect("checked user is admin"), + "sender is not admin" + ); let mutex_state = Arc::clone( services() @@ -194,34 +224,33 @@ async fn respond_to_room(output_content: RoomMessageEventContent) -> Result<()> .roomid_mutex_state .write() .await - .entry(admin_room.clone()) + .entry(room_id.to_owned()) .or_default(), ); let state_lock = mutex_state.lock().await; let response_pdu = PduBuilder { event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&output_content).expect("event is valid, we just created it"), + content: to_raw_value(content).expect("event is valid, we just created it"), unsigned: None, state_key: None, redacts: None, }; - let server_user = &services().globals.server_user; if let Err(e) = services() .rooms .timeline - .build_and_append_pdu(response_pdu, server_user, &admin_room, &state_lock) + .build_and_append_pdu(response_pdu, user_id, room_id, &state_lock) .await { - handle_response_error(&e, &admin_room, server_user, &state_lock).await?; + if let Err(e) = handle_response_error(&e, room_id, user_id, &state_lock).await { + error!("{e}"); + } } - - Ok(()) } async fn handle_response_error( - e: &Error, admin_room: &RoomId, server_user: &UserId, state_lock: &MutexGuard<'_, ()>, + e: &Error, room_id: &RoomId, user_id: &UserId, state_lock: &MutexGuard<'_, ()>, ) -> Result<()> { error!("Failed to build and append admin room response PDU: \"{e}\""); let error_room_message = RoomMessageEventContent::text_plain(format!( @@ -240,8 +269,63 @@ async fn handle_response_error( services() .rooms .timeline - .build_and_append_pdu(response_pdu, server_user, admin_room, state_lock) + .build_and_append_pdu(response_pdu, user_id, room_id, state_lock) .await?; Ok(()) } + +pub async fn is_admin_command(pdu: &PduEvent, body: &str) -> bool { + // Server-side command-escape with public echo + let is_escape = body.starts_with('\\'); + let is_public_escape = is_escape && body.trim_start_matches('\\').starts_with("!admin"); + + // Admin command with public echo (in admin room) + let server_user = &services().globals.server_user; + let is_public_prefix = body.starts_with("!admin") || body.starts_with(server_user.as_str()); + + // Expected backward branch + if !is_public_escape && !is_public_prefix { + return false; + } + + // Check if server-side command-escape is disabled by configuration + if is_public_escape && !services().globals.config.admin_escape_commands { + return false; + } + + // Prevent unescaped !admin from being used outside of the admin room + if is_public_prefix && !is_admin_room(&pdu.room_id) { + return false; + } + + // Only senders who are admin can proceed + if !services() + .admin + .user_is_admin(&pdu.sender) + .await + .unwrap_or(false) + { + return false; + } + + // This will evaluate to false if the emergency password is set up so that + // the administrator can execute commands as conduit + let emergency_password_set = services().globals.emergency_password().is_some(); + let from_server = pdu.sender == *server_user && !emergency_password_set; + if from_server && is_admin_room(&pdu.room_id) { + return false; + } + + // Authentic admin command + true +} + +#[must_use] +pub fn is_admin_room(room_id: &RoomId) -> bool { + if let Ok(Some(admin_room_id)) = Service::get_admin_room() { + admin_room_id == room_id + } else { + false + } +} diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 31a687a0..7deba4de 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -35,10 +35,10 @@ use tracing::{debug, error, info, warn}; use super::state_compressor::CompressedStateEvent; use crate::{ + admin, server_is_ours, //api::server_server, service::{ - self, appservice::NamespaceRegex, pdu::{EventHash, PduBuilder}, rooms::event_handler::parse_incoming_pdu, @@ -477,30 +477,11 @@ impl Service { .search .index_pdu(shortroomid, &pdu_id, &body)?; - let server_user = &services().globals.server_user; - - let to_conduit = body.starts_with(&format!("{server_user}: ")) - || body.starts_with(&format!("{server_user} ")) - || body.starts_with("!admin") - || body == format!("{server_user}:") - || body == *server_user; - - // This will evaluate to false if the emergency password is set up so that - // the administrator can execute commands as conduit - let from_conduit = pdu.sender == *server_user && services().globals.emergency_password().is_none(); - if let Some(admin_room) = service::admin::Service::get_admin_room()? { - if to_conduit - && !from_conduit && admin_room == pdu.room_id - && services() - .rooms - .state_cache - .is_joined(server_user, &admin_room)? - { - services() - .admin - .command(body, Some(pdu.event_id.clone())) - .await; - } + if admin::is_admin_command(pdu, &body).await { + services() + .admin + .command(body, Some(pdu.event_id.clone())) + .await; } } }, @@ -795,7 +776,7 @@ impl Service { state_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex ) -> Result> { let (pdu, pdu_json) = self.create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock)?; - if let Some(admin_room) = service::admin::Service::get_admin_room()? { + if let Some(admin_room) = admin::Service::get_admin_room()? { if admin_room == room_id { match pdu.event_type() { TimelineEventType::RoomEncryption => { @@ -1222,6 +1203,7 @@ impl Service { Ok(()) } } + #[cfg(test)] mod tests { use super::*; From 8b68d6306c33639076c6ee3d543fe2fc081e082f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 14 Jun 2024 21:11:31 +0000 Subject: [PATCH 0189/2091] add MutexMap to utils Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 2 ++ src/core/utils/mutex_map.rs | 56 +++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 src/core/utils/mutex_map.rs diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 9ffbbbd0..27d47fa5 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -4,6 +4,7 @@ pub mod defer; pub mod hash; pub mod html; pub mod json; +pub mod mutex_map; pub mod sys; use std::{ @@ -14,6 +15,7 @@ use std::{ pub use debug::slice_truncated as debug_slice_truncated; pub use html::Escape as HtmlEscape; pub use json::{deserialize_from_str, to_canonical_object}; +pub use mutex_map::MutexMap; use rand::prelude::*; use ring::digest; use ruma::OwnedUserId; diff --git a/src/core/utils/mutex_map.rs b/src/core/utils/mutex_map.rs new file mode 100644 index 00000000..f102487c --- /dev/null +++ b/src/core/utils/mutex_map.rs @@ -0,0 +1,56 @@ +use std::{hash::Hash, sync::Arc}; + +type Value = tokio::sync::Mutex; +type ArcMutex = Arc>; +type HashMap = std::collections::HashMap>; +type MapMutex = std::sync::Mutex>; +type Map = MapMutex; + +/// Map of Mutexes +pub struct MutexMap { + map: Map, +} + +pub struct Guard { + _guard: tokio::sync::OwnedMutexGuard, +} + +impl MutexMap +where + Key: Send + Hash + Eq + Clone, + Val: Send + Default, +{ + #[must_use] + pub fn new() -> Self { + Self { + map: Map::::new(HashMap::::new()), + } + } + + pub async fn lock(&self, k: &K) -> Guard + where + K: ?Sized + Send + Sync, + Key: for<'a> From<&'a K>, + { + let val = self + .map + .lock() + .expect("map mutex locked") + .entry(k.into()) + .or_default() + .clone(); + + let guard = val.lock_owned().await; + Guard:: { + _guard: guard, + } + } +} + +impl Default for MutexMap +where + Key: Send + Hash + Eq + Clone, + Val: Send + Default, +{ + fn default() -> Self { Self::new() } +} From 22272bdc16110a3a1d29b99d9413043b93e05236 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 14 Jun 2024 21:28:00 +0000 Subject: [PATCH 0190/2091] reduce roomid_mutex_insert Signed-off-by: Jason Volk --- src/api/client/sync.rs | 54 ++++++------------------------- src/service/globals/mod.rs | 6 ++-- src/service/rooms/timeline/mod.rs | 26 ++++----------- 3 files changed, 20 insertions(+), 66 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 5d881b45..84cf11b6 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1,7 +1,6 @@ use std::{ cmp::Ordering, collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}, - sync::Arc, time::Duration, }; @@ -197,20 +196,9 @@ pub(crate) async fn sync_events_route( for result in all_invited_rooms { let (room_id, invite_state_events) = result?; - { - // Get and drop the lock to wait for remaining operations to finish - let mutex_insert = Arc::clone( - services() - .globals - .roomid_mutex_insert - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let insert_lock = mutex_insert.lock().await; - drop(insert_lock); - }; + // Get and drop the lock to wait for remaining operations to finish + let insert_lock = services().globals.roomid_mutex_insert.lock(&room_id).await; + drop(insert_lock); let invite_count = services() .rooms @@ -332,20 +320,9 @@ async fn handle_left_room( since: u64, room_id: &RoomId, sender_user: &UserId, left_rooms: &mut BTreeMap, next_batch_string: &str, full_state: bool, lazy_load_enabled: bool, ) -> Result<()> { - { - // Get and drop the lock to wait for remaining operations to finish - let mutex_insert = Arc::clone( - services() - .globals - .roomid_mutex_insert - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let insert_lock = mutex_insert.lock().await; - drop(insert_lock); - }; + // Get and drop the lock to wait for remaining operations to finish + let insert_lock = services().globals.roomid_mutex_insert.lock(room_id).await; + drop(insert_lock); let left_count = services() .rooms @@ -544,21 +521,10 @@ async fn load_joined_room( next_batch: u64, next_batchcount: PduCount, lazy_load_enabled: bool, lazy_load_send_redundant: bool, full_state: bool, device_list_updates: &mut HashSet, left_encrypted_users: &mut HashSet, ) -> Result { - { - // Get and drop the lock to wait for remaining operations to finish - // This will make sure the we have all events until next_batch - let mutex_insert = Arc::clone( - services() - .globals - .roomid_mutex_insert - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let insert_lock = mutex_insert.lock().await; - drop(insert_lock); - }; + // Get and drop the lock to wait for remaining operations to finish + // This will make sure the we have all events until next_batch + let insert_lock = services().globals.roomid_mutex_insert.lock(room_id).await; + drop(insert_lock); let (timeline_pdus, limited) = load_timeline(sender_user, room_id, sincecount, 10)?; diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 317a0d56..494d1821 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -14,6 +14,7 @@ use std::{ }; use base64::{engine::general_purpose, Engine as _}; +use conduit::utils; use data::Data; use hickory_resolver::TokioAsyncResolver; use ipaddress::IPAddress; @@ -33,6 +34,7 @@ use tokio::{ }; use tracing::{error, trace}; use url::Url; +use utils::MutexMap; use crate::{services, Config, Result}; @@ -52,7 +54,7 @@ pub struct Service { pub bad_event_ratelimiter: Arc>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub bad_query_ratelimiter: Arc>>, - pub roomid_mutex_insert: RwLock>>>, + pub roomid_mutex_insert: MutexMap, pub roomid_mutex_state: RwLock>>>, pub roomid_mutex_federation: RwLock>>>, // this lock will be held longer pub roomid_federationhandletime: RwLock>, @@ -115,7 +117,7 @@ impl Service { bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), roomid_mutex_state: RwLock::new(HashMap::new()), - roomid_mutex_insert: RwLock::new(HashMap::new()), + roomid_mutex_insert: MutexMap::::new(), roomid_mutex_federation: RwLock::new(HashMap::new()), roomid_federationhandletime: RwLock::new(HashMap::new()), updates_handle: Mutex::new(None), diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 7deba4de..4fbfb5df 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -271,16 +271,11 @@ impl Service { .state .set_forward_extremities(&pdu.room_id, leaves, state_lock)?; - let mutex_insert = Arc::clone( - services() - .globals - .roomid_mutex_insert - .write() - .await - .entry(pdu.room_id.clone()) - .or_default(), - ); - let insert_lock = mutex_insert.lock().await; + let insert_lock = services() + .globals + .roomid_mutex_insert + .lock(&pdu.room_id) + .await; let count1 = services().globals.next_count()?; // Mark as read first so the sending client doesn't get a notification even if @@ -1165,16 +1160,7 @@ impl Service { .get_shortroomid(&room_id)? .expect("room exists"); - let mutex_insert = Arc::clone( - services() - .globals - .roomid_mutex_insert - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let insert_lock = mutex_insert.lock().await; + let insert_lock = services().globals.roomid_mutex_insert.lock(&room_id).await; let count = services().globals.next_count()?; let mut pdu_id = shortroomid.to_be_bytes().to_vec(); From 539aa278150ddeef2822d4d52ef74d1873c37db6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 14 Jun 2024 21:39:37 +0000 Subject: [PATCH 0191/2091] reduce roomid_mutex_federation Signed-off-by: Jason Volk --- src/api/server/send.rs | 17 ++++++----------- src/api/server/send_join.rs | 17 ++++++----------- src/api/server/send_leave.rs | 17 ++++++----------- src/service/globals/mod.rs | 4 ++-- src/service/rooms/timeline/mod.rs | 15 +++++---------- 5 files changed, 25 insertions(+), 45 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index e2413a4a..6faa7c25 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, sync::Arc, time::Instant}; +use std::{collections::BTreeMap, time::Instant}; use axum_client_ip::InsecureClientIp; use conduit::debug_warn; @@ -107,16 +107,11 @@ pub(crate) async fn send_transaction_message_route( let mut resolved_map = BTreeMap::new(); for (event_id, value, room_id) in parsed_pdus { let pdu_start_time = Instant::now(); - let mutex = Arc::clone( - services() - .globals - .roomid_mutex_federation - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let mutex_lock = mutex.lock().await; + let mutex_lock = services() + .globals + .roomid_mutex_federation + .lock(&room_id) + .await; resolved_map.insert( event_id.clone(), services() diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 18d075b4..388280e9 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -1,6 +1,6 @@ #![allow(deprecated)] -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_join_event}, @@ -148,16 +148,11 @@ async fn create_join_event( .fetch_required_signing_keys([&value], &pub_key_map) .await?; - let mutex = Arc::clone( - services() - .globals - .roomid_mutex_federation - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let mutex_lock = mutex.lock().await; + let mutex_lock = services() + .globals + .roomid_mutex_federation + .lock(room_id) + .await; let pdu_id: Vec = services() .rooms .event_handler diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index ec843588..2680378f 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -1,6 +1,6 @@ #![allow(deprecated)] -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_leave_event}, @@ -154,16 +154,11 @@ async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJson .fetch_required_signing_keys([&value], &pub_key_map) .await?; - let mutex = Arc::clone( - services() - .globals - .roomid_mutex_federation - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let mutex_lock = mutex.lock().await; + let mutex_lock = services() + .globals + .roomid_mutex_federation + .lock(room_id) + .await; let pdu_id: Vec = services() .rooms .event_handler diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 494d1821..716735df 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -56,7 +56,7 @@ pub struct Service { pub bad_query_ratelimiter: Arc>>, pub roomid_mutex_insert: MutexMap, pub roomid_mutex_state: RwLock>>>, - pub roomid_mutex_federation: RwLock>>>, // this lock will be held longer + pub roomid_mutex_federation: MutexMap, pub roomid_federationhandletime: RwLock>, pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, @@ -118,7 +118,7 @@ impl Service { bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), roomid_mutex_state: RwLock::new(HashMap::new()), roomid_mutex_insert: MutexMap::::new(), - roomid_mutex_federation: RwLock::new(HashMap::new()), + roomid_mutex_federation: MutexMap::::new(), roomid_federationhandletime: RwLock::new(HashMap::new()), updates_handle: Mutex::new(None), stateres_mutex: Arc::new(Mutex::new(())), diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 4fbfb5df..35c1ffbb 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1122,16 +1122,11 @@ impl Service { let (event_id, value, room_id) = parse_incoming_pdu(&pdu)?; // Lock so we cannot backfill the same pdu twice at the same time - let mutex = Arc::clone( - services() - .globals - .roomid_mutex_federation - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let mutex_lock = mutex.lock().await; + let mutex_lock = services() + .globals + .roomid_mutex_federation + .lock(&room_id) + .await; // Skip the PDU if we already have it as a timeline event if let Some(pdu_id) = self.get_pdu_id(&event_id)? { From 08bf074cbbb7f2a50fcd4675b5e1e9b267e347a6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 14 Jun 2024 22:08:44 +0000 Subject: [PATCH 0192/2091] reduce roomid_mutex_state Signed-off-by: Jason Volk --- src/admin/debug/debug_commands.rs | 12 +-- src/api/client/membership.rs | 97 ++++++------------------- src/api/client/message.rs | 20 ++--- src/api/client/profile.rs | 14 +--- src/api/client/redact.rs | 17 ++--- src/api/client/room.rs | 49 ++++--------- src/api/client/state.rs | 13 +--- src/api/server/make_join.rs | 18 ++--- src/api/server/make_leave.rs | 19 ++--- src/service/admin/create.rs | 15 +--- src/service/admin/grant.rs | 13 +--- src/service/admin/mod.rs | 21 +----- src/service/globals/mod.rs | 4 +- src/service/rooms/event_handler/mod.rs | 12 +-- src/service/rooms/state/data.rs | 10 +-- src/service/rooms/state/mod.rs | 10 +-- src/service/rooms/state_accessor/mod.rs | 4 +- src/service/rooms/timeline/mod.rs | 14 ++-- 18 files changed, 93 insertions(+), 269 deletions(-) diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 0316abc2..0b9ff62c 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -595,17 +595,7 @@ pub(crate) async fn force_set_room_state_from_server( .state_compressor .save_state(room_id.clone().as_ref(), new_room_state)?; - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.clone().into()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - + let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; services() .rooms .state diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index f781808c..111d03c4 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -7,6 +7,7 @@ use std::{ }; use axum_client_ip::InsecureClientIp; +use conduit::utils::mutex_map; use ruma::{ api::{ client::{ @@ -32,7 +33,7 @@ use ruma::{ OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::sync::{MutexGuard, RwLock}; +use tokio::sync::RwLock; use tracing::{debug, error, info, trace, warn}; use super::get_alias_helper; @@ -373,16 +374,11 @@ pub(crate) async fn kick_user_route(body: Ruma) -> Resul event.membership = MembershipState::Leave; event.reason.clone_from(&body.reason); - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; services() .rooms @@ -442,16 +438,11 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< }, )?; - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; services() .rooms @@ -496,16 +487,11 @@ pub(crate) async fn unban_user_route(body: Ruma) -> Res event.reason.clone_from(&body.reason); event.join_authorized_via_users_server = None; - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; services() .rooms @@ -670,16 +656,7 @@ pub async fn join_room_by_id_helper( }); } - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; // Ask a remote server if we are not participating in this room if !services() @@ -695,7 +672,7 @@ pub async fn join_room_by_id_helper( async fn join_room_by_id_helper_remote( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, state_lock: MutexGuard<'_, ()>, + _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, ) -> Result { info!("Joining {room_id} over federation."); @@ -1030,7 +1007,7 @@ async fn join_room_by_id_helper_remote( async fn join_room_by_id_helper_local( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, state_lock: MutexGuard<'_, ()>, + _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, ) -> Result { info!("We can join locally"); @@ -1413,17 +1390,7 @@ pub(crate) async fn invite_helper( if !user_is_local(user_id) { let (pdu, pdu_json, invite_room_state) = { - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - + let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; let content = to_raw_value(&RoomMemberEventContent { avatar_url: services().users.avatar_url(user_id)?, displayname: None, @@ -1535,16 +1502,7 @@ pub(crate) async fn invite_helper( )); } - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; services() .rooms @@ -1638,16 +1596,7 @@ pub async fn leave_room(user_id: &UserId, room_id: &RoomId, reason: Option, user_id: OwnedUserId) { for (pdu_builder, room_id) in all_joined_rooms { - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - + let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; if let Err(e) = services() .rooms .timeline diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index d92ef002..4cb24c33 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use ruma::{ api::client::redact::redact_event, events::{room::redaction::RoomRedactionEventContent, TimelineEventType}, @@ -17,16 +15,11 @@ pub(crate) async fn redact_event_route(body: Ruma) -> let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let body = body.body; - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; let event_id = services() .rooms diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 14071d39..7090fdc8 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -1,4 +1,4 @@ -use std::{cmp::max, collections::BTreeMap, sync::Arc}; +use std::{cmp::max, collections::BTreeMap}; use conduit::{debug_info, debug_warn}; use ruma::{ @@ -89,18 +89,8 @@ pub(crate) async fn create_room_route(body: Ruma) -> R )); } - services().rooms.short.get_or_create_shortroomid(&room_id)?; - - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let _short_id = services().rooms.short.get_or_create_shortroomid(&room_id)?; + let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; let alias: Option = if let Some(alias) = &body.room_alias_name { Some(room_alias_check(alias, &body.appservice_info).await?) @@ -577,21 +567,17 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Create a replacement room let replacement_room = RoomId::new(services().globals.server_name()); - services() + + let _short_id = services() .rooms .short .get_or_create_shortroomid(&replacement_room)?; - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; // Send a m.room.tombstone event to the old room to indicate that it is not // intended to be used any further Fail if the sender does not have the required @@ -619,16 +605,11 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Change lock to replacement room drop(state_lock); - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(replacement_room.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&replacement_room) + .await; // Get the old room creation event let mut create_event_content = serde_json::from_str::( diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 32e0fb68..17ae7be4 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -172,18 +172,7 @@ async fn send_state_event_for_key_helper( sender: &UserId, room_id: &RoomId, event_type: &StateEventType, json: &Raw, state_key: String, ) -> Result> { allowed_to_send_state_event(room_id, event_type, json).await?; - - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - + let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; let event_id = services() .rooms .timeline diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index 0a6b1992..d3d934f7 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_join_event}, events::{ @@ -74,17 +72,11 @@ pub(crate) async fn create_join_event_template_route( } } - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; let join_rules_event = services() .rooms diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 72c931c5..62c09717 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_leave_event}, events::{ @@ -37,18 +35,11 @@ pub(crate) async fn create_leave_event_template_route( .acl_check(origin, &body.room_id)?; let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; - - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(body.room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; let content = to_raw_value(&RoomMemberEventContent { avatar_url: None, blurhash: None, diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index c65bb6fa..ad70fe0c 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; use conduit::{Error, Result}; use ruma::{ @@ -32,18 +32,9 @@ use crate::{pdu::PduBuilder, services}; pub async fn create_admin_room() -> Result<()> { let room_id = RoomId::new(services().globals.server_name()); - services().rooms.short.get_or_create_shortroomid(&room_id)?; + let _short_id = services().rooms.short.get_or_create_shortroomid(&room_id)?; - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; // Create a user for the server let server_user = &services().globals.server_user; diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 601bc486..ca48ce0d 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; use conduit::Result; use ruma::{ @@ -22,16 +22,7 @@ use crate::{pdu::PduBuilder, services}; /// In conduit, this is equivalent to granting admin privileges. pub async fn make_user_admin(user_id: &UserId, displayname: String) -> Result<()> { if let Some(room_id) = Service::get_admin_room()? { - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.clone()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; + let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; // Use the server user to grant the new admin's power level let server_user = &services().globals.server_user; diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index aa4c2c21..f5f818d3 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -4,7 +4,7 @@ mod grant; use std::{future::Future, pin::Pin, sync::Arc}; -use conduit::{Error, Result}; +use conduit::{utils::mutex_map, Error, Result}; pub use create::create_admin_room; pub use grant::make_user_admin; use ruma::{ @@ -15,10 +15,7 @@ use ruma::{ EventId, OwnedRoomId, RoomId, UserId, }; use serde_json::value::to_raw_value; -use tokio::{ - sync::{Mutex, MutexGuard}, - task::JoinHandle, -}; +use tokio::{sync::Mutex, task::JoinHandle}; use tracing::error; use crate::{pdu::PduBuilder, services, PduEvent}; @@ -218,17 +215,7 @@ async fn respond_to_room(content: &RoomMessageEventContent, room_id: &RoomId, us "sender is not admin" ); - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - let state_lock = mutex_state.lock().await; - + let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; let response_pdu = PduBuilder { event_type: TimelineEventType::RoomMessage, content: to_raw_value(content).expect("event is valid, we just created it"), @@ -250,7 +237,7 @@ async fn respond_to_room(content: &RoomMessageEventContent, room_id: &RoomId, us } async fn handle_response_error( - e: &Error, room_id: &RoomId, user_id: &UserId, state_lock: &MutexGuard<'_, ()>, + e: &Error, room_id: &RoomId, user_id: &UserId, state_lock: &mutex_map::Guard<()>, ) -> Result<()> { error!("Failed to build and append admin room response PDU: \"{e}\""); let error_room_message = RoomMessageEventContent::text_plain(format!( diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 716735df..626b2779 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -55,7 +55,7 @@ pub struct Service { pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub bad_query_ratelimiter: Arc>>, pub roomid_mutex_insert: MutexMap, - pub roomid_mutex_state: RwLock>>>, + pub roomid_mutex_state: MutexMap, pub roomid_mutex_federation: MutexMap, pub roomid_federationhandletime: RwLock>, pub updates_handle: Mutex>>, @@ -116,7 +116,7 @@ impl Service { bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), - roomid_mutex_state: RwLock::new(HashMap::new()), + roomid_mutex_state: MutexMap::::new(), roomid_mutex_insert: MutexMap::::new(), roomid_mutex_federation: MutexMap::::new(), roomid_federationhandletime: RwLock::new(HashMap::new()), diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index d1413cb5..e2cd5066 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -530,18 +530,8 @@ impl Service { // 13. Use state resolution to find new room state // We start looking at current room state now, so lets lock the room - let mutex_state = Arc::clone( - services() - .globals - .roomid_mutex_state - .write() - .await - .entry(room_id.to_owned()) - .or_default(), - ); - trace!("Locking the room"); - let state_lock = mutex_state.lock().await; + let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; // Now we calculate the set of extremities this room has after the incoming // event has been applied. We start with the previous extremities (aka leaves) diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index f0fef086..f8c7f6a6 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -1,7 +1,7 @@ use std::{collections::HashSet, sync::Arc}; +use conduit::utils::mutex_map; use ruma::{EventId, OwnedEventId, RoomId}; -use tokio::sync::MutexGuard; use crate::{utils, Error, KeyValueDatabase, Result}; @@ -14,7 +14,7 @@ pub trait Data: Send + Sync { &self, room_id: &RoomId, new_shortstatehash: u64, - _mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result<()>; /// Associates a state with an event. @@ -28,7 +28,7 @@ pub trait Data: Send + Sync { &self, room_id: &RoomId, event_ids: Vec, - _mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result<()>; } @@ -47,7 +47,7 @@ impl Data for KeyValueDatabase { &self, room_id: &RoomId, new_shortstatehash: u64, - _mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { self.roomid_shortstatehash .insert(room_id.as_bytes(), &new_shortstatehash.to_be_bytes())?; @@ -80,7 +80,7 @@ impl Data for KeyValueDatabase { &self, room_id: &RoomId, event_ids: Vec, - _mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 8a32d9ca..199ee87f 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -4,6 +4,7 @@ use std::{ sync::Arc, }; +use conduit::utils::mutex_map; use data::Data; use ruma::{ api::client::error::ErrorKind, @@ -15,7 +16,6 @@ use ruma::{ state_res::{self, StateMap}, EventId, OwnedEventId, RoomId, RoomVersionId, UserId, }; -use tokio::sync::MutexGuard; use tracing::warn; use super::state_compressor::CompressedStateEvent; @@ -33,7 +33,7 @@ impl Service { shortstatehash: u64, statediffnew: Arc>, _statediffremoved: Arc>, - state_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { for event_id in statediffnew.iter().filter_map(|new| { services() @@ -299,12 +299,12 @@ impl Service { } /// Set the state hash to a new version, but does not update state_cache. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self, mutex_lock))] pub fn set_room_state( &self, room_id: &RoomId, shortstatehash: u64, - mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { self.db.set_room_state(room_id, shortstatehash, mutex_lock) } @@ -343,7 +343,7 @@ impl Service { &self, room_id: &RoomId, event_ids: Vec, - state_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { self.db .set_forward_extremities(room_id, event_ids, state_lock) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index d3dc92ef..ab290cbd 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -4,6 +4,7 @@ use std::{ sync::{Arc, Mutex}, }; +use conduit::utils::mutex_map; use data::Data; use lru_cache::LruCache; use ruma::{ @@ -22,7 +23,6 @@ use ruma::{ EventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; use serde_json::value::to_raw_value; -use tokio::sync::MutexGuard; use tracing::{error, warn}; use crate::{service::pdu::PduBuilder, services, Error, PduEvent, Result}; @@ -285,7 +285,7 @@ impl Service { } pub async fn user_can_invite( - &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &MutexGuard<'_, ()>, + &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &mutex_map::Guard<()>, ) -> Result { let content = to_raw_value(&RoomMemberEventContent::new(MembershipState::Invite)) .expect("Event content always serializes"); diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 35c1ffbb..e8774cdc 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -30,7 +30,7 @@ use ruma::{ }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::sync::{Mutex, MutexGuard, RwLock}; +use tokio::sync::{Mutex, RwLock}; use tracing::{debug, error, info, warn}; use super::state_compressor::CompressedStateEvent; @@ -44,7 +44,7 @@ use crate::{ rooms::event_handler::parse_incoming_pdu, }, services, - utils::{self}, + utils::{self, mutex_map}, Error, PduCount, PduEvent, @@ -200,13 +200,13 @@ impl Service { /// happens in `append_pdu`. /// /// Returns pdu id - #[tracing::instrument(skip(self, pdu, pdu_json, leaves))] + #[tracing::instrument(skip_all)] pub async fn append_pdu( &self, pdu: &PduEvent, mut pdu_json: CanonicalJsonObject, leaves: Vec, - state_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result> { // Coalesce database writes for the remainder of this scope. let _cork = services().globals.db.cork_and_flush(); @@ -581,7 +581,7 @@ impl Service { pdu_builder: PduBuilder, sender: &UserId, room_id: &RoomId, - _mutex_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result<(PduEvent, CanonicalJsonObject)> { let PduBuilder { event_type, @@ -768,7 +768,7 @@ impl Service { pdu_builder: PduBuilder, sender: &UserId, room_id: &RoomId, - state_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result> { let (pdu, pdu_json) = self.create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock)?; if let Some(admin_room) = admin::Service::get_admin_room()? { @@ -909,7 +909,7 @@ impl Service { new_room_leaves: Vec, state_ids_compressed: Arc>, soft_fail: bool, - state_lock: &MutexGuard<'_, ()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result>> { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't From 03c04ce0a18b6d6f5f701210e20d98814dacb23a Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 14 Jun 2024 23:46:07 -0400 Subject: [PATCH 0193/2091] dont allow "remote admins" to run public escaped cmds on behalf of others Signed-off-by: strawberry --- src/service/admin/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index f5f818d3..dca3ac8d 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -18,7 +18,7 @@ use serde_json::value::to_raw_value; use tokio::{sync::Mutex, task::JoinHandle}; use tracing::error; -use crate::{pdu::PduBuilder, services, PduEvent}; +use crate::{pdu::PduBuilder, services, user_is_local, PduEvent}; pub type HandlerResult = Pin> + Send>>; pub type Handler = fn(AdminEvent) -> HandlerResult; @@ -276,6 +276,11 @@ pub async fn is_admin_command(pdu: &PduEvent, body: &str) -> bool { return false; } + // only allow public escaped commands by local admins + if is_public_escape && !user_is_local(&pdu.sender) { + return false; + } + // Check if server-side command-escape is disabled by configuration if is_public_escape && !services().globals.config.admin_escape_commands { return false; From 332fc74fb818a14292d8e90063b6236cdd4052f7 Mon Sep 17 00:00:00 2001 From: Kimiblock Moe Date: Sun, 16 Jun 2024 08:16:59 +0800 Subject: [PATCH 0194/2091] Add systemd service for Arch --- arch/conduwuit.service | 61 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 arch/conduwuit.service diff --git a/arch/conduwuit.service b/arch/conduwuit.service new file mode 100644 index 00000000..ecf06d01 --- /dev/null +++ b/arch/conduwuit.service @@ -0,0 +1,61 @@ +[Unit] +Description=conduwuit Matrix homeserver +After=network.target +Documentation=https://conduwuit.puppyirl.gay/ + +[Service] +DynamicUser=yes +Type=notify + +AmbientCapabilities= +CapabilityBoundingSet= + +DevicePolicy=closed +LockPersonality=yes +MemoryDenyWriteExecute=yes +NoNewPrivileges=yes +ProcSubset=pid +ProtectClock=yes +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelLogs=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectProc=invisible +ProtectSystem=strict +PrivateDevices=yes +PrivateMounts=yes +PrivateTmp=yes +PrivateUsers=yes +PrivateIPC=yes +RemoveIPC=yes +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +SystemCallArchitectures=native +SystemCallFilter=@system-service @resources +SystemCallFilter=~@clock @debug @module @mount @reboot @swap @cpu-emulation @obsolete @timer @chown @setuid @privileged @keyring @ipc +SystemCallErrorNumber=EPERM +StateDirectory=matrix-conduit + +RuntimeDirectory=conduwuit +RuntimeDirectoryMode=0750 + +Environment="CONDUIT_CONFIG=/etc/conduwuit/conduwuit.toml" + +ExecStart=/usr/bin/conduwuit +Restart=on-failure +RestartSec=1 +RestartSteps=1 +RestartMaxDelaySec=6000s + +TimeoutStopSec=4m +TimeoutStartSec=4m + +StartLimitInterval=1m +StartLimitBurst=5 + +[Install] +WantedBy=multi-user.target From 1323506c884a8f4233f7503dc72af6101c9d20fa Mon Sep 17 00:00:00 2001 From: Kimiblock Moe Date: Sun, 16 Jun 2024 09:48:25 +0800 Subject: [PATCH 0195/2091] Fix naming issues, directories will be moved in the post_upgrade function --- arch/conduwuit.service | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index ecf06d01..18fa04f9 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -2,6 +2,7 @@ Description=conduwuit Matrix homeserver After=network.target Documentation=https://conduwuit.puppyirl.gay/ +RequiresMountsFor=/var/lib/private/conduwuit [Service] DynamicUser=yes @@ -38,7 +39,7 @@ SystemCallArchitectures=native SystemCallFilter=@system-service @resources SystemCallFilter=~@clock @debug @module @mount @reboot @swap @cpu-emulation @obsolete @timer @chown @setuid @privileged @keyring @ipc SystemCallErrorNumber=EPERM -StateDirectory=matrix-conduit +StateDirectory=conduwuit RuntimeDirectory=conduwuit RuntimeDirectoryMode=0750 @@ -47,9 +48,7 @@ Environment="CONDUIT_CONFIG=/etc/conduwuit/conduwuit.toml" ExecStart=/usr/bin/conduwuit Restart=on-failure -RestartSec=1 -RestartSteps=1 -RestartMaxDelaySec=6000s +RestartSec=5 TimeoutStopSec=4m TimeoutStartSec=4m From 7b7593a6f583bed12d3416f0bf55ba52b34a4bc9 Mon Sep 17 00:00:00 2001 From: Kimiblock Moe Date: Sun, 16 Jun 2024 10:08:02 +0800 Subject: [PATCH 0196/2091] Change the env name --- arch/conduwuit.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index 18fa04f9..4cc6f850 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -44,7 +44,7 @@ StateDirectory=conduwuit RuntimeDirectory=conduwuit RuntimeDirectoryMode=0750 -Environment="CONDUIT_CONFIG=/etc/conduwuit/conduwuit.toml" +Environment="CONDUWUIT_CONFIG=/etc/conduwuit/conduwuit.toml" ExecStart=/usr/bin/conduwuit Restart=on-failure From 3813628acd3191f27645825c0f830710a5a4131c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 14 Jun 2024 03:27:27 +0000 Subject: [PATCH 0197/2091] log full custom edu rather than just content Signed-off-by: Jason Volk --- src/api/server/send.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 6faa7c25..e147f652 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -352,8 +352,8 @@ pub(crate) async fn send_transaction_message_route( .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true)?; } }, - Edu::_Custom(custom) => { - debug_warn!(?custom, "received custom/unknown EDU"); + Edu::_Custom(ref _custom) => { + debug_warn!(?edu, "received custom/unknown EDU"); }, } } From 02bd67dc4b2d6444e70fd7aa0d28c11d8e61b81c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 15 Jun 2024 23:52:55 +0000 Subject: [PATCH 0198/2091] admin command path simplifications; fix message to room Signed-off-by: Jason Volk --- src/admin/handler.rs | 23 ++-- src/service/admin/mod.rs | 175 ++++++++++++++++-------------- src/service/rooms/timeline/mod.rs | 2 +- 3 files changed, 103 insertions(+), 97 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index e283def5..85270db2 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -12,8 +12,8 @@ use ruma::{ extern crate conduit_service as service; use conduit::Result; -use service::admin::HandlerResult; -pub(crate) use service::admin::{AdminEvent, Service}; +pub(crate) use service::admin::{Command, Service}; +use service::admin::{CommandResult, HandlerResult}; use self::{fsck::FsckCommand, tester::TesterCommands}; use crate::{ @@ -68,25 +68,18 @@ pub(crate) enum AdminCommand { } #[must_use] -pub fn handle(event: AdminEvent) -> HandlerResult { Box::pin(handle_event(event)) } +pub fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(command)) } #[tracing::instrument(skip_all, name = "admin")] -async fn handle_event(event: AdminEvent) -> Result { Ok(AdminEvent::Reply(process_event(event).await)) } - -async fn process_event(event: AdminEvent) -> Option { - let (mut message_content, reply_id) = match event { - AdminEvent::Command(room_message, reply_id) => (Box::pin(process_admin_message(room_message)).await, reply_id), - AdminEvent::Notice(content) => (content, None), - AdminEvent::Reply(_) => return None, - }; - - message_content.relates_to = reply_id.map(|reply_id| Reply { +async fn handle_command(command: Command) -> CommandResult { + let mut content = process_admin_message(command.command).await; + content.relates_to = command.reply_id.map(|event_id| Reply { in_reply_to: InReplyTo { - event_id: reply_id.into(), + event_id, }, }); - Some(message_content) + Ok(Some(content)) } // Parse and process a message from the admin room diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index dca3ac8d..ddc3606f 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -7,12 +7,13 @@ use std::{future::Future, pin::Pin, sync::Arc}; use conduit::{utils::mutex_map, Error, Result}; pub use create::create_admin_room; pub use grant::make_user_admin; +use loole::{Receiver, Sender}; use ruma::{ events::{ room::message::{Relation, RoomMessageEventContent}, TimelineEventType, }, - EventId, OwnedRoomId, RoomId, UserId, + OwnedEventId, OwnedRoomId, RoomId, UserId, }; use serde_json::value::to_raw_value; use tokio::{sync::Mutex, task::JoinHandle}; @@ -20,12 +21,16 @@ use tracing::error; use crate::{pdu::PduBuilder, services, user_is_local, PduEvent}; -pub type HandlerResult = Pin> + Send>>; -pub type Handler = fn(AdminEvent) -> HandlerResult; +const COMMAND_QUEUE_LIMIT: usize = 512; + +pub type CommandOutput = Option; +pub type CommandResult = Result; +pub type HandlerResult = Pin + Send>>; +pub type Handler = fn(Command) -> HandlerResult; pub struct Service { - sender: loole::Sender, - receiver: Mutex>, + sender: Sender, + receiver: Mutex>, handler_join: Mutex>>, pub handle: Mutex>, #[cfg(feature = "console")] @@ -33,16 +38,15 @@ pub struct Service { } #[derive(Debug)] -pub enum AdminEvent { - Command(String, Option>), - Reply(Option), - Notice(RoomMessageEventContent), +pub struct Command { + pub command: String, + pub reply_id: Option, } impl Service { #[must_use] pub fn build() -> Arc { - let (sender, receiver) = loole::unbounded(); + let (sender, receiver) = loole::bounded(COMMAND_QUEUE_LIMIT); Arc::new(Self { sender, receiver: Mutex::new(receiver), @@ -53,6 +57,18 @@ impl Service { }) } + pub async fn start_handler(self: &Arc) { + let self_ = Arc::clone(self); + let handle = services().server.runtime().spawn(async move { + self_ + .handler() + .await + .expect("Failed to initialize admin room handler"); + }); + + _ = self.handler_join.lock().await.insert(handle); + } + pub fn interrupt(&self) { #[cfg(feature = "console")] self.console.interrupt(); @@ -75,16 +91,39 @@ impl Service { } } - pub async fn start_handler(self: &Arc) { - let self_ = Arc::clone(self); - let handle = services().server.runtime().spawn(async move { - self_ - .handler() - .await - .expect("Failed to initialize admin room handler"); - }); + pub async fn send_text(&self, body: &str) { + self.send_message(RoomMessageEventContent::text_plain(body)) + .await; + } - _ = self.handler_join.lock().await.insert(handle); + pub async fn send_message(&self, message_content: RoomMessageEventContent) { + if let Ok(Some(room_id)) = Self::get_admin_room() { + let user_id = &services().globals.server_user; + respond_to_room(message_content, &room_id, user_id).await; + } + } + + pub async fn command(&self, command: String, reply_id: Option) { + self.send(Command { + command, + reply_id, + }) + .await; + } + + pub async fn command_in_place( + &self, command: String, reply_id: Option, + ) -> Result> { + self.process_command(Command { + command, + reply_id, + }) + .await + } + + async fn send(&self, message: Command) { + debug_assert!(!self.sender.is_closed(), "channel closed"); + self.sender.send_async(message).await.expect("message sent"); } async fn handler(self: &Arc) -> Result<()> { @@ -93,8 +132,8 @@ impl Service { loop { debug_assert!(!receiver.is_closed(), "channel closed"); tokio::select! { - event = receiver.recv_async() => match event { - Ok(event) => self.receive(event).await, + command = receiver.recv_async() => match command { + Ok(command) => self.handle_command(command).await, Err(_) => return Ok(()), }, sig = signals.recv() => match sig { @@ -105,48 +144,6 @@ impl Service { } } - pub async fn send_text(&self, body: &str) { - self.send_message(RoomMessageEventContent::text_plain(body)) - .await; - } - - pub async fn send_message(&self, message_content: RoomMessageEventContent) { - self.send(AdminEvent::Notice(message_content)).await; - } - - pub async fn command(&self, command: String, event_id: Option>) { - self.send(AdminEvent::Command(command, event_id)).await; - } - - pub async fn command_in_place( - &self, command: String, event_id: Option>, - ) -> Result> { - match self.handle(AdminEvent::Command(command, event_id)).await? { - AdminEvent::Reply(content) => Ok(content), - _ => Ok(None), - } - } - - async fn send(&self, message: AdminEvent) { - debug_assert!(!self.sender.is_full(), "channel full"); - debug_assert!(!self.sender.is_closed(), "channel closed"); - self.sender.send(message).expect("message sent"); - } - - async fn receive(&self, event: AdminEvent) { - if let Ok(AdminEvent::Reply(content)) = self.handle(event).await { - handle_response(content).await; - } - } - - async fn handle(&self, event: AdminEvent) -> Result { - if let Some(handle) = self.handle.lock().await.as_ref() { - handle(event).await - } else { - Err(Error::Err("Admin module is not loaded.".into())) - } - } - async fn handle_signal(&self, #[allow(unused_variables)] sig: &'static str) { #[cfg(feature = "console")] if sig == "SIGINT" && services().server.running() { @@ -154,6 +151,20 @@ impl Service { } } + async fn handle_command(&self, command: Command) { + if let Ok(Some(output)) = self.process_command(command).await { + handle_response(output).await; + } + } + + async fn process_command(&self, command: Command) -> CommandResult { + if let Some(handle) = self.handle.lock().await.as_ref() { + handle(command).await + } else { + Err(Error::Err("Admin module is not loaded.".into())) + } + } + /// Checks whether a given user is an admin of this server pub async fn user_is_admin(&self, user_id: &UserId) -> Result { if let Ok(Some(admin_room)) = Self::get_admin_room() { @@ -186,26 +197,28 @@ impl Service { } } -async fn handle_response(content: Option) { - if let Some(content) = content.as_ref() { - if let Some(Relation::Reply { - in_reply_to, - }) = content.relates_to.as_ref() - { - if let Ok(Some(pdu)) = services().rooms.timeline.get_pdu(&in_reply_to.event_id) { - let response_sender = if is_admin_room(&pdu.room_id) { - &services().globals.server_user - } else { - &pdu.sender - }; +async fn handle_response(content: RoomMessageEventContent) { + let Some(Relation::Reply { + in_reply_to, + }) = content.relates_to.as_ref() + else { + return; + }; - respond_to_room(content, &pdu.room_id, response_sender).await; - } - } - } + let Ok(Some(pdu)) = services().rooms.timeline.get_pdu(&in_reply_to.event_id) else { + return; + }; + + let response_sender = if is_admin_room(&pdu.room_id) { + &services().globals.server_user + } else { + &pdu.sender + }; + + respond_to_room(content, &pdu.room_id, response_sender).await; } -async fn respond_to_room(content: &RoomMessageEventContent, room_id: &RoomId, user_id: &UserId) { +async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, user_id: &UserId) { assert!( services() .admin @@ -218,7 +231,7 @@ async fn respond_to_room(content: &RoomMessageEventContent, room_id: &RoomId, us let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; let response_pdu = PduBuilder { event_type: TimelineEventType::RoomMessage, - content: to_raw_value(content).expect("event is valid, we just created it"), + content: to_raw_value(&content).expect("event is valid, we just created it"), unsigned: None, state_key: None, redacts: None, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index e8774cdc..ed369334 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -475,7 +475,7 @@ impl Service { if admin::is_admin_command(pdu, &body).await { services() .admin - .command(body, Some(pdu.event_id.clone())) + .command(body, Some((*pdu.event_id).into())) .await; } } From 48d96779590ed145e4301df5a7786937536fcccd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 00:09:32 +0000 Subject: [PATCH 0199/2091] rustfmt nightly/stable compats Signed-off-by: Jason Volk --- src/admin/room/room_moderation_commands.rs | 29 +++++++++++----------- src/core/config/proxy.rs | 4 +-- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 5989ff73..1876bf5e 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -111,15 +111,15 @@ async fn ban_room( user.ok().filter(|local_user| { user_is_local(local_user) // additional wrapped check here is to avoid adding remote users - // who are in the admin room to the list of local users (would fail auth check) + // who are in the admin room to the list of local users (would + // fail auth check) && (user_is_local(local_user) + // since this is a force operation, assume user is an admin + // if somehow this fails && services() .users .is_admin(local_user) - .unwrap_or(true)) // since this is a force - // operation, assume user - // is an admin if somehow - // this fails + .unwrap_or(true)) }) }) .collect::>() @@ -311,19 +311,17 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo .filter_map(|user| { user.ok().filter(|local_user| { local_user.server_name() == services().globals.server_name() - // additional wrapped check here is to avoid adding remote users - // who are in the admin room to the list of local users (would fail auth check) + // additional wrapped check here is to avoid adding remote + // users who are in the admin room to the list of local + // users (would fail auth check) && (local_user.server_name() == services().globals.server_name() + // since this is a force operation, assume user is an + // admin if somehow this fails && services() .users .is_admin(local_user) - .unwrap_or(true)) // since this is a - // force operation, - // assume user is - // an admin if - // somehow this - // fails + .unwrap_or(true)) }) }) .collect::>() @@ -344,8 +342,9 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo .filter_map(|user| { user.ok().filter(|local_user| { local_user.server_name() == services().globals.server_name() - // additional wrapped check here is to avoid adding remote users - // who are in the admin room to the list of local users (would fail auth check) + // additional wrapped check here is to avoid adding remote + // users who are in the admin room to the list of local + // users (would fail auth check) && (local_user.server_name() == services().globals.server_name() && !services() diff --git a/src/core/config/proxy.rs b/src/core/config/proxy.rs index a2d64fa6..d823e5e4 100644 --- a/src/core/config/proxy.rs +++ b/src/core/config/proxy.rs @@ -47,8 +47,8 @@ impl ProxyConfig { url, } => Some(Proxy::all(url)?), Self::ByDomain(proxies) => Some(Proxy::custom(move |url| { - proxies.iter().find_map(|proxy| proxy.for_url(url)).cloned() // first matching - // proxy + // first matching proxy + proxies.iter().find_map(|proxy| proxy.for_url(url)).cloned() })), }) } From 0923b6f4282445302613102ce61730fdaddddd21 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 00:10:53 +0000 Subject: [PATCH 0200/2091] fix use std::mem related lint Signed-off-by: Jason Volk --- src/service/rooms/pdu_metadata/data.rs | 4 ++-- src/service/rooms/read_receipt/data.rs | 6 +++--- src/service/rooms/threads/data.rs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index b5bea331..e0309172 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,4 +1,4 @@ -use std::{mem, sync::Arc}; +use std::{mem::size_of, sync::Arc}; use ruma::{EventId, RoomId, UserId}; @@ -44,7 +44,7 @@ impl Data for KeyValueDatabase { .iter_from(¤t, true) .take_while(move |(k, _)| k.starts_with(&prefix)) .map(move |(tofrom, _data)| { - let from = utils::u64_from_bytes(&tofrom[(mem::size_of::())..]) + let from = utils::u64_from_bytes(&tofrom[(size_of::())..]) .map_err(|_| Error::bad_database("Invalid count in tofrom_relation."))?; let mut pduid = shortroomid.to_be_bytes().to_vec(); diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 00d03e8c..702efd98 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -1,4 +1,4 @@ -use std::mem; +use std::mem::size_of; use ruma::{ events::{receipt::ReceiptEvent, AnySyncEphemeralRoomEvent}, @@ -86,10 +86,10 @@ impl Data for KeyValueDatabase { .iter_from(&first_possible_edu, false) .take_while(move |(k, _)| k.starts_with(&prefix2)) .map(move |(k, v)| { - let count = utils::u64_from_bytes(&k[prefix.len()..prefix.len() + mem::size_of::()]) + let count = utils::u64_from_bytes(&k[prefix.len()..prefix.len() + size_of::()]) .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?; let user_id = UserId::parse( - utils::string_from_bytes(&k[prefix.len() + mem::size_of::() + 1..]) + utils::string_from_bytes(&k[prefix.len() + size_of::() + 1..]) .map_err(|_| Error::bad_database("Invalid readreceiptid userid bytes in db."))?, ) .map_err(|_| Error::bad_database("Invalid readreceiptid userid in db."))?; diff --git a/src/service/rooms/threads/data.rs b/src/service/rooms/threads/data.rs index 3974ca02..8ed5c770 100644 --- a/src/service/rooms/threads/data.rs +++ b/src/service/rooms/threads/data.rs @@ -1,4 +1,4 @@ -use std::mem; +use std::mem::size_of; use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId}; @@ -35,7 +35,7 @@ impl Data for KeyValueDatabase { .iter_from(¤t, true) .take_while(move |(k, _)| k.starts_with(&prefix)) .map(move |(pduid, _users)| { - let count = utils::u64_from_bytes(&pduid[(mem::size_of::())..]) + let count = utils::u64_from_bytes(&pduid[(size_of::())..]) .map_err(|_| Error::bad_database("Invalid pduid in threadid_userids."))?; let mut pdu = services() .rooms From 3872ae80f75e754e265e96d9ef3aa072bb36a013 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 00:12:56 +0000 Subject: [PATCH 0201/2091] fix PanicInfo deprecation Signed-off-by: Jason Volk --- src/core/debug.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/debug.rs b/src/core/debug.rs index 207f08fa..ad12099b 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] // this is a developer's toolbox -use std::{panic, panic::PanicInfo}; +use std::panic; /// Export all of the ancillary tools from here as well. pub use crate::utils::debug::*; @@ -59,7 +59,8 @@ pub fn set_panic_trap() { } #[inline(always)] -fn panic_handler(info: &PanicInfo<'_>, next: &dyn Fn(&PanicInfo<'_>)) { +#[allow(deprecated_in_future)] +fn panic_handler(info: &panic::PanicInfo<'_>, next: &dyn Fn(&panic::PanicInfo<'_>)) { trap(); next(info); } From 83565007bba8501bf6783a9fbf08b59481f042ee Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 00:25:49 +0000 Subject: [PATCH 0202/2091] fix lint group priority Signed-off-by: Jason Volk --- Cargo.toml | 45 +++++++++++++++++++-------------------------- 1 file changed, 19 insertions(+), 26 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 801aa5a5..da0c06fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -741,34 +741,27 @@ significant_drop_tightening = { level = "allow", priority = 1 } # TODO pedantic = "warn" ## some sadness -cast_possible_truncation = "allow" -cast_precision_loss = "allow" -cast_sign_loss = "allow" -doc_markdown = "allow" -error_impl_error = "allow" -expect_used = "allow" -if_not_else = "allow" -if_then_some_else_none = "allow" -implicit_return = "allow" -inline_always = "allow" -map_err_ignore = "allow" -missing_docs_in_private_items = "allow" -missing_errors_doc = "allow" -missing_panics_doc = "allow" -mod_module_files = "allow" -module_name_repetitions = "allow" -multiple_inherent_impl = "allow" -no_effect_underscore_binding = "allow" -ref_patterns = "allow" -same_name_method = "allow" +cast_possible_truncation = { level = "allow", priority = 1 } +cast_precision_loss = { level = "allow", priority = 1 } +cast_sign_loss = { level = "allow", priority = 1 } +doc_markdown = { level = "allow", priority = 1 } +error_impl_error = { level = "allow", priority = 1 } +expect_used = { level = "allow", priority = 1 } +if_not_else = { level = "allow", priority = 1 } +if_then_some_else_none = { level = "allow", priority = 1 } +implicit_return = { level = "allow", priority = 1 } +inline_always = { level = "allow", priority = 1 } +map_err_ignore = { level = "allow", priority = 1 } +missing_docs_in_private_items = { level = "allow", priority = 1 } +missing_errors_doc = { level = "allow", priority = 1 } +missing_panics_doc = { level = "allow", priority = 1 } +mod_module_files = { level = "allow", priority = 1 } +module_name_repetitions = { level = "allow", priority = 1 } +no_effect_underscore_binding = { level = "allow", priority = 1 } similar_names = { level = "allow", priority = 1 } -single_call_fn = "allow" -string_add = "allow" -string_slice = "allow" struct_field_names = { level = "allow", priority = 1 } unnecessary_wraps = { level = "allow", priority = 1 } unused_async = { level = "allow", priority = 1 } -unwrap_used = "allow" ################### perf = "warn" @@ -822,10 +815,10 @@ style = "warn" ## some sadness # trivial assertions are quite alright -assertions_on_constants = "allow" +assertions_on_constants = { level = "allow", priority = 1 } ################### suspicious = "warn" ## some sadness -let_underscore_future = "allow" +let_underscore_future = { level = "allow", priority = 1 } From 30e7298dd70cc9fb9c2a60844beb60aa65f5ce3c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 00:36:49 +0000 Subject: [PATCH 0203/2091] fix doc-lazy-continuation Signed-off-by: Jason Volk --- src/admin/debug/mod.rs | 7 +++---- src/api/client/context.rs | 3 +-- src/api/client/message.rs | 3 +-- src/api/client/profile.rs | 10 +++++----- src/api/client/sync.rs | 9 +++------ src/api/client/user_directory.rs | 3 +-- src/api/server/key.rs | 6 ++---- 7 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index afcc49b0..b993872b 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -35,8 +35,8 @@ pub(crate) enum DebugCommand { }, /// - Attempts to retrieve a PDU from a remote server. Inserts it into our - /// database/timeline if found and we do not have this PDU already - /// (following normal event auth rules, handles it as an incoming PDU). + /// database/timeline if found and we do not have this PDU already + /// (following normal event auth rules, handles it as an incoming PDU). GetRemotePdu { /// An event ID (a $ followed by the base64 reference hash) event_id: Box, @@ -47,8 +47,7 @@ pub(crate) enum DebugCommand { }, /// - Same as `get-remote-pdu` but accepts a codeblock newline delimited - /// list - /// of PDUs and a single server to fetch from + /// list of PDUs and a single server to fetch from GetRemotePduList { /// Argument for us to attempt to fetch all the events from the /// specified remote server. diff --git a/src/api/client/context.rs b/src/api/client/context.rs index a137bf8d..2ca161c9 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -13,8 +13,7 @@ use crate::{services, Error, Result, Ruma}; /// Allows loading room history around an event. /// /// - Only works if the user is joined (TODO: always allow, but only show events -/// if the user was -/// joined, depending on history_visibility) +/// if the user was joined, depending on history_visibility) pub(crate) async fn get_context_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); diff --git a/src/api/client/message.rs b/src/api/client/message.rs index a57defb2..9548f084 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -107,8 +107,7 @@ pub(crate) async fn send_message_event_route( /// Allows paginating through room history. /// /// - Only works if the user is joined (TODO: always allow, but only show events -/// where the user was -/// joined, depending on `history_visibility`) +/// where the user was joined, depending on `history_visibility`) pub(crate) async fn get_message_events_route( body: Ruma, ) -> Result { diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index b71280f1..2b23344e 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -50,8 +50,8 @@ pub(crate) async fn set_displayname_route( /// /// Returns the displayname of the user. /// -/// - If user is on another server and we do not have a local copy already -/// fetch displayname over federation +/// - If user is on another server and we do not have a local copy already fetch +/// displayname over federation pub(crate) async fn get_displayname_route( body: Ruma, ) -> Result { @@ -140,8 +140,8 @@ pub(crate) async fn set_avatar_url_route( /// /// Returns the `avatar_url` and `blurhash` of the user. /// -/// - If user is on another server and we do not have a local copy already -/// fetch `avatar_url` and blurhash over federation +/// - If user is on another server and we do not have a local copy already fetch +/// `avatar_url` and blurhash over federation pub(crate) async fn get_avatar_url_route( body: Ruma, ) -> Result { @@ -199,7 +199,7 @@ pub(crate) async fn get_avatar_url_route( /// Returns the displayname, avatar_url and blurhash of the user. /// /// - If user is on another server and we do not have a local copy already, -/// fetch profile over federation. +/// fetch profile over federation. pub(crate) async fn get_profile_route(body: Ruma) -> Result { if !user_is_local(&body.user_id) { // Create and update our local copy of the user diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 84cf11b6..9e9a9ffd 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -36,8 +36,7 @@ use crate::{service::pdu::EventHash, services, utils, Error, PduEvent, Result, R /// Synchronize the client's state with the latest state on the server. /// /// - This endpoint takes a `since` parameter which should be the `next_batch` -/// value from a -/// previous request for incremental syncs. +/// value from a previous request for incremental syncs. /// /// Calling this endpoint without a `since` parameter returns: /// - Some of the most recent events of each timeline @@ -49,11 +48,9 @@ use crate::{service::pdu::EventHash, services, utils, Error, PduEvent, Result, R /// returns: For joined rooms: /// - Some of the most recent events of each timeline that happened after since /// - If user joined the room after since: All state events (unless lazy loading -/// is activated) and -/// all device list updates in that room +/// is activated) and all device list updates in that room /// - If the user was already in the room: A list of all events that are in the -/// state now, but were -/// not in the state at `since` +/// state now, but were not in the state at `since` /// - If the state we send contains a member event: Joined and invited member /// counts, heroes /// - Device list updates that happened after `since` diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index 4d9947f8..ca5f834a 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -13,8 +13,7 @@ use crate::{services, Result, Ruma}; /// Searches all known users for a match. /// /// - Hides any local users that aren't in any public rooms (i.e. those that -/// have the join rule set to public) -/// and don't share a room with the sender +/// have the join rule set to public) and don't share a room with the sender pub(crate) async fn search_users_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let limit = usize::try_from(body.limit).unwrap_or(10); // default limit is 10 diff --git a/src/api/server/key.rs b/src/api/server/key.rs index ddc8e8ef..27d86639 100644 --- a/src/api/server/key.rs +++ b/src/api/server/key.rs @@ -20,8 +20,7 @@ use crate::{services, Result}; /// Gets the public signing keys of this server. /// /// - Matrix does not support invalidating public keys, so the key returned by -/// this will be valid -/// forever. +/// this will be valid forever. // Response type for this endpoint is Json because we need to calculate a // signature for the response pub(crate) async fn get_server_keys_route() -> Result { @@ -71,6 +70,5 @@ pub(crate) async fn get_server_keys_route() -> Result { /// Gets the public signing keys of this server. /// /// - Matrix does not support invalidating public keys, so the key returned by -/// this will be valid -/// forever. +/// this will be valid forever. pub(crate) async fn get_server_keys_deprecated_route() -> impl IntoResponse { get_server_keys_route().await } From 08f2b8579c0b15e9cb7b383ade71a8a7be810e77 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 01:39:14 +0000 Subject: [PATCH 0204/2091] abstract shutdown/reload functionality. Signed-off-by: Jason Volk --- src/core/server.rs | 34 +++++++++++++++++++++++++++++++++- src/main/main.rs | 30 ++++++++++++------------------ src/main/mods.rs | 1 + src/router/request.rs | 4 ++-- src/router/run.rs | 1 - src/service/services.rs | 5 +---- 6 files changed, 49 insertions(+), 26 deletions(-) diff --git a/src/core/server.rs b/src/core/server.rs index 95eb7e01..ca62b64c 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -5,7 +5,7 @@ use std::{ use tokio::{runtime, sync::broadcast}; -use crate::{config::Config, log}; +use crate::{config::Config, log, Error, Result}; /// Server runtime state; public portion pub struct Server { @@ -59,6 +59,38 @@ impl Server { } } + pub fn reload(&self) -> Result<()> { + if cfg!(not(conduit_mods)) { + return Err(Error::Err("Reloading not enabled".into())); + } + + if self.reloading.swap(true, Ordering::AcqRel) { + return Err(Error::Err("Reloading already in progress".into())); + } + + if self.stopping.swap(true, Ordering::AcqRel) { + return Err(Error::Err("Shutdown already in progress".into())); + } + + self.signal("SIGINT") + } + + pub fn shutdown(&self) -> Result<()> { + if self.stopping.swap(true, Ordering::AcqRel) { + return Err(Error::Err("Shutdown already in progress".into())); + } + + self.signal("SIGTERM") + } + + pub fn signal(&self, sig: &'static str) -> Result<()> { + if let Err(e) = self.signal.send(sig) { + return Err(Error::Err(format!("Failed to send signal: {e}"))); + } + + Ok(()) + } + #[inline] pub fn runtime(&self) -> &runtime::Handle { self.runtime diff --git a/src/main/main.rs b/src/main/main.rs index 6e1bfe38..566b0d67 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -4,11 +4,7 @@ mod server; extern crate conduit_core as conduit; -use std::{ - cmp, - sync::{atomic::Ordering, Arc}, - time::Duration, -}; +use std::{cmp, sync::Arc, time::Duration}; use conduit::{debug_error, debug_info, error, trace, utils::available_parallelism, warn, Error, Result}; use server::Server; @@ -107,7 +103,7 @@ async fn signal(server: Arc) { use unix::SignalKind; const CONSOLE: bool = cfg!(feature = "console"); - const RELOADING: bool = cfg!(all(unix, debug_assertions, not(CONSOLE))); + const RELOADING: bool = cfg!(all(conduit_mods, not(CONSOLE))); let mut quit = unix::signal(SignalKind::quit()).expect("SIGQUIT handler"); let mut term = unix::signal(SignalKind::terminate()).expect("SIGTERM handler"); @@ -120,19 +116,17 @@ async fn signal(server: Arc) { _ = term.recv() => { sig = "SIGTERM"; }, } - // Indicate the SIGINT is requesting a hot-reload. - if RELOADING && sig == "SIGINT" { - server.server.reloading.store(true, Ordering::Release); - } - - // Indicate the signal is requesting a shutdown - if matches!(sig, "SIGQUIT" | "SIGTERM") || (!CONSOLE && sig == "SIGINT") { - server.server.stopping.store(true, Ordering::Release); - } - warn!("Received {sig}"); - if let Err(e) = server.server.signal.send(sig) { - debug_error!("signal channel: {e}"); + let result = if RELOADING && sig == "SIGINT" { + server.server.reload() + } else if matches!(sig, "SIGQUIT" | "SIGTERM") || (!CONSOLE && sig == "SIGINT") { + server.server.shutdown() + } else { + server.server.signal(sig) + }; + + if let Err(e) = result { + debug_error!(?sig, "signal: {e}"); } } } diff --git a/src/main/mods.rs b/src/main/mods.rs index 125cd54a..f39527f7 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -39,6 +39,7 @@ pub(crate) async fn run(server: &Arc, starts: bool) -> Result<(bool, boo return Err(error); } } + server.server.stopping.store(false, Ordering::Release); let run = main_mod.get::("run")?; if let Err(error) = run(&server.server).await { error!("Running server: {error}"); diff --git a/src/router/request.rs b/src/router/request.rs index 0693e671..2bdd06f8 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -13,7 +13,7 @@ use tracing::{debug, error, trace}; pub(crate) async fn spawn( State(server): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { - if server.stopping.load(Ordering::Relaxed) { + if !server.running() { debug_warn!("unavailable pending shutdown"); return Err(StatusCode::SERVICE_UNAVAILABLE); } @@ -35,7 +35,7 @@ pub(crate) async fn spawn( pub(crate) async fn handle( State(server): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { - if server.stopping.load(Ordering::Relaxed) { + if !server.running() { debug_warn!( method = %req.method(), uri = %req.uri(), diff --git a/src/router/run.rs b/src/router/run.rs index e9876ef3..fb59c797 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -25,7 +25,6 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { _ = services().admin.handle.lock().await.insert(admin::handle); // Setup shutdown/signal handling - server.stopping.store(false, Ordering::Release); let handle = ServerHandle::new(); let (tx, _) = broadcast::channel::<()>(1); let sigs = server diff --git a/src/service/services.rs b/src/service/services.rs index d9805010..e1c27d52 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,6 +1,6 @@ use std::{ collections::{BTreeMap, HashMap}, - sync::{atomic, Arc, Mutex as StdMutex}, + sync::{Arc, Mutex as StdMutex}, }; use conduit::{debug_info, Result, Server}; @@ -301,8 +301,6 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} pub async fn interrupt(&self) { trace!("Interrupting services..."); - self.server.stopping.store(true, atomic::Ordering::Release); - self.sending.interrupt(); self.presence.interrupt(); self.admin.interrupt(); @@ -310,7 +308,6 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} trace!("Services interrupt complete."); } - #[tracing::instrument(skip_all)] pub async fn stop(&self) { info!("Shutting down services"); self.interrupt().await; From 3d3d63fdf44549d8fe997906900f42fbf51ff0df Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 01:44:41 +0000 Subject: [PATCH 0205/2091] admin commands for shutdown/reload Signed-off-by: Jason Volk --- src/admin/mod.rs | 2 ++ src/admin/server/mod.rs | 15 +++++++++++---- src/admin/server/server_commands.rs | 15 +++++++++++++++ src/service/admin/console.rs | 2 +- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 1832cc9b..33158b5c 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1,3 +1,5 @@ +#![allow(clippy::wildcard_imports)] + pub(crate) mod appservice; pub(crate) mod debug; pub(crate) mod federation; diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index e591f7c5..254735a3 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -3,10 +3,7 @@ pub(crate) mod server_commands; use clap::Subcommand; use ruma::events::room::message::RoomMessageEventContent; -use self::server_commands::{ - backup_database, clear_database_caches, clear_service_caches, list_backups, list_database_files, memory_usage, - show_config, uptime, -}; +use self::server_commands::*; use crate::Result; #[cfg_attr(test, derive(Debug))] @@ -42,6 +39,13 @@ pub(crate) enum ServerCommand { /// - List database files ListDatabaseFiles, + + #[cfg(conduit_mods)] + /// - Hot-reload the server + Reload, + + /// - Shutdown the server + Shutdown, } pub(crate) async fn process(command: ServerCommand, body: Vec<&str>) -> Result { @@ -58,5 +62,8 @@ pub(crate) async fn process(command: ServerCommand, body: Vec<&str>) -> Result list_backups(body).await?, ServerCommand::BackupDatabase => backup_database(body).await?, ServerCommand::ListDatabaseFiles => list_database_files(body).await?, + #[cfg(conduit_mods)] + ServerCommand::Reload => reload(body).await?, + ServerCommand::Shutdown => shutdown(body).await?, }) } diff --git a/src/admin/server/server_commands.rs b/src/admin/server/server_commands.rs index d80cc3d7..c9daedc4 100644 --- a/src/admin/server/server_commands.rs +++ b/src/admin/server/server_commands.rs @@ -1,3 +1,4 @@ +use conduit::warn; use ruma::events::room::message::RoomMessageEventContent; use crate::{services, Result}; @@ -96,3 +97,17 @@ pub(crate) async fn list_database_files(_body: Vec<&str>) -> Result) -> Result { + services().server.reload()?; + + Ok(RoomMessageEventContent::notice_plain(String::new())) +} + +pub(crate) async fn shutdown(_body: Vec<&str>) -> Result { + warn!("shutdown command"); + services().server.shutdown()?; + + Ok(RoomMessageEventContent::notice_plain(String::new())) +} diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 66faf2c1..13cfdb27 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -73,7 +73,7 @@ impl Console { #[tracing::instrument(skip_all, name = "console")] async fn worker(self: Arc) { - loop { + while services().server.running() { let mut input = self.input.lock().await; let suppression = log::Suppress::new(&services().server); From 483f0a9c865f7f1fd2d16c3d5b24e73a3e2c2088 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 02:10:47 +0000 Subject: [PATCH 0206/2091] add admin command for admin room notices Signed-off-by: Jason Volk --- src/admin/server/mod.rs | 8 ++++++++ src/admin/server/server_commands.rs | 7 +++++++ src/service/admin/mod.rs | 2 +- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 254735a3..584d3234 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -40,6 +40,11 @@ pub(crate) enum ServerCommand { /// - List database files ListDatabaseFiles, + /// - Send a message to the admin room. + AdminNotice { + message: Vec, + }, + #[cfg(conduit_mods)] /// - Hot-reload the server Reload, @@ -62,6 +67,9 @@ pub(crate) async fn process(command: ServerCommand, body: Vec<&str>) -> Result list_backups(body).await?, ServerCommand::BackupDatabase => backup_database(body).await?, ServerCommand::ListDatabaseFiles => list_database_files(body).await?, + ServerCommand::AdminNotice { + message, + } => admin_notice(body, message).await?, #[cfg(conduit_mods)] ServerCommand::Reload => reload(body).await?, ServerCommand::Shutdown => shutdown(body).await?, diff --git a/src/admin/server/server_commands.rs b/src/admin/server/server_commands.rs index c9daedc4..79e02e17 100644 --- a/src/admin/server/server_commands.rs +++ b/src/admin/server/server_commands.rs @@ -98,6 +98,13 @@ pub(crate) async fn list_database_files(_body: Vec<&str>) -> Result, message: Vec) -> Result { + let message = message.join(" "); + services().admin.send_text(&message).await; + + Ok(RoomMessageEventContent::notice_plain("Notice was sent to #admins")) +} + #[cfg(conduit_mods)] pub(crate) async fn reload(_body: Vec<&str>) -> Result { services().server.reload()?; diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index ddc3606f..0c949937 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -92,7 +92,7 @@ impl Service { } pub async fn send_text(&self, body: &str) { - self.send_message(RoomMessageEventContent::text_plain(body)) + self.send_message(RoomMessageEventContent::text_markdown(body)) .await; } From 1d1b1644e9a3fd31782eff4a7cc59567e657933e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 02:14:19 +0000 Subject: [PATCH 0207/2091] add admin debug echo command Signed-off-by: Jason Volk --- src/admin/debug/debug_commands.rs | 6 ++++++ src/admin/debug/mod.rs | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/debug_commands.rs index 0b9ff62c..681dc6fb 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/debug_commands.rs @@ -21,6 +21,12 @@ use tracing_subscriber::EnvFilter; use crate::api::client::validate_and_add_event_id; +pub(crate) async fn echo(_body: Vec<&str>, message: Vec) -> Result { + let message = message.join(" "); + + Ok(RoomMessageEventContent::notice_plain(message)) +} + pub(crate) async fn get_auth_chain(_body: Vec<&str>, event_id: Box) -> Result { let event_id = Arc::::from(event_id); if let Some(event) = services().rooms.timeline.get_pdu_json(&event_id)? { diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index b993872b..52c49c20 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -3,7 +3,7 @@ use debug_commands::{first_pdu_in_room, force_set_room_state_from_server, latest use ruma::{events::room::message::RoomMessageEventContent, EventId, RoomId, ServerName}; use self::debug_commands::{ - change_log_level, force_device_list_updates, get_auth_chain, get_pdu, get_remote_pdu, get_remote_pdu_list, + change_log_level, echo, force_device_list_updates, get_auth_chain, get_pdu, get_remote_pdu, get_remote_pdu_list, get_room_state, memory_stats, parse_pdu, ping, resolve_true_destination, sign_json, verify_json, }; use crate::Result; @@ -13,6 +13,11 @@ pub(crate) mod debug_commands; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] pub(crate) enum DebugCommand { + /// - Echo input of admin command + Echo { + message: Vec, + }, + /// - Get the auth_chain of a PDU GetAuthChain { /// An event ID (the $ character followed by the base64 reference hash) @@ -160,6 +165,9 @@ pub(crate) enum DebugCommand { pub(crate) async fn process(command: DebugCommand, body: Vec<&str>) -> Result { Ok(match command { + DebugCommand::Echo { + message, + } => echo(body, message).await?, DebugCommand::GetAuthChain { event_id, } => get_auth_chain(body, event_id).await?, From a22524496d6e6f0fa8e9730c6deb6dca83e07fbd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 05:17:15 +0000 Subject: [PATCH 0208/2091] console command interruption Signed-off-by: Jason Volk --- src/service/admin/console.rs | 49 +++++++++++++++++++++++------------- src/service/admin/mod.rs | 1 + 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 13cfdb27..ba0d7ecc 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -1,7 +1,8 @@ #![cfg(feature = "console")] use std::sync::Arc; -use conduit::{error, log, trace}; +use conduit::{debug, defer, error, log, trace}; +use futures_util::future::{AbortHandle, Abortable}; use ruma::events::room::message::RoomMessageEventContent; use rustyline::{error::ReadlineError, history, Editor}; use termimad::MadSkin; @@ -12,6 +13,7 @@ use crate::services; pub struct Console { join: Mutex>>, input: Mutex>, + abort: std::sync::Mutex>, output: MadSkin, } @@ -49,6 +51,7 @@ impl Console { Arc::new(Self { join: None.into(), input: Mutex::new(input), + abort: None.into(), output, }) } @@ -62,8 +65,6 @@ impl Console { } } - pub fn interrupt(self: &Arc) { Self::handle_interrupt(); } - #[allow(clippy::let_underscore_must_use)] pub async fn close(self: &Arc) { if let Some(join) = self.join.lock().await.take() { @@ -71,8 +72,16 @@ impl Console { } } + pub fn interrupt(self: &Arc) { + if let Some(abort) = self.abort.lock().expect("locked").take() { + debug!("Interrupting console command..."); + abort.abort(); + } + } + #[tracing::instrument(skip_all, name = "console")] async fn worker(self: Arc) { + debug!("session starting"); while services().server.running() { let mut input = self.input.lock().await; @@ -80,25 +89,39 @@ impl Console { let line = tokio::task::block_in_place(|| input.readline("uwu> ")); drop(suppression); + trace!(?line, "input"); match line { - Ok(string) => self.handle(string).await, + Ok(string) => self.clone().handle(string).await, Err(e) => match e { - ReadlineError::Eof => break, - ReadlineError::Interrupted => Self::handle_interrupt(), - ReadlineError::WindowResized => Self::handle_winch(), + ReadlineError::Interrupted | ReadlineError::Eof => break, + ReadlineError::WindowResized => continue, _ => error!("console: {e:?}"), }, } } + debug!("session ending"); self.join.lock().await.take(); } - async fn handle(&self, line: String) { + #[allow(clippy::let_underscore_must_use)] + async fn handle(self: Arc, line: String) { if line.is_empty() { return; } + let future = self.clone().process(line); + let (abort, abort_reg) = AbortHandle::new_pair(); + let future = Abortable::new(future, abort_reg); + _ = self.abort.lock().expect("locked").insert(abort); + defer! {{ + _ = self.abort.lock().expect("locked").take(); + }} + + _ = future.await; + } + + async fn process(self: Arc, line: String) { match services().admin.command_in_place(line, None).await { Ok(Some(content)) => self.output(content).await, Err(e) => error!("processing command: {e}"), @@ -106,16 +129,8 @@ impl Console { } } - async fn output(&self, output_content: RoomMessageEventContent) { + async fn output(self: Arc, output_content: RoomMessageEventContent) { let output = self.output.term_text(output_content.body()); println!("{output}"); } - - fn handle_interrupt() { - trace!("interrupted"); - } - - fn handle_winch() { - trace!("winch"); - } } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 0c949937..3f983b75 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -147,6 +147,7 @@ impl Service { async fn handle_signal(&self, #[allow(unused_variables)] sig: &'static str) { #[cfg(feature = "console")] if sig == "SIGINT" && services().server.running() { + self.console.interrupt(); self.console.start().await; } } From 3596fe0e18dc0283d9f52eab39c590a670223c0d Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 16:52:28 -0400 Subject: [PATCH 0209/2091] ci: dont let renovate publish docker images Signed-off-by: strawberry --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6f6eb0b..e0e77118 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -255,7 +255,7 @@ jobs: name: Docker publish runs-on: ubuntu-latest needs: build - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.actor != 'renovate' env: DOCKER_ARM64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 DOCKER_AMD64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 From 169617ab871a74f30d505b648eaf093f2ec62b78 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 16:53:28 -0400 Subject: [PATCH 0210/2091] ci: remove erroneous `DIRENV_DEVSHELL` engage already sets `DIRENV_DEVSHELL` as necessary Signed-off-by: strawberry --- .github/workflows/ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0e77118..a7382a8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,9 +35,6 @@ env: # Custom nix binary cache if fork is being used ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} - # Use the all-features devshell instead of default, to ensure that features - # match between nix and cargo - DIRENV_DEVSHELL: all-features # Get error output from nix that we can actually use NIX_CONFIG: show-trace = true From ce4517269be0f796a346a035aae43ca65ff9b4d6 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 17:01:32 -0400 Subject: [PATCH 0211/2091] ci: output complement diff results to summary again Signed-off-by: strawberry --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a7382a8d..ed4547db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,9 +139,6 @@ jobs: - name: Diff Complement results with checked-in repo results run: | diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log) - - - name: Add Complement diff result to Job Summary - run: | echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY echo '```diff' >> $GITHUB_STEP_SUMMARY tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY @@ -154,7 +151,7 @@ jobs: echo '# ✅ completed suwuccessfully' >> $GITHUB_STEP_SUMMARY else echo '```' >> $GITHUB_STEP_SUMMARY - tail -n 20 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + tail -n 40 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY fi From cad1d34611442c9f6e1c5dec5b4ca66085e4b1c3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 17:11:25 -0400 Subject: [PATCH 0212/2091] ci: disable detsys telemetry, use our binary cache URL as upstream Signed-off-by: strawberry --- .github/workflows/ci.yml | 3 +++ .github/workflows/documentation.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed4547db..d1da9211 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,9 @@ jobs: - name: Configure Magic Nix Cache uses: DeterminateSystems/magic-nix-cache-action@main + with: + diagnostic-endpoint: "" + upstream-cache: "https://attic.kennel.juneis.dog" - name: Apply Nix binary cache configuration run: | diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 1759ab32..4dbd34d3 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -73,6 +73,9 @@ jobs: - name: Pop/push Magic Nix Cache uses: DeterminateSystems/magic-nix-cache-action@main + with: + diagnostic-endpoint: "" + upstream-cache: "https://attic.kennel.juneis.dog" - name: Configure `nix-direnv` run: | From e83b426caf76c0251bcb3aecb42aa1f51871af3f Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 17:13:04 -0400 Subject: [PATCH 0213/2091] ci: set `SOURCE_DATE_EPOCH` for at least debian pkgs Signed-off-by: strawberry --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d1da9211..aa9f28a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -209,6 +209,7 @@ jobs: - name: Build static ${{ matrix.target }} run: | CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) bin/nix-build-and-cache just .#static-${{ matrix.target }} mkdir -v -p target/release/ From 4312ac91c1252fa795a74ab206285589e14ee26e Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 17:16:34 -0400 Subject: [PATCH 0214/2091] ci: set `NIX_CONFIG` for documentation workflow too Signed-off-by: strawberry --- .github/workflows/documentation.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 4dbd34d3..168ec6c8 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -19,6 +19,8 @@ env: # Custom nix binary cache if fork is being used ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} + # Get error output from nix that we can actually use + NIX_CONFIG: show-trace = true # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. From a589a34d15e855523d6f65e1025d33d4593b8f3b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 17:31:09 -0400 Subject: [PATCH 0215/2091] ci: unify/simplify documentation and ci nix steps Signed-off-by: strawberry --- .github/workflows/ci.yml | 3 ++ .github/workflows/documentation.yml | 60 ++++++++++++++--------------- 2 files changed, 31 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa9f28a0..b9e3233c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,9 @@ jobs: - name: Install Nix uses: DeterminateSystems/nix-installer-action@main + with: + diagnostic-endpoint: "" + extra-conf: "experimental-features = nix-command flakes" - name: Enable Cachix binary cache run: | diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 168ec6c8..da5b383a 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -31,7 +31,6 @@ concurrency: jobs: docs: name: Documentation and GitHub Pages - runs-on: ubuntu-latest permissions: @@ -50,52 +49,49 @@ jobs: if: github.event_name != 'pull_request' uses: actions/configure-pages@v5 - - name: Install Nix (with flakes and nix-command enabled) - uses: cachix/install-nix-action@v27 + - name: Install Nix + uses: DeterminateSystems/nix-installer-action@main with: - nix_path: nixpkgs=channel:nixos-unstable + diagnostic-endpoint: "" + extra-conf: "experimental-features = nix-command flakes" - # Add `nix-community`, Crane, upstream Conduit, and conduwuit binary caches - extra_nix_config: | - experimental-features = nix-command flakes - extra-substituters = https://nix-community.cachix.org - extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= - extra-substituters = https://crane.cachix.org - extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= - extra-substituters = https://attic.kennel.juneis.dog/conduit - extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= - extra-substituters = https://attic.kennel.juneis.dog/conduwuit - extra-trusted-public-keys = conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= - - - name: Add alternative Nix binary caches if specified - if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }} + - name: Enable Cachix binary cache run: | - echo "extra-substituters = ${{ env.ATTIC_ENDPOINT }}" >> /etc/nix/nix.conf - echo "extra-trusted-public-keys = ${{ env.ATTIC_PUBLIC_KEY }}" >> /etc/nix/nix.conf + nix profile install nixpkgs#cachix + cachix use crane + cachix use nix-community - - name: Pop/push Magic Nix Cache + - name: Configure Magic Nix Cache uses: DeterminateSystems/magic-nix-cache-action@main with: diagnostic-endpoint: "" upstream-cache: "https://attic.kennel.juneis.dog" - - name: Configure `nix-direnv` + - name: Apply Nix binary cache configuration run: | - echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + sudo tee -a /etc/nix/nix.conf > /dev/null <" -iA direnv -iA nix-direnv + - name: Use alternative Nix binary caches if specified + if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }} + run: | + sudo tee -a /etc/nix/nix.conf > /dev/null < "$HOME/.direnvrc" + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + direnv allow + nix develop --command true - name: Cache CI dependencies run: | - ./bin/nix-build-and-cache ci + bin/nix-build-and-cache ci - name: Build documentation (book) run: | From a543bec1a5595cb8f9f8f4b42d323a2c18143ce4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 17:32:51 -0400 Subject: [PATCH 0216/2091] ci: add conduwuit cachix nix binary cache Signed-off-by: strawberry --- .github/workflows/ci.yml | 6 ++++-- .github/workflows/documentation.yml | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9e3233c..dfecd34f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,8 @@ env: TERM: ansi # Publishing to my nix binary cache ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + # conduwuit.cachix.org + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} # Just in case incremental is still being set to true, speeds up CI CARGO_INCREMENTAL: 0 # Custom nix binary cache if fork is being used @@ -83,8 +85,8 @@ jobs: - name: Apply Nix binary cache configuration run: | sudo tee -a /etc/nix/nix.conf > /dev/null < /dev/null < Date: Sat, 15 Jun 2024 21:24:25 -0400 Subject: [PATCH 0217/2091] nix: support pushing to conduwuit.cachix.org Signed-off-by: strawberry --- bin/nix-build-and-cache | 9 +- flake.lock | 355 +++++++++++++++++++++++++++++++++++++++- flake.nix | 1 + 3 files changed, 361 insertions(+), 4 deletions(-) diff --git a/bin/nix-build-and-cache b/bin/nix-build-and-cache index e466bc83..3e39e91f 100755 --- a/bin/nix-build-and-cache +++ b/bin/nix-build-and-cache @@ -52,7 +52,7 @@ just() { "${ATTIC_ENDPOINT:-https://attic.kennel.juneis.dog/conduwuit}" \ "$ATTIC_TOKEN" - # Upload them to Attic (conduwuit store) + # Upload them to Attic (conduwuit store) and Cachix # # Use `xargs` and a here-string because something would probably explode if # several thousand arguments got passed to a command at once. Hopefully no @@ -61,6 +61,12 @@ just() { IFS=$'\n' nix shell --inputs-from "$toplevel" attic -c xargs \ attic push conduwuit <<< "${cache[*]}" + + # push to cachix if available + if [ "$CACHIX_AUTH_TOKEN" ]; then + nix shell --inputs-from "$toplevel" cachix -c xargs \ + cachix push conduwuit <<< "${cache[*]}" + fi ) } @@ -73,6 +79,7 @@ ci() { "$toplevel#devShells.x86_64-linux.default" "$toplevel#devShells.x86_64-linux.all-features" attic#default + cachix#default nixpkgs#direnv nixpkgs#jq nixpkgs#nix-direnv diff --git a/flake.lock b/flake.lock index 0a4efdbe..7cd38d07 100644 --- a/flake.lock +++ b/flake.lock @@ -23,6 +23,28 @@ "type": "github" } }, + "cachix": { + "inputs": { + "devenv": "devenv", + "flake-compat": "flake-compat_3", + "nixpkgs": "nixpkgs_3", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1717420532, + "narHash": "sha256-OCCmI69EMaA4BcxRKrXJsx5Ozua2f/PKEy4aJbE7ziM=", + "owner": "cachix", + "repo": "cachix", + "rev": "5727f0676f08a4b41ed13d403ec64dcce989f6e5", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "master", + "repo": "cachix", + "type": "github" + } + }, "complement": { "flake": false, "locked": { @@ -82,6 +104,35 @@ "type": "github" } }, + "devenv": { + "inputs": { + "flake-compat": [ + "cachix", + "flake-compat" + ], + "nix": "nix", + "nixpkgs": "nixpkgs_2", + "poetry2nix": "poetry2nix", + "pre-commit-hooks": [ + "cachix", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1708704632, + "narHash": "sha256-w+dOIW60FKMaHI1q5714CSibk99JfYxm0CzTinYWr+Q=", + "owner": "cachix", + "repo": "devenv", + "rev": "2ee4450b0f4b95a1b90f2eb5ffea98b90e48c196", + "type": "github" + }, + "original": { + "owner": "cachix", + "ref": "python-rewrite", + "repo": "devenv", + "type": "github" + } + }, "fenix": { "inputs": { "nixpkgs": [ @@ -121,6 +172,54 @@ } }, "flake-compat_2": { + "flake": false, + "locked": { + "lastModified": 1673956053, + "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_3": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_4": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_5": { "flake": false, "locked": { "lastModified": 1696426674, @@ -156,6 +255,42 @@ "inputs": { "systems": "systems" }, + "locked": { + "lastModified": 1689068808, + "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_3": { + "inputs": { + "systems": "systems_2" + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_4": { + "inputs": { + "systems": "systems_3" + }, "locked": { "lastModified": 1710146030, "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", @@ -171,6 +306,28 @@ "type": "github" } }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "cachix", + "pre-commit-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, "liburing": { "flake": false, "locked": { @@ -188,6 +345,31 @@ "type": "github" } }, + "nix": { + "inputs": { + "flake-compat": "flake-compat_2", + "nixpkgs": [ + "cachix", + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1708577783, + "narHash": "sha256-92xq7eXlxIT5zFNccLpjiP7sdQqQI30Gyui2p/PfKZM=", + "owner": "domenkozar", + "repo": "nix", + "rev": "ecd0af0c1f56de32cbad14daa1d82a132bf298f8", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.21", + "repo": "nix", + "type": "github" + } + }, "nix-filter": { "locked": { "lastModified": 1710156097, @@ -204,6 +386,29 @@ "type": "github" } }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "cachix", + "devenv", + "poetry2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688870561, + "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1711401922, @@ -220,6 +425,22 @@ "type": "github" } }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, "nixpkgs-stable": { "locked": { "lastModified": 1711460390, @@ -236,7 +457,55 @@ "type": "github" } }, + "nixpkgs-stable_2": { + "locked": { + "lastModified": 1710695816, + "narHash": "sha256-3Eh7fhEID17pv9ZxrPwCLfqXnYP006RKzSs0JptsN84=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "614b4613980a522ba49f0d194531beddbb7220d3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-23.11", + "repo": "nixpkgs", + "type": "github" + } + }, "nixpkgs_2": { + "locked": { + "lastModified": 1692808169, + "narHash": "sha256-x9Opq06rIiwdwGeK2Ykj69dNc2IvUH1fY55Wm7atwrE=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9201b5ff357e781bf014d0330d18555695df7ba8", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { + "locked": { + "lastModified": 1715534503, + "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "2057814051972fa1453ddfb0d98badbea9b83c06", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { "locked": { "lastModified": 1716330097, "narHash": "sha256-8BO3B7e3BiyIDsaKA0tY8O88rClYRTjvAp66y+VBUeU=", @@ -252,6 +521,55 @@ "type": "github" } }, + "poetry2nix": { + "inputs": { + "flake-utils": "flake-utils_2", + "nix-github-actions": "nix-github-actions", + "nixpkgs": [ + "cachix", + "devenv", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1692876271, + "narHash": "sha256-IXfZEkI0Mal5y1jr6IRWMqK8GW2/f28xJenZIPQqkY0=", + "owner": "nix-community", + "repo": "poetry2nix", + "rev": "d5006be9c2c2417dafb2e2e5034d83fabd207ee3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "poetry2nix", + "type": "github" + } + }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": "flake-compat_4", + "flake-utils": "flake-utils_3", + "gitignore": "gitignore", + "nixpkgs": [ + "cachix", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable_2" + }, + "locked": { + "lastModified": 1715609711, + "narHash": "sha256-/5u29K0c+4jyQ8x7dUIEUWlz2BoTSZWUP2quPwFCE7M=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "c182c876690380f8d3b9557c4609472ebfa1b141", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, "rocksdb": { "flake": false, "locked": { @@ -272,14 +590,15 @@ "root": { "inputs": { "attic": "attic", + "cachix": "cachix", "complement": "complement", "crane": "crane_2", "fenix": "fenix", - "flake-compat": "flake-compat_2", - "flake-utils": "flake-utils_2", + "flake-compat": "flake-compat_5", + "flake-utils": "flake-utils_4", "liburing": "liburing", "nix-filter": "nix-filter", - "nixpkgs": "nixpkgs_2", + "nixpkgs": "nixpkgs_4", "rocksdb": "rocksdb" } }, @@ -314,6 +633,36 @@ "repo": "default", "type": "github" } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index a9399cd8..acaf0dae 100644 --- a/flake.nix +++ b/flake.nix @@ -1,6 +1,7 @@ { inputs = { attic.url = "github:zhaofengli/attic?ref=main"; + cachix.url = "github:cachix/cachix?ref=master"; complement = { url = "github:matrix-org/complement?ref=main"; flake = false; }; crane = { url = "github:ipetkov/crane?ref=master"; inputs.nixpkgs.follows = "nixpkgs"; }; fenix = { url = "github:nix-community/fenix?ref=main"; inputs.nixpkgs.follows = "nixpkgs"; }; From 2be2a0dc914ba65b970b4c44e0468bf8084eee8a Mon Sep 17 00:00:00 2001 From: PedroHLC Date: Sat, 15 Jun 2024 21:27:12 -0400 Subject: [PATCH 0218/2091] fix: add tracing-log patch, use pinned tracing revs original patch from https://github.com/chaotic-cx/nyx/commit/637ff3ce0c8c19673d5f06d31c18a61db9d84459#diff-73188cdc15fe4e672c637dfc8a06ca08c30f789aa31dc8e1a5297f76779bc369 Signed-off-by: strawberry --- Cargo.lock | 25 +++++++------------------ Cargo.toml | 9 ++++++--- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 07148055..d4cc7b8c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4401,7 +4401,7 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" version = "0.1.40" -source = "git+https://github.com/girlbossceo/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "log", "pin-project-lite", @@ -4412,7 +4412,7 @@ dependencies = [ [[package]] name = "tracing-attributes" version = "0.1.27" -source = "git+https://github.com/girlbossceo/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "proc-macro2", "quote", @@ -4422,7 +4422,7 @@ dependencies = [ [[package]] name = "tracing-core" version = "0.1.32" -source = "git+https://github.com/girlbossceo/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "once_cell", "valuable", @@ -4442,18 +4442,7 @@ dependencies = [ [[package]] name = "tracing-log" version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "git+https://github.com/girlbossceo/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "log", "once_cell", @@ -4473,7 +4462,7 @@ dependencies = [ "smallvec", "tracing", "tracing-core", - "tracing-log 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tracing-log", "tracing-subscriber", "web-time 0.2.4", ] @@ -4481,7 +4470,7 @@ dependencies = [ [[package]] name = "tracing-subscriber" version = "0.3.18" -source = "git+https://github.com/girlbossceo/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" dependencies = [ "matchers", "nu-ansi-term", @@ -4492,7 +4481,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log 0.2.0 (git+https://github.com/girlbossceo/tracing?branch=tracing-subscriber/env-filter-clone-0.1.x-backport)", + "tracing-log", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index da0c06fe..b7994847 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -429,13 +429,16 @@ version = "0.29.2" # we can switch back to upstream if #2956 is merged and backported in the upstream repo. [patch.crates-io.tracing-subscriber] git = "https://github.com/girlbossceo/tracing" -branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" +rev = "b348dca742af641c47bc390261f60711c2af573c" [patch.crates-io.tracing] git = "https://github.com/girlbossceo/tracing" -branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" +rev = "b348dca742af641c47bc390261f60711c2af573c" [patch.crates-io.tracing-core] git = "https://github.com/girlbossceo/tracing" -branch = "tracing-subscriber/env-filter-clone-0.1.x-backport" +rev = "b348dca742af641c47bc390261f60711c2af573c" +[patch.crates-io.tracing-log] +git = "https://github.com/girlbossceo/tracing" +rev = "b348dca742af641c47bc390261f60711c2af573c" # fixes hyper graceful shutdowns [https://github.com/programatik29/axum-server/issues/114] [patch.crates-io.axum-server] From 502856a4a268f096c841d0fb77f05a3cacd7cb98 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 21:31:33 -0400 Subject: [PATCH 0219/2091] use pinned revs of git cargo deps where possible should reduce all but rocksdb from here: https://github.com/chaotic-cx/nyx/blob/0a48586e8c81ce5647915a1664d104616d5704b6/pkgs/conduwuit-git/pins.json Co-authored-by: PedroHLC Signed-off-by: strawberry --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 9 ++++++--- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4cc7b8c..b716b1c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,7 +302,7 @@ dependencies = [ [[package]] name = "axum-server" version = "0.6.0" -source = "git+https://github.com/girlbossceo/axum-server?branch=phlip9/fix-graceful-shutdown#8e3368d899079818934e61cc9c839abcbbcada8a" +source = "git+https://github.com/girlbossceo/axum-server?rev=8e3368d899079818934e61cc9c839abcbbcada8a#8e3368d899079818934e61cc9c839abcbbcada8a" dependencies = [ "arc-swap", "bytes", @@ -3075,7 +3075,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "assign", "js_int", @@ -3095,7 +3095,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "ruma-common", @@ -3107,7 +3107,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "as_variant", "assign", @@ -3130,7 +3130,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "as_variant", "base64 0.22.1", @@ -3160,7 +3160,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "as_variant", "indexmap 2.2.6", @@ -3183,7 +3183,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "ruma-common", @@ -3195,7 +3195,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "thiserror", @@ -3204,7 +3204,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "ruma-common", @@ -3214,7 +3214,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "once_cell", "proc-macro-crate", @@ -3229,7 +3229,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "js_int", "ruma-common", @@ -3241,7 +3241,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3257,7 +3257,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?branch=conduwuit-changes#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index b7994847..4c10bb0c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -273,7 +273,8 @@ version = "0.1.2" # Used for matrix spec type definitions and helpers [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" -branch = "conduwuit-changes" +#branch = "conduwuit-changes" +rev = "fd686e77950680462377c9105dfb4136dd49c7a0" features = [ "compat", "rand", @@ -300,7 +301,7 @@ features = [ [workspace.dependencies.ruma-identifiers-validation] git = "https://github.com/girlbossceo/ruwuma" -branch = "conduwuit-changes" +rev = "fd686e77950680462377c9105dfb4136dd49c7a0" [workspace.dependencies.rust-rocksdb] path = "deps/rust-rocksdb" @@ -427,6 +428,7 @@ version = "0.29.2" # backport of [https://github.com/tokio-rs/tracing/pull/2956] to the 0.1.x branch of tracing. # we can switch back to upstream if #2956 is merged and backported in the upstream repo. +# https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c [patch.crates-io.tracing-subscriber] git = "https://github.com/girlbossceo/tracing" rev = "b348dca742af641c47bc390261f60711c2af573c" @@ -441,9 +443,10 @@ git = "https://github.com/girlbossceo/tracing" rev = "b348dca742af641c47bc390261f60711c2af573c" # fixes hyper graceful shutdowns [https://github.com/programatik29/axum-server/issues/114] +# https://github.com/girlbossceo/axum-server/commit/8e3368d899079818934e61cc9c839abcbbcada8a [patch.crates-io.axum-server] git = "https://github.com/girlbossceo/axum-server" -branch = "phlip9/fix-graceful-shutdown" +rev = "8e3368d899079818934e61cc9c839abcbbcada8a" # # Our crates From e5c9bf1992ccd107ffcb2597fc8f1891e4da60d4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 22:09:27 -0400 Subject: [PATCH 0220/2091] misc build changes to mdbook, add favicon/logo Signed-off-by: strawberry --- book.toml | 6 ++++++ nix/pkgs/book/default.nix | 4 ++-- theme/favicon.png | Bin 0 -> 29856 bytes 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 theme/favicon.png diff --git a/book.toml b/book.toml index 8d52c75a..82c2966d 100644 --- a/book.toml +++ b/book.toml @@ -2,12 +2,18 @@ title = "conduwuit" description = "conduwuit, which is a well-maintained fork of Conduit, is a simple, fast and reliable chat server for the Matrix protocol" language = "en" +authors = ["strawberry (June)"] +text-direction = "ltr" multilingual = false src = "docs" [build] build-dir = "public" create-missing = true +extra-watch-dirs = ["debian", "docs"] + +[rust] +edition = "2021" [output.html] git-repository-url = "https://github.com/girlbossceo/conduwuit" diff --git a/nix/pkgs/book/default.nix b/nix/pkgs/book/default.nix index c1e5b8e4..4745e432 100644 --- a/nix/pkgs/book/default.nix +++ b/nix/pkgs/book/default.nix @@ -19,6 +19,7 @@ stdenv.mkDerivation { "debian/conduwuit.service" "debian/README.md" "docs" + "theme" ]; }; @@ -27,7 +28,6 @@ stdenv.mkDerivation { ]; buildPhase = '' - mdbook build - mv public $out + mdbook build -d $out ''; } diff --git a/theme/favicon.png b/theme/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..150aea82ca57c59833b5364c0a883c2ae2b11469 GIT binary patch literal 29856 zcmeFZbySpJ7%mDUNJ&X4AtOjjC@F}HBAwDbfD(#;Al)OSAfN(*N+~Jb-5?^Mbhk(k zAu#m7-9xB%t+Vc3_uN0uTIc-!a4pxzo;~~9`;F&)o_Eg+Rb?4sLRvyBEG%L<*}H03 zSZ9ne|M1UZVPVO?Axgr+f?~)bPji%8=Hk&fOi zGS#mS?-Mao7ef6v$j_9Ym%YR_GAPUk3mf9i#V4>^<8>pK0X$I+seOf|MVzTcaPn6i@Gv*k2g_XLp9io6R&4A{ zR?E?UpTV#S@J{E%d`dOk2TO~h(B|J~P@lmwn7<2@>EN3*KG~`1ctkTKs4! z?)c|I4h}~(j+cfY#KFlWR27u_=ktgv@bOLDf9Ava|NEsUD?_{o-}XMiQSzT}!^UpA z^klW6`Q_7%1JA&jkpA6C=>K+S^56DA{+~?tzwP;Nd;aGG-hbBfzs7o0 z18T7oLy(+_AGd|h4mj=`_nw6L_nc54K}WNf!GF^e;s;1MvHEHhOn;+(!e@{O)U}&R z|4E{AF+Ny*Wh{c1|K>WJ|EBwI@BU9U6qpq*$pwa;?7q;kS^#5JlK~D*&My8J(XI*s zgyr-f!aBl_{{hhdaZ&JJj{IMfBmZW*|L-N`StdS#p~YpyMrL;%$N;_t{rvxwY~mDT zwQj`|DGGl}t=b7tebo&vzWXQd#(__Y>p$scIbDfmG|1XhgT1-eE-IUW1Z-AF%X<1Z z5JYOnX+Y76|4;hG|9>r75tR_5(MWCiGEaPb^F6Bb-(@h6v5R2_4^XE3 zeO(_|8AKB^c*L=b_MbI#Q>Tf7!Q5l?0m}c(d;{W$861P^BK&tSWg2E??vX12ivJA0 zhPaIxJW&2w>7S>;IA<9uadN4xsU`nd4HO1dO5gymt)MSa^v@nyB4FLmqOkg&Dcyg5 z5@Rr5+ejcF2)@s5F-_o~RS;8Kx~-nCrLA;57xK@Mc|fcQz|K{pG=lzldX}(|8%#kw zayQ`a-^JA);11DN;_!)m_-AMzexW!Rnlj8KbY3h+xEldf2PIXIGT1a=mP>?AMshv&Sl?U{_}pf@$rq^ zwP|Y4=Fy@4@;C{J;3Ee#T3Cy7O~TxNZO)Mp-)N^0H`ha1p8U@Op}92JbzC<3_{Mfd z@_)Aj%Et+HNwz+Nef7K2KT}jul2rzAS72$isaXG=xQdLdvPIV%JCj}}>n~X|m*g{{ zxtvdEJni=A@2SGraBy-Sy}gH((WII2cSrO%pe~vfkAa(1>-}}<4npj@j5xFNwF`EA zM1MC*8oUUKCm2gBr)2D}$qC$ntaRy6P=mRt zXY9ntYnxcq{v4wZ%pDge$2=<*t1lD9NBj3BlfgX6d~e5|V`+&#q@kmyjqO?HQErPl zT|g`aS>^kxB#2Hy|JS+GhdB7Z&XZNzUn8!3g(bAg!iV*DzIn(q?ebCRBSK?e)qb|W z>wN~E->${h1gDCVqyKk22Y4Q&_jCc@_`5C<^WU(Os{-DG?t>)OVw33mKhzye2E2t0 zcP3U}7E0g`tB42%FTtSOES_@N$MLD*_djlxkPK#5py8R##m$XT4f<;yX=M1GFayVoSA(DJrL)@P7OO@6fCBqKL^=KWNa3!H8R#=xmUrpEV=&2T)u#p`x#*P z_%_z!&-C{cr<+$>3v3I~S@P*Vb|zKpgI9mXE75?nslN05LOfRAOGiG!lU;@7*LLis z_z|poO=j&%@=G7?HA^;GTW!rOnap+{j<>fi>6F=xh!P?VVXOK4rXYu2;5Iu&*k>rC zOC6k|`DVR|C`e+-S?N)v^?u$W{EE@y89n6I-U~aq?ho__ez(B`4S7TSVh_>Nx$DbO z3QLJxAp}N-y5HPpFZ=W^W(l7zh6iy*nSq2t<>pnz-DKy$7cywmXpz|t15`R^1a3jB z#14G&J7eVqr;&lwO7cWjxe9##mXh<4b=;rlIJ0mJ5&zGi_wnE@EfI7%Q zD~1H8^S#ff#QhGV^L97UT^@XTf4F)*%4BcN(M(WRLF}0}cU#rcU982Wt`9Dw5Q-Vb z^8w~cUKzt@NIsC0h&Z^^NItpwWRxpO0&J-U^DBlFyNOwqxnI56m8OH5d0AVBCAmwj z-koy|VmLXKT+)o>s2QpF%1|XL z3opK43gK)$Kv`LltYI1g{3C%9yKXV|rtjkv`(}lXtY5!MF!w2tMZZ}u&)9HDSdm{HEL8ybN1Ec+|{fPriLF=6r z!48LEagA#PGf+2YKT&$`d^g&@>)sh}##WcM`=qxmR;=cKzR;NZK0XeDb%G578C-To zo1T%xz{O+Ku%x)=>T~Pr$;qL3a*9g53Nz7bPp~s7S}o6O5wsC$FJBP{7Npl;AHK=jr5o=Lce)X(jJE9*mIrkDRP+#Et(8z?SFe9^>HGyoM}Px zGSo-&WBT_#LdEKZIlk#BM%wLl14nH05{_!Sesz;Q?f%FE6YYNb4In2VZWAaPIvPY5 zZzl~^w0k+NY=1i>6pPzg%$@p7Y_i2Q-0-6U&SOST5%uF>N3Kd2_Q9d&V-dW!_1UlH zpp^*wy~+vAmf{ZCZvgUsOL%V;4|8fi_(c>(g2xj*yr6$BxH-&_%6*FTlCz5pxvV1r z*_=OYRJ$sFcs)Pdw!p&8d88t5leaCu;tp$^i%#^0!U_q-=Zf1~vc?ZgTs#Nw%dJyY zy8TwkD+;Hq)vItgYouX}^pD2#+}$3k8RZh)HL zDau-hr6V7WH|GnM<*X7ISs(QG#^(_kqjjzL(fS~3k86=NuHE2}@_|X<v$!{DjZI`_0k)4!)`^bNa9^3ubivs_Rg;y(C_Ai)DtD#p@nZMUbT(u0}4ZYm0c z>m8Rsur|fbSH0%3^HrfnQ@o;R8Kr-qkdf^)T}0sIy6{2%It zy{D6(oPhdY4Dao6Xn}PInqEBji4&1hMajQ6F;L{XsNX$$!|hq`fcJ{^0Y%vUmrl}b z4qJ3DRQco3aCH9geWRM)cKh*QcyIZmC}8Q1_z#?w&cVH;KO2VN<}9`LM-JIg&}A&V zJ+%G+Cq4YS;=!}u*?*Y!qFaa+GDV1QJawL(5rBFbP6h6G-@jHmCfw&+3!I>a}x#I$eQP&hqpk@R; zp1oE`{4L?b{V%!eH8lo%y&oS}Q?$o1uzI>|xJ-097c0g*wbYH+vu3M!U*fuRrl`H- z?YDV9EQAqJ1Vi;rg}s8Mh|%{VOXD$G3sYE8yfxQ79)6*!8p~lFzXiU<)5^SYkGwgX zj2P~%HLdNRuP=>95Yv}xVG?;vkK2rfgV1WyMLi}5g?|DayP9gn#aCh_Z+Nj*|oBt8|qIg0CxG~ zaPYH6`0OR4cS^6L%(R5qi9t|n$WmRvOWoxh?*UN2fAB`SUy zJF~lWIP9CIo$b!>bbgdcHOlC%Rh*TerIXp?g__-DK2)2x+U{qKi!b%h>sN`Ni-WD; zwh+Rz-M&~SBVCs+_Isp9w6k2k`on1cq`u{NTv4ZnNSNi#EO>!aED zu1)#I2a~TYAuO5MtN_!2x=(kq6X%Fh|0 zMBUW!8l`zs!BGMFS~;8}p=a?mSb$@)WQ@w=wp5y#JWepI`Wb0)IUg#7cJ|Kd;oECs zuxePGz4bQL-3+XZj|_2rY+Z2EbOPkqa}P)i7Ykp~z%68Yc=al;cuXyeRW#kd#XwG0 zE1Lhx^S$&%#%S9u`ro52S!Gun9s^zg{2 zehynqUso3Jn*2gM84<2Rve+-7{vBE*pWp+I7E7gf_s9sinVGq&yLTG|Ta5hk$GhLG z=oL?$_5z@vG5JMS$qzBZ5}v~@mgMdYYCXVh%F%D)ACuVqUB3E})XRAPM+8*00L;i= zy4-a%pe=@FD}F-S@>TJi?r?+gP3SUYw)Yb!*H^+iXRAFz8$U$pCR*aJ-_&`R@iFTY$UOIMqqr#8cTL_Aw{Ztf^Qd%^*5-9? zQ1xS>utmdjmRq<3 zYj=x(a!0S+E{VT@0WE8?p2n=*QrcJDU_nJetl9`Hge;K%LypH4 zbAi4eU}5f;Xk+)5FSU>NRB);O(!I=fsn`OZWHjlqIw@2Ca9#Of``wU2b#QE{Zwp4l z<{g#*N|R<~sU8(Q2m+>+kPgz9xfq=pm3%Yr5n5=f%C+J*C10Kiyf#N+J1W z*|>(XCxhtlMpfyC^Vr{sByBa%G81~wbh*u1;=8^IGgF5&q9S?N zl3O)a=-E$!6G83*x>H<99?LhYH@~gErUa)`0HKFP@c+C+;6%(Pk_otf<5ROlp z26^Q%;cJ(HJNudA6aMJ)h&MQD%b8DF9?5NUTe40Ek28ne&#Ylp=4m>)9H?bO3oSFk4Oh)ZV&wOR!emjRZH9A#8(lQ4 z<&quU&)T;c!VE;lRk{mx${Pv8D(CQcZr*xCmhB~FSCsywMU$@`QvB!z@mm--II(RJ zkHojh4>j00wB<$v2GUrNBgQznKe!+Avu$K6(FA;){Cv;%>7P>>q{z}YKf|i}8P+l% zDB*b+VzN~*%*hbd^nvBoTJm^)Fb{#Sfwy%5tEoUNCZ88Xe2WpAJ3o2Dj3@4woQCC8T($TEPgnmg!Kk@OpFLVft+Cy8`Y?t@MU11qd$$=TqE5se~#BhjwSI*T5KxM|3R$ z{qsidr-g+T1t5M1b5hCfAZ80IVAy=Kfemb|D&uT!$-5&=05yewvDC zr)x(Dn$C;fz{k!EZykz}`(m`#F1<|GGLs2sV3u>07#Tn3%2ys*)QN{+{gW_9BsbQk zM>Jx&FYL8VguwML-twNrjYaY09b>;0srH)Oy`O2W<>lmGdSBFoGZJSWc#$kj0P)8$ zm4NS$us4prXvx!7VubHK*ls5H>S(&m+Ibb=W!=Tj&z&w33LppMe@nW_HYd>7@>a3| zNAEJ9-8_GW2w{dz$v)G~uKeI4ed2jA272!EdkbSU%ym6X@r=A}aAH^fYROq$lzregm)l z?YHll4@07NY$Oy@Y&^dV*s>H5xNGI2T5TXWZ*Wa(Cf0Y}D2|o|<&|Xf=v4}g`;a{M*;Z{h zo!3gf!kTl!6$Cvn9g-_jKXKLK4(+9w$I280O>?5%NHP^NY-LMjyO@xRl|@gY|3|#2XPk~&*A3YPTNXoA5hrOHs3H-f64zP zA|quC&wQM0E*{px-Rh;aVH=OMYRZcjghW@V`vY|5pDxr;A!K+VARu+-?dnYxoAe*l ziaW_I)>5&fB&^cL@sK>3j+IM;v|gf98FvXPS;#89>7{I^G>-U)p>-j3uh&lQvhBj( z$CV{E%SiA_283~T*~s>YHa1h^NSVKmbxW}Gk7dAGR@fyaB?Sl~K)*|5%jdiZ+o*g| z8`3=I>xh9Z#ErqW>dPGxhynd(pkXb&{%uabckUyWP+VF~7zk#jUm_T_p_{nxZqa{b zSyDm_1SYzlx&H|V{|b>ylgPX50xI0xA3WAptgIL&W{AII0RG8Sq}akb9@MH){TCdMCd&0i)1`rNb(~zDi>p72A3TESC{U*>HN32P_Pcv)9E16@l+*?K2ysm%E^yHYr2y6i6dQdA zsjIGpdGY+&K9QAJ?};GMN~-0s$5>BsJm(cU7S7(6xS%ht>&pg9um`KL|JmnAws(Ws zr^x07orE!VCZI~I0ad!BU%(pTGW6aRgMlA*y?W`7g`g%9d06r#Si7?WH#f*w@Pj@K zt5qr4_!45^extLE2TP^f0OQO=? z?)@MMj@BCmelhx!VTD)7XU7=19UvPWhxOJc1*mR&l&d%HKSgqoxu^c#PAg3=c0<^x zB=Lp(%V6J)u}jK@UF^>C6=K-qg6fhh=@#0x02=)OMmr5tyHW!~^0rsMYqQO`a?F#f zDnd86A{fywhT9!@5~`W%MFdDNcJO>MVdH@@MXmgF|Apg`xr_+8s7vJL2A)nfiC_C~ zUA$#qZ&%jJB?}D$G^dQ*yAklJHk`P}lsMu&t|_UT+X+Pe>>5$NSPsq%MV0O9m52R! ztGREr`yxCGJU=C>+4<0~`5Pj_etZ*0(bW`R_c*90vX9b!1-k^$Vc47N_aJ#+D-OPQ z6y00mI#nlS$)n4bEBWkxd@Rc_I)SslcygeKn_w-MSj;|wkjT#Cf4U?bQ*RmFLYIf7 zW~lV{(y1S+fe`JfJ5M`%aj#)1vRlg^l*>ILcMMMy4^N8co{CKCb~v8nPIBVwR6uxI z*5u8L+Jc_|6r+ppuAXdT&gp5jj?NSD+TgEvwpTpGto{Vsmf;4(-(*vab9Y%@`%wMF z7Q>ul-tI)^AOqnIE0BWhzJFvpV9cf?)p$x^NYYY3yU`Z&xH38K5uVD3rnqB8^YwfJ zHwB#q*`PBiYJ6%itY92<4-dqE^W(=SAd$AlgymFsA~<5lNvaq3)83OzJO@bU4){c^ znm~N*uZ&QVhN$@0IjfeR8B}U0jl_0~N2aN+9E%6|jrfhX4kL?}utG1GnB6~?Jf0FH z#a#boXk2DV#dJLEJdPgx$jM; zC^4JNv|TJ42FH;JzKbixg9kAXPUal;m{=nAcdvmp*Qs{g%>2rJZ}JpOuM6ayLtFvo z_d+T8H;x*~PZ!6!ER)$$-mw0{np0u&*Ljh-O^OBtQCPdCQ)b60B-3ExRkNNW;_ivAtS*bq?8Mwd^G` z*`85X=8Y1@AuUHRnVJ@4f(8X(Zsnmt?+U zmsk}-L}b7RC+VKf{vZY{QNtV9H1NvlM)b0ASF()XMeTS7|J4)Xids{~1aJ-*y%@&& zY7H6HAdHL^1vHvyk^7J~()zhk%Sd;K>Cn-TgoIOP3$~fw=$5#2$z`IV6Qu!t#RT<9 zPWw#3^2(m>XLHbA!{+qjr`>ze$M!Rqb0qL#a4g9A(w9qjF~&y#)1E-#? z$84Q&{5G}5IBM0%@6v$YXKRpA7O~;;?X$84FE+;GSw*2z#^%1K-W~{ufzP0#I0=P% zY@njE6_VpMvKdu#Hm!x|u)WRO32$C*_3Gbkmm=LLB3i~5)3SL9R&cUDQbdnFsrX)R z7`n4jm(WC;cDE8Y>R9^$K3cfze`((|8z$)Q|3t0Y`V#wVi zu@RFl%8Mz;6TfL3Hw47e$cm%*e9L}YaW%~HRwk@VvkoW%dwsXqHF!@6ux5PIiBCVx zL{-rsBS~{eD&j|}HC0ZI5tOG5DsdF7?cz7xp7BVK-1?5yo+JY9in%vW z8c_YLy_zpz?|>P-dq32|{mPBWv=5 zmX}ieNh*_c_~+v|s>cOJqUXLY*0Z&Xwq^xpjbe||m16Z(30i-DnZM=S=CM)9XR?_& zcMT)y@Yy59F-Cv<*&$}|S+jOe?h}T0nExU3QYFZ^J1jPvIBbS)ccbl4gL1wO0-onT z$RE2A6-25*Tj*PB6#Qa!;m;tgwtnayGt>uVkwqug9TCMJreyx$3mT)ihQxkAoFv8H z5Hr6V{TeU#?X30~n(;DP98PMoTF$e5``eJgtmj7rp+=%M%?Pd27^okr$A`QNHdKQGQH*{WwVg_utmYKDlQ#Mf5$1P;d)h^;szWnyLz0uahWB`i_u)A zsy~rh01G$YxzMx?F$*~Ibs=7FD6(>I`nQ3SnNxdB8yL$=@Ia*dY76~zuPI_RaGGa{+v4=zDHMTW zV?XxUE?1^#zB90jvY@&ZeLr&xIhgxh_$GZZFFZ;Jdfe>>YsX2+7UfaY_fShN_)*_~ zBYyWmvZrNCEfj3;?X<;$T$rv*PmL#3Ot|{I5FC|CGS6NqN{jJf#4Y?%NME3)&{s|{ zmQ$lhHyB1;pEB-fKL2&tT&AzaZvBw?d2fD@D5l#7QB0)4&}ighCt14vQ=#%n*%IBC zZEvvugXgknPV=#Rd>~Xy^z&H^{7q@yC zv~ha78#sAga)L)ei-9`r`g2#F%m%tt8Fh+Ux55F#_(y|*Pm%L1&mBGS-Z5g#HiAP{ zlk5og!DF1W!^v9>u3Gnk8n?G4s~{aWf3}{a{Gd_ZMvk7Y0;+}Ld=!s4j`Az7aP#I{ zRd|;+ezvi?3lD%$r+~lMB|YD1fRlTP+bKRpulK2mt?-EbdZNkV8@AoXN7={WUN-^x z6_@_U>q<&a5*-*8-6~|R+5lU-wguI5-0{e+4&95n1^jce*8gi%*JCGFB-Rf)WQgigR+(G!w_bEK3^D_Zr8AgI{{O&8N?F#sX2Q2!$6XX}$a?qxHE zLk2H1V4};U_BVW!O&xu3vc;JZcGZ&5iZG9%1$9ih!=d?=Do5>x;BoU{6IJuA?o@E< z7>?eqbvXy-#P^khEgtF6cqfQc8{l0{?o6~iP<49vmk&eK;N}zcbX$af_F# z_g~&gY&PoK2V^!^Pe2P!I@q=JH@!A27)i9u?e^qeib7xo1>_LtiCyebJV8Jq4WezOrC@#u{Y02NIddq zdkuCVQr_F4A?)sbR^7?t4!s%sEL}`Tfi+DrVQvMRWIFBfLxC7}s1Hm1XDy)GrZSkz zwAunW&!TEUSEveuIlr@B07Vs}>(>)mm7zujRetWkpO!Jlz z!_jac91>6xo3!$jUm1CVF6t##ZT%a5jdz(XsJ*_e0SL;#3qS0d&-(_EO!OTd{e?|S z2Nt%;{mMTV!{EJOcfa&_d8Ev!D<&yb?F2(`DkaPXUnrQZGSYX@9&p`iO%9QhJaPe_ z+6mmCN;YI&AR|HQcQLcoaK2|ofNHHWIxttRZYJ$}EM2e7-`|^doJ4gYQq^gWT&B7i z39&~&Rk=zYCunS(+e0(|>XFf7LqaGnc&)`?`kOkZ+)gxICoa>%7{U`2m6{9dloa;M zsaHri+EQo-Edsd}aEhYCcA9ds&c)ZPKAUsnBx9EZ3XJ4Ex@3Gseg31qFF9)hxZGpT zqq9b65h)soinUcfH0h#OmOxKUBJip|U=W)Wa1185g}1DrV~n{TXwAxxRJP@T+H(}a z9qt$}q)*iseqobp|58!+>6->SNbXz&YW(vMDVxj9Zl0hlQx;R4)eb)zUua}JQ@&o` zcq34{{eI`qo`D3l1KSKa+kp545RVo9x9<`=UG#+)hTn zJT1=2)u*%amG8b}RsZM%UI1w94o7$$Z+!h-lCyKAQyl}QD`7r1_pNFwow(hMoDkwF`Yaxq7|zDSULC&iojf2H_{&f_wdXE#r?`jOXYoD#x)0uU!^R z$*Fz{kr_vSg{Hy@a~M4J9mJ`Q>VxU(-N)CL(?kg=WPyaC*3%;g$^!1% ztiW%-IFCpVgb`uVO`;juT_c;wAvSKGUgfbB5_O|;WKV@TET83*#?NmPk>*;Pt}WowaDF#LkIC|y+1N@ zGM7|XOH}O%+o&9a!wI&J)1-H97+1}x5;@50{``0xyCig4sPU9jYnd~m!x_VHe3X$y zN%Yl|3)<7IpUxcRw+=eQz%bdVxPfZ=dUp}Tzy;K@nQfj-t@#Ricm8G0W>-tn38S_K zG%-nfT3SwTsNNOOB*@&N!E{HKtHft<(@CVBN(dZ5yPB!lpk9R6r^ z3BpmrSZzO)p1W8_e%h;x=)y5wY5q;G^KgT92!xgb7EY??Xp@qu`KWp;nQ^W8Xn~U! z?hO@8%njgRj171HNqTEvR>XRYC4*kId+Fc)$?5)*yoBo~w;Vj^ldS*H@vdw|wyqRTibkbcoT_hcNH zZp}Fd*Rzv4Ac?s03cPYFB1yIq)?#*WQc>A^SO9|k$l`JA@iDy8l}7rspu1ojJ0EcV zB7g3A(ApZcp2m4Xfl-n_N243%o=X+>jz)psPq5WsBHJVG#x(E>#e2Y6?su{uhjkc~ zBHCe|+@z`3Vz$ZUl+3HBe1@|IoZ2-a;sv#cOM`77L4n0|_Uum1h{kEP{sNL0CcBR* zTLW)xBO1pvm~)*$Q^t?9yUzIpT!Y!oPyw=p}d6hXtCE4+ry}0j%&(j<2hZZY0~e@3_W%>Y)B+H1HZ*Lj(lN14TWb}#E-a2i^Xw!Bn>^af7!ZB8Ik{XyG zdCm*xh>5A@l?@>)WMs9EgVaDp`v>3RM_dIe*tzsFOOQ0*EBd3s-f!ZaAPWt|$1Np- z6}`iC+H*w&tu7ZC#Q= z7(RdYEwQavmri8e>3=i?d$8+epD9NIPoe)$OrLQVj?oQk%>mSe?8Bv{oR7%TfYG9j zq;N{aK>G#kt5UkLr)q2jkfeqIL|PU$^ID|c-=BlqnC&iyzv|X89Qh0 z^o&eHABzg5wKSRph*Ew#r`oJ2-&1wVH-H98>YZ!WF%5I30zH4LEK1Cq4S7j6f|5S4 zK-^-_!};8uUwfiACDmKG$o9m7qWESbHF>0gNYJS!6a@lRNqqHT6KK@4E*sKH+XJY6 ztAF4eK@azY(^mPH4};^UB=e&uGM8L6bZQRXVbb{p6;P$AG>^*=Ra$lXn^?L$T6SCh z-mcUe{48%Tm4=-a=w>F^@J%Y#$DU`xb3{ckJ-ExRkB>W1ULVCCP+qwvN3;SSANT%g za$Vk}LKef2R6RCU(G1vt?%U4fP{^-ZYP@wHiPN~5?bt2D<0I(luP0JHfHgBYN6lj| zC8=B{p7iOXcv30{Q1kb@qzXZO>R7c4Ohi+c@jcMB+ zu{JQzDn2VSz8OwR&yQtv?ThE!wBi$xJ48piK zAVKQAn>zmDea9&YzQ8_03`{fXROssn`Sw9-3Iv5FZn-_Y2>W-3y|^U-vCcIk2j4#2 zzKr-2rHFZ(0R({}lnM9SX^hUTxNe!0&CYlNj}X;-GUusYU(x5y^GLaRA{d16V(%BE*=YJfa3YvJgi_c@4ZWiuN4ouA!$hjY6 zP$-NK$l)7@7`}HRy}EhjE}mVXdsXWN^r?5IfBtxxltl``p_p(VQd5Ea`2_6Q0!IPw{e zD0i;Av(p{L0|^qZ7TaF`HymM7H@8=rCktIu9ZYs-I#dB3_(;^fX_9R4-Q^@ATgR9G zW+MdBnBz)GpWoA4sBf682TS0BjJwZC_^W#_{YsnPu6*(Hk>g}UHz_aN%WS(Vh)Yl0 z*4MTeI3p7EaA{60Zq`m;)+(xEwN`yIn6$YbNek^DK#bSy2j%`u=av5cMd4jysh*+8 z=5*`e0|trl1s2;c6S%orSJlPvF1kxK%xf$e%wCs{UGv$PPxZcebG6pF6@%^Y3w0*Y zD-zdluD7XtG6(mLtX!rWG8&dGN~C(#OAR3$&Qqo6TtlmozD`18Yr`_?mtvdm;vUN@ zr#`By*Wzm&z)OMyV##I2eax^~k=XQ{B|3j?N{7Wp53-OrS>AB%1QaxQz~pt zOTcgO<2j~rg%V67r4(xb;i%E6AihRpol2NI;x&i98ao7Gl}rFJHR`%W>~2>M<*u~Q z8+#N0ICNzZvGfsGec#Nie5jAIO&?iD+bq zhct89v->EO%XTA`eiVGk(*4;-s)1#NJ}=}B{7U##Zu}sAEs;yAc28H{qz4}xk_Z_z zp;r+{X+&SUFVfP0nL~?GdB=3+cacg!Sj-axt{ID0OF-=$stE{*yWL{_y!!RyjW=&a zOCx;sZbA(g&Rf2Or2^@2su_@-V)H_v$M6ALk`Y?(M1~G!kCyHR*)1hU6|X4@$W|%5 zBXnbxGi4-gW0ae0m@4I{Crs8x0<@?_!*XzLZRUAjI@nODp%9-vl_)@s`JF=6btNj? z@k2?dA>(U|(r#0`577C@LC?C$uK{;jJ~zoUc_?TIL+7UVS{heKCf*+!t+l(THifH} z{md-f?Zmi4;^VQG_5W1d6lDcWAWuI7LH{MDRXt(&ZlKuph(PP%3qYRFjEyDB*PD<80niQ{}(P`2i z7o?AR4}g`N$ncKyo?j_hMMh_{g7dwZ-%km z4{MHCd(`47`%tXdDMCPYAps-#U~Z^o%*(RZ*OEx1~4daVW{S<*+;*FQSfce@~*CJLZtLJ>-&Kd#q(XLQA#^#Ya~d zPKsCVP_=A;(JuPMc24^t98$BNfu`KzJDYmX0C`Sx)RQV5Xba%JqEBraY{9+m`ps0P zVhf`0C!O1K*b*l}=BWGjBG}!FK|{^luk07!Z9SWwvG%Czp-4aFNQSp}E&`DE{ron% zH3y$7d)Krp!DAwlJDLw-Vd)?0MMv!6lk=@@8=H0i#0cn@Ep zDi!VtYYp@@Y+hz{6+yHRy)i>CC`43+Wowu-B5p`JHX>Tt3Y8g;vIBYJ$6< zEFsqkdJE*Bj!b>>R(;=cy%v(OXq2y%+_*;hu4X^yd?0KK_h3y%f`oCuwo{IRs{YM@ zd`^UCTwCd(@EelR=Cf0M4P8-peP4k6XNr0B1&5UL;WD>y+gzrUGp}bT>ls>6~J|S@=xIz`_llDRY zahjqXrjc>`{h`&|z)MkpC7|aQ`}hcuei$`$fwAxsXlKf-twFD8*PFphMO?j@xt2+X zqjD6ob4=gEI#;KA8U&n+(gB=JFX&wpfqYK=Y4*MP1i@Uh$%Spc+BniO>v`dx*4tIB zJaXw7=K-SLWP8#se0U`Wy=cM0OkncOL5?aoAP3x+Inn!)4m6omPdeeAf_2ewF4hJb z+sq6RQM)cLo(8z3`+eY6I-0E6sAHeas_%*f>06F6hP=2cHR)}8F?VsRKP9xdSZ^(ZlRYz^N2uv1zjDKQjjK{)_JG8zLNgn zG7@DL=BlD|`HQm_&stngULQsd5t;rKF0Np(He{I%$|TGkbV^Egd#+D3X4H+mdAom9 z#o+4CiWq(o0fE$b?@*b1(S%>^xPyPom+*I0qW$Jr^JJ?;+{VJ2yrz6fVNmq_^n_3I zg!IV+Y$)H@qj3$vRFbuWb=4d$z0^bV*urx!_v(Gz!rMNupxTGtHxLwe5)jVA8E`{wo9<4!8>vs^w! z&P$?VYfOP657_VMcl$jL8(d=Nn}wfhp7*!6u*4yG@CjhZjZ(Qrk9pTGony2##!boS;} z2LrlxMLzW8Bh&efzIsr$_u(~faLx46l@{tfPyVcwi( zOXjp0jp2El(t{yF=~OvTZBtUP^ZS`Ge~Tj?-kh$PLj$S1U<}W5Qd_~b&+4(ie~ziU z;ZhygDDZ0lRwRf|f%Xp!ACw?oW^OAVHDYa~HM>B}DlhOaHsH;;bREQji=;#mcA`Ir zXJ2_6ig~h1P&!lC*kFm-_ngn(^(Kuh_MW#>>yI4ZWk80D`>pepY6GM6UoLLw9B6+w}^D_oX}NG zU=KL>Jy$WFQt#OB#nKji&EQe{{oBaz`tge+vuae@l})P*V;T!WJp(=f{)21FLdxJY zX3XB%Et^$;klGF^(0-4Fn8Dwl*;fhV%HB5Jp1=#`@l+k0?fQG!vlBHqNvnSA;A5Av z&k*uj8EuiBpeMzb z%;j(|>>Rv$hHDZDm&nLpE)Wwe_MGLF{E!`FR+6f+gCh0&R^#^Iqi;czBzmp4KVZB9 z`Q~@fjyog^Pz!o_if3=Is+!}s+8hqpH!zdT4cvD(D+yKH6HbkEl8;=S9_Ph|7@lvA zBdXYV6D(iDVyBXL)FtNq*mL0bKtPk>#SAV7Ggb_7U~toIph_-fQf1al>wWOw5?rk3TlNo(lms+3-@_#NmG}y?ym&;$<=Hum)P)LUF0{dG^;`_mIU$|I{NFAeDz)1$^yY% zujIkm7=j2__mSwiKHEdS%4%fZki@|I+)uh`;&TOug+0SEemSg}nb$DC?^Vm_fl65| z8`sq{qj9$utMI-5RpmhY==TL;+bijv&|$CD#$4Zdk_(%R@%o)yi5F4ao!$zNlVQ-) zpP>El*r|81g)7It(PK$WALFh|m_?qpIGEe#bC+ZTldG(9s`?6y zUfTqfU~W@^jvl9;`*n*=Y<>ItajWSnDJ{#dgM= z_xEv5`dAmKn<*;CZlxfN_N{Wz@-FQaq!|urgD*6OFQMgJ)<4e5yRMeC)rIG>UNz%MD8?t7d+7t+{EtLX7)zDp!qqBF8caNX}!fsSWSm=R=!1@j)gbZ#5<}JDVe(6M#Ng z<2uU7(ohRt&r5sZxP@r+H%K3Lfkn0E1F{v9!QlanWeQyfzi1_Tgfr^Dwu>-3{7%<# zDq55fU$dAD9X^nBnLw;K85b!cz>h6KbjsrNf+^EDU6yXVMEWj&3Vs{uW#IYCL0^BQ zz$nX5^bF`cp9I-4Plfj;Z_Gvp+Yv7i5au-AKu2V_HNIt?I$7*QMO?`P*2?n`y)TX^ z2R4Nq>povY4z99~DH4OsmIN-GwI6l`dfxyIbU8u?LSK;C&yn)|iQb!@AGsXsdKZ%c zq0nl^IFf`%1#Dv~Z>!K_1|Flow{GGa9=5WBxd@wKQ~qs<`L_c`IRND}ay4`+RN}J7 zc9aBtXx!7D+LfQpakFkezopg}Bz|og-&(41XSCo5@5M)40%s_$hrF#^L{MkdzC&c{ zK2H9uA9|QPhxv(!J(^~l&xe7BPn@>DNRkqNWe^iSN2t~E|VA^}{YY!(qSDq`7P zt5M3eY@(=nT1o&3DGG{!ae*2EOTZD8MbwHQ+spL1gA!4Ykpx2(uz-pn3MvpqLr+=#3-5fo$ROYbk zl2vJa$F+-vkwxz6J@=Im1L~he1>^}}D*fD?pM|e1??rX(vu*B`@BeYTP4tYK8xIWf zepCj;GIP0uCGG9eBL#*a1f1A99NNQq^@@4AyupC%ZoTI0rS3N)mn*X$a29V8mh|&a z`Zqr*=`Cp0^Rpjr3^=eQ_fHQz3w~UWsk1uk6t=`wTjm>~-j3bbVdF$c^Ch7+(IeG& zUYBwUonn6UlV6;CyLgDL_uhSQ%Vv#y*20P|hg&(|!Qk}<6USl<*b#?=Afqs!edT#U`oSgY?i6Yc z2!~yiJ(!!UX#DBG;x#JlfSouucx6uEID05&!En%~&aruwp7(-acuq z(zI=*u4Z})bq$={W)@C4*3Oi|&a)=79OSA6a-}3O7tqSB-B$aMU$_gLWEPnZnx!Zj zj~|%tNj@2bp-jshX$1-PN1qeqG7rIeX{O|@jg_$R!tfu1rl!L;OLZR0ay6vU*l;eh zC~POaAi?&~bMO-^FL{>%WWRuRV34>dNw6tUG#)+RFrJ*s2)3n-UcgNlW%a*KkJpF| z0t1Le+tOe^#l_gYiIzdBNqtwrV||SNOblnhPMHY6`s!MF$*)lCoF?x6br3ta*!i7Lw>P zSbxY zMCmZ#+>h?No9?OH)@8f^t}YVhrD`nnL8yG4h;1qWhWwiYU!l;c8W)qb)Zk6PXlt#J zLwR1FU#*lF5ZJ^UYl1a*Uho)cPt`7^&JlhHN;-uOplWQ6A*o^Xdx?v2egdv%>_q_K zJXH-zcvgJPa-U{}kJ%iEe4RNkNIjzivm1};+Dc!g&yP>HCF==-+=lfHC;mrc4E0x# zTD49-;WcrUs-*o4atrFiRY;>^tTn7XS_aa3QKS0SjtT+nH=sWKuKIQk^ykOKX}cwL zo8HWr;%L&;iNn>o>{dU5_gsI)bePksg}Wc+M7Ubfi7U`71X&j6lDuwY$xvoX+fZ8i zFDdKtbs^h$L%0B`>Ib4ur6;tGe44dKpXb<0Ig3~p77l51j_)pC`f57iF>vuzK_E#r zX?uuJb>18Dupj#g5R&@gFukkyMHU+|lMmpsK16kq6$l~k6O7uC7qfx%x|kR)ZRPU> zGGx-aP0&=8BtU!w<00JCLTa|*NJoJe{UGdJlcRwh1gEYBg9sBihFn+svJ0rnK%&an z5{AszS72$^Uhx%LDfF!>ALkLCMW_MV5u!?zbYKi{GC)Yys<}{t_xudn-w3qs5kcpX zUDK)pSz6Sxky9xHTDmqa^INt(iVZv7IO*z}a=yvIo(NI}Xu)(tiI1Q`qY|JSTpO{- zc4FHbLQQClTD`gatF31dt(I^xil13o17b)SMjUytYz^X%_*dlN`c2^`0YgpMq;c{~ zMdnwB=_RTlIyx`JFnU<>6Ba^OvHo2C(*V&RGQm^>j=b{x1jGx$P}JhjL;c@j#qyO4 zTp#puk6=k6r)|E+Bh-01s-V(K)#Jvv7^6|s6{CcD)K5n=?ly?Ocd`%hY5?BQm_pCD z@YsWdCZ@euRN=mP1)?^-G(*YB*)3=|mx`#yGmfcm7puJA*2x_CUr$ue4^{0&%D(NZ zpW>zCMtPMlrO>;x|G5rv=a@nwqW+LkJqcOmV2W&LyLbN_VhyYvV1mm$J9Nght%bJE z&ISZY<`S^y>1W@MiMdNG#iE?PJ}9rGHoz>#K*b+(R-_fYnSN5Id%30^=-dT}X4{03 zfgTD*i>Tk3OZpI>aR(6HRVdNb@C6KZfSC8n?;-Ah<&IHme*I+*Lfi=ua|dM^QL`1q zY(}BmSn6jXV&>S1MQNXj`uijGwS`i+ueWbQz;Dr`N*lKxY8u_6^=xgzSx^usBL2F``(vuc0j=!MnM)$3l+K;E3iNW2h#i&3m6Jn}&_ z@TKlt{@znf>V#+fi+?du96QPNgl?I1$YYEdEC(~NH<+eICcs#!U!Sh+kc(XrIRzx% ziX`7HDO4Ze^E>lmvX(O@2G?Lc@o@;k=Vsv3XKxeoi*9j&0REDV=TPu1#G;^%?}uc+4*s!| zJIcY3onpq9!Y|hYF1(p=SHYa^Jv9D=cFKhldF7i+{Kb zp6l|?ep_TBrqEg}l3Bcm12NJImAGM)(#i+di53^eK!B^uZT?{=x}ifVey%`tcc$t! zF|W-d{)Z17xfJO@V~hu8oa-uy(cl9{>BFma5HDU1v&EtbE2e2G5~v=~Yp0PN;MMS` zh`N{-PVS(<9EJ`8(ole^&%|gj9fOSgCf&SX-Pa?-Av2y?ibd~go>kBo8|@TMW$8bC z{6-~9juph6aa}f~U}^6m7)m|EG({gk7T5&{hNk&f5y{1S0*C@KLaKjBDsgS zBGx(xhBdd{5G6SQ>UpM|jqZbt^#XX`&qRSZ{+CF}C)Kb+v-q|5tF++H!*#98 Date: Sat, 15 Jun 2024 22:11:13 -0400 Subject: [PATCH 0221/2091] docs: add conduwuit-bin to arch-linux.md Signed-off-by: strawberry --- docs/deploying/arch-linux.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/deploying/arch-linux.md b/docs/deploying/arch-linux.md index ec9fe25c..e65c2c92 100644 --- a/docs/deploying/arch-linux.md +++ b/docs/deploying/arch-linux.md @@ -6,3 +6,4 @@ The conduwuit AUR packages are community maintained and are not maintained by co - [conduwuit](https://aur.archlinux.org/packages/conduwuit) - latest tagged conduwuit - [conduwuit-git](https://aur.archlinux.org/packages/conduwuit-git) - latest git conduwuit from `main` branch +- [conduwuit-bin](https://aur.archlinux.org/packages/conduwuit-bin) - latest tagged conduwuit static binary From 4942d684eae65ac28ef44a7c8e50382665d9ce4a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 15 Jun 2024 23:13:32 -0400 Subject: [PATCH 0222/2091] bump gitlab nix ci to 2.23.0 Signed-off-by: strawberry --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f94d7b6f..cab57c6f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ before_script: ci: stage: ci - image: nixos/nix:2.22.1 + image: nixos/nix:2.23.0 script: # Cache CI dependencies - ./bin/nix-build-and-cache ci @@ -83,7 +83,7 @@ ci: artifacts: stage: artifacts - image: nixos/nix:2.22.1 + image: nixos/nix:2.23.0 script: - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl - cp result/bin/conduit x86_64-unknown-linux-musl From 3341a8e56e7dfe1f8c2fb9ee9e28be152d7da274 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 16 Jun 2024 12:39:07 -0400 Subject: [PATCH 0223/2091] ci: allow drafts to run artifact builds again Signed-off-by: strawberry --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dfecd34f..0fe17593 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,7 +167,6 @@ jobs: name: Build runs-on: ubuntu-latest needs: tests - if: github.event.pull_request.draft != true strategy: matrix: include: From ddf327e8b61c6f70d896a45d559d65361952fe11 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 16 Jun 2024 13:07:23 -0400 Subject: [PATCH 0224/2091] bump termimad Signed-off-by: strawberry --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b716b1c9..00d23a74 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4010,9 +4010,9 @@ dependencies = [ [[package]] name = "termimad" -version = "0.29.2" +version = "0.29.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aab6c8572830b10362f27e242c7c5e749f062ec310b76a0d0b56670eca81f28e" +checksum = "30d4260ceb786f1d101630049b2a914479f3c049013f9323bd031dfed3ff70ff" dependencies = [ "coolor", "crokey", diff --git a/Cargo.toml b/Cargo.toml index 4c10bb0c..0c6c6e25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -420,7 +420,7 @@ version = "14.0.0" default-features = false [workspace.dependencies.termimad] -version = "0.29.2" +version = "0.29.3" # # Patches From 9e9b256b83d1d74249e890d20bb2d81852171969 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 16 Jun 2024 15:20:57 -0400 Subject: [PATCH 0225/2091] temp: comment out hardened_malloc code sorry but this is just non-functional in our build system, i haven't had time to figure out why nix doesn't like what i'm doing with it, and i haven't heard anyone using this (the user who i anticipated would use this has not updated their conduwuit in many months) Signed-off-by: strawberry --- src/core/alloc/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/core/alloc/mod.rs b/src/core/alloc/mod.rs index ceb9f498..4c42b1e6 100644 --- a/src/core/alloc/mod.rs +++ b/src/core/alloc/mod.rs @@ -7,10 +7,12 @@ pub mod je; pub use je::{memory_stats, memory_usage}; // hardened_malloc -#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", target_os = "linux", not(feature = "jemalloc")))] -pub mod hardened; -#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", target_os = "linux", not(feature = "jemalloc")))] -pub use hardened::{memory_stats, memory_usage}; +//#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", target_os = +//#[cfg(all(not(target_env "linux", not(feature = "jemalloc")))] +//pub mod hardened; +//#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", target_os = +//#[cfg(all(not(target_env "linux", not(feature = "jemalloc")))] +//pub use hardened::{memory_stats, memory_usage}; // default, enabled when none or multiple of the above are enabled #[cfg(any( From a5de4d30bb398a84ecf83e8c84e8deef1e502ced Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 16 Jun 2024 15:34:41 -0400 Subject: [PATCH 0226/2091] ci: add forgejo/gitea actions file Signed-off-by: strawberry --- .gitea/workflows/ci.yml | 254 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 .gitea/workflows/ci.yml diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 00000000..d2175cd2 --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,254 @@ +name: CI and Artifacts + +on: + pull_request: + push: + # documentation workflow deals with this or is not relevant for this workflow + paths-ignore: + - '*.md' + - 'conduwuit-example.toml' + - 'book.toml' + - '.gitlab-ci.yml' + - '.gitignore' + - 'renovate.json' + - 'docs/**' + - 'debian/**' + - 'docker/**' + branches: + - main + tags: + - '*' + # Allows you to run this workflow manually from the Actions tab + #workflow_dispatch: + +#concurrency: +# group: ${{ gitea.head_ref || gitea.ref_name }} +# cancel-in-progress: true + +env: + # Required to make some things output color + TERM: ansi + # Publishing to my nix binary cache + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + # conduwuit.cachix.org + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + # Just in case incremental is still being set to true, speeds up CI + CARGO_INCREMENTAL: 0 + # Custom nix binary cache if fork is being used + ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} + ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} + # Get error output from nix that we can actually use + NIX_CONFIG: show-trace = true + +#permissions: +# packages: write +# contents: read + +jobs: + tests: + name: Test + runs-on: ubuntu-latest + steps: + - name: Sync repository + uses: https://github.com/https://github.com/actions/checkout@v4 + + - name: Tag comparison check + if: startsWith(gitea.ref, 'refs/tags/v') + run: | + # Tag mismatch with latest repo tag check to prevent potential downgrades + LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) + + if [ $LATEST_TAG != ${{ gitea.ref_name }} ]; then + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + - name: Install Nix + uses: https://github.com/DeterminateSystems/nix-installer-action@main + with: + diagnostic-endpoint: "" + extra-conf: "experimental-features = nix-command flakes" + + - name: Enable Cachix binary cache + run: | + nix profile install nixpkgs#cachix + cachix use crane + cachix use nix-community + + - name: Configure Magic Nix Cache + uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main + with: + diagnostic-endpoint: "" + upstream-cache: "https://attic.kennel.juneis.dog" + + - name: Apply Nix binary cache configuration + run: | + sudo tee -a /etc/nix/nix.conf > /dev/null < /dev/null < "$HOME/.direnvrc" + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + direnv allow + nix develop .#all-features --command true + + - name: Cache CI dependencies + run: | + bin/nix-build-and-cache ci + + - name: Run CI tests + run: | + direnv exec . engage > >(tee -a test_output.log) + + - name: Sync Complement repository + uses: https://github.com/actions/checkout@v4 + with: + repository: 'matrix-org/complement' + path: complement_src + + - name: Run Complement tests + run: | + direnv exec . bin/complement 'complement_src' 'complement_test_logs.jsonl' 'complement_test_results.jsonl' + cp -v -f result complement_oci_image.tar.gz + + - name: Upload Complement OCI image + uses: https://github.com/actions/upload-artifact@v4 + with: + name: complement_oci_image.tar.gz + path: complement_oci_image.tar.gz + if-no-files-found: error + + - name: Upload Complement logs + uses: https://github.com/actions/upload-artifact@v4 + with: + name: complement_test_logs.jsonl + path: complement_test_logs.jsonl + if-no-files-found: error + + - name: Upload Complement results + uses: https://github.com/actions/upload-artifact@v4 + with: + name: complement_test_results.jsonl + path: complement_test_results.jsonl + if-no-files-found: error + + - name: Diff Complement results with checked-in repo results + run: | + diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log) + echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + - name: Update Job Summary + if: success() || failure() + run: | + if [ ${{ job.status }} == 'success' ]; then + echo '# ✅ completed suwuccessfully' >> $GITHUB_STEP_SUMMARY + else + echo '```' >> $GITHUB_STEP_SUMMARY + tail -n 40 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + + build: + name: Build + runs-on: ubuntu-latest + needs: tests + strategy: + matrix: + include: + - target: aarch64-unknown-linux-musl + - target: x86_64-unknown-linux-musl + steps: + - name: Sync repository + uses: https://github.com/actions/checkout@v4 + + - name: Install Nix + uses: https://github.com/DeterminateSystems/nix-installer-action@main + + - name: Install and enable Cachix binary cache + run: | + nix profile install nixpkgs#cachix + cachix use crane + cachix use nix-community + + - name: Configure Magic Nix Cache + uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main + + - name: Apply Nix binary cache configuration + run: | + sudo tee -a /etc/nix/nix.conf > /dev/null < /dev/null < "$HOME/.direnvrc" + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + direnv allow + nix develop .#all-features --command true + + - name: Build static ${{ matrix.target }} + run: | + CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }} + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduit target/release/conduwuit + cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + # -p conduit is the main crate name + direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb + mv -v target/release/conduwuit static-${{ matrix.target }} + mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb + + - name: Upload static-${{ matrix.target }} + uses: https://github.com/actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }} + path: static-${{ matrix.target }} + if-no-files-found: error + + - name: Upload deb ${{ matrix.target }} + uses: https://github.com/actions/upload-artifact@v4 + with: + name: deb-${{ matrix.target }} + path: ${{ matrix.target }}.deb + if-no-files-found: error + compression-level: 0 + + - name: Build OCI image ${{ matrix.target }} + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }} + cp -v -f result oci-image-${{ matrix.target }}.tar.gz + + - name: Upload OCI image ${{ matrix.target }} + uses: https://github.com/actions/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }} + path: oci-image-${{ matrix.target }}.tar.gz + if-no-files-found: error + compression-level: 0 From c921deaee2e905159a4e44c842d3250e9eb5188b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 16 Jun 2024 15:22:58 -0400 Subject: [PATCH 0227/2091] docs: update differences.md Signed-off-by: strawberry --- docs/differences.md | 97 ++++++++++++++++++++++++++++++--------------- 1 file changed, 65 insertions(+), 32 deletions(-) diff --git a/docs/differences.md b/docs/differences.md index 0f0716e1..37d84517 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -5,43 +5,56 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! ## Performance: -- Concurrency support for key fetching for faster remote room joins and room joins that will error less frequently +- Concurrency support for individual homeserver key fetching for faster remote room joins and room joins that will error less frequently - Send `Cache-Control` response header with `immutable` and 1 year cache length for all media requests (download and thumbnail) to instruct clients to cache media, and reduce server load from media requests that could be otherwise cached - Add feature flags and config options to enable/build with zstd, brotli, and/or gzip HTTP body compression (response and request) - Eliminate all usage of the thread-blocking `getaddrinfo(3)` call upon DNS queries, significantly improving federation latency/ping and cache DNS results (NXDOMAINs, successful queries, etc) using hickory-dns / hickory-resolver +- Enable HTTP/2 support on all requests - Vastly improve RocksDB default settings to use new features that help with performance significantly, uses settings tailored to SSDs, various ways to tweak RocksDB, and a conduwuit setting to tell RocksDB to use settings that are tailored to HDDs or slow spinning rust storage or buggy filesystems. -- Add a Cargo build profile for aggressive build-time performance optimisations for release builds (1 codegen unit, no debug, fat LTO, etc, and optimise all crates with same) - Implement database flush and cleanup conduwuit operations when using RocksDB - Implement RocksDB write buffer corking and coalescing in database write-heavy areas - Perform connection pooling and keepalives where necessary to significantly improve federation performance and latency - Various config options to tweak connection pooling, request timeouts, connection timeouts, DNS timeouts and settings, etc with good defaults which also help huge with performance via reusing connections and retrying where needed -- Implement building conduwuit with jemalloc (which extends to the RocksDB jemalloc feature for maximum gains) or hardened_malloc light variant, and produce CI builds with jemalloc for performance (Nix doesn't seem to build [hardened_malloc-rs](https://github.com/girlbossceo/hardened_malloc-rs) properly) -- Add support for caching DNS results with hickory-dns / hickory-resolver in conduwuit (not a replacement for a proper resolver cache, but still far better than nothing) +- Properly get and use the amount of parallelism / tokio workers +- Implement building conduwuit with jemalloc (which extends to the RocksDB jemalloc feature for maximum gains) or hardened_malloc light variant, and io_uring support, and produce CI builds with jemalloc and io_uring by default for performance (Nix doesn't seem to build [hardened_malloc-rs](https://github.com/girlbossceo/hardened_malloc-rs) properly) +- Add support for caching DNS results with hickory-dns / hickory-resolver in conduwuit (not a replacement for a proper resolver cache, but still far better than nothing), also properly falls back on TCP for UDP errors or if a SRV response is too large - Add config option for using DNS over TCP, and config option for controlling A/AAAA record lookup strategy (e.g. don't query AAAA records if you don't have IPv6 connectivity) - Overall significant database, Client-Server, and federation performance and latency improvements (check out the ping room leaderboards if you don't believe me :>) - Add config options for RocksDB compression and bottommost compression, including choosing the algorithm and compression level - Use [loole](https://github.com/mahdi-shojaee/loole) MPSC channels instead of tokio MPSC channels for huge performance boosts in sending channels (mainly relevant for federation) and presence channels - Use `tracing`/`log`'s `release_max_level_info` feature to improve performance, build speeds, binary size, and CPU usage in release builds by avoid compiling debug/trace log level macros that users will generally never use (can be disabled with a build-time feature flag) -- Enable RocksDB async read I/O via `io_uring` by default +- Remove some unnecessary checks on EDU handling for incoming transactions, effectively speeding them up +- Simplify, dedupe, etc huge chunks of the codebase, including some that were unnecessary overhead, binary bloats, or preventing compiler/linker optimisations -## General Fixes: +## General Fixes/Features: +- Add legacy Element client hack fixing password changes and deactivations on legacy Element Android/iOS due to usage of an unspecced `user` field for UIAA - Raise and improve all the various request timeouts making some things like room joins and client bugs error less or none at all than they should, and make them all user configurable - Add missing `reason` field to user ban events (`/ban`) -- Fixed spec compliance issue with room version 8 - 11 joins (https://github.com/matrix-org/synapse/issues/16717 / https://github.com/matrix-org/matrix-spec/issues/1708) -- Safer and cleaner shutdowns on both database side as we run cleanup on shutdown and exits database loop better (no potential hanging issues in database loop), overall cleaner shutdown logic +- Safer and cleaner shutdowns across incoming/outgoing requests (graceful shutdown) and the database - Stop sending `make_join` requests on room joins if 15 servers respond with `M_UNSUPPORTED_ROOM_VERSION` or `M_INVALID_ROOM_VERSION` - Stop sending `make_join` requests if 50 servers cannot provide `make_join` for us - Respect *most* client parameters for `/media/` requests (`allow_redirect` still needs work) -- Increased graceful shutdown timeout from a low 60 seconds to 180 seconds to avoid killing connections and let the remaining ones finish processing - Return joined member count of rooms for push rules/conditions instead of a hardcoded value of 10 - Make `CONDUIT_CONFIG` optional, relevant for container users that configure only by environment variables and no longer need to set `CONDUIT_CONFIG` to an empty string. - Allow HEAD and PATCH (MSC4138) HTTP requests in CORS for clients (despite not being explicity mentioned in Matrix spec, HTTP spec says all HEAD requests need to behave the same as GET requests, Synapse supports HEAD requests) +- Fix using conduwuit with flake-compat on NixOS - Resolve and remove some "features" from upstream that result in concurrency hazards, exponential backoff issues, or arbitrary performance limiters - Find more servers for outbound federation `/hierarchy` requests instead of just the room ID server name - Support for suggesting servers to join through at `/_matrix/client/v3/directory/room/{roomAlias}` - Support for suggesting servers to join through us at `/_matrix/federation/v1/query/directory` -- Add workaround for [Out Of Your Element](https://gitdab.com/cadence/out-of-your-element) appservice bridge to make it functional on conduwuit (bug has already been reported) +- Misc edge-case search fixes (e.g. potentially missing some events) +- Misc `/sync` fixes (e.g. returning unnecessary data or incorrect/invalid responses) +- Add `replaces_state` and `prev_sender` in `unsigned` for state event changes which primarily makes Element's "See history" button on a state event functional +- Fix Conduit not allowing incoming federation requests for various world readable rooms +- Fix Conduit not respecting the client-requested file name on media requests +- Prevent sending junk / non-membership events to `/send_join` and `/send_leave` endpoints +- Only allow the requested membership type on `/send_join` and `/send_leave` endpoints (e.g. don't allow leave memberships on join endpoints) +- Prevent state key impersonation on `/send_join` and `/send_leave` endpoints +- Validate `X-Matrix` origin and request body `"origin"` field on incoming transactions +- Add `GET /_matrix/client/v1/register/m.login.registration_token/validity` endpoint +- Explicitly define support for sliding sync at `/_matrix/client/versions` (`org.matrix.msc3575`) +- Fix seeing empty status messages on user presences ## Moderation: @@ -50,13 +63,15 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Add support for serving `support` well-known from `[well_known.support]` (MSC1929) - Config option to forbid publishing rooms to the room directory (`lockdown_public_room_directory`) except for admins - Admin commands to delete room aliases and unpublish rooms from our room directory -- For all [`/report`](https://spec.matrix.org/v1.9/client-server-api/#post_matrixclientv3roomsroomidreporteventid) requests: check if the reported event ID belongs to the reported room ID, raise report reasoning character limit to 750, fix broken formatting, make a small delayed random response per spec suggestion on privacy, and check if the sender user is in the reported room. +- For all [`/report`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3roomsroomidreporteventid) requests: check if the reported event ID belongs to the reported room ID, raise report reasoning character limit to 750, fix broken formatting, make a small delayed random response per spec suggestion on privacy, and check if the sender user is in the reported room. - Support blocking servers from downloading remote media from, returning a 404 - Don't allow `m.call.invite` events to be sent in public rooms (prevents calling the entire room) -- On new public room creations, only allow moderators to send `m.call.invite`, `org.matrix.msc3401.call`, and `org.matrix.msc3401.call.member` events +- On new public room creations, only allow moderators to send `m.call.invite`, `org.matrix.msc3401.call`, and `org.matrix.msc3401.call.member` events to prevent unprivileged users from calling the entire room - Add support for a "global ACLs" feature (`forbidden_remote_server_names`) that blocks inbound remote room invites, room joins by room ID on server name, room joins by room alias on server name, incoming federated joins, and incoming federated room directory requests. This is very helpful for blocking servers that are purely toxic/bad and serve no value in allowing our users to suffer from things like room invite spam or such. Please note that this is not a substitute for room ACLs. - Add support for a config option to forbid our local users from sending federated room directory requests for (`forbidden_remote_room_directory_server_names`). Similar to above, useful for blocking servers that help prevent our users from wandering into bad areas of Matrix via room directories of those malicious servers. - Add config option for auto remediating/deactivating local non-admin users who attempt to join bad/forbidden rooms (`auto_deactivate_banned_room_attempts`) +- Deactivating users will remove their profile picture, blurhash, display name, and leave all rooms by default just like Synapse and for additional privacy +- Reject some EDUs from ACL'd users such as read receipts and typing indicators ## Privacy/Security: @@ -69,10 +84,10 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Config option to block non-admin users from sending room invites or receiving remote room invites. Admin users are still allowed. - Config option to disable incoming and/or outgoing remote read receipts - Config option to disable incoming and/or outgoing remote typing indicators -- Config option to disable incoming, outgoing, and/or local presence +- Config option to disable incoming, outgoing, and/or local presence and for timing out remote users - Sanitise file names for the `Content-Disposition` header for all media requests (thumbnails, downloads, uploads) -- Return `inline` or `attachment` based on the detected file MIME type for the `Content-Disposition` and only allow images/videos/text/audio to be `inline` -- Send secure default HTTP headers such as a strong restrictive CSP, deny iframes, disable `X-XSS-Protection`, disable interest cohort in `Permission-Policy`, etc to mitigate any potential attack surface such as from untrusted media +- Media repository on handling `Content-Disposition` and `Content-Type` is fully spec compliant and secured +- Send secure default HTTP headers such as a strong restrictive CSP (see MSC4149), deny iframes, disable `X-XSS-Protection`, disable interest cohort in `Permission-Policy`, etc to mitigate any potential attack surface such as from untrusted media ## Administration/Logging: @@ -81,36 +96,45 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Substantially clean up, improve, and fix logging (less noisy dead server logging, registration attempts, more useful troubleshooting logging, proper error propagation, etc) - Configurable RocksDB logging (`LOG` files) with proper defaults (rotate, max size, verbosity, etc) to stop LOG files from accumulating so much - Explicit startup error if your configuration allows open registration without a token or such like Synapse with a way to bypass it if needed -- Replace the lightning bolt emoji option with support for setting any arbitrary text (e.g. another emoji) to suffix to all new user registrations, with a conduwuit default of 🏳️‍⚧️ +- Replace the lightning bolt emoji option with support for setting any arbitrary text (e.g. another emoji) to suffix to all new user registrations, with a conduwuit default of "🏳️‍⚧️" - Implement config option to auto join rooms upon registration - Warn on unknown config options specified - Add `/_conduwuit/server_version` route to return the version of conduwuit without relying on the federation API `/_matrix/federation/v1/version` +- Add `/_conduwuit/local_user_count` route to return the amount of registered active local users on your homeserver *if federation is enabled* - Add configurable RocksDB recovery modes to aid in recovering corrupted RocksDB databases - Support config options via `CONDUWUIT_` prefix and accessing non-global struct config options with the `__` split (e.g. `CONDUWUIT_WELL_KNOWN__SERVER`) -- Add support for listening on multiple TCP ports -- Disable update check by default as it's not useful for conduwuit +- Add support for listening on multiple TCP ports and multiple addresses - **Opt-in** Sentry.io telemetry and metrics, mainly used for crash reporting +- Log the client IP on various requests such as registrations, banned room join attempts, logins, deactivations, federation transactions, etc +- Fix Conduit dropping some remote server federation response errors ## Maintenance/Stability: - GitLab CI ported to GitHub Actions -- Repo is mirrored to GitHub, GitLab, git.gay, git.girlcock.ceo, sourcehut, and Codeberg (see README.md for their links) +- Add support for the Matrix spec compliance test suite [Complement](https://github.com/matrix-org/complement/) via the Nix flake and various other fixes for it +- Implement running and diff'ing Complement results in CI and error if any mismatch occurs to prevent large cases of conduwuit regressions +- Repo is (officially) mirrored to GitHub, GitLab, git.gay, git.girlcock.ceo, sourcehut, and Codeberg (see README.md for their links) - Docker container images published to GitLab Container Registry, GitHub Container Registry, and Dockerhub - Extensively revamp the example config to be extremely helpful and useful to both new users and power users - Fixed every single clippy (default lints) and rustc warnings, including some that were performance related or potential safety issues / unsoundness - Add a **lot** of other clippy and rustc lints and a rustfmt.toml file -- Has [Renovate](https://docs.renovatebot.com/), [Trivy](https://github.com/aquasecurity/trivy-action), and keeps ALL dependencies as up to date as possible +- Repo uses [Renovate](https://docs.renovatebot.com/), [Trivy](https://github.com/aquasecurity/trivy-action), and keeps ALL dependencies as up to date as possible - Attempts and interest in removing extreme and unnecessary panics/unwraps/expects that can lead to denial of service or such (upstream and upstream contributors want this unusual behaviour for some reason) - Purge unmaintained/irrelevant/broken database backends (heed, sled, persy) and other unnecessary code or overhead - webp support for images - Add cargo audit support to CI -- CI tests with all features -- Add timestamp by commit date support to building OCI images for keeping image build reproducibility and still have a meaningful "last modified date" for OCI image metadata -- Update rusqlite/sqlite (not that you should be using it) +- CI tests for all sorts of feature matrixes (jemalloc, non-defaullt, all features, etc) +- Add static and dynamic linking smoke tests in CI to prevent any potential linking regressions for Complement, static binaries, Nix devshells, etc +- Add timestamp by commit date when building OCI images for keeping image build reproducibility and still have a meaningful "last modified date" for OCI image +- Add timestamp by commit date via `SOURCE_DATE_EPOCH` for Debian packages - Startup check if conduwuit running in a container and is listening on 127.0.0.1 (generally containers are using NAT networking and 0.0.0.0 is the intended listening address) +- Add a panic catcher layer to return panic messages in HTTP responses if a panic occurs ## Admin Room: +- Add support for a console CLI interface that can issue admin commands and output them in your terminal +- Add support for an admin-user-only commandline admin room interface that can be issued in any room with the `\\!admin` or `\!admin` prefix and returns the response as yourself in the same room +- Add admin commands for uptime, server startup, server shutdown, and server restart - Fix admin room handler to not panic/crash if the admin room command response fails (e.g. too large message) - Add command to dynamically change conduwuit's tracing log level filter on the fly - Add admin command to fetch a server's `/.well-known/matrix/support` file @@ -124,17 +148,26 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Add admin command to bulk delete media via a codeblock list of MXC URLs. - Add admin command to delete both the thumbnail and media MXC URLs from an event ID (e.g. from an abuse report) - Add admin command to list all the rooms a local user is joined in -- Add admin command to delete all remote media in the past X minutes as a form of deleting media that you don't want on your server that a remote user posted in a room -- Add admin command to return a room's state +- Add admin command to list joined members in a room +- Add admin command to view the room topic of a room +- Add admin command to delete all remote media in the past X minutes as a form of deleting media that you don't want on your server that a remote user posted in a room, a `--force` flag to ignore errors, and support for reading `last modified time` instead of `creation time` for filesystems that don't support file created metadata +- Add admin command to return a room's full/complete state - Admin debug command to fetch a PDU from a remote server and inserts it into our database/timeline as backfill - Add admin command to delete media via a specific MXC. This deletes the MXC from our database, and the file locally. - Add admin commands for banning (blocking) room IDs from our local users joining (admins are always allowed) and evicts all our local users from that room, in addition to bulk room banning support, and blocks room invites (remote and local) to the banned room, as a moderation feature - Add admin commands to output jemalloc memory stats and memory usage -- Add admin command to get conduwuit's uptime - Add admin command to get rooms a *remote* user shares with us +- Add debug admin commands to get the earliest and latest PDU in a room +- Add debug admin command to echo a message +- Add admin command to insert rooms tags for a user, most useful for inserting the `m.server_notice` tag on your admin room to make it "persistent" in the "System Alerts" section of Element +- Add experimental admin debug command for Dendrite's `AdminDownloadState` (`/admin/downloadState/{serverName}/{roomID}`) admin API endpoint to download and use a remote server's room state in the room +- Disable URL previews by default in the admin room due to various command outputs having "URLs" in them that clients may needlessly render/request +- Extend memory usage admin server command to support showing memory allocator stats such as jemalloc's +- Add admin debug command to see memory allocator's full extended debug statistics such as jemalloc's ## Misc: +- Add guest support for accessing TURN servers via `turn_allow_guests` like Synapse - Support for creating rooms with custom room IDs like Maunium Synapse (`room_id` request body field to `/createRoom`) - Query parameter `?format=event|content` for returning either the room state event's content (default) for the full room state event on ` /_matrix/client/v3/rooms/{roomId}/state/{eventType}[/{stateKey}]` requests (see https://github.com/matrix-org/matrix-spec/issues/1047) - Add **optional** feature flag to use SHA256 key names for media instead of base64 to overcome filesystem file name length limitations (OS error file name too long) @@ -146,15 +179,15 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Implement legacy Matrix `/v1/` media endpoints that some clients and servers may still call - Config option to change Conduit's behaviour of homeserver key fetching (`query_trusted_key_servers_first`). This option sets whether conduwuit will query trusted notary key servers first before the individual homeserver(s), or vice versa which may help in joining certain rooms. - Implement unstable MSC2666 support for querying mutual rooms with a user -- Assume well-knowns are broken if they exceed past 10000 characters. -- Add support for the Matrix spec compliance test suite [Complement](https://github.com/matrix-org/complement/) via the Nix flake and various other fixes for it +- Implement unstable MSC4125 support for specifying servers to join via on federated invites +- Make conduwuit build and be functional under Nix + macOS +- Log out all sessions after unsetting the emergency password +- Assume well-knowns are broken if they exceed past 12288 characters. - Add support for listening on both HTTP and HTTPS if using direct TLS with conduwuit for usecases such as Complement -- Implement running and diff'ing Complement results in CI -- Interest in supporting other operating systems such as macOS, BSDs, and Windows, and getting them added into CI and doing builds for them - Add config option for disabling RocksDB Direct IO if needed -- Add various documentation on maintaining conduwuit, using RocksDB online backups, some troubleshooting, using admin commands, etc +- Add various documentation on maintaining conduwuit, using RocksDB online backups, some troubleshooting, using admin commands, moderation documentation, etc - (Developers): Add support for [hot reloadable/"live" modular development](development/hot_reload.md) - (Developers): Add support for tokio-console - (Developers): Add support for tracing flame graphs -- Add `release-debuginfo` Cargo build profile - No cryptocurrency donations allowed, conduwuit is fully maintained by independent queer maintainers, and with a strong priority on inclusitivity and comfort for protected groups 🏳️‍⚧️ +- [Add a community Code of Conduct for all conduwuit community spaces, primarily the Matrix space](https://conduwuit.puppyirl.gay/conduwuit_coc.html) From 73dd4b1f8f3e55063a523a28b347cc48a60e748b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 16 Jun 2024 18:54:03 -0400 Subject: [PATCH 0228/2091] ci: enable accept-flake-config, use the same args everywhere Signed-off-by: strawberry --- .gitea/workflows/ci.yml | 16 +++++++++++++--- .github/workflows/ci.yml | 16 +++++++++++++--- .github/workflows/documentation.yml | 4 +++- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index d2175cd2..c501dd2c 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -68,7 +68,9 @@ jobs: uses: https://github.com/DeterminateSystems/nix-installer-action@main with: diagnostic-endpoint: "" - extra-conf: "experimental-features = nix-command flakes" + extra-conf: | + experimental-features = nix-command flakes + accept-flake-config = true - name: Enable Cachix binary cache run: | @@ -178,6 +180,11 @@ jobs: - name: Install Nix uses: https://github.com/DeterminateSystems/nix-installer-action@main + with: + diagnostic-endpoint: "" + extra-conf: | + experimental-features = nix-command flakes + accept-flake-config = true - name: Install and enable Cachix binary cache run: | @@ -187,12 +194,15 @@ jobs: - name: Configure Magic Nix Cache uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main + with: + diagnostic-endpoint: "" + upstream-cache: "https://attic.kennel.juneis.dog" - name: Apply Nix binary cache configuration run: | sudo tee -a /etc/nix/nix.conf > /dev/null < /dev/null < Date: Sun, 16 Jun 2024 19:17:29 -0400 Subject: [PATCH 0229/2091] ci: typo, store path is `/conduwuit` Signed-off-by: strawberry --- .gitea/workflows/ci.yml | 4 ++-- .github/workflows/ci.yml | 4 ++-- .github/workflows/documentation.yml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index c501dd2c..6d063ee7 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main with: diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog" + upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" - name: Apply Nix binary cache configuration run: | @@ -196,7 +196,7 @@ jobs: uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main with: diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog" + upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" - name: Apply Nix binary cache configuration run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a7c10cb..a23ae428 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: uses: DeterminateSystems/magic-nix-cache-action@main with: diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog" + upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" - name: Apply Nix binary cache configuration run: | @@ -196,7 +196,7 @@ jobs: uses: DeterminateSystems/magic-nix-cache-action@main with: diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog" + upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" - name: Apply Nix binary cache configuration run: | diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 4a48b37f..81a9d687 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -69,12 +69,12 @@ jobs: uses: DeterminateSystems/magic-nix-cache-action@main with: diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog" + upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" - name: Apply Nix binary cache configuration run: | sudo tee -a /etc/nix/nix.conf > /dev/null < Date: Sun, 16 Jun 2024 20:58:17 -0400 Subject: [PATCH 0230/2091] docs: add slight theme changes, better title Signed-off-by: strawberry --- book.toml | 2 +- theme/css/chrome.css | 608 ++++++++++++++++++++++++++++++++++++++++ theme/css/general.css | 234 ++++++++++++++++ theme/css/variables.css | 279 ++++++++++++++++++ 4 files changed, 1122 insertions(+), 1 deletion(-) create mode 100644 theme/css/chrome.css create mode 100644 theme/css/general.css create mode 100644 theme/css/variables.css diff --git a/book.toml b/book.toml index 82c2966d..1d32c766 100644 --- a/book.toml +++ b/book.toml @@ -1,5 +1,5 @@ [book] -title = "conduwuit" +title = "conduwuit 🏳️‍⚧️ 💜 🦴" description = "conduwuit, which is a well-maintained fork of Conduit, is a simple, fast and reliable chat server for the Matrix protocol" language = "en" authors = ["strawberry (June)"] diff --git a/theme/css/chrome.css b/theme/css/chrome.css new file mode 100644 index 00000000..52b35c2c --- /dev/null +++ b/theme/css/chrome.css @@ -0,0 +1,608 @@ +/* CSS for UI elements (a.k.a. chrome) */ + +@import 'variables.css'; + +html { + scrollbar-color: var(--scrollbar) var(--bg); +} +#searchresults a, +.content a:link, +a:visited, +a > .hljs { + color: var(--links); +} + +/* + body-container is necessary because mobile browsers don't seem to like + overflow-x on the body tag when there is a tag. +*/ +#body-container { + /* + This is used when the sidebar pushes the body content off the side of + the screen on small screens. Without it, dragging on mobile Safari + will want to reposition the viewport in a weird way. + */ + overflow-x: clip; +} + +/* Menu Bar */ + +#menu-bar, +#menu-bar-hover-placeholder { + z-index: 101; + margin: auto calc(0px - var(--page-padding)); +} +#menu-bar { + position: relative; + display: flex; + flex-wrap: wrap; + background-color: var(--bg); + border-block-end-color: var(--bg); + border-block-end-width: 1px; + border-block-end-style: solid; +} +#menu-bar.sticky, +.js #menu-bar-hover-placeholder:hover + #menu-bar, +.js #menu-bar:hover, +.js.sidebar-visible #menu-bar { + position: -webkit-sticky; + position: sticky; + top: 0 !important; +} +#menu-bar-hover-placeholder { + position: sticky; + position: -webkit-sticky; + top: 0; + height: var(--menu-bar-height); +} +#menu-bar.bordered { + border-block-end-color: var(--table-border-color); +} +#menu-bar i, #menu-bar .icon-button { + position: relative; + padding: 0 8px; + z-index: 10; + line-height: var(--menu-bar-height); + cursor: pointer; + transition: color 0.5s; +} +@media only screen and (max-width: 420px) { + #menu-bar i, #menu-bar .icon-button { + padding: 0 5px; + } +} + +.icon-button { + border: none; + background: none; + padding: 0; + color: inherit; +} +.icon-button i { + margin: 0; +} + +.right-buttons { + margin: 0 15px; +} +.right-buttons a { + text-decoration: none; +} + +.left-buttons { + display: flex; + margin: 0 5px; +} +.no-js .left-buttons button { + display: none; +} + +.menu-title { + display: inline-block; + font-weight: 400; + font-size: 2.4rem; + line-height: var(--menu-bar-height); + text-align: center; + margin: 0; + flex: 1; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + color: #bb7fb2; +} +.js .menu-title { + cursor: pointer; +} + +.menu-bar, +.menu-bar:visited, +.nav-chapters, +.nav-chapters:visited, +.mobile-nav-chapters, +.mobile-nav-chapters:visited, +.menu-bar .icon-button, +.menu-bar a i { + color: var(--icons); +} + +.menu-bar i:hover, +.menu-bar .icon-button:hover, +.nav-chapters:hover, +.mobile-nav-chapters i:hover { + color: var(--icons-hover); +} + +/* Nav Icons */ + +.nav-chapters { + font-size: 2.5em; + text-align: center; + text-decoration: none; + + position: fixed; + top: 0; + bottom: 0; + margin: 0; + max-width: 150px; + min-width: 90px; + + display: flex; + justify-content: center; + align-content: center; + flex-direction: column; + + transition: color 0.5s, background-color 0.5s; +} + +.nav-chapters:hover { + text-decoration: none; + background-color: var(--theme-hover); + transition: background-color 0.15s, color 0.15s; +} + +.nav-wrapper { + margin-block-start: 50px; + display: none; +} + +.mobile-nav-chapters { + font-size: 2.5em; + text-align: center; + text-decoration: none; + width: 90px; + border-radius: 5px; + background-color: var(--sidebar-bg); +} + +/* Only Firefox supports flow-relative values */ +.previous { float: left; } +[dir=rtl] .previous { float: right; } + +/* Only Firefox supports flow-relative values */ +.next { + float: right; + right: var(--page-padding); +} +[dir=rtl] .next { + float: left; + right: unset; + left: var(--page-padding); +} + +/* Use the correct buttons for RTL layouts*/ +[dir=rtl] .previous i.fa-angle-left:before {content:"\f105";} +[dir=rtl] .next i.fa-angle-right:before { content:"\f104"; } + +@media only screen and (max-width: 1080px) { + .nav-wide-wrapper { display: none; } + .nav-wrapper { display: block; } +} + +/* sidebar-visible */ +@media only screen and (max-width: 1380px) { + #sidebar-toggle-anchor:checked ~ .page-wrapper .nav-wide-wrapper { display: none; } + #sidebar-toggle-anchor:checked ~ .page-wrapper .nav-wrapper { display: block; } +} + +/* Inline code */ + +:not(pre) > .hljs { + display: inline; + padding: 0.1em 0.3em; + border-radius: 3px; +} + +:not(pre):not(a) > .hljs { + color: var(--inline-code-color); + overflow-x: initial; +} + +a:hover > .hljs { + text-decoration: underline; +} + +pre { + position: relative; +} +pre > .buttons { + position: absolute; + z-index: 100; + right: 0px; + top: 2px; + margin: 0px; + padding: 2px 0px; + + color: var(--sidebar-fg); + cursor: pointer; + visibility: hidden; + opacity: 0; + transition: visibility 0.1s linear, opacity 0.1s linear; +} +pre:hover > .buttons { + visibility: visible; + opacity: 1 +} +pre > .buttons :hover { + color: var(--sidebar-active); + border-color: var(--icons-hover); + background-color: var(--theme-hover); +} +pre > .buttons i { + margin-inline-start: 8px; +} +pre > .buttons button { + cursor: inherit; + margin: 0px 5px; + padding: 3px 5px; + font-size: 14px; + + border-style: solid; + border-width: 1px; + border-radius: 4px; + border-color: var(--icons); + background-color: var(--theme-popup-bg); + transition: 100ms; + transition-property: color,border-color,background-color; + color: var(--icons); +} +@media (pointer: coarse) { + pre > .buttons button { + /* On mobile, make it easier to tap buttons. */ + padding: 0.3rem 1rem; + } + + .sidebar-resize-indicator { + /* Hide resize indicator on devices with limited accuracy */ + display: none; + } +} +pre > code { + display: block; + padding: 1rem; +} + +/* FIXME: ACE editors overlap their buttons because ACE does absolute + positioning within the code block which breaks padding. The only solution I + can think of is to move the padding to the outer pre tag (or insert a div + wrapper), but that would require fixing a whole bunch of CSS rules. +*/ +.hljs.ace_editor { + padding: 0rem 0rem; +} + +pre > .result { + margin-block-start: 10px; +} + +/* Search */ + +#searchresults a { + text-decoration: none; +} + +mark { + border-radius: 2px; + padding-block-start: 0; + padding-block-end: 1px; + padding-inline-start: 3px; + padding-inline-end: 3px; + margin-block-start: 0; + margin-block-end: -1px; + margin-inline-start: -3px; + margin-inline-end: -3px; + background-color: var(--search-mark-bg); + transition: background-color 300ms linear; + cursor: pointer; +} + +mark.fade-out { + background-color: rgba(0,0,0,0) !important; + cursor: auto; +} + +.searchbar-outer { + margin-inline-start: auto; + margin-inline-end: auto; + max-width: var(--content-max-width); +} + +#searchbar { + width: 100%; + margin-block-start: 5px; + margin-block-end: 0; + margin-inline-start: auto; + margin-inline-end: auto; + padding: 10px 16px; + transition: box-shadow 300ms ease-in-out; + border: 1px solid var(--searchbar-border-color); + border-radius: 3px; + background-color: var(--searchbar-bg); + color: var(--searchbar-fg); +} +#searchbar:focus, +#searchbar.active { + box-shadow: 0 0 3px var(--searchbar-shadow-color); +} + +.searchresults-header { + font-weight: bold; + font-size: 1em; + padding-block-start: 18px; + padding-block-end: 0; + padding-inline-start: 5px; + padding-inline-end: 0; + color: var(--searchresults-header-fg); +} + +.searchresults-outer { + margin-inline-start: auto; + margin-inline-end: auto; + max-width: var(--content-max-width); + border-block-end: 1px dashed var(--searchresults-border-color); +} + +ul#searchresults { + list-style: none; + padding-inline-start: 20px; +} +ul#searchresults li { + margin: 10px 0px; + padding: 2px; + border-radius: 2px; +} +ul#searchresults li.focus { + background-color: var(--searchresults-li-bg); +} +ul#searchresults span.teaser { + display: block; + clear: both; + margin-block-start: 5px; + margin-block-end: 0; + margin-inline-start: 20px; + margin-inline-end: 0; + font-size: 0.8em; +} +ul#searchresults span.teaser em { + font-weight: bold; + font-style: normal; +} + +/* Sidebar */ + +.sidebar { + position: fixed; + left: 0; + top: 0; + bottom: 0; + width: var(--sidebar-width); + font-size: 0.875em; + box-sizing: border-box; + -webkit-overflow-scrolling: touch; + overscroll-behavior-y: contain; + background-color: var(--sidebar-bg); + color: var(--sidebar-fg); +} +[dir=rtl] .sidebar { left: unset; right: 0; } +.sidebar-resizing { + -moz-user-select: none; + -webkit-user-select: none; + -ms-user-select: none; + user-select: none; +} +.no-js .sidebar, +.js:not(.sidebar-resizing) .sidebar { + transition: transform 0.3s; /* Animation: slide away */ +} +.sidebar code { + line-height: 2em; +} +.sidebar .sidebar-scrollbox { + overflow-y: auto; + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + padding: 10px 10px; +} +.sidebar .sidebar-resize-handle { + position: absolute; + cursor: col-resize; + width: 0; + right: calc(var(--sidebar-resize-indicator-width) * -1); + top: 0; + bottom: 0; + display: flex; + align-items: center; +} + +.sidebar-resize-handle .sidebar-resize-indicator { + width: 100%; + height: 12px; + background-color: var(--icons); + margin-inline-start: var(--sidebar-resize-indicator-space); +} + +[dir=rtl] .sidebar .sidebar-resize-handle { + left: calc(var(--sidebar-resize-indicator-width) * -1); + right: unset; +} +.js .sidebar .sidebar-resize-handle { + cursor: col-resize; + width: calc(var(--sidebar-resize-indicator-width) - var(--sidebar-resize-indicator-space)); +} +/* sidebar-hidden */ +#sidebar-toggle-anchor:not(:checked) ~ .sidebar { + transform: translateX(calc(0px - var(--sidebar-width) - var(--sidebar-resize-indicator-width))); + z-index: -1; +} +[dir=rtl] #sidebar-toggle-anchor:not(:checked) ~ .sidebar { + transform: translateX(calc(var(--sidebar-width) + var(--sidebar-resize-indicator-width))); +} +.sidebar::-webkit-scrollbar { + background: var(--sidebar-bg); +} +.sidebar::-webkit-scrollbar-thumb { + background: var(--scrollbar); +} + +/* sidebar-visible */ +#sidebar-toggle-anchor:checked ~ .page-wrapper { + transform: translateX(calc(var(--sidebar-width) + var(--sidebar-resize-indicator-width))); +} +[dir=rtl] #sidebar-toggle-anchor:checked ~ .page-wrapper { + transform: translateX(calc(0px - var(--sidebar-width) - var(--sidebar-resize-indicator-width))); +} +@media only screen and (min-width: 620px) { + #sidebar-toggle-anchor:checked ~ .page-wrapper { + transform: none; + margin-inline-start: calc(var(--sidebar-width) + var(--sidebar-resize-indicator-width)); + } + [dir=rtl] #sidebar-toggle-anchor:checked ~ .page-wrapper { + transform: none; + } +} + +.chapter { + list-style: none outside none; + padding-inline-start: 0; + line-height: 2.2em; +} + +.chapter ol { + width: 100%; +} + +.chapter li { + display: flex; + color: var(--sidebar-non-existant); +} +.chapter li a { + display: block; + padding: 0; + text-decoration: none; + color: var(--sidebar-fg); +} + +.chapter li a:hover { + color: var(--sidebar-active); +} + +.chapter li a.active { + color: var(--sidebar-active); +} + +.chapter li > a.toggle { + cursor: pointer; + display: block; + margin-inline-start: auto; + padding: 0 10px; + user-select: none; + opacity: 0.68; +} + +.chapter li > a.toggle div { + transition: transform 0.5s; +} + +/* collapse the section */ +.chapter li:not(.expanded) + li > ol { + display: none; +} + +.chapter li.chapter-item { + line-height: 1.5em; + margin-block-start: 0.6em; +} + +.chapter li.expanded > a.toggle div { + transform: rotate(90deg); +} + +.spacer { + width: 100%; + height: 3px; + margin: 5px 0px; +} +.chapter .spacer { + background-color: var(--sidebar-spacer); +} + +@media (-moz-touch-enabled: 1), (pointer: coarse) { + .chapter li a { padding: 5px 0; } + .spacer { margin: 10px 0; } +} + +.section { + list-style: none outside none; + padding-inline-start: 20px; + line-height: 1.9em; +} + +/* Theme Menu Popup */ + +.theme-popup { + position: absolute; + left: 10px; + top: var(--menu-bar-height); + z-index: 1000; + border-radius: 4px; + font-size: 0.7em; + color: var(--fg); + background: var(--theme-popup-bg); + border: 1px solid var(--theme-popup-border); + margin: 0; + padding: 0; + list-style: none; + display: none; + /* Don't let the children's background extend past the rounded corners. */ + overflow: hidden; +} +[dir=rtl] .theme-popup { left: unset; right: 10px; } +.theme-popup .default { + color: var(--icons); +} +.theme-popup .theme { + width: 100%; + border: 0; + margin: 0; + padding: 2px 20px; + line-height: 25px; + white-space: nowrap; + text-align: start; + cursor: pointer; + color: inherit; + background: inherit; + font-size: inherit; +} +.theme-popup .theme:hover { + background-color: var(--theme-hover); +} + +.theme-selected::before { + display: inline-block; + content: "✓"; + margin-inline-start: -14px; + width: 14px; +} + diff --git a/theme/css/general.css b/theme/css/general.css new file mode 100644 index 00000000..e7d20da7 --- /dev/null +++ b/theme/css/general.css @@ -0,0 +1,234 @@ +/* Base styles and content styles */ + +@import 'variables.css'; + +:root { + /* Browser default font-size is 16px, this way 1 rem = 10px */ + font-size: 62.5%; + color-scheme: var(--color-scheme); +} + +html { + font-family: "Open Sans", sans-serif; + color: var(--fg); + background-color: var(--bg); + text-size-adjust: none; + -webkit-text-size-adjust: none; +} + +body { + margin: 0; + font-size: 1.6rem; + overflow-x: hidden; +} + +code { + font-family: var(--mono-font) !important; + font-size: var(--code-font-size); + direction: ltr !important; +} + +/* make long words/inline code not x overflow */ +main { + overflow-wrap: break-word; +} + +/* make wide tables scroll if they overflow */ +.table-wrapper { + overflow-x: auto; +} + +/* Don't change font size in headers. */ +h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { + font-size: unset; +} + +.left { float: left; } +.right { float: right; } +.boring { opacity: 0.6; } +.hide-boring .boring { display: none; } +.hidden { display: none !important; } + +h2, h3 { margin-block-start: 2.5em; } +h4, h5 { margin-block-start: 2em; } + +.header + .header h3, +.header + .header h4, +.header + .header h5 { + margin-block-start: 1em; +} + +h1:target::before, +h2:target::before, +h3:target::before, +h4:target::before, +h5:target::before, +h6:target::before { + display: inline-block; + content: "»"; + margin-inline-start: -30px; + width: 30px; +} + +/* This is broken on Safari as of version 14, but is fixed + in Safari Technology Preview 117 which I think will be Safari 14.2. + https://bugs.webkit.org/show_bug.cgi?id=218076 +*/ +:target { + /* Safari does not support logical properties */ + scroll-margin-top: calc(var(--menu-bar-height) + 0.5em); +} + +.page { + outline: 0; + padding: 0 var(--page-padding); + margin-block-start: calc(0px - var(--menu-bar-height)); /* Compensate for the #menu-bar-hover-placeholder */ +} +.page-wrapper { + box-sizing: border-box; + background-color: var(--bg); +} +.no-js .page-wrapper, +.js:not(.sidebar-resizing) .page-wrapper { + transition: margin-left 0.3s ease, transform 0.3s ease; /* Animation: slide away */ +} +[dir=rtl] .js:not(.sidebar-resizing) .page-wrapper { + transition: margin-right 0.3s ease, transform 0.3s ease; /* Animation: slide away */ +} + +.content { + overflow-y: auto; + padding: 0 5px 50px 5px; +} +.content main { + margin-inline-start: auto; + margin-inline-end: auto; + max-width: var(--content-max-width); +} +.content p { line-height: 1.45em; } +.content ol { line-height: 1.45em; } +.content ul { line-height: 1.45em; } +.content a { text-decoration: none; } +.content a:hover { text-decoration: underline; } +.content img, .content video { max-width: 100%; } +.content .header:link, +.content .header:visited { + color: var(--fg); +} +.content .header:link, +.content .header:visited:hover { + text-decoration: none; +} + +table { + margin: 0 auto; + border-collapse: collapse; +} +table td { + padding: 3px 20px; + border: 1px var(--table-border-color) solid; +} +table thead { + background: var(--table-header-bg); +} +table thead td { + font-weight: 700; + border: none; +} +table thead th { + padding: 3px 20px; +} +table thead tr { + border: 1px var(--table-header-bg) solid; +} +/* Alternate background colors for rows */ +table tbody tr:nth-child(2n) { + background: var(--table-alternate-bg); +} + + +blockquote { + margin: 20px 0; + padding: 0 20px; + color: var(--fg); + background-color: var(--quote-bg); + border-block-start: .1em solid var(--quote-border); + border-block-end: .1em solid var(--quote-border); +} + +.warning { + margin: 20px; + padding: 0 20px; + border-inline-start: 2px solid var(--warning-border); +} + +.warning:before { + position: absolute; + width: 3rem; + height: 3rem; + margin-inline-start: calc(-1.5rem - 21px); + content: "ⓘ"; + text-align: center; + background-color: var(--bg); + color: var(--warning-border); + font-weight: bold; + font-size: 2rem; +} + +blockquote .warning:before { + background-color: var(--quote-bg); +} + +kbd { + background-color: var(--table-border-color); + border-radius: 4px; + border: solid 1px var(--theme-popup-border); + box-shadow: inset 0 -1px 0 var(--theme-hover); + display: inline-block; + font-size: var(--code-font-size); + font-family: var(--mono-font); + line-height: 10px; + padding: 4px 5px; + vertical-align: middle; +} + +:not(.footnote-definition) + .footnote-definition, +.footnote-definition + :not(.footnote-definition) { + margin-block-start: 2em; +} +.footnote-definition { + font-size: 0.9em; + margin: 0.5em 0; +} +.footnote-definition p { + display: inline; +} + +.tooltiptext { + position: absolute; + visibility: hidden; + color: #fff; + background-color: #333; + transform: translateX(-50%); /* Center by moving tooltip 50% of its width left */ + left: -8px; /* Half of the width of the icon */ + top: -35px; + font-size: 0.8em; + text-align: center; + border-radius: 6px; + padding: 5px 8px; + margin: 5px; + z-index: 1000; +} +.tooltipped .tooltiptext { + visibility: visible; +} + +.chapter li.part-title { + color: var(--sidebar-fg); + margin: 5px 0px; + font-weight: bold; +} + +.result-no-output { + font-style: italic; +} diff --git a/theme/css/variables.css b/theme/css/variables.css new file mode 100644 index 00000000..e7feed98 --- /dev/null +++ b/theme/css/variables.css @@ -0,0 +1,279 @@ +/* Globals */ + +:root { + --sidebar-width: 300px; + --sidebar-resize-indicator-width: 8px; + --sidebar-resize-indicator-space: 2px; + --page-padding: 15px; + --content-max-width: 750px; + --menu-bar-height: 50px; + --mono-font: "Source Code Pro", Consolas, "Ubuntu Mono", Menlo, "DejaVu Sans Mono", monospace, monospace; + --code-font-size: 0.875em + /* please adjust the ace font size accordingly in editor.js */ +} + +/* Themes */ + +.ayu { + --bg: hsl(210, 25%, 8%); + --fg: #c5c5c5; + + --sidebar-bg: #14191f; + --sidebar-fg: #c8c9db; + --sidebar-non-existant: #5c6773; + --sidebar-active: #ffb454; + --sidebar-spacer: #2d334f; + + --scrollbar: var(--sidebar-fg); + + --icons: #737480; + --icons-hover: #b7b9cc; + + --links: #0096cf; + + --inline-code-color: #ffb454; + + --theme-popup-bg: #14191f; + --theme-popup-border: #5c6773; + --theme-hover: #191f26; + + --quote-bg: hsl(226, 15%, 17%); + --quote-border: hsl(226, 15%, 22%); + + --warning-border: #ff8e00; + + --table-border-color: hsl(210, 25%, 13%); + --table-header-bg: hsl(210, 25%, 28%); + --table-alternate-bg: hsl(210, 25%, 11%); + + --searchbar-border-color: #848484; + --searchbar-bg: #424242; + --searchbar-fg: #fff; + --searchbar-shadow-color: #d4c89f; + --searchresults-header-fg: #666; + --searchresults-border-color: #888; + --searchresults-li-bg: #252932; + --search-mark-bg: #e3b171; + + --color-scheme: dark; +} + +.coal { + --bg: hsl(200, 7%, 8%); + --fg: #98a3ad; + + --sidebar-bg: #292c2f; + --sidebar-fg: #a1adb8; + --sidebar-non-existant: #505254; + --sidebar-active: #3473ad; + --sidebar-spacer: #393939; + + --scrollbar: var(--sidebar-fg); + + --icons: #43484d; + --icons-hover: #b3c0cc; + + --links: #2b79a2; + + --inline-code-color: #c5c8c6; + + --theme-popup-bg: #141617; + --theme-popup-border: #43484d; + --theme-hover: #1f2124; + + --quote-bg: hsl(234, 21%, 18%); + --quote-border: hsl(234, 21%, 23%); + + --warning-border: #ff8e00; + + --table-border-color: hsl(200, 7%, 13%); + --table-header-bg: hsl(200, 7%, 28%); + --table-alternate-bg: hsl(200, 7%, 11%); + + --searchbar-border-color: #aaa; + --searchbar-bg: #b7b7b7; + --searchbar-fg: #000; + --searchbar-shadow-color: #aaa; + --searchresults-header-fg: #666; + --searchresults-border-color: #98a3ad; + --searchresults-li-bg: #2b2b2f; + --search-mark-bg: #355c7d; + + --color-scheme: dark; +} + +.light { + --bg: hsl(0, 0%, 100%); + --fg: hsl(0, 0%, 0%); + + --sidebar-bg: #fafafa; + --sidebar-fg: #AE518E; + --sidebar-non-existant: #aaaaaa; + --sidebar-active: #2F7E86; + --sidebar-spacer: #f4f4f4; + + --scrollbar: #8F8F8F; + + --icons: #747474; + --icons-hover: #000000; + + --links: #429EC2; + + --inline-code-color: #9d579d; + + --theme-popup-bg: #fafafa; + --theme-popup-border: #cccccc; + --theme-hover: #e6e6e6; + + --quote-bg: hsl(197, 37%, 96%); + --quote-border: hsl(197, 37%, 91%); + + --warning-border: #ff8e00; + + --table-border-color: hsl(0, 0%, 95%); + --table-header-bg: hsl(0, 0%, 80%); + --table-alternate-bg: hsl(0, 0%, 97%); + + --searchbar-border-color: #aaa; + --searchbar-bg: #fafafa; + --searchbar-fg: #000; + --searchbar-shadow-color: #aaa; + --searchresults-header-fg: #666; + --searchresults-border-color: #888; + --searchresults-li-bg: #e4f2fe; + --search-mark-bg: #a2cff5; + + --color-scheme: light; +} + +.navy { + --bg: hsl(226, 23%, 11%); + --fg: #dfd3db; + + --sidebar-bg: #282d3f; + --sidebar-fg: #fdcbec; + --sidebar-non-existant: #505274; + --sidebar-active: #5BCEFA; + --sidebar-spacer: #2d334f; + + --scrollbar: var(--sidebar-fg); + + --icons: #fdcbec; + --icons-hover: #5BCEFA; + + --links: #5BCEFA; + + --inline-code-color: #bd66bc; + + --theme-popup-bg: #161923; + --theme-popup-border: #737480; + --theme-hover: #282e40; + + --quote-bg: hsl(226, 15%, 17%); + --quote-border: hsl(226, 15%, 22%); + + --warning-border: #ff8e00; + + --table-border-color: hsl(226, 23%, 16%); + --table-header-bg: hsl(226, 23%, 31%); + --table-alternate-bg: hsl(226, 23%, 14%); + + --searchbar-border-color: #aaa; + --searchbar-bg: #aeaec6; + --searchbar-fg: #000; + --searchbar-shadow-color: #aaa; + --searchresults-header-fg: #5f5f71; + --searchresults-border-color: #5c5c68; + --searchresults-li-bg: #242430; + --search-mark-bg: #a2cff5; + + --color-scheme: dark; +} + +.rust { + --bg: hsl(60, 9%, 87%); + --fg: #262625; + + --sidebar-bg: #3b2e2a; + --sidebar-fg: #c8c9db; + --sidebar-non-existant: #505254; + --sidebar-active: #e69f67; + --sidebar-spacer: #45373a; + + --scrollbar: var(--sidebar-fg); + + --icons: #737480; + --icons-hover: #262625; + + --links: #2b79a2; + + --inline-code-color: #6e6b5e; + + --theme-popup-bg: #e1e1db; + --theme-popup-border: #b38f6b; + --theme-hover: #99908a; + + --quote-bg: hsl(60, 5%, 75%); + --quote-border: hsl(60, 5%, 70%); + + --warning-border: #ff8e00; + + --table-border-color: hsl(60, 9%, 82%); + --table-header-bg: #b3a497; + --table-alternate-bg: hsl(60, 9%, 84%); + + --searchbar-border-color: #aaa; + --searchbar-bg: #fafafa; + --searchbar-fg: #000; + --searchbar-shadow-color: #aaa; + --searchresults-header-fg: #666; + --searchresults-border-color: #888; + --searchresults-li-bg: #dec2a2; + --search-mark-bg: #e69f67; + + --color-scheme: light; +} + +@media (prefers-color-scheme: dark) { + .light.no-js { + --bg: hsl(200, 7%, 8%); + --fg: #98a3ad; + + --sidebar-bg: #292c2f; + --sidebar-fg: #a1adb8; + --sidebar-non-existant: #505254; + --sidebar-active: #3473ad; + --sidebar-spacer: #393939; + + --scrollbar: var(--sidebar-fg); + + --icons: #43484d; + --icons-hover: #b3c0cc; + + --links: #2b79a2; + + --inline-code-color: #c5c8c6; + + --theme-popup-bg: #141617; + --theme-popup-border: #43484d; + --theme-hover: #1f2124; + + --quote-bg: hsl(234, 21%, 18%); + --quote-border: hsl(234, 21%, 23%); + + --warning-border: #ff8e00; + + --table-border-color: hsl(200, 7%, 13%); + --table-header-bg: hsl(200, 7%, 28%); + --table-alternate-bg: hsl(200, 7%, 11%); + + --searchbar-border-color: #aaa; + --searchbar-bg: #b7b7b7; + --searchbar-fg: #000; + --searchbar-shadow-color: #aaa; + --searchresults-header-fg: #666; + --searchresults-border-color: #98a3ad; + --searchresults-li-bg: #2b2b2f; + --search-mark-bg: #355c7d; + } +} From f40a3ea4a64094a26890137b464a3f1fb271b17c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 16 Jun 2024 21:30:51 -0400 Subject: [PATCH 0231/2091] docs: more improvements, fixes, cleanup, etc Signed-off-by: strawberry --- docs/appservices.md | 4 ++-- docs/conduwuit_coc.md | 8 ++++---- docs/configuration.md | 27 ++++++++++++++++++++++++--- docs/deploying.md | 2 +- docs/deploying/docker.md | 13 +++---------- docs/deploying/generic.md | 2 +- docs/deploying/nixos.md | 2 +- docs/development.md | 4 ++-- docs/introduction.md | 4 ++-- docs/maintenance.md | 8 ++++---- docs/troubleshooting.md | 4 +++- docs/turn.md | 16 +--------------- nix/pkgs/book/default.nix | 1 + 13 files changed, 49 insertions(+), 46 deletions(-) diff --git a/docs/appservices.md b/docs/appservices.md index 8d646921..839001eb 100644 --- a/docs/appservices.md +++ b/docs/appservices.md @@ -18,7 +18,7 @@ First, go into the `#admins` room of your homeserver. The first person that registered on the homeserver automatically joins it. Then send a message into the room like this: - @conduit:your.server.name: register-appservice + !admin appservices register ``` paste the @@ -33,7 +33,7 @@ the room like this: You can confirm it worked by sending a message like this: `!admin appservices list` -The `@conduit` bot should answer with `Appservices (1): your-bridge` +The server bot should answer with `Appservices (1): your-bridge` Then you are done. conduwuit will send messages to the appservices and the appservice can send requests to the homeserver. You don't need to restart diff --git a/docs/conduwuit_coc.md b/docs/conduwuit_coc.md index 3a5b8777..2547e64f 100644 --- a/docs/conduwuit_coc.md +++ b/docs/conduwuit_coc.md @@ -1,6 +1,6 @@ -# Conduwuit Community Code of Conduct +# conduwuit Community Code of Conduct -Welcome to the conduwuit community! We’re excited to have you here. Conduwuit is a hard-fork of the Conduit homeserver, +Welcome to the conduwuit community! We’re excited to have you here. conduwuit is a hard-fork of the Conduit homeserver, aimed at making Matrix more accessible and inclusive for everyone. This space is dedicated to fostering a positive, supportive, and inclusive environment for everyone. This Code of @@ -39,7 +39,7 @@ additional guidelines specific to the conduwuit community. ## Matrix Community -This Code of Conduct applies to the entire [Conduwuit Matrix Space](https://matrix.to/#/#conduwuit-space:puppygock.gay) +This Code of Conduct applies to the entire [conduwuit Matrix Space](https://matrix.to/#/#conduwuit-space:puppygock.gay) and its rooms, including: ### [#conduwuit:puppygock.gay](https://matrix.to/#/#conduwuit:puppygock.gay) @@ -74,4 +74,4 @@ a DM to one of the moderators directly. Together, let’s build a community where everyone feels valued and respected. -- The Conduwuit Moderation Team +- The conduwuit Moderation Team diff --git a/docs/configuration.md b/docs/configuration.md index 6627c3b6..b0176c47 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,11 +1,32 @@ -# Example configuration +## Example configuration -``` toml +
    +Example configuration + +```toml {{#include ../conduwuit-example.toml}} ``` -# Example systemd unit file +
    + +## Debian systemd unit file + +
    +Debian systemd unit file ``` {{#include ../debian/conduwuit.service}} ``` + +
    + +## Arch Linux systemd unit file + +
    +Arch Linux systemd unit file + +``` +{{#include ../arch/conduwuit.service}} +``` + +
    diff --git a/docs/deploying.md b/docs/deploying.md index fecf7647..86277aba 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -1,3 +1,3 @@ # Deploying -This chapter describes various ways to deploy Conduwuit. +This chapter describes various ways to deploy conduwuit. diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index e6826768..e5e6a1f7 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -4,7 +4,6 @@ To run conduwuit with Docker you can either build the image yourself or pull it from a registry. - ### Use a registry OCI images for conduwuit are available in the registries listed below. @@ -24,14 +23,12 @@ OCI images for conduwuit are available in the registries listed below. [shield-latest]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/latest [shield-main]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/main - Use ```bash docker image pull ``` to pull it to your machine. - ### Run When you have the image you can simply run it with @@ -39,13 +36,9 @@ When you have the image you can simply run it with ```bash docker run -d -p 8448:6167 \ -v db:/var/lib/conduwuit/ \ - -e CONDUIT_SERVER_NAME="your.server.name" \ - -e CONDUIT_DATABASE_BACKEND="rocksdb" \ - -e CONDUIT_ALLOW_REGISTRATION=false \ - -e CONDUIT_ALLOW_FEDERATION=true \ - -e CONDUIT_MAX_REQUEST_SIZE="40000000" \ - -e CONDUIT_TRUSTED_SERVERS="[\"matrix.org\"]" \ - -e CONDUIT_LOG="warn,ruma_state_res=warn" \ + -e CONDUWUIT_SERVER_NAME="your.server.name" \ + -e CONDUWUIT_DATABASE_BACKEND="rocksdb" \ + -e CONDUWUIT_ALLOW_REGISTRATION=false \ --name conduit ``` diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index dc47f51b..31f61f6a 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -114,7 +114,7 @@ $ sudo systemctl enable conduwuit You can open [a Matrix client](https://matrix.org/ecosystem/clients), enter your homeserver and try to register. -You can also use these commands as a quick health check. +You can also use these commands as a quick health check (replace `your.server.name`). ```bash $ curl https://your.server.name/_conduwuit/server_version diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index 59f95900..d6fd3929 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -30,7 +30,7 @@ The `flake.nix` and `default.nix` do not (currently) provide a NixOS module, so (for now) [`services.matrix-conduit`][module] from Nixpkgs should be used to configure conduwuit. -If you want to run the latest code, you should get Conduwuit from the `flake.nix` +If you want to run the latest code, you should get conduwuit from the `flake.nix` or `default.nix` and set [`services.matrix-conduit.package`][package] appropriately. diff --git a/docs/development.md b/docs/development.md index 5bffa39b..fc177c8f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -7,7 +7,7 @@ it, you can safely ignore this section. If you plan on contributing, see the ## Debugging with `tokio-console` [`tokio-console`][1] can be a useful tool for debugging and profiling. To make -a `tokio-console`-enabled build of Conduwuit, enable the `tokio_console` feature, +a `tokio-console`-enabled build of conduwuit, enable the `tokio_console` feature, disable the default `release_max_log_level` feature, and set the `--cfg tokio_unstable` flag to enable experimental tokio APIs. A build might look like this: @@ -16,7 +16,7 @@ look like this: RUSTFLAGS="--cfg tokio_unstable" cargo build \ --release \ --no-default-features \ - --features=rocksdb,systemd,element_hacks,sentry_telemetry,gzip_compression,brotli_compression,zstd_compression,tokio_console + --features=rocksdb,systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console ``` [1]: https://docs.rs/tokio-console/latest/tokio_console/ diff --git a/docs/introduction.md b/docs/introduction.md index 17d7e83f..6b1c35e3 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -1,4 +1,4 @@ -# Conduwuit +# conduwuit {{#include ../README.md:catchphrase}} @@ -12,7 +12,7 @@ See the [differences](differences.md) page - [Deployment options](deploying.md) -If you want to connect an Appservice to Conduwuit, take a look at the [appservices documentation](appservices.md). +If you want to connect an appservice to conduwuit, take a look at the [appservices documentation](appservices.md). #### How can I contribute? diff --git a/docs/maintenance.md b/docs/maintenance.md index 407340bf..0e61d675 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -41,7 +41,7 @@ To restore a backup from an online RocksDB backup: - create a new directory for merging together the data - in the online backup created, copy all `.sst` files in `$DATABASE_BACKUP_PATH/shared_checksum` to your new directory - trim all the strings so instead of `######_sxxxxxxxxx.sst`, it reads `######.sst`. A way of doing this with sed and bash is `for file in *.sst; do mv "$file" "$(echo "$file" | sed 's/_s.*/.sst/')"; done` -- copy all the files in `$DATABASE_BACKUP_PATH/1` to your new directory +- copy all the files in `$DATABASE_BACKUP_PATH/1` (or the latest backup number if you have multiple) to your new directory - set your `database_path` config option to your new directory, or replace your old one with the new one you crafted - start up conduwuit again and it should open as normal @@ -52,12 +52,12 @@ Backing up media is also just copying the `media/` directory from your database ## Media Media still needs various work, however conduwuit implements media deletion via: -- MXC URI +- MXC URI or Event ID (unencrypted and attempts to find the MXC URI in the event) - Delete list of MXC URIs -- Delete remote media in the past `N` seconds/minutes +- Delete remote media in the past `N` seconds/minutes via filesystem metadata on the file created time (`btime`) or file modified time (`mtime`) See the `!admin media` command for further information. All media in conduwuit is stored at `$DATABASE_DIR/media`. This will be configurable soon. If you are finding yourself needing extensive granular control over media, we recommend looking into [Matrix Media Repo](https://github.com/t2bot/matrix-media-repo). conduwuit intends to implement various utilities for media, but MMR is dedicated to extensive media management. -Built-in S3 support is also planned, but for now using a "S3 filesystem" on `media/` works. conduwuit also sends a `Cache-Control` header of 1 year and immutable for all media requests (download and thumbnail) to reduce unnecessary media requests from browsers. +Built-in S3 support is also planned, but for now using a "S3 filesystem" on `media/` works. conduwuit also sends a `Cache-Control` header of 1 year and immutable for all media requests (download and thumbnail) to reduce unnecessary media requests from browsers, reduce bandwidth usage, and reduce load. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 59ee32c7..9f817a68 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -16,7 +16,9 @@ Some filesystems may not like RocksDB using [Direct IO](https://github.com/faceb #### Database corruption -If your database is corrupted and is failing to start (e.g. checksum mismatch), it may be recoverable but careful steps must be taken, and there is no guarantee it may be recoverable. +If your database is corrupted *and* is failing to start (e.g. checksum mismatch), it may be recoverable but careful steps must be taken, and there is no guarantee it may be recoverable. + +The first thing that can be done is launching conduwuit with the `rocksdb_repair` config option set to true. This will tell RocksDB to attempt to repair itself at launch. If this does not work, disable the option and continue reading. RocksDB has the following recovery modes: diff --git a/docs/turn.md b/docs/turn.md index 3b143d80..7fc8a45a 100644 --- a/docs/turn.md +++ b/docs/turn.md @@ -13,21 +13,7 @@ realm= ``` A common way to generate a suitable alphanumeric secret key is by using `pwgen -s 64 1`. -These same values need to be set in conduwuit. You can either modify conduwuit.toml to include these lines: - -``` -turn_uris = ["turn:?transport=udp", "turn:?transport=tcp"] -turn_secret = "" -``` - -or append the following to the docker environment variables dependig on which configuration method you used earlier: - -```yml -CONDUIT_TURN_URIS: '["turn:?transport=udp", "turn:?transport=tcp"]' -CONDUIT_TURN_SECRET: "" -``` - -Restart conduwuit to apply these changes. +These same values need to be set in conduwuit. See the [example config](configuration.md) in the TURN section for configuring these and restart conduwuit after. ### Run Run the [Coturn](https://hub.docker.com/r/coturn/coturn) image using diff --git a/nix/pkgs/book/default.nix b/nix/pkgs/book/default.nix index 4745e432..cafd162f 100644 --- a/nix/pkgs/book/default.nix +++ b/nix/pkgs/book/default.nix @@ -18,6 +18,7 @@ stdenv.mkDerivation { "README.md" "debian/conduwuit.service" "debian/README.md" + "arch/conduwuit.service" "docs" "theme" ]; From 959fd2e6c4a7218bb748f9fe9c5ce7cdbb7b8a01 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 19:41:03 +0000 Subject: [PATCH 0232/2091] split main signal handler to unit Signed-off-by: Jason Volk --- src/main/main.rs | 57 ++++------------------------------------------ src/main/signal.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 53 deletions(-) create mode 100644 src/main/signal.rs diff --git a/src/main/main.rs b/src/main/main.rs index 566b0d67..618871a7 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -1,14 +1,15 @@ pub(crate) mod clap; mod mods; mod server; +mod signal; extern crate conduit_core as conduit; use std::{cmp, sync::Arc, time::Duration}; -use conduit::{debug_error, debug_info, error, trace, utils::available_parallelism, warn, Error, Result}; +use conduit::{debug_info, error, utils::available_parallelism, Error, Result}; use server::Server; -use tokio::{runtime, signal}; +use tokio::runtime; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; @@ -26,7 +27,7 @@ fn main() -> Result<(), Error> { .expect("built runtime"); let server: Arc = Server::build(args, Some(runtime.handle()))?; - runtime.spawn(signal(server.clone())); + runtime.spawn(signal::signal(server.clone())); runtime.block_on(async_main(&server))?; // explicit drop here to trace thread and tls dtors @@ -95,53 +96,3 @@ async fn async_main(server: &Arc) -> Result<(), Error> { debug_info!("Exit runtime"); Ok(()) } - -#[cfg(unix)] -#[tracing::instrument(skip_all)] -async fn signal(server: Arc) { - use signal::unix; - use unix::SignalKind; - - const CONSOLE: bool = cfg!(feature = "console"); - const RELOADING: bool = cfg!(all(conduit_mods, not(CONSOLE))); - - let mut quit = unix::signal(SignalKind::quit()).expect("SIGQUIT handler"); - let mut term = unix::signal(SignalKind::terminate()).expect("SIGTERM handler"); - loop { - trace!("Installed signal handlers"); - let sig: &'static str; - tokio::select! { - _ = signal::ctrl_c() => { sig = "SIGINT"; }, - _ = quit.recv() => { sig = "SIGQUIT"; }, - _ = term.recv() => { sig = "SIGTERM"; }, - } - - warn!("Received {sig}"); - let result = if RELOADING && sig == "SIGINT" { - server.server.reload() - } else if matches!(sig, "SIGQUIT" | "SIGTERM") || (!CONSOLE && sig == "SIGINT") { - server.server.shutdown() - } else { - server.server.signal(sig) - }; - - if let Err(e) = result { - debug_error!(?sig, "signal: {e}"); - } - } -} - -#[cfg(not(unix))] -#[tracing::instrument(skip_all)] -async fn signal(server: Arc) { - loop { - tokio::select! { - _ = signal::ctrl_c() => { - warn!("Received Ctrl+C"); - if let Err(e) = server.server.signal.send("SIGINT") { - debug_error!("signal channel: {e}"); - } - }, - } - } -} diff --git a/src/main/signal.rs b/src/main/signal.rs new file mode 100644 index 00000000..139aab78 --- /dev/null +++ b/src/main/signal.rs @@ -0,0 +1,56 @@ +use std::sync::Arc; + +use conduit::{debug_error, trace, warn}; +use tokio::signal; + +use super::server::Server; + +#[cfg(unix)] +#[tracing::instrument(skip_all)] +pub(super) async fn signal(server: Arc) { + use signal::unix; + use unix::SignalKind; + + const CONSOLE: bool = cfg!(feature = "console"); + const RELOADING: bool = cfg!(all(conduit_mods, not(CONSOLE))); + + let mut quit = unix::signal(SignalKind::quit()).expect("SIGQUIT handler"); + let mut term = unix::signal(SignalKind::terminate()).expect("SIGTERM handler"); + loop { + trace!("Installed signal handlers"); + let sig: &'static str; + tokio::select! { + _ = signal::ctrl_c() => { sig = "SIGINT"; }, + _ = quit.recv() => { sig = "SIGQUIT"; }, + _ = term.recv() => { sig = "SIGTERM"; }, + } + + warn!("Received {sig}"); + let result = if RELOADING && sig == "SIGINT" { + server.server.reload() + } else if matches!(sig, "SIGQUIT" | "SIGTERM") || (!CONSOLE && sig == "SIGINT") { + server.server.shutdown() + } else { + server.server.signal(sig) + }; + + if let Err(e) = result { + debug_error!(?sig, "signal: {e}"); + } + } +} + +#[cfg(not(unix))] +#[tracing::instrument(skip_all)] +pub(super) async fn signal(server: Arc) { + loop { + tokio::select! { + _ = signal::ctrl_c() => { + warn!("Received Ctrl+C"); + if let Err(e) = server.server.signal.send("SIGINT") { + debug_error!("signal channel: {e}"); + } + }, + } + } +} From 5aee03d14adc6470e27be3f881ab7cf92ea255e2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 19:42:16 +0000 Subject: [PATCH 0233/2091] switch to crate rustyline_async improve console signal and interrupt stack Signed-off-by: Jason Volk --- Cargo.lock | 77 +++++------------ Cargo.toml | 5 +- src/service/Cargo.toml | 6 +- src/service/admin/console.rs | 158 ++++++++++++++++++++++++----------- src/service/admin/mod.rs | 6 +- 5 files changed, 137 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00d23a74..71540f8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -510,12 +510,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "cfg_aliases" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" - [[package]] name = "cfg_aliases" version = "0.2.1" @@ -580,15 +574,6 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" -[[package]] -name = "clipboard-win" -version = "5.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79f4473f5144e20d9aceaf2972478f06ddf687831eafeeb434fbaf0acc4144ad" -dependencies = [ - "error-code", -] - [[package]] name = "color_quant" version = "1.1.0" @@ -696,7 +681,7 @@ dependencies = [ "itertools 0.13.0", "libloading", "log", - "nix 0.29.0", + "nix", "parking_lot", "rand", "regex", @@ -796,7 +781,7 @@ dependencies = [ "reqwest", "ruma", "ruma-identifiers-validation", - "rustyline", + "rustyline-async", "serde", "serde_json", "serde_yaml", @@ -990,6 +975,7 @@ checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ "bitflags 2.5.0", "crossterm_winapi", + "futures-core", "libc", "mio", "parking_lot", @@ -1166,12 +1152,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "error-code" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0474425d51df81997e2f90a21591180b38eccf27292d755f3e30750225c175b" - [[package]] name = "fallible-iterator" version = "0.3.0" @@ -2341,18 +2321,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" -[[package]] -name = "nix" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" -dependencies = [ - "bitflags 2.5.0", - "cfg-if", - "cfg_aliases 0.1.1", - "libc", -] - [[package]] name = "nix" version = "0.29.0" @@ -2361,7 +2329,7 @@ checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ "bitflags 2.5.0", "cfg-if", - "cfg_aliases 0.2.1", + "cfg_aliases", "libc", ] @@ -3420,22 +3388,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] -name = "rustyline" -version = "14.0.0" +name = "rustyline-async" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" +checksum = "8b6eb06391513b2184f0a5405c11a4a0a5302e8be442f4c5c35267187c2b37d5" dependencies = [ - "bitflags 2.5.0", - "cfg-if", - "clipboard-win", - "libc", - "log", - "memchr", - "nix 0.28.0", + "crossterm", + "futures-channel", + "futures-util", + "pin-project", + "thingbuf", + "thiserror", "unicode-segmentation", "unicode-width", - "utf8parse", - "windows-sys 0.52.0", ] [[package]] @@ -4024,6 +3989,16 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thingbuf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "662b54ef6f7b4e71f683dadc787bbb2d8e8ef2f91b682ebed3164a5a7abca905" +dependencies = [ + "parking_lot", + "pin-project", +] + [[package]] name = "thiserror" version = "1.0.61" @@ -4635,12 +4610,6 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - [[package]] name = "uuid" version = "1.8.0" diff --git a/Cargo.toml b/Cargo.toml index 0c6c6e25..70599fcf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -415,9 +415,8 @@ features = [ "light", ] -[workspace.dependencies.rustyline] -version = "14.0.0" -default-features = false +[workspace.dependencies.rustyline-async] +version = "0.4.2" [workspace.dependencies.termimad] version = "0.29.3" diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 8a254ef8..80335181 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -21,7 +21,7 @@ brotli_compression = [ "reqwest/brotli", ] console = [ - "dep:rustyline", + "dep:rustyline-async", "dep:termimad", ] dev_release_log_level = [] @@ -61,8 +61,8 @@ regex.workspace = true reqwest.workspace = true ruma-identifiers-validation.workspace = true ruma.workspace = true -rustyline.workspace = true -rustyline.optional = true +rustyline-async.workspace = true +rustyline-async.optional = true serde_json.workspace = true serde.workspace = true serde_yaml.workspace = true diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index ba0d7ecc..dc967d00 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -1,44 +1,35 @@ #![cfg(feature = "console")] -use std::sync::Arc; +use std::{ + collections::VecDeque, + sync::{Arc, Mutex}, +}; -use conduit::{debug, defer, error, log, trace}; +use conduit::{debug, defer, error, log}; use futures_util::future::{AbortHandle, Abortable}; use ruma::events::room::message::RoomMessageEventContent; -use rustyline::{error::ReadlineError, history, Editor}; +use rustyline_async::{Readline, ReadlineError, ReadlineEvent}; use termimad::MadSkin; -use tokio::{sync::Mutex, task::JoinHandle}; +use tokio::task::JoinHandle; use crate::services; pub struct Console { - join: Mutex>>, - input: Mutex>, - abort: std::sync::Mutex>, + worker_join: Mutex>>, + input_abort: Mutex>, + command_abort: Mutex>, + history: Mutex>, output: MadSkin, } +const PROMPT: &str = "uwu> "; +const HISTORY_LIMIT: usize = 48; + impl Console { #[must_use] pub fn new() -> Arc { - use rustyline::config::{Behavior, BellStyle}; use termimad::{crossterm::style::Color, Alignment, CompoundStyle, LineStyle}; - let config = rustyline::Config::builder() - .enable_signals(false) - .behavior(Behavior::PreferTerm) - .bell_style(BellStyle::Visible) - .auto_add_history(true) - .max_history_size(100) - .expect("valid history size") - .indent_size(4) - .tab_stop(4) - .build(); - - let history = history::MemHistory::with_config(config); - let input = Editor::with_history(config, history).expect("builder configuration succeeded"); - let mut output = MadSkin::default_dark(); - let code_style = CompoundStyle::with_fgbg(Color::AnsiValue(40), Color::AnsiValue(234)); output.inline_code = code_style.clone(); output.code_block = LineStyle { @@ -49,33 +40,63 @@ impl Console { }; Arc::new(Self { - join: None.into(), - input: Mutex::new(input), - abort: None.into(), + worker_join: None.into(), + input_abort: None.into(), + command_abort: None.into(), + history: VecDeque::with_capacity(HISTORY_LIMIT).into(), output, }) } + pub(super) async fn handle_signal(self: &Arc, sig: &'static str) { + if !services().server.running() { + self.interrupt(); + } else if sig == "SIGINT" { + self.interrupt_command(); + self.start().await; + } + } + #[allow(clippy::let_underscore_must_use)] pub async fn start(self: &Arc) { - let mut join = self.join.lock().await; - if join.is_none() { + let mut worker_join = self.worker_join.lock().expect("locked"); + if worker_join.is_none() { let self_ = Arc::clone(self); - _ = join.insert(services().server.runtime().spawn(self_.worker())); + _ = worker_join.insert(services().server.runtime().spawn(self_.worker())); } } #[allow(clippy::let_underscore_must_use)] pub async fn close(self: &Arc) { - if let Some(join) = self.join.lock().await.take() { - _ = join.await; - } + self.interrupt(); + let Some(worker_join) = self.worker_join.lock().expect("locked").take() else { + return; + }; + + _ = worker_join.await; } pub fn interrupt(self: &Arc) { - if let Some(abort) = self.abort.lock().expect("locked").take() { + self.interrupt_command(); + self.interrupt_readline(); + self.worker_join + .lock() + .expect("locked") + .as_ref() + .map(JoinHandle::abort); + } + + pub fn interrupt_readline(self: &Arc) { + if let Some(input_abort) = self.input_abort.lock().expect("locked").take() { + debug!("Interrupting console readline..."); + input_abort.abort(); + } + } + + pub fn interrupt_command(self: &Arc) { + if let Some(command_abort) = self.command_abort.lock().expect("locked").take() { debug!("Interrupting console command..."); - abort.abort(); + command_abort.abort(); } } @@ -83,25 +104,48 @@ impl Console { async fn worker(self: Arc) { debug!("session starting"); while services().server.running() { - let mut input = self.input.lock().await; - - let suppression = log::Suppress::new(&services().server); - let line = tokio::task::block_in_place(|| input.readline("uwu> ")); - drop(suppression); - - trace!(?line, "input"); - match line { - Ok(string) => self.clone().handle(string).await, - Err(e) => match e { - ReadlineError::Interrupted | ReadlineError::Eof => break, - ReadlineError::WindowResized => continue, - _ => error!("console: {e:?}"), + match self.readline().await { + Ok(event) => match event { + ReadlineEvent::Line(string) => self.clone().handle(string).await, + ReadlineEvent::Interrupted => continue, + ReadlineEvent::Eof => break, + }, + Err(error) => match error { + ReadlineError::Closed => break, + ReadlineError::IO(error) => { + error!("console I/O: {error:?}"); + break; + }, }, } } debug!("session ending"); - self.join.lock().await.take(); + self.worker_join.lock().expect("locked").take(); + } + + #[allow(clippy::let_underscore_must_use)] + async fn readline(self: &Arc) -> Result { + let _suppression = log::Suppress::new(&services().server); + + let (mut readline, _writer) = Readline::new(PROMPT.to_owned())?; + self.set_history(&mut readline); + + let future = readline.readline(); + + let (abort, abort_reg) = AbortHandle::new_pair(); + let future = Abortable::new(future, abort_reg); + _ = self.input_abort.lock().expect("locked").insert(abort); + defer! {{ + _ = self.input_abort.lock().expect("locked").take(); + }} + + let Ok(result) = future.await else { + return Ok(ReadlineEvent::Eof); + }; + + readline.flush()?; + result } #[allow(clippy::let_underscore_must_use)] @@ -110,12 +154,13 @@ impl Console { return; } + self.add_history(line.clone()); let future = self.clone().process(line); let (abort, abort_reg) = AbortHandle::new_pair(); let future = Abortable::new(future, abort_reg); - _ = self.abort.lock().expect("locked").insert(abort); + _ = self.command_abort.lock().expect("locked").insert(abort); defer! {{ - _ = self.abort.lock().expect("locked").take(); + _ = self.command_abort.lock().expect("locked").take(); }} _ = future.await; @@ -133,4 +178,17 @@ impl Console { let output = self.output.term_text(output_content.body()); println!("{output}"); } + + fn set_history(&self, readline: &mut Readline) { + let history = self.history.lock().expect("locked"); + for entry in history.iter().rev() { + readline.add_history_entry(entry.clone()); + } + } + + fn add_history(&self, line: String) { + let mut history = self.history.lock().expect("locked"); + history.push_front(line); + history.truncate(HISTORY_LIMIT); + } } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 3f983b75..730163ac 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -130,7 +130,6 @@ impl Service { let receiver = self.receiver.lock().await; let mut signals = services().server.signal.subscribe(); loop { - debug_assert!(!receiver.is_closed(), "channel closed"); tokio::select! { command = receiver.recv_async() => match command { Ok(command) => self.handle_command(command).await, @@ -146,10 +145,7 @@ impl Service { async fn handle_signal(&self, #[allow(unused_variables)] sig: &'static str) { #[cfg(feature = "console")] - if sig == "SIGINT" && services().server.running() { - self.console.interrupt(); - self.console.start().await; - } + self.console.handle_signal(sig).await; } async fn handle_command(&self, command: Command) { From 2cb31275f0836018789f0cb39f2542fff4847d0a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 19:44:46 +0000 Subject: [PATCH 0234/2091] fix missing command response; use non-empty command responses Signed-off-by: Jason Volk --- src/admin/server/server_commands.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/admin/server/server_commands.rs b/src/admin/server/server_commands.rs index 79e02e17..cf69d79e 100644 --- a/src/admin/server/server_commands.rs +++ b/src/admin/server/server_commands.rs @@ -18,7 +18,7 @@ pub(crate) async fn uptime(_body: Vec<&str>) -> Result seconds % 60, ); - Ok(RoomMessageEventContent::notice_html(String::new(), result)) + Ok(RoomMessageEventContent::notice_plain(result)) } pub(crate) async fn show_config(_body: Vec<&str>) -> Result { @@ -109,12 +109,12 @@ pub(crate) async fn admin_notice(_body: Vec<&str>, message: Vec) -> Resu pub(crate) async fn reload(_body: Vec<&str>) -> Result { services().server.reload()?; - Ok(RoomMessageEventContent::notice_plain(String::new())) + Ok(RoomMessageEventContent::notice_plain("Reloading server...")) } pub(crate) async fn shutdown(_body: Vec<&str>) -> Result { warn!("shutdown command"); services().server.shutdown()?; - Ok(RoomMessageEventContent::notice_plain(String::new())) + Ok(RoomMessageEventContent::notice_plain("Shutting down server...")) } From c6f4b20e17a2a4ca9824ddf1495197e826426b91 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 19:46:32 +0000 Subject: [PATCH 0235/2091] add server restart support w/ admin command Signed-off-by: Jason Volk --- src/admin/server/mod.rs | 6 ++++++ src/admin/server/server_commands.rs | 7 +++++++ src/core/server.rs | 12 ++++++++++++ src/main/main.rs | 14 ++++++++++++-- src/main/restart.rs | 17 +++++++++++++++++ 5 files changed, 54 insertions(+), 2 deletions(-) create mode 100644 src/main/restart.rs diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 584d3234..c3a517c0 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -49,6 +49,10 @@ pub(crate) enum ServerCommand { /// - Hot-reload the server Reload, + #[cfg(unix)] + /// - Restart the server + Restart, + /// - Shutdown the server Shutdown, } @@ -72,6 +76,8 @@ pub(crate) async fn process(command: ServerCommand, body: Vec<&str>) -> Result admin_notice(body, message).await?, #[cfg(conduit_mods)] ServerCommand::Reload => reload(body).await?, + #[cfg(unix)] + ServerCommand::Restart => restart(body).await?, ServerCommand::Shutdown => shutdown(body).await?, }) } diff --git a/src/admin/server/server_commands.rs b/src/admin/server/server_commands.rs index cf69d79e..0e01d4bf 100644 --- a/src/admin/server/server_commands.rs +++ b/src/admin/server/server_commands.rs @@ -112,6 +112,13 @@ pub(crate) async fn reload(_body: Vec<&str>) -> Result Ok(RoomMessageEventContent::notice_plain("Reloading server...")) } +#[cfg(unix)] +pub(crate) async fn restart(_body: Vec<&str>) -> Result { + services().server.restart()?; + + Ok(RoomMessageEventContent::notice_plain("Restarting server...")) +} + pub(crate) async fn shutdown(_body: Vec<&str>) -> Result { warn!("shutdown command"); services().server.shutdown()?; diff --git a/src/core/server.rs b/src/core/server.rs index ca62b64c..2aa821fa 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -23,6 +23,9 @@ pub struct Server { /// is an observable used on shutdown and modifying is not recommended. pub reloading: AtomicBool, + /// Restart desired; when true, restart it desired after shutdown. + pub restarting: AtomicBool, + /// Handle to the runtime pub runtime: Option, @@ -48,6 +51,7 @@ impl Server { started: SystemTime::now(), stopping: AtomicBool::new(false), reloading: AtomicBool::new(false), + restarting: AtomicBool::new(false), runtime, signal: broadcast::channel::<&'static str>(1).0, log, @@ -75,6 +79,14 @@ impl Server { self.signal("SIGINT") } + pub fn restart(&self) -> Result<()> { + if self.restarting.swap(true, Ordering::AcqRel) { + return Err(Error::Err("Restart already in progress".into())); + } + + self.shutdown() + } + pub fn shutdown(&self) -> Result<()> { if self.stopping.swap(true, Ordering::AcqRel) { return Err(Error::Err("Shutdown already in progress".into())); diff --git a/src/main/main.rs b/src/main/main.rs index 618871a7..1bb46c8e 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -1,11 +1,16 @@ pub(crate) mod clap; mod mods; +mod restart; mod server; mod signal; extern crate conduit_core as conduit; -use std::{cmp, sync::Arc, time::Duration}; +use std::{ + cmp, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; use conduit::{debug_info, error, utils::available_parallelism, Error, Result}; use server::Server; @@ -32,8 +37,13 @@ fn main() -> Result<(), Error> { // explicit drop here to trace thread and tls dtors drop(runtime); - debug_info!("Exit"); + #[cfg(unix)] + if server.server.restarting.load(Ordering::Acquire) { + restart::restart(); + } + + debug_info!("Exit"); Ok(()) } diff --git a/src/main/restart.rs b/src/main/restart.rs new file mode 100644 index 00000000..559d7446 --- /dev/null +++ b/src/main/restart.rs @@ -0,0 +1,17 @@ +#![cfg(unix)] + +use std::{env, os::unix::process::CommandExt, process::Command}; + +use conduit::{debug, info}; + +pub(super) fn restart() -> ! { + let exe = env::current_exe().expect("program path must be identified and available"); + let envs = env::vars(); + let args = env::args().skip(1); + debug!(?exe, ?args, ?envs, "Restart"); + + info!("Restart"); + + let error = Command::new(exe).args(args).envs(envs).exec(); + panic!("{error:?}"); +} From 50ce87161b61f2d539fd178f67f186b3935ce7ec Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 22:26:52 +0000 Subject: [PATCH 0236/2091] refactor admin command visibilities and use statements Signed-off-by: Jason Volk --- .../{appservice_command.rs => commands.rs} | 8 ++-- src/admin/appservice/mod.rs | 10 ++--- .../debug/{debug_commands.rs => commands.rs} | 37 +++++++++--------- src/admin/debug/mod.rs | 16 +++----- .../{federation_commands.rs => commands.rs} | 10 ++--- src/admin/federation/mod.rs | 14 +++---- .../fsck/{fsck_commands.rs => commands.rs} | 5 ++- src/admin/fsck/mod.rs | 12 +++--- .../media/{media_commands.rs => commands.rs} | 9 +++-- src/admin/media/mod.rs | 12 +++--- src/admin/query/account_data.rs | 2 +- src/admin/query/appservice.rs | 2 +- src/admin/query/globals.rs | 2 +- src/admin/query/mod.rs | 38 +++++++++---------- src/admin/query/presence.rs | 2 +- src/admin/query/room_alias.rs | 2 +- src/admin/query/room_state_cache.rs | 2 +- src/admin/query/sending.rs | 2 +- src/admin/query/users.rs | 2 +- src/admin/room/mod.rs | 26 ++++++------- src/admin/room/room_alias_commands.rs | 2 +- src/admin/room/room_commands.rs | 2 +- src/admin/room/room_directory_commands.rs | 2 +- src/admin/room/room_info_commands.rs | 2 +- src/admin/room/room_moderation_commands.rs | 2 +- .../{server_commands.rs => commands.rs} | 28 +++++++------- src/admin/server/mod.rs | 10 ++--- src/admin/tester/mod.rs | 4 +- .../user/{user_commands.rs => commands.rs} | 21 +++++----- src/admin/user/mod.rs | 11 +++--- 30 files changed, 145 insertions(+), 152 deletions(-) rename src/admin/appservice/{appservice_command.rs => commands.rs} (89%) rename src/admin/debug/{debug_commands.rs => commands.rs} (95%) rename src/admin/federation/{federation_commands.rs => commands.rs} (92%) rename src/admin/fsck/{fsck_commands.rs => commands.rs} (89%) rename src/admin/media/{media_commands.rs => commands.rs} (97%) rename src/admin/server/{server_commands.rs => commands.rs} (77%) rename src/admin/user/{user_commands.rs => commands.rs} (96%) diff --git a/src/admin/appservice/appservice_command.rs b/src/admin/appservice/commands.rs similarity index 89% rename from src/admin/appservice/appservice_command.rs rename to src/admin/appservice/commands.rs index 6f36ebf7..5b764296 100644 --- a/src/admin/appservice/appservice_command.rs +++ b/src/admin/appservice/commands.rs @@ -2,7 +2,7 @@ use ruma::{api::appservice::Registration, events::room::message::RoomMessageEven use crate::{escape_html, services, Result}; -pub(crate) async fn register(body: Vec<&str>) -> Result { +pub(super) async fn register(body: Vec<&str>) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -26,7 +26,7 @@ pub(crate) async fn register(body: Vec<&str>) -> Result } } -pub(crate) async fn unregister(_body: Vec<&str>, appservice_identifier: String) -> Result { +pub(super) async fn unregister(_body: Vec<&str>, appservice_identifier: String) -> Result { match services() .appservice .unregister_appservice(&appservice_identifier) @@ -39,7 +39,7 @@ pub(crate) async fn unregister(_body: Vec<&str>, appservice_identifier: String) } } -pub(crate) async fn show(_body: Vec<&str>, appservice_identifier: String) -> Result { +pub(super) async fn show(_body: Vec<&str>, appservice_identifier: String) -> Result { match services() .appservice .get_registration(&appservice_identifier) @@ -59,7 +59,7 @@ pub(crate) async fn show(_body: Vec<&str>, appservice_identifier: String) -> Res } } -pub(crate) async fn list(_body: Vec<&str>) -> Result { +pub(super) async fn list(_body: Vec<&str>) -> Result { let appservices = services().appservice.iter_ids().await; let output = format!("Appservices ({}): {}", appservices.len(), appservices.join(", ")); Ok(RoomMessageEventContent::text_plain(output)) diff --git a/src/admin/appservice/mod.rs b/src/admin/appservice/mod.rs index 8cf246b9..87ab1b6d 100644 --- a/src/admin/appservice/mod.rs +++ b/src/admin/appservice/mod.rs @@ -1,14 +1,14 @@ +mod commands; + use clap::Subcommand; use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; -use self::appservice_command::{list, register, show, unregister}; - -pub(crate) mod appservice_command; +use self::commands::*; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum AppserviceCommand { +pub(super) enum AppserviceCommand { /// - Register an appservice using its registration YAML /// /// This command needs a YAML generated by an appservice (such as a bridge), @@ -38,7 +38,7 @@ pub(crate) enum AppserviceCommand { List, } -pub(crate) async fn process(command: AppserviceCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: AppserviceCommand, body: Vec<&str>) -> Result { Ok(match command { AppserviceCommand::Register => register(body).await?, AppserviceCommand::Unregister { diff --git a/src/admin/debug/debug_commands.rs b/src/admin/debug/commands.rs similarity index 95% rename from src/admin/debug/debug_commands.rs rename to src/admin/debug/commands.rs index 681dc6fb..0409af32 100644 --- a/src/admin/debug/debug_commands.rs +++ b/src/admin/debug/commands.rs @@ -4,6 +4,7 @@ use std::{ time::Instant, }; +use api::client::validate_and_add_event_id; use conduit::{ debug, info, log, log::{capture, Capture}, @@ -19,15 +20,13 @@ use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve::resolv use tokio::sync::RwLock; use tracing_subscriber::EnvFilter; -use crate::api::client::validate_and_add_event_id; - -pub(crate) async fn echo(_body: Vec<&str>, message: Vec) -> Result { +pub(super) async fn echo(_body: Vec<&str>, message: Vec) -> Result { let message = message.join(" "); Ok(RoomMessageEventContent::notice_plain(message)) } -pub(crate) async fn get_auth_chain(_body: Vec<&str>, event_id: Box) -> Result { +pub(super) async fn get_auth_chain(_body: Vec<&str>, event_id: Box) -> Result { let event_id = Arc::::from(event_id); if let Some(event) = services().rooms.timeline.get_pdu_json(&event_id)? { let room_id_str = event @@ -53,7 +52,7 @@ pub(crate) async fn get_auth_chain(_body: Vec<&str>, event_id: Box) -> } } -pub(crate) async fn parse_pdu(body: Vec<&str>) -> Result { +pub(super) async fn parse_pdu(body: Vec<&str>) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -81,7 +80,7 @@ pub(crate) async fn parse_pdu(body: Vec<&str>) -> Result, event_id: Box) -> Result { +pub(super) async fn get_pdu(_body: Vec<&str>, event_id: Box) -> Result { let mut outlier = false; let mut pdu_json = services() .rooms @@ -119,7 +118,7 @@ pub(crate) async fn get_pdu(_body: Vec<&str>, event_id: Box) -> Result< } } -pub(crate) async fn get_remote_pdu_list( +pub(super) async fn get_remote_pdu_list( body: Vec<&str>, server: Box, force: bool, ) -> Result { if !services().globals.config.allow_federation { @@ -166,7 +165,7 @@ pub(crate) async fn get_remote_pdu_list( Ok(RoomMessageEventContent::text_plain("Fetched list of remote PDUs.")) } -pub(crate) async fn get_remote_pdu( +pub(super) async fn get_remote_pdu( _body: Vec<&str>, event_id: Box, server: Box, ) -> Result { if !services().globals.config.allow_federation { @@ -256,7 +255,7 @@ pub(crate) async fn get_remote_pdu( } } -pub(crate) async fn get_room_state(_body: Vec<&str>, room_id: Box) -> Result { +pub(super) async fn get_room_state(_body: Vec<&str>, room_id: Box) -> Result { let room_state = services() .rooms .state_accessor @@ -289,7 +288,7 @@ pub(crate) async fn get_room_state(_body: Vec<&str>, room_id: Box) -> Re )) } -pub(crate) async fn ping(_body: Vec<&str>, server: Box) -> Result { +pub(super) async fn ping(_body: Vec<&str>, server: Box) -> Result { if server == services().globals.server_name() { return Ok(RoomMessageEventContent::text_plain( "Not allowed to send federation requests to ourselves.", @@ -332,7 +331,7 @@ pub(crate) async fn ping(_body: Vec<&str>, server: Box) -> Result) -> Result { +pub(super) async fn force_device_list_updates(_body: Vec<&str>) -> Result { // Force E2EE device list updates for all users for user_id in services().users.iter().filter_map(Result::ok) { services().users.mark_device_key_update(&user_id)?; @@ -342,7 +341,7 @@ pub(crate) async fn force_device_list_updates(_body: Vec<&str>) -> Result, filter: Option, reset: bool, ) -> Result { if reset { @@ -395,7 +394,7 @@ pub(crate) async fn change_log_level( Ok(RoomMessageEventContent::text_plain("No log level was specified.")) } -pub(crate) async fn sign_json(body: Vec<&str>) -> Result { +pub(super) async fn sign_json(body: Vec<&str>) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -418,7 +417,7 @@ pub(crate) async fn sign_json(body: Vec<&str>) -> Result) -> Result { +pub(super) async fn verify_json(body: Vec<&str>) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -449,7 +448,7 @@ pub(crate) async fn verify_json(body: Vec<&str>) -> Result, room_id: Box) -> Result { +pub(super) async fn first_pdu_in_room(_body: Vec<&str>, room_id: Box) -> Result { if !services() .rooms .state_cache @@ -470,7 +469,7 @@ pub(crate) async fn first_pdu_in_room(_body: Vec<&str>, room_id: Box) -> } #[tracing::instrument(skip(_body))] -pub(crate) async fn latest_pdu_in_room(_body: Vec<&str>, room_id: Box) -> Result { +pub(super) async fn latest_pdu_in_room(_body: Vec<&str>, room_id: Box) -> Result { if !services() .rooms .state_cache @@ -491,7 +490,7 @@ pub(crate) async fn latest_pdu_in_room(_body: Vec<&str>, room_id: Box) - } #[tracing::instrument(skip(_body))] -pub(crate) async fn force_set_room_state_from_server( +pub(super) async fn force_set_room_state_from_server( _body: Vec<&str>, server_name: Box, room_id: Box, ) -> Result { if !services() @@ -621,7 +620,7 @@ pub(crate) async fn force_set_room_state_from_server( )) } -pub(crate) async fn resolve_true_destination( +pub(super) async fn resolve_true_destination( _body: Vec<&str>, server_name: Box, no_cache: bool, ) -> Result { if !services().globals.config.allow_federation { @@ -659,7 +658,7 @@ pub(crate) async fn resolve_true_destination( } #[must_use] -pub(crate) fn memory_stats() -> RoomMessageEventContent { +pub(super) fn memory_stats() -> RoomMessageEventContent { let html_body = conduit::alloc::memory_stats(); if html_body.is_empty() { diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 52c49c20..19d3712a 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -1,18 +1,14 @@ +mod commands; + use clap::Subcommand; -use debug_commands::{first_pdu_in_room, force_set_room_state_from_server, latest_pdu_in_room}; +use conduit::Result; use ruma::{events::room::message::RoomMessageEventContent, EventId, RoomId, ServerName}; -use self::debug_commands::{ - change_log_level, echo, force_device_list_updates, get_auth_chain, get_pdu, get_remote_pdu, get_remote_pdu_list, - get_room_state, memory_stats, parse_pdu, ping, resolve_true_destination, sign_json, verify_json, -}; -use crate::Result; - -pub(crate) mod debug_commands; +use self::commands::*; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum DebugCommand { +pub(super) enum DebugCommand { /// - Echo input of admin command Echo { message: Vec, @@ -163,7 +159,7 @@ pub(crate) enum DebugCommand { MemoryStats, } -pub(crate) async fn process(command: DebugCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: DebugCommand, body: Vec<&str>) -> Result { Ok(match command { DebugCommand::Echo { message, diff --git a/src/admin/federation/federation_commands.rs b/src/admin/federation/commands.rs similarity index 92% rename from src/admin/federation/federation_commands.rs rename to src/admin/federation/commands.rs index b52a62f2..4084350d 100644 --- a/src/admin/federation/federation_commands.rs +++ b/src/admin/federation/commands.rs @@ -4,17 +4,17 @@ use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId, use crate::{escape_html, get_room_info, services, utils::HtmlEscape, Result}; -pub(crate) async fn disable_room(_body: Vec<&str>, room_id: Box) -> Result { +pub(super) async fn disable_room(_body: Vec<&str>, room_id: Box) -> Result { services().rooms.metadata.disable_room(&room_id, true)?; Ok(RoomMessageEventContent::text_plain("Room disabled.")) } -pub(crate) async fn enable_room(_body: Vec<&str>, room_id: Box) -> Result { +pub(super) async fn enable_room(_body: Vec<&str>, room_id: Box) -> Result { services().rooms.metadata.disable_room(&room_id, false)?; Ok(RoomMessageEventContent::text_plain("Room enabled.")) } -pub(crate) async fn incoming_federation(_body: Vec<&str>) -> Result { +pub(super) async fn incoming_federation(_body: Vec<&str>) -> Result { let map = services().globals.roomid_federationhandletime.read().await; let mut msg = format!("Handling {} incoming pdus:\n", map.len()); @@ -26,7 +26,7 @@ pub(crate) async fn incoming_federation(_body: Vec<&str>) -> Result, server_name: Box, ) -> Result { let response = services() @@ -72,7 +72,7 @@ pub(crate) async fn fetch_support_well_known( )) } -pub(crate) async fn remote_user_in_rooms(_body: Vec<&str>, user_id: Box) -> Result { +pub(super) async fn remote_user_in_rooms(_body: Vec<&str>, user_id: Box) -> Result { if user_id.server_name() == services().globals.config.server_name { return Ok(RoomMessageEventContent::text_plain( "User belongs to our server, please use `list-joined-rooms` user admin command instead.", diff --git a/src/admin/federation/mod.rs b/src/admin/federation/mod.rs index 81469feb..46e323b2 100644 --- a/src/admin/federation/mod.rs +++ b/src/admin/federation/mod.rs @@ -1,16 +1,14 @@ +mod commands; + use clap::Subcommand; +use conduit::Result; use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId}; -use self::federation_commands::{ - disable_room, enable_room, fetch_support_well_known, incoming_federation, remote_user_in_rooms, -}; -use crate::Result; - -pub(crate) mod federation_commands; +use self::commands::*; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum FederationCommand { +pub(super) enum FederationCommand { /// - List all rooms we are currently handling an incoming pdu from IncomingFederation, @@ -43,7 +41,7 @@ pub(crate) enum FederationCommand { }, } -pub(crate) async fn process(command: FederationCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: FederationCommand, body: Vec<&str>) -> Result { Ok(match command { FederationCommand::DisableRoom { room_id, diff --git a/src/admin/fsck/fsck_commands.rs b/src/admin/fsck/commands.rs similarity index 89% rename from src/admin/fsck/fsck_commands.rs rename to src/admin/fsck/commands.rs index 2eab94de..9fb0281a 100644 --- a/src/admin/fsck/fsck_commands.rs +++ b/src/admin/fsck/commands.rs @@ -1,11 +1,12 @@ +use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; -use crate::{services, Result}; +use crate::services; /// Uses the iterator in `src/database/key_value/users.rs` to iterator over /// every user in our database (remote and local). Reports total count, any /// errors if there were any, etc -pub(crate) async fn check_all_users(_body: Vec<&str>) -> Result { +pub(super) async fn check_all_users(_body: Vec<&str>) -> Result { let timer = tokio::time::Instant::now(); let results = services().users.db.iter(); let query_time = timer.elapsed(); diff --git a/src/admin/fsck/mod.rs b/src/admin/fsck/mod.rs index e618fdc5..4252718d 100644 --- a/src/admin/fsck/mod.rs +++ b/src/admin/fsck/mod.rs @@ -1,18 +1,18 @@ +mod commands; + use clap::Subcommand; +use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; -use self::fsck_commands::check_all_users; -use crate::Result; - -pub(crate) mod fsck_commands; +use self::commands::*; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum FsckCommand { +pub(super) enum FsckCommand { CheckAllUsers, } -pub(crate) async fn process(command: FsckCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: FsckCommand, body: Vec<&str>) -> Result { Ok(match command { FsckCommand::CheckAllUsers => check_all_users(body).await?, }) diff --git a/src/admin/media/media_commands.rs b/src/admin/media/commands.rs similarity index 97% rename from src/admin/media/media_commands.rs rename to src/admin/media/commands.rs index a78ee387..c1026ae2 100644 --- a/src/admin/media/media_commands.rs +++ b/src/admin/media/commands.rs @@ -1,9 +1,10 @@ +use conduit::Result; use ruma::{events::room::message::RoomMessageEventContent, EventId, MxcUri}; use tracing::{debug, info}; -use crate::{services, Result}; +use crate::services; -pub(crate) async fn delete( +pub(super) async fn delete( _body: Vec<&str>, mxc: Option>, event_id: Option>, ) -> Result { if event_id.is_some() && mxc.is_some() { @@ -137,7 +138,7 @@ pub(crate) async fn delete( )) } -pub(crate) async fn delete_list(body: Vec<&str>) -> Result { +pub(super) async fn delete_list(body: Vec<&str>) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -164,7 +165,7 @@ pub(crate) async fn delete_list(body: Vec<&str>) -> Result, duration: String, force: bool, ) -> Result { let deleted_count = services() diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index b5957370..60f7a9bb 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -1,14 +1,14 @@ +mod commands; + use clap::Subcommand; +use conduit::Result; use ruma::{events::room::message::RoomMessageEventContent, EventId, MxcUri}; -use self::media_commands::{delete, delete_list, delete_past_remote_media}; -use crate::Result; - -pub(crate) mod media_commands; +use self::commands::*; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum MediaCommand { +pub(super) enum MediaCommand { /// - Deletes a single media file from our database and on the filesystem /// via a single MXC URL Delete { @@ -38,7 +38,7 @@ pub(crate) enum MediaCommand { }, } -pub(crate) async fn process(command: MediaCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: MediaCommand, body: Vec<&str>) -> Result { Ok(match command { MediaCommand::Delete { mxc, diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index eb7cd0d1..290d5a04 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -4,7 +4,7 @@ use super::AccountData; use crate::{services, Result}; /// All the getters and iterators from src/database/key_value/account_data.rs -pub(crate) async fn account_data(subcommand: AccountData) -> Result { +pub(super) async fn account_data(subcommand: AccountData) -> Result { match subcommand { AccountData::ChangesSince { user_id, diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index d108ca8d..a94f22b8 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -4,7 +4,7 @@ use super::Appservice; use crate::{services, Result}; /// All the getters and iterators from src/database/key_value/appservice.rs -pub(crate) async fn appservice(subcommand: Appservice) -> Result { +pub(super) async fn appservice(subcommand: Appservice) -> Result { match subcommand { Appservice::GetRegistration { appservice_id, diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index e5249118..bc8bcdef 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -4,7 +4,7 @@ use super::Globals; use crate::{services, Result}; /// All the getters and iterators from src/database/key_value/globals.rs -pub(crate) async fn globals(subcommand: Globals) -> Result { +pub(super) async fn globals(subcommand: Globals) -> Result { match subcommand { Globals::DatabaseVersion => { let timer = tokio::time::Instant::now(); diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index 2e5b0bb8..946e6ec8 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -1,13 +1,14 @@ -pub(crate) mod account_data; -pub(crate) mod appservice; -pub(crate) mod globals; -pub(crate) mod presence; -pub(crate) mod room_alias; -pub(crate) mod room_state_cache; -pub(crate) mod sending; -pub(crate) mod users; +mod account_data; +mod appservice; +mod globals; +mod presence; +mod room_alias; +mod room_state_cache; +mod sending; +mod users; use clap::Subcommand; +use conduit::Result; use room_state_cache::room_state_cache; use ruma::{ events::{room::message::RoomMessageEventContent, RoomAccountDataEventType}, @@ -18,12 +19,11 @@ use self::{ account_data::account_data, appservice::appservice, globals::globals, presence::presence, room_alias::room_alias, sending::sending, users::users, }; -use crate::Result; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] /// Query tables from database -pub(crate) enum QueryCommand { +pub(super) enum QueryCommand { /// - account_data.rs iterators and getters #[command(subcommand)] AccountData(AccountData), @@ -60,7 +60,7 @@ pub(crate) enum QueryCommand { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] /// All the getters and iterators from src/database/key_value/account_data.rs -pub(crate) enum AccountData { +pub(super) enum AccountData { /// - Returns all changes to the account data that happened after `since`. ChangesSince { /// Full user ID @@ -85,7 +85,7 @@ pub(crate) enum AccountData { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] /// All the getters and iterators from src/database/key_value/appservice.rs -pub(crate) enum Appservice { +pub(super) enum Appservice { /// - Gets the appservice registration info/details from the ID as a string GetRegistration { /// Appservice registration ID @@ -99,7 +99,7 @@ pub(crate) enum Appservice { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] /// All the getters and iterators from src/database/key_value/presence.rs -pub(crate) enum Presence { +pub(super) enum Presence { /// - Returns the latest presence event for the given user. GetPresence { /// Full user ID @@ -117,7 +117,7 @@ pub(crate) enum Presence { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] /// All the getters and iterators from src/database/key_value/rooms/alias.rs -pub(crate) enum RoomAlias { +pub(super) enum RoomAlias { ResolveLocalAlias { /// Full room alias alias: Box, @@ -135,7 +135,7 @@ pub(crate) enum RoomAlias { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum RoomStateCache { +pub(super) enum RoomStateCache { ServerInRoom { server: Box, room_id: Box, @@ -208,7 +208,7 @@ pub(crate) enum RoomStateCache { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] /// All the getters and iterators from src/database/key_value/globals.rs -pub(crate) enum Globals { +pub(super) enum Globals { DatabaseVersion, CurrentCount, @@ -227,7 +227,7 @@ pub(crate) enum Globals { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] /// All the getters and iterators from src/database/key_value/sending.rs -pub(crate) enum Sending { +pub(super) enum Sending { /// - Queries database for all `servercurrentevent_data` ActiveRequests, @@ -283,12 +283,12 @@ pub(crate) enum Sending { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] /// All the getters and iterators from src/database/key_value/users.rs -pub(crate) enum Users { +pub(super) enum Users { Iter, } /// Processes admin query commands -pub(crate) async fn process(command: QueryCommand, _body: Vec<&str>) -> Result { +pub(super) async fn process(command: QueryCommand, _body: Vec<&str>) -> Result { Ok(match command { QueryCommand::AccountData(command) => account_data(command).await?, QueryCommand::Appservice(command) => appservice(command).await?, diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index f69dd19a..c54cad63 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -4,7 +4,7 @@ use super::Presence; use crate::{services, Result}; /// All the getters and iterators in key_value/presence.rs -pub(crate) async fn presence(subcommand: Presence) -> Result { +pub(super) async fn presence(subcommand: Presence) -> Result { match subcommand { Presence::GetPresence { user_id, diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index 62d05899..eaca285e 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -4,7 +4,7 @@ use super::RoomAlias; use crate::{services, Result}; /// All the getters and iterators in src/database/key_value/rooms/alias.rs -pub(crate) async fn room_alias(subcommand: RoomAlias) -> Result { +pub(super) async fn room_alias(subcommand: RoomAlias) -> Result { match subcommand { RoomAlias::ResolveLocalAlias { alias, diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index c062497e..63cf355f 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -3,7 +3,7 @@ use ruma::events::room::message::RoomMessageEventContent; use super::RoomStateCache; use crate::{services, Result}; -pub(crate) async fn room_state_cache(subcommand: RoomStateCache) -> Result { +pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result { match subcommand { RoomStateCache::ServerInRoom { server, diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 5a21ec87..8c52b45f 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -4,7 +4,7 @@ use super::Sending; use crate::{service::sending::Destination, services, Result}; /// All the getters and iterators in key_value/sending.rs -pub(crate) async fn sending(subcommand: Sending) -> Result { +pub(super) async fn sending(subcommand: Sending) -> Result { match subcommand { Sending::ActiveRequests => { let timer = tokio::time::Instant::now(); diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index ef99d880..7f0980ae 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -4,7 +4,7 @@ use super::Users; use crate::{services, Result}; /// All the getters and iterators in key_value/users.rs -pub(crate) async fn users(subcommand: Users) -> Result { +pub(super) async fn users(subcommand: Users) -> Result { match subcommand { Users::Iter => { let timer = tokio::time::Instant::now(); diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index ced96c95..72e72793 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -1,18 +1,18 @@ +mod room_alias_commands; +mod room_commands; +mod room_directory_commands; +mod room_info_commands; +mod room_moderation_commands; + use clap::Subcommand; +use conduit::Result; use ruma::{events::room::message::RoomMessageEventContent, RoomId, RoomOrAliasId}; use self::room_commands::list; -use crate::Result; - -pub(crate) mod room_alias_commands; -pub(crate) mod room_commands; -pub(crate) mod room_directory_commands; -pub(crate) mod room_info_commands; -pub(crate) mod room_moderation_commands; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum RoomCommand { +pub(super) enum RoomCommand { /// - List all rooms the server knows about List { page: Option, @@ -37,7 +37,7 @@ pub(crate) enum RoomCommand { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum RoomInfoCommand { +pub(super) enum RoomInfoCommand { /// - List joined members in a room ListJoinedMembers { room_id: Box, @@ -54,7 +54,7 @@ pub(crate) enum RoomInfoCommand { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum RoomAliasCommand { +pub(super) enum RoomAliasCommand { /// - Make an alias point to a room. Set { #[arg(short, long)] @@ -90,7 +90,7 @@ pub(crate) enum RoomAliasCommand { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum RoomDirectoryCommand { +pub(super) enum RoomDirectoryCommand { /// - Publish a room to the room directory Publish { /// The room id of the room to publish @@ -111,7 +111,7 @@ pub(crate) enum RoomDirectoryCommand { #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum RoomModerationCommand { +pub(super) enum RoomModerationCommand { /// - Bans a room from local users joining and evicts all our local users /// from the room. Also blocks any invites (local and remote) for the /// banned room. @@ -167,7 +167,7 @@ pub(crate) enum RoomModerationCommand { ListBannedRooms, } -pub(crate) async fn process(command: RoomCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: RoomCommand, body: Vec<&str>) -> Result { Ok(match command { RoomCommand::Info(command) => room_info_commands::process(command, body).await?, diff --git a/src/admin/room/room_alias_commands.rs b/src/admin/room/room_alias_commands.rs index bde13e53..4f43ac6e 100644 --- a/src/admin/room/room_alias_commands.rs +++ b/src/admin/room/room_alias_commands.rs @@ -5,7 +5,7 @@ use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId}; use super::RoomAliasCommand; use crate::{escape_html, services, Result}; -pub(crate) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Result { +pub(super) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Result { let server_user = &services().globals.server_user; match command { diff --git a/src/admin/room/room_commands.rs b/src/admin/room/room_commands.rs index 50372e32..f64ccf30 100644 --- a/src/admin/room/room_commands.rs +++ b/src/admin/room/room_commands.rs @@ -4,7 +4,7 @@ use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId}; use crate::{escape_html, get_room_info, handler::PAGE_SIZE, services, Result}; -pub(crate) async fn list(_body: Vec<&str>, page: Option) -> Result { +pub(super) async fn list(_body: Vec<&str>, page: Option) -> Result { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); let mut rooms = services() diff --git a/src/admin/room/room_directory_commands.rs b/src/admin/room/room_directory_commands.rs index 37c09791..5e81bd89 100644 --- a/src/admin/room/room_directory_commands.rs +++ b/src/admin/room/room_directory_commands.rs @@ -5,7 +5,7 @@ use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId}; use super::RoomDirectoryCommand; use crate::{escape_html, get_room_info, handler::PAGE_SIZE, services, Result}; -pub(crate) async fn process(command: RoomDirectoryCommand, _body: Vec<&str>) -> Result { +pub(super) async fn process(command: RoomDirectoryCommand, _body: Vec<&str>) -> Result { match command { RoomDirectoryCommand::Publish { room_id, diff --git a/src/admin/room/room_info_commands.rs b/src/admin/room/room_info_commands.rs index 0bcbc7ae..3b35d82c 100644 --- a/src/admin/room/room_info_commands.rs +++ b/src/admin/room/room_info_commands.rs @@ -6,7 +6,7 @@ use service::services; use super::RoomInfoCommand; use crate::{escape_html, Result}; -pub(crate) async fn process(command: RoomInfoCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: RoomInfoCommand, body: Vec<&str>) -> Result { match command { RoomInfoCommand::ListJoinedMembers { room_id, diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 1876bf5e..095ccf61 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -9,7 +9,7 @@ use tracing::{debug, error, info, warn}; use super::{super::Service, RoomModerationCommand}; use crate::{escape_html, get_room_info, services, user_is_local, Result}; -pub(crate) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> Result { match command { RoomModerationCommand::BanRoom { force, diff --git a/src/admin/server/server_commands.rs b/src/admin/server/commands.rs similarity index 77% rename from src/admin/server/server_commands.rs rename to src/admin/server/commands.rs index 0e01d4bf..eb3ae20c 100644 --- a/src/admin/server/server_commands.rs +++ b/src/admin/server/commands.rs @@ -1,9 +1,9 @@ -use conduit::warn; +use conduit::{warn, Result}; use ruma::events::room::message::RoomMessageEventContent; -use crate::{services, Result}; +use crate::services; -pub(crate) async fn uptime(_body: Vec<&str>) -> Result { +pub(super) async fn uptime(_body: Vec<&str>) -> Result { let seconds = services() .server .started @@ -21,12 +21,12 @@ pub(crate) async fn uptime(_body: Vec<&str>) -> Result Ok(RoomMessageEventContent::notice_plain(result)) } -pub(crate) async fn show_config(_body: Vec<&str>) -> Result { +pub(super) async fn show_config(_body: Vec<&str>) -> Result { // Construct and send the response Ok(RoomMessageEventContent::text_plain(format!("{}", services().globals.config))) } -pub(crate) async fn memory_usage(_body: Vec<&str>) -> Result { +pub(super) async fn memory_usage(_body: Vec<&str>) -> Result { let response0 = services().memory_usage().await; let response1 = services().globals.db.memory_usage(); let response2 = conduit::alloc::memory_usage(); @@ -41,19 +41,19 @@ pub(crate) async fn memory_usage(_body: Vec<&str>) -> Result, amount: u32) -> Result { +pub(super) async fn clear_database_caches(_body: Vec<&str>, amount: u32) -> Result { services().globals.db.clear_caches(amount); Ok(RoomMessageEventContent::text_plain("Done.")) } -pub(crate) async fn clear_service_caches(_body: Vec<&str>, amount: u32) -> Result { +pub(super) async fn clear_service_caches(_body: Vec<&str>, amount: u32) -> Result { services().clear_caches(amount).await; Ok(RoomMessageEventContent::text_plain("Done.")) } -pub(crate) async fn list_backups(_body: Vec<&str>) -> Result { +pub(super) async fn list_backups(_body: Vec<&str>) -> Result { let result = services().globals.db.backup_list()?; if result.is_empty() { @@ -63,7 +63,7 @@ pub(crate) async fn list_backups(_body: Vec<&str>) -> Result) -> Result { +pub(super) async fn backup_database(_body: Vec<&str>) -> Result { if !cfg!(feature = "rocksdb") { return Ok(RoomMessageEventContent::text_plain( "Only RocksDB supports online backups in conduwuit.", @@ -87,7 +87,7 @@ pub(crate) async fn backup_database(_body: Vec<&str>) -> Result) -> Result { +pub(super) async fn list_database_files(_body: Vec<&str>) -> Result { if !cfg!(feature = "rocksdb") { return Ok(RoomMessageEventContent::text_plain( "Only RocksDB supports listing files in conduwuit.", @@ -98,7 +98,7 @@ pub(crate) async fn list_database_files(_body: Vec<&str>) -> Result, message: Vec) -> Result { +pub(super) async fn admin_notice(_body: Vec<&str>, message: Vec) -> Result { let message = message.join(" "); services().admin.send_text(&message).await; @@ -106,20 +106,20 @@ pub(crate) async fn admin_notice(_body: Vec<&str>, message: Vec) -> Resu } #[cfg(conduit_mods)] -pub(crate) async fn reload(_body: Vec<&str>) -> Result { +pub(super) async fn reload(_body: Vec<&str>) -> Result { services().server.reload()?; Ok(RoomMessageEventContent::notice_plain("Reloading server...")) } #[cfg(unix)] -pub(crate) async fn restart(_body: Vec<&str>) -> Result { +pub(super) async fn restart(_body: Vec<&str>) -> Result { services().server.restart()?; Ok(RoomMessageEventContent::notice_plain("Restarting server...")) } -pub(crate) async fn shutdown(_body: Vec<&str>) -> Result { +pub(super) async fn shutdown(_body: Vec<&str>) -> Result { warn!("shutdown command"); services().server.shutdown()?; diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index c3a517c0..13f6e265 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -1,14 +1,14 @@ -pub(crate) mod server_commands; +mod commands; use clap::Subcommand; +use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; -use self::server_commands::*; -use crate::Result; +use self::commands::*; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum ServerCommand { +pub(super) enum ServerCommand { /// - Time elapsed since startup Uptime, @@ -57,7 +57,7 @@ pub(crate) enum ServerCommand { Shutdown, } -pub(crate) async fn process(command: ServerCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: ServerCommand, body: Vec<&str>) -> Result { Ok(match command { ServerCommand::Uptime => uptime(body).await?, ServerCommand::ShowConfig => show_config(body).await?, diff --git a/src/admin/tester/mod.rs b/src/admin/tester/mod.rs index f7b4ecea..8350fe14 100644 --- a/src/admin/tester/mod.rs +++ b/src/admin/tester/mod.rs @@ -4,10 +4,10 @@ use crate::Result; #[cfg_attr(test, derive(Debug))] #[derive(clap::Subcommand)] -pub(crate) enum TesterCommands { +pub(super) enum TesterCommands { Tester, } -pub(crate) async fn process(command: TesterCommands, _body: Vec<&str>) -> Result { +pub(super) async fn process(command: TesterCommands, _body: Vec<&str>) -> Result { Ok(match command { TesterCommands::Tester => RoomMessageEventContent::notice_plain(String::from("completed")), }) diff --git a/src/admin/user/user_commands.rs b/src/admin/user/commands.rs similarity index 96% rename from src/admin/user/user_commands.rs rename to src/admin/user/commands.rs index cada5022..26ec1af5 100644 --- a/src/admin/user/user_commands.rs +++ b/src/admin/user/commands.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{join_room_by_id_helper, leave_all_rooms, update_avatar_url, update_displayname}; -use conduit::utils; +use conduit::{utils, Result}; use ruma::{ events::{ room::message::RoomMessageEventContent, @@ -15,12 +15,11 @@ use tracing::{error, info, warn}; use crate::{ escape_html, get_room_info, services, utils::{parse_active_local_user_id, parse_local_user_id}, - Result, }; const AUTO_GEN_PASSWORD_LENGTH: usize = 25; -pub(crate) async fn list(_body: Vec<&str>) -> Result { +pub(super) async fn list(_body: Vec<&str>) -> Result { match services().users.list_local_users() { Ok(users) => { let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); @@ -36,7 +35,7 @@ pub(crate) async fn list(_body: Vec<&str>) -> Result { } } -pub(crate) async fn create( +pub(super) async fn create( _body: Vec<&str>, username: String, password: Option, ) -> Result { // Validate user id @@ -127,7 +126,7 @@ pub(crate) async fn create( ))) } -pub(crate) async fn deactivate( +pub(super) async fn deactivate( _body: Vec<&str>, no_leave_rooms: bool, user_id: String, ) -> Result { // Validate user id @@ -166,7 +165,7 @@ pub(crate) async fn deactivate( ))) } -pub(crate) async fn reset_password(_body: Vec<&str>, username: String) -> Result { +pub(super) async fn reset_password(_body: Vec<&str>, username: String) -> Result { let user_id = parse_local_user_id(&username)?; if user_id == services().globals.server_user { @@ -190,7 +189,7 @@ pub(crate) async fn reset_password(_body: Vec<&str>, username: String) -> Result } } -pub(crate) async fn deactivate_all( +pub(super) async fn deactivate_all( body: Vec<&str>, no_leave_rooms: bool, force: bool, ) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { @@ -284,7 +283,7 @@ pub(crate) async fn deactivate_all( } } -pub(crate) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Result { +pub(super) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Result { // Validate user id let user_id = parse_local_user_id(&user_id)?; @@ -335,7 +334,7 @@ pub(crate) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Resu Ok(RoomMessageEventContent::text_html(output_plain, output_html)) } -pub(crate) async fn put_room_tag( +pub(super) async fn put_room_tag( _body: Vec<&str>, user_id: String, room_id: Box, tag: String, ) -> Result { let user_id = parse_active_local_user_id(&user_id)?; @@ -370,7 +369,7 @@ pub(crate) async fn put_room_tag( ))) } -pub(crate) async fn delete_room_tag( +pub(super) async fn delete_room_tag( _body: Vec<&str>, user_id: String, room_id: Box, tag: String, ) -> Result { let user_id = parse_active_local_user_id(&user_id)?; @@ -402,7 +401,7 @@ pub(crate) async fn delete_room_tag( ))) } -pub(crate) async fn get_room_tags( +pub(super) async fn get_room_tags( _body: Vec<&str>, user_id: String, room_id: Box, ) -> Result { let user_id = parse_active_local_user_id(&user_id)?; diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index e11429de..31bf57d6 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -1,15 +1,14 @@ -pub(crate) mod user_commands; +mod commands; use clap::Subcommand; +use conduit::Result; use ruma::{events::room::message::RoomMessageEventContent, RoomId}; -use user_commands::{delete_room_tag, get_room_tags, put_room_tag}; -use self::user_commands::{create, deactivate, deactivate_all, list, list_joined_rooms, reset_password}; -use crate::Result; +use self::commands::*; #[cfg_attr(test, derive(Debug))] #[derive(Subcommand)] -pub(crate) enum UserCommand { +pub(super) enum UserCommand { /// - Create a new user Create { /// Username of the new user @@ -93,7 +92,7 @@ pub(crate) enum UserCommand { }, } -pub(crate) async fn process(command: UserCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: UserCommand, body: Vec<&str>) -> Result { Ok(match command { UserCommand::List => list(body).await?, UserCommand::Create { From 1ccdba892143fb1002cf6918645523c6ed40cc7b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 23:09:54 +0000 Subject: [PATCH 0237/2091] use markdown for list-database-files command Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 3 ++- src/database/rocksdb/mod.rs | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index eb3ae20c..bd465e22 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -95,7 +95,8 @@ pub(super) async fn list_database_files(_body: Vec<&str>) -> Result, message: Vec) -> Result { diff --git a/src/database/rocksdb/mod.rs b/src/database/rocksdb/mod.rs index c9493cab..52fd303d 100644 --- a/src/database/rocksdb/mod.rs +++ b/src/database/rocksdb/mod.rs @@ -238,10 +238,12 @@ impl KeyValueDatabaseEngine for Arc { Err(e) => Ok(String::from(e)), Ok(files) => { let mut res = String::new(); + writeln!(res, "| lev | sst | keys | dels | size | column |").expect("written to string buffer"); + writeln!(res, "| ---: | :--- | ---: | ---: | ---: | :--- |").expect("written to string buffer"); for file in files { writeln!( res, - "L{} {:<13} {:7}+ {:4}- {:9} {}
    ", + "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", file.level, file.name, file.num_entries, file.num_deletions, file.size, file.column_family_name, ) .expect("should be able to writeln to string buffer"); From 8affdc43a677cf51f04dc16a36978e5adde12fb3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 00:19:17 +0000 Subject: [PATCH 0238/2091] additional termimad config Signed-off-by: Jason Volk --- src/service/admin/console.rs | 40 ++++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index dc967d00..1603a057 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -27,24 +27,12 @@ const HISTORY_LIMIT: usize = 48; impl Console { #[must_use] pub fn new() -> Arc { - use termimad::{crossterm::style::Color, Alignment, CompoundStyle, LineStyle}; - - let mut output = MadSkin::default_dark(); - let code_style = CompoundStyle::with_fgbg(Color::AnsiValue(40), Color::AnsiValue(234)); - output.inline_code = code_style.clone(); - output.code_block = LineStyle { - left_margin: 0, - right_margin: 0, - align: Alignment::Left, - compound_style: code_style, - }; - Arc::new(Self { worker_join: None.into(), input_abort: None.into(), command_abort: None.into(), history: VecDeque::with_capacity(HISTORY_LIMIT).into(), - output, + output: configure_output(MadSkin::default_dark()), }) } @@ -175,8 +163,7 @@ impl Console { } async fn output(self: Arc, output_content: RoomMessageEventContent) { - let output = self.output.term_text(output_content.body()); - println!("{output}"); + self.output.print_text(output_content.body()); } fn set_history(&self, readline: &mut Readline) { @@ -192,3 +179,26 @@ impl Console { history.truncate(HISTORY_LIMIT); } } + +fn configure_output(mut output: MadSkin) -> MadSkin { + use termimad::{crossterm::style::Color, Alignment, CompoundStyle, LineStyle}; + + let code_style = CompoundStyle::with_fgbg(Color::AnsiValue(40), Color::AnsiValue(234)); + output.inline_code = code_style.clone(); + output.code_block = LineStyle { + left_margin: 0, + right_margin: 0, + align: Alignment::Left, + compound_style: code_style, + }; + + let table_style = CompoundStyle::default(); + output.table = LineStyle { + left_margin: 1, + right_margin: 1, + align: Alignment::Left, + compound_style: table_style, + }; + + output +} From 64705fa27d593255a2ca32cca1a058197d61b9f1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 21:29:59 +0000 Subject: [PATCH 0239/2091] rename router to routes Signed-off-by: Jason Volk --- src/api/mod.rs | 2 +- src/api/{router.rs => routes.rs} | 0 src/router/router.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename src/api/{router.rs => routes.rs} (100%) diff --git a/src/api/mod.rs b/src/api/mod.rs index 2fef4dbf..b9614f25 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,5 +1,5 @@ pub mod client; -pub mod router; +pub mod routes; mod ruma_wrapper; pub mod server; diff --git a/src/api/router.rs b/src/api/routes.rs similarity index 100% rename from src/api/router.rs rename to src/api/routes.rs diff --git a/src/router/router.rs b/src/router/router.rs index c0a382b6..da31ffea 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -15,7 +15,7 @@ pub(crate) fn build(server: &Arc) -> Router { .fallback(not_found) .with_state(state); - api::router::build(router, server) + api::routes::build(router, server) } async fn not_found(_uri: Uri) -> impl IntoResponse { From 1c0ed91f6f35218ebbc3fdf3e6ee7f593460ebea Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 16 Jun 2024 21:32:08 +0000 Subject: [PATCH 0240/2091] rename ruma_wrapper to router Signed-off-by: Jason Volk --- src/api/mod.rs | 5 +++-- src/api/{ruma_wrapper => router}/auth.rs | 0 src/api/{ruma_wrapper => router}/handler.rs | 0 src/api/{ruma_wrapper => router}/mod.rs | 0 src/api/{ruma_wrapper => router}/request.rs | 0 src/api/{ruma_wrapper => router}/xmatrix.rs | 0 src/api/routes.rs | 2 +- 7 files changed, 4 insertions(+), 3 deletions(-) rename src/api/{ruma_wrapper => router}/auth.rs (100%) rename src/api/{ruma_wrapper => router}/handler.rs (100%) rename src/api/{ruma_wrapper => router}/mod.rs (100%) rename src/api/{ruma_wrapper => router}/request.rs (100%) rename src/api/{ruma_wrapper => router}/xmatrix.rs (100%) diff --git a/src/api/mod.rs b/src/api/mod.rs index b9614f25..8e30a518 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,14 +1,15 @@ pub mod client; +mod router; pub mod routes; -mod ruma_wrapper; pub mod server; extern crate conduit_core as conduit; extern crate conduit_service as service; pub(crate) use conduit::{debug_info, debug_warn, utils, Error, Result}; -pub(crate) use ruma_wrapper::{Ruma, RumaResponse}; pub(crate) use service::{pdu::PduEvent, services, user_is_local}; +pub(crate) use crate::router::{Ruma, RumaResponse}; + conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/api/ruma_wrapper/auth.rs b/src/api/router/auth.rs similarity index 100% rename from src/api/ruma_wrapper/auth.rs rename to src/api/router/auth.rs diff --git a/src/api/ruma_wrapper/handler.rs b/src/api/router/handler.rs similarity index 100% rename from src/api/ruma_wrapper/handler.rs rename to src/api/router/handler.rs diff --git a/src/api/ruma_wrapper/mod.rs b/src/api/router/mod.rs similarity index 100% rename from src/api/ruma_wrapper/mod.rs rename to src/api/router/mod.rs diff --git a/src/api/ruma_wrapper/request.rs b/src/api/router/request.rs similarity index 100% rename from src/api/ruma_wrapper/request.rs rename to src/api/router/request.rs diff --git a/src/api/ruma_wrapper/xmatrix.rs b/src/api/router/xmatrix.rs similarity index 100% rename from src/api/ruma_wrapper/xmatrix.rs rename to src/api/router/xmatrix.rs diff --git a/src/api/routes.rs b/src/api/routes.rs index 3157f10d..a733ebe0 100644 --- a/src/api/routes.rs +++ b/src/api/routes.rs @@ -7,7 +7,7 @@ use conduit::{Error, Server}; use http::Uri; use ruma::api::client::error::ErrorKind; -use crate::{client, ruma_wrapper::RouterExt, server}; +use crate::{client, router::RouterExt, server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; From 98d96b89a5a28c9434e4a294df1cf157fd774b26 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 04:57:41 +0000 Subject: [PATCH 0241/2091] minor reductions Signed-off-by: Jason Volk --- src/router/layers.rs | 27 +++++++++++++-------------- src/service/admin/console.rs | 14 ++++++++++---- 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index 3c7a5387..c419d653 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -84,28 +84,28 @@ fn compression_layer(server: &Server) -> tower_http::compression::CompressionLay #[cfg(feature = "zstd_compression")] { - if server.config.zstd_compression { - compression_layer = compression_layer.zstd(true); + compression_layer = if server.config.zstd_compression { + compression_layer.zstd(true) } else { - compression_layer = compression_layer.no_zstd(); + compression_layer.no_zstd() }; }; #[cfg(feature = "gzip_compression")] { - if server.config.gzip_compression { - compression_layer = compression_layer.gzip(true); + compression_layer = if server.config.gzip_compression { + compression_layer.gzip(true) } else { - compression_layer = compression_layer.no_gzip(); + compression_layer.no_gzip() }; }; #[cfg(feature = "brotli_compression")] { - if server.config.brotli_compression { - compression_layer = compression_layer.br(true); + compression_layer = if server.config.brotli_compression { + compression_layer.br(true) } else { - compression_layer = compression_layer.no_br(); + compression_layer.no_br() }; }; @@ -179,11 +179,10 @@ fn catch_panic(err: Box) -> http::Response(request: &http::Request) -> tracing::Span { - let path = if let Some(path) = request.extensions().get::() { - path.as_str() - } else { - request.uri().path() - }; + let path = request + .extensions() + .get::() + .map_or_else(|| request.uri().path(), |p| p.as_str()); tracing::info_span!("router:", %path) } diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 1603a057..31432cda 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -167,10 +167,16 @@ impl Console { } fn set_history(&self, readline: &mut Readline) { - let history = self.history.lock().expect("locked"); - for entry in history.iter().rev() { - readline.add_history_entry(entry.clone()); - } + self.history + .lock() + .expect("locked") + .iter() + .rev() + .for_each(|entry| { + readline + .add_history_entry(entry.clone()) + .expect("added history entry"); + }); } fn add_history(&self, line: String) { From 213a6d441fc9133c986ebeda45d2fb9f403fc2e6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 04:12:11 +0000 Subject: [PATCH 0242/2091] shorten several request span names. Signed-off-by: Jason Volk --- src/api/client/account.rs | 25 ++++++++++++------------- src/api/client/directory.rs | 12 ++++++------ src/api/client/membership.rs | 16 ++++++++-------- src/api/client/sync.rs | 2 +- src/api/server/invite.rs | 4 ++-- src/api/server/publicrooms.rs | 8 ++++---- src/api/server/send.rs | 4 ++-- 7 files changed, 35 insertions(+), 36 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 42049b56..71324c1b 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -40,9 +40,9 @@ const RANDOM_USER_ID_LENGTH: usize = 10; /// /// Note: This will not reserve the username, so the username might become /// invalid when trying to register -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "register_available")] pub(crate) async fn get_register_available_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { // Validate user id let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), services().globals.server_name()) @@ -89,9 +89,9 @@ pub(crate) async fn get_register_available_route( /// - If `inhibit_login` is false: Creates a device and returns device id and /// access_token #[allow(clippy::doc_markdown)] -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "register")] pub(crate) async fn register_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if !services().globals.allow_registration() && body.appservice_info.is_none() { info!( @@ -302,7 +302,7 @@ pub(crate) async fn register_route( services() .admin .send_message(RoomMessageEventContent::notice_plain(format!( - "New user \"{user_id}\" registered on this server from IP {client_ip}." + "New user \"{user_id}\" registered on this server from IP {client}." ))) .await; } @@ -321,7 +321,7 @@ pub(crate) async fn register_route( .admin .send_message(RoomMessageEventContent::notice_plain(format!( "Guest user \"{user_id}\" with device display name `{device_display_name}` registered on this \ - server from IP {client_ip}." + server from IP {client}." ))) .await; } else { @@ -329,7 +329,7 @@ pub(crate) async fn register_route( .admin .send_message(RoomMessageEventContent::notice_plain(format!( "Guest user \"{user_id}\" with no device display name registered on this server from IP \ - {client_ip}.", + {client}.", ))) .await; } @@ -337,8 +337,7 @@ pub(crate) async fn register_route( services() .admin .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with no device display name registered on this server from IP \ - {client_ip}.", + "Guest user \"{user_id}\" with no device display name registered on this server from IP {client}.", ))) .await; } @@ -420,9 +419,9 @@ pub(crate) async fn register_route( /// last seen ts) /// - Forgets to-device events /// - Triggers device list updates -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "change_password")] pub(crate) async fn change_password_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -509,9 +508,9 @@ pub(crate) async fn whoami_route(body: Ruma) -> Result, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 20f0ccca..7671b90a 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -28,9 +28,9 @@ use crate::{service::server_is_ours, services, Error, Result, Ruma}; /// Lists the public rooms on this server. /// /// - Rooms are ordered by the number of joined members -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] pub(crate) async fn get_public_rooms_filtered_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if let Some(server) = &body.server { if services() @@ -66,9 +66,9 @@ pub(crate) async fn get_public_rooms_filtered_route( /// Lists the public rooms on this server. /// /// - Rooms are ordered by the number of joined members -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] pub(crate) async fn get_public_rooms_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if let Some(server) = &body.server { if services() @@ -109,9 +109,9 @@ pub(crate) async fn get_public_rooms_route( /// Sets the visibility of a given room in the room directory. /// /// - TODO: Access control checks -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "room_directory")] pub(crate) async fn set_room_visibility_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 111d03c4..88cbdd44 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -165,7 +165,7 @@ async fn banned_room_check( /// rules locally /// - If the server does not know about the room: asks other servers over /// federation -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client_ip), name = "join")] pub(crate) async fn join_room_by_id_route( InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { @@ -218,16 +218,16 @@ pub(crate) async fn join_room_by_id_route( /// - If the server does not know about the room: use the server name query /// param if specified. if not specified, asks other servers over federation /// via room alias server name and room ID server name -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "join")] pub(crate) async fn join_room_by_id_or_alias_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_deref().expect("user is authenticated"); let body = body.body; let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { Ok(room_id) => { - banned_room_check(sender_user, Some(&room_id), room_id.server_name(), client_ip).await?; + banned_room_check(sender_user, Some(&room_id), room_id.server_name(), client).await?; let mut servers = body.server_name.clone(); servers.extend( @@ -261,7 +261,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( Err(room_alias) => { let response = get_alias_helper(room_alias.clone(), Some(body.server_name.clone())).await?; - banned_room_check(sender_user, Some(&response.room_id), Some(room_alias.server_name()), client_ip).await?; + banned_room_check(sender_user, Some(&response.room_id), Some(room_alias.server_name()), client).await?; let mut servers = body.server_name; servers.extend(response.servers); @@ -321,9 +321,9 @@ pub(crate) async fn leave_room_route(body: Ruma) -> Res /// # `POST /_matrix/client/r0/rooms/{roomId}/invite` /// /// Tries to send an invite event into the room. -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "invite")] pub(crate) async fn invite_user_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -338,7 +338,7 @@ pub(crate) async fn invite_user_route( )); } - banned_room_check(sender_user, Some(&body.room_id), body.room_id.server_name(), client_ip).await?; + banned_room_check(sender_user, Some(&body.room_id), body.room_id.server_name(), client).await?; if let invite_user::v3::InvitationRecipient::UserId { user_id, diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 9e9a9ffd..738c94d7 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -312,7 +312,7 @@ pub(crate) async fn sync_events_route( Ok(response) } -#[tracing::instrument(skip_all, fields(user_id = %sender_user, room_id = %room_id))] +#[tracing::instrument(skip_all, fields(user_id = %sender_user, room_id = %room_id), name = "left_room")] async fn handle_left_room( since: u64, room_id: &RoomId, sender_user: &UserId, left_rooms: &mut BTreeMap, next_batch_string: &str, full_state: bool, lazy_load_enabled: bool, diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 1b256b4a..333ebee6 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -17,9 +17,9 @@ use crate::{ /// # `PUT /_matrix/federation/v2/invite/{roomId}/{eventId}` /// /// Invites a remote user to a room. -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "invite")] pub(crate) async fn create_invite_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index 6b5010e5..13edd8d6 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -12,9 +12,9 @@ use crate::{services, Error, Result, Ruma}; /// # `POST /_matrix/federation/v1/publicRooms` /// /// Lists the public rooms on this server. -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] pub(crate) async fn get_public_rooms_filtered_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if !services() .globals @@ -44,9 +44,9 @@ pub(crate) async fn get_public_rooms_filtered_route( /// # `GET /_matrix/federation/v1/publicRooms` /// /// Lists the public rooms on this server. -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), "publicrooms")] pub(crate) async fn get_public_rooms_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if !services() .globals diff --git a/src/api/server/send.rs b/src/api/server/send.rs index e147f652..1168dd82 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -26,9 +26,9 @@ use crate::{ /// # `PUT /_matrix/federation/v1/send/{txnId}` /// /// Push EDUs and PDUs to this server. -#[tracing::instrument(skip_all, fields(%client_ip))] +#[tracing::instrument(skip_all, fields(%client), name = "send")] pub(crate) async fn send_transaction_message_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); From 14dcc8db15358df57ec2bc43aa112bfc9baa2326 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 05:24:08 +0000 Subject: [PATCH 0243/2091] truncate MatchedPath for span field Signed-off-by: Jason Volk --- src/router/layers.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index c419d653..4fe35164 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -182,7 +182,13 @@ fn tracing_span(request: &http::Request) -> tracing::Span { let path = request .extensions() .get::() - .map_or_else(|| request.uri().path(), |p| p.as_str()); + .map_or_else(|| request.uri().path(), truncated_matched_path); tracing::info_span!("router:", %path) } + +fn truncated_matched_path(path: &MatchedPath) -> &str { + path.as_str() + .rsplit_once(':') + .map_or(path.as_str(), |path| path.0.strip_suffix('/').unwrap_or(path.0)) +} From cbb97b4fdf8186e3386b9d3a5fd239c4d88e02b8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 06:07:26 +0000 Subject: [PATCH 0244/2091] bump termimad Signed-off-by: Jason Volk --- Cargo.lock | 4 ++-- Cargo.toml | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 71540f8d..07f5e827 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3975,9 +3975,9 @@ dependencies = [ [[package]] name = "termimad" -version = "0.29.3" +version = "0.29.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d4260ceb786f1d101630049b2a914479f3c049013f9323bd031dfed3ff70ff" +checksum = "eff854b510d076163c3b8628ebfe6bfbe6f01c3254b8b8e77cac7ad2d8c72b06" dependencies = [ "coolor", "crokey", diff --git a/Cargo.toml b/Cargo.toml index 70599fcf..0db139ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -419,7 +419,9 @@ features = [ version = "0.4.2" [workspace.dependencies.termimad] -version = "0.29.3" +version = "0.29.4" +default-features = false + # # Patches From 5f46623371952dafa064c59466c13787c616d208 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 22 Jun 2024 21:21:16 +0000 Subject: [PATCH 0245/2091] redaction fixes --- src/service/rooms/event_handler/mod.rs | 15 +++++++- src/service/rooms/state_accessor/mod.rs | 46 +++++++++++++++++++++++ src/service/rooms/timeline/mod.rs | 49 +++++++++++++++++++------ 3 files changed, 97 insertions(+), 13 deletions(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index e2cd5066..813b8d4a 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -19,7 +19,7 @@ use ruma::{ }, events::{ room::{create::RoomCreateEventContent, server_acl::RoomServerAclEventContent}, - StateEventType, + StateEventType, TimelineEventType, }, int, serde::Base64, @@ -525,7 +525,18 @@ impl Service { let soft_fail = !state_res::event_auth::auth_check(&room_version, &incoming_pdu, None::, |k, s| { auth_events.get(&(k.clone(), s.to_owned())) }) - .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))?; + .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))? + || if let Some(redact_id) = &incoming_pdu.redacts { + incoming_pdu.kind == TimelineEventType::RoomRedaction + && !services().rooms.state_accessor.user_can_redact( + redact_id, + &incoming_pdu.sender, + &incoming_pdu.room_id, + true, + )? + } else { + false + }; // 13. Use state resolution to find new room state diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index ab290cbd..06293ea3 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -16,6 +16,7 @@ use ruma::{ history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, member::{MembershipState, RoomMemberEventContent}, name::RoomNameEventContent, + power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, topic::RoomTopicEventContent, }, StateEventType, @@ -355,4 +356,49 @@ impl Service { }) }) } + + /// Checks if a given user can redact a given event + /// + /// If federation is true, it allows redaction events from any user of the + /// same server as the original event sender + pub fn user_can_redact( + &self, redacts: &EventId, sender: &UserId, room_id: &RoomId, federation: bool, + ) -> Result { + self.room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? + .map_or_else( + || { + // Falling back on m.room.create to judge power level + if let Some(pdu) = self.room_state_get(room_id, &StateEventType::RoomCreate, "")? { + Ok(pdu.sender == sender + || if let Ok(Some(pdu)) = services().rooms.timeline.get_pdu(redacts) { + pdu.sender == sender + } else { + false + }) + } else { + Err(Error::bad_database( + "No m.room.power_levels or m.room.create events in database for room", + )) + } + }, + |event| { + serde_json::from_str(event.content.get()) + .map(|content: RoomPowerLevelsEventContent| content.into()) + .map(|event: RoomPowerLevels| { + event.user_can_redact_event_of_other(sender) + || event.user_can_redact_own_event(sender) + && if let Ok(Some(pdu)) = services().rooms.timeline.get_pdu(redacts) { + if federation { + pdu.sender.server_name() == sender.server_name() + } else { + pdu.sender == sender + } + } else { + false + } + }) + .map_err(|_| Error::bad_database("Invalid m.room.power_levels event in database")) + }, + ) + } } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index ed369334..7712ab90 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -397,21 +397,36 @@ impl Service { | RoomVersionId::V9 | RoomVersionId::V10 => { if let Some(redact_id) = &pdu.redacts { - self.redact_pdu(redact_id, pdu, shortroomid)?; + if services().rooms.state_accessor.user_can_redact( + redact_id, + &pdu.sender, + &pdu.room_id, + false, + )? { + self.redact_pdu(redact_id, pdu, shortroomid)?; + } } }, RoomVersionId::V11 => { let content = serde_json::from_str::(pdu.content.get()).map_err(|e| { warn!("Invalid content in redaction pdu: {e}"); - Error::bad_database("Invalid content in redaction pdu.") + Error::bad_database("Invalid content in redaction pdu") })?; + if let Some(redact_id) = &content.redacts { - self.redact_pdu(redact_id, pdu, shortroomid)?; + if services().rooms.state_accessor.user_can_redact( + redact_id, + &pdu.sender, + &pdu.room_id, + false, + )? { + self.redact_pdu(redact_id, pdu, shortroomid)?; + } } }, _ => { - warn!("Unexpected or unsupported room version {}", room_version_id); + warn!("Unexpected or unsupported room version {room_version_id}"); return Err(Error::BadRequest( ErrorKind::BadJson, "Unexpected or unsupported room version found", @@ -703,7 +718,7 @@ impl Service { // Hash and sign let mut pdu_json = utils::to_canonical_object(&pdu).map_err(|e| { - error!("Failed to convert PDU to canonical JSON: {}", e); + error!("Failed to convert PDU to canonical JSON: {e}"); Error::bad_database("Failed to convert PDU to canonical JSON.") })?; @@ -778,7 +793,7 @@ impl Service { warn!("Encryption is not allowed in the admins room"); return Err(Error::BadRequest( ErrorKind::forbidden(), - "Encryption is not allowed in the admins room.", + "Encryption is not allowed in the admins room", )); }, TimelineEventType::RoomMember => { @@ -789,14 +804,14 @@ impl Service { let server_user = &services().globals.server_user.to_string(); let content = serde_json::from_str::(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid content in pdu."))?; + .map_err(|_| Error::bad_database("Invalid content in pdu"))?; if content.membership == MembershipState::Leave { if target == server_user { - warn!("Conduit user cannot leave from admins room"); + warn!("Server user cannot leave from admins room"); return Err(Error::BadRequest( ErrorKind::forbidden(), - "Conduit user cannot leave from admins room.", + "Server user cannot leave from admins room.", )); } @@ -818,10 +833,10 @@ impl Service { if content.membership == MembershipState::Ban && pdu.state_key().is_some() { if target == server_user { - warn!("Conduit user cannot be banned in admins room"); + warn!("Server user cannot be banned in admins room"); return Err(Error::BadRequest( ErrorKind::forbidden(), - "Conduit user cannot be banned in admins room.", + "Server user cannot be banned in admins room.", )); } @@ -846,6 +861,18 @@ impl Service { } } + // If redaction event is not authorized, do not append it to the timeline + if let Some(redact_id) = &pdu.redacts { + if pdu.kind == TimelineEventType::RoomRedaction + && !services() + .rooms + .state_accessor + .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false)? + { + return Err(Error::BadRequest(ErrorKind::forbidden(), "User cannot redact this event")); + } + }; + // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't // fail. From 7eee88160a8c42d44fddb394a0274e5e45841347 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 06:32:53 +0000 Subject: [PATCH 0246/2091] truncate other span names and fields Signed-off-by: Jason Volk --- src/service/rooms/auth_chain/mod.rs | 4 ++-- src/service/sending/sender.rs | 11 +++-------- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 84e423e3..b93ea8c5 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -8,7 +8,7 @@ use data::Data; use ruma::{api::client::error::ErrorKind, EventId, RoomId}; use tracing::{debug, error, trace, warn}; -use crate::{services, utils::debug_slice_truncated, Error, Result}; +use crate::{services, Error, Result}; pub struct Service { pub db: Arc, @@ -30,7 +30,7 @@ impl Service { .filter_map(move |sid| services().rooms.short.get_eventid_from_short(sid).ok())) } - #[tracing::instrument(skip(self), fields(starting_events = debug_slice_truncated(starting_events, 5)))] + #[tracing::instrument(skip_all, name = "auth_chain")] pub async fn get_auth_chain(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result> { const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 7efe3c72..8479a869 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -541,7 +541,7 @@ async fn send_events_dest_push( #[tracing::instrument(skip(dest, events), name = "")] async fn send_events_dest_normal( - dest: &Destination, server_name: &OwnedServerName, events: Vec, + dest: &Destination, server: &OwnedServerName, events: Vec, ) -> SendingResult { let mut edu_jsons = Vec::new(); let mut pdu_jsons = Vec::new(); @@ -557,12 +557,7 @@ async fn send_events_dest_normal( .get_pdu_json_from_id(pdu_id) .map_err(|e| (dest.clone(), e))? .ok_or_else(|| { - error!( - dest = ?dest, - server_name = ?server_name, - pdu_id = ?pdu_id, - "event not found" - ); + error!(?dest, ?server, ?pdu_id, "event not found"); ( dest.clone(), Error::bad_database("[Normal] Event in servernameevent_data not found in db."), @@ -587,7 +582,7 @@ async fn send_events_dest_normal( // transaction"); send::send( client, - server_name, + server, send_transaction_message::v1::Request { origin: services().globals.server_name().to_owned(), pdus: pdu_jsons, From 3b34e72456adae1704b191ea7fef546b9d1b391b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 10:12:44 +0000 Subject: [PATCH 0247/2091] improve output scheme for admin query commands Signed-off-by: Jason Volk --- src/admin/query/account_data.rs | 14 ++-- src/admin/query/appservice.rs | 14 ++-- src/admin/query/globals.rs | 35 ++++----- src/admin/query/presence.rs | 14 ++-- src/admin/query/room_alias.rs | 21 +++--- src/admin/query/room_state_cache.rs | 112 ++++++++++++---------------- src/admin/query/sending.rs | 28 +++---- src/admin/query/users.rs | 7 +- 8 files changed, 105 insertions(+), 140 deletions(-) diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index 290d5a04..9f5b1482 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -18,10 +18,9 @@ pub(super) async fn account_data(subcommand: AccountData) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, AccountData::Get { user_id, @@ -35,10 +34,9 @@ pub(super) async fn account_data(subcommand: AccountData) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, } } diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index a94f22b8..7331462a 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -16,20 +16,18 @@ pub(super) async fn appservice(subcommand: Appservice) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, Appservice::All => { let timer = tokio::time::Instant::now(); let results = services().appservice.db.all(); let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, } } diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index bc8bcdef..2e22d688 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -11,40 +11,36 @@ pub(super) async fn globals(subcommand: Globals) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, Globals::CurrentCount => { let timer = tokio::time::Instant::now(); let results = services().globals.db.current_count(); let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, Globals::LastCheckForUpdatesId => { let timer = tokio::time::Instant::now(); let results = services().globals.db.last_check_for_updates_id(); let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, Globals::LoadKeypair => { let timer = tokio::time::Instant::now(); let results = services().globals.db.load_keypair(); let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, Globals::SigningKeysFor { origin, @@ -53,10 +49,9 @@ pub(super) async fn globals(subcommand: Globals) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, } } diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index c54cad63..bd853ada 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -13,10 +13,9 @@ pub(super) async fn presence(subcommand: Presence) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, Presence::PresenceSince { since, @@ -27,10 +26,9 @@ pub(super) async fn presence(subcommand: Presence) -> Result = results.collect(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{presence_since:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {presence_since:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{presence_since:#?}\n```" + ))) }, } } diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index eaca285e..3262303a 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -13,10 +13,9 @@ pub(super) async fn room_alias(subcommand: RoomAlias) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomAlias::LocalAliasesForRoom { room_id, @@ -27,10 +26,9 @@ pub(super) async fn room_alias(subcommand: RoomAlias) -> Result = results.collect(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{aliases:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {aliases:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```" + ))) }, RoomAlias::AllLocalAliases => { let timer = tokio::time::Instant::now(); @@ -39,10 +37,9 @@ pub(super) async fn room_alias(subcommand: RoomAlias) -> Result = results.collect(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{aliases:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {aliases:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```" + ))) }, } } diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index 63cf355f..aed2b4a2 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -16,10 +16,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {result:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) }, RoomStateCache::RoomServers { room_id, @@ -32,10 +31,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::ServerRooms { server, @@ -44,10 +42,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result> = services().rooms.state_cache.server_rooms(&server).collect(); let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::RoomMembers { room_id, @@ -60,10 +57,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::LocalUsersInRoom { room_id, @@ -76,10 +72,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::ActiveLocalUsersInRoom { room_id, @@ -92,10 +87,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::RoomJoinedCount { room_id, @@ -104,10 +98,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::RoomInvitedCount { room_id, @@ -116,10 +109,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::RoomUserOnceJoined { room_id, @@ -132,10 +124,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::RoomMembersInvited { room_id, @@ -148,10 +139,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::GetInviteCount { room_id, @@ -164,10 +154,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::GetLeftCount { room_id, @@ -180,10 +169,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::RoomsJoined { user_id, @@ -196,10 +184,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::RoomsInvited { user_id, @@ -212,10 +199,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::RoomsLeft { user_id, @@ -224,10 +210,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result> = services().rooms.state_cache.rooms_left(&user_id).collect(); let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{results:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, RoomStateCache::InviteState { user_id, @@ -240,10 +225,9 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, } } diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 8c52b45f..9690f8f6 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -13,10 +13,9 @@ pub(super) async fn sending(subcommand: Sending) -> Result> = results.collect(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{active_requests:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {active_requests:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" + ))) }, Sending::QueuedRequests { appservice_id, @@ -92,10 +91,9 @@ pub(super) async fn sending(subcommand: Sending) -> Result>>(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{queued_requests:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {queued_requests:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{queued_requests:#?}\n```" + ))) }, Sending::ActiveRequestsFor { appservice_id, @@ -171,10 +169,9 @@ pub(super) async fn sending(subcommand: Sending) -> Result>>(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{active_requests:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {active_requests:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" + ))) }, Sending::GetLatestEduCount { server_name, @@ -183,10 +180,9 @@ pub(super) async fn sending(subcommand: Sending) -> ResultQuery completed in {query_time:?}:

    \n
    {results:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) }, } } diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index 7f0980ae..8f88808a 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -13,10 +13,9 @@ pub(super) async fn users(subcommand: Users) -> Result let users = results.collect::>(); - Ok(RoomMessageEventContent::text_html( - format!("Query completed in {query_time:?}:\n\n```\n{users:?}```"), - format!("

    Query completed in {query_time:?}:

    \n
    {users:?}\n
    "), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{users:#?}\n```" + ))) }, } } From 2f84bc895d40b485bef0408baa9b086908008758 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 19 Jun 2024 01:55:21 +0000 Subject: [PATCH 0248/2091] Improve additional command outputs containing codeblocks. Signed-off-by: Jason Volk --- src/admin/appservice/commands.rs | 9 +-- src/admin/debug/commands.rs | 68 +++++++--------------- src/admin/federation/commands.rs | 12 ++-- src/admin/fsck/commands.rs | 2 +- src/admin/handler.rs | 4 +- src/admin/room/room_info_commands.rs | 34 ++--------- src/admin/room/room_moderation_commands.rs | 25 +------- src/admin/user/commands.rs | 13 ++--- src/admin/utils.rs | 1 - 9 files changed, 40 insertions(+), 128 deletions(-) diff --git a/src/admin/appservice/commands.rs b/src/admin/appservice/commands.rs index 5b764296..5cce8351 100644 --- a/src/admin/appservice/commands.rs +++ b/src/admin/appservice/commands.rs @@ -1,6 +1,6 @@ use ruma::{api::appservice::Registration, events::room::message::RoomMessageEventContent}; -use crate::{escape_html, services, Result}; +use crate::{services, Result}; pub(super) async fn register(body: Vec<&str>) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { @@ -48,12 +48,7 @@ pub(super) async fn show(_body: Vec<&str>, appservice_identifier: String) -> Res Some(config) => { let config_str = serde_yaml::to_string(&config).expect("config should've been validated on register"); let output = format!("Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```",); - let output_html = format!( - "Config for {}:\n\n
    {}
    ", - escape_html(&appservice_identifier), - escape_html(&config_str), - ); - Ok(RoomMessageEventContent::text_html(output, output_html)) + Ok(RoomMessageEventContent::notice_markdown(output)) }, None => Ok(RoomMessageEventContent::text_plain("Appservice does not exist.")), } diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 0409af32..40aa0278 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -8,7 +8,6 @@ use api::client::validate_and_add_event_id; use conduit::{ debug, info, log, log::{capture, Capture}, - utils::HtmlEscape, warn, Error, Result, }; use ruma::{ @@ -93,26 +92,15 @@ pub(super) async fn get_pdu(_body: Vec<&str>, event_id: Box) -> Result< match pdu_json { Some(json) => { let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json"); - Ok(RoomMessageEventContent::text_html( - format!( - "{}\n```json\n{}\n```", - if outlier { - "Outlier PDU found in our database" - } else { - "PDU found in our database" - }, - json_text - ), - format!( - "

    {}

    \n
    {}\n
    \n", - if outlier { - "Outlier PDU found in our database" - } else { - "PDU found in our database" - }, - HtmlEscape(&json_text) - ), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "{}\n```json\n{}\n```", + if outlier { + "Outlier PDU found in our database" + } else { + "PDU found in our database" + }, + json_text + ))) }, None => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")), } @@ -237,17 +225,10 @@ pub(super) async fn get_remote_pdu( let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json"); - Ok(RoomMessageEventContent::text_html( - format!( - "{}\n```json\n{}\n```", - "Got PDU from specified server and handled as backfilled PDU successfully. Event body:", json_text - ), - format!( - "

    {}

    \n
    {}\n
    \n", - "Got PDU from specified server and handled as backfilled PDU successfully. Event body:", - HtmlEscape(&json_text) - ), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "{}\n```json\n{}\n```", + "Got PDU from specified server and handled as backfilled PDU successfully. Event body:", json_text + ))) }, Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "Remote server did not have PDU or failed sending request to remote server: {e}" @@ -278,14 +259,10 @@ pub(super) async fn get_room_state(_body: Vec<&str>, room_id: Box) -> Re ) })?; - Ok(RoomMessageEventContent::text_html( - format!("{}\n```json\n{}\n```", "Found full room state", json_text), - format!( - "

    {}

    \n
    {}\n
    \n", - "Found full room state", - HtmlEscape(&json_text) - ), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "{}\n```json\n{}\n```", + "Found full room state", json_text + ))) } pub(super) async fn ping(_body: Vec<&str>, server: Box) -> Result { @@ -308,14 +285,9 @@ pub(super) async fn ping(_body: Vec<&str>, server: Box) -> ResultGot response which took {ping_time:?} time:

    \n
    {}\n
    \n", - HtmlEscape(&json) - ), - )); + return Ok(RoomMessageEventContent::notice_markdown(format!( + "Got response which took {ping_time:?} time:\n```json\n{json}\n```" + ))); } Ok(RoomMessageEventContent::text_plain(format!( diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 4084350d..293efdc5 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -2,7 +2,7 @@ use std::fmt::Write; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId, ServerName, UserId}; -use crate::{escape_html, get_room_info, services, utils::HtmlEscape, Result}; +use crate::{escape_html, get_room_info, services, Result}; pub(super) async fn disable_room(_body: Vec<&str>, room_id: Box) -> Result { services().rooms.metadata.disable_room(&room_id, true)?; @@ -63,13 +63,9 @@ pub(super) async fn fetch_support_well_known( }, }; - Ok(RoomMessageEventContent::text_html( - format!("Got JSON response:\n\n```json\n{pretty_json}\n```"), - format!( - "

    Got JSON response:

    \n
    {}\n
    \n", - HtmlEscape(&pretty_json) - ), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Got JSON response:\n\n```json\n{pretty_json}\n```" + ))) } pub(super) async fn remote_user_in_rooms(_body: Vec<&str>, user_id: Box) -> Result { diff --git a/src/admin/fsck/commands.rs b/src/admin/fsck/commands.rs index 9fb0281a..f3a2972a 100644 --- a/src/admin/fsck/commands.rs +++ b/src/admin/fsck/commands.rs @@ -22,5 +22,5 @@ pub(super) async fn check_all_users(_body: Vec<&str>) -> Result RoomMessageEventContent { Ok(reply_message) => reply_message, Err(error) => { let markdown_message = format!("Encountered an error while handling the command:\n```\n{error}\n```",); - let html_message = format!("Encountered an error while handling the command:\n
    \n{error}\n
    ",); - - RoomMessageEventContent::text_html(markdown_message, html_message) + RoomMessageEventContent::notice_markdown(markdown_message) }, } } diff --git a/src/admin/room/room_info_commands.rs b/src/admin/room/room_info_commands.rs index 3b35d82c..d985fb87 100644 --- a/src/admin/room/room_info_commands.rs +++ b/src/admin/room/room_info_commands.rs @@ -1,10 +1,8 @@ -use std::fmt::Write; - use ruma::{events::room::message::RoomMessageEventContent, RoomId}; use service::services; use super::RoomInfoCommand; -use crate::{escape_html, Result}; +use crate::Result; pub(super) async fn process(command: RoomInfoCommand, body: Vec<&str>) -> Result { match command { @@ -57,26 +55,7 @@ async fn list_joined_members(_body: Vec<&str>, room_id: Box) -> Result{} Members in Room \"{}\" \nMXID\tDisplay \ - Name\n{}", - member_info.len(), - room_name, - member_info - .iter() - .fold(String::new(), |mut output, (mxid, displayname)| { - writeln!( - output, - "{}\t{}", - mxid, - escape_html(displayname.as_ref()) - ) - .expect("should be able to write to string buffer"); - output - }) - ); - - Ok(RoomMessageEventContent::text_html(output_plain, output_html)) + Ok(RoomMessageEventContent::notice_markdown(output_plain)) } async fn view_room_topic(_body: Vec<&str>, room_id: Box) -> Result { @@ -84,10 +63,7 @@ async fn view_room_topic(_body: Vec<&str>, room_id: Box) -> ResultRoom topic:

    \n
    \n{}
    ", escape_html(&room_topic)); - - Ok(RoomMessageEventContent::text_html( - format!("Room topic:\n\n```{room_topic}\n```"), - output_html, - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "Room topic:\n\n```{room_topic}\n```" + ))) } diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 095ccf61..75a20584 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -1,5 +1,3 @@ -use std::fmt::Write; - use api::client::{get_alias_helper, leave_room}; use ruma::{ events::room::message::RoomMessageEventContent, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, RoomOrAliasId, @@ -7,7 +5,7 @@ use ruma::{ use tracing::{debug, error, info, warn}; use super::{super::Service, RoomModerationCommand}; -use crate::{escape_html, get_room_info, services, user_is_local, Result}; +use crate::{get_room_info, services, user_is_local, Result}; pub(super) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> Result { match command { @@ -492,26 +490,7 @@ async fn list_banned_rooms(_body: Vec<&str>) -> Result .join("\n") ); - let output_html = format!( - "\n\t\t\n{}
    Rooms Banned ({}) \ -
    idmembersname
    ", - rooms.len(), - rooms - .iter() - .fold(String::new(), |mut output, (id, members, name)| { - writeln!( - output, - "{}\t{}\t{}", - id, - members, - escape_html(name.as_ref()) - ) - .expect("should be able to write to string buffer"); - output - }) - ); - - Ok(RoomMessageEventContent::text_html(output_plain, output_html)) + Ok(RoomMessageEventContent::notice_markdown(output_plain)) }, Err(e) => { error!("Failed to list banned rooms: {}", e); diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 26ec1af5..6dc60713 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -26,10 +26,7 @@ pub(super) async fn list(_body: Vec<&str>) -> Result { plain_msg += &users.join("\n"); plain_msg += "\n```"; - let mut html_msg = format!("

    Found {} local user account(s):

    ", users.len());
-			html_msg += &users.join("\n");
-			html_msg += "\n
    "; - Ok(RoomMessageEventContent::text_html(&plain_msg, &html_msg)) + Ok(RoomMessageEventContent::notice_markdown(plain_msg)) }, Err(e) => Ok(RoomMessageEventContent::text_plain(e.to_string())), } @@ -419,8 +416,8 @@ pub(super) async fn get_room_tags( |e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"), ); - Ok(RoomMessageEventContent::text_html( - format!("
    \n{:?}\n
    ", tags_event.content.tags), - format!("```\n{:?}\n```", tags_event.content.tags), - )) + Ok(RoomMessageEventContent::notice_markdown(format!( + "```\n{:#?}\n```", + tags_event.content.tags + ))) } diff --git a/src/admin/utils.rs b/src/admin/utils.rs index 4c30e5a8..91982fe3 100644 --- a/src/admin/utils.rs +++ b/src/admin/utils.rs @@ -1,4 +1,3 @@ -pub(crate) use conduit::utils::HtmlEscape; use conduit_core::Error; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; use service::user_is_local; From 0f07b861594e0b959e0bda8053795f1e86f60261 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 22 Jun 2024 17:51:51 +0000 Subject: [PATCH 0249/2091] bump Cargo.lock --- Cargo.lock | 470 ++++++++++++++++------------------------------------- 1 file changed, 136 insertions(+), 334 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 07f5e827..f6f849dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,7 +130,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -141,7 +141,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.72" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17c6a35df3749d2e8bb1b7b21a976d82b15548788d2735b9d82f329268f71a11" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -389,7 +389,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -451,9 +451,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.16.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78834c15cb5d5efe3452d58b1e8ba890dd62d21907f867f383358198e56ebca5" +checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" [[package]] name = "byteorder" @@ -565,7 +565,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -887,9 +887,9 @@ dependencies = [ [[package]] name = "crokey" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b48209802ec5862bb034cb16719eec24d1c759e62921be7d3c899d0d85f3344b" +checksum = "b015414137feca6f4137550d60d233b580a09307e2de329412b8e8c661b6dfab" dependencies = [ "crokey-proc_macros", "crossterm", @@ -900,9 +900,9 @@ dependencies = [ [[package]] name = "crokey-proc_macros" -version = "0.6.4" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397d3c009d8df93c4b063ddaa44a81ee7098feb056f99b00896c36e2cee9a9f7" +checksum = "6085c6385034f0847d2b3120547a88da1e5e35727970a70f32b555eb3c47051c" dependencies = [ "crossterm", "proc-macro2", @@ -1005,16 +1005,15 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.2" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", "cpufeatures", "curve25519-dalek-derive", "digest", "fiat-crypto", - "platforms", "rustc_version", "subtle", "zeroize", @@ -1028,7 +1027,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -1089,17 +1088,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "displaydoc" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "ed25519" version = "2.2.3" @@ -1143,7 +1131,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -1291,7 +1279,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -1584,7 +1572,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -1645,9 +1633,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -1708,19 +1696,21 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.26.0" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", "hyper 1.3.1", "hyper-util", - "rustls 0.22.4", + "rustls 0.23.10", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tower-service", + "webpki-roots", ] [[package]] @@ -1755,124 +1745,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "icu_collections" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locid" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - -[[package]] -name = "icu_normalizer" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "utf16_iter", - "utf8_iter", - "write16", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" - -[[package]] -name = "icu_properties" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locid_transform", - "icu_properties_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" - -[[package]] -name = "icu_provider" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_provider_macros", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "idna" version = "0.4.0" @@ -1885,14 +1757,12 @@ dependencies = [ [[package]] name = "idna" -version = "1.0.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" dependencies = [ - "icu_normalizer", - "icu_properties", - "smallvec", - "utf8_iter", + "unicode-bidi", + "unicode-normalization", ] [[package]] @@ -2102,14 +1972,14 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] name = "lazy_static" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lazycell" @@ -2160,12 +2030,6 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" -[[package]] -name = "litemap" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" - [[package]] name = "lock_api" version = "0.4.12" @@ -2268,9 +2132,9 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "mime" @@ -2295,9 +2159,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", "simd-adler32", @@ -2450,9 +2314,9 @@ dependencies = [ [[package]] name = "object" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ec7ab813848ba4522158d5517a6093db1ded27575b070f4177b8d12b41db5e" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] @@ -2627,7 +2491,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2720,7 +2584,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2751,12 +2615,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" -[[package]] -name = "platforms" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db23d408679286588f4d4644f965003d056e3dd5abcaaa938116871d7ce2fee7" - [[package]] name = "png" version = "0.17.13" @@ -2799,9 +2657,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.85" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] @@ -2814,7 +2672,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", "version_check", "yansi", ] @@ -2839,7 +2697,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -2875,6 +2733,53 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quinn" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.10", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", + "rustls 0.23.10", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46" +dependencies = [ + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -2916,9 +2821,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ "bitflags 2.5.0", ] @@ -2969,9 +2874,9 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" dependencies = [ "async-compression", "base64 0.22.1", @@ -2994,16 +2899,17 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.22.4", + "quinn", + "rustls 0.23.10", "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls 0.26.0", "tokio-socks", "tokio-util", "tower-service", @@ -3190,7 +3096,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.66", + "syn 2.0.67", "toml", ] @@ -3331,6 +3237,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls" +version = "0.23.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.102.4", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" version = "0.7.0" @@ -3631,7 +3551,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -3864,12 +3784,6 @@ dependencies = [ "der", ] -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - [[package]] name = "strict" version = "0.2.0" @@ -3913,9 +3827,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "0d0208408ba0c3df17ed26eb06992cb1a1268d41b2c0e12e65203fbe3972cee5" [[package]] name = "syn" @@ -3930,9 +3844,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" dependencies = [ "proc-macro2", "quote", @@ -3951,17 +3865,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" -[[package]] -name = "synstructure" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "tendril" version = "0.4.3" @@ -4016,7 +3919,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -4113,16 +4016,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tinystr" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" -dependencies = [ - "displaydoc", - "zerovec", -] - [[package]] name = "tinyvec" version = "1.6.0" @@ -4175,7 +4068,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -4201,11 +4094,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.25.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.22.4", + "rustls 0.23.10", "rustls-pki-types", "tokio", ] @@ -4391,7 +4284,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390 dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", ] [[package]] @@ -4576,12 +4469,12 @@ dependencies = [ [[package]] name = "url" -version = "2.5.1" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", - "idna 1.0.0", + "idna 0.5.0", "percent-encoding", "serde", ] @@ -4598,18 +4491,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "uuid" version = "1.8.0" @@ -4674,7 +4555,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", "wasm-bindgen-shared", ] @@ -4708,7 +4589,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.67", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4763,9 +4644,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.2" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c452ad30530b54a4d8e71952716a212b08efd0f3562baa66c29a618b07da7c3" +checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" dependencies = [ "rustls-pki-types", ] @@ -5006,18 +4887,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - -[[package]] -name = "writeable" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - [[package]] name = "xml5ever" version = "0.18.1" @@ -5035,30 +4904,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" -[[package]] -name = "yoke" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", - "synstructure", -] - [[package]] name = "zerocopy" version = "0.7.34" @@ -5076,28 +4921,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", -] - -[[package]] -name = "zerofrom" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", - "synstructure", + "syn 2.0.67", ] [[package]] @@ -5106,28 +4930,6 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -[[package]] -name = "zerovec" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.66", -] - [[package]] name = "zstd" version = "0.13.1" @@ -5148,9 +4950,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.11+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" dependencies = [ "cc", "pkg-config", From 2dae3052e2716c41ee7661733dfa5520e11947e8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 22 Jun 2024 18:30:47 +0000 Subject: [PATCH 0250/2091] remove unnecessary html render Signed-off-by: Jason Volk --- src/admin/handler.rs | 92 ++++---------------------------------------- 1 file changed, 7 insertions(+), 85 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 37afd554..15039377 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,12 +1,8 @@ use clap::Parser; use conduit::trace; -use regex::Regex; -use ruma::{ - events::{ - relation::InReplyTo, - room::message::{Relation::Reply, RoomMessageEventContent}, - }, - ServerName, +use ruma::events::{ + relation::InReplyTo, + room::message::{Relation::Reply, RoomMessageEventContent}, }; extern crate conduit_service as service; @@ -17,9 +13,9 @@ use service::admin::{CommandResult, HandlerResult}; use self::{fsck::FsckCommand, tester::TesterCommands}; use crate::{ - appservice, appservice::AppserviceCommand, debug, debug::DebugCommand, escape_html, federation, - federation::FederationCommand, fsck, media, media::MediaCommand, query, query::QueryCommand, room, - room::RoomCommand, server, server::ServerCommand, services, tester, user, user::UserCommand, + appservice, appservice::AppserviceCommand, debug, debug::DebugCommand, federation, federation::FederationCommand, + fsck, media, media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, server, + server::ServerCommand, services, tester, user, user::UserCommand, }; pub(crate) const PAGE_SIZE: usize = 100; @@ -93,9 +89,7 @@ async fn process_admin_message(msg: String) -> RoomMessageEventContent { Err(error) => { let server_name = services().globals.server_name(); let message = error.replace("server.name", server_name.as_str()); - let html_message = usage_to_html(&message, server_name); - - return RoomMessageEventContent::text_html(message, html_message); + return RoomMessageEventContent::notice_markdown(message); }, }; @@ -173,75 +167,3 @@ async fn process_admin_command(command: AdminCommand, body: Vec<&str>) -> Result Ok(reply_message_content) } - -// Utility to turn clap's `--help` text to HTML. -fn usage_to_html(text: &str, server_name: &ServerName) -> String { - // Replace `@conduit:servername:-subcmdname` with `@conduit:servername: - // subcmdname` - let text = text.replace(&format!("@conduit:{server_name}:-"), &format!("@conduit:{server_name}: ")); - - // For the conduit admin room, subcommands become main commands - let text = text.replace("SUBCOMMAND", "COMMAND"); - let text = text.replace("subcommand", "command"); - - // Escape option names (e.g. ``) since they look like HTML tags - let text = escape_html(&text); - - // Italicize the first line (command name and version text) - let re = Regex::new("^(.*?)\n").expect("Regex compilation should not fail"); - let text = re.replace_all(&text, "$1\n"); - - // Unmerge wrapped lines - let text = text.replace("\n ", " "); - - // Wrap option names in backticks. The lines look like: - // -V, --version Prints version information - // And are converted to: - // -V, --version: Prints version information - // (?m) enables multi-line mode for ^ and $ - let re = Regex::new("(?m)^ {4}(([a-zA-Z_&;-]+(, )?)+) +(.*)$").expect("Regex compilation should not fail"); - let text = re.replace_all(&text, "$1: $4"); - - // Look for a `[commandbody]` tag. If it exists, use all lines below it that - // start with a `#` in the USAGE section. - let mut text_lines = text.lines().collect::>(); - let mut command_body = String::new(); - - if let Some(line_index) = text_lines.iter().position(|line| *line == "[commandbody]") { - text_lines.remove(line_index); - - while text_lines - .get(line_index) - .is_some_and(|line| line.starts_with('#')) - { - command_body += if text_lines[line_index].starts_with("# ") { - &text_lines[line_index][2..] - } else { - &text_lines[line_index][1..] - }; - command_body += "[nobr]\n"; - text_lines.remove(line_index); - } - } - - let text = text_lines.join("\n"); - - // Improve the usage section - let text = if command_body.is_empty() { - // Wrap the usage line in code tags - let re = Regex::new("(?m)^USAGE:\n {4}(@conduit:.*)$").expect("Regex compilation should not fail"); - re.replace_all(&text, "USAGE:\n$1").to_string() - } else { - // Wrap the usage line in a code block, and add a yaml block example - // This makes the usage of e.g. `register-appservice` more accurate - let re = Regex::new("(?m)^USAGE:\n {4}(.*?)\n\n").expect("Regex compilation should not fail"); - re.replace_all(&text, "USAGE:\n
    $1[nobr]\n[commandbodyblock]
    ") - .replace("[commandbodyblock]", &command_body) - }; - - // Add HTML line-breaks - - text.replace("\n\n\n", "\n\n") - .replace('\n', "
    \n") - .replace("[nobr]
    ", "") -} From bc093e95448d12508646c8fb04d7176079ab3163 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 22 Jun 2024 17:56:34 -0400 Subject: [PATCH 0251/2091] bump conduwuit version to 0.4.3 Signed-off-by: strawberry --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f6f849dd..de91f932 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -582,7 +582,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" -version = "0.4.2" +version = "0.4.3" dependencies = [ "clap", "conduit_admin", @@ -610,7 +610,7 @@ dependencies = [ [[package]] name = "conduit_admin" -version = "0.4.2" +version = "0.4.3" dependencies = [ "clap", "conduit_api", @@ -633,7 +633,7 @@ dependencies = [ [[package]] name = "conduit_api" -version = "0.4.2" +version = "0.4.3" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -666,7 +666,7 @@ dependencies = [ [[package]] name = "conduit_core" -version = "0.4.2" +version = "0.4.3" dependencies = [ "argon2", "axum 0.7.5", @@ -708,7 +708,7 @@ dependencies = [ [[package]] name = "conduit_database" -version = "0.4.2" +version = "0.4.3" dependencies = [ "chrono", "conduit_core", @@ -725,7 +725,7 @@ dependencies = [ [[package]] name = "conduit_router" -version = "0.4.2" +version = "0.4.3" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -758,7 +758,7 @@ dependencies = [ [[package]] name = "conduit_service" -version = "0.4.2" +version = "0.4.3" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 0db139ad..45cc8cf7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" rust-version = "1.77.0" -version = "0.4.2" +version = "0.4.3" [workspace.metadata.crane] name = "conduit" From 189688994bcb64c7ddc63c31f3fc1ea207411dc4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 22 Jun 2024 18:35:30 -0400 Subject: [PATCH 0252/2091] ci: properly exclude renovate from docker publishing Signed-off-by: strawberry --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a23ae428..0eddce4d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -267,7 +267,7 @@ jobs: name: Docker publish runs-on: ubuntu-latest needs: build - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.actor != 'renovate' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate' env: DOCKER_ARM64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 DOCKER_AMD64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 From 7a71012589c21213954a1c904feb1f010efdf3a2 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Tue, 18 Jun 2024 02:45:03 +0000 Subject: [PATCH 0253/2091] chore(deps): update aquasecurity/trivy-action action to v0.23.0 --- .github/workflows/trivy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 631e010f..e9cc9a30 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@v4 - name: Run Trivy code and vulnerability scanner on repo - uses: aquasecurity/trivy-action@0.22.0 + uses: aquasecurity/trivy-action@0.23.0 with: scan-type: repo format: sarif @@ -34,7 +34,7 @@ jobs: severity: CRITICAL,HIGH,MEDIUM,LOW - name: Run Trivy code and vulnerability scanner on filesystem - uses: aquasecurity/trivy-action@0.22.0 + uses: aquasecurity/trivy-action@0.23.0 with: scan-type: fs format: sarif From d875e0c1c00d8e42b83dd562c3945c077898746d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 04:33:50 +0000 Subject: [PATCH 0254/2091] fix handling of empty admin command lines Signed-off-by: Jason Volk --- src/admin/handler.rs | 15 +++++++++------ src/service/admin/console.rs | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 15039377..8a767cb8 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -9,7 +9,7 @@ extern crate conduit_service as service; use conduit::Result; pub(crate) use service::admin::{Command, Service}; -use service::admin::{CommandResult, HandlerResult}; +use service::admin::{CommandOutput, CommandResult, HandlerResult}; use self::{fsck::FsckCommand, tester::TesterCommands}; use crate::{ @@ -68,7 +68,10 @@ pub fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(comma #[tracing::instrument(skip_all, name = "admin")] async fn handle_command(command: Command) -> CommandResult { - let mut content = process_admin_message(command.command).await; + let Some(mut content) = process_admin_message(command.command).await else { + return Ok(None); + }; + content.relates_to = command.reply_id.map(|event_id| Reply { in_reply_to: InReplyTo { event_id, @@ -79,7 +82,7 @@ async fn handle_command(command: Command) -> CommandResult { } // Parse and process a message from the admin room -async fn process_admin_message(msg: String) -> RoomMessageEventContent { +async fn process_admin_message(msg: String) -> CommandOutput { let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); let command_line = lines.next().expect("each string has at least one line"); let body = lines.collect::>(); @@ -89,15 +92,15 @@ async fn process_admin_message(msg: String) -> RoomMessageEventContent { Err(error) => { let server_name = services().globals.server_name(); let message = error.replace("server.name", server_name.as_str()); - return RoomMessageEventContent::notice_markdown(message); + return Some(RoomMessageEventContent::notice_markdown(message)); }, }; match process_admin_command(admin_command, body).await { - Ok(reply_message) => reply_message, + Ok(reply_message) => Some(reply_message), Err(error) => { let markdown_message = format!("Encountered an error while handling the command:\n```\n{error}\n```",); - RoomMessageEventContent::notice_markdown(markdown_message) + Some(RoomMessageEventContent::notice_markdown(markdown_message)) }, } } diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 31432cda..c0562c52 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -138,7 +138,7 @@ impl Console { #[allow(clippy::let_underscore_must_use)] async fn handle(self: Arc, line: String) { - if line.is_empty() { + if line.trim().is_empty() { return; } From 6f9d7b5667c9d4ae62712d4c8e6622c97bcf93c8 Mon Sep 17 00:00:00 2001 From: Integral Date: Mon, 24 Jun 2024 00:08:54 +0800 Subject: [PATCH 0255/2091] docs: fix broken link to development.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2a32c0ec..d6e23912 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing guide -This page is for about contributing to conduwuit. The [development](docs/development.md) page may be of interest for you as well. +This page is for about contributing to conduwuit. The [development](development.md) page may be of interest for you as well. If you would like to work on an [issue][issues] that is not assigned, preferably ask in the Matrix room first at [#conduwuit:puppygock.gay][conduwuit-matrix], and comment on it. From 0a28d80cf25ee99aa64990fd5080d214993ade84 Mon Sep 17 00:00:00 2001 From: morguldir Date: Thu, 27 Jun 2024 04:08:04 +0200 Subject: [PATCH 0256/2091] ci: try another upstream cache Signed-off-by: morguldir --- .github/workflows/ci.yml | 4 ++-- .github/workflows/documentation.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0eddce4d..905a1b45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: uses: DeterminateSystems/magic-nix-cache-action@main with: diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" + upstream-cache: "https://cache.nixos.org" - name: Apply Nix binary cache configuration run: | @@ -196,7 +196,7 @@ jobs: uses: DeterminateSystems/magic-nix-cache-action@main with: diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" + upstream-cache: "https://cache.nixos.org" - name: Apply Nix binary cache configuration run: | diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 81a9d687..baeca35c 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -69,7 +69,7 @@ jobs: uses: DeterminateSystems/magic-nix-cache-action@main with: diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" + upstream-cache: "https://cache.nixos.org" - name: Apply Nix binary cache configuration run: | From 563618cc9ed7bed63d51e0d5548d7b7f1203cee3 Mon Sep 17 00:00:00 2001 From: morguldir Date: Thu, 27 Jun 2024 12:48:14 +0200 Subject: [PATCH 0257/2091] ci: free disk space before starting tests --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 905a1b45..b96e58dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,8 @@ jobs: name: Test runs-on: ubuntu-latest steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main - name: Sync repository uses: actions/checkout@v4 From 5d138a7e5ed30c8e6e1e54c34580716e3451c1e6 Mon Sep 17 00:00:00 2001 From: morguldir Date: Thu, 27 Jun 2024 18:36:35 +0200 Subject: [PATCH 0258/2091] ci: skip development.md file when running lychee --- engage.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engage.toml b/engage.toml index 39da78a8..41c4a649 100644 --- a/engage.toml +++ b/engage.toml @@ -136,7 +136,7 @@ cargo clippy \ [[task]] name = "lychee" group = "lints" -script = "lychee --verbose --offline docs *.md" +script = "lychee --verbose --offline docs *.md --exclude development.md" [[task]] name = "cargo/all" From fb758f5f4bc8ceefbedc25f71b26e013df21d626 Mon Sep 17 00:00:00 2001 From: morguldir Date: Thu, 27 Jun 2024 18:37:44 +0200 Subject: [PATCH 0259/2091] chore(deps): bump Cargo.lock (subtle crate was yanked) --- Cargo.lock | 94 +++++++++++++++++++++++++++--------------------------- 1 file changed, 47 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index de91f932..30f46a50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,7 +130,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -141,7 +141,7 @@ checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -378,7 +378,7 @@ version = "0.69.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -389,7 +389,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -400,9 +400,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "blake2" @@ -486,9 +486,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.99" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" +checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" dependencies = [ "jobserver", "libc", @@ -565,7 +565,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -973,7 +973,7 @@ version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "crossterm_winapi", "futures-core", "libc", @@ -1027,7 +1027,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -1115,9 +1115,9 @@ dependencies = [ [[package]] name = "either" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" dependencies = [ "serde", ] @@ -1131,7 +1131,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -1279,7 +1279,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -1572,7 +1572,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -1972,7 +1972,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -1995,9 +1995,9 @@ checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", "windows-targets 0.52.5", @@ -2191,7 +2191,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "cfg-if", "cfg_aliases", "libc", @@ -2491,7 +2491,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2584,7 +2584,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2672,7 +2672,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "version_check", "yansi", ] @@ -2697,7 +2697,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -2715,7 +2715,7 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "memchr", "pulldown-cmark-escape", "unicase", @@ -2825,7 +2825,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -3096,7 +3096,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.67", + "syn 2.0.68", "toml", ] @@ -3148,7 +3148,7 @@ name = "rusqlite" version = "0.31.0" source = "git+https://github.com/rusqlite/rusqlite?rev=e00b626e2b1c67347d789fb7f600281705c89381#e00b626e2b1c67347d789fb7f600281705c89381" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -3376,7 +3376,7 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -3551,7 +3551,7 @@ checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -3569,9 +3569,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.117" +version = "1.0.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" +checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" dependencies = [ "itoa", "ryu", @@ -3827,9 +3827,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.6.0" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d0208408ba0c3df17ed26eb06992cb1a1268d41b2c0e12e65203fbe3972cee5" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -3844,9 +3844,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.67" +version = "2.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff8655ed1d86f3af4ee3fd3263786bc14245ad17c4c7e85ba7187fb3ae028c90" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" dependencies = [ "proc-macro2", "quote", @@ -3919,7 +3919,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -4018,9 +4018,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" dependencies = [ "tinyvec_macros", ] @@ -4068,7 +4068,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -4238,7 +4238,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" dependencies = [ "async-compression", - "bitflags 2.5.0", + "bitflags 2.6.0", "bytes", "futures-core", "futures-util", @@ -4284,7 +4284,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390 dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] @@ -4493,9 +4493,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" -version = "1.8.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" dependencies = [ "getrandom", "serde", @@ -4555,7 +4555,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "wasm-bindgen-shared", ] @@ -4589,7 +4589,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4921,7 +4921,7 @@ checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.67", + "syn 2.0.68", ] [[package]] From 624cd2acfae8af654f9dd0cfcb919b078a20b1e7 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 19:11:15 -0400 Subject: [PATCH 0260/2091] remove sqlite code --- src/core/error.rs | 10 -- src/database/kvdatabase.rs | 20 +-- src/database/mod.rs | 5 +- src/database/sqlite/mod.rs | 337 ------------------------------------- 4 files changed, 2 insertions(+), 370 deletions(-) delete mode 100644 src/database/sqlite/mod.rs diff --git a/src/core/error.rs b/src/core/error.rs index 5847fb38..4fafd1e3 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -25,12 +25,6 @@ pub type Result = std::result::Result; #[derive(Error)] pub enum Error { - #[cfg(feature = "sqlite")] - #[error("There was a problem with the connection to the sqlite database: {source}")] - Sqlite { - #[from] - source: rusqlite::Error, - }, #[cfg(feature = "rocksdb")] #[error("There was a problem with the connection to the rocksdb database: {source}")] RocksDb { @@ -120,10 +114,6 @@ impl Error { let db_error = String::from("Database or I/O error occurred."); match self { - #[cfg(feature = "sqlite")] - Self::Sqlite { - .. - } => db_error, #[cfg(feature = "rocksdb")] Self::RocksDb { .. diff --git a/src/database/kvdatabase.rs b/src/database/kvdatabase.rs index ef34fec4..4f5fbacd 100644 --- a/src/database/kvdatabase.rs +++ b/src/database/kvdatabase.rs @@ -274,13 +274,6 @@ impl KeyValueDatabase { fn build(config: &Config) -> Result> { match &*config.database_backend { - "sqlite" => { - debug!("Got sqlite database backend"); - #[cfg(not(feature = "sqlite"))] - return Err(Error::bad_config("Database backend not found.")); - #[cfg(feature = "sqlite")] - Ok(Arc::new(Arc::::open(config)?)) - }, "rocksdb" => { debug!("Got rocksdb database backend"); #[cfg(not(feature = "rocksdb"))] @@ -289,7 +282,7 @@ fn build(config: &Config) -> Result> { Ok(Arc::new(Arc::::open(config)?)) }, _ => Err(Error::bad_config( - "Database backend not found. sqlite (not recommended) and rocksdb are the only supported backends.", + "Database backend not found. rocksdb is the only supported backend.", )), } } @@ -297,19 +290,8 @@ fn build(config: &Config) -> Result> { fn check_db_setup(config: &Config) -> Result<()> { let path = Path::new(&config.database_path); - let sqlite_exists = path.join("conduit.db").exists(); let rocksdb_exists = path.join("IDENTITY").exists(); - if sqlite_exists && rocksdb_exists { - return Err(Error::bad_config("Multiple databases at database_path detected.")); - } - - if sqlite_exists && config.database_backend != "sqlite" { - return Err(Error::bad_config( - "Found sqlite at database_path, but is not specified in config.", - )); - } - if rocksdb_exists && config.database_backend != "rocksdb" { return Err(Error::bad_config( "Found rocksdb at database_path, but is not specified in config.", diff --git a/src/database/mod.rs b/src/database/mod.rs index 506f7ac1..68b9b47a 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -6,10 +6,7 @@ mod kvtree; #[cfg(feature = "rocksdb")] pub(crate) mod rocksdb; -#[cfg(feature = "sqlite")] -pub(crate) mod sqlite; - -#[cfg(any(feature = "sqlite", feature = "rocksdb"))] +#[cfg(any(feature = "rocksdb"))] pub(crate) mod watchers; extern crate conduit_core as conduit; diff --git a/src/database/sqlite/mod.rs b/src/database/sqlite/mod.rs deleted file mode 100644 index 562d2a10..00000000 --- a/src/database/sqlite/mod.rs +++ /dev/null @@ -1,337 +0,0 @@ -use std::{ - cell::RefCell, - future::Future, - path::{Path, PathBuf}, - pin::Pin, - sync::Arc, -}; - -use conduit::{Config, Result}; -use parking_lot::{Mutex, MutexGuard}; -use rusqlite::{Connection, DatabaseName::Main, OptionalExtension}; -use thread_local::ThreadLocal; -use tracing::debug; - -use super::{watchers::Watchers, KeyValueDatabaseEngine, KvTree}; - -thread_local! { - static READ_CONNECTION: RefCell> = const { RefCell::new(None) }; - static READ_CONNECTION_ITERATOR: RefCell> = const { RefCell::new(None) }; -} - -struct PreparedStatementIterator<'a> { - iterator: Box + 'a>, - _statement_ref: AliasableBox>, -} - -impl Iterator for PreparedStatementIterator<'_> { - type Item = TupleOfBytes; - - fn next(&mut self) -> Option { self.iterator.next() } -} - -struct AliasableBox(*mut T); -impl Drop for AliasableBox { - fn drop(&mut self) { - // SAFETY: This is cursed and relies on non-local reasoning. - // - // In order for this to be safe: - // - // * All aliased references to this value must have been dropped first, for - // example by coming after its referrers in struct fields, because struct - // fields are automatically dropped in order from top to bottom in the absence - // of an explicit Drop impl. Otherwise, the referrers may read into - // deallocated memory. - // * This type must not be copyable or cloneable. Otherwise, double-free can - // occur. - // - // These conditions are met, but again, note that changing safe code in - // this module can result in unsoundness if any of these constraints are - // violated. - unsafe { drop(Box::from_raw(self.0)) } - } -} - -pub(crate) struct Engine { - writer: Mutex, - read_conn_tls: ThreadLocal, - read_iterator_conn_tls: ThreadLocal, - - path: PathBuf, - cache_size_per_thread: u32, -} - -impl Engine { - fn prepare_conn(path: &Path, cache_size_kb: u32) -> Result { - let conn = Connection::open(path)?; - - conn.pragma_update(Some(Main), "page_size", 2048)?; - conn.pragma_update(Some(Main), "journal_mode", "WAL")?; - conn.pragma_update(Some(Main), "synchronous", "NORMAL")?; - conn.pragma_update(Some(Main), "cache_size", -i64::from(cache_size_kb))?; - conn.pragma_update(Some(Main), "wal_autocheckpoint", 0)?; - - Ok(conn) - } - - fn write_lock(&self) -> MutexGuard<'_, Connection> { self.writer.lock() } - - fn read_lock(&self) -> &Connection { - self.read_conn_tls - .get_or(|| Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap()) - } - - fn read_lock_iterator(&self) -> &Connection { - self.read_iterator_conn_tls - .get_or(|| Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap()) - } - - fn flush_wal(self: &Arc) -> Result<()> { - self.write_lock() - .pragma_update(Some(Main), "wal_checkpoint", "RESTART")?; - Ok(()) - } -} - -impl KeyValueDatabaseEngine for Arc { - fn open(config: &Config) -> Result { - let path = Path::new(&config.database_path).join("conduit.db"); - - // calculates cache-size per permanent connection - // 1. convert MB to KiB - // 2. divide by permanent connections + permanent iter connections + write - // connection - // 3. round down to nearest integer - #[allow( - clippy::as_conversions, - clippy::cast_possible_truncation, - clippy::cast_precision_loss, - clippy::cast_sign_loss - )] - let cache_size_per_thread = ((config.db_cache_capacity_mb * 1024.0) - / (conduit::utils::available_parallelism() as f64).mul_add(2.0, 1.0)) as u32; - - let writer = Mutex::new(Engine::prepare_conn(&path, cache_size_per_thread)?); - - let arc = Self::new(Engine { - writer, - read_conn_tls: ThreadLocal::new(), - read_iterator_conn_tls: ThreadLocal::new(), - path, - cache_size_per_thread, - }); - - Ok(arc) - } - - fn open_tree(&self, name: &str) -> Result> { - self.write_lock().execute( - &format!("CREATE TABLE IF NOT EXISTS {name} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )"), - [], - )?; - - Ok(Arc::new(SqliteTable { - engine: Self::clone(self), - name: name.to_owned(), - watchers: Watchers::default(), - })) - } - - fn flush(&self) -> Result<()> { - // we enabled PRAGMA synchronous=normal, so this should not be necessary - Ok(()) - } - - fn cleanup(&self) -> Result<()> { self.flush_wal() } -} - -struct SqliteTable { - engine: Arc, - name: String, - watchers: Watchers, -} - -type TupleOfBytes = (Vec, Vec); - -impl SqliteTable { - fn get_with_guard(&self, guard: &Connection, key: &[u8]) -> Result>> { - Ok(guard - .prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())? - .query_row([key], |row| row.get(0)) - .optional()?) - } - - fn insert_with_guard(&self, guard: &Connection, key: &[u8], value: &[u8]) -> Result<()> { - guard.execute( - format!("INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)", self.name).as_str(), - [key, value], - )?; - Ok(()) - } - - fn iter_with_guard<'a>(&'a self, guard: &'a Connection) -> Box + 'a> { - let statement = Box::leak(Box::new( - guard - .prepare(&format!("SELECT key, value FROM {} ORDER BY key ASC", &self.name)) - .unwrap(), - )); - - let statement_ref = AliasableBox(statement); - - //let name = self.name.clone(); - - let iterator = Box::new( - statement - .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) - .unwrap() - .map(Result::unwrap), - ); - - Box::new(PreparedStatementIterator { - iterator, - _statement_ref: statement_ref, - }) - } -} - -impl KvTree for SqliteTable { - fn get(&self, key: &[u8]) -> Result>> { self.get_with_guard(self.engine.read_lock(), key) } - - fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { - let guard = self.engine.write_lock(); - self.insert_with_guard(&guard, key, value)?; - drop(guard); - self.watchers.wake(key); - Ok(()) - } - - fn insert_batch(&self, iter: &mut dyn Iterator, Vec)>) -> Result<()> { - let guard = self.engine.write_lock(); - - guard.execute("BEGIN", [])?; - for (key, value) in iter { - self.insert_with_guard(&guard, &key, &value)?; - } - guard.execute("COMMIT", [])?; - - drop(guard); - - Ok(()) - } - - fn increment_batch(&self, iter: &mut dyn Iterator>) -> Result<()> { - let guard = self.engine.write_lock(); - - guard.execute("BEGIN", [])?; - for key in iter { - let old = self.get_with_guard(&guard, &key)?; - let new = conduit::utils::increment(old.as_deref()); - self.insert_with_guard(&guard, &key, &new)?; - } - guard.execute("COMMIT", [])?; - - drop(guard); - - Ok(()) - } - - fn remove(&self, key: &[u8]) -> Result<()> { - let guard = self.engine.write_lock(); - - guard.execute(format!("DELETE FROM {} WHERE key = ?", self.name).as_str(), [key])?; - - Ok(()) - } - - fn iter<'a>(&'a self) -> Box + 'a> { - let guard = self.engine.read_lock_iterator(); - - self.iter_with_guard(guard) - } - - fn iter_from<'a>(&'a self, from: &[u8], backwards: bool) -> Box + 'a> { - let guard = self.engine.read_lock_iterator(); - let from = from.to_vec(); // TODO change interface? - - //let name = self.name.clone(); - - if backwards { - let statement = Box::leak(Box::new( - guard - .prepare(&format!( - "SELECT key, value FROM {} WHERE key <= ? ORDER BY key DESC", - &self.name - )) - .unwrap(), - )); - - let statement_ref = AliasableBox(statement); - - let iterator = Box::new( - statement - .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) - .unwrap() - .map(Result::unwrap), - ); - Box::new(PreparedStatementIterator { - iterator, - _statement_ref: statement_ref, - }) - } else { - let statement = Box::leak(Box::new( - guard - .prepare(&format!( - "SELECT key, value FROM {} WHERE key >= ? ORDER BY key ASC", - &self.name - )) - .unwrap(), - )); - - let statement_ref = AliasableBox(statement); - - let iterator = Box::new( - statement - .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1)))) - .unwrap() - .map(Result::unwrap), - ); - - Box::new(PreparedStatementIterator { - iterator, - _statement_ref: statement_ref, - }) - } - } - - fn increment(&self, key: &[u8]) -> Result> { - let guard = self.engine.write_lock(); - - let old = self.get_with_guard(&guard, key)?; - - let new = conduit::utils::increment(old.as_deref()); - - self.insert_with_guard(&guard, key, &new)?; - - Ok(new) - } - - fn scan_prefix<'a>(&'a self, prefix: Vec) -> Box + 'a> { - Box::new( - self.iter_from(&prefix, false) - .take_while(move |(key, _)| key.starts_with(&prefix)), - ) - } - - fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { - self.watchers.watch(prefix) - } - - fn clear(&self) -> Result<()> { - debug!("clear: running"); - self.engine - .write_lock() - .execute(format!("DELETE FROM {}", self.name).as_str(), [])?; - debug!("clear: ran"); - Ok(()) - } -} From 68f1b4789aab5004eafc9d071e9f566e2bea43e4 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 19:20:31 -0400 Subject: [PATCH 0261/2091] drop the sqlite feature from main --- src/main/Cargo.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index bc3e3951..97575c68 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -89,7 +89,7 @@ io_uring = [ jemalloc = [ "conduit-admin/jemalloc", "conduit-core/jemalloc", - "conduit-database/jemalloc", + "conduit-database/jemalloc", ] jemalloc_prof = [ "conduit-core/jemalloc_prof", @@ -132,10 +132,6 @@ sentry_telemetry = [ sha256_media = [ "conduit-service/sha256_media", ] -sqlite = [ - "conduit-core/sqlite", - "conduit-database/sqlite", -] systemd = [ "conduit-router/systemd", ] From 325f1c0bd2c1a61f70364364a1f5139f041ca781 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 19:20:54 -0400 Subject: [PATCH 0262/2091] drop the sqlite feature from database --- Cargo.lock | 1 - src/database/Cargo.toml | 7 ------- 2 files changed, 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 30f46a50..ece610f3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -716,7 +716,6 @@ dependencies = [ "lru-cache", "parking_lot", "ruma", - "rusqlite", "rust-rocksdb-uwu", "thread_local", "tokio", diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 18e073ed..6455fc65 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -24,11 +24,6 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] -sqlite = [ - "dep:rusqlite", - "dep:parking_lot", - "dep:thread_local", -] rocksdb = [ "dep:rust-rocksdb", ] @@ -50,8 +45,6 @@ lru-cache.workspace = true parking_lot.optional = true parking_lot.workspace = true ruma.workspace = true -rusqlite.optional = true -rusqlite.workspace = true rust-rocksdb.optional = true rust-rocksdb.workspace = true thread_local.optional = true From 65177d237a25a425e2049391f802e19570dd2bb4 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 19:21:23 -0400 Subject: [PATCH 0263/2091] drop the sqlite feature from core --- Cargo.lock | 1 - src/core/Cargo.toml | 7 ------- 2 files changed, 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ece610f3..8b1b052b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -688,7 +688,6 @@ dependencies = [ "reqwest", "ring", "ruma", - "rusqlite", "rust-rocksdb-uwu", "sanitize-filename", "serde", diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 4a694d90..420897b6 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -24,11 +24,6 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] -sqlite = [ - "dep:rusqlite", - "dep:parking_lot", - "dep:thread_local", -] rocksdb = [ "dep:rust-rocksdb", ] @@ -79,8 +74,6 @@ regex.workspace = true reqwest.workspace = true ring.workspace = true ruma.workspace = true -rusqlite.optional = true -rusqlite.workspace = true rust-rocksdb.optional = true rust-rocksdb.workspace = true sanitize-filename.workspace = true From c61258fa0d524cfa319f7e90e0db036d5012a8cf Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 19:25:35 -0400 Subject: [PATCH 0264/2091] drop rusqlite dependency --- Cargo.toml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 45cc8cf7..3f7c1333 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -378,12 +378,6 @@ version = "0.5.4" default-features = false features = ["use_std"] -[workspace.dependencies.rusqlite] -git = "https://github.com/rusqlite/rusqlite" -#branch = "master" -rev = "e00b626e2b1c67347d789fb7f600281705c89381" -features = ["bundled"] - # used only by rusqlite [workspace.dependencies.parking_lot] version = "0.12.3" From e24b7d2f3e6328657e56146fe531f3a3fdca8016 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 19:25:44 -0400 Subject: [PATCH 0265/2091] drop sqlite from docs --- conduwuit-example.toml | 4 +--- docs/deploying/generic.md | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 97652eee..37aaf2fe 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -64,9 +64,7 @@ # Note: this was previously "/var/lib/matrix-conduit" database_path = "/var/lib/conduwuit" -# Database backend: Only rocksdb and sqlite are supported. Please note that sqlite -# will perform significantly worse than rocksdb as it is not intended to be used the -# way it is by conduwuit. sqlite only exists for historical reasons. +# Database backend: Only rocksdb is supported. database_backend = "rocksdb" diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 31f61f6a..28918980 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -49,7 +49,7 @@ The systemd unit for conduwuit can be found [here](../configuration.md#example-s Now we need to create the conduwuit's config file in `/etc/conduwuit/conduwuit.toml`. The example config can be found at [conduwuit-example.toml](../configuration.md).**Please take a moment to read it. You need to change at least the server name.** -RocksDB is the only supported database backend. SQLite only exists for historical reasons, is not recommended, and will be removed soon (likely in v0.5.0). Any performance issues, storage issues, database issues, etc will not be assisted if using SQLite and you will be asked to migrate to RocksDB first. +RocksDB is the only supported database backend. ## Setting the correct file permissions From 5e6b9ec1cfcef76e998e158424d2a33384c75305 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 19:26:57 -0400 Subject: [PATCH 0266/2091] drop thread_local feature --- Cargo.lock | 2 -- Cargo.toml | 4 ---- src/core/Cargo.toml | 2 -- src/database/Cargo.toml | 2 -- 4 files changed, 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b1b052b..4854d01d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -694,7 +694,6 @@ dependencies = [ "serde_json", "serde_regex", "thiserror", - "thread_local", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -716,7 +715,6 @@ dependencies = [ "parking_lot", "ruma", "rust-rocksdb-uwu", - "thread_local", "tokio", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index 3f7c1333..446acb82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -382,10 +382,6 @@ features = ["use_std"] [workspace.dependencies.parking_lot] version = "0.12.3" -# used only by rusqlite -[workspace.dependencies.thread_local] -version = "1.1.8" - [workspace.dependencies.tokio-metrics] version = "0.3.1" default-features = false diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 420897b6..884d908e 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -81,8 +81,6 @@ serde_json.workspace = true serde_regex.workspace = true serde.workspace = true thiserror.workspace = true -thread_local.optional = true -thread_local.workspace = true tikv-jemallocator.optional = true tikv-jemallocator.workspace = true tikv-jemalloc-ctl.optional = true diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 6455fc65..547cdc38 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -47,8 +47,6 @@ parking_lot.workspace = true ruma.workspace = true rust-rocksdb.optional = true rust-rocksdb.workspace = true -thread_local.optional = true -thread_local.workspace = true tokio.workspace = true tracing.workspace = true From 73b85372d64f8e9562d88ef66562d11e7e002dc3 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 19:27:48 -0400 Subject: [PATCH 0267/2091] drop parking_lot feature --- Cargo.lock | 2 -- Cargo.toml | 4 ---- src/core/Cargo.toml | 2 -- src/database/Cargo.toml | 2 -- 4 files changed, 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4854d01d..b2871b85 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -682,7 +682,6 @@ dependencies = [ "libloading", "log", "nix", - "parking_lot", "rand", "regex", "reqwest", @@ -712,7 +711,6 @@ dependencies = [ "conduit_core", "log", "lru-cache", - "parking_lot", "ruma", "rust-rocksdb-uwu", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 446acb82..d9329273 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -378,10 +378,6 @@ version = "0.5.4" default-features = false features = ["use_std"] -# used only by rusqlite -[workspace.dependencies.parking_lot] -version = "0.12.3" - [workspace.dependencies.tokio-metrics] version = "0.3.1" default-features = false diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 884d908e..dc3fc1d4 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -67,8 +67,6 @@ ipaddress.workspace = true itertools.workspace = true libloading.workspace = true log.workspace = true -parking_lot.optional = true -parking_lot.workspace = true rand.workspace = true regex.workspace = true reqwest.workspace = true diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 547cdc38..1bc382a0 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -42,8 +42,6 @@ chrono.workspace = true conduit-core.workspace = true log.workspace = true lru-cache.workspace = true -parking_lot.optional = true -parking_lot.workspace = true ruma.workspace = true rust-rocksdb.optional = true rust-rocksdb.workspace = true From 8d5305c769c77a4ea6fb1b102310cfc8ada395cf Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Sun, 30 Jun 2024 10:40:35 -0400 Subject: [PATCH 0268/2091] lock cleanup --- Cargo.lock | 79 ------------------------------------------------------ 1 file changed, 79 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2871b85..6333c49e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,18 +17,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "ahash" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" -dependencies = [ - "cfg-if", - "once_cell", - "version_check", - "zerocopy", -] - [[package]] name = "aho-corasick" version = "1.1.3" @@ -1134,18 +1122,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" -[[package]] -name = "fallible-iterator" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - [[package]] name = "fdeflate" version = "0.3.4" @@ -1405,18 +1381,6 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", -] - -[[package]] -name = "hashlink" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" -dependencies = [ - "hashbrown 0.14.5", -] [[package]] name = "hdrhistogram" @@ -1997,16 +1961,6 @@ dependencies = [ "windows-targets 0.52.5", ] -[[package]] -name = "libsqlite3-sys" -version = "0.28.0" -source = "git+https://github.com/rusqlite/rusqlite?rev=e00b626e2b1c67347d789fb7f600281705c89381#e00b626e2b1c67347d789fb7f600281705c89381" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - [[package]] name = "libz-sys" version = "1.1.18" @@ -3137,19 +3091,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "rusqlite" -version = "0.31.0" -source = "git+https://github.com/rusqlite/rusqlite?rev=e00b626e2b1c67347d789fb7f600281705c89381#e00b626e2b1c67347d789fb7f600281705c89381" -dependencies = [ - "bitflags 2.6.0", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - [[package]] name = "rust-librocksdb-sys" version = "0.22.0+9.2.1" @@ -4898,26 +4839,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" -[[package]] -name = "zerocopy" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.7.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.68", -] - [[package]] name = "zeroize" version = "1.8.1" From 19d8f0b27e05d7bc88d70468f6e36b463b6ad8d8 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Sun, 30 Jun 2024 10:40:54 -0400 Subject: [PATCH 0269/2091] dont use any for a single feature check --- src/database/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/mod.rs b/src/database/mod.rs index 68b9b47a..48366e25 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -6,7 +6,7 @@ mod kvtree; #[cfg(feature = "rocksdb")] pub(crate) mod rocksdb; -#[cfg(any(feature = "rocksdb"))] +#[cfg(feature = "rocksdb")] pub(crate) mod watchers; extern crate conduit_core as conduit; From efbdced535e3be3e8505fa01e4356f46cfa9a21e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 19:55:17 +0000 Subject: [PATCH 0270/2091] cleanup migration function a bit Signed-off-by: Jason Volk --- src/service/globals/migrations.rs | 324 +++++++++++++++--------------- 1 file changed, 165 insertions(+), 159 deletions(-) diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index bc9eaaa8..591aed33 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -6,7 +6,7 @@ use std::{ sync::Arc, }; -use conduit::{debug_info, debug_warn}; +use conduit::{debug, debug_info, debug_warn, error, info, utils, warn, Config, Error, Result}; use database::KeyValueDatabase; use itertools::Itertools; use ruma::{ @@ -14,9 +14,14 @@ use ruma::{ push::Ruleset, EventId, OwnedRoomId, RoomId, UserId, }; -use tracing::{debug, error, info, warn}; -use crate::{services, utils, Config, Error, Result}; +use crate::services; + +/// The current schema version. +/// * If the database is opened at lesser version we apply migrations up to this +/// version. +/// * If the database is opened at greater version we reject with error. +const DATABASE_VERSION: u64 = 13 + cfg!(feature = "sha256_media") as u64; pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result<()> { // Matrix resource ownership is based on the server name; changing it @@ -32,114 +37,156 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result } } - // If the database has any data, perform data migrations before starting - // do not increment the db version if the user is not using sha256_media - let latest_database_version = if cfg!(feature = "sha256_media") { - 14 - } else { - 13 - }; - if services().users.count()? > 0 { - // MIGRATIONS - if services().globals.database_version()? < 1 { - db_lt_1(db, config).await?; + migrate(db, config).await + } else { + fresh(db, config).await + } +} + +async fn fresh(db: &KeyValueDatabase, config: &Config) -> Result<()> { + services().globals.bump_database_version(DATABASE_VERSION)?; + + db.global + .insert(b"fix_bad_double_separator_in_state_cache", &[])?; + db.global + .insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; + + // Create the admin room and server user on first run + crate::admin::create_admin_room().await?; + + warn!( + "Created new {} database with version {DATABASE_VERSION}", + config.database_backend, + ); + + Ok(()) +} + +/// Apply any migrations +async fn migrate(db: &KeyValueDatabase, config: &Config) -> Result<()> { + if services().globals.database_version()? < 1 { + db_lt_1(db, config).await?; + } + + if services().globals.database_version()? < 2 { + db_lt_2(db, config).await?; + } + + if services().globals.database_version()? < 3 { + db_lt_3(db, config).await?; + } + + if services().globals.database_version()? < 4 { + db_lt_4(db, config).await?; + } + + if services().globals.database_version()? < 5 { + db_lt_5(db, config).await?; + } + + if services().globals.database_version()? < 6 { + db_lt_6(db, config).await?; + } + + if services().globals.database_version()? < 7 { + db_lt_7(db, config).await?; + } + + if services().globals.database_version()? < 8 { + db_lt_8(db, config).await?; + } + + if services().globals.database_version()? < 9 { + db_lt_9(db, config).await?; + } + + if services().globals.database_version()? < 10 { + db_lt_10(db, config).await?; + } + + if services().globals.database_version()? < 11 { + db_lt_11(db, config).await?; + } + + if services().globals.database_version()? < 12 { + db_lt_12(db, config).await?; + } + + // This migration can be reused as-is anytime the server-default rules are + // updated. + if services().globals.database_version()? < 13 { + db_lt_13(db, config).await?; + } + + #[cfg(feature = "sha256_media")] + if services().globals.database_version()? < 14 { + feat_sha256_media(db, config).await?; + } + + if db + .global + .get(b"fix_bad_double_separator_in_state_cache")? + .is_none() + { + fix_bad_double_separator_in_state_cache(db, config).await?; + } + + if db + .global + .get(b"retroactively_fix_bad_data_from_roomuserid_joined")? + .is_none() + { + retroactively_fix_bad_data_from_roomuserid_joined(db, config).await?; + } + + assert_eq!( + services().globals.database_version().unwrap(), + DATABASE_VERSION, + "Failed asserting local database version {} is equal to known latest conduwuit database version {}", + services().globals.database_version().unwrap(), + DATABASE_VERSION, + ); + + { + let patterns = services().globals.forbidden_usernames(); + if !patterns.is_empty() { + for user_id in services() + .users + .iter() + .filter_map(Result::ok) + .filter(|user| !services().users.is_deactivated(user).unwrap_or(true)) + .filter(|user| user.server_name() == config.server_name) + { + let matches = patterns.matches(user_id.localpart()); + if matches.matched_any() { + warn!( + "User {} matches the following forbidden username patterns: {}", + user_id.to_string(), + matches + .into_iter() + .map(|x| &patterns.patterns()[x]) + .join(", ") + ); + } + } } + } - if services().globals.database_version()? < 2 { - db_lt_2(db, config).await?; - } - - if services().globals.database_version()? < 3 { - db_lt_3(db, config).await?; - } - - if services().globals.database_version()? < 4 { - db_lt_4(db, config).await?; - } - - if services().globals.database_version()? < 5 { - db_lt_5(db, config).await?; - } - - if services().globals.database_version()? < 6 { - db_lt_6(db, config).await?; - } - - if services().globals.database_version()? < 7 { - db_lt_7(db, config).await?; - } - - if services().globals.database_version()? < 8 { - db_lt_8(db, config).await?; - } - - if services().globals.database_version()? < 9 { - db_lt_9(db, config).await?; - } - - if services().globals.database_version()? < 10 { - db_lt_10(db, config).await?; - } - - if services().globals.database_version()? < 11 { - db_lt_11(db, config).await?; - } - - if services().globals.database_version()? < 12 { - db_lt_12(db, config).await?; - } - - // This migration can be reused as-is anytime the server-default rules are - // updated. - if services().globals.database_version()? < 13 { - db_lt_13(db, config).await?; - } - - #[cfg(feature = "sha256_media")] - if services().globals.database_version()? < 14 && cfg!(feature = "sha256_media") { - feat_sha256_media(db, config).await?; - } - - if db - .global - .get(b"fix_bad_double_separator_in_state_cache")? - .is_none() - { - fix_bad_double_separator_in_state_cache(db, config).await?; - } - - if db - .global - .get(b"retroactively_fix_bad_data_from_roomuserid_joined")? - .is_none() - { - retroactively_fix_bad_data_from_roomuserid_joined(db, config).await?; - } - - assert_eq!( - services().globals.database_version().unwrap(), - latest_database_version, - "Failed asserting local database version {} is equal to known latest conduwuit database version {}", - services().globals.database_version().unwrap(), - latest_database_version - ); - - { - let patterns = services().globals.forbidden_usernames(); - if !patterns.is_empty() { - for user_id in services() - .users - .iter() - .filter_map(Result::ok) - .filter(|user| !services().users.is_deactivated(user).unwrap_or(true)) - .filter(|user| user.server_name() == config.server_name) - { - let matches = patterns.matches(user_id.localpart()); + { + let patterns = services().globals.forbidden_alias_names(); + if !patterns.is_empty() { + for address in services().rooms.metadata.iter_ids() { + let room_id = address?; + let room_aliases = services().rooms.alias.local_aliases_for_room(&room_id); + for room_alias_result in room_aliases { + let room_alias = room_alias_result?; + let matches = patterns.matches(room_alias.alias()); if matches.matched_any() { warn!( - "User {} matches the following forbidden username patterns: {}", - user_id.to_string(), + "Room with alias {} ({}) matches the following forbidden room name patterns: {}", + room_alias, + &room_id, matches .into_iter() .map(|x| &patterns.patterns()[x]) @@ -149,55 +196,13 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result } } } - - { - let patterns = services().globals.forbidden_alias_names(); - if !patterns.is_empty() { - for address in services().rooms.metadata.iter_ids() { - let room_id = address?; - let room_aliases = services().rooms.alias.local_aliases_for_room(&room_id); - for room_alias_result in room_aliases { - let room_alias = room_alias_result?; - let matches = patterns.matches(room_alias.alias()); - if matches.matched_any() { - warn!( - "Room with alias {} ({}) matches the following forbidden room name patterns: {}", - room_alias, - &room_id, - matches - .into_iter() - .map(|x| &patterns.patterns()[x]) - .join(", ") - ); - } - } - } - } - } - - info!( - "Loaded {} database with schema version {}", - config.database_backend, latest_database_version - ); - } else { - services() - .globals - .bump_database_version(latest_database_version)?; - - db.global - .insert(b"fix_bad_double_separator_in_state_cache", &[])?; - db.global - .insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; - - // Create the admin room and server user on first run - crate::admin::create_admin_room().await?; - - warn!( - "Created new {} database with version {}", - config.database_backend, latest_database_version - ); } + info!( + "Loaded {} database with schema version {DATABASE_VERSION}", + config.database_backend, + ); + Ok(()) } @@ -686,15 +691,16 @@ async fn db_lt_13(_db: &KeyValueDatabase, config: &Config) -> Result<()> { #[cfg(feature = "sha256_media")] async fn feat_sha256_media(db: &KeyValueDatabase, _config: &Config) -> Result<()> { use std::path::PathBuf; - warn!("sha256_media feature flag is enabled, migrating legacy base64 file names to sha256 file names"); + + warn!("Mgrating legacy base64 file names to sha256 file names"); + // Move old media files to new names let mut changes = Vec::<(PathBuf, PathBuf)>::new(); for (key, _) in db.mediaid_file.iter() { - let old_path = services().globals.get_media_file(&key); - debug!("Old file path: {old_path:?}"); - let path = services().globals.get_media_file_new(&key); - debug!("New file path: {path:?}"); - changes.push((old_path, path)); + let old = services().globals.get_media_file(&key); + let new = services().globals.get_media_file_new(&key); + changes.push((old, new)); + debug!(?old, ?new, num = changes.len(), "change"); } // move the file to the new location for (old_path, path) in changes { From 3480074f61d27bf615d5af79b355f004b3c2ab61 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 20:48:40 +0000 Subject: [PATCH 0271/2091] sha256_media integration Signed-off-by: Jason Volk --- src/core/config/mod.rs | 7 + src/main/Cargo.toml | 3 - src/service/Cargo.toml | 4 - src/service/globals/migrations.rs | 123 ++++++++++-- src/service/globals/mod.rs | 38 ---- src/service/media/mod.rs | 316 ++++++++++-------------------- src/service/media/tests.rs | 84 ++++++++ src/service/services.rs | 6 +- 8 files changed, 299 insertions(+), 282 deletions(-) create mode 100644 src/service/media/tests.rs diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 58696f88..bbf44a1a 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -300,8 +300,13 @@ pub struct Config { #[serde(default)] pub allow_guests_auto_join_rooms: bool, + #[serde(default = "true_fn")] + pub media_startup_check: bool, + #[serde(default = "true_fn")] + pub media_compat_file_link: bool, #[serde(default = "Vec::new")] pub prevent_media_downloads_from: Vec, + #[serde(default = "Vec::new")] pub forbidden_remote_server_names: Vec, #[serde(default = "Vec::new")] @@ -758,6 +763,8 @@ impl fmt::Display for Config { "RocksDB Compaction Idle IOPriority", &self.rocksdb_compaction_ioprio_idle.to_string(), ), + ("Media integrity checks on startup", &self.media_startup_check.to_string()), + ("Media compatibility filesystem links", &self.media_compat_file_link.to_string()), ("Prevent Media Downloads From", { let mut lst = vec![]; for domain in &self.prevent_media_downloads_from { diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 97575c68..9ef51545 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -129,9 +129,6 @@ sentry_telemetry = [ "conduit-core/sentry_telemetry", "conduit-router/sentry_telemetry", ] -sha256_media = [ - "conduit-service/sha256_media", -] systemd = [ "conduit-router/systemd", ] diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 80335181..0151b823 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -35,9 +35,6 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] -sha256_media = [ - "dep:sha2", -] [dependencies] async-trait.workspace = true @@ -66,7 +63,6 @@ rustyline-async.optional = true serde_json.workspace = true serde.workspace = true serde_yaml.workspace = true -sha2.optional = true sha2.workspace = true termimad.workspace = true termimad.optional = true diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 591aed33..a281ce2f 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -1,9 +1,12 @@ use std::{ collections::{HashMap, HashSet}, + ffi::{OsStr, OsString}, fs::{self}, io::Write, mem::size_of, + path::PathBuf, sync::Arc, + time::Instant, }; use conduit::{debug, debug_info, debug_warn, error, info, utils, warn, Config, Error, Result}; @@ -18,10 +21,12 @@ use ruma::{ use crate::services; /// The current schema version. -/// * If the database is opened at lesser version we apply migrations up to this -/// version. -/// * If the database is opened at greater version we reject with error. -const DATABASE_VERSION: u64 = 13 + cfg!(feature = "sha256_media") as u64; +/// - If database is opened at greater version we reject with error. The +/// software must be updated for backward-incompatible changes. +/// - If database is opened at lesser version we apply migrations up to this. +/// Note that named-feature migrations may also be performed when opening at +/// equal or lesser version. These are expected to be backward-compatible. +const DATABASE_VERSION: u64 = 13; pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result<()> { // Matrix resource ownership is based on the server name; changing it @@ -119,9 +124,10 @@ async fn migrate(db: &KeyValueDatabase, config: &Config) -> Result<()> { db_lt_13(db, config).await?; } - #[cfg(feature = "sha256_media")] - if services().globals.database_version()? < 14 { - feat_sha256_media(db, config).await?; + if db.global.get(b"feat_sha256_media")?.is_none() { + migrate_sha256_media(db, config).await?; + } else if config.media_startup_check { + checkup_sha256_media(db, config).await?; } if db @@ -250,7 +256,7 @@ async fn db_lt_3(db: &KeyValueDatabase, _config: &Config) -> Result<()> { } #[allow(deprecated)] - let path = services().globals.get_media_file(&key); + let path = services().media.get_media_file(&key); let mut file = fs::File::create(path)?; file.write_all(&content)?; db.mediaid_file.insert(&key, &[])?; @@ -688,29 +694,110 @@ async fn db_lt_13(_db: &KeyValueDatabase, config: &Config) -> Result<()> { Ok(()) } -#[cfg(feature = "sha256_media")] -async fn feat_sha256_media(db: &KeyValueDatabase, _config: &Config) -> Result<()> { - use std::path::PathBuf; - - warn!("Mgrating legacy base64 file names to sha256 file names"); +/// Migrates a media directory from legacy base64 file names to sha2 file names. +/// All errors are fatal. Upon success the database is keyed to not perform this +/// again. +async fn migrate_sha256_media(db: &KeyValueDatabase, _config: &Config) -> Result<()> { + warn!("Migrating legacy base64 file names to sha256 file names"); // Move old media files to new names let mut changes = Vec::<(PathBuf, PathBuf)>::new(); for (key, _) in db.mediaid_file.iter() { - let old = services().globals.get_media_file(&key); - let new = services().globals.get_media_file_new(&key); + let old = services().media.get_media_file_b64(&key); + let new = services().media.get_media_file_sha256(&key); + debug!(?key, ?old, ?new, num = changes.len(), "change"); changes.push((old, new)); - debug!(?old, ?new, num = changes.len(), "change"); } // move the file to the new location for (old_path, path) in changes { if old_path.exists() { tokio::fs::rename(&old_path, &path).await?; + tokio::fs::symlink(&path, &old_path).await?; } } - services().globals.bump_database_version(14)?; - info!("Migration: 13 -> 14 finished"); + // Apply fix from when sha256_media was backward-incompat and bumped the schema + // version from 13 to 14. For users satisfying these conditions we can go back. + if services().globals.database_version()? == 14 && DATABASE_VERSION == 13 { + services().globals.bump_database_version(13)?; + } + + db.global.insert(b"feat_sha256_media", &[])?; + info!("Finished applying sha256_media"); + Ok(()) +} + +/// Check is run on startup for prior-migrated media directories. This handles: +/// - Going back and forth to non-sha256 legacy binaries (e.g. upstream). +/// - Deletion of artifacts in the media directory which will then fall out of +/// sync with the database. +async fn checkup_sha256_media(db: &KeyValueDatabase, config: &Config) -> Result<()> { + use crate::media::encode_key; + + debug!("Checking integrity of media directory"); + let media = &services().media; + let timer = Instant::now(); + + let dir = media.get_media_dir(); + let files: HashSet = fs::read_dir(dir)? + .filter_map(|ent| ent.map_or(None, |ent| Some(ent.path().into_os_string()))) + .collect(); + + for key in media.db.get_all_media_keys() { + let new_path = media.get_media_file_sha256(&key).into_os_string(); + let old_path = media.get_media_file_b64(&key).into_os_string(); + if let Err(e) = handle_media_check(db, config, &files, &key, &new_path, &old_path).await { + error!( + media_id = ?encode_key(&key), ?new_path, ?old_path, + "Failed to resolve media check failure: {e}" + ); + } + } + + debug_info!( + elapsed = ?timer.elapsed(), + "Finished checking media directory" + ); + + Ok(()) +} + +async fn handle_media_check( + db: &KeyValueDatabase, config: &Config, files: &HashSet, key: &[u8], new_path: &OsStr, old_path: &OsStr, +) -> Result<()> { + use crate::media::encode_key; + + let old_exists = files.contains(old_path); + let new_exists = files.contains(new_path); + if !old_exists && !new_exists { + error!( + media_id = ?encode_key(key), ?new_path, ?old_path, + "Media is missing at all paths. Removing from database..." + ); + + db.mediaid_file.remove(key)?; + db.mediaid_user.remove(key)?; + } + + if config.media_compat_file_link && !old_exists && new_exists { + debug_warn!( + media_id = ?encode_key(key), ?new_path, ?old_path, + "Media found but missing legacy link. Fixing..." + ); + + tokio::fs::symlink(&new_path, &old_path).await?; + } + + if config.media_compat_file_link && !new_exists && old_exists { + debug_warn!( + media_id = ?encode_key(key), ?new_path, ?old_path, + "Legacy media found without sha256 migration. Fixing..." + ); + + tokio::fs::rename(&old_path, &new_path).await?; + tokio::fs::symlink(&new_path, &old_path).await?; + } + Ok(()) } diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 626b2779..2e106030 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,13 +7,10 @@ pub(super) mod updates; use std::{ collections::{BTreeMap, HashMap}, - fs, - path::PathBuf, sync::Arc, time::Instant, }; -use base64::{engine::general_purpose, Engine as _}; use conduit::utils; use data::Data; use hickory_resolver::TokioAsyncResolver; @@ -128,8 +125,6 @@ impl Service { .expect("@conduit:server_name is valid"), }; - fs::create_dir_all(s.get_media_folder())?; - if !s .supported_room_versions() .contains(&s.config.default_room_version) @@ -314,39 +309,6 @@ impl Service { pub fn bump_database_version(&self, new_version: u64) -> Result<()> { self.db.bump_database_version(new_version) } - pub fn get_media_folder(&self) -> PathBuf { - let mut r = PathBuf::new(); - r.push(self.config.database_path.clone()); - r.push("media"); - r - } - - /// new SHA256 file name media function, requires "sha256_media" feature - /// flag enabled and database migrated uses SHA256 hash of the base64 key as - /// the file name - #[cfg(feature = "sha256_media")] - pub fn get_media_file_new(&self, key: &[u8]) -> PathBuf { - let mut r = PathBuf::new(); - r.push(self.config.database_path.clone()); - r.push("media"); - // Using the hash of the base64 key as the filename - // This is to prevent the total length of the path from exceeding the maximum - // length in most filesystems - r.push(general_purpose::URL_SAFE_NO_PAD.encode(::digest(key))); - r - } - - /// old base64 file name media function - /// This is the old version of `get_media_file` that uses the full base64 - /// key as the filename. - pub fn get_media_file(&self, key: &[u8]) -> PathBuf { - let mut r = PathBuf::new(); - r.push(self.config.database_path.clone()); - r.push("media"); - r.push(general_purpose::URL_SAFE_NO_PAD.encode(key)); - r - } - pub fn well_known_client(&self) -> &Option { &self.config.well_known.client } pub fn well_known_server(&self) -> &Option { &self.config.well_known.server } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index d23e082c..17a1a6dc 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,18 +1,22 @@ mod data; -use std::{collections::HashMap, io::Cursor, sync::Arc, time::SystemTime}; +mod tests; +use std::{collections::HashMap, io::Cursor, path::PathBuf, sync::Arc, time::SystemTime}; + +use base64::{engine::general_purpose, Engine as _}; +use conduit::{debug, debug_error, error, utils, Error, Result, Server}; use data::Data; +use database::KeyValueDatabase; use image::imageops::FilterType; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; use tokio::{ - fs::{self, File}, + fs, io::{AsyncReadExt, AsyncWriteExt, BufReader}, sync::{Mutex, RwLock}, }; -use tracing::{debug, error}; -use crate::{services, utils, Error, Result}; +use crate::services; #[derive(Debug)] pub struct FileMeta { @@ -39,11 +43,20 @@ pub struct UrlPreviewData { } pub struct Service { + server: Arc, pub(super) db: Arc, pub url_preview_mutex: RwLock>>>, } impl Service { + pub fn build(server: &Arc, db: &Arc) -> Self { + Self { + server: server.clone(), + db: db.clone(), + url_preview_mutex: RwLock::new(HashMap::new()), + } + } + /// Uploads a file. pub async fn create( &self, sender_user: Option, mxc: String, content_disposition: Option<&str>, @@ -58,21 +71,8 @@ impl Service { .create_file_metadata(None, mxc, 0, 0, content_disposition, content_type)? }; - let path; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(feature = "sha256_media")] - { - path = services().globals.get_media_file_new(&key); - }; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sha256_media"))] - { - path = services().globals.get_media_file(&key); - }; - - let mut f = File::create(path).await?; + //TODO: Dangling metadata in database if creation fails + let mut f = self.create_media_file(&key).await?; f.write_all(file).await?; Ok(()) @@ -82,24 +82,7 @@ impl Service { pub async fn delete(&self, mxc: String) -> Result<()> { if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc.clone()) { for key in keys { - let file_path; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(feature = "sha256_media")] - { - file_path = services().globals.get_media_file_new(&key); - }; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sha256_media"))] - { - file_path = services().globals.get_media_file(&key); - }; - - debug!("Got local file path: {:?}", file_path); - - debug!("Deleting local file {:?} from filesystem, original MXC: {}", file_path, mxc); - fs::remove_file(file_path).await?; + self.remove_media_file(&key).await?; debug!("Deleting MXC {mxc} from database"); self.db.delete_file_mxc(mxc.clone())?; @@ -128,21 +111,8 @@ impl Service { .create_file_metadata(None, mxc, width, height, content_disposition, content_type)? }; - let path; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(feature = "sha256_media")] - { - path = services().globals.get_media_file_new(&key); - }; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sha256_media"))] - { - path = services().globals.get_media_file(&key); - }; - - let mut f = File::create(path).await?; + //TODO: Dangling metadata in database if creation fails + let mut f = self.create_media_file(&key).await?; f.write_all(file).await?; Ok(()) @@ -151,22 +121,9 @@ impl Service { /// Downloads a file. pub async fn get(&self, mxc: String) -> Result> { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { - let path; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(feature = "sha256_media")] - { - path = services().globals.get_media_file_new(&key); - }; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sha256_media"))] - { - path = services().globals.get_media_file(&key); - }; - let mut file = Vec::new(); - BufReader::new(File::open(path).await?) + let path = self.get_media_file(&key); + BufReader::new(fs::File::open(path).await?) .read_to_end(&mut file) .await?; @@ -233,24 +190,11 @@ impl Service { continue; } - let path; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(feature = "sha256_media")] - { - path = services().globals.get_media_file_new(&key); - }; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sha256_media"))] - { - path = services().globals.get_media_file(&key); - }; - - debug!("MXC path: {:?}", path); + let path = self.get_media_file(&key); + debug!("MXC path: {path:?}"); let file_metadata = fs::metadata(path.clone()).await?; - debug!("File metadata: {:?}", file_metadata); + debug!("File metadata: {file_metadata:?}"); let file_created_at = match file_metadata.created() { Ok(value) => value, @@ -328,22 +272,9 @@ impl Service { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), width, height) { // Using saved thumbnail - let path; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(feature = "sha256_media")] - { - path = services().globals.get_media_file_new(&key); - }; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sha256_media"))] - { - path = services().globals.get_media_file(&key); - }; - let mut file = Vec::new(); - File::open(path).await?.read_to_end(&mut file).await?; + let path = self.get_media_file(&key); + fs::File::open(path).await?.read_to_end(&mut file).await?; Ok(Some(FileMeta { content_disposition, @@ -352,22 +283,9 @@ impl Service { })) } else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), 0, 0) { // Generate a thumbnail - let path; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(feature = "sha256_media")] - { - path = services().globals.get_media_file_new(&key); - }; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sha256_media"))] - { - path = services().globals.get_media_file(&key); - }; - let mut file = Vec::new(); - File::open(path).await?.read_to_end(&mut file).await?; + let path = self.get_media_file(&key); + fs::File::open(path).await?.read_to_end(&mut file).await?; if let Ok(image) = image::load_from_memory(&file) { let original_width = image.width(); @@ -433,21 +351,7 @@ impl Service { content_type.as_deref(), )?; - let path; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(feature = "sha256_media")] - { - path = services().globals.get_media_file_new(&thumbnail_key); - }; - - #[allow(clippy::unnecessary_operation)] // error[E0658]: attributes on expressions are experimental - #[cfg(not(feature = "sha256_media"))] - { - path = services().globals.get_media_file(&thumbnail_key); - }; - - let mut f = File::create(path).await?; + let mut f = self.create_media_file(&thumbnail_key).await?; f.write_all(&thumbnail_bytes).await?; Ok(Some(FileMeta { @@ -483,99 +387,81 @@ impl Service { .expect("valid system time"); self.db.set_url_preview(url, data, now) } -} -#[cfg(test)] -mod tests { - #[cfg(feature = "sha256_media")] - #[tokio::test] - async fn long_file_names_works() { - use std::path::PathBuf; + pub async fn create_media_dir(&self) -> Result<()> { + let dir = self.get_media_dir(); + Ok(fs::create_dir_all(dir).await?) + } - use base64::{engine::general_purpose, Engine as _}; + async fn remove_media_file(&self, key: &[u8]) -> Result<()> { + let path = self.get_media_file(key); + let legacy = self.get_media_file_b64(key); + debug!(?key, ?path, ?legacy, "Removing media file"); - use super::*; - - struct MockedKVDatabase; - - impl Data for MockedKVDatabase { - fn create_file_metadata( - &self, _sender_user: Option<&str>, mxc: String, width: u32, height: u32, - content_disposition: Option<&str>, content_type: Option<&str>, - ) -> Result> { - // copied from src/database/key_value/media.rs - let mut key = mxc.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(&width.to_be_bytes()); - key.extend_from_slice(&height.to_be_bytes()); - key.push(0xFF); - key.extend_from_slice( - content_disposition - .as_ref() - .map(|f| f.as_bytes()) - .unwrap_or_default(), - ); - key.push(0xFF); - key.extend_from_slice( - content_type - .as_ref() - .map(|c| c.as_bytes()) - .unwrap_or_default(), - ); - - Ok(key) + let file_rm = fs::remove_file(&path); + let legacy_rm = fs::remove_file(&legacy); + let (file_rm, legacy_rm) = tokio::join!(file_rm, legacy_rm); + if let Err(e) = legacy_rm { + if self.server.config.media_compat_file_link { + debug_error!(?key, ?legacy, "Failed to remove legacy media symlink: {e}"); } - - fn delete_file_mxc(&self, _mxc: String) -> Result<()> { todo!() } - - fn search_mxc_metadata_prefix(&self, _mxc: String) -> Result>> { todo!() } - - fn get_all_media_keys(&self) -> Vec> { todo!() } - - fn search_file_metadata( - &self, _mxc: String, _width: u32, _height: u32, - ) -> Result<(Option, Option, Vec)> { - todo!() - } - - fn remove_url_preview(&self, _url: &str) -> Result<()> { todo!() } - - fn set_url_preview( - &self, _url: &str, _data: &UrlPreviewData, _timestamp: std::time::Duration, - ) -> Result<()> { - todo!() - } - - fn get_url_preview(&self, _url: &str) -> Option { todo!() } } - let db: Arc = Arc::new(MockedKVDatabase); - let media = Service { - db, - url_preview_mutex: RwLock::new(HashMap::new()), - }; + Ok(file_rm?) + } - let mxc = "mxc://example.com/ascERGshawAWawugaAcauga".to_owned(); - let width = 100; - let height = 100; - let content_disposition = "attachment; filename=\"this is a very long file name with spaces and special \ - characters like äöüß and even emoji like 🦀.png\""; - let content_type = "image/png"; - let key = media - .db - .create_file_metadata(None, mxc, width, height, Some(content_disposition), Some(content_type)) - .unwrap(); - let mut r = PathBuf::from("/tmp/media"); - // r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD)); - // use the sha256 hash of the key as the file name instead of the key itself - // this is because the base64 encoded key can be longer than 255 characters. - r.push(general_purpose::URL_SAFE_NO_PAD.encode(::digest(key))); - // Check that the file path is not longer than 255 characters - // (255 is the maximum length of a file path on most file systems) - assert!( - r.to_str().unwrap().len() <= 255, - "File path is too long: {}", - r.to_str().unwrap().len() - ); + async fn create_media_file(&self, key: &[u8]) -> Result { + let path = self.get_media_file(key); + debug!(?key, ?path, "Creating media file"); + + let file = fs::File::create(&path).await?; + if self.server.config.media_compat_file_link { + let legacy = self.get_media_file_b64(key); + if let Err(e) = fs::symlink(&path, &legacy).await { + debug_error!( + key = ?encode_key(key), ?path, ?legacy, + "Failed to create legacy media symlink: {e}" + ); + } + } + + Ok(file) + } + + #[inline] + pub fn get_media_file(&self, key: &[u8]) -> PathBuf { self.get_media_file_sha256(key) } + + /// new SHA256 file name media function. requires database migrated. uses + /// SHA256 hash of the base64 key as the file name + pub fn get_media_file_sha256(&self, key: &[u8]) -> PathBuf { + let mut r = self.get_media_dir(); + // Using the hash of the base64 key as the filename + // This is to prevent the total length of the path from exceeding the maximum + // length in most filesystems + let digest = ::digest(key); + let encoded = encode_key(&digest); + r.push(encoded); + r + } + + /// old base64 file name media function + /// This is the old version of `get_media_file` that uses the full base64 + /// key as the filename. + pub fn get_media_file_b64(&self, key: &[u8]) -> PathBuf { + let mut r = self.get_media_dir(); + let encoded = encode_key(key); + r.push(encoded); + r + } + + pub fn get_media_dir(&self) -> PathBuf { + let mut r = PathBuf::new(); + r.push(self.server.config.database_path.clone()); + r.push("media"); + r } } + +#[inline] +#[must_use] +pub fn encode_key(key: &[u8]) -> String { general_purpose::URL_SAFE_NO_PAD.encode(key) } diff --git a/src/service/media/tests.rs b/src/service/media/tests.rs new file mode 100644 index 00000000..9a51bf43 --- /dev/null +++ b/src/service/media/tests.rs @@ -0,0 +1,84 @@ +#![cfg(test)] + +#[tokio::test] +async fn long_file_names_works() { + use std::path::PathBuf; + + use base64::{engine::general_purpose, Engine as _}; + + use super::*; + + struct MockedKVDatabase; + + impl Data for MockedKVDatabase { + fn create_file_metadata( + &self, _sender_user: Option<&str>, mxc: String, width: u32, height: u32, content_disposition: Option<&str>, + content_type: Option<&str>, + ) -> Result> { + // copied from src/database/key_value/media.rs + let mut key = mxc.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(&width.to_be_bytes()); + key.extend_from_slice(&height.to_be_bytes()); + key.push(0xFF); + key.extend_from_slice( + content_disposition + .as_ref() + .map(|f| f.as_bytes()) + .unwrap_or_default(), + ); + key.push(0xFF); + key.extend_from_slice( + content_type + .as_ref() + .map(|c| c.as_bytes()) + .unwrap_or_default(), + ); + + Ok(key) + } + + fn delete_file_mxc(&self, _mxc: String) -> Result<()> { todo!() } + + fn search_mxc_metadata_prefix(&self, _mxc: String) -> Result>> { todo!() } + + fn get_all_media_keys(&self) -> Vec> { todo!() } + + fn search_file_metadata( + &self, _mxc: String, _width: u32, _height: u32, + ) -> Result<(Option, Option, Vec)> { + todo!() + } + + fn remove_url_preview(&self, _url: &str) -> Result<()> { todo!() } + + fn set_url_preview(&self, _url: &str, _data: &UrlPreviewData, _timestamp: std::time::Duration) -> Result<()> { + todo!() + } + + fn get_url_preview(&self, _url: &str) -> Option { todo!() } + } + + let db: Arc = Arc::new(MockedKVDatabase); + let mxc = "mxc://example.com/ascERGshawAWawugaAcauga".to_owned(); + let width = 100; + let height = 100; + let content_disposition = "attachment; filename=\"this is a very long file name with spaces and special \ + characters like äöüß and even emoji like 🦀.png\""; + let content_type = "image/png"; + let key = db + .create_file_metadata(None, mxc, width, height, Some(content_disposition), Some(content_type)) + .unwrap(); + let mut r = PathBuf::from("/tmp/media"); + // r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD)); + // use the sha256 hash of the key as the file name instead of the key itself + // this is because the base64 encoded key can be longer than 255 characters. + r.push(general_purpose::URL_SAFE_NO_PAD.encode(::digest(key))); + // Check that the file path is not longer than 255 characters + // (255 is the maximum length of a file path on most file systems) + assert!( + r.to_str().unwrap().len() <= 255, + "File path is too long: {}", + r.to_str().unwrap().len() + ); +} diff --git a/src/service/services.rs b/src/service/services.rs index e1c27d52..7629b207 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -136,10 +136,7 @@ impl Services { key_backups: key_backups::Service { db: db.clone(), }, - media: media::Service { - db: db.clone(), - url_preview_mutex: RwLock::new(HashMap::new()), - }, + media: media::Service::build(&server, &db), sending: sending::Service::build(db.clone(), config), globals: globals::Service::load(db.clone(), config)?, server, @@ -277,6 +274,7 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} pub async fn start(&self) -> Result<()> { debug_info!("Starting services"); + self.media.create_media_dir().await?; globals::migrations::migrations(&self.db, &self.globals.config).await?; globals::emerg_access::init_emergency_access(); From c1712d4d8b57407493651ca10aa8f65624036b23 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 25 Jun 2024 01:31:52 +0000 Subject: [PATCH 0272/2091] optimize increment Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 19 +++++++++-------- src/core/utils/tests.rs | 37 ++++++++++++++++++++++++++++++++++ src/database/rocksdb/kvtree.rs | 4 ++-- 3 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 src/core/utils/tests.rs diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 27d47fa5..1e0e01c0 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -6,6 +6,7 @@ pub mod html; pub mod json; pub mod mutex_map; pub mod sys; +mod tests; use std::{ cmp::{self, Ordering}, @@ -34,16 +35,14 @@ pub fn millis_since_unix_epoch() -> u64 { .as_millis() as u64 } -pub fn increment(old: Option<&[u8]>) -> Vec { - let number = match old.map(TryInto::try_into) { - Some(Ok(bytes)) => { - let number = u64::from_be_bytes(bytes); - number + 1 - }, - _ => 1, // Start at one. since 0 should return the first event in the db - }; - - number.to_be_bytes().to_vec() +#[inline] +#[must_use] +pub fn increment(old: Option<&[u8]>) -> [u8; 8] { + const ZERO: u64 = 0; + old.map(TryInto::try_into) + .map_or(ZERO, |val| val.map_or(ZERO, u64::from_be_bytes)) + .wrapping_add(1) + .to_be_bytes() } #[must_use] diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs new file mode 100644 index 00000000..b226bd41 --- /dev/null +++ b/src/core/utils/tests.rs @@ -0,0 +1,37 @@ +#![cfg(test)] + +use crate::utils; + +#[test] +fn increment_none() { + let bytes: [u8; 8] = utils::increment(None); + let res = u64::from_be_bytes(bytes); + assert_eq!(res, 1); +} + +#[test] +fn increment_fault() { + let start: u8 = 127; + let bytes: [u8; 1] = start.to_be_bytes(); + let bytes: [u8; 8] = utils::increment(Some(&bytes)); + let res = u64::from_be_bytes(bytes); + assert_eq!(res, 1); +} + +#[test] +fn increment_norm() { + let start: u64 = 1_234_567; + let bytes: [u8; 8] = start.to_be_bytes(); + let bytes: [u8; 8] = utils::increment(Some(&bytes)); + let res = u64::from_be_bytes(bytes); + assert_eq!(res, 1_234_568); +} + +#[test] +fn increment_wrap() { + let start = u64::MAX; + let bytes: [u8; 8] = start.to_be_bytes(); + let bytes: [u8; 8] = utils::increment(Some(&bytes)); + let res = u64::from_be_bytes(bytes); + assert_eq!(res, 0); +} diff --git a/src/database/rocksdb/kvtree.rs b/src/database/rocksdb/kvtree.rs index 00d01f8e..253b10c6 100644 --- a/src/database/rocksdb/kvtree.rs +++ b/src/database/rocksdb/kvtree.rs @@ -155,13 +155,13 @@ impl KvTree for RocksDbEngineTree<'_> { let new = utils::increment(old.as_deref()); self.db .rocks - .put_cf_opt(&self.cf(), key, &new, &writeoptions)?; + .put_cf_opt(&self.cf(), key, new, &writeoptions)?; if !self.db.corked() { self.db.flush()?; } - Ok(new) + Ok(new.to_vec()) } fn increment_batch(&self, iter: &mut dyn Iterator>) -> Result<()> { From 01f4455ceb5395c78a5302503d1ce589569e4079 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 12 Jun 2024 01:08:07 +0000 Subject: [PATCH 0273/2091] convert rocksdb errors locally; remove from Error. Signed-off-by: Jason Volk --- Cargo.lock | 1 - src/core/Cargo.toml | 16 +++++------ src/core/error.rs | 11 +++----- src/database/rocksdb/kvtree.rs | 49 +++++++++++++++++++++++----------- src/database/rocksdb/mod.rs | 44 +++++++++++++++--------------- src/main/Cargo.toml | 2 -- 6 files changed, 65 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6333c49e..3294de97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -675,7 +675,6 @@ dependencies = [ "reqwest", "ring", "ruma", - "rust-rocksdb-uwu", "sanitize-filename", "serde", "serde_json", diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index dc3fc1d4..46ca00c7 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -24,14 +24,10 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] -rocksdb = [ - "dep:rust-rocksdb", -] jemalloc = [ "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl", "dep:tikv-jemallocator", - "rust-rocksdb/jemalloc", ] jemalloc_prof = [ "tikv-jemalloc-sys/profiling", @@ -39,18 +35,13 @@ jemalloc_prof = [ hardened_malloc = [ "dep:hardened_malloc-rs" ] -io_uring = [ - "rust-rocksdb/io-uring", -] -zstd_compression = [ - "rust-rocksdb/zstd", -] gzip_compression = [ "reqwest/gzip", ] brotli_compression = [ "reqwest/brotli", ] +zstd_compression =[] perf_measurements = [] sentry_telemetry = [] @@ -72,8 +63,13 @@ regex.workspace = true reqwest.workspace = true ring.workspace = true ruma.workspace = true +<<<<<<< HEAD rust-rocksdb.optional = true rust-rocksdb.workspace = true +======= +rusqlite.optional = true +rusqlite.workspace = true +>>>>>>> 6963b38f (convert rocksdb errors locally; remove from Error.) sanitize-filename.workspace = true serde_json.workspace = true serde_regex.workspace = true diff --git a/src/core/error.rs b/src/core/error.rs index 4fafd1e3..ac01cc96 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -25,12 +25,8 @@ pub type Result = std::result::Result; #[derive(Error)] pub enum Error { - #[cfg(feature = "rocksdb")] - #[error("There was a problem with the connection to the rocksdb database: {source}")] - RocksDb { - #[from] - source: rust_rocksdb::Error, - }, + #[error("{0}")] + Database(String), #[error("Could not generate an image: {source}")] Image { #[from] @@ -114,8 +110,7 @@ impl Error { let db_error = String::from("Database or I/O error occurred."); match self { - #[cfg(feature = "rocksdb")] - Self::RocksDb { + Self::Database { .. } => db_error, Self::Io { diff --git a/src/database/rocksdb/kvtree.rs b/src/database/rocksdb/kvtree.rs index 253b10c6..949db84e 100644 --- a/src/database/rocksdb/kvtree.rs +++ b/src/database/rocksdb/kvtree.rs @@ -2,7 +2,10 @@ use std::{future::Future, pin::Pin, sync::Arc}; use conduit::{utils, Result}; -use super::{rust_rocksdb::WriteBatchWithTransaction, watchers::Watchers, Engine, KeyValueDatabaseEngine, KvTree}; +use super::{ + or_else, result, rust_rocksdb::WriteBatchWithTransaction, watchers::Watchers, Engine, KeyValueDatabaseEngine, + KvTree, +}; pub(crate) struct RocksDbEngineTree<'a> { pub(crate) db: Arc, @@ -19,7 +22,7 @@ impl KvTree for RocksDbEngineTree<'_> { let mut readoptions = rust_rocksdb::ReadOptions::default(); readoptions.set_total_order_seek(true); - Ok(self.db.rocks.get_cf_opt(&self.cf(), key, &readoptions)?) + result(self.db.rocks.get_cf_opt(&self.cf(), key, &readoptions)) } fn multi_get(&self, keys: &[&[u8]]) -> Result>>> { @@ -36,9 +39,10 @@ impl KvTree for RocksDbEngineTree<'_> { .rocks .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, &readoptions) { - match res? { - Some(res) => ret.push(Some((*res).to_vec())), - None => ret.push(None), + match res { + Ok(Some(res)) => ret.push(Some((*res).to_vec())), + Ok(None) => ret.push(None), + Err(e) => return or_else(e), } } @@ -50,7 +54,8 @@ impl KvTree for RocksDbEngineTree<'_> { self.db .rocks - .put_cf_opt(&self.cf(), key, value, &writeoptions)?; + .put_cf_opt(&self.cf(), key, value, &writeoptions) + .or_else(or_else)?; if !self.db.corked() { self.db.flush()?; @@ -70,25 +75,25 @@ impl KvTree for RocksDbEngineTree<'_> { batch.put_cf(&self.cf(), key, value); } - let result = self.db.rocks.write_opt(batch, &writeoptions); + let res = self.db.rocks.write_opt(batch, &writeoptions); if !self.db.corked() { self.db.flush()?; } - Ok(result?) + result(res) } fn remove(&self, key: &[u8]) -> Result<()> { let writeoptions = rust_rocksdb::WriteOptions::default(); - let result = self.db.rocks.delete_cf_opt(&self.cf(), key, &writeoptions); + let res = self.db.rocks.delete_cf_opt(&self.cf(), key, &writeoptions); if !self.db.corked() { self.db.flush()?; } - Ok(result?) + result(res) } fn remove_batch(&self, iter: &mut dyn Iterator>) -> Result<()> { @@ -100,13 +105,13 @@ impl KvTree for RocksDbEngineTree<'_> { batch.delete_cf(&self.cf(), key); } - let result = self.db.rocks.write_opt(batch, &writeoptions); + let res = self.db.rocks.write_opt(batch, &writeoptions); if !self.db.corked() { self.db.flush()?; } - Ok(result?) + result(res) } fn iter<'a>(&'a self) -> Box, Vec)> + 'a> { @@ -151,11 +156,16 @@ impl KvTree for RocksDbEngineTree<'_> { readoptions.set_total_order_seek(true); let writeoptions = rust_rocksdb::WriteOptions::default(); - let old = self.db.rocks.get_cf_opt(&self.cf(), key, &readoptions)?; + let old = self + .db + .rocks + .get_cf_opt(&self.cf(), key, &readoptions) + .or_else(or_else)?; let new = utils::increment(old.as_deref()); self.db .rocks - .put_cf_opt(&self.cf(), key, new, &writeoptions)?; + .put_cf_opt(&self.cf(), key, new, &writeoptions) + .or_else(or_else)?; if !self.db.corked() { self.db.flush()?; @@ -172,12 +182,19 @@ impl KvTree for RocksDbEngineTree<'_> { let mut batch = WriteBatchWithTransaction::::default(); for key in iter { - let old = self.db.rocks.get_cf_opt(&self.cf(), &key, &readoptions)?; + let old = self + .db + .rocks + .get_cf_opt(&self.cf(), &key, &readoptions) + .or_else(or_else)?; let new = utils::increment(old.as_deref()); batch.put_cf(&self.cf(), key, new); } - self.db.rocks.write_opt(batch, &writeoptions)?; + self.db + .rocks + .write_opt(batch, &writeoptions) + .or_else(or_else)?; if !self.db.corked() { self.db.flush()?; diff --git a/src/database/rocksdb/mod.rs b/src/database/rocksdb/mod.rs index 52fd303d..943b6745 100644 --- a/src/database/rocksdb/mod.rs +++ b/src/database/rocksdb/mod.rs @@ -51,7 +51,7 @@ impl KeyValueDatabaseEngine for Arc { let mut col_cache = HashMap::new(); col_cache.insert("primary".to_owned(), Cache::new_lru_cache(col_cache_capacity_bytes)); - let mut db_env = Env::new()?; + let mut db_env = Env::new().or_else(or_else)?; let row_cache = Cache::new_lru_cache(row_cache_capacity_bytes); let db_opts = db_options(config, &mut db_env, &row_cache, col_cache.get("primary").expect("cache")); @@ -73,12 +73,13 @@ impl KeyValueDatabaseEngine for Arc { .collect::>(); debug!("Opening database..."); - let db = if config.rocksdb_read_only { - Db::::open_cf_for_read_only(&db_opts, &config.database_path, cfs.clone(), false)? + let res = if config.rocksdb_read_only { + Db::::open_cf_for_read_only(&db_opts, &config.database_path, cfs.clone(), false) } else { - Db::::open_cf_descriptors(&db_opts, &config.database_path, cfds)? + Db::::open_cf_descriptors(&db_opts, &config.database_path, cfds) }; + let db = res.or_else(or_else)?; info!( "Opened database at sequence number {} in {:?}", db.latest_sequence_number(), @@ -115,17 +116,9 @@ impl KeyValueDatabaseEngine for Arc { })) } - fn flush(&self) -> Result<()> { - DBCommon::flush_wal(&self.rocks, false)?; + fn flush(&self) -> Result<()> { result(DBCommon::flush_wal(&self.rocks, false)) } - Ok(()) - } - - fn sync(&self) -> Result<()> { - DBCommon::flush_wal(&self.rocks, true)?; - - Ok(()) - } + fn sync(&self) -> Result<()> { result(DBCommon::flush_wal(&self.rocks, true)) } fn corked(&self) -> bool { self.corks.load(std::sync::atomic::Ordering::Relaxed) > 0 } @@ -146,7 +139,7 @@ impl KeyValueDatabaseEngine for Arc { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] fn memory_usage(&self) -> Result { let mut res = String::new(); - let stats = get_memory_usage_stats(Some(&[&self.rocks]), Some(&[&self.row_cache]))?; + let stats = get_memory_usage_stats(Some(&[&self.rocks]), Some(&[&self.row_cache])).or_else(or_else)?; writeln!( res, "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow cache: {:.2} MiB", @@ -168,10 +161,7 @@ impl KeyValueDatabaseEngine for Arc { fn cleanup(&self) -> Result<()> { debug!("Running flush_opt"); let flushoptions = rust_rocksdb::FlushOptions::default(); - - DBCommon::flush_opt(&self.rocks, &flushoptions)?; - - Ok(()) + result(DBCommon::flush_opt(&self.rocks, &flushoptions)) } fn backup(&self) -> Result<(), Box> { @@ -214,8 +204,8 @@ impl KeyValueDatabaseEngine for Arc { } let mut res = String::new(); - let options = BackupEngineOptions::new(path.unwrap())?; - let engine = BackupEngine::open(&options, &self.env)?; + let options = BackupEngineOptions::new(path.expect("valid path")).or_else(or_else)?; + let engine = BackupEngine::open(&options, &self.env).or_else(or_else)?; for info in engine.get_backup_info() { writeln!( res, @@ -275,3 +265,15 @@ impl Drop for Engine { self.env.join_all_threads(); } } + +#[inline] +fn result(r: std::result::Result) -> Result { + r.map_or_else(or_else, and_then) +} + +#[inline(always)] +fn and_then(t: T) -> Result { Ok(t) } + +fn or_else(e: rust_rocksdb::Error) -> Result { Err(map_err(e)) } + +fn map_err(e: rust_rocksdb::Error) -> conduit::Error { conduit::Error::Database(e.into_string()) } diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 9ef51545..cb7cb9a9 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -83,7 +83,6 @@ hardened_malloc = [ ] io_uring = [ "conduit-admin/io_uring", - "conduit-core/io_uring", "conduit-database/io_uring", ] jemalloc = [ @@ -119,7 +118,6 @@ release_max_log_level = [ ] rocksdb = [ "conduit-admin/rocksdb", - "conduit-core/rocksdb", "conduit-database/rocksdb", ] sentry_telemetry = [ From fbcdb3860ae951083a65f40ca001e265ea6e51b3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 12 Jun 2024 01:13:19 +0000 Subject: [PATCH 0274/2091] eliminate dep:rust-rocksdb et al from admin crate Signed-off-by: Jason Volk --- Cargo.lock | 1 - src/admin/Cargo.toml | 15 +-------------- src/main/Cargo.toml | 4 ---- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3294de97..929fc542 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -610,7 +610,6 @@ dependencies = [ "loole", "regex", "ruma", - "rust-rocksdb-uwu", "serde", "serde_json", "serde_yaml", diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 42327430..98e3a5b4 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -24,18 +24,7 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] -rocksdb = [ - "dep:rust-rocksdb", -] -jemalloc = [ - "rust-rocksdb/jemalloc", -] -io_uring = [ - "rust-rocksdb/io-uring", -] -zstd_compression = [ - "rust-rocksdb/zstd", -] +rocksdb = [] [dependencies] clap.workspace = true @@ -48,8 +37,6 @@ log.workspace = true loole.workspace = true regex.workspace = true ruma.workspace = true -rust-rocksdb.optional = true -rust-rocksdb.workspace = true serde_json.workspace = true serde.workspace = true serde_yaml.workspace = true diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index cb7cb9a9..b2ead09e 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -82,11 +82,9 @@ hardened_malloc = [ "conduit-core/hardened_malloc", ] io_uring = [ - "conduit-admin/io_uring", "conduit-database/io_uring", ] jemalloc = [ - "conduit-admin/jemalloc", "conduit-core/jemalloc", "conduit-database/jemalloc", ] @@ -117,7 +115,6 @@ release_max_log_level = [ "conduit-service/release_max_log_level", ] rocksdb = [ - "conduit-admin/rocksdb", "conduit-database/rocksdb", ] sentry_telemetry = [ @@ -136,7 +133,6 @@ tokio_console = [ "tokio/tracing", ] zstd_compression = [ - "conduit-admin/zstd_compression", "conduit-core/zstd_compression", "conduit-database/zstd_compression", "conduit-router/zstd_compression", From a6edaad6fcebb5f57c50611fe8354f5df2c537a7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 14 Jun 2024 18:29:21 +0000 Subject: [PATCH 0275/2091] error enum cleanup Signed-off-by: Jason Volk --- src/core/Cargo.toml | 7 --- src/core/error.rs | 108 ++++++++++++++++------------------ src/core/mod.rs | 4 +- src/service/appservice/mod.rs | 2 +- 4 files changed, 54 insertions(+), 67 deletions(-) diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 46ca00c7..ec4eaf9b 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -63,13 +63,6 @@ regex.workspace = true reqwest.workspace = true ring.workspace = true ruma.workspace = true -<<<<<<< HEAD -rust-rocksdb.optional = true -rust-rocksdb.workspace = true -======= -rusqlite.optional = true -rusqlite.workspace = true ->>>>>>> 6963b38f (convert rocksdb errors locally; remove from Error.) sanitize-filename.workspace = true serde_json.workspace = true serde_regex.workspace = true diff --git a/src/core/error.rs b/src/core/error.rs index ac01cc96..6942d1be 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -7,7 +7,11 @@ use http_body_util::Full; use ruma::{ api::{ client::{ - error::{Error as RumaError, ErrorBody, ErrorKind}, + error::ErrorKind::{ + Forbidden, GuestAccessForbidden, LimitExceeded, MissingToken, NotFound, ThreepidAuthFailed, + ThreepidDenied, TooLarge, Unauthorized, Unknown, UnknownToken, Unrecognized, UserDeactivated, + WrongRoomKeysVersion, + }, uiaa::{UiaaInfo, UiaaResponse}, }, OutgoingResponse, @@ -16,68 +20,56 @@ use ruma::{ }; use thiserror::Error; use tracing::error; -use ErrorKind::{ - Forbidden, GuestAccessForbidden, LimitExceeded, MissingToken, NotFound, ThreepidAuthFailed, ThreepidDenied, - TooLarge, Unauthorized, Unknown, UnknownToken, Unrecognized, UserDeactivated, WrongRoomKeysVersion, -}; - -pub type Result = std::result::Result; #[derive(Error)] pub enum Error { + // std #[error("{0}")] - Database(String), - #[error("Could not generate an image: {source}")] - Image { - #[from] - source: image::error::ImageError, - }, - #[error("Could not connect to server: {source}")] - Reqwest { - #[from] - source: reqwest::Error, - }, - #[error("Could build regular expression: {source}")] - Regex { - #[from] - source: regex::Error, - }, - #[error("Remote server {0} responded with: {1}")] - Federation(OwnedServerName, RumaError), - #[error("Could not do this io: {source}")] - Io { - #[from] - source: std::io::Error, - }, - #[error("There was a problem with your configuration: {0}")] - BadConfig(String), - #[error("{0}")] - BadServerResponse(&'static str), - #[error("{0}")] - /// Don't create this directly. Use Error::bad_database instead. - BadDatabase(&'static str), - #[error("uiaa")] - Uiaa(UiaaInfo), - #[error("{0}: {1}")] - BadRequest(ErrorKind, &'static str), - #[error("{0}")] - Conflict(&'static str), // This is only needed for when a room alias already exists + Fmt(#[from] fmt::Error), + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + + // third-party + #[error("Regex error: {0}")] + Regex(#[from] regex::Error), + #[error("Tracing filter error: {0}")] + TracingFilter(#[from] tracing_subscriber::filter::ParseError), + #[error("Image error: {0}")] + Image(#[from] image::error::ImageError), + #[error("Request error: {0}")] + Reqwest(#[from] reqwest::Error), #[error("{0}")] Extension(#[from] axum::extract::rejection::ExtensionRejection), #[error("{0}")] Path(#[from] axum::extract::rejection::PathRejection), - #[error("from {0}: {1}")] - Redaction(OwnedServerName, ruma::canonical_json::RedactionError), - #[error("{0} in {1}")] - InconsistentRoomState(&'static str, ruma::OwnedRoomId), - #[error("{0}")] - TracingFilter(#[from] tracing_subscriber::filter::ParseError), - #[error("{0}")] - AdminCommand(&'static str), - #[error("{0}")] - Fmt(#[from] fmt::Error), + + // ruma #[error("{0}")] Mxid(#[from] ruma::IdParseError), + #[error("{0}: {1}")] + BadRequest(ruma::api::client::error::ErrorKind, &'static str), + #[error("from {0}: {1}")] + Redaction(OwnedServerName, ruma::canonical_json::RedactionError), + #[error("Remote server {0} responded with: {1}")] + Federation(OwnedServerName, ruma::api::client::error::Error), + #[error("{0} in {1}")] + InconsistentRoomState(&'static str, ruma::OwnedRoomId), + + // conduwuit + #[error("There was a problem with your configuration: {0}")] + BadConfig(String), + #[error("{0}")] + BadDatabase(&'static str), + #[error("{0}")] + Database(String), + #[error("{0}")] + BadServerResponse(&'static str), + #[error("{0}")] + Conflict(&'static str), // This is only needed for when a room alias already exists + #[error("uiaa")] + Uiaa(UiaaInfo), + + // unique / untyped #[error("{0}")] Err(String), } @@ -94,7 +86,7 @@ impl Error { } /// Returns the Matrix error code / error kind - pub fn error_code(&self) -> ErrorKind { + pub fn error_code(&self) -> ruma::api::client::error::ErrorKind { if let Self::Federation(_, error) = self { return error.error_kind().unwrap_or_else(|| &Unknown).clone(); } @@ -107,15 +99,13 @@ impl Error { /// Sanitizes public-facing errors that can leak sensitive information. pub fn sanitized_error(&self) -> String { - let db_error = String::from("Database or I/O error occurred."); - match self { Self::Database { .. - } => db_error, + } => String::from("Database error occurred."), Self::Io { .. - } => db_error, + } => String::from("I/O error occurred."), _ => self.to_string(), } } @@ -142,6 +132,8 @@ impl From for RumaResponse { impl Error { pub fn to_response(&self) -> RumaResponse { + use ruma::api::client::error::{Error as RumaError, ErrorBody}; + if let Self::Uiaa(uiaainfo) = self { return RumaResponse(UiaaResponse::AuthResponse(uiaainfo.clone())); } diff --git a/src/core/mod.rs b/src/core/mod.rs index baebbb8f..383b45ee 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -10,10 +10,12 @@ pub mod utils; pub mod version; pub use config::Config; -pub use error::{Error, Result, RumaResponse}; +pub use error::{Error, RumaResponse}; pub use pducount::PduCount; pub use server::Server; +pub type Result = std::result::Result; + #[cfg(not(conduit_mods))] pub mod mods { #[macro_export] diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 1f29c415..94133396 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -159,7 +159,7 @@ impl Service { .write() .await .remove(service_name) - .ok_or_else(|| crate::Error::AdminCommand("Appservice not found"))?; + .ok_or_else(|| crate::Error::Err("Appservice not found".to_owned()))?; // remove the appservice from the database self.db.unregister_appservice(service_name)?; From 7ad7badd60809e6b63cd6190c982f22ce1a448d0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 27 May 2024 03:17:20 +0000 Subject: [PATCH 0276/2091] devirtualize service Data traits Signed-off-by: Jason Volk --- src/admin/query/account_data.rs | 2 - src/service/account_data/data.rs | 46 ++-- src/service/account_data/mod.rs | 24 +- src/service/admin/mod.rs | 10 +- src/service/appservice/data.rs | 39 ++- src/service/appservice/mod.rs | 8 +- src/service/globals/data.rs | 135 +++++----- src/service/globals/mod.rs | 10 +- src/service/key_backups/data.rs | 88 +++---- src/service/key_backups/mod.rs | 14 +- src/service/media/data.rs | 61 ++--- src/service/media/mod.rs | 10 +- src/service/mod.rs | 2 +- src/service/presence/data.rs | 39 ++- src/service/presence/mod.rs | 16 +- src/service/pusher/data.rs | 31 ++- src/service/pusher/mod.rs | 11 +- src/service/rooms/alias/data.rs | 49 ++-- src/service/rooms/alias/mod.rs | 11 +- src/service/rooms/auth_chain/data.rs | 29 ++- src/service/rooms/auth_chain/mod.rs | 11 +- src/service/rooms/directory/data.rs | 37 +-- src/service/rooms/directory/mod.rs | 11 +- src/service/rooms/event_handler/mod.rs | 8 +- src/service/rooms/lazy_loading/data.rs | 30 +-- src/service/rooms/lazy_loading/mod.rs | 12 +- src/service/rooms/metadata/data.rs | 43 ++-- src/service/rooms/metadata/mod.rs | 11 +- src/service/rooms/outlier/data.rs | 23 +- src/service/rooms/outlier/mod.rs | 11 +- src/service/rooms/pdu_metadata/data.rs | 37 +-- src/service/rooms/pdu_metadata/mod.rs | 11 +- src/service/rooms/read_receipt/data.rs | 51 ++-- src/service/rooms/read_receipt/mod.rs | 11 +- src/service/rooms/search/data.rs | 45 ++-- src/service/rooms/search/mod.rs | 11 +- src/service/rooms/short/data.rs | 62 ++--- src/service/rooms/short/mod.rs | 11 +- src/service/rooms/spaces/mod.rs | 13 + src/service/rooms/state/data.rs | 50 ++-- src/service/rooms/state/mod.rs | 11 +- src/service/rooms/state_accessor/data.rs | 76 ++---- src/service/rooms/state_accessor/mod.rs | 20 +- src/service/rooms/state_cache/data.rs | 201 +++++++-------- src/service/rooms/state_cache/mod.rs | 10 +- src/service/rooms/state_compressor/data.rs | 27 +- src/service/rooms/state_compressor/mod.rs | 20 +- src/service/rooms/threads/data.rs | 26 +- src/service/rooms/threads/mod.rs | 11 +- src/service/rooms/timeline/data.rs | 127 ++++----- src/service/rooms/timeline/mod.rs | 12 +- src/service/rooms/typing/mod.rs | 12 +- src/service/rooms/user/data.rs | 54 ++-- src/service/rooms/user/mod.rs | 11 +- src/service/sending/data.rs | 58 ++--- src/service/sending/mod.rs | 15 +- src/service/sending/sender.rs | 2 +- src/service/services.rs | 143 +++-------- src/service/transaction_ids/data.rs | 28 +- src/service/transaction_ids/mod.rs | 11 +- src/service/uiaa/data.rs | 46 ++-- src/service/uiaa/mod.rs | 11 +- src/service/users/data.rs | 283 ++++++++------------- src/service/users/mod.rs | 17 +- 64 files changed, 1190 insertions(+), 1176 deletions(-) diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index 9f5b1482..ce9caedb 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -14,7 +14,6 @@ pub(super) async fn account_data(subcommand: AccountData) -> Result Result, user_id: &UserId, event_type: RoomAccountDataEventType, - data: &serde_json::Value, - ) -> Result<()>; - - /// Searches the account data for a specific kind. - fn get( - &self, room_id: Option<&RoomId>, user_id: &UserId, kind: RoomAccountDataEventType, - ) -> Result>>; - - /// Returns all changes to the account data that happened after `since`. - fn changes_since( - &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, - ) -> Result>>; +pub(super) struct Data { + roomuserdataid_accountdata: Arc, + roomusertype_roomuserdataid: Arc, } -impl Data for KeyValueDatabase { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + roomuserdataid_accountdata: db.roomuserdataid_accountdata.clone(), + roomusertype_roomuserdataid: db.roomusertype_roomuserdataid.clone(), + } + } + /// Places one event in the account data of the user and removes the /// previous entry. - #[tracing::instrument(skip(self, room_id, user_id, event_type, data))] - fn update( - &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: RoomAccountDataEventType, + pub(super) fn update( + &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: &RoomAccountDataEventType, data: &serde_json::Value, ) -> Result<()> { let mut prefix = room_id @@ -80,9 +72,8 @@ impl Data for KeyValueDatabase { } /// Searches the account data for a specific kind. - #[tracing::instrument(skip(self, room_id, user_id, kind))] - fn get( - &self, room_id: Option<&RoomId>, user_id: &UserId, kind: RoomAccountDataEventType, + pub(super) fn get( + &self, room_id: Option<&RoomId>, user_id: &UserId, kind: &RoomAccountDataEventType, ) -> Result>> { let mut key = room_id .map(ToString::to_string) @@ -107,8 +98,7 @@ impl Data for KeyValueDatabase { } /// Returns all changes to the account data that happened after `since`. - #[tracing::instrument(skip(self, room_id, user_id, since))] - fn changes_since( + pub(super) fn changes_since( &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, ) -> Result>> { let mut userdata = HashMap::new(); diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 7e17e145..fc330ead 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -2,40 +2,46 @@ mod data; use std::{collections::HashMap, sync::Arc}; -pub use data::Data; +use conduit::{Result, Server}; +use data::Data; +use database::KeyValueDatabase; use ruma::{ events::{AnyEphemeralRoomEvent, RoomAccountDataEventType}, serde::Raw, RoomId, UserId, }; -use crate::Result; - pub struct Service { - pub db: Arc, + db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + /// Places one event in the account data of the user and removes the /// previous entry. - #[tracing::instrument(skip(self, room_id, user_id, event_type, data))] + #[allow(clippy::needless_pass_by_value)] pub fn update( &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: RoomAccountDataEventType, data: &serde_json::Value, ) -> Result<()> { - self.db.update(room_id, user_id, event_type, data) + self.db.update(room_id, user_id, &event_type, data) } /// Searches the account data for a specific kind. - #[tracing::instrument(skip(self, room_id, user_id, event_type))] + #[allow(clippy::needless_pass_by_value)] pub fn get( &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: RoomAccountDataEventType, ) -> Result>> { - self.db.get(room_id, user_id, event_type) + self.db.get(room_id, user_id, &event_type) } /// Returns all changes to the account data that happened after `since`. - #[tracing::instrument(skip(self, room_id, user_id, since))] + #[tracing::instrument(skip_all, name = "since")] pub fn changes_since( &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, ) -> Result>> { diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 730163ac..5a1c161c 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + pub mod console; mod create; mod grant; @@ -44,17 +47,16 @@ pub struct Command { } impl Service { - #[must_use] - pub fn build() -> Arc { + pub fn build(_server: &Arc, _db: &Arc) -> Result> { let (sender, receiver) = loole::bounded(COMMAND_QUEUE_LIMIT); - Arc::new(Self { + Ok(Arc::new(Self { sender, receiver: Mutex::new(receiver), handler_join: Mutex::new(None), handle: Mutex::new(None), #[cfg(feature = "console")] console: console::Console::new(), - }) + })) } pub async fn start_handler(self: &Arc) { diff --git a/src/service/appservice/data.rs b/src/service/appservice/data.rs index e81cb7ac..59fabe96 100644 --- a/src/service/appservice/data.rs +++ b/src/service/appservice/data.rs @@ -1,28 +1,23 @@ +use std::sync::Arc; + +use database::KvTree; use ruma::api::appservice::Registration; use crate::{utils, Error, KeyValueDatabase, Result}; -pub trait Data: Send + Sync { - /// Registers an appservice and returns the ID to the caller - fn register_appservice(&self, yaml: Registration) -> Result; - - /// Remove an appservice registration - /// - /// # Arguments - /// - /// * `service_name` - the name you send to register the service previously - fn unregister_appservice(&self, service_name: &str) -> Result<()>; - - fn get_registration(&self, id: &str) -> Result>; - - fn iter_ids<'a>(&'a self) -> Result> + 'a>>; - - fn all(&self) -> Result>; +pub struct Data { + id_appserviceregistrations: Arc, } -impl Data for KeyValueDatabase { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + id_appserviceregistrations: db.id_appserviceregistrations.clone(), + } + } + /// Registers an appservice and returns the ID to the caller - fn register_appservice(&self, yaml: Registration) -> Result { + pub(super) fn register_appservice(&self, yaml: Registration) -> Result { let id = yaml.id.as_str(); self.id_appserviceregistrations .insert(id.as_bytes(), serde_yaml::to_string(&yaml).unwrap().as_bytes())?; @@ -35,13 +30,13 @@ impl Data for KeyValueDatabase { /// # Arguments /// /// * `service_name` - the name you send to register the service previously - fn unregister_appservice(&self, service_name: &str) -> Result<()> { + pub(super) fn unregister_appservice(&self, service_name: &str) -> Result<()> { self.id_appserviceregistrations .remove(service_name.as_bytes())?; Ok(()) } - fn get_registration(&self, id: &str) -> Result> { + pub fn get_registration(&self, id: &str) -> Result> { self.id_appserviceregistrations .get(id.as_bytes())? .map(|bytes| { @@ -51,14 +46,14 @@ impl Data for KeyValueDatabase { .transpose() } - fn iter_ids<'a>(&'a self) -> Result> + 'a>> { + pub(super) fn iter_ids<'a>(&'a self) -> Result> + 'a>> { Ok(Box::new(self.id_appserviceregistrations.iter().map(|(id, _)| { utils::string_from_bytes(&id) .map_err(|_| Error::bad_database("Invalid id bytes in id_appserviceregistrations.")) }))) } - fn all(&self) -> Result> { + pub fn all(&self) -> Result> { self.iter_ids()? .filter_map(Result::ok) .map(move |id| { diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 94133396..83023151 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -113,13 +113,17 @@ impl TryFrom for RegistrationInfo { } pub struct Service { - pub db: Arc, + pub db: Data, registration_info: RwLock>, } +use conduit::Server; +use database::KeyValueDatabase; + impl Service { - pub fn build(db: Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { let mut registration_info = BTreeMap::new(); + let db = Data::new(db); // Inserting registrations into cache for appservice in db.all()? { registration_info.insert( diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 9851bc0a..1645c8d6 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -1,6 +1,9 @@ -use std::collections::{BTreeMap, HashMap}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; -use async_trait::async_trait; +use database::{Cork, KeyValueDatabase, KvTree}; use futures_util::{stream::FuturesUnordered, StreamExt}; use lru_cache::LruCache; use ruma::{ @@ -10,56 +13,60 @@ use ruma::{ }; use tracing::trace; -use crate::{database::Cork, services, utils, Error, KeyValueDatabase, Result}; +use crate::{services, utils, Error, Result}; const COUNTER: &[u8] = b"c"; const LAST_CHECK_FOR_UPDATES_COUNT: &[u8] = b"u"; -#[async_trait] -pub trait Data: Send + Sync { - fn next_count(&self) -> Result; - fn current_count(&self) -> Result; - fn last_check_for_updates_id(&self) -> Result; - fn update_check_for_updates_id(&self, id: u64) -> Result<()>; - #[allow(unused_qualifications)] // async traits - async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()>; - fn cleanup(&self) -> Result<()>; - - fn cork(&self) -> Cork; - fn cork_and_flush(&self) -> Cork; - - fn memory_usage(&self) -> String; - fn clear_caches(&self, amount: u32); - fn load_keypair(&self) -> Result; - fn remove_keypair(&self) -> Result<()>; - fn add_signing_key( - &self, origin: &ServerName, new_keys: ServerSigningKeys, - ) -> Result>; - - /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found - /// for the server. - fn signing_keys_for(&self, origin: &ServerName) -> Result>; - fn database_version(&self) -> Result; - fn bump_database_version(&self, new_version: u64) -> Result<()>; - fn backup(&self) -> Result<(), Box> { unimplemented!() } - fn backup_list(&self) -> Result { Ok(String::new()) } - fn file_list(&self) -> Result { Ok(String::new()) } +pub struct Data { + global: Arc, + todeviceid_events: Arc, + userroomid_joined: Arc, + userroomid_invitestate: Arc, + userroomid_leftstate: Arc, + userroomid_notificationcount: Arc, + userroomid_highlightcount: Arc, + pduid_pdu: Arc, + keychangeid_userid: Arc, + roomusertype_roomuserdataid: Arc, + server_signingkeys: Arc, + readreceiptid_readreceipt: Arc, + userid_lastonetimekeyupdate: Arc, + pub(super) db: Arc, } -#[async_trait] -impl Data for KeyValueDatabase { - fn next_count(&self) -> Result { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + global: db.global.clone(), + todeviceid_events: db.todeviceid_events.clone(), + userroomid_joined: db.userroomid_joined.clone(), + userroomid_invitestate: db.userroomid_invitestate.clone(), + userroomid_leftstate: db.userroomid_leftstate.clone(), + userroomid_notificationcount: db.userroomid_notificationcount.clone(), + userroomid_highlightcount: db.userroomid_highlightcount.clone(), + pduid_pdu: db.pduid_pdu.clone(), + keychangeid_userid: db.keychangeid_userid.clone(), + roomusertype_roomuserdataid: db.roomusertype_roomuserdataid.clone(), + server_signingkeys: db.server_signingkeys.clone(), + readreceiptid_readreceipt: db.readreceiptid_readreceipt.clone(), + userid_lastonetimekeyupdate: db.userid_lastonetimekeyupdate.clone(), + db: db.clone(), + } + } + + pub fn next_count(&self) -> Result { utils::u64_from_bytes(&self.global.increment(COUNTER)?) .map_err(|_| Error::bad_database("Count has invalid bytes.")) } - fn current_count(&self) -> Result { + pub fn current_count(&self) -> Result { self.global.get(COUNTER)?.map_or(Ok(0_u64), |bytes| { utils::u64_from_bytes(&bytes).map_err(|_| Error::bad_database("Count has invalid bytes.")) }) } - fn last_check_for_updates_id(&self) -> Result { + pub fn last_check_for_updates_id(&self) -> Result { self.global .get(LAST_CHECK_FOR_UPDATES_COUNT)? .map_or(Ok(0_u64), |bytes| { @@ -68,16 +75,15 @@ impl Data for KeyValueDatabase { }) } - fn update_check_for_updates_id(&self, id: u64) -> Result<()> { + pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { self.global .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes())?; Ok(()) } - #[allow(unused_qualifications)] // async traits #[tracing::instrument(skip(self))] - async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { + pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { let userid_bytes = user_id.as_bytes().to_vec(); let mut userid_prefix = userid_bytes.clone(); userid_prefix.push(0xFF); @@ -177,20 +183,20 @@ impl Data for KeyValueDatabase { Ok(()) } - fn cleanup(&self) -> Result<()> { self.db.cleanup() } + pub fn cleanup(&self) -> Result<()> { self.db.db.cleanup() } - fn cork(&self) -> Cork { Cork::new(&self.db, false, false) } + pub fn cork(&self) -> Cork { Cork::new(&self.db.db, false, false) } - fn cork_and_flush(&self) -> Cork { Cork::new(&self.db, true, false) } + pub fn cork_and_flush(&self) -> Cork { Cork::new(&self.db.db, true, false) } - fn memory_usage(&self) -> String { - let auth_chain_cache = self.auth_chain_cache.lock().unwrap().len(); - let appservice_in_room_cache = self.appservice_in_room_cache.read().unwrap().len(); - let lasttimelinecount_cache = self.lasttimelinecount_cache.lock().unwrap().len(); + pub fn memory_usage(&self) -> String { + let auth_chain_cache = self.db.auth_chain_cache.lock().unwrap().len(); + let appservice_in_room_cache = self.db.appservice_in_room_cache.read().unwrap().len(); + let lasttimelinecount_cache = self.db.lasttimelinecount_cache.lock().unwrap().len(); - let max_auth_chain_cache = self.auth_chain_cache.lock().unwrap().capacity(); - let max_appservice_in_room_cache = self.appservice_in_room_cache.read().unwrap().capacity(); - let max_lasttimelinecount_cache = self.lasttimelinecount_cache.lock().unwrap().capacity(); + let max_auth_chain_cache = self.db.auth_chain_cache.lock().unwrap().capacity(); + let max_appservice_in_room_cache = self.db.appservice_in_room_cache.read().unwrap().capacity(); + let max_lasttimelinecount_cache = self.db.lasttimelinecount_cache.lock().unwrap().capacity(); format!( "\ @@ -198,26 +204,26 @@ auth_chain_cache: {auth_chain_cache} / {max_auth_chain_cache} appservice_in_room_cache: {appservice_in_room_cache} / {max_appservice_in_room_cache} lasttimelinecount_cache: {lasttimelinecount_cache} / {max_lasttimelinecount_cache}\n\n {}", - self.db.memory_usage().unwrap_or_default() + self.db.db.memory_usage().unwrap_or_default() ) } - fn clear_caches(&self, amount: u32) { + pub fn clear_caches(&self, amount: u32) { if amount > 1 { - let c = &mut *self.auth_chain_cache.lock().unwrap(); + let c = &mut *self.db.auth_chain_cache.lock().unwrap(); *c = LruCache::new(c.capacity()); } if amount > 2 { - let c = &mut *self.appservice_in_room_cache.write().unwrap(); + let c = &mut *self.db.appservice_in_room_cache.write().unwrap(); *c = HashMap::new(); } if amount > 3 { - let c = &mut *self.lasttimelinecount_cache.lock().unwrap(); + let c = &mut *self.db.lasttimelinecount_cache.lock().unwrap(); *c = HashMap::new(); } } - fn load_keypair(&self) -> Result { + pub fn load_keypair(&self) -> Result { let keypair_bytes = self.global.get(b"keypair")?.map_or_else( || { let keypair = utils::generate_keypair(); @@ -249,9 +255,9 @@ lasttimelinecount_cache: {lasttimelinecount_cache} / {max_lasttimelinecount_cach }) } - fn remove_keypair(&self) -> Result<()> { self.global.remove(b"keypair") } + pub fn remove_keypair(&self) -> Result<()> { self.global.remove(b"keypair") } - fn add_signing_key( + pub fn add_signing_key( &self, origin: &ServerName, new_keys: ServerSigningKeys, ) -> Result> { // Not atomic, but this is not critical @@ -290,8 +296,9 @@ lasttimelinecount_cache: {lasttimelinecount_cache} / {max_lasttimelinecount_cach /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. - fn signing_keys_for(&self, origin: &ServerName) -> Result> { + pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { let signingkeys = self + .db .server_signingkeys .get(origin.as_bytes())? .and_then(|bytes| serde_json::from_slice(&bytes).ok()) @@ -308,20 +315,20 @@ lasttimelinecount_cache: {lasttimelinecount_cache} / {max_lasttimelinecount_cach Ok(signingkeys) } - fn database_version(&self) -> Result { + pub fn database_version(&self) -> Result { self.global.get(b"version")?.map_or(Ok(0), |version| { utils::u64_from_bytes(&version).map_err(|_| Error::bad_database("Database version id is invalid.")) }) } - fn bump_database_version(&self, new_version: u64) -> Result<()> { + pub fn bump_database_version(&self, new_version: u64) -> Result<()> { self.global.insert(b"version", &new_version.to_be_bytes())?; Ok(()) } - fn backup(&self) -> Result<(), Box> { self.db.backup() } + pub fn backup(&self) -> Result<(), Box> { self.db.db.backup() } - fn backup_list(&self) -> Result { self.db.backup_list() } + pub fn backup_list(&self) -> Result { self.db.db.backup_list() } - fn file_list(&self) -> Result { self.db.file_list() } + pub fn file_list(&self) -> Result { self.db.db.file_list() } } diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 2e106030..9a6aef18 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -1,3 +1,5 @@ +use conduit::Server; + mod client; mod data; pub(super) mod emerg_access; @@ -33,12 +35,12 @@ use tracing::{error, trace}; use url::Url; use utils::MutexMap; -use crate::{services, Config, Result}; +use crate::{services, Config, KeyValueDatabase, Result}; type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries pub struct Service { - pub db: Arc, + pub db: Data, pub config: Config, pub cidr_range_denylist: Vec, @@ -62,7 +64,9 @@ pub struct Service { } impl Service { - pub fn load(db: Arc, config: &Config) -> Result { + pub fn build(server: &Arc, db: &Arc) -> Result { + let config = &server.config; + let db = Data::new(db); let keypair = db.load_keypair(); let keypair = match keypair { diff --git a/src/service/key_backups/data.rs b/src/service/key_backups/data.rs index 491fd0e4..78042ccf 100644 --- a/src/service/key_backups/data.rs +++ b/src/service/key_backups/data.rs @@ -1,4 +1,4 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, sync::Arc}; use ruma::{ api::client::{ @@ -9,48 +9,24 @@ use ruma::{ OwnedRoomId, RoomId, UserId, }; -use crate::{services, utils, Error, KeyValueDatabase, Result}; +use crate::{services, utils, Error, KeyValueDatabase, KvTree, Result}; -pub(crate) trait Data: Send + Sync { - fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result; - - fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()>; - - fn update_backup(&self, user_id: &UserId, version: &str, backup_metadata: &Raw) -> Result; - - fn get_latest_backup_version(&self, user_id: &UserId) -> Result>; - - fn get_latest_backup(&self, user_id: &UserId) -> Result)>>; - - fn get_backup(&self, user_id: &UserId, version: &str) -> Result>>; - - fn add_key( - &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, key_data: &Raw, - ) -> Result<()>; - - fn count_keys(&self, user_id: &UserId, version: &str) -> Result; - - fn get_etag(&self, user_id: &UserId, version: &str) -> Result; - - fn get_all(&self, user_id: &UserId, version: &str) -> Result>; - - fn get_room( - &self, user_id: &UserId, version: &str, room_id: &RoomId, - ) -> Result>>; - - fn get_session( - &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, - ) -> Result>>; - - fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()>; - - fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) -> Result<()>; - - fn delete_room_key(&self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str) -> Result<()>; +pub(crate) struct Data { + backupid_algorithm: Arc, + backupid_etag: Arc, + backupkeyid_backup: Arc, } -impl Data for KeyValueDatabase { - fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + backupid_algorithm: db.backupid_algorithm.clone(), + backupid_etag: db.backupid_etag.clone(), + backupkeyid_backup: db.backupkeyid_backup.clone(), + } + } + + pub(super) fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { let version = services().globals.next_count()?.to_string(); let mut key = user_id.as_bytes().to_vec(); @@ -66,7 +42,7 @@ impl Data for KeyValueDatabase { Ok(version) } - fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> { + pub(super) fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(version.as_bytes()); @@ -83,7 +59,9 @@ impl Data for KeyValueDatabase { Ok(()) } - fn update_backup(&self, user_id: &UserId, version: &str, backup_metadata: &Raw) -> Result { + pub(super) fn update_backup( + &self, user_id: &UserId, version: &str, backup_metadata: &Raw, + ) -> Result { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(version.as_bytes()); @@ -99,7 +77,7 @@ impl Data for KeyValueDatabase { Ok(version.to_owned()) } - fn get_latest_backup_version(&self, user_id: &UserId) -> Result> { + pub(super) fn get_latest_backup_version(&self, user_id: &UserId) -> Result> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); let mut last_possible_key = prefix.clone(); @@ -120,7 +98,7 @@ impl Data for KeyValueDatabase { .transpose() } - fn get_latest_backup(&self, user_id: &UserId) -> Result)>> { + pub(super) fn get_latest_backup(&self, user_id: &UserId) -> Result)>> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); let mut last_possible_key = prefix.clone(); @@ -147,7 +125,7 @@ impl Data for KeyValueDatabase { .transpose() } - fn get_backup(&self, user_id: &UserId, version: &str) -> Result>> { + pub(super) fn get_backup(&self, user_id: &UserId, version: &str) -> Result>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(version.as_bytes()); @@ -160,7 +138,7 @@ impl Data for KeyValueDatabase { }) } - fn add_key( + pub(super) fn add_key( &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, key_data: &Raw, ) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); @@ -185,7 +163,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn count_keys(&self, user_id: &UserId, version: &str) -> Result { + pub(super) fn count_keys(&self, user_id: &UserId, version: &str) -> Result { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); prefix.extend_from_slice(version.as_bytes()); @@ -193,7 +171,7 @@ impl Data for KeyValueDatabase { Ok(self.backupkeyid_backup.scan_prefix(prefix).count()) } - fn get_etag(&self, user_id: &UserId, version: &str) -> Result { + pub(super) fn get_etag(&self, user_id: &UserId, version: &str) -> Result { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(version.as_bytes()); @@ -208,7 +186,7 @@ impl Data for KeyValueDatabase { .to_string()) } - fn get_all(&self, user_id: &UserId, version: &str) -> Result> { + pub(super) fn get_all(&self, user_id: &UserId, version: &str) -> Result> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); prefix.extend_from_slice(version.as_bytes()); @@ -257,7 +235,7 @@ impl Data for KeyValueDatabase { Ok(rooms) } - fn get_room( + pub(super) fn get_room( &self, user_id: &UserId, version: &str, room_id: &RoomId, ) -> Result>> { let mut prefix = user_id.as_bytes().to_vec(); @@ -289,7 +267,7 @@ impl Data for KeyValueDatabase { .collect()) } - fn get_session( + pub(super) fn get_session( &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, ) -> Result>> { let mut key = user_id.as_bytes().to_vec(); @@ -309,7 +287,7 @@ impl Data for KeyValueDatabase { .transpose() } - fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> { + pub(super) fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(version.as_bytes()); @@ -322,7 +300,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) -> Result<()> { + pub(super) fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(version.as_bytes()); @@ -337,7 +315,9 @@ impl Data for KeyValueDatabase { Ok(()) } - fn delete_room_key(&self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str) -> Result<()> { + pub(super) fn delete_room_key( + &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, + ) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(version.as_bytes()); diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 9b88e293..19b5d53d 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -1,20 +1,28 @@ +use conduit::Server; + mod data; use std::{collections::BTreeMap, sync::Arc}; +use conduit::Result; use data::Data; +use database::KeyValueDatabase; use ruma::{ api::client::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, serde::Raw, OwnedRoomId, RoomId, UserId, }; -use crate::Result; - pub struct Service { - pub(super) db: Arc, + pub(super) db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + pub fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { self.db.create_backup(user_id, backup_metadata) } diff --git a/src/service/media/data.rs b/src/service/media/data.rs index f2a83a53..7b8d0b5a 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -1,37 +1,28 @@ +use std::sync::Arc; + use conduit::debug_info; +use database::{KeyValueDatabase, KvTree}; use ruma::api::client::error::ErrorKind; use tracing::debug; -use crate::{media::UrlPreviewData, utils::string_from_bytes, Error, KeyValueDatabase, Result}; +use crate::{media::UrlPreviewData, utils::string_from_bytes, Error, Result}; -pub(crate) trait Data: Send + Sync { - fn create_file_metadata( - &self, sender_user: Option<&str>, mxc: String, width: u32, height: u32, content_disposition: Option<&str>, - content_type: Option<&str>, - ) -> Result>; - - fn delete_file_mxc(&self, mxc: String) -> Result<()>; - - /// Returns content_disposition, content_type and the metadata key. - fn search_file_metadata( - &self, mxc: String, width: u32, height: u32, - ) -> Result<(Option, Option, Vec)>; - - fn search_mxc_metadata_prefix(&self, mxc: String) -> Result>>; - - fn get_all_media_keys(&self) -> Vec>; - - // TODO: use this - #[allow(dead_code)] - fn remove_url_preview(&self, url: &str) -> Result<()>; - - fn set_url_preview(&self, url: &str, data: &UrlPreviewData, timestamp: std::time::Duration) -> Result<()>; - - fn get_url_preview(&self, url: &str) -> Option; +pub struct Data { + mediaid_file: Arc, + mediaid_user: Arc, + url_previews: Arc, } -impl Data for KeyValueDatabase { - fn create_file_metadata( +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + mediaid_file: db.mediaid_file.clone(), + mediaid_user: db.mediaid_user.clone(), + url_previews: db.url_previews.clone(), + } + } + + pub(super) fn create_file_metadata( &self, sender_user: Option<&str>, mxc: String, width: u32, height: u32, content_disposition: Option<&str>, content_type: Option<&str>, ) -> Result> { @@ -65,7 +56,7 @@ impl Data for KeyValueDatabase { Ok(key) } - fn delete_file_mxc(&self, mxc: String) -> Result<()> { + pub(super) fn delete_file_mxc(&self, mxc: String) -> Result<()> { debug!("MXC URI: {:?}", mxc); let mut prefix = mxc.as_bytes().to_vec(); @@ -91,7 +82,7 @@ impl Data for KeyValueDatabase { } /// Searches for all files with the given MXC - fn search_mxc_metadata_prefix(&self, mxc: String) -> Result>> { + pub(super) fn search_mxc_metadata_prefix(&self, mxc: String) -> Result>> { debug!("MXC URI: {:?}", mxc); let mut prefix = mxc.as_bytes().to_vec(); @@ -114,7 +105,7 @@ impl Data for KeyValueDatabase { Ok(keys) } - fn search_file_metadata( + pub(super) fn search_file_metadata( &self, mxc: String, width: u32, height: u32, ) -> Result<(Option, Option, Vec)> { let mut prefix = mxc.as_bytes().to_vec(); @@ -156,11 +147,13 @@ impl Data for KeyValueDatabase { /// Gets all the media keys in our database (this includes all the metadata /// associated with it such as width, height, content-type, etc) - fn get_all_media_keys(&self) -> Vec> { self.mediaid_file.iter().map(|(key, _)| key).collect() } + pub(crate) fn get_all_media_keys(&self) -> Vec> { self.mediaid_file.iter().map(|(key, _)| key).collect() } - fn remove_url_preview(&self, url: &str) -> Result<()> { self.url_previews.remove(url.as_bytes()) } + pub(super) fn remove_url_preview(&self, url: &str) -> Result<()> { self.url_previews.remove(url.as_bytes()) } - fn set_url_preview(&self, url: &str, data: &UrlPreviewData, timestamp: std::time::Duration) -> Result<()> { + pub(super) fn set_url_preview( + &self, url: &str, data: &UrlPreviewData, timestamp: std::time::Duration, + ) -> Result<()> { let mut value = Vec::::new(); value.extend_from_slice(×tamp.as_secs().to_be_bytes()); value.push(0xFF); @@ -194,7 +187,7 @@ impl Data for KeyValueDatabase { self.url_previews.insert(url.as_bytes(), &value) } - fn get_url_preview(&self, url: &str) -> Option { + pub(super) fn get_url_preview(&self, url: &str) -> Option { let values = self.url_previews.get(url.as_bytes()).ok()??; let mut values = values.split(|&b| b == 0xFF); diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 17a1a6dc..a6e2a2a9 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -44,17 +44,17 @@ pub struct UrlPreviewData { pub struct Service { server: Arc, - pub(super) db: Arc, + pub db: Data, pub url_preview_mutex: RwLock>>>, } impl Service { - pub fn build(server: &Arc, db: &Arc) -> Self { - Self { + pub fn build(server: &Arc, db: &Arc) -> Result { + Ok(Self { server: server.clone(), - db: db.clone(), + db: Data::new(db), url_preview_mutex: RwLock::new(HashMap::new()), - } + }) } /// Uploads a file. diff --git a/src/service/mod.rs b/src/service/mod.rs index 2178316a..d1db2c25 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -20,7 +20,7 @@ extern crate conduit_database as database; use std::sync::{Arc, RwLock}; pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, PduCount, Result, Server}; -pub(crate) use database::KeyValueDatabase; +pub(crate) use database::{KeyValueDatabase, KvTree}; pub use globals::{server_is_ours, user_is_local}; pub use pdu::PduEvent; pub use services::Services; diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index bf730398..ade891a9 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -1,4 +1,7 @@ +use std::sync::Arc; + use conduit::debug_warn; +use database::KvTree; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; use crate::{ @@ -8,26 +11,20 @@ use crate::{ Error, KeyValueDatabase, Result, }; -pub trait Data: Send + Sync { - /// Returns the latest presence event for the given user. - fn get_presence(&self, user_id: &UserId) -> Result>; - - /// Adds a presence event which will be saved until a new event replaces it. - fn set_presence( - &self, user_id: &UserId, presence_state: &PresenceState, currently_active: Option, - last_active_ago: Option, status_msg: Option, - ) -> Result<()>; - - /// Removes the presence record for the given user from the database. - fn remove_presence(&self, user_id: &UserId) -> Result<()>; - - /// Returns the most recent presence updates that happened after the event - /// with id `since`. - fn presence_since<'a>(&'a self, since: u64) -> Box)> + 'a>; +pub struct Data { + presenceid_presence: Arc, + userid_presenceid: Arc, } -impl Data for KeyValueDatabase { - fn get_presence(&self, user_id: &UserId) -> Result> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + presenceid_presence: db.presenceid_presence.clone(), + userid_presenceid: db.userid_presenceid.clone(), + } + } + + pub fn get_presence(&self, user_id: &UserId) -> Result> { if let Some(count_bytes) = self.userid_presenceid.get(user_id.as_bytes())? { let count = utils::u64_from_bytes(&count_bytes) .map_err(|_e| Error::bad_database("No 'count' bytes in presence key"))?; @@ -44,7 +41,7 @@ impl Data for KeyValueDatabase { } } - fn set_presence( + pub(super) fn set_presence( &self, user_id: &UserId, presence_state: &PresenceState, currently_active: Option, last_active_ago: Option, status_msg: Option, ) -> Result<()> { @@ -105,7 +102,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn remove_presence(&self, user_id: &UserId) -> Result<()> { + pub(super) fn remove_presence(&self, user_id: &UserId) -> Result<()> { if let Some(count_bytes) = self.userid_presenceid.get(user_id.as_bytes())? { let count = utils::u64_from_bytes(&count_bytes) .map_err(|_e| Error::bad_database("No 'count' bytes in presence key"))?; @@ -117,7 +114,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn presence_since<'a>(&'a self, since: u64) -> Box)> + 'a> { + pub fn presence_since<'a>(&'a self, since: u64) -> Box)> + 'a> { Box::new( self.presenceid_presence .iter() diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index de5030f6..046614c9 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -1,3 +1,5 @@ +use conduit::Server; + mod data; use std::{sync::Arc, time::Duration}; @@ -14,9 +16,10 @@ use tokio::{sync::Mutex, task::JoinHandle, time::sleep}; use tracing::{debug, error}; use crate::{ + database::KeyValueDatabase, services, user_is_local, utils::{self}, - Config, Error, Result, + Error, Result, }; /// Represents data required to be kept in order to implement the presence @@ -77,7 +80,7 @@ impl Presence { } pub struct Service { - pub db: Arc, + pub db: Data, pub timer_sender: loole::Sender<(OwnedUserId, Duration)>, timer_receiver: Mutex>, handler_join: Mutex>>, @@ -85,15 +88,16 @@ pub struct Service { } impl Service { - pub fn build(db: Arc, config: &Config) -> Arc { + pub fn build(server: &Arc, db: &Arc) -> Result> { + let config = &server.config; let (timer_sender, timer_receiver) = loole::unbounded(); - Arc::new(Self { - db, + Ok(Arc::new(Self { + db: Data::new(db), timer_sender, timer_receiver: Mutex::new(timer_receiver), handler_join: Mutex::new(None), timeout_remote_users: config.presence_timeout_remote_users, - }) + })) } pub async fn start_handler(self: &Arc) { diff --git a/src/service/pusher/data.rs b/src/service/pusher/data.rs index d8c75f4d..0892073f 100644 --- a/src/service/pusher/data.rs +++ b/src/service/pusher/data.rs @@ -1,22 +1,25 @@ +use std::sync::Arc; + +use database::{KeyValueDatabase, KvTree}; use ruma::{ api::client::push::{set_pusher, Pusher}, UserId, }; -use crate::{utils, Error, KeyValueDatabase, Result}; +use crate::{utils, Error, Result}; -pub(crate) trait Data: Send + Sync { - fn set_pusher(&self, sender: &UserId, pusher: set_pusher::v3::PusherAction) -> Result<()>; - - fn get_pusher(&self, sender: &UserId, pushkey: &str) -> Result>; - - fn get_pushers(&self, sender: &UserId) -> Result>; - - fn get_pushkeys<'a>(&'a self, sender: &UserId) -> Box> + 'a>; +pub struct Data { + senderkey_pusher: Arc, } -impl Data for KeyValueDatabase { - fn set_pusher(&self, sender: &UserId, pusher: set_pusher::v3::PusherAction) -> Result<()> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + senderkey_pusher: db.senderkey_pusher.clone(), + } + } + + pub(super) fn set_pusher(&self, sender: &UserId, pusher: set_pusher::v3::PusherAction) -> Result<()> { match &pusher { set_pusher::v3::PusherAction::Post(data) => { let mut key = sender.as_bytes().to_vec(); @@ -35,7 +38,7 @@ impl Data for KeyValueDatabase { } } - fn get_pusher(&self, sender: &UserId, pushkey: &str) -> Result> { + pub(super) fn get_pusher(&self, sender: &UserId, pushkey: &str) -> Result> { let mut senderkey = sender.as_bytes().to_vec(); senderkey.push(0xFF); senderkey.extend_from_slice(pushkey.as_bytes()); @@ -46,7 +49,7 @@ impl Data for KeyValueDatabase { .transpose() } - fn get_pushers(&self, sender: &UserId) -> Result> { + pub(super) fn get_pushers(&self, sender: &UserId) -> Result> { let mut prefix = sender.as_bytes().to_vec(); prefix.push(0xFF); @@ -56,7 +59,7 @@ impl Data for KeyValueDatabase { .collect() } - fn get_pushkeys<'a>(&'a self, sender: &UserId) -> Box> + 'a> { + pub(super) fn get_pushkeys<'a>(&'a self, sender: &UserId) -> Box> + 'a> { let mut prefix = sender.as_bytes().to_vec(); prefix.push(0xFF); diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 5736ef1d..ff167918 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::{fmt::Debug, mem, sync::Arc}; @@ -25,10 +28,16 @@ use tracing::{info, trace, warn}; use crate::{debug_info, services, Error, PduEvent, Result}; pub struct Service { - pub(super) db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + pub fn set_pusher(&self, sender: &UserId, pusher: set_pusher::v3::PusherAction) -> Result<()> { self.db.set_pusher(sender, pusher) } diff --git a/src/service/rooms/alias/data.rs b/src/service/rooms/alias/data.rs index e0f4335e..bc0c9cc6 100644 --- a/src/service/rooms/alias/data.rs +++ b/src/service/rooms/alias/data.rs @@ -1,31 +1,26 @@ +use std::sync::Arc; + +use database::KvTree; use ruma::{api::client::error::ErrorKind, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, UserId}; use crate::{services, utils, Error, KeyValueDatabase, Result}; -pub trait Data: Send + Sync { - /// Creates or updates the alias to the given room id. - fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()>; - - /// Forgets about an alias. Returns an error if the alias did not exist. - fn remove_alias(&self, alias: &RoomAliasId) -> Result<()>; - - /// Looks up the roomid for the given alias. - fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result>; - - /// Finds the user who assigned the given alias to a room - fn who_created_alias(&self, alias: &RoomAliasId) -> Result>; - - /// Returns all local aliases that point to the given room - fn local_aliases_for_room<'a>( - &'a self, room_id: &RoomId, - ) -> Box> + 'a>; - - /// Returns all local aliases on the server - fn all_local_aliases<'a>(&'a self) -> Box> + 'a>; +pub struct Data { + alias_userid: Arc, + alias_roomid: Arc, + aliasid_alias: Arc, } -impl Data for KeyValueDatabase { - fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + alias_userid: db.alias_userid.clone(), + alias_roomid: db.alias_roomid.clone(), + aliasid_alias: db.aliasid_alias.clone(), + } + } + + pub(super) fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { // Comes first as we don't want a stuck alias self.alias_userid .insert(alias.alias().as_bytes(), user_id.as_bytes())?; @@ -41,7 +36,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn remove_alias(&self, alias: &RoomAliasId) -> Result<()> { + pub(super) fn remove_alias(&self, alias: &RoomAliasId) -> Result<()> { if let Some(room_id) = self.alias_roomid.get(alias.alias().as_bytes())? { let mut prefix = room_id; prefix.push(0xFF); @@ -60,7 +55,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { + pub fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { self.alias_roomid .get(alias.alias().as_bytes())? .map(|bytes| { @@ -73,7 +68,7 @@ impl Data for KeyValueDatabase { .transpose() } - fn who_created_alias(&self, alias: &RoomAliasId) -> Result> { + pub(super) fn who_created_alias(&self, alias: &RoomAliasId) -> Result> { self.alias_userid .get(alias.alias().as_bytes())? .map(|bytes| { @@ -86,7 +81,7 @@ impl Data for KeyValueDatabase { .transpose() } - fn local_aliases_for_room<'a>( + pub fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { let mut prefix = room_id.as_bytes().to_vec(); @@ -100,7 +95,7 @@ impl Data for KeyValueDatabase { })) } - fn all_local_aliases<'a>(&'a self) -> Box> + 'a> { + pub fn all_local_aliases<'a>(&'a self) -> Box> + 'a> { Box::new( self.alias_roomid .iter() diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 80f125dc..5c4f5cdb 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -15,10 +18,16 @@ use ruma::{ use crate::{services, Error, Result}; pub struct Service { - pub db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + #[tracing::instrument(skip(self))] pub fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { if alias == services().globals.admin_alias && user_id != services().globals.server_user { diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index c3f046fc..bb6e6516 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -1,16 +1,25 @@ use std::{mem::size_of, sync::Arc}; +use database::KvTree; + use crate::{utils, KeyValueDatabase, Result}; -pub trait Data: Send + Sync { - fn get_cached_eventid_authchain(&self, shorteventid: &[u64]) -> Result>>; - fn cache_auth_chain(&self, shorteventid: Vec, auth_chain: Arc<[u64]>) -> Result<()>; +pub(super) struct Data { + shorteventid_authchain: Arc, + db: Arc, } -impl Data for KeyValueDatabase { - fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result>> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + shorteventid_authchain: db.shorteventid_authchain.clone(), + db: db.clone(), + } + } + + pub(super) fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result>> { // Check RAM cache - if let Some(result) = self.auth_chain_cache.lock().unwrap().get_mut(key) { + if let Some(result) = self.db.auth_chain_cache.lock().unwrap().get_mut(key) { return Ok(Some(Arc::clone(result))); } @@ -29,7 +38,8 @@ impl Data for KeyValueDatabase { if let Some(chain) = chain { // Cache in RAM - self.auth_chain_cache + self.db + .auth_chain_cache .lock() .unwrap() .insert(vec![key[0]], Arc::clone(&chain)); @@ -41,7 +51,7 @@ impl Data for KeyValueDatabase { Ok(None) } - fn cache_auth_chain(&self, key: Vec, auth_chain: Arc<[u64]>) -> Result<()> { + pub(super) fn cache_auth_chain(&self, key: Vec, auth_chain: Arc<[u64]>) -> Result<()> { // Only persist single events in db if key.len() == 1 { self.shorteventid_authchain.insert( @@ -54,7 +64,8 @@ impl Data for KeyValueDatabase { } // Cache in RAM - self.auth_chain_cache + self.db + .auth_chain_cache .lock() .unwrap() .insert(key, auth_chain); diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index b93ea8c5..6cadee0f 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::{ collections::{BTreeSet, HashSet}, @@ -11,10 +14,16 @@ use tracing::{debug, error, trace, warn}; use crate::{services, Error, Result}; pub struct Service { - pub db: Arc, + db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + pub async fn event_ids_iter<'a>( &self, room_id: &RoomId, starting_events_: Vec>, ) -> Result> + 'a> { diff --git a/src/service/rooms/directory/data.rs b/src/service/rooms/directory/data.rs index 67afbb6c..f4c453eb 100644 --- a/src/service/rooms/directory/data.rs +++ b/src/service/rooms/directory/data.rs @@ -1,31 +1,34 @@ +use std::sync::Arc; + +use database::KvTree; use ruma::{OwnedRoomId, RoomId}; use crate::{utils, Error, KeyValueDatabase, Result}; -pub trait Data: Send + Sync { - /// Adds the room to the public room directory - fn set_public(&self, room_id: &RoomId) -> Result<()>; - - /// Removes the room from the public room directory. - fn set_not_public(&self, room_id: &RoomId) -> Result<()>; - - /// Returns true if the room is in the public room directory. - fn is_public_room(&self, room_id: &RoomId) -> Result; - - /// Returns the unsorted public room directory - fn public_rooms<'a>(&'a self) -> Box> + 'a>; +pub(super) struct Data { + publicroomids: Arc, } -impl Data for KeyValueDatabase { - fn set_public(&self, room_id: &RoomId) -> Result<()> { self.publicroomids.insert(room_id.as_bytes(), &[]) } +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + publicroomids: db.publicroomids.clone(), + } + } - fn set_not_public(&self, room_id: &RoomId) -> Result<()> { self.publicroomids.remove(room_id.as_bytes()) } + pub(super) fn set_public(&self, room_id: &RoomId) -> Result<()> { + self.publicroomids.insert(room_id.as_bytes(), &[]) + } - fn is_public_room(&self, room_id: &RoomId) -> Result { + pub(super) fn set_not_public(&self, room_id: &RoomId) -> Result<()> { + self.publicroomids.remove(room_id.as_bytes()) + } + + pub(super) fn is_public_room(&self, room_id: &RoomId) -> Result { Ok(self.publicroomids.get(room_id.as_bytes())?.is_some()) } - fn public_rooms<'a>(&'a self) -> Box> + 'a> { + pub(super) fn public_rooms<'a>(&'a self) -> Box> + 'a> { Box::new(self.publicroomids.iter().map(|(bytes, _)| { RoomId::parse( utils::string_from_bytes(&bytes) diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 85909c74..a0c4caf7 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -8,10 +11,16 @@ use ruma::{OwnedRoomId, RoomId}; use crate::Result; pub struct Service { - pub db: Arc, + db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + #[tracing::instrument(skip(self))] pub fn set_public(&self, room_id: &RoomId) -> Result<()> { self.db.set_public(room_id) } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 813b8d4a..c7e819ac 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1,6 +1,8 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod parse_incoming_pdu; mod signing_keys; -pub struct Service; use std::{ cmp, @@ -32,6 +34,8 @@ use tracing::{debug, error, info, trace, warn}; use super::state_compressor::CompressedStateEvent; use crate::{debug_error, debug_info, pdu, services, Error, PduEvent, Result}; +pub struct Service; + // We use some AsyncRecursiveType hacks here so we can call async funtion // recursively. type AsyncRecursiveType<'a, T> = Pin + 'a + Send>>; @@ -41,6 +45,8 @@ type AsyncRecursiveCanonicalJsonResult<'a> = AsyncRecursiveType<'a, Result<(Arc, BTreeMap)>>; impl Service { + pub fn build(_server: &Arc, _db: &Arc) -> Result { Ok(Self {}) } + /// When receiving an event one needs to: /// 0. Check the server is in the room /// 1. Skip the PDU if we already know about it diff --git a/src/service/rooms/lazy_loading/data.rs b/src/service/rooms/lazy_loading/data.rs index 7c237901..63823c4a 100644 --- a/src/service/rooms/lazy_loading/data.rs +++ b/src/service/rooms/lazy_loading/data.rs @@ -1,22 +1,22 @@ +use std::sync::Arc; + +use database::KvTree; use ruma::{DeviceId, RoomId, UserId}; use crate::{KeyValueDatabase, Result}; -pub trait Data: Send + Sync { - fn lazy_load_was_sent_before( - &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, - ) -> Result; - - fn lazy_load_confirm_delivery( - &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, - confirmed_user_ids: &mut dyn Iterator, - ) -> Result<()>; - - fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId) -> Result<()>; +pub struct Data { + lazyloadedids: Arc, } -impl Data for KeyValueDatabase { - fn lazy_load_was_sent_before( +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + lazyloadedids: db.lazyloadedids.clone(), + } + } + + pub(super) fn lazy_load_was_sent_before( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, ) -> Result { let mut key = user_id.as_bytes().to_vec(); @@ -29,7 +29,7 @@ impl Data for KeyValueDatabase { Ok(self.lazyloadedids.get(&key)?.is_some()) } - fn lazy_load_confirm_delivery( + pub(super) fn lazy_load_confirm_delivery( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, confirmed_user_ids: &mut dyn Iterator, ) -> Result<()> { @@ -49,7 +49,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId) -> Result<()> { + pub(super) fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId) -> Result<()> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); prefix.extend_from_slice(device_id.as_bytes()); diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 283a03c1..dba78500 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::{ collections::{HashMap, HashSet}, @@ -11,13 +14,20 @@ use tokio::sync::Mutex; use crate::{PduCount, Result}; pub struct Service { - pub db: Arc, + pub db: Data, #[allow(clippy::type_complexity)] pub lazy_load_waiting: Mutex>>, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + lazy_load_waiting: Mutex::new(HashMap::new()), + }) + } + #[tracing::instrument(skip(self))] pub fn lazy_load_was_sent_before( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, diff --git a/src/service/rooms/metadata/data.rs b/src/service/rooms/metadata/data.rs index a6bb701e..b5f2fcc3 100644 --- a/src/service/rooms/metadata/data.rs +++ b/src/service/rooms/metadata/data.rs @@ -1,20 +1,29 @@ +use std::sync::Arc; + +use database::KvTree; use ruma::{OwnedRoomId, RoomId}; use tracing::error; use crate::{services, utils, Error, KeyValueDatabase, Result}; -pub trait Data: Send + Sync { - fn exists(&self, room_id: &RoomId) -> Result; - fn iter_ids<'a>(&'a self) -> Box> + 'a>; - fn is_disabled(&self, room_id: &RoomId) -> Result; - fn disable_room(&self, room_id: &RoomId, disabled: bool) -> Result<()>; - fn is_banned(&self, room_id: &RoomId) -> Result; - fn ban_room(&self, room_id: &RoomId, banned: bool) -> Result<()>; - fn list_banned_rooms<'a>(&'a self) -> Box> + 'a>; +pub struct Data { + disabledroomids: Arc, + bannedroomids: Arc, + roomid_shortroomid: Arc, + pduid_pdu: Arc, } -impl Data for KeyValueDatabase { - fn exists(&self, room_id: &RoomId) -> Result { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + disabledroomids: db.disabledroomids.clone(), + bannedroomids: db.bannedroomids.clone(), + roomid_shortroomid: db.roomid_shortroomid.clone(), + pduid_pdu: db.pduid_pdu.clone(), + } + } + + pub(super) fn exists(&self, room_id: &RoomId) -> Result { let prefix = match services().rooms.short.get_shortroomid(room_id)? { Some(b) => b.to_be_bytes().to_vec(), None => return Ok(false), @@ -29,7 +38,7 @@ impl Data for KeyValueDatabase { .is_some()) } - fn iter_ids<'a>(&'a self) -> Box> + 'a> { + pub(super) fn iter_ids<'a>(&'a self) -> Box> + 'a> { Box::new(self.roomid_shortroomid.iter().map(|(bytes, _)| { RoomId::parse( utils::string_from_bytes(&bytes) @@ -39,11 +48,11 @@ impl Data for KeyValueDatabase { })) } - fn is_disabled(&self, room_id: &RoomId) -> Result { + pub(super) fn is_disabled(&self, room_id: &RoomId) -> Result { Ok(self.disabledroomids.get(room_id.as_bytes())?.is_some()) } - fn disable_room(&self, room_id: &RoomId, disabled: bool) -> Result<()> { + pub(super) fn disable_room(&self, room_id: &RoomId, disabled: bool) -> Result<()> { if disabled { self.disabledroomids.insert(room_id.as_bytes(), &[])?; } else { @@ -53,9 +62,11 @@ impl Data for KeyValueDatabase { Ok(()) } - fn is_banned(&self, room_id: &RoomId) -> Result { Ok(self.bannedroomids.get(room_id.as_bytes())?.is_some()) } + pub(super) fn is_banned(&self, room_id: &RoomId) -> Result { + Ok(self.bannedroomids.get(room_id.as_bytes())?.is_some()) + } - fn ban_room(&self, room_id: &RoomId, banned: bool) -> Result<()> { + pub(super) fn ban_room(&self, room_id: &RoomId, banned: bool) -> Result<()> { if banned { self.bannedroomids.insert(room_id.as_bytes(), &[])?; } else { @@ -65,7 +76,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn list_banned_rooms<'a>(&'a self) -> Box> + 'a> { + pub(super) fn list_banned_rooms<'a>(&'a self) -> Box> + 'a> { Box::new(self.bannedroomids.iter().map( |(room_id_bytes, _ /* non-banned rooms should not be in this table */)| { let room_id = utils::string_from_bytes(&room_id_bytes) diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 2a6f7724..21f946b6 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -8,10 +11,16 @@ use ruma::{OwnedRoomId, RoomId}; use crate::Result; pub struct Service { - pub db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + /// Checks if a room exists. #[tracing::instrument(skip(self))] pub fn exists(&self, room_id: &RoomId) -> Result { self.db.exists(room_id) } diff --git a/src/service/rooms/outlier/data.rs b/src/service/rooms/outlier/data.rs index a278161c..f89e9fb4 100644 --- a/src/service/rooms/outlier/data.rs +++ b/src/service/rooms/outlier/data.rs @@ -1,15 +1,22 @@ +use std::sync::Arc; + +use database::KvTree; use ruma::{CanonicalJsonObject, EventId}; use crate::{Error, KeyValueDatabase, PduEvent, Result}; -pub trait Data: Send + Sync { - fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result>; - fn get_outlier_pdu(&self, event_id: &EventId) -> Result>; - fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) -> Result<()>; +pub struct Data { + eventid_outlierpdu: Arc, } -impl Data for KeyValueDatabase { - fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + eventid_outlierpdu: db.eventid_outlierpdu.clone(), + } + } + + pub(super) fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result> { self.eventid_outlierpdu .get(event_id.as_bytes())? .map_or(Ok(None), |pdu| { @@ -17,7 +24,7 @@ impl Data for KeyValueDatabase { }) } - fn get_outlier_pdu(&self, event_id: &EventId) -> Result> { + pub(super) fn get_outlier_pdu(&self, event_id: &EventId) -> Result> { self.eventid_outlierpdu .get(event_id.as_bytes())? .map_or(Ok(None), |pdu| { @@ -25,7 +32,7 @@ impl Data for KeyValueDatabase { }) } - fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) -> Result<()> { + pub(super) fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) -> Result<()> { self.eventid_outlierpdu.insert( event_id.as_bytes(), &serde_json::to_vec(&pdu).expect("CanonicalJsonObject is valid"), diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 3e8b4ed9..ee1f9466 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -8,10 +11,16 @@ use ruma::{CanonicalJsonObject, EventId}; use crate::{PduEvent, Result}; pub struct Service { - pub db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + /// Returns the pdu from the outlier tree. pub fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result> { self.db.get_outlier_pdu_json(event_id) diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index e0309172..e652c9fe 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,30 +1,33 @@ use std::{mem::size_of, sync::Arc}; +use database::KvTree; use ruma::{EventId, RoomId, UserId}; use crate::{services, utils, Error, KeyValueDatabase, PduCount, PduEvent, Result}; -pub trait Data: Send + Sync { - fn add_relation(&self, from: u64, to: u64) -> Result<()>; - #[allow(clippy::type_complexity)] - fn relations_until<'a>( - &'a self, user_id: &'a UserId, room_id: u64, target: u64, until: PduCount, - ) -> Result> + 'a>>; - fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) -> Result<()>; - fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> Result; - fn mark_event_soft_failed(&self, event_id: &EventId) -> Result<()>; - fn is_event_soft_failed(&self, event_id: &EventId) -> Result; +pub(super) struct Data { + tofrom_relation: Arc, + referencedevents: Arc, + softfailedeventids: Arc, } -impl Data for KeyValueDatabase { - fn add_relation(&self, from: u64, to: u64) -> Result<()> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + tofrom_relation: db.tofrom_relation.clone(), + referencedevents: db.referencedevents.clone(), + softfailedeventids: db.softfailedeventids.clone(), + } + } + + pub(super) fn add_relation(&self, from: u64, to: u64) -> Result<()> { let mut key = to.to_be_bytes().to_vec(); key.extend_from_slice(&from.to_be_bytes()); self.tofrom_relation.insert(&key, &[])?; Ok(()) } - fn relations_until<'a>( + pub(super) fn relations_until<'a>( &'a self, user_id: &'a UserId, shortroomid: u64, target: u64, until: PduCount, ) -> Result> + 'a>> { let prefix = target.to_be_bytes().to_vec(); @@ -63,7 +66,7 @@ impl Data for KeyValueDatabase { )) } - fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) -> Result<()> { + pub(super) fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) -> Result<()> { for prev in event_ids { let mut key = room_id.as_bytes().to_vec(); key.extend_from_slice(prev.as_bytes()); @@ -73,17 +76,17 @@ impl Data for KeyValueDatabase { Ok(()) } - fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> Result { + pub(super) fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> Result { let mut key = room_id.as_bytes().to_vec(); key.extend_from_slice(event_id.as_bytes()); Ok(self.referencedevents.get(&key)?.is_some()) } - fn mark_event_soft_failed(&self, event_id: &EventId) -> Result<()> { + pub(super) fn mark_event_soft_failed(&self, event_id: &EventId) -> Result<()> { self.softfailedeventids.insert(event_id.as_bytes(), &[]) } - fn is_event_soft_failed(&self, event_id: &EventId) -> Result { + pub(super) fn is_event_soft_failed(&self, event_id: &EventId) -> Result { self.softfailedeventids .get(event_id.as_bytes()) .map(|o| o.is_some()) diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index c4f45848..488cf03e 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -13,7 +16,7 @@ use serde::Deserialize; use crate::{services, PduCount, PduEvent, Result}; pub struct Service { - pub db: Arc, + db: Data, } #[derive(Clone, Debug, Deserialize)] @@ -27,6 +30,12 @@ struct ExtractRelatesToEventId { } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + #[tracing::instrument(skip(self, from, to))] pub fn add_relation(&self, from: PduCount, to: PduCount) -> Result<()> { match (from, to) { diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 702efd98..cf9d0ee3 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -1,5 +1,6 @@ -use std::mem::size_of; +use std::{mem::size_of, sync::Arc}; +use database::KvTree; use ruma::{ events::{receipt::ReceiptEvent, AnySyncEphemeralRoomEvent}, serde::Raw, @@ -11,32 +12,22 @@ use crate::{services, utils, Error, KeyValueDatabase, Result}; type AnySyncEphemeralRoomEventIter<'a> = Box)>> + 'a>; -pub trait Data: Send + Sync { - /// Replaces the previous read receipt. - fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: ReceiptEvent) -> Result<()>; - - /// Returns an iterator over the most recent read_receipts in a room that - /// happened after the event with id `since`. - fn readreceipts_since(&self, room_id: &RoomId, since: u64) -> AnySyncEphemeralRoomEventIter<'_>; - - /// Sets a private read marker at `count`. - fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) -> Result<()>; - - /// Returns the private read marker. - /// - /// TODO: use this? - #[allow(dead_code)] - fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result>; - - /// Returns the count of the last typing update in this room. - /// - /// TODO: use this? - #[allow(dead_code)] - fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result; +pub(super) struct Data { + roomuserid_privateread: Arc, + roomuserid_lastprivatereadupdate: Arc, + readreceiptid_readreceipt: Arc, } -impl Data for KeyValueDatabase { - fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: ReceiptEvent) -> Result<()> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + roomuserid_privateread: db.roomuserid_privateread.clone(), + roomuserid_lastprivatereadupdate: db.roomuserid_lastprivatereadupdate.clone(), + readreceiptid_readreceipt: db.readreceiptid_readreceipt.clone(), + } + } + + pub(super) fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: ReceiptEvent) -> Result<()> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -71,9 +62,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn readreceipts_since<'a>( - &'a self, room_id: &RoomId, since: u64, - ) -> Box)>> + 'a> { + pub(super) fn readreceipts_since<'a>(&'a self, room_id: &RoomId, since: u64) -> AnySyncEphemeralRoomEventIter<'a> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); let prefix2 = prefix.clone(); @@ -107,7 +96,7 @@ impl Data for KeyValueDatabase { ) } - fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) -> Result<()> { + pub(super) fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) -> Result<()> { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(user_id.as_bytes()); @@ -119,7 +108,7 @@ impl Data for KeyValueDatabase { .insert(&key, &services().globals.next_count()?.to_be_bytes()) } - fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { + pub(super) fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(user_id.as_bytes()); @@ -133,7 +122,7 @@ impl Data for KeyValueDatabase { }) } - fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result { + pub(super) fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(user_id.as_bytes()); diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 9afc1fd2..56401fe4 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -8,10 +11,16 @@ use ruma::{events::receipt::ReceiptEvent, serde::Raw, OwnedUserId, RoomId, UserI use crate::{services, Result}; pub struct Service { - pub db: Arc, + db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + /// Replaces the previous read receipt. pub fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: ReceiptEvent) -> Result<()> { self.db.readreceipt_update(user_id, room_id, event)?; diff --git a/src/service/rooms/search/data.rs b/src/service/rooms/search/data.rs index 7fa49989..ff42cc9f 100644 --- a/src/service/rooms/search/data.rs +++ b/src/service/rooms/search/data.rs @@ -1,30 +1,24 @@ +use std::sync::Arc; + +use database::KvTree; use ruma::RoomId; use crate::{services, utils, KeyValueDatabase, Result}; type SearchPdusResult<'a> = Result> + 'a>, Vec)>>; -pub trait Data: Send + Sync { - fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()>; - - fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()>; - - fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a>; +pub struct Data { + tokenids: Arc, } -/// Splits a string into tokens used as keys in the search inverted index -/// -/// This may be used to tokenize both message bodies (for indexing) or search -/// queries (for querying). -fn tokenize(body: &str) -> impl Iterator + '_ { - body.split_terminator(|c: char| !c.is_alphanumeric()) - .filter(|s| !s.is_empty()) - .filter(|word| word.len() <= 50) - .map(str::to_lowercase) -} +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + tokenids: db.tokenids.clone(), + } + } -impl Data for KeyValueDatabase { - fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { + pub(super) fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { let mut batch = tokenize(message_body).map(|word| { let mut key = shortroomid.to_be_bytes().to_vec(); key.extend_from_slice(word.as_bytes()); @@ -36,7 +30,7 @@ impl Data for KeyValueDatabase { self.tokenids.insert_batch(&mut batch) } - fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { + pub(super) fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { let batch = tokenize(message_body).map(|word| { let mut key = shortroomid.to_be_bytes().to_vec(); key.extend_from_slice(word.as_bytes()); @@ -52,7 +46,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a> { + pub(super) fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a> { let prefix = services() .rooms .short @@ -88,3 +82,14 @@ impl Data for KeyValueDatabase { Ok(Some((Box::new(common_elements), words))) } } + +/// Splits a string into tokens used as keys in the search inverted index +/// +/// This may be used to tokenize both message bodies (for indexing) or search +/// queries (for querying). +fn tokenize(body: &str) -> impl Iterator + '_ { + body.split_terminator(|c: char| !c.is_alphanumeric()) + .filter(|s| !s.is_empty()) + .filter(|word| word.len() <= 50) + .map(str::to_lowercase) +} diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 9e7f14c7..9762c8fc 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -8,10 +11,16 @@ use ruma::RoomId; use crate::Result; pub struct Service { - pub db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + #[tracing::instrument(skip(self))] pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { self.db.index_pdu(shortroomid, pdu_id, message_body) diff --git a/src/service/rooms/short/data.rs b/src/service/rooms/short/data.rs index 11ac39b5..13088758 100644 --- a/src/service/rooms/short/data.rs +++ b/src/service/rooms/short/data.rs @@ -1,33 +1,33 @@ use std::sync::Arc; +use conduit::warn; +use database::{KeyValueDatabase, KvTree}; use ruma::{events::StateEventType, EventId, RoomId}; -use tracing::warn; -use crate::{services, utils, Error, KeyValueDatabase, Result}; +use crate::{services, utils, Error, Result}; -pub trait Data: Send + Sync { - fn get_or_create_shorteventid(&self, event_id: &EventId) -> Result; - - fn multi_get_or_create_shorteventid(&self, event_id: &[&EventId]) -> Result>; - - fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result>; - - fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result; - - fn get_eventid_from_short(&self, shorteventid: u64) -> Result>; - - fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)>; - - /// Returns (shortstatehash, already_existed) - fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> Result<(u64, bool)>; - - fn get_shortroomid(&self, room_id: &RoomId) -> Result>; - - fn get_or_create_shortroomid(&self, room_id: &RoomId) -> Result; +pub(super) struct Data { + eventid_shorteventid: Arc, + shorteventid_eventid: Arc, + statekey_shortstatekey: Arc, + shortstatekey_statekey: Arc, + roomid_shortroomid: Arc, + statehash_shortstatehash: Arc, } -impl Data for KeyValueDatabase { - fn get_or_create_shorteventid(&self, event_id: &EventId) -> Result { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + eventid_shorteventid: db.eventid_shorteventid.clone(), + shorteventid_eventid: db.shorteventid_eventid.clone(), + statekey_shortstatekey: db.statekey_shortstatekey.clone(), + shortstatekey_statekey: db.shortstatekey_statekey.clone(), + roomid_shortroomid: db.roomid_shortroomid.clone(), + statehash_shortstatehash: db.statehash_shortstatehash.clone(), + } + } + + pub(super) fn get_or_create_shorteventid(&self, event_id: &EventId) -> Result { let short = if let Some(shorteventid) = self.eventid_shorteventid.get(event_id.as_bytes())? { utils::u64_from_bytes(&shorteventid).map_err(|_| Error::bad_database("Invalid shorteventid in db."))? } else { @@ -42,7 +42,7 @@ impl Data for KeyValueDatabase { Ok(short) } - fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Result> { + pub(super) fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Result> { let mut ret: Vec = Vec::with_capacity(event_ids.len()); let keys = event_ids .iter() @@ -74,7 +74,7 @@ impl Data for KeyValueDatabase { Ok(ret) } - fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result> { + pub(super) fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result> { let mut statekey_vec = event_type.to_string().as_bytes().to_vec(); statekey_vec.push(0xFF); statekey_vec.extend_from_slice(state_key.as_bytes()); @@ -90,7 +90,7 @@ impl Data for KeyValueDatabase { Ok(short) } - fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { + pub(super) fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { let mut statekey_vec = event_type.to_string().as_bytes().to_vec(); statekey_vec.push(0xFF); statekey_vec.extend_from_slice(state_key.as_bytes()); @@ -109,7 +109,7 @@ impl Data for KeyValueDatabase { Ok(short) } - fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { + pub(super) fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { let bytes = self .shorteventid_eventid .get(&shorteventid.to_be_bytes())? @@ -124,7 +124,7 @@ impl Data for KeyValueDatabase { Ok(event_id) } - fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { + pub(super) fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { let bytes = self .shortstatekey_statekey .get(&shortstatekey.to_be_bytes())? @@ -150,7 +150,7 @@ impl Data for KeyValueDatabase { } /// Returns (shortstatehash, already_existed) - fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> Result<(u64, bool)> { + pub(super) fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> Result<(u64, bool)> { Ok(if let Some(shortstatehash) = self.statehash_shortstatehash.get(state_hash)? { ( utils::u64_from_bytes(&shortstatehash) @@ -165,14 +165,14 @@ impl Data for KeyValueDatabase { }) } - fn get_shortroomid(&self, room_id: &RoomId) -> Result> { + pub(super) fn get_shortroomid(&self, room_id: &RoomId) -> Result> { self.roomid_shortroomid .get(room_id.as_bytes())? .map(|bytes| utils::u64_from_bytes(&bytes).map_err(|_| Error::bad_database("Invalid shortroomid in db."))) .transpose() } - fn get_or_create_shortroomid(&self, room_id: &RoomId) -> Result { + pub(super) fn get_or_create_shortroomid(&self, room_id: &RoomId) -> Result { Ok(if let Some(short) = self.roomid_shortroomid.get(room_id.as_bytes())? { utils::u64_from_bytes(&short).map_err(|_| Error::bad_database("Invalid shortroomid in db."))? } else { diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 2e994c3c..0cd676a9 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -7,10 +10,16 @@ use ruma::{events::StateEventType, EventId, RoomId}; use crate::Result; pub struct Service { - pub db: Arc, + db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + pub fn get_or_create_shorteventid(&self, event_id: &EventId) -> Result { self.db.get_or_create_shorteventid(event_id) } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index ffe1935d..e11bade0 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -1,8 +1,11 @@ use std::{ fmt::{Display, Formatter}, str::FromStr, + sync::Arc, }; +use conduit::Server; +use database::KeyValueDatabase; use lru_cache::LruCache; use ruma::{ api::{ @@ -330,6 +333,16 @@ impl From for SpaceHierarchyRoomsChunk { } impl Service { + pub fn build(server: &Arc, _db: &Arc) -> Result { + let config = &server.config; + Ok(Self { + roomid_spacehierarchy_cache: Mutex::new(LruCache::new( + (f64::from(config.roomid_spacehierarchy_cache_capacity) * config.conduit_cache_capacity_modifier) + as usize, + )), + }) + } + ///Gets the response for the space hierarchy over federation request /// ///Panics if the room does not exist, so a check if the room exists should diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index f8c7f6a6..3a0f93b5 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -1,39 +1,27 @@ use std::{collections::HashSet, sync::Arc}; use conduit::utils::mutex_map; +use database::KvTree; use ruma::{EventId, OwnedEventId, RoomId}; use crate::{utils, Error, KeyValueDatabase, Result}; -pub trait Data: Send + Sync { - /// Returns the last state hash key added to the db for the given room. - fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result>; - - /// Set the state hash to a new version, but does not update `state_cache`. - fn set_room_state( - &self, - room_id: &RoomId, - new_shortstatehash: u64, - _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex - ) -> Result<()>; - - /// Associates a state with an event. - fn set_event_state(&self, shorteventid: u64, shortstatehash: u64) -> Result<()>; - - /// Returns all events we would send as the `prev_events` of the next event. - fn get_forward_extremities(&self, room_id: &RoomId) -> Result>>; - - /// Replace the forward extremities of the room. - fn set_forward_extremities( - &self, - room_id: &RoomId, - event_ids: Vec, - _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex - ) -> Result<()>; +pub struct Data { + shorteventid_shortstatehash: Arc, + roomid_pduleaves: Arc, + roomid_shortstatehash: Arc, } -impl Data for KeyValueDatabase { - fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + shorteventid_shortstatehash: db.shorteventid_shortstatehash.clone(), + roomid_pduleaves: db.roomid_pduleaves.clone(), + roomid_shortstatehash: db.roomid_shortstatehash.clone(), + } + } + + pub(super) fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result> { self.roomid_shortstatehash .get(room_id.as_bytes())? .map_or(Ok(None), |bytes| { @@ -43,7 +31,7 @@ impl Data for KeyValueDatabase { }) } - fn set_room_state( + pub(super) fn set_room_state( &self, room_id: &RoomId, new_shortstatehash: u64, @@ -54,13 +42,13 @@ impl Data for KeyValueDatabase { Ok(()) } - fn set_event_state(&self, shorteventid: u64, shortstatehash: u64) -> Result<()> { + pub(super) fn set_event_state(&self, shorteventid: u64, shortstatehash: u64) -> Result<()> { self.shorteventid_shortstatehash .insert(&shorteventid.to_be_bytes(), &shortstatehash.to_be_bytes())?; Ok(()) } - fn get_forward_extremities(&self, room_id: &RoomId) -> Result>> { + pub(super) fn get_forward_extremities(&self, room_id: &RoomId) -> Result>> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -76,7 +64,7 @@ impl Data for KeyValueDatabase { .collect() } - fn set_forward_extremities( + pub(super) fn set_forward_extremities( &self, room_id: &RoomId, event_ids: Vec, diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 199ee87f..e0d75c80 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::{ collections::{HashMap, HashSet}, @@ -22,10 +25,16 @@ use super::state_compressor::CompressedStateEvent; use crate::{services, utils::calculate_hash, Error, PduEvent, Result}; pub struct Service { - pub db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + /// Set the room to the given statehash and update caches. pub async fn force_state( &self, diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index a7b39e8c..4b6dd131 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,56 +1,25 @@ use std::{collections::HashMap, sync::Arc}; -use async_trait::async_trait; +use database::KvTree; use ruma::{events::StateEventType, EventId, RoomId}; use crate::{services, utils, Error, KeyValueDatabase, PduEvent, Result}; -#[async_trait] -pub trait Data: Send + Sync { - /// Builds a StateMap by iterating over all keys that start - /// with state_hash, this gives the full state for the given state_hash. - #[allow(unused_qualifications)] // async traits - async fn state_full_ids(&self, shortstatehash: u64) -> Result>>; - - #[allow(unused_qualifications)] // async traits - async fn state_full(&self, shortstatehash: u64) -> Result>>; - - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - fn state_get_id( - &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, - ) -> Result>>; - - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - fn state_get( - &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, - ) -> Result>>; - - /// Returns the state hash for this pdu. - fn pdu_shortstatehash(&self, event_id: &EventId) -> Result>; - - /// Returns the full room state. - #[allow(unused_qualifications)] // async traits - async fn room_state_full(&self, room_id: &RoomId) -> Result>>; - - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - fn room_state_get_id( - &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result>>; - - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - fn room_state_get( - &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result>>; +pub struct Data { + eventid_shorteventid: Arc, + shorteventid_shortstatehash: Arc, } -#[async_trait] -impl Data for KeyValueDatabase { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + eventid_shorteventid: db.eventid_shorteventid.clone(), + shorteventid_shortstatehash: db.shorteventid_shortstatehash.clone(), + } + } + #[allow(unused_qualifications)] // async traits - async fn state_full_ids(&self, shortstatehash: u64) -> Result>> { + pub(super) async fn state_full_ids(&self, shortstatehash: u64) -> Result>> { let full_state = services() .rooms .state_compressor @@ -76,7 +45,9 @@ impl Data for KeyValueDatabase { } #[allow(unused_qualifications)] // async traits - async fn state_full(&self, shortstatehash: u64) -> Result>> { + pub(super) async fn state_full( + &self, shortstatehash: u64, + ) -> Result>> { let full_state = services() .rooms .state_compressor @@ -116,7 +87,8 @@ impl Data for KeyValueDatabase { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - fn state_get_id( + #[allow(clippy::unused_self)] + pub(super) fn state_get_id( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, ) -> Result>> { let Some(shortstatekey) = services() @@ -148,7 +120,7 @@ impl Data for KeyValueDatabase { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - fn state_get( + pub(super) fn state_get( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, ) -> Result>> { self.state_get_id(shortstatehash, event_type, state_key)? @@ -156,7 +128,7 @@ impl Data for KeyValueDatabase { } /// Returns the state hash for this pdu. - fn pdu_shortstatehash(&self, event_id: &EventId) -> Result> { + pub(super) fn pdu_shortstatehash(&self, event_id: &EventId) -> Result> { self.eventid_shorteventid .get(event_id.as_bytes())? .map_or(Ok(None), |shorteventid| { @@ -173,7 +145,9 @@ impl Data for KeyValueDatabase { /// Returns the full room state. #[allow(unused_qualifications)] // async traits - async fn room_state_full(&self, room_id: &RoomId) -> Result>> { + pub(super) async fn room_state_full( + &self, room_id: &RoomId, + ) -> Result>> { if let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? { self.state_full(current_shortstatehash).await } else { @@ -183,7 +157,7 @@ impl Data for KeyValueDatabase { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - fn room_state_get_id( + pub(super) fn room_state_get_id( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result>> { if let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? { @@ -195,7 +169,7 @@ impl Data for KeyValueDatabase { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - fn room_state_get( + pub(super) fn room_state_get( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result>> { if let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? { diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 06293ea3..e1466d75 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -1,3 +1,8 @@ +use std::sync::Mutex as StdMutex; + +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::{ collections::HashMap, @@ -29,12 +34,25 @@ use tracing::{error, warn}; use crate::{service::pdu::PduBuilder, services, Error, PduEvent, Result}; pub struct Service { - pub db: Arc, + pub db: Data, pub server_visibility_cache: Mutex>, pub user_visibility_cache: Mutex>, } impl Service { + pub fn build(server: &Arc, db: &Arc) -> Result { + let config = &server.config; + Ok(Self { + db: Data::new(db), + server_visibility_cache: StdMutex::new(LruCache::new( + (f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, + )), + user_visibility_cache: StdMutex::new(LruCache::new( + (f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, + )), + }) + } + /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[tracing::instrument(skip(self))] diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index d380c1ab..03c04abb 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -17,101 +17,53 @@ use crate::{ type StrippedStateEventIter<'a> = Box>)>> + 'a>; type AnySyncStateEventIter<'a> = Box>)>> + 'a>; -pub trait Data: Send + Sync { - fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()>; - fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()>; - fn mark_as_invited( - &self, user_id: &UserId, room_id: &RoomId, last_state: Option>>, - invite_via: Option>, - ) -> Result<()>; - fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()>; +use std::sync::Arc; - fn update_joined_count(&self, room_id: &RoomId) -> Result<()>; +use database::KvTree; - fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result; - - /// Makes a user forget a room. - fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()>; - - /// Returns an iterator of all servers participating in this room. - fn room_servers<'a>(&'a self, room_id: &RoomId) -> Box> + 'a>; - - fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result; - - /// Returns an iterator of all rooms a server participates in (as far as we - /// know). - fn server_rooms<'a>(&'a self, server: &ServerName) -> Box> + 'a>; - - /// Returns an iterator of all joined members of a room. - fn room_members<'a>(&'a self, room_id: &RoomId) -> Box> + 'a>; - - /// Returns an iterator of all our local users - /// in the room, even if they're deactivated/guests - fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a>; - - /// Returns an iterator of all our local joined users in a room who are - /// active (not deactivated, not guest) - fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a>; - - fn room_joined_count(&self, room_id: &RoomId) -> Result>; - - fn room_invited_count(&self, room_id: &RoomId) -> Result>; - - /// Returns an iterator over all User IDs who ever joined a room. - /// - /// TODO: use this? - #[allow(dead_code)] - fn room_useroncejoined<'a>(&'a self, room_id: &RoomId) -> Box> + 'a>; - - /// Returns an iterator over all invited members of a room. - fn room_members_invited<'a>(&'a self, room_id: &RoomId) -> Box> + 'a>; - - fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result>; - - fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result>; - - /// Returns an iterator over all rooms this user joined. - fn rooms_joined(&self, user_id: &UserId) -> Box> + '_>; - - /// Returns an iterator over all rooms a user was invited to. - fn rooms_invited<'a>(&'a self, user_id: &UserId) -> StrippedStateEventIter<'a>; - - fn invite_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>>; - - fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>>; - - /// Returns an iterator over all rooms a user left. - fn rooms_left<'a>(&'a self, user_id: &UserId) -> AnySyncStateEventIter<'a>; - - fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result; - - fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result; - - fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result; - - fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result; - - /// Gets the servers to either accept or decline invites via for a given - /// room. - fn servers_invite_via<'a>(&'a self, room_id: &RoomId) -> Box> + 'a>; - - /// Add the given servers the list to accept or decline invites via for a - /// given room. - /// - /// TODO: use this? - #[allow(dead_code)] - fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) -> Result<()>; +pub struct Data { + userroomid_joined: Arc, + roomuserid_joined: Arc, + userroomid_invitestate: Arc, + roomuserid_invitecount: Arc, + userroomid_leftstate: Arc, + roomuserid_leftcount: Arc, + roomid_inviteviaservers: Arc, + roomuseroncejoinedids: Arc, + roomid_joinedcount: Arc, + roomid_invitedcount: Arc, + roomserverids: Arc, + serverroomids: Arc, + db: Arc, } -impl Data for KeyValueDatabase { - fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + userroomid_joined: db.userroomid_joined.clone(), + roomuserid_joined: db.roomuserid_joined.clone(), + userroomid_invitestate: db.userroomid_invitestate.clone(), + roomuserid_invitecount: db.roomuserid_invitecount.clone(), + userroomid_leftstate: db.userroomid_leftstate.clone(), + roomuserid_leftcount: db.roomuserid_leftcount.clone(), + roomid_inviteviaservers: db.roomid_inviteviaservers.clone(), + roomuseroncejoinedids: db.roomuseroncejoinedids.clone(), + roomid_joinedcount: db.roomid_joinedcount.clone(), + roomid_invitedcount: db.roomid_invitedcount.clone(), + roomserverids: db.roomserverids.clone(), + serverroomids: db.serverroomids.clone(), + db: db.clone(), + } + } + + pub(super) fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); self.roomuseroncejoinedids.insert(&userroom_id, &[]) } - fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { + pub(super) fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { let roomid = room_id.as_bytes().to_vec(); let mut roomuser_id = roomid.clone(); @@ -134,7 +86,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn mark_as_invited( + pub(super) fn mark_as_invited( &self, user_id: &UserId, room_id: &RoomId, last_state: Option>>, invite_via: Option>, ) -> Result<()> { @@ -179,7 +131,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { + pub(super) fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { let roomid = room_id.as_bytes().to_vec(); let mut roomuser_id = roomid.clone(); @@ -206,7 +158,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn update_joined_count(&self, room_id: &RoomId) -> Result<()> { + pub(super) fn update_joined_count(&self, room_id: &RoomId) -> Result<()> { let mut joinedcount = 0_u64; let mut invitedcount = 0_u64; let mut joined_servers = HashSet::new(); @@ -256,7 +208,8 @@ impl Data for KeyValueDatabase { self.serverroomids.insert(&serverroom_id, &[])?; } - self.appservice_in_room_cache + self.db + .appservice_in_room_cache .write() .unwrap() .remove(room_id); @@ -265,8 +218,9 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self, room_id, appservice))] - fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { + pub(super) fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { let maybe = self + .db .appservice_in_room_cache .read() .unwrap() @@ -288,7 +242,8 @@ impl Data for KeyValueDatabase { .room_members(room_id) .any(|userid| userid.map_or(false, |userid| appservice.users.is_match(userid.as_str()))); - self.appservice_in_room_cache + self.db + .appservice_in_room_cache .write() .unwrap() .entry(room_id.to_owned()) @@ -301,7 +256,7 @@ impl Data for KeyValueDatabase { /// Makes a user forget a room. #[tracing::instrument(skip(self))] - fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { + pub(super) fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -318,7 +273,9 @@ impl Data for KeyValueDatabase { /// Returns an iterator of all servers participating in this room. #[tracing::instrument(skip(self))] - fn room_servers<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { + pub(super) fn room_servers<'a>( + &'a self, room_id: &RoomId, + ) -> Box> + 'a> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -336,7 +293,7 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result { + pub(super) fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result { let mut key = server.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(room_id.as_bytes()); @@ -347,7 +304,9 @@ impl Data for KeyValueDatabase { /// Returns an iterator of all rooms a server participates in (as far as we /// know). #[tracing::instrument(skip(self))] - fn server_rooms<'a>(&'a self, server: &ServerName) -> Box> + 'a> { + pub(super) fn server_rooms<'a>( + &'a self, server: &ServerName, + ) -> Box> + 'a> { let mut prefix = server.as_bytes().to_vec(); prefix.push(0xFF); @@ -366,7 +325,7 @@ impl Data for KeyValueDatabase { /// Returns an iterator of all joined members of a room. #[tracing::instrument(skip(self))] - fn room_members<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { + pub(super) fn room_members<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -385,7 +344,7 @@ impl Data for KeyValueDatabase { /// Returns an iterator of all our local users in the room, even if they're /// deactivated/guests - fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a> { + pub(super) fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a> { Box::new( self.room_members(room_id) .filter_map(Result::ok) @@ -396,7 +355,9 @@ impl Data for KeyValueDatabase { /// Returns an iterator of all our local joined users in a room who are /// active (not deactivated, not guest) #[tracing::instrument(skip(self))] - fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a> { + pub(super) fn active_local_users_in_room<'a>( + &'a self, room_id: &RoomId, + ) -> Box + 'a> { Box::new( self.local_users_in_room(room_id) .filter(|user| !services().users.is_deactivated(user).unwrap_or(true)), @@ -405,7 +366,7 @@ impl Data for KeyValueDatabase { /// Returns the number of users which are currently in a room #[tracing::instrument(skip(self))] - fn room_joined_count(&self, room_id: &RoomId) -> Result> { + pub(super) fn room_joined_count(&self, room_id: &RoomId) -> Result> { self.roomid_joinedcount .get(room_id.as_bytes())? .map(|b| utils::u64_from_bytes(&b).map_err(|_| Error::bad_database("Invalid joinedcount in db."))) @@ -414,7 +375,7 @@ impl Data for KeyValueDatabase { /// Returns the number of users which are currently invited to a room #[tracing::instrument(skip(self))] - fn room_invited_count(&self, room_id: &RoomId) -> Result> { + pub(super) fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.roomid_invitedcount .get(room_id.as_bytes())? .map(|b| utils::u64_from_bytes(&b).map_err(|_| Error::bad_database("Invalid joinedcount in db."))) @@ -423,7 +384,9 @@ impl Data for KeyValueDatabase { /// Returns an iterator over all User IDs who ever joined a room. #[tracing::instrument(skip(self))] - fn room_useroncejoined<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { + pub(super) fn room_useroncejoined<'a>( + &'a self, room_id: &RoomId, + ) -> Box> + 'a> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -446,7 +409,9 @@ impl Data for KeyValueDatabase { /// Returns an iterator over all invited members of a room. #[tracing::instrument(skip(self))] - fn room_members_invited<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { + pub(super) fn room_members_invited<'a>( + &'a self, room_id: &RoomId, + ) -> Box> + 'a> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -468,7 +433,7 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { + pub(super) fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(user_id.as_bytes()); @@ -483,7 +448,7 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { + pub(super) fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(user_id.as_bytes()); @@ -496,7 +461,7 @@ impl Data for KeyValueDatabase { /// Returns an iterator over all rooms this user joined. #[tracing::instrument(skip(self))] - fn rooms_joined(&self, user_id: &UserId) -> Box> + '_> { + pub(super) fn rooms_joined(&self, user_id: &UserId) -> Box> + '_> { Box::new( self.userroomid_joined .scan_prefix(user_id.as_bytes().to_vec()) @@ -516,7 +481,7 @@ impl Data for KeyValueDatabase { /// Returns an iterator over all rooms a user was invited to. #[tracing::instrument(skip(self))] - fn rooms_invited<'a>(&'a self, user_id: &UserId) -> StrippedStateEventIter<'a> { + pub(super) fn rooms_invited<'a>(&'a self, user_id: &UserId) -> StrippedStateEventIter<'a> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -543,7 +508,9 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn invite_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>> { + pub(super) fn invite_state( + &self, user_id: &UserId, room_id: &RoomId, + ) -> Result>>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(room_id.as_bytes()); @@ -560,7 +527,9 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>> { + pub(super) fn left_state( + &self, user_id: &UserId, room_id: &RoomId, + ) -> Result>>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(room_id.as_bytes()); @@ -578,7 +547,7 @@ impl Data for KeyValueDatabase { /// Returns an iterator over all rooms a user left. #[tracing::instrument(skip(self))] - fn rooms_left<'a>(&'a self, user_id: &UserId) -> AnySyncStateEventIter<'a> { + pub(super) fn rooms_left<'a>(&'a self, user_id: &UserId) -> AnySyncStateEventIter<'a> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -605,7 +574,7 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { + pub(super) fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -614,7 +583,7 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { + pub(super) fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -623,7 +592,7 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { + pub(super) fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -632,7 +601,7 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { + pub(super) fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -641,7 +610,9 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn servers_invite_via<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { + pub(super) fn servers_invite_via<'a>( + &'a self, room_id: &RoomId, + ) -> Box> + 'a> { let key = room_id.as_bytes().to_vec(); Box::new( @@ -665,7 +636,7 @@ impl Data for KeyValueDatabase { } #[tracing::instrument(skip(self))] - fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) -> Result<()> { + pub(super) fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) -> Result<()> { let mut prev_servers = self .servers_invite_via(room_id) .filter_map(Result::ok) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index c9d1bbd7..c4eb0dff 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -1,6 +1,8 @@ use std::sync::Arc; +use conduit::Server; use data::Data; +use database::KeyValueDatabase; use itertools::Itertools; use ruma::{ events::{ @@ -24,10 +26,16 @@ use crate::{service::appservice::RegistrationInfo, services, user_is_local, Erro mod data; pub struct Service { - pub db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + /// Update current membership data. #[tracing::instrument(skip(self, last_state))] #[allow(clippy::too_many_arguments)] diff --git a/src/service/rooms/state_compressor/data.rs b/src/service/rooms/state_compressor/data.rs index 4612ebc6..2bf93f38 100644 --- a/src/service/rooms/state_compressor/data.rs +++ b/src/service/rooms/state_compressor/data.rs @@ -1,21 +1,28 @@ use std::{collections::HashSet, mem::size_of, sync::Arc}; +use database::KvTree; + use super::CompressedStateEvent; use crate::{utils, Error, KeyValueDatabase, Result}; -pub struct StateDiff { - pub parent: Option, - pub added: Arc>, - pub removed: Arc>, +pub(super) struct StateDiff { + pub(super) parent: Option, + pub(super) added: Arc>, + pub(super) removed: Arc>, } -pub trait Data: Send + Sync { - fn get_statediff(&self, shortstatehash: u64) -> Result; - fn save_statediff(&self, shortstatehash: u64, diff: StateDiff) -> Result<()>; +pub struct Data { + shortstatehash_statediff: Arc, } -impl Data for KeyValueDatabase { - fn get_statediff(&self, shortstatehash: u64) -> Result { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + shortstatehash_statediff: db.shortstatehash_statediff.clone(), + } + } + + pub(super) fn get_statediff(&self, shortstatehash: u64) -> Result { let value = self .shortstatehash_statediff .get(&shortstatehash.to_be_bytes())? @@ -53,7 +60,7 @@ impl Data for KeyValueDatabase { }) } - fn save_statediff(&self, shortstatehash: u64, diff: StateDiff) -> Result<()> { + pub(super) fn save_statediff(&self, shortstatehash: u64, diff: StateDiff) -> Result<()> { let mut value = diff.parent.unwrap_or(0).to_be_bytes().to_vec(); for new in diff.added.iter() { value.extend_from_slice(&new[..]); diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 3f025b47..8dde2890 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -1,3 +1,8 @@ +use std::sync::Mutex as StdMutex; + +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::{ collections::HashSet, @@ -41,16 +46,25 @@ type ParentStatesVec = Vec<( )>; type HashSetCompressStateEvent = Result<(u64, Arc>, Arc>)>; +pub type CompressedStateEvent = [u8; 2 * size_of::()]; pub struct Service { - pub db: Arc, + pub db: Data, pub stateinfo_cache: StateInfoLruCache, } -pub type CompressedStateEvent = [u8; 2 * size_of::()]; - impl Service { + pub fn build(server: &Arc, db: &Arc) -> Result { + let config = &server.config; + Ok(Self { + db: Data::new(db), + stateinfo_cache: StdMutex::new(LruCache::new( + (f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, + )), + }) + } + /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. #[tracing::instrument(skip(self))] diff --git a/src/service/rooms/threads/data.rs b/src/service/rooms/threads/data.rs index 8ed5c770..a4044294 100644 --- a/src/service/rooms/threads/data.rs +++ b/src/service/rooms/threads/data.rs @@ -1,22 +1,24 @@ -use std::mem::size_of; +use std::{mem::size_of, sync::Arc}; +use database::KvTree; use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId}; use crate::{services, utils, Error, KeyValueDatabase, PduEvent, Result}; type PduEventIterResult<'a> = Result> + 'a>>; -pub trait Data: Send + Sync { - fn threads_until<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, include: &'a IncludeThreads, - ) -> PduEventIterResult<'a>; - - fn update_participants(&self, root_id: &[u8], participants: &[OwnedUserId]) -> Result<()>; - fn get_participants(&self, root_id: &[u8]) -> Result>>; +pub struct Data { + threadid_userids: Arc, } -impl Data for KeyValueDatabase { - fn threads_until<'a>( +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + threadid_userids: db.threadid_userids.clone(), + } + } + + pub(super) fn threads_until<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, _include: &'a IncludeThreads, ) -> PduEventIterResult<'a> { let prefix = services() @@ -50,7 +52,7 @@ impl Data for KeyValueDatabase { )) } - fn update_participants(&self, root_id: &[u8], participants: &[OwnedUserId]) -> Result<()> { + pub(super) fn update_participants(&self, root_id: &[u8], participants: &[OwnedUserId]) -> Result<()> { let users = participants .iter() .map(|user| user.as_bytes()) @@ -62,7 +64,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn get_participants(&self, root_id: &[u8]) -> Result>> { + pub(super) fn get_participants(&self, root_id: &[u8]) -> Result>> { if let Some(users) = self.threadid_userids.get(root_id)? { Ok(Some( users diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index 6c48e842..606cc001 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::{collections::BTreeMap, sync::Arc}; @@ -13,10 +16,16 @@ use serde_json::json; use crate::{services, Error, PduEvent, Result}; pub struct Service { - pub db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + pub fn threads_until<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, include: &'a IncludeThreads, ) -> Result> + 'a> { diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 9fb1eea4..8264b6f3 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -1,76 +1,36 @@ use std::{collections::hash_map, mem::size_of, sync::Arc}; +use database::KvTree; use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedUserId, RoomId, UserId}; use tracing::error; use super::PduCount; use crate::{services, utils, Error, KeyValueDatabase, PduEvent, Result}; -pub trait Data: Send + Sync { - fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result; - - /// Returns the `count` of this pdu's id. - fn get_pdu_count(&self, event_id: &EventId) -> Result>; - - /// Returns the json of a pdu. - fn get_pdu_json(&self, event_id: &EventId) -> Result>; - - /// Returns the json of a pdu. - fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result>; - - /// Returns the pdu's id. - fn get_pdu_id(&self, event_id: &EventId) -> Result>>; - - /// Returns the pdu. - /// - /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result>; - - /// Returns the pdu. - /// - /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - fn get_pdu(&self, event_id: &EventId) -> Result>>; - - /// Returns the pdu. - /// - /// This does __NOT__ check the outliers `Tree`. - fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result>; - - /// Returns the pdu as a `BTreeMap`. - fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result>; - - /// Adds a new pdu to the timeline - fn append_pdu(&self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64) -> Result<()>; - - // Adds a new pdu to the backfilled timeline - fn prepend_backfill_pdu(&self, pdu_id: &[u8], event_id: &EventId, json: &CanonicalJsonObject) -> Result<()>; - - /// Removes a pdu and creates a new one with the same id. - fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, pdu: &PduEvent) -> Result<()>; - - /// Returns an iterator over all events and their tokens in a room that - /// happened before the event with id `until` in reverse-chronological - /// order. - #[allow(clippy::type_complexity)] - fn pdus_until<'a>( - &'a self, user_id: &UserId, room_id: &RoomId, until: PduCount, - ) -> Result> + 'a>>; - - /// Returns an iterator over all events in a room that happened after the - /// event with id `from` in chronological order. - #[allow(clippy::type_complexity)] - fn pdus_after<'a>( - &'a self, user_id: &UserId, room_id: &RoomId, from: PduCount, - ) -> Result> + 'a>>; - - fn increment_notification_counts( - &self, room_id: &RoomId, notifies: Vec, highlights: Vec, - ) -> Result<()>; +pub struct Data { + eventid_pduid: Arc, + pduid_pdu: Arc, + eventid_outlierpdu: Arc, + userroomid_notificationcount: Arc, + userroomid_highlightcount: Arc, + db: Arc, } -impl Data for KeyValueDatabase { - fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + eventid_pduid: db.eventid_pduid.clone(), + pduid_pdu: db.pduid_pdu.clone(), + eventid_outlierpdu: db.eventid_outlierpdu.clone(), + userroomid_notificationcount: db.userroomid_notificationcount.clone(), + userroomid_highlightcount: db.userroomid_highlightcount.clone(), + db: db.clone(), + } + } + + pub(super) fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { match self + .db .lasttimelinecount_cache .lock() .unwrap() @@ -96,7 +56,7 @@ impl Data for KeyValueDatabase { } /// Returns the `count` of this pdu's id. - fn get_pdu_count(&self, event_id: &EventId) -> Result> { + pub(super) fn get_pdu_count(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? .map(|pdu_id| pdu_count(&pdu_id)) @@ -104,7 +64,7 @@ impl Data for KeyValueDatabase { } /// Returns the json of a pdu. - fn get_pdu_json(&self, event_id: &EventId) -> Result> { + pub(super) fn get_pdu_json(&self, event_id: &EventId) -> Result> { self.get_non_outlier_pdu_json(event_id)?.map_or_else( || { self.eventid_outlierpdu @@ -117,7 +77,7 @@ impl Data for KeyValueDatabase { } /// Returns the json of a pdu. - fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result> { + pub(super) fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? .map(|pduid| { @@ -131,10 +91,12 @@ impl Data for KeyValueDatabase { } /// Returns the pdu's id. - fn get_pdu_id(&self, event_id: &EventId) -> Result>> { self.eventid_pduid.get(event_id.as_bytes()) } + pub(super) fn get_pdu_id(&self, event_id: &EventId) -> Result>> { + self.eventid_pduid.get(event_id.as_bytes()) + } /// Returns the pdu. - fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { + pub(super) fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? .map(|pduid| { @@ -150,7 +112,7 @@ impl Data for KeyValueDatabase { /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - fn get_pdu(&self, event_id: &EventId) -> Result>> { + pub(super) fn get_pdu(&self, event_id: &EventId) -> Result>> { if let Some(pdu) = self .get_non_outlier_pdu(event_id)? .map_or_else( @@ -173,7 +135,7 @@ impl Data for KeyValueDatabase { /// Returns the pdu. /// /// This does __NOT__ check the outliers `Tree`. - fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result> { + pub(super) fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result> { self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| { Ok(Some( serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))?, @@ -182,7 +144,7 @@ impl Data for KeyValueDatabase { } /// Returns the pdu as a `BTreeMap`. - fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result> { + pub(super) fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result> { self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| { Ok(Some( serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))?, @@ -190,13 +152,16 @@ impl Data for KeyValueDatabase { }) } - fn append_pdu(&self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64) -> Result<()> { + pub(super) fn append_pdu( + &self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64, + ) -> Result<()> { self.pduid_pdu.insert( pdu_id, &serde_json::to_vec(json).expect("CanonicalJsonObject is always a valid"), )?; - self.lasttimelinecount_cache + self.db + .lasttimelinecount_cache .lock() .unwrap() .insert(pdu.room_id.clone(), PduCount::Normal(count)); @@ -207,7 +172,9 @@ impl Data for KeyValueDatabase { Ok(()) } - fn prepend_backfill_pdu(&self, pdu_id: &[u8], event_id: &EventId, json: &CanonicalJsonObject) -> Result<()> { + pub(super) fn prepend_backfill_pdu( + &self, pdu_id: &[u8], event_id: &EventId, json: &CanonicalJsonObject, + ) -> Result<()> { self.pduid_pdu.insert( pdu_id, &serde_json::to_vec(json).expect("CanonicalJsonObject is always a valid"), @@ -220,7 +187,7 @@ impl Data for KeyValueDatabase { } /// Removes a pdu and creates a new one with the same id. - fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, _pdu: &PduEvent) -> Result<()> { + pub(super) fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, _pdu: &PduEvent) -> Result<()> { if self.pduid_pdu.get(pdu_id)?.is_some() { self.pduid_pdu.insert( pdu_id, @@ -236,7 +203,7 @@ impl Data for KeyValueDatabase { /// Returns an iterator over all events and their tokens in a room that /// happened before the event with id `until` in reverse-chronological /// order. - fn pdus_until<'a>( + pub(super) fn pdus_until<'a>( &'a self, user_id: &UserId, room_id: &RoomId, until: PduCount, ) -> Result> + 'a>> { let (prefix, current) = count_to_id(room_id, until, 1, true)?; @@ -260,7 +227,7 @@ impl Data for KeyValueDatabase { )) } - fn pdus_after<'a>( + pub(super) fn pdus_after<'a>( &'a self, user_id: &UserId, room_id: &RoomId, from: PduCount, ) -> Result> + 'a>> { let (prefix, current) = count_to_id(room_id, from, 1, false)?; @@ -284,7 +251,7 @@ impl Data for KeyValueDatabase { )) } - fn increment_notification_counts( + pub(super) fn increment_notification_counts( &self, room_id: &RoomId, notifies: Vec, highlights: Vec, ) -> Result<()> { let mut notifies_batch = Vec::new(); @@ -311,7 +278,7 @@ impl Data for KeyValueDatabase { } /// Returns the `count` of this pdu's id. -fn pdu_count(pdu_id: &[u8]) -> Result { +pub(super) fn pdu_count(pdu_id: &[u8]) -> Result { let last_u64 = utils::u64_from_bytes(&pdu_id[pdu_id.len() - size_of::()..]) .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?; let second_last_u64 = @@ -324,7 +291,9 @@ fn pdu_count(pdu_id: &[u8]) -> Result { } } -fn count_to_id(room_id: &RoomId, count: PduCount, offset: u64, subtract: bool) -> Result<(Vec, Vec)> { +pub(super) fn count_to_id( + room_id: &RoomId, count: PduCount, offset: u64, subtract: bool, +) -> Result<(Vec, Vec)> { let prefix = services() .rooms .short diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 7712ab90..994b5cd4 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::{ @@ -74,12 +77,19 @@ struct ExtractBody { } pub struct Service { - pub db: Arc, + pub db: Data, pub lasttimelinecount_cache: Mutex>, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + lasttimelinecount_cache: Mutex::new(HashMap::new()), + }) + } + #[tracing::instrument(skip(self))] pub fn first_pdu_in_room(&self, room_id: &RoomId) -> Result>> { self.all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id)? diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 0b89dfdd..614d5af2 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -1,5 +1,7 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, sync::Arc}; +use conduit::Server; +use database::KeyValueDatabase; use ruma::{ api::federation::transactions::edu::{Edu, TypingContent}, events::SyncEphemeralRoomEvent, @@ -23,6 +25,14 @@ pub struct Service { } impl Service { + pub fn build(_server: &Arc, _db: &Arc) -> Result { + Ok(Self { + typing: RwLock::new(BTreeMap::new()), + last_typing_update: RwLock::new(BTreeMap::new()), + typing_update_sender: broadcast::channel(100).0, + }) + } + /// Sets a user as typing until the timeout timestamp is reached or /// roomtyping_remove is called. pub async fn typing_add(&self, user_id: &UserId, room_id: &RoomId, timeout: u64) -> Result<()> { diff --git a/src/service/rooms/user/data.rs b/src/service/rooms/user/data.rs index 3e8587da..e9339917 100644 --- a/src/service/rooms/user/data.rs +++ b/src/service/rooms/user/data.rs @@ -1,28 +1,30 @@ +use std::sync::Arc; + +use database::KvTree; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; use crate::{services, utils, Error, KeyValueDatabase, Result}; -pub trait Data: Send + Sync { - fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) -> Result<()>; - - fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> Result; - - fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> Result; - - // Returns the count at which the last reset_notification_counts was called - fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> Result; - - fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: u64) -> Result<()>; - - fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result>; - - fn get_shared_rooms<'a>( - &'a self, users: Vec, - ) -> Result> + 'a>>; +pub struct Data { + userroomid_notificationcount: Arc, + userroomid_highlightcount: Arc, + roomuserid_lastnotificationread: Arc, + roomsynctoken_shortstatehash: Arc, + userroomid_joined: Arc, } -impl Data for KeyValueDatabase { - fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + userroomid_notificationcount: db.userroomid_notificationcount.clone(), + userroomid_highlightcount: db.userroomid_highlightcount.clone(), + roomuserid_lastnotificationread: db.roomuserid_lastnotificationread.clone(), + roomsynctoken_shortstatehash: db.roomsynctoken_shortstatehash.clone(), + userroomid_joined: db.userroomid_joined.clone(), + } + } + + pub(super) fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -41,7 +43,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> Result { + pub(super) fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -53,7 +55,7 @@ impl Data for KeyValueDatabase { }) } - fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> Result { + pub(super) fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -65,7 +67,7 @@ impl Data for KeyValueDatabase { }) } - fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> Result { + pub(super) fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(user_id.as_bytes()); @@ -81,7 +83,9 @@ impl Data for KeyValueDatabase { .unwrap_or(0)) } - fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: u64) -> Result<()> { + pub(super) fn associate_token_shortstatehash( + &self, room_id: &RoomId, token: u64, shortstatehash: u64, + ) -> Result<()> { let shortroomid = services() .rooms .short @@ -95,7 +99,7 @@ impl Data for KeyValueDatabase { .insert(&key, &shortstatehash.to_be_bytes()) } - fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result> { + pub(super) fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result> { let shortroomid = services() .rooms .short @@ -114,7 +118,7 @@ impl Data for KeyValueDatabase { .transpose() } - fn get_shared_rooms<'a>( + pub(super) fn get_shared_rooms<'a>( &'a self, users: Vec, ) -> Result> + 'a>> { let iterators = users.into_iter().map(move |user_id| { diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index e589a444..36e78982 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -8,10 +11,16 @@ use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; use crate::Result; pub struct Service { - pub db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + pub fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { self.db.reset_notification_counts(user_id, room_id) } diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 9057c603..13ad4d6e 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -1,31 +1,31 @@ +use std::sync::Arc; + use ruma::{ServerName, UserId}; use super::{Destination, SendingEvent}; -use crate::{services, utils, Error, KeyValueDatabase, Result}; +use crate::{services, utils, Error, KeyValueDatabase, KvTree, Result}; type OutgoingSendingIter<'a> = Box, Destination, SendingEvent)>> + 'a>; type SendingEventIter<'a> = Box, SendingEvent)>> + 'a>; -pub trait Data: Send + Sync { - fn active_requests(&self) -> OutgoingSendingIter<'_>; - fn active_requests_for(&self, destination: &Destination) -> SendingEventIter<'_>; - fn delete_active_request(&self, key: Vec) -> Result<()>; - fn delete_all_active_requests_for(&self, destination: &Destination) -> Result<()>; - - /// TODO: use this? - #[allow(dead_code)] - fn delete_all_requests_for(&self, destination: &Destination) -> Result<()>; - fn queue_requests(&self, requests: &[(&Destination, SendingEvent)]) -> Result>>; - fn queued_requests<'a>( - &'a self, destination: &Destination, - ) -> Box)>> + 'a>; - fn mark_as_active(&self, events: &[(SendingEvent, Vec)]) -> Result<()>; - fn set_latest_educount(&self, server_name: &ServerName, educount: u64) -> Result<()>; - fn get_latest_educount(&self, server_name: &ServerName) -> Result; +pub struct Data { + servercurrentevent_data: Arc, + servernameevent_data: Arc, + servername_educount: Arc, + _db: Arc, } -impl Data for KeyValueDatabase { - fn active_requests<'a>(&'a self) -> Box, Destination, SendingEvent)>> + 'a> { +impl Data { + pub(super) fn new(db: Arc) -> Self { + Self { + servercurrentevent_data: db.servercurrentevent_data.clone(), + servernameevent_data: db.servernameevent_data.clone(), + servername_educount: db.servername_educount.clone(), + _db: db, + } + } + + pub fn active_requests(&self) -> OutgoingSendingIter<'_> { Box::new( self.servercurrentevent_data .iter() @@ -33,9 +33,7 @@ impl Data for KeyValueDatabase { ) } - fn active_requests_for<'a>( - &'a self, destination: &Destination, - ) -> Box, SendingEvent)>> + 'a> { + pub fn active_requests_for<'a>(&'a self, destination: &Destination) -> SendingEventIter<'a> { let prefix = destination.get_prefix(); Box::new( self.servercurrentevent_data @@ -44,9 +42,9 @@ impl Data for KeyValueDatabase { ) } - fn delete_active_request(&self, key: Vec) -> Result<()> { self.servercurrentevent_data.remove(&key) } + pub(super) fn delete_active_request(&self, key: &[u8]) -> Result<()> { self.servercurrentevent_data.remove(key) } - fn delete_all_active_requests_for(&self, destination: &Destination) -> Result<()> { + pub(super) fn delete_all_active_requests_for(&self, destination: &Destination) -> Result<()> { let prefix = destination.get_prefix(); for (key, _) in self.servercurrentevent_data.scan_prefix(prefix) { self.servercurrentevent_data.remove(&key)?; @@ -55,7 +53,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn delete_all_requests_for(&self, destination: &Destination) -> Result<()> { + pub(super) fn delete_all_requests_for(&self, destination: &Destination) -> Result<()> { let prefix = destination.get_prefix(); for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) { self.servercurrentevent_data.remove(&key).unwrap(); @@ -68,7 +66,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn queue_requests(&self, requests: &[(&Destination, SendingEvent)]) -> Result>> { + pub(super) fn queue_requests(&self, requests: &[(&Destination, SendingEvent)]) -> Result>> { let mut batch = Vec::new(); let mut keys = Vec::new(); for (destination, event) in requests { @@ -91,7 +89,7 @@ impl Data for KeyValueDatabase { Ok(keys) } - fn queued_requests<'a>( + pub fn queued_requests<'a>( &'a self, destination: &Destination, ) -> Box)>> + 'a> { let prefix = destination.get_prefix(); @@ -102,7 +100,7 @@ impl Data for KeyValueDatabase { ); } - fn mark_as_active(&self, events: &[(SendingEvent, Vec)]) -> Result<()> { + pub(super) fn mark_as_active(&self, events: &[(SendingEvent, Vec)]) -> Result<()> { for (e, key) in events { if key.is_empty() { continue; @@ -120,12 +118,12 @@ impl Data for KeyValueDatabase { Ok(()) } - fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) -> Result<()> { + pub(super) fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) -> Result<()> { self.servername_educount .insert(server_name.as_bytes(), &last_count.to_be_bytes()) } - fn get_latest_educount(&self, server_name: &ServerName) -> Result { + pub fn get_latest_educount(&self, server_name: &ServerName) -> Result { self.servername_educount .get(server_name.as_bytes())? .map_or(Ok(0), |bytes| { diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 225d37c7..b9a0144f 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -1,3 +1,5 @@ +use conduit::Server; + mod appservice; mod data; pub mod resolve; @@ -15,10 +17,10 @@ use ruma::{ use tokio::{sync::Mutex, task::JoinHandle}; use tracing::{error, warn}; -use crate::{server_is_ours, services, Config, Error, Result}; +use crate::{server_is_ours, services, Error, KeyValueDatabase, Result}; pub struct Service { - pub db: Arc, + pub db: Data, /// The state for a given state hash. sender: loole::Sender, @@ -51,16 +53,17 @@ pub enum SendingEvent { } impl Service { - pub fn build(db: Arc, config: &Config) -> Arc { + pub fn build(server: &Arc, db: &Arc) -> Result> { + let config = &server.config; let (sender, receiver) = loole::unbounded(); - Arc::new(Self { - db, + Ok(Arc::new(Self { + db: Data::new(db.clone()), sender, receiver: Mutex::new(receiver), handler_join: Mutex::new(None), startup_netburst: config.startup_netburst, startup_netburst_keep: config.startup_netburst_keep, - }) + })) } pub async fn close(&self) { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 8479a869..7edb4db5 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -144,7 +144,7 @@ impl Service { if self.startup_netburst_keep >= 0 && entry.len() >= keep { warn!("Dropping unsent event {:?} {:?}", dest, String::from_utf8_lossy(&key)); self.db - .delete_active_request(key) + .delete_active_request(&key) .expect("active request deleted"); } else { entry.push(event); diff --git a/src/service/services.rs b/src/service/services.rs index 7629b207..6f07c1fe 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,12 +1,7 @@ -use std::{ - collections::{BTreeMap, HashMap}, - sync::{Arc, Mutex as StdMutex}, -}; +use std::sync::Arc; use conduit::{debug_info, Result, Server}; use database::KeyValueDatabase; -use lru_cache::LruCache; -use tokio::sync::{broadcast, Mutex, RwLock}; use tracing::{debug, info, trace}; use crate::{ @@ -15,9 +10,9 @@ use crate::{ }; pub struct Services { + pub rooms: rooms::Service, pub appservice: appservice::Service, pub pusher: pusher::Service, - pub rooms: rooms::Service, pub transaction_ids: transaction_ids::Service, pub uiaa: uiaa::Service, pub users: users::Service, @@ -34,111 +29,41 @@ pub struct Services { impl Services { pub async fn build(server: Arc, db: Arc) -> Result { - let config = &server.config; Ok(Self { - appservice: appservice::Service::build(db.clone())?, - pusher: pusher::Service { - db: db.clone(), - }, rooms: rooms::Service { - alias: rooms::alias::Service { - db: db.clone(), - }, - auth_chain: rooms::auth_chain::Service { - db: db.clone(), - }, - directory: rooms::directory::Service { - db: db.clone(), - }, - event_handler: rooms::event_handler::Service, - lazy_loading: rooms::lazy_loading::Service { - db: db.clone(), - lazy_load_waiting: Mutex::new(HashMap::new()), - }, - metadata: rooms::metadata::Service { - db: db.clone(), - }, - outlier: rooms::outlier::Service { - db: db.clone(), - }, - pdu_metadata: rooms::pdu_metadata::Service { - db: db.clone(), - }, - read_receipt: rooms::read_receipt::Service { - db: db.clone(), - }, - search: rooms::search::Service { - db: db.clone(), - }, - short: rooms::short::Service { - db: db.clone(), - }, - state: rooms::state::Service { - db: db.clone(), - }, - state_accessor: rooms::state_accessor::Service { - db: db.clone(), - server_visibility_cache: StdMutex::new(LruCache::new( - (f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) - as usize, - )), - user_visibility_cache: StdMutex::new(LruCache::new( - (f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) - as usize, - )), - }, - state_cache: rooms::state_cache::Service { - db: db.clone(), - }, - state_compressor: rooms::state_compressor::Service { - db: db.clone(), - stateinfo_cache: StdMutex::new(LruCache::new( - (f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, - )), - }, - timeline: rooms::timeline::Service { - db: db.clone(), - lasttimelinecount_cache: Mutex::new(HashMap::new()), - }, - threads: rooms::threads::Service { - db: db.clone(), - }, - typing: rooms::typing::Service { - typing: RwLock::new(BTreeMap::new()), - last_typing_update: RwLock::new(BTreeMap::new()), - typing_update_sender: broadcast::channel(100).0, - }, - spaces: rooms::spaces::Service { - roomid_spacehierarchy_cache: Mutex::new(LruCache::new( - (f64::from(config.roomid_spacehierarchy_cache_capacity) - * config.conduit_cache_capacity_modifier) as usize, - )), - }, - user: rooms::user::Service { - db: db.clone(), - }, + alias: rooms::alias::Service::build(&server, &db)?, + auth_chain: rooms::auth_chain::Service::build(&server, &db)?, + directory: rooms::directory::Service::build(&server, &db)?, + event_handler: rooms::event_handler::Service::build(&server, &db)?, + lazy_loading: rooms::lazy_loading::Service::build(&server, &db)?, + metadata: rooms::metadata::Service::build(&server, &db)?, + outlier: rooms::outlier::Service::build(&server, &db)?, + pdu_metadata: rooms::pdu_metadata::Service::build(&server, &db)?, + read_receipt: rooms::read_receipt::Service::build(&server, &db)?, + search: rooms::search::Service::build(&server, &db)?, + short: rooms::short::Service::build(&server, &db)?, + state: rooms::state::Service::build(&server, &db)?, + state_accessor: rooms::state_accessor::Service::build(&server, &db)?, + state_cache: rooms::state_cache::Service::build(&server, &db)?, + state_compressor: rooms::state_compressor::Service::build(&server, &db)?, + timeline: rooms::timeline::Service::build(&server, &db)?, + threads: rooms::threads::Service::build(&server, &db)?, + typing: rooms::typing::Service::build(&server, &db)?, + spaces: rooms::spaces::Service::build(&server, &db)?, + user: rooms::user::Service::build(&server, &db)?, }, - transaction_ids: transaction_ids::Service { - db: db.clone(), - }, - uiaa: uiaa::Service { - db: db.clone(), - }, - users: users::Service { - db: db.clone(), - connections: StdMutex::new(BTreeMap::new()), - }, - account_data: account_data::Service { - db: db.clone(), - }, - presence: presence::Service::build(db.clone(), config), - admin: admin::Service::build(), - key_backups: key_backups::Service { - db: db.clone(), - }, - media: media::Service::build(&server, &db), - sending: sending::Service::build(db.clone(), config), - globals: globals::Service::load(db.clone(), config)?, + appservice: appservice::Service::build(&server, &db)?, + pusher: pusher::Service::build(&server, &db)?, + transaction_ids: transaction_ids::Service::build(&server, &db)?, + uiaa: uiaa::Service::build(&server, &db)?, + users: users::Service::build(&server, &db)?, + account_data: account_data::Service::build(&server, &db)?, + presence: presence::Service::build(&server, &db)?, + admin: admin::Service::build(&server, &db)?, + key_backups: key_backups::Service::build(&server, &db)?, + media: media::Service::build(&server, &db)?, + sending: sending::Service::build(&server, &db)?, + globals: globals::Service::build(&server, &db)?, server, db, }) diff --git a/src/service/transaction_ids/data.rs b/src/service/transaction_ids/data.rs index 8ea3b8fd..5b29f211 100644 --- a/src/service/transaction_ids/data.rs +++ b/src/service/transaction_ids/data.rs @@ -1,19 +1,21 @@ +use std::sync::Arc; + +use conduit::Result; +use database::{KeyValueDatabase, KvTree}; use ruma::{DeviceId, TransactionId, UserId}; -use crate::{KeyValueDatabase, Result}; - -pub(crate) trait Data: Send + Sync { - fn add_txnid( - &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, data: &[u8], - ) -> Result<()>; - - fn existing_txnid( - &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, - ) -> Result>>; +pub struct Data { + userdevicetxnid_response: Arc, } -impl Data for KeyValueDatabase { - fn add_txnid( +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + userdevicetxnid_response: db.userdevicetxnid_response.clone(), + } + } + + pub(super) fn add_txnid( &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, data: &[u8], ) -> Result<()> { let mut key = user_id.as_bytes().to_vec(); @@ -27,7 +29,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn existing_txnid( + pub(super) fn existing_txnid( &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, ) -> Result>> { let mut key = user_id.as_bytes().to_vec(); diff --git a/src/service/transaction_ids/mod.rs b/src/service/transaction_ids/mod.rs index e986f0ac..5fb85d4a 100644 --- a/src/service/transaction_ids/mod.rs +++ b/src/service/transaction_ids/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -8,10 +11,16 @@ use ruma::{DeviceId, TransactionId, UserId}; use crate::Result; pub struct Service { - pub(super) db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + pub fn add_txnid( &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, data: &[u8], ) -> Result<()> { diff --git a/src/service/uiaa/data.rs b/src/service/uiaa/data.rs index d6d745bc..b0710b6a 100644 --- a/src/service/uiaa/data.rs +++ b/src/service/uiaa/data.rs @@ -1,29 +1,30 @@ +use std::sync::Arc; + +use conduit::{Error, Result}; +use database::{KeyValueDatabase, KvTree}; use ruma::{ api::client::{error::ErrorKind, uiaa::UiaaInfo}, CanonicalJsonValue, DeviceId, UserId, }; -use crate::{Error, KeyValueDatabase, Result}; - -pub(crate) trait Data: Send + Sync { - fn set_uiaa_request( - &self, user_id: &UserId, device_id: &DeviceId, session: &str, request: &CanonicalJsonValue, - ) -> Result<()>; - - fn get_uiaa_request(&self, user_id: &UserId, device_id: &DeviceId, session: &str) -> Option; - - fn update_uiaa_session( - &self, user_id: &UserId, device_id: &DeviceId, session: &str, uiaainfo: Option<&UiaaInfo>, - ) -> Result<()>; - - fn get_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: &str) -> Result; +pub struct Data { + userdevicesessionid_uiaainfo: Arc, + db: Arc, } -impl Data for KeyValueDatabase { - fn set_uiaa_request( +impl Data { + pub(super) fn new(db: &Arc) -> Self { + Self { + userdevicesessionid_uiaainfo: db.userdevicesessionid_uiaainfo.clone(), + db: db.clone(), + } + } + + pub(super) fn set_uiaa_request( &self, user_id: &UserId, device_id: &DeviceId, session: &str, request: &CanonicalJsonValue, ) -> Result<()> { - self.userdevicesessionid_uiaarequest + self.db + .userdevicesessionid_uiaarequest .write() .unwrap() .insert( @@ -34,15 +35,18 @@ impl Data for KeyValueDatabase { Ok(()) } - fn get_uiaa_request(&self, user_id: &UserId, device_id: &DeviceId, session: &str) -> Option { - self.userdevicesessionid_uiaarequest + pub(super) fn get_uiaa_request( + &self, user_id: &UserId, device_id: &DeviceId, session: &str, + ) -> Option { + self.db + .userdevicesessionid_uiaarequest .read() .unwrap() .get(&(user_id.to_owned(), device_id.to_owned(), session.to_owned())) .map(ToOwned::to_owned) } - fn update_uiaa_session( + pub(super) fn update_uiaa_session( &self, user_id: &UserId, device_id: &DeviceId, session: &str, uiaainfo: Option<&UiaaInfo>, ) -> Result<()> { let mut userdevicesessionid = user_id.as_bytes().to_vec(); @@ -64,7 +68,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn get_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: &str) -> Result { + pub(super) fn get_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: &str) -> Result { let mut userdevicesessionid = user_id.as_bytes().to_vec(); userdevicesessionid.push(0xFF); userdevicesessionid.extend_from_slice(device_id.as_bytes()); diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index bec6c15b..4539c762 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -1,3 +1,6 @@ +use conduit::Server; +use database::KeyValueDatabase; + mod data; use std::sync::Arc; @@ -18,10 +21,16 @@ use crate::services; pub const SESSION_ID_LENGTH: usize = 32; pub struct Service { - pub(super) db: Arc, + pub db: Data, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db), + }) + } + /// Creates a new Uiaa session. Make sure the session token is unique. pub fn create( &self, user_id: &UserId, device_id: &DeviceId, uiaainfo: &UiaaInfo, json_body: &CanonicalJsonValue, diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 5d3eadd8..5644b3e3 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, mem::size_of}; +use std::{collections::BTreeMap, mem::size_of, sync::Arc}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, @@ -10,149 +10,60 @@ use ruma::{ }; use tracing::warn; -use crate::{services, users::clean_signatures, utils, Error, KeyValueDatabase, Result}; +use crate::{services, users::clean_signatures, utils, Error, KeyValueDatabase, KvTree, Result}; -pub trait Data: Send + Sync { - /// Check if a user has an account on this homeserver. - fn exists(&self, user_id: &UserId) -> Result; - - /// Check if account is deactivated - fn is_deactivated(&self, user_id: &UserId) -> Result; - - /// Returns the number of users registered on this server. - fn count(&self) -> Result; - - /// Find out which user an access token belongs to. - fn find_from_token(&self, token: &str) -> Result>; - - /// Returns an iterator over all users on this homeserver. - fn iter<'a>(&'a self) -> Box> + 'a>; - - /// Returns a list of local users as list of usernames. - /// - /// A user account is considered `local` if the length of it's password is - /// greater then zero. - fn list_local_users(&self) -> Result>; - - /// Returns the password hash for the given user. - fn password_hash(&self, user_id: &UserId) -> Result>; - - /// Hash and set the user's password to the Argon2 hash - fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()>; - - /// Returns the displayname of a user on this homeserver. - fn displayname(&self, user_id: &UserId) -> Result>; - - /// Sets a new displayname or removes it if displayname is None. You still - /// need to nofify all rooms of this change. - fn set_displayname(&self, user_id: &UserId, displayname: Option) -> Result<()>; - - /// Get the avatar_url of a user. - fn avatar_url(&self, user_id: &UserId) -> Result>; - - /// Sets a new avatar_url or removes it if avatar_url is None. - fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) -> Result<()>; - - /// Get the blurhash of a user. - fn blurhash(&self, user_id: &UserId) -> Result>; - - /// Sets a new avatar_url or removes it if avatar_url is None. - fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()>; - - /// Adds a new device to a user. - fn create_device( - &self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option, - ) -> Result<()>; - - /// Removes a device from a user. - fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()>; - - /// Returns an iterator over all device ids of this user. - fn all_device_ids<'a>(&'a self, user_id: &UserId) -> Box> + 'a>; - - /// Replaces the access token of one device. - fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()>; - - fn add_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, one_time_key_key: &DeviceKeyId, - one_time_key_value: &Raw, - ) -> Result<()>; - - fn last_one_time_keys_update(&self, user_id: &UserId) -> Result; - - fn take_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, key_algorithm: &DeviceKeyAlgorithm, - ) -> Result)>>; - - fn count_one_time_keys(&self, user_id: &UserId, device_id: &DeviceId) - -> Result>; - - fn add_device_keys(&self, user_id: &UserId, device_id: &DeviceId, device_keys: &Raw) -> Result<()>; - - fn add_cross_signing_keys( - &self, user_id: &UserId, master_key: &Raw, self_signing_key: &Option>, - user_signing_key: &Option>, notify: bool, - ) -> Result<()>; - - fn sign_key(&self, target_id: &UserId, key_id: &str, signature: (String, String), sender_id: &UserId) - -> Result<()>; - - fn keys_changed<'a>( - &'a self, user_or_room_id: &str, from: u64, to: Option, - ) -> Box> + 'a>; - - fn mark_device_key_update(&self, user_id: &UserId) -> Result<()>; - - fn get_device_keys(&self, user_id: &UserId, device_id: &DeviceId) -> Result>>; - - fn parse_master_key( - &self, user_id: &UserId, master_key: &Raw, - ) -> Result<(Vec, CrossSigningKey)>; - - fn get_key( - &self, key: &[u8], sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, - ) -> Result>>; - - fn get_master_key( - &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, - ) -> Result>>; - - fn get_self_signing_key( - &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, - ) -> Result>>; - - fn get_user_signing_key(&self, user_id: &UserId) -> Result>>; - - fn add_to_device_event( - &self, sender: &UserId, target_user_id: &UserId, target_device_id: &DeviceId, event_type: &str, - content: serde_json::Value, - ) -> Result<()>; - - fn get_to_device_events(&self, user_id: &UserId, device_id: &DeviceId) -> Result>>; - - fn remove_to_device_events(&self, user_id: &UserId, device_id: &DeviceId, until: u64) -> Result<()>; - - fn update_device_metadata(&self, user_id: &UserId, device_id: &DeviceId, device: &Device) -> Result<()>; - - /// Get device metadata. - fn get_device_metadata(&self, user_id: &UserId, device_id: &DeviceId) -> Result>; - - fn get_devicelist_version(&self, user_id: &UserId) -> Result>; - - fn all_devices_metadata<'a>(&'a self, user_id: &UserId) -> Box> + 'a>; - - /// Creates a new sync filter. Returns the filter id. - fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> Result; - - fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result>; +pub struct Data { + userid_password: Arc, + token_userdeviceid: Arc, + userid_displayname: Arc, + userid_avatarurl: Arc, + userid_blurhash: Arc, + userid_devicelistversion: Arc, + userdeviceid_token: Arc, + userdeviceid_metadata: Arc, + onetimekeyid_onetimekeys: Arc, + userid_lastonetimekeyupdate: Arc, + keyid_key: Arc, + userid_masterkeyid: Arc, + userid_selfsigningkeyid: Arc, + userid_usersigningkeyid: Arc, + keychangeid_userid: Arc, + todeviceid_events: Arc, + userfilterid_filter: Arc, + _db: Arc, } -impl Data for KeyValueDatabase { +impl Data { + pub(super) fn new(db: Arc) -> Self { + Self { + userid_password: db.userid_password.clone(), + token_userdeviceid: db.token_userdeviceid.clone(), + userid_displayname: db.userid_displayname.clone(), + userid_avatarurl: db.userid_avatarurl.clone(), + userid_blurhash: db.userid_blurhash.clone(), + userid_devicelistversion: db.userid_devicelistversion.clone(), + userdeviceid_token: db.userdeviceid_token.clone(), + userdeviceid_metadata: db.userdeviceid_metadata.clone(), + onetimekeyid_onetimekeys: db.onetimekeyid_onetimekeys.clone(), + userid_lastonetimekeyupdate: db.userid_lastonetimekeyupdate.clone(), + keyid_key: db.keyid_key.clone(), + userid_masterkeyid: db.userid_masterkeyid.clone(), + userid_selfsigningkeyid: db.userid_selfsigningkeyid.clone(), + userid_usersigningkeyid: db.userid_usersigningkeyid.clone(), + keychangeid_userid: db.keychangeid_userid.clone(), + todeviceid_events: db.todeviceid_events.clone(), + userfilterid_filter: db.userfilterid_filter.clone(), + _db: db, + } + } + /// Check if a user has an account on this homeserver. - fn exists(&self, user_id: &UserId) -> Result { Ok(self.userid_password.get(user_id.as_bytes())?.is_some()) } + pub(super) fn exists(&self, user_id: &UserId) -> Result { + Ok(self.userid_password.get(user_id.as_bytes())?.is_some()) + } /// Check if account is deactivated - fn is_deactivated(&self, user_id: &UserId) -> Result { + pub(super) fn is_deactivated(&self, user_id: &UserId) -> Result { Ok(self .userid_password .get(user_id.as_bytes())? @@ -161,10 +72,10 @@ impl Data for KeyValueDatabase { } /// Returns the number of users registered on this server. - fn count(&self) -> Result { Ok(self.userid_password.iter().count()) } + pub(super) fn count(&self) -> Result { Ok(self.userid_password.iter().count()) } /// Find out which user an access token belongs to. - fn find_from_token(&self, token: &str) -> Result> { + pub(super) fn find_from_token(&self, token: &str) -> Result> { self.token_userdeviceid .get(token.as_bytes())? .map_or(Ok(None), |bytes| { @@ -189,7 +100,7 @@ impl Data for KeyValueDatabase { } /// Returns an iterator over all users on this homeserver. - fn iter<'a>(&'a self) -> Box> + 'a> { + pub fn iter<'a>(&'a self) -> Box> + 'a> { Box::new(self.userid_password.iter().map(|(bytes, _)| { UserId::parse( utils::string_from_bytes(&bytes) @@ -203,7 +114,7 @@ impl Data for KeyValueDatabase { /// /// A user account is considered `local` if the length of it's password is /// greater then zero. - fn list_local_users(&self) -> Result> { + pub(super) fn list_local_users(&self) -> Result> { let users: Vec = self .userid_password .iter() @@ -213,7 +124,7 @@ impl Data for KeyValueDatabase { } /// Returns the password hash for the given user. - fn password_hash(&self, user_id: &UserId) -> Result> { + pub(super) fn password_hash(&self, user_id: &UserId) -> Result> { self.userid_password .get(user_id.as_bytes())? .map_or(Ok(None), |bytes| { @@ -224,7 +135,7 @@ impl Data for KeyValueDatabase { } /// Hash and set the user's password to the Argon2 hash - fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { + pub(super) fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { if let Some(password) = password { if let Ok(hash) = utils::hash::password(password) { self.userid_password @@ -243,7 +154,7 @@ impl Data for KeyValueDatabase { } /// Returns the displayname of a user on this homeserver. - fn displayname(&self, user_id: &UserId) -> Result> { + pub(super) fn displayname(&self, user_id: &UserId) -> Result> { self.userid_displayname .get(user_id.as_bytes())? .map_or(Ok(None), |bytes| { @@ -256,7 +167,7 @@ impl Data for KeyValueDatabase { /// Sets a new displayname or removes it if displayname is None. You still /// need to nofify all rooms of this change. - fn set_displayname(&self, user_id: &UserId, displayname: Option) -> Result<()> { + pub(super) fn set_displayname(&self, user_id: &UserId, displayname: Option) -> Result<()> { if let Some(displayname) = displayname { self.userid_displayname .insert(user_id.as_bytes(), displayname.as_bytes())?; @@ -268,7 +179,7 @@ impl Data for KeyValueDatabase { } /// Get the `avatar_url` of a user. - fn avatar_url(&self, user_id: &UserId) -> Result> { + pub(super) fn avatar_url(&self, user_id: &UserId) -> Result> { self.userid_avatarurl .get(user_id.as_bytes())? .map(|bytes| { @@ -283,7 +194,7 @@ impl Data for KeyValueDatabase { } /// Sets a new avatar_url or removes it if avatar_url is None. - fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) -> Result<()> { + pub(super) fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) -> Result<()> { if let Some(avatar_url) = avatar_url { self.userid_avatarurl .insert(user_id.as_bytes(), avatar_url.to_string().as_bytes())?; @@ -295,7 +206,7 @@ impl Data for KeyValueDatabase { } /// Get the blurhash of a user. - fn blurhash(&self, user_id: &UserId) -> Result> { + pub(super) fn blurhash(&self, user_id: &UserId) -> Result> { self.userid_blurhash .get(user_id.as_bytes())? .map(|bytes| { @@ -307,8 +218,8 @@ impl Data for KeyValueDatabase { .transpose() } - /// Sets a new blurhash or removes it if blurhash is None. - fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()> { + /// Sets a new avatar_url or removes it if avatar_url is None. + pub(super) fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()> { if let Some(blurhash) = blurhash { self.userid_blurhash .insert(user_id.as_bytes(), blurhash.as_bytes())?; @@ -320,7 +231,7 @@ impl Data for KeyValueDatabase { } /// Adds a new device to a user. - fn create_device( + pub(super) fn create_device( &self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option, ) -> Result<()> { // This method should never be called for nonexistent users. We shouldn't assert @@ -354,7 +265,7 @@ impl Data for KeyValueDatabase { } /// Removes a device from a user. - fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { + pub(super) fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { let mut userdeviceid = user_id.as_bytes().to_vec(); userdeviceid.push(0xFF); userdeviceid.extend_from_slice(device_id.as_bytes()); @@ -384,7 +295,9 @@ impl Data for KeyValueDatabase { } /// Returns an iterator over all device ids of this user. - fn all_device_ids<'a>(&'a self, user_id: &UserId) -> Box> + 'a> { + pub(super) fn all_device_ids<'a>( + &'a self, user_id: &UserId, + ) -> Box> + 'a> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); // All devices have metadata @@ -405,7 +318,7 @@ impl Data for KeyValueDatabase { } /// Replaces the access token of one device. - fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> { + pub(super) fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> { let mut userdeviceid = user_id.as_bytes().to_vec(); userdeviceid.push(0xFF); userdeviceid.extend_from_slice(device_id.as_bytes()); @@ -436,7 +349,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn add_one_time_key( + pub(super) fn add_one_time_key( &self, user_id: &UserId, device_id: &DeviceId, one_time_key_key: &DeviceKeyId, one_time_key_value: &Raw, ) -> Result<()> { @@ -478,7 +391,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn last_one_time_keys_update(&self, user_id: &UserId) -> Result { + pub(super) fn last_one_time_keys_update(&self, user_id: &UserId) -> Result { self.userid_lastonetimekeyupdate .get(user_id.as_bytes())? .map_or(Ok(0), |bytes| { @@ -487,7 +400,7 @@ impl Data for KeyValueDatabase { }) } - fn take_one_time_key( + pub(super) fn take_one_time_key( &self, user_id: &UserId, device_id: &DeviceId, key_algorithm: &DeviceKeyAlgorithm, ) -> Result)>> { let mut prefix = user_id.as_bytes().to_vec(); @@ -521,7 +434,7 @@ impl Data for KeyValueDatabase { .transpose() } - fn count_one_time_keys( + pub(super) fn count_one_time_keys( &self, user_id: &UserId, device_id: &DeviceId, ) -> Result> { let mut userdeviceid = user_id.as_bytes().to_vec(); @@ -551,7 +464,9 @@ impl Data for KeyValueDatabase { Ok(counts) } - fn add_device_keys(&self, user_id: &UserId, device_id: &DeviceId, device_keys: &Raw) -> Result<()> { + pub(super) fn add_device_keys( + &self, user_id: &UserId, device_id: &DeviceId, device_keys: &Raw, + ) -> Result<()> { let mut userdeviceid = user_id.as_bytes().to_vec(); userdeviceid.push(0xFF); userdeviceid.extend_from_slice(device_id.as_bytes()); @@ -566,7 +481,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn add_cross_signing_keys( + pub(super) fn add_cross_signing_keys( &self, user_id: &UserId, master_key: &Raw, self_signing_key: &Option>, user_signing_key: &Option>, notify: bool, ) -> Result<()> { @@ -574,7 +489,7 @@ impl Data for KeyValueDatabase { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); - let (master_key_key, _) = self.parse_master_key(user_id, master_key)?; + let (master_key_key, _) = Self::parse_master_key(user_id, master_key)?; self.keyid_key .insert(&master_key_key, master_key.json().get().as_bytes())?; @@ -647,7 +562,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn sign_key( + pub(super) fn sign_key( &self, target_id: &UserId, key_id: &str, signature: (String, String), sender_id: &UserId, ) -> Result<()> { let mut key = target_id.as_bytes().to_vec(); @@ -685,7 +600,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn keys_changed<'a>( + pub(super) fn keys_changed<'a>( &'a self, user_or_room_id: &str, from: u64, to: Option, ) -> Box> + 'a> { let mut prefix = user_or_room_id.as_bytes().to_vec(); @@ -724,7 +639,7 @@ impl Data for KeyValueDatabase { ) } - fn mark_device_key_update(&self, user_id: &UserId) -> Result<()> { + pub(super) fn mark_device_key_update(&self, user_id: &UserId) -> Result<()> { let count = services().globals.next_count()?.to_be_bytes(); for room_id in services() .rooms @@ -757,7 +672,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn get_device_keys(&self, user_id: &UserId, device_id: &DeviceId) -> Result>> { + pub(super) fn get_device_keys(&self, user_id: &UserId, device_id: &DeviceId) -> Result>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(device_id.as_bytes()); @@ -769,8 +684,8 @@ impl Data for KeyValueDatabase { }) } - fn parse_master_key( - &self, user_id: &UserId, master_key: &Raw, + pub(super) fn parse_master_key( + user_id: &UserId, master_key: &Raw, ) -> Result<(Vec, CrossSigningKey)> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -793,7 +708,7 @@ impl Data for KeyValueDatabase { Ok((master_key_key, master_key)) } - fn get_key( + pub(super) fn get_key( &self, key: &[u8], sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, ) -> Result>> { self.keyid_key.get(key)?.map_or(Ok(None), |bytes| { @@ -807,7 +722,7 @@ impl Data for KeyValueDatabase { }) } - fn get_master_key( + pub(super) fn get_master_key( &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, ) -> Result>> { self.userid_masterkeyid @@ -815,7 +730,7 @@ impl Data for KeyValueDatabase { .map_or(Ok(None), |key| self.get_key(&key, sender_user, user_id, allowed_signatures)) } - fn get_self_signing_key( + pub(super) fn get_self_signing_key( &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, ) -> Result>> { self.userid_selfsigningkeyid @@ -823,7 +738,7 @@ impl Data for KeyValueDatabase { .map_or(Ok(None), |key| self.get_key(&key, sender_user, user_id, allowed_signatures)) } - fn get_user_signing_key(&self, user_id: &UserId) -> Result>> { + pub(super) fn get_user_signing_key(&self, user_id: &UserId) -> Result>> { self.userid_usersigningkeyid .get(user_id.as_bytes())? .map_or(Ok(None), |key| { @@ -836,7 +751,7 @@ impl Data for KeyValueDatabase { }) } - fn add_to_device_event( + pub(super) fn add_to_device_event( &self, sender: &UserId, target_user_id: &UserId, target_device_id: &DeviceId, event_type: &str, content: serde_json::Value, ) -> Result<()> { @@ -858,7 +773,9 @@ impl Data for KeyValueDatabase { Ok(()) } - fn get_to_device_events(&self, user_id: &UserId, device_id: &DeviceId) -> Result>> { + pub(super) fn get_to_device_events( + &self, user_id: &UserId, device_id: &DeviceId, + ) -> Result>> { let mut events = Vec::new(); let mut prefix = user_id.as_bytes().to_vec(); @@ -876,7 +793,7 @@ impl Data for KeyValueDatabase { Ok(events) } - fn remove_to_device_events(&self, user_id: &UserId, device_id: &DeviceId, until: u64) -> Result<()> { + pub(super) fn remove_to_device_events(&self, user_id: &UserId, device_id: &DeviceId, until: u64) -> Result<()> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); prefix.extend_from_slice(device_id.as_bytes()); @@ -905,7 +822,7 @@ impl Data for KeyValueDatabase { Ok(()) } - fn update_device_metadata(&self, user_id: &UserId, device_id: &DeviceId, device: &Device) -> Result<()> { + pub(super) fn update_device_metadata(&self, user_id: &UserId, device_id: &DeviceId, device: &Device) -> Result<()> { let mut userdeviceid = user_id.as_bytes().to_vec(); userdeviceid.push(0xFF); userdeviceid.extend_from_slice(device_id.as_bytes()); @@ -935,7 +852,7 @@ impl Data for KeyValueDatabase { } /// Get device metadata. - fn get_device_metadata(&self, user_id: &UserId, device_id: &DeviceId) -> Result> { + pub(super) fn get_device_metadata(&self, user_id: &UserId, device_id: &DeviceId) -> Result> { let mut userdeviceid = user_id.as_bytes().to_vec(); userdeviceid.push(0xFF); userdeviceid.extend_from_slice(device_id.as_bytes()); @@ -949,7 +866,7 @@ impl Data for KeyValueDatabase { }) } - fn get_devicelist_version(&self, user_id: &UserId) -> Result> { + pub(super) fn get_devicelist_version(&self, user_id: &UserId) -> Result> { self.userid_devicelistversion .get(user_id.as_bytes())? .map_or(Ok(None), |bytes| { @@ -959,7 +876,9 @@ impl Data for KeyValueDatabase { }) } - fn all_devices_metadata<'a>(&'a self, user_id: &UserId) -> Box> + 'a> { + pub(super) fn all_devices_metadata<'a>( + &'a self, user_id: &UserId, + ) -> Box> + 'a> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); @@ -974,7 +893,7 @@ impl Data for KeyValueDatabase { } /// Creates a new sync filter. Returns the filter id. - fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> Result { + pub(super) fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> Result { let filter_id = utils::random_string(4); let mut key = user_id.as_bytes().to_vec(); @@ -987,7 +906,7 @@ impl Data for KeyValueDatabase { Ok(filter_id) } - fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result> { + pub(super) fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(filter_id.as_bytes()); @@ -1006,7 +925,7 @@ impl Data for KeyValueDatabase { /// username could be successfully parsed. /// If `utils::string_from_bytes`(...) returns an error that username will be /// skipped and the error will be logged. -fn get_username_with_valid_password(username: &[u8], password: &[u8]) -> Option { +pub(super) fn get_username_with_valid_password(username: &[u8], password: &[u8]) -> Option { // A valid password is not empty if password.is_empty() { None diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index c3de86a0..a5349d97 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,8 +1,10 @@ +use conduit::Server; + mod data; use std::{ collections::{BTreeMap, BTreeSet}, mem, - sync::{Arc, Mutex}, + sync::{Arc, Mutex, Mutex as StdMutex}, }; use data::Data; @@ -22,7 +24,7 @@ use ruma::{ UInt, UserId, }; -use crate::{service, services, Error, Result}; +use crate::{database::KeyValueDatabase, service, services, Error, Result}; pub struct SlidingSyncCache { lists: BTreeMap, @@ -34,11 +36,18 @@ pub struct SlidingSyncCache { type DbConnections = Mutex>>>; pub struct Service { - pub db: Arc, + pub db: Data, pub connections: DbConnections, } impl Service { + pub fn build(_server: &Arc, db: &Arc) -> Result { + Ok(Self { + db: Data::new(db.clone()), + connections: StdMutex::new(BTreeMap::new()), + }) + } + /// Check if a user has an account on this homeserver. pub fn exists(&self, user_id: &UserId) -> Result { self.db.exists(user_id) } @@ -381,7 +390,7 @@ impl Service { pub fn parse_master_key( &self, user_id: &UserId, master_key: &Raw, ) -> Result<(Vec, CrossSigningKey)> { - self.db.parse_master_key(user_id, master_key) + Data::parse_master_key(user_id, master_key) } pub fn get_key( From de21f7442a8600b08c4e44cb69d616a3656bc6ff Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 28 May 2024 06:59:50 +0000 Subject: [PATCH 0277/2091] devirtualize database Signed-off-by: Jason Volk --- src/database/cork.rs | 10 +- src/database/database.rs | 51 ++++ src/database/{rocksdb/mod.rs => engine.rs} | 152 +++++------ src/database/kvdatabase.rs | 302 --------------------- src/database/kvengine.rs | 38 --- src/database/kvtree.rs | 62 ----- src/database/map.rs | 242 +++++++++++++++++ src/database/maps.rs | 99 +++++++ src/database/mod.rs | 26 +- src/database/{rocksdb => }/opts.rs | 19 +- src/database/rocksdb/kvtree.rs | 227 ---------------- src/database/util.rs | 13 + 12 files changed, 497 insertions(+), 744 deletions(-) create mode 100644 src/database/database.rs rename src/database/{rocksdb/mod.rs => engine.rs} (63%) delete mode 100644 src/database/kvdatabase.rs delete mode 100644 src/database/kvengine.rs delete mode 100644 src/database/kvtree.rs create mode 100644 src/database/map.rs create mode 100644 src/database/maps.rs rename src/database/{rocksdb => }/opts.rs (96%) delete mode 100644 src/database/rocksdb/kvtree.rs create mode 100644 src/database/util.rs diff --git a/src/database/cork.rs b/src/database/cork.rs index dcabd707..faa3a491 100644 --- a/src/database/cork.rs +++ b/src/database/cork.rs @@ -1,16 +1,16 @@ use std::sync::Arc; -use super::KeyValueDatabaseEngine; +use crate::Engine; pub struct Cork { - db: Arc, + db: Arc, flush: bool, sync: bool, } impl Cork { - pub fn new(db: &Arc, flush: bool, sync: bool) -> Self { - db.cork().unwrap(); + pub fn new(db: &Arc, flush: bool, sync: bool) -> Self { + db.cork(); Self { db: db.clone(), flush, @@ -21,7 +21,7 @@ impl Cork { impl Drop for Cork { fn drop(&mut self) { - self.db.uncork().ok(); + self.db.uncork(); if self.flush { self.db.flush().ok(); } diff --git a/src/database/database.rs b/src/database/database.rs new file mode 100644 index 00000000..e995d987 --- /dev/null +++ b/src/database/database.rs @@ -0,0 +1,51 @@ +use std::{ + collections::{BTreeMap, HashMap}, + ops::Index, + sync::{Arc, Mutex, RwLock}, +}; + +use conduit::{PduCount, Result, Server}; +use lru_cache::LruCache; +use ruma::{CanonicalJsonValue, OwnedDeviceId, OwnedRoomId, OwnedUserId}; + +use crate::{maps, maps::Maps, Engine, Map}; + +pub struct Database { + pub db: Arc, + pub map: Maps, + + //TODO: not a database + pub userdevicesessionid_uiaarequest: RwLock>, + pub auth_chain_cache: Mutex, Arc<[u64]>>>, + pub appservice_in_room_cache: RwLock>>, + pub lasttimelinecount_cache: Mutex>, +} + +impl Database { + /// Load an existing database or create a new one. + pub async fn open(server: &Arc) -> Result { + let config = &server.config; + let db = Engine::open(server)?; + Ok(Self { + db: db.clone(), + map: maps::open(&db)?, + + userdevicesessionid_uiaarequest: RwLock::new(BTreeMap::new()), + appservice_in_room_cache: RwLock::new(HashMap::new()), + lasttimelinecount_cache: Mutex::new(HashMap::new()), + auth_chain_cache: Mutex::new(LruCache::new( + (f64::from(config.auth_chain_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, + )), + }) + } +} + +impl Index<&str> for Database { + type Output = Arc; + + fn index(&self, name: &str) -> &Self::Output { + self.map + .get(name) + .expect("column in database does not exist") + } +} diff --git a/src/database/rocksdb/mod.rs b/src/database/engine.rs similarity index 63% rename from src/database/rocksdb/mod.rs rename to src/database/engine.rs index 943b6745..396d327d 100644 --- a/src/database/rocksdb/mod.rs +++ b/src/database/engine.rs @@ -1,45 +1,36 @@ -// no_link to prevent double-inclusion of librocksdb.a here and with -// libconduit_core.so -#[no_link] -extern crate rust_rocksdb; - use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, fmt::Write, - sync::{atomic::AtomicU32, Arc}, + sync::{atomic::AtomicU32, Arc, Mutex, RwLock}, }; use chrono::{DateTime, Utc}; -use rust_rocksdb::{ +use conduit::{debug, error, info, warn, Result, Server}; +use rocksdb::{ backup::{BackupEngine, BackupEngineOptions}, perf::get_memory_usage_stats, - Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode as Db, Env, MultiThreaded, Options, + BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode as Db, Env, MultiThreaded, Options, }; -use tracing::{debug, error, info, warn}; -use crate::{watchers::Watchers, Config, KeyValueDatabaseEngine, KvTree, Result}; +use crate::{ + opts::{cf_options, db_options}, + or_else, result, +}; -pub(crate) mod kvtree; -pub(crate) mod opts; - -use kvtree::RocksDbEngineTree; -use opts::{cf_options, db_options}; - -use super::watchers; - -pub(crate) struct Engine { - config: Config, +pub struct Engine { + server: Arc, row_cache: Cache, - col_cache: HashMap, + col_cache: RwLock>, opts: Options, env: Env, - old_cfs: Vec, - rocks: Db, + cfs: Mutex>, + pub(crate) db: Db, corks: AtomicU32, } -impl KeyValueDatabaseEngine for Arc { - fn open(config: &Config) -> Result { +impl Engine { + pub(crate) fn open(server: &Arc) -> Result> { + let config = &server.config; let cache_capacity_bytes = config.db_cache_capacity_mb * 1024.0 * 1024.0; #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] @@ -85,61 +76,64 @@ impl KeyValueDatabaseEngine for Arc { db.latest_sequence_number(), load_time.elapsed() ); - Ok(Self::new(Engine { - config: config.clone(), + + let cfs = HashSet::::from_iter(cfs); + Ok(Arc::new(Self { + server: server.clone(), row_cache, - col_cache, + col_cache: RwLock::new(col_cache), opts: db_opts, env: db_env, - old_cfs: cfs, - rocks: db, + cfs: Mutex::new(cfs), + db, corks: AtomicU32::new(0), })) } - fn open_tree(&self, name: &'static str) -> Result> { - if !self.old_cfs.contains(&name.to_owned()) { - // Create if it didn't exist + pub(crate) fn open_cf(&self, name: &str) -> Result>> { + let mut cfs = self.cfs.lock().expect("locked"); + if !cfs.contains(name) { debug!("Creating new column family in database: {}", name); - // TODO: the workaround for this needs to be extended to rocksdb caches, but i - // dont know that code to safely do that - #[allow(clippy::let_underscore_must_use)] - #[allow(clippy::let_underscore_untyped)] // attributes on expressions are experimental - let _ = self.rocks.create_cf(name, &self.opts); + let mut col_cache = self.col_cache.write().expect("locked"); + let opts = cf_options(&self.server.config, name, self.opts.clone(), &mut col_cache); + if let Err(e) = self.db.create_cf(name, &opts) { + error!("Failed to create new column family: {e}"); + return or_else(e); + } + + cfs.insert(name.to_owned()); } - Ok(Arc::new(RocksDbEngineTree { - name, - db: Self::clone(self), - watchers: Watchers::default(), - })) + Ok(self.cf(name)) } - fn flush(&self) -> Result<()> { result(DBCommon::flush_wal(&self.rocks, false)) } + pub(crate) fn cf<'db>(&'db self, name: &str) -> Arc> { + self.db + .cf_handle(name) + .expect("column was created and exists") + } - fn sync(&self) -> Result<()> { result(DBCommon::flush_wal(&self.rocks, true)) } + pub fn flush(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, false)) } - fn corked(&self) -> bool { self.corks.load(std::sync::atomic::Ordering::Relaxed) > 0 } + pub fn sync(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, true)) } - fn cork(&self) -> Result<()> { + pub(crate) fn corked(&self) -> bool { self.corks.load(std::sync::atomic::Ordering::Relaxed) > 0 } + + pub(crate) fn cork(&self) { self.corks .fetch_add(1, std::sync::atomic::Ordering::Relaxed); - - Ok(()) } - fn uncork(&self) -> Result<()> { + pub(crate) fn uncork(&self) { self.corks .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); - - Ok(()) } #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] - fn memory_usage(&self) -> Result { + pub fn memory_usage(&self) -> Result { let mut res = String::new(); - let stats = get_memory_usage_stats(Some(&[&self.rocks]), Some(&[&self.row_cache])).or_else(or_else)?; + let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&self.row_cache])).or_else(or_else)?; writeln!( res, "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow cache: {:.2} MiB", @@ -150,7 +144,7 @@ impl KeyValueDatabaseEngine for Arc { ) .expect("should be able to write to string buffer"); - for (name, cache) in &self.col_cache { + for (name, cache) in &*self.col_cache.read().expect("locked") { writeln!(res, "{} cache: {:.2} MiB", name, cache.get_usage() as f64 / 1024.0 / 1024.0,) .expect("should be able to write to string buffer"); } @@ -158,22 +152,23 @@ impl KeyValueDatabaseEngine for Arc { Ok(res) } - fn cleanup(&self) -> Result<()> { + pub fn cleanup(&self) -> Result<()> { debug!("Running flush_opt"); - let flushoptions = rust_rocksdb::FlushOptions::default(); - result(DBCommon::flush_opt(&self.rocks, &flushoptions)) + let flushoptions = rocksdb::FlushOptions::default(); + result(DBCommon::flush_opt(&self.db, &flushoptions)) } - fn backup(&self) -> Result<(), Box> { - let path = self.config.database_backup_path.as_ref(); + pub fn backup(&self) -> Result<(), Box> { + let config = &self.server.config; + let path = config.database_backup_path.as_ref(); if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { return Ok(()); } let options = BackupEngineOptions::new(path.unwrap())?; let mut engine = BackupEngine::open(&options, &self.env)?; - if self.config.database_backups_to_keep > 0 { - if let Err(e) = engine.create_new_backup_flush(&self.rocks, true) { + if config.database_backups_to_keep > 0 { + if let Err(e) = engine.create_new_backup_flush(&self.db, true) { return Err(Box::new(e)); } @@ -185,8 +180,8 @@ impl KeyValueDatabaseEngine for Arc { ); } - if self.config.database_backups_to_keep >= 0 { - let keep = u32::try_from(self.config.database_backups_to_keep)?; + if config.database_backups_to_keep >= 0 { + let keep = u32::try_from(config.database_backups_to_keep)?; if let Err(e) = engine.purge_old_backups(keep.try_into()?) { error!("Failed to purge old backup: {:?}", e.to_string()); } @@ -195,8 +190,9 @@ impl KeyValueDatabaseEngine for Arc { Ok(()) } - fn backup_list(&self) -> Result { - let path = self.config.database_backup_path.as_ref(); + pub fn backup_list(&self) -> Result { + let config = &self.server.config; + let path = config.database_backup_path.as_ref(); if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { return Ok( "Configure database_backup_path to enable backups, or the path specified is not valid".to_owned(), @@ -223,8 +219,8 @@ impl KeyValueDatabaseEngine for Arc { Ok(res) } - fn file_list(&self) -> Result { - match self.rocks.live_files() { + pub fn file_list(&self) -> Result { + match self.db.live_files() { Err(e) => Ok(String::from(e)), Ok(files) => { let mut res = String::new(); @@ -242,10 +238,6 @@ impl KeyValueDatabaseEngine for Arc { }, } } - - // TODO: figure out if this is needed for rocksdb - #[allow(dead_code)] - fn clear_caches(&self) {} } impl Drop for Engine { @@ -253,7 +245,7 @@ impl Drop for Engine { const BLOCKING: bool = true; debug!("Waiting for background tasks to finish..."); - self.rocks.cancel_all_background_work(BLOCKING); + self.db.cancel_all_background_work(BLOCKING); debug!("Shutting down background threads"); self.env.set_high_priority_background_threads(0); @@ -265,15 +257,3 @@ impl Drop for Engine { self.env.join_all_threads(); } } - -#[inline] -fn result(r: std::result::Result) -> Result { - r.map_or_else(or_else, and_then) -} - -#[inline(always)] -fn and_then(t: T) -> Result { Ok(t) } - -fn or_else(e: rust_rocksdb::Error) -> Result { Err(map_err(e)) } - -fn map_err(e: rust_rocksdb::Error) -> conduit::Error { conduit::Error::Database(e.into_string()) } diff --git a/src/database/kvdatabase.rs b/src/database/kvdatabase.rs deleted file mode 100644 index 4f5fbacd..00000000 --- a/src/database/kvdatabase.rs +++ /dev/null @@ -1,302 +0,0 @@ -use std::{ - collections::{BTreeMap, HashMap}, - path::Path, - sync::{Arc, Mutex, RwLock}, -}; - -use conduit::{Config, Error, PduCount, Result, Server}; -use lru_cache::LruCache; -use ruma::{CanonicalJsonValue, OwnedDeviceId, OwnedRoomId, OwnedUserId}; -use tracing::debug; - -use crate::{KeyValueDatabaseEngine, KvTree}; - -pub struct KeyValueDatabase { - pub db: Arc, - - //pub globals: globals::Globals, - pub global: Arc, - pub server_signingkeys: Arc, - - pub roomid_inviteviaservers: Arc, - - //pub users: users::Users, - pub userid_password: Arc, - pub userid_displayname: Arc, - pub userid_avatarurl: Arc, - pub userid_blurhash: Arc, - pub userdeviceid_token: Arc, - pub userdeviceid_metadata: Arc, // This is also used to check if a device exists - pub userid_devicelistversion: Arc, // DevicelistVersion = u64 - pub token_userdeviceid: Arc, - - pub onetimekeyid_onetimekeys: Arc, // OneTimeKeyId = UserId + DeviceKeyId - pub userid_lastonetimekeyupdate: Arc, // LastOneTimeKeyUpdate = Count - pub keychangeid_userid: Arc, // KeyChangeId = UserId/RoomId + Count - pub keyid_key: Arc, // KeyId = UserId + KeyId (depends on key type) - pub userid_masterkeyid: Arc, - pub userid_selfsigningkeyid: Arc, - pub userid_usersigningkeyid: Arc, - - pub userfilterid_filter: Arc, // UserFilterId = UserId + FilterId - pub todeviceid_events: Arc, // ToDeviceId = UserId + DeviceId + Count - pub userid_presenceid: Arc, // UserId => Count - pub presenceid_presence: Arc, // Count + UserId => Presence - - //pub uiaa: uiaa::Uiaa, - pub userdevicesessionid_uiaainfo: Arc, // User-interactive authentication - pub userdevicesessionid_uiaarequest: RwLock>, - - //pub edus: RoomEdus, - pub readreceiptid_readreceipt: Arc, // ReadReceiptId = RoomId + Count + UserId - pub roomuserid_privateread: Arc, // RoomUserId = Room + User, PrivateRead = Count - pub roomuserid_lastprivatereadupdate: Arc, // LastPrivateReadUpdate = Count - - //pub rooms: rooms::Rooms, - pub pduid_pdu: Arc, // PduId = ShortRoomId + Count - pub eventid_pduid: Arc, - pub roomid_pduleaves: Arc, - pub alias_roomid: Arc, - pub alias_userid: Arc, // UserId = AliasId (User who created the alias) - pub aliasid_alias: Arc, // AliasId = RoomId + Count - pub publicroomids: Arc, - - pub threadid_userids: Arc, // ThreadId = RoomId + Count - - pub tokenids: Arc, // TokenId = ShortRoomId + Token + PduIdCount - - /// Participating servers in a room. - pub roomserverids: Arc, // RoomServerId = RoomId + ServerName - pub serverroomids: Arc, // ServerRoomId = ServerName + RoomId - - pub userroomid_joined: Arc, - pub roomuserid_joined: Arc, - pub roomid_joinedcount: Arc, - pub roomid_invitedcount: Arc, - pub roomuseroncejoinedids: Arc, - pub userroomid_invitestate: Arc, // InviteState = Vec> - pub roomuserid_invitecount: Arc, // InviteCount = Count - pub userroomid_leftstate: Arc, - pub roomuserid_leftcount: Arc, - - pub disabledroomids: Arc, // Rooms where incoming federation handling is disabled - - pub bannedroomids: Arc, // Rooms where local users are not allowed to join - - pub lazyloadedids: Arc, // LazyLoadedIds = UserId + DeviceId + RoomId + LazyLoadedUserId - - pub userroomid_notificationcount: Arc, // NotifyCount = u64 - pub userroomid_highlightcount: Arc, // HightlightCount = u64 - pub roomuserid_lastnotificationread: Arc, // LastNotificationRead = u64 - - /// Remember the current state hash of a room. - pub roomid_shortstatehash: Arc, - pub roomsynctoken_shortstatehash: Arc, - /// Remember the state hash at events in the past. - pub shorteventid_shortstatehash: Arc, - pub statekey_shortstatekey: Arc, /* StateKey = EventType + StateKey, ShortStateKey = - * Count */ - pub shortstatekey_statekey: Arc, - - pub roomid_shortroomid: Arc, - - pub shorteventid_eventid: Arc, - pub eventid_shorteventid: Arc, - - pub statehash_shortstatehash: Arc, - pub shortstatehash_statediff: Arc, /* StateDiff = parent (or 0) + - * (shortstatekey+shorteventid++) + 0_u64 + - * (shortstatekey+shorteventid--) */ - - pub shorteventid_authchain: Arc, - - /// RoomId + EventId -> outlier PDU. - /// Any pdu that has passed the steps 1-8 in the incoming event - /// /federation/send/txn. - pub eventid_outlierpdu: Arc, - pub softfailedeventids: Arc, - - /// ShortEventId + ShortEventId -> (). - pub tofrom_relation: Arc, - /// RoomId + EventId -> Parent PDU EventId. - pub referencedevents: Arc, - - //pub account_data: account_data::AccountData, - pub roomuserdataid_accountdata: Arc, // RoomUserDataId = Room + User + Count + Type - pub roomusertype_roomuserdataid: Arc, // RoomUserType = Room + User + Type - - //pub media: media::Media, - pub mediaid_file: Arc, // MediaId = MXC + WidthHeight + ContentDisposition + ContentType - pub url_previews: Arc, - pub mediaid_user: Arc, - //pub key_backups: key_backups::KeyBackups, - pub backupid_algorithm: Arc, // BackupId = UserId + Version(Count) - pub backupid_etag: Arc, // BackupId = UserId + Version(Count) - pub backupkeyid_backup: Arc, // BackupKeyId = UserId + Version + RoomId + SessionId - - //pub transaction_ids: transaction_ids::TransactionIds, - pub userdevicetxnid_response: Arc, /* Response can be empty (/sendToDevice) or the event id - * (/send) */ - //pub sending: sending::Sending, - pub servername_educount: Arc, // EduCount: Count of last EDU sync - pub servernameevent_data: Arc, /* ServernameEvent = (+ / $)SenderKey / ServerName / UserId + - * PduId / Id (for edus), Data = EDU content */ - pub servercurrentevent_data: Arc, /* ServerCurrentEvents = (+ / $)ServerName / UserId + PduId - * / Id (for edus), Data = EDU content */ - - //pub appservice: appservice::Appservice, - pub id_appserviceregistrations: Arc, - - //pub pusher: pusher::PushData, - pub senderkey_pusher: Arc, - - pub auth_chain_cache: Mutex, Arc<[u64]>>>, - pub appservice_in_room_cache: RwLock>>, - pub lasttimelinecount_cache: Mutex>, -} - -impl KeyValueDatabase { - /// Load an existing database or create a new one. - #[allow(clippy::too_many_lines)] - pub async fn load_or_create(server: &Arc) -> Result { - let config = &server.config; - check_db_setup(config)?; - let builder = build(config)?; - Ok(Self { - db: builder.clone(), - userid_password: builder.open_tree("userid_password")?, - userid_displayname: builder.open_tree("userid_displayname")?, - userid_avatarurl: builder.open_tree("userid_avatarurl")?, - userid_blurhash: builder.open_tree("userid_blurhash")?, - userdeviceid_token: builder.open_tree("userdeviceid_token")?, - userdeviceid_metadata: builder.open_tree("userdeviceid_metadata")?, - userid_devicelistversion: builder.open_tree("userid_devicelistversion")?, - token_userdeviceid: builder.open_tree("token_userdeviceid")?, - onetimekeyid_onetimekeys: builder.open_tree("onetimekeyid_onetimekeys")?, - userid_lastonetimekeyupdate: builder.open_tree("userid_lastonetimekeyupdate")?, - keychangeid_userid: builder.open_tree("keychangeid_userid")?, - keyid_key: builder.open_tree("keyid_key")?, - userid_masterkeyid: builder.open_tree("userid_masterkeyid")?, - userid_selfsigningkeyid: builder.open_tree("userid_selfsigningkeyid")?, - userid_usersigningkeyid: builder.open_tree("userid_usersigningkeyid")?, - userfilterid_filter: builder.open_tree("userfilterid_filter")?, - todeviceid_events: builder.open_tree("todeviceid_events")?, - userid_presenceid: builder.open_tree("userid_presenceid")?, - presenceid_presence: builder.open_tree("presenceid_presence")?, - - userdevicesessionid_uiaainfo: builder.open_tree("userdevicesessionid_uiaainfo")?, - userdevicesessionid_uiaarequest: RwLock::new(BTreeMap::new()), - readreceiptid_readreceipt: builder.open_tree("readreceiptid_readreceipt")?, - roomuserid_privateread: builder.open_tree("roomuserid_privateread")?, // "Private" read receipt - roomuserid_lastprivatereadupdate: builder.open_tree("roomuserid_lastprivatereadupdate")?, - pduid_pdu: builder.open_tree("pduid_pdu")?, - eventid_pduid: builder.open_tree("eventid_pduid")?, - roomid_pduleaves: builder.open_tree("roomid_pduleaves")?, - - alias_roomid: builder.open_tree("alias_roomid")?, - alias_userid: builder.open_tree("alias_userid")?, - aliasid_alias: builder.open_tree("aliasid_alias")?, - publicroomids: builder.open_tree("publicroomids")?, - - threadid_userids: builder.open_tree("threadid_userids")?, - - tokenids: builder.open_tree("tokenids")?, - - roomserverids: builder.open_tree("roomserverids")?, - serverroomids: builder.open_tree("serverroomids")?, - userroomid_joined: builder.open_tree("userroomid_joined")?, - roomuserid_joined: builder.open_tree("roomuserid_joined")?, - roomid_joinedcount: builder.open_tree("roomid_joinedcount")?, - roomid_invitedcount: builder.open_tree("roomid_invitedcount")?, - roomuseroncejoinedids: builder.open_tree("roomuseroncejoinedids")?, - userroomid_invitestate: builder.open_tree("userroomid_invitestate")?, - roomuserid_invitecount: builder.open_tree("roomuserid_invitecount")?, - userroomid_leftstate: builder.open_tree("userroomid_leftstate")?, - roomuserid_leftcount: builder.open_tree("roomuserid_leftcount")?, - - disabledroomids: builder.open_tree("disabledroomids")?, - - bannedroomids: builder.open_tree("bannedroomids")?, - - lazyloadedids: builder.open_tree("lazyloadedids")?, - - userroomid_notificationcount: builder.open_tree("userroomid_notificationcount")?, - userroomid_highlightcount: builder.open_tree("userroomid_highlightcount")?, - roomuserid_lastnotificationread: builder.open_tree("userroomid_highlightcount")?, - - statekey_shortstatekey: builder.open_tree("statekey_shortstatekey")?, - shortstatekey_statekey: builder.open_tree("shortstatekey_statekey")?, - - shorteventid_authchain: builder.open_tree("shorteventid_authchain")?, - - roomid_shortroomid: builder.open_tree("roomid_shortroomid")?, - - shortstatehash_statediff: builder.open_tree("shortstatehash_statediff")?, - eventid_shorteventid: builder.open_tree("eventid_shorteventid")?, - shorteventid_eventid: builder.open_tree("shorteventid_eventid")?, - shorteventid_shortstatehash: builder.open_tree("shorteventid_shortstatehash")?, - roomid_shortstatehash: builder.open_tree("roomid_shortstatehash")?, - roomsynctoken_shortstatehash: builder.open_tree("roomsynctoken_shortstatehash")?, - statehash_shortstatehash: builder.open_tree("statehash_shortstatehash")?, - - eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?, - softfailedeventids: builder.open_tree("softfailedeventids")?, - - tofrom_relation: builder.open_tree("tofrom_relation")?, - referencedevents: builder.open_tree("referencedevents")?, - roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?, - roomusertype_roomuserdataid: builder.open_tree("roomusertype_roomuserdataid")?, - mediaid_file: builder.open_tree("mediaid_file")?, - url_previews: builder.open_tree("url_previews")?, - mediaid_user: builder.open_tree("mediaid_user")?, - backupid_algorithm: builder.open_tree("backupid_algorithm")?, - backupid_etag: builder.open_tree("backupid_etag")?, - backupkeyid_backup: builder.open_tree("backupkeyid_backup")?, - userdevicetxnid_response: builder.open_tree("userdevicetxnid_response")?, - servername_educount: builder.open_tree("servername_educount")?, - servernameevent_data: builder.open_tree("servernameevent_data")?, - servercurrentevent_data: builder.open_tree("servercurrentevent_data")?, - id_appserviceregistrations: builder.open_tree("id_appserviceregistrations")?, - senderkey_pusher: builder.open_tree("senderkey_pusher")?, - global: builder.open_tree("global")?, - server_signingkeys: builder.open_tree("server_signingkeys")?, - - roomid_inviteviaservers: builder.open_tree("roomid_inviteviaservers")?, - - auth_chain_cache: Mutex::new(LruCache::new( - (f64::from(config.auth_chain_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, - )), - appservice_in_room_cache: RwLock::new(HashMap::new()), - lasttimelinecount_cache: Mutex::new(HashMap::new()), - }) - } -} - -fn build(config: &Config) -> Result> { - match &*config.database_backend { - "rocksdb" => { - debug!("Got rocksdb database backend"); - #[cfg(not(feature = "rocksdb"))] - return Err(Error::bad_config("Database backend not found.")); - #[cfg(feature = "rocksdb")] - Ok(Arc::new(Arc::::open(config)?)) - }, - _ => Err(Error::bad_config( - "Database backend not found. rocksdb is the only supported backend.", - )), - } -} - -fn check_db_setup(config: &Config) -> Result<()> { - let path = Path::new(&config.database_path); - - let rocksdb_exists = path.join("IDENTITY").exists(); - - if rocksdb_exists && config.database_backend != "rocksdb" { - return Err(Error::bad_config( - "Found rocksdb at database_path, but is not specified in config.", - )); - } - - Ok(()) -} diff --git a/src/database/kvengine.rs b/src/database/kvengine.rs deleted file mode 100644 index 1b27c571..00000000 --- a/src/database/kvengine.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::{error::Error, sync::Arc}; - -use super::{Config, KvTree}; -use crate::Result; - -pub trait KeyValueDatabaseEngine: Send + Sync { - fn open(config: &Config) -> Result - where - Self: Sized; - - fn open_tree(&self, name: &'static str) -> Result>; - - fn flush(&self) -> Result<()>; - - #[allow(dead_code)] - fn sync(&self) -> Result<()> { Ok(()) } - - fn cork(&self) -> Result<()> { Ok(()) } - - fn uncork(&self) -> Result<()> { Ok(()) } - - fn corked(&self) -> bool { false } - - fn cleanup(&self) -> Result<()> { Ok(()) } - - fn memory_usage(&self) -> Result { - Ok("Current database engine does not support memory usage reporting.".to_owned()) - } - - #[allow(dead_code)] - fn clear_caches(&self) {} - - fn backup(&self) -> Result<(), Box> { unimplemented!() } - - fn backup_list(&self) -> Result { Ok(String::new()) } - - fn file_list(&self) -> Result { Ok(String::new()) } -} diff --git a/src/database/kvtree.rs b/src/database/kvtree.rs deleted file mode 100644 index 009e45d5..00000000 --- a/src/database/kvtree.rs +++ /dev/null @@ -1,62 +0,0 @@ -use std::{future::Future, pin::Pin}; - -use crate::Result; - -pub trait KvTree: Send + Sync { - fn get(&self, key: &[u8]) -> Result>>; - - #[allow(dead_code)] - fn multi_get(&self, keys: &[&[u8]]) -> Result>>> { - let mut ret: Vec>> = Vec::with_capacity(keys.len()); - for key in keys { - ret.push(self.get(key)?); - } - - Ok(ret) - } - - fn insert(&self, key: &[u8], value: &[u8]) -> Result<()>; - fn insert_batch(&self, iter: &mut dyn Iterator, Vec)>) -> Result<()> { - for (key, value) in iter { - self.insert(&key, &value)?; - } - - Ok(()) - } - - fn remove(&self, key: &[u8]) -> Result<()>; - - #[allow(dead_code)] - fn remove_batch(&self, iter: &mut dyn Iterator>) -> Result<()> { - for key in iter { - self.remove(&key)?; - } - - Ok(()) - } - - fn iter<'a>(&'a self) -> Box, Vec)> + 'a>; - - fn iter_from<'a>(&'a self, from: &[u8], backwards: bool) -> Box, Vec)> + 'a>; - - fn increment(&self, key: &[u8]) -> Result>; - fn increment_batch(&self, iter: &mut dyn Iterator>) -> Result<()> { - for key in iter { - self.increment(&key)?; - } - - Ok(()) - } - - fn scan_prefix<'a>(&'a self, prefix: Vec) -> Box, Vec)> + 'a>; - - fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>>; - - fn clear(&self) -> Result<()> { - for (key, _) in self.iter() { - self.remove(&key)?; - } - - Ok(()) - } -} diff --git a/src/database/map.rs b/src/database/map.rs new file mode 100644 index 00000000..591de04b --- /dev/null +++ b/src/database/map.rs @@ -0,0 +1,242 @@ +use std::{future::Future, pin::Pin, sync::Arc}; + +use conduit::{utils, Result}; +use rocksdb::{BoundColumnFamily, Direction, IteratorMode, ReadOptions, WriteBatchWithTransaction, WriteOptions}; + +use super::{or_else, result, watchers::Watchers, Engine}; + +pub struct Map { + db: Arc, + name: String, + watchers: Watchers, +} + +type Key = Vec; +type Val = Vec; +type KeyVal = (Key, Val); + +impl Map { + pub(crate) fn open(db: &Arc, name: &str) -> Result> { + db.open_cf(name)?; + Ok(Arc::new(Self { + db: db.clone(), + name: name.to_owned(), + watchers: Watchers::default(), + })) + } + + pub fn get(&self, key: &[u8]) -> Result>> { + let mut readoptions = ReadOptions::default(); + readoptions.set_total_order_seek(true); + + result(self.db.db.get_cf_opt(&self.cf(), key, &readoptions)) + } + + pub fn multi_get(&self, keys: &[&[u8]]) -> Result>>> { + // Optimization can be `true` if key vector is pre-sorted **by the column + // comparator**. + const SORTED: bool = false; + + let mut readoptions = ReadOptions::default(); + readoptions.set_total_order_seek(true); + + let mut ret: Vec>> = Vec::with_capacity(keys.len()); + for res in self + .db + .db + .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, &readoptions) + { + match res { + Ok(Some(res)) => ret.push(Some((*res).to_vec())), + Ok(None) => ret.push(None), + Err(e) => return or_else(e), + } + } + + Ok(ret) + } + + pub fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { + let writeoptions = WriteOptions::default(); + + self.db + .db + .put_cf_opt(&self.cf(), key, value, &writeoptions) + .or_else(or_else)?; + + if !self.db.corked() { + self.db.flush()?; + } + + self.watchers.wake(key); + + Ok(()) + } + + pub fn insert_batch(&self, iter: &mut dyn Iterator) -> Result<()> { + let writeoptions = WriteOptions::default(); + + let mut batch = WriteBatchWithTransaction::::default(); + + for (key, value) in iter { + batch.put_cf(&self.cf(), key, value); + } + + let res = self.db.db.write_opt(batch, &writeoptions); + + if !self.db.corked() { + self.db.flush()?; + } + + result(res) + } + + pub fn remove(&self, key: &[u8]) -> Result<()> { + let writeoptions = WriteOptions::default(); + + let res = self.db.db.delete_cf_opt(&self.cf(), key, &writeoptions); + + if !self.db.corked() { + self.db.flush()?; + } + + result(res) + } + + pub fn remove_batch(&self, iter: &mut dyn Iterator) -> Result<()> { + let writeoptions = WriteOptions::default(); + + let mut batch = WriteBatchWithTransaction::::default(); + + for key in iter { + batch.delete_cf(&self.cf(), key); + } + + let res = self.db.db.write_opt(batch, &writeoptions); + + if !self.db.corked() { + self.db.flush()?; + } + + result(res) + } + + pub fn iter<'a>(&'a self) -> Box + 'a> { + let mut readoptions = ReadOptions::default(); + readoptions.set_total_order_seek(true); + + let it = self + .db + .db + .iterator_cf_opt(&self.cf(), readoptions, IteratorMode::Start) + .map(Result::unwrap) + .map(|(k, v)| (Vec::from(k), Vec::from(v))); + + Box::new(it) + } + + pub fn iter_from<'a>(&'a self, from: &[u8], backwards: bool) -> Box + 'a> { + let mut readoptions = ReadOptions::default(); + readoptions.set_total_order_seek(true); + + let it = self + .db + .db + .iterator_cf_opt( + &self.cf(), + readoptions, + IteratorMode::From( + from, + if backwards { + Direction::Reverse + } else { + Direction::Forward + }, + ), + ) + .map(Result::unwrap) + .map(|(k, v)| (Vec::from(k), Vec::from(v))); + + Box::new(it) + } + + pub fn increment(&self, key: &[u8]) -> Result> { + let mut readoptions = ReadOptions::default(); + readoptions.set_total_order_seek(true); + let writeoptions = WriteOptions::default(); + + let old = self + .db + .db + .get_cf_opt(&self.cf(), key, &readoptions) + .or_else(or_else)?; + let new = utils::increment(old.as_deref()); + self.db + .db + .put_cf_opt(&self.cf(), key, new, &writeoptions) + .or_else(or_else)?; + + if !self.db.corked() { + self.db.flush()?; + } + + Ok(new.to_vec()) + } + + pub fn increment_batch(&self, iter: &mut dyn Iterator) -> Result<()> { + let mut readoptions = ReadOptions::default(); + readoptions.set_total_order_seek(true); + let writeoptions = WriteOptions::default(); + + let mut batch = WriteBatchWithTransaction::::default(); + + for key in iter { + let old = self + .db + .db + .get_cf_opt(&self.cf(), &key, &readoptions) + .or_else(or_else)?; + let new = utils::increment(old.as_deref()); + batch.put_cf(&self.cf(), key, new); + } + + self.db + .db + .write_opt(batch, &writeoptions) + .or_else(or_else)?; + + if !self.db.corked() { + self.db.flush()?; + } + + Ok(()) + } + + pub fn scan_prefix<'a>(&'a self, prefix: Vec) -> Box + 'a> { + let mut readoptions = ReadOptions::default(); + readoptions.set_total_order_seek(true); + + let it = self + .db + .db + .iterator_cf_opt(&self.cf(), readoptions, IteratorMode::From(&prefix, Direction::Forward)) + .map(Result::unwrap) + .map(|(k, v)| (Vec::from(k), Vec::from(v))) + .take_while(move |(k, _)| k.starts_with(&prefix)); + + Box::new(it) + } + + pub fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { + self.watchers.watch(prefix) + } + + fn cf(&self) -> Arc> { self.db.cf(&self.name) } +} + +impl<'a> IntoIterator for &'a Map { + type IntoIter = Box + 'a>; + type Item = KeyVal; + + fn into_iter(self) -> Self::IntoIter { self.iter() } +} diff --git a/src/database/maps.rs b/src/database/maps.rs new file mode 100644 index 00000000..94e8ba99 --- /dev/null +++ b/src/database/maps.rs @@ -0,0 +1,99 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use conduit::Result; + +use crate::{Engine, Map}; + +pub type Maps = BTreeMap>; + +pub(crate) fn open(db: &Arc) -> Result { open_list(db, MAPS) } + +pub(crate) fn open_list(db: &Arc, maps: &[&str]) -> Result { + Ok(maps + .iter() + .map(|&name| (name.to_owned(), Map::open(db, name).expect("valid column opened"))) + .collect::()) +} + +pub const MAPS: &[&str] = &[ + "alias_roomid", + "alias_userid", + "aliasid_alias", + "backupid_algorithm", + "backupid_etag", + "backupkeyid_backup", + "bannedroomids", + "disabledroomids", + "eventid_outlierpdu", + "eventid_pduid", + "eventid_shorteventid", + "global", + "id_appserviceregistrations", + "keychangeid_userid", + "keyid_key", + "lazyloadedids", + "mediaid_file", + "mediaid_user", + "onetimekeyid_onetimekeys", + "pduid_pdu", + "presenceid_presence", + "publicroomids", + "readreceiptid_readreceipt", + "referencedevents", + "roomid_invitedcount", + "roomid_inviteviaservers", + "roomid_joinedcount", + "roomid_pduleaves", + "roomid_shortroomid", + "roomid_shortstatehash", + "roomserverids", + "roomsynctoken_shortstatehash", + "roomuserdataid_accountdata", + "roomuserid_invitecount", + "roomuserid_joined", + "roomuserid_lastprivatereadupdate", + "roomuserid_leftcount", + "roomuserid_privateread", + "roomuseroncejoinedids", + "roomusertype_roomuserdataid", + "senderkey_pusher", + "server_signingkeys", + "servercurrentevent_data", + "servername_educount", + "servernameevent_data", + "serverroomids", + "shorteventid_authchain", + "shorteventid_eventid", + "shorteventid_shortstatehash", + "shortstatehash_statediff", + "shortstatekey_statekey", + "softfailedeventids", + "statehash_shortstatehash", + "statekey_shortstatekey", + "threadid_userids", + "todeviceid_events", + "tofrom_relation", + "token_userdeviceid", + "tokenids", + "url_previews", + "userdeviceid_metadata", + "userdeviceid_token", + "userdevicesessionid_uiaainfo", + "userdevicetxnid_response", + "userfilterid_filter", + "userid_avatarurl", + "userid_blurhash", + "userid_devicelistversion", + "userid_displayname", + "userid_lastonetimekeyupdate", + "userid_masterkeyid", + "userid_password", + "userid_presenceid", + "userid_selfsigningkeyid", + "userid_usersigningkeyid", + "userroomid_highlightcount", + "userroomid_invitestate", + "userroomid_joined", + "userroomid_leftstate", + "userroomid_notificationcount", +]; diff --git a/src/database/mod.rs b/src/database/mod.rs index 48366e25..8b65e230 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,20 +1,20 @@ pub mod cork; -mod kvdatabase; -mod kvengine; -mod kvtree; - -#[cfg(feature = "rocksdb")] -pub(crate) mod rocksdb; - -#[cfg(feature = "rocksdb")] -pub(crate) mod watchers; +mod database; +mod engine; +mod map; +pub mod maps; +mod opts; +mod util; +mod watchers; extern crate conduit_core as conduit; -pub(crate) use conduit::{Config, Result}; +extern crate rust_rocksdb as rocksdb; + pub use cork::Cork; -pub use kvdatabase::KeyValueDatabase; -pub use kvengine::KeyValueDatabaseEngine; -pub use kvtree::KvTree; +pub use database::Database; +pub(crate) use engine::Engine; +pub use map::Map; +pub(crate) use util::{or_else, result}; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/database/rocksdb/opts.rs b/src/database/opts.rs similarity index 96% rename from src/database/rocksdb/opts.rs rename to src/database/opts.rs index 06c90a77..4f9dece5 100644 --- a/src/database/rocksdb/opts.rs +++ b/src/database/opts.rs @@ -1,14 +1,9 @@ -#![allow(dead_code)] use std::{cmp, collections::HashMap}; -use conduit::utils; - -use super::{ - rust_rocksdb::{ - BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, LogLevel, Options, - UniversalCompactOptions, UniversalCompactionStopStyle, - }, - Config, +use conduit::{utils, Config}; +use rocksdb::{ + BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, LogLevel, Options, + UniversalCompactOptions, UniversalCompactionStopStyle, }; /// Create database-wide options suitable for opening the database. This also @@ -150,7 +145,7 @@ pub(crate) fn cf_options(cfg: &Config, name: &str, mut opts: Options, cache: &mu ), #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] - "pduid_pdu" => set_table_with_new_cache( + "eventid_outlierpdu" => set_table_with_new_cache( &mut opts, cfg, cache, @@ -158,7 +153,7 @@ pub(crate) fn cf_options(cfg: &Config, name: &str, mut opts: Options, cache: &mu (cfg.pdu_cache_capacity as usize).saturating_mul(1536), ), - "eventid_outlierpdu" => set_table_with_shared_cache(&mut opts, cfg, cache, name, "pduid_pdu"), + "pduid_pdu" => set_table_with_shared_cache(&mut opts, cfg, cache, name, "eventid_outlierpdu"), &_ => {}, } @@ -210,6 +205,7 @@ fn set_compression_defaults(opts: &mut Options, config: &Config) { opts.set_compression_type(rocksdb_compression_algo); } +#[allow(dead_code)] fn set_for_random_small_uc(opts: &mut Options, config: &Config) { let uco = uc_options(config); set_for_random_small(opts, config); @@ -224,6 +220,7 @@ fn set_for_sequential_small_uc(opts: &mut Options, config: &Config) { opts.set_compaction_style(DBCompactionStyle::Universal); } +#[allow(dead_code)] fn set_for_random_small(opts: &mut Options, config: &Config) { set_for_random(opts, config); diff --git a/src/database/rocksdb/kvtree.rs b/src/database/rocksdb/kvtree.rs deleted file mode 100644 index 949db84e..00000000 --- a/src/database/rocksdb/kvtree.rs +++ /dev/null @@ -1,227 +0,0 @@ -use std::{future::Future, pin::Pin, sync::Arc}; - -use conduit::{utils, Result}; - -use super::{ - or_else, result, rust_rocksdb::WriteBatchWithTransaction, watchers::Watchers, Engine, KeyValueDatabaseEngine, - KvTree, -}; - -pub(crate) struct RocksDbEngineTree<'a> { - pub(crate) db: Arc, - pub(crate) name: &'a str, - pub(crate) watchers: Watchers, -} - -impl RocksDbEngineTree<'_> { - fn cf(&self) -> Arc> { self.db.rocks.cf_handle(self.name).unwrap() } -} - -impl KvTree for RocksDbEngineTree<'_> { - fn get(&self, key: &[u8]) -> Result>> { - let mut readoptions = rust_rocksdb::ReadOptions::default(); - readoptions.set_total_order_seek(true); - - result(self.db.rocks.get_cf_opt(&self.cf(), key, &readoptions)) - } - - fn multi_get(&self, keys: &[&[u8]]) -> Result>>> { - // Optimization can be `true` if key vector is pre-sorted **by the column - // comparator**. - const SORTED: bool = false; - - let mut readoptions = rust_rocksdb::ReadOptions::default(); - readoptions.set_total_order_seek(true); - - let mut ret: Vec>> = Vec::with_capacity(keys.len()); - for res in self - .db - .rocks - .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, &readoptions) - { - match res { - Ok(Some(res)) => ret.push(Some((*res).to_vec())), - Ok(None) => ret.push(None), - Err(e) => return or_else(e), - } - } - - Ok(ret) - } - - fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { - let writeoptions = rust_rocksdb::WriteOptions::default(); - - self.db - .rocks - .put_cf_opt(&self.cf(), key, value, &writeoptions) - .or_else(or_else)?; - - if !self.db.corked() { - self.db.flush()?; - } - - self.watchers.wake(key); - - Ok(()) - } - - fn insert_batch(&self, iter: &mut dyn Iterator, Vec)>) -> Result<()> { - let writeoptions = rust_rocksdb::WriteOptions::default(); - - let mut batch = WriteBatchWithTransaction::::default(); - - for (key, value) in iter { - batch.put_cf(&self.cf(), key, value); - } - - let res = self.db.rocks.write_opt(batch, &writeoptions); - - if !self.db.corked() { - self.db.flush()?; - } - - result(res) - } - - fn remove(&self, key: &[u8]) -> Result<()> { - let writeoptions = rust_rocksdb::WriteOptions::default(); - - let res = self.db.rocks.delete_cf_opt(&self.cf(), key, &writeoptions); - - if !self.db.corked() { - self.db.flush()?; - } - - result(res) - } - - fn remove_batch(&self, iter: &mut dyn Iterator>) -> Result<()> { - let writeoptions = rust_rocksdb::WriteOptions::default(); - - let mut batch = WriteBatchWithTransaction::::default(); - - for key in iter { - batch.delete_cf(&self.cf(), key); - } - - let res = self.db.rocks.write_opt(batch, &writeoptions); - - if !self.db.corked() { - self.db.flush()?; - } - - result(res) - } - - fn iter<'a>(&'a self) -> Box, Vec)> + 'a> { - let mut readoptions = rust_rocksdb::ReadOptions::default(); - readoptions.set_total_order_seek(true); - - Box::new( - self.db - .rocks - .iterator_cf_opt(&self.cf(), readoptions, rust_rocksdb::IteratorMode::Start) - .map(Result::unwrap) - .map(|(k, v)| (Vec::from(k), Vec::from(v))), - ) - } - - fn iter_from<'a>(&'a self, from: &[u8], backwards: bool) -> Box, Vec)> + 'a> { - let mut readoptions = rust_rocksdb::ReadOptions::default(); - readoptions.set_total_order_seek(true); - - Box::new( - self.db - .rocks - .iterator_cf_opt( - &self.cf(), - readoptions, - rust_rocksdb::IteratorMode::From( - from, - if backwards { - rust_rocksdb::Direction::Reverse - } else { - rust_rocksdb::Direction::Forward - }, - ), - ) - .map(Result::unwrap) - .map(|(k, v)| (Vec::from(k), Vec::from(v))), - ) - } - - fn increment(&self, key: &[u8]) -> Result> { - let mut readoptions = rust_rocksdb::ReadOptions::default(); - readoptions.set_total_order_seek(true); - let writeoptions = rust_rocksdb::WriteOptions::default(); - - let old = self - .db - .rocks - .get_cf_opt(&self.cf(), key, &readoptions) - .or_else(or_else)?; - let new = utils::increment(old.as_deref()); - self.db - .rocks - .put_cf_opt(&self.cf(), key, new, &writeoptions) - .or_else(or_else)?; - - if !self.db.corked() { - self.db.flush()?; - } - - Ok(new.to_vec()) - } - - fn increment_batch(&self, iter: &mut dyn Iterator>) -> Result<()> { - let mut readoptions = rust_rocksdb::ReadOptions::default(); - readoptions.set_total_order_seek(true); - let writeoptions = rust_rocksdb::WriteOptions::default(); - - let mut batch = WriteBatchWithTransaction::::default(); - - for key in iter { - let old = self - .db - .rocks - .get_cf_opt(&self.cf(), &key, &readoptions) - .or_else(or_else)?; - let new = utils::increment(old.as_deref()); - batch.put_cf(&self.cf(), key, new); - } - - self.db - .rocks - .write_opt(batch, &writeoptions) - .or_else(or_else)?; - - if !self.db.corked() { - self.db.flush()?; - } - - Ok(()) - } - - fn scan_prefix<'a>(&'a self, prefix: Vec) -> Box, Vec)> + 'a> { - let mut readoptions = rust_rocksdb::ReadOptions::default(); - readoptions.set_total_order_seek(true); - - Box::new( - self.db - .rocks - .iterator_cf_opt( - &self.cf(), - readoptions, - rust_rocksdb::IteratorMode::From(&prefix, rust_rocksdb::Direction::Forward), - ) - .map(Result::unwrap) - .map(|(k, v)| (Vec::from(k), Vec::from(v))) - .take_while(move |(k, _)| k.starts_with(&prefix)), - ) - } - - fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { - self.watchers.watch(prefix) - } -} diff --git a/src/database/util.rs b/src/database/util.rs new file mode 100644 index 00000000..513cedc8 --- /dev/null +++ b/src/database/util.rs @@ -0,0 +1,13 @@ +use conduit::Result; + +#[inline] +pub(crate) fn result(r: std::result::Result) -> Result { + r.map_or_else(or_else, and_then) +} + +#[inline(always)] +pub(crate) fn and_then(t: T) -> Result { Ok(t) } + +pub(crate) fn or_else(e: rocksdb::Error) -> Result { Err(map_err(e)) } + +pub(crate) fn map_err(e: rocksdb::Error) -> conduit::Error { conduit::Error::Database(e.into_string()) } From 875d9e8b07dd82b9fd9e6243c9529b7d210feff4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 28 Jun 2024 23:05:45 +0000 Subject: [PATCH 0278/2091] fix complex types Signed-off-by: Jason Volk --- src/service/rooms/pdu_metadata/data.rs | 5 ++++- src/service/rooms/timeline/data.rs | 11 +++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index e652c9fe..7be183a0 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -11,6 +11,9 @@ pub(super) struct Data { softfailedeventids: Arc, } +type PdusIterItem = Result<(PduCount, PduEvent)>; +type PdusIterator<'a> = Box + 'a>; + impl Data { pub(super) fn new(db: &Arc) -> Self { Self { @@ -29,7 +32,7 @@ impl Data { pub(super) fn relations_until<'a>( &'a self, user_id: &'a UserId, shortroomid: u64, target: u64, until: PduCount, - ) -> Result> + 'a>> { + ) -> Result> { let prefix = target.to_be_bytes().to_vec(); let mut current = prefix.clone(); diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 8264b6f3..41a58f67 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -16,6 +16,9 @@ pub struct Data { db: Arc, } +type PdusIterItem = Result<(PduCount, PduEvent)>; +type PdusIterator<'a> = Box + 'a>; + impl Data { pub(super) fn new(db: &Arc) -> Self { Self { @@ -203,9 +206,7 @@ impl Data { /// Returns an iterator over all events and their tokens in a room that /// happened before the event with id `until` in reverse-chronological /// order. - pub(super) fn pdus_until<'a>( - &'a self, user_id: &UserId, room_id: &RoomId, until: PduCount, - ) -> Result> + 'a>> { + pub(super) fn pdus_until(&self, user_id: &UserId, room_id: &RoomId, until: PduCount) -> Result> { let (prefix, current) = count_to_id(room_id, until, 1, true)?; let user_id = user_id.to_owned(); @@ -227,9 +228,7 @@ impl Data { )) } - pub(super) fn pdus_after<'a>( - &'a self, user_id: &UserId, room_id: &RoomId, from: PduCount, - ) -> Result> + 'a>> { + pub(super) fn pdus_after(&self, user_id: &UserId, room_id: &RoomId, from: PduCount) -> Result> { let (prefix, current) = count_to_id(room_id, from, 1, false)?; let user_id = user_id.to_owned(); From 57acc4f6551b13e6ef7836d5f3e1ce248e99fd31 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 28 Jun 2024 23:23:59 +0000 Subject: [PATCH 0279/2091] fix needless pass by value Signed-off-by: Jason Volk --- src/admin/media/commands.rs | 6 +++--- src/api/client/media.rs | 14 +++++++------- src/api/client/push.rs | 4 +--- src/api/client/read_marker.rs | 4 ++-- src/api/server/send.rs | 2 +- src/service/appservice/data.rs | 2 +- src/service/appservice/mod.rs | 2 +- src/service/media/data.rs | 8 ++++---- src/service/media/mod.rs | 20 ++++++++++---------- src/service/pusher/data.rs | 6 +++--- src/service/pusher/mod.rs | 2 +- src/service/rooms/read_receipt/data.rs | 4 ++-- src/service/rooms/read_receipt/mod.rs | 2 +- src/service/rooms/state_compressor/data.rs | 2 +- src/service/rooms/state_compressor/mod.rs | 4 ++-- 15 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index c1026ae2..d29d5f47 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -15,7 +15,7 @@ pub(super) async fn delete( if let Some(mxc) = mxc { debug!("Got MXC URL: {mxc}"); - services().media.delete(mxc.to_string()).await?; + services().media.delete(mxc.as_ref()).await?; return Ok(RoomMessageEventContent::text_plain( "Deleted the MXC from our database and on our filesystem.", @@ -123,7 +123,7 @@ pub(super) async fn delete( } for mxc_url in mxc_urls { - services().media.delete(mxc_url).await?; + services().media.delete(&mxc_url).await?; mxc_deletion_count = mxc_deletion_count.saturating_add(1); } @@ -154,7 +154,7 @@ pub(super) async fn delete_list(body: Vec<&str>) -> Result) -> R content_type, file, content_disposition, - }) = services().media.get(mxc.clone()).await? + }) = services().media.get(&mxc).await? { let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); @@ -264,7 +264,7 @@ pub(crate) async fn get_content_as_filename_route( content_type, file, content_disposition, - }) = services().media.get(mxc.clone()).await? + }) = services().media.get(&mxc).await? { let content_disposition = Some(make_content_disposition( &content_type, @@ -352,7 +352,7 @@ pub(crate) async fn get_content_thumbnail_route( }) = services() .media .get_thumbnail( - mxc.clone(), + &mxc, body.width .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, @@ -405,7 +405,7 @@ pub(crate) async fn get_content_thumbnail_route( .media .upload_thumbnail( None, - mxc, + &mxc, None, get_thumbnail_response.content_type.as_deref(), body.width.try_into().expect("all UInts are valid u32s"), @@ -494,7 +494,7 @@ async fn get_remote_content( .media .create( None, - mxc.to_owned(), + mxc, content_disposition.as_deref(), content_response.content_type.as_deref(), &content_response.file, @@ -520,7 +520,7 @@ async fn download_image(client: &reqwest::Client, url: &str) -> Result) -> R pub(crate) async fn set_pushers_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services() - .pusher - .set_pusher(sender_user, body.action.clone())?; + services().pusher.set_pusher(sender_user, &body.action)?; Ok(set_pusher::v3::Response::default()) } diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index 0f43eeef..cbb36b81 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -85,7 +85,7 @@ pub(crate) async fn set_read_marker_route( services().rooms.read_receipt.readreceipt_update( sender_user, &body.room_id, - ruma::events::receipt::ReceiptEvent { + &ruma::events::receipt::ReceiptEvent { content: ruma::events::receipt::ReceiptEventContent(receipt_content), room_id: body.room_id.clone(), }, @@ -145,7 +145,7 @@ pub(crate) async fn create_receipt_route( services().rooms.read_receipt.readreceipt_update( sender_user, &body.room_id, - ruma::events::receipt::ReceiptEvent { + &ruma::events::receipt::ReceiptEvent { content: ruma::events::receipt::ReceiptEventContent(receipt_content), room_id: body.room_id.clone(), }, diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 1168dd82..5fad790f 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -208,7 +208,7 @@ pub(crate) async fn send_transaction_message_route( services() .rooms .read_receipt - .readreceipt_update(&user_id, &room_id, event)?; + .readreceipt_update(&user_id, &room_id, &event)?; } } else { debug_warn!(%user_id, %room_id, %origin, "received read receipt EDU from server who does not have a single member from their server in the room"); diff --git a/src/service/appservice/data.rs b/src/service/appservice/data.rs index 59fabe96..c70a35e2 100644 --- a/src/service/appservice/data.rs +++ b/src/service/appservice/data.rs @@ -17,7 +17,7 @@ impl Data { } /// Registers an appservice and returns the ID to the caller - pub(super) fn register_appservice(&self, yaml: Registration) -> Result { + pub(super) fn register_appservice(&self, yaml: &Registration) -> Result { let id = yaml.id.as_str(); self.id_appserviceregistrations .insert(id.as_bytes(), serde_yaml::to_string(&yaml).unwrap().as_bytes())?; diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 83023151..17f951d4 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -149,7 +149,7 @@ impl Service { .await .insert(yaml.id.clone(), yaml.clone().try_into()?); - self.db.register_appservice(yaml) + self.db.register_appservice(&yaml) } /// Remove an appservice registration diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 7b8d0b5a..e52244f3 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -23,7 +23,7 @@ impl Data { } pub(super) fn create_file_metadata( - &self, sender_user: Option<&str>, mxc: String, width: u32, height: u32, content_disposition: Option<&str>, + &self, sender_user: Option<&str>, mxc: &str, width: u32, height: u32, content_disposition: Option<&str>, content_type: Option<&str>, ) -> Result> { let mut key = mxc.as_bytes().to_vec(); @@ -56,7 +56,7 @@ impl Data { Ok(key) } - pub(super) fn delete_file_mxc(&self, mxc: String) -> Result<()> { + pub(super) fn delete_file_mxc(&self, mxc: &str) -> Result<()> { debug!("MXC URI: {:?}", mxc); let mut prefix = mxc.as_bytes().to_vec(); @@ -82,7 +82,7 @@ impl Data { } /// Searches for all files with the given MXC - pub(super) fn search_mxc_metadata_prefix(&self, mxc: String) -> Result>> { + pub(super) fn search_mxc_metadata_prefix(&self, mxc: &str) -> Result>> { debug!("MXC URI: {:?}", mxc); let mut prefix = mxc.as_bytes().to_vec(); @@ -106,7 +106,7 @@ impl Data { } pub(super) fn search_file_metadata( - &self, mxc: String, width: u32, height: u32, + &self, mxc: &str, width: u32, height: u32, ) -> Result<(Option, Option, Vec)> { let mut prefix = mxc.as_bytes().to_vec(); prefix.push(0xFF); diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index a6e2a2a9..f631acb3 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -59,7 +59,7 @@ impl Service { /// Uploads a file. pub async fn create( - &self, sender_user: Option, mxc: String, content_disposition: Option<&str>, + &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, content_type: Option<&str>, file: &[u8], ) -> Result<()> { // Width, Height = 0 if it's not a thumbnail @@ -79,13 +79,13 @@ impl Service { } /// Deletes a file in the database and from the media directory via an MXC - pub async fn delete(&self, mxc: String) -> Result<()> { - if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc.clone()) { + pub async fn delete(&self, mxc: &str) -> Result<()> { + if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc) { for key in keys { self.remove_media_file(&key).await?; debug!("Deleting MXC {mxc} from database"); - self.db.delete_file_mxc(mxc.clone())?; + self.db.delete_file_mxc(mxc)?; } Ok(()) @@ -100,7 +100,7 @@ impl Service { /// Uploads or replaces a file thumbnail. #[allow(clippy::too_many_arguments)] pub async fn upload_thumbnail( - &self, sender_user: Option, mxc: String, content_disposition: Option<&str>, + &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, content_type: Option<&str>, width: u32, height: u32, file: &[u8], ) -> Result<()> { let key = if let Some(user) = sender_user { @@ -119,7 +119,7 @@ impl Service { } /// Downloads a file. - pub async fn get(&self, mxc: String) -> Result> { + pub async fn get(&self, mxc: &str) -> Result> { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { let mut file = Vec::new(); let path = self.get_media_file(&key); @@ -232,7 +232,7 @@ impl Service { for mxc in remote_mxcs { debug!("Deleting MXC {mxc} from database and filesystem"); - self.delete(mxc).await?; + self.delete(&mxc).await?; deletion_count = deletion_count.saturating_add(1); } @@ -265,12 +265,12 @@ impl Service { /// /// For width,height <= 96 the server uses another thumbnailing algorithm /// which crops the image afterwards. - pub async fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result> { + pub async fn get_thumbnail(&self, mxc: &str, width: u32, height: u32) -> Result> { let (width, height, crop) = self .thumbnail_properties(width, height) .unwrap_or((0, 0, false)); // 0, 0 because that's the original file - if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), width, height) { + if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, width, height) { // Using saved thumbnail let mut file = Vec::new(); let path = self.get_media_file(&key); @@ -281,7 +281,7 @@ impl Service { content_type, file: file.clone(), })) - } else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc.clone(), 0, 0) { + } else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { // Generate a thumbnail let mut file = Vec::new(); let path = self.get_media_file(&key); diff --git a/src/service/pusher/data.rs b/src/service/pusher/data.rs index 0892073f..a8a9366e 100644 --- a/src/service/pusher/data.rs +++ b/src/service/pusher/data.rs @@ -19,14 +19,14 @@ impl Data { } } - pub(super) fn set_pusher(&self, sender: &UserId, pusher: set_pusher::v3::PusherAction) -> Result<()> { - match &pusher { + pub(super) fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result<()> { + match pusher { set_pusher::v3::PusherAction::Post(data) => { let mut key = sender.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(data.pusher.ids.pushkey.as_bytes()); self.senderkey_pusher - .insert(&key, &serde_json::to_vec(&pusher).expect("Pusher is valid JSON value"))?; + .insert(&key, &serde_json::to_vec(pusher).expect("Pusher is valid JSON value"))?; Ok(()) }, set_pusher::v3::PusherAction::Delete(ids) => { diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index ff167918..25faecd5 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -38,7 +38,7 @@ impl Service { }) } - pub fn set_pusher(&self, sender: &UserId, pusher: set_pusher::v3::PusherAction) -> Result<()> { + pub fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result<()> { self.db.set_pusher(sender, pusher) } diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index cf9d0ee3..a33c64ac 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -27,7 +27,7 @@ impl Data { } } - pub(super) fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: ReceiptEvent) -> Result<()> { + pub(super) fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) -> Result<()> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -56,7 +56,7 @@ impl Data { self.readreceiptid_readreceipt.insert( &room_latest_id, - &serde_json::to_vec(&event).expect("EduEvent::to_string always works"), + &serde_json::to_vec(event).expect("EduEvent::to_string always works"), )?; Ok(()) diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 56401fe4..5da374ef 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -22,7 +22,7 @@ impl Service { } /// Replaces the previous read receipt. - pub fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: ReceiptEvent) -> Result<()> { + pub fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) -> Result<()> { self.db.readreceipt_update(user_id, room_id, event)?; services().sending.flush_room(room_id)?; diff --git a/src/service/rooms/state_compressor/data.rs b/src/service/rooms/state_compressor/data.rs index 2bf93f38..454a3b57 100644 --- a/src/service/rooms/state_compressor/data.rs +++ b/src/service/rooms/state_compressor/data.rs @@ -60,7 +60,7 @@ impl Data { }) } - pub(super) fn save_statediff(&self, shortstatehash: u64, diff: StateDiff) -> Result<()> { + pub(super) fn save_statediff(&self, shortstatehash: u64, diff: &StateDiff) -> Result<()> { let mut value = diff.parent.unwrap_or(0).to_be_bytes().to_vec(); for new in diff.added.iter() { value.extend_from_slice(&new[..]); diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 8dde2890..ee0e4cc9 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -200,7 +200,7 @@ impl Service { // There is no parent layer, create a new state self.db.save_statediff( shortstatehash, - StateDiff { + &StateDiff { parent: None, added: statediffnew, removed: statediffremoved, @@ -251,7 +251,7 @@ impl Service { // Diff small enough, we add diff as layer on top of parent self.db.save_statediff( shortstatehash, - StateDiff { + &StateDiff { parent: Some(parent.0), added: statediffnew, removed: statediffremoved, From cb48e2578355ce98d3e06bacfa77143e3ad122a2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 28 Jun 2024 22:51:39 +0000 Subject: [PATCH 0280/2091] refactor dyn KvTree out of services Signed-off-by: Jason Volk --- src/service/account_data/data.rs | 15 +- src/service/account_data/mod.rs | 4 +- src/service/admin/mod.rs | 9 +- src/service/appservice/data.rs | 11 +- src/service/appservice/mod.rs | 11 +- src/service/globals/client.rs | 2 +- src/service/globals/data.rs | 63 +++--- src/service/globals/migrations.rs | 179 ++++++++++-------- src/service/globals/mod.rs | 11 +- src/service/globals/resolver.rs | 4 +- src/service/key_backups/data.rs | 20 +- src/service/key_backups/mod.rs | 11 +- src/service/media/data.rs | 23 ++- src/service/media/mod.rs | 6 +- src/service/media/tests.rs | 1 + src/service/mod.rs | 14 +- src/service/pdu.rs | 4 +- src/service/presence/data.rs | 23 +-- src/service/presence/mod.rs | 14 +- src/service/pusher/data.rs | 13 +- src/service/pusher/mod.rs | 13 +- src/service/rooms/alias/data.rs | 27 +-- src/service/rooms/alias/mod.rs | 11 +- src/service/rooms/auth_chain/data.rs | 13 +- src/service/rooms/auth_chain/mod.rs | 11 +- src/service/rooms/directory/data.rs | 11 +- src/service/rooms/directory/mod.rs | 7 +- src/service/rooms/event_handler/mod.rs | 9 +- .../rooms/event_handler/parse_incoming_pdu.rs | 3 +- src/service/rooms/lazy_loading/data.rs | 13 +- src/service/rooms/lazy_loading/mod.rs | 10 +- src/service/rooms/metadata/data.rs | 26 +-- src/service/rooms/metadata/mod.rs | 11 +- src/service/rooms/outlier/data.rs | 13 +- src/service/rooms/outlier/mod.rs | 11 +- src/service/rooms/pdu_metadata/data.rs | 19 +- src/service/rooms/pdu_metadata/mod.rs | 9 +- src/service/rooms/read_receipt/data.rs | 19 +- src/service/rooms/read_receipt/mod.rs | 9 +- src/service/rooms/search/data.rs | 13 +- src/service/rooms/search/mod.rs | 11 +- src/service/rooms/short/data.rs | 32 ++-- src/service/rooms/short/mod.rs | 10 +- src/service/rooms/spaces/mod.rs | 8 +- src/service/rooms/state/data.rs | 23 ++- src/service/rooms/state/mod.rs | 17 +- src/service/rooms/state_accessor/data.rs | 17 +- src/service/rooms/state_accessor/mod.rs | 18 +- src/service/rooms/state_cache/data.rs | 69 +++---- src/service/rooms/state_cache/mod.rs | 15 +- src/service/rooms/state_compressor/data.rs | 12 +- src/service/rooms/state_compressor/mod.rs | 16 +- src/service/rooms/threads/data.rs | 13 +- src/service/rooms/threads/mod.rs | 11 +- src/service/rooms/timeline/data.rs | 33 ++-- src/service/rooms/timeline/mod.rs | 28 +-- src/service/rooms/typing/mod.rs | 13 +- src/service/rooms/user/data.rs | 29 +-- src/service/rooms/user/mod.rs | 11 +- src/service/sending/data.rs | 20 +- src/service/sending/mod.rs | 8 +- src/service/sending/sender.rs | 2 +- src/service/services.rs | 6 +- src/service/transaction_ids/data.rs | 8 +- src/service/transaction_ids/mod.rs | 9 +- src/service/uiaa/data.rs | 10 +- src/service/uiaa/mod.rs | 8 +- src/service/users/data.rs | 77 ++++---- src/service/users/mod.rs | 11 +- 69 files changed, 594 insertions(+), 647 deletions(-) diff --git a/src/service/account_data/data.rs b/src/service/account_data/data.rs index 7ab39918..7b3a3dee 100644 --- a/src/service/account_data/data.rs +++ b/src/service/account_data/data.rs @@ -1,25 +1,26 @@ use std::{collections::HashMap, sync::Arc}; +use conduit::{utils, warn, Error, Result}; +use database::{Database, Map}; use ruma::{ api::client::error::ErrorKind, events::{AnyEphemeralRoomEvent, RoomAccountDataEventType}, serde::Raw, RoomId, UserId, }; -use tracing::warn; -use crate::{services, utils, Error, KeyValueDatabase, KvTree, Result}; +use crate::services; pub(super) struct Data { - roomuserdataid_accountdata: Arc, - roomusertype_roomuserdataid: Arc, + roomuserdataid_accountdata: Arc, + roomusertype_roomuserdataid: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - roomuserdataid_accountdata: db.roomuserdataid_accountdata.clone(), - roomusertype_roomuserdataid: db.roomusertype_roomuserdataid.clone(), + roomuserdataid_accountdata: db["roomuserdataid_accountdata"].clone(), + roomusertype_roomuserdataid: db["roomusertype_roomuserdataid"].clone(), } } diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index fc330ead..62560500 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -4,7 +4,7 @@ use std::{collections::HashMap, sync::Arc}; use conduit::{Result, Server}; use data::Data; -use database::KeyValueDatabase; +use database::Database; use ruma::{ events::{AnyEphemeralRoomEvent, RoomAccountDataEventType}, serde::Raw, @@ -16,7 +16,7 @@ pub struct Service { } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 5a1c161c..b5cb8af4 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -1,14 +1,12 @@ -use conduit::Server; -use database::KeyValueDatabase; - pub mod console; mod create; mod grant; use std::{future::Future, pin::Pin, sync::Arc}; -use conduit::{utils::mutex_map, Error, Result}; +use conduit::{error, utils::mutex_map, Error, Result, Server}; pub use create::create_admin_room; +use database::Database; pub use grant::make_user_admin; use loole::{Receiver, Sender}; use ruma::{ @@ -20,7 +18,6 @@ use ruma::{ }; use serde_json::value::to_raw_value; use tokio::{sync::Mutex, task::JoinHandle}; -use tracing::error; use crate::{pdu::PduBuilder, services, user_is_local, PduEvent}; @@ -47,7 +44,7 @@ pub struct Command { } impl Service { - pub fn build(_server: &Arc, _db: &Arc) -> Result> { + pub fn build(_server: &Arc, _db: &Arc) -> Result> { let (sender, receiver) = loole::bounded(COMMAND_QUEUE_LIMIT); Ok(Arc::new(Self { sender, diff --git a/src/service/appservice/data.rs b/src/service/appservice/data.rs index c70a35e2..52c89454 100644 --- a/src/service/appservice/data.rs +++ b/src/service/appservice/data.rs @@ -1,18 +1,17 @@ use std::sync::Arc; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::api::appservice::Registration; -use crate::{utils, Error, KeyValueDatabase, Result}; - pub struct Data { - id_appserviceregistrations: Arc, + id_appserviceregistrations: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - id_appserviceregistrations: db.id_appserviceregistrations.clone(), + id_appserviceregistrations: db["id_appserviceregistrations"].clone(), } } diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 17f951d4..05bc7675 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -2,7 +2,9 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -pub use data::Data; +use conduit::{Result, Server}; +use data::Data; +use database::Database; use futures_util::Future; use regex::RegexSet; use ruma::{ @@ -11,7 +13,7 @@ use ruma::{ }; use tokio::sync::RwLock; -use crate::{services, Result}; +use crate::services; /// Compiled regular expressions for a namespace #[derive(Clone, Debug)] @@ -117,11 +119,8 @@ pub struct Service { registration_info: RwLock>, } -use conduit::Server; -use database::KeyValueDatabase; - impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { let mut registration_info = BTreeMap::new(); let db = Data::new(db); // Inserting registrations into cache diff --git a/src/service/globals/client.rs b/src/service/globals/client.rs index 5e1b129d..7c63618e 100644 --- a/src/service/globals/client.rs +++ b/src/service/globals/client.rs @@ -2,7 +2,7 @@ use std::{sync::Arc, time::Duration}; use reqwest::redirect; -use crate::{service::globals::resolver, Config, Result}; +use crate::{globals::resolver, Config, Result}; pub struct Client { pub default: reqwest::Client, diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 1645c8d6..bf3eb68b 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -3,7 +3,8 @@ use std::{ sync::Arc, }; -use database::{Cork, KeyValueDatabase, KvTree}; +use conduit::{trace, utils, Error, Result}; +use database::{Cork, Database, Map}; use futures_util::{stream::FuturesUnordered, StreamExt}; use lru_cache::LruCache; use ruma::{ @@ -11,46 +12,45 @@ use ruma::{ signatures::Ed25519KeyPair, DeviceId, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, ServerName, UserId, }; -use tracing::trace; -use crate::{services, utils, Error, Result}; +use crate::services; const COUNTER: &[u8] = b"c"; const LAST_CHECK_FOR_UPDATES_COUNT: &[u8] = b"u"; pub struct Data { - global: Arc, - todeviceid_events: Arc, - userroomid_joined: Arc, - userroomid_invitestate: Arc, - userroomid_leftstate: Arc, - userroomid_notificationcount: Arc, - userroomid_highlightcount: Arc, - pduid_pdu: Arc, - keychangeid_userid: Arc, - roomusertype_roomuserdataid: Arc, - server_signingkeys: Arc, - readreceiptid_readreceipt: Arc, - userid_lastonetimekeyupdate: Arc, - pub(super) db: Arc, + global: Arc, + todeviceid_events: Arc, + userroomid_joined: Arc, + userroomid_invitestate: Arc, + userroomid_leftstate: Arc, + userroomid_notificationcount: Arc, + userroomid_highlightcount: Arc, + pduid_pdu: Arc, + keychangeid_userid: Arc, + roomusertype_roomuserdataid: Arc, + server_signingkeys: Arc, + readreceiptid_readreceipt: Arc, + userid_lastonetimekeyupdate: Arc, + pub(super) db: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - global: db.global.clone(), - todeviceid_events: db.todeviceid_events.clone(), - userroomid_joined: db.userroomid_joined.clone(), - userroomid_invitestate: db.userroomid_invitestate.clone(), - userroomid_leftstate: db.userroomid_leftstate.clone(), - userroomid_notificationcount: db.userroomid_notificationcount.clone(), - userroomid_highlightcount: db.userroomid_highlightcount.clone(), - pduid_pdu: db.pduid_pdu.clone(), - keychangeid_userid: db.keychangeid_userid.clone(), - roomusertype_roomuserdataid: db.roomusertype_roomuserdataid.clone(), - server_signingkeys: db.server_signingkeys.clone(), - readreceiptid_readreceipt: db.readreceiptid_readreceipt.clone(), - userid_lastonetimekeyupdate: db.userid_lastonetimekeyupdate.clone(), + global: db["global"].clone(), + todeviceid_events: db["todeviceid_events"].clone(), + userroomid_joined: db["userroomid_joined"].clone(), + userroomid_invitestate: db["userroomid_invitestate"].clone(), + userroomid_leftstate: db["userroomid_leftstate"].clone(), + userroomid_notificationcount: db["userroomid_notificationcount"].clone(), + userroomid_highlightcount: db["userroomid_highlightcount"].clone(), + pduid_pdu: db["pduid_pdu"].clone(), + keychangeid_userid: db["keychangeid_userid"].clone(), + roomusertype_roomuserdataid: db["roomusertype_roomuserdataid"].clone(), + server_signingkeys: db["server_signingkeys"].clone(), + readreceiptid_readreceipt: db["readreceiptid_readreceipt"].clone(), + userid_lastonetimekeyupdate: db["userid_lastonetimekeyupdate"].clone(), db: db.clone(), } } @@ -298,7 +298,6 @@ lasttimelinecount_cache: {lasttimelinecount_cache} / {max_lasttimelinecount_cach /// for the server. pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { let signingkeys = self - .db .server_signingkeys .get(origin.as_bytes())? .and_then(|bytes| serde_json::from_slice(&bytes).ok()) diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index a281ce2f..0f52d5af 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -10,7 +10,7 @@ use std::{ }; use conduit::{debug, debug_info, debug_warn, error, info, utils, warn, Config, Error, Result}; -use database::KeyValueDatabase; +use database::Database; use itertools::Itertools; use ruma::{ events::{push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType}, @@ -28,7 +28,7 @@ use crate::services; /// equal or lesser version. These are expected to be backward-compatible. const DATABASE_VERSION: u64 = 13; -pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result<()> { +pub(crate) async fn migrations(db: &Arc, config: &Config) -> Result<()> { // Matrix resource ownership is based on the server name; changing it // requires recreating the database from scratch. if services().users.count()? > 0 { @@ -49,13 +49,11 @@ pub(crate) async fn migrations(db: &KeyValueDatabase, config: &Config) -> Result } } -async fn fresh(db: &KeyValueDatabase, config: &Config) -> Result<()> { +async fn fresh(db: &Arc, config: &Config) -> Result<()> { services().globals.bump_database_version(DATABASE_VERSION)?; - db.global - .insert(b"fix_bad_double_separator_in_state_cache", &[])?; - db.global - .insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; + db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[])?; + db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; // Create the admin room and server user on first run crate::admin::create_admin_room().await?; @@ -69,7 +67,7 @@ async fn fresh(db: &KeyValueDatabase, config: &Config) -> Result<()> { } /// Apply any migrations -async fn migrate(db: &KeyValueDatabase, config: &Config) -> Result<()> { +async fn migrate(db: &Arc, config: &Config) -> Result<()> { if services().globals.database_version()? < 1 { db_lt_1(db, config).await?; } @@ -124,22 +122,20 @@ async fn migrate(db: &KeyValueDatabase, config: &Config) -> Result<()> { db_lt_13(db, config).await?; } - if db.global.get(b"feat_sha256_media")?.is_none() { + if db["global"].get(b"feat_sha256_media")?.is_none() { migrate_sha256_media(db, config).await?; } else if config.media_startup_check { checkup_sha256_media(db, config).await?; } - if db - .global + if db["global"] .get(b"fix_bad_double_separator_in_state_cache")? .is_none() { fix_bad_double_separator_in_state_cache(db, config).await?; } - if db - .global + if db["global"] .get(b"retroactively_fix_bad_data_from_roomuserid_joined")? .is_none() { @@ -212,8 +208,10 @@ async fn migrate(db: &KeyValueDatabase, config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_1(db: &KeyValueDatabase, _config: &Config) -> Result<()> { - for (roomserverid, _) in db.roomserverids.iter() { +async fn db_lt_1(db: &Arc, _config: &Config) -> Result<()> { + let roomserverids = &db["roomserverids"]; + let serverroomids = &db["serverroomids"]; + for (roomserverid, _) in roomserverids.iter() { let mut parts = roomserverid.split(|&b| b == 0xFF); let room_id = parts.next().expect("split always returns one element"); let Some(servername) = parts.next() else { @@ -224,7 +222,7 @@ async fn db_lt_1(db: &KeyValueDatabase, _config: &Config) -> Result<()> { serverroomid.push(0xFF); serverroomid.extend_from_slice(room_id); - db.serverroomids.insert(&serverroomid, &[])?; + serverroomids.insert(&serverroomid, &[])?; } services().globals.bump_database_version(1)?; @@ -232,14 +230,15 @@ async fn db_lt_1(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_2(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn db_lt_2(db: &Arc, _config: &Config) -> Result<()> { // We accidentally inserted hashed versions of "" into the db instead of just "" - for (userid, password) in db.userid_password.iter() { + let userid_password = &db["roomserverids"]; + for (userid, password) in userid_password.iter() { let empty_pass = utils::hash::password("").expect("our own password to be properly hashed"); let password = std::str::from_utf8(&password).expect("password is valid utf-8"); let empty_hashed_password = utils::hash::verify_password(password, &empty_pass).is_ok(); if empty_hashed_password { - db.userid_password.insert(&userid, b"")?; + userid_password.insert(&userid, b"")?; } } @@ -248,9 +247,10 @@ async fn db_lt_2(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_3(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn db_lt_3(db: &Arc, _config: &Config) -> Result<()> { // Move media to filesystem - for (key, content) in db.mediaid_file.iter() { + let mediaid_file = &db["mediaid_file"]; + for (key, content) in mediaid_file.iter() { if content.is_empty() { continue; } @@ -259,7 +259,7 @@ async fn db_lt_3(db: &KeyValueDatabase, _config: &Config) -> Result<()> { let path = services().media.get_media_file(&key); let mut file = fs::File::create(path)?; file.write_all(&content)?; - db.mediaid_file.insert(&key, &[])?; + mediaid_file.insert(&key, &[])?; } services().globals.bump_database_version(3)?; @@ -267,7 +267,7 @@ async fn db_lt_3(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_4(_db: &KeyValueDatabase, config: &Config) -> Result<()> { +async fn db_lt_4(_db: &Arc, config: &Config) -> Result<()> { // Add federated users to services() as deactivated for our_user in services().users.iter() { let our_user = our_user?; @@ -290,9 +290,11 @@ async fn db_lt_4(_db: &KeyValueDatabase, config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_5(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn db_lt_5(db: &Arc, _config: &Config) -> Result<()> { // Upgrade user data store - for (roomuserdataid, _) in db.roomuserdataid_accountdata.iter() { + let roomuserdataid_accountdata = &db["roomuserdataid_accountdata"]; + let roomusertype_roomuserdataid = &db["roomusertype_roomuserdataid"]; + for (roomuserdataid, _) in roomuserdataid_accountdata.iter() { let mut parts = roomuserdataid.split(|&b| b == 0xFF); let room_id = parts.next().unwrap(); let user_id = parts.next().unwrap(); @@ -304,8 +306,7 @@ async fn db_lt_5(db: &KeyValueDatabase, _config: &Config) -> Result<()> { key.push(0xFF); key.extend_from_slice(event_type); - db.roomusertype_roomuserdataid - .insert(&key, &roomuserdataid)?; + roomusertype_roomuserdataid.insert(&key, &roomuserdataid)?; } services().globals.bump_database_version(5)?; @@ -313,9 +314,10 @@ async fn db_lt_5(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_6(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn db_lt_6(db: &Arc, _config: &Config) -> Result<()> { // Set room member count - for (roomid, _) in db.roomid_shortstatehash.iter() { + let roomid_shortstatehash = &db["roomid_shortstatehash"]; + for (roomid, _) in roomid_shortstatehash.iter() { let string = utils::string_from_bytes(&roomid).unwrap(); let room_id = <&RoomId>::try_from(string.as_str()).unwrap(); services().rooms.state_cache.update_joined_count(room_id)?; @@ -326,7 +328,7 @@ async fn db_lt_6(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_7(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn db_lt_7(db: &Arc, _config: &Config) -> Result<()> { // Upgrade state store let mut last_roomstates: HashMap = HashMap::new(); let mut current_sstatehash: Option = None; @@ -399,7 +401,9 @@ async fn db_lt_7(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Ok::<_, Error>(()) }; - for (k, seventid) in db.db.open_tree("stateid_shorteventid")?.iter() { + let stateid_shorteventid = &db["stateid_shorteventid"]; + let shorteventid_eventid = &db["shorteventid_eventid"]; + for (k, seventid) in stateid_shorteventid.iter() { let sstatehash = utils::u64_from_bytes(&k[0..size_of::()]).expect("number of bytes is correct"); let sstatekey = k[size_of::()..].to_vec(); if Some(sstatehash) != current_sstatehash { @@ -415,7 +419,7 @@ async fn db_lt_7(db: &KeyValueDatabase, _config: &Config) -> Result<()> { current_state = HashSet::new(); current_sstatehash = Some(sstatehash); - let event_id = db.shorteventid_eventid.get(&seventid).unwrap().unwrap(); + let event_id = shorteventid_eventid.get(&seventid).unwrap().unwrap(); let string = utils::string_from_bytes(&event_id).unwrap(); let event_id = <&EventId>::try_from(string.as_str()).unwrap(); let pdu = services() @@ -449,15 +453,20 @@ async fn db_lt_7(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_8(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn db_lt_8(db: &Arc, _config: &Config) -> Result<()> { + let roomid_shortstatehash = &db["roomid_shortstatehash"]; + let roomid_shortroomid = &db["roomid_shortroomid"]; + let pduid_pdu = &db["pduid_pdu"]; + let eventid_pduid = &db["eventid_pduid"]; + // Generate short room ids for all rooms - for (room_id, _) in db.roomid_shortstatehash.iter() { + for (room_id, _) in roomid_shortstatehash.iter() { let shortroomid = services().globals.next_count()?.to_be_bytes(); - db.roomid_shortroomid.insert(&room_id, &shortroomid)?; + roomid_shortroomid.insert(&room_id, &shortroomid)?; info!("Migration: 8"); } // Update pduids db layout - let mut batch = db.pduid_pdu.iter().filter_map(|(key, v)| { + let mut batch = pduid_pdu.iter().filter_map(|(key, v)| { if !key.starts_with(b"!") { return None; } @@ -465,8 +474,7 @@ async fn db_lt_8(db: &KeyValueDatabase, _config: &Config) -> Result<()> { let room_id = parts.next().unwrap(); let count = parts.next().unwrap(); - let short_room_id = db - .roomid_shortroomid + let short_room_id = roomid_shortroomid .get(room_id) .unwrap() .expect("shortroomid should exist"); @@ -477,9 +485,9 @@ async fn db_lt_8(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Some((new_key, v)) }); - db.pduid_pdu.insert_batch(&mut batch)?; + pduid_pdu.insert_batch(&mut batch)?; - let mut batch2 = db.eventid_pduid.iter().filter_map(|(k, value)| { + let mut batch2 = eventid_pduid.iter().filter_map(|(k, value)| { if !value.starts_with(b"!") { return None; } @@ -487,8 +495,7 @@ async fn db_lt_8(db: &KeyValueDatabase, _config: &Config) -> Result<()> { let room_id = parts.next().unwrap(); let count = parts.next().unwrap(); - let short_room_id = db - .roomid_shortroomid + let short_room_id = roomid_shortroomid .get(room_id) .unwrap() .expect("shortroomid should exist"); @@ -499,17 +506,19 @@ async fn db_lt_8(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Some((k, new_value)) }); - db.eventid_pduid.insert_batch(&mut batch2)?; + eventid_pduid.insert_batch(&mut batch2)?; services().globals.bump_database_version(8)?; info!("Migration: 7 -> 8 finished"); Ok(()) } -async fn db_lt_9(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn db_lt_9(db: &Arc, _config: &Config) -> Result<()> { + let tokenids = &db["tokenids"]; + let roomid_shortroomid = &db["roomid_shortroomid"]; + // Update tokenids db layout - let mut iter = db - .tokenids + let mut iter = tokenids .iter() .filter_map(|(key, _)| { if !key.starts_with(b"!") { @@ -521,8 +530,7 @@ async fn db_lt_9(db: &KeyValueDatabase, _config: &Config) -> Result<()> { let _pdu_id_room = parts.next().unwrap(); let pdu_id_count = parts.next().unwrap(); - let short_room_id = db - .roomid_shortroomid + let short_room_id = roomid_shortroomid .get(room_id) .unwrap() .expect("shortroomid should exist"); @@ -535,14 +543,13 @@ async fn db_lt_9(db: &KeyValueDatabase, _config: &Config) -> Result<()> { .peekable(); while iter.peek().is_some() { - db.tokenids.insert_batch(&mut iter.by_ref().take(1000))?; + tokenids.insert_batch(&mut iter.by_ref().take(1000))?; debug!("Inserted smaller batch"); } info!("Deleting starts"); - let batch2: Vec<_> = db - .tokenids + let batch2: Vec<_> = tokenids .iter() .filter_map(|(key, _)| { if key.starts_with(b"!") { @@ -554,7 +561,7 @@ async fn db_lt_9(db: &KeyValueDatabase, _config: &Config) -> Result<()> { .collect(); for key in batch2 { - db.tokenids.remove(&key)?; + tokenids.remove(&key)?; } services().globals.bump_database_version(9)?; @@ -562,11 +569,13 @@ async fn db_lt_9(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_10(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn db_lt_10(db: &Arc, _config: &Config) -> Result<()> { + let statekey_shortstatekey = &db["statekey_shortstatekey"]; + let shortstatekey_statekey = &db["shortstatekey_statekey"]; + // Add other direction for shortstatekeys - for (statekey, shortstatekey) in db.statekey_shortstatekey.iter() { - db.shortstatekey_statekey - .insert(&shortstatekey, &statekey)?; + for (statekey, shortstatekey) in statekey_shortstatekey.iter() { + shortstatekey_statekey.insert(&shortstatekey, &statekey)?; } // Force E2EE device list updates so we can send them over federation @@ -579,17 +588,16 @@ async fn db_lt_10(db: &KeyValueDatabase, _config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_11(db: &KeyValueDatabase, _config: &Config) -> Result<()> { - db.db - .open_tree("userdevicesessionid_uiaarequest")? - .clear()?; +async fn db_lt_11(db: &Arc, _config: &Config) -> Result<()> { + let _userdevicesessionid_uiaarequest = &db["userdevicesessionid_uiaarequest"]; + //userdevicesessionid_uiaarequest.clear()?; services().globals.bump_database_version(11)?; info!("Migration: 10 -> 11 finished"); Ok(()) } -async fn db_lt_12(_db: &KeyValueDatabase, config: &Config) -> Result<()> { +async fn db_lt_12(_db: &Arc, config: &Config) -> Result<()> { for username in services().users.list_local_users()? { let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { Ok(u) => u, @@ -657,7 +665,7 @@ async fn db_lt_12(_db: &KeyValueDatabase, config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_13(_db: &KeyValueDatabase, config: &Config) -> Result<()> { +async fn db_lt_13(_db: &Arc, config: &Config) -> Result<()> { for username in services().users.list_local_users()? { let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { Ok(u) => u, @@ -697,12 +705,13 @@ async fn db_lt_13(_db: &KeyValueDatabase, config: &Config) -> Result<()> { /// Migrates a media directory from legacy base64 file names to sha2 file names. /// All errors are fatal. Upon success the database is keyed to not perform this /// again. -async fn migrate_sha256_media(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn migrate_sha256_media(db: &Arc, _config: &Config) -> Result<()> { warn!("Migrating legacy base64 file names to sha256 file names"); + let mediaid_file = &db["mediaid_file"]; // Move old media files to new names let mut changes = Vec::<(PathBuf, PathBuf)>::new(); - for (key, _) in db.mediaid_file.iter() { + for (key, _) in mediaid_file.iter() { let old = services().media.get_media_file_b64(&key); let new = services().media.get_media_file_sha256(&key); debug!(?key, ?old, ?new, num = changes.len(), "change"); @@ -722,7 +731,7 @@ async fn migrate_sha256_media(db: &KeyValueDatabase, _config: &Config) -> Result services().globals.bump_database_version(13)?; } - db.global.insert(b"feat_sha256_media", &[])?; + db["global"].insert(b"feat_sha256_media", &[])?; info!("Finished applying sha256_media"); Ok(()) } @@ -731,10 +740,13 @@ async fn migrate_sha256_media(db: &KeyValueDatabase, _config: &Config) -> Result /// - Going back and forth to non-sha256 legacy binaries (e.g. upstream). /// - Deletion of artifacts in the media directory which will then fall out of /// sync with the database. -async fn checkup_sha256_media(db: &KeyValueDatabase, config: &Config) -> Result<()> { +async fn checkup_sha256_media(db: &Arc, config: &Config) -> Result<()> { use crate::media::encode_key; debug!("Checking integrity of media directory"); + let mediaid_file = &db["mediaid_file"]; + let mediaid_user = &db["mediaid_user"]; + let dbs = (mediaid_file, mediaid_user); let media = &services().media; let timer = Instant::now(); @@ -746,7 +758,7 @@ async fn checkup_sha256_media(db: &KeyValueDatabase, config: &Config) -> Result< for key in media.db.get_all_media_keys() { let new_path = media.get_media_file_sha256(&key).into_os_string(); let old_path = media.get_media_file_b64(&key).into_os_string(); - if let Err(e) = handle_media_check(db, config, &files, &key, &new_path, &old_path).await { + if let Err(e) = handle_media_check(&dbs, config, &files, &key, &new_path, &old_path).await { error!( media_id = ?encode_key(&key), ?new_path, ?old_path, "Failed to resolve media check failure: {e}" @@ -763,9 +775,11 @@ async fn checkup_sha256_media(db: &KeyValueDatabase, config: &Config) -> Result< } async fn handle_media_check( - db: &KeyValueDatabase, config: &Config, files: &HashSet, key: &[u8], new_path: &OsStr, old_path: &OsStr, + dbs: &(&Arc, &Arc), config: &Config, files: &HashSet, key: &[u8], + new_path: &OsStr, old_path: &OsStr, ) -> Result<()> { use crate::media::encode_key; + let (mediaid_file, mediaid_user) = dbs; let old_exists = files.contains(old_path); let new_exists = files.contains(new_path); @@ -775,8 +789,8 @@ async fn handle_media_check( "Media is missing at all paths. Removing from database..." ); - db.mediaid_file.remove(key)?; - db.mediaid_user.remove(key)?; + mediaid_file.remove(key)?; + mediaid_user.remove(key)?; } if config.media_compat_file_link && !old_exists && new_exists { @@ -801,13 +815,13 @@ async fn handle_media_check( Ok(()) } -async fn fix_bad_double_separator_in_state_cache(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn fix_bad_double_separator_in_state_cache(db: &Arc, _config: &Config) -> Result<()> { warn!("Fixing bad double separator in state_cache roomuserid_joined"); + let roomuserid_joined = &db["roomuserid_joined"]; + let _cork = database::Cork::new(&db.db, true, true); + let mut iter_count: usize = 0; - - let _cork = db.db.cork(); - - for (mut key, value) in db.roomuserid_joined.iter() { + for (mut key, value) in roomuserid_joined.iter() { iter_count = iter_count.saturating_add(1); debug_info!(%iter_count); let first_sep_index = key.iter().position(|&i| i == 0xFF).unwrap(); @@ -820,24 +834,24 @@ async fn fix_bad_double_separator_in_state_cache(db: &KeyValueDatabase, _config: == vec![0xFF, 0xFF] { debug_warn!("Found bad key: {key:?}"); - db.roomuserid_joined.remove(&key)?; + roomuserid_joined.remove(&key)?; key.remove(first_sep_index); debug_warn!("Fixed key: {key:?}"); - db.roomuserid_joined.insert(&key, &value)?; + roomuserid_joined.insert(&key, &value)?; } } db.db.cleanup()?; - db.global - .insert(b"fix_bad_double_separator_in_state_cache", &[])?; + db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[])?; info!("Finished fixing"); Ok(()) } -async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &KeyValueDatabase, _config: &Config) -> Result<()> { +async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &Arc, _config: &Config) -> Result<()> { warn!("Retroactively fixing bad data from broken roomuserid_joined"); + let _cork = database::Cork::new(&db.db, true, true); let room_ids = services() .rooms @@ -846,8 +860,6 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &KeyValueDatabase .filter_map(Result::ok) .collect_vec(); - let _cork = db.db.cork(); - for room_id in room_ids.clone() { debug_info!("Fixing room {room_id}"); @@ -910,8 +922,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &KeyValueDatabase } db.db.cleanup()?; - db.global - .insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; + db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; info!("Finished fixing"); Ok(()) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 9a6aef18..0cf0fccf 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -1,5 +1,3 @@ -use conduit::Server; - mod client; mod data; pub(super) mod emerg_access; @@ -13,8 +11,9 @@ use std::{ time::Instant, }; -use conduit::utils; +use conduit::{error, trace, utils::MutexMap, Config, Result, Server}; use data::Data; +use database::Database; use hickory_resolver::TokioAsyncResolver; use ipaddress::IPAddress; use regex::RegexSet; @@ -31,11 +30,9 @@ use tokio::{ sync::{Mutex, RwLock}, task::JoinHandle, }; -use tracing::{error, trace}; use url::Url; -use utils::MutexMap; -use crate::{services, Config, KeyValueDatabase, Result}; +use crate::services; type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries @@ -64,7 +61,7 @@ pub struct Service { } impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { + pub fn build(server: &Arc, db: &Arc) -> Result { let config = &server.config; let db = Data::new(db); let keypair = db.load_keypair(); diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index 39e4cfb0..7cf88cc0 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -6,13 +6,13 @@ use std::{ time::Duration, }; +use conduit::{error, Config, Error}; use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use ruma::OwnedServerName; use tokio::sync::RwLock; -use tracing::error; -use crate::{service::sending::FedDest, Config, Error}; +use crate::sending::FedDest; pub(crate) type WellKnownMap = HashMap; type TlsNameMap = HashMap, u16)>; diff --git a/src/service/key_backups/data.rs b/src/service/key_backups/data.rs index 78042ccf..f1794895 100644 --- a/src/service/key_backups/data.rs +++ b/src/service/key_backups/data.rs @@ -1,5 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{ api::client::{ backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, @@ -9,20 +11,20 @@ use ruma::{ OwnedRoomId, RoomId, UserId, }; -use crate::{services, utils, Error, KeyValueDatabase, KvTree, Result}; +use crate::services; -pub(crate) struct Data { - backupid_algorithm: Arc, - backupid_etag: Arc, - backupkeyid_backup: Arc, +pub(super) struct Data { + backupid_algorithm: Arc, + backupid_etag: Arc, + backupkeyid_backup: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - backupid_algorithm: db.backupid_algorithm.clone(), - backupid_etag: db.backupid_etag.clone(), - backupkeyid_backup: db.backupkeyid_backup.clone(), + backupid_algorithm: db["backupid_algorithm"].clone(), + backupid_etag: db["backupid_etag"].clone(), + backupkeyid_backup: db["backupkeyid_backup"].clone(), } } diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 19b5d53d..650aa6b6 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -1,11 +1,10 @@ -use conduit::Server; - mod data; + use std::{collections::BTreeMap, sync::Arc}; -use conduit::Result; +use conduit::{Result, Server}; use data::Data; -use database::KeyValueDatabase; +use database::Database; use ruma::{ api::client::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, serde::Raw, @@ -13,11 +12,11 @@ use ruma::{ }; pub struct Service { - pub(super) db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/media/data.rs b/src/service/media/data.rs index e52244f3..4cabf167 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -1,24 +1,23 @@ use std::sync::Arc; -use conduit::debug_info; -use database::{KeyValueDatabase, KvTree}; +use conduit::{debug, debug_info, Error, Result}; +use database::{Database, Map}; use ruma::api::client::error::ErrorKind; -use tracing::debug; -use crate::{media::UrlPreviewData, utils::string_from_bytes, Error, Result}; +use crate::{media::UrlPreviewData, utils::string_from_bytes}; -pub struct Data { - mediaid_file: Arc, - mediaid_user: Arc, - url_previews: Arc, +pub(crate) struct Data { + mediaid_file: Arc, + mediaid_user: Arc, + url_previews: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - mediaid_file: db.mediaid_file.clone(), - mediaid_user: db.mediaid_user.clone(), - url_previews: db.url_previews.clone(), + mediaid_file: db["mediaid_file"].clone(), + mediaid_user: db["mediaid_user"].clone(), + url_previews: db["url_previews"].clone(), } } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index f631acb3..0323bfc5 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -6,7 +6,7 @@ use std::{collections::HashMap, io::Cursor, path::PathBuf, sync::Arc, time::Syst use base64::{engine::general_purpose, Engine as _}; use conduit::{debug, debug_error, error, utils, Error, Result, Server}; use data::Data; -use database::KeyValueDatabase; +use database::Database; use image::imageops::FilterType; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; @@ -44,12 +44,12 @@ pub struct UrlPreviewData { pub struct Service { server: Arc, - pub db: Data, + pub(crate) db: Data, pub url_preview_mutex: RwLock>>>, } impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { + pub fn build(server: &Arc, db: &Arc) -> Result { Ok(Self { server: server.clone(), db: Data::new(db), diff --git a/src/service/media/tests.rs b/src/service/media/tests.rs index 9a51bf43..b2f31e6f 100644 --- a/src/service/media/tests.rs +++ b/src/service/media/tests.rs @@ -1,6 +1,7 @@ #![cfg(test)] #[tokio::test] +#[cfg(disable)] //TODO: fixme async fn long_file_names_works() { use std::path::PathBuf; diff --git a/src/service/mod.rs b/src/service/mod.rs index d1db2c25..c80c9862 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -17,15 +17,17 @@ pub mod users; extern crate conduit_core as conduit; extern crate conduit_database as database; + use std::sync::{Arc, RwLock}; pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, PduCount, Result, Server}; -pub(crate) use database::{KeyValueDatabase, KvTree}; -pub use globals::{server_is_ours, user_is_local}; -pub use pdu::PduEvent; -pub use services::Services; +use database::Database; -pub(crate) use crate as service; +pub use crate::{ + globals::{server_is_ours, user_is_local}, + pdu::PduEvent, + services::Services, +}; conduit::mod_ctor! {} conduit::mod_dtor! {} @@ -34,7 +36,7 @@ static SERVICES: RwLock> = RwLock::new(None); #[allow(clippy::let_underscore_must_use)] pub async fn init(server: &Arc) -> Result<()> { - let d = Arc::new(KeyValueDatabase::load_or_create(server).await?); + let d = Arc::new(Database::open(server).await?); let s = Box::new(Services::build(server.clone(), d.clone()).await?); _ = SERVICES.write().expect("write locked").insert(Box::leak(s)); diff --git a/src/service/pdu.rs b/src/service/pdu.rs index 135ae1fd..b5650c0a 100644 --- a/src/service/pdu.rs +++ b/src/service/pdu.rs @@ -1,5 +1,6 @@ use std::{cmp::Ordering, collections::BTreeMap, sync::Arc}; +use conduit::{warn, Error}; use ruma::{ canonical_json::redact_content_in_place, events::{ @@ -17,9 +18,8 @@ use serde_json::{ json, value::{to_raw_value, RawValue as RawJsonValue}, }; -use tracing::warn; -use crate::{services, Error}; +use crate::services; #[derive(Deserialize)] struct ExtractRedactedBecause { diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index ade891a9..5cb4032f 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -1,26 +1,21 @@ use std::sync::Arc; -use conduit::debug_warn; -use database::KvTree; +use conduit::{debug_warn, utils, Error, Result}; +use database::{Database, Map}; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; -use crate::{ - presence::Presence, - services, - utils::{self, user_id_from_bytes}, - Error, KeyValueDatabase, Result, -}; +use crate::{presence::Presence, services}; pub struct Data { - presenceid_presence: Arc, - userid_presenceid: Arc, + presenceid_presence: Arc, + userid_presenceid: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - presenceid_presence: db.presenceid_presence.clone(), - userid_presenceid: db.userid_presenceid.clone(), + presenceid_presence: db["presenceid_presence"].clone(), + userid_presenceid: db["userid_presenceid"].clone(), } } @@ -135,7 +130,7 @@ fn presenceid_key(count: u64, user_id: &UserId) -> Vec { #[inline] fn presenceid_parse(key: &[u8]) -> Result<(u64, OwnedUserId)> { let (count, user_id) = key.split_at(8); - let user_id = user_id_from_bytes(user_id)?; + let user_id = utils::user_id_from_bytes(user_id)?; let count = utils::u64_from_bytes(count).unwrap(); Ok((count, user_id)) diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 046614c9..5065efdd 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -1,10 +1,10 @@ -use conduit::Server; - mod data; use std::{sync::Arc, time::Duration}; +use conduit::{debug, error, utils, Error, Result, Server}; use data::Data; +use database::Database; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ events::presence::{PresenceEvent, PresenceEventContent}, @@ -13,14 +13,8 @@ use ruma::{ }; use serde::{Deserialize, Serialize}; use tokio::{sync::Mutex, task::JoinHandle, time::sleep}; -use tracing::{debug, error}; -use crate::{ - database::KeyValueDatabase, - services, user_is_local, - utils::{self}, - Error, Result, -}; +use crate::{services, user_is_local}; /// Represents data required to be kept in order to implement the presence /// specification. @@ -88,7 +82,7 @@ pub struct Service { } impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result> { + pub fn build(server: &Arc, db: &Arc) -> Result> { let config = &server.config; let (timer_sender, timer_receiver) = loole::unbounded(); Ok(Arc::new(Self { diff --git a/src/service/pusher/data.rs b/src/service/pusher/data.rs index a8a9366e..f9734334 100644 --- a/src/service/pusher/data.rs +++ b/src/service/pusher/data.rs @@ -1,21 +1,20 @@ use std::sync::Arc; -use database::{KeyValueDatabase, KvTree}; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{ api::client::push::{set_pusher, Pusher}, UserId, }; -use crate::{utils, Error, Result}; - -pub struct Data { - senderkey_pusher: Arc, +pub(super) struct Data { + senderkey_pusher: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - senderkey_pusher: db.senderkey_pusher.clone(), + senderkey_pusher: db["senderkey_pusher"].clone(), } } diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 25faecd5..28280273 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -1,11 +1,11 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; + use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; +use conduit::{debug_info, info, trace, warn, Error, Result, Server}; use data::Data; +use database::Database; use ipaddress::IPAddress; use ruma::{ api::{ @@ -23,16 +23,15 @@ use ruma::{ serde::Raw, uint, RoomId, UInt, UserId, }; -use tracing::{info, trace, warn}; -use crate::{debug_info, services, Error, PduEvent, Result}; +use crate::{services, PduEvent}; pub struct Service { - pub db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/alias/data.rs b/src/service/rooms/alias/data.rs index bc0c9cc6..04f5ca7f 100644 --- a/src/service/rooms/alias/data.rs +++ b/src/service/rooms/alias/data.rs @@ -1,22 +1,23 @@ use std::sync::Arc; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{api::client::error::ErrorKind, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, UserId}; -use crate::{services, utils, Error, KeyValueDatabase, Result}; +use crate::services; -pub struct Data { - alias_userid: Arc, - alias_roomid: Arc, - aliasid_alias: Arc, +pub(super) struct Data { + alias_userid: Arc, + alias_roomid: Arc, + aliasid_alias: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - alias_userid: db.alias_userid.clone(), - alias_roomid: db.alias_roomid.clone(), - aliasid_alias: db.aliasid_alias.clone(), + alias_userid: db["alias_userid"].clone(), + alias_roomid: db["alias_roomid"].clone(), + aliasid_alias: db["aliasid_alias"].clone(), } } @@ -55,7 +56,7 @@ impl Data { Ok(()) } - pub fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { + pub(super) fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { self.alias_roomid .get(alias.alias().as_bytes())? .map(|bytes| { @@ -81,7 +82,7 @@ impl Data { .transpose() } - pub fn local_aliases_for_room<'a>( + pub(super) fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { let mut prefix = room_id.as_bytes().to_vec(); @@ -95,7 +96,7 @@ impl Data { })) } - pub fn all_local_aliases<'a>(&'a self) -> Box> + 'a> { + pub(super) fn all_local_aliases<'a>(&'a self) -> Box> + 'a> { Box::new( self.alias_roomid .iter() diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 5c4f5cdb..cfa4edb9 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -1,11 +1,10 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; +use conduit::{Error, Result, Server}; use data::Data; +use database::Database; use ruma::{ api::client::error::ErrorKind, events::{ @@ -15,14 +14,14 @@ use ruma::{ OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId, UserId, }; -use crate::{services, Error, Result}; +use crate::services; pub struct Service { - pub db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index bb6e6516..4e844d6c 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -1,18 +1,17 @@ use std::{mem::size_of, sync::Arc}; -use database::KvTree; - -use crate::{utils, KeyValueDatabase, Result}; +use conduit::{utils, Result}; +use database::{Database, Map}; pub(super) struct Data { - shorteventid_authchain: Arc, - db: Arc, + shorteventid_authchain: Arc, + db: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - shorteventid_authchain: db.shorteventid_authchain.clone(), + shorteventid_authchain: db["shorteventid_authchain"].clone(), db: db.clone(), } } diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 6cadee0f..ca9cf5f2 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -1,24 +1,23 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; + use std::{ collections::{BTreeSet, HashSet}, sync::Arc, }; +use conduit::{debug, error, trace, warn, Error, Result, Server}; use data::Data; +use database::Database; use ruma::{api::client::error::ErrorKind, EventId, RoomId}; -use tracing::{debug, error, trace, warn}; -use crate::{services, Error, Result}; +use crate::services; pub struct Service { db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/directory/data.rs b/src/service/rooms/directory/data.rs index f4c453eb..713ee057 100644 --- a/src/service/rooms/directory/data.rs +++ b/src/service/rooms/directory/data.rs @@ -1,18 +1,17 @@ use std::sync::Arc; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{OwnedRoomId, RoomId}; -use crate::{utils, Error, KeyValueDatabase, Result}; - pub(super) struct Data { - publicroomids: Arc, + publicroomids: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - publicroomids: db.publicroomids.clone(), + publicroomids: db["publicroomids"].clone(), } } diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index a0c4caf7..87c7cf92 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -1,11 +1,10 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; +use conduit::Server; use data::Data; +use database::Database; use ruma::{OwnedRoomId, RoomId}; use crate::Result; @@ -15,7 +14,7 @@ pub struct Service { } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index c7e819ac..ca9c9c14 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1,6 +1,3 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod parse_incoming_pdu; mod signing_keys; @@ -12,6 +9,8 @@ use std::{ time::{Duration, Instant}, }; +use conduit::{debug_error, debug_info, Error, Result, Server}; +use database::Database; use futures_util::Future; pub use parse_incoming_pdu::parse_incoming_pdu; use ruma::{ @@ -32,7 +31,7 @@ use tokio::sync::RwLock; use tracing::{debug, error, info, trace, warn}; use super::state_compressor::CompressedStateEvent; -use crate::{debug_error, debug_info, pdu, services, Error, PduEvent, Result}; +use crate::{pdu, services, PduEvent}; pub struct Service; @@ -45,7 +44,7 @@ type AsyncRecursiveCanonicalJsonResult<'a> = AsyncRecursiveType<'a, Result<(Arc, BTreeMap)>>; impl Service { - pub fn build(_server: &Arc, _db: &Arc) -> Result { Ok(Self {}) } + pub fn build(_server: &Arc, _db: &Arc) -> Result { Ok(Self {}) } /// When receiving an event one needs to: /// 0. Check the server is in the room diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index d9d9f063..4c907e51 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,8 +1,9 @@ +use conduit::{Error, Result}; use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId}; use serde_json::value::RawValue as RawJsonValue; use tracing::warn; -use crate::{service::pdu::gen_event_id_canonical_json, services, Error, Result}; +use crate::{pdu::gen_event_id_canonical_json, services}; pub fn parse_incoming_pdu(pdu: &RawJsonValue) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { diff --git a/src/service/rooms/lazy_loading/data.rs b/src/service/rooms/lazy_loading/data.rs index 63823c4a..073d45f5 100644 --- a/src/service/rooms/lazy_loading/data.rs +++ b/src/service/rooms/lazy_loading/data.rs @@ -1,18 +1,17 @@ use std::sync::Arc; -use database::KvTree; +use conduit::Result; +use database::{Database, Map}; use ruma::{DeviceId, RoomId, UserId}; -use crate::{KeyValueDatabase, Result}; - -pub struct Data { - lazyloadedids: Arc, +pub(super) struct Data { + lazyloadedids: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - lazyloadedids: db.lazyloadedids.clone(), + lazyloadedids: db["lazyloadedids"].clone(), } } diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index dba78500..1f2ae6dd 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -1,27 +1,27 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; + use std::{ collections::{HashMap, HashSet}, sync::Arc, }; +use conduit::Server; use data::Data; +use database::Database; use ruma::{DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, RoomId, UserId}; use tokio::sync::Mutex; use crate::{PduCount, Result}; pub struct Service { - pub db: Data, + db: Data, #[allow(clippy::type_complexity)] pub lazy_load_waiting: Mutex>>, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), lazy_load_waiting: Mutex::new(HashMap::new()), diff --git a/src/service/rooms/metadata/data.rs b/src/service/rooms/metadata/data.rs index b5f2fcc3..df9b6b10 100644 --- a/src/service/rooms/metadata/data.rs +++ b/src/service/rooms/metadata/data.rs @@ -1,25 +1,25 @@ use std::sync::Arc; -use database::KvTree; +use conduit::{error, utils, Error, Result}; +use database::{Database, Map}; use ruma::{OwnedRoomId, RoomId}; -use tracing::error; -use crate::{services, utils, Error, KeyValueDatabase, Result}; +use crate::services; -pub struct Data { - disabledroomids: Arc, - bannedroomids: Arc, - roomid_shortroomid: Arc, - pduid_pdu: Arc, +pub(super) struct Data { + disabledroomids: Arc, + bannedroomids: Arc, + roomid_shortroomid: Arc, + pduid_pdu: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - disabledroomids: db.disabledroomids.clone(), - bannedroomids: db.bannedroomids.clone(), - roomid_shortroomid: db.roomid_shortroomid.clone(), - pduid_pdu: db.pduid_pdu.clone(), + disabledroomids: db["disabledroomids"].clone(), + bannedroomids: db["bannedroomids"].clone(), + roomid_shortroomid: db["roomid_shortroomid"].clone(), + pduid_pdu: db["pduid_pdu"].clone(), } } diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 21f946b6..b91fc67e 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -1,21 +1,18 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; +use conduit::{Result, Server}; use data::Data; +use database::Database; use ruma::{OwnedRoomId, RoomId}; -use crate::Result; - pub struct Service { - pub db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/outlier/data.rs b/src/service/rooms/outlier/data.rs index f89e9fb4..aa804721 100644 --- a/src/service/rooms/outlier/data.rs +++ b/src/service/rooms/outlier/data.rs @@ -1,18 +1,19 @@ use std::sync::Arc; -use database::KvTree; +use conduit::{Error, Result}; +use database::{Database, Map}; use ruma::{CanonicalJsonObject, EventId}; -use crate::{Error, KeyValueDatabase, PduEvent, Result}; +use crate::PduEvent; -pub struct Data { - eventid_outlierpdu: Arc, +pub(super) struct Data { + eventid_outlierpdu: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - eventid_outlierpdu: db.eventid_outlierpdu.clone(), + eventid_outlierpdu: db["eventid_outlierpdu"].clone(), } } diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index ee1f9466..0da12a14 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -1,21 +1,20 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; +use conduit::{Result, Server}; use data::Data; +use database::Database; use ruma::{CanonicalJsonObject, EventId}; -use crate::{PduEvent, Result}; +use crate::PduEvent; pub struct Service { - pub db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 7be183a0..24c756fd 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,25 +1,26 @@ use std::{mem::size_of, sync::Arc}; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{EventId, RoomId, UserId}; -use crate::{services, utils, Error, KeyValueDatabase, PduCount, PduEvent, Result}; +use crate::{services, PduCount, PduEvent}; pub(super) struct Data { - tofrom_relation: Arc, - referencedevents: Arc, - softfailedeventids: Arc, + tofrom_relation: Arc, + referencedevents: Arc, + softfailedeventids: Arc, } type PdusIterItem = Result<(PduCount, PduEvent)>; type PdusIterator<'a> = Box + 'a>; impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - tofrom_relation: db.tofrom_relation.clone(), - referencedevents: db.referencedevents.clone(), - softfailedeventids: db.softfailedeventids.clone(), + tofrom_relation: db["tofrom_relation"].clone(), + referencedevents: db["referencedevents"].clone(), + softfailedeventids: db["softfailedeventids"].clone(), } } diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 488cf03e..d77fbf74 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -1,11 +1,10 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; +use conduit::{Result, Server}; use data::Data; +use database::Database; use ruma::{ api::{client::relations::get_relating_events, Direction}, events::{relation::RelationType, TimelineEventType}, @@ -13,7 +12,7 @@ use ruma::{ }; use serde::Deserialize; -use crate::{services, PduCount, PduEvent, Result}; +use crate::{services, PduCount, PduEvent}; pub struct Service { db: Data, @@ -30,7 +29,7 @@ struct ExtractRelatesToEventId { } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index a33c64ac..17acb0b3 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -1,29 +1,30 @@ use std::{mem::size_of, sync::Arc}; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{ events::{receipt::ReceiptEvent, AnySyncEphemeralRoomEvent}, serde::Raw, CanonicalJsonObject, OwnedUserId, RoomId, UserId, }; -use crate::{services, utils, Error, KeyValueDatabase, Result}; +use crate::services; type AnySyncEphemeralRoomEventIter<'a> = Box)>> + 'a>; pub(super) struct Data { - roomuserid_privateread: Arc, - roomuserid_lastprivatereadupdate: Arc, - readreceiptid_readreceipt: Arc, + roomuserid_privateread: Arc, + roomuserid_lastprivatereadupdate: Arc, + readreceiptid_readreceipt: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - roomuserid_privateread: db.roomuserid_privateread.clone(), - roomuserid_lastprivatereadupdate: db.roomuserid_lastprivatereadupdate.clone(), - readreceiptid_readreceipt: db.readreceiptid_readreceipt.clone(), + roomuserid_privateread: db["roomuserid_privateread"].clone(), + roomuserid_lastprivatereadupdate: db["roomuserid_lastprivatereadupdate"].clone(), + readreceiptid_readreceipt: db["readreceiptid_readreceipt"].clone(), } } diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 5da374ef..ccc17d3a 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -1,21 +1,20 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; +use conduit::{Result, Server}; use data::Data; +use database::Database; use ruma::{events::receipt::ReceiptEvent, serde::Raw, OwnedUserId, RoomId, UserId}; -use crate::{services, Result}; +use crate::services; pub struct Service { db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/search/data.rs b/src/service/rooms/search/data.rs index ff42cc9f..fe5d3edf 100644 --- a/src/service/rooms/search/data.rs +++ b/src/service/rooms/search/data.rs @@ -1,20 +1,21 @@ use std::sync::Arc; -use database::KvTree; +use conduit::{utils, Result}; +use database::{Database, Map}; use ruma::RoomId; -use crate::{services, utils, KeyValueDatabase, Result}; +use crate::services; type SearchPdusResult<'a> = Result> + 'a>, Vec)>>; -pub struct Data { - tokenids: Arc, +pub(super) struct Data { + tokenids: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - tokenids: db.tokenids.clone(), + tokenids: db["tokenids"].clone(), } } diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 9762c8fc..e2ef0f80 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,21 +1,18 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; +use conduit::{Result, Server}; use data::Data; +use database::Database; use ruma::RoomId; -use crate::Result; - pub struct Service { - pub db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/short/data.rs b/src/service/rooms/short/data.rs index 13088758..883c3c1d 100644 --- a/src/service/rooms/short/data.rs +++ b/src/service/rooms/short/data.rs @@ -1,29 +1,29 @@ use std::sync::Arc; -use conduit::warn; -use database::{KeyValueDatabase, KvTree}; +use conduit::{utils, warn, Error, Result}; +use database::{Database, Map}; use ruma::{events::StateEventType, EventId, RoomId}; -use crate::{services, utils, Error, Result}; +use crate::services; pub(super) struct Data { - eventid_shorteventid: Arc, - shorteventid_eventid: Arc, - statekey_shortstatekey: Arc, - shortstatekey_statekey: Arc, - roomid_shortroomid: Arc, - statehash_shortstatehash: Arc, + eventid_shorteventid: Arc, + shorteventid_eventid: Arc, + statekey_shortstatekey: Arc, + shortstatekey_statekey: Arc, + roomid_shortroomid: Arc, + statehash_shortstatehash: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - eventid_shorteventid: db.eventid_shorteventid.clone(), - shorteventid_eventid: db.shorteventid_eventid.clone(), - statekey_shortstatekey: db.statekey_shortstatekey.clone(), - shortstatekey_statekey: db.shortstatekey_statekey.clone(), - roomid_shortroomid: db.roomid_shortroomid.clone(), - statehash_shortstatehash: db.statehash_shortstatehash.clone(), + eventid_shorteventid: db["eventid_shorteventid"].clone(), + shorteventid_eventid: db["shorteventid_eventid"].clone(), + statekey_shortstatekey: db["statekey_shortstatekey"].clone(), + shortstatekey_statekey: db["shortstatekey_statekey"].clone(), + roomid_shortroomid: db["roomid_shortroomid"].clone(), + statehash_shortstatehash: db["statehash_shortstatehash"].clone(), } } diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 0cd676a9..638b6c1c 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,20 +1,18 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; + use std::sync::Arc; +use conduit::{Result, Server}; use data::Data; +use database::Database; use ruma::{events::StateEventType, EventId, RoomId}; -use crate::Result; - pub struct Service { db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index e11bade0..bc7703a3 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -4,8 +4,8 @@ use std::{ sync::Arc, }; -use conduit::Server; -use database::KeyValueDatabase; +use conduit::{debug_info, Error, Result, Server}; +use database::Database; use lru_cache::LruCache; use ruma::{ api::{ @@ -31,7 +31,7 @@ use ruma::{ use tokio::sync::Mutex; use tracing::{debug, error, warn}; -use crate::{debug_info, server_is_ours, services, Error, Result}; +use crate::{server_is_ours, services}; pub struct CachedSpaceHierarchySummary { summary: SpaceHierarchyParentSummary, @@ -333,7 +333,7 @@ impl From for SpaceHierarchyRoomsChunk { } impl Service { - pub fn build(server: &Arc, _db: &Arc) -> Result { + pub fn build(server: &Arc, _db: &Arc) -> Result { let config = &server.config; Ok(Self { roomid_spacehierarchy_cache: Mutex::new(LruCache::new( diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index 3a0f93b5..dad613d2 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -1,23 +1,22 @@ use std::{collections::HashSet, sync::Arc}; -use conduit::utils::mutex_map; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{EventId, OwnedEventId, RoomId}; +use utils::mutex_map; -use crate::{utils, Error, KeyValueDatabase, Result}; - -pub struct Data { - shorteventid_shortstatehash: Arc, - roomid_pduleaves: Arc, - roomid_shortstatehash: Arc, +pub(super) struct Data { + shorteventid_shortstatehash: Arc, + roomid_pduleaves: Arc, + roomid_shortstatehash: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - shorteventid_shortstatehash: db.shorteventid_shortstatehash.clone(), - roomid_pduleaves: db.roomid_pduleaves.clone(), - roomid_shortstatehash: db.roomid_shortstatehash.clone(), + shorteventid_shortstatehash: db["shorteventid_shortstatehash"].clone(), + roomid_pduleaves: db["roomid_pduleaves"].clone(), + roomid_shortstatehash: db["roomid_shortstatehash"].clone(), } } diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index e0d75c80..42483c97 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -1,14 +1,16 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; + use std::{ collections::{HashMap, HashSet}, sync::Arc, }; -use conduit::utils::mutex_map; +use conduit::{ + utils::{calculate_hash, mutex_map}, + warn, Error, Result, Server, +}; use data::Data; +use database::Database; use ruma::{ api::client::error::ErrorKind, events::{ @@ -19,17 +21,16 @@ use ruma::{ state_res::{self, StateMap}, EventId, OwnedEventId, RoomId, RoomVersionId, UserId, }; -use tracing::warn; use super::state_compressor::CompressedStateEvent; -use crate::{services, utils::calculate_hash, Error, PduEvent, Result}; +use crate::{services, PduEvent}; pub struct Service { - pub db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 4b6dd131..7e9daeda 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,20 +1,21 @@ use std::{collections::HashMap, sync::Arc}; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{events::StateEventType, EventId, RoomId}; -use crate::{services, utils, Error, KeyValueDatabase, PduEvent, Result}; +use crate::{services, PduEvent}; -pub struct Data { - eventid_shorteventid: Arc, - shorteventid_shortstatehash: Arc, +pub(super) struct Data { + eventid_shorteventid: Arc, + shorteventid_shortstatehash: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - eventid_shorteventid: db.eventid_shorteventid.clone(), - shorteventid_shortstatehash: db.shorteventid_shortstatehash.clone(), + eventid_shorteventid: db["eventid_shorteventid"].clone(), + shorteventid_shortstatehash: db["shorteventid_shortstatehash"].clone(), } } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index e1466d75..3df0e123 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -1,16 +1,13 @@ -use std::sync::Mutex as StdMutex; - -use conduit::Server; -use database::KeyValueDatabase; - mod data; + use std::{ collections::HashMap, - sync::{Arc, Mutex}, + sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::utils::mutex_map; +use conduit::{error, utils::mutex_map, warn, Error, Result, Server}; use data::Data; +use database::Database; use lru_cache::LruCache; use ruma::{ events::{ @@ -29,18 +26,17 @@ use ruma::{ EventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; use serde_json::value::to_raw_value; -use tracing::{error, warn}; -use crate::{service::pdu::PduBuilder, services, Error, PduEvent, Result}; +use crate::{pdu::PduBuilder, services, PduEvent}; pub struct Service { - pub db: Data, + db: Data, pub server_visibility_cache: Mutex>, pub user_visibility_cache: Mutex>, } impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { + pub fn build(server: &Arc, db: &Arc) -> Result { let config = &server.config; Ok(Self { db: Data::new(db), diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 03c04abb..c8d05ab0 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -1,5 +1,7 @@ -use std::collections::HashSet; +use std::{collections::HashSet, sync::Arc}; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use itertools::Itertools; use ruma::{ events::{AnyStrippedStateEvent, AnySyncStateEvent}, @@ -7,51 +9,42 @@ use ruma::{ OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; -use crate::{ - appservice::RegistrationInfo, - services, user_is_local, - utils::{self}, - Error, KeyValueDatabase, Result, -}; +use crate::{appservice::RegistrationInfo, services, user_is_local}; type StrippedStateEventIter<'a> = Box>)>> + 'a>; type AnySyncStateEventIter<'a> = Box>)>> + 'a>; -use std::sync::Arc; - -use database::KvTree; - -pub struct Data { - userroomid_joined: Arc, - roomuserid_joined: Arc, - userroomid_invitestate: Arc, - roomuserid_invitecount: Arc, - userroomid_leftstate: Arc, - roomuserid_leftcount: Arc, - roomid_inviteviaservers: Arc, - roomuseroncejoinedids: Arc, - roomid_joinedcount: Arc, - roomid_invitedcount: Arc, - roomserverids: Arc, - serverroomids: Arc, - db: Arc, +pub(super) struct Data { + userroomid_joined: Arc, + roomuserid_joined: Arc, + userroomid_invitestate: Arc, + roomuserid_invitecount: Arc, + userroomid_leftstate: Arc, + roomuserid_leftcount: Arc, + roomid_inviteviaservers: Arc, + roomuseroncejoinedids: Arc, + roomid_joinedcount: Arc, + roomid_invitedcount: Arc, + roomserverids: Arc, + serverroomids: Arc, + db: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - userroomid_joined: db.userroomid_joined.clone(), - roomuserid_joined: db.roomuserid_joined.clone(), - userroomid_invitestate: db.userroomid_invitestate.clone(), - roomuserid_invitecount: db.roomuserid_invitecount.clone(), - userroomid_leftstate: db.userroomid_leftstate.clone(), - roomuserid_leftcount: db.roomuserid_leftcount.clone(), - roomid_inviteviaservers: db.roomid_inviteviaservers.clone(), - roomuseroncejoinedids: db.roomuseroncejoinedids.clone(), - roomid_joinedcount: db.roomid_joinedcount.clone(), - roomid_invitedcount: db.roomid_invitedcount.clone(), - roomserverids: db.roomserverids.clone(), - serverroomids: db.serverroomids.clone(), + userroomid_joined: db["userroomid_joined"].clone(), + roomuserid_joined: db["roomuserid_joined"].clone(), + userroomid_invitestate: db["userroomid_invitestate"].clone(), + roomuserid_invitecount: db["roomuserid_invitecount"].clone(), + userroomid_leftstate: db["userroomid_leftstate"].clone(), + roomuserid_leftcount: db["roomuserid_leftcount"].clone(), + roomid_inviteviaservers: db["roomid_inviteviaservers"].clone(), + roomuseroncejoinedids: db["roomuseroncejoinedids"].clone(), + roomid_joinedcount: db["roomid_joinedcount"].clone(), + roomid_invitedcount: db["roomid_invitedcount"].clone(), + roomserverids: db["roomserverids"].clone(), + serverroomids: db["serverroomids"].clone(), db: db.clone(), } } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index c4eb0dff..eeecc9c7 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -1,8 +1,10 @@ +mod data; + use std::sync::Arc; -use conduit::Server; +use conduit::{error, warn, Error, Result, Server}; use data::Data; -use database::KeyValueDatabase; +use database::Database; use itertools::Itertools; use ruma::{ events::{ @@ -19,18 +21,15 @@ use ruma::{ serde::Raw, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; -use tracing::{error, warn}; -use crate::{service::appservice::RegistrationInfo, services, user_is_local, Error, Result}; - -mod data; +use crate::{appservice::RegistrationInfo, services, user_is_local}; pub struct Service { - pub db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/state_compressor/data.rs b/src/service/rooms/state_compressor/data.rs index 454a3b57..61c7d6e6 100644 --- a/src/service/rooms/state_compressor/data.rs +++ b/src/service/rooms/state_compressor/data.rs @@ -1,9 +1,9 @@ use std::{collections::HashSet, mem::size_of, sync::Arc}; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use super::CompressedStateEvent; -use crate::{utils, Error, KeyValueDatabase, Result}; pub(super) struct StateDiff { pub(super) parent: Option, @@ -11,14 +11,14 @@ pub(super) struct StateDiff { pub(super) removed: Arc>, } -pub struct Data { - shortstatehash_statediff: Arc, +pub(super) struct Data { + shortstatehash_statediff: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - shortstatehash_statediff: db.shortstatehash_statediff.clone(), + shortstatehash_statediff: db["shortstatehash_statediff"].clone(), } } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index ee0e4cc9..08112995 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -1,21 +1,19 @@ -use std::sync::Mutex as StdMutex; - -use conduit::Server; -use database::KeyValueDatabase; - mod data; + use std::{ collections::HashSet, mem::size_of, - sync::{Arc, Mutex}, + sync::{Arc, Mutex as StdMutex, Mutex}, }; +use conduit::{utils, Result, Server}; use data::Data; +use database::Database; use lru_cache::LruCache; use ruma::{EventId, RoomId}; use self::data::StateDiff; -use crate::{services, utils, Result}; +use crate::services; type StateInfoLruCache = Mutex< LruCache< @@ -49,13 +47,13 @@ type HashSetCompressStateEvent = Result<(u64, Arc> pub type CompressedStateEvent = [u8; 2 * size_of::()]; pub struct Service { - pub db: Data, + db: Data, pub stateinfo_cache: StateInfoLruCache, } impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { + pub fn build(server: &Arc, db: &Arc) -> Result { let config = &server.config; Ok(Self { db: Data::new(db), diff --git a/src/service/rooms/threads/data.rs b/src/service/rooms/threads/data.rs index a4044294..29539847 100644 --- a/src/service/rooms/threads/data.rs +++ b/src/service/rooms/threads/data.rs @@ -1,20 +1,21 @@ use std::{mem::size_of, sync::Arc}; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId}; -use crate::{services, utils, Error, KeyValueDatabase, PduEvent, Result}; +use crate::{services, PduEvent}; type PduEventIterResult<'a> = Result> + 'a>>; -pub struct Data { - threadid_userids: Arc, +pub(super) struct Data { + threadid_userids: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - threadid_userids: db.threadid_userids.clone(), + threadid_userids: db["threadid_userids"].clone(), } } diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index 606cc001..f47fc499 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -1,11 +1,10 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::{collections::BTreeMap, sync::Arc}; +use conduit::{Error, Result, Server}; use data::Data; +use database::Database; use ruma::{ api::client::{error::ErrorKind, threads::get_threads::v1::IncludeThreads}, events::relation::BundledThread, @@ -13,14 +12,14 @@ use ruma::{ }; use serde_json::json; -use crate::{services, Error, PduEvent, Result}; +use crate::{services, PduEvent}; pub struct Service { - pub db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 41a58f67..ba74b1aa 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -1,32 +1,31 @@ use std::{collections::hash_map, mem::size_of, sync::Arc}; -use database::KvTree; +use conduit::{error, utils, Error, Result}; +use database::{Database, Map}; use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedUserId, RoomId, UserId}; -use tracing::error; -use super::PduCount; -use crate::{services, utils, Error, KeyValueDatabase, PduEvent, Result}; +use crate::{services, PduCount, PduEvent}; -pub struct Data { - eventid_pduid: Arc, - pduid_pdu: Arc, - eventid_outlierpdu: Arc, - userroomid_notificationcount: Arc, - userroomid_highlightcount: Arc, - db: Arc, +pub(super) struct Data { + eventid_pduid: Arc, + pduid_pdu: Arc, + eventid_outlierpdu: Arc, + userroomid_notificationcount: Arc, + userroomid_highlightcount: Arc, + db: Arc, } type PdusIterItem = Result<(PduCount, PduEvent)>; type PdusIterator<'a> = Box + 'a>; impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - eventid_pduid: db.eventid_pduid.clone(), - pduid_pdu: db.pduid_pdu.clone(), - eventid_outlierpdu: db.eventid_outlierpdu.clone(), - userroomid_notificationcount: db.userroomid_notificationcount.clone(), - userroomid_highlightcount: db.userroomid_highlightcount.clone(), + eventid_pduid: db["eventid_pduid"].clone(), + pduid_pdu: db["pduid_pdu"].clone(), + eventid_outlierpdu: db["eventid_outlierpdu"].clone(), + userroomid_notificationcount: db["userroomid_notificationcount"].clone(), + userroomid_highlightcount: db["userroomid_highlightcount"].clone(), db: db.clone(), } } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 994b5cd4..2680c7f4 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1,6 +1,3 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::{ @@ -8,7 +5,9 @@ use std::{ sync::Arc, }; +use conduit::{debug, error, info, utils, utils::mutex_map, warn, Error, Result, Server}; use data::Data; +use database::Database; use itertools::Itertools; use rand::prelude::SliceRandom; use ruma::{ @@ -34,24 +33,13 @@ use ruma::{ use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tokio::sync::{Mutex, RwLock}; -use tracing::{debug, error, info, warn}; -use super::state_compressor::CompressedStateEvent; use crate::{ admin, - server_is_ours, - //api::server_server, - service::{ - appservice::NamespaceRegex, - pdu::{EventHash, PduBuilder}, - rooms::event_handler::parse_incoming_pdu, - }, - services, - utils::{self, mutex_map}, - Error, - PduCount, - PduEvent, - Result, + appservice::NamespaceRegex, + pdu::{EventHash, PduBuilder}, + rooms::{event_handler::parse_incoming_pdu, state_compressor::CompressedStateEvent}, + server_is_ours, services, PduCount, PduEvent, }; // Update Relationships @@ -77,13 +65,13 @@ struct ExtractBody { } pub struct Service { - pub db: Data, + db: Data, pub lasttimelinecount_cache: Mutex>, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), lasttimelinecount_cache: Mutex::new(HashMap::new()), diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 614d5af2..dd4e7fe2 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -1,20 +1,15 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduit::Server; -use database::KeyValueDatabase; +use conduit::{debug_info, trace, utils, Result, Server}; +use database::Database; use ruma::{ api::federation::transactions::edu::{Edu, TypingContent}, events::SyncEphemeralRoomEvent, OwnedRoomId, OwnedUserId, RoomId, UserId, }; use tokio::sync::{broadcast, RwLock}; -use tracing::trace; -use crate::{ - debug_info, services, user_is_local, - utils::{self}, - Result, -}; +use crate::{services, user_is_local}; pub struct Service { pub typing: RwLock>>, // u64 is unix timestamp of timeout @@ -25,7 +20,7 @@ pub struct Service { } impl Service { - pub fn build(_server: &Arc, _db: &Arc) -> Result { + pub fn build(_server: &Arc, _db: &Arc) -> Result { Ok(Self { typing: RwLock::new(BTreeMap::new()), last_typing_update: RwLock::new(BTreeMap::new()), diff --git a/src/service/rooms/user/data.rs b/src/service/rooms/user/data.rs index e9339917..618caae0 100644 --- a/src/service/rooms/user/data.rs +++ b/src/service/rooms/user/data.rs @@ -1,26 +1,27 @@ use std::sync::Arc; -use database::KvTree; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; -use crate::{services, utils, Error, KeyValueDatabase, Result}; +use crate::services; -pub struct Data { - userroomid_notificationcount: Arc, - userroomid_highlightcount: Arc, - roomuserid_lastnotificationread: Arc, - roomsynctoken_shortstatehash: Arc, - userroomid_joined: Arc, +pub(super) struct Data { + userroomid_notificationcount: Arc, + userroomid_highlightcount: Arc, + roomuserid_lastnotificationread: Arc, + roomsynctoken_shortstatehash: Arc, + userroomid_joined: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - userroomid_notificationcount: db.userroomid_notificationcount.clone(), - userroomid_highlightcount: db.userroomid_highlightcount.clone(), - roomuserid_lastnotificationread: db.roomuserid_lastnotificationread.clone(), - roomsynctoken_shortstatehash: db.roomsynctoken_shortstatehash.clone(), - userroomid_joined: db.userroomid_joined.clone(), + userroomid_notificationcount: db["userroomid_notificationcount"].clone(), + userroomid_highlightcount: db["userroomid_highlightcount"].clone(), + roomuserid_lastnotificationread: db["userroomid_highlightcount"].clone(), //< NOTE: known bug from conduit + roomsynctoken_shortstatehash: db["roomsynctoken_shortstatehash"].clone(), + userroomid_joined: db["userroomid_joined"].clone(), } } diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 36e78982..505e0662 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -1,21 +1,18 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; +use conduit::{Result, Server}; use data::Data; +use database::Database; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; -use crate::Result; - pub struct Service { - pub db: Data, + db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 13ad4d6e..20e5b77a 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -1,26 +1,28 @@ use std::sync::Arc; +use conduit::{utils, Error, Result}; +use database::{Database, Map}; use ruma::{ServerName, UserId}; use super::{Destination, SendingEvent}; -use crate::{services, utils, Error, KeyValueDatabase, KvTree, Result}; +use crate::services; type OutgoingSendingIter<'a> = Box, Destination, SendingEvent)>> + 'a>; type SendingEventIter<'a> = Box, SendingEvent)>> + 'a>; pub struct Data { - servercurrentevent_data: Arc, - servernameevent_data: Arc, - servername_educount: Arc, - _db: Arc, + servercurrentevent_data: Arc, + servernameevent_data: Arc, + servername_educount: Arc, + _db: Arc, } impl Data { - pub(super) fn new(db: Arc) -> Self { + pub(super) fn new(db: Arc) -> Self { Self { - servercurrentevent_data: db.servercurrentevent_data.clone(), - servernameevent_data: db.servernameevent_data.clone(), - servername_educount: db.servername_educount.clone(), + servercurrentevent_data: db["servercurrentevent_data"].clone(), + servernameevent_data: db["servernameevent_data"].clone(), + servername_educount: db["servername_educount"].clone(), _db: db, } } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index b9a0144f..3894ded1 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -1,5 +1,3 @@ -use conduit::Server; - mod appservice; mod data; pub mod resolve; @@ -8,7 +6,9 @@ mod sender; use std::{fmt::Debug, sync::Arc}; +use conduit::{Error, Result, Server}; use data::Data; +use database::Database; pub use resolve::FedDest; use ruma::{ api::{appservice::Registration, OutgoingRequest}, @@ -17,7 +17,7 @@ use ruma::{ use tokio::{sync::Mutex, task::JoinHandle}; use tracing::{error, warn}; -use crate::{server_is_ours, services, Error, KeyValueDatabase, Result}; +use crate::{server_is_ours, services}; pub struct Service { pub db: Data, @@ -53,7 +53,7 @@ pub enum SendingEvent { } impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result> { + pub fn build(server: &Arc, db: &Arc) -> Result> { let config = &server.config; let (sender, receiver) = loole::unbounded(); Ok(Arc::new(Self { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 7edb4db5..e1a14dfd 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -23,7 +23,7 @@ use ruma::{ use tracing::{debug, error, warn}; use super::{appservice, send, Destination, Msg, SendingEvent, Service}; -use crate::{service::presence::Presence, services, user_is_local, utils::calculate_hash, Error, PduEvent, Result}; +use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, PduEvent, Result}; #[derive(Debug)] enum TransactionStatus { diff --git a/src/service/services.rs b/src/service/services.rs index 6f07c1fe..4a3d6d2e 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use conduit::{debug_info, Result, Server}; -use database::KeyValueDatabase; +use database::Database; use tracing::{debug, info, trace}; use crate::{ @@ -24,11 +24,11 @@ pub struct Services { pub media: media::Service, pub sending: Arc, pub server: Arc, - pub db: Arc, + pub db: Arc, } impl Services { - pub async fn build(server: Arc, db: Arc) -> Result { + pub async fn build(server: Arc, db: Arc) -> Result { Ok(Self { rooms: rooms::Service { alias: rooms::alias::Service::build(&server, &db)?, diff --git a/src/service/transaction_ids/data.rs b/src/service/transaction_ids/data.rs index 5b29f211..668852ca 100644 --- a/src/service/transaction_ids/data.rs +++ b/src/service/transaction_ids/data.rs @@ -1,17 +1,17 @@ use std::sync::Arc; use conduit::Result; -use database::{KeyValueDatabase, KvTree}; +use database::{Database, Map}; use ruma::{DeviceId, TransactionId, UserId}; pub struct Data { - userdevicetxnid_response: Arc, + userdevicetxnid_response: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - userdevicetxnid_response: db.userdevicetxnid_response.clone(), + userdevicetxnid_response: db["userdevicetxnid_response"].clone(), } } diff --git a/src/service/transaction_ids/mod.rs b/src/service/transaction_ids/mod.rs index 5fb85d4a..5b94ef5b 100644 --- a/src/service/transaction_ids/mod.rs +++ b/src/service/transaction_ids/mod.rs @@ -1,21 +1,18 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; +use conduit::{Result, Server}; use data::Data; +use database::Database; use ruma::{DeviceId, TransactionId, UserId}; -use crate::Result; - pub struct Service { pub db: Data, } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/uiaa/data.rs b/src/service/uiaa/data.rs index b0710b6a..71d43dc2 100644 --- a/src/service/uiaa/data.rs +++ b/src/service/uiaa/data.rs @@ -1,21 +1,21 @@ use std::sync::Arc; use conduit::{Error, Result}; -use database::{KeyValueDatabase, KvTree}; +use database::{Database, Map}; use ruma::{ api::client::{error::ErrorKind, uiaa::UiaaInfo}, CanonicalJsonValue, DeviceId, UserId, }; pub struct Data { - userdevicesessionid_uiaainfo: Arc, - db: Arc, + userdevicesessionid_uiaainfo: Arc, + db: Arc, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(db: &Arc) -> Self { Self { - userdevicesessionid_uiaainfo: db.userdevicesessionid_uiaainfo.clone(), + userdevicesessionid_uiaainfo: db["userdevicesessionid_uiaainfo"].clone(), db: db.clone(), } } diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 4539c762..0d2f8bf7 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -1,12 +1,10 @@ -use conduit::Server; -use database::KeyValueDatabase; - mod data; use std::sync::Arc; -use conduit::{utils, utils::hash, Error, Result}; +use conduit::{utils, utils::hash, Error, Result, Server}; use data::Data; +use database::Database; use ruma::{ api::client::{ error::ErrorKind, @@ -25,7 +23,7 @@ pub struct Service { } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), }) diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 5644b3e3..cc40aa52 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -1,5 +1,7 @@ use std::{collections::BTreeMap, mem::size_of, sync::Arc}; +use conduit::{utils, warn, Error, Result}; +use database::{Database, Map}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, @@ -8,51 +10,50 @@ use ruma::{ uint, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedDeviceKeyId, OwnedMxcUri, OwnedUserId, UInt, UserId, }; -use tracing::warn; -use crate::{services, users::clean_signatures, utils, Error, KeyValueDatabase, KvTree, Result}; +use crate::{services, users::clean_signatures}; pub struct Data { - userid_password: Arc, - token_userdeviceid: Arc, - userid_displayname: Arc, - userid_avatarurl: Arc, - userid_blurhash: Arc, - userid_devicelistversion: Arc, - userdeviceid_token: Arc, - userdeviceid_metadata: Arc, - onetimekeyid_onetimekeys: Arc, - userid_lastonetimekeyupdate: Arc, - keyid_key: Arc, - userid_masterkeyid: Arc, - userid_selfsigningkeyid: Arc, - userid_usersigningkeyid: Arc, - keychangeid_userid: Arc, - todeviceid_events: Arc, - userfilterid_filter: Arc, - _db: Arc, + userid_password: Arc, + token_userdeviceid: Arc, + userid_displayname: Arc, + userid_avatarurl: Arc, + userid_blurhash: Arc, + userid_devicelistversion: Arc, + userdeviceid_token: Arc, + userdeviceid_metadata: Arc, + onetimekeyid_onetimekeys: Arc, + userid_lastonetimekeyupdate: Arc, + keyid_key: Arc, + userid_masterkeyid: Arc, + userid_selfsigningkeyid: Arc, + userid_usersigningkeyid: Arc, + keychangeid_userid: Arc, + todeviceid_events: Arc, + userfilterid_filter: Arc, + _db: Arc, } impl Data { - pub(super) fn new(db: Arc) -> Self { + pub(super) fn new(db: Arc) -> Self { Self { - userid_password: db.userid_password.clone(), - token_userdeviceid: db.token_userdeviceid.clone(), - userid_displayname: db.userid_displayname.clone(), - userid_avatarurl: db.userid_avatarurl.clone(), - userid_blurhash: db.userid_blurhash.clone(), - userid_devicelistversion: db.userid_devicelistversion.clone(), - userdeviceid_token: db.userdeviceid_token.clone(), - userdeviceid_metadata: db.userdeviceid_metadata.clone(), - onetimekeyid_onetimekeys: db.onetimekeyid_onetimekeys.clone(), - userid_lastonetimekeyupdate: db.userid_lastonetimekeyupdate.clone(), - keyid_key: db.keyid_key.clone(), - userid_masterkeyid: db.userid_masterkeyid.clone(), - userid_selfsigningkeyid: db.userid_selfsigningkeyid.clone(), - userid_usersigningkeyid: db.userid_usersigningkeyid.clone(), - keychangeid_userid: db.keychangeid_userid.clone(), - todeviceid_events: db.todeviceid_events.clone(), - userfilterid_filter: db.userfilterid_filter.clone(), + userid_password: db["userid_password"].clone(), + token_userdeviceid: db["token_userdeviceid"].clone(), + userid_displayname: db["userid_displayname"].clone(), + userid_avatarurl: db["userid_avatarurl"].clone(), + userid_blurhash: db["userid_blurhash"].clone(), + userid_devicelistversion: db["userid_devicelistversion"].clone(), + userdeviceid_token: db["userdeviceid_token"].clone(), + userdeviceid_metadata: db["userdeviceid_metadata"].clone(), + onetimekeyid_onetimekeys: db["onetimekeyid_onetimekeys"].clone(), + userid_lastonetimekeyupdate: db["userid_lastonetimekeyupdate"].clone(), + keyid_key: db["keyid_key"].clone(), + userid_masterkeyid: db["userid_masterkeyid"].clone(), + userid_selfsigningkeyid: db["userid_selfsigningkeyid"].clone(), + userid_usersigningkeyid: db["userid_usersigningkeyid"].clone(), + keychangeid_userid: db["keychangeid_userid"].clone(), + todeviceid_events: db["todeviceid_events"].clone(), + userfilterid_filter: db["userfilterid_filter"].clone(), _db: db, } } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index a5349d97..e7560436 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,13 +1,14 @@ -use conduit::Server; - mod data; + use std::{ collections::{BTreeMap, BTreeSet}, mem, sync::{Arc, Mutex, Mutex as StdMutex}, }; +use conduit::{Error, Result, Server}; use data::Data; +use database::Database; use ruma::{ api::client::{ device::Device, @@ -24,7 +25,7 @@ use ruma::{ UInt, UserId, }; -use crate::{database::KeyValueDatabase, service, services, Error, Result}; +use crate::services; pub struct SlidingSyncCache { lists: BTreeMap, @@ -41,7 +42,7 @@ pub struct Service { } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db.clone()), connections: StdMutex::new(BTreeMap::new()), @@ -242,7 +243,7 @@ impl Service { /// Check if a user is an admin pub fn is_admin(&self, user_id: &UserId) -> Result { - if let Some(admin_room_id) = service::admin::Service::get_admin_room()? { + if let Some(admin_room_id) = crate::admin::Service::get_admin_room()? { services() .rooms .state_cache From 30b5ad387041515cd3ea07fcf774e97d76330bb2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 29 Jun 2024 00:43:45 +0000 Subject: [PATCH 0281/2091] de-feature rocksdb --- src/admin/Cargo.toml | 1 - src/admin/server/commands.rs | 12 ------------ src/core/config/check.rs | 2 +- src/core/config/mod.rs | 17 ----------------- src/database/Cargo.toml | 4 ---- src/main/Cargo.toml | 4 ---- 6 files changed, 1 insertion(+), 39 deletions(-) diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 98e3a5b4..413a4613 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -24,7 +24,6 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] -rocksdb = [] [dependencies] clap.workspace = true diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index bd465e22..28962ef2 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -64,12 +64,6 @@ pub(super) async fn list_backups(_body: Vec<&str>) -> Result) -> Result { - if !cfg!(feature = "rocksdb") { - return Ok(RoomMessageEventContent::text_plain( - "Only RocksDB supports online backups in conduwuit.", - )); - } - let mut result = services() .server .runtime() @@ -88,12 +82,6 @@ pub(super) async fn backup_database(_body: Vec<&str>) -> Result) -> Result { - if !cfg!(feature = "rocksdb") { - return Ok(RoomMessageEventContent::text_plain( - "Only RocksDB supports listing files in conduwuit.", - )); - } - let result = services().globals.db.file_list()?; Ok(RoomMessageEventContent::notice_markdown(result)) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 403aa27c..9ee25962 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -66,7 +66,7 @@ pub fn check(config: &Config) -> Result<(), Error> { }); // rocksdb does not allow max_log_files to be 0 - if config.rocksdb_max_log_files == 0 && cfg!(feature = "rocksdb") { + if config.rocksdb_max_log_files == 0 { return Err(Error::bad_config( "When using RocksDB, rocksdb_max_log_files cannot be 0. Please set a value at least 1.", )); diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index bbf44a1a..636f0092 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -711,54 +711,37 @@ impl fmt::Display for Config { ("Gzip HTTP Compression", &self.gzip_compression.to_string()), #[cfg(feature = "brotli_compression")] ("Brotli HTTP Compression", &self.brotli_compression.to_string()), - #[cfg(feature = "rocksdb")] ("RocksDB database LOG level", &self.rocksdb_log_level), - #[cfg(feature = "rocksdb")] ("RocksDB database LOG to stderr", &self.rocksdb_log_stderr.to_string()), - #[cfg(feature = "rocksdb")] ("RocksDB database LOG time-to-roll", &self.rocksdb_log_time_to_roll.to_string()), - #[cfg(feature = "rocksdb")] ("RocksDB Max LOG Files", &self.rocksdb_max_log_files.to_string()), - #[cfg(feature = "rocksdb")] ( "RocksDB database max LOG file size", &self.rocksdb_max_log_file_size.to_string(), ), - #[cfg(feature = "rocksdb")] ( "RocksDB database optimize for spinning disks", &self.rocksdb_optimize_for_spinning_disks.to_string(), ), - #[cfg(feature = "rocksdb")] ("RocksDB Direct-IO", &self.rocksdb_direct_io.to_string()), - #[cfg(feature = "rocksdb")] ("RocksDB Parallelism Threads", &self.rocksdb_parallelism_threads.to_string()), - #[cfg(feature = "rocksdb")] ("RocksDB Compression Algorithm", &self.rocksdb_compression_algo), - #[cfg(feature = "rocksdb")] ("RocksDB Compression Level", &self.rocksdb_compression_level.to_string()), - #[cfg(feature = "rocksdb")] ( "RocksDB Bottommost Compression Level", &self.rocksdb_bottommost_compression_level.to_string(), ), - #[cfg(feature = "rocksdb")] ( "RocksDB Bottommost Level Compression", &self.rocksdb_bottommost_compression.to_string(), ), - #[cfg(feature = "rocksdb")] ("RocksDB Recovery Mode", &self.rocksdb_recovery_mode.to_string()), - #[cfg(feature = "rocksdb")] ("RocksDB Repair Mode", &self.rocksdb_repair.to_string()), - #[cfg(feature = "rocksdb")] ("RocksDB Read-only Mode", &self.rocksdb_read_only.to_string()), - #[cfg(feature = "rocksdb")] ( "RocksDB Compaction Idle Priority", &self.rocksdb_compaction_prio_idle.to_string(), ), - #[cfg(feature = "rocksdb")] ( "RocksDB Compaction Idle IOPriority", &self.rocksdb_compaction_ioprio_idle.to_string(), diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 1bc382a0..22858e01 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -24,9 +24,6 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] -rocksdb = [ - "dep:rust-rocksdb", -] jemalloc = [ "rust-rocksdb/jemalloc", ] @@ -43,7 +40,6 @@ conduit-core.workspace = true log.workspace = true lru-cache.workspace = true ruma.workspace = true -rust-rocksdb.optional = true rust-rocksdb.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index b2ead09e..0479311f 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -43,7 +43,6 @@ default = [ "io_uring", "jemalloc", "release_max_log_level", - "rocksdb", "sentry_telemetry", "systemd", "zstd_compression", @@ -114,9 +113,6 @@ release_max_log_level = [ "conduit-router/release_max_log_level", "conduit-service/release_max_log_level", ] -rocksdb = [ - "conduit-database/rocksdb", -] sentry_telemetry = [ "dep:sentry", "dep:sentry-tracing", From 0f1432f448f9569045b10f0c8481788e2d430444 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 29 Jun 2024 01:19:23 +0000 Subject: [PATCH 0282/2091] improve appservice::Data interface encap Signed-off-by: Jason Volk --- src/admin/query/appservice.rs | 2 +- src/admin/query/room_alias.rs | 6 +++--- src/service/appservice/data.rs | 13 ------------- src/service/appservice/mod.rs | 17 ++++++++++++++++- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 7331462a..77ac9641 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -22,7 +22,7 @@ pub(super) async fn appservice(subcommand: Appservice) -> Result { let timer = tokio::time::Instant::now(); - let results = services().appservice.db.all(); + let results = services().appservice.all(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index 3262303a..2d3b10ce 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -10,7 +10,7 @@ pub(super) async fn room_alias(subcommand: RoomAlias) -> Result { let timer = tokio::time::Instant::now(); - let results = services().rooms.alias.db.resolve_local_alias(&alias); + let results = services().rooms.alias.resolve_local_alias(&alias); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -21,7 +21,7 @@ pub(super) async fn room_alias(subcommand: RoomAlias) -> Result { let timer = tokio::time::Instant::now(); - let results = services().rooms.alias.db.local_aliases_for_room(&room_id); + let results = services().rooms.alias.local_aliases_for_room(&room_id); let query_time = timer.elapsed(); let aliases: Vec<_> = results.collect(); @@ -32,7 +32,7 @@ pub(super) async fn room_alias(subcommand: RoomAlias) -> Result { let timer = tokio::time::Instant::now(); - let results = services().rooms.alias.db.all_local_aliases(); + let results = services().rooms.alias.all_local_aliases(); let query_time = timer.elapsed(); let aliases: Vec<_> = results.collect(); diff --git a/src/service/appservice/data.rs b/src/service/appservice/data.rs index 52c89454..40e641a1 100644 --- a/src/service/appservice/data.rs +++ b/src/service/appservice/data.rs @@ -51,17 +51,4 @@ impl Data { .map_err(|_| Error::bad_database("Invalid id bytes in id_appserviceregistrations.")) }))) } - - pub fn all(&self) -> Result> { - self.iter_ids()? - .filter_map(Result::ok) - .map(move |id| { - Ok(( - id.clone(), - self.get_registration(&id)? - .expect("iter_ids only returns appservices that exist"), - )) - }) - .collect() - } } diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 05bc7675..916e7df3 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -124,7 +124,7 @@ impl Service { let mut registration_info = BTreeMap::new(); let db = Data::new(db); // Inserting registrations into cache - for appservice in db.all()? { + for appservice in iter_ids(&db)? { registration_info.insert( appservice.0, appservice @@ -140,6 +140,8 @@ impl Service { }) } + pub fn all(&self) -> Result> { iter_ids(&self.db) } + /// Registers an appservice and returns the ID to the caller pub async fn register_appservice(&self, yaml: Registration) -> Result { //TODO: Check for collisions between exclusive appservice namespaces @@ -231,3 +233,16 @@ impl Service { self.registration_info.read() } } + +fn iter_ids(db: &Data) -> Result> { + db.iter_ids()? + .filter_map(Result::ok) + .map(move |id| { + Ok(( + id.clone(), + db.get_registration(&id)? + .expect("iter_ids only returns appservices that exist"), + )) + }) + .collect() +} From dc633100da896bfd0a286b2a2e75312e0a7ea93e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 29 Jun 2024 03:25:13 +0000 Subject: [PATCH 0283/2091] hoist read_options/write_options instances Signed-off-by: Jason Volk --- src/database/map.rs | 88 ++++++++++++++++++++++----------------------- 1 file changed, 43 insertions(+), 45 deletions(-) diff --git a/src/database/map.rs b/src/database/map.rs index 591de04b..9229d7a9 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -9,6 +9,8 @@ pub struct Map { db: Arc, name: String, watchers: Watchers, + write_options: WriteOptions, + read_options: ReadOptions, } type Key = Vec; @@ -22,14 +24,16 @@ impl Map { db: db.clone(), name: name.to_owned(), watchers: Watchers::default(), + write_options: write_options_default(), + read_options: read_options_default(), })) } pub fn get(&self, key: &[u8]) -> Result>> { - let mut readoptions = ReadOptions::default(); - readoptions.set_total_order_seek(true); + let read_options = &self.read_options; + let res = self.db.db.get_cf_opt(&self.cf(), key, read_options); - result(self.db.db.get_cf_opt(&self.cf(), key, &readoptions)) + result(res) } pub fn multi_get(&self, keys: &[&[u8]]) -> Result>>> { @@ -37,14 +41,12 @@ impl Map { // comparator**. const SORTED: bool = false; - let mut readoptions = ReadOptions::default(); - readoptions.set_total_order_seek(true); - let mut ret: Vec>> = Vec::with_capacity(keys.len()); + let read_options = &self.read_options; for res in self .db .db - .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, &readoptions) + .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) { match res { Ok(Some(res)) => ret.push(Some((*res).to_vec())), @@ -57,11 +59,10 @@ impl Map { } pub fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { - let writeoptions = WriteOptions::default(); - + let write_options = &self.write_options; self.db .db - .put_cf_opt(&self.cf(), key, value, &writeoptions) + .put_cf_opt(&self.cf(), key, value, write_options) .or_else(or_else)?; if !self.db.corked() { @@ -74,15 +75,13 @@ impl Map { } pub fn insert_batch(&self, iter: &mut dyn Iterator) -> Result<()> { - let writeoptions = WriteOptions::default(); - let mut batch = WriteBatchWithTransaction::::default(); - for (key, value) in iter { batch.put_cf(&self.cf(), key, value); } - let res = self.db.db.write_opt(batch, &writeoptions); + let write_options = &self.write_options; + let res = self.db.db.write_opt(batch, write_options); if !self.db.corked() { self.db.flush()?; @@ -92,9 +91,8 @@ impl Map { } pub fn remove(&self, key: &[u8]) -> Result<()> { - let writeoptions = WriteOptions::default(); - - let res = self.db.db.delete_cf_opt(&self.cf(), key, &writeoptions); + let write_options = &self.write_options; + let res = self.db.db.delete_cf_opt(&self.cf(), key, write_options); if !self.db.corked() { self.db.flush()?; @@ -104,15 +102,13 @@ impl Map { } pub fn remove_batch(&self, iter: &mut dyn Iterator) -> Result<()> { - let writeoptions = WriteOptions::default(); - let mut batch = WriteBatchWithTransaction::::default(); - for key in iter { batch.delete_cf(&self.cf(), key); } - let res = self.db.db.write_opt(batch, &writeoptions); + let write_options = &self.write_options; + let res = self.db.db.write_opt(batch, write_options); if !self.db.corked() { self.db.flush()?; @@ -122,13 +118,11 @@ impl Map { } pub fn iter<'a>(&'a self) -> Box + 'a> { - let mut readoptions = ReadOptions::default(); - readoptions.set_total_order_seek(true); - + let read_options = read_options_default(); let it = self .db .db - .iterator_cf_opt(&self.cf(), readoptions, IteratorMode::Start) + .iterator_cf_opt(&self.cf(), read_options, IteratorMode::Start) .map(Result::unwrap) .map(|(k, v)| (Vec::from(k), Vec::from(v))); @@ -136,15 +130,13 @@ impl Map { } pub fn iter_from<'a>(&'a self, from: &[u8], backwards: bool) -> Box + 'a> { - let mut readoptions = ReadOptions::default(); - readoptions.set_total_order_seek(true); - + let read_options = read_options_default(); let it = self .db .db .iterator_cf_opt( &self.cf(), - readoptions, + read_options, IteratorMode::From( from, if backwards { @@ -161,19 +153,19 @@ impl Map { } pub fn increment(&self, key: &[u8]) -> Result> { - let mut readoptions = ReadOptions::default(); - readoptions.set_total_order_seek(true); - let writeoptions = WriteOptions::default(); - + let read_options = &self.read_options; let old = self .db .db - .get_cf_opt(&self.cf(), key, &readoptions) + .get_cf_opt(&self.cf(), key, read_options) .or_else(or_else)?; + let new = utils::increment(old.as_deref()); + + let write_options = &self.write_options; self.db .db - .put_cf_opt(&self.cf(), key, new, &writeoptions) + .put_cf_opt(&self.cf(), key, new, write_options) .or_else(or_else)?; if !self.db.corked() { @@ -184,25 +176,23 @@ impl Map { } pub fn increment_batch(&self, iter: &mut dyn Iterator) -> Result<()> { - let mut readoptions = ReadOptions::default(); - readoptions.set_total_order_seek(true); - let writeoptions = WriteOptions::default(); - let mut batch = WriteBatchWithTransaction::::default(); + let read_options = &self.read_options; for key in iter { let old = self .db .db - .get_cf_opt(&self.cf(), &key, &readoptions) + .get_cf_opt(&self.cf(), &key, read_options) .or_else(or_else)?; let new = utils::increment(old.as_deref()); batch.put_cf(&self.cf(), key, new); } + let write_options = &self.write_options; self.db .db - .write_opt(batch, &writeoptions) + .write_opt(batch, write_options) .or_else(or_else)?; if !self.db.corked() { @@ -213,13 +203,11 @@ impl Map { } pub fn scan_prefix<'a>(&'a self, prefix: Vec) -> Box + 'a> { - let mut readoptions = ReadOptions::default(); - readoptions.set_total_order_seek(true); - + let read_options = read_options_default(); let it = self .db .db - .iterator_cf_opt(&self.cf(), readoptions, IteratorMode::From(&prefix, Direction::Forward)) + .iterator_cf_opt(&self.cf(), read_options, IteratorMode::From(&prefix, Direction::Forward)) .map(Result::unwrap) .map(|(k, v)| (Vec::from(k), Vec::from(v))) .take_while(move |(k, _)| k.starts_with(&prefix)); @@ -240,3 +228,13 @@ impl<'a> IntoIterator for &'a Map { fn into_iter(self) -> Self::IntoIter { self.iter() } } + +#[inline] +fn read_options_default() -> ReadOptions { + let mut read_options = ReadOptions::default(); + read_options.set_total_order_seek(true); + read_options +} + +#[inline] +fn write_options_default() -> WriteOptions { WriteOptions::default() } From df7a1dee1524b88bcd2330334f951274d236eefe Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 29 Jun 2024 04:18:07 +0000 Subject: [PATCH 0284/2091] remove --features=rocksdb from smoketest; use separate target dir Signed-off-by: Jason Volk --- tests/cargo_smoke.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/cargo_smoke.sh b/tests/cargo_smoke.sh index e0eb3ad3..6ee20ee2 100755 --- a/tests/cargo_smoke.sh +++ b/tests/cargo_smoke.sh @@ -1,10 +1,12 @@ #!/bin/bash +export CARGO_TARGET_DIR="target/smokes" run () { RUN_COMMAND=$@ - echo -e "\033[1;33mTEST\033[0m $RUN_COMMAND" + echo -ne "\033[1;33mTEST\033[0m $RUN_COMMAND" ERRORS=$($RUN_COMMAND 2>&1>/tmp/uwu_smoketest.out) RESULT=$? + echo if test $RESULT -ne 0; then cat /tmp/uwu_smoketest.out echo -e "$ERRORS" @@ -44,17 +46,17 @@ element () { vector () { TOOLCHAIN=$1; shift VECTOR_OPTS=$@ - element "$TOOLCHAIN" $VECTOR_OPTS --no-default-features --features="rocksdb" + element "$TOOLCHAIN" $VECTOR_OPTS --no-default-features element "$TOOLCHAIN" $VECTOR_OPTS --features=default element "$TOOLCHAIN" $VECTOR_OPTS --all-features } matrix () { run cargo +nightly fmt --all --check - vector +nightly --profile=dev - vector +nightly --profile=release vector "" --profile=dev vector "" --profile=release + vector +nightly --profile=dev + vector +nightly --profile=release } BUILD=${1:-build} From 7db8478dc09476b9cf9787ad832b10547d0c7846 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 29 Jun 2024 05:41:20 +0000 Subject: [PATCH 0285/2091] hoist iterator modes; group fns Signed-off-by: Jason Volk --- src/database/map.rs | 46 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 25 deletions(-) diff --git a/src/database/map.rs b/src/database/map.rs index 9229d7a9..0a628f9c 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -130,28 +130,37 @@ impl Map { } pub fn iter_from<'a>(&'a self, from: &[u8], backwards: bool) -> Box + 'a> { + let direction = if backwards { + Direction::Reverse + } else { + Direction::Forward + }; + let mode = IteratorMode::From(from, direction); let read_options = read_options_default(); let it = self .db .db - .iterator_cf_opt( - &self.cf(), - read_options, - IteratorMode::From( - from, - if backwards { - Direction::Reverse - } else { - Direction::Forward - }, - ), - ) + .iterator_cf_opt(&self.cf(), read_options, mode) .map(Result::unwrap) .map(|(k, v)| (Vec::from(k), Vec::from(v))); Box::new(it) } + pub fn scan_prefix<'a>(&'a self, prefix: Vec) -> Box + 'a> { + let mode = IteratorMode::From(&prefix, Direction::Forward); + let read_options = read_options_default(); + let it = self + .db + .db + .iterator_cf_opt(&self.cf(), read_options, mode) + .map(Result::unwrap) + .map(|(k, v)| (Vec::from(k), Vec::from(v))) + .take_while(move |(k, _)| k.starts_with(&prefix)); + + Box::new(it) + } + pub fn increment(&self, key: &[u8]) -> Result> { let read_options = &self.read_options; let old = self @@ -202,19 +211,6 @@ impl Map { Ok(()) } - pub fn scan_prefix<'a>(&'a self, prefix: Vec) -> Box + 'a> { - let read_options = read_options_default(); - let it = self - .db - .db - .iterator_cf_opt(&self.cf(), read_options, IteratorMode::From(&prefix, Direction::Forward)) - .map(Result::unwrap) - .map(|(k, v)| (Vec::from(k), Vec::from(v))) - .take_while(move |(k, _)| k.starts_with(&prefix)); - - Box::new(it) - } - pub fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { self.watchers.watch(prefix) } From 35049c94de17b1a4b153309026a292842c24e742 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 29 Jun 2024 19:57:22 +0000 Subject: [PATCH 0286/2091] simplify db ThreadMode typedef Signed-off-by: Jason Volk --- src/database/engine.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 396d327d..c8d70304 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -9,7 +9,7 @@ use conduit::{debug, error, info, warn, Result, Server}; use rocksdb::{ backup::{BackupEngine, BackupEngineOptions}, perf::get_memory_usage_stats, - BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode as Db, Env, MultiThreaded, Options, + BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode, Env, MultiThreaded, Options, }; use crate::{ @@ -24,10 +24,12 @@ pub struct Engine { opts: Options, env: Env, cfs: Mutex>, - pub(crate) db: Db, + pub(crate) db: Db, corks: AtomicU32, } +pub(crate) type Db = DBWithThreadMode; + impl Engine { pub(crate) fn open(server: &Arc) -> Result> { let config = &server.config; @@ -49,13 +51,13 @@ impl Engine { let load_time = std::time::Instant::now(); if config.rocksdb_repair { warn!("Starting database repair. This may take a long time..."); - if let Err(e) = Db::::repair(&db_opts, &config.database_path) { + if let Err(e) = Db::repair(&db_opts, &config.database_path) { error!("Repair failed: {:?}", e); } } debug!("Listing column families in database"); - let cfs = Db::::list_cf(&db_opts, &config.database_path).unwrap_or_default(); + let cfs = Db::list_cf(&db_opts, &config.database_path).unwrap_or_default(); debug!("Opening {} column family descriptors in database", cfs.len()); let cfds = cfs @@ -65,9 +67,9 @@ impl Engine { debug!("Opening database..."); let res = if config.rocksdb_read_only { - Db::::open_cf_for_read_only(&db_opts, &config.database_path, cfs.clone(), false) + Db::open_cf_for_read_only(&db_opts, &config.database_path, cfs.clone(), false) } else { - Db::::open_cf_descriptors(&db_opts, &config.database_path, cfds) + Db::open_cf_descriptors(&db_opts, &config.database_path, cfds) }; let db = res.or_else(or_else)?; From 7638bbc49c5f1bcc7417796b3c5688b27be9dd4e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 29 Jun 2024 22:07:57 +0000 Subject: [PATCH 0287/2091] keep column list lexically sorted Signed-off-by: Jason Volk --- src/database/engine.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index c8d70304..40a5438c 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -1,5 +1,5 @@ use std::{ - collections::{HashMap, HashSet}, + collections::{BTreeSet, HashMap}, fmt::Write, sync::{atomic::AtomicU32, Arc, Mutex, RwLock}, }; @@ -23,7 +23,7 @@ pub struct Engine { col_cache: RwLock>, opts: Options, env: Env, - cfs: Mutex>, + cfs: Mutex>, pub(crate) db: Db, corks: AtomicU32, } @@ -57,7 +57,10 @@ impl Engine { } debug!("Listing column families in database"); - let cfs = Db::list_cf(&db_opts, &config.database_path).unwrap_or_default(); + let cfs = Db::list_cf(&db_opts, &config.database_path) + .unwrap_or_default() + .into_iter() + .collect::>(); debug!("Opening {} column family descriptors in database", cfs.len()); let cfds = cfs @@ -79,7 +82,6 @@ impl Engine { load_time.elapsed() ); - let cfs = HashSet::::from_iter(cfs); Ok(Arc::new(Self { server: server.clone(), row_cache, From 3a51e18ce68d39a541977c62447654f99d681bf6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 24 Jun 2024 22:06:20 +0000 Subject: [PATCH 0288/2091] pre-format version strings Signed-off-by: Jason Volk --- src/api/client/unversioned.rs | 6 ++--- src/api/server/version.rs | 4 ++-- src/core/mod.rs | 1 + src/core/version.rs | 43 +++++++++++++++++++---------------- src/main/clap.rs | 2 +- src/main/server.rs | 2 +- src/service/globals/client.rs | 6 +---- 7 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 56ab9a90..2b703f94 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -145,7 +145,7 @@ pub(crate) async fn syncv3_client_server_json() -> Result { Ok(Json(serde_json::json!({ "server": server_url, - "version": conduit::version::conduwuit(), + "version": conduit::version(), }))) } @@ -155,8 +155,8 @@ pub(crate) async fn syncv3_client_server_json() -> Result { /// `/_matrix/federation/v1/version` pub(crate) async fn conduwuit_server_version() -> Result { Ok(Json(serde_json::json!({ - "name": "conduwuit", - "version": conduit::version::conduwuit(), + "name": conduit::version::name(), + "version": conduit::version::version(), }))) } diff --git a/src/api/server/version.rs b/src/api/server/version.rs index 557b10dc..92837571 100644 --- a/src/api/server/version.rs +++ b/src/api/server/version.rs @@ -10,8 +10,8 @@ pub(crate) async fn get_server_version_route( ) -> Result { Ok(get_server_version::v1::Response { server: Some(get_server_version::v1::Server { - name: Some("Conduwuit".to_owned()), - version: Some(conduit::version::conduwuit()), + name: Some(conduit::version::name().into()), + version: Some(conduit::version::version().into()), }), }) } diff --git a/src/core/mod.rs b/src/core/mod.rs index 383b45ee..5ffe4cb9 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -13,6 +13,7 @@ pub use config::Config; pub use error::{Error, RumaResponse}; pub use pducount::PduCount; pub use server::Server; +pub use version::version; pub type Result = std::result::Result; diff --git a/src/core/version.rs b/src/core/version.rs index f65bac99..4a4e8426 100644 --- a/src/core/version.rs +++ b/src/core/version.rs @@ -4,25 +4,28 @@ /// Set the environment variable `CONDUWUIT_VERSION_EXTRA` to any UTF-8 string /// to include it in parenthesis after the SemVer version. A common value are /// git commit hashes. +use std::sync::OnceLock; + +static BRANDING: &str = "Conduwuit"; +static SEMANTIC: &str = env!("CARGO_PKG_VERSION"); + +static VERSION: OnceLock = OnceLock::new(); +static USER_AGENT: OnceLock = OnceLock::new(); + +#[inline] #[must_use] -pub fn conduwuit() -> String { - match option_env!("CONDUWUIT_VERSION_EXTRA") { - Some(extra) => { - if extra.is_empty() { - env!("CARGO_PKG_VERSION").to_owned() - } else { - format!("{} ({})", env!("CARGO_PKG_VERSION"), extra) - } - }, - None => match option_env!("CONDUIT_VERSION_EXTRA") { - Some(extra) => { - if extra.is_empty() { - env!("CARGO_PKG_VERSION").to_owned() - } else { - format!("{} ({})", env!("CARGO_PKG_VERSION"), extra) - } - }, - None => env!("CARGO_PKG_VERSION").to_owned(), - }, - } +pub fn name() -> &'static str { BRANDING } + +#[inline] +pub fn version() -> &'static str { VERSION.get_or_init(init_version) } + +#[inline] +pub fn user_agent() -> &'static str { USER_AGENT.get_or_init(init_user_agent) } + +fn init_user_agent() -> String { format!("{}/{}", name(), version()) } + +fn init_version() -> String { + option_env!("CONDUWUIT_VERSION_EXTRA") + .or(option_env!("CONDUIT_VERSION_EXTRA")) + .map_or(SEMANTIC.to_owned(), |extra| format!("{BRANDING} ({extra})")) } diff --git a/src/main/clap.rs b/src/main/clap.rs index a2fb588e..6ce164f5 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -6,7 +6,7 @@ use clap::Parser; /// Commandline arguments #[derive(Parser, Debug)] -#[clap(version = conduit::version::conduwuit(), about, long_about = None)] +#[clap(version = conduit::version(), about, long_about = None)] pub(crate) struct Args { #[arg(short, long)] /// Optional argument to the path of a conduwuit config TOML file diff --git a/src/main/server.rs b/src/main/server.rs index ac93da17..456a8756 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -46,7 +46,7 @@ impl Server { database_path = ?config.database_path, log_levels = %config.log, "{}", - conduit::version::conduwuit(), + conduit::version(), ); Ok(Arc::new(Self { diff --git a/src/service/globals/client.rs b/src/service/globals/client.rs index 7c63618e..1986d7d6 100644 --- a/src/service/globals/client.rs +++ b/src/service/globals/client.rs @@ -87,10 +87,6 @@ impl Client { } fn base(config: &Config) -> Result { - let version = conduit::version::conduwuit(); - - let user_agent = format!("Conduwuit/{version}"); - let mut builder = reqwest::Client::builder() .hickory_dns(true) .connect_timeout(Duration::from_secs(config.request_conn_timeout)) @@ -98,7 +94,7 @@ impl Client { .timeout(Duration::from_secs(config.request_total_timeout)) .pool_idle_timeout(Duration::from_secs(config.request_idle_timeout)) .pool_max_idle_per_host(config.request_idle_per_host.into()) - .user_agent(user_agent) + .user_agent(conduit::version::user_agent()) .redirect(redirect::Policy::limited(6)) .connection_verbose(true); From 0e46b750a04578dca87fb71cfea466fbcf551e60 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 22 Jun 2024 19:28:26 +0000 Subject: [PATCH 0289/2091] default to jemalloc when both hardened_malloc and jemalloc enabled. Signed-off-by: Jason Volk --- src/core/alloc/hardened.rs | 2 ++ src/core/alloc/je.rs | 2 ++ src/core/alloc/mod.rs | 26 ++++++++------------------ src/core/config/check.rs | 5 +---- 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/src/core/alloc/hardened.rs b/src/core/alloc/hardened.rs index 4c9563cf..6727407f 100644 --- a/src/core/alloc/hardened.rs +++ b/src/core/alloc/hardened.rs @@ -1,3 +1,5 @@ +//! hmalloc allocator + #[global_allocator] static HMALLOC: hardened_malloc_rs::HardenedMalloc = hardened_malloc_rs::HardenedMalloc; diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index ebf3575f..e0628ef3 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -1,3 +1,5 @@ +//! jemalloc allocator + use std::ffi::{c_char, c_void}; use tikv_jemalloc_ctl as mallctl; diff --git a/src/core/alloc/mod.rs b/src/core/alloc/mod.rs index 4c42b1e6..7941a99c 100644 --- a/src/core/alloc/mod.rs +++ b/src/core/alloc/mod.rs @@ -1,27 +1,17 @@ //! Integration with allocators // jemalloc -#[cfg(all(not(target_env = "msvc"), feature = "jemalloc", not(feature = "hardened_malloc")))] +#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] pub mod je; -#[cfg(all(not(target_env = "msvc"), feature = "jemalloc", not(feature = "hardened_malloc")))] +#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] pub use je::{memory_stats, memory_usage}; -// hardened_malloc -//#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", target_os = -//#[cfg(all(not(target_env "linux", not(feature = "jemalloc")))] -//pub mod hardened; -//#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", target_os = -//#[cfg(all(not(target_env "linux", not(feature = "jemalloc")))] -//pub use hardened::{memory_stats, memory_usage}; +#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", not(feature = "jemalloc")))] +pub mod hardened; +#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", not(feature = "jemalloc")))] +pub use hardened::{memory_stats, memory_usage}; -// default, enabled when none or multiple of the above are enabled -#[cfg(any( - not(any(feature = "jemalloc", feature = "hardened_malloc")), - all(feature = "jemalloc", feature = "hardened_malloc"), -))] +#[cfg(any(target_env = "msvc", all(not(feature = "hardened_malloc"), not(feature = "jemalloc"))))] pub mod default; -#[cfg(any( - not(any(feature = "jemalloc", feature = "hardened_malloc")), - all(feature = "jemalloc", feature = "hardened_malloc"), -))] +#[cfg(any(target_env = "msvc", all(not(feature = "hardened_malloc"), not(feature = "jemalloc"))))] pub use default::{memory_stats, memory_usage}; diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 9ee25962..24a9ac9c 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -14,10 +14,7 @@ pub fn check(config: &Config) -> Result<(), Error> { } if cfg!(feature = "hardened_malloc") && cfg!(feature = "jemalloc") { - warn!( - "hardened_malloc and jemalloc were built together, this causes neither to be used. Conduwuit will still \ - function, but consider rebuilding and pick one as this is now no-op." - ); + warn!("hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used."); } if config.unix_socket_path.is_some() && !cfg!(unix) { From c05f00661b136ada748cfdd04ad01d90505faaba Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 24 Jun 2024 22:20:07 +0000 Subject: [PATCH 0290/2091] use extern namespace in log wrapper macros convert use tracing statements Signed-off-by: Jason Volk --- src/api/client/state.rs | 2 +- src/core/debug.rs | 10 +++++----- src/core/log/mod.rs | 10 +++++----- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 17ae7be4..9247b123 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use conduit::{error, warn}; use ruma::{ api::client::{ error::ErrorKind, @@ -16,7 +17,6 @@ use ruma::{ serde::Raw, EventId, RoomId, UserId, }; -use tracing::{error, log::warn}; use crate::{ service::{pdu::PduBuilder, server_is_ours}, diff --git a/src/core/debug.rs b/src/core/debug.rs index ad12099b..7522aa77 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -14,9 +14,9 @@ pub use crate::utils::debug::*; macro_rules! debug_event { ( $level:expr, $($x:tt)+ ) => { if cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) { - tracing::event!( $level, $($x)+ ); + ::tracing::event!( $level, $($x)+ ); } else { - tracing::debug!( $($x)+ ); + ::tracing::debug!( $($x)+ ); } } } @@ -27,7 +27,7 @@ macro_rules! debug_event { #[macro_export] macro_rules! debug_error { ( $($x:tt)+ ) => { - $crate::debug_event!(tracing::Level::ERROR, $($x)+ ); + $crate::debug_event!(::tracing::Level::ERROR, $($x)+ ); } } @@ -37,7 +37,7 @@ macro_rules! debug_error { #[macro_export] macro_rules! debug_warn { ( $($x:tt)+ ) => { - $crate::debug_event!(tracing::Level::WARN, $($x)+ ); + $crate::debug_event!(::tracing::Level::WARN, $($x)+ ); } } @@ -47,7 +47,7 @@ macro_rules! debug_warn { #[macro_export] macro_rules! debug_info { ( $($x:tt)+ ) => { - $crate::debug_event!(tracing::Level::INFO, $($x)+ ); + $crate::debug_event!(::tracing::Level::INFO, $($x)+ ); } } diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 97da6b40..49efe56d 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -20,25 +20,25 @@ pub use tracing_subscriber::EnvFilter; #[macro_export] macro_rules! error { - ( $($x:tt)+ ) => { tracing::error!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::error!( $($x)+ ); } } #[macro_export] macro_rules! warn { - ( $($x:tt)+ ) => { tracing::warn!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::warn!( $($x)+ ); } } #[macro_export] macro_rules! info { - ( $($x:tt)+ ) => { tracing::info!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::info!( $($x)+ ); } } #[macro_export] macro_rules! debug { - ( $($x:tt)+ ) => { tracing::debug!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::debug!( $($x)+ ); } } #[macro_export] macro_rules! trace { - ( $($x:tt)+ ) => { tracing::trace!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::trace!( $($x)+ ); } } From 703c2752666060fabec3b1241a0e44c59c227ea7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 24 Jun 2024 22:10:34 +0000 Subject: [PATCH 0291/2091] split sentry init; add user-agent, trace hooks. Signed-off-by: Jason Volk --- src/main/main.rs | 1 + src/main/sentry.rs | 45 +++++++++++++++++++++++++++++++++++++++++++++ src/main/server.rs | 35 ++++------------------------------- 3 files changed, 50 insertions(+), 31 deletions(-) create mode 100644 src/main/sentry.rs diff --git a/src/main/main.rs b/src/main/main.rs index 1bb46c8e..d579cd4a 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -1,6 +1,7 @@ pub(crate) mod clap; mod mods; mod restart; +mod sentry; mod server; mod signal; diff --git a/src/main/sentry.rs b/src/main/sentry.rs new file mode 100644 index 00000000..6ed4bb8a --- /dev/null +++ b/src/main/sentry.rs @@ -0,0 +1,45 @@ +#![cfg(feature = "sentry_telemetry")] + +use std::{str::FromStr, sync::Arc}; + +use conduit::{config::Config, trace}; +use sentry::{ + types::{protocol::v7::Event, Dsn}, + Breadcrumb, ClientOptions, +}; + +pub(crate) fn init(config: &Config) -> Option { + config.sentry.then(|| sentry::init(options(config))) +} + +fn options(config: &Config) -> ClientOptions { + let dsn = config + .sentry_endpoint + .as_ref() + .expect("init_sentry should only be called if sentry is enabled and this is not None") + .as_str(); + + ClientOptions { + dsn: Some(Dsn::from_str(dsn).expect("sentry_endpoint must be a valid URL")), + server_name: config + .sentry_send_server_name + .then(|| config.server_name.to_string().into()), + traces_sample_rate: config.sentry_traces_sample_rate, + debug: cfg!(debug_assertions), + release: sentry::release_name!(), + user_agent: conduit::version::user_agent().into(), + before_send: Some(Arc::new(before_send)), + before_breadcrumb: Some(Arc::new(before_breadcrumb)), + ..Default::default() + } +} + +fn before_send(event: Event<'static>) -> Option> { + trace!("Sending sentry event: {event:?}"); + Some(event) +} + +fn before_breadcrumb(crumb: Breadcrumb) -> Option { + trace!("Adding sentry breadcrumb: {crumb:?}"); + Some(crumb) +} diff --git a/src/main/server.rs b/src/main/server.rs index 456a8756..064357c1 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -21,7 +21,7 @@ pub(crate) struct Server { _tracing_flame_guard: TracingFlameGuard, #[cfg(feature = "sentry_telemetry")] - _sentry_guard: Option, + _sentry_guard: Option<::sentry::ClientInitGuard>, #[cfg(conduit_mods)] // Module instances; TODO: move to mods::loaded mgmt vector @@ -33,10 +33,12 @@ impl Server { let config = Config::new(args.config)?; #[cfg(feature = "sentry_telemetry")] - let sentry_guard = init_sentry(&config); + let sentry_guard = crate::sentry::init(&config); + let (tracing_reload_handle, tracing_flame_guard, capture) = init_tracing(&config); config.check()?; + #[cfg(unix)] sys::maximize_fd_limit().expect("Unable to increase maximum soft and hard file descriptor limit"); hash::init(); @@ -70,35 +72,6 @@ impl Server { } } -#[cfg(feature = "sentry_telemetry")] -fn init_sentry(config: &Config) -> Option { - if !config.sentry { - return None; - } - - let sentry_endpoint = config - .sentry_endpoint - .as_ref() - .expect("init_sentry should only be called if sentry is enabled and this is not None") - .as_str(); - - let server_name = if config.sentry_send_server_name { - Some(config.server_name.to_string().into()) - } else { - None - }; - - Some(sentry::init(( - sentry_endpoint, - sentry::ClientOptions { - release: sentry::release_name!(), - traces_sample_rate: config.sentry_traces_sample_rate, - server_name, - ..Default::default() - }, - ))) -} - #[cfg(feature = "perf_measurements")] type TracingFlameGuard = Option>>; #[cfg(not(feature = "perf_measurements"))] From bd19fac4c523f2d9437f632b5caa5b808c182aa5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 24 Jun 2024 22:22:18 +0000 Subject: [PATCH 0292/2091] split tracing init into unit Signed-off-by: Jason Volk --- src/main/main.rs | 2 +- src/main/server.rs | 109 ++------------------------------------------ src/main/tracing.rs | 107 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 111 insertions(+), 107 deletions(-) create mode 100644 src/main/tracing.rs diff --git a/src/main/main.rs b/src/main/main.rs index d579cd4a..23f53b4e 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -4,6 +4,7 @@ mod restart; mod sentry; mod server; mod signal; +mod tracing; extern crate conduit_core as conduit; @@ -53,7 +54,6 @@ fn main() -> Result<(), Error> { #[cfg(not(conduit_mods))] async fn async_main(server: &Arc) -> Result<(), Error> { extern crate conduit_router as router; - use tracing::error; if let Err(error) = router::start(&server.server).await { error!("Critical error starting server: {error}"); diff --git a/src/main/server.rs b/src/main/server.rs index 064357c1..08d81d01 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,17 +1,15 @@ use std::sync::Arc; use conduit::{ - config, config::Config, info, - log::{self, capture, LogLevelReloadHandles, ReloadHandle}, + log::{self}, utils::{hash, sys}, Error, Result, }; use tokio::runtime; -use tracing_subscriber::{prelude::*, reload, EnvFilter, Registry}; -use crate::clap::Args; +use crate::{clap::Args, tracing::TracingFlameGuard}; /// Server runtime state; complete pub(crate) struct Server { @@ -35,7 +33,7 @@ impl Server { #[cfg(feature = "sentry_telemetry")] let sentry_guard = crate::sentry::init(&config); - let (tracing_reload_handle, tracing_flame_guard, capture) = init_tracing(&config); + let (tracing_reload_handle, tracing_flame_guard, capture) = crate::tracing::init(&config); config.check()?; @@ -71,104 +69,3 @@ impl Server { })) } } - -#[cfg(feature = "perf_measurements")] -type TracingFlameGuard = Option>>; -#[cfg(not(feature = "perf_measurements"))] -type TracingFlameGuard = (); - -// clippy thinks the filter_layer clones are redundant if the next usage is -// behind a disabled feature. -#[allow(clippy::redundant_clone)] -fn init_tracing(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard, Arc) { - let registry = Registry::default(); - let fmt_layer = tracing_subscriber::fmt::Layer::new(); - let filter_layer = match EnvFilter::try_new(&config.log) { - Ok(s) => s, - Err(e) => { - eprintln!("It looks like your config is invalid. The following error occured while parsing it: {e}"); - EnvFilter::try_new(config::default_log()).expect("failed to set default EnvFilter") - }, - }; - - let mut reload_handles = Vec:: + Send + Sync>>::new(); - let subscriber = registry; - - #[cfg(all(feature = "tokio_console", tokio_unstable))] - let subscriber = { - let console_layer = console_subscriber::spawn(); - subscriber.with(console_layer) - }; - - let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.push(Box::new(fmt_reload_handle)); - - let cap_state = Arc::new(capture::State::new()); - let cap_layer = capture::Layer::new(&cap_state); - let subscriber = subscriber - .with(fmt_layer.with_filter(fmt_reload_filter)) - .with(cap_layer); - - #[cfg(feature = "sentry_telemetry")] - let subscriber = { - let sentry_layer = sentry_tracing::layer(); - let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.push(Box::new(sentry_reload_handle)); - subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) - }; - - #[cfg(feature = "perf_measurements")] - let (subscriber, flame_guard) = { - let (flame_layer, flame_guard) = if config.tracing_flame { - let flame_filter = match EnvFilter::try_new(&config.tracing_flame_filter) { - Ok(flame_filter) => flame_filter, - Err(e) => panic!("tracing_flame_filter config value is invalid: {e}"), - }; - - let (flame_layer, flame_guard) = - match tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) { - Ok(ok) => ok, - Err(e) => panic!("failed to initialize tracing-flame: {e}"), - }; - let flame_layer = flame_layer - .with_empty_samples(false) - .with_filter(flame_filter); - (Some(flame_layer), Some(flame_guard)) - } else { - (None, None) - }; - - let jaeger_layer = if config.allow_jaeger { - opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); - let tracer = opentelemetry_jaeger::new_agent_pipeline() - .with_auto_split_batch(true) - .with_service_name("conduwuit") - .install_batch(opentelemetry_sdk::runtime::Tokio) - .unwrap(); - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - - let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(filter_layer); - reload_handles.push(Box::new(jaeger_reload_handle)); - Some(telemetry.with_filter(jaeger_reload_filter)) - } else { - None - }; - - let subscriber = subscriber.with(flame_layer).with(jaeger_layer); - (subscriber, flame_guard) - }; - - #[cfg(not(feature = "perf_measurements"))] - #[cfg_attr(not(feature = "perf_measurements"), allow(clippy::let_unit_value))] - let flame_guard = (); - - tracing::subscriber::set_global_default(subscriber).expect("failed to set global tracing subscriber"); - - #[cfg(all(feature = "tokio_console", feature = "release_max_log_level", tokio_unstable))] - tracing::error!( - "'tokio_console' feature and 'release_max_log_level' feature are incompatible, because console-subscriber \ - needs access to trace-level events. 'release_max_log_level' must be disabled to use tokio-console." - ); - - (LogLevelReloadHandles::new(reload_handles), flame_guard, cap_state) -} diff --git a/src/main/tracing.rs b/src/main/tracing.rs new file mode 100644 index 00000000..b5f7a7d1 --- /dev/null +++ b/src/main/tracing.rs @@ -0,0 +1,107 @@ +use std::sync::Arc; + +use conduit::{ + config, + config::Config, + log::{capture, LogLevelReloadHandles, ReloadHandle}, +}; +use tracing_subscriber::{prelude::*, reload, EnvFilter, Registry}; + +#[cfg(feature = "perf_measurements")] +pub(crate) type TracingFlameGuard = Option>>; +#[cfg(not(feature = "perf_measurements"))] +pub(crate) type TracingFlameGuard = (); + +#[allow(clippy::redundant_clone)] +pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard, Arc) { + let registry = Registry::default(); + let fmt_layer = tracing_subscriber::fmt::Layer::new(); + let filter_layer = match EnvFilter::try_new(&config.log) { + Ok(s) => s, + Err(e) => { + eprintln!("It looks like your config is invalid. The following error occured while parsing it: {e}"); + EnvFilter::try_new(config::default_log()).expect("failed to set default EnvFilter") + }, + }; + + let mut reload_handles = Vec:: + Send + Sync>>::new(); + let subscriber = registry; + + #[cfg(all(feature = "tokio_console", tokio_unstable))] + let subscriber = { + let console_layer = console_subscriber::spawn(); + subscriber.with(console_layer) + }; + + let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); + reload_handles.push(Box::new(fmt_reload_handle)); + + let cap_state = Arc::new(capture::State::new()); + let cap_layer = capture::Layer::new(&cap_state); + let subscriber = subscriber + .with(fmt_layer.with_filter(fmt_reload_filter)) + .with(cap_layer); + + #[cfg(feature = "sentry_telemetry")] + let subscriber = { + let sentry_layer = sentry_tracing::layer(); + let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer.clone()); + reload_handles.push(Box::new(sentry_reload_handle)); + subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) + }; + + #[cfg(feature = "perf_measurements")] + let (subscriber, flame_guard) = { + let (flame_layer, flame_guard) = if config.tracing_flame { + let flame_filter = match EnvFilter::try_new(&config.tracing_flame_filter) { + Ok(flame_filter) => flame_filter, + Err(e) => panic!("tracing_flame_filter config value is invalid: {e}"), + }; + + let (flame_layer, flame_guard) = + match tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) { + Ok(ok) => ok, + Err(e) => panic!("failed to initialize tracing-flame: {e}"), + }; + let flame_layer = flame_layer + .with_empty_samples(false) + .with_filter(flame_filter); + (Some(flame_layer), Some(flame_guard)) + } else { + (None, None) + }; + + let jaeger_layer = if config.allow_jaeger { + opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); + let tracer = opentelemetry_jaeger::new_agent_pipeline() + .with_auto_split_batch(true) + .with_service_name("conduwuit") + .install_batch(opentelemetry_sdk::runtime::Tokio) + .unwrap(); + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + + let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(filter_layer.clone()); + reload_handles.push(Box::new(jaeger_reload_handle)); + Some(telemetry.with_filter(jaeger_reload_filter)) + } else { + None + }; + + let subscriber = subscriber.with(flame_layer).with(jaeger_layer); + (subscriber, flame_guard) + }; + + #[cfg(not(feature = "perf_measurements"))] + #[cfg_attr(not(feature = "perf_measurements"), allow(clippy::let_unit_value))] + let flame_guard = (); + + tracing::subscriber::set_global_default(subscriber).expect("failed to set global tracing subscriber"); + + #[cfg(all(feature = "tokio_console", feature = "release_max_log_level", tokio_unstable))] + tracing::error!( + "'tokio_console' feature and 'release_max_log_level' feature are incompatible, because console-subscriber \ + needs access to trace-level events. 'release_max_log_level' must be disabled to use tokio-console." + ); + + (LogLevelReloadHandles::new(reload_handles), flame_guard, cap_state) +} From 06f01d384f6ef1e5279a04a18bee8a8087245dac Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 25 Jun 2024 00:03:05 +0000 Subject: [PATCH 0293/2091] simplify argon init Signed-off-by: Jason Volk --- src/core/utils/hash.rs | 48 +++++++++++++++++------------------------- src/main/server.rs | 3 +-- 2 files changed, 20 insertions(+), 31 deletions(-) diff --git a/src/core/utils/hash.rs b/src/core/utils/hash.rs index 9412228f..d6efaff9 100644 --- a/src/core/utils/hash.rs +++ b/src/core/utils/hash.rs @@ -1,4 +1,4 @@ -use std::sync::Mutex; +use std::sync::OnceLock; use argon2::{ password_hash, password_hash::SaltString, Algorithm, Argon2, Params, PasswordHash, PasswordHasher, @@ -9,10 +9,24 @@ const M_COST: u32 = Params::DEFAULT_M_COST; // memory size in 1 KiB blocks const T_COST: u32 = Params::DEFAULT_T_COST; // nr of iterations const P_COST: u32 = Params::DEFAULT_P_COST; // parallelism -static STATE: Mutex>> = Mutex::new(None); +static ARGON: OnceLock> = OnceLock::new(); -#[allow(clippy::let_underscore_must_use)] -pub fn init() { +pub fn password(password: &str) -> Result { + let salt = SaltString::generate(rand::thread_rng()); + ARGON + .get_or_init(init_argon) + .hash_password(password.as_bytes(), &salt) + .map(|it| it.to_string()) +} + +pub fn verify_password(password: &str, password_hash: &str) -> Result<(), password_hash::Error> { + let password_hash = PasswordHash::new(password_hash)?; + ARGON + .get_or_init(init_argon) + .verify_password(password.as_bytes(), &password_hash) +} + +fn init_argon() -> Argon2<'static> { // 19456 Kib blocks, iterations = 2, parallelism = 1 // * debug_assert!(M_COST == 19_456, "M_COST default changed"); @@ -23,29 +37,7 @@ pub fn init() { let version = Version::default(); let out_len: Option = None; let params = Params::new(M_COST, T_COST, P_COST, out_len).expect("valid parameters"); - let state = Argon2::new(algorithm, version, params); - _ = STATE.lock().expect("hashing state locked").insert(state); -} - -pub fn password(password: &str) -> Result { - let salt = SaltString::generate(rand::thread_rng()); - STATE - .lock() - .expect("hashing state locked") - .as_ref() - .expect("hashing state initialized") - .hash_password(password.as_bytes(), &salt) - .map(|it| it.to_string()) -} - -pub fn verify_password(password: &str, password_hash: &str) -> Result<(), password_hash::Error> { - let password_hash = PasswordHash::new(password_hash)?; - STATE - .lock() - .expect("hashing state locked") - .as_ref() - .expect("hashing state initialized") - .verify_password(password.as_bytes(), &password_hash) + Argon2::new(algorithm, version, params) } #[cfg(test)] @@ -53,7 +45,6 @@ mod tests { #[test] fn password_hash_and_verify() { use crate::utils::hash; - hash::init(); let preimage = "temp123"; let digest = hash::password(preimage).expect("digest"); hash::verify_password(preimage, &digest).expect("verified"); @@ -63,7 +54,6 @@ mod tests { #[should_panic(expected = "unverified")] fn password_hash_and_verify_fail() { use crate::utils::hash; - hash::init(); let preimage = "temp123"; let fakeimage = "temp321"; let digest = hash::password(preimage).expect("digest"); diff --git a/src/main/server.rs b/src/main/server.rs index 08d81d01..dfe75541 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -4,7 +4,7 @@ use conduit::{ config::Config, info, log::{self}, - utils::{hash, sys}, + utils::sys, Error, Result, }; use tokio::runtime; @@ -39,7 +39,6 @@ impl Server { #[cfg(unix)] sys::maximize_fd_limit().expect("Unable to increase maximum soft and hard file descriptor limit"); - hash::init(); info!( server_name = %config.server_name, From 20fe84379c11ce345459871c23986ba4ccc4fbbf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 26 Jun 2024 00:19:35 +0000 Subject: [PATCH 0294/2091] rename log::Server to log::Log Signed-off-by: Jason Volk --- src/core/log/mod.rs | 13 +++++++++++-- src/core/log/server.rs | 14 -------------- src/core/server.rs | 4 ++-- src/main/server.rs | 10 ++-------- 4 files changed, 15 insertions(+), 26 deletions(-) delete mode 100644 src/core/log/server.rs diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 49efe56d..daa6b8e8 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -2,17 +2,26 @@ pub mod capture; pub mod color; pub mod fmt; mod reload; -mod server; mod suppress; pub use capture::Capture; pub use reload::{LogLevelReloadHandles, ReloadHandle}; -pub use server::Server; pub use suppress::Suppress; pub use tracing::Level; pub use tracing_core::{Event, Metadata}; pub use tracing_subscriber::EnvFilter; +/// Logging subsystem. This is a singleton member of super::Server which holds +/// all logging and tracing related state rather than shoving it all in +/// super::Server directly. +pub struct Log { + /// General log level reload handles. + pub reload: LogLevelReloadHandles, + + /// Tracing capture state for ephemeral/oneshot uses. + pub capture: std::sync::Arc, +} + // Wraps for logging macros. Use these macros rather than extern tracing:: or // log:: crates in project code. ::log and ::tracing can still be used if // necessary but discouraged. Remember debug_ log macros are also exported to diff --git a/src/core/log/server.rs b/src/core/log/server.rs deleted file mode 100644 index 644cecd1..00000000 --- a/src/core/log/server.rs +++ /dev/null @@ -1,14 +0,0 @@ -use std::sync::Arc; - -use super::{capture, reload::LogLevelReloadHandles}; - -/// Logging subsystem. This is a singleton member of super::Server which holds -/// all logging and tracing related state rather than shoving it all in -/// super::Server directly. -pub struct Server { - /// General log level reload handles. - pub reload: LogLevelReloadHandles, - - /// Tracing capture state for ephemeral/oneshot uses. - pub capture: Arc, -} diff --git a/src/core/server.rs b/src/core/server.rs index 2aa821fa..e76e4d57 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -33,7 +33,7 @@ pub struct Server { pub signal: broadcast::Sender<&'static str>, /// Logging subsystem state - pub log: log::Server, + pub log: log::Log, /// TODO: move stats pub requests_spawn_active: AtomicU32, @@ -45,7 +45,7 @@ pub struct Server { impl Server { #[must_use] - pub fn new(config: Config, runtime: Option, log: log::Server) -> Self { + pub fn new(config: Config, runtime: Option, log: log::Log) -> Self { Self { config, started: SystemTime::now(), diff --git a/src/main/server.rs b/src/main/server.rs index dfe75541..f72b3ef3 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,12 +1,6 @@ use std::sync::Arc; -use conduit::{ - config::Config, - info, - log::{self}, - utils::sys, - Error, Result, -}; +use conduit::{config::Config, info, log::Log, utils::sys, Error, Result}; use tokio::runtime; use crate::{clap::Args, tracing::TracingFlameGuard}; @@ -52,7 +46,7 @@ impl Server { server: Arc::new(conduit::Server::new( config, runtime.cloned(), - log::Server { + Log { reload: tracing_reload_handle, capture, }, From bc5003417387c8a02ba7abeaea95d86ff52aaf82 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 25 Jun 2024 00:52:56 +0000 Subject: [PATCH 0295/2091] idiomize various expressions Signed-off-by: Jason Volk --- src/core/config/mod.rs | 98 +++++++++++++++++++----------------------- 1 file changed, 45 insertions(+), 53 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 636f0092..4260c46e 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -518,10 +518,9 @@ impl fmt::Display for Config { ("Database path", &self.database_path.to_string_lossy()), ( "Database backup path", - match &self.database_backup_path { - Some(path) => path.to_str().unwrap(), - None => "", - }, + self.database_backup_path + .as_ref() + .map_or("", |path| path.to_str().unwrap_or("")), ), ("Database backups to keep", &self.database_backups_to_keep.to_string()), ("Database cache capacity (MB)", &self.db_cache_capacity_mb.to_string()), @@ -573,9 +572,10 @@ impl fmt::Display for Config { ("Allow registration", &self.allow_registration.to_string()), ( "Registration token", - match self.registration_token { - Some(_) => "set", - None => "not set (open registration!)", + if self.registration_token.is_some() { + "set" + } else { + "not set (open registration!)" }, ), ( @@ -656,13 +656,14 @@ impl fmt::Display for Config { None => "not set", }, ), - ("Trusted key servers", { - let mut lst = vec![]; - for server in &self.trusted_servers { - lst.push(server.host()); - } - &lst.join(", ") - }), + ( + "Trusted key servers", + &self + .trusted_servers + .iter() + .map(|server| server.host()) + .join(", "), + ), ( "Query Trusted Key Servers First", &self.query_trusted_key_servers_first.to_string(), @@ -814,51 +815,45 @@ impl fmt::Display for Config { ("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()), ( "Well-known server name", - &if let Some(server) = &self.well_known.server { - server.to_string() - } else { - String::new() - }, + self.well_known + .server + .as_ref() + .map_or("", |server| server.as_str()), ), ( "Well-known client URL", - &if let Some(server) = &self.well_known.client { - server.to_string() - } else { - String::new() - }, + self.well_known + .client + .as_ref() + .map_or("", |url| url.as_str()), ), ( "Well-known support email", - &if let Some(support_email) = &self.well_known.support_email { - support_email.to_string() - } else { - String::new() - }, + self.well_known + .support_email + .as_ref() + .map_or("", |str| str.as_ref()), ), ( "Well-known support Matrix ID", - &if let Some(support_mxid) = &self.well_known.support_mxid { - support_mxid.to_string() - } else { - String::new() - }, + self.well_known + .support_mxid + .as_ref() + .map_or("", |mxid| mxid.as_str()), ), ( "Well-known support role", - &if let Some(support_role) = &self.well_known.support_role { - support_role.to_string() - } else { - String::new() - }, + self.well_known + .support_role + .as_ref() + .map_or("", |role| role.as_str()), ), ( "Well-known support page/URL", - &if let Some(support_page) = &self.well_known.support_page { - support_page.to_string() - } else { - String::new() - }, + self.well_known + .support_page + .as_ref() + .map_or("", |url| url.as_str()), ), ]; @@ -974,14 +969,13 @@ fn default_tracing_flame_output_path() -> String { "./tracing.folded".to_owned() fn default_trusted_servers() -> Vec { vec![OwnedServerName::try_from("matrix.org").unwrap()] } +/// do debug logging by default for debug builds #[must_use] pub fn default_log() -> String { - // do debug logging by default for debug builds - if cfg!(debug_assertions) { - "debug".to_owned() - } else { - "info".to_owned() - } + cfg!(debug_assertions) + .then_some("debug") + .unwrap_or("info") + .to_owned() } fn default_notification_push_path() -> String { "/_matrix/push/v1/notify".to_owned() } @@ -1062,9 +1056,7 @@ fn default_url_preview_max_spider_size() -> usize { fn default_new_user_displayname_suffix() -> String { "🏳️‍⚧️".to_owned() } fn default_sentry_endpoint() -> Option { - Url::parse("https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536") - .unwrap() - .into() + Url::parse("https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536").ok() } fn default_sentry_traces_sample_rate() -> f32 { 0.15 } From db288b169e52021faa9aa44c3c6b5d2e3f78b917 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 26 Jun 2024 23:00:33 +0000 Subject: [PATCH 0296/2091] add config option for tokio_console runtime enablement Signed-off-by: Jason Volk --- conduwuit-example.toml | 5 +++ src/core/config/mod.rs | 4 +++ src/main/tracing.rs | 72 ++++++++++++++++++++++++++++++------------ 3 files changed, 61 insertions(+), 20 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 37aaf2fe..f8f93070 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -371,6 +371,11 @@ allow_profile_lookup_federation_requests = true # If 'tracing_flame' is enabled, set the path to write the generated profile. # tracing_flame_output_path = "./tracing.folded" +# Enable the tokio-console. This option is only relevant to developers. +# See: docs/development.md#debugging-with-tokio-console for more information. +#tokio_console = false + + ### Generic database options # Set this to any float value to multiply conduwuit's in-memory LRU caches with. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 4260c46e..0b9cb299 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -355,6 +355,9 @@ pub struct Config { #[serde(default = "default_sentry_traces_sample_rate")] pub sentry_traces_sample_rate: f32, + #[serde(default)] + pub tokio_console: bool, + #[serde(flatten)] #[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime catchall: BTreeMap, @@ -855,6 +858,7 @@ impl fmt::Display for Config { .as_ref() .map_or("", |url| url.as_str()), ), + ("Enable the tokio-console", &self.tokio_console.to_string()), ]; let mut msg: String = "Active config values:\n\n".to_owned(); diff --git a/src/main/tracing.rs b/src/main/tracing.rs index b5f7a7d1..bbfe4dc4 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -3,9 +3,10 @@ use std::sync::Arc; use conduit::{ config, config::Config, + debug_warn, log::{capture, LogLevelReloadHandles, ReloadHandle}, }; -use tracing_subscriber::{prelude::*, reload, EnvFilter, Registry}; +use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; #[cfg(feature = "perf_measurements")] pub(crate) type TracingFlameGuard = Option>>; @@ -14,7 +15,6 @@ pub(crate) type TracingFlameGuard = (); #[allow(clippy::redundant_clone)] pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard, Arc) { - let registry = Registry::default(); let fmt_layer = tracing_subscriber::fmt::Layer::new(); let filter_layer = match EnvFilter::try_new(&config.log) { Ok(s) => s, @@ -25,22 +25,14 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard }; let mut reload_handles = Vec:: + Send + Sync>>::new(); - let subscriber = registry; - - #[cfg(all(feature = "tokio_console", tokio_unstable))] - let subscriber = { - let console_layer = console_subscriber::spawn(); - subscriber.with(console_layer) - }; - let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); reload_handles.push(Box::new(fmt_reload_handle)); + let subscriber = Registry::default().with(fmt_layer.with_filter(fmt_reload_filter)); + let cap_state = Arc::new(capture::State::new()); let cap_layer = capture::Layer::new(&cap_state); - let subscriber = subscriber - .with(fmt_layer.with_filter(fmt_reload_filter)) - .with(cap_layer); + let subscriber = subscriber.with(cap_layer); #[cfg(feature = "sentry_telemetry")] let subscriber = { @@ -95,13 +87,53 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard #[cfg_attr(not(feature = "perf_measurements"), allow(clippy::let_unit_value))] let flame_guard = (); - tracing::subscriber::set_global_default(subscriber).expect("failed to set global tracing subscriber"); + let ret = (LogLevelReloadHandles::new(reload_handles), flame_guard, cap_state); - #[cfg(all(feature = "tokio_console", feature = "release_max_log_level", tokio_unstable))] - tracing::error!( - "'tokio_console' feature and 'release_max_log_level' feature are incompatible, because console-subscriber \ - needs access to trace-level events. 'release_max_log_level' must be disabled to use tokio-console." - ); + // Enable the tokio console. This is slightly kludgy because we're judggling + // compile-time and runtime conditions to elide it, each of those changing the + // subscriber's type. + let (console_enabled, console_disabled_reason) = tokio_console_enabled(config); + #[cfg(all(feature = "tokio_console", tokio_unstable))] + if console_enabled { + let console_layer = console_subscriber::ConsoleLayer::builder() + .with_default_env() + .spawn(); - (LogLevelReloadHandles::new(reload_handles), flame_guard, cap_state) + set_global_default(subscriber.with(console_layer)); + return ret; + } + + set_global_default(subscriber); + + // If there's a reason the tokio console was disabled when it might be desired + // we output that here after initializing logging + if !console_enabled && !console_disabled_reason.is_empty() { + debug_warn!("{console_disabled_reason}"); + } + + ret +} + +fn tokio_console_enabled(config: &Config) -> (bool, &'static str) { + if !cfg!(all(feature = "tokio_console", tokio_unstable)) { + return (false, ""); + } + + if cfg!(feature = "release_max_log_level") && !cfg!(debug_assertions) { + return ( + false, + "'tokio_console' feature and 'release_max_log_level' feature are incompatible.", + ); + } + + if !config.tokio_console { + return (false, "tokio console is available but disabled by the configuration."); + } + + (true, "") +} + +fn set_global_default(subscriber: S) { + tracing::subscriber::set_global_default(subscriber) + .expect("the global default tracing subscriber failed to be initialized"); } From d9054f5364f78d6ca7c46af88d5a922fcdfb4d77 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 14 Jun 2024 17:53:40 +0000 Subject: [PATCH 0297/2091] add backreference to layer in capture data struct Signed-off-by: Jason Volk --- src/core/log/capture/data.rs | 3 ++- src/core/log/capture/layer.rs | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/core/log/capture/data.rs b/src/core/log/capture/data.rs index ea104f57..3f92101d 100644 --- a/src/core/log/capture/data.rs +++ b/src/core/log/capture/data.rs @@ -1,9 +1,10 @@ use tracing::Level; use tracing_core::{span::Current, Event}; -use super::layer::Value; +use super::{layer::Value, Layer}; pub struct Data<'a> { + pub layer: &'a Layer, pub event: &'a Event<'a>, pub current: &'a Current, pub values: Option<&'a mut [Value]>, diff --git a/src/core/log/capture/layer.rs b/src/core/log/capture/layer.rs index 48b125da..19d76771 100644 --- a/src/core/log/capture/layer.rs +++ b/src/core/log/capture/layer.rs @@ -40,12 +40,12 @@ where .read() .expect("shared lock") .iter() - .filter(|capture| filter(capture, event, &ctx)) - .for_each(|capture| handle(capture, event, &ctx)); + .filter(|capture| filter(self, capture, event, &ctx)) + .for_each(|capture| handle(self, capture, event, &ctx)); } } -fn handle(capture: &Capture, event: &Event<'_>, ctx: &Context<'_, S>) +fn handle(layer: &Layer, capture: &Capture, event: &Event<'_>, ctx: &Context<'_, S>) where S: Subscriber + for<'a> LookupSpan<'a>, { @@ -56,18 +56,20 @@ where let mut closure = capture.closure.lock().expect("exclusive lock"); closure(Data { + layer, event, current: &ctx.current_span(), values: Some(&mut visitor.values), }); } -fn filter(capture: &Capture, event: &Event<'_>, ctx: &Context<'_, S>) -> bool +fn filter(layer: &Layer, capture: &Capture, event: &Event<'_>, ctx: &Context<'_, S>) -> bool where S: Subscriber + for<'a> LookupSpan<'a>, { capture.filter.as_ref().map_or(true, |filter| { filter(Data { + layer, event, current: &ctx.current_span(), values: None, From 96a16190c586ecf2eb8ef670a5918b036a145b66 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 27 Jun 2024 23:19:20 +0000 Subject: [PATCH 0298/2091] mute dev_release_log_level from --all-features for now Signed-off-by: Jason Volk --- src/admin/Cargo.toml | 2 +- src/api/Cargo.toml | 2 +- src/core/Cargo.toml | 2 +- src/database/Cargo.toml | 2 +- src/main/Cargo.toml | 16 ++++++++-------- src/router/Cargo.toml | 2 +- src/service/Cargo.toml | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 413a4613..97de55ff 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -17,7 +17,7 @@ crate-type = [ ] [features] -dev_release_log_level = [] +#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index fb486a71..a27924c6 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -18,7 +18,7 @@ crate-type = [ [features] element_hacks = [] -dev_release_log_level = [] +#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index ec4eaf9b..76a4084e 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -17,7 +17,7 @@ crate-type = [ ] [features] -dev_release_log_level = [] +#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 22858e01..104ef93d 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -17,7 +17,7 @@ crate-type = [ ] [features] -dev_release_log_level = [] +#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 0479311f..fa0e5874 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -60,14 +60,14 @@ brotli_compression = [ console = [ "conduit-service/console", ] -dev_release_log_level = [ - "conduit-admin/dev_release_log_level", - "conduit-api/dev_release_log_level", - "conduit-core/dev_release_log_level", - "conduit-database/dev_release_log_level", - "conduit-router/dev_release_log_level", - "conduit-service/dev_release_log_level", -] +#dev_release_log_level = [ +# "conduit-admin/dev_release_log_level", +# "conduit-api/dev_release_log_level", +# "conduit-core/dev_release_log_level", +# "conduit-database/dev_release_log_level", +# "conduit-router/dev_release_log_level", +# "conduit-service/dev_release_log_level", +#] element_hacks = [ "conduit-api/element_hacks", "conduit-service/element_hacks", diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index f613826b..2fac3858 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -17,7 +17,7 @@ crate-type = [ ] [features] -dev_release_log_level = [] +#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 0151b823..f59a6036 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -24,7 +24,7 @@ console = [ "dep:rustyline-async", "dep:termimad", ] -dev_release_log_level = [] +#dev_release_log_level = [] element_hacks = [] gzip_compression = [ "reqwest/gzip", From fa02d7b7e33840de6cd8ec58f3ab2e9c3b46486c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 02:55:00 +0000 Subject: [PATCH 0299/2091] diffuse get_alias_helper into services::rooms::alias Signed-off-by: Jason Volk --- src/admin/room/room_moderation_commands.rs | 46 ++++-- src/api/client/alias.rs | 162 +++------------------ src/api/client/membership.rs | 20 ++- src/api/client/mod.rs | 1 - src/service/rooms/alias/mod.rs | 75 +++++++++- src/service/rooms/alias/remote.rs | 71 +++++++++ 6 files changed, 204 insertions(+), 171 deletions(-) create mode 100644 src/service/rooms/alias/remote.rs diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 75a20584..39eb7c47 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -1,4 +1,4 @@ -use api::client::{get_alias_helper, leave_room}; +use api::client::leave_room; use ruma::{ events::room::message::RoomMessageEventContent, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, RoomOrAliasId, }; @@ -76,13 +76,18 @@ async fn ban_room( } else { debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); - match get_alias_helper(room_alias, None).await { - Ok(response) => { - debug!("Got federation response fetching room ID for room {room}: {:?}", response); - response.room_id + match services() + .rooms + .alias + .resolve_alias(&room_alias, None) + .await + { + Ok((room_id, servers)) => { + debug!(?room_id, ?servers, "Got federation response fetching room ID for {room}"); + room_id }, Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( + return Ok(RoomMessageEventContent::notice_plain(format!( "Failed to resolve room alias {room} to a room ID: {e}" ))); }, @@ -237,13 +242,19 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo ID over federation" ); - match get_alias_helper(room_alias, None).await { - Ok(response) => { + match services() + .rooms + .alias + .resolve_alias(&room_alias, None) + .await + { + Ok((room_id, servers)) => { debug!( - "Got federation response fetching room ID for room {room}: {:?}", - response + ?room_id, + ?servers, + "Got federation response fetching room ID for {room}", ); - response.room_id + room_id }, Err(e) => { // don't fail if force blocking @@ -426,10 +437,15 @@ async fn unban_room( } else { debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); - match get_alias_helper(room_alias, None).await { - Ok(response) => { - debug!("Got federation response fetching room ID for room {room}: {:?}", response); - response.room_id + match services() + .rooms + .alias + .resolve_alias(&room_alias, None) + .await + { + Ok((room_id, servers)) => { + debug!(?room_id, ?servers, "Got federation response fetching room ID for room {room}"); + room_id }, Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 5752ea9b..5230c25b 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -1,22 +1,14 @@ use rand::seq::SliceRandom; use ruma::{ - api::{ - appservice, - client::{ - alias::{create_alias, delete_alias, get_alias}, - error::ErrorKind, - }, - federation, + api::client::{ + alias::{create_alias, delete_alias, get_alias}, + error::ErrorKind, }, - OwnedRoomAliasId, OwnedServerName, RoomAliasId, RoomId, + OwnedServerName, RoomAliasId, RoomId, }; use tracing::debug; -use crate::{ - debug_info, debug_warn, - service::{appservice::RegistrationInfo, server_is_ours}, - services, Error, Result, Ruma, -}; +use crate::{service::server_is_ours, services, Error, Result, Ruma}; /// # `PUT /_matrix/client/v3/directory/room/{roomAlias}` /// @@ -24,7 +16,7 @@ use crate::{ pub(crate) async fn create_alias_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - alias_checks(&body.room_alias, &body.appservice_info).await?; + service::rooms::alias::appservice_checks(&body.room_alias, &body.appservice_info).await?; // this isn't apart of alias_checks or delete alias route because we should // allow removing forbidden room aliases @@ -61,7 +53,7 @@ pub(crate) async fn create_alias_route(body: Ruma) -> pub(crate) async fn delete_alias_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - alias_checks(&body.room_alias, &body.appservice_info).await?; + service::rooms::alias::appservice_checks(&body.room_alias, &body.appservice_info).await?; if services() .rooms @@ -87,124 +79,20 @@ pub(crate) async fn delete_alias_route(body: Ruma) -> /// /// Resolve an alias locally or over federation. pub(crate) async fn get_alias_route(body: Ruma) -> Result { - get_alias_helper(body.body.room_alias, None).await -} + let room_alias = body.body.room_alias; + let servers = None; -pub async fn get_alias_helper( - room_alias: OwnedRoomAliasId, servers: Option>, -) -> Result { - debug!("get_alias_helper servers: {servers:?}"); - if !server_is_ours(room_alias.server_name()) - && (!servers - .as_ref() - .is_some_and(|servers| servers.contains(&services().globals.server_name().to_owned())) - || servers.as_ref().is_none()) - { - let mut response = services() - .sending - .send_federation_request( - room_alias.server_name(), - federation::query::get_room_information::v1::Request { - room_alias: room_alias.clone(), - }, - ) - .await; - - debug!("room alias server_name get_alias_helper response: {response:?}"); - - if let Err(ref e) = response { - debug_info!( - "Server {} of the original room alias failed to assist in resolving room alias: {e}", - room_alias.server_name() - ); - } - - if response.as_ref().is_ok_and(|resp| resp.servers.is_empty()) || response.as_ref().is_err() { - if let Some(servers) = servers { - for server in servers { - response = services() - .sending - .send_federation_request( - &server, - federation::query::get_room_information::v1::Request { - room_alias: room_alias.clone(), - }, - ) - .await; - debug!("Got response from server {server} for room aliases: {response:?}"); - - if let Ok(ref response) = response { - if !response.servers.is_empty() { - break; - } - debug_warn!( - "Server {server} responded with room aliases, but was empty? Response: {response:?}" - ); - } - } - } - } - - if let Ok(response) = response { - let room_id = response.room_id; - - let mut pre_servers = response.servers; - // since the room alis server responded, insert it into the list - pre_servers.push(room_alias.server_name().into()); - - let servers = room_available_servers(&room_id, &room_alias, &Some(pre_servers)); - debug!( - "room alias servers from federation response for room ID {room_id} and room alias {room_alias}: \ - {servers:?}" - ); - - return Ok(get_alias::v3::Response::new(room_id, servers)); - } - - return Err(Error::BadRequest( - ErrorKind::NotFound, - "No servers could assist in resolving the room alias", - )); - } - - let mut room_id = None; - match services().rooms.alias.resolve_local_alias(&room_alias)? { - Some(r) => room_id = Some(r), - None => { - for appservice in services().appservice.read().await.values() { - if appservice.aliases.is_match(room_alias.as_str()) - && matches!( - services() - .sending - .send_appservice_request( - appservice.registration.clone(), - appservice::query::query_room_alias::v1::Request { - room_alias: room_alias.clone(), - }, - ) - .await, - Ok(Some(_opt_result)) - ) { - room_id = Some( - services() - .rooms - .alias - .resolve_local_alias(&room_alias)? - .ok_or_else(|| Error::bad_config("Room does not exist."))?, - ); - break; - } - } - }, - }; - - let Some(room_id) = room_id else { + let Ok((room_id, pre_servers)) = services() + .rooms + .alias + .resolve_alias(&room_alias, servers.as_ref()) + .await + else { return Err(Error::BadRequest(ErrorKind::NotFound, "Room with alias not found.")); }; - let servers = room_available_servers(&room_id, &room_alias, &None); - - debug!("room alias servers for room ID {room_id} and room alias {room_alias}"); + let servers = room_available_servers(&room_id, &room_alias, &pre_servers); + debug!(?room_alias, ?room_id, "available servers: {servers:?}"); Ok(get_alias::v3::Response::new(room_id, servers)) } @@ -250,19 +138,3 @@ fn room_available_servers( servers } - -async fn alias_checks(room_alias: &RoomAliasId, appservice_info: &Option) -> Result<()> { - if !server_is_ours(room_alias.server_name()) { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Alias is from another server.")); - } - - if let Some(ref info) = appservice_info { - if !info.aliases.is_match(room_alias.as_str()) { - return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias is not in namespace.")); - } - } else if services().appservice.is_exclusive_alias(room_alias).await { - return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias reserved by appservice.")); - } - - Ok(()) -} diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 88cbdd44..b09d8e08 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -36,7 +36,6 @@ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tokio::sync::RwLock; use tracing::{debug, error, info, trace, warn}; -use super::get_alias_helper; use crate::{ client::{update_avatar_url, update_displayname}, service::{ @@ -259,17 +258,24 @@ pub(crate) async fn join_room_by_id_or_alias_route( (servers, room_id) }, Err(room_alias) => { - let response = get_alias_helper(room_alias.clone(), Some(body.server_name.clone())).await?; + let response = services() + .rooms + .alias + .resolve_alias(&room_alias, Some(&body.server_name.clone())) + .await?; + let (room_id, mut pre_servers) = response; - banned_room_check(sender_user, Some(&response.room_id), Some(room_alias.server_name()), client).await?; + banned_room_check(sender_user, Some(&room_id), Some(room_alias.server_name()), client).await?; let mut servers = body.server_name; - servers.extend(response.servers); + if let Some(pre_servers) = &mut pre_servers { + servers.append(pre_servers); + } servers.extend( services() .rooms .state_cache - .servers_invite_via(&response.room_id) + .servers_invite_via(&room_id) .filter_map(Result::ok), ); @@ -277,7 +283,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( services() .rooms .state_cache - .invite_state(sender_user, &response.room_id)? + .invite_state(sender_user, &room_id)? .unwrap_or_default() .iter() .filter_map(|event| serde_json::from_str(event.json().get()).ok()) @@ -287,7 +293,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( .map(|user| user.server_name().to_owned()), ); - (servers, response.room_id) + (servers, room_id) }, }; diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index 1461be94..79088a5b 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -35,7 +35,6 @@ pub(super) mod user_directory; pub(super) mod voip; pub(super) use account::*; -pub use alias::get_alias_helper; pub(super) use alias::*; pub(super) use backup::*; pub(super) use capabilities::*; diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index cfa4edb9..6397ef7f 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -1,4 +1,5 @@ mod data; +mod remote; use std::sync::Arc; @@ -6,15 +7,15 @@ use conduit::{Error, Result, Server}; use data::Data; use database::Database; use ruma::{ - api::client::error::ErrorKind, + api::{appservice, client::error::ErrorKind}, events::{ room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, StateEventType, }, - OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId, UserId, + OwnedRoomAliasId, OwnedRoomId, OwnedServerName, RoomAliasId, RoomId, UserId, }; -use crate::services; +use crate::{appservice::RegistrationInfo, server_is_ours, services}; pub struct Service { db: Data, @@ -51,6 +52,30 @@ impl Service { } } + #[tracing::instrument(skip(self), name = "resolve")] + pub async fn resolve_alias( + &self, room_alias: &RoomAliasId, servers: Option<&Vec>, + ) -> Result<(OwnedRoomId, Option>)> { + if !server_is_ours(room_alias.server_name()) + && (!servers + .as_ref() + .is_some_and(|servers| servers.contains(&services().globals.server_name().to_owned())) + || servers.as_ref().is_none()) + { + return remote::resolve(room_alias, servers).await; + } + + let room_id: Option = match self.resolve_local_alias(room_alias)? { + Some(r) => Some(r), + None => self.resolve_appservice_alias(room_alias).await?, + }; + + room_id.map_or_else( + || Err(Error::BadRequest(ErrorKind::NotFound, "Room with alias not found.")), + |room_id| Ok((room_id, None)), + ) + } + #[tracing::instrument(skip(self))] pub fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { self.db.resolve_local_alias(alias) @@ -112,4 +137,48 @@ impl Service { Err(Error::bad_database("Room has no m.room.create event")) } } + + async fn resolve_appservice_alias(&self, room_alias: &RoomAliasId) -> Result> { + for appservice in services().appservice.read().await.values() { + if appservice.aliases.is_match(room_alias.as_str()) + && matches!( + services() + .sending + .send_appservice_request( + appservice.registration.clone(), + appservice::query::query_room_alias::v1::Request { + room_alias: room_alias.to_owned(), + }, + ) + .await, + Ok(Some(_opt_result)) + ) { + return Ok(Some( + services() + .rooms + .alias + .resolve_local_alias(room_alias)? + .ok_or_else(|| Error::bad_config("Room does not exist."))?, + )); + } + } + + Ok(None) + } +} + +pub async fn appservice_checks(room_alias: &RoomAliasId, appservice_info: &Option) -> Result<()> { + if !server_is_ours(room_alias.server_name()) { + return Err(Error::BadRequest(ErrorKind::InvalidParam, "Alias is from another server.")); + } + + if let Some(ref info) = appservice_info { + if !info.aliases.is_match(room_alias.as_str()) { + return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias is not in namespace.")); + } + } else if services().appservice.is_exclusive_alias(room_alias).await { + return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias reserved by appservice.")); + } + + Ok(()) } diff --git a/src/service/rooms/alias/remote.rs b/src/service/rooms/alias/remote.rs new file mode 100644 index 00000000..7fcd27f5 --- /dev/null +++ b/src/service/rooms/alias/remote.rs @@ -0,0 +1,71 @@ +use conduit::{debug, debug_info, debug_warn, Error, Result}; +use ruma::{ + api::{client::error::ErrorKind, federation}, + OwnedRoomId, OwnedServerName, RoomAliasId, +}; + +use crate::services; + +pub(super) async fn resolve( + room_alias: &RoomAliasId, servers: Option<&Vec>, +) -> Result<(OwnedRoomId, Option>)> { + debug!(?room_alias, ?servers, "resolve"); + + let mut response = services() + .sending + .send_federation_request( + room_alias.server_name(), + federation::query::get_room_information::v1::Request { + room_alias: room_alias.to_owned(), + }, + ) + .await; + + debug!("room alias server_name get_alias_helper response: {response:?}"); + + if let Err(ref e) = response { + debug_info!( + "Server {} of the original room alias failed to assist in resolving room alias: {e}", + room_alias.server_name() + ); + } + + if response.as_ref().is_ok_and(|resp| resp.servers.is_empty()) || response.as_ref().is_err() { + if let Some(servers) = servers { + for server in servers { + response = services() + .sending + .send_federation_request( + server, + federation::query::get_room_information::v1::Request { + room_alias: room_alias.to_owned(), + }, + ) + .await; + debug!("Got response from server {server} for room aliases: {response:?}"); + + if let Ok(ref response) = response { + if !response.servers.is_empty() { + break; + } + debug_warn!("Server {server} responded with room aliases, but was empty? Response: {response:?}"); + } + } + } + } + + if let Ok(response) = response { + let room_id = response.room_id; + + let mut pre_servers = response.servers; + // since the room alis server responded, insert it into the list + pre_servers.push(room_alias.server_name().into()); + + return Ok((room_id, Some(pre_servers))); + } + + Err(Error::BadRequest( + ErrorKind::NotFound, + "No servers could assist in resolving the room alias", + )) +} From f127987c7a34a318b92dfbe7433fcfaa496173a2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 02:57:32 +0000 Subject: [PATCH 0300/2091] add convenience alias resolver to interface Signed-off-by: Jason Volk --- src/service/rooms/alias/mod.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 6397ef7f..9f66cf88 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -12,7 +12,7 @@ use ruma::{ room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, StateEventType, }, - OwnedRoomAliasId, OwnedRoomId, OwnedServerName, RoomAliasId, RoomId, UserId, + OwnedRoomAliasId, OwnedRoomId, OwnedServerName, RoomAliasId, RoomId, RoomOrAliasId, UserId, }; use crate::{appservice::RegistrationInfo, server_is_ours, services}; @@ -52,6 +52,16 @@ impl Service { } } + pub async fn resolve(&self, room: &RoomOrAliasId) -> Result { + if room.is_room_id() { + let room_id: &RoomId = &RoomId::parse(room).expect("valid RoomId"); + Ok(room_id.to_owned()) + } else { + let alias: &RoomAliasId = &RoomAliasId::parse(room).expect("valid RoomAliasId"); + Ok(self.resolve_alias(alias, None).await?.0) + } + } + #[tracing::instrument(skip(self), name = "resolve")] pub async fn resolve_alias( &self, room_alias: &RoomAliasId, servers: Option<&Vec>, From b28d216e0070f5deccaa70dcd8bbe8f328472b96 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 03:12:12 +0000 Subject: [PATCH 0301/2091] improve debug get-room-state; use RoomOrAliasId Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 12 +++++------- src/admin/debug/mod.rs | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 40aa0278..c5b944bf 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -13,7 +13,7 @@ use conduit::{ use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, events::room::message::RoomMessageEventContent, - CanonicalJsonObject, EventId, RoomId, RoomVersionId, ServerName, + CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, }; use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve::resolve_actual_dest, services, PduEvent}; use tokio::sync::RwLock; @@ -236,7 +236,8 @@ pub(super) async fn get_remote_pdu( } } -pub(super) async fn get_room_state(_body: Vec<&str>, room_id: Box) -> Result { +pub(super) async fn get_room_state(_body: Vec<&str>, room: OwnedRoomOrAliasId) -> Result { + let room_id = services().rooms.alias.resolve(&room).await?; let room_state = services() .rooms .state_accessor @@ -252,17 +253,14 @@ pub(super) async fn get_room_state(_body: Vec<&str>, room_id: Box) -> Re )); } - let json_text = serde_json::to_string_pretty(&room_state).map_err(|e| { + let json = serde_json::to_string_pretty(&room_state).map_err(|e| { warn!("Failed converting room state vector in our database to pretty JSON: {e}"); Error::bad_database( "Failed to convert room state events to pretty JSON, possible invalid room state events in our database", ) })?; - Ok(RoomMessageEventContent::notice_markdown(format!( - "{}\n```json\n{}\n```", - "Found full room state", json_text - ))) + Ok(RoomMessageEventContent::notice_markdown(format!("```json\n{json}\n```"))) } pub(super) async fn ping(_body: Vec<&str>, server: Box) -> Result { diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 19d3712a..27e36a73 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -2,7 +2,7 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{events::room::message::RoomMessageEventContent, EventId, RoomId, ServerName}; +use ruma::{events::room::message::RoomMessageEventContent, EventId, OwnedRoomOrAliasId, RoomId, ServerName}; use self::commands::*; @@ -71,7 +71,7 @@ pub(super) enum DebugCommand { /// Of course the check is still done on the actual client API. GetRoomState { /// Room ID - room_id: Box, + room_id: OwnedRoomOrAliasId, }, /// - Sends a federation request to the remote server's From 3301cde8e7ae8eb2e7bd8c2df49120bfd3b171d4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 07:10:29 +0000 Subject: [PATCH 0302/2091] add timer around admin command processing Signed-off-by: Jason Volk --- src/admin/handler.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 8a767cb8..bda295c3 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,3 +1,5 @@ +use std::time::Instant; + use clap::Parser; use conduit::trace; use ruma::events::{ @@ -84,11 +86,10 @@ async fn handle_command(command: Command) -> CommandResult { // Parse and process a message from the admin room async fn process_admin_message(msg: String) -> CommandOutput { let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); - let command_line = lines.next().expect("each string has at least one line"); + let command = lines.next().expect("each string has at least one line"); let body = lines.collect::>(); - - let admin_command = match parse_admin_command(command_line) { - Ok(command) => command, + let parsed = match parse_admin_command(command) { + Ok(parsed) => parsed, Err(error) => { let server_name = services().globals.server_name(); let message = error.replace("server.name", server_name.as_str()); @@ -96,12 +97,15 @@ async fn process_admin_message(msg: String) -> CommandOutput { }, }; - match process_admin_command(admin_command, body).await { - Ok(reply_message) => Some(reply_message), - Err(error) => { - let markdown_message = format!("Encountered an error while handling the command:\n```\n{error}\n```",); - Some(RoomMessageEventContent::notice_markdown(markdown_message)) - }, + let timer = Instant::now(); + let result = process_admin_command(parsed, body).await; + let elapsed = timer.elapsed(); + conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}"); + match result { + Ok(reply) => Some(reply), + Err(error) => Some(RoomMessageEventContent::notice_markdown(format!( + "Encountered an error while handling the command:\n```\n{error}\n```" + ))), } } From 5805394ca52867b6c339db1eebb3733b8f51b3e6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 07:11:00 +0000 Subject: [PATCH 0303/2091] add tester timer stub Signed-off-by: Jason Volk --- src/admin/tester/mod.rs | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/admin/tester/mod.rs b/src/admin/tester/mod.rs index 8350fe14..69dc7540 100644 --- a/src/admin/tester/mod.rs +++ b/src/admin/tester/mod.rs @@ -2,13 +2,41 @@ use ruma::events::room::message::RoomMessageEventContent; use crate::Result; -#[cfg_attr(test, derive(Debug))] #[derive(clap::Subcommand)] +#[cfg_attr(test, derive(Debug))] pub(super) enum TesterCommands { Tester, + Timer, } -pub(super) async fn process(command: TesterCommands, _body: Vec<&str>) -> Result { - Ok(match command { - TesterCommands::Tester => RoomMessageEventContent::notice_plain(String::from("completed")), - }) + +pub(super) async fn process(command: TesterCommands, body: Vec<&str>) -> Result { + match command { + TesterCommands::Tester => tester(body).await, + TesterCommands::Timer => timer(body).await, + } +} + +#[inline(never)] +#[rustfmt::skip] +#[allow(unused_variables)] +async fn tester(body: Vec<&str>) -> Result { + + Ok(RoomMessageEventContent::notice_plain("completed")) +} + +#[inline(never)] +#[rustfmt::skip] +async fn timer(body: Vec<&str>) -> Result { + let started = std::time::Instant::now(); + timed(&body); + + let elapsed = started.elapsed(); + Ok(RoomMessageEventContent::notice_plain(format!("completed in {elapsed:#?}"))) +} + +#[inline(never)] +#[rustfmt::skip] +#[allow(unused_variables)] +fn timed(body: &[&str]) { + } From 2387f7f955d4dc3e9f3d962a225f5f66dca6f6aa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 09:06:25 +0000 Subject: [PATCH 0304/2091] move tester command under debug Signed-off-by: Jason Volk --- src/admin/debug/mod.rs | 7 +++++++ src/admin/{tester/mod.rs => debug/tester.rs} | 8 ++++---- src/admin/handler.rs | 9 ++------- src/admin/mod.rs | 1 - 4 files changed, 13 insertions(+), 12 deletions(-) rename src/admin/{tester/mod.rs => debug/tester.rs} (76%) diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 27e36a73..eed3b633 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -1,8 +1,10 @@ mod commands; +pub(crate) mod tester; use clap::Subcommand; use conduit::Result; use ruma::{events::room::message::RoomMessageEventContent, EventId, OwnedRoomOrAliasId, RoomId, ServerName}; +use tester::TesterCommand; use self::commands::*; @@ -157,6 +159,10 @@ pub(super) enum DebugCommand { /// - Print extended memory usage MemoryStats, + + /// - Developer test stubs + #[command(subcommand)] + Tester(TesterCommand), } pub(super) async fn process(command: DebugCommand, body: Vec<&str>) -> Result { @@ -207,5 +213,6 @@ pub(super) async fn process(command: DebugCommand, body: Vec<&str>) -> Result resolve_true_destination(body, server_name, no_cache).await?, DebugCommand::MemoryStats => memory_stats(), + DebugCommand::Tester(command) => tester::process(command, body).await?, }) } diff --git a/src/admin/tester/mod.rs b/src/admin/debug/tester.rs similarity index 76% rename from src/admin/tester/mod.rs rename to src/admin/debug/tester.rs index 69dc7540..6982ef52 100644 --- a/src/admin/tester/mod.rs +++ b/src/admin/debug/tester.rs @@ -4,15 +4,15 @@ use crate::Result; #[derive(clap::Subcommand)] #[cfg_attr(test, derive(Debug))] -pub(super) enum TesterCommands { +pub(crate) enum TesterCommand { Tester, Timer, } -pub(super) async fn process(command: TesterCommands, body: Vec<&str>) -> Result { +pub(super) async fn process(command: TesterCommand, body: Vec<&str>) -> Result { match command { - TesterCommands::Tester => tester(body).await, - TesterCommands::Timer => timer(body).await, + TesterCommand::Tester => tester(body).await, + TesterCommand::Timer => timer(body).await, } } diff --git a/src/admin/handler.rs b/src/admin/handler.rs index bda295c3..ca8545d9 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -13,11 +13,10 @@ use conduit::Result; pub(crate) use service::admin::{Command, Service}; use service::admin::{CommandOutput, CommandResult, HandlerResult}; -use self::{fsck::FsckCommand, tester::TesterCommands}; use crate::{ appservice, appservice::AppserviceCommand, debug, debug::DebugCommand, federation, federation::FederationCommand, - fsck, media, media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, server, - server::ServerCommand, services, tester, user, user::UserCommand, + fsck, fsck::FsckCommand, media, media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, server, + server::ServerCommand, services, user, user::UserCommand, }; pub(crate) const PAGE_SIZE: usize = 100; @@ -60,9 +59,6 @@ pub(crate) enum AdminCommand { #[command(subcommand)] /// - Query all the database getters and iterators Fsck(FsckCommand), - - #[command(subcommand)] - Tester(TesterCommands), } #[must_use] @@ -169,7 +165,6 @@ async fn process_admin_command(command: AdminCommand, body: Vec<&str>) -> Result AdminCommand::Debug(command) => debug::process(command, body).await?, AdminCommand::Query(command) => query::process(command, body).await?, AdminCommand::Fsck(command) => fsck::process(command, body).await?, - AdminCommand::Tester(command) => tester::process(command, body).await?, }; Ok(reply_message_content) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 33158b5c..29830a86 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -9,7 +9,6 @@ pub(crate) mod media; pub(crate) mod query; pub(crate) mod room; pub(crate) mod server; -pub(crate) mod tester; pub(crate) mod user; pub(crate) mod utils; From 9d1db7d171a0b5a0782789d1eef1f04f1754e33d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 23 Jun 2024 09:55:51 +0000 Subject: [PATCH 0305/2091] rename fsck to check Signed-off-by: Jason Volk --- src/admin/{fsck => check}/commands.rs | 2 +- src/admin/{fsck => check}/mod.rs | 8 ++++---- src/admin/handler.rs | 18 +++++++++--------- src/admin/mod.rs | 2 +- 4 files changed, 15 insertions(+), 15 deletions(-) rename src/admin/{fsck => check}/commands.rs (93%) rename src/admin/{fsck => check}/mod.rs (51%) diff --git a/src/admin/fsck/commands.rs b/src/admin/check/commands.rs similarity index 93% rename from src/admin/fsck/commands.rs rename to src/admin/check/commands.rs index f3a2972a..1fbea8f6 100644 --- a/src/admin/fsck/commands.rs +++ b/src/admin/check/commands.rs @@ -19,7 +19,7 @@ pub(super) async fn check_all_users(_body: Vec<&str>) -> Result) -> Result { +pub(super) async fn process(command: CheckCommand, body: Vec<&str>) -> Result { Ok(match command { - FsckCommand::CheckAllUsers => check_all_users(body).await?, + CheckCommand::AllUsers => check_all_users(body).await?, }) } diff --git a/src/admin/handler.rs b/src/admin/handler.rs index ca8545d9..0bd8af9f 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -14,9 +14,9 @@ pub(crate) use service::admin::{Command, Service}; use service::admin::{CommandOutput, CommandResult, HandlerResult}; use crate::{ - appservice, appservice::AppserviceCommand, debug, debug::DebugCommand, federation, federation::FederationCommand, - fsck, fsck::FsckCommand, media, media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, server, - server::ServerCommand, services, user, user::UserCommand, + appservice, appservice::AppserviceCommand, check, check::CheckCommand, debug, debug::DebugCommand, federation, + federation::FederationCommand, media, media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, + server, server::ServerCommand, services, user, user::UserCommand, }; pub(crate) const PAGE_SIZE: usize = 100; @@ -48,17 +48,17 @@ pub(crate) enum AdminCommand { /// - Commands for managing media Media(MediaCommand), + #[command(subcommand)] + /// - Commands for checking integrity + Check(CheckCommand), + #[command(subcommand)] /// - Commands for debugging things Debug(DebugCommand), #[command(subcommand)] - /// - Query all the database getters and iterators + /// - Low-level queries for database getters and iterators Query(QueryCommand), - - #[command(subcommand)] - /// - Query all the database getters and iterators - Fsck(FsckCommand), } #[must_use] @@ -164,7 +164,7 @@ async fn process_admin_command(command: AdminCommand, body: Vec<&str>) -> Result AdminCommand::Server(command) => server::process(command, body).await?, AdminCommand::Debug(command) => debug::process(command, body).await?, AdminCommand::Query(command) => query::process(command, body).await?, - AdminCommand::Fsck(command) => fsck::process(command, body).await?, + AdminCommand::Check(command) => check::process(command, body).await?, }; Ok(reply_message_content) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 29830a86..f2034724 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1,9 +1,9 @@ #![allow(clippy::wildcard_imports)] pub(crate) mod appservice; +pub(crate) mod check; pub(crate) mod debug; pub(crate) mod federation; -pub(crate) mod fsck; pub(crate) mod handler; pub(crate) mod media; pub(crate) mod query; From 6c461025e507635cc8634eb2fca479dd703cd738 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 30 Jun 2024 23:26:36 -0400 Subject: [PATCH 0306/2091] use proper redacts field Signed-off-by: strawberry --- src/service/rooms/event_handler/mod.rs | 54 ++++++++++++++++++++------ src/service/rooms/timeline/mod.rs | 46 ++++++++++++++++++---- 2 files changed, 80 insertions(+), 20 deletions(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index ca9c9c14..0468ab38 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -19,7 +19,9 @@ use ruma::{ federation::event::{get_event, get_room_state_ids}, }, events::{ - room::{create::RoomCreateEventContent, server_acl::RoomServerAclEventContent}, + room::{ + create::RoomCreateEventContent, redaction::RoomRedactionEventContent, server_acl::RoomServerAclEventContent, + }, StateEventType, TimelineEventType, }, int, @@ -531,17 +533,45 @@ impl Service { auth_events.get(&(k.clone(), s.to_owned())) }) .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))? - || if let Some(redact_id) = &incoming_pdu.redacts { - incoming_pdu.kind == TimelineEventType::RoomRedaction - && !services().rooms.state_accessor.user_can_redact( - redact_id, - &incoming_pdu.sender, - &incoming_pdu.room_id, - true, - )? - } else { - false - }; + || incoming_pdu.kind == TimelineEventType::RoomRedaction + && match room_version_id { + RoomVersionId::V1 + | RoomVersionId::V2 + | RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + | RoomVersionId::V7 + | RoomVersionId::V8 + | RoomVersionId::V9 + | RoomVersionId::V10 => { + if let Some(redact_id) = &incoming_pdu.redacts { + !services().rooms.state_accessor.user_can_redact( + redact_id, + &incoming_pdu.sender, + &incoming_pdu.room_id, + true, + )? + } else { + false + } + }, + _ => { + let content = serde_json::from_str::(incoming_pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; + + if let Some(redact_id) = &content.redacts { + !services().rooms.state_accessor.user_can_redact( + redact_id, + &incoming_pdu.sender, + &incoming_pdu.room_id, + true, + )? + } else { + false + } + }, + }; // 13. Use state resolution to find new room state diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 2680c7f4..d18c76ef 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -860,14 +860,44 @@ impl Service { } // If redaction event is not authorized, do not append it to the timeline - if let Some(redact_id) = &pdu.redacts { - if pdu.kind == TimelineEventType::RoomRedaction - && !services() - .rooms - .state_accessor - .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false)? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "User cannot redact this event")); + if pdu.kind == TimelineEventType::RoomRedaction { + match services().rooms.state.get_room_version(&pdu.room_id)? { + RoomVersionId::V1 + | RoomVersionId::V2 + | RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + | RoomVersionId::V7 + | RoomVersionId::V8 + | RoomVersionId::V9 + | RoomVersionId::V10 => { + if let Some(redact_id) = &pdu.redacts { + if !services().rooms.state_accessor.user_can_redact( + redact_id, + &pdu.sender, + &pdu.room_id, + false, + )? { + return Err(Error::BadRequest(ErrorKind::forbidden(), "User cannot redact this event.")); + } + }; + }, + _ => { + let content = serde_json::from_str::(pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; + + if let Some(redact_id) = &content.redacts { + if !services().rooms.state_accessor.user_can_redact( + redact_id, + &pdu.sender, + &pdu.room_id, + false, + )? { + return Err(Error::BadRequest(ErrorKind::forbidden(), "User cannot redact this event.")); + } + } + }, } }; From 792d4aa6ce691773c995f989c4d6a7a3042b1972 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 30 Jun 2024 23:26:59 -0400 Subject: [PATCH 0307/2091] fix nightly clippy lint Signed-off-by: strawberry --- src/api/client/search.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/search.rs b/src/api/client/search.rs index e88558fd..d2a1668f 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -118,7 +118,7 @@ pub(crate) async fn search_events_route(body: Ruma) }; let mut results = Vec::new(); - let next_batch: usize = skip.saturating_add(limit); + let next_batch = skip.saturating_add(limit); for _ in 0..next_batch { if let Some(s) = searches From 2b35f830895922a6b123364cf0f5ba29a1704197 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 30 Jun 2024 23:36:07 -0400 Subject: [PATCH 0308/2091] ci: exclude `renovate[bot]` Signed-off-by: strawberry --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b96e58dd..e2daf176 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -269,7 +269,7 @@ jobs: name: Docker publish runs-on: ubuntu-latest needs: build - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' env: DOCKER_ARM64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 DOCKER_AMD64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 From b1150cded9e23467ed099cd5eba1e4e81e313762 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 22:04:28 -0400 Subject: [PATCH 0309/2091] rename configuruation.md to configuration/examples.md --- debian/README.md | 2 +- docs/SUMMARY.md | 2 +- docs/{configuration.md => configuration/examples.md} | 6 +++--- docs/deploying/docker.md | 2 +- docs/deploying/generic.md | 4 ++-- docs/maintenance.md | 2 +- docs/troubleshooting.md | 5 +++-- docs/turn.md | 2 +- 8 files changed, 13 insertions(+), 12 deletions(-) rename docs/{configuration.md => configuration/examples.md} (71%) diff --git a/debian/README.md b/debian/README.md index 3fe62212..62aa2112 100644 --- a/debian/README.md +++ b/debian/README.md @@ -15,7 +15,7 @@ in `/etc/conduwuit/conduwuit.toml`. ### Running -The package uses the [`conduwuit.service`](../configuration.md#example-systemd-unit-file) systemd unit file to start and stop conduwuit. The binary is installed at `/usr/sbin/conduwuit`. +The package uses the [`conduwuit.service`](../configuration/examples.md#example-systemd-unit-file) systemd unit file to start and stop conduwuit. The binary is installed at `/usr/sbin/conduwuit`. This package assumes by default that conduwuit will be placed behind a reverse proxy. The default config options apply (listening on `localhost` and TCP port `6167`). Matrix federation requires a valid domain name and TLS, so you will need to set up TLS certificates and renewal for it to work properly if you intend to federate. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 6f3bd570..fcd8f898 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -2,7 +2,7 @@ - [Introduction](introduction.md) - [Differences from upstream Conduit](differences.md) -- [Example configuration](configuration.md) +- [Example configuration](configuration/examples.md) - [Deploying](deploying.md) - [Generic](deploying/generic.md) - [NixOS](deploying/nixos.md) diff --git a/docs/configuration.md b/docs/configuration/examples.md similarity index 71% rename from docs/configuration.md rename to docs/configuration/examples.md index b0176c47..54aa8bd7 100644 --- a/docs/configuration.md +++ b/docs/configuration/examples.md @@ -4,7 +4,7 @@ Example configuration ```toml -{{#include ../conduwuit-example.toml}} +{{#include ../../conduwuit-example.toml}} ``` @@ -15,7 +15,7 @@ Debian systemd unit file ``` -{{#include ../debian/conduwuit.service}} +{{#include ../../debian/conduwuit.service}} ``` @@ -26,7 +26,7 @@ Arch Linux systemd unit file ``` -{{#include ../arch/conduwuit.service}} +{{#include ../../arch/conduwuit.service}} ``` diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index e5e6a1f7..6687e069 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -44,7 +44,7 @@ docker run -d -p 8448:6167 \ or you can use [docker compose](#docker-compose). -The `-d` flag lets the container run in detached mode. You may supply an optional `conduwuit.toml` config file, the example config can be found [here](../configuration.md). +The `-d` flag lets the container run in detached mode. You may supply an optional `conduwuit.toml` config file, the example config can be found [here](../configuration/examples.md). You can pass in different env vars to change config values on the fly. You can even configure conduwuit completely by using env vars. For an overview of possible values, please take a look at the [`docker-compose.yml`](docker-compose.yml) file. diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 28918980..0e846099 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -43,11 +43,11 @@ If conduwuit runs behind a router or in a container and has a different public I ## Setting up a systemd service -The systemd unit for conduwuit can be found [here](../configuration.md#example-systemd-unit-file). You may need to change the `ExecStart=` path to where you placed the conduwuit binary. +The systemd unit for conduwuit can be found [here](../configuration/examples.md#example-systemd-unit-file). You may need to change the `ExecStart=` path to where you placed the conduwuit binary. ## Creating the conduwuit configuration file -Now we need to create the conduwuit's config file in `/etc/conduwuit/conduwuit.toml`. The example config can be found at [conduwuit-example.toml](../configuration.md).**Please take a moment to read it. You need to change at least the server name.** +Now we need to create the conduwuit's config file in `/etc/conduwuit/conduwuit.toml`. The example config can be found at [conduwuit-example.toml](../configuration/examples.md).**Please take a moment to read it. You need to change at least the server name.** RocksDB is the only supported database backend. diff --git a/docs/maintenance.md b/docs/maintenance.md index 0e61d675..b67c8ccb 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -28,7 +28,7 @@ Any commands with `-list` in them will require a codeblock in the message with e If using RocksDB, there's very little you need to do. Compaction is ran automatically based on various defined thresholds tuned for conduwuit to be high performance with the least I/O amplifcation or overhead. Manually running compaction is not recommended, or compaction via a timer. RocksDB is built with io_uring support via liburing for async read I/O. -Some RocksDB settings can be adjusted such as the compression method chosen. See the RocksDB section in the [example config](configuration.md). btrfs users may benefit from disabling compression on RocksDB if CoW is in use. +Some RocksDB settings can be adjusted such as the compression method chosen. See the RocksDB section in the [example config](configuration/examples.md). btrfs users may benefit from disabling compression on RocksDB if CoW is in use. RocksDB troubleshooting can be found [in the RocksDB section of troubleshooting](troubleshooting.md). diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 9f817a68..2f5f989f 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -12,7 +12,7 @@ #### Direct IO -Some filesystems may not like RocksDB using [Direct IO](https://github.com/facebook/rocksdb/wiki/Direct-IO). Direct IO is for non-buffered I/O which improves conduwuit performance, but at least FUSE is a filesystem potentially known to not like this. See the [example config](configuration.md) for disabling it if needed. Issues from Direct IO on unsupported filesystems are usually shown as startup errors. +Some filesystems may not like RocksDB using [Direct IO](https://github.com/facebook/rocksdb/wiki/Direct-IO). Direct IO is for non-buffered I/O which improves conduwuit performance, but at least FUSE is a filesystem potentially known to not like this. See the [example config](configuration/examples.md) for disabling it if needed. Issues from Direct IO on unsupported filesystems are usually shown as startup errors. #### Database corruption @@ -32,7 +32,8 @@ By default, conduwuit uses `TolerateCorruptedTailRecords` as generally these may `AbsoluteConsistency` will fail to start the database if any sign of corruption is detected. `SkipAnyCorruptedRecord` will skip all forms of corruption unless it forbids the database from opening (e.g. too severe). Usage of `SkipAnyCorruptedRecord` voids any support as this may cause more damage and/or leave your database in a permanently inconsistent state, but it may do something if `PointInTime` does not work as a last ditch effort. With this in mind: -- First start conduwuit with the `PointInTime` recovery method. See the [example config](configuration.md) for how to do this using `rocksdb_recovery_mode` + +- First start conduwuit with the `PointInTime` recovery method. See the [example config](configuration/examples.md) for how to do this using `rocksdb_recovery_mode` - If your database successfully opens, clients are recommended to clear their client cache to account for the rollback - Leave your conduwuit running in `PointInTime` for at least 30-60 minutes so as much possible corruption is restored - If all goes will, you should be able to restore back to using `TolerateCorruptedTailRecords` and you have successfully recovered your database diff --git a/docs/turn.md b/docs/turn.md index 7fc8a45a..68c15495 100644 --- a/docs/turn.md +++ b/docs/turn.md @@ -13,7 +13,7 @@ realm= ``` A common way to generate a suitable alphanumeric secret key is by using `pwgen -s 64 1`. -These same values need to be set in conduwuit. See the [example config](configuration.md) in the TURN section for configuring these and restart conduwuit after. +These same values need to be set in conduwuit. See the [example config](configuration/examples.md) in the TURN section for configuring these and restart conduwuit after. ### Run Run the [Coturn](https://hub.docker.com/r/coturn/coturn) image using From 1ca484f90d9d6fb54a87fa9a187c2f6b0251ba85 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 22:08:18 -0400 Subject: [PATCH 0310/2091] explain how to configure conduwuit --- docs/SUMMARY.md | 3 ++- docs/configuration.md | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 docs/configuration.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index fcd8f898..868810f6 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -2,7 +2,8 @@ - [Introduction](introduction.md) - [Differences from upstream Conduit](differences.md) -- [Example configuration](configuration/examples.md) +- [Configuration](configuration.md) + - [Examples](configuration/examples.md) - [Deploying](deploying.md) - [Generic](deploying/generic.md) - [NixOS](deploying/nixos.md) diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 00000000..8b3fec8a --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,18 @@ +# Configuration + +This chapter describes various ways to configure conduwuit. + +## Basics + +Conduwuit uses a config file for the majority of the settings. Please refer to the +[example config file](./configuration/examples.md#example-configuration) for all of those settings. +The config file to use can either be specified on the command line when running conduwuit by specifying the +`-c`, `--config` flag. Alternatively, you can use the environment variable `CONDUWUIT_CONFIG` to specify the config +file to used. + +## Environment variables + +All of the settings that are found in the config file can be specified by using environment variables. +The environment variable names should be all caps and prefixed with `CONDUWUIT_`. +For example, if the setting you are changing is `max_request_size`, then the environment variable to set is +`CONDUWUIT_MAX_REQUEST_SIZE`. From c952db72e7ad011ed464bbdb5cf5b8ee7e8d1319 Mon Sep 17 00:00:00 2001 From: Paul Robertson Date: Wed, 26 Jun 2024 20:50:32 -0400 Subject: [PATCH 0311/2091] remove unused CONDUWUIT_CONFIG_FILE variable --- debian/postinst | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/postinst b/debian/postinst index 223d322b..edc99bd3 100644 --- a/debian/postinst +++ b/debian/postinst @@ -6,7 +6,6 @@ set -e CONDUWUIT_DATABASE_PATH=/var/lib/conduwuit CONDUWUIT_CONFIG_PATH=/etc/conduwuit -CONDUWUIT_CONFIG_FILE="${CONDUWUIT_CONFIG_PATH}/conduwuit.toml" case "$1" in configure) From c2c36197e93242e1d37c45b6eeb4b541475aa560 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Thu, 27 Jun 2024 12:07:09 +0000 Subject: [PATCH 0312/2091] chore(deps): update nixos/nix docker tag to v2.23.1 --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cab57c6f..33d0c6ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ before_script: ci: stage: ci - image: nixos/nix:2.23.0 + image: nixos/nix:2.23.1 script: # Cache CI dependencies - ./bin/nix-build-and-cache ci @@ -83,7 +83,7 @@ ci: artifacts: stage: artifacts - image: nixos/nix:2.23.0 + image: nixos/nix:2.23.1 script: - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl - cp result/bin/conduit x86_64-unknown-linux-musl From 39a1e994eb1b907ceac366695954e26d80590404 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 28 Jun 2024 01:11:26 +0000 Subject: [PATCH 0313/2091] chore(deps): update rust crate log to v0.4.22 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 929fc542..938ae2c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1988,9 +1988,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "loole" From 6bae136720467cb8eeccb8502ff33b9b02d42ad4 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 01:06:26 +0000 Subject: [PATCH 0314/2091] Update Rust crate clap to v4.5.8 --- Cargo.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 938ae2c5..b8b3a9c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -526,9 +526,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" dependencies = [ "clap_builder", "clap_derive", @@ -536,9 +536,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.7" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" dependencies = [ "anstyle", "clap_lex", @@ -546,9 +546,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.5" +version = "4.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" dependencies = [ "heck 0.5.0", "proc-macro2", From e9f4cd58a9cd83440784748cb6587d85b5c56c9f Mon Sep 17 00:00:00 2001 From: Kimiblock Moe Date: Tue, 2 Jul 2024 01:13:31 +0800 Subject: [PATCH 0315/2091] Bind Mount the old matrix-conduit directory --- arch/conduwuit.service | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index 4cc6f850..e2040a18 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -45,6 +45,8 @@ RuntimeDirectory=conduwuit RuntimeDirectoryMode=0750 Environment="CONDUWUIT_CONFIG=/etc/conduwuit/conduwuit.toml" +BindPaths=/var/lib/private/conduwuit:/var/lib/matrix-conduit +BindPaths=/var/lib/private/conduwuit:/var/lib/private/matrix-conduit ExecStart=/usr/bin/conduwuit Restart=on-failure From 20aab1792be1b637bee4e268a7bd8d163321fb63 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 1 Jul 2024 19:25:23 +0000 Subject: [PATCH 0316/2091] fix version format regression 3a51e18ce6 Signed-off-by: Jason Volk --- src/core/version.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/version.rs b/src/core/version.rs index 4a4e8426..bd89bee2 100644 --- a/src/core/version.rs +++ b/src/core/version.rs @@ -27,5 +27,5 @@ fn init_user_agent() -> String { format!("{}/{}", name(), version()) } fn init_version() -> String { option_env!("CONDUWUIT_VERSION_EXTRA") .or(option_env!("CONDUIT_VERSION_EXTRA")) - .map_or(SEMANTIC.to_owned(), |extra| format!("{BRANDING} ({extra})")) + .map_or(SEMANTIC.to_owned(), |extra| format!("{SEMANTIC} ({extra})")) } From 75731671d10d72319f1f6ca3426f6fe678b1b27a Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 2 Jul 2024 14:46:53 -0400 Subject: [PATCH 0317/2091] use `RoomMemberEventContent` for send_join/send_leave Signed-off-by: strawberry --- src/api/server/send_join.rs | 25 +++++++++++-------------- src/api/server/send_leave.rs | 23 ++++++++++------------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 388280e9..7e10307d 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -4,7 +4,10 @@ use std::collections::BTreeMap; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_join_event}, - events::{room::member::MembershipState, StateEventType}, + events::{ + room::member::{MembershipState, RoomMemberEventContent}, + StateEventType, + }, CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; @@ -63,22 +66,16 @@ async fn create_join_event( )); } - let content = value - .get("content") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property."))? - .as_object() - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid."))?; - - let membership: MembershipState = serde_json::from_value( - content - .get("membership") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event content missing membership property."))? + let content: RoomMemberEventContent = serde_json::from_value( + value + .get("content") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property"))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Event has an invalid membership state."))?; + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid"))?; - if membership != MembershipState::Join { + if content.membership != MembershipState::Join { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Not allowed to send a non-join membership event to join endpoint.", @@ -122,7 +119,7 @@ async fn create_join_event( ErrorKind::InvalidParam, "State key does not match sender user", )); - } + }; ruma::signatures::hash_and_sign_event( services().globals.server_name().as_str(), diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index 2680378f..4fdde515 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -4,7 +4,10 @@ use std::collections::BTreeMap; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_leave_event}, - events::{room::member::MembershipState, StateEventType}, + events::{ + room::member::{MembershipState, RoomMemberEventContent}, + StateEventType, + }, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::RawValue as RawJsonValue; @@ -62,22 +65,16 @@ async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJson )); }; - let content = value - .get("content") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property."))? - .as_object() - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event content not an object."))?; - - let membership: MembershipState = serde_json::from_value( - content - .get("membership") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event membership is missing."))? + let content: RoomMemberEventContent = serde_json::from_value( + value + .get("content") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property"))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Event membership state is not valid."))?; + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid"))?; - if membership != MembershipState::Leave { + if content.membership != MembershipState::Leave { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Not allowed to send a non-leave membership event to leave endpoint.", From 339a1fc4e84936c34783cc30fd34f09d2e172eef Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 2 Jul 2024 15:01:11 -0400 Subject: [PATCH 0318/2091] only sign /send_join events if restricted join via room version and `join_authorised_via_users_server` Co-authored-by: Matthias Ahouansou Signed-off-by: strawberry --- src/api/server/send_join.rs | 45 ++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 7e10307d..19172498 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -8,9 +8,10 @@ use ruma::{ room::member::{MembershipState, RoomMemberEventContent}, StateEventType, }, - CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, + CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use service::user_is_local; use tokio::sync::RwLock; use tracing::warn; @@ -121,13 +122,33 @@ async fn create_join_event( )); }; - ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), - &mut value, - &room_version_id, - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; + if content + .join_authorized_via_users_server + .is_some_and(|user| user_is_local(&user)) + && !matches!( + room_version_id, + RoomVersionId::V1 + | RoomVersionId::V2 + | RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + | RoomVersionId::V7 + ) { + ruma::signatures::hash_and_sign_event( + services().globals.server_name().as_str(), + services().globals.keypair(), + &mut value, + &room_version_id, + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; + } + + services() + .rooms + .event_handler + .fetch_required_signing_keys([&value], &pub_key_map) + .await?; let origin: OwnedServerName = serde_json::from_value( serde_json::to_value( @@ -139,12 +160,6 @@ async fn create_join_event( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; - services() - .rooms - .event_handler - .fetch_required_signing_keys([&value], &pub_key_map) - .await?; - let mutex_lock = services() .globals .roomid_mutex_federation @@ -183,7 +198,7 @@ async fn create_join_event( .collect(), // Event field is required if the room version supports restricted join rules. event: Some( - to_raw_value(&CanonicalJsonValue::Object(value.clone())) + to_raw_value(&CanonicalJsonValue::Object(value)) .expect("To raw json should not fail since only change was adding signature"), ), }) From bb27f21ac10949bc5049884b732b41b4764b63c8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 2 Jul 2024 15:32:40 -0400 Subject: [PATCH 0319/2091] simplify getting join_authorized_via_users_server for make/send_join, remove unnecessary async Co-authored-by: Matthias Ahouansou Signed-off-by: strawberry --- src/api/client/membership.rs | 7 +- src/api/server/make_join.rs | 182 +++++++++++++----------- src/api/server/send_join.rs | 14 +- src/service/rooms/state_accessor/mod.rs | 2 +- 4 files changed, 104 insertions(+), 101 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index b09d8e08..8529a9df 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1055,7 +1055,7 @@ async fn join_room_by_id_helper_local( .filter(|user| user_is_local(user)) .collect::>(); - let mut authorized_user: Option = None; + let mut join_authorized_via_users_server: Option = None; if restriction_rooms.iter().any(|restriction_room_id| { services() @@ -1069,10 +1069,9 @@ async fn join_room_by_id_helper_local( .rooms .state_accessor .user_can_invite(room_id, &user, sender_user, &state_lock) - .await .unwrap_or(false) { - authorized_user = Some(user); + join_authorized_via_users_server = Some(user); break; } } @@ -1086,7 +1085,7 @@ async fn join_room_by_id_helper_local( third_party_invite: None, blurhash: services().users.blurhash(sender_user)?, reason: reason.clone(), - join_authorized_via_users_server: authorized_user, + join_authorized_via_users_server, }; // Try normal join first diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index d3d934f7..ca50dcbe 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -7,15 +7,12 @@ use ruma::{ }, StateEventType, TimelineEventType, }, - RoomVersionId, + RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; use tracing::warn; -use crate::{ - service::{pdu::PduBuilder, user_is_local}, - services, Error, Result, Ruma, -}; +use crate::{service::pdu::PduBuilder, services, Error, Result, Ruma}; /// # `GET /_matrix/federation/v1/make_join/{roomId}/{userId}` /// @@ -72,95 +69,42 @@ pub(crate) async fn create_join_event_template_route( } } + let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; + let state_lock = services() .globals .roomid_mutex_state .lock(&body.room_id) .await; - let join_rules_event = - services() + + let join_authorized_via_users_server = if (services() + .rooms + .state_cache + .is_left(&body.user_id, &body.room_id) + .unwrap_or(true)) + && user_can_perform_restricted_join(&body.user_id, &body.room_id, &room_version_id)? + { + let auth_user = services() .rooms - .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomJoinRules, "")?; - - let join_rules_event_content: Option = join_rules_event - .as_ref() - .map(|join_rules_event| { - serde_json::from_str(join_rules_event.content.get()) - .map_err(|_| Error::bad_database("Invalid join rules event in db.")) - }) - .transpose()?; - - let join_authorized_via_users_server = if let Some(join_rules_event_content) = join_rules_event_content { - if let JoinRule::Restricted(r) | JoinRule::KnockRestricted(r) = join_rules_event_content.join_rule { - if r.allow - .iter() - .filter_map(|rule| { - if let AllowRule::RoomMembership(membership) = rule { - Some(membership) - } else { - None - } - }) - .any(|m| { - services() - .rooms - .state_cache - .is_joined(&body.user_id, &m.room_id) - .unwrap_or(false) - }) { - if services() + .state_cache + .room_members(&body.room_id) + .filter_map(Result::ok) + .filter(|user| user.server_name() == services().globals.server_name()) + .find(|user| { + services() .rooms - .state_cache - .is_left(&body.user_id, &body.room_id) - .unwrap_or(true) - { - let members: Vec<_> = services() - .rooms - .state_cache - .room_members(&body.room_id) - .filter_map(Result::ok) - .filter(|user| user_is_local(user)) - .collect(); + .state_accessor + .user_can_invite(&body.room_id, user, &body.user_id, &state_lock) + .unwrap_or(false) + }); - let mut auth_user = None; - - for user in members { - if services() - .rooms - .state_accessor - .user_can_invite(&body.room_id, &user, &body.user_id, &state_lock) - .await - .unwrap_or(false) - { - auth_user = Some(user); - break; - } - } - if auth_user.is_some() { - auth_user - } else { - return Err(Error::BadRequest( - ErrorKind::UnableToGrantJoin, - "No user on this server is able to assist in joining.", - )); - } - } else { - // If the user has any state other than leave, either: - // - the auth_check will deny them (ban, knock - (until/unless MSC4123 is - // merged)) - // - they are able to join via other methods (invite) - // - they are already in the room (join) - None - } - } else { - return Err(Error::BadRequest( - ErrorKind::UnableToAuthorizeJoin, - "User is not known to be in any required room.", - )); - } + if auth_user.is_some() { + auth_user } else { - None + return Err(Error::BadRequest( + ErrorKind::UnableToGrantJoin, + "No user on this server is able to assist in joining.", + )); } } else { None @@ -231,3 +175,71 @@ pub(crate) async fn create_join_event_template_route( event: to_raw_value(&pdu_json).expect("CanonicalJson can be serialized to JSON"), }) } + +/// Checks whether the given user can join the given room via a restricted join. +/// This doesn't check the current user's membership. This should be done +/// externally, either by using the state cache or attempting to authorize the +/// event. +pub(crate) fn user_can_perform_restricted_join( + user_id: &UserId, room_id: &RoomId, room_version_id: &RoomVersionId, +) -> Result { + let join_rules_event = + services() + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomJoinRules, "")?; + + let Some(join_rules_event_content) = join_rules_event + .as_ref() + .map(|join_rules_event| { + serde_json::from_str::(join_rules_event.content.get()).map_err(|e| { + warn!("Invalid join rules event in database: {e}"); + Error::bad_database("Invalid join rules event in database") + }) + }) + .transpose()? + else { + return Ok(false); + }; + + if matches!( + room_version_id, + RoomVersionId::V1 + | RoomVersionId::V2 + | RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + | RoomVersionId::V7 + ) { + return Ok(false); + } + + let (JoinRule::Restricted(r) | JoinRule::KnockRestricted(r)) = join_rules_event_content.join_rule else { + return Ok(false); + }; + + if r.allow + .iter() + .filter_map(|rule| { + if let AllowRule::RoomMembership(membership) = rule { + Some(membership) + } else { + None + } + }) + .any(|m| { + services() + .rooms + .state_cache + .is_joined(user_id, &m.room_id) + .unwrap_or(false) + }) { + Ok(true) + } else { + Err(Error::BadRequest( + ErrorKind::UnableToAuthorizeJoin, + "User is not known to be in any required room.", + )) + } +} diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 19172498..82cfc45e 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -8,7 +8,7 @@ use ruma::{ room::member::{MembershipState, RoomMemberEventContent}, StateEventType, }, - CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, + CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::user_is_local; @@ -125,16 +125,8 @@ async fn create_join_event( if content .join_authorized_via_users_server .is_some_and(|user| user_is_local(&user)) - && !matches!( - room_version_id, - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - ) { + && super::user_can_perform_restricted_join(&sender, room_id, &room_version_id).unwrap_or_default() + { ruma::signatures::hash_and_sign_event( services().globals.server_name().as_str(), services().globals.keypair(), diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 3df0e123..ce884fe5 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -299,7 +299,7 @@ impl Service { }) } - pub async fn user_can_invite( + pub fn user_can_invite( &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &mutex_map::Guard<()>, ) -> Result { let content = to_raw_value(&RoomMemberEventContent::new(MembershipState::Invite)) From 8deba94a1297e96431e3e7722be190da5161ced5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 2 Jul 2024 15:40:45 -0400 Subject: [PATCH 0320/2091] update various deps, update rocksdb to v9.3.1 Signed-off-by: strawberry --- Cargo.lock | 52 ++++----- Cargo.toml | 6 +- deps/rust-rocksdb/Cargo.toml | 2 +- flake.lock | 217 ++++++++++++++++++++++++----------- flake.nix | 2 +- 5 files changed, 182 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8b3a9c8..d2599be8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -194,7 +194,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-util", "itoa", "matchit", @@ -298,7 +298,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-util", "pin-project-lite", "rustls 0.21.12", @@ -474,9 +474,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.101" +version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" +checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" dependencies = [ "jobserver", "libc", @@ -633,7 +633,7 @@ dependencies = [ "futures-util", "hmac", "http 1.1.0", - "hyper 1.3.1", + "hyper 1.4.0", "image", "ipaddress", "jsonwebtoken", @@ -720,7 +720,7 @@ dependencies = [ "conduit_service", "http 1.1.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-util", "log", "regex", @@ -1631,9 +1631,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" dependencies = [ "bytes", "futures-channel", @@ -1658,7 +1658,7 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-util", "rustls 0.23.10", "rustls-native-certs", @@ -1683,16 +1683,16 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" dependencies = [ "bytes", "futures-channel", "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.3.1", + "hyper 1.4.0", "pin-project-lite", "socket2", "tokio", @@ -2185,9 +2185,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c165a9ab64cf766f73521c0dd2cfdff64f488b8f0b3e621face3462d3db536d7" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", @@ -2260,9 +2260,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" +checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" dependencies = [ "memchr", ] @@ -2334,7 +2334,7 @@ dependencies = [ "glob", "once_cell", "opentelemetry", - "ordered-float 4.2.0", + "ordered-float 4.2.1", "percent-encoding", "rand", "thiserror", @@ -2353,9 +2353,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.0" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" +checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be" dependencies = [ "num-traits", ] @@ -2835,7 +2835,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.3.1", + "hyper 1.4.0", "hyper-rustls", "hyper-util", "ipnet", @@ -3091,8 +3091,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.22.0+9.2.1" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=e9e1cb5ba92a44ea225fe8d13b31aa23621b9035#e9e1cb5ba92a44ea225fe8d13b31aa23621b9035" +version = "0.23.0+9.3.1" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=b4887edfb84771336930855727390edec07d63fa#b4887edfb84771336930855727390edec07d63fa" dependencies = [ "bindgen", "bzip2-sys", @@ -3108,8 +3108,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.26.0" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=e9e1cb5ba92a44ea225fe8d13b31aa23621b9035#e9e1cb5ba92a44ea225fe8d13b31aa23621b9035" +version = "0.27.0" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=b4887edfb84771336930855727390edec07d63fa#b4887edfb84771336930855727390edec07d63fa" dependencies = [ "libc", "rust-librocksdb-sys", @@ -3502,9 +3502,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.118" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d947f6b3163d8857ea16c4fa0dd4840d52f3041039a85decd46867eb1abef2e4" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "itoa", "ryu", diff --git a/Cargo.toml b/Cargo.toml index d9329273..49aa2a45 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -117,7 +117,7 @@ version = "1.0.203" features = ["rc"] [workspace.dependencies.serde_json] -version = "1.0.117" +version = "1.0.120" features = ["raw_value"] # Used for appservice registration files @@ -212,7 +212,7 @@ features = ["alloc"] default-features = false [workspace.dependencies.hyper] -version = "1.3.1" +version = "1.4.0" features = [ "server", "http1", @@ -220,7 +220,7 @@ features = [ ] [workspace.dependencies.hyper-util] -version = "0.1.5" +version = "0.1.6" features = [ "client", "server-auto", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index ff549845..03efc61c 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/zaidoon1/rust-rocksdb" -rev = "e9e1cb5ba92a44ea225fe8d13b31aa23621b9035" +rev = "b4887edfb84771336930855727390edec07d63fa" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index 7cd38d07..1a064510 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1711742460, - "narHash": "sha256-0O4v6e4a1toxXZ2gf5INhg4WPE5C5T+SVvsBt+45Mcc=", + "lastModified": 1717279440, + "narHash": "sha256-kH04ReTjxOpQumgWnqy40vvQLSnLGxWP6RF3nq5Esrk=", "owner": "zhaofengli", "repo": "attic", - "rev": "4dbdbee45728d8ce5788db6461aaaa89d98081f0", + "rev": "717cc95983cdc357bc347d70be20ced21f935843", "type": "github" }, "original": { @@ -31,11 +31,11 @@ "pre-commit-hooks": "pre-commit-hooks" }, "locked": { - "lastModified": 1717420532, - "narHash": "sha256-OCCmI69EMaA4BcxRKrXJsx5Ozua2f/PKEy4aJbE7ziM=", + "lastModified": 1719923519, + "narHash": "sha256-7Rhljj2fsklFRsu+eq7N683Z9qukmreMEj5C1GqCrSA=", "owner": "cachix", "repo": "cachix", - "rev": "5727f0676f08a4b41ed13d403ec64dcce989f6e5", + "rev": "4e9e71f78b9500fa6210cf1eaa4d75bdbab777c3", "type": "github" }, "original": { @@ -45,14 +45,47 @@ "type": "github" } }, + "cachix_2": { + "inputs": { + "devenv": "devenv_2", + "flake-compat": [ + "cachix", + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "cachix", + "devenv", + "nixpkgs" + ], + "pre-commit-hooks": [ + "cachix", + "devenv", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1712055811, + "narHash": "sha256-7FcfMm5A/f02yyzuavJe06zLa9hcMHsagE28ADcmQvk=", + "owner": "cachix", + "repo": "cachix", + "rev": "02e38da89851ec7fec3356a5c04bc8349cae0e30", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "cachix", + "type": "github" + } + }, "complement": { "flake": false, "locked": { - "lastModified": 1715700731, - "narHash": "sha256-cie+b5N/TQAFD8vF/XbqfyFJkFU0qUPDbtJQDm/TfQc=", + "lastModified": 1719903368, + "narHash": "sha256-PPzgxM4Bir+Zh9FUV/v+RBxEYeJxYVmi/BYo3uqt268=", "owner": "matrix-org", "repo": "complement", - "rev": "8587fb3cbe746754b2c883ff6c818ca4d987d0a5", + "rev": "bc97f1ddc1cd7485faf80c8935ee2641f3e1b57c", "type": "github" }, "original": { @@ -70,11 +103,11 @@ ] }, "locked": { - "lastModified": 1702918879, - "narHash": "sha256-tWJqzajIvYcaRWxn+cLUB9L9Pv4dQ3Bfit/YjU5ze3g=", + "lastModified": 1717025063, + "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=", "owner": "ipetkov", "repo": "crane", - "rev": "7195c00c272fdd92fc74e7d5a0a2844b9fadb2fb", + "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e", "type": "github" }, "original": { @@ -106,14 +139,49 @@ }, "devenv": { "inputs": { + "cachix": "cachix_2", "flake-compat": [ "cachix", "flake-compat" ], + "nix": "nix_2", + "nixpkgs": [ + "cachix", + "nixpkgs" + ], + "pre-commit-hooks": [ + "cachix", + "pre-commit-hooks" + ] + }, + "locked": { + "lastModified": 1719759336, + "narHash": "sha256-3a34VL/QnHprl5gMy9xlx6d8J+iNp+W88Ex8smkgH9M=", + "owner": "cachix", + "repo": "devenv", + "rev": "bb32aa986f2f695385e54428d0eaf7d05b31466e", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "devenv_2": { + "inputs": { + "flake-compat": [ + "cachix", + "devenv", + "cachix", + "flake-compat" + ], "nix": "nix", "nixpkgs": "nixpkgs_2", "poetry2nix": "poetry2nix", "pre-commit-hooks": [ + "cachix", + "devenv", "cachix", "pre-commit-hooks" ] @@ -281,24 +349,6 @@ "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", "type": "github" }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_4": { - "inputs": { - "systems": "systems_3" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", - "type": "github" - }, "original": { "owner": "numtide", "ref": "main", @@ -331,11 +381,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1716565485, - "narHash": "sha256-4R19aJNQYs6vb0/Hz4bWT56YN1P1DkFL/sxdE4Yj0CE=", + "lastModified": 1719025212, + "narHash": "sha256-kD0yhjNStqC6uFqC1AxBwUpc/HlSFtiKrV+gwDyroDc=", "owner": "axboe", "repo": "liburing", - "rev": "b90c0e670a93caabbebe2d9e24ff85cece4cfe0e", + "rev": "7b3245583069bd481190c9da18f22e9fc8c3a805", "type": "github" }, "original": { @@ -349,6 +399,8 @@ "inputs": { "flake-compat": "flake-compat_2", "nixpkgs": [ + "cachix", + "devenv", "cachix", "devenv", "nixpkgs" @@ -356,11 +408,11 @@ "nixpkgs-regression": "nixpkgs-regression" }, "locked": { - "lastModified": 1708577783, - "narHash": "sha256-92xq7eXlxIT5zFNccLpjiP7sdQqQI30Gyui2p/PfKZM=", + "lastModified": 1712911606, + "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", "owner": "domenkozar", "repo": "nix", - "rev": "ecd0af0c1f56de32cbad14daa1d82a132bf298f8", + "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", "type": "github" }, "original": { @@ -389,6 +441,8 @@ "nix-github-actions": { "inputs": { "nixpkgs": [ + "cachix", + "devenv", "cachix", "devenv", "poetry2nix", @@ -409,6 +463,35 @@ "type": "github" } }, + "nix_2": { + "inputs": { + "flake-compat": [ + "cachix", + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "cachix", + "devenv", + "nixpkgs" + ], + "nixpkgs-regression": "nixpkgs-regression_2" + }, + "locked": { + "lastModified": 1712911606, + "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", + "owner": "domenkozar", + "repo": "nix", + "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.21", + "repo": "nix", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1711401922, @@ -441,6 +524,22 @@ "type": "github" } }, + "nixpkgs-regression_2": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, "nixpkgs-stable": { "locked": { "lastModified": 1711460390, @@ -459,11 +558,11 @@ }, "nixpkgs-stable_2": { "locked": { - "lastModified": 1710695816, - "narHash": "sha256-3Eh7fhEID17pv9ZxrPwCLfqXnYP006RKzSs0JptsN84=", + "lastModified": 1718811006, + "narHash": "sha256-0Y8IrGhRmBmT7HHXlxxepg2t8j1X90++qRN3lukGaIk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "614b4613980a522ba49f0d194531beddbb7220d3", + "rev": "03d771e513ce90147b65fe922d87d3a0356fc125", "type": "github" }, "original": { @@ -491,11 +590,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1715534503, - "narHash": "sha256-5ZSVkFadZbFP1THataCaSf0JH2cAH3S29hU9rrxTEqk=", + "lastModified": 1719848872, + "narHash": "sha256-H3+EC5cYuq+gQW8y0lSrrDZfH71LB4DAf+TDFyvwCNA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2057814051972fa1453ddfb0d98badbea9b83c06", + "rev": "00d80d13810dbfea8ab4ed1009b09100cca86ba8", "type": "github" }, "original": { @@ -526,6 +625,8 @@ "flake-utils": "flake-utils_2", "nix-github-actions": "nix-github-actions", "nixpkgs": [ + "cachix", + "devenv", "cachix", "devenv", "nixpkgs" @@ -548,7 +649,6 @@ "pre-commit-hooks": { "inputs": { "flake-compat": "flake-compat_4", - "flake-utils": "flake-utils_3", "gitignore": "gitignore", "nixpkgs": [ "cachix", @@ -557,11 +657,11 @@ "nixpkgs-stable": "nixpkgs-stable_2" }, "locked": { - "lastModified": 1715609711, - "narHash": "sha256-/5u29K0c+4jyQ8x7dUIEUWlz2BoTSZWUP2quPwFCE7M=", + "lastModified": 1719259945, + "narHash": "sha256-F1h+XIsGKT9TkGO3omxDLEb/9jOOsI6NnzsXFsZhry4=", "owner": "cachix", "repo": "pre-commit-hooks.nix", - "rev": "c182c876690380f8d3b9557c4609472ebfa1b141", + "rev": "0ff4381bbb8f7a52ca4a851660fc7a437a4c6e07", "type": "github" }, "original": { @@ -573,16 +673,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1716773462, - "narHash": "sha256-5kUH+XK+2lbFfUgbxuNy3YMLHbp6scfWPdtc8za1wDM=", + "lastModified": 1719949653, + "narHash": "sha256-DYx7XHH2GEh17GukKhXs6laM6l+eugCmRkF0adpi9wk=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "c8a1450231e9c608edf535538dbe8ca1a8d2f3bc", + "rev": "a935c0273e1ba44eacf88ce3685a9b9831486155", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.2.1", + "ref": "v9.3.1", "repo": "rocksdb", "type": "github" } @@ -595,7 +695,7 @@ "crane": "crane_2", "fenix": "fenix", "flake-compat": "flake-compat_5", - "flake-utils": "flake-utils_4", + "flake-utils": "flake-utils_3", "liburing": "liburing", "nix-filter": "nix-filter", "nixpkgs": "nixpkgs_4", @@ -648,21 +748,6 @@ "repo": "default", "type": "github" } - }, - "systems_3": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index acaf0dae..b9249daa 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,7 @@ nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-unstable"; # https://github.com/girlbossceo/rocksdb/commit/db6df0b185774778457dabfcbd822cb81760cade - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.2.1"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.3.1"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From 652faafe6e6824fea1081b2a1f7d7a0560401f4c Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 2 Jul 2024 15:43:10 -0400 Subject: [PATCH 0321/2091] bump ruma Signed-off-by: strawberry --- Cargo.lock | 47 ++++++++++++++++++++++++----------------- Cargo.toml | 2 +- src/api/client/media.rs | 3 +++ 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d2599be8..edc41835 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -760,7 +760,7 @@ dependencies = [ "regex", "reqwest", "ruma", - "ruma-identifiers-validation", + "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0)", "rustyline-async", "serde", "serde_json", @@ -2657,9 +2657,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993" +checksum = "8746739f11d39ce5ad5c2520a9b75285310dbfe78c541ccf832d38615765aec0" dependencies = [ "bitflags 2.6.0", "memchr", @@ -2669,9 +2669,9 @@ dependencies = [ [[package]] name = "pulldown-cmark-escape" -version = "0.10.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd348ff538bc9caeda7ee8cad2d1d48236a1f443c1fa3913c6a02fe0043b1dd3" +checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" [[package]] name = "quick-error" @@ -2895,7 +2895,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "assign", "js_int", @@ -2915,7 +2915,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "js_int", "ruma-common", @@ -2927,7 +2927,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "as_variant", "assign", @@ -2950,7 +2950,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "as_variant", "base64 0.22.1", @@ -2963,7 +2963,7 @@ dependencies = [ "percent-encoding", "rand", "regex", - "ruma-identifiers-validation", + "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0)", "ruma-macros", "serde", "serde_html_form", @@ -2980,7 +2980,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "as_variant", "indexmap 2.2.6", @@ -2990,7 +2990,7 @@ dependencies = [ "pulldown-cmark", "regex", "ruma-common", - "ruma-identifiers-validation", + "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0)", "ruma-macros", "serde", "serde_json", @@ -3003,7 +3003,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "js_int", "ruma-common", @@ -3012,6 +3012,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ruma-identifiers-validation" +version = "0.9.5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +dependencies = [ + "js_int", + "thiserror", +] + [[package]] name = "ruma-identifiers-validation" version = "0.9.5" @@ -3024,7 +3033,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "js_int", "ruma-common", @@ -3034,13 +3043,13 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "ruma-identifiers-validation", + "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0)", "serde", "syn 2.0.68", "toml", @@ -3049,7 +3058,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "js_int", "ruma-common", @@ -3061,7 +3070,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3077,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 49aa2a45..755ff46a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -274,7 +274,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "fd686e77950680462377c9105dfb4136dd49c7a0" +rev = "9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" features = [ "compat", "rand", diff --git a/src/api/client/media.rs b/src/api/client/media.rs index d454ae9c..cfb10c3d 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -1,3 +1,5 @@ +#![allow(deprecated)] + use std::{io::Cursor, sync::Arc, time::Duration}; use image::io::Reader as ImgReader; @@ -396,6 +398,7 @@ pub(crate) async fn get_content_thumbnail_route( media_id: body.media_id.clone(), timeout_ms: body.timeout_ms, allow_redirect: body.allow_redirect, + animated: body.animated, }, ) .await From 9115901c66bcac350d38522061343f63fa969212 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 2 Jul 2024 15:49:17 -0400 Subject: [PATCH 0322/2091] update complement results Signed-off-by: strawberry --- .../complement/test_results.jsonl | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 51cd57d4..d1e322ab 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -42,6 +42,7 @@ {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/knock_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/non-state_membership_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/regular_event"} +{"Action":"fail","Test":"TestContentMediaV1"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation/good_connectivity"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation/interrupted_connectivity"} @@ -111,8 +112,8 @@ {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_in#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to#01"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock#01"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_in_the_room_can_reject_a_knock"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_in_the_room_can_reject_a_knock#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_that_has_already_knocked_is_allowed_to_knock_again_on_the_same_room"} @@ -120,8 +121,8 @@ {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_that_has_knocked_on_a_local_room_can_rescind_their_knock_and_then_knock_again"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it#01"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail#01"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail#01"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/Change_the_join_rule_of_a_room_from_'invite'_to_'knock'"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/Change_the_join_rule_of_a_room_from_'invite'_to_'knock'#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail"} @@ -135,23 +136,38 @@ {"Action":"fail","Test":"TestMediaFilenames/Parallel"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'_over_/_matrix/client/v1/media/download"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'_over_/_matrix/client/v1/media/download"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'_over_/_matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name_over__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_upload_with_ASCII_file_name"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name_over__matrix/client/v1/media/download"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally_over__matrix/client/v1/media/download"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation_via__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_upload_with_Unicode_file_name"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline_via__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline_via__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments"} +{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments_via__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaWithoutFileName"} {"Action":"pass","Test":"TestMediaWithoutFileName/parallel"} {"Action":"pass","Test":"TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_locally"} {"Action":"pass","Test":"TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_over_federation"} {"Action":"pass","Test":"TestMediaWithoutFileName/parallel/Can_upload_without_a_file_name"} +{"Action":"fail","Test":"TestMediaWithoutFileNameCSMediaV1"} +{"Action":"fail","Test":"TestMediaWithoutFileNameCSMediaV1/parallel"} +{"Action":"fail","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_locally"} +{"Action":"fail","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_over_federation"} +{"Action":"pass","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_upload_without_a_file_name"} {"Action":"fail","Test":"TestNetworkPartitionOrdering"} {"Action":"fail","Test":"TestOutboundFederationIgnoresMissingEventWithBadJSONForRoomVersion6"} {"Action":"pass","Test":"TestOutboundFederationProfile"} @@ -191,7 +207,7 @@ {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinLocalUser"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinLocalUserInMSC3787Room"} -{"Action":"fail","Test":"TestRestrictedRoomsSpacesSummaryFederation"} +{"Action":"pass","Test":"TestRestrictedRoomsSpacesSummaryFederation"} {"Action":"fail","Test":"TestRestrictedRoomsSpacesSummaryLocal"} {"Action":"skip","Test":"TestSendJoinPartialStateResponse"} {"Action":"pass","Test":"TestSyncOmitsStateChangeOnFilteredEvents"} From 28ac3790c281bd164d124dfc3983a11b18ed4ab6 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 2 Jul 2024 16:42:07 -0400 Subject: [PATCH 0323/2091] sync upstream spaces/hierarchy federation MR also had to fix a million clippy lints fix(spaces): deal with hierarchy recursion fix(spaces): properly handle max_depth refactor(spaces): token scheme to prevent clients from modifying max_depth and suggested_only perf(spaces): use tokens to skip to room to start populating results at feat(spaces): request hierarchy from servers in via field of child event Co-authored-by: Matthias Ahouansou Signed-off-by: strawberry --- src/api/client/space.rs | 18 +- src/service/rooms/spaces/mod.rs | 1014 ++++++----------------- src/service/rooms/spaces/tests.rs | 145 ++++ src/service/rooms/state_accessor/mod.rs | 38 +- src/service/rooms/state_cache/mod.rs | 2 + 5 files changed, 433 insertions(+), 784 deletions(-) create mode 100644 src/service/rooms/spaces/tests.rs diff --git a/src/api/client/space.rs b/src/api/client/space.rs index 61431c01..e00171c3 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -2,10 +2,10 @@ use std::str::FromStr; use ruma::{ api::client::{error::ErrorKind, space::get_hierarchy}, - uint, UInt, + UInt, }; -use crate::{service::rooms::spaces::PagnationToken, services, Error, Result, Ruma}; +use crate::{service::rooms::spaces::PaginationToken, services, Error, Result, Ruma}; /// # `GET /_matrix/client/v1/rooms/{room_id}/hierarchy` /// @@ -14,12 +14,10 @@ use crate::{service::rooms::spaces::PagnationToken, services, Error, Result, Rum pub(crate) async fn get_hierarchy_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let limit: usize = body + let limit = body .limit - .unwrap_or_else(|| uint!(10)) - .try_into() - .unwrap_or(10) - .min(100); + .unwrap_or_else(|| UInt::from(10_u32)) + .min(UInt::from(100_u32)); let max_depth = body .max_depth @@ -29,7 +27,7 @@ pub(crate) async fn get_hierarchy_route(body: Ruma) let key = body .from .as_ref() - .and_then(|s| PagnationToken::from_str(s).ok()); + .and_then(|s| PaginationToken::from_str(s).ok()); // Should prevent unexpeded behaviour in (bad) clients if let Some(ref token) = key { @@ -47,8 +45,8 @@ pub(crate) async fn get_hierarchy_route(body: Ruma) .get_client_hierarchy( sender_user, &body.room_id, - limit, - key.map_or(0, |token| token.skip.try_into().unwrap_or(0)), + limit.try_into().unwrap_or(10), + key.map_or(vec![], |token| token.short_room_ids), max_depth.try_into().unwrap_or(3), body.suggested_only, ) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index bc7703a3..3dfbcf3c 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -1,10 +1,13 @@ +mod tests; + use std::{ + collections::VecDeque, fmt::{Display, Formatter}, str::FromStr, sync::Arc, }; -use conduit::{debug_info, Error, Result, Server}; +use conduit::{debug_info, Server}; use database::Database; use lru_cache::LruCache; use ruma::{ @@ -18,8 +21,10 @@ use ruma::{ events::{ room::{ avatar::RoomAvatarEventContent, + canonical_alias::RoomCanonicalAliasEventContent, create::RoomCreateEventContent, - join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent, RoomMembership}, + join_rules::{JoinRule, RoomJoinRulesEventContent}, + topic::RoomTopicEventContent, }, space::child::{HierarchySpaceChildEvent, SpaceChildEventContent}, StateEventType, @@ -31,225 +36,43 @@ use ruma::{ use tokio::sync::Mutex; use tracing::{debug, error, warn}; -use crate::{server_is_ours, services}; +use crate::{services, Error, Result}; pub struct CachedSpaceHierarchySummary { summary: SpaceHierarchyParentSummary, } -enum SummaryAccessibility { +pub enum SummaryAccessibility { Accessible(Box), Inaccessible, } -struct Arena { - nodes: Vec, - max_depth: usize, - first_untraversed: Option, -} - -struct Node { - parent: Option, - // Next meaning: - // --> - // o o o o - next_sibling: Option, - // First meaning: - // | - // v - // o o o o - first_child: Option, - room_id: OwnedRoomId, - via: Vec, - traversed: bool, -} - -#[derive(Clone, Copy, PartialEq, Debug, PartialOrd)] -struct NodeId { - index: usize, -} - -impl Arena { - /// Checks if a given node is traversed - fn traversed(&self, id: NodeId) -> Option { Some(self.get(id)?.traversed) } - - /// Gets the previous sibling of a given node - fn next_sibling(&self, id: NodeId) -> Option { self.get(id)?.next_sibling } - - /// Gets the parent of a given node - fn parent(&self, id: NodeId) -> Option { self.get(id)?.parent } - - /// Gets the last child of a given node - fn first_child(&self, id: NodeId) -> Option { self.get(id)?.first_child } - - /// Sets traversed to true for a given node - fn traverse(&mut self, id: NodeId) { self.nodes[id.index].traversed = true; } - - /// Gets the node of a given id - fn get(&self, id: NodeId) -> Option<&Node> { self.nodes.get(id.index) } - - /// Gets a mutable reference of a node of a given id - fn get_mut(&mut self, id: NodeId) -> Option<&mut Node> { self.nodes.get_mut(id.index) } - - /// Returns the first untraversed node, marking it as traversed in the - /// process - fn first_untraversed(&mut self) -> Option { - if self.nodes.is_empty() { - None - } else if let Some(untraversed) = self.first_untraversed { - let mut current = untraversed; - - self.traverse(untraversed); - - // Possible paths: - // 1) Next child exists, and hence is not traversed - // 2) Next child does not exist, so go to the parent, then repeat - // 3) If both the parent and child do not exist, then we have just traversed the - // whole space tree. - // - // You should only ever encounter a traversed node when going up through parents - while self.traversed(current) == Some(true) { - if let Some(next) = self.next_sibling(current) { - current = next; - } else if let Some(parent) = self.parent(current) { - current = parent; - } else { - break; - } - } - - // Traverses down the children until it reaches one without children - while let Some(child) = self.first_child(current) { - current = child; - } - - if self.traversed(current)? { - self.first_untraversed = None; - } else { - self.first_untraversed = Some(current); - } - - Some(untraversed) - } else { - None - } - } - - /// Adds all the given nodes as children of the parent node - fn push(&mut self, parent: NodeId, mut children: Vec<(OwnedRoomId, Vec)>) { - if children.is_empty() { - self.traverse(parent); - } else if self.nodes.get(parent.index).is_some() { - let mut parents = vec![( - parent, - self.get(parent) - .expect("It is some, as above") - .room_id - // Cloning cause otherwise when iterating over the parents, below, there would - // be a mutable and immutable reference to self.nodes - .clone(), - )]; - - while let Some(parent) = self.parent(parents.last().expect("Has at least one value, as above").0) { - parents.push(( - parent, - self.get(parent) - .expect("It is some, as above") - .room_id - .clone(), - )); - } - - // If at max_depth, don't add new rooms - if self.max_depth < parents.len() { - return; - } - - children.reverse(); - - let mut next_id = None; - - for (child, via) in children { - // Prevent adding a child which is a parent (recursion) - if !parents.iter().any(|parent| parent.1 == child) { - self.nodes.push(Node { - parent: Some(parent), - next_sibling: next_id, - first_child: None, - room_id: child, - traversed: false, - via, - }); - - next_id = Some(NodeId { - index: self.nodes.len() - 1, - }); - } - } - - if self.first_untraversed.is_none() - || parent - >= self - .first_untraversed - .expect("Should have already continued if none") - { - self.first_untraversed = next_id; - } - - self.traverse(parent); - - // This is done as if we use an if-let above, we cannot reference self.nodes - // above as then we would have multiple mutable references - let node = self - .get_mut(parent) - .expect("Must be some, as inside this block"); - - node.first_child = next_id; - } - } - - fn new(root: OwnedRoomId, max_depth: usize) -> Self { - let zero_depth = max_depth == 0; - - Self { - nodes: vec![Node { - parent: None, - next_sibling: None, - first_child: None, - room_id: root, - traversed: zero_depth, - via: vec![], - }], - max_depth, - first_untraversed: if zero_depth { - None - } else { - Some(NodeId { - index: 0, - }) - }, - } - } -} - -// Note: perhaps use some better form of token rather than just room count +// TODO: perhaps use some better form of token rather than just room count #[derive(Debug, Eq, PartialEq)] -pub struct PagnationToken { - pub skip: UInt, +pub struct PaginationToken { + /// Path down the hierarchy of the room to start the response at, + /// excluding the root space. + pub short_room_ids: Vec, pub limit: UInt, pub max_depth: UInt, pub suggested_only: bool, } -impl FromStr for PagnationToken { +impl FromStr for PaginationToken { type Err = Error; fn from_str(value: &str) -> Result { let mut values = value.split('_'); let mut pag_tok = || { + let mut rooms = vec![]; + + for room in values.next()?.split(',') { + rooms.push(u64::from_str(room).ok()?); + } + Some(Self { - skip: UInt::from_str(values.next()?).ok()?, + short_room_ids: rooms, limit: UInt::from_str(values.next()?).ok()?, max_depth: UInt::from_str(values.next()?).ok()?, suggested_only: { @@ -278,9 +101,20 @@ impl FromStr for PagnationToken { } } -impl Display for PagnationToken { +impl Display for PaginationToken { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!(f, "{}_{}_{}_{}", self.skip, self.limit, self.max_depth, self.suggested_only) + write!( + f, + "{}_{}_{}_{}", + self.short_room_ids + .iter() + .map(ToString::to_string) + .collect::>() + .join(","), + self.limit, + self.max_depth, + self.suggested_only + ) } } @@ -290,7 +124,6 @@ impl Display for PagnationToken { enum Identifier<'a> { UserId(&'a UserId), ServerName(&'a ServerName), - None, } pub struct Service { @@ -343,15 +176,15 @@ impl Service { }) } - ///Gets the response for the space hierarchy over federation request + /// Gets the response for the space hierarchy over federation request /// - ///Panics if the room does not exist, so a check if the room exists should + /// Errors if the room does not exist, so a check if the room exists should /// be done pub async fn get_federation_hierarchy( &self, room_id: &RoomId, server_name: &ServerName, suggested_only: bool, ) -> Result { match self - .get_summary_and_children_local(&room_id.to_owned(), Identifier::None) + .get_summary_and_children_local(&room_id.to_owned(), Identifier::ServerName(server_name)) .await? { Some(SummaryAccessibility::Accessible(room)) => { @@ -386,6 +219,7 @@ impl Service { } } + /// Gets the summary of a space using solely local information async fn get_summary_and_children_local( &self, current_room: &OwnedRoomId, identifier: Identifier<'_>, ) -> Result> { @@ -402,7 +236,7 @@ impl Service { &cached.summary.join_rule, &identifier, &cached.summary.allowed_room_ids, - )? { + ) { Some(SummaryAccessibility::Accessible(Box::new(cached.summary.clone()))) } else { Some(SummaryAccessibility::Inaccessible) @@ -433,11 +267,11 @@ impl Service { ) } + /// Gets the summary of a space using solely federation + #[tracing::instrument(skip(self))] async fn get_summary_and_children_federation( &self, current_room: &OwnedRoomId, suggested_only: bool, user_id: &UserId, via: &[OwnedServerName], ) -> Result> { - debug_info!("servers via for federation hierarchy: {via:?}"); - for server in via { debug_info!("Asking {server} for /hierarchy"); if let Ok(response) = services() @@ -506,21 +340,24 @@ impl Service { &response.room.join_rule, &Identifier::UserId(user_id), &response.room.allowed_room_ids, - )? { + ) { return Ok(Some(SummaryAccessibility::Accessible(Box::new(summary.clone())))); } return Ok(Some(SummaryAccessibility::Inaccessible)); } - - self.roomid_spacehierarchy_cache - .lock() - .await - .insert(current_room.clone(), None); } + + self.roomid_spacehierarchy_cache + .lock() + .await + .insert(current_room.clone(), None); + Ok(None) } + /// Gets the summary of a space using either local or remote (federation) + /// sources async fn get_summary_and_children_client( &self, current_room: &OwnedRoomId, suggested_only: bool, user_id: &UserId, via: &[OwnedServerName], ) -> Result> { @@ -555,9 +392,12 @@ impl Service { .transpose()? .unwrap_or(JoinRule::Invite); - let allowed_room_ids = allowed_room_ids(join_rule.clone()); + let allowed_room_ids = services() + .rooms + .state_accessor + .allowed_room_ids(join_rule.clone()); - if !is_accessable_child(current_room, &join_rule.clone().into(), identifier, &allowed_room_ids)? { + if !is_accessable_child(current_room, &join_rule.clone().into(), identifier, &allowed_room_ids) { debug!("User is not allowed to see room {room_id}"); // This error will be caught later return Err(Error::BadRequest(ErrorKind::forbidden(), "User is not allowed to see the room")); @@ -569,7 +409,12 @@ impl Service { canonical_alias: services() .rooms .state_accessor - .get_canonical_alias(room_id)?, + .room_state_get(room_id, &StateEventType::RoomCanonicalAlias, "")? + .map_or(Ok(None), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomCanonicalAliasEventContent| c.alias) + .map_err(|_| Error::bad_database("Invalid canonical alias event in database.")) + })?, name: services().rooms.state_accessor.get_name(room_id)?, num_joined_members: services() .rooms @@ -585,8 +430,15 @@ impl Service { topic: services() .rooms .state_accessor - .get_room_topic(room_id) - .unwrap_or(None), + .room_state_get(room_id, &StateEventType::RoomTopic, "")? + .map_or(Ok(None), |s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomTopicEventContent| Some(c.topic)) + .map_err(|_| { + error!("Invalid room topic event in database for room {}", room_id); + Error::bad_database("Invalid room topic event in database.") + }) + })?, world_readable: services().rooms.state_accessor.is_world_readable(room_id)?, guest_can_join: services().rooms.state_accessor.guest_can_join(room_id)?, avatar_url: services() @@ -619,116 +471,134 @@ impl Service { }) } - // TODO: make this a lot less messy pub async fn get_client_hierarchy( - &self, sender_user: &UserId, room_id: &RoomId, limit: usize, skip: usize, max_depth: usize, + &self, sender_user: &UserId, room_id: &RoomId, limit: usize, short_room_ids: Vec, max_depth: usize, suggested_only: bool, ) -> Result { - // try to find more servers to fetch hierachy from if the only - // choice is the room ID's server name (usually dead) - // - // all spaces are normal rooms, so they should always have at least - // 1 admin in it which has a far higher chance of their server still - // being alive - let power_levels: ruma::events::room::power_levels::RoomPowerLevelsEventContent = services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? - .map(|ev| { - serde_json::from_str(ev.content.get()) - .map_err(|_| Error::bad_database("invalid m.room.power_levels event")) - }) - .transpose()? - .unwrap_or_default(); + let mut parents = VecDeque::new(); - // add server names of the list of admins in the room for backfill server - let mut via = power_levels - .users - .iter() - .filter(|(_, level)| **level > power_levels.users_default) - .map(|(user_id, _)| user_id.server_name()) - .filter(|server| !server_is_ours(server)) - .map(ToOwned::to_owned) - .collect::>(); + // Don't start populating the results if we have to start at a specific room. + let mut populate_results = short_room_ids.is_empty(); - if let Some(server_name) = room_id.server_name() { - via.push(server_name.to_owned()); - } + let mut stack = vec![vec![( + room_id.to_owned(), + match room_id.server_name() { + Some(server_name) => vec![server_name.into()], + None => vec![], + }, + )]]; - debug_info!("servers via for hierarchy: {via:?}"); + let mut results = Vec::new(); - match self - .get_summary_and_children_client(&room_id.to_owned(), suggested_only, sender_user, &via) - .await? - { - Some(SummaryAccessibility::Accessible(summary)) => { - let mut left_to_skip = skip; - let mut arena = Arena::new(summary.room_id.clone(), max_depth); + while let Some((current_room, via)) = { next_room_to_traverse(&mut stack, &mut parents) } { + if limit > results.len() { + match ( + self.get_summary_and_children_client(¤t_room, suggested_only, sender_user, &via) + .await?, + current_room == room_id, + ) { + (Some(SummaryAccessibility::Accessible(summary)), _) => { + let mut children: Vec<(OwnedRoomId, Vec)> = + get_parent_children_via(&summary, suggested_only) + .into_iter() + .filter(|(room, _)| parents.iter().all(|parent| parent != room)) + .rev() + .collect(); - let mut results = Vec::new(); - let root = arena - .first_untraversed() - .expect("The node just added is not traversed"); + if populate_results { + results.push(summary_to_chunk(*summary.clone())); + } else { + children = children + .into_iter() + .rev() + .skip_while(|(room, _)| { + if let Ok(short) = services().rooms.short.get_shortroomid(room) + { + short.as_ref() != short_room_ids.get(parents.len()) + } else { + false + } + }) + .collect::>() + // skip_while doesn't implement DoubleEndedIterator, which is needed for rev + .into_iter() + .rev() + .collect(); - arena.push(root, get_parent_children_via(&summary, suggested_only)); - if left_to_skip > 0 { - left_to_skip -= 1; - } else { - results.push(summary_to_chunk(*summary.clone())); - } + if children.is_empty() { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Room IDs in token were not found.", + )); + } - while let Some(current_room) = arena.first_untraversed() { - if limit > results.len() { - let node = arena - .get(current_room) - .expect("We added this node, it must exist"); - if let Some(SummaryAccessibility::Accessible(summary)) = self - .get_summary_and_children_client(&node.room_id, suggested_only, sender_user, &node.via) - .await? - { - let children = get_parent_children_via(&summary, suggested_only); - arena.push(current_room, children); - - if left_to_skip > 0 { - left_to_skip -= 1; - } else { - results.push(summary_to_chunk(*summary.clone())); + // We have reached the room after where we last left off + if parents.len() + 1 == short_room_ids.len() { + populate_results = true; } } - } else { - break; - } + + if !children.is_empty() && parents.len() < max_depth { + parents.push_back(current_room.clone()); + stack.push(children); + } + // Root room in the space hierarchy, we return an error + // if this one fails. + }, + (Some(SummaryAccessibility::Inaccessible), true) => { + return Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room is inaccessible")); + }, + (None, true) => { + return Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room was not found")); + }, + // Just ignore other unavailable rooms + (None | Some(SummaryAccessibility::Inaccessible), false) => (), + } + } else { + break; + } + } + + Ok(client::space::get_hierarchy::v1::Response { + next_batch: if let Some((room, _)) = next_room_to_traverse(&mut stack, &mut parents) { + parents.pop_front(); + parents.push_back(room); + + let mut short_room_ids = vec![]; + + for room in parents { + short_room_ids.push(services().rooms.short.get_or_create_shortroomid(&room)?); } - Ok(client::space::get_hierarchy::v1::Response { - next_batch: if results.len() < limit { - None - } else { - let skip = UInt::new((skip + limit) as u64); - - skip.map(|skip| { - PagnationToken { - skip, - limit: UInt::new(max_depth as u64) - .expect("When sent in request it must have been valid UInt"), - max_depth: UInt::new(max_depth as u64) - .expect("When sent in request it must have been valid UInt"), - suggested_only, - } - .to_string() - }) - }, - rooms: results, - }) + Some( + PaginationToken { + short_room_ids, + limit: UInt::new(max_depth as u64).expect("When sent in request it must have been valid UInt"), + max_depth: UInt::new(max_depth as u64) + .expect("When sent in request it must have been valid UInt"), + suggested_only, + } + .to_string(), + ) + } else { + None }, - Some(SummaryAccessibility::Inaccessible) => { - Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room is inaccessible")) - }, - None => Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room was not found")), - } + rooms: results, + }) } } +fn next_room_to_traverse( + stack: &mut Vec)>>, parents: &mut VecDeque, +) -> Option<(OwnedRoomId, Vec)> { + while stack.last().map_or(false, Vec::is_empty) { + stack.pop(); + parents.pop_back(); + } + + stack.last_mut().and_then(Vec::pop) +} + /// Simply returns the stripped m.space.child events of a room async fn get_stripped_space_child_events( room_id: &RoomId, @@ -774,96 +644,74 @@ async fn get_stripped_space_child_events( fn is_accessable_child( current_room: &OwnedRoomId, join_rule: &SpaceRoomJoinRule, identifier: &Identifier<'_>, allowed_room_ids: &Vec, -) -> Result { - is_accessable_child_recurse(current_room, join_rule, identifier, allowed_room_ids, 0) -} +) -> bool { + // Note: unwrap_or_default for bool means false + match identifier { + Identifier::ServerName(server_name) => { + let room_id: &RoomId = current_room; -fn is_accessable_child_recurse( - current_room: &OwnedRoomId, join_rule: &SpaceRoomJoinRule, identifier: &Identifier<'_>, - allowed_room_ids: &Vec, recurse_num: usize, -) -> Result { - // Set limit at 10, as we cannot keep going up parents forever - // and it is very unlikely to have 10 space parents - if recurse_num < 10 { - match identifier { - Identifier::ServerName(server_name) => { - let room_id: &RoomId = current_room; - - // Checks if ACLs allow for the server to participate - if services() - .rooms - .event_handler - .acl_check(server_name, room_id) - .is_err() - { - return Ok(false); - } - }, - Identifier::UserId(user_id) => { - if services() + // Checks if ACLs allow for the server to participate + if services() + .rooms + .event_handler + .acl_check(server_name, room_id) + .is_err() + { + return false; + } + }, + Identifier::UserId(user_id) => { + if services() + .rooms + .state_cache + .is_joined(user_id, current_room) + .unwrap_or_default() + || services() .rooms .state_cache - .is_joined(user_id, current_room)? - || services() - .rooms - .state_cache - .is_invited(user_id, current_room)? - { - return Ok(true); - } - }, - Identifier::None => (), - } // Takes care of joinrules - Ok(match join_rule { - SpaceRoomJoinRule::Restricted => { - for room in allowed_room_ids { - if let Ok((join_rule, allowed_room_ids)) = get_join_rule(room) { - if matches!( - is_accessable_child_recurse( - room, - &join_rule, - identifier, - &allowed_room_ids, - recurse_num + 1, - ), - Ok(true) - ) { - return Ok(true); + .is_invited(user_id, current_room) + .unwrap_or_default() + { + return true; + } + }, + } // Takes care of join rules + match join_rule { + SpaceRoomJoinRule::Restricted => { + for room in allowed_room_ids { + match identifier { + Identifier::UserId(user) => { + if services() + .rooms + .state_cache + .is_joined(user, room) + .unwrap_or_default() + { + return true; } - } + }, + Identifier::ServerName(server) => { + if services() + .rooms + .state_cache + .server_in_room(server, room) + .unwrap_or_default() + { + return true; + } + }, } - false - }, - SpaceRoomJoinRule::Public | SpaceRoomJoinRule::Knock | SpaceRoomJoinRule::KnockRestricted => true, - // Custom join rules, Invite, or Private - _ => false, - }) - } else { - // If you need to go up 10 parents, we just assume it is inaccessable - Ok(false) + } + false + }, + SpaceRoomJoinRule::Public | SpaceRoomJoinRule::Knock | SpaceRoomJoinRule::KnockRestricted => true, + // Invite only, Private, or Custom join rule + _ => false, } } -/// Returns the join rule for a given room -fn get_join_rule(current_room: &RoomId) -> Result<(SpaceRoomJoinRule, Vec), Error> { - Ok(services() - .rooms - .state_accessor - .room_state_get(current_room, &StateEventType::RoomJoinRules, "")? - .map(|s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomJoinRulesEventContent| (c.join_rule.clone().into(), allowed_room_ids(c.join_rule))) - .map_err(|e| { - error!("Invalid room join rule event in database: {}", e); - Error::BadDatabase("Invalid room join rule event in database.") - }) - }) - .transpose()? - .unwrap_or((SpaceRoomJoinRule::Invite, vec![]))) -} - -// Here because cannot implement `From` across ruma-federation-api and -// ruma-client-api types +/// Here because cannot implement `From` across ruma-federation-api and +/// ruma-client-api types fn summary_to_chunk(summary: SpaceHierarchyParentSummary) -> SpaceHierarchyRoomsChunk { let SpaceHierarchyParentSummary { canonical_alias, @@ -895,22 +743,6 @@ fn summary_to_chunk(summary: SpaceHierarchyParentSummary) -> SpaceHierarchyRooms } } -/// Returns an empty vec if not a restricted room -fn allowed_room_ids(join_rule: JoinRule) -> Vec { - let mut room_ids = vec![]; - if let JoinRule::Restricted(r) | JoinRule::KnockRestricted(r) = join_rule { - for rule in r.allow { - if let AllowRule::RoomMembership(RoomMembership { - room_id: membership, - }) = rule - { - room_ids.push(membership.clone()); - } - } - } - room_ids -} - /// Returns the children of a SpaceHierarchyParentSummary, making use of the /// children_state field fn get_parent_children_via( @@ -930,367 +762,3 @@ fn get_parent_children_via( }) .collect() } - -#[cfg(test)] -mod tests { - use ruma::{ - api::federation::space::SpaceHierarchyParentSummaryInit, events::room::join_rules::Restricted, owned_room_id, - owned_server_name, - }; - - use super::*; - - fn first(arena: &mut Arena, room_id: &OwnedRoomId) { - let first_untrav = arena.first_untraversed().unwrap(); - - assert_eq!(arena.get(first_untrav).unwrap().room_id, *room_id); - } - - #[test] - fn zero_depth() { - let mut arena = Arena::new(owned_room_id!("!foo:example.org"), 0); - - assert_eq!(arena.first_untraversed(), None); - } - - #[test] - fn two_depth() { - let mut arena = Arena::new(owned_room_id!("!root:example.org"), 2); - - let root = arena.first_untraversed().unwrap(); - arena.push( - root, - vec![ - (owned_room_id!("!subspace1:example.org"), vec![]), - (owned_room_id!("!subspace2:example.org"), vec![]), - (owned_room_id!("!foo:example.org"), vec![]), - ], - ); - - let subspace1 = arena.first_untraversed().unwrap(); - let subspace2 = arena.first_untraversed().unwrap(); - - arena.push( - subspace1, - vec![ - (owned_room_id!("!room1:example.org"), vec![]), - (owned_room_id!("!room2:example.org"), vec![]), - ], - ); - - first(&mut arena, &owned_room_id!("!room1:example.org")); - first(&mut arena, &owned_room_id!("!room2:example.org")); - - arena.push( - subspace2, - vec![ - (owned_room_id!("!room3:example.org"), vec![]), - (owned_room_id!("!room4:example.org"), vec![]), - ], - ); - first(&mut arena, &owned_room_id!("!room3:example.org")); - first(&mut arena, &owned_room_id!("!room4:example.org")); - - let foo_node = NodeId { - index: 1, - }; - - assert_eq!(arena.first_untraversed(), Some(foo_node)); - assert_eq!( - arena.get(foo_node).map(|node| node.room_id.clone()), - Some(owned_room_id!("!foo:example.org")) - ); - } - - #[test] - fn empty_push() { - let mut arena = Arena::new(owned_room_id!("!root:example.org"), 5); - - let root = arena.first_untraversed().unwrap(); - arena.push( - root, - vec![ - (owned_room_id!("!room1:example.org"), vec![]), - (owned_room_id!("!room2:example.org"), vec![]), - ], - ); - - let room1 = arena.first_untraversed().unwrap(); - arena.push(room1, vec![]); - - first(&mut arena, &owned_room_id!("!room2:example.org")); - assert!(arena.first_untraversed().is_none()); - } - - #[test] - fn beyond_max_depth() { - let mut arena = Arena::new(owned_room_id!("!root:example.org"), 0); - - let root = NodeId { - index: 0, - }; - - arena.push(root, vec![(owned_room_id!("!too_deep:example.org"), vec![])]); - - assert_eq!(arena.first_child(root), None); - assert_eq!(arena.nodes.len(), 1); - } - - #[test] - fn order_check() { - let mut arena = Arena::new(owned_room_id!("!root:example.org"), 3); - - let root = arena.first_untraversed().unwrap(); - arena.push( - root, - vec![ - (owned_room_id!("!subspace1:example.org"), vec![]), - (owned_room_id!("!subspace2:example.org"), vec![]), - (owned_room_id!("!foo:example.org"), vec![]), - ], - ); - - let subspace1 = arena.first_untraversed().unwrap(); - arena.push( - subspace1, - vec![ - (owned_room_id!("!room1:example.org"), vec![]), - (owned_room_id!("!room3:example.org"), vec![]), - (owned_room_id!("!room5:example.org"), vec![]), - ], - ); - - first(&mut arena, &owned_room_id!("!room1:example.org")); - first(&mut arena, &owned_room_id!("!room3:example.org")); - first(&mut arena, &owned_room_id!("!room5:example.org")); - - let subspace2 = arena.first_untraversed().unwrap(); - - assert_eq!(arena.get(subspace2).unwrap().room_id, owned_room_id!("!subspace2:example.org")); - - arena.push( - subspace2, - vec![ - (owned_room_id!("!room1:example.org"), vec![]), - (owned_room_id!("!room2:example.org"), vec![]), - ], - ); - - first(&mut arena, &owned_room_id!("!room1:example.org")); - first(&mut arena, &owned_room_id!("!room2:example.org")); - first(&mut arena, &owned_room_id!("!foo:example.org")); - - assert_eq!(arena.first_untraversed(), None); - } - - #[test] - fn get_summary_children() { - let summary: SpaceHierarchyParentSummary = SpaceHierarchyParentSummaryInit { - num_joined_members: UInt::from(1_u32), - room_id: owned_room_id!("!root:example.org"), - world_readable: true, - guest_can_join: true, - join_rule: SpaceRoomJoinRule::Public, - children_state: vec![ - serde_json::from_str( - r#"{ - "content": { - "via": [ - "example.org" - ], - "suggested": false - }, - "origin_server_ts": 1629413349153, - "sender": "@alice:example.org", - "state_key": "!foo:example.org", - "type": "m.space.child" - }"#, - ) - .unwrap(), - serde_json::from_str( - r#"{ - "content": { - "via": [ - "example.org" - ], - "suggested": true - }, - "origin_server_ts": 1629413349157, - "sender": "@alice:example.org", - "state_key": "!bar:example.org", - "type": "m.space.child" - }"#, - ) - .unwrap(), - serde_json::from_str( - r#"{ - "content": { - "via": [ - "example.org" - ] - }, - "origin_server_ts": 1629413349160, - "sender": "@alice:example.org", - "state_key": "!baz:example.org", - "type": "m.space.child" - }"#, - ) - .unwrap(), - ], - allowed_room_ids: vec![], - } - .into(); - - assert_eq!( - get_parent_children_via(&summary, false), - vec![ - (owned_room_id!("!foo:example.org"), vec![owned_server_name!("example.org")]), - (owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")]), - (owned_room_id!("!baz:example.org"), vec![owned_server_name!("example.org")]) - ] - ); - assert_eq!( - get_parent_children_via(&summary, true), - vec![(owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")])] - ); - } - - #[test] - fn allowed_room_ids_from_join_rule() { - let restricted_join_rule = JoinRule::Restricted(Restricted { - allow: vec![ - AllowRule::RoomMembership(RoomMembership { - room_id: owned_room_id!("!foo:example.org"), - }), - AllowRule::RoomMembership(RoomMembership { - room_id: owned_room_id!("!bar:example.org"), - }), - AllowRule::RoomMembership(RoomMembership { - room_id: owned_room_id!("!baz:example.org"), - }), - ], - }); - - let invite_join_rule = JoinRule::Invite; - - assert_eq!( - allowed_room_ids(restricted_join_rule), - vec![ - owned_room_id!("!foo:example.org"), - owned_room_id!("!bar:example.org"), - owned_room_id!("!baz:example.org") - ] - ); - - let empty_vec: Vec = vec![]; - - assert_eq!(allowed_room_ids(invite_join_rule), empty_vec); - } - - #[test] - fn invalid_pagnation_tokens() { - fn token_is_err(token: &str) { - let token: Result = PagnationToken::from_str(token); - token.unwrap_err(); - } - - token_is_err("231_2_noabool"); - token_is_err(""); - token_is_err("111_3_"); - token_is_err("foo_not_int"); - token_is_err("11_4_true_"); - token_is_err("___"); - token_is_err("__false"); - } - - #[test] - fn valid_pagnation_tokens() { - assert_eq!( - PagnationToken { - skip: UInt::from(40_u32), - limit: UInt::from(20_u32), - max_depth: UInt::from(1_u32), - suggested_only: true - }, - PagnationToken::from_str("40_20_1_true").unwrap() - ); - - assert_eq!( - PagnationToken { - skip: UInt::from(27645_u32), - limit: UInt::from(97_u32), - max_depth: UInt::from(10539_u32), - suggested_only: false - }, - PagnationToken::from_str("27645_97_10539_false").unwrap() - ); - } - - #[test] - fn pagnation_token_to_string() { - assert_eq!( - PagnationToken { - skip: UInt::from(27645_u32), - limit: UInt::from(97_u32), - max_depth: UInt::from(9420_u32), - suggested_only: false - } - .to_string(), - "27645_97_9420_false" - ); - - assert_eq!( - PagnationToken { - skip: UInt::from(12_u32), - limit: UInt::from(3_u32), - max_depth: UInt::from(1_u32), - suggested_only: true - } - .to_string(), - "12_3_1_true" - ); - } - - #[test] - fn forbid_recursion() { - let mut arena = Arena::new(owned_room_id!("!root:example.org"), 5); - let root_node_id = arena.first_untraversed().unwrap(); - - arena.push( - root_node_id, - vec![ - (owned_room_id!("!subspace1:example.org"), vec![]), - (owned_room_id!("!room1:example.org"), vec![]), - (owned_room_id!("!subspace2:example.org"), vec![]), - ], - ); - - let subspace1_node_id = arena.first_untraversed().unwrap(); - arena.push( - subspace1_node_id, - vec![ - (owned_room_id!("!subspace2:example.org"), vec![]), - (owned_room_id!("!room1:example.org"), vec![]), - ], - ); - - let subspace2_node_id = arena.first_untraversed().unwrap(); - // Here, both subspaces should be ignored and not added, as they are both - // parents of subspace2 - arena.push( - subspace2_node_id, - vec![ - (owned_room_id!("!subspace1:example.org"), vec![]), - (owned_room_id!("!subspace2:example.org"), vec![]), - (owned_room_id!("!room1:example.org"), vec![]), - ], - ); - - assert_eq!(arena.nodes.len(), 7); - first(&mut arena, &owned_room_id!("!room1:example.org")); - first(&mut arena, &owned_room_id!("!room1:example.org")); - first(&mut arena, &owned_room_id!("!room1:example.org")); - first(&mut arena, &owned_room_id!("!subspace2:example.org")); - assert!(arena.first_untraversed().is_none()); - } -} diff --git a/src/service/rooms/spaces/tests.rs b/src/service/rooms/spaces/tests.rs new file mode 100644 index 00000000..71640035 --- /dev/null +++ b/src/service/rooms/spaces/tests.rs @@ -0,0 +1,145 @@ +#![cfg(test)] + +use std::str::FromStr; + +use ruma::{ + api::federation::space::{SpaceHierarchyParentSummary, SpaceHierarchyParentSummaryInit}, + owned_room_id, owned_server_name, + space::SpaceRoomJoinRule, + UInt, +}; + +use crate::rooms::spaces::{get_parent_children_via, PaginationToken}; + +#[test] +fn get_summary_children() { + let summary: SpaceHierarchyParentSummary = SpaceHierarchyParentSummaryInit { + num_joined_members: UInt::from(1_u32), + room_id: owned_room_id!("!root:example.org"), + world_readable: true, + guest_can_join: true, + join_rule: SpaceRoomJoinRule::Public, + children_state: vec![ + serde_json::from_str( + r#"{ + "content": { + "via": [ + "example.org" + ], + "suggested": false + }, + "origin_server_ts": 1629413349153, + "sender": "@alice:example.org", + "state_key": "!foo:example.org", + "type": "m.space.child" + }"#, + ) + .unwrap(), + serde_json::from_str( + r#"{ + "content": { + "via": [ + "example.org" + ], + "suggested": true + }, + "origin_server_ts": 1629413349157, + "sender": "@alice:example.org", + "state_key": "!bar:example.org", + "type": "m.space.child" + }"#, + ) + .unwrap(), + serde_json::from_str( + r#"{ + "content": { + "via": [ + "example.org" + ] + }, + "origin_server_ts": 1629413349160, + "sender": "@alice:example.org", + "state_key": "!baz:example.org", + "type": "m.space.child" + }"#, + ) + .unwrap(), + ], + allowed_room_ids: vec![], + } + .into(); + + assert_eq!( + get_parent_children_via(&summary, false), + vec![ + (owned_room_id!("!foo:example.org"), vec![owned_server_name!("example.org")]), + (owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")]), + (owned_room_id!("!baz:example.org"), vec![owned_server_name!("example.org")]) + ] + ); + assert_eq!( + get_parent_children_via(&summary, true), + vec![(owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")])] + ); +} + +#[test] +fn invalid_pagination_tokens() { + fn token_is_err(token: &str) { PaginationToken::from_str(token).unwrap_err(); } + + token_is_err("231_2_noabool"); + token_is_err(""); + token_is_err("111_3_"); + token_is_err("foo_not_int"); + token_is_err("11_4_true_"); + token_is_err("___"); + token_is_err("__false"); +} + +#[test] +fn valid_pagination_tokens() { + assert_eq!( + PaginationToken { + short_room_ids: vec![5383, 42934, 283, 423], + limit: UInt::from(20_u32), + max_depth: UInt::from(1_u32), + suggested_only: true + }, + PaginationToken::from_str("5383,42934,283,423_20_1_true").unwrap() + ); + + assert_eq!( + PaginationToken { + short_room_ids: vec![740], + limit: UInt::from(97_u32), + max_depth: UInt::from(10539_u32), + suggested_only: false + }, + PaginationToken::from_str("740_97_10539_false").unwrap() + ); +} + +#[test] +fn pagination_token_to_string() { + assert_eq!( + PaginationToken { + short_room_ids: vec![740], + limit: UInt::from(97_u32), + max_depth: UInt::from(10539_u32), + suggested_only: false + } + .to_string(), + "740_97_10539_false" + ); + + assert_eq!( + PaginationToken { + short_room_ids: vec![9, 34], + limit: UInt::from(3_u32), + max_depth: UInt::from(1_u32), + suggested_only: true + } + .to_string(), + "9,34_3_1_true" + ); +} diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index ce884fe5..65c43905 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -16,6 +16,7 @@ use ruma::{ canonical_alias::RoomCanonicalAliasEventContent, guest_access::{GuestAccess, RoomGuestAccessEventContent}, history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent, RoomMembership}, member::{MembershipState, RoomMemberEventContent}, name::RoomNameEventContent, power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, @@ -23,7 +24,8 @@ use ruma::{ }, StateEventType, }, - EventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, + space::SpaceRoomJoinRule, + EventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; use serde_json::value::to_raw_value; @@ -415,4 +417,38 @@ impl Service { }, ) } + + /// Returns the join rule for a given room + pub fn get_join_rule(&self, current_room: &RoomId) -> Result<(SpaceRoomJoinRule, Vec), Error> { + Ok(self + .room_state_get(current_room, &StateEventType::RoomJoinRules, "")? + .map(|s| { + serde_json::from_str(s.content.get()) + .map(|c: RoomJoinRulesEventContent| { + (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule)) + }) + .map_err(|e| { + error!("Invalid room join rule event in database: {e}"); + Error::BadDatabase("Invalid room join rule event in database.") + }) + }) + .transpose()? + .unwrap_or((SpaceRoomJoinRule::Invite, vec![]))) + } + + /// Returns an empty vec if not a restricted room + pub fn allowed_room_ids(&self, join_rule: JoinRule) -> Vec { + let mut room_ids = vec![]; + if let JoinRule::Restricted(r) | JoinRule::KnockRestricted(r) = join_rule { + for rule in r.allow { + if let AllowRule::RoomMembership(RoomMembership { + room_id: membership, + }) = rule + { + room_ids.push(membership.clone()); + } + } + } + room_ids + } } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index eeecc9c7..6572fcba 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -293,6 +293,7 @@ impl Service { self.db.room_members(room_id) } + /// Returns the number of users which are currently in a room #[tracing::instrument(skip(self))] pub fn room_joined_count(&self, room_id: &RoomId) -> Result> { self.db.room_joined_count(room_id) } @@ -310,6 +311,7 @@ impl Service { self.db.active_local_users_in_room(room_id) } + /// Returns the number of users which are currently invited to a room #[tracing::instrument(skip(self))] pub fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.db.room_invited_count(room_id) } From ed3a464722890448a6b93d3c5886686a1a48eee2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 2 Jul 2024 19:13:54 -0400 Subject: [PATCH 0324/2091] remove uselsss partial_cmp with cmp Signed-off-by: strawberry --- src/service/rooms/pdu_metadata/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index d77fbf74..978a0ec0 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -210,7 +210,7 @@ impl Service { } } - pdus.sort_by(|a, b| a.0.partial_cmp(&b.0).expect("u64s can always be compared")); + pdus.sort_by(|a, b| a.0.cmp(&b.0)); pdus }) } From 9c0d7b1da057d7e60ce723cca72087d8c03c28da Mon Sep 17 00:00:00 2001 From: morguldir Date: Wed, 3 Jul 2024 01:08:30 +0200 Subject: [PATCH 0325/2091] ci: change to cache-nix-action --- .github/workflows/ci.yml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e2daf176..0c228e4f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,8 +49,6 @@ jobs: name: Test runs-on: ubuntu-latest steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - name: Sync repository uses: actions/checkout@v4 @@ -66,13 +64,7 @@ jobs: exit 1 fi - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@main - with: - diagnostic-endpoint: "" - extra-conf: | - experimental-features = nix-command flakes - accept-flake-config = true + - uses: nixbuild/nix-quick-install-action@v27 - name: Enable Cachix binary cache run: | @@ -80,18 +72,36 @@ jobs: cachix use crane cachix use nix-community - - name: Configure Magic Nix Cache - uses: DeterminateSystems/magic-nix-cache-action@main + - name: Restore and cache Nix store + uses: nix-community/cache-nix-action@v5 with: - diagnostic-endpoint: "" - upstream-cache: "https://cache.nixos.org" + # restore and save a cache using this key + primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }} + # if there's no cache hit, restore a cache by this prefix + restore-prefixes-first-match: nix-${{ runner.os }}- + # collect garbage until Nix store size (in bytes) is at most this number + # before trying to save a new cache + gc-max-store-size-linux: 1073741824 + # do purge caches + purge: true + # purge all versions of the cache + purge-prefixes: cache-${{ runner.os }}- + # created more than this number of seconds ago relative to the start of the `Post Restore` phase + purge-created: 0 + # except the version with the `primary-key`, if it exists + purge-primary-key: never - name: Apply Nix binary cache configuration run: | + echo "before: " && cat /etc/nix/nix.conf + cat /etc/nix/nix.conf sudo tee -a /etc/nix/nix.conf > /dev/null < Date: Wed, 3 Jul 2024 01:56:14 +0200 Subject: [PATCH 0326/2091] ci: use correct config file also had to try putting it at the end again --- .github/workflows/ci.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c228e4f..e83c86a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,6 +16,7 @@ on: - 'docker/**' branches: - main + - change-ci-cache tags: - '*' # Allows you to run this workflow manually from the Actions tab @@ -64,7 +65,7 @@ jobs: exit 1 fi - - uses: nixbuild/nix-quick-install-action@v27 + - uses: nixbuild/nix-quick-install-action@v28 - name: Enable Cachix binary cache run: | @@ -76,37 +77,34 @@ jobs: uses: nix-community/cache-nix-action@v5 with: # restore and save a cache using this key - primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix') }} + primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/.lock') }} # if there's no cache hit, restore a cache by this prefix restore-prefixes-first-match: nix-${{ runner.os }}- # collect garbage until Nix store size (in bytes) is at most this number # before trying to save a new cache - gc-max-store-size-linux: 1073741824 + gc-max-store-size-linux: 2073741824 # do purge caches purge: true # purge all versions of the cache - purge-prefixes: cache-${{ runner.os }}- + purge-prefixes: nix-${{ runner.os }}- # created more than this number of seconds ago relative to the start of the `Post Restore` phase - purge-created: 0 + purge-last-accessed: 86400 # except the version with the `primary-key`, if it exists purge-primary-key: never + # always save the cache + save-always: true - name: Apply Nix binary cache configuration run: | - echo "before: " && cat /etc/nix/nix.conf - cat /etc/nix/nix.conf - sudo tee -a /etc/nix/nix.conf > /dev/null < /dev/null < /dev/null < /dev/null < Date: Wed, 3 Jul 2024 03:39:49 +0200 Subject: [PATCH 0327/2091] ci: try out cargo caching --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e83c86a3..50e4f51f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,6 +119,8 @@ jobs: - name: Cache CI dependencies run: | bin/nix-build-and-cache ci + + - uses: Swatinem/rust-cache@v2 - name: Run CI tests run: | From b035cd7cbe5b241cf3ce8d8ccec722a8f2fcc6bf Mon Sep 17 00:00:00 2001 From: morguldir Date: Wed, 3 Jul 2024 04:51:59 +0200 Subject: [PATCH 0328/2091] ci: remove detsys from documentation/build workflow too Signed-off-by: morguldir --- .github/workflows/ci.yml | 37 ++++++++++++++++++----------- .github/workflows/documentation.yml | 37 ++++++++++++++++++----------- 2 files changed, 46 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50e4f51f..cc11a226 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,29 +190,38 @@ jobs: - name: Sync repository uses: actions/checkout@v4 - - name: Install Nix - uses: DeterminateSystems/nix-installer-action@main - with: - diagnostic-endpoint: "" - extra-conf: | - experimental-features = nix-command flakes - accept-flake-config = true + - uses: nixbuild/nix-quick-install-action@v28 - - name: Install and enable Cachix binary cache + - name: Enable Cachix binary cache run: | nix profile install nixpkgs#cachix cachix use crane cachix use nix-community - - name: Configure Magic Nix Cache - uses: DeterminateSystems/magic-nix-cache-action@main + - name: Restore and cache Nix store + uses: nix-community/cache-nix-action@v5 with: - diagnostic-endpoint: "" - upstream-cache: "https://cache.nixos.org" + # restore and save a cache using this key + primary-key: nix-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/*.nix', '**/.lock') }} + # if there's no cache hit, restore a cache by this prefix + restore-prefixes-first-match: nix-${{ runner.os }}- + # collect garbage until Nix store size (in bytes) is at most this number + # before trying to save a new cache + gc-max-store-size-linux: 2073741824 + # do purge caches + purge: true + # purge all versions of the cache + purge-prefixes: nix-${{ runner.os }}- + # created more than this number of seconds ago relative to the start of the `Post Restore` phase + purge-last-accessed: 86400 + # except the version with the `primary-key`, if it exists + purge-primary-key: never + # always save the cache + save-always: true - name: Apply Nix binary cache configuration run: | - sudo tee -a /etc/nix/nix.conf > /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < Date: Wed, 3 Jul 2024 06:13:39 +0200 Subject: [PATCH 0329/2091] ci: pin cache-nix-action Signed-off-by: morguldir --- .github/workflows/ci.yml | 4 ++-- .github/workflows/documentation.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc11a226..efb2a9c0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: cachix use nix-community - name: Restore and cache Nix store - uses: nix-community/cache-nix-action@v5 + uses: nix-community/cache-nix-action@v5.1.0 with: # restore and save a cache using this key primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/.lock') }} @@ -199,7 +199,7 @@ jobs: cachix use nix-community - name: Restore and cache Nix store - uses: nix-community/cache-nix-action@v5 + uses: nix-community/cache-nix-action@v5.1.0 with: # restore and save a cache using this key primary-key: nix-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/*.nix', '**/.lock') }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 9051d048..9beb5064 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -60,7 +60,7 @@ jobs: cachix use nix-community - name: Restore and cache Nix store - uses: nix-community/cache-nix-action@v5 + uses: nix-community/cache-nix-action@v5.1.0 with: # restore and save a cache using this key primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/.lock') }} From 0613140130695fe38dd8f89099ce71a0966d5fd6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 30 Jun 2024 18:20:39 +0000 Subject: [PATCH 0330/2091] optimize bytes conversion utils Signed-off-by: Jason Volk --- src/core/error.rs | 6 ++++++ src/core/utils/mod.rs | 34 ++++++++++++++++++---------------- src/service/presence/data.rs | 4 ++-- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/core/error.rs b/src/core/error.rs index 6942d1be..ac2e176d 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -28,6 +28,12 @@ pub enum Error { Fmt(#[from] fmt::Error), #[error("I/O error: {0}")] Io(#[from] std::io::Error), + #[error("{0}")] + Utf8Error(#[from] std::str::Utf8Error), + #[error("{0}")] + FromUtf8Error(#[from] std::string::FromUtf8Error), + #[error("{0}")] + TryFromSliceError(#[from] std::array::TryFromSliceError), // third-party #[error("Regex error: {0}")] diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 1e0e01c0..bbaaf2ec 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -19,10 +19,10 @@ pub use json::{deserialize_from_str, to_canonical_object}; pub use mutex_map::MutexMap; use rand::prelude::*; use ring::digest; -use ruma::OwnedUserId; +use ruma::UserId; pub use sys::available_parallelism; -use crate::{Error, Result}; +use crate::Result; pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), max) } @@ -38,9 +38,8 @@ pub fn millis_since_unix_epoch() -> u64 { #[inline] #[must_use] pub fn increment(old: Option<&[u8]>) -> [u8; 8] { - const ZERO: u64 = 0; old.map(TryInto::try_into) - .map_or(ZERO, |val| val.map_or(ZERO, u64::from_be_bytes)) + .map_or(0_u64, |val| val.map_or(0_u64, u64::from_be_bytes)) .wrapping_add(1) .to_be_bytes() } @@ -55,23 +54,26 @@ pub fn generate_keypair() -> Vec { value } -/// Parses the bytes into an u64. -pub fn u64_from_bytes(bytes: &[u8]) -> Result { - let array: [u8; 8] = bytes.try_into()?; - Ok(u64::from_be_bytes(array)) +/// Parses a `UserId` from bytes. +pub fn user_id_from_bytes(bytes: &[u8]) -> Result<&UserId> { + let str: &str = str_from_bytes(bytes)?; + let user_id: &UserId = str.try_into()?; + Ok(user_id) } /// Parses the bytes into a string. -pub fn string_from_bytes(bytes: &[u8]) -> Result { - String::from_utf8(bytes.to_vec()) +pub fn string_from_bytes(bytes: &[u8]) -> Result { + let str: &str = str_from_bytes(bytes)?; + Ok(str.to_owned()) } -/// Parses a `OwnedUserId` from bytes. -pub fn user_id_from_bytes(bytes: &[u8]) -> Result { - OwnedUserId::try_from( - string_from_bytes(bytes).map_err(|_| Error::bad_database("Failed to parse string from bytes"))?, - ) - .map_err(|_| Error::bad_database("Failed to parse user id from bytes")) +/// Parses the bytes into a string. +pub fn str_from_bytes(bytes: &[u8]) -> Result<&str> { Ok(std::str::from_utf8(bytes)?) } + +/// Parses the bytes into an u64. +pub fn u64_from_bytes(bytes: &[u8]) -> Result { + let array: [u8; 8] = bytes.try_into()?; + Ok(u64::from_be_bytes(array)) } pub fn random_string(length: usize) -> String { diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 5cb4032f..42b827d3 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -115,7 +115,7 @@ impl Data { .iter() .flat_map(|(key, presence_bytes)| -> Result<(OwnedUserId, u64, Vec)> { let (count, user_id) = presenceid_parse(&key)?; - Ok((user_id, count, presence_bytes)) + Ok((user_id.to_owned(), count, presence_bytes)) }) .filter(move |(_, count, _)| *count > since), ) @@ -128,7 +128,7 @@ fn presenceid_key(count: u64, user_id: &UserId) -> Vec { } #[inline] -fn presenceid_parse(key: &[u8]) -> Result<(u64, OwnedUserId)> { +fn presenceid_parse(key: &[u8]) -> Result<(u64, &UserId)> { let (count, user_id) = key.split_at(8); let user_id = utils::user_id_from_bytes(user_id)?; let count = utils::u64_from_bytes(count).unwrap(); From 972037dcd9e4bb0bc35f1ec4f308fa2e192239eb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 30 Jun 2024 23:06:01 +0000 Subject: [PATCH 0331/2091] optimize column family handles Signed-off-by: Jason Volk --- src/database/engine.rs | 2 +- src/database/map.rs | 51 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 40a5438c..c6f9eaa5 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -112,7 +112,7 @@ impl Engine { Ok(self.cf(name)) } - pub(crate) fn cf<'db>(&'db self, name: &str) -> Arc> { + pub(crate) fn cf(&self, name: &str) -> Arc> { self.db .cf_handle(name) .expect("column was created and exists") diff --git a/src/database/map.rs b/src/database/map.rs index 0a628f9c..8f21f76d 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -1,13 +1,16 @@ use std::{future::Future, pin::Pin, sync::Arc}; use conduit::{utils, Result}; -use rocksdb::{BoundColumnFamily, Direction, IteratorMode, ReadOptions, WriteBatchWithTransaction, WriteOptions}; +use rocksdb::{ + AsColumnFamilyRef, ColumnFamily, Direction, IteratorMode, ReadOptions, WriteBatchWithTransaction, WriteOptions, +}; use super::{or_else, result, watchers::Watchers, Engine}; pub struct Map { - db: Arc, name: String, + db: Arc, + cf: Arc, watchers: Watchers, write_options: WriteOptions, read_options: ReadOptions, @@ -19,10 +22,10 @@ type KeyVal = (Key, Val); impl Map { pub(crate) fn open(db: &Arc, name: &str) -> Result> { - db.open_cf(name)?; Ok(Arc::new(Self { - db: db.clone(), name: name.to_owned(), + db: db.clone(), + cf: open(db, name)?, watchers: Watchers::default(), write_options: write_options_default(), read_options: read_options_default(), @@ -215,7 +218,10 @@ impl Map { self.watchers.watch(prefix) } - fn cf(&self) -> Arc> { self.db.cf(&self.name) } + #[inline] + pub fn name(&self) -> &str { &self.name } + + fn cf(&self) -> impl AsColumnFamilyRef + '_ { &*self.cf } } impl<'a> IntoIterator for &'a Map { @@ -225,6 +231,41 @@ impl<'a> IntoIterator for &'a Map { fn into_iter(self) -> Self::IntoIter { self.iter() } } +fn open(db: &Arc, name: &str) -> Result> { + let bounded_arc = db.open_cf(name)?; + let bounded_ptr = Arc::into_raw(bounded_arc); + let cf_ptr = bounded_ptr.cast::(); + + // SAFETY: After thorough contemplation this appears to be the best solution, + // even by a significant margin. + // + // BACKGROUND: Column family handles out of RocksDB are basic pointers and can + // be invalidated: 1. when the database closes. 2. when the column is dropped or + // closed. rust_rocksdb wraps this for us by storing handles in their own + // `RwLock` map and returning an Arc>` to + // provide expected safety. Similarly in "single-threaded mode" we would + // receive `&'_ ColumnFamily`. + // + // PROBLEM: We need to hold these handles in a field, otherwise we have to take + // a lock and get them by name from this map for every query, which is what + // conduit was doing, but we're not going to make a query for every query so we + // need to be holding it right. The lifetime parameter on these references makes + // that complicated. If this can be done without polluting the userspace + // with lifetimes on every instance of `Map` then this `unsafe` might not be + // necessary. + // + // SOLUTION: After investigating the underlying types it appears valid to + // Arc-swap `BoundColumnFamily<'_>` for `ColumnFamily`. They have the + // same inner data, the same Drop behavior, Deref, etc. We're just losing the + // lifetime parameter. We should not hold this handle, even in its Arc, after + // closing the database (dropping `Engine`). Since `Arc` is a sibling + // member along with this handle in `Map`, that is prevented. + Ok(unsafe { + Arc::decrement_strong_count(cf_ptr); + Arc::from_raw(cf_ptr) + }) +} + #[inline] fn read_options_default() -> ReadOptions { let mut read_options = ReadOptions::default(); From 0171262581fbd000846dece01f0fd26df48fd1d6 Mon Sep 17 00:00:00 2001 From: Jacob Taylor Date: Sun, 30 Jun 2024 21:38:52 -0700 Subject: [PATCH 0332/2091] make pduid_pdu/eventid_outlierpdu scale with conduit_cache_capacity_modifier via cache_size(), and fix lint --- src/database/opts.rs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/database/opts.rs b/src/database/opts.rs index 4f9dece5..1b9c163f 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -144,14 +144,9 @@ pub(crate) fn cf_options(cfg: &Config, name: &str, mut opts: Options, cache: &mu cache_size(cfg, cfg.statekeyshort_cache_capacity, 1024), ), - #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] - "eventid_outlierpdu" => set_table_with_new_cache( - &mut opts, - cfg, - cache, - name, - (cfg.pdu_cache_capacity as usize).saturating_mul(1536), - ), + "eventid_outlierpdu" => { + set_table_with_new_cache(&mut opts, cfg, cache, name, cache_size(cfg, cfg.pdu_cache_capacity, 1536)); + }, "pduid_pdu" => set_table_with_shared_cache(&mut opts, cfg, cache, name, "eventid_outlierpdu"), From a1ced0a56fbcec139eaffc503dd950790e298ed3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 1 Jul 2024 20:35:39 +0000 Subject: [PATCH 0333/2091] fix admin query timers not covering full iteration Signed-off-by: Jason Volk --- src/admin/query/presence.rs | 3 +- src/admin/query/room_alias.rs | 6 +-- src/admin/query/sending.rs | 72 +++++++++++------------------------ src/admin/query/users.rs | 3 +- 4 files changed, 27 insertions(+), 57 deletions(-) diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index bd853ada..c47b7a51 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -22,9 +22,8 @@ pub(super) async fn presence(subcommand: Presence) -> Result { let timer = tokio::time::Instant::now(); let results = services().presence.db.presence_since(since); - let query_time = timer.elapsed(); - let presence_since: Vec<(_, _, _)> = results.collect(); + let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{presence_since:#?}\n```" diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index 2d3b10ce..d2c16801 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -22,9 +22,8 @@ pub(super) async fn room_alias(subcommand: RoomAlias) -> Result { let timer = tokio::time::Instant::now(); let results = services().rooms.alias.local_aliases_for_room(&room_id); - let query_time = timer.elapsed(); - let aliases: Vec<_> = results.collect(); + let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```" @@ -33,9 +32,8 @@ pub(super) async fn room_alias(subcommand: RoomAlias) -> Result { let timer = tokio::time::Instant::now(); let results = services().rooms.alias.all_local_aliases(); - let query_time = timer.elapsed(); - let aliases: Vec<_> = results.collect(); + let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```" diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 9690f8f6..4e82695d 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -9,9 +9,8 @@ pub(super) async fn sending(subcommand: Sending) -> Result { let timer = tokio::time::Instant::now(); let results = services().sending.db.active_requests(); - let query_time = timer.elapsed(); - let active_requests: Result> = results.collect(); + let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" @@ -29,8 +28,8 @@ pub(super) async fn sending(subcommand: Sending) -> Result { if appservice_id.is_empty() { return Ok(RoomMessageEventContent::text_plain( @@ -39,25 +38,15 @@ pub(super) async fn sending(subcommand: Sending) -> Result { - let timer = tokio::time::Instant::now(); - let results = services() - .sending - .db - .queued_requests(&Destination::Normal(server_name.into())); - let query_time = timer.elapsed(); - - (results, query_time) + .queued_requests(&Destination::Appservice(appservice_id)) }, + (None, Some(server_name), None, None) => services() + .sending + .db + .queued_requests(&Destination::Normal(server_name.into())), (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { return Ok(RoomMessageEventContent::text_plain( @@ -66,14 +55,10 @@ pub(super) async fn sending(subcommand: Sending) -> Result { return Ok(RoomMessageEventContent::text_plain( @@ -90,6 +75,7 @@ pub(super) async fn sending(subcommand: Sending) -> Result>>(); + let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{queued_requests:#?}\n```" @@ -108,7 +94,8 @@ pub(super) async fn sending(subcommand: Sending) -> Result { if appservice_id.is_empty() { return Ok(RoomMessageEventContent::text_plain( @@ -117,25 +104,15 @@ pub(super) async fn sending(subcommand: Sending) -> Result { - let timer = tokio::time::Instant::now(); - let results = services() - .sending - .db - .active_requests_for(&Destination::Normal(server_name.into())); - let query_time = timer.elapsed(); - - (results, query_time) + .active_requests_for(&Destination::Appservice(appservice_id)) }, + (None, Some(server_name), None, None) => services() + .sending + .db + .active_requests_for(&Destination::Normal(server_name.into())), (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { return Ok(RoomMessageEventContent::text_plain( @@ -144,14 +121,10 @@ pub(super) async fn sending(subcommand: Sending) -> Result { return Ok(RoomMessageEventContent::text_plain( @@ -168,6 +141,7 @@ pub(super) async fn sending(subcommand: Sending) -> Result>>(); + let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index 8f88808a..2e73bff3 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -9,9 +9,8 @@ pub(super) async fn users(subcommand: Users) -> Result Users::Iter => { let timer = tokio::time::Instant::now(); let results = services().users.db.iter(); - let query_time = timer.elapsed(); - let users = results.collect::>(); + let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{users:#?}\n```" From 6dd6e4bfaf3e4826febae3de8ff8425f8f13079e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 1 Jul 2024 20:54:38 +0000 Subject: [PATCH 0334/2091] simplify cork interface related Signed-off-by: Jason Volk --- src/api/client/sync.rs | 2 +- src/database/cork.rs | 2 +- src/database/database.rs | 11 ++++++++++- src/database/engine.rs | 2 +- src/database/mod.rs | 3 +-- src/service/globals/data.rs | 6 +----- src/service/globals/migrations.rs | 4 ++-- src/service/rooms/timeline/mod.rs | 2 +- src/service/sending/mod.rs | 10 +++++----- src/service/sending/sender.rs | 6 +++--- 10 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 738c94d7..2ea766a4 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -138,7 +138,7 @@ pub(crate) async fn sync_events_route( .collect::>(); // Coalesce database writes for the remainder of this scope. - let _cork = services().globals.db.cork_and_flush(); + let _cork = services().db.cork_and_flush(); for room_id in all_joined_rooms { let room_id = room_id?; diff --git a/src/database/cork.rs b/src/database/cork.rs index faa3a491..b7e1f60f 100644 --- a/src/database/cork.rs +++ b/src/database/cork.rs @@ -9,7 +9,7 @@ pub struct Cork { } impl Cork { - pub fn new(db: &Arc, flush: bool, sync: bool) -> Self { + pub(super) fn new(db: &Arc, flush: bool, sync: bool) -> Self { db.cork(); Self { db: db.clone(), diff --git a/src/database/database.rs b/src/database/database.rs index e995d987..833b0047 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -8,7 +8,7 @@ use conduit::{PduCount, Result, Server}; use lru_cache::LruCache; use ruma::{CanonicalJsonValue, OwnedDeviceId, OwnedRoomId, OwnedUserId}; -use crate::{maps, maps::Maps, Engine, Map}; +use crate::{cork::Cork, maps, maps::Maps, Engine, Map}; pub struct Database { pub db: Arc, @@ -38,6 +38,15 @@ impl Database { )), }) } + + #[must_use] + pub fn cork(&self) -> Cork { Cork::new(&self.db, false, false) } + + #[must_use] + pub fn cork_and_flush(&self) -> Cork { Cork::new(&self.db, true, false) } + + #[must_use] + pub fn cork_and_sync(&self) -> Cork { Cork::new(&self.db, true, true) } } impl Index<&str> for Database { diff --git a/src/database/engine.rs b/src/database/engine.rs index c6f9eaa5..7acd470f 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -122,7 +122,7 @@ impl Engine { pub fn sync(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, true)) } - pub(crate) fn corked(&self) -> bool { self.corks.load(std::sync::atomic::Ordering::Relaxed) > 0 } + pub fn corked(&self) -> bool { self.corks.load(std::sync::atomic::Ordering::Relaxed) > 0 } pub(crate) fn cork(&self) { self.corks diff --git a/src/database/mod.rs b/src/database/mod.rs index 8b65e230..e10f6cee 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,4 +1,4 @@ -pub mod cork; +mod cork; mod database; mod engine; mod map; @@ -10,7 +10,6 @@ mod watchers; extern crate conduit_core as conduit; extern crate rust_rocksdb as rocksdb; -pub use cork::Cork; pub use database::Database; pub(crate) use engine::Engine; pub use map::Map; diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index bf3eb68b..ccd50ee9 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -4,7 +4,7 @@ use std::{ }; use conduit::{trace, utils, Error, Result}; -use database::{Cork, Database, Map}; +use database::{Database, Map}; use futures_util::{stream::FuturesUnordered, StreamExt}; use lru_cache::LruCache; use ruma::{ @@ -185,10 +185,6 @@ impl Data { pub fn cleanup(&self) -> Result<()> { self.db.db.cleanup() } - pub fn cork(&self) -> Cork { Cork::new(&self.db.db, false, false) } - - pub fn cork_and_flush(&self) -> Cork { Cork::new(&self.db.db, true, false) } - pub fn memory_usage(&self) -> String { let auth_chain_cache = self.db.auth_chain_cache.lock().unwrap().len(); let appservice_in_room_cache = self.db.appservice_in_room_cache.read().unwrap().len(); diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 0f52d5af..c05e8df8 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -818,7 +818,7 @@ async fn handle_media_check( async fn fix_bad_double_separator_in_state_cache(db: &Arc, _config: &Config) -> Result<()> { warn!("Fixing bad double separator in state_cache roomuserid_joined"); let roomuserid_joined = &db["roomuserid_joined"]; - let _cork = database::Cork::new(&db.db, true, true); + let _cork = db.cork_and_sync(); let mut iter_count: usize = 0; for (mut key, value) in roomuserid_joined.iter() { @@ -851,7 +851,7 @@ async fn fix_bad_double_separator_in_state_cache(db: &Arc, _config: &C async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &Arc, _config: &Config) -> Result<()> { warn!("Retroactively fixing bad data from broken roomuserid_joined"); - let _cork = database::Cork::new(&db.db, true, true); + let _cork = db.cork_and_sync(); let room_ids = services() .rooms diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index d18c76ef..aba712fa 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -207,7 +207,7 @@ impl Service { state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex ) -> Result> { // Coalesce database writes for the remainder of this scope. - let _cork = services().globals.db.cork_and_flush(); + let _cork = services().db.cork_and_flush(); let shortroomid = services() .rooms diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 3894ded1..9bda6cad 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -85,7 +85,7 @@ impl Service { pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { let dest = Destination::Push(user.to_owned(), pushkey); let event = SendingEvent::Pdu(pdu_id.to_owned()); - let _cork = services().globals.db.cork(); + let _cork = services().db.cork(); let keys = self.db.queue_requests(&[(&dest, event.clone())])?; self.dispatch(Msg { dest, @@ -98,7 +98,7 @@ impl Service { pub fn send_pdu_appservice(&self, appservice_id: String, pdu_id: Vec) -> Result<()> { let dest = Destination::Appservice(appservice_id); let event = SendingEvent::Pdu(pdu_id); - let _cork = services().globals.db.cork(); + let _cork = services().db.cork(); let keys = self.db.queue_requests(&[(&dest, event.clone())])?; self.dispatch(Msg { dest, @@ -125,7 +125,7 @@ impl Service { .into_iter() .map(|server| (Destination::Normal(server), SendingEvent::Pdu(pdu_id.to_owned()))) .collect::>(); - let _cork = services().globals.db.cork(); + let _cork = services().db.cork(); let keys = self.db.queue_requests( &requests .iter() @@ -147,7 +147,7 @@ impl Service { pub fn send_edu_server(&self, server: &ServerName, serialized: Vec) -> Result<()> { let dest = Destination::Normal(server.to_owned()); let event = SendingEvent::Edu(serialized); - let _cork = services().globals.db.cork(); + let _cork = services().db.cork(); let keys = self.db.queue_requests(&[(&dest, event.clone())])?; self.dispatch(Msg { dest, @@ -174,7 +174,7 @@ impl Service { .into_iter() .map(|server| (Destination::Normal(server), SendingEvent::Edu(serialized.clone()))) .collect::>(); - let _cork = services().globals.db.cork(); + let _cork = services().db.cork(); let keys = self.db.queue_requests( &requests .iter() diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index e1a14dfd..aa2865ec 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -100,7 +100,7 @@ impl Service { fn handle_response_ok( &self, dest: &Destination, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, ) { - let _cork = services().globals.db.cork(); + let _cork = services().db.cork(); self.db .delete_all_active_requests_for(dest) .expect("all active requests deleted"); @@ -173,7 +173,7 @@ impl Service { return Ok(None); } - let _cork = services().globals.db.cork(); + let _cork = services().db.cork(); let mut events = Vec::new(); // Must retry any previous transaction for this remote. @@ -187,7 +187,7 @@ impl Service { } // Compose the next transaction - let _cork = services().globals.db.cork(); + let _cork = services().db.cork(); if !new_events.is_empty() { self.db.mark_as_active(&new_events)?; for (e, _) in new_events { From 0522fe7d92abda7fe1478609f6cb4d334209a9fb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 2 Jul 2024 00:03:54 +0000 Subject: [PATCH 0335/2091] reimplement iterator from lowlevel Signed-off-by: Jason Volk --- src/database/iter.rs | 105 +++++++++++++++++++++++++++++++++++++++++++ src/database/map.rs | 37 ++++----------- src/database/mod.rs | 2 + 3 files changed, 115 insertions(+), 29 deletions(-) create mode 100644 src/database/iter.rs diff --git a/src/database/iter.rs b/src/database/iter.rs new file mode 100644 index 00000000..bbbab491 --- /dev/null +++ b/src/database/iter.rs @@ -0,0 +1,105 @@ +use std::{iter::FusedIterator, sync::Arc}; + +use conduit::Result; +use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode, Direction, IteratorMode, ReadOptions}; + +use crate::{engine::Db, map::KeyVal, result, Engine}; + +type Cursor<'cursor> = DBRawIteratorWithThreadMode<'cursor, Db>; +type Key<'item> = &'item [u8]; +type Val<'item> = &'item [u8]; +type Item<'item> = (Key<'item>, Val<'item>); + +struct State<'cursor> { + cursor: Cursor<'cursor>, + direction: Direction, + valid: bool, + init: bool, +} + +impl<'cursor> State<'cursor> { + pub(crate) fn new( + db: &'cursor Arc, cf: &'cursor Arc, opts: ReadOptions, mode: &IteratorMode<'_>, + ) -> Self { + let mut cursor = db.db.raw_iterator_cf_opt(&**cf, opts); + let direction = into_direction(mode); + let valid = seek_init(&mut cursor, mode); + Self { + cursor, + direction, + valid, + init: true, + } + } +} + +pub struct Iter<'cursor> { + state: State<'cursor>, +} + +impl<'cursor> Iter<'cursor> { + pub(crate) fn new( + db: &'cursor Arc, cf: &'cursor Arc, opts: ReadOptions, mode: &IteratorMode<'_>, + ) -> Self { + Self { + state: State::new(db, cf, opts, mode), + } + } +} + +impl Iterator for Iter<'_> { + type Item = KeyVal; + + fn next(&mut self) -> Option { + if !self.state.init && self.state.valid { + seek_next(&mut self.state.cursor, self.state.direction); + } else if self.state.init { + self.state.init = false; + } + + self.state.cursor.item().map(into_keyval).or_else(|| { + when_invalid(&mut self.state).expect("iterator invalidated due to error"); + None + }) + } +} + +impl FusedIterator for Iter<'_> {} + +fn when_invalid(state: &mut State<'_>) -> Result<()> { + state.valid = false; + result(state.cursor.status()) +} + +fn seek_next(cursor: &mut Cursor<'_>, direction: Direction) { + match direction { + Direction::Forward => cursor.next(), + Direction::Reverse => cursor.prev(), + } +} + +fn seek_init(cursor: &mut Cursor<'_>, mode: &IteratorMode<'_>) -> bool { + use Direction::{Forward, Reverse}; + use IteratorMode::{End, From, Start}; + + match mode { + Start => cursor.seek_to_first(), + End => cursor.seek_to_last(), + From(key, Forward) => cursor.seek(key), + From(key, Reverse) => cursor.seek_for_prev(key), + }; + + cursor.valid() +} + +fn into_direction(mode: &IteratorMode<'_>) -> Direction { + use Direction::{Forward, Reverse}; + use IteratorMode::{End, From, Start}; + + match mode { + Start | From(_, Forward) => Forward, + End | From(_, Reverse) => Reverse, + } +} + +fn into_keyval((key, val): Item<'_>) -> KeyVal { (Vec::from(key), Vec::from(val)) } diff --git a/src/database/map.rs b/src/database/map.rs index 8f21f76d..afd74080 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -5,7 +5,7 @@ use rocksdb::{ AsColumnFamilyRef, ColumnFamily, Direction, IteratorMode, ReadOptions, WriteBatchWithTransaction, WriteOptions, }; -use super::{or_else, result, watchers::Watchers, Engine}; +use crate::{or_else, result, watchers::Watchers, Engine, Iter}; pub struct Map { name: String, @@ -16,9 +16,9 @@ pub struct Map { read_options: ReadOptions, } -type Key = Vec; -type Val = Vec; -type KeyVal = (Key, Val); +pub(crate) type KeyVal = (Key, Val); +pub(crate) type Val = Vec; +pub(crate) type Key = Vec; impl Map { pub(crate) fn open(db: &Arc, name: &str) -> Result> { @@ -121,15 +121,9 @@ impl Map { } pub fn iter<'a>(&'a self) -> Box + 'a> { + let mode = IteratorMode::Start; let read_options = read_options_default(); - let it = self - .db - .db - .iterator_cf_opt(&self.cf(), read_options, IteratorMode::Start) - .map(Result::unwrap) - .map(|(k, v)| (Vec::from(k), Vec::from(v))); - - Box::new(it) + Box::new(Iter::new(&self.db, &self.cf, read_options, &mode)) } pub fn iter_from<'a>(&'a self, from: &[u8], backwards: bool) -> Box + 'a> { @@ -140,28 +134,13 @@ impl Map { }; let mode = IteratorMode::From(from, direction); let read_options = read_options_default(); - let it = self - .db - .db - .iterator_cf_opt(&self.cf(), read_options, mode) - .map(Result::unwrap) - .map(|(k, v)| (Vec::from(k), Vec::from(v))); - - Box::new(it) + Box::new(Iter::new(&self.db, &self.cf, read_options, &mode)) } pub fn scan_prefix<'a>(&'a self, prefix: Vec) -> Box + 'a> { let mode = IteratorMode::From(&prefix, Direction::Forward); let read_options = read_options_default(); - let it = self - .db - .db - .iterator_cf_opt(&self.cf(), read_options, mode) - .map(Result::unwrap) - .map(|(k, v)| (Vec::from(k), Vec::from(v))) - .take_while(move |(k, _)| k.starts_with(&prefix)); - - Box::new(it) + Box::new(Iter::new(&self.db, &self.cf, read_options, &mode).take_while(move |(k, _)| k.starts_with(&prefix))) } pub fn increment(&self, key: &[u8]) -> Result> { diff --git a/src/database/mod.rs b/src/database/mod.rs index e10f6cee..74e598b6 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,6 +1,7 @@ mod cork; mod database; mod engine; +mod iter; mod map; pub mod maps; mod opts; @@ -12,6 +13,7 @@ extern crate rust_rocksdb as rocksdb; pub use database::Database; pub(crate) use engine::Engine; +pub use iter::Iter; pub use map::Map; pub(crate) use util::{or_else, result}; From ee64fb149c2927450f8755df1ab8b31bc3c24b97 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 2 Jul 2024 05:56:10 +0000 Subject: [PATCH 0336/2091] optimize get w/ zero-copy ref handle Signed-off-by: Jason Volk --- src/database/handle.rs | 25 +++++++++++++++++++++++++ src/database/map.rs | 8 ++++---- src/database/mod.rs | 2 ++ src/service/globals/data.rs | 2 +- src/service/globals/migrations.rs | 6 +++--- src/service/rooms/alias/data.rs | 2 +- src/service/rooms/event_handler/mod.rs | 4 ++-- src/service/rooms/timeline/data.rs | 2 +- src/service/rooms/timeline/mod.rs | 7 +++++-- src/service/transaction_ids/data.rs | 2 +- src/service/transaction_ids/mod.rs | 2 +- 11 files changed, 46 insertions(+), 16 deletions(-) create mode 100644 src/database/handle.rs diff --git a/src/database/handle.rs b/src/database/handle.rs new file mode 100644 index 00000000..787a5cea --- /dev/null +++ b/src/database/handle.rs @@ -0,0 +1,25 @@ +use std::ops::Deref; + +use rocksdb::DBPinnableSlice; + +pub struct Handle<'a> { + val: DBPinnableSlice<'a>, +} + +impl<'a> From> for Handle<'a> { + fn from(val: DBPinnableSlice<'a>) -> Self { + Self { + val, + } + } +} + +impl Deref for Handle<'_> { + type Target = [u8]; + + fn deref(&self) -> &Self::Target { &self.val } +} + +impl AsRef<[u8]> for Handle<'_> { + fn as_ref(&self) -> &[u8] { &self.val } +} diff --git a/src/database/map.rs b/src/database/map.rs index afd74080..40a810bc 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -5,7 +5,7 @@ use rocksdb::{ AsColumnFamilyRef, ColumnFamily, Direction, IteratorMode, ReadOptions, WriteBatchWithTransaction, WriteOptions, }; -use crate::{or_else, result, watchers::Watchers, Engine, Iter}; +use crate::{or_else, result, watchers::Watchers, Engine, Handle, Iter}; pub struct Map { name: String, @@ -32,11 +32,11 @@ impl Map { })) } - pub fn get(&self, key: &[u8]) -> Result>> { + pub fn get(&self, key: &[u8]) -> Result>> { let read_options = &self.read_options; - let res = self.db.db.get_cf_opt(&self.cf(), key, read_options); + let res = self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options); - result(res) + Ok(result(res)?.map(Handle::from)) } pub fn multi_get(&self, keys: &[&[u8]]) -> Result>>> { diff --git a/src/database/mod.rs b/src/database/mod.rs index 74e598b6..39566896 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,6 +1,7 @@ mod cork; mod database; mod engine; +mod handle; mod iter; mod map; pub mod maps; @@ -13,6 +14,7 @@ extern crate rust_rocksdb as rocksdb; pub use database::Database; pub(crate) use engine::Engine; +pub use handle::Handle; pub use iter::Iter; pub use map::Map; pub(crate) use util::{or_else, result}; diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index ccd50ee9..59df5e7f 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -226,7 +226,7 @@ lasttimelinecount_cache: {lasttimelinecount_cache} / {max_lasttimelinecount_cach self.global.insert(b"keypair", &keypair)?; Ok::<_, Error>(keypair) }, - Ok, + |val| Ok(val.to_vec()), )?; let mut parts = keypair_bytes.splitn(2, |&b| b == 0xFF); diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index c05e8df8..4109017d 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -479,7 +479,7 @@ async fn db_lt_8(db: &Arc, _config: &Config) -> Result<()> { .unwrap() .expect("shortroomid should exist"); - let mut new_key = short_room_id; + let mut new_key = short_room_id.to_vec(); new_key.extend_from_slice(count); Some((new_key, v)) @@ -500,7 +500,7 @@ async fn db_lt_8(db: &Arc, _config: &Config) -> Result<()> { .unwrap() .expect("shortroomid should exist"); - let mut new_value = short_room_id; + let mut new_value = short_room_id.to_vec(); new_value.extend_from_slice(count); Some((k, new_value)) @@ -534,7 +534,7 @@ async fn db_lt_9(db: &Arc, _config: &Config) -> Result<()> { .get(room_id) .unwrap() .expect("shortroomid should exist"); - let mut new_key = short_room_id; + let mut new_key = short_room_id.to_vec(); new_key.extend_from_slice(word); new_key.push(0xFF); new_key.extend_from_slice(pdu_id_count); diff --git a/src/service/rooms/alias/data.rs b/src/service/rooms/alias/data.rs index 04f5ca7f..eb99e543 100644 --- a/src/service/rooms/alias/data.rs +++ b/src/service/rooms/alias/data.rs @@ -39,7 +39,7 @@ impl Data { pub(super) fn remove_alias(&self, alias: &RoomAliasId) -> Result<()> { if let Some(room_id) = self.alias_roomid.get(alias.alias().as_bytes())? { - let mut prefix = room_id; + let mut prefix = room_id.to_vec(); prefix.push(0xFF); for (key, _) in self.aliasid_alias.scan_prefix(prefix) { diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 0468ab38..ed09e2e6 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -83,7 +83,7 @@ impl Service { ) -> Result>> { // 1. Skip the PDU if we already have it as a timeline event if let Some(pdu_id) = services().rooms.timeline.get_pdu_id(event_id)? { - return Ok(Some(pdu_id)); + return Ok(Some(pdu_id.to_vec())); } // 1.1 Check the server is in the room @@ -449,7 +449,7 @@ impl Service { ) -> Result>> { // Skip the PDU if we already have it as a timeline event if let Ok(Some(pduid)) = services().rooms.timeline.get_pdu_id(&incoming_pdu.event_id) { - return Ok(Some(pduid)); + return Ok(Some(pduid.to_vec())); } if services() diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index ba74b1aa..6609bde8 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -93,7 +93,7 @@ impl Data { } /// Returns the pdu's id. - pub(super) fn get_pdu_id(&self, event_id: &EventId) -> Result>> { + pub(super) fn get_pdu_id(&self, event_id: &EventId) -> Result>> { self.eventid_pduid.get(event_id.as_bytes()) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index aba712fa..7aed2c0d 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -159,7 +159,9 @@ impl Service { } /// Returns the pdu's id. - pub fn get_pdu_id(&self, event_id: &EventId) -> Result>> { self.db.get_pdu_id(event_id) } + pub fn get_pdu_id(&self, event_id: &EventId) -> Result>> { + self.db.get_pdu_id(event_id) + } /// Returns the pdu. /// @@ -1185,7 +1187,8 @@ impl Service { // Skip the PDU if we already have it as a timeline event if let Some(pdu_id) = self.get_pdu_id(&event_id)? { - info!("We already know {event_id} at {pdu_id:?}"); + let pdu_id = pdu_id.to_vec(); + debug!("We already know {event_id} at {pdu_id:?}"); return Ok(()); } diff --git a/src/service/transaction_ids/data.rs b/src/service/transaction_ids/data.rs index 668852ca..791b46f0 100644 --- a/src/service/transaction_ids/data.rs +++ b/src/service/transaction_ids/data.rs @@ -31,7 +31,7 @@ impl Data { pub(super) fn existing_txnid( &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, - ) -> Result>> { + ) -> Result>> { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(device_id.map(DeviceId::as_bytes).unwrap_or_default()); diff --git a/src/service/transaction_ids/mod.rs b/src/service/transaction_ids/mod.rs index 5b94ef5b..f18bbe64 100644 --- a/src/service/transaction_ids/mod.rs +++ b/src/service/transaction_ids/mod.rs @@ -26,7 +26,7 @@ impl Service { pub fn existing_txnid( &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, - ) -> Result>> { + ) -> Result>> { self.db.existing_txnid(user_id, device_id, txn_id) } } From 2e2cf08bb21689b85dcc45a00e5ccf0564f4b965 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 2 Jul 2024 06:24:01 +0000 Subject: [PATCH 0337/2091] optimize increment / increment batch Signed-off-by: Jason Volk --- src/database/map.rs | 34 +++++++--------------------------- 1 file changed, 7 insertions(+), 27 deletions(-) diff --git a/src/database/map.rs b/src/database/map.rs index 40a810bc..f7b16b7e 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -143,54 +143,34 @@ impl Map { Box::new(Iter::new(&self.db, &self.cf, read_options, &mode).take_while(move |(k, _)| k.starts_with(&prefix))) } - pub fn increment(&self, key: &[u8]) -> Result> { - let read_options = &self.read_options; - let old = self - .db - .db - .get_cf_opt(&self.cf(), key, read_options) - .or_else(or_else)?; - + pub fn increment(&self, key: &[u8]) -> Result<[u8; 8]> { + let old = self.get(key)?; let new = utils::increment(old.as_deref()); - - let write_options = &self.write_options; - self.db - .db - .put_cf_opt(&self.cf(), key, new, write_options) - .or_else(or_else)?; + self.insert(key, &new)?; if !self.db.corked() { self.db.flush()?; } - Ok(new.to_vec()) + Ok(new) } pub fn increment_batch(&self, iter: &mut dyn Iterator) -> Result<()> { let mut batch = WriteBatchWithTransaction::::default(); - - let read_options = &self.read_options; for key in iter { - let old = self - .db - .db - .get_cf_opt(&self.cf(), &key, read_options) - .or_else(or_else)?; + let old = self.get(&key)?; let new = utils::increment(old.as_deref()); batch.put_cf(&self.cf(), key, new); } let write_options = &self.write_options; - self.db - .db - .write_opt(batch, write_options) - .or_else(or_else)?; + let res = self.db.db.write_opt(batch, write_options); if !self.db.corked() { self.db.flush()?; } - Ok(()) + result(res) } pub fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { From 46423cab4f2fc0d22d0c721470257f4f458002a6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 2 Jul 2024 07:56:45 +0000 Subject: [PATCH 0338/2091] fix counter increment race Signed-off-by: Jason Volk --- src/service/globals/data.rs | 38 ++++++++++++++++++++++++++++++------- src/service/globals/mod.rs | 6 +++--- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 59df5e7f..823adb46 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -1,6 +1,6 @@ use std::{ collections::{BTreeMap, HashMap}, - sync::Arc, + sync::{Arc, RwLock}, }; use conduit::{trace, utils, Error, Result}; @@ -33,6 +33,7 @@ pub struct Data { readreceiptid_readreceipt: Arc, userid_lastonetimekeyupdate: Arc, pub(super) db: Arc, + counter: RwLock, } impl Data { @@ -52,18 +53,41 @@ impl Data { readreceiptid_readreceipt: db["readreceiptid_readreceipt"].clone(), userid_lastonetimekeyupdate: db["userid_lastonetimekeyupdate"].clone(), db: db.clone(), + counter: RwLock::new(Self::stored_count(&db["global"]).expect("initialized global counter")), } } pub fn next_count(&self) -> Result { - utils::u64_from_bytes(&self.global.increment(COUNTER)?) - .map_err(|_| Error::bad_database("Count has invalid bytes.")) + let mut lock = self.counter.write().expect("locked"); + let counter: &mut u64 = &mut lock; + debug_assert!( + *counter == Self::stored_count(&self.global).expect("database failure"), + "counter mismatch" + ); + + *counter = counter.wrapping_add(1); + self.global.insert(COUNTER, &counter.to_be_bytes())?; + + Ok(*counter) } - pub fn current_count(&self) -> Result { - self.global.get(COUNTER)?.map_or(Ok(0_u64), |bytes| { - utils::u64_from_bytes(&bytes).map_err(|_| Error::bad_database("Count has invalid bytes.")) - }) + #[inline] + pub fn current_count(&self) -> u64 { + let lock = self.counter.read().expect("locked"); + let counter: &u64 = &lock; + debug_assert!( + *counter == Self::stored_count(&self.global).expect("database failure"), + "counter mismatch" + ); + + *counter + } + + fn stored_count(global: &Arc) -> Result { + global + .get(COUNTER)? + .as_deref() + .map_or(Ok(0_u64), utils::u64_from_bytes) } pub fn last_check_for_updates_id(&self) -> Result { diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 0cf0fccf..979f671f 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -140,11 +140,11 @@ impl Service { /// Returns this server's keypair. pub fn keypair(&self) -> &ruma::signatures::Ed25519KeyPair { &self.keypair } - #[tracing::instrument(skip(self))] + #[inline] pub fn next_count(&self) -> Result { self.db.next_count() } - #[tracing::instrument(skip(self))] - pub fn current_count(&self) -> Result { self.db.current_count() } + #[inline] + pub fn current_count(&self) -> Result { Ok(self.db.current_count()) } #[tracing::instrument(skip(self))] pub fn last_check_for_updates_id(&self) -> Result { self.db.last_check_for_updates_id() } From a2d25215a35e832735b0ddb75c06c4c138e6a940 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 2 Jul 2024 09:51:00 +0000 Subject: [PATCH 0339/2091] consolidate key/value types; consistent interface arguments Signed-off-by: Jason Volk --- src/database/iter.rs | 27 ++++++----- src/database/map.rs | 58 ++++++++++++++--------- src/database/mod.rs | 2 + src/database/slice.rs | 64 +++++++++++++++++++++++++ src/service/globals/migrations.rs | 75 ++++++++++++++++-------------- src/service/rooms/search/data.rs | 19 ++++---- src/service/rooms/timeline/data.rs | 4 +- src/service/sending/data.rs | 2 +- 8 files changed, 172 insertions(+), 79 deletions(-) create mode 100644 src/database/slice.rs diff --git a/src/database/iter.rs b/src/database/iter.rs index bbbab491..4845e977 100644 --- a/src/database/iter.rs +++ b/src/database/iter.rs @@ -3,12 +3,14 @@ use std::{iter::FusedIterator, sync::Arc}; use conduit::Result; use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode, Direction, IteratorMode, ReadOptions}; -use crate::{engine::Db, map::KeyVal, result, Engine}; +use crate::{ + engine::Db, + result, + slice::{OwnedKeyVal, OwnedKeyValPair}, + Engine, +}; type Cursor<'cursor> = DBRawIteratorWithThreadMode<'cursor, Db>; -type Key<'item> = &'item [u8]; -type Val<'item> = &'item [u8]; -type Item<'item> = (Key<'item>, Val<'item>); struct State<'cursor> { cursor: Cursor<'cursor>, @@ -48,7 +50,7 @@ impl<'cursor> Iter<'cursor> { } impl Iterator for Iter<'_> { - type Item = KeyVal; + type Item = OwnedKeyValPair; fn next(&mut self) -> Option { if !self.state.init && self.state.valid { @@ -57,10 +59,15 @@ impl Iterator for Iter<'_> { self.state.init = false; } - self.state.cursor.item().map(into_keyval).or_else(|| { - when_invalid(&mut self.state).expect("iterator invalidated due to error"); - None - }) + self.state + .cursor + .item() + .map(OwnedKeyVal::from) + .map(OwnedKeyVal::to_tuple) + .or_else(|| { + when_invalid(&mut self.state).expect("iterator invalidated due to error"); + None + }) } } @@ -101,5 +108,3 @@ fn into_direction(mode: &IteratorMode<'_>) -> Direction { End | From(_, Reverse) => Reverse, } } - -fn into_keyval((key, val): Item<'_>) -> KeyVal { (Vec::from(key), Vec::from(val)) } diff --git a/src/database/map.rs b/src/database/map.rs index f7b16b7e..73cf5107 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -1,11 +1,16 @@ -use std::{future::Future, pin::Pin, sync::Arc}; +use std::{future::Future, mem::size_of, pin::Pin, sync::Arc}; use conduit::{utils, Result}; use rocksdb::{ AsColumnFamilyRef, ColumnFamily, Direction, IteratorMode, ReadOptions, WriteBatchWithTransaction, WriteOptions, }; -use crate::{or_else, result, watchers::Watchers, Engine, Handle, Iter}; +use crate::{ + or_else, result, + slice::{Byte, Key, KeyVal, OwnedKey, OwnedKeyValPair, OwnedVal, Val}, + watchers::Watchers, + Engine, Handle, Iter, +}; pub struct Map { name: String, @@ -16,9 +21,7 @@ pub struct Map { read_options: ReadOptions, } -pub(crate) type KeyVal = (Key, Val); -pub(crate) type Val = Vec; -pub(crate) type Key = Vec; +type OwnedKeyValPairIter<'a> = Box + Send + 'a>; impl Map { pub(crate) fn open(db: &Arc, name: &str) -> Result> { @@ -32,19 +35,19 @@ impl Map { })) } - pub fn get(&self, key: &[u8]) -> Result>> { + pub fn get(&self, key: &Key) -> Result>> { let read_options = &self.read_options; let res = self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options); Ok(result(res)?.map(Handle::from)) } - pub fn multi_get(&self, keys: &[&[u8]]) -> Result>>> { + pub fn multi_get(&self, keys: &[&Key]) -> Result>> { // Optimization can be `true` if key vector is pre-sorted **by the column // comparator**. const SORTED: bool = false; - let mut ret: Vec>> = Vec::with_capacity(keys.len()); + let mut ret: Vec> = Vec::with_capacity(keys.len()); let read_options = &self.read_options; for res in self .db @@ -61,7 +64,7 @@ impl Map { Ok(ret) } - pub fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> { + pub fn insert(&self, key: &Key, value: &Val) -> Result<()> { let write_options = &self.write_options; self.db .db @@ -77,9 +80,12 @@ impl Map { Ok(()) } - pub fn insert_batch(&self, iter: &mut dyn Iterator) -> Result<()> { + pub fn insert_batch<'a, I>(&'a self, iter: I) -> Result<()> + where + I: Iterator>, + { let mut batch = WriteBatchWithTransaction::::default(); - for (key, value) in iter { + for KeyVal(key, value) in iter { batch.put_cf(&self.cf(), key, value); } @@ -93,7 +99,7 @@ impl Map { result(res) } - pub fn remove(&self, key: &[u8]) -> Result<()> { + pub fn remove(&self, key: &Key) -> Result<()> { let write_options = &self.write_options; let res = self.db.db.delete_cf_opt(&self.cf(), key, write_options); @@ -104,7 +110,10 @@ impl Map { result(res) } - pub fn remove_batch(&self, iter: &mut dyn Iterator) -> Result<()> { + pub fn remove_batch<'a, I>(&'a self, iter: I) -> Result<()> + where + I: Iterator, + { let mut batch = WriteBatchWithTransaction::::default(); for key in iter { batch.delete_cf(&self.cf(), key); @@ -120,14 +129,14 @@ impl Map { result(res) } - pub fn iter<'a>(&'a self) -> Box + 'a> { + pub fn iter(&self) -> OwnedKeyValPairIter<'_> { let mode = IteratorMode::Start; let read_options = read_options_default(); Box::new(Iter::new(&self.db, &self.cf, read_options, &mode)) } - pub fn iter_from<'a>(&'a self, from: &[u8], backwards: bool) -> Box + 'a> { - let direction = if backwards { + pub fn iter_from(&self, from: &Key, reverse: bool) -> OwnedKeyValPairIter<'_> { + let direction = if reverse { Direction::Reverse } else { Direction::Forward @@ -137,13 +146,13 @@ impl Map { Box::new(Iter::new(&self.db, &self.cf, read_options, &mode)) } - pub fn scan_prefix<'a>(&'a self, prefix: Vec) -> Box + 'a> { + pub fn scan_prefix(&self, prefix: OwnedKey) -> OwnedKeyValPairIter<'_> { let mode = IteratorMode::From(&prefix, Direction::Forward); let read_options = read_options_default(); Box::new(Iter::new(&self.db, &self.cf, read_options, &mode).take_while(move |(k, _)| k.starts_with(&prefix))) } - pub fn increment(&self, key: &[u8]) -> Result<[u8; 8]> { + pub fn increment(&self, key: &Key) -> Result<[Byte; size_of::()]> { let old = self.get(key)?; let new = utils::increment(old.as_deref()); self.insert(key, &new)?; @@ -155,10 +164,13 @@ impl Map { Ok(new) } - pub fn increment_batch(&self, iter: &mut dyn Iterator) -> Result<()> { + pub fn increment_batch<'a, I>(&'a self, iter: I) -> Result<()> + where + I: Iterator, + { let mut batch = WriteBatchWithTransaction::::default(); for key in iter { - let old = self.get(&key)?; + let old = self.get(key)?; let new = utils::increment(old.as_deref()); batch.put_cf(&self.cf(), key, new); } @@ -173,7 +185,7 @@ impl Map { result(res) } - pub fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { + pub fn watch_prefix<'a>(&'a self, prefix: &Key) -> Pin + Send + 'a>> { self.watchers.watch(prefix) } @@ -184,8 +196,8 @@ impl Map { } impl<'a> IntoIterator for &'a Map { - type IntoIter = Box + 'a>; - type Item = KeyVal; + type IntoIter = Box + Send + 'a>; + type Item = OwnedKeyValPair; fn into_iter(self) -> Self::IntoIter { self.iter() } } diff --git a/src/database/mod.rs b/src/database/mod.rs index 39566896..283224f6 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -6,6 +6,7 @@ mod iter; mod map; pub mod maps; mod opts; +mod slice; mod util; mod watchers; @@ -17,6 +18,7 @@ pub(crate) use engine::Engine; pub use handle::Handle; pub use iter::Iter; pub use map::Map; +pub use slice::{Key, KeyVal, OwnedKey, OwnedKeyVal, OwnedVal, Val}; pub(crate) use util::{or_else, result}; conduit::mod_ctor! {} diff --git a/src/database/slice.rs b/src/database/slice.rs new file mode 100644 index 00000000..770f1957 --- /dev/null +++ b/src/database/slice.rs @@ -0,0 +1,64 @@ +pub struct OwnedKeyVal(pub OwnedKey, pub OwnedVal); +pub(crate) type OwnedKeyValPair = (OwnedKey, OwnedVal); +pub type OwnedVal = Vec; +pub type OwnedKey = Vec; + +pub struct KeyVal<'item>(pub &'item Key, pub &'item Val); +pub(crate) type KeyValPair<'item> = (&'item Key, &'item Val); +pub type Val = [Byte]; +pub type Key = [Byte]; + +pub(crate) type Byte = u8; + +impl OwnedKeyVal { + #[inline] + #[must_use] + pub fn as_slice(&self) -> KeyVal<'_> { KeyVal(&self.0, &self.1) } + + #[inline] + #[must_use] + pub fn to_tuple(self) -> OwnedKeyValPair { (self.0, self.1) } +} + +impl From for OwnedKeyVal { + #[inline] + fn from((key, val): OwnedKeyValPair) -> Self { Self(key, val) } +} + +impl From<&KeyVal<'_>> for OwnedKeyVal { + fn from(slice: &KeyVal<'_>) -> Self { slice.to_owned() } +} + +impl From> for OwnedKeyVal { + fn from((key, val): KeyValPair<'_>) -> Self { Self(Vec::from(key), Vec::from(val)) } +} + +impl From for OwnedKeyValPair { + #[inline] + fn from(val: OwnedKeyVal) -> Self { val.to_tuple() } +} + +impl KeyVal<'_> { + #[inline] + #[must_use] + pub fn to_owned(&self) -> OwnedKeyVal { OwnedKeyVal::from(self) } + + #[inline] + #[must_use] + pub fn as_tuple(&self) -> KeyValPair<'_> { (self.0, self.1) } +} + +impl<'a> From<&'a OwnedKeyVal> for KeyVal<'a> { + #[inline] + fn from(owned: &'a OwnedKeyVal) -> Self { owned.as_slice() } +} + +impl<'a> From<&'a OwnedKeyValPair> for KeyVal<'a> { + #[inline] + fn from((key, val): &'a OwnedKeyValPair) -> Self { KeyVal(key.as_slice(), val.as_slice()) } +} + +impl<'a> From> for KeyVal<'a> { + #[inline] + fn from((key, val): KeyValPair<'a>) -> Self { KeyVal(key, val) } +} diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 4109017d..a98eec4e 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -466,47 +466,53 @@ async fn db_lt_8(db: &Arc, _config: &Config) -> Result<()> { info!("Migration: 8"); } // Update pduids db layout - let mut batch = pduid_pdu.iter().filter_map(|(key, v)| { - if !key.starts_with(b"!") { - return None; - } - let mut parts = key.splitn(2, |&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let count = parts.next().unwrap(); + let batch = pduid_pdu + .iter() + .filter_map(|(key, v)| { + if !key.starts_with(b"!") { + return None; + } + let mut parts = key.splitn(2, |&b| b == 0xFF); + let room_id = parts.next().unwrap(); + let count = parts.next().unwrap(); - let short_room_id = roomid_shortroomid - .get(room_id) - .unwrap() - .expect("shortroomid should exist"); + let short_room_id = roomid_shortroomid + .get(room_id) + .unwrap() + .expect("shortroomid should exist"); - let mut new_key = short_room_id.to_vec(); - new_key.extend_from_slice(count); + let mut new_key = short_room_id.to_vec(); + new_key.extend_from_slice(count); - Some((new_key, v)) - }); + Some(database::OwnedKeyVal(new_key, v)) + }) + .collect::>(); - pduid_pdu.insert_batch(&mut batch)?; + pduid_pdu.insert_batch(batch.iter().map(database::KeyVal::from))?; - let mut batch2 = eventid_pduid.iter().filter_map(|(k, value)| { - if !value.starts_with(b"!") { - return None; - } - let mut parts = value.splitn(2, |&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let count = parts.next().unwrap(); + let batch2 = eventid_pduid + .iter() + .filter_map(|(k, value)| { + if !value.starts_with(b"!") { + return None; + } + let mut parts = value.splitn(2, |&b| b == 0xFF); + let room_id = parts.next().unwrap(); + let count = parts.next().unwrap(); - let short_room_id = roomid_shortroomid - .get(room_id) - .unwrap() - .expect("shortroomid should exist"); + let short_room_id = roomid_shortroomid + .get(room_id) + .unwrap() + .expect("shortroomid should exist"); - let mut new_value = short_room_id.to_vec(); - new_value.extend_from_slice(count); + let mut new_value = short_room_id.to_vec(); + new_value.extend_from_slice(count); - Some((k, new_value)) - }); + Some(database::OwnedKeyVal(k, new_value)) + }) + .collect::>(); - eventid_pduid.insert_batch(&mut batch2)?; + eventid_pduid.insert_batch(batch2.iter().map(database::KeyVal::from))?; services().globals.bump_database_version(8)?; info!("Migration: 7 -> 8 finished"); @@ -538,12 +544,13 @@ async fn db_lt_9(db: &Arc, _config: &Config) -> Result<()> { new_key.extend_from_slice(word); new_key.push(0xFF); new_key.extend_from_slice(pdu_id_count); - Some((new_key, Vec::new())) + Some(database::OwnedKeyVal(new_key, Vec::::new())) }) .peekable(); while iter.peek().is_some() { - tokenids.insert_batch(&mut iter.by_ref().take(1000))?; + let batch = iter.by_ref().take(1000).collect::>(); + tokenids.insert_batch(batch.iter().map(database::KeyVal::from))?; debug!("Inserted smaller batch"); } diff --git a/src/service/rooms/search/data.rs b/src/service/rooms/search/data.rs index fe5d3edf..79b23cba 100644 --- a/src/service/rooms/search/data.rs +++ b/src/service/rooms/search/data.rs @@ -20,15 +20,18 @@ impl Data { } pub(super) fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { - let mut batch = tokenize(message_body).map(|word| { - let mut key = shortroomid.to_be_bytes().to_vec(); - key.extend_from_slice(word.as_bytes()); - key.push(0xFF); - key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here - (key, Vec::new()) - }); + let batch = tokenize(message_body) + .map(|word| { + let mut key = shortroomid.to_be_bytes().to_vec(); + key.extend_from_slice(word.as_bytes()); + key.push(0xFF); + key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here + (key, Vec::::new()) + }) + .collect::>(); - self.tokenids.insert_batch(&mut batch) + self.tokenids + .insert_batch(batch.iter().map(database::KeyVal::from)) } pub(super) fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 6609bde8..023457fa 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -268,9 +268,9 @@ impl Data { } self.userroomid_notificationcount - .increment_batch(&mut notifies_batch.into_iter())?; + .increment_batch(notifies_batch.iter().map(Vec::as_slice))?; self.userroomid_highlightcount - .increment_batch(&mut highlights_batch.into_iter())?; + .increment_batch(highlights_batch.iter().map(Vec::as_slice))?; Ok(()) } } diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 20e5b77a..22b69818 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -87,7 +87,7 @@ impl Data { keys.push(key); } self.servernameevent_data - .insert_batch(&mut batch.into_iter())?; + .insert_batch(batch.iter().map(database::KeyVal::from))?; Ok(keys) } From e1d1dac95ef21cf063f6106222cb922410bd2210 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 2 Jul 2024 21:39:29 +0000 Subject: [PATCH 0340/2091] add missing media compat example config Signed-off-by: Jason Volk --- conduwuit-example.toml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index f8f93070..9c6b7f0c 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -375,6 +375,20 @@ allow_profile_lookup_federation_requests = true # See: docs/development.md#debugging-with-tokio-console for more information. #tokio_console = false +# Enable backward-compatibility with Conduit's media directory by creating symlinks of media. This +# option is only necessary if you plan on using Conduit again. Otherwise setting this to false +# reduces filesystem clutter and overhead for managing these symlinks in the directory. +#media_compat_file_link = true + +# Checks consistency of the media directory at startup: +# 1. When `media_compat_file_link` is enbled, this check will upgrade media when switching back +# and forth between Conduit and Conduwuit. Both options must be enabled to handle this. +# 2. When media is deleted from the directory, this check will also delete its database entry. +# +# If none of these checks apply to your use cases, and your media directory is significantly large +# setting this to false may reduce startup time. +#media_statup_check = true + ### Generic database options From 473b29d5244d55f0b279b4d1bd6fa75c171faca3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 2 Jul 2024 22:40:58 +0000 Subject: [PATCH 0341/2091] move remaining runtime caches into their respective service Signed-off-by: Jason Volk --- Cargo.lock | 1 - src/database/Cargo.toml | 1 - src/database/database.rs | 24 ++-------------- src/service/globals/data.rs | 40 +++++++++++++-------------- src/service/globals/migrations.rs | 6 ++-- src/service/rooms/auth_chain/data.rs | 29 +++++++++++-------- src/service/rooms/auth_chain/mod.rs | 11 ++++++-- src/service/rooms/state_cache/data.rs | 17 ++++++------ src/service/rooms/state_cache/mod.rs | 13 +++++++++ src/service/rooms/timeline/data.rs | 21 ++++++++------ src/service/rooms/timeline/mod.rs | 13 +++++++++ src/service/uiaa/data.rs | 17 ++++++------ 12 files changed, 108 insertions(+), 85 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index edc41835..05547769 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -696,7 +696,6 @@ dependencies = [ "chrono", "conduit_core", "log", - "lru-cache", "ruma", "rust-rocksdb-uwu", "tokio", diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 104ef93d..f7f3c773 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -38,7 +38,6 @@ zstd_compression = [ chrono.workspace = true conduit-core.workspace = true log.workspace = true -lru-cache.workspace = true ruma.workspace = true rust-rocksdb.workspace = true tokio.workspace = true diff --git a/src/database/database.rs b/src/database/database.rs index 833b0047..2c6c6808 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -1,41 +1,21 @@ -use std::{ - collections::{BTreeMap, HashMap}, - ops::Index, - sync::{Arc, Mutex, RwLock}, -}; +use std::{ops::Index, sync::Arc}; -use conduit::{PduCount, Result, Server}; -use lru_cache::LruCache; -use ruma::{CanonicalJsonValue, OwnedDeviceId, OwnedRoomId, OwnedUserId}; +use conduit::{Result, Server}; use crate::{cork::Cork, maps, maps::Maps, Engine, Map}; pub struct Database { pub db: Arc, pub map: Maps, - - //TODO: not a database - pub userdevicesessionid_uiaarequest: RwLock>, - pub auth_chain_cache: Mutex, Arc<[u64]>>>, - pub appservice_in_room_cache: RwLock>>, - pub lasttimelinecount_cache: Mutex>, } impl Database { /// Load an existing database or create a new one. pub async fn open(server: &Arc) -> Result { - let config = &server.config; let db = Engine::open(server)?; Ok(Self { db: db.clone(), map: maps::open(&db)?, - - userdevicesessionid_uiaarequest: RwLock::new(BTreeMap::new()), - appservice_in_room_cache: RwLock::new(HashMap::new()), - lasttimelinecount_cache: Mutex::new(HashMap::new()), - auth_chain_cache: Mutex::new(LruCache::new( - (f64::from(config.auth_chain_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, - )), }) } diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 823adb46..022762f4 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -1,12 +1,11 @@ use std::{ - collections::{BTreeMap, HashMap}, + collections::BTreeMap, sync::{Arc, RwLock}, }; use conduit::{trace, utils, Error, Result}; use database::{Database, Map}; use futures_util::{stream::FuturesUnordered, StreamExt}; -use lru_cache::LruCache; use ruma::{ api::federation::discovery::{ServerSigningKeys, VerifyKey}, signatures::Ed25519KeyPair, @@ -210,36 +209,37 @@ impl Data { pub fn cleanup(&self) -> Result<()> { self.db.db.cleanup() } pub fn memory_usage(&self) -> String { - let auth_chain_cache = self.db.auth_chain_cache.lock().unwrap().len(); - let appservice_in_room_cache = self.db.appservice_in_room_cache.read().unwrap().len(); - let lasttimelinecount_cache = self.db.lasttimelinecount_cache.lock().unwrap().len(); - - let max_auth_chain_cache = self.db.auth_chain_cache.lock().unwrap().capacity(); - let max_appservice_in_room_cache = self.db.appservice_in_room_cache.read().unwrap().capacity(); - let max_lasttimelinecount_cache = self.db.lasttimelinecount_cache.lock().unwrap().capacity(); + let (auth_chain_cache, max_auth_chain_cache) = services().rooms.auth_chain.get_cache_usage(); + let (appservice_in_room_cache, max_appservice_in_room_cache) = services() + .rooms + .state_cache + .get_appservice_in_room_cache_usage(); + let (lasttimelinecount_cache, max_lasttimelinecount_cache) = services() + .rooms + .timeline + .get_lasttimelinecount_cache_usage(); format!( - "\ -auth_chain_cache: {auth_chain_cache} / {max_auth_chain_cache} -appservice_in_room_cache: {appservice_in_room_cache} / {max_appservice_in_room_cache} -lasttimelinecount_cache: {lasttimelinecount_cache} / {max_lasttimelinecount_cache}\n\n -{}", + "auth_chain_cache: {auth_chain_cache} / {max_auth_chain_cache}\nappservice_in_room_cache: \ + {appservice_in_room_cache} / {max_appservice_in_room_cache}\nlasttimelinecount_cache: \ + {lasttimelinecount_cache} / {max_lasttimelinecount_cache}\n\n{}", self.db.db.memory_usage().unwrap_or_default() ) } + #[allow(clippy::unused_self)] pub fn clear_caches(&self, amount: u32) { if amount > 1 { - let c = &mut *self.db.auth_chain_cache.lock().unwrap(); - *c = LruCache::new(c.capacity()); + services().rooms.auth_chain.clear_cache(); } if amount > 2 { - let c = &mut *self.db.appservice_in_room_cache.write().unwrap(); - *c = HashMap::new(); + services() + .rooms + .state_cache + .clear_appservice_in_room_cache(); } if amount > 3 { - let c = &mut *self.db.lasttimelinecount_cache.lock().unwrap(); - *c = HashMap::new(); + services().rooms.timeline.clear_lasttimelinecount_cache(); } } diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index a98eec4e..5cc25a1d 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -595,8 +595,10 @@ async fn db_lt_10(db: &Arc, _config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_11(db: &Arc, _config: &Config) -> Result<()> { - let _userdevicesessionid_uiaarequest = &db["userdevicesessionid_uiaarequest"]; +#[allow(unreachable_code)] +async fn db_lt_11(_db: &Arc, _config: &Config) -> Result<()> { + todo!("Dropping a column to clear data is not implemented yet."); + //let userdevicesessionid_uiaarequest = &db["userdevicesessionid_uiaarequest"]; //userdevicesessionid_uiaarequest.clear()?; services().globals.bump_database_version(11)?; diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index 4e844d6c..b2e80b7b 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -1,24 +1,31 @@ -use std::{mem::size_of, sync::Arc}; +use std::{ + mem::size_of, + sync::{Arc, Mutex}, +}; -use conduit::{utils, Result}; +use conduit::{utils, Result, Server}; use database::{Database, Map}; +use lru_cache::LruCache; pub(super) struct Data { shorteventid_authchain: Arc, - db: Arc, + pub(super) auth_chain_cache: Mutex, Arc<[u64]>>>, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(server: &Arc, db: &Arc) -> Self { + let config = &server.config; + let cache_size = f64::from(config.auth_chain_cache_capacity); + let cache_size = (cache_size * config.conduit_cache_capacity_modifier) as usize; Self { shorteventid_authchain: db["shorteventid_authchain"].clone(), - db: db.clone(), + auth_chain_cache: Mutex::new(LruCache::new(cache_size)), } } pub(super) fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result>> { // Check RAM cache - if let Some(result) = self.db.auth_chain_cache.lock().unwrap().get_mut(key) { + if let Some(result) = self.auth_chain_cache.lock().unwrap().get_mut(key) { return Ok(Some(Arc::clone(result))); } @@ -37,10 +44,9 @@ impl Data { if let Some(chain) = chain { // Cache in RAM - self.db - .auth_chain_cache + self.auth_chain_cache .lock() - .unwrap() + .expect("locked") .insert(vec![key[0]], Arc::clone(&chain)); return Ok(Some(chain)); @@ -63,10 +69,9 @@ impl Data { } // Cache in RAM - self.db - .auth_chain_cache + self.auth_chain_cache .lock() - .unwrap() + .expect("locked") .insert(key, auth_chain); Ok(()) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index ca9cf5f2..6b14fda0 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -17,9 +17,9 @@ pub struct Service { } impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { + pub fn build(server: &Arc, db: &Arc) -> Result { Ok(Self { - db: Data::new(db), + db: Data::new(server, db), }) } @@ -181,4 +181,11 @@ impl Service { self.db .cache_auth_chain(key, auth_chain.iter().copied().collect::>()) } + + pub fn get_cache_usage(&self) -> (usize, usize) { + let cache = self.db.auth_chain_cache.lock().expect("locked"); + (cache.len(), cache.capacity()) + } + + pub fn clear_cache(&self) { self.db.auth_chain_cache.lock().expect("locked").clear(); } } diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index c8d05ab0..f79ee678 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -1,4 +1,7 @@ -use std::{collections::HashSet, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, RwLock}, +}; use conduit::{utils, Error, Result}; use database::{Database, Map}; @@ -13,6 +16,7 @@ use crate::{appservice::RegistrationInfo, services, user_is_local}; type StrippedStateEventIter<'a> = Box>)>> + 'a>; type AnySyncStateEventIter<'a> = Box>)>> + 'a>; +type AppServiceInRoomCache = RwLock>>; pub(super) struct Data { userroomid_joined: Arc, @@ -27,7 +31,7 @@ pub(super) struct Data { roomid_invitedcount: Arc, roomserverids: Arc, serverroomids: Arc, - db: Arc, + pub(super) appservice_in_room_cache: AppServiceInRoomCache, } impl Data { @@ -45,7 +49,7 @@ impl Data { roomid_invitedcount: db["roomid_invitedcount"].clone(), roomserverids: db["roomserverids"].clone(), serverroomids: db["serverroomids"].clone(), - db: db.clone(), + appservice_in_room_cache: RwLock::new(HashMap::new()), } } @@ -201,8 +205,7 @@ impl Data { self.serverroomids.insert(&serverroom_id, &[])?; } - self.db - .appservice_in_room_cache + self.appservice_in_room_cache .write() .unwrap() .remove(room_id); @@ -213,7 +216,6 @@ impl Data { #[tracing::instrument(skip(self, room_id, appservice))] pub(super) fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { let maybe = self - .db .appservice_in_room_cache .read() .unwrap() @@ -235,8 +237,7 @@ impl Data { .room_members(room_id) .any(|userid| userid.map_or(false, |userid| appservice.users.is_match(userid.as_str()))); - self.db - .appservice_in_room_cache + self.appservice_in_room_cache .write() .unwrap() .entry(room_id.to_owned()) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 6572fcba..5038ef1c 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -441,4 +441,17 @@ impl Service { Ok(servers) } + + pub fn get_appservice_in_room_cache_usage(&self) -> (usize, usize) { + let cache = self.db.appservice_in_room_cache.read().expect("locked"); + (cache.len(), cache.capacity()) + } + + pub fn clear_appservice_in_room_cache(&self) { + self.db + .appservice_in_room_cache + .write() + .expect("locked") + .clear(); + } } diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 023457fa..2054caf7 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -1,8 +1,12 @@ -use std::{collections::hash_map, mem::size_of, sync::Arc}; +use std::{ + collections::{hash_map, HashMap}, + mem::size_of, + sync::{Arc, Mutex}, +}; use conduit::{error, utils, Error, Result}; use database::{Database, Map}; -use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedUserId, RoomId, UserId}; +use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; use crate::{services, PduCount, PduEvent}; @@ -12,11 +16,12 @@ pub(super) struct Data { eventid_outlierpdu: Arc, userroomid_notificationcount: Arc, userroomid_highlightcount: Arc, - db: Arc, + pub(super) lasttimelinecount_cache: LastTimelineCountCache, } type PdusIterItem = Result<(PduCount, PduEvent)>; type PdusIterator<'a> = Box + 'a>; +type LastTimelineCountCache = Mutex>; impl Data { pub(super) fn new(db: &Arc) -> Self { @@ -26,16 +31,15 @@ impl Data { eventid_outlierpdu: db["eventid_outlierpdu"].clone(), userroomid_notificationcount: db["userroomid_notificationcount"].clone(), userroomid_highlightcount: db["userroomid_highlightcount"].clone(), - db: db.clone(), + lasttimelinecount_cache: Mutex::new(HashMap::new()), } } pub(super) fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { match self - .db .lasttimelinecount_cache .lock() - .unwrap() + .expect("locked") .entry(room_id.to_owned()) { hash_map::Entry::Vacant(v) => { @@ -162,10 +166,9 @@ impl Data { &serde_json::to_vec(json).expect("CanonicalJsonObject is always a valid"), )?; - self.db - .lasttimelinecount_cache + self.lasttimelinecount_cache .lock() - .unwrap() + .expect("locked") .insert(pdu.room_id.clone(), PduCount::Normal(count)); self.eventid_pduid.insert(pdu.event_id.as_bytes(), pdu_id)?; diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 7aed2c0d..b9c0d7ba 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1241,6 +1241,19 @@ impl Service { debug!("Prepended backfill pdu"); Ok(()) } + + pub fn get_lasttimelinecount_cache_usage(&self) -> (usize, usize) { + let cache = self.db.lasttimelinecount_cache.lock().expect("locked"); + (cache.len(), cache.capacity()) + } + + pub fn clear_lasttimelinecount_cache(&self) { + self.db + .lasttimelinecount_cache + .lock() + .expect("locked") + .clear(); + } } #[cfg(test)] diff --git a/src/service/uiaa/data.rs b/src/service/uiaa/data.rs index 71d43dc2..ce071da0 100644 --- a/src/service/uiaa/data.rs +++ b/src/service/uiaa/data.rs @@ -1,30 +1,32 @@ -use std::sync::Arc; +use std::{ + collections::BTreeMap, + sync::{Arc, RwLock}, +}; use conduit::{Error, Result}; use database::{Database, Map}; use ruma::{ api::client::{error::ErrorKind, uiaa::UiaaInfo}, - CanonicalJsonValue, DeviceId, UserId, + CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedUserId, UserId, }; pub struct Data { + userdevicesessionid_uiaarequest: RwLock>, userdevicesessionid_uiaainfo: Arc, - db: Arc, } impl Data { pub(super) fn new(db: &Arc) -> Self { Self { + userdevicesessionid_uiaarequest: RwLock::new(BTreeMap::new()), userdevicesessionid_uiaainfo: db["userdevicesessionid_uiaainfo"].clone(), - db: db.clone(), } } pub(super) fn set_uiaa_request( &self, user_id: &UserId, device_id: &DeviceId, session: &str, request: &CanonicalJsonValue, ) -> Result<()> { - self.db - .userdevicesessionid_uiaarequest + self.userdevicesessionid_uiaarequest .write() .unwrap() .insert( @@ -38,8 +40,7 @@ impl Data { pub(super) fn get_uiaa_request( &self, user_id: &UserId, device_id: &DeviceId, session: &str, ) -> Option { - self.db - .userdevicesessionid_uiaarequest + self.userdevicesessionid_uiaarequest .read() .unwrap() .get(&(user_id.to_owned(), device_id.to_owned(), session.to_owned())) From df0328f43ff163bd18cf972858b47033a13a2749 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 00:06:00 +0000 Subject: [PATCH 0342/2091] split send txn handler Signed-off-by: Jason Volk --- src/api/server/send.rs | 568 +++++++++++++++++++++++------------------ 1 file changed, 321 insertions(+), 247 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 5fad790f..90225a14 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, time::Instant}; +use std::{collections::BTreeMap, net::IpAddr, time::Instant}; use axum_client_ip::InsecureClientIp; use conduit::debug_warn; @@ -6,12 +6,16 @@ use ruma::{ api::{ client::error::ErrorKind, federation::transactions::{ - edu::{DeviceListUpdateContent, DirectDeviceContent, Edu, SigningKeyUpdateContent}, + edu::{ + DeviceListUpdateContent, DirectDeviceContent, Edu, PresenceContent, ReceiptContent, + SigningKeyUpdateContent, TypingContent, + }, send_transaction_message, }, }, events::receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType}, to_device::DeviceIdOrAllDevices, + OwnedEventId, ServerName, }; use tokio::sync::RwLock; use tracing::{debug, error, trace, warn}; @@ -23,6 +27,8 @@ use crate::{ Error, Result, Ruma, }; +type ResolvedMap = BTreeMap>; + /// # `PUT /_matrix/federation/v1/send/{txnId}` /// /// Push EDUs and PDUs to this server. @@ -53,15 +59,39 @@ pub(crate) async fn send_transaction_message_route( )); } - // This is all the auth_events that have been recursively fetched so they don't - // have to be deserialized over and over again. - // TODO: make this persist across requests but not in a DB Tree (in globals?) - // TODO: This could potentially also be some sort of trie (suffix tree) like - // structure so that once an auth event is known it would know (using indexes - // maybe) all of the auth events that it references. - // let mut auth_cache = EventMap::new(); - let txn_start_time = Instant::now(); + trace!( + pdus = ?body.pdus.len(), + edus = ?body.edus.len(), + elapsed = ?txn_start_time.elapsed(), + id = ?body.transaction_id, + origin =?body.origin, + "Starting txn", + ); + + let resolved_map = handle_pdus(&client, &body, origin, &txn_start_time).await?; + handle_edus(&client, &body, origin).await?; + + debug!( + pdus = ?body.pdus.len(), + edus = ?body.edus.len(), + elapsed = ?txn_start_time.elapsed(), + id = ?body.transaction_id, + origin =?body.origin, + "Finished txn", + ); + + Ok(send_transaction_message::v1::Response { + pdus: resolved_map + .into_iter() + .map(|(e, r)| (e, r.map_err(|e| e.sanitized_error()))) + .collect(), + }) +} + +async fn handle_pdus( + _client: &IpAddr, body: &Ruma, origin: &ServerName, txn_start_time: &Instant, +) -> Result { let mut parsed_pdus = Vec::with_capacity(body.pdus.len()); for pdu in &body.pdus { parsed_pdus.push(match parse_incoming_pdu(pdu) { @@ -76,15 +106,6 @@ pub(crate) async fn send_transaction_message_route( // and hashes checks } - trace!( - pdus = ?parsed_pdus.len(), - edus = ?body.edus.len(), - elapsed = ?txn_start_time.elapsed(), - id = ?body.transaction_id, - origin =?body.origin, - "Starting txn", - ); - // We go through all the signatures we see on the PDUs and fetch the // corresponding signing keys let pub_key_map = RwLock::new(BTreeMap::new()); @@ -94,9 +115,7 @@ pub(crate) async fn send_transaction_message_route( .event_handler .fetch_required_signing_keys(parsed_pdus.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) .await - .unwrap_or_else(|e| { - warn!("Could not fetch all signatures for PDUs from {origin}: {:?}", e); - }); + .unwrap_or_else(|e| warn!("Could not fetch all signatures for PDUs from {origin}: {e:?}")); debug!( elapsed = ?txn_start_time.elapsed(), @@ -133,244 +152,299 @@ pub(crate) async fn send_transaction_message_route( for pdu in &resolved_map { if let Err(e) = pdu.1 { if matches!(e, Error::BadRequest(ErrorKind::NotFound, _)) { - warn!("Incoming PDU failed {:?}", pdu); + warn!("Incoming PDU failed {pdu:?}"); } } } + Ok(resolved_map) +} + +async fn handle_edus( + client: &IpAddr, body: &Ruma, origin: &ServerName, +) -> Result<()> { for edu in body .edus .iter() .filter_map(|edu| serde_json::from_str::(edu.json().get()).ok()) { match edu { - Edu::Presence(presence) => { - if !services().globals.allow_incoming_presence() { - continue; - } - - for update in presence.push { - if update.user_id.server_name() != origin { - debug_warn!(%update.user_id, %origin, "received presence EDU for user not belonging to origin"); - continue; - } - - services().presence.set_presence( - &update.user_id, - &update.presence, - Some(update.currently_active), - Some(update.last_active_ago), - update.status_msg.clone(), - )?; - } - }, - Edu::Receipt(receipt) => { - if !services().globals.allow_incoming_read_receipts() { - continue; - } - - for (room_id, room_updates) in receipt.receipts { - if services() - .rooms - .event_handler - .acl_check(origin, &room_id) - .is_err() - { - debug_warn!(%origin, %room_id, "received read receipt EDU from ACL'd server"); - continue; - } - - for (user_id, user_updates) in room_updates.read { - if user_id.server_name() != origin { - debug_warn!(%user_id, %origin, "received read receipt EDU for user not belonging to origin"); - continue; - } - - if services() - .rooms - .state_cache - .room_members(&room_id) - .filter_map(Result::ok) - .any(|member| member.server_name() == user_id.server_name()) - { - for event_id in &user_updates.event_ids { - let user_receipts = BTreeMap::from([(user_id.clone(), user_updates.data.clone())]); - - let receipts = BTreeMap::from([(ReceiptType::Read, user_receipts)]); - - let receipt_content = BTreeMap::from([(event_id.to_owned(), receipts)]); - - let event = ReceiptEvent { - content: ReceiptEventContent(receipt_content), - room_id: room_id.clone(), - }; - - services() - .rooms - .read_receipt - .readreceipt_update(&user_id, &room_id, &event)?; - } - } else { - debug_warn!(%user_id, %room_id, %origin, "received read receipt EDU from server who does not have a single member from their server in the room"); - continue; - } - } - } - }, - Edu::Typing(typing) => { - if !services().globals.config.allow_incoming_typing { - continue; - } - - if typing.user_id.server_name() != origin { - debug_warn!(%typing.user_id, %origin, "received typing EDU for user not belonging to origin"); - continue; - } - - if services() - .rooms - .event_handler - .acl_check(typing.user_id.server_name(), &typing.room_id) - .is_err() - { - debug_warn!(%typing.user_id, %typing.room_id, %origin, "received typing EDU for ACL'd user's server"); - continue; - } - - if services() - .rooms - .state_cache - .is_joined(&typing.user_id, &typing.room_id)? - { - if typing.typing { - let timeout = utils::millis_since_unix_epoch().saturating_add( - services() - .globals - .config - .typing_federation_timeout_s - .saturating_mul(1000), - ); - services() - .rooms - .typing - .typing_add(&typing.user_id, &typing.room_id, timeout) - .await?; - } else { - services() - .rooms - .typing - .typing_remove(&typing.user_id, &typing.room_id) - .await?; - } - } else { - debug_warn!(%typing.user_id, %typing.room_id, %origin, "received typing EDU for user not in room"); - continue; - } - }, - Edu::DeviceListUpdate(DeviceListUpdateContent { - user_id, - .. - }) => { - if user_id.server_name() != origin { - debug_warn!(%user_id, %origin, "received device list update EDU for user not belonging to origin"); - continue; - } - - services().users.mark_device_key_update(&user_id)?; - }, - Edu::DirectToDevice(DirectDeviceContent { - sender, - ev_type, - message_id, - messages, - }) => { - if sender.server_name() != origin { - debug_warn!(%sender, %origin, "received direct to device EDU for user not belonging to origin"); - continue; - } - - // Check if this is a new transaction id - if services() - .transaction_ids - .existing_txnid(&sender, None, &message_id)? - .is_some() - { - continue; - } - - for (target_user_id, map) in &messages { - for (target_device_id_maybe, event) in map { - match target_device_id_maybe { - DeviceIdOrAllDevices::DeviceId(target_device_id) => { - services().users.add_to_device_event( - &sender, - target_user_id, - target_device_id, - &ev_type.to_string(), - event.deserialize_as().map_err(|e| { - error!("To-Device event is invalid: {event:?} {e}"); - Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") - })?, - )?; - }, - - DeviceIdOrAllDevices::AllDevices => { - for target_device_id in services().users.all_device_ids(target_user_id) { - services().users.add_to_device_event( - &sender, - target_user_id, - &target_device_id?, - &ev_type.to_string(), - event.deserialize_as().map_err(|_| { - Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") - })?, - )?; - } - }, - } - } - } - - // Save transaction id with empty data - services() - .transaction_ids - .add_txnid(&sender, None, &message_id, &[])?; - }, - Edu::SigningKeyUpdate(SigningKeyUpdateContent { - user_id, - master_key, - self_signing_key, - }) => { - if user_id.server_name() != origin { - debug_warn!(%user_id, %origin, "received signing key update EDU from server that does not belong to user's server"); - continue; - } - - if let Some(master_key) = master_key { - services() - .users - .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true)?; - } - }, + Edu::Presence(presence) => handle_edu_presence(client, origin, presence).await?, + Edu::Receipt(receipt) => handle_edu_receipt(client, origin, receipt).await?, + Edu::Typing(typing) => handle_edu_typing(client, origin, typing).await?, + Edu::DeviceListUpdate(content) => handle_edu_device_list_update(client, origin, content).await?, + Edu::DirectToDevice(content) => handle_edu_direct_to_device(client, origin, content).await?, + Edu::SigningKeyUpdate(content) => handle_edu_signing_key_update(client, origin, content).await?, Edu::_Custom(ref _custom) => { - debug_warn!(?edu, "received custom/unknown EDU"); + debug_warn!(?body.edus, "received custom/unknown EDU"); }, } } - debug!( - pdus = ?body.pdus.len(), - edus = ?body.edus.len(), - elapsed = ?txn_start_time.elapsed(), - id = ?body.transaction_id, - origin =?body.origin, - "Finished txn", - ); - - Ok(send_transaction_message::v1::Response { - pdus: resolved_map - .into_iter() - .map(|(e, r)| (e, r.map_err(|e| e.sanitized_error()))) - .collect(), - }) + Ok(()) +} + +async fn handle_edu_presence(_client: &IpAddr, origin: &ServerName, presence: PresenceContent) -> Result<()> { + if !services().globals.allow_incoming_presence() { + return Ok(()); + } + + for update in presence.push { + if update.user_id.server_name() != origin { + debug_warn!( + %update.user_id, %origin, + "received presence EDU for user not belonging to origin" + ); + continue; + } + + services().presence.set_presence( + &update.user_id, + &update.presence, + Some(update.currently_active), + Some(update.last_active_ago), + update.status_msg.clone(), + )?; + } + + Ok(()) +} + +async fn handle_edu_receipt(_client: &IpAddr, origin: &ServerName, receipt: ReceiptContent) -> Result<()> { + if !services().globals.allow_incoming_read_receipts() { + return Ok(()); + } + + for (room_id, room_updates) in receipt.receipts { + if services() + .rooms + .event_handler + .acl_check(origin, &room_id) + .is_err() + { + debug_warn!( + %origin, %room_id, + "received read receipt EDU from ACL'd server" + ); + continue; + } + + for (user_id, user_updates) in room_updates.read { + if user_id.server_name() != origin { + debug_warn!( + %user_id, %origin, + "received read receipt EDU for user not belonging to origin" + ); + continue; + } + + if services() + .rooms + .state_cache + .room_members(&room_id) + .filter_map(Result::ok) + .any(|member| member.server_name() == user_id.server_name()) + { + for event_id in &user_updates.event_ids { + let user_receipts = BTreeMap::from([(user_id.clone(), user_updates.data.clone())]); + let receipts = BTreeMap::from([(ReceiptType::Read, user_receipts)]); + let receipt_content = BTreeMap::from([(event_id.to_owned(), receipts)]); + let event = ReceiptEvent { + content: ReceiptEventContent(receipt_content), + room_id: room_id.clone(), + }; + + services() + .rooms + .read_receipt + .readreceipt_update(&user_id, &room_id, &event)?; + } + } else { + debug_warn!( + %user_id, %room_id, %origin, + "received read receipt EDU from server who does not have a member in the room", + ); + continue; + } + } + } + + Ok(()) +} + +async fn handle_edu_typing(_client: &IpAddr, origin: &ServerName, typing: TypingContent) -> Result<()> { + if !services().globals.config.allow_incoming_typing { + return Ok(()); + } + + if typing.user_id.server_name() != origin { + debug_warn!( + %typing.user_id, %origin, + "received typing EDU for user not belonging to origin" + ); + return Ok(()); + } + + if services() + .rooms + .event_handler + .acl_check(typing.user_id.server_name(), &typing.room_id) + .is_err() + { + debug_warn!( + %typing.user_id, %typing.room_id, %origin, + "received typing EDU for ACL'd user's server" + ); + return Ok(()); + } + + if services() + .rooms + .state_cache + .is_joined(&typing.user_id, &typing.room_id)? + { + if typing.typing { + let timeout = utils::millis_since_unix_epoch().saturating_add( + services() + .globals + .config + .typing_federation_timeout_s + .saturating_mul(1000), + ); + services() + .rooms + .typing + .typing_add(&typing.user_id, &typing.room_id, timeout) + .await?; + } else { + services() + .rooms + .typing + .typing_remove(&typing.user_id, &typing.room_id) + .await?; + } + } else { + debug_warn!( + %typing.user_id, %typing.room_id, %origin, + "received typing EDU for user not in room" + ); + return Ok(()); + } + + Ok(()) +} + +async fn handle_edu_device_list_update( + _client: &IpAddr, origin: &ServerName, content: DeviceListUpdateContent, +) -> Result<()> { + let DeviceListUpdateContent { + user_id, + .. + } = content; + + if user_id.server_name() != origin { + debug_warn!( + %user_id, %origin, + "received device list update EDU for user not belonging to origin" + ); + return Ok(()); + } + + services().users.mark_device_key_update(&user_id)?; + + Ok(()) +} + +async fn handle_edu_direct_to_device( + _client: &IpAddr, origin: &ServerName, content: DirectDeviceContent, +) -> Result<()> { + let DirectDeviceContent { + sender, + ev_type, + message_id, + messages, + } = content; + + if sender.server_name() != origin { + debug_warn!( + %sender, %origin, + "received direct to device EDU for user not belonging to origin" + ); + return Ok(()); + } + + // Check if this is a new transaction id + if services() + .transaction_ids + .existing_txnid(&sender, None, &message_id)? + .is_some() + { + return Ok(()); + } + + for (target_user_id, map) in &messages { + for (target_device_id_maybe, event) in map { + match target_device_id_maybe { + DeviceIdOrAllDevices::DeviceId(target_device_id) => { + services().users.add_to_device_event( + &sender, + target_user_id, + target_device_id, + &ev_type.to_string(), + event.deserialize_as().map_err(|e| { + error!("To-Device event is invalid: {event:?} {e}"); + Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") + })?, + )?; + }, + + DeviceIdOrAllDevices::AllDevices => { + for target_device_id in services().users.all_device_ids(target_user_id) { + services().users.add_to_device_event( + &sender, + target_user_id, + &target_device_id?, + &ev_type.to_string(), + event + .deserialize_as() + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid"))?, + )?; + } + }, + } + } + } + + // Save transaction id with empty data + services() + .transaction_ids + .add_txnid(&sender, None, &message_id, &[])?; + + Ok(()) +} + +async fn handle_edu_signing_key_update( + _client: &IpAddr, origin: &ServerName, content: SigningKeyUpdateContent, +) -> Result<()> { + let SigningKeyUpdateContent { + user_id, + master_key, + self_signing_key, + } = content; + + if user_id.server_name() != origin { + debug_warn!( + %user_id, %origin, + "received signing key update EDU from server that does not belong to user's server" + ); + return Ok(()); + } + + if let Some(master_key) = master_key { + services() + .users + .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true)?; + } + + Ok(()) } From 52d470058a870e26754071ce58dc989f65ca8bf3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 00:44:00 +0000 Subject: [PATCH 0343/2091] split hash utils into directory Signed-off-by: Jason Volk --- src/core/utils/hash.rs | 67 ++++++---------------------------- src/core/utils/hash/argon.rs | 68 +++++++++++++++++++++++++++++++++++ src/core/utils/hash/sha256.rs | 9 +++++ src/core/utils/mod.rs | 9 +---- 4 files changed, 88 insertions(+), 65 deletions(-) create mode 100644 src/core/utils/hash/argon.rs create mode 100644 src/core/utils/hash/sha256.rs diff --git a/src/core/utils/hash.rs b/src/core/utils/hash.rs index d6efaff9..d1219bfe 100644 --- a/src/core/utils/hash.rs +++ b/src/core/utils/hash.rs @@ -1,62 +1,15 @@ -use std::sync::OnceLock; +mod argon; +mod sha256; -use argon2::{ - password_hash, password_hash::SaltString, Algorithm, Argon2, Params, PasswordHash, PasswordHasher, - PasswordVerifier, Version, -}; +use crate::Result; -const M_COST: u32 = Params::DEFAULT_M_COST; // memory size in 1 KiB blocks -const T_COST: u32 = Params::DEFAULT_T_COST; // nr of iterations -const P_COST: u32 = Params::DEFAULT_P_COST; // parallelism +#[inline] +pub fn password(password: &str) -> Result { argon::password(password) } -static ARGON: OnceLock> = OnceLock::new(); - -pub fn password(password: &str) -> Result { - let salt = SaltString::generate(rand::thread_rng()); - ARGON - .get_or_init(init_argon) - .hash_password(password.as_bytes(), &salt) - .map(|it| it.to_string()) +#[inline] +pub fn verify_password(password: &str, password_hash: &str) -> Result<()> { + argon::verify_password(password, password_hash) } -pub fn verify_password(password: &str, password_hash: &str) -> Result<(), password_hash::Error> { - let password_hash = PasswordHash::new(password_hash)?; - ARGON - .get_or_init(init_argon) - .verify_password(password.as_bytes(), &password_hash) -} - -fn init_argon() -> Argon2<'static> { - // 19456 Kib blocks, iterations = 2, parallelism = 1 - // * - debug_assert!(M_COST == 19_456, "M_COST default changed"); - debug_assert!(T_COST == 2, "T_COST default changed"); - debug_assert!(P_COST == 1, "P_COST default changed"); - - let algorithm = Algorithm::Argon2id; - let version = Version::default(); - let out_len: Option = None; - let params = Params::new(M_COST, T_COST, P_COST, out_len).expect("valid parameters"); - Argon2::new(algorithm, version, params) -} - -#[cfg(test)] -mod tests { - #[test] - fn password_hash_and_verify() { - use crate::utils::hash; - let preimage = "temp123"; - let digest = hash::password(preimage).expect("digest"); - hash::verify_password(preimage, &digest).expect("verified"); - } - - #[test] - #[should_panic(expected = "unverified")] - fn password_hash_and_verify_fail() { - use crate::utils::hash; - let preimage = "temp123"; - let fakeimage = "temp321"; - let digest = hash::password(preimage).expect("digest"); - hash::verify_password(fakeimage, &digest).expect("unverified"); - } -} +#[inline] +pub fn calculate_hash(keys: &[&[u8]]) -> Vec { sha256::hash(keys) } diff --git a/src/core/utils/hash/argon.rs b/src/core/utils/hash/argon.rs new file mode 100644 index 00000000..98cef00e --- /dev/null +++ b/src/core/utils/hash/argon.rs @@ -0,0 +1,68 @@ +use std::sync::OnceLock; + +use argon2::{ + password_hash, password_hash::SaltString, Algorithm, Argon2, Params, PasswordHash, PasswordHasher, + PasswordVerifier, Version, +}; + +use crate::{Error, Result}; + +const M_COST: u32 = Params::DEFAULT_M_COST; // memory size in 1 KiB blocks +const T_COST: u32 = Params::DEFAULT_T_COST; // nr of iterations +const P_COST: u32 = Params::DEFAULT_P_COST; // parallelism + +static ARGON: OnceLock> = OnceLock::new(); + +fn init_argon() -> Argon2<'static> { + // 19456 Kib blocks, iterations = 2, parallelism = 1 + // * + debug_assert!(M_COST == 19_456, "M_COST default changed"); + debug_assert!(T_COST == 2, "T_COST default changed"); + debug_assert!(P_COST == 1, "P_COST default changed"); + + let algorithm = Algorithm::Argon2id; + let version = Version::default(); + let out_len: Option = None; + let params = Params::new(M_COST, T_COST, P_COST, out_len).expect("valid parameters"); + Argon2::new(algorithm, version, params) +} + +pub(super) fn password(password: &str) -> Result { + let salt = SaltString::generate(rand::thread_rng()); + ARGON + .get_or_init(init_argon) + .hash_password(password.as_bytes(), &salt) + .map(|it| it.to_string()) + .map_err(map_err) +} + +pub(super) fn verify_password(password: &str, password_hash: &str) -> Result<()> { + let password_hash = PasswordHash::new(password_hash).map_err(map_err)?; + ARGON + .get_or_init(init_argon) + .verify_password(password.as_bytes(), &password_hash) + .map_err(map_err) +} + +fn map_err(e: password_hash::Error) -> Error { Error::Err(e.to_string()) } + +#[cfg(test)] +mod tests { + #[test] + fn password_hash_and_verify() { + use crate::utils::hash; + let preimage = "temp123"; + let digest = hash::password(preimage).expect("digest"); + hash::verify_password(preimage, &digest).expect("verified"); + } + + #[test] + #[should_panic(expected = "unverified")] + fn password_hash_and_verify_fail() { + use crate::utils::hash; + let preimage = "temp123"; + let fakeimage = "temp321"; + let digest = hash::password(preimage).expect("digest"); + hash::verify_password(fakeimage, &digest).expect("unverified"); + } +} diff --git a/src/core/utils/hash/sha256.rs b/src/core/utils/hash/sha256.rs new file mode 100644 index 00000000..6a1f1879 --- /dev/null +++ b/src/core/utils/hash/sha256.rs @@ -0,0 +1,9 @@ +use ring::{digest, digest::SHA256}; + +#[tracing::instrument(skip_all)] +pub(super) fn hash(keys: &[&[u8]]) -> Vec { + // We only hash the pdu's event ids, not the whole pdu + let bytes = keys.join(&0xFF); + let hash = digest::digest(&SHA256, &bytes); + hash.as_ref().to_owned() +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index bbaaf2ec..721ed977 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -14,11 +14,11 @@ use std::{ }; pub use debug::slice_truncated as debug_slice_truncated; +pub use hash::calculate_hash; pub use html::Escape as HtmlEscape; pub use json::{deserialize_from_str, to_canonical_object}; pub use mutex_map::MutexMap; use rand::prelude::*; -use ring::digest; use ruma::UserId; pub use sys::available_parallelism; @@ -84,13 +84,6 @@ pub fn random_string(length: usize) -> String { .collect() } -#[tracing::instrument(skip(keys))] -pub fn calculate_hash(keys: &[&[u8]]) -> Vec { - // We only hash the pdu's event ids, not the whole pdu - let bytes = keys.join(&0xFF); - let hash = digest::digest(&digest::SHA256, &bytes); - hash.as_ref().to_owned() -} #[allow(clippy::impl_trait_in_params)] pub fn common_elements( From af81baae441047d65c544b7118cf5a197b54ad70 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 00:47:58 +0000 Subject: [PATCH 0344/2091] split string utils into unit Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 26 ++------------------------ src/core/utils/string.rs | 22 ++++++++++++++++++++++ src/service/presence/data.rs | 10 +++++++++- 3 files changed, 33 insertions(+), 25 deletions(-) create mode 100644 src/core/utils/string.rs diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 721ed977..1ff36b02 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -5,6 +5,7 @@ pub mod hash; pub mod html; pub mod json; pub mod mutex_map; +pub mod string; pub mod sys; mod tests; @@ -18,8 +19,7 @@ pub use hash::calculate_hash; pub use html::Escape as HtmlEscape; pub use json::{deserialize_from_str, to_canonical_object}; pub use mutex_map::MutexMap; -use rand::prelude::*; -use ruma::UserId; +pub use string::{random_string, str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; use crate::Result; @@ -54,34 +54,12 @@ pub fn generate_keypair() -> Vec { value } -/// Parses a `UserId` from bytes. -pub fn user_id_from_bytes(bytes: &[u8]) -> Result<&UserId> { - let str: &str = str_from_bytes(bytes)?; - let user_id: &UserId = str.try_into()?; - Ok(user_id) -} - -/// Parses the bytes into a string. -pub fn string_from_bytes(bytes: &[u8]) -> Result { - let str: &str = str_from_bytes(bytes)?; - Ok(str.to_owned()) -} - -/// Parses the bytes into a string. -pub fn str_from_bytes(bytes: &[u8]) -> Result<&str> { Ok(std::str::from_utf8(bytes)?) } - /// Parses the bytes into an u64. pub fn u64_from_bytes(bytes: &[u8]) -> Result { let array: [u8; 8] = bytes.try_into()?; Ok(u64::from_be_bytes(array)) } -pub fn random_string(length: usize) -> String { - thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(length) - .map(char::from) - .collect() } diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs new file mode 100644 index 00000000..7f6f6531 --- /dev/null +++ b/src/core/utils/string.rs @@ -0,0 +1,22 @@ +use rand::prelude::*; + +use crate::Result; + +pub fn random_string(length: usize) -> String { + thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(length) + .map(char::from) + .collect() +} + +/// Parses the bytes into a string. +#[inline] +pub fn string_from_bytes(bytes: &[u8]) -> Result { + let str: &str = str_from_bytes(bytes)?; + Ok(str.to_owned()) +} + +/// Parses the bytes into a string. +#[inline] +pub fn str_from_bytes(bytes: &[u8]) -> Result<&str> { Ok(std::str::from_utf8(bytes)?) } diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 42b827d3..dac8becf 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -130,8 +130,16 @@ fn presenceid_key(count: u64, user_id: &UserId) -> Vec { #[inline] fn presenceid_parse(key: &[u8]) -> Result<(u64, &UserId)> { let (count, user_id) = key.split_at(8); - let user_id = utils::user_id_from_bytes(user_id)?; + let user_id = user_id_from_bytes(user_id)?; let count = utils::u64_from_bytes(count).unwrap(); Ok((count, user_id)) } + +/// Parses a `UserId` from bytes. +fn user_id_from_bytes(bytes: &[u8]) -> Result<&UserId> { + let str: &str = utils::str_from_bytes(bytes)?; + let user_id: &UserId = str.try_into()?; + + Ok(user_id) +} From 3b9fba233cf68fbc8e9c177ed679790e09456dd4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 00:54:36 +0000 Subject: [PATCH 0345/2091] split bytes utils into unit Signed-off-by: Jason Volk --- src/core/utils/bytes.rs | 29 ++++++++++++++++++ src/core/utils/hash.rs | 1 + src/core/utils/mod.rs | 44 +++++++++------------------- src/service/rooms/auth_chain/data.rs | 2 +- 4 files changed, 45 insertions(+), 31 deletions(-) create mode 100644 src/core/utils/bytes.rs diff --git a/src/core/utils/bytes.rs b/src/core/utils/bytes.rs new file mode 100644 index 00000000..6ebf4c4f --- /dev/null +++ b/src/core/utils/bytes.rs @@ -0,0 +1,29 @@ +use crate::Result; + +#[inline] +#[must_use] +pub fn increment(old: Option<&[u8]>) -> [u8; 8] { + old.map(TryInto::try_into) + .map_or(0_u64, |val| val.map_or(0_u64, u64::from_be_bytes)) + .wrapping_add(1) + .to_be_bytes() +} + +/// Parses the big-endian bytes into an u64. +#[inline] +pub fn u64_from_bytes(bytes: &[u8]) -> Result { + let array: [u8; 8] = bytes.try_into()?; + Ok(u64_from_u8x8(array)) +} + +/// Parses the 8 big-endian bytes into an u64. +#[inline] +#[must_use] +pub fn u64_from_u8(bytes: &[u8]) -> u64 { + let bytes: &[u8; 8] = bytes.try_into().expect("must slice at least 8 bytes"); + u64_from_u8x8(*bytes) +} + +#[inline] +#[must_use] +pub fn u64_from_u8x8(bytes: [u8; 8]) -> u64 { u64::from_be_bytes(bytes) } diff --git a/src/core/utils/hash.rs b/src/core/utils/hash.rs index d1219bfe..9469321f 100644 --- a/src/core/utils/hash.rs +++ b/src/core/utils/hash.rs @@ -12,4 +12,5 @@ pub fn verify_password(password: &str, password_hash: &str) -> Result<()> { } #[inline] +#[must_use] pub fn calculate_hash(keys: &[&[u8]]) -> Vec { sha256::hash(keys) } diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 1ff36b02..6ebab2c6 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -1,3 +1,4 @@ +pub mod bytes; pub mod content_disposition; pub mod debug; pub mod defer; @@ -14,6 +15,7 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; +pub use bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}; pub use debug::slice_truncated as debug_slice_truncated; pub use hash::calculate_hash; pub use html::Escape as HtmlEscape; @@ -26,6 +28,18 @@ use crate::Result; pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), max) } +/// Boilerplate for wraps which are typed to never error. +/// +/// * +#[must_use] +#[inline(always)] +pub fn unwrap_infallible(result: Result) -> T { + match result { + Ok(val) => val, + Err(err) => match err {}, + } +} + #[must_use] #[allow(clippy::as_conversions)] pub fn millis_since_unix_epoch() -> u64 { @@ -35,15 +49,6 @@ pub fn millis_since_unix_epoch() -> u64 { .as_millis() as u64 } -#[inline] -#[must_use] -pub fn increment(old: Option<&[u8]>) -> [u8; 8] { - old.map(TryInto::try_into) - .map_or(0_u64, |val| val.map_or(0_u64, u64::from_be_bytes)) - .wrapping_add(1) - .to_be_bytes() -} - #[must_use] pub fn generate_keypair() -> Vec { let mut value = random_string(8).as_bytes().to_vec(); @@ -54,15 +59,6 @@ pub fn generate_keypair() -> Vec { value } -/// Parses the bytes into an u64. -pub fn u64_from_bytes(bytes: &[u8]) -> Result { - let array: [u8; 8] = bytes.try_into()?; - Ok(u64::from_be_bytes(array)) -} - -} - - #[allow(clippy::impl_trait_in_params)] pub fn common_elements( mut iterators: impl Iterator>>, check_order: impl Fn(&[u8], &[u8]) -> Ordering, @@ -86,15 +82,3 @@ pub fn common_elements( }) })) } - -/// Boilerplate for wraps which are typed to never error. -/// -/// * -#[must_use] -#[inline(always)] -pub fn unwrap_infallible(result: Result) -> T { - match result { - Ok(val) => val, - Err(err) => match err {}, - } -} diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index b2e80b7b..a5771f4a 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -38,7 +38,7 @@ impl Data { .map(|chain| { chain .chunks_exact(size_of::()) - .map(|chunk| utils::u64_from_bytes(chunk).expect("byte length is correct")) + .map(utils::u64_from_u8) .collect::>() }); From 8cf55c702f2b953c78cae83cf7107d16b02746cd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 00:57:50 +0000 Subject: [PATCH 0346/2091] split time utils into unit Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 16 +++------------- src/core/utils/time.rs | 11 +++++++++++ 2 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 src/core/utils/time.rs diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 6ebab2c6..401775f7 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -9,11 +9,9 @@ pub mod mutex_map; pub mod string; pub mod sys; mod tests; +pub mod time; -use std::{ - cmp::{self, Ordering}, - time::{SystemTime, UNIX_EPOCH}, -}; +use std::cmp::{self, Ordering}; pub use bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}; pub use debug::slice_truncated as debug_slice_truncated; @@ -23,6 +21,7 @@ pub use json::{deserialize_from_str, to_canonical_object}; pub use mutex_map::MutexMap; pub use string::{random_string, str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; +pub use time::millis_since_unix_epoch; use crate::Result; @@ -40,15 +39,6 @@ pub fn unwrap_infallible(result: Result) -> T { } } -#[must_use] -#[allow(clippy::as_conversions)] -pub fn millis_since_unix_epoch() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("time is valid") - .as_millis() as u64 -} - #[must_use] pub fn generate_keypair() -> Vec { let mut value = random_string(8).as_bytes().to_vec(); diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs new file mode 100644 index 00000000..043950a8 --- /dev/null +++ b/src/core/utils/time.rs @@ -0,0 +1,11 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + +#[inline] +#[must_use] +#[allow(clippy::as_conversions)] +pub fn millis_since_unix_epoch() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time is valid") + .as_millis() as u64 +} From be2d1c722b0ede35ff852092068f402f44aaa14b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 01:08:54 +0000 Subject: [PATCH 0347/2091] encap dep:chrono in time utils Signed-off-by: Jason Volk --- Cargo.lock | 2 +- src/core/Cargo.toml | 1 + src/core/utils/time.rs | 9 +++++++++ src/database/Cargo.toml | 1 - src/database/engine.rs | 7 ++----- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 05547769..d009df4b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -658,6 +658,7 @@ dependencies = [ "argon2", "axum 0.7.5", "bytes", + "chrono", "either", "figment", "hardened_malloc-rs", @@ -693,7 +694,6 @@ dependencies = [ name = "conduit_database" version = "0.4.3" dependencies = [ - "chrono", "conduit_core", "log", "ruma", diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 76a4084e..0214762a 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -49,6 +49,7 @@ sentry_telemetry = [] argon2.workspace = true axum.workspace = true bytes.workspace = true +chrono.workspace = true either.workspace = true figment.workspace = true http-body-util.workspace = true diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 043950a8..d7cab1fb 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -9,3 +9,12 @@ pub fn millis_since_unix_epoch() -> u64 { .expect("time is valid") .as_millis() as u64 } + +#[must_use] +pub fn rfc2822_from_seconds(epoch: i64) -> String { + use chrono::{DateTime, Utc}; + + DateTime::::from_timestamp(epoch, 0) + .unwrap_or_default() + .to_rfc2822() +} diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index f7f3c773..6e95236b 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -35,7 +35,6 @@ zstd_compression = [ ] [dependencies] -chrono.workspace = true conduit-core.workspace = true log.workspace = true ruma.workspace = true diff --git a/src/database/engine.rs b/src/database/engine.rs index 7acd470f..fe16d29f 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -4,8 +4,7 @@ use std::{ sync::{atomic::AtomicU32, Arc, Mutex, RwLock}, }; -use chrono::{DateTime, Utc}; -use conduit::{debug, error, info, warn, Result, Server}; +use conduit::{debug, error, info, utils::time::rfc2822_from_seconds, warn, Result, Server}; use rocksdb::{ backup::{BackupEngine, BackupEngineOptions}, perf::get_memory_usage_stats, @@ -211,9 +210,7 @@ impl Engine { res, "#{} {}: {} bytes, {} files", info.backup_id, - DateTime::::from_timestamp(info.timestamp, 0) - .unwrap_or_default() - .to_rfc2822(), + rfc2822_from_seconds(info.timestamp), info.size, info.num_files, ) From 7658387a74fb003b92b9a47fcb73b5941cc78a59 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 01:24:16 +0000 Subject: [PATCH 0348/2091] remove resolver wrapper; use std mutex Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 2 -- src/service/globals/resolver.rs | 25 +++++++++++++++---------- src/service/sending/resolve.rs | 5 +++-- src/service/sending/send.rs | 5 +++-- src/service/services.rs | 30 ++++++++++++++++++++++++++---- 5 files changed, 47 insertions(+), 20 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 979f671f..7e1ccb5e 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -196,8 +196,6 @@ impl Service { pub fn dns_resolver(&self) -> &TokioAsyncResolver { &self.resolver.resolver } - pub fn actual_destinations(&self) -> &Arc> { &self.resolver.destinations } - pub fn jwt_decoding_key(&self) -> Option<&jsonwebtoken::DecodingKey> { self.jwt_decoding_key.as_ref() } pub fn turn_password(&self) -> &String { &self.config.turn_password } diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index 7cf88cc0..86fa6700 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -2,7 +2,7 @@ use std::{ collections::HashMap, future, iter, net::{IpAddr, SocketAddr}, - sync::{Arc, RwLock as StdRwLock}, + sync::{Arc, RwLock}, time::Duration, }; @@ -10,22 +10,21 @@ use conduit::{error, Config, Error}; use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use ruma::OwnedServerName; -use tokio::sync::RwLock; use crate::sending::FedDest; -pub(crate) type WellKnownMap = HashMap; +type WellKnownMap = HashMap; type TlsNameMap = HashMap, u16)>; pub struct Resolver { pub destinations: Arc>, // actual_destination, host - pub overrides: Arc>, + pub overrides: Arc>, pub resolver: Arc, pub hooked: Arc, } pub struct Hooked { - pub overrides: Arc>, + pub overrides: Arc>, pub resolver: Arc, } @@ -34,10 +33,10 @@ impl Resolver { pub fn new(config: &Config) -> Self { let (sys_conf, mut opts) = hickory_resolver::system_conf::read_system_conf() .map_err(|e| { - error!("Failed to set up hickory dns resolver with system config: {}", e); + error!("Failed to set up hickory dns resolver with system config: {e}"); Error::bad_config("Failed to set up hickory dns resolver with system config.") }) - .unwrap(); + .expect("DNS system config must be valid"); let mut conf = hickory_resolver::config::ResolverConfig::new(); @@ -82,7 +81,7 @@ impl Resolver { opts.authentic_data = false; let resolver = Arc::new(TokioAsyncResolver::tokio(conf, opts)); - let overrides = Arc::new(StdRwLock::new(TlsNameMap::new())); + let overrides = Arc::new(RwLock::new(TlsNameMap::new())); Self { destinations: Arc::new(RwLock::new(WellKnownMap::new())), overrides: overrides.clone(), @@ -101,7 +100,13 @@ impl Resolve for Resolver { impl Resolve for Hooked { fn resolve(&self, name: Name) -> Resolving { - let addr_port = self.overrides.read().unwrap().get(name.as_str()).cloned(); + let addr_port = self + .overrides + .read() + .expect("locked for reading") + .get(name.as_str()) + .cloned(); + if let Some((addr, port)) = addr_port { cached_to_reqwest(&addr, port) } else { @@ -118,7 +123,7 @@ fn cached_to_reqwest(override_name: &[IpAddr], port: u16) -> Resolving { let result: Box + Send> = Box::new(iter::once(saddr)); Box::pin(future::ready(Ok(result))) }) - .unwrap() + .expect("must provide at least one override name") } fn resolve_to_reqwest(resolver: Arc, name: Name) -> Resolving { diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 23080bed..294ac09f 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -47,9 +47,10 @@ pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result 6 { - self.globals.resolver.overrides.write().unwrap().clear(); - self.globals.resolver.destinations.write().await.clear(); + self.globals + .resolver + .overrides + .write() + .expect("locked for writing") + .clear(); + self.globals + .resolver + .destinations + .write() + .expect("locked for writing") + .clear(); } if amount > 7 { self.globals.resolver.resolver.clear_cache(); From 5edd391e83ac908f009e494996eb6b0591df857f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 04:46:50 +0000 Subject: [PATCH 0349/2091] support admin server restart --force Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 14 ++++++++++++-- src/admin/server/mod.rs | 9 +++++++-- src/core/utils/sys.rs | 31 +++++++++++++++++++++++++++++++ src/main/restart.rs | 14 ++++++++++++-- 4 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 28962ef2..77b20377 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,4 +1,4 @@ -use conduit::{warn, Result}; +use conduit::{warn, Error, Result}; use ruma::events::room::message::RoomMessageEventContent; use crate::services; @@ -102,7 +102,17 @@ pub(super) async fn reload(_body: Vec<&str>) -> Result } #[cfg(unix)] -pub(super) async fn restart(_body: Vec<&str>) -> Result { +pub(super) async fn restart(_body: Vec<&str>, force: bool) -> Result { + use conduit::utils::sys::current_exe_deleted; + + if !force && current_exe_deleted() { + return Err(Error::Err( + "The server cannot be restarted because the executable was tampered with. If this is expected use --force \ + to override." + .to_owned(), + )); + } + services().server.restart()?; Ok(RoomMessageEventContent::notice_plain("Restarting server...")) diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 13f6e265..b4222251 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -51,7 +51,10 @@ pub(super) enum ServerCommand { #[cfg(unix)] /// - Restart the server - Restart, + Restart { + #[arg(short, long)] + force: bool, + }, /// - Shutdown the server Shutdown, @@ -77,7 +80,9 @@ pub(super) async fn process(command: ServerCommand, body: Vec<&str>) -> Result reload(body).await?, #[cfg(unix)] - ServerCommand::Restart => restart(body).await?, + ServerCommand::Restart { + force, + } => restart(body, force).await?, ServerCommand::Shutdown => shutdown(body).await?, }) } diff --git a/src/core/utils/sys.rs b/src/core/utils/sys.rs index 825ec903..6c396921 100644 --- a/src/core/utils/sys.rs +++ b/src/core/utils/sys.rs @@ -34,3 +34,34 @@ pub fn available_parallelism() -> usize { .expect("Unable to query for available parallelism.") .get() } + +/// Return a possibly corrected std::env::current_exe() even if the path is +/// marked deleted. +/// +/// # Safety +/// This function is declared unsafe because the original result was altered for +/// security purposes, and altering it back ignores those urposes and should be +/// understood by the user. +pub unsafe fn current_exe() -> Result { + use std::path::PathBuf; + + let exe = std::env::current_exe()?; + match exe.to_str() { + None => Ok(exe), + Some(str) => Ok(str + .strip_suffix(" (deleted)") + .map(PathBuf::from) + .unwrap_or(exe)), + } +} + +/// Determine if the server's executable was removed or replaced. This is a +/// specific check; useful for successful restarts. May not be available or +/// accurate on all platforms; defaults to false. +#[must_use] +pub fn current_exe_deleted() -> bool { + std::env::current_exe().map_or(false, |exe| { + exe.to_str() + .map_or(false, |exe| exe.ends_with(" (deleted)")) + }) +} diff --git a/src/main/restart.rs b/src/main/restart.rs index 559d7446..00de9f21 100644 --- a/src/main/restart.rs +++ b/src/main/restart.rs @@ -2,10 +2,20 @@ use std::{env, os::unix::process::CommandExt, process::Command}; -use conduit::{debug, info}; +use conduit::{debug, info, utils}; pub(super) fn restart() -> ! { - let exe = env::current_exe().expect("program path must be identified and available"); + // SAFETY: We have allowed an override for the case where the current_exe() has + // been replaced or removed. By default the server will fail to restart if the + // binary has been replaced (i.e. by cargo); this is for security purposes. + // Command::exec() used to panic in that case. + // + // We can (and do) prevent that panic by checking the result of current_exe() + // prior to committing to restart, returning an error to the user without any + // unexpected shutdown. In a nutshell that is the execuse for this unsafety. + // Nevertheless, we still want a way to override the restart preventation (i.e. + // admin server restart --force). + let exe = unsafe { utils::sys::current_exe().expect("program path must be available") }; let envs = env::vars(); let args = env::args().skip(1); debug!(?exe, ?args, ?envs, "Restart"); From 1a64e42cfe03db40295400a185c5032e4a8a9044 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 2 Jul 2024 21:51:11 -0400 Subject: [PATCH 0350/2091] add /openid/request_token and /openid/userinfo routes heavily changed and improved by me Co-authored-by: mikoto Signed-off-by: strawberry --- conduwuit-example.toml | 7 ++++++ src/api/client/mod.rs | 2 ++ src/api/client/openid.rs | 41 ++++++++++++++++++++++++++++++++++ src/api/router/auth.rs | 26 +++++++++++++++++----- src/api/routes.rs | 2 ++ src/api/server/mod.rs | 2 ++ src/api/server/openid.rs | 16 +++++++++++++ src/core/config/mod.rs | 5 +++++ src/database/maps.rs | 1 + src/service/users/data.rs | 47 ++++++++++++++++++++++++++++++++++++++- src/service/users/mod.rs | 9 ++++++++ 11 files changed, 151 insertions(+), 7 deletions(-) create mode 100644 src/api/client/openid.rs create mode 100644 src/api/server/openid.rs diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 9c6b7f0c..516a75a2 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -389,6 +389,13 @@ allow_profile_lookup_federation_requests = true # setting this to false may reduce startup time. #media_statup_check = true +# OpenID token expiration/TTL in seconds +# +# These are the OpenID tokens that are primarily used for Matrix account integrations, *not* OIDC/OpenID Connect/etc +# +# Defaults to 3600 (1 hour) +#openid_token_ttl = 3600 + ### Generic database options diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index 79088a5b..cf13cf7e 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -11,6 +11,7 @@ pub(super) mod keys; pub(super) mod media; pub(super) mod membership; pub(super) mod message; +pub(super) mod openid; pub(super) mod presence; pub(super) mod profile; pub(super) mod push; @@ -48,6 +49,7 @@ pub(super) use media::*; pub(super) use membership::*; pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room, validate_and_add_event_id}; pub(super) use message::*; +pub(super) use openid::*; pub(super) use presence::*; pub(super) use profile::*; pub use profile::{update_all_rooms, update_avatar_url, update_displayname}; diff --git a/src/api/client/openid.rs b/src/api/client/openid.rs new file mode 100644 index 00000000..a1932052 --- /dev/null +++ b/src/api/client/openid.rs @@ -0,0 +1,41 @@ +use std::time::Duration; + +use conduit::utils; +use ruma::{ + api::client::{account, error::ErrorKind}, + authentication::TokenType, +}; + +use super::TOKEN_LENGTH; +use crate::{services, Error, Result, Ruma}; + +/// # `POST /_matrix/client/v3/user/{userId}/openid/request_token` +/// +/// Request an OpenID token to verify identity with third-party services. +/// +/// - The token generated is only valid for the OpenID API +pub(crate) async fn create_openid_token_route( + body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if sender_user != &body.user_id { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to request OpenID tokens on behalf of other users", + )); + } + + let access_token = utils::random_string(TOKEN_LENGTH); + + let expires_in = services() + .users + .create_openid_token(&body.user_id, &access_token)?; + + Ok(account::request_openid_token::v3::Response { + access_token, + token_type: TokenType::Bearer, + matrix_server_name: services().globals.config.server_name.clone(), + expires_in: Duration::from_secs(expires_in), + }) +} diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index f6dcc01f..da36c82c 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -76,12 +76,26 @@ pub(super) async fn auth( } match (metadata.authentication, token) { - (_, Token::Invalid) => Err(Error::BadRequest( - ErrorKind::UnknownToken { - soft_logout: false, - }, - "Unknown access token.", - )), + (_, Token::Invalid) => { + // OpenID endpoint uses a query param with the same name, drop this once query + // params for user auth are removed from the spec. This is required to make + // integration manager work. + if request.query.access_token.is_some() && request.parts.uri.path().contains("/openid/") { + Ok(Auth { + origin: None, + sender_user: None, + sender_device: None, + appservice_info: None, + }) + } else { + Err(Error::BadRequest( + ErrorKind::UnknownToken { + soft_logout: false, + }, + "Unknown access token.", + )) + } + }, (AuthScheme::AccessToken, Token::Appservice(info)) => Ok(auth_appservice(request, info)?), (AuthScheme::None | AuthScheme::AccessTokenOptional | AuthScheme::AppserviceToken, Token::Appservice(info)) => { Ok(Auth { diff --git a/src/api/routes.rs b/src/api/routes.rs index a733ebe0..7510c6b9 100644 --- a/src/api/routes.rs +++ b/src/api/routes.rs @@ -39,6 +39,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::get_room_aliases_route) .ruma_route(client::get_filter_route) .ruma_route(client::create_filter_route) + .ruma_route(client::create_openid_token_route) .ruma_route(client::set_global_account_data_route) .ruma_route(client::set_room_account_data_route) .ruma_route(client::get_global_account_data_route) @@ -212,6 +213,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(server::get_profile_information_route) .ruma_route(server::get_keys_route) .ruma_route(server::claim_keys_route) + .ruma_route(server::get_openid_userinfo_route) .ruma_route(server::get_hierarchy_route) .ruma_route(server::well_known_server) .route("/_conduwuit/local_user_count", get(client::conduwuit_local_user_count)) diff --git a/src/api/server/mod.rs b/src/api/server/mod.rs index dce9c6ff..5d46a866 100644 --- a/src/api/server/mod.rs +++ b/src/api/server/mod.rs @@ -7,6 +7,7 @@ pub(super) mod invite; pub(super) mod key; pub(super) mod make_join; pub(super) mod make_leave; +pub(super) mod openid; pub(super) mod publicrooms; pub(super) mod query; pub(super) mod send; @@ -27,6 +28,7 @@ pub(super) use invite::*; pub(super) use key::*; pub(super) use make_join::*; pub(super) use make_leave::*; +pub(super) use openid::*; pub(super) use publicrooms::*; pub(super) use query::*; pub(super) use send::*; diff --git a/src/api/server/openid.rs b/src/api/server/openid.rs new file mode 100644 index 00000000..322c7343 --- /dev/null +++ b/src/api/server/openid.rs @@ -0,0 +1,16 @@ +use ruma::api::federation::openid::get_openid_userinfo; + +use crate::{services, Result, Ruma}; + +/// # `GET /_matrix/federation/v1/openid/userinfo` +/// +/// Get information about the user that generated the OpenID token. +pub(crate) async fn get_openid_userinfo_route( + body: Ruma, +) -> Result { + Ok(get_openid_userinfo::v1::Response::new( + services() + .users + .find_from_openid_token(&body.access_token)?, + )) +} diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 0b9cb299..b808f196 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -201,6 +201,8 @@ pub struct Config { pub query_trusted_key_servers_first: bool, #[serde(default = "default_log")] pub log: String, + #[serde(default = "default_openid_token_ttl")] + pub openid_token_ttl: u64, #[serde(default)] pub turn_username: String, #[serde(default)] @@ -671,6 +673,7 @@ impl fmt::Display for Config { "Query Trusted Key Servers First", &self.query_trusted_key_servers_first.to_string(), ), + ("OpenID Token TTL", &self.openid_token_ttl.to_string()), ( "TURN username", if self.turn_username.is_empty() { @@ -984,6 +987,8 @@ pub fn default_log() -> String { fn default_notification_push_path() -> String { "/_matrix/push/v1/notify".to_owned() } +fn default_openid_token_ttl() -> u64 { 60 * 60 } + fn default_turn_ttl() -> u64 { 60 * 60 * 24 } fn default_presence_idle_timeout_s() -> u64 { 5 * 60 } diff --git a/src/database/maps.rs b/src/database/maps.rs index 94e8ba99..1e09041c 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -91,6 +91,7 @@ pub const MAPS: &[&str] = &[ "userid_presenceid", "userid_selfsigningkeyid", "userid_usersigningkeyid", + "openidtoken_expiresatuserid", "userroomid_highlightcount", "userroomid_invitestate", "userroomid_joined", diff --git a/src/service/users/data.rs b/src/service/users/data.rs index cc40aa52..14aa8006 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, mem::size_of, sync::Arc}; -use conduit::{utils, warn, Error, Result}; +use conduit::{debug_info, utils, warn, Error, Result}; use database::{Database, Map}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, @@ -28,6 +28,7 @@ pub struct Data { userid_masterkeyid: Arc, userid_selfsigningkeyid: Arc, userid_usersigningkeyid: Arc, + openidtoken_expiresatuserid: Arc, keychangeid_userid: Arc, todeviceid_events: Arc, userfilterid_filter: Arc, @@ -51,6 +52,7 @@ impl Data { userid_masterkeyid: db["userid_masterkeyid"].clone(), userid_selfsigningkeyid: db["userid_selfsigningkeyid"].clone(), userid_usersigningkeyid: db["userid_usersigningkeyid"].clone(), + openidtoken_expiresatuserid: db["openidtoken_expiresatuserid"].clone(), keychangeid_userid: db["keychangeid_userid"].clone(), todeviceid_events: db["todeviceid_events"].clone(), userfilterid_filter: db["userfilterid_filter"].clone(), @@ -920,6 +922,49 @@ impl Data { Ok(None) } } + + /// Creates an OpenID token, which can be used to prove that a user has + /// access to an account (primarily for integrations) + pub(super) fn create_openid_token(&self, user_id: &UserId, token: &str) -> Result { + let expires_in = services().globals.config.openid_token_ttl; + let expires_at = utils::millis_since_unix_epoch().saturating_add(expires_in * 1000); + + let mut value = expires_at.to_be_bytes().to_vec(); + value.extend_from_slice(user_id.as_bytes()); + + self.openidtoken_expiresatuserid + .insert(token.as_bytes(), value.as_slice())?; + + Ok(expires_in) + } + + /// Find out which user an OpenID access token belongs to. + pub(super) fn find_from_openid_token(&self, token: &str) -> Result { + let Some(value) = self.openidtoken_expiresatuserid.get(token.as_bytes())? else { + return Err(Error::BadRequest(ErrorKind::Unauthorized, "OpenID token is unrecognised")); + }; + + let (expires_at_bytes, user_bytes) = value.split_at(0_u64.to_be_bytes().len()); + + let expires_at = u64::from_be_bytes( + expires_at_bytes + .try_into() + .map_err(|_| Error::bad_database("expires_at in openid_userid is invalid u64."))?, + ); + + if expires_at < utils::millis_since_unix_epoch() { + debug_info!("OpenID token is expired, removing"); + self.openidtoken_expiresatuserid.remove(token.as_bytes())?; + + return Err(Error::BadRequest(ErrorKind::Unauthorized, "OpenID token is expired")); + } + + UserId::parse( + utils::string_from_bytes(user_bytes) + .map_err(|_| Error::bad_database("User ID in openid_userid is invalid unicode."))?, + ) + .map_err(|_| Error::bad_database("User ID in openid_userid is invalid.")) + } } /// Will only return with Some(username) if the password was not empty and the diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index e7560436..8444d538 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -477,6 +477,15 @@ impl Service { pub fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result> { self.db.get_filter(user_id, filter_id) } + + /// Creates an OpenID token, which can be used to prove that a user has + /// access to an account (primarily for integrations) + pub fn create_openid_token(&self, user_id: &UserId, token: &str) -> Result { + self.db.create_openid_token(user_id, token) + } + + /// Find out which user an OpenID access token belongs to. + pub fn find_from_openid_token(&self, token: &str) -> Result { self.db.find_from_openid_token(token) } } /// Ensure that a user only sees signatures from themselves and the target user From 74791642c126529f2cfb21bc6e26f5e61057afa2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 03:32:18 -0400 Subject: [PATCH 0351/2091] add markdownlint and run lychee in documentation CI workflow Signed-off-by: strawberry --- .editorconfig | 7 ++++++- .github/workflows/documentation.yml | 5 +++++ .markdownlintignore | 1 + CONTRIBUTING.md | 2 ++ engage.toml | 10 ++++++++++ flake.nix | 3 +++ 6 files changed, 27 insertions(+), 1 deletion(-) create mode 120000 .markdownlintignore diff --git a/.editorconfig b/.editorconfig index dba82b95..cea657e2 100644 --- a/.editorconfig +++ b/.editorconfig @@ -11,5 +11,10 @@ indent_style = space insert_final_newline = true max_line_length = 120 +[*.{md,txt}] +indent_style = space +indent_size = 4 +max_line_length = 80 + [*.nix] -indent_size = 2 \ No newline at end of file +indent_size = 2 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 9beb5064..ed0e898d 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -106,6 +106,11 @@ jobs: run: | bin/nix-build-and-cache ci + - name: Run lychee and markdownlint + run: | + direnv exec . engage just lints lychee + direnv exec . engage just lints markdownlint + - name: Build documentation (book) run: | ./bin/nix-build-and-cache just .#book diff --git a/.markdownlintignore b/.markdownlintignore new file mode 120000 index 00000000..3e4e48b0 --- /dev/null +++ b/.markdownlintignore @@ -0,0 +1 @@ +.gitignore \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d6e23912..d8bbe1fc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,6 +37,7 @@ The following binaries are used in [`engage.toml`][engage.toml]: - [`cargo-audit`][cargo-audit] - [`cargo-deb`][cargo-deb] - [`lychee`][lychee] +- [`markdownlint-cli`][markdownlint-cli] ### Matrix tests @@ -86,6 +87,7 @@ By sending a pull request or patch, you are agreeing that your changes are allow [sytest]: https://github.com/matrix-org/sytest/ [cargo-deb]: https://github.com/kornelski/cargo-deb [lychee]: https://github.com/lycheeverse/lychee +[markdownlint-cli]: https://github.com/igorshubovych/markdownlint-cli [cargo-audit]: https://github.com/RustSec/rustsec/tree/main/cargo-audit [direnv]: https://direnv.net/ [mdbook]: https://rust-lang.github.io/mdBook/ diff --git a/engage.toml b/engage.toml index 41c4a649..d63d822d 100644 --- a/engage.toml +++ b/engage.toml @@ -55,6 +55,11 @@ name = "lychee" group = "versions" script = "lychee --version" +[[task]] +name = "markdownlint" +group = "versions" +script = "markdownlint --version" + [[task]] name = "cargo-audit" group = "security" @@ -138,6 +143,11 @@ name = "lychee" group = "lints" script = "lychee --verbose --offline docs *.md --exclude development.md" +[[task]] +name = "markdownlint" +group = "lints" +script = "markdownlint docs *.md" + [[task]] name = "cargo/all" group = "tests" diff --git a/flake.nix b/flake.nix index b9249daa..499d828d 100644 --- a/flake.nix +++ b/flake.nix @@ -108,6 +108,9 @@ # Needed for finding broken markdown links lychee + # Needed for linting markdown files + markdownlint-cli + # Useful for editing the book locally mdbook ]) From daa46a0dc4ba21d4e28908b8092a55e2f44a566f Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 03:49:33 -0400 Subject: [PATCH 0352/2091] auto-fix some markdown lints Signed-off-by: strawberry --- CODE_OF_CONDUCT.md | 3 +-- CONTRIBUTING.md | 1 + README.md | 1 + docs/SUMMARY.md | 18 +++++++++--------- docs/deploying/docker.md | 4 +++- docs/deploying/generic.md | 8 ++++---- docs/deploying/nixos.md | 1 + docs/development/hot_reload.md | 3 +++ docs/differences.md | 21 +++++++++++---------- docs/maintenance.md | 3 +++ docs/turn.md | 3 +++ engage.toml | 2 +- 12 files changed, 41 insertions(+), 27 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index a8682537..e77154e7 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -61,7 +61,7 @@ representative at an online or offline event. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement over email at -strawberry@puppygock.gay or over Matrix at @strawberry:puppygock.gay. + or over Matrix at @strawberry:puppygock.gay. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the @@ -131,4 +131,3 @@ For answers to common questions about this code of conduct, see the FAQ at [Mozilla CoC]: https://github.com/mozilla/diversity [FAQ]: https://www.contributor-covenant.org/faq [translations]: https://www.contributor-covenant.org/translations - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d8bbe1fc..89fdb165 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,6 +19,7 @@ To test, format, lint, etc that CI would do, install engage, allow the `.envrc` All of the tasks are defined at the [engage.toml][engage.toml] file. You can view all of them neatly by running `engage list` If you would like to run only a specific engage task group, use `just`: + - `engage just ` - Example: `engage just lints` diff --git a/README.md b/README.md index 92648bf8..7da24dc9 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ from time to time. #### Contact If you run into any question, feel free to + - Ask us in `#conduwuit:puppygock.gay` on Matrix - [Open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new) diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 868810f6..ca0feeba 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -3,19 +3,19 @@ - [Introduction](introduction.md) - [Differences from upstream Conduit](differences.md) - [Configuration](configuration.md) - - [Examples](configuration/examples.md) + - [Examples](configuration/examples.md) - [Deploying](deploying.md) - - [Generic](deploying/generic.md) - - [NixOS](deploying/nixos.md) - - [Docker](deploying/docker.md) - - [Arch Linux](deploying/arch-linux.md) - - [Debian](deploying/debian.md) + - [Generic](deploying/generic.md) + - [NixOS](deploying/nixos.md) + - [Docker](deploying/docker.md) + - [Arch Linux](deploying/arch-linux.md) + - [Debian](deploying/debian.md) - [TURN](turn.md) - [Appservices](appservices.md) - [Maintenance](maintenance.md) - [Troubleshooting](troubleshooting.md) - [Development](development.md) - - [Contributing](contributing.md) - - [Testing](development/testing.md) - - [Hot Reloading ("Live" Development)](development/hot_reload.md) + - [Contributing](contributing.md) + - [Testing](development/testing.md) + - [Hot Reloading ("Live" Development)](development/hot_reload.md) - [conduwuit Community Code of Conduct](conduwuit_coc.md) diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index 6687e069..5cb89308 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -24,9 +24,11 @@ OCI images for conduwuit are available in the registries listed below. [shield-main]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/main Use + ```bash docker image pull ``` + to pull it to your machine. ### Run @@ -55,6 +57,7 @@ If you just want to test conduwuit for a short time, you can use the `--rm` flag If the `docker run` command is not for you or your setup, you can also use one of the provided `docker-compose` files. Depending on your proxy setup, you can use one of the following files; + - If you already have a `traefik` instance set up, use [`docker-compose.for-traefik.yml`](docker-compose.for-traefik.yml) - If you don't have a `traefik` instance set up (or any other reverse proxy), use [`docker-compose.with-traefik.yml`](docker-compose.with-traefik.yml) - For any other reverse proxy, use [`docker-compose.yml`](docker-compose.yml) @@ -100,7 +103,6 @@ either expose ports `443` and `8448` or serve two endpoints `.well-known/matrix/ With the service `well-known` we use a single `nginx` container that will serve those two files. - ## Voice communication See the [TURN](../turn.md) page. diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 0e846099..75b285d1 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -93,21 +93,21 @@ your.server.name, your.server.name:8448 { That's it! Just start and enable the service and you're set. ```bash -$ sudo systemctl enable --now caddy +sudo systemctl enable --now caddy ``` -## You're done! +## You're done Now you can start conduwuit with: ```bash -$ sudo systemctl start conduwuit +sudo systemctl start conduwuit ``` Set it to start automatically when your system boots with: ```bash -$ sudo systemctl enable conduwuit +sudo systemctl enable conduwuit ``` ## How do I know it works? diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index d6fd3929..9f247ac4 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -8,6 +8,7 @@ conduwuit can be acquired by [Lix][lix] from various places: A binary cache for conduwuit that the CI/CD publishes to is available at the following places (both are the same just different names): + ``` https://attic.kennel.juneis.dog/conduit conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= diff --git a/docs/development/hot_reload.md b/docs/development/hot_reload.md index 554d5d08..426e7c07 100644 --- a/docs/development/hot_reload.md +++ b/docs/development/hot_reload.md @@ -65,14 +65,17 @@ When a symbol is referenced between crates they become bound: **crates cannot be Proper resource management is essential for reliable reloading to occur. This is a very basic ask in RAII-idiomatic Rust and the exposure to reloading hazards is remarkably low, generally stemming from poor patterns and practices. Unfortunately static analysis doesn't enforce reload-safety programmatically (though it could one day), for now hazards can be avoided by knowing a few basic do's and dont's: 1. Understand that code is memory. Just like one is forbidden from referencing free'd memory, one must not transfer control to free'd code. Exposure to this is primarily from two things: + - Callbacks, which this project makes very little use of. - Async tasks, which are addressed below. 2. Tie all resources to a scope or object lifetime with greatest possible symmetry (locality). For our purposes this applies to code resources, which means async blocks and tokio tasks. + - **Never spawn a task without receiving and storing its JoinHandle**. - **Always wait on join handles** before leaving a scope or in another cleanup function called by an owning scope. 3. Know any minor specific quirks documented in code or here: + - Don't use `tokio::spawn`, instead use our `Handle` in `core/server.rs`, which is reachable in most of the codebase via `services()` or other state. This is due to some bugs or assumptions made in tokio, as it happens in `unsafe {}` blocks, which are mitigated by circumventing some thread-local variables. Using runtime handles is good practice in any case. The initial implementation PR is available [here][1]. diff --git a/docs/differences.md b/docs/differences.md index 37d84517..5c93deb3 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -1,10 +1,11 @@ #### **Note: This list may not up to date. There are rapidly more and more improvements, fixes, changes, etc being made that it is becoming more difficult to maintain this list. I recommend that you give conduwuit a try and see the differences for yourself. If you have any concerns, feel free to join the conduwuit Matrix room and ask any pre-usage questions.** -### list of features, bug fixes, etc that conduwuit does that Conduit does not: +### list of features, bug fixes, etc that conduwuit does that Conduit does not Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! -## Performance: +## Performance + - Concurrency support for individual homeserver key fetching for faster remote room joins and room joins that will error less frequently - Send `Cache-Control` response header with `immutable` and 1 year cache length for all media requests (download and thumbnail) to instruct clients to cache media, and reduce server load from media requests that could be otherwise cached - Add feature flags and config options to enable/build with zstd, brotli, and/or gzip HTTP body compression (response and request) @@ -26,8 +27,8 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Remove some unnecessary checks on EDU handling for incoming transactions, effectively speeding them up - Simplify, dedupe, etc huge chunks of the codebase, including some that were unnecessary overhead, binary bloats, or preventing compiler/linker optimisations +## General Fixes/Features -## General Fixes/Features: - Add legacy Element client hack fixing password changes and deactivations on legacy Element Android/iOS due to usage of an unspecced `user` field for UIAA - Raise and improve all the various request timeouts making some things like room joins and client bugs error less or none at all than they should, and make them all user configurable - Add missing `reason` field to user ban events (`/ban`) @@ -56,8 +57,8 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Explicitly define support for sliding sync at `/_matrix/client/versions` (`org.matrix.msc3575`) - Fix seeing empty status messages on user presences +## Moderation -## Moderation: - (Also see [Admin Room](#admin-room) for all the admin commands pertaining to moderation, there's a lot!) - Add support for room banning/blocking by ID using admin command - Add support for serving `support` well-known from `[well_known.support]` (MSC1929) @@ -73,8 +74,8 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Deactivating users will remove their profile picture, blurhash, display name, and leave all rooms by default just like Synapse and for additional privacy - Reject some EDUs from ACL'd users such as read receipts and typing indicators +## Privacy/Security -## Privacy/Security: - Add config option for device name federation with a privacy-friendly default (disabled) - Add config option for requiring authentication to the `/publicRooms` endpoint (room directory) with a default enabled for privacy - Add config option for federating `/publicRooms` endpoint (room directory) to other servers with a default disabled for privacy @@ -89,8 +90,8 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Media repository on handling `Content-Disposition` and `Content-Type` is fully spec compliant and secured - Send secure default HTTP headers such as a strong restrictive CSP (see MSC4149), deny iframes, disable `X-XSS-Protection`, disable interest cohort in `Permission-Policy`, etc to mitigate any potential attack surface such as from untrusted media +## Administration/Logging -## Administration/Logging: - Commandline argument to specify the path to a config file instead of relying on `CONDUIT_CONFIG` - Revamped admin room infrastructure and commands - Substantially clean up, improve, and fix logging (less noisy dead server logging, registration attempts, more useful troubleshooting logging, proper error propagation, etc) @@ -108,8 +109,8 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Log the client IP on various requests such as registrations, banned room join attempts, logins, deactivations, federation transactions, etc - Fix Conduit dropping some remote server federation response errors +## Maintenance/Stability -## Maintenance/Stability: - GitLab CI ported to GitHub Actions - Add support for the Matrix spec compliance test suite [Complement](https://github.com/matrix-org/complement/) via the Nix flake and various other fixes for it - Implement running and diff'ing Complement results in CI and error if any mismatch occurs to prevent large cases of conduwuit regressions @@ -130,8 +131,8 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Startup check if conduwuit running in a container and is listening on 127.0.0.1 (generally containers are using NAT networking and 0.0.0.0 is the intended listening address) - Add a panic catcher layer to return panic messages in HTTP responses if a panic occurs +## Admin Room -## Admin Room: - Add support for a console CLI interface that can issue admin commands and output them in your terminal - Add support for an admin-user-only commandline admin room interface that can be issued in any room with the `\\!admin` or `\!admin` prefix and returns the response as yourself in the same room - Add admin commands for uptime, server startup, server shutdown, and server restart @@ -165,11 +166,11 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Extend memory usage admin server command to support showing memory allocator stats such as jemalloc's - Add admin debug command to see memory allocator's full extended debug statistics such as jemalloc's +## Misc -## Misc: - Add guest support for accessing TURN servers via `turn_allow_guests` like Synapse - Support for creating rooms with custom room IDs like Maunium Synapse (`room_id` request body field to `/createRoom`) -- Query parameter `?format=event|content` for returning either the room state event's content (default) for the full room state event on ` /_matrix/client/v3/rooms/{roomId}/state/{eventType}[/{stateKey}]` requests (see https://github.com/matrix-org/matrix-spec/issues/1047) +- Query parameter `?format=event|content` for returning either the room state event's content (default) for the full room state event on `/_matrix/client/v3/rooms/{roomId}/state/{eventType}[/{stateKey}]` requests (see ) - Add **optional** feature flag to use SHA256 key names for media instead of base64 to overcome filesystem file name length limitations (OS error file name too long) - Send a User-Agent on all of our requests - Send `avatar_url` on invite room membership events/changes diff --git a/docs/maintenance.md b/docs/maintenance.md index b67c8ccb..3cf2cf98 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -5,6 +5,7 @@ conduwuit has moderation through admin room commands. "binary commands" (medium priority) and an admin API (low priority) is planned. Some moderation-related config options are available in the example config such as "global ACLs" and blocking media requests to certain servers. See the example config for the moderation config options under the "Moderation / Privacy / Security" section. conduwuit has moderation admin commands for: + - managing room aliases (`!admin rooms alias`) - managing room directory (`!admin rooms directory`) - managing room banning/blocking and user removal (`!admin rooms moderation`) @@ -37,6 +38,7 @@ RocksDB troubleshooting can be found [in the RocksDB section of troubleshooting] Currently only RocksDB supports online backups. If you'd like to backup your database online without any downtime, see the `!admin server` command for the backup commands and the `database_backup_path` config options in the example config. Please note that the format of the database backup is not the exact same. This is unfortunately a bad design choice by Facebook as we are using the database backup engine API from RocksDB, however the data is still there and can still be joined together. To restore a backup from an online RocksDB backup: + - shutdown conduwuit - create a new directory for merging together the data - in the online backup created, copy all `.sst` files in `$DATABASE_BACKUP_PATH/shared_checksum` to your new directory @@ -52,6 +54,7 @@ Backing up media is also just copying the `media/` directory from your database ## Media Media still needs various work, however conduwuit implements media deletion via: + - MXC URI or Event ID (unencrypted and attempts to find the MXC URI in the event) - Delete list of MXC URIs - Delete remote media in the past `N` seconds/minutes via filesystem metadata on the file created time (`btime`) or file modified time (`mtime`) diff --git a/docs/turn.md b/docs/turn.md index 68c15495..29ddb79c 100644 --- a/docs/turn.md +++ b/docs/turn.md @@ -11,12 +11,15 @@ use-auth-secret static-auth-secret= realm= ``` + A common way to generate a suitable alphanumeric secret key is by using `pwgen -s 64 1`. These same values need to be set in conduwuit. See the [example config](configuration/examples.md) in the TURN section for configuring these and restart conduwuit after. ### Run + Run the [Coturn](https://hub.docker.com/r/coturn/coturn) image using + ```bash docker run -d --network=host -v $(pwd)/coturn.conf:/etc/coturn/turnserver.conf coturn/coturn ``` diff --git a/engage.toml b/engage.toml index d63d822d..365c0179 100644 --- a/engage.toml +++ b/engage.toml @@ -146,7 +146,7 @@ script = "lychee --verbose --offline docs *.md --exclude development.md" [[task]] name = "markdownlint" group = "lints" -script = "markdownlint docs *.md" +script = "markdownlint docs *.md || true" # TODO: fix the ton of markdown lints so we can drop `|| true` [[task]] name = "cargo/all" From deca2c52c65e84b40475018c77c61949614c25eb Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 03:53:50 -0400 Subject: [PATCH 0353/2091] switch to ruma's X-Matrix parser Signed-off-by: strawberry --- Cargo.lock | 23 +++++++++++++++ Cargo.toml | 1 + src/api/router/auth.rs | 18 +++++++++--- src/api/router/mod.rs | 1 - src/api/router/xmatrix.rs | 61 --------------------------------------- 5 files changed, 38 insertions(+), 66 deletions(-) delete mode 100644 src/api/router/xmatrix.rs diff --git a/Cargo.lock b/Cargo.lock index d009df4b..ac079d91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1552,6 +1552,15 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-auth" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643c9bbf6a4ea8a656d6b4cd53d34f79e3f841ad5203c1a55fb7d761923bc255" +dependencies = [ + "memchr", +] + [[package]] name = "http-body" version = "0.4.6" @@ -2906,6 +2915,7 @@ dependencies = [ "ruma-federation-api", "ruma-identity-service-api", "ruma-push-gateway-api", + "ruma-server-util", "ruma-signatures", "ruma-state-res", "web-time 1.1.0", @@ -3066,6 +3076,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "ruma-server-util" +version = "0.3.0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +dependencies = [ + "headers", + "http 1.1.0", + "http-auth", + "ruma-common", + "thiserror", + "tracing", +] + [[package]] name = "ruma-signatures" version = "0.15.0" diff --git a/Cargo.toml b/Cargo.toml index 755ff46a..e63dcef5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -284,6 +284,7 @@ features = [ "markdown", "push-gateway-api-c", "state-res", + "server-util", "unstable-exhaustive-types", "ring-compat", "unstable-unspecified", diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index da36c82c..08a08e08 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -9,11 +9,12 @@ use axum_extra::{ use http::uri::PathAndQuery; use ruma::{ api::{client::error::ErrorKind, AuthScheme, Metadata}, + server_util::authorization::XMatrix, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; use tracing::warn; -use super::{request::Request, xmatrix::XMatrix}; +use super::request::Request; use crate::{service::appservice::RegistrationInfo, services, Error, Result}; enum Token { @@ -202,8 +203,17 @@ async fn auth_server(request: &mut Request, json_body: &Option, - pub(crate) key: String, // KeyName? - pub(crate) sig: String, -} - -impl Credentials for XMatrix { - const SCHEME: &'static str = "X-Matrix"; - - fn decode(value: &http::HeaderValue) -> Option { - debug_assert!( - value.as_bytes().starts_with(b"X-Matrix "), - "HeaderValue to decode should start with \"X-Matrix ..\", received = {value:?}", - ); - - let parameters = str::from_utf8(&value.as_bytes()["X-Matrix ".len()..]) - .ok()? - .trim_start(); - - let mut origin = None; - let mut destination = None; - let mut key = None; - let mut sig = None; - - for entry in parameters.split_terminator(',') { - let (name, value) = entry.split_once('=')?; - - // It's not at all clear why some fields are quoted and others not in the spec, - // let's simply accept either form for every field. - let value = value - .strip_prefix('"') - .and_then(|rest| rest.strip_suffix('"')) - .unwrap_or(value); - - // FIXME: Catch multiple fields of the same name - match name { - "origin" => origin = Some(value.try_into().ok()?), - "key" => key = Some(value.to_owned()), - "sig" => sig = Some(value.to_owned()), - "destination" => destination = Some(value.to_owned()), - _ => debug!("Unexpected field `{name}` in X-Matrix Authorization header"), - } - } - - Some(Self { - origin: origin?, - key: key?, - sig: sig?, - destination, - }) - } - - fn encode(&self) -> http::HeaderValue { todo!() } -} From 3502e6ef8725b3369b9041c4107ef9865b0e010c Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 03:56:25 -0400 Subject: [PATCH 0354/2091] bump conduwuit to 0.4.4 Signed-off-by: strawberry --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac079d91..f1b25072 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -570,7 +570,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" -version = "0.4.3" +version = "0.4.4" dependencies = [ "clap", "conduit_admin", @@ -598,7 +598,7 @@ dependencies = [ [[package]] name = "conduit_admin" -version = "0.4.3" +version = "0.4.4" dependencies = [ "clap", "conduit_api", @@ -620,7 +620,7 @@ dependencies = [ [[package]] name = "conduit_api" -version = "0.4.3" +version = "0.4.4" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -653,7 +653,7 @@ dependencies = [ [[package]] name = "conduit_core" -version = "0.4.3" +version = "0.4.4" dependencies = [ "argon2", "axum 0.7.5", @@ -692,7 +692,7 @@ dependencies = [ [[package]] name = "conduit_database" -version = "0.4.3" +version = "0.4.4" dependencies = [ "conduit_core", "log", @@ -704,7 +704,7 @@ dependencies = [ [[package]] name = "conduit_router" -version = "0.4.3" +version = "0.4.4" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -737,7 +737,7 @@ dependencies = [ [[package]] name = "conduit_service" -version = "0.4.3" +version = "0.4.4" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index e63dcef5..b8b69d71 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" rust-version = "1.77.0" -version = "0.4.3" +version = "0.4.4" [workspace.metadata.crane] name = "conduit" From c9384e9c78dea4ffe6b430b274dd87dcf9c142c9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 11:30:34 -0400 Subject: [PATCH 0355/2091] allow all complement tests again Signed-off-by: strawberry --- bin/complement | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/complement b/bin/complement index 7c591076..b579e063 100755 --- a/bin/complement +++ b/bin/complement @@ -20,8 +20,9 @@ OCI_IMAGE="complement-conduit:main" # Complement tests that are skipped due to flakiness/reliability issues (likely # Complement itself induced based on various open issues) # +#go test -tags="conduwuit_blacklist" "$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests | tee "$LOG_FILE" # According to Go docs, these are separated by forward slashes and not pipes (why) -SKIPPED_COMPLEMENT_TESTS='-skip=TestJumpToDateEndpoint.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestFederationRoomsInvite.*|TestClientSpacesSummary.*' +#SKIPPED_COMPLEMENT_TESTS='-skip=TestJumpToDateEndpoint.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*' toplevel="$(git rev-parse --show-toplevel)" @@ -37,7 +38,7 @@ set +o pipefail env \ -C "$COMPLEMENT_SRC" \ COMPLEMENT_BASE_IMAGE="$OCI_IMAGE" \ - go test -tags="conduwuit_blacklist" "$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests | tee "$LOG_FILE" + go test -tags="conduwuit_blacklist" -v -timeout 1h -json ./tests | tee "$LOG_FILE" set -o pipefail # Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results From 82acc7c0b0d30294328a4d471a58a62e1b8de780 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 12:12:11 -0400 Subject: [PATCH 0356/2091] ci: add back ubuntu free disk space Signed-off-by: strawberry --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index efb2a9c0..f01e865d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,9 @@ jobs: name: Test runs-on: ubuntu-latest steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + - name: Sync repository uses: actions/checkout@v4 @@ -119,7 +122,7 @@ jobs: - name: Cache CI dependencies run: | bin/nix-build-and-cache ci - + - uses: Swatinem/rust-cache@v2 - name: Run CI tests From b0eec60724575f56f3098bdc95ee407883947b8a Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 12:40:08 -0400 Subject: [PATCH 0357/2091] add MSC3266 room summary API support Signed-off-by: strawberry --- Cargo.lock | 32 +++---- Cargo.toml | 3 +- src/api/client/unstable.rs | 121 +++++++++++++++++++++++- src/api/routes.rs | 6 +- src/service/rooms/state_accessor/mod.rs | 33 ++++++- 5 files changed, 174 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f1b25072..df340175 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2903,7 +2903,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "assign", "js_int", @@ -2924,7 +2924,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "js_int", "ruma-common", @@ -2936,7 +2936,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "as_variant", "assign", @@ -2959,7 +2959,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "as_variant", "base64 0.22.1", @@ -2972,7 +2972,7 @@ dependencies = [ "percent-encoding", "rand", "regex", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0)", + "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", "ruma-macros", "serde", "serde_html_form", @@ -2989,7 +2989,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "as_variant", "indexmap 2.2.6", @@ -2999,7 +2999,7 @@ dependencies = [ "pulldown-cmark", "regex", "ruma-common", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0)", + "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", "ruma-macros", "serde", "serde_json", @@ -3012,7 +3012,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "js_int", "ruma-common", @@ -3024,7 +3024,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "js_int", "thiserror", @@ -3042,7 +3042,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "js_int", "ruma-common", @@ -3052,13 +3052,13 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0)", + "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", "serde", "syn 2.0.68", "toml", @@ -3067,7 +3067,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "js_int", "ruma-common", @@ -3079,7 +3079,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "headers", "http 1.1.0", @@ -3092,7 +3092,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3108,7 +3108,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9a5bfad8494b7a4e6c40421c3d0675db4b434ec0#9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index b8b69d71..a52b4138 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -274,7 +274,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "9a5bfad8494b7a4e6c40421c3d0675db4b434ec0" +rev = "c51ccb2c68d2e3557eb12b1a49036531711ec0e5" features = [ "compat", "rand", @@ -294,6 +294,7 @@ features = [ "unstable-msc2870", "unstable-msc3026", "unstable-msc3061", + "unstable-msc3266", "unstable-msc3575", "unstable-msc4121", "unstable-msc4125", diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 5fecfb71..f53460a8 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -1,5 +1,8 @@ +use axum_client_ip::InsecureClientIp; +use conduit::{warn, RumaResponse}; use ruma::{ - api::client::{error::ErrorKind, membership::mutual_rooms}, + api::client::{error::ErrorKind, membership::mutual_rooms, room::get_summary}, + events::room::member::MembershipState, OwnedRoomId, }; @@ -12,8 +15,9 @@ use crate::{services, Error, Result, Ruma}; /// TODO: Implement pagination, currently this just returns everything /// /// An implementation of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) +#[tracing::instrument(skip_all, fields(%client), name = "mutual_rooms")] pub(crate) async fn get_mutual_rooms_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -43,3 +47,116 @@ pub(crate) async fn get_mutual_rooms_route( next_batch_token: None, }) } + +/// # `GET /_matrix/client/unstable/im.nheko.summary/rooms/{roomIdOrAlias}/summary` +/// +/// Returns a short description of the state of a room. +/// +/// This is the "wrong" endpoint that some implementations/clients may use +/// according to the MSC. Request and response bodies are the same as +/// `get_room_summary`. +/// +/// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) +pub(crate) async fn get_room_summary_legacy( + InsecureClientIp(client): InsecureClientIp, body: Ruma, +) -> Result> { + get_room_summary(InsecureClientIp(client), body) + .await + .map(RumaResponse) +} + +/// # `GET /_matrix/client/unstable/im.nheko.summary/summary/{roomIdOrAlias}` +/// +/// Returns a short description of the state of a room. +/// +/// TODO: support fetching remote room info if we don't know the room +/// +/// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) +#[tracing::instrument(skip_all, fields(%client), name = "room_summary")] +pub(crate) async fn get_room_summary( + InsecureClientIp(client): InsecureClientIp, body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref(); + + let room_id = services() + .rooms + .alias + .resolve(&body.room_id_or_alias) + .await?; + + if !services().rooms.metadata.exists(&room_id)? { + return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server")); + } + + if sender_user.is_none() + && !services() + .rooms + .state_accessor + .is_world_readable(&room_id) + .unwrap_or(false) + { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Room is not world readable, authentication is required", + )); + } + + Ok(get_summary::msc3266::Response { + room_id: room_id.clone(), + canonical_alias: services() + .rooms + .state_accessor + .get_canonical_alias(&room_id)?, + avatar_url: services() + .rooms + .state_accessor + .get_avatar(&room_id)? + .into_option() + .unwrap_or_default() + .url, + guest_can_join: services().rooms.state_accessor.guest_can_join(&room_id)?, + name: services() + .rooms + .state_accessor + .get_name(&room_id) + .unwrap_or(None), + num_joined_members: services() + .rooms + .state_cache + .room_joined_count(&room_id) + .unwrap_or_default() + .unwrap_or_else(|| { + warn!("Room {room_id} has no member count"); + 0 + }) + .try_into() + .expect("user count should not be that big"), + topic: services() + .rooms + .state_accessor + .get_room_topic(&room_id) + .unwrap_or(None), + world_readable: services() + .rooms + .state_accessor + .is_world_readable(&room_id) + .unwrap_or(false), + join_rule: services().rooms.state_accessor.get_join_rule(&room_id)?.0, + room_type: services().rooms.state_accessor.get_room_type(&room_id)?, + room_version: Some(services().rooms.state.get_room_version(&room_id)?), + membership: if let Some(sender_user) = sender_user { + services() + .rooms + .state_accessor + .get_member(&room_id, sender_user)? + .map_or_else(|| Some(MembershipState::Leave), |content| Some(content.membership)) + } else { + None + }, + encryption: services() + .rooms + .state_accessor + .get_room_encryption(&room_id) + .unwrap_or_else(|_e| None), + }) +} diff --git a/src/api/routes.rs b/src/api/routes.rs index 7510c6b9..94951aec 100644 --- a/src/api/routes.rs +++ b/src/api/routes.rs @@ -172,7 +172,6 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::get_key_changes_route) .ruma_route(client::get_pushers_route) .ruma_route(client::set_pushers_route) - // .ruma_route(client::third_party_route) .ruma_route(client::upgrade_room_route) .ruma_route(client::get_threads_route) .ruma_route(client::get_relating_events_with_rel_type_and_event_type_route) @@ -180,6 +179,11 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::get_relating_events_route) .ruma_route(client::get_hierarchy_route) .ruma_route(client::get_mutual_rooms_route) + .ruma_route(client::get_room_summary) + .route( + "/_matrix/client/unstable/im.nheko.summary/rooms/:room_id_or_alias/summary", + get(client::get_room_summary_legacy) + ) .ruma_route(client::well_known_support) .ruma_route(client::well_known_client) .route("/_conduwuit/server_version", get(client::conduwuit_server_version)) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 65c43905..cdd8bec9 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -14,6 +14,8 @@ use ruma::{ room::{ avatar::RoomAvatarEventContent, canonical_alias::RoomCanonicalAliasEventContent, + create::RoomCreateEventContent, + encryption::RoomEncryptionEventContent, guest_access::{GuestAccess, RoomGuestAccessEventContent}, history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent, RoomMembership}, @@ -24,8 +26,10 @@ use ruma::{ }, StateEventType, }, + room::RoomType, space::SpaceRoomJoinRule, - EventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, + EventEncryptionAlgorithm, EventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, + UserId, }; use serde_json::value::to_raw_value; @@ -451,4 +455,31 @@ impl Service { } room_ids } + + pub fn get_room_type(&self, room_id: &RoomId) -> Result> { + Ok(self + .room_state_get(room_id, &StateEventType::RoomCreate, "")? + .map(|s| { + serde_json::from_str::(s.content.get()).map_err(|e| { + error!("Invalid room create event in database: {e}"); + Error::BadDatabase("Invalid room create event in database.") + }) + }) + .transpose()? + .and_then(|e| e.room_type)) + } + + /// Gets the room's encryption algorithm if `m.room.encryption` state event + /// is found + pub fn get_room_encryption(&self, room_id: &RoomId) -> Result> { + self.room_state_get(room_id, &StateEventType::RoomEncryption, "")? + .map_or(Ok(None), |s| { + serde_json::from_str::(s.content.get()) + .map(|content| Some(content.algorithm)) + .map_err(|e| { + error!("Invalid room encryption event in database: {e}"); + Error::BadDatabase("Invalid room encryption event in database.") + }) + }) + } } From 0832c28a40af751390540c95059a502a45ba78cc Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 12:53:51 -0400 Subject: [PATCH 0358/2091] dedupe some state accessors in various places Signed-off-by: strawberry --- src/api/client/directory.rs | 29 +++-------- src/api/client/unstable.rs | 3 +- src/service/rooms/spaces/mod.rs | 66 +++++++------------------ src/service/rooms/state_accessor/mod.rs | 6 +-- 4 files changed, 30 insertions(+), 74 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 7671b90a..512dface 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -10,11 +10,7 @@ use ruma::{ }, directory::{Filter, PublicRoomJoinRule, PublicRoomsChunk, RoomNetwork}, events::{ - room::{ - avatar::RoomAvatarEventContent, - create::RoomCreateEventContent, - join_rules::{JoinRule, RoomJoinRulesEventContent}, - }, + room::join_rules::{JoinRule, RoomJoinRulesEventContent}, StateEventType, }, uint, ServerName, UInt, @@ -256,15 +252,10 @@ pub(crate) async fn get_public_rooms_filtered_helper( avatar_url: services() .rooms .state_accessor - .room_state_get(&room_id, &StateEventType::RoomAvatar, "")? - .map(|s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomAvatarEventContent| c.url) - .map_err(|_| Error::bad_database("Invalid room avatar event in database.")) - }) - .transpose()? - // url is now an Option so we must flatten - .flatten(), + .get_avatar(&room_id)? + .into_option() + .unwrap_or_default() + .url, join_rule: services() .rooms .state_accessor @@ -287,15 +278,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( room_type: services() .rooms .state_accessor - .room_state_get(&room_id, &StateEventType::RoomCreate, "")? - .map(|s| { - serde_json::from_str::(s.content.get()).map_err(|e| { - error!("Invalid room create event in database: {}", e); - Error::BadDatabase("Invalid room create event in database.") - }) - }) - .transpose()? - .and_then(|e| e.room_type), + .get_room_type(&room_id)?, room_id, }; Ok(chunk) diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index f53460a8..77cac0fa 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -106,7 +106,8 @@ pub(crate) async fn get_room_summary( canonical_alias: services() .rooms .state_accessor - .get_canonical_alias(&room_id)?, + .get_canonical_alias(&room_id) + .unwrap_or(None), avatar_url: services() .rooms .state_accessor diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 3dfbcf3c..6a87ac98 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -19,13 +19,7 @@ use ruma::{ }, }, events::{ - room::{ - avatar::RoomAvatarEventContent, - canonical_alias::RoomCanonicalAliasEventContent, - create::RoomCreateEventContent, - join_rules::{JoinRule, RoomJoinRulesEventContent}, - topic::RoomTopicEventContent, - }, + room::join_rules::{JoinRule, RoomJoinRulesEventContent}, space::child::{HierarchySpaceChildEvent, SpaceChildEventContent}, StateEventType, }, @@ -409,19 +403,20 @@ impl Service { canonical_alias: services() .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomCanonicalAlias, "")? - .map_or(Ok(None), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomCanonicalAliasEventContent| c.alias) - .map_err(|_| Error::bad_database("Invalid canonical alias event in database.")) - })?, - name: services().rooms.state_accessor.get_name(room_id)?, + .get_canonical_alias(room_id) + .unwrap_or(None), + name: services() + .rooms + .state_accessor + .get_name(room_id) + .unwrap_or(None), num_joined_members: services() .rooms .state_cache - .room_joined_count(room_id)? + .room_joined_count(room_id) + .unwrap_or_default() .unwrap_or_else(|| { - warn!("Room {} has no member count", room_id); + warn!("Room {room_id} has no member count"); 0 }) .try_into() @@ -430,42 +425,19 @@ impl Service { topic: services() .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomTopic, "")? - .map_or(Ok(None), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomTopicEventContent| Some(c.topic)) - .map_err(|_| { - error!("Invalid room topic event in database for room {}", room_id); - Error::bad_database("Invalid room topic event in database.") - }) - })?, + .get_room_topic(room_id) + .unwrap_or(None), world_readable: services().rooms.state_accessor.is_world_readable(room_id)?, guest_can_join: services().rooms.state_accessor.guest_can_join(room_id)?, avatar_url: services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomAvatar, "")? - .map(|s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomAvatarEventContent| c.url) - .map_err(|_| Error::bad_database("Invalid room avatar event in database.")) - }) - .transpose()? - // url is now an Option so we must flatten - .flatten(), - join_rule, - room_type: services() .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "")? - .map(|s| { - serde_json::from_str::(s.content.get()).map_err(|e| { - error!("Invalid room create event in database: {}", e); - Error::BadDatabase("Invalid room create event in database.") - }) - }) - .transpose()? - .and_then(|e| e.room_type), + .get_avatar(room_id)? + .into_option() + .unwrap_or_default() + .url, + join_rule, + room_type: services().rooms.state_accessor.get_room_type(room_id)?, children_state, allowed_room_ids, }) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index cdd8bec9..03e22187 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -422,10 +422,10 @@ impl Service { ) } - /// Returns the join rule for a given room - pub fn get_join_rule(&self, current_room: &RoomId) -> Result<(SpaceRoomJoinRule, Vec), Error> { + /// Returns the join rule (`SpaceRoomJoinRule`) for a given room + pub fn get_join_rule(&self, room_id: &RoomId) -> Result<(SpaceRoomJoinRule, Vec), Error> { Ok(self - .room_state_get(current_room, &StateEventType::RoomJoinRules, "")? + .room_state_get(room_id, &StateEventType::RoomJoinRules, "")? .map(|s| { serde_json::from_str(s.content.get()) .map(|c: RoomJoinRulesEventContent| { From 6b433392a4c48436f1e271358969a5a277177828 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 13:16:27 -0400 Subject: [PATCH 0359/2091] slight docs updates Signed-off-by: strawberry --- docs/development.md | 2 +- docs/differences.md | 9 ++++++--- docs/troubleshooting.md | 4 ---- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/development.md b/docs/development.md index fc177c8f..a23fe859 100644 --- a/docs/development.md +++ b/docs/development.md @@ -16,7 +16,7 @@ look like this: RUSTFLAGS="--cfg tokio_unstable" cargo build \ --release \ --no-default-features \ - --features=rocksdb,systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console + --features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console ``` [1]: https://docs.rs/tokio-console/latest/tokio_console/ diff --git a/docs/differences.md b/docs/differences.md index 5c93deb3..7dfbedc1 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -26,6 +26,7 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Use `tracing`/`log`'s `release_max_level_info` feature to improve performance, build speeds, binary size, and CPU usage in release builds by avoid compiling debug/trace log level macros that users will generally never use (can be disabled with a build-time feature flag) - Remove some unnecessary checks on EDU handling for incoming transactions, effectively speeding them up - Simplify, dedupe, etc huge chunks of the codebase, including some that were unnecessary overhead, binary bloats, or preventing compiler/linker optimisations +- Implement zero-copy RocksDB database accessors, substantially improving performance caused by unnecessary memory allocations ## General Fixes/Features @@ -61,7 +62,7 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - (Also see [Admin Room](#admin-room) for all the admin commands pertaining to moderation, there's a lot!) - Add support for room banning/blocking by ID using admin command -- Add support for serving `support` well-known from `[well_known.support]` (MSC1929) +- Add support for serving `support` well-known from `[global.well_known]` (MSC1929) (`/.well-known/matrix/support`) - Config option to forbid publishing rooms to the room directory (`lockdown_public_room_directory`) except for admins - Admin commands to delete room aliases and unpublish rooms from our room directory - For all [`/report`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3roomsroomidreporteventid) requests: check if the reported event ID belongs to the reported room ID, raise report reasoning character limit to 750, fix broken formatting, make a small delayed random response per spec suggestion on privacy, and check if the sender user is in the reported room. @@ -120,16 +121,18 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Fixed every single clippy (default lints) and rustc warnings, including some that were performance related or potential safety issues / unsoundness - Add a **lot** of other clippy and rustc lints and a rustfmt.toml file - Repo uses [Renovate](https://docs.renovatebot.com/), [Trivy](https://github.com/aquasecurity/trivy-action), and keeps ALL dependencies as up to date as possible -- Attempts and interest in removing extreme and unnecessary panics/unwraps/expects that can lead to denial of service or such (upstream and upstream contributors want this unusual behaviour for some reason) - Purge unmaintained/irrelevant/broken database backends (heed, sled, persy) and other unnecessary code or overhead - webp support for images - Add cargo audit support to CI +- Add documentation lints via lychee and markdownlint-cli to CI - CI tests for all sorts of feature matrixes (jemalloc, non-defaullt, all features, etc) - Add static and dynamic linking smoke tests in CI to prevent any potential linking regressions for Complement, static binaries, Nix devshells, etc - Add timestamp by commit date when building OCI images for keeping image build reproducibility and still have a meaningful "last modified date" for OCI image - Add timestamp by commit date via `SOURCE_DATE_EPOCH` for Debian packages - Startup check if conduwuit running in a container and is listening on 127.0.0.1 (generally containers are using NAT networking and 0.0.0.0 is the intended listening address) - Add a panic catcher layer to return panic messages in HTTP responses if a panic occurs +- Add full compatibility support for SHA256 media file names instead of base64 file names to overcome filesystem file name length limitations (OS error file name too long) while still retaining upstream database compatibility +- Remove SQLite support due to being very poor performance, difficult to maintain against RocksDB, and is a blocker to significantly improved database code ## Admin Room @@ -171,7 +174,6 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Add guest support for accessing TURN servers via `turn_allow_guests` like Synapse - Support for creating rooms with custom room IDs like Maunium Synapse (`room_id` request body field to `/createRoom`) - Query parameter `?format=event|content` for returning either the room state event's content (default) for the full room state event on `/_matrix/client/v3/rooms/{roomId}/state/{eventType}[/{stateKey}]` requests (see ) -- Add **optional** feature flag to use SHA256 key names for media instead of base64 to overcome filesystem file name length limitations (OS error file name too long) - Send a User-Agent on all of our requests - Send `avatar_url` on invite room membership events/changes - Support sending [`well_known` response to client login responses](https://spec.matrix.org/v1.10/client-server-api/#post_matrixclientv3login) if using config option `[well_known.client]` @@ -180,6 +182,7 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - Implement legacy Matrix `/v1/` media endpoints that some clients and servers may still call - Config option to change Conduit's behaviour of homeserver key fetching (`query_trusted_key_servers_first`). This option sets whether conduwuit will query trusted notary key servers first before the individual homeserver(s), or vice versa which may help in joining certain rooms. - Implement unstable MSC2666 support for querying mutual rooms with a user +- Implement unstable MSC3266 room summary API support - Implement unstable MSC4125 support for specifying servers to join via on federated invites - Make conduwuit build and be functional under Nix + macOS - Log out all sessions after unsetting the emergency password diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 2f5f989f..81b09e81 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -40,10 +40,6 @@ With this in mind: ## Media -#### "File name too long" - -If you are running into the "file name is too long" OS error for media requests, your filesystem cannot handle file name lengths >=255 characters. This is unfortuntely due to Conduit (upstream) using base64 for file name keys which is very problematic for some filesystems as the base64 input is untrusted and long file names or specific inputs can cause this. If you would like to avoid this, you may build conduwuit yourself with the `sha256_media` feature. **This will lose database compatibility with upstream**. - ## Debugging Note that users should not really be debugging things. If you find yourself debugging and find the issue, please let us know and/or how we can fix it. Various debug commands can be found in `!admin debug`. From e2033fb6c195acdf87ed4690ba8606fd661f9343 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 13:20:37 -0400 Subject: [PATCH 0360/2091] notify users to update their build scripts if using `rocksdb` or `sha256_media` Signed-off-by: strawberry --- src/core/Cargo.toml | 4 ++++ src/core/config/check.rs | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 0214762a..2c0fc47d 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -45,6 +45,10 @@ zstd_compression =[] perf_measurements = [] sentry_telemetry = [] +# these do nothing, these are purely for informing users to update their build scripts if they use one +rocksdb = [] +sha256_media = [] + [dependencies] argon2.workspace = true axum.workspace = true diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 24a9ac9c..795f0c43 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -6,6 +6,17 @@ use tracing::{debug, error, info, warn}; use crate::{error::Error, Config}; pub fn check(config: &Config) -> Result<(), Error> { + #[cfg(feature = "rocksdb")] + warn!( + "Note the rocksdb feature was deleted from conduwuit, sqlite was deleted and RocksDB is the only supported \ + backend now. Please update your build script to remove this feature." + ); + #[cfg(feature = "sha256_media")] + warn!( + "Note the sha256_media feature was deleted from conduwuit, it is now fully integrated in a \ + forwards-compatible way. Please update your build script to remove this feature." + ); + config.warn_deprecated(); config.warn_unknown_key(); From b2dbc3bc10355d2d4c59480da3d74e7ba0af6d05 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 13:33:46 -0400 Subject: [PATCH 0361/2091] update complement results Signed-off-by: strawberry --- .../complement/test_results.jsonl | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index d1e322ab..74ad7cd2 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -42,6 +42,13 @@ {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/knock_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/non-state_membership_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/regular_event"} +{"Action":"fail","Test":"TestClientSpacesSummary"} +{"Action":"fail","Test":"TestClientSpacesSummary/max_depth"} +{"Action":"fail","Test":"TestClientSpacesSummary/pagination"} +{"Action":"fail","Test":"TestClientSpacesSummary/query_whole_graph"} +{"Action":"fail","Test":"TestClientSpacesSummary/redact_link"} +{"Action":"fail","Test":"TestClientSpacesSummary/suggested_only"} +{"Action":"fail","Test":"TestClientSpacesSummaryJoinRules"} {"Action":"fail","Test":"TestContentMediaV1"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation/good_connectivity"} @@ -57,6 +64,13 @@ {"Action":"fail","Test":"TestFederationKeyUploadQuery/Can_query_remote_device_keys_using_POST"} {"Action":"pass","Test":"TestFederationRedactSendsWithoutEvent"} {"Action":"pass","Test":"TestFederationRejectInvite"} +{"Action":"pass","Test":"TestFederationRoomsInvite"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_for_empty_room"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_several_times"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_has_'is_direct'_flag_in_prev_content_after_joining"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_see_room_metadata"} {"Action":"fail","Test":"TestGetMissingEventsGapFilling"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility"} @@ -71,12 +85,29 @@ {"Action":"pass","Test":"TestIsDirectFlagFederation"} {"Action":"pass","Test":"TestIsDirectFlagLocal"} {"Action":"pass","Test":"TestJoinFederatedRoomFailOver"} +{"Action":"fail","Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser"} +{"Action":"fail","Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser/join_remote_federated_room_as_application_service_user"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_missing_signatures_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_bad_signatures_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_state_with_unverifiable_auth_events_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_unobtainable_keys_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinViaRoomIDAndServerName"} +{"Action":"fail","Test":"TestJumpToDateEndpoint"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/can_paginate_after_getting_remote_event_from_timestamp_to_event_endpoint"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/looking_backwards,_should_be_able_to_find_event_that_was_sent_before_we_joined"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/looking_forwards,_should_be_able_to_find_event_that_was_sent_before_we_joined"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/when_looking_backwards_before_the_room_was_created,_should_be_able_to_find_event_that_was_imported"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_event_after_given_timestmap"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_event_before_given_timestmap"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_after_given_timestmap_when_all_message_timestamps_are_the_same"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_before_given_timestamp_when_all_message_timestamps_are_the_same"} +{"Action":"pass","Test":"TestJumpToDateEndpoint/parallel/should_find_nothing_after_the_latest_timestmap"} +{"Action":"pass","Test":"TestJumpToDateEndpoint/parallel/should_find_nothing_before_the_earliest_timestmap"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_private_room_you_are_not_a_member_of"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_public_room_you_are_not_a_member_of"} {"Action":"fail","Test":"TestKnockRoomsInPublicRoomsDirectory"} {"Action":"fail","Test":"TestKnockRoomsInPublicRoomsDirectoryInMSC3787Room"} {"Action":"fail","Test":"TestKnocking"} From 4014cb44131979bbf7043d272e9a5f2b9b029ff9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 3 Jul 2024 15:23:52 -0400 Subject: [PATCH 0362/2091] document emergency_password Signed-off-by: strawberry --- conduwuit-example.toml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 516a75a2..7e5c9710 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -378,6 +378,8 @@ allow_profile_lookup_federation_requests = true # Enable backward-compatibility with Conduit's media directory by creating symlinks of media. This # option is only necessary if you plan on using Conduit again. Otherwise setting this to false # reduces filesystem clutter and overhead for managing these symlinks in the directory. +# +# Enabled by default. #media_compat_file_link = true # Checks consistency of the media directory at startup: @@ -387,6 +389,8 @@ allow_profile_lookup_federation_requests = true # # If none of these checks apply to your use cases, and your media directory is significantly large # setting this to false may reduce startup time. +# +# Enabled by default. #media_statup_check = true # OpenID token expiration/TTL in seconds @@ -396,6 +400,12 @@ allow_profile_lookup_federation_requests = true # Defaults to 3600 (1 hour) #openid_token_ttl = 3600 +# Emergency password feature. This password set here will let you login to the server service account (e.g. `@conduit`) +# and let you run admin commands, invite yourself to the admin room, etc. +# +# no default. +#emergency_password = "" + ### Generic database options From d1b3faf4bd80e14aa27bea08d05c9e01d86a118b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 3 Jul 2024 18:23:31 +0000 Subject: [PATCH 0363/2091] Update Rust crate sd-notify to v0.4.2 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index df340175..f89c9f35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3330,9 +3330,9 @@ dependencies = [ [[package]] name = "sd-notify" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "621e3680f3e07db4c9c2c3fb07c6223ab2fab2e54bd3c04c3ae037990f428c32" +checksum = "4646d6f919800cd25c50edb49438a1381e2cd4833c027e75e8897981c50b8b5e" [[package]] name = "security-framework" From 9a0ffd55da7ebc530040be6f0cc2b64c70447bfa Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 4 Jul 2024 01:51:32 -0400 Subject: [PATCH 0364/2091] skip complement test `TestClientSpacesSummary` still too flakey still Signed-off-by: strawberry --- bin/complement | 5 ++--- tests/test_results/complement/test_results.jsonl | 7 ------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/bin/complement b/bin/complement index b579e063..f190fef8 100755 --- a/bin/complement +++ b/bin/complement @@ -20,9 +20,8 @@ OCI_IMAGE="complement-conduit:main" # Complement tests that are skipped due to flakiness/reliability issues (likely # Complement itself induced based on various open issues) # -#go test -tags="conduwuit_blacklist" "$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests | tee "$LOG_FILE" # According to Go docs, these are separated by forward slashes and not pipes (why) -#SKIPPED_COMPLEMENT_TESTS='-skip=TestJumpToDateEndpoint.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*' +SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*' toplevel="$(git rev-parse --show-toplevel)" @@ -38,7 +37,7 @@ set +o pipefail env \ -C "$COMPLEMENT_SRC" \ COMPLEMENT_BASE_IMAGE="$OCI_IMAGE" \ - go test -tags="conduwuit_blacklist" -v -timeout 1h -json ./tests | tee "$LOG_FILE" + go test -tags="conduwuit_blacklist" "$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests | tee "$LOG_FILE" set -o pipefail # Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 74ad7cd2..d8205bb9 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -42,13 +42,6 @@ {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/knock_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/non-state_membership_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/regular_event"} -{"Action":"fail","Test":"TestClientSpacesSummary"} -{"Action":"fail","Test":"TestClientSpacesSummary/max_depth"} -{"Action":"fail","Test":"TestClientSpacesSummary/pagination"} -{"Action":"fail","Test":"TestClientSpacesSummary/query_whole_graph"} -{"Action":"fail","Test":"TestClientSpacesSummary/redact_link"} -{"Action":"fail","Test":"TestClientSpacesSummary/suggested_only"} -{"Action":"fail","Test":"TestClientSpacesSummaryJoinRules"} {"Action":"fail","Test":"TestContentMediaV1"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation/good_connectivity"} From 229f2fde7a0dec19d923d470f1a40dccddac4275 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 4 Jul 2024 11:17:13 -0400 Subject: [PATCH 0365/2091] skip 2 more flakey complement tests still Signed-off-by: strawberry --- bin/complement | 2 +- .../test_results/complement/test_results.jsonl | 17 ----------------- 2 files changed, 1 insertion(+), 18 deletions(-) diff --git a/bin/complement b/bin/complement index f190fef8..c679035a 100755 --- a/bin/complement +++ b/bin/complement @@ -21,7 +21,7 @@ OCI_IMAGE="complement-conduit:main" # Complement itself induced based on various open issues) # # According to Go docs, these are separated by forward slashes and not pipes (why) -SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*' +SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*' toplevel="$(git rev-parse --show-toplevel)" diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index d8205bb9..2f3db95d 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -78,29 +78,12 @@ {"Action":"pass","Test":"TestIsDirectFlagFederation"} {"Action":"pass","Test":"TestIsDirectFlagLocal"} {"Action":"pass","Test":"TestJoinFederatedRoomFailOver"} -{"Action":"fail","Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser"} -{"Action":"fail","Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser/join_remote_federated_room_as_application_service_user"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_missing_signatures_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_bad_signatures_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_state_with_unverifiable_auth_events_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_unobtainable_keys_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinViaRoomIDAndServerName"} -{"Action":"fail","Test":"TestJumpToDateEndpoint"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/can_paginate_after_getting_remote_event_from_timestamp_to_event_endpoint"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/looking_backwards,_should_be_able_to_find_event_that_was_sent_before_we_joined"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/looking_forwards,_should_be_able_to_find_event_that_was_sent_before_we_joined"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/when_looking_backwards_before_the_room_was_created,_should_be_able_to_find_event_that_was_imported"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_event_after_given_timestmap"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_event_before_given_timestmap"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_after_given_timestmap_when_all_message_timestamps_are_the_same"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_before_given_timestamp_when_all_message_timestamps_are_the_same"} -{"Action":"pass","Test":"TestJumpToDateEndpoint/parallel/should_find_nothing_after_the_latest_timestmap"} -{"Action":"pass","Test":"TestJumpToDateEndpoint/parallel/should_find_nothing_before_the_earliest_timestmap"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_private_room_you_are_not_a_member_of"} -{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_public_room_you_are_not_a_member_of"} {"Action":"fail","Test":"TestKnockRoomsInPublicRoomsDirectory"} {"Action":"fail","Test":"TestKnockRoomsInPublicRoomsDirectoryInMSC3787Room"} {"Action":"fail","Test":"TestKnocking"} From 0cea64309a6ffe3617452b1c97e8ba651ac9b453 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 21:05:24 +0000 Subject: [PATCH 0366/2091] move PduEvent from services to core Signed-off-by: Jason Volk --- src/api/client/membership.rs | 9 +-- src/api/server/backfill.rs | 6 +- src/api/server/event.rs | 6 +- src/api/server/event_auth.rs | 6 +- src/api/server/get_missing_events.rs | 6 +- src/api/server/invite.rs | 12 ++-- src/api/server/send_join.rs | 11 ++-- src/api/server/state.rs | 8 ++- src/core/mod.rs | 4 +- src/core/pdu/builder.rs | 16 ++++++ src/core/{pducount.rs => pdu/count.rs} | 0 src/{service/pdu.rs => core/pdu/mod.rs} | 55 ++----------------- src/service/mod.rs | 5 +- src/service/sending/mod.rs | 1 + src/service/sending/sender.rs | 73 ++++++++++++++++++------- 15 files changed, 118 insertions(+), 100 deletions(-) create mode 100644 src/core/pdu/builder.rs rename src/core/{pducount.rs => pdu/count.rs} (100%) rename src/{service/pdu.rs => core/pdu/mod.rs} (88%) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 8529a9df..bb19ee29 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -33,6 +33,7 @@ use ruma::{ OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use service::sending::convert_to_outgoing_federation_event; use tokio::sync::RwLock; use tracing::{debug, error, info, trace, warn}; @@ -779,7 +780,7 @@ async fn join_room_by_id_helper_remote( federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), event_id: event_id.to_owned(), - pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), + pdu: convert_to_outgoing_federation_event(join_event.clone()), omit_members: false, }, ) @@ -1207,7 +1208,7 @@ async fn join_room_by_id_helper_local( federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), event_id: event_id.to_owned(), - pdu: PduEvent::convert_to_outgoing_federation_event(join_event.clone()), + pdu: convert_to_outgoing_federation_event(join_event.clone()), omit_members: false, }, ) @@ -1438,7 +1439,7 @@ pub(crate) async fn invite_helper( room_id: room_id.to_owned(), event_id: (*pdu.event_id).to_owned(), room_version: room_version_id.clone(), - event: PduEvent::convert_to_outgoing_federation_event(pdu_json.clone()), + event: convert_to_outgoing_federation_event(pdu_json.clone()), invite_room_state, via: services().rooms.state_cache.servers_route_via(room_id).ok(), }, @@ -1775,7 +1776,7 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { federation::membership::create_leave_event::v2::Request { room_id: room_id.to_owned(), event_id, - pdu: PduEvent::convert_to_outgoing_federation_event(leave_event.clone()), + pdu: convert_to_outgoing_federation_event(leave_event.clone()), }, ) .await?; diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index e3ff82e4..b432ae20 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -1,9 +1,11 @@ +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::backfill::get_backfill}, uint, user_id, MilliSecondsSinceUnixEpoch, }; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/backfill/` /// @@ -62,7 +64,7 @@ pub(crate) async fn get_backfill_route(body: Ruma) -> }) .map(|(_, pdu)| services().rooms.timeline.get_pdu_json(&pdu.event_id)) .filter_map(|r| r.ok().flatten()) - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) .collect(); Ok(get_backfill::v1::Response { diff --git a/src/api/server/event.rs b/src/api/server/event.rs index 29f538b4..f4c9d145 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -1,9 +1,11 @@ +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_event}, MilliSecondsSinceUnixEpoch, RoomId, }; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/event/{eventId}` /// @@ -48,6 +50,6 @@ pub(crate) async fn get_event_route(body: Ruma) -> Resul Ok(get_event::v1::Response { origin: services().globals.server_name().to_owned(), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - pdu: PduEvent::convert_to_outgoing_federation_event(event), + pdu: convert_to_outgoing_federation_event(event), }) } diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 1ddf2ce3..bef5116b 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -1,11 +1,13 @@ use std::sync::Arc; +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::authorization::get_event_authorization}, RoomId, }; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/event_auth/{roomId}/{eventId}` /// @@ -57,7 +59,7 @@ pub(crate) async fn get_event_authorization_route( Ok(get_event_authorization::v1::Response { auth_chain: auth_chain_ids .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok()?) - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) .collect(), }) } diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index 1c9a6a38..5ab9abf8 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -1,9 +1,11 @@ +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_missing_events}, OwnedEventId, RoomId, }; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `POST /_matrix/federation/v1/get_missing_events/{roomId}` /// @@ -79,7 +81,7 @@ pub(crate) async fn get_missing_events_route( ) .map_err(|_| Error::bad_database("Invalid prev_events in event in database."))?, ); - events.push(PduEvent::convert_to_outgoing_federation_event(pdu)); + events.push(convert_to_outgoing_federation_event(pdu)); } i = i.saturating_add(1); } diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 333ebee6..89b90058 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -1,18 +1,14 @@ use axum_client_ip::InsecureClientIp; +use conduit::{utils, warn, Error, PduEvent, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_invite}, events::room::member::{MembershipState, RoomMemberEventContent}, serde::JsonObject, CanonicalJsonValue, EventId, OwnedUserId, }; -use tracing::warn; +use service::{sending::convert_to_outgoing_federation_event, server_is_ours, services}; -use crate::{ - service::server_is_ours, - services, - utils::{self}, - Error, PduEvent, Result, Ruma, -}; +use crate::Ruma; /// # `PUT /_matrix/federation/v2/invite/{roomId}/{eventId}` /// @@ -176,6 +172,6 @@ pub(crate) async fn create_invite_route( } Ok(create_invite::v2::Response { - event: PduEvent::convert_to_outgoing_federation_event(signed_event), + event: convert_to_outgoing_federation_event(signed_event), }) } diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 82cfc45e..ff362f64 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_join_event}, events::{ @@ -11,11 +12,13 @@ use ruma::{ CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use service::user_is_local; +use service::{ + pdu::gen_event_id_canonical_json, sending::convert_to_outgoing_federation_event, services, user_is_local, +}; use tokio::sync::RwLock; use tracing::warn; -use crate::{service::pdu::gen_event_id_canonical_json, services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// helper method for /send_join v1 and v2 async fn create_join_event( @@ -181,12 +184,12 @@ async fn create_join_event( Ok(create_join_event::v1::RoomState { auth_chain: auth_chain_ids .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok().flatten()) - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) .collect(), state: state_ids .iter() .filter_map(|(_, id)| services().rooms.timeline.get_pdu_json(id).ok().flatten()) - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) .collect(), // Event field is required if the room version supports restricted join rules. event: Some( diff --git a/src/api/server/state.rs b/src/api/server/state.rs index c858f6fd..22044840 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -1,8 +1,10 @@ use std::sync::Arc; +use conduit::{Error, Result}; use ruma::api::{client::error::ErrorKind, federation::event::get_room_state}; +use service::{sending::convert_to_outgoing_federation_event, services}; -use crate::{services, Error, PduEvent, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/state/{roomId}` /// @@ -42,7 +44,7 @@ pub(crate) async fn get_room_state_route( .await? .into_values() .map(|id| { - PduEvent::convert_to_outgoing_federation_event( + convert_to_outgoing_federation_event( services() .rooms .timeline @@ -67,7 +69,7 @@ pub(crate) async fn get_room_state_route( .timeline .get_pdu_json(&id) .ok()? - .map(PduEvent::convert_to_outgoing_federation_event) + .map(convert_to_outgoing_federation_event) }) .collect(), pdus, diff --git a/src/core/mod.rs b/src/core/mod.rs index 5ffe4cb9..ec536ee2 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -4,14 +4,14 @@ pub mod debug; pub mod error; pub mod log; pub mod mods; -pub mod pducount; +pub mod pdu; pub mod server; pub mod utils; pub mod version; pub use config::Config; pub use error::{Error, RumaResponse}; -pub use pducount::PduCount; +pub use pdu::{PduBuilder, PduCount, PduEvent}; pub use server::Server; pub use version::version; diff --git a/src/core/pdu/builder.rs b/src/core/pdu/builder.rs new file mode 100644 index 00000000..a8bad677 --- /dev/null +++ b/src/core/pdu/builder.rs @@ -0,0 +1,16 @@ +use std::{collections::BTreeMap, sync::Arc}; + +use ruma::{events::TimelineEventType, EventId}; +use serde::Deserialize; +use serde_json::value::RawValue as RawJsonValue; + +/// Build the start of a PDU in order to add it to the Database. +#[derive(Debug, Deserialize)] +pub struct PduBuilder { + #[serde(rename = "type")] + pub event_type: TimelineEventType, + pub content: Box, + pub unsigned: Option>, + pub state_key: Option, + pub redacts: Option>, +} diff --git a/src/core/pducount.rs b/src/core/pdu/count.rs similarity index 100% rename from src/core/pducount.rs rename to src/core/pdu/count.rs diff --git a/src/service/pdu.rs b/src/core/pdu/mod.rs similarity index 88% rename from src/service/pdu.rs rename to src/core/pdu/mod.rs index b5650c0a..c7d93608 100644 --- a/src/service/pdu.rs +++ b/src/core/pdu/mod.rs @@ -1,6 +1,10 @@ +mod builder; +mod count; + use std::{cmp::Ordering, collections::BTreeMap, sync::Arc}; -use conduit::{warn, Error}; +pub use builder::PduBuilder; +pub use count::PduCount; use ruma::{ canonical_json::redact_content_in_place, events::{ @@ -19,7 +23,7 @@ use serde_json::{ value::{to_raw_value, RawValue as RawJsonValue}, }; -use crate::services; +use crate::{warn, Error}; #[derive(Deserialize)] struct ExtractRedactedBecause { @@ -336,42 +340,6 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - /// This does not return a full `Pdu` it is only to satisfy ruma's types. - #[tracing::instrument] - pub fn convert_to_outgoing_federation_event(mut pdu_json: CanonicalJsonObject) -> Box { - if let Some(unsigned) = pdu_json - .get_mut("unsigned") - .and_then(|val| val.as_object_mut()) - { - unsigned.remove("transaction_id"); - } - - // room v3 and above removed the "event_id" field from remote PDU format - if let Some(room_id) = pdu_json - .get("room_id") - .and_then(|val| RoomId::parse(val.as_str()?).ok()) - { - match services().rooms.state.get_room_version(&room_id) { - Ok(room_version_id) => match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => _ = pdu_json.remove("event_id"), - }, - Err(_) => _ = pdu_json.remove("event_id"), - } - } else { - pdu_json.remove("event_id"); - } - - // TODO: another option would be to convert it to a canonical string to validate - // size and return a Result> - // serde_json::from_str::>( - // ruma::serde::to_canonical_json_string(pdu_json).expect("CanonicalJson is - // valid serde_json::Value"), ) - // .expect("Raw::from_value always works") - - to_raw_value(&pdu_json).expect("CanonicalJson is valid serde_json::Value") - } - pub fn from_id_val(event_id: &EventId, mut json: CanonicalJsonObject) -> Result { json.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); @@ -438,14 +406,3 @@ pub fn gen_event_id_canonical_json( Ok((event_id, value)) } - -/// Build the start of a PDU in order to add it to the Database. -#[derive(Debug, Deserialize)] -pub struct PduBuilder { - #[serde(rename = "type")] - pub event_type: TimelineEventType, - pub content: Box, - pub unsigned: Option>, - pub state_key: Option, - pub redacts: Option>, -} diff --git a/src/service/mod.rs b/src/service/mod.rs index c80c9862..641dd36a 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,4 +1,3 @@ -pub mod pdu; pub mod services; pub mod account_data; @@ -20,12 +19,12 @@ extern crate conduit_database as database; use std::sync::{Arc, RwLock}; -pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, PduCount, Result, Server}; +pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, Result, Server}; +pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; use database::Database; pub use crate::{ globals::{server_is_ours, user_is_local}, - pdu::PduEvent, services::Services, }; diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 9bda6cad..c57ca14d 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -14,6 +14,7 @@ use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; +pub use sender::convert_to_outgoing_federation_event; use tokio::{sync::Mutex, task::JoinHandle}; use tracing::{error, warn}; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index aa2865ec..61cb9f6d 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -18,12 +18,14 @@ use ruma::{ }, device_id, events::{push_rules::PushRulesEvent, receipt::ReceiptType, AnySyncEphemeralRoomEvent, GlobalAccountDataEventType}, - push, uint, MilliSecondsSinceUnixEpoch, OwnedServerName, OwnedUserId, RoomId, ServerName, UInt, + push, uint, CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, + ServerName, UInt, }; +use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tracing::{debug, error, warn}; use super::{appservice, send, Destination, Msg, SendingEvent, Service}; -use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, PduEvent, Result}; +use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; #[derive(Debug)] enum TransactionStatus { @@ -548,24 +550,21 @@ async fn send_events_dest_normal( for event in &events { match event { - SendingEvent::Pdu(pdu_id) => { + SendingEvent::Pdu(pdu_id) => pdu_jsons.push(convert_to_outgoing_federation_event( // TODO: check room version and remove event_id if needed - let raw = PduEvent::convert_to_outgoing_federation_event( - services() - .rooms - .timeline - .get_pdu_json_from_id(pdu_id) - .map_err(|e| (dest.clone(), e))? - .ok_or_else(|| { - error!(?dest, ?server, ?pdu_id, "event not found"); - ( - dest.clone(), - Error::bad_database("[Normal] Event in servernameevent_data not found in db."), - ) - })?, - ); - pdu_jsons.push(raw); - }, + services() + .rooms + .timeline + .get_pdu_json_from_id(pdu_id) + .map_err(|e| (dest.clone(), e))? + .ok_or_else(|| { + error!(?dest, ?server, ?pdu_id, "event not found"); + ( + dest.clone(), + Error::bad_database("[Normal] Event in servernameevent_data not found in db."), + ) + })?, + )), SendingEvent::Edu(edu) => { if let Ok(raw) = serde_json::from_slice(edu) { edu_jsons.push(raw); @@ -611,3 +610,39 @@ async fn send_events_dest_normal( }) .map_err(|e| (dest.clone(), e)) } + +/// This does not return a full `Pdu` it is only to satisfy ruma's types. +#[tracing::instrument] +pub fn convert_to_outgoing_federation_event(mut pdu_json: CanonicalJsonObject) -> Box { + if let Some(unsigned) = pdu_json + .get_mut("unsigned") + .and_then(|val| val.as_object_mut()) + { + unsigned.remove("transaction_id"); + } + + // room v3 and above removed the "event_id" field from remote PDU format + if let Some(room_id) = pdu_json + .get("room_id") + .and_then(|val| RoomId::parse(val.as_str()?).ok()) + { + match services().rooms.state.get_room_version(&room_id) { + Ok(room_version_id) => match room_version_id { + RoomVersionId::V1 | RoomVersionId::V2 => {}, + _ => _ = pdu_json.remove("event_id"), + }, + Err(_) => _ = pdu_json.remove("event_id"), + } + } else { + pdu_json.remove("event_id"); + } + + // TODO: another option would be to convert it to a canonical string to validate + // size and return a Result> + // serde_json::from_str::>( + // ruma::serde::to_canonical_json_string(pdu_json).expect("CanonicalJson is + // valid serde_json::Value"), ) + // .expect("Raw::from_value always works") + + to_raw_value(&pdu_json).expect("CanonicalJson is valid serde_json::Value") +} From f104ced55dc19559fc380846ee57684c6f670428 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 20:19:20 +0000 Subject: [PATCH 0367/2091] remove unnecessary wrapper Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 3 --- src/service/sending/resolve.rs | 6 ++++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 7e1ccb5e..27be1516 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -14,7 +14,6 @@ use std::{ use conduit::{error, trace, utils::MutexMap, Config, Result, Server}; use data::Data; use database::Database; -use hickory_resolver::TokioAsyncResolver; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ @@ -194,8 +193,6 @@ impl Service { pub fn query_trusted_key_servers_first(&self) -> bool { self.config.query_trusted_key_servers_first } - pub fn dns_resolver(&self) -> &TokioAsyncResolver { &self.resolver.resolver } - pub fn jwt_decoding_key(&self) -> Option<&jsonwebtoken::DecodingKey> { self.jwt_decoding_key.as_ref() } pub fn turn_password(&self) -> &String { &self.config.turn_password } diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 294ac09f..51e7a92f 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -261,7 +261,8 @@ async fn conditional_query_and_cache_override(overname: &str, hostname: &str, po async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { match services() .globals - .dns_resolver() + .resolver + .resolver .lookup_ip(hostname.to_owned()) .await { @@ -299,7 +300,8 @@ async fn query_srv_record(hostname: &'_ str) -> Result> { let hostname = hostname.trim_end_matches('.'); services() .globals - .dns_resolver() + .resolver + .resolver .srv_lookup(hostname.to_owned()) .await } From 25c004f08cc13ef830d380056e326078a4fe6708 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 23:11:05 +0000 Subject: [PATCH 0368/2091] remove dead cache Signed-off-by: Jason Volk --- src/service/rooms/timeline/mod.rs | 11 ++++------- src/service/services.rs | 10 ++-------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index b9c0d7ba..dfcfedea 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1,7 +1,7 @@ mod data; use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashSet}, sync::Arc, }; @@ -27,12 +27,12 @@ use ruma::{ push::{Action, Ruleset, Tweak}, serde::Base64, state_res::{self, Event, RoomVersion}, - uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, - RoomId, RoomVersionId, ServerName, UserId, + uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedServerName, RoomId, + RoomVersionId, ServerName, UserId, }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::RwLock; use crate::{ admin, @@ -66,15 +66,12 @@ struct ExtractBody { pub struct Service { db: Data, - - pub lasttimelinecount_cache: Mutex>, } impl Service { pub fn build(_server: &Arc, db: &Arc) -> Result { Ok(Self { db: Data::new(db), - lasttimelinecount_cache: Mutex::new(HashMap::new()), }) } diff --git a/src/service/services.rs b/src/service/services.rs index 70e6a6d5..81aba231 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -95,10 +95,7 @@ impl Services { let lasttimelinecount_cache = self .rooms .timeline - .lasttimelinecount_cache - .lock() - .await - .len(); + .get_lasttimelinecount_cache_usage().0; let roomid_spacehierarchy_cache = self .rooms .spaces @@ -177,10 +174,7 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} if amount > 4 { self.rooms .timeline - .lasttimelinecount_cache - .lock() - .await - .clear(); + .clear_lasttimelinecount_cache(); } if amount > 5 { self.rooms From 4f5c6de8534e35455d44c55211741975aefb0e2a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 08:44:59 +0000 Subject: [PATCH 0369/2091] start rand utils suite Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 6 ++++-- src/core/utils/rand.rs | 24 ++++++++++++++++++++++++ src/core/utils/string.rs | 10 ---------- 3 files changed, 28 insertions(+), 12 deletions(-) create mode 100644 src/core/utils/rand.rs diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 401775f7..75f55c9a 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -6,6 +6,7 @@ pub mod hash; pub mod html; pub mod json; pub mod mutex_map; +pub mod rand; pub mod string; pub mod sys; mod tests; @@ -19,7 +20,8 @@ pub use hash::calculate_hash; pub use html::Escape as HtmlEscape; pub use json::{deserialize_from_str, to_canonical_object}; pub use mutex_map::MutexMap; -pub use string::{random_string, str_from_bytes, string_from_bytes}; +pub use rand::string as random_string; +pub use string::{str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; pub use time::millis_since_unix_epoch; @@ -41,7 +43,7 @@ pub fn unwrap_infallible(result: Result) -> T { #[must_use] pub fn generate_keypair() -> Vec { - let mut value = random_string(8).as_bytes().to_vec(); + let mut value = rand::string(8).as_bytes().to_vec(); value.push(0xFF); value.extend_from_slice( &ruma::signatures::Ed25519KeyPair::generate().expect("Ed25519KeyPair generation always works (?)"), diff --git a/src/core/utils/rand.rs b/src/core/utils/rand.rs new file mode 100644 index 00000000..1ded8a6d --- /dev/null +++ b/src/core/utils/rand.rs @@ -0,0 +1,24 @@ +use std::{ + ops::Range, + time::{Duration, SystemTime}, +}; + +use rand::{thread_rng, Rng}; + +pub fn string(length: usize) -> String { + thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(length) + .map(char::from) + .collect() +} + +#[inline] +#[must_use] +pub fn timepoint_secs(range: Range) -> SystemTime { SystemTime::now() + secs(range) } + +#[must_use] +pub fn secs(range: Range) -> Duration { + let mut rng = thread_rng(); + Duration::from_secs(rng.gen_range(range)) +} diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 7f6f6531..84a928a0 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -1,15 +1,5 @@ -use rand::prelude::*; - use crate::Result; -pub fn random_string(length: usize) -> String { - thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(length) - .map(char::from) - .collect() -} - /// Parses the bytes into a string. #[inline] pub fn string_from_bytes(bytes: &[u8]) -> Result { From 177c9e8bfa87aab3d483990c333c4fe4e2646dd2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 23:12:43 +0000 Subject: [PATCH 0370/2091] add split_once_infallible string util Signed-off-by: Jason Volk --- src/core/utils/string.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 84a928a0..78765154 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -1,5 +1,13 @@ use crate::Result; +pub const EMPTY: &str = ""; + +#[inline] +#[must_use] +pub fn split_once_infallible<'a>(input: &'a str, delim: &'_ str) -> (&'a str, &'a str) { + input.split_once(delim).unwrap_or((input, EMPTY)) +} + /// Parses the bytes into a string. #[inline] pub fn string_from_bytes(bytes: &[u8]) -> Result { From e125af620eba5ed15c1278bf9b6bad1f05c3086a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 03:26:19 +0000 Subject: [PATCH 0371/2091] impl crate::Service for Service Signed-off-by: Jason Volk --- src/admin/federation/commands.rs | 10 +- src/admin/server/commands.rs | 16 +- src/admin/server/mod.rs | 20 +- src/api/client/keys.rs | 4 +- src/api/client/membership.rs | 4 +- src/core/alloc/je.rs | 2 +- src/service/account_data/mod.rs | 17 +- src/service/admin/mod.rs | 23 +- src/service/appservice/mod.rs | 17 +- src/service/globals/data.rs | 35 --- src/service/globals/mod.rs | 71 +++++- src/service/globals/resolver.rs | 21 +- src/service/key_backups/mod.rs | 17 +- src/service/media/mod.rs | 17 +- src/service/mod.rs | 6 +- src/service/presence/mod.rs | 29 ++- src/service/pusher/mod.rs | 17 +- src/service/rooms/alias/mod.rs | 17 +- src/service/rooms/auth_chain/mod.rs | 17 +- src/service/rooms/directory/mod.rs | 16 +- src/service/rooms/event_handler/mod.rs | 29 ++- src/service/rooms/lazy_loading/mod.rs | 33 ++- src/service/rooms/metadata/mod.rs | 17 +- src/service/rooms/mod.rs | 42 ++-- src/service/rooms/outlier/mod.rs | 17 +- src/service/rooms/pdu_metadata/mod.rs | 17 +- src/service/rooms/read_receipt/mod.rs | 17 +- src/service/rooms/search/mod.rs | 17 +- src/service/rooms/short/mod.rs | 17 +- src/service/rooms/spaces/mod.rs | 17 +- src/service/rooms/state/mod.rs | 17 +- src/service/rooms/state_accessor/mod.rs | 35 ++- src/service/rooms/state_cache/mod.rs | 17 +- src/service/rooms/state_compressor/mod.rs | 29 ++- src/service/rooms/threads/mod.rs | 17 +- src/service/rooms/timeline/mod.rs | 51 ++-- src/service/rooms/typing/mod.rs | 15 +- src/service/rooms/user/mod.rs | 17 +- src/service/sending/mod.rs | 27 +- src/service/service.rs | 53 ++++ src/service/services.rs | 293 +++++++--------------- src/service/transaction_ids/mod.rs | 17 +- src/service/uiaa/mod.rs | 17 +- src/service/users/mod.rs | 17 +- 44 files changed, 673 insertions(+), 548 deletions(-) create mode 100644 src/service/service.rs diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 293efdc5..24f4bc23 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -15,14 +15,18 @@ pub(super) async fn enable_room(_body: Vec<&str>, room_id: Box) -> Resul } pub(super) async fn incoming_federation(_body: Vec<&str>) -> Result { - let map = services().globals.roomid_federationhandletime.read().await; + let map = services() + .globals + .roomid_federationhandletime + .read() + .expect("locked"); let mut msg = format!("Handling {} incoming pdus:\n", map.len()); for (r, (e, i)) in map.iter() { let elapsed = i.elapsed(); - writeln!(msg, "{} {}: {}m{}s", r, e, elapsed.as_secs() / 60, elapsed.as_secs() % 60,) - .expect("should be able to write to string buffer"); + writeln!(msg, "{} {}: {}m{}s", r, e, elapsed.as_secs() / 60, elapsed.as_secs() % 60)?; } + Ok(RoomMessageEventContent::text_plain(&msg)) } diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 77b20377..2ab71033 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -27,12 +27,12 @@ pub(super) async fn show_config(_body: Vec<&str>) -> Result) -> Result { - let response0 = services().memory_usage().await; - let response1 = services().globals.db.memory_usage(); + let response0 = services().memory_usage().await?; + let response1 = services().db.db.memory_usage()?; let response2 = conduit::alloc::memory_usage(); Ok(RoomMessageEventContent::text_plain(format!( - "Services:\n{response0}\n\nDatabase:\n{response1}\n{}", + "Services:\n{response0}\nDatabase:\n{response1}\n{}", if !response2.is_empty() { format!("Allocator:\n {response2}") } else { @@ -41,14 +41,8 @@ pub(super) async fn memory_usage(_body: Vec<&str>) -> Result, amount: u32) -> Result { - services().globals.db.clear_caches(amount); - - Ok(RoomMessageEventContent::text_plain("Done.")) -} - -pub(super) async fn clear_service_caches(_body: Vec<&str>, amount: u32) -> Result { - services().clear_caches(amount).await; +pub(super) async fn clear_caches(_body: Vec<&str>) -> Result { + services().clear_cache().await; Ok(RoomMessageEventContent::text_plain("Done.")) } diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index b4222251..41b10180 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -18,17 +18,8 @@ pub(super) enum ServerCommand { /// - Print database memory usage statistics MemoryUsage, - /// - Clears all of Conduit's database caches with index smaller than the - /// amount - ClearDatabaseCaches { - amount: u32, - }, - - /// - Clears all of Conduit's service caches with index smaller than the - /// amount - ClearServiceCaches { - amount: u32, - }, + /// - Clears all of Conduwuit's caches + ClearCaches, /// - Performs an online backup of the database (only available for RocksDB /// at the moment) @@ -65,12 +56,7 @@ pub(super) async fn process(command: ServerCommand, body: Vec<&str>) -> Result uptime(body).await?, ServerCommand::ShowConfig => show_config(body).await?, ServerCommand::MemoryUsage => memory_usage(body).await?, - ServerCommand::ClearDatabaseCaches { - amount, - } => clear_database_caches(body, amount).await?, - ServerCommand::ClearServiceCaches { - amount, - } => clear_service_caches(body, amount).await?, + ServerCommand::ClearCaches => clear_caches(body).await?, ServerCommand::ListBackups => list_backups(body).await?, ServerCommand::BackupDatabase => backup_database(body).await?, ServerCommand::ListDatabaseFiles => list_database_files(body).await?, diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 0cfb0138..6f089875 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -334,7 +334,7 @@ pub(crate) async fn get_keys_helper bool + Send>( .globals .bad_query_ratelimiter .write() - .await + .expect("locked") .entry(id) { hash_map::Entry::Vacant(e) => { @@ -353,7 +353,7 @@ pub(crate) async fn get_keys_helper bool + Send>( .globals .bad_query_ratelimiter .read() - .await + .expect("locked") .get(server) { // Exponential backoff diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index bb19ee29..07a585fd 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1343,7 +1343,7 @@ pub async fn validate_and_add_event_id( .globals .bad_event_ratelimiter .write() - .await + .expect("locked") .entry(id) { Entry::Vacant(e) => { @@ -1359,7 +1359,7 @@ pub async fn validate_and_add_event_id( .globals .bad_event_ratelimiter .read() - .await + .expect("locked") .get(&event_id) { // Exponential backoff diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index e0628ef3..966cbde3 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -19,7 +19,7 @@ pub fn memory_usage() -> String { let resident = stats::resident::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; let retained = stats::retained::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; format!( - " allocated: {allocated:.2} MiB\n active: {active:.2} MiB\n mapped: {mapped:.2} MiB\n metadata: {metadata:.2} \ + "allocated: {allocated:.2} MiB\n active: {active:.2} MiB\n mapped: {mapped:.2} MiB\n metadata: {metadata:.2} \ MiB\n resident: {resident:.2} MiB\n retained: {retained:.2} MiB\n " ) } diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 62560500..66474370 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -2,9 +2,8 @@ mod data; use std::{collections::HashMap, sync::Arc}; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{ events::{AnyEphemeralRoomEvent, RoomAccountDataEventType}, serde::Raw, @@ -15,13 +14,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Places one event in the account data of the user and removes the /// previous entry. #[allow(clippy::needless_pass_by_value)] diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index b5cb8af4..ed2ac725 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -4,9 +4,9 @@ mod grant; use std::{future::Future, pin::Pin, sync::Arc}; -use conduit::{error, utils::mutex_map, Error, Result, Server}; +use async_trait::async_trait; +use conduit::{error, utils::mutex_map, Error, Result}; pub use create::create_admin_room; -use database::Database; pub use grant::make_user_admin; use loole::{Receiver, Sender}; use ruma::{ @@ -43,8 +43,9 @@ pub struct Command { pub reply_id: Option, } -impl Service { - pub fn build(_server: &Arc, _db: &Arc) -> Result> { +#[async_trait] +impl crate::Service for Service { + fn build(_args: crate::Args<'_>) -> Result> { let (sender, receiver) = loole::bounded(COMMAND_QUEUE_LIMIT); Ok(Arc::new(Self { sender, @@ -56,8 +57,8 @@ impl Service { })) } - pub async fn start_handler(self: &Arc) { - let self_ = Arc::clone(self); + async fn start(self: Arc) -> Result<()> { + let self_ = Arc::clone(&self); let handle = services().server.runtime().spawn(async move { self_ .handler() @@ -66,9 +67,11 @@ impl Service { }); _ = self.handler_join.lock().await.insert(handle); + + Ok(()) } - pub fn interrupt(&self) { + fn interrupt(&self) { #[cfg(feature = "console")] self.console.interrupt(); @@ -77,7 +80,7 @@ impl Service { } } - pub async fn close(&self) { + async fn stop(&self) { self.interrupt(); #[cfg(feature = "console")] @@ -90,6 +93,10 @@ impl Service { } } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub async fn send_text(&self, body: &str) { self.send_message(RoomMessageEventContent::text_markdown(body)) .await; diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 916e7df3..9bb00df1 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -2,9 +2,8 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use futures_util::Future; use regex::RegexSet; use ruma::{ @@ -119,10 +118,10 @@ pub struct Service { registration_info: RwLock>, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { let mut registration_info = BTreeMap::new(); - let db = Data::new(db); + let db = Data::new(args.db); // Inserting registrations into cache for appservice in iter_ids(&db)? { registration_info.insert( @@ -134,12 +133,16 @@ impl Service { ); } - Ok(Self { + Ok(Arc::new(Self { db, registration_info: RwLock::new(registration_info), - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn all(&self) -> Result> { iter_ids(&self.db) } /// Registers an appservice and returns the ID to the caller diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 022762f4..43614854 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -208,41 +208,6 @@ impl Data { pub fn cleanup(&self) -> Result<()> { self.db.db.cleanup() } - pub fn memory_usage(&self) -> String { - let (auth_chain_cache, max_auth_chain_cache) = services().rooms.auth_chain.get_cache_usage(); - let (appservice_in_room_cache, max_appservice_in_room_cache) = services() - .rooms - .state_cache - .get_appservice_in_room_cache_usage(); - let (lasttimelinecount_cache, max_lasttimelinecount_cache) = services() - .rooms - .timeline - .get_lasttimelinecount_cache_usage(); - - format!( - "auth_chain_cache: {auth_chain_cache} / {max_auth_chain_cache}\nappservice_in_room_cache: \ - {appservice_in_room_cache} / {max_appservice_in_room_cache}\nlasttimelinecount_cache: \ - {lasttimelinecount_cache} / {max_lasttimelinecount_cache}\n\n{}", - self.db.db.memory_usage().unwrap_or_default() - ) - } - - #[allow(clippy::unused_self)] - pub fn clear_caches(&self, amount: u32) { - if amount > 1 { - services().rooms.auth_chain.clear_cache(); - } - if amount > 2 { - services() - .rooms - .state_cache - .clear_appservice_in_room_cache(); - } - if amount > 3 { - services().rooms.timeline.clear_lasttimelinecount_cache(); - } - } - pub fn load_keypair(&self) -> Result { let keypair_bytes = self.global.get(b"keypair")?.map_or_else( || { diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 27be1516..2a6fbc2f 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,13 +7,13 @@ pub(super) mod updates; use std::{ collections::{BTreeMap, HashMap}, - sync::Arc, + fmt::Write, + sync::{Arc, RwLock}, time::Instant, }; -use conduit::{error, trace, utils::MutexMap, Config, Result, Server}; +use conduit::{error, trace, utils::MutexMap, Config, Result}; use data::Data; -use database::Database; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ @@ -25,10 +25,7 @@ use ruma::{ DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, }; -use tokio::{ - sync::{Mutex, RwLock}, - task::JoinHandle, -}; +use tokio::{sync::Mutex, task::JoinHandle}; use url::Url; use crate::services; @@ -59,10 +56,10 @@ pub struct Service { pub admin_alias: OwnedRoomAliasId, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - let config = &server.config; - let db = Data::new(db); +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + let db = Data::new(args.db); let keypair = db.load_keypair(); let keypair = match keypair { @@ -133,9 +130,59 @@ impl Service { s.config.default_room_version = crate::config::default_default_room_version(); }; - Ok(s) + Ok(Arc::new(s)) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + self.resolver.memory_usage(out)?; + + let bad_event_ratelimiter = self + .bad_event_ratelimiter + .read() + .expect("locked for reading") + .len(); + writeln!(out, "bad_event_ratelimiter: {bad_event_ratelimiter}")?; + + let bad_query_ratelimiter = self + .bad_query_ratelimiter + .read() + .expect("locked for reading") + .len(); + writeln!(out, "bad_query_ratelimiter: {bad_query_ratelimiter}")?; + + let bad_signature_ratelimiter = self + .bad_signature_ratelimiter + .read() + .expect("locked for reading") + .len(); + writeln!(out, "bad_signature_ratelimiter: {bad_signature_ratelimiter}")?; + + Ok(()) + } + + fn clear_cache(&self) { + self.resolver.clear_cache(); + + self.bad_event_ratelimiter + .write() + .expect("locked for writing") + .clear(); + + self.bad_query_ratelimiter + .write() + .expect("locked for writing") + .clear(); + + self.bad_signature_ratelimiter + .write() + .expect("locked for writing") + .clear(); + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Returns this server's keypair. pub fn keypair(&self) -> &ruma::signatures::Ed25519KeyPair { &self.keypair } diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index 86fa6700..e5199162 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -1,12 +1,13 @@ use std::{ collections::HashMap, + fmt::Write, future, iter, net::{IpAddr, SocketAddr}, sync::{Arc, RwLock}, time::Duration, }; -use conduit::{error, Config, Error}; +use conduit::{error, Config, Error, Result}; use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use ruma::OwnedServerName; @@ -30,7 +31,7 @@ pub struct Hooked { impl Resolver { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] - pub fn new(config: &Config) -> Self { + pub(super) fn new(config: &Config) -> Self { let (sys_conf, mut opts) = hickory_resolver::system_conf::read_system_conf() .map_err(|e| { error!("Failed to set up hickory dns resolver with system config: {e}"); @@ -92,6 +93,22 @@ impl Resolver { }), } } + + pub(super) fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let resolver_overrides_cache = self.overrides.read().expect("locked for reading").len(); + writeln!(out, "resolver_overrides_cache: {resolver_overrides_cache}")?; + + let resolver_destinations_cache = self.destinations.read().expect("locked for reading").len(); + writeln!(out, "resolver_destinations_cache: {resolver_destinations_cache}")?; + + Ok(()) + } + + pub(super) fn clear_cache(&self) { + self.overrides.write().expect("write locked").clear(); + self.destinations.write().expect("write locked").clear(); + self.resolver.clear_cache(); + } } impl Resolve for Resolver { diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 650aa6b6..d83d4497 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -2,9 +2,8 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{ api::client::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, serde::Raw, @@ -15,13 +14,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { self.db.create_backup(user_id, backup_metadata) } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 0323bfc5..caa75d4e 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -6,7 +6,6 @@ use std::{collections::HashMap, io::Cursor, path::PathBuf, sync::Arc, time::Syst use base64::{engine::general_purpose, Engine as _}; use conduit::{debug, debug_error, error, utils, Error, Result, Server}; use data::Data; -use database::Database; use image::imageops::FilterType; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; @@ -48,15 +47,19 @@ pub struct Service { pub url_preview_mutex: RwLock>>>, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - Ok(Self { - server: server.clone(), - db: Data::new(db), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + server: args.server.clone(), + db: Data::new(args.db), url_preview_mutex: RwLock::new(HashMap::new()), - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Uploads a file. pub async fn create( &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, diff --git a/src/service/mod.rs b/src/service/mod.rs index 641dd36a..9c83b25c 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,3 +1,6 @@ +#![allow(refining_impl_trait)] + +mod service; pub mod services; pub mod account_data; @@ -22,6 +25,7 @@ use std::sync::{Arc, RwLock}; pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, Result, Server}; pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; use database::Database; +pub(crate) use service::{Args, Service}; pub use crate::{ globals::{server_is_ours, user_is_local}, @@ -36,7 +40,7 @@ static SERVICES: RwLock> = RwLock::new(None); #[allow(clippy::let_underscore_must_use)] pub async fn init(server: &Arc) -> Result<()> { let d = Arc::new(Database::open(server).await?); - let s = Box::new(Services::build(server.clone(), d.clone()).await?); + let s = Box::new(Services::build(server.clone(), d)?); _ = SERVICES.write().expect("write locked").insert(Box::leak(s)); Ok(()) diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 5065efdd..6f05d402 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -2,9 +2,9 @@ mod data; use std::{sync::Arc, time::Duration}; -use conduit::{debug, error, utils, Error, Result, Server}; +use async_trait::async_trait; +use conduit::{debug, error, utils, Error, Result}; use data::Data; -use database::Database; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ events::presence::{PresenceEvent, PresenceEventContent}, @@ -81,12 +81,13 @@ pub struct Service { timeout_remote_users: bool, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result> { - let config = &server.config; +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; let (timer_sender, timer_receiver) = loole::unbounded(); Ok(Arc::new(Self { - db: Data::new(db), + db: Data::new(args.db), timer_sender, timer_receiver: Mutex::new(timer_receiver), handler_join: Mutex::new(None), @@ -94,8 +95,10 @@ impl Service { })) } - pub async fn start_handler(self: &Arc) { - let self_ = Arc::clone(self); + async fn start(self: Arc) -> Result<()> { + //TODO: if self.globals.config.allow_local_presence { return; } + + let self_ = Arc::clone(&self); let handle = services().server.runtime().spawn(async move { self_ .handler() @@ -104,9 +107,11 @@ impl Service { }); _ = self.handler_join.lock().await.insert(handle); + + Ok(()) } - pub async fn close(&self) { + async fn stop(&self) { self.interrupt(); if let Some(handler_join) = self.handler_join.lock().await.take() { if let Err(e) = handler_join.await { @@ -115,12 +120,16 @@ impl Service { } } - pub fn interrupt(&self) { + fn interrupt(&self) { if !self.timer_sender.is_closed() { self.timer_sender.close(); } } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Returns the latest presence event for the given user. pub fn get_presence(&self, user_id: &UserId) -> Result> { if let Some((_, presence)) = self.db.get_presence(user_id)? { diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 28280273..27b49c28 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -3,9 +3,8 @@ mod data; use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; -use conduit::{debug_info, info, trace, warn, Error, Result, Server}; +use conduit::{debug_info, info, trace, warn, Error, Result}; use data::Data; -use database::Database; use ipaddress::IPAddress; use ruma::{ api::{ @@ -30,13 +29,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result<()> { self.db.set_pusher(sender, pusher) } diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 9f66cf88..d375561e 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -3,9 +3,8 @@ mod remote; use std::sync::Arc; -use conduit::{Error, Result, Server}; +use conduit::{Error, Result}; use data::Data; -use database::Database; use ruma::{ api::{appservice, client::error::ErrorKind}, events::{ @@ -21,13 +20,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { if alias == services().globals.admin_alias && user_id != services().globals.server_user { diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 6b14fda0..ca04b1e5 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -5,9 +5,8 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, trace, warn, Error, Result, Server}; +use conduit::{debug, error, trace, warn, Error, Result}; use data::Data; -use database::Database; use ruma::{api::client::error::ErrorKind, EventId, RoomId}; use crate::services; @@ -16,13 +15,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(server, db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.server, args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub async fn event_ids_iter<'a>( &self, room_id: &RoomId, starting_events_: Vec>, ) -> Result> + 'a> { diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 87c7cf92..3e60831c 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -2,9 +2,7 @@ mod data; use std::sync::Arc; -use conduit::Server; use data::Data; -use database::Database; use ruma::{OwnedRoomId, RoomId}; use crate::Result; @@ -13,13 +11,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn set_public(&self, room_id: &RoomId) -> Result<()> { self.db.set_public(room_id) } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index ed09e2e6..bb22c6c8 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -9,8 +9,7 @@ use std::{ time::{Duration, Instant}, }; -use conduit::{debug_error, debug_info, Error, Result, Server}; -use database::Database; +use conduit::{debug_error, debug_info, Error, Result}; use futures_util::Future; pub use parse_incoming_pdu::parse_incoming_pdu; use ruma::{ @@ -45,9 +44,13 @@ type AsyncRecursiveCanonicalJsonVec<'a> = type AsyncRecursiveCanonicalJsonResult<'a> = AsyncRecursiveType<'a, Result<(Arc, BTreeMap)>>; -impl Service { - pub fn build(_server: &Arc, _db: &Arc) -> Result { Ok(Self {}) } +impl crate::Service for Service { + fn build(_args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self {})) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// When receiving an event one needs to: /// 0. Check the server is in the room /// 1. Skip the PDU if we already know about it @@ -180,7 +183,7 @@ impl Service { .globals .bad_event_ratelimiter .write() - .await + .expect("locked") .entry((*prev_id).to_owned()) { hash_map::Entry::Vacant(e) => { @@ -200,7 +203,7 @@ impl Service { .globals .roomid_federationhandletime .write() - .await + .expect("locked") .insert(room_id.to_owned(), (event_id.to_owned(), start_time)); let r = self @@ -211,7 +214,7 @@ impl Service { .globals .roomid_federationhandletime .write() - .await + .expect("locked") .remove(&room_id.to_owned()); r @@ -245,7 +248,7 @@ impl Service { .globals .bad_event_ratelimiter .read() - .await + .expect("locked") .get(prev_id) { // Exponential backoff @@ -274,7 +277,7 @@ impl Service { .globals .roomid_federationhandletime .write() - .await + .expect("locked") .insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time)); self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id, pub_key_map) @@ -284,7 +287,7 @@ impl Service { .globals .roomid_federationhandletime .write() - .await + .expect("locked") .remove(&room_id.to_owned()); debug!( @@ -1043,7 +1046,7 @@ impl Service { .globals .bad_event_ratelimiter .write() - .await + .expect("locked") .entry(id) { hash_map::Entry::Vacant(e) => { @@ -1076,7 +1079,7 @@ impl Service { .globals .bad_event_ratelimiter .read() - .await + .expect("locked") .get(&*next_id) { // Exponential backoff @@ -1184,7 +1187,7 @@ impl Service { .globals .bad_event_ratelimiter .read() - .await + .expect("locked") .get(&**next_id) { // Exponential backoff diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 1f2ae6dd..185cfd8c 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -2,14 +2,12 @@ mod data; use std::{ collections::{HashMap, HashSet}, - sync::Arc, + fmt::Write, + sync::{Arc, Mutex}, }; -use conduit::Server; use data::Data; -use database::Database; use ruma::{DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, RoomId, UserId}; -use tokio::sync::Mutex; use crate::{PduCount, Result}; @@ -20,14 +18,27 @@ pub struct Service { pub lazy_load_waiting: Mutex>>, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), lazy_load_waiting: Mutex::new(HashMap::new()), - }) + })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let lazy_load_waiting = self.lazy_load_waiting.lock().expect("locked").len(); + writeln!(out, "lazy_load_waiting: {lazy_load_waiting}")?; + + Ok(()) + } + + fn clear_cache(&self) { self.lazy_load_waiting.lock().expect("locked").clear(); } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn lazy_load_was_sent_before( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, @@ -43,7 +54,7 @@ impl Service { ) { self.lazy_load_waiting .lock() - .await + .expect("locked") .insert((user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), count), lazy_load); } @@ -51,7 +62,7 @@ impl Service { pub async fn lazy_load_confirm_delivery( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, since: PduCount, ) -> Result<()> { - if let Some(user_ids) = self.lazy_load_waiting.lock().await.remove(&( + if let Some(user_ids) = self.lazy_load_waiting.lock().expect("locked").remove(&( user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index b91fc67e..3a96b89f 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{OwnedRoomId, RoomId}; pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Checks if a room exists. #[tracing::instrument(skip(self))] pub fn exists(&self, room_id: &RoomId) -> Result { self.db.exists(room_id) } diff --git a/src/service/rooms/mod.rs b/src/service/rooms/mod.rs index bef56a25..ef50b094 100644 --- a/src/service/rooms/mod.rs +++ b/src/service/rooms/mod.rs @@ -19,25 +19,27 @@ pub mod timeline; pub mod typing; pub mod user; +use std::sync::Arc; + pub struct Service { - pub alias: alias::Service, - pub auth_chain: auth_chain::Service, - pub directory: directory::Service, - pub event_handler: event_handler::Service, - pub lazy_loading: lazy_loading::Service, - pub metadata: metadata::Service, - pub outlier: outlier::Service, - pub pdu_metadata: pdu_metadata::Service, - pub read_receipt: read_receipt::Service, - pub search: search::Service, - pub short: short::Service, - pub state: state::Service, - pub state_accessor: state_accessor::Service, - pub state_cache: state_cache::Service, - pub state_compressor: state_compressor::Service, - pub timeline: timeline::Service, - pub threads: threads::Service, - pub typing: typing::Service, - pub spaces: spaces::Service, - pub user: user::Service, + pub alias: Arc, + pub auth_chain: Arc, + pub directory: Arc, + pub event_handler: Arc, + pub lazy_loading: Arc, + pub metadata: Arc, + pub outlier: Arc, + pub pdu_metadata: Arc, + pub read_receipt: Arc, + pub search: Arc, + pub short: Arc, + pub state: Arc, + pub state_accessor: Arc, + pub state_cache: Arc, + pub state_compressor: Arc, + pub timeline: Arc, + pub threads: Arc, + pub typing: Arc, + pub spaces: Arc, + pub user: Arc, } diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 0da12a14..a7326f42 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{CanonicalJsonObject, EventId}; use crate::PduEvent; @@ -13,13 +12,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Returns the pdu from the outlier tree. pub fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result> { self.db.get_outlier_pdu_json(event_id) diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 978a0ec0..e35c969d 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{ api::{client::relations::get_relating_events, Direction}, events::{relation::RelationType, TimelineEventType}, @@ -28,13 +27,17 @@ struct ExtractRelatesToEventId { relates_to: ExtractRelType, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self, from, to))] pub fn add_relation(&self, from: PduCount, to: PduCount) -> Result<()> { match (from, to) { diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index ccc17d3a..e46027b7 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{events::receipt::ReceiptEvent, serde::Raw, OwnedUserId, RoomId, UserId}; use crate::services; @@ -13,13 +12,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Replaces the previous read receipt. pub fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) -> Result<()> { self.db.readreceipt_update(user_id, room_id, event)?; diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index e2ef0f80..7573d218 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::RoomId; pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { self.db.index_pdu(shortroomid, pdu_id, message_body) diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 638b6c1c..0979fb4f 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{events::StateEventType, EventId, RoomId}; pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn get_or_create_shorteventid(&self, event_id: &EventId) -> Result { self.db.get_or_create_shorteventid(event_id) } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 6a87ac98..bf6cc873 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,8 +7,7 @@ use std::{ sync::Arc, }; -use conduit::{debug_info, Server}; -use database::Database; +use conduit::debug_info; use lru_cache::LruCache; use ruma::{ api::{ @@ -159,17 +158,21 @@ impl From for SpaceHierarchyRoomsChunk { } } -impl Service { - pub fn build(server: &Arc, _db: &Arc) -> Result { - let config = &server.config; - Ok(Self { +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + Ok(Arc::new(Self { roomid_spacehierarchy_cache: Mutex::new(LruCache::new( (f64::from(config.roomid_spacehierarchy_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, )), - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Gets the response for the space hierarchy over federation request /// /// Errors if the room does not exist, so a check if the room exists should diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 42483c97..bf3b29b7 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -7,10 +7,9 @@ use std::{ use conduit::{ utils::{calculate_hash, mutex_map}, - warn, Error, Result, Server, + warn, Error, Result, }; use data::Data; -use database::Database; use ruma::{ api::client::error::ErrorKind, events::{ @@ -29,13 +28,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Set the room to the given statehash and update caches. pub async fn force_state( &self, diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 03e22187..ac127b35 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -2,12 +2,12 @@ mod data; use std::{ collections::HashMap, + fmt::Write, sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{error, utils::mutex_map, warn, Error, Result, Server}; +use conduit::{error, utils::mutex_map, warn, Error, Result}; use data::Data; -use database::Database; use lru_cache::LruCache; use ruma::{ events::{ @@ -41,20 +41,39 @@ pub struct Service { pub user_visibility_cache: Mutex>, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - let config = &server.config; - Ok(Self { - db: Data::new(db), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + Ok(Arc::new(Self { + db: Data::new(args.db), server_visibility_cache: StdMutex::new(LruCache::new( (f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, )), user_visibility_cache: StdMutex::new(LruCache::new( (f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, )), - }) + })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let server_visibility_cache = self.server_visibility_cache.lock().expect("locked").len(); + writeln!(out, "server_visibility_cache: {server_visibility_cache}")?; + + let user_visibility_cache = self.user_visibility_cache.lock().expect("locked").len(); + writeln!(out, "user_visibility_cache: {user_visibility_cache}")?; + + Ok(()) + } + + fn clear_cache(&self) { + self.server_visibility_cache.lock().expect("locked").clear(); + self.user_visibility_cache.lock().expect("locked").clear(); + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[tracing::instrument(skip(self))] diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 5038ef1c..45e05c29 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{error, warn, Error, Result, Server}; +use conduit::{error, warn, Error, Result}; use data::Data; -use database::Database; use itertools::Itertools; use ruma::{ events::{ @@ -28,13 +27,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Update current membership data. #[tracing::instrument(skip(self, last_state))] #[allow(clippy::too_many_arguments)] diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 08112995..564a0bee 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -2,13 +2,13 @@ mod data; use std::{ collections::HashSet, + fmt::Write, mem::size_of, sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{utils, Result, Server}; +use conduit::{utils, Result}; use data::Data; -use database::Database; use lru_cache::LruCache; use ruma::{EventId, RoomId}; @@ -52,17 +52,30 @@ pub struct Service { pub stateinfo_cache: StateInfoLruCache, } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result { - let config = &server.config; - Ok(Self { - db: Data::new(db), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + Ok(Arc::new(Self { + db: Data::new(args.db), stateinfo_cache: StdMutex::new(LruCache::new( (f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, )), - }) + })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let stateinfo_cache = self.stateinfo_cache.lock().expect("locked").len(); + writeln!(out, "stateinfo_cache: {stateinfo_cache}")?; + + Ok(()) + } + + fn clear_cache(&self) { self.stateinfo_cache.lock().expect("locked").clear(); } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. #[tracing::instrument(skip(self))] diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index f47fc499..f3cefe21 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -2,9 +2,8 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{Error, Result, Server}; +use conduit::{Error, Result}; use data::Data; -use database::Database; use ruma::{ api::client::{error::ErrorKind, threads::get_threads::v1::IncludeThreads}, events::relation::BundledThread, @@ -18,13 +17,17 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn threads_until<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, include: &'a IncludeThreads, ) -> Result> + 'a> { diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index dfcfedea..f7e8384b 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -2,12 +2,12 @@ mod data; use std::{ collections::{BTreeMap, HashSet}, + fmt::Write, sync::Arc, }; -use conduit::{debug, error, info, utils, utils::mutex_map, warn, Error, Result, Server}; +use conduit::{debug, error, info, utils, utils::mutex_map, warn, Error, Result}; use data::Data; -use database::Database; use itertools::Itertools; use rand::prelude::SliceRandom; use ruma::{ @@ -68,13 +68,37 @@ pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let lasttimelinecount_cache = self + .db + .lasttimelinecount_cache + .lock() + .expect("locked") + .len(); + writeln!(out, "lasttimelinecount_cache: {lasttimelinecount_cache}")?; + + Ok(()) + } + + fn clear_cache(&self) { + self.db + .lasttimelinecount_cache + .lock() + .expect("locked") + .clear(); + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self))] pub fn first_pdu_in_room(&self, room_id: &RoomId) -> Result>> { self.all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id)? @@ -1238,19 +1262,6 @@ impl Service { debug!("Prepended backfill pdu"); Ok(()) } - - pub fn get_lasttimelinecount_cache_usage(&self) -> (usize, usize) { - let cache = self.db.lasttimelinecount_cache.lock().expect("locked"); - (cache.len(), cache.capacity()) - } - - pub fn clear_lasttimelinecount_cache(&self) { - self.db - .lasttimelinecount_cache - .lock() - .expect("locked") - .clear(); - } } #[cfg(test)] diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index dd4e7fe2..715e3162 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -1,7 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduit::{debug_info, trace, utils, Result, Server}; -use database::Database; +use conduit::{debug_info, trace, utils, Result}; use ruma::{ api::federation::transactions::edu::{Edu, TypingContent}, events::SyncEphemeralRoomEvent, @@ -19,15 +18,19 @@ pub struct Service { pub typing_update_sender: broadcast::Sender, } -impl Service { - pub fn build(_server: &Arc, _db: &Arc) -> Result { - Ok(Self { +impl crate::Service for Service { + fn build(_args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { typing: RwLock::new(BTreeMap::new()), last_typing_update: RwLock::new(BTreeMap::new()), typing_update_sender: broadcast::channel(100).0, - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Sets a user as typing until the timeout timestamp is reached or /// roomtyping_remove is called. pub async fn typing_add(&self, user_id: &UserId, room_id: &RoomId, timeout: u64) -> Result<()> { diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 505e0662..12124a57 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; pub struct Service { db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { self.db.reset_notification_counts(user_id, room_id) } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index c57ca14d..b75ca8b4 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -6,9 +6,9 @@ mod sender; use std::{fmt::Debug, sync::Arc}; -use conduit::{Error, Result, Server}; +use async_trait::async_trait; +use conduit::{Error, Result}; use data::Data; -use database::Database; pub use resolve::FedDest; use ruma::{ api::{appservice::Registration, OutgoingRequest}, @@ -53,12 +53,13 @@ pub enum SendingEvent { Flush, // none } -impl Service { - pub fn build(server: &Arc, db: &Arc) -> Result> { - let config = &server.config; +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; let (sender, receiver) = loole::unbounded(); Ok(Arc::new(Self { - db: Data::new(db.clone()), + db: Data::new(args.db.clone()), sender, receiver: Mutex::new(receiver), handler_join: Mutex::new(None), @@ -67,7 +68,13 @@ impl Service { })) } - pub async fn close(&self) { + async fn start(self: Arc) -> Result<()> { + self.start_handler().await; + + Ok(()) + } + + async fn stop(&self) { self.interrupt(); if let Some(handler_join) = self.handler_join.lock().await.take() { if let Err(e) = handler_join.await { @@ -76,12 +83,16 @@ impl Service { } } - pub fn interrupt(&self) { + fn interrupt(&self) { if !self.sender.is_closed() { self.sender.close(); } } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { #[tracing::instrument(skip(self, pdu_id, user, pushkey))] pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { let dest = Destination::Push(user.to_owned(), pushkey); diff --git a/src/service/service.rs b/src/service/service.rs new file mode 100644 index 00000000..ef60f359 --- /dev/null +++ b/src/service/service.rs @@ -0,0 +1,53 @@ +use std::{collections::BTreeMap, fmt::Write, sync::Arc}; + +use async_trait::async_trait; +use conduit::{utils::string::split_once_infallible, Result, Server}; +use database::Database; + +#[async_trait] +pub(crate) trait Service: Send + Sync { + /// Implement the construction of the service instance. Services are + /// generally singletons so expect this to only be called once for a + /// service type. Note that it may be called again after a server reload, + /// but the prior instance will have been dropped first. Failure will + /// shutdown the server with an error. + fn build(args: Args<'_>) -> Result> + where + Self: Sized; + + /// Start the service. Implement the spawning of any service workers. This + /// is called after all other services have been constructed. Failure will + /// shutdown the server with an error. + async fn start(self: Arc) -> Result<()> { Ok(()) } + + /// Stop the service. Implement the joining of any service workers and + /// cleanup of any other state. This function is asynchronous to allow that + /// gracefully, but errors cannot propagate. + async fn stop(&self) {} + + /// Interrupt the service. This may be sent prior to `stop()` as a + /// notification to improve the shutdown sequence. Implementations must be + /// robust to this being called multiple times. + fn interrupt(&self) {} + + /// Clear any caches or similar runtime state. + fn clear_cache(&self) {} + + /// Memory usage report in a markdown string. + fn memory_usage(&self, _out: &mut dyn Write) -> Result<()> { Ok(()) } + + /// Return the name of the service. + /// i.e. `crate::service::make_name(std::module_path!())` + fn name(&self) -> &str; +} + +pub(crate) struct Args<'a> { + pub(crate) server: &'a Arc, + pub(crate) db: &'a Arc, + pub(crate) _service: &'a Map, +} + +pub(crate) type Map = BTreeMap>; + +#[inline] +pub(crate) fn make_name(module_path: &str) -> &str { split_once_infallible(module_path, "::").1 } diff --git a/src/service/services.rs b/src/service/services.rs index 81aba231..0ba86693 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,101 +1,97 @@ -use std::sync::Arc; +use std::{collections::BTreeMap, fmt::Write, sync::Arc}; -use conduit::{debug_info, Result, Server}; +use conduit::{debug, debug_info, info, trace, Result, Server}; use database::Database; -use tracing::{debug, info, trace}; use crate::{ - account_data, admin, appservice, globals, key_backups, media, presence, pusher, rooms, sending, transaction_ids, - uiaa, users, + account_data, admin, appservice, globals, key_backups, media, presence, pusher, rooms, sending, + service::{Args, Map, Service}, + transaction_ids, uiaa, users, }; pub struct Services { pub rooms: rooms::Service, - pub appservice: appservice::Service, - pub pusher: pusher::Service, - pub transaction_ids: transaction_ids::Service, - pub uiaa: uiaa::Service, - pub users: users::Service, - pub account_data: account_data::Service, + pub appservice: Arc, + pub pusher: Arc, + pub transaction_ids: Arc, + pub uiaa: Arc, + pub users: Arc, + pub account_data: Arc, pub presence: Arc, pub admin: Arc, - pub globals: globals::Service, - pub key_backups: key_backups::Service, - pub media: media::Service, + pub globals: Arc, + pub key_backups: Arc, + pub media: Arc, pub sending: Arc, + + pub(crate) service: Map, pub server: Arc, pub db: Arc, } impl Services { - pub async fn build(server: Arc, db: Arc) -> Result { + pub fn build(server: Arc, db: Arc) -> Result { + let mut service: Map = BTreeMap::new(); + macro_rules! build { + ($tyname:ty) => {{ + let built = <$tyname>::build(Args { + server: &server, + db: &db, + _service: &service, + })?; + service.insert(built.name().to_owned(), built.clone()); + built + }}; + } + Ok(Self { rooms: rooms::Service { - alias: rooms::alias::Service::build(&server, &db)?, - auth_chain: rooms::auth_chain::Service::build(&server, &db)?, - directory: rooms::directory::Service::build(&server, &db)?, - event_handler: rooms::event_handler::Service::build(&server, &db)?, - lazy_loading: rooms::lazy_loading::Service::build(&server, &db)?, - metadata: rooms::metadata::Service::build(&server, &db)?, - outlier: rooms::outlier::Service::build(&server, &db)?, - pdu_metadata: rooms::pdu_metadata::Service::build(&server, &db)?, - read_receipt: rooms::read_receipt::Service::build(&server, &db)?, - search: rooms::search::Service::build(&server, &db)?, - short: rooms::short::Service::build(&server, &db)?, - state: rooms::state::Service::build(&server, &db)?, - state_accessor: rooms::state_accessor::Service::build(&server, &db)?, - state_cache: rooms::state_cache::Service::build(&server, &db)?, - state_compressor: rooms::state_compressor::Service::build(&server, &db)?, - timeline: rooms::timeline::Service::build(&server, &db)?, - threads: rooms::threads::Service::build(&server, &db)?, - typing: rooms::typing::Service::build(&server, &db)?, - spaces: rooms::spaces::Service::build(&server, &db)?, - user: rooms::user::Service::build(&server, &db)?, + alias: build!(rooms::alias::Service), + auth_chain: build!(rooms::auth_chain::Service), + directory: build!(rooms::directory::Service), + event_handler: build!(rooms::event_handler::Service), + lazy_loading: build!(rooms::lazy_loading::Service), + metadata: build!(rooms::metadata::Service), + outlier: build!(rooms::outlier::Service), + pdu_metadata: build!(rooms::pdu_metadata::Service), + read_receipt: build!(rooms::read_receipt::Service), + search: build!(rooms::search::Service), + short: build!(rooms::short::Service), + state: build!(rooms::state::Service), + state_accessor: build!(rooms::state_accessor::Service), + state_cache: build!(rooms::state_cache::Service), + state_compressor: build!(rooms::state_compressor::Service), + timeline: build!(rooms::timeline::Service), + threads: build!(rooms::threads::Service), + typing: build!(rooms::typing::Service), + spaces: build!(rooms::spaces::Service), + user: build!(rooms::user::Service), }, - appservice: appservice::Service::build(&server, &db)?, - pusher: pusher::Service::build(&server, &db)?, - transaction_ids: transaction_ids::Service::build(&server, &db)?, - uiaa: uiaa::Service::build(&server, &db)?, - users: users::Service::build(&server, &db)?, - account_data: account_data::Service::build(&server, &db)?, - presence: presence::Service::build(&server, &db)?, - admin: admin::Service::build(&server, &db)?, - key_backups: key_backups::Service::build(&server, &db)?, - media: media::Service::build(&server, &db)?, - sending: sending::Service::build(&server, &db)?, - globals: globals::Service::build(&server, &db)?, + appservice: build!(appservice::Service), + pusher: build!(pusher::Service), + transaction_ids: build!(transaction_ids::Service), + uiaa: build!(uiaa::Service), + users: build!(users::Service), + account_data: build!(account_data::Service), + presence: build!(presence::Service), + admin: build!(admin::Service), + key_backups: build!(key_backups::Service), + media: build!(media::Service), + sending: build!(sending::Service), + globals: build!(globals::Service), + service, server, db, }) } - pub async fn memory_usage(&self) -> String { - let lazy_load_waiting = self.rooms.lazy_loading.lazy_load_waiting.lock().await.len(); - let server_visibility_cache = self - .rooms - .state_accessor - .server_visibility_cache - .lock() - .unwrap() - .len(); - let user_visibility_cache = self - .rooms - .state_accessor - .user_visibility_cache - .lock() - .unwrap() - .len(); - let stateinfo_cache = self - .rooms - .state_compressor - .stateinfo_cache - .lock() - .unwrap() - .len(); - let lasttimelinecount_cache = self - .rooms - .timeline - .get_lasttimelinecount_cache_usage().0; + pub async fn memory_usage(&self) -> Result { + let mut out = String::new(); + for service in self.service.values() { + service.memory_usage(&mut out)?; + } + + //TODO let roomid_spacehierarchy_cache = self .rooms .spaces @@ -103,113 +99,23 @@ impl Services { .lock() .await .len(); - let resolver_overrides_cache = self - .globals - .resolver - .overrides - .read() - .expect("locked for reading") - .len(); - let resolver_destinations_cache = self - .globals - .resolver - .destinations - .read() - .expect("locked for reading") - .len(); - let bad_event_ratelimiter = self.globals.bad_event_ratelimiter.read().await.len(); - let bad_query_ratelimiter = self.globals.bad_query_ratelimiter.read().await.len(); - let bad_signature_ratelimiter = self.globals.bad_signature_ratelimiter.read().await.len(); + writeln!(out, "roomid_spacehierarchy_cache: {roomid_spacehierarchy_cache}")?; - format!( - "\ -lazy_load_waiting: {lazy_load_waiting} -server_visibility_cache: {server_visibility_cache} -user_visibility_cache: {user_visibility_cache} -stateinfo_cache: {stateinfo_cache} -lasttimelinecount_cache: {lasttimelinecount_cache} -roomid_spacehierarchy_cache: {roomid_spacehierarchy_cache} -resolver_overrides_cache: {resolver_overrides_cache} -resolver_destinations_cache: {resolver_destinations_cache} -bad_event_ratelimiter: {bad_event_ratelimiter} -bad_query_ratelimiter: {bad_query_ratelimiter} -bad_signature_ratelimiter: {bad_signature_ratelimiter} -" - ) + Ok(out) } - pub async fn clear_caches(&self, amount: u32) { - if amount > 0 { - self.rooms - .lazy_loading - .lazy_load_waiting - .lock() - .await - .clear(); - } - if amount > 1 { - self.rooms - .state_accessor - .server_visibility_cache - .lock() - .unwrap() - .clear(); - } - if amount > 2 { - self.rooms - .state_accessor - .user_visibility_cache - .lock() - .unwrap() - .clear(); - } - if amount > 3 { - self.rooms - .state_compressor - .stateinfo_cache - .lock() - .unwrap() - .clear(); - } - if amount > 4 { - self.rooms - .timeline - .clear_lasttimelinecount_cache(); - } - if amount > 5 { - self.rooms - .spaces - .roomid_spacehierarchy_cache - .lock() - .await - .clear(); - } - if amount > 6 { - self.globals - .resolver - .overrides - .write() - .expect("locked for writing") - .clear(); - self.globals - .resolver - .destinations - .write() - .expect("locked for writing") - .clear(); - } - if amount > 7 { - self.globals.resolver.resolver.clear_cache(); - } - if amount > 8 { - self.globals.bad_event_ratelimiter.write().await.clear(); - } - if amount > 9 { - self.globals.bad_query_ratelimiter.write().await.clear(); - } - if amount > 10 { - self.globals.bad_signature_ratelimiter.write().await.clear(); + pub async fn clear_cache(&self) { + for service in self.service.values() { + service.clear_cache(); } + + //TODO + self.rooms + .spaces + .roomid_spacehierarchy_cache + .lock() + .await + .clear(); } pub async fn start(&self) -> Result<()> { @@ -219,10 +125,9 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} globals::migrations::migrations(&self.db, &self.globals.config).await?; globals::emerg_access::init_emergency_access(); - self.admin.start_handler().await; - self.sending.start_handler().await; - if self.globals.config.allow_local_presence { - self.presence.start_handler().await; + for (name, service) in &self.service { + debug!("Starting {name}"); + service.clone().start().await?; } if self.globals.allow_check_for_updates() { @@ -238,18 +143,18 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} Ok(()) } - pub async fn interrupt(&self) { - trace!("Interrupting services..."); - self.sending.interrupt(); - self.presence.interrupt(); - self.admin.interrupt(); + pub fn interrupt(&self) { + debug!("Interrupting services..."); - trace!("Services interrupt complete."); + for (name, service) in &self.service { + trace!("Interrupting {name}"); + service.interrupt(); + } } pub async fn stop(&self) { info!("Shutting down services"); - self.interrupt().await; + self.interrupt(); debug!("Waiting for update worker..."); if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { @@ -261,14 +166,10 @@ bad_signature_ratelimiter: {bad_signature_ratelimiter} } } - debug!("Waiting for admin worker..."); - self.admin.close().await; - - debug!("Waiting for presence worker..."); - self.presence.close().await; - - debug!("Waiting for sender..."); - self.sending.close().await; + for (name, service) in &self.service { + debug!("Waiting for {name} ..."); + service.stop().await; + } debug_info!("Services shutdown complete."); } diff --git a/src/service/transaction_ids/mod.rs b/src/service/transaction_ids/mod.rs index f18bbe64..78e6337f 100644 --- a/src/service/transaction_ids/mod.rs +++ b/src/service/transaction_ids/mod.rs @@ -2,22 +2,25 @@ mod data; use std::sync::Arc; -use conduit::{Result, Server}; +use conduit::Result; use data::Data; -use database::Database; use ruma::{DeviceId, TransactionId, UserId}; pub struct Service { pub db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub fn add_txnid( &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, data: &[u8], ) -> Result<()> { diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 0d2f8bf7..4b953ffb 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -2,9 +2,8 @@ mod data; use std::sync::Arc; -use conduit::{utils, utils::hash, Error, Result, Server}; +use conduit::{utils, utils::hash, Error, Result}; use data::Data; -use database::Database; use ruma::{ api::client::{ error::ErrorKind, @@ -22,13 +21,17 @@ pub struct Service { pub db: Data, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db), - }) +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db), + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Creates a new Uiaa session. Make sure the session token is unique. pub fn create( &self, user_id: &UserId, device_id: &DeviceId, uiaainfo: &UiaaInfo, json_body: &CanonicalJsonValue, diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 8444d538..cd580994 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -6,9 +6,8 @@ use std::{ sync::{Arc, Mutex, Mutex as StdMutex}, }; -use conduit::{Error, Result, Server}; +use conduit::{Error, Result}; use data::Data; -use database::Database; use ruma::{ api::client::{ device::Device, @@ -41,14 +40,18 @@ pub struct Service { pub connections: DbConnections, } -impl Service { - pub fn build(_server: &Arc, db: &Arc) -> Result { - Ok(Self { - db: Data::new(db.clone()), +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: Data::new(args.db.clone()), connections: StdMutex::new(BTreeMap::new()), - }) + })) } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { /// Check if a user has an account on this homeserver. pub fn exists(&self, user_id: &UserId) -> Result { self.db.exists(user_id) } From 08274150e513482f442ce5a84a5908bd47f3a6ab Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 09:44:52 +0000 Subject: [PATCH 0372/2091] cork database during global counter increment Signed-off-by: Jason Volk --- src/service/globals/data.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 43614854..7c430a81 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -57,6 +57,7 @@ impl Data { } pub fn next_count(&self) -> Result { + let _cork = self.db.cork(); let mut lock = self.counter.write().expect("locked"); let counter: &mut u64 = &mut lock; debug_assert!( From db46d6dd6b5db90b26302749df6ed99bc0404756 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 19:13:49 +0000 Subject: [PATCH 0373/2091] optimize millis_since_unix_epoch() Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 2 +- src/core/utils/time.rs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 75f55c9a..b34611df 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -23,7 +23,7 @@ pub use mutex_map::MutexMap; pub use rand::string as random_string; pub use string::{str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; -pub use time::millis_since_unix_epoch; +pub use time::now_millis as millis_since_unix_epoch; use crate::Result; diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index d7cab1fb..f6208b31 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,12 +1,12 @@ -use std::time::{SystemTime, UNIX_EPOCH}; - #[inline] #[must_use] #[allow(clippy::as_conversions)] -pub fn millis_since_unix_epoch() -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .expect("time is valid") +pub fn now_millis() -> u64 { + use std::time::UNIX_EPOCH; + + UNIX_EPOCH + .elapsed() + .expect("positive duration after epoch") .as_millis() as u64 } From 1e8b8cce0f6ac62f4dc00ed904b55497e99f79a4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 20:04:15 +0000 Subject: [PATCH 0374/2091] even more byte utils optimizations Signed-off-by: Jason Volk --- src/core/utils/bytes.rs | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/core/utils/bytes.rs b/src/core/utils/bytes.rs index 6ebf4c4f..e8975a49 100644 --- a/src/core/utils/bytes.rs +++ b/src/core/utils/bytes.rs @@ -3,27 +3,28 @@ use crate::Result; #[inline] #[must_use] pub fn increment(old: Option<&[u8]>) -> [u8; 8] { - old.map(TryInto::try_into) - .map_or(0_u64, |val| val.map_or(0_u64, u64::from_be_bytes)) + old.map_or(0_u64, u64_from_bytes_or_zero) .wrapping_add(1) .to_be_bytes() } -/// Parses the big-endian bytes into an u64. -#[inline] -pub fn u64_from_bytes(bytes: &[u8]) -> Result { - let array: [u8; 8] = bytes.try_into()?; - Ok(u64_from_u8x8(array)) -} - -/// Parses the 8 big-endian bytes into an u64. +/// Parses 8 big-endian bytes into an u64; panic on invalid argument #[inline] #[must_use] -pub fn u64_from_u8(bytes: &[u8]) -> u64 { - let bytes: &[u8; 8] = bytes.try_into().expect("must slice at least 8 bytes"); - u64_from_u8x8(*bytes) -} +pub fn u64_from_u8(bytes: &[u8]) -> u64 { u64_from_bytes(bytes).expect("must slice at least 8 bytes") } + +/// Parses the big-endian bytes into an u64. +#[inline] +#[must_use] +pub fn u64_from_bytes_or_zero(bytes: &[u8]) -> u64 { u64_from_bytes(bytes).unwrap_or(0) } + +/// Parses the big-endian bytes into an u64. +#[inline] +pub fn u64_from_bytes(bytes: &[u8]) -> Result { Ok(u64_from_u8x8(*u8x8_from_bytes(bytes)?)) } #[inline] #[must_use] pub fn u64_from_u8x8(bytes: [u8; 8]) -> u64 { u64::from_be_bytes(bytes) } + +#[inline] +pub fn u8x8_from_bytes(bytes: &[u8]) -> Result<&[u8; 8]> { Ok(bytes.try_into()?) } From eeda96d94a8138274d6ee344611e426ec3e3ec9f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 3 Jul 2024 20:06:43 +0000 Subject: [PATCH 0375/2091] inline analysis and symbol reduction; emits smaller than 64 bytes marked inline Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 2 +- src/core/error.rs | 1 + src/core/log/capture/guard.rs | 1 + src/core/log/capture/layer.rs | 2 + src/core/utils/hash.rs | 3 - src/core/utils/mod.rs | 1 - src/core/utils/string.rs | 1 - src/database/cork.rs | 1 + src/database/database.rs | 3 + src/database/engine.rs | 2 + src/database/handle.rs | 2 + src/database/map.rs | 1 + src/database/slice.rs | 9 +-- src/main/restart.rs | 1 + src/service/appservice/mod.rs | 4 ++ src/service/globals/data.rs | 15 ++++- src/service/globals/migrations.rs | 65 ++++++++++--------- src/service/globals/mod.rs | 24 +------ src/service/media/data.rs | 1 + src/service/presence/mod.rs | 2 + src/service/rooms/event_handler/mod.rs | 1 + .../rooms/event_handler/signing_keys.rs | 8 ++- src/service/rooms/metadata/data.rs | 4 ++ src/service/rooms/metadata/mod.rs | 7 +- src/service/rooms/state/data.rs | 1 + src/service/rooms/state/mod.rs | 2 + src/service/rooms/state_accessor/mod.rs | 3 + src/service/rooms/state_compressor/mod.rs | 1 + src/service/rooms/timeline/data.rs | 1 + src/service/rooms/timeline/mod.rs | 4 ++ src/service/sending/data.rs | 2 + src/service/sending/mod.rs | 4 +- src/service/sending/resolve.rs | 1 + src/service/users/data.rs | 2 + src/service/users/mod.rs | 8 +++ 35 files changed, 117 insertions(+), 73 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index c5b944bf..23197431 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -15,7 +15,7 @@ use ruma::{ events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, }; -use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve::resolve_actual_dest, services, PduEvent}; +use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve_actual_dest, services, PduEvent}; use tokio::sync::RwLock; use tracing_subscriber::EnvFilter; diff --git a/src/core/error.rs b/src/core/error.rs index ac2e176d..f977803d 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -92,6 +92,7 @@ impl Error { } /// Returns the Matrix error code / error kind + #[inline] pub fn error_code(&self) -> ruma::api::client::error::ErrorKind { if let Self::Federation(_, error) = self { return error.error_kind().unwrap_or_else(|| &Unknown).clone(); diff --git a/src/core/log/capture/guard.rs b/src/core/log/capture/guard.rs index a126fedc..b5a6d8b3 100644 --- a/src/core/log/capture/guard.rs +++ b/src/core/log/capture/guard.rs @@ -8,5 +8,6 @@ pub struct Guard { } impl Drop for Guard { + #[inline] fn drop(&mut self) { self.capture.stop(); } } diff --git a/src/core/log/capture/layer.rs b/src/core/log/capture/layer.rs index 19d76771..57f22dc7 100644 --- a/src/core/log/capture/layer.rs +++ b/src/core/log/capture/layer.rs @@ -17,6 +17,7 @@ struct Visitor { } impl Layer { + #[inline] pub fn new(state: &Arc) -> Self { Self { state: state.clone(), @@ -25,6 +26,7 @@ impl Layer { } impl fmt::Debug for Layer { + #[inline] fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { formatter.debug_struct("capture::Layer").finish() } diff --git a/src/core/utils/hash.rs b/src/core/utils/hash.rs index 9469321f..5a3664cb 100644 --- a/src/core/utils/hash.rs +++ b/src/core/utils/hash.rs @@ -3,14 +3,11 @@ mod sha256; use crate::Result; -#[inline] pub fn password(password: &str) -> Result { argon::password(password) } -#[inline] pub fn verify_password(password: &str, password_hash: &str) -> Result<()> { argon::verify_password(password, password_hash) } -#[inline] #[must_use] pub fn calculate_hash(keys: &[&[u8]]) -> Vec { sha256::hash(keys) } diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index b34611df..f9f1b87e 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -33,7 +33,6 @@ pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), /// /// * #[must_use] -#[inline(always)] pub fn unwrap_infallible(result: Result) -> T { match result { Ok(val) => val, diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 78765154..1a5b6b72 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -9,7 +9,6 @@ pub fn split_once_infallible<'a>(input: &'a str, delim: &'_ str) -> (&'a str, &' } /// Parses the bytes into a string. -#[inline] pub fn string_from_bytes(bytes: &[u8]) -> Result { let str: &str = str_from_bytes(bytes)?; Ok(str.to_owned()) diff --git a/src/database/cork.rs b/src/database/cork.rs index b7e1f60f..26c520a2 100644 --- a/src/database/cork.rs +++ b/src/database/cork.rs @@ -9,6 +9,7 @@ pub struct Cork { } impl Cork { + #[inline] pub(super) fn new(db: &Arc, flush: bool, sync: bool) -> Self { db.cork(); Self { diff --git a/src/database/database.rs b/src/database/database.rs index 2c6c6808..06a2d88f 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -19,12 +19,15 @@ impl Database { }) } + #[inline] #[must_use] pub fn cork(&self) -> Cork { Cork::new(&self.db, false, false) } + #[inline] #[must_use] pub fn cork_and_flush(&self) -> Cork { Cork::new(&self.db, true, false) } + #[inline] #[must_use] pub fn cork_and_sync(&self) -> Cork { Cork::new(&self.db, true, true) } } diff --git a/src/database/engine.rs b/src/database/engine.rs index fe16d29f..7c9522e1 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -121,6 +121,7 @@ impl Engine { pub fn sync(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, true)) } + #[inline] pub fn corked(&self) -> bool { self.corks.load(std::sync::atomic::Ordering::Relaxed) > 0 } pub(crate) fn cork(&self) { @@ -242,6 +243,7 @@ impl Engine { } impl Drop for Engine { + #[cold] fn drop(&mut self) { const BLOCKING: bool = true; diff --git a/src/database/handle.rs b/src/database/handle.rs index 787a5cea..0b45a75f 100644 --- a/src/database/handle.rs +++ b/src/database/handle.rs @@ -17,9 +17,11 @@ impl<'a> From> for Handle<'a> { impl Deref for Handle<'_> { type Target = [u8]; + #[inline] fn deref(&self) -> &Self::Target { &self.val } } impl AsRef<[u8]> for Handle<'_> { + #[inline] fn as_ref(&self) -> &[u8] { &self.val } } diff --git a/src/database/map.rs b/src/database/map.rs index 73cf5107..0b007307 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -199,6 +199,7 @@ impl<'a> IntoIterator for &'a Map { type IntoIter = Box + Send + 'a>; type Item = OwnedKeyValPair; + #[inline] fn into_iter(self) -> Self::IntoIter { self.iter() } } diff --git a/src/database/slice.rs b/src/database/slice.rs index 770f1957..448d969d 100644 --- a/src/database/slice.rs +++ b/src/database/slice.rs @@ -11,21 +11,19 @@ pub type Key = [Byte]; pub(crate) type Byte = u8; impl OwnedKeyVal { - #[inline] #[must_use] pub fn as_slice(&self) -> KeyVal<'_> { KeyVal(&self.0, &self.1) } - #[inline] #[must_use] pub fn to_tuple(self) -> OwnedKeyValPair { (self.0, self.1) } } impl From for OwnedKeyVal { - #[inline] fn from((key, val): OwnedKeyValPair) -> Self { Self(key, val) } } impl From<&KeyVal<'_>> for OwnedKeyVal { + #[inline] fn from(slice: &KeyVal<'_>) -> Self { slice.to_owned() } } @@ -34,7 +32,6 @@ impl From> for OwnedKeyVal { } impl From for OwnedKeyValPair { - #[inline] fn from(val: OwnedKeyVal) -> Self { val.to_tuple() } } @@ -43,22 +40,18 @@ impl KeyVal<'_> { #[must_use] pub fn to_owned(&self) -> OwnedKeyVal { OwnedKeyVal::from(self) } - #[inline] #[must_use] pub fn as_tuple(&self) -> KeyValPair<'_> { (self.0, self.1) } } impl<'a> From<&'a OwnedKeyVal> for KeyVal<'a> { - #[inline] fn from(owned: &'a OwnedKeyVal) -> Self { owned.as_slice() } } impl<'a> From<&'a OwnedKeyValPair> for KeyVal<'a> { - #[inline] fn from((key, val): &'a OwnedKeyValPair) -> Self { KeyVal(key.as_slice(), val.as_slice()) } } impl<'a> From> for KeyVal<'a> { - #[inline] fn from((key, val): KeyValPair<'a>) -> Self { KeyVal(key, val) } } diff --git a/src/main/restart.rs b/src/main/restart.rs index 00de9f21..009e5228 100644 --- a/src/main/restart.rs +++ b/src/main/restart.rs @@ -4,6 +4,7 @@ use std::{env, os::unix::process::CommandExt, process::Command}; use conduit::{debug, info, utils}; +#[cold] pub(super) fn restart() -> ! { // SAFETY: We have allowed an override for the case where the current_exe() has // been replaced or removed. By default the server will fail to restart if the diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 9bb00df1..f7cf0b6b 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -23,6 +23,7 @@ pub struct NamespaceRegex { impl NamespaceRegex { /// Checks if this namespace has rights to a namespace + #[inline] #[must_use] pub fn is_match(&self, heystack: &str) -> bool { if self.is_exclusive_match(heystack) { @@ -38,6 +39,7 @@ impl NamespaceRegex { } /// Checks if this namespace has exlusive rights to a namespace + #[inline] #[must_use] pub fn is_exclusive_match(&self, heystack: &str) -> bool { if let Some(exclusive) = &self.exclusive { @@ -55,6 +57,7 @@ impl RegistrationInfo { self.users.is_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() } + #[inline] #[must_use] pub fn is_exclusive_user_match(&self, user_id: &UserId) -> bool { self.users.is_exclusive_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() @@ -143,6 +146,7 @@ impl crate::Service for Service { } impl Service { + #[inline] pub fn all(&self) -> Result> { iter_ids(&self.db) } /// Registers an appservice and returns the ID to the caller diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 7c430a81..ecb827fe 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -99,6 +99,7 @@ impl Data { }) } + #[inline] pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { self.global .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes())?; @@ -207,8 +208,6 @@ impl Data { Ok(()) } - pub fn cleanup(&self) -> Result<()> { self.db.db.cleanup() } - pub fn load_keypair(&self) -> Result { let keypair_bytes = self.global.get(b"keypair")?.map_or_else( || { @@ -241,8 +240,16 @@ impl Data { }) } + #[inline] pub fn remove_keypair(&self) -> Result<()> { self.global.remove(b"keypair") } + /// TODO: the key valid until timestamp (`valid_until_ts`) is only honored + /// in room version > 4 + /// + /// Remove the outdated keys and insert the new ones. + /// + /// This doesn't actually check that the keys provided are newer than the + /// old set. pub fn add_signing_key( &self, origin: &ServerName, new_keys: ServerSigningKeys, ) -> Result> { @@ -306,14 +313,18 @@ impl Data { }) } + #[inline] pub fn bump_database_version(&self, new_version: u64) -> Result<()> { self.global.insert(b"version", &new_version.to_be_bytes())?; Ok(()) } + #[inline] pub fn backup(&self) -> Result<(), Box> { self.db.db.backup() } + #[inline] pub fn backup_list(&self) -> Result { self.db.db.backup_list() } + #[inline] pub fn file_list(&self) -> Result { self.db.db.file_list() } } diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 5cc25a1d..e171cb9d 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -50,7 +50,10 @@ pub(crate) async fn migrations(db: &Arc, config: &Config) -> Result<() } async fn fresh(db: &Arc, config: &Config) -> Result<()> { - services().globals.bump_database_version(DATABASE_VERSION)?; + services() + .globals + .db + .bump_database_version(DATABASE_VERSION)?; db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[])?; db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; @@ -68,57 +71,57 @@ async fn fresh(db: &Arc, config: &Config) -> Result<()> { /// Apply any migrations async fn migrate(db: &Arc, config: &Config) -> Result<()> { - if services().globals.database_version()? < 1 { + if services().globals.db.database_version()? < 1 { db_lt_1(db, config).await?; } - if services().globals.database_version()? < 2 { + if services().globals.db.database_version()? < 2 { db_lt_2(db, config).await?; } - if services().globals.database_version()? < 3 { + if services().globals.db.database_version()? < 3 { db_lt_3(db, config).await?; } - if services().globals.database_version()? < 4 { + if services().globals.db.database_version()? < 4 { db_lt_4(db, config).await?; } - if services().globals.database_version()? < 5 { + if services().globals.db.database_version()? < 5 { db_lt_5(db, config).await?; } - if services().globals.database_version()? < 6 { + if services().globals.db.database_version()? < 6 { db_lt_6(db, config).await?; } - if services().globals.database_version()? < 7 { + if services().globals.db.database_version()? < 7 { db_lt_7(db, config).await?; } - if services().globals.database_version()? < 8 { + if services().globals.db.database_version()? < 8 { db_lt_8(db, config).await?; } - if services().globals.database_version()? < 9 { + if services().globals.db.database_version()? < 9 { db_lt_9(db, config).await?; } - if services().globals.database_version()? < 10 { + if services().globals.db.database_version()? < 10 { db_lt_10(db, config).await?; } - if services().globals.database_version()? < 11 { + if services().globals.db.database_version()? < 11 { db_lt_11(db, config).await?; } - if services().globals.database_version()? < 12 { + if services().globals.db.database_version()? < 12 { db_lt_12(db, config).await?; } // This migration can be reused as-is anytime the server-default rules are // updated. - if services().globals.database_version()? < 13 { + if services().globals.db.database_version()? < 13 { db_lt_13(db, config).await?; } @@ -143,10 +146,10 @@ async fn migrate(db: &Arc, config: &Config) -> Result<()> { } assert_eq!( - services().globals.database_version().unwrap(), + services().globals.db.database_version().unwrap(), DATABASE_VERSION, "Failed asserting local database version {} is equal to known latest conduwuit database version {}", - services().globals.database_version().unwrap(), + services().globals.db.database_version().unwrap(), DATABASE_VERSION, ); @@ -225,7 +228,7 @@ async fn db_lt_1(db: &Arc, _config: &Config) -> Result<()> { serverroomids.insert(&serverroomid, &[])?; } - services().globals.bump_database_version(1)?; + services().globals.db.bump_database_version(1)?; info!("Migration: 0 -> 1 finished"); Ok(()) } @@ -242,7 +245,7 @@ async fn db_lt_2(db: &Arc, _config: &Config) -> Result<()> { } } - services().globals.bump_database_version(2)?; + services().globals.db.bump_database_version(2)?; info!("Migration: 1 -> 2 finished"); Ok(()) } @@ -262,7 +265,7 @@ async fn db_lt_3(db: &Arc, _config: &Config) -> Result<()> { mediaid_file.insert(&key, &[])?; } - services().globals.bump_database_version(3)?; + services().globals.db.bump_database_version(3)?; info!("Migration: 2 -> 3 finished"); Ok(()) } @@ -285,7 +288,7 @@ async fn db_lt_4(_db: &Arc, config: &Config) -> Result<()> { } } - services().globals.bump_database_version(4)?; + services().globals.db.bump_database_version(4)?; info!("Migration: 3 -> 4 finished"); Ok(()) } @@ -309,7 +312,7 @@ async fn db_lt_5(db: &Arc, _config: &Config) -> Result<()> { roomusertype_roomuserdataid.insert(&key, &roomuserdataid)?; } - services().globals.bump_database_version(5)?; + services().globals.db.bump_database_version(5)?; info!("Migration: 4 -> 5 finished"); Ok(()) } @@ -323,7 +326,7 @@ async fn db_lt_6(db: &Arc, _config: &Config) -> Result<()> { services().rooms.state_cache.update_joined_count(room_id)?; } - services().globals.bump_database_version(6)?; + services().globals.db.bump_database_version(6)?; info!("Migration: 5 -> 6 finished"); Ok(()) } @@ -448,7 +451,7 @@ async fn db_lt_7(db: &Arc, _config: &Config) -> Result<()> { )?; } - services().globals.bump_database_version(7)?; + services().globals.db.bump_database_version(7)?; info!("Migration: 6 -> 7 finished"); Ok(()) } @@ -514,7 +517,7 @@ async fn db_lt_8(db: &Arc, _config: &Config) -> Result<()> { eventid_pduid.insert_batch(batch2.iter().map(database::KeyVal::from))?; - services().globals.bump_database_version(8)?; + services().globals.db.bump_database_version(8)?; info!("Migration: 7 -> 8 finished"); Ok(()) } @@ -571,7 +574,7 @@ async fn db_lt_9(db: &Arc, _config: &Config) -> Result<()> { tokenids.remove(&key)?; } - services().globals.bump_database_version(9)?; + services().globals.db.bump_database_version(9)?; info!("Migration: 8 -> 9 finished"); Ok(()) } @@ -590,7 +593,7 @@ async fn db_lt_10(db: &Arc, _config: &Config) -> Result<()> { services().users.mark_device_key_update(&user_id)?; } - services().globals.bump_database_version(10)?; + services().globals.db.bump_database_version(10)?; info!("Migration: 9 -> 10 finished"); Ok(()) } @@ -601,7 +604,7 @@ async fn db_lt_11(_db: &Arc, _config: &Config) -> Result<()> { //let userdevicesessionid_uiaarequest = &db["userdevicesessionid_uiaarequest"]; //userdevicesessionid_uiaarequest.clear()?; - services().globals.bump_database_version(11)?; + services().globals.db.bump_database_version(11)?; info!("Migration: 10 -> 11 finished"); Ok(()) } @@ -669,7 +672,7 @@ async fn db_lt_12(_db: &Arc, config: &Config) -> Result<()> { )?; } - services().globals.bump_database_version(12)?; + services().globals.db.bump_database_version(12)?; info!("Migration: 11 -> 12 finished"); Ok(()) } @@ -706,7 +709,7 @@ async fn db_lt_13(_db: &Arc, config: &Config) -> Result<()> { )?; } - services().globals.bump_database_version(13)?; + services().globals.db.bump_database_version(13)?; info!("Migration: 12 -> 13 finished"); Ok(()) } @@ -736,8 +739,8 @@ async fn migrate_sha256_media(db: &Arc, _config: &Config) -> Result<() // Apply fix from when sha256_media was backward-incompat and bumped the schema // version from 13 to 14. For users satisfying these conditions we can go back. - if services().globals.database_version()? == 14 && DATABASE_VERSION == 13 { - services().globals.bump_database_version(13)?; + if services().globals.db.database_version()? == 14 && DATABASE_VERSION == 13 { + services().globals.db.bump_database_version(13)?; } db["global"].insert(b"feat_sha256_media", &[])?; diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 2a6fbc2f..12843b9d 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -17,10 +17,7 @@ use data::Data; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ - api::{ - client::discovery::discover_support::ContactRole, - federation::discovery::{ServerSigningKeys, VerifyKey}, - }, + api::{client::discovery::discover_support::ContactRole, federation::discovery::VerifyKey}, serde::Base64, DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, @@ -230,6 +227,7 @@ impl Service { pub fn allow_unstable_room_versions(&self) -> bool { self.config.allow_unstable_room_versions } + #[inline] pub fn default_room_version(&self) -> RoomVersionId { self.config.default_room_version.clone() } pub fn new_user_displayname_suffix(&self) -> &String { &self.config.new_user_displayname_suffix } @@ -317,19 +315,6 @@ impl Service { room_versions } - /// TODO: the key valid until timestamp (`valid_until_ts`) is only honored - /// in room version > 4 - /// - /// Remove the outdated keys and insert the new ones. - /// - /// This doesn't actually check that the keys provided are newer than the - /// old set. - pub fn add_signing_key( - &self, origin: &ServerName, new_keys: ServerSigningKeys, - ) -> Result> { - self.db.add_signing_key(origin, new_keys) - } - /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { @@ -348,14 +333,11 @@ impl Service { Ok(keys) } - pub fn database_version(&self) -> Result { self.db.database_version() } - - pub fn bump_database_version(&self, new_version: u64) -> Result<()> { self.db.bump_database_version(new_version) } - pub fn well_known_client(&self) -> &Option { &self.config.well_known.client } pub fn well_known_server(&self) -> &Option { &self.config.well_known.server } + #[inline] pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool { for cidr in &self.cidr_range_denylist { if cidr.includes(ip) { diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 4cabf167..186d502d 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -148,6 +148,7 @@ impl Data { /// associated with it such as width, height, content-type, etc) pub(crate) fn get_all_media_keys(&self) -> Vec> { self.mediaid_file.iter().map(|(key, _)| key).collect() } + #[inline] pub(super) fn remove_url_preview(&self, url: &str) -> Result<()> { self.url_previews.remove(url.as_bytes()) } pub(super) fn set_url_preview( diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 6f05d402..584f1a6d 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -131,6 +131,7 @@ impl crate::Service for Service { impl Service { /// Returns the latest presence event for the given user. + #[inline] pub fn get_presence(&self, user_id: &UserId) -> Result> { if let Some((_, presence)) = self.db.get_presence(user_id)? { Ok(Some(presence)) @@ -207,6 +208,7 @@ impl Service { /// Returns the most recent presence updates that happened after the event /// with id `since`. + #[inline] pub fn presence_since(&self, since: u64) -> Box)> + '_> { self.db.presence_since(since) } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index bb22c6c8..9f50ef58 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1381,6 +1381,7 @@ impl Service { Ok(create_event_content.room_version) } + #[inline] fn to_room_version(room_version_id: &RoomVersionId) -> RoomVersion { RoomVersion::new(room_version_id).expect("room version is supported") } diff --git a/src/service/rooms/event_handler/signing_keys.rs b/src/service/rooms/event_handler/signing_keys.rs index d22af9bd..fdb2be0e 100644 --- a/src/service/rooms/event_handler/signing_keys.rs +++ b/src/service/rooms/event_handler/signing_keys.rs @@ -201,6 +201,7 @@ impl super::Service { let result = services() .globals + .db .add_signing_key(&k.server_name, k.clone())? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) @@ -249,6 +250,7 @@ impl super::Service { if let Ok(key) = get_keys_response.server_key.deserialize() { let result: BTreeMap<_, _> = services() .globals + .db .add_signing_key(&origin, key)? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) @@ -392,7 +394,7 @@ impl super::Service { }) { debug!("Got signing keys: {:?}", server_keys); for k in server_keys { - services().globals.add_signing_key(origin, k.clone())?; + services().globals.db.add_signing_key(origin, k.clone())?; result.extend( k.verify_keys .into_iter() @@ -421,6 +423,7 @@ impl super::Service { { services() .globals + .db .add_signing_key(origin, server_key.clone())?; result.extend( @@ -453,6 +456,7 @@ impl super::Service { { services() .globals + .db .add_signing_key(origin, server_key.clone())?; result.extend( @@ -499,7 +503,7 @@ impl super::Service { }) { debug!("Got signing keys: {:?}", server_keys); for k in server_keys { - services().globals.add_signing_key(origin, k.clone())?; + services().globals.db.add_signing_key(origin, k.clone())?; result.extend( k.verify_keys .into_iter() diff --git a/src/service/rooms/metadata/data.rs b/src/service/rooms/metadata/data.rs index df9b6b10..763dd0e8 100644 --- a/src/service/rooms/metadata/data.rs +++ b/src/service/rooms/metadata/data.rs @@ -48,10 +48,12 @@ impl Data { })) } + #[inline] pub(super) fn is_disabled(&self, room_id: &RoomId) -> Result { Ok(self.disabledroomids.get(room_id.as_bytes())?.is_some()) } + #[inline] pub(super) fn disable_room(&self, room_id: &RoomId, disabled: bool) -> Result<()> { if disabled { self.disabledroomids.insert(room_id.as_bytes(), &[])?; @@ -62,10 +64,12 @@ impl Data { Ok(()) } + #[inline] pub(super) fn is_banned(&self, room_id: &RoomId) -> Result { Ok(self.bannedroomids.get(room_id.as_bytes())?.is_some()) } + #[inline] pub(super) fn ban_room(&self, room_id: &RoomId, banned: bool) -> Result<()> { if banned { self.bannedroomids.insert(room_id.as_bytes(), &[])?; diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 3a96b89f..ec34a82c 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -22,22 +22,27 @@ impl crate::Service for Service { impl Service { /// Checks if a room exists. - #[tracing::instrument(skip(self))] + #[inline] pub fn exists(&self, room_id: &RoomId) -> Result { self.db.exists(room_id) } #[must_use] pub fn iter_ids<'a>(&'a self) -> Box> + 'a> { self.db.iter_ids() } + #[inline] pub fn is_disabled(&self, room_id: &RoomId) -> Result { self.db.is_disabled(room_id) } + #[inline] pub fn disable_room(&self, room_id: &RoomId, disabled: bool) -> Result<()> { self.db.disable_room(room_id, disabled) } + #[inline] pub fn is_banned(&self, room_id: &RoomId) -> Result { self.db.is_banned(room_id) } + #[inline] pub fn ban_room(&self, room_id: &RoomId, banned: bool) -> Result<()> { self.db.ban_room(room_id, banned) } + #[inline] #[must_use] pub fn list_banned_rooms<'a>(&'a self) -> Box> + 'a> { self.db.list_banned_rooms() diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index dad613d2..aad3bede 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -30,6 +30,7 @@ impl Data { }) } + #[inline] pub(super) fn set_room_state( &self, room_id: &RoomId, diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index bf3b29b7..7964a5fb 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -200,6 +200,7 @@ impl Service { if let Some(state_key) = &new_pdu.state_key { let states_parents = previous_shortstatehash.map_or_else( || Ok(Vec::new()), + #[inline] |p| { services() .rooms @@ -344,6 +345,7 @@ impl Service { Ok(create_event_content.room_version) } + #[inline] pub fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result> { self.db.get_room_shortstatehash(room_id) } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index ac127b35..7fe61134 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -96,6 +96,7 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). + #[inline] pub fn state_get( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, ) -> Result>> { @@ -113,6 +114,7 @@ impl Service { } /// The user was a joined member at this state (potentially in the past) + #[inline] fn user_was_joined(&self, shortstatehash: u64, user_id: &UserId) -> bool { self.user_membership(shortstatehash, user_id) .is_ok_and(|s| s == MembershipState::Join) @@ -122,6 +124,7 @@ impl Service { /// The user was an invited or joined room member at this state (potentially /// in the past) + #[inline] fn user_was_invited(&self, shortstatehash: u64, user_id: &UserId) -> bool { self.user_membership(shortstatehash, user_id) .is_ok_and(|s| s == MembershipState::Join || s == MembershipState::Invite) diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 564a0bee..97f3fb80 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -135,6 +135,7 @@ impl Service { } /// Returns shortstatekey, event id + #[inline] pub fn parse_compressed_state_event(&self, compressed_event: &CompressedStateEvent) -> Result<(u64, Arc)> { Ok(( utils::u64_from_bytes(&compressed_event[0..size_of::()]).expect("bytes have right length"), diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 2054caf7..0d4d945e 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -97,6 +97,7 @@ impl Data { } /// Returns the pdu's id. + #[inline] pub(super) fn get_pdu_id(&self, event_id: &EventId) -> Result>> { self.eventid_pduid.get(event_id.as_bytes()) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index f7e8384b..ba987dbd 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -175,11 +175,13 @@ impl Service { } /// Returns the json of a pdu. + #[inline] pub fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result> { self.db.get_non_outlier_pdu_json(event_id) } /// Returns the pdu's id. + #[inline] pub fn get_pdu_id(&self, event_id: &EventId) -> Result>> { self.db.get_pdu_id(event_id) } @@ -190,6 +192,7 @@ impl Service { /// /// TODO: use this? #[allow(dead_code)] + #[inline] pub fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { self.db.get_non_outlier_pdu(event_id) } @@ -1017,6 +1020,7 @@ impl Service { } /// Returns an iterator over all PDUs in a room. + #[inline] pub fn all_pdus<'a>( &'a self, user_id: &UserId, room_id: &RoomId, ) -> Result> + 'a> { diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 22b69818..ed579817 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -27,6 +27,7 @@ impl Data { } } + #[inline] pub fn active_requests(&self) -> OutgoingSendingIter<'_> { Box::new( self.servercurrentevent_data @@ -35,6 +36,7 @@ impl Data { ) } + #[inline] pub fn active_requests_for<'a>(&'a self, destination: &Destination) -> SendingEventIter<'a> { let prefix = destination.get_prefix(); Box::new( diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index b75ca8b4..d3f3cbdd 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -1,6 +1,6 @@ mod appservice; mod data; -pub mod resolve; +mod resolve; mod send; mod sender; @@ -9,7 +9,7 @@ use std::{fmt::Debug, sync::Arc}; use async_trait::async_trait; use conduit::{Error, Result}; use data::Data; -pub use resolve::FedDest; +pub use resolve::{resolve_actual_dest, FedDest}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 51e7a92f..5f257217 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -414,6 +414,7 @@ impl FedDest { } } + #[inline] fn port(&self) -> Option { match &self { Self::Literal(addr) => Some(addr.port()), diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 14aa8006..3ba54a93 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -61,6 +61,7 @@ impl Data { } /// Check if a user has an account on this homeserver. + #[inline] pub(super) fn exists(&self, user_id: &UserId) -> Result { Ok(self.userid_password.get(user_id.as_bytes())?.is_some()) } @@ -75,6 +76,7 @@ impl Data { } /// Returns the number of users registered on this server. + #[inline] pub(super) fn count(&self) -> Result { Ok(self.userid_password.iter().count()) } /// Find out which user an access token belongs to. diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index cd580994..0196e1aa 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -53,6 +53,7 @@ impl crate::Service for Service { impl Service { /// Check if a user has an account on this homeserver. + #[inline] pub fn exists(&self, user_id: &UserId) -> Result { self.db.exists(user_id) } pub fn forget_sync_request_connection(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) { @@ -257,12 +258,14 @@ impl Service { } /// Create a new user account on this homeserver. + #[inline] pub fn create(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { self.db.set_password(user_id, password)?; Ok(()) } /// Returns the number of users registered on this server. + #[inline] pub fn count(&self) -> Result { self.db.count() } /// Find out which user an access token belongs to. @@ -283,6 +286,7 @@ impl Service { pub fn password_hash(&self, user_id: &UserId) -> Result> { self.db.password_hash(user_id) } /// Hash and set the user's password to the Argon2 hash + #[inline] pub fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { self.db.set_password(user_id, password) } @@ -331,6 +335,7 @@ impl Service { } /// Replaces the access token of one device. + #[inline] pub fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> { self.db.set_token(user_id, device_id, token) } @@ -385,18 +390,21 @@ impl Service { self.db.keys_changed(user_or_room_id, from, to) } + #[inline] pub fn mark_device_key_update(&self, user_id: &UserId) -> Result<()> { self.db.mark_device_key_update(user_id) } pub fn get_device_keys(&self, user_id: &UserId, device_id: &DeviceId) -> Result>> { self.db.get_device_keys(user_id, device_id) } + #[inline] pub fn parse_master_key( &self, user_id: &UserId, master_key: &Raw, ) -> Result<(Vec, CrossSigningKey)> { Data::parse_master_key(user_id, master_key) } + #[inline] pub fn get_key( &self, key: &[u8], sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, ) -> Result>> { From eaf1cf38a52caa6e82d5beb9539fa35d8b818a3e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 11:44:39 +0000 Subject: [PATCH 0376/2091] refactor resolver tuples into structs Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 2 +- src/service/globals/resolver.rs | 22 ++++---- src/service/sending/mod.rs | 2 +- src/service/sending/resolve.rs | 93 ++++++++++++++++++++++++--------- src/service/sending/send.rs | 21 ++++---- 5 files changed, 93 insertions(+), 47 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 12843b9d..3c0e66c0 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -2,7 +2,7 @@ mod client; mod data; pub(super) mod emerg_access; pub(super) mod migrations; -mod resolver; +pub(crate) mod resolver; pub(super) mod updates; use std::{ diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index e5199162..0cfaa1b9 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -12,21 +12,21 @@ use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use ruma::OwnedServerName; -use crate::sending::FedDest; +use crate::sending::{CachedDest, CachedOverride}; -type WellKnownMap = HashMap; -type TlsNameMap = HashMap, u16)>; +type WellKnownMap = HashMap; +type TlsNameMap = HashMap; pub struct Resolver { pub destinations: Arc>, // actual_destination, host pub overrides: Arc>, - pub resolver: Arc, - pub hooked: Arc, + pub(crate) resolver: Arc, + pub(crate) hooked: Arc, } -pub struct Hooked { - pub overrides: Arc>, - pub resolver: Arc, +pub(crate) struct Hooked { + overrides: Arc>, + resolver: Arc, } impl Resolver { @@ -117,15 +117,15 @@ impl Resolve for Resolver { impl Resolve for Hooked { fn resolve(&self, name: Name) -> Resolving { - let addr_port = self + let cached = self .overrides .read() .expect("locked for reading") .get(name.as_str()) .cloned(); - if let Some((addr, port)) = addr_port { - cached_to_reqwest(&addr, port) + if let Some(cached) = cached { + cached_to_reqwest(&cached.ips, cached.port) } else { resolve_to_reqwest(self.resolver.clone(), name) } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index d3f3cbdd..0dcebcec 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -9,7 +9,7 @@ use std::{fmt::Debug, sync::Arc}; use async_trait::async_trait; use conduit::{Error, Result}; use data::Data; -pub use resolve::{resolve_actual_dest, FedDest}; +pub use resolve::{resolve_actual_dest, CachedDest, CachedOverride, FedDest}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 5f257217..01943cc0 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -6,7 +6,7 @@ use std::{ use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; -use ruma::ServerName; +use ruma::{OwnedServerName, ServerName}; use tracing::{debug, error, trace}; use crate::{debug_error, debug_info, debug_warn, services, Error, Result}; @@ -35,6 +35,7 @@ pub enum FedDest { Named(String, String), } +#[derive(Clone, Debug)] pub(crate) struct ActualDest { pub(crate) dest: FedDest, pub(crate) host: String, @@ -42,19 +43,31 @@ pub(crate) struct ActualDest { pub(crate) cached: bool, } +#[derive(Clone, Debug)] +pub struct CachedDest { + pub dest: FedDest, + pub host: String, +} + +#[derive(Clone, Debug)] +pub struct CachedOverride { + pub ips: Vec, + pub port: u16, +} + #[tracing::instrument(skip_all, name = "resolve")] pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result { let cached; let cached_result = services() .globals .resolver - .destinations - .read() - .expect("locked for reading") - .get(server_name) - .cloned(); + .get_cached_destination(server_name); - let (dest, host) = if let Some(result) = cached_result { + let CachedDest { + dest, + host, + .. + } = if let Some(result) = cached_result { cached = true; result } else { @@ -77,7 +90,7 @@ pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result Result<(FedDest, String)> { +pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result { trace!("Finding actual destination for {dest}"); let mut host = dest.as_str().to_owned(); let actual_dest = match get_ip_with_port(dest.as_str()) { @@ -109,7 +122,10 @@ pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result<(FedD }; debug!("Actual destination: {actual_dest:?} hostname: {host:?}"); - Ok((actual_dest, host.into_uri_string())) + Ok(CachedDest { + dest: actual_dest, + host: host.into_uri_string(), + }) } fn actual_dest_1(host_port: FedDest) -> Result { @@ -193,14 +209,7 @@ async fn actual_dest_5(dest: &ServerName, cache: bool) -> Result { #[tracing::instrument(skip_all, name = "well-known")] async fn request_well_known(dest: &str) -> Result> { trace!("Requesting well known for {dest}"); - if !services() - .globals - .resolver - .overrides - .read() - .unwrap() - .contains_key(dest) - { + if !services().globals.resolver.has_cached_override(dest) { query_and_cache_override(dest, dest, 8448).await?; } @@ -269,15 +278,16 @@ async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u1 Err(e) => handle_resolve_error(&e), Ok(override_ip) => { if hostname != overname { - debug_info!("{:?} overriden by {:?}", overname, hostname); + debug_info!("{overname:?} overriden by {hostname:?}"); } - services() - .globals - .resolver - .overrides - .write() - .unwrap() - .insert(overname.to_owned(), (override_ip.iter().collect(), port)); + + services().globals.resolver.set_cached_override( + overname.to_owned(), + CachedOverride { + ips: override_ip.iter().collect(), + port, + }, + ); Ok(()) }, @@ -392,6 +402,39 @@ fn add_port_to_hostname(dest_str: &str) -> FedDest { FedDest::Named(host.to_owned(), port.to_owned()) } +impl crate::globals::resolver::Resolver { + pub(crate) fn set_cached_destination(&self, name: OwnedServerName, dest: CachedDest) -> Option { + trace!(?name, ?dest, "set cached destination"); + self.destinations + .write() + .expect("locked for writing") + .insert(name, dest) + } + + pub(crate) fn get_cached_destination(&self, name: &ServerName) -> Option { + self.destinations + .read() + .expect("locked for reading") + .get(name) + .cloned() + } + + pub(crate) fn set_cached_override(&self, name: String, over: CachedOverride) -> Option { + trace!(?name, ?over, "set cached override"); + self.overrides + .write() + .expect("locked for writing") + .insert(name, over) + } + + pub(crate) fn has_cached_override(&self, name: &str) -> bool { + self.overrides + .read() + .expect("locked for reading") + .contains_key(name) + } +} + impl FedDest { fn into_https_string(self) -> String { match self { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 57ff9127..1b977a73 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -8,11 +8,14 @@ use ruma::{ client::error::Error as RumaError, EndpointError, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, }, - OwnedServerName, ServerName, + ServerName, }; use tracing::{debug, trace}; -use super::{resolve, resolve::ActualDest}; +use super::{ + resolve, + resolve::{ActualDest, CachedDest}, +}; use crate::{debug_error, debug_warn, services, Error, Result}; #[tracing::instrument(skip_all, name = "send")] @@ -103,13 +106,13 @@ where let response = T::IncomingResponse::try_from_http_response(http_response); if response.is_ok() && !actual.cached { - services() - .globals - .resolver - .destinations - .write() - .expect("locked for writing") - .insert(OwnedServerName::from(dest), (actual.dest.clone(), actual.host.clone())); + services().globals.resolver.set_cached_destination( + dest.to_owned(), + CachedDest { + dest: actual.dest.clone(), + host: actual.host.clone(), + }, + ); } match response { From dc18f89c0bd851aa861b59cdf78342176b31f426 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 11:46:30 +0000 Subject: [PATCH 0377/2091] don't cache server name lookups indefinitely (#436) Signed-off-by: Jason Volk --- src/service/globals/resolver.rs | 1 + src/service/sending/resolve.rs | 30 ++++++++++++++++++++++++++++-- src/service/sending/send.rs | 1 + 3 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index 0cfaa1b9..3082f2fd 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -122,6 +122,7 @@ impl Resolve for Hooked { .read() .expect("locked for reading") .get(name.as_str()) + .filter(|cached| cached.valid()) .cloned(); if let Some(cached) = cached { diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 01943cc0..77311006 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -2,6 +2,7 @@ use std::{ fmt, fmt::Debug, net::{IpAddr, SocketAddr}, + time::SystemTime, }; use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; @@ -9,7 +10,7 @@ use ipaddress::IPAddress; use ruma::{OwnedServerName, ServerName}; use tracing::{debug, error, trace}; -use crate::{debug_error, debug_info, debug_warn, services, Error, Result}; +use crate::{debug_error, debug_info, debug_warn, services, utils::rand, Error, Result}; /// Wraps either an literal IP address plus port, or a hostname plus complement /// (colon-plus-port if it was specified). @@ -47,12 +48,14 @@ pub(crate) struct ActualDest { pub struct CachedDest { pub dest: FedDest, pub host: String, + pub expire: SystemTime, } #[derive(Clone, Debug)] pub struct CachedOverride { pub ips: Vec, pub port: u16, + pub expire: SystemTime, } #[tracing::instrument(skip_all, name = "resolve")] @@ -125,6 +128,7 @@ pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result bool { self.expire > SystemTime::now() } + + #[must_use] + pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } +} + +impl CachedOverride { + #[inline] + #[must_use] + pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + + #[must_use] + pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } +} + impl FedDest { fn into_https_string(self) -> String { match self { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 1b977a73..a89ea2f8 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -111,6 +111,7 @@ where CachedDest { dest: actual.dest.clone(), host: actual.host.clone(), + expire: CachedDest::default_expire(), }, ); } From 2dd68d3fa5fa48cb19864aa0f35bf825cf28d96e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 12:58:53 +0000 Subject: [PATCH 0378/2091] add time format string util Signed-off-by: Jason Volk --- Cargo.toml | 2 +- src/core/utils/time.rs | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index a52b4138..44f43849 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -208,7 +208,7 @@ features = ["serde"] # standard date and time tools [workspace.dependencies.chrono] version = "0.4.38" -features = ["alloc"] +features = ["alloc", "std"] default-features = false [workspace.dependencies.hyper] diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index f6208b31..7de00e9e 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,9 +1,9 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + #[inline] #[must_use] #[allow(clippy::as_conversions)] pub fn now_millis() -> u64 { - use std::time::UNIX_EPOCH; - UNIX_EPOCH .elapsed() .expect("positive duration after epoch") @@ -18,3 +18,11 @@ pub fn rfc2822_from_seconds(epoch: i64) -> String { .unwrap_or_default() .to_rfc2822() } + +#[must_use] +pub fn format(ts: SystemTime, str: &str) -> String { + use chrono::{DateTime, Utc}; + + let dt: DateTime = ts.into(); + dt.format(str).to_string() +} From 17a3ed4c560c0890a20776769ea1758acb6b1920 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 11:49:13 +0000 Subject: [PATCH 0379/2091] add admin query resolver commands Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 15 ++++--- src/admin/query/mod.rs | 27 ++++++++++-- src/admin/query/resolver.rs | 87 +++++++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 10 deletions(-) create mode 100644 src/admin/query/resolver.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 23197431..3efe283b 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -614,15 +614,16 @@ pub(super) async fn resolve_true_destination( let state = &services().server.log.capture; let logs = Arc::new(Mutex::new(String::new())); let capture = Capture::new(state, Some(filter), capture::fmt_markdown(logs.clone())); - let (actual_dest, hostname_uri); - { - let _capture_scope = capture.start(); - (actual_dest, hostname_uri) = resolve_actual_dest(&server_name, !no_cache).await?; - }; + + let capture_scope = capture.start(); + let actual = resolve_actual_dest(&server_name, !no_cache).await?; + drop(capture_scope); let msg = format!( - "{}\nDestination: {actual_dest}\nHostname URI: {hostname_uri}", - logs.lock().expect("locked") + "{}\nDestination: {}\nHostname URI: {}", + logs.lock().expect("locked"), + actual.dest, + actual.host, ); Ok(RoomMessageEventContent::text_markdown(msg)) } diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index 946e6ec8..ea7036d0 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -2,6 +2,7 @@ mod account_data; mod appservice; mod globals; mod presence; +mod resolver; mod room_alias; mod room_state_cache; mod sending; @@ -12,12 +13,12 @@ use conduit::Result; use room_state_cache::room_state_cache; use ruma::{ events::{room::message::RoomMessageEventContent, RoomAccountDataEventType}, - RoomAliasId, RoomId, ServerName, UserId, + OwnedServerName, RoomAliasId, RoomId, ServerName, UserId, }; use self::{ - account_data::account_data, appservice::appservice, globals::globals, presence::presence, room_alias::room_alias, - sending::sending, users::users, + account_data::account_data, appservice::appservice, globals::globals, presence::presence, resolver::resolver, + room_alias::room_alias, sending::sending, users::users, }; #[cfg_attr(test, derive(Debug))] @@ -55,6 +56,10 @@ pub(super) enum QueryCommand { /// - users.rs iterators and getters #[command(subcommand)] Users(Users), + + /// - resolver service + #[command(subcommand)] + Resolver(Resolver), } #[cfg_attr(test, derive(Debug))] @@ -287,6 +292,21 @@ pub(super) enum Users { Iter, } +#[cfg_attr(test, derive(Debug))] +#[derive(Subcommand)] +/// Resolver service and caches +pub(super) enum Resolver { + /// Query the destinations cache + DestinationsCache { + server_name: Option, + }, + + /// Query the overrides cache + OverridesCache { + name: Option, + }, +} + /// Processes admin query commands pub(super) async fn process(command: QueryCommand, _body: Vec<&str>) -> Result { Ok(match command { @@ -298,5 +318,6 @@ pub(super) async fn process(command: QueryCommand, _body: Vec<&str>) -> Result globals(command).await?, QueryCommand::Sending(command) => sending(command).await?, QueryCommand::Users(command) => users(command).await?, + QueryCommand::Resolver(command) => resolver(command).await?, }) } diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs new file mode 100644 index 00000000..2a2554b5 --- /dev/null +++ b/src/admin/query/resolver.rs @@ -0,0 +1,87 @@ +use std::fmt::Write; + +use conduit::{utils::time, Result}; +use ruma::{events::room::message::RoomMessageEventContent, OwnedServerName}; + +use super::Resolver; +use crate::services; + +/// All the getters and iterators in key_value/users.rs +pub(super) async fn resolver(subcommand: Resolver) -> Result { + match subcommand { + Resolver::DestinationsCache { + server_name, + } => destinations_cache(server_name).await, + Resolver::OverridesCache { + name, + } => overrides_cache(name).await, + } +} + +async fn destinations_cache(server_name: Option) -> Result { + use service::sending::CachedDest; + + let mut out = String::new(); + writeln!(out, "| Server Name | Destination | Hostname | Expires |")?; + writeln!(out, "| ----------- | ----------- | -------- | ------- |")?; + let row = |( + name, + &CachedDest { + ref dest, + ref host, + expire, + }, + )| { + let expire = time::format(expire, "%+"); + writeln!(out, "| {name} | {dest} | {host} | {expire} |").expect("wrote line"); + }; + + let map = services() + .globals + .resolver + .destinations + .read() + .expect("locked"); + + if let Some(server_name) = server_name.as_ref() { + map.get_key_value(server_name).map(row); + } else { + map.iter().for_each(row); + } + + Ok(RoomMessageEventContent::notice_markdown(out)) +} + +async fn overrides_cache(server_name: Option) -> Result { + use service::sending::CachedOverride; + + let mut out = String::new(); + writeln!(out, "| Server Name | IP | Port | Expires |")?; + writeln!(out, "| ----------- | --- | ----:| ------- |")?; + let row = |( + name, + &CachedOverride { + ref ips, + port, + expire, + }, + )| { + let expire = time::format(expire, "%+"); + writeln!(out, "| {name} | {ips:?} | {port} | {expire} |").expect("wrote line"); + }; + + let map = services() + .globals + .resolver + .overrides + .read() + .expect("locked"); + + if let Some(server_name) = server_name.as_ref() { + map.get_key_value(server_name).map(row); + } else { + map.iter().for_each(row); + } + + Ok(RoomMessageEventContent::notice_markdown(out)) +} From 8691141237c459ffe66ec13083d22757c318d302 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 02:48:36 -0400 Subject: [PATCH 0380/2091] ci: try running `nix-build-and-cache` prior dynamic build test Signed-off-by: strawberry --- engage.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engage.toml b/engage.toml index 365c0179..d28ecff1 100644 --- a/engage.toml +++ b/engage.toml @@ -184,5 +184,6 @@ cargo test \ name = "nix-default" group = "tests" script = """ -nix run .#default -- --help +bin/nix-build-and-cache just .#default +nix run -L .#default -- --help """ From 2bc53139faed06653b05ab6ce3f94e9d89905fe5 Mon Sep 17 00:00:00 2001 From: Christoph Dittmann Date: Sat, 6 Jul 2024 14:08:14 +0200 Subject: [PATCH 0381/2091] Don't send empty presence EDUs I run a homeserver whose logs show a high number of incoming empty presence EDUs originating from the user agent "Conduwuit/0.4.4". They arrive at a rate of about 2 queries per second per Conduwuit server. The empty EDUs all look the same, only with `origin_server_ts` increasing: ``` {"origin":"example.com","origin_server_ts":1720266475601,"edus":[{"edu_type":"m.presence","content":{"push":[]}}]} ``` These updates are unnecessary because they don't do anything. They only increase network traffic and CPU usage on both sides. After this commit, the empty presence updates are no longer inserted into the outgoing event queue. --- src/service/sending/sender.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 61cb9f6d..54302fd5 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -324,8 +324,10 @@ fn select_edus_presence( } } - let presence_content = Edu::Presence(PresenceContent::new(presence_updates)); - events.push(serde_json::to_vec(&presence_content).expect("PresenceEvent can be serialized")); + if !presence_updates.is_empty() { + let presence_content = Edu::Presence(PresenceContent::new(presence_updates)); + events.push(serde_json::to_vec(&presence_content).expect("PresenceEvent can be serialized")); + } Ok(true) } From 04e3de08eba355557bfa7ace886e7f4231fb7a26 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 4 Jul 2024 23:53:30 +0000 Subject: [PATCH 0382/2091] add log to error functors for Result::map_or_else Signed-off-by: Jason Volk --- src/core/error.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/core/error.rs b/src/core/error.rs index f977803d..ea7acda5 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -19,7 +19,8 @@ use ruma::{ OwnedServerName, }; use thiserror::Error; -use tracing::error; + +use crate::{debug_error, error}; #[derive(Error)] pub enum Error { @@ -126,6 +127,18 @@ impl fmt::Debug for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } } +#[inline] +pub fn log(e: Error) { + error!("{e}"); + drop(e); +} + +#[inline] +pub fn debug_log(e: Error) { + debug_error!("{e}"); + drop(e); +} + #[derive(Clone)] pub struct RumaResponse(pub T); From 8350aced396e610f70d2cc7a1997e81a8b5407ee Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 01:27:13 +0000 Subject: [PATCH 0383/2091] add ctrl-\ quit support to admin console Signed-off-by: Jason Volk --- Cargo.lock | 3 +-- Cargo.toml | 3 ++- src/service/admin/console.rs | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f89c9f35..563b87ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3274,8 +3274,7 @@ checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "rustyline-async" version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6eb06391513b2184f0a5405c11a4a0a5302e8be442f4c5c35267187c2b37d5" +source = "git+https://github.com/girlbossceo/rustyline-async?rev=de26100b0db03e419a3d8e1dd26895d170d1fe50#de26100b0db03e419a3d8e1dd26895d170d1fe50" dependencies = [ "crossterm", "futures-channel", diff --git a/Cargo.toml b/Cargo.toml index 44f43849..166a359d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -404,7 +404,8 @@ features = [ ] [workspace.dependencies.rustyline-async] -version = "0.4.2" +git = "https://github.com/girlbossceo/rustyline-async" +rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50" [workspace.dependencies.termimad] version = "0.29.4" diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index c0562c52..3d96e078 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -97,6 +97,7 @@ impl Console { ReadlineEvent::Line(string) => self.clone().handle(string).await, ReadlineEvent::Interrupted => continue, ReadlineEvent::Eof => break, + ReadlineEvent::Quit => services().server.shutdown().unwrap_or_else(error::log), }, Err(error) => match error { ReadlineError::Closed => break, From 38a24e017089b859b68463cec721f12bd2214567 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 01:31:06 +0000 Subject: [PATCH 0384/2091] remove unused deps in member crates Signed-off-by: Jason Volk --- Cargo.lock | 1 - src/router/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 563b87ae..446a4518 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -711,7 +711,6 @@ dependencies = [ "axum-server", "axum-server-dual-protocol", "bytes", - "clap", "conduit_admin", "conduit_api", "conduit_core", diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 2fac3858..ecb09dd7 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -61,7 +61,6 @@ tokio.workspace = true tower.workspace = true tracing.workspace = true bytes.workspace = true -clap.workspace = true http-body-util.workspace = true http.workspace = true hyper.workspace = true From 0e580292a6fe4da501347fa7d51be3809759bbd6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 01:44:43 +0000 Subject: [PATCH 0385/2091] encap admin handler init/fini in crate Signed-off-by: Jason Volk --- src/admin/mod.rs | 13 ++++++++++++- src/router/run.rs | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index f2034724..7bb22418 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -17,7 +17,6 @@ extern crate conduit_core as conduit; extern crate conduit_service as service; pub(crate) use conduit::{mod_ctor, mod_dtor, Result}; -pub use handler::handle; pub(crate) use service::{services, user_is_local}; pub(crate) use crate::{ @@ -28,6 +27,18 @@ pub(crate) use crate::{ mod_ctor! {} mod_dtor! {} +/// Install the admin command handler +#[allow(clippy::let_underscore_must_use)] +pub async fn init() { + _ = services().admin.handle.lock().await.insert(handler::handle); +} + +/// Uninstall the admin command handler +#[allow(clippy::let_underscore_must_use)] +pub async fn fini() { + _ = services().admin.handle.lock().await.take(); +} + #[cfg(test)] mod test { use clap::Parser; diff --git a/src/router/run.rs b/src/router/run.rs index fb59c797..4f7853d8 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -22,7 +22,7 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { let app = layers::build(&server)?; // Install the admin room callback here for now - _ = services().admin.handle.lock().await.insert(admin::handle); + admin::init().await; // Setup shutdown/signal handling let handle = ServerHandle::new(); @@ -39,7 +39,7 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { _ = sigs.await; // Remove the admin room callback - _ = services().admin.handle.lock().await.take(); + admin::fini().await; debug_info!("Finished"); res From b31e81a469567a9064d9a96cfa8cf4507e2efb42 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 07:01:11 +0000 Subject: [PATCH 0386/2091] add common_prefix string util Signed-off-by: Jason Volk --- src/core/utils/string.rs | 19 +++++++++++++++++++ src/core/utils/tests.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 1a5b6b72..1f2a6572 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -2,6 +2,25 @@ use crate::Result; pub const EMPTY: &str = ""; +/// Find the common prefix from a collection of strings and return a slice +/// ``` +/// use conduit_core::utils::string::common_prefix; +/// let input = ["conduwuit", "conduit", "construct"]; +/// common_prefix(&input) == "con"; +/// ``` +#[must_use] +pub fn common_prefix<'a>(choice: &'a [&str]) -> &'a str { + choice.first().map_or(EMPTY, move |best| { + choice.iter().skip(1).fold(*best, |best, choice| { + &best[0..choice + .chars() + .zip(best.chars()) + .take_while(|&(a, b)| a == b) + .count()] + }) + }) +} + #[inline] #[must_use] pub fn split_once_infallible<'a>(input: &'a str, delim: &'_ str) -> (&'a str, &'a str) { diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index b226bd41..f5cd0a07 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -35,3 +35,30 @@ fn increment_wrap() { let res = u64::from_be_bytes(bytes); assert_eq!(res, 0); } + +#[test] +fn common_prefix() { + use utils::string; + + let input = ["conduwuit", "conduit", "construct"]; + let output = string::common_prefix(&input); + assert_eq!(output, "con"); +} + +#[test] +fn common_prefix_empty() { + use utils::string; + + let input = ["abcdefg", "hijklmn", "opqrstu"]; + let output = string::common_prefix(&input); + assert_eq!(output, ""); +} + +#[test] +fn common_prefix_none() { + use utils::string; + + let input = []; + let output = string::common_prefix(&input); + assert_eq!(output, ""); +} From 5254eb4f721079a9f6666b274b109bad618a6203 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 07:52:05 +0000 Subject: [PATCH 0387/2091] add basic tab completion to console Signed-off-by: Jason Volk --- src/admin/handler.rs | 150 +++++++++++++++++++++++------------ src/admin/mod.rs | 12 +++ src/service/admin/console.rs | 8 ++ src/service/admin/mod.rs | 16 +++- 4 files changed, 134 insertions(+), 52 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 0bd8af9f..1466c040 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,6 +1,6 @@ use std::time::Instant; -use clap::Parser; +use clap::{CommandFactory, Parser}; use conduit::trace; use ruma::events::{ relation::InReplyTo, @@ -9,7 +9,7 @@ use ruma::events::{ extern crate conduit_service as service; -use conduit::Result; +use conduit::{utils::string::common_prefix, Result}; pub(crate) use service::admin::{Command, Service}; use service::admin::{CommandOutput, CommandResult, HandlerResult}; @@ -62,7 +62,10 @@ pub(crate) enum AdminCommand { } #[must_use] -pub fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(command)) } +pub(crate) fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(command)) } + +#[must_use] +pub(crate) fn complete(line: &str) -> String { complete_admin_command(AdminCommand::command(), line) } #[tracing::instrument(skip_all, name = "admin")] async fn handle_command(command: Command) -> CommandResult { @@ -105,54 +108,6 @@ async fn process_admin_message(msg: String) -> CommandOutput { } } -// Parse chat messages from the admin room into an AdminCommand object -fn parse_admin_command(command_line: &str) -> Result { - let mut argv = command_line.split_whitespace().collect::>(); - - // Remove any escapes that came with a server-side escape command - if !argv.is_empty() && argv[0].ends_with("admin") { - argv[0] = argv[0].trim_start_matches('\\'); - } - - // First indice has to be "admin" but for console convenience we add it here - let server_user = services().globals.server_user.as_str(); - if !argv.is_empty() && !argv[0].ends_with("admin") && !argv[0].starts_with(server_user) { - argv.insert(0, "admin"); - } - - // Replace `help command` with `command --help` - // Clap has a help subcommand, but it omits the long help description. - if argv.len() > 1 && argv[1] == "help" { - argv.remove(1); - argv.push("--help"); - } - - // Backwards compatibility with `register_appservice`-style commands - let command_with_dashes_argv1; - if argv.len() > 1 && argv[1].contains('_') { - command_with_dashes_argv1 = argv[1].replace('_', "-"); - argv[1] = &command_with_dashes_argv1; - } - - // Backwards compatibility with `register_appservice`-style commands - let command_with_dashes_argv2; - if argv.len() > 2 && argv[2].contains('_') { - command_with_dashes_argv2 = argv[2].replace('_', "-"); - argv[2] = &command_with_dashes_argv2; - } - - // if the user is using the `query` command (argv[1]), replace the database - // function/table calls with underscores to match the codebase - let command_with_dashes_argv3; - if argv.len() > 3 && argv[1].eq("query") { - command_with_dashes_argv3 = argv[3].replace('_', "-"); - argv[3] = &command_with_dashes_argv3; - } - - trace!(?command_line, ?argv, "parse"); - AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) -} - #[tracing::instrument(skip_all, name = "command")] async fn process_admin_command(command: AdminCommand, body: Vec<&str>) -> Result { let reply_message_content = match command { @@ -169,3 +124,96 @@ async fn process_admin_command(command: AdminCommand, body: Vec<&str>) -> Result Ok(reply_message_content) } + +// Parse chat messages from the admin room into an AdminCommand object +fn parse_admin_command(command_line: &str) -> Result { + let argv = parse_command_line(command_line); + AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) +} + +fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { + let mut ret = Vec::::new(); + let argv = parse_command_line(line); + 'token: for token in argv.into_iter().skip(1) { + let mut choice = Vec::new(); + let cmd_ = cmd.clone(); + for sub in cmd_.get_subcommands() { + let name = sub.get_name(); + if *name == token { + // token already complete; recurse to subcommand + ret.push(token); + cmd.clone_from(sub); + continue 'token; + } + if name.starts_with(&token) { + // partial match; add to choices + choice.push(name); + } + } + + if choice.is_empty() { + // Nothing found, return original string + ret.push(token); + } else if choice.len() == 1 { + // One choice. Add extra space because it's complete + ret.push((*choice.first().expect("only choice")).to_owned()); + ret.push(String::new()); + } else { + // Find the common prefix + ret.push(common_prefix(&choice).into()); + } + + // Return from completion + return ret.join(" "); + } + + // Return from no completion. Needs a space though. + let mut ret = ret.join(" "); + ret.push(' '); + ret +} + +// Parse chat messages from the admin room into an AdminCommand object +fn parse_command_line(command_line: &str) -> Vec { + let mut argv = command_line + .split_whitespace() + .map(str::to_owned) + .collect::>(); + + // Remove any escapes that came with a server-side escape command + if !argv.is_empty() && argv[0].ends_with("admin") { + argv[0] = argv[0].trim_start_matches('\\').into(); + } + + // First indice has to be "admin" but for console convenience we add it here + let server_user = services().globals.server_user.as_str(); + if !argv.is_empty() && !argv[0].ends_with("admin") && !argv[0].starts_with(server_user) { + argv.insert(0, "admin".to_owned()); + } + + // Replace `help command` with `command --help` + // Clap has a help subcommand, but it omits the long help description. + if argv.len() > 1 && argv[1] == "help" { + argv.remove(1); + argv.push("--help".to_owned()); + } + + // Backwards compatibility with `register_appservice`-style commands + if argv.len() > 1 && argv[1].contains('_') { + argv[1] = argv[1].replace('_', "-"); + } + + // Backwards compatibility with `register_appservice`-style commands + if argv.len() > 2 && argv[2].contains('_') { + argv[2] = argv[2].replace('_', "-"); + } + + // if the user is using the `query` command (argv[1]), replace the database + // function/table calls with underscores to match the codebase + if argv.len() > 3 && argv[1].eq("query") { + argv[3] = argv[3].replace('_', "-"); + } + + trace!(?command_line, ?argv, "parse"); + argv +} diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 7bb22418..2de42420 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -30,6 +30,12 @@ mod_dtor! {} /// Install the admin command handler #[allow(clippy::let_underscore_must_use)] pub async fn init() { + _ = services() + .admin + .complete + .write() + .expect("locked for writing") + .insert(handler::complete); _ = services().admin.handle.lock().await.insert(handler::handle); } @@ -37,6 +43,12 @@ pub async fn init() { #[allow(clippy::let_underscore_must_use)] pub async fn fini() { _ = services().admin.handle.lock().await.take(); + _ = services() + .admin + .complete + .write() + .expect("locked for writing") + .take(); } #[cfg(test)] diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 3d96e078..0a200cae 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -118,6 +118,7 @@ impl Console { let _suppression = log::Suppress::new(&services().server); let (mut readline, _writer) = Readline::new(PROMPT.to_owned())?; + readline.set_tab_completer(Self::tab_complete); self.set_history(&mut readline); let future = readline.readline(); @@ -185,6 +186,13 @@ impl Console { history.push_front(line); history.truncate(HISTORY_LIMIT); } + + fn tab_complete(line: &str) -> String { + services() + .admin + .complete_command(line) + .unwrap_or_else(|| line.to_owned()) + } } fn configure_output(mut output: MadSkin) -> MadSkin { diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index ed2ac725..d01a6f55 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -2,7 +2,11 @@ pub mod console; mod create; mod grant; -use std::{future::Future, pin::Pin, sync::Arc}; +use std::{ + future::Future, + pin::Pin, + sync::{Arc, RwLock as StdRwLock}, +}; use async_trait::async_trait; use conduit::{error, utils::mutex_map, Error, Result}; @@ -27,12 +31,14 @@ pub type CommandOutput = Option; pub type CommandResult = Result; pub type HandlerResult = Pin + Send>>; pub type Handler = fn(Command) -> HandlerResult; +pub type Completer = fn(&str) -> String; pub struct Service { sender: Sender, receiver: Mutex>, handler_join: Mutex>>, pub handle: Mutex>, + pub complete: StdRwLock>, #[cfg(feature = "console")] pub console: Arc, } @@ -52,6 +58,7 @@ impl crate::Service for Service { receiver: Mutex::new(receiver), handler_join: Mutex::new(None), handle: Mutex::new(None), + complete: StdRwLock::new(None), #[cfg(feature = "console")] console: console::Console::new(), })) @@ -127,6 +134,13 @@ impl Service { .await } + pub fn complete_command(&self, command: &str) -> Option { + self.complete + .read() + .expect("locked for reading") + .map(|complete| complete(command)) + } + async fn send(&self, message: Command) { debug_assert!(!self.sender.is_closed(), "channel closed"); self.sender.send_async(message).await.expect("message sent"); From a3638dbb15fc9e320f24f6e2d4e3fab2aac08856 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 08:39:37 +0000 Subject: [PATCH 0388/2091] use rwlock for command handler. Signed-off-by: Jason Volk --- src/admin/mod.rs | 9 +++++++-- src/service/admin/mod.rs | 11 +++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 2de42420..f2e35d80 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -36,13 +36,18 @@ pub async fn init() { .write() .expect("locked for writing") .insert(handler::complete); - _ = services().admin.handle.lock().await.insert(handler::handle); + _ = services() + .admin + .handle + .write() + .await + .insert(handler::handle); } /// Uninstall the admin command handler #[allow(clippy::let_underscore_must_use)] pub async fn fini() { - _ = services().admin.handle.lock().await.take(); + _ = services().admin.handle.write().await.take(); _ = services() .admin .complete diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index d01a6f55..ca0e551b 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -21,7 +21,10 @@ use ruma::{ OwnedEventId, OwnedRoomId, RoomId, UserId, }; use serde_json::value::to_raw_value; -use tokio::{sync::Mutex, task::JoinHandle}; +use tokio::{ + sync::{Mutex, RwLock}, + task::JoinHandle, +}; use crate::{pdu::PduBuilder, services, user_is_local, PduEvent}; @@ -37,7 +40,7 @@ pub struct Service { sender: Sender, receiver: Mutex>, handler_join: Mutex>>, - pub handle: Mutex>, + pub handle: RwLock>, pub complete: StdRwLock>, #[cfg(feature = "console")] pub console: Arc, @@ -57,7 +60,7 @@ impl crate::Service for Service { sender, receiver: Mutex::new(receiver), handler_join: Mutex::new(None), - handle: Mutex::new(None), + handle: RwLock::new(None), complete: StdRwLock::new(None), #[cfg(feature = "console")] console: console::Console::new(), @@ -175,7 +178,7 @@ impl Service { } async fn process_command(&self, command: Command) -> CommandResult { - if let Some(handle) = self.handle.lock().await.as_ref() { + if let Some(handle) = self.handle.read().await.as_ref() { handle(command).await } else { Err(Error::Err("Admin module is not loaded.".into())) From 72d9e8ed2bd2f26ce27aef421d2e6a4be36500da Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 14:32:41 -0400 Subject: [PATCH 0389/2091] bump conduwuit version to 0.4.5 Signed-off-by: strawberry --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 446a4518..3415cac0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -570,7 +570,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" -version = "0.4.4" +version = "0.4.5" dependencies = [ "clap", "conduit_admin", @@ -598,7 +598,7 @@ dependencies = [ [[package]] name = "conduit_admin" -version = "0.4.4" +version = "0.4.5" dependencies = [ "clap", "conduit_api", @@ -620,7 +620,7 @@ dependencies = [ [[package]] name = "conduit_api" -version = "0.4.4" +version = "0.4.5" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -653,7 +653,7 @@ dependencies = [ [[package]] name = "conduit_core" -version = "0.4.4" +version = "0.4.5" dependencies = [ "argon2", "axum 0.7.5", @@ -692,7 +692,7 @@ dependencies = [ [[package]] name = "conduit_database" -version = "0.4.4" +version = "0.4.5" dependencies = [ "conduit_core", "log", @@ -704,7 +704,7 @@ dependencies = [ [[package]] name = "conduit_router" -version = "0.4.4" +version = "0.4.5" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -736,7 +736,7 @@ dependencies = [ [[package]] name = "conduit_service" -version = "0.4.4" +version = "0.4.5" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 166a359d..29454e60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" rust-version = "1.77.0" -version = "0.4.4" +version = "0.4.5" [workspace.metadata.crane] name = "conduit" From c1c084dda1c6b3801e81b53ceaa8bfa26a9e9245 Mon Sep 17 00:00:00 2001 From: AlexPewMaster <68469103+AlexPewMaster@users.noreply.github.com> Date: Mon, 8 Jul 2024 18:31:08 +0200 Subject: [PATCH 0390/2091] Introduce a new way of deploying conduwuit using caddy in Docker --- docs/deploying/docker-compose.with-caddy.yml | 54 ++++++++++++++++++++ docs/deploying/docker.md | 11 +++- 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 docs/deploying/docker-compose.with-caddy.yml diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml new file mode 100644 index 00000000..41d8856d --- /dev/null +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -0,0 +1,54 @@ +services: + caddy: + # This compose file uses caddy-docker-proxy as the reverse proxy for conduwuit! + # For more info, visit https://github.com/lucaslorentz/caddy-docker-proxy + image: lucaslorentz/caddy-docker-proxy:ci-alpine + ports: + - 80:80 + - 443:443 + environment: + - CADDY_INGRESS_NETWORKS=caddy + networks: + - caddy + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./data:/data + restart: unless-stopped + labels: + caddy: example.com + caddy.0_respond: /.well-known/matrix/server {"m.server":"matrix.example.com:443"} + caddy.1_respond: /.well-known/matrix/client {"m.server":{"base_url":"https://matrix.example.com"},"m.homeserver":{"base_url":"https://matrix.example.com"},"org.matrix.msc3575.proxy":{"url":"https://matrix.example.com"}} + + homeserver: + ### If you already built the conduwuit image with 'docker build' or want to use a registry image, + ### then you are ready to go. + image: girlbossceo/conduwuit:latest + restart: unless-stopped + volumes: + - db:/var/lib/conduwuit + #- ./conduwuit.toml:/etc/conduwuit.toml + environment: + CONDUWUIT_SERVER_NAME: example.com # EDIT THIS + CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit + CONDUWUIT_DATABASE_BACKEND: rocksdb + CONDUWUIT_PORT: 6167 + CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB + CONDUWUIT_ALLOW_REGISTRATION: 'true' + CONDUWUIT_ALLOW_FEDERATION: 'true' + CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' + CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' + #CONDUWUIT_LOG: warn,state_res=warn + CONDUWUIT_ADDRESS: 0.0.0.0 + #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above + networks: + - caddy + labels: + caddy: matrix.example.com + caddy.reverse_proxy: "{{upstreams 6167}}" + +volumes: + db: + +networks: + caddy: + external: true diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index 5cb89308..ffbad5b4 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -59,13 +59,22 @@ If the `docker run` command is not for you or your setup, you can also use one o Depending on your proxy setup, you can use one of the following files; - If you already have a `traefik` instance set up, use [`docker-compose.for-traefik.yml`](docker-compose.for-traefik.yml) -- If you don't have a `traefik` instance set up (or any other reverse proxy), use [`docker-compose.with-traefik.yml`](docker-compose.with-traefik.yml) +- If you don't have a `traefik` instance set up and would like to use it, use [`docker-compose.with-traefik.yml`](docker-compose.with-traefik.yml) +- If you want a setup that works out of the box with `caddy-docker-proxy`, use [`docker-compose.with-caddy.yml`](docker-compose.with-caddy.yml) and replace all `example.com` placeholders with your own domain - For any other reverse proxy, use [`docker-compose.yml`](docker-compose.yml) When picking the traefik-related compose file, rename it so it matches `docker-compose.yml`, and rename the override file to `docker-compose.override.yml`. Edit the latter with the values you want for your server. +When picking the `caddy-docker-proxy` compose file, it's important to first create the `caddy` network before spinning up the containers: + +```bash +docker network create caddy +``` + +After that, you can rename it so it matches `docker-compose.yml` and spin up the containers! + Additional info about deploying conduwuit can be found [here](generic.md). ### Build From a309ef55c910351368687ce02080bddf35d4f0bb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 6 Jul 2024 13:53:21 +0000 Subject: [PATCH 0391/2091] restore signal state after channel failures Signed-off-by: Jason Volk --- src/core/server.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/server.rs b/src/core/server.rs index e76e4d57..575924d3 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -76,7 +76,10 @@ impl Server { return Err(Error::Err("Shutdown already in progress".into())); } - self.signal("SIGINT") + self.signal("SIGINT").inspect_err(|_| { + self.stopping.store(false, Ordering::Release); + self.reloading.store(false, Ordering::Release); + }) } pub fn restart(&self) -> Result<()> { @@ -85,6 +88,7 @@ impl Server { } self.shutdown() + .inspect_err(|_| self.restarting.store(false, Ordering::Release)) } pub fn shutdown(&self) -> Result<()> { @@ -93,6 +97,7 @@ impl Server { } self.signal("SIGTERM") + .inspect_err(|_| self.stopping.store(false, Ordering::Release)) } pub fn signal(&self, sig: &'static str) -> Result<()> { From 24b37e03a01d0df5d53775f2519523fa7b23333b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 6 Jul 2024 06:47:42 +0000 Subject: [PATCH 0392/2091] add pretty time util Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 17 +++----- src/core/utils/time.rs | 79 +++++++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 2ab71033..06007fb8 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,24 +1,17 @@ -use conduit::{warn, Error, Result}; +use conduit::{utils::time, warn, Error, Result}; use ruma::events::room::message::RoomMessageEventContent; use crate::services; pub(super) async fn uptime(_body: Vec<&str>) -> Result { - let seconds = services() + let elapsed = services() .server .started .elapsed() - .expect("standard duration") - .as_secs(); - let result = format!( - "up {} days, {} hours, {} minutes, {} seconds.", - seconds / 86400, - (seconds % 86400) / 60 / 60, - (seconds % 3600) / 60, - seconds % 60, - ); + .expect("standard duration"); - Ok(RoomMessageEventContent::notice_plain(result)) + let result = time::pretty(elapsed); + Ok(RoomMessageEventContent::notice_plain(format!("{result}."))) } pub(super) async fn show_config(_body: Vec<&str>) -> Result { diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 7de00e9e..39ceb3c3 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,4 +1,6 @@ -use std::time::{SystemTime, UNIX_EPOCH}; +#![allow(clippy::enum_glob_use)] + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[inline] #[must_use] @@ -26,3 +28,78 @@ pub fn format(ts: SystemTime, str: &str) -> String { let dt: DateTime = ts.into(); dt.format(str).to_string() } + +#[must_use] +pub fn pretty(d: Duration) -> String { + use Unit::*; + + let fmt = |w, f, u| format!("{w}.{f} {u}"); + let gen64 = |w, f, u| fmt(w, (f * 100.0) as u64, u); + let gen128 = |w, f, u| gen64(u64::try_from(w).expect("u128 to u64"), f, u); + match whole_and_frac(d) { + (Days(whole), frac) => gen64(whole, frac, "days"), + (Hours(whole), frac) => gen64(whole, frac, "hours"), + (Mins(whole), frac) => gen64(whole, frac, "minutes"), + (Secs(whole), frac) => gen64(whole, frac, "seconds"), + (Millis(whole), frac) => gen128(whole, frac, "milliseconds"), + (Micros(whole), frac) => gen128(whole, frac, "microseconds"), + (Nanos(whole), frac) => gen128(whole, frac, "nanoseconds"), + } +} + +/// Return a pair of (whole part, frac part) from a duration where. The whole +/// part is the largest Unit containing a non-zero value, the frac part is a +/// rational remainder left over. +#[must_use] +pub fn whole_and_frac(d: Duration) -> (Unit, f64) { + use Unit::*; + + let whole = whole_unit(d); + ( + whole, + match whole { + Days(_) => (d.as_secs() % 86_400) as f64 / 86_400.0, + Hours(_) => (d.as_secs() % 3_600) as f64 / 3_600.0, + Mins(_) => (d.as_secs() % 60) as f64 / 60.0, + Secs(_) => f64::from(d.subsec_millis()) / 1000.0, + Millis(_) => f64::from(d.subsec_micros()) / 1000.0, + Micros(_) => f64::from(d.subsec_nanos()) / 1000.0, + Nanos(_) => 0.0, + }, + ) +} + +/// Return the largest Unit which represents the duration. The value is +/// rounded-down, but never zero. +#[must_use] +pub fn whole_unit(d: Duration) -> Unit { + use Unit::*; + + match d.as_secs() { + 86_400.. => Days(d.as_secs() / 86_400), + 3_600..=86_399 => Hours(d.as_secs() / 3_600), + 60..=3_599 => Mins(d.as_secs() / 60), + + _ => match d.as_micros() { + 1_000_000.. => Secs(d.as_secs()), + 1_000..=999_999 => Millis(d.subsec_millis().into()), + + _ => match d.as_nanos() { + 1_000.. => Micros(d.subsec_micros().into()), + + _ => Nanos(d.subsec_nanos().into()), + }, + }, + } +} + +#[derive(Eq, PartialEq, Clone, Copy, Debug)] +pub enum Unit { + Days(u64), + Hours(u64), + Mins(u64), + Secs(u64), + Millis(u128), + Micros(u128), + Nanos(u128), +} From a388c2e06edf48671d244e73777fb5f3be8c65bc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 6 Jul 2024 06:50:29 +0000 Subject: [PATCH 0393/2091] allow clippy::enum_glob_use Signed-off-by: Jason Volk --- Cargo.toml | 1 + src/core/utils/time.rs | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 29454e60..f6b23102 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -741,6 +741,7 @@ cast_possible_truncation = { level = "allow", priority = 1 } cast_precision_loss = { level = "allow", priority = 1 } cast_sign_loss = { level = "allow", priority = 1 } doc_markdown = { level = "allow", priority = 1 } +enum_glob_use = { level = "allow", priority = 1 } error_impl_error = { level = "allow", priority = 1 } expect_used = { level = "allow", priority = 1 } if_not_else = { level = "allow", priority = 1 } diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 39ceb3c3..2fe90b31 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,5 +1,3 @@ -#![allow(clippy::enum_glob_use)] - use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[inline] From 611f09829efd839ea290e8806ac43dfcd19aa0f6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 08:48:24 +0000 Subject: [PATCH 0394/2091] use shorthand constraint syntax and formatting Signed-off-by: Jason Volk --- src/core/utils/defer.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/core/utils/defer.rs b/src/core/utils/defer.rs index 2762d4fa..9d42e679 100644 --- a/src/core/utils/defer.rs +++ b/src/core/utils/defer.rs @@ -1,17 +1,11 @@ #[macro_export] macro_rules! defer { ($body:block) => { - struct _Defer_ - where - F: FnMut(), - { + struct _Defer_ { closure: F, } - impl Drop for _Defer_ - where - F: FnMut(), - { + impl Drop for _Defer_ { fn drop(&mut self) { (self.closure)(); } } From 5e72d368006247e664f5c9c2fe52d21aa91c2f88 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 03:36:50 +0000 Subject: [PATCH 0395/2091] add math utils; integrate checked expression macros Signed-off-by: Jason Volk --- Cargo.lock | 10 ++++++++++ Cargo.toml | 3 +++ src/core/Cargo.toml | 1 + src/core/error.rs | 8 ++++++++ src/core/utils/math.rs | 26 ++++++++++++++++++++++++++ src/core/utils/mod.rs | 1 + src/core/utils/tests.rs | 19 +++++++++++++++++++ 7 files changed, 68 insertions(+) create mode 100644 src/core/utils/math.rs diff --git a/Cargo.lock b/Cargo.lock index 3415cac0..7152c66d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -504,6 +504,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "checked_ops" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b491d76efc1d99d74de3c8529bee64c62312c275c7eb124f9185291de45801d5" +dependencies = [ + "num-traits", +] + [[package]] name = "chrono" version = "0.4.38" @@ -658,6 +667,7 @@ dependencies = [ "argon2", "axum 0.7.5", "bytes", + "checked_ops", "chrono", "either", "figment", diff --git a/Cargo.toml b/Cargo.toml index f6b23102..c93c6736 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -411,6 +411,9 @@ rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50" version = "0.29.4" default-features = false +[workspace.dependencies.checked_ops] +version = "0.1" + # # Patches diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 2c0fc47d..e47f673e 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -53,6 +53,7 @@ sha256_media = [] argon2.workspace = true axum.workspace = true bytes.workspace = true +checked_ops.workspace = true chrono.workspace = true either.workspace = true figment.workspace = true diff --git a/src/core/error.rs b/src/core/error.rs index ea7acda5..63729f31 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -35,6 +35,12 @@ pub enum Error { FromUtf8Error(#[from] std::string::FromUtf8Error), #[error("{0}")] TryFromSliceError(#[from] std::array::TryFromSliceError), + #[error("{0}")] + TryFromIntError(#[from] std::num::TryFromIntError), + #[error("{0}")] + ParseIntError(#[from] std::num::ParseIntError), + #[error("{0}")] + ParseFloatError(#[from] std::num::ParseFloatError), // third-party #[error("Regex error: {0}")] @@ -63,6 +69,8 @@ pub enum Error { InconsistentRoomState(&'static str, ruma::OwnedRoomId), // conduwuit + #[error("Arithmetic operation failed: {0}")] + Arithmetic(&'static str), #[error("There was a problem with your configuration: {0}")] BadConfig(String), #[error("{0}")] diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs new file mode 100644 index 00000000..d5dbf3a6 --- /dev/null +++ b/src/core/utils/math.rs @@ -0,0 +1,26 @@ +use std::{cmp, time::Duration}; + +pub use checked_ops::checked_ops; + +/// Checked arithmetic expression. Returns a Result +#[macro_export] +macro_rules! checked { + ($($input:tt)*) => { + $crate::utils::math::checked_ops!($($input)*) + .ok_or_else(|| $crate::Error::Arithmetic("operation overflowed or result invalid")) + } +} + +/// in release-mode. Use for performance when the expression is obviously safe. +/// The check remains in debug-mode for regression analysis. +#[cfg(not(debug_assertions))] +#[macro_export] +macro_rules! validated { + ($($input:tt)*) => { Ok($($input)*) } +} + +#[cfg(debug_assertions)] +#[macro_export] +macro_rules! validated { + ($($input:tt)*) => { $crate::checked!($($input)*) } +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index f9f1b87e..2b79c3c4 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -5,6 +5,7 @@ pub mod defer; pub mod hash; pub mod html; pub mod json; +pub mod math; pub mod mutex_map; pub mod rand; pub mod string; diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index f5cd0a07..239e27e9 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -62,3 +62,22 @@ fn common_prefix_none() { let output = string::common_prefix(&input); assert_eq!(output, ""); } + +#[test] +fn checked_add() { + use utils::math::checked; + + let a = 1234; + let res = checked!(a + 1).unwrap(); + assert_eq!(res, 1235); +} + +#[test] +#[should_panic(expected = "overflow")] +fn checked_add_overflow() { + use utils::math::checked; + + let a: u64 = u64::MAX; + let res = checked!(a + 1).expect("overflow"); + assert_eq!(res, 0); +} From 52a561ff9e8196794995ccc2556c9976a636fb8b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 03:39:35 +0000 Subject: [PATCH 0396/2091] abstract expoential backoff to math utils. Signed-off-by: Jason Volk --- src/api/client/keys.rs | 21 +++++-------- src/api/client/membership.rs | 21 ++++++------- src/core/utils/math.rs | 18 +++++++++++ src/service/rooms/event_handler/mod.rs | 41 +++++++++++--------------- src/service/sending/sender.rs | 12 ++++---- 5 files changed, 60 insertions(+), 53 deletions(-) diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 6f089875..7bb02a60 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -1,9 +1,9 @@ use std::{ - cmp, collections::{hash_map, BTreeMap, HashMap, HashSet}, - time::{Duration, Instant}, + time::Instant, }; +use conduit::{utils, utils::math::continue_exponential_backoff_secs, Error, Result}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::{ @@ -18,15 +18,11 @@ use ruma::{ DeviceKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, }; use serde_json::json; +use service::user_is_local; use tracing::debug; use super::SESSION_ID_LENGTH; -use crate::{ - service::user_is_local, - services, - utils::{self}, - Error, Result, Ruma, -}; +use crate::{services, Ruma}; /// # `POST /_matrix/client/r0/keys/upload` /// @@ -357,11 +353,10 @@ pub(crate) async fn get_keys_helper bool + Send>( .get(server) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_elapsed_duration = cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - - if time.elapsed() < min_elapsed_duration { - debug!("Backing off query from {:?}", server); + const MIN: u64 = 5 * 60; + const MAX: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) { + debug!("Backing off query from {server:?}"); return (server, Err(Error::BadServerResponse("bad query, still backing off"))); } } diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 07a585fd..e2ba4c9a 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1,13 +1,16 @@ use std::{ - cmp, collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, net::IpAddr, sync::Arc, - time::{Duration, Instant}, + time::Instant, }; use axum_client_ip::InsecureClientIp; -use conduit::utils::mutex_map; +use conduit::{ + debug, error, info, trace, utils, + utils::{math::continue_exponential_backoff_secs, mutex_map}, + warn, Error, PduEvent, Result, +}; use ruma::{ api::{ client::{ @@ -35,7 +38,6 @@ use ruma::{ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::sending::convert_to_outgoing_federation_event; use tokio::sync::RwLock; -use tracing::{debug, error, info, trace, warn}; use crate::{ client::{update_avatar_url, update_displayname}, @@ -43,7 +45,7 @@ use crate::{ pdu::{gen_event_id_canonical_json, PduBuilder}, server_is_ours, user_is_local, }, - services, utils, Error, PduEvent, Result, Ruma, + services, Ruma, }; /// Checks if the room is banned in any way possible and the sender user is not @@ -1363,11 +1365,10 @@ pub async fn validate_and_add_event_id( .get(&event_id) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_elapsed_duration = cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - - if time.elapsed() < min_elapsed_duration { - debug!("Backing off from {}", event_id); + const MIN: u64 = 60 * 5; + const MAX: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) { + debug!("Backing off from {event_id}"); return Err(Error::BadServerResponse("bad event, still backing off")); } } diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index d5dbf3a6..a77f8e26 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -24,3 +24,21 @@ macro_rules! validated { macro_rules! validated { ($($input:tt)*) => { $crate::checked!($($input)*) } } + +/// Returns false if the exponential backoff has expired based on the inputs +#[inline] +#[must_use] +pub fn continue_exponential_backoff_secs(min: u64, max: u64, elapsed: Duration, tries: u32) -> bool { + let min = Duration::from_secs(min); + let max = Duration::from_secs(max); + continue_exponential_backoff(min, max, elapsed, tries) +} + +/// Returns false if the exponential backoff has expired based on the inputs +#[inline] +#[must_use] +pub fn continue_exponential_backoff(min: Duration, max: Duration, elapsed: Duration, tries: u32) -> bool { + let min = min.saturating_mul(tries).saturating_mul(tries); + let min = cmp::min(min, max); + elapsed < min +} diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 9f50ef58..395b70f2 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -2,14 +2,16 @@ mod parse_incoming_pdu; mod signing_keys; use std::{ - cmp, collections::{hash_map, BTreeMap, HashMap, HashSet}, pin::Pin, sync::Arc, - time::{Duration, Instant}, + time::Instant, }; -use conduit::{debug_error, debug_info, Error, Result}; +use conduit::{ + debug, debug_error, debug_info, error, info, trace, utils::math::continue_exponential_backoff_secs, warn, Error, + Result, +}; use futures_util::Future; pub use parse_incoming_pdu::parse_incoming_pdu; use ruma::{ @@ -29,7 +31,6 @@ use ruma::{ uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedUserId, RoomId, RoomVersionId, ServerName, }; use tokio::sync::RwLock; -use tracing::{debug, error, info, trace, warn}; use super::state_compressor::CompressedStateEvent; use crate::{pdu, services, PduEvent}; @@ -252,14 +253,12 @@ impl Service { .get(prev_id) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_duration = cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - let duration = time.elapsed(); - - if duration < min_duration { + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { debug!( - duration = ?duration, - min_duration = ?min_duration, + ?tries, + duration = ?time.elapsed(), "Backing off from prev_event" ); return Ok(()); @@ -1083,12 +1082,10 @@ impl Service { .get(&*next_id) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_elapsed_duration = - cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - - if time.elapsed() < min_elapsed_duration { - info!("Backing off from {}", next_id); + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + info!("Backing off from {next_id}"); continue; } } @@ -1191,12 +1188,10 @@ impl Service { .get(&**next_id) { // Exponential backoff - const MAX_DURATION: Duration = Duration::from_secs(60 * 60 * 24); - let min_elapsed_duration = - cmp::min(MAX_DURATION, Duration::from_secs(5 * 60) * (*tries) * (*tries)); - - if time.elapsed() < min_elapsed_duration { - debug!("Backing off from {}", next_id); + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + debug!("Backing off from {next_id}"); continue; } } diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 54302fd5..0f4fa17a 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -3,10 +3,11 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, sync::Arc, - time::{Duration, Instant}, + time::Instant, }; use base64::{engine::general_purpose, Engine as _}; +use conduit::{debug, error, utils::math::continue_exponential_backoff_secs, warn}; use federation::transactions::send_transaction_message; use futures_util::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; use ruma::{ @@ -22,7 +23,6 @@ use ruma::{ ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tracing::{debug, error, warn}; use super::{appservice, send, Destination, Msg, SendingEvent, Service}; use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; @@ -216,11 +216,9 @@ impl Service { .and_modify(|e| match e { TransactionStatus::Failed(tries, time) => { // Fail if a request has failed recently (exponential backoff) - let max_duration = Duration::from_secs(services().globals.config.sender_retry_backoff_limit); - let min_duration = Duration::from_secs(services().globals.config.sender_timeout); - let min_elapsed_duration = min_duration * (*tries) * (*tries); - let min_elapsed_duration = cmp::min(min_elapsed_duration, max_duration); - if time.elapsed() < min_elapsed_duration { + let min = services().globals.config.sender_timeout; + let max = services().globals.config.sender_retry_backoff_limit; + if continue_exponential_backoff_secs(min, max, time.elapsed(), *tries) { allow = false; } else { retry = true; From 7397064edd4d1e6d7bc9cb4d2c5b19794e713458 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 04:46:16 +0000 Subject: [PATCH 0397/2091] fix arithmetic side-effects Signed-off-by: Jason Volk --- Cargo.toml | 2 +- src/admin/debug/commands.rs | 2 +- src/admin/room/room_moderation_commands.rs | 5 +- src/admin/user/commands.rs | 7 ++- src/core/utils/html.rs | 2 +- src/core/utils/math.rs | 10 ++- src/core/utils/rand.rs | 6 +- src/core/utils/tests.rs | 6 +- src/service/globals/data.rs | 5 +- src/service/globals/migrations.rs | 7 ++- src/service/media/mod.rs | 34 +++------- src/service/presence/mod.rs | 73 +++++++++++----------- src/service/rooms/auth_chain/mod.rs | 12 ++-- src/service/rooms/event_handler/mod.rs | 6 +- src/service/rooms/pdu_metadata/mod.rs | 2 +- src/service/rooms/read_receipt/data.rs | 6 +- src/service/rooms/spaces/mod.rs | 5 +- src/service/rooms/state_compressor/data.rs | 11 ++-- src/service/rooms/state_compressor/mod.rs | 16 +++-- src/service/rooms/threads/data.rs | 4 +- src/service/rooms/threads/mod.rs | 2 +- src/service/rooms/timeline/data.rs | 8 ++- src/service/rooms/timeline/mod.rs | 7 ++- src/service/sending/sender.rs | 2 +- src/service/users/data.rs | 13 ++-- 25 files changed, 139 insertions(+), 114 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c93c6736..19ccbb00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -769,7 +769,7 @@ perf = "warn" ################### #restriction = "warn" -#arithmetic_side_effects = "warn" # TODO +arithmetic_side_effects = "warn" #as_conversions = "warn" # TODO assertions_on_result_states = "warn" dbg_macro = "warn" diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 3efe283b..53009566 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -58,7 +58,7 @@ pub(super) async fn parse_pdu(body: Vec<&str>) -> Result match ruma::signatures::reference_hash(&value, &RoomVersionId::V6) { Ok(hash) => { diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 39eb7c47..92e3de47 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -191,7 +191,10 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo )); } - let rooms_s = body.clone().drain(1..body.len() - 1).collect::>(); + let rooms_s = body + .clone() + .drain(1..body.len().saturating_sub(1)) + .collect::>(); let admin_room_alias = &services().globals.admin_alias; diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 6dc60713..884e1d29 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -23,7 +23,7 @@ pub(super) async fn list(_body: Vec<&str>) -> Result { match services().users.list_local_users() { Ok(users) => { let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); - plain_msg += &users.join("\n"); + plain_msg += users.join("\n").as_str(); plain_msg += "\n```"; Ok(RoomMessageEventContent::notice_markdown(plain_msg)) @@ -195,7 +195,10 @@ pub(super) async fn deactivate_all( )); } - let usernames = body.clone().drain(1..body.len() - 1).collect::>(); + let usernames = body + .clone() + .drain(1..body.len().saturating_sub(1)) + .collect::>(); let mut user_ids: Vec = Vec::with_capacity(usernames.len()); let mut admins = Vec::new(); diff --git a/src/core/utils/html.rs b/src/core/utils/html.rs index 3b44a31b..938e50ec 100644 --- a/src/core/utils/html.rs +++ b/src/core/utils/html.rs @@ -26,7 +26,7 @@ impl fmt::Display for Escape<'_> { fmt.write_str(s)?; // NOTE: we only expect single byte characters here - which is fine as long as // we only match single byte characters - last = i + 1; + last = i.saturating_add(1); } if last < s.len() { diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index a77f8e26..f00b9055 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -16,7 +16,15 @@ macro_rules! checked { #[cfg(not(debug_assertions))] #[macro_export] macro_rules! validated { - ($($input:tt)*) => { Ok($($input)*) } + ($($input:tt)*) => { + //#[allow(clippy::arithmetic_side_effects)] { + //Some($($input)*) + // .ok_or_else(|| $crate::Error::Arithmetic("this error should never been seen")) + //} + + //NOTE: remove me when stmt_expr_attributes is stable + $crate::checked!($($input)*) + } } #[cfg(debug_assertions)] diff --git a/src/core/utils/rand.rs b/src/core/utils/rand.rs index 1ded8a6d..b80671eb 100644 --- a/src/core/utils/rand.rs +++ b/src/core/utils/rand.rs @@ -15,7 +15,11 @@ pub fn string(length: usize) -> String { #[inline] #[must_use] -pub fn timepoint_secs(range: Range) -> SystemTime { SystemTime::now() + secs(range) } +pub fn timepoint_secs(range: Range) -> SystemTime { + SystemTime::now() + .checked_add(secs(range)) + .expect("range does not overflow SystemTime") +} #[must_use] pub fn secs(range: Range) -> Duration { diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index 239e27e9..add15861 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -65,7 +65,7 @@ fn common_prefix_none() { #[test] fn checked_add() { - use utils::math::checked; + use crate::checked; let a = 1234; let res = checked!(a + 1).unwrap(); @@ -75,9 +75,9 @@ fn checked_add() { #[test] #[should_panic(expected = "overflow")] fn checked_add_overflow() { - use utils::math::checked; + use crate::checked; - let a: u64 = u64::MAX; + let a = u64::MAX; let res = checked!(a + 1).expect("overflow"); assert_eq!(res, 0); } diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index ecb827fe..083aec3c 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -65,7 +65,10 @@ impl Data { "counter mismatch" ); - *counter = counter.wrapping_add(1); + *counter = counter + .checked_add(1) + .expect("counter must not overflow u64"); + self.global.insert(COUNTER, &counter.to_be_bytes())?; Ok(*counter) diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index e171cb9d..3948d1f5 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -836,11 +836,14 @@ async fn fix_bad_double_separator_in_state_cache(db: &Arc, _config: &C for (mut key, value) in roomuserid_joined.iter() { iter_count = iter_count.saturating_add(1); debug_info!(%iter_count); - let first_sep_index = key.iter().position(|&i| i == 0xFF).unwrap(); + let first_sep_index = key + .iter() + .position(|&i| i == 0xFF) + .expect("found 0xFF delim"); if key .iter() - .get(first_sep_index..=first_sep_index + 1) + .get(first_sep_index..=first_sep_index.saturating_add(1)) .copied() .collect_vec() == vec![0xFF, 0xFF] diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index caa75d4e..3cb8fda8 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,10 +1,10 @@ mod data; mod tests; -use std::{collections::HashMap, io::Cursor, path::PathBuf, sync::Arc, time::SystemTime}; +use std::{collections::HashMap, io::Cursor, num::Saturating as Sat, path::PathBuf, sync::Arc, time::SystemTime}; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, debug_error, error, utils, Error, Result, Server}; +use conduit::{checked, debug, debug_error, error, utils, Error, Result, Server}; use data::Data; use image::imageops::FilterType; use ruma::{OwnedMxcUri, OwnedUserId}; @@ -305,36 +305,20 @@ impl Service { image.resize_to_fill(width, height, FilterType::CatmullRom) } else { let (exact_width, exact_height) = { - // Copied from image::dynimage::resize_dimensions - // - // https://github.com/image-rs/image/blob/6edf8ae492c4bb1dacb41da88681ea74dab1bab3/src/math/utils.rs#L5-L11 - // Calculates the width and height an image should be - // resized to. This preserves aspect ratio, and based - // on the `fill` parameter will either fill the - // dimensions to fit inside the smaller constraint - // (will overflow the specified bounds on one axis to - // preserve aspect ratio), or will shrink so that both - // dimensions are completely contained within the given - // `width` and `height`, with empty space on one axis. - let ratio = u64::from(original_width) * u64::from(height); - let nratio = u64::from(width) * u64::from(original_height); + let ratio = Sat(original_width) * Sat(height); + let nratio = Sat(width) * Sat(original_height); let use_width = nratio <= ratio; let intermediate = if use_width { - u64::from(original_height) * u64::from(width) / u64::from(original_width) + Sat(original_height) * Sat(checked!(width / original_width)?) } else { - u64::from(original_width) * u64::from(height) / u64::from(original_height) + Sat(original_width) * Sat(checked!(height / original_height)?) }; + if use_width { - if u32::try_from(intermediate).is_ok() { - (width, intermediate as u32) - } else { - ((u64::from(width) * u64::from(u32::MAX) / intermediate) as u32, u32::MAX) - } - } else if u32::try_from(intermediate).is_ok() { - (intermediate as u32, height) + (width, intermediate.0) } else { - (u32::MAX, (u64::from(height) * u64::from(u32::MAX) / intermediate) as u32) + (intermediate.0, height) } }; diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 584f1a6d..f5400379 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -3,7 +3,7 @@ mod data; use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{debug, error, utils, Error, Result}; +use conduit::{checked, debug, error, utils, Error, Result}; use data::Data; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ @@ -79,12 +79,16 @@ pub struct Service { timer_receiver: Mutex>, handler_join: Mutex>>, timeout_remote_users: bool, + idle_timeout: u64, + offline_timeout: u64, } #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; + let idle_timeout_s = config.presence_idle_timeout_s; + let offline_timeout_s = config.presence_offline_timeout_s; let (timer_sender, timer_receiver) = loole::unbounded(); Ok(Arc::new(Self { db: Data::new(args.db), @@ -92,6 +96,8 @@ impl crate::Service for Service { timer_receiver: Mutex::new(timer_receiver), handler_join: Mutex::new(None), timeout_remote_users: config.presence_timeout_remote_users, + idle_timeout: checked!(idle_timeout_s * 1_000)?, + offline_timeout: checked!(offline_timeout_s * 1_000)?, })) } @@ -219,7 +225,7 @@ impl Service { loop { debug_assert!(!receiver.is_closed(), "channel error"); tokio::select! { - Some(user_id) = presence_timers.next() => process_presence_timer(&user_id)?, + Some(user_id) = presence_timers.next() => self.process_presence_timer(&user_id)?, event = receiver.recv_async() => match event { Err(_e) => return Ok(()), Ok((user_id, timeout)) => { @@ -230,6 +236,36 @@ impl Service { } } } + + fn process_presence_timer(&self, user_id: &OwnedUserId) -> Result<()> { + let mut presence_state = PresenceState::Offline; + let mut last_active_ago = None; + let mut status_msg = None; + + let presence_event = self.get_presence(user_id)?; + + if let Some(presence_event) = presence_event { + presence_state = presence_event.content.presence; + last_active_ago = presence_event.content.last_active_ago; + status_msg = presence_event.content.status_msg; + } + + let new_state = match (&presence_state, last_active_ago.map(u64::from)) { + (PresenceState::Online, Some(ago)) if ago >= self.idle_timeout => Some(PresenceState::Unavailable), + (PresenceState::Unavailable, Some(ago)) if ago >= self.offline_timeout => Some(PresenceState::Offline), + _ => None, + }; + + debug!( + "Processed presence timer for user '{user_id}': Old state = {presence_state}, New state = {new_state:?}" + ); + + if let Some(new_state) = new_state { + self.set_presence(user_id, &new_state, Some(false), last_active_ago, status_msg)?; + } + + Ok(()) + } } async fn presence_timer(user_id: OwnedUserId, timeout: Duration) -> OwnedUserId { @@ -237,36 +273,3 @@ async fn presence_timer(user_id: OwnedUserId, timeout: Duration) -> OwnedUserId user_id } - -fn process_presence_timer(user_id: &OwnedUserId) -> Result<()> { - let idle_timeout = services().globals.config.presence_idle_timeout_s * 1_000; - let offline_timeout = services().globals.config.presence_offline_timeout_s * 1_000; - - let mut presence_state = PresenceState::Offline; - let mut last_active_ago = None; - let mut status_msg = None; - - let presence_event = services().presence.get_presence(user_id)?; - - if let Some(presence_event) = presence_event { - presence_state = presence_event.content.presence; - last_active_ago = presence_event.content.last_active_ago; - status_msg = presence_event.content.status_msg; - } - - let new_state = match (&presence_state, last_active_ago.map(u64::from)) { - (PresenceState::Online, Some(ago)) if ago >= idle_timeout => Some(PresenceState::Unavailable), - (PresenceState::Unavailable, Some(ago)) if ago >= offline_timeout => Some(PresenceState::Offline), - _ => None, - }; - - debug!("Processed presence timer for user '{user_id}': Old state = {presence_state}, New state = {new_state:?}"); - - if let Some(new_state) = new_state { - services() - .presence - .set_presence(user_id, &new_state, Some(false), last_active_ago, status_msg)?; - } - - Ok(()) -} diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index ca04b1e5..18cdb70f 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -5,7 +5,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, trace, warn, Error, Result}; +use conduit::{debug, error, trace, validated, warn, Error, Result}; use data::Data; use ruma::{api::client::error::ErrorKind, EventId, RoomId}; @@ -43,20 +43,20 @@ impl Service { #[tracing::instrument(skip_all, name = "auth_chain")] pub async fn get_auth_chain(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result> { - const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? + const NUM_BUCKETS: u64 = 50; //TODO: change possible w/o disrupting db? const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); let started = std::time::Instant::now(); - let mut buckets = [BUCKET; NUM_BUCKETS]; - for (i, short) in services() + let mut buckets = [BUCKET; NUM_BUCKETS as usize]; + for (i, &short) in services() .rooms .short .multi_get_or_create_shorteventid(starting_events)? .iter() .enumerate() { - let bucket = short % NUM_BUCKETS as u64; - buckets[bucket as usize].insert((*short, starting_events[i])); + let bucket = validated!(short % NUM_BUCKETS)?; + buckets[bucket as usize].insert((short, starting_events[i])); } debug!( diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 395b70f2..0f7919dd 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -191,7 +191,7 @@ impl Service { e.insert((Instant::now(), 1)); }, hash_map::Entry::Occupied(mut e) => { - *e.get_mut() = (Instant::now(), e.get().1 + 1); + *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); }, }; }, @@ -1072,7 +1072,7 @@ impl Service { let mut todo_auth_events = vec![Arc::clone(id)]; let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); let mut events_all = HashSet::with_capacity(todo_auth_events.len()); - let mut i = 0; + let mut i: u64 = 0; while let Some(next_id) = todo_auth_events.pop() { if let Some((time, tries)) = services() .globals @@ -1094,7 +1094,7 @@ impl Service { continue; } - i += 1; + i = i.saturating_add(1); if i % 100 == 0 { tokio::task::yield_now().await; } diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index e35c969d..cdb2fc29 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -205,7 +205,7 @@ impl Service { if let Ok(relations) = self.db.relations_until(user_id, room_id, target, until) { for relation in relations.flatten() { if stack_pdu.1 < max_depth { - stack.push((relation.clone(), stack_pdu.1 + 1)); + stack.push((relation.clone(), stack_pdu.1.saturating_add(1))); } pdus.push(relation); diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 17acb0b3..06eaf655 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -76,10 +76,12 @@ impl Data { .iter_from(&first_possible_edu, false) .take_while(move |(k, _)| k.starts_with(&prefix2)) .map(move |(k, v)| { - let count = utils::u64_from_bytes(&k[prefix.len()..prefix.len() + size_of::()]) + let count_offset = prefix.len().saturating_add(size_of::()); + let count = utils::u64_from_bytes(&k[prefix.len()..count_offset]) .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?; + let user_id_offset = count_offset.saturating_add(1); let user_id = UserId::parse( - utils::string_from_bytes(&k[prefix.len() + size_of::() + 1..]) + utils::string_from_bytes(&k[user_id_offset..]) .map_err(|_| Error::bad_database("Invalid readreceiptid userid bytes in db."))?, ) .map_err(|_| Error::bad_database("Invalid readreceiptid userid in db."))?; diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index bf6cc873..6924c50e 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::Arc, }; -use conduit::debug_info; +use conduit::{checked, debug_info}; use lru_cache::LruCache; use ruma::{ api::{ @@ -508,7 +508,8 @@ impl Service { } // We have reached the room after where we last left off - if parents.len() + 1 == short_room_ids.len() { + let parents_len = parents.len(); + if checked!(parents_len + 1)? == short_room_ids.len() { populate_results = true; } } diff --git a/src/service/rooms/state_compressor/data.rs b/src/service/rooms/state_compressor/data.rs index 61c7d6e6..33773001 100644 --- a/src/service/rooms/state_compressor/data.rs +++ b/src/service/rooms/state_compressor/data.rs @@ -1,6 +1,6 @@ use std::{collections::HashSet, mem::size_of, sync::Arc}; -use conduit::{utils, Error, Result}; +use conduit::{checked, utils, Error, Result}; use database::{Database, Map}; use super::CompressedStateEvent; @@ -38,11 +38,12 @@ impl Data { let mut added = HashSet::new(); let mut removed = HashSet::new(); - let mut i = size_of::(); - while let Some(v) = value.get(i..i + 2 * size_of::()) { + let stride = size_of::(); + let mut i = stride; + while let Some(v) = value.get(i..checked!(i + 2 * stride)?) { if add_mode && v.starts_with(&0_u64.to_be_bytes()) { add_mode = false; - i += size_of::(); + i = checked!(i + stride)?; continue; } if add_mode { @@ -50,7 +51,7 @@ impl Data { } else { removed.insert(v.try_into().expect("we checked the size above")); } - i += 2 * size_of::(); + i = checked!(i + 2 * stride)?; } Ok(StateDiff { diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 97f3fb80..ca076fc9 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{utils, Result}; +use conduit::{checked, utils, Result}; use data::Data; use lru_cache::LruCache; use ruma::{EventId, RoomId}; @@ -169,12 +169,14 @@ impl Service { statediffremoved: Arc>, diff_to_sibling: usize, mut parent_states: ParentStatesVec, ) -> Result<()> { - let diffsum = statediffnew.len() + statediffremoved.len(); + let statediffnew_len = statediffnew.len(); + let statediffremoved_len = statediffremoved.len(); + let diffsum = checked!(statediffnew_len + statediffremoved_len)?; if parent_states.len() > 3 { // Number of layers // To many layers, we have to go deeper - let parent = parent_states.pop().unwrap(); + let parent = parent_states.pop().expect("parent must have a state"); let mut parent_new = (*parent.2).clone(); let mut parent_removed = (*parent.3).clone(); @@ -226,10 +228,12 @@ impl Service { // 1. We add the current diff on top of the parent layer. // 2. We replace a layer above - let parent = parent_states.pop().unwrap(); - let parent_diff = parent.2.len() + parent.3.len(); + let parent = parent_states.pop().expect("parent must have a state"); + let parent_2_len = parent.2.len(); + let parent_3_len = parent.3.len(); + let parent_diff = checked!(parent_2_len + parent_3_len)?; - if diffsum * diffsum >= 2 * diff_to_sibling * parent_diff { + if checked!(diffsum * diffsum)? >= checked!(2 * diff_to_sibling * parent_diff)? { // Diff too big, we replace above layer(s) let mut parent_new = (*parent.2).clone(); let mut parent_removed = (*parent.3).clone(); diff --git a/src/service/rooms/threads/data.rs b/src/service/rooms/threads/data.rs index 29539847..c4a1a294 100644 --- a/src/service/rooms/threads/data.rs +++ b/src/service/rooms/threads/data.rs @@ -1,6 +1,6 @@ use std::{mem::size_of, sync::Arc}; -use conduit::{utils, Error, Result}; +use conduit::{checked, utils, Error, Result}; use database::{Database, Map}; use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId}; @@ -31,7 +31,7 @@ impl Data { .to_vec(); let mut current = prefix.clone(); - current.extend_from_slice(&(until - 1).to_be_bytes()); + current.extend_from_slice(&(checked!(until - 1)?).to_be_bytes()); Ok(Box::new( self.threadid_userids diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index f3cefe21..dd2686b0 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -64,7 +64,7 @@ impl Service { .and_then(|relations| serde_json::from_value::(relations.clone().into()).ok()) { // Thread already existed - relations.count += uint!(1); + relations.count = relations.count.saturating_add(uint!(1)); relations.latest_event = pdu.to_message_like_event(); let content = serde_json::to_value(relations).expect("to_value always works"); diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 0d4d945e..ec975b99 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -4,7 +4,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{error, utils, Error, Result}; +use conduit::{checked, error, utils, Error, Result}; use database::{Database, Map}; use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; @@ -281,10 +281,12 @@ impl Data { /// Returns the `count` of this pdu's id. pub(super) fn pdu_count(pdu_id: &[u8]) -> Result { - let last_u64 = utils::u64_from_bytes(&pdu_id[pdu_id.len() - size_of::()..]) + let stride = size_of::(); + let pdu_id_len = pdu_id.len(); + let last_u64 = utils::u64_from_bytes(&pdu_id[checked!(pdu_id_len - stride)?..]) .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?; let second_last_u64 = - utils::u64_from_bytes(&pdu_id[pdu_id.len() - 2 * size_of::()..pdu_id.len() - size_of::()]); + utils::u64_from_bytes(&pdu_id[checked!(pdu_id_len - 2 * stride)?..checked!(pdu_id_len - stride)?]); if matches!(second_last_u64, Ok(0)) { Ok(PduCount::Backfilled(u64::MAX.saturating_sub(last_u64))) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index ba987dbd..70f4423c 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -6,7 +6,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, info, utils, utils::mutex_map, warn, Error, Result}; +use conduit::{debug, error, info, utils, utils::mutex_map, validated, warn, Error, Result}; use data::Data; use itertools::Itertools; use rand::prelude::SliceRandom; @@ -670,7 +670,7 @@ impl Service { .filter_map(|event_id| Some(self.get_pdu(event_id).ok()??.depth)) .max() .unwrap_or_else(|| uint!(0)) - + uint!(1); + .saturating_add(uint!(1)); let mut unsigned = unsigned.unwrap_or_default(); @@ -1240,10 +1240,11 @@ impl Service { let insert_lock = services().globals.roomid_mutex_insert.lock(&room_id).await; + let max = u64::MAX; let count = services().globals.next_count()?; let mut pdu_id = shortroomid.to_be_bytes().to_vec(); pdu_id.extend_from_slice(&0_u64.to_be_bytes()); - pdu_id.extend_from_slice(&(u64::MAX - count).to_be_bytes()); + pdu_id.extend_from_slice(&(validated!(max - count)?).to_be_bytes()); // Insert pdu self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value)?; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 0f4fa17a..0fb0d9dc 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -93,7 +93,7 @@ impl Service { statuses.entry(dest).and_modify(|e| { *e = match e { TransactionStatus::Running => TransactionStatus::Failed(1, Instant::now()), - TransactionStatus::Retrying(n) => TransactionStatus::Failed(*n + 1, Instant::now()), + TransactionStatus::Retrying(ref n) => TransactionStatus::Failed(n.saturating_add(1), Instant::now()), TransactionStatus::Failed(..) => panic!("Request that was not even running failed?!"), } }); diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 3ba54a93..302eae9d 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -463,7 +463,8 @@ impl Data { .algorithm(), ) }) { - *counts.entry(algorithm?).or_default() += uint!(1); + let count: &mut UInt = counts.entry(algorithm?).or_default(); + *count = count.saturating_add(uint!(1)); } Ok(counts) @@ -814,7 +815,7 @@ impl Data { .map(|(key, _)| { Ok::<_, Error>(( key.clone(), - utils::u64_from_bytes(&key[key.len() - size_of::()..key.len()]) + utils::u64_from_bytes(&key[key.len().saturating_sub(size_of::())..key.len()]) .map_err(|_| Error::bad_database("ToDeviceId has invalid count bytes."))?, )) }) @@ -928,10 +929,12 @@ impl Data { /// Creates an OpenID token, which can be used to prove that a user has /// access to an account (primarily for integrations) pub(super) fn create_openid_token(&self, user_id: &UserId, token: &str) -> Result { - let expires_in = services().globals.config.openid_token_ttl; - let expires_at = utils::millis_since_unix_epoch().saturating_add(expires_in * 1000); + use std::num::Saturating as Sat; - let mut value = expires_at.to_be_bytes().to_vec(); + let expires_in = services().globals.config.openid_token_ttl; + let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in) * Sat(1000); + + let mut value = expires_at.0.to_be_bytes().to_vec(); value.extend_from_slice(user_id.as_bytes()); self.openidtoken_expiresatuserid From dcd7422c45a0d124bd6689e339760bd330facb2b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 06:17:58 +0000 Subject: [PATCH 0398/2091] fix as conversions Signed-off-by: Jason Volk --- Cargo.toml | 2 +- src/api/client/space.rs | 2 +- src/api/client/sync.rs | 32 ++++++++++++--------- src/core/alloc/je.rs | 25 ++++++++++------ src/core/utils/math.rs | 35 +++++++++++++++++++++++ src/core/utils/time.rs | 4 ++- src/service/rooms/auth_chain/data.rs | 4 +-- src/service/rooms/auth_chain/mod.rs | 9 +++--- src/service/rooms/spaces/mod.rs | 19 ++++++------ src/service/rooms/state_accessor/mod.rs | 19 +++++++----- src/service/rooms/state_compressor/mod.rs | 7 ++--- 11 files changed, 107 insertions(+), 51 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 19ccbb00..bf4f71c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -770,7 +770,7 @@ perf = "warn" #restriction = "warn" arithmetic_side_effects = "warn" -#as_conversions = "warn" # TODO +as_conversions = "warn" assertions_on_result_states = "warn" dbg_macro = "warn" default_union_representation = "warn" diff --git a/src/api/client/space.rs b/src/api/client/space.rs index e00171c3..0cf1b107 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -47,7 +47,7 @@ pub(crate) async fn get_hierarchy_route(body: Ruma) &body.room_id, limit.try_into().unwrap_or(10), key.map_or(vec![], |token| token.short_room_ids), - max_depth.try_into().unwrap_or(3), + max_depth.into(), body.suggested_only, ) .await diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 2ea766a4..9ef1fab0 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -4,7 +4,11 @@ use std::{ time::Duration, }; -use conduit::PduCount; +use conduit::{ + error, + utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, + PduCount, +}; use ruma::{ api::client::{ filter::{FilterDefinition, LazyLoadOptions}, @@ -27,7 +31,7 @@ use ruma::{ serde::Raw, uint, DeviceId, EventId, OwnedUserId, RoomId, UInt, UserId, }; -use tracing::{error, Instrument as _, Span}; +use tracing::{Instrument as _, Span}; use crate::{service::pdu::EventHash, services, utils, Error, PduEvent, Result, Ruma, RumaResponse}; @@ -975,8 +979,8 @@ async fn load_joined_room( }, summary: RoomSummary { heroes, - joined_member_count: joined_member_count.map(|n| (n as u32).into()), - invited_member_count: invited_member_count.map(|n| (n as u32).into()), + joined_member_count: joined_member_count.map(ruma_from_u64), + invited_member_count: invited_member_count.map(ruma_from_u64), }, unread_notifications: UnreadNotificationsCount { highlight_count, @@ -1026,7 +1030,7 @@ fn load_timeline( // Take the last events for the timeline timeline_pdus = non_timeline_pdus .by_ref() - .take(limit as usize) + .take(usize_from_u64_truncated(limit)) .collect::>() .into_iter() .rev() @@ -1300,7 +1304,7 @@ pub(crate) async fn sync_events_v4_route( r.0, UInt::try_from(all_joined_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), ); - let room_ids = all_joined_rooms[(u64::from(r.0) as usize)..=(u64::from(r.1) as usize)].to_vec(); + let room_ids = all_joined_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec(); new_known_rooms.extend(room_ids.iter().cloned()); for room_id in &room_ids { let todo_room = todo_rooms @@ -1333,7 +1337,7 @@ pub(crate) async fn sync_events_v4_route( } }) .collect(), - count: UInt::from(all_joined_rooms.len() as u32), + count: ruma_from_usize(all_joined_rooms.len()), }, ); @@ -1529,20 +1533,22 @@ pub(crate) async fn sync_events_v4_route( prev_batch, limited, joined_count: Some( - (services() + services() .rooms .state_cache .room_joined_count(room_id)? - .unwrap_or(0) as u32) - .into(), + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), ), invited_count: Some( - (services() + services() .rooms .state_cache .room_invited_count(room_id)? - .unwrap_or(0) as u32) - .into(), + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), ), num_live: None, // Count events in timeline greater than global sync counter timestamp: None, diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 966cbde3..5e3c361f 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -12,15 +12,24 @@ static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc; #[must_use] pub fn memory_usage() -> String { use mallctl::stats; - let allocated = stats::allocated::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let active = stats::active::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let mapped = stats::mapped::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let metadata = stats::metadata::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let resident = stats::resident::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; - let retained = stats::retained::read().unwrap_or_default() as f64 / 1024.0 / 1024.0; + + let mibs = |input: Result| { + let input = input.unwrap_or_default(); + let kibs = input / 1024; + let kibs = u32::try_from(kibs).unwrap_or_default(); + let kibs = f64::from(kibs); + kibs / 1024.0 + }; + + let allocated = mibs(stats::allocated::read()); + let active = mibs(stats::active::read()); + let mapped = mibs(stats::mapped::read()); + let metadata = mibs(stats::metadata::read()); + let resident = mibs(stats::resident::read()); + let retained = mibs(stats::retained::read()); format!( - "allocated: {allocated:.2} MiB\n active: {active:.2} MiB\n mapped: {mapped:.2} MiB\n metadata: {metadata:.2} \ - MiB\n resident: {resident:.2} MiB\n retained: {retained:.2} MiB\n " + "allocated: {allocated:.2} MiB\nactive: {active:.2} MiB\nmapped: {mapped:.2} MiB\nmetadata: {metadata:.2} \ + MiB\nresident: {resident:.2} MiB\nretained: {retained:.2} MiB\n" ) } diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index f00b9055..3dc7e27f 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -2,6 +2,8 @@ use std::{cmp, time::Duration}; pub use checked_ops::checked_ops; +use crate::{Error, Result}; + /// Checked arithmetic expression. Returns a Result #[macro_export] macro_rules! checked { @@ -50,3 +52,36 @@ pub fn continue_exponential_backoff(min: Duration, max: Duration, elapsed: Durat let min = cmp::min(min, max); elapsed < min } + +#[inline] +#[allow(clippy::as_conversions)] +pub fn usize_from_f64(val: f64) -> Result { + if val < 0.0 { + return Err(Error::Arithmetic("Converting negative float to unsigned integer")); + } + + Ok(val as usize) +} + +#[inline] +#[must_use] +pub fn usize_from_ruma(val: ruma::UInt) -> usize { + usize::try_from(val).expect("failed conversion from ruma::UInt to usize") +} + +#[inline] +#[must_use] +pub fn ruma_from_u64(val: u64) -> ruma::UInt { + ruma::UInt::try_from(val).expect("failed conversion from u64 to ruma::UInt") +} + +#[inline] +#[must_use] +pub fn ruma_from_usize(val: usize) -> ruma::UInt { + ruma::UInt::try_from(val).expect("failed conversion from usize to ruma::UInt") +} + +#[inline] +#[must_use] +#[allow(clippy::as_conversions)] +pub fn usize_from_u64_truncated(val: u64) -> usize { val as usize } diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 2fe90b31..bf6192b2 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -28,11 +28,12 @@ pub fn format(ts: SystemTime, str: &str) -> String { } #[must_use] +#[allow(clippy::as_conversions)] pub fn pretty(d: Duration) -> String { use Unit::*; let fmt = |w, f, u| format!("{w}.{f} {u}"); - let gen64 = |w, f, u| fmt(w, (f * 100.0) as u64, u); + let gen64 = |w, f, u| fmt(w, (f * 100.0) as u32, u); let gen128 = |w, f, u| gen64(u64::try_from(w).expect("u128 to u64"), f, u); match whole_and_frac(d) { (Days(whole), frac) => gen64(whole, frac, "days"), @@ -49,6 +50,7 @@ pub fn pretty(d: Duration) -> String { /// part is the largest Unit containing a non-zero value, the frac part is a /// rational remainder left over. #[must_use] +#[allow(clippy::as_conversions)] pub fn whole_and_frac(d: Duration) -> (Unit, f64) { use Unit::*; diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index a5771f4a..5efb36c2 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{utils, Result, Server}; +use conduit::{utils, utils::math::usize_from_f64, Result, Server}; use database::{Database, Map}; use lru_cache::LruCache; @@ -16,7 +16,7 @@ impl Data { pub(super) fn new(server: &Arc, db: &Arc) -> Self { let config = &server.config; let cache_size = f64::from(config.auth_chain_cache_capacity); - let cache_size = (cache_size * config.conduit_cache_capacity_modifier) as usize; + let cache_size = usize_from_f64(cache_size * config.conduit_cache_capacity_modifier).expect("valid cache size"); Self { shorteventid_authchain: db["shorteventid_authchain"].clone(), auth_chain_cache: Mutex::new(LruCache::new(cache_size)), diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 18cdb70f..7b2a4f01 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -43,11 +43,11 @@ impl Service { #[tracing::instrument(skip_all, name = "auth_chain")] pub async fn get_auth_chain(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result> { - const NUM_BUCKETS: u64 = 50; //TODO: change possible w/o disrupting db? + const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); let started = std::time::Instant::now(); - let mut buckets = [BUCKET; NUM_BUCKETS as usize]; + let mut buckets = [BUCKET; NUM_BUCKETS]; for (i, &short) in services() .rooms .short @@ -55,8 +55,9 @@ impl Service { .iter() .enumerate() { - let bucket = validated!(short % NUM_BUCKETS)?; - buckets[bucket as usize].insert((short, starting_events[i])); + let bucket: usize = short.try_into()?; + let bucket: usize = validated!(bucket % NUM_BUCKETS)?; + buckets[bucket].insert((short, starting_events[i])); } debug!( diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 6924c50e..03d0d43f 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::Arc, }; -use conduit::{checked, debug_info}; +use conduit::{checked, debug_info, utils::math::usize_from_f64}; use lru_cache::LruCache; use ruma::{ api::{ @@ -161,11 +161,10 @@ impl From for SpaceHierarchyRoomsChunk { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; + let cache_size = f64::from(config.roomid_spacehierarchy_cache_capacity); + let cache_size = cache_size * config.conduit_cache_capacity_modifier; Ok(Arc::new(Self { - roomid_spacehierarchy_cache: Mutex::new(LruCache::new( - (f64::from(config.roomid_spacehierarchy_cache_capacity) * config.conduit_cache_capacity_modifier) - as usize, - )), + roomid_spacehierarchy_cache: Mutex::new(LruCache::new(usize_from_f64(cache_size)?)), })) } @@ -447,7 +446,7 @@ impl Service { } pub async fn get_client_hierarchy( - &self, sender_user: &UserId, room_id: &RoomId, limit: usize, short_room_ids: Vec, max_depth: usize, + &self, sender_user: &UserId, room_id: &RoomId, limit: usize, short_room_ids: Vec, max_depth: u64, suggested_only: bool, ) -> Result { let mut parents = VecDeque::new(); @@ -514,7 +513,8 @@ impl Service { } } - if !children.is_empty() && parents.len() < max_depth { + let parents_len: u64 = parents.len().try_into()?; + if !children.is_empty() && parents_len < max_depth { parents.push_back(current_room.clone()); stack.push(children); } @@ -549,9 +549,8 @@ impl Service { Some( PaginationToken { short_room_ids, - limit: UInt::new(max_depth as u64).expect("When sent in request it must have been valid UInt"), - max_depth: UInt::new(max_depth as u64) - .expect("When sent in request it must have been valid UInt"), + limit: UInt::new(max_depth).expect("When sent in request it must have been valid UInt"), + max_depth: UInt::new(max_depth).expect("When sent in request it must have been valid UInt"), suggested_only, } .to_string(), diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 7fe61134..e8554500 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -6,7 +6,11 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{error, utils::mutex_map, warn, Error, Result}; +use conduit::{ + error, + utils::{math::usize_from_f64, mutex_map}, + warn, Error, Result, +}; use data::Data; use lru_cache::LruCache; use ruma::{ @@ -44,14 +48,15 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; + let server_visibility_cache_capacity = + f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier; + let user_visibility_cache_capacity = + f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier; + Ok(Arc::new(Self { db: Data::new(args.db), - server_visibility_cache: StdMutex::new(LruCache::new( - (f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, - )), - user_visibility_cache: StdMutex::new(LruCache::new( - (f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, - )), + server_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64(server_visibility_cache_capacity)?)), + user_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64(user_visibility_cache_capacity)?)), })) } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index ca076fc9..197efbea 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{checked, utils, Result}; +use conduit::{checked, utils, utils::math::usize_from_f64, Result}; use data::Data; use lru_cache::LruCache; use ruma::{EventId, RoomId}; @@ -55,11 +55,10 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; + let cache_capacity = f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier; Ok(Arc::new(Self { db: Data::new(args.db), - stateinfo_cache: StdMutex::new(LruCache::new( - (f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier) as usize, - )), + stateinfo_cache: StdMutex::new(LruCache::new(usize_from_f64(cache_capacity)?)), })) } From dfd13780dff7a144c46c16be228d10a9c41da2ad Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 07:14:09 +0000 Subject: [PATCH 0399/2091] mitigate additional cast lints Signed-off-by: Jason Volk --- Cargo.toml | 3 --- src/core/utils/math.rs | 5 +++-- src/core/utils/time.rs | 6 +++--- src/database/engine.rs | 27 ++++++++++++--------------- 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index bf4f71c7..af9c719e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -740,9 +740,6 @@ significant_drop_tightening = { level = "allow", priority = 1 } # TODO pedantic = "warn" ## some sadness -cast_possible_truncation = { level = "allow", priority = 1 } -cast_precision_loss = { level = "allow", priority = 1 } -cast_sign_loss = { level = "allow", priority = 1 } doc_markdown = { level = "allow", priority = 1 } enum_glob_use = { level = "allow", priority = 1 } error_impl_error = { level = "allow", priority = 1 } diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index 3dc7e27f..155721e7 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -60,7 +60,8 @@ pub fn usize_from_f64(val: f64) -> Result { return Err(Error::Arithmetic("Converting negative float to unsigned integer")); } - Ok(val as usize) + //SAFETY: + Ok(unsafe { val.to_int_unchecked::() }) } #[inline] @@ -83,5 +84,5 @@ pub fn ruma_from_usize(val: usize) -> ruma::UInt { #[inline] #[must_use] -#[allow(clippy::as_conversions)] +#[allow(clippy::as_conversions, clippy::cast_possible_truncation)] pub fn usize_from_u64_truncated(val: u64) -> usize { val as usize } diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index bf6192b2..9a31632e 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -2,7 +2,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; #[inline] #[must_use] -#[allow(clippy::as_conversions)] +#[allow(clippy::as_conversions, clippy::cast_possible_truncation)] pub fn now_millis() -> u64 { UNIX_EPOCH .elapsed() @@ -28,7 +28,7 @@ pub fn format(ts: SystemTime, str: &str) -> String { } #[must_use] -#[allow(clippy::as_conversions)] +#[allow(clippy::as_conversions, clippy::cast_possible_truncation, clippy::cast_sign_loss)] pub fn pretty(d: Duration) -> String { use Unit::*; @@ -50,7 +50,7 @@ pub fn pretty(d: Duration) -> String { /// part is the largest Unit containing a non-zero value, the frac part is a /// rational remainder left over. #[must_use] -#[allow(clippy::as_conversions)] +#[allow(clippy::as_conversions, clippy::cast_precision_loss)] pub fn whole_and_frac(d: Duration) -> (Unit, f64) { use Unit::*; diff --git a/src/database/engine.rs b/src/database/engine.rs index 7c9522e1..53e39abc 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -134,23 +134,21 @@ impl Engine { .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); } - #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] pub fn memory_usage(&self) -> Result { let mut res = String::new(); let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&self.row_cache])).or_else(or_else)?; + let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0; writeln!( res, "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow cache: {:.2} MiB", - stats.mem_table_total as f64 / 1024.0 / 1024.0, - stats.mem_table_unflushed as f64 / 1024.0 / 1024.0, - stats.mem_table_readers_total as f64 / 1024.0 / 1024.0, - self.row_cache.get_usage() as f64 / 1024.0 / 1024.0, - ) - .expect("should be able to write to string buffer"); + mibs(stats.mem_table_total), + mibs(stats.mem_table_unflushed), + mibs(stats.mem_table_readers_total), + mibs(u64::try_from(self.row_cache.get_usage())?), + )?; for (name, cache) in &*self.col_cache.read().expect("locked") { - writeln!(res, "{} cache: {:.2} MiB", name, cache.get_usage() as f64 / 1024.0 / 1024.0,) - .expect("should be able to write to string buffer"); + writeln!(res, "{} cache: {:.2} MiB", name, mibs(u64::try_from(cache.get_usage())?))?; } Ok(res) @@ -214,8 +212,7 @@ impl Engine { rfc2822_from_seconds(info.timestamp), info.size, info.num_files, - ) - .expect("should be able to write to string buffer"); + )?; } Ok(res) @@ -226,16 +223,16 @@ impl Engine { Err(e) => Ok(String::from(e)), Ok(files) => { let mut res = String::new(); - writeln!(res, "| lev | sst | keys | dels | size | column |").expect("written to string buffer"); - writeln!(res, "| ---: | :--- | ---: | ---: | ---: | :--- |").expect("written to string buffer"); + writeln!(res, "| lev | sst | keys | dels | size | column |")?; + writeln!(res, "| ---: | :--- | ---: | ---: | ---: | :--- |")?; for file in files { writeln!( res, "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", file.level, file.name, file.num_entries, file.num_deletions, file.size, file.column_family_name, - ) - .expect("should be able to writeln to string buffer"); + )?; } + Ok(res) }, } From 5722c4ae396368d0b7c696b7106bbcf4b0f213a4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 07:39:18 +0000 Subject: [PATCH 0400/2091] fix needless collect Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/admin/room/room_moderation_commands.rs | 20 +++++--------------- src/api/client/membership.rs | 3 +-- src/service/rooms/state_cache/data.rs | 4 +++- src/service/rooms/state_cache/mod.rs | 2 +- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index af9c719e..f6149086 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -730,7 +730,6 @@ nursery = "warn" ## some sadness missing_const_for_fn = { level = "allow", priority = 1 } # TODO -needless_collect = { level = "allow", priority = 1 } # TODO option_if_let_else = { level = "allow", priority = 1 } # TODO redundant_pub_crate = { level = "allow", priority = 1 } # TODO significant_drop_in_scrutinee = { level = "allow", priority = 1 } # TODO diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 92e3de47..30c30c6e 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -1,7 +1,5 @@ use api::client::leave_room; -use ruma::{ - events::room::message::RoomMessageEventContent, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, RoomOrAliasId, -}; +use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId}; use tracing::{debug, error, info, warn}; use super::{super::Service, RoomModerationCommand}; @@ -124,9 +122,7 @@ async fn ban_room( .is_admin(local_user) .unwrap_or(true)) }) - }) - .collect::>() - { + }) { debug!( "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", &local_user, &room_id @@ -153,9 +149,7 @@ async fn ban_room( .is_admin(local_user) .unwrap_or(false)) }) - }) - .collect::>() - { + }) { debug!("Attempting leave for user {} in room {}", &local_user, &room_id); if let Err(e) = leave_room(&local_user, &room_id, None).await { error!( @@ -335,9 +329,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo .is_admin(local_user) .unwrap_or(true)) }) - }) - .collect::>() - { + }) { debug!( "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", &local_user, room_id @@ -364,9 +356,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo .is_admin(local_user) .unwrap_or(false)) }) - }) - .collect::>() - { + }) { debug!("Attempting leave for user {} in room {}", &local_user, &room_id); if let Err(e) = leave_room(&local_user, &room_id, None).await { error!( diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index e2ba4c9a..0792ed89 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1682,8 +1682,7 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { .filter_map(|event: serde_json::Value| event.get("sender").cloned()) .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) .filter_map(|sender| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()) - .collect::>(), + .map(|user| user.server_name().to_owned()), ); debug!("servers in remote_leave_room: {servers:?}"); diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index f79ee678..0120d5a6 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -319,7 +319,9 @@ impl Data { /// Returns an iterator of all joined members of a room. #[tracing::instrument(skip(self))] - pub(super) fn room_members<'a>(&'a self, room_id: &RoomId) -> Box> + 'a> { + pub(super) fn room_members<'a>( + &'a self, room_id: &RoomId, + ) -> Box> + Send + 'a> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 45e05c29..a0910c7c 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -292,7 +292,7 @@ impl Service { /// Returns an iterator over all joined members of a room. #[tracing::instrument(skip(self))] - pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> + '_ { + pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> + Send + '_ { self.db.room_members(room_id) } From 56a1b0e7615f83fde2b7a6fc909f835c5e0f7ee8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 11:20:03 +0000 Subject: [PATCH 0401/2091] restrict untripped clippies Signed-off-by: Jason Volk --- Cargo.toml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f6149086..dad0adeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -741,17 +741,12 @@ pedantic = "warn" ## some sadness doc_markdown = { level = "allow", priority = 1 } enum_glob_use = { level = "allow", priority = 1 } -error_impl_error = { level = "allow", priority = 1 } -expect_used = { level = "allow", priority = 1 } if_not_else = { level = "allow", priority = 1 } if_then_some_else_none = { level = "allow", priority = 1 } -implicit_return = { level = "allow", priority = 1 } inline_always = { level = "allow", priority = 1 } -map_err_ignore = { level = "allow", priority = 1 } missing_docs_in_private_items = { level = "allow", priority = 1 } missing_errors_doc = { level = "allow", priority = 1 } missing_panics_doc = { level = "allow", priority = 1 } -mod_module_files = { level = "allow", priority = 1 } module_name_repetitions = { level = "allow", priority = 1 } no_effect_underscore_binding = { level = "allow", priority = 1 } similar_names = { level = "allow", priority = 1 } @@ -765,8 +760,10 @@ perf = "warn" ################### #restriction = "warn" +allow_attributes = "warn" arithmetic_side_effects = "warn" as_conversions = "warn" +as_underscore = "warn" assertions_on_result_states = "warn" dbg_macro = "warn" default_union_representation = "warn" @@ -804,6 +801,7 @@ unnecessary_safety_doc = "warn" unnecessary_self_imports = "warn" unneeded_field_pattern = "warn" unseparated_literal_suffix = "warn" +#unwrap_used = "warn" # TODO verbose_file_reads = "warn" ################### From 5af880e4f4b5861cc484c628cd89a5c44d181c95 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 20:44:40 +0000 Subject: [PATCH 0402/2091] fix reference count on punned ColumnFamily Arc Signed-off-by: Jason Volk --- src/database/database.rs | 2 +- src/database/map.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/database.rs b/src/database/database.rs index 06a2d88f..44bb655c 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -6,7 +6,7 @@ use crate::{cork::Cork, maps, maps::Maps, Engine, Map}; pub struct Database { pub db: Arc, - pub map: Maps, + map: Maps, } impl Database { diff --git a/src/database/map.rs b/src/database/map.rs index 0b007307..1b35a72a 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -233,7 +233,7 @@ fn open(db: &Arc, name: &str) -> Result> { // closing the database (dropping `Engine`). Since `Arc` is a sibling // member along with this handle in `Map`, that is prevented. Ok(unsafe { - Arc::decrement_strong_count(cf_ptr); + Arc::increment_strong_count(cf_ptr); Arc::from_raw(cf_ptr) }) } From 113a27c1d5b4b3b373577c0124016da00eb5879c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 19:59:04 +0000 Subject: [PATCH 0403/2091] use usize for max_request_size config item Signed-off-by: Jason Volk --- src/api/client/media.rs | 4 ++-- src/api/router/request.rs | 7 +------ src/core/config/mod.rs | 4 ++-- src/router/layers.rs | 10 +--------- src/service/globals/mod.rs | 2 -- 5 files changed, 6 insertions(+), 21 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index cfb10c3d..44978caf 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -2,6 +2,7 @@ use std::{io::Cursor, sync::Arc, time::Duration}; +use conduit::{debug, error, utils::math::ruma_from_usize, warn}; use image::io::Reader as ImgReader; use ipaddress::IPAddress; use reqwest::Url; @@ -12,7 +13,6 @@ use ruma::api::client::{ get_media_preview, }, }; -use tracing::{debug, error, warn}; use webpage::HTML; use crate::{ @@ -44,7 +44,7 @@ pub(crate) async fn get_media_config_route( _body: Ruma, ) -> Result { Ok(get_media_config::v3::Response { - upload_size: services().globals.max_request_size().into(), + upload_size: ruma_from_usize(services().globals.config.max_request_size), }) } diff --git a/src/api/router/request.rs b/src/api/router/request.rs index 59639eaa..56c76619 100644 --- a/src/api/router/request.rs +++ b/src/api/router/request.rs @@ -29,12 +29,7 @@ pub(super) async fn from(request: hyper::Request) -> Result u64 { 10 } fn default_ip_lookup_strategy() -> u8 { 5 } -fn default_max_request_size() -> u32 { +fn default_max_request_size() -> usize { 20 * 1024 * 1024 // Default to 20 MB } diff --git a/src/router/layers.rs b/src/router/layers.rs index 4fe35164..db664b38 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -138,15 +138,7 @@ fn cors_layer(_server: &Server) -> CorsLayer { .max_age(Duration::from_secs(86400)) } -fn body_limit_layer(server: &Server) -> DefaultBodyLimit { - DefaultBodyLimit::max( - server - .config - .max_request_size - .try_into() - .expect("failed to convert max request size"), - ) -} +fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max(server.config.max_request_size) } #[allow(clippy::needless_pass_by_value)] #[tracing::instrument(skip_all)] diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 3c0e66c0..f28299b2 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -201,8 +201,6 @@ impl Service { pub fn server_name(&self) -> &ServerName { self.config.server_name.as_ref() } - pub fn max_request_size(&self) -> u32 { self.config.max_request_size } - pub fn max_fetch_prev_events(&self) -> u16 { self.config.max_fetch_prev_events } pub fn allow_registration(&self) -> bool { self.config.allow_registration } From 59c406230507be37482bdc457ddb2f2a630f3a20 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 19:03:15 +0000 Subject: [PATCH 0404/2091] set trivial/leaf spans to debug level Signed-off-by: Jason Volk --- src/core/pdu/mod.rs | 20 +++---- src/core/utils/content_disposition.rs | 2 +- src/core/utils/hash/sha256.rs | 2 +- src/router/request.rs | 2 +- src/service/account_data/mod.rs | 2 +- src/service/globals/data.rs | 2 +- src/service/globals/mod.rs | 4 +- src/service/pusher/mod.rs | 2 +- src/service/rooms/alias/mod.rs | 6 +- src/service/rooms/auth_chain/mod.rs | 4 +- src/service/rooms/directory/mod.rs | 8 +-- .../rooms/event_handler/signing_keys.rs | 2 +- src/service/rooms/lazy_loading/mod.rs | 8 +-- src/service/rooms/outlier/mod.rs | 2 +- src/service/rooms/pdu_metadata/mod.rs | 10 ++-- src/service/rooms/read_receipt/mod.rs | 6 +- src/service/rooms/search/mod.rs | 6 +- src/service/rooms/state/mod.rs | 12 ++-- src/service/rooms/state_accessor/mod.rs | 10 ++-- src/service/rooms/state_cache/data.rs | 48 +++++++-------- src/service/rooms/state_cache/mod.rs | 58 +++++++++---------- src/service/rooms/state_compressor/mod.rs | 4 +- src/service/rooms/timeline/mod.rs | 12 ++-- src/service/sending/data.rs | 2 +- src/service/sending/mod.rs | 22 +++---- 25 files changed, 128 insertions(+), 128 deletions(-) diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index c7d93608..0c54812e 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -64,7 +64,7 @@ pub struct PduEvent { } impl PduEvent { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> crate::Result<()> { self.unsigned = None; @@ -158,7 +158,7 @@ impl PduEvent { (self.redacts.clone(), self.content.clone()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_sync_room_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -183,7 +183,7 @@ impl PduEvent { } /// This only works for events that are also AnyRoomEvents. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_any_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -208,7 +208,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_room_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -233,7 +233,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_message_like_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -258,7 +258,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_state_event(&self) -> Raw { let mut json = json!({ "content": self.content, @@ -277,7 +277,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_sync_state_event(&self) -> Raw { let mut json = json!({ "content": self.content, @@ -295,7 +295,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_stripped_state_event(&self) -> Raw { let json = json!({ "content": self.content, @@ -307,7 +307,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_stripped_spacechild_state_event(&self) -> Raw { let json = json!({ "content": self.content, @@ -320,7 +320,7 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn to_member_event(&self) -> Raw> { let mut json = json!({ "content": self.content, diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index 1c2b066d..be17a731 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -66,7 +66,7 @@ pub fn content_disposition_type(content_type: &Option) -> &'static str { /// sanitises the file name for the Content-Disposition using /// `sanitize_filename` crate -#[tracing::instrument] +#[tracing::instrument(level = "debug")] pub fn sanitise_filename(filename: String) -> String { let options = sanitize_filename::Options { truncate: false, diff --git a/src/core/utils/hash/sha256.rs b/src/core/utils/hash/sha256.rs index 6a1f1879..b2e5a94c 100644 --- a/src/core/utils/hash/sha256.rs +++ b/src/core/utils/hash/sha256.rs @@ -1,6 +1,6 @@ use ring::{digest, digest::SHA256}; -#[tracing::instrument(skip_all)] +#[tracing::instrument(skip_all, level = "debug")] pub(super) fn hash(keys: &[&[u8]]) -> Vec { // We only hash the pdu's event ids, not the whole pdu let bytes = keys.join(&0xFF); diff --git a/src/router/request.rs b/src/router/request.rs index 2bdd06f8..49a31c29 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -9,7 +9,7 @@ use ruma::api::client::{ }; use tracing::{debug, error, trace}; -#[tracing::instrument(skip_all)] +#[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( State(server): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 66474370..69d2f799 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -44,7 +44,7 @@ impl Service { } /// Returns all changes to the account data that happened after `since`. - #[tracing::instrument(skip_all, name = "since")] + #[tracing::instrument(skip_all, name = "since", level = "debug")] pub fn changes_since( &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, ) -> Result>> { diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 083aec3c..254a3d9c 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -110,7 +110,7 @@ impl Data { Ok(()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { let userid_bytes = user_id.as_bytes().to_vec(); let mut userid_prefix = userid_bytes.clone(); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index f28299b2..11bfc88c 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -189,10 +189,10 @@ impl Service { #[inline] pub fn current_count(&self) -> Result { Ok(self.db.current_count()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn last_check_for_updates_id(&self) -> Result { self.db.last_check_for_updates_id() } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { self.db.update_check_for_updates_id(id) } pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 27b49c28..ea48ea7c 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -186,7 +186,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, user, ruleset, pdu))] + #[tracing::instrument(skip(self, user, ruleset, pdu), level = "debug")] pub fn get_actions<'a>( &self, user: &UserId, ruleset: &'a Ruleset, power_levels: &RoomPowerLevelsEventContent, pdu: &Raw, room_id: &RoomId, diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index d375561e..706bf2f8 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -89,19 +89,19 @@ impl Service { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { self.db.resolve_local_alias(alias) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { self.db.local_aliases_for_room(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn all_local_aliases<'a>(&'a self) -> Box> + 'a> { self.db.all_local_aliases() } diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 7b2a4f01..2919ea7e 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -174,13 +174,13 @@ impl Service { self.db.get_cached_eventid_authchain(key) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) -> Result<()> { self.db .cache_auth_chain(key, auth_chain.iter().copied().collect::>()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &Vec) -> Result<()> { self.db .cache_auth_chain(key, auth_chain.iter().copied().collect::>()) diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 3e60831c..23ec6b6b 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -22,15 +22,15 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn set_public(&self, room_id: &RoomId) -> Result<()> { self.db.set_public(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn set_not_public(&self, room_id: &RoomId) -> Result<()> { self.db.set_not_public(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_public_room(&self, room_id: &RoomId) -> Result { self.db.is_public_room(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn public_rooms(&self) -> impl Iterator> + '_ { self.db.public_rooms() } } diff --git a/src/service/rooms/event_handler/signing_keys.rs b/src/service/rooms/event_handler/signing_keys.rs index fdb2be0e..2fa5b0df 100644 --- a/src/service/rooms/event_handler/signing_keys.rs +++ b/src/service/rooms/event_handler/signing_keys.rs @@ -3,6 +3,7 @@ use std::{ time::{Duration, SystemTime}, }; +use conduit::{debug, error, info, trace, warn}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::federation::{ @@ -19,7 +20,6 @@ use ruma::{ }; use serde_json::value::RawValue as RawJsonValue; use tokio::sync::{RwLock, RwLockWriteGuard}; -use tracing::{debug, error, info, trace, warn}; use crate::{services, Error, Result}; diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 185cfd8c..96f623f2 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -39,7 +39,7 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn lazy_load_was_sent_before( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, ) -> Result { @@ -47,7 +47,7 @@ impl Service { .lazy_load_was_sent_before(user_id, device_id, room_id, ll_user) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn lazy_load_mark_sent( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, lazy_load: HashSet, count: PduCount, @@ -58,7 +58,7 @@ impl Service { .insert((user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), count), lazy_load); } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn lazy_load_confirm_delivery( &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, since: PduCount, ) -> Result<()> { @@ -77,7 +77,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId) -> Result<()> { self.db.lazy_load_reset(user_id, device_id, room_id) } diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index a7326f42..22bd2092 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -35,7 +35,7 @@ impl Service { pub fn get_pdu_outlier(&self, event_id: &EventId) -> Result> { self.db.get_outlier_pdu(event_id) } /// Append the PDU as an outlier. - #[tracing::instrument(skip(self, pdu))] + #[tracing::instrument(skip(self, pdu), level = "debug")] pub fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) -> Result<()> { self.db.add_pdu_outlier(event_id, pdu) } diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index cdb2fc29..05067aa8 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -38,7 +38,7 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self, from, to))] + #[tracing::instrument(skip(self, from, to), level = "debug")] pub fn add_relation(&self, from: PduCount, to: PduCount) -> Result<()> { match (from, to) { (PduCount::Normal(f), PduCount::Normal(t)) => self.db.add_relation(f, t), @@ -218,19 +218,19 @@ impl Service { }) } - #[tracing::instrument(skip(self, room_id, event_ids))] + #[tracing::instrument(skip_all, level = "debug")] pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) -> Result<()> { self.db.mark_as_referenced(room_id, event_ids) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> Result { self.db.is_event_referenced(room_id, event_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn mark_event_soft_failed(&self, event_id: &EventId) -> Result<()> { self.db.mark_event_soft_failed(event_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_event_soft_failed(&self, event_id: &EventId) -> Result { self.db.is_event_soft_failed(event_id) } } diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index e46027b7..9375276e 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -33,7 +33,7 @@ impl Service { /// Returns an iterator over the most recent read_receipts in a room that /// happened after the event with id `since`. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn readreceipts_since<'a>( &'a self, room_id: &RoomId, since: u64, ) -> impl Iterator)>> + 'a { @@ -41,13 +41,13 @@ impl Service { } /// Sets a private read marker at `count`. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) -> Result<()> { self.db.private_read_set(room_id, user_id, count) } /// Returns the private read marker. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { self.db.private_read_get(room_id, user_id) } diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 7573d218..082dd432 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -21,17 +21,17 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { self.db.index_pdu(shortroomid, pdu_id, message_body) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { self.db.deindex_pdu(shortroomid, pdu_id, message_body) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn search_pdus<'a>( &'a self, room_id: &RoomId, search_string: &str, ) -> Result> + 'a, Vec)>> { diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 7964a5fb..52ee89d1 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -116,7 +116,7 @@ impl Service { /// /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `eventid_statehash`. - #[tracing::instrument(skip(self, state_ids_compressed))] + #[tracing::instrument(skip(self, state_ids_compressed), level = "debug")] pub fn set_event_state( &self, event_id: &EventId, room_id: &RoomId, state_ids_compressed: Arc>, ) -> Result { @@ -184,7 +184,7 @@ impl Service { /// /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `eventid_statehash`. - #[tracing::instrument(skip(self, new_pdu))] + #[tracing::instrument(skip(self, new_pdu), level = "debug")] pub fn append_to_state(&self, new_pdu: &PduEvent) -> Result { let shorteventid = services() .rooms @@ -257,7 +257,7 @@ impl Service { } } - #[tracing::instrument(skip(self, invite_event))] + #[tracing::instrument(skip(self, invite_event), level = "debug")] pub fn calculate_invite_state(&self, invite_event: &PduEvent) -> Result>> { let mut state = Vec::new(); // Add recommended events @@ -313,7 +313,7 @@ impl Service { } /// Set the state hash to a new version, but does not update state_cache. - #[tracing::instrument(skip(self, mutex_lock))] + #[tracing::instrument(skip(self, mutex_lock), level = "debug")] pub fn set_room_state( &self, room_id: &RoomId, @@ -324,7 +324,7 @@ impl Service { } /// Returns the room's version. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn get_room_version(&self, room_id: &RoomId) -> Result { let create_event = services() .rooms @@ -365,7 +365,7 @@ impl Service { } /// This fetches auth events from the current state. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn get_auth_events( &self, room_id: &RoomId, kind: &TimelineEventType, sender: &UserId, state_key: Option<&str>, content: &serde_json::value::RawValue, diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index e8554500..a3567857 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -81,7 +81,7 @@ impl crate::Service for Service { impl Service { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn state_full_ids(&self, shortstatehash: u64) -> Result>> { self.db.state_full_ids(shortstatehash).await } @@ -92,7 +92,7 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn state_get_id( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, ) -> Result>> { @@ -286,14 +286,14 @@ impl Service { pub fn pdu_shortstatehash(&self, event_id: &EventId) -> Result> { self.db.pdu_shortstatehash(event_id) } /// Returns the full room state. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn room_state_full(&self, room_id: &RoomId) -> Result>> { self.db.room_state_full(room_id).await } /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_state_get_id( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result>> { @@ -302,7 +302,7 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_state_get( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result>> { diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 0120d5a6..2b9fbe94 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -213,7 +213,7 @@ impl Data { Ok(()) } - #[tracing::instrument(skip(self, room_id, appservice))] + #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] pub(super) fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { let maybe = self .appservice_in_room_cache @@ -249,7 +249,7 @@ impl Data { } /// Makes a user forget a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -266,7 +266,7 @@ impl Data { } /// Returns an iterator of all servers participating in this room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_servers<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { @@ -286,7 +286,7 @@ impl Data { })) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result { let mut key = server.as_bytes().to_vec(); key.push(0xFF); @@ -297,7 +297,7 @@ impl Data { /// Returns an iterator of all rooms a server participates in (as far as we /// know). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn server_rooms<'a>( &'a self, server: &ServerName, ) -> Box> + 'a> { @@ -318,7 +318,7 @@ impl Data { } /// Returns an iterator of all joined members of a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_members<'a>( &'a self, room_id: &RoomId, ) -> Box> + Send + 'a> { @@ -350,7 +350,7 @@ impl Data { /// Returns an iterator of all our local joined users in a room who are /// active (not deactivated, not guest) - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn active_local_users_in_room<'a>( &'a self, room_id: &RoomId, ) -> Box + 'a> { @@ -361,7 +361,7 @@ impl Data { } /// Returns the number of users which are currently in a room - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_joined_count(&self, room_id: &RoomId) -> Result> { self.roomid_joinedcount .get(room_id.as_bytes())? @@ -370,7 +370,7 @@ impl Data { } /// Returns the number of users which are currently invited to a room - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.roomid_invitedcount .get(room_id.as_bytes())? @@ -379,7 +379,7 @@ impl Data { } /// Returns an iterator over all User IDs who ever joined a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_useroncejoined<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { @@ -404,7 +404,7 @@ impl Data { } /// Returns an iterator over all invited members of a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn room_members_invited<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { @@ -428,7 +428,7 @@ impl Data { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); @@ -443,7 +443,7 @@ impl Data { }) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); @@ -456,7 +456,7 @@ impl Data { } /// Returns an iterator over all rooms this user joined. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn rooms_joined(&self, user_id: &UserId) -> Box> + '_> { Box::new( self.userroomid_joined @@ -476,7 +476,7 @@ impl Data { } /// Returns an iterator over all rooms a user was invited to. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn rooms_invited<'a>(&'a self, user_id: &UserId) -> StrippedStateEventIter<'a> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -503,7 +503,7 @@ impl Data { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn invite_state( &self, user_id: &UserId, room_id: &RoomId, ) -> Result>>> { @@ -522,7 +522,7 @@ impl Data { .transpose() } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn left_state( &self, user_id: &UserId, room_id: &RoomId, ) -> Result>>> { @@ -542,7 +542,7 @@ impl Data { } /// Returns an iterator over all rooms a user left. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn rooms_left<'a>(&'a self, user_id: &UserId) -> AnySyncStateEventIter<'a> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -569,7 +569,7 @@ impl Data { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -578,7 +578,7 @@ impl Data { Ok(self.roomuseroncejoinedids.get(&userroom_id)?.is_some()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -587,7 +587,7 @@ impl Data { Ok(self.userroomid_joined.get(&userroom_id)?.is_some()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -596,7 +596,7 @@ impl Data { Ok(self.userroomid_invitestate.get(&userroom_id)?.is_some()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); @@ -605,7 +605,7 @@ impl Data { Ok(self.userroomid_leftstate.get(&userroom_id)?.is_some()) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn servers_invite_via<'a>( &'a self, room_id: &RoomId, ) -> Box> + 'a> { @@ -631,7 +631,7 @@ impl Data { ) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub(super) fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) -> Result<()> { let mut prev_servers = self .servers_invite_via(room_id) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index a0910c7c..30fd7bf4 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -219,10 +219,10 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, room_id))] + #[tracing::instrument(skip(self, room_id), level = "debug")] pub fn update_joined_count(&self, room_id: &RoomId) -> Result<()> { self.db.update_joined_count(room_id) } - #[tracing::instrument(skip(self, room_id, appservice))] + #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] pub fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { self.db.appservice_in_room(room_id, appservice) } @@ -230,7 +230,7 @@ impl Service { /// Direct DB function to directly mark a user as left. It is not /// recommended to use this directly. You most likely should use /// `update_membership` instead - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { self.db.mark_as_left(user_id, room_id) } @@ -238,35 +238,35 @@ impl Service { /// Direct DB function to directly mark a user as joined. It is not /// recommended to use this directly. You most likely should use /// `update_membership` instead - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { self.db.mark_as_joined(user_id, room_id) } /// Makes a user forget a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { self.db.forget(room_id, user_id) } /// Returns an iterator of all servers participating in this room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_servers(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_servers(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result { self.db.server_in_room(server, room_id) } /// Returns an iterator of all rooms a server participates in (as far as we /// know). - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn server_rooms(&self, server: &ServerName) -> impl Iterator> + '_ { self.db.server_rooms(server) } /// Returns true if server can see user by sharing at least one room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn server_sees_user(&self, server: &ServerName, user_id: &UserId) -> Result { Ok(self .server_rooms(server) @@ -275,7 +275,7 @@ impl Service { } /// Returns true if user_a and user_b share at least one room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> Result { // Minimize number of point-queries by iterating user with least nr rooms let (a, b) = if self.rooms_joined(user_a).count() < self.rooms_joined(user_b).count() { @@ -291,23 +291,23 @@ impl Service { } /// Returns an iterator over all joined members of a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> + Send + '_ { self.db.room_members(room_id) } /// Returns the number of users which are currently in a room - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_joined_count(&self, room_id: &RoomId) -> Result> { self.db.room_joined_count(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] /// Returns an iterator of all our local users in the room, even if they're /// deactivated/guests pub fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> impl Iterator + 'a { self.db.local_users_in_room(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] /// Returns an iterator of all our local joined users in a room who are /// active (not deactivated, not guest) pub fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> impl Iterator + 'a { @@ -315,80 +315,80 @@ impl Service { } /// Returns the number of users which are currently invited to a room - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.db.room_invited_count(room_id) } /// Returns an iterator over all User IDs who ever joined a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_useroncejoined(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_useroncejoined(room_id) } /// Returns an iterator over all invited members of a room. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.room_members_invited(room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { self.db.get_invite_count(room_id, user_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { self.db.get_left_count(room_id, user_id) } /// Returns an iterator over all rooms this user joined. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn rooms_joined(&self, user_id: &UserId) -> impl Iterator> + '_ { self.db.rooms_joined(user_id) } /// Returns an iterator over all rooms a user was invited to. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn rooms_invited( &self, user_id: &UserId, ) -> impl Iterator>)>> + '_ { self.db.rooms_invited(user_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn invite_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>> { self.db.invite_state(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>> { self.db.left_state(user_id, room_id) } /// Returns an iterator over all rooms a user left. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn rooms_left( &self, user_id: &UserId, ) -> impl Iterator>)>> + '_ { self.db.rooms_left(user_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.once_joined(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.is_joined(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.is_invited(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.is_left(user_id, room_id) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn servers_invite_via(&self, room_id: &RoomId) -> impl Iterator> + '_ { self.db.servers_invite_via(room_id) } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 197efbea..4b4ea7d4 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -77,7 +77,7 @@ impl crate::Service for Service { impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn load_shortstatehash_info(&self, shortstatehash: u64) -> ShortStateInfoResult { if let Some(r) = self .stateinfo_cache @@ -162,7 +162,7 @@ impl Service { /// for this layer /// * `parent_states` - A stack with info on shortstatehash, full state, /// added diff and removed diff for each parent layer - #[tracing::instrument(skip(self, statediffnew, statediffremoved, diff_to_sibling, parent_states))] + #[tracing::instrument(skip(self, statediffnew, statediffremoved, diff_to_sibling, parent_states), level = "debug")] pub fn save_state_from_diff( &self, shortstatehash: u64, statediffnew: Arc>, statediffremoved: Arc>, diff_to_sibling: usize, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 70f4423c..db7af004 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -99,7 +99,7 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn first_pdu_in_room(&self, room_id: &RoomId) -> Result>> { self.all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id)? .next() @@ -107,7 +107,7 @@ impl Service { .transpose() } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result>> { self.all_pdus(user_id!("@placeholder:conduwuit.placeholder"), room_id)? .last() @@ -115,7 +115,7 @@ impl Service { .transpose() } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { self.db.last_timeline_count(sender_user, room_id) } @@ -213,7 +213,7 @@ impl Service { } /// Removes a pdu and creates a new one with the same id. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, pdu: &PduEvent) -> Result<()> { self.db.replace_pdu(pdu_id, pdu_json, pdu) } @@ -1030,7 +1030,7 @@ impl Service { /// Returns an iterator over all events and their tokens in a room that /// happened before the event with id `until` in reverse-chronological /// order. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn pdus_until<'a>( &'a self, user_id: &UserId, room_id: &RoomId, until: PduCount, ) -> Result> + 'a> { @@ -1039,7 +1039,7 @@ impl Service { /// Returns an iterator over all events and their token in a room that /// happened after the event with id `from` in chronological order. - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn pdus_after<'a>( &'a self, user_id: &UserId, room_id: &RoomId, from: PduCount, ) -> Result> + 'a> { diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index ed579817..65725618 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -136,7 +136,7 @@ impl Data { } } -#[tracing::instrument(skip(key))] +#[tracing::instrument(skip(key), level = "debug")] fn parse_servercurrentevent(key: &[u8], value: Vec) -> Result<(Destination, SendingEvent)> { // Appservices start with a plus Ok::<_, Error>(if key.starts_with(b"+") { diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 0dcebcec..a18240bd 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -93,7 +93,7 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip(self, pdu_id, user, pushkey))] + #[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")] pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { let dest = Destination::Push(user.to_owned(), pushkey); let event = SendingEvent::Pdu(pdu_id.to_owned()); @@ -106,7 +106,7 @@ impl Service { }) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn send_pdu_appservice(&self, appservice_id: String, pdu_id: Vec) -> Result<()> { let dest = Destination::Appservice(appservice_id); let event = SendingEvent::Pdu(pdu_id); @@ -119,7 +119,7 @@ impl Service { }) } - #[tracing::instrument(skip(self, room_id, pdu_id))] + #[tracing::instrument(skip(self, room_id, pdu_id), level = "debug")] pub fn send_pdu_room(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result<()> { let servers = services() .rooms @@ -131,7 +131,7 @@ impl Service { self.send_pdu_servers(servers, pdu_id) } - #[tracing::instrument(skip(self, servers, pdu_id))] + #[tracing::instrument(skip(self, servers, pdu_id), level = "debug")] pub fn send_pdu_servers>(&self, servers: I, pdu_id: &[u8]) -> Result<()> { let requests = servers .into_iter() @@ -155,7 +155,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, server, serialized))] + #[tracing::instrument(skip(self, server, serialized), level = "debug")] pub fn send_edu_server(&self, server: &ServerName, serialized: Vec) -> Result<()> { let dest = Destination::Normal(server.to_owned()); let event = SendingEvent::Edu(serialized); @@ -168,7 +168,7 @@ impl Service { }) } - #[tracing::instrument(skip(self, room_id, serialized))] + #[tracing::instrument(skip(self, room_id, serialized), level = "debug")] pub fn send_edu_room(&self, room_id: &RoomId, serialized: Vec) -> Result<()> { let servers = services() .rooms @@ -180,7 +180,7 @@ impl Service { self.send_edu_servers(servers, serialized) } - #[tracing::instrument(skip(self, servers, serialized))] + #[tracing::instrument(skip(self, servers, serialized), level = "debug")] pub fn send_edu_servers>(&self, servers: I, serialized: Vec) -> Result<()> { let requests = servers .into_iter() @@ -205,7 +205,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, room_id))] + #[tracing::instrument(skip(self, room_id), level = "debug")] pub fn flush_room(&self, room_id: &RoomId) -> Result<()> { let servers = services() .rooms @@ -217,7 +217,7 @@ impl Service { self.flush_servers(servers) } - #[tracing::instrument(skip(self, servers))] + #[tracing::instrument(skip(self, servers), level = "debug")] pub fn flush_servers>(&self, servers: I) -> Result<()> { let requests = servers.into_iter().map(Destination::Normal); for dest in requests { @@ -255,7 +255,7 @@ impl Service { /// Cleanup event data /// Used for instance after we remove an appservice registration - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub fn cleanup_events(&self, appservice_id: String) -> Result<()> { self.db .delete_all_requests_for(&Destination::Appservice(appservice_id))?; @@ -271,7 +271,7 @@ impl Service { } impl Destination { - #[tracing::instrument(skip(self))] + #[must_use] pub fn get_prefix(&self) -> Vec { let mut prefix = match self { Self::Appservice(server) => { From 8e3be6feb0ff01d83122c26fcf071fa2922ff2bd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 19:43:41 +0000 Subject: [PATCH 0405/2091] slightly optimize Destination::get_prefix() Signed-off-by: Jason Volk --- src/service/sending/mod.rs | 44 +++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index a18240bd..d7a9c0fc 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -273,27 +273,45 @@ impl Service { impl Destination { #[must_use] pub fn get_prefix(&self) -> Vec { - let mut prefix = match self { - Self::Appservice(server) => { - let mut p = b"+".to_vec(); + match self { + Self::Normal(server) => { + let len = server.as_bytes().len().saturating_add(1); + + let mut p = Vec::with_capacity(len); p.extend_from_slice(server.as_bytes()); + p.push(0xFF); + p + }, + Self::Appservice(server) => { + let sigil = b"+"; + let len = sigil + .len() + .saturating_add(server.as_bytes().len()) + .saturating_add(1); + + let mut p = Vec::with_capacity(len); + p.extend_from_slice(sigil); + p.extend_from_slice(server.as_bytes()); + p.push(0xFF); p }, Self::Push(user, pushkey) => { - let mut p = b"$".to_vec(); + let sigil = b"$"; + let len = sigil + .len() + .saturating_add(user.as_bytes().len()) + .saturating_add(1) + .saturating_add(pushkey.as_bytes().len()) + .saturating_add(1); + + let mut p = Vec::with_capacity(len); + p.extend_from_slice(sigil); p.extend_from_slice(user.as_bytes()); p.push(0xFF); p.extend_from_slice(pushkey.as_bytes()); + p.push(0xFF); p }, - Self::Normal(server) => { - let mut p = Vec::new(); - p.extend_from_slice(server.as_bytes()); - p - }, - }; - prefix.push(0xFF); - - prefix + } } } From 8296e0ed676f9a4f152de9d2f0e4ae0c1688341d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 02:59:58 +0000 Subject: [PATCH 0406/2091] slightly optimize command completer Signed-off-by: Jason Volk --- src/admin/handler.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 1466c040..22ec81c5 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -132,11 +132,13 @@ fn parse_admin_command(command_line: &str) -> Result { } fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { - let mut ret = Vec::::new(); let argv = parse_command_line(line); + let mut ret = Vec::::with_capacity(argv.len().saturating_add(1)); + 'token: for token in argv.into_iter().skip(1) { - let mut choice = Vec::new(); let cmd_ = cmd.clone(); + let mut choice = Vec::new(); + for sub in cmd_.get_subcommands() { let name = sub.get_name(); if *name == token { @@ -144,20 +146,20 @@ fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { ret.push(token); cmd.clone_from(sub); continue 'token; - } - if name.starts_with(&token) { + } else if name.starts_with(&token) { // partial match; add to choices choice.push(name); } } - if choice.is_empty() { + if choice.len() == 1 { + // One choice. Add extra space because it's complete + let choice = *choice.first().expect("only choice"); + ret.push(choice.to_owned()); + ret.push(String::new()); + } else if choice.is_empty() { // Nothing found, return original string ret.push(token); - } else if choice.len() == 1 { - // One choice. Add extra space because it's complete - ret.push((*choice.first().expect("only choice")).to_owned()); - ret.push(String::new()); } else { // Find the common prefix ret.push(common_prefix(&choice).into()); @@ -168,9 +170,8 @@ fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { } // Return from no completion. Needs a space though. - let mut ret = ret.join(" "); - ret.push(' '); - ret + ret.push(String::new()); + ret.join(" ") } // Parse chat messages from the admin room into an AdminCommand object From eb5556e74ee3d98d1cd0fcf78b908d0e3c482b31 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 7 Jul 2024 20:58:26 +0000 Subject: [PATCH 0407/2091] additional tracing spans / log cleanup. Signed-off-by: Jason Volk --- src/database/engine.rs | 18 +++++++++++------- src/database/maps.rs | 1 + 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 53e39abc..d02ecf58 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -30,6 +30,7 @@ pub struct Engine { pub(crate) type Db = DBWithThreadMode; impl Engine { + #[tracing::instrument(skip_all)] pub(crate) fn open(server: &Arc) -> Result> { let config = &server.config; let cache_capacity_bytes = config.db_cache_capacity_mb * 1024.0 * 1024.0; @@ -51,7 +52,7 @@ impl Engine { if config.rocksdb_repair { warn!("Starting database repair. This may take a long time..."); if let Err(e) = Db::repair(&db_opts, &config.database_path) { - error!("Repair failed: {:?}", e); + error!("Repair failed: {e:?}"); } } @@ -76,9 +77,9 @@ impl Engine { let db = res.or_else(or_else)?; info!( - "Opened database at sequence number {} in {:?}", - db.latest_sequence_number(), - load_time.elapsed() + sequence = %db.latest_sequence_number(), + time = ?load_time.elapsed(), + "Opened database." ); Ok(Arc::new(Self { @@ -93,15 +94,16 @@ impl Engine { })) } + #[tracing::instrument(skip(self))] pub(crate) fn open_cf(&self, name: &str) -> Result>> { let mut cfs = self.cfs.lock().expect("locked"); if !cfs.contains(name) { - debug!("Creating new column family in database: {}", name); + debug!("Creating new column family in database: {name}"); let mut col_cache = self.col_cache.write().expect("locked"); let opts = cf_options(&self.server.config, name, self.opts.clone(), &mut col_cache); if let Err(e) = self.db.create_cf(name, &opts) { - error!("Failed to create new column family: {e}"); + error!(?name, "Failed to create new column family: {e}"); return or_else(e); } @@ -148,18 +150,20 @@ impl Engine { )?; for (name, cache) in &*self.col_cache.read().expect("locked") { - writeln!(res, "{} cache: {:.2} MiB", name, mibs(u64::try_from(cache.get_usage())?))?; + writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?; } Ok(res) } + #[tracing::instrument(skip(self), level = "debug")] pub fn cleanup(&self) -> Result<()> { debug!("Running flush_opt"); let flushoptions = rocksdb::FlushOptions::default(); result(DBCommon::flush_opt(&self.db, &flushoptions)) } + #[tracing::instrument(skip(self))] pub fn backup(&self) -> Result<(), Box> { let config = &self.server.config; let path = config.database_backup_path.as_ref(); diff --git a/src/database/maps.rs b/src/database/maps.rs index 1e09041c..de78eaed 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -8,6 +8,7 @@ pub type Maps = BTreeMap>; pub(crate) fn open(db: &Arc) -> Result { open_list(db, MAPS) } +#[tracing::instrument(skip_all, level = "debug")] pub(crate) fn open_list(db: &Arc, maps: &[&str]) -> Result { Ok(maps .iter() From 100c6f572bee8fac288290674cfa16eb62aec71a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 01:28:38 +0000 Subject: [PATCH 0408/2091] trim unused dependencies in member crates Signed-off-by: Jason Volk --- Cargo.lock | 9 --------- src/admin/Cargo.toml | 5 ----- src/api/Cargo.toml | 1 - src/database/Cargo.toml | 1 - src/router/Cargo.toml | 2 -- 5 files changed, 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7152c66d..36da41e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,14 +612,9 @@ dependencies = [ "clap", "conduit_api", "conduit_core", - "conduit_database", "conduit_service", - "futures-util", "log", - "loole", - "regex", "ruma", - "serde", "serde_json", "serde_yaml", "tokio", @@ -654,7 +649,6 @@ dependencies = [ "serde_html_form", "serde_json", "sha-1", - "thiserror", "tokio", "tracing", "webpage", @@ -706,7 +700,6 @@ version = "0.4.5" dependencies = [ "conduit_core", "log", - "ruma", "rust-rocksdb-uwu", "tokio", "tracing", @@ -724,14 +717,12 @@ dependencies = [ "conduit_admin", "conduit_api", "conduit_core", - "conduit_database", "conduit_service", "http 1.1.0", "http-body-util", "hyper 1.4.0", "hyper-util", "log", - "regex", "ruma", "sd-notify", "sentry", diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 97de55ff..f84fbb92 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -29,15 +29,10 @@ release_max_log_level = [ clap.workspace = true conduit-api.workspace = true conduit-core.workspace = true -conduit-database.workspace = true conduit-service.workspace = true -futures-util.workspace = true log.workspace = true -loole.workspace = true -regex.workspace = true ruma.workspace = true serde_json.workspace = true -serde.workspace = true serde_yaml.workspace = true tokio.workspace = true tracing-subscriber.workspace = true diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index a27924c6..2f8bce54 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -56,7 +56,6 @@ serde_html_form.workspace = true serde_json.workspace = true serde.workspace = true sha-1.workspace = true -thiserror.workspace = true tokio.workspace = true tracing.workspace = true webpage.workspace = true diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 6e95236b..8b0d3fc3 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -37,7 +37,6 @@ zstd_compression = [ [dependencies] conduit-core.workspace = true log.workspace = true -ruma.workspace = true rust-rocksdb.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index ecb09dd7..5312984a 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -54,7 +54,6 @@ axum.workspace = true conduit-admin.workspace = true conduit-api.workspace = true conduit-core.workspace = true -conduit-database.workspace = true conduit-service.workspace = true log.workspace = true tokio.workspace = true @@ -65,7 +64,6 @@ http-body-util.workspace = true http.workspace = true hyper.workspace = true hyper-util.workspace = true -regex.workspace = true ruma.workspace = true sentry.optional = true sentry-tower.optional = true From 23a9055199a3e7fb876177ecad8c99d4e57b2513 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 16:10:18 +0000 Subject: [PATCH 0409/2091] relax and de-clutter let_underscore_must_use Signed-off-by: Jason Volk --- Cargo.toml | 1 - src/admin/mod.rs | 2 -- src/api/client/sync.rs | 11 +++-------- src/router/request.rs | 1 - src/router/run.rs | 1 - src/router/serve/unix.rs | 1 - src/service/admin/console.rs | 4 ---- src/service/mod.rs | 1 - src/service/services.rs | 12 ++---------- 9 files changed, 5 insertions(+), 29 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dad0adeb..1ef39370 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -777,7 +777,6 @@ fn_to_numeric_cast_any = "warn" format_push_string = "warn" get_unwrap = "warn" impl_trait_in_params = "warn" -let_underscore_must_use = "warn" let_underscore_untyped = "warn" lossy_float_literal = "warn" mem_forget = "warn" diff --git a/src/admin/mod.rs b/src/admin/mod.rs index f2e35d80..6a47bc74 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -28,7 +28,6 @@ mod_ctor! {} mod_dtor! {} /// Install the admin command handler -#[allow(clippy::let_underscore_must_use)] pub async fn init() { _ = services() .admin @@ -45,7 +44,6 @@ pub async fn init() { } /// Uninstall the admin command handler -#[allow(clippy::let_underscore_must_use)] pub async fn fini() { _ = services().admin.handle.write().await.take(); _ = services() diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 9ef1fab0..59813cb6 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -307,10 +307,7 @@ pub(crate) async fn sync_events_route( duration = Duration::from_secs(30); } - #[allow(clippy::let_underscore_must_use)] - { - _ = tokio::time::timeout(duration, watcher).await; - } + _ = tokio::time::timeout(duration, watcher).await; } Ok(response) @@ -1567,10 +1564,8 @@ pub(crate) async fn sync_events_v4_route( if duration.as_secs() > 30 { duration = Duration::from_secs(30); } - #[allow(clippy::let_underscore_must_use)] - { - _ = tokio::time::timeout(duration, watcher).await; - } + + _ = tokio::time::timeout(duration, watcher).await; } Ok(sync_events::v4::Response { diff --git a/src/router/request.rs b/src/router/request.rs index 49a31c29..079106ef 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -71,7 +71,6 @@ fn handle_result( } } -#[allow(clippy::unnecessary_wraps)] fn handle_result_403( _method: &Method, _uri: &Uri, result: &axum::response::Response, ) -> Result { diff --git a/src/router/run.rs b/src/router/run.rs index 4f7853d8..3e09823a 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -17,7 +17,6 @@ use crate::{layers, serve}; /// Main loop base #[tracing::instrument(skip_all)] -#[allow(clippy::let_underscore_must_use)] // various of these are intended pub(crate) async fn run(server: Arc) -> Result<(), Error> { let app = layers::build(&server)?; diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 6c406d28..8373b749 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -54,7 +54,6 @@ pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broad Ok(()) } -#[allow(clippy::let_underscore_must_use)] async fn accept( server: &Arc, listener: &UnixListener, tasks: &mut JoinSet<()>, mut app: MakeService, builder: server::conn::auto::Builder, conn: (UnixStream, SocketAddr), diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 0a200cae..27a4b6c7 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -45,7 +45,6 @@ impl Console { } } - #[allow(clippy::let_underscore_must_use)] pub async fn start(self: &Arc) { let mut worker_join = self.worker_join.lock().expect("locked"); if worker_join.is_none() { @@ -54,7 +53,6 @@ impl Console { } } - #[allow(clippy::let_underscore_must_use)] pub async fn close(self: &Arc) { self.interrupt(); let Some(worker_join) = self.worker_join.lock().expect("locked").take() else { @@ -113,7 +111,6 @@ impl Console { self.worker_join.lock().expect("locked").take(); } - #[allow(clippy::let_underscore_must_use)] async fn readline(self: &Arc) -> Result { let _suppression = log::Suppress::new(&services().server); @@ -138,7 +135,6 @@ impl Console { result } - #[allow(clippy::let_underscore_must_use)] async fn handle(self: Arc, line: String) { if line.trim().is_empty() { return; diff --git a/src/service/mod.rs b/src/service/mod.rs index 9c83b25c..4b19073d 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -37,7 +37,6 @@ conduit::mod_dtor! {} static SERVICES: RwLock> = RwLock::new(None); -#[allow(clippy::let_underscore_must_use)] pub async fn init(server: &Arc) -> Result<()> { let d = Arc::new(Database::open(server).await?); let s = Box::new(Services::build(server.clone(), d)?); diff --git a/src/service/services.rs b/src/service/services.rs index 0ba86693..aeed8204 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -132,11 +132,7 @@ impl Services { if self.globals.allow_check_for_updates() { let handle = globals::updates::start_check_for_updates_task(); - - #[allow(clippy::let_underscore_must_use)] // needed for shutdown - { - _ = self.globals.updates_handle.lock().await.insert(handle); - } + _ = self.globals.updates_handle.lock().await.insert(handle); } debug_info!("Services startup complete."); @@ -159,11 +155,7 @@ impl Services { debug!("Waiting for update worker..."); if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { updates_handle.abort(); - - #[allow(clippy::let_underscore_must_use)] - { - _ = updates_handle.await; - } + _ = updates_handle.await; } for (name, service) in &self.service { From 51df946911204536fad2940d5d35904510ad3bb6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 16:13:31 +0000 Subject: [PATCH 0410/2091] de-branch duration limit statement Signed-off-by: Jason Volk --- src/api/client/sync.rs | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 59813cb6..45a1c75b 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1,4 +1,5 @@ use std::{ + cmp, cmp::Ordering, collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}, time::Duration, @@ -302,11 +303,8 @@ pub(crate) async fn sync_events_route( { // Hang a few seconds so requests are not spammed // Stop hanging if new info arrives - let mut duration = body.timeout.unwrap_or_default(); - if duration.as_secs() > 30 { - duration = Duration::from_secs(30); - } - + let default = Duration::from_secs(30); + let duration = cmp::min(body.timeout.unwrap_or(default), default); _ = tokio::time::timeout(duration, watcher).await; } @@ -1560,11 +1558,8 @@ pub(crate) async fn sync_events_v4_route( { // Hang a few seconds so requests are not spammed // Stop hanging if new info arrives - let mut duration = body.timeout.unwrap_or(Duration::from_secs(30)); - if duration.as_secs() > 30 { - duration = Duration::from_secs(30); - } - + let default = Duration::from_secs(30); + let duration = cmp::min(body.timeout.unwrap_or(default), default); _ = tokio::time::timeout(duration, watcher).await; } From 59d86d364137617e850a6ef6563aa954c8188587 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 16:30:59 +0000 Subject: [PATCH 0411/2091] restrict clippy::string_slice Signed-off-by: Jason Volk --- Cargo.toml | 1 + src/core/config/proxy.rs | 1 + src/core/utils/html.rs | 1 + src/core/utils/string.rs | 5 +++-- src/service/sending/resolve.rs | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1ef39370..0d3d59fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -790,6 +790,7 @@ rest_pat_in_fully_bound_structs = "warn" semicolon_outside_block = "warn" str_to_string = "warn" string_lit_chars_any = "warn" +string_slice = "warn" string_to_string = "warn" suspicious_xor_used_as_pow = "warn" tests_outside_test_module = "warn" diff --git a/src/core/config/proxy.rs b/src/core/config/proxy.rs index d823e5e4..48f883c6 100644 --- a/src/core/config/proxy.rs +++ b/src/core/config/proxy.rs @@ -127,6 +127,7 @@ impl WildCardedDomain { impl std::str::FromStr for WildCardedDomain { type Err = std::convert::Infallible; + #[allow(clippy::string_slice)] fn from_str(s: &str) -> Result { // maybe do some domain validation? Ok(if s.starts_with("*.") { diff --git a/src/core/utils/html.rs b/src/core/utils/html.rs index 938e50ec..fe07b2dd 100644 --- a/src/core/utils/html.rs +++ b/src/core/utils/html.rs @@ -6,6 +6,7 @@ pub struct Escape<'a>(pub &'a str); /// Copied from librustdoc: /// * +#[allow(clippy::string_slice)] impl fmt::Display for Escape<'_> { fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result { // Because the internet is always right, turns out there's not that many diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 1f2a6572..ec423d53 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -9,12 +9,13 @@ pub const EMPTY: &str = ""; /// common_prefix(&input) == "con"; /// ``` #[must_use] +#[allow(clippy::string_slice)] pub fn common_prefix<'a>(choice: &'a [&str]) -> &'a str { choice.first().map_or(EMPTY, move |best| { choice.iter().skip(1).fold(*best, |best, choice| { &best[0..choice - .chars() - .zip(best.chars()) + .char_indices() + .zip(best.char_indices()) .take_while(|&(a, b)| a == b) .count()] }) diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 77311006..d38509ba 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -484,6 +484,7 @@ impl FedDest { } #[inline] + #[allow(clippy::string_slice)] fn port(&self) -> Option { match &self { Self::Literal(addr) => Some(addr.port()), From 7ba0777bd304e09d8f34ac0e2e5714409d0772c7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 02:50:10 +0000 Subject: [PATCH 0412/2091] move RumaResponse out of core Error; cleanup Error conversions. Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/api/Cargo.toml | 1 + src/api/client/unstable.rs | 4 +- src/api/mod.rs | 3 +- src/api/router/mod.rs | 3 +- src/api/router/response.rs | 22 +++++ src/core/error.rs | 192 +++++++++++++++++-------------------- src/core/mod.rs | 2 +- src/router/request.rs | 4 +- 9 files changed, 122 insertions(+), 110 deletions(-) create mode 100644 src/api/router/response.rs diff --git a/Cargo.lock b/Cargo.lock index 36da41e0..dd0c0436 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -637,6 +637,7 @@ dependencies = [ "futures-util", "hmac", "http 1.1.0", + "http-body-util", "hyper 1.4.0", "image", "ipaddress", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 2f8bce54..45cae73d 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -44,6 +44,7 @@ conduit-service.workspace = true futures-util.workspace = true hmac.workspace = true http.workspace = true +http-body-util.workspace = true hyper.workspace = true image.workspace = true ipaddress.workspace = true diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 77cac0fa..e39db94e 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -1,12 +1,12 @@ use axum_client_ip::InsecureClientIp; -use conduit::{warn, RumaResponse}; +use conduit::warn; use ruma::{ api::client::{error::ErrorKind, membership::mutual_rooms, room::get_summary}, events::room::member::MembershipState, OwnedRoomId, }; -use crate::{services, Error, Result, Ruma}; +use crate::{services, Error, Result, Ruma, RumaResponse}; /// # `GET /_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms` /// diff --git a/src/api/mod.rs b/src/api/mod.rs index 8e30a518..0395da7a 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -9,7 +9,8 @@ extern crate conduit_service as service; pub(crate) use conduit::{debug_info, debug_warn, utils, Error, Result}; pub(crate) use service::{pdu::PduEvent, services, user_is_local}; -pub(crate) use crate::router::{Ruma, RumaResponse}; +pub(crate) use self::router::Ruma; +pub use self::router::RumaResponse; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/api/router/mod.rs b/src/api/router/mod.rs index 2c439d65..f167a606 100644 --- a/src/api/router/mod.rs +++ b/src/api/router/mod.rs @@ -1,12 +1,12 @@ mod auth; mod handler; mod request; +mod response; use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, BytesMut}; -pub(super) use conduit::error::RumaResponse; use conduit::{debug, debug_warn, trace, warn}; use ruma::{ api::{client::error::ErrorKind, IncomingRequest}, @@ -14,6 +14,7 @@ use ruma::{ }; pub(super) use self::handler::RouterExt; +pub use self::response::RumaResponse; use self::{auth::Auth, request::Request}; use crate::{service::appservice::RegistrationInfo, services, Error, Result}; diff --git a/src/api/router/response.rs b/src/api/router/response.rs new file mode 100644 index 00000000..38e58ba9 --- /dev/null +++ b/src/api/router/response.rs @@ -0,0 +1,22 @@ +use axum::response::{IntoResponse, Response}; +use bytes::BytesMut; +use conduit::Error; +use http::StatusCode; +use http_body_util::Full; +use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; + +#[derive(Clone)] +pub struct RumaResponse(pub T); + +impl From for RumaResponse { + fn from(t: Error) -> Self { Self(t.into()) } +} + +impl IntoResponse for RumaResponse { + fn into_response(self) -> Response { + match self.0.try_into_http_response::() { + Ok(res) => res.map(BytesMut::freeze).map(Full::new).into_response(), + Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), + } + } +} diff --git a/src/core/error.rs b/src/core/error.rs index 63729f31..cb74d531 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -1,19 +1,11 @@ use std::{convert::Infallible, fmt}; -use axum::response::{IntoResponse, Response}; use bytes::BytesMut; use http::StatusCode; use http_body_util::Full; use ruma::{ api::{ - client::{ - error::ErrorKind::{ - Forbidden, GuestAccessForbidden, LimitExceeded, MissingToken, NotFound, ThreepidAuthFailed, - ThreepidDenied, TooLarge, Unauthorized, Unknown, UnknownToken, Unrecognized, UserDeactivated, - WrongRoomKeysVersion, - }, - uiaa::{UiaaInfo, UiaaResponse}, - }, + client::uiaa::{UiaaInfo, UiaaResponse}, OutgoingResponse, }, OwnedServerName, @@ -57,6 +49,8 @@ pub enum Error { Path(#[from] axum::extract::rejection::PathRejection), // ruma + #[error("uiaa")] + Uiaa(UiaaInfo), #[error("{0}")] Mxid(#[from] ruma::IdParseError), #[error("{0}: {1}")] @@ -81,8 +75,6 @@ pub enum Error { BadServerResponse(&'static str), #[error("{0}")] Conflict(&'static str), // This is only needed for when a room alias already exists - #[error("uiaa")] - Uiaa(UiaaInfo), // unique / untyped #[error("{0}")] @@ -103,11 +95,10 @@ impl Error { /// Returns the Matrix error code / error kind #[inline] pub fn error_code(&self) -> ruma::api::client::error::ErrorKind { - if let Self::Federation(_, error) = self { - return error.error_kind().unwrap_or_else(|| &Unknown).clone(); - } + use ruma::api::client::error::ErrorKind::Unknown; match self { + Self::Federation(_, err) => err.error_kind().unwrap_or(&Unknown).clone(), Self::BadRequest(kind, _) => kind.clone(), _ => Unknown, } @@ -116,12 +107,8 @@ impl Error { /// Sanitizes public-facing errors that can leak sensitive information. pub fn sanitized_error(&self) -> String { match self { - Self::Database { - .. - } => String::from("Database error occurred."), - Self::Io { - .. - } => String::from("I/O error occurred."), + Self::Database(..) => String::from("Database error occurred."), + Self::Io(..) => String::from("I/O error occurred."), _ => self.to_string(), } } @@ -135,6 +122,88 @@ impl fmt::Debug for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } } +impl axum::response::IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + let response: UiaaResponse = self.into(); + response.try_into_http_response::().map_or_else( + |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |r| r.map(BytesMut::freeze).map(Full::new).into_response(), + ) + } +} + +impl From for UiaaResponse { + fn from(error: Error) -> Self { + use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind::Unknown}; + + if let Error::Uiaa(uiaainfo) = error { + return Self::AuthResponse(uiaainfo); + } + + let kind = match &error { + Error::Federation(_, ref error) => error.error_kind().unwrap_or(&Unknown), + Error::BadRequest(kind, _) => kind, + _ => &Unknown, + }; + + let status_code = match &error { + Error::Federation(_, ref error) => error.status_code, + Error::BadRequest(ref kind, _) => bad_request_code(kind), + Error::Conflict(_) => StatusCode::CONFLICT, + _ => StatusCode::INTERNAL_SERVER_ERROR, + }; + + let message = if let Error::Federation(ref origin, ref error) = &error { + format!("Answer from {origin}: {error}") + } else { + format!("{error}") + }; + + let body = ErrorBody::Standard { + kind: kind.clone(), + message, + }; + + Self::MatrixError(RumaError { + status_code, + body, + }) + } +} + +fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { + use ruma::api::client::error::ErrorKind::*; + + match kind { + GuestAccessForbidden + | ThreepidAuthFailed + | UserDeactivated + | ThreepidDenied + | WrongRoomKeysVersion { + .. + } + | Forbidden { + .. + } => StatusCode::FORBIDDEN, + + UnknownToken { + .. + } + | MissingToken + | Unauthorized => StatusCode::UNAUTHORIZED, + + LimitExceeded { + .. + } => StatusCode::TOO_MANY_REQUESTS, + + TooLarge => StatusCode::PAYLOAD_TOO_LARGE, + + NotFound | Unrecognized => StatusCode::NOT_FOUND, + + _ => StatusCode::BAD_REQUEST, + } +} + #[inline] pub fn log(e: Error) { error!("{e}"); @@ -146,86 +215,3 @@ pub fn debug_log(e: Error) { debug_error!("{e}"); drop(e); } - -#[derive(Clone)] -pub struct RumaResponse(pub T); - -impl From for RumaResponse { - fn from(t: T) -> Self { Self(t) } -} - -impl From for RumaResponse { - fn from(t: Error) -> Self { t.to_response() } -} - -impl Error { - pub fn to_response(&self) -> RumaResponse { - use ruma::api::client::error::{Error as RumaError, ErrorBody}; - - if let Self::Uiaa(uiaainfo) = self { - return RumaResponse(UiaaResponse::AuthResponse(uiaainfo.clone())); - } - - if let Self::Federation(origin, error) = self { - let mut error = error.clone(); - error.body = ErrorBody::Standard { - kind: error.error_kind().unwrap_or_else(|| &Unknown).clone(), - message: format!("Answer from {origin}: {error}"), - }; - return RumaResponse(UiaaResponse::MatrixError(error)); - } - - let message = format!("{self}"); - let (kind, status_code) = match self { - Self::BadRequest(kind, _) => ( - kind.clone(), - match kind { - WrongRoomKeysVersion { - .. - } - | Forbidden { - .. - } - | GuestAccessForbidden - | ThreepidAuthFailed - | UserDeactivated - | ThreepidDenied => StatusCode::FORBIDDEN, - Unauthorized - | UnknownToken { - .. - } - | MissingToken => StatusCode::UNAUTHORIZED, - NotFound | Unrecognized => StatusCode::NOT_FOUND, - LimitExceeded { - .. - } => StatusCode::TOO_MANY_REQUESTS, - TooLarge => StatusCode::PAYLOAD_TOO_LARGE, - _ => StatusCode::BAD_REQUEST, - }, - ), - Self::Conflict(_) => (Unknown, StatusCode::CONFLICT), - _ => (Unknown, StatusCode::INTERNAL_SERVER_ERROR), - }; - - RumaResponse(UiaaResponse::MatrixError(RumaError { - body: ErrorBody::Standard { - kind, - message, - }, - status_code, - })) - } -} - -impl ::axum::response::IntoResponse for Error { - fn into_response(self) -> ::axum::response::Response { self.to_response().into_response() } -} - -impl IntoResponse for RumaResponse { - fn into_response(self) -> Response { - match self.0.try_into_http_response::() { - Ok(res) => res.map(BytesMut::freeze).map(Full::new).into_response(), - Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), - } - } -} diff --git a/src/core/mod.rs b/src/core/mod.rs index ec536ee2..de8057fa 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -10,7 +10,7 @@ pub mod utils; pub mod version; pub use config::Config; -pub use error::{Error, RumaResponse}; +pub use error::Error; pub use pdu::{PduBuilder, PduCount, PduEvent}; pub use server::Server; pub use version::version; diff --git a/src/router/request.rs b/src/router/request.rs index 079106ef..567de81f 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -1,13 +1,13 @@ use std::sync::{atomic::Ordering, Arc}; use axum::{extract::State, response::IntoResponse}; -use conduit::{debug_error, debug_warn, defer, Result, RumaResponse, Server}; +use conduit::{debug, debug_error, debug_warn, defer, error, trace, Result, Server}; +use conduit_api::RumaResponse; use http::{Method, StatusCode, Uri}; use ruma::api::client::{ error::{Error as RumaError, ErrorBody, ErrorKind}, uiaa::UiaaResponse, }; -use tracing::{debug, error, trace}; #[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( From a43c78e55f2127cac83a8ebb525cf69eb692b809 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 8 Jul 2024 18:54:42 +0000 Subject: [PATCH 0413/2091] add RumaError to Error; encapsulate RumaResponse in api Signed-off-by: Jason Volk --- src/api/mod.rs | 3 +-- src/api/router/mod.rs | 3 +-- src/api/router/response.rs | 3 +-- src/core/error.rs | 55 +++++++++++++++++++++++++------------- src/router/request.rs | 16 +++++------ 5 files changed, 45 insertions(+), 35 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 0395da7a..6adf2d39 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -9,8 +9,7 @@ extern crate conduit_service as service; pub(crate) use conduit::{debug_info, debug_warn, utils, Error, Result}; pub(crate) use service::{pdu::PduEvent, services, user_is_local}; -pub(crate) use self::router::Ruma; -pub use self::router::RumaResponse; +pub(crate) use self::router::{Ruma, RumaResponse}; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/api/router/mod.rs b/src/api/router/mod.rs index f167a606..c3e08c5b 100644 --- a/src/api/router/mod.rs +++ b/src/api/router/mod.rs @@ -13,9 +13,8 @@ use ruma::{ CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; -pub(super) use self::handler::RouterExt; -pub use self::response::RumaResponse; use self::{auth::Auth, request::Request}; +pub(super) use self::{handler::RouterExt, response::RumaResponse}; use crate::{service::appservice::RegistrationInfo, services, Error, Result}; /// Extractor for Ruma request structs diff --git a/src/api/router/response.rs b/src/api/router/response.rs index 38e58ba9..9b67f37b 100644 --- a/src/api/router/response.rs +++ b/src/api/router/response.rs @@ -5,8 +5,7 @@ use http::StatusCode; use http_body_util::Full; use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; -#[derive(Clone)] -pub struct RumaResponse(pub T); +pub(crate) struct RumaResponse(pub(crate) T); impl From for RumaResponse { fn from(t: Error) -> Self { Self(t.into()) } diff --git a/src/core/error.rs b/src/core/error.rs index cb74d531..5f4d4798 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -4,17 +4,13 @@ use bytes::BytesMut; use http::StatusCode; use http_body_util::Full; use ruma::{ - api::{ - client::uiaa::{UiaaInfo, UiaaResponse}, - OutgoingResponse, - }, + api::{client::uiaa::UiaaResponse, OutgoingResponse}, OwnedServerName, }; -use thiserror::Error; use crate::{debug_error, error}; -#[derive(Error)] +#[derive(thiserror::Error)] pub enum Error { // std #[error("{0}")] @@ -47,10 +43,16 @@ pub enum Error { Extension(#[from] axum::extract::rejection::ExtensionRejection), #[error("{0}")] Path(#[from] axum::extract::rejection::PathRejection), + #[error("{0}")] + Http(#[from] http::Error), // ruma + #[error("{0}")] + IntoHttpError(#[from] ruma::api::error::IntoHttpError), + #[error("{0}")] + RumaError(#[from] ruma::api::client::error::Error), #[error("uiaa")] - Uiaa(UiaaInfo), + Uiaa(ruma::api::client::uiaa::UiaaInfo), #[error("{0}")] Mxid(#[from] ruma::IdParseError), #[error("{0}: {1}")] @@ -98,7 +100,7 @@ impl Error { use ruma::api::client::error::ErrorKind::Unknown; match self { - Self::Federation(_, err) => err.error_kind().unwrap_or(&Unknown).clone(), + Self::Federation(_, error) => ruma_error_kind(error).clone(), Self::BadRequest(kind, _) => kind.clone(), _ => Unknown, } @@ -134,37 +136,35 @@ impl axum::response::IntoResponse for Error { impl From for UiaaResponse { fn from(error: Error) -> Self { - use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind::Unknown}; - if let Error::Uiaa(uiaainfo) = error { return Self::AuthResponse(uiaainfo); } let kind = match &error { - Error::Federation(_, ref error) => error.error_kind().unwrap_or(&Unknown), + Error::Federation(_, ref error) | Error::RumaError(ref error) => ruma_error_kind(error), Error::BadRequest(kind, _) => kind, - _ => &Unknown, + _ => &ruma::api::client::error::ErrorKind::Unknown, }; let status_code = match &error { - Error::Federation(_, ref error) => error.status_code, + Error::Federation(_, ref error) | Error::RumaError(ref error) => error.status_code, Error::BadRequest(ref kind, _) => bad_request_code(kind), Error::Conflict(_) => StatusCode::CONFLICT, _ => StatusCode::INTERNAL_SERVER_ERROR, }; - let message = if let Error::Federation(ref origin, ref error) = &error { - format!("Answer from {origin}: {error}") - } else { - format!("{error}") + let message = match &error { + Error::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), + Error::RumaError(ref error) => ruma_error_message(error), + _ => format!("{error}"), }; - let body = ErrorBody::Standard { + let body = ruma::api::client::error::ErrorBody::Standard { kind: kind.clone(), message, }; - Self::MatrixError(RumaError { + Self::MatrixError(ruma::api::client::error::Error { status_code, body, }) @@ -204,6 +204,23 @@ fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { } } +fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { + if let ruma::api::client::error::ErrorBody::Standard { + message, + .. + } = &error.body + { + return message.to_string(); + } + + format!("{error}") +} + +fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::error::ErrorKind { + e.error_kind() + .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) +} + #[inline] pub fn log(e: Error) { error!("{e}"); diff --git a/src/router/request.rs b/src/router/request.rs index 567de81f..9256fb9c 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -1,13 +1,9 @@ use std::sync::{atomic::Ordering, Arc}; use axum::{extract::State, response::IntoResponse}; -use conduit::{debug, debug_error, debug_warn, defer, error, trace, Result, Server}; -use conduit_api::RumaResponse; +use conduit::{debug, debug_error, debug_warn, defer, error, trace, Error, Result, Server}; use http::{Method, StatusCode, Uri}; -use ruma::api::client::{ - error::{Error as RumaError, ErrorBody, ErrorKind}, - uiaa::UiaaResponse, -}; +use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind}; #[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( @@ -66,15 +62,15 @@ fn handle_result( ) -> Result { handle_result_log(method, uri, &result); match result.status() { - StatusCode::METHOD_NOT_ALLOWED => handle_result_403(method, uri, &result), + StatusCode::METHOD_NOT_ALLOWED => handle_result_405(method, uri, &result), _ => Ok(result), } } -fn handle_result_403( +fn handle_result_405( _method: &Method, _uri: &Uri, result: &axum::response::Response, ) -> Result { - let error = UiaaResponse::MatrixError(RumaError { + let error = Error::RumaError(RumaError { status_code: result.status(), body: ErrorBody::Standard { kind: ErrorKind::Unrecognized, @@ -82,7 +78,7 @@ fn handle_result_403( }, }); - Ok(RumaResponse(error).into_response()) + Ok(error.into_response()) } fn handle_result_log(method: &Method, uri: &Uri, result: &axum::response::Response) { From 4718387dbe4c3da072b2652393762f335d5ae888 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 03:31:58 +0000 Subject: [PATCH 0414/2091] fix wrapper macro semicolons Signed-off-by: Jason Volk --- src/core/debug.rs | 10 +++++----- src/core/log/mod.rs | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/core/debug.rs b/src/core/debug.rs index 7522aa77..1f855e52 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -14,9 +14,9 @@ pub use crate::utils::debug::*; macro_rules! debug_event { ( $level:expr, $($x:tt)+ ) => { if cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) { - ::tracing::event!( $level, $($x)+ ); + ::tracing::event!( $level, $($x)+ ) } else { - ::tracing::debug!( $($x)+ ); + ::tracing::debug!( $($x)+ ) } } } @@ -27,7 +27,7 @@ macro_rules! debug_event { #[macro_export] macro_rules! debug_error { ( $($x:tt)+ ) => { - $crate::debug_event!(::tracing::Level::ERROR, $($x)+ ); + $crate::debug_event!(::tracing::Level::ERROR, $($x)+ ) } } @@ -37,7 +37,7 @@ macro_rules! debug_error { #[macro_export] macro_rules! debug_warn { ( $($x:tt)+ ) => { - $crate::debug_event!(::tracing::Level::WARN, $($x)+ ); + $crate::debug_event!(::tracing::Level::WARN, $($x)+ ) } } @@ -47,7 +47,7 @@ macro_rules! debug_warn { #[macro_export] macro_rules! debug_info { ( $($x:tt)+ ) => { - $crate::debug_event!(::tracing::Level::INFO, $($x)+ ); + $crate::debug_event!(::tracing::Level::INFO, $($x)+ ) } } diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index daa6b8e8..04d250a6 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -29,25 +29,25 @@ pub struct Log { #[macro_export] macro_rules! error { - ( $($x:tt)+ ) => { ::tracing::error!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::error!( $($x)+ ) } } #[macro_export] macro_rules! warn { - ( $($x:tt)+ ) => { ::tracing::warn!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::warn!( $($x)+ ) } } #[macro_export] macro_rules! info { - ( $($x:tt)+ ) => { ::tracing::info!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::info!( $($x)+ ) } } #[macro_export] macro_rules! debug { - ( $($x:tt)+ ) => { ::tracing::debug!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::debug!( $($x)+ ) } } #[macro_export] macro_rules! trace { - ( $($x:tt)+ ) => { ::tracing::trace!( $($x)+ ); } + ( $($x:tt)+ ) => { ::tracing::trace!( $($x)+ ) } } From 158de10fe6582b2fe3fa3e45c60edfe1626f14c0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 03:36:08 +0000 Subject: [PATCH 0415/2091] log erroring errors; improve inspection functors. Signed-off-by: Jason Volk --- src/api/router/response.rs | 13 ++++++---- src/core/error.rs | 49 +++++++++++++++++++++++------------- src/service/admin/console.rs | 2 +- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/src/api/router/response.rs b/src/api/router/response.rs index 9b67f37b..2aaa79fa 100644 --- a/src/api/router/response.rs +++ b/src/api/router/response.rs @@ -1,6 +1,6 @@ use axum::response::{IntoResponse, Response}; use bytes::BytesMut; -use conduit::Error; +use conduit::{error, Error}; use http::StatusCode; use http_body_util::Full; use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; @@ -13,9 +13,12 @@ impl From for RumaResponse { impl IntoResponse for RumaResponse { fn into_response(self) -> Response { - match self.0.try_into_http_response::() { - Ok(res) => res.map(BytesMut::freeze).map(Full::new).into_response(), - Err(_) => StatusCode::INTERNAL_SERVER_ERROR.into_response(), - } + self.0 + .try_into_http_response::() + .inspect_err(|e| error!("response error: {e}")) + .map_or_else( + |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |r| r.map(BytesMut::freeze).map(Full::new).into_response(), + ) } } diff --git a/src/core/error.rs b/src/core/error.rs index 5f4d4798..1959081a 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -116,21 +116,46 @@ impl Error { } } -impl From for Error { - fn from(i: Infallible) -> Self { match i {} } +#[inline] +pub fn log(e: &Error) { + error!(?e); +} + +#[inline] +pub fn debug_log(e: &Error) { + debug_error!(?e); +} + +#[inline] +pub fn into_log(e: Error) { + error!(?e); + drop(e); +} + +#[inline] +pub fn into_debug_log(e: Error) { + debug_error!(?e); + drop(e); } impl fmt::Debug for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } } +impl From for Error { + fn from(i: Infallible) -> Self { match i {} } +} + impl axum::response::IntoResponse for Error { fn into_response(self) -> axum::response::Response { let response: UiaaResponse = self.into(); - response.try_into_http_response::().map_or_else( - |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), - |r| r.map(BytesMut::freeze).map(Full::new).into_response(), - ) + response + .try_into_http_response::() + .inspect_err(|e| error!(?e)) + .map_or_else( + |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |r| r.map(BytesMut::freeze).map(Full::new).into_response(), + ) } } @@ -220,15 +245,3 @@ fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::e e.error_kind() .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) } - -#[inline] -pub fn log(e: Error) { - error!("{e}"); - drop(e); -} - -#[inline] -pub fn debug_log(e: Error) { - debug_error!("{e}"); - drop(e); -} diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 27a4b6c7..2f66b1d5 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -95,7 +95,7 @@ impl Console { ReadlineEvent::Line(string) => self.clone().handle(string).await, ReadlineEvent::Interrupted => continue, ReadlineEvent::Eof => break, - ReadlineEvent::Quit => services().server.shutdown().unwrap_or_else(error::log), + ReadlineEvent::Quit => services().server.shutdown().unwrap_or_else(error::into_log), }, Err(error) => match error { ReadlineError::Closed => break, From 3259ea08b5643a2e362179947420a6dbd1e49d8b Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 15:18:03 -0400 Subject: [PATCH 0416/2091] use cond compilation for config check, fix docker check w/unix sockets, use our logging instead of tracing Signed-off-by: strawberry --- src/core/config/check.rs | 106 +++++++++++++++++++-------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 795f0c43..95b4df4e 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -1,17 +1,12 @@ -#[cfg(unix)] -use std::path::Path; // not unix specific, just only for UNIX sockets stuff and *nix container checks - -use tracing::{debug, error, info, warn}; - -use crate::{error::Error, Config}; +use crate::{error::Error, warn, Config}; pub fn check(config: &Config) -> Result<(), Error> { - #[cfg(feature = "rocksdb")] + #[cfg(all(feature = "rocksdb", not(feature = "sha256_media")))] // prevents catching this in `--all-features` warn!( - "Note the rocksdb feature was deleted from conduwuit, sqlite was deleted and RocksDB is the only supported \ - backend now. Please update your build script to remove this feature." + "Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \ + supported backend now. Please update your build script to remove this feature." ); - #[cfg(feature = "sha256_media")] + #[cfg(all(feature = "sha256_media", not(feature = "rocksdb")))] // prevents catching this in `--all-features` warn!( "Note the sha256_media feature was deleted from conduwuit, it is now fully integrated in a \ forwards-compatible way. Please update your build script to remove this feature." @@ -24,72 +19,77 @@ pub fn check(config: &Config) -> Result<(), Error> { return Err(Error::bad_config("Sentry cannot be enabled without an endpoint set")); } - if cfg!(feature = "hardened_malloc") && cfg!(feature = "jemalloc") { - warn!("hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used."); - } + #[cfg(all(feature = "hardened_malloc", feature = "jemalloc"))] + warn!( + "hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used. If using --all-features, \ + this is harmless." + ); - if config.unix_socket_path.is_some() && !cfg!(unix) { + #[cfg(not(unix))] + if config.unix_socket_path.is_some() { return Err(Error::bad_config( "UNIX socket support is only available on *nix platforms. Please remove \"unix_socket_path\" from your \ config.", )); } - config.get_bind_addrs().iter().for_each(|addr| { - if addr.ip().is_loopback() && cfg!(unix) { - debug!("Found loopback listening address {addr}, running checks if we're in a container.",); + #[cfg(unix)] + if config.unix_socket_path.is_none() { + config.get_bind_addrs().iter().for_each(|addr| { + use std::path::Path; - #[cfg(unix)] - if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() - /* Host */ - { - error!( - "You are detected using OpenVZ with a loopback/localhost listening address of {addr}. If you are \ - using OpenVZ for containers and you use NAT-based networking to communicate with the host and \ - guest, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, you can \ - ignore.", - ); - } + if addr.ip().is_loopback() { + crate::debug_info!("Found loopback listening address {addr}, running checks if we're in a container.",); - #[cfg(unix)] - if Path::new("/.dockerenv").exists() { - error!( - "You are detected using Docker with a loopback/localhost listening address of {addr}. If you are \ - using a reverse proxy on the host and require communication to conduwuit in the Docker container \ - via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". If this is \ - expected, you can ignore.", - ); - } + if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() + /* Host */ + { + crate::error!( + "You are detected using OpenVZ with a loopback/localhost listening address of {addr}. If you \ + are using OpenVZ for containers and you use NAT-based networking to communicate with the \ + host and guest, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, \ + you can ignore.", + ); + } - #[cfg(unix)] - if Path::new("/run/.containerenv").exists() { - error!( - "You are detected using Podman with a loopback/localhost listening address of {addr}. If you are \ - using a reverse proxy on the host and require communication to conduwuit in the Podman container \ - via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". If this is \ - expected, you can ignore.", - ); + if Path::new("/.dockerenv").exists() { + crate::error!( + "You are detected using Docker with a loopback/localhost listening address of {addr}. If you \ + are using a reverse proxy on the host and require communication to conduwuit in the Docker \ + container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ + If this is expected, you can ignore.", + ); + } + + if Path::new("/run/.containerenv").exists() { + crate::error!( + "You are detected using Podman with a loopback/localhost listening address of {addr}. If you \ + are using a reverse proxy on the host and require communication to conduwuit in the Podman \ + container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ + If this is expected, you can ignore.", + ); + } } - } - }); + }); + } // rocksdb does not allow max_log_files to be 0 if config.rocksdb_max_log_files == 0 { return Err(Error::bad_config( - "When using RocksDB, rocksdb_max_log_files cannot be 0. Please set a value at least 1.", + "rocksdb_max_log_files cannot be 0. Please set a value at least 1.", )); } // yeah, unless the user built a debug build hopefully for local testing only - if config.server_name == "your.server.name" && !cfg!(debug_assertions) { + #[cfg(not(debug_assertions))] + if config.server_name == "your.server.name" { return Err(Error::bad_config( "You must specify a valid server name for production usage of conduwuit.", )); } - if cfg!(debug_assertions) { - info!("Note: conduwuit was built without optimisations (i.e. debug build)"); - } + #[cfg(debug_assertions)] + crate::info!("Note: conduwuit was built without optimisations (i.e. debug build)"); // check if the user specified a registration token as `""` if config.registration_token == Some(String::new()) { @@ -103,7 +103,7 @@ pub fn check(config: &Config) -> Result<(), Error> { // check if user specified valid IP CIDR ranges on startup for cidr in &config.ip_range_denylist { if let Err(e) = ipaddress::IPAddress::parse(cidr) { - error!("Error parsing specified IP CIDR range from string: {e}"); + crate::error!("Error parsing specified IP CIDR range from string: {e}"); return Err(Error::bad_config("Error parsing specified IP CIDR ranges from strings")); } } From daa5c34ea354523446eaca948d2dcfbcd75b3187 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 15:41:16 -0400 Subject: [PATCH 0417/2091] fix empty version string for NixOS users Signed-off-by: strawberry --- src/core/version.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core/version.rs b/src/core/version.rs index bd89bee2..2876cea8 100644 --- a/src/core/version.rs +++ b/src/core/version.rs @@ -27,5 +27,11 @@ fn init_user_agent() -> String { format!("{}/{}", name(), version()) } fn init_version() -> String { option_env!("CONDUWUIT_VERSION_EXTRA") .or(option_env!("CONDUIT_VERSION_EXTRA")) - .map_or(SEMANTIC.to_owned(), |extra| format!("{SEMANTIC} ({extra})")) + .map_or(SEMANTIC.to_owned(), |extra| { + if extra.is_empty() { + SEMANTIC.to_owned() + } else { + format!("{SEMANTIC} ({extra})") + } + }) } From e54f4d43978b5eee5356b390144a89fecfb4121d Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 15:50:49 -0400 Subject: [PATCH 0418/2091] ci: move complement diff results test output Signed-off-by: strawberry --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f01e865d..71d8d2a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,10 +164,6 @@ jobs: - name: Diff Complement results with checked-in repo results run: | diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log) - echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - name: Update Job Summary if: success() || failure() @@ -175,9 +171,15 @@ jobs: if [ ${{ job.status }} == 'success' ]; then echo '# ✅ completed suwuccessfully' >> $GITHUB_STEP_SUMMARY else + echo '# CI failure' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY tail -n 40 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY + + echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY fi build: From da03de1d322dbca83683e4baf9b1cff20f1da584 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:28:23 -0400 Subject: [PATCH 0419/2091] bump flake.lock fully MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'crane': 'github:ipetkov/crane/109987da061a1bf452f435f1653c47511587d919' (2024-05-24) → 'github:ipetkov/crane/087e08a41009bf083d51ab35d8e30b1b7eafa7b0' (2024-07-03) • Updated input 'fenix': 'github:nix-community/fenix/b6fc5035b28e36a98370d0eac44f4ef3fd323df6' (2024-05-22) → 'github:nix-community/fenix/f6994934e25396d3a70ddb908cefccd8d3c37ac4' (2024-07-05) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/21ec8f523812b88418b2bfc64240c62b3dd967bd' (2024-05-19) → 'github:rust-lang/rust-analyzer/cae997e3380363a906588f14c7b4587f39cf09f5' (2024-07-03) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/5710852ba686cc1fd0d3b8e22b3117d43ba374c2' (2024-05-21) → 'github:NixOS/nixpkgs/9f4128e00b0ae8ec65918efeba59db998750ead6' (2024-07-03) Signed-off-by: strawberry --- flake.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 1a064510..06a4cea1 100644 --- a/flake.lock +++ b/flake.lock @@ -123,11 +123,11 @@ ] }, "locked": { - "lastModified": 1716569590, - "narHash": "sha256-5eDbq8TuXFGGO3mqJFzhUbt5zHVTf5zilQoyW5jnJwo=", + "lastModified": 1720025378, + "narHash": "sha256-zlIdj0oLvMEHlllP/7tvY+kE987xsN4FPux6WHSOh00=", "owner": "ipetkov", "repo": "crane", - "rev": "109987da061a1bf452f435f1653c47511587d919", + "rev": "087e08a41009bf083d51ab35d8e30b1b7eafa7b0", "type": "github" }, "original": { @@ -209,11 +209,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1716359173, - "narHash": "sha256-pYcjP6Gy7i6jPWrjiWAVV0BCQp+DdmGaI/k65lBb/kM=", + "lastModified": 1720160983, + "narHash": "sha256-GPl5qug68zcCBvkakakTdzuA/LIOdWyJbAjXkoeM+FE=", "owner": "nix-community", "repo": "fenix", - "rev": "b6fc5035b28e36a98370d0eac44f4ef3fd323df6", + "rev": "f6994934e25396d3a70ddb908cefccd8d3c37ac4", "type": "github" }, "original": { @@ -606,11 +606,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1716330097, - "narHash": "sha256-8BO3B7e3BiyIDsaKA0tY8O88rClYRTjvAp66y+VBUeU=", + "lastModified": 1720031269, + "narHash": "sha256-rwz8NJZV+387rnWpTYcXaRNvzUSnnF9aHONoJIYmiUQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "5710852ba686cc1fd0d3b8e22b3117d43ba374c2", + "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", "type": "github" }, "original": { @@ -705,11 +705,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1716107283, - "narHash": "sha256-NJgrwLiLGHDrCia5AeIvZUHUY7xYGVryee0/9D3Ir1I=", + "lastModified": 1719997308, + "narHash": "sha256-dQx1p/2ObV+iDriPWTBvELCwxe9ZbOimKTJKE3MA2FQ=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "21ec8f523812b88418b2bfc64240c62b3dd967bd", + "rev": "cae997e3380363a906588f14c7b4587f39cf09f5", "type": "github" }, "original": { From 391bfd986e00f07063e74deac2fe38947c75d207 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:29:19 -0400 Subject: [PATCH 0420/2091] use ruma's X-Matrix to construct the sending X-Matrix header value Signed-off-by: strawberry --- src/service/sending/send.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index a89ea2f8..f4825ff3 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -8,6 +8,8 @@ use ruma::{ client::error::Error as RumaError, EndpointError, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, }, + serde::Base64, + server_util::authorization::XMatrix, ServerName, }; use tracing::{debug, trace}; @@ -196,16 +198,20 @@ where for signature_server in signatures { for s in signature_server { + let key = + s.0.as_str() + .try_into() + .expect("valid homeserver signing key ID"); + let sig = Base64::parse(s.1).expect("valid base64"); + http_request.headers_mut().insert( AUTHORIZATION, - HeaderValue::from_str(&format!( - "X-Matrix origin=\"{}\",destination=\"{}\",key=\"{}\",sig=\"{}\"", - services().globals.config.server_name, - dest, - s.0, - s.1 - )) - .expect("formatted X-Matrix header"), + HeaderValue::from(&XMatrix::new( + services().globals.config.server_name.clone(), + dest.to_owned(), + key, + sig, + )), ); } } From 60141950f746467e1de5f3153909ff8f82e8a26d Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:31:54 -0400 Subject: [PATCH 0421/2091] fix wrong deactivate-all argument comment Signed-off-by: strawberry --- src/admin/user/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 31bf57d6..088133a5 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -49,7 +49,7 @@ pub(super) enum UserCommand { /// Markdown code block below the command. DeactivateAll { #[arg(short, long)] - /// Remove users from their joined rooms + /// Does not leave any rooms the user is in on deactivation no_leave_rooms: bool, #[arg(short, long)] /// Also deactivate admin accounts and will assume leave all rooms too From 1c453b1b5511406e554f6827e4e124aaeb17e8b5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:36:05 -0400 Subject: [PATCH 0422/2091] require authentication on change password and deactivation routes Signed-off-by: strawberry --- src/api/client/account.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 71324c1b..0d8d6075 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -423,7 +423,12 @@ pub(crate) async fn register_route( pub(crate) async fn change_password_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + // Authentication for this endpoint was made optional, but we need + // authentication currently + let sender_user = body + .sender_user + .as_ref() + .ok_or_else(|| Error::BadRequest(ErrorKind::MissingToken, "Missing access token."))?; let sender_device = body.sender_device.as_ref().expect("user is authenticated"); let mut uiaainfo = UiaaInfo { @@ -512,7 +517,12 @@ pub(crate) async fn whoami_route(body: Ruma) -> Result, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + // Authentication for this endpoint was made optional, but we need + // authentication currently + let sender_user = body + .sender_user + .as_ref() + .ok_or_else(|| Error::BadRequest(ErrorKind::MissingToken, "Missing access token."))?; let sender_device = body.sender_device.as_ref().expect("user is authenticated"); let mut uiaainfo = UiaaInfo { From 83e853e7a341a7b10c8e9bf32e78db21ced1830d Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 16:39:57 -0400 Subject: [PATCH 0423/2091] add "unstable" endpoint legacy Element Android/iOS seems to call Signed-off-by: strawberry --- src/api/client/thirdparty.rs | 12 +++++++++++- src/api/routes.rs | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/api/client/thirdparty.rs b/src/api/client/thirdparty.rs index de24c0ec..f6af8729 100644 --- a/src/api/client/thirdparty.rs +++ b/src/api/client/thirdparty.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use ruma::api::client::thirdparty::get_protocols; -use crate::{Result, Ruma}; +use crate::{Result, Ruma, RumaResponse}; /// # `GET /_matrix/client/r0/thirdparty/protocols` /// @@ -15,3 +15,13 @@ pub(crate) async fn get_protocols_route( protocols: BTreeMap::new(), }) } + +/// # `GET /_matrix/client/unstable/thirdparty/protocols` +/// +/// Same as `get_protocols_route`, except for some reason Element Android legacy +/// calls this +pub(crate) async fn get_protocols_route_unstable( + body: Ruma, +) -> Result> { + get_protocols_route(body).await.map(RumaResponse) +} diff --git a/src/api/routes.rs b/src/api/routes.rs index 94951aec..b22a32cb 100644 --- a/src/api/routes.rs +++ b/src/api/routes.rs @@ -94,6 +94,8 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::search_users_route) .ruma_route(client::get_member_events_route) .ruma_route(client::get_protocols_route) + .route("/_matrix/client/unstable/thirdparty/protocols", + get(client::get_protocols_route_unstable)) .ruma_route(client::send_message_event_route) .ruma_route(client::send_state_event_for_key_route) .ruma_route(client::get_state_events_route) From 68ad351f84b3edcadf364f1dc3c2ba6110a6acf9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 20:13:21 -0400 Subject: [PATCH 0424/2091] nix: partially remove some liburing overrides the argument one is weird and causing build script invalid arguments Signed-off-by: strawberry --- flake.nix | 10 ---------- nix/pkgs/main/default.nix | 5 ++--- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/flake.nix b/flake.nix index 499d828d..b252193e 100644 --- a/flake.nix +++ b/flake.nix @@ -50,16 +50,6 @@ configureFlags = pkgs.lib.subtractLists [ "--enable-static" "--disable-shared" ] old.configureFlags; - - postInstall = old.postInstall + '' - # we remove the extra outputs - # - # we need to do this to prevent rocksdb from trying to link the - # static library in a dynamic stdenv - rm $out/lib/liburing*${ - if pkgs.stdenv.hostPlatform.isStatic then ".so*" else ".a" - } - ''; }); }); diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index f91a9cdd..5c8ade13 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -43,7 +43,7 @@ features'' = lib.subtractLists disable_features' features'; featureEnabled = feature : builtins.elem feature features''; -enableLiburing = featureEnabled "io_uring" && stdenv.isLinux; +enableLiburing = featureEnabled "io_uring" && !stdenv.isDarwin; # This derivation will set the JEMALLOC_OVERRIDE variable, causing the # tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's @@ -74,8 +74,7 @@ buildDepsOnlyEnv = # TODO: static rocksdb fails to build on darwin # build log at meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin; - # TODO: switch to enableUring option once https://github.com/NixOS/nixpkgs/pull/314945 is available - buildInputs = old.buildInputs ++ lib.optional enableLiburing liburing; + enableLiburing = enableLiburing; }); in { From 373991a8d6952185109e722031563c2b7f1e1f92 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 21:11:54 -0400 Subject: [PATCH 0425/2091] cleanup and fix backfill from server getting Signed-off-by: strawberry --- src/api/client/account.rs | 2 +- src/service/rooms/timeline/mod.rs | 90 +++++++++++-------------------- 2 files changed, 33 insertions(+), 59 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 0d8d6075..66dc7a26 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -309,7 +309,7 @@ pub(crate) async fn register_route( // log in conduit admin channel if a guest registered if body.appservice_info.is_none() && is_guest && services().globals.log_guest_registrations() { - info!("New guest user \"{user_id}\" registered on this server from IP."); + info!("New guest user \"{user_id}\" registered on this server."); if let Some(device_display_name) = &body.initial_device_display_name { if body diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index db7af004..96f137e8 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -9,7 +9,6 @@ use std::{ use conduit::{debug, error, info, utils, utils::mutex_map, validated, warn, Error, Result}; use data::Data; use itertools::Itertools; -use rand::prelude::SliceRandom; use ruma::{ api::{client::error::ErrorKind, federation}, canonical_json::to_canonical_value, @@ -1081,7 +1080,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, room_id))] + #[tracing::instrument(skip(self))] pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { let first_pdu = self .all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id)? @@ -1093,41 +1092,6 @@ impl Service { return Ok(()); } - let mut servers: Vec = vec![]; - - // add server names of any trusted key servers if they're in the room - servers.extend( - services() - .rooms - .state_cache - .room_servers(room_id) - .filter_map(Result::ok) - .filter(|server_name| { - services().globals.trusted_servers().contains(server_name) && !server_is_ours(server_name) - }), - ); - - // add server names from room aliases on the room ID - let room_aliases = services() - .rooms - .alias - .local_aliases_for_room(room_id) - .collect::, _>>(); - if let Ok(aliases) = &room_aliases { - for alias in aliases { - if !server_is_ours(alias.server_name()) { - servers.push(alias.server_name().to_owned()); - } - } - } - - // add room ID server name for backfill server - if let Some(server_name) = room_id.server_name() { - if !server_is_ours(server_name) { - servers.push(server_name.to_owned()); - } - } - let power_levels: RoomPowerLevelsEventContent = services() .rooms .state_accessor @@ -1139,29 +1103,39 @@ impl Service { .transpose()? .unwrap_or_default(); - // add server names of the list of admins in the room for backfill server - servers.extend( - power_levels - .users - .iter() - .filter(|(_, level)| **level > power_levels.users_default) - .map(|(user_id, _)| user_id.server_name()) - .filter(|server_name| !server_is_ours(server_name)) - .map(ToOwned::to_owned), - ); + let room_mods = power_levels.users.iter().filter_map(|(user_id, level)| { + if level > &power_levels.users_default && !server_is_ours(user_id.server_name()) { + Some(user_id.server_name().to_owned()) + } else { + None + } + }); - // don't backfill from ourselves (might be noop if we checked it above already) - if let Some(server_index) = servers - .clone() - .into_iter() - .position(|server_name| server_is_ours(&server_name)) - { - servers.swap_remove(server_index); - } + let room_alias_servers = services() + .rooms + .alias + .local_aliases_for_room(room_id) + .filter_map(|alias| { + alias + .ok() + .filter(|alias| !server_is_ours(alias.server_name())) + .map(|alias| alias.server_name().to_owned()) + }); - servers.sort_unstable(); - servers.dedup(); - servers.shuffle(&mut rand::thread_rng()); + let servers = room_mods + .chain(room_alias_servers) + .chain(services().globals.config.trusted_servers.clone()) + .filter(|server_name| { + if server_is_ours(server_name) { + return false; + } + + services() + .rooms + .state_cache + .server_in_room(server_name, room_id) + .unwrap_or(false) + }); for backfill_server in servers { info!("Asking {backfill_server} for backfill"); From 6abc4ad798db2751bb1a0c235a640f170e69f670 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 6 Jul 2024 14:43:17 -0400 Subject: [PATCH 0426/2091] make `local_aliases_for_room` db iterator Send Signed-off-by: strawberry --- src/service/rooms/alias/data.rs | 2 +- src/service/rooms/alias/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/alias/data.rs b/src/service/rooms/alias/data.rs index eb99e543..302c21ae 100644 --- a/src/service/rooms/alias/data.rs +++ b/src/service/rooms/alias/data.rs @@ -84,7 +84,7 @@ impl Data { pub(super) fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, - ) -> Box> + 'a> { + ) -> Box> + 'a + Send> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 706bf2f8..4af1035e 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -97,7 +97,7 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub fn local_aliases_for_room<'a>( &'a self, room_id: &RoomId, - ) -> Box> + 'a> { + ) -> Box> + 'a + Send> { self.db.local_aliases_for_room(room_id) } From 0873e18e14bbcbafa2a92dd881e76f6dd4351df9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 21:16:41 -0400 Subject: [PATCH 0427/2091] remove random duplicate function Signed-off-by: strawberry --- src/service/rooms/timeline/mod.rs | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 96f137e8..c82098ba 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -145,29 +145,6 @@ impl Service { } */ - /// Returns the version of a room, if known - /// - /// TODO: use this? - #[allow(dead_code)] - pub fn get_room_version(&self, room_id: &RoomId) -> Result> { - let create_event = services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "")?; - - let create_event_content: Option = create_event - .as_ref() - .map(|create_event| { - serde_json::from_str(create_event.content.get()).map_err(|e| { - warn!("Invalid create event: {}", e); - Error::bad_database("Invalid create event in db.") - }) - }) - .transpose()?; - - Ok(create_event_content.map(|content| content.room_version)) - } - /// Returns the json of a pdu. pub fn get_pdu_json(&self, event_id: &EventId) -> Result> { self.db.get_pdu_json(event_id) @@ -188,9 +165,6 @@ impl Service { /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - /// - /// TODO: use this? - #[allow(dead_code)] #[inline] pub fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { self.db.get_non_outlier_pdu(event_id) From b5ee15a216e48a53b5b469d6350901fcffe530f4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 21:19:27 -0400 Subject: [PATCH 0428/2091] dont skip_all tracing instrument on fetch_state Signed-off-by: strawberry --- src/service/rooms/event_handler/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 0f7919dd..53ce6f8d 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -953,7 +953,7 @@ impl Service { /// Call /state_ids to find out what the state at this pdu is. We trust the /// server's response to some extend (sic), but we still do a lot of checks /// on the events - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip(self, pub_key_map, create_event, room_version_id))] async fn fetch_state( &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, pub_key_map: &RwLock>>, event_id: &EventId, From efea13a675178e052966c21e079dfa57b8560b26 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 5 Jul 2024 22:05:52 -0400 Subject: [PATCH 0429/2091] add access control checks for room directory publishing/removing Signed-off-by: strawberry --- src/api/client/directory.rs | 40 +++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 512dface..7d2aff0d 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -10,10 +10,13 @@ use ruma::{ }, directory::{Filter, PublicRoomJoinRule, PublicRoomsChunk, RoomNetwork}, events::{ - room::join_rules::{JoinRule, RoomJoinRulesEventContent}, + room::{ + join_rules::{JoinRule, RoomJoinRulesEventContent}, + power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, + }, StateEventType, }, - uint, ServerName, UInt, + uint, RoomId, ServerName, UInt, UserId, }; use tracing::{error, info, warn}; @@ -103,8 +106,6 @@ pub(crate) async fn get_public_rooms_route( /// # `PUT /_matrix/client/r0/directory/list/room/{roomId}` /// /// Sets the visibility of a given room in the room directory. -/// -/// - TODO: Access control checks #[tracing::instrument(skip_all, fields(%client), name = "room_directory")] pub(crate) async fn set_room_visibility_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, @@ -116,6 +117,8 @@ pub(crate) async fn set_room_visibility_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } + user_can_publish_room(sender_user, &body.room_id)?; + match &body.visibility { room::Visibility::Public => { if services().globals.config.lockdown_public_room_directory && !services().users.is_admin(sender_user)? { @@ -351,3 +354,32 @@ pub(crate) async fn get_public_rooms_filtered_helper( total_room_count_estimate: Some(total_room_count_estimate), }) } + +/// Check whether the user can publish to the room directory via power levels of +/// room history visibility event or room creator +fn user_can_publish_room(user_id: &UserId, room_id: &RoomId) -> Result { + if let Some(event) = + services() + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? + { + serde_json::from_str(event.content.get()) + .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) + .map(|content: RoomPowerLevelsEventContent| { + RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomHistoryVisibility) + }) + } else if let Some(event) = + services() + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomCreate, "")? + { + Ok(event.sender == user_id) + } else { + return Err(Error::BadRequest( + ErrorKind::Unauthorized, + "You are not allowed to publish this room to the room directory", + )); + } +} From 35336eb68698f6e5fdd2775ca3e3097a30ba074c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 6 Jul 2024 21:15:44 -0400 Subject: [PATCH 0430/2091] ci: use `$COMPLEMENT_SRC` from nix devshell for a pinned complement rev Signed-off-by: strawberry --- .github/workflows/ci.yml | 13 ++++--------- bin/complement | 17 ++++++++++++----- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 71d8d2a2..4c6b3bb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,15 +129,10 @@ jobs: run: | direnv exec . engage > >(tee -a test_output.log) - - name: Sync Complement repository - uses: actions/checkout@v4 - with: - repository: 'matrix-org/complement' - path: complement_src - - name: Run Complement tests run: | - direnv exec . bin/complement 'complement_src' 'complement_test_logs.jsonl' 'complement_test_results.jsonl' + # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op + direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) cp -v -f result complement_oci_image.tar.gz - name: Upload Complement OCI image @@ -163,7 +158,7 @@ jobs: - name: Diff Complement results with checked-in repo results run: | - diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log) + diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log) - name: Update Job Summary if: success() || failure() @@ -178,7 +173,7 @@ jobs: echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY fi diff --git a/bin/complement b/bin/complement index c679035a..a715b14b 100755 --- a/bin/complement +++ b/bin/complement @@ -7,7 +7,7 @@ set -euo pipefail # The `COMPLEMENT_SRC` environment variable is set in the Nix dev shell, which # points to a store path containing the Complement source code. It's likely you # want to just pass that as the first argument to use it here. -COMPLEMENT_SRC="$1" +COMPLEMENT_SRC="${COMPLEMENT_SRC:-$1}" # A `.jsonl` file to write test logs to LOG_FILE="$2" @@ -17,12 +17,19 @@ RESULTS_FILE="$3" OCI_IMAGE="complement-conduit:main" -# Complement tests that are skipped due to flakiness/reliability issues (likely -# Complement itself induced based on various open issues) -# -# According to Go docs, these are separated by forward slashes and not pipes (why) +# Complement tests that are skipped due to flakiness/reliability issues SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*' +# $COMPLEMENT_SRC needs to be a directory to Complement source code +if [ -f "$COMPLEMENT_SRC" ]; then + echo "\$COMPLEMENT_SRC must be a directory/path to Complement source code" + exit 1 +fi + +# quick test to make sure we can actually write to $LOG_FILE and $RESULTS_FILE +touch $LOG_FILE && rm -v $LOG_FILE +touch $RESULTS_FILE && rm -v $RESULTS_FILE + toplevel="$(git rev-parse --show-toplevel)" pushd "$toplevel" > /dev/null From 0dae9280d9c0046063fa97b18c5979ce0ae42558 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 6 Jul 2024 21:19:52 -0400 Subject: [PATCH 0431/2091] nix: bump flake.lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'crane': 'github:ipetkov/crane/087e08a41009bf083d51ab35d8e30b1b7eafa7b0' (2024-07-03) → 'github:ipetkov/crane/0aed560c5c0a61c9385bddff471a13036203e11c' (2024-07-06) • Updated input 'fenix': 'github:nix-community/fenix/f6994934e25396d3a70ddb908cefccd8d3c37ac4' (2024-07-05) → 'github:nix-community/fenix/27cbad7cc093c5298231b87daa04db9610053651' (2024-07-06) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/cae997e3380363a906588f14c7b4587f39cf09f5' (2024-07-03) → 'github:rust-lang/rust-analyzer/f2afcb874e7410121c366ae601660abe327e320b' (2024-07-05) • Updated input 'fenix': 'github:nix-community/fenix/27cbad7cc093c5298231b87daa04db9610053651' (2024-07-06) → 'github:nix-community/fenix/abc0549e3560189462a7d394cc9d50af4608d103' (2024-07-08) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/f2afcb874e7410121c366ae601660abe327e320b' (2024-07-05) → 'github:rust-lang/rust-analyzer/a5b21ea0aa644dffd7cf958b43f11f221d53404e' (2024-07-07) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/9f4128e00b0ae8ec65918efeba59db998750ead6' (2024-07-03) → 'github:NixOS/nixpkgs/655a58a72a6601292512670343087c2d75d859c1' (2024-07-08) Signed-off-by: strawberry --- flake.lock | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index 06a4cea1..4e6721b2 100644 --- a/flake.lock +++ b/flake.lock @@ -123,11 +123,11 @@ ] }, "locked": { - "lastModified": 1720025378, - "narHash": "sha256-zlIdj0oLvMEHlllP/7tvY+kE987xsN4FPux6WHSOh00=", + "lastModified": 1720226507, + "narHash": "sha256-yHVvNsgrpyNTXZBEokL8uyB2J6gB1wEx0KOJzoeZi1A=", "owner": "ipetkov", "repo": "crane", - "rev": "087e08a41009bf083d51ab35d8e30b1b7eafa7b0", + "rev": "0aed560c5c0a61c9385bddff471a13036203e11c", "type": "github" }, "original": { @@ -209,11 +209,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1720160983, - "narHash": "sha256-GPl5qug68zcCBvkakakTdzuA/LIOdWyJbAjXkoeM+FE=", + "lastModified": 1720420198, + "narHash": "sha256-OIuDb6pHDyGpo7YMFyuRzMLcHm7mRvlYOz0Ht7ps2sU=", "owner": "nix-community", "repo": "fenix", - "rev": "f6994934e25396d3a70ddb908cefccd8d3c37ac4", + "rev": "abc0549e3560189462a7d394cc9d50af4608d103", "type": "github" }, "original": { @@ -606,11 +606,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1720031269, - "narHash": "sha256-rwz8NJZV+387rnWpTYcXaRNvzUSnnF9aHONoJIYmiUQ=", + "lastModified": 1720418205, + "narHash": "sha256-cPJoFPXU44GlhWg4pUk9oUPqurPlCFZ11ZQPk21GTPU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9f4128e00b0ae8ec65918efeba59db998750ead6", + "rev": "655a58a72a6601292512670343087c2d75d859c1", "type": "github" }, "original": { @@ -705,11 +705,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1719997308, - "narHash": "sha256-dQx1p/2ObV+iDriPWTBvELCwxe9ZbOimKTJKE3MA2FQ=", + "lastModified": 1720344064, + "narHash": "sha256-STmaV9Zu74QtkGGrbr9uMhskwagfCjJqOAYapXabiuk=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "cae997e3380363a906588f14c7b4587f39cf09f5", + "rev": "a5b21ea0aa644dffd7cf958b43f11f221d53404e", "type": "github" }, "original": { From d2facaee0b65f5e965651b9c4bbab3b716c01317 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 6 Jul 2024 21:20:51 -0400 Subject: [PATCH 0432/2091] bump various dependencies Signed-off-by: strawberry --- .github/workflows/trivy.yml | 4 +- .gitlab-ci.yml | 4 +- Cargo.lock | 216 ++++++++++++++++++------------------ Cargo.toml | 6 +- 4 files changed, 115 insertions(+), 115 deletions(-) diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index e9cc9a30..e0871324 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@v4 - name: Run Trivy code and vulnerability scanner on repo - uses: aquasecurity/trivy-action@0.23.0 + uses: aquasecurity/trivy-action@0.24.0 with: scan-type: repo format: sarif @@ -34,7 +34,7 @@ jobs: severity: CRITICAL,HIGH,MEDIUM,LOW - name: Run Trivy code and vulnerability scanner on filesystem - uses: aquasecurity/trivy-action@0.23.0 + uses: aquasecurity/trivy-action@0.24.0 with: scan-type: fs format: sarif diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 33d0c6ed..78449e36 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ before_script: ci: stage: ci - image: nixos/nix:2.23.1 + image: nixos/nix:2.23.3 script: # Cache CI dependencies - ./bin/nix-build-and-cache ci @@ -83,7 +83,7 @@ ci: artifacts: stage: artifacts - image: nixos/nix:2.23.1 + image: nixos/nix:2.23.3 script: - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl - cp result/bin/conduit x86_64-unknown-linux-musl diff --git a/Cargo.lock b/Cargo.lock index dd0c0436..eb24aa21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,18 +118,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -166,7 +166,7 @@ dependencies = [ "futures-util", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "itoa", "matchit", "memchr", @@ -194,7 +194,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "itoa", "matchit", @@ -298,7 +298,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "pin-project-lite", "rustls 0.21.12", @@ -377,7 +377,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -474,9 +474,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.104" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" dependencies = [ "jobserver", "libc", @@ -535,9 +535,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.8" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" +checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" dependencies = [ "clap_builder", "clap_derive", @@ -545,9 +545,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.8" +version = "4.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" +checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" dependencies = [ "anstyle", "clap_lex", @@ -562,7 +562,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -638,7 +638,7 @@ dependencies = [ "hmac", "http 1.1.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "image", "ipaddress", "jsonwebtoken", @@ -721,7 +721,7 @@ dependencies = [ "conduit_service", "http 1.1.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", "log", "ruma", @@ -1007,7 +1007,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1111,7 +1111,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1247,7 +1247,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1528,7 +1528,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1616,9 +1616,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.29" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", @@ -1640,9 +1640,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.4.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4fe55fb7a772d59a5ff1dfbff4fe0258d19b89fec4b233e75d35d5d2316badc" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" dependencies = [ "bytes", "futures-channel", @@ -1667,9 +1667,9 @@ checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", "http 1.1.0", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-util", - "rustls 0.23.10", + "rustls 0.23.11", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -1684,7 +1684,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper 0.14.29", + "hyper 0.14.30", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -1701,7 +1701,7 @@ dependencies = [ "futures-util", "http 1.1.0", "http-body 1.0.0", - "hyper 1.4.0", + "hyper 1.4.1", "pin-project-lite", "socket2", "tokio", @@ -1937,7 +1937,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -1965,7 +1965,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" dependencies = [ "cfg-if", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2406,7 +2406,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2446,7 +2446,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2539,7 +2539,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2627,7 +2627,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "version_check", "yansi", ] @@ -2652,7 +2652,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -2699,7 +2699,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.10", + "rustls 0.23.11", "thiserror", "tokio", "tracing", @@ -2715,7 +2715,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls 0.23.10", + "rustls 0.23.11", "slab", "thiserror", "tinyvec", @@ -2844,7 +2844,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.4.0", + "hyper 1.4.1", "hyper-rustls", "hyper-util", "ipnet", @@ -2855,7 +2855,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.10", + "rustls 0.23.11", "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", @@ -3061,7 +3061,7 @@ dependencies = [ "quote", "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", "serde", - "syn 2.0.68", + "syn 2.0.70", "toml", ] @@ -3197,30 +3197,31 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.5", "subtle", "zeroize", ] [[package]] name = "rustls" -version = "0.23.10" +version = "0.23.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05cff451f60db80f490f3c182b77c35260baace73209e9cdbbe526bfe3a4d402" +checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" dependencies = [ + "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.4", + "rustls-webpki 0.102.5", "subtle", "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -3257,9 +3258,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.4" +version = "0.102.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff448f7e92e913c4b7d4c6d8e4540a1724b319b4152b8aef6d4cf8339712b33e" +checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" dependencies = [ "ring", "rustls-pki-types", @@ -3500,22 +3501,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.203" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3808,9 +3809,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.68" +version = "2.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" dependencies = [ "proc-macro2", "quote", @@ -3883,7 +3884,7 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -3982,9 +3983,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.6.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -4032,7 +4033,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4062,7 +4063,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.10", + "rustls 0.23.11", "rustls-pki-types", "tokio", ] @@ -4112,7 +4113,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.14", + "toml_edit 0.22.15", ] [[package]] @@ -4137,9 +4138,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.14" +version = "0.22.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" dependencies = [ "indexmap 2.2.6", "serde", @@ -4162,7 +4163,7 @@ dependencies = [ "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", - "hyper 0.14.29", + "hyper 0.14.30", "hyper-timeout", "percent-encoding", "pin-project", @@ -4248,7 +4249,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390 dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", ] [[package]] @@ -4417,16 +4418,15 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.9.7" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11a831e3c0b56e438a28308e7c810799e3c118417f342d30ecec080105395cd" +checksum = "72139d247e5f97a3eff96229a7ae85ead5328a39efe76f8bf5a06313d505b6ea" dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.22.4", + "rustls 0.23.11", "rustls-pki-types", - "rustls-webpki 0.102.4", "url", "webpki-roots", ] @@ -4457,9 +4457,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" -version = "1.9.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de17fd2f7da591098415cff336e12965a28061ddace43b59cb3c430179c9439" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", "serde", @@ -4519,7 +4519,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-shared", ] @@ -4553,7 +4553,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.68", + "syn 2.0.70", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4662,7 +4662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4671,7 +4671,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4689,7 +4689,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -4709,18 +4709,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -4731,9 +4731,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -4743,9 +4743,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -4755,15 +4755,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -4773,9 +4773,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -4785,9 +4785,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -4797,9 +4797,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -4809,9 +4809,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" @@ -4876,27 +4876,27 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.11+zstd.1.5.6" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75652c55c0b6f3e6f12eb786fe1bc960396bf05a1eb3bf1f3691c3610ac2e6d4" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 0d3d59fd..38065bcc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -113,7 +113,7 @@ features = [ ] [workspace.dependencies.serde] -version = "1.0.203" +version = "1.0.204" features = ["rc"] [workspace.dependencies.serde_json] @@ -169,7 +169,7 @@ default-features = false # used for conduit's CLI and admin room command parsing [workspace.dependencies.clap] -version = "4.5.4" +version = "4.5.9" default-features = false features = [ "std", @@ -265,7 +265,7 @@ version = "2.1.1" version = "0.3.1" [workspace.dependencies.async-trait] -version = "0.1.80" +version = "0.1.81" [workspace.dependencies.lru-cache] version = "0.1.2" From 53fa7c3729d5f80a2b1a17660509bbbf6719adc3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 02:47:07 -0400 Subject: [PATCH 0433/2091] nix: add `all-features` outputs/packages Signed-off-by: strawberry --- flake.nix | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/flake.nix b/flake.nix index b252193e..ae8533b9 100644 --- a/flake.nix +++ b/flake.nix @@ -114,9 +114,15 @@ { packages = { default = scopeHost.main; + all-features = scopeHost.main.override { all_features = true; }; hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; }; oci-image = scopeHost.oci-image; + oci-image-all-features = scopeHost.oci-image.override { + main = scopeHost.main.override { + all_features = true; + }; + }; oci-image-hmalloc = scopeHost.oci-image.override { main = scopeHost.main.override { features = ["hardened_malloc"]; @@ -151,6 +157,14 @@ value = scopeCrossStatic.main; } + # An output for a statically-linked binary with `--all-features` + { + name = "${binaryName}-all-features"; + value = scopeCrossStatic.main.override { + all_features = true; + }; + } + # An output for a statically-linked binary with hardened_malloc { name = "${binaryName}-hmalloc"; @@ -165,6 +179,16 @@ value = scopeCrossStatic.oci-image; } + # An output for an OCI image based on that binary with `--all-features` + { + name = "oci-image-${crossSystem}-all-features"; + value = scopeCrossStatic.oci-image.override { + main = scopeCrossStatic.main.override { + all_features = true; + }; + }; + } + # An output for an OCI image based on that binary with hardened_malloc { name = "oci-image-${crossSystem}-hmalloc"; From bd71435a2222fe7afe9587003e1956f558b08cd1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 02:48:44 -0400 Subject: [PATCH 0434/2091] ci: build static binaries and OCI imgs with `--all-features` Signed-off-by: strawberry --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c6b3bb6..66ecdce8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,7 +246,7 @@ jobs: CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - bin/nix-build-and-cache just .#static-${{ matrix.target }} + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features mkdir -v -p target/release/ mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ cp -v -f result/bin/conduit target/release/conduwuit @@ -273,7 +273,7 @@ jobs: - name: Build OCI image ${{ matrix.target }} run: | - bin/nix-build-and-cache just .#oci-image-${{ matrix.target }} + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features cp -v -f result oci-image-${{ matrix.target }}.tar.gz - name: Upload OCI image ${{ matrix.target }} From 438911c18d4c93c18ddfb645ad9ec881bb2f7d90 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 12:34:28 -0400 Subject: [PATCH 0435/2091] nix: remove unnecessary dependencies gathering from allFeatures Signed-off-by: strawberry --- nix/pkgs/main/default.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 5c8ade13..2897e675 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -25,11 +25,7 @@ let # on the nix side depend on feature values. crateFeatures = path: let manifest = lib.importTOML "${path}/Cargo.toml"; in - lib.remove "default" (lib.attrNames manifest.features) ++ - lib.attrNames - (lib.filterAttrs - (_: dependency: dependency.optional or false) - manifest.dependencies); + lib.remove "default" (lib.attrNames manifest.features); crateDefaultFeatures = path: (lib.importTOML "${path}/Cargo.toml").features.default; allDefaultFeatures = crateDefaultFeatures "${inputs.self}/src/main"; From 7a4bbe2ff665568aa0cfa807e75551baf08f8aec Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 13:38:38 -0400 Subject: [PATCH 0436/2091] nix: drop hardened_malloc feature from all-features builds this is non-functional and i've spent far too long trying to figure out what the issue is Signed-off-by: strawberry --- flake.nix | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index ae8533b9..8315c6e1 100644 --- a/flake.nix +++ b/flake.nix @@ -114,13 +114,19 @@ { packages = { default = scopeHost.main; - all-features = scopeHost.main.override { all_features = true; }; + all-features = scopeHost.main.override { + all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; + }; hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; }; oci-image = scopeHost.oci-image; oci-image-all-features = scopeHost.oci-image.override { main = scopeHost.main.override { all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; }; }; oci-image-hmalloc = scopeHost.oci-image.override { @@ -162,6 +168,8 @@ name = "${binaryName}-all-features"; value = scopeCrossStatic.main.override { all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; }; } @@ -185,6 +193,8 @@ value = scopeCrossStatic.oci-image.override { main = scopeCrossStatic.main.override { all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; }; }; } @@ -210,7 +220,11 @@ devShells.default = mkDevShell scopeHostStatic; devShells.all-features = mkDevShell (scopeHostStatic.overrideScope (final: prev: { - main = prev.main.override { all_features = true; }; + main = prev.main.override { + all_features = true; + # this is non-functional on nix for some reason + disable_features = ["hardened_malloc"]; + }; })); devShells.no-features = mkDevShell (scopeHostStatic.overrideScope (final: prev: { From d036d8adcbaa74eee50290a1655be2c2746813e7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 14:20:35 -0400 Subject: [PATCH 0437/2091] bump rust-rocksdb to 0.27.1 Signed-off-by: strawberry --- Cargo.lock | 8 ++++---- deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eb24aa21..1846c82d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3123,8 +3123,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.23.0+9.3.1" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=b4887edfb84771336930855727390edec07d63fa#b4887edfb84771336930855727390edec07d63fa" +version = "0.23.1+9.3.1" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=db1ba33c2b78ad228e2525e8902d059c24fc81a1#db1ba33c2b78ad228e2525e8902d059c24fc81a1" dependencies = [ "bindgen", "bzip2-sys", @@ -3140,8 +3140,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.27.0" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=b4887edfb84771336930855727390edec07d63fa#b4887edfb84771336930855727390edec07d63fa" +version = "0.27.1" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=db1ba33c2b78ad228e2525e8902d059c24fc81a1#db1ba33c2b78ad228e2525e8902d059c24fc81a1" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 03efc61c..82b36ff2 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/zaidoon1/rust-rocksdb" -rev = "b4887edfb84771336930855727390edec07d63fa" +rev = "db1ba33c2b78ad228e2525e8902d059c24fc81a1" #branch = "master" default-features = false From 93e7cf461d7702ea4d9d6cd622c29fcc95eb3d0b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 14:56:18 -0400 Subject: [PATCH 0438/2091] add client IP logging to media requests Signed-off-by: strawberry --- src/api/client/media.rs | 60 +++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 44978caf..9d3fbabe 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -2,6 +2,7 @@ use std::{io::Cursor, sync::Arc, time::Duration}; +use axum_client_ip::InsecureClientIp; use conduit::{debug, error, utils::math::ruma_from_usize, warn}; use image::io::Reader as ImgReader; use ipaddress::IPAddress; @@ -64,18 +65,22 @@ pub(crate) async fn get_media_config_v1_route( /// # `GET /_matrix/media/v3/preview_url` /// /// Returns URL preview. +#[tracing::instrument(skip_all, fields(%client), name = "url_preview")] pub(crate) async fn get_media_preview_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let url = &body.url; if !url_preview_allowed(url) { + warn!(%sender_user, "URL is not allowed to be previewed: {url}"); return Err(Error::BadRequest(ErrorKind::forbidden(), "URL is not allowed to be previewed")); } match get_url_preview(url).await { Ok(preview) => { let res = serde_json::value::to_raw_value(&preview).map_err(|e| { - error!("Failed to convert UrlPreviewData into a serde json value: {}", e); + error!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}"); Error::BadRequest( ErrorKind::LimitExceeded { retry_after: Some(RetryAfter::Delay(Duration::from_secs(5))), @@ -87,7 +92,7 @@ pub(crate) async fn get_media_preview_route( Ok(get_media_preview::v3::Response::from_raw_value(res)) }, Err(e) => { - warn!("Failed to generate a URL preview: {e}"); + warn!(%sender_user, "Failed to generate a URL preview: {e}"); // there doesn't seem to be an agreed-upon error code in the spec. // the only response codes in the preview_url spec page are 200 and 429. @@ -108,10 +113,13 @@ pub(crate) async fn get_media_preview_route( /// See /// /// Returns URL preview. +#[tracing::instrument(skip_all, fields(%client), name = "url_preview")] pub(crate) async fn get_media_preview_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - get_media_preview_route(body).await.map(RumaResponse) + get_media_preview_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } /// # `POST /_matrix/media/v3/upload` @@ -120,8 +128,9 @@ pub(crate) async fn get_media_preview_v1_route( /// /// - Some metadata will be saved in the database /// - Media will be saved in the media/ directory +#[tracing::instrument(skip_all, fields(%client), name = "media_upload")] pub(crate) async fn create_content_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -167,10 +176,13 @@ pub(crate) async fn create_content_route( /// /// - Some metadata will be saved in the database /// - Media will be saved in the media/ directory +#[tracing::instrument(skip_all, fields(%client), name = "media_upload")] pub(crate) async fn create_content_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - create_content_route(body).await.map(RumaResponse) + create_content_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } /// # `GET /_matrix/media/v3/download/{serverName}/{mediaId}` @@ -181,7 +193,10 @@ pub(crate) async fn create_content_v1_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds -pub(crate) async fn get_content_route(body: Ruma) -> Result { +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] +pub(crate) async fn get_content_route( + InsecureClientIp(client): InsecureClientIp, body: Ruma, +) -> Result { let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { @@ -243,10 +258,13 @@ pub(crate) async fn get_content_route(body: Ruma) -> R /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - get_content_route(body).await.map(RumaResponse) + get_content_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } /// # `GET /_matrix/media/v3/download/{serverName}/{mediaId}/{fileName}` @@ -257,8 +275,9 @@ pub(crate) async fn get_content_v1_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_as_filename_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); @@ -328,10 +347,13 @@ pub(crate) async fn get_content_as_filename_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_as_filename_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - get_content_as_filename_route(body).await.map(RumaResponse) + get_content_as_filename_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } /// # `GET /_matrix/media/v3/thumbnail/{serverName}/{mediaId}` @@ -342,8 +364,9 @@ pub(crate) async fn get_content_as_filename_v1_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] pub(crate) async fn get_content_thumbnail_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); @@ -453,10 +476,13 @@ pub(crate) async fn get_content_thumbnail_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] pub(crate) async fn get_content_thumbnail_v1_route( - body: Ruma, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { - get_content_thumbnail_route(body).await.map(RumaResponse) + get_content_thumbnail_route(InsecureClientIp(client), body) + .await + .map(RumaResponse) } async fn get_remote_content( From 0fa6976d868b3a1f5c146a2c64e5c796c12a6f32 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 15:02:09 -0400 Subject: [PATCH 0439/2091] add client IP and user logging on join, remove unnecessary Option Signed-off-by: strawberry --- src/admin/user/commands.rs | 2 +- src/api/client/account.rs | 2 +- src/api/client/membership.rs | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 884e1d29..12aa0170 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -95,7 +95,7 @@ pub(super) async fn create( if let Some(room_id_server_name) = room.server_name() { match join_room_by_id_helper( - Some(&user_id), + &user_id, room, Some("Automatically joining this room upon registration".to_owned()), &[room_id_server_name.to_owned(), services().globals.server_name().to_owned()], diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 66dc7a26..d34211bf 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -376,7 +376,7 @@ pub(crate) async fn register_route( if let Some(room_id_server_name) = room.server_name() { if let Err(e) = join_room_by_id_helper( - Some(&user_id), + &user_id, room, Some("Automatically joining this room upon registration".to_owned()), &[room_id_server_name.to_owned(), services().globals.server_name().to_owned()], diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 0792ed89..6ebcd95b 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -202,7 +202,7 @@ pub(crate) async fn join_room_by_id_route( } join_room_by_id_helper( - body.sender_user.as_deref(), + sender_user, &body.room_id, body.reason.clone(), &servers, @@ -301,7 +301,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( }; let join_room_response = join_room_by_id_helper( - Some(sender_user), + sender_user, &room_id, body.reason.clone(), &servers, @@ -653,11 +653,9 @@ pub(crate) async fn joined_members_route( } pub async fn join_room_by_id_helper( - sender_user: Option<&UserId>, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], + sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], third_party_signed: Option<&ThirdPartySigned>, ) -> Result { - let sender_user = sender_user.expect("user is authenticated"); - if matches!(services().rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { info!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { @@ -679,6 +677,7 @@ pub async fn join_room_by_id_helper( } } +#[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote")] async fn join_room_by_id_helper_remote( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, @@ -1014,6 +1013,7 @@ async fn join_room_by_id_helper_remote( Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) } +#[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_local")] async fn join_room_by_id_helper_local( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, From 192c1e08da38b8be621fd39a9a67be85493136c8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 7 Jul 2024 15:18:07 -0400 Subject: [PATCH 0440/2091] add `exclude_disabled` and `exclude_banned` room list admin cmd arguments Signed-off-by: strawberry --- src/admin/room/mod.rs | 12 ++++++++++- src/admin/room/room_commands.rs | 36 +++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index 72e72793..b4fa15bd 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -16,6 +16,14 @@ pub(super) enum RoomCommand { /// - List all rooms the server knows about List { page: Option, + + /// Excludes rooms that we have federation disabled with + #[arg(long)] + exclude_disabled: bool, + + /// Excludes rooms that we have banned + #[arg(long)] + exclude_banned: bool, }, #[command(subcommand)] @@ -179,6 +187,8 @@ pub(super) async fn process(command: RoomCommand, body: Vec<&str>) -> Result list(body, page).await?, + exclude_disabled, + exclude_banned, + } => list(body, page, exclude_disabled, exclude_banned).await?, }) } diff --git a/src/admin/room/room_commands.rs b/src/admin/room/room_commands.rs index f64ccf30..d47edce2 100644 --- a/src/admin/room/room_commands.rs +++ b/src/admin/room/room_commands.rs @@ -1,18 +1,46 @@ use std::fmt::Write; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId}; +use ruma::events::room::message::RoomMessageEventContent; use crate::{escape_html, get_room_info, handler::PAGE_SIZE, services, Result}; -pub(super) async fn list(_body: Vec<&str>, page: Option) -> Result { +pub(super) async fn list( + _body: Vec<&str>, page: Option, exclude_disabled: bool, exclude_banned: bool, +) -> Result { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); let mut rooms = services() .rooms .metadata .iter_ids() - .filter_map(Result::ok) - .map(|id: OwnedRoomId| get_room_info(&id)) + .filter_map(|room_id| { + room_id + .ok() + .filter(|room_id| { + if exclude_disabled + && services() + .rooms + .metadata + .is_disabled(room_id) + .unwrap_or(false) + { + return false; + } + + if exclude_banned + && services() + .rooms + .metadata + .is_banned(room_id) + .unwrap_or(false) + { + return false; + } + + true + }) + .map(|room_id| get_room_info(&room_id)) + }) .collect::>(); rooms.sort_by_key(|r| r.1); rooms.reverse(); From a8e690f22b4678c81d08a62841597ab7a2e73955 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 8 Jul 2024 17:07:31 -0400 Subject: [PATCH 0441/2091] nix: unset `patches` in rocksdb we have the patch already and i dont want to make it nix exclusive by reverting it in my rocksdb fork Signed-off-by: strawberry --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index 8315c6e1..9dc60a99 100644 --- a/flake.nix +++ b/flake.nix @@ -42,6 +42,9 @@ "v" (builtins.fromJSON (builtins.readFile ./flake.lock)) .nodes.rocksdb.original.ref; + # we have this already at https://github.com/girlbossceo/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155 + # unsetting this so i don't have to revert it and make this nix exclusive + patches = []; }); # TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available liburing = pkgs.liburing.overrideAttrs (old: { From 05befa4ba248f578643b46bd92450ade849b622c Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 8 Jul 2024 17:08:11 -0400 Subject: [PATCH 0442/2091] nix: make enableLiburing in rocksdb conditional instead of default true for some reason enableLiburing in nixpkgs rocksdb is default true which breaks Darwin entirely Signed-off-by: strawberry --- nix/pkgs/main/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 2897e675..e3d23a43 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -66,6 +66,10 @@ buildDepsOnlyEnv = # # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17 enableJemalloc = featureEnabled "jemalloc" && !stdenv.isDarwin; + + # for some reason enableLiburing in nixpkgs rocksdb is default true + # which breaks Darwin entirely + enableLiburing = enableLiburing; }).overrideAttrs (old: { # TODO: static rocksdb fails to build on darwin # build log at From 53223a4d5ff94e63822ceea567bf5ff1612afeb7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 8 Jul 2024 19:17:10 -0400 Subject: [PATCH 0443/2091] remove snappy as a default feature Signed-off-by: strawberry --- Cargo.toml | 1 - deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 38065bcc..fcbe53d5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -311,7 +311,6 @@ package = "rust-rocksdb-uwu" features = [ "multi-threaded-cf", "mt_static", - "snappy", "lz4", "zstd", "zlib", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 82b36ff2..57b71491 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true version = "0.0.1" [features] -default = ["snappy", "lz4", "zstd", "zlib", "bzip2"] +default = ["lz4", "zstd", "zlib", "bzip2"] jemalloc = ["rust-rocksdb/jemalloc"] io-uring = ["rust-rocksdb/io-uring"] valgrind = ["rust-rocksdb/valgrind"] From f129d90900fe238a114578bb74c0fc219230cfe2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 8 Jul 2024 19:51:05 -0400 Subject: [PATCH 0444/2091] nix: delete `-DWITH_SNAPPY=1` from rocksdb cmakeFlags Signed-off-by: strawberry --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 9dc60a99..e20da52b 100644 --- a/flake.nix +++ b/flake.nix @@ -45,6 +45,10 @@ # we have this already at https://github.com/girlbossceo/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155 # unsetting this so i don't have to revert it and make this nix exclusive patches = []; + # no real reason to have snappy, no one uses this + cmakeFlags = pkgs.lib.subtractLists + [ "-DWITH_SNAPPY=1" ] + old.cmakeFlags; }); # TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available liburing = pkgs.liburing.overrideAttrs (old: { From 080975ab0e862a40a6c035c5198244cff8761e1f Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 9 Jul 2024 08:44:24 -0400 Subject: [PATCH 0445/2091] adjust/update complement conduwuit config.toml Signed-off-by: strawberry --- nix/pkgs/complement/config.toml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index db1f2d81..15f939bb 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -5,13 +5,17 @@ allow_guest_registration = true allow_public_room_directory_over_federation = true allow_public_room_directory_without_auth = true allow_registration = true -allow_unstable_room_versions = true -database_backend = "rocksdb" database_path = "/database" -log = "trace" +log = "trace,h2=warn,hyper=warn" port = [8008, 8448] trusted_servers = [] +query_trusted_key_servers_first = false yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true +ip_range_denylist = [] +url_preview_domain_contains_allowlist = ["*"] +media_compat_file_link = false +media_statup_check = false +rocksdb_direct_io = false [global.tls] certs = "/certificate.crt" From aa9540af21d7533f39a8b6d44705f79e7e45935f Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 10 Jul 2024 19:56:34 -0400 Subject: [PATCH 0446/2091] slightly simplify send_state_event route Signed-off-by: strawberry --- src/api/client/state.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 9247b123..c63ddba5 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -36,18 +36,16 @@ pub(crate) async fn send_state_event_for_key_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event_id = send_state_event_for_key_helper( - sender_user, - &body.room_id, - &body.event_type, - &body.body.body, - body.state_key.clone(), - ) - .await?; - - let event_id = (*event_id).to_owned(); Ok(send_state_event::v3::Response { - event_id, + event_id: send_state_event_for_key_helper( + sender_user, + &body.room_id, + &body.event_type, + &body.body.body, + body.state_key.clone(), + ) + .await? + .into(), }) } From b5d4a1c1b0c1e5cd8e399caa01744d46a8e34ff4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 10 Jul 2024 19:57:10 -0400 Subject: [PATCH 0447/2091] drop harmless state event not found message to debug_info Signed-off-by: strawberry --- src/api/client/state.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index c63ddba5..abff9218 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{error, warn}; +use conduit::{debug_info, error}; use ruma::{ api::client::{ error::ErrorKind, @@ -126,7 +126,7 @@ pub(crate) async fn get_state_events_for_key_route( .state_accessor .room_state_get(&body.room_id, &body.event_type, &body.state_key)? .ok_or_else(|| { - warn!("State event {:?} not found in room {:?}", &body.event_type, &body.room_id); + debug_info!("State event {:?} not found in room {:?}", &body.event_type, &body.room_id); Error::BadRequest(ErrorKind::NotFound, "State event not found.") })?; if body From 8077e910f648078da2eb0e6b1d8733165775265d Mon Sep 17 00:00:00 2001 From: AlexPewMaster Date: Sat, 13 Jul 2024 22:59:07 +0200 Subject: [PATCH 0448/2091] Fix CONDUWUIT_CONFIG variable in docker-compose.yml files (+ remove deprecated compose version) --- docs/deploying/docker-compose.for-traefik.yml | 3 +-- docs/deploying/docker-compose.override.yml | 1 - docs/deploying/docker-compose.with-caddy.yml | 2 +- docs/deploying/docker-compose.with-traefik.yml | 4 +--- docs/deploying/docker-compose.yml | 4 +--- 5 files changed, 4 insertions(+), 10 deletions(-) diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index ec3e720c..d10e5815 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -1,5 +1,4 @@ # conduwuit - Behind Traefik Reverse Proxy -version: '2.4' # uses '2.4' for cpuset services: homeserver: @@ -24,7 +23,7 @@ services: CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' #CONDUWUIT_LOG: warn,state_res=warn CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above + #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above #cpuset: "0-4" # Uncomment to limit to specific CPU cores # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container diff --git a/docs/deploying/docker-compose.override.yml b/docs/deploying/docker-compose.override.yml index 2e937e75..23d6a90b 100644 --- a/docs/deploying/docker-compose.override.yml +++ b/docs/deploying/docker-compose.override.yml @@ -1,5 +1,4 @@ # conduwuit - Traefik Reverse Proxy Labels -version: '2.4' # uses '2.4' for cpuset services: homeserver: diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index 41d8856d..899f4d67 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -39,7 +39,7 @@ services: CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' #CONDUWUIT_LOG: warn,state_res=warn CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above + #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above networks: - caddy labels: diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index c93f5414..79d20051 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -1,5 +1,4 @@ # conduwuit - Behind Traefik Reverse Proxy -version: '2.4' # uses '2.4' for cpuset services: homeserver: @@ -16,7 +15,7 @@ services: CONDUWUIT_SERVER_NAME: your.server.name # EDIT THIS CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' CONDUWUIT_ALLOW_REGISTRATION : 'true' - #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above + #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above ### Uncomment and change values as desired # CONDUWUIT_ADDRESS: 0.0.0.0 # CONDUWUIT_PORT: 6167 @@ -28,7 +27,6 @@ services: # CONDUWUIT_DATABASE_PATH: /srv/conduwuit/.local/share/conduwuit # CONDUWUIT_WORKERS: 10 # CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB - #cpuset: "0-4" # Uncomment to limit to specific CPU cores # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container # to serve those two as static files. If you want to use a different way, delete or comment the below service, here diff --git a/docs/deploying/docker-compose.yml b/docs/deploying/docker-compose.yml index 066c8fe1..bc9f2477 100644 --- a/docs/deploying/docker-compose.yml +++ b/docs/deploying/docker-compose.yml @@ -1,5 +1,4 @@ # conduwuit -version: '2.4' # uses '2.4' for cpuset services: homeserver: @@ -24,8 +23,7 @@ services: CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' #CONDUWUIT_LOG: warn,state_res=warn CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: './conduwuit.toml' # Uncomment if you mapped config toml above - #cpuset: "0-4" # Uncomment to limit to specific CPU cores + #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above # ### Uncomment if you want to use your own Element-Web App. ### Note: You need to provide a config.json for Element and you also need a second From 454dd43d4ca8561cf92a6e0ed89457590311dab4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:04:05 +0000 Subject: [PATCH 0449/2091] fix membership route ABA's Signed-off-by: Jason Volk --- src/api/client/membership.rs | 42 ++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 6ebcd95b..da96acc8 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -7,7 +7,7 @@ use std::{ use axum_client_ip::InsecureClientIp; use conduit::{ - debug, error, info, trace, utils, + debug, debug_warn, error, info, trace, utils, utils::{math::continue_exponential_backoff_secs, mutex_map}, warn, Error, PduEvent, Result, }; @@ -366,6 +366,12 @@ pub(crate) async fn invite_user_route( pub(crate) async fn kick_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; + let mut event: RoomMemberEventContent = serde_json::from_str( services() .rooms @@ -383,12 +389,6 @@ pub(crate) async fn kick_user_route(body: Ruma) -> Resul event.membership = MembershipState::Leave; event.reason.clone_from(&body.reason); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; - services() .rooms .timeline @@ -417,6 +417,12 @@ pub(crate) async fn kick_user_route(body: Ruma) -> Resul pub(crate) async fn ban_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; + let event = services() .rooms .state_accessor @@ -447,12 +453,6 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< }, )?; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; - services() .rooms .timeline @@ -481,6 +481,12 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< pub(crate) async fn unban_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let state_lock = services() + .globals + .roomid_mutex_state + .lock(&body.room_id) + .await; + let mut event: RoomMemberEventContent = serde_json::from_str( services() .rooms @@ -496,12 +502,6 @@ pub(crate) async fn unban_user_route(body: Ruma) -> Res event.reason.clone_from(&body.reason); event.join_authorized_via_users_server = None; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; - services() .rooms .timeline @@ -656,6 +656,8 @@ pub async fn join_room_by_id_helper( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], third_party_signed: Option<&ThirdPartySigned>, ) -> Result { + let state_lock = services().rooms.state.mutex.lock(room_id).await; + if matches!(services().rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { info!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { @@ -663,8 +665,6 @@ pub async fn join_room_by_id_helper( }); } - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; - // Ask a remote server if we are not participating in this room if !services() .rooms From c62d653989ffc1d0e6176d5d33fbc44a6f3df4eb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:08:40 +0000 Subject: [PATCH 0450/2091] tweak some log levels Signed-off-by: Jason Volk --- src/api/client/membership.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index da96acc8..4b664c5b 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -659,7 +659,7 @@ pub async fn join_room_by_id_helper( let state_lock = services().rooms.state.mutex.lock(room_id).await; if matches!(services().rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { - info!("{sender_user} is already joined in {room_id}"); + debug_warn!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { room_id: room_id.into(), }); @@ -1018,7 +1018,7 @@ async fn join_room_by_id_helper_local( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, ) -> Result { - info!("We can join locally"); + debug!("We can join locally"); let join_rules_event = services() @@ -1118,7 +1118,7 @@ async fn join_room_by_id_helper_local( .iter() .any(|server_name| !server_is_ours(server_name)) { - info!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); + warn!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); let (make_join_response, remote_server) = make_join_request(sender_user, room_id, servers).await?; let room_version_id = match make_join_response.room_version { From 5dcdafe2070fb362cb0477ec4ce8b6068b9eb147 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:09:12 +0000 Subject: [PATCH 0451/2091] take local join branch when remote join would fail Signed-off-by: Jason Volk --- src/api/client/membership.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 4b664c5b..5c56faa0 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -665,15 +665,17 @@ pub async fn join_room_by_id_helper( }); } - // Ask a remote server if we are not participating in this room - if !services() + if services() .rooms .state_cache .server_in_room(services().globals.server_name(), room_id)? + || servers.is_empty() + || (servers.len() == 1 && server_is_ours(&servers[0])) { - join_room_by_id_helper_remote(sender_user, room_id, reason, servers, third_party_signed, state_lock).await - } else { join_room_by_id_helper_local(sender_user, room_id, reason, servers, third_party_signed, state_lock).await + } else { + // Ask a remote server if we are not participating in this room + join_room_by_id_helper_remote(sender_user, room_id, reason, servers, third_party_signed, state_lock).await } } From 50c2d2b80151c888ebb49153d1e8b0c9dee02c44 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:15:45 +0000 Subject: [PATCH 0452/2091] add command to force join user to room (#136) Signed-off-by: Jason Volk --- src/admin/user/commands.rs | 16 +++++++++++++++- src/admin/user/mod.rs | 12 +++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 12aa0170..218dadb9 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -8,7 +8,7 @@ use ruma::{ tag::{TagEvent, TagEventContent, TagInfo}, RoomAccountDataEventType, }, - OwnedRoomId, OwnedUserId, RoomId, + OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, }; use tracing::{error, info, warn}; @@ -334,6 +334,20 @@ pub(super) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Resu Ok(RoomMessageEventContent::text_html(output_plain, output_html)) } +pub(super) async fn force_join_room( + _body: Vec<&str>, user_id: String, room_id: OwnedRoomOrAliasId, +) -> Result { + let user_id = parse_local_user_id(&user_id)?; + let room_id = services().rooms.alias.resolve(&room_id).await?; + + assert!(service::user_is_local(&user_id), "Parsed user_id must be a local user"); + join_room_by_id_helper(&user_id, &room_id, None, &[], None).await?; + + Ok(RoomMessageEventContent::notice_markdown(format!( + "{user_id} has been joined to {room_id}.", + ))) +} + pub(super) async fn put_room_tag( _body: Vec<&str>, user_id: String, room_id: Box, tag: String, ) -> Result { diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 088133a5..e7d3d251 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -2,7 +2,7 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{events::room::message::RoomMessageEventContent, RoomId}; +use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomOrAliasId, RoomId}; use self::commands::*; @@ -65,6 +65,12 @@ pub(super) enum UserCommand { user_id: String, }, + /// - Manually join a local user to a room. + ForceJoinRoom { + user_id: String, + room_id: OwnedRoomOrAliasId, + }, + /// - Puts a room tag for the specified user and room ID. /// /// This is primarily useful if you'd like to set your admin room @@ -113,6 +119,10 @@ pub(super) async fn process(command: UserCommand, body: Vec<&str>) -> Result list_joined_rooms(body, user_id).await?, + UserCommand::ForceJoinRoom { + user_id, + room_id, + } => force_join_room(body, user_id, room_id).await?, UserCommand::PutRoomTag { user_id, room_id, From 01b2928d5581a53cc0e38895b90a3226327907fa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 06:41:29 +0000 Subject: [PATCH 0453/2091] add make user admin command (#136) Signed-off-by: Jason Volk --- src/admin/user/commands.rs | 15 +++++++++++++++ src/admin/user/mod.rs | 8 ++++++++ 2 files changed, 23 insertions(+) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 218dadb9..9e4b348b 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -348,6 +348,21 @@ pub(super) async fn force_join_room( ))) } +pub(super) async fn make_user_admin(_body: Vec<&str>, user_id: String) -> Result { + let user_id = parse_local_user_id(&user_id)?; + let displayname = services() + .users + .displayname(&user_id)? + .unwrap_or_else(|| user_id.to_string()); + + assert!(service::user_is_local(&user_id), "Parsed user_id must be a local user"); + service::admin::make_user_admin(&user_id, displayname).await?; + + Ok(RoomMessageEventContent::notice_markdown(format!( + "{user_id} has been granted admin privileges.", + ))) +} + pub(super) async fn put_room_tag( _body: Vec<&str>, user_id: String, room_id: Box, tag: String, ) -> Result { diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index e7d3d251..cdb5fa5e 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -71,6 +71,11 @@ pub(super) enum UserCommand { room_id: OwnedRoomOrAliasId, }, + /// - Grant server-admin privileges to a user. + MakeUserAdmin { + user_id: String, + }, + /// - Puts a room tag for the specified user and room ID. /// /// This is primarily useful if you'd like to set your admin room @@ -123,6 +128,9 @@ pub(super) async fn process(command: UserCommand, body: Vec<&str>) -> Result force_join_room(body, user_id, room_id).await?, + UserCommand::MakeUserAdmin { + user_id, + } => make_user_admin(body, user_id).await?, UserCommand::PutRoomTag { user_id, room_id, From 2d251eb19ca734baa0fdafbe8c5d0993ec771be9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 20:04:43 +0000 Subject: [PATCH 0454/2091] cleanup on drop for utils::mutex_map. Signed-off-by: Jason Volk --- src/api/client/membership.rs | 12 +++--- src/core/utils/mod.rs | 2 +- src/core/utils/mutex_map.rs | 53 ++++++++++++++++++------- src/core/utils/tests.rs | 53 +++++++++++++++++++++++++ src/service/admin/mod.rs | 6 +-- src/service/globals/mod.rs | 18 ++++++--- src/service/rooms/state/data.rs | 7 ++-- src/service/rooms/state/mod.rs | 13 +++--- src/service/rooms/state_accessor/mod.rs | 10 ++--- src/service/rooms/timeline/mod.rs | 11 ++--- 10 files changed, 131 insertions(+), 54 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 5c56faa0..2ac4c723 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -7,9 +7,8 @@ use std::{ use axum_client_ip::InsecureClientIp; use conduit::{ - debug, debug_warn, error, info, trace, utils, - utils::{math::continue_exponential_backoff_secs, mutex_map}, - warn, Error, PduEvent, Result, + debug, debug_warn, error, info, trace, utils, utils::math::continue_exponential_backoff_secs, warn, Error, + PduEvent, Result, }; use ruma::{ api::{ @@ -36,13 +35,14 @@ use ruma::{ OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use service::sending::convert_to_outgoing_federation_event; use tokio::sync::RwLock; use crate::{ client::{update_avatar_url, update_displayname}, service::{ + globals::RoomMutexGuard, pdu::{gen_event_id_canonical_json, PduBuilder}, + sending::convert_to_outgoing_federation_event, server_is_ours, user_is_local, }, services, Ruma, @@ -682,7 +682,7 @@ pub async fn join_room_by_id_helper( #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote")] async fn join_room_by_id_helper_remote( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, + _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, ) -> Result { info!("Joining {room_id} over federation."); @@ -1018,7 +1018,7 @@ async fn join_room_by_id_helper_remote( #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_local")] async fn join_room_by_id_helper_local( sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, state_lock: mutex_map::Guard<()>, + _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, ) -> Result { debug!("We can join locally"); diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 2b79c3c4..4e5ba480 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -20,7 +20,7 @@ pub use debug::slice_truncated as debug_slice_truncated; pub use hash::calculate_hash; pub use html::Escape as HtmlEscape; pub use json::{deserialize_from_str, to_canonical_object}; -pub use mutex_map::MutexMap; +pub use mutex_map::{Guard as MutexMapGuard, MutexMap}; pub use rand::string as random_string; pub use string::{str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; diff --git a/src/core/utils/mutex_map.rs b/src/core/utils/mutex_map.rs index f102487c..c3c51798 100644 --- a/src/core/utils/mutex_map.rs +++ b/src/core/utils/mutex_map.rs @@ -1,20 +1,22 @@ -use std::{hash::Hash, sync::Arc}; +use std::{fmt::Debug, hash::Hash, sync::Arc}; -type Value = tokio::sync::Mutex; -type ArcMutex = Arc>; -type HashMap = std::collections::HashMap>; -type MapMutex = std::sync::Mutex>; -type Map = MapMutex; +use tokio::sync::OwnedMutexGuard as Omg; /// Map of Mutexes pub struct MutexMap { map: Map, } -pub struct Guard { - _guard: tokio::sync::OwnedMutexGuard, +pub struct Guard { + map: Map, + val: Omg, } +type Map = Arc>; +type MapMutex = std::sync::Mutex>; +type HashMap = std::collections::HashMap>; +type Value = Arc>; + impl MutexMap where Key: Send + Hash + Eq + Clone, @@ -23,28 +25,38 @@ where #[must_use] pub fn new() -> Self { Self { - map: Map::::new(HashMap::::new()), + map: Map::new(MapMutex::new(HashMap::new())), } } - pub async fn lock(&self, k: &K) -> Guard + #[tracing::instrument(skip(self), level = "debug")] + pub async fn lock(&self, k: &K) -> Guard where - K: ?Sized + Send + Sync, + K: ?Sized + Send + Sync + Debug, Key: for<'a> From<&'a K>, { let val = self .map .lock() - .expect("map mutex locked") + .expect("locked") .entry(k.into()) .or_default() .clone(); - let guard = val.lock_owned().await; - Guard:: { - _guard: guard, + Guard:: { + map: Arc::clone(&self.map), + val: val.lock_owned().await, } } + + #[must_use] + pub fn contains(&self, k: &Key) -> bool { self.map.lock().expect("locked").contains_key(k) } + + #[must_use] + pub fn is_empty(&self) -> bool { self.map.lock().expect("locked").is_empty() } + + #[must_use] + pub fn len(&self) -> usize { self.map.lock().expect("locked").len() } } impl Default for MutexMap @@ -54,3 +66,14 @@ where { fn default() -> Self { Self::new() } } + +impl Drop for Guard { + fn drop(&mut self) { + if Arc::strong_count(Omg::mutex(&self.val)) <= 2 { + self.map + .lock() + .expect("locked") + .retain(|_, val| !Arc::ptr_eq(val, Omg::mutex(&self.val)) || Arc::strong_count(val) > 2); + } + } +} diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index add15861..43968947 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -81,3 +81,56 @@ fn checked_add_overflow() { let res = checked!(a + 1).expect("overflow"); assert_eq!(res, 0); } + +#[tokio::test] +async fn mutex_map_cleanup() { + use crate::utils::MutexMap; + + let map = MutexMap::::new(); + + let lock = map.lock("foo").await; + assert!(!map.is_empty(), "map must not be empty"); + + drop(lock); + assert!(map.is_empty(), "map must be empty"); +} + +#[tokio::test] +async fn mutex_map_contend() { + use std::sync::Arc; + + use tokio::sync::Barrier; + + use crate::utils::MutexMap; + + let map = Arc::new(MutexMap::::new()); + let seq = Arc::new([Barrier::new(2), Barrier::new(2)]); + let str = "foo".to_owned(); + + let seq_ = seq.clone(); + let map_ = map.clone(); + let str_ = str.clone(); + let join_a = tokio::spawn(async move { + let _lock = map_.lock(&str_).await; + assert!(!map_.is_empty(), "A0 must not be empty"); + seq_[0].wait().await; + assert!(map_.contains(&str_), "A1 must contain key"); + }); + + let seq_ = seq.clone(); + let map_ = map.clone(); + let str_ = str.clone(); + let join_b = tokio::spawn(async move { + let _lock = map_.lock(&str_).await; + assert!(!map_.is_empty(), "B0 must not be empty"); + seq_[1].wait().await; + assert!(map_.contains(&str_), "B1 must contain key"); + }); + + seq[0].wait().await; + assert!(map.contains(&str), "Must contain key"); + seq[1].wait().await; + + tokio::try_join!(join_b, join_a).expect("joined"); + assert!(map.is_empty(), "Must be empty"); +} diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index ca0e551b..e0dd1760 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,7 +9,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{error, utils::mutex_map, Error, Result}; +use conduit::{error, Error, Result}; pub use create::create_admin_room; pub use grant::make_user_admin; use loole::{Receiver, Sender}; @@ -26,7 +26,7 @@ use tokio::{ task::JoinHandle, }; -use crate::{pdu::PduBuilder, services, user_is_local, PduEvent}; +use crate::{globals::RoomMutexGuard, pdu::PduBuilder, services, user_is_local, PduEvent}; const COMMAND_QUEUE_LIMIT: usize = 512; @@ -270,7 +270,7 @@ async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, use } async fn handle_response_error( - e: &Error, room_id: &RoomId, user_id: &UserId, state_lock: &mutex_map::Guard<()>, + e: &Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, ) -> Result<()> { error!("Failed to build and append admin room response PDU: \"{e}\""); let error_room_message = RoomMessageEventContent::text_plain(format!( diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 11bfc88c..a5b70835 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -12,7 +12,11 @@ use std::{ time::Instant, }; -use conduit::{error, trace, utils::MutexMap, Config, Result}; +use conduit::{ + error, trace, + utils::{MutexMap, MutexMapGuard}, + Config, Result, +}; use data::Data; use ipaddress::IPAddress; use regex::RegexSet; @@ -27,8 +31,6 @@ use url::Url; use crate::services; -type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries - pub struct Service { pub db: Data, @@ -43,9 +45,9 @@ pub struct Service { pub bad_event_ratelimiter: Arc>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub bad_query_ratelimiter: Arc>>, - pub roomid_mutex_insert: MutexMap, - pub roomid_mutex_state: MutexMap, - pub roomid_mutex_federation: MutexMap, + pub roomid_mutex_insert: RoomMutexMap, + pub roomid_mutex_state: RoomMutexMap, + pub roomid_mutex_federation: RoomMutexMap, pub roomid_federationhandletime: RwLock>, pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, @@ -53,6 +55,10 @@ pub struct Service { pub admin_alias: OwnedRoomAliasId, } +pub type RoomMutexMap = MutexMap; +pub type RoomMutexGuard = MutexMapGuard; +type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index aad3bede..b62adf60 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -3,7 +3,8 @@ use std::{collections::HashSet, sync::Arc}; use conduit::{utils, Error, Result}; use database::{Database, Map}; use ruma::{EventId, OwnedEventId, RoomId}; -use utils::mutex_map; + +use crate::globals::RoomMutexGuard; pub(super) struct Data { shorteventid_shortstatehash: Arc, @@ -35,7 +36,7 @@ impl Data { &self, room_id: &RoomId, new_shortstatehash: u64, - _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { self.roomid_shortstatehash .insert(room_id.as_bytes(), &new_shortstatehash.to_be_bytes())?; @@ -68,7 +69,7 @@ impl Data { &self, room_id: &RoomId, event_ids: Vec, - _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 52ee89d1..b46a9d04 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -5,10 +5,7 @@ use std::{ sync::Arc, }; -use conduit::{ - utils::{calculate_hash, mutex_map}, - warn, Error, Result, -}; +use conduit::{utils::calculate_hash, warn, Error, Result}; use data::Data; use ruma::{ api::client::error::ErrorKind, @@ -22,7 +19,7 @@ use ruma::{ }; use super::state_compressor::CompressedStateEvent; -use crate::{services, PduEvent}; +use crate::{globals::RoomMutexGuard, services, PduEvent}; pub struct Service { db: Data, @@ -46,7 +43,7 @@ impl Service { shortstatehash: u64, statediffnew: Arc>, _statediffremoved: Arc>, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { for event_id in statediffnew.iter().filter_map(|new| { services() @@ -318,7 +315,7 @@ impl Service { &self, room_id: &RoomId, shortstatehash: u64, - mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { self.db.set_room_state(room_id, shortstatehash, mutex_lock) } @@ -358,7 +355,7 @@ impl Service { &self, room_id: &RoomId, event_ids: Vec, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { self.db .set_forward_extremities(room_id, event_ids, state_lock) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index a3567857..35719c15 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -6,11 +6,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{ - error, - utils::{math::usize_from_f64, mutex_map}, - warn, Error, Result, -}; +use conduit::{error, utils::math::usize_from_f64, warn, Error, Result}; use data::Data; use lru_cache::LruCache; use ruma::{ @@ -37,7 +33,7 @@ use ruma::{ }; use serde_json::value::to_raw_value; -use crate::{pdu::PduBuilder, services, PduEvent}; +use crate::{globals::RoomMutexGuard, pdu::PduBuilder, services, PduEvent}; pub struct Service { db: Data, @@ -333,7 +329,7 @@ impl Service { } pub fn user_can_invite( - &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &mutex_map::Guard<()>, + &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &RoomMutexGuard, ) -> Result { let content = to_raw_value(&RoomMemberEventContent::new(MembershipState::Invite)) .expect("Event content always serializes"); diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index c82098ba..9bfc2715 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -6,7 +6,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, info, utils, utils::mutex_map, validated, warn, Error, Result}; +use conduit::{debug, error, info, utils, validated, warn, Error, Result}; use data::Data; use itertools::Itertools; use ruma::{ @@ -36,6 +36,7 @@ use tokio::sync::RwLock; use crate::{ admin, appservice::NamespaceRegex, + globals::RoomMutexGuard, pdu::{EventHash, PduBuilder}, rooms::{event_handler::parse_incoming_pdu, state_compressor::CompressedStateEvent}, server_is_ours, services, PduCount, PduEvent, @@ -203,7 +204,7 @@ impl Service { pdu: &PduEvent, mut pdu_json: CanonicalJsonObject, leaves: Vec, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result> { // Coalesce database writes for the remainder of this scope. let _cork = services().db.cork_and_flush(); @@ -593,7 +594,7 @@ impl Service { pdu_builder: PduBuilder, sender: &UserId, room_id: &RoomId, - _mutex_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<(PduEvent, CanonicalJsonObject)> { let PduBuilder { event_type, @@ -780,7 +781,7 @@ impl Service { pdu_builder: PduBuilder, sender: &UserId, room_id: &RoomId, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result> { let (pdu, pdu_json) = self.create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock)?; if let Some(admin_room) = admin::Service::get_admin_room()? { @@ -963,7 +964,7 @@ impl Service { new_room_leaves: Vec, state_ids_compressed: Arc>, soft_fail: bool, - state_lock: &mutex_map::Guard<()>, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result>> { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't From 271f720286280876eeb2be1ba8082dbc1d93274e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 21:10:14 +0000 Subject: [PATCH 0455/2091] move mutex maps out of globals into respective service Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 2 +- src/admin/federation/commands.rs | 5 +-- src/api/client/membership.rs | 26 ++++----------- src/api/client/message.rs | 6 +--- src/api/client/profile.rs | 2 +- src/api/client/redact.rs | 6 +--- src/api/client/room.rs | 14 ++------ src/api/client/state.rs | 2 +- src/api/client/sync.rs | 6 ++-- src/api/server/make_join.rs | 6 +--- src/api/server/make_leave.rs | 6 +--- src/api/server/send.rs | 5 +-- src/api/server/send_join.rs | 5 +-- src/api/server/send_leave.rs | 5 +-- src/service/admin/create.rs | 2 +- src/service/admin/grant.rs | 2 +- src/service/admin/mod.rs | 4 +-- src/service/globals/mod.rs | 20 ++---------- src/service/rooms/event_handler/mod.rs | 43 ++++++++++++++----------- src/service/rooms/state/data.rs | 2 +- src/service/rooms/state/mod.rs | 14 ++++++-- src/service/rooms/state_accessor/mod.rs | 2 +- src/service/rooms/timeline/mod.rs | 29 ++++++++++------- 23 files changed, 93 insertions(+), 121 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 53009566..62741f38 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -570,7 +570,7 @@ pub(super) async fn force_set_room_state_from_server( .state_compressor .save_state(room_id.clone().as_ref(), new_room_state)?; - let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; + let state_lock = services().rooms.state.mutex.lock(&room_id).await; services() .rooms .state diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 24f4bc23..a97e7582 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -16,8 +16,9 @@ pub(super) async fn enable_room(_body: Vec<&str>, room_id: Box) -> Resul pub(super) async fn incoming_federation(_body: Vec<&str>) -> Result { let map = services() - .globals - .roomid_federationhandletime + .rooms + .event_handler + .federation_handletime .read() .expect("locked"); let mut msg = format!("Handling {} incoming pdus:\n", map.len()); diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 2ac4c723..1b70fdec 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -40,8 +40,8 @@ use tokio::sync::RwLock; use crate::{ client::{update_avatar_url, update_displayname}, service::{ - globals::RoomMutexGuard, pdu::{gen_event_id_canonical_json, PduBuilder}, + rooms::state::RoomMutexGuard, sending::convert_to_outgoing_federation_event, server_is_ours, user_is_local, }, @@ -366,11 +366,7 @@ pub(crate) async fn invite_user_route( pub(crate) async fn kick_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let mut event: RoomMemberEventContent = serde_json::from_str( services() @@ -417,11 +413,7 @@ pub(crate) async fn kick_user_route(body: Ruma) -> Resul pub(crate) async fn ban_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let event = services() .rooms @@ -481,11 +473,7 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< pub(crate) async fn unban_user_route(body: Ruma) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let mut event: RoomMemberEventContent = serde_json::from_str( services() @@ -1399,7 +1387,7 @@ pub(crate) async fn invite_helper( if !user_is_local(user_id) { let (pdu, pdu_json, invite_room_state) = { - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; let content = to_raw_value(&RoomMemberEventContent { avatar_url: services().users.avatar_url(user_id)?, displayname: None, @@ -1511,7 +1499,7 @@ pub(crate) async fn invite_helper( )); } - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; services() .rooms @@ -1605,7 +1593,7 @@ pub async fn leave_room(user_id: &UserId, room_id: &RoomId, reason: Option, user_id: OwnedUserId) { for (pdu_builder, room_id) in all_joined_rooms { - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; if let Err(e) = services() .rooms .timeline diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index 4cb24c33..308d12e5 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -15,11 +15,7 @@ pub(crate) async fn redact_event_route(body: Ruma) -> let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let body = body.body; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let event_id = services() .rooms diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 7090fdc8..ccdf4dc0 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -90,7 +90,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R } let _short_id = services().rooms.short.get_or_create_shortroomid(&room_id)?; - let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; + let state_lock = services().rooms.state.mutex.lock(&room_id).await; let alias: Option = if let Some(alias) = &body.room_alias_name { Some(room_alias_check(alias, &body.appservice_info).await?) @@ -573,11 +573,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> .short .get_or_create_shortroomid(&replacement_room)?; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; // Send a m.room.tombstone event to the old room to indicate that it is not // intended to be used any further Fail if the sender does not have the required @@ -605,11 +601,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Change lock to replacement room drop(state_lock); - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&replacement_room) - .await; + let state_lock = services().rooms.state.mutex.lock(&replacement_room).await; // Get the old room creation event let mut create_event_content = serde_json::from_str::( diff --git a/src/api/client/state.rs b/src/api/client/state.rs index abff9218..25b77fe3 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -170,7 +170,7 @@ async fn send_state_event_for_key_helper( sender: &UserId, room_id: &RoomId, event_type: &StateEventType, json: &Raw, state_key: String, ) -> Result> { allowed_to_send_state_event(room_id, event_type, json).await?; - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; let event_id = services() .rooms .timeline diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 45a1c75b..dd1f3401 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -199,7 +199,7 @@ pub(crate) async fn sync_events_route( let (room_id, invite_state_events) = result?; // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services().globals.roomid_mutex_insert.lock(&room_id).await; + let insert_lock = services().rooms.timeline.mutex_insert.lock(&room_id).await; drop(insert_lock); let invite_count = services() @@ -317,7 +317,7 @@ async fn handle_left_room( next_batch_string: &str, full_state: bool, lazy_load_enabled: bool, ) -> Result<()> { // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services().globals.roomid_mutex_insert.lock(room_id).await; + let insert_lock = services().rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); let left_count = services() @@ -519,7 +519,7 @@ async fn load_joined_room( ) -> Result { // Get and drop the lock to wait for remaining operations to finish // This will make sure the we have all events until next_batch - let insert_lock = services().globals.roomid_mutex_insert.lock(room_id).await; + let insert_lock = services().rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); let (timeline_pdus, limited) = load_timeline(sender_user, room_id, sincecount, 10)?; diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index ca50dcbe..c909ea33 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -71,11 +71,7 @@ pub(crate) async fn create_join_event_template_route( let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let join_authorized_via_users_server = if (services() .rooms diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 62c09717..eea3e4f8 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -35,11 +35,7 @@ pub(crate) async fn create_leave_event_template_route( .acl_check(origin, &body.room_id)?; let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; - let state_lock = services() - .globals - .roomid_mutex_state - .lock(&body.room_id) - .await; + let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; let content = to_raw_value(&RoomMemberEventContent { avatar_url: None, blurhash: None, diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 90225a14..08caf1b4 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -127,8 +127,9 @@ async fn handle_pdus( for (event_id, value, room_id) in parsed_pdus { let pdu_start_time = Instant::now(); let mutex_lock = services() - .globals - .roomid_mutex_federation + .rooms + .event_handler + .mutex_federation .lock(&room_id) .await; resolved_map.insert( diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index ff362f64..577833d5 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -156,8 +156,9 @@ async fn create_join_event( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; let mutex_lock = services() - .globals - .roomid_mutex_federation + .rooms + .event_handler + .mutex_federation .lock(room_id) .await; let pdu_id: Vec = services() diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index 4fdde515..c4e17bbc 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -152,8 +152,9 @@ async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJson .await?; let mutex_lock = services() - .globals - .roomid_mutex_federation + .rooms + .event_handler + .mutex_federation .lock(room_id) .await; let pdu_id: Vec = services() diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index ad70fe0c..57df344a 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -34,7 +34,7 @@ pub async fn create_admin_room() -> Result<()> { let _short_id = services().rooms.short.get_or_create_shortroomid(&room_id)?; - let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; + let state_lock = services().rooms.state.mutex.lock(&room_id).await; // Create a user for the server let server_user = &services().globals.server_user; diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index ca48ce0d..9a4ef242 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -22,7 +22,7 @@ use crate::{pdu::PduBuilder, services}; /// In conduit, this is equivalent to granting admin privileges. pub async fn make_user_admin(user_id: &UserId, displayname: String) -> Result<()> { if let Some(room_id) = Service::get_admin_room()? { - let state_lock = services().globals.roomid_mutex_state.lock(&room_id).await; + let state_lock = services().rooms.state.mutex.lock(&room_id).await; // Use the server user to grant the new admin's power level let server_user = &services().globals.server_user; diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index e0dd1760..5c8a98bb 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -26,7 +26,7 @@ use tokio::{ task::JoinHandle, }; -use crate::{globals::RoomMutexGuard, pdu::PduBuilder, services, user_is_local, PduEvent}; +use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, user_is_local, PduEvent}; const COMMAND_QUEUE_LIMIT: usize = 512; @@ -248,7 +248,7 @@ async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, use "sender is not admin" ); - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; let response_pdu = PduBuilder { event_type: TimelineEventType::RoomMessage, content: to_raw_value(&content).expect("event is valid, we just created it"), diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index a5b70835..16830d87 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -12,19 +12,15 @@ use std::{ time::Instant, }; -use conduit::{ - error, trace, - utils::{MutexMap, MutexMapGuard}, - Config, Result, -}; +use conduit::{error, trace, Config, Result}; use data::Data; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ api::{client::discovery::discover_support::ContactRole, federation::discovery::VerifyKey}, serde::Base64, - DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, - RoomAliasId, RoomVersionId, ServerName, UserId, + DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, + RoomVersionId, ServerName, UserId, }; use tokio::{sync::Mutex, task::JoinHandle}; use url::Url; @@ -45,18 +41,12 @@ pub struct Service { pub bad_event_ratelimiter: Arc>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub bad_query_ratelimiter: Arc>>, - pub roomid_mutex_insert: RoomMutexMap, - pub roomid_mutex_state: RoomMutexMap, - pub roomid_mutex_federation: RoomMutexMap, - pub roomid_federationhandletime: RwLock>, pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, } -pub type RoomMutexMap = MutexMap; -pub type RoomMutexGuard = MutexMapGuard; type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries impl crate::Service for Service { @@ -113,10 +103,6 @@ impl crate::Service for Service { bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), - roomid_mutex_state: MutexMap::::new(), - roomid_mutex_insert: MutexMap::::new(), - roomid_mutex_federation: MutexMap::::new(), - roomid_federationhandletime: RwLock::new(HashMap::new()), updates_handle: Mutex::new(None), stateres_mutex: Arc::new(Mutex::new(())), admin_alias: RoomAliasId::parse(format!("#admins:{}", &config.server_name)) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 53ce6f8d..bf553d5c 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -4,13 +4,14 @@ mod signing_keys; use std::{ collections::{hash_map, BTreeMap, HashMap, HashSet}, pin::Pin, - sync::Arc, + sync::{Arc, RwLock as StdRwLock}, time::Instant, }; use conduit::{ - debug, debug_error, debug_info, error, info, trace, utils::math::continue_exponential_backoff_secs, warn, Error, - Result, + debug, debug_error, debug_info, error, info, trace, + utils::{math::continue_exponential_backoff_secs, MutexMap}, + warn, Error, Result, }; use futures_util::Future; pub use parse_incoming_pdu::parse_incoming_pdu; @@ -28,14 +29,21 @@ use ruma::{ int, serde::Base64, state_res::{self, RoomVersion, StateMap}, - uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedUserId, RoomId, RoomVersionId, ServerName, + uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, + RoomVersionId, ServerName, }; use tokio::sync::RwLock; use super::state_compressor::CompressedStateEvent; use crate::{pdu, services, PduEvent}; -pub struct Service; +pub struct Service { + pub federation_handletime: StdRwLock, + pub mutex_federation: RoomMutexMap, +} + +type RoomMutexMap = MutexMap; +type HandleTimeMap = HashMap; // We use some AsyncRecursiveType hacks here so we can call async funtion // recursively. @@ -46,7 +54,12 @@ type AsyncRecursiveCanonicalJsonResult<'a> = AsyncRecursiveType<'a, Result<(Arc, BTreeMap)>>; impl crate::Service for Service { - fn build(_args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self {})) } + fn build(_args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + federation_handletime: HandleTimeMap::new().into(), + mutex_federation: RoomMutexMap::new(), + })) + } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } @@ -200,9 +213,7 @@ impl Service { // Done with prev events, now handling the incoming event let start_time = Instant::now(); - services() - .globals - .roomid_federationhandletime + self.federation_handletime .write() .expect("locked") .insert(room_id.to_owned(), (event_id.to_owned(), start_time)); @@ -211,9 +222,7 @@ impl Service { .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, &create_event, origin, room_id, pub_key_map) .await; - services() - .globals - .roomid_federationhandletime + self.federation_handletime .write() .expect("locked") .remove(&room_id.to_owned()); @@ -272,9 +281,7 @@ impl Service { } let start_time = Instant::now(); - services() - .globals - .roomid_federationhandletime + self.federation_handletime .write() .expect("locked") .insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time)); @@ -282,9 +289,7 @@ impl Service { self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id, pub_key_map) .await?; - services() - .globals - .roomid_federationhandletime + self.federation_handletime .write() .expect("locked") .remove(&room_id.to_owned()); @@ -579,7 +584,7 @@ impl Service { // We start looking at current room state now, so lets lock the room trace!("Locking the room"); - let state_lock = services().globals.roomid_mutex_state.lock(room_id).await; + let state_lock = services().rooms.state.mutex.lock(room_id).await; // Now we calculate the set of extremities this room has after the incoming // event has been applied. We start with the previous extremities (aka leaves) diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index b62adf60..3c110afc 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -4,7 +4,7 @@ use conduit::{utils, Error, Result}; use database::{Database, Map}; use ruma::{EventId, OwnedEventId, RoomId}; -use crate::globals::RoomMutexGuard; +use super::RoomMutexGuard; pub(super) struct Data { shorteventid_shortstatehash: Arc, diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index b46a9d04..7d89ee33 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -5,7 +5,10 @@ use std::{ sync::Arc, }; -use conduit::{utils::calculate_hash, warn, Error, Result}; +use conduit::{ + utils::{calculate_hash, MutexMap, MutexMapGuard}, + warn, Error, Result, +}; use data::Data; use ruma::{ api::client::error::ErrorKind, @@ -15,20 +18,25 @@ use ruma::{ }, serde::Raw, state_res::{self, StateMap}, - EventId, OwnedEventId, RoomId, RoomVersionId, UserId, + EventId, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, UserId, }; use super::state_compressor::CompressedStateEvent; -use crate::{globals::RoomMutexGuard, services, PduEvent}; +use crate::{services, PduEvent}; pub struct Service { db: Data, + pub mutex: RoomMutexMap, } +type RoomMutexMap = MutexMap; +pub type RoomMutexGuard = MutexMapGuard; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { db: Data::new(args.db), + mutex: RoomMutexMap::new(), })) } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 35719c15..389f12c3 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -33,7 +33,7 @@ use ruma::{ }; use serde_json::value::to_raw_value; -use crate::{globals::RoomMutexGuard, pdu::PduBuilder, services, PduEvent}; +use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, PduEvent}; pub struct Service { db: Data, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 9bfc2715..df2d46bd 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -6,7 +6,11 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, info, utils, validated, warn, Error, Result}; +use conduit::{ + debug, error, info, utils, + utils::{MutexMap, MutexMapGuard}, + validated, warn, Error, Result, +}; use data::Data; use itertools::Itertools; use ruma::{ @@ -26,8 +30,8 @@ use ruma::{ push::{Action, Ruleset, Tweak}, serde::Base64, state_res::{self, Event, RoomVersion}, - uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedServerName, RoomId, - RoomVersionId, ServerName, UserId, + uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, + RoomId, RoomVersionId, ServerName, UserId, }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; @@ -36,7 +40,6 @@ use tokio::sync::RwLock; use crate::{ admin, appservice::NamespaceRegex, - globals::RoomMutexGuard, pdu::{EventHash, PduBuilder}, rooms::{event_handler::parse_incoming_pdu, state_compressor::CompressedStateEvent}, server_is_ours, services, PduCount, PduEvent, @@ -66,12 +69,17 @@ struct ExtractBody { pub struct Service { db: Data, + pub mutex_insert: RoomMutexMap, } +type RoomMutexMap = MutexMap; +pub type RoomMutexGuard = MutexMapGuard; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { db: Data::new(args.db), + mutex_insert: RoomMutexMap::new(), })) } @@ -269,11 +277,7 @@ impl Service { .state .set_forward_extremities(&pdu.room_id, leaves, state_lock)?; - let insert_lock = services() - .globals - .roomid_mutex_insert - .lock(&pdu.room_id) - .await; + let insert_lock = self.mutex_insert.lock(&pdu.room_id).await; let count1 = services().globals.next_count()?; // Mark as read first so the sending client doesn't get a notification even if @@ -1154,8 +1158,9 @@ impl Service { // Lock so we cannot backfill the same pdu twice at the same time let mutex_lock = services() - .globals - .roomid_mutex_federation + .rooms + .event_handler + .mutex_federation .lock(&room_id) .await; @@ -1187,7 +1192,7 @@ impl Service { .get_shortroomid(&room_id)? .expect("room exists"); - let insert_lock = services().globals.roomid_mutex_insert.lock(&room_id).await; + let insert_lock = self.mutex_insert.lock(&room_id).await; let max = u64::MAX; let count = services().globals.next_count()?; From 2a304c2b6c4f0b29afe1786e299f7cb1d1836f86 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 21:22:44 +0000 Subject: [PATCH 0456/2091] add some usage stats output on the mutex maps Signed-off-by: Jason Volk --- src/service/rooms/event_handler/mod.rs | 15 +++++++++++++++ src/service/rooms/state/mod.rs | 8 ++++++++ src/service/rooms/timeline/mod.rs | 3 +++ 3 files changed, 26 insertions(+) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index bf553d5c..9490a404 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -3,6 +3,7 @@ mod signing_keys; use std::{ collections::{hash_map, BTreeMap, HashMap, HashSet}, + fmt::Write, pin::Pin, sync::{Arc, RwLock as StdRwLock}, time::Instant, @@ -61,6 +62,20 @@ impl crate::Service for Service { })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let mutex_federation = self.mutex_federation.len(); + writeln!(out, "federation_mutex: {mutex_federation}")?; + + let federation_handletime = self + .federation_handletime + .read() + .expect("locked for reading") + .len(); + writeln!(out, "federation_handletime: {federation_handletime}")?; + + Ok(()) + } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 7d89ee33..a3a317a5 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -2,6 +2,7 @@ mod data; use std::{ collections::{HashMap, HashSet}, + fmt::Write, sync::Arc, }; @@ -40,6 +41,13 @@ impl crate::Service for Service { })) } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let mutex = self.mutex.len(); + writeln!(out, "state_mutex: {mutex}")?; + + Ok(()) + } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index df2d46bd..e255e978 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -92,6 +92,9 @@ impl crate::Service for Service { .len(); writeln!(out, "lasttimelinecount_cache: {lasttimelinecount_cache}")?; + let mutex_insert = self.mutex_insert.len(); + writeln!(out, "insert_mutex: {mutex_insert}")?; + Ok(()) } From 16e76d45cb22d9c96d80a57a5ed5af810cc64946 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 21:55:56 +0000 Subject: [PATCH 0457/2091] improve alloc stats interface; fix admin command formatting Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 4 ++-- src/admin/server/commands.rs | 13 ++++--------- src/core/alloc/default.rs | 8 ++++---- src/core/alloc/hardened.rs | 9 +++++---- src/core/alloc/je.rs | 10 +++++----- 5 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 62741f38..5fd0360d 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -632,12 +632,12 @@ pub(super) async fn resolve_true_destination( pub(super) fn memory_stats() -> RoomMessageEventContent { let html_body = conduit::alloc::memory_stats(); - if html_body.is_empty() { + if html_body.is_none() { return RoomMessageEventContent::text_plain("malloc stats are not supported on your compiled malloc."); } RoomMessageEventContent::text_html( "This command's output can only be viewed by clients that render HTML.".to_owned(), - html_body, + html_body.expect("string result"), ) } diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 06007fb8..229f05e1 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -20,17 +20,12 @@ pub(super) async fn show_config(_body: Vec<&str>) -> Result) -> Result { - let response0 = services().memory_usage().await?; - let response1 = services().db.db.memory_usage()?; - let response2 = conduit::alloc::memory_usage(); + let services_usage = services().memory_usage().await?; + let database_usage = services().db.db.memory_usage()?; + let allocator_usage = conduit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}")); Ok(RoomMessageEventContent::text_plain(format!( - "Services:\n{response0}\nDatabase:\n{response1}\n{}", - if !response2.is_empty() { - format!("Allocator:\n {response2}") - } else { - String::new() - } + "Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}", ))) } diff --git a/src/core/alloc/default.rs b/src/core/alloc/default.rs index 4e2f8d7e..83bfca7d 100644 --- a/src/core/alloc/default.rs +++ b/src/core/alloc/default.rs @@ -1,9 +1,9 @@ //! Default allocator with no special features -/// Always returns the empty string +/// Always returns None #[must_use] -pub fn memory_stats() -> String { String::default() } +pub fn memory_stats() -> Option { None } -/// Always returns the empty string +/// Always returns None #[must_use] -pub fn memory_usage() -> String { String::default() } +pub fn memory_usage() -> Option { None } diff --git a/src/core/alloc/hardened.rs b/src/core/alloc/hardened.rs index 6727407f..335a3307 100644 --- a/src/core/alloc/hardened.rs +++ b/src/core/alloc/hardened.rs @@ -4,9 +4,10 @@ static HMALLOC: hardened_malloc_rs::HardenedMalloc = hardened_malloc_rs::HardenedMalloc; #[must_use] -pub fn memory_usage() -> String { - String::default() //TODO: get usage -} +//TODO: get usage +pub fn memory_usage() -> Option { None } #[must_use] -pub fn memory_stats() -> String { "Extended statistics are not available from hardened_malloc.".to_owned() } +pub fn memory_stats() -> Option { + Some("Extended statistics are not available from hardened_malloc.".to_owned()) +} diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 5e3c361f..08bfc49a 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -10,7 +10,7 @@ use tikv_jemallocator as jemalloc; static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc; #[must_use] -pub fn memory_usage() -> String { +pub fn memory_usage() -> Option { use mallctl::stats; let mibs = |input: Result| { @@ -27,14 +27,14 @@ pub fn memory_usage() -> String { let metadata = mibs(stats::metadata::read()); let resident = mibs(stats::resident::read()); let retained = mibs(stats::retained::read()); - format!( + Some(format!( "allocated: {allocated:.2} MiB\nactive: {active:.2} MiB\nmapped: {mapped:.2} MiB\nmetadata: {metadata:.2} \ MiB\nresident: {resident:.2} MiB\nretained: {retained:.2} MiB\n" - ) + )) } #[must_use] -pub fn memory_stats() -> String { +pub fn memory_stats() -> Option { const MAX_LENGTH: usize = 65536 - 4096; let opts_s = "d"; @@ -51,7 +51,7 @@ pub fn memory_stats() -> String { unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts_p) }; str.truncate(MAX_LENGTH); - format!("
    {str}
    ") + Some(format!("
    {str}
    ")) } extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { From f10f5319db5b0189901008487f0b9b5b4b375e58 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 06:34:03 +0000 Subject: [PATCH 0458/2091] elaborate error log functor stack Signed-off-by: Jason Volk --- src/core/error.rs | 70 ++++++++++++++++++++++++++++++------ src/service/admin/console.rs | 5 ++- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/src/core/error.rs b/src/core/error.rs index 1959081a..7069d1e6 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -117,25 +117,73 @@ impl Error { } #[inline] -pub fn log(e: &Error) { - error!(?e); +pub fn else_log(error: E) -> Result +where + T: Default, + Error: From, +{ + Ok(default_log(error)) } #[inline] -pub fn debug_log(e: &Error) { - debug_error!(?e); +pub fn else_debug_log(error: E) -> Result +where + T: Default, + Error: From, +{ + Ok(default_debug_log(error)) } #[inline] -pub fn into_log(e: Error) { - error!(?e); - drop(e); +pub fn default_log(error: E) -> T +where + T: Default, + Error: From, +{ + let error = Error::from(error); + inspect_log(&error); + T::default() } #[inline] -pub fn into_debug_log(e: Error) { - debug_error!(?e); - drop(e); +pub fn default_debug_log(error: E) -> T +where + T: Default, + Error: From, +{ + let error = Error::from(error); + inspect_debug_log(&error); + T::default() +} + +#[inline] +pub fn map_log(error: E) -> Error +where + Error: From, +{ + let error = Error::from(error); + inspect_log(&error); + error +} + +#[inline] +pub fn map_debug_log(error: E) -> Error +where + Error: From, +{ + let error = Error::from(error); + inspect_debug_log(&error); + error +} + +#[inline] +pub fn inspect_log(error: &E) { + error!("{error}"); +} + +#[inline] +pub fn inspect_debug_log(error: &E) { + debug_error!("{error:?}"); } impl fmt::Debug for Error { @@ -151,7 +199,7 @@ impl axum::response::IntoResponse for Error { let response: UiaaResponse = self.into(); response .try_into_http_response::() - .inspect_err(|e| error!(?e)) + .inspect_err(|e| error!("error response error: {e}")) .map_or_else( |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), |r| r.map(BytesMut::freeze).map(Full::new).into_response(), diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 2f66b1d5..c590b928 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -95,7 +95,10 @@ impl Console { ReadlineEvent::Line(string) => self.clone().handle(string).await, ReadlineEvent::Interrupted => continue, ReadlineEvent::Eof => break, - ReadlineEvent::Quit => services().server.shutdown().unwrap_or_else(error::into_log), + ReadlineEvent::Quit => services() + .server + .shutdown() + .unwrap_or_else(error::default_log), }, Err(error) => match error { ReadlineError::Closed => break, From 0627b46f4018e5a589bc106ebae24cb60cc91ae2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 06:35:11 +0000 Subject: [PATCH 0459/2091] add panic suite to Error Signed-off-by: Jason Volk --- src/api/server/send.rs | 2 +- src/core/debug.rs | 7 +++-- src/core/error.rs | 71 ++++++++++++++++++++++++++++++++---------- 3 files changed, 60 insertions(+), 20 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 08caf1b4..122f564f 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -84,7 +84,7 @@ pub(crate) async fn send_transaction_message_route( Ok(send_transaction_message::v1::Response { pdus: resolved_map .into_iter() - .map(|(e, r)| (e, r.map_err(|e| e.sanitized_error()))) + .map(|(e, r)| (e, r.map_err(|e| e.sanitized_string()))) .collect(), }) } diff --git a/src/core/debug.rs b/src/core/debug.rs index 1f855e52..14d0be87 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -1,6 +1,4 @@ -#![allow(dead_code)] // this is a developer's toolbox - -use std::panic; +use std::{any::Any, panic}; /// Export all of the ancillary tools from here as well. pub use crate::utils::debug::*; @@ -79,3 +77,6 @@ pub fn trap() { std::arch::asm!("int3"); } } + +#[must_use] +pub fn panic_str(p: &Box) -> &'static str { p.downcast_ref::<&str>().copied().unwrap_or_default() } diff --git a/src/core/error.rs b/src/core/error.rs index 7069d1e6..26bac6e0 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -1,4 +1,9 @@ -use std::{convert::Infallible, fmt}; +use std::{ + any::Any, + convert::Infallible, + fmt, + panic::{RefUnwindSafe, UnwindSafe}, +}; use bytes::BytesMut; use http::StatusCode; @@ -8,10 +13,15 @@ use ruma::{ OwnedServerName, }; -use crate::{debug_error, error}; +use crate::{debug::panic_str, debug_error, error}; #[derive(thiserror::Error)] pub enum Error { + #[error("PANIC!")] + PanicAny(Box), + #[error("PANIC! {0}")] + Panic(&'static str, Box), + // std #[error("{0}")] Fmt(#[from] fmt::Error), @@ -31,6 +41,8 @@ pub enum Error { ParseFloatError(#[from] std::num::ParseFloatError), // third-party + #[error("Join error: {0}")] + JoinError(#[from] tokio::task::JoinError), #[error("Regex error: {0}")] Regex(#[from] regex::Error), #[error("Tracing filter error: {0}")] @@ -94,6 +106,29 @@ impl Error { Self::BadConfig(message.to_owned()) } + #[must_use] + pub fn from_panic(e: Box) -> Self { Self::Panic(panic_str(&e), e) } + + pub fn into_panic(self) -> Box { + match self { + Self::Panic(_, e) | Self::PanicAny(e) => e, + Self::JoinError(e) => e.into_panic(), + _ => std::panic::panic_any(self), + } + } + + /// Sanitizes public-facing errors that can leak sensitive information. + pub fn sanitized_string(&self) -> String { + match self { + Self::Database(..) => String::from("Database error occurred."), + Self::Io(..) => String::from("I/O error occurred."), + _ => self.to_string(), + } + } + + /// Get the panic message string. + pub fn panic_str(self) -> Option<&'static str> { self.is_panic().then_some(panic_str(&self.into_panic())) } + /// Returns the Matrix error code / error kind #[inline] pub fn error_code(&self) -> ruma::api::client::error::ErrorKind { @@ -106,16 +141,28 @@ impl Error { } } - /// Sanitizes public-facing errors that can leak sensitive information. - pub fn sanitized_error(&self) -> String { - match self { - Self::Database(..) => String::from("Database error occurred."), - Self::Io(..) => String::from("I/O error occurred."), - _ => self.to_string(), + /// Check if the Error is trafficking a panic object. + #[inline] + pub fn is_panic(&self) -> bool { + match &self { + Self::Panic(..) | Self::PanicAny(..) => true, + Self::JoinError(e) => e.is_panic(), + _ => false, } } } +impl UnwindSafe for Error {} +impl RefUnwindSafe for Error {} + +impl From for Error { + fn from(i: Infallible) -> Self { match i {} } +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } +} + #[inline] pub fn else_log(error: E) -> Result where @@ -186,14 +233,6 @@ pub fn inspect_debug_log(error: &E) { debug_error!("{error:?}"); } -impl fmt::Debug for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } -} - -impl From for Error { - fn from(i: Infallible) -> Self { match i {} } -} - impl axum::response::IntoResponse for Error { fn into_response(self) -> axum::response::Response { let response: UiaaResponse = self.into(); From 61f2a3c68b433be1a8acb96f79f31cc746a9b311 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 10:43:41 +0000 Subject: [PATCH 0460/2091] catch panic from admin commands Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/admin/Cargo.toml | 1 + src/admin/handler.rs | 46 ++++++++++++++++++++++++++++++++------------ 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1846c82d..02da0b6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -613,6 +613,7 @@ dependencies = [ "conduit_api", "conduit_core", "conduit_service", + "futures-util", "log", "ruma", "serde_json", diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index f84fbb92..7a092237 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -30,6 +30,7 @@ clap.workspace = true conduit-api.workspace = true conduit-core.workspace = true conduit-service.workspace = true +futures-util.workspace = true log.workspace = true ruma.workspace = true serde_json.workspace = true diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 22ec81c5..2d1f96b5 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,10 +1,14 @@ -use std::time::Instant; +use std::{panic::AssertUnwindSafe, time::Instant}; use clap::{CommandFactory, Parser}; -use conduit::trace; -use ruma::events::{ - relation::InReplyTo, - room::message::{Relation::Reply, RoomMessageEventContent}, +use conduit::{error, trace, Error}; +use futures_util::future::FutureExt; +use ruma::{ + events::{ + relation::InReplyTo, + room::message::{Relation::Reply, RoomMessageEventContent}, + }, + OwnedEventId, }; extern crate conduit_service as service; @@ -69,21 +73,39 @@ pub(crate) fn complete(line: &str) -> String { complete_admin_command(AdminComma #[tracing::instrument(skip_all, name = "admin")] async fn handle_command(command: Command) -> CommandResult { - let Some(mut content) = process_admin_message(command.command).await else { - return Ok(None); - }; + AssertUnwindSafe(process_command(&command)) + .catch_unwind() + .await + .map_err(Error::from_panic) + .or_else(|error| handle_panic(&error, command)) +} - content.relates_to = command.reply_id.map(|event_id| Reply { +async fn process_command(command: &Command) -> CommandOutput { + process_admin_message(&command.command) + .await + .and_then(|content| reply(content, command.reply_id.clone())) +} + +fn handle_panic(error: &Error, command: Command) -> CommandResult { + let link = "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; + let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}"); + let content = RoomMessageEventContent::notice_markdown(msg); + error!("Panic while processing command: {error:?}"); + Ok(reply(content, command.reply_id)) +} + +fn reply(mut content: RoomMessageEventContent, reply_id: Option) -> Option { + content.relates_to = reply_id.map(|event_id| Reply { in_reply_to: InReplyTo { event_id, }, }); - Ok(Some(content)) + Some(content) } // Parse and process a message from the admin room -async fn process_admin_message(msg: String) -> CommandOutput { +async fn process_admin_message(msg: &str) -> CommandOutput { let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); let command = lines.next().expect("each string has at least one line"); let body = lines.collect::>(); @@ -103,7 +125,7 @@ async fn process_admin_message(msg: String) -> CommandOutput { match result { Ok(reply) => Some(reply), Err(error) => Some(RoomMessageEventContent::notice_markdown(format!( - "Encountered an error while handling the command:\n```\n{error}\n```" + "Encountered an error while handling the command:\n```\n{error:#?}\n```" ))), } } From e73aa2aa2146ebd99f56dea9ad4c5f5d51bf9fa6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 02:29:06 +0000 Subject: [PATCH 0461/2091] log propagated errors in admin service; minor cleanup Signed-off-by: Jason Volk --- src/service/admin/mod.rs | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 5c8a98bb..e9729d2d 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,7 +9,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{error, Error, Result}; +use conduit::{debug, error, Error, Result}; pub use create::create_admin_room; pub use grant::make_user_admin; use loole::{Receiver, Sender}; @@ -28,14 +28,6 @@ use tokio::{ use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, user_is_local, PduEvent}; -const COMMAND_QUEUE_LIMIT: usize = 512; - -pub type CommandOutput = Option; -pub type CommandResult = Result; -pub type HandlerResult = Pin + Send>>; -pub type Handler = fn(Command) -> HandlerResult; -pub type Completer = fn(&str) -> String; - pub struct Service { sender: Sender, receiver: Mutex>, @@ -52,6 +44,14 @@ pub struct Command { pub reply_id: Option, } +pub type Completer = fn(&str) -> String; +pub type Handler = fn(Command) -> HandlerResult; +pub type HandlerResult = Pin + Send>>; +pub type CommandResult = Result; +pub type CommandOutput = Option; + +const COMMAND_QUEUE_LIMIT: usize = 512; + #[async_trait] impl crate::Service for Service { fn build(_args: crate::Args<'_>) -> Result> { @@ -172,8 +172,10 @@ impl Service { } async fn handle_command(&self, command: Command) { - if let Ok(Some(output)) = self.process_command(command).await { - handle_response(output).await; + match self.process_command(command).await { + Ok(Some(output)) => handle_response(output).await, + Ok(None) => debug!("Command successful with no response"), + Err(e) => error!("Command processing error: {e}"), } } @@ -263,14 +265,14 @@ async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, use .build_and_append_pdu(response_pdu, user_id, room_id, &state_lock) .await { - if let Err(e) = handle_response_error(&e, room_id, user_id, &state_lock).await { + if let Err(e) = handle_response_error(e, room_id, user_id, &state_lock).await { error!("{e}"); } } } async fn handle_response_error( - e: &Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, + e: Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, ) -> Result<()> { error!("Failed to build and append admin room response PDU: \"{e}\""); let error_room_message = RoomMessageEventContent::text_plain(format!( From db3c718ddc224a15b0dcbc208413c8a8a8242766 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 20:27:17 +0000 Subject: [PATCH 0462/2091] add config for sentry stacktraces Signed-off-by: Jason Volk --- conduwuit-example.toml | 3 +++ src/core/config/mod.rs | 3 +++ src/main/sentry.rs | 1 + 3 files changed, 7 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 7e5c9710..cb6dc8e6 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -57,6 +57,9 @@ # Defaults to 0.15 #sentry_traces_sample_rate = 0.15 +# Whether to attach a stacktrace to Sentry reports. +#sentry_attach_stacktrace = false + ### Database configuration diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index f9a5ec4c..8b864c84 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -356,6 +356,8 @@ pub struct Config { pub sentry_send_server_name: bool, #[serde(default = "default_sentry_traces_sample_rate")] pub sentry_traces_sample_rate: f32, + #[serde(default)] + pub sentry_attach_stacktrace: bool, #[serde(default)] pub tokio_console: bool, @@ -819,6 +821,7 @@ impl fmt::Display for Config { ("Sentry.io send server_name in logs", &self.sentry_send_server_name.to_string()), #[cfg(feature = "sentry_telemetry")] ("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()), + ("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()), ( "Well-known server name", self.well_known diff --git a/src/main/sentry.rs b/src/main/sentry.rs index 6ed4bb8a..59f88a0c 100644 --- a/src/main/sentry.rs +++ b/src/main/sentry.rs @@ -28,6 +28,7 @@ fn options(config: &Config) -> ClientOptions { debug: cfg!(debug_assertions), release: sentry::release_name!(), user_agent: conduit::version::user_agent().into(), + attach_stacktrace: config.sentry_attach_stacktrace, before_send: Some(Arc::new(before_send)), before_breadcrumb: Some(Arc::new(before_breadcrumb)), ..Default::default() From 0023b09f5b644cf51597cf010f643e37af3d98bb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 10 Jul 2024 22:28:56 +0000 Subject: [PATCH 0463/2091] make tracing reload handles into a named map Signed-off-by: Jason Volk --- src/core/error.rs | 2 ++ src/core/log/reload.rs | 50 ++++++++++++++++++++++++++---------------- src/main/tracing.rs | 13 ++++++----- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/core/error.rs b/src/core/error.rs index 26bac6e0..8567f303 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -47,6 +47,8 @@ pub enum Error { Regex(#[from] regex::Error), #[error("Tracing filter error: {0}")] TracingFilter(#[from] tracing_subscriber::filter::ParseError), + #[error("Tracing reload error: {0}")] + TracingReload(#[from] tracing_subscriber::reload::Error), #[error("Image error: {0}")] Image(#[from] image::error::ImageError), #[error("Request error: {0}")] diff --git a/src/core/log/reload.rs b/src/core/log/reload.rs index 7646254e..cfed785d 100644 --- a/src/core/log/reload.rs +++ b/src/core/log/reload.rs @@ -1,7 +1,12 @@ -use std::sync::Arc; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; use tracing_subscriber::{reload, EnvFilter}; +use crate::{error, Result}; + /// We need to store a reload::Handle value, but can't name it's type explicitly /// because the S type parameter depends on the subscriber's previous layers. In /// our case, this includes unnameable 'impl Trait' types. @@ -24,32 +29,39 @@ impl ReloadHandle for reload::Handle { fn reload(&self, new_value: L) -> Result<(), reload::Error> { Self::reload(self, new_value) } } -struct LogLevelReloadHandlesInner { - handles: Vec + Send + Sync>>, -} - -/// Wrapper to allow reloading the filter on several several -/// [`tracing_subscriber::reload::Handle`]s at once, with the same value. #[derive(Clone)] pub struct LogLevelReloadHandles { - inner: Arc, + handles: Arc>, } +type HandleMap = HashMap; +type Handle = Box + Send + Sync>; + impl LogLevelReloadHandles { - #[must_use] - pub fn new(handles: Vec + Send + Sync>>) -> Self { - Self { - inner: Arc::new(LogLevelReloadHandlesInner { - handles, - }), - } + pub fn add(&self, name: &str, handle: Handle) { + self.handles + .lock() + .expect("locked") + .insert(name.into(), handle); } - pub fn reload(&self, new_value: &EnvFilter) -> Result<(), reload::Error> { - for handle in &self.inner.handles { - handle.reload(new_value.clone())?; - } + pub fn reload(&self, new_value: &EnvFilter) -> Result<()> { + self.handles + .lock() + .expect("locked") + .values() + .for_each(|handle| { + _ = handle.reload(new_value.clone()).or_else(error::else_log); + }); Ok(()) } } + +impl Default for LogLevelReloadHandles { + fn default() -> Self { + Self { + handles: Arc::new(HandleMap::new().into()), + } + } +} diff --git a/src/main/tracing.rs b/src/main/tracing.rs index bbfe4dc4..4c8fde5a 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -4,7 +4,7 @@ use conduit::{ config, config::Config, debug_warn, - log::{capture, LogLevelReloadHandles, ReloadHandle}, + log::{capture, LogLevelReloadHandles}, }; use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; @@ -15,6 +15,8 @@ pub(crate) type TracingFlameGuard = (); #[allow(clippy::redundant_clone)] pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard, Arc) { + let reload_handles = LogLevelReloadHandles::default(); + let fmt_layer = tracing_subscriber::fmt::Layer::new(); let filter_layer = match EnvFilter::try_new(&config.log) { Ok(s) => s, @@ -24,9 +26,8 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard }, }; - let mut reload_handles = Vec:: + Send + Sync>>::new(); let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.push(Box::new(fmt_reload_handle)); + reload_handles.add("format", Box::new(fmt_reload_handle)); let subscriber = Registry::default().with(fmt_layer.with_filter(fmt_reload_filter)); @@ -38,7 +39,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard let subscriber = { let sentry_layer = sentry_tracing::layer(); let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.push(Box::new(sentry_reload_handle)); + reload_handles.add("sentry", Box::new(sentry_reload_handle)); subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) }; @@ -73,7 +74,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.push(Box::new(jaeger_reload_handle)); + reload_handles.add("jaeger", Box::new(jaeger_reload_handle)); Some(telemetry.with_filter(jaeger_reload_filter)) } else { None @@ -87,7 +88,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard #[cfg_attr(not(feature = "perf_measurements"), allow(clippy::let_unit_value))] let flame_guard = (); - let ret = (LogLevelReloadHandles::new(reload_handles), flame_guard, cap_state); + let ret = (reload_handles, flame_guard, cap_state); // Enable the tokio console. This is slightly kludgy because we're judggling // compile-time and runtime conditions to elide it, each of those changing the From 160f48043e55cc579d6bb77eb12924a11e134e65 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 00:42:49 +0000 Subject: [PATCH 0464/2091] add selective log handle reload Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 16 ++++++++++++++-- src/core/log/reload.rs | 7 ++++--- src/core/log/suppress.rs | 4 ++-- src/main/tracing.rs | 2 +- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 5fd0360d..a5087467 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -314,6 +314,8 @@ pub(super) async fn force_device_list_updates(_body: Vec<&str>) -> Result, filter: Option, reset: bool, ) -> Result { + let handles = &["console"]; + if reset { let old_filter_layer = match EnvFilter::try_new(&services().globals.config.log) { Ok(s) => s, @@ -324,7 +326,12 @@ pub(super) async fn change_log_level( }, }; - match services().server.log.reload.reload(&old_filter_layer) { + match services() + .server + .log + .reload + .reload(&old_filter_layer, Some(handles)) + { Ok(()) => { return Ok(RoomMessageEventContent::text_plain(format!( "Successfully changed log level back to config value {}", @@ -349,7 +356,12 @@ pub(super) async fn change_log_level( }, }; - match services().server.log.reload.reload(&new_filter_layer) { + match services() + .server + .log + .reload + .reload(&new_filter_layer, Some(handles)) + { Ok(()) => { return Ok(RoomMessageEventContent::text_plain("Successfully changed log level")); }, diff --git a/src/core/log/reload.rs b/src/core/log/reload.rs index cfed785d..0b72d787 100644 --- a/src/core/log/reload.rs +++ b/src/core/log/reload.rs @@ -45,12 +45,13 @@ impl LogLevelReloadHandles { .insert(name.into(), handle); } - pub fn reload(&self, new_value: &EnvFilter) -> Result<()> { + pub fn reload(&self, new_value: &EnvFilter, names: Option<&[&str]>) -> Result<()> { self.handles .lock() .expect("locked") - .values() - .for_each(|handle| { + .iter() + .filter(|(name, _)| names.map_or(false, |names| names.contains(&name.as_str()))) + .for_each(|(_, handle)| { _ = handle.reload(new_value.clone()).or_else(error::else_log); }); diff --git a/src/core/log/suppress.rs b/src/core/log/suppress.rs index 6e883086..ca2d5485 100644 --- a/src/core/log/suppress.rs +++ b/src/core/log/suppress.rs @@ -18,7 +18,7 @@ impl Suppress { server .log .reload - .reload(suppress) + .reload(suppress, Some(&["console"])) .expect("log filter reloaded"); Self { server: server.clone(), @@ -32,7 +32,7 @@ impl Drop for Suppress { self.server .log .reload - .reload(&self.restore) + .reload(&self.restore, Some(&["console"])) .expect("log filter reloaded"); } } diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 4c8fde5a..78c35c66 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -27,7 +27,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard }; let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.add("format", Box::new(fmt_reload_handle)); + reload_handles.add("console", Box::new(fmt_reload_handle)); let subscriber = Registry::default().with(fmt_layer.with_filter(fmt_reload_filter)); From bc58e5002d95ae86831fd04f8e059798188b7bb8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 01:02:05 +0000 Subject: [PATCH 0465/2091] add interface for current log filter; fix console suppression Signed-off-by: Jason Volk --- src/core/log/reload.rs | 15 ++++++++++++++- src/core/log/suppress.rs | 13 +++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/core/log/reload.rs b/src/core/log/reload.rs index 0b72d787..6d651065 100644 --- a/src/core/log/reload.rs +++ b/src/core/log/reload.rs @@ -22,10 +22,14 @@ use crate::{error, Result}; /// /// [1]: pub trait ReloadHandle { + fn current(&self) -> Option; + fn reload(&self, new_value: L) -> Result<(), reload::Error>; } -impl ReloadHandle for reload::Handle { +impl ReloadHandle for reload::Handle { + fn current(&self) -> Option { Self::clone_current(self) } + fn reload(&self, new_value: L) -> Result<(), reload::Error> { Self::reload(self, new_value) } } @@ -57,6 +61,15 @@ impl LogLevelReloadHandles { Ok(()) } + + #[must_use] + pub fn current(&self, name: &str) -> Option { + self.handles + .lock() + .expect("locked") + .get(name) + .map(|handle| handle.current())? + } } impl Default for LogLevelReloadHandles { diff --git a/src/core/log/suppress.rs b/src/core/log/suppress.rs index ca2d5485..b13ee99e 100644 --- a/src/core/log/suppress.rs +++ b/src/core/log/suppress.rs @@ -10,16 +10,21 @@ pub struct Suppress { impl Suppress { pub fn new(server: &Arc) -> Self { + let handle = "console"; let config = &server.config.log; - Self::from_filters(server, EnvFilter::try_new(config).unwrap_or_default(), &EnvFilter::default()) - } + let suppress = EnvFilter::default(); + let restore = server + .log + .reload + .current(handle) + .unwrap_or_else(|| EnvFilter::try_new(config).unwrap_or_default()); - fn from_filters(server: &Arc, restore: EnvFilter, suppress: &EnvFilter) -> Self { server .log .reload - .reload(suppress, Some(&["console"])) + .reload(&suppress, Some(&[handle])) .expect("log filter reloaded"); + Self { server: server.clone(), restore, From d88ab371200c4bebf21a32c3afb43897af985df4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 02:04:21 +0000 Subject: [PATCH 0466/2091] add configuration for sentry to send panics and errors Signed-off-by: Jason Volk --- conduwuit-example.toml | 7 +++++ src/core/config/mod.rs | 6 +++++ src/main/sentry.rs | 61 ++++++++++++++++++++++++++++++++++++------ 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index cb6dc8e6..ea095bfa 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -60,6 +60,13 @@ # Whether to attach a stacktrace to Sentry reports. #sentry_attach_stacktrace = false +# Send panics to sentry. This is true by default, but sentry has to be enabled. +#sentry_send_panic = true + +# Send errors to sentry. This is true by default, but sentry has to be enabled. This option is +# only effective in release-mode; forced to false in debug-mode. +#sentry_send_error = true + ### Database configuration diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 8b864c84..1997a39e 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -358,6 +358,10 @@ pub struct Config { pub sentry_traces_sample_rate: f32, #[serde(default)] pub sentry_attach_stacktrace: bool, + #[serde(default = "true_fn")] + pub sentry_send_panic: bool, + #[serde(default = "true_fn")] + pub sentry_send_error: bool, #[serde(default)] pub tokio_console: bool, @@ -822,6 +826,8 @@ impl fmt::Display for Config { #[cfg(feature = "sentry_telemetry")] ("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()), ("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()), + ("Sentry.io send panics", &self.sentry_send_panic.to_string()), + ("Sentry.io send errors", &self.sentry_send_error.to_string()), ( "Well-known server name", self.well_known diff --git a/src/main/sentry.rs b/src/main/sentry.rs index 59f88a0c..04ad8654 100644 --- a/src/main/sentry.rs +++ b/src/main/sentry.rs @@ -1,18 +1,34 @@ #![cfg(feature = "sentry_telemetry")] -use std::{str::FromStr, sync::Arc}; - -use conduit::{config::Config, trace}; -use sentry::{ - types::{protocol::v7::Event, Dsn}, - Breadcrumb, ClientOptions, +use std::{ + str::FromStr, + sync::{Arc, OnceLock}, }; +use conduit::{config::Config, debug, trace}; +use sentry::{ + types::{ + protocol::v7::{Context, Event}, + Dsn, + }, + Breadcrumb, ClientOptions, Level, +}; + +static SEND_PANIC: OnceLock = OnceLock::new(); +static SEND_ERROR: OnceLock = OnceLock::new(); + pub(crate) fn init(config: &Config) -> Option { config.sentry.then(|| sentry::init(options(config))) } fn options(config: &Config) -> ClientOptions { + SEND_PANIC + .set(config.sentry_send_panic) + .expect("SEND_PANIC was not previously set"); + SEND_ERROR + .set(config.sentry_send_error) + .expect("SEND_ERROR was not previously set"); + let dsn = config .sentry_endpoint .as_ref() @@ -36,11 +52,40 @@ fn options(config: &Config) -> ClientOptions { } fn before_send(event: Event<'static>) -> Option> { - trace!("Sending sentry event: {event:?}"); + if event.exception.iter().any(|e| e.ty == "panic") && !SEND_PANIC.get().unwrap_or(&true) { + return None; + } + + if event.level == Level::Error { + if !SEND_ERROR.get().unwrap_or(&true) { + return None; + } + + if cfg!(debug_assertions) { + return None; + } + + //NOTE: we can enable this to specify error!(sentry = true, ...) + if let Some(Context::Other(context)) = event.contexts.get("Rust Tracing Fields") { + if !context.contains_key("sentry") { + //return None; + } + } + } + + if event.level == Level::Fatal { + trace!("{event:#?}"); + } + + debug!("Sending sentry event: {event:?}"); Some(event) } fn before_breadcrumb(crumb: Breadcrumb) -> Option { - trace!("Adding sentry breadcrumb: {crumb:?}"); + if crumb.ty == "log" && crumb.level == Level::Debug { + return None; + } + + trace!("Sentry breadcrumb: {crumb:?}"); Some(crumb) } From 57969f9480f5ad2fc4d392043be4003071a52890 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 02:22:37 +0000 Subject: [PATCH 0467/2091] fix large stack array exceeding limit Signed-off-by: Jason Volk --- src/core/config/mod.rs | 710 ++++++++++++++++++++--------------------- 1 file changed, 353 insertions(+), 357 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 1997a39e..df13ad1f 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1,6 +1,6 @@ use std::{ collections::BTreeMap, - fmt::{self, Write as _}, + fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, path::PathBuf, }; @@ -522,364 +522,360 @@ impl Config { impl fmt::Display for Config { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - // Prepare a list of config values to show - let lines = [ - ("Server name", self.server_name.host()), - ("Database backend", &self.database_backend), - ("Database path", &self.database_path.to_string_lossy()), - ( - "Database backup path", - self.database_backup_path - .as_ref() - .map_or("", |path| path.to_str().unwrap_or("")), - ), - ("Database backups to keep", &self.database_backups_to_keep.to_string()), - ("Database cache capacity (MB)", &self.db_cache_capacity_mb.to_string()), - ("Cache capacity modifier", &self.conduit_cache_capacity_modifier.to_string()), - ("PDU cache capacity", &self.pdu_cache_capacity.to_string()), - ("Auth chain cache capacity", &self.auth_chain_cache_capacity.to_string()), - ("Short eventid cache capacity", &self.shorteventid_cache_capacity.to_string()), - ("Eventid short cache capacity", &self.eventidshort_cache_capacity.to_string()), - ("Short statekey cache capacity", &self.shortstatekey_cache_capacity.to_string()), - ("Statekey short cache capacity", &self.statekeyshort_cache_capacity.to_string()), - ( - "Server visibility cache capacity", - &self.server_visibility_cache_capacity.to_string(), - ), - ( - "User visibility cache capacity", - &self.user_visibility_cache_capacity.to_string(), - ), - ("Stateinfo cache capacity", &self.stateinfo_cache_capacity.to_string()), - ( - "Roomid space hierarchy cache capacity", - &self.roomid_spacehierarchy_cache_capacity.to_string(), - ), - ("DNS cache entry limit", &self.dns_cache_entries.to_string()), - ("DNS minimum TTL", &self.dns_min_ttl.to_string()), - ("DNS minimum NXDOMAIN TTL", &self.dns_min_ttl_nxdomain.to_string()), - ("DNS attempts", &self.dns_attempts.to_string()), - ("DNS timeout", &self.dns_timeout.to_string()), - ("DNS fallback to TCP", &self.dns_tcp_fallback.to_string()), - ("DNS query over TCP only", &self.query_over_tcp_only.to_string()), - ("Query all nameservers", &self.query_all_nameservers.to_string()), - ("Maximum request size (bytes)", &self.max_request_size.to_string()), - ("Sender retry backoff limit", &self.sender_retry_backoff_limit.to_string()), - ("Request connect timeout", &self.request_conn_timeout.to_string()), - ("Request timeout", &self.request_timeout.to_string()), - ("Request total timeout", &self.request_total_timeout.to_string()), - ("Idle connections per host", &self.request_idle_per_host.to_string()), - ("Request pool idle timeout", &self.request_idle_timeout.to_string()), - ("Well_known connect timeout", &self.well_known_conn_timeout.to_string()), - ("Well_known timeout", &self.well_known_timeout.to_string()), - ("Federation timeout", &self.federation_timeout.to_string()), - ("Federation pool idle per host", &self.federation_idle_per_host.to_string()), - ("Federation pool idle timeout", &self.federation_idle_timeout.to_string()), - ("Sender timeout", &self.sender_timeout.to_string()), - ("Sender pool idle timeout", &self.sender_idle_timeout.to_string()), - ("Appservice timeout", &self.appservice_timeout.to_string()), - ("Appservice pool idle timeout", &self.appservice_idle_timeout.to_string()), - ("Pusher pool idle timeout", &self.pusher_idle_timeout.to_string()), - ("Allow registration", &self.allow_registration.to_string()), - ( - "Registration token", - if self.registration_token.is_some() { - "set" - } else { - "not set (open registration!)" - }, - ), - ( - "Allow guest registration (inherently false if allow registration is false)", - &self.allow_guest_registration.to_string(), - ), - ( - "Log guest registrations in admin room", - &self.log_guest_registrations.to_string(), - ), - ( - "Allow guests to auto join rooms", - &self.allow_guests_auto_join_rooms.to_string(), - ), - ("New user display name suffix", &self.new_user_displayname_suffix), - ("Allow encryption", &self.allow_encryption.to_string()), - ("Allow federation", &self.allow_federation.to_string()), - ( - "Allow incoming federated presence requests (updates)", - &self.allow_incoming_presence.to_string(), - ), - ( - "Allow outgoing federated presence requests (updates)", - &self.allow_outgoing_presence.to_string(), - ), - ( - "Allow local presence requests (updates)", - &self.allow_local_presence.to_string(), - ), - ( - "Allow incoming remote read receipts", - &self.allow_incoming_read_receipts.to_string(), - ), - ( - "Allow outgoing remote read receipts", - &self.allow_outgoing_read_receipts.to_string(), - ), - ( - "Block non-admin room invites (local and remote, admins can still send and receive invites)", - &self.block_non_admin_invites.to_string(), - ), - ("Enable admin escape commands", &self.admin_escape_commands.to_string()), - ("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()), - ("Allow incoming federated typing", &self.allow_incoming_typing.to_string()), - ( - "Incoming federated typing timeout", - &self.typing_federation_timeout_s.to_string(), - ), - ("Client typing timeout minimum", &self.typing_client_timeout_min_s.to_string()), - ("Client typing timeout maxmimum", &self.typing_client_timeout_max_s.to_string()), - ("Allow device name federation", &self.allow_device_name_federation.to_string()), - ( - "Allow incoming profile lookup federation requests", - &self.allow_profile_lookup_federation_requests.to_string(), - ), - ( - "Auto deactivate banned room join attempts", - &self.auto_deactivate_banned_room_attempts.to_string(), - ), - ("Notification push path", &self.notification_push_path), - ("Allow room creation", &self.allow_room_creation.to_string()), - ( - "Allow public room directory over federation", - &self.allow_public_room_directory_over_federation.to_string(), - ), - ( - "Allow public room directory without authentication", - &self.allow_public_room_directory_without_auth.to_string(), - ), - ( - "Lockdown public room directory (only allow admins to publish)", - &self.lockdown_public_room_directory.to_string(), - ), - ( - "JWT secret", - match self.jwt_secret { - Some(_) => "set", - None => "not set", - }, - ), - ( - "Trusted key servers", - &self - .trusted_servers - .iter() - .map(|server| server.host()) - .join(", "), - ), - ( - "Query Trusted Key Servers First", - &self.query_trusted_key_servers_first.to_string(), - ), - ("OpenID Token TTL", &self.openid_token_ttl.to_string()), - ( - "TURN username", - if self.turn_username.is_empty() { - "not set" - } else { - &self.turn_username - }, - ), - ("TURN password", { - if self.turn_password.is_empty() { - "not set" - } else { - "set" - } - }), - ("TURN secret", { - if self.turn_secret.is_empty() { - "not set" - } else { - "set" - } - }), - ("Turn TTL", &self.turn_ttl.to_string()), - ("Turn URIs", { - let mut lst = vec![]; - for item in self.turn_uris.iter().cloned().enumerate() { - let (_, uri): (usize, String) = item; - lst.push(uri); - } - &lst.join(", ") - }), - ("Auto Join Rooms", { - let mut lst = vec![]; - for room in &self.auto_join_rooms { - lst.push(room); - } - &lst.into_iter().join(", ") - }), - #[cfg(feature = "zstd_compression")] - ("Zstd HTTP Compression", &self.zstd_compression.to_string()), - #[cfg(feature = "gzip_compression")] - ("Gzip HTTP Compression", &self.gzip_compression.to_string()), - #[cfg(feature = "brotli_compression")] - ("Brotli HTTP Compression", &self.brotli_compression.to_string()), - ("RocksDB database LOG level", &self.rocksdb_log_level), - ("RocksDB database LOG to stderr", &self.rocksdb_log_stderr.to_string()), - ("RocksDB database LOG time-to-roll", &self.rocksdb_log_time_to_roll.to_string()), - ("RocksDB Max LOG Files", &self.rocksdb_max_log_files.to_string()), - ( - "RocksDB database max LOG file size", - &self.rocksdb_max_log_file_size.to_string(), - ), - ( - "RocksDB database optimize for spinning disks", - &self.rocksdb_optimize_for_spinning_disks.to_string(), - ), - ("RocksDB Direct-IO", &self.rocksdb_direct_io.to_string()), - ("RocksDB Parallelism Threads", &self.rocksdb_parallelism_threads.to_string()), - ("RocksDB Compression Algorithm", &self.rocksdb_compression_algo), - ("RocksDB Compression Level", &self.rocksdb_compression_level.to_string()), - ( - "RocksDB Bottommost Compression Level", - &self.rocksdb_bottommost_compression_level.to_string(), - ), - ( - "RocksDB Bottommost Level Compression", - &self.rocksdb_bottommost_compression.to_string(), - ), - ("RocksDB Recovery Mode", &self.rocksdb_recovery_mode.to_string()), - ("RocksDB Repair Mode", &self.rocksdb_repair.to_string()), - ("RocksDB Read-only Mode", &self.rocksdb_read_only.to_string()), - ( - "RocksDB Compaction Idle Priority", - &self.rocksdb_compaction_prio_idle.to_string(), - ), - ( - "RocksDB Compaction Idle IOPriority", - &self.rocksdb_compaction_ioprio_idle.to_string(), - ), - ("Media integrity checks on startup", &self.media_startup_check.to_string()), - ("Media compatibility filesystem links", &self.media_compat_file_link.to_string()), - ("Prevent Media Downloads From", { - let mut lst = vec![]; - for domain in &self.prevent_media_downloads_from { - lst.push(domain.host()); - } - &lst.join(", ") - }), - ("Forbidden Remote Server Names (\"Global\" ACLs)", { - let mut lst = vec![]; - for domain in &self.forbidden_remote_server_names { - lst.push(domain.host()); - } - &lst.join(", ") - }), - ("Forbidden Remote Room Directory Server Names", { - let mut lst = vec![]; - for domain in &self.forbidden_remote_room_directory_server_names { - lst.push(domain.host()); - } - &lst.join(", ") - }), - ("Outbound Request IP Range Denylist", { - let mut lst = vec![]; - for item in self.ip_range_denylist.iter().cloned().enumerate() { - let (_, ip): (usize, String) = item; - lst.push(ip); - } - &lst.join(", ") - }), - ("Forbidden usernames", { - &self.forbidden_usernames.patterns().iter().join(", ") - }), - ("Forbidden room aliases", { - &self.forbidden_alias_names.patterns().iter().join(", ") - }), - ( - "URL preview domain contains allowlist", - &self.url_preview_domain_contains_allowlist.join(", "), - ), - ( - "URL preview domain explicit allowlist", - &self.url_preview_domain_explicit_allowlist.join(", "), - ), - ( - "URL preview domain explicit denylist", - &self.url_preview_domain_explicit_denylist.join(", "), - ), - ( - "URL preview URL contains allowlist", - &self.url_preview_url_contains_allowlist.join(", "), - ), - ("URL preview maximum spider size", &self.url_preview_max_spider_size.to_string()), - ("URL preview check root domain", &self.url_preview_check_root_domain.to_string()), - ( - "Allow check for updates / announcements check", - &self.allow_check_for_updates.to_string(), - ), - ("Enable netburst on startup", &self.startup_netburst.to_string()), - #[cfg(feature = "sentry_telemetry")] - ("Sentry.io reporting and tracing", &self.sentry.to_string()), - #[cfg(feature = "sentry_telemetry")] - ("Sentry.io send server_name in logs", &self.sentry_send_server_name.to_string()), - #[cfg(feature = "sentry_telemetry")] - ("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()), - ("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()), - ("Sentry.io send panics", &self.sentry_send_panic.to_string()), - ("Sentry.io send errors", &self.sentry_send_error.to_string()), - ( - "Well-known server name", - self.well_known - .server - .as_ref() - .map_or("", |server| server.as_str()), - ), - ( - "Well-known client URL", - self.well_known - .client - .as_ref() - .map_or("", |url| url.as_str()), - ), - ( - "Well-known support email", - self.well_known - .support_email - .as_ref() - .map_or("", |str| str.as_ref()), - ), - ( - "Well-known support Matrix ID", - self.well_known - .support_mxid - .as_ref() - .map_or("", |mxid| mxid.as_str()), - ), - ( - "Well-known support role", - self.well_known - .support_role - .as_ref() - .map_or("", |role| role.as_str()), - ), - ( - "Well-known support page/URL", - self.well_known - .support_page - .as_ref() - .map_or("", |url| url.as_str()), - ), - ("Enable the tokio-console", &self.tokio_console.to_string()), - ]; + writeln!(f, "Active config values:\n\n").expect("wrote line to formatter stream"); + let mut line = |key: &str, val: &str| { + writeln!(f, "{key}: {val}").expect("wrote line to formatter stream"); + }; - let mut msg: String = "Active config values:\n\n".to_owned(); + line("Server name", self.server_name.host()); + line("Database backend", &self.database_backend); + line("Database path", &self.database_path.to_string_lossy()); + line( + "Database backup path", + self.database_backup_path + .as_ref() + .map_or("", |path| path.to_str().unwrap_or("")), + ); + line("Database backups to keep", &self.database_backups_to_keep.to_string()); + line("Database cache capacity (MB)", &self.db_cache_capacity_mb.to_string()); + line("Cache capacity modifier", &self.conduit_cache_capacity_modifier.to_string()); + line("PDU cache capacity", &self.pdu_cache_capacity.to_string()); + line("Auth chain cache capacity", &self.auth_chain_cache_capacity.to_string()); + line("Short eventid cache capacity", &self.shorteventid_cache_capacity.to_string()); + line("Eventid short cache capacity", &self.eventidshort_cache_capacity.to_string()); + line("Short statekey cache capacity", &self.shortstatekey_cache_capacity.to_string()); + line("Statekey short cache capacity", &self.statekeyshort_cache_capacity.to_string()); + line( + "Server visibility cache capacity", + &self.server_visibility_cache_capacity.to_string(), + ); + line( + "User visibility cache capacity", + &self.user_visibility_cache_capacity.to_string(), + ); + line("Stateinfo cache capacity", &self.stateinfo_cache_capacity.to_string()); + line( + "Roomid space hierarchy cache capacity", + &self.roomid_spacehierarchy_cache_capacity.to_string(), + ); + line("DNS cache entry limit", &self.dns_cache_entries.to_string()); + line("DNS minimum TTL", &self.dns_min_ttl.to_string()); + line("DNS minimum NXDOMAIN TTL", &self.dns_min_ttl_nxdomain.to_string()); + line("DNS attempts", &self.dns_attempts.to_string()); + line("DNS timeout", &self.dns_timeout.to_string()); + line("DNS fallback to TCP", &self.dns_tcp_fallback.to_string()); + line("DNS query over TCP only", &self.query_over_tcp_only.to_string()); + line("Query all nameservers", &self.query_all_nameservers.to_string()); + line("Maximum request size (bytes)", &self.max_request_size.to_string()); + line("Sender retry backoff limit", &self.sender_retry_backoff_limit.to_string()); + line("Request connect timeout", &self.request_conn_timeout.to_string()); + line("Request timeout", &self.request_timeout.to_string()); + line("Request total timeout", &self.request_total_timeout.to_string()); + line("Idle connections per host", &self.request_idle_per_host.to_string()); + line("Request pool idle timeout", &self.request_idle_timeout.to_string()); + line("Well_known connect timeout", &self.well_known_conn_timeout.to_string()); + line("Well_known timeout", &self.well_known_timeout.to_string()); + line("Federation timeout", &self.federation_timeout.to_string()); + line("Federation pool idle per host", &self.federation_idle_per_host.to_string()); + line("Federation pool idle timeout", &self.federation_idle_timeout.to_string()); + line("Sender timeout", &self.sender_timeout.to_string()); + line("Sender pool idle timeout", &self.sender_idle_timeout.to_string()); + line("Appservice timeout", &self.appservice_timeout.to_string()); + line("Appservice pool idle timeout", &self.appservice_idle_timeout.to_string()); + line("Pusher pool idle timeout", &self.pusher_idle_timeout.to_string()); + line("Allow registration", &self.allow_registration.to_string()); + line( + "Registration token", + if self.registration_token.is_some() { + "set" + } else { + "not set (open registration!)" + }, + ); + line( + "Allow guest registration (inherently false if allow registration is false)", + &self.allow_guest_registration.to_string(), + ); + line( + "Log guest registrations in admin room", + &self.log_guest_registrations.to_string(), + ); + line( + "Allow guests to auto join rooms", + &self.allow_guests_auto_join_rooms.to_string(), + ); + line("New user display name suffix", &self.new_user_displayname_suffix); + line("Allow encryption", &self.allow_encryption.to_string()); + line("Allow federation", &self.allow_federation.to_string()); + line( + "Allow incoming federated presence requests (updates)", + &self.allow_incoming_presence.to_string(), + ); + line( + "Allow outgoing federated presence requests (updates)", + &self.allow_outgoing_presence.to_string(), + ); + line( + "Allow local presence requests (updates)", + &self.allow_local_presence.to_string(), + ); + line( + "Allow incoming remote read receipts", + &self.allow_incoming_read_receipts.to_string(), + ); + line( + "Allow outgoing remote read receipts", + &self.allow_outgoing_read_receipts.to_string(), + ); + line( + "Block non-admin room invites (local and remote, admins can still send and receive invites)", + &self.block_non_admin_invites.to_string(), + ); + line("Enable admin escape commands", &self.admin_escape_commands.to_string()); + line("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()); + line("Allow incoming federated typing", &self.allow_incoming_typing.to_string()); + line( + "Incoming federated typing timeout", + &self.typing_federation_timeout_s.to_string(), + ); + line("Client typing timeout minimum", &self.typing_client_timeout_min_s.to_string()); + line("Client typing timeout maxmimum", &self.typing_client_timeout_max_s.to_string()); + line("Allow device name federation", &self.allow_device_name_federation.to_string()); + line( + "Allow incoming profile lookup federation requests", + &self.allow_profile_lookup_federation_requests.to_string(), + ); + line( + "Auto deactivate banned room join attempts", + &self.auto_deactivate_banned_room_attempts.to_string(), + ); + line("Notification push path", &self.notification_push_path); + line("Allow room creation", &self.allow_room_creation.to_string()); + line( + "Allow public room directory over federation", + &self.allow_public_room_directory_over_federation.to_string(), + ); + line( + "Allow public room directory without authentication", + &self.allow_public_room_directory_without_auth.to_string(), + ); + line( + "Lockdown public room directory (only allow admins to publish)", + &self.lockdown_public_room_directory.to_string(), + ); + line( + "JWT secret", + match self.jwt_secret { + Some(_) => "set", + None => "not set", + }, + ); + line( + "Trusted key servers", + &self + .trusted_servers + .iter() + .map(|server| server.host()) + .join(", "), + ); + line( + "Query Trusted Key Servers First", + &self.query_trusted_key_servers_first.to_string(), + ); + line("OpenID Token TTL", &self.openid_token_ttl.to_string()); + line( + "TURN username", + if self.turn_username.is_empty() { + "not set" + } else { + &self.turn_username + }, + ); + line("TURN password", { + if self.turn_password.is_empty() { + "not set" + } else { + "set" + } + }); + line("TURN secret", { + if self.turn_secret.is_empty() { + "not set" + } else { + "set" + } + }); + line("Turn TTL", &self.turn_ttl.to_string()); + line("Turn URIs", { + let mut lst = vec![]; + for item in self.turn_uris.iter().cloned().enumerate() { + let (_, uri): (usize, String) = item; + lst.push(uri); + } + &lst.join(", ") + }); + line("Auto Join Rooms", { + let mut lst = vec![]; + for room in &self.auto_join_rooms { + lst.push(room); + } + &lst.into_iter().join(", ") + }); + #[cfg(feature = "zstd_compression")] + line("Zstd HTTP Compression", &self.zstd_compression.to_string()); + #[cfg(feature = "gzip_compression")] + line("Gzip HTTP Compression", &self.gzip_compression.to_string()); + #[cfg(feature = "brotli_compression")] + line("Brotli HTTP Compression", &self.brotli_compression.to_string()); + line("RocksDB database LOG level", &self.rocksdb_log_level); + line("RocksDB database LOG to stderr", &self.rocksdb_log_stderr.to_string()); + line("RocksDB database LOG time-to-roll", &self.rocksdb_log_time_to_roll.to_string()); + line("RocksDB Max LOG Files", &self.rocksdb_max_log_files.to_string()); + line( + "RocksDB database max LOG file size", + &self.rocksdb_max_log_file_size.to_string(), + ); + line( + "RocksDB database optimize for spinning disks", + &self.rocksdb_optimize_for_spinning_disks.to_string(), + ); + line("RocksDB Direct-IO", &self.rocksdb_direct_io.to_string()); + line("RocksDB Parallelism Threads", &self.rocksdb_parallelism_threads.to_string()); + line("RocksDB Compression Algorithm", &self.rocksdb_compression_algo); + line("RocksDB Compression Level", &self.rocksdb_compression_level.to_string()); + line( + "RocksDB Bottommost Compression Level", + &self.rocksdb_bottommost_compression_level.to_string(), + ); + line( + "RocksDB Bottommost Level Compression", + &self.rocksdb_bottommost_compression.to_string(), + ); + line("RocksDB Recovery Mode", &self.rocksdb_recovery_mode.to_string()); + line("RocksDB Repair Mode", &self.rocksdb_repair.to_string()); + line("RocksDB Read-only Mode", &self.rocksdb_read_only.to_string()); + line( + "RocksDB Compaction Idle Priority", + &self.rocksdb_compaction_prio_idle.to_string(), + ); + line( + "RocksDB Compaction Idle IOPriority", + &self.rocksdb_compaction_ioprio_idle.to_string(), + ); + line("Media integrity checks on startup", &self.media_startup_check.to_string()); + line("Media compatibility filesystem links", &self.media_compat_file_link.to_string()); + line("Prevent Media Downloads From", { + let mut lst = vec![]; + for domain in &self.prevent_media_downloads_from { + lst.push(domain.host()); + } + &lst.join(", ") + }); + line("Forbidden Remote Server Names (\"Global\" ACLs)", { + let mut lst = vec![]; + for domain in &self.forbidden_remote_server_names { + lst.push(domain.host()); + } + &lst.join(", ") + }); + line("Forbidden Remote Room Directory Server Names", { + let mut lst = vec![]; + for domain in &self.forbidden_remote_room_directory_server_names { + lst.push(domain.host()); + } + &lst.join(", ") + }); + line("Outbound Request IP Range Denylist", { + let mut lst = vec![]; + for item in self.ip_range_denylist.iter().cloned().enumerate() { + let (_, ip): (usize, String) = item; + lst.push(ip); + } + &lst.join(", ") + }); + line("Forbidden usernames", { + &self.forbidden_usernames.patterns().iter().join(", ") + }); + line("Forbidden room aliases", { + &self.forbidden_alias_names.patterns().iter().join(", ") + }); + line( + "URL preview domain contains allowlist", + &self.url_preview_domain_contains_allowlist.join(", "), + ); + line( + "URL preview domain explicit allowlist", + &self.url_preview_domain_explicit_allowlist.join(", "), + ); + line( + "URL preview domain explicit denylist", + &self.url_preview_domain_explicit_denylist.join(", "), + ); + line( + "URL preview URL contains allowlist", + &self.url_preview_url_contains_allowlist.join(", "), + ); + line("URL preview maximum spider size", &self.url_preview_max_spider_size.to_string()); + line("URL preview check root domain", &self.url_preview_check_root_domain.to_string()); + line( + "Allow check for updates / announcements check", + &self.allow_check_for_updates.to_string(), + ); + line("Enable netburst on startup", &self.startup_netburst.to_string()); + #[cfg(feature = "sentry_telemetry")] + line("Sentry.io reporting and tracing", &self.sentry.to_string()); + #[cfg(feature = "sentry_telemetry")] + line("Sentry.io send server_name in logs", &self.sentry_send_server_name.to_string()); + #[cfg(feature = "sentry_telemetry")] + line("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()); + line("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()); + line("Sentry.io send panics", &self.sentry_send_panic.to_string()); + line("Sentry.io send errors", &self.sentry_send_error.to_string()); + line( + "Well-known server name", + self.well_known + .server + .as_ref() + .map_or("", |server| server.as_str()), + ); + line( + "Well-known client URL", + self.well_known + .client + .as_ref() + .map_or("", |url| url.as_str()), + ); + line( + "Well-known support email", + self.well_known + .support_email + .as_ref() + .map_or("", |str| str.as_ref()), + ); + line( + "Well-known support Matrix ID", + self.well_known + .support_mxid + .as_ref() + .map_or("", |mxid| mxid.as_str()), + ); + line( + "Well-known support role", + self.well_known + .support_role + .as_ref() + .map_or("", |role| role.as_str()), + ); + line( + "Well-known support page/URL", + self.well_known + .support_page + .as_ref() + .map_or("", |url| url.as_str()), + ); + line("Enable the tokio-console", &self.tokio_console.to_string()); - for line in lines.into_iter().enumerate() { - writeln!(msg, "{}: {}", line.1 .0, line.1 .1).expect("should be able to write to string buffer"); - } - - write!(f, "{msg}") + Ok(()) } } From 899b79873e51b5746ca466f3b7bdcaa864d036be Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 05:03:16 +0000 Subject: [PATCH 0468/2091] propagate errors from tracing init Signed-off-by: Jason Volk --- src/main/server.rs | 2 +- src/main/tracing.rs | 57 +++++++++++++++++---------------------------- 2 files changed, 22 insertions(+), 37 deletions(-) diff --git a/src/main/server.rs b/src/main/server.rs index f72b3ef3..73c06f0c 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -27,7 +27,7 @@ impl Server { #[cfg(feature = "sentry_telemetry")] let sentry_guard = crate::sentry::init(&config); - let (tracing_reload_handle, tracing_flame_guard, capture) = crate::tracing::init(&config); + let (tracing_reload_handle, tracing_flame_guard, capture) = crate::tracing::init(&config)?; config.check()?; diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 78c35c66..31870661 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -1,10 +1,10 @@ use std::sync::Arc; use conduit::{ - config, config::Config, debug_warn, log::{capture, LogLevelReloadHandles}, + Error, Result, }; use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; @@ -14,26 +14,21 @@ pub(crate) type TracingFlameGuard = Option (LogLevelReloadHandles, TracingFlameGuard, Arc) { +pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFlameGuard, Arc)> { let reload_handles = LogLevelReloadHandles::default(); - let fmt_layer = tracing_subscriber::fmt::Layer::new(); - let filter_layer = match EnvFilter::try_new(&config.log) { - Ok(s) => s, - Err(e) => { - eprintln!("It looks like your config is invalid. The following error occured while parsing it: {e}"); - EnvFilter::try_new(config::default_log()).expect("failed to set default EnvFilter") - }, - }; - - let (fmt_reload_filter, fmt_reload_handle) = reload::Layer::new(filter_layer.clone()); - reload_handles.add("console", Box::new(fmt_reload_handle)); - - let subscriber = Registry::default().with(fmt_layer.with_filter(fmt_reload_filter)); + let console_filter = + EnvFilter::try_new(&config.log).map_err(|e| Error::BadConfig(format!("in the 'log' setting: {e}.")))?; + let console_layer = tracing_subscriber::fmt::Layer::new(); + let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); + reload_handles.add("console", Box::new(console_reload_handle)); let cap_state = Arc::new(capture::State::new()); let cap_layer = capture::Layer::new(&cap_state); - let subscriber = subscriber.with(cap_layer); + + let subscriber = Registry::default() + .with(console_layer.with_filter(console_reload_filter)) + .with(cap_layer); #[cfg(feature = "sentry_telemetry")] let subscriber = { @@ -46,16 +41,10 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard #[cfg(feature = "perf_measurements")] let (subscriber, flame_guard) = { let (flame_layer, flame_guard) = if config.tracing_flame { - let flame_filter = match EnvFilter::try_new(&config.tracing_flame_filter) { - Ok(flame_filter) => flame_filter, - Err(e) => panic!("tracing_flame_filter config value is invalid: {e}"), - }; - - let (flame_layer, flame_guard) = - match tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) { - Ok(ok) => ok, - Err(e) => panic!("failed to initialize tracing-flame: {e}"), - }; + let flame_filter = EnvFilter::try_new(&config.tracing_flame_filter) + .map_err(|e| Error::BadConfig(format!("in the 'tracing_flame_filter' setting: {e}.")))?; + let (flame_layer, flame_guard) = tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) + .map_err(|e| Error::BadConfig(format!("in the 'tracing_flame_output_path' setting: {e}.")))?; let flame_layer = flame_layer .with_empty_samples(false) .with_filter(flame_filter); @@ -64,22 +53,18 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard (None, None) }; - let jaeger_layer = if config.allow_jaeger { + let jaeger_layer = config.allow_jaeger.then(|| { opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); let tracer = opentelemetry_jaeger::new_agent_pipeline() .with_auto_split_batch(true) .with_service_name("conduwuit") .install_batch(opentelemetry_sdk::runtime::Tokio) - .unwrap(); + .expect("jaeger agent pipeline"); let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - - let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(filter_layer.clone()); + let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(console_filter.clone()); reload_handles.add("jaeger", Box::new(jaeger_reload_handle)); Some(telemetry.with_filter(jaeger_reload_filter)) - } else { - None - }; - + }); let subscriber = subscriber.with(flame_layer).with(jaeger_layer); (subscriber, flame_guard) }; @@ -101,7 +86,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard .spawn(); set_global_default(subscriber.with(console_layer)); - return ret; + return Ok(ret); } set_global_default(subscriber); @@ -112,7 +97,7 @@ pub(crate) fn init(config: &Config) -> (LogLevelReloadHandles, TracingFlameGuard debug_warn!("{console_disabled_reason}"); } - ret + Ok(ret) } fn tokio_console_enabled(config: &Config) -> (bool, &'static str) { From 5570220c8901d6a423a1c2ea4d86f8ed5d8b1ba4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 05:03:41 +0000 Subject: [PATCH 0469/2091] use separate but configurable envfilter for sentry Signed-off-by: Jason Volk --- src/core/config/mod.rs | 5 +++++ src/main/tracing.rs | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index df13ad1f..4eefddcf 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -362,6 +362,8 @@ pub struct Config { pub sentry_send_panic: bool, #[serde(default = "true_fn")] pub sentry_send_error: bool, + #[serde(default = "default_sentry_filter")] + pub sentry_filter: String, #[serde(default)] pub tokio_console: bool, @@ -831,6 +833,7 @@ impl fmt::Display for Config { line("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()); line("Sentry.io send panics", &self.sentry_send_panic.to_string()); line("Sentry.io send errors", &self.sentry_send_error.to_string()); + line("Sentry.io tracing filter", &self.sentry_filter); line( "Well-known server name", self.well_known @@ -1075,4 +1078,6 @@ fn default_sentry_endpoint() -> Option { fn default_sentry_traces_sample_rate() -> f32 { 0.15 } +fn default_sentry_filter() -> String { "info".to_owned() } + fn default_startup_netburst_keep() -> i64 { 50 } diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 31870661..ccfe188d 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -32,8 +32,10 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla #[cfg(feature = "sentry_telemetry")] let subscriber = { + let sentry_filter = EnvFilter::try_new(&config.sentry_filter) + .map_err(|e| Error::BadConfig(format!("in the 'sentry_filter' setting: {e}.")))?; let sentry_layer = sentry_tracing::layer(); - let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(filter_layer.clone()); + let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(sentry_filter); reload_handles.add("sentry", Box::new(sentry_reload_handle)); subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) }; From a35b6cbfdd8cbea73056b94652bc43807d0f1521 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 05:26:43 +0000 Subject: [PATCH 0470/2091] use separate configurable jaeger envfilter Signed-off-by: Jason Volk --- src/core/config/mod.rs | 11 +++++++++++ src/main/tracing.rs | 5 ++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 4eefddcf..aadfe564 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -183,6 +183,9 @@ pub struct Config { #[serde(default)] #[cfg(feature = "perf_measurements")] pub allow_jaeger: bool, + #[serde(default = "default_jaeger_filter")] + #[cfg(feature = "perf_measurements")] + pub jaeger_filter: String, #[serde(default)] #[cfg(feature = "perf_measurements")] pub tracing_flame: bool, @@ -979,6 +982,14 @@ fn default_max_fetch_prev_events() -> u16 { 100_u16 } #[cfg(feature = "perf_measurements")] fn default_tracing_flame_filter() -> String { "trace,h2=off".to_owned() } +#[cfg(feature = "perf_measurements")] +fn default_jaeger_filter() -> String { + cfg!(debug_assertions) + .then_some("trace,h2=off") + .unwrap_or("info") + .to_owned() +} + #[cfg(feature = "perf_measurements")] fn default_tracing_flame_output_path() -> String { "./tracing.folded".to_owned() } diff --git a/src/main/tracing.rs b/src/main/tracing.rs index ccfe188d..67977088 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -55,6 +55,8 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla (None, None) }; + let jaeger_filter = EnvFilter::try_new(&config.jaeger_filter) + .map_err(|e| Error::BadConfig(format!("in the 'jaeger_filter' setting: {e}.")))?; let jaeger_layer = config.allow_jaeger.then(|| { opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); let tracer = opentelemetry_jaeger::new_agent_pipeline() @@ -63,10 +65,11 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla .install_batch(opentelemetry_sdk::runtime::Tokio) .expect("jaeger agent pipeline"); let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(console_filter.clone()); + let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(jaeger_filter.clone()); reload_handles.add("jaeger", Box::new(jaeger_reload_handle)); Some(telemetry.with_filter(jaeger_reload_filter)) }); + let subscriber = subscriber.with(flame_layer).with(jaeger_layer); (subscriber, flame_guard) }; From 03d890cd498e8ce55d9e86d38f0b0525c18ce61a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 08:54:33 +0000 Subject: [PATCH 0471/2091] move admin tests into unit; fix Signed-off-by: Jason Volk --- src/admin/handler.rs | 1 - src/admin/mod.rs | 28 +--------------------------- src/admin/tests.rs | 26 ++++++++++++++++++++++++++ 3 files changed, 27 insertions(+), 28 deletions(-) create mode 100644 src/admin/tests.rs diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 2d1f96b5..95c7ed41 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -24,7 +24,6 @@ use crate::{ }; pub(crate) const PAGE_SIZE: usize = 100; -#[cfg_attr(test, derive(Debug))] #[derive(Parser)] #[command(name = "admin", version = env!("CARGO_PKG_VERSION"))] pub(crate) enum AdminCommand { diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 6a47bc74..14856811 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -9,6 +9,7 @@ pub(crate) mod media; pub(crate) mod query; pub(crate) mod room; pub(crate) mod server; +mod tests; pub(crate) mod user; pub(crate) mod utils; @@ -53,30 +54,3 @@ pub async fn fini() { .expect("locked for writing") .take(); } - -#[cfg(test)] -mod test { - use clap::Parser; - - use crate::handler::AdminCommand; - - #[test] - fn get_help_short() { get_help_inner("-h"); } - - #[test] - fn get_help_long() { get_help_inner("--help"); } - - #[test] - fn get_help_subcommand() { get_help_inner("help"); } - - fn get_help_inner(input: &str) { - let error = AdminCommand::try_parse_from(["argv[0] doesn't matter", input]) - .unwrap_err() - .to_string(); - - // Search for a handful of keywords that suggest the help printed properly - assert!(error.contains("Usage:")); - assert!(error.contains("Commands:")); - assert!(error.contains("Options:")); - } -} diff --git a/src/admin/tests.rs b/src/admin/tests.rs new file mode 100644 index 00000000..69ccd896 --- /dev/null +++ b/src/admin/tests.rs @@ -0,0 +1,26 @@ +#![cfg(test)] + +#[test] +fn get_help_short() { get_help_inner("-h"); } + +#[test] +fn get_help_long() { get_help_inner("--help"); } + +#[test] +fn get_help_subcommand() { get_help_inner("help"); } + +fn get_help_inner(input: &str) { + use clap::Parser; + + use crate::handler::AdminCommand; + + let Err(error) = AdminCommand::try_parse_from(["argv[0] doesn't matter", input]) else { + panic!("no error!"); + }; + + let error = error.to_string(); + // Search for a handful of keywords that suggest the help printed properly + assert!(error.contains("Usage:")); + assert!(error.contains("Commands:")); + assert!(error.contains("Options:")); +} From c111d2e39536672224e4a2efe19d674e5ac7c6a0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 11 Jul 2024 21:00:30 +0000 Subject: [PATCH 0472/2091] abstract service worker pattern; restart on panic. Signed-off-by: Jason Volk --- src/router/run.rs | 6 +- src/service/admin/mod.rs | 64 +++-------- src/service/mod.rs | 6 +- src/service/presence/mod.rs | 57 +++------ src/service/sending/mod.rs | 50 +------- src/service/sending/sender.rs | 39 ++++--- src/service/service.rs | 17 +-- src/service/services.rs | 210 +++++++++++++++++++++++++--------- 8 files changed, 233 insertions(+), 216 deletions(-) diff --git a/src/router/run.rs b/src/router/run.rs index 3e09823a..02cec781 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -11,7 +11,6 @@ extern crate conduit_service as service; use std::sync::atomic::Ordering; use conduit::{debug_info, trace, Error, Result, Server}; -use service::services; use crate::{layers, serve}; @@ -50,7 +49,6 @@ pub(crate) async fn start(server: Arc) -> Result<(), Error> { debug!("Starting..."); service::init(&server).await?; - services().start().await?; #[cfg(feature = "systemd")] sd_notify::notify(true, &[sd_notify::NotifyState::Ready]).expect("failed to notify systemd of ready state"); @@ -66,9 +64,7 @@ pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { // Wait for all completions before dropping or we'll lose them to the module // unload and explode. - services().stop().await; - // Deactivate services(). Any further use will panic the caller. - service::fini(); + service::fini().await; debug!("Cleaning up..."); diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index e9729d2d..41019cd1 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -21,17 +21,13 @@ use ruma::{ OwnedEventId, OwnedRoomId, RoomId, UserId, }; use serde_json::value::to_raw_value; -use tokio::{ - sync::{Mutex, RwLock}, - task::JoinHandle, -}; +use tokio::sync::{Mutex, RwLock}; use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, user_is_local, PduEvent}; pub struct Service { sender: Sender, receiver: Mutex>, - handler_join: Mutex>>, pub handle: RwLock>, pub complete: StdRwLock>, #[cfg(feature = "console")] @@ -59,7 +55,6 @@ impl crate::Service for Service { Ok(Arc::new(Self { sender, receiver: Mutex::new(receiver), - handler_join: Mutex::new(None), handle: RwLock::new(None), complete: StdRwLock::new(None), #[cfg(feature = "console")] @@ -67,16 +62,25 @@ impl crate::Service for Service { })) } - async fn start(self: Arc) -> Result<()> { - let self_ = Arc::clone(&self); - let handle = services().server.runtime().spawn(async move { - self_ - .handler() - .await - .expect("Failed to initialize admin room handler"); - }); + async fn worker(self: Arc) -> Result<()> { + let receiver = self.receiver.lock().await; + let mut signals = services().server.signal.subscribe(); + loop { + tokio::select! { + command = receiver.recv_async() => match command { + Ok(command) => self.handle_command(command).await, + Err(_) => break, + }, + sig = signals.recv() => match sig { + Ok(sig) => self.handle_signal(sig).await, + Err(_) => continue, + }, + } + } - _ = self.handler_join.lock().await.insert(handle); + //TODO: not unwind safe + #[cfg(feature = "console")] + self.console.close().await; Ok(()) } @@ -90,19 +94,6 @@ impl crate::Service for Service { } } - async fn stop(&self) { - self.interrupt(); - - #[cfg(feature = "console")] - self.console.close().await; - - if let Some(handler_join) = self.handler_join.lock().await.take() { - if let Err(e) = handler_join.await { - error!("Failed to shutdown: {e:?}"); - } - } - } - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } @@ -149,23 +140,6 @@ impl Service { self.sender.send_async(message).await.expect("message sent"); } - async fn handler(self: &Arc) -> Result<()> { - let receiver = self.receiver.lock().await; - let mut signals = services().server.signal.subscribe(); - loop { - tokio::select! { - command = receiver.recv_async() => match command { - Ok(command) => self.handle_command(command).await, - Err(_) => return Ok(()), - }, - sig = signals.recv() => match sig { - Ok(sig) => self.handle_signal(sig).await, - Err(_) => continue, - }, - } - } - } - async fn handle_signal(&self, #[allow(unused_variables)] sig: &'static str) { #[cfg(feature = "console")] self.console.handle_signal(sig).await; diff --git a/src/service/mod.rs b/src/service/mod.rs index 4b19073d..15c4cc35 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -42,10 +42,12 @@ pub async fn init(server: &Arc) -> Result<()> { let s = Box::new(Services::build(server.clone(), d)?); _ = SERVICES.write().expect("write locked").insert(Box::leak(s)); - Ok(()) + services().start().await } -pub fn fini() { +pub async fn fini() { + services().stop().await; + // Deactivate services(). Any further use will panic the caller. let s = SERVICES .write() diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index f5400379..254304ba 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -12,7 +12,7 @@ use ruma::{ OwnedUserId, UInt, UserId, }; use serde::{Deserialize, Serialize}; -use tokio::{sync::Mutex, task::JoinHandle, time::sleep}; +use tokio::{sync::Mutex, time::sleep}; use crate::{services, user_is_local}; @@ -77,7 +77,6 @@ pub struct Service { pub db: Data, pub timer_sender: loole::Sender<(OwnedUserId, Duration)>, timer_receiver: Mutex>, - handler_join: Mutex>>, timeout_remote_users: bool, idle_timeout: u64, offline_timeout: u64, @@ -94,34 +93,26 @@ impl crate::Service for Service { db: Data::new(args.db), timer_sender, timer_receiver: Mutex::new(timer_receiver), - handler_join: Mutex::new(None), timeout_remote_users: config.presence_timeout_remote_users, idle_timeout: checked!(idle_timeout_s * 1_000)?, offline_timeout: checked!(offline_timeout_s * 1_000)?, })) } - async fn start(self: Arc) -> Result<()> { - //TODO: if self.globals.config.allow_local_presence { return; } - - let self_ = Arc::clone(&self); - let handle = services().server.runtime().spawn(async move { - self_ - .handler() - .await - .expect("Failed to start presence handler"); - }); - - _ = self.handler_join.lock().await.insert(handle); - - Ok(()) - } - - async fn stop(&self) { - self.interrupt(); - if let Some(handler_join) = self.handler_join.lock().await.take() { - if let Err(e) = handler_join.await { - error!("Failed to shutdown: {e:?}"); + async fn worker(self: Arc) -> Result<()> { + let mut presence_timers = FuturesUnordered::new(); + let receiver = self.timer_receiver.lock().await; + loop { + debug_assert!(!receiver.is_closed(), "channel error"); + tokio::select! { + Some(user_id) = presence_timers.next() => self.process_presence_timer(&user_id)?, + event = receiver.recv_async() => match event { + Err(_e) => return Ok(()), + Ok((user_id, timeout)) => { + debug!("Adding timer {}: {user_id} timeout:{timeout:?}", presence_timers.len()); + presence_timers.push(presence_timer(user_id, timeout)); + }, + }, } } } @@ -219,24 +210,6 @@ impl Service { self.db.presence_since(since) } - async fn handler(&self) -> Result<()> { - let mut presence_timers = FuturesUnordered::new(); - let receiver = self.timer_receiver.lock().await; - loop { - debug_assert!(!receiver.is_closed(), "channel error"); - tokio::select! { - Some(user_id) = presence_timers.next() => self.process_presence_timer(&user_id)?, - event = receiver.recv_async() => match event { - Err(_e) => return Ok(()), - Ok((user_id, timeout)) => { - debug!("Adding timer {}: {user_id} timeout:{timeout:?}", presence_timers.len()); - presence_timers.push(presence_timer(user_id, timeout)); - }, - }, - } - } - } - fn process_presence_timer(&self, user_id: &OwnedUserId) -> Result<()> { let mut presence_state = PresenceState::Offline; let mut last_active_ago = None; diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index d7a9c0fc..eb708fcf 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -4,29 +4,26 @@ mod resolve; mod send; mod sender; -use std::{fmt::Debug, sync::Arc}; +use std::fmt::Debug; -use async_trait::async_trait; use conduit::{Error, Result}; -use data::Data; pub use resolve::{resolve_actual_dest, CachedDest, CachedOverride, FedDest}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; pub use sender::convert_to_outgoing_federation_event; -use tokio::{sync::Mutex, task::JoinHandle}; -use tracing::{error, warn}; +use tokio::sync::Mutex; +use tracing::warn; use crate::{server_is_ours, services}; pub struct Service { - pub db: Data, + pub db: data::Data, /// The state for a given state hash. sender: loole::Sender, receiver: Mutex>, - handler_join: Mutex>>, startup_netburst: bool, startup_netburst_keep: i64, } @@ -53,45 +50,6 @@ pub enum SendingEvent { Flush, // none } -#[async_trait] -impl crate::Service for Service { - fn build(args: crate::Args<'_>) -> Result> { - let config = &args.server.config; - let (sender, receiver) = loole::unbounded(); - Ok(Arc::new(Self { - db: Data::new(args.db.clone()), - sender, - receiver: Mutex::new(receiver), - handler_join: Mutex::new(None), - startup_netburst: config.startup_netburst, - startup_netburst_keep: config.startup_netburst_keep, - })) - } - - async fn start(self: Arc) -> Result<()> { - self.start_handler().await; - - Ok(()) - } - - async fn stop(&self) { - self.interrupt(); - if let Some(handler_join) = self.handler_join.lock().await.take() { - if let Err(e) = handler_join.await { - error!("Failed to shutdown: {e:?}"); - } - } - } - - fn interrupt(&self) { - if !self.sender.is_closed() { - self.sender.close(); - } - } - - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } -} - impl Service { #[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")] pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 0fb0d9dc..cfd5b4bc 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -6,6 +6,7 @@ use std::{ time::Instant, }; +use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; use conduit::{debug, error, utils::math::continue_exponential_backoff_secs, warn}; use federation::transactions::send_transaction_message; @@ -23,8 +24,9 @@ use ruma::{ ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use tokio::sync::Mutex; -use super::{appservice, send, Destination, Msg, SendingEvent, Service}; +use super::{appservice, data::Data, send, Destination, Msg, SendingEvent, Service}; use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; #[derive(Debug)] @@ -43,21 +45,22 @@ type CurTransactionStatus = HashMap; const DEQUEUE_LIMIT: usize = 48; const SELECT_EDU_LIMIT: usize = 16; -impl Service { - pub async fn start_handler(self: &Arc) { - let self_ = Arc::clone(self); - let handle = services().server.runtime().spawn(async move { - self_ - .handler() - .await - .expect("Failed to start sending handler"); - }); - - _ = self.handler_join.lock().await.insert(handle); +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + let (sender, receiver) = loole::unbounded(); + Ok(Arc::new(Self { + db: Data::new(args.db.clone()), + sender, + receiver: Mutex::new(receiver), + startup_netburst: config.startup_netburst, + startup_netburst_keep: config.startup_netburst_keep, + })) } #[tracing::instrument(skip_all, name = "sender")] - async fn handler(&self) -> Result<()> { + async fn worker(self: Arc) -> Result<()> { let receiver = self.receiver.lock().await; let mut futures: SendingFutures<'_> = FuturesUnordered::new(); let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); @@ -77,6 +80,16 @@ impl Service { } } + fn interrupt(&self) { + if !self.sender.is_closed() { + self.sender.close(); + } + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { fn handle_response( &self, response: SendingResult, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, ) { diff --git a/src/service/service.rs b/src/service/service.rs index ef60f359..3b8f4231 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -15,19 +15,12 @@ pub(crate) trait Service: Send + Sync { where Self: Sized; - /// Start the service. Implement the spawning of any service workers. This - /// is called after all other services have been constructed. Failure will - /// shutdown the server with an error. - async fn start(self: Arc) -> Result<()> { Ok(()) } + /// Implement the service's worker loop. The service manager spawns a + /// task and calls this function after all services have been built. + async fn worker(self: Arc) -> Result<()> { Ok(()) } - /// Stop the service. Implement the joining of any service workers and - /// cleanup of any other state. This function is asynchronous to allow that - /// gracefully, but errors cannot propagate. - async fn stop(&self) {} - - /// Interrupt the service. This may be sent prior to `stop()` as a - /// notification to improve the shutdown sequence. Implementations must be - /// robust to this being called multiple times. + /// Interrupt the service. This is sent to initiate a graceful shutdown. + /// The service worker should return from its work loop. fn interrupt(&self) {} /// Clear any caches or similar runtime state. diff --git a/src/service/services.rs b/src/service/services.rs index aeed8204..13689008 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,7 +1,13 @@ -use std::{collections::BTreeMap, fmt::Write, sync::Arc}; +use std::{collections::BTreeMap, fmt::Write, panic::AssertUnwindSafe, sync::Arc, time::Duration}; -use conduit::{debug, debug_info, info, trace, Result, Server}; +use conduit::{debug, debug_info, error, info, trace, utils::time, warn, Error, Result, Server}; use database::Database; +use futures_util::FutureExt; +use tokio::{ + sync::{Mutex, MutexGuard}, + task::{JoinHandle, JoinSet}, + time::sleep, +}; use crate::{ account_data, admin, appservice, globals, key_backups, media, presence, pusher, rooms, sending, @@ -24,11 +30,19 @@ pub struct Services { pub media: Arc, pub sending: Arc, + workers: Mutex, + manager: Mutex>>>, pub(crate) service: Map, pub server: Arc, pub db: Arc, } +type Workers = JoinSet; +type WorkerResult = (Arc, Result<()>); +type WorkersLocked<'a> = MutexGuard<'a, Workers>; + +const RESTART_DELAY_MS: u64 = 2500; + impl Services { pub fn build(server: Arc, db: Arc) -> Result { let mut service: Map = BTreeMap::new(); @@ -79,12 +93,74 @@ impl Services { media: build!(media::Service), sending: build!(sending::Service), globals: build!(globals::Service), + workers: Mutex::new(JoinSet::new()), + manager: Mutex::new(None), service, server, db, }) } + pub async fn start(&self) -> Result<()> { + debug_info!("Starting services..."); + + self.media.create_media_dir().await?; + globals::migrations::migrations(&self.db, &self.globals.config).await?; + globals::emerg_access::init_emergency_access(); + + let mut workers = self.workers.lock().await; + for service in self.service.values() { + self.start_worker(&mut workers, service).await?; + } + + debug!("Starting service manager..."); + let manager = async move { crate::services().manager().await }; + let manager = self.server.runtime().spawn(manager); + _ = self.manager.lock().await.insert(manager); + + if self.globals.allow_check_for_updates() { + let handle = globals::updates::start_check_for_updates_task(); + _ = self.globals.updates_handle.lock().await.insert(handle); + } + + debug_info!("Services startup complete."); + Ok(()) + } + + pub async fn stop(&self) { + info!("Shutting down services..."); + self.interrupt(); + + debug!("Waiting for update worker..."); + if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { + updates_handle.abort(); + _ = updates_handle.await; + } + + debug!("Stopping service manager..."); + if let Some(manager) = self.manager.lock().await.take() { + if let Err(e) = manager.await { + error!("Manager shutdown error: {e:?}"); + } + } + + debug_info!("Services shutdown complete."); + } + + pub async fn clear_cache(&self) { + for service in self.service.values() { + service.clear_cache(); + } + + //TODO + self.rooms + .spaces + .roomid_spacehierarchy_cache + .lock() + .await + .clear(); + } + pub async fn memory_usage(&self) -> Result { let mut out = String::new(); for service in self.service.values() { @@ -104,65 +180,97 @@ impl Services { Ok(out) } - pub async fn clear_cache(&self) { - for service in self.service.values() { - service.clear_cache(); - } - - //TODO - self.rooms - .spaces - .roomid_spacehierarchy_cache - .lock() - .await - .clear(); - } - - pub async fn start(&self) -> Result<()> { - debug_info!("Starting services"); - - self.media.create_media_dir().await?; - globals::migrations::migrations(&self.db, &self.globals.config).await?; - globals::emerg_access::init_emergency_access(); - - for (name, service) in &self.service { - debug!("Starting {name}"); - service.clone().start().await?; - } - - if self.globals.allow_check_for_updates() { - let handle = globals::updates::start_check_for_updates_task(); - _ = self.globals.updates_handle.lock().await.insert(handle); - } - - debug_info!("Services startup complete."); - Ok(()) - } - - pub fn interrupt(&self) { + fn interrupt(&self) { debug!("Interrupting services..."); - for (name, service) in &self.service { trace!("Interrupting {name}"); service.interrupt(); } } - pub async fn stop(&self) { - info!("Shutting down services"); - self.interrupt(); - - debug!("Waiting for update worker..."); - if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { - updates_handle.abort(); - _ = updates_handle.await; + async fn manager(&self) -> Result<()> { + loop { + let mut workers = self.workers.lock().await; + tokio::select! { + result = workers.join_next() => match result { + Some(Ok(result)) => self.handle_result(&mut workers, result).await?, + Some(Err(error)) => self.handle_abort(&mut workers, Error::from(error)).await?, + None => break, + } + } } - for (name, service) in &self.service { - debug!("Waiting for {name} ..."); - service.stop().await; + debug!("Worker manager finished"); + Ok(()) + } + + async fn handle_abort(&self, _workers: &mut WorkersLocked<'_>, error: Error) -> Result<()> { + // not supported until service can be associated with abort + unimplemented!("unexpected worker task abort {error:?}"); + } + + async fn handle_result(&self, workers: &mut WorkersLocked<'_>, result: WorkerResult) -> Result<()> { + let (service, result) = result; + match result { + Ok(()) => self.handle_finished(workers, &service).await, + Err(error) => self.handle_error(workers, &service, error).await, + } + } + + async fn handle_finished(&self, _workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + debug!("service {:?} worker finished", service.name()); + Ok(()) + } + + async fn handle_error( + &self, workers: &mut WorkersLocked<'_>, service: &Arc, error: Error, + ) -> Result<()> { + let name = service.name(); + error!("service {name:?} worker error: {error}"); + + if !error.is_panic() { + return Ok(()); } - debug_info!("Services shutdown complete."); + if !self.server.running() { + return Ok(()); + } + + let delay = Duration::from_millis(RESTART_DELAY_MS); + warn!("service {name:?} worker restarting after {} delay", time::pretty(delay)); + sleep(delay).await; + + self.start_worker(workers, service).await + } + + /// Start the worker in a task for the service. + async fn start_worker(&self, workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + if !self.server.running() { + return Err(Error::Err(format!( + "Service {:?} worker not starting during server shutdown.", + service.name() + ))); + } + + debug!("Service {:?} worker starting...", service.name()); + workers.spawn_on(worker(service.clone()), self.server.runtime()); + + Ok(()) } } + +/// Base frame for service worker. This runs in a tokio::task. All errors and +/// panics from the worker are caught and returned cleanly. The JoinHandle +/// should never error with a panic, and if so it should propagate, but it may +/// error with an Abort which the manager should handle along with results to +/// determine if the worker should be restarted. +async fn worker(service: Arc) -> WorkerResult { + let service_ = Arc::clone(&service); + let result = AssertUnwindSafe(service_.worker()) + .catch_unwind() + .await + .map_err(Error::from_panic); + + // flattens JoinError for panic into worker's Error + (service, result.unwrap_or_else(Err)) +} From 93ec4e579bd9fc3c26216e659a673f9ff64d67bf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 12 Jul 2024 07:37:46 +0000 Subject: [PATCH 0473/2091] error macro suite Signed-off-by: Jason Volk --- src/core/error.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/core/error.rs b/src/core/error.rs index 8567f303..a9f44dc9 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -15,6 +15,48 @@ use ruma::{ use crate::{debug::panic_str, debug_error, error}; +#[macro_export] +macro_rules! err { + (error!($($args:tt),+)) => {{ + error!($($args),+); + $crate::error::Error::Err(format!($($args),+)) + }}; + + (debug_error!($($args:tt),+)) => {{ + debug_error!($($args),+); + $crate::error::Error::Err(format!($($args),+)) + }}; + + ($variant:ident(error!($($args:tt),+))) => {{ + error!($($args),+); + $crate::error::Error::$variant(format!($($args),+)) + }}; + + ($variant:ident(debug_error!($($args:tt),+))) => {{ + debug_error!($($args),+); + $crate::error::Error::$variant(format!($($args),+)) + }}; + + ($variant:ident(format!($($args:tt),+))) => { + $crate::error::Error::$variant(format!($($args),+)) + }; + + ($variant:ident($($args:tt),+)) => { + $crate::error::Error::$variant(format!($($args),+)) + }; + + ($string:literal$(,)? $($args:tt),*) => { + $crate::error::Error::Err(format!($string, $($args),*)) + }; +} + +#[macro_export] +macro_rules! Err { + ($($args:tt)*) => { + Err($crate::err!($($args)*)) + }; +} + #[derive(thiserror::Error)] pub enum Error { #[error("PANIC!")] From 4cc92dd17571329bd138f2be0552db26b7026055 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 12 Jul 2024 07:41:01 +0000 Subject: [PATCH 0474/2091] refactor Error::bad_config Signed-off-by: Jason Volk --- src/api/router/auth.rs | 3 +- src/api/routes.rs | 4 +-- src/core/config/check.rs | 52 ++++++++++++++++++++------------- src/core/config/mod.rs | 6 ++-- src/core/error.rs | 36 ++++++++++------------- src/main/tracing.rs | 19 ++++++------ src/service/globals/resolver.rs | 7 ++--- src/service/rooms/alias/mod.rs | 4 +-- src/service/sending/resolve.rs | 3 +- src/service/sending/send.rs | 3 +- 10 files changed, 71 insertions(+), 66 deletions(-) diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 08a08e08..6c2922b9 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -6,6 +6,7 @@ use axum_extra::{ typed_header::TypedHeaderRejectionReason, TypedHeader, }; +use conduit::Err; use http::uri::PathAndQuery; use ruma::{ api::{client::error::ErrorKind, AuthScheme, Metadata}, @@ -183,7 +184,7 @@ fn auth_appservice(request: &Request, info: Box) -> Result) -> Result { if !services().globals.allow_federation() { - return Err(Error::bad_config("Federation is disabled.")); + return Err!(Config("allow_federation", "Federation is disabled.")); } let TypedHeader(Authorization(x_matrix)) = request diff --git a/src/api/routes.rs b/src/api/routes.rs index b22a32cb..3a8b2c74 100644 --- a/src/api/routes.rs +++ b/src/api/routes.rs @@ -3,7 +3,7 @@ use axum::{ routing::{any, get, post}, Router, }; -use conduit::{Error, Server}; +use conduit::{err, Error, Server}; use http::Uri; use ruma::api::client::error::ErrorKind; @@ -236,4 +236,4 @@ async fn initial_sync(_uri: Uri) -> impl IntoResponse { Error::BadRequest(ErrorKind::GuestAccessForbidden, "Guest access not implemented") } -async fn federation_disabled() -> impl IntoResponse { Error::bad_config("Federation is disabled.") } +async fn federation_disabled() -> impl IntoResponse { err!(Config("allow_federation", "Federation is disabled.")) } diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 95b4df4e..67d19c72 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -1,6 +1,9 @@ -use crate::{error::Error, warn, Config}; +use crate::{error, error::Error, info, warn, Config, Err}; pub fn check(config: &Config) -> Result<(), Error> { + #[cfg(debug_assertions)] + info!("Note: conduwuit was built without optimisations (i.e. debug build)"); + #[cfg(all(feature = "rocksdb", not(feature = "sha256_media")))] // prevents catching this in `--all-features` warn!( "Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \ @@ -16,7 +19,7 @@ pub fn check(config: &Config) -> Result<(), Error> { config.warn_unknown_key(); if config.sentry && config.sentry_endpoint.is_none() { - return Err(Error::bad_config("Sentry cannot be enabled without an endpoint set")); + return Err!(Config("sentry_endpoint", "Sentry cannot be enabled without an endpoint set")); } #[cfg(all(feature = "hardened_malloc", feature = "jemalloc"))] @@ -27,7 +30,8 @@ pub fn check(config: &Config) -> Result<(), Error> { #[cfg(not(unix))] if config.unix_socket_path.is_some() { - return Err(Error::bad_config( + return Err!(Config( + "unix_socket_path", "UNIX socket support is only available on *nix platforms. Please remove \"unix_socket_path\" from your \ config.", )); @@ -44,7 +48,7 @@ pub fn check(config: &Config) -> Result<(), Error> { if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() /* Host */ { - crate::error!( + error!( "You are detected using OpenVZ with a loopback/localhost listening address of {addr}. If you \ are using OpenVZ for containers and you use NAT-based networking to communicate with the \ host and guest, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, \ @@ -53,7 +57,7 @@ pub fn check(config: &Config) -> Result<(), Error> { } if Path::new("/.dockerenv").exists() { - crate::error!( + error!( "You are detected using Docker with a loopback/localhost listening address of {addr}. If you \ are using a reverse proxy on the host and require communication to conduwuit in the Docker \ container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ @@ -62,7 +66,7 @@ pub fn check(config: &Config) -> Result<(), Error> { } if Path::new("/run/.containerenv").exists() { - crate::error!( + error!( "You are detected using Podman with a loopback/localhost listening address of {addr}. If you \ are using a reverse proxy on the host and require communication to conduwuit in the Podman \ container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ @@ -75,36 +79,40 @@ pub fn check(config: &Config) -> Result<(), Error> { // rocksdb does not allow max_log_files to be 0 if config.rocksdb_max_log_files == 0 { - return Err(Error::bad_config( - "rocksdb_max_log_files cannot be 0. Please set a value at least 1.", + return Err!(Config( + "max_log_files", + "rocksdb_max_log_files cannot be 0. Please set a value at least 1." )); } // yeah, unless the user built a debug build hopefully for local testing only #[cfg(not(debug_assertions))] if config.server_name == "your.server.name" { - return Err(Error::bad_config( - "You must specify a valid server name for production usage of conduwuit.", + return Err!(Config( + "server_name", + "You must specify a valid server name for production usage of conduwuit." )); } - #[cfg(debug_assertions)] - crate::info!("Note: conduwuit was built without optimisations (i.e. debug build)"); - // check if the user specified a registration token as `""` if config.registration_token == Some(String::new()) { - return Err(Error::bad_config("Registration token was specified but is empty (\"\")")); + return Err!(Config( + "registration_token", + "Registration token was specified but is empty (\"\")" + )); } if config.max_request_size < 5_120_000 { - return Err(Error::bad_config("Max request size is less than 5MB. Please increase it.")); + return Err!(Config( + "max_request_size", + "Max request size is less than 5MB. Please increase it." + )); } // check if user specified valid IP CIDR ranges on startup for cidr in &config.ip_range_denylist { if let Err(e) = ipaddress::IPAddress::parse(cidr) { - crate::error!("Error parsing specified IP CIDR range from string: {e}"); - return Err(Error::bad_config("Error parsing specified IP CIDR ranges from strings")); + return Err!(Config("ip_range_denylist", "Parsing specified IP CIDR range from string: {e}.")); } } @@ -112,13 +120,14 @@ pub fn check(config: &Config) -> Result<(), Error> { && !config.yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse && config.registration_token.is_none() { - return Err(Error::bad_config( + return Err!(Config( + "registration_token", "!! You have `allow_registration` enabled without a token configured in your config which means you are \ allowing ANYONE to register on your conduwuit instance without any 2nd-step (e.g. registration token).\n If this is not the intended behaviour, please set a registration token with the `registration_token` config option.\n For security and safety reasons, conduwuit will shut down. If you are extra sure this is the desired behaviour you \ want, please set the following config option to true: -`yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`", +`yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`" )); } @@ -135,8 +144,9 @@ For security and safety reasons, conduwuit will shut down. If you are extra sure } if config.allow_outgoing_presence && !config.allow_local_presence { - return Err(Error::bad_config( - "Outgoing presence requires allowing local presence. Please enable \"allow_local_presence\".", + return Err!(Config( + "allow_local_presence", + "Outgoing presence requires allowing local presence. Please enable 'allow_local_presence'." )); } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index aadfe564..d12e7780 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -24,7 +24,7 @@ use url::Url; pub use self::check::check; use self::proxy::ProxyConfig; -use crate::error::Error; +use crate::{error::Error, Err}; pub mod check; pub mod proxy; @@ -433,13 +433,13 @@ impl Config { }; let config = match raw_config.extract::() { - Err(e) => return Err(Error::BadConfig(format!("{e}"))), + Err(e) => return Err!("There was a problem with your configuration file: {e}"), Ok(config) => config, }; // don't start if we're listening on both UNIX sockets and TCP at same time if Self::is_dual_listening(&raw_config) { - return Err(Error::bad_config("dual listening on UNIX and TCP sockets not allowed.")); + return Err!(Config("address", "dual listening on UNIX and TCP sockets not allowed.")); }; Ok(config) diff --git a/src/core/error.rs b/src/core/error.rs index a9f44dc9..77faae3a 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -18,35 +18,36 @@ use crate::{debug::panic_str, debug_error, error}; #[macro_export] macro_rules! err { (error!($($args:tt),+)) => {{ - error!($($args),+); - $crate::error::Error::Err(format!($($args),+)) + $crate::error!($($args),+); + $crate::error::Error::Err(std::format!($($args),+)) }}; (debug_error!($($args:tt),+)) => {{ - debug_error!($($args),+); - $crate::error::Error::Err(format!($($args),+)) + $crate::debug_error!($($args),+); + $crate::error::Error::Err(std::format!($($args),+)) }}; ($variant:ident(error!($($args:tt),+))) => {{ - error!($($args),+); - $crate::error::Error::$variant(format!($($args),+)) + $crate::error!($($args),+); + $crate::error::Error::$variant(std::format!($($args),+)) }}; ($variant:ident(debug_error!($($args:tt),+))) => {{ - debug_error!($($args),+); - $crate::error::Error::$variant(format!($($args),+)) + $crate::debug_error!($($args),+); + $crate::error::Error::$variant(std::format!($($args),+)) }}; - ($variant:ident(format!($($args:tt),+))) => { - $crate::error::Error::$variant(format!($($args),+)) - }; + (Config($item:literal, $($args:tt),+)) => {{ + $crate::error!(config = %$item, $($args),+); + $crate::error::Error::Config($item, std::format!($($args),+)) + }}; ($variant:ident($($args:tt),+)) => { - $crate::error::Error::$variant(format!($($args),+)) + $crate::error::Error::$variant(std::format!($($args),+)) }; ($string:literal$(,)? $($args:tt),*) => { - $crate::error::Error::Err(format!($string, $($args),*)) + $crate::error::Error::Err(std::format!($string, $($args),*)) }; } @@ -123,8 +124,8 @@ pub enum Error { // conduwuit #[error("Arithmetic operation failed: {0}")] Arithmetic(&'static str), - #[error("There was a problem with your configuration: {0}")] - BadConfig(String), + #[error("There was a problem with the '{0}' directive in your configuration: {1}")] + Config(&'static str, String), #[error("{0}")] BadDatabase(&'static str), #[error("{0}")] @@ -145,11 +146,6 @@ impl Error { Self::BadDatabase(message) } - pub fn bad_config(message: &str) -> Self { - error!("BadConfig: {}", message); - Self::BadConfig(message.to_owned()) - } - #[must_use] pub fn from_panic(e: Box) -> Self { Self::Panic(panic_str(&e), e) } diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 67977088..0217f38a 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -2,9 +2,9 @@ use std::sync::Arc; use conduit::{ config::Config, - debug_warn, + debug_warn, err, log::{capture, LogLevelReloadHandles}, - Error, Result, + Result, }; use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; @@ -17,8 +17,7 @@ pub(crate) type TracingFlameGuard = (); pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFlameGuard, Arc)> { let reload_handles = LogLevelReloadHandles::default(); - let console_filter = - EnvFilter::try_new(&config.log).map_err(|e| Error::BadConfig(format!("in the 'log' setting: {e}.")))?; + let console_filter = EnvFilter::try_new(&config.log).map_err(|e| err!(Config("log", "{e}.")))?; let console_layer = tracing_subscriber::fmt::Layer::new(); let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); reload_handles.add("console", Box::new(console_reload_handle)); @@ -32,8 +31,8 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla #[cfg(feature = "sentry_telemetry")] let subscriber = { - let sentry_filter = EnvFilter::try_new(&config.sentry_filter) - .map_err(|e| Error::BadConfig(format!("in the 'sentry_filter' setting: {e}.")))?; + let sentry_filter = + EnvFilter::try_new(&config.sentry_filter).map_err(|e| err!(Config("sentry_filter", "{e}.")))?; let sentry_layer = sentry_tracing::layer(); let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(sentry_filter); reload_handles.add("sentry", Box::new(sentry_reload_handle)); @@ -44,9 +43,9 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla let (subscriber, flame_guard) = { let (flame_layer, flame_guard) = if config.tracing_flame { let flame_filter = EnvFilter::try_new(&config.tracing_flame_filter) - .map_err(|e| Error::BadConfig(format!("in the 'tracing_flame_filter' setting: {e}.")))?; + .map_err(|e| err!(Config("tracing_flame_filter", "{e}.")))?; let (flame_layer, flame_guard) = tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) - .map_err(|e| Error::BadConfig(format!("in the 'tracing_flame_output_path' setting: {e}.")))?; + .map_err(|e| err!(Config("tracing_flame_output_path", "{e}.")))?; let flame_layer = flame_layer .with_empty_samples(false) .with_filter(flame_filter); @@ -55,8 +54,8 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla (None, None) }; - let jaeger_filter = EnvFilter::try_new(&config.jaeger_filter) - .map_err(|e| Error::BadConfig(format!("in the 'jaeger_filter' setting: {e}.")))?; + let jaeger_filter = + EnvFilter::try_new(&config.jaeger_filter).map_err(|e| err!(Config("jaeger_filter", "{e}.")))?; let jaeger_layer = config.allow_jaeger.then(|| { opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); let tracer = opentelemetry_jaeger::new_agent_pipeline() diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs index 3082f2fd..3002decf 100644 --- a/src/service/globals/resolver.rs +++ b/src/service/globals/resolver.rs @@ -7,7 +7,7 @@ use std::{ time::Duration, }; -use conduit::{error, Config, Error, Result}; +use conduit::{error, Config, Result}; use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use ruma::OwnedServerName; @@ -33,10 +33,7 @@ impl Resolver { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] pub(super) fn new(config: &Config) -> Self { let (sys_conf, mut opts) = hickory_resolver::system_conf::read_system_conf() - .map_err(|e| { - error!("Failed to set up hickory dns resolver with system config: {e}"); - Error::bad_config("Failed to set up hickory dns resolver with system config.") - }) + .inspect_err(|e| error!("Failed to set up hickory dns resolver with system config: {e}")) .expect("DNS system config must be valid"); let mut conf = hickory_resolver::config::ResolverConfig::new(); diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 4af1035e..97eb0f48 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -3,7 +3,7 @@ mod remote; use std::sync::Arc; -use conduit::{Error, Result}; +use conduit::{err, Error, Result}; use data::Data; use ruma::{ api::{appservice, client::error::ErrorKind}, @@ -171,7 +171,7 @@ impl Service { .rooms .alias .resolve_local_alias(room_alias)? - .ok_or_else(|| Error::bad_config("Room does not exist."))?, + .ok_or_else(|| err!(BadConfig("Room does not exist.")))?, )); } } diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index d38509ba..91041a2f 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -5,6 +5,7 @@ use std::{ time::SystemTime, }; +use conduit::Err; use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; use ruma::{OwnedServerName, ServerName}; @@ -354,7 +355,7 @@ fn handle_resolve_error(e: &ResolveError) -> Result<()> { fn validate_dest(dest: &ServerName) -> Result<()> { if dest == services().globals.server_name() { - return Err(Error::bad_config("Won't send federation request to ourselves")); + return Err!("Won't send federation request to ourselves"); } if dest.is_ip_literal() || IPAddress::is_valid(dest.host()) { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index f4825ff3..18a98828 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,5 +1,6 @@ use std::{fmt::Debug, mem}; +use conduit::Err; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; use reqwest::{Client, Method, Request, Response, Url}; @@ -26,7 +27,7 @@ where T: OutgoingRequest + Debug + Send, { if !services().globals.allow_federation() { - return Err(Error::bad_config("Federation is disabled.")); + return Err!(Config("allow_federation", "Federation is disabled.")); } let actual = resolve::get_actual_dest(dest).await?; From 4600c7f32de2683c465d3dcfaf4ddac8a7c250c6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 12 Jul 2024 20:13:55 +0000 Subject: [PATCH 0475/2091] move infallible handling into error Signed-off-by: Jason Volk --- src/core/error.rs | 12 +++++++++++- src/core/utils/mod.rs | 13 ------------- src/router/serve/unix.rs | 12 ++++++++---- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/core/error.rs b/src/core/error.rs index 77faae3a..01460217 100644 --- a/src/core/error.rs +++ b/src/core/error.rs @@ -196,7 +196,11 @@ impl UnwindSafe for Error {} impl RefUnwindSafe for Error {} impl From for Error { - fn from(i: Infallible) -> Self { match i {} } + #[cold] + #[inline(never)] + fn from(_i: Infallible) -> Self { + panic!("infallible error should never exist"); + } } impl fmt::Debug for Error { @@ -273,6 +277,12 @@ pub fn inspect_debug_log(error: &E) { debug_error!("{error:?}"); } +#[cold] +#[inline(never)] +pub fn infallible(_e: &Infallible) { + panic!("infallible error should never exist"); +} + impl axum::response::IntoResponse for Error { fn into_response(self) -> axum::response::Response { let response: UiaaResponse = self.into(); diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 4e5ba480..bbd52829 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -26,21 +26,8 @@ pub use string::{str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; pub use time::now_millis as millis_since_unix_epoch; -use crate::Result; - pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), max) } -/// Boilerplate for wraps which are typed to never error. -/// -/// * -#[must_use] -pub fn unwrap_infallible(result: Result) -> T { - match result { - Ok(val) => val, - Err(err) => match err {}, - } -} - #[must_use] pub fn generate_keypair() -> Vec { let mut value = rand::string(8).as_bytes().to_vec(); diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 8373b749..e876b2ac 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -10,7 +10,7 @@ use axum::{ extract::{connect_info::IntoMakeServiceWithConnectInfo, Request}, Router, }; -use conduit::{debug_error, trace, utils, Error, Result, Server}; +use conduit::{debug_error, error::infallible, trace, Error, Result, Server}; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, @@ -24,7 +24,6 @@ use tokio::{ }; use tower::{Service, ServiceExt}; use tracing::{debug, info, warn}; -use utils::unwrap_infallible; type MakeService = IntoMakeServiceWithConnectInfo; @@ -62,9 +61,14 @@ async fn accept( let socket = TokioIo::new(socket); trace!(?listener, ?socket, ?remote, "accepted"); - let called = unwrap_infallible(app.call(NULL_ADDR).await); - let handler = service_fn(move |req: Request| called.clone().oneshot(req)); + let called = app + .call(NULL_ADDR) + .await + .inspect_err(infallible) + .expect("infallible"); + let service = move |req: Request| called.clone().oneshot(req); + let handler = service_fn(service); let task = async move { // bug on darwin causes all results to be errors. do not unwrap this _ = builder.serve_connection(socket, handler).await; From 4a6f089b23fccb5279a200546e81937199ad56f3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 01:24:37 +0000 Subject: [PATCH 0476/2091] move some config checks into check unit Signed-off-by: Jason Volk --- src/core/config/check.rs | 61 ++++++++++++++++++++++++++-- src/core/config/mod.rs | 88 ++++++++-------------------------------- 2 files changed, 74 insertions(+), 75 deletions(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 67d19c72..20a61b70 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -1,6 +1,10 @@ -use crate::{error, error::Error, info, warn, Config, Err}; +use figment::Figment; -pub fn check(config: &Config) -> Result<(), Error> { +use super::DEPRECATED_KEYS; +use crate::{debug, error, info, warn, Config, Err, Result}; + +#[allow(clippy::cognitive_complexity)] +pub fn check(config: &Config) -> Result<()> { #[cfg(debug_assertions)] info!("Note: conduwuit was built without optimisations (i.e. debug build)"); @@ -15,8 +19,8 @@ pub fn check(config: &Config) -> Result<(), Error> { forwards-compatible way. Please update your build script to remove this feature." ); - config.warn_deprecated(); - config.warn_unknown_key(); + warn_deprecated(config); + warn_unknown_key(config); if config.sentry && config.sentry_endpoint.is_none() { return Err!(Config("sentry_endpoint", "Sentry cannot be enabled without an endpoint set")); @@ -183,3 +187,52 @@ For security and safety reasons, conduwuit will shut down. If you are extra sure Ok(()) } + +/// Iterates over all the keys in the config file and warns if there is a +/// deprecated key specified +fn warn_deprecated(config: &Config) { + debug!("Checking for deprecated config keys"); + let mut was_deprecated = false; + for key in config + .catchall + .keys() + .filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key)) + { + warn!("Config parameter \"{}\" is deprecated, ignoring.", key); + was_deprecated = true; + } + + if was_deprecated { + warn!( + "Read conduwuit config documentation at https://conduwuit.puppyirl.gay/configuration.html and check your \ + configuration if any new configuration parameters should be adjusted" + ); + } +} + +/// iterates over all the catchall keys (unknown config options) and warns +/// if there are any. +fn warn_unknown_key(config: &Config) { + debug!("Checking for unknown config keys"); + for key in config + .catchall + .keys() + .filter(|key| "config".to_owned().ne(key.to_owned()) /* "config" is expected */) + { + warn!("Config parameter \"{}\" is unknown to conduwuit, ignoring.", key); + } +} + +/// Checks the presence of the `address` and `unix_socket_path` keys in the +/// raw_config, exiting the process if both keys were detected. +pub(super) fn is_dual_listening(raw_config: &Figment) -> Result<()> { + let contains_address = raw_config.contains("address"); + let contains_unix_socket = raw_config.contains("unix_socket_path"); + if contains_address && contains_unix_socket { + return Err!( + "TOML keys \"address\" and \"unix_socket_path\" were both defined. Please specify only one option." + ); + } + + Ok(()) +} diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index d12e7780..16c7b1a8 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -19,30 +19,15 @@ use ruma::{ api::client::discovery::discover_support::ContactRole, OwnedRoomId, OwnedServerName, OwnedUserId, RoomVersionId, }; use serde::{de::IgnoredAny, Deserialize}; -use tracing::{debug, error, warn}; use url::Url; pub use self::check::check; use self::proxy::ProxyConfig; -use crate::{error::Error, Err}; +use crate::{error::Error, Err, Result}; pub mod check; pub mod proxy; -#[derive(Deserialize, Clone, Debug)] -#[serde(transparent)] -struct ListeningPort { - #[serde(with = "either::serde_untagged")] - ports: Either>, -} - -#[derive(Deserialize, Clone, Debug)] -#[serde(transparent)] -struct ListeningAddr { - #[serde(with = "either::serde_untagged")] - addrs: Either>, -} - /// all the config options for conduwuit #[derive(Clone, Debug, Deserialize)] #[allow(clippy::struct_excessive_bools)] @@ -397,6 +382,20 @@ pub struct WellKnownConfig { pub support_mxid: Option, } +#[derive(Deserialize, Clone, Debug)] +#[serde(transparent)] +struct ListeningPort { + #[serde(with = "either::serde_untagged")] + ports: Either>, +} + +#[derive(Deserialize, Clone, Debug)] +#[serde(transparent)] +struct ListeningAddr { + #[serde(with = "either::serde_untagged")] + addrs: Either>, +} + const DEPRECATED_KEYS: &[&str] = &[ "cache_capacity", "max_concurrent_requests", @@ -410,7 +409,7 @@ const DEPRECATED_KEYS: &[&str] = &[ impl Config { /// Initialize config - pub fn new(path: Option) -> Result { + pub fn new(path: Option) -> Result { let raw_config = if let Some(config_file_env) = Env::var("CONDUIT_CONFIG") { Figment::new() .merge(Toml::file(config_file_env).nested()) @@ -438,64 +437,11 @@ impl Config { }; // don't start if we're listening on both UNIX sockets and TCP at same time - if Self::is_dual_listening(&raw_config) { - return Err!(Config("address", "dual listening on UNIX and TCP sockets not allowed.")); - }; + check::is_dual_listening(&raw_config)?; Ok(config) } - /// Iterates over all the keys in the config file and warns if there is a - /// deprecated key specified - pub(crate) fn warn_deprecated(&self) { - debug!("Checking for deprecated config keys"); - let mut was_deprecated = false; - for key in self - .catchall - .keys() - .filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key)) - { - warn!("Config parameter \"{}\" is deprecated, ignoring.", key); - was_deprecated = true; - } - - if was_deprecated { - warn!( - "Read conduwuit config documentation at https://conduwuit.puppyirl.gay/configuration.html and check \ - your configuration if any new configuration parameters should be adjusted" - ); - } - } - - /// iterates over all the catchall keys (unknown config options) and warns - /// if there are any. - pub(crate) fn warn_unknown_key(&self) { - debug!("Checking for unknown config keys"); - for key in self - .catchall - .keys() - .filter(|key| "config".to_owned().ne(key.to_owned()) /* "config" is expected */) - { - warn!("Config parameter \"{}\" is unknown to conduwuit, ignoring.", key); - } - } - - /// Checks the presence of the `address` and `unix_socket_path` keys in the - /// raw_config, exiting the process if both keys were detected. - fn is_dual_listening(raw_config: &Figment) -> bool { - let check_address = raw_config.find_value("address"); - let check_unix_socket = raw_config.find_value("unix_socket_path"); - - // are the check_address and check_unix_socket keys both Ok (specified) at the - // same time? - if check_address.is_ok() && check_unix_socket.is_ok() { - error!("TOML keys \"address\" and \"unix_socket_path\" were both defined. Please specify only one option."); - return true; - } - - false - } - #[must_use] pub fn get_bind_addrs(&self) -> Vec { let mut addrs = Vec::new(); From b87f1649d4e1f2e212188e93c58d8d52fbf1ad84 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 02:15:15 +0000 Subject: [PATCH 0477/2091] split error into directory Signed-off-by: Jason Volk --- src/api/client/membership.rs | 10 +- src/core/error.rs | 384 ----------------------------------- src/core/error/err.rs | 42 ++++ src/core/error/log.rs | 74 +++++++ src/core/error/mod.rs | 156 ++++++++++++++ src/core/error/panic.rs | 41 ++++ src/core/error/response.rs | 88 ++++++++ 7 files changed, 404 insertions(+), 391 deletions(-) delete mode 100644 src/core/error.rs create mode 100644 src/core/error/err.rs create mode 100644 src/core/error/log.rs create mode 100644 src/core/error/mod.rs create mode 100644 src/core/error/panic.rs create mode 100644 src/core/error/response.rs diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 1b70fdec..c25c08d1 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1273,16 +1273,12 @@ async fn make_join_request( make_join_counter = make_join_counter.saturating_add(1); if let Err(ref e) = make_join_response { - trace!("make_join ErrorKind string: {:?}", e.error_code().to_string()); + trace!("make_join ErrorKind string: {:?}", e.kind().to_string()); // converting to a string is necessary (i think) because ruma is forcing us to // fill in the struct for M_INCOMPATIBLE_ROOM_VERSION - if e.error_code() - .to_string() - .contains("M_INCOMPATIBLE_ROOM_VERSION") - || e.error_code() - .to_string() - .contains("M_UNSUPPORTED_ROOM_VERSION") + if e.kind().to_string().contains("M_INCOMPATIBLE_ROOM_VERSION") + || e.kind().to_string().contains("M_UNSUPPORTED_ROOM_VERSION") { incompatible_room_version_count = incompatible_room_version_count.saturating_add(1); } diff --git a/src/core/error.rs b/src/core/error.rs deleted file mode 100644 index 01460217..00000000 --- a/src/core/error.rs +++ /dev/null @@ -1,384 +0,0 @@ -use std::{ - any::Any, - convert::Infallible, - fmt, - panic::{RefUnwindSafe, UnwindSafe}, -}; - -use bytes::BytesMut; -use http::StatusCode; -use http_body_util::Full; -use ruma::{ - api::{client::uiaa::UiaaResponse, OutgoingResponse}, - OwnedServerName, -}; - -use crate::{debug::panic_str, debug_error, error}; - -#[macro_export] -macro_rules! err { - (error!($($args:tt),+)) => {{ - $crate::error!($($args),+); - $crate::error::Error::Err(std::format!($($args),+)) - }}; - - (debug_error!($($args:tt),+)) => {{ - $crate::debug_error!($($args),+); - $crate::error::Error::Err(std::format!($($args),+)) - }}; - - ($variant:ident(error!($($args:tt),+))) => {{ - $crate::error!($($args),+); - $crate::error::Error::$variant(std::format!($($args),+)) - }}; - - ($variant:ident(debug_error!($($args:tt),+))) => {{ - $crate::debug_error!($($args),+); - $crate::error::Error::$variant(std::format!($($args),+)) - }}; - - (Config($item:literal, $($args:tt),+)) => {{ - $crate::error!(config = %$item, $($args),+); - $crate::error::Error::Config($item, std::format!($($args),+)) - }}; - - ($variant:ident($($args:tt),+)) => { - $crate::error::Error::$variant(std::format!($($args),+)) - }; - - ($string:literal$(,)? $($args:tt),*) => { - $crate::error::Error::Err(std::format!($string, $($args),*)) - }; -} - -#[macro_export] -macro_rules! Err { - ($($args:tt)*) => { - Err($crate::err!($($args)*)) - }; -} - -#[derive(thiserror::Error)] -pub enum Error { - #[error("PANIC!")] - PanicAny(Box), - #[error("PANIC! {0}")] - Panic(&'static str, Box), - - // std - #[error("{0}")] - Fmt(#[from] fmt::Error), - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), - #[error("{0}")] - Utf8Error(#[from] std::str::Utf8Error), - #[error("{0}")] - FromUtf8Error(#[from] std::string::FromUtf8Error), - #[error("{0}")] - TryFromSliceError(#[from] std::array::TryFromSliceError), - #[error("{0}")] - TryFromIntError(#[from] std::num::TryFromIntError), - #[error("{0}")] - ParseIntError(#[from] std::num::ParseIntError), - #[error("{0}")] - ParseFloatError(#[from] std::num::ParseFloatError), - - // third-party - #[error("Join error: {0}")] - JoinError(#[from] tokio::task::JoinError), - #[error("Regex error: {0}")] - Regex(#[from] regex::Error), - #[error("Tracing filter error: {0}")] - TracingFilter(#[from] tracing_subscriber::filter::ParseError), - #[error("Tracing reload error: {0}")] - TracingReload(#[from] tracing_subscriber::reload::Error), - #[error("Image error: {0}")] - Image(#[from] image::error::ImageError), - #[error("Request error: {0}")] - Reqwest(#[from] reqwest::Error), - #[error("{0}")] - Extension(#[from] axum::extract::rejection::ExtensionRejection), - #[error("{0}")] - Path(#[from] axum::extract::rejection::PathRejection), - #[error("{0}")] - Http(#[from] http::Error), - - // ruma - #[error("{0}")] - IntoHttpError(#[from] ruma::api::error::IntoHttpError), - #[error("{0}")] - RumaError(#[from] ruma::api::client::error::Error), - #[error("uiaa")] - Uiaa(ruma::api::client::uiaa::UiaaInfo), - #[error("{0}")] - Mxid(#[from] ruma::IdParseError), - #[error("{0}: {1}")] - BadRequest(ruma::api::client::error::ErrorKind, &'static str), - #[error("from {0}: {1}")] - Redaction(OwnedServerName, ruma::canonical_json::RedactionError), - #[error("Remote server {0} responded with: {1}")] - Federation(OwnedServerName, ruma::api::client::error::Error), - #[error("{0} in {1}")] - InconsistentRoomState(&'static str, ruma::OwnedRoomId), - - // conduwuit - #[error("Arithmetic operation failed: {0}")] - Arithmetic(&'static str), - #[error("There was a problem with the '{0}' directive in your configuration: {1}")] - Config(&'static str, String), - #[error("{0}")] - BadDatabase(&'static str), - #[error("{0}")] - Database(String), - #[error("{0}")] - BadServerResponse(&'static str), - #[error("{0}")] - Conflict(&'static str), // This is only needed for when a room alias already exists - - // unique / untyped - #[error("{0}")] - Err(String), -} - -impl Error { - pub fn bad_database(message: &'static str) -> Self { - error!("BadDatabase: {}", message); - Self::BadDatabase(message) - } - - #[must_use] - pub fn from_panic(e: Box) -> Self { Self::Panic(panic_str(&e), e) } - - pub fn into_panic(self) -> Box { - match self { - Self::Panic(_, e) | Self::PanicAny(e) => e, - Self::JoinError(e) => e.into_panic(), - _ => std::panic::panic_any(self), - } - } - - /// Sanitizes public-facing errors that can leak sensitive information. - pub fn sanitized_string(&self) -> String { - match self { - Self::Database(..) => String::from("Database error occurred."), - Self::Io(..) => String::from("I/O error occurred."), - _ => self.to_string(), - } - } - - /// Get the panic message string. - pub fn panic_str(self) -> Option<&'static str> { self.is_panic().then_some(panic_str(&self.into_panic())) } - - /// Returns the Matrix error code / error kind - #[inline] - pub fn error_code(&self) -> ruma::api::client::error::ErrorKind { - use ruma::api::client::error::ErrorKind::Unknown; - - match self { - Self::Federation(_, error) => ruma_error_kind(error).clone(), - Self::BadRequest(kind, _) => kind.clone(), - _ => Unknown, - } - } - - /// Check if the Error is trafficking a panic object. - #[inline] - pub fn is_panic(&self) -> bool { - match &self { - Self::Panic(..) | Self::PanicAny(..) => true, - Self::JoinError(e) => e.is_panic(), - _ => false, - } - } -} - -impl UnwindSafe for Error {} -impl RefUnwindSafe for Error {} - -impl From for Error { - #[cold] - #[inline(never)] - fn from(_i: Infallible) -> Self { - panic!("infallible error should never exist"); - } -} - -impl fmt::Debug for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } -} - -#[inline] -pub fn else_log(error: E) -> Result -where - T: Default, - Error: From, -{ - Ok(default_log(error)) -} - -#[inline] -pub fn else_debug_log(error: E) -> Result -where - T: Default, - Error: From, -{ - Ok(default_debug_log(error)) -} - -#[inline] -pub fn default_log(error: E) -> T -where - T: Default, - Error: From, -{ - let error = Error::from(error); - inspect_log(&error); - T::default() -} - -#[inline] -pub fn default_debug_log(error: E) -> T -where - T: Default, - Error: From, -{ - let error = Error::from(error); - inspect_debug_log(&error); - T::default() -} - -#[inline] -pub fn map_log(error: E) -> Error -where - Error: From, -{ - let error = Error::from(error); - inspect_log(&error); - error -} - -#[inline] -pub fn map_debug_log(error: E) -> Error -where - Error: From, -{ - let error = Error::from(error); - inspect_debug_log(&error); - error -} - -#[inline] -pub fn inspect_log(error: &E) { - error!("{error}"); -} - -#[inline] -pub fn inspect_debug_log(error: &E) { - debug_error!("{error:?}"); -} - -#[cold] -#[inline(never)] -pub fn infallible(_e: &Infallible) { - panic!("infallible error should never exist"); -} - -impl axum::response::IntoResponse for Error { - fn into_response(self) -> axum::response::Response { - let response: UiaaResponse = self.into(); - response - .try_into_http_response::() - .inspect_err(|e| error!("error response error: {e}")) - .map_or_else( - |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), - |r| r.map(BytesMut::freeze).map(Full::new).into_response(), - ) - } -} - -impl From for UiaaResponse { - fn from(error: Error) -> Self { - if let Error::Uiaa(uiaainfo) = error { - return Self::AuthResponse(uiaainfo); - } - - let kind = match &error { - Error::Federation(_, ref error) | Error::RumaError(ref error) => ruma_error_kind(error), - Error::BadRequest(kind, _) => kind, - _ => &ruma::api::client::error::ErrorKind::Unknown, - }; - - let status_code = match &error { - Error::Federation(_, ref error) | Error::RumaError(ref error) => error.status_code, - Error::BadRequest(ref kind, _) => bad_request_code(kind), - Error::Conflict(_) => StatusCode::CONFLICT, - _ => StatusCode::INTERNAL_SERVER_ERROR, - }; - - let message = match &error { - Error::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), - Error::RumaError(ref error) => ruma_error_message(error), - _ => format!("{error}"), - }; - - let body = ruma::api::client::error::ErrorBody::Standard { - kind: kind.clone(), - message, - }; - - Self::MatrixError(ruma::api::client::error::Error { - status_code, - body, - }) - } -} - -fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { - use ruma::api::client::error::ErrorKind::*; - - match kind { - GuestAccessForbidden - | ThreepidAuthFailed - | UserDeactivated - | ThreepidDenied - | WrongRoomKeysVersion { - .. - } - | Forbidden { - .. - } => StatusCode::FORBIDDEN, - - UnknownToken { - .. - } - | MissingToken - | Unauthorized => StatusCode::UNAUTHORIZED, - - LimitExceeded { - .. - } => StatusCode::TOO_MANY_REQUESTS, - - TooLarge => StatusCode::PAYLOAD_TOO_LARGE, - - NotFound | Unrecognized => StatusCode::NOT_FOUND, - - _ => StatusCode::BAD_REQUEST, - } -} - -fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { - if let ruma::api::client::error::ErrorBody::Standard { - message, - .. - } = &error.body - { - return message.to_string(); - } - - format!("{error}") -} - -fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::error::ErrorKind { - e.error_kind() - .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) -} diff --git a/src/core/error/err.rs b/src/core/error/err.rs new file mode 100644 index 00000000..12dd4f3b --- /dev/null +++ b/src/core/error/err.rs @@ -0,0 +1,42 @@ +#[macro_export] +macro_rules! Err { + ($($args:tt)*) => { + Err($crate::err!($($args)*)) + }; +} + +#[macro_export] +macro_rules! err { + (error!($($args:tt),+)) => {{ + $crate::error!($($args),+); + $crate::error::Error::Err(std::format!($($args),+)) + }}; + + (debug_error!($($args:tt),+)) => {{ + $crate::debug_error!($($args),+); + $crate::error::Error::Err(std::format!($($args),+)) + }}; + + ($variant:ident(error!($($args:tt),+))) => {{ + $crate::error!($($args),+); + $crate::error::Error::$variant(std::format!($($args),+)) + }}; + + ($variant:ident(debug_error!($($args:tt),+))) => {{ + $crate::debug_error!($($args),+); + $crate::error::Error::$variant(std::format!($($args),+)) + }}; + + (Config($item:literal, $($args:tt),+)) => {{ + $crate::error!(config = %$item, $($args),+); + $crate::error::Error::Config($item, std::format!($($args),+)) + }}; + + ($variant:ident($($args:tt),+)) => { + $crate::error::Error::$variant(std::format!($($args),+)) + }; + + ($string:literal$(,)? $($args:tt),*) => { + $crate::error::Error::Err(std::format!($string, $($args),*)) + }; +} diff --git a/src/core/error/log.rs b/src/core/error/log.rs new file mode 100644 index 00000000..c272bf73 --- /dev/null +++ b/src/core/error/log.rs @@ -0,0 +1,74 @@ +use std::{convert::Infallible, fmt}; + +use super::Error; +use crate::{debug_error, error}; + +#[inline] +pub fn else_log(error: E) -> Result +where + T: Default, + Error: From, +{ + Ok(default_log(error)) +} + +#[inline] +pub fn else_debug_log(error: E) -> Result +where + T: Default, + Error: From, +{ + Ok(default_debug_log(error)) +} + +#[inline] +pub fn default_log(error: E) -> T +where + T: Default, + Error: From, +{ + let error = Error::from(error); + inspect_log(&error); + T::default() +} + +#[inline] +pub fn default_debug_log(error: E) -> T +where + T: Default, + Error: From, +{ + let error = Error::from(error); + inspect_debug_log(&error); + T::default() +} + +#[inline] +pub fn map_log(error: E) -> Error +where + Error: From, +{ + let error = Error::from(error); + inspect_log(&error); + error +} + +#[inline] +pub fn map_debug_log(error: E) -> Error +where + Error: From, +{ + let error = Error::from(error); + inspect_debug_log(&error); + error +} + +#[inline] +pub fn inspect_log(error: &E) { + error!("{error}"); +} + +#[inline] +pub fn inspect_debug_log(error: &E) { + debug_error!("{error:?}"); +} diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs new file mode 100644 index 00000000..cbc22f3e --- /dev/null +++ b/src/core/error/mod.rs @@ -0,0 +1,156 @@ +mod err; +mod log; +mod panic; +mod response; + +use std::{any::Any, convert::Infallible, fmt}; + +pub use log::*; + +use crate::error; + +#[derive(thiserror::Error)] +pub enum Error { + #[error("PANIC!")] + PanicAny(Box), + #[error("PANIC! {0}")] + Panic(&'static str, Box), + + // std + #[error("{0}")] + Fmt(#[from] fmt::Error), + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + #[error("{0}")] + Utf8Error(#[from] std::str::Utf8Error), + #[error("{0}")] + FromUtf8Error(#[from] std::string::FromUtf8Error), + #[error("{0}")] + TryFromSliceError(#[from] std::array::TryFromSliceError), + #[error("{0}")] + TryFromIntError(#[from] std::num::TryFromIntError), + #[error("{0}")] + ParseIntError(#[from] std::num::ParseIntError), + #[error("{0}")] + ParseFloatError(#[from] std::num::ParseFloatError), + + // third-party + #[error("Join error: {0}")] + JoinError(#[from] tokio::task::JoinError), + #[error("Regex error: {0}")] + Regex(#[from] regex::Error), + #[error("Tracing filter error: {0}")] + TracingFilter(#[from] tracing_subscriber::filter::ParseError), + #[error("Tracing reload error: {0}")] + TracingReload(#[from] tracing_subscriber::reload::Error), + #[error("Image error: {0}")] + Image(#[from] image::error::ImageError), + #[error("Request error: {0}")] + Reqwest(#[from] reqwest::Error), + #[error("{0}")] + Extension(#[from] axum::extract::rejection::ExtensionRejection), + #[error("{0}")] + Path(#[from] axum::extract::rejection::PathRejection), + #[error("{0}")] + Http(#[from] http::Error), + + // ruma + #[error("{0}")] + IntoHttpError(#[from] ruma::api::error::IntoHttpError), + #[error("{0}")] + RumaError(#[from] ruma::api::client::error::Error), + #[error("uiaa")] + Uiaa(ruma::api::client::uiaa::UiaaInfo), + #[error("{0}")] + Mxid(#[from] ruma::IdParseError), + #[error("{0}: {1}")] + BadRequest(ruma::api::client::error::ErrorKind, &'static str), + #[error("from {0}: {1}")] + Redaction(ruma::OwnedServerName, ruma::canonical_json::RedactionError), + #[error("Remote server {0} responded with: {1}")] + Federation(ruma::OwnedServerName, ruma::api::client::error::Error), + #[error("{0} in {1}")] + InconsistentRoomState(&'static str, ruma::OwnedRoomId), + + // conduwuit + #[error("Arithmetic operation failed: {0}")] + Arithmetic(&'static str), + #[error("There was a problem with the '{0}' directive in your configuration: {1}")] + Config(&'static str, String), + #[error("{0}")] + BadDatabase(&'static str), + #[error("{0}")] + Database(String), + #[error("{0}")] + BadServerResponse(&'static str), + #[error("{0}")] + Conflict(&'static str), // This is only needed for when a room alias already exists + + // unique / untyped + #[error("{0}")] + Err(String), +} + +impl Error { + pub fn bad_database(message: &'static str) -> Self { + error!("BadDatabase: {}", message); + Self::BadDatabase(message) + } + + /// Sanitizes public-facing errors that can leak sensitive information. + pub fn sanitized_string(&self) -> String { + match self { + Self::Database(..) => String::from("Database error occurred."), + Self::Io(..) => String::from("I/O error occurred."), + _ => self.to_string(), + } + } + + pub fn message(&self) -> String { + match self { + Self::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), + Self::RumaError(ref error) => response::ruma_error_message(error), + _ => format!("{self}"), + } + } + + /// Returns the Matrix error code / error kind + #[inline] + pub fn kind(&self) -> ruma::api::client::error::ErrorKind { + use ruma::api::client::error::ErrorKind::Unknown; + + match self { + Self::Federation(_, error) => response::ruma_error_kind(error).clone(), + Self::BadRequest(kind, _) => kind.clone(), + _ => Unknown, + } + } + + pub fn status_code(&self) -> http::StatusCode { + match self { + Self::Federation(_, ref error) | Self::RumaError(ref error) => error.status_code, + Self::BadRequest(ref kind, _) => response::bad_request_code(kind), + Self::Conflict(_) => http::StatusCode::CONFLICT, + _ => http::StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl fmt::Debug for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } +} + +#[allow(clippy::fallible_impl_from)] +impl From for Error { + #[cold] + #[inline(never)] + fn from(_e: Infallible) -> Self { + panic!("infallible error should never exist"); + } +} + +#[cold] +#[inline(never)] +pub fn infallible(_e: &Infallible) { + panic!("infallible error should never exist"); +} diff --git a/src/core/error/panic.rs b/src/core/error/panic.rs new file mode 100644 index 00000000..c070f786 --- /dev/null +++ b/src/core/error/panic.rs @@ -0,0 +1,41 @@ +use std::{ + any::Any, + panic::{panic_any, RefUnwindSafe, UnwindSafe}, +}; + +use super::Error; +use crate::debug; + +impl UnwindSafe for Error {} +impl RefUnwindSafe for Error {} + +impl Error { + pub fn panic(self) -> ! { panic_any(self.into_panic()) } + + #[must_use] + pub fn from_panic(e: Box) -> Self { Self::Panic(debug::panic_str(&e), e) } + + pub fn into_panic(self) -> Box { + match self { + Self::Panic(_, e) | Self::PanicAny(e) => e, + Self::JoinError(e) => e.into_panic(), + _ => Box::new(self), + } + } + + /// Get the panic message string. + pub fn panic_str(self) -> Option<&'static str> { + self.is_panic() + .then_some(debug::panic_str(&self.into_panic())) + } + + /// Check if the Error is trafficking a panic object. + #[inline] + pub fn is_panic(&self) -> bool { + match &self { + Self::Panic(..) | Self::PanicAny(..) => true, + Self::JoinError(e) => e.is_panic(), + _ => false, + } + } +} diff --git a/src/core/error/response.rs b/src/core/error/response.rs new file mode 100644 index 00000000..4ea76e26 --- /dev/null +++ b/src/core/error/response.rs @@ -0,0 +1,88 @@ +use bytes::BytesMut; +use http::StatusCode; +use http_body_util::Full; +use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; + +use super::Error; +use crate::error; + +impl axum::response::IntoResponse for Error { + fn into_response(self) -> axum::response::Response { + let response: UiaaResponse = self.into(); + response + .try_into_http_response::() + .inspect_err(|e| error!("error response error: {e}")) + .map_or_else( + |_| StatusCode::INTERNAL_SERVER_ERROR.into_response(), + |r| r.map(BytesMut::freeze).map(Full::new).into_response(), + ) + } +} + +impl From for UiaaResponse { + fn from(error: Error) -> Self { + if let Error::Uiaa(uiaainfo) = error { + return Self::AuthResponse(uiaainfo); + } + + let body = ruma::api::client::error::ErrorBody::Standard { + kind: error.kind(), + message: error.message(), + }; + + Self::MatrixError(ruma::api::client::error::Error { + status_code: error.status_code(), + body, + }) + } +} + +pub(super) fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { + use ruma::api::client::error::ErrorKind::*; + + match kind { + GuestAccessForbidden + | ThreepidAuthFailed + | UserDeactivated + | ThreepidDenied + | WrongRoomKeysVersion { + .. + } + | Forbidden { + .. + } => StatusCode::FORBIDDEN, + + UnknownToken { + .. + } + | MissingToken + | Unauthorized => StatusCode::UNAUTHORIZED, + + LimitExceeded { + .. + } => StatusCode::TOO_MANY_REQUESTS, + + TooLarge => StatusCode::PAYLOAD_TOO_LARGE, + + NotFound | Unrecognized => StatusCode::NOT_FOUND, + + _ => StatusCode::BAD_REQUEST, + } +} + +pub(super) fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { + if let ruma::api::client::error::ErrorBody::Standard { + message, + .. + } = &error.body + { + return message.to_string(); + } + + format!("{error}") +} + +pub(super) fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::error::ErrorKind { + e.error_kind() + .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) +} From b8baa1223d8083b48e1127bb29378eed32033c7e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 12 Jul 2024 01:08:53 +0000 Subject: [PATCH 0478/2091] reduce RoomVersionId patterns Signed-off-by: Jason Volk --- src/api/client/membership.rs | 11 +--- src/api/client/room.rs | 85 ++++++++++---------------- src/api/server/make_join.rs | 57 ++++++++--------- src/api/server/make_leave.rs | 26 ++------ src/service/admin/create.rs | 33 +++++----- src/service/rooms/event_handler/mod.rs | 79 +++++++++++------------- src/service/rooms/timeline/mod.rs | 27 ++------ 7 files changed, 121 insertions(+), 197 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index c25c08d1..3adee631 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -780,14 +780,9 @@ async fn join_room_by_id_helper_remote( info!("send_join finished"); if join_authorized_via_users_server.is_some() { + use RoomVersionId::*; match &room_version_id { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 => { + V1 | V2 | V3 | V4 | V5 | V6 | V7 => { warn!( "Found `join_authorised_via_users_server` but room {} is version {}. Ignoring.", room_id, &room_version_id @@ -795,7 +790,7 @@ async fn join_room_by_id_helper_remote( }, // only room versions 8 and above using `join_authorized_via_users_server` (restricted joins) need to // validate and send signatures - RoomVersionId::V8 | RoomVersionId::V9 | RoomVersionId::V10 | RoomVersionId::V11 => { + V8 | V9 | V10 | V11 => { if let Some(signed_raw) = &send_join_response.room_state.event { info!( "There is a signed event. This room is probably using restricted joins. Adding signature to \ diff --git a/src/api/client/room.rs b/src/api/client/room.rs index ccdf4dc0..adf58b04 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -118,6 +118,8 @@ pub(crate) async fn create_room_route(body: Ruma) -> R let content = match &body.creation_content { Some(content) => { + use RoomVersionId::*; + let mut content = content .deserialize_as::() .map_err(|e| { @@ -125,16 +127,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R Error::bad_database("Failed to deserialise content as canonical JSON.") })?; match room_version { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { content.insert( "creator".into(), json!(&sender_user).try_into().map_err(|e| { @@ -143,7 +136,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R })?, ); }, - RoomVersionId::V11 => {}, // V11 removed the "creator" key + V11 => {}, // V11 removed the "creator" key _ => { warn!("Unexpected or unsupported room version {room_version}"); return Err(Error::BadRequest( @@ -152,7 +145,6 @@ pub(crate) async fn create_room_route(body: Ruma) -> R )); }, } - content.insert( "room_version".into(), json!(room_version.as_str()) @@ -162,18 +154,11 @@ pub(crate) async fn create_room_route(body: Ruma) -> R content }, None => { + use RoomVersionId::*; + let content = match room_version { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => RoomCreateEventContent::new_v1(sender_user.clone()), - RoomVersionId::V11 => RoomCreateEventContent::new_v11(), + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => RoomCreateEventContent::new_v1(sender_user.clone()), + V11 => RoomCreateEventContent::new_v11(), _ => { warn!("Unexpected or unsupported room version {room_version}"); return Err(Error::BadRequest( @@ -623,36 +608,30 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Send a m.room.create event containing a predecessor field and the applicable // room_version - match body.new_version { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { - create_event_content.insert( - "creator".into(), - json!(&sender_user).try_into().map_err(|e| { - info!("Error forming creation event: {e}"); - Error::BadRequest(ErrorKind::BadJson, "Error forming creation event") - })?, - ); - }, - RoomVersionId::V11 => { - // "creator" key no longer exists in V11 rooms - create_event_content.remove("creator"); - }, - _ => { - warn!("Unexpected or unsupported room version {}", body.new_version); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, + { + use RoomVersionId::*; + match body.new_version { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + create_event_content.insert( + "creator".into(), + json!(&sender_user).try_into().map_err(|e| { + info!("Error forming creation event: {e}"); + Error::BadRequest(ErrorKind::BadJson, "Error forming creation event") + })?, + ); + }, + V11 => { + // "creator" key no longer exists in V11 rooms + create_event_content.remove("creator"); + }, + _ => { + warn!("Unexpected or unsupported room version {}", body.new_version); + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + } } create_event_content.insert( diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index c909ea33..b5dadf7f 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -7,7 +7,7 @@ use ruma::{ }, StateEventType, TimelineEventType, }, - RoomId, RoomVersionId, UserId, + CanonicalJsonObject, RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; use tracing::warn; @@ -144,27 +144,7 @@ pub(crate) async fn create_join_event_template_route( drop(state_lock); // room v3 and above removed the "event_id" field from remote PDU format - match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 - | RoomVersionId::V11 => { - pdu_json.remove("event_id"); - }, - _ => { - warn!("Unexpected or unsupported room version {room_version_id}"); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, - }; + maybe_strip_event_id(&mut pdu_json, &room_version_id)?; Ok(prepare_join_event::v1::Response { room_version: Some(room_version_id), @@ -179,6 +159,8 @@ pub(crate) async fn create_join_event_template_route( pub(crate) fn user_can_perform_restricted_join( user_id: &UserId, room_id: &RoomId, room_version_id: &RoomVersionId, ) -> Result { + use RoomVersionId::*; + let join_rules_event = services() .rooms @@ -198,16 +180,7 @@ pub(crate) fn user_can_perform_restricted_join( return Ok(false); }; - if matches!( - room_version_id, - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - ) { + if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { return Ok(false); } @@ -239,3 +212,23 @@ pub(crate) fn user_can_perform_restricted_join( )) } } + +pub(crate) fn maybe_strip_event_id(pdu_json: &mut CanonicalJsonObject, room_version_id: &RoomVersionId) -> Result<()> { + use RoomVersionId::*; + + match room_version_id { + V1 | V2 => {}, + V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 | V11 => { + pdu_json.remove("event_id"); + }, + _ => { + warn!("Unexpected or unsupported room version {room_version_id}"); + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + }; + + Ok(()) +} diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index eea3e4f8..63fc2b2e 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -1,14 +1,15 @@ +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_leave_event}, events::{ room::member::{MembershipState, RoomMemberEventContent}, TimelineEventType, }, - RoomVersionId, }; use serde_json::value::to_raw_value; -use crate::{service::pdu::PduBuilder, services, Error, Result, Ruma}; +use super::make_join::maybe_strip_event_id; +use crate::{service::pdu::PduBuilder, services, Ruma}; /// # `PUT /_matrix/federation/v1/make_leave/{roomId}/{eventId}` /// @@ -64,26 +65,7 @@ pub(crate) async fn create_leave_event_template_route( drop(state_lock); // room v3 and above removed the "event_id" field from remote PDU format - match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 - | RoomVersionId::V11 => { - pdu_json.remove("event_id"); - }, - _ => { - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, - }; + maybe_strip_event_id(&mut pdu_json, &room_version_id)?; Ok(prepare_leave_event::v1::Response { room_version: Some(room_version_id), diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 57df344a..fbb6a078 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -41,25 +41,20 @@ pub async fn create_admin_room() -> Result<()> { services().users.create(server_user, None)?; let room_version = services().globals.default_room_version(); - let mut content = match room_version { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => RoomCreateEventContent::new_v1(server_user.clone()), - RoomVersionId::V11 => RoomCreateEventContent::new_v11(), - _ => { - warn!("Unexpected or unsupported room version {}", room_version); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, + + let mut content = { + use RoomVersionId::*; + match room_version { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => RoomCreateEventContent::new_v1(server_user.clone()), + V11 => RoomCreateEventContent::new_v11(), + _ => { + warn!("Unexpected or unsupported room version {}", room_version); + return Err(Error::BadRequest( + ErrorKind::BadJson, + "Unexpected or unsupported room version found", + )); + }, + } }; content.federate = true; diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 9490a404..33c00643 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -551,49 +551,44 @@ impl Service { // Soft fail check before doing state res debug!("Performing soft-fail check"); - let soft_fail = !state_res::event_auth::auth_check(&room_version, &incoming_pdu, None::, |k, s| { - auth_events.get(&(k.clone(), s.to_owned())) - }) - .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))? - || incoming_pdu.kind == TimelineEventType::RoomRedaction - && match room_version_id { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { - if let Some(redact_id) = &incoming_pdu.redacts { - !services().rooms.state_accessor.user_can_redact( - redact_id, - &incoming_pdu.sender, - &incoming_pdu.room_id, - true, - )? - } else { - false - } - }, - _ => { - let content = serde_json::from_str::(incoming_pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; + let soft_fail = { + use RoomVersionId::*; - if let Some(redact_id) = &content.redacts { - !services().rooms.state_accessor.user_can_redact( - redact_id, - &incoming_pdu.sender, - &incoming_pdu.room_id, - true, - )? - } else { - false - } - }, - }; + !state_res::event_auth::auth_check(&room_version, &incoming_pdu, None::, |k, s| { + auth_events.get(&(k.clone(), s.to_owned())) + }) + .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))? + || incoming_pdu.kind == TimelineEventType::RoomRedaction + && match room_version_id { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + if let Some(redact_id) = &incoming_pdu.redacts { + !services().rooms.state_accessor.user_can_redact( + redact_id, + &incoming_pdu.sender, + &incoming_pdu.room_id, + true, + )? + } else { + false + } + }, + _ => { + let content = serde_json::from_str::(incoming_pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; + + if let Some(redact_id) = &content.redacts { + !services().rooms.state_accessor.user_can_redact( + redact_id, + &incoming_pdu.sender, + &incoming_pdu.room_id, + true, + )? + } else { + false + } + }, + } + }; // 13. Use state resolution to find new room state diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index e255e978..0bc5ade1 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -389,18 +389,11 @@ impl Service { match pdu.kind { TimelineEventType::RoomRedaction => { + use RoomVersionId::*; + let room_version_id = services().rooms.state.get_room_version(&pdu.room_id)?; match room_version_id { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { if services().rooms.state_accessor.user_can_redact( redact_id, @@ -412,7 +405,7 @@ impl Service { } } }, - RoomVersionId::V11 => { + V11 => { let content = serde_json::from_str::(pdu.content.get()).map_err(|e| { warn!("Invalid content in redaction pdu: {e}"); @@ -868,17 +861,9 @@ impl Service { // If redaction event is not authorized, do not append it to the timeline if pdu.kind == TimelineEventType::RoomRedaction { + use RoomVersionId::*; match services().rooms.state.get_room_version(&pdu.room_id)? { - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - | RoomVersionId::V7 - | RoomVersionId::V8 - | RoomVersionId::V9 - | RoomVersionId::V10 => { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { if !services().rooms.state_accessor.user_can_redact( redact_id, From 5be679e17b67e22f8a91fb2906be091198f4c713 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 06:07:09 +0000 Subject: [PATCH 0479/2091] refactor main task stack through service mgr Signed-off-by: Jason Volk --- src/router/run.rs | 54 ++++++++++++++++++++++++++++--------- src/router/serve/mod.rs | 16 +++++------ src/service/mod.rs | 4 +-- src/service/services.rs | 60 ++++++++++++++++++++++++++++------------- 4 files changed, 93 insertions(+), 41 deletions(-) diff --git a/src/router/run.rs b/src/router/run.rs index 02cec781..b3ec2285 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -1,8 +1,10 @@ use std::{sync::Arc, time::Duration}; use axum_server::Handle as ServerHandle; -use tokio::sync::broadcast::{self, Sender}; -use tracing::{debug, error, info}; +use tokio::{ + sync::broadcast::{self, Sender}, + task::JoinHandle, +}; extern crate conduit_admin as admin; extern crate conduit_core as conduit; @@ -10,14 +12,14 @@ extern crate conduit_service as service; use std::sync::atomic::Ordering; -use conduit::{debug_info, trace, Error, Result, Server}; +use conduit::{debug, debug_info, error, info, trace, Error, Result, Server}; -use crate::{layers, serve}; +use crate::serve; /// Main loop base #[tracing::instrument(skip_all)] -pub(crate) async fn run(server: Arc) -> Result<(), Error> { - let app = layers::build(&server)?; +pub(crate) async fn run(server: Arc) -> Result<()> { + debug!("Start"); // Install the admin room callback here for now admin::init().await; @@ -29,8 +31,16 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { .runtime() .spawn(signal(server.clone(), tx.clone(), handle.clone())); - // Serve clients - let res = serve::serve(&server, app, handle, tx.subscribe()).await; + let mut listener = server + .runtime() + .spawn(serve::serve(server.clone(), handle.clone(), tx.subscribe())); + + // Focal point + debug!("Running"); + let res = tokio::select! { + res = &mut listener => res.map_err(Error::from).unwrap_or_else(Err), + res = service::services().poll() => handle_services_poll(&server, res, listener).await, + }; // Join the signal handler before we leave. sigs.abort(); @@ -39,16 +49,16 @@ pub(crate) async fn run(server: Arc) -> Result<(), Error> { // Remove the admin room callback admin::fini().await; - debug_info!("Finished"); + debug_info!("Finish"); res } /// Async initializations #[tracing::instrument(skip_all)] -pub(crate) async fn start(server: Arc) -> Result<(), Error> { +pub(crate) async fn start(server: Arc) -> Result<()> { debug!("Starting..."); - service::init(&server).await?; + service::start(&server).await?; #[cfg(feature = "systemd")] sd_notify::notify(true, &[sd_notify::NotifyState::Ready]).expect("failed to notify systemd of ready state"); @@ -59,12 +69,12 @@ pub(crate) async fn start(server: Arc) -> Result<(), Error> { /// Async destructions #[tracing::instrument(skip_all)] -pub(crate) async fn stop(_server: Arc) -> Result<(), Error> { +pub(crate) async fn stop(_server: Arc) -> Result<()> { debug!("Shutting down..."); // Wait for all completions before dropping or we'll lose them to the module // unload and explode. - service::fini().await; + service::stop().await; debug!("Cleaning up..."); @@ -108,3 +118,21 @@ async fn handle_shutdown(server: &Arc, tx: &Sender<()>, handle: &axum_se handle.shutdown(); } } + +async fn handle_services_poll( + server: &Arc, result: Result<()>, listener: JoinHandle>, +) -> Result<()> { + debug!("Service manager finished: {result:?}"); + + if server.running() { + if let Err(e) = server.shutdown() { + error!("Failed to send shutdown signal: {e}"); + } + } + + if let Err(e) = listener.await { + error!("Client listener task finished with error: {e}"); + } + + result +} diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index 47f2fd43..4e923444 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -4,23 +4,23 @@ mod unix; use std::sync::Arc; -use axum::Router; use axum_server::Handle as ServerHandle; -use conduit::{Error, Result, Server}; +use conduit::{Result, Server}; use tokio::sync::broadcast; +use crate::layers; + /// Serve clients -pub(super) async fn serve( - server: &Arc, app: Router, handle: ServerHandle, shutdown: broadcast::Receiver<()>, -) -> Result<(), Error> { +pub(super) async fn serve(server: Arc, handle: ServerHandle, shutdown: broadcast::Receiver<()>) -> Result<()> { let config = &server.config; let addrs = config.get_bind_addrs(); + let app = layers::build(&server)?; if cfg!(unix) && config.unix_socket_path.is_some() { - unix::serve(server, app, shutdown).await + unix::serve(&server, app, shutdown).await } else if config.tls.is_some() { - tls::serve(server, app, handle, addrs).await + tls::serve(&server, app, handle, addrs).await } else { - plain::serve(server, app, handle, addrs).await + plain::serve(&server, app, handle, addrs).await } } diff --git a/src/service/mod.rs b/src/service/mod.rs index 15c4cc35..3a1eaf57 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -37,7 +37,7 @@ conduit::mod_dtor! {} static SERVICES: RwLock> = RwLock::new(None); -pub async fn init(server: &Arc) -> Result<()> { +pub async fn start(server: &Arc) -> Result<()> { let d = Arc::new(Database::open(server).await?); let s = Box::new(Services::build(server.clone(), d)?); _ = SERVICES.write().expect("write locked").insert(Box::leak(s)); @@ -45,7 +45,7 @@ pub async fn init(server: &Arc) -> Result<()> { services().start().await } -pub async fn fini() { +pub async fn stop() { services().stop().await; // Deactivate services(). Any further use will panic the caller. diff --git a/src/service/services.rs b/src/service/services.rs index 13689008..e9629ef7 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,6 +1,6 @@ -use std::{collections::BTreeMap, fmt::Write, panic::AssertUnwindSafe, sync::Arc, time::Duration}; +use std::{collections::BTreeMap, fmt::Write, ops::DerefMut, panic::AssertUnwindSafe, sync::Arc, time::Duration}; -use conduit::{debug, debug_info, error, info, trace, utils::time, warn, Error, Result, Server}; +use conduit::{debug, debug_info, debug_warn, error, info, trace, utils::time, warn, Error, Result, Server}; use database::Database; use futures_util::FutureExt; use tokio::{ @@ -30,8 +30,8 @@ pub struct Services { pub media: Arc, pub sending: Arc, - workers: Mutex, manager: Mutex>>>, + workers: Mutex, pub(crate) service: Map, pub server: Arc, pub db: Arc, @@ -93,15 +93,15 @@ impl Services { media: build!(media::Service), sending: build!(sending::Service), globals: build!(globals::Service), - workers: Mutex::new(JoinSet::new()), manager: Mutex::new(None), + workers: Mutex::new(JoinSet::new()), service, server, db, }) } - pub async fn start(&self) -> Result<()> { + pub(super) async fn start(&self) -> Result<()> { debug_info!("Starting services..."); self.media.create_media_dir().await?; @@ -114,9 +114,7 @@ impl Services { } debug!("Starting service manager..."); - let manager = async move { crate::services().manager().await }; - let manager = self.server.runtime().spawn(manager); - _ = self.manager.lock().await.insert(manager); + self.manager_start().await?; if self.globals.allow_check_for_updates() { let handle = globals::updates::start_check_for_updates_task(); @@ -127,7 +125,7 @@ impl Services { Ok(()) } - pub async fn stop(&self) { + pub(super) async fn stop(&self) { info!("Shutting down services..."); self.interrupt(); @@ -138,15 +136,20 @@ impl Services { } debug!("Stopping service manager..."); - if let Some(manager) = self.manager.lock().await.take() { - if let Err(e) = manager.await { - error!("Manager shutdown error: {e:?}"); - } - } + self.manager_stop().await; debug_info!("Services shutdown complete."); } + pub async fn poll(&self) -> Result<()> { + if let Some(manager) = self.manager.lock().await.deref_mut() { + trace!("Polling service manager..."); + return manager.await?; + } + + Ok(()) + } + pub async fn clear_cache(&self) { for service in self.service.values() { service.clear_cache(); @@ -188,6 +191,26 @@ impl Services { } } + async fn manager_start(&self) -> Result<()> { + debug!("Starting service manager..."); + self.manager.lock().await.get_or_insert_with(|| { + self.server + .runtime() + .spawn(async move { crate::services().manager().await }) + }); + + Ok(()) + } + + async fn manager_stop(&self) { + if let Some(manager) = self.manager.lock().await.take() { + debug!("Waiting for service manager..."); + if let Err(e) = manager.await { + error!("Manager shutdown error: {e:?}"); + } + } + } + async fn manager(&self) -> Result<()> { loop { let mut workers = self.workers.lock().await; @@ -226,14 +249,15 @@ impl Services { &self, workers: &mut WorkersLocked<'_>, service: &Arc, error: Error, ) -> Result<()> { let name = service.name(); - error!("service {name:?} worker error: {error}"); + error!("service {name:?} aborted: {error}"); - if !error.is_panic() { + if !self.server.running() { + debug_warn!("service {name:?} error ignored on shutdown."); return Ok(()); } - if !self.server.running() { - return Ok(()); + if !error.is_panic() { + return Err(error); } let delay = Duration::from_millis(RESTART_DELAY_MS); From 38c989a07efa5670c9690dc064e3f4955758c43d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 07:01:45 +0000 Subject: [PATCH 0480/2091] split service manager into unit Signed-off-by: Jason Volk --- src/service/manager.rs | 159 ++++++++++++++++++++++++++++++++++++++++ src/service/mod.rs | 1 + src/service/services.rs | 156 +++++---------------------------------- 3 files changed, 179 insertions(+), 137 deletions(-) create mode 100644 src/service/manager.rs diff --git a/src/service/manager.rs b/src/service/manager.rs new file mode 100644 index 00000000..84a28a76 --- /dev/null +++ b/src/service/manager.rs @@ -0,0 +1,159 @@ +use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; + +use conduit::{debug, debug_warn, error, trace, utils::time, warn, Error, Result, Server}; +use futures_util::FutureExt; +use tokio::{ + sync::{Mutex, MutexGuard}, + task::{JoinHandle, JoinSet}, + time::sleep, +}; + +use crate::{service::Service, Services}; + +pub(crate) struct Manager { + manager: Mutex>>>, + workers: Mutex, + server: Arc, + services: &'static Services, +} + +type Workers = JoinSet; +type WorkerResult = (Arc, Result<()>); +type WorkersLocked<'a> = MutexGuard<'a, Workers>; + +const RESTART_DELAY_MS: u64 = 2500; + +impl Manager { + pub(super) fn new(services: &Services) -> Arc { + Arc::new(Self { + manager: Mutex::new(None), + workers: Mutex::new(JoinSet::new()), + server: services.server.clone(), + services: crate::services(), + }) + } + + pub(super) async fn poll(&self) -> Result<()> { + if let Some(manager) = &mut *self.manager.lock().await { + trace!("Polling service manager..."); + return manager.await?; + } + + Ok(()) + } + + pub(super) async fn start(self: Arc) -> Result<()> { + let mut workers = self.workers.lock().await; + + debug!("Starting service manager..."); + let self_ = self.clone(); + _ = self.manager.lock().await.insert( + self.server + .runtime() + .spawn(async move { self_.worker().await }), + ); + + debug!("Starting service workers..."); + for service in self.services.service.values() { + self.start_worker(&mut workers, service).await?; + } + + Ok(()) + } + + pub(super) async fn stop(&self) { + if let Some(manager) = self.manager.lock().await.take() { + debug!("Waiting for service manager..."); + if let Err(e) = manager.await { + error!("Manager shutdown error: {e:?}"); + } + } + } + + async fn worker(&self) -> Result<()> { + loop { + let mut workers = self.workers.lock().await; + tokio::select! { + result = workers.join_next() => match result { + Some(Ok(result)) => self.handle_result(&mut workers, result).await?, + Some(Err(error)) => self.handle_abort(&mut workers, Error::from(error)).await?, + None => break, + } + } + } + + debug!("Worker manager finished"); + Ok(()) + } + + async fn handle_abort(&self, _workers: &mut WorkersLocked<'_>, error: Error) -> Result<()> { + // not supported until service can be associated with abort + unimplemented!("unexpected worker task abort {error:?}"); + } + + async fn handle_result(&self, workers: &mut WorkersLocked<'_>, result: WorkerResult) -> Result<()> { + let (service, result) = result; + match result { + Ok(()) => self.handle_finished(workers, &service).await, + Err(error) => self.handle_error(workers, &service, error).await, + } + } + + async fn handle_finished(&self, _workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + debug!("service {:?} worker finished", service.name()); + Ok(()) + } + + async fn handle_error( + &self, workers: &mut WorkersLocked<'_>, service: &Arc, error: Error, + ) -> Result<()> { + let name = service.name(); + error!("service {name:?} aborted: {error}"); + + if !self.server.running() { + debug_warn!("service {name:?} error ignored on shutdown."); + return Ok(()); + } + + if !error.is_panic() { + return Err(error); + } + + let delay = Duration::from_millis(RESTART_DELAY_MS); + warn!("service {name:?} worker restarting after {} delay", time::pretty(delay)); + sleep(delay).await; + + self.start_worker(workers, service).await + } + + /// Start the worker in a task for the service. + async fn start_worker(&self, workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + if !self.server.running() { + return Err(Error::Err(format!( + "Service {:?} worker not starting during server shutdown.", + service.name() + ))); + } + + debug!("Service {:?} worker starting...", service.name()); + workers.spawn_on(worker(service.clone()), self.server.runtime()); + + Ok(()) + } +} + +/// Base frame for service worker. This runs in a tokio::task. All errors and +/// panics from the worker are caught and returned cleanly. The JoinHandle +/// should never error with a panic, and if so it should propagate, but it may +/// error with an Abort which the manager should handle along with results to +/// determine if the worker should be restarted. +async fn worker(service: Arc) -> WorkerResult { + let service_ = Arc::clone(&service); + let result = AssertUnwindSafe(service_.worker()) + .catch_unwind() + .await + .map_err(Error::from_panic); + + // flattens JoinError for panic into worker's Error + (service, result.unwrap_or_else(Err)) +} diff --git a/src/service/mod.rs b/src/service/mod.rs index 3a1eaf57..ba68fae2 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,5 +1,6 @@ #![allow(refining_impl_trait)] +mod manager; mod service; pub mod services; diff --git a/src/service/services.rs b/src/service/services.rs index e9629ef7..62e73fd1 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,16 +1,13 @@ -use std::{collections::BTreeMap, fmt::Write, ops::DerefMut, panic::AssertUnwindSafe, sync::Arc, time::Duration}; +use std::{collections::BTreeMap, fmt::Write, sync::Arc}; -use conduit::{debug, debug_info, debug_warn, error, info, trace, utils::time, warn, Error, Result, Server}; +use conduit::{debug, debug_info, info, trace, Result, Server}; use database::Database; -use futures_util::FutureExt; -use tokio::{ - sync::{Mutex, MutexGuard}, - task::{JoinHandle, JoinSet}, - time::sleep, -}; +use tokio::sync::Mutex; use crate::{ - account_data, admin, appservice, globals, key_backups, media, presence, pusher, rooms, sending, + account_data, admin, appservice, globals, key_backups, + manager::Manager, + media, presence, pusher, rooms, sending, service::{Args, Map, Service}, transaction_ids, uiaa, users, }; @@ -30,19 +27,12 @@ pub struct Services { pub media: Arc, pub sending: Arc, - manager: Mutex>>>, - workers: Mutex, + manager: Mutex>>, pub(crate) service: Map, pub server: Arc, pub db: Arc, } -type Workers = JoinSet; -type WorkerResult = (Arc, Result<()>); -type WorkersLocked<'a> = MutexGuard<'a, Workers>; - -const RESTART_DELAY_MS: u64 = 2500; - impl Services { pub fn build(server: Arc, db: Arc) -> Result { let mut service: Map = BTreeMap::new(); @@ -94,7 +84,6 @@ impl Services { sending: build!(sending::Service), globals: build!(globals::Service), manager: Mutex::new(None), - workers: Mutex::new(JoinSet::new()), service, server, db, @@ -108,13 +97,13 @@ impl Services { globals::migrations::migrations(&self.db, &self.globals.config).await?; globals::emerg_access::init_emergency_access(); - let mut workers = self.workers.lock().await; - for service in self.service.values() { - self.start_worker(&mut workers, service).await?; - } - - debug!("Starting service manager..."); - self.manager_start().await?; + self.manager + .lock() + .await + .insert(Manager::new(self)) + .clone() + .start() + .await?; if self.globals.allow_check_for_updates() { let handle = globals::updates::start_check_for_updates_task(); @@ -135,16 +124,16 @@ impl Services { _ = updates_handle.await; } - debug!("Stopping service manager..."); - self.manager_stop().await; + if let Some(manager) = self.manager.lock().await.as_ref() { + manager.stop().await; + } debug_info!("Services shutdown complete."); } pub async fn poll(&self) -> Result<()> { - if let Some(manager) = self.manager.lock().await.deref_mut() { - trace!("Polling service manager..."); - return manager.await?; + if let Some(manager) = self.manager.lock().await.as_ref() { + return manager.poll().await; } Ok(()) @@ -190,111 +179,4 @@ impl Services { service.interrupt(); } } - - async fn manager_start(&self) -> Result<()> { - debug!("Starting service manager..."); - self.manager.lock().await.get_or_insert_with(|| { - self.server - .runtime() - .spawn(async move { crate::services().manager().await }) - }); - - Ok(()) - } - - async fn manager_stop(&self) { - if let Some(manager) = self.manager.lock().await.take() { - debug!("Waiting for service manager..."); - if let Err(e) = manager.await { - error!("Manager shutdown error: {e:?}"); - } - } - } - - async fn manager(&self) -> Result<()> { - loop { - let mut workers = self.workers.lock().await; - tokio::select! { - result = workers.join_next() => match result { - Some(Ok(result)) => self.handle_result(&mut workers, result).await?, - Some(Err(error)) => self.handle_abort(&mut workers, Error::from(error)).await?, - None => break, - } - } - } - - debug!("Worker manager finished"); - Ok(()) - } - - async fn handle_abort(&self, _workers: &mut WorkersLocked<'_>, error: Error) -> Result<()> { - // not supported until service can be associated with abort - unimplemented!("unexpected worker task abort {error:?}"); - } - - async fn handle_result(&self, workers: &mut WorkersLocked<'_>, result: WorkerResult) -> Result<()> { - let (service, result) = result; - match result { - Ok(()) => self.handle_finished(workers, &service).await, - Err(error) => self.handle_error(workers, &service, error).await, - } - } - - async fn handle_finished(&self, _workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { - debug!("service {:?} worker finished", service.name()); - Ok(()) - } - - async fn handle_error( - &self, workers: &mut WorkersLocked<'_>, service: &Arc, error: Error, - ) -> Result<()> { - let name = service.name(); - error!("service {name:?} aborted: {error}"); - - if !self.server.running() { - debug_warn!("service {name:?} error ignored on shutdown."); - return Ok(()); - } - - if !error.is_panic() { - return Err(error); - } - - let delay = Duration::from_millis(RESTART_DELAY_MS); - warn!("service {name:?} worker restarting after {} delay", time::pretty(delay)); - sleep(delay).await; - - self.start_worker(workers, service).await - } - - /// Start the worker in a task for the service. - async fn start_worker(&self, workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { - if !self.server.running() { - return Err(Error::Err(format!( - "Service {:?} worker not starting during server shutdown.", - service.name() - ))); - } - - debug!("Service {:?} worker starting...", service.name()); - workers.spawn_on(worker(service.clone()), self.server.runtime()); - - Ok(()) - } -} - -/// Base frame for service worker. This runs in a tokio::task. All errors and -/// panics from the worker are caught and returned cleanly. The JoinHandle -/// should never error with a panic, and if so it should propagate, but it may -/// error with an Abort which the manager should handle along with results to -/// determine if the worker should be restarted. -async fn worker(service: Arc) -> WorkerResult { - let service_ = Arc::clone(&service); - let result = AssertUnwindSafe(service_.worker()) - .catch_unwind() - .await - .map_err(Error::from_panic); - - // flattens JoinError for panic into worker's Error - (service, result.unwrap_or_else(Err)) } From aa7a3102001b2b9609db7a96d10232d63b8ee24f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 07:05:51 +0000 Subject: [PATCH 0481/2091] move media startup into service Signed-off-by: Jason Volk --- src/service/media/mod.rs | 8 ++++++++ src/service/services.rs | 1 - 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 3cb8fda8..9b1cbe06 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -3,6 +3,7 @@ mod tests; use std::{collections::HashMap, io::Cursor, num::Saturating as Sat, path::PathBuf, sync::Arc, time::SystemTime}; +use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; use conduit::{checked, debug, debug_error, error, utils, Error, Result, Server}; use data::Data; @@ -47,6 +48,7 @@ pub struct Service { pub url_preview_mutex: RwLock>>>, } +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -56,6 +58,12 @@ impl crate::Service for Service { })) } + async fn worker(self: Arc) -> Result<()> { + self.create_media_dir().await?; + + Ok(()) + } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/services.rs b/src/service/services.rs index 62e73fd1..88b58299 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -93,7 +93,6 @@ impl Services { pub(super) async fn start(&self) -> Result<()> { debug_info!("Starting services..."); - self.media.create_media_dir().await?; globals::migrations::migrations(&self.db, &self.globals.config).await?; globals::emerg_access::init_emergency_access(); From 084751ae381fe6f304f5335251ed1ef4b254b064 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 08:07:49 +0000 Subject: [PATCH 0482/2091] refactor globals::updates into a Service Signed-off-by: Jason Volk --- src/admin/query/globals.rs | 2 +- src/service/globals/data.rs | 22 +------ src/service/globals/mod.rs | 21 +++---- src/service/globals/updates.rs | 76 ---------------------- src/service/mod.rs | 1 + src/service/services.rs | 24 +++---- src/service/updates/mod.rs | 112 +++++++++++++++++++++++++++++++++ 7 files changed, 133 insertions(+), 125 deletions(-) delete mode 100644 src/service/globals/updates.rs create mode 100644 src/service/updates/mod.rs diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 2e22d688..9bdd38fc 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -26,7 +26,7 @@ pub(super) async fn globals(subcommand: Globals) -> Result { let timer = tokio::time::Instant::now(); - let results = services().globals.db.last_check_for_updates_id(); + let results = services().updates.last_check_for_updates_id(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 254a3d9c..281c2a94 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -14,9 +14,6 @@ use ruma::{ use crate::services; -const COUNTER: &[u8] = b"c"; -const LAST_CHECK_FOR_UPDATES_COUNT: &[u8] = b"u"; - pub struct Data { global: Arc, todeviceid_events: Arc, @@ -35,6 +32,8 @@ pub struct Data { counter: RwLock, } +const COUNTER: &[u8] = b"c"; + impl Data { pub(super) fn new(db: &Arc) -> Self { Self { @@ -93,23 +92,6 @@ impl Data { .map_or(Ok(0_u64), utils::u64_from_bytes) } - pub fn last_check_for_updates_id(&self) -> Result { - self.global - .get(LAST_CHECK_FOR_UPDATES_COUNT)? - .map_or(Ok(0_u64), |bytes| { - utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("last check for updates count has invalid bytes.")) - }) - } - - #[inline] - pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { - self.global - .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes())?; - - Ok(()) - } - #[tracing::instrument(skip(self), level = "debug")] pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { let userid_bytes = user_id.as_bytes().to_vec(); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 16830d87..0a0d0d8e 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -1,9 +1,8 @@ mod client; mod data; -pub(super) mod emerg_access; +mod emerg_access; pub(super) mod migrations; pub(crate) mod resolver; -pub(super) mod updates; use std::{ collections::{BTreeMap, HashMap}, @@ -12,6 +11,7 @@ use std::{ time::Instant, }; +use async_trait::async_trait; use conduit::{error, trace, Config, Result}; use data::Data; use ipaddress::IPAddress; @@ -22,7 +22,7 @@ use ruma::{ DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, }; -use tokio::{sync::Mutex, task::JoinHandle}; +use tokio::sync::Mutex; use url::Url; use crate::services; @@ -41,7 +41,6 @@ pub struct Service { pub bad_event_ratelimiter: Arc>>, pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub bad_query_ratelimiter: Arc>>, - pub updates_handle: Mutex>>, pub stateres_mutex: Arc>, pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, @@ -49,6 +48,7 @@ pub struct Service { type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; @@ -103,7 +103,6 @@ impl crate::Service for Service { bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), - updates_handle: Mutex::new(None), stateres_mutex: Arc::new(Mutex::new(())), admin_alias: RoomAliasId::parse(format!("#admins:{}", &config.server_name)) .expect("#admins:server_name is valid alias name"), @@ -122,6 +121,12 @@ impl crate::Service for Service { Ok(Arc::new(s)) } + async fn worker(self: Arc) -> Result<()> { + emerg_access::init_emergency_access(); + + Ok(()) + } + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { self.resolver.memory_usage(out)?; @@ -181,12 +186,6 @@ impl Service { #[inline] pub fn current_count(&self) -> Result { Ok(self.db.current_count()) } - #[tracing::instrument(skip(self), level = "debug")] - pub fn last_check_for_updates_id(&self) -> Result { self.db.last_check_for_updates_id() } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { self.db.update_check_for_updates_id(id) } - pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { self.db.watch(user_id, device_id).await } diff --git a/src/service/globals/updates.rs b/src/service/globals/updates.rs deleted file mode 100644 index c6ac9fff..00000000 --- a/src/service/globals/updates.rs +++ /dev/null @@ -1,76 +0,0 @@ -use std::time::Duration; - -use ruma::events::room::message::RoomMessageEventContent; -use serde::Deserialize; -use tokio::{task::JoinHandle, time::interval}; -use tracing::{error, warn}; - -use crate::{ - conduit::{Error, Result}, - services, -}; - -const CHECK_FOR_UPDATES_URL: &str = "https://pupbrain.dev/check-for-updates/stable"; -const CHECK_FOR_UPDATES_INTERVAL: u64 = 7200; // 2 hours - -#[derive(Deserialize)] -struct CheckForUpdatesResponseEntry { - id: u64, - date: String, - message: String, -} -#[derive(Deserialize)] -struct CheckForUpdatesResponse { - updates: Vec, -} - -#[tracing::instrument] -pub fn start_check_for_updates_task() -> JoinHandle<()> { - let timer_interval = Duration::from_secs(CHECK_FOR_UPDATES_INTERVAL); - - services().server.runtime().spawn(async move { - let mut i = interval(timer_interval); - - loop { - i.tick().await; - - if let Err(e) = try_handle_updates().await { - warn!(%e, "Failed to check for updates"); - } - } - }) -} - -#[tracing::instrument(skip_all)] -async fn try_handle_updates() -> Result<()> { - let response = services() - .globals - .client - .default - .get(CHECK_FOR_UPDATES_URL) - .send() - .await?; - - let response = serde_json::from_str::(&response.text().await?) - .map_err(|e| Error::Err(format!("Bad check for updates response: {e}")))?; - - let mut last_update_id = services().globals.last_check_for_updates_id()?; - for update in response.updates { - last_update_id = last_update_id.max(update.id); - if update.id > services().globals.last_check_for_updates_id()? { - error!("{}", update.message); - services() - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "@room: the following is a message from the conduwuit puppy. it was sent on '{}':\n\n{}", - update.date, update.message - ))) - .await; - } - } - services() - .globals - .update_check_for_updates_id(last_update_id)?; - - Ok(()) -} diff --git a/src/service/mod.rs b/src/service/mod.rs index ba68fae2..6f2f4ee5 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -16,6 +16,7 @@ pub mod rooms; pub mod sending; pub mod transaction_ids; pub mod uiaa; +pub mod updates; pub mod users; extern crate conduit_core as conduit; diff --git a/src/service/services.rs b/src/service/services.rs index 88b58299..cc9ec290 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -9,7 +9,7 @@ use crate::{ manager::Manager, media, presence, pusher, rooms, sending, service::{Args, Map, Service}, - transaction_ids, uiaa, users, + transaction_ids, uiaa, updates, users, }; pub struct Services { @@ -22,10 +22,11 @@ pub struct Services { pub account_data: Arc, pub presence: Arc, pub admin: Arc, - pub globals: Arc, pub key_backups: Arc, pub media: Arc, pub sending: Arc, + pub updates: Arc, + pub globals: Arc, manager: Mutex>>, pub(crate) service: Map, @@ -82,6 +83,7 @@ impl Services { key_backups: build!(key_backups::Service), media: build!(media::Service), sending: build!(sending::Service), + updates: build!(updates::Service), globals: build!(globals::Service), manager: Mutex::new(None), service, @@ -93,9 +95,7 @@ impl Services { pub(super) async fn start(&self) -> Result<()> { debug_info!("Starting services..."); - globals::migrations::migrations(&self.db, &self.globals.config).await?; - globals::emerg_access::init_emergency_access(); - + globals::migrations::migrations(&self.db, &self.server.config).await?; self.manager .lock() .await @@ -104,25 +104,14 @@ impl Services { .start() .await?; - if self.globals.allow_check_for_updates() { - let handle = globals::updates::start_check_for_updates_task(); - _ = self.globals.updates_handle.lock().await.insert(handle); - } - debug_info!("Services startup complete."); Ok(()) } pub(super) async fn stop(&self) { info!("Shutting down services..."); + self.interrupt(); - - debug!("Waiting for update worker..."); - if let Some(updates_handle) = self.globals.updates_handle.lock().await.take() { - updates_handle.abort(); - _ = updates_handle.await; - } - if let Some(manager) = self.manager.lock().await.as_ref() { manager.stop().await; } @@ -173,6 +162,7 @@ impl Services { fn interrupt(&self) { debug!("Interrupting services..."); + for (name, service) in &self.service { trace!("Interrupting {name}"); service.interrupt(); diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs new file mode 100644 index 00000000..a7088aba --- /dev/null +++ b/src/service/updates/mod.rs @@ -0,0 +1,112 @@ +use std::{sync::Arc, time::Duration}; + +use async_trait::async_trait; +use conduit::{err, info, utils, warn, Error, Result}; +use database::Map; +use ruma::events::room::message::RoomMessageEventContent; +use serde::Deserialize; +use tokio::{sync::Notify, time::interval}; + +use crate::services; + +pub struct Service { + db: Arc, + interrupt: Notify, + interval: Duration, +} + +#[derive(Deserialize)] +struct CheckForUpdatesResponse { + updates: Vec, +} + +#[derive(Deserialize)] +struct CheckForUpdatesResponseEntry { + id: u64, + date: String, + message: String, +} + +const CHECK_FOR_UPDATES_URL: &str = "https://pupbrain.dev/check-for-updates/stable"; +const CHECK_FOR_UPDATES_INTERVAL: u64 = 7200; // 2 hours +const LAST_CHECK_FOR_UPDATES_COUNT: &[u8] = b"u"; + +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + db: args.db["global"].clone(), + interrupt: Notify::new(), + interval: Duration::from_secs(CHECK_FOR_UPDATES_INTERVAL), + })) + } + + async fn worker(self: Arc) -> Result<()> { + let mut i = interval(self.interval); + loop { + tokio::select! { + () = self.interrupt.notified() => return Ok(()), + _ = i.tick() => (), + } + + if let Err(e) = self.handle_updates().await { + warn!(%e, "Failed to check for updates"); + } + } + } + + fn interrupt(&self) { self.interrupt.notify_waiters(); } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { + #[tracing::instrument(skip_all)] + async fn handle_updates(&self) -> Result<()> { + let response = services() + .globals + .client + .default + .get(CHECK_FOR_UPDATES_URL) + .send() + .await?; + + let response = serde_json::from_str::(&response.text().await?) + .map_err(|e| Error::Err(format!("Bad check for updates response: {e}")))?; + + let mut last_update_id = self.last_check_for_updates_id()?; + for update in response.updates { + last_update_id = last_update_id.max(update.id); + if update.id > self.last_check_for_updates_id()? { + info!("{:#}", update.message); + services() + .admin + .send_message(RoomMessageEventContent::text_markdown(format!( + "### the following is a message from the conduwuit puppy\n\nit was sent on `{}`:\n\n@room: {}", + update.date, update.message + ))) + .await; + } + } + self.update_check_for_updates_id(last_update_id)?; + + Ok(()) + } + + #[inline] + pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { + self.db + .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes())?; + + Ok(()) + } + + pub fn last_check_for_updates_id(&self) -> Result { + self.db + .get(LAST_CHECK_FOR_UPDATES_COUNT)? + .map_or(Ok(0_u64), |bytes| { + utils::u64_from_bytes(&bytes) + .map_err(|_| Error::bad_database("last check for updates count has invalid bytes.")) + }) + } +} From b3f2288d07d3bbb730ca3a81ae8763bb7e622984 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 21:02:43 +0000 Subject: [PATCH 0483/2091] add constant-expression string utils Signed-off-by: Jason Volk --- Cargo.lock | 13 +++++++++++++ Cargo.toml | 3 +++ src/admin/Cargo.toml | 1 + src/api/Cargo.toml | 1 + src/core/Cargo.toml | 1 + src/core/utils/string.rs | 24 ++++++++++++++++++++++++ src/database/Cargo.toml | 1 + src/main/Cargo.toml | 1 + src/router/Cargo.toml | 1 + src/service/Cargo.toml | 1 + 10 files changed, 47 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 02da0b6c..f2036284 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -589,6 +589,7 @@ dependencies = [ "conduit_router", "conduit_service", "console-subscriber", + "const-str", "hardened_malloc-rs", "log", "opentelemetry", @@ -613,6 +614,7 @@ dependencies = [ "conduit_api", "conduit_core", "conduit_service", + "const-str", "futures-util", "log", "ruma", @@ -635,6 +637,7 @@ dependencies = [ "conduit_core", "conduit_database", "conduit_service", + "const-str", "futures-util", "hmac", "http 1.1.0", @@ -665,6 +668,7 @@ dependencies = [ "bytes", "checked_ops", "chrono", + "const-str", "either", "figment", "hardened_malloc-rs", @@ -701,6 +705,7 @@ name = "conduit_database" version = "0.4.5" dependencies = [ "conduit_core", + "const-str", "log", "rust-rocksdb-uwu", "tokio", @@ -720,6 +725,7 @@ dependencies = [ "conduit_api", "conduit_core", "conduit_service", + "const-str", "http 1.1.0", "http-body-util", "hyper 1.4.1", @@ -746,6 +752,7 @@ dependencies = [ "bytes", "conduit_core", "conduit_database", + "const-str", "cyborgtime", "futures-util", "hickory-resolver", @@ -817,6 +824,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-str" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3618cccc083bb987a415d85c02ca6c9994ea5b44731ec28b9ecf09658655fba9" + [[package]] name = "const_panic" version = "0.2.8" diff --git a/Cargo.toml b/Cargo.toml index fcbe53d5..33e2b9d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,9 @@ version = "0.4.5" [workspace.metadata.crane] name = "conduit" +[workspace.dependencies.const-str] +version = "0.5.7" + [workspace.dependencies.sanitize-filename] version = "0.5.0" diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 7a092237..1e13fb7a 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -30,6 +30,7 @@ clap.workspace = true conduit-api.workspace = true conduit-core.workspace = true conduit-service.workspace = true +const-str.workspace = true futures-util.workspace = true log.workspace = true ruma.workspace = true diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 45cae73d..356adc1f 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -41,6 +41,7 @@ bytes.workspace = true conduit-core.workspace = true conduit-database.workspace = true conduit-service.workspace = true +const-str.workspace = true futures-util.workspace = true hmac.workspace = true http.workspace = true diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index e47f673e..73ee0152 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -55,6 +55,7 @@ axum.workspace = true bytes.workspace = true checked_ops.workspace = true chrono.workspace = true +const-str.workspace = true either.workspace = true figment.workspace = true http-body-util.workspace = true diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index ec423d53..106d0cb7 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -2,6 +2,30 @@ use crate::Result; pub const EMPTY: &str = ""; +/// Constant expression to bypass format! if the argument is a string literal +/// but not a format string. If the literal is a format string then String is +/// returned otherwise the input (i.e. &'static str) is returned. If multiple +/// arguments are provided the first is assumed to be a format string. +#[macro_export] +macro_rules! format_maybe { + ($s:literal) => { + if $crate::is_format!($s) { std::format!($s).into() } else { $s.into() } + }; + + ($($args:expr),*) => { + std::format!($($args),*).into() + }; +} + +/// Constant expression to decide if a literal is a format string. Note: could +/// use some improvement. +#[macro_export] +macro_rules! is_format { + ($s:literal) => { + ::const_str::contains!($s, "{") && ::const_str::contains!($s, "}") + }; +} + /// Find the common prefix from a collection of strings and return a slice /// ``` /// use conduit_core::utils::string::common_prefix; diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 8b0d3fc3..34d98416 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -36,6 +36,7 @@ zstd_compression = [ [dependencies] conduit-core.workspace = true +const-str.workspace = true log.workspace = true rust-rocksdb.workspace = true tokio.workspace = true diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index fa0e5874..8dc2a34d 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -147,6 +147,7 @@ log.workspace = true tracing.workspace = true tracing-subscriber.workspace = true clap.workspace = true +const-str.workspace = true opentelemetry.workspace = true opentelemetry.optional = true diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 5312984a..38e6adc7 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -55,6 +55,7 @@ conduit-admin.workspace = true conduit-api.workspace = true conduit-core.workspace = true conduit-service.workspace = true +const-str.workspace = true log.workspace = true tokio.workspace = true tower.workspace = true diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index f59a6036..d2c9785f 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -42,6 +42,7 @@ base64.workspace = true bytes.workspace = true conduit-core.workspace = true conduit-database.workspace = true +const-str.workspace = true cyborgtime.workspace = true futures-util.workspace = true hickory-resolver.workspace = true From 05efd9b04471fc6e2dd9f373bc8ef505d65cf572 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 13 Jul 2024 21:11:05 +0000 Subject: [PATCH 0484/2091] elaborate error macro and apply at various callsites Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 11 +- src/admin/utils.rs | 10 +- src/api/client/directory.rs | 7 +- src/api/client/sync.rs | 5 +- src/core/error/err.rs | 103 +++++++++++++----- src/core/error/mod.rs | 23 ++-- src/core/server.rs | 14 +-- src/core/utils/hash/argon.rs | 4 +- src/database/util.rs | 7 +- src/router/serve/unix.rs | 9 +- src/service/appservice/mod.rs | 4 +- src/service/manager.rs | 7 +- src/service/mod.rs | 2 +- src/service/rooms/alias/mod.rs | 2 +- src/service/rooms/auth_chain/mod.rs | 11 +- src/service/rooms/event_handler/mod.rs | 9 +- .../rooms/event_handler/parse_incoming_pdu.rs | 9 +- src/service/rooms/spaces/mod.rs | 10 +- src/service/rooms/state_accessor/mod.rs | 18 +-- src/service/rooms/state_cache/mod.rs | 20 ++-- src/service/sending/mod.rs | 4 +- src/service/sending/resolve.rs | 10 +- src/service/updates/mod.rs | 2 +- 23 files changed, 161 insertions(+), 140 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 229f05e1..e4503736 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,4 +1,4 @@ -use conduit::{utils::time, warn, Error, Result}; +use conduit::{utils::time, warn, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; use crate::services; @@ -88,11 +88,10 @@ pub(super) async fn restart(_body: Vec<&str>, force: bool) -> Result (OwnedRoomId, u64, String) { /// Parses user ID pub(crate) fn parse_user_id(user_id: &str) -> Result { UserId::parse_with_server_name(user_id.to_lowercase(), services().globals.server_name()) - .map_err(|e| Error::Err(format!("The supplied username is not a valid username: {e}"))) + .map_err(|e| err!("The supplied username is not a valid username: {e}")) } /// Parses user ID as our local user @@ -41,7 +41,7 @@ pub(crate) fn parse_local_user_id(user_id: &str) -> Result { let user_id = parse_user_id(user_id)?; if !user_is_local(&user_id) { - return Err(Error::Err(String::from("User does not belong to our server."))); + return Err!("User {user_id:?} does not belong to our server."); } Ok(user_id) @@ -52,11 +52,11 @@ pub(crate) fn parse_active_local_user_id(user_id: &str) -> Result { let user_id = parse_local_user_id(user_id)?; if !services().users.exists(&user_id)? { - return Err(Error::Err(String::from("User does not exist on this server."))); + return Err!("User {user_id:?} does not exist on this server."); } if services().users.is_deactivated(&user_id)? { - return Err(Error::Err(String::from("User is deactivated."))); + return Err!("User {user_id:?} is deactivated."); } Ok(user_id) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 7d2aff0d..68bd0dff 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,4 +1,5 @@ use axum_client_ip::InsecureClientIp; +use conduit::{err, info, warn, Error, Result}; use ruma::{ api::{ client::{ @@ -18,9 +19,8 @@ use ruma::{ }, uint, RoomId, ServerName, UInt, UserId, }; -use tracing::{error, info, warn}; -use crate::{service::server_is_ours, services, Error, Result, Ruma}; +use crate::{service::server_is_ours, services, Ruma}; /// # `POST /_matrix/client/v3/publicRooms` /// @@ -271,8 +271,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( _ => None, }) .map_err(|e| { - error!("Invalid room join rule event in database: {}", e); - Error::BadDatabase("Invalid room join rule event in database.") + err!(Database(error!("Invalid room join rule event in database: {e}"))) }) }) .transpose()? diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index dd1f3401..e425616b 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -8,7 +8,7 @@ use std::{ use conduit::{ error, utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, - PduCount, + Err, PduCount, }; use ruma::{ api::client::{ @@ -545,8 +545,7 @@ async fn load_joined_room( // Database queries: let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? else { - error!("Room {} has no state", room_id); - return Err(Error::BadDatabase("Room has no state")); + return Err!(Database(error!("Room {room_id} has no state"))); }; let since_shortstatehash = services() diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 12dd4f3b..ea596644 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -1,3 +1,36 @@ +//! Error construction macros +//! +//! These are specialized macros specific to this project's patterns for +//! throwing Errors; they make Error construction succinct and reduce clutter. +//! They are developed from folding existing patterns into the macro while +//! fixing several anti-patterns in the codebase. +//! +//! - The primary macros `Err!` and `err!` are provided. `Err!` simply wraps +//! `err!` in the Result variant to reduce `Err(err!(...))` boilerplate, thus +//! `err!` can be used in any case. +//! +//! 1. The macro makes the general Error construction easy: `return +//! Err!("something went wrong")` replaces the prior `return +//! Err(Error::Err("something went wrong".to_owned()))`. +//! +//! 2. The macro integrates format strings automatically: `return +//! Err!("something bad: {msg}")` replaces the prior `return +//! Err(Error::Err(format!("something bad: {msg}")))`. +//! +//! 3. The macro scopes variants of Error: `return Err!(Database("problem with +//! bad database."))` replaces the prior `return Err(Error::Database("problem +//! with bad database."))`. +//! +//! 4. The macro matches and scopes some special-case sub-variants, for example +//! with ruma ErrorKind: `return Err!(Request(MissingToken("you must provide +//! an access token")))`. +//! +//! 5. The macro fixes the anti-pattern of repeating messages in an error! log +//! and then again in an Error construction, often slightly different due to +//! the Error variant not supporting a format string. Instead `return +//! Err(Database(error!("problem with db: {msg}")))` logs the error at the +//! callsite and then returns the error with the same string. Caller has the +//! option of replacing `error!` with `debug_error!`. #[macro_export] macro_rules! Err { ($($args:tt)*) => { @@ -7,36 +40,56 @@ macro_rules! Err { #[macro_export] macro_rules! err { - (error!($($args:tt),+)) => {{ - $crate::error!($($args),+); - $crate::error::Error::Err(std::format!($($args),+)) + (Config($item:literal, $($args:expr),*)) => {{ + $crate::error!(config = %$item, $($args),*); + $crate::error::Error::Config($item, $crate::format_maybe!($($args),*)) }}; - (debug_error!($($args:tt),+)) => {{ - $crate::debug_error!($($args),+); - $crate::error::Error::Err(std::format!($($args),+)) + (Request(Forbidden($level:ident!($($args:expr),*)))) => {{ + $crate::$level!($($args),*); + $crate::error::Error::Request( + ::ruma::api::client::error::ErrorKind::forbidden(), + $crate::format_maybe!($($args),*) + ) }}; - ($variant:ident(error!($($args:tt),+))) => {{ - $crate::error!($($args),+); - $crate::error::Error::$variant(std::format!($($args),+)) - }}; - - ($variant:ident(debug_error!($($args:tt),+))) => {{ - $crate::debug_error!($($args),+); - $crate::error::Error::$variant(std::format!($($args),+)) - }}; - - (Config($item:literal, $($args:tt),+)) => {{ - $crate::error!(config = %$item, $($args),+); - $crate::error::Error::Config($item, std::format!($($args),+)) - }}; - - ($variant:ident($($args:tt),+)) => { - $crate::error::Error::$variant(std::format!($($args),+)) + (Request(Forbidden($($args:expr),*))) => { + $crate::error::Error::Request( + ::ruma::api::client::error::ErrorKind::forbidden(), + $crate::format_maybe!($($args),*) + ) }; - ($string:literal$(,)? $($args:tt),*) => { - $crate::error::Error::Err(std::format!($string, $($args),*)) + (Request($variant:ident($level:ident!($($args:expr),*)))) => {{ + $crate::$level!($($args),*); + $crate::error::Error::Request( + ::ruma::api::client::error::ErrorKind::$variant, + $crate::format_maybe!($($args),*) + ) + }}; + + (Request($variant:ident($($args:expr),*))) => { + $crate::error::Error::Request( + ::ruma::api::client::error::ErrorKind::$variant, + $crate::format_maybe!($($args),*) + ) + }; + + ($variant:ident($level:ident!($($args:expr),*))) => {{ + $crate::$level!($($args),*); + $crate::error::Error::$variant($crate::format_maybe!($($args),*)) + }}; + + ($variant:ident($($args:expr),*)) => { + $crate::error::Error::$variant($crate::format_maybe!($($args),*)) + }; + + ($level:ident!($($args:expr),*)) => {{ + $crate::$level!($($args),*); + $crate::error::Error::Err($crate::format_maybe!($($args),*)) + }}; + + ($($args:expr),*) => { + $crate::error::Error::Err($crate::format_maybe!($($args),*)) }; } diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index cbc22f3e..3adfde97 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -3,7 +3,7 @@ mod log; mod panic; mod response; -use std::{any::Any, convert::Infallible, fmt}; +use std::{any::Any, borrow::Cow, convert::Infallible, fmt}; pub use log::*; @@ -64,7 +64,9 @@ pub enum Error { #[error("{0}")] Mxid(#[from] ruma::IdParseError), #[error("{0}: {1}")] - BadRequest(ruma::api::client::error::ErrorKind, &'static str), + BadRequest(ruma::api::client::error::ErrorKind, &'static str), //TODO: remove + #[error("{0}: {1}")] + Request(ruma::api::client::error::ErrorKind, Cow<'static, str>), #[error("from {0}: {1}")] Redaction(ruma::OwnedServerName, ruma::canonical_json::RedactionError), #[error("Remote server {0} responded with: {1}")] @@ -76,11 +78,9 @@ pub enum Error { #[error("Arithmetic operation failed: {0}")] Arithmetic(&'static str), #[error("There was a problem with the '{0}' directive in your configuration: {1}")] - Config(&'static str, String), + Config(&'static str, Cow<'static, str>), #[error("{0}")] - BadDatabase(&'static str), - #[error("{0}")] - Database(String), + Database(Cow<'static, str>), #[error("{0}")] BadServerResponse(&'static str), #[error("{0}")] @@ -88,14 +88,11 @@ pub enum Error { // unique / untyped #[error("{0}")] - Err(String), + Err(Cow<'static, str>), } impl Error { - pub fn bad_database(message: &'static str) -> Self { - error!("BadDatabase: {}", message); - Self::BadDatabase(message) - } + pub fn bad_database(message: &'static str) -> Self { crate::err!(Database(error!("{message}"))) } /// Sanitizes public-facing errors that can leak sensitive information. pub fn sanitized_string(&self) -> String { @@ -121,7 +118,7 @@ impl Error { match self { Self::Federation(_, error) => response::ruma_error_kind(error).clone(), - Self::BadRequest(kind, _) => kind.clone(), + Self::BadRequest(kind, _) | Self::Request(kind, _) => kind.clone(), _ => Unknown, } } @@ -129,7 +126,7 @@ impl Error { pub fn status_code(&self) -> http::StatusCode { match self { Self::Federation(_, ref error) | Self::RumaError(ref error) => error.status_code, - Self::BadRequest(ref kind, _) => response::bad_request_code(kind), + Self::BadRequest(ref kind, _) | Self::Request(ref kind, _) => response::bad_request_code(kind), Self::Conflict(_) => http::StatusCode::CONFLICT, _ => http::StatusCode::INTERNAL_SERVER_ERROR, } diff --git a/src/core/server.rs b/src/core/server.rs index 575924d3..752680ad 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -5,7 +5,7 @@ use std::{ use tokio::{runtime, sync::broadcast}; -use crate::{config::Config, log, Error, Result}; +use crate::{config::Config, log, Err, Result}; /// Server runtime state; public portion pub struct Server { @@ -65,15 +65,15 @@ impl Server { pub fn reload(&self) -> Result<()> { if cfg!(not(conduit_mods)) { - return Err(Error::Err("Reloading not enabled".into())); + return Err!("Reloading not enabled"); } if self.reloading.swap(true, Ordering::AcqRel) { - return Err(Error::Err("Reloading already in progress".into())); + return Err!("Reloading already in progress"); } if self.stopping.swap(true, Ordering::AcqRel) { - return Err(Error::Err("Shutdown already in progress".into())); + return Err!("Shutdown already in progress"); } self.signal("SIGINT").inspect_err(|_| { @@ -84,7 +84,7 @@ impl Server { pub fn restart(&self) -> Result<()> { if self.restarting.swap(true, Ordering::AcqRel) { - return Err(Error::Err("Restart already in progress".into())); + return Err!("Restart already in progress"); } self.shutdown() @@ -93,7 +93,7 @@ impl Server { pub fn shutdown(&self) -> Result<()> { if self.stopping.swap(true, Ordering::AcqRel) { - return Err(Error::Err("Shutdown already in progress".into())); + return Err!("Shutdown already in progress"); } self.signal("SIGTERM") @@ -102,7 +102,7 @@ impl Server { pub fn signal(&self, sig: &'static str) -> Result<()> { if let Err(e) = self.signal.send(sig) { - return Err(Error::Err(format!("Failed to send signal: {e}"))); + return Err!("Failed to send signal: {e}"); } Ok(()) diff --git a/src/core/utils/hash/argon.rs b/src/core/utils/hash/argon.rs index 98cef00e..0a1e1e14 100644 --- a/src/core/utils/hash/argon.rs +++ b/src/core/utils/hash/argon.rs @@ -5,7 +5,7 @@ use argon2::{ PasswordVerifier, Version, }; -use crate::{Error, Result}; +use crate::{err, Error, Result}; const M_COST: u32 = Params::DEFAULT_M_COST; // memory size in 1 KiB blocks const T_COST: u32 = Params::DEFAULT_T_COST; // nr of iterations @@ -44,7 +44,7 @@ pub(super) fn verify_password(password: &str, password_hash: &str) -> Result<()> .map_err(map_err) } -fn map_err(e: password_hash::Error) -> Error { Error::Err(e.to_string()) } +fn map_err(e: password_hash::Error) -> Error { err!("{e}") } #[cfg(test)] mod tests { diff --git a/src/database/util.rs b/src/database/util.rs index 513cedc8..f0ccbcbe 100644 --- a/src/database/util.rs +++ b/src/database/util.rs @@ -1,4 +1,4 @@ -use conduit::Result; +use conduit::{err, Result}; #[inline] pub(crate) fn result(r: std::result::Result) -> Result { @@ -10,4 +10,7 @@ pub(crate) fn and_then(t: T) -> Result { Ok(t) } pub(crate) fn or_else(e: rocksdb::Error) -> Result { Err(map_err(e)) } -pub(crate) fn map_err(e: rocksdb::Error) -> conduit::Error { conduit::Error::Database(e.into_string()) } +pub(crate) fn map_err(e: rocksdb::Error) -> conduit::Error { + let string = e.into_string(); + err!(Database(error!("{string}"))) +} diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index e876b2ac..266936dd 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -10,7 +10,7 @@ use axum::{ extract::{connect_info::IntoMakeServiceWithConnectInfo, Request}, Router, }; -use conduit::{debug_error, error::infallible, trace, Error, Result, Server}; +use conduit::{debug, debug_error, error::infallible, info, trace, warn, Err, Result, Server}; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, @@ -23,7 +23,6 @@ use tokio::{ task::JoinSet, }; use tower::{Service, ServiceExt}; -use tracing::{debug, info, warn}; type MakeService = IntoMakeServiceWithConnectInfo; @@ -97,19 +96,19 @@ async fn init(server: &Arc) -> Result { let dir = path.parent().unwrap_or_else(|| Path::new("/")); if let Err(e) = fs::create_dir_all(dir).await { - return Err(Error::Err(format!("Failed to create {dir:?} for socket {path:?}: {e}"))); + return Err!("Failed to create {dir:?} for socket {path:?}: {e}"); } let listener = UnixListener::bind(path); if let Err(e) = listener { - return Err(Error::Err(format!("Failed to bind listener {path:?}: {e}"))); + return Err!("Failed to bind listener {path:?}: {e}"); } let socket_perms = config.unix_socket_perms.to_string(); let octal_perms = u32::from_str_radix(&socket_perms, 8).expect("failed to convert octal permissions"); let perms = std::fs::Permissions::from_mode(octal_perms); if let Err(e) = fs::set_permissions(&path, perms).await { - return Err(Error::Err(format!("Failed to set socket {path:?} permissions: {e}"))); + return Err!("Failed to set socket {path:?} permissions: {e}"); } info!("Listening at {:?}", path); diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index f7cf0b6b..24c9b8b0 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -2,7 +2,7 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::Result; +use conduit::{err, Result}; use data::Data; use futures_util::Future; use regex::RegexSet; @@ -171,7 +171,7 @@ impl Service { .write() .await .remove(service_name) - .ok_or_else(|| crate::Error::Err("Appservice not found".to_owned()))?; + .ok_or(err!("Appservice not found"))?; // remove the appservice from the database self.db.unregister_appservice(service_name)?; diff --git a/src/service/manager.rs b/src/service/manager.rs index 84a28a76..af59b4a4 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -1,6 +1,6 @@ use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; -use conduit::{debug, debug_warn, error, trace, utils::time, warn, Error, Result, Server}; +use conduit::{debug, debug_warn, error, trace, utils::time, warn, Err, Error, Result, Server}; use futures_util::FutureExt; use tokio::{ sync::{Mutex, MutexGuard}, @@ -129,10 +129,7 @@ impl Manager { /// Start the worker in a task for the service. async fn start_worker(&self, workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { if !self.server.running() { - return Err(Error::Err(format!( - "Service {:?} worker not starting during server shutdown.", - service.name() - ))); + return Err!("Service {:?} worker not starting during server shutdown.", service.name()); } debug!("Service {:?} worker starting...", service.name()); diff --git a/src/service/mod.rs b/src/service/mod.rs index 6f2f4ee5..81e0be3b 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -24,7 +24,7 @@ extern crate conduit_database as database; use std::sync::{Arc, RwLock}; -pub(crate) use conduit::{config, debug_error, debug_info, debug_warn, utils, Config, Error, Result, Server}; +pub(crate) use conduit::{config, debug_error, debug_warn, utils, Config, Error, Result, Server}; pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; use database::Database; pub(crate) use service::{Args, Service}; diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 97eb0f48..792f5c98 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -171,7 +171,7 @@ impl Service { .rooms .alias .resolve_local_alias(room_alias)? - .ok_or_else(|| err!(BadConfig("Room does not exist.")))?, + .ok_or_else(|| err!(Request(NotFound("Room does not exist."))))?, )); } } diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 2919ea7e..4e8c7bb2 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -5,9 +5,9 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, trace, validated, warn, Error, Result}; +use conduit::{debug, error, trace, validated, warn, Err, Result}; use data::Data; -use ruma::{api::client::error::ErrorKind, EventId, RoomId}; +use ruma::{EventId, RoomId}; use crate::services; @@ -143,8 +143,11 @@ impl Service { match services().rooms.timeline.get_pdu(&event_id) { Ok(Some(pdu)) => { if pdu.room_id != room_id { - error!(?event_id, ?pdu, "auth event for incorrect room_id"); - return Err(Error::BadRequest(ErrorKind::forbidden(), "Evil event in db")); + return Err!(Request(Forbidden( + "auth event {event_id:?} for incorrect room {} which is not {}", + pdu.room_id, + room_id + ))); } for auth_event in &pdu.auth_events { let sauthevent = services() diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 33c00643..6cb23b9f 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -10,7 +10,7 @@ use std::{ }; use conduit::{ - debug, debug_error, debug_info, error, info, trace, + debug, debug_error, debug_info, err, error, info, trace, utils::{math::continue_exponential_backoff_secs, MutexMap}, warn, Error, Result, }; @@ -1382,11 +1382,8 @@ impl Service { } fn get_room_version_id(create_event: &PduEvent) -> Result { - let create_event_content: RoomCreateEventContent = - serde_json::from_str(create_event.content.get()).map_err(|e| { - error!("Invalid create event: {}", e); - Error::BadDatabase("Invalid create event in db") - })?; + let create_event_content: RoomCreateEventContent = serde_json::from_str(create_event.content.get()) + .map_err(|e| err!(Database("Invalid create event: {e}")))?; Ok(create_event_content.room_version) } diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 4c907e51..8fcd8549 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,4 +1,4 @@ -use conduit::{Error, Result}; +use conduit::{Err, Error, Result}; use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId}; use serde_json::value::RawValue as RawJsonValue; use tracing::warn; @@ -17,15 +17,12 @@ pub fn parse_incoming_pdu(pdu: &RawJsonValue) -> Result<(OwnedEventId, Canonical .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid room id in pdu"))?; let Ok(room_version_id) = services().rooms.state.get_room_version(&room_id) else { - return Err(Error::Err(format!("Server is not in room {room_id}"))); + return Err!("Server is not in room {room_id}"); }; let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); + return Err!(Request(InvalidParam("Could not convert event to canonical json."))); }; Ok((event_id, value, room_id)) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 03d0d43f..18133fc1 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::Arc, }; -use conduit::{checked, debug_info, utils::math::usize_from_f64}; +use conduit::{checked, debug, debug_info, err, utils::math::usize_from_f64, warn, Error, Result}; use lru_cache::LruCache; use ruma::{ api::{ @@ -27,9 +27,8 @@ use ruma::{ OwnedRoomId, OwnedServerName, RoomId, ServerName, UInt, UserId, }; use tokio::sync::Mutex; -use tracing::{debug, error, warn}; -use crate::{services, Error, Result}; +use crate::services; pub struct CachedSpaceHierarchySummary { summary: SpaceHierarchyParentSummary, @@ -380,10 +379,7 @@ impl Service { .map(|s| { serde_json::from_str(s.content.get()) .map(|c: RoomJoinRulesEventContent| c.join_rule) - .map_err(|e| { - error!("Invalid room join rule event in database: {}", e); - Error::BadDatabase("Invalid room join rule event in database.") - }) + .map_err(|e| err!(Database(error!("Invalid room join rule event in database: {e}")))) }) .transpose()? .unwrap_or(JoinRule::Invite); diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 389f12c3..7abe5e0f 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -6,7 +6,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{error, utils::math::usize_from_f64, warn, Error, Result}; +use conduit::{err, error, utils::math::usize_from_f64, warn, Error, Result}; use data::Data; use lru_cache::LruCache; use ruma::{ @@ -454,10 +454,7 @@ impl Service { .map(|c: RoomJoinRulesEventContent| { (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule)) }) - .map_err(|e| { - error!("Invalid room join rule event in database: {e}"); - Error::BadDatabase("Invalid room join rule event in database.") - }) + .map_err(|e| err!(Database(error!("Invalid room join rule event in database: {e}")))) }) .transpose()? .unwrap_or((SpaceRoomJoinRule::Invite, vec![]))) @@ -483,10 +480,8 @@ impl Service { Ok(self .room_state_get(room_id, &StateEventType::RoomCreate, "")? .map(|s| { - serde_json::from_str::(s.content.get()).map_err(|e| { - error!("Invalid room create event in database: {e}"); - Error::BadDatabase("Invalid room create event in database.") - }) + serde_json::from_str::(s.content.get()) + .map_err(|e| err!(Database(error!("Invalid room create event in database: {e}")))) }) .transpose()? .and_then(|e| e.room_type)) @@ -499,10 +494,7 @@ impl Service { .map_or(Ok(None), |s| { serde_json::from_str::(s.content.get()) .map(|content| Some(content.algorithm)) - .map_err(|e| { - error!("Invalid room encryption event in database: {e}"); - Error::BadDatabase("Invalid room encryption event in database.") - }) + .map_err(|e| err!(Database(error!("Invalid room encryption event in database: {e}")))) }) } } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 30fd7bf4..48215817 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -2,7 +2,7 @@ mod data; use std::sync::Arc; -use conduit::{error, warn, Error, Result}; +use conduit::{err, error, warn, Error, Result}; use data::Data; use itertools::Itertools; use ruma::{ @@ -128,10 +128,8 @@ impl Service { .account_data .get(Some(&predecessor.room_id), user_id, RoomAccountDataEventType::Tag)? .map(|event| { - serde_json::from_str(event.get()).map_err(|e| { - warn!("Invalid account data event in db: {e:?}"); - Error::BadDatabase("Invalid account data event in db.") - }) + serde_json::from_str(event.get()) + .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) { services() .account_data @@ -144,10 +142,8 @@ impl Service { .account_data .get(None, user_id, GlobalAccountDataEventType::Direct.to_string().into())? .map(|event| { - serde_json::from_str::(event.get()).map_err(|e| { - warn!("Invalid account data event in db: {e:?}"); - Error::BadDatabase("Invalid account data event in db.") - }) + serde_json::from_str::(event.get()) + .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) { let mut direct_event = direct_event?; let mut room_ids_updated = false; @@ -185,10 +181,8 @@ impl Service { .into(), )? .map(|event| { - serde_json::from_str::(event.get()).map_err(|e| { - warn!("Invalid account data event in db: {e:?}"); - Error::BadDatabase("Invalid account data event in db.") - }) + serde_json::from_str::(event.get()) + .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) .transpose()? .map_or(false, |ignored| { diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index eb708fcf..88b8b189 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -6,7 +6,7 @@ mod sender; use std::fmt::Debug; -use conduit::{Error, Result}; +use conduit::{err, Result}; pub use resolve::{resolve_actual_dest, CachedDest, CachedOverride, FedDest}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, @@ -224,7 +224,7 @@ impl Service { fn dispatch(&self, msg: Msg) -> Result<()> { debug_assert!(!self.sender.is_full(), "channel full"); debug_assert!(!self.sender.is_closed(), "channel closed"); - self.sender.send(msg).map_err(|e| Error::Err(e.to_string())) + self.sender.send(msg).map_err(|e| err!("{e}")) } } diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 91041a2f..8b6bfc95 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -5,13 +5,12 @@ use std::{ time::SystemTime, }; -use conduit::Err; +use conduit::{debug, debug_error, debug_info, debug_warn, trace, utils::rand, Err, Error, Result}; use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; use ruma::{OwnedServerName, ServerName}; -use tracing::{debug, error, trace}; -use crate::{debug_error, debug_info, debug_warn, services, utils::rand, Error, Result}; +use crate::services; /// Wraps either an literal IP address plus port, or a hostname plus complement /// (colon-plus-port if it was specified). @@ -346,10 +345,7 @@ fn handle_resolve_error(e: &ResolveError) -> Result<()> { debug!("{e}"); Ok(()) }, - _ => { - error!("DNS {e}"); - Err(Error::Err(e.to_string())) - }, + _ => Err!(error!("DNS {e}")), } } diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index a7088aba..3fb680d6 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -72,7 +72,7 @@ impl Service { .await?; let response = serde_json::from_str::(&response.text().await?) - .map_err(|e| Error::Err(format!("Bad check for updates response: {e}")))?; + .map_err(|e| err!("Bad check for updates response: {e}"))?; let mut last_update_id = self.last_check_for_updates_id()?; for update in response.updates { From d2fb6d04c9cb6583814f86366de389fc02e3cf73 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 01:11:03 +0000 Subject: [PATCH 0485/2091] cleanup pending transactions before sender worker completes Signed-off-by: Jason Volk --- src/service/sending/sender.rs | 40 +++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index cfd5b4bc..2f542dfe 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -3,12 +3,12 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, sync::Arc, - time::Instant, + time::{Duration, Instant}, }; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, error, utils::math::continue_exponential_backoff_secs, warn}; +use conduit::{debug, debug_warn, error, trace, utils::math::continue_exponential_backoff_secs, warn}; use federation::transactions::send_transaction_message; use futures_util::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; use ruma::{ @@ -24,7 +24,7 @@ use ruma::{ ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::sync::Mutex; +use tokio::{sync::Mutex, time::sleep_until}; use super::{appservice, data::Data, send, Destination, Msg, SendingEvent, Service}; use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; @@ -44,6 +44,7 @@ type CurTransactionStatus = HashMap; const DEQUEUE_LIMIT: usize = 48; const SELECT_EDU_LIMIT: usize = 16; +const CLEANUP_TIMEOUT_MS: u64 = 3500; #[async_trait] impl crate::Service for Service { @@ -65,19 +66,22 @@ impl crate::Service for Service { let mut futures: SendingFutures<'_> = FuturesUnordered::new(); let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); - self.initial_transactions(&futures, &mut statuses); + self.initial_requests(&futures, &mut statuses); loop { debug_assert!(!receiver.is_closed(), "channel error"); tokio::select! { request = receiver.recv_async() => match request { Ok(request) => self.handle_request(request, &futures, &mut statuses), - Err(_) => return Ok(()), + Err(_) => break, }, Some(response) = futures.next() => { - self.handle_response(response, &mut futures, &mut statuses); + self.handle_response(response, &futures, &mut statuses); }, } } + self.finish_responses(&mut futures, &mut statuses).await; + + Ok(()) } fn interrupt(&self) { @@ -91,7 +95,7 @@ impl crate::Service for Service { impl Service { fn handle_response( - &self, response: SendingResult, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, + &self, response: SendingResult, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, ) { match response { Ok(dest) => self.handle_response_ok(&dest, futures, statuses), @@ -100,7 +104,7 @@ impl Service { } fn handle_response_err( - dest: Destination, _futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus, e: &Error, + dest: Destination, _futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, e: &Error, ) { debug!(dest = ?dest, "{e:?}"); statuses.entry(dest).and_modify(|e| { @@ -151,7 +155,25 @@ impl Service { } } - fn initial_transactions(&self, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus) { + async fn finish_responses(&self, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus) { + let now = Instant::now(); + let timeout = Duration::from_millis(CLEANUP_TIMEOUT_MS); + let deadline = now.checked_add(timeout).unwrap_or(now); + loop { + trace!("Waiting for {} requests to complete...", futures.len()); + tokio::select! { + () = sleep_until(deadline.into()) => break, + response = futures.next() => match response { + Some(response) => self.handle_response(response, futures, statuses), + None => return, + } + } + } + + debug_warn!("Leaving with {} unfinished requests...", futures.len()); + } + + fn initial_requests(&self, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus) { let keep = usize::try_from(self.startup_netburst_keep).unwrap_or(usize::MAX); let mut txns = HashMap::>::new(); for (key, dest, event) in self.db.active_requests().filter_map(Result::ok) { From e4dc4a1ba59d09ba037a7160ae0e39c0eeb9cc03 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 06:07:54 +0000 Subject: [PATCH 0486/2091] fix graceful shutdown on unix socket Signed-off-by: Jason Volk --- src/router/serve/unix.rs | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 266936dd..b5938673 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -3,7 +3,7 @@ use std::{ net::{self, IpAddr, Ipv4Addr}, path::Path, - sync::Arc, + sync::{atomic::Ordering, Arc}, }; use axum::{ @@ -21,12 +21,14 @@ use tokio::{ net::{unix::SocketAddr, UnixListener, UnixStream}, sync::broadcast::{self}, task::JoinSet, + time::{sleep, Duration}, }; use tower::{Service, ServiceExt}; type MakeService = IntoMakeServiceWithConnectInfo; -static NULL_ADDR: net::SocketAddr = net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); +const NULL_ADDR: net::SocketAddr = net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); +const FINI_POLL_INTERVAL: Duration = Duration::from_millis(750); #[tracing::instrument(skip_all)] pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broadcast::Receiver<()>) -> Result<()> { @@ -47,7 +49,7 @@ pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broad } } - fini(listener, tasks).await; + fini(server, listener, tasks).await; Ok(()) } @@ -111,15 +113,26 @@ async fn init(server: &Arc) -> Result { return Err!("Failed to set socket {path:?} permissions: {e}"); } - info!("Listening at {:?}", path); + info!("Listening at {path:?}"); Ok(listener.unwrap()) } -async fn fini(listener: UnixListener, mut tasks: JoinSet<()>) { +async fn fini(server: &Arc, listener: UnixListener, mut tasks: JoinSet<()>) { let local = listener.local_addr(); + debug!("Closing listener at {local:?} ..."); drop(listener); + + debug!("Waiting for requests to finish..."); + while server.metrics.requests_spawn_active.load(Ordering::Relaxed) > 0 { + tokio::select! { + _ = tasks.join_next() => {} + () = sleep(FINI_POLL_INTERVAL) => {} + } + } + + debug!("Shutting down..."); tasks.shutdown().await; if let Ok(local) = local { From 5ec49b3f622be21ebb7a68a2f712aab230111097 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 05:11:58 +0000 Subject: [PATCH 0487/2091] split csp into array; integrate error; cleanup type Signed-off-by: Jason Volk --- src/core/error/mod.rs | 2 ++ src/router/layers.rs | 24 ++++++++++++++++-------- src/router/request.rs | 19 +++++++++---------- 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 3adfde97..069fe60e 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -53,6 +53,8 @@ pub enum Error { Path(#[from] axum::extract::rejection::PathRejection), #[error("{0}")] Http(#[from] http::Error), + #[error("{0}")] + HttpHeader(#[from] http::header::InvalidHeaderValue), // ruma #[error("{0}")] diff --git a/src/router/layers.rs b/src/router/layers.rs index db664b38..8c2e114b 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -1,11 +1,11 @@ -use std::{any::Any, io, sync::Arc, time::Duration}; +use std::{any::Any, sync::Arc, time::Duration}; use axum::{ extract::{DefaultBodyLimit, MatchedPath}, Router, }; use axum_client_ip::SecureClientIpSource; -use conduit::Server; +use conduit::{Result, Server}; use http::{ header::{self, HeaderName}, HeaderValue, Method, StatusCode, @@ -22,11 +22,19 @@ use tracing::Level; use crate::{request, router}; -const CONDUWUIT_CSP: &str = "sandbox; default-src 'none'; font-src 'none'; script-src 'none'; frame-ancestors 'none'; \ - form-action 'none'; base-uri 'none';"; -const CONDUWUIT_PERMISSIONS_POLICY: &str = "interest-cohort=(),browsing-topics=()"; +const CONDUWUIT_CSP: &[&str] = &[ + "sandbox", + "default-src 'none'", + "font-src 'none'", + "script-src 'none'", + "frame-ancestors 'none'", + "form-action 'none'", + "base-uri 'none'", +]; -pub(crate) fn build(server: &Arc) -> io::Result { +const CONDUWUIT_PERMISSIONS_POLICY: &[&str] = &["interest-cohort=()", "browsing-topics=()"]; + +pub(crate) fn build(server: &Arc) -> Result { let layers = ServiceBuilder::new(); #[cfg(feature = "sentry_telemetry")] @@ -65,11 +73,11 @@ pub(crate) fn build(server: &Arc) -> io::Result { )) .layer(SetResponseHeaderLayer::if_not_present( HeaderName::from_static("permissions-policy"), - HeaderValue::from_static(CONDUWUIT_PERMISSIONS_POLICY), + HeaderValue::from_str(&CONDUWUIT_PERMISSIONS_POLICY.join(","))?, )) .layer(SetResponseHeaderLayer::if_not_present( header::CONTENT_SECURITY_POLICY, - HeaderValue::from_static(CONDUWUIT_CSP), + HeaderValue::from_str(&CONDUWUIT_CSP.join("; "))?, )) .layer(cors_layer(server)) .layer(body_limit_layer(server)) diff --git a/src/router/request.rs b/src/router/request.rs index 9256fb9c..851bd168 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -1,6 +1,9 @@ use std::sync::{atomic::Ordering, Arc}; -use axum::{extract::State, response::IntoResponse}; +use axum::{ + extract::State, + response::{IntoResponse, Response}, +}; use conduit::{debug, debug_error, debug_warn, defer, error, trace, Error, Result, Server}; use http::{Method, StatusCode, Uri}; use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind}; @@ -8,7 +11,7 @@ use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind}; #[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( State(server): State>, req: http::Request, next: axum::middleware::Next, -) -> Result { +) -> Result { if !server.running() { debug_warn!("unavailable pending shutdown"); return Err(StatusCode::SERVICE_UNAVAILABLE); @@ -30,7 +33,7 @@ pub(crate) async fn spawn( #[tracing::instrument(skip_all, name = "handle")] pub(crate) async fn handle( State(server): State>, req: http::Request, next: axum::middleware::Next, -) -> Result { +) -> Result { if !server.running() { debug_warn!( method = %req.method(), @@ -57,9 +60,7 @@ pub(crate) async fn handle( handle_result(&method, &uri, result) } -fn handle_result( - method: &Method, uri: &Uri, result: axum::response::Response, -) -> Result { +fn handle_result(method: &Method, uri: &Uri, result: Response) -> Result { handle_result_log(method, uri, &result); match result.status() { StatusCode::METHOD_NOT_ALLOWED => handle_result_405(method, uri, &result), @@ -67,9 +68,7 @@ fn handle_result( } } -fn handle_result_405( - _method: &Method, _uri: &Uri, result: &axum::response::Response, -) -> Result { +fn handle_result_405(_method: &Method, _uri: &Uri, result: &Response) -> Result { let error = Error::RumaError(RumaError { status_code: result.status(), body: ErrorBody::Standard { @@ -81,7 +80,7 @@ fn handle_result_405( Ok(error.into_response()) } -fn handle_result_log(method: &Method, uri: &Uri, result: &axum::response::Response) { +fn handle_result_log(method: &Method, uri: &Uri, result: &Response) { let status = result.status(); let reason = status.canonical_reason().unwrap_or("Unknown Reason"); let code = status.as_u16(); From cce270d93823859daecb25d4b9682fcaf494943c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 25 Jun 2024 05:05:02 +0000 Subject: [PATCH 0488/2091] tokio metrics Signed-off-by: Jason Volk --- Cargo.lock | 2 ++ Cargo.toml | 7 ++-- src/admin/debug/commands.rs | 34 ++++++++++++++++++ src/admin/debug/mod.rs | 9 +++++ src/core/Cargo.toml | 1 + src/core/metrics/mod.rs | 72 +++++++++++++++++++++++++++++++++++++ src/core/mod.rs | 1 + src/core/server.rs | 24 +++++-------- src/main/main.rs | 4 +-- src/router/layers.rs | 1 + src/router/request.rs | 14 +++++--- src/router/run.rs | 2 +- src/router/serve/plain.rs | 19 +++++++--- 13 files changed, 157 insertions(+), 33 deletions(-) create mode 100644 src/core/metrics/mod.rs diff --git a/Cargo.lock b/Cargo.lock index f2036284..48d1a91f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -694,6 +694,7 @@ dependencies = [ "tikv-jemalloc-sys", "tikv-jemallocator", "tokio", + "tokio-metrics", "tracing", "tracing-core", "tracing-subscriber", @@ -4058,6 +4059,7 @@ checksum = "eace09241d62c98b7eeb1107d4c5c64ca3bd7da92e8c218c153ab3a78f9be112" dependencies = [ "futures-util", "pin-project-lite", + "tokio", "tokio-stream", ] diff --git a/Cargo.toml b/Cargo.toml index 33e2b9d9..e24d84b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -200,6 +200,9 @@ features = [ "io-util", ] +[workspace.dependencies.tokio-metrics] +version = "0.3.1" + [workspace.dependencies.libloading] version = "0.8.3" @@ -382,10 +385,6 @@ version = "0.5.4" default-features = false features = ["use_std"] -[workspace.dependencies.tokio-metrics] -version = "0.3.1" -default-features = false - [workspace.dependencies.console-subscriber] version = "0.3" diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index a5087467..46f71622 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -653,3 +653,37 @@ pub(super) fn memory_stats() -> RoomMessageEventContent { html_body.expect("string result"), ) } + +#[cfg(tokio_unstable)] +pub(super) async fn runtime_metrics(_body: Vec<&str>) -> Result { + let out = services().server.metrics.runtime_metrics().map_or_else( + || "Runtime metrics are not available.".to_owned(), + |metrics| format!("```rs\n{metrics:#?}\n```"), + ); + + Ok(RoomMessageEventContent::text_markdown(out)) +} + +#[cfg(not(tokio_unstable))] +pub(super) async fn runtime_metrics(_body: Vec<&str>) -> Result { + Ok(RoomMessageEventContent::text_markdown( + "Runtime metrics require building with `tokio_unstable`.", + )) +} + +#[cfg(tokio_unstable)] +pub(super) async fn runtime_interval(_body: Vec<&str>) -> Result { + let out = services().server.metrics.runtime_interval().map_or_else( + || "Runtime metrics are not available.".to_owned(), + |metrics| format!("```rs\n{metrics:#?}\n```"), + ); + + Ok(RoomMessageEventContent::text_markdown(out)) +} + +#[cfg(not(tokio_unstable))] +pub(super) async fn runtime_interval(_body: Vec<&str>) -> Result { + Ok(RoomMessageEventContent::text_markdown( + "Runtime metrics require building with `tokio_unstable`.", + )) +} diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index eed3b633..7d6cafa7 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -160,6 +160,13 @@ pub(super) enum DebugCommand { /// - Print extended memory usage MemoryStats, + /// - Print general tokio runtime metric totals. + RuntimeMetrics, + + /// - Print detailed tokio runtime metrics accumulated since last command + /// invocation. + RuntimeInterval, + /// - Developer test stubs #[command(subcommand)] Tester(TesterCommand), @@ -213,6 +220,8 @@ pub(super) async fn process(command: DebugCommand, body: Vec<&str>) -> Result resolve_true_destination(body, server_name, no_cache).await?, DebugCommand::MemoryStats => memory_stats(), + DebugCommand::RuntimeMetrics => runtime_metrics(body).await?, + DebugCommand::RuntimeInterval => runtime_interval(body).await?, DebugCommand::Tester(command) => tester::process(command, body).await?, }) } diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 73ee0152..453d7b13 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -82,6 +82,7 @@ tikv-jemalloc-ctl.workspace = true tikv-jemalloc-sys.optional = true tikv-jemalloc-sys.workspace = true tokio.workspace = true +tokio-metrics.workspace = true tracing-core.workspace = true tracing-subscriber.workspace = true tracing.workspace = true diff --git a/src/core/metrics/mod.rs b/src/core/metrics/mod.rs new file mode 100644 index 00000000..3ae139a8 --- /dev/null +++ b/src/core/metrics/mod.rs @@ -0,0 +1,72 @@ +use std::sync::atomic::AtomicU32; + +use tokio::runtime; +use tokio_metrics::TaskMonitor; +#[cfg(tokio_unstable)] +use tokio_metrics::{RuntimeIntervals, RuntimeMonitor}; + +pub struct Metrics { + _runtime: Option, + + runtime_metrics: Option, + + task_monitor: Option, + + #[cfg(tokio_unstable)] + _runtime_monitor: Option, + + #[cfg(tokio_unstable)] + runtime_intervals: std::sync::Mutex>, + + // TODO: move stats + pub requests_spawn_active: AtomicU32, + pub requests_spawn_finished: AtomicU32, + pub requests_handle_active: AtomicU32, + pub requests_handle_finished: AtomicU32, + pub requests_panic: AtomicU32, +} + +impl Metrics { + #[must_use] + pub fn new(runtime: Option) -> Self { + #[cfg(tokio_unstable)] + let runtime_monitor = runtime.as_ref().map(RuntimeMonitor::new); + + #[cfg(tokio_unstable)] + let runtime_intervals = runtime_monitor.as_ref().map(RuntimeMonitor::intervals); + + Self { + _runtime: runtime.clone(), + + runtime_metrics: runtime.as_ref().map(runtime::Handle::metrics), + + task_monitor: runtime.map(|_| TaskMonitor::new()), + + #[cfg(tokio_unstable)] + _runtime_monitor: runtime_monitor, + + #[cfg(tokio_unstable)] + runtime_intervals: std::sync::Mutex::new(runtime_intervals), + + requests_spawn_active: AtomicU32::new(0), + requests_spawn_finished: AtomicU32::new(0), + requests_handle_active: AtomicU32::new(0), + requests_handle_finished: AtomicU32::new(0), + requests_panic: AtomicU32::new(0), + } + } + + #[cfg(tokio_unstable)] + pub fn runtime_interval(&self) -> Option { + self.runtime_intervals + .lock() + .expect("locked") + .as_mut() + .map(Iterator::next) + .expect("next interval") + } + + pub fn task_root(&self) -> Option<&TaskMonitor> { self.task_monitor.as_ref() } + + pub fn runtime_metrics(&self) -> Option<&runtime::RuntimeMetrics> { self.runtime_metrics.as_ref() } +} diff --git a/src/core/mod.rs b/src/core/mod.rs index de8057fa..9716b46e 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -3,6 +3,7 @@ pub mod config; pub mod debug; pub mod error; pub mod log; +pub mod metrics; pub mod mods; pub mod pdu; pub mod server; diff --git a/src/core/server.rs b/src/core/server.rs index 752680ad..bf0ab99d 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -1,11 +1,11 @@ use std::{ - sync::atomic::{AtomicBool, AtomicU32, Ordering}, + sync::atomic::{AtomicBool, Ordering}, time::SystemTime, }; use tokio::{runtime, sync::broadcast}; -use crate::{config::Config, log, Err, Result}; +use crate::{config::Config, log::Log, metrics::Metrics, Err, Result}; /// Server runtime state; public portion pub struct Server { @@ -33,33 +33,25 @@ pub struct Server { pub signal: broadcast::Sender<&'static str>, /// Logging subsystem state - pub log: log::Log, + pub log: Log, - /// TODO: move stats - pub requests_spawn_active: AtomicU32, - pub requests_spawn_finished: AtomicU32, - pub requests_handle_active: AtomicU32, - pub requests_handle_finished: AtomicU32, - pub requests_panic: AtomicU32, + /// Metrics subsystem state + pub metrics: Metrics, } impl Server { #[must_use] - pub fn new(config: Config, runtime: Option, log: log::Log) -> Self { + pub fn new(config: Config, runtime: Option, log: Log) -> Self { Self { config, started: SystemTime::now(), stopping: AtomicBool::new(false), reloading: AtomicBool::new(false), restarting: AtomicBool::new(false), - runtime, + runtime: runtime.clone(), signal: broadcast::channel::<&'static str>(1).0, log, - requests_spawn_active: AtomicU32::new(0), - requests_spawn_finished: AtomicU32::new(0), - requests_handle_active: AtomicU32::new(0), - requests_handle_finished: AtomicU32::new(0), - requests_panic: AtomicU32::new(0), + metrics: Metrics::new(runtime), } } diff --git a/src/main/main.rs b/src/main/main.rs index 23f53b4e..959e8610 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -20,7 +20,7 @@ use tokio::runtime; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; -const WORKER_KEEPALIVE_MS: u64 = 2500; +const WORKER_KEEPALIVE: u64 = 36; fn main() -> Result<(), Error> { let args = clap::parse(); @@ -29,7 +29,7 @@ fn main() -> Result<(), Error> { .enable_time() .thread_name(WORKER_NAME) .worker_threads(cmp::max(WORKER_MIN, available_parallelism())) - .thread_keep_alive(Duration::from_millis(WORKER_KEEPALIVE_MS)) + .thread_keep_alive(Duration::from_secs(WORKER_KEEPALIVE)) .build() .expect("built runtime"); diff --git a/src/router/layers.rs b/src/router/layers.rs index 8c2e114b..073940f1 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -153,6 +153,7 @@ fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max fn catch_panic(err: Box) -> http::Response> { conduit_service::services() .server + .metrics .requests_panic .fetch_add(1, std::sync::atomic::Ordering::Release); diff --git a/src/router/request.rs b/src/router/request.rs index 851bd168..ae739984 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -17,11 +17,14 @@ pub(crate) async fn spawn( return Err(StatusCode::SERVICE_UNAVAILABLE); } - let active = server.requests_spawn_active.fetch_add(1, Ordering::Relaxed); + let active = server + .metrics + .requests_spawn_active + .fetch_add(1, Ordering::Relaxed); trace!(active, "enter"); defer! {{ - let active = server.requests_spawn_active.fetch_sub(1, Ordering::Relaxed); - let finished = server.requests_spawn_finished.fetch_add(1, Ordering::Relaxed); + let active = server.metrics.requests_spawn_active.fetch_sub(1, Ordering::Relaxed); + let finished = server.metrics.requests_spawn_finished.fetch_add(1, Ordering::Relaxed); trace!(active, finished, "leave"); }}; @@ -45,12 +48,13 @@ pub(crate) async fn handle( } let active = server + .metrics .requests_handle_active .fetch_add(1, Ordering::Relaxed); trace!(active, "enter"); defer! {{ - let active = server.requests_handle_active.fetch_sub(1, Ordering::Relaxed); - let finished = server.requests_handle_finished.fetch_add(1, Ordering::Relaxed); + let active = server.metrics.requests_handle_active.fetch_sub(1, Ordering::Relaxed); + let finished = server.metrics.requests_handle_finished.fetch_add(1, Ordering::Relaxed); trace!(active, finished, "leave"); }}; diff --git a/src/router/run.rs b/src/router/run.rs index b3ec2285..91507772 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -108,7 +108,7 @@ async fn handle_shutdown(server: &Arc, tx: &Sender<()>, handle: &axum_se error!("failed sending shutdown transaction to channel: {e}"); } - let pending = server.requests_spawn_active.load(Ordering::Relaxed); + let pending = server.metrics.requests_spawn_active.load(Ordering::Relaxed); if pending > 0 { let timeout = Duration::from_secs(36); trace!(pending, ?timeout, "Notifying for graceful shutdown"); diff --git a/src/router/serve/plain.rs b/src/router/serve/plain.rs index b79d342d..08263353 100644 --- a/src/router/serve/plain.rs +++ b/src/router/serve/plain.rs @@ -21,12 +21,21 @@ pub(super) async fn serve( info!("Listening on {addrs:?}"); while join_set.join_next().await.is_some() {} - let spawn_active = server.requests_spawn_active.load(Ordering::Relaxed); - let handle_active = server.requests_handle_active.load(Ordering::Relaxed); + let spawn_active = server.metrics.requests_spawn_active.load(Ordering::Relaxed); + let handle_active = server + .metrics + .requests_handle_active + .load(Ordering::Relaxed); debug_info!( - spawn_finished = server.requests_spawn_finished.load(Ordering::Relaxed), - handle_finished = server.requests_handle_finished.load(Ordering::Relaxed), - panics = server.requests_panic.load(Ordering::Relaxed), + spawn_finished = server + .metrics + .requests_spawn_finished + .load(Ordering::Relaxed), + handle_finished = server + .metrics + .requests_handle_finished + .load(Ordering::Relaxed), + panics = server.metrics.requests_panic.load(Ordering::Relaxed), spawn_active, handle_active, "Stopped listening on {addrs:?}", From 95006f7e46f4865955d38950e212bac50076bad2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 10:55:39 +0000 Subject: [PATCH 0489/2091] fix unnecessary preprocessing cfgs Signed-off-by: Jason Volk --- src/core/config/check.rs | 58 +++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 20a61b70..b36b9c5e 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -1,23 +1,29 @@ use figment::Figment; use super::DEPRECATED_KEYS; -use crate::{debug, error, info, warn, Config, Err, Result}; +use crate::{debug, debug_info, error, info, warn, Config, Err, Result}; #[allow(clippy::cognitive_complexity)] pub fn check(config: &Config) -> Result<()> { - #[cfg(debug_assertions)] - info!("Note: conduwuit was built without optimisations (i.e. debug build)"); + if cfg!(debug_assertions) { + info!("Note: conduwuit was built without optimisations (i.e. debug build)"); + } - #[cfg(all(feature = "rocksdb", not(feature = "sha256_media")))] // prevents catching this in `--all-features` - warn!( - "Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \ - supported backend now. Please update your build script to remove this feature." - ); - #[cfg(all(feature = "sha256_media", not(feature = "rocksdb")))] // prevents catching this in `--all-features` - warn!( - "Note the sha256_media feature was deleted from conduwuit, it is now fully integrated in a \ - forwards-compatible way. Please update your build script to remove this feature." - ); + // prevents catching this in `--all-features` + if cfg!(all(feature = "rocksdb", not(feature = "sha256_media"))) { + warn!( + "Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \ + supported backend now. Please update your build script to remove this feature." + ); + } + + // prevents catching this in `--all-features` + if cfg!(all(feature = "sha256_media", not(feature = "rocksdb"))) { + warn!( + "Note the sha256_media feature was deleted from conduwuit, it is now fully integrated in a \ + forwards-compatible way. Please update your build script to remove this feature." + ); + } warn_deprecated(config); warn_unknown_key(config); @@ -26,28 +32,27 @@ pub fn check(config: &Config) -> Result<()> { return Err!(Config("sentry_endpoint", "Sentry cannot be enabled without an endpoint set")); } - #[cfg(all(feature = "hardened_malloc", feature = "jemalloc"))] - warn!( - "hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used. If using --all-features, \ - this is harmless." - ); + if cfg!(all(feature = "hardened_malloc", feature = "jemalloc")) { + warn!( + "hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used. If using --all-features, \ + this is harmless." + ); + } - #[cfg(not(unix))] - if config.unix_socket_path.is_some() { + if cfg!(not(unix)) && config.unix_socket_path.is_some() { return Err!(Config( "unix_socket_path", - "UNIX socket support is only available on *nix platforms. Please remove \"unix_socket_path\" from your \ - config.", + "UNIX socket support is only available on *nix platforms. Please remove 'unix_socket_path' from your \ + config." )); } - #[cfg(unix)] - if config.unix_socket_path.is_none() { + if cfg!(unix) && config.unix_socket_path.is_none() { config.get_bind_addrs().iter().for_each(|addr| { use std::path::Path; if addr.ip().is_loopback() { - crate::debug_info!("Found loopback listening address {addr}, running checks if we're in a container.",); + debug_info!("Found loopback listening address {addr}, running checks if we're in a container."); if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() /* Host */ @@ -90,8 +95,7 @@ pub fn check(config: &Config) -> Result<()> { } // yeah, unless the user built a debug build hopefully for local testing only - #[cfg(not(debug_assertions))] - if config.server_name == "your.server.name" { + if cfg!(not(debug_assertions)) && config.server_name == "your.server.name" { return Err!(Config( "server_name", "You must specify a valid server name for production usage of conduwuit." From e53c2fbc5ab0232b4907db0d7cf21f45d362cd7c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 11:30:10 +0000 Subject: [PATCH 0490/2091] fix tracing flame envfilter defaults; remove preprocessed cfgs Signed-off-by: Jason Volk --- src/core/config/mod.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 16c7b1a8..336144db 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -166,19 +166,14 @@ pub struct Config { #[serde(default)] pub well_known: WellKnownConfig, #[serde(default)] - #[cfg(feature = "perf_measurements")] pub allow_jaeger: bool, #[serde(default = "default_jaeger_filter")] - #[cfg(feature = "perf_measurements")] pub jaeger_filter: String, #[serde(default)] - #[cfg(feature = "perf_measurements")] pub tracing_flame: bool, #[serde(default = "default_tracing_flame_filter")] - #[cfg(feature = "perf_measurements")] pub tracing_flame_filter: String, #[serde(default = "default_tracing_flame_output_path")] - #[cfg(feature = "perf_measurements")] pub tracing_flame_output_path: String, #[serde(default)] pub proxy: ProxyConfig, @@ -672,11 +667,8 @@ impl fmt::Display for Config { } &lst.into_iter().join(", ") }); - #[cfg(feature = "zstd_compression")] line("Zstd HTTP Compression", &self.zstd_compression.to_string()); - #[cfg(feature = "gzip_compression")] line("Gzip HTTP Compression", &self.gzip_compression.to_string()); - #[cfg(feature = "brotli_compression")] line("Brotli HTTP Compression", &self.brotli_compression.to_string()); line("RocksDB database LOG level", &self.rocksdb_log_level); line("RocksDB database LOG to stderr", &self.rocksdb_log_stderr.to_string()); @@ -925,10 +917,13 @@ fn default_pusher_idle_timeout() -> u64 { 15 } fn default_max_fetch_prev_events() -> u16 { 100_u16 } -#[cfg(feature = "perf_measurements")] -fn default_tracing_flame_filter() -> String { "trace,h2=off".to_owned() } +fn default_tracing_flame_filter() -> String { + cfg!(debug_assertions) + .then_some("trace,h2=off") + .unwrap_or("info") + .to_owned() +} -#[cfg(feature = "perf_measurements")] fn default_jaeger_filter() -> String { cfg!(debug_assertions) .then_some("trace,h2=off") @@ -936,7 +931,6 @@ fn default_jaeger_filter() -> String { .to_owned() } -#[cfg(feature = "perf_measurements")] fn default_tracing_flame_output_path() -> String { "./tracing.folded".to_owned() } fn default_trusted_servers() -> Vec { vec![OwnedServerName::try_from("matrix.org").unwrap()] } From 101fdbc9b1a1587e1eef7e95286ef73633b0af83 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:14:28 -0400 Subject: [PATCH 0491/2091] bump cargo.lock Updating bytes v1.6.0 -> v1.6.1 Updating cc v1.1.0 -> v1.1.3 Updating http-body v1.0.0 -> v1.0.1 Updating syn v2.0.70 -> v2.0.71 Updating thiserror v1.0.61 -> v1.0.62 Updating thiserror-impl v1.0.61 -> v1.0.62 Signed-off-by: strawberry Co-authored-by: Jason Volk --- Cargo.lock | 82 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 4 +-- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 48d1a91f..7f8e3f77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,7 +118,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -129,7 +129,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -192,7 +192,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "hyper 1.4.1", "hyper-util", @@ -253,7 +253,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -276,7 +276,7 @@ dependencies = [ "futures-util", "headers", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", @@ -296,7 +296,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "hyper 1.4.1", "hyper-util", @@ -377,7 +377,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -457,9 +457,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "bzip2-sys" @@ -474,9 +474,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.0" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaff6f8ce506b9773fa786672d63fc7a191ffea1be33f72bbd4aeacefca9ffc8" +checksum = "18e2d530f35b40a84124146478cd16f34225306a8441998836466a2e2961c950" dependencies = [ "jobserver", "libc", @@ -562,7 +562,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1022,7 +1022,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1126,7 +1126,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1262,7 +1262,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1543,7 +1543,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -1590,9 +1590,9 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", "http 1.1.0", @@ -1607,7 +1607,7 @@ dependencies = [ "bytes", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1664,7 +1664,7 @@ dependencies = [ "futures-util", "h2 0.4.5", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -1715,7 +1715,7 @@ dependencies = [ "futures-channel", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "hyper 1.4.1", "pin-project-lite", "socket2", @@ -1952,7 +1952,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -2461,7 +2461,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -2554,7 +2554,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -2642,7 +2642,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", "version_check", "yansi", ] @@ -2667,7 +2667,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -2857,7 +2857,7 @@ dependencies = [ "h2 0.4.5", "hickory-resolver", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "hyper 1.4.1", "hyper-rustls", @@ -3076,7 +3076,7 @@ dependencies = [ "quote", "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", "serde", - "syn 2.0.70", + "syn 2.0.71", "toml", ] @@ -3531,7 +3531,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -3824,9 +3824,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.70" +version = "2.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0209b68b3613b093e0ec905354eccaedcfe83b8cb37cbdeae64026c3064c16" +checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" dependencies = [ "proc-macro2", "quote", @@ -3884,22 +3884,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" +checksum = "f2675633b1499176c2dff06b0856a27976a8f9d436737b4cf4f312d4d91d8bbb" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" +checksum = "d20468752b09f49e909e55a5d338caa8bedf615594e9d80bc4c565d30faf798c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -4048,7 +4048,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -4224,7 +4224,7 @@ dependencies = [ "futures-core", "futures-util", "http 1.1.0", - "http-body 1.0.0", + "http-body 1.0.1", "http-body-util", "pin-project-lite", "tokio", @@ -4265,7 +4265,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390 dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", ] [[package]] @@ -4535,7 +4535,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", "wasm-bindgen-shared", ] @@ -4569,7 +4569,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.70", + "syn 2.0.71", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/Cargo.toml b/Cargo.toml index e24d84b8..f9696252 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ version = "0.8.5" # Used for the http request / response body type for Ruma endpoints used with reqwest [workspace.dependencies.bytes] -version = "1.6.0" +version = "1.6.1" [workspace.dependencies.http-body-util] version = "0.1.1" @@ -251,7 +251,7 @@ default-features = false # Used for conduit::Error type [workspace.dependencies.thiserror] -version = "1.0.61" +version = "1.0.62" # Used when hashing the state [workspace.dependencies.ring] From 3d73b53136daef69c2f023aaec353e6c1ad1c4a1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 22:16:48 -0400 Subject: [PATCH 0492/2091] update complement results we dont implement authenticated media yet Signed-off-by: strawberry --- tests/test_results/complement/test_results.jsonl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 2f3db95d..873397ba 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -138,7 +138,8 @@ {"Action":"fail","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} -{"Action":"pass","Test":"TestLocalPngThumbnail"} +{"Action":"fail","Test":"TestLocalPngThumbnail"} +{"Action":"fail","Test":"TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint"} {"Action":"fail","Test":"TestMediaFilenames"} {"Action":"fail","Test":"TestMediaFilenames/Parallel"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII"} @@ -181,7 +182,8 @@ {"Action":"pass","Test":"TestOutboundFederationProfile/Outbound_federation_can_query_profile_data"} {"Action":"pass","Test":"TestOutboundFederationSend"} {"Action":"pass","Test":"TestRemoteAliasRequestsUnderstandUnicode"} -{"Action":"pass","Test":"TestRemotePngThumbnail"} +{"Action":"fail","Test":"TestRemotePngThumbnail"} +{"Action":"fail","Test":"TestRemotePngThumbnail/test_/_matrix/client/v1/media_endpoint"} {"Action":"fail","Test":"TestRemotePresence"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members"} From 1ab77aeb9143cd0c3ff6ac750a8bef68b0455250 Mon Sep 17 00:00:00 2001 From: Charles Hall Date: Sat, 13 Jul 2024 15:48:07 -0400 Subject: [PATCH 0493/2091] don't strip binary for dev builds Signed-off-by: strawberry --- nix/pkgs/main/default.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index e3d23a43..ffb1de96 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -126,6 +126,8 @@ commonAttrs = { ]; }; + dontStrip = profile == "dev"; + buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'; nativeBuildInputs = [ From fcb6c8a1138cfe58e31c7841c07e5a5a2908935c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 15:54:14 -0400 Subject: [PATCH 0494/2091] bump rust-rocksdb Signed-off-by: strawberry --- Cargo.lock | 8 ++++---- deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7f8e3f77..12b96665 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3138,8 +3138,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.23.1+9.3.1" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=db1ba33c2b78ad228e2525e8902d059c24fc81a1#db1ba33c2b78ad228e2525e8902d059c24fc81a1" +version = "0.24.0+9.4.0" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=4056a3b0f823013fec49f6d0b3e5698856e6476a#4056a3b0f823013fec49f6d0b3e5698856e6476a" dependencies = [ "bindgen", "bzip2-sys", @@ -3155,8 +3155,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.27.1" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=db1ba33c2b78ad228e2525e8902d059c24fc81a1#db1ba33c2b78ad228e2525e8902d059c24fc81a1" +version = "0.28.0" +source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=4056a3b0f823013fec49f6d0b3e5698856e6476a#4056a3b0f823013fec49f6d0b3e5698856e6476a" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 57b71491..cf49e1ad 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/zaidoon1/rust-rocksdb" -rev = "db1ba33c2b78ad228e2525e8902d059c24fc81a1" +rev = "4056a3b0f823013fec49f6d0b3e5698856e6476a" #branch = "master" default-features = false From 2c0bfac43e1610b215d92d4cb35466a6b0e3d10a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:01:58 -0400 Subject: [PATCH 0495/2091] nix: bump flake.lock and rocksdb to v9.4.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'attic': 'github:zhaofengli/attic/717cc95983cdc357bc347d70be20ced21f935843' (2024-06-01) → 'github:zhaofengli/attic/6139576a3ce6bb992e0f6c3022528ec233e45f00' (2024-07-09) • Updated input 'complement': 'github:matrix-org/complement/bc97f1ddc1cd7485faf80c8935ee2641f3e1b57c' (2024-07-02) → 'github:matrix-org/complement/0d14432e010482ea9e13a6f7c47c1533c0c9d62f' (2024-07-10) • Updated input 'crane': 'github:ipetkov/crane/0aed560c5c0a61c9385bddff471a13036203e11c' (2024-07-06) → 'github:ipetkov/crane/2d83156f23c43598cf44e152c33a59d3892f8b29' (2024-07-09) • Updated input 'fenix': 'github:nix-community/fenix/abc0549e3560189462a7d394cc9d50af4608d103' (2024-07-08) → 'github:nix-community/fenix/5087b12a595ee73131a944d922f24d81dae05725' (2024-07-13) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/a5b21ea0aa644dffd7cf958b43f11f221d53404e' (2024-07-07) → 'github:rust-lang/rust-analyzer/ffbc5ad993d5cd2f3b8bcf9a511165470944ab91' (2024-07-11) • Updated input 'liburing': 'github:axboe/liburing/7b3245583069bd481190c9da18f22e9fc8c3a805' (2024-06-22) → 'github:axboe/liburing/1d674f83b7d0f07553ac44d99a401b05853d9dbe' (2024-07-12) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/655a58a72a6601292512670343087c2d75d859c1' (2024-07-08) → 'github:NixOS/nixpkgs/7e7c39ea35c5cdd002cd4588b03a3fb9ece6fad9' (2024-07-12) • Updated input 'rocksdb': 'github:girlbossceo/rocksdb/a935c0273e1ba44eacf88ce3685a9b9831486155' (2024-07-02) → 'github:girlbossceo/rocksdb/911f4243e69c2e320a7a209bf1f5f3ff5f825495' (2024-07-13) Signed-off-by: strawberry --- flake.lock | 50 +++++++++++++++++++++++++------------------------- flake.nix | 3 +-- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/flake.lock b/flake.lock index 4e6721b2..9d7a162e 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1717279440, - "narHash": "sha256-kH04ReTjxOpQumgWnqy40vvQLSnLGxWP6RF3nq5Esrk=", + "lastModified": 1720542474, + "narHash": "sha256-aKjJ/4l2I9+wNGTaOGRsuS3M1+IoTibqgEMPDikXm04=", "owner": "zhaofengli", "repo": "attic", - "rev": "717cc95983cdc357bc347d70be20ced21f935843", + "rev": "6139576a3ce6bb992e0f6c3022528ec233e45f00", "type": "github" }, "original": { @@ -81,11 +81,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1719903368, - "narHash": "sha256-PPzgxM4Bir+Zh9FUV/v+RBxEYeJxYVmi/BYo3uqt268=", + "lastModified": 1720637557, + "narHash": "sha256-oZz6nCmFmdJZpC+K1iOG2KkzTI6rlAmndxANPDVU7X0=", "owner": "matrix-org", "repo": "complement", - "rev": "bc97f1ddc1cd7485faf80c8935ee2641f3e1b57c", + "rev": "0d14432e010482ea9e13a6f7c47c1533c0c9d62f", "type": "github" }, "original": { @@ -123,11 +123,11 @@ ] }, "locked": { - "lastModified": 1720226507, - "narHash": "sha256-yHVvNsgrpyNTXZBEokL8uyB2J6gB1wEx0KOJzoeZi1A=", + "lastModified": 1720546058, + "narHash": "sha256-iU2yVaPIZm5vMGdlT0+57vdB/aPq/V5oZFBRwYw+HBM=", "owner": "ipetkov", "repo": "crane", - "rev": "0aed560c5c0a61c9385bddff471a13036203e11c", + "rev": "2d83156f23c43598cf44e152c33a59d3892f8b29", "type": "github" }, "original": { @@ -209,11 +209,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1720420198, - "narHash": "sha256-OIuDb6pHDyGpo7YMFyuRzMLcHm7mRvlYOz0Ht7ps2sU=", + "lastModified": 1720852044, + "narHash": "sha256-3NBYz8VuXuKU+8ONd9NFafCNjPEGHIZQ2Mdoam1a4mY=", "owner": "nix-community", "repo": "fenix", - "rev": "abc0549e3560189462a7d394cc9d50af4608d103", + "rev": "5087b12a595ee73131a944d922f24d81dae05725", "type": "github" }, "original": { @@ -381,11 +381,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1719025212, - "narHash": "sha256-kD0yhjNStqC6uFqC1AxBwUpc/HlSFtiKrV+gwDyroDc=", + "lastModified": 1720798442, + "narHash": "sha256-gtPppAoksMLW4GuruQ36nf4EAqIA1Bs6V9Xcx8dBxrQ=", "owner": "axboe", "repo": "liburing", - "rev": "7b3245583069bd481190c9da18f22e9fc8c3a805", + "rev": "1d674f83b7d0f07553ac44d99a401b05853d9dbe", "type": "github" }, "original": { @@ -606,11 +606,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1720418205, - "narHash": "sha256-cPJoFPXU44GlhWg4pUk9oUPqurPlCFZ11ZQPk21GTPU=", + "lastModified": 1720768451, + "narHash": "sha256-EYekUHJE2gxeo2pM/zM9Wlqw1Uw2XTJXOSAO79ksc4Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "655a58a72a6601292512670343087c2d75d859c1", + "rev": "7e7c39ea35c5cdd002cd4588b03a3fb9ece6fad9", "type": "github" }, "original": { @@ -673,16 +673,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1719949653, - "narHash": "sha256-DYx7XHH2GEh17GukKhXs6laM6l+eugCmRkF0adpi9wk=", + "lastModified": 1720900786, + "narHash": "sha256-Vta9Um/RRuWwZ46BjXftV06iWLm/j/9MX39emXUvSAY=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "a935c0273e1ba44eacf88ce3685a9b9831486155", + "rev": "911f4243e69c2e320a7a209bf1f5f3ff5f825495", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.3.1", + "ref": "v9.4.0", "repo": "rocksdb", "type": "github" } @@ -705,11 +705,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1720344064, - "narHash": "sha256-STmaV9Zu74QtkGGrbr9uMhskwagfCjJqOAYapXabiuk=", + "lastModified": 1720717809, + "narHash": "sha256-6I+fm+nTLF/iaj7ffiFGlSY7POmubwUaPA/Wq0Bm53M=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "a5b21ea0aa644dffd7cf958b43f11f221d53404e", + "rev": "ffbc5ad993d5cd2f3b8bcf9a511165470944ab91", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index e20da52b..36ebc997 100644 --- a/flake.nix +++ b/flake.nix @@ -9,8 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-unstable"; - # https://github.com/girlbossceo/rocksdb/commit/db6df0b185774778457dabfcbd822cb81760cade - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.3.1"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.4.0"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From 7009f56a7ae6dae52da692c0bd8bee76d5aa95f5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:02:44 -0400 Subject: [PATCH 0496/2091] fix user directory publishing access controls Signed-off-by: strawberry --- src/api/client/directory.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 68bd0dff..8e12c034 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -117,7 +117,12 @@ pub(crate) async fn set_room_visibility_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } - user_can_publish_room(sender_user, &body.room_id)?; + if !user_can_publish_room(sender_user, &body.room_id)? { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "User is not allowed to publish this room", + )); + } match &body.visibility { room::Visibility::Public => { @@ -377,8 +382,8 @@ fn user_can_publish_room(user_id: &UserId, room_id: &RoomId) -> Result { Ok(event.sender == user_id) } else { return Err(Error::BadRequest( - ErrorKind::Unauthorized, - "You are not allowed to publish this room to the room directory", + ErrorKind::forbidden(), + "User is not allowed to publish this room", )); } } From a1bfd7a018afe2135da01ecc0ddd7ce88c444316 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:09:24 -0400 Subject: [PATCH 0497/2091] nix: dont include `experimental` feature on all-features builds this is just future infra for it Signed-off-by: strawberry --- flake.nix | 40 ++++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/flake.nix b/flake.nix index 36ebc997..0024e176 100644 --- a/flake.nix +++ b/flake.nix @@ -122,8 +122,12 @@ default = scopeHost.main; all-features = scopeHost.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; }; @@ -131,8 +135,12 @@ oci-image-all-features = scopeHost.oci-image.override { main = scopeHost.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; }; oci-image-hmalloc = scopeHost.oci-image.override { @@ -174,8 +182,12 @@ name = "${binaryName}-all-features"; value = scopeCrossStatic.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; } @@ -199,8 +211,12 @@ value = scopeCrossStatic.oci-image.override { main = scopeCrossStatic.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; }; } @@ -228,8 +244,12 @@ (scopeHostStatic.overrideScope (final: prev: { main = prev.main.override { all_features = true; - # this is non-functional on nix for some reason - disable_features = ["hardened_malloc"]; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; })); devShells.no-features = mkDevShell From bacffd617427d835d0792b87ef64cba8aad95bae Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 16:46:13 -0400 Subject: [PATCH 0498/2091] rename `conduit_cache_capacity_modifier` to `cache_capacity_modifier` this prefix causes you to require setting the environment variable to `CONDUWUIT_CONDUIT_CACHE_CAPACITY_MODIFIER` alias this so we dont break any configs Signed-off-by: strawberry --- conduwuit-example.toml | 5 ++++- src/core/config/mod.rs | 11 ++++++----- src/database/opts.rs | 2 +- src/service/rooms/auth_chain/data.rs | 2 +- src/service/rooms/spaces/mod.rs | 2 +- src/service/rooms/state_accessor/mod.rs | 4 ++-- src/service/rooms/state_compressor/mod.rs | 2 +- 7 files changed, 16 insertions(+), 12 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index ea095bfa..57093128 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -421,8 +421,11 @@ allow_profile_lookup_federation_requests = true # Set this to any float value to multiply conduwuit's in-memory LRU caches with. # May be useful if you have significant memory to spare to increase performance. +# +# This was previously called `conduit_cache_capacity_modifier` +# # Defaults to 1.0. -#conduit_cache_capacity_modifier = 1.0 +#cache_capacity_modifier = 1.0 # Set this to any float value in megabytes for conduwuit to tell the database engine that this much memory is available for database-related caches. # May be useful if you have significant memory to spare to increase performance. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 336144db..b40ebb65 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -58,8 +58,8 @@ pub struct Config { #[serde(default = "default_pdu_cache_capacity")] pub pdu_cache_capacity: u32, - #[serde(default = "default_conduit_cache_capacity_modifier")] - pub conduit_cache_capacity_modifier: f64, + #[serde(default = "default_cache_capacity_modifier", alias = "conduit_cache_capacity_modifier")] + pub cache_capacity_modifier: f64, #[serde(default = "default_auth_chain_cache_capacity")] pub auth_chain_cache_capacity: u32, #[serde(default = "default_shorteventid_cache_capacity")] @@ -391,8 +391,9 @@ struct ListeningAddr { addrs: Either>, } -const DEPRECATED_KEYS: &[&str] = &[ +const DEPRECATED_KEYS: &[&str; 9] = &[ "cache_capacity", + "conduit_cache_capacity_modifier", "max_concurrent_requests", "well_known_client", "well_known_server", @@ -484,7 +485,7 @@ impl fmt::Display for Config { ); line("Database backups to keep", &self.database_backups_to_keep.to_string()); line("Database cache capacity (MB)", &self.db_cache_capacity_mb.to_string()); - line("Cache capacity modifier", &self.conduit_cache_capacity_modifier.to_string()); + line("Cache capacity modifier", &self.cache_capacity_modifier.to_string()); line("PDU cache capacity", &self.pdu_cache_capacity.to_string()); line("Auth chain cache capacity", &self.auth_chain_cache_capacity.to_string()); line("Short eventid cache capacity", &self.shorteventid_cache_capacity.to_string()); @@ -847,7 +848,7 @@ fn default_db_cache_capacity_mb() -> f64 { 256.0 } fn default_pdu_cache_capacity() -> u32 { 150_000 } -fn default_conduit_cache_capacity_modifier() -> f64 { 1.0 } +fn default_cache_capacity_modifier() -> f64 { 1.0 } fn default_auth_chain_cache_capacity() -> u32 { 100_000 } diff --git a/src/database/opts.rs b/src/database/opts.rs index 1b9c163f..d2236454 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -310,7 +310,7 @@ fn set_table_with_shared_cache( } fn cache_size(config: &Config, base_size: u32, entity_size: usize) -> usize { - let ents = f64::from(base_size) * config.conduit_cache_capacity_modifier; + let ents = f64::from(base_size) * config.cache_capacity_modifier; #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] (ents as usize) diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index 5efb36c2..4e468234 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -16,7 +16,7 @@ impl Data { pub(super) fn new(server: &Arc, db: &Arc) -> Self { let config = &server.config; let cache_size = f64::from(config.auth_chain_cache_capacity); - let cache_size = usize_from_f64(cache_size * config.conduit_cache_capacity_modifier).expect("valid cache size"); + let cache_size = usize_from_f64(cache_size * config.cache_capacity_modifier).expect("valid cache size"); Self { shorteventid_authchain: db["shorteventid_authchain"].clone(), auth_chain_cache: Mutex::new(LruCache::new(cache_size)), diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 18133fc1..02db7fab 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -161,7 +161,7 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; let cache_size = f64::from(config.roomid_spacehierarchy_cache_capacity); - let cache_size = cache_size * config.conduit_cache_capacity_modifier; + let cache_size = cache_size * config.cache_capacity_modifier; Ok(Arc::new(Self { roomid_spacehierarchy_cache: Mutex::new(LruCache::new(usize_from_f64(cache_size)?)), })) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 7abe5e0f..bd3eb0a1 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -45,9 +45,9 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; let server_visibility_cache_capacity = - f64::from(config.server_visibility_cache_capacity) * config.conduit_cache_capacity_modifier; + f64::from(config.server_visibility_cache_capacity) * config.cache_capacity_modifier; let user_visibility_cache_capacity = - f64::from(config.user_visibility_cache_capacity) * config.conduit_cache_capacity_modifier; + f64::from(config.user_visibility_cache_capacity) * config.cache_capacity_modifier; Ok(Arc::new(Self { db: Data::new(args.db), diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 4b4ea7d4..422c562b 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -55,7 +55,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; - let cache_capacity = f64::from(config.stateinfo_cache_capacity) * config.conduit_cache_capacity_modifier; + let cache_capacity = f64::from(config.stateinfo_cache_capacity) * config.cache_capacity_modifier; Ok(Arc::new(Self { db: Data::new(args.db), stateinfo_cache: StdMutex::new(LruCache::new(usize_from_f64(cache_capacity)?)), From bdd58454907690ca0f9b827b21fd062e1a5feb64 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 22:15:16 -0400 Subject: [PATCH 0499/2091] docs: suggest recursive chmod Signed-off-by: strawberry --- docs/deploying/generic.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 75b285d1..0cbd8ef8 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -23,7 +23,7 @@ Otherwise, follow standard Rust project build guides (installing git and cloning While conduwuit can run as any user it is better to use dedicated users for different services. This also allows you to make sure that the file permissions are correctly set up. -In Debian or RHEL, you can use this command to create a conduwuit user: +In Debian or Fedora/RHEL, you can use this command to create a conduwuit user: ```bash sudo adduser --system conduwuit --group --disabled-login --no-create-home @@ -53,13 +53,11 @@ RocksDB is the only supported database backend. ## Setting the correct file permissions -If you are using a dedicated user for conduwuit, you will need to allow it to read the config. To do that you can run this command on - -Debian or RHEL: +If you are using a dedicated user for conduwuit, you will need to allow it to read the config. To do that you can run this: ```bash sudo chown -R root:root /etc/conduwuit -sudo chmod 755 /etc/conduwuit +sudo chmod -R 755 /etc/conduwuit ``` If you use the default database path you also need to run this: From 9370e93a8dd2e9cc72c6bc48b7689146bd3a4405 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 23:03:29 -0400 Subject: [PATCH 0500/2091] nix: try to make x86-64-v2 optimised builds if target is x86_64 Signed-off-by: strawberry --- nix/pkgs/main/cross-compilation-env.nix | 3 +++ nix/pkgs/main/default.nix | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index 83fe6ed6..70d0e51c 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -13,6 +13,9 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { lib.concatStringsSep " " ([] + ++ lib.optionals + stdenv.targetPlatform.isx86_64 + [ "-C" "target-cpu=x86-64-v2" ] # This disables PIE for static builds, which isn't great in terms # of security. Unfortunately, my hand is forced because nixpkgs' # `libstdc++.a` is built without `-fPIE`, which precludes us from diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index ffb1de96..6e3e82d7 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -71,10 +71,23 @@ buildDepsOnlyEnv = # which breaks Darwin entirely enableLiburing = enableLiburing; }).overrideAttrs (old: { - # TODO: static rocksdb fails to build on darwin + # TODO: static rocksdb fails to build on darwin, also see # build log at meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin; + enableLiburing = enableLiburing; + + sse42Support = stdenv.targetPlatform.isx86_64; + + cmakeFlags = if stdenv.targetPlatform.isx86_64 + then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags + ++ lib.optionals stdenv.targetPlatform.isx86_64 [ + "-DPORTABLE=x86-64-v2" + "-DUSE_SSE=1" + "-DHAVE_SSE=1" + "-DHAVE_SSE42=1" + ] + else old.cmakeFlags; }); in { @@ -101,7 +114,9 @@ buildPackageEnv = { # Only needed in static stdenv because these are transitive dependencies of rocksdb CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic) - " -L${lib.getLib liburing}/lib -luring"; + " -L${lib.getLib liburing}/lib -luring" + + lib.optionalString stdenv.targetPlatform.isx86_64 + " -Ctarget-cpu=x86-64-v2"; }; From df28359a197ec09cbefbe3be496a8eb19597f553 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 23:23:40 -0400 Subject: [PATCH 0501/2091] nix: dont build rocksdb core tools (ldb / sst_dump) (`-DWITH_CORE_TOOLS=1`) Signed-off-by: strawberry --- flake.nix | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index 0024e176..e34aaad0 100644 --- a/flake.nix +++ b/flake.nix @@ -44,10 +44,21 @@ # we have this already at https://github.com/girlbossceo/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155 # unsetting this so i don't have to revert it and make this nix exclusive patches = []; - # no real reason to have snappy, no one uses this cmakeFlags = pkgs.lib.subtractLists - [ "-DWITH_SNAPPY=1" ] - old.cmakeFlags; + [ + # no real reason to have snappy, no one uses this + "-DWITH_SNAPPY=1" + # we dont need to use ldb or sst_dump (core_tools) + "-DWITH_CORE_TOOLS=1" + ] + old.cmakeFlags + ++ [ "-DWITH_CORE_TOOLS=0" ]; + + # outputs has "tools" which we dont need or use + outputs = [ "out" ]; + + # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use + preInstall = ""; }); # TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available liburing = pkgs.liburing.overrideAttrs (old: { From 649e9da1f8f6404123f240af99cd801a7d16ed3a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 13 Jul 2024 23:24:25 -0400 Subject: [PATCH 0502/2091] nix: try to make armv8.2-a / cortex-a55 optimised builds if target is aarch64 Signed-off-by: strawberry --- nix/pkgs/main/cross-compilation-env.nix | 3 +++ nix/pkgs/main/default.nix | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index 70d0e51c..75b5862f 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -16,6 +16,9 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { ++ lib.optionals stdenv.targetPlatform.isx86_64 [ "-C" "target-cpu=x86-64-v2" ] + ++ lib.optionals + stdenv.targetPlatform.isAarch64 + [ "-C" "target-cpu=cortex-a55" ] # cortex-a55 == ARMv8.2-a # This disables PIE for static builds, which isn't great in terms # of security. Unfortunately, my hand is forced because nixpkgs' # `libstdc++.a` is built without `-fPIE`, which precludes us from diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 6e3e82d7..629957c8 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -87,6 +87,12 @@ buildDepsOnlyEnv = "-DHAVE_SSE=1" "-DHAVE_SSE42=1" ] + else if stdenv.targetPlatform.isAarch64 + then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags + ++ lib.optionals stdenv.targetPlatform.isAarch64 [ + # cortex-a55 == ARMv8.2-a + "-DPORTABLE=armv8.2-a" + ] else old.cmakeFlags; }); in @@ -116,7 +122,9 @@ buildPackageEnv = { + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic) " -L${lib.getLib liburing}/lib -luring" + lib.optionalString stdenv.targetPlatform.isx86_64 - " -Ctarget-cpu=x86-64-v2"; + " -Ctarget-cpu=x86-64-v2" + + lib.optionalString stdenv.targetPlatform.isAarch64 + " -Ctarget-cpu=cortex-a55"; # cortex-a55 == ARMv8.2-a }; From f0c63c539bf1fd7347e2e5bddec7e6e9da7cde1d Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 00:11:37 -0400 Subject: [PATCH 0503/2091] nix: dont build more unnecessary tests/tools for rocksdb Signed-off-by: strawberry --- flake.nix | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index e34aaad0..8ac5836f 100644 --- a/flake.nix +++ b/flake.nix @@ -50,9 +50,18 @@ "-DWITH_SNAPPY=1" # we dont need to use ldb or sst_dump (core_tools) "-DWITH_CORE_TOOLS=1" + # we dont need to build rocksdb tests + "-DWITH_TESTS=1" ] old.cmakeFlags - ++ [ "-DWITH_CORE_TOOLS=0" ]; + ++ [ + # we dont need to use ldb or sst_dump (core_tools) + "-DWITH_CORE_TOOLS=0" + # we dont need trace tools + "-DWITH_TRACE_TOOLS=0" + # we dont need to build rocksdb tests + "-DWITH_TESTS=0" + ]; # outputs has "tools" which we dont need or use outputs = [ "out" ]; From 188fa5a07397a01137bd1402ed7ae7b826ee4b27 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 00:14:23 -0400 Subject: [PATCH 0504/2091] patch rustyline-async and use no default features Signed-off-by: strawberry --- Cargo.toml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f9696252..104535c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -405,8 +405,8 @@ features = [ ] [workspace.dependencies.rustyline-async] -git = "https://github.com/girlbossceo/rustyline-async" -rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50" +version = "0.4.2" +default-features = false [workspace.dependencies.termimad] version = "0.29.4" @@ -442,6 +442,12 @@ rev = "b348dca742af641c47bc390261f60711c2af573c" git = "https://github.com/girlbossceo/axum-server" rev = "8e3368d899079818934e61cc9c839abcbbcada8a" +# adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50 +# adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b +[patch.crates-io.rustyline-async] +git = "https://github.com/girlbossceo/rustyline-async" +rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50" + # # Our crates # From 1797fec3c9109a8b02c782934a8677ce3b6f21ea Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 00:38:29 -0400 Subject: [PATCH 0505/2091] nix: disable USE_RTTI for rocksdb Signed-off-by: strawberry --- flake.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flake.nix b/flake.nix index 8ac5836f..a56d1bc0 100644 --- a/flake.nix +++ b/flake.nix @@ -52,6 +52,8 @@ "-DWITH_CORE_TOOLS=1" # we dont need to build rocksdb tests "-DWITH_TESTS=1" + # we use rust-rocksdb via C interface and dont need C++ RTTI + "-DUSE_RTTI=1" ] old.cmakeFlags ++ [ @@ -61,6 +63,8 @@ "-DWITH_TRACE_TOOLS=0" # we dont need to build rocksdb tests "-DWITH_TESTS=0" + # we use rust-rocksdb via C interface and dont need C++ RTTI + "-DUSE_RTTI=0" ]; # outputs has "tools" which we dont need or use From bdf2de076a0af8c5ac795bbabaa45feb6cc206bf Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 00:47:56 -0400 Subject: [PATCH 0506/2091] ci: set NIX_CONFIG with our binary caches for the earlier CI steps Signed-off-by: strawberry --- .github/workflows/ci.yml | 7 +++++-- .github/workflows/documentation.yml | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66ecdce8..19aa68cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,8 +38,11 @@ env: # Custom nix binary cache if fork is being used ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} - # Get error output from nix that we can actually use - NIX_CONFIG: show-trace = true + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= permissions: packages: write diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ed0e898d..cab1710f 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -21,8 +21,11 @@ env: # Custom nix binary cache if fork is being used ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} - # Get error output from nix that we can actually use - NIX_CONFIG: show-trace = true + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. From e2280aa1a53c9403fffe53e866c641f9dbbafb66 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 01:33:38 -0400 Subject: [PATCH 0507/2091] ci: try removing --impure Signed-off-by: strawberry --- .github/workflows/ci.yml | 4 ++-- .github/workflows/documentation.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19aa68cd..9def2cf9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,7 +118,7 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop .#all-features --command true @@ -240,7 +240,7 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop .#all-features --command true diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index cab1710f..c6f9133a 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -101,7 +101,7 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop --command true From bda44b16b11d27dc83d45d8a5af25a5e4e785327 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 02:07:07 -0400 Subject: [PATCH 0508/2091] ci: run nix dynamic build test with `DIRENV_DEVSHELL=dynamic` Signed-off-by: strawberry --- engage.toml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/engage.toml b/engage.toml index d28ecff1..8bba0eaf 100644 --- a/engage.toml +++ b/engage.toml @@ -184,6 +184,10 @@ cargo test \ name = "nix-default" group = "tests" script = """ -bin/nix-build-and-cache just .#default -nix run -L .#default -- --help +env DIRENV_DEVSHELL=dynamic \ + direnv exec . \ + bin/nix-build-and-cache just .#default +env DIRENV_DEVSHELL=dynamic \ + direnv exec . \ + nix run -L .#default -- --help """ From 80ec0e31b19dcf19549b8fd0f1b70d562e99245c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 14 Jul 2024 20:33:22 -0400 Subject: [PATCH 0509/2091] bump tracing fork Signed-off-by: strawberry --- Cargo.lock | 10 +++++----- Cargo.toml | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 12b96665..2c637a34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4250,7 +4250,7 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" version = "0.1.40" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "log", "pin-project-lite", @@ -4261,7 +4261,7 @@ dependencies = [ [[package]] name = "tracing-attributes" version = "0.1.27" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "proc-macro2", "quote", @@ -4271,7 +4271,7 @@ dependencies = [ [[package]] name = "tracing-core" version = "0.1.32" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "once_cell", "valuable", @@ -4291,7 +4291,7 @@ dependencies = [ [[package]] name = "tracing-log" version = "0.2.0" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "log", "once_cell", @@ -4319,7 +4319,7 @@ dependencies = [ [[package]] name = "tracing-subscriber" version = "0.3.18" -source = "git+https://github.com/girlbossceo/tracing?rev=b348dca742af641c47bc390261f60711c2af573c#b348dca742af641c47bc390261f60711c2af573c" +source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" dependencies = [ "matchers", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index 104535c8..db48b42e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -425,16 +425,16 @@ version = "0.1" # https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c [patch.crates-io.tracing-subscriber] git = "https://github.com/girlbossceo/tracing" -rev = "b348dca742af641c47bc390261f60711c2af573c" +rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" [patch.crates-io.tracing] git = "https://github.com/girlbossceo/tracing" -rev = "b348dca742af641c47bc390261f60711c2af573c" +rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" [patch.crates-io.tracing-core] git = "https://github.com/girlbossceo/tracing" -rev = "b348dca742af641c47bc390261f60711c2af573c" +rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" [patch.crates-io.tracing-log] git = "https://github.com/girlbossceo/tracing" -rev = "b348dca742af641c47bc390261f60711c2af573c" +rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" # fixes hyper graceful shutdowns [https://github.com/programatik29/axum-server/issues/114] # https://github.com/girlbossceo/axum-server/commit/8e3368d899079818934e61cc9c839abcbbcada8a From f79bd2ac72e4135d170b526657027b458a921da2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 15 Jul 2024 21:45:10 -0400 Subject: [PATCH 0510/2091] bump ruma-identifiers-validation to c51ccb2c68d2e3557eb12b1a49036531711ec0e5 Signed-off-by: strawberry --- Cargo.lock | 17 ++++------------- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c637a34..61cf48bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -769,7 +769,7 @@ dependencies = [ "regex", "reqwest", "ruma", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0)", + "ruma-identifiers-validation", "rustyline-async", "serde", "serde_json", @@ -2988,7 +2988,7 @@ dependencies = [ "percent-encoding", "rand", "regex", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", + "ruma-identifiers-validation", "ruma-macros", "serde", "serde_html_form", @@ -3015,7 +3015,7 @@ dependencies = [ "pulldown-cmark", "regex", "ruma-common", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", + "ruma-identifiers-validation", "ruma-macros", "serde", "serde_json", @@ -3046,15 +3046,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "ruma-identifiers-validation" -version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=fd686e77950680462377c9105dfb4136dd49c7a0#fd686e77950680462377c9105dfb4136dd49c7a0" -dependencies = [ - "js_int", - "thiserror", -] - [[package]] name = "ruma-identity-service-api" version = "0.9.0" @@ -3074,7 +3065,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "ruma-identifiers-validation 0.9.5 (git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5)", + "ruma-identifiers-validation", "serde", "syn 2.0.71", "toml", diff --git a/Cargo.toml b/Cargo.toml index db48b42e..7c17d0f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -309,7 +309,7 @@ features = [ [workspace.dependencies.ruma-identifiers-validation] git = "https://github.com/girlbossceo/ruwuma" -rev = "fd686e77950680462377c9105dfb4136dd49c7a0" +rev = "c51ccb2c68d2e3557eb12b1a49036531711ec0e5" [workspace.dependencies.rust-rocksdb] path = "deps/rust-rocksdb" From 59503553485da33d4521612917f103e9ad88e6f2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 02:50:09 +0000 Subject: [PATCH 0511/2091] break from wait loop on empty taskset Signed-off-by: Jason Volk --- src/router/serve/unix.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index b5938673..fb011f18 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -127,8 +127,8 @@ async fn fini(server: &Arc, listener: UnixListener, mut tasks: JoinSet<( debug!("Waiting for requests to finish..."); while server.metrics.requests_spawn_active.load(Ordering::Relaxed) > 0 { tokio::select! { - _ = tasks.join_next() => {} - () = sleep(FINI_POLL_INTERVAL) => {} + task = tasks.join_next() => if task.is_none() { break; }, + () = sleep(FINI_POLL_INTERVAL) => {}, } } From c42cb90dd3ccd8f3b531c75c46b08c7891186cde Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 01:49:30 +0000 Subject: [PATCH 0512/2091] simplify result handler / 405 error interposition Signed-off-by: Jason Volk --- src/core/error/err.rs | 12 ++++++--- src/core/error/mod.rs | 7 ++--- src/core/error/response.rs | 54 +++++++++++++++++++++++++++----------- src/router/request.rs | 31 ++++++---------------- src/router/router.rs | 4 +-- 5 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/core/error/err.rs b/src/core/error/err.rs index ea596644..66634f2b 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -49,14 +49,16 @@ macro_rules! err { $crate::$level!($($args),*); $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::forbidden(), - $crate::format_maybe!($($args),*) + $crate::format_maybe!($($args),*), + ::http::StatusCode::BAD_REQUEST ) }}; (Request(Forbidden($($args:expr),*))) => { $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::forbidden(), - $crate::format_maybe!($($args),*) + $crate::format_maybe!($($args),*), + ::http::StatusCode::BAD_REQUEST ) }; @@ -64,14 +66,16 @@ macro_rules! err { $crate::$level!($($args),*); $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::$variant, - $crate::format_maybe!($($args),*) + $crate::format_maybe!($($args),*), + ::http::StatusCode::BAD_REQUEST ) }}; (Request($variant:ident($($args:expr),*))) => { $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::$variant, - $crate::format_maybe!($($args),*) + $crate::format_maybe!($($args),*), + ::http::StatusCode::BAD_REQUEST ) }; diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 069fe60e..1f0d56c1 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -68,7 +68,7 @@ pub enum Error { #[error("{0}: {1}")] BadRequest(ruma::api::client::error::ErrorKind, &'static str), //TODO: remove #[error("{0}: {1}")] - Request(ruma::api::client::error::ErrorKind, Cow<'static, str>), + Request(ruma::api::client::error::ErrorKind, Cow<'static, str>, http::StatusCode), #[error("from {0}: {1}")] Redaction(ruma::OwnedServerName, ruma::canonical_json::RedactionError), #[error("Remote server {0} responded with: {1}")] @@ -120,7 +120,7 @@ impl Error { match self { Self::Federation(_, error) => response::ruma_error_kind(error).clone(), - Self::BadRequest(kind, _) | Self::Request(kind, _) => kind.clone(), + Self::BadRequest(kind, ..) | Self::Request(kind, ..) => kind.clone(), _ => Unknown, } } @@ -128,7 +128,8 @@ impl Error { pub fn status_code(&self) -> http::StatusCode { match self { Self::Federation(_, ref error) | Self::RumaError(ref error) => error.status_code, - Self::BadRequest(ref kind, _) | Self::Request(ref kind, _) => response::bad_request_code(kind), + Self::Request(ref kind, _, code) => response::status_code(kind, *code), + Self::BadRequest(ref kind, ..) => response::bad_request_code(kind), Self::Conflict(_) => http::StatusCode::CONFLICT, _ => http::StatusCode::INTERNAL_SERVER_ERROR, } diff --git a/src/core/error/response.rs b/src/core/error/response.rs index 4ea76e26..7568a1c0 100644 --- a/src/core/error/response.rs +++ b/src/core/error/response.rs @@ -1,7 +1,13 @@ use bytes::BytesMut; use http::StatusCode; use http_body_util::Full; -use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; +use ruma::api::{ + client::{ + error::{ErrorBody, ErrorKind}, + uiaa::UiaaResponse, + }, + OutgoingResponse, +}; use super::Error; use crate::error; @@ -25,7 +31,7 @@ impl From for UiaaResponse { return Self::AuthResponse(uiaainfo); } - let body = ruma::api::client::error::ErrorBody::Standard { + let body = ErrorBody::Standard { kind: error.kind(), message: error.message(), }; @@ -37,10 +43,33 @@ impl From for UiaaResponse { } } -pub(super) fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> StatusCode { - use ruma::api::client::error::ErrorKind::*; +pub(super) fn status_code(kind: &ErrorKind, hint: StatusCode) -> StatusCode { + if hint == StatusCode::BAD_REQUEST { + bad_request_code(kind) + } else { + hint + } +} + +pub(super) fn bad_request_code(kind: &ErrorKind) -> StatusCode { + use ErrorKind::*; match kind { + // 429 + LimitExceeded { + .. + } => StatusCode::TOO_MANY_REQUESTS, + + // 413 + TooLarge => StatusCode::PAYLOAD_TOO_LARGE, + + // 405 + Unrecognized => StatusCode::METHOD_NOT_ALLOWED, + + // 404 + NotFound => StatusCode::NOT_FOUND, + + // 403 GuestAccessForbidden | ThreepidAuthFailed | UserDeactivated @@ -52,26 +81,20 @@ pub(super) fn bad_request_code(kind: &ruma::api::client::error::ErrorKind) -> St .. } => StatusCode::FORBIDDEN, + // 401 UnknownToken { .. } | MissingToken | Unauthorized => StatusCode::UNAUTHORIZED, - LimitExceeded { - .. - } => StatusCode::TOO_MANY_REQUESTS, - - TooLarge => StatusCode::PAYLOAD_TOO_LARGE, - - NotFound | Unrecognized => StatusCode::NOT_FOUND, - + // 400 _ => StatusCode::BAD_REQUEST, } } pub(super) fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { - if let ruma::api::client::error::ErrorBody::Standard { + if let ErrorBody::Standard { message, .. } = &error.body @@ -82,7 +105,6 @@ pub(super) fn ruma_error_message(error: &ruma::api::client::error::Error) -> Str format!("{error}") } -pub(super) fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ruma::api::client::error::ErrorKind { - e.error_kind() - .unwrap_or(&ruma::api::client::error::ErrorKind::Unknown) +pub(super) fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ErrorKind { + e.error_kind().unwrap_or(&ErrorKind::Unknown) } diff --git a/src/router/request.rs b/src/router/request.rs index ae739984..4d5c61e7 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -4,9 +4,8 @@ use axum::{ extract::State, response::{IntoResponse, Response}, }; -use conduit::{debug, debug_error, debug_warn, defer, error, trace, Error, Result, Server}; +use conduit::{debug, debug_error, debug_warn, defer, err, error, trace, Result, Server}; use http::{Method, StatusCode, Uri}; -use ruma::api::client::error::{Error as RumaError, ErrorBody, ErrorKind}; #[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( @@ -58,33 +57,13 @@ pub(crate) async fn handle( trace!(active, finished, "leave"); }}; - let method = req.method().clone(); let uri = req.uri().clone(); + let method = req.method().clone(); let result = next.run(req).await; handle_result(&method, &uri, result) } fn handle_result(method: &Method, uri: &Uri, result: Response) -> Result { - handle_result_log(method, uri, &result); - match result.status() { - StatusCode::METHOD_NOT_ALLOWED => handle_result_405(method, uri, &result), - _ => Ok(result), - } -} - -fn handle_result_405(_method: &Method, _uri: &Uri, result: &Response) -> Result { - let error = Error::RumaError(RumaError { - status_code: result.status(), - body: ErrorBody::Standard { - kind: ErrorKind::Unrecognized, - message: "M_UNRECOGNIZED: Method not allowed for endpoint".to_owned(), - }, - }); - - Ok(error.into_response()) -} - -fn handle_result_log(method: &Method, uri: &Uri, result: &Response) { let status = result.status(); let reason = status.canonical_reason().unwrap_or("Unknown Reason"); let code = status.as_u16(); @@ -97,4 +76,10 @@ fn handle_result_log(method: &Method, uri: &Uri, result: &Response) { } else { trace!(method = ?method, uri = ?uri, "{code} {reason}"); } + + if status == StatusCode::METHOD_NOT_ALLOWED { + return Ok(err!(Request(Unrecognized("Method Not Allowed"))).into_response()); + } + + Ok(result) } diff --git a/src/router/router.rs b/src/router/router.rs index da31ffea..2f6d5c46 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use axum::{response::IntoResponse, routing::get, Router}; use conduit::{Error, Server}; use conduit_service as service; -use http::Uri; +use http::{StatusCode, Uri}; use ruma::api::client::error::ErrorKind; extern crate conduit_api as api; @@ -19,7 +19,7 @@ pub(crate) fn build(server: &Arc) -> Router { } async fn not_found(_uri: Uri) -> impl IntoResponse { - Error::BadRequest(ErrorKind::Unrecognized, "Unrecognized request") + Error::Request(ErrorKind::Unrecognized, "Not Found".into(), StatusCode::NOT_FOUND) } async fn it_works() -> &'static str { "hewwo from conduwuit woof!" } From 720fbd09c212d1fdbf93484c27de9abcd239127a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 14 Jul 2024 23:46:03 +0000 Subject: [PATCH 0513/2091] move routes into api router top level Signed-off-by: Jason Volk --- src/api/mod.rs | 9 ++++----- src/api/{routes.rs => router.rs} | 32 +++++++++++++++++++----------- src/api/router/{mod.rs => args.rs} | 18 ++++++----------- src/router/router.rs | 2 +- 4 files changed, 31 insertions(+), 30 deletions(-) rename src/api/{routes.rs => router.rs} (93%) rename src/api/router/{mod.rs => args.rs} (90%) diff --git a/src/api/mod.rs b/src/api/mod.rs index 6adf2d39..dda37caf 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,15 +1,14 @@ pub mod client; -mod router; -pub mod routes; +pub mod router; pub mod server; extern crate conduit_core as conduit; extern crate conduit_service as service; -pub(crate) use conduit::{debug_info, debug_warn, utils, Error, Result}; -pub(crate) use service::{pdu::PduEvent, services, user_is_local}; +pub(crate) use conduit::{debug_info, debug_warn, pdu::PduEvent, utils, Error, Result}; +pub(crate) use service::{services, user_is_local}; -pub(crate) use self::router::{Ruma, RumaResponse}; +pub(crate) use crate::router::{Ruma, RumaResponse}; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/api/routes.rs b/src/api/router.rs similarity index 93% rename from src/api/routes.rs rename to src/api/router.rs index 3a8b2c74..9b6f62a5 100644 --- a/src/api/routes.rs +++ b/src/api/router.rs @@ -1,3 +1,9 @@ +mod args; +mod auth; +mod handler; +mod request; +mod response; + use axum::{ response::IntoResponse, routing::{any, get, post}, @@ -7,7 +13,9 @@ use conduit::{err, Error, Server}; use http::Uri; use ruma::api::client::error::ErrorKind; -use crate::{client, router::RouterExt, server}; +use self::handler::RouterExt; +pub(super) use self::{ar::Ruma, response::RumaResponse}; +use crate::{client, server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; @@ -95,7 +103,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::get_member_events_route) .ruma_route(client::get_protocols_route) .route("/_matrix/client/unstable/thirdparty/protocols", - get(client::get_protocols_route_unstable)) + get(client::get_protocols_route_unstable)) .ruma_route(client::send_message_event_route) .ruma_route(client::send_state_event_for_key_route) .ruma_route(client::get_state_events_route) @@ -180,15 +188,15 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::get_relating_events_with_rel_type_route) .ruma_route(client::get_relating_events_route) .ruma_route(client::get_hierarchy_route) - .ruma_route(client::get_mutual_rooms_route) - .ruma_route(client::get_room_summary) - .route( - "/_matrix/client/unstable/im.nheko.summary/rooms/:room_id_or_alias/summary", - get(client::get_room_summary_legacy) - ) - .ruma_route(client::well_known_support) - .ruma_route(client::well_known_client) - .route("/_conduwuit/server_version", get(client::conduwuit_server_version)) + .ruma_route(client::get_mutual_rooms_route) + .ruma_route(client::get_room_summary) + .route( + "/_matrix/client/unstable/im.nheko.summary/rooms/:room_id_or_alias/summary", + get(client::get_room_summary_legacy) + ) + .ruma_route(client::well_known_support) + .ruma_route(client::well_known_client) + .route("/_conduwuit/server_version", get(client::conduwuit_server_version)) .route("/_matrix/client/r0/rooms/:room_id/initialSync", get(initial_sync)) .route("/_matrix/client/v3/rooms/:room_id/initialSync", get(initial_sync)) .route("/client/server.json", get(client::syncv3_client_server_json)); @@ -233,7 +241,7 @@ pub fn build(router: Router, server: &Server) -> Router { } async fn initial_sync(_uri: Uri) -> impl IntoResponse { - Error::BadRequest(ErrorKind::GuestAccessForbidden, "Guest access not implemented") + err!(Request(GuestAccessForbidden("Guest access not implemented"))) } async fn federation_disabled() -> impl IntoResponse { err!(Config("allow_federation", "Federation is disabled.")) } diff --git a/src/api/router/mod.rs b/src/api/router/args.rs similarity index 90% rename from src/api/router/mod.rs rename to src/api/router/args.rs index c3e08c5b..b54a6f80 100644 --- a/src/api/router/mod.rs +++ b/src/api/router/args.rs @@ -1,24 +1,18 @@ -mod auth; -mod handler; -mod request; -mod response; - use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, BytesMut}; -use conduit::{debug, debug_warn, trace, warn}; +use conduit::{debug, debug_warn, trace, warn, Error, Result}; use ruma::{ api::{client::error::ErrorKind, IncomingRequest}, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; -use self::{auth::Auth, request::Request}; -pub(super) use self::{handler::RouterExt, response::RumaResponse}; -use crate::{service::appservice::RegistrationInfo, services, Error, Result}; +use super::{auth, auth::Auth, request, request::Request}; +use crate::{service::appservice::RegistrationInfo, services}; /// Extractor for Ruma request structs -pub(crate) struct Ruma { +pub(crate) struct Args { /// Request struct body pub(crate) body: T, @@ -44,7 +38,7 @@ pub(crate) struct Ruma { } #[async_trait] -impl FromRequest for Ruma +impl FromRequest for Args where T: IncomingRequest, { @@ -65,7 +59,7 @@ where } } -impl Deref for Ruma { +impl Deref for Args { type Target = T; fn deref(&self) -> &Self::Target { &self.body } diff --git a/src/router/router.rs b/src/router/router.rs index 2f6d5c46..7dfb089b 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -10,7 +10,7 @@ extern crate conduit_api as api; pub(crate) fn build(server: &Arc) -> Router { let state = service::services(); - let router = Router::new() + api::router::build(Router::new(), server) .route("/", get(it_works)) .fallback(not_found) .with_state(state); From 038b71fc9db286025614fc425add2e478abf79fc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 03:56:27 +0000 Subject: [PATCH 0514/2091] add state to router Signed-off-by: Jason Volk --- src/api/mod.rs | 1 + src/api/router.rs | 9 +++++---- src/api/router/handler.rs | 16 ++++++++-------- src/router/router.rs | 11 +++++------ 4 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index dda37caf..79382934 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -8,6 +8,7 @@ extern crate conduit_service as service; pub(crate) use conduit::{debug_info, debug_warn, pdu::PduEvent, utils, Error, Result}; pub(crate) use service::{services, user_is_local}; +pub use crate::router::State; pub(crate) use crate::router::{Ruma, RumaResponse}; conduit::mod_ctor! {} diff --git a/src/api/router.rs b/src/api/router.rs index 9b6f62a5..761c173c 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -9,15 +9,16 @@ use axum::{ routing::{any, get, post}, Router, }; -use conduit::{err, Error, Server}; +use conduit::{err, Server}; use http::Uri; -use ruma::api::client::error::ErrorKind; use self::handler::RouterExt; -pub(super) use self::{ar::Ruma, response::RumaResponse}; +pub(super) use self::{args::Args as Ruma, response::RumaResponse}; use crate::{client, server}; -pub fn build(router: Router, server: &Server) -> Router { +pub type State = &'static service::Services; + +pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; let router = router .ruma_route(client::get_supported_versions_route) diff --git a/src/api/router/handler.rs b/src/api/router/handler.rs index 73361989..d112ec58 100644 --- a/src/api/router/handler.rs +++ b/src/api/router/handler.rs @@ -10,7 +10,7 @@ use conduit::Result; use http::Method; use ruma::api::IncomingRequest; -use super::{Ruma, RumaResponse}; +use super::{Ruma, RumaResponse, State}; pub(in super::super) trait RouterExt { fn ruma_route(self, handler: H) -> Self @@ -18,7 +18,7 @@ pub(in super::super) trait RouterExt { H: RumaHandler; } -impl RouterExt for Router { +impl RouterExt for Router { fn ruma_route(self, handler: H) -> Self where H: RumaHandler, @@ -28,9 +28,9 @@ impl RouterExt for Router { } pub(in super::super) trait RumaHandler { - fn add_routes(&self, router: Router) -> Router; + fn add_routes(&self, router: Router) -> Router; - fn add_route(&self, router: Router, path: &str) -> Router; + fn add_route(&self, router: Router, path: &str) -> Router; } macro_rules! ruma_handler { @@ -41,17 +41,17 @@ macro_rules! ruma_handler { Req: IncomingRequest + Send + 'static, Ret: IntoResponse, Fut: Future> + Send, - Fun: FnOnce($($tx,)* Ruma) -> Fut + Clone + Send + Sync + 'static, - $( $tx: FromRequestParts<()> + Send + 'static, )* + Fun: FnOnce($($tx,)* Ruma,) -> Fut + Clone + Send + Sync + 'static, + $( $tx: FromRequestParts + Send + 'static, )* { - fn add_routes(&self, router: Router) -> Router { + fn add_routes(&self, router: Router) -> Router { Req::METADATA .history .all_paths() .fold(router, |router, path| self.add_route(router, path)) } - fn add_route(&self, router: Router, path: &str) -> Router { + fn add_route(&self, router: Router, path: &str) -> Router { let handle = self.clone(); let method = method_to_filter(&Req::METADATA.method); let action = |$($tx,)* req| async { handle($($tx,)* req).await.map(RumaResponse) }; diff --git a/src/router/router.rs b/src/router/router.rs index 7dfb089b..7c374b47 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -2,20 +2,19 @@ use std::sync::Arc; use axum::{response::IntoResponse, routing::get, Router}; use conduit::{Error, Server}; -use conduit_service as service; use http::{StatusCode, Uri}; use ruma::api::client::error::ErrorKind; extern crate conduit_api as api; +extern crate conduit_service as service; pub(crate) fn build(server: &Arc) -> Router { - let state = service::services(); - api::router::build(Router::new(), server) + let router = Router::::new(); + + api::router::build(router, server) .route("/", get(it_works)) .fallback(not_found) - .with_state(state); - - api::routes::build(router, server) + .with_state(service::services()) } async fn not_found(_uri: Uri) -> impl IntoResponse { From 838e4b9d8db24ee6533f3174cdcc56bfc247b293 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 04:40:14 +0000 Subject: [PATCH 0515/2091] log more details for panic in tower handler Signed-off-by: Jason Volk --- src/router/layers.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index 073940f1..67342eb3 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -5,7 +5,7 @@ use axum::{ Router, }; use axum_client_ip::SecureClientIpSource; -use conduit::{Result, Server}; +use conduit::{error, Result, Server}; use http::{ header::{self, HeaderName}, HeaderValue, Method, StatusCode, @@ -149,7 +149,7 @@ fn cors_layer(_server: &Server) -> CorsLayer { fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max(server.config.max_request_size) } #[allow(clippy::needless_pass_by_value)] -#[tracing::instrument(skip_all)] +#[tracing::instrument(skip_all, name = "panic")] fn catch_panic(err: Box) -> http::Response> { conduit_service::services() .server @@ -165,17 +165,17 @@ fn catch_panic(err: Box) -> http::Response Date: Mon, 24 Jun 2024 08:27:06 +0000 Subject: [PATCH 0516/2091] rename FileMeta::file to Option Signed-off-by: Jason Volk --- src/api/client/media.rs | 9 +++++--- src/service/media/mod.rs | 45 ++++++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 9d3fbabe..39640b23 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -200,12 +200,13 @@ pub(crate) async fn get_content_route( let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { + content, content_type, - file, content_disposition, }) = services().media.get(&mxc).await? { let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); + let file = content.expect("content"); Ok(get_content::v3::Response { file, @@ -282,8 +283,8 @@ pub(crate) async fn get_content_as_filename_route( let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { + content, content_type, - file, content_disposition, }) = services().media.get(&mxc).await? { @@ -293,6 +294,7 @@ pub(crate) async fn get_content_as_filename_route( Some(body.filename.clone()), )); + let file = content.expect("content"); Ok(get_content_as_filename::v3::Response { file, content_type, @@ -371,8 +373,8 @@ pub(crate) async fn get_content_thumbnail_route( let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); if let Some(FileMeta { + content, content_type, - file, content_disposition, }) = services() .media @@ -388,6 +390,7 @@ pub(crate) async fn get_content_thumbnail_route( .await? { let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); + let file = content.expect("content"); Ok(get_content_thumbnail::v3::Response { file, diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 9b1cbe06..2f687925 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -20,10 +20,9 @@ use crate::services; #[derive(Debug)] pub struct FileMeta { - #[allow(dead_code)] - pub content_disposition: Option, + pub content: Option>, pub content_type: Option, - pub file: Vec, + pub content_disposition: Option, } #[derive(Serialize, Default)] @@ -132,16 +131,16 @@ impl Service { /// Downloads a file. pub async fn get(&self, mxc: &str) -> Result> { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { - let mut file = Vec::new(); + let mut content = Vec::new(); let path = self.get_media_file(&key); BufReader::new(fs::File::open(path).await?) - .read_to_end(&mut file) + .read_to_end(&mut content) .await?; Ok(Some(FileMeta { - content_disposition, + content: Some(content), content_type, - file, + content_disposition, })) } else { Ok(None) @@ -283,29 +282,35 @@ impl Service { if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, width, height) { // Using saved thumbnail - let mut file = Vec::new(); + let mut content = Vec::new(); let path = self.get_media_file(&key); - fs::File::open(path).await?.read_to_end(&mut file).await?; + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; Ok(Some(FileMeta { - content_disposition, + content: Some(content), content_type, - file: file.clone(), + content_disposition, })) } else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { // Generate a thumbnail - let mut file = Vec::new(); + let mut content = Vec::new(); let path = self.get_media_file(&key); - fs::File::open(path).await?.read_to_end(&mut file).await?; + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; - if let Ok(image) = image::load_from_memory(&file) { + if let Ok(image) = image::load_from_memory(&content) { let original_width = image.width(); let original_height = image.height(); if width > original_width || height > original_height { return Ok(Some(FileMeta { - content_disposition, + content: Some(content), content_type, - file: file.clone(), + content_disposition, })); } @@ -350,16 +355,16 @@ impl Service { f.write_all(&thumbnail_bytes).await?; Ok(Some(FileMeta { - content_disposition, + content: Some(thumbnail_bytes), content_type, - file: thumbnail_bytes.clone(), + content_disposition, })) } else { // Couldn't parse file to generate thumbnail, send original Ok(Some(FileMeta { - content_disposition, + content: Some(content), content_type, - file: file.clone(), + content_disposition, })) } } else { From b903b46d16352e536e7fbecfbb2ab1f97ace8d4a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 07:21:10 +0000 Subject: [PATCH 0517/2091] split thumbnailing related into unit Signed-off-by: Jason Volk --- src/service/media/data.rs | 18 +++- src/service/media/mod.rs | 159 ++---------------------------- src/service/media/thumbnail.rs | 171 +++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 153 deletions(-) create mode 100644 src/service/media/thumbnail.rs diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 186d502d..e5856bbf 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -12,6 +12,13 @@ pub(crate) struct Data { url_previews: Arc, } +#[derive(Debug)] +pub(super) struct Metadata { + pub(super) content_disposition: Option, + pub(super) content_type: Option, + pub(super) key: Vec, +} + impl Data { pub(super) fn new(db: &Arc) -> Self { Self { @@ -104,9 +111,7 @@ impl Data { Ok(keys) } - pub(super) fn search_file_metadata( - &self, mxc: &str, width: u32, height: u32, - ) -> Result<(Option, Option, Vec)> { + pub(super) fn search_file_metadata(&self, mxc: &str, width: u32, height: u32) -> Result { let mut prefix = mxc.as_bytes().to_vec(); prefix.push(0xFF); prefix.extend_from_slice(&width.to_be_bytes()); @@ -141,7 +146,12 @@ impl Data { .map_err(|_| Error::bad_database("Content Disposition in mediaid_file is invalid unicode."))?, ) }; - Ok((content_disposition, content_type, key)) + + Ok(Metadata { + content_disposition, + content_type, + key, + }) } /// Gets all the media keys in our database (this includes all the metadata diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 2f687925..c969d1e8 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,13 +1,13 @@ mod data; mod tests; +mod thumbnail; -use std::{collections::HashMap, io::Cursor, num::Saturating as Sat, path::PathBuf, sync::Arc, time::SystemTime}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{checked, debug, debug_error, error, utils, Error, Result, Server}; -use data::Data; -use image::imageops::FilterType; +use conduit::{debug, debug_error, err, error, utils, Result, Server}; +use data::{Data, Metadata}; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; use tokio::{ @@ -107,30 +107,14 @@ impl Service { } } - /// Uploads or replaces a file thumbnail. - #[allow(clippy::too_many_arguments)] - pub async fn upload_thumbnail( - &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, - content_type: Option<&str>, width: u32, height: u32, file: &[u8], - ) -> Result<()> { - let key = if let Some(user) = sender_user { - self.db - .create_file_metadata(Some(user.as_str()), mxc, width, height, content_disposition, content_type)? - } else { - self.db - .create_file_metadata(None, mxc, width, height, content_disposition, content_type)? - }; - - //TODO: Dangling metadata in database if creation fails - let mut f = self.create_media_file(&key).await?; - f.write_all(file).await?; - - Ok(()) - } - /// Downloads a file. pub async fn get(&self, mxc: &str) -> Result> { - if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { + if let Ok(Metadata { + content_disposition, + content_type, + key, + }) = self.db.search_file_metadata(mxc, 0, 0) + { let mut content = Vec::new(); let path = self.get_media_file(&key); BufReader::new(fs::File::open(path).await?) @@ -249,129 +233,6 @@ impl Service { Ok(deletion_count) } - /// Returns width, height of the thumbnail and whether it should be cropped. - /// Returns None when the server should send the original file. - pub fn thumbnail_properties(&self, width: u32, height: u32) -> Option<(u32, u32, bool)> { - match (width, height) { - (0..=32, 0..=32) => Some((32, 32, true)), - (0..=96, 0..=96) => Some((96, 96, true)), - (0..=320, 0..=240) => Some((320, 240, false)), - (0..=640, 0..=480) => Some((640, 480, false)), - (0..=800, 0..=600) => Some((800, 600, false)), - _ => None, - } - } - - /// Downloads a file's thumbnail. - /// - /// Here's an example on how it works: - /// - /// - Client requests an image with width=567, height=567 - /// - Server rounds that up to (800, 600), so it doesn't have to save too - /// many thumbnails - /// - Server rounds that up again to (958, 600) to fix the aspect ratio - /// (only for width,height>96) - /// - Server creates the thumbnail and sends it to the user - /// - /// For width,height <= 96 the server uses another thumbnailing algorithm - /// which crops the image afterwards. - pub async fn get_thumbnail(&self, mxc: &str, width: u32, height: u32) -> Result> { - let (width, height, crop) = self - .thumbnail_properties(width, height) - .unwrap_or((0, 0, false)); // 0, 0 because that's the original file - - if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, width, height) { - // Using saved thumbnail - let mut content = Vec::new(); - let path = self.get_media_file(&key); - fs::File::open(path) - .await? - .read_to_end(&mut content) - .await?; - - Ok(Some(FileMeta { - content: Some(content), - content_type, - content_disposition, - })) - } else if let Ok((content_disposition, content_type, key)) = self.db.search_file_metadata(mxc, 0, 0) { - // Generate a thumbnail - let mut content = Vec::new(); - let path = self.get_media_file(&key); - fs::File::open(path) - .await? - .read_to_end(&mut content) - .await?; - - if let Ok(image) = image::load_from_memory(&content) { - let original_width = image.width(); - let original_height = image.height(); - if width > original_width || height > original_height { - return Ok(Some(FileMeta { - content: Some(content), - content_type, - content_disposition, - })); - } - - let thumbnail = if crop { - image.resize_to_fill(width, height, FilterType::CatmullRom) - } else { - let (exact_width, exact_height) = { - let ratio = Sat(original_width) * Sat(height); - let nratio = Sat(width) * Sat(original_height); - - let use_width = nratio <= ratio; - let intermediate = if use_width { - Sat(original_height) * Sat(checked!(width / original_width)?) - } else { - Sat(original_width) * Sat(checked!(height / original_height)?) - }; - - if use_width { - (width, intermediate.0) - } else { - (intermediate.0, height) - } - }; - - image.thumbnail_exact(exact_width, exact_height) - }; - - let mut thumbnail_bytes = Vec::new(); - thumbnail.write_to(&mut Cursor::new(&mut thumbnail_bytes), image::ImageFormat::Png)?; - - // Save thumbnail in database so we don't have to generate it again next time - let thumbnail_key = self.db.create_file_metadata( - None, - mxc, - width, - height, - content_disposition.as_deref(), - content_type.as_deref(), - )?; - - let mut f = self.create_media_file(&thumbnail_key).await?; - f.write_all(&thumbnail_bytes).await?; - - Ok(Some(FileMeta { - content: Some(thumbnail_bytes), - content_type, - content_disposition, - })) - } else { - // Couldn't parse file to generate thumbnail, send original - Ok(Some(FileMeta { - content: Some(content), - content_type, - content_disposition, - })) - } - } else { - Ok(None) - } - } - pub async fn get_url_preview(&self, url: &str) -> Option { self.db.get_url_preview(url) } /// TODO: use this? diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs new file mode 100644 index 00000000..50be4ee1 --- /dev/null +++ b/src/service/media/thumbnail.rs @@ -0,0 +1,171 @@ +use std::{io::Cursor, num::Saturating as Sat}; + +use conduit::{checked, Result}; +use image::{imageops::FilterType, DynamicImage}; +use ruma::OwnedUserId; +use tokio::{ + fs, + io::{AsyncReadExt, AsyncWriteExt}, +}; + +use super::{data::Metadata, FileMeta}; + +impl super::Service { + /// Uploads or replaces a file thumbnail. + #[allow(clippy::too_many_arguments)] + pub async fn upload_thumbnail( + &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, + content_type: Option<&str>, width: u32, height: u32, file: &[u8], + ) -> Result<()> { + let key = if let Some(user) = sender_user { + self.db + .create_file_metadata(Some(user.as_str()), mxc, width, height, content_disposition, content_type)? + } else { + self.db + .create_file_metadata(None, mxc, width, height, content_disposition, content_type)? + }; + + //TODO: Dangling metadata in database if creation fails + let mut f = self.create_media_file(&key).await?; + f.write_all(file).await?; + + Ok(()) + } + + /// Downloads a file's thumbnail. + /// + /// Here's an example on how it works: + /// + /// - Client requests an image with width=567, height=567 + /// - Server rounds that up to (800, 600), so it doesn't have to save too + /// many thumbnails + /// - Server rounds that up again to (958, 600) to fix the aspect ratio + /// (only for width,height>96) + /// - Server creates the thumbnail and sends it to the user + /// + /// For width,height <= 96 the server uses another thumbnailing algorithm + /// which crops the image afterwards. + #[tracing::instrument(skip(self), name = "thumbnail", level = "debug")] + pub async fn get_thumbnail(&self, mxc: &str, width: u32, height: u32) -> Result> { + // 0, 0 because that's the original file + let (width, height, crop) = thumbnail_properties(width, height).unwrap_or((0, 0, false)); + + if let Ok(metadata) = self.db.search_file_metadata(mxc, width, height) { + self.get_thumbnail_saved(metadata).await + } else if let Ok(metadata) = self.db.search_file_metadata(mxc, 0, 0) { + self.get_thumbnail_generate(mxc, width, height, crop, metadata) + .await + } else { + Ok(None) + } + } + + /// Using saved thumbnail + #[tracing::instrument(skip(self), name = "saved", level = "debug")] + async fn get_thumbnail_saved(&self, data: Metadata) -> Result> { + let mut content = Vec::new(); + let path = self.get_media_file(&data.key); + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; + + Ok(Some(into_filemeta(data, content))) + } + + /// Generate a thumbnail + #[tracing::instrument(skip(self), name = "generate", level = "debug")] + async fn get_thumbnail_generate( + &self, mxc: &str, width: u32, height: u32, crop: bool, data: Metadata, + ) -> Result> { + let mut content = Vec::new(); + let path = self.get_media_file(&data.key); + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; + + let Ok(image) = image::load_from_memory(&content) else { + // Couldn't parse file to generate thumbnail, send original + return Ok(Some(into_filemeta(data, content))); + }; + + if width > image.width() || height > image.height() { + return Ok(Some(into_filemeta(data, content))); + } + + let mut thumbnail_bytes = Vec::new(); + let thumbnail = thumbnail_generate(&image, width, height, crop)?; + thumbnail.write_to(&mut Cursor::new(&mut thumbnail_bytes), image::ImageFormat::Png)?; + + // Save thumbnail in database so we don't have to generate it again next time + let thumbnail_key = self.db.create_file_metadata( + None, + mxc, + width, + height, + data.content_disposition.as_deref(), + data.content_type.as_deref(), + )?; + + let mut f = self.create_media_file(&thumbnail_key).await?; + f.write_all(&thumbnail_bytes).await?; + + Ok(Some(into_filemeta(data, thumbnail_bytes))) + } +} + +fn thumbnail_generate(image: &DynamicImage, width: u32, height: u32, crop: bool) -> Result { + let thumbnail = if crop { + image.resize_to_fill(width, height, FilterType::CatmullRom) + } else { + let (exact_width, exact_height) = thumbnail_dimension(image, width, height)?; + image.thumbnail_exact(exact_width, exact_height) + }; + + Ok(thumbnail) +} + +fn thumbnail_dimension(image: &DynamicImage, width: u32, height: u32) -> Result<(u32, u32)> { + let original_width = image.width(); + let original_height = image.height(); + + let ratio = Sat(original_width) * Sat(height); + let nratio = Sat(width) * Sat(original_height); + let use_width = nratio <= ratio; + + let intermediate = if use_width { + Sat(original_height) * Sat(checked!(width / original_width)?) + } else { + Sat(original_width) * Sat(checked!(height / original_height)?) + }; + + let dims = if use_width { + (width, intermediate.0) + } else { + (intermediate.0, height) + }; + + Ok(dims) +} + +/// Returns width, height of the thumbnail and whether it should be cropped. +/// Returns None when the server should send the original file. +fn thumbnail_properties(width: u32, height: u32) -> Option<(u32, u32, bool)> { + match (width, height) { + (0..=32, 0..=32) => Some((32, 32, true)), + (0..=96, 0..=96) => Some((96, 96, true)), + (0..=320, 0..=240) => Some((320, 240, false)), + (0..=640, 0..=480) => Some((640, 480, false)), + (0..=800, 0..=600) => Some((800, 600, false)), + _ => None, + } +} + +fn into_filemeta(data: Metadata, content: Vec) -> FileMeta { + FileMeta { + content: Some(content), + content_type: data.content_type, + content_disposition: data.content_disposition, + } +} From d67f19a55d98a535622f98015fab434ee2ec0278 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 04:19:43 +0000 Subject: [PATCH 0518/2091] cleanup some error callsites Signed-off-by: Jason Volk --- src/api/router/args.rs | 30 ++++-------- src/api/router/request.rs | 11 +++-- src/api/server/send.rs | 29 ++++-------- src/core/error/err.rs | 1 + src/core/error/mod.rs | 2 +- src/core/utils/math.rs | 8 ++-- src/service/admin/mod.rs | 8 ++-- src/service/media/mod.rs | 52 +++++++------------- src/service/users/data.rs | 99 +++++++++++++++++---------------------- 9 files changed, 96 insertions(+), 144 deletions(-) diff --git a/src/api/router/args.rs b/src/api/router/args.rs index b54a6f80..776ce4f4 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -2,11 +2,8 @@ use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, BytesMut}; -use conduit::{debug, debug_warn, trace, warn, Error, Result}; -use ruma::{ - api::{client::error::ErrorKind, IncomingRequest}, - CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, -}; +use conduit::{debug, err, trace, Error, Result}; +use ruma::{api::IncomingRequest, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId}; use super::{auth, auth::Auth, request, request::Request}; use crate::{service::appservice::RegistrationInfo, services}; @@ -103,21 +100,14 @@ where let mut http_request = hyper::Request::builder() .uri(request.parts.uri.clone()) .method(request.parts.method.clone()); - *http_request.headers_mut().unwrap() = request.parts.headers.clone(); - let http_request = http_request.body(body).unwrap(); - debug!( - "{:?} {:?} {:?}", - http_request.method(), - http_request.uri(), - http_request.headers() - ); + *http_request.headers_mut().expect("mutable http headers") = request.parts.headers.clone(); + let http_request = http_request.body(body).expect("http request body"); - trace!("{:?} {:?} {:?}", http_request.method(), http_request.uri(), json_body); - let body = T::try_from_http_request(http_request, &request.path).map_err(|e| { - warn!("try_from_http_request failed: {e:?}",); - debug_warn!("JSON body: {:?}", json_body); - Error::BadRequest(ErrorKind::BadJson, "Failed to deserialize request.") - })?; + let headers = http_request.headers(); + let method = http_request.method(); + let uri = http_request.uri(); + debug!("{method:?} {uri:?} {headers:?}"); + trace!("{method:?} {uri:?} {json_body:?}"); - Ok(body) + T::try_from_http_request(http_request, &request.path).map_err(|e| err!(Request(BadJson(debug_warn!("{e}"))))) } diff --git a/src/api/router/request.rs b/src/api/router/request.rs index 56c76619..bed8d057 100644 --- a/src/api/router/request.rs +++ b/src/api/router/request.rs @@ -2,11 +2,11 @@ use std::str; use axum::{extract::Path, RequestExt, RequestPartsExt}; use bytes::Bytes; +use conduit::err; use http::request::Parts; -use ruma::api::client::error::ErrorKind; use serde::Deserialize; -use crate::{services, Error, Result}; +use crate::{services, Result}; #[derive(Deserialize)] pub(super) struct QueryParams { @@ -26,14 +26,15 @@ pub(super) async fn from(request: hyper::Request) -> Result> = parts.extract().await?; - let query = serde_html_form::from_str(parts.uri.query().unwrap_or_default()) - .map_err(|_| Error::BadRequest(ErrorKind::Unknown, "Failed to read query parameters"))?; + let query = parts.uri.query().unwrap_or_default(); + let query = + serde_html_form::from_str(query).map_err(|e| err!(Request(Unknown("Failed to read query parameters: {e}"))))?; let max_body_size = services().globals.config.max_request_size; let body = axum::body::to_bytes(body, max_body_size) .await - .map_err(|_| Error::BadRequest(ErrorKind::TooLarge, "Request body too large"))?; + .map_err(|e| err!(Request(TooLarge("Request body too large: {e}"))))?; Ok(Request { path, diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 122f564f..31f8e73b 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; use axum_client_ip::InsecureClientIp; -use conduit::debug_warn; +use conduit::{debug, debug_warn, err, trace, warn, Err}; use ruma::{ api::{ client::error::ErrorKind, @@ -18,7 +18,6 @@ use ruma::{ OwnedEventId, ServerName, }; use tokio::sync::RwLock; -use tracing::{debug, error, trace, warn}; use crate::{ service::rooms::event_handler::parse_incoming_pdu, @@ -39,24 +38,17 @@ pub(crate) async fn send_transaction_message_route( let origin = body.origin.as_ref().expect("server is authenticated"); if *origin != body.body.origin { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Not allowed to send transactions on behalf of other servers", - )); + return Err!(Request(Forbidden( + "Not allowed to send transactions on behalf of other servers" + ))); } if body.pdus.len() > 50_usize { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Not allowed to send more than 50 PDUs in one transaction", - )); + return Err!(Request(Forbidden("Not allowed to send more than 50 PDUs in one transaction"))); } if body.edus.len() > 100_usize { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Not allowed to send more than 100 EDUs in one transaction", - )); + return Err!(Request(Forbidden("Not allowed to send more than 100 EDUs in one transaction"))); } let txn_start_time = Instant::now(); @@ -392,10 +384,9 @@ async fn handle_edu_direct_to_device( target_user_id, target_device_id, &ev_type.to_string(), - event.deserialize_as().map_err(|e| { - error!("To-Device event is invalid: {event:?} {e}"); - Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid") - })?, + event + .deserialize_as() + .map_err(|e| err!(Request(InvalidParam(error!("To-Device event is invalid: {e}")))))?, )?; }, @@ -408,7 +399,7 @@ async fn handle_edu_direct_to_device( &ev_type.to_string(), event .deserialize_as() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid"))?, + .map_err(|e| err!(Request(InvalidParam("Event is invalid: {e}"))))?, )?; } }, diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 66634f2b..55e38f18 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -31,6 +31,7 @@ //! Err(Database(error!("problem with db: {msg}")))` logs the error at the //! callsite and then returns the error with the same string. Caller has the //! option of replacing `error!` with `debug_error!`. + #[macro_export] macro_rules! Err { ($($args:tt)*) => { diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 1f0d56c1..9439261e 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -78,7 +78,7 @@ pub enum Error { // conduwuit #[error("Arithmetic operation failed: {0}")] - Arithmetic(&'static str), + Arithmetic(Cow<'static, str>), #[error("There was a problem with the '{0}' directive in your configuration: {1}")] Config(&'static str, Cow<'static, str>), #[error("{0}")] diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index 155721e7..96ac6dc2 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -2,14 +2,14 @@ use std::{cmp, time::Duration}; pub use checked_ops::checked_ops; -use crate::{Error, Result}; +use crate::{Err, Error, Result}; /// Checked arithmetic expression. Returns a Result #[macro_export] macro_rules! checked { ($($input:tt)*) => { $crate::utils::math::checked_ops!($($input)*) - .ok_or_else(|| $crate::Error::Arithmetic("operation overflowed or result invalid")) + .ok_or_else(|| $crate::err!(Arithmetic("operation overflowed or result invalid"))) } } @@ -21,7 +21,7 @@ macro_rules! validated { ($($input:tt)*) => { //#[allow(clippy::arithmetic_side_effects)] { //Some($($input)*) - // .ok_or_else(|| $crate::Error::Arithmetic("this error should never been seen")) + // .ok_or_else(|| $crate::err!(Arithmetic("this error should never been seen"))) //} //NOTE: remove me when stmt_expr_attributes is stable @@ -57,7 +57,7 @@ pub fn continue_exponential_backoff(min: Duration, max: Duration, elapsed: Durat #[allow(clippy::as_conversions)] pub fn usize_from_f64(val: f64) -> Result { if val < 0.0 { - return Err(Error::Arithmetic("Converting negative float to unsigned integer")); + return Err!(Arithmetic("Converting negative float to unsigned integer")); } //SAFETY: diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 41019cd1..f63ebf09 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,7 +9,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{debug, error, Error, Result}; +use conduit::{debug, error, error::default_log, Error, Result}; pub use create::create_admin_room; pub use grant::make_user_admin; use loole::{Receiver, Sender}; @@ -239,9 +239,9 @@ async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, use .build_and_append_pdu(response_pdu, user_id, room_id, &state_lock) .await { - if let Err(e) = handle_response_error(e, room_id, user_id, &state_lock).await { - error!("{e}"); - } + handle_response_error(e, room_id, user_id, &state_lock) + .await + .unwrap_or_else(default_log); } } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index c969d1e8..1638235b 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -6,7 +6,7 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, debug_error, err, error, utils, Result, Server}; +use conduit::{debug, debug_error, err, error, utils, Err, Result, Server}; use data::{Data, Metadata}; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; @@ -100,10 +100,9 @@ impl Service { Ok(()) } else { - error!("Failed to find any media keys for MXC \"{mxc}\" in our database (MXC does not exist)"); - Err(Error::bad_database( - "Failed to find any media keys for the provided MXC in our database (MXC does not exist)", - )) + Err!(Database(error!( + "Failed to find any media keys for MXC {mxc:?} in our database." + ))) } } @@ -137,23 +136,16 @@ impl Service { let all_keys = self.db.get_all_media_keys(); let user_duration: SystemTime = match cyborgtime::parse_duration(&time) { - Ok(duration) => { - debug!("Parsed duration: {:?}", duration); - debug!("System time now: {:?}", SystemTime::now()); - SystemTime::now().checked_sub(duration).ok_or_else(|| { - Error::bad_database("Duration specified is not valid against the current system time") - })? - }, - Err(e) => { - error!("Failed to parse user-specified time duration: {}", e); - return Err(Error::bad_database("Failed to parse user-specified time duration.")); - }, + Err(e) => return Err!(Database(error!("Failed to parse specified time duration: {e}"))), + Ok(duration) => SystemTime::now() + .checked_sub(duration) + .ok_or(err!(Arithmetic("Duration {duration:?} is too large")))?, }; let mut remote_mxcs: Vec = vec![]; for key in all_keys { - debug!("Full MXC key from database: {:?}", key); + debug!("Full MXC key from database: {key:?}"); // we need to get the MXC URL from the first part of the key (the first 0xff / // 255 push). this is all necessary because of conduit using magic keys for @@ -162,24 +154,19 @@ impl Service { let mxc = parts .next() .map(|bytes| { - utils::string_from_bytes(bytes).map_err(|e| { - error!("Failed to parse MXC unicode bytes from our database: {}", e); - Error::bad_database("Failed to parse MXC unicode bytes from our database") - }) + utils::string_from_bytes(bytes) + .map_err(|e| err!(Database(error!("Failed to parse MXC unicode bytes from our database: {e}")))) }) .transpose()?; let Some(mxc_s) = mxc else { - return Err(Error::bad_database( - "Parsed MXC URL unicode bytes from database but still is None", - )); + return Err!(Database("Parsed MXC URL unicode bytes from database but still is None")); }; - debug!("Parsed MXC key to URL: {}", mxc_s); - + debug!("Parsed MXC key to URL: {mxc_s}"); let mxc = OwnedMxcUri::from(mxc_s); if mxc.server_name() == Ok(services().globals.server_name()) { - debug!("Ignoring local media MXC: {}", mxc); + debug!("Ignoring local media MXC: {mxc}"); // ignore our own MXC URLs as this would be local media. continue; } @@ -198,14 +185,14 @@ impl Service { }, Err(err) => { if force { - error!("Could not delete MXC path {:?}: {:?}. Skipping...", path, err); + error!("Could not delete MXC path {path:?}: {err:?}. Skipping..."); continue; } return Err(err.into()); }, }; - debug!("File created at: {:?}", file_created_at); + debug!("File created at: {file_created_at:?}"); if file_created_at <= user_duration { debug!("File is within user duration, pushing to list of file paths and keys to delete."); remote_mxcs.push(mxc.to_string()); @@ -215,15 +202,12 @@ impl Service { debug!( "Finished going through all our media in database for eligible keys to delete, checking if these are empty" ); - if remote_mxcs.is_empty() { - return Err(Error::bad_database("Did not found any eligible MXCs to delete.")); + return Err!(Database("Did not found any eligible MXCs to delete.")); } - debug!("Deleting media now in the past \"{:?}\".", user_duration); - + debug!("Deleting media now in the past {user_duration:?}."); let mut deletion_count: usize = 0; - for mxc in remote_mxcs { debug!("Deleting MXC {mxc} from database and filesystem"); self.delete(&mxc).await?; diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 302eae9d..5546adb1 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, mem::size_of, sync::Arc}; -use conduit::{debug_info, utils, warn, Error, Result}; +use conduit::{debug_info, err, utils, warn, Err, Error, Result}; use database::{Database, Map}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, @@ -87,19 +87,19 @@ impl Data { let mut parts = bytes.split(|&b| b == 0xFF); let user_bytes = parts .next() - .ok_or_else(|| Error::bad_database("User ID in token_userdeviceid is invalid."))?; + .ok_or_else(|| err!(Database("User ID in token_userdeviceid is invalid.")))?; let device_bytes = parts .next() - .ok_or_else(|| Error::bad_database("Device ID in token_userdeviceid is invalid."))?; + .ok_or_else(|| err!(Database("Device ID in token_userdeviceid is invalid.")))?; Ok(Some(( UserId::parse( utils::string_from_bytes(user_bytes) - .map_err(|_| Error::bad_database("User ID in token_userdeviceid is invalid unicode."))?, + .map_err(|e| err!(Database("User ID in token_userdeviceid is invalid unicode. {e}")))?, ) - .map_err(|_| Error::bad_database("User ID in token_userdeviceid is invalid."))?, + .map_err(|e| err!(Database("User ID in token_userdeviceid is invalid. {e}")))?, utils::string_from_bytes(device_bytes) - .map_err(|_| Error::bad_database("Device ID in token_userdeviceid is invalid."))?, + .map_err(|e| err!(Database("Device ID in token_userdeviceid is invalid. {e}")))?, ))) }) } @@ -109,9 +109,9 @@ impl Data { Box::new(self.userid_password.iter().map(|(bytes, _)| { UserId::parse( utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("User ID in userid_password is invalid unicode."))?, + .map_err(|e| err!(Database("User ID in userid_password is invalid unicode. {e}")))?, ) - .map_err(|_| Error::bad_database("User ID in userid_password is invalid.")) + .map_err(|e| err!(Database("User ID in userid_password is invalid. {e}"))) })) } @@ -165,7 +165,7 @@ impl Data { .map_or(Ok(None), |bytes| { Ok(Some( utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Displayname in db is invalid."))?, + .map_err(|e| err!(Database("Displayname in db is invalid. {e}")))?, )) }) } @@ -188,10 +188,8 @@ impl Data { self.userid_avatarurl .get(user_id.as_bytes())? .map(|bytes| { - let s_bytes = utils::string_from_bytes(&bytes).map_err(|e| { - warn!("Avatar URL in db is invalid: {}", e); - Error::bad_database("Avatar URL in db is invalid.") - })?; + let s_bytes = utils::string_from_bytes(&bytes) + .map_err(|e| err!(Database(warn!("Avatar URL in db is invalid: {e}"))))?; let mxc_uri: OwnedMxcUri = s_bytes.into(); Ok(mxc_uri) }) @@ -215,10 +213,7 @@ impl Data { self.userid_blurhash .get(user_id.as_bytes())? .map(|bytes| { - let s = utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?; - - Ok(s) + utils::string_from_bytes(&bytes).map_err(|e| err!(Database("Avatar URL in db is invalid. {e}"))) }) .transpose() } @@ -314,9 +309,9 @@ impl Data { bytes .rsplit(|&b| b == 0xFF) .next() - .ok_or_else(|| Error::bad_database("UserDevice ID in db is invalid."))?, + .ok_or_else(|| err!(Database("UserDevice ID in db is invalid.")))?, ) - .map_err(|_| Error::bad_database("Device ID in userdeviceid_metadata is invalid."))? + .map_err(|e| err!(Database("Device ID in userdeviceid_metadata is invalid. {e}")))? .into()) }), ) @@ -330,13 +325,9 @@ impl Data { // should not be None, but we shouldn't assert either lol... if self.userdeviceid_metadata.get(&userdeviceid)?.is_none() { - warn!( - "Called set_token for a non-existent user \"{}\" and/or device ID \"{}\" with no metadata in database", - user_id, device_id - ); - return Err(Error::bad_database( - "User does not exist or device ID has no metadata in database.", - )); + return Err!(Database(error!( + "User {user_id:?} does not exist or device ID {device_id:?} has no metadata." + ))); } // Remove old token @@ -366,14 +357,9 @@ impl Data { // Only existing devices should be able to call this, but we shouldn't assert // either... if self.userdeviceid_metadata.get(&key)?.is_none() { - warn!( - "Called add_one_time_key for a non-existent user \"{}\" and/or device ID \"{}\" with no metadata in \ - database", - user_id, device_id - ); - return Err(Error::bad_database( - "User does not exist or device ID has no metadata in database.", - )); + return Err!(Database(error!( + "User {user_id:?} does not exist or device ID {device_id:?} has no metadata." + ))); } key.push(0xFF); @@ -401,7 +387,7 @@ impl Data { .get(user_id.as_bytes())? .map_or(Ok(0), |bytes| { utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")) + .map_err(|e| err!(Database("Count in roomid_lastroomactiveupdate is invalid. {e}"))) }) } @@ -429,11 +415,10 @@ impl Data { serde_json::from_slice( key.rsplit(|&b| b == 0xFF) .next() - .ok_or_else(|| Error::bad_database("OneTimeKeyId in db is invalid."))?, + .ok_or_else(|| err!(Database("OneTimeKeyId in db is invalid.")))?, ) - .map_err(|_| Error::bad_database("OneTimeKeyId in db is invalid."))?, - serde_json::from_slice(&value) - .map_err(|_| Error::bad_database("OneTimeKeys in db are invalid."))?, + .map_err(|e| err!(Database("OneTimeKeyId in db is invalid. {e}")))?, + serde_json::from_slice(&value).map_err(|e| err!(Database("OneTimeKeys in db are invalid. {e}")))?, )) }) .transpose() @@ -457,9 +442,9 @@ impl Data { bytes .rsplit(|&b| b == 0xFF) .next() - .ok_or_else(|| Error::bad_database("OneTimeKey ID in db is invalid."))?, + .ok_or_else(|| err!(Database("OneTimeKey ID in db is invalid.")))?, ) - .map_err(|_| Error::bad_database("DeviceKeyId in db is invalid."))? + .map_err(|e| err!(Database("DeviceKeyId in db is invalid. {e}")))? .algorithm(), ) }) { @@ -581,19 +566,19 @@ impl Data { .get(&key)? .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Tried to sign nonexistent key."))?, ) - .map_err(|_| Error::bad_database("key in keyid_key is invalid."))?; + .map_err(|e| err!(Database("key in keyid_key is invalid. {e}")))?; let signatures = cross_signing_key .get_mut("signatures") - .ok_or_else(|| Error::bad_database("key in keyid_key has no signatures field."))? + .ok_or_else(|| err!(Database("key in keyid_key has no signatures field.")))? .as_object_mut() - .ok_or_else(|| Error::bad_database("key in keyid_key has invalid signatures field."))? + .ok_or_else(|| err!(Database("key in keyid_key has invalid signatures field.")))? .entry(sender_id.to_string()) .or_insert_with(|| serde_json::Map::new().into()); signatures .as_object_mut() - .ok_or_else(|| Error::bad_database("signatures in keyid_key for a user is invalid."))? + .ok_or_else(|| err!(Database("signatures in keyid_key for a user is invalid.")))? .insert(signature.0, signature.1.into()); self.keyid_key.insert( @@ -640,7 +625,7 @@ impl Data { Error::bad_database("User ID in devicekeychangeid_userid is invalid unicode.") })?, ) - .map_err(|_| Error::bad_database("User ID in devicekeychangeid_userid is invalid.")) + .map_err(|e| err!(Database("User ID in devicekeychangeid_userid is invalid. {e}"))) }), ) } @@ -685,7 +670,7 @@ impl Data { self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| { Ok(Some( - serde_json::from_slice(&bytes).map_err(|_| Error::bad_database("DeviceKeys in db are invalid."))?, + serde_json::from_slice(&bytes).map_err(|e| err!(Database("DeviceKeys in db are invalid. {e}")))?, )) }) } @@ -719,7 +704,7 @@ impl Data { ) -> Result>> { self.keyid_key.get(key)?.map_or(Ok(None), |bytes| { let mut cross_signing_key = serde_json::from_slice::(&bytes) - .map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?; + .map_err(|e| err!(Database("CrossSigningKey in db is invalid. {e}")))?; clean_signatures(&mut cross_signing_key, sender_user, user_id, allowed_signatures)?; Ok(Some(Raw::from_json( @@ -751,7 +736,7 @@ impl Data { self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| { Ok(Some( serde_json::from_slice(&bytes) - .map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?, + .map_err(|e| err!(Database("CrossSigningKey in db is invalid. {e}")))?, )) }) }) @@ -792,7 +777,7 @@ impl Data { for (_, value) in self.todeviceid_events.scan_prefix(prefix) { events.push( serde_json::from_slice(&value) - .map_err(|_| Error::bad_database("Event in todeviceid_events is invalid."))?, + .map_err(|e| err!(Database("Event in todeviceid_events is invalid. {e}")))?, ); } @@ -816,7 +801,7 @@ impl Data { Ok::<_, Error>(( key.clone(), utils::u64_from_bytes(&key[key.len().saturating_sub(size_of::())..key.len()]) - .map_err(|_| Error::bad_database("ToDeviceId has invalid count bytes."))?, + .map_err(|e| err!(Database("ToDeviceId has invalid count bytes. {e}")))?, )) }) .filter_map(Result::ok) @@ -877,7 +862,7 @@ impl Data { .get(user_id.as_bytes())? .map_or(Ok(None), |bytes| { utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Invalid devicelistversion in db.")) + .map_err(|e| err!(Database("Invalid devicelistversion in db. {e}"))) .map(Some) }) } @@ -893,7 +878,7 @@ impl Data { .scan_prefix(key) .map(|(_, bytes)| { serde_json::from_slice::(&bytes) - .map_err(|_| Error::bad_database("Device in userdeviceid_metadata is invalid.")) + .map_err(|e| err!(Database("Device in userdeviceid_metadata is invalid. {e}"))) }), ) } @@ -920,7 +905,7 @@ impl Data { let raw = self.userfilterid_filter.get(&key)?; if let Some(raw) = raw { - serde_json::from_slice(&raw).map_err(|_| Error::bad_database("Invalid filter event in db.")) + serde_json::from_slice(&raw).map_err(|e| err!(Database("Invalid filter event in db. {e}"))) } else { Ok(None) } @@ -954,7 +939,7 @@ impl Data { let expires_at = u64::from_be_bytes( expires_at_bytes .try_into() - .map_err(|_| Error::bad_database("expires_at in openid_userid is invalid u64."))?, + .map_err(|e| err!(Database("expires_at in openid_userid is invalid u64. {e}")))?, ); if expires_at < utils::millis_since_unix_epoch() { @@ -966,9 +951,9 @@ impl Data { UserId::parse( utils::string_from_bytes(user_bytes) - .map_err(|_| Error::bad_database("User ID in openid_userid is invalid unicode."))?, + .map_err(|e| err!(Database("User ID in openid_userid is invalid unicode. {e}")))?, ) - .map_err(|_| Error::bad_database("User ID in openid_userid is invalid.")) + .map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) } } From 4430e4dee037c02ba02a3e5f463edbc0f66d800a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 10:37:16 +0000 Subject: [PATCH 0519/2091] de-global some services() Signed-off-by: Jason Volk --- src/api/server/send.rs | 89 +++++++++++++++++++++++------------------- 1 file changed, 49 insertions(+), 40 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 31f8e73b..7c699e95 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; +use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{debug, debug_warn, err, trace, warn, Err}; use ruma::{ @@ -21,7 +22,7 @@ use tokio::sync::RwLock; use crate::{ service::rooms::event_handler::parse_incoming_pdu, - services, + services::Services, utils::{self}, Error, Result, Ruma, }; @@ -33,7 +34,8 @@ type ResolvedMap = BTreeMap>; /// Push EDUs and PDUs to this server. #[tracing::instrument(skip_all, fields(%client), name = "send")] pub(crate) async fn send_transaction_message_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State<&Services>, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); @@ -61,8 +63,8 @@ pub(crate) async fn send_transaction_message_route( "Starting txn", ); - let resolved_map = handle_pdus(&client, &body, origin, &txn_start_time).await?; - handle_edus(&client, &body, origin).await?; + let resolved_map = handle_pdus(services, &client, &body, origin, &txn_start_time).await?; + handle_edus(services, &client, &body, origin).await?; debug!( pdus = ?body.pdus.len(), @@ -82,7 +84,8 @@ pub(crate) async fn send_transaction_message_route( } async fn handle_pdus( - _client: &IpAddr, body: &Ruma, origin: &ServerName, txn_start_time: &Instant, + services: &Services, _client: &IpAddr, body: &Ruma, origin: &ServerName, + txn_start_time: &Instant, ) -> Result { let mut parsed_pdus = Vec::with_capacity(body.pdus.len()); for pdu in &body.pdus { @@ -102,7 +105,7 @@ async fn handle_pdus( // corresponding signing keys let pub_key_map = RwLock::new(BTreeMap::new()); if !parsed_pdus.is_empty() { - services() + services .rooms .event_handler .fetch_required_signing_keys(parsed_pdus.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) @@ -118,7 +121,7 @@ async fn handle_pdus( let mut resolved_map = BTreeMap::new(); for (event_id, value, room_id) in parsed_pdus { let pdu_start_time = Instant::now(); - let mutex_lock = services() + let mutex_lock = services .rooms .event_handler .mutex_federation @@ -126,7 +129,7 @@ async fn handle_pdus( .await; resolved_map.insert( event_id.clone(), - services() + services .rooms .event_handler .handle_incoming_pdu(origin, &room_id, &event_id, value, true, &pub_key_map) @@ -154,7 +157,7 @@ async fn handle_pdus( } async fn handle_edus( - client: &IpAddr, body: &Ruma, origin: &ServerName, + services: &Services, client: &IpAddr, body: &Ruma, origin: &ServerName, ) -> Result<()> { for edu in body .edus @@ -162,12 +165,12 @@ async fn handle_edus( .filter_map(|edu| serde_json::from_str::(edu.json().get()).ok()) { match edu { - Edu::Presence(presence) => handle_edu_presence(client, origin, presence).await?, - Edu::Receipt(receipt) => handle_edu_receipt(client, origin, receipt).await?, - Edu::Typing(typing) => handle_edu_typing(client, origin, typing).await?, - Edu::DeviceListUpdate(content) => handle_edu_device_list_update(client, origin, content).await?, - Edu::DirectToDevice(content) => handle_edu_direct_to_device(client, origin, content).await?, - Edu::SigningKeyUpdate(content) => handle_edu_signing_key_update(client, origin, content).await?, + Edu::Presence(presence) => handle_edu_presence(services, client, origin, presence).await?, + Edu::Receipt(receipt) => handle_edu_receipt(services, client, origin, receipt).await?, + Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await?, + Edu::DeviceListUpdate(content) => handle_edu_device_list_update(services, client, origin, content).await?, + Edu::DirectToDevice(content) => handle_edu_direct_to_device(services, client, origin, content).await?, + Edu::SigningKeyUpdate(content) => handle_edu_signing_key_update(services, client, origin, content).await?, Edu::_Custom(ref _custom) => { debug_warn!(?body.edus, "received custom/unknown EDU"); }, @@ -177,8 +180,10 @@ async fn handle_edus( Ok(()) } -async fn handle_edu_presence(_client: &IpAddr, origin: &ServerName, presence: PresenceContent) -> Result<()> { - if !services().globals.allow_incoming_presence() { +async fn handle_edu_presence( + services: &Services, _client: &IpAddr, origin: &ServerName, presence: PresenceContent, +) -> Result<()> { + if !services.globals.allow_incoming_presence() { return Ok(()); } @@ -191,7 +196,7 @@ async fn handle_edu_presence(_client: &IpAddr, origin: &ServerName, presence: Pr continue; } - services().presence.set_presence( + services.presence.set_presence( &update.user_id, &update.presence, Some(update.currently_active), @@ -203,13 +208,15 @@ async fn handle_edu_presence(_client: &IpAddr, origin: &ServerName, presence: Pr Ok(()) } -async fn handle_edu_receipt(_client: &IpAddr, origin: &ServerName, receipt: ReceiptContent) -> Result<()> { - if !services().globals.allow_incoming_read_receipts() { +async fn handle_edu_receipt( + services: &Services, _client: &IpAddr, origin: &ServerName, receipt: ReceiptContent, +) -> Result<()> { + if !services.globals.allow_incoming_read_receipts() { return Ok(()); } for (room_id, room_updates) in receipt.receipts { - if services() + if services .rooms .event_handler .acl_check(origin, &room_id) @@ -231,7 +238,7 @@ async fn handle_edu_receipt(_client: &IpAddr, origin: &ServerName, receipt: Rece continue; } - if services() + if services .rooms .state_cache .room_members(&room_id) @@ -247,7 +254,7 @@ async fn handle_edu_receipt(_client: &IpAddr, origin: &ServerName, receipt: Rece room_id: room_id.clone(), }; - services() + services .rooms .read_receipt .readreceipt_update(&user_id, &room_id, &event)?; @@ -265,8 +272,10 @@ async fn handle_edu_receipt(_client: &IpAddr, origin: &ServerName, receipt: Rece Ok(()) } -async fn handle_edu_typing(_client: &IpAddr, origin: &ServerName, typing: TypingContent) -> Result<()> { - if !services().globals.config.allow_incoming_typing { +async fn handle_edu_typing( + services: &Services, _client: &IpAddr, origin: &ServerName, typing: TypingContent, +) -> Result<()> { + if !services.globals.config.allow_incoming_typing { return Ok(()); } @@ -278,7 +287,7 @@ async fn handle_edu_typing(_client: &IpAddr, origin: &ServerName, typing: Typing return Ok(()); } - if services() + if services .rooms .event_handler .acl_check(typing.user_id.server_name(), &typing.room_id) @@ -291,26 +300,26 @@ async fn handle_edu_typing(_client: &IpAddr, origin: &ServerName, typing: Typing return Ok(()); } - if services() + if services .rooms .state_cache .is_joined(&typing.user_id, &typing.room_id)? { if typing.typing { let timeout = utils::millis_since_unix_epoch().saturating_add( - services() + services .globals .config .typing_federation_timeout_s .saturating_mul(1000), ); - services() + services .rooms .typing .typing_add(&typing.user_id, &typing.room_id, timeout) .await?; } else { - services() + services .rooms .typing .typing_remove(&typing.user_id, &typing.room_id) @@ -328,7 +337,7 @@ async fn handle_edu_typing(_client: &IpAddr, origin: &ServerName, typing: Typing } async fn handle_edu_device_list_update( - _client: &IpAddr, origin: &ServerName, content: DeviceListUpdateContent, + services: &Services, _client: &IpAddr, origin: &ServerName, content: DeviceListUpdateContent, ) -> Result<()> { let DeviceListUpdateContent { user_id, @@ -343,13 +352,13 @@ async fn handle_edu_device_list_update( return Ok(()); } - services().users.mark_device_key_update(&user_id)?; + services.users.mark_device_key_update(&user_id)?; Ok(()) } async fn handle_edu_direct_to_device( - _client: &IpAddr, origin: &ServerName, content: DirectDeviceContent, + services: &Services, _client: &IpAddr, origin: &ServerName, content: DirectDeviceContent, ) -> Result<()> { let DirectDeviceContent { sender, @@ -367,7 +376,7 @@ async fn handle_edu_direct_to_device( } // Check if this is a new transaction id - if services() + if services .transaction_ids .existing_txnid(&sender, None, &message_id)? .is_some() @@ -379,7 +388,7 @@ async fn handle_edu_direct_to_device( for (target_device_id_maybe, event) in map { match target_device_id_maybe { DeviceIdOrAllDevices::DeviceId(target_device_id) => { - services().users.add_to_device_event( + services.users.add_to_device_event( &sender, target_user_id, target_device_id, @@ -391,8 +400,8 @@ async fn handle_edu_direct_to_device( }, DeviceIdOrAllDevices::AllDevices => { - for target_device_id in services().users.all_device_ids(target_user_id) { - services().users.add_to_device_event( + for target_device_id in services.users.all_device_ids(target_user_id) { + services.users.add_to_device_event( &sender, target_user_id, &target_device_id?, @@ -408,7 +417,7 @@ async fn handle_edu_direct_to_device( } // Save transaction id with empty data - services() + services .transaction_ids .add_txnid(&sender, None, &message_id, &[])?; @@ -416,7 +425,7 @@ async fn handle_edu_direct_to_device( } async fn handle_edu_signing_key_update( - _client: &IpAddr, origin: &ServerName, content: SigningKeyUpdateContent, + services: &Services, _client: &IpAddr, origin: &ServerName, content: SigningKeyUpdateContent, ) -> Result<()> { let SigningKeyUpdateContent { user_id, @@ -433,7 +442,7 @@ async fn handle_edu_signing_key_update( } if let Some(master_key) = master_key { - services() + services .users .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true)?; } From 923a98eb66aa74aaa1094d1efcf110ff6ee4246c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 23:32:50 +0000 Subject: [PATCH 0520/2091] partially revert dc18f89c0b for now Signed-off-by: Jason Volk --- src/service/sending/resolve.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 8b6bfc95..e7a71af1 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -443,7 +443,9 @@ impl crate::globals::resolver::Resolver { impl CachedDest { #[inline] #[must_use] - pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + pub fn valid(&self) -> bool { true } + + //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } #[must_use] pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } @@ -452,7 +454,9 @@ impl CachedDest { impl CachedOverride { #[inline] #[must_use] - pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + pub fn valid(&self) -> bool { true } + + //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } #[must_use] pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } From 739eab46d534e897779ced2a189aa0384375302c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 15 Jul 2024 08:35:02 +0000 Subject: [PATCH 0521/2091] refactor thumbnail math Signed-off-by: Jason Volk --- src/service/media/thumbnail.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 50be4ee1..01bf73f6 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -1,4 +1,4 @@ -use std::{io::Cursor, num::Saturating as Sat}; +use std::{cmp, io::Cursor, num::Saturating as Sat}; use conduit::{checked, Result}; use image::{imageops::FilterType, DynamicImage}; @@ -127,26 +127,29 @@ fn thumbnail_generate(image: &DynamicImage, width: u32, height: u32, crop: bool) } fn thumbnail_dimension(image: &DynamicImage, width: u32, height: u32) -> Result<(u32, u32)> { - let original_width = image.width(); - let original_height = image.height(); + let image_width = image.width(); + let image_height = image.height(); - let ratio = Sat(original_width) * Sat(height); - let nratio = Sat(width) * Sat(original_height); - let use_width = nratio <= ratio; + let width = cmp::min(width, image_width); + let height = cmp::min(height, image_height); - let intermediate = if use_width { - Sat(original_height) * Sat(checked!(width / original_width)?) + let use_width = Sat(width) * Sat(image_height) < Sat(height) * Sat(image_width); + + let x = if use_width { + let dividend = (Sat(height) * Sat(image_width)).0; + checked!(dividend / image_height)? } else { - Sat(original_width) * Sat(checked!(height / original_height)?) + width }; - let dims = if use_width { - (width, intermediate.0) + let y = if !use_width { + let dividend = (Sat(width) * Sat(image_height)).0; + checked!(dividend / image_width)? } else { - (intermediate.0, height) + height }; - Ok(dims) + Ok((x, y)) } /// Returns width, height of the thumbnail and whether it should be cropped. From c29197b3f457cf72197ef5251f9815107b2526d7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 16 Jul 2024 00:44:47 -0400 Subject: [PATCH 0522/2091] ci: dont publish `-rc` tags as `latest` docker branches Signed-off-by: strawberry --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9def2cf9..ef1c2b2e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,7 @@ jobs: uses: actions/checkout@v4 - name: Tag comparison check - if: startsWith(github.ref, 'refs/tags/v') + if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} run: | # Tag mismatch with latest repo tag check to prevent potential downgrades LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) @@ -296,15 +296,15 @@ jobs: DOCKER_ARM64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 DOCKER_AMD64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 DOCKER_TAG: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - DOCKER_BRANCH: docker.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + DOCKER_BRANCH: docker.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} GHCR_ARM64: ghcr.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 GHCR_AMD64: ghcr.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 GHCR_TAG: ghcr.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - GHCR_BRANCH: ghcr.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + GHCR_BRANCH: ghcr.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} GLCR_ARM64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 GLCR_AMD64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 GLCR_TAG: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - GLCR_BRANCH: registry.gitlab.com/conduwuit/conduwuit:${{ (startsWith(github.ref, 'refs/tags/v') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + GLCR_BRANCH: registry.gitlab.com/conduwuit/conduwuit:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} From 06531993f69b8cefd5d67d3589791ca9cd6785c2 Mon Sep 17 00:00:00 2001 From: Matthew Scheffel Date: Wed, 17 Jul 2024 22:47:59 -0300 Subject: [PATCH 0523/2091] Calculate Age Rather than Hardcoding to 1 (#531) --- src/core/pdu/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 0c54812e..32ec19c6 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -116,7 +116,12 @@ impl PduEvent { .map_or_else(|| Ok(BTreeMap::new()), |u| serde_json::from_str(u.get())) .map_err(|_| Error::bad_database("Invalid unsigned in pdu event"))?; - unsigned.insert("age".to_owned(), to_raw_value(&1).unwrap()); + // deliberately allowing for the possibility of negative age + let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into(); + let then: i128 = self.origin_server_ts.into(); + let this_age: i128 = now.saturating_sub(then); + + unsigned.insert("age".to_owned(), to_raw_value(&this_age).unwrap()); self.unsigned = Some(to_raw_value(&unsigned).expect("unsigned is valid")); Ok(()) From 478804045805230ac0d34e1ea9ad6da4a3580109 Mon Sep 17 00:00:00 2001 From: Myned Date: Sun, 21 Jul 2024 18:41:44 -0500 Subject: [PATCH 0524/2091] fix media_startup_check typo Signed-off-by: Myned --- conduwuit-example.toml | 2 +- nix/pkgs/complement/config.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 57093128..7a7a0e7a 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -401,7 +401,7 @@ allow_profile_lookup_federation_requests = true # setting this to false may reduce startup time. # # Enabled by default. -#media_statup_check = true +#media_startup_check = true # OpenID token expiration/TTL in seconds # diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index 15f939bb..08c538bb 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -14,7 +14,7 @@ yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true ip_range_denylist = [] url_preview_domain_contains_allowlist = ["*"] media_compat_file_link = false -media_statup_check = false +media_startup_check = false rocksdb_direct_io = false [global.tls] From db494e0d684ca891302ecdd6ba8839d0c2760e84 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 17 Jul 2024 22:09:36 -0400 Subject: [PATCH 0525/2091] bump conduwuit to 0.4.6 Signed-off-by: strawberry --- Cargo.lock | 14 +++++++------- Cargo.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 61cf48bd..ed1357df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -579,7 +579,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" -version = "0.4.5" +version = "0.4.6" dependencies = [ "clap", "conduit_admin", @@ -608,7 +608,7 @@ dependencies = [ [[package]] name = "conduit_admin" -version = "0.4.5" +version = "0.4.6" dependencies = [ "clap", "conduit_api", @@ -627,7 +627,7 @@ dependencies = [ [[package]] name = "conduit_api" -version = "0.4.5" +version = "0.4.6" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -661,7 +661,7 @@ dependencies = [ [[package]] name = "conduit_core" -version = "0.4.5" +version = "0.4.6" dependencies = [ "argon2", "axum 0.7.5", @@ -703,7 +703,7 @@ dependencies = [ [[package]] name = "conduit_database" -version = "0.4.5" +version = "0.4.6" dependencies = [ "conduit_core", "const-str", @@ -715,7 +715,7 @@ dependencies = [ [[package]] name = "conduit_router" -version = "0.4.5" +version = "0.4.6" dependencies = [ "axum 0.7.5", "axum-client-ip", @@ -746,7 +746,7 @@ dependencies = [ [[package]] name = "conduit_service" -version = "0.4.5" +version = "0.4.6" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 7c17d0f7..60d82264 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" rust-version = "1.77.0" -version = "0.4.5" +version = "0.4.6" [workspace.metadata.crane] name = "conduit" From 463fa4fd536c2c53663edb3338424a8c95e6b34b Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 17 Jul 2024 22:38:34 -0400 Subject: [PATCH 0526/2091] nix: dont build unnecessary jemalloc docs or cxx integration Signed-off-by: strawberry --- nix/pkgs/main/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 629957c8..5f21176e 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -51,6 +51,10 @@ rust-jemalloc-sys' = (rust-jemalloc-sys.override { unprefixed = true; }).overrideAttrs (old: { configureFlags = old.configureFlags ++ + # we dont need docs + [ "--disable-doc" ] ++ + # we dont need cxx/C++ integration + [ "--disable-cxx" ] ++ # tikv-jemalloc-sys/profiling feature lib.optional (featureEnabled "jemalloc_prof") "--enable-prof"; }); From 328759a60f1475af575b60b098b81cb1c4cbea1c Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 17 Jul 2024 22:57:51 -0400 Subject: [PATCH 0527/2091] ci: run quick smoke test of x86_64 static binary Signed-off-by: strawberry --- .github/workflows/ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef1c2b2e..cfa6adee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -259,6 +259,14 @@ jobs: mv -v target/release/conduwuit static-${{ matrix.target }} mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb + # quick smoke test of the x86_64 static binary + - name: Run x86_64 static binary + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduit | grep x86-64; then + result/bin/conduit --version + fi + - name: Upload static-${{ matrix.target }} uses: actions/upload-artifact@v4 with: From 4851ad26e295b59dfca2f2c44a1a0d69e26717ec Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 22 Jul 2024 23:07:41 -0400 Subject: [PATCH 0528/2091] bump cargo.lock, lock jemallocator to 0.5.4 Signed-off-by: strawberry --- Cargo.lock | 138 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 15 +++--- 2 files changed, 79 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed1357df..061e8d70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,9 +85,9 @@ checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" [[package]] name = "async-compression" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" +checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" dependencies = [ "brotli", "flate2", @@ -118,7 +118,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -129,7 +129,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -377,7 +377,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -474,13 +474,12 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.3" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e2d530f35b40a84124146478cd16f34225306a8441998836466a2e2961c950" +checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" dependencies = [ "jobserver", "libc", - "once_cell", ] [[package]] @@ -562,7 +561,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1022,7 +1021,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1126,7 +1125,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1262,7 +1261,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1543,7 +1542,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1747,12 +1746,12 @@ dependencies = [ [[package]] name = "image" -version = "0.25.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd54d660e773627692c524beaad361aca785a4f9f5730ce91f42aabe5bce3d11" +checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10" dependencies = [ "bytemuck", - "byteorder", + "byteorder-lite", "color_quant", "gif", "image-webp", @@ -1764,12 +1763,12 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d730b085583c4d789dfd07fdcf185be59501666a90c97c40162b37e4fdad272d" +checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904" dependencies = [ "byteorder-lite", - "thiserror", + "quick-error 2.0.1", ] [[package]] @@ -1952,7 +1951,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1975,9 +1974,9 @@ checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e310b3a6b5907f99202fcdb4960ff45b93735d7c7d96b760fcff8db2dc0e103d" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", "windows-targets 0.52.6", @@ -2033,9 +2032,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9764018d143cc854c9f17f0b907de70f14393b1f502da6375dce70f00514eb3" +checksum = "109de74d5d2353660401699a4174a4ff23fcc649caf553df71933c7fb45ad868" dependencies = [ "cc", "libc", @@ -2461,7 +2460,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2554,7 +2553,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2642,7 +2641,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", "version_check", "yansi", ] @@ -2667,7 +2666,7 @@ dependencies = [ "itertools 0.12.1", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -2703,6 +2702,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quinn" version = "0.11.2" @@ -2739,14 +2744,13 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9096629c45860fc7fb143e125eb826b5e721e10be3263160c7d60ca832cf8c46" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" dependencies = [ "libc", "once_cell", "socket2", - "tracing", "windows-sys 0.52.0", ] @@ -2791,9 +2795,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ "bitflags 2.6.0", ] @@ -2898,7 +2902,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" dependencies = [ "hostname 0.3.1", - "quick-error", + "quick-error 1.2.3", ] [[package]] @@ -3067,7 +3071,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.71", + "syn 2.0.72", "toml", ] @@ -3203,7 +3207,7 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.5", + "rustls-webpki 0.102.6", "subtle", "zeroize", ] @@ -3218,7 +3222,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.5", + "rustls-webpki 0.102.6", "subtle", "zeroize", ] @@ -3264,9 +3268,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.5" +version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a6fccd794a42c2c105b513a2f62bc3fd8f3ba57a4593677ceb0bd035164d78" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ "ring", "rustls-pki-types", @@ -3343,9 +3347,9 @@ checksum = "4646d6f919800cd25c50edb49438a1381e2cd4833c027e75e8897981c50b8b5e" [[package]] name = "security-framework" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -3356,9 +3360,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" dependencies = [ "core-foundation-sys", "libc", @@ -3522,7 +3526,7 @@ checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -3815,9 +3819,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.71" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b146dcf730474b4bcd16c311627b31ede9ab149045db4d6088b3becaea046462" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -3875,22 +3879,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.62" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2675633b1499176c2dff06b0856a27976a8f9d436737b4cf4f312d4d91d8bbb" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.62" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d20468752b09f49e909e55a5d338caa8bedf615594e9d80bc4c565d30faf798c" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -4004,9 +4008,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.0" +version = "1.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" +checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" dependencies = [ "backtrace", "bytes", @@ -4039,7 +4043,7 @@ checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -4113,14 +4117,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.15", + "toml_edit 0.22.16", ] [[package]] @@ -4145,15 +4149,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.15" +version = "0.22.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" dependencies = [ "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.13", + "winnow 0.6.15", ] [[package]] @@ -4256,7 +4260,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b47 dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -4526,7 +4530,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", "wasm-bindgen-shared", ] @@ -4560,7 +4564,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4831,9 +4835,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.13" +version = "0.6.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +checksum = "557404e450152cd6795bb558bca69e43c585055f4606e3bcae5894fc6dac9ba0" dependencies = [ "memchr", ] @@ -4917,9 +4921,9 @@ checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" [[package]] name = "zune-jpeg" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec866b44a2a1fd6133d363f073ca1b179f438f99e7e5bfb1e33f7181facfe448" +checksum = "16099418600b4d8f028622f73ff6e3deaabdff330fb9a2a131dea781ee8b0768" dependencies = [ "zune-core", ] diff --git a/Cargo.toml b/Cargo.toml index 60d82264..12f0e93d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,7 @@ categories = ["network-programming"] description = "a very cool fork of Conduit, a Matrix homeserver written in Rust" edition = "2021" homepage = "https://conduwuit.puppyirl.gay/" -keywords = ["chat", "matrix", "server"] +keywords = ["chat", "matrix", "server", "uwu"] license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" @@ -204,7 +204,7 @@ features = [ version = "0.3.1" [workspace.dependencies.libloading] -version = "0.8.3" +version = "0.8.5" # Validating urls in config, was already a transitive dependency [workspace.dependencies.url] @@ -218,7 +218,7 @@ features = ["alloc", "std"] default-features = false [workspace.dependencies.hyper] -version = "1.4.0" +version = "1.4.1" features = [ "server", "http1", @@ -251,7 +251,7 @@ default-features = false # Used for conduit::Error type [workspace.dependencies.thiserror] -version = "1.0.62" +version = "1.0.63" # Used when hashing the state [workspace.dependencies.ring] @@ -372,16 +372,17 @@ version = "0.34.0" version = "0.34.0" # jemalloc usage +# locked to 0.5.4 due to static binary linking breakage [workspace.dependencies.tikv-jemalloc-sys] -version = "0.5.4" +version = "=0.5.4" default-features = false features = ["stats", "unprefixed_malloc_on_supported_platforms"] [workspace.dependencies.tikv-jemallocator] -version = "0.5.4" +version = "=0.5.4" default-features = false features = ["stats", "unprefixed_malloc_on_supported_platforms"] [workspace.dependencies.tikv-jemalloc-ctl] -version = "0.5.4" +version = "=0.5.4" default-features = false features = ["use_std"] From ce0ca96df4643f1336fdd470130ff0f5873357fc Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 22 Jul 2024 23:19:09 -0400 Subject: [PATCH 0529/2091] clarify that sentry_traces_sample_rate is a percentage Signed-off-by: strawberry --- conduwuit-example.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 7a7a0e7a..27e4cb0b 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -52,9 +52,10 @@ # Performance monitoring/tracing sample rate for Sentry.io # -# Note that too high values may impact performance, and can be disabled by setting it to 0.0 +# Note that too high values may impact performance, and can be disabled by setting it to 0.0 (0%) +# This value is read as a percentage to Sentry, represented as a decimal # -# Defaults to 0.15 +# Defaults to 15% of traces (0.15) #sentry_traces_sample_rate = 0.15 # Whether to attach a stacktrace to Sentry reports. From 30ad5da5f131390532fd1d2c9bbcbe63cae27cfa Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 23 Jul 2024 22:11:51 -0400 Subject: [PATCH 0530/2091] use ruma-identifiers-validation feature from our ruma Signed-off-by: strawberry --- Cargo.lock | 28 ++++++++++++++-------------- Cargo.toml | 7 ++----- src/service/Cargo.toml | 1 - src/service/sending/resolve.rs | 2 +- 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 061e8d70..d8f54efe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -768,7 +768,6 @@ dependencies = [ "regex", "reqwest", "ruma", - "ruma-identifiers-validation", "rustyline-async", "serde", "serde_json", @@ -2923,7 +2922,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "assign", "js_int", @@ -2933,6 +2932,7 @@ dependencies = [ "ruma-common", "ruma-events", "ruma-federation-api", + "ruma-identifiers-validation", "ruma-identity-service-api", "ruma-push-gateway-api", "ruma-server-util", @@ -2944,7 +2944,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "js_int", "ruma-common", @@ -2956,7 +2956,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "as_variant", "assign", @@ -2979,7 +2979,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "as_variant", "base64 0.22.1", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "as_variant", "indexmap 2.2.6", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "js_int", "ruma-common", @@ -3044,7 +3044,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "js_int", "thiserror", @@ -3053,7 +3053,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "js_int", "ruma-common", @@ -3063,7 +3063,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "once_cell", "proc-macro-crate", @@ -3078,7 +3078,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "js_int", "ruma-common", @@ -3090,7 +3090,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "headers", "http 1.1.0", @@ -3103,7 +3103,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3119,7 +3119,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c51ccb2c68d2e3557eb12b1a49036531711ec0e5#c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 12f0e93d..3f9f8e23 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -280,7 +280,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "c51ccb2c68d2e3557eb12b1a49036531711ec0e5" +rev = "c76e2873c1593a3308d4ba3e0e4a1db65acf8536" features = [ "compat", "rand", @@ -293,6 +293,7 @@ features = [ "server-util", "unstable-exhaustive-types", "ring-compat", + "identifiers-validation", "unstable-unspecified", "unstable-msc2448", "unstable-msc2666", @@ -307,10 +308,6 @@ features = [ "unstable-extensible-events", ] -[workspace.dependencies.ruma-identifiers-validation] -git = "https://github.com/girlbossceo/ruwuma" -rev = "c51ccb2c68d2e3557eb12b1a49036531711ec0e5" - [workspace.dependencies.rust-rocksdb] path = "deps/rust-rocksdb" package = "rust-rocksdb-uwu" diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index d2c9785f..c444e5f5 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -57,7 +57,6 @@ lru-cache.workspace = true rand.workspace = true regex.workspace = true reqwest.workspace = true -ruma-identifiers-validation.workspace = true ruma.workspace = true rustyline-async.workspace = true rustyline-async.optional = true diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index e7a71af1..48dddf35 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -252,7 +252,7 @@ async fn request_well_known(dest: &str) -> Result> { .as_str() .unwrap_or_default(); - if ruma_identifiers_validation::server_name::validate(m_server).is_err() { + if ruma::identifiers_validation::server_name::validate(m_server).is_err() { debug_error!("response content missing or invalid"); return Ok(None); } From 96f9d641110045f255c3bc4dc94b471c53a2d1f2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 23 Jul 2024 22:28:15 -0400 Subject: [PATCH 0531/2091] add back is_historical check on admin user creations from accidental removal Signed-off-by: strawberry --- src/admin/user/commands.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 9e4b348b..709e1f57 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -42,6 +42,12 @@ pub(super) async fn create( return Ok(RoomMessageEventContent::text_plain(format!("Userid {user_id} already exists"))); } + if user_id.is_historical() { + return Ok(RoomMessageEventContent::text_plain(format!( + "User ID {user_id} does not conform to new Matrix identifier spec" + ))); + } + let password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); // Create user From 100fc5e0f7fac80ab2bb8a8bccc0a5c190773c51 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 23 Jul 2024 22:29:50 -0400 Subject: [PATCH 0532/2091] bump tokio to 1.39.1 Signed-off-by: strawberry --- Cargo.lock | 29 ++++++++++++++++++++--------- Cargo.toml | 2 +- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8f54efe..e9730053 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -970,7 +970,7 @@ dependencies = [ "crossterm_winapi", "futures-core", "libc", - "mio", + "mio 0.8.11", "parking_lot", "signal-hook", "signal-hook-mio", @@ -2147,6 +2147,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mio" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +dependencies = [ + "hermit-abi", + "libc", + "wasi", + "windows-sys 0.52.0", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -3672,7 +3684,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" dependencies = [ "libc", - "mio", + "mio 0.8.11", "signal-hook", ] @@ -4008,21 +4020,20 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.38.1" +version = "1.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2caba9f80616f438e09748d5acda951967e1ea58508ef53d9c6402485a46df" +checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a" dependencies = [ "backtrace", "bytes", "libc", - "mio", - "num_cpus", + "mio 1.0.1", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", "tracing", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4037,9 +4048,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 3f9f8e23..559965ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -188,7 +188,7 @@ version = "0.3.30" default-features = false [workspace.dependencies.tokio] -version = "1.38.0" +version = "1.39.1" features = [ "fs", "net", From b13ea4ff458d19dd70e911c760c76018283f2428 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 23 Jul 2024 22:36:06 -0400 Subject: [PATCH 0533/2091] fix non-working admin room topic accessor output Signed-off-by: strawberry --- src/admin/room/room_info_commands.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/room/room_info_commands.rs b/src/admin/room/room_info_commands.rs index d985fb87..9182616f 100644 --- a/src/admin/room/room_info_commands.rs +++ b/src/admin/room/room_info_commands.rs @@ -64,6 +64,6 @@ async fn view_room_topic(_body: Vec<&str>, room_id: Box) -> Result Date: Tue, 23 Jul 2024 22:46:20 -0400 Subject: [PATCH 0534/2091] add harmless check for presence PUT matching sender user this is already done but we just don't error and always use the sender user. match synapse behaviour where we check and error. Signed-off-by: strawberry --- src/api/client/presence.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/api/client/presence.rs b/src/api/client/presence.rs index 066de1cc..775f82ec 100644 --- a/src/api/client/presence.rs +++ b/src/api/client/presence.rs @@ -16,6 +16,14 @@ pub(crate) async fn set_presence_route(body: Ruma) -> } let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if sender_user != &body.user_id && body.appservice_info.is_none() { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to set presence of other users", + )); + } + services() .presence .set_presence(sender_user, &body.presence, None, None, body.status_msg.clone())?; From 39c0f4ea3d1156e7308f74d3fb51b94187630382 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 24 Jul 2024 18:59:36 -0400 Subject: [PATCH 0535/2091] delete unused bad_signature_ratelimiter rm'd in 67f9553790b7e06b1cfb66b036dd3c8f7ee8c31a Signed-off-by: strawberry --- src/service/globals/mod.rs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 0a0d0d8e..6f26161e 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -39,7 +39,6 @@ pub struct Service { pub stable_room_versions: Vec, pub unstable_room_versions: Vec, pub bad_event_ratelimiter: Arc>>, - pub bad_signature_ratelimiter: Arc, RateLimitState>>>, pub bad_query_ratelimiter: Arc>>, pub stateres_mutex: Arc>, pub server_user: OwnedUserId, @@ -101,7 +100,6 @@ impl crate::Service for Service { stable_room_versions, unstable_room_versions, bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), - bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), stateres_mutex: Arc::new(Mutex::new(())), admin_alias: RoomAliasId::parse(format!("#admins:{}", &config.server_name)) @@ -144,13 +142,6 @@ impl crate::Service for Service { .len(); writeln!(out, "bad_query_ratelimiter: {bad_query_ratelimiter}")?; - let bad_signature_ratelimiter = self - .bad_signature_ratelimiter - .read() - .expect("locked for reading") - .len(); - writeln!(out, "bad_signature_ratelimiter: {bad_signature_ratelimiter}")?; - Ok(()) } @@ -166,11 +157,6 @@ impl crate::Service for Service { .write() .expect("locked for writing") .clear(); - - self.bad_signature_ratelimiter - .write() - .expect("locked for writing") - .clear(); } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } From 39f19c483a12f44a7d3fd90b5cef8af8123df53a Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 24 Jul 2024 19:07:25 -0400 Subject: [PATCH 0536/2091] nix: use all-features for complement OCI images Signed-off-by: strawberry --- nix/pkgs/complement/default.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index 22c91367..0100dbb6 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -15,7 +15,14 @@ let main' = main.override { profile = "dev"; - features = ["axum_dual_protocol"]; + all_features = true; + disable_release_max_log_level = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; start = writeShellScriptBin "start" '' From 09680f653f873687728e44461d85b7c3ad747f3e Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 24 Jul 2024 19:17:37 -0400 Subject: [PATCH 0537/2091] nix: add outputs for debug builds Signed-off-by: strawberry --- flake.nix | 89 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/flake.nix b/flake.nix index a56d1bc0..bd948c86 100644 --- a/flake.nix +++ b/flake.nix @@ -144,6 +144,11 @@ { packages = { default = scopeHost.main; + default-debug = scopeHost.main.override { + profile = "dev"; + # debug build users expect full logs + disable_release_max_log_level = true; + }; all-features = scopeHost.main.override { all_features = true; disable_features = [ @@ -153,6 +158,18 @@ "experimental" ]; }; + all-features-debug = scopeHost.main.override { + profile = "dev"; + all_features = true; + # debug build users expect full logs + disable_release_max_log_level = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; + }; hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; }; oci-image = scopeHost.oci-image; @@ -167,6 +184,20 @@ ]; }; }; + oci-image-all-features-debug = scopeHost.oci-image.override { + main = scopeHost.main.override { + profile = "dev"; + all_features = true; + # debug build users expect full logs + disable_release_max_log_level = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; + }; + }; oci-image-hmalloc = scopeHost.oci-image.override { main = scopeHost.main.override { features = ["hardened_malloc"]; @@ -201,6 +232,16 @@ value = scopeCrossStatic.main; } + # An output for a statically-linked unstripped debug ("dev") binary + { + name = "${binaryName}-debug"; + value = scopeCrossStatic.main.override { + profile = "dev"; + # debug build users expect full logs + disable_release_max_log_level = true; + }; + } + # An output for a statically-linked binary with `--all-features` { name = "${binaryName}-all-features"; @@ -215,6 +256,23 @@ }; } + # An output for a statically-linked unstripped debug ("dev") binary with `--all-features` + { + name = "${binaryName}-all-features-debug"; + value = scopeCrossStatic.main.override { + profile = "dev"; + all_features = true; + # debug build users expect full logs + disable_release_max_log_level = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; + }; + } + # An output for a statically-linked binary with hardened_malloc { name = "${binaryName}-hmalloc"; @@ -229,6 +287,18 @@ value = scopeCrossStatic.oci-image; } + # An output for an OCI image based on that unstripped debug ("dev") binary + { + name = "oci-image-${crossSystem}-debug"; + value = scopeCrossStatic.oci-image.override { + main = scopeCrossStatic.main.override { + profile = "dev"; + # debug build users expect full logs + disable_release_max_log_level = true; + }; + }; + } + # An output for an OCI image based on that binary with `--all-features` { name = "oci-image-${crossSystem}-all-features"; @@ -245,6 +315,25 @@ }; } + # An output for an OCI image based on that unstripped debug ("dev") binary with `--all-features` + { + name = "oci-image-${crossSystem}-all-features-debug"; + value = scopeCrossStatic.oci-image.override { + main = scopeCrossStatic.main.override { + profile = "dev"; + all_features = true; + # debug build users expect full logs + disable_release_max_log_level = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; + }; + }; + } + # An output for an OCI image based on that binary with hardened_malloc { name = "oci-image-${crossSystem}-hmalloc"; From 1378399f9d9d7590067d7350c54e67264eba54a5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 24 Jul 2024 19:32:26 -0400 Subject: [PATCH 0538/2091] ci: build and upload static debug/dev binaries and OCI images Signed-off-by: strawberry --- .github/workflows/ci.yml | 109 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 104 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfa6adee..0343d49b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -259,8 +259,34 @@ jobs: mv -v target/release/conduwuit static-${{ matrix.target }} mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb - # quick smoke test of the x86_64 static binary - - name: Run x86_64 static binary + # quick smoke test of the x86_64 static release binary + - name: Run x86_64 static release binary + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduit | grep x86-64; then + result/bin/conduit --version + fi + + - name: Build static debug ${{ matrix.target }} + run: | + CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug + + # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles. + # so we need to coerce cargo-deb into thinking this is a release binary + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduit target/release/conduwuit + cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + # -p conduit is the main crate name + direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}-debug.deb + mv -v target/release/conduwuit static-${{ matrix.target }}-debug + mv -v target/release/${{ matrix.target }}-debug.deb ${{ matrix.target }}-debug.deb + + # quick smoke test of the x86_64 static debug binary + - name: Run x86_64 static debug binary run: | # GH actions default runners are x86_64 only if file result/bin/conduit | grep x86-64; then @@ -282,11 +308,31 @@ jobs: if-no-files-found: error compression-level: 0 + - name: Upload static-${{ matrix.target }}-debug + uses: actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }}-debug + path: static-${{ matrix.target }}-debug + if-no-files-found: error + + - name: Upload deb ${{ matrix.target }}-debug + uses: actions/upload-artifact@v4 + with: + name: deb-${{ matrix.target }}-debug + path: ${{ matrix.target }}-debug.deb + if-no-files-found: error + compression-level: 0 + - name: Build OCI image ${{ matrix.target }} run: | bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features cp -v -f result oci-image-${{ matrix.target }}.tar.gz + - name: Build debug OCI image ${{ matrix.target }} + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug + cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz + - name: Upload OCI image ${{ matrix.target }} uses: actions/upload-artifact@v4 with: @@ -295,6 +341,14 @@ jobs: if-no-files-found: error compression-level: 0 + - name: Upload OCI image ${{ matrix.target }}-debug + uses: actions/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }}-debug + path: oci-image-${{ matrix.target }}-debug.tar.gz + if-no-files-found: error + compression-level: 0 + docker: name: Docker publish runs-on: ubuntu-latest @@ -345,8 +399,10 @@ jobs: - name: Move OCI images into position run: | - mv -v oci-image-x86_64-*/*.tar.gz oci-image-amd64.tar.gz - mv -v oci-image-aarch64-*/*.tar.gz oci-image-arm64v8.tar.gz + mv -v oci-image-x86_64-unknown-linux-musl/*.tar.gz oci-image-amd64.tar.gz + mv -v oci-image-aarch64-unknown-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz + mv -v oci-image-x86_64-unknown-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz + mv -v oci-image-aarch64-unknown-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz - name: Load and push amd64 image if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} @@ -370,6 +426,28 @@ jobs: docker push ${{ env.GHCR_ARM64 }} docker push ${{ env.GLCR_ARM64 }} + - name: Load and push amd64 debug image + if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + run: | + docker load -i oci-image-amd64-debug.tar.gz + docker tag $(docker images -q conduit:main) ${{ env.DOCKER_AMD64 }}-debug + docker tag $(docker images -q conduit:main) ${{ env.GHCR_AMD64 }}-debug + docker tag $(docker images -q conduit:main) ${{ env.GLCR_AMD64 }}-debug + docker push ${{ env.DOCKER_AMD64 }}-debug + docker push ${{ env.GHCR_AMD64 }}-debug + docker push ${{ env.GLCR_AMD64 }}-debug + + - name: Load and push arm64 debug image + if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + run: | + docker load -i oci-image-arm64v8-debug.tar.gz + docker tag $(docker images -q conduit:main) ${{ env.DOCKER_ARM64 }}-debug + docker tag $(docker images -q conduit:main) ${{ env.GHCR_ARM64 }}-debug + docker tag $(docker images -q conduit:main) ${{ env.GLCR_ARM64 }}-debug + docker push ${{ env.DOCKER_ARM64 }}-debug + docker push ${{ env.GHCR_ARM64 }}-debug + docker push ${{ env.GLCR_ARM64 }}-debug + - name: Create Docker combined manifests run: | # Dockerhub Container Registry @@ -379,9 +457,21 @@ jobs: docker manifest create ${{ env.GHCR_TAG }} --amend ${{ env.GHCR_ARM64 }} --amend ${{ env.GHCR_AMD64 }} docker manifest create ${{ env.GHCR_BRANCH }} --amend ${{ env.GHCR_ARM64 }} --amend ${{ env.GHCR_AMD64 }} # GitLab Container Registry - docker manifest create ${{ env.GLCR_TAG }} --amend ${{ env.GLCR_ARM64 }} --amend ${{ env.GCCR_AMD64 }} + docker manifest create ${{ env.GLCR_TAG }} --amend ${{ env.GLCR_ARM64 }} --amend ${{ env.GLCR_AMD64 }} docker manifest create ${{ env.GLCR_BRANCH }} --amend ${{ env.GLCR_ARM64 }} --amend ${{ env.GLCR_AMD64 }} + - name: Create Docker combined debug manifests + run: | + # Dockerhub Container Registry + docker manifest create ${{ env.DOCKER_TAG }}-debug --amend ${{ env.DOCKER_ARM64 }}-debug --amend ${{ env.DOCKER_AMD64 }}-debug + docker manifest create ${{ env.DOCKER_BRANCH }}-debug --amend ${{ env.DOCKER_ARM64 }}-debug --amend ${{ env.DOCKER_AMD64 }}-debug + # GitHub Container Registry + docker manifest create ${{ env.GHCR_TAG }}-debug --amend ${{ env.GHCR_ARM64 }}-debug --amend ${{ env.GHCR_AMD64 }}-debug + docker manifest create ${{ env.GHCR_BRANCH }}-debug --amend ${{ env.GHCR_ARM64 }}-debug --amend ${{ env.GHCR_AMD64 }}-debug + # GitLab Container Registry + docker manifest create ${{ env.GLCR_TAG }}-debug --amend ${{ env.GLCR_ARM64 }}-debug --amend ${{ env.GLCR_AMD64 }}-debug + docker manifest create ${{ env.GLCR_BRANCH }}-debug --amend ${{ env.GLCR_ARM64 }}-debug --amend ${{ env.GLCR_AMD64 }}-debug + - name: Push manifests to Docker registries if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | @@ -391,6 +481,12 @@ jobs: docker manifest push ${{ env.GHCR_BRANCH }} docker manifest push ${{ env.GLCR_TAG }} docker manifest push ${{ env.GLCR_BRANCH }} + docker manifest push ${{ env.DOCKER_TAG }}-debug + docker manifest push ${{ env.DOCKER_BRANCH }}-debug + docker manifest push ${{ env.GHCR_TAG }}-debug + docker manifest push ${{ env.GHCR_BRANCH }}-debug + docker manifest push ${{ env.GLCR_TAG }}-debug + docker manifest push ${{ env.GLCR_BRANCH }}-debug - name: Add Image Links to Job Summary if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} @@ -398,3 +494,6 @@ jobs: echo "- \`docker pull ${{ env.DOCKER_TAG }}\`" >> $GITHUB_STEP_SUMMARY echo "- \`docker pull ${{ env.GHCR_TAG }}\`" >> $GITHUB_STEP_SUMMARY echo "- \`docker pull ${{ env.GLCR_TAG }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${{ env.DOCKER_TAG }}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${{ env.GHCR_TAG }}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${{ env.GLCR_TAG }}-debug\`" >> $GITHUB_STEP_SUMMARY From 934ab6a4fe55e3b8ab964be7dad364b4ba3c449f Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 24 Jul 2024 19:34:19 -0400 Subject: [PATCH 0539/2091] nix: add dpkg to devshell packages for dpkg-deb Signed-off-by: strawberry --- CONTRIBUTING.md | 1 + engage.toml | 5 +++++ flake.nix | 3 +++ 3 files changed, 9 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 89fdb165..0cc2bfef 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,6 +39,7 @@ The following binaries are used in [`engage.toml`][engage.toml]: - [`cargo-deb`][cargo-deb] - [`lychee`][lychee] - [`markdownlint-cli`][markdownlint-cli] +- `dpkg` ### Matrix tests diff --git a/engage.toml b/engage.toml index 8bba0eaf..7413a8e3 100644 --- a/engage.toml +++ b/engage.toml @@ -60,6 +60,11 @@ name = "markdownlint" group = "versions" script = "markdownlint --version" +[[task]] +name = "dpkg" +group = "versions" +script = "dpkg --version" + [[task]] name = "cargo-audit" group = "security" diff --git a/flake.nix b/flake.nix index bd948c86..225aef09 100644 --- a/flake.nix +++ b/flake.nix @@ -119,6 +119,9 @@ # Needed for producing Debian packages cargo-deb + # Needed for CI to check validity of produced Debian packages (dpkg-deb) + dpkg + # Needed for Complement go From 03296412ab3884fde427ecaca5104e8bebb183df Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 24 Jul 2024 19:40:53 -0400 Subject: [PATCH 0540/2091] ci: validate deb packages produced Signed-off-by: strawberry --- .github/workflows/ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0343d49b..a30fb061 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,6 +293,16 @@ jobs: result/bin/conduit --version fi + # check validity of produced deb package, invalid debs will error on these commands + - name: Validate produced deb package + run: | + # List contents + dpkg-deb --contents ${{ matrix.target }}.deb + dpkg-deb --contents ${{ matrix.target }}-debug.deb + # List info + dpkg-deb --info ${{ matrix.target }}.deb + dpkg-deb --info ${{ matrix.target }}-debug.deb + - name: Upload static-${{ matrix.target }} uses: actions/upload-artifact@v4 with: From 60927c1c727941d0aea73bdc1a33132d92ce7956 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 24 Jul 2024 22:22:45 -0400 Subject: [PATCH 0541/2091] ci: try using sccache Signed-off-by: strawberry --- .github/workflows/ci.yml | 22 ++++++++++++++++++++++ flake.nix | 2 ++ 2 files changed, 24 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a30fb061..d95147ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,16 @@ concurrency: cancel-in-progress: true env: + # sccache + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + SCCACHE_BUCKET: "sccache" + SCCACHE_S3_USE_SSL: ${{ vars.SCCACHE_S3_USE_SSL }} + SCCACHE_REGION: ${{ vars.SCCACHE_REGION }} + SCCACHE_ENDPOINT: ${{ vars.SCCACHE_ENDPOINT }} + SCCACHE_CACHE_MULTIARCH: ${{ vars.SCCACHE_CACHE_MULTIARCH }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # Required to make some things output color TERM: ansi # Publishing to my nix binary cache @@ -126,6 +136,11 @@ jobs: run: | bin/nix-build-and-cache ci + # use sccache for Rust + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@main + + # use rust-cache - uses: Swatinem/rust-cache@v2 - name: Run CI tests @@ -244,6 +259,13 @@ jobs: direnv allow nix develop .#all-features --command true + # use sccache for Rust + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@main + + # use rust-cache + - uses: Swatinem/rust-cache@v2 + - name: Build static ${{ matrix.target }} run: | CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') diff --git a/flake.nix b/flake.nix index 225aef09..e480fd82 100644 --- a/flake.nix +++ b/flake.nix @@ -136,6 +136,8 @@ # Useful for editing the book locally mdbook + + sccache ]) ++ scope.main.buildInputs ++ scope.main.propagatedBuildInputs From f098532b094dfa9d803a2d013144d4883e7758a9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 24 Jul 2024 22:45:27 -0400 Subject: [PATCH 0542/2091] ci: retry nix-build-and-cache 3 times before failing due to attic issues attic just sucks and every 5 CI runs guarantees at least 2 false gateway errors that i can't do anything about. Signed-off-by: strawberry --- .github/workflows/ci.yml | 93 +++++++++++++++++++++++++++-- .github/workflows/documentation.yml | 40 ++++++++++++- 2 files changed, 125 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d95147ae..2e01808e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -134,7 +134,23 @@ jobs: - name: Cache CI dependencies run: | - bin/nix-build-and-cache ci + # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + ATTEMPTS=3 + SUCCESS=false + while (( ATTEMPTS-- > 0 )) + do + bin/nix-build-and-cache ci + if [[ $? == 0 ]]; then + SUCCESS=true + break + else + sleep 3 + fi + done + + if [[ $SUCCESS == "false" ]]; then + exit 1 + fi # use sccache for Rust - name: Run sccache-cache @@ -271,7 +287,24 @@ jobs: CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features + # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + ATTEMPTS=3 + SUCCESS=false + while (( ATTEMPTS-- > 0 )) + do + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features + if [[ $? == 0 ]]; then + SUCCESS=true + break + else + sleep 3 + fi + done + + if [[ $SUCCESS == "false" ]]; then + exit 1 + fi + mkdir -v -p target/release/ mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ cp -v -f result/bin/conduit target/release/conduwuit @@ -294,7 +327,23 @@ jobs: CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug + # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + ATTEMPTS=3 + SUCCESS=false + while (( ATTEMPTS-- > 0 )) + do + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug + if [[ $? == 0 ]]; then + SUCCESS=true + break + else + sleep 3 + fi + done + + if [[ $SUCCESS == "false" ]]; then + exit 1 + fi # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles. # so we need to coerce cargo-deb into thinking this is a release binary @@ -357,12 +406,46 @@ jobs: - name: Build OCI image ${{ matrix.target }} run: | - bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features + # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + ATTEMPTS=3 + SUCCESS=false + while (( ATTEMPTS-- > 0 )) + do + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features + if [[ $? == 0 ]]; then + SUCCESS=true + break + else + sleep 3 + fi + done + + if [[ $SUCCESS == "false" ]]; then + exit 1 + fi + cp -v -f result oci-image-${{ matrix.target }}.tar.gz - name: Build debug OCI image ${{ matrix.target }} run: | - bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug + # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + ATTEMPTS=3 + SUCCESS=false + while (( ATTEMPTS-- > 0 )) + do + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug + if [[ $? == 0 ]]; then + SUCCESS=true + break + else + sleep 3 + fi + done + + if [[ $SUCCESS == "false" ]]; then + exit 1 + fi + cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz - name: Upload OCI image ${{ matrix.target }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index c6f9133a..f9e8000b 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -107,7 +107,23 @@ jobs: - name: Cache CI dependencies run: | - bin/nix-build-and-cache ci + # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + ATTEMPTS=3 + SUCCESS=false + while (( ATTEMPTS-- > 0 )) + do + bin/nix-build-and-cache ci + if [[ $? == 0 ]]; then + SUCCESS=true + break + else + sleep 3 + fi + done + + if [[ $SUCCESS == "false" ]]; then + exit 1 + fi - name: Run lychee and markdownlint run: | @@ -116,8 +132,26 @@ jobs: - name: Build documentation (book) run: | - ./bin/nix-build-and-cache just .#book - cp -r --dereference result public + # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + ATTEMPTS=3 + SUCCESS=false + while (( ATTEMPTS-- > 0 )) + do + bin/nix-build-and-cache just .#book + if [[ $? == 0 ]]; then + SUCCESS=true + break + else + sleep 3 + fi + done + + if [[ $SUCCESS == "false" ]]; then + exit 1 + fi + + cp -r --dereference result public + - name: Upload generated documentation (book) as normal artifact uses: actions/upload-artifact@v4 with: From 04971d043022e7cda67347f1849c0e4068fdd595 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 24 Jul 2024 23:33:38 -0400 Subject: [PATCH 0543/2091] ci: cache all crates in rust-cache Signed-off-by: strawberry --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e01808e..57d7e592 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -158,6 +158,8 @@ jobs: # use rust-cache - uses: Swatinem/rust-cache@v2 + with: + cache-all-crates: "true" - name: Run CI tests run: | @@ -281,6 +283,8 @@ jobs: # use rust-cache - uses: Swatinem/rust-cache@v2 + with: + cache-all-crates: "true" - name: Build static ${{ matrix.target }} run: | From fb8a2846df70885d9a0e148b50308f59a8d2ced4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 25 Jul 2024 18:06:45 -0400 Subject: [PATCH 0544/2091] ci/nix: use a `"test"` cargo profile Signed-off-by: strawberry --- .github/workflows/ci.yml | 2 ++ Cargo.toml | 9 +++++++++ engage.toml | 16 +++++++++++++--- flake.nix | 14 ++++++++++++++ nix/pkgs/complement/default.nix | 2 +- nix/pkgs/main/default.nix | 2 +- 6 files changed, 40 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57d7e592..0448b3e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,8 @@ jobs: tests: name: Test runs-on: ubuntu-latest + env: + CARGO_PROFILE: "test" steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main diff --git a/Cargo.toml b/Cargo.toml index 559965ab..02c83600 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -657,7 +657,16 @@ opt-level = 'z' # '-Clink-arg=-Wl,-z,nodelete', #] +# primarily used for CI [profile.test] +inherits = "dev" +codegen-units = 16 +incremental = false + +[profile.test.package.'*'] +inherits = "dev" +debug = 0 +codegen-units = 16 incremental = false ############################################################################### diff --git a/engage.toml b/engage.toml index 7413a8e3..e2428aea 100644 --- a/engage.toml +++ b/engage.toml @@ -73,7 +73,9 @@ script = "cargo audit -D warnings -D unmaintained -D unsound -D yanked" [[task]] name = "cargo-fmt" group = "lints" -script = "cargo fmt --check -- --color=always" +script = """ +cargo fmt --check -- --color=always +""" [[task]] name = "cargo-doc" @@ -84,6 +86,7 @@ env DIRENV_DEVSHELL=all-features \ direnv exec . \ cargo doc \ --workspace \ + --profile test \ --all-features \ --no-deps \ --document-private-items \ @@ -96,6 +99,7 @@ group = "lints" script = """ cargo clippy \ --workspace \ + --profile test \ --all-targets \ --color=always \ -- \ @@ -110,6 +114,7 @@ env DIRENV_DEVSHELL=all-features \ direnv exec . \ cargo clippy \ --workspace \ + --profile test \ --all-targets \ --all-features \ --color=always \ @@ -123,6 +128,7 @@ group = "lints" script = """ cargo clippy \ --workspace \ + --profile test \ --features jemalloc \ --all-targets \ --color=always \ @@ -161,6 +167,7 @@ env DIRENV_DEVSHELL=all-features \ direnv exec . \ cargo test \ --workspace \ + --profile test \ --all-targets \ --all-features \ --color=always \ @@ -174,6 +181,7 @@ group = "tests" script = """ cargo test \ --workspace \ + --profile test \ --all-targets \ --color=always \ -- \ @@ -190,9 +198,11 @@ name = "nix-default" group = "tests" script = """ env DIRENV_DEVSHELL=dynamic \ + CARGO_PROFILE="test" \ direnv exec . \ - bin/nix-build-and-cache just .#default + bin/nix-build-and-cache just .#default-test env DIRENV_DEVSHELL=dynamic \ + CARGO_PROFILE="test" \ direnv exec . \ - nix run -L .#default -- --help + nix run -L .#default-test -- --help && nix run -L .#default-test -- --version """ diff --git a/flake.nix b/flake.nix index e480fd82..9d9b103c 100644 --- a/flake.nix +++ b/flake.nix @@ -154,6 +154,10 @@ # debug build users expect full logs disable_release_max_log_level = true; }; + default-test = scopeHost.main.override { + profile = "test"; + disable_release_max_log_level = true; + }; all-features = scopeHost.main.override { all_features = true; disable_features = [ @@ -247,6 +251,16 @@ }; } + # An output for a statically-linked unstripped debug binary with the + # "test" profile (for CI usage only) + { + name = "${binaryName}-test"; + value = scopeCrossStatic.main.override { + profile = "test"; + disable_release_max_log_level = true; + }; + } + # An output for a statically-linked binary with `--all-features` { name = "${binaryName}-all-features"; diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index 0100dbb6..5ebe9eb3 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -14,7 +14,7 @@ let main' = main.override { - profile = "dev"; + profile = "test"; all_features = true; disable_release_max_log_level = true; disable_features = [ diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 5f21176e..72d67465 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -153,7 +153,7 @@ commonAttrs = { ]; }; - dontStrip = profile == "dev"; + dontStrip = profile == "dev" || profile == "test"; buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'; From b20c4e0539a254546efaa580d2beafd3566a49aa Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 25 Jul 2024 18:08:15 -0400 Subject: [PATCH 0545/2091] bump MSRV and rust toolchain to 1.80.0 Signed-off-by: strawberry --- Cargo.toml | 2 +- flake.nix | 2 +- rust-toolchain.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 02c83600..175d11b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" -rust-version = "1.77.0" +rust-version = "1.80.0" version = "0.4.6" [workspace.metadata.crane] diff --git a/flake.nix b/flake.nix index 9d9b103c..aadfef03 100644 --- a/flake.nix +++ b/flake.nix @@ -24,7 +24,7 @@ file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` - sha256 = "sha256-+syqAd2kX8KVa8/U2gz3blIQTTsYYt3U63xBWaGOSc8"; + sha256 = "sha256-6eN/GKzjVSjEhGO9FhWObkRFaE1Jf+uqMSdQnb8lcB4="; }; mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 0ed021bd..098a99e6 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -11,7 +11,7 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.77.0" +channel = "1.80.0" components = [ # For rust-analyzer "rust-src", From 74826dcb9437ebaee7bf17612167659c42c39580 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 25 Jul 2024 19:43:48 -0400 Subject: [PATCH 0546/2091] resolve lint_groups_priority lint (https://github.com/rust-lang/cargo/issues/12918) Signed-off-by: strawberry --- Cargo.toml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 175d11b4..f5c2ad24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -728,19 +728,19 @@ variant_size_differences = "allow" [workspace.lints.clippy] ################### -cargo = "warn" +cargo = { level = "warn", priority = -1 } ## some sadness multiple_crate_versions = { level = "allow", priority = 1 } ################### -complexity = "warn" +complexity = { level = "warn", priority = -1 } ################### -correctness = "warn" +correctness = { level = "warn", priority = -1 } ################### -nursery = "warn" +nursery = { level = "warn", priority = -1 } ## some sadness missing_const_for_fn = { level = "allow", priority = 1 } # TODO @@ -750,7 +750,7 @@ significant_drop_in_scrutinee = { level = "allow", priority = 1 } # TODO significant_drop_tightening = { level = "allow", priority = 1 } # TODO ################### -pedantic = "warn" +pedantic = { level = "warn", priority = -1 } ## some sadness doc_markdown = { level = "allow", priority = 1 } @@ -769,7 +769,7 @@ unnecessary_wraps = { level = "allow", priority = 1 } unused_async = { level = "allow", priority = 1 } ################### -perf = "warn" +perf = { level = "warn", priority = -1 } ################### #restriction = "warn" @@ -819,14 +819,14 @@ unseparated_literal_suffix = "warn" verbose_file_reads = "warn" ################### -style = "warn" +style = { level = "warn", priority = -1 } ## some sadness # trivial assertions are quite alright assertions_on_constants = { level = "allow", priority = 1 } ################### -suspicious = "warn" +suspicious = { level = "warn", priority = -1 } ## some sadness let_underscore_future = { level = "allow", priority = 1 } From dc573f4562237b8d8603aedeb262fb9e5762129e Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 25 Jul 2024 19:39:47 -0400 Subject: [PATCH 0547/2091] ci: run free disk space for documentation workflow too Signed-off-by: strawberry --- .github/workflows/documentation.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index f9e8000b..37f2c0f0 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -47,6 +47,9 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + - name: Sync repository uses: actions/checkout@v4 From 04fc87ad0765b996d0f344e7bc4308cfb0bec378 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 25 Jul 2024 21:33:34 -0400 Subject: [PATCH 0548/2091] ci: try to free more runner storage/space Signed-off-by: strawberry --- .github/workflows/ci.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0448b3e6..96719735 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -215,6 +215,10 @@ jobs: echo '```' >> $GITHUB_STEP_SUMMARY fi + - name: Run cargo clean test artifacts + run: | + cargo clean --profile test + build: name: Build runs-on: ubuntu-latest @@ -225,6 +229,9 @@ jobs: - target: aarch64-unknown-linux-musl - target: x86_64-unknown-linux-musl steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + - name: Sync repository uses: actions/checkout@v4 From 463f1a12875d909afd901de9cabb038793e403e7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 16 Jul 2024 06:49:47 +0000 Subject: [PATCH 0549/2091] only use graceful shutdown w/ axum-server fixed Signed-off-by: Jason Volk --- src/router/run.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/router/run.rs b/src/router/run.rs index 91507772..cb5d2abf 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -12,7 +12,7 @@ extern crate conduit_service as service; use std::sync::atomic::Ordering; -use conduit::{debug, debug_info, error, info, trace, Error, Result, Server}; +use conduit::{debug, debug_info, error, info, Error, Result, Server}; use crate::serve; @@ -103,20 +103,20 @@ async fn signal(server: Arc, tx: Sender<()>, handle: axum_server::Handle } async fn handle_shutdown(server: &Arc, tx: &Sender<()>, handle: &axum_server::Handle, sig: &str) { - debug!("Received signal {}", sig); + debug!("Received signal {sig}"); if let Err(e) = tx.send(()) { error!("failed sending shutdown transaction to channel: {e}"); } - let pending = server.metrics.requests_spawn_active.load(Ordering::Relaxed); - if pending > 0 { - let timeout = Duration::from_secs(36); - trace!(pending, ?timeout, "Notifying for graceful shutdown"); - handle.graceful_shutdown(Some(timeout)); - } else { - debug!(pending, "Notifying for immediate shutdown"); - handle.shutdown(); - } + let timeout = Duration::from_secs(36); + debug!( + ?timeout, + spawn_active = ?server.metrics.requests_spawn_active.load(Ordering::Relaxed), + handle_active = ?server.metrics.requests_handle_active.load(Ordering::Relaxed), + "Notifying for graceful shutdown" + ); + + handle.graceful_shutdown(Some(timeout)); } async fn handle_services_poll( From 8b6018d77da5a0caf5800410354b07314a836a3f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 16 Jul 2024 08:05:25 +0000 Subject: [PATCH 0550/2091] de-global services() from api Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 4 +- src/admin/room/room_moderation_commands.rs | 8 +- src/admin/user/commands.rs | 15 +- src/api/client/account.rs | 129 +++---- src/api/client/alias.rs | 38 +- src/api/client/backup.rs | 111 +++--- src/api/client/capabilities.rs | 11 +- src/api/client/config.rs | 29 +- src/api/client/context.rs | 39 +- src/api/client/device.rs | 43 ++- src/api/client/directory.rs | 75 ++-- src/api/client/filter.rs | 15 +- src/api/client/keys.rs | 86 +++-- src/api/client/media.rs | 143 ++++---- src/api/client/membership.rs | 398 +++++++++++---------- src/api/client/message.rs | 57 +-- src/api/client/openid.rs | 9 +- src/api/client/presence.rs | 22 +- src/api/client/profile.rs | 113 +++--- src/api/client/push.rs | 61 ++-- src/api/client/read_marker.rs | 27 +- src/api/client/redact.rs | 11 +- src/api/client/relations.rs | 90 +++-- src/api/client/report.rs | 30 +- src/api/client/room.rs | 139 +++---- src/api/client/search.rs | 31 +- src/api/client/session.rs | 45 ++- src/api/client/space.rs | 9 +- src/api/client/state.rs | 47 +-- src/api/client/sync.rs | 281 ++++++++------- src/api/client/tag.rs | 25 +- src/api/client/threads.rs | 11 +- src/api/client/to_device.rs | 19 +- src/api/client/typing.rs | 15 +- src/api/client/unstable.rs | 52 +-- src/api/client/unversioned.rs | 30 +- src/api/client/user_directory.rs | 19 +- src/api/client/voip.rs | 19 +- src/api/router/args.rs | 23 +- src/api/router/auth.rs | 36 +- src/api/router/request.rs | 9 +- src/api/server/backfill.rs | 23 +- src/api/server/event.rs | 20 +- src/api/server/event_auth.rs | 17 +- src/api/server/get_missing_events.rs | 15 +- src/api/server/hierarchy.rs | 11 +- src/api/server/invite.rs | 30 +- src/api/server/key.rs | 20 +- src/api/server/make_join.rs | 47 +-- src/api/server/make_leave.rs | 15 +- src/api/server/openid.rs | 9 +- src/api/server/publicrooms.rs | 15 +- src/api/server/query.rs | 32 +- src/api/server/send.rs | 2 +- src/api/server/send_join.rs | 53 +-- src/api/server/send_leave.rs | 35 +- src/api/server/state.rs | 30 +- src/api/server/state_ids.rs | 17 +- src/api/server/user.rs | 32 +- src/api/server/well_known.rs | 7 +- src/service/globals/mod.rs | 1 + 61 files changed, 1485 insertions(+), 1320 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 46f71622..7c7f9331 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -530,7 +530,7 @@ pub(super) async fn force_set_room_state_from_server( for result in remote_state_response .pdus .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map)) + .map(|pdu| validate_and_add_event_id(services(), pdu, &room_version, &pub_key_map)) { let Ok((event_id, value)) = result.await else { continue; @@ -558,7 +558,7 @@ pub(super) async fn force_set_room_state_from_server( for result in remote_state_response .auth_chain .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map)) + .map(|pdu| validate_and_add_event_id(services(), pdu, &room_version, &pub_key_map)) { let Ok((event_id, value)) = result.await else { continue; diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 30c30c6e..c9be4473 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -128,7 +128,7 @@ async fn ban_room( &local_user, &room_id ); - if let Err(e) = leave_room(&local_user, &room_id, None).await { + if let Err(e) = leave_room(services(), &local_user, &room_id, None).await { warn!(%e, "Failed to leave room"); } } @@ -151,7 +151,7 @@ async fn ban_room( }) }) { debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(&local_user, &room_id, None).await { + if let Err(e) = leave_room(services(), &local_user, &room_id, None).await { error!( "Error attempting to make local user {} leave room {} during room banning: {}", &local_user, &room_id, e @@ -334,7 +334,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", &local_user, room_id ); - if let Err(e) = leave_room(&local_user, &room_id, None).await { + if let Err(e) = leave_room(services(), &local_user, &room_id, None).await { warn!(%e, "Failed to leave room"); } } @@ -358,7 +358,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo }) }) { debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(&local_user, &room_id, None).await { + if let Err(e) = leave_room(services(), &local_user, &room_id, None).await { error!( "Error attempting to make local user {} leave room {} during bulk room banning: {}", &local_user, &room_id, e diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 709e1f57..f6387ee8 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -101,6 +101,7 @@ pub(super) async fn create( if let Some(room_id_server_name) = room.server_name() { match join_room_by_id_helper( + services(), &user_id, room, Some("Automatically joining this room upon registration".to_owned()), @@ -158,9 +159,9 @@ pub(super) async fn deactivate( .rooms_joined(&user_id) .filter_map(Result::ok) .collect(); - update_displayname(user_id.clone(), None, all_joined_rooms.clone()).await?; - update_avatar_url(user_id.clone(), None, None, all_joined_rooms).await?; - leave_all_rooms(&user_id).await; + update_displayname(services(), user_id.clone(), None, all_joined_rooms.clone()).await?; + update_avatar_url(services(), user_id.clone(), None, None, all_joined_rooms).await?; + leave_all_rooms(services(), &user_id).await; } Ok(RoomMessageEventContent::text_plain(format!( @@ -262,9 +263,9 @@ pub(super) async fn deactivate_all( .rooms_joined(&user_id) .filter_map(Result::ok) .collect(); - update_displayname(user_id.clone(), None, all_joined_rooms.clone()).await?; - update_avatar_url(user_id.clone(), None, None, all_joined_rooms).await?; - leave_all_rooms(&user_id).await; + update_displayname(services(), user_id.clone(), None, all_joined_rooms.clone()).await?; + update_avatar_url(services(), user_id.clone(), None, None, all_joined_rooms).await?; + leave_all_rooms(services(), &user_id).await; } }, Err(e) => { @@ -347,7 +348,7 @@ pub(super) async fn force_join_room( let room_id = services().rooms.alias.resolve(&room_id).await?; assert!(service::user_is_local(&user_id), "Parsed user_id must be a local user"); - join_room_by_id_helper(&user_id, &room_id, None, &[], None).await?; + join_room_by_id_helper(services(), &user_id, &room_id, None, &[], None).await?; Ok(RoomMessageEventContent::notice_markdown(format!( "{user_id} has been joined to {room_id}.", diff --git a/src/api/client/account.rs b/src/api/client/account.rs index d34211bf..19ac89a0 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -1,5 +1,6 @@ use std::fmt::Write; +use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::debug_info; use register::RegistrationKind; @@ -22,7 +23,6 @@ use tracing::{error, info, warn}; use super::{join_room_by_id_helper, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; use crate::{ service::user_is_local, - services, utils::{self}, Error, Result, Ruma, }; @@ -42,20 +42,21 @@ const RANDOM_USER_ID_LENGTH: usize = 10; /// invalid when trying to register #[tracing::instrument(skip_all, fields(%client), name = "register_available")] pub(crate) async fn get_register_available_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { // Validate user id - let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), services().globals.server_name()) + let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), services.globals.server_name()) .ok() .filter(|user_id| !user_id.is_historical() && user_is_local(user_id)) .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; // Check if username is creative enough - if services().users.exists(&user_id)? { + if services.users.exists(&user_id)? { return Err(Error::BadRequest(ErrorKind::UserInUse, "Desired user ID is already taken.")); } - if services() + if services .globals .forbidden_usernames() .is_match(user_id.localpart()) @@ -91,9 +92,9 @@ pub(crate) async fn get_register_available_route( #[allow(clippy::doc_markdown)] #[tracing::instrument(skip_all, fields(%client), name = "register")] pub(crate) async fn register_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - if !services().globals.allow_registration() && body.appservice_info.is_none() { + if !services.globals.allow_registration() && body.appservice_info.is_none() { info!( "Registration disabled and request not from known appservice, rejecting registration attempt for username \ {:?}", @@ -105,8 +106,8 @@ pub(crate) async fn register_route( let is_guest = body.kind == RegistrationKind::Guest; if is_guest - && (!services().globals.allow_guest_registration() - || (services().globals.allow_registration() && services().globals.config.registration_token.is_some())) + && (!services.globals.allow_guest_registration() + || (services.globals.allow_registration() && services.globals.config.registration_token.is_some())) { info!( "Guest registration disabled / registration enabled with token configured, rejecting guest registration \ @@ -121,7 +122,7 @@ pub(crate) async fn register_route( // forbid guests from registering if there is not a real admin user yet. give // generic user error. - if is_guest && services().users.count()? < 2 { + if is_guest && services.users.count()? < 2 { warn!( "Guest account attempted to register before a real admin user has been registered, rejecting \ registration. Guest's initial device name: {:?}", @@ -133,16 +134,16 @@ pub(crate) async fn register_route( let user_id = match (&body.username, is_guest) { (Some(username), false) => { let proposed_user_id = - UserId::parse_with_server_name(username.to_lowercase(), services().globals.server_name()) + UserId::parse_with_server_name(username.to_lowercase(), services.globals.server_name()) .ok() .filter(|user_id| !user_id.is_historical() && user_is_local(user_id)) .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; - if services().users.exists(&proposed_user_id)? { + if services.users.exists(&proposed_user_id)? { return Err(Error::BadRequest(ErrorKind::UserInUse, "Desired user ID is already taken.")); } - if services() + if services .globals .forbidden_usernames() .is_match(proposed_user_id.localpart()) @@ -155,10 +156,10 @@ pub(crate) async fn register_route( _ => loop { let proposed_user_id = UserId::parse_with_server_name( utils::random_string(RANDOM_USER_ID_LENGTH).to_lowercase(), - services().globals.server_name(), + services.globals.server_name(), ) .unwrap(); - if !services().users.exists(&proposed_user_id)? { + if !services.users.exists(&proposed_user_id)? { break proposed_user_id; } }, @@ -172,13 +173,13 @@ pub(crate) async fn register_route( } else { return Err(Error::BadRequest(ErrorKind::MissingToken, "Missing appservice token.")); } - } else if services().appservice.is_exclusive_user_id(&user_id).await { + } else if services.appservice.is_exclusive_user_id(&user_id).await { return Err(Error::BadRequest(ErrorKind::Exclusive, "User ID reserved by appservice.")); } // UIAA let mut uiaainfo; - let skip_auth = if services().globals.config.registration_token.is_some() { + let skip_auth = if services.globals.config.registration_token.is_some() { // Registration token required uiaainfo = UiaaInfo { flows: vec![AuthFlow { @@ -206,8 +207,8 @@ pub(crate) async fn register_route( if !skip_auth { if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services().uiaa.try_auth( - &UserId::parse_with_server_name("", services().globals.server_name()).expect("we know this is valid"), + let (worked, uiaainfo) = services.uiaa.try_auth( + &UserId::parse_with_server_name("", services.globals.server_name()).expect("we know this is valid"), "".into(), auth, &uiaainfo, @@ -218,8 +219,8 @@ pub(crate) async fn register_route( // Success! } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services().uiaa.create( - &UserId::parse_with_server_name("", services().globals.server_name()).expect("we know this is valid"), + services.uiaa.create( + &UserId::parse_with_server_name("", services.globals.server_name()).expect("we know this is valid"), "".into(), &uiaainfo, &json, @@ -237,25 +238,25 @@ pub(crate) async fn register_route( }; // Create user - services().users.create(&user_id, password)?; + services.users.create(&user_id, password)?; // Default to pretty displayname let mut displayname = user_id.localpart().to_owned(); // If `new_user_displayname_suffix` is set, registration will push whatever // content is set to the user's display name with a space before it - if !services().globals.new_user_displayname_suffix().is_empty() { - write!(displayname, " {}", services().globals.config.new_user_displayname_suffix) + if !services.globals.new_user_displayname_suffix().is_empty() { + write!(displayname, " {}", services.globals.config.new_user_displayname_suffix) .expect("should be able to write to string buffer"); } - services() + services .users .set_displayname(&user_id, Some(displayname.clone())) .await?; // Initial account data - services().account_data.update( + services.account_data.update( None, &user_id, GlobalAccountDataEventType::PushRules.to_string().into(), @@ -290,7 +291,7 @@ pub(crate) async fn register_route( let token = utils::random_string(TOKEN_LENGTH); // Create device for this account - services() + services .users .create_device(&user_id, &device_id, &token, body.initial_device_display_name.clone())?; @@ -299,7 +300,7 @@ pub(crate) async fn register_route( // log in conduit admin channel if a non-guest user registered if body.appservice_info.is_none() && !is_guest { info!("New user \"{user_id}\" registered on this server."); - services() + services .admin .send_message(RoomMessageEventContent::notice_plain(format!( "New user \"{user_id}\" registered on this server from IP {client}." @@ -308,7 +309,7 @@ pub(crate) async fn register_route( } // log in conduit admin channel if a guest registered - if body.appservice_info.is_none() && is_guest && services().globals.log_guest_registrations() { + if body.appservice_info.is_none() && is_guest && services.globals.log_guest_registrations() { info!("New guest user \"{user_id}\" registered on this server."); if let Some(device_display_name) = &body.initial_device_display_name { @@ -317,7 +318,7 @@ pub(crate) async fn register_route( .as_ref() .is_some_and(|device_display_name| !device_display_name.is_empty()) { - services() + services .admin .send_message(RoomMessageEventContent::notice_plain(format!( "Guest user \"{user_id}\" with device display name `{device_display_name}` registered on this \ @@ -325,7 +326,7 @@ pub(crate) async fn register_route( ))) .await; } else { - services() + services .admin .send_message(RoomMessageEventContent::notice_plain(format!( "Guest user \"{user_id}\" with no device display name registered on this server from IP \ @@ -334,7 +335,7 @@ pub(crate) async fn register_route( .await; } } else { - services() + services .admin .send_message(RoomMessageEventContent::notice_plain(format!( "Guest user \"{user_id}\" with no device display name registered on this server from IP {client}.", @@ -347,12 +348,7 @@ pub(crate) async fn register_route( // users Note: the server user, @conduit:servername, is generated first if !is_guest { if let Some(admin_room) = service::admin::Service::get_admin_room()? { - if services() - .rooms - .state_cache - .room_joined_count(&admin_room)? - == Some(1) - { + if services.rooms.state_cache.room_joined_count(&admin_room)? == Some(1) { service::admin::make_user_admin(&user_id, displayname).await?; warn!("Granting {user_id} admin privileges as the first user"); @@ -361,14 +357,14 @@ pub(crate) async fn register_route( } if body.appservice_info.is_none() - && !services().globals.config.auto_join_rooms.is_empty() - && (services().globals.allow_guests_auto_join_rooms() || !is_guest) + && !services.globals.config.auto_join_rooms.is_empty() + && (services.globals.allow_guests_auto_join_rooms() || !is_guest) { - for room in &services().globals.config.auto_join_rooms { - if !services() + for room in &services.globals.config.auto_join_rooms { + if !services .rooms .state_cache - .server_in_room(services().globals.server_name(), room)? + .server_in_room(services.globals.server_name(), room)? { warn!("Skipping room {room} to automatically join as we have never joined before."); continue; @@ -376,10 +372,11 @@ pub(crate) async fn register_route( if let Some(room_id_server_name) = room.server_name() { if let Err(e) = join_room_by_id_helper( + services, &user_id, room, Some("Automatically joining this room upon registration".to_owned()), - &[room_id_server_name.to_owned(), services().globals.server_name().to_owned()], + &[room_id_server_name.to_owned(), services.globals.server_name().to_owned()], None, ) .await @@ -421,7 +418,8 @@ pub(crate) async fn register_route( /// - Triggers device list updates #[tracing::instrument(skip_all, fields(%client), name = "change_password")] pub(crate) async fn change_password_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { // Authentication for this endpoint was made optional, but we need // authentication currently @@ -442,7 +440,7 @@ pub(crate) async fn change_password_route( }; if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services() + let (worked, uiaainfo) = services .uiaa .try_auth(sender_user, sender_device, auth, &uiaainfo)?; if !worked { @@ -451,7 +449,7 @@ pub(crate) async fn change_password_route( // Success! } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services() + services .uiaa .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); @@ -459,24 +457,24 @@ pub(crate) async fn change_password_route( return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } - services() + services .users .set_password(sender_user, Some(&body.new_password))?; if body.logout_devices { // Logout all devices except the current one - for id in services() + for id in services .users .all_device_ids(sender_user) .filter_map(Result::ok) .filter(|id| id != sender_device) { - services().users.remove_device(sender_user, &id)?; + services.users.remove_device(sender_user, &id)?; } } info!("User {sender_user} changed their password."); - services() + services .admin .send_message(RoomMessageEventContent::notice_plain(format!( "User {sender_user} changed their password." @@ -491,14 +489,16 @@ pub(crate) async fn change_password_route( /// Get `user_id` of the sender user. /// /// Note: Also works for Application Services -pub(crate) async fn whoami_route(body: Ruma) -> Result { +pub(crate) async fn whoami_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let device_id = body.sender_device.clone(); Ok(whoami::v3::Response { user_id: sender_user.clone(), device_id, - is_guest: services().users.is_deactivated(sender_user)? && body.appservice_info.is_none(), + is_guest: services.users.is_deactivated(sender_user)? && body.appservice_info.is_none(), }) } @@ -515,7 +515,8 @@ pub(crate) async fn whoami_route(body: Ruma) -> Result, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { // Authentication for this endpoint was made optional, but we need // authentication currently @@ -536,7 +537,7 @@ pub(crate) async fn deactivate_route( }; if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services() + let (worked, uiaainfo) = services .uiaa .try_auth(sender_user, sender_device, auth, &uiaainfo)?; if !worked { @@ -545,7 +546,7 @@ pub(crate) async fn deactivate_route( // Success! } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services() + services .uiaa .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); @@ -554,23 +555,23 @@ pub(crate) async fn deactivate_route( } // Remove devices and mark account as deactivated - services().users.deactivate_account(sender_user)?; + services.users.deactivate_account(sender_user)?; // Remove profile pictures and display name - let all_joined_rooms: Vec = services() + let all_joined_rooms: Vec = services .rooms .state_cache .rooms_joined(sender_user) .filter_map(Result::ok) .collect(); - super::update_displayname(sender_user.clone(), None, all_joined_rooms.clone()).await?; - super::update_avatar_url(sender_user.clone(), None, None, all_joined_rooms).await?; + super::update_displayname(services, sender_user.clone(), None, all_joined_rooms.clone()).await?; + super::update_avatar_url(services, sender_user.clone(), None, None, all_joined_rooms).await?; // Make the user leave all rooms before deactivation - super::leave_all_rooms(sender_user).await; + super::leave_all_rooms(services, sender_user).await; info!("User {sender_user} deactivated their account."); - services() + services .admin .send_message(RoomMessageEventContent::notice_plain(format!( "User {sender_user} deactivated their account." @@ -632,9 +633,9 @@ pub(crate) async fn request_3pid_management_token_via_msisdn_route( /// Currently does not have any ratelimiting, and this isn't very practical as /// there is only one registration token allowed. pub(crate) async fn check_registration_token_validity( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { - let Some(reg_token) = services().globals.config.registration_token.clone() else { + let Some(reg_token) = services.globals.config.registration_token.clone() else { return Err(Error::BadRequest( ErrorKind::forbidden(), "Server does not allow token registration.", diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 5230c25b..88d1a4e6 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use rand::seq::SliceRandom; use ruma::{ api::client::{ @@ -8,19 +9,24 @@ use ruma::{ }; use tracing::debug; -use crate::{service::server_is_ours, services, Error, Result, Ruma}; +use crate::{ + service::{server_is_ours, Services}, + Error, Result, Ruma, +}; /// # `PUT /_matrix/client/v3/directory/room/{roomAlias}` /// /// Creates a new room alias on this server. -pub(crate) async fn create_alias_route(body: Ruma) -> Result { +pub(crate) async fn create_alias_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); service::rooms::alias::appservice_checks(&body.room_alias, &body.appservice_info).await?; // this isn't apart of alias_checks or delete alias route because we should // allow removing forbidden room aliases - if services() + if services .globals .forbidden_alias_names() .is_match(body.room_alias.alias()) @@ -28,7 +34,7 @@ pub(crate) async fn create_alias_route(body: Ruma) -> return Err(Error::BadRequest(ErrorKind::forbidden(), "Room alias is forbidden.")); } - if services() + if services .rooms .alias .resolve_local_alias(&body.room_alias)? @@ -37,7 +43,7 @@ pub(crate) async fn create_alias_route(body: Ruma) -> return Err(Error::Conflict("Alias already exists.")); } - services() + services .rooms .alias .set_alias(&body.room_alias, &body.room_id, sender_user)?; @@ -50,12 +56,14 @@ pub(crate) async fn create_alias_route(body: Ruma) -> /// Deletes a room alias from this server. /// /// - TODO: Update canonical alias event -pub(crate) async fn delete_alias_route(body: Ruma) -> Result { +pub(crate) async fn delete_alias_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); service::rooms::alias::appservice_checks(&body.room_alias, &body.appservice_info).await?; - if services() + if services .rooms .alias .resolve_local_alias(&body.room_alias)? @@ -64,7 +72,7 @@ pub(crate) async fn delete_alias_route(body: Ruma) -> return Err(Error::BadRequest(ErrorKind::NotFound, "Alias does not exist.")); } - services() + services .rooms .alias .remove_alias(&body.room_alias, sender_user) @@ -78,11 +86,13 @@ pub(crate) async fn delete_alias_route(body: Ruma) -> /// # `GET /_matrix/client/v3/directory/room/{roomAlias}` /// /// Resolve an alias locally or over federation. -pub(crate) async fn get_alias_route(body: Ruma) -> Result { +pub(crate) async fn get_alias_route( + State(services): State, body: Ruma, +) -> Result { let room_alias = body.body.room_alias; let servers = None; - let Ok((room_id, pre_servers)) = services() + let Ok((room_id, pre_servers)) = services .rooms .alias .resolve_alias(&room_alias, servers.as_ref()) @@ -91,17 +101,17 @@ pub(crate) async fn get_alias_route(body: Ruma) -> Resul return Err(Error::BadRequest(ErrorKind::NotFound, "Room with alias not found.")); }; - let servers = room_available_servers(&room_id, &room_alias, &pre_servers); + let servers = room_available_servers(services, &room_id, &room_alias, &pre_servers); debug!(?room_alias, ?room_id, "available servers: {servers:?}"); Ok(get_alias::v3::Response::new(room_id, servers)) } fn room_available_servers( - room_id: &RoomId, room_alias: &RoomAliasId, pre_servers: &Option>, + services: &Services, room_id: &RoomId, room_alias: &RoomAliasId, pre_servers: &Option>, ) -> Vec { // find active servers in room state cache to suggest - let mut servers: Vec = services() + let mut servers: Vec = services .rooms .state_cache .room_servers(room_id) @@ -127,7 +137,7 @@ fn room_available_servers( .position(|server_name| server_is_ours(server_name)) { servers.swap_remove(server_index); - servers.insert(0, services().globals.server_name().to_owned()); + servers.insert(0, services.globals.server_name().to_owned()); } else if let Some(alias_server_index) = servers .iter() .position(|server| server == room_alias.server_name()) diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index fb7e7f31..4ead8777 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use ruma::{ api::client::{ backup::{ @@ -11,16 +12,16 @@ use ruma::{ UInt, }; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `POST /_matrix/client/r0/room_keys/version` /// /// Creates a new backup. pub(crate) async fn create_backup_version_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let version = services() + let version = services .key_backups .create_backup(sender_user, &body.algorithm)?; @@ -34,10 +35,10 @@ pub(crate) async fn create_backup_version_route( /// Update information about an existing backup. Only `auth_data` can be /// modified. pub(crate) async fn update_backup_version_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services() + services .key_backups .update_backup(sender_user, &body.version, &body.algorithm)?; @@ -48,20 +49,20 @@ pub(crate) async fn update_backup_version_route( /// /// Get information about the latest backup version. pub(crate) async fn get_latest_backup_info_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let (version, algorithm) = services() + let (version, algorithm) = services .key_backups .get_latest_backup(sender_user)? .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Key backup does not exist."))?; Ok(get_latest_backup_info::v3::Response { algorithm, - count: (UInt::try_from(services().key_backups.count_keys(sender_user, &version)?) + count: (UInt::try_from(services.key_backups.count_keys(sender_user, &version)?) .expect("user backup keys count should not be that high")), - etag: services().key_backups.get_etag(sender_user, &version)?, + etag: services.key_backups.get_etag(sender_user, &version)?, version, }) } @@ -70,10 +71,10 @@ pub(crate) async fn get_latest_backup_info_route( /// /// Get information about an existing backup. pub(crate) async fn get_backup_info_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let algorithm = services() + let algorithm = services .key_backups .get_backup(sender_user, &body.version)? .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Key backup does not exist."))?; @@ -81,14 +82,12 @@ pub(crate) async fn get_backup_info_route( Ok(get_backup_info::v3::Response { algorithm, count: (UInt::try_from( - services() + services .key_backups .count_keys(sender_user, &body.version)?, ) .expect("user backup keys count should not be that high")), - etag: services() - .key_backups - .get_etag(sender_user, &body.version)?, + etag: services.key_backups.get_etag(sender_user, &body.version)?, version: body.version.clone(), }) } @@ -100,11 +99,11 @@ pub(crate) async fn get_backup_info_route( /// - Deletes both information about the backup, as well as all key data related /// to the backup pub(crate) async fn delete_backup_version_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services() + services .key_backups .delete_backup(sender_user, &body.version)?; @@ -120,12 +119,12 @@ pub(crate) async fn delete_backup_version_route( /// - Adds the keys to the backup /// - Returns the new number of keys in this backup and the etag pub(crate) async fn add_backup_keys_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); if Some(&body.version) - != services() + != services .key_backups .get_latest_backup_version(sender_user)? .as_ref() @@ -138,7 +137,7 @@ pub(crate) async fn add_backup_keys_route( for (room_id, room) in &body.rooms { for (session_id, key_data) in &room.sessions { - services() + services .key_backups .add_key(sender_user, &body.version, room_id, session_id, key_data)?; } @@ -146,14 +145,12 @@ pub(crate) async fn add_backup_keys_route( Ok(add_backup_keys::v3::Response { count: (UInt::try_from( - services() + services .key_backups .count_keys(sender_user, &body.version)?, ) .expect("user backup keys count should not be that high")), - etag: services() - .key_backups - .get_etag(sender_user, &body.version)?, + etag: services.key_backups.get_etag(sender_user, &body.version)?, }) } @@ -166,12 +163,12 @@ pub(crate) async fn add_backup_keys_route( /// - Adds the keys to the backup /// - Returns the new number of keys in this backup and the etag pub(crate) async fn add_backup_keys_for_room_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); if Some(&body.version) - != services() + != services .key_backups .get_latest_backup_version(sender_user)? .as_ref() @@ -183,21 +180,19 @@ pub(crate) async fn add_backup_keys_for_room_route( } for (session_id, key_data) in &body.sessions { - services() + services .key_backups .add_key(sender_user, &body.version, &body.room_id, session_id, key_data)?; } Ok(add_backup_keys_for_room::v3::Response { count: (UInt::try_from( - services() + services .key_backups .count_keys(sender_user, &body.version)?, ) .expect("user backup keys count should not be that high")), - etag: services() - .key_backups - .get_etag(sender_user, &body.version)?, + etag: services.key_backups.get_etag(sender_user, &body.version)?, }) } @@ -210,12 +205,12 @@ pub(crate) async fn add_backup_keys_for_room_route( /// - Adds the keys to the backup /// - Returns the new number of keys in this backup and the etag pub(crate) async fn add_backup_keys_for_session_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); if Some(&body.version) - != services() + != services .key_backups .get_latest_backup_version(sender_user)? .as_ref() @@ -226,20 +221,18 @@ pub(crate) async fn add_backup_keys_for_session_route( )); } - services() + services .key_backups .add_key(sender_user, &body.version, &body.room_id, &body.session_id, &body.session_data)?; Ok(add_backup_keys_for_session::v3::Response { count: (UInt::try_from( - services() + services .key_backups .count_keys(sender_user, &body.version)?, ) .expect("user backup keys count should not be that high")), - etag: services() - .key_backups - .get_etag(sender_user, &body.version)?, + etag: services.key_backups.get_etag(sender_user, &body.version)?, }) } @@ -247,11 +240,11 @@ pub(crate) async fn add_backup_keys_for_session_route( /// /// Retrieves all keys from the backup. pub(crate) async fn get_backup_keys_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let rooms = services().key_backups.get_all(sender_user, &body.version)?; + let rooms = services.key_backups.get_all(sender_user, &body.version)?; Ok(get_backup_keys::v3::Response { rooms, @@ -262,11 +255,11 @@ pub(crate) async fn get_backup_keys_route( /// /// Retrieves all keys from the backup for a given room. pub(crate) async fn get_backup_keys_for_room_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sessions = services() + let sessions = services .key_backups .get_room(sender_user, &body.version, &body.room_id)?; @@ -279,11 +272,11 @@ pub(crate) async fn get_backup_keys_for_room_route( /// /// Retrieves a key from the backup. pub(crate) async fn get_backup_keys_for_session_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let key_data = services() + let key_data = services .key_backups .get_session(sender_user, &body.version, &body.room_id, &body.session_id)? .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Backup key not found for this user's session."))?; @@ -297,24 +290,22 @@ pub(crate) async fn get_backup_keys_for_session_route( /// /// Delete the keys from the backup. pub(crate) async fn delete_backup_keys_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services() + services .key_backups .delete_all_keys(sender_user, &body.version)?; Ok(delete_backup_keys::v3::Response { count: (UInt::try_from( - services() + services .key_backups .count_keys(sender_user, &body.version)?, ) .expect("user backup keys count should not be that high")), - etag: services() - .key_backups - .get_etag(sender_user, &body.version)?, + etag: services.key_backups.get_etag(sender_user, &body.version)?, }) } @@ -322,24 +313,22 @@ pub(crate) async fn delete_backup_keys_route( /// /// Delete the keys from the backup for a given room. pub(crate) async fn delete_backup_keys_for_room_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services() + services .key_backups .delete_room_keys(sender_user, &body.version, &body.room_id)?; Ok(delete_backup_keys_for_room::v3::Response { count: (UInt::try_from( - services() + services .key_backups .count_keys(sender_user, &body.version)?, ) .expect("user backup keys count should not be that high")), - etag: services() - .key_backups - .get_etag(sender_user, &body.version)?, + etag: services.key_backups.get_etag(sender_user, &body.version)?, }) } @@ -347,23 +336,21 @@ pub(crate) async fn delete_backup_keys_for_room_route( /// /// Delete a key from the backup. pub(crate) async fn delete_backup_keys_for_session_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services() + services .key_backups .delete_room_key(sender_user, &body.version, &body.room_id, &body.session_id)?; Ok(delete_backup_keys_for_session::v3::Response { count: (UInt::try_from( - services() + services .key_backups .count_keys(sender_user, &body.version)?, ) .expect("user backup keys count should not be that high")), - etag: services() - .key_backups - .get_etag(sender_user, &body.version)?, + etag: services.key_backups.get_etag(sender_user, &body.version)?, }) } diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index f8572fe8..c04347f8 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -1,29 +1,30 @@ use std::collections::BTreeMap; +use axum::extract::State; use ruma::api::client::discovery::get_capabilities::{ self, Capabilities, RoomVersionStability, RoomVersionsCapability, ThirdPartyIdChangesCapability, }; -use crate::{services, Result, Ruma}; +use crate::{Result, Ruma}; /// # `GET /_matrix/client/v3/capabilities` /// /// Get information on the supported feature set and other relevent capabilities /// of this server. pub(crate) async fn get_capabilities_route( - _body: Ruma, + State(services): State, _body: Ruma, ) -> Result { let mut available = BTreeMap::new(); - for room_version in &services().globals.unstable_room_versions { + for room_version in &services.globals.unstable_room_versions { available.insert(room_version.clone(), RoomVersionStability::Unstable); } - for room_version in &services().globals.stable_room_versions { + for room_version in &services.globals.stable_room_versions { available.insert(room_version.clone(), RoomVersionStability::Stable); } let mut capabilities = Capabilities::default(); capabilities.room_versions = RoomVersionsCapability { - default: services().globals.default_room_version(), + default: services.globals.default_room_version(), available, }; diff --git a/src/api/client/config.rs b/src/api/client/config.rs index 603c192a..56d33ba7 100644 --- a/src/api/client/config.rs +++ b/src/api/client/config.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use ruma::{ api::client::{ config::{get_global_account_data, get_room_account_data, set_global_account_data, set_room_account_data}, @@ -10,15 +11,21 @@ use ruma::{ use serde::Deserialize; use serde_json::{json, value::RawValue as RawJsonValue}; -use crate::{services, Error, Result, Ruma}; +use crate::{service::Services, Error, Result, Ruma}; /// # `PUT /_matrix/client/r0/user/{userId}/account_data/{type}` /// /// Sets some account data for the sender user. pub(crate) async fn set_global_account_data_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { - set_account_data(None, &body.sender_user, &body.event_type.to_string(), body.data.json())?; + set_account_data( + services, + None, + &body.sender_user, + &body.event_type.to_string(), + body.data.json(), + )?; Ok(set_global_account_data::v3::Response {}) } @@ -27,9 +34,10 @@ pub(crate) async fn set_global_account_data_route( /// /// Sets some room account data for the sender user. pub(crate) async fn set_room_account_data_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { set_account_data( + services, Some(&body.room_id), &body.sender_user, &body.event_type.to_string(), @@ -43,11 +51,11 @@ pub(crate) async fn set_room_account_data_route( /// /// Gets some account data for the sender user. pub(crate) async fn get_global_account_data_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event: Box = services() + let event: Box = services .account_data .get(None, sender_user, body.event_type.to_string().into())? .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Data not found."))?; @@ -65,11 +73,11 @@ pub(crate) async fn get_global_account_data_route( /// /// Gets some room account data for the sender user. pub(crate) async fn get_room_account_data_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event: Box = services() + let event: Box = services .account_data .get(Some(&body.room_id), sender_user, body.event_type.clone())? .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Data not found."))?; @@ -84,14 +92,15 @@ pub(crate) async fn get_room_account_data_route( } fn set_account_data( - room_id: Option<&RoomId>, sender_user: &Option, event_type: &str, data: &RawJsonValue, + services: &Services, room_id: Option<&RoomId>, sender_user: &Option, event_type: &str, + data: &RawJsonValue, ) -> Result<()> { let sender_user = sender_user.as_ref().expect("user is authenticated"); let data: serde_json::Value = serde_json::from_str(data.get()).map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?; - services().account_data.update( + services.account_data.update( room_id, sender_user, event_type.into(), diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 2ca161c9..89a08153 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,12 +1,13 @@ use std::collections::HashSet; +use axum::extract::State; use ruma::{ api::client::{context::get_context, error::ErrorKind, filter::LazyLoadOptions}, events::StateEventType, }; use tracing::error; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `GET /_matrix/client/r0/rooms/{roomId}/context` /// @@ -14,7 +15,9 @@ use crate::{services, Error, Result, Ruma}; /// /// - Only works if the user is joined (TODO: always allow, but only show events /// if the user was joined, depending on history_visibility) -pub(crate) async fn get_context_route(body: Ruma) -> Result { +pub(crate) async fn get_context_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -27,13 +30,13 @@ pub(crate) async fn get_context_route(body: Ruma) -> R let mut lazy_loaded = HashSet::new(); - let base_token = services() + let base_token = services .rooms .timeline .get_pdu_count(&body.event_id)? .ok_or(Error::BadRequest(ErrorKind::NotFound, "Base event id not found."))?; - let base_event = services() + let base_event = services .rooms .timeline .get_pdu(&body.event_id)? @@ -41,7 +44,7 @@ pub(crate) async fn get_context_route(body: Ruma) -> R let room_id = base_event.room_id.clone(); - if !services() + if !services .rooms .state_accessor .user_can_see_event(sender_user, &room_id, &body.event_id)? @@ -52,7 +55,7 @@ pub(crate) async fn get_context_route(body: Ruma) -> R )); } - if !services().rooms.lazy_loading.lazy_load_was_sent_before( + if !services.rooms.lazy_loading.lazy_load_was_sent_before( sender_user, sender_device, &room_id, @@ -67,14 +70,14 @@ pub(crate) async fn get_context_route(body: Ruma) -> R let base_event = base_event.to_room_event(); - let events_before: Vec<_> = services() + let events_before: Vec<_> = services .rooms .timeline .pdus_until(sender_user, &room_id, base_token)? .take(limit / 2) .filter_map(Result::ok) // Remove buggy events .filter(|(_, pdu)| { - services() + services .rooms .state_accessor .user_can_see_event(sender_user, &room_id, &pdu.event_id) @@ -83,7 +86,7 @@ pub(crate) async fn get_context_route(body: Ruma) -> R .collect(); for (_, event) in &events_before { - if !services().rooms.lazy_loading.lazy_load_was_sent_before( + if !services.rooms.lazy_loading.lazy_load_was_sent_before( sender_user, sender_device, &room_id, @@ -103,14 +106,14 @@ pub(crate) async fn get_context_route(body: Ruma) -> R .map(|(_, pdu)| pdu.to_room_event()) .collect(); - let events_after: Vec<_> = services() + let events_after: Vec<_> = services .rooms .timeline .pdus_after(sender_user, &room_id, base_token)? .take(limit / 2) .filter_map(Result::ok) // Remove buggy events .filter(|(_, pdu)| { - services() + services .rooms .state_accessor .user_can_see_event(sender_user, &room_id, &pdu.event_id) @@ -119,7 +122,7 @@ pub(crate) async fn get_context_route(body: Ruma) -> R .collect(); for (_, event) in &events_after { - if !services().rooms.lazy_loading.lazy_load_was_sent_before( + if !services.rooms.lazy_loading.lazy_load_was_sent_before( sender_user, sender_device, &room_id, @@ -130,7 +133,7 @@ pub(crate) async fn get_context_route(body: Ruma) -> R } } - let shortstatehash = services() + let shortstatehash = services .rooms .state_accessor .pdu_shortstatehash( @@ -139,7 +142,7 @@ pub(crate) async fn get_context_route(body: Ruma) -> R .map_or(&*body.event_id, |(_, e)| &*e.event_id), )? .map_or( - services() + services .rooms .state .get_room_shortstatehash(&room_id)? @@ -147,7 +150,7 @@ pub(crate) async fn get_context_route(body: Ruma) -> R |hash| hash, ); - let state_ids = services() + let state_ids = services .rooms .state_accessor .state_full_ids(shortstatehash) @@ -165,20 +168,20 @@ pub(crate) async fn get_context_route(body: Ruma) -> R let mut state = Vec::with_capacity(state_ids.len()); for (shortstatekey, id) in state_ids { - let (event_type, state_key) = services() + let (event_type, state_key) = services .rooms .short .get_statekey_from_short(shortstatekey)?; if event_type != StateEventType::RoomMember { - let Some(pdu) = services().rooms.timeline.get_pdu(&id)? else { + let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { error!("Pdu in state not found: {}", id); continue; }; state.push(pdu.to_state_event()); } else if !lazy_load_enabled || lazy_loaded.contains(&state_key) { - let Some(pdu) = services().rooms.timeline.get_pdu(&id)? else { + let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { error!("Pdu in state not found: {}", id); continue; }; diff --git a/src/api/client/device.rs b/src/api/client/device.rs index c917ba77..bad7f284 100644 --- a/src/api/client/device.rs +++ b/src/api/client/device.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use ruma::api::client::{ device::{self, delete_device, delete_devices, get_device, get_devices, update_device}, error::ErrorKind, @@ -5,15 +6,17 @@ use ruma::api::client::{ }; use super::SESSION_ID_LENGTH; -use crate::{services, utils, Error, Result, Ruma}; +use crate::{utils, Error, Result, Ruma}; /// # `GET /_matrix/client/r0/devices` /// /// Get metadata on all devices of the sender user. -pub(crate) async fn get_devices_route(body: Ruma) -> Result { +pub(crate) async fn get_devices_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let devices: Vec = services() + let devices: Vec = services .users .all_devices_metadata(sender_user) .filter_map(Result::ok) // Filter out buggy devices @@ -27,10 +30,12 @@ pub(crate) async fn get_devices_route(body: Ruma) -> R /// # `GET /_matrix/client/r0/devices/{deviceId}` /// /// Get metadata on a single device of the sender user. -pub(crate) async fn get_device_route(body: Ruma) -> Result { +pub(crate) async fn get_device_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let device = services() + let device = services .users .get_device_metadata(sender_user, &body.body.device_id)? .ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?; @@ -43,17 +48,19 @@ pub(crate) async fn get_device_route(body: Ruma) -> Res /// # `PUT /_matrix/client/r0/devices/{deviceId}` /// /// Updates the metadata on a given device of the sender user. -pub(crate) async fn update_device_route(body: Ruma) -> Result { +pub(crate) async fn update_device_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let mut device = services() + let mut device = services .users .get_device_metadata(sender_user, &body.device_id)? .ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?; device.display_name.clone_from(&body.display_name); - services() + services .users .update_device_metadata(sender_user, &body.device_id, &device)?; @@ -70,7 +77,9 @@ pub(crate) async fn update_device_route(body: Ruma) /// last seen ts) /// - Forgets to-device events /// - Triggers device list updates -pub(crate) async fn delete_device_route(body: Ruma) -> Result { +pub(crate) async fn delete_device_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -86,7 +95,7 @@ pub(crate) async fn delete_device_route(body: Ruma) }; if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services() + let (worked, uiaainfo) = services .uiaa .try_auth(sender_user, sender_device, auth, &uiaainfo)?; if !worked { @@ -95,7 +104,7 @@ pub(crate) async fn delete_device_route(body: Ruma) // Success! } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services() + services .uiaa .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); @@ -103,9 +112,7 @@ pub(crate) async fn delete_device_route(body: Ruma) return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } - services() - .users - .remove_device(sender_user, &body.device_id)?; + services.users.remove_device(sender_user, &body.device_id)?; Ok(delete_device::v3::Response {}) } @@ -123,7 +130,7 @@ pub(crate) async fn delete_device_route(body: Ruma) /// - Forgets to-device events /// - Triggers device list updates pub(crate) async fn delete_devices_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -140,7 +147,7 @@ pub(crate) async fn delete_devices_route( }; if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services() + let (worked, uiaainfo) = services .uiaa .try_auth(sender_user, sender_device, auth, &uiaainfo)?; if !worked { @@ -149,7 +156,7 @@ pub(crate) async fn delete_devices_route( // Success! } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services() + services .uiaa .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); @@ -158,7 +165,7 @@ pub(crate) async fn delete_devices_route( } for device_id in &body.devices { - services().users.remove_device(sender_user, device_id)?; + services.users.remove_device(sender_user, device_id)?; } Ok(delete_devices::v3::Response {}) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 8e12c034..deebd250 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{err, info, warn, Error, Result}; use ruma::{ @@ -20,7 +21,10 @@ use ruma::{ uint, RoomId, ServerName, UInt, UserId, }; -use crate::{service::server_is_ours, services, Ruma}; +use crate::{ + service::{server_is_ours, Services}, + Ruma, +}; /// # `POST /_matrix/client/v3/publicRooms` /// @@ -29,10 +33,11 @@ use crate::{service::server_is_ours, services, Ruma}; /// - Rooms are ordered by the number of joined members #[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] pub(crate) async fn get_public_rooms_filtered_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { if let Some(server) = &body.server { - if services() + if services .globals .forbidden_remote_room_directory_server_names() .contains(server) @@ -45,6 +50,7 @@ pub(crate) async fn get_public_rooms_filtered_route( } let response = get_public_rooms_filtered_helper( + services, body.server.as_deref(), body.limit, body.since.as_deref(), @@ -67,10 +73,11 @@ pub(crate) async fn get_public_rooms_filtered_route( /// - Rooms are ordered by the number of joined members #[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] pub(crate) async fn get_public_rooms_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { if let Some(server) = &body.server { - if services() + if services .globals .forbidden_remote_room_directory_server_names() .contains(server) @@ -83,6 +90,7 @@ pub(crate) async fn get_public_rooms_route( } let response = get_public_rooms_filtered_helper( + services, body.server.as_deref(), body.limit, body.since.as_deref(), @@ -108,16 +116,17 @@ pub(crate) async fn get_public_rooms_route( /// Sets the visibility of a given room in the room directory. #[tracing::instrument(skip_all, fields(%client), name = "room_directory")] pub(crate) async fn set_room_visibility_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services().rooms.metadata.exists(&body.room_id)? { + if !services.rooms.metadata.exists(&body.room_id)? { // Return 404 if the room doesn't exist return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } - if !user_can_publish_room(sender_user, &body.room_id)? { + if !user_can_publish_room(services, sender_user, &body.room_id)? { return Err(Error::BadRequest( ErrorKind::forbidden(), "User is not allowed to publish this room", @@ -126,7 +135,7 @@ pub(crate) async fn set_room_visibility_route( match &body.visibility { room::Visibility::Public => { - if services().globals.config.lockdown_public_room_directory && !services().users.is_admin(sender_user)? { + if services.globals.config.lockdown_public_room_directory && !services.users.is_admin(sender_user)? { info!( "Non-admin user {sender_user} tried to publish {0} to the room directory while \ \"lockdown_public_room_directory\" is enabled", @@ -139,10 +148,10 @@ pub(crate) async fn set_room_visibility_route( )); } - services().rooms.directory.set_public(&body.room_id)?; + services.rooms.directory.set_public(&body.room_id)?; info!("{sender_user} made {0} public", body.room_id); }, - room::Visibility::Private => services().rooms.directory.set_not_public(&body.room_id)?, + room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id)?, _ => { return Err(Error::BadRequest( ErrorKind::InvalidParam, @@ -158,15 +167,15 @@ pub(crate) async fn set_room_visibility_route( /// /// Gets the visibility of a given room in the room directory. pub(crate) async fn get_room_visibility_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { - if !services().rooms.metadata.exists(&body.room_id)? { + if !services.rooms.metadata.exists(&body.room_id)? { // Return 404 if the room doesn't exist return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } Ok(get_room_visibility::v3::Response { - visibility: if services().rooms.directory.is_public_room(&body.room_id)? { + visibility: if services.rooms.directory.is_public_room(&body.room_id)? { room::Visibility::Public } else { room::Visibility::Private @@ -175,10 +184,11 @@ pub(crate) async fn get_room_visibility_route( } pub(crate) async fn get_public_rooms_filtered_helper( - server: Option<&ServerName>, limit: Option, since: Option<&str>, filter: &Filter, _network: &RoomNetwork, + services: &Services, server: Option<&ServerName>, limit: Option, since: Option<&str>, filter: &Filter, + _network: &RoomNetwork, ) -> Result { if let Some(other_server) = server.filter(|server_name| !server_is_ours(server_name)) { - let response = services() + let response = services .sending .send_federation_request( other_server, @@ -224,7 +234,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( } } - let mut all_rooms: Vec<_> = services() + let mut all_rooms: Vec<_> = services .rooms .directory .public_rooms() @@ -232,12 +242,12 @@ pub(crate) async fn get_public_rooms_filtered_helper( let room_id = room_id?; let chunk = PublicRoomsChunk { - canonical_alias: services() + canonical_alias: services .rooms .state_accessor .get_canonical_alias(&room_id)?, - name: services().rooms.state_accessor.get_name(&room_id)?, - num_joined_members: services() + name: services.rooms.state_accessor.get_name(&room_id)?, + num_joined_members: services .rooms .state_cache .room_joined_count(&room_id)? @@ -247,24 +257,24 @@ pub(crate) async fn get_public_rooms_filtered_helper( }) .try_into() .expect("user count should not be that big"), - topic: services() + topic: services .rooms .state_accessor .get_room_topic(&room_id) .unwrap_or(None), - world_readable: services().rooms.state_accessor.is_world_readable(&room_id)?, - guest_can_join: services() + world_readable: services.rooms.state_accessor.is_world_readable(&room_id)?, + guest_can_join: services .rooms .state_accessor .guest_can_join(&room_id)?, - avatar_url: services() + avatar_url: services .rooms .state_accessor .get_avatar(&room_id)? .into_option() .unwrap_or_default() .url, - join_rule: services() + join_rule: services .rooms .state_accessor .room_state_get(&room_id, &StateEventType::RoomJoinRules, "")? @@ -282,7 +292,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( .transpose()? .flatten() .ok_or_else(|| Error::bad_database("Missing room join rule event for room."))?, - room_type: services() + room_type: services .rooms .state_accessor .get_room_type(&room_id)?, @@ -361,12 +371,11 @@ pub(crate) async fn get_public_rooms_filtered_helper( /// Check whether the user can publish to the room directory via power levels of /// room history visibility event or room creator -fn user_can_publish_room(user_id: &UserId, room_id: &RoomId) -> Result { - if let Some(event) = - services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? +fn user_can_publish_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result { + if let Some(event) = services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? { serde_json::from_str(event.content.get()) .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) @@ -374,7 +383,7 @@ fn user_can_publish_room(user_id: &UserId, room_id: &RoomId) -> Result { RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomHistoryVisibility) }) } else if let Some(event) = - services() + services .rooms .state_accessor .room_state_get(room_id, &StateEventType::RoomCreate, "")? diff --git a/src/api/client/filter.rs b/src/api/client/filter.rs index 5a161400..8b2690c6 100644 --- a/src/api/client/filter.rs +++ b/src/api/client/filter.rs @@ -1,18 +1,21 @@ +use axum::extract::State; use ruma::api::client::{ error::ErrorKind, filter::{create_filter, get_filter}, }; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `GET /_matrix/client/r0/user/{userId}/filter/{filterId}` /// /// Loads a filter that was previously created. /// /// - A user can only access their own filters -pub(crate) async fn get_filter_route(body: Ruma) -> Result { +pub(crate) async fn get_filter_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let Some(filter) = services().users.get_filter(sender_user, &body.filter_id)? else { + let Some(filter) = services.users.get_filter(sender_user, &body.filter_id)? else { return Err(Error::BadRequest(ErrorKind::NotFound, "Filter not found.")); }; @@ -22,9 +25,11 @@ pub(crate) async fn get_filter_route(body: Ruma) -> Res /// # `PUT /_matrix/client/r0/user/{userId}/filter` /// /// Creates a new filter to be used by other endpoints. -pub(crate) async fn create_filter_route(body: Ruma) -> Result { +pub(crate) async fn create_filter_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); Ok(create_filter::v3::Response::new( - services().users.create_filter(sender_user, &body.filter)?, + services.users.create_filter(sender_user, &body.filter)?, )) } diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 7bb02a60..728ea7a9 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -3,6 +3,7 @@ use std::{ time::Instant, }; +use axum::extract::State; use conduit::{utils, utils::math::continue_exponential_backoff_secs, Error, Result}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ @@ -22,7 +23,7 @@ use service::user_is_local; use tracing::debug; use super::SESSION_ID_LENGTH; -use crate::{services, Ruma}; +use crate::{service::Services, Ruma}; /// # `POST /_matrix/client/r0/keys/upload` /// @@ -31,12 +32,14 @@ use crate::{services, Ruma}; /// - Adds one time keys /// - If there are no device keys yet: Adds device keys (TODO: merge with /// existing keys?) -pub(crate) async fn upload_keys_route(body: Ruma) -> Result { +pub(crate) async fn upload_keys_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); for (key_key, key_value) in &body.one_time_keys { - services() + services .users .add_one_time_key(sender_user, sender_device, key_key, key_value)?; } @@ -44,19 +47,19 @@ pub(crate) async fn upload_keys_route(body: Ruma) -> R if let Some(device_keys) = &body.device_keys { // TODO: merge this and the existing event? // This check is needed to assure that signatures are kept - if services() + if services .users .get_device_keys(sender_user, sender_device)? .is_none() { - services() + services .users .add_device_keys(sender_user, sender_device, device_keys)?; } } Ok(upload_keys::v3::Response { - one_time_key_counts: services() + one_time_key_counts: services .users .count_one_time_keys(sender_user, sender_device)?, }) @@ -70,10 +73,13 @@ pub(crate) async fn upload_keys_route(body: Ruma) -> R /// - Gets master keys, self-signing keys, user signing keys and device keys. /// - The master and self-signing keys contain signatures that the user is /// allowed to see -pub(crate) async fn get_keys_route(body: Ruma) -> Result { +pub(crate) async fn get_keys_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); get_keys_helper( + services, Some(sender_user), &body.device_keys, |u| u == sender_user, @@ -85,8 +91,10 @@ pub(crate) async fn get_keys_route(body: Ruma) -> Result< /// # `POST /_matrix/client/r0/keys/claim` /// /// Claims one-time keys -pub(crate) async fn claim_keys_route(body: Ruma) -> Result { - claim_keys_helper(&body.one_time_keys).await +pub(crate) async fn claim_keys_route( + State(services): State, body: Ruma, +) -> Result { + claim_keys_helper(services, &body.one_time_keys).await } /// # `POST /_matrix/client/r0/keys/device_signing/upload` @@ -95,7 +103,7 @@ pub(crate) async fn claim_keys_route(body: Ruma) -> Res /// /// - Requires UIAA to verify password pub(crate) async fn upload_signing_keys_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -112,7 +120,7 @@ pub(crate) async fn upload_signing_keys_route( }; if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services() + let (worked, uiaainfo) = services .uiaa .try_auth(sender_user, sender_device, auth, &uiaainfo)?; if !worked { @@ -121,7 +129,7 @@ pub(crate) async fn upload_signing_keys_route( // Success! } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services() + services .uiaa .create(sender_user, sender_device, &uiaainfo, &json)?; return Err(Error::Uiaa(uiaainfo)); @@ -130,7 +138,7 @@ pub(crate) async fn upload_signing_keys_route( } if let Some(master_key) = &body.master_key { - services().users.add_cross_signing_keys( + services.users.add_cross_signing_keys( sender_user, master_key, &body.self_signing_key, @@ -146,7 +154,7 @@ pub(crate) async fn upload_signing_keys_route( /// /// Uploads end-to-end key signatures from the sender user. pub(crate) async fn upload_signatures_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -173,7 +181,7 @@ pub(crate) async fn upload_signatures_route( .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid signature value."))? .to_owned(), ); - services() + services .users .sign_key(user_id, key_id, signature, sender_user)?; } @@ -192,14 +200,14 @@ pub(crate) async fn upload_signatures_route( /// /// - TODO: left users pub(crate) async fn get_key_changes_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let mut device_list_updates = HashSet::new(); device_list_updates.extend( - services() + services .users .keys_changed( sender_user.as_str(), @@ -215,14 +223,14 @@ pub(crate) async fn get_key_changes_route( .filter_map(Result::ok), ); - for room_id in services() + for room_id in services .rooms .state_cache .rooms_joined(sender_user) .filter_map(Result::ok) { device_list_updates.extend( - services() + services .users .keys_changed( room_id.as_ref(), @@ -245,8 +253,8 @@ pub(crate) async fn get_key_changes_route( } pub(crate) async fn get_keys_helper bool + Send>( - sender_user: Option<&UserId>, device_keys_input: &BTreeMap>, allowed_signatures: F, - include_display_names: bool, + services: &Services, sender_user: Option<&UserId>, device_keys_input: &BTreeMap>, + allowed_signatures: F, include_display_names: bool, ) -> Result { let mut master_keys = BTreeMap::new(); let mut self_signing_keys = BTreeMap::new(); @@ -268,10 +276,10 @@ pub(crate) async fn get_keys_helper bool + Send>( if device_ids.is_empty() { let mut container = BTreeMap::new(); - for device_id in services().users.all_device_ids(user_id) { + for device_id in services.users.all_device_ids(user_id) { let device_id = device_id?; - if let Some(mut keys) = services().users.get_device_keys(user_id, &device_id)? { - let metadata = services() + if let Some(mut keys) = services.users.get_device_keys(user_id, &device_id)? { + let metadata = services .users .get_device_metadata(user_id, &device_id)? .ok_or_else(|| Error::bad_database("all_device_keys contained nonexistent device."))?; @@ -286,8 +294,8 @@ pub(crate) async fn get_keys_helper bool + Send>( } else { for device_id in device_ids { let mut container = BTreeMap::new(); - if let Some(mut keys) = services().users.get_device_keys(user_id, device_id)? { - let metadata = services() + if let Some(mut keys) = services.users.get_device_keys(user_id, device_id)? { + let metadata = services .users .get_device_metadata(user_id, device_id)? .ok_or(Error::BadRequest( @@ -303,21 +311,21 @@ pub(crate) async fn get_keys_helper bool + Send>( } } - if let Some(master_key) = services() + if let Some(master_key) = services .users .get_master_key(sender_user, user_id, &allowed_signatures)? { master_keys.insert(user_id.to_owned(), master_key); } if let Some(self_signing_key) = - services() + services .users .get_self_signing_key(sender_user, user_id, &allowed_signatures)? { self_signing_keys.insert(user_id.to_owned(), self_signing_key); } if Some(user_id) == sender_user { - if let Some(user_signing_key) = services().users.get_user_signing_key(user_id)? { + if let Some(user_signing_key) = services.users.get_user_signing_key(user_id)? { user_signing_keys.insert(user_id.to_owned(), user_signing_key); } } @@ -326,7 +334,7 @@ pub(crate) async fn get_keys_helper bool + Send>( let mut failures = BTreeMap::new(); let back_off = |id| async { - match services() + match services .globals .bad_query_ratelimiter .write() @@ -345,7 +353,7 @@ pub(crate) async fn get_keys_helper bool + Send>( let mut futures: FuturesUnordered<_> = get_over_federation .into_iter() .map(|(server, vec)| async move { - if let Some((time, tries)) = services() + if let Some((time, tries)) = services .globals .bad_query_ratelimiter .read() @@ -369,7 +377,7 @@ pub(crate) async fn get_keys_helper bool + Send>( let request = federation::keys::get_keys::v1::Request { device_keys: device_keys_input_fed, }; - let response = services() + let response = services .sending .send_federation_request(server, request) .await; @@ -381,19 +389,19 @@ pub(crate) async fn get_keys_helper bool + Send>( while let Some((server, response)) = futures.next().await { if let Ok(Ok(response)) = response { for (user, masterkey) in response.master_keys { - let (master_key_id, mut master_key) = services().users.parse_master_key(&user, &masterkey)?; + let (master_key_id, mut master_key) = services.users.parse_master_key(&user, &masterkey)?; if let Some(our_master_key) = - services() + services .users .get_key(&master_key_id, sender_user, &user, &allowed_signatures)? { - let (_, our_master_key) = services().users.parse_master_key(&user, &our_master_key)?; + let (_, our_master_key) = services.users.parse_master_key(&user, &our_master_key)?; master_key.signatures.extend(our_master_key.signatures); } let json = serde_json::to_value(master_key).expect("to_value always works"); let raw = serde_json::from_value(json).expect("Raw::from_value always works"); - services().users.add_cross_signing_keys( + services.users.add_cross_signing_keys( &user, &raw, &None, &None, false, /* Dont notify. A notification would trigger another key request resulting in an * endless loop */ @@ -444,7 +452,7 @@ fn add_unsigned_device_display_name( } pub(crate) async fn claim_keys_helper( - one_time_keys_input: &BTreeMap>, + services: &Services, one_time_keys_input: &BTreeMap>, ) -> Result { let mut one_time_keys = BTreeMap::new(); @@ -460,7 +468,7 @@ pub(crate) async fn claim_keys_helper( let mut container = BTreeMap::new(); for (device_id, key_algorithm) in map { - if let Some(one_time_keys) = services() + if let Some(one_time_keys) = services .users .take_one_time_key(user_id, device_id, key_algorithm)? { @@ -483,7 +491,7 @@ pub(crate) async fn claim_keys_helper( } ( server, - services() + services .sending .send_federation_request( server, diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 39640b23..1adcefdd 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -2,6 +2,7 @@ use std::{io::Cursor, sync::Arc, time::Duration}; +use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{debug, error, utils::math::ruma_from_usize, warn}; use image::io::Reader as ImgReader; @@ -20,9 +21,8 @@ use crate::{ debug_warn, service::{ media::{FileMeta, UrlPreviewData}, - server_is_ours, + server_is_ours, Services, }, - services, utils::{ self, content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename}, @@ -42,10 +42,10 @@ const CORP_CROSS_ORIGIN: &str = "cross-origin"; /// /// Returns max upload size. pub(crate) async fn get_media_config_route( - _body: Ruma, + State(services): State, _body: Ruma, ) -> Result { Ok(get_media_config::v3::Response { - upload_size: ruma_from_usize(services().globals.config.max_request_size), + upload_size: ruma_from_usize(services.globals.config.max_request_size), }) } @@ -57,9 +57,11 @@ pub(crate) async fn get_media_config_route( /// /// Returns max upload size. pub(crate) async fn get_media_config_v1_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result> { - get_media_config_route(body).await.map(RumaResponse) + get_media_config_route(State(services), body) + .await + .map(RumaResponse) } /// # `GET /_matrix/media/v3/preview_url` @@ -67,17 +69,18 @@ pub(crate) async fn get_media_config_v1_route( /// Returns URL preview. #[tracing::instrument(skip_all, fields(%client), name = "url_preview")] pub(crate) async fn get_media_preview_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let url = &body.url; - if !url_preview_allowed(url) { + if !url_preview_allowed(services, url) { warn!(%sender_user, "URL is not allowed to be previewed: {url}"); return Err(Error::BadRequest(ErrorKind::forbidden(), "URL is not allowed to be previewed")); } - match get_url_preview(url).await { + match get_url_preview(services, url).await { Ok(preview) => { let res = serde_json::value::to_raw_value(&preview).map_err(|e| { error!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}"); @@ -115,9 +118,10 @@ pub(crate) async fn get_media_preview_route( /// Returns URL preview. #[tracing::instrument(skip_all, fields(%client), name = "url_preview")] pub(crate) async fn get_media_preview_v1_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result> { - get_media_preview_route(InsecureClientIp(client), body) + get_media_preview_route(State(services), InsecureClientIp(client), body) .await .map(RumaResponse) } @@ -130,17 +134,14 @@ pub(crate) async fn get_media_preview_v1_route( /// - Media will be saved in the media/ directory #[tracing::instrument(skip_all, fields(%client), name = "media_upload")] pub(crate) async fn create_content_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let mxc = format!( - "mxc://{}/{}", - services().globals.server_name(), - utils::random_string(MXC_LENGTH) - ); + let mxc = format!("mxc://{}/{}", services.globals.server_name(), utils::random_string(MXC_LENGTH)); - services() + services .media .create( Some(sender_user.clone()), @@ -178,9 +179,10 @@ pub(crate) async fn create_content_route( /// - Media will be saved in the media/ directory #[tracing::instrument(skip_all, fields(%client), name = "media_upload")] pub(crate) async fn create_content_v1_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result> { - create_content_route(InsecureClientIp(client), body) + create_content_route(State(services), InsecureClientIp(client), body) .await .map(RumaResponse) } @@ -195,7 +197,8 @@ pub(crate) async fn create_content_v1_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); @@ -203,7 +206,7 @@ pub(crate) async fn get_content_route( content, content_type, content_disposition, - }) = services().media.get(&mxc).await? + }) = services.media.get(&mxc).await? { let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); let file = content.expect("content"); @@ -217,6 +220,7 @@ pub(crate) async fn get_content_route( }) } else if !server_is_ours(&body.server_name) && body.allow_remote { let response = get_remote_content( + services, &mxc, &body.server_name, body.media_id.clone(), @@ -261,9 +265,10 @@ pub(crate) async fn get_content_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_v1_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result> { - get_content_route(InsecureClientIp(client), body) + get_content_route(State(services), InsecureClientIp(client), body) .await .map(RumaResponse) } @@ -278,7 +283,8 @@ pub(crate) async fn get_content_v1_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_as_filename_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); @@ -286,7 +292,7 @@ pub(crate) async fn get_content_as_filename_route( content, content_type, content_disposition, - }) = services().media.get(&mxc).await? + }) = services.media.get(&mxc).await? { let content_disposition = Some(make_content_disposition( &content_type, @@ -304,6 +310,7 @@ pub(crate) async fn get_content_as_filename_route( }) } else if !server_is_ours(&body.server_name) && body.allow_remote { match get_remote_content( + services, &mxc, &body.server_name, body.media_id.clone(), @@ -351,9 +358,10 @@ pub(crate) async fn get_content_as_filename_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_as_filename_v1_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result> { - get_content_as_filename_route(InsecureClientIp(client), body) + get_content_as_filename_route(State(services), InsecureClientIp(client), body) .await .map(RumaResponse) } @@ -368,7 +376,8 @@ pub(crate) async fn get_content_as_filename_v1_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] pub(crate) async fn get_content_thumbnail_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); @@ -376,7 +385,7 @@ pub(crate) async fn get_content_thumbnail_route( content, content_type, content_disposition, - }) = services() + }) = services .media .get_thumbnail( &mxc, @@ -400,7 +409,7 @@ pub(crate) async fn get_content_thumbnail_route( content_disposition, }) } else if !server_is_ours(&body.server_name) && body.allow_remote { - if services() + if services .globals .prevent_media_downloads_from() .contains(&body.server_name) @@ -411,7 +420,7 @@ pub(crate) async fn get_content_thumbnail_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")); } - match services() + match services .sending .send_federation_request( &body.server_name, @@ -430,7 +439,7 @@ pub(crate) async fn get_content_thumbnail_route( .await { Ok(get_thumbnail_response) => { - services() + services .media .upload_thumbnail( None, @@ -481,17 +490,19 @@ pub(crate) async fn get_content_thumbnail_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] pub(crate) async fn get_content_thumbnail_v1_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result> { - get_content_thumbnail_route(InsecureClientIp(client), body) + get_content_thumbnail_route(State(services), InsecureClientIp(client), body) .await .map(RumaResponse) } async fn get_remote_content( - mxc: &str, server_name: &ruma::ServerName, media_id: String, allow_redirect: bool, timeout_ms: Duration, + services: &Services, mxc: &str, server_name: &ruma::ServerName, media_id: String, allow_redirect: bool, + timeout_ms: Duration, ) -> Result { - if services() + if services .globals .prevent_media_downloads_from() .contains(&server_name.to_owned()) @@ -502,7 +513,7 @@ async fn get_remote_content( return Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")); } - let content_response = services() + let content_response = services .sending .send_federation_request( server_name, @@ -522,7 +533,7 @@ async fn get_remote_content( None, )); - services() + services .media .create( None, @@ -542,15 +553,11 @@ async fn get_remote_content( }) } -async fn download_image(client: &reqwest::Client, url: &str) -> Result { +async fn download_image(services: &Services, client: &reqwest::Client, url: &str) -> Result { let image = client.get(url).send().await?.bytes().await?; - let mxc = format!( - "mxc://{}/{}", - services().globals.server_name(), - utils::random_string(MXC_LENGTH) - ); + let mxc = format!("mxc://{}/{}", services.globals.server_name(), utils::random_string(MXC_LENGTH)); - services() + services .media .create(None, &mxc, None, None, &image) .await?; @@ -572,18 +579,18 @@ async fn download_image(client: &reqwest::Client, url: &str) -> Result Result { +async fn download_html(services: &Services, client: &reqwest::Client, url: &str) -> Result { let mut response = client.get(url).send().await?; let mut bytes: Vec = Vec::new(); while let Some(chunk) = response.chunk().await? { bytes.extend_from_slice(&chunk); - if bytes.len() > services().globals.url_preview_max_spider_size() { + if bytes.len() > services.globals.url_preview_max_spider_size() { debug!( "Response body from URL {} exceeds url_preview_max_spider_size ({}), not processing the rest of the \ response body and assuming our necessary data is in this range.", url, - services().globals.url_preview_max_spider_size() + services.globals.url_preview_max_spider_size() ); break; } @@ -595,7 +602,7 @@ async fn download_html(client: &reqwest::Client, url: &str) -> Result UrlPreviewData::default(), - Some(obj) => download_image(client, &obj.url).await?, + Some(obj) => download_image(services, client, &obj.url).await?, }; let props = html.opengraph.properties; @@ -607,19 +614,19 @@ async fn download_html(client: &reqwest::Client, url: &str) -> Result Result { +async fn request_url_preview(services: &Services, url: &str) -> Result { if let Ok(ip) = IPAddress::parse(url) { - if !services().globals.valid_cidr_range(&ip) { + if !services.globals.valid_cidr_range(&ip) { return Err(Error::BadServerResponse("Requesting from this address is forbidden")); } } - let client = &services().globals.client.url_preview; + let client = &services.globals.client.url_preview; let response = client.head(url).send().await?; if let Some(remote_addr) = response.remote_addr() { if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { - if !services().globals.valid_cidr_range(&ip) { + if !services.globals.valid_cidr_range(&ip) { return Err(Error::BadServerResponse("Requesting from this address is forbidden")); } } @@ -633,24 +640,24 @@ async fn request_url_preview(url: &str) -> Result { return Err(Error::BadRequest(ErrorKind::Unknown, "Unknown Content-Type")); }; let data = match content_type { - html if html.starts_with("text/html") => download_html(client, url).await?, - img if img.starts_with("image/") => download_image(client, url).await?, + html if html.starts_with("text/html") => download_html(services, client, url).await?, + img if img.starts_with("image/") => download_image(services, client, url).await?, _ => return Err(Error::BadRequest(ErrorKind::Unknown, "Unsupported Content-Type")), }; - services().media.set_url_preview(url, &data).await?; + services.media.set_url_preview(url, &data).await?; Ok(data) } -async fn get_url_preview(url: &str) -> Result { - if let Some(preview) = services().media.get_url_preview(url).await { +async fn get_url_preview(services: &Services, url: &str) -> Result { + if let Some(preview) = services.media.get_url_preview(url).await { return Ok(preview); } // ensure that only one request is made per URL let mutex_request = Arc::clone( - services() + services .media .url_preview_mutex .write() @@ -660,13 +667,13 @@ async fn get_url_preview(url: &str) -> Result { ); let _request_lock = mutex_request.lock().await; - match services().media.get_url_preview(url).await { + match services.media.get_url_preview(url).await { Some(preview) => Ok(preview), - None => request_url_preview(url).await, + None => request_url_preview(services, url).await, } } -fn url_preview_allowed(url_str: &str) -> bool { +fn url_preview_allowed(services: &Services, url_str: &str) -> bool { let url: Url = match Url::parse(url_str) { Ok(u) => u, Err(e) => { @@ -691,10 +698,10 @@ fn url_preview_allowed(url_str: &str) -> bool { Some(h) => h.to_owned(), }; - let allowlist_domain_contains = services().globals.url_preview_domain_contains_allowlist(); - let allowlist_domain_explicit = services().globals.url_preview_domain_explicit_allowlist(); - let denylist_domain_explicit = services().globals.url_preview_domain_explicit_denylist(); - let allowlist_url_contains = services().globals.url_preview_url_contains_allowlist(); + let allowlist_domain_contains = services.globals.url_preview_domain_contains_allowlist(); + let allowlist_domain_explicit = services.globals.url_preview_domain_explicit_allowlist(); + let denylist_domain_explicit = services.globals.url_preview_domain_explicit_denylist(); + let allowlist_url_contains = services.globals.url_preview_url_contains_allowlist(); if allowlist_domain_contains.contains(&"*".to_owned()) || allowlist_domain_explicit.contains(&"*".to_owned()) @@ -735,7 +742,7 @@ fn url_preview_allowed(url_str: &str) -> bool { } // check root domain if available and if user has root domain checks - if services().globals.url_preview_check_root_domain() { + if services.globals.url_preview_check_root_domain() { debug!("Checking root domain"); match host.split_once('.') { None => return false, diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 3adee631..9fde99a4 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -5,6 +5,7 @@ use std::{ time::Instant, }; +use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ debug, debug_warn, error, info, trace, utils, utils::math::continue_exponential_backoff_secs, warn, Error, @@ -43,9 +44,9 @@ use crate::{ pdu::{gen_event_id_canonical_json, PduBuilder}, rooms::state::RoomMutexGuard, sending::convert_to_outgoing_federation_event, - server_is_ours, user_is_local, + server_is_ours, user_is_local, Services, }, - services, Ruma, + Ruma, }; /// Checks if the room is banned in any way possible and the sender user is not @@ -53,14 +54,15 @@ use crate::{ /// /// Performs automatic deactivation if `auto_deactivate_banned_room_attempts` is /// enabled -#[tracing::instrument] +#[tracing::instrument(skip(services))] async fn banned_room_check( - user_id: &UserId, room_id: Option<&RoomId>, server_name: Option<&ServerName>, client_ip: IpAddr, + services: &Services, user_id: &UserId, room_id: Option<&RoomId>, server_name: Option<&ServerName>, + client_ip: IpAddr, ) -> Result<()> { - if !services().users.is_admin(user_id)? { + if !services.users.is_admin(user_id)? { if let Some(room_id) = room_id { - if services().rooms.metadata.is_banned(room_id)? - || services() + if services.rooms.metadata.is_banned(room_id)? + || services .globals .config .forbidden_remote_server_names @@ -71,13 +73,9 @@ async fn banned_room_check( room or banned room server name: {room_id}" ); - if services() - .globals - .config - .auto_deactivate_banned_room_attempts - { + if services.globals.config.auto_deactivate_banned_room_attempts { warn!("Automatically deactivating user {user_id} due to attempted banned room join"); - services() + services .admin .send_message(RoomMessageEventContent::text_plain(format!( "Automatically deactivating user {user_id} due to attempted banned room join from IP \ @@ -85,20 +83,20 @@ async fn banned_room_check( ))) .await; - if let Err(e) = services().users.deactivate_account(user_id) { + if let Err(e) = services.users.deactivate_account(user_id) { warn!(%user_id, %e, "Failed to deactivate account"); } - let all_joined_rooms: Vec = services() + let all_joined_rooms: Vec = services .rooms .state_cache .rooms_joined(user_id) .filter_map(Result::ok) .collect(); - update_displayname(user_id.into(), None, all_joined_rooms.clone()).await?; - update_avatar_url(user_id.into(), None, None, all_joined_rooms).await?; - leave_all_rooms(user_id).await; + update_displayname(services, user_id.into(), None, all_joined_rooms.clone()).await?; + update_avatar_url(services, user_id.into(), None, None, all_joined_rooms).await?; + leave_all_rooms(services, user_id).await; } return Err(Error::BadRequest( @@ -107,7 +105,7 @@ async fn banned_room_check( )); } } else if let Some(server_name) = server_name { - if services() + if services .globals .config .forbidden_remote_server_names @@ -118,13 +116,9 @@ async fn banned_room_check( that is globally forbidden. Rejecting.", ); - if services() - .globals - .config - .auto_deactivate_banned_room_attempts - { + if services.globals.config.auto_deactivate_banned_room_attempts { warn!("Automatically deactivating user {user_id} due to attempted banned room join"); - services() + services .admin .send_message(RoomMessageEventContent::text_plain(format!( "Automatically deactivating user {user_id} due to attempted banned room join from IP \ @@ -132,20 +126,20 @@ async fn banned_room_check( ))) .await; - if let Err(e) = services().users.deactivate_account(user_id) { + if let Err(e) = services.users.deactivate_account(user_id) { warn!(%user_id, %e, "Failed to deactivate account"); } - let all_joined_rooms: Vec = services() + let all_joined_rooms: Vec = services .rooms .state_cache .rooms_joined(user_id) .filter_map(Result::ok) .collect(); - update_displayname(user_id.into(), None, all_joined_rooms.clone()).await?; - update_avatar_url(user_id.into(), None, None, all_joined_rooms).await?; - leave_all_rooms(user_id).await; + update_displayname(services, user_id.into(), None, all_joined_rooms.clone()).await?; + update_avatar_url(services, user_id.into(), None, None, all_joined_rooms).await?; + leave_all_rooms(services, user_id).await; } return Err(Error::BadRequest( @@ -169,14 +163,22 @@ async fn banned_room_check( /// federation #[tracing::instrument(skip_all, fields(%client_ip), name = "join")] pub(crate) async fn join_room_by_id_route( - InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client_ip): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - banned_room_check(sender_user, Some(&body.room_id), body.room_id.server_name(), client_ip).await?; + banned_room_check( + services, + sender_user, + Some(&body.room_id), + body.room_id.server_name(), + client_ip, + ) + .await?; // There is no body.server_name for /roomId/join - let mut servers = services() + let mut servers = services .rooms .state_cache .servers_invite_via(&body.room_id) @@ -184,7 +186,7 @@ pub(crate) async fn join_room_by_id_route( .collect::>(); servers.extend( - services() + services .rooms .state_cache .invite_state(sender_user, &body.room_id)? @@ -202,6 +204,7 @@ pub(crate) async fn join_room_by_id_route( } join_room_by_id_helper( + services, sender_user, &body.room_id, body.reason.clone(), @@ -222,18 +225,19 @@ pub(crate) async fn join_room_by_id_route( /// via room alias server name and room ID server name #[tracing::instrument(skip_all, fields(%client), name = "join")] pub(crate) async fn join_room_by_id_or_alias_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_deref().expect("user is authenticated"); let body = body.body; let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { Ok(room_id) => { - banned_room_check(sender_user, Some(&room_id), room_id.server_name(), client).await?; + banned_room_check(services, sender_user, Some(&room_id), room_id.server_name(), client).await?; let mut servers = body.server_name.clone(); servers.extend( - services() + services .rooms .state_cache .servers_invite_via(&room_id) @@ -241,7 +245,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( ); servers.extend( - services() + services .rooms .state_cache .invite_state(sender_user, &room_id)? @@ -261,21 +265,21 @@ pub(crate) async fn join_room_by_id_or_alias_route( (servers, room_id) }, Err(room_alias) => { - let response = services() + let response = services .rooms .alias .resolve_alias(&room_alias, Some(&body.server_name.clone())) .await?; let (room_id, mut pre_servers) = response; - banned_room_check(sender_user, Some(&room_id), Some(room_alias.server_name()), client).await?; + banned_room_check(services, sender_user, Some(&room_id), Some(room_alias.server_name()), client).await?; let mut servers = body.server_name; if let Some(pre_servers) = &mut pre_servers { servers.append(pre_servers); } servers.extend( - services() + services .rooms .state_cache .servers_invite_via(&room_id) @@ -283,7 +287,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( ); servers.extend( - services() + services .rooms .state_cache .invite_state(sender_user, &room_id)? @@ -301,6 +305,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( }; let join_room_response = join_room_by_id_helper( + services, sender_user, &room_id, body.reason.clone(), @@ -319,10 +324,12 @@ pub(crate) async fn join_room_by_id_or_alias_route( /// Tries to leave the sender user from a room. /// /// - This should always work if the user is currently joined. -pub(crate) async fn leave_room_route(body: Ruma) -> Result { +pub(crate) async fn leave_room_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - leave_room(sender_user, &body.room_id, body.reason.clone()).await?; + leave_room(services, sender_user, &body.room_id, body.reason.clone()).await?; Ok(leave_room::v3::Response::new()) } @@ -332,11 +339,12 @@ pub(crate) async fn leave_room_route(body: Ruma) -> Res /// Tries to send an invite event into the room. #[tracing::instrument(skip_all, fields(%client), name = "invite")] pub(crate) async fn invite_user_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services().users.is_admin(sender_user)? && services().globals.block_non_admin_invites() { + if !services.users.is_admin(sender_user)? && services.globals.block_non_admin_invites() { info!( "User {sender_user} is not an admin and attempted to send an invite to room {}", &body.room_id @@ -347,13 +355,13 @@ pub(crate) async fn invite_user_route( )); } - banned_room_check(sender_user, Some(&body.room_id), body.room_id.server_name(), client).await?; + banned_room_check(services, sender_user, Some(&body.room_id), body.room_id.server_name(), client).await?; if let invite_user::v3::InvitationRecipient::UserId { user_id, } = &body.recipient { - invite_helper(sender_user, user_id, &body.room_id, body.reason.clone(), false).await?; + invite_helper(services, sender_user, user_id, &body.room_id, body.reason.clone(), false).await?; Ok(invite_user::v3::Response {}) } else { Err(Error::BadRequest(ErrorKind::NotFound, "User not found.")) @@ -363,13 +371,15 @@ pub(crate) async fn invite_user_route( /// # `POST /_matrix/client/r0/rooms/{roomId}/kick` /// /// Tries to send a kick event into the room. -pub(crate) async fn kick_user_route(body: Ruma) -> Result { +pub(crate) async fn kick_user_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; let mut event: RoomMemberEventContent = serde_json::from_str( - services() + services .rooms .state_accessor .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref())? @@ -385,7 +395,7 @@ pub(crate) async fn kick_user_route(body: Ruma) -> Resul event.membership = MembershipState::Leave; event.reason.clone_from(&body.reason); - services() + services .rooms .timeline .build_and_append_pdu( @@ -410,12 +420,14 @@ pub(crate) async fn kick_user_route(body: Ruma) -> Resul /// # `POST /_matrix/client/r0/rooms/{roomId}/ban` /// /// Tries to send a ban event into the room. -pub(crate) async fn ban_user_route(body: Ruma) -> Result { +pub(crate) async fn ban_user_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let event = services() + let event = services .rooms .state_accessor .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref())? @@ -426,7 +438,7 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< avatar_url: None, is_direct: None, third_party_invite: None, - blurhash: services().users.blurhash(&body.user_id).unwrap_or_default(), + blurhash: services.users.blurhash(&body.user_id).unwrap_or_default(), reason: body.reason.clone(), join_authorized_via_users_server: None, }), @@ -436,7 +448,7 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< membership: MembershipState::Ban, displayname: None, avatar_url: None, - blurhash: services().users.blurhash(&body.user_id).unwrap_or_default(), + blurhash: services.users.blurhash(&body.user_id).unwrap_or_default(), reason: body.reason.clone(), join_authorized_via_users_server: None, ..event @@ -445,7 +457,7 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< }, )?; - services() + services .rooms .timeline .build_and_append_pdu( @@ -470,13 +482,15 @@ pub(crate) async fn ban_user_route(body: Ruma) -> Result< /// # `POST /_matrix/client/r0/rooms/{roomId}/unban` /// /// Tries to send an unban event into the room. -pub(crate) async fn unban_user_route(body: Ruma) -> Result { +pub(crate) async fn unban_user_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; let mut event: RoomMemberEventContent = serde_json::from_str( - services() + services .rooms .state_accessor .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref())? @@ -490,7 +504,7 @@ pub(crate) async fn unban_user_route(body: Ruma) -> Res event.reason.clone_from(&body.reason); event.join_authorized_via_users_server = None; - services() + services .rooms .timeline .build_and_append_pdu( @@ -521,10 +535,12 @@ pub(crate) async fn unban_user_route(body: Ruma) -> Res /// /// Note: Other devices of the user have no way of knowing the room was /// forgotten, so this has to be called from every device -pub(crate) async fn forget_room_route(body: Ruma) -> Result { +pub(crate) async fn forget_room_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if services() + if services .rooms .state_cache .is_joined(sender_user, &body.room_id)? @@ -535,7 +551,7 @@ pub(crate) async fn forget_room_route(body: Ruma) -> R )); } - services() + services .rooms .state_cache .forget(&body.room_id, sender_user)?; @@ -546,11 +562,13 @@ pub(crate) async fn forget_room_route(body: Ruma) -> R /// # `POST /_matrix/client/r0/joined_rooms` /// /// Lists all rooms the user has joined. -pub(crate) async fn joined_rooms_route(body: Ruma) -> Result { +pub(crate) async fn joined_rooms_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); Ok(joined_rooms::v3::Response { - joined_rooms: services() + joined_rooms: services .rooms .state_cache .rooms_joined(sender_user) @@ -566,11 +584,11 @@ pub(crate) async fn joined_rooms_route(body: Ruma) -> /// /// - Only works if the user is currently joined pub(crate) async fn get_member_events_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services() + if !services .rooms .state_accessor .user_can_see_state_events(sender_user, &body.room_id)? @@ -582,7 +600,7 @@ pub(crate) async fn get_member_events_route( } Ok(get_member_events::v3::Response { - chunk: services() + chunk: services .rooms .state_accessor .room_state_full(&body.room_id) @@ -601,11 +619,11 @@ pub(crate) async fn get_member_events_route( /// - The sender user must be in the room /// - TODO: An appservice just needs a puppet joined pub(crate) async fn joined_members_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services() + if !services .rooms .state_accessor .user_can_see_state_events(sender_user, &body.room_id)? @@ -617,14 +635,14 @@ pub(crate) async fn joined_members_route( } let mut joined = BTreeMap::new(); - for user_id in services() + for user_id in services .rooms .state_cache .room_members(&body.room_id) .filter_map(Result::ok) { - let display_name = services().users.displayname(&user_id)?; - let avatar_url = services().users.avatar_url(&user_id)?; + let display_name = services.users.displayname(&user_id)?; + let avatar_url = services.users.avatar_url(&user_id)?; joined.insert( user_id, @@ -641,46 +659,48 @@ pub(crate) async fn joined_members_route( } pub async fn join_room_by_id_helper( - sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], + services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], third_party_signed: Option<&ThirdPartySigned>, ) -> Result { - let state_lock = services().rooms.state.mutex.lock(room_id).await; + let state_lock = services.rooms.state.mutex.lock(room_id).await; - if matches!(services().rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { + if matches!(services.rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { debug_warn!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { room_id: room_id.into(), }); } - if services() + if services .rooms .state_cache - .server_in_room(services().globals.server_name(), room_id)? + .server_in_room(services.globals.server_name(), room_id)? || servers.is_empty() || (servers.len() == 1 && server_is_ours(&servers[0])) { - join_room_by_id_helper_local(sender_user, room_id, reason, servers, third_party_signed, state_lock).await + join_room_by_id_helper_local(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) + .await } else { // Ask a remote server if we are not participating in this room - join_room_by_id_helper_remote(sender_user, room_id, reason, servers, third_party_signed, state_lock).await + join_room_by_id_helper_remote(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) + .await } } #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote")] async fn join_room_by_id_helper_remote( - sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], + services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, ) -> Result { info!("Joining {room_id} over federation."); - let (make_join_response, remote_server) = make_join_request(sender_user, room_id, servers).await?; + let (make_join_response, remote_server) = make_join_request(services, sender_user, room_id, servers).await?; info!("make_join finished"); let room_version_id = match make_join_response.room_version { Some(room_version) - if services() + if services .globals .supported_room_versions() .contains(&room_version) => @@ -705,7 +725,7 @@ async fn join_room_by_id_helper_remote( // TODO: Is origin needed? join_event_stub.insert( "origin".to_owned(), - CanonicalJsonValue::String(services().globals.server_name().as_str().to_owned()), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), ); join_event_stub.insert( "origin_server_ts".to_owned(), @@ -719,11 +739,11 @@ async fn join_room_by_id_helper_remote( "content".to_owned(), to_canonical_value(RoomMemberEventContent { membership: MembershipState::Join, - displayname: services().users.displayname(sender_user)?, - avatar_url: services().users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user)?, + avatar_url: services.users.avatar_url(sender_user)?, is_direct: None, third_party_invite: None, - blurhash: services().users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user)?, reason, join_authorized_via_users_server: join_authorized_via_users_server.clone(), }) @@ -742,8 +762,8 @@ async fn join_room_by_id_helper_remote( // In order to create a compatible ref hash (EventID) the `hashes` field needs // to be present ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), + services.globals.server_name().as_str(), + services.globals.keypair(), &mut join_event_stub, &room_version_id, ) @@ -764,7 +784,7 @@ async fn join_room_by_id_helper_remote( let mut join_event = join_event_stub; info!("Asking {remote_server} for send_join in room {room_id}"); - let send_join_response = services() + let send_join_response = services .sending .send_federation_request( &remote_server, @@ -852,7 +872,7 @@ async fn join_room_by_id_helper_remote( } } - services().rooms.short.get_or_create_shortroomid(room_id)?; + services.rooms.short.get_or_create_shortroomid(room_id)?; info!("Parsing join event"); let parsed_join_pdu = PduEvent::from_id_val(event_id, join_event.clone()) @@ -862,7 +882,7 @@ async fn join_room_by_id_helper_remote( let pub_key_map = RwLock::new(BTreeMap::new()); info!("Fetching join signing keys"); - services() + services .rooms .event_handler .fetch_join_signing_keys(&send_join_response, &room_version_id, &pub_key_map) @@ -873,7 +893,7 @@ async fn join_room_by_id_helper_remote( .room_state .state .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version_id, &pub_key_map)) + .map(|pdu| validate_and_add_event_id(services, pdu, &room_version_id, &pub_key_map)) { let Ok((event_id, value)) = result.await else { continue; @@ -884,12 +904,9 @@ async fn join_room_by_id_helper_remote( Error::BadServerResponse("Invalid PDU in send_join response.") })?; - services() - .rooms - .outlier - .add_pdu_outlier(&event_id, &value)?; + services.rooms.outlier.add_pdu_outlier(&event_id, &value)?; if let Some(state_key) = &pdu.state_key { - let shortstatekey = services() + let shortstatekey = services .rooms .short .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key)?; @@ -902,16 +919,13 @@ async fn join_room_by_id_helper_remote( .room_state .auth_chain .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version_id, &pub_key_map)) + .map(|pdu| validate_and_add_event_id(services, pdu, &room_version_id, &pub_key_map)) { let Ok((event_id, value)) = result.await else { continue; }; - services() - .rooms - .outlier - .add_pdu_outlier(&event_id, &value)?; + services.rooms.outlier.add_pdu_outlier(&event_id, &value)?; } debug!("Running send_join auth check"); @@ -921,12 +935,12 @@ async fn join_room_by_id_helper_remote( &parsed_join_pdu, None::, // TODO: third party invite |k, s| { - services() + services .rooms .timeline .get_pdu( state.get( - &services() + &services .rooms .short .get_or_create_shortstatekey(&k.to_string().into(), s) @@ -946,37 +960,32 @@ async fn join_room_by_id_helper_remote( } info!("Saving state from send_join"); - let (statehash_before_join, new, removed) = services().rooms.state_compressor.save_state( + let (statehash_before_join, new, removed) = services.rooms.state_compressor.save_state( room_id, Arc::new( state .into_iter() - .map(|(k, id)| { - services() - .rooms - .state_compressor - .compress_state_event(k, &id) - }) + .map(|(k, id)| services.rooms.state_compressor.compress_state_event(k, &id)) .collect::>()?, ), )?; - services() + services .rooms .state .force_state(room_id, statehash_before_join, new, removed, &state_lock) .await?; info!("Updating joined counts for new room"); - services().rooms.state_cache.update_joined_count(room_id)?; + services.rooms.state_cache.update_joined_count(room_id)?; // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't // fail. - let statehash_after_join = services().rooms.state.append_to_state(&parsed_join_pdu)?; + let statehash_after_join = services.rooms.state.append_to_state(&parsed_join_pdu)?; info!("Appending new room join event"); - services() + services .rooms .timeline .append_pdu( @@ -990,7 +999,7 @@ async fn join_room_by_id_helper_remote( info!("Setting final room state for new room"); // We set the room state after inserting the pdu, so that we never have a moment // in time where events in the current room state do not exist - services() + services .rooms .state .set_room_state(room_id, statehash_after_join, &state_lock)?; @@ -1000,16 +1009,15 @@ async fn join_room_by_id_helper_remote( #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_local")] async fn join_room_by_id_helper_local( - sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], + services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, ) -> Result { debug!("We can join locally"); - let join_rules_event = - services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomJoinRules, "")?; + let join_rules_event = services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomJoinRules, "")?; let join_rules_event_content: Option = join_rules_event .as_ref() @@ -1035,7 +1043,7 @@ async fn join_room_by_id_helper_local( _ => Vec::new(), }; - let local_members = services() + let local_members = services .rooms .state_cache .room_members(room_id) @@ -1046,14 +1054,14 @@ async fn join_room_by_id_helper_local( let mut join_authorized_via_users_server: Option = None; if restriction_rooms.iter().any(|restriction_room_id| { - services() + services .rooms .state_cache .is_joined(sender_user, restriction_room_id) .unwrap_or(false) }) { for user in local_members { - if services() + if services .rooms .state_accessor .user_can_invite(room_id, &user, sender_user, &state_lock) @@ -1067,17 +1075,17 @@ async fn join_room_by_id_helper_local( let event = RoomMemberEventContent { membership: MembershipState::Join, - displayname: services().users.displayname(sender_user)?, - avatar_url: services().users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user)?, + avatar_url: services.users.avatar_url(sender_user)?, is_direct: None, third_party_invite: None, - blurhash: services().users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user)?, reason: reason.clone(), join_authorized_via_users_server, }; // Try normal join first - let error = match services() + let error = match services .rooms .timeline .build_and_append_pdu( @@ -1104,11 +1112,11 @@ async fn join_room_by_id_helper_local( .any(|server_name| !server_is_ours(server_name)) { warn!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); - let (make_join_response, remote_server) = make_join_request(sender_user, room_id, servers).await?; + let (make_join_response, remote_server) = make_join_request(services, sender_user, room_id, servers).await?; let room_version_id = match make_join_response.room_version { Some(room_version_id) - if services() + if services .globals .supported_room_versions() .contains(&room_version_id) => @@ -1130,7 +1138,7 @@ async fn join_room_by_id_helper_local( // TODO: Is origin needed? join_event_stub.insert( "origin".to_owned(), - CanonicalJsonValue::String(services().globals.server_name().as_str().to_owned()), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), ); join_event_stub.insert( "origin_server_ts".to_owned(), @@ -1144,11 +1152,11 @@ async fn join_room_by_id_helper_local( "content".to_owned(), to_canonical_value(RoomMemberEventContent { membership: MembershipState::Join, - displayname: services().users.displayname(sender_user)?, - avatar_url: services().users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user)?, + avatar_url: services.users.avatar_url(sender_user)?, is_direct: None, third_party_invite: None, - blurhash: services().users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user)?, reason, join_authorized_via_users_server, }) @@ -1167,8 +1175,8 @@ async fn join_room_by_id_helper_local( // In order to create a compatible ref hash (EventID) the `hashes` field needs // to be present ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), + services.globals.server_name().as_str(), + services.globals.keypair(), &mut join_event_stub, &room_version_id, ) @@ -1188,7 +1196,7 @@ async fn join_room_by_id_helper_local( // It has enough fields to be called a proper event now let join_event = join_event_stub; - let send_join_response = services() + let send_join_response = services .sending .send_federation_request( &remote_server, @@ -1219,12 +1227,12 @@ async fn join_room_by_id_helper_local( drop(state_lock); let pub_key_map = RwLock::new(BTreeMap::new()); - services() + services .rooms .event_handler .fetch_required_signing_keys([&signed_value], &pub_key_map) .await?; - services() + services .rooms .event_handler .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true, &pub_key_map) @@ -1240,7 +1248,7 @@ async fn join_room_by_id_helper_local( } async fn make_join_request( - sender_user: &UserId, room_id: &RoomId, servers: &[OwnedServerName], + services: &Services, sender_user: &UserId, room_id: &RoomId, servers: &[OwnedServerName], ) -> Result<(federation::membership::prepare_join_event::v1::Response, OwnedServerName)> { let mut make_join_response_and_server = Err(Error::BadServerResponse("No server available to assist in joining.")); @@ -1252,14 +1260,14 @@ async fn make_join_request( continue; } info!("Asking {remote_server} for make_join ({make_join_counter})"); - let make_join_response = services() + let make_join_response = services .sending .send_federation_request( remote_server, federation::membership::prepare_join_event::v1::Request { room_id: room_id.to_owned(), user_id: sender_user.to_owned(), - ver: services().globals.supported_room_versions(), + ver: services.globals.supported_room_versions(), }, ) .await; @@ -1309,7 +1317,8 @@ async fn make_join_request( } pub async fn validate_and_add_event_id( - pdu: &RawJsonValue, room_version: &RoomVersionId, pub_key_map: &RwLock>>, + services: &Services, pdu: &RawJsonValue, room_version: &RoomVersionId, + pub_key_map: &RwLock>>, ) -> Result<(OwnedEventId, CanonicalJsonObject)> { let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); @@ -1322,7 +1331,7 @@ pub async fn validate_and_add_event_id( .expect("ruma's reference hashes are valid event ids"); let back_off = |id| async { - match services() + match services .globals .bad_event_ratelimiter .write() @@ -1338,7 +1347,7 @@ pub async fn validate_and_add_event_id( } }; - if let Some((time, tries)) = services() + if let Some((time, tries)) = services .globals .bad_event_ratelimiter .read() @@ -1366,9 +1375,10 @@ pub async fn validate_and_add_event_id( } pub(crate) async fn invite_helper( - sender_user: &UserId, user_id: &UserId, room_id: &RoomId, reason: Option, is_direct: bool, + services: &Services, sender_user: &UserId, user_id: &UserId, room_id: &RoomId, reason: Option, + is_direct: bool, ) -> Result<()> { - if !services().users.is_admin(user_id)? && services().globals.block_non_admin_invites() { + if !services.users.is_admin(user_id)? && services.globals.block_non_admin_invites() { info!("User {sender_user} is not an admin and attempted to send an invite to room {room_id}"); return Err(Error::BadRequest( ErrorKind::forbidden(), @@ -1378,9 +1388,9 @@ pub(crate) async fn invite_helper( if !user_is_local(user_id) { let (pdu, pdu_json, invite_room_state) = { - let state_lock = services().rooms.state.mutex.lock(room_id).await; + let state_lock = services.rooms.state.mutex.lock(room_id).await; let content = to_raw_value(&RoomMemberEventContent { - avatar_url: services().users.avatar_url(user_id)?, + avatar_url: services.users.avatar_url(user_id)?, displayname: None, is_direct: Some(is_direct), membership: MembershipState::Invite, @@ -1391,7 +1401,7 @@ pub(crate) async fn invite_helper( }) .expect("member event is valid value"); - let (pdu, pdu_json) = services().rooms.timeline.create_hash_and_sign_event( + let (pdu, pdu_json) = services.rooms.timeline.create_hash_and_sign_event( PduBuilder { event_type: TimelineEventType::RoomMember, content, @@ -1404,16 +1414,16 @@ pub(crate) async fn invite_helper( &state_lock, )?; - let invite_room_state = services().rooms.state.calculate_invite_state(&pdu)?; + let invite_room_state = services.rooms.state.calculate_invite_state(&pdu)?; drop(state_lock); (pdu, pdu_json, invite_room_state) }; - let room_version_id = services().rooms.state.get_room_version(room_id)?; + let room_version_id = services.rooms.state.get_room_version(room_id)?; - let response = services() + let response = services .sending .send_federation_request( user_id.server_name(), @@ -1423,7 +1433,7 @@ pub(crate) async fn invite_helper( room_version: room_version_id.clone(), event: convert_to_outgoing_federation_event(pdu_json.clone()), invite_room_state, - via: services().rooms.state_cache.servers_route_via(room_id).ok(), + via: services.rooms.state_cache.servers_route_via(room_id).ok(), }, ) .await?; @@ -1459,13 +1469,13 @@ pub(crate) async fn invite_helper( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Origin field is invalid."))?; - services() + services .rooms .event_handler .fetch_required_signing_keys([&value], &pub_key_map) .await?; - let pdu_id: Vec = services() + let pdu_id: Vec = services .rooms .event_handler .handle_incoming_pdu(&origin, room_id, &event_id, value, true, &pub_key_map) @@ -1475,24 +1485,20 @@ pub(crate) async fn invite_helper( "Could not accept incoming PDU as timeline event.", ))?; - services().sending.send_pdu_room(room_id, &pdu_id)?; + services.sending.send_pdu_room(room_id, &pdu_id)?; return Ok(()); } - if !services() - .rooms - .state_cache - .is_joined(sender_user, room_id)? - { + if !services.rooms.state_cache.is_joined(sender_user, room_id)? { return Err(Error::BadRequest( ErrorKind::forbidden(), "You don't have permission to view this room.", )); } - let state_lock = services().rooms.state.mutex.lock(room_id).await; + let state_lock = services.rooms.state.mutex.lock(room_id).await; - services() + services .rooms .timeline .build_and_append_pdu( @@ -1500,11 +1506,11 @@ pub(crate) async fn invite_helper( event_type: TimelineEventType::RoomMember, content: to_raw_value(&RoomMemberEventContent { membership: MembershipState::Invite, - displayname: services().users.displayname(user_id)?, - avatar_url: services().users.avatar_url(user_id)?, + displayname: services.users.displayname(user_id)?, + avatar_url: services.users.avatar_url(user_id)?, is_direct: Some(is_direct), third_party_invite: None, - blurhash: services().users.blurhash(user_id)?, + blurhash: services.users.blurhash(user_id)?, reason, join_authorized_via_users_server: None, }) @@ -1526,13 +1532,13 @@ pub(crate) async fn invite_helper( // Make a user leave all their joined rooms, forgets all rooms, and ignores // errors -pub async fn leave_all_rooms(user_id: &UserId) { - let all_rooms = services() +pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { + let all_rooms = services .rooms .state_cache .rooms_joined(user_id) .chain( - services() + services .rooms .state_cache .rooms_invited(user_id) @@ -1546,35 +1552,35 @@ pub async fn leave_all_rooms(user_id: &UserId) { }; // ignore errors - if let Err(e) = leave_room(user_id, &room_id, None).await { + if let Err(e) = leave_room(services, user_id, &room_id, None).await { warn!(%room_id, %user_id, %e, "Failed to leave room"); } - if let Err(e) = services().rooms.state_cache.forget(&room_id, user_id) { + if let Err(e) = services.rooms.state_cache.forget(&room_id, user_id) { warn!(%room_id, %user_id, %e, "Failed to forget room"); } } } -pub async fn leave_room(user_id: &UserId, room_id: &RoomId, reason: Option) -> Result<()> { +pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, reason: Option) -> Result<()> { // Ask a remote server if we don't have this room - if !services() + if !services .rooms .state_cache - .server_in_room(services().globals.server_name(), room_id)? + .server_in_room(services.globals.server_name(), room_id)? { - if let Err(e) = remote_leave_room(user_id, room_id).await { + if let Err(e) = remote_leave_room(services, user_id, room_id).await { warn!("Failed to leave room {} remotely: {}", user_id, e); // Don't tell the client about this error } - let last_state = services() + let last_state = services .rooms .state_cache .invite_state(user_id, room_id)? - .map_or_else(|| services().rooms.state_cache.left_state(user_id, room_id), |s| Ok(Some(s)))?; + .map_or_else(|| services.rooms.state_cache.left_state(user_id, room_id), |s| Ok(Some(s)))?; // We always drop the invite, we can't rely on other servers - services().rooms.state_cache.update_membership( + services.rooms.state_cache.update_membership( room_id, user_id, RoomMemberEventContent::new(MembershipState::Leave), @@ -1584,10 +1590,10 @@ pub async fn leave_room(user_id: &UserId, room_id: &RoomId, reason: Option { error!("Trying to leave a room you are not a member of."); - services().rooms.state_cache.update_membership( + services.rooms.state_cache.update_membership( room_id, user_id, RoomMemberEventContent::new(MembershipState::Leave), @@ -1619,7 +1625,7 @@ pub async fn leave_room(user_id: &UserId, room_id: &RoomId, reason: Option Result<()> { +async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result<()> { let mut make_leave_response_and_server = Err(Error::BadServerResponse("No server available to assist in leaving.")); - let invite_state = services() + let invite_state = services .rooms .state_cache .invite_state(user_id, room_id)? .ok_or(Error::BadRequest(ErrorKind::BadState, "User is not invited."))?; - let mut servers: HashSet = services() + let mut servers: HashSet = services .rooms .state_cache .servers_invite_via(room_id) @@ -1669,7 +1675,7 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { debug!("servers in remote_leave_room: {servers:?}"); for remote_server in servers { - let make_leave_response = services() + let make_leave_response = services .sending .send_federation_request( &remote_server, @@ -1691,7 +1697,7 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { let room_version_id = match make_leave_response.room_version { Some(version) - if services() + if services .globals .supported_room_versions() .contains(&version) => @@ -1707,7 +1713,7 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { // TODO: Is origin needed? leave_event_stub.insert( "origin".to_owned(), - CanonicalJsonValue::String(services().globals.server_name().as_str().to_owned()), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), ); leave_event_stub.insert( "origin_server_ts".to_owned(), @@ -1729,8 +1735,8 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { // In order to create a compatible ref hash (EventID) the `hashes` field needs // to be present ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), + services.globals.server_name().as_str(), + services.globals.keypair(), &mut leave_event_stub, &room_version_id, ) @@ -1750,7 +1756,7 @@ async fn remote_leave_room(user_id: &UserId, room_id: &RoomId) -> Result<()> { // It has enough fields to be called a proper event now let leave_event = leave_event_stub; - services() + services .sending .send_federation_request( &remote_server, diff --git a/src/api/client/message.rs b/src/api/client/message.rs index c376ee52..c0b5cf0c 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,5 +1,6 @@ use std::collections::{BTreeMap, HashSet}; +use axum::extract::State; use conduit::PduCount; use ruma::{ api::client::{ @@ -12,7 +13,10 @@ use ruma::{ }; use serde_json::{from_str, Value}; -use crate::{service::pdu::PduBuilder, services, utils, Error, PduEvent, Result, Ruma}; +use crate::{ + service::{pdu::PduBuilder, Services}, + utils, Error, PduEvent, Result, Ruma, +}; /// # `PUT /_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}` /// @@ -24,21 +28,19 @@ use crate::{service::pdu::PduBuilder, services, utils, Error, PduEvent, Result, /// - Tries to send the event into the room, auth rules will determine if it is /// allowed pub(crate) async fn send_message_event_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_deref(); - let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; // Forbid m.room.encrypted if encryption is disabled - if MessageLikeEventType::RoomEncrypted == body.event_type && !services().globals.allow_encryption() { + if MessageLikeEventType::RoomEncrypted == body.event_type && !services.globals.allow_encryption() { return Err(Error::BadRequest(ErrorKind::forbidden(), "Encryption has been disabled")); } - if body.event_type == MessageLikeEventType::CallInvite - && services().rooms.directory.is_public_room(&body.room_id)? - { + if body.event_type == MessageLikeEventType::CallInvite && services.rooms.directory.is_public_room(&body.room_id)? { return Err(Error::BadRequest( ErrorKind::forbidden(), "Room call invites are not allowed in public rooms", @@ -46,7 +48,7 @@ pub(crate) async fn send_message_event_route( } // Check if this is a new transaction id - if let Some(response) = services() + if let Some(response) = services .transaction_ids .existing_txnid(sender_user, sender_device, &body.txn_id)? { @@ -71,7 +73,7 @@ pub(crate) async fn send_message_event_route( let mut unsigned = BTreeMap::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.to_string().into()); - let event_id = services() + let event_id = services .rooms .timeline .build_and_append_pdu( @@ -89,7 +91,7 @@ pub(crate) async fn send_message_event_route( ) .await?; - services() + services .transaction_ids .add_txnid(sender_user, sender_device, &body.txn_id, event_id.as_bytes())?; @@ -105,7 +107,7 @@ pub(crate) async fn send_message_event_route( /// - Only works if the user is joined (TODO: always allow, but only show events /// where the user was joined, depending on `history_visibility`) pub(crate) async fn get_message_events_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -123,7 +125,7 @@ pub(crate) async fn get_message_events_route( .as_ref() .and_then(|t| PduCount::try_from_string(t).ok()); - services() + services .rooms .lazy_loading .lazy_load_confirm_delivery(sender_user, sender_device, &body.room_id, from) @@ -139,12 +141,12 @@ pub(crate) async fn get_message_events_route( match body.dir { ruma::api::Direction::Forward => { - let events_after: Vec<_> = services() + let events_after: Vec<_> = services .rooms .timeline .pdus_after(sender_user, &body.room_id, from)? .filter_map(Result::ok) // Filter out buggy events - .filter(|(_, pdu)| { contains_url_filter(pdu, &body.filter) && visibility_filter(pdu, sender_user, &body.room_id) + .filter(|(_, pdu)| { contains_url_filter(pdu, &body.filter) && visibility_filter(services, pdu, sender_user, &body.room_id) }) .take_while(|&(k, _)| Some(k) != to) // Stop at `to` @@ -157,7 +159,7 @@ pub(crate) async fn get_message_events_route( * https://github.com/vector-im/element-web/issues/21034 */ if !cfg!(feature = "element_hacks") - && !services().rooms.lazy_loading.lazy_load_was_sent_before( + && !services.rooms.lazy_loading.lazy_load_was_sent_before( sender_user, sender_device, &body.room_id, @@ -181,17 +183,17 @@ pub(crate) async fn get_message_events_route( resp.chunk = events_after; }, ruma::api::Direction::Backward => { - services() + services .rooms .timeline .backfill_if_required(&body.room_id, from) .await?; - let events_before: Vec<_> = services() + let events_before: Vec<_> = services .rooms .timeline .pdus_until(sender_user, &body.room_id, from)? .filter_map(Result::ok) // Filter out buggy events - .filter(|(_, pdu)| {contains_url_filter(pdu, &body.filter) && visibility_filter(pdu, sender_user, &body.room_id)}) + .filter(|(_, pdu)| {contains_url_filter(pdu, &body.filter) && visibility_filter(services, pdu, sender_user, &body.room_id)}) .take_while(|&(k, _)| Some(k) != to) // Stop at `to` .take(limit) .collect(); @@ -202,7 +204,7 @@ pub(crate) async fn get_message_events_route( * https://github.com/vector-im/element-web/issues/21034 */ if !cfg!(feature = "element_hacks") - && !services().rooms.lazy_loading.lazy_load_was_sent_before( + && !services.rooms.lazy_loading.lazy_load_was_sent_before( sender_user, sender_device, &body.room_id, @@ -229,11 +231,12 @@ pub(crate) async fn get_message_events_route( resp.state = Vec::new(); for ll_id in &lazy_loaded { - if let Some(member_event) = services().rooms.state_accessor.room_state_get( - &body.room_id, - &StateEventType::RoomMember, - ll_id.as_str(), - )? { + if let Some(member_event) = + services + .rooms + .state_accessor + .room_state_get(&body.room_id, &StateEventType::RoomMember, ll_id.as_str())? + { resp.state.push(member_event.to_state_event()); } } @@ -241,7 +244,7 @@ pub(crate) async fn get_message_events_route( // remove the feature check when we are sure clients like element can handle it if !cfg!(feature = "element_hacks") { if let Some(next_token) = next_token { - services() + services .rooms .lazy_loading .lazy_load_mark_sent(sender_user, sender_device, &body.room_id, lazy_loaded, next_token) @@ -252,8 +255,8 @@ pub(crate) async fn get_message_events_route( Ok(resp) } -fn visibility_filter(pdu: &PduEvent, user_id: &UserId, room_id: &RoomId) -> bool { - services() +fn visibility_filter(services: &Services, pdu: &PduEvent, user_id: &UserId, room_id: &RoomId) -> bool { + services .rooms .state_accessor .user_can_see_event(user_id, room_id, &pdu.event_id) diff --git a/src/api/client/openid.rs b/src/api/client/openid.rs index a1932052..3e4c6ca8 100644 --- a/src/api/client/openid.rs +++ b/src/api/client/openid.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use axum::extract::State; use conduit::utils; use ruma::{ api::client::{account, error::ErrorKind}, @@ -7,7 +8,7 @@ use ruma::{ }; use super::TOKEN_LENGTH; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `POST /_matrix/client/v3/user/{userId}/openid/request_token` /// @@ -15,7 +16,7 @@ use crate::{services, Error, Result, Ruma}; /// /// - The token generated is only valid for the OpenID API pub(crate) async fn create_openid_token_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -28,14 +29,14 @@ pub(crate) async fn create_openid_token_route( let access_token = utils::random_string(TOKEN_LENGTH); - let expires_in = services() + let expires_in = services .users .create_openid_token(&body.user_id, &access_token)?; Ok(account::request_openid_token::v3::Response { access_token, token_type: TokenType::Bearer, - matrix_server_name: services().globals.config.server_name.clone(), + matrix_server_name: services.globals.config.server_name.clone(), expires_in: Duration::from_secs(expires_in), }) } diff --git a/src/api/client/presence.rs b/src/api/client/presence.rs index 775f82ec..8384d5ac 100644 --- a/src/api/client/presence.rs +++ b/src/api/client/presence.rs @@ -1,22 +1,24 @@ use std::time::Duration; +use axum::extract::State; use ruma::api::client::{ error::ErrorKind, presence::{get_presence, set_presence}, }; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `PUT /_matrix/client/r0/presence/{userId}/status` /// /// Sets the presence state of the sender user. -pub(crate) async fn set_presence_route(body: Ruma) -> Result { - if !services().globals.allow_local_presence() { +pub(crate) async fn set_presence_route( + State(services): State, body: Ruma, +) -> Result { + if !services.globals.allow_local_presence() { return Err(Error::BadRequest(ErrorKind::forbidden(), "Presence is disabled on this server")); } let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if sender_user != &body.user_id && body.appservice_info.is_none() { return Err(Error::BadRequest( ErrorKind::InvalidParam, @@ -24,7 +26,7 @@ pub(crate) async fn set_presence_route(body: Ruma) -> )); } - services() + services .presence .set_presence(sender_user, &body.presence, None, None, body.status_msg.clone())?; @@ -36,8 +38,10 @@ pub(crate) async fn set_presence_route(body: Ruma) -> /// Gets the presence state of the given user. /// /// - Only works if you share a room with the user -pub(crate) async fn get_presence_route(body: Ruma) -> Result { - if !services().globals.allow_local_presence() { +pub(crate) async fn get_presence_route( + State(services): State, body: Ruma, +) -> Result { + if !services.globals.allow_local_presence() { return Err(Error::BadRequest(ErrorKind::forbidden(), "Presence is disabled on this server")); } @@ -45,12 +49,12 @@ pub(crate) async fn get_presence_route(body: Ruma) -> let mut presence_event = None; - for _room_id in services() + for _room_id in services .rooms .user .get_shared_rooms(vec![sender_user.clone(), body.user_id.clone()])? { - if let Some(presence) = services().presence.get_presence(&body.user_id)? { + if let Some(presence) = services.presence.get_presence(&body.user_id)? { presence_event = Some(presence); break; } diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index b0305939..3b2c32ec 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use ruma::{ api::{ client::{ @@ -14,8 +15,8 @@ use serde_json::value::to_raw_value; use tracing::warn; use crate::{ - service::{pdu::PduBuilder, user_is_local}, - services, Error, Result, Ruma, + service::{pdu::PduBuilder, user_is_local, Services}, + Error, Result, Ruma, }; /// # `PUT /_matrix/client/r0/profile/{userId}/displayname` @@ -24,21 +25,21 @@ use crate::{ /// /// - Also makes sure other users receive the update using presence EDUs pub(crate) async fn set_displayname_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let all_joined_rooms: Vec = services() + let all_joined_rooms: Vec = services .rooms .state_cache .rooms_joined(sender_user) .filter_map(Result::ok) .collect(); - update_displayname(sender_user.clone(), body.displayname.clone(), all_joined_rooms).await?; + update_displayname(services, sender_user.clone(), body.displayname.clone(), all_joined_rooms).await?; - if services().globals.allow_local_presence() { + if services.globals.allow_local_presence() { // Presence update - services() + services .presence .ping_presence(sender_user, &PresenceState::Online)?; } @@ -53,11 +54,11 @@ pub(crate) async fn set_displayname_route( /// - If user is on another server and we do not have a local copy already fetch /// displayname over federation pub(crate) async fn get_displayname_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { if !user_is_local(&body.user_id) { // Create and update our local copy of the user - if let Ok(response) = services() + if let Ok(response) = services .sending .send_federation_request( body.user_id.server_name(), @@ -68,19 +69,19 @@ pub(crate) async fn get_displayname_route( ) .await { - if !services().users.exists(&body.user_id)? { - services().users.create(&body.user_id, None)?; + if !services.users.exists(&body.user_id)? { + services.users.create(&body.user_id, None)?; } - services() + services .users .set_displayname(&body.user_id, response.displayname.clone()) .await?; - services() + services .users .set_avatar_url(&body.user_id, response.avatar_url.clone()) .await?; - services() + services .users .set_blurhash(&body.user_id, response.blurhash.clone()) .await?; @@ -91,14 +92,14 @@ pub(crate) async fn get_displayname_route( } } - if !services().users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id)? { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); } Ok(get_display_name::v3::Response { - displayname: services().users.displayname(&body.user_id)?, + displayname: services.users.displayname(&body.user_id)?, }) } @@ -108,10 +109,10 @@ pub(crate) async fn get_displayname_route( /// /// - Also makes sure other users receive the update using presence EDUs pub(crate) async fn set_avatar_url_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let all_joined_rooms: Vec = services() + let all_joined_rooms: Vec = services .rooms .state_cache .rooms_joined(sender_user) @@ -119,6 +120,7 @@ pub(crate) async fn set_avatar_url_route( .collect(); update_avatar_url( + services, sender_user.clone(), body.avatar_url.clone(), body.blurhash.clone(), @@ -126,9 +128,9 @@ pub(crate) async fn set_avatar_url_route( ) .await?; - if services().globals.allow_local_presence() { + if services.globals.allow_local_presence() { // Presence update - services() + services .presence .ping_presence(sender_user, &PresenceState::Online)?; } @@ -143,11 +145,11 @@ pub(crate) async fn set_avatar_url_route( /// - If user is on another server and we do not have a local copy already fetch /// `avatar_url` and blurhash over federation pub(crate) async fn get_avatar_url_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { if !user_is_local(&body.user_id) { // Create and update our local copy of the user - if let Ok(response) = services() + if let Ok(response) = services .sending .send_federation_request( body.user_id.server_name(), @@ -158,19 +160,19 @@ pub(crate) async fn get_avatar_url_route( ) .await { - if !services().users.exists(&body.user_id)? { - services().users.create(&body.user_id, None)?; + if !services.users.exists(&body.user_id)? { + services.users.create(&body.user_id, None)?; } - services() + services .users .set_displayname(&body.user_id, response.displayname.clone()) .await?; - services() + services .users .set_avatar_url(&body.user_id, response.avatar_url.clone()) .await?; - services() + services .users .set_blurhash(&body.user_id, response.blurhash.clone()) .await?; @@ -182,15 +184,15 @@ pub(crate) async fn get_avatar_url_route( } } - if !services().users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id)? { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); } Ok(get_avatar_url::v3::Response { - avatar_url: services().users.avatar_url(&body.user_id)?, - blurhash: services().users.blurhash(&body.user_id)?, + avatar_url: services.users.avatar_url(&body.user_id)?, + blurhash: services.users.blurhash(&body.user_id)?, }) } @@ -200,10 +202,12 @@ pub(crate) async fn get_avatar_url_route( /// /// - If user is on another server and we do not have a local copy already, /// fetch profile over federation. -pub(crate) async fn get_profile_route(body: Ruma) -> Result { +pub(crate) async fn get_profile_route( + State(services): State, body: Ruma, +) -> Result { if !user_is_local(&body.user_id) { // Create and update our local copy of the user - if let Ok(response) = services() + if let Ok(response) = services .sending .send_federation_request( body.user_id.server_name(), @@ -214,19 +218,19 @@ pub(crate) async fn get_profile_route(body: Ruma) -> R ) .await { - if !services().users.exists(&body.user_id)? { - services().users.create(&body.user_id, None)?; + if !services.users.exists(&body.user_id)? { + services.users.create(&body.user_id, None)?; } - services() + services .users .set_displayname(&body.user_id, response.displayname.clone()) .await?; - services() + services .users .set_avatar_url(&body.user_id, response.avatar_url.clone()) .await?; - services() + services .users .set_blurhash(&body.user_id, response.blurhash.clone()) .await?; @@ -239,23 +243,23 @@ pub(crate) async fn get_profile_route(body: Ruma) -> R } } - if !services().users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id)? { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); } Ok(get_profile::v3::Response { - avatar_url: services().users.avatar_url(&body.user_id)?, - blurhash: services().users.blurhash(&body.user_id)?, - displayname: services().users.displayname(&body.user_id)?, + avatar_url: services.users.avatar_url(&body.user_id)?, + blurhash: services.users.blurhash(&body.user_id)?, + displayname: services.users.displayname(&body.user_id)?, }) } pub async fn update_displayname( - user_id: OwnedUserId, displayname: Option, all_joined_rooms: Vec, + services: &Services, user_id: OwnedUserId, displayname: Option, all_joined_rooms: Vec, ) -> Result<()> { - services() + services .users .set_displayname(&user_id, displayname.clone()) .await?; @@ -271,7 +275,7 @@ pub async fn update_displayname( displayname: displayname.clone(), join_authorized_via_users_server: None, ..serde_json::from_str( - services() + services .rooms .state_accessor .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())? @@ -294,19 +298,20 @@ pub async fn update_displayname( .filter_map(Result::ok) .collect(); - update_all_rooms(all_joined_rooms, user_id).await; + update_all_rooms(services, all_joined_rooms, user_id).await; Ok(()) } pub async fn update_avatar_url( - user_id: OwnedUserId, avatar_url: Option, blurhash: Option, all_joined_rooms: Vec, + services: &Services, user_id: OwnedUserId, avatar_url: Option, blurhash: Option, + all_joined_rooms: Vec, ) -> Result<()> { - services() + services .users .set_avatar_url(&user_id, avatar_url.clone()) .await?; - services() + services .users .set_blurhash(&user_id, blurhash.clone()) .await?; @@ -323,7 +328,7 @@ pub async fn update_avatar_url( blurhash: blurhash.clone(), join_authorized_via_users_server: None, ..serde_json::from_str( - services() + services .rooms .state_accessor .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())? @@ -346,15 +351,17 @@ pub async fn update_avatar_url( .filter_map(Result::ok) .collect(); - update_all_rooms(all_joined_rooms, user_id).await; + update_all_rooms(services, all_joined_rooms, user_id).await; Ok(()) } -pub async fn update_all_rooms(all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: OwnedUserId) { +pub async fn update_all_rooms( + services: &Services, all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: OwnedUserId, +) { for (pdu_builder, room_id) in all_joined_rooms { - let state_lock = services().rooms.state.mutex.lock(room_id).await; - if let Err(e) = services() + let state_lock = services.rooms.state.mutex.lock(room_id).await; + if let Err(e) = services .rooms .timeline .build_and_append_pdu(pdu_builder, &user_id, room_id, &state_lock) diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 5d7fd630..26462e79 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use ruma::{ api::client::{ error::ErrorKind, @@ -10,18 +11,18 @@ use ruma::{ push::{InsertPushRuleError, RemovePushRuleError, Ruleset}, }; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `GET /_matrix/client/r0/pushrules/` /// /// Retrieves the push rules event for this user. pub(crate) async fn get_pushrules_all_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let event = - services() + services .account_data .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())?; @@ -34,7 +35,7 @@ pub(crate) async fn get_pushrules_all_route( global: account_data.global, }) } else { - services().account_data.update( + services.account_data.update( None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into(), @@ -55,10 +56,12 @@ pub(crate) async fn get_pushrules_all_route( /// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}` /// /// Retrieves a single specified push rule for this user. -pub(crate) async fn get_pushrule_route(body: Ruma) -> Result { +pub(crate) async fn get_pushrule_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = services() + let event = services .account_data .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; @@ -84,7 +87,9 @@ pub(crate) async fn get_pushrule_route(body: Ruma) -> /// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}` /// /// Creates a single specified push rule for this user. -pub(crate) async fn set_pushrule_route(body: Ruma) -> Result { +pub(crate) async fn set_pushrule_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let body = body.body; @@ -95,7 +100,7 @@ pub(crate) async fn set_pushrule_route(body: Ruma) -> )); } - let event = services() + let event = services .account_data .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; @@ -134,7 +139,7 @@ pub(crate) async fn set_pushrule_route(body: Ruma) -> return Err(err); } - services().account_data.update( + services.account_data.update( None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into(), @@ -148,7 +153,7 @@ pub(crate) async fn set_pushrule_route(body: Ruma) -> /// /// Gets the actions of a single specified push rule for this user. pub(crate) async fn get_pushrule_actions_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -159,7 +164,7 @@ pub(crate) async fn get_pushrule_actions_route( )); } - let event = services() + let event = services .account_data .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; @@ -183,7 +188,7 @@ pub(crate) async fn get_pushrule_actions_route( /// /// Sets the actions of a single specified push rule for this user. pub(crate) async fn set_pushrule_actions_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -194,7 +199,7 @@ pub(crate) async fn set_pushrule_actions_route( )); } - let event = services() + let event = services .account_data .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; @@ -211,7 +216,7 @@ pub(crate) async fn set_pushrule_actions_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")); } - services().account_data.update( + services.account_data.update( None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into(), @@ -225,7 +230,7 @@ pub(crate) async fn set_pushrule_actions_route( /// /// Gets the enabled status of a single specified push rule for this user. pub(crate) async fn get_pushrule_enabled_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -236,7 +241,7 @@ pub(crate) async fn get_pushrule_enabled_route( )); } - let event = services() + let event = services .account_data .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; @@ -259,7 +264,7 @@ pub(crate) async fn get_pushrule_enabled_route( /// /// Sets the enabled status of a single specified push rule for this user. pub(crate) async fn set_pushrule_enabled_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -270,7 +275,7 @@ pub(crate) async fn set_pushrule_enabled_route( )); } - let event = services() + let event = services .account_data .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; @@ -287,7 +292,7 @@ pub(crate) async fn set_pushrule_enabled_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")); } - services().account_data.update( + services.account_data.update( None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into(), @@ -301,7 +306,7 @@ pub(crate) async fn set_pushrule_enabled_route( /// /// Deletes a single specified push rule for this user. pub(crate) async fn delete_pushrule_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -312,7 +317,7 @@ pub(crate) async fn delete_pushrule_route( )); } - let event = services() + let event = services .account_data .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; @@ -336,7 +341,7 @@ pub(crate) async fn delete_pushrule_route( return Err(err); } - services().account_data.update( + services.account_data.update( None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into(), @@ -349,11 +354,13 @@ pub(crate) async fn delete_pushrule_route( /// # `GET /_matrix/client/r0/pushers` /// /// Gets all currently active pushers for the sender user. -pub(crate) async fn get_pushers_route(body: Ruma) -> Result { +pub(crate) async fn get_pushers_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); Ok(get_pushers::v3::Response { - pushers: services().pusher.get_pushers(sender_user)?, + pushers: services.pusher.get_pushers(sender_user)?, }) } @@ -362,10 +369,12 @@ pub(crate) async fn get_pushers_route(body: Ruma) -> R /// Adds a pusher for the sender user. /// /// - TODO: Handle `append` -pub(crate) async fn set_pushers_route(body: Ruma) -> Result { +pub(crate) async fn set_pushers_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services().pusher.set_pusher(sender_user, &body.action)?; + services.pusher.set_pusher(sender_user, &body.action)?; Ok(set_pusher::v3::Response::default()) } diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index cbb36b81..f40f2493 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use axum::extract::State; use conduit::PduCount; use ruma::{ api::client::{error::ErrorKind, read_marker::set_read_marker, receipt::create_receipt}, @@ -10,7 +11,7 @@ use ruma::{ MilliSecondsSinceUnixEpoch, }; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `POST /_matrix/client/r0/rooms/{roomId}/read_markers` /// @@ -20,7 +21,7 @@ use crate::{services, Error, Result, Ruma}; /// - If `read_receipt` is set: Update private marker and public read receipt /// EDU pub(crate) async fn set_read_marker_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -30,7 +31,7 @@ pub(crate) async fn set_read_marker_route( event_id: fully_read.clone(), }, }; - services().account_data.update( + services.account_data.update( Some(&body.room_id), sender_user, RoomAccountDataEventType::FullyRead, @@ -39,14 +40,14 @@ pub(crate) async fn set_read_marker_route( } if body.private_read_receipt.is_some() || body.read_receipt.is_some() { - services() + services .rooms .user .reset_notification_counts(sender_user, &body.room_id)?; } if let Some(event) = &body.private_read_receipt { - let count = services() + let count = services .rooms .timeline .get_pdu_count(event)? @@ -60,7 +61,7 @@ pub(crate) async fn set_read_marker_route( }, PduCount::Normal(c) => c, }; - services() + services .rooms .read_receipt .private_read_set(&body.room_id, sender_user, count)?; @@ -82,7 +83,7 @@ pub(crate) async fn set_read_marker_route( let mut receipt_content = BTreeMap::new(); receipt_content.insert(event.to_owned(), receipts); - services().rooms.read_receipt.readreceipt_update( + services.rooms.read_receipt.readreceipt_update( sender_user, &body.room_id, &ruma::events::receipt::ReceiptEvent { @@ -99,7 +100,7 @@ pub(crate) async fn set_read_marker_route( /// /// Sets private read marker and public read receipt EDU. pub(crate) async fn create_receipt_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -107,7 +108,7 @@ pub(crate) async fn create_receipt_route( &body.receipt_type, create_receipt::v3::ReceiptType::Read | create_receipt::v3::ReceiptType::ReadPrivate ) { - services() + services .rooms .user .reset_notification_counts(sender_user, &body.room_id)?; @@ -120,7 +121,7 @@ pub(crate) async fn create_receipt_route( event_id: body.event_id.clone(), }, }; - services().account_data.update( + services.account_data.update( Some(&body.room_id), sender_user, RoomAccountDataEventType::FullyRead, @@ -142,7 +143,7 @@ pub(crate) async fn create_receipt_route( let mut receipt_content = BTreeMap::new(); receipt_content.insert(body.event_id.clone(), receipts); - services().rooms.read_receipt.readreceipt_update( + services.rooms.read_receipt.readreceipt_update( sender_user, &body.room_id, &ruma::events::receipt::ReceiptEvent { @@ -152,7 +153,7 @@ pub(crate) async fn create_receipt_route( )?; }, create_receipt::v3::ReceiptType::ReadPrivate => { - let count = services() + let count = services .rooms .timeline .get_pdu_count(&body.event_id)? @@ -166,7 +167,7 @@ pub(crate) async fn create_receipt_route( }, PduCount::Normal(c) => c, }; - services() + services .rooms .read_receipt .private_read_set(&body.room_id, sender_user, count)?; diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index 308d12e5..89446754 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -1,23 +1,26 @@ +use axum::extract::State; use ruma::{ api::client::redact::redact_event, events::{room::redaction::RoomRedactionEventContent, TimelineEventType}, }; use serde_json::value::to_raw_value; -use crate::{service::pdu::PduBuilder, services, Result, Ruma}; +use crate::{service::pdu::PduBuilder, Result, Ruma}; /// # `PUT /_matrix/client/r0/rooms/{roomId}/redact/{eventId}/{txnId}` /// /// Tries to send a redaction event into the room. /// /// - TODO: Handle txn id -pub(crate) async fn redact_event_route(body: Ruma) -> Result { +pub(crate) async fn redact_event_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let body = body.body; - let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let event_id = services() + let event_id = services .rooms .timeline .build_and_append_pdu( diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index fb366bdc..ae645940 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,30 +1,28 @@ +use axum::extract::State; use ruma::api::client::relations::{ get_relating_events, get_relating_events_with_rel_type, get_relating_events_with_rel_type_and_event_type, }; -use crate::{services, Result, Ruma}; +use crate::{Result, Ruma}; /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}` pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let res = services() - .rooms - .pdu_metadata - .paginate_relations_with_filter( - sender_user, - &body.room_id, - &body.event_id, - &Some(body.event_type.clone()), - &Some(body.rel_type.clone()), - &body.from, - &body.to, - &body.limit, - body.recurse, - body.dir, - )?; + let res = services.rooms.pdu_metadata.paginate_relations_with_filter( + sender_user, + &body.room_id, + &body.event_id, + &Some(body.event_type.clone()), + &Some(body.rel_type.clone()), + &body.from, + &body.to, + &body.limit, + body.recurse, + body.dir, + )?; Ok(get_relating_events_with_rel_type_and_event_type::v1::Response { chunk: res.chunk, @@ -36,25 +34,22 @@ pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}/{relType}` pub(crate) async fn get_relating_events_with_rel_type_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let res = services() - .rooms - .pdu_metadata - .paginate_relations_with_filter( - sender_user, - &body.room_id, - &body.event_id, - &None, - &Some(body.rel_type.clone()), - &body.from, - &body.to, - &body.limit, - body.recurse, - body.dir, - )?; + let res = services.rooms.pdu_metadata.paginate_relations_with_filter( + sender_user, + &body.room_id, + &body.event_id, + &None, + &Some(body.rel_type.clone()), + &body.from, + &body.to, + &body.limit, + body.recurse, + body.dir, + )?; Ok(get_relating_events_with_rel_type::v1::Response { chunk: res.chunk, @@ -66,23 +61,20 @@ pub(crate) async fn get_relating_events_with_rel_type_route( /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}` pub(crate) async fn get_relating_events_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services() - .rooms - .pdu_metadata - .paginate_relations_with_filter( - sender_user, - &body.room_id, - &body.event_id, - &None, - &None, - &body.from, - &body.to, - &body.limit, - body.recurse, - body.dir, - ) + services.rooms.pdu_metadata.paginate_relations_with_filter( + sender_user, + &body.room_id, + &body.event_id, + &None, + &None, + &body.from, + &body.to, + &body.limit, + body.recurse, + body.dir, + ) } diff --git a/src/api/client/report.rs b/src/api/client/report.rs index dae6086f..a16df444 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use axum::extract::State; use rand::Rng; use ruma::{ api::client::{error::ErrorKind, room::report_content}, @@ -9,13 +10,18 @@ use ruma::{ use tokio::time::sleep; use tracing::info; -use crate::{debug_info, service::pdu::PduEvent, services, utils::HtmlEscape, Error, Result, Ruma}; +use crate::{ + debug_info, + service::{pdu::PduEvent, Services}, + utils::HtmlEscape, + Error, Result, Ruma, +}; /// # `POST /_matrix/client/v3/rooms/{roomId}/report/{eventId}` /// /// Reports an inappropriate event to homeserver admins pub(crate) async fn report_event_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { // user authentication let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -26,18 +32,26 @@ pub(crate) async fn report_event_route( ); // check if we know about the reported event ID or if it's invalid - let Some(pdu) = services().rooms.timeline.get_pdu(&body.event_id)? else { + let Some(pdu) = services.rooms.timeline.get_pdu(&body.event_id)? else { return Err(Error::BadRequest( ErrorKind::NotFound, "Event ID is not known to us or Event ID is invalid", )); }; - is_report_valid(&pdu.event_id, &body.room_id, sender_user, &body.reason, body.score, &pdu)?; + is_report_valid( + services, + &pdu.event_id, + &body.room_id, + sender_user, + &body.reason, + body.score, + &pdu, + )?; // send admin room message that we received the report with an @room ping for // urgency - services() + services .admin .send_message(message::RoomMessageEventContent::text_html( format!( @@ -79,8 +93,8 @@ pub(crate) async fn report_event_route( /// check if score is in valid range /// check if report reasoning is less than or equal to 750 characters fn is_report_valid( - event_id: &EventId, room_id: &RoomId, sender_user: &UserId, reason: &Option, score: Option, - pdu: &std::sync::Arc, + services: &Services, event_id: &EventId, room_id: &RoomId, sender_user: &UserId, reason: &Option, + score: Option, pdu: &std::sync::Arc, ) -> Result { debug_info!("Checking if report from user {sender_user} for event {event_id} in room {room_id} is valid"); @@ -91,7 +105,7 @@ fn is_report_valid( )); } - if !services() + if !services .rooms .state_cache .room_members(&pdu.room_id) diff --git a/src/api/client/room.rs b/src/api/client/room.rs index adf58b04..c4d82822 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -1,5 +1,6 @@ use std::{cmp::max, collections::BTreeMap}; +use axum::extract::State; use conduit::{debug_info, debug_warn}; use ruma::{ api::client::{ @@ -30,8 +31,8 @@ use tracing::{error, info, warn}; use super::invite_helper; use crate::{ - service::{appservice::RegistrationInfo, pdu::PduBuilder}, - services, Error, Result, Ruma, + service::{appservice::RegistrationInfo, pdu::PduBuilder, Services}, + Error, Result, Ruma, }; /// Recommended transferable state events list from the spec @@ -63,44 +64,46 @@ const TRANSFERABLE_STATE_EVENTS: &[StateEventType; 9] = &[ /// - Send events listed in initial state /// - Send events implied by `name` and `topic` /// - Send invite events -pub(crate) async fn create_room_route(body: Ruma) -> Result { +pub(crate) async fn create_room_route( + State(services): State, body: Ruma, +) -> Result { use create_room::v3::RoomPreset; let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services().globals.allow_room_creation() + if !services.globals.allow_room_creation() && body.appservice_info.is_none() - && !services().users.is_admin(sender_user)? + && !services.users.is_admin(sender_user)? { return Err(Error::BadRequest(ErrorKind::forbidden(), "Room creation has been disabled.")); } let room_id: OwnedRoomId = if let Some(custom_room_id) = &body.room_id { - custom_room_id_check(custom_room_id)? + custom_room_id_check(services, custom_room_id)? } else { - RoomId::new(&services().globals.config.server_name) + RoomId::new(&services.globals.config.server_name) }; // check if room ID doesn't already exist instead of erroring on auth check - if services().rooms.short.get_shortroomid(&room_id)?.is_some() { + if services.rooms.short.get_shortroomid(&room_id)?.is_some() { return Err(Error::BadRequest( ErrorKind::RoomInUse, "Room with that custom room ID already exists", )); } - let _short_id = services().rooms.short.get_or_create_shortroomid(&room_id)?; - let state_lock = services().rooms.state.mutex.lock(&room_id).await; + let _short_id = services.rooms.short.get_or_create_shortroomid(&room_id)?; + let state_lock = services.rooms.state.mutex.lock(&room_id).await; let alias: Option = if let Some(alias) = &body.room_alias_name { - Some(room_alias_check(alias, &body.appservice_info).await?) + Some(room_alias_check(services, alias, &body.appservice_info).await?) } else { None }; let room_version = match body.room_version.clone() { Some(room_version) => { - if services() + if services .globals .supported_room_versions() .contains(&room_version) @@ -113,7 +116,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R )); } }, - None => services().globals.default_room_version(), + None => services.globals.default_room_version(), }; let content = match &body.creation_content { @@ -184,7 +187,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R }; // 1. The room create event - services() + services .rooms .timeline .build_and_append_pdu( @@ -202,7 +205,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R .await?; // 2. Let the room creator join - services() + services .rooms .timeline .build_and_append_pdu( @@ -210,11 +213,11 @@ pub(crate) async fn create_room_route(body: Ruma) -> R event_type: TimelineEventType::RoomMember, content: to_raw_value(&RoomMemberEventContent { membership: MembershipState::Join, - displayname: services().users.displayname(sender_user)?, - avatar_url: services().users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user)?, + avatar_url: services.users.avatar_url(sender_user)?, is_direct: Some(body.is_direct), third_party_invite: None, - blurhash: services().users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user)?, reason: None, join_authorized_via_users_server: None, }) @@ -249,7 +252,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R let power_levels_content = default_power_levels_content(&body.power_level_content_override, &body.visibility, users)?; - services() + services .rooms .timeline .build_and_append_pdu( @@ -268,7 +271,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R // 4. Canonical room alias if let Some(room_alias_id) = &alias { - services() + services .rooms .timeline .build_and_append_pdu( @@ -293,7 +296,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R // 5. Events set by preset // 5.1 Join Rules - services() + services .rooms .timeline .build_and_append_pdu( @@ -316,7 +319,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R .await?; // 5.2 History Visibility - services() + services .rooms .timeline .build_and_append_pdu( @@ -335,7 +338,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R .await?; // 5.3 Guest Access - services() + services .rooms .timeline .build_and_append_pdu( @@ -378,11 +381,11 @@ pub(crate) async fn create_room_route(body: Ruma) -> R pdu_builder.state_key.get_or_insert_with(String::new); // Silently skip encryption events if they are not allowed - if pdu_builder.event_type == TimelineEventType::RoomEncryption && !services().globals.allow_encryption() { + if pdu_builder.event_type == TimelineEventType::RoomEncryption && !services.globals.allow_encryption() { continue; } - services() + services .rooms .timeline .build_and_append_pdu(pdu_builder, sender_user, &room_id, &state_lock) @@ -391,7 +394,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R // 7. Events implied by name and topic if let Some(name) = &body.name { - services() + services .rooms .timeline .build_and_append_pdu( @@ -411,7 +414,7 @@ pub(crate) async fn create_room_route(body: Ruma) -> R } if let Some(topic) = &body.topic { - services() + services .rooms .timeline .build_and_append_pdu( @@ -435,21 +438,21 @@ pub(crate) async fn create_room_route(body: Ruma) -> R // 8. Events implied by invite (and TODO: invite_3pid) drop(state_lock); for user_id in &body.invite { - if let Err(e) = invite_helper(sender_user, user_id, &room_id, None, body.is_direct).await { + if let Err(e) = invite_helper(services, sender_user, user_id, &room_id, None, body.is_direct).await { warn!(%e, "Failed to send invite"); } } // Homeserver specific stuff if let Some(alias) = alias { - services() + services .rooms .alias .set_alias(&alias, &room_id, sender_user)?; } if body.visibility == room::Visibility::Public { - services().rooms.directory.set_public(&room_id)?; + services.rooms.directory.set_public(&room_id)?; } info!("{sender_user} created a room with room ID {room_id}"); @@ -464,11 +467,11 @@ pub(crate) async fn create_room_route(body: Ruma) -> R /// - You have to currently be joined to the room (TODO: Respect history /// visibility) pub(crate) async fn get_room_event_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = services() + let event = services .rooms .timeline .get_pdu(&body.event_id)? @@ -477,7 +480,7 @@ pub(crate) async fn get_room_event_route( Error::BadRequest(ErrorKind::NotFound, "Event not found.") })?; - if !services() + if !services .rooms .state_accessor .user_can_see_event(sender_user, &event.room_id, &body.event_id)? @@ -502,10 +505,12 @@ pub(crate) async fn get_room_event_route( /// /// - Only users joined to the room are allowed to call this, or if /// `history_visibility` is world readable in the room -pub(crate) async fn get_room_aliases_route(body: Ruma) -> Result { +pub(crate) async fn get_room_aliases_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services() + if !services .rooms .state_accessor .user_can_see_state_events(sender_user, &body.room_id)? @@ -517,7 +522,7 @@ pub(crate) async fn get_room_aliases_route(body: Ruma) -> } Ok(aliases::v3::Response { - aliases: services() + aliases: services .rooms .alias .local_aliases_for_room(&body.room_id) @@ -536,10 +541,12 @@ pub(crate) async fn get_room_aliases_route(body: Ruma) -> /// - Transfers some state events /// - Moves local aliases /// - Modifies old room power levels to prevent users from speaking -pub(crate) async fn upgrade_room_route(body: Ruma) -> Result { +pub(crate) async fn upgrade_room_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services() + if !services .globals .supported_room_versions() .contains(&body.new_version) @@ -551,19 +558,19 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> } // Create a replacement room - let replacement_room = RoomId::new(services().globals.server_name()); + let replacement_room = RoomId::new(services.globals.server_name()); - let _short_id = services() + let _short_id = services .rooms .short .get_or_create_shortroomid(&replacement_room)?; - let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; // Send a m.room.tombstone event to the old room to indicate that it is not // intended to be used any further Fail if the sender does not have the required // permissions - let tombstone_event_id = services() + let tombstone_event_id = services .rooms .timeline .build_and_append_pdu( @@ -586,11 +593,11 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Change lock to replacement room drop(state_lock); - let state_lock = services().rooms.state.mutex.lock(&replacement_room).await; + let state_lock = services.rooms.state.mutex.lock(&replacement_room).await; // Get the old room creation event let mut create_event_content = serde_json::from_str::( - services() + services .rooms .state_accessor .room_state_get(&body.room_id, &StateEventType::RoomCreate, "")? @@ -658,7 +665,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> return Err(Error::BadRequest(ErrorKind::BadJson, "Error forming creation event")); } - services() + services .rooms .timeline .build_and_append_pdu( @@ -676,7 +683,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> .await?; // Join the new room - services() + services .rooms .timeline .build_and_append_pdu( @@ -684,11 +691,11 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> event_type: TimelineEventType::RoomMember, content: to_raw_value(&RoomMemberEventContent { membership: MembershipState::Join, - displayname: services().users.displayname(sender_user)?, - avatar_url: services().users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user)?, + avatar_url: services.users.avatar_url(sender_user)?, is_direct: None, third_party_invite: None, - blurhash: services().users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user)?, reason: None, join_authorized_via_users_server: None, }) @@ -705,7 +712,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Replicate transferable state events to the new room for event_type in TRANSFERABLE_STATE_EVENTS { - let event_content = match services() + let event_content = match services .rooms .state_accessor .room_state_get(&body.room_id, event_type, "")? @@ -714,7 +721,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> None => continue, // Skipping missing events. }; - services() + services .rooms .timeline .build_and_append_pdu( @@ -733,13 +740,13 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> } // Moves any local aliases to the new room - for alias in services() + for alias in services .rooms .alias .local_aliases_for_room(&body.room_id) .filter_map(Result::ok) { - services() + services .rooms .alias .set_alias(&alias, &replacement_room, sender_user)?; @@ -747,7 +754,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Get the old room power levels let mut power_levels_event_content: RoomPowerLevelsEventContent = serde_json::from_str( - services() + services .rooms .state_accessor .room_state_get(&body.room_id, &StateEventType::RoomPowerLevels, "")? @@ -772,7 +779,7 @@ pub(crate) async fn upgrade_room_route(body: Ruma) -> // Modify the power levels in the old room to prevent sending of events and // inviting new users - services() + services .rooms .timeline .build_and_append_pdu( @@ -841,7 +848,7 @@ fn default_power_levels_content( /// if a room is being created with a room alias, run our checks async fn room_alias_check( - room_alias_name: &str, appservice_info: &Option, + services: &Services, room_alias_name: &str, appservice_info: &Option, ) -> Result { // Basic checks on the room alias validity if room_alias_name.contains(':') { @@ -858,7 +865,7 @@ async fn room_alias_check( } // check if room alias is forbidden - if services() + if services .globals .forbidden_alias_names() .is_match(room_alias_name) @@ -866,13 +873,13 @@ async fn room_alias_check( return Err(Error::BadRequest(ErrorKind::Unknown, "Room alias name is forbidden.")); } - let full_room_alias = RoomAliasId::parse(format!("#{}:{}", room_alias_name, services().globals.config.server_name)) + let full_room_alias = RoomAliasId::parse(format!("#{}:{}", room_alias_name, services.globals.config.server_name)) .map_err(|e| { - info!("Failed to parse room alias {room_alias_name}: {e}"); - Error::BadRequest(ErrorKind::InvalidParam, "Invalid room alias specified.") - })?; + info!("Failed to parse room alias {room_alias_name}: {e}"); + Error::BadRequest(ErrorKind::InvalidParam, "Invalid room alias specified.") + })?; - if services() + if services .rooms .alias .resolve_local_alias(&full_room_alias)? @@ -885,7 +892,7 @@ async fn room_alias_check( if !info.aliases.is_match(full_room_alias.as_str()) { return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias is not in namespace.")); } - } else if services() + } else if services .appservice .is_exclusive_alias(&full_room_alias) .await @@ -899,9 +906,9 @@ async fn room_alias_check( } /// if a room is being created with a custom room ID, run our checks against it -fn custom_room_id_check(custom_room_id: &str) -> Result { +fn custom_room_id_check(services: &Services, custom_room_id: &str) -> Result { // apply forbidden room alias checks to custom room IDs too - if services() + if services .globals .forbidden_alias_names() .is_match(custom_room_id) @@ -922,7 +929,7 @@ fn custom_room_id_check(custom_room_id: &str) -> Result { )); } - let full_room_id = format!("!{}:{}", custom_room_id, services().globals.config.server_name); + let full_room_id = format!("!{}:{}", custom_room_id, services.globals.config.server_name); debug_info!("Full custom room ID: {full_room_id}"); diff --git a/src/api/client/search.rs b/src/api/client/search.rs index d2a1668f..b143bd2c 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use axum::extract::State; use ruma::{ api::client::{ error::ErrorKind, @@ -14,7 +15,7 @@ use ruma::{ }; use tracing::debug; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `POST /_matrix/client/r0/search` /// @@ -22,7 +23,9 @@ use crate::{services, Error, Result, Ruma}; /// /// - Only works if the user is currently joined to the room (TODO: Respect /// history visibility) -pub(crate) async fn search_events_route(body: Ruma) -> Result { +pub(crate) async fn search_events_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let search_criteria = body.search_categories.room_events.as_ref().unwrap(); @@ -30,7 +33,7 @@ pub(crate) async fn search_events_route(body: Ruma) let include_state = &search_criteria.include_state; let room_ids = filter.rooms.clone().unwrap_or_else(|| { - services() + services .rooms .state_cache .rooms_joined(sender_user) @@ -50,11 +53,7 @@ pub(crate) async fn search_events_route(body: Ruma) if include_state.is_some_and(|include_state| include_state) { for room_id in &room_ids { - if !services() - .rooms - .state_cache - .is_joined(sender_user, room_id)? - { + if !services.rooms.state_cache.is_joined(sender_user, room_id)? { return Err(Error::BadRequest( ErrorKind::forbidden(), "You don't have permission to view this room.", @@ -62,12 +61,12 @@ pub(crate) async fn search_events_route(body: Ruma) } // check if sender_user can see state events - if services() + if services .rooms .state_accessor .user_can_see_state_events(sender_user, room_id)? { - let room_state = services() + let room_state = services .rooms .state_accessor .room_state_full(room_id) @@ -91,18 +90,14 @@ pub(crate) async fn search_events_route(body: Ruma) let mut searches = Vec::new(); for room_id in &room_ids { - if !services() - .rooms - .state_cache - .is_joined(sender_user, room_id)? - { + if !services.rooms.state_cache.is_joined(sender_user, room_id)? { return Err(Error::BadRequest( ErrorKind::forbidden(), "You don't have permission to view this room.", )); } - if let Some(search) = services() + if let Some(search) = services .rooms .search .search_pdus(room_id, &search_criteria.search_term)? @@ -135,14 +130,14 @@ pub(crate) async fn search_events_route(body: Ruma) .iter() .skip(skip) .filter_map(|result| { - services() + services .rooms .timeline .get_pdu_from_id(result) .ok()? .filter(|pdu| { !pdu.is_redacted() - && services() + && services .rooms .state_accessor .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 6df46549..32f3ed29 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use ruma::{ api::client::{ error::ErrorKind, @@ -20,7 +21,7 @@ use serde::Deserialize; use tracing::{debug, info, warn}; use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; -use crate::{services, utils, utils::hash, Error, Result, Ruma}; +use crate::{utils, utils::hash, Error, Result, Ruma}; #[derive(Debug, Deserialize)] struct Claims { @@ -55,7 +56,9 @@ pub(crate) async fn get_login_types_route( /// Note: You can use [`GET /// /_matrix/client/r0/login`](fn.get_supported_versions_route.html) to see /// supported login types. -pub(crate) async fn login_route(body: Ruma) -> Result { +pub(crate) async fn login_route( + State(services): State, body: Ruma, +) -> Result { // Validate login method // TODO: Other login methods let user_id = match &body.login_info { @@ -68,7 +71,7 @@ pub(crate) async fn login_route(body: Ruma) -> Result { debug!("Got password login type"); let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { - UserId::parse_with_server_name(user_id.to_lowercase(), services().globals.server_name()) + UserId::parse_with_server_name(user_id.to_lowercase(), services.globals.server_name()) } else if let Some(user) = user { UserId::parse(user) } else { @@ -77,7 +80,7 @@ pub(crate) async fn login_route(body: Ruma) -> Result) -> Result { debug!("Got token login type"); - if let Some(jwt_decoding_key) = services().globals.jwt_decoding_key() { + if let Some(jwt_decoding_key) = services.globals.jwt_decoding_key() { let token = jsonwebtoken::decode::(token, jwt_decoding_key, &jsonwebtoken::Validation::default()) .map_err(|e| { @@ -106,7 +109,7 @@ pub(crate) async fn login_route(body: Ruma) -> Result) -> Result { debug!("Got appservice login type"); let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { - UserId::parse_with_server_name(user_id.to_lowercase(), services().globals.server_name()) + UserId::parse_with_server_name(user_id.to_lowercase(), services.globals.server_name()) } else if let Some(user) = user { UserId::parse(user) } else { @@ -164,22 +167,22 @@ pub(crate) async fn login_route(body: Ruma) -> Result = services() + let client_discovery_info: Option = services .globals .well_known_client() .as_ref() @@ -197,7 +200,7 @@ pub(crate) async fn login_route(body: Ruma) -> Result) -> Result) -> Result { +pub(crate) async fn logout_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); - services().users.remove_device(sender_user, sender_device)?; + services.users.remove_device(sender_user, sender_device)?; // send device list update for user after logout - services().users.mark_device_key_update(sender_user)?; + services.users.mark_device_key_update(sender_user)?; Ok(logout::v3::Response::new()) } @@ -236,15 +241,17 @@ pub(crate) async fn logout_route(body: Ruma) -> Result) -> Result { +pub(crate) async fn logout_all_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - for device_id in services().users.all_device_ids(sender_user).flatten() { - services().users.remove_device(sender_user, &device_id)?; + for device_id in services.users.all_device_ids(sender_user).flatten() { + services.users.remove_device(sender_user, &device_id)?; } // send device list update for user after logout - services().users.mark_device_key_update(sender_user)?; + services.users.mark_device_key_update(sender_user)?; Ok(logout_all::v3::Response::new()) } diff --git a/src/api/client/space.rs b/src/api/client/space.rs index 0cf1b107..a3031a3a 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -1,17 +1,20 @@ use std::str::FromStr; +use axum::extract::State; use ruma::{ api::client::{error::ErrorKind, space::get_hierarchy}, UInt, }; -use crate::{service::rooms::spaces::PaginationToken, services, Error, Result, Ruma}; +use crate::{service::rooms::spaces::PaginationToken, Error, Result, Ruma}; /// # `GET /_matrix/client/v1/rooms/{room_id}/hierarchy` /// /// Paginates over the space tree in a depth-first manner to locate child rooms /// of a given space. -pub(crate) async fn get_hierarchy_route(body: Ruma) -> Result { +pub(crate) async fn get_hierarchy_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let limit = body @@ -39,7 +42,7 @@ pub(crate) async fn get_hierarchy_route(body: Ruma) } } - services() + services .rooms .spaces .get_client_hierarchy( diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 25b77fe3..51217d00 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use axum::extract::State; use conduit::{debug_info, error}; use ruma::{ api::client::{ @@ -19,8 +20,8 @@ use ruma::{ }; use crate::{ - service::{pdu::PduBuilder, server_is_ours}, - services, Error, Result, Ruma, RumaResponse, + service::{pdu::PduBuilder, server_is_ours, Services}, + Error, Result, Ruma, RumaResponse, }; /// # `PUT /_matrix/client/*/rooms/{roomId}/state/{eventType}/{stateKey}` @@ -32,12 +33,13 @@ use crate::{ /// allowed /// - If event is new `canonical_alias`: Rejects if alias is incorrect pub(crate) async fn send_state_event_for_key_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); Ok(send_state_event::v3::Response { event_id: send_state_event_for_key_helper( + services, sender_user, &body.room_id, &body.event_type, @@ -58,9 +60,11 @@ pub(crate) async fn send_state_event_for_key_route( /// allowed /// - If event is new `canonical_alias`: Rejects if alias is incorrect pub(crate) async fn send_state_event_for_empty_key_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result> { - send_state_event_for_key_route(body).await.map(RumaResponse) + send_state_event_for_key_route(State(services), body) + .await + .map(RumaResponse) } /// # `GET /_matrix/client/v3/rooms/{roomid}/state` @@ -70,11 +74,11 @@ pub(crate) async fn send_state_event_for_empty_key_route( /// - If not joined: Only works if current room history visibility is world /// readable pub(crate) async fn get_state_events_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services() + if !services .rooms .state_accessor .user_can_see_state_events(sender_user, &body.room_id)? @@ -86,7 +90,7 @@ pub(crate) async fn get_state_events_route( } Ok(get_state_events::v3::Response { - room_state: services() + room_state: services .rooms .state_accessor .room_state_full(&body.room_id) @@ -106,11 +110,11 @@ pub(crate) async fn get_state_events_route( /// - If not joined: Only works if current room history visibility is world /// readable pub(crate) async fn get_state_events_for_key_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services() + if !services .rooms .state_accessor .user_can_see_state_events(sender_user, &body.room_id)? @@ -121,7 +125,7 @@ pub(crate) async fn get_state_events_for_key_route( )); } - let event = services() + let event = services .rooms .state_accessor .room_state_get(&body.room_id, &body.event_type, &body.state_key)? @@ -161,17 +165,20 @@ pub(crate) async fn get_state_events_for_key_route( /// - If not joined: Only works if current room history visibility is world /// readable pub(crate) async fn get_state_events_for_empty_key_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result> { - get_state_events_for_key_route(body).await.map(RumaResponse) + get_state_events_for_key_route(State(services), body) + .await + .map(RumaResponse) } async fn send_state_event_for_key_helper( - sender: &UserId, room_id: &RoomId, event_type: &StateEventType, json: &Raw, state_key: String, + services: &Services, sender: &UserId, room_id: &RoomId, event_type: &StateEventType, + json: &Raw, state_key: String, ) -> Result> { - allowed_to_send_state_event(room_id, event_type, json).await?; - let state_lock = services().rooms.state.mutex.lock(room_id).await; - let event_id = services() + allowed_to_send_state_event(services, room_id, event_type, json).await?; + let state_lock = services.rooms.state.mutex.lock(room_id).await; + let event_id = services .rooms .timeline .build_and_append_pdu( @@ -192,12 +199,12 @@ async fn send_state_event_for_key_helper( } async fn allowed_to_send_state_event( - room_id: &RoomId, event_type: &StateEventType, json: &Raw, + services: &Services, room_id: &RoomId, event_type: &StateEventType, json: &Raw, ) -> Result<()> { match event_type { // Forbid m.room.encryption if encryption is disabled StateEventType::RoomEncryption => { - if !services().globals.allow_encryption() { + if !services.globals.allow_encryption() { return Err(Error::BadRequest(ErrorKind::forbidden(), "Encryption has been disabled")); } }, @@ -244,7 +251,7 @@ async fn allowed_to_send_state_event( for alias in aliases { if !server_is_ours(alias.server_name()) - || services() + || services .rooms .alias .resolve_local_alias(&alias)? diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index e425616b..5739052c 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -5,6 +5,7 @@ use std::{ time::Duration, }; +use axum::extract::State; use conduit::{ error, utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, @@ -17,7 +18,7 @@ use ruma::{ self, v3::{ Ephemeral, Filter, GlobalAccountData, InviteState, InvitedRoom, JoinedRoom, LeftRoom, Presence, - RoomAccountData, RoomSummary, Rooms, State, Timeline, ToDevice, + RoomAccountData, RoomSummary, Rooms, State as RoomState, Timeline, ToDevice, }, v4::SlidingOp, DeviceLists, UnreadNotificationsCount, @@ -34,7 +35,10 @@ use ruma::{ }; use tracing::{Instrument as _, Span}; -use crate::{service::pdu::EventHash, services, utils, Error, PduEvent, Result, Ruma, RumaResponse}; +use crate::{ + service::{pdu::EventHash, Services}, + utils, Error, PduEvent, Result, Ruma, RumaResponse, +}; /// # `GET /_matrix/client/r0/sync` /// @@ -72,23 +76,23 @@ use crate::{service::pdu::EventHash, services, utils, Error, PduEvent, Result, R /// - If the user left after `since`: `prev_batch` token, empty state (TODO: /// subset of the state at the point of the leave) pub(crate) async fn sync_events_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result> { let sender_user = body.sender_user.expect("user is authenticated"); let sender_device = body.sender_device.expect("user is authenticated"); let body = body.body; // Presence update - if services().globals.allow_local_presence() { - services() + if services.globals.allow_local_presence() { + services .presence .ping_presence(&sender_user, &body.set_presence)?; } // Setup watchers, so if there's no response, we can wait for them - let watcher = services().globals.watch(&sender_user, &sender_device); + let watcher = services.globals.watch(&sender_user, &sender_device); - let next_batch = services().globals.current_count()?; + let next_batch = services.globals.current_count()?; let next_batchcount = PduCount::Normal(next_batch); let next_batch_string = next_batch.to_string(); @@ -96,7 +100,7 @@ pub(crate) async fn sync_events_route( let filter = match body.filter { None => FilterDefinition::default(), Some(Filter::FilterDefinition(filter)) => filter, - Some(Filter::FilterId(filter_id)) => services() + Some(Filter::FilterId(filter_id)) => services .users .get_filter(&sender_user, &filter_id)? .unwrap_or_default(), @@ -126,28 +130,29 @@ pub(crate) async fn sync_events_route( // Look for device list updates of this account device_list_updates.extend( - services() + services .users .keys_changed(sender_user.as_ref(), since, None) .filter_map(Result::ok), ); - if services().globals.allow_local_presence() { - process_presence_updates(&mut presence_updates, since, &sender_user).await?; + if services.globals.allow_local_presence() { + process_presence_updates(services, &mut presence_updates, since, &sender_user).await?; } - let all_joined_rooms = services() + let all_joined_rooms = services .rooms .state_cache .rooms_joined(&sender_user) .collect::>(); // Coalesce database writes for the remainder of this scope. - let _cork = services().db.cork_and_flush(); + let _cork = services.db.cork_and_flush(); for room_id in all_joined_rooms { let room_id = room_id?; if let Ok(joined_room) = load_joined_room( + services, &sender_user, &sender_device, &room_id, @@ -170,13 +175,14 @@ pub(crate) async fn sync_events_route( } let mut left_rooms = BTreeMap::new(); - let all_left_rooms: Vec<_> = services() + let all_left_rooms: Vec<_> = services .rooms .state_cache .rooms_left(&sender_user) .collect(); for result in all_left_rooms { handle_left_room( + services, since, &result?.0, &sender_user, @@ -190,7 +196,7 @@ pub(crate) async fn sync_events_route( } let mut invited_rooms = BTreeMap::new(); - let all_invited_rooms: Vec<_> = services() + let all_invited_rooms: Vec<_> = services .rooms .state_cache .rooms_invited(&sender_user) @@ -199,10 +205,10 @@ pub(crate) async fn sync_events_route( let (room_id, invite_state_events) = result?; // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services().rooms.timeline.mutex_insert.lock(&room_id).await; + let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; drop(insert_lock); - let invite_count = services() + let invite_count = services .rooms .state_cache .get_invite_count(&room_id, &sender_user)?; @@ -223,14 +229,14 @@ pub(crate) async fn sync_events_route( } for user_id in left_encrypted_users { - let dont_share_encrypted_room = services() + let dont_share_encrypted_room = services .rooms .user .get_shared_rooms(vec![sender_user.clone(), user_id.clone()])? .filter_map(Result::ok) .filter_map(|other_room_id| { Some( - services() + services .rooms .state_accessor .room_state_get(&other_room_id, &StateEventType::RoomEncryption, "") @@ -247,7 +253,7 @@ pub(crate) async fn sync_events_route( } // Remove all to-device events the device received *last time* - services() + services .users .remove_to_device_events(&sender_user, &sender_device, since)?; @@ -266,7 +272,7 @@ pub(crate) async fn sync_events_route( .collect(), }, account_data: GlobalAccountData { - events: services() + events: services .account_data .changes_since(None, &sender_user, since)? .into_iter() @@ -281,11 +287,11 @@ pub(crate) async fn sync_events_route( changed: device_list_updates.into_iter().collect(), left: device_list_left.into_iter().collect(), }, - device_one_time_keys_count: services() + device_one_time_keys_count: services .users .count_one_time_keys(&sender_user, &sender_device)?, to_device: ToDevice { - events: services() + events: services .users .get_to_device_events(&sender_user, &sender_device)?, }, @@ -311,16 +317,18 @@ pub(crate) async fn sync_events_route( Ok(response) } +#[allow(clippy::too_many_arguments)] #[tracing::instrument(skip_all, fields(user_id = %sender_user, room_id = %room_id), name = "left_room")] async fn handle_left_room( - since: u64, room_id: &RoomId, sender_user: &UserId, left_rooms: &mut BTreeMap, - next_batch_string: &str, full_state: bool, lazy_load_enabled: bool, + services: &Services, since: u64, room_id: &RoomId, sender_user: &UserId, + left_rooms: &mut BTreeMap, next_batch_string: &str, full_state: bool, + lazy_load_enabled: bool, ) -> Result<()> { // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services().rooms.timeline.mutex_insert.lock(room_id).await; + let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); - let left_count = services() + let left_count = services .rooms .state_cache .get_left_count(room_id, sender_user)?; @@ -330,11 +338,11 @@ async fn handle_left_room( return Ok(()); } - if !services().rooms.metadata.exists(room_id)? { + if !services.rooms.metadata.exists(room_id)? { // This is just a rejected invite, not a room we know // Insert a leave event anyways let event = PduEvent { - event_id: EventId::new(services().globals.server_name()).into(), + event_id: EventId::new(services.globals.server_name()).into(), sender: sender_user.to_owned(), origin: None, origin_server_ts: utils::millis_since_unix_epoch() @@ -367,7 +375,7 @@ async fn handle_left_room( prev_batch: Some(next_batch_string.to_owned()), events: Vec::new(), }, - state: State { + state: RoomState { events: vec![event.to_sync_state_event()], }, }, @@ -377,27 +385,27 @@ async fn handle_left_room( let mut left_state_events = Vec::new(); - let since_shortstatehash = services() + let since_shortstatehash = services .rooms .user .get_token_shortstatehash(room_id, since)?; let since_state_ids = match since_shortstatehash { - Some(s) => services().rooms.state_accessor.state_full_ids(s).await?, + Some(s) => services.rooms.state_accessor.state_full_ids(s).await?, None => HashMap::new(), }; - let Some(left_event_id) = services().rooms.state_accessor.room_state_get_id( - room_id, - &StateEventType::RoomMember, - sender_user.as_str(), - )? + let Some(left_event_id) = + services + .rooms + .state_accessor + .room_state_get_id(room_id, &StateEventType::RoomMember, sender_user.as_str())? else { error!("Left room but no left state event"); return Ok(()); }; - let Some(left_shortstatehash) = services() + let Some(left_shortstatehash) = services .rooms .state_accessor .pdu_shortstatehash(&left_event_id)? @@ -406,13 +414,13 @@ async fn handle_left_room( return Ok(()); }; - let mut left_state_ids = services() + let mut left_state_ids = services .rooms .state_accessor .state_full_ids(left_shortstatehash) .await?; - let leave_shortstatekey = services() + let leave_shortstatekey = services .rooms .short .get_or_create_shortstatekey(&StateEventType::RoomMember, sender_user.as_str())?; @@ -422,7 +430,7 @@ async fn handle_left_room( let mut i: u8 = 0; for (key, id) in left_state_ids { if full_state || since_state_ids.get(&key) != Some(&id) { - let (event_type, state_key) = services().rooms.short.get_statekey_from_short(key)?; + let (event_type, state_key) = services.rooms.short.get_statekey_from_short(key)?; if !lazy_load_enabled || event_type != StateEventType::RoomMember @@ -430,7 +438,7 @@ async fn handle_left_room( // TODO: Delete the following line when this is resolved: https://github.com/vector-im/element-web/issues/22565 || (cfg!(feature = "element_hacks") && *sender_user == state_key) { - let Some(pdu) = services().rooms.timeline.get_pdu(&id)? else { + let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { error!("Pdu in state not found: {}", id); continue; }; @@ -456,7 +464,7 @@ async fn handle_left_room( prev_batch: Some(next_batch_string.to_owned()), events: Vec::new(), }, - state: State { + state: RoomState { events: left_state_events, }, }, @@ -465,13 +473,13 @@ async fn handle_left_room( } async fn process_presence_updates( - presence_updates: &mut HashMap, since: u64, syncing_user: &UserId, + services: &Services, presence_updates: &mut HashMap, since: u64, syncing_user: &UserId, ) -> Result<()> { use crate::service::presence::Presence; // Take presence updates - for (user_id, _, presence_bytes) in services().presence.presence_since(since) { - if !services() + for (user_id, _, presence_bytes) in services.presence.presence_since(since) { + if !services .rooms .state_cache .user_sees_user(syncing_user, &user_id)? @@ -513,19 +521,20 @@ async fn process_presence_updates( #[allow(clippy::too_many_arguments)] async fn load_joined_room( - sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, since: u64, sincecount: PduCount, - next_batch: u64, next_batchcount: PduCount, lazy_load_enabled: bool, lazy_load_send_redundant: bool, - full_state: bool, device_list_updates: &mut HashSet, left_encrypted_users: &mut HashSet, + services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, since: u64, + sincecount: PduCount, next_batch: u64, next_batchcount: PduCount, lazy_load_enabled: bool, + lazy_load_send_redundant: bool, full_state: bool, device_list_updates: &mut HashSet, + left_encrypted_users: &mut HashSet, ) -> Result { // Get and drop the lock to wait for remaining operations to finish // This will make sure the we have all events until next_batch - let insert_lock = services().rooms.timeline.mutex_insert.lock(room_id).await; + let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); - let (timeline_pdus, limited) = load_timeline(sender_user, room_id, sincecount, 10)?; + let (timeline_pdus, limited) = load_timeline(services, sender_user, room_id, sincecount, 10)?; let send_notification_counts = !timeline_pdus.is_empty() - || services() + || services .rooms .user .last_notification_read(sender_user, room_id)? @@ -536,7 +545,7 @@ async fn load_joined_room( timeline_users.insert(event.sender.as_str().to_owned()); } - services() + services .rooms .lazy_loading .lazy_load_confirm_delivery(sender_user, sender_device, room_id, sincecount) @@ -544,11 +553,11 @@ async fn load_joined_room( // Database queries: - let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? else { + let Some(current_shortstatehash) = services.rooms.state.get_room_shortstatehash(room_id)? else { return Err!(Database(error!("Room {room_id} has no state"))); }; - let since_shortstatehash = services() + let since_shortstatehash = services .rooms .user .get_token_shortstatehash(room_id, since)?; @@ -560,12 +569,12 @@ async fn load_joined_room( } else { // Calculates joined_member_count, invited_member_count and heroes let calculate_counts = || { - let joined_member_count = services() + let joined_member_count = services .rooms .state_cache .room_joined_count(room_id)? .unwrap_or(0); - let invited_member_count = services() + let invited_member_count = services .rooms .state_cache .room_invited_count(room_id)? @@ -578,7 +587,7 @@ async fn load_joined_room( // Go through all PDUs and for each member event, check if the user is still // joined or invited until we have 5 or we reach the end - for hero in services() + for hero in services .rooms .timeline .all_pdus(sender_user, room_id)? @@ -594,8 +603,8 @@ async fn load_joined_room( // The membership was and still is invite or join if matches!(content.membership, MembershipState::Join | MembershipState::Invite) - && (services().rooms.state_cache.is_joined(&user_id, room_id)? - || services().rooms.state_cache.is_invited(&user_id, room_id)?) + && (services.rooms.state_cache.is_joined(&user_id, room_id)? + || services.rooms.state_cache.is_invited(&user_id, room_id)?) { Ok::<_, Error>(Some(user_id)) } else { @@ -622,7 +631,7 @@ async fn load_joined_room( let since_sender_member: Option = since_shortstatehash .and_then(|shortstatehash| { - services() + services .rooms .state_accessor .state_get(shortstatehash, &StateEventType::RoomMember, sender_user.as_str()) @@ -643,7 +652,7 @@ async fn load_joined_room( let (joined_member_count, invited_member_count, heroes) = calculate_counts()?; - let current_state_ids = services() + let current_state_ids = services .rooms .state_accessor .state_full_ids(current_shortstatehash) @@ -654,13 +663,13 @@ async fn load_joined_room( let mut i: u8 = 0; for (shortstatekey, id) in current_state_ids { - let (event_type, state_key) = services() + let (event_type, state_key) = services .rooms .short .get_statekey_from_short(shortstatekey)?; if event_type != StateEventType::RoomMember { - let Some(pdu) = services().rooms.timeline.get_pdu(&id)? else { + let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { error!("Pdu in state not found: {}", id); continue; }; @@ -676,7 +685,7 @@ async fn load_joined_room( // TODO: Delete the following line when this is resolved: https://github.com/vector-im/element-web/issues/22565 || (cfg!(feature = "element_hacks") && *sender_user == state_key) { - let Some(pdu) = services().rooms.timeline.get_pdu(&id)? else { + let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { error!("Pdu in state not found: {}", id); continue; }; @@ -695,14 +704,14 @@ async fn load_joined_room( } // Reset lazy loading because this is an initial sync - services() + services .rooms .lazy_loading .lazy_load_reset(sender_user, sender_device, room_id)?; // The state_events above should contain all timeline_users, let's mark them as // lazy loaded. - services() + services .rooms .lazy_loading .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy_loaded, next_batchcount) @@ -716,12 +725,12 @@ async fn load_joined_room( let mut delta_state_events = Vec::new(); if since_shortstatehash != current_shortstatehash { - let current_state_ids = services() + let current_state_ids = services .rooms .state_accessor .state_full_ids(current_shortstatehash) .await?; - let since_state_ids = services() + let since_state_ids = services .rooms .state_accessor .state_full_ids(since_shortstatehash) @@ -729,7 +738,7 @@ async fn load_joined_room( for (key, id) in current_state_ids { if full_state || since_state_ids.get(&key) != Some(&id) { - let Some(pdu) = services().rooms.timeline.get_pdu(&id)? else { + let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { error!("Pdu in state not found: {}", id); continue; }; @@ -740,13 +749,13 @@ async fn load_joined_room( } } - let encrypted_room = services() + let encrypted_room = services .rooms .state_accessor .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "")? .is_some(); - let since_encryption = services().rooms.state_accessor.state_get( + let since_encryption = services.rooms.state_accessor.state_get( since_shortstatehash, &StateEventType::RoomEncryption, "", @@ -781,7 +790,7 @@ async fn load_joined_room( match new_membership { MembershipState::Join => { // A new user joined an encrypted room - if !share_encrypted_room(sender_user, &user_id, room_id)? { + if !share_encrypted_room(services, sender_user, &user_id, room_id)? { device_list_updates.insert(user_id); } }, @@ -798,7 +807,7 @@ async fn load_joined_room( if joined_since_last_sync && encrypted_room || new_encrypted_room { // If the user is in a new encrypted room, give them all joined users device_list_updates.extend( - services() + services .rooms .state_cache .room_members(room_id) @@ -810,7 +819,7 @@ async fn load_joined_room( .filter(|user_id| { // Only send keys if the sender doesn't share an encrypted room with the target // already - !share_encrypted_room(sender_user, user_id, room_id).unwrap_or(false) + !share_encrypted_room(services, sender_user, user_id, room_id).unwrap_or(false) }), ); } @@ -848,14 +857,14 @@ async fn load_joined_room( continue; } - if !services().rooms.lazy_loading.lazy_load_was_sent_before( + if !services.rooms.lazy_loading.lazy_load_was_sent_before( sender_user, sender_device, room_id, &event.sender, )? || lazy_load_send_redundant { - if let Some(member_event) = services().rooms.state_accessor.room_state_get( + if let Some(member_event) = services.rooms.state_accessor.room_state_get( room_id, &StateEventType::RoomMember, event.sender.as_str(), @@ -866,7 +875,7 @@ async fn load_joined_room( } } - services() + services .rooms .lazy_loading .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy_loaded, next_batchcount) @@ -884,7 +893,7 @@ async fn load_joined_room( // Look for device list updates in this room device_list_updates.extend( - services() + services .users .keys_changed(room_id.as_ref(), since, None) .filter_map(Result::ok), @@ -892,7 +901,7 @@ async fn load_joined_room( let notification_count = if send_notification_counts { Some( - services() + services .rooms .user .notification_count(sender_user, room_id)? @@ -905,7 +914,7 @@ async fn load_joined_room( let highlight_count = if send_notification_counts { Some( - services() + services .rooms .user .highlight_count(sender_user, room_id)? @@ -933,7 +942,7 @@ async fn load_joined_room( .map(|(_, pdu)| pdu.to_sync_room_event()) .collect(); - let mut edus: Vec<_> = services() + let mut edus: Vec<_> = services .rooms .read_receipt .readreceipts_since(room_id, since) @@ -941,10 +950,10 @@ async fn load_joined_room( .map(|(_, _, v)| v) .collect(); - if services().rooms.typing.last_typing_update(room_id).await? > since { + if services.rooms.typing.last_typing_update(room_id).await? > since { edus.push( serde_json::from_str( - &serde_json::to_string(&services().rooms.typing.typings_all(room_id).await?) + &serde_json::to_string(&services.rooms.typing.typings_all(room_id).await?) .expect("event is valid, we just created it"), ) .expect("event is valid, we just created it"), @@ -953,14 +962,14 @@ async fn load_joined_room( // Save the state after this sync so we can send the correct state diff next // sync - services() + services .rooms .user .associate_token_shortstatehash(room_id, next_batch, current_shortstatehash)?; Ok(JoinedRoom { account_data: RoomAccountData { - events: services() + events: services .account_data .changes_since(Some(room_id), sender_user, since)? .into_iter() @@ -985,7 +994,7 @@ async fn load_joined_room( prev_batch, events: room_events, }, - state: State { + state: RoomState { events: state_events .iter() .map(|pdu| pdu.to_sync_state_event()) @@ -999,16 +1008,16 @@ async fn load_joined_room( } fn load_timeline( - sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: u64, + services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: u64, ) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> { let timeline_pdus; - let limited = if services() + let limited = if services .rooms .timeline .last_timeline_count(sender_user, room_id)? > roomsincecount { - let mut non_timeline_pdus = services() + let mut non_timeline_pdus = services .rooms .timeline .pdus_until(sender_user, room_id, PduCount::max())? @@ -1040,8 +1049,10 @@ fn load_timeline( Ok((timeline_pdus, limited)) } -fn share_encrypted_room(sender_user: &UserId, user_id: &UserId, ignore_room: &RoomId) -> Result { - Ok(services() +fn share_encrypted_room( + services: &Services, sender_user: &UserId, user_id: &UserId, ignore_room: &RoomId, +) -> Result { + Ok(services .rooms .user .get_shared_rooms(vec![sender_user.to_owned(), user_id.to_owned()])? @@ -1049,7 +1060,7 @@ fn share_encrypted_room(sender_user: &UserId, user_id: &UserId, ignore_room: &Ro .filter(|room_id| room_id != ignore_room) .filter_map(|other_room_id| { Some( - services() + services .rooms .state_accessor .room_state_get(&other_room_id, &StateEventType::RoomEncryption, "") @@ -1064,15 +1075,15 @@ fn share_encrypted_room(sender_user: &UserId, user_id: &UserId, ignore_room: &Ro /// /// Sliding Sync endpoint (future endpoint: `/_matrix/client/v4/sync`) pub(crate) async fn sync_events_v4_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result> { let sender_user = body.sender_user.expect("user is authenticated"); let sender_device = body.sender_device.expect("user is authenticated"); let mut body = body.body; // Setup watchers, so if there's no response, we can wait for them - let watcher = services().globals.watch(&sender_user, &sender_device); + let watcher = services.globals.watch(&sender_user, &sender_device); - let next_batch = services().globals.next_count()?; + let next_batch = services.globals.next_count()?; let globalsince = body .pos @@ -1082,21 +1093,19 @@ pub(crate) async fn sync_events_v4_route( if globalsince == 0 { if let Some(conn_id) = &body.conn_id { - services().users.forget_sync_request_connection( - sender_user.clone(), - sender_device.clone(), - conn_id.clone(), - ); + services + .users + .forget_sync_request_connection(sender_user.clone(), sender_device.clone(), conn_id.clone()); } } // Get sticky parameters from cache let known_rooms = - services() + services .users .update_sync_request_with_cache(sender_user.clone(), sender_device.clone(), &mut body); - let all_joined_rooms = services() + let all_joined_rooms = services .rooms .state_cache .rooms_joined(&sender_user) @@ -1104,7 +1113,7 @@ pub(crate) async fn sync_events_v4_route( .collect::>(); if body.extensions.to_device.enabled.unwrap_or(false) { - services() + services .users .remove_to_device_events(&sender_user, &sender_device, globalsince)?; } @@ -1116,26 +1125,26 @@ pub(crate) async fn sync_events_v4_route( if body.extensions.e2ee.enabled.unwrap_or(false) { // Look for device list updates of this account device_list_changes.extend( - services() + services .users .keys_changed(sender_user.as_ref(), globalsince, None) .filter_map(Result::ok), ); for room_id in &all_joined_rooms { - let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? else { + let Some(current_shortstatehash) = services.rooms.state.get_room_shortstatehash(room_id)? else { error!("Room {} has no state", room_id); continue; }; - let since_shortstatehash = services() + let since_shortstatehash = services .rooms .user .get_token_shortstatehash(room_id, globalsince)?; let since_sender_member: Option = since_shortstatehash .and_then(|shortstatehash| { - services() + services .rooms .state_accessor .state_get(shortstatehash, &StateEventType::RoomMember, sender_user.as_str()) @@ -1148,7 +1157,7 @@ pub(crate) async fn sync_events_v4_route( .ok() }); - let encrypted_room = services() + let encrypted_room = services .rooms .state_accessor .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "")? @@ -1160,7 +1169,7 @@ pub(crate) async fn sync_events_v4_route( continue; } - let since_encryption = services().rooms.state_accessor.state_get( + let since_encryption = services.rooms.state_accessor.state_get( since_shortstatehash, &StateEventType::RoomEncryption, "", @@ -1171,12 +1180,12 @@ pub(crate) async fn sync_events_v4_route( let new_encrypted_room = encrypted_room && since_encryption.is_none(); if encrypted_room { - let current_state_ids = services() + let current_state_ids = services .rooms .state_accessor .state_full_ids(current_shortstatehash) .await?; - let since_state_ids = services() + let since_state_ids = services .rooms .state_accessor .state_full_ids(since_shortstatehash) @@ -1184,7 +1193,7 @@ pub(crate) async fn sync_events_v4_route( for (key, id) in current_state_ids { if since_state_ids.get(&key) != Some(&id) { - let Some(pdu) = services().rooms.timeline.get_pdu(&id)? else { + let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { error!("Pdu in state not found: {}", id); continue; }; @@ -1205,7 +1214,7 @@ pub(crate) async fn sync_events_v4_route( match new_membership { MembershipState::Join => { // A new user joined an encrypted room - if !share_encrypted_room(&sender_user, &user_id, room_id)? { + if !share_encrypted_room(services, &sender_user, &user_id, room_id)? { device_list_changes.insert(user_id); } }, @@ -1222,7 +1231,7 @@ pub(crate) async fn sync_events_v4_route( if joined_since_last_sync || new_encrypted_room { // If the user is in a new encrypted room, give them all joined users device_list_changes.extend( - services() + services .rooms .state_cache .room_members(room_id) @@ -1234,7 +1243,7 @@ pub(crate) async fn sync_events_v4_route( .filter(|user_id| { // Only send keys if the sender doesn't share an encrypted room with the target // already - !share_encrypted_room(&sender_user, user_id, room_id).unwrap_or(false) + !share_encrypted_room(services, &sender_user, user_id, room_id).unwrap_or(false) }), ); } @@ -1242,21 +1251,21 @@ pub(crate) async fn sync_events_v4_route( } // Look for device list updates in this room device_list_changes.extend( - services() + services .users .keys_changed(room_id.as_ref(), globalsince, None) .filter_map(Result::ok), ); } for user_id in left_encrypted_users { - let dont_share_encrypted_room = services() + let dont_share_encrypted_room = services .rooms .user .get_shared_rooms(vec![sender_user.clone(), user_id.clone()])? .filter_map(Result::ok) .filter_map(|other_room_id| { Some( - services() + services .rooms .state_accessor .room_state_get(&other_room_id, &StateEventType::RoomEncryption, "") @@ -1336,7 +1345,7 @@ pub(crate) async fn sync_events_v4_route( ); if let Some(conn_id) = &body.conn_id { - services().users.update_sync_known_rooms( + services.users.update_sync_known_rooms( sender_user.clone(), sender_device.clone(), conn_id.clone(), @@ -1349,7 +1358,7 @@ pub(crate) async fn sync_events_v4_route( let mut known_subscription_rooms = BTreeSet::new(); for (room_id, room) in &body.room_subscriptions { - if !services().rooms.metadata.exists(room_id)? { + if !services.rooms.metadata.exists(room_id)? { continue; } let todo_room = todo_rooms @@ -1375,7 +1384,7 @@ pub(crate) async fn sync_events_v4_route( } if let Some(conn_id) = &body.conn_id { - services().users.update_sync_known_rooms( + services.users.update_sync_known_rooms( sender_user.clone(), sender_device.clone(), conn_id.clone(), @@ -1386,7 +1395,7 @@ pub(crate) async fn sync_events_v4_route( } if let Some(conn_id) = &body.conn_id { - services().users.update_sync_subscriptions( + services.users.update_sync_subscriptions( sender_user.clone(), sender_device.clone(), conn_id.clone(), @@ -1398,7 +1407,7 @@ pub(crate) async fn sync_events_v4_route( for (room_id, (required_state_request, timeline_limit, roomsince)) in &todo_rooms { let roomsincecount = PduCount::Normal(*roomsince); - let (timeline_pdus, limited) = load_timeline(&sender_user, room_id, roomsincecount, *timeline_limit)?; + let (timeline_pdus, limited) = load_timeline(services, &sender_user, room_id, roomsincecount, *timeline_limit)?; if roomsince != &0 && timeline_pdus.is_empty() { continue; @@ -1431,7 +1440,7 @@ pub(crate) async fn sync_events_v4_route( let required_state = required_state_request .iter() .map(|state| { - services() + services .rooms .state_accessor .room_state_get(room_id, &state.0, &state.1) @@ -1442,7 +1451,7 @@ pub(crate) async fn sync_events_v4_route( .collect(); // Heroes - let heroes = services() + let heroes = services .rooms .state_cache .room_members(room_id) @@ -1450,7 +1459,7 @@ pub(crate) async fn sync_events_v4_route( .filter(|member| member != &sender_user) .map(|member| { Ok::<_, Error>( - services() + services .rooms .state_accessor .get_member(room_id, &member)? @@ -1491,11 +1500,11 @@ pub(crate) async fn sync_events_v4_route( rooms.insert( room_id.clone(), sync_events::v4::SlidingSyncRoom { - name: services().rooms.state_accessor.get_name(room_id)?.or(name), + name: services.rooms.state_accessor.get_name(room_id)?.or(name), avatar: if let Some(heroes_avatar) = heroes_avatar { ruma::JsOption::Some(heroes_avatar) } else { - match services().rooms.state_accessor.get_avatar(room_id)? { + match services.rooms.state_accessor.get_avatar(room_id)? { ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url), ruma::JsOption::Null => ruma::JsOption::Null, ruma::JsOption::Undefined => ruma::JsOption::Undefined, @@ -1506,7 +1515,7 @@ pub(crate) async fn sync_events_v4_route( invite_state: None, unread_notifications: UnreadNotificationsCount { highlight_count: Some( - services() + services .rooms .user .highlight_count(&sender_user, room_id)? @@ -1514,7 +1523,7 @@ pub(crate) async fn sync_events_v4_route( .expect("notification count can't go that high"), ), notification_count: Some( - services() + services .rooms .user .notification_count(&sender_user, room_id)? @@ -1527,7 +1536,7 @@ pub(crate) async fn sync_events_v4_route( prev_batch, limited, joined_count: Some( - services() + services .rooms .state_cache .room_joined_count(room_id)? @@ -1536,7 +1545,7 @@ pub(crate) async fn sync_events_v4_route( .unwrap_or_else(|_| uint!(0)), ), invited_count: Some( - services() + services .rooms .state_cache .room_invited_count(room_id)? @@ -1571,7 +1580,7 @@ pub(crate) async fn sync_events_v4_route( extensions: sync_events::v4::Extensions { to_device: if body.extensions.to_device.enabled.unwrap_or(false) { Some(sync_events::v4::ToDevice { - events: services() + events: services .users .get_to_device_events(&sender_user, &sender_device)?, next_batch: next_batch.to_string(), @@ -1584,7 +1593,7 @@ pub(crate) async fn sync_events_v4_route( changed: device_list_changes.into_iter().collect(), left: device_list_left.into_iter().collect(), }, - device_one_time_keys_count: services() + device_one_time_keys_count: services .users .count_one_time_keys(&sender_user, &sender_device)?, // Fallback keys are not yet supported @@ -1592,7 +1601,7 @@ pub(crate) async fn sync_events_v4_route( }, account_data: sync_events::v4::AccountData { global: if body.extensions.account_data.enabled.unwrap_or(false) { - services() + services .account_data .changes_since(None, &sender_user, globalsince)? .into_iter() diff --git a/src/api/client/tag.rs b/src/api/client/tag.rs index 9fb60bc6..301568e5 100644 --- a/src/api/client/tag.rs +++ b/src/api/client/tag.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use axum::extract::State; use ruma::{ api::client::tag::{create_tag, delete_tag, get_tags}, events::{ @@ -8,17 +9,19 @@ use ruma::{ }, }; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `PUT /_matrix/client/r0/user/{userId}/rooms/{roomId}/tags/{tag}` /// /// Adds a tag to the room. /// /// - Inserts the tag into the tag event of the room account data. -pub(crate) async fn update_tag_route(body: Ruma) -> Result { +pub(crate) async fn update_tag_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = services() + let event = services .account_data .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag)?; @@ -38,7 +41,7 @@ pub(crate) async fn update_tag_route(body: Ruma) -> Res .tags .insert(body.tag.clone().into(), body.tag_info.clone()); - services().account_data.update( + services.account_data.update( Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag, @@ -53,10 +56,12 @@ pub(crate) async fn update_tag_route(body: Ruma) -> Res /// Deletes a tag from the room. /// /// - Removes the tag from the tag event of the room account data. -pub(crate) async fn delete_tag_route(body: Ruma) -> Result { +pub(crate) async fn delete_tag_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = services() + let event = services .account_data .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag)?; @@ -73,7 +78,7 @@ pub(crate) async fn delete_tag_route(body: Ruma) -> Res tags_event.content.tags.remove(&body.tag.clone().into()); - services().account_data.update( + services.account_data.update( Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag, @@ -88,10 +93,12 @@ pub(crate) async fn delete_tag_route(body: Ruma) -> Res /// Returns tags on the room. /// /// - Gets the tag event of the room account data. -pub(crate) async fn get_tags_route(body: Ruma) -> Result { +pub(crate) async fn get_tags_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = services() + let event = services .account_data .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag)?; diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index 2895573a..8100f0e6 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -1,12 +1,15 @@ +use axum::extract::State; use ruma::{ api::client::{error::ErrorKind, threads::get_threads}, uint, }; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `GET /_matrix/client/r0/rooms/{roomId}/threads` -pub(crate) async fn get_threads_route(body: Ruma) -> Result { +pub(crate) async fn get_threads_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); // Use limit or else 10, with maximum 100 @@ -24,14 +27,14 @@ pub(crate) async fn get_threads_route(body: Ruma) -> R u64::MAX }; - let threads = services() + let threads = services .rooms .threads .threads_until(sender_user, &body.room_id, from, &body.include)? .take(limit) .filter_map(Result::ok) .filter(|(_, pdu)| { - services() + services .rooms .state_accessor .user_can_see_event(sender_user, &body.room_id, &pdu.event_id) diff --git a/src/api/client/to_device.rs b/src/api/client/to_device.rs index 011e08f7..8476ff41 100644 --- a/src/api/client/to_device.rs +++ b/src/api/client/to_device.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use axum::extract::State; use ruma::{ api::{ client::{error::ErrorKind, to_device::send_event_to_device}, @@ -8,19 +9,19 @@ use ruma::{ to_device::DeviceIdOrAllDevices, }; -use crate::{services, user_is_local, Error, Result, Ruma}; +use crate::{user_is_local, Error, Result, Ruma}; /// # `PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}` /// /// Send a to-device event to a set of client devices. pub(crate) async fn send_event_to_device_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_deref(); // Check if this is a new transaction id - if services() + if services .transaction_ids .existing_txnid(sender_user, sender_device, &body.txn_id)? .is_some() @@ -35,9 +36,9 @@ pub(crate) async fn send_event_to_device_route( map.insert(target_device_id_maybe.clone(), event.clone()); let mut messages = BTreeMap::new(); messages.insert(target_user_id.clone(), map); - let count = services().globals.next_count()?; + let count = services.globals.next_count()?; - services().sending.send_edu_server( + services.sending.send_edu_server( target_user_id.server_name(), serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice(DirectDeviceContent { sender: sender_user.clone(), @@ -53,7 +54,7 @@ pub(crate) async fn send_event_to_device_route( match target_device_id_maybe { DeviceIdOrAllDevices::DeviceId(target_device_id) => { - services().users.add_to_device_event( + services.users.add_to_device_event( sender_user, target_user_id, target_device_id, @@ -65,8 +66,8 @@ pub(crate) async fn send_event_to_device_route( }, DeviceIdOrAllDevices::AllDevices => { - for target_device_id in services().users.all_device_ids(target_user_id) { - services().users.add_to_device_event( + for target_device_id in services.users.all_device_ids(target_user_id) { + services.users.add_to_device_event( sender_user, target_user_id, &target_device_id?, @@ -82,7 +83,7 @@ pub(crate) async fn send_event_to_device_route( } // Save transaction id with empty data - services() + services .transaction_ids .add_txnid(sender_user, sender_device, &body.txn_id, &[])?; diff --git a/src/api/client/typing.rs b/src/api/client/typing.rs index 52c8b353..a06648e0 100644 --- a/src/api/client/typing.rs +++ b/src/api/client/typing.rs @@ -1,18 +1,19 @@ +use axum::extract::State; use ruma::api::client::{error::ErrorKind, typing::create_typing_event}; -use crate::{services, utils, Error, Result, Ruma}; +use crate::{utils, Error, Result, Ruma}; /// # `PUT /_matrix/client/r0/rooms/{roomId}/typing/{userId}` /// /// Sets the typing state of the sender user. pub(crate) async fn create_typing_event_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { use create_typing_event::v3::Typing; let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services() + if !services .rooms .state_cache .is_joined(sender_user, &body.room_id)? @@ -23,20 +24,20 @@ pub(crate) async fn create_typing_event_route( if let Typing::Yes(duration) = body.state { let duration = utils::clamp( duration.as_millis().try_into().unwrap_or(u64::MAX), - services() + services .globals .config .typing_client_timeout_min_s .checked_mul(1000) .unwrap(), - services() + services .globals .config .typing_client_timeout_max_s .checked_mul(1000) .unwrap(), ); - services() + services .rooms .typing .typing_add( @@ -48,7 +49,7 @@ pub(crate) async fn create_typing_event_route( ) .await?; } else { - services() + services .rooms .typing .typing_remove(sender_user, &body.room_id) diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index e39db94e..89650951 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::warn; use ruma::{ @@ -6,7 +7,7 @@ use ruma::{ OwnedRoomId, }; -use crate::{services, Error, Result, Ruma, RumaResponse}; +use crate::{Error, Result, Ruma, RumaResponse}; /// # `GET /_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms` /// @@ -17,7 +18,8 @@ use crate::{services, Error, Result, Ruma, RumaResponse}; /// An implementation of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) #[tracing::instrument(skip_all, fields(%client), name = "mutual_rooms")] pub(crate) async fn get_mutual_rooms_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -28,14 +30,14 @@ pub(crate) async fn get_mutual_rooms_route( )); } - if !services().users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id)? { return Ok(mutual_rooms::unstable::Response { joined: vec![], next_batch_token: None, }); } - let mutual_rooms: Vec = services() + let mutual_rooms: Vec = services .rooms .user .get_shared_rooms(vec![sender_user.clone(), body.user_id.clone()])? @@ -58,9 +60,10 @@ pub(crate) async fn get_mutual_rooms_route( /// /// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) pub(crate) async fn get_room_summary_legacy( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result> { - get_room_summary(InsecureClientIp(client), body) + get_room_summary(State(services), InsecureClientIp(client), body) .await .map(RumaResponse) } @@ -74,22 +77,19 @@ pub(crate) async fn get_room_summary_legacy( /// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) #[tracing::instrument(skip_all, fields(%client), name = "room_summary")] pub(crate) async fn get_room_summary( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref(); - let room_id = services() - .rooms - .alias - .resolve(&body.room_id_or_alias) - .await?; + let room_id = services.rooms.alias.resolve(&body.room_id_or_alias).await?; - if !services().rooms.metadata.exists(&room_id)? { + if !services.rooms.metadata.exists(&room_id)? { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server")); } if sender_user.is_none() - && !services() + && !services .rooms .state_accessor .is_world_readable(&room_id) @@ -103,25 +103,25 @@ pub(crate) async fn get_room_summary( Ok(get_summary::msc3266::Response { room_id: room_id.clone(), - canonical_alias: services() + canonical_alias: services .rooms .state_accessor .get_canonical_alias(&room_id) .unwrap_or(None), - avatar_url: services() + avatar_url: services .rooms .state_accessor .get_avatar(&room_id)? .into_option() .unwrap_or_default() .url, - guest_can_join: services().rooms.state_accessor.guest_can_join(&room_id)?, - name: services() + guest_can_join: services.rooms.state_accessor.guest_can_join(&room_id)?, + name: services .rooms .state_accessor .get_name(&room_id) .unwrap_or(None), - num_joined_members: services() + num_joined_members: services .rooms .state_cache .room_joined_count(&room_id) @@ -132,21 +132,21 @@ pub(crate) async fn get_room_summary( }) .try_into() .expect("user count should not be that big"), - topic: services() + topic: services .rooms .state_accessor .get_room_topic(&room_id) .unwrap_or(None), - world_readable: services() + world_readable: services .rooms .state_accessor .is_world_readable(&room_id) .unwrap_or(false), - join_rule: services().rooms.state_accessor.get_join_rule(&room_id)?.0, - room_type: services().rooms.state_accessor.get_room_type(&room_id)?, - room_version: Some(services().rooms.state.get_room_version(&room_id)?), + join_rule: services.rooms.state_accessor.get_join_rule(&room_id)?.0, + room_type: services.rooms.state_accessor.get_room_type(&room_id)?, + room_version: Some(services.rooms.state.get_room_version(&room_id)?), membership: if let Some(sender_user) = sender_user { - services() + services .rooms .state_accessor .get_member(&room_id, sender_user)? @@ -154,7 +154,7 @@ pub(crate) async fn get_room_summary( } else { None }, - encryption: services() + encryption: services .rooms .state_accessor .get_room_encryption(&room_id) diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 2b703f94..9a8f3220 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use axum::{response::IntoResponse, Json}; +use axum::{extract::State, response::IntoResponse, Json}; use ruma::api::client::{ discovery::{ discover_homeserver::{self, HomeserverInfo, SlidingSyncProxyInfo}, @@ -10,7 +10,7 @@ use ruma::api::client::{ error::ErrorKind, }; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `GET /_matrix/client/versions` /// @@ -62,9 +62,9 @@ pub(crate) async fn get_supported_versions_route( /// /// Returns the .well-known URL if it is configured, otherwise returns 404. pub(crate) async fn well_known_client( - _body: Ruma, + State(services): State, _body: Ruma, ) -> Result { - let client_url = match services().globals.well_known_client() { + let client_url = match services.globals.well_known_client() { Some(url) => url.to_string(), None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), }; @@ -84,22 +84,24 @@ pub(crate) async fn well_known_client( /// # `GET /.well-known/matrix/support` /// /// Server support contact and support page of a homeserver's domain. -pub(crate) async fn well_known_support(_body: Ruma) -> Result { - let support_page = services() +pub(crate) async fn well_known_support( + State(services): State, _body: Ruma, +) -> Result { + let support_page = services .globals .well_known_support_page() .as_ref() .map(ToString::to_string); - let role = services().globals.well_known_support_role().clone(); + let role = services.globals.well_known_support_role().clone(); // support page or role must be either defined for this to be valid if support_page.is_none() && role.is_none() { return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); } - let email_address = services().globals.well_known_support_email().clone(); - let matrix_id = services().globals.well_known_support_mxid().clone(); + let email_address = services.globals.well_known_support_email().clone(); + let matrix_id = services.globals.well_known_support_mxid().clone(); // if a role is specified, an email address or matrix id is required if role.is_some() && (email_address.is_none() && matrix_id.is_none()) { @@ -134,10 +136,10 @@ pub(crate) async fn well_known_support(_body: Ruma) - /// /// Endpoint provided by sliding sync proxy used by some clients such as Element /// Web as a non-standard health check. -pub(crate) async fn syncv3_client_server_json() -> Result { - let server_url = match services().globals.well_known_client() { +pub(crate) async fn syncv3_client_server_json(State(services): State) -> Result { + let server_url = match services.globals.well_known_client() { Some(url) => url.to_string(), - None => match services().globals.well_known_server() { + None => match services.globals.well_known_server() { Some(url) => url.to_string(), None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), }, @@ -165,8 +167,8 @@ pub(crate) async fn conduwuit_server_version() -> Result { /// conduwuit-specific API to return the amount of users registered on this /// homeserver. Endpoint is disabled if federation is disabled for privacy. This /// only includes active users (not deactivated, no guests, etc) -pub(crate) async fn conduwuit_local_user_count() -> Result { - let user_count = services().users.list_local_users()?.len(); +pub(crate) async fn conduwuit_local_user_count(State(services): State) -> Result { + let user_count = services.users.list_local_users()?.len(); Ok(Json(serde_json::json!({ "count": user_count diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index ca5f834a..87d4062c 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use ruma::{ api::client::user_directory::search_users, events::{ @@ -6,7 +7,7 @@ use ruma::{ }, }; -use crate::{services, Result, Ruma}; +use crate::{Result, Ruma}; /// # `POST /_matrix/client/r0/user_directory/search` /// @@ -14,18 +15,20 @@ use crate::{services, Result, Ruma}; /// /// - Hides any local users that aren't in any public rooms (i.e. those that /// have the join rule set to public) and don't share a room with the sender -pub(crate) async fn search_users_route(body: Ruma) -> Result { +pub(crate) async fn search_users_route( + State(services): State, body: Ruma, +) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let limit = usize::try_from(body.limit).unwrap_or(10); // default limit is 10 - let mut users = services().users.iter().filter_map(|user_id| { + let mut users = services.users.iter().filter_map(|user_id| { // Filter out buggy users (they should not exist, but you never know...) let user_id = user_id.ok()?; let user = search_users::v3::User { user_id: user_id.clone(), - display_name: services().users.displayname(&user_id).ok()?, - avatar_url: services().users.avatar_url(&user_id).ok()?, + display_name: services.users.displayname(&user_id).ok()?, + avatar_url: services.users.avatar_url(&user_id).ok()?, }; let user_id_matches = user @@ -50,13 +53,13 @@ pub(crate) async fn search_users_route(body: Ruma) -> // It's a matching user, but is the sender allowed to see them? let mut user_visible = false; - let user_is_in_public_rooms = services() + let user_is_in_public_rooms = services .rooms .state_cache .rooms_joined(&user_id) .filter_map(Result::ok) .any(|room| { - services() + services .rooms .state_accessor .room_state_get(&room, &StateEventType::RoomJoinRules, "") @@ -71,7 +74,7 @@ pub(crate) async fn search_users_route(body: Ruma) -> if user_is_in_public_rooms { user_visible = true; } else { - let user_is_in_shared_rooms = services() + let user_is_in_shared_rooms = services .rooms .user .get_shared_rooms(vec![sender_user.clone(), user_id]) diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index 9608dc88..ed6971ee 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -1,12 +1,13 @@ use std::time::{Duration, SystemTime}; +use axum::extract::State; use base64::{engine::general_purpose, Engine as _}; use conduit::utils; use hmac::{Hmac, Mac}; use ruma::{api::client::voip::get_turn_server_info, SecondsSinceUnixEpoch, UserId}; use sha1::Sha1; -use crate::{services, Result, Ruma}; +use crate::{Result, Ruma}; const RANDOM_USER_ID_LENGTH: usize = 10; @@ -16,14 +17,14 @@ type HmacSha1 = Hmac; /// /// TODO: Returns information about the recommended turn server. pub(crate) async fn turn_server_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { - let turn_secret = services().globals.turn_secret().clone(); + let turn_secret = services.globals.turn_secret().clone(); let (username, password) = if !turn_secret.is_empty() { let expiry = SecondsSinceUnixEpoch::from_system_time( SystemTime::now() - .checked_add(Duration::from_secs(services().globals.turn_ttl())) + .checked_add(Duration::from_secs(services.globals.turn_ttl())) .expect("TURN TTL should not get this high"), ) .expect("time is valid"); @@ -31,7 +32,7 @@ pub(crate) async fn turn_server_route( let user = body.sender_user.unwrap_or_else(|| { UserId::parse_with_server_name( utils::random_string(RANDOM_USER_ID_LENGTH).to_lowercase(), - &services().globals.config.server_name, + &services.globals.config.server_name, ) .unwrap() }); @@ -46,15 +47,15 @@ pub(crate) async fn turn_server_route( (username, password) } else { ( - services().globals.turn_username().clone(), - services().globals.turn_password().clone(), + services.globals.turn_username().clone(), + services.globals.turn_password().clone(), ) }; Ok(get_turn_server_info::v3::Response { username, password, - uris: services().globals.turn_uris().to_vec(), - ttl: Duration::from_secs(services().globals.turn_ttl()), + uris: services.globals.turn_uris().to_vec(), + ttl: Duration::from_secs(services.globals.turn_ttl()), }) } diff --git a/src/api/router/args.rs b/src/api/router/args.rs index 776ce4f4..fa5b1e43 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -2,11 +2,12 @@ use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, BytesMut}; -use conduit::{debug, err, trace, Error, Result}; +use conduit::{debug, err, trace, utils::string::EMPTY, Error, Result}; use ruma::{api::IncomingRequest, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId}; +use service::Services; use super::{auth, auth::Auth, request, request::Request}; -use crate::{service::appservice::RegistrationInfo, services}; +use crate::service::appservice::RegistrationInfo; /// Extractor for Ruma request structs pub(crate) struct Args { @@ -42,11 +43,12 @@ where type Rejection = Error; async fn from_request(request: hyper::Request, _: &S) -> Result { - let mut request = request::from(request).await?; + let services = service::services(); // ??? + let mut request = request::from(services, request).await?; let mut json_body = serde_json::from_slice::(&request.body).ok(); - let auth = auth::auth(&mut request, &json_body, &T::METADATA).await?; + let auth = auth::auth(services, &mut request, &json_body, &T::METADATA).await?; Ok(Self { - body: make_body::(&mut request, &mut json_body, &auth)?, + body: make_body::(services, &mut request, &mut json_body, &auth)?, origin: auth.origin, sender_user: auth.sender_user, sender_device: auth.sender_device, @@ -62,13 +64,16 @@ impl Deref for Args { fn deref(&self) -> &Self::Target { &self.body } } -fn make_body(request: &mut Request, json_body: &mut Option, auth: &Auth) -> Result +fn make_body( + services: &Services, request: &mut Request, json_body: &mut Option, auth: &Auth, +) -> Result where T: IncomingRequest, { let body = if let Some(CanonicalJsonValue::Object(json_body)) = json_body { let user_id = auth.sender_user.clone().unwrap_or_else(|| { - UserId::parse_with_server_name("", services().globals.server_name()).expect("we know this is valid") + let server_name = services.globals.server_name(); + UserId::parse_with_server_name(EMPTY, server_name).expect("valid user_id") }); let uiaa_request = json_body @@ -77,9 +82,9 @@ where .and_then(|auth| auth.get("session")) .and_then(|session| session.as_str()) .and_then(|session| { - services().uiaa.get_uiaa_request( + services.uiaa.get_uiaa_request( &user_id, - &auth.sender_device.clone().unwrap_or_else(|| "".into()), + &auth.sender_device.clone().unwrap_or_else(|| EMPTY.into()), session, ) }); diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 6c2922b9..3f08ddba 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -6,17 +6,17 @@ use axum_extra::{ typed_header::TypedHeaderRejectionReason, TypedHeader, }; -use conduit::Err; +use conduit::{warn, Err, Error, Result}; use http::uri::PathAndQuery; use ruma::{ api::{client::error::ErrorKind, AuthScheme, Metadata}, server_util::authorization::XMatrix, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; -use tracing::warn; +use service::Services; use super::request::Request; -use crate::{service::appservice::RegistrationInfo, services, Error, Result}; +use crate::service::appservice::RegistrationInfo; enum Token { Appservice(Box), @@ -33,7 +33,7 @@ pub(super) struct Auth { } pub(super) async fn auth( - request: &mut Request, json_body: &Option, metadata: &Metadata, + services: &Services, request: &mut Request, json_body: &Option, metadata: &Metadata, ) -> Result { let bearer: Option>> = request.parts.extract().await?; let token = match &bearer { @@ -42,9 +42,9 @@ pub(super) async fn auth( }; let token = if let Some(token) = token { - if let Some(reg_info) = services().appservice.find_from_token(token).await { + if let Some(reg_info) = services.appservice.find_from_token(token).await { Token::Appservice(Box::new(reg_info)) - } else if let Some((user_id, device_id)) = services().users.find_from_token(token)? { + } else if let Some((user_id, device_id)) = services.users.find_from_token(token)? { Token::User((user_id, OwnedDeviceId::from(device_id))) } else { Token::Invalid @@ -57,7 +57,7 @@ pub(super) async fn auth( match request.parts.uri.path() { // TODO: can we check this better? "/_matrix/client/v3/publicRooms" | "/_matrix/client/r0/publicRooms" => { - if !services() + if !services .globals .config .allow_public_room_directory_without_auth @@ -98,7 +98,7 @@ pub(super) async fn auth( )) } }, - (AuthScheme::AccessToken, Token::Appservice(info)) => Ok(auth_appservice(request, info)?), + (AuthScheme::AccessToken, Token::Appservice(info)) => Ok(auth_appservice(services, request, info)?), (AuthScheme::None | AuthScheme::AccessTokenOptional | AuthScheme::AppserviceToken, Token::Appservice(info)) => { Ok(Auth { origin: None, @@ -110,7 +110,7 @@ pub(super) async fn auth( (AuthScheme::AccessToken, Token::None) => match request.parts.uri.path() { // TODO: can we check this better? "/_matrix/client/v3/voip/turnServer" | "/_matrix/client/r0/voip/turnServer" => { - if services().globals.config.turn_allow_guests { + if services.globals.config.turn_allow_guests { Ok(Auth { origin: None, sender_user: None, @@ -132,7 +132,7 @@ pub(super) async fn auth( sender_device: Some(device_id), appservice_info: None, }), - (AuthScheme::ServerSignatures, Token::None) => Ok(auth_server(request, json_body).await?), + (AuthScheme::ServerSignatures, Token::None) => Ok(auth_server(services, request, json_body).await?), (AuthScheme::None | AuthScheme::AppserviceToken | AuthScheme::AccessTokenOptional, Token::None) => Ok(Auth { sender_user: None, sender_device: None, @@ -150,7 +150,7 @@ pub(super) async fn auth( } } -fn auth_appservice(request: &Request, info: Box) -> Result { +fn auth_appservice(services: &Services, request: &Request, info: Box) -> Result { let user_id = request .query .user_id @@ -159,7 +159,7 @@ fn auth_appservice(request: &Request, info: Box) -> Result) -> Result) -> Result) -> Result { - if !services().globals.allow_federation() { +async fn auth_server( + services: &Services, request: &mut Request, json_body: &Option, +) -> Result { + if !services.globals.allow_federation() { return Err!(Config("allow_federation", "Federation is disabled.")); } @@ -216,7 +218,7 @@ async fn auth_server(request: &mut Request, json_body: &Option) -> Result { +pub(super) async fn from(services: &Services, request: hyper::Request) -> Result { let limited = request.with_limited_body(); let (mut parts, body) = limited.into_parts(); @@ -30,7 +29,7 @@ pub(super) async fn from(request: hyper::Request) -> Result) -> Result { +pub(crate) async fn get_backfill_route( + State(services): State, body: Ruma, +) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - services() + services .rooms .event_handler .acl_check(origin, &body.room_id)?; - if !services() + if !services .rooms .state_accessor .is_world_readable(&body.room_id)? - && !services() + && !services .rooms .state_cache .server_in_room(origin, &body.room_id)? @@ -34,7 +37,7 @@ pub(crate) async fn get_backfill_route(body: Ruma) -> let until = body .v .iter() - .map(|event_id| services().rooms.timeline.get_pdu_count(event_id)) + .map(|event_id| services.rooms.timeline.get_pdu_count(event_id)) .filter_map(|r| r.ok().flatten()) .max() .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event not found."))?; @@ -45,7 +48,7 @@ pub(crate) async fn get_backfill_route(body: Ruma) -> .try_into() .expect("UInt could not be converted to usize"); - let all_events = services() + let all_events = services .rooms .timeline .pdus_until(user_id!("@doesntmatter:conduit.rs"), &body.room_id, until)? @@ -55,20 +58,20 @@ pub(crate) async fn get_backfill_route(body: Ruma) -> .filter_map(Result::ok) .filter(|(_, e)| { matches!( - services() + services .rooms .state_accessor .server_can_see_event(origin, &e.room_id, &e.event_id,), Ok(true), ) }) - .map(|(_, pdu)| services().rooms.timeline.get_pdu_json(&pdu.event_id)) + .map(|(_, pdu)| services.rooms.timeline.get_pdu_json(&pdu.event_id)) .filter_map(|r| r.ok().flatten()) .map(convert_to_outgoing_federation_event) .collect(); Ok(get_backfill::v1::Response { - origin: services().globals.server_name().to_owned(), + origin: services.globals.server_name().to_owned(), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), pdus: events, }) diff --git a/src/api/server/event.rs b/src/api/server/event.rs index f4c9d145..e8e08c81 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -1,9 +1,10 @@ +use axum::extract::State; use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_event}, MilliSecondsSinceUnixEpoch, RoomId, }; -use service::{sending::convert_to_outgoing_federation_event, services}; +use service::sending::convert_to_outgoing_federation_event; use crate::Ruma; @@ -13,10 +14,12 @@ use crate::Ruma; /// /// - Only works if a user of this server is currently invited or joined the /// room -pub(crate) async fn get_event_route(body: Ruma) -> Result { +pub(crate) async fn get_event_route( + State(services): State, body: Ruma, +) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - let event = services() + let event = services .rooms .timeline .get_pdu_json(&body.event_id)? @@ -30,16 +33,13 @@ pub(crate) async fn get_event_route(body: Ruma) -> Resul let room_id = <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; - if !services().rooms.state_accessor.is_world_readable(room_id)? - && !services() - .rooms - .state_cache - .server_in_room(origin, room_id)? + if !services.rooms.state_accessor.is_world_readable(room_id)? + && !services.rooms.state_cache.server_in_room(origin, room_id)? { return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); } - if !services() + if !services .rooms .state_accessor .server_can_see_event(origin, room_id, &body.event_id)? @@ -48,7 +48,7 @@ pub(crate) async fn get_event_route(body: Ruma) -> Resul } Ok(get_event::v1::Response { - origin: services().globals.server_name().to_owned(), + origin: services.globals.server_name().to_owned(), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), pdu: convert_to_outgoing_federation_event(event), }) diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index bef5116b..8d26b73a 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -1,11 +1,12 @@ use std::sync::Arc; +use axum::extract::State; use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::authorization::get_event_authorization}, RoomId, }; -use service::{sending::convert_to_outgoing_federation_event, services}; +use service::sending::convert_to_outgoing_federation_event; use crate::Ruma; @@ -15,20 +16,20 @@ use crate::Ruma; /// /// - This does not include the event itself pub(crate) async fn get_event_authorization_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - services() + services .rooms .event_handler .acl_check(origin, &body.room_id)?; - if !services() + if !services .rooms .state_accessor .is_world_readable(&body.room_id)? - && !services() + && !services .rooms .state_cache .server_in_room(origin, &body.room_id)? @@ -36,7 +37,7 @@ pub(crate) async fn get_event_authorization_route( return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); } - let event = services() + let event = services .rooms .timeline .get_pdu_json(&body.event_id)? @@ -50,7 +51,7 @@ pub(crate) async fn get_event_authorization_route( let room_id = <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; - let auth_chain_ids = services() + let auth_chain_ids = services .rooms .auth_chain .event_ids_iter(room_id, vec![Arc::from(&*body.event_id)]) @@ -58,7 +59,7 @@ pub(crate) async fn get_event_authorization_route( Ok(get_event_authorization::v1::Response { auth_chain: auth_chain_ids - .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok()?) + .filter_map(|id| services.rooms.timeline.get_pdu_json(&id).ok()?) .map(convert_to_outgoing_federation_event) .collect(), }) diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index 5ab9abf8..378cd4fe 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -1,9 +1,10 @@ +use axum::extract::State; use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_missing_events}, OwnedEventId, RoomId, }; -use service::{sending::convert_to_outgoing_federation_event, services}; +use service::sending::convert_to_outgoing_federation_event; use crate::Ruma; @@ -11,20 +12,20 @@ use crate::Ruma; /// /// Retrieves events that the sender is missing. pub(crate) async fn get_missing_events_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - services() + services .rooms .event_handler .acl_check(origin, &body.room_id)?; - if !services() + if !services .rooms .state_accessor .is_world_readable(&body.room_id)? - && !services() + && !services .rooms .state_cache .server_in_room(origin, &body.room_id)? @@ -43,7 +44,7 @@ pub(crate) async fn get_missing_events_route( let mut i: usize = 0; while i < queued_events.len() && events.len() < limit { - if let Some(pdu) = services().rooms.timeline.get_pdu_json(&queued_events[i])? { + if let Some(pdu) = services.rooms.timeline.get_pdu_json(&queued_events[i])? { let room_id_str = pdu .get("room_id") .and_then(|val| val.as_str()) @@ -61,7 +62,7 @@ pub(crate) async fn get_missing_events_route( continue; } - if !services() + if !services .rooms .state_accessor .server_can_see_event(origin, &body.room_id, &queued_events[i])? diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs index 84b30e80..530ed145 100644 --- a/src/api/server/hierarchy.rs +++ b/src/api/server/hierarchy.rs @@ -1,16 +1,19 @@ +use axum::extract::State; use ruma::api::{client::error::ErrorKind, federation::space::get_hierarchy}; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `GET /_matrix/federation/v1/hierarchy/{roomId}` /// /// Gets the space tree in a depth-first manner to locate child rooms of a given /// space. -pub(crate) async fn get_hierarchy_route(body: Ruma) -> Result { +pub(crate) async fn get_hierarchy_route( + State(services): State, body: Ruma, +) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - if services().rooms.metadata.exists(&body.room_id)? { - services() + if services.rooms.metadata.exists(&body.room_id)? { + services .rooms .spaces .get_federation_hierarchy(&body.room_id, origin, body.suggested_only) diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 89b90058..982a2a01 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{utils, warn, Error, PduEvent, Result}; use ruma::{ @@ -6,7 +7,7 @@ use ruma::{ serde::JsonObject, CanonicalJsonValue, EventId, OwnedUserId, }; -use service::{sending::convert_to_outgoing_federation_event, server_is_ours, services}; +use service::{sending::convert_to_outgoing_federation_event, server_is_ours}; use crate::Ruma; @@ -15,17 +16,18 @@ use crate::Ruma; /// Invites a remote user to a room. #[tracing::instrument(skip_all, fields(%client), name = "invite")] pub(crate) async fn create_invite_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); // ACL check origin - services() + services .rooms .event_handler .acl_check(origin, &body.room_id)?; - if !services() + if !services .globals .supported_room_versions() .contains(&body.room_version) @@ -39,7 +41,7 @@ pub(crate) async fn create_invite_route( } if let Some(server) = body.room_id.server_name() { - if services() + if services .globals .config .forbidden_remote_server_names @@ -52,7 +54,7 @@ pub(crate) async fn create_invite_route( } } - if services() + if services .globals .config .forbidden_remote_server_names @@ -94,14 +96,14 @@ pub(crate) async fn create_invite_route( } // Make sure we're not ACL'ed from their room. - services() + services .rooms .event_handler .acl_check(invited_user.server_name(), &body.room_id)?; ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), + services.globals.server_name().as_str(), + services.globals.keypair(), &mut signed_event, &body.room_version, ) @@ -127,14 +129,14 @@ pub(crate) async fn create_invite_route( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "sender is not a user ID."))?; - if services().rooms.metadata.is_banned(&body.room_id)? && !services().users.is_admin(&invited_user)? { + if services.rooms.metadata.is_banned(&body.room_id)? && !services.users.is_admin(&invited_user)? { return Err(Error::BadRequest( ErrorKind::forbidden(), "This room is banned on this homeserver.", )); } - if services().globals.block_non_admin_invites() && !services().users.is_admin(&invited_user)? { + if services.globals.block_non_admin_invites() && !services.users.is_admin(&invited_user)? { return Err(Error::BadRequest( ErrorKind::forbidden(), "This server does not allow room invites.", @@ -155,12 +157,12 @@ pub(crate) async fn create_invite_route( // If we are active in the room, the remote server will notify us about the join // via /send - if !services() + if !services .rooms .state_cache - .server_in_room(services().globals.server_name(), &body.room_id)? + .server_in_room(services.globals.server_name(), &body.room_id)? { - services().rooms.state_cache.update_membership( + services.rooms.state_cache.update_membership( &body.room_id, &invited_user, RoomMemberEventContent::new(MembershipState::Invite), diff --git a/src/api/server/key.rs b/src/api/server/key.rs index 27d86639..686e4424 100644 --- a/src/api/server/key.rs +++ b/src/api/server/key.rs @@ -3,7 +3,7 @@ use std::{ time::{Duration, SystemTime}, }; -use axum::{response::IntoResponse, Json}; +use axum::{extract::State, response::IntoResponse, Json}; use ruma::{ api::{ federation::discovery::{get_server_keys, ServerSigningKeys, VerifyKey}, @@ -13,7 +13,7 @@ use ruma::{ MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, }; -use crate::{services, Result}; +use crate::Result; /// # `GET /_matrix/key/v2/server` /// @@ -23,20 +23,20 @@ use crate::{services, Result}; /// this will be valid forever. // Response type for this endpoint is Json because we need to calculate a // signature for the response -pub(crate) async fn get_server_keys_route() -> Result { +pub(crate) async fn get_server_keys_route(State(services): State) -> Result { let verify_keys: BTreeMap = BTreeMap::from([( - format!("ed25519:{}", services().globals.keypair().version()) + format!("ed25519:{}", services.globals.keypair().version()) .try_into() .expect("found invalid server signing keys in DB"), VerifyKey { - key: Base64::new(services().globals.keypair().public_key().to_vec()), + key: Base64::new(services.globals.keypair().public_key().to_vec()), }, )]); let mut response = serde_json::from_slice( get_server_keys::v2::Response { server_key: Raw::new(&ServerSigningKeys { - server_name: services().globals.server_name().to_owned(), + server_name: services.globals.server_name().to_owned(), verify_keys, old_verify_keys: BTreeMap::new(), signatures: BTreeMap::new(), @@ -56,8 +56,8 @@ pub(crate) async fn get_server_keys_route() -> Result { .unwrap(); ruma::signatures::sign_json( - services().globals.server_name().as_str(), - services().globals.keypair(), + services.globals.server_name().as_str(), + services.globals.keypair(), &mut response, ) .unwrap(); @@ -71,4 +71,6 @@ pub(crate) async fn get_server_keys_route() -> Result { /// /// - Matrix does not support invalidating public keys, so the key returned by /// this will be valid forever. -pub(crate) async fn get_server_keys_deprecated_route() -> impl IntoResponse { get_server_keys_route().await } +pub(crate) async fn get_server_keys_deprecated_route(State(services): State) -> impl IntoResponse { + get_server_keys_route(State(services)).await +} diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index b5dadf7f..e1beaa33 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_join_event}, events::{ @@ -12,15 +13,18 @@ use ruma::{ use serde_json::value::to_raw_value; use tracing::warn; -use crate::{service::pdu::PduBuilder, services, Error, Result, Ruma}; +use crate::{ + service::{pdu::PduBuilder, Services}, + Error, Result, Ruma, +}; /// # `GET /_matrix/federation/v1/make_join/{roomId}/{userId}` /// /// Creates a join template. pub(crate) async fn create_join_event_template_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { - if !services().rooms.metadata.exists(&body.room_id)? { + if !services.rooms.metadata.exists(&body.room_id)? { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } @@ -33,12 +37,12 @@ pub(crate) async fn create_join_event_template_route( } // ACL check origin server - services() + services .rooms .event_handler .acl_check(origin, &body.room_id)?; - if services() + if services .globals .config .forbidden_remote_server_names @@ -56,7 +60,7 @@ pub(crate) async fn create_join_event_template_route( } if let Some(server) = body.room_id.server_name() { - if services() + if services .globals .config .forbidden_remote_server_names @@ -69,25 +73,25 @@ pub(crate) async fn create_join_event_template_route( } } - let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; + let room_version_id = services.rooms.state.get_room_version(&body.room_id)?; - let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let join_authorized_via_users_server = if (services() + let join_authorized_via_users_server = if (services .rooms .state_cache .is_left(&body.user_id, &body.room_id) .unwrap_or(true)) - && user_can_perform_restricted_join(&body.user_id, &body.room_id, &room_version_id)? + && user_can_perform_restricted_join(services, &body.user_id, &body.room_id, &room_version_id)? { - let auth_user = services() + let auth_user = services .rooms .state_cache .room_members(&body.room_id) .filter_map(Result::ok) - .filter(|user| user.server_name() == services().globals.server_name()) + .filter(|user| user.server_name() == services.globals.server_name()) .find(|user| { - services() + services .rooms .state_accessor .user_can_invite(&body.room_id, user, &body.user_id, &state_lock) @@ -106,7 +110,7 @@ pub(crate) async fn create_join_event_template_route( None }; - let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; + let room_version_id = services.rooms.state.get_room_version(&body.room_id)?; if !body.ver.contains(&room_version_id) { return Err(Error::BadRequest( ErrorKind::IncompatibleRoomVersion { @@ -128,7 +132,7 @@ pub(crate) async fn create_join_event_template_route( }) .expect("member event is valid value"); - let (_pdu, mut pdu_json) = services().rooms.timeline.create_hash_and_sign_event( + let (_pdu, mut pdu_json) = services.rooms.timeline.create_hash_and_sign_event( PduBuilder { event_type: TimelineEventType::RoomMember, content, @@ -157,15 +161,14 @@ pub(crate) async fn create_join_event_template_route( /// externally, either by using the state cache or attempting to authorize the /// event. pub(crate) fn user_can_perform_restricted_join( - user_id: &UserId, room_id: &RoomId, room_version_id: &RoomVersionId, + services: &Services, user_id: &UserId, room_id: &RoomId, room_version_id: &RoomVersionId, ) -> Result { use RoomVersionId::*; - let join_rules_event = - services() - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomJoinRules, "")?; + let join_rules_event = services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomJoinRules, "")?; let Some(join_rules_event_content) = join_rules_event .as_ref() @@ -198,7 +201,7 @@ pub(crate) fn user_can_perform_restricted_join( } }) .any(|m| { - services() + services .rooms .state_cache .is_joined(user_id, &m.room_id) diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 63fc2b2e..ae7237ad 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_leave_event}, @@ -9,15 +10,15 @@ use ruma::{ use serde_json::value::to_raw_value; use super::make_join::maybe_strip_event_id; -use crate::{service::pdu::PduBuilder, services, Ruma}; +use crate::{service::pdu::PduBuilder, Ruma}; /// # `PUT /_matrix/federation/v1/make_leave/{roomId}/{eventId}` /// /// Creates a leave template. pub(crate) async fn create_leave_event_template_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { - if !services().rooms.metadata.exists(&body.room_id)? { + if !services.rooms.metadata.exists(&body.room_id)? { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } @@ -30,13 +31,13 @@ pub(crate) async fn create_leave_event_template_route( } // ACL check origin - services() + services .rooms .event_handler .acl_check(origin, &body.room_id)?; - let room_version_id = services().rooms.state.get_room_version(&body.room_id)?; - let state_lock = services().rooms.state.mutex.lock(&body.room_id).await; + let room_version_id = services.rooms.state.get_room_version(&body.room_id)?; + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; let content = to_raw_value(&RoomMemberEventContent { avatar_url: None, blurhash: None, @@ -49,7 +50,7 @@ pub(crate) async fn create_leave_event_template_route( }) .expect("member event is valid value"); - let (_pdu, mut pdu_json) = services().rooms.timeline.create_hash_and_sign_event( + let (_pdu, mut pdu_json) = services.rooms.timeline.create_hash_and_sign_event( PduBuilder { event_type: TimelineEventType::RoomMember, content, diff --git a/src/api/server/openid.rs b/src/api/server/openid.rs index 322c7343..6a1b99b7 100644 --- a/src/api/server/openid.rs +++ b/src/api/server/openid.rs @@ -1,16 +1,15 @@ +use axum::extract::State; use ruma::api::federation::openid::get_openid_userinfo; -use crate::{services, Result, Ruma}; +use crate::{Result, Ruma}; /// # `GET /_matrix/federation/v1/openid/userinfo` /// /// Get information about the user that generated the OpenID token. pub(crate) async fn get_openid_userinfo_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { Ok(get_openid_userinfo::v1::Response::new( - services() - .users - .find_from_openid_token(&body.access_token)?, + services.users.find_from_openid_token(&body.access_token)?, )) } diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index 13edd8d6..1876dde1 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use axum_client_ip::InsecureClientIp; use ruma::{ api::{ @@ -7,16 +8,17 @@ use ruma::{ directory::Filter, }; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `POST /_matrix/federation/v1/publicRooms` /// /// Lists the public rooms on this server. #[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] pub(crate) async fn get_public_rooms_filtered_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { - if !services() + if !services .globals .allow_public_room_directory_over_federation() { @@ -24,6 +26,7 @@ pub(crate) async fn get_public_rooms_filtered_route( } let response = crate::client::get_public_rooms_filtered_helper( + services, None, body.limit, body.since.as_deref(), @@ -46,9 +49,10 @@ pub(crate) async fn get_public_rooms_filtered_route( /// Lists the public rooms on this server. #[tracing::instrument(skip_all, fields(%client), "publicrooms")] pub(crate) async fn get_public_rooms_route( - InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { - if !services() + if !services .globals .allow_public_room_directory_over_federation() { @@ -56,6 +60,7 @@ pub(crate) async fn get_public_rooms_route( } let response = crate::client::get_public_rooms_filtered_helper( + services, None, body.limit, body.since.as_deref(), diff --git a/src/api/server/query.rs b/src/api/server/query.rs index 89baedb3..dddf23e7 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use get_profile_information::v1::ProfileField; use rand::seq::SliceRandom; use ruma::{ @@ -8,21 +9,21 @@ use ruma::{ OwnedServerName, }; -use crate::{service::server_is_ours, services, Error, Result, Ruma}; +use crate::{service::server_is_ours, Error, Result, Ruma}; /// # `GET /_matrix/federation/v1/query/directory` /// /// Resolve a room alias to a room id. pub(crate) async fn get_room_information_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { - let room_id = services() + let room_id = services .rooms .alias .resolve_local_alias(&body.room_alias)? .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Room alias not found."))?; - let mut servers: Vec = services() + let mut servers: Vec = services .rooms .state_cache .room_servers(&room_id) @@ -37,10 +38,10 @@ pub(crate) async fn get_room_information_route( // insert our server as the very first choice if in list if let Some(server_index) = servers .iter() - .position(|server| server == services().globals.server_name()) + .position(|server| server == services.globals.server_name()) { servers.swap_remove(server_index); - servers.insert(0, services().globals.server_name().to_owned()); + servers.insert(0, services.globals.server_name().to_owned()); } Ok(get_room_information::v1::Response { @@ -54,12 +55,9 @@ pub(crate) async fn get_room_information_route( /// /// Gets information on a profile. pub(crate) async fn get_profile_information_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { - if !services() - .globals - .allow_profile_lookup_federation_requests() - { + if !services.globals.allow_profile_lookup_federation_requests() { return Err(Error::BadRequest( ErrorKind::forbidden(), "Profile lookup over federation is not allowed on this homeserver.", @@ -79,18 +77,18 @@ pub(crate) async fn get_profile_information_route( match &body.field { Some(ProfileField::DisplayName) => { - displayname = services().users.displayname(&body.user_id)?; + displayname = services.users.displayname(&body.user_id)?; }, Some(ProfileField::AvatarUrl) => { - avatar_url = services().users.avatar_url(&body.user_id)?; - blurhash = services().users.blurhash(&body.user_id)?; + avatar_url = services.users.avatar_url(&body.user_id)?; + blurhash = services.users.blurhash(&body.user_id)?; }, // TODO: what to do with custom Some(_) => {}, None => { - displayname = services().users.displayname(&body.user_id)?; - avatar_url = services().users.avatar_url(&body.user_id)?; - blurhash = services().users.blurhash(&body.user_id)?; + displayname = services.users.displayname(&body.user_id)?; + avatar_url = services.users.avatar_url(&body.user_id)?; + blurhash = services.users.blurhash(&body.user_id)?; }, } diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 7c699e95..f2934480 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -34,7 +34,7 @@ type ResolvedMap = BTreeMap>; /// Push EDUs and PDUs to this server. #[tracing::instrument(skip_all, fields(%client), name = "send")] pub(crate) async fn send_transaction_message_route( - State(services): State<&Services>, InsecureClientIp(client): InsecureClientIp, + State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 577833d5..b72bfa03 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; +use axum::extract::State; use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_join_event}, @@ -13,7 +14,7 @@ use ruma::{ }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::{ - pdu::gen_event_id_canonical_json, sending::convert_to_outgoing_federation_event, services, user_is_local, + pdu::gen_event_id_canonical_json, sending::convert_to_outgoing_federation_event, user_is_local, Services, }; use tokio::sync::RwLock; use tracing::warn; @@ -22,18 +23,18 @@ use crate::Ruma; /// helper method for /send_join v1 and v2 async fn create_join_event( - origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, + services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, ) -> Result { - if !services().rooms.metadata.exists(room_id)? { + if !services.rooms.metadata.exists(room_id)? { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } // ACL check origin server - services().rooms.event_handler.acl_check(origin, room_id)?; + services.rooms.event_handler.acl_check(origin, room_id)?; // We need to return the state prior to joining, let's keep a reference to that // here - let shortstatehash = services() + let shortstatehash = services .rooms .state .get_room_shortstatehash(room_id)? @@ -44,7 +45,7 @@ async fn create_join_event( // We do not add the event_id field to the pdu here because of signature and // hashes checks - let room_version_id = services().rooms.state.get_room_version(room_id)?; + let room_version_id = services.rooms.state.get_room_version(room_id)?; let Ok((event_id, mut value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json @@ -96,7 +97,7 @@ async fn create_join_event( ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "sender is not a valid user ID."))?; - services() + services .rooms .event_handler .acl_check(sender.server_name(), room_id)?; @@ -128,18 +129,18 @@ async fn create_join_event( if content .join_authorized_via_users_server .is_some_and(|user| user_is_local(&user)) - && super::user_can_perform_restricted_join(&sender, room_id, &room_version_id).unwrap_or_default() + && super::user_can_perform_restricted_join(services, &sender, room_id, &room_version_id).unwrap_or_default() { ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), + services.globals.server_name().as_str(), + services.globals.keypair(), &mut value, &room_version_id, ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; } - services() + services .rooms .event_handler .fetch_required_signing_keys([&value], &pub_key_map) @@ -155,13 +156,13 @@ async fn create_join_event( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; - let mutex_lock = services() + let mutex_lock = services .rooms .event_handler .mutex_federation .lock(room_id) .await; - let pdu_id: Vec = services() + let pdu_id: Vec = services .rooms .event_handler .handle_incoming_pdu(&origin, room_id, &event_id, value.clone(), true, &pub_key_map) @@ -169,27 +170,27 @@ async fn create_join_event( .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; drop(mutex_lock); - let state_ids = services() + let state_ids = services .rooms .state_accessor .state_full_ids(shortstatehash) .await?; - let auth_chain_ids = services() + let auth_chain_ids = services .rooms .auth_chain .event_ids_iter(room_id, state_ids.values().cloned().collect()) .await?; - services().sending.send_pdu_room(room_id, &pdu_id)?; + services.sending.send_pdu_room(room_id, &pdu_id)?; Ok(create_join_event::v1::RoomState { auth_chain: auth_chain_ids - .filter_map(|id| services().rooms.timeline.get_pdu_json(&id).ok().flatten()) + .filter_map(|id| services.rooms.timeline.get_pdu_json(&id).ok().flatten()) .map(convert_to_outgoing_federation_event) .collect(), state: state_ids .iter() - .filter_map(|(_, id)| services().rooms.timeline.get_pdu_json(id).ok().flatten()) + .filter_map(|(_, id)| services.rooms.timeline.get_pdu_json(id).ok().flatten()) .map(convert_to_outgoing_federation_event) .collect(), // Event field is required if the room version supports restricted join rules. @@ -204,11 +205,11 @@ async fn create_join_event( /// /// Submits a signed join event. pub(crate) async fn create_join_event_v1_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - if services() + if services .globals .config .forbidden_remote_server_names @@ -225,7 +226,7 @@ pub(crate) async fn create_join_event_v1_route( } if let Some(server) = body.room_id.server_name() { - if services() + if services .globals .config .forbidden_remote_server_names @@ -243,7 +244,7 @@ pub(crate) async fn create_join_event_v1_route( } } - let room_state = create_join_event(origin, &body.room_id, &body.pdu).await?; + let room_state = create_join_event(services, origin, &body.room_id, &body.pdu).await?; Ok(create_join_event::v1::Response { room_state, @@ -254,11 +255,11 @@ pub(crate) async fn create_join_event_v1_route( /// /// Submits a signed join event. pub(crate) async fn create_join_event_v2_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - if services() + if services .globals .config .forbidden_remote_server_names @@ -271,7 +272,7 @@ pub(crate) async fn create_join_event_v2_route( } if let Some(server) = body.room_id.server_name() { - if services() + if services .globals .config .forbidden_remote_server_names @@ -288,7 +289,7 @@ pub(crate) async fn create_join_event_v2_route( auth_chain, state, event, - } = create_join_event(origin, &body.room_id, &body.pdu).await?; + } = create_join_event(services, origin, &body.room_id, &body.pdu).await?; let room_state = create_join_event::v2::RoomState { members_omitted: false, auth_chain, diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index c4e17bbc..f7484a33 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; +use axum::extract::State; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_leave_event}, events::{ @@ -14,19 +15,19 @@ use serde_json::value::RawValue as RawJsonValue; use tokio::sync::RwLock; use crate::{ - service::{pdu::gen_event_id_canonical_json, server_is_ours}, - services, Error, Result, Ruma, + service::{pdu::gen_event_id_canonical_json, server_is_ours, Services}, + Error, Result, Ruma, }; /// # `PUT /_matrix/federation/v1/send_leave/{roomId}/{eventId}` /// /// Submits a signed leave event. pub(crate) async fn create_leave_event_v1_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - create_leave_event(origin, &body.room_id, &body.pdu).await?; + create_leave_event(services, origin, &body.room_id, &body.pdu).await?; Ok(create_leave_event::v1::Response::new()) } @@ -35,28 +36,30 @@ pub(crate) async fn create_leave_event_v1_route( /// /// Submits a signed leave event. pub(crate) async fn create_leave_event_v2_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - create_leave_event(origin, &body.room_id, &body.pdu).await?; + create_leave_event(services, origin, &body.room_id, &body.pdu).await?; Ok(create_leave_event::v2::Response::new()) } -async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue) -> Result<()> { - if !services().rooms.metadata.exists(room_id)? { +async fn create_leave_event( + services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, +) -> Result<()> { + if !services.rooms.metadata.exists(room_id)? { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } // ACL check origin - services().rooms.event_handler.acl_check(origin, room_id)?; + services.rooms.event_handler.acl_check(origin, room_id)?; let pub_key_map = RwLock::new(BTreeMap::new()); // We do not add the event_id field to the pdu here because of signature and // hashes checks - let room_version_id = services().rooms.state.get_room_version(room_id)?; + let room_version_id = services.rooms.state.get_room_version(room_id)?; let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json return Err(Error::BadRequest( @@ -107,7 +110,7 @@ async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJson ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "User ID in sender is invalid."))?; - services() + services .rooms .event_handler .acl_check(sender.server_name(), room_id)?; @@ -145,19 +148,19 @@ async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJson ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; - services() + services .rooms .event_handler .fetch_required_signing_keys([&value], &pub_key_map) .await?; - let mutex_lock = services() + let mutex_lock = services .rooms .event_handler .mutex_federation .lock(room_id) .await; - let pdu_id: Vec = services() + let pdu_id: Vec = services .rooms .event_handler .handle_incoming_pdu(&origin, room_id, &event_id, value, true, &pub_key_map) @@ -166,14 +169,14 @@ async fn create_leave_event(origin: &ServerName, room_id: &RoomId, pdu: &RawJson drop(mutex_lock); - let servers = services() + let servers = services .rooms .state_cache .room_servers(room_id) .filter_map(Result::ok) .filter(|server| !server_is_ours(server)); - services().sending.send_pdu_servers(servers, &pdu_id)?; + services.sending.send_pdu_servers(servers, &pdu_id)?; Ok(()) } diff --git a/src/api/server/state.rs b/src/api/server/state.rs index 22044840..24a11cca 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -1,8 +1,9 @@ use std::sync::Arc; +use axum::extract::State; use conduit::{Error, Result}; use ruma::api::{client::error::ErrorKind, federation::event::get_room_state}; -use service::{sending::convert_to_outgoing_federation_event, services}; +use service::sending::convert_to_outgoing_federation_event; use crate::Ruma; @@ -10,20 +11,20 @@ use crate::Ruma; /// /// Retrieves a snapshot of a room's state at a given event. pub(crate) async fn get_room_state_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - services() + services .rooms .event_handler .acl_check(origin, &body.room_id)?; - if !services() + if !services .rooms .state_accessor .is_world_readable(&body.room_id)? - && !services() + && !services .rooms .state_cache .server_in_room(origin, &body.room_id)? @@ -31,31 +32,22 @@ pub(crate) async fn get_room_state_route( return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); } - let shortstatehash = services() + let shortstatehash = services .rooms .state_accessor .pdu_shortstatehash(&body.event_id)? .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Pdu state not found."))?; - let pdus = services() + let pdus = services .rooms .state_accessor .state_full_ids(shortstatehash) .await? .into_values() - .map(|id| { - convert_to_outgoing_federation_event( - services() - .rooms - .timeline - .get_pdu_json(&id) - .unwrap() - .unwrap(), - ) - }) + .map(|id| convert_to_outgoing_federation_event(services.rooms.timeline.get_pdu_json(&id).unwrap().unwrap())) .collect(); - let auth_chain_ids = services() + let auth_chain_ids = services .rooms .auth_chain .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) @@ -64,7 +56,7 @@ pub(crate) async fn get_room_state_route( Ok(get_room_state::v1::Response { auth_chain: auth_chain_ids .filter_map(|id| { - services() + services .rooms .timeline .get_pdu_json(&id) diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index 9fe695dc..d22f2df4 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -1,28 +1,29 @@ use std::sync::Arc; +use axum::extract::State; use ruma::api::{client::error::ErrorKind, federation::event::get_room_state_ids}; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `GET /_matrix/federation/v1/state_ids/{roomId}` /// /// Retrieves a snapshot of a room's state at a given event, in the form of /// event IDs. pub(crate) async fn get_room_state_ids_route( - body: Ruma, + State(services): State, body: Ruma, ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - services() + services .rooms .event_handler .acl_check(origin, &body.room_id)?; - if !services() + if !services .rooms .state_accessor .is_world_readable(&body.room_id)? - && !services() + && !services .rooms .state_cache .server_in_room(origin, &body.room_id)? @@ -30,13 +31,13 @@ pub(crate) async fn get_room_state_ids_route( return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); } - let shortstatehash = services() + let shortstatehash = services .rooms .state_accessor .pdu_shortstatehash(&body.event_id)? .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Pdu state not found."))?; - let pdu_ids = services() + let pdu_ids = services .rooms .state_accessor .state_full_ids(shortstatehash) @@ -45,7 +46,7 @@ pub(crate) async fn get_room_state_ids_route( .map(|id| (*id).to_owned()) .collect(); - let auth_chain_ids = services() + let auth_chain_ids = services .rooms .auth_chain .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) diff --git a/src/api/server/user.rs b/src/api/server/user.rs index 5bfc070c..949e5f38 100644 --- a/src/api/server/user.rs +++ b/src/api/server/user.rs @@ -1,3 +1,4 @@ +use axum::extract::State; use ruma::api::{ client::error::ErrorKind, federation::{ @@ -9,13 +10,15 @@ use ruma::api::{ use crate::{ client::{claim_keys_helper, get_keys_helper}, service::user_is_local, - services, Error, Result, Ruma, + Error, Result, Ruma, }; /// # `GET /_matrix/federation/v1/user/devices/{userId}` /// /// Gets information on all devices of the user. -pub(crate) async fn get_devices_route(body: Ruma) -> Result { +pub(crate) async fn get_devices_route( + State(services): State, body: Ruma, +) -> Result { if !user_is_local(&body.user_id) { return Err(Error::BadRequest( ErrorKind::InvalidParam, @@ -27,25 +30,25 @@ pub(crate) async fn get_devices_route(body: Ruma) -> R Ok(get_devices::v1::Response { user_id: body.user_id.clone(), - stream_id: services() + stream_id: services .users .get_devicelist_version(&body.user_id)? .unwrap_or(0) .try_into() .expect("version will not grow that large"), - devices: services() + devices: services .users .all_devices_metadata(&body.user_id) .filter_map(Result::ok) .filter_map(|metadata| { let device_id_string = metadata.device_id.as_str().to_owned(); - let device_display_name = if services().globals.allow_device_name_federation() { + let device_display_name = if services.globals.allow_device_name_federation() { metadata.display_name } else { Some(device_id_string) }; Some(UserDevice { - keys: services() + keys: services .users .get_device_keys(&body.user_id, &metadata.device_id) .ok()??, @@ -54,10 +57,10 @@ pub(crate) async fn get_devices_route(body: Ruma) -> R }) }) .collect(), - master_key: services() + master_key: services .users .get_master_key(None, &body.user_id, &|u| u.server_name() == origin)?, - self_signing_key: services() + self_signing_key: services .users .get_self_signing_key(None, &body.user_id, &|u| u.server_name() == origin)?, }) @@ -66,7 +69,9 @@ pub(crate) async fn get_devices_route(body: Ruma) -> R /// # `POST /_matrix/federation/v1/user/keys/query` /// /// Gets devices and identity keys for the given users. -pub(crate) async fn get_keys_route(body: Ruma) -> Result { +pub(crate) async fn get_keys_route( + State(services): State, body: Ruma, +) -> Result { if body.device_keys.iter().any(|(u, _)| !user_is_local(u)) { return Err(Error::BadRequest( ErrorKind::InvalidParam, @@ -75,10 +80,11 @@ pub(crate) async fn get_keys_route(body: Ruma) -> Result< } let result = get_keys_helper( + services, None, &body.device_keys, |u| Some(u.server_name()) == body.origin.as_deref(), - services().globals.allow_device_name_federation(), + services.globals.allow_device_name_federation(), ) .await?; @@ -92,7 +98,9 @@ pub(crate) async fn get_keys_route(body: Ruma) -> Result< /// # `POST /_matrix/federation/v1/user/keys/claim` /// /// Claims one-time keys. -pub(crate) async fn claim_keys_route(body: Ruma) -> Result { +pub(crate) async fn claim_keys_route( + State(services): State, body: Ruma, +) -> Result { if body.one_time_keys.iter().any(|(u, _)| !user_is_local(u)) { return Err(Error::BadRequest( ErrorKind::InvalidParam, @@ -100,7 +108,7 @@ pub(crate) async fn claim_keys_route(body: Ruma) -> Res )); } - let result = claim_keys_helper(&body.one_time_keys).await?; + let result = claim_keys_helper(services, &body.one_time_keys).await?; Ok(claim_keys::v1::Response { one_time_keys: result.one_time_keys, diff --git a/src/api/server/well_known.rs b/src/api/server/well_known.rs index 35862662..2cc8f238 100644 --- a/src/api/server/well_known.rs +++ b/src/api/server/well_known.rs @@ -1,15 +1,16 @@ +use axum::extract::State; use ruma::api::{client::error::ErrorKind, federation::discovery::discover_homeserver}; -use crate::{services, Error, Result, Ruma}; +use crate::{Error, Result, Ruma}; /// # `GET /.well-known/matrix/server` /// /// Returns the .well-known URL if it is configured, otherwise returns 404. pub(crate) async fn well_known_server( - _body: Ruma, + State(services): State, _body: Ruma, ) -> Result { Ok(discover_homeserver::Response { - server: match services().globals.well_known_server() { + server: match services.globals.well_known_server() { Some(server_name) => server_name.to_owned(), None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), }, diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 6f26161e..75cae822 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -176,6 +176,7 @@ impl Service { self.db.watch(user_id, device_id).await } + #[inline] pub fn server_name(&self) -> &ServerName { self.config.server_name.as_ref() } pub fn max_fetch_prev_events(&self) -> u16 { self.config.max_fetch_prev_events } From 2fd6f6b0ff1301544cdb3886069799ca103e3361 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 16 Jul 2024 21:33:28 +0000 Subject: [PATCH 0551/2091] add polymorphism to Services Signed-off-by: Jason Volk --- src/service/manager.rs | 2 +- src/service/service.rs | 15 ++++++++++++--- src/service/services.rs | 14 ++++++++------ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/service/manager.rs b/src/service/manager.rs index af59b4a4..447cd6fe 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -54,7 +54,7 @@ impl Manager { ); debug!("Starting service workers..."); - for service in self.services.service.values() { + for (service, ..) in self.services.service.values() { self.start_worker(&mut workers, service).await?; } diff --git a/src/service/service.rs b/src/service/service.rs index 3b8f4231..ab41753a 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -1,11 +1,11 @@ -use std::{collections::BTreeMap, fmt::Write, sync::Arc}; +use std::{any::Any, collections::BTreeMap, fmt::Write, sync::Arc}; use async_trait::async_trait; use conduit::{utils::string::split_once_infallible, Result, Server}; use database::Database; #[async_trait] -pub(crate) trait Service: Send + Sync { +pub(crate) trait Service: Any + Send + Sync { /// Implement the construction of the service instance. Services are /// generally singletons so expect this to only be called once for a /// service type. Note that it may be called again after a server reload, @@ -40,7 +40,16 @@ pub(crate) struct Args<'a> { pub(crate) _service: &'a Map, } -pub(crate) type Map = BTreeMap>; +pub(crate) type Map = BTreeMap; +pub(crate) type MapVal = (Arc, Arc); + +pub(crate) fn get(map: &Map, name: &str) -> Option> { + map.get(name).map(|(_, s)| { + s.clone() + .downcast::() + .expect("Service must be correctly downcast.") + }) +} #[inline] pub(crate) fn make_name(module_path: &str) -> &str { split_once_infallible(module_path, "::").1 } diff --git a/src/service/services.rs b/src/service/services.rs index cc9ec290..0aac10dc 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,4 +1,4 @@ -use std::{collections::BTreeMap, fmt::Write, sync::Arc}; +use std::{any::Any, collections::BTreeMap, fmt::Write, sync::Arc}; use conduit::{debug, debug_info, info, trace, Result, Server}; use database::Database; @@ -7,7 +7,7 @@ use tokio::sync::Mutex; use crate::{ account_data, admin, appservice, globals, key_backups, manager::Manager, - media, presence, pusher, rooms, sending, + media, presence, pusher, rooms, sending, service, service::{Args, Map, Service}, transaction_ids, uiaa, updates, users, }; @@ -44,7 +44,7 @@ impl Services { db: &db, _service: &service, })?; - service.insert(built.name().to_owned(), built.clone()); + service.insert(built.name().to_owned(), (built.clone(), built.clone())); built }}; } @@ -128,7 +128,7 @@ impl Services { } pub async fn clear_cache(&self) { - for service in self.service.values() { + for (service, ..) in self.service.values() { service.clear_cache(); } @@ -143,7 +143,7 @@ impl Services { pub async fn memory_usage(&self) -> Result { let mut out = String::new(); - for service in self.service.values() { + for (service, ..) in self.service.values() { service.memory_usage(&mut out)?; } @@ -163,9 +163,11 @@ impl Services { fn interrupt(&self) { debug!("Interrupting services..."); - for (name, service) in &self.service { + for (name, (service, ..)) in &self.service { trace!("Interrupting {name}"); service.interrupt(); } } + + pub fn get(&self, name: &str) -> Option> { service::get::(&self.service, name) } } From f465d77ad39d3d095346e30daf9d51270e3976be Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 16 Jul 2024 22:00:54 +0000 Subject: [PATCH 0552/2091] convert Resolver into a Service. Signed-off-by: Jason Volk --- src/admin/query/resolver.rs | 18 +- src/service/globals/client.rs | 4 +- src/service/globals/mod.rs | 16 +- src/service/globals/resolver.rs | 156 -------------- src/service/mod.rs | 1 + src/service/resolver/mod.rs | 349 ++++++++++++++++++++++++++++++++ src/service/sending/mod.rs | 2 +- src/service/sending/resolve.rs | 223 +------------------- src/service/sending/send.rs | 9 +- src/service/service.rs | 2 +- src/service/services.rs | 10 +- 11 files changed, 381 insertions(+), 409 deletions(-) delete mode 100644 src/service/globals/resolver.rs create mode 100644 src/service/resolver/mod.rs diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index 2a2554b5..06cd8ba9 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -19,7 +19,7 @@ pub(super) async fn resolver(subcommand: Resolver) -> Result) -> Result { - use service::sending::CachedDest; + use service::resolver::CachedDest; let mut out = String::new(); writeln!(out, "| Server Name | Destination | Hostname | Expires |")?; @@ -36,12 +36,7 @@ async fn destinations_cache(server_name: Option) -> Result) -> Result) -> Result { - use service::sending::CachedOverride; + use service::resolver::CachedOverride; let mut out = String::new(); writeln!(out, "| Server Name | IP | Port | Expires |")?; @@ -70,12 +65,7 @@ async fn overrides_cache(server_name: Option) -> Result) -> Self { + pub fn new(config: &Config, resolver: &Arc) -> Self { Self { default: Self::base(config) .unwrap() diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 75cae822..a2fba4d9 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -2,7 +2,6 @@ mod client; mod data; mod emerg_access; pub(super) mod migrations; -pub(crate) mod resolver; use std::{ collections::{BTreeMap, HashMap}, @@ -25,7 +24,7 @@ use ruma::{ use tokio::sync::Mutex; use url::Url; -use crate::services; +use crate::{resolver, service, services}; pub struct Service { pub db: Data, @@ -34,7 +33,6 @@ pub struct Service { pub cidr_range_denylist: Vec, keypair: Arc, jwt_decoding_key: Option, - pub resolver: Arc, pub client: client::Client, pub stable_room_versions: Vec, pub unstable_room_versions: Vec, @@ -68,8 +66,6 @@ impl crate::Service for Service { .as_ref() .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes())); - let resolver = Arc::new(resolver::Resolver::new(config)); - // Supported and stable room versions let stable_room_versions = vec![ RoomVersionId::V6, @@ -89,12 +85,14 @@ impl crate::Service for Service { cidr_range_denylist.push(cidr); } + let resolver = service::get::(args.service, "resolver") + .expect("resolver must be built prior to globals"); + let mut s = Self { db, config: config.clone(), cidr_range_denylist, keypair: Arc::new(keypair), - resolver: resolver.clone(), client: client::Client::new(config, &resolver), jwt_decoding_key, stable_room_versions, @@ -126,8 +124,6 @@ impl crate::Service for Service { } fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { - self.resolver.memory_usage(out)?; - let bad_event_ratelimiter = self .bad_event_ratelimiter .read() @@ -146,8 +142,6 @@ impl crate::Service for Service { } fn clear_cache(&self) { - self.resolver.clear_cache(); - self.bad_event_ratelimiter .write() .expect("locked for writing") @@ -159,7 +153,7 @@ impl crate::Service for Service { .clear(); } - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } + fn name(&self) -> &str { service::make_name(std::module_path!()) } } impl Service { diff --git a/src/service/globals/resolver.rs b/src/service/globals/resolver.rs deleted file mode 100644 index 3002decf..00000000 --- a/src/service/globals/resolver.rs +++ /dev/null @@ -1,156 +0,0 @@ -use std::{ - collections::HashMap, - fmt::Write, - future, iter, - net::{IpAddr, SocketAddr}, - sync::{Arc, RwLock}, - time::Duration, -}; - -use conduit::{error, Config, Result}; -use hickory_resolver::TokioAsyncResolver; -use reqwest::dns::{Addrs, Name, Resolve, Resolving}; -use ruma::OwnedServerName; - -use crate::sending::{CachedDest, CachedOverride}; - -type WellKnownMap = HashMap; -type TlsNameMap = HashMap; - -pub struct Resolver { - pub destinations: Arc>, // actual_destination, host - pub overrides: Arc>, - pub(crate) resolver: Arc, - pub(crate) hooked: Arc, -} - -pub(crate) struct Hooked { - overrides: Arc>, - resolver: Arc, -} - -impl Resolver { - #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] - pub(super) fn new(config: &Config) -> Self { - let (sys_conf, mut opts) = hickory_resolver::system_conf::read_system_conf() - .inspect_err(|e| error!("Failed to set up hickory dns resolver with system config: {e}")) - .expect("DNS system config must be valid"); - - let mut conf = hickory_resolver::config::ResolverConfig::new(); - - if let Some(domain) = sys_conf.domain() { - conf.set_domain(domain.clone()); - } - - for sys_conf in sys_conf.search() { - conf.add_search(sys_conf.clone()); - } - - for sys_conf in sys_conf.name_servers() { - let mut ns = sys_conf.clone(); - - if config.query_over_tcp_only { - ns.protocol = hickory_resolver::config::Protocol::Tcp; - } - - ns.trust_negative_responses = !config.query_all_nameservers; - - conf.add_name_server(ns); - } - - opts.cache_size = config.dns_cache_entries as usize; - opts.negative_min_ttl = Some(Duration::from_secs(config.dns_min_ttl_nxdomain)); - opts.negative_max_ttl = Some(Duration::from_secs(60 * 60 * 24 * 30)); - opts.positive_min_ttl = Some(Duration::from_secs(config.dns_min_ttl)); - opts.positive_max_ttl = Some(Duration::from_secs(60 * 60 * 24 * 7)); - opts.timeout = Duration::from_secs(config.dns_timeout); - opts.attempts = config.dns_attempts as usize; - opts.try_tcp_on_error = config.dns_tcp_fallback; - opts.num_concurrent_reqs = 1; - opts.shuffle_dns_servers = true; - opts.rotate = true; - opts.ip_strategy = match config.ip_lookup_strategy { - 1 => hickory_resolver::config::LookupIpStrategy::Ipv4Only, - 2 => hickory_resolver::config::LookupIpStrategy::Ipv6Only, - 3 => hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6, - 4 => hickory_resolver::config::LookupIpStrategy::Ipv6thenIpv4, - _ => hickory_resolver::config::LookupIpStrategy::Ipv4thenIpv6, - }; - opts.authentic_data = false; - - let resolver = Arc::new(TokioAsyncResolver::tokio(conf, opts)); - let overrides = Arc::new(RwLock::new(TlsNameMap::new())); - Self { - destinations: Arc::new(RwLock::new(WellKnownMap::new())), - overrides: overrides.clone(), - resolver: resolver.clone(), - hooked: Arc::new(Hooked { - overrides, - resolver, - }), - } - } - - pub(super) fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { - let resolver_overrides_cache = self.overrides.read().expect("locked for reading").len(); - writeln!(out, "resolver_overrides_cache: {resolver_overrides_cache}")?; - - let resolver_destinations_cache = self.destinations.read().expect("locked for reading").len(); - writeln!(out, "resolver_destinations_cache: {resolver_destinations_cache}")?; - - Ok(()) - } - - pub(super) fn clear_cache(&self) { - self.overrides.write().expect("write locked").clear(); - self.destinations.write().expect("write locked").clear(); - self.resolver.clear_cache(); - } -} - -impl Resolve for Resolver { - fn resolve(&self, name: Name) -> Resolving { resolve_to_reqwest(self.resolver.clone(), name) } -} - -impl Resolve for Hooked { - fn resolve(&self, name: Name) -> Resolving { - let cached = self - .overrides - .read() - .expect("locked for reading") - .get(name.as_str()) - .filter(|cached| cached.valid()) - .cloned(); - - if let Some(cached) = cached { - cached_to_reqwest(&cached.ips, cached.port) - } else { - resolve_to_reqwest(self.resolver.clone(), name) - } - } -} - -fn cached_to_reqwest(override_name: &[IpAddr], port: u16) -> Resolving { - override_name - .first() - .map(|first_name| -> Resolving { - let saddr = SocketAddr::new(*first_name, port); - let result: Box + Send> = Box::new(iter::once(saddr)); - Box::pin(future::ready(Ok(result))) - }) - .expect("must provide at least one override name") -} - -fn resolve_to_reqwest(resolver: Arc, name: Name) -> Resolving { - Box::pin(async move { - let results = resolver - .lookup_ip(name.as_str()) - .await? - .into_iter() - .map(|ip| SocketAddr::new(ip, 0)); - - let results: Addrs = Box::new(results); - - Ok(results) - }) -} diff --git a/src/service/mod.rs b/src/service/mod.rs index 81e0be3b..870d865b 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -12,6 +12,7 @@ pub mod key_backups; pub mod media; pub mod presence; pub mod pusher; +pub mod resolver; pub mod rooms; pub mod sending; pub mod transaction_ids; diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs new file mode 100644 index 00000000..62fd1625 --- /dev/null +++ b/src/service/resolver/mod.rs @@ -0,0 +1,349 @@ +use std::{ + collections::HashMap, + fmt, + fmt::Write, + future, iter, + net::{IpAddr, SocketAddr}, + sync::{Arc, RwLock}, + time::{Duration, SystemTime}, +}; + +use conduit::{err, trace, Result}; +use hickory_resolver::TokioAsyncResolver; +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; +use ruma::{OwnedServerName, ServerName}; + +use crate::utils::rand; + +pub struct Service { + pub destinations: Arc>, // actual_destination, host + pub overrides: Arc>, + pub(crate) resolver: Arc, + pub(crate) hooked: Arc, +} + +pub(crate) struct Hooked { + overrides: Arc>, + resolver: Arc, +} + +#[derive(Clone, Debug)] +pub struct CachedDest { + pub dest: FedDest, + pub host: String, + pub expire: SystemTime, +} + +#[derive(Clone, Debug)] +pub struct CachedOverride { + pub ips: Vec, + pub port: u16, + pub expire: SystemTime, +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FedDest { + Literal(SocketAddr), + Named(String, String), +} + +type WellKnownMap = HashMap; +type TlsNameMap = HashMap; + +impl crate::Service for Service { + #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] + fn build(args: crate::Args<'_>) -> Result> { + let config = &args.server.config; + let (sys_conf, mut opts) = hickory_resolver::system_conf::read_system_conf() + .map_err(|e| err!(error!("Failed to configure DNS resolver from system: {e}")))?; + + let mut conf = hickory_resolver::config::ResolverConfig::new(); + + if let Some(domain) = sys_conf.domain() { + conf.set_domain(domain.clone()); + } + + for sys_conf in sys_conf.search() { + conf.add_search(sys_conf.clone()); + } + + for sys_conf in sys_conf.name_servers() { + let mut ns = sys_conf.clone(); + + if config.query_over_tcp_only { + ns.protocol = hickory_resolver::config::Protocol::Tcp; + } + + ns.trust_negative_responses = !config.query_all_nameservers; + + conf.add_name_server(ns); + } + + opts.cache_size = config.dns_cache_entries as usize; + opts.negative_min_ttl = Some(Duration::from_secs(config.dns_min_ttl_nxdomain)); + opts.negative_max_ttl = Some(Duration::from_secs(60 * 60 * 24 * 30)); + opts.positive_min_ttl = Some(Duration::from_secs(config.dns_min_ttl)); + opts.positive_max_ttl = Some(Duration::from_secs(60 * 60 * 24 * 7)); + opts.timeout = Duration::from_secs(config.dns_timeout); + opts.attempts = config.dns_attempts as usize; + opts.try_tcp_on_error = config.dns_tcp_fallback; + opts.num_concurrent_reqs = 1; + opts.shuffle_dns_servers = true; + opts.rotate = true; + opts.ip_strategy = match config.ip_lookup_strategy { + 1 => hickory_resolver::config::LookupIpStrategy::Ipv4Only, + 2 => hickory_resolver::config::LookupIpStrategy::Ipv6Only, + 3 => hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6, + 4 => hickory_resolver::config::LookupIpStrategy::Ipv6thenIpv4, + _ => hickory_resolver::config::LookupIpStrategy::Ipv4thenIpv6, + }; + opts.authentic_data = false; + + let resolver = Arc::new(TokioAsyncResolver::tokio(conf, opts)); + let overrides = Arc::new(RwLock::new(TlsNameMap::new())); + Ok(Arc::new(Self { + destinations: Arc::new(RwLock::new(WellKnownMap::new())), + overrides: overrides.clone(), + resolver: resolver.clone(), + hooked: Arc::new(Hooked { + overrides, + resolver, + }), + })) + } + + fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + let resolver_overrides_cache = self.overrides.read().expect("locked for reading").len(); + writeln!(out, "resolver_overrides_cache: {resolver_overrides_cache}")?; + + let resolver_destinations_cache = self.destinations.read().expect("locked for reading").len(); + writeln!(out, "resolver_destinations_cache: {resolver_destinations_cache}")?; + + Ok(()) + } + + fn clear_cache(&self) { + self.overrides.write().expect("write locked").clear(); + self.destinations.write().expect("write locked").clear(); + self.resolver.clear_cache(); + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { + pub fn set_cached_destination(&self, name: OwnedServerName, dest: CachedDest) -> Option { + trace!(?name, ?dest, "set cached destination"); + self.destinations + .write() + .expect("locked for writing") + .insert(name, dest) + } + + #[must_use] + pub fn get_cached_destination(&self, name: &ServerName) -> Option { + self.destinations + .read() + .expect("locked for reading") + .get(name) + .cloned() + } + + pub fn set_cached_override(&self, name: String, over: CachedOverride) -> Option { + trace!(?name, ?over, "set cached override"); + self.overrides + .write() + .expect("locked for writing") + .insert(name, over) + } + + #[must_use] + pub fn get_cached_override(&self, name: &str) -> Option { + self.overrides + .read() + .expect("locked for reading") + .get(name) + .cloned() + } + + #[must_use] + pub fn has_cached_override(&self, name: &str) -> bool { + self.overrides + .read() + .expect("locked for reading") + .contains_key(name) + } +} + +impl Resolve for Service { + fn resolve(&self, name: Name) -> Resolving { resolve_to_reqwest(self.resolver.clone(), name) } +} + +impl Resolve for Hooked { + fn resolve(&self, name: Name) -> Resolving { + let cached = self + .overrides + .read() + .expect("locked for reading") + .get(name.as_str()) + .cloned(); + + if let Some(cached) = cached { + cached_to_reqwest(&cached.ips, cached.port) + } else { + resolve_to_reqwest(self.resolver.clone(), name) + } + } +} + +fn cached_to_reqwest(override_name: &[IpAddr], port: u16) -> Resolving { + override_name + .first() + .map(|first_name| -> Resolving { + let saddr = SocketAddr::new(*first_name, port); + let result: Box + Send> = Box::new(iter::once(saddr)); + Box::pin(future::ready(Ok(result))) + }) + .expect("must provide at least one override name") +} + +fn resolve_to_reqwest(resolver: Arc, name: Name) -> Resolving { + Box::pin(async move { + let results = resolver + .lookup_ip(name.as_str()) + .await? + .into_iter() + .map(|ip| SocketAddr::new(ip, 0)); + + let results: Addrs = Box::new(results); + + Ok(results) + }) +} + +impl CachedDest { + #[inline] + #[must_use] + pub fn valid(&self) -> bool { true } + + //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + + #[must_use] + pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } +} + +impl CachedOverride { + #[inline] + #[must_use] + pub fn valid(&self) -> bool { true } + + //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + + #[must_use] + pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } +} + +pub(crate) fn get_ip_with_port(dest_str: &str) -> Option { + if let Ok(dest) = dest_str.parse::() { + Some(FedDest::Literal(dest)) + } else if let Ok(ip_addr) = dest_str.parse::() { + Some(FedDest::Literal(SocketAddr::new(ip_addr, 8448))) + } else { + None + } +} + +pub(crate) fn add_port_to_hostname(dest_str: &str) -> FedDest { + let (host, port) = match dest_str.find(':') { + None => (dest_str, ":8448"), + Some(pos) => dest_str.split_at(pos), + }; + + FedDest::Named(host.to_owned(), port.to_owned()) +} + +impl FedDest { + pub(crate) fn into_https_string(self) -> String { + match self { + Self::Literal(addr) => format!("https://{addr}"), + Self::Named(host, port) => format!("https://{host}{port}"), + } + } + + pub(crate) fn into_uri_string(self) -> String { + match self { + Self::Literal(addr) => addr.to_string(), + Self::Named(host, port) => format!("{host}{port}"), + } + } + + pub(crate) fn hostname(&self) -> String { + match &self { + Self::Literal(addr) => addr.ip().to_string(), + Self::Named(host, _) => host.clone(), + } + } + + #[inline] + #[allow(clippy::string_slice)] + pub(crate) fn port(&self) -> Option { + match &self { + Self::Literal(addr) => Some(addr.port()), + Self::Named(_, port) => port[1..].parse().ok(), + } + } +} + +impl fmt::Display for FedDest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Named(host, port) => write!(f, "{host}{port}"), + Self::Literal(addr) => write!(f, "{addr}"), + } + } +} + +#[cfg(test)] +mod tests { + use super::{add_port_to_hostname, get_ip_with_port, FedDest}; + + #[test] + fn ips_get_default_ports() { + assert_eq!( + get_ip_with_port("1.1.1.1"), + Some(FedDest::Literal("1.1.1.1:8448".parse().unwrap())) + ); + assert_eq!( + get_ip_with_port("dead:beef::"), + Some(FedDest::Literal("[dead:beef::]:8448".parse().unwrap())) + ); + } + + #[test] + fn ips_keep_custom_ports() { + assert_eq!( + get_ip_with_port("1.1.1.1:1234"), + Some(FedDest::Literal("1.1.1.1:1234".parse().unwrap())) + ); + assert_eq!( + get_ip_with_port("[dead::beef]:8933"), + Some(FedDest::Literal("[dead::beef]:8933".parse().unwrap())) + ); + } + + #[test] + fn hostnames_get_default_ports() { + assert_eq!( + add_port_to_hostname("example.com"), + FedDest::Named(String::from("example.com"), String::from(":8448")) + ); + } + + #[test] + fn hostnames_keep_custom_ports() { + assert_eq!( + add_port_to_hostname("example.com:1337"), + FedDest::Named(String::from("example.com"), String::from(":1337")) + ); + } +} diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 88b8b189..fd9acd3d 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -7,7 +7,7 @@ mod sender; use std::fmt::Debug; use conduit::{err, Result}; -pub use resolve::{resolve_actual_dest, CachedDest, CachedOverride, FedDest}; +pub use resolve::resolve_actual_dest; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 48dddf35..4c1e4758 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -1,40 +1,17 @@ use std::{ - fmt, fmt::Debug, net::{IpAddr, SocketAddr}, - time::SystemTime, }; -use conduit::{debug, debug_error, debug_info, debug_warn, trace, utils::rand, Err, Error, Result}; +use conduit::{debug, debug_error, debug_info, debug_warn, trace, Err, Error, Result}; use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; -use ruma::{OwnedServerName, ServerName}; +use ruma::ServerName; -use crate::services; - -/// Wraps either an literal IP address plus port, or a hostname plus complement -/// (colon-plus-port if it was specified). -/// -/// Note: A `FedDest::Named` might contain an IP address in string form if there -/// was no port specified to construct a `SocketAddr` with. -/// -/// # Examples: -/// ```rust -/// # use conduit_service::sending::FedDest; -/// # fn main() -> Result<(), std::net::AddrParseError> { -/// FedDest::Literal("198.51.100.3:8448".parse()?); -/// FedDest::Literal("[2001:db8::4:5]:443".parse()?); -/// FedDest::Named("matrix.example.org".to_owned(), String::new()); -/// FedDest::Named("matrix.example.org".to_owned(), ":8448".to_owned()); -/// FedDest::Named("198.51.100.5".to_owned(), String::new()); -/// # Ok(()) -/// # } -/// ``` -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum FedDest { - Literal(SocketAddr), - Named(String, String), -} +use crate::{ + resolver::{add_port_to_hostname, get_ip_with_port, CachedDest, CachedOverride, FedDest}, + services, +}; #[derive(Clone, Debug)] pub(crate) struct ActualDest { @@ -44,27 +21,10 @@ pub(crate) struct ActualDest { pub(crate) cached: bool, } -#[derive(Clone, Debug)] -pub struct CachedDest { - pub dest: FedDest, - pub host: String, - pub expire: SystemTime, -} - -#[derive(Clone, Debug)] -pub struct CachedOverride { - pub ips: Vec, - pub port: u16, - pub expire: SystemTime, -} - #[tracing::instrument(skip_all, name = "resolve")] pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result { let cached; - let cached_result = services() - .globals - .resolver - .get_cached_destination(server_name); + let cached_result = services().resolver.get_cached_destination(server_name); let CachedDest { dest, @@ -213,7 +173,7 @@ async fn actual_dest_5(dest: &ServerName, cache: bool) -> Result { #[tracing::instrument(skip_all, name = "well-known")] async fn request_well_known(dest: &str) -> Result> { trace!("Requesting well known for {dest}"); - if !services().globals.resolver.has_cached_override(dest) { + if !services().resolver.has_cached_override(dest) { query_and_cache_override(dest, dest, 8448).await?; } @@ -273,7 +233,6 @@ async fn conditional_query_and_cache_override(overname: &str, hostname: &str, po #[tracing::instrument(skip_all, name = "ip")] async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { match services() - .globals .resolver .resolver .lookup_ip(hostname.to_owned()) @@ -285,7 +244,7 @@ async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u1 debug_info!("{overname:?} overriden by {hostname:?}"); } - services().globals.resolver.set_cached_override( + services().resolver.set_cached_override( overname.to_owned(), CachedOverride { ips: override_ip.iter().collect(), @@ -314,7 +273,6 @@ async fn query_srv_record(hostname: &'_ str) -> Result> { debug!("querying SRV for {:?}", hostname); let hostname = hostname.trim_end_matches('.'); services() - .globals .resolver .resolver .srv_lookup(hostname.to_owned()) @@ -384,166 +342,3 @@ pub(crate) fn validate_ip(ip: &IPAddress) -> Result<()> { Ok(()) } - -fn get_ip_with_port(dest_str: &str) -> Option { - if let Ok(dest) = dest_str.parse::() { - Some(FedDest::Literal(dest)) - } else if let Ok(ip_addr) = dest_str.parse::() { - Some(FedDest::Literal(SocketAddr::new(ip_addr, 8448))) - } else { - None - } -} - -fn add_port_to_hostname(dest_str: &str) -> FedDest { - let (host, port) = match dest_str.find(':') { - None => (dest_str, ":8448"), - Some(pos) => dest_str.split_at(pos), - }; - - FedDest::Named(host.to_owned(), port.to_owned()) -} - -impl crate::globals::resolver::Resolver { - pub(crate) fn set_cached_destination(&self, name: OwnedServerName, dest: CachedDest) -> Option { - trace!(?name, ?dest, "set cached destination"); - self.destinations - .write() - .expect("locked for writing") - .insert(name, dest) - } - - pub(crate) fn get_cached_destination(&self, name: &ServerName) -> Option { - self.destinations - .read() - .expect("locked for reading") - .get(name) - .filter(|cached| cached.valid()) - .cloned() - } - - pub(crate) fn set_cached_override(&self, name: String, over: CachedOverride) -> Option { - trace!(?name, ?over, "set cached override"); - self.overrides - .write() - .expect("locked for writing") - .insert(name, over) - } - - pub(crate) fn has_cached_override(&self, name: &str) -> bool { - self.overrides - .read() - .expect("locked for reading") - .get(name) - .filter(|cached| cached.valid()) - .is_some() - } -} - -impl CachedDest { - #[inline] - #[must_use] - pub fn valid(&self) -> bool { true } - - //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } - - #[must_use] - pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } -} - -impl CachedOverride { - #[inline] - #[must_use] - pub fn valid(&self) -> bool { true } - - //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } - - #[must_use] - pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } -} - -impl FedDest { - fn into_https_string(self) -> String { - match self { - Self::Literal(addr) => format!("https://{addr}"), - Self::Named(host, port) => format!("https://{host}{port}"), - } - } - - fn into_uri_string(self) -> String { - match self { - Self::Literal(addr) => addr.to_string(), - Self::Named(host, port) => format!("{host}{port}"), - } - } - - fn hostname(&self) -> String { - match &self { - Self::Literal(addr) => addr.ip().to_string(), - Self::Named(host, _) => host.clone(), - } - } - - #[inline] - #[allow(clippy::string_slice)] - fn port(&self) -> Option { - match &self { - Self::Literal(addr) => Some(addr.port()), - Self::Named(_, port) => port[1..].parse().ok(), - } - } -} - -impl fmt::Display for FedDest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Named(host, port) => write!(f, "{host}{port}"), - Self::Literal(addr) => write!(f, "{addr}"), - } - } -} - -#[cfg(test)] -mod tests { - use super::{add_port_to_hostname, get_ip_with_port, FedDest}; - - #[test] - fn ips_get_default_ports() { - assert_eq!( - get_ip_with_port("1.1.1.1"), - Some(FedDest::Literal("1.1.1.1:8448".parse().unwrap())) - ); - assert_eq!( - get_ip_with_port("dead:beef::"), - Some(FedDest::Literal("[dead:beef::]:8448".parse().unwrap())) - ); - } - - #[test] - fn ips_keep_custom_ports() { - assert_eq!( - get_ip_with_port("1.1.1.1:1234"), - Some(FedDest::Literal("1.1.1.1:1234".parse().unwrap())) - ); - assert_eq!( - get_ip_with_port("[dead::beef]:8933"), - Some(FedDest::Literal("[dead::beef]:8933".parse().unwrap())) - ); - } - - #[test] - fn hostnames_get_default_ports() { - assert_eq!( - add_port_to_hostname("example.com"), - FedDest::Named(String::from("example.com"), String::from(":8448")) - ); - } - - #[test] - fn hostnames_keep_custom_ports() { - assert_eq!( - add_port_to_hostname("example.com:1337"), - FedDest::Named(String::from("example.com"), String::from(":1337")) - ); - } -} diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 18a98828..dc4541d6 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -15,11 +15,8 @@ use ruma::{ }; use tracing::{debug, trace}; -use super::{ - resolve, - resolve::{ActualDest, CachedDest}, -}; -use crate::{debug_error, debug_warn, services, Error, Result}; +use super::{resolve, resolve::ActualDest}; +use crate::{debug_error, debug_warn, resolver::CachedDest, services, Error, Result}; #[tracing::instrument(skip_all, name = "send")] pub async fn send(client: &Client, dest: &ServerName, req: T) -> Result @@ -109,7 +106,7 @@ where let response = T::IncomingResponse::try_from_http_response(http_response); if response.is_ok() && !actual.cached { - services().globals.resolver.set_cached_destination( + services().resolver.set_cached_destination( dest.to_owned(), CachedDest { dest: actual.dest.clone(), diff --git a/src/service/service.rs b/src/service/service.rs index ab41753a..8b49d455 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -37,7 +37,7 @@ pub(crate) trait Service: Any + Send + Sync { pub(crate) struct Args<'a> { pub(crate) server: &'a Arc, pub(crate) db: &'a Arc, - pub(crate) _service: &'a Map, + pub(crate) service: &'a Map, } pub(crate) type Map = BTreeMap; diff --git a/src/service/services.rs b/src/service/services.rs index 0aac10dc..fb13f24a 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -7,12 +7,14 @@ use tokio::sync::Mutex; use crate::{ account_data, admin, appservice, globals, key_backups, manager::Manager, - media, presence, pusher, rooms, sending, service, + media, presence, pusher, resolver, rooms, sending, service, service::{Args, Map, Service}, transaction_ids, uiaa, updates, users, }; pub struct Services { + pub resolver: Arc, + pub globals: Arc, pub rooms: rooms::Service, pub appservice: Arc, pub pusher: Arc, @@ -26,7 +28,6 @@ pub struct Services { pub media: Arc, pub sending: Arc, pub updates: Arc, - pub globals: Arc, manager: Mutex>>, pub(crate) service: Map, @@ -42,7 +43,7 @@ impl Services { let built = <$tyname>::build(Args { server: &server, db: &db, - _service: &service, + service: &service, })?; service.insert(built.name().to_owned(), (built.clone(), built.clone())); built @@ -50,6 +51,8 @@ impl Services { } Ok(Self { + resolver: build!(resolver::Service), + globals: build!(globals::Service), rooms: rooms::Service { alias: build!(rooms::alias::Service), auth_chain: build!(rooms::auth_chain::Service), @@ -84,7 +87,6 @@ impl Services { media: build!(media::Service), sending: build!(sending::Service), updates: build!(updates::Service), - globals: build!(globals::Service), manager: Mutex::new(None), service, server, From 8a2ae401df8a34b9a2ae823aedee8857d5737b48 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 16 Jul 2024 22:29:42 +0000 Subject: [PATCH 0553/2091] convert Client into Service Signed-off-by: Jason Volk --- src/admin/federation/commands.rs | 1 - src/api/client/media.rs | 2 +- src/service/client/mod.rs | 143 ++++++++++++++++++++++++++++++ src/service/globals/mod.rs | 8 +- src/service/mod.rs | 3 +- src/service/pusher/mod.rs | 7 +- src/service/sending/appservice.rs | 1 - src/service/sending/mod.rs | 2 +- src/service/sending/resolve.rs | 1 - src/service/sending/sender.rs | 3 +- src/service/service.rs | 6 ++ src/service/services.rs | 4 +- src/service/updates/mod.rs | 1 - 13 files changed, 159 insertions(+), 23 deletions(-) create mode 100644 src/service/client/mod.rs diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index a97e7582..331231ae 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -35,7 +35,6 @@ pub(super) async fn fetch_support_well_known( _body: Vec<&str>, server_name: Box, ) -> Result { let response = services() - .globals .client .default .get(format!("https://{server_name}/.well-known/matrix/support")) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 1adcefdd..46f8152b 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -621,7 +621,7 @@ async fn request_url_preview(services: &Services, url: &str) -> Result) -> Result> { + let config = &args.server.config; + let resolver = args + .get_service::("resolver") + .expect("resolver must be built prior to client"); + + Ok(Arc::new(Self { + default: base(config) + .unwrap() + .dns_resolver(resolver.clone()) + .build() + .unwrap(), + + url_preview: base(config) + .unwrap() + .dns_resolver(resolver.clone()) + .redirect(redirect::Policy::limited(3)) + .build() + .unwrap(), + + well_known: base(config) + .unwrap() + .dns_resolver(resolver.hooked.clone()) + .connect_timeout(Duration::from_secs(config.well_known_conn_timeout)) + .read_timeout(Duration::from_secs(config.well_known_timeout)) + .timeout(Duration::from_secs(config.well_known_timeout)) + .pool_max_idle_per_host(0) + .redirect(redirect::Policy::limited(4)) + .build() + .unwrap(), + + federation: base(config) + .unwrap() + .dns_resolver(resolver.hooked.clone()) + .read_timeout(Duration::from_secs(config.federation_timeout)) + .timeout(Duration::from_secs(config.federation_timeout)) + .pool_max_idle_per_host(config.federation_idle_per_host.into()) + .pool_idle_timeout(Duration::from_secs(config.federation_idle_timeout)) + .redirect(redirect::Policy::limited(3)) + .build() + .unwrap(), + + sender: base(config) + .unwrap() + .dns_resolver(resolver.hooked.clone()) + .read_timeout(Duration::from_secs(config.sender_timeout)) + .timeout(Duration::from_secs(config.sender_timeout)) + .pool_max_idle_per_host(1) + .pool_idle_timeout(Duration::from_secs(config.sender_idle_timeout)) + .redirect(redirect::Policy::limited(2)) + .build() + .unwrap(), + + appservice: base(config) + .unwrap() + .dns_resolver(resolver.clone()) + .connect_timeout(Duration::from_secs(5)) + .read_timeout(Duration::from_secs(config.appservice_timeout)) + .timeout(Duration::from_secs(config.appservice_timeout)) + .pool_max_idle_per_host(1) + .pool_idle_timeout(Duration::from_secs(config.appservice_idle_timeout)) + .redirect(redirect::Policy::limited(2)) + .build() + .unwrap(), + + pusher: base(config) + .unwrap() + .dns_resolver(resolver.clone()) + .pool_max_idle_per_host(1) + .pool_idle_timeout(Duration::from_secs(config.pusher_idle_timeout)) + .redirect(redirect::Policy::limited(2)) + .build() + .unwrap(), + })) + } + + fn name(&self) -> &str { service::make_name(std::module_path!()) } +} + +fn base(config: &Config) -> Result { + let mut builder = reqwest::Client::builder() + .hickory_dns(true) + .connect_timeout(Duration::from_secs(config.request_conn_timeout)) + .read_timeout(Duration::from_secs(config.request_timeout)) + .timeout(Duration::from_secs(config.request_total_timeout)) + .pool_idle_timeout(Duration::from_secs(config.request_idle_timeout)) + .pool_max_idle_per_host(config.request_idle_per_host.into()) + .user_agent(conduit::version::user_agent()) + .redirect(redirect::Policy::limited(6)) + .connection_verbose(true); + + #[cfg(feature = "gzip_compression")] + { + builder = if config.gzip_compression { + builder.gzip(true) + } else { + builder.gzip(false).no_gzip() + }; + }; + + #[cfg(feature = "brotli_compression")] + { + builder = if config.brotli_compression { + builder.brotli(true) + } else { + builder.brotli(false).no_brotli() + }; + }; + + #[cfg(not(feature = "gzip_compression"))] + { + builder = builder.no_gzip(); + }; + + #[cfg(not(feature = "brotli_compression"))] + { + builder = builder.no_brotli(); + }; + + if let Some(proxy) = config.proxy.to_proxy()? { + Ok(builder.proxy(proxy)) + } else { + Ok(builder) + } +} diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index a2fba4d9..811aff3a 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -1,4 +1,3 @@ -mod client; mod data; mod emerg_access; pub(super) mod migrations; @@ -24,7 +23,7 @@ use ruma::{ use tokio::sync::Mutex; use url::Url; -use crate::{resolver, service, services}; +use crate::{service, services}; pub struct Service { pub db: Data, @@ -33,7 +32,6 @@ pub struct Service { pub cidr_range_denylist: Vec, keypair: Arc, jwt_decoding_key: Option, - pub client: client::Client, pub stable_room_versions: Vec, pub unstable_room_versions: Vec, pub bad_event_ratelimiter: Arc>>, @@ -85,15 +83,11 @@ impl crate::Service for Service { cidr_range_denylist.push(cidr); } - let resolver = service::get::(args.service, "resolver") - .expect("resolver must be built prior to globals"); - let mut s = Self { db, config: config.clone(), cidr_range_denylist, keypair: Arc::new(keypair), - client: client::Client::new(config, &resolver), jwt_decoding_key, stable_room_versions, unstable_room_versions, diff --git a/src/service/mod.rs b/src/service/mod.rs index 870d865b..46adb072 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -7,6 +7,7 @@ pub mod services; pub mod account_data; pub mod admin; pub mod appservice; +pub mod client; pub mod globals; pub mod key_backups; pub mod media; @@ -25,7 +26,7 @@ extern crate conduit_database as database; use std::sync::{Arc, RwLock}; -pub(crate) use conduit::{config, debug_error, debug_warn, utils, Config, Error, Result, Server}; +pub(crate) use conduit::{config, debug_error, debug_warn, utils, Error, Result, Server}; pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; use database::Database; pub(crate) use service::{Args, Service}; diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index ea48ea7c..38ea5b9a 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -84,12 +84,7 @@ impl Service { } } - let response = services() - .globals - .client - .pusher - .execute(reqwest_request) - .await; + let response = services().client.pusher.execute(reqwest_request).await; match response { Ok(mut response) => { diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 721424e1..9e060e81 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -49,7 +49,6 @@ where let reqwest_request = reqwest::Request::try_from(http_request)?; let mut response = services() - .globals .client .appservice .execute(reqwest_request) diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index fd9acd3d..be10184d 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -194,7 +194,7 @@ impl Service { where T: OutgoingRequest + Debug + Send, { - let client = &services().globals.client.federation; + let client = &services().client.federation; send::send(client, dest, request).await } diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs index 4c1e4758..f8d1c51c 100644 --- a/src/service/sending/resolve.rs +++ b/src/service/sending/resolve.rs @@ -178,7 +178,6 @@ async fn request_well_known(dest: &str) -> Result> { } let response = services() - .globals .client .well_known .get(&format!("https://{dest}/.well-known/matrix/server")) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 2f542dfe..e6b68e9e 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -611,11 +611,10 @@ async fn send_events_dest_normal( } } - let client = &services().globals.client.sender; //debug_assert!(pdu_jsons.len() + edu_jsons.len() > 0, "sending empty // transaction"); send::send( - client, + &services().client.sender, server, send_transaction_message::v1::Request { origin: services().globals.server_name().to_owned(), diff --git a/src/service/service.rs b/src/service/service.rs index 8b49d455..99b8723a 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -43,6 +43,12 @@ pub(crate) struct Args<'a> { pub(crate) type Map = BTreeMap; pub(crate) type MapVal = (Arc, Arc); +impl Args<'_> { + pub(crate) fn get_service(&self, name: &str) -> Option> { + get::(self.service, name) + } +} + pub(crate) fn get(map: &Map, name: &str) -> Option> { map.get(name).map(|(_, s)| { s.clone() diff --git a/src/service/services.rs b/src/service/services.rs index fb13f24a..03f3a9ba 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -5,7 +5,7 @@ use database::Database; use tokio::sync::Mutex; use crate::{ - account_data, admin, appservice, globals, key_backups, + account_data, admin, appservice, client, globals, key_backups, manager::Manager, media, presence, pusher, resolver, rooms, sending, service, service::{Args, Map, Service}, @@ -14,6 +14,7 @@ use crate::{ pub struct Services { pub resolver: Arc, + pub client: Arc, pub globals: Arc, pub rooms: rooms::Service, pub appservice: Arc, @@ -52,6 +53,7 @@ impl Services { Ok(Self { resolver: build!(resolver::Service), + client: build!(client::Service), globals: build!(globals::Service), rooms: rooms::Service { alias: build!(rooms::alias::Service), diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index 3fb680d6..db69d9b0 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -64,7 +64,6 @@ impl Service { #[tracing::instrument(skip_all)] async fn handle_updates(&self) -> Result<()> { let response = services() - .globals .client .default .get(CHECK_FOR_UPDATES_URL) From 3ccd9ea326cefaf80946b4c334f314d3c87b0598 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 16 Jul 2024 23:38:48 +0000 Subject: [PATCH 0554/2091] consolidate all resolution in resolver; split units Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 7 +- src/admin/query/resolver.rs | 13 +- src/service/client/mod.rs | 14 +- src/service/resolver/actual.rs | 356 +++++++++++++++++++++++++++++++++ src/service/resolver/cache.rs | 114 +++++++++++ src/service/resolver/dns.rs | 129 ++++++++++++ src/service/resolver/fed.rs | 70 +++++++ src/service/resolver/mod.rs | 351 ++++---------------------------- src/service/resolver/tests.rs | 43 ++++ src/service/sending/mod.rs | 2 - src/service/sending/send.rs | 11 +- 11 files changed, 774 insertions(+), 336 deletions(-) create mode 100644 src/service/resolver/actual.rs create mode 100644 src/service/resolver/cache.rs create mode 100644 src/service/resolver/dns.rs create mode 100644 src/service/resolver/fed.rs create mode 100644 src/service/resolver/tests.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 7c7f9331..f319f5a5 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -15,7 +15,7 @@ use ruma::{ events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, }; -use service::{rooms::event_handler::parse_incoming_pdu, sending::resolve_actual_dest, services, PduEvent}; +use service::{rooms::event_handler::parse_incoming_pdu, services, PduEvent}; use tokio::sync::RwLock; use tracing_subscriber::EnvFilter; @@ -628,7 +628,10 @@ pub(super) async fn resolve_true_destination( let capture = Capture::new(state, Some(filter), capture::fmt_markdown(logs.clone())); let capture_scope = capture.start(); - let actual = resolve_actual_dest(&server_name, !no_cache).await?; + let actual = services() + .resolver + .resolve_actual_dest(&server_name, !no_cache) + .await?; drop(capture_scope); let msg = format!( diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index 06cd8ba9..37d17960 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -19,7 +19,7 @@ pub(super) async fn resolver(subcommand: Resolver) -> Result) -> Result { - use service::resolver::CachedDest; + use service::resolver::cache::CachedDest; let mut out = String::new(); writeln!(out, "| Server Name | Destination | Hostname | Expires |")?; @@ -36,7 +36,12 @@ async fn destinations_cache(server_name: Option) -> Result) -> Result) -> Result { - use service::resolver::CachedOverride; + use service::resolver::cache::CachedOverride; let mut out = String::new(); writeln!(out, "| Server Name | IP | Port | Expires |")?; @@ -65,7 +70,7 @@ async fn overrides_cache(server_name: Option) -> Result Result { + let cached; + let cached_result = self.get_cached_destination(server_name); + + let CachedDest { + dest, + host, + .. + } = if let Some(result) = cached_result { + cached = true; + result + } else { + cached = false; + validate_dest(server_name)?; + self.resolve_actual_dest(server_name, true).await? + }; + + let string = dest.clone().into_https_string(); + Ok(ActualDest { + dest, + host, + string, + cached, + }) + } + + /// Returns: `actual_destination`, host header + /// Implemented according to the specification at + /// Numbers in comments below refer to bullet points in linked section of + /// specification + #[tracing::instrument(skip_all, name = "actual")] + pub async fn resolve_actual_dest(&self, dest: &ServerName, cache: bool) -> Result { + trace!("Finding actual destination for {dest}"); + let mut host = dest.as_str().to_owned(); + let actual_dest = match get_ip_with_port(dest.as_str()) { + Some(host_port) => Self::actual_dest_1(host_port)?, + None => { + if let Some(pos) = dest.as_str().find(':') { + self.actual_dest_2(dest, cache, pos).await? + } else if let Some(delegated) = self.request_well_known(dest.as_str()).await? { + self.actual_dest_3(&mut host, cache, delegated).await? + } else if let Some(overrider) = self.query_srv_record(dest.as_str()).await? { + self.actual_dest_4(&host, cache, overrider).await? + } else { + self.actual_dest_5(dest, cache).await? + } + }, + }; + + // Can't use get_ip_with_port here because we don't want to add a port + // to an IP address if it wasn't specified + let host = if let Ok(addr) = host.parse::() { + FedDest::Literal(addr) + } else if let Ok(addr) = host.parse::() { + FedDest::Named(addr.to_string(), ":8448".to_owned()) + } else if let Some(pos) = host.find(':') { + let (host, port) = host.split_at(pos); + FedDest::Named(host.to_owned(), port.to_owned()) + } else { + FedDest::Named(host, ":8448".to_owned()) + }; + + debug!("Actual destination: {actual_dest:?} hostname: {host:?}"); + Ok(CachedDest { + dest: actual_dest, + host: host.into_uri_string(), + expire: CachedDest::default_expire(), + }) + } + + fn actual_dest_1(host_port: FedDest) -> Result { + debug!("1: IP literal with provided or default port"); + Ok(host_port) + } + + async fn actual_dest_2(&self, dest: &ServerName, cache: bool, pos: usize) -> Result { + debug!("2: Hostname with included port"); + let (host, port) = dest.as_str().split_at(pos); + self.conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache) + .await?; + Ok(FedDest::Named(host.to_owned(), port.to_owned())) + } + + async fn actual_dest_3(&self, host: &mut String, cache: bool, delegated: String) -> Result { + debug!("3: A .well-known file is available"); + *host = add_port_to_hostname(&delegated).into_uri_string(); + match get_ip_with_port(&delegated) { + Some(host_and_port) => Self::actual_dest_3_1(host_and_port), + None => { + if let Some(pos) = delegated.find(':') { + self.actual_dest_3_2(cache, delegated, pos).await + } else { + trace!("Delegated hostname has no port in this branch"); + if let Some(overrider) = self.query_srv_record(&delegated).await? { + self.actual_dest_3_3(cache, delegated, overrider).await + } else { + self.actual_dest_3_4(cache, delegated).await + } + } + }, + } + } + + fn actual_dest_3_1(host_and_port: FedDest) -> Result { + debug!("3.1: IP literal in .well-known file"); + Ok(host_and_port) + } + + async fn actual_dest_3_2(&self, cache: bool, delegated: String, pos: usize) -> Result { + debug!("3.2: Hostname with port in .well-known file"); + let (host, port) = delegated.split_at(pos); + self.conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache) + .await?; + Ok(FedDest::Named(host.to_owned(), port.to_owned())) + } + + async fn actual_dest_3_3(&self, cache: bool, delegated: String, overrider: FedDest) -> Result { + debug!("3.3: SRV lookup successful"); + let force_port = overrider.port(); + self.conditional_query_and_cache_override(&delegated, &overrider.hostname(), force_port.unwrap_or(8448), cache) + .await?; + if let Some(port) = force_port { + Ok(FedDest::Named(delegated, format!(":{port}"))) + } else { + Ok(add_port_to_hostname(&delegated)) + } + } + + async fn actual_dest_3_4(&self, cache: bool, delegated: String) -> Result { + debug!("3.4: No SRV records, just use the hostname from .well-known"); + self.conditional_query_and_cache_override(&delegated, &delegated, 8448, cache) + .await?; + Ok(add_port_to_hostname(&delegated)) + } + + async fn actual_dest_4(&self, host: &str, cache: bool, overrider: FedDest) -> Result { + debug!("4: No .well-known; SRV record found"); + let force_port = overrider.port(); + self.conditional_query_and_cache_override(host, &overrider.hostname(), force_port.unwrap_or(8448), cache) + .await?; + if let Some(port) = force_port { + Ok(FedDest::Named(host.to_owned(), format!(":{port}"))) + } else { + Ok(add_port_to_hostname(host)) + } + } + + async fn actual_dest_5(&self, dest: &ServerName, cache: bool) -> Result { + debug!("5: No SRV record found"); + self.conditional_query_and_cache_override(dest.as_str(), dest.as_str(), 8448, cache) + .await?; + Ok(add_port_to_hostname(dest.as_str())) + } + + #[tracing::instrument(skip_all, name = "well-known")] + async fn request_well_known(&self, dest: &str) -> Result> { + trace!("Requesting well known for {dest}"); + if !self.has_cached_override(dest) { + self.query_and_cache_override(dest, dest, 8448).await?; + } + + let response = services() + .client + .well_known + .get(&format!("https://{dest}/.well-known/matrix/server")) + .send() + .await; + + trace!("response: {:?}", response); + if let Err(e) = &response { + debug!("error: {e:?}"); + return Ok(None); + } + + let response = response?; + if !response.status().is_success() { + debug!("response not 2XX"); + return Ok(None); + } + + let text = response.text().await?; + trace!("response text: {:?}", text); + if text.len() >= 12288 { + debug_warn!("response contains junk"); + return Ok(None); + } + + let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); + + let m_server = body + .get("m.server") + .unwrap_or(&serde_json::Value::Null) + .as_str() + .unwrap_or_default(); + + if ruma::identifiers_validation::server_name::validate(m_server).is_err() { + debug_error!("response content missing or invalid"); + return Ok(None); + } + + debug_info!("{:?} found at {:?}", dest, m_server); + Ok(Some(m_server.to_owned())) + } + + #[inline] + async fn conditional_query_and_cache_override( + &self, overname: &str, hostname: &str, port: u16, cache: bool, + ) -> Result<()> { + if cache { + self.query_and_cache_override(overname, hostname, port) + .await + } else { + Ok(()) + } + } + + #[tracing::instrument(skip_all, name = "ip")] + async fn query_and_cache_override(&self, overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { + match services() + .resolver + .raw() + .lookup_ip(hostname.to_owned()) + .await + { + Err(e) => handle_resolve_error(&e), + Ok(override_ip) => { + if hostname != overname { + debug_info!("{overname:?} overriden by {hostname:?}"); + } + + services().resolver.set_cached_override( + overname.to_owned(), + CachedOverride { + ips: override_ip.iter().collect(), + port, + expire: CachedOverride::default_expire(), + }, + ); + + Ok(()) + }, + } + } + + #[tracing::instrument(skip_all, name = "srv")] + async fn query_srv_record(&self, hostname: &'_ str) -> Result> { + fn handle_successful_srv(srv: &SrvLookup) -> Option { + srv.iter().next().map(|result| { + FedDest::Named( + result.target().to_string().trim_end_matches('.').to_owned(), + format!(":{}", result.port()), + ) + }) + } + + async fn lookup_srv( + resolver: Arc, hostname: &str, + ) -> Result { + debug!("querying SRV for {hostname:?}"); + let hostname = hostname.trim_end_matches('.'); + resolver.srv_lookup(hostname.to_owned()).await + } + + let hostnames = [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")]; + + for hostname in hostnames { + match lookup_srv(self.raw(), &hostname).await { + Ok(result) => return Ok(handle_successful_srv(&result)), + Err(e) => handle_resolve_error(&e)?, + } + } + + Ok(None) + } +} + +#[allow(clippy::single_match_else)] +fn handle_resolve_error(e: &ResolveError) -> Result<()> { + use hickory_resolver::error::ResolveErrorKind; + + match *e.kind() { + ResolveErrorKind::NoRecordsFound { + .. + } => { + // Raise to debug_warn if we can find out the result wasn't from cache + debug!("{e}"); + Ok(()) + }, + _ => Err!(error!("DNS {e}")), + } +} + +fn validate_dest(dest: &ServerName) -> Result<()> { + if dest == services().globals.server_name() { + return Err!("Won't send federation request to ourselves"); + } + + if dest.is_ip_literal() || IPAddress::is_valid(dest.host()) { + validate_dest_ip_literal(dest)?; + } + + Ok(()) +} + +fn validate_dest_ip_literal(dest: &ServerName) -> Result<()> { + trace!("Destination is an IP literal, checking against IP range denylist.",); + debug_assert!( + dest.is_ip_literal() || !IPAddress::is_valid(dest.host()), + "Destination is not an IP literal." + ); + let ip = IPAddress::parse(dest.host()).map_err(|e| { + debug_error!("Failed to parse IP literal from string: {}", e); + Error::BadServerResponse("Invalid IP address") + })?; + + validate_ip(&ip)?; + + Ok(()) +} + +pub(crate) fn validate_ip(ip: &IPAddress) -> Result<()> { + if !services().globals.valid_cidr_range(ip) { + return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); + } + + Ok(()) +} diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs new file mode 100644 index 00000000..0fba2400 --- /dev/null +++ b/src/service/resolver/cache.rs @@ -0,0 +1,114 @@ +use std::{ + collections::HashMap, + net::IpAddr, + sync::{Arc, RwLock}, + time::SystemTime, +}; + +use conduit::trace; +use ruma::{OwnedServerName, ServerName}; + +use super::fed::FedDest; +use crate::utils::rand; + +pub struct Cache { + pub destinations: RwLock, // actual_destination, host + pub overrides: RwLock, +} + +#[derive(Clone, Debug)] +pub struct CachedDest { + pub dest: FedDest, + pub host: String, + pub expire: SystemTime, +} + +#[derive(Clone, Debug)] +pub struct CachedOverride { + pub ips: Vec, + pub port: u16, + pub expire: SystemTime, +} + +pub type WellKnownMap = HashMap; +pub type TlsNameMap = HashMap; + +impl Cache { + pub(super) fn new() -> Arc { + Arc::new(Self { + destinations: RwLock::new(WellKnownMap::new()), + overrides: RwLock::new(TlsNameMap::new()), + }) + } +} + +impl super::Service { + pub fn set_cached_destination(&self, name: OwnedServerName, dest: CachedDest) -> Option { + trace!(?name, ?dest, "set cached destination"); + self.cache + .destinations + .write() + .expect("locked for writing") + .insert(name, dest) + } + + #[must_use] + pub fn get_cached_destination(&self, name: &ServerName) -> Option { + self.cache + .destinations + .read() + .expect("locked for reading") + .get(name) + .cloned() + } + + pub fn set_cached_override(&self, name: String, over: CachedOverride) -> Option { + trace!(?name, ?over, "set cached override"); + self.cache + .overrides + .write() + .expect("locked for writing") + .insert(name, over) + } + + #[must_use] + pub fn get_cached_override(&self, name: &str) -> Option { + self.cache + .overrides + .read() + .expect("locked for reading") + .get(name) + .cloned() + } + + #[must_use] + pub fn has_cached_override(&self, name: &str) -> bool { + self.cache + .overrides + .read() + .expect("locked for reading") + .contains_key(name) + } +} + +impl CachedDest { + #[inline] + #[must_use] + pub fn valid(&self) -> bool { true } + + //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + + #[must_use] + pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } +} + +impl CachedOverride { + #[inline] + #[must_use] + pub fn valid(&self) -> bool { true } + + //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + + #[must_use] + pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } +} diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs new file mode 100644 index 00000000..b77bbb84 --- /dev/null +++ b/src/service/resolver/dns.rs @@ -0,0 +1,129 @@ +use std::{ + future, iter, + net::{IpAddr, SocketAddr}, + sync::Arc, + time::Duration, +}; + +use conduit::{err, Result, Server}; +use hickory_resolver::TokioAsyncResolver; +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; + +use super::cache::Cache; + +pub struct Resolver { + pub(crate) resolver: Arc, + pub(crate) hooked: Arc, +} + +pub(crate) struct Hooked { + resolver: Arc, + cache: Arc, +} + +impl Resolver { + #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] + pub(super) fn build(server: &Arc, cache: Arc) -> Result> { + let config = &server.config; + let (sys_conf, mut opts) = hickory_resolver::system_conf::read_system_conf() + .map_err(|e| err!(error!("Failed to configure DNS resolver from system: {e}")))?; + + let mut conf = hickory_resolver::config::ResolverConfig::new(); + + if let Some(domain) = sys_conf.domain() { + conf.set_domain(domain.clone()); + } + + for sys_conf in sys_conf.search() { + conf.add_search(sys_conf.clone()); + } + + for sys_conf in sys_conf.name_servers() { + let mut ns = sys_conf.clone(); + + if config.query_over_tcp_only { + ns.protocol = hickory_resolver::config::Protocol::Tcp; + } + + ns.trust_negative_responses = !config.query_all_nameservers; + + conf.add_name_server(ns); + } + + opts.cache_size = config.dns_cache_entries as usize; + opts.negative_min_ttl = Some(Duration::from_secs(config.dns_min_ttl_nxdomain)); + opts.negative_max_ttl = Some(Duration::from_secs(60 * 60 * 24 * 30)); + opts.positive_min_ttl = Some(Duration::from_secs(config.dns_min_ttl)); + opts.positive_max_ttl = Some(Duration::from_secs(60 * 60 * 24 * 7)); + opts.timeout = Duration::from_secs(config.dns_timeout); + opts.attempts = config.dns_attempts as usize; + opts.try_tcp_on_error = config.dns_tcp_fallback; + opts.num_concurrent_reqs = 1; + opts.shuffle_dns_servers = true; + opts.rotate = true; + opts.ip_strategy = match config.ip_lookup_strategy { + 1 => hickory_resolver::config::LookupIpStrategy::Ipv4Only, + 2 => hickory_resolver::config::LookupIpStrategy::Ipv6Only, + 3 => hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6, + 4 => hickory_resolver::config::LookupIpStrategy::Ipv6thenIpv4, + _ => hickory_resolver::config::LookupIpStrategy::Ipv4thenIpv6, + }; + opts.authentic_data = false; + + let resolver = Arc::new(TokioAsyncResolver::tokio(conf, opts)); + Ok(Arc::new(Self { + resolver: resolver.clone(), + hooked: Arc::new(Hooked { + resolver, + cache, + }), + })) + } +} + +impl Resolve for Resolver { + fn resolve(&self, name: Name) -> Resolving { resolve_to_reqwest(self.resolver.clone(), name) } +} + +impl Resolve for Hooked { + fn resolve(&self, name: Name) -> Resolving { + let cached = self + .cache + .overrides + .read() + .expect("locked for reading") + .get(name.as_str()) + .cloned(); + + if let Some(cached) = cached { + cached_to_reqwest(&cached.ips, cached.port) + } else { + resolve_to_reqwest(self.resolver.clone(), name) + } + } +} + +fn cached_to_reqwest(override_name: &[IpAddr], port: u16) -> Resolving { + override_name + .first() + .map(|first_name| -> Resolving { + let saddr = SocketAddr::new(*first_name, port); + let result: Box + Send> = Box::new(iter::once(saddr)); + Box::pin(future::ready(Ok(result))) + }) + .expect("must provide at least one override name") +} + +fn resolve_to_reqwest(resolver: Arc, name: Name) -> Resolving { + Box::pin(async move { + let results = resolver + .lookup_ip(name.as_str()) + .await? + .into_iter() + .map(|ip| SocketAddr::new(ip, 0)); + + let results: Addrs = Box::new(results); + + Ok(results) + }) +} diff --git a/src/service/resolver/fed.rs b/src/service/resolver/fed.rs new file mode 100644 index 00000000..10cbbbdd --- /dev/null +++ b/src/service/resolver/fed.rs @@ -0,0 +1,70 @@ +use std::{ + fmt, + net::{IpAddr, SocketAddr}, +}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FedDest { + Literal(SocketAddr), + Named(String, String), +} + +pub(crate) fn get_ip_with_port(dest_str: &str) -> Option { + if let Ok(dest) = dest_str.parse::() { + Some(FedDest::Literal(dest)) + } else if let Ok(ip_addr) = dest_str.parse::() { + Some(FedDest::Literal(SocketAddr::new(ip_addr, 8448))) + } else { + None + } +} + +pub(crate) fn add_port_to_hostname(dest_str: &str) -> FedDest { + let (host, port) = match dest_str.find(':') { + None => (dest_str, ":8448"), + Some(pos) => dest_str.split_at(pos), + }; + + FedDest::Named(host.to_owned(), port.to_owned()) +} + +impl FedDest { + pub(crate) fn into_https_string(self) -> String { + match self { + Self::Literal(addr) => format!("https://{addr}"), + Self::Named(host, port) => format!("https://{host}{port}"), + } + } + + pub(crate) fn into_uri_string(self) -> String { + match self { + Self::Literal(addr) => addr.to_string(), + Self::Named(host, port) => format!("{host}{port}"), + } + } + + pub(crate) fn hostname(&self) -> String { + match &self { + Self::Literal(addr) => addr.ip().to_string(), + Self::Named(host, _) => host.clone(), + } + } + + #[inline] + #[allow(clippy::string_slice)] + pub(crate) fn port(&self) -> Option { + match &self { + Self::Literal(addr) => Some(addr.port()), + Self::Named(_, port) => port[1..].parse().ok(), + } + } +} + +impl fmt::Display for FedDest { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Named(host, port) => write!(f, "{host}{port}"), + Self::Literal(addr) => write!(f, "{addr}"), + } + } +} diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 62fd1625..48ff8813 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -1,349 +1,66 @@ -use std::{ - collections::HashMap, - fmt, - fmt::Write, - future, iter, - net::{IpAddr, SocketAddr}, - sync::{Arc, RwLock}, - time::{Duration, SystemTime}, -}; +pub mod actual; +pub mod cache; +mod dns; +pub mod fed; +mod tests; -use conduit::{err, trace, Result}; +use std::{fmt::Write, sync::Arc}; + +use conduit::Result; use hickory_resolver::TokioAsyncResolver; -use reqwest::dns::{Addrs, Name, Resolve, Resolving}; -use ruma::{OwnedServerName, ServerName}; -use crate::utils::rand; +use self::{cache::Cache, dns::Resolver}; pub struct Service { - pub destinations: Arc>, // actual_destination, host - pub overrides: Arc>, - pub(crate) resolver: Arc, - pub(crate) hooked: Arc, + pub cache: Arc, + pub resolver: Arc, } -pub(crate) struct Hooked { - overrides: Arc>, - resolver: Arc, -} - -#[derive(Clone, Debug)] -pub struct CachedDest { - pub dest: FedDest, - pub host: String, - pub expire: SystemTime, -} - -#[derive(Clone, Debug)] -pub struct CachedOverride { - pub ips: Vec, - pub port: u16, - pub expire: SystemTime, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum FedDest { - Literal(SocketAddr), - Named(String, String), -} - -type WellKnownMap = HashMap; -type TlsNameMap = HashMap; - impl crate::Service for Service { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] fn build(args: crate::Args<'_>) -> Result> { - let config = &args.server.config; - let (sys_conf, mut opts) = hickory_resolver::system_conf::read_system_conf() - .map_err(|e| err!(error!("Failed to configure DNS resolver from system: {e}")))?; - - let mut conf = hickory_resolver::config::ResolverConfig::new(); - - if let Some(domain) = sys_conf.domain() { - conf.set_domain(domain.clone()); - } - - for sys_conf in sys_conf.search() { - conf.add_search(sys_conf.clone()); - } - - for sys_conf in sys_conf.name_servers() { - let mut ns = sys_conf.clone(); - - if config.query_over_tcp_only { - ns.protocol = hickory_resolver::config::Protocol::Tcp; - } - - ns.trust_negative_responses = !config.query_all_nameservers; - - conf.add_name_server(ns); - } - - opts.cache_size = config.dns_cache_entries as usize; - opts.negative_min_ttl = Some(Duration::from_secs(config.dns_min_ttl_nxdomain)); - opts.negative_max_ttl = Some(Duration::from_secs(60 * 60 * 24 * 30)); - opts.positive_min_ttl = Some(Duration::from_secs(config.dns_min_ttl)); - opts.positive_max_ttl = Some(Duration::from_secs(60 * 60 * 24 * 7)); - opts.timeout = Duration::from_secs(config.dns_timeout); - opts.attempts = config.dns_attempts as usize; - opts.try_tcp_on_error = config.dns_tcp_fallback; - opts.num_concurrent_reqs = 1; - opts.shuffle_dns_servers = true; - opts.rotate = true; - opts.ip_strategy = match config.ip_lookup_strategy { - 1 => hickory_resolver::config::LookupIpStrategy::Ipv4Only, - 2 => hickory_resolver::config::LookupIpStrategy::Ipv6Only, - 3 => hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6, - 4 => hickory_resolver::config::LookupIpStrategy::Ipv6thenIpv4, - _ => hickory_resolver::config::LookupIpStrategy::Ipv4thenIpv6, - }; - opts.authentic_data = false; - - let resolver = Arc::new(TokioAsyncResolver::tokio(conf, opts)); - let overrides = Arc::new(RwLock::new(TlsNameMap::new())); + let cache = Cache::new(); Ok(Arc::new(Self { - destinations: Arc::new(RwLock::new(WellKnownMap::new())), - overrides: overrides.clone(), - resolver: resolver.clone(), - hooked: Arc::new(Hooked { - overrides, - resolver, - }), + cache: cache.clone(), + resolver: Resolver::build(args.server, cache)?, })) } fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { - let resolver_overrides_cache = self.overrides.read().expect("locked for reading").len(); + let resolver_overrides_cache = self + .cache + .overrides + .read() + .expect("locked for reading") + .len(); writeln!(out, "resolver_overrides_cache: {resolver_overrides_cache}")?; - let resolver_destinations_cache = self.destinations.read().expect("locked for reading").len(); + let resolver_destinations_cache = self + .cache + .destinations + .read() + .expect("locked for reading") + .len(); writeln!(out, "resolver_destinations_cache: {resolver_destinations_cache}")?; Ok(()) } fn clear_cache(&self) { - self.overrides.write().expect("write locked").clear(); - self.destinations.write().expect("write locked").clear(); - self.resolver.clear_cache(); + self.cache.overrides.write().expect("write locked").clear(); + self.cache + .destinations + .write() + .expect("write locked") + .clear(); + self.resolver.resolver.clear_cache(); } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } impl Service { - pub fn set_cached_destination(&self, name: OwnedServerName, dest: CachedDest) -> Option { - trace!(?name, ?dest, "set cached destination"); - self.destinations - .write() - .expect("locked for writing") - .insert(name, dest) - } - - #[must_use] - pub fn get_cached_destination(&self, name: &ServerName) -> Option { - self.destinations - .read() - .expect("locked for reading") - .get(name) - .cloned() - } - - pub fn set_cached_override(&self, name: String, over: CachedOverride) -> Option { - trace!(?name, ?over, "set cached override"); - self.overrides - .write() - .expect("locked for writing") - .insert(name, over) - } - - #[must_use] - pub fn get_cached_override(&self, name: &str) -> Option { - self.overrides - .read() - .expect("locked for reading") - .get(name) - .cloned() - } - - #[must_use] - pub fn has_cached_override(&self, name: &str) -> bool { - self.overrides - .read() - .expect("locked for reading") - .contains_key(name) - } -} - -impl Resolve for Service { - fn resolve(&self, name: Name) -> Resolving { resolve_to_reqwest(self.resolver.clone(), name) } -} - -impl Resolve for Hooked { - fn resolve(&self, name: Name) -> Resolving { - let cached = self - .overrides - .read() - .expect("locked for reading") - .get(name.as_str()) - .cloned(); - - if let Some(cached) = cached { - cached_to_reqwest(&cached.ips, cached.port) - } else { - resolve_to_reqwest(self.resolver.clone(), name) - } - } -} - -fn cached_to_reqwest(override_name: &[IpAddr], port: u16) -> Resolving { - override_name - .first() - .map(|first_name| -> Resolving { - let saddr = SocketAddr::new(*first_name, port); - let result: Box + Send> = Box::new(iter::once(saddr)); - Box::pin(future::ready(Ok(result))) - }) - .expect("must provide at least one override name") -} - -fn resolve_to_reqwest(resolver: Arc, name: Name) -> Resolving { - Box::pin(async move { - let results = resolver - .lookup_ip(name.as_str()) - .await? - .into_iter() - .map(|ip| SocketAddr::new(ip, 0)); - - let results: Addrs = Box::new(results); - - Ok(results) - }) -} - -impl CachedDest { #[inline] #[must_use] - pub fn valid(&self) -> bool { true } - - //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } - - #[must_use] - pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } -} - -impl CachedOverride { - #[inline] - #[must_use] - pub fn valid(&self) -> bool { true } - - //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } - - #[must_use] - pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } -} - -pub(crate) fn get_ip_with_port(dest_str: &str) -> Option { - if let Ok(dest) = dest_str.parse::() { - Some(FedDest::Literal(dest)) - } else if let Ok(ip_addr) = dest_str.parse::() { - Some(FedDest::Literal(SocketAddr::new(ip_addr, 8448))) - } else { - None - } -} - -pub(crate) fn add_port_to_hostname(dest_str: &str) -> FedDest { - let (host, port) = match dest_str.find(':') { - None => (dest_str, ":8448"), - Some(pos) => dest_str.split_at(pos), - }; - - FedDest::Named(host.to_owned(), port.to_owned()) -} - -impl FedDest { - pub(crate) fn into_https_string(self) -> String { - match self { - Self::Literal(addr) => format!("https://{addr}"), - Self::Named(host, port) => format!("https://{host}{port}"), - } - } - - pub(crate) fn into_uri_string(self) -> String { - match self { - Self::Literal(addr) => addr.to_string(), - Self::Named(host, port) => format!("{host}{port}"), - } - } - - pub(crate) fn hostname(&self) -> String { - match &self { - Self::Literal(addr) => addr.ip().to_string(), - Self::Named(host, _) => host.clone(), - } - } - - #[inline] - #[allow(clippy::string_slice)] - pub(crate) fn port(&self) -> Option { - match &self { - Self::Literal(addr) => Some(addr.port()), - Self::Named(_, port) => port[1..].parse().ok(), - } - } -} - -impl fmt::Display for FedDest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Named(host, port) => write!(f, "{host}{port}"), - Self::Literal(addr) => write!(f, "{addr}"), - } - } -} - -#[cfg(test)] -mod tests { - use super::{add_port_to_hostname, get_ip_with_port, FedDest}; - - #[test] - fn ips_get_default_ports() { - assert_eq!( - get_ip_with_port("1.1.1.1"), - Some(FedDest::Literal("1.1.1.1:8448".parse().unwrap())) - ); - assert_eq!( - get_ip_with_port("dead:beef::"), - Some(FedDest::Literal("[dead:beef::]:8448".parse().unwrap())) - ); - } - - #[test] - fn ips_keep_custom_ports() { - assert_eq!( - get_ip_with_port("1.1.1.1:1234"), - Some(FedDest::Literal("1.1.1.1:1234".parse().unwrap())) - ); - assert_eq!( - get_ip_with_port("[dead::beef]:8933"), - Some(FedDest::Literal("[dead::beef]:8933".parse().unwrap())) - ); - } - - #[test] - fn hostnames_get_default_ports() { - assert_eq!( - add_port_to_hostname("example.com"), - FedDest::Named(String::from("example.com"), String::from(":8448")) - ); - } - - #[test] - fn hostnames_keep_custom_ports() { - assert_eq!( - add_port_to_hostname("example.com:1337"), - FedDest::Named(String::from("example.com"), String::from(":1337")) - ); - } + pub fn raw(&self) -> Arc { self.resolver.resolver.clone() } } diff --git a/src/service/resolver/tests.rs b/src/service/resolver/tests.rs new file mode 100644 index 00000000..55cf0345 --- /dev/null +++ b/src/service/resolver/tests.rs @@ -0,0 +1,43 @@ +#![cfg(test)] + +use super::fed::{add_port_to_hostname, get_ip_with_port, FedDest}; + +#[test] +fn ips_get_default_ports() { + assert_eq!( + get_ip_with_port("1.1.1.1"), + Some(FedDest::Literal("1.1.1.1:8448".parse().unwrap())) + ); + assert_eq!( + get_ip_with_port("dead:beef::"), + Some(FedDest::Literal("[dead:beef::]:8448".parse().unwrap())) + ); +} + +#[test] +fn ips_keep_custom_ports() { + assert_eq!( + get_ip_with_port("1.1.1.1:1234"), + Some(FedDest::Literal("1.1.1.1:1234".parse().unwrap())) + ); + assert_eq!( + get_ip_with_port("[dead::beef]:8933"), + Some(FedDest::Literal("[dead::beef]:8933".parse().unwrap())) + ); +} + +#[test] +fn hostnames_get_default_ports() { + assert_eq!( + add_port_to_hostname("example.com"), + FedDest::Named(String::from("example.com"), String::from(":8448")) + ); +} + +#[test] +fn hostnames_keep_custom_ports() { + assert_eq!( + add_port_to_hostname("example.com:1337"), + FedDest::Named(String::from("example.com"), String::from(":1337")) + ); +} diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index be10184d..1eacca77 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -1,13 +1,11 @@ mod appservice; mod data; -mod resolve; mod send; mod sender; use std::fmt::Debug; use conduit::{err, Result}; -pub use resolve::resolve_actual_dest; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index dc4541d6..df3139c3 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -15,8 +15,11 @@ use ruma::{ }; use tracing::{debug, trace}; -use super::{resolve, resolve::ActualDest}; -use crate::{debug_error, debug_warn, resolver::CachedDest, services, Error, Result}; +use crate::{ + debug_error, debug_warn, resolver, + resolver::{actual::ActualDest, cache::CachedDest}, + services, Error, Result, +}; #[tracing::instrument(skip_all, name = "send")] pub async fn send(client: &Client, dest: &ServerName, req: T) -> Result @@ -27,7 +30,7 @@ where return Err!(Config("allow_federation", "Federation is disabled.")); } - let actual = resolve::get_actual_dest(dest).await?; + let actual = services().resolver.get_actual_dest(dest).await?; let request = prepare::(dest, &actual, req).await?; execute::(client, dest, &actual, request).await } @@ -219,7 +222,7 @@ fn validate_url(url: &Url) -> Result<()> { if let Some(url_host) = url.host_str() { if let Ok(ip) = IPAddress::parse(url_host) { trace!("Checking request URL IP {ip:?}"); - resolve::validate_ip(&ip)?; + resolver::actual::validate_ip(&ip)?; } } From 29fc5b9b524db556a5b45d899ff25fb9883bf097 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 17 Jul 2024 01:00:57 +0000 Subject: [PATCH 0555/2091] de-global some services in services Signed-off-by: Jason Volk --- src/service/sending/data.rs | 4 ++-- src/service/sending/mod.rs | 17 ++++++++--------- src/service/sending/sender.rs | 26 ++++++++++++-------------- src/service/services.rs | 2 +- 4 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 65725618..9cb1c267 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -14,7 +14,7 @@ pub struct Data { servercurrentevent_data: Arc, servernameevent_data: Arc, servername_educount: Arc, - _db: Arc, + pub(super) db: Arc, } impl Data { @@ -23,7 +23,7 @@ impl Data { servercurrentevent_data: db["servercurrentevent_data"].clone(), servernameevent_data: db["servernameevent_data"].clone(), servername_educount: db["servername_educount"].clone(), - _db: db, + db, } } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 1eacca77..a6c3411f 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -3,9 +3,9 @@ mod data; mod send; mod sender; -use std::fmt::Debug; +use std::{fmt::Debug, sync::Arc}; -use conduit::{err, Result}; +use conduit::{err, Result, Server}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, @@ -18,12 +18,11 @@ use crate::{server_is_ours, services}; pub struct Service { pub db: data::Data, + server: Arc, /// The state for a given state hash. sender: loole::Sender, receiver: Mutex>, - startup_netburst: bool, - startup_netburst_keep: i64, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -53,7 +52,7 @@ impl Service { pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { let dest = Destination::Push(user.to_owned(), pushkey); let event = SendingEvent::Pdu(pdu_id.to_owned()); - let _cork = services().db.cork(); + let _cork = self.db.db.cork(); let keys = self.db.queue_requests(&[(&dest, event.clone())])?; self.dispatch(Msg { dest, @@ -66,7 +65,7 @@ impl Service { pub fn send_pdu_appservice(&self, appservice_id: String, pdu_id: Vec) -> Result<()> { let dest = Destination::Appservice(appservice_id); let event = SendingEvent::Pdu(pdu_id); - let _cork = services().db.cork(); + let _cork = self.db.db.cork(); let keys = self.db.queue_requests(&[(&dest, event.clone())])?; self.dispatch(Msg { dest, @@ -93,7 +92,7 @@ impl Service { .into_iter() .map(|server| (Destination::Normal(server), SendingEvent::Pdu(pdu_id.to_owned()))) .collect::>(); - let _cork = services().db.cork(); + let _cork = self.db.db.cork(); let keys = self.db.queue_requests( &requests .iter() @@ -115,7 +114,7 @@ impl Service { pub fn send_edu_server(&self, server: &ServerName, serialized: Vec) -> Result<()> { let dest = Destination::Normal(server.to_owned()); let event = SendingEvent::Edu(serialized); - let _cork = services().db.cork(); + let _cork = self.db.db.cork(); let keys = self.db.queue_requests(&[(&dest, event.clone())])?; self.dispatch(Msg { dest, @@ -142,7 +141,7 @@ impl Service { .into_iter() .map(|server| (Destination::Normal(server), SendingEvent::Edu(serialized.clone()))) .collect::>(); - let _cork = services().db.cork(); + let _cork = self.db.db.cork(); let keys = self.db.queue_requests( &requests .iter() diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index e6b68e9e..a924ce55 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -49,14 +49,12 @@ const CLEANUP_TIMEOUT_MS: u64 = 3500; #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { - let config = &args.server.config; let (sender, receiver) = loole::unbounded(); Ok(Arc::new(Self { db: Data::new(args.db.clone()), + server: args.server.clone(), sender, receiver: Mutex::new(receiver), - startup_netburst: config.startup_netburst, - startup_netburst_keep: config.startup_netburst_keep, })) } @@ -119,7 +117,7 @@ impl Service { fn handle_response_ok( &self, dest: &Destination, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, ) { - let _cork = services().db.cork(); + let _cork = self.db.db.cork(); self.db .delete_all_active_requests_for(dest) .expect("all active requests deleted"); @@ -174,11 +172,11 @@ impl Service { } fn initial_requests(&self, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus) { - let keep = usize::try_from(self.startup_netburst_keep).unwrap_or(usize::MAX); + let keep = usize::try_from(self.server.config.startup_netburst_keep).unwrap_or(usize::MAX); let mut txns = HashMap::>::new(); for (key, dest, event) in self.db.active_requests().filter_map(Result::ok) { let entry = txns.entry(dest.clone()).or_default(); - if self.startup_netburst_keep >= 0 && entry.len() >= keep { + if self.server.config.startup_netburst_keep >= 0 && entry.len() >= keep { warn!("Dropping unsent event {:?} {:?}", dest, String::from_utf8_lossy(&key)); self.db .delete_active_request(&key) @@ -189,7 +187,7 @@ impl Service { } for (dest, events) in txns { - if self.startup_netburst && !events.is_empty() { + if self.server.config.startup_netburst && !events.is_empty() { statuses.insert(dest.clone(), TransactionStatus::Running); futures.push(Box::pin(send_events(dest.clone(), events))); } @@ -210,7 +208,7 @@ impl Service { return Ok(None); } - let _cork = services().db.cork(); + let _cork = self.db.db.cork(); let mut events = Vec::new(); // Must retry any previous transaction for this remote. @@ -224,7 +222,7 @@ impl Service { } // Compose the next transaction - let _cork = services().db.cork(); + let _cork = self.db.db.cork(); if !new_events.is_empty() { self.db.mark_as_active(&new_events)?; for (e, _) in new_events { @@ -251,8 +249,8 @@ impl Service { .and_modify(|e| match e { TransactionStatus::Failed(tries, time) => { // Fail if a request has failed recently (exponential backoff) - let min = services().globals.config.sender_timeout; - let max = services().globals.config.sender_retry_backoff_limit; + let min = self.server.config.sender_timeout; + let max = self.server.config.sender_retry_backoff_limit; if continue_exponential_backoff_secs(min, max, time.elapsed(), *tries) { allow = false; } else { @@ -288,7 +286,7 @@ impl Service { .filter(|user_id| user_is_local(user_id)), ); - if services().globals.allow_outgoing_read_receipts() + if self.server.config.allow_outgoing_read_receipts && !select_edus_receipts(&room_id, since, &mut max_edu_count, &mut events)? { break; @@ -311,7 +309,7 @@ impl Service { events.push(serde_json::to_vec(&edu).expect("json can be serialized")); } - if services().globals.allow_outgoing_presence() { + if self.server.config.allow_outgoing_presence { select_edus_presence(server_name, since, &mut max_edu_count, &mut events)?; } @@ -617,7 +615,7 @@ async fn send_events_dest_normal( &services().client.sender, server, send_transaction_message::v1::Request { - origin: services().globals.server_name().to_owned(), + origin: services().server.config.server_name.clone(), pdus: pdu_jsons, edus: edu_jsons, origin_server_ts: MilliSecondsSinceUnixEpoch::now(), diff --git a/src/service/services.rs b/src/service/services.rs index 03f3a9ba..d0f74e13 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -52,9 +52,9 @@ impl Services { } Ok(Self { + globals: build!(globals::Service), resolver: build!(resolver::Service), client: build!(client::Service), - globals: build!(globals::Service), rooms: rooms::Service { alias: build!(rooms::alias::Service), auth_chain: build!(rooms::auth_chain::Service), From b0ac5255c8a6d211df86dde8a22cccfd9189b8c0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 17 Jul 2024 02:19:03 +0000 Subject: [PATCH 0556/2091] move sending service impl properly back to mod root Signed-off-by: Jason Volk --- src/service/sending/mod.rs | 27 +++++++++++++++++++++++++++ src/service/sending/sender.rs | 31 ++++--------------------------- 2 files changed, 31 insertions(+), 27 deletions(-) diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index a6c3411f..26f43fd3 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -5,6 +5,7 @@ mod sender; use std::{fmt::Debug, sync::Arc}; +use async_trait::async_trait; use conduit::{err, Result, Server}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, @@ -47,6 +48,32 @@ pub enum SendingEvent { Flush, // none } +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + let (sender, receiver) = loole::unbounded(); + Ok(Arc::new(Self { + db: data::Data::new(args.db.clone()), + server: args.server.clone(), + sender, + receiver: Mutex::new(receiver), + })) + } + + async fn worker(self: Arc) -> Result<()> { + // trait impl can't be split between files so this just glues to mod sender + self.sender().await + } + + fn interrupt(&self) { + if !self.sender.is_closed() { + self.sender.close(); + } + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + impl Service { #[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")] pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index a924ce55..df41db28 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -2,11 +2,9 @@ use std::{ cmp, collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, - sync::Arc, time::{Duration, Instant}, }; -use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; use conduit::{debug, debug_warn, error, trace, utils::math::continue_exponential_backoff_secs, warn}; use federation::transactions::send_transaction_message; @@ -24,9 +22,9 @@ use ruma::{ ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::{sync::Mutex, time::sleep_until}; +use tokio::time::sleep_until; -use super::{appservice, data::Data, send, Destination, Msg, SendingEvent, Service}; +use super::{appservice, send, Destination, Msg, SendingEvent, Service}; use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; #[derive(Debug)] @@ -46,20 +44,9 @@ const DEQUEUE_LIMIT: usize = 48; const SELECT_EDU_LIMIT: usize = 16; const CLEANUP_TIMEOUT_MS: u64 = 3500; -#[async_trait] -impl crate::Service for Service { - fn build(args: crate::Args<'_>) -> Result> { - let (sender, receiver) = loole::unbounded(); - Ok(Arc::new(Self { - db: Data::new(args.db.clone()), - server: args.server.clone(), - sender, - receiver: Mutex::new(receiver), - })) - } - +impl Service { #[tracing::instrument(skip_all, name = "sender")] - async fn worker(self: Arc) -> Result<()> { + pub(super) async fn sender(&self) -> Result<()> { let receiver = self.receiver.lock().await; let mut futures: SendingFutures<'_> = FuturesUnordered::new(); let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); @@ -82,16 +69,6 @@ impl crate::Service for Service { Ok(()) } - fn interrupt(&self) { - if !self.sender.is_closed() { - self.sender.close(); - } - } - - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } -} - -impl Service { fn handle_response( &self, response: SendingResult, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, ) { From b116984e46aa267e306da500db1b06ad7bc39ce1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 17 Jul 2024 02:48:43 +0000 Subject: [PATCH 0557/2091] use mutex_map for url preview lock Signed-off-by: Jason Volk --- src/api/client/media.rs | 13 ++----------- src/service/media/mod.rs | 13 ++++++++----- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 46f8152b..e219e457 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -1,6 +1,6 @@ #![allow(deprecated)] -use std::{io::Cursor, sync::Arc, time::Duration}; +use std::{io::Cursor, time::Duration}; use axum::extract::State; use axum_client_ip::InsecureClientIp; @@ -656,16 +656,7 @@ async fn get_url_preview(services: &Services, url: &str) -> Result Ok(preview), diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 1638235b..4a7d38a4 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -2,18 +2,17 @@ mod data; mod tests; mod thumbnail; -use std::{collections::HashMap, path::PathBuf, sync::Arc, time::SystemTime}; +use std::{path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, debug_error, err, error, utils, Err, Result, Server}; +use conduit::{debug, debug_error, err, error, utils, utils::MutexMap, Err, Result, Server}; use data::{Data, Metadata}; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt, BufReader}, - sync::{Mutex, RwLock}, }; use crate::services; @@ -44,7 +43,7 @@ pub struct UrlPreviewData { pub struct Service { server: Arc, pub(crate) db: Data, - pub url_preview_mutex: RwLock>>>, + pub url_preview_mutex: MutexMap, } #[async_trait] @@ -53,7 +52,7 @@ impl crate::Service for Service { Ok(Arc::new(Self { server: args.server.clone(), db: Data::new(args.db), - url_preview_mutex: RwLock::new(HashMap::new()), + url_preview_mutex: MutexMap::new(), })) } @@ -274,10 +273,12 @@ impl Service { } #[inline] + #[must_use] pub fn get_media_file(&self, key: &[u8]) -> PathBuf { self.get_media_file_sha256(key) } /// new SHA256 file name media function. requires database migrated. uses /// SHA256 hash of the base64 key as the file name + #[must_use] pub fn get_media_file_sha256(&self, key: &[u8]) -> PathBuf { let mut r = self.get_media_dir(); // Using the hash of the base64 key as the filename @@ -292,6 +293,7 @@ impl Service { /// old base64 file name media function /// This is the old version of `get_media_file` that uses the full base64 /// key as the filename. + #[must_use] pub fn get_media_file_b64(&self, key: &[u8]) -> PathBuf { let mut r = self.get_media_dir(); let encoded = encode_key(key); @@ -299,6 +301,7 @@ impl Service { r } + #[must_use] pub fn get_media_dir(&self) -> PathBuf { let mut r = PathBuf::new(); r.push(self.server.config.database_path.clone()); From 43432189578bc6327566748c24866c838358c6cc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 17 Jul 2024 07:39:14 +0000 Subject: [PATCH 0558/2091] initialize some containers with_capacity Signed-off-by: Jason Volk --- src/service/sending/send.rs | 2 +- src/service/sending/sender.rs | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index df3139c3..7901de48 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -154,7 +154,7 @@ fn sign_request(dest: &ServerName, http_request: &mut http::Request>) where T: OutgoingRequest + Debug + Send, { - let mut req_map = serde_json::Map::new(); + let mut req_map = serde_json::Map::with_capacity(8); if !http_request.body().is_empty() { req_map.insert( "content".to_owned(), diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index df41db28..774c3d69 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -555,8 +555,18 @@ async fn send_events_dest_push( async fn send_events_dest_normal( dest: &Destination, server: &OwnedServerName, events: Vec, ) -> SendingResult { - let mut edu_jsons = Vec::new(); - let mut pdu_jsons = Vec::new(); + let mut pdu_jsons = Vec::with_capacity( + events + .iter() + .filter(|event| matches!(event, SendingEvent::Pdu(_))) + .count(), + ); + let mut edu_jsons = Vec::with_capacity( + events + .iter() + .filter(|event| matches!(event, SendingEvent::Edu(_))) + .count(), + ); for event in &events { match event { From a88f913a178ab278c81b65b8d918c5cbdac92149 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 17 Jul 2024 22:31:21 +0000 Subject: [PATCH 0559/2091] enrich interface for inter-service referencing Signed-off-by: Jason Volk --- src/service/client/mod.rs | 4 +--- src/service/service.rs | 23 ++++++++++++++++++++--- src/service/services.rs | 14 +++++++++++++- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index cc8d52d1..03b0a142 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -18,9 +18,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; - let resolver = args - .get_service::("resolver") - .expect("resolver must be built prior to client"); + let resolver = args.require_service::("resolver"); Ok(Arc::new(Self { default: base(config) diff --git a/src/service/service.rs b/src/service/service.rs index 99b8723a..863b955b 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -1,7 +1,7 @@ use std::{any::Any, collections::BTreeMap, fmt::Write, sync::Arc}; use async_trait::async_trait; -use conduit::{utils::string::split_once_infallible, Result, Server}; +use conduit::{err, error::inspect_log, utils::string::split_once_infallible, Err, Result, Server}; use database::Database; #[async_trait] @@ -44,9 +44,26 @@ pub(crate) type Map = BTreeMap; pub(crate) type MapVal = (Arc, Arc); impl Args<'_> { - pub(crate) fn get_service(&self, name: &str) -> Option> { - get::(self.service, name) + pub(crate) fn require_service(&self, name: &str) -> Arc { + self.try_get_service::(name) + .inspect_err(inspect_log) + .expect("Failure to reference service required by another service.") } + + pub(crate) fn try_get_service(&self, name: &str) -> Result> { + try_get::(self.service, name) + } +} + +pub(crate) fn try_get(map: &Map, name: &str) -> Result> { + map.get(name).map_or_else( + || Err!("Service {name:?} does not exist or has not been built yet."), + |(_, s)| { + s.clone() + .downcast::() + .map_err(|_| err!("Service {name:?} must be correctly downcast.")) + }, + ) } pub(crate) fn get(map: &Map, name: &str) -> Option> { diff --git a/src/service/services.rs b/src/service/services.rs index d0f74e13..136059cd 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -173,5 +173,17 @@ impl Services { } } - pub fn get(&self, name: &str) -> Option> { service::get::(&self.service, name) } + pub fn try_get(&self, name: &str) -> Result> + where + T: Any + Send + Sync, + { + service::try_get::(&self.service, name) + } + + pub fn get(&self, name: &str) -> Option> + where + T: Any + Send + Sync, + { + service::get::(&self.service, name) + } } From 3dc91525cec658cd7288658b756a15228450bbee Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 21 Jul 2024 01:08:03 +0000 Subject: [PATCH 0560/2091] fix over-tabulation Signed-off-by: Jason Volk --- src/service/rooms/spaces/mod.rs | 277 ++++++++++++++++---------------- 1 file changed, 140 insertions(+), 137 deletions(-) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 02db7fab..19a3ebbb 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -269,7 +269,7 @@ impl Service { ) -> Result> { for server in via { debug_info!("Asking {server} for /hierarchy"); - if let Ok(response) = services() + let Ok(response) = services() .sending .send_federation_request( server, @@ -279,68 +279,70 @@ impl Service { }, ) .await - { - debug_info!("Got response from {server} for /hierarchy\n{response:?}"); - let summary = response.room.clone(); + else { + continue; + }; - self.roomid_spacehierarchy_cache.lock().await.insert( - current_room.clone(), - Some(CachedSpaceHierarchySummary { - summary: summary.clone(), - }), - ); + debug_info!("Got response from {server} for /hierarchy\n{response:?}"); + let summary = response.room.clone(); - for child in response.children { - let mut guard = self.roomid_spacehierarchy_cache.lock().await; - if !guard.contains_key(current_room) { - guard.insert( - current_room.clone(), - Some(CachedSpaceHierarchySummary { - summary: { - let SpaceHierarchyChildSummary { - canonical_alias, - name, - num_joined_members, - room_id, - topic, - world_readable, - guest_can_join, - avatar_url, - join_rule, - room_type, - allowed_room_ids, - } = child; + self.roomid_spacehierarchy_cache.lock().await.insert( + current_room.clone(), + Some(CachedSpaceHierarchySummary { + summary: summary.clone(), + }), + ); - SpaceHierarchyParentSummary { - canonical_alias, - name, - num_joined_members, - room_id: room_id.clone(), - topic, - world_readable, - guest_can_join, - avatar_url, - join_rule, - room_type, - children_state: get_stripped_space_child_events(&room_id).await?.unwrap(), - allowed_room_ids, - } - }, - }), - ); - } + for child in response.children { + let mut guard = self.roomid_spacehierarchy_cache.lock().await; + if !guard.contains_key(current_room) { + guard.insert( + current_room.clone(), + Some(CachedSpaceHierarchySummary { + summary: { + let SpaceHierarchyChildSummary { + canonical_alias, + name, + num_joined_members, + room_id, + topic, + world_readable, + guest_can_join, + avatar_url, + join_rule, + room_type, + allowed_room_ids, + } = child; + + SpaceHierarchyParentSummary { + canonical_alias, + name, + num_joined_members, + room_id: room_id.clone(), + topic, + world_readable, + guest_can_join, + avatar_url, + join_rule, + room_type, + children_state: get_stripped_space_child_events(&room_id).await?.unwrap(), + allowed_room_ids, + } + }, + }), + ); } - if is_accessable_child( - current_room, - &response.room.join_rule, - &Identifier::UserId(user_id), - &response.room.allowed_room_ids, - ) { - return Ok(Some(SummaryAccessibility::Accessible(Box::new(summary.clone())))); - } - - return Ok(Some(SummaryAccessibility::Inaccessible)); } + if is_accessable_child( + current_room, + &response.room.join_rule, + &Identifier::UserId(user_id), + &response.room.allowed_room_ids, + ) { + return Ok(Some(SummaryAccessibility::Accessible(Box::new(summary.clone())))); + } + + return Ok(Some(SummaryAccessibility::Inaccessible)); } self.roomid_spacehierarchy_cache @@ -461,24 +463,27 @@ impl Service { let mut results = Vec::new(); while let Some((current_room, via)) = { next_room_to_traverse(&mut stack, &mut parents) } { - if limit > results.len() { - match ( - self.get_summary_and_children_client(¤t_room, suggested_only, sender_user, &via) - .await?, - current_room == room_id, - ) { - (Some(SummaryAccessibility::Accessible(summary)), _) => { - let mut children: Vec<(OwnedRoomId, Vec)> = - get_parent_children_via(&summary, suggested_only) - .into_iter() - .filter(|(room, _)| parents.iter().all(|parent| parent != room)) - .rev() - .collect(); + if results.len() >= limit { + break; + } - if populate_results { - results.push(summary_to_chunk(*summary.clone())); - } else { - children = children + match ( + self.get_summary_and_children_client(¤t_room, suggested_only, sender_user, &via) + .await?, + current_room == room_id, + ) { + (Some(SummaryAccessibility::Accessible(summary)), _) => { + let mut children: Vec<(OwnedRoomId, Vec)> = + get_parent_children_via(&summary, suggested_only) + .into_iter() + .filter(|(room, _)| parents.iter().all(|parent| parent != room)) + .rev() + .collect(); + + if populate_results { + results.push(summary_to_chunk(*summary.clone())); + } else { + children = children .into_iter() .rev() .skip_while(|(room, _)| { @@ -495,39 +500,36 @@ impl Service { .rev() .collect(); - if children.is_empty() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Room IDs in token were not found.", - )); - } - - // We have reached the room after where we last left off - let parents_len = parents.len(); - if checked!(parents_len + 1)? == short_room_ids.len() { - populate_results = true; - } + if children.is_empty() { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Room IDs in token were not found.", + )); } - let parents_len: u64 = parents.len().try_into()?; - if !children.is_empty() && parents_len < max_depth { - parents.push_back(current_room.clone()); - stack.push(children); + // We have reached the room after where we last left off + let parents_len = parents.len(); + if checked!(parents_len + 1)? == short_room_ids.len() { + populate_results = true; } - // Root room in the space hierarchy, we return an error - // if this one fails. - }, - (Some(SummaryAccessibility::Inaccessible), true) => { - return Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room is inaccessible")); - }, - (None, true) => { - return Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room was not found")); - }, - // Just ignore other unavailable rooms - (None | Some(SummaryAccessibility::Inaccessible), false) => (), - } - } else { - break; + } + + let parents_len: u64 = parents.len().try_into()?; + if !children.is_empty() && parents_len < max_depth { + parents.push_back(current_room.clone()); + stack.push(children); + } + // Root room in the space hierarchy, we return an error + // if this one fails. + }, + (Some(SummaryAccessibility::Inaccessible), true) => { + return Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room is inaccessible")); + }, + (None, true) => { + return Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room was not found")); + }, + // Just ignore other unavailable rooms + (None | Some(SummaryAccessibility::Inaccessible), false) => (), } } @@ -574,41 +576,42 @@ fn next_room_to_traverse( async fn get_stripped_space_child_events( room_id: &RoomId, ) -> Result>>, Error> { - if let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? { - let state = services() - .rooms - .state_accessor - .state_full_ids(current_shortstatehash) - .await?; - let mut children_pdus = Vec::new(); - for (key, id) in state { - let (event_type, state_key) = services().rooms.short.get_statekey_from_short(key)?; - if event_type != StateEventType::SpaceChild { - continue; - } + let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? else { + return Ok(None); + }; - let pdu = services() - .rooms - .timeline - .get_pdu(&id)? - .ok_or_else(|| Error::bad_database("Event in space state not found"))?; - - if serde_json::from_str::(pdu.content.get()) - .ok() - .map(|c| c.via) - .map_or(true, |v| v.is_empty()) - { - continue; - } - - if OwnedRoomId::try_from(state_key).is_ok() { - children_pdus.push(pdu.to_stripped_spacechild_state_event()); - } + let state = services() + .rooms + .state_accessor + .state_full_ids(current_shortstatehash) + .await?; + let mut children_pdus = Vec::new(); + for (key, id) in state { + let (event_type, state_key) = services().rooms.short.get_statekey_from_short(key)?; + if event_type != StateEventType::SpaceChild { + continue; + } + + let pdu = services() + .rooms + .timeline + .get_pdu(&id)? + .ok_or_else(|| Error::bad_database("Event in space state not found"))?; + + if serde_json::from_str::(pdu.content.get()) + .ok() + .map(|c| c.via) + .map_or(true, |v| v.is_empty()) + { + continue; + } + + if OwnedRoomId::try_from(state_key).is_ok() { + children_pdus.push(pdu.to_stripped_spacechild_state_event()); } - Ok(Some(children_pdus)) - } else { - Ok(None) } + + Ok(Some(children_pdus)) } /// With the given identifier, checks if a room is accessable From 9b20c6918f86c14b541356d2ae40a2c4a8558cf6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 21 Jul 2024 00:21:34 +0000 Subject: [PATCH 0561/2091] add indirection for circular-dependencies between services Signed-off-by: Jason Volk --- src/service/mod.rs | 2 +- src/service/service.rs | 51 +++++++++++++++++++++++++++++++++-------- src/service/services.rs | 36 +++++++++++++++++++---------- 3 files changed, 66 insertions(+), 23 deletions(-) diff --git a/src/service/mod.rs b/src/service/mod.rs index 46adb072..21d1f594 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -29,7 +29,7 @@ use std::sync::{Arc, RwLock}; pub(crate) use conduit::{config, debug_error, debug_warn, utils, Error, Result, Server}; pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; use database::Database; -pub(crate) use service::{Args, Service}; +pub(crate) use service::{Args, Dep, Service}; pub use crate::{ globals::{server_is_ours, user_is_local}, diff --git a/src/service/service.rs b/src/service/service.rs index 863b955b..ce4f15b2 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -1,9 +1,16 @@ -use std::{any::Any, collections::BTreeMap, fmt::Write, sync::Arc}; +use std::{ + any::Any, + collections::BTreeMap, + fmt::Write, + ops::Deref, + sync::{Arc, OnceLock}, +}; use async_trait::async_trait; use conduit::{err, error::inspect_log, utils::string::split_once_infallible, Err, Result, Server}; use database::Database; +/// Abstract interface for a Service #[async_trait] pub(crate) trait Service: Any + Send + Sync { /// Implement the construction of the service instance. Services are @@ -34,27 +41,51 @@ pub(crate) trait Service: Any + Send + Sync { fn name(&self) -> &str; } +/// Args are passed to `Service::build` when a service is constructed. This +/// allows for arguments to change with limited impact to the many services. pub(crate) struct Args<'a> { pub(crate) server: &'a Arc, pub(crate) db: &'a Arc, - pub(crate) service: &'a Map, + pub(crate) service: &'a Arc, +} + +/// Dep is a reference to a service used within another service. +/// Circular-dependencies between services require this indirection to allow the +/// referenced service construction after the referencing service. +pub(crate) struct Dep { + dep: OnceLock>, + service: Arc, + name: &'static str, } pub(crate) type Map = BTreeMap; pub(crate) type MapVal = (Arc, Arc); -impl Args<'_> { - pub(crate) fn require_service(&self, name: &str) -> Arc { - self.try_get_service::(name) - .inspect_err(inspect_log) - .expect("Failure to reference service required by another service.") - } +impl Deref for Dep { + type Target = Arc; - pub(crate) fn try_get_service(&self, name: &str) -> Result> { - try_get::(self.service, name) + fn deref(&self) -> &Self::Target { + self.dep + .get_or_init(|| require::(&self.service, self.name)) } } +impl Args<'_> { + pub(crate) fn depend_service(&self, name: &'static str) -> Dep { + Dep:: { + dep: OnceLock::new(), + service: self.service.clone(), + name, + } + } +} + +pub(crate) fn require(map: &Map, name: &str) -> Arc { + try_get::(map, name) + .inspect_err(inspect_log) + .expect("Failure to reference service required by another service.") +} + pub(crate) fn try_get(map: &Map, name: &str) -> Result> { map.get(name).map_or_else( || Err!("Service {name:?} does not exist or has not been built yet."), diff --git a/src/service/services.rs b/src/service/services.rs index 136059cd..68205323 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -31,24 +31,36 @@ pub struct Services { pub updates: Arc, manager: Mutex>>, - pub(crate) service: Map, + pub(crate) service: Arc, pub server: Arc, pub db: Arc, } +macro_rules! build_service { + ($map:ident, $server:ident, $db:ident, $tyname:ty) => {{ + let built = <$tyname>::build(Args { + server: &$server, + db: &$db, + service: &$map, + })?; + + Arc::get_mut(&mut $map) + .expect("must have mutable reference to services collection") + .insert(built.name().to_owned(), (built.clone(), built.clone())); + + trace!("built service #{}: {:?}", $map.len(), built.name()); + built + }}; +} + impl Services { + #[allow(clippy::cognitive_complexity)] pub fn build(server: Arc, db: Arc) -> Result { - let mut service: Map = BTreeMap::new(); + let mut service: Arc = Arc::new(BTreeMap::new()); macro_rules! build { - ($tyname:ty) => {{ - let built = <$tyname>::build(Args { - server: &server, - db: &db, - service: &service, - })?; - service.insert(built.name().to_owned(), (built.clone(), built.clone())); - built - }}; + ($srv:ty) => { + build_service!(service, server, db, $srv) + }; } Ok(Self { @@ -167,7 +179,7 @@ impl Services { fn interrupt(&self) { debug!("Interrupting services..."); - for (name, (service, ..)) in &self.service { + for (name, (service, ..)) in self.service.iter() { trace!("Interrupting {name}"); service.interrupt(); } From 992c0a1e583e94589bad46ee56bf464b046931d8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 20 Jul 2024 23:38:20 +0000 Subject: [PATCH 0562/2091] de-global services for admin Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 2 +- src/admin/handler.rs | 2 +- src/admin/mod.rs | 5 +- src/admin/room/room_moderation_commands.rs | 6 +- src/admin/user/commands.rs | 5 +- src/api/client/account.rs | 7 +- src/api/client/state.rs | 4 +- src/service/admin/console.rs | 20 +- src/service/admin/create.rs | 47 ++-- src/service/admin/grant.rs | 43 ++- src/service/admin/mod.rs | 298 ++++++++++----------- src/service/globals/migrations.rs | 2 +- src/service/rooms/timeline/mod.rs | 5 +- src/service/users/mod.rs | 2 +- 14 files changed, 221 insertions(+), 227 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index f319f5a5..f0aa23cb 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -8,7 +8,7 @@ use api::client::validate_and_add_event_id; use conduit::{ debug, info, log, log::{capture, Capture}, - warn, Error, Result, + warn, Error, PduEvent, Result, }; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 95c7ed41..409abc18 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -14,7 +14,7 @@ use ruma::{ extern crate conduit_service as service; use conduit::{utils::string::common_prefix, Result}; -pub(crate) use service::admin::{Command, Service}; +pub(crate) use service::admin::Command; use service::admin::{CommandOutput, CommandResult, HandlerResult}; use crate::{ diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 14856811..e020ed43 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -20,10 +20,7 @@ extern crate conduit_service as service; pub(crate) use conduit::{mod_ctor, mod_dtor, Result}; pub(crate) use service::{services, user_is_local}; -pub(crate) use crate::{ - handler::Service, - utils::{escape_html, get_room_info}, -}; +pub(crate) use crate::utils::{escape_html, get_room_info}; mod_ctor! {} mod_dtor! {} diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index c9be4473..46354c0f 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -2,7 +2,7 @@ use api::client::leave_room; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId}; use tracing::{debug, error, info, warn}; -use super::{super::Service, RoomModerationCommand}; +use super::RoomModerationCommand; use crate::{get_room_info, services, user_is_local, Result}; pub(super) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> Result { @@ -31,7 +31,7 @@ async fn ban_room( let admin_room_alias = &services().globals.admin_alias; - if let Some(admin_room_id) = Service::get_admin_room()? { + if let Some(admin_room_id) = services().admin.get_admin_room()? { if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) { return Ok(RoomMessageEventContent::text_plain("Not allowed to ban the admin room.")); } @@ -198,7 +198,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo for &room in &rooms_s { match <&RoomOrAliasId>::try_from(room) { Ok(room_alias_or_id) => { - if let Some(admin_room_id) = Service::get_admin_room()? { + if let Some(admin_room_id) = services().admin.get_admin_room()? { if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias) { info!("User specified admin room in bulk ban list, ignoring"); continue; diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index f6387ee8..fa1c5288 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -363,7 +363,10 @@ pub(super) async fn make_user_admin(_body: Vec<&str>, user_id: String) -> Result .unwrap_or_else(|| user_id.to_string()); assert!(service::user_is_local(&user_id), "Parsed user_id must be a local user"); - service::admin::make_user_admin(&user_id, displayname).await?; + services() + .admin + .make_user_admin(&user_id, displayname) + .await?; Ok(RoomMessageEventContent::notice_markdown(format!( "{user_id} has been granted admin privileges.", diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 19ac89a0..b3495b42 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -347,9 +347,12 @@ pub(crate) async fn register_route( // If this is the first real user, grant them admin privileges except for guest // users Note: the server user, @conduit:servername, is generated first if !is_guest { - if let Some(admin_room) = service::admin::Service::get_admin_room()? { + if let Some(admin_room) = services.admin.get_admin_room()? { if services.rooms.state_cache.room_joined_count(&admin_room)? == Some(1) { - service::admin::make_user_admin(&user_id, displayname).await?; + services + .admin + .make_user_admin(&user_id, displayname) + .await?; warn!("Granting {user_id} admin privileges as the first user"); } diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 51217d00..56ffd2ac 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -210,7 +210,7 @@ async fn allowed_to_send_state_event( }, // admin room is a sensitive room, it should not ever be made public StateEventType::RoomJoinRules => { - if let Some(admin_room_id) = service::admin::Service::get_admin_room()? { + if let Some(admin_room_id) = services.admin.get_admin_room()? { if admin_room_id == room_id { if let Ok(join_rule) = serde_json::from_str::(json.json().get()) { if join_rule.join_rule == JoinRule::Public { @@ -225,7 +225,7 @@ async fn allowed_to_send_state_event( }, // admin room is a sensitive room, it should not ever be made world readable StateEventType::RoomHistoryVisibility => { - if let Some(admin_room_id) = service::admin::Service::get_admin_room()? { + if let Some(admin_room_id) = services.admin.get_admin_room()? { if admin_room_id == room_id { if let Ok(visibility_content) = serde_json::from_str::(json.json().get()) diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index c590b928..9c335791 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -4,7 +4,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{debug, defer, error, log}; +use conduit::{debug, defer, error, log, Server}; use futures_util::future::{AbortHandle, Abortable}; use ruma::events::room::message::RoomMessageEventContent; use rustyline_async::{Readline, ReadlineError, ReadlineEvent}; @@ -14,6 +14,7 @@ use tokio::task::JoinHandle; use crate::services; pub struct Console { + server: Arc, worker_join: Mutex>>, input_abort: Mutex>, command_abort: Mutex>, @@ -25,9 +26,9 @@ const PROMPT: &str = "uwu> "; const HISTORY_LIMIT: usize = 48; impl Console { - #[must_use] - pub fn new() -> Arc { + pub(super) fn new(args: &crate::Args<'_>) -> Arc { Arc::new(Self { + server: args.server.clone(), worker_join: None.into(), input_abort: None.into(), command_abort: None.into(), @@ -37,7 +38,7 @@ impl Console { } pub(super) async fn handle_signal(self: &Arc, sig: &'static str) { - if !services().server.running() { + if !self.server.running() { self.interrupt(); } else if sig == "SIGINT" { self.interrupt_command(); @@ -49,7 +50,7 @@ impl Console { let mut worker_join = self.worker_join.lock().expect("locked"); if worker_join.is_none() { let self_ = Arc::clone(self); - _ = worker_join.insert(services().server.runtime().spawn(self_.worker())); + _ = worker_join.insert(self.server.runtime().spawn(self_.worker())); } } @@ -89,16 +90,13 @@ impl Console { #[tracing::instrument(skip_all, name = "console")] async fn worker(self: Arc) { debug!("session starting"); - while services().server.running() { + while self.server.running() { match self.readline().await { Ok(event) => match event { ReadlineEvent::Line(string) => self.clone().handle(string).await, ReadlineEvent::Interrupted => continue, ReadlineEvent::Eof => break, - ReadlineEvent::Quit => services() - .server - .shutdown() - .unwrap_or_else(error::default_log), + ReadlineEvent::Quit => self.server.shutdown().unwrap_or_else(error::default_log), }, Err(error) => match error { ReadlineError::Closed => break, @@ -115,7 +113,7 @@ impl Console { } async fn readline(self: &Arc) -> Result { - let _suppression = log::Suppress::new(&services().server); + let _suppression = log::Suppress::new(&self.server); let (mut readline, _writer) = Readline::new(PROMPT.to_owned())?; readline.set_tab_completer(Self::tab_complete); diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index fbb6a078..18cbe039 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use conduit::{Error, Result}; +use conduit::{pdu::PduBuilder, warn, Error, Result}; use ruma::{ api::client::error::ErrorKind, events::{ @@ -21,26 +21,25 @@ use ruma::{ RoomId, RoomVersionId, }; use serde_json::value::to_raw_value; -use tracing::warn; -use crate::{pdu::PduBuilder, services}; +use crate::Services; /// Create the admin room. /// /// Users in this room are considered admins by conduit, and the room can be /// used to issue admin commands by talking to the server user inside it. -pub async fn create_admin_room() -> Result<()> { - let room_id = RoomId::new(services().globals.server_name()); +pub async fn create_admin_room(services: &Services) -> Result<()> { + let room_id = RoomId::new(services.globals.server_name()); - let _short_id = services().rooms.short.get_or_create_shortroomid(&room_id)?; + let _short_id = services.rooms.short.get_or_create_shortroomid(&room_id)?; - let state_lock = services().rooms.state.mutex.lock(&room_id).await; + let state_lock = services.rooms.state.mutex.lock(&room_id).await; // Create a user for the server - let server_user = &services().globals.server_user; - services().users.create(server_user, None)?; + let server_user = &services.globals.server_user; + services.users.create(server_user, None)?; - let room_version = services().globals.default_room_version(); + let room_version = services.globals.default_room_version(); let mut content = { use RoomVersionId::*; @@ -62,7 +61,7 @@ pub async fn create_admin_room() -> Result<()> { content.room_version = room_version; // 1. The room create event - services() + services .rooms .timeline .build_and_append_pdu( @@ -80,7 +79,7 @@ pub async fn create_admin_room() -> Result<()> { .await?; // 2. Make conduit bot join - services() + services .rooms .timeline .build_and_append_pdu( @@ -111,7 +110,7 @@ pub async fn create_admin_room() -> Result<()> { let mut users = BTreeMap::new(); users.insert(server_user.clone(), 100.into()); - services() + services .rooms .timeline .build_and_append_pdu( @@ -133,7 +132,7 @@ pub async fn create_admin_room() -> Result<()> { .await?; // 4.1 Join Rules - services() + services .rooms .timeline .build_and_append_pdu( @@ -152,7 +151,7 @@ pub async fn create_admin_room() -> Result<()> { .await?; // 4.2 History Visibility - services() + services .rooms .timeline .build_and_append_pdu( @@ -171,7 +170,7 @@ pub async fn create_admin_room() -> Result<()> { .await?; // 4.3 Guest Access - services() + services .rooms .timeline .build_and_append_pdu( @@ -190,8 +189,8 @@ pub async fn create_admin_room() -> Result<()> { .await?; // 5. Events implied by name and topic - let room_name = format!("{} Admin Room", services().globals.server_name()); - services() + let room_name = format!("{} Admin Room", services.globals.server_name()); + services .rooms .timeline .build_and_append_pdu( @@ -209,14 +208,14 @@ pub async fn create_admin_room() -> Result<()> { ) .await?; - services() + services .rooms .timeline .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomTopic, content: to_raw_value(&RoomTopicEventContent { - topic: format!("Manage {}", services().globals.server_name()), + topic: format!("Manage {}", services.globals.server_name()), }) .expect("event is valid, we just created it"), unsigned: None, @@ -230,9 +229,9 @@ pub async fn create_admin_room() -> Result<()> { .await?; // 6. Room alias - let alias = &services().globals.admin_alias; + let alias = &services.globals.admin_alias; - services() + services .rooms .timeline .build_and_append_pdu( @@ -253,13 +252,13 @@ pub async fn create_admin_room() -> Result<()> { ) .await?; - services() + services .rooms .alias .set_alias(alias, &room_id, server_user)?; // 7. (ad-hoc) Disable room previews for everyone by default - services() + services .rooms .timeline .build_and_append_pdu( diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 9a4ef242..213225ac 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -14,23 +14,24 @@ use ruma::{ }; use serde_json::value::to_raw_value; -use super::Service; -use crate::{pdu::PduBuilder, services}; +use crate::pdu::PduBuilder; -/// Invite the user to the conduit admin room. -/// -/// In conduit, this is equivalent to granting admin privileges. -pub async fn make_user_admin(user_id: &UserId, displayname: String) -> Result<()> { - if let Some(room_id) = Service::get_admin_room()? { - let state_lock = services().rooms.state.mutex.lock(&room_id).await; +impl super::Service { + /// Invite the user to the conduit admin room. + /// + /// In conduit, this is equivalent to granting admin privileges. + pub async fn make_user_admin(&self, user_id: &UserId, displayname: String) -> Result<()> { + let Some(room_id) = self.get_admin_room()? else { + return Ok(()); + }; + + let state_lock = self.state.mutex.lock(&room_id).await; // Use the server user to grant the new admin's power level - let server_user = &services().globals.server_user; + let server_user = &self.globals.server_user; // Invite and join the real user - services() - .rooms - .timeline + self.timeline .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomMember, @@ -54,9 +55,7 @@ pub async fn make_user_admin(user_id: &UserId, displayname: String) -> Result<() &state_lock, ) .await?; - services() - .rooms - .timeline + self.timeline .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomMember, @@ -86,9 +85,7 @@ pub async fn make_user_admin(user_id: &UserId, displayname: String) -> Result<() users.insert(server_user.clone(), 100.into()); users.insert(user_id.to_owned(), 100.into()); - services() - .rooms - .timeline + self.timeline .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomPowerLevels, @@ -108,12 +105,12 @@ pub async fn make_user_admin(user_id: &UserId, displayname: String) -> Result<() .await?; // Send welcome message - services().rooms.timeline.build_and_append_pdu( + self.timeline.build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomMessage, content: to_raw_value(&RoomMessageEventContent::text_html( - format!("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `@conduit:{}: --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`", services().globals.server_name()), - format!("

    Thank you for trying out conduwuit!

    \n

    conduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

    \n

    Helpful links:

    \n
    \n

    Git and Documentation: https://github.com/girlbossceo/conduwuit
    Report issues: https://github.com/girlbossceo/conduwuit/issues

    \n
    \n

    For a list of available commands, send the following message in this room: @conduit:{}: --help

    \n

    Here are some rooms you can join (by typing the command):

    \n

    conduwuit room (Ask questions and get notified on updates):
    /join #conduwuit:puppygock.gay

    \n", services().globals.server_name()), + format!("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `@conduit:{}: --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`", self.globals.server_name()), + format!("

    Thank you for trying out conduwuit!

    \n

    conduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

    \n

    Helpful links:

    \n
    \n

    Git and Documentation: https://github.com/girlbossceo/conduwuit
    Report issues: https://github.com/girlbossceo/conduwuit/issues

    \n
    \n

    For a list of available commands, send the following message in this room: @conduit:{}: --help

    \n

    Here are some rooms you can join (by typing the command):

    \n

    conduwuit room (Ask questions and get notified on updates):
    /join #conduwuit:puppygock.gay

    \n", self.globals.server_name()), )) .expect("event is valid, we just created it"), unsigned: None, @@ -124,7 +121,7 @@ pub async fn make_user_admin(user_id: &UserId, displayname: String) -> Result<() &room_id, &state_lock, ).await?; - } - Ok(()) + Ok(()) + } } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index f63ebf09..fcb34212 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,9 +9,8 @@ use std::{ }; use async_trait::async_trait; -use conduit::{debug, error, error::default_log, Error, Result}; +use conduit::{debug, error, error::default_log, pdu::PduBuilder, Error, PduEvent, Result, Server}; pub use create::create_admin_room; -pub use grant::make_user_admin; use loole::{Receiver, Sender}; use ruma::{ events::{ @@ -23,9 +22,15 @@ use ruma::{ use serde_json::value::to_raw_value; use tokio::sync::{Mutex, RwLock}; -use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, user_is_local, PduEvent}; +use crate::{globals, rooms, rooms::state::RoomMutexGuard, user_is_local}; pub struct Service { + server: Arc, + globals: Arc, + alias: Arc, + timeline: Arc, + state: Arc, + state_cache: Arc, sender: Sender, receiver: Mutex>, pub handle: RwLock>, @@ -50,21 +55,27 @@ const COMMAND_QUEUE_LIMIT: usize = 512; #[async_trait] impl crate::Service for Service { - fn build(_args: crate::Args<'_>) -> Result> { + fn build(args: crate::Args<'_>) -> Result> { let (sender, receiver) = loole::bounded(COMMAND_QUEUE_LIMIT); Ok(Arc::new(Self { + server: args.server.clone(), + globals: args.require_service::("globals"), + alias: args.require_service::("rooms::alias"), + timeline: args.require_service::("rooms::timeline"), + state: args.require_service::("rooms::state"), + state_cache: args.require_service::("rooms::state_cache"), sender, receiver: Mutex::new(receiver), handle: RwLock::new(None), complete: StdRwLock::new(None), #[cfg(feature = "console")] - console: console::Console::new(), + console: console::Console::new(&args), })) } async fn worker(self: Arc) -> Result<()> { let receiver = self.receiver.lock().await; - let mut signals = services().server.signal.subscribe(); + let mut signals = self.server.signal.subscribe(); loop { tokio::select! { command = receiver.recv_async() => match command { @@ -104,9 +115,10 @@ impl Service { } pub async fn send_message(&self, message_content: RoomMessageEventContent) { - if let Ok(Some(room_id)) = Self::get_admin_room() { - let user_id = &services().globals.server_user; - respond_to_room(message_content, &room_id, user_id).await; + if let Ok(Some(room_id)) = self.get_admin_room() { + let user_id = &self.globals.server_user; + self.respond_to_room(message_content, &room_id, user_id) + .await; } } @@ -147,7 +159,7 @@ impl Service { async fn handle_command(&self, command: Command) { match self.process_command(command).await { - Ok(Some(output)) => handle_response(output).await, + Ok(Some(output)) => self.handle_response(output).await, Ok(None) => debug!("Command successful with no response"), Err(e) => error!("Command processing error: {e}"), } @@ -163,8 +175,8 @@ impl Service { /// Checks whether a given user is an admin of this server pub async fn user_is_admin(&self, user_id: &UserId) -> Result { - if let Ok(Some(admin_room)) = Self::get_admin_room() { - services().rooms.state_cache.is_joined(user_id, &admin_room) + if let Ok(Some(admin_room)) = self.get_admin_room() { + self.state_cache.is_joined(user_id, &admin_room) } else { Ok(false) } @@ -174,16 +186,11 @@ impl Service { /// /// Errors are propagated from the database, and will have None if there is /// no admin room - pub fn get_admin_room() -> Result> { - if let Some(room_id) = services() - .rooms - .alias - .resolve_local_alias(&services().globals.admin_alias)? - { - if services() - .rooms + pub fn get_admin_room(&self) -> Result> { + if let Some(room_id) = self.alias.resolve_local_alias(&self.globals.admin_alias)? { + if self .state_cache - .is_joined(&services().globals.server_user, &room_id)? + .is_joined(&self.globals.server_user, &room_id)? { return Ok(Some(room_id)); } @@ -191,142 +198,133 @@ impl Service { Ok(None) } -} -async fn handle_response(content: RoomMessageEventContent) { - let Some(Relation::Reply { - in_reply_to, - }) = content.relates_to.as_ref() - else { - return; - }; + async fn handle_response(&self, content: RoomMessageEventContent) { + let Some(Relation::Reply { + in_reply_to, + }) = content.relates_to.as_ref() + else { + return; + }; - let Ok(Some(pdu)) = services().rooms.timeline.get_pdu(&in_reply_to.event_id) else { - return; - }; + let Ok(Some(pdu)) = self.timeline.get_pdu(&in_reply_to.event_id) else { + return; + }; - let response_sender = if is_admin_room(&pdu.room_id) { - &services().globals.server_user - } else { - &pdu.sender - }; + let response_sender = if self.is_admin_room(&pdu.room_id) { + &self.globals.server_user + } else { + &pdu.sender + }; - respond_to_room(content, &pdu.room_id, response_sender).await; -} + self.respond_to_room(content, &pdu.room_id, response_sender) + .await; + } -async fn respond_to_room(content: RoomMessageEventContent, room_id: &RoomId, user_id: &UserId) { - assert!( - services() - .admin - .user_is_admin(user_id) + async fn respond_to_room(&self, content: RoomMessageEventContent, room_id: &RoomId, user_id: &UserId) { + assert!( + self.user_is_admin(user_id) + .await + .expect("checked user is admin"), + "sender is not admin" + ); + + let state_lock = self.state.mutex.lock(room_id).await; + let response_pdu = PduBuilder { + event_type: TimelineEventType::RoomMessage, + content: to_raw_value(&content).expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }; + + if let Err(e) = self + .timeline + .build_and_append_pdu(response_pdu, user_id, room_id, &state_lock) .await - .expect("checked user is admin"), - "sender is not admin" - ); + { + self.handle_response_error(e, room_id, user_id, &state_lock) + .await + .unwrap_or_else(default_log); + } + } - let state_lock = services().rooms.state.mutex.lock(room_id).await; - let response_pdu = PduBuilder { - event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&content).expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - }; + async fn handle_response_error( + &self, e: Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, + ) -> Result<()> { + error!("Failed to build and append admin room response PDU: \"{e}\""); + let error_room_message = RoomMessageEventContent::text_plain(format!( + "Failed to build and append admin room PDU: \"{e}\"\n\nThe original admin command may have finished \ + successfully, but we could not return the output." + )); - if let Err(e) = services() - .rooms - .timeline - .build_and_append_pdu(response_pdu, user_id, room_id, &state_lock) - .await - { - handle_response_error(e, room_id, user_id, &state_lock) - .await - .unwrap_or_else(default_log); - } -} - -async fn handle_response_error( - e: Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, -) -> Result<()> { - error!("Failed to build and append admin room response PDU: \"{e}\""); - let error_room_message = RoomMessageEventContent::text_plain(format!( - "Failed to build and append admin room PDU: \"{e}\"\n\nThe original admin command may have finished \ - successfully, but we could not return the output." - )); - - let response_pdu = PduBuilder { - event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&error_room_message).expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - }; - - services() - .rooms - .timeline - .build_and_append_pdu(response_pdu, user_id, room_id, state_lock) - .await?; - - Ok(()) -} - -pub async fn is_admin_command(pdu: &PduEvent, body: &str) -> bool { - // Server-side command-escape with public echo - let is_escape = body.starts_with('\\'); - let is_public_escape = is_escape && body.trim_start_matches('\\').starts_with("!admin"); - - // Admin command with public echo (in admin room) - let server_user = &services().globals.server_user; - let is_public_prefix = body.starts_with("!admin") || body.starts_with(server_user.as_str()); - - // Expected backward branch - if !is_public_escape && !is_public_prefix { - return false; - } - - // only allow public escaped commands by local admins - if is_public_escape && !user_is_local(&pdu.sender) { - return false; - } - - // Check if server-side command-escape is disabled by configuration - if is_public_escape && !services().globals.config.admin_escape_commands { - return false; - } - - // Prevent unescaped !admin from being used outside of the admin room - if is_public_prefix && !is_admin_room(&pdu.room_id) { - return false; - } - - // Only senders who are admin can proceed - if !services() - .admin - .user_is_admin(&pdu.sender) - .await - .unwrap_or(false) - { - return false; - } - - // This will evaluate to false if the emergency password is set up so that - // the administrator can execute commands as conduit - let emergency_password_set = services().globals.emergency_password().is_some(); - let from_server = pdu.sender == *server_user && !emergency_password_set; - if from_server && is_admin_room(&pdu.room_id) { - return false; - } - - // Authentic admin command - true -} - -#[must_use] -pub fn is_admin_room(room_id: &RoomId) -> bool { - if let Ok(Some(admin_room_id)) = Service::get_admin_room() { - admin_room_id == room_id - } else { - false + let response_pdu = PduBuilder { + event_type: TimelineEventType::RoomMessage, + content: to_raw_value(&error_room_message).expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: None, + }; + + self.timeline + .build_and_append_pdu(response_pdu, user_id, room_id, state_lock) + .await?; + + Ok(()) + } + + pub async fn is_admin_command(&self, pdu: &PduEvent, body: &str) -> bool { + // Server-side command-escape with public echo + let is_escape = body.starts_with('\\'); + let is_public_escape = is_escape && body.trim_start_matches('\\').starts_with("!admin"); + + // Admin command with public echo (in admin room) + let server_user = &self.globals.server_user; + let is_public_prefix = body.starts_with("!admin") || body.starts_with(server_user.as_str()); + + // Expected backward branch + if !is_public_escape && !is_public_prefix { + return false; + } + + // only allow public escaped commands by local admins + if is_public_escape && !user_is_local(&pdu.sender) { + return false; + } + + // Check if server-side command-escape is disabled by configuration + if is_public_escape && !self.globals.config.admin_escape_commands { + return false; + } + + // Prevent unescaped !admin from being used outside of the admin room + if is_public_prefix && !self.is_admin_room(&pdu.room_id) { + return false; + } + + // Only senders who are admin can proceed + if !self.user_is_admin(&pdu.sender).await.unwrap_or(false) { + return false; + } + + // This will evaluate to false if the emergency password is set up so that + // the administrator can execute commands as conduit + let emergency_password_set = self.globals.emergency_password().is_some(); + let from_server = pdu.sender == *server_user && !emergency_password_set; + if from_server && self.is_admin_room(&pdu.room_id) { + return false; + } + + // Authentic admin command + true + } + + #[must_use] + pub fn is_admin_room(&self, room_id: &RoomId) -> bool { + if let Ok(Some(admin_room_id)) = self.get_admin_room() { + admin_room_id == room_id + } else { + false + } } } diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 3948d1f5..d8c5f29b 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -59,7 +59,7 @@ async fn fresh(db: &Arc, config: &Config) -> Result<()> { db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; // Create the admin room and server user on first run - crate::admin::create_admin_room().await?; + crate::admin::create_admin_room(services).await?; warn!( "Created new {} database with version {DATABASE_VERSION}", diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 0bc5ade1..4c5e407a 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -38,7 +38,6 @@ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tokio::sync::RwLock; use crate::{ - admin, appservice::NamespaceRegex, pdu::{EventHash, PduBuilder}, rooms::{event_handler::parse_incoming_pdu, state_compressor::CompressedStateEvent}, @@ -485,7 +484,7 @@ impl Service { .search .index_pdu(shortroomid, &pdu_id, &body)?; - if admin::is_admin_command(pdu, &body).await { + if services().admin.is_admin_command(pdu, &body).await { services() .admin .command(body, Some((*pdu.event_id).into())) @@ -784,7 +783,7 @@ impl Service { state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result> { let (pdu, pdu_json) = self.create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock)?; - if let Some(admin_room) = admin::Service::get_admin_room()? { + if let Some(admin_room) = services().admin.get_admin_room()? { if admin_room == room_id { match pdu.event_type() { TimelineEventType::RoomEncryption => { diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 0196e1aa..e0a4dd1c 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -247,7 +247,7 @@ impl Service { /// Check if a user is an admin pub fn is_admin(&self, user_id: &UserId) -> Result { - if let Some(admin_room_id) = crate::admin::Service::get_admin_room()? { + if let Some(admin_room_id) = services().admin.get_admin_room()? { services() .rooms .state_cache From 010e4ee35a372018e56e9572a927cf06394c4291 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 18 Jul 2024 06:37:47 +0000 Subject: [PATCH 0563/2091] de-global services for services Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 9 +- src/admin/mod.rs | 1 + src/api/client/alias.rs | 12 +- src/api/client/membership.rs | 17 +- src/api/client/sync.rs | 6 +- src/api/mod.rs | 2 + src/api/server/backfill.rs | 3 +- src/api/server/event.rs | 3 +- src/api/server/event_auth.rs | 3 +- src/api/server/get_missing_events.rs | 3 +- src/api/server/invite.rs | 6 +- src/api/server/send.rs | 3 +- src/api/server/send_join.rs | 8 +- src/api/server/state.rs | 9 +- src/router/mod.rs | 2 + src/service/account_data/data.rs | 17 +- src/service/account_data/mod.rs | 2 +- src/service/admin/console.rs | 14 +- src/service/admin/grant.rs | 19 +- src/service/admin/mod.rs | 63 +- src/service/appservice/mod.rs | 14 +- src/service/client/mod.rs | 2 +- src/service/emergency/mod.rs | 83 ++ src/service/globals/data.rs | 41 +- src/service/globals/emerg_access.rs | 54 -- src/service/globals/migrations.rs | 274 ++++--- src/service/globals/mod.rs | 15 +- src/service/key_backups/data.rs | 23 +- src/service/key_backups/mod.rs | 2 +- src/service/manager.rs | 20 +- src/service/media/data.rs | 4 +- src/service/media/mod.rs | 22 +- src/service/mod.rs | 4 +- src/service/presence/data.rs | 24 +- src/service/presence/mod.rs | 38 +- src/service/pusher/mod.rs | 46 +- src/service/resolver/actual.rs | 111 ++- src/service/resolver/cache.rs | 3 +- src/service/resolver/mod.rs | 15 +- src/service/rooms/alias/data.rs | 17 +- src/service/rooms/alias/mod.rs | 86 +- src/service/rooms/alias/remote.rs | 114 +-- src/service/rooms/auth_chain/data.rs | 9 +- src/service/rooms/auth_chain/mod.rs | 31 +- src/service/rooms/directory/mod.rs | 4 +- src/service/rooms/event_handler/mod.rs | 229 +++--- .../rooms/event_handler/parse_incoming_pdu.rs | 41 +- .../rooms/event_handler/signing_keys.rs | 57 +- src/service/rooms/lazy_loading/mod.rs | 4 +- src/service/rooms/metadata/data.rs | 17 +- src/service/rooms/metadata/mod.rs | 5 +- src/service/rooms/mod.rs | 4 +- src/service/rooms/pdu_metadata/data.rs | 21 +- src/service/rooms/pdu_metadata/mod.rs | 30 +- src/service/rooms/read_receipt/data.rs | 19 +- src/service/rooms/read_receipt/mod.rs | 14 +- src/service/rooms/search/data.rs | 19 +- src/service/rooms/search/mod.rs | 2 +- src/service/rooms/short/data.rs | 25 +- src/service/rooms/short/mod.rs | 5 +- src/service/rooms/spaces/mod.rs | 381 ++++----- src/service/rooms/state/mod.rs | 110 ++- src/service/rooms/state_accessor/data.rs | 60 +- src/service/rooms/state_accessor/mod.rs | 32 +- src/service/rooms/state_cache/data.rs | 73 +- src/service/rooms/state_cache/mod.rs | 55 +- src/service/rooms/state_compressor/mod.rs | 25 +- src/service/rooms/threads/data.rs | 27 +- src/service/rooms/threads/mod.rs | 39 +- src/service/rooms/timeline/data.rs | 99 +-- src/service/rooms/timeline/mod.rs | 303 +++---- src/service/rooms/typing/mod.rs | 44 +- src/service/rooms/user/data.rs | 27 +- src/service/rooms/user/mod.rs | 5 +- src/service/sending/appservice.rs | 22 +- src/service/sending/data.rs | 17 +- src/service/sending/mod.rs | 65 +- src/service/sending/send.rs | 158 ++-- src/service/sending/sender.rs | 745 +++++++++--------- src/service/service.rs | 66 +- src/service/services.rs | 118 +-- src/service/uiaa/mod.rs | 23 +- src/service/updates/mod.rs | 17 +- src/service/users/data.rs | 83 +- src/service/users/mod.rs | 23 +- 85 files changed, 2480 insertions(+), 1887 deletions(-) create mode 100644 src/service/emergency/mod.rs delete mode 100644 src/service/globals/emerg_access.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index f0aa23cb..cbe52473 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -15,7 +15,7 @@ use ruma::{ events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, }; -use service::{rooms::event_handler::parse_incoming_pdu, services, PduEvent}; +use service::services; use tokio::sync::RwLock; use tracing_subscriber::EnvFilter; @@ -189,7 +189,10 @@ pub(super) async fn get_remote_pdu( debug!("Attempting to parse PDU: {:?}", &response.pdu); let parsed_pdu = { - let parsed_result = parse_incoming_pdu(&response.pdu); + let parsed_result = services() + .rooms + .event_handler + .parse_incoming_pdu(&response.pdu); let (event_id, value, room_id) = match parsed_result { Ok(t) => t, Err(e) => { @@ -510,7 +513,7 @@ pub(super) async fn force_set_room_state_from_server( let mut events = Vec::with_capacity(remote_state_response.pdus.len()); for pdu in remote_state_response.pdus.clone() { - events.push(match parse_incoming_pdu(&pdu) { + events.push(match services().rooms.event_handler.parse_incoming_pdu(&pdu) { Ok(t) => t, Err(e) => { warn!("Could not parse PDU, ignoring: {e}"); diff --git a/src/admin/mod.rs b/src/admin/mod.rs index e020ed43..c57659c1 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1,3 +1,4 @@ +#![recursion_limit = "168"] #![allow(clippy::wildcard_imports)] pub(crate) mod appservice; diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 88d1a4e6..11617a0e 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -22,7 +22,11 @@ pub(crate) async fn create_alias_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - service::rooms::alias::appservice_checks(&body.room_alias, &body.appservice_info).await?; + services + .rooms + .alias + .appservice_checks(&body.room_alias, &body.appservice_info) + .await?; // this isn't apart of alias_checks or delete alias route because we should // allow removing forbidden room aliases @@ -61,7 +65,11 @@ pub(crate) async fn delete_alias_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - service::rooms::alias::appservice_checks(&body.room_alias, &body.appservice_info).await?; + services + .rooms + .alias + .appservice_checks(&body.room_alias, &body.appservice_info) + .await?; if services .rooms diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 9fde99a4..e3630c72 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -43,7 +43,6 @@ use crate::{ service::{ pdu::{gen_event_id_canonical_json, PduBuilder}, rooms::state::RoomMutexGuard, - sending::convert_to_outgoing_federation_event, server_is_ours, user_is_local, Services, }, Ruma, @@ -791,7 +790,9 @@ async fn join_room_by_id_helper_remote( federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), event_id: event_id.to_owned(), - pdu: convert_to_outgoing_federation_event(join_event.clone()), + pdu: services + .sending + .convert_to_outgoing_federation_event(join_event.clone()), omit_members: false, }, ) @@ -1203,7 +1204,9 @@ async fn join_room_by_id_helper_local( federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), event_id: event_id.to_owned(), - pdu: convert_to_outgoing_federation_event(join_event.clone()), + pdu: services + .sending + .convert_to_outgoing_federation_event(join_event.clone()), omit_members: false, }, ) @@ -1431,7 +1434,9 @@ pub(crate) async fn invite_helper( room_id: room_id.to_owned(), event_id: (*pdu.event_id).to_owned(), room_version: room_version_id.clone(), - event: convert_to_outgoing_federation_event(pdu_json.clone()), + event: services + .sending + .convert_to_outgoing_federation_event(pdu_json.clone()), invite_room_state, via: services.rooms.state_cache.servers_route_via(room_id).ok(), }, @@ -1763,7 +1768,9 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room federation::membership::create_leave_event::v2::Request { room_id: room_id.to_owned(), event_id, - pdu: convert_to_outgoing_federation_event(leave_event.clone()), + pdu: services + .sending + .convert_to_outgoing_federation_event(leave_event.clone()), }, ) .await?; diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 5739052c..6eeb8fff 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -475,8 +475,6 @@ async fn handle_left_room( async fn process_presence_updates( services: &Services, presence_updates: &mut HashMap, since: u64, syncing_user: &UserId, ) -> Result<()> { - use crate::service::presence::Presence; - // Take presence updates for (user_id, _, presence_bytes) in services.presence.presence_since(since) { if !services @@ -487,7 +485,9 @@ async fn process_presence_updates( continue; } - let presence_event = Presence::from_json_bytes_to_event(&presence_bytes, &user_id)?; + let presence_event = services + .presence + .from_json_bytes_to_event(&presence_bytes, &user_id)?; match presence_updates.entry(user_id) { Entry::Vacant(slot) => { slot.insert(presence_event); diff --git a/src/api/mod.rs b/src/api/mod.rs index 79382934..0d80e581 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "160"] + pub mod client; pub mod router; pub mod server; diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 8dd38cad..1b665c19 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -4,7 +4,6 @@ use ruma::{ api::{client::error::ErrorKind, federation::backfill::get_backfill}, uint, user_id, MilliSecondsSinceUnixEpoch, }; -use service::sending::convert_to_outgoing_federation_event; use crate::Ruma; @@ -67,7 +66,7 @@ pub(crate) async fn get_backfill_route( }) .map(|(_, pdu)| services.rooms.timeline.get_pdu_json(&pdu.event_id)) .filter_map(|r| r.ok().flatten()) - .map(convert_to_outgoing_federation_event) + .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) .collect(); Ok(get_backfill::v1::Response { diff --git a/src/api/server/event.rs b/src/api/server/event.rs index e8e08c81..e11a01a2 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -4,7 +4,6 @@ use ruma::{ api::{client::error::ErrorKind, federation::event::get_event}, MilliSecondsSinceUnixEpoch, RoomId, }; -use service::sending::convert_to_outgoing_federation_event; use crate::Ruma; @@ -50,6 +49,6 @@ pub(crate) async fn get_event_route( Ok(get_event::v1::Response { origin: services.globals.server_name().to_owned(), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - pdu: convert_to_outgoing_federation_event(event), + pdu: services.sending.convert_to_outgoing_federation_event(event), }) } diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 8d26b73a..4b0f6bc0 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -6,7 +6,6 @@ use ruma::{ api::{client::error::ErrorKind, federation::authorization::get_event_authorization}, RoomId, }; -use service::sending::convert_to_outgoing_federation_event; use crate::Ruma; @@ -60,7 +59,7 @@ pub(crate) async fn get_event_authorization_route( Ok(get_event_authorization::v1::Response { auth_chain: auth_chain_ids .filter_map(|id| services.rooms.timeline.get_pdu_json(&id).ok()?) - .map(convert_to_outgoing_federation_event) + .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) .collect(), }) } diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index 378cd4fe..e2c3c93c 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -4,7 +4,6 @@ use ruma::{ api::{client::error::ErrorKind, federation::event::get_missing_events}, OwnedEventId, RoomId, }; -use service::sending::convert_to_outgoing_federation_event; use crate::Ruma; @@ -82,7 +81,7 @@ pub(crate) async fn get_missing_events_route( ) .map_err(|_| Error::bad_database("Invalid prev_events in event in database."))?, ); - events.push(convert_to_outgoing_federation_event(pdu)); + events.push(services.sending.convert_to_outgoing_federation_event(pdu)); } i = i.saturating_add(1); } diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 982a2a01..17e21920 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -7,7 +7,7 @@ use ruma::{ serde::JsonObject, CanonicalJsonValue, EventId, OwnedUserId, }; -use service::{sending::convert_to_outgoing_federation_event, server_is_ours}; +use service::server_is_ours; use crate::Ruma; @@ -174,6 +174,8 @@ pub(crate) async fn create_invite_route( } Ok(create_invite::v2::Response { - event: convert_to_outgoing_federation_event(signed_event), + event: services + .sending + .convert_to_outgoing_federation_event(signed_event), }) } diff --git a/src/api/server/send.rs b/src/api/server/send.rs index f2934480..2f698d33 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -21,7 +21,6 @@ use ruma::{ use tokio::sync::RwLock; use crate::{ - service::rooms::event_handler::parse_incoming_pdu, services::Services, utils::{self}, Error, Result, Ruma, @@ -89,7 +88,7 @@ async fn handle_pdus( ) -> Result { let mut parsed_pdus = Vec::with_capacity(body.pdus.len()); for pdu in &body.pdus { - parsed_pdus.push(match parse_incoming_pdu(pdu) { + parsed_pdus.push(match services.rooms.event_handler.parse_incoming_pdu(pdu) { Ok(t) => t, Err(e) => { debug_warn!("Could not parse PDU: {e}"); diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index b72bfa03..7f79a1d9 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -13,9 +13,7 @@ use ruma::{ CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use service::{ - pdu::gen_event_id_canonical_json, sending::convert_to_outgoing_federation_event, user_is_local, Services, -}; +use service::{pdu::gen_event_id_canonical_json, user_is_local, Services}; use tokio::sync::RwLock; use tracing::warn; @@ -186,12 +184,12 @@ async fn create_join_event( Ok(create_join_event::v1::RoomState { auth_chain: auth_chain_ids .filter_map(|id| services.rooms.timeline.get_pdu_json(&id).ok().flatten()) - .map(convert_to_outgoing_federation_event) + .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) .collect(), state: state_ids .iter() .filter_map(|(_, id)| services.rooms.timeline.get_pdu_json(id).ok().flatten()) - .map(convert_to_outgoing_federation_event) + .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) .collect(), // Event field is required if the room version supports restricted join rules. event: Some( diff --git a/src/api/server/state.rs b/src/api/server/state.rs index 24a11cca..d215236a 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use axum::extract::State; use conduit::{Error, Result}; use ruma::api::{client::error::ErrorKind, federation::event::get_room_state}; -use service::sending::convert_to_outgoing_federation_event; use crate::Ruma; @@ -44,7 +43,11 @@ pub(crate) async fn get_room_state_route( .state_full_ids(shortstatehash) .await? .into_values() - .map(|id| convert_to_outgoing_federation_event(services.rooms.timeline.get_pdu_json(&id).unwrap().unwrap())) + .map(|id| { + services + .sending + .convert_to_outgoing_federation_event(services.rooms.timeline.get_pdu_json(&id).unwrap().unwrap()) + }) .collect(); let auth_chain_ids = services @@ -61,7 +64,7 @@ pub(crate) async fn get_room_state_route( .timeline .get_pdu_json(&id) .ok()? - .map(convert_to_outgoing_federation_event) + .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) }) .collect(), pdus, diff --git a/src/router/mod.rs b/src/router/mod.rs index e9bae3c5..03c70f6d 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "160"] + mod layers; mod request; mod router; diff --git a/src/service/account_data/data.rs b/src/service/account_data/data.rs index 7b3a3dee..439603be 100644 --- a/src/service/account_data/data.rs +++ b/src/service/account_data/data.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, sync::Arc}; use conduit::{utils, warn, Error, Result}; -use database::{Database, Map}; +use database::Map; use ruma::{ api::client::error::ErrorKind, events::{AnyEphemeralRoomEvent, RoomAccountDataEventType}, @@ -9,18 +9,27 @@ use ruma::{ RoomId, UserId, }; -use crate::services; +use crate::{globals, Dep}; pub(super) struct Data { roomuserdataid_accountdata: Arc, roomusertype_roomuserdataid: Arc, + services: Services, +} + +struct Services { + globals: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { roomuserdataid_accountdata: db["roomuserdataid_accountdata"].clone(), roomusertype_roomuserdataid: db["roomusertype_roomuserdataid"].clone(), + services: Services { + globals: args.depend::("globals"), + }, } } @@ -40,7 +49,7 @@ impl Data { prefix.push(0xFF); let mut roomuserdataid = prefix.clone(); - roomuserdataid.extend_from_slice(&services().globals.next_count()?.to_be_bytes()); + roomuserdataid.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); roomuserdataid.push(0xFF); roomuserdataid.extend_from_slice(event_type.to_string().as_bytes()); diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 69d2f799..c569889e 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -17,7 +17,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data::new(&args), })) } diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 9c335791..c9a288d9 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -11,10 +11,11 @@ use rustyline_async::{Readline, ReadlineError, ReadlineEvent}; use termimad::MadSkin; use tokio::task::JoinHandle; -use crate::services; +use crate::{admin, Dep}; pub struct Console { server: Arc, + admin: Dep, worker_join: Mutex>>, input_abort: Mutex>, command_abort: Mutex>, @@ -29,6 +30,7 @@ impl Console { pub(super) fn new(args: &crate::Args<'_>) -> Arc { Arc::new(Self { server: args.server.clone(), + admin: args.depend::("admin"), worker_join: None.into(), input_abort: None.into(), command_abort: None.into(), @@ -116,7 +118,8 @@ impl Console { let _suppression = log::Suppress::new(&self.server); let (mut readline, _writer) = Readline::new(PROMPT.to_owned())?; - readline.set_tab_completer(Self::tab_complete); + let self_ = Arc::clone(self); + readline.set_tab_completer(move |line| self_.tab_complete(line)); self.set_history(&mut readline); let future = readline.readline(); @@ -154,7 +157,7 @@ impl Console { } async fn process(self: Arc, line: String) { - match services().admin.command_in_place(line, None).await { + match self.admin.command_in_place(line, None).await { Ok(Some(content)) => self.output(content).await, Err(e) => error!("processing command: {e}"), _ => (), @@ -184,9 +187,8 @@ impl Console { history.truncate(HISTORY_LIMIT); } - fn tab_complete(line: &str) -> String { - services() - .admin + fn tab_complete(&self, line: &str) -> String { + self.admin .complete_command(line) .unwrap_or_else(|| line.to_owned()) } diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 213225ac..c35f8c42 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -25,13 +25,14 @@ impl super::Service { return Ok(()); }; - let state_lock = self.state.mutex.lock(&room_id).await; + let state_lock = self.services.state.mutex.lock(&room_id).await; // Use the server user to grant the new admin's power level - let server_user = &self.globals.server_user; + let server_user = &self.services.globals.server_user; // Invite and join the real user - self.timeline + self.services + .timeline .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomMember, @@ -55,7 +56,8 @@ impl super::Service { &state_lock, ) .await?; - self.timeline + self.services + .timeline .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomMember, @@ -85,7 +87,8 @@ impl super::Service { users.insert(server_user.clone(), 100.into()); users.insert(user_id.to_owned(), 100.into()); - self.timeline + self.services + .timeline .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomPowerLevels, @@ -105,12 +108,12 @@ impl super::Service { .await?; // Send welcome message - self.timeline.build_and_append_pdu( + self.services.timeline.build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomMessage, content: to_raw_value(&RoomMessageEventContent::text_html( - format!("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `@conduit:{}: --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`", self.globals.server_name()), - format!("

    Thank you for trying out conduwuit!

    \n

    conduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

    \n

    Helpful links:

    \n
    \n

    Git and Documentation: https://github.com/girlbossceo/conduwuit
    Report issues: https://github.com/girlbossceo/conduwuit/issues

    \n
    \n

    For a list of available commands, send the following message in this room: @conduit:{}: --help

    \n

    Here are some rooms you can join (by typing the command):

    \n

    conduwuit room (Ask questions and get notified on updates):
    /join #conduwuit:puppygock.gay

    \n", self.globals.server_name()), + format!("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `@conduit:{}: --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`", self.services.globals.server_name()), + format!("

    Thank you for trying out conduwuit!

    \n

    conduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

    \n

    Helpful links:

    \n
    \n

    Git and Documentation: https://github.com/girlbossceo/conduwuit
    Report issues: https://github.com/girlbossceo/conduwuit/issues

    \n
    \n

    For a list of available commands, send the following message in this room: @conduit:{}: --help

    \n

    Here are some rooms you can join (by typing the command):

    \n

    conduwuit room (Ask questions and get notified on updates):
    /join #conduwuit:puppygock.gay

    \n", self.services.globals.server_name()), )) .expect("event is valid, we just created it"), unsigned: None, diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index fcb34212..8b9473a2 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -22,15 +22,10 @@ use ruma::{ use serde_json::value::to_raw_value; use tokio::sync::{Mutex, RwLock}; -use crate::{globals, rooms, rooms::state::RoomMutexGuard, user_is_local}; +use crate::{globals, rooms, rooms::state::RoomMutexGuard, user_is_local, Dep}; pub struct Service { - server: Arc, - globals: Arc, - alias: Arc, - timeline: Arc, - state: Arc, - state_cache: Arc, + services: Services, sender: Sender, receiver: Mutex>, pub handle: RwLock>, @@ -39,6 +34,15 @@ pub struct Service { pub console: Arc, } +struct Services { + server: Arc, + globals: Dep, + alias: Dep, + timeline: Dep, + state: Dep, + state_cache: Dep, +} + #[derive(Debug)] pub struct Command { pub command: String, @@ -58,12 +62,14 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let (sender, receiver) = loole::bounded(COMMAND_QUEUE_LIMIT); Ok(Arc::new(Self { - server: args.server.clone(), - globals: args.require_service::("globals"), - alias: args.require_service::("rooms::alias"), - timeline: args.require_service::("rooms::timeline"), - state: args.require_service::("rooms::state"), - state_cache: args.require_service::("rooms::state_cache"), + services: Services { + server: args.server.clone(), + globals: args.depend::("globals"), + alias: args.depend::("rooms::alias"), + timeline: args.depend::("rooms::timeline"), + state: args.depend::("rooms::state"), + state_cache: args.depend::("rooms::state_cache"), + }, sender, receiver: Mutex::new(receiver), handle: RwLock::new(None), @@ -75,7 +81,7 @@ impl crate::Service for Service { async fn worker(self: Arc) -> Result<()> { let receiver = self.receiver.lock().await; - let mut signals = self.server.signal.subscribe(); + let mut signals = self.services.server.signal.subscribe(); loop { tokio::select! { command = receiver.recv_async() => match command { @@ -116,7 +122,7 @@ impl Service { pub async fn send_message(&self, message_content: RoomMessageEventContent) { if let Ok(Some(room_id)) = self.get_admin_room() { - let user_id = &self.globals.server_user; + let user_id = &self.services.globals.server_user; self.respond_to_room(message_content, &room_id, user_id) .await; } @@ -176,7 +182,7 @@ impl Service { /// Checks whether a given user is an admin of this server pub async fn user_is_admin(&self, user_id: &UserId) -> Result { if let Ok(Some(admin_room)) = self.get_admin_room() { - self.state_cache.is_joined(user_id, &admin_room) + self.services.state_cache.is_joined(user_id, &admin_room) } else { Ok(false) } @@ -187,10 +193,15 @@ impl Service { /// Errors are propagated from the database, and will have None if there is /// no admin room pub fn get_admin_room(&self) -> Result> { - if let Some(room_id) = self.alias.resolve_local_alias(&self.globals.admin_alias)? { + if let Some(room_id) = self + .services + .alias + .resolve_local_alias(&self.services.globals.admin_alias)? + { if self + .services .state_cache - .is_joined(&self.globals.server_user, &room_id)? + .is_joined(&self.services.globals.server_user, &room_id)? { return Ok(Some(room_id)); } @@ -207,12 +218,12 @@ impl Service { return; }; - let Ok(Some(pdu)) = self.timeline.get_pdu(&in_reply_to.event_id) else { + let Ok(Some(pdu)) = self.services.timeline.get_pdu(&in_reply_to.event_id) else { return; }; let response_sender = if self.is_admin_room(&pdu.room_id) { - &self.globals.server_user + &self.services.globals.server_user } else { &pdu.sender }; @@ -229,7 +240,7 @@ impl Service { "sender is not admin" ); - let state_lock = self.state.mutex.lock(room_id).await; + let state_lock = self.services.state.mutex.lock(room_id).await; let response_pdu = PduBuilder { event_type: TimelineEventType::RoomMessage, content: to_raw_value(&content).expect("event is valid, we just created it"), @@ -239,6 +250,7 @@ impl Service { }; if let Err(e) = self + .services .timeline .build_and_append_pdu(response_pdu, user_id, room_id, &state_lock) .await @@ -266,7 +278,8 @@ impl Service { redacts: None, }; - self.timeline + self.services + .timeline .build_and_append_pdu(response_pdu, user_id, room_id, state_lock) .await?; @@ -279,7 +292,7 @@ impl Service { let is_public_escape = is_escape && body.trim_start_matches('\\').starts_with("!admin"); // Admin command with public echo (in admin room) - let server_user = &self.globals.server_user; + let server_user = &self.services.globals.server_user; let is_public_prefix = body.starts_with("!admin") || body.starts_with(server_user.as_str()); // Expected backward branch @@ -293,7 +306,7 @@ impl Service { } // Check if server-side command-escape is disabled by configuration - if is_public_escape && !self.globals.config.admin_escape_commands { + if is_public_escape && !self.services.globals.config.admin_escape_commands { return false; } @@ -309,7 +322,7 @@ impl Service { // This will evaluate to false if the emergency password is set up so that // the administrator can execute commands as conduit - let emergency_password_set = self.globals.emergency_password().is_some(); + let emergency_password_set = self.services.globals.emergency_password().is_some(); let from_server = pdu.sender == *server_user && !emergency_password_set; if from_server && self.is_admin_room(&pdu.room_id) { return false; diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 24c9b8b0..c0752d56 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -12,7 +12,7 @@ use ruma::{ }; use tokio::sync::RwLock; -use crate::services; +use crate::{sending, Dep}; /// Compiled regular expressions for a namespace #[derive(Clone, Debug)] @@ -118,9 +118,14 @@ impl TryFrom for RegistrationInfo { pub struct Service { pub db: Data, + services: Services, registration_info: RwLock>, } +struct Services { + sending: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let mut registration_info = BTreeMap::new(); @@ -138,6 +143,9 @@ impl crate::Service for Service { Ok(Arc::new(Self { db, + services: Services { + sending: args.depend::("sending"), + }, registration_info: RwLock::new(registration_info), })) } @@ -178,7 +186,9 @@ impl Service { // deletes all active requests for the appservice if there are any so we stop // sending to the URL - services().sending.cleanup_events(service_name.to_owned())?; + self.services + .sending + .cleanup_events(service_name.to_owned())?; Ok(()) } diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index 03b0a142..386bd33c 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -18,7 +18,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; - let resolver = args.require_service::("resolver"); + let resolver = args.require::("resolver"); Ok(Arc::new(Self { default: base(config) diff --git a/src/service/emergency/mod.rs b/src/service/emergency/mod.rs new file mode 100644 index 00000000..1bb0843d --- /dev/null +++ b/src/service/emergency/mod.rs @@ -0,0 +1,83 @@ +use std::sync::Arc; + +use async_trait::async_trait; +use conduit::{error, warn, Result}; +use ruma::{ + events::{push_rules::PushRulesEventContent, GlobalAccountDataEvent, GlobalAccountDataEventType}, + push::Ruleset, +}; + +use crate::{account_data, globals, users, Dep}; + +pub struct Service { + services: Services, +} + +struct Services { + account_data: Dep, + globals: Dep, + users: Dep, +} + +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + services: Services { + account_data: args.depend::("account_data"), + globals: args.depend::("globals"), + users: args.depend::("users"), + }, + })) + } + + async fn worker(self: Arc) -> Result<()> { + self.set_emergency_access() + .inspect_err(|e| error!("Could not set the configured emergency password for the conduit user: {e}"))?; + + Ok(()) + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { + /// Sets the emergency password and push rules for the @conduit account in + /// case emergency password is set + fn set_emergency_access(&self) -> Result { + let conduit_user = &self.services.globals.server_user; + + self.services + .users + .set_password(conduit_user, self.services.globals.emergency_password().as_deref())?; + + let (ruleset, pwd_set) = match self.services.globals.emergency_password() { + Some(_) => (Ruleset::server_default(conduit_user), true), + None => (Ruleset::new(), false), + }; + + self.services.account_data.update( + None, + conduit_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(&GlobalAccountDataEvent { + content: PushRulesEventContent { + global: ruleset, + }, + }) + .expect("to json value always works"), + )?; + + if pwd_set { + warn!( + "The server account emergency password is set! Please unset it as soon as you finish admin account \ + recovery! You will be logged out of the server service account when you finish." + ); + } else { + // logs out any users still in the server service account and removes sessions + self.services.users.deactivate_account(conduit_user)?; + } + + Ok(pwd_set) + } +} diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 281c2a94..5d6240cd 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, RwLock}, }; -use conduit::{trace, utils, Error, Result}; +use conduit::{trace, utils, Error, Result, Server}; use database::{Database, Map}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ @@ -12,7 +12,7 @@ use ruma::{ DeviceId, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, ServerName, UserId, }; -use crate::services; +use crate::{rooms, Dep}; pub struct Data { global: Arc, @@ -28,14 +28,23 @@ pub struct Data { server_signingkeys: Arc, readreceiptid_readreceipt: Arc, userid_lastonetimekeyupdate: Arc, - pub(super) db: Arc, counter: RwLock, + pub(super) db: Arc, + services: Services, +} + +struct Services { + server: Arc, + short: Dep, + state_cache: Dep, + typing: Dep, } const COUNTER: &[u8] = b"c"; impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { global: db["global"].clone(), todeviceid_events: db["todeviceid_events"].clone(), @@ -50,8 +59,14 @@ impl Data { server_signingkeys: db["server_signingkeys"].clone(), readreceiptid_readreceipt: db["readreceiptid_readreceipt"].clone(), userid_lastonetimekeyupdate: db["userid_lastonetimekeyupdate"].clone(), - db: db.clone(), counter: RwLock::new(Self::stored_count(&db["global"]).expect("initialized global counter")), + db: args.db.clone(), + services: Services { + server: args.server.clone(), + short: args.depend::("rooms::short"), + state_cache: args.depend::("rooms::state_cache"), + typing: args.depend::("rooms::typing"), + }, } } @@ -118,14 +133,14 @@ impl Data { futures.push(self.userroomid_highlightcount.watch_prefix(&userid_prefix)); // Events for rooms we are in - for room_id in services() - .rooms + for room_id in self + .services .state_cache .rooms_joined(user_id) .filter_map(Result::ok) { - let short_roomid = services() - .rooms + let short_roomid = self + .services .short .get_shortroomid(&room_id) .ok() @@ -143,7 +158,7 @@ impl Data { // EDUs futures.push(Box::pin(async move { - let _result = services().rooms.typing.wait_for_update(&room_id).await; + let _result = self.services.typing.wait_for_update(&room_id).await; })); futures.push(self.readreceiptid_readreceipt.watch_prefix(&roomid_prefix)); @@ -176,12 +191,12 @@ impl Data { futures.push(self.userid_lastonetimekeyupdate.watch_prefix(&userid_bytes)); futures.push(Box::pin(async move { - while services().server.running() { - let _result = services().server.signal.subscribe().recv().await; + while self.services.server.running() { + let _result = self.services.server.signal.subscribe().recv().await; } })); - if !services().server.running() { + if !self.services.server.running() { return Ok(()); } diff --git a/src/service/globals/emerg_access.rs b/src/service/globals/emerg_access.rs deleted file mode 100644 index 50c5f8c3..00000000 --- a/src/service/globals/emerg_access.rs +++ /dev/null @@ -1,54 +0,0 @@ -use conduit::Result; -use ruma::{ - events::{push_rules::PushRulesEventContent, GlobalAccountDataEvent, GlobalAccountDataEventType}, - push::Ruleset, -}; -use tracing::{error, warn}; - -use crate::services; - -/// Set emergency access for the conduit user -pub(crate) fn init_emergency_access() { - if let Err(e) = set_emergency_access() { - error!("Could not set the configured emergency password for the conduit user: {e}"); - } -} - -/// Sets the emergency password and push rules for the @conduit account in case -/// emergency password is set -fn set_emergency_access() -> Result { - let conduit_user = &services().globals.server_user; - - services() - .users - .set_password(conduit_user, services().globals.emergency_password().as_deref())?; - - let (ruleset, pwd_set) = match services().globals.emergency_password() { - Some(_) => (Ruleset::server_default(conduit_user), true), - None => (Ruleset::new(), false), - }; - - services().account_data.update( - None, - conduit_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(&GlobalAccountDataEvent { - content: PushRulesEventContent { - global: ruleset, - }, - }) - .expect("to json value always works"), - )?; - - if pwd_set { - warn!( - "The server account emergency password is set! Please unset it as soon as you finish admin account \ - recovery! You will be logged out of the server service account when you finish." - ); - } else { - // logs out any users still in the server service account and removes sessions - services().users.deactivate_account(conduit_user)?; - } - - Ok(pwd_set) -} diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index d8c5f29b..2fe22b0e 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -10,7 +10,6 @@ use std::{ }; use conduit::{debug, debug_info, debug_warn, error, info, utils, warn, Config, Error, Result}; -use database::Database; use itertools::Itertools; use ruma::{ events::{push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType}, @@ -18,7 +17,7 @@ use ruma::{ EventId, OwnedRoomId, RoomId, UserId, }; -use crate::services; +use crate::Services; /// The current schema version. /// - If database is opened at greater version we reject with error. The @@ -28,13 +27,13 @@ use crate::services; /// equal or lesser version. These are expected to be backward-compatible. const DATABASE_VERSION: u64 = 13; -pub(crate) async fn migrations(db: &Arc, config: &Config) -> Result<()> { +pub(crate) async fn migrations(services: &Services) -> Result<()> { // Matrix resource ownership is based on the server name; changing it // requires recreating the database from scratch. - if services().users.count()? > 0 { - let conduit_user = &services().globals.server_user; + if services.users.count()? > 0 { + let conduit_user = &services.globals.server_user; - if !services().users.exists(conduit_user)? { + if !services.users.exists(conduit_user)? { error!("The {} server user does not exist, and the database is not new.", conduit_user); return Err(Error::bad_database( "Cannot reuse an existing database after changing the server name, please delete the old one first.", @@ -42,15 +41,18 @@ pub(crate) async fn migrations(db: &Arc, config: &Config) -> Result<() } } - if services().users.count()? > 0 { - migrate(db, config).await + if services.users.count()? > 0 { + migrate(services).await } else { - fresh(db, config).await + fresh(services).await } } -async fn fresh(db: &Arc, config: &Config) -> Result<()> { - services() +async fn fresh(services: &Services) -> Result<()> { + let db = &services.db; + let config = &services.server.config; + + services .globals .db .bump_database_version(DATABASE_VERSION)?; @@ -70,97 +72,100 @@ async fn fresh(db: &Arc, config: &Config) -> Result<()> { } /// Apply any migrations -async fn migrate(db: &Arc, config: &Config) -> Result<()> { - if services().globals.db.database_version()? < 1 { - db_lt_1(db, config).await?; +async fn migrate(services: &Services) -> Result<()> { + let db = &services.db; + let config = &services.server.config; + + if services.globals.db.database_version()? < 1 { + db_lt_1(services).await?; } - if services().globals.db.database_version()? < 2 { - db_lt_2(db, config).await?; + if services.globals.db.database_version()? < 2 { + db_lt_2(services).await?; } - if services().globals.db.database_version()? < 3 { - db_lt_3(db, config).await?; + if services.globals.db.database_version()? < 3 { + db_lt_3(services).await?; } - if services().globals.db.database_version()? < 4 { - db_lt_4(db, config).await?; + if services.globals.db.database_version()? < 4 { + db_lt_4(services).await?; } - if services().globals.db.database_version()? < 5 { - db_lt_5(db, config).await?; + if services.globals.db.database_version()? < 5 { + db_lt_5(services).await?; } - if services().globals.db.database_version()? < 6 { - db_lt_6(db, config).await?; + if services.globals.db.database_version()? < 6 { + db_lt_6(services).await?; } - if services().globals.db.database_version()? < 7 { - db_lt_7(db, config).await?; + if services.globals.db.database_version()? < 7 { + db_lt_7(services).await?; } - if services().globals.db.database_version()? < 8 { - db_lt_8(db, config).await?; + if services.globals.db.database_version()? < 8 { + db_lt_8(services).await?; } - if services().globals.db.database_version()? < 9 { - db_lt_9(db, config).await?; + if services.globals.db.database_version()? < 9 { + db_lt_9(services).await?; } - if services().globals.db.database_version()? < 10 { - db_lt_10(db, config).await?; + if services.globals.db.database_version()? < 10 { + db_lt_10(services).await?; } - if services().globals.db.database_version()? < 11 { - db_lt_11(db, config).await?; + if services.globals.db.database_version()? < 11 { + db_lt_11(services).await?; } - if services().globals.db.database_version()? < 12 { - db_lt_12(db, config).await?; + if services.globals.db.database_version()? < 12 { + db_lt_12(services).await?; } // This migration can be reused as-is anytime the server-default rules are // updated. - if services().globals.db.database_version()? < 13 { - db_lt_13(db, config).await?; + if services.globals.db.database_version()? < 13 { + db_lt_13(services).await?; } if db["global"].get(b"feat_sha256_media")?.is_none() { - migrate_sha256_media(db, config).await?; + migrate_sha256_media(services).await?; } else if config.media_startup_check { - checkup_sha256_media(db, config).await?; + checkup_sha256_media(services).await?; } if db["global"] .get(b"fix_bad_double_separator_in_state_cache")? .is_none() { - fix_bad_double_separator_in_state_cache(db, config).await?; + fix_bad_double_separator_in_state_cache(services).await?; } if db["global"] .get(b"retroactively_fix_bad_data_from_roomuserid_joined")? .is_none() { - retroactively_fix_bad_data_from_roomuserid_joined(db, config).await?; + retroactively_fix_bad_data_from_roomuserid_joined(services).await?; } assert_eq!( - services().globals.db.database_version().unwrap(), + services.globals.db.database_version().unwrap(), DATABASE_VERSION, "Failed asserting local database version {} is equal to known latest conduwuit database version {}", - services().globals.db.database_version().unwrap(), + services.globals.db.database_version().unwrap(), DATABASE_VERSION, ); { - let patterns = services().globals.forbidden_usernames(); + let patterns = services.globals.forbidden_usernames(); if !patterns.is_empty() { - for user_id in services() + for user_id in services .users .iter() .filter_map(Result::ok) - .filter(|user| !services().users.is_deactivated(user).unwrap_or(true)) + .filter(|user| !services.users.is_deactivated(user).unwrap_or(true)) .filter(|user| user.server_name() == config.server_name) { let matches = patterns.matches(user_id.localpart()); @@ -179,11 +184,11 @@ async fn migrate(db: &Arc, config: &Config) -> Result<()> { } { - let patterns = services().globals.forbidden_alias_names(); + let patterns = services.globals.forbidden_alias_names(); if !patterns.is_empty() { - for address in services().rooms.metadata.iter_ids() { + for address in services.rooms.metadata.iter_ids() { let room_id = address?; - let room_aliases = services().rooms.alias.local_aliases_for_room(&room_id); + let room_aliases = services.rooms.alias.local_aliases_for_room(&room_id); for room_alias_result in room_aliases { let room_alias = room_alias_result?; let matches = patterns.matches(room_alias.alias()); @@ -211,7 +216,9 @@ async fn migrate(db: &Arc, config: &Config) -> Result<()> { Ok(()) } -async fn db_lt_1(db: &Arc, _config: &Config) -> Result<()> { +async fn db_lt_1(services: &Services) -> Result<()> { + let db = &services.db; + let roomserverids = &db["roomserverids"]; let serverroomids = &db["serverroomids"]; for (roomserverid, _) in roomserverids.iter() { @@ -228,12 +235,14 @@ async fn db_lt_1(db: &Arc, _config: &Config) -> Result<()> { serverroomids.insert(&serverroomid, &[])?; } - services().globals.db.bump_database_version(1)?; + services.globals.db.bump_database_version(1)?; info!("Migration: 0 -> 1 finished"); Ok(()) } -async fn db_lt_2(db: &Arc, _config: &Config) -> Result<()> { +async fn db_lt_2(services: &Services) -> Result<()> { + let db = &services.db; + // We accidentally inserted hashed versions of "" into the db instead of just "" let userid_password = &db["roomserverids"]; for (userid, password) in userid_password.iter() { @@ -245,12 +254,14 @@ async fn db_lt_2(db: &Arc, _config: &Config) -> Result<()> { } } - services().globals.db.bump_database_version(2)?; + services.globals.db.bump_database_version(2)?; info!("Migration: 1 -> 2 finished"); Ok(()) } -async fn db_lt_3(db: &Arc, _config: &Config) -> Result<()> { +async fn db_lt_3(services: &Services) -> Result<()> { + let db = &services.db; + // Move media to filesystem let mediaid_file = &db["mediaid_file"]; for (key, content) in mediaid_file.iter() { @@ -259,41 +270,45 @@ async fn db_lt_3(db: &Arc, _config: &Config) -> Result<()> { } #[allow(deprecated)] - let path = services().media.get_media_file(&key); + let path = services.media.get_media_file(&key); let mut file = fs::File::create(path)?; file.write_all(&content)?; mediaid_file.insert(&key, &[])?; } - services().globals.db.bump_database_version(3)?; + services.globals.db.bump_database_version(3)?; info!("Migration: 2 -> 3 finished"); Ok(()) } -async fn db_lt_4(_db: &Arc, config: &Config) -> Result<()> { - // Add federated users to services() as deactivated - for our_user in services().users.iter() { +async fn db_lt_4(services: &Services) -> Result<()> { + let config = &services.server.config; + + // Add federated users to services as deactivated + for our_user in services.users.iter() { let our_user = our_user?; - if services().users.is_deactivated(&our_user)? { + if services.users.is_deactivated(&our_user)? { continue; } - for room in services().rooms.state_cache.rooms_joined(&our_user) { - for user in services().rooms.state_cache.room_members(&room?) { + for room in services.rooms.state_cache.rooms_joined(&our_user) { + for user in services.rooms.state_cache.room_members(&room?) { let user = user?; if user.server_name() != config.server_name { info!(?user, "Migration: creating user"); - services().users.create(&user, None)?; + services.users.create(&user, None)?; } } } } - services().globals.db.bump_database_version(4)?; + services.globals.db.bump_database_version(4)?; info!("Migration: 3 -> 4 finished"); Ok(()) } -async fn db_lt_5(db: &Arc, _config: &Config) -> Result<()> { +async fn db_lt_5(services: &Services) -> Result<()> { + let db = &services.db; + // Upgrade user data store let roomuserdataid_accountdata = &db["roomuserdataid_accountdata"]; let roomusertype_roomuserdataid = &db["roomusertype_roomuserdataid"]; @@ -312,26 +327,30 @@ async fn db_lt_5(db: &Arc, _config: &Config) -> Result<()> { roomusertype_roomuserdataid.insert(&key, &roomuserdataid)?; } - services().globals.db.bump_database_version(5)?; + services.globals.db.bump_database_version(5)?; info!("Migration: 4 -> 5 finished"); Ok(()) } -async fn db_lt_6(db: &Arc, _config: &Config) -> Result<()> { +async fn db_lt_6(services: &Services) -> Result<()> { + let db = &services.db; + // Set room member count let roomid_shortstatehash = &db["roomid_shortstatehash"]; for (roomid, _) in roomid_shortstatehash.iter() { let string = utils::string_from_bytes(&roomid).unwrap(); let room_id = <&RoomId>::try_from(string.as_str()).unwrap(); - services().rooms.state_cache.update_joined_count(room_id)?; + services.rooms.state_cache.update_joined_count(room_id)?; } - services().globals.db.bump_database_version(6)?; + services.globals.db.bump_database_version(6)?; info!("Migration: 5 -> 6 finished"); Ok(()) } -async fn db_lt_7(db: &Arc, _config: &Config) -> Result<()> { +async fn db_lt_7(services: &Services) -> Result<()> { + let db = &services.db; + // Upgrade state store let mut last_roomstates: HashMap = HashMap::new(); let mut current_sstatehash: Option = None; @@ -347,7 +366,7 @@ async fn db_lt_7(db: &Arc, _config: &Config) -> Result<()> { let states_parents = last_roomsstatehash.map_or_else( || Ok(Vec::new()), |&last_roomsstatehash| { - services() + services .rooms .state_compressor .load_shortstatehash_info(last_roomsstatehash) @@ -371,7 +390,7 @@ async fn db_lt_7(db: &Arc, _config: &Config) -> Result<()> { (current_state, HashSet::new()) }; - services().rooms.state_compressor.save_state_from_diff( + services.rooms.state_compressor.save_state_from_diff( current_sstatehash, Arc::new(statediffnew), Arc::new(statediffremoved), @@ -380,7 +399,7 @@ async fn db_lt_7(db: &Arc, _config: &Config) -> Result<()> { )?; /* - let mut tmp = services().rooms.load_shortstatehash_info(¤t_sstatehash)?; + let mut tmp = services.rooms.load_shortstatehash_info(¤t_sstatehash)?; let state = tmp.pop().unwrap(); println!( "{}\t{}{:?}: {:?} + {:?} - {:?}", @@ -425,12 +444,7 @@ async fn db_lt_7(db: &Arc, _config: &Config) -> Result<()> { let event_id = shorteventid_eventid.get(&seventid).unwrap().unwrap(); let string = utils::string_from_bytes(&event_id).unwrap(); let event_id = <&EventId>::try_from(string.as_str()).unwrap(); - let pdu = services() - .rooms - .timeline - .get_pdu(event_id) - .unwrap() - .unwrap(); + let pdu = services.rooms.timeline.get_pdu(event_id).unwrap().unwrap(); if Some(&pdu.room_id) != current_room.as_ref() { current_room = Some(pdu.room_id.clone()); @@ -451,12 +465,14 @@ async fn db_lt_7(db: &Arc, _config: &Config) -> Result<()> { )?; } - services().globals.db.bump_database_version(7)?; + services.globals.db.bump_database_version(7)?; info!("Migration: 6 -> 7 finished"); Ok(()) } -async fn db_lt_8(db: &Arc, _config: &Config) -> Result<()> { +async fn db_lt_8(services: &Services) -> Result<()> { + let db = &services.db; + let roomid_shortstatehash = &db["roomid_shortstatehash"]; let roomid_shortroomid = &db["roomid_shortroomid"]; let pduid_pdu = &db["pduid_pdu"]; @@ -464,7 +480,7 @@ async fn db_lt_8(db: &Arc, _config: &Config) -> Result<()> { // Generate short room ids for all rooms for (room_id, _) in roomid_shortstatehash.iter() { - let shortroomid = services().globals.next_count()?.to_be_bytes(); + let shortroomid = services.globals.next_count()?.to_be_bytes(); roomid_shortroomid.insert(&room_id, &shortroomid)?; info!("Migration: 8"); } @@ -517,12 +533,14 @@ async fn db_lt_8(db: &Arc, _config: &Config) -> Result<()> { eventid_pduid.insert_batch(batch2.iter().map(database::KeyVal::from))?; - services().globals.db.bump_database_version(8)?; + services.globals.db.bump_database_version(8)?; info!("Migration: 7 -> 8 finished"); Ok(()) } -async fn db_lt_9(db: &Arc, _config: &Config) -> Result<()> { +async fn db_lt_9(services: &Services) -> Result<()> { + let db = &services.db; + let tokenids = &db["tokenids"]; let roomid_shortroomid = &db["roomid_shortroomid"]; @@ -574,12 +592,14 @@ async fn db_lt_9(db: &Arc, _config: &Config) -> Result<()> { tokenids.remove(&key)?; } - services().globals.db.bump_database_version(9)?; + services.globals.db.bump_database_version(9)?; info!("Migration: 8 -> 9 finished"); Ok(()) } -async fn db_lt_10(db: &Arc, _config: &Config) -> Result<()> { +async fn db_lt_10(services: &Services) -> Result<()> { + let db = &services.db; + let statekey_shortstatekey = &db["statekey_shortstatekey"]; let shortstatekey_statekey = &db["shortstatekey_statekey"]; @@ -589,28 +609,30 @@ async fn db_lt_10(db: &Arc, _config: &Config) -> Result<()> { } // Force E2EE device list updates so we can send them over federation - for user_id in services().users.iter().filter_map(Result::ok) { - services().users.mark_device_key_update(&user_id)?; + for user_id in services.users.iter().filter_map(Result::ok) { + services.users.mark_device_key_update(&user_id)?; } - services().globals.db.bump_database_version(10)?; + services.globals.db.bump_database_version(10)?; info!("Migration: 9 -> 10 finished"); Ok(()) } #[allow(unreachable_code)] -async fn db_lt_11(_db: &Arc, _config: &Config) -> Result<()> { - todo!("Dropping a column to clear data is not implemented yet."); +async fn db_lt_11(services: &Services) -> Result<()> { + error!("Dropping a column to clear data is not implemented yet."); //let userdevicesessionid_uiaarequest = &db["userdevicesessionid_uiaarequest"]; //userdevicesessionid_uiaarequest.clear()?; - services().globals.db.bump_database_version(11)?; + services.globals.db.bump_database_version(11)?; info!("Migration: 10 -> 11 finished"); Ok(()) } -async fn db_lt_12(_db: &Arc, config: &Config) -> Result<()> { - for username in services().users.list_local_users()? { +async fn db_lt_12(services: &Services) -> Result<()> { + let config = &services.server.config; + + for username in services.users.list_local_users()? { let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { Ok(u) => u, Err(e) => { @@ -619,7 +641,7 @@ async fn db_lt_12(_db: &Arc, config: &Config) -> Result<()> { }, }; - let raw_rules_list = services() + let raw_rules_list = services .account_data .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) .unwrap() @@ -664,7 +686,7 @@ async fn db_lt_12(_db: &Arc, config: &Config) -> Result<()> { } } - services().account_data.update( + services.account_data.update( None, &user, GlobalAccountDataEventType::PushRules.to_string().into(), @@ -672,13 +694,15 @@ async fn db_lt_12(_db: &Arc, config: &Config) -> Result<()> { )?; } - services().globals.db.bump_database_version(12)?; + services.globals.db.bump_database_version(12)?; info!("Migration: 11 -> 12 finished"); Ok(()) } -async fn db_lt_13(_db: &Arc, config: &Config) -> Result<()> { - for username in services().users.list_local_users()? { +async fn db_lt_13(services: &Services) -> Result<()> { + let config = &services.server.config; + + for username in services.users.list_local_users()? { let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { Ok(u) => u, Err(e) => { @@ -687,7 +711,7 @@ async fn db_lt_13(_db: &Arc, config: &Config) -> Result<()> { }, }; - let raw_rules_list = services() + let raw_rules_list = services .account_data .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) .unwrap() @@ -701,7 +725,7 @@ async fn db_lt_13(_db: &Arc, config: &Config) -> Result<()> { .global .update_with_server_default(user_default_rules); - services().account_data.update( + services.account_data.update( None, &user, GlobalAccountDataEventType::PushRules.to_string().into(), @@ -709,7 +733,7 @@ async fn db_lt_13(_db: &Arc, config: &Config) -> Result<()> { )?; } - services().globals.db.bump_database_version(13)?; + services.globals.db.bump_database_version(13)?; info!("Migration: 12 -> 13 finished"); Ok(()) } @@ -717,15 +741,17 @@ async fn db_lt_13(_db: &Arc, config: &Config) -> Result<()> { /// Migrates a media directory from legacy base64 file names to sha2 file names. /// All errors are fatal. Upon success the database is keyed to not perform this /// again. -async fn migrate_sha256_media(db: &Arc, _config: &Config) -> Result<()> { +async fn migrate_sha256_media(services: &Services) -> Result<()> { + let db = &services.db; + warn!("Migrating legacy base64 file names to sha256 file names"); let mediaid_file = &db["mediaid_file"]; // Move old media files to new names let mut changes = Vec::<(PathBuf, PathBuf)>::new(); for (key, _) in mediaid_file.iter() { - let old = services().media.get_media_file_b64(&key); - let new = services().media.get_media_file_sha256(&key); + let old = services.media.get_media_file_b64(&key); + let new = services.media.get_media_file_sha256(&key); debug!(?key, ?old, ?new, num = changes.len(), "change"); changes.push((old, new)); } @@ -739,8 +765,8 @@ async fn migrate_sha256_media(db: &Arc, _config: &Config) -> Result<() // Apply fix from when sha256_media was backward-incompat and bumped the schema // version from 13 to 14. For users satisfying these conditions we can go back. - if services().globals.db.database_version()? == 14 && DATABASE_VERSION == 13 { - services().globals.db.bump_database_version(13)?; + if services.globals.db.database_version()? == 14 && DATABASE_VERSION == 13 { + services.globals.db.bump_database_version(13)?; } db["global"].insert(b"feat_sha256_media", &[])?; @@ -752,14 +778,16 @@ async fn migrate_sha256_media(db: &Arc, _config: &Config) -> Result<() /// - Going back and forth to non-sha256 legacy binaries (e.g. upstream). /// - Deletion of artifacts in the media directory which will then fall out of /// sync with the database. -async fn checkup_sha256_media(db: &Arc, config: &Config) -> Result<()> { +async fn checkup_sha256_media(services: &Services) -> Result<()> { use crate::media::encode_key; debug!("Checking integrity of media directory"); + let db = &services.db; + let media = &services.media; + let config = &services.server.config; let mediaid_file = &db["mediaid_file"]; let mediaid_user = &db["mediaid_user"]; let dbs = (mediaid_file, mediaid_user); - let media = &services().media; let timer = Instant::now(); let dir = media.get_media_dir(); @@ -791,6 +819,7 @@ async fn handle_media_check( new_path: &OsStr, old_path: &OsStr, ) -> Result<()> { use crate::media::encode_key; + let (mediaid_file, mediaid_user) = dbs; let old_exists = files.contains(old_path); @@ -827,8 +856,10 @@ async fn handle_media_check( Ok(()) } -async fn fix_bad_double_separator_in_state_cache(db: &Arc, _config: &Config) -> Result<()> { +async fn fix_bad_double_separator_in_state_cache(services: &Services) -> Result<()> { warn!("Fixing bad double separator in state_cache roomuserid_joined"); + + let db = &services.db; let roomuserid_joined = &db["roomuserid_joined"]; let _cork = db.cork_and_sync(); @@ -864,11 +895,13 @@ async fn fix_bad_double_separator_in_state_cache(db: &Arc, _config: &C Ok(()) } -async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &Arc, _config: &Config) -> Result<()> { +async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) -> Result<()> { warn!("Retroactively fixing bad data from broken roomuserid_joined"); + + let db = &services.db; let _cork = db.cork_and_sync(); - let room_ids = services() + let room_ids = services .rooms .metadata .iter_ids() @@ -878,7 +911,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &Arc, _ for room_id in room_ids.clone() { debug_info!("Fixing room {room_id}"); - let users_in_room = services() + let users_in_room = services .rooms .state_cache .room_members(&room_id) @@ -888,7 +921,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &Arc, _ let joined_members = users_in_room .iter() .filter(|user_id| { - services() + services .rooms .state_accessor .get_member(&room_id, user_id) @@ -900,7 +933,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &Arc, _ let non_joined_members = users_in_room .iter() .filter(|user_id| { - services() + services .rooms .state_accessor .get_member(&room_id, user_id) @@ -913,7 +946,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &Arc, _ for user_id in joined_members { debug_info!("User is joined, marking as joined"); - services() + services .rooms .state_cache .mark_as_joined(user_id, &room_id)?; @@ -921,10 +954,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &Arc, _ for user_id in non_joined_members { debug_info!("User is left or banned, marking as left"); - services() - .rooms - .state_cache - .mark_as_left(user_id, &room_id)?; + services.rooms.state_cache.mark_as_left(user_id, &room_id)?; } } @@ -933,7 +963,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(db: &Arc, _ "Updating joined count for room {room_id} to fix servers in room after correcting membership states" ); - services().rooms.state_cache.update_joined_count(&room_id)?; + services.rooms.state_cache.update_joined_count(&room_id)?; } db.db.cleanup()?; diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 811aff3a..eab156ee 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -1,5 +1,4 @@ mod data; -mod emerg_access; pub(super) mod migrations; use std::{ @@ -9,7 +8,6 @@ use std::{ time::Instant, }; -use async_trait::async_trait; use conduit::{error, trace, Config, Result}; use data::Data; use ipaddress::IPAddress; @@ -43,11 +41,10 @@ pub struct Service { type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries -#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { + let db = Data::new(&args); let config = &args.server.config; - let db = Data::new(args.db); let keypair = db.load_keypair(); let keypair = match keypair { @@ -104,19 +101,13 @@ impl crate::Service for Service { .supported_room_versions() .contains(&s.config.default_room_version) { - error!(config=?s.config.default_room_version, fallback=?crate::config::default_default_room_version(), "Room version in config isn't supported, falling back to default version"); - s.config.default_room_version = crate::config::default_default_room_version(); + error!(config=?s.config.default_room_version, fallback=?conduit::config::default_default_room_version(), "Room version in config isn't supported, falling back to default version"); + s.config.default_room_version = conduit::config::default_default_room_version(); }; Ok(Arc::new(s)) } - async fn worker(self: Arc) -> Result<()> { - emerg_access::init_emergency_access(); - - Ok(()) - } - fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { let bad_event_ratelimiter = self .bad_event_ratelimiter diff --git a/src/service/key_backups/data.rs b/src/service/key_backups/data.rs index f1794895..30ac593b 100644 --- a/src/service/key_backups/data.rs +++ b/src/service/key_backups/data.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; use conduit::{utils, Error, Result}; -use database::{Database, Map}; +use database::Map; use ruma::{ api::client::{ backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, @@ -11,25 +11,34 @@ use ruma::{ OwnedRoomId, RoomId, UserId, }; -use crate::services; +use crate::{globals, Dep}; pub(super) struct Data { backupid_algorithm: Arc, backupid_etag: Arc, backupkeyid_backup: Arc, + services: Services, +} + +struct Services { + globals: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { backupid_algorithm: db["backupid_algorithm"].clone(), backupid_etag: db["backupid_etag"].clone(), backupkeyid_backup: db["backupkeyid_backup"].clone(), + services: Services { + globals: args.depend::("globals"), + }, } } pub(super) fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { - let version = services().globals.next_count()?.to_string(); + let version = self.services.globals.next_count()?.to_string(); let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); @@ -40,7 +49,7 @@ impl Data { &serde_json::to_vec(backup_metadata).expect("BackupAlgorithm::to_vec always works"), )?; self.backupid_etag - .insert(&key, &services().globals.next_count()?.to_be_bytes())?; + .insert(&key, &self.services.globals.next_count()?.to_be_bytes())?; Ok(version) } @@ -75,7 +84,7 @@ impl Data { self.backupid_algorithm .insert(&key, backup_metadata.json().get().as_bytes())?; self.backupid_etag - .insert(&key, &services().globals.next_count()?.to_be_bytes())?; + .insert(&key, &self.services.globals.next_count()?.to_be_bytes())?; Ok(version.to_owned()) } @@ -152,7 +161,7 @@ impl Data { } self.backupid_etag - .insert(&key, &services().globals.next_count()?.to_be_bytes())?; + .insert(&key, &self.services.globals.next_count()?.to_be_bytes())?; key.push(0xFF); key.extend_from_slice(room_id.as_bytes()); diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index d83d4497..65d3c065 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -17,7 +17,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data::new(&args), })) } diff --git a/src/service/manager.rs b/src/service/manager.rs index 447cd6fe..087fd3fa 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -8,13 +8,13 @@ use tokio::{ time::sleep, }; -use crate::{service::Service, Services}; +use crate::{service, service::Service, Services}; pub(crate) struct Manager { manager: Mutex>>>, workers: Mutex, server: Arc, - services: &'static Services, + service: Arc, } type Workers = JoinSet; @@ -29,7 +29,7 @@ impl Manager { manager: Mutex::new(None), workers: Mutex::new(JoinSet::new()), server: services.server.clone(), - services: crate::services(), + service: services.service.clone(), }) } @@ -53,9 +53,19 @@ impl Manager { .spawn(async move { self_.worker().await }), ); + // we can't hold the lock during the iteration with start_worker so the values + // are snapshotted here + let services: Vec> = self + .service + .read() + .expect("locked for reading") + .values() + .map(|v| v.0.clone()) + .collect(); + debug!("Starting service workers..."); - for (service, ..) in self.services.service.values() { - self.start_worker(&mut workers, service).await?; + for service in services { + self.start_worker(&mut workers, &service).await?; } Ok(()) diff --git a/src/service/media/data.rs b/src/service/media/data.rs index e5856bbf..617ec526 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -1,10 +1,10 @@ use std::sync::Arc; -use conduit::{debug, debug_info, Error, Result}; +use conduit::{debug, debug_info, utils::string_from_bytes, Error, Result}; use database::{Database, Map}; use ruma::api::client::error::ErrorKind; -use crate::{media::UrlPreviewData, utils::string_from_bytes}; +use crate::media::UrlPreviewData; pub(crate) struct Data { mediaid_file: Arc, diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 4a7d38a4..d5d518dc 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -15,7 +15,7 @@ use tokio::{ io::{AsyncReadExt, AsyncWriteExt, BufReader}, }; -use crate::services; +use crate::{globals, Dep}; #[derive(Debug)] pub struct FileMeta { @@ -41,16 +41,24 @@ pub struct UrlPreviewData { } pub struct Service { - server: Arc, + services: Services, pub(crate) db: Data, pub url_preview_mutex: MutexMap, } +struct Services { + server: Arc, + globals: Dep, +} + #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - server: args.server.clone(), + services: Services { + server: args.server.clone(), + globals: args.depend::("globals"), + }, db: Data::new(args.db), url_preview_mutex: MutexMap::new(), })) @@ -164,7 +172,7 @@ impl Service { debug!("Parsed MXC key to URL: {mxc_s}"); let mxc = OwnedMxcUri::from(mxc_s); - if mxc.server_name() == Ok(services().globals.server_name()) { + if mxc.server_name() == Ok(self.services.globals.server_name()) { debug!("Ignoring local media MXC: {mxc}"); // ignore our own MXC URLs as this would be local media. continue; @@ -246,7 +254,7 @@ impl Service { let legacy_rm = fs::remove_file(&legacy); let (file_rm, legacy_rm) = tokio::join!(file_rm, legacy_rm); if let Err(e) = legacy_rm { - if self.server.config.media_compat_file_link { + if self.services.server.config.media_compat_file_link { debug_error!(?key, ?legacy, "Failed to remove legacy media symlink: {e}"); } } @@ -259,7 +267,7 @@ impl Service { debug!(?key, ?path, "Creating media file"); let file = fs::File::create(&path).await?; - if self.server.config.media_compat_file_link { + if self.services.server.config.media_compat_file_link { let legacy = self.get_media_file_b64(key); if let Err(e) = fs::symlink(&path, &legacy).await { debug_error!( @@ -304,7 +312,7 @@ impl Service { #[must_use] pub fn get_media_dir(&self) -> PathBuf { let mut r = PathBuf::new(); - r.push(self.server.config.database_path.clone()); + r.push(self.services.server.config.database_path.clone()); r.push("media"); r } diff --git a/src/service/mod.rs b/src/service/mod.rs index 21d1f594..6e749c99 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,3 +1,4 @@ +#![recursion_limit = "160"] #![allow(refining_impl_trait)] mod manager; @@ -8,6 +9,7 @@ pub mod account_data; pub mod admin; pub mod appservice; pub mod client; +pub mod emergency; pub mod globals; pub mod key_backups; pub mod media; @@ -26,8 +28,8 @@ extern crate conduit_database as database; use std::sync::{Arc, RwLock}; -pub(crate) use conduit::{config, debug_error, debug_warn, utils, Error, Result, Server}; pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; +use conduit::{Result, Server}; use database::Database; pub(crate) use service::{Args, Dep, Service}; diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index dac8becf..53f9d8c7 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -1,21 +1,32 @@ use std::sync::Arc; use conduit::{debug_warn, utils, Error, Result}; -use database::{Database, Map}; +use database::Map; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; -use crate::{presence::Presence, services}; +use crate::{globals, presence::Presence, users, Dep}; pub struct Data { presenceid_presence: Arc, userid_presenceid: Arc, + services: Services, +} + +struct Services { + globals: Dep, + users: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { presenceid_presence: db["presenceid_presence"].clone(), userid_presenceid: db["userid_presenceid"].clone(), + services: Services { + globals: args.depend::("globals"), + users: args.depend::("users"), + }, } } @@ -28,7 +39,10 @@ impl Data { self.presenceid_presence .get(&key)? .map(|presence_bytes| -> Result<(u64, PresenceEvent)> { - Ok((count, Presence::from_json_bytes(&presence_bytes)?.to_presence_event(user_id)?)) + Ok(( + count, + Presence::from_json_bytes(&presence_bytes)?.to_presence_event(user_id, &self.services.users)?, + )) }) .transpose() } else { @@ -80,7 +94,7 @@ impl Data { last_active_ts, status_msg, ); - let count = services().globals.next_count()?; + let count = self.services.globals.next_count()?; let key = presenceid_key(count, user_id); self.presenceid_presence diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 254304ba..705ac4ff 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -3,8 +3,7 @@ mod data; use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{checked, debug, error, utils, Error, Result}; -use data::Data; +use conduit::{checked, debug, error, utils, Error, Result, Server}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ events::presence::{PresenceEvent, PresenceEventContent}, @@ -14,7 +13,8 @@ use ruma::{ use serde::{Deserialize, Serialize}; use tokio::{sync::Mutex, time::sleep}; -use crate::{services, user_is_local}; +use self::data::Data; +use crate::{user_is_local, users, Dep}; /// Represents data required to be kept in order to implement the presence /// specification. @@ -37,11 +37,6 @@ impl Presence { } } - pub fn from_json_bytes_to_event(bytes: &[u8], user_id: &UserId) -> Result { - let presence = Self::from_json_bytes(bytes)?; - presence.to_presence_event(user_id) - } - pub fn from_json_bytes(bytes: &[u8]) -> Result { serde_json::from_slice(bytes).map_err(|_| Error::bad_database("Invalid presence data in database")) } @@ -51,7 +46,7 @@ impl Presence { } /// Creates a PresenceEvent from available data. - pub fn to_presence_event(&self, user_id: &UserId) -> Result { + pub fn to_presence_event(&self, user_id: &UserId, users: &users::Service) -> Result { let now = utils::millis_since_unix_epoch(); let last_active_ago = if self.currently_active { None @@ -66,14 +61,15 @@ impl Presence { status_msg: self.status_msg.clone(), currently_active: Some(self.currently_active), last_active_ago, - displayname: services().users.displayname(user_id)?, - avatar_url: services().users.avatar_url(user_id)?, + displayname: users.displayname(user_id)?, + avatar_url: users.avatar_url(user_id)?, }, }) } } pub struct Service { + services: Services, pub db: Data, pub timer_sender: loole::Sender<(OwnedUserId, Duration)>, timer_receiver: Mutex>, @@ -82,6 +78,11 @@ pub struct Service { offline_timeout: u64, } +struct Services { + server: Arc, + users: Dep, +} + #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -90,7 +91,11 @@ impl crate::Service for Service { let offline_timeout_s = config.presence_offline_timeout_s; let (timer_sender, timer_receiver) = loole::unbounded(); Ok(Arc::new(Self { - db: Data::new(args.db), + services: Services { + server: args.server.clone(), + users: args.depend::("users"), + }, + db: Data::new(&args), timer_sender, timer_receiver: Mutex::new(timer_receiver), timeout_remote_users: config.presence_timeout_remote_users, @@ -182,8 +187,8 @@ impl Service { if self.timeout_remote_users || user_is_local(user_id) { let timeout = match presence_state { - PresenceState::Online => services().globals.config.presence_idle_timeout_s, - _ => services().globals.config.presence_offline_timeout_s, + PresenceState::Online => self.services.server.config.presence_idle_timeout_s, + _ => self.services.server.config.presence_offline_timeout_s, }; self.timer_sender @@ -210,6 +215,11 @@ impl Service { self.db.presence_since(since) } + pub fn from_json_bytes_to_event(&self, bytes: &[u8], user_id: &UserId) -> Result { + let presence = Presence::from_json_bytes(bytes)?; + presence.to_presence_event(user_id, &self.services.users) + } + fn process_presence_timer(&self, user_id: &OwnedUserId) -> Result<()> { let mut presence_state = PresenceState::Offline; let mut last_active_ago = None; diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 38ea5b9a..873f0f49 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -3,8 +3,7 @@ mod data; use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; -use conduit::{debug_info, info, trace, warn, Error, Result}; -use data::Data; +use conduit::{debug_info, info, trace, utils::string_from_bytes, warn, Error, PduEvent, Result}; use ipaddress::IPAddress; use ruma::{ api::{ @@ -23,15 +22,32 @@ use ruma::{ uint, RoomId, UInt, UserId, }; -use crate::{services, PduEvent}; +use self::data::Data; +use crate::{client, globals, rooms, users, Dep}; pub struct Service { + services: Services, db: Data, } +struct Services { + globals: Dep, + client: Dep, + state_accessor: Dep, + state_cache: Dep, + users: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + services: Services { + globals: args.depend::("globals"), + client: args.depend::("client"), + state_accessor: args.depend::("rooms::state_accessor"), + state_cache: args.depend::("rooms::state_cache"), + users: args.depend::("users"), + }, db: Data::new(args.db), })) } @@ -62,7 +78,7 @@ impl Service { { const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_0]; - let dest = dest.replace(services().globals.notification_push_path(), ""); + let dest = dest.replace(self.services.globals.notification_push_path(), ""); trace!("Push gateway destination: {dest}"); let http_request = request @@ -78,13 +94,13 @@ impl Service { if let Some(url_host) = reqwest_request.url().host_str() { trace!("Checking request URL for IP"); if let Ok(ip) = IPAddress::parse(url_host) { - if !services().globals.valid_cidr_range(&ip) { + if !self.services.globals.valid_cidr_range(&ip) { return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); } } } - let response = services().client.pusher.execute(reqwest_request).await; + let response = self.services.client.pusher.execute(reqwest_request).await; match response { Ok(mut response) => { @@ -93,7 +109,7 @@ impl Service { trace!("Checking response destination's IP"); if let Some(remote_addr) = response.remote_addr() { if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { - if !services().globals.valid_cidr_range(&ip) { + if !self.services.globals.valid_cidr_range(&ip) { return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); } } @@ -114,7 +130,7 @@ impl Service { if !status.is_success() { info!("Push gateway {dest} returned unsuccessful HTTP response ({status})"); - debug_info!("Push gateway response body: {:?}", crate::utils::string_from_bytes(&body)); + debug_info!("Push gateway response body: {:?}", string_from_bytes(&body)); return Err(Error::BadServerResponse("Push gateway returned unsuccessful response")); } @@ -143,8 +159,8 @@ impl Service { let mut notify = None; let mut tweaks = Vec::new(); - let power_levels: RoomPowerLevelsEventContent = services() - .rooms + let power_levels: RoomPowerLevelsEventContent = self + .services .state_accessor .room_state_get(&pdu.room_id, &StateEventType::RoomPowerLevels, "")? .map(|ev| { @@ -195,15 +211,15 @@ impl Service { let ctx = PushConditionRoomCtx { room_id: room_id.to_owned(), member_count: UInt::try_from( - services() - .rooms + self.services .state_cache .room_joined_count(room_id)? .unwrap_or(1), ) .unwrap_or_else(|_| uint!(0)), user_id: user.to_owned(), - user_display_name: services() + user_display_name: self + .services .users .displayname(user)? .unwrap_or_else(|| user.localpart().to_owned()), @@ -263,9 +279,9 @@ impl Service { notifi.user_is_target = event.state_key.as_deref() == Some(event.sender.as_str()); } - notifi.sender_display_name = services().users.displayname(&event.sender)?; + notifi.sender_display_name = self.services.users.displayname(&event.sender)?; - notifi.room_name = services().rooms.state_accessor.get_name(&event.room_id)?; + notifi.room_name = self.services.state_accessor.get_name(&event.room_id)?; self.send_request(&http.url, send_event_notification::v1::Request::new(notifi)) .await?; diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index b2a00023..4d5c132f 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -9,12 +9,9 @@ use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; use ruma::ServerName; -use crate::{ - resolver::{ - cache::{CachedDest, CachedOverride}, - fed::{add_port_to_hostname, get_ip_with_port, FedDest}, - }, - services, +use crate::resolver::{ + cache::{CachedDest, CachedOverride}, + fed::{add_port_to_hostname, get_ip_with_port, FedDest}, }; #[derive(Clone, Debug)] @@ -40,7 +37,7 @@ impl super::Service { result } else { cached = false; - validate_dest(server_name)?; + self.validate_dest(server_name)?; self.resolve_actual_dest(server_name, true).await? }; @@ -188,7 +185,8 @@ impl super::Service { self.query_and_cache_override(dest, dest, 8448).await?; } - let response = services() + let response = self + .services .client .well_known .get(&format!("https://{dest}/.well-known/matrix/server")) @@ -245,19 +243,14 @@ impl super::Service { #[tracing::instrument(skip_all, name = "ip")] async fn query_and_cache_override(&self, overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { - match services() - .resolver - .raw() - .lookup_ip(hostname.to_owned()) - .await - { - Err(e) => handle_resolve_error(&e), + match self.raw().lookup_ip(hostname.to_owned()).await { + Err(e) => Self::handle_resolve_error(&e), Ok(override_ip) => { if hostname != overname { debug_info!("{overname:?} overriden by {hostname:?}"); } - services().resolver.set_cached_override( + self.set_cached_override( overname.to_owned(), CachedOverride { ips: override_ip.iter().collect(), @@ -295,62 +288,62 @@ impl super::Service { for hostname in hostnames { match lookup_srv(self.raw(), &hostname).await { Ok(result) => return Ok(handle_successful_srv(&result)), - Err(e) => handle_resolve_error(&e)?, + Err(e) => Self::handle_resolve_error(&e)?, } } Ok(None) } -} -#[allow(clippy::single_match_else)] -fn handle_resolve_error(e: &ResolveError) -> Result<()> { - use hickory_resolver::error::ResolveErrorKind; + #[allow(clippy::single_match_else)] + fn handle_resolve_error(e: &ResolveError) -> Result<()> { + use hickory_resolver::error::ResolveErrorKind; - match *e.kind() { - ResolveErrorKind::NoRecordsFound { - .. - } => { - // Raise to debug_warn if we can find out the result wasn't from cache - debug!("{e}"); - Ok(()) - }, - _ => Err!(error!("DNS {e}")), - } -} - -fn validate_dest(dest: &ServerName) -> Result<()> { - if dest == services().globals.server_name() { - return Err!("Won't send federation request to ourselves"); + match *e.kind() { + ResolveErrorKind::NoRecordsFound { + .. + } => { + // Raise to debug_warn if we can find out the result wasn't from cache + debug!("{e}"); + Ok(()) + }, + _ => Err!(error!("DNS {e}")), + } } - if dest.is_ip_literal() || IPAddress::is_valid(dest.host()) { - validate_dest_ip_literal(dest)?; + fn validate_dest(&self, dest: &ServerName) -> Result<()> { + if dest == self.services.server.config.server_name { + return Err!("Won't send federation request to ourselves"); + } + + if dest.is_ip_literal() || IPAddress::is_valid(dest.host()) { + self.validate_dest_ip_literal(dest)?; + } + + Ok(()) } - Ok(()) -} + fn validate_dest_ip_literal(&self, dest: &ServerName) -> Result<()> { + trace!("Destination is an IP literal, checking against IP range denylist.",); + debug_assert!( + dest.is_ip_literal() || !IPAddress::is_valid(dest.host()), + "Destination is not an IP literal." + ); + let ip = IPAddress::parse(dest.host()).map_err(|e| { + debug_error!("Failed to parse IP literal from string: {}", e); + Error::BadServerResponse("Invalid IP address") + })?; -fn validate_dest_ip_literal(dest: &ServerName) -> Result<()> { - trace!("Destination is an IP literal, checking against IP range denylist.",); - debug_assert!( - dest.is_ip_literal() || !IPAddress::is_valid(dest.host()), - "Destination is not an IP literal." - ); - let ip = IPAddress::parse(dest.host()).map_err(|e| { - debug_error!("Failed to parse IP literal from string: {}", e); - Error::BadServerResponse("Invalid IP address") - })?; + self.validate_ip(&ip)?; - validate_ip(&ip)?; - - Ok(()) -} - -pub(crate) fn validate_ip(ip: &IPAddress) -> Result<()> { - if !services().globals.valid_cidr_range(ip) { - return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); + Ok(()) } - Ok(()) + pub(crate) fn validate_ip(&self, ip: &IPAddress) -> Result<()> { + if !self.services.globals.valid_cidr_range(ip) { + return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); + } + + Ok(()) + } } diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index 0fba2400..465b5985 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -5,11 +5,10 @@ use std::{ time::SystemTime, }; -use conduit::trace; +use conduit::{trace, utils::rand}; use ruma::{OwnedServerName, ServerName}; use super::fed::FedDest; -use crate::utils::rand; pub struct Cache { pub destinations: RwLock, // actual_destination, host diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 48ff8813..457ea9cc 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -6,14 +6,22 @@ mod tests; use std::{fmt::Write, sync::Arc}; -use conduit::Result; +use conduit::{Result, Server}; use hickory_resolver::TokioAsyncResolver; use self::{cache::Cache, dns::Resolver}; +use crate::{client, globals, Dep}; pub struct Service { pub cache: Arc, pub resolver: Arc, + services: Services, +} + +struct Services { + server: Arc, + client: Dep, + globals: Dep, } impl crate::Service for Service { @@ -23,6 +31,11 @@ impl crate::Service for Service { Ok(Arc::new(Self { cache: cache.clone(), resolver: Resolver::build(args.server, cache)?, + services: Services { + server: args.server.clone(), + client: args.depend::("client"), + globals: args.depend::("globals"), + }, })) } diff --git a/src/service/rooms/alias/data.rs b/src/service/rooms/alias/data.rs index 302c21ae..efd2b5b7 100644 --- a/src/service/rooms/alias/data.rs +++ b/src/service/rooms/alias/data.rs @@ -1,23 +1,32 @@ use std::sync::Arc; use conduit::{utils, Error, Result}; -use database::{Database, Map}; +use database::Map; use ruma::{api::client::error::ErrorKind, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, UserId}; -use crate::services; +use crate::{globals, Dep}; pub(super) struct Data { alias_userid: Arc, alias_roomid: Arc, aliasid_alias: Arc, + services: Services, +} + +struct Services { + globals: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { alias_userid: db["alias_userid"].clone(), alias_roomid: db["alias_roomid"].clone(), aliasid_alias: db["aliasid_alias"].clone(), + services: Services { + globals: args.depend::("globals"), + }, } } @@ -31,7 +40,7 @@ impl Data { let mut aliasid = room_id.as_bytes().to_vec(); aliasid.push(0xFF); - aliasid.extend_from_slice(&services().globals.next_count()?.to_be_bytes()); + aliasid.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); self.aliasid_alias.insert(&aliasid, alias.as_bytes())?; Ok(()) diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 792f5c98..344ab6d2 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -4,9 +4,8 @@ mod remote; use std::sync::Arc; use conduit::{err, Error, Result}; -use data::Data; use ruma::{ - api::{appservice, client::error::ErrorKind}, + api::client::error::ErrorKind, events::{ room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, StateEventType, @@ -14,16 +13,33 @@ use ruma::{ OwnedRoomAliasId, OwnedRoomId, OwnedServerName, RoomAliasId, RoomId, RoomOrAliasId, UserId, }; -use crate::{appservice::RegistrationInfo, server_is_ours, services}; +use self::data::Data; +use crate::{admin, appservice, appservice::RegistrationInfo, globals, rooms, sending, server_is_ours, Dep}; pub struct Service { db: Data, + services: Services, +} + +struct Services { + admin: Dep, + appservice: Dep, + globals: Dep, + sending: Dep, + state_accessor: Dep, } impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data::new(&args), + services: Services { + admin: args.depend::("admin"), + appservice: args.depend::("appservice"), + globals: args.depend::("globals"), + sending: args.depend::("sending"), + state_accessor: args.depend::("rooms::state_accessor"), + }, })) } @@ -33,7 +49,7 @@ impl crate::Service for Service { impl Service { #[tracing::instrument(skip(self))] pub fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { - if alias == services().globals.admin_alias && user_id != services().globals.server_user { + if alias == self.services.globals.admin_alias && user_id != self.services.globals.server_user { Err(Error::BadRequest( ErrorKind::forbidden(), "Only the server user can set this alias", @@ -72,10 +88,10 @@ impl Service { if !server_is_ours(room_alias.server_name()) && (!servers .as_ref() - .is_some_and(|servers| servers.contains(&services().globals.server_name().to_owned())) + .is_some_and(|servers| servers.contains(&self.services.globals.server_name().to_owned())) || servers.as_ref().is_none()) { - return remote::resolve(room_alias, servers).await; + return self.remote_resolve(room_alias, servers).await; } let room_id: Option = match self.resolve_local_alias(room_alias)? { @@ -111,7 +127,7 @@ impl Service { return Err(Error::BadRequest(ErrorKind::NotFound, "Alias not found.")); }; - let server_user = &services().globals.server_user; + let server_user = &self.services.globals.server_user; // The creator of an alias can remove it if self @@ -119,7 +135,7 @@ impl Service { .who_created_alias(alias)? .is_some_and(|user| user == user_id) // Server admins can remove any local alias - || services().admin.user_is_admin(user_id).await? + || self.services.admin.user_is_admin(user_id).await? // Always allow the server service account to remove the alias, since there may not be an admin room || server_user == user_id { @@ -127,8 +143,7 @@ impl Service { // Checking whether the user is able to change canonical aliases of the // room } else if let Some(event) = - services() - .rooms + self.services .state_accessor .room_state_get(&room_id, &StateEventType::RoomPowerLevels, "")? { @@ -140,8 +155,7 @@ impl Service { // If there is no power levels event, only the room creator can change // canonical aliases } else if let Some(event) = - services() - .rooms + self.services .state_accessor .room_state_get(&room_id, &StateEventType::RoomCreate, "")? { @@ -152,14 +166,16 @@ impl Service { } async fn resolve_appservice_alias(&self, room_alias: &RoomAliasId) -> Result> { - for appservice in services().appservice.read().await.values() { + use ruma::api::appservice::query::query_room_alias; + + for appservice in self.services.appservice.read().await.values() { if appservice.aliases.is_match(room_alias.as_str()) && matches!( - services() + self.services .sending .send_appservice_request( appservice.registration.clone(), - appservice::query::query_room_alias::v1::Request { + query_room_alias::v1::Request { room_alias: room_alias.to_owned(), }, ) @@ -167,10 +183,7 @@ impl Service { Ok(Some(_opt_result)) ) { return Ok(Some( - services() - .rooms - .alias - .resolve_local_alias(room_alias)? + self.resolve_local_alias(room_alias)? .ok_or_else(|| err!(Request(NotFound("Room does not exist."))))?, )); } @@ -178,20 +191,27 @@ impl Service { Ok(None) } -} -pub async fn appservice_checks(room_alias: &RoomAliasId, appservice_info: &Option) -> Result<()> { - if !server_is_ours(room_alias.server_name()) { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Alias is from another server.")); - } - - if let Some(ref info) = appservice_info { - if !info.aliases.is_match(room_alias.as_str()) { - return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias is not in namespace.")); + pub async fn appservice_checks( + &self, room_alias: &RoomAliasId, appservice_info: &Option, + ) -> Result<()> { + if !server_is_ours(room_alias.server_name()) { + return Err(Error::BadRequest(ErrorKind::InvalidParam, "Alias is from another server.")); } - } else if services().appservice.is_exclusive_alias(room_alias).await { - return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias reserved by appservice.")); - } - Ok(()) + if let Some(ref info) = appservice_info { + if !info.aliases.is_match(room_alias.as_str()) { + return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias is not in namespace.")); + } + } else if self + .services + .appservice + .is_exclusive_alias(room_alias) + .await + { + return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias reserved by appservice.")); + } + + Ok(()) + } } diff --git a/src/service/rooms/alias/remote.rs b/src/service/rooms/alias/remote.rs index 7fcd27f5..5d835240 100644 --- a/src/service/rooms/alias/remote.rs +++ b/src/service/rooms/alias/remote.rs @@ -1,71 +1,75 @@ -use conduit::{debug, debug_info, debug_warn, Error, Result}; +use conduit::{debug, debug_warn, Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation}, OwnedRoomId, OwnedServerName, RoomAliasId, }; -use crate::services; +impl super::Service { + pub(super) async fn remote_resolve( + &self, room_alias: &RoomAliasId, servers: Option<&Vec>, + ) -> Result<(OwnedRoomId, Option>)> { + debug!(?room_alias, ?servers, "resolve"); -pub(super) async fn resolve( - room_alias: &RoomAliasId, servers: Option<&Vec>, -) -> Result<(OwnedRoomId, Option>)> { - debug!(?room_alias, ?servers, "resolve"); + let mut response = self + .services + .sending + .send_federation_request( + room_alias.server_name(), + federation::query::get_room_information::v1::Request { + room_alias: room_alias.to_owned(), + }, + ) + .await; - let mut response = services() - .sending - .send_federation_request( - room_alias.server_name(), - federation::query::get_room_information::v1::Request { - room_alias: room_alias.to_owned(), - }, - ) - .await; + debug!("room alias server_name get_alias_helper response: {response:?}"); - debug!("room alias server_name get_alias_helper response: {response:?}"); + if let Err(ref e) = response { + debug_warn!( + "Server {} of the original room alias failed to assist in resolving room alias: {e}", + room_alias.server_name(), + ); + } - if let Err(ref e) = response { - debug_info!( - "Server {} of the original room alias failed to assist in resolving room alias: {e}", - room_alias.server_name() - ); - } + if response.as_ref().is_ok_and(|resp| resp.servers.is_empty()) || response.as_ref().is_err() { + if let Some(servers) = servers { + for server in servers { + response = self + .services + .sending + .send_federation_request( + server, + federation::query::get_room_information::v1::Request { + room_alias: room_alias.to_owned(), + }, + ) + .await; + debug!("Got response from server {server} for room aliases: {response:?}"); - if response.as_ref().is_ok_and(|resp| resp.servers.is_empty()) || response.as_ref().is_err() { - if let Some(servers) = servers { - for server in servers { - response = services() - .sending - .send_federation_request( - server, - federation::query::get_room_information::v1::Request { - room_alias: room_alias.to_owned(), - }, - ) - .await; - debug!("Got response from server {server} for room aliases: {response:?}"); - - if let Ok(ref response) = response { - if !response.servers.is_empty() { - break; + if let Ok(ref response) = response { + if !response.servers.is_empty() { + break; + } + debug_warn!( + "Server {server} responded with room aliases, but was empty? Response: {response:?}" + ); } - debug_warn!("Server {server} responded with room aliases, but was empty? Response: {response:?}"); } } } + + if let Ok(response) = response { + let room_id = response.room_id; + + let mut pre_servers = response.servers; + // since the room alis server responded, insert it into the list + pre_servers.push(room_alias.server_name().into()); + + return Ok((room_id, Some(pre_servers))); + } + + Err(Error::BadRequest( + ErrorKind::NotFound, + "No servers could assist in resolving the room alias", + )) } - - if let Ok(response) = response { - let room_id = response.room_id; - - let mut pre_servers = response.servers; - // since the room alis server responded, insert it into the list - pre_servers.push(room_alias.server_name().into()); - - return Ok((room_id, Some(pre_servers))); - } - - Err(Error::BadRequest( - ErrorKind::NotFound, - "No servers could assist in resolving the room alias", - )) } diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index 4e468234..6e7c7835 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -3,8 +3,8 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{utils, utils::math::usize_from_f64, Result, Server}; -use database::{Database, Map}; +use conduit::{utils, utils::math::usize_from_f64, Result}; +use database::Map; use lru_cache::LruCache; pub(super) struct Data { @@ -13,8 +13,9 @@ pub(super) struct Data { } impl Data { - pub(super) fn new(server: &Arc, db: &Arc) -> Self { - let config = &server.config; + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; + let config = &args.server.config; let cache_size = f64::from(config.auth_chain_cache_capacity); let cache_size = usize_from_f64(cache_size * config.cache_capacity_modifier).expect("valid cache size"); Self { diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 4e8c7bb2..9a1e7e67 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -6,19 +6,29 @@ use std::{ }; use conduit::{debug, error, trace, validated, warn, Err, Result}; -use data::Data; use ruma::{EventId, RoomId}; -use crate::services; +use self::data::Data; +use crate::{rooms, Dep}; pub struct Service { + services: Services, db: Data, } +struct Services { + short: Dep, + timeline: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.server, args.db), + services: Services { + short: args.depend::("rooms::short"), + timeline: args.depend::("rooms::timeline"), + }, + db: Data::new(&args), })) } @@ -27,7 +37,7 @@ impl crate::Service for Service { impl Service { pub async fn event_ids_iter<'a>( - &self, room_id: &RoomId, starting_events_: Vec>, + &'a self, room_id: &RoomId, starting_events_: Vec>, ) -> Result> + 'a> { let mut starting_events: Vec<&EventId> = Vec::with_capacity(starting_events_.len()); for starting_event in &starting_events_ { @@ -38,7 +48,7 @@ impl Service { .get_auth_chain(room_id, &starting_events) .await? .into_iter() - .filter_map(move |sid| services().rooms.short.get_eventid_from_short(sid).ok())) + .filter_map(move |sid| self.services.short.get_eventid_from_short(sid).ok())) } #[tracing::instrument(skip_all, name = "auth_chain")] @@ -48,8 +58,8 @@ impl Service { let started = std::time::Instant::now(); let mut buckets = [BUCKET; NUM_BUCKETS]; - for (i, &short) in services() - .rooms + for (i, &short) in self + .services .short .multi_get_or_create_shorteventid(starting_events)? .iter() @@ -140,7 +150,7 @@ impl Service { while let Some(event_id) = todo.pop() { trace!(?event_id, "processing auth event"); - match services().rooms.timeline.get_pdu(&event_id) { + match self.services.timeline.get_pdu(&event_id) { Ok(Some(pdu)) => { if pdu.room_id != room_id { return Err!(Request(Forbidden( @@ -150,10 +160,7 @@ impl Service { ))); } for auth_event in &pdu.auth_events { - let sauthevent = services() - .rooms - .short - .get_or_create_shorteventid(auth_event)?; + let sauthevent = self.services.short.get_or_create_shorteventid(auth_event)?; if found.insert(sauthevent) { trace!(?event_id, ?auth_event, "adding auth event to processing queue"); diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 23ec6b6b..706e6c2e 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -2,10 +2,10 @@ mod data; use std::sync::Arc; -use data::Data; +use conduit::Result; use ruma::{OwnedRoomId, RoomId}; -use crate::Result; +use self::data::Data; pub struct Service { db: Data, diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 6cb23b9f..fd8e2185 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -10,12 +10,11 @@ use std::{ }; use conduit::{ - debug, debug_error, debug_info, err, error, info, trace, + debug, debug_error, debug_info, err, error, info, pdu, trace, utils::{math::continue_exponential_backoff_secs, MutexMap}, - warn, Error, Result, + warn, Error, PduEvent, Result, }; use futures_util::Future; -pub use parse_incoming_pdu::parse_incoming_pdu; use ruma::{ api::{ client::error::ErrorKind, @@ -36,13 +35,28 @@ use ruma::{ use tokio::sync::RwLock; use super::state_compressor::CompressedStateEvent; -use crate::{pdu, services, PduEvent}; +use crate::{globals, rooms, sending, Dep}; pub struct Service { + services: Services, pub federation_handletime: StdRwLock, pub mutex_federation: RoomMutexMap, } +struct Services { + globals: Dep, + sending: Dep, + auth_chain: Dep, + metadata: Dep, + outlier: Dep, + pdu_metadata: Dep, + short: Dep, + state: Dep, + state_accessor: Dep, + state_compressor: Dep, + timeline: Dep, +} + type RoomMutexMap = MutexMap; type HandleTimeMap = HashMap; @@ -55,8 +69,21 @@ type AsyncRecursiveCanonicalJsonResult<'a> = AsyncRecursiveType<'a, Result<(Arc, BTreeMap)>>; impl crate::Service for Service { - fn build(_args: crate::Args<'_>) -> Result> { + fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + services: Services { + globals: args.depend::("globals"), + sending: args.depend::("sending"), + auth_chain: args.depend::("rooms::auth_chain"), + metadata: args.depend::("rooms::metadata"), + outlier: args.depend::("rooms::outlier"), + pdu_metadata: args.depend::("rooms::pdu_metadata"), + short: args.depend::("rooms::short"), + state: args.depend::("rooms::state"), + state_accessor: args.depend::("rooms::state_accessor"), + state_compressor: args.depend::("rooms::state_compressor"), + timeline: args.depend::("rooms::timeline"), + }, federation_handletime: HandleTimeMap::new().into(), mutex_federation: RoomMutexMap::new(), })) @@ -114,17 +141,17 @@ impl Service { pub_key_map: &'a RwLock>>, ) -> Result>> { // 1. Skip the PDU if we already have it as a timeline event - if let Some(pdu_id) = services().rooms.timeline.get_pdu_id(event_id)? { + if let Some(pdu_id) = self.services.timeline.get_pdu_id(event_id)? { return Ok(Some(pdu_id.to_vec())); } // 1.1 Check the server is in the room - if !services().rooms.metadata.exists(room_id)? { + if !self.services.metadata.exists(room_id)? { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server")); } // 1.2 Check if the room is disabled - if services().rooms.metadata.is_disabled(room_id)? { + if self.services.metadata.is_disabled(room_id)? { return Err(Error::BadRequest( ErrorKind::forbidden(), "Federation of this room is currently disabled on this server.", @@ -147,8 +174,8 @@ impl Service { self.acl_check(sender.server_name(), room_id)?; // Fetch create event - let create_event = services() - .rooms + let create_event = self + .services .state_accessor .room_state_get(room_id, &StateEventType::RoomCreate, "")? .ok_or_else(|| Error::bad_database("Failed to find create event in db."))?; @@ -156,8 +183,8 @@ impl Service { // Procure the room version let room_version_id = Self::get_room_version_id(&create_event)?; - let first_pdu_in_room = services() - .rooms + let first_pdu_in_room = self + .services .timeline .first_pdu_in_room(room_id)? .ok_or_else(|| Error::bad_database("Failed to find first pdu in db."))?; @@ -208,7 +235,8 @@ impl Service { Ok(()) => continue, Err(e) => { warn!("Prev event {} failed: {}", prev_id, e); - match services() + match self + .services .globals .bad_event_ratelimiter .write() @@ -258,7 +286,7 @@ impl Service { create_event: &Arc, first_pdu_in_room: &Arc, prev_id: &EventId, ) -> Result<()> { // Check for disabled again because it might have changed - if services().rooms.metadata.is_disabled(room_id)? { + if self.services.metadata.is_disabled(room_id)? { debug!( "Federaton of room {room_id} is currently disabled on this server. Request by origin {origin} and \ event ID {event_id}" @@ -269,7 +297,8 @@ impl Service { )); } - if let Some((time, tries)) = services() + if let Some((time, tries)) = self + .services .globals .bad_event_ratelimiter .read() @@ -349,7 +378,7 @@ impl Service { }; // Skip the PDU if it is redacted and we already have it as an outlier event - if services().rooms.timeline.get_pdu_json(event_id)?.is_some() { + if self.services.timeline.get_pdu_json(event_id)?.is_some() { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Event was redacted and we already knew about it", @@ -401,7 +430,7 @@ impl Service { // Build map of auth events let mut auth_events = HashMap::with_capacity(incoming_pdu.auth_events.len()); for id in &incoming_pdu.auth_events { - let Some(auth_event) = services().rooms.timeline.get_pdu(id)? else { + let Some(auth_event) = self.services.timeline.get_pdu(id)? else { warn!("Could not find auth event {}", id); continue; }; @@ -454,8 +483,7 @@ impl Service { trace!("Validation successful."); // 7. Persist the event as an outlier. - services() - .rooms + self.services .outlier .add_pdu_outlier(&incoming_pdu.event_id, &val)?; @@ -470,12 +498,12 @@ impl Service { origin: &ServerName, room_id: &RoomId, pub_key_map: &RwLock>>, ) -> Result>> { // Skip the PDU if we already have it as a timeline event - if let Ok(Some(pduid)) = services().rooms.timeline.get_pdu_id(&incoming_pdu.event_id) { + if let Ok(Some(pduid)) = self.services.timeline.get_pdu_id(&incoming_pdu.event_id) { return Ok(Some(pduid.to_vec())); } - if services() - .rooms + if self + .services .pdu_metadata .is_event_soft_failed(&incoming_pdu.event_id)? { @@ -521,14 +549,13 @@ impl Service { &incoming_pdu, None::, // TODO: third party invite |k, s| { - services() - .rooms + self.services .short .get_shortstatekey(&k.to_string().into(), s) .ok() .flatten() .and_then(|shortstatekey| state_at_incoming_event.get(&shortstatekey)) - .and_then(|event_id| services().rooms.timeline.get_pdu(event_id).ok().flatten()) + .and_then(|event_id| self.services.timeline.get_pdu(event_id).ok().flatten()) }, ) .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))?; @@ -541,7 +568,7 @@ impl Service { } debug!("Gathering auth events"); - let auth_events = services().rooms.state.get_auth_events( + let auth_events = self.services.state.get_auth_events( room_id, &incoming_pdu.kind, &incoming_pdu.sender, @@ -562,7 +589,7 @@ impl Service { && match room_version_id { V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &incoming_pdu.redacts { - !services().rooms.state_accessor.user_can_redact( + !self.services.state_accessor.user_can_redact( redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, @@ -577,7 +604,7 @@ impl Service { .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; if let Some(redact_id) = &content.redacts { - !services().rooms.state_accessor.user_can_redact( + !self.services.state_accessor.user_can_redact( redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, @@ -594,12 +621,12 @@ impl Service { // We start looking at current room state now, so lets lock the room trace!("Locking the room"); - let state_lock = services().rooms.state.mutex.lock(room_id).await; + let state_lock = self.services.state.mutex.lock(room_id).await; // Now we calculate the set of extremities this room has after the incoming // event has been applied. We start with the previous extremities (aka leaves) trace!("Calculating extremities"); - let mut extremities = services().rooms.state.get_forward_extremities(room_id)?; + let mut extremities = self.services.state.get_forward_extremities(room_id)?; trace!("Calculated {} extremities", extremities.len()); // Remove any forward extremities that are referenced by this incoming event's @@ -609,22 +636,13 @@ impl Service { } // Only keep those extremities were not referenced yet - extremities.retain(|id| { - !matches!( - services() - .rooms - .pdu_metadata - .is_event_referenced(room_id, id), - Ok(true) - ) - }); + extremities.retain(|id| !matches!(self.services.pdu_metadata.is_event_referenced(room_id, id), Ok(true))); debug!("Retained {} extremities. Compressing state", extremities.len()); let state_ids_compressed = Arc::new( state_at_incoming_event .iter() .map(|(shortstatekey, id)| { - services() - .rooms + self.services .state_compressor .compress_state_event(*shortstatekey, id) }) @@ -637,8 +655,8 @@ impl Service { // We also add state after incoming event to the fork states let mut state_after = state_at_incoming_event.clone(); if let Some(state_key) = &incoming_pdu.state_key { - let shortstatekey = services() - .rooms + let shortstatekey = self + .services .short .get_or_create_shortstatekey(&incoming_pdu.kind.to_string().into(), state_key)?; @@ -651,13 +669,12 @@ impl Service { // Set the new room state to the resolved state debug!("Forcing new room state"); - let (sstatehash, new, removed) = services() - .rooms + let (sstatehash, new, removed) = self + .services .state_compressor .save_state(room_id, new_room_state)?; - services() - .rooms + self.services .state .force_state(room_id, sstatehash, new, removed, &state_lock) .await?; @@ -667,8 +684,7 @@ impl Service { // if not soft fail it if soft_fail { debug!("Soft failing event"); - services() - .rooms + self.services .timeline .append_incoming_pdu( &incoming_pdu, @@ -682,8 +698,7 @@ impl Service { // Soft fail, we keep the event as an outlier but don't add it to the timeline warn!("Event was soft failed: {:?}", incoming_pdu); - services() - .rooms + self.services .pdu_metadata .mark_event_soft_failed(&incoming_pdu.event_id)?; @@ -696,8 +711,8 @@ impl Service { // Now that the event has passed all auth it is added into the timeline. // We use the `state_at_event` instead of `state_after` so we accurately // represent the state for this event. - let pdu_id = services() - .rooms + let pdu_id = self + .services .timeline .append_incoming_pdu( &incoming_pdu, @@ -723,14 +738,14 @@ impl Service { &self, room_id: &RoomId, room_version_id: &RoomVersionId, incoming_state: HashMap>, ) -> Result>> { debug!("Loading current room state ids"); - let current_sstatehash = services() - .rooms + let current_sstatehash = self + .services .state .get_room_shortstatehash(room_id)? .expect("every room has state"); - let current_state_ids = services() - .rooms + let current_state_ids = self + .services .state_accessor .state_full_ids(current_sstatehash) .await?; @@ -740,8 +755,7 @@ impl Service { let mut auth_chain_sets = Vec::with_capacity(fork_states.len()); for state in &fork_states { auth_chain_sets.push( - services() - .rooms + self.services .auth_chain .event_ids_iter(room_id, state.iter().map(|(_, id)| id.clone()).collect()) .await? @@ -755,8 +769,7 @@ impl Service { .map(|map| { map.into_iter() .filter_map(|(k, id)| { - services() - .rooms + self.services .short .get_statekey_from_short(k) .map(|(ty, st_key)| ((ty.to_string().into(), st_key), id)) @@ -766,11 +779,11 @@ impl Service { }) .collect(); - let lock = services().globals.stateres_mutex.lock(); + let lock = self.services.globals.stateres_mutex.lock(); debug!("Resolving state"); let state_resolve = state_res::resolve(room_version_id, &fork_states, auth_chain_sets, |id| { - let res = services().rooms.timeline.get_pdu(id); + let res = self.services.timeline.get_pdu(id); if let Err(e) = &res { error!("Failed to fetch event: {}", e); } @@ -793,12 +806,11 @@ impl Service { let new_room_state = state .into_iter() .map(|((event_type, state_key), event_id)| { - let shortstatekey = services() - .rooms + let shortstatekey = self + .services .short .get_or_create_shortstatekey(&event_type.to_string().into(), &state_key)?; - services() - .rooms + self.services .state_compressor .compress_state_event(shortstatekey, &event_id) }) @@ -814,15 +826,14 @@ impl Service { &self, incoming_pdu: &Arc, ) -> Result>>> { let prev_event = &*incoming_pdu.prev_events[0]; - let prev_event_sstatehash = services() - .rooms + let prev_event_sstatehash = self + .services .state_accessor .pdu_shortstatehash(prev_event)?; let state = if let Some(shortstatehash) = prev_event_sstatehash { Some( - services() - .rooms + self.services .state_accessor .state_full_ids(shortstatehash) .await, @@ -833,8 +844,8 @@ impl Service { if let Some(Ok(mut state)) = state { debug!("Using cached state"); - let prev_pdu = services() - .rooms + let prev_pdu = self + .services .timeline .get_pdu(prev_event) .ok() @@ -842,8 +853,8 @@ impl Service { .ok_or_else(|| Error::bad_database("Could not find prev event, but we know the state."))?; if let Some(state_key) = &prev_pdu.state_key { - let shortstatekey = services() - .rooms + let shortstatekey = self + .services .short .get_or_create_shortstatekey(&prev_pdu.kind.to_string().into(), state_key)?; @@ -866,13 +877,13 @@ impl Service { let mut okay = true; for prev_eventid in &incoming_pdu.prev_events { - let Ok(Some(prev_event)) = services().rooms.timeline.get_pdu(prev_eventid) else { + let Ok(Some(prev_event)) = self.services.timeline.get_pdu(prev_eventid) else { okay = false; break; }; - let Ok(Some(sstatehash)) = services() - .rooms + let Ok(Some(sstatehash)) = self + .services .state_accessor .pdu_shortstatehash(prev_eventid) else { @@ -891,15 +902,15 @@ impl Service { let mut auth_chain_sets = Vec::with_capacity(extremity_sstatehashes.len()); for (sstatehash, prev_event) in extremity_sstatehashes { - let mut leaf_state: HashMap<_, _> = services() - .rooms + let mut leaf_state: HashMap<_, _> = self + .services .state_accessor .state_full_ids(sstatehash) .await?; if let Some(state_key) = &prev_event.state_key { - let shortstatekey = services() - .rooms + let shortstatekey = self + .services .short .get_or_create_shortstatekey(&prev_event.kind.to_string().into(), state_key)?; leaf_state.insert(shortstatekey, Arc::from(&*prev_event.event_id)); @@ -910,7 +921,7 @@ impl Service { let mut starting_events = Vec::with_capacity(leaf_state.len()); for (k, id) in leaf_state { - if let Ok((ty, st_key)) = services().rooms.short.get_statekey_from_short(k) { + if let Ok((ty, st_key)) = self.services.short.get_statekey_from_short(k) { // FIXME: Undo .to_string().into() when StateMap // is updated to use StateEventType state.insert((ty.to_string().into(), st_key), id.clone()); @@ -921,8 +932,7 @@ impl Service { } auth_chain_sets.push( - services() - .rooms + self.services .auth_chain .event_ids_iter(room_id, starting_events) .await? @@ -932,9 +942,9 @@ impl Service { fork_states.push(state); } - let lock = services().globals.stateres_mutex.lock(); + let lock = self.services.globals.stateres_mutex.lock(); let result = state_res::resolve(room_version_id, &fork_states, auth_chain_sets, |id| { - let res = services().rooms.timeline.get_pdu(id); + let res = self.services.timeline.get_pdu(id); if let Err(e) = &res { error!("Failed to fetch event: {}", e); } @@ -947,8 +957,8 @@ impl Service { new_state .into_iter() .map(|((event_type, state_key), event_id)| { - let shortstatekey = services() - .rooms + let shortstatekey = self + .services .short .get_or_create_shortstatekey(&event_type.to_string().into(), &state_key)?; Ok((shortstatekey, event_id)) @@ -974,7 +984,8 @@ impl Service { pub_key_map: &RwLock>>, event_id: &EventId, ) -> Result>>> { debug!("Fetching state ids"); - match services() + match self + .services .sending .send_federation_request( origin, @@ -1004,8 +1015,8 @@ impl Service { .clone() .ok_or_else(|| Error::bad_database("Found non-state pdu in state events."))?; - let shortstatekey = services() - .rooms + let shortstatekey = self + .services .short .get_or_create_shortstatekey(&pdu.kind.to_string().into(), &state_key)?; @@ -1022,8 +1033,8 @@ impl Service { } // The original create event must still be in the state - let create_shortstatekey = services() - .rooms + let create_shortstatekey = self + .services .short .get_shortstatekey(&StateEventType::RoomCreate, "")? .expect("Room exists"); @@ -1056,7 +1067,8 @@ impl Service { ) -> AsyncRecursiveCanonicalJsonVec<'a> { Box::pin(async move { let back_off = |id| async { - match services() + match self + .services .globals .bad_event_ratelimiter .write() @@ -1075,7 +1087,7 @@ impl Service { // a. Look in the main timeline (pduid_pdu tree) // b. Look at outlier pdu tree // (get_pdu_json checks both) - if let Ok(Some(local_pdu)) = services().rooms.timeline.get_pdu(id) { + if let Ok(Some(local_pdu)) = self.services.timeline.get_pdu(id) { trace!("Found {} in db", id); events_with_auth_events.push((id, Some(local_pdu), vec![])); continue; @@ -1089,7 +1101,8 @@ impl Service { let mut events_all = HashSet::with_capacity(todo_auth_events.len()); let mut i: u64 = 0; while let Some(next_id) = todo_auth_events.pop() { - if let Some((time, tries)) = services() + if let Some((time, tries)) = self + .services .globals .bad_event_ratelimiter .read() @@ -1114,13 +1127,14 @@ impl Service { tokio::task::yield_now().await; } - if let Ok(Some(_)) = services().rooms.timeline.get_pdu(&next_id) { + if let Ok(Some(_)) = self.services.timeline.get_pdu(&next_id) { trace!("Found {} in db", next_id); continue; } debug!("Fetching {} over federation.", next_id); - match services() + match self + .services .sending .send_federation_request( origin, @@ -1195,7 +1209,8 @@ impl Service { pdus.push((local_pdu, None)); } for (next_id, value) in events_in_reverse_order.iter().rev() { - if let Some((time, tries)) = services() + if let Some((time, tries)) = self + .services .globals .bad_event_ratelimiter .read() @@ -1244,8 +1259,8 @@ impl Service { let mut eventid_info = HashMap::new(); let mut todo_outlier_stack: Vec> = initial_set; - let first_pdu_in_room = services() - .rooms + let first_pdu_in_room = self + .services .timeline .first_pdu_in_room(room_id)? .ok_or_else(|| Error::bad_database("Failed to find first pdu in db."))?; @@ -1267,19 +1282,18 @@ impl Service { { Self::check_room_id(room_id, &pdu)?; - if amount > services().globals.max_fetch_prev_events() { + if amount > self.services.globals.max_fetch_prev_events() { // Max limit reached debug!( "Max prev event limit reached! Limit: {}", - services().globals.max_fetch_prev_events() + self.services.globals.max_fetch_prev_events() ); graph.insert(prev_event_id.clone(), HashSet::new()); continue; } if let Some(json) = json_opt.or_else(|| { - services() - .rooms + self.services .outlier .get_outlier_pdu_json(&prev_event_id) .ok() @@ -1335,8 +1349,7 @@ impl Service { #[tracing::instrument(skip_all)] pub fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result<()> { let acl_event = if let Some(acl) = - services() - .rooms + self.services .state_accessor .room_state_get(room_id, &StateEventType::RoomServerAcl, "")? { diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 8fcd8549..a19862a5 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,29 +1,28 @@ -use conduit::{Err, Error, Result}; +use conduit::{pdu::gen_event_id_canonical_json, warn, Err, Error, Result}; use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId}; use serde_json::value::RawValue as RawJsonValue; -use tracing::warn; -use crate::{pdu::gen_event_id_canonical_json, services}; +impl super::Service { + pub fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { + let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { + warn!("Error parsing incoming event {pdu:?}: {e:?}"); + Error::BadServerResponse("Invalid PDU in server response") + })?; -pub fn parse_incoming_pdu(pdu: &RawJsonValue) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { - let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { - warn!("Error parsing incoming event {pdu:?}: {e:?}"); - Error::BadServerResponse("Invalid PDU in server response") - })?; + let room_id: OwnedRoomId = value + .get("room_id") + .and_then(|id| RoomId::parse(id.as_str()?).ok()) + .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid room id in pdu"))?; - let room_id: OwnedRoomId = value - .get("room_id") - .and_then(|id| RoomId::parse(id.as_str()?).ok()) - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid room id in pdu"))?; + let Ok(room_version_id) = self.services.state.get_room_version(&room_id) else { + return Err!("Server is not in room {room_id}"); + }; - let Ok(room_version_id) = services().rooms.state.get_room_version(&room_id) else { - return Err!("Server is not in room {room_id}"); - }; + let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { + // Event could not be converted to canonical json + return Err!(Request(InvalidParam("Could not convert event to canonical json."))); + }; - let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { - // Event could not be converted to canonical json - return Err!(Request(InvalidParam("Could not convert event to canonical json."))); - }; - - Ok((event_id, value, room_id)) + Ok((event_id, value, room_id)) + } } diff --git a/src/service/rooms/event_handler/signing_keys.rs b/src/service/rooms/event_handler/signing_keys.rs index 2fa5b0df..1ebcbefb 100644 --- a/src/service/rooms/event_handler/signing_keys.rs +++ b/src/service/rooms/event_handler/signing_keys.rs @@ -3,7 +3,7 @@ use std::{ time::{Duration, SystemTime}, }; -use conduit::{debug, error, info, trace, warn}; +use conduit::{debug, error, info, trace, warn, Error, Result}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::federation::{ @@ -21,8 +21,6 @@ use ruma::{ use serde_json::value::RawValue as RawJsonValue; use tokio::sync::{RwLock, RwLockWriteGuard}; -use crate::{services, Error, Result}; - impl super::Service { pub async fn fetch_required_signing_keys<'a, E>( &'a self, events: E, pub_key_map: &RwLock>>, @@ -147,7 +145,8 @@ impl super::Service { debug!("Loading signing keys for {}", origin); - let result: BTreeMap<_, _> = services() + let result: BTreeMap<_, _> = self + .services .globals .signing_keys_for(origin)? .into_iter() @@ -171,9 +170,10 @@ impl super::Service { &self, mut servers: BTreeMap>, pub_key_map: &RwLock>>, ) -> Result<()> { - for server in services().globals.trusted_servers() { + for server in self.services.globals.trusted_servers() { debug!("Asking batch signing keys from trusted server {}", server); - match services() + match self + .services .sending .send_federation_request( server, @@ -199,7 +199,8 @@ impl super::Service { // TODO: Check signature from trusted server? servers.remove(&k.server_name); - let result = services() + let result = self + .services .globals .db .add_signing_key(&k.server_name, k.clone())? @@ -234,7 +235,7 @@ impl super::Service { .into_keys() .map(|server| async move { ( - services() + self.services .sending .send_federation_request(&server, get_server_keys::v2::Request::new()) .await, @@ -248,7 +249,8 @@ impl super::Service { if let (Ok(get_keys_response), origin) = result { debug!("Result is from {origin}"); if let Ok(key) = get_keys_response.server_key.deserialize() { - let result: BTreeMap<_, _> = services() + let result: BTreeMap<_, _> = self + .services .globals .db .add_signing_key(&origin, key)? @@ -297,7 +299,7 @@ impl super::Service { return Ok(()); } - if services().globals.query_trusted_key_servers_first() { + if self.services.globals.query_trusted_key_servers_first() { info!( "query_trusted_key_servers_first is set to true, querying notary trusted key servers first for \ homeserver signing keys." @@ -349,7 +351,8 @@ impl super::Service { ) -> Result> { let contains_all_ids = |keys: &BTreeMap| signature_ids.iter().all(|id| keys.contains_key(id)); - let mut result: BTreeMap<_, _> = services() + let mut result: BTreeMap<_, _> = self + .services .globals .signing_keys_for(origin)? .into_iter() @@ -362,15 +365,16 @@ impl super::Service { } // i didnt split this out into their own functions because it's relatively small - if services().globals.query_trusted_key_servers_first() { + if self.services.globals.query_trusted_key_servers_first() { info!( "query_trusted_key_servers_first is set to true, querying notary trusted servers first for {origin} \ keys" ); - for server in services().globals.trusted_servers() { + for server in self.services.globals.trusted_servers() { debug!("Asking notary server {server} for {origin}'s signing key"); - if let Some(server_keys) = services() + if let Some(server_keys) = self + .services .sending .send_federation_request( server, @@ -394,7 +398,10 @@ impl super::Service { }) { debug!("Got signing keys: {:?}", server_keys); for k in server_keys { - services().globals.db.add_signing_key(origin, k.clone())?; + self.services + .globals + .db + .add_signing_key(origin, k.clone())?; result.extend( k.verify_keys .into_iter() @@ -414,14 +421,15 @@ impl super::Service { } debug!("Asking {origin} for their signing keys over federation"); - if let Some(server_key) = services() + if let Some(server_key) = self + .services .sending .send_federation_request(origin, get_server_keys::v2::Request::new()) .await .ok() .and_then(|resp| resp.server_key.deserialize().ok()) { - services() + self.services .globals .db .add_signing_key(origin, server_key.clone())?; @@ -447,14 +455,15 @@ impl super::Service { info!("query_trusted_key_servers_first is set to false, querying {origin} first"); debug!("Asking {origin} for their signing keys over federation"); - if let Some(server_key) = services() + if let Some(server_key) = self + .services .sending .send_federation_request(origin, get_server_keys::v2::Request::new()) .await .ok() .and_then(|resp| resp.server_key.deserialize().ok()) { - services() + self.services .globals .db .add_signing_key(origin, server_key.clone())?; @@ -477,9 +486,10 @@ impl super::Service { } } - for server in services().globals.trusted_servers() { + for server in self.services.globals.trusted_servers() { debug!("Asking notary server {server} for {origin}'s signing key"); - if let Some(server_keys) = services() + if let Some(server_keys) = self + .services .sending .send_federation_request( server, @@ -503,7 +513,10 @@ impl super::Service { }) { debug!("Got signing keys: {:?}", server_keys); for k in server_keys { - services().globals.db.add_signing_key(origin, k.clone())?; + self.services + .globals + .db + .add_signing_key(origin, k.clone())?; result.extend( k.verify_keys .into_iter() diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 96f623f2..64764198 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -6,10 +6,10 @@ use std::{ sync::{Arc, Mutex}, }; -use data::Data; +use conduit::{PduCount, Result}; use ruma::{DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, RoomId, UserId}; -use crate::{PduCount, Result}; +use self::data::Data; pub struct Service { db: Data, diff --git a/src/service/rooms/metadata/data.rs b/src/service/rooms/metadata/data.rs index 763dd0e8..efe681b1 100644 --- a/src/service/rooms/metadata/data.rs +++ b/src/service/rooms/metadata/data.rs @@ -1,30 +1,39 @@ use std::sync::Arc; use conduit::{error, utils, Error, Result}; -use database::{Database, Map}; +use database::Map; use ruma::{OwnedRoomId, RoomId}; -use crate::services; +use crate::{rooms, Dep}; pub(super) struct Data { disabledroomids: Arc, bannedroomids: Arc, roomid_shortroomid: Arc, pduid_pdu: Arc, + services: Services, +} + +struct Services { + short: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { disabledroomids: db["disabledroomids"].clone(), bannedroomids: db["bannedroomids"].clone(), roomid_shortroomid: db["roomid_shortroomid"].clone(), pduid_pdu: db["pduid_pdu"].clone(), + services: Services { + short: args.depend::("rooms::short"), + }, } } pub(super) fn exists(&self, room_id: &RoomId) -> Result { - let prefix = match services().rooms.short.get_shortroomid(room_id)? { + let prefix = match self.services.short.get_shortroomid(room_id)? { Some(b) => b.to_be_bytes().to_vec(), None => return Ok(false), }; diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index ec34a82c..7415c53b 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -3,9 +3,10 @@ mod data; use std::sync::Arc; use conduit::Result; -use data::Data; use ruma::{OwnedRoomId, RoomId}; +use self::data::Data; + pub struct Service { db: Data, } @@ -13,7 +14,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data::new(&args), })) } diff --git a/src/service/rooms/mod.rs b/src/service/rooms/mod.rs index ef50b094..44a83582 100644 --- a/src/service/rooms/mod.rs +++ b/src/service/rooms/mod.rs @@ -33,13 +33,13 @@ pub struct Service { pub read_receipt: Arc, pub search: Arc, pub short: Arc, + pub spaces: Arc, pub state: Arc, pub state_accessor: Arc, pub state_cache: Arc, pub state_compressor: Arc, - pub timeline: Arc, pub threads: Arc, + pub timeline: Arc, pub typing: Arc, - pub spaces: Arc, pub user: Arc, } diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 24c756fd..d1649da8 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,26 +1,35 @@ use std::{mem::size_of, sync::Arc}; -use conduit::{utils, Error, Result}; -use database::{Database, Map}; +use conduit::{utils, Error, PduCount, PduEvent, Result}; +use database::Map; use ruma::{EventId, RoomId, UserId}; -use crate::{services, PduCount, PduEvent}; +use crate::{rooms, Dep}; pub(super) struct Data { tofrom_relation: Arc, referencedevents: Arc, softfailedeventids: Arc, + services: Services, +} + +struct Services { + timeline: Dep, } type PdusIterItem = Result<(PduCount, PduEvent)>; type PdusIterator<'a> = Box + 'a>; impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { tofrom_relation: db["tofrom_relation"].clone(), referencedevents: db["referencedevents"].clone(), softfailedeventids: db["softfailedeventids"].clone(), + services: Services { + timeline: args.depend::("rooms::timeline"), + }, } } @@ -57,8 +66,8 @@ impl Data { let mut pduid = shortroomid.to_be_bytes().to_vec(); pduid.extend_from_slice(&from.to_be_bytes()); - let mut pdu = services() - .rooms + let mut pdu = self + .services .timeline .get_pdu_from_id(&pduid)? .ok_or_else(|| Error::bad_database("Pdu in tofrom_relation is invalid."))?; diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 05067aa8..7546dcb2 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -2,8 +2,7 @@ mod data; use std::sync::Arc; -use conduit::Result; -use data::Data; +use conduit::{PduCount, PduEvent, Result}; use ruma::{ api::{client::relations::get_relating_events, Direction}, events::{relation::RelationType, TimelineEventType}, @@ -11,12 +10,20 @@ use ruma::{ }; use serde::Deserialize; -use crate::{services, PduCount, PduEvent}; +use self::data::Data; +use crate::{rooms, Dep}; pub struct Service { + services: Services, db: Data, } +struct Services { + short: Dep, + state_accessor: Dep, + timeline: Dep, +} + #[derive(Clone, Debug, Deserialize)] struct ExtractRelType { rel_type: RelationType, @@ -30,7 +37,12 @@ struct ExtractRelatesToEventId { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + services: Services { + short: args.depend::("rooms::short"), + state_accessor: args.depend::("rooms::state_accessor"), + timeline: args.depend::("rooms::timeline"), + }, + db: Data::new(&args), })) } @@ -101,8 +113,7 @@ impl Service { }) .take(limit) .filter(|(_, pdu)| { - services() - .rooms + self.services .state_accessor .user_can_see_event(sender_user, room_id, &pdu.event_id) .unwrap_or(false) @@ -147,8 +158,7 @@ impl Service { }) .take(limit) .filter(|(_, pdu)| { - services() - .rooms + self.services .state_accessor .user_can_see_event(sender_user, room_id, &pdu.event_id) .unwrap_or(false) @@ -180,10 +190,10 @@ impl Service { pub fn relations_until<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, target: &'a EventId, until: PduCount, max_depth: u8, ) -> Result> { - let room_id = services().rooms.short.get_or_create_shortroomid(room_id)?; + let room_id = self.services.short.get_or_create_shortroomid(room_id)?; #[allow(unknown_lints)] #[allow(clippy::manual_unwrap_or_default)] - let target = match services().rooms.timeline.get_pdu_count(target)? { + let target = match self.services.timeline.get_pdu_count(target)? { Some(PduCount::Normal(c)) => c, // TODO: Support backfilled relations _ => 0, // This will result in an empty iterator diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 06eaf655..0f400ff3 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -1,14 +1,14 @@ use std::{mem::size_of, sync::Arc}; use conduit::{utils, Error, Result}; -use database::{Database, Map}; +use database::Map; use ruma::{ events::{receipt::ReceiptEvent, AnySyncEphemeralRoomEvent}, serde::Raw, CanonicalJsonObject, OwnedUserId, RoomId, UserId, }; -use crate::services; +use crate::{globals, Dep}; type AnySyncEphemeralRoomEventIter<'a> = Box)>> + 'a>; @@ -16,15 +16,24 @@ type AnySyncEphemeralRoomEventIter<'a> = pub(super) struct Data { roomuserid_privateread: Arc, roomuserid_lastprivatereadupdate: Arc, + services: Services, readreceiptid_readreceipt: Arc, } +struct Services { + globals: Dep, +} + impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { roomuserid_privateread: db["roomuserid_privateread"].clone(), roomuserid_lastprivatereadupdate: db["roomuserid_lastprivatereadupdate"].clone(), readreceiptid_readreceipt: db["readreceiptid_readreceipt"].clone(), + services: Services { + globals: args.depend::("globals"), + }, } } @@ -51,7 +60,7 @@ impl Data { } let mut room_latest_id = prefix; - room_latest_id.extend_from_slice(&services().globals.next_count()?.to_be_bytes()); + room_latest_id.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); room_latest_id.push(0xFF); room_latest_id.extend_from_slice(user_id.as_bytes()); @@ -108,7 +117,7 @@ impl Data { .insert(&key, &count.to_be_bytes())?; self.roomuserid_lastprivatereadupdate - .insert(&key, &services().globals.next_count()?.to_be_bytes()) + .insert(&key, &self.services.globals.next_count()?.to_be_bytes()) } pub(super) fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 9375276e..d202d893 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -6,16 +6,24 @@ use conduit::Result; use data::Data; use ruma::{events::receipt::ReceiptEvent, serde::Raw, OwnedUserId, RoomId, UserId}; -use crate::services; +use crate::{sending, Dep}; pub struct Service { + services: Services, db: Data, } +struct Services { + sending: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + services: Services { + sending: args.depend::("sending"), + }, + db: Data::new(&args), })) } @@ -26,7 +34,7 @@ impl Service { /// Replaces the previous read receipt. pub fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) -> Result<()> { self.db.readreceipt_update(user_id, room_id, event)?; - services().sending.flush_room(room_id)?; + self.services.sending.flush_room(room_id)?; Ok(()) } diff --git a/src/service/rooms/search/data.rs b/src/service/rooms/search/data.rs index 79b23cba..a0086095 100644 --- a/src/service/rooms/search/data.rs +++ b/src/service/rooms/search/data.rs @@ -1,21 +1,30 @@ use std::sync::Arc; use conduit::{utils, Result}; -use database::{Database, Map}; +use database::Map; use ruma::RoomId; -use crate::services; +use crate::{rooms, Dep}; type SearchPdusResult<'a> = Result> + 'a>, Vec)>>; pub(super) struct Data { tokenids: Arc, + services: Services, +} + +struct Services { + short: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { tokenids: db["tokenids"].clone(), + services: Services { + short: args.depend::("rooms::short"), + }, } } @@ -51,8 +60,8 @@ impl Data { } pub(super) fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a> { - let prefix = services() - .rooms + let prefix = self + .services .short .get_shortroomid(room_id)? .expect("room exists") diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 082dd432..8caa0ce3 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -13,7 +13,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data::new(&args), })) } diff --git a/src/service/rooms/short/data.rs b/src/service/rooms/short/data.rs index 883c3c1d..963bd927 100644 --- a/src/service/rooms/short/data.rs +++ b/src/service/rooms/short/data.rs @@ -1,10 +1,10 @@ use std::sync::Arc; use conduit::{utils, warn, Error, Result}; -use database::{Database, Map}; +use database::Map; use ruma::{events::StateEventType, EventId, RoomId}; -use crate::services; +use crate::{globals, Dep}; pub(super) struct Data { eventid_shorteventid: Arc, @@ -13,10 +13,16 @@ pub(super) struct Data { shortstatekey_statekey: Arc, roomid_shortroomid: Arc, statehash_shortstatehash: Arc, + services: Services, +} + +struct Services { + globals: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { eventid_shorteventid: db["eventid_shorteventid"].clone(), shorteventid_eventid: db["shorteventid_eventid"].clone(), @@ -24,6 +30,9 @@ impl Data { shortstatekey_statekey: db["shortstatekey_statekey"].clone(), roomid_shortroomid: db["roomid_shortroomid"].clone(), statehash_shortstatehash: db["statehash_shortstatehash"].clone(), + services: Services { + globals: args.depend::("globals"), + }, } } @@ -31,7 +40,7 @@ impl Data { let short = if let Some(shorteventid) = self.eventid_shorteventid.get(event_id.as_bytes())? { utils::u64_from_bytes(&shorteventid).map_err(|_| Error::bad_database("Invalid shorteventid in db."))? } else { - let shorteventid = services().globals.next_count()?; + let shorteventid = self.services.globals.next_count()?; self.eventid_shorteventid .insert(event_id.as_bytes(), &shorteventid.to_be_bytes())?; self.shorteventid_eventid @@ -59,7 +68,7 @@ impl Data { utils::u64_from_bytes(short).map_err(|_| Error::bad_database("Invalid shorteventid in db."))?, ), None => { - let short = services().globals.next_count()?; + let short = self.services.globals.next_count()?; self.eventid_shorteventid .insert(keys[i], &short.to_be_bytes())?; self.shorteventid_eventid @@ -98,7 +107,7 @@ impl Data { let short = if let Some(shortstatekey) = self.statekey_shortstatekey.get(&statekey_vec)? { utils::u64_from_bytes(&shortstatekey).map_err(|_| Error::bad_database("Invalid shortstatekey in db."))? } else { - let shortstatekey = services().globals.next_count()?; + let shortstatekey = self.services.globals.next_count()?; self.statekey_shortstatekey .insert(&statekey_vec, &shortstatekey.to_be_bytes())?; self.shortstatekey_statekey @@ -158,7 +167,7 @@ impl Data { true, ) } else { - let shortstatehash = services().globals.next_count()?; + let shortstatehash = self.services.globals.next_count()?; self.statehash_shortstatehash .insert(state_hash, &shortstatehash.to_be_bytes())?; (shortstatehash, false) @@ -176,7 +185,7 @@ impl Data { Ok(if let Some(short) = self.roomid_shortroomid.get(room_id.as_bytes())? { utils::u64_from_bytes(&short).map_err(|_| Error::bad_database("Invalid shortroomid in db."))? } else { - let short = services().globals.next_count()?; + let short = self.services.globals.next_count()?; self.roomid_shortroomid .insert(room_id.as_bytes(), &short.to_be_bytes())?; short diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 0979fb4f..bfe0e9a0 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -3,9 +3,10 @@ mod data; use std::sync::Arc; use conduit::Result; -use data::Data; use ruma::{events::StateEventType, EventId, RoomId}; +use self::data::Data; + pub struct Service { db: Data, } @@ -13,7 +14,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data::new(&args), })) } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 19a3ebbb..24d612d8 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -28,7 +28,7 @@ use ruma::{ }; use tokio::sync::Mutex; -use crate::services; +use crate::{rooms, sending, Dep}; pub struct CachedSpaceHierarchySummary { summary: SpaceHierarchyParentSummary, @@ -119,42 +119,18 @@ enum Identifier<'a> { } pub struct Service { + services: Services, pub roomid_spacehierarchy_cache: Mutex>>, } -// Here because cannot implement `From` across ruma-federation-api and -// ruma-client-api types -impl From for SpaceHierarchyRoomsChunk { - fn from(value: CachedSpaceHierarchySummary) -> Self { - let SpaceHierarchyParentSummary { - canonical_alias, - name, - num_joined_members, - room_id, - topic, - world_readable, - guest_can_join, - avatar_url, - join_rule, - room_type, - children_state, - .. - } = value.summary; - - Self { - canonical_alias, - name, - num_joined_members, - room_id, - topic, - world_readable, - guest_can_join, - avatar_url, - join_rule, - room_type, - children_state, - } - } +struct Services { + state_accessor: Dep, + state_cache: Dep, + state: Dep, + short: Dep, + event_handler: Dep, + timeline: Dep, + sending: Dep, } impl crate::Service for Service { @@ -163,6 +139,15 @@ impl crate::Service for Service { let cache_size = f64::from(config.roomid_spacehierarchy_cache_capacity); let cache_size = cache_size * config.cache_capacity_modifier; Ok(Arc::new(Self { + services: Services { + state_accessor: args.depend::("rooms::state_accessor"), + state_cache: args.depend::("rooms::state_cache"), + state: args.depend::("rooms::state"), + short: args.depend::("rooms::short"), + event_handler: args.depend::("rooms::event_handler"), + timeline: args.depend::("rooms::timeline"), + sending: args.depend::("sending"), + }, roomid_spacehierarchy_cache: Mutex::new(LruCache::new(usize_from_f64(cache_size)?)), })) } @@ -226,7 +211,7 @@ impl Service { .as_ref() { return Ok(if let Some(cached) = cached { - if is_accessable_child( + if self.is_accessible_child( current_room, &cached.summary.join_rule, &identifier, @@ -242,8 +227,8 @@ impl Service { } Ok( - if let Some(children_pdus) = get_stripped_space_child_events(current_room).await? { - let summary = Self::get_room_summary(current_room, children_pdus, &identifier); + if let Some(children_pdus) = self.get_stripped_space_child_events(current_room).await? { + let summary = self.get_room_summary(current_room, children_pdus, &identifier); if let Ok(summary) = summary { self.roomid_spacehierarchy_cache.lock().await.insert( current_room.clone(), @@ -269,7 +254,8 @@ impl Service { ) -> Result> { for server in via { debug_info!("Asking {server} for /hierarchy"); - let Ok(response) = services() + let Ok(response) = self + .services .sending .send_federation_request( server, @@ -325,7 +311,10 @@ impl Service { avatar_url, join_rule, room_type, - children_state: get_stripped_space_child_events(&room_id).await?.unwrap(), + children_state: self + .get_stripped_space_child_events(&room_id) + .await? + .unwrap(), allowed_room_ids, } }, @@ -333,7 +322,7 @@ impl Service { ); } } - if is_accessable_child( + if self.is_accessible_child( current_room, &response.room.join_rule, &Identifier::UserId(user_id), @@ -370,12 +359,13 @@ impl Service { } fn get_room_summary( - current_room: &OwnedRoomId, children_state: Vec>, identifier: &Identifier<'_>, + &self, current_room: &OwnedRoomId, children_state: Vec>, + identifier: &Identifier<'_>, ) -> Result { let room_id: &RoomId = current_room; - let join_rule = services() - .rooms + let join_rule = self + .services .state_accessor .room_state_get(room_id, &StateEventType::RoomJoinRules, "")? .map(|s| { @@ -386,12 +376,12 @@ impl Service { .transpose()? .unwrap_or(JoinRule::Invite); - let allowed_room_ids = services() - .rooms + let allowed_room_ids = self + .services .state_accessor .allowed_room_ids(join_rule.clone()); - if !is_accessable_child(current_room, &join_rule.clone().into(), identifier, &allowed_room_ids) { + if !self.is_accessible_child(current_room, &join_rule.clone().into(), identifier, &allowed_room_ids) { debug!("User is not allowed to see room {room_id}"); // This error will be caught later return Err(Error::BadRequest(ErrorKind::forbidden(), "User is not allowed to see the room")); @@ -400,18 +390,18 @@ impl Service { let join_rule = join_rule.into(); Ok(SpaceHierarchyParentSummary { - canonical_alias: services() - .rooms + canonical_alias: self + .services .state_accessor .get_canonical_alias(room_id) .unwrap_or(None), - name: services() - .rooms + name: self + .services .state_accessor .get_name(room_id) .unwrap_or(None), - num_joined_members: services() - .rooms + num_joined_members: self + .services .state_cache .room_joined_count(room_id) .unwrap_or_default() @@ -422,22 +412,22 @@ impl Service { .try_into() .expect("user count should not be that big"), room_id: room_id.to_owned(), - topic: services() - .rooms + topic: self + .services .state_accessor .get_room_topic(room_id) .unwrap_or(None), - world_readable: services().rooms.state_accessor.is_world_readable(room_id)?, - guest_can_join: services().rooms.state_accessor.guest_can_join(room_id)?, - avatar_url: services() - .rooms + world_readable: self.services.state_accessor.is_world_readable(room_id)?, + guest_can_join: self.services.state_accessor.guest_can_join(room_id)?, + avatar_url: self + .services .state_accessor .get_avatar(room_id)? .into_option() .unwrap_or_default() .url, join_rule, - room_type: services().rooms.state_accessor.get_room_type(room_id)?, + room_type: self.services.state_accessor.get_room_type(room_id)?, children_state, allowed_room_ids, }) @@ -487,7 +477,7 @@ impl Service { .into_iter() .rev() .skip_while(|(room, _)| { - if let Ok(short) = services().rooms.short.get_shortroomid(room) + if let Ok(short) = self.services.short.get_shortroomid(room) { short.as_ref() != short_room_ids.get(parents.len()) } else { @@ -541,7 +531,7 @@ impl Service { let mut short_room_ids = vec![]; for room in parents { - short_room_ids.push(services().rooms.short.get_or_create_shortroomid(&room)?); + short_room_ids.push(self.services.short.get_or_create_shortroomid(&room)?); } Some( @@ -559,128 +549,152 @@ impl Service { rooms: results, }) } -} -fn next_room_to_traverse( - stack: &mut Vec)>>, parents: &mut VecDeque, -) -> Option<(OwnedRoomId, Vec)> { - while stack.last().map_or(false, Vec::is_empty) { - stack.pop(); - parents.pop_back(); + /// Simply returns the stripped m.space.child events of a room + async fn get_stripped_space_child_events( + &self, room_id: &RoomId, + ) -> Result>>, Error> { + let Some(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id)? else { + return Ok(None); + }; + + let state = self + .services + .state_accessor + .state_full_ids(current_shortstatehash) + .await?; + let mut children_pdus = Vec::new(); + for (key, id) in state { + let (event_type, state_key) = self.services.short.get_statekey_from_short(key)?; + if event_type != StateEventType::SpaceChild { + continue; + } + + let pdu = self + .services + .timeline + .get_pdu(&id)? + .ok_or_else(|| Error::bad_database("Event in space state not found"))?; + + if serde_json::from_str::(pdu.content.get()) + .ok() + .map(|c| c.via) + .map_or(true, |v| v.is_empty()) + { + continue; + } + + if OwnedRoomId::try_from(state_key).is_ok() { + children_pdus.push(pdu.to_stripped_spacechild_state_event()); + } + } + + Ok(Some(children_pdus)) } - stack.last_mut().and_then(Vec::pop) -} + /// With the given identifier, checks if a room is accessable + fn is_accessible_child( + &self, current_room: &OwnedRoomId, join_rule: &SpaceRoomJoinRule, identifier: &Identifier<'_>, + allowed_room_ids: &Vec, + ) -> bool { + // Note: unwrap_or_default for bool means false + match identifier { + Identifier::ServerName(server_name) => { + let room_id: &RoomId = current_room; -/// Simply returns the stripped m.space.child events of a room -async fn get_stripped_space_child_events( - room_id: &RoomId, -) -> Result>>, Error> { - let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? else { - return Ok(None); - }; - - let state = services() - .rooms - .state_accessor - .state_full_ids(current_shortstatehash) - .await?; - let mut children_pdus = Vec::new(); - for (key, id) in state { - let (event_type, state_key) = services().rooms.short.get_statekey_from_short(key)?; - if event_type != StateEventType::SpaceChild { - continue; - } - - let pdu = services() - .rooms - .timeline - .get_pdu(&id)? - .ok_or_else(|| Error::bad_database("Event in space state not found"))?; - - if serde_json::from_str::(pdu.content.get()) - .ok() - .map(|c| c.via) - .map_or(true, |v| v.is_empty()) - { - continue; - } - - if OwnedRoomId::try_from(state_key).is_ok() { - children_pdus.push(pdu.to_stripped_spacechild_state_event()); - } - } - - Ok(Some(children_pdus)) -} - -/// With the given identifier, checks if a room is accessable -fn is_accessable_child( - current_room: &OwnedRoomId, join_rule: &SpaceRoomJoinRule, identifier: &Identifier<'_>, - allowed_room_ids: &Vec, -) -> bool { - // Note: unwrap_or_default for bool means false - match identifier { - Identifier::ServerName(server_name) => { - let room_id: &RoomId = current_room; - - // Checks if ACLs allow for the server to participate - if services() - .rooms - .event_handler - .acl_check(server_name, room_id) - .is_err() - { - return false; - } - }, - Identifier::UserId(user_id) => { - if services() - .rooms - .state_cache - .is_joined(user_id, current_room) - .unwrap_or_default() - || services() - .rooms - .state_cache - .is_invited(user_id, current_room) - .unwrap_or_default() - { - return true; - } - }, - } // Takes care of join rules - match join_rule { - SpaceRoomJoinRule::Restricted => { - for room in allowed_room_ids { - match identifier { - Identifier::UserId(user) => { - if services() - .rooms - .state_cache - .is_joined(user, room) - .unwrap_or_default() - { - return true; - } - }, - Identifier::ServerName(server) => { - if services() - .rooms - .state_cache - .server_in_room(server, room) - .unwrap_or_default() - { - return true; - } - }, + // Checks if ACLs allow for the server to participate + if self + .services + .event_handler + .acl_check(server_name, room_id) + .is_err() + { + return false; } - } - false - }, - SpaceRoomJoinRule::Public | SpaceRoomJoinRule::Knock | SpaceRoomJoinRule::KnockRestricted => true, - // Invite only, Private, or Custom join rule - _ => false, + }, + Identifier::UserId(user_id) => { + if self + .services + .state_cache + .is_joined(user_id, current_room) + .unwrap_or_default() + || self + .services + .state_cache + .is_invited(user_id, current_room) + .unwrap_or_default() + { + return true; + } + }, + } // Takes care of join rules + match join_rule { + SpaceRoomJoinRule::Restricted => { + for room in allowed_room_ids { + match identifier { + Identifier::UserId(user) => { + if self + .services + .state_cache + .is_joined(user, room) + .unwrap_or_default() + { + return true; + } + }, + Identifier::ServerName(server) => { + if self + .services + .state_cache + .server_in_room(server, room) + .unwrap_or_default() + { + return true; + } + }, + } + } + false + }, + SpaceRoomJoinRule::Public | SpaceRoomJoinRule::Knock | SpaceRoomJoinRule::KnockRestricted => true, + // Invite only, Private, or Custom join rule + _ => false, + } + } +} + +// Here because cannot implement `From` across ruma-federation-api and +// ruma-client-api types +impl From for SpaceHierarchyRoomsChunk { + fn from(value: CachedSpaceHierarchySummary) -> Self { + let SpaceHierarchyParentSummary { + canonical_alias, + name, + num_joined_members, + room_id, + topic, + world_readable, + guest_can_join, + avatar_url, + join_rule, + room_type, + children_state, + .. + } = value.summary; + + Self { + canonical_alias, + name, + num_joined_members, + room_id, + topic, + world_readable, + guest_can_join, + avatar_url, + join_rule, + room_type, + children_state, + } } } @@ -736,3 +750,14 @@ fn get_parent_children_via( }) .collect() } + +fn next_room_to_traverse( + stack: &mut Vec)>>, parents: &mut VecDeque, +) -> Option<(OwnedRoomId, Vec)> { + while stack.last().map_or(false, Vec::is_empty) { + stack.pop(); + parents.pop_back(); + } + + stack.last_mut().and_then(Vec::pop) +} diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index a3a317a5..cb219bc0 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -8,7 +8,7 @@ use std::{ use conduit::{ utils::{calculate_hash, MutexMap, MutexMapGuard}, - warn, Error, Result, + warn, Error, PduEvent, Result, }; use data::Data; use ruma::{ @@ -23,19 +23,39 @@ use ruma::{ }; use super::state_compressor::CompressedStateEvent; -use crate::{services, PduEvent}; +use crate::{globals, rooms, Dep}; pub struct Service { + services: Services, db: Data, pub mutex: RoomMutexMap, } +struct Services { + globals: Dep, + short: Dep, + spaces: Dep, + state_cache: Dep, + state_accessor: Dep, + state_compressor: Dep, + timeline: Dep, +} + type RoomMutexMap = MutexMap; pub type RoomMutexGuard = MutexMapGuard; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + services: Services { + globals: args.depend::("globals"), + short: args.depend::("rooms::short"), + spaces: args.depend::("rooms::spaces"), + state_cache: args.depend::("rooms::state_cache"), + state_accessor: args.depend::("rooms::state_accessor"), + state_compressor: args.depend::("rooms::state_compressor"), + timeline: args.depend::("rooms::timeline"), + }, db: Data::new(args.db), mutex: RoomMutexMap::new(), })) @@ -62,14 +82,13 @@ impl Service { state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { for event_id in statediffnew.iter().filter_map(|new| { - services() - .rooms + self.services .state_compressor .parse_compressed_state_event(new) .ok() .map(|(_, id)| id) }) { - let Some(pdu) = services().rooms.timeline.get_pdu_json(&event_id)? else { + let Some(pdu) = self.services.timeline.get_pdu_json(&event_id)? else { continue; }; @@ -94,7 +113,7 @@ impl Service { continue; }; - services().rooms.state_cache.update_membership( + self.services.state_cache.update_membership( room_id, &user_id, membership_event, @@ -105,8 +124,7 @@ impl Service { )?; }, TimelineEventType::SpaceChild => { - services() - .rooms + self.services .spaces .roomid_spacehierarchy_cache .lock() @@ -117,7 +135,7 @@ impl Service { } } - services().rooms.state_cache.update_joined_count(room_id)?; + self.services.state_cache.update_joined_count(room_id)?; self.db .set_room_state(room_id, shortstatehash, state_lock)?; @@ -133,10 +151,7 @@ impl Service { pub fn set_event_state( &self, event_id: &EventId, room_id: &RoomId, state_ids_compressed: Arc>, ) -> Result { - let shorteventid = services() - .rooms - .short - .get_or_create_shorteventid(event_id)?; + let shorteventid = self.services.short.get_or_create_shorteventid(event_id)?; let previous_shortstatehash = self.db.get_room_shortstatehash(room_id)?; @@ -147,20 +162,15 @@ impl Service { .collect::>(), ); - let (shortstatehash, already_existed) = services() - .rooms + let (shortstatehash, already_existed) = self + .services .short .get_or_create_shortstatehash(&state_hash)?; if !already_existed { let states_parents = previous_shortstatehash.map_or_else( || Ok(Vec::new()), - |p| { - services() - .rooms - .state_compressor - .load_shortstatehash_info(p) - }, + |p| self.services.state_compressor.load_shortstatehash_info(p), )?; let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { @@ -179,7 +189,7 @@ impl Service { } else { (state_ids_compressed, Arc::new(HashSet::new())) }; - services().rooms.state_compressor.save_state_from_diff( + self.services.state_compressor.save_state_from_diff( shortstatehash, statediffnew, statediffremoved, @@ -199,8 +209,8 @@ impl Service { /// to `stateid_pduid` and adds the incoming event to `eventid_statehash`. #[tracing::instrument(skip(self, new_pdu), level = "debug")] pub fn append_to_state(&self, new_pdu: &PduEvent) -> Result { - let shorteventid = services() - .rooms + let shorteventid = self + .services .short .get_or_create_shorteventid(&new_pdu.event_id)?; @@ -214,21 +224,16 @@ impl Service { let states_parents = previous_shortstatehash.map_or_else( || Ok(Vec::new()), #[inline] - |p| { - services() - .rooms - .state_compressor - .load_shortstatehash_info(p) - }, + |p| self.services.state_compressor.load_shortstatehash_info(p), )?; - let shortstatekey = services() - .rooms + let shortstatekey = self + .services .short .get_or_create_shortstatekey(&new_pdu.kind.to_string().into(), state_key)?; - let new = services() - .rooms + let new = self + .services .state_compressor .compress_state_event(shortstatekey, &new_pdu.event_id)?; @@ -246,7 +251,7 @@ impl Service { } // TODO: statehash with deterministic inputs - let shortstatehash = services().globals.next_count()?; + let shortstatehash = self.services.globals.next_count()?; let mut statediffnew = HashSet::new(); statediffnew.insert(new); @@ -256,7 +261,7 @@ impl Service { statediffremoved.insert(*replaces); } - services().rooms.state_compressor.save_state_from_diff( + self.services.state_compressor.save_state_from_diff( shortstatehash, Arc::new(statediffnew), Arc::new(statediffremoved), @@ -275,22 +280,20 @@ impl Service { let mut state = Vec::new(); // Add recommended events if let Some(e) = - services() - .rooms + self.services .state_accessor .room_state_get(&invite_event.room_id, &StateEventType::RoomCreate, "")? { state.push(e.to_stripped_state_event()); } if let Some(e) = - services() - .rooms + self.services .state_accessor .room_state_get(&invite_event.room_id, &StateEventType::RoomJoinRules, "")? { state.push(e.to_stripped_state_event()); } - if let Some(e) = services().rooms.state_accessor.room_state_get( + if let Some(e) = self.services.state_accessor.room_state_get( &invite_event.room_id, &StateEventType::RoomCanonicalAlias, "", @@ -298,22 +301,20 @@ impl Service { state.push(e.to_stripped_state_event()); } if let Some(e) = - services() - .rooms + self.services .state_accessor .room_state_get(&invite_event.room_id, &StateEventType::RoomAvatar, "")? { state.push(e.to_stripped_state_event()); } if let Some(e) = - services() - .rooms + self.services .state_accessor .room_state_get(&invite_event.room_id, &StateEventType::RoomName, "")? { state.push(e.to_stripped_state_event()); } - if let Some(e) = services().rooms.state_accessor.room_state_get( + if let Some(e) = self.services.state_accessor.room_state_get( &invite_event.room_id, &StateEventType::RoomMember, invite_event.sender.as_str(), @@ -339,8 +340,8 @@ impl Service { /// Returns the room's version. #[tracing::instrument(skip(self), level = "debug")] pub fn get_room_version(&self, room_id: &RoomId) -> Result { - let create_event = services() - .rooms + let create_event = self + .services .state_accessor .room_state_get(room_id, &StateEventType::RoomCreate, "")?; @@ -393,8 +394,7 @@ impl Service { let mut sauthevents = auth_events .into_iter() .filter_map(|(event_type, state_key)| { - services() - .rooms + self.services .short .get_shortstatekey(&event_type.to_string().into(), &state_key) .ok() @@ -403,8 +403,8 @@ impl Service { }) .collect::>(); - let full_state = services() - .rooms + let full_state = self + .services .state_compressor .load_shortstatehash_info(shortstatehash)? .pop() @@ -414,16 +414,14 @@ impl Service { Ok(full_state .iter() .filter_map(|compressed| { - services() - .rooms + self.services .state_compressor .parse_compressed_state_event(compressed) .ok() }) .filter_map(|(shortstatekey, event_id)| sauthevents.remove(&shortstatekey).map(|k| (k, event_id))) .filter_map(|(k, event_id)| { - services() - .rooms + self.services .timeline .get_pdu(&event_id) .ok() diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 7e9daeda..4c85148d 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,28 +1,43 @@ use std::{collections::HashMap, sync::Arc}; -use conduit::{utils, Error, Result}; -use database::{Database, Map}; +use conduit::{utils, Error, PduEvent, Result}; +use database::Map; use ruma::{events::StateEventType, EventId, RoomId}; -use crate::{services, PduEvent}; +use crate::{rooms, Dep}; pub(super) struct Data { eventid_shorteventid: Arc, shorteventid_shortstatehash: Arc, + services: Services, +} + +struct Services { + short: Dep, + state: Dep, + state_compressor: Dep, + timeline: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { eventid_shorteventid: db["eventid_shorteventid"].clone(), shorteventid_shortstatehash: db["shorteventid_shortstatehash"].clone(), + services: Services { + short: args.depend::("rooms::short"), + state: args.depend::("rooms::state"), + state_compressor: args.depend::("rooms::state_compressor"), + timeline: args.depend::("rooms::timeline"), + }, } } #[allow(unused_qualifications)] // async traits pub(super) async fn state_full_ids(&self, shortstatehash: u64) -> Result>> { - let full_state = services() - .rooms + let full_state = self + .services .state_compressor .load_shortstatehash_info(shortstatehash)? .pop() @@ -31,8 +46,8 @@ impl Data { let mut result = HashMap::new(); let mut i: u8 = 0; for compressed in full_state.iter() { - let parsed = services() - .rooms + let parsed = self + .services .state_compressor .parse_compressed_state_event(compressed)?; result.insert(parsed.0, parsed.1); @@ -49,8 +64,8 @@ impl Data { pub(super) async fn state_full( &self, shortstatehash: u64, ) -> Result>> { - let full_state = services() - .rooms + let full_state = self + .services .state_compressor .load_shortstatehash_info(shortstatehash)? .pop() @@ -60,11 +75,11 @@ impl Data { let mut result = HashMap::new(); let mut i: u8 = 0; for compressed in full_state.iter() { - let (_, eventid) = services() - .rooms + let (_, eventid) = self + .services .state_compressor .parse_compressed_state_event(compressed)?; - if let Some(pdu) = services().rooms.timeline.get_pdu(&eventid)? { + if let Some(pdu) = self.services.timeline.get_pdu(&eventid)? { result.insert( ( pdu.kind.to_string().into(), @@ -92,15 +107,15 @@ impl Data { pub(super) fn state_get_id( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, ) -> Result>> { - let Some(shortstatekey) = services() - .rooms + let Some(shortstatekey) = self + .services .short .get_shortstatekey(event_type, state_key)? else { return Ok(None); }; - let full_state = services() - .rooms + let full_state = self + .services .state_compressor .load_shortstatehash_info(shortstatehash)? .pop() @@ -110,8 +125,7 @@ impl Data { .iter() .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) .and_then(|compressed| { - services() - .rooms + self.services .state_compressor .parse_compressed_state_event(compressed) .ok() @@ -125,7 +139,7 @@ impl Data { &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, ) -> Result>> { self.state_get_id(shortstatehash, event_type, state_key)? - .map_or(Ok(None), |event_id| services().rooms.timeline.get_pdu(&event_id)) + .map_or(Ok(None), |event_id| self.services.timeline.get_pdu(&event_id)) } /// Returns the state hash for this pdu. @@ -149,7 +163,7 @@ impl Data { pub(super) async fn room_state_full( &self, room_id: &RoomId, ) -> Result>> { - if let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? { + if let Some(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id)? { self.state_full(current_shortstatehash).await } else { Ok(HashMap::new()) @@ -161,7 +175,7 @@ impl Data { pub(super) fn room_state_get_id( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result>> { - if let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? { + if let Some(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id)? { self.state_get_id(current_shortstatehash, event_type, state_key) } else { Ok(None) @@ -173,7 +187,7 @@ impl Data { pub(super) fn room_state_get( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result>> { - if let Some(current_shortstatehash) = services().rooms.state.get_room_shortstatehash(room_id)? { + if let Some(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id)? { self.state_get(current_shortstatehash, event_type, state_key) } else { Ok(None) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index bd3eb0a1..2526f1bd 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -6,7 +6,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{err, error, utils::math::usize_from_f64, warn, Error, Result}; +use conduit::{err, error, pdu::PduBuilder, utils::math::usize_from_f64, warn, Error, PduEvent, Result}; use data::Data; use lru_cache::LruCache; use ruma::{ @@ -33,14 +33,20 @@ use ruma::{ }; use serde_json::value::to_raw_value; -use crate::{pdu::PduBuilder, rooms::state::RoomMutexGuard, services, PduEvent}; +use crate::{rooms, rooms::state::RoomMutexGuard, Dep}; pub struct Service { + services: Services, db: Data, pub server_visibility_cache: Mutex>, pub user_visibility_cache: Mutex>, } +struct Services { + state_cache: Dep, + timeline: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; @@ -50,7 +56,11 @@ impl crate::Service for Service { f64::from(config.user_visibility_cache_capacity) * config.cache_capacity_modifier; Ok(Arc::new(Self { - db: Data::new(args.db), + services: Services { + state_cache: args.depend::("rooms::state_cache"), + timeline: args.depend::("rooms::timeline"), + }, + db: Data::new(&args), server_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64(server_visibility_cache_capacity)?)), user_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64(user_visibility_cache_capacity)?)), })) @@ -164,8 +174,8 @@ impl Service { }) .unwrap_or(HistoryVisibility::Shared); - let mut current_server_members = services() - .rooms + let mut current_server_members = self + .services .state_cache .room_members(room_id) .filter_map(Result::ok) @@ -212,7 +222,7 @@ impl Service { return Ok(*visibility); } - let currently_member = services().rooms.state_cache.is_joined(user_id, room_id)?; + let currently_member = self.services.state_cache.is_joined(user_id, room_id)?; let history_visibility = self .state_get(shortstatehash, &StateEventType::RoomHistoryVisibility, "")? @@ -258,7 +268,7 @@ impl Service { /// the room's history_visibility at that event's state. #[tracing::instrument(skip(self, user_id, room_id))] pub fn user_can_see_state_events(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let currently_member = services().rooms.state_cache.is_joined(user_id, room_id)?; + let currently_member = self.services.state_cache.is_joined(user_id, room_id)?; let history_visibility = self .room_state_get(room_id, &StateEventType::RoomHistoryVisibility, "")? @@ -342,8 +352,8 @@ impl Service { redacts: None, }; - Ok(services() - .rooms + Ok(self + .services .timeline .create_hash_and_sign_event(new_event, sender, room_id, state_lock) .is_ok()) @@ -413,7 +423,7 @@ impl Service { // Falling back on m.room.create to judge power level if let Some(pdu) = self.room_state_get(room_id, &StateEventType::RoomCreate, "")? { Ok(pdu.sender == sender - || if let Ok(Some(pdu)) = services().rooms.timeline.get_pdu(redacts) { + || if let Ok(Some(pdu)) = self.services.timeline.get_pdu(redacts) { pdu.sender == sender } else { false @@ -430,7 +440,7 @@ impl Service { .map(|event: RoomPowerLevels| { event.user_can_redact_event_of_other(sender) || event.user_can_redact_own_event(sender) - && if let Ok(Some(pdu)) = services().rooms.timeline.get_pdu(redacts) { + && if let Ok(Some(pdu)) = self.services.timeline.get_pdu(redacts) { if federation { pdu.sender.server_name() == sender.server_name() } else { diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 2b9fbe94..cbda73cf 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -4,7 +4,7 @@ use std::{ }; use conduit::{utils, Error, Result}; -use database::{Database, Map}; +use database::Map; use itertools::Itertools; use ruma::{ events::{AnyStrippedStateEvent, AnySyncStateEvent}, @@ -12,44 +12,55 @@ use ruma::{ OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; -use crate::{appservice::RegistrationInfo, services, user_is_local}; +use crate::{appservice::RegistrationInfo, globals, user_is_local, users, Dep}; type StrippedStateEventIter<'a> = Box>)>> + 'a>; type AnySyncStateEventIter<'a> = Box>)>> + 'a>; type AppServiceInRoomCache = RwLock>>; pub(super) struct Data { - userroomid_joined: Arc, - roomuserid_joined: Arc, - userroomid_invitestate: Arc, - roomuserid_invitecount: Arc, - userroomid_leftstate: Arc, - roomuserid_leftcount: Arc, - roomid_inviteviaservers: Arc, - roomuseroncejoinedids: Arc, - roomid_joinedcount: Arc, - roomid_invitedcount: Arc, - roomserverids: Arc, - serverroomids: Arc, pub(super) appservice_in_room_cache: AppServiceInRoomCache, + roomid_invitedcount: Arc, + roomid_inviteviaservers: Arc, + roomid_joinedcount: Arc, + roomserverids: Arc, + roomuserid_invitecount: Arc, + roomuserid_joined: Arc, + roomuserid_leftcount: Arc, + roomuseroncejoinedids: Arc, + serverroomids: Arc, + userroomid_invitestate: Arc, + userroomid_joined: Arc, + userroomid_leftstate: Arc, + services: Services, +} + +struct Services { + globals: Dep, + users: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { - userroomid_joined: db["userroomid_joined"].clone(), - roomuserid_joined: db["roomuserid_joined"].clone(), - userroomid_invitestate: db["userroomid_invitestate"].clone(), - roomuserid_invitecount: db["roomuserid_invitecount"].clone(), - userroomid_leftstate: db["userroomid_leftstate"].clone(), - roomuserid_leftcount: db["roomuserid_leftcount"].clone(), - roomid_inviteviaservers: db["roomid_inviteviaservers"].clone(), - roomuseroncejoinedids: db["roomuseroncejoinedids"].clone(), - roomid_joinedcount: db["roomid_joinedcount"].clone(), - roomid_invitedcount: db["roomid_invitedcount"].clone(), - roomserverids: db["roomserverids"].clone(), - serverroomids: db["serverroomids"].clone(), appservice_in_room_cache: RwLock::new(HashMap::new()), + roomid_invitedcount: db["roomid_invitedcount"].clone(), + roomid_inviteviaservers: db["roomid_inviteviaservers"].clone(), + roomid_joinedcount: db["roomid_joinedcount"].clone(), + roomserverids: db["roomserverids"].clone(), + roomuserid_invitecount: db["roomuserid_invitecount"].clone(), + roomuserid_joined: db["roomuserid_joined"].clone(), + roomuserid_leftcount: db["roomuserid_leftcount"].clone(), + roomuseroncejoinedids: db["roomuseroncejoinedids"].clone(), + serverroomids: db["serverroomids"].clone(), + userroomid_invitestate: db["userroomid_invitestate"].clone(), + userroomid_joined: db["userroomid_joined"].clone(), + userroomid_leftstate: db["userroomid_leftstate"].clone(), + services: Services { + globals: args.depend::("globals"), + users: args.depend::("users"), + }, } } @@ -100,7 +111,7 @@ impl Data { &serde_json::to_vec(&last_state.unwrap_or_default()).expect("state to bytes always works"), )?; self.roomuserid_invitecount - .insert(&roomuser_id, &services().globals.next_count()?.to_be_bytes())?; + .insert(&roomuser_id, &self.services.globals.next_count()?.to_be_bytes())?; self.userroomid_joined.remove(&userroom_id)?; self.roomuserid_joined.remove(&roomuser_id)?; self.userroomid_leftstate.remove(&userroom_id)?; @@ -144,7 +155,7 @@ impl Data { &serde_json::to_vec(&Vec::>::new()).unwrap(), )?; // TODO self.roomuserid_leftcount - .insert(&roomuser_id, &services().globals.next_count()?.to_be_bytes())?; + .insert(&roomuser_id, &self.services.globals.next_count()?.to_be_bytes())?; self.userroomid_joined.remove(&userroom_id)?; self.roomuserid_joined.remove(&roomuser_id)?; self.userroomid_invitestate.remove(&userroom_id)?; @@ -228,7 +239,7 @@ impl Data { } else { let bridge_user_id = UserId::parse_with_server_name( appservice.registration.sender_localpart.as_str(), - services().globals.server_name(), + self.services.globals.server_name(), ) .ok(); @@ -356,7 +367,7 @@ impl Data { ) -> Box + 'a> { Box::new( self.local_users_in_room(room_id) - .filter(|user| !services().users.is_deactivated(user).unwrap_or(true)), + .filter(|user| !self.services.users.is_deactivated(user).unwrap_or(true)), ) } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 48215817..ac2f688e 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -21,16 +21,28 @@ use ruma::{ OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; -use crate::{appservice::RegistrationInfo, services, user_is_local}; +use crate::{account_data, appservice::RegistrationInfo, rooms, user_is_local, users, Dep}; pub struct Service { + services: Services, db: Data, } +struct Services { + account_data: Dep, + state_accessor: Dep, + users: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + services: Services { + account_data: args.depend::("account_data"), + state_accessor: args.depend::("rooms::state_accessor"), + users: args.depend::("users"), + }, + db: Data::new(&args), })) } @@ -54,18 +66,18 @@ impl Service { // update #[allow(clippy::collapsible_if)] if !user_is_local(user_id) { - if !services().users.exists(user_id)? { - services().users.create(user_id, None)?; + if !self.services.users.exists(user_id)? { + self.services.users.create(user_id, None)?; } /* // Try to update our local copy of the user if ours does not match - if ((services().users.displayname(user_id)? != membership_event.displayname) - || (services().users.avatar_url(user_id)? != membership_event.avatar_url) - || (services().users.blurhash(user_id)? != membership_event.blurhash)) + if ((self.services.users.displayname(user_id)? != membership_event.displayname) + || (self.services.users.avatar_url(user_id)? != membership_event.avatar_url) + || (self.services.users.blurhash(user_id)? != membership_event.blurhash)) && (membership != MembershipState::Leave) { - let response = services() + let response = self.services .sending .send_federation_request( user_id.server_name(), @@ -76,9 +88,9 @@ impl Service { ) .await; - services().users.set_displayname(user_id, response.displayname.clone()).await?; - services().users.set_avatar_url(user_id, response.avatar_url).await?; - services().users.set_blurhash(user_id, response.blurhash).await?; + self.services.users.set_displayname(user_id, response.displayname.clone()).await?; + self.services.users.set_avatar_url(user_id, response.avatar_url).await?; + self.services.users.set_blurhash(user_id, response.blurhash).await?; }; */ } @@ -91,8 +103,8 @@ impl Service { self.db.mark_as_once_joined(user_id, room_id)?; // Check if the room has a predecessor - if let Some(predecessor) = services() - .rooms + if let Some(predecessor) = self + .services .state_accessor .room_state_get(room_id, &StateEventType::RoomCreate, "")? .and_then(|create| serde_json::from_str(create.content.get()).ok()) @@ -124,21 +136,23 @@ impl Service { // .ok(); // Copy old tags to new room - if let Some(tag_event) = services() + if let Some(tag_event) = self + .services .account_data .get(Some(&predecessor.room_id), user_id, RoomAccountDataEventType::Tag)? .map(|event| { serde_json::from_str(event.get()) .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) { - services() + self.services .account_data .update(Some(room_id), user_id, RoomAccountDataEventType::Tag, &tag_event?) .ok(); }; // Copy direct chat flag - if let Some(direct_event) = services() + if let Some(direct_event) = self + .services .account_data .get(None, user_id, GlobalAccountDataEventType::Direct.to_string().into())? .map(|event| { @@ -156,7 +170,7 @@ impl Service { } if room_ids_updated { - services().account_data.update( + self.services.account_data.update( None, user_id, GlobalAccountDataEventType::Direct.to_string().into(), @@ -171,7 +185,8 @@ impl Service { }, MembershipState::Invite => { // We want to know if the sender is ignored by the receiver - let is_ignored = services() + let is_ignored = self + .services .account_data .get( None, // Ignored users are in global account data @@ -393,8 +408,8 @@ impl Service { /// See #[tracing::instrument(skip(self))] pub fn servers_route_via(&self, room_id: &RoomId) -> Result> { - let most_powerful_user_server = services() - .rooms + let most_powerful_user_server = self + .services .state_accessor .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? .map(|pdu| { diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 422c562b..2550774e 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -13,7 +13,7 @@ use lru_cache::LruCache; use ruma::{EventId, RoomId}; use self::data::StateDiff; -use crate::services; +use crate::{rooms, Dep}; type StateInfoLruCache = Mutex< LruCache< @@ -48,16 +48,25 @@ pub type CompressedStateEvent = [u8; 2 * size_of::()]; pub struct Service { db: Data, - + services: Services, pub stateinfo_cache: StateInfoLruCache, } +struct Services { + short: Dep, + state: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; let cache_capacity = f64::from(config.stateinfo_cache_capacity) * config.cache_capacity_modifier; Ok(Arc::new(Self { db: Data::new(args.db), + services: Services { + short: args.depend::("rooms::short"), + state: args.depend::("rooms::state"), + }, stateinfo_cache: StdMutex::new(LruCache::new(usize_from_f64(cache_capacity)?)), })) } @@ -124,8 +133,8 @@ impl Service { pub fn compress_state_event(&self, shortstatekey: u64, event_id: &EventId) -> Result { let mut v = shortstatekey.to_be_bytes().to_vec(); v.extend_from_slice( - &services() - .rooms + &self + .services .short .get_or_create_shorteventid(event_id)? .to_be_bytes(), @@ -138,7 +147,7 @@ impl Service { pub fn parse_compressed_state_event(&self, compressed_event: &CompressedStateEvent) -> Result<(u64, Arc)> { Ok(( utils::u64_from_bytes(&compressed_event[0..size_of::()]).expect("bytes have right length"), - services().rooms.short.get_eventid_from_short( + self.services.short.get_eventid_from_short( utils::u64_from_bytes(&compressed_event[size_of::()..]).expect("bytes have right length"), )?, )) @@ -282,7 +291,7 @@ impl Service { pub fn save_state( &self, room_id: &RoomId, new_state_ids_compressed: Arc>, ) -> HashSetCompressStateEvent { - let previous_shortstatehash = services().rooms.state.get_room_shortstatehash(room_id)?; + let previous_shortstatehash = self.services.state.get_room_shortstatehash(room_id)?; let state_hash = utils::calculate_hash( &new_state_ids_compressed @@ -291,8 +300,8 @@ impl Service { .collect::>(), ); - let (new_shortstatehash, already_existed) = services() - .rooms + let (new_shortstatehash, already_existed) = self + .services .short .get_or_create_shortstatehash(&state_hash)?; diff --git a/src/service/rooms/threads/data.rs b/src/service/rooms/threads/data.rs index c4a1a294..fb279a00 100644 --- a/src/service/rooms/threads/data.rs +++ b/src/service/rooms/threads/data.rs @@ -1,29 +1,40 @@ use std::{mem::size_of, sync::Arc}; -use conduit::{checked, utils, Error, Result}; -use database::{Database, Map}; +use conduit::{checked, utils, Error, PduEvent, Result}; +use database::Map; use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId}; -use crate::{services, PduEvent}; +use crate::{rooms, Dep}; type PduEventIterResult<'a> = Result> + 'a>>; pub(super) struct Data { threadid_userids: Arc, + services: Services, +} + +struct Services { + short: Dep, + timeline: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { threadid_userids: db["threadid_userids"].clone(), + services: Services { + short: args.depend::("rooms::short"), + timeline: args.depend::("rooms::timeline"), + }, } } pub(super) fn threads_until<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, _include: &'a IncludeThreads, ) -> PduEventIterResult<'a> { - let prefix = services() - .rooms + let prefix = self + .services .short .get_shortroomid(room_id)? .expect("room exists") @@ -40,8 +51,8 @@ impl Data { .map(move |(pduid, _users)| { let count = utils::u64_from_bytes(&pduid[(size_of::())..]) .map_err(|_| Error::bad_database("Invalid pduid in threadid_userids."))?; - let mut pdu = services() - .rooms + let mut pdu = self + .services .timeline .get_pdu_from_id(&pduid)? .ok_or_else(|| Error::bad_database("Invalid pduid reference in threadid_userids"))?; diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index dd2686b0..ae51cd0f 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -2,7 +2,7 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{Error, Result}; +use conduit::{Error, PduEvent, Result}; use data::Data; use ruma::{ api::client::{error::ErrorKind, threads::get_threads::v1::IncludeThreads}, @@ -11,16 +11,24 @@ use ruma::{ }; use serde_json::json; -use crate::{services, PduEvent}; +use crate::{rooms, Dep}; pub struct Service { + services: Services, db: Data, } +struct Services { + timeline: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + services: Services { + timeline: args.depend::("rooms::timeline"), + }, + db: Data::new(&args), })) } @@ -35,22 +43,22 @@ impl Service { } pub fn add_to_thread(&self, root_event_id: &EventId, pdu: &PduEvent) -> Result<()> { - let root_id = &services() - .rooms + let root_id = self + .services .timeline .get_pdu_id(root_event_id)? .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Invalid event id in thread message"))?; - let root_pdu = services() - .rooms + let root_pdu = self + .services .timeline - .get_pdu_from_id(root_id)? + .get_pdu_from_id(&root_id)? .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Thread root pdu not found"))?; - let mut root_pdu_json = services() - .rooms + let mut root_pdu_json = self + .services .timeline - .get_pdu_json_from_id(root_id)? + .get_pdu_json_from_id(&root_id)? .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Thread root pdu not found"))?; if let CanonicalJsonValue::Object(unsigned) = root_pdu_json @@ -93,20 +101,19 @@ impl Service { ); } - services() - .rooms + self.services .timeline - .replace_pdu(root_id, &root_pdu_json, &root_pdu)?; + .replace_pdu(&root_id, &root_pdu_json, &root_pdu)?; } let mut users = Vec::new(); - if let Some(userids) = self.db.get_participants(root_id)? { + if let Some(userids) = self.db.get_participants(&root_id)? { users.extend_from_slice(&userids); } else { users.push(root_pdu.sender); } users.push(pdu.sender.clone()); - self.db.update_participants(root_id, &users) + self.db.update_participants(&root_id, &users) } } diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index ec975b99..5917e96b 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -4,19 +4,25 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{checked, error, utils, Error, Result}; +use conduit::{checked, error, utils, Error, PduCount, PduEvent, Result}; use database::{Database, Map}; use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; -use crate::{services, PduCount, PduEvent}; +use crate::{rooms, Dep}; pub(super) struct Data { + eventid_outlierpdu: Arc, eventid_pduid: Arc, pduid_pdu: Arc, - eventid_outlierpdu: Arc, - userroomid_notificationcount: Arc, userroomid_highlightcount: Arc, + userroomid_notificationcount: Arc, pub(super) lasttimelinecount_cache: LastTimelineCountCache, + pub(super) db: Arc, + services: Services, +} + +struct Services { + short: Dep, } type PdusIterItem = Result<(PduCount, PduEvent)>; @@ -24,14 +30,19 @@ type PdusIterator<'a> = Box + 'a>; type LastTimelineCountCache = Mutex>; impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { + eventid_outlierpdu: db["eventid_outlierpdu"].clone(), eventid_pduid: db["eventid_pduid"].clone(), pduid_pdu: db["pduid_pdu"].clone(), - eventid_outlierpdu: db["eventid_outlierpdu"].clone(), - userroomid_notificationcount: db["userroomid_notificationcount"].clone(), userroomid_highlightcount: db["userroomid_highlightcount"].clone(), + userroomid_notificationcount: db["userroomid_notificationcount"].clone(), lasttimelinecount_cache: Mutex::new(HashMap::new()), + db: args.db.clone(), + services: Services { + short: args.depend::("rooms::short"), + }, } } @@ -210,7 +221,7 @@ impl Data { /// happened before the event with id `until` in reverse-chronological /// order. pub(super) fn pdus_until(&self, user_id: &UserId, room_id: &RoomId, until: PduCount) -> Result> { - let (prefix, current) = count_to_id(room_id, until, 1, true)?; + let (prefix, current) = self.count_to_id(room_id, until, 1, true)?; let user_id = user_id.to_owned(); @@ -232,7 +243,7 @@ impl Data { } pub(super) fn pdus_after(&self, user_id: &UserId, room_id: &RoomId, from: PduCount) -> Result> { - let (prefix, current) = count_to_id(room_id, from, 1, false)?; + let (prefix, current) = self.count_to_id(room_id, from, 1, false)?; let user_id = user_id.to_owned(); @@ -277,6 +288,41 @@ impl Data { .increment_batch(highlights_batch.iter().map(Vec::as_slice))?; Ok(()) } + + pub(super) fn count_to_id( + &self, room_id: &RoomId, count: PduCount, offset: u64, subtract: bool, + ) -> Result<(Vec, Vec)> { + let prefix = self + .services + .short + .get_shortroomid(room_id)? + .ok_or_else(|| Error::bad_database("Looked for bad shortroomid in timeline"))? + .to_be_bytes() + .to_vec(); + let mut pdu_id = prefix.clone(); + // +1 so we don't send the base event + let count_raw = match count { + PduCount::Normal(x) => { + if subtract { + x.saturating_sub(offset) + } else { + x.saturating_add(offset) + } + }, + PduCount::Backfilled(x) => { + pdu_id.extend_from_slice(&0_u64.to_be_bytes()); + let num = u64::MAX.saturating_sub(x); + if subtract { + num.saturating_sub(offset) + } else { + num.saturating_add(offset) + } + }, + }; + pdu_id.extend_from_slice(&count_raw.to_be_bytes()); + + Ok((prefix, pdu_id)) + } } /// Returns the `count` of this pdu's id. @@ -294,38 +340,3 @@ pub(super) fn pdu_count(pdu_id: &[u8]) -> Result { Ok(PduCount::Normal(last_u64)) } } - -pub(super) fn count_to_id( - room_id: &RoomId, count: PduCount, offset: u64, subtract: bool, -) -> Result<(Vec, Vec)> { - let prefix = services() - .rooms - .short - .get_shortroomid(room_id)? - .ok_or_else(|| Error::bad_database("Looked for bad shortroomid in timeline"))? - .to_be_bytes() - .to_vec(); - let mut pdu_id = prefix.clone(); - // +1 so we don't send the base event - let count_raw = match count { - PduCount::Normal(x) => { - if subtract { - x.saturating_sub(offset) - } else { - x.saturating_add(offset) - } - }, - PduCount::Backfilled(x) => { - pdu_id.extend_from_slice(&0_u64.to_be_bytes()); - let num = u64::MAX.saturating_sub(x); - if subtract { - num.saturating_sub(offset) - } else { - num.saturating_add(offset) - } - }, - }; - pdu_id.extend_from_slice(&count_raw.to_be_bytes()); - - Ok((prefix, pdu_id)) -} diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 4c5e407a..50d29475 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -7,11 +7,12 @@ use std::{ }; use conduit::{ - debug, error, info, utils, + debug, error, info, + pdu::{EventHash, PduBuilder, PduCount, PduEvent}, + utils, utils::{MutexMap, MutexMapGuard}, - validated, warn, Error, Result, + validated, warn, Error, Result, Server, }; -use data::Data; use itertools::Itertools; use ruma::{ api::{client::error::ErrorKind, federation}, @@ -37,11 +38,10 @@ use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tokio::sync::RwLock; +use self::data::Data; use crate::{ - appservice::NamespaceRegex, - pdu::{EventHash, PduBuilder}, - rooms::{event_handler::parse_incoming_pdu, state_compressor::CompressedStateEvent}, - server_is_ours, services, PduCount, PduEvent, + account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms, + rooms::state_compressor::CompressedStateEvent, sending, server_is_ours, Dep, }; // Update Relationships @@ -67,17 +67,61 @@ struct ExtractBody { } pub struct Service { + services: Services, db: Data, pub mutex_insert: RoomMutexMap, } +struct Services { + server: Arc, + account_data: Dep, + appservice: Dep, + admin: Dep, + alias: Dep, + globals: Dep, + short: Dep, + state: Dep, + state_cache: Dep, + state_accessor: Dep, + pdu_metadata: Dep, + read_receipt: Dep, + sending: Dep, + user: Dep, + pusher: Dep, + threads: Dep, + search: Dep, + spaces: Dep, + event_handler: Dep, +} + type RoomMutexMap = MutexMap; pub type RoomMutexGuard = MutexMapGuard; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + services: Services { + server: args.server.clone(), + account_data: args.depend::("account_data"), + appservice: args.depend::("appservice"), + admin: args.depend::("admin"), + alias: args.depend::("rooms::alias"), + globals: args.depend::("globals"), + short: args.depend::("rooms::short"), + state: args.depend::("rooms::state"), + state_cache: args.depend::("rooms::state_cache"), + state_accessor: args.depend::("rooms::state_accessor"), + pdu_metadata: args.depend::("rooms::pdu_metadata"), + read_receipt: args.depend::("rooms::read_receipt"), + sending: args.depend::("sending"), + user: args.depend::("rooms::user"), + pusher: args.depend::("pusher"), + threads: args.depend::("rooms::threads"), + search: args.depend::("rooms::search"), + spaces: args.depend::("rooms::spaces"), + event_handler: args.depend::("rooms::event_handler"), + }, + db: Data::new(&args), mutex_insert: RoomMutexMap::new(), })) } @@ -217,10 +261,10 @@ impl Service { state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result> { // Coalesce database writes for the remainder of this scope. - let _cork = services().db.cork_and_flush(); + let _cork = self.db.db.cork_and_flush(); - let shortroomid = services() - .rooms + let shortroomid = self + .services .short .get_shortroomid(&pdu.room_id)? .expect("room exists"); @@ -233,14 +277,14 @@ impl Service { .entry("unsigned".to_owned()) .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::default())) { - if let Some(shortstatehash) = services() - .rooms + if let Some(shortstatehash) = self + .services .state_accessor .pdu_shortstatehash(&pdu.event_id) .unwrap() { - if let Some(prev_state) = services() - .rooms + if let Some(prev_state) = self + .services .state_accessor .state_get(shortstatehash, &pdu.kind.to_string().into(), state_key) .unwrap() @@ -270,30 +314,26 @@ impl Service { } // We must keep track of all events that have been referenced. - services() - .rooms + self.services .pdu_metadata .mark_as_referenced(&pdu.room_id, &pdu.prev_events)?; - services() - .rooms + self.services .state .set_forward_extremities(&pdu.room_id, leaves, state_lock)?; let insert_lock = self.mutex_insert.lock(&pdu.room_id).await; - let count1 = services().globals.next_count()?; + let count1 = self.services.globals.next_count()?; // Mark as read first so the sending client doesn't get a notification even if // appending fails - services() - .rooms + self.services .read_receipt .private_read_set(&pdu.room_id, &pdu.sender, count1)?; - services() - .rooms + self.services .user .reset_notification_counts(&pdu.sender, &pdu.room_id)?; - let count2 = services().globals.next_count()?; + let count2 = self.services.globals.next_count()?; let mut pdu_id = shortroomid.to_be_bytes().to_vec(); pdu_id.extend_from_slice(&count2.to_be_bytes()); @@ -303,8 +343,8 @@ impl Service { drop(insert_lock); // See if the event matches any known pushers - let power_levels: RoomPowerLevelsEventContent = services() - .rooms + let power_levels: RoomPowerLevelsEventContent = self + .services .state_accessor .room_state_get(&pdu.room_id, &StateEventType::RoomPowerLevels, "")? .map(|ev| { @@ -319,8 +359,8 @@ impl Service { let mut notifies = Vec::new(); let mut highlights = Vec::new(); - let mut push_target = services() - .rooms + let mut push_target = self + .services .state_cache .active_local_users_in_room(&pdu.room_id) .collect_vec(); @@ -341,7 +381,8 @@ impl Service { continue; } - let rules_for_user = services() + let rules_for_user = self + .services .account_data .get(None, user, GlobalAccountDataEventType::PushRules.to_string().into())? .map(|event| { @@ -357,7 +398,7 @@ impl Service { let mut notify = false; for action in - services() + self.services .pusher .get_actions(user, &rules_for_user, &power_levels, &sync_pdu, &pdu.room_id)? { @@ -378,8 +419,10 @@ impl Service { highlights.push(user.clone()); } - for push_key in services().pusher.get_pushkeys(user) { - services().sending.send_pdu_push(&pdu_id, user, push_key?)?; + for push_key in self.services.pusher.get_pushkeys(user) { + self.services + .sending + .send_pdu_push(&pdu_id, user, push_key?)?; } } @@ -390,11 +433,11 @@ impl Service { TimelineEventType::RoomRedaction => { use RoomVersionId::*; - let room_version_id = services().rooms.state.get_room_version(&pdu.room_id)?; + let room_version_id = self.services.state.get_room_version(&pdu.room_id)?; match room_version_id { V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { - if services().rooms.state_accessor.user_can_redact( + if self.services.state_accessor.user_can_redact( redact_id, &pdu.sender, &pdu.room_id, @@ -412,7 +455,7 @@ impl Service { })?; if let Some(redact_id) = &content.redacts { - if services().rooms.state_accessor.user_can_redact( + if self.services.state_accessor.user_can_redact( redact_id, &pdu.sender, &pdu.room_id, @@ -433,8 +476,7 @@ impl Service { }, TimelineEventType::SpaceChild => { if let Some(_state_key) = &pdu.state_key { - services() - .rooms + self.services .spaces .roomid_spacehierarchy_cache .lock() @@ -455,7 +497,7 @@ impl Service { let invite_state = match content.membership { MembershipState::Invite => { - let state = services().rooms.state.calculate_invite_state(pdu)?; + let state = self.services.state.calculate_invite_state(pdu)?; Some(state) }, _ => None, @@ -463,7 +505,7 @@ impl Service { // Update our membership info, we do this here incase a user is invited // and immediately leaves we need the DB to record the invite event for auth - services().rooms.state_cache.update_membership( + self.services.state_cache.update_membership( &pdu.room_id, &target_user_id, content, @@ -479,13 +521,12 @@ impl Service { .map_err(|_| Error::bad_database("Invalid content in pdu."))?; if let Some(body) = content.body { - services() - .rooms + self.services .search .index_pdu(shortroomid, &pdu_id, &body)?; - if services().admin.is_admin_command(pdu, &body).await { - services() + if self.services.admin.is_admin_command(pdu, &body).await { + self.services .admin .command(body, Some((*pdu.event_id).into())) .await; @@ -497,8 +538,7 @@ impl Service { if let Ok(content) = serde_json::from_str::(pdu.content.get()) { if let Some(related_pducount) = self.get_pdu_count(&content.relates_to.event_id)? { - services() - .rooms + self.services .pdu_metadata .add_relation(PduCount::Normal(count2), related_pducount)?; } @@ -512,29 +552,25 @@ impl Service { // We need to do it again here, because replies don't have // event_id as a top level field if let Some(related_pducount) = self.get_pdu_count(&in_reply_to.event_id)? { - services() - .rooms + self.services .pdu_metadata .add_relation(PduCount::Normal(count2), related_pducount)?; } }, Relation::Thread(thread) => { - services() - .rooms - .threads - .add_to_thread(&thread.event_id, pdu)?; + self.services.threads.add_to_thread(&thread.event_id, pdu)?; }, _ => {}, // TODO: Aggregate other types } } - for appservice in services().appservice.read().await.values() { - if services() - .rooms + for appservice in self.services.appservice.read().await.values() { + if self + .services .state_cache .appservice_in_room(&pdu.room_id, appservice)? { - services() + self.services .sending .send_pdu_appservice(appservice.registration.id.clone(), pdu_id.clone())?; continue; @@ -550,7 +586,7 @@ impl Service { { let appservice_uid = appservice.registration.sender_localpart.as_str(); if state_key_uid == appservice_uid { - services() + self.services .sending .send_pdu_appservice(appservice.registration.id.clone(), pdu_id.clone())?; continue; @@ -567,8 +603,7 @@ impl Service { .map_or(false, |state_key| users.is_match(state_key)) }; let matching_aliases = |aliases: &NamespaceRegex| { - services() - .rooms + self.services .alias .local_aliases_for_room(&pdu.room_id) .filter_map(Result::ok) @@ -579,7 +614,7 @@ impl Service { || appservice.rooms.is_match(pdu.room_id.as_str()) || matching_users(&appservice.users) { - services() + self.services .sending .send_pdu_appservice(appservice.registration.id.clone(), pdu_id.clone())?; } @@ -603,8 +638,8 @@ impl Service { redacts, } = pdu_builder; - let prev_events: Vec<_> = services() - .rooms + let prev_events: Vec<_> = self + .services .state .get_forward_extremities(room_id)? .into_iter() @@ -612,28 +647,23 @@ impl Service { .collect(); // If there was no create event yet, assume we are creating a room - let room_version_id = services() - .rooms - .state - .get_room_version(room_id) - .or_else(|_| { - if event_type == TimelineEventType::RoomCreate { - let content = serde_json::from_str::(content.get()) - .expect("Invalid content in RoomCreate pdu."); - Ok(content.room_version) - } else { - Err(Error::InconsistentRoomState( - "non-create event for room of unknown version", - room_id.to_owned(), - )) - } - })?; + let room_version_id = self.services.state.get_room_version(room_id).or_else(|_| { + if event_type == TimelineEventType::RoomCreate { + let content = serde_json::from_str::(content.get()) + .expect("Invalid content in RoomCreate pdu."); + Ok(content.room_version) + } else { + Err(Error::InconsistentRoomState( + "non-create event for room of unknown version", + room_id.to_owned(), + )) + } + })?; let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); let auth_events = - services() - .rooms + self.services .state .get_auth_events(room_id, &event_type, sender, state_key.as_deref(), &content)?; @@ -649,8 +679,7 @@ impl Service { if let Some(state_key) = &state_key { if let Some(prev_pdu) = - services() - .rooms + self.services .state_accessor .room_state_get(room_id, &event_type.to_string().into(), state_key)? { @@ -730,12 +759,12 @@ impl Service { // Add origin because synapse likes that (and it's required in the spec) pdu_json.insert( "origin".to_owned(), - to_canonical_value(services().globals.server_name()).expect("server name is a valid CanonicalJsonValue"), + to_canonical_value(self.services.globals.server_name()).expect("server name is a valid CanonicalJsonValue"), ); match ruma::signatures::hash_and_sign_event( - services().globals.server_name().as_str(), - services().globals.keypair(), + self.services.globals.server_name().as_str(), + self.services.globals.keypair(), &mut pdu_json, &room_version_id, ) { @@ -763,8 +792,8 @@ impl Service { ); // Generate short event id - let _shorteventid = services() - .rooms + let _shorteventid = self + .services .short .get_or_create_shorteventid(&pdu.event_id)?; @@ -783,7 +812,7 @@ impl Service { state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result> { let (pdu, pdu_json) = self.create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock)?; - if let Some(admin_room) = services().admin.get_admin_room()? { + if let Some(admin_room) = self.services.admin.get_admin_room()? { if admin_room == room_id { match pdu.event_type() { TimelineEventType::RoomEncryption => { @@ -798,7 +827,7 @@ impl Service { .state_key() .filter(|v| v.starts_with('@')) .unwrap_or(sender.as_str()); - let server_user = &services().globals.server_user.to_string(); + let server_user = &self.services.globals.server_user.to_string(); let content = serde_json::from_str::(pdu.content.get()) .map_err(|_| Error::bad_database("Invalid content in pdu"))?; @@ -812,8 +841,8 @@ impl Service { )); } - let count = services() - .rooms + let count = self + .services .state_cache .room_members(room_id) .filter_map(Result::ok) @@ -837,8 +866,8 @@ impl Service { )); } - let count = services() - .rooms + let count = self + .services .state_cache .room_members(room_id) .filter_map(Result::ok) @@ -861,15 +890,14 @@ impl Service { // If redaction event is not authorized, do not append it to the timeline if pdu.kind == TimelineEventType::RoomRedaction { use RoomVersionId::*; - match services().rooms.state.get_room_version(&pdu.room_id)? { + match self.services.state.get_room_version(&pdu.room_id)? { V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { - if !services().rooms.state_accessor.user_can_redact( - redact_id, - &pdu.sender, - &pdu.room_id, - false, - )? { + if !self + .services + .state_accessor + .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false)? + { return Err(Error::BadRequest(ErrorKind::forbidden(), "User cannot redact this event.")); } }; @@ -879,12 +907,11 @@ impl Service { .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; if let Some(redact_id) = &content.redacts { - if !services().rooms.state_accessor.user_can_redact( - redact_id, - &pdu.sender, - &pdu.room_id, - false, - )? { + if !self + .services + .state_accessor + .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false)? + { return Err(Error::BadRequest(ErrorKind::forbidden(), "User cannot redact this event.")); } } @@ -895,7 +922,7 @@ impl Service { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't // fail. - let statehashid = services().rooms.state.append_to_state(&pdu)?; + let statehashid = self.services.state.append_to_state(&pdu)?; let pdu_id = self .append_pdu( @@ -910,13 +937,12 @@ impl Service { // We set the room state after inserting the pdu, so that we never have a moment // in time where events in the current room state do not exist - services() - .rooms + self.services .state .set_room_state(room_id, statehashid, state_lock)?; - let mut servers: HashSet = services() - .rooms + let mut servers: HashSet = self + .services .state_cache .room_servers(room_id) .filter_map(Result::ok) @@ -936,9 +962,9 @@ impl Service { // Remove our server from the server list since it will be added to it by // room_servers() and/or the if statement above - servers.remove(services().globals.server_name()); + servers.remove(self.services.globals.server_name()); - services() + self.services .sending .send_pdu_servers(servers.into_iter(), &pdu_id)?; @@ -960,18 +986,15 @@ impl Service { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't // fail. - services() - .rooms + self.services .state .set_event_state(&pdu.event_id, &pdu.room_id, state_ids_compressed)?; if soft_fail { - services() - .rooms + self.services .pdu_metadata .mark_as_referenced(&pdu.room_id, &pdu.prev_events)?; - services() - .rooms + self.services .state .set_forward_extremities(&pdu.room_id, new_room_leaves, state_lock)?; return Ok(None); @@ -1022,14 +1045,13 @@ impl Service { if let Ok(content) = serde_json::from_str::(pdu.content.get()) { if let Some(body) = content.body { - services() - .rooms + self.services .search .deindex_pdu(shortroomid, &pdu_id, &body)?; } } - let room_version_id = services().rooms.state.get_room_version(&pdu.room_id)?; + let room_version_id = self.services.state.get_room_version(&pdu.room_id)?; pdu.redact(room_version_id, reason)?; @@ -1058,8 +1080,8 @@ impl Service { return Ok(()); } - let power_levels: RoomPowerLevelsEventContent = services() - .rooms + let power_levels: RoomPowerLevelsEventContent = self + .services .state_accessor .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? .map(|ev| { @@ -1077,8 +1099,8 @@ impl Service { } }); - let room_alias_servers = services() - .rooms + let room_alias_servers = self + .services .alias .local_aliases_for_room(room_id) .filter_map(|alias| { @@ -1090,14 +1112,13 @@ impl Service { let servers = room_mods .chain(room_alias_servers) - .chain(services().globals.config.trusted_servers.clone()) + .chain(self.services.server.config.trusted_servers.clone()) .filter(|server_name| { if server_is_ours(server_name) { return false; } - services() - .rooms + self.services .state_cache .server_in_room(server_name, room_id) .unwrap_or(false) @@ -1105,7 +1126,8 @@ impl Service { for backfill_server in servers { info!("Asking {backfill_server} for backfill"); - let response = services() + let response = self + .services .sending .send_federation_request( &backfill_server, @@ -1141,11 +1163,11 @@ impl Service { &self, origin: &ServerName, pdu: Box, pub_key_map: &RwLock>>, ) -> Result<()> { - let (event_id, value, room_id) = parse_incoming_pdu(&pdu)?; + let (event_id, value, room_id) = self.services.event_handler.parse_incoming_pdu(&pdu)?; // Lock so we cannot backfill the same pdu twice at the same time - let mutex_lock = services() - .rooms + let mutex_lock = self + .services .event_handler .mutex_federation .lock(&room_id) @@ -1158,14 +1180,12 @@ impl Service { return Ok(()); } - services() - .rooms + self.services .event_handler .fetch_required_signing_keys([&value], pub_key_map) .await?; - services() - .rooms + self.services .event_handler .handle_incoming_pdu(origin, &room_id, &event_id, value, false, pub_key_map) .await?; @@ -1173,8 +1193,8 @@ impl Service { let value = self.get_pdu_json(&event_id)?.expect("We just created it"); let pdu = self.get_pdu(&event_id)?.expect("We just created it"); - let shortroomid = services() - .rooms + let shortroomid = self + .services .short .get_shortroomid(&room_id)? .expect("room exists"); @@ -1182,7 +1202,7 @@ impl Service { let insert_lock = self.mutex_insert.lock(&room_id).await; let max = u64::MAX; - let count = services().globals.next_count()?; + let count = self.services.globals.next_count()?; let mut pdu_id = shortroomid.to_be_bytes().to_vec(); pdu_id.extend_from_slice(&0_u64.to_be_bytes()); pdu_id.extend_from_slice(&(validated!(max - count)?).to_be_bytes()); @@ -1197,8 +1217,7 @@ impl Service { .map_err(|_| Error::bad_database("Invalid content in pdu."))?; if let Some(body) = content.body { - services() - .rooms + self.services .search .index_pdu(shortroomid, &pdu_id, &body)?; } diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 715e3162..d863f217 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduit::{debug_info, trace, utils, Result}; +use conduit::{debug_info, trace, utils, Result, Server}; use ruma::{ api::federation::transactions::edu::{Edu, TypingContent}, events::SyncEphemeralRoomEvent, @@ -8,19 +8,31 @@ use ruma::{ }; use tokio::sync::{broadcast, RwLock}; -use crate::{services, user_is_local}; +use crate::{globals, sending, user_is_local, Dep}; pub struct Service { - pub typing: RwLock>>, // u64 is unix timestamp of timeout - pub last_typing_update: RwLock>, /* timestamp of the last change to - * typing - * users */ + server: Arc, + services: Services, + /// u64 is unix timestamp of timeout + pub typing: RwLock>>, + /// timestamp of the last change to typing users + pub last_typing_update: RwLock>, pub typing_update_sender: broadcast::Sender, } +struct Services { + globals: Dep, + sending: Dep, +} + impl crate::Service for Service { - fn build(_args: crate::Args<'_>) -> Result> { + fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + server: args.server.clone(), + services: Services { + globals: args.depend::("globals"), + sending: args.depend::("sending"), + }, typing: RwLock::new(BTreeMap::new()), last_typing_update: RwLock::new(BTreeMap::new()), typing_update_sender: broadcast::channel(100).0, @@ -45,14 +57,14 @@ impl Service { self.last_typing_update .write() .await - .insert(room_id.to_owned(), services().globals.next_count()?); + .insert(room_id.to_owned(), self.services.globals.next_count()?); if self.typing_update_sender.send(room_id.to_owned()).is_err() { trace!("receiver found what it was looking for and is no longer interested"); } // update federation if user_is_local(user_id) { - Self::federation_send(room_id, user_id, true)?; + self.federation_send(room_id, user_id, true)?; } Ok(()) @@ -71,14 +83,14 @@ impl Service { self.last_typing_update .write() .await - .insert(room_id.to_owned(), services().globals.next_count()?); + .insert(room_id.to_owned(), self.services.globals.next_count()?); if self.typing_update_sender.send(room_id.to_owned()).is_err() { trace!("receiver found what it was looking for and is no longer interested"); } // update federation if user_is_local(user_id) { - Self::federation_send(room_id, user_id, false)?; + self.federation_send(room_id, user_id, false)?; } Ok(()) @@ -126,7 +138,7 @@ impl Service { self.last_typing_update .write() .await - .insert(room_id.to_owned(), services().globals.next_count()?); + .insert(room_id.to_owned(), self.services.globals.next_count()?); if self.typing_update_sender.send(room_id.to_owned()).is_err() { trace!("receiver found what it was looking for and is no longer interested"); } @@ -134,7 +146,7 @@ impl Service { // update federation for user in removable { if user_is_local(&user) { - Self::federation_send(room_id, &user, false)?; + self.federation_send(room_id, &user, false)?; } } } @@ -171,15 +183,15 @@ impl Service { }) } - fn federation_send(room_id: &RoomId, user_id: &UserId, typing: bool) -> Result<()> { + fn federation_send(&self, room_id: &RoomId, user_id: &UserId, typing: bool) -> Result<()> { debug_assert!(user_is_local(user_id), "tried to broadcast typing status of remote user",); - if !services().globals.config.allow_outgoing_typing { + if !self.server.config.allow_outgoing_typing { return Ok(()); } let edu = Edu::Typing(TypingContent::new(room_id.to_owned(), user_id.to_owned(), typing)); - services() + self.services .sending .send_edu_room(room_id, serde_json::to_vec(&edu).expect("Serialized Edu::Typing"))?; diff --git a/src/service/rooms/user/data.rs b/src/service/rooms/user/data.rs index 618caae0..c7131615 100644 --- a/src/service/rooms/user/data.rs +++ b/src/service/rooms/user/data.rs @@ -1,10 +1,10 @@ use std::sync::Arc; use conduit::{utils, Error, Result}; -use database::{Database, Map}; +use database::Map; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; -use crate::services; +use crate::{globals, rooms, Dep}; pub(super) struct Data { userroomid_notificationcount: Arc, @@ -12,16 +12,27 @@ pub(super) struct Data { roomuserid_lastnotificationread: Arc, roomsynctoken_shortstatehash: Arc, userroomid_joined: Arc, + services: Services, +} + +struct Services { + globals: Dep, + short: Dep, } impl Data { - pub(super) fn new(db: &Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { userroomid_notificationcount: db["userroomid_notificationcount"].clone(), userroomid_highlightcount: db["userroomid_highlightcount"].clone(), roomuserid_lastnotificationread: db["userroomid_highlightcount"].clone(), //< NOTE: known bug from conduit roomsynctoken_shortstatehash: db["roomsynctoken_shortstatehash"].clone(), userroomid_joined: db["userroomid_joined"].clone(), + services: Services { + globals: args.depend::("globals"), + short: args.depend::("rooms::short"), + }, } } @@ -39,7 +50,7 @@ impl Data { .insert(&userroom_id, &0_u64.to_be_bytes())?; self.roomuserid_lastnotificationread - .insert(&roomuser_id, &services().globals.next_count()?.to_be_bytes())?; + .insert(&roomuser_id, &self.services.globals.next_count()?.to_be_bytes())?; Ok(()) } @@ -87,8 +98,8 @@ impl Data { pub(super) fn associate_token_shortstatehash( &self, room_id: &RoomId, token: u64, shortstatehash: u64, ) -> Result<()> { - let shortroomid = services() - .rooms + let shortroomid = self + .services .short .get_shortroomid(room_id)? .expect("room exists"); @@ -101,8 +112,8 @@ impl Data { } pub(super) fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result> { - let shortroomid = services() - .rooms + let shortroomid = self + .services .short .get_shortroomid(room_id)? .expect("room exists"); diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 12124a57..93d38470 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -3,9 +3,10 @@ mod data; use std::sync::Arc; use conduit::Result; -use data::Data; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; +use self::data::Data; + pub struct Service { db: Data, } @@ -13,7 +14,7 @@ pub struct Service { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data::new(&args), })) } diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 9e060e81..5ed40ad9 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -1,16 +1,17 @@ use std::{fmt::Debug, mem}; use bytes::BytesMut; +use conduit::{debug_error, trace, utils, warn, Error, Result}; +use reqwest::Client; use ruma::api::{appservice::Registration, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken}; -use tracing::{trace, warn}; - -use crate::{debug_error, services, utils, Error, Result}; /// Sends a request to an appservice /// /// Only returns Ok(None) if there is no url specified in the appservice /// registration file -pub(crate) async fn send_request(registration: Registration, request: T) -> Result> +pub(crate) async fn send_request( + client: &Client, registration: Registration, request: T, +) -> Result> where T: OutgoingRequest + Debug + Send, { @@ -48,15 +49,10 @@ where let reqwest_request = reqwest::Request::try_from(http_request)?; - let mut response = services() - .client - .appservice - .execute(reqwest_request) - .await - .map_err(|e| { - warn!("Could not send request to appservice \"{}\" at {dest}: {e}", registration.id); - e - })?; + let mut response = client.execute(reqwest_request).await.map_err(|e| { + warn!("Could not send request to appservice \"{}\" at {dest}: {e}", registration.id); + e + })?; // reqwest::Response -> http::Response conversion let status = response.status(); diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 9cb1c267..6c8e2544 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -5,7 +5,7 @@ use database::{Database, Map}; use ruma::{ServerName, UserId}; use super::{Destination, SendingEvent}; -use crate::services; +use crate::{globals, Dep}; type OutgoingSendingIter<'a> = Box, Destination, SendingEvent)>> + 'a>; type SendingEventIter<'a> = Box, SendingEvent)>> + 'a>; @@ -15,15 +15,24 @@ pub struct Data { servernameevent_data: Arc, servername_educount: Arc, pub(super) db: Arc, + services: Services, +} + +struct Services { + globals: Dep, } impl Data { - pub(super) fn new(db: Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { servercurrentevent_data: db["servercurrentevent_data"].clone(), servernameevent_data: db["servernameevent_data"].clone(), servername_educount: db["servername_educount"].clone(), - db, + db: args.db.clone(), + services: Services { + globals: args.depend::("globals"), + }, } } @@ -78,7 +87,7 @@ impl Data { if let SendingEvent::Pdu(value) = &event { key.extend_from_slice(value); } else { - key.extend_from_slice(&services().globals.next_count()?.to_be_bytes()); + key.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); } let value = if let SendingEvent::Edu(value) = &event { &**value diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 26f43fd3..6f091b04 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -6,26 +6,39 @@ mod sender; use std::{fmt::Debug, sync::Arc}; use async_trait::async_trait; -use conduit::{err, Result, Server}; +use conduit::{err, warn, Result, Server}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; -pub use sender::convert_to_outgoing_federation_event; use tokio::sync::Mutex; -use tracing::warn; -use crate::{server_is_ours, services}; +use crate::{account_data, client, globals, presence, pusher, resolver, rooms, server_is_ours, users, Dep}; pub struct Service { - pub db: data::Data, server: Arc, - - /// The state for a given state hash. + services: Services, + pub db: data::Data, sender: loole::Sender, receiver: Mutex>, } +struct Services { + client: Dep, + globals: Dep, + resolver: Dep, + state: Dep, + state_cache: Dep, + user: Dep, + users: Dep, + presence: Dep, + read_receipt: Dep, + timeline: Dep, + account_data: Dep, + appservice: Dep, + pusher: Dep, +} + #[derive(Clone, Debug, PartialEq, Eq)] struct Msg { dest: Destination, @@ -53,8 +66,23 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let (sender, receiver) = loole::unbounded(); Ok(Arc::new(Self { - db: data::Data::new(args.db.clone()), server: args.server.clone(), + services: Services { + client: args.depend::("client"), + globals: args.depend::("globals"), + resolver: args.depend::("resolver"), + state: args.depend::("rooms::state"), + state_cache: args.depend::("rooms::state_cache"), + user: args.depend::("rooms::user"), + users: args.depend::("users"), + presence: args.depend::("presence"), + read_receipt: args.depend::("rooms::read_receipt"), + timeline: args.depend::("rooms::timeline"), + account_data: args.depend::("account_data"), + appservice: args.depend::("appservice"), + pusher: args.depend::("pusher"), + }, + db: data::Data::new(&args), sender, receiver: Mutex::new(receiver), })) @@ -103,8 +131,8 @@ impl Service { #[tracing::instrument(skip(self, room_id, pdu_id), level = "debug")] pub fn send_pdu_room(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result<()> { - let servers = services() - .rooms + let servers = self + .services .state_cache .room_servers(room_id) .filter_map(Result::ok) @@ -152,8 +180,8 @@ impl Service { #[tracing::instrument(skip(self, room_id, serialized), level = "debug")] pub fn send_edu_room(&self, room_id: &RoomId, serialized: Vec) -> Result<()> { - let servers = services() - .rooms + let servers = self + .services .state_cache .room_servers(room_id) .filter_map(Result::ok) @@ -189,8 +217,8 @@ impl Service { #[tracing::instrument(skip(self, room_id), level = "debug")] pub fn flush_room(&self, room_id: &RoomId) -> Result<()> { - let servers = services() - .rooms + let servers = self + .services .state_cache .room_servers(room_id) .filter_map(Result::ok) @@ -213,13 +241,13 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, request), name = "request")] + #[tracing::instrument(skip_all, name = "request")] pub async fn send_federation_request(&self, dest: &ServerName, request: T) -> Result where T: OutgoingRequest + Debug + Send, { - let client = &services().client.federation; - send::send(client, dest, request).await + let client = &self.services.client.federation; + self.send(client, dest, request).await } /// Sends a request to an appservice @@ -232,7 +260,8 @@ impl Service { where T: OutgoingRequest + Debug + Send, { - appservice::send_request(registration, request).await + let client = &self.services.client.appservice; + appservice::send_request(client, registration, request).await } /// Cleanup event data diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 7901de48..b3a84d62 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,6 +1,8 @@ use std::{fmt::Debug, mem}; -use conduit::Err; +use conduit::{ + debug, debug_error, debug_warn, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error, Result, +}; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; use reqwest::{Client, Method, Request, Response, Url}; @@ -13,75 +15,91 @@ use ruma::{ server_util::authorization::XMatrix, ServerName, }; -use tracing::{debug, trace}; use crate::{ - debug_error, debug_warn, resolver, + globals, resolver, resolver::{actual::ActualDest, cache::CachedDest}, - services, Error, Result, }; -#[tracing::instrument(skip_all, name = "send")] -pub async fn send(client: &Client, dest: &ServerName, req: T) -> Result -where - T: OutgoingRequest + Debug + Send, -{ - if !services().globals.allow_federation() { - return Err!(Config("allow_federation", "Federation is disabled.")); +impl super::Service { + #[tracing::instrument(skip(self, client, req), name = "send")] + pub async fn send(&self, client: &Client, dest: &ServerName, req: T) -> Result + where + T: OutgoingRequest + Debug + Send, + { + if !self.server.config.allow_federation { + return Err!(Config("allow_federation", "Federation is disabled.")); + } + + let actual = self.services.resolver.get_actual_dest(dest).await?; + let request = self.prepare::(dest, &actual, req).await?; + self.execute::(dest, &actual, request, client).await } - let actual = services().resolver.get_actual_dest(dest).await?; - let request = prepare::(dest, &actual, req).await?; - execute::(client, dest, &actual, request).await -} + async fn execute( + &self, dest: &ServerName, actual: &ActualDest, request: Request, client: &Client, + ) -> Result + where + T: OutgoingRequest + Debug + Send, + { + let url = request.url().clone(); + let method = request.method().clone(); -async fn execute( - client: &Client, dest: &ServerName, actual: &ActualDest, request: Request, -) -> Result -where - T: OutgoingRequest + Debug + Send, -{ - let method = request.method().clone(); - let url = request.url().clone(); - debug!( - method = ?method, - url = ?url, - "Sending request", - ); - match client.execute(request).await { - Ok(response) => handle_response::(dest, actual, &method, &url, response).await, - Err(e) => handle_error::(dest, actual, &method, &url, e), + debug!(?method, ?url, "Sending request"); + match client.execute(request).await { + Ok(response) => handle_response::(&self.services.resolver, dest, actual, &method, &url, response).await, + Err(error) => handle_error::(dest, actual, &method, &url, error), + } } -} -async fn prepare(dest: &ServerName, actual: &ActualDest, req: T) -> Result -where - T: OutgoingRequest + Debug + Send, -{ - const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_5]; + async fn prepare(&self, dest: &ServerName, actual: &ActualDest, req: T) -> Result + where + T: OutgoingRequest + Debug + Send, + { + const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_5]; + const SATIR: SendAccessToken<'_> = SendAccessToken::IfRequired(EMPTY); - trace!("Preparing request"); + trace!("Preparing request"); + let mut http_request = req + .try_into_http_request::>(&actual.string, SATIR, &VERSIONS) + .map_err(|_| Error::BadServerResponse("Invalid destination"))?; - let mut http_request = req - .try_into_http_request::>(&actual.string, SendAccessToken::IfRequired(""), &VERSIONS) - .map_err(|_e| Error::BadServerResponse("Invalid destination"))?; + sign_request::(&self.services.globals, dest, &mut http_request); - sign_request::(dest, &mut http_request); + let request = Request::try_from(http_request)?; + self.validate_url(request.url())?; - let request = Request::try_from(http_request)?; - validate_url(request.url())?; + Ok(request) + } - Ok(request) + fn validate_url(&self, url: &Url) -> Result<()> { + if let Some(url_host) = url.host_str() { + if let Ok(ip) = IPAddress::parse(url_host) { + trace!("Checking request URL IP {ip:?}"); + self.services.resolver.validate_ip(&ip)?; + } + } + + Ok(()) + } } async fn handle_response( - dest: &ServerName, actual: &ActualDest, method: &Method, url: &Url, mut response: Response, + resolver: &resolver::Service, dest: &ServerName, actual: &ActualDest, method: &Method, url: &Url, + mut response: Response, ) -> Result where T: OutgoingRequest + Debug + Send, { - trace!("Received response from {} for {} with {}", actual.string, url, response.url()); let status = response.status(); + trace!( + ?status, ?method, + request_url = ?url, + response_url = ?response.url(), + "Received response from {}", + actual.string, + ); + let mut http_response_builder = http::Response::builder() .status(status) .version(response.version()); @@ -92,11 +110,13 @@ where .expect("http::response::Builder is usable"), ); - trace!("Waiting for response body"); - let body = response.bytes().await.unwrap_or_else(|e| { - debug_error!("server error {}", e); - Vec::new().into() - }); // TODO: handle timeout + // TODO: handle timeout + trace!("Waiting for response body..."); + let body = response + .bytes() + .await + .inspect_err(inspect_debug_log) + .unwrap_or_else(|_| Vec::new().into()); let http_response = http_response_builder .body(body) @@ -109,7 +129,7 @@ where let response = T::IncomingResponse::try_from_http_response(http_response); if response.is_ok() && !actual.cached { - services().resolver.set_cached_destination( + resolver.set_cached_destination( dest.to_owned(), CachedDest { dest: actual.dest.clone(), @@ -120,7 +140,7 @@ where } match response { - Err(_e) => Err(Error::BadServerResponse("Server returned bad 200 response.")), + Err(_) => Err(Error::BadServerResponse("Server returned bad 200 response.")), Ok(response) => Ok(response), } } @@ -150,7 +170,7 @@ where Err(e.into()) } -fn sign_request(dest: &ServerName, http_request: &mut http::Request>) +fn sign_request(globals: &globals::Service, dest: &ServerName, http_request: &mut http::Request>) where T: OutgoingRequest + Debug + Send, { @@ -172,16 +192,12 @@ where .to_string() .into(), ); - req_map.insert("origin".to_owned(), services().globals.server_name().as_str().into()); + req_map.insert("origin".to_owned(), globals.server_name().as_str().into()); req_map.insert("destination".to_owned(), dest.as_str().into()); let mut req_json = serde_json::from_value(req_map.into()).expect("valid JSON is valid BTreeMap"); - ruma::signatures::sign_json( - services().globals.server_name().as_str(), - services().globals.keypair(), - &mut req_json, - ) - .expect("our request json is what ruma expects"); + ruma::signatures::sign_json(globals.server_name().as_str(), globals.keypair(), &mut req_json) + .expect("our request json is what ruma expects"); let req_json: serde_json::Map = serde_json::from_slice(&serde_json::to_vec(&req_json).unwrap()).unwrap(); @@ -207,24 +223,8 @@ where http_request.headers_mut().insert( AUTHORIZATION, - HeaderValue::from(&XMatrix::new( - services().globals.config.server_name.clone(), - dest.to_owned(), - key, - sig, - )), + HeaderValue::from(&XMatrix::new(globals.config.server_name.clone(), dest.to_owned(), key, sig)), ); } } } - -fn validate_url(url: &Url) -> Result<()> { - if let Some(url_host) = url.host_str() { - if let Ok(ip) = IPAddress::parse(url_host) { - trace!("Checking request URL IP {ip:?}"); - resolver::actual::validate_ip(&ip)?; - } - } - - Ok(()) -} diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 774c3d69..0668ce24 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -6,7 +6,11 @@ use std::{ }; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, debug_warn, error, trace, utils::math::continue_exponential_backoff_secs, warn}; +use conduit::{ + debug, debug_warn, error, trace, + utils::{calculate_hash, math::continue_exponential_backoff_secs}, + warn, Error, Result, +}; use federation::transactions::send_transaction_message; use futures_util::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; use ruma::{ @@ -24,8 +28,8 @@ use ruma::{ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tokio::time::sleep_until; -use super::{appservice, send, Destination, Msg, SendingEvent, Service}; -use crate::{presence::Presence, services, user_is_local, utils::calculate_hash, Error, Result}; +use super::{appservice, Destination, Msg, SendingEvent, Service}; +use crate::user_is_local; #[derive(Debug)] enum TransactionStatus { @@ -69,8 +73,8 @@ impl Service { Ok(()) } - fn handle_response( - &self, response: SendingResult, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, + fn handle_response<'a>( + &'a self, response: SendingResult, futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { match response { Ok(dest) => self.handle_response_ok(&dest, futures, statuses), @@ -91,8 +95,8 @@ impl Service { }); } - fn handle_response_ok( - &self, dest: &Destination, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, + fn handle_response_ok<'a>( + &'a self, dest: &Destination, futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { let _cork = self.db.db.cork(); self.db @@ -113,24 +117,24 @@ impl Service { .mark_as_active(&new_events) .expect("marked as active"); let new_events_vec = new_events.into_iter().map(|(event, _)| event).collect(); - futures.push(Box::pin(send_events(dest.clone(), new_events_vec))); + futures.push(Box::pin(self.send_events(dest.clone(), new_events_vec))); } else { statuses.remove(dest); } } - fn handle_request(&self, msg: Msg, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus) { + fn handle_request<'a>(&'a self, msg: Msg, futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus) { let iv = vec![(msg.event, msg.queue_id)]; if let Ok(Some(events)) = self.select_events(&msg.dest, iv, statuses) { if !events.is_empty() { - futures.push(Box::pin(send_events(msg.dest, events))); + futures.push(Box::pin(self.send_events(msg.dest, events))); } else { statuses.remove(&msg.dest); } } } - async fn finish_responses(&self, futures: &mut SendingFutures<'_>, statuses: &mut CurTransactionStatus) { + async fn finish_responses<'a>(&'a self, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus) { let now = Instant::now(); let timeout = Duration::from_millis(CLEANUP_TIMEOUT_MS); let deadline = now.checked_add(timeout).unwrap_or(now); @@ -148,7 +152,7 @@ impl Service { debug_warn!("Leaving with {} unfinished requests...", futures.len()); } - fn initial_requests(&self, futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus) { + fn initial_requests<'a>(&'a self, futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus) { let keep = usize::try_from(self.server.config.startup_netburst_keep).unwrap_or(usize::MAX); let mut txns = HashMap::>::new(); for (key, dest, event) in self.db.active_requests().filter_map(Result::ok) { @@ -166,12 +170,12 @@ impl Service { for (dest, events) in txns { if self.server.config.startup_netburst && !events.is_empty() { statuses.insert(dest.clone(), TransactionStatus::Running); - futures.push(Box::pin(send_events(dest.clone(), events))); + futures.push(Box::pin(self.send_events(dest.clone(), events))); } } } - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip_all, level = "debug")] fn select_events( &self, dest: &Destination, @@ -218,7 +222,7 @@ impl Service { Ok(Some(events)) } - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip_all, level = "debug")] fn select_events_current(&self, dest: Destination, statuses: &mut CurTransactionStatus) -> Result<(bool, bool)> { let (mut allow, mut retry) = (true, false); statuses @@ -244,7 +248,7 @@ impl Service { Ok((allow, retry)) } - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip_all, level = "debug")] fn select_edus(&self, server_name: &ServerName) -> Result<(Vec>, u64)> { // u64: count of last edu let since = self.db.get_latest_educount(server_name)?; @@ -252,11 +256,11 @@ impl Service { let mut max_edu_count = since; let mut device_list_changes = HashSet::new(); - for room_id in services().rooms.state_cache.server_rooms(server_name) { + for room_id in self.services.state_cache.server_rooms(server_name) { let room_id = room_id?; // Look for device list updates in this room device_list_changes.extend( - services() + self.services .users .keys_changed(room_id.as_ref(), since, None) .filter_map(Result::ok) @@ -264,7 +268,7 @@ impl Service { ); if self.server.config.allow_outgoing_read_receipts - && !select_edus_receipts(&room_id, since, &mut max_edu_count, &mut events)? + && !self.select_edus_receipts(&room_id, since, &mut max_edu_count, &mut events)? { break; } @@ -287,381 +291,390 @@ impl Service { } if self.server.config.allow_outgoing_presence { - select_edus_presence(server_name, since, &mut max_edu_count, &mut events)?; + self.select_edus_presence(server_name, since, &mut max_edu_count, &mut events)?; } Ok((events, max_edu_count)) } -} -/// Look for presence -fn select_edus_presence( - server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, -) -> Result { - // Look for presence updates for this server - let mut presence_updates = Vec::new(); - for (user_id, count, presence_bytes) in services().presence.presence_since(since) { - *max_edu_count = cmp::max(count, *max_edu_count); + /// Look for presence + fn select_edus_presence( + &self, server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, + ) -> Result { + // Look for presence updates for this server + let mut presence_updates = Vec::new(); + for (user_id, count, presence_bytes) in self.services.presence.presence_since(since) { + *max_edu_count = cmp::max(count, *max_edu_count); - if !user_is_local(&user_id) { - continue; - } + if !user_is_local(&user_id) { + continue; + } - if !services() - .rooms - .state_cache - .server_sees_user(server_name, &user_id)? - { - continue; - } + if !self + .services + .state_cache + .server_sees_user(server_name, &user_id)? + { + continue; + } - let presence_event = Presence::from_json_bytes_to_event(&presence_bytes, &user_id)?; - presence_updates.push(PresenceUpdate { - user_id, - presence: presence_event.content.presence, - currently_active: presence_event.content.currently_active.unwrap_or(false), - last_active_ago: presence_event - .content - .last_active_ago - .unwrap_or_else(|| uint!(0)), - status_msg: presence_event.content.status_msg, - }); - - if presence_updates.len() >= SELECT_EDU_LIMIT { - break; - } - } - - if !presence_updates.is_empty() { - let presence_content = Edu::Presence(PresenceContent::new(presence_updates)); - events.push(serde_json::to_vec(&presence_content).expect("PresenceEvent can be serialized")); - } - - Ok(true) -} - -/// Look for read receipts in this room -fn select_edus_receipts( - room_id: &RoomId, since: u64, max_edu_count: &mut u64, events: &mut Vec>, -) -> Result { - for r in services() - .rooms - .read_receipt - .readreceipts_since(room_id, since) - { - let (user_id, count, read_receipt) = r?; - *max_edu_count = cmp::max(count, *max_edu_count); - - if !user_is_local(&user_id) { - continue; - } - - let event = serde_json::from_str(read_receipt.json().get()) - .map_err(|_| Error::bad_database("Invalid edu event in read_receipts."))?; - let federation_event = if let AnySyncEphemeralRoomEvent::Receipt(r) = event { - let mut read = BTreeMap::new(); - - let (event_id, mut receipt) = r - .content - .0 - .into_iter() - .next() - .expect("we only use one event per read receipt"); - let receipt = receipt - .remove(&ReceiptType::Read) - .expect("our read receipts always set this") - .remove(&user_id) - .expect("our read receipts always have the user here"); - - read.insert( + let presence_event = self + .services + .presence + .from_json_bytes_to_event(&presence_bytes, &user_id)?; + presence_updates.push(PresenceUpdate { user_id, - ReceiptData { - data: receipt.clone(), - event_ids: vec![event_id.clone()], - }, - ); + presence: presence_event.content.presence, + currently_active: presence_event.content.currently_active.unwrap_or(false), + last_active_ago: presence_event + .content + .last_active_ago + .unwrap_or_else(|| uint!(0)), + status_msg: presence_event.content.status_msg, + }); - let receipt_map = ReceiptMap { - read, - }; - - let mut receipts = BTreeMap::new(); - receipts.insert(room_id.to_owned(), receipt_map); - - Edu::Receipt(ReceiptContent { - receipts, - }) - } else { - Error::bad_database("Invalid event type in read_receipts"); - continue; - }; - - events.push(serde_json::to_vec(&federation_event).expect("json can be serialized")); - - if events.len() >= SELECT_EDU_LIMIT { - return Ok(false); - } - } - - Ok(true) -} - -async fn send_events(dest: Destination, events: Vec) -> SendingResult { - //debug_assert!(!events.is_empty(), "sending empty transaction"); - match dest { - Destination::Normal(ref server) => send_events_dest_normal(&dest, server, events).await, - Destination::Appservice(ref id) => send_events_dest_appservice(&dest, id, events).await, - Destination::Push(ref userid, ref pushkey) => send_events_dest_push(&dest, userid, pushkey, events).await, - } -} - -#[tracing::instrument(skip(dest, events))] -async fn send_events_dest_appservice(dest: &Destination, id: &str, events: Vec) -> SendingResult { - let mut pdu_jsons = Vec::new(); - - for event in &events { - match event { - SendingEvent::Pdu(pdu_id) => { - pdu_jsons.push( - services() - .rooms - .timeline - .get_pdu_from_id(pdu_id) - .map_err(|e| (dest.clone(), e))? - .ok_or_else(|| { - ( - dest.clone(), - Error::bad_database("[Appservice] Event in servernameevent_data not found in db."), - ) - })? - .to_room_event(), - ); - }, - SendingEvent::Edu(_) | SendingEvent::Flush => { - // Appservices don't need EDUs (?) and flush only; - // no new content - }, - } - } - - //debug_assert!(!pdu_jsons.is_empty(), "sending empty transaction"); - match appservice::send_request( - services() - .appservice - .get_registration(id) - .await - .ok_or_else(|| { - ( - dest.clone(), - Error::bad_database("[Appservice] Could not load registration from db."), - ) - })?, - ruma::api::appservice::event::push_events::v1::Request { - events: pdu_jsons, - txn_id: (&*general_purpose::URL_SAFE_NO_PAD.encode(calculate_hash( - &events - .iter() - .map(|e| match e { - SendingEvent::Edu(b) | SendingEvent::Pdu(b) => &**b, - SendingEvent::Flush => &[], - }) - .collect::>(), - ))) - .into(), - }, - ) - .await - { - Ok(_) => Ok(dest.clone()), - Err(e) => Err((dest.clone(), e)), - } -} - -#[tracing::instrument(skip(dest, events))] -async fn send_events_dest_push( - dest: &Destination, userid: &OwnedUserId, pushkey: &str, events: Vec, -) -> SendingResult { - let mut pdus = Vec::new(); - - for event in &events { - match event { - SendingEvent::Pdu(pdu_id) => { - pdus.push( - services() - .rooms - .timeline - .get_pdu_from_id(pdu_id) - .map_err(|e| (dest.clone(), e))? - .ok_or_else(|| { - ( - dest.clone(), - Error::bad_database("[Push] Event in servernameevent_data not found in db."), - ) - })?, - ); - }, - SendingEvent::Edu(_) | SendingEvent::Flush => { - // Push gateways don't need EDUs (?) and flush only; - // no new content - }, - } - } - - for pdu in pdus { - // Redacted events are not notification targets (we don't send push for them) - if let Some(unsigned) = &pdu.unsigned { - if let Ok(unsigned) = serde_json::from_str::(unsigned.get()) { - if unsigned.get("redacted_because").is_some() { - continue; - } + if presence_updates.len() >= SELECT_EDU_LIMIT { + break; } } - let Some(pusher) = services() - .pusher - .get_pusher(userid, pushkey) - .map_err(|e| (dest.clone(), e))? - else { - continue; - }; + if !presence_updates.is_empty() { + let presence_content = Edu::Presence(PresenceContent::new(presence_updates)); + events.push(serde_json::to_vec(&presence_content).expect("PresenceEvent can be serialized")); + } - let rules_for_user = services() - .account_data - .get(None, userid, GlobalAccountDataEventType::PushRules.to_string().into()) - .unwrap_or_default() - .and_then(|event| serde_json::from_str::(event.get()).ok()) - .map_or_else(|| push::Ruleset::server_default(userid), |ev: PushRulesEvent| ev.content.global); - - let unread: UInt = services() - .rooms - .user - .notification_count(userid, &pdu.room_id) - .map_err(|e| (dest.clone(), e))? - .try_into() - .expect("notification count can't go that high"); - - let _response = services() - .pusher - .send_push_notice(userid, unread, &pusher, rules_for_user, &pdu) - .await - .map(|_response| dest.clone()) - .map_err(|e| (dest.clone(), e)); + Ok(true) } - Ok(dest.clone()) -} + /// Look for read receipts in this room + fn select_edus_receipts( + &self, room_id: &RoomId, since: u64, max_edu_count: &mut u64, events: &mut Vec>, + ) -> Result { + for r in self + .services + .read_receipt + .readreceipts_since(room_id, since) + { + let (user_id, count, read_receipt) = r?; + *max_edu_count = cmp::max(count, *max_edu_count); -#[tracing::instrument(skip(dest, events), name = "")] -async fn send_events_dest_normal( - dest: &Destination, server: &OwnedServerName, events: Vec, -) -> SendingResult { - let mut pdu_jsons = Vec::with_capacity( - events - .iter() - .filter(|event| matches!(event, SendingEvent::Pdu(_))) - .count(), - ); - let mut edu_jsons = Vec::with_capacity( - events - .iter() - .filter(|event| matches!(event, SendingEvent::Edu(_))) - .count(), - ); + if !user_is_local(&user_id) { + continue; + } - for event in &events { - match event { - SendingEvent::Pdu(pdu_id) => pdu_jsons.push(convert_to_outgoing_federation_event( - // TODO: check room version and remove event_id if needed - services() - .rooms - .timeline - .get_pdu_json_from_id(pdu_id) - .map_err(|e| (dest.clone(), e))? - .ok_or_else(|| { - error!(?dest, ?server, ?pdu_id, "event not found"); - ( - dest.clone(), - Error::bad_database("[Normal] Event in servernameevent_data not found in db."), - ) - })?, - )), - SendingEvent::Edu(edu) => { - if let Ok(raw) = serde_json::from_slice(edu) { - edu_jsons.push(raw); - } - }, - SendingEvent::Flush => { - // flush only; no new content + let event = serde_json::from_str(read_receipt.json().get()) + .map_err(|_| Error::bad_database("Invalid edu event in read_receipts."))?; + let federation_event = if let AnySyncEphemeralRoomEvent::Receipt(r) = event { + let mut read = BTreeMap::new(); + + let (event_id, mut receipt) = r + .content + .0 + .into_iter() + .next() + .expect("we only use one event per read receipt"); + let receipt = receipt + .remove(&ReceiptType::Read) + .expect("our read receipts always set this") + .remove(&user_id) + .expect("our read receipts always have the user here"); + + read.insert( + user_id, + ReceiptData { + data: receipt.clone(), + event_ids: vec![event_id.clone()], + }, + ); + + let receipt_map = ReceiptMap { + read, + }; + + let mut receipts = BTreeMap::new(); + receipts.insert(room_id.to_owned(), receipt_map); + + Edu::Receipt(ReceiptContent { + receipts, + }) + } else { + Error::bad_database("Invalid event type in read_receipts"); + continue; + }; + + events.push(serde_json::to_vec(&federation_event).expect("json can be serialized")); + + if events.len() >= SELECT_EDU_LIMIT { + return Ok(false); + } + } + + Ok(true) + } + + async fn send_events(&self, dest: Destination, events: Vec) -> SendingResult { + //debug_assert!(!events.is_empty(), "sending empty transaction"); + match dest { + Destination::Normal(ref server) => self.send_events_dest_normal(&dest, server, events).await, + Destination::Appservice(ref id) => self.send_events_dest_appservice(&dest, id, events).await, + Destination::Push(ref userid, ref pushkey) => { + self.send_events_dest_push(&dest, userid, pushkey, events) + .await }, } } - //debug_assert!(pdu_jsons.len() + edu_jsons.len() > 0, "sending empty - // transaction"); - send::send( - &services().client.sender, - server, - send_transaction_message::v1::Request { - origin: services().server.config.server_name.clone(), + #[tracing::instrument(skip(self, dest, events), name = "appservice")] + async fn send_events_dest_appservice( + &self, dest: &Destination, id: &str, events: Vec, + ) -> SendingResult { + let mut pdu_jsons = Vec::new(); + + for event in &events { + match event { + SendingEvent::Pdu(pdu_id) => { + pdu_jsons.push( + self.services + .timeline + .get_pdu_from_id(pdu_id) + .map_err(|e| (dest.clone(), e))? + .ok_or_else(|| { + ( + dest.clone(), + Error::bad_database("[Appservice] Event in servernameevent_data not found in db."), + ) + })? + .to_room_event(), + ); + }, + SendingEvent::Edu(_) | SendingEvent::Flush => { + // Appservices don't need EDUs (?) and flush only; + // no new content + }, + } + } + + //debug_assert!(!pdu_jsons.is_empty(), "sending empty transaction"); + let client = &self.services.client.appservice; + match appservice::send_request( + client, + self.services + .appservice + .get_registration(id) + .await + .ok_or_else(|| { + ( + dest.clone(), + Error::bad_database("[Appservice] Could not load registration from db."), + ) + })?, + ruma::api::appservice::event::push_events::v1::Request { + events: pdu_jsons, + txn_id: (&*general_purpose::URL_SAFE_NO_PAD.encode(calculate_hash( + &events + .iter() + .map(|e| match e { + SendingEvent::Edu(b) | SendingEvent::Pdu(b) => &**b, + SendingEvent::Flush => &[], + }) + .collect::>(), + ))) + .into(), + }, + ) + .await + { + Ok(_) => Ok(dest.clone()), + Err(e) => Err((dest.clone(), e)), + } + } + + #[tracing::instrument(skip(self, dest, events), name = "push")] + async fn send_events_dest_push( + &self, dest: &Destination, userid: &OwnedUserId, pushkey: &str, events: Vec, + ) -> SendingResult { + let mut pdus = Vec::new(); + + for event in &events { + match event { + SendingEvent::Pdu(pdu_id) => { + pdus.push( + self.services + .timeline + .get_pdu_from_id(pdu_id) + .map_err(|e| (dest.clone(), e))? + .ok_or_else(|| { + ( + dest.clone(), + Error::bad_database("[Push] Event in servernameevent_data not found in db."), + ) + })?, + ); + }, + SendingEvent::Edu(_) | SendingEvent::Flush => { + // Push gateways don't need EDUs (?) and flush only; + // no new content + }, + } + } + + for pdu in pdus { + // Redacted events are not notification targets (we don't send push for them) + if let Some(unsigned) = &pdu.unsigned { + if let Ok(unsigned) = serde_json::from_str::(unsigned.get()) { + if unsigned.get("redacted_because").is_some() { + continue; + } + } + } + + let Some(pusher) = self + .services + .pusher + .get_pusher(userid, pushkey) + .map_err(|e| (dest.clone(), e))? + else { + continue; + }; + + let rules_for_user = self + .services + .account_data + .get(None, userid, GlobalAccountDataEventType::PushRules.to_string().into()) + .unwrap_or_default() + .and_then(|event| serde_json::from_str::(event.get()).ok()) + .map_or_else(|| push::Ruleset::server_default(userid), |ev: PushRulesEvent| ev.content.global); + + let unread: UInt = self + .services + .user + .notification_count(userid, &pdu.room_id) + .map_err(|e| (dest.clone(), e))? + .try_into() + .expect("notification count can't go that high"); + + let _response = self + .services + .pusher + .send_push_notice(userid, unread, &pusher, rules_for_user, &pdu) + .await + .map(|_response| dest.clone()) + .map_err(|e| (dest.clone(), e)); + } + + Ok(dest.clone()) + } + + #[tracing::instrument(skip(self, dest, events), name = "", level = "debug")] + async fn send_events_dest_normal( + &self, dest: &Destination, server: &OwnedServerName, events: Vec, + ) -> SendingResult { + let mut pdu_jsons = Vec::with_capacity( + events + .iter() + .filter(|event| matches!(event, SendingEvent::Pdu(_))) + .count(), + ); + let mut edu_jsons = Vec::with_capacity( + events + .iter() + .filter(|event| matches!(event, SendingEvent::Edu(_))) + .count(), + ); + + for event in &events { + match event { + // TODO: check room version and remove event_id if needed + SendingEvent::Pdu(pdu_id) => pdu_jsons.push( + self.convert_to_outgoing_federation_event( + self.services + .timeline + .get_pdu_json_from_id(pdu_id) + .map_err(|e| (dest.clone(), e))? + .ok_or_else(|| { + error!(?dest, ?server, ?pdu_id, "event not found"); + ( + dest.clone(), + Error::bad_database("[Normal] Event in servernameevent_data not found in db."), + ) + })?, + ), + ), + SendingEvent::Edu(edu) => { + if let Ok(raw) = serde_json::from_slice(edu) { + edu_jsons.push(raw); + } + }, + SendingEvent::Flush => {}, // flush only; no new content + } + } + + //debug_assert!(pdu_jsons.len() + edu_jsons.len() > 0, "sending empty + // transaction"); + let transaction_id = &*general_purpose::URL_SAFE_NO_PAD.encode(calculate_hash( + &events + .iter() + .map(|e| match e { + SendingEvent::Edu(b) | SendingEvent::Pdu(b) => &**b, + SendingEvent::Flush => &[], + }) + .collect::>(), + )); + + let request = send_transaction_message::v1::Request { + origin: self.server.config.server_name.clone(), pdus: pdu_jsons, edus: edu_jsons, origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - transaction_id: (&*general_purpose::URL_SAFE_NO_PAD.encode(calculate_hash( - &events + transaction_id: transaction_id.into(), + }; + + let client = &self.services.client.sender; + self.send(client, server, request) + .await + .inspect(|response| { + response + .pdus .iter() - .map(|e| match e { - SendingEvent::Edu(b) | SendingEvent::Pdu(b) => &**b, - SendingEvent::Flush => &[], - }) - .collect::>(), - ))) - .into(), - }, - ) - .await - .map(|response| { - for pdu in response.pdus { - if pdu.1.is_err() { - warn!("error for {} from remote: {:?}", pdu.0, pdu.1); + .filter(|(_, res)| res.is_err()) + .for_each(|(pdu_id, res)| warn!("error for {pdu_id} from remote: {res:?}")); + }) + .map(|_| dest.clone()) + .map_err(|e| (dest.clone(), e)) + } + + /// This does not return a full `Pdu` it is only to satisfy ruma's types. + pub fn convert_to_outgoing_federation_event(&self, mut pdu_json: CanonicalJsonObject) -> Box { + if let Some(unsigned) = pdu_json + .get_mut("unsigned") + .and_then(|val| val.as_object_mut()) + { + unsigned.remove("transaction_id"); + } + + // room v3 and above removed the "event_id" field from remote PDU format + if let Some(room_id) = pdu_json + .get("room_id") + .and_then(|val| RoomId::parse(val.as_str()?).ok()) + { + match self.services.state.get_room_version(&room_id) { + Ok(room_version_id) => match room_version_id { + RoomVersionId::V1 | RoomVersionId::V2 => {}, + _ => _ = pdu_json.remove("event_id"), + }, + Err(_) => _ = pdu_json.remove("event_id"), } + } else { + pdu_json.remove("event_id"); } - dest.clone() - }) - .map_err(|e| (dest.clone(), e)) -} -/// This does not return a full `Pdu` it is only to satisfy ruma's types. -#[tracing::instrument] -pub fn convert_to_outgoing_federation_event(mut pdu_json: CanonicalJsonObject) -> Box { - if let Some(unsigned) = pdu_json - .get_mut("unsigned") - .and_then(|val| val.as_object_mut()) - { - unsigned.remove("transaction_id"); + // TODO: another option would be to convert it to a canonical string to validate + // size and return a Result> + // serde_json::from_str::>( + // ruma::serde::to_canonical_json_string(pdu_json).expect("CanonicalJson is + // valid serde_json::Value"), ) + // .expect("Raw::from_value always works") + + to_raw_value(&pdu_json).expect("CanonicalJson is valid serde_json::Value") } - - // room v3 and above removed the "event_id" field from remote PDU format - if let Some(room_id) = pdu_json - .get("room_id") - .and_then(|val| RoomId::parse(val.as_str()?).ok()) - { - match services().rooms.state.get_room_version(&room_id) { - Ok(room_version_id) => match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => _ = pdu_json.remove("event_id"), - }, - Err(_) => _ = pdu_json.remove("event_id"), - } - } else { - pdu_json.remove("event_id"); - } - - // TODO: another option would be to convert it to a canonical string to validate - // size and return a Result> - // serde_json::from_str::>( - // ruma::serde::to_canonical_json_string(pdu_json).expect("CanonicalJson is - // valid serde_json::Value"), ) - // .expect("Raw::from_value always works") - - to_raw_value(&pdu_json).expect("CanonicalJson is valid serde_json::Value") } diff --git a/src/service/service.rs b/src/service/service.rs index ce4f15b2..bf3b891b 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -3,7 +3,7 @@ use std::{ collections::BTreeMap, fmt::Write, ops::Deref, - sync::{Arc, OnceLock}, + sync::{Arc, OnceLock, RwLock}, }; use async_trait::async_trait; @@ -50,20 +50,20 @@ pub(crate) struct Args<'a> { } /// Dep is a reference to a service used within another service. -/// Circular-dependencies between services require this indirection to allow the -/// referenced service construction after the referencing service. +/// Circular-dependencies between services require this indirection. pub(crate) struct Dep { dep: OnceLock>, service: Arc, name: &'static str, } -pub(crate) type Map = BTreeMap; +pub(crate) type Map = RwLock>; pub(crate) type MapVal = (Arc, Arc); -impl Deref for Dep { +impl Deref for Dep { type Target = Arc; + /// Dereference a dependency. The dependency must be ready or panics. fn deref(&self) -> &Self::Target { self.dep .get_or_init(|| require::(&self.service, self.name)) @@ -71,39 +71,61 @@ impl Deref for Dep { } impl Args<'_> { - pub(crate) fn depend_service(&self, name: &'static str) -> Dep { + /// Create a lazy-reference to a service when constructing another Service. + pub(crate) fn depend(&self, name: &'static str) -> Dep { Dep:: { dep: OnceLock::new(), service: self.service.clone(), name, } } + + /// Create a reference immediately to a service when constructing another + /// Service. The other service must be constructed. + pub(crate) fn require(&self, name: &str) -> Arc { require::(self.service, name) } } -pub(crate) fn require(map: &Map, name: &str) -> Arc { +/// Reference a Service by name. Panics if the Service does not exist or was +/// incorrectly cast. +pub(crate) fn require(map: &Map, name: &str) -> Arc { try_get::(map, name) .inspect_err(inspect_log) .expect("Failure to reference service required by another service.") } -pub(crate) fn try_get(map: &Map, name: &str) -> Result> { - map.get(name).map_or_else( - || Err!("Service {name:?} does not exist or has not been built yet."), - |(_, s)| { +/// Reference a Service by name. Returns Err if the Service does not exist or +/// was incorrectly cast. +pub(crate) fn try_get(map: &Map, name: &str) -> Result> { + map.read() + .expect("locked for reading") + .get(name) + .map_or_else( + || Err!("Service {name:?} does not exist or has not been built yet."), + |(_, s)| { + s.clone() + .downcast::() + .map_err(|_| err!("Service {name:?} must be correctly downcast.")) + }, + ) +} + +/// Reference a Service by name. Returns None if the Service does not exist, but +/// panics if incorrectly cast. +/// +/// # Panics +/// Incorrect type is not a silent failure (None) as the type never has a reason +/// to be incorrect. +pub(crate) fn get(map: &Map, name: &str) -> Option> { + map.read() + .expect("locked for reading") + .get(name) + .map(|(_, s)| { s.clone() .downcast::() - .map_err(|_| err!("Service {name:?} must be correctly downcast.")) - }, - ) -} - -pub(crate) fn get(map: &Map, name: &str) -> Option> { - map.get(name).map(|(_, s)| { - s.clone() - .downcast::() - .expect("Service must be correctly downcast.") - }) + .expect("Service must be correctly downcast.") + }) } +/// Utility for service implementations; see Service::name() in the trait. #[inline] pub(crate) fn make_name(module_path: &str) -> &str { split_once_infallible(module_path, "::").1 } diff --git a/src/service/services.rs b/src/service/services.rs index 68205323..59909f8c 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,11 +1,16 @@ -use std::{any::Any, collections::BTreeMap, fmt::Write, sync::Arc}; +use std::{ + any::Any, + collections::BTreeMap, + fmt::Write, + sync::{Arc, RwLock}, +}; use conduit::{debug, debug_info, info, trace, Result, Server}; use database::Database; use tokio::sync::Mutex; use crate::{ - account_data, admin, appservice, client, globals, key_backups, + account_data, admin, appservice, client, emergency, globals, key_backups, manager::Manager, media, presence, pusher, resolver, rooms, sending, service, service::{Args, Map, Service}, @@ -13,22 +18,23 @@ use crate::{ }; pub struct Services { - pub resolver: Arc, - pub client: Arc, - pub globals: Arc, - pub rooms: rooms::Service, - pub appservice: Arc, - pub pusher: Arc, - pub transaction_ids: Arc, - pub uiaa: Arc, - pub users: Arc, pub account_data: Arc, - pub presence: Arc, pub admin: Arc, + pub appservice: Arc, + pub client: Arc, + pub emergency: Arc, + pub globals: Arc, pub key_backups: Arc, pub media: Arc, + pub presence: Arc, + pub pusher: Arc, + pub resolver: Arc, + pub rooms: rooms::Service, pub sending: Arc, + pub transaction_ids: Arc, + pub uiaa: Arc, pub updates: Arc, + pub users: Arc, manager: Mutex>>, pub(crate) service: Arc, @@ -36,37 +42,34 @@ pub struct Services { pub db: Arc, } -macro_rules! build_service { - ($map:ident, $server:ident, $db:ident, $tyname:ty) => {{ - let built = <$tyname>::build(Args { - server: &$server, - db: &$db, - service: &$map, - })?; - - Arc::get_mut(&mut $map) - .expect("must have mutable reference to services collection") - .insert(built.name().to_owned(), (built.clone(), built.clone())); - - trace!("built service #{}: {:?}", $map.len(), built.name()); - built - }}; -} - impl Services { #[allow(clippy::cognitive_complexity)] pub fn build(server: Arc, db: Arc) -> Result { - let mut service: Arc = Arc::new(BTreeMap::new()); + let service: Arc = Arc::new(RwLock::new(BTreeMap::new())); macro_rules! build { - ($srv:ty) => { - build_service!(service, server, db, $srv) - }; + ($tyname:ty) => {{ + let built = <$tyname>::build(Args { + db: &db, + server: &server, + service: &service, + })?; + add_service(&service, built.clone(), built.clone()); + built + }}; } Ok(Self { - globals: build!(globals::Service), + account_data: build!(account_data::Service), + admin: build!(admin::Service), + appservice: build!(appservice::Service), resolver: build!(resolver::Service), client: build!(client::Service), + emergency: build!(emergency::Service), + globals: build!(globals::Service), + key_backups: build!(key_backups::Service), + media: build!(media::Service), + presence: build!(presence::Service), + pusher: build!(pusher::Service), rooms: rooms::Service { alias: build!(rooms::alias::Service), auth_chain: build!(rooms::auth_chain::Service), @@ -79,28 +82,22 @@ impl Services { read_receipt: build!(rooms::read_receipt::Service), search: build!(rooms::search::Service), short: build!(rooms::short::Service), + spaces: build!(rooms::spaces::Service), state: build!(rooms::state::Service), state_accessor: build!(rooms::state_accessor::Service), state_cache: build!(rooms::state_cache::Service), state_compressor: build!(rooms::state_compressor::Service), - timeline: build!(rooms::timeline::Service), threads: build!(rooms::threads::Service), + timeline: build!(rooms::timeline::Service), typing: build!(rooms::typing::Service), - spaces: build!(rooms::spaces::Service), user: build!(rooms::user::Service), }, - appservice: build!(appservice::Service), - pusher: build!(pusher::Service), + sending: build!(sending::Service), transaction_ids: build!(transaction_ids::Service), uiaa: build!(uiaa::Service), - users: build!(users::Service), - account_data: build!(account_data::Service), - presence: build!(presence::Service), - admin: build!(admin::Service), - key_backups: build!(key_backups::Service), - media: build!(media::Service), - sending: build!(sending::Service), updates: build!(updates::Service), + users: build!(users::Service), + manager: Mutex::new(None), service, server, @@ -111,7 +108,7 @@ impl Services { pub(super) async fn start(&self) -> Result<()> { debug_info!("Starting services..."); - globals::migrations::migrations(&self.db, &self.server.config).await?; + globals::migrations::migrations(self).await?; self.manager .lock() .await @@ -144,7 +141,7 @@ impl Services { } pub async fn clear_cache(&self) { - for (service, ..) in self.service.values() { + for (service, ..) in self.service.read().expect("locked for reading").values() { service.clear_cache(); } @@ -159,7 +156,7 @@ impl Services { pub async fn memory_usage(&self) -> Result { let mut out = String::new(); - for (service, ..) in self.service.values() { + for (service, ..) in self.service.read().expect("locked for reading").values() { service.memory_usage(&mut out)?; } @@ -179,23 +176,26 @@ impl Services { fn interrupt(&self) { debug!("Interrupting services..."); - for (name, (service, ..)) in self.service.iter() { + for (name, (service, ..)) in self.service.read().expect("locked for reading").iter() { trace!("Interrupting {name}"); service.interrupt(); } } - pub fn try_get(&self, name: &str) -> Result> - where - T: Any + Send + Sync, - { + pub fn try_get(&self, name: &str) -> Result> { service::try_get::(&self.service, name) } - pub fn get(&self, name: &str) -> Option> - where - T: Any + Send + Sync, - { - service::get::(&self.service, name) - } + pub fn get(&self, name: &str) -> Option> { service::get::(&self.service, name) } +} + +fn add_service(map: &Arc, s: Arc, a: Arc) { + let name = s.name(); + let len = map.read().expect("locked for reading").len(); + + trace!("built service #{len}: {name:?}"); + + map.write() + .expect("locked for writing") + .insert(name.to_owned(), (s, a)); } diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 4b953ffb..6041bbd3 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -2,7 +2,7 @@ mod data; use std::sync::Arc; -use conduit::{utils, utils::hash, Error, Result}; +use conduit::{error, utils, utils::hash, Error, Result, Server}; use data::Data; use ruma::{ api::client::{ @@ -11,19 +11,30 @@ use ruma::{ }, CanonicalJsonValue, DeviceId, UserId, }; -use tracing::error; -use crate::services; +use crate::{globals, users, Dep}; pub const SESSION_ID_LENGTH: usize = 32; pub struct Service { + server: Arc, + services: Services, pub db: Data, } +struct Services { + globals: Dep, + users: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + server: args.server.clone(), + services: Services { + globals: args.depend::("globals"), + users: args.depend::("users"), + }, db: Data::new(args.db), })) } @@ -87,11 +98,11 @@ impl Service { return Err(Error::BadRequest(ErrorKind::Unrecognized, "Identifier type not recognized.")); }; - let user_id = UserId::parse_with_server_name(username.clone(), services().globals.server_name()) + let user_id = UserId::parse_with_server_name(username.clone(), self.services.globals.server_name()) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid."))?; // Check if password is correct - if let Some(hash) = services().users.password_hash(&user_id)? { + if let Some(hash) = self.services.users.password_hash(&user_id)? { let hash_matches = hash::verify_password(password, &hash).is_ok(); if !hash_matches { uiaainfo.auth_error = Some(ruma::api::client::error::StandardErrorBody { @@ -106,7 +117,7 @@ impl Service { uiaainfo.completed.push(AuthType::Password); }, AuthData::RegistrationToken(t) => { - if Some(t.token.trim()) == services().globals.config.registration_token.as_deref() { + if Some(t.token.trim()) == self.server.config.registration_token.as_deref() { uiaainfo.completed.push(AuthType::RegistrationToken); } else { uiaainfo.auth_error = Some(ruma::api::client::error::StandardErrorBody { diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index db69d9b0..f89471cd 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -7,14 +7,20 @@ use ruma::events::room::message::RoomMessageEventContent; use serde::Deserialize; use tokio::{sync::Notify, time::interval}; -use crate::services; +use crate::{admin, client, Dep}; pub struct Service { + services: Services, db: Arc, interrupt: Notify, interval: Duration, } +struct Services { + admin: Dep, + client: Dep, +} + #[derive(Deserialize)] struct CheckForUpdatesResponse { updates: Vec, @@ -35,6 +41,10 @@ const LAST_CHECK_FOR_UPDATES_COUNT: &[u8] = b"u"; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + services: Services { + admin: args.depend::("admin"), + client: args.depend::("client"), + }, db: args.db["global"].clone(), interrupt: Notify::new(), interval: Duration::from_secs(CHECK_FOR_UPDATES_INTERVAL), @@ -63,7 +73,8 @@ impl crate::Service for Service { impl Service { #[tracing::instrument(skip_all)] async fn handle_updates(&self) -> Result<()> { - let response = services() + let response = self + .services .client .default .get(CHECK_FOR_UPDATES_URL) @@ -78,7 +89,7 @@ impl Service { last_update_id = last_update_id.max(update.id); if update.id > self.last_check_for_updates_id()? { info!("{:#}", update.message); - services() + self.services .admin .send_message(RoomMessageEventContent::text_markdown(format!( "### the following is a message from the conduwuit puppy\n\nit was sent on `{}`:\n\n@room: {}", diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 5546adb1..2dcde7ce 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, mem::size_of, sync::Arc}; -use conduit::{debug_info, err, utils, warn, Err, Error, Result}; -use database::{Database, Map}; +use conduit::{debug_info, err, utils, warn, Err, Error, Result, Server}; +use database::Map; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, @@ -11,52 +11,65 @@ use ruma::{ OwnedMxcUri, OwnedUserId, UInt, UserId, }; -use crate::{services, users::clean_signatures}; +use crate::{globals, rooms, users::clean_signatures, Dep}; pub struct Data { - userid_password: Arc, + keychangeid_userid: Arc, + keyid_key: Arc, + onetimekeyid_onetimekeys: Arc, + openidtoken_expiresatuserid: Arc, + todeviceid_events: Arc, token_userdeviceid: Arc, - userid_displayname: Arc, + userdeviceid_metadata: Arc, + userdeviceid_token: Arc, + userfilterid_filter: Arc, userid_avatarurl: Arc, userid_blurhash: Arc, userid_devicelistversion: Arc, - userdeviceid_token: Arc, - userdeviceid_metadata: Arc, - onetimekeyid_onetimekeys: Arc, + userid_displayname: Arc, userid_lastonetimekeyupdate: Arc, - keyid_key: Arc, userid_masterkeyid: Arc, + userid_password: Arc, userid_selfsigningkeyid: Arc, userid_usersigningkeyid: Arc, - openidtoken_expiresatuserid: Arc, - keychangeid_userid: Arc, - todeviceid_events: Arc, - userfilterid_filter: Arc, - _db: Arc, + services: Services, +} + +struct Services { + server: Arc, + globals: Dep, + state_cache: Dep, + state_accessor: Dep, } impl Data { - pub(super) fn new(db: Arc) -> Self { + pub(super) fn new(args: &crate::Args<'_>) -> Self { + let db = &args.db; Self { - userid_password: db["userid_password"].clone(), + keychangeid_userid: db["keychangeid_userid"].clone(), + keyid_key: db["keyid_key"].clone(), + onetimekeyid_onetimekeys: db["onetimekeyid_onetimekeys"].clone(), + openidtoken_expiresatuserid: db["openidtoken_expiresatuserid"].clone(), + todeviceid_events: db["todeviceid_events"].clone(), token_userdeviceid: db["token_userdeviceid"].clone(), - userid_displayname: db["userid_displayname"].clone(), + userdeviceid_metadata: db["userdeviceid_metadata"].clone(), + userdeviceid_token: db["userdeviceid_token"].clone(), + userfilterid_filter: db["userfilterid_filter"].clone(), userid_avatarurl: db["userid_avatarurl"].clone(), userid_blurhash: db["userid_blurhash"].clone(), userid_devicelistversion: db["userid_devicelistversion"].clone(), - userdeviceid_token: db["userdeviceid_token"].clone(), - userdeviceid_metadata: db["userdeviceid_metadata"].clone(), - onetimekeyid_onetimekeys: db["onetimekeyid_onetimekeys"].clone(), + userid_displayname: db["userid_displayname"].clone(), userid_lastonetimekeyupdate: db["userid_lastonetimekeyupdate"].clone(), - keyid_key: db["keyid_key"].clone(), userid_masterkeyid: db["userid_masterkeyid"].clone(), + userid_password: db["userid_password"].clone(), userid_selfsigningkeyid: db["userid_selfsigningkeyid"].clone(), userid_usersigningkeyid: db["userid_usersigningkeyid"].clone(), - openidtoken_expiresatuserid: db["openidtoken_expiresatuserid"].clone(), - keychangeid_userid: db["keychangeid_userid"].clone(), - todeviceid_events: db["todeviceid_events"].clone(), - userfilterid_filter: db["userfilterid_filter"].clone(), - _db: db, + services: Services { + server: args.server.clone(), + globals: args.depend::("globals"), + state_cache: args.depend::("rooms::state_cache"), + state_accessor: args.depend::("rooms::state_accessor"), + }, } } @@ -377,7 +390,7 @@ impl Data { )?; self.userid_lastonetimekeyupdate - .insert(user_id.as_bytes(), &services().globals.next_count()?.to_be_bytes())?; + .insert(user_id.as_bytes(), &self.services.globals.next_count()?.to_be_bytes())?; Ok(()) } @@ -403,7 +416,7 @@ impl Data { prefix.push(b':'); self.userid_lastonetimekeyupdate - .insert(user_id.as_bytes(), &services().globals.next_count()?.to_be_bytes())?; + .insert(user_id.as_bytes(), &self.services.globals.next_count()?.to_be_bytes())?; self.onetimekeyid_onetimekeys .scan_prefix(prefix) @@ -631,16 +644,16 @@ impl Data { } pub(super) fn mark_device_key_update(&self, user_id: &UserId) -> Result<()> { - let count = services().globals.next_count()?.to_be_bytes(); - for room_id in services() - .rooms + let count = self.services.globals.next_count()?.to_be_bytes(); + for room_id in self + .services .state_cache .rooms_joined(user_id) .filter_map(Result::ok) { // Don't send key updates to unencrypted rooms - if services() - .rooms + if self + .services .state_accessor .room_state_get(&room_id, &StateEventType::RoomEncryption, "")? .is_none() @@ -750,7 +763,7 @@ impl Data { key.push(0xFF); key.extend_from_slice(target_device_id.as_bytes()); key.push(0xFF); - key.extend_from_slice(&services().globals.next_count()?.to_be_bytes()); + key.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); let mut json = serde_json::Map::new(); json.insert("type".to_owned(), event_type.to_owned().into()); @@ -916,7 +929,7 @@ impl Data { pub(super) fn create_openid_token(&self, user_id: &UserId, token: &str) -> Result { use std::num::Saturating as Sat; - let expires_in = services().globals.config.openid_token_ttl; + let expires_in = self.services.server.config.openid_token_ttl; let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in) * Sat(1000); let mut value = expires_at.0.to_be_bytes().to_vec(); diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index e0a4dd1c..4c80d0d3 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -7,7 +7,6 @@ use std::{ }; use conduit::{Error, Result}; -use data::Data; use ruma::{ api::client::{ device::Device, @@ -24,7 +23,8 @@ use ruma::{ UInt, UserId, }; -use crate::services; +use self::data::Data; +use crate::{admin, rooms, Dep}; pub struct SlidingSyncCache { lists: BTreeMap, @@ -36,14 +36,24 @@ pub struct SlidingSyncCache { type DbConnections = Mutex>>>; pub struct Service { + services: Services, pub db: Data, pub connections: DbConnections, } +struct Services { + admin: Dep, + state_cache: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db.clone()), + services: Services { + admin: args.depend::("admin"), + state_cache: args.depend::("rooms::state_cache"), + }, + db: Data::new(&args), connections: StdMutex::new(BTreeMap::new()), })) } @@ -247,11 +257,8 @@ impl Service { /// Check if a user is an admin pub fn is_admin(&self, user_id: &UserId) -> Result { - if let Some(admin_room_id) = services().admin.get_admin_room()? { - services() - .rooms - .state_cache - .is_joined(user_id, &admin_room_id) + if let Some(admin_room_id) = self.services.admin.get_admin_room()? { + self.services.state_cache.is_joined(user_id, &admin_room_id) } else { Ok(false) } From 59efabbbc2feee08589a2118683d29dddb942186 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 22 Jul 2024 07:43:51 +0000 Subject: [PATCH 0564/2091] de-global server_is_ours / user_is_local Signed-off-by: Jason Volk --- src/admin/federation/commands.rs | 2 +- src/admin/mod.rs | 2 +- src/admin/room/room_commands.rs | 2 +- src/admin/room/room_directory_commands.rs | 2 +- src/admin/room/room_moderation_commands.rs | 10 +++---- src/admin/user/commands.rs | 32 ++++++++++++--------- src/admin/utils.rs | 30 +++++++++----------- src/api/client/account.rs | 13 +++------ src/api/client/alias.rs | 10 +++---- src/api/client/directory.rs | 8 ++---- src/api/client/keys.rs | 8 ++---- src/api/client/media.rs | 33 +++++++++++----------- src/api/client/membership.rs | 23 ++++++++------- src/api/client/profile.rs | 14 ++++----- src/api/client/state.rs | 10 +++---- src/api/client/to_device.rs | 5 ++-- src/api/mod.rs | 4 +-- src/api/server/invite.rs | 3 +- src/api/server/query.rs | 5 ++-- src/api/server/send_join.rs | 7 ++--- src/api/server/send_leave.rs | 7 +++-- src/api/server/user.rs | 18 ++++++++---- src/core/server.rs | 3 ++ src/service/admin/mod.rs | 4 +-- src/service/globals/mod.rs | 18 ++++++------ src/service/mod.rs | 5 +--- src/service/presence/mod.rs | 6 ++-- src/service/rooms/alias/mod.rs | 13 +++++++-- src/service/rooms/state_cache/data.rs | 4 +-- src/service/rooms/state_cache/mod.rs | 6 ++-- src/service/rooms/timeline/mod.rs | 12 ++++---- src/service/rooms/typing/mod.rs | 14 +++++---- src/service/sending/mod.rs | 8 +++--- src/service/sending/sender.rs | 7 ++--- 34 files changed, 179 insertions(+), 169 deletions(-) diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 331231ae..d6ecd3f7 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -90,7 +90,7 @@ pub(super) async fn remote_user_in_rooms(_body: Vec<&str>, user_id: Box) .state_cache .rooms_joined(&user_id) .filter_map(Result::ok) - .map(|room_id| get_room_info(&room_id)) + .map(|room_id| get_room_info(services(), &room_id)) .collect(); if rooms.is_empty() { diff --git a/src/admin/mod.rs b/src/admin/mod.rs index c57659c1..b183f3f6 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -19,7 +19,7 @@ extern crate conduit_core as conduit; extern crate conduit_service as service; pub(crate) use conduit::{mod_ctor, mod_dtor, Result}; -pub(crate) use service::{services, user_is_local}; +pub(crate) use service::services; pub(crate) use crate::utils::{escape_html, get_room_info}; diff --git a/src/admin/room/room_commands.rs b/src/admin/room/room_commands.rs index d47edce2..1a387c7e 100644 --- a/src/admin/room/room_commands.rs +++ b/src/admin/room/room_commands.rs @@ -39,7 +39,7 @@ pub(super) async fn list( true }) - .map(|room_id| get_room_info(&room_id)) + .map(|room_id| get_room_info(services(), &room_id)) }) .collect::>(); rooms.sort_by_key(|r| r.1); diff --git a/src/admin/room/room_directory_commands.rs b/src/admin/room/room_directory_commands.rs index 5e81bd89..c9b4eb9e 100644 --- a/src/admin/room/room_directory_commands.rs +++ b/src/admin/room/room_directory_commands.rs @@ -29,7 +29,7 @@ pub(super) async fn process(command: RoomDirectoryCommand, _body: Vec<&str>) -> .directory .public_rooms() .filter_map(Result::ok) - .map(|id: OwnedRoomId| get_room_info(&id)) + .map(|id: OwnedRoomId| get_room_info(services(), &id)) .collect::>(); rooms.sort_by_key(|r| r.1); rooms.reverse(); diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/room_moderation_commands.rs index 46354c0f..8ad8295b 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/room_moderation_commands.rs @@ -1,9 +1,9 @@ use api::client::leave_room; +use conduit::{debug, error, info, warn, Result}; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId}; -use tracing::{debug, error, info, warn}; use super::RoomModerationCommand; -use crate::{get_room_info, services, user_is_local, Result}; +use crate::{get_room_info, services}; pub(super) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> Result { match command { @@ -110,11 +110,11 @@ async fn ban_room( .room_members(&room_id) .filter_map(|user| { user.ok().filter(|local_user| { - user_is_local(local_user) + services().globals.user_is_local(local_user) // additional wrapped check here is to avoid adding remote users // who are in the admin room to the list of local users (would // fail auth check) - && (user_is_local(local_user) + && (services().globals.user_is_local(local_user) // since this is a force operation, assume user is an admin // if somehow this fails && services() @@ -484,7 +484,7 @@ async fn list_banned_rooms(_body: Vec<&str>) -> Result let mut rooms = room_ids .into_iter() - .map(|room_id| get_room_info(&room_id)) + .map(|room_id| get_room_info(services(), &room_id)) .collect::>(); rooms.sort_by_key(|r| r.1); rooms.reverse(); diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index fa1c5288..69019d79 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -36,7 +36,7 @@ pub(super) async fn create( _body: Vec<&str>, username: String, password: Option, ) -> Result { // Validate user id - let user_id = parse_local_user_id(&username)?; + let user_id = parse_local_user_id(services(), &username)?; if services().users.exists(&user_id)? { return Ok(RoomMessageEventContent::text_plain(format!("Userid {user_id} already exists"))); @@ -134,7 +134,7 @@ pub(super) async fn deactivate( _body: Vec<&str>, no_leave_rooms: bool, user_id: String, ) -> Result { // Validate user id - let user_id = parse_local_user_id(&user_id)?; + let user_id = parse_local_user_id(services(), &user_id)?; // don't deactivate the server service account if user_id == services().globals.server_user { @@ -170,7 +170,7 @@ pub(super) async fn deactivate( } pub(super) async fn reset_password(_body: Vec<&str>, username: String) -> Result { - let user_id = parse_local_user_id(&username)?; + let user_id = parse_local_user_id(services(), &username)?; if user_id == services().globals.server_user { return Ok(RoomMessageEventContent::text_plain( @@ -211,7 +211,7 @@ pub(super) async fn deactivate_all( let mut admins = Vec::new(); for username in usernames { - match parse_active_local_user_id(username) { + match parse_active_local_user_id(services(), username) { Ok(user_id) => { if services().users.is_admin(&user_id)? && !force { services() @@ -292,14 +292,14 @@ pub(super) async fn deactivate_all( pub(super) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Result { // Validate user id - let user_id = parse_local_user_id(&user_id)?; + let user_id = parse_local_user_id(services(), &user_id)?; let mut rooms: Vec<(OwnedRoomId, u64, String)> = services() .rooms .state_cache .rooms_joined(&user_id) .filter_map(Result::ok) - .map(|room_id| get_room_info(&room_id)) + .map(|room_id| get_room_info(services(), &room_id)) .collect(); if rooms.is_empty() { @@ -344,10 +344,13 @@ pub(super) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Resu pub(super) async fn force_join_room( _body: Vec<&str>, user_id: String, room_id: OwnedRoomOrAliasId, ) -> Result { - let user_id = parse_local_user_id(&user_id)?; + let user_id = parse_local_user_id(services(), &user_id)?; let room_id = services().rooms.alias.resolve(&room_id).await?; - assert!(service::user_is_local(&user_id), "Parsed user_id must be a local user"); + assert!( + services().globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); join_room_by_id_helper(services(), &user_id, &room_id, None, &[], None).await?; Ok(RoomMessageEventContent::notice_markdown(format!( @@ -356,13 +359,16 @@ pub(super) async fn force_join_room( } pub(super) async fn make_user_admin(_body: Vec<&str>, user_id: String) -> Result { - let user_id = parse_local_user_id(&user_id)?; + let user_id = parse_local_user_id(services(), &user_id)?; let displayname = services() .users .displayname(&user_id)? .unwrap_or_else(|| user_id.to_string()); - assert!(service::user_is_local(&user_id), "Parsed user_id must be a local user"); + assert!( + services().globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); services() .admin .make_user_admin(&user_id, displayname) @@ -376,7 +382,7 @@ pub(super) async fn make_user_admin(_body: Vec<&str>, user_id: String) -> Result pub(super) async fn put_room_tag( _body: Vec<&str>, user_id: String, room_id: Box, tag: String, ) -> Result { - let user_id = parse_active_local_user_id(&user_id)?; + let user_id = parse_active_local_user_id(services(), &user_id)?; let event = services() .account_data @@ -411,7 +417,7 @@ pub(super) async fn put_room_tag( pub(super) async fn delete_room_tag( _body: Vec<&str>, user_id: String, room_id: Box, tag: String, ) -> Result { - let user_id = parse_active_local_user_id(&user_id)?; + let user_id = parse_active_local_user_id(services(), &user_id)?; let event = services() .account_data @@ -443,7 +449,7 @@ pub(super) async fn delete_room_tag( pub(super) async fn get_room_tags( _body: Vec<&str>, user_id: String, room_id: Box, ) -> Result { - let user_id = parse_active_local_user_id(&user_id)?; + let user_id = parse_active_local_user_id(services(), &user_id)?; let event = services() .account_data diff --git a/src/admin/utils.rs b/src/admin/utils.rs index fda42e6e..8d3d15ae 100644 --- a/src/admin/utils.rs +++ b/src/admin/utils.rs @@ -1,8 +1,6 @@ -use conduit_core::{err, Err}; +use conduit_core::{err, Err, Result}; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; -use service::user_is_local; - -use crate::{services, Result}; +use service::Services; pub(crate) fn escape_html(s: &str) -> String { s.replace('&', "&") @@ -10,17 +8,17 @@ pub(crate) fn escape_html(s: &str) -> String { .replace('>', ">") } -pub(crate) fn get_room_info(id: &RoomId) -> (OwnedRoomId, u64, String) { +pub(crate) fn get_room_info(services: &Services, id: &RoomId) -> (OwnedRoomId, u64, String) { ( id.into(), - services() + services .rooms .state_cache .room_joined_count(id) .ok() .flatten() .unwrap_or(0), - services() + services .rooms .state_accessor .get_name(id) @@ -31,16 +29,16 @@ pub(crate) fn get_room_info(id: &RoomId) -> (OwnedRoomId, u64, String) { } /// Parses user ID -pub(crate) fn parse_user_id(user_id: &str) -> Result { - UserId::parse_with_server_name(user_id.to_lowercase(), services().globals.server_name()) +pub(crate) fn parse_user_id(services: &Services, user_id: &str) -> Result { + UserId::parse_with_server_name(user_id.to_lowercase(), services.globals.server_name()) .map_err(|e| err!("The supplied username is not a valid username: {e}")) } /// Parses user ID as our local user -pub(crate) fn parse_local_user_id(user_id: &str) -> Result { - let user_id = parse_user_id(user_id)?; +pub(crate) fn parse_local_user_id(services: &Services, user_id: &str) -> Result { + let user_id = parse_user_id(services, user_id)?; - if !user_is_local(&user_id) { + if !services.globals.user_is_local(&user_id) { return Err!("User {user_id:?} does not belong to our server."); } @@ -48,14 +46,14 @@ pub(crate) fn parse_local_user_id(user_id: &str) -> Result { } /// Parses user ID that is an active (not guest or deactivated) local user -pub(crate) fn parse_active_local_user_id(user_id: &str) -> Result { - let user_id = parse_local_user_id(user_id)?; +pub(crate) fn parse_active_local_user_id(services: &Services, user_id: &str) -> Result { + let user_id = parse_local_user_id(services, user_id)?; - if !services().users.exists(&user_id)? { + if !services.users.exists(&user_id)? { return Err!("User {user_id:?} does not exist on this server."); } - if services().users.is_deactivated(&user_id)? { + if services.users.is_deactivated(&user_id)? { return Err!("User {user_id:?} is deactivated."); } diff --git a/src/api/client/account.rs b/src/api/client/account.rs index b3495b42..7c2bb0b6 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -2,7 +2,7 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::debug_info; +use conduit::{debug_info, error, info, utils, warn, Error, Result}; use register::RegistrationKind; use ruma::{ api::client::{ @@ -18,14 +18,9 @@ use ruma::{ events::{room::message::RoomMessageEventContent, GlobalAccountDataEventType}, push, OwnedRoomId, UserId, }; -use tracing::{error, info, warn}; use super::{join_room_by_id_helper, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; -use crate::{ - service::user_is_local, - utils::{self}, - Error, Result, Ruma, -}; +use crate::Ruma; const RANDOM_USER_ID_LENGTH: usize = 10; @@ -48,7 +43,7 @@ pub(crate) async fn get_register_available_route( // Validate user id let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), services.globals.server_name()) .ok() - .filter(|user_id| !user_id.is_historical() && user_is_local(user_id)) + .filter(|user_id| !user_id.is_historical() && services.globals.user_is_local(user_id)) .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; // Check if username is creative enough @@ -136,7 +131,7 @@ pub(crate) async fn register_route( let proposed_user_id = UserId::parse_with_server_name(username.to_lowercase(), services.globals.server_name()) .ok() - .filter(|user_id| !user_id.is_historical() && user_is_local(user_id)) + .filter(|user_id| !user_id.is_historical() && services.globals.user_is_local(user_id)) .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; if services.users.exists(&proposed_user_id)? { diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 11617a0e..dbc75e64 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -1,4 +1,5 @@ use axum::extract::State; +use conduit::{debug, Error, Result}; use rand::seq::SliceRandom; use ruma::{ api::client::{ @@ -7,12 +8,9 @@ use ruma::{ }, OwnedServerName, RoomAliasId, RoomId, }; -use tracing::debug; +use service::Services; -use crate::{ - service::{server_is_ours, Services}, - Error, Result, Ruma, -}; +use crate::Ruma; /// # `PUT /_matrix/client/v3/directory/room/{roomAlias}` /// @@ -142,7 +140,7 @@ fn room_available_servers( // prefer the room alias server first if let Some(server_index) = servers .iter() - .position(|server_name| server_is_ours(server_name)) + .position(|server_name| services.globals.server_is_ours(server_name)) { servers.swap_remove(server_index); servers.insert(0, services.globals.server_name().to_owned()); diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index deebd250..cb30b60a 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -20,11 +20,9 @@ use ruma::{ }, uint, RoomId, ServerName, UInt, UserId, }; +use service::Services; -use crate::{ - service::{server_is_ours, Services}, - Ruma, -}; +use crate::Ruma; /// # `POST /_matrix/client/v3/publicRooms` /// @@ -187,7 +185,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( services: &Services, server: Option<&ServerName>, limit: Option, since: Option<&str>, filter: &Filter, _network: &RoomNetwork, ) -> Result { - if let Some(other_server) = server.filter(|server_name| !server_is_ours(server_name)) { + if let Some(other_server) = server.filter(|server_name| !services.globals.server_is_ours(server_name)) { let response = services .sending .send_federation_request( diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 728ea7a9..8489dde3 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -4,7 +4,7 @@ use std::{ }; use axum::extract::State; -use conduit::{utils, utils::math::continue_exponential_backoff_secs, Error, Result}; +use conduit::{debug, utils, utils::math::continue_exponential_backoff_secs, Error, Result}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::{ @@ -19,8 +19,6 @@ use ruma::{ DeviceKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, }; use serde_json::json; -use service::user_is_local; -use tracing::debug; use super::SESSION_ID_LENGTH; use crate::{service::Services, Ruma}; @@ -266,7 +264,7 @@ pub(crate) async fn get_keys_helper bool + Send>( for (user_id, device_ids) in device_keys_input { let user_id: &UserId = user_id; - if !user_is_local(user_id) { + if !services.globals.user_is_local(user_id) { get_over_federation .entry(user_id.server_name()) .or_insert_with(Vec::new) @@ -459,7 +457,7 @@ pub(crate) async fn claim_keys_helper( let mut get_over_federation = BTreeMap::new(); for (user_id, map) in one_time_keys_input { - if !user_is_local(user_id) { + if !services.globals.user_is_local(user_id) { get_over_federation .entry(user_id.server_name()) .or_insert_with(Vec::new) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index e219e457..78463fc6 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -4,7 +4,15 @@ use std::{io::Cursor, time::Duration}; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{debug, error, utils::math::ruma_from_usize, warn}; +use conduit::{ + debug, debug_warn, error, + utils::{ + self, + content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename}, + math::ruma_from_usize, + }, + warn, Error, Result, +}; use image::io::Reader as ImgReader; use ipaddress::IPAddress; use reqwest::Url; @@ -15,20 +23,13 @@ use ruma::api::client::{ get_media_preview, }, }; +use service::{ + media::{FileMeta, UrlPreviewData}, + Services, +}; use webpage::HTML; -use crate::{ - debug_warn, - service::{ - media::{FileMeta, UrlPreviewData}, - server_is_ours, Services, - }, - utils::{ - self, - content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename}, - }, - Error, Result, Ruma, RumaResponse, -}; +use crate::{Ruma, RumaResponse}; /// generated MXC ID (`media-id`) length const MXC_LENGTH: usize = 32; @@ -218,7 +219,7 @@ pub(crate) async fn get_content_route( cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) - } else if !server_is_ours(&body.server_name) && body.allow_remote { + } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { let response = get_remote_content( services, &mxc, @@ -308,7 +309,7 @@ pub(crate) async fn get_content_as_filename_route( cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) - } else if !server_is_ours(&body.server_name) && body.allow_remote { + } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { match get_remote_content( services, &mxc, @@ -408,7 +409,7 @@ pub(crate) async fn get_content_thumbnail_route( cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), content_disposition, }) - } else if !server_is_ours(&body.server_name) && body.allow_remote { + } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { if services .globals .prevent_media_downloads_from() diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index e3630c72..d3b2d8f6 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -8,8 +8,11 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - debug, debug_warn, error, info, trace, utils, utils::math::continue_exponential_backoff_secs, warn, Error, - PduEvent, Result, + debug, debug_warn, error, info, + pdu::{gen_event_id_canonical_json, PduBuilder}, + trace, utils, + utils::math::continue_exponential_backoff_secs, + warn, Error, PduEvent, Result, }; use ruma::{ api::{ @@ -36,15 +39,11 @@ use ruma::{ OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use service::{rooms::state::RoomMutexGuard, Services}; use tokio::sync::RwLock; use crate::{ client::{update_avatar_url, update_displayname}, - service::{ - pdu::{gen_event_id_canonical_json, PduBuilder}, - rooms::state::RoomMutexGuard, - server_is_ours, user_is_local, Services, - }, Ruma, }; @@ -675,7 +674,7 @@ pub async fn join_room_by_id_helper( .state_cache .server_in_room(services.globals.server_name(), room_id)? || servers.is_empty() - || (servers.len() == 1 && server_is_ours(&servers[0])) + || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])) { join_room_by_id_helper_local(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) .await @@ -1049,7 +1048,7 @@ async fn join_room_by_id_helper_local( .state_cache .room_members(room_id) .filter_map(Result::ok) - .filter(|user| user_is_local(user)) + .filter(|user| services.globals.user_is_local(user)) .collect::>(); let mut join_authorized_via_users_server: Option = None; @@ -1110,7 +1109,7 @@ async fn join_room_by_id_helper_local( if !restriction_rooms.is_empty() && servers .iter() - .any(|server_name| !server_is_ours(server_name)) + .any(|server_name| !services.globals.server_is_ours(server_name)) { warn!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); let (make_join_response, remote_server) = make_join_request(services, sender_user, room_id, servers).await?; @@ -1259,7 +1258,7 @@ async fn make_join_request( let mut incompatible_room_version_count: u8 = 0; for remote_server in servers { - if server_is_ours(remote_server) { + if services.globals.server_is_ours(remote_server) { continue; } info!("Asking {remote_server} for make_join ({make_join_counter})"); @@ -1389,7 +1388,7 @@ pub(crate) async fn invite_helper( )); } - if !user_is_local(user_id) { + if !services.globals.user_is_local(user_id) { let (pdu, pdu_json, invite_room_state) = { let state_lock = services.rooms.state.mutex.lock(room_id).await; let content = to_raw_value(&RoomMemberEventContent { diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 3b2c32ec..9e9bcf8e 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -1,4 +1,5 @@ use axum::extract::State; +use conduit::{pdu::PduBuilder, warn, Error, Result}; use ruma::{ api::{ client::{ @@ -12,12 +13,9 @@ use ruma::{ OwnedMxcUri, OwnedRoomId, OwnedUserId, }; use serde_json::value::to_raw_value; -use tracing::warn; +use service::Services; -use crate::{ - service::{pdu::PduBuilder, user_is_local, Services}, - Error, Result, Ruma, -}; +use crate::Ruma; /// # `PUT /_matrix/client/r0/profile/{userId}/displayname` /// @@ -56,7 +54,7 @@ pub(crate) async fn set_displayname_route( pub(crate) async fn get_displayname_route( State(services): State, body: Ruma, ) -> Result { - if !user_is_local(&body.user_id) { + if !services.globals.user_is_local(&body.user_id) { // Create and update our local copy of the user if let Ok(response) = services .sending @@ -147,7 +145,7 @@ pub(crate) async fn set_avatar_url_route( pub(crate) async fn get_avatar_url_route( State(services): State, body: Ruma, ) -> Result { - if !user_is_local(&body.user_id) { + if !services.globals.user_is_local(&body.user_id) { // Create and update our local copy of the user if let Ok(response) = services .sending @@ -205,7 +203,7 @@ pub(crate) async fn get_avatar_url_route( pub(crate) async fn get_profile_route( State(services): State, body: Ruma, ) -> Result { - if !user_is_local(&body.user_id) { + if !services.globals.user_is_local(&body.user_id) { // Create and update our local copy of the user if let Ok(response) = services .sending diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 56ffd2ac..7af4f5f9 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use axum::extract::State; -use conduit::{debug_info, error}; +use conduit::{debug_info, error, pdu::PduBuilder, Error, Result}; use ruma::{ api::client::{ error::ErrorKind, @@ -18,11 +18,9 @@ use ruma::{ serde::Raw, EventId, RoomId, UserId, }; +use service::Services; -use crate::{ - service::{pdu::PduBuilder, server_is_ours, Services}, - Error, Result, Ruma, RumaResponse, -}; +use crate::{Ruma, RumaResponse}; /// # `PUT /_matrix/client/*/rooms/{roomId}/state/{eventType}/{stateKey}` /// @@ -250,7 +248,7 @@ async fn allowed_to_send_state_event( } for alias in aliases { - if !server_is_ours(alias.server_name()) + if !services.globals.server_is_ours(alias.server_name()) || services .rooms .alias diff --git a/src/api/client/to_device.rs b/src/api/client/to_device.rs index 8476ff41..1f557ad7 100644 --- a/src/api/client/to_device.rs +++ b/src/api/client/to_device.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; +use conduit::{Error, Result}; use ruma::{ api::{ client::{error::ErrorKind, to_device::send_event_to_device}, @@ -9,7 +10,7 @@ use ruma::{ to_device::DeviceIdOrAllDevices, }; -use crate::{user_is_local, Error, Result, Ruma}; +use crate::Ruma; /// # `PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}` /// @@ -31,7 +32,7 @@ pub(crate) async fn send_event_to_device_route( for (target_user_id, map) in &body.messages { for (target_device_id_maybe, event) in map { - if !user_is_local(target_user_id) { + if !services.globals.user_is_local(target_user_id) { let mut map = BTreeMap::new(); map.insert(target_device_id_maybe.clone(), event.clone()); let mut messages = BTreeMap::new(); diff --git a/src/api/mod.rs b/src/api/mod.rs index 0d80e581..c7411b6c 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,8 +7,8 @@ pub mod server; extern crate conduit_core as conduit; extern crate conduit_service as service; -pub(crate) use conduit::{debug_info, debug_warn, pdu::PduEvent, utils, Error, Result}; -pub(crate) use service::{services, user_is_local}; +pub(crate) use conduit::{debug_info, pdu::PduEvent, utils, Error, Result}; +pub(crate) use service::services; pub use crate::router::State; pub(crate) use crate::router::{Ruma, RumaResponse}; diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 17e21920..688e026c 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -7,7 +7,6 @@ use ruma::{ serde::JsonObject, CanonicalJsonValue, EventId, OwnedUserId, }; -use service::server_is_ours; use crate::Ruma; @@ -88,7 +87,7 @@ pub(crate) async fn create_invite_route( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "state_key is not a user ID."))?; - if !server_is_ours(invited_user.server_name()) { + if !services.globals.server_is_ours(invited_user.server_name()) { return Err(Error::BadRequest( ErrorKind::InvalidParam, "User does not belong to this homeserver.", diff --git a/src/api/server/query.rs b/src/api/server/query.rs index dddf23e7..5712f46a 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -1,4 +1,5 @@ use axum::extract::State; +use conduit::{Error, Result}; use get_profile_information::v1::ProfileField; use rand::seq::SliceRandom; use ruma::{ @@ -9,7 +10,7 @@ use ruma::{ OwnedServerName, }; -use crate::{service::server_is_ours, Error, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/query/directory` /// @@ -64,7 +65,7 @@ pub(crate) async fn get_profile_information_route( )); } - if !server_is_ours(body.user_id.server_name()) { + if !services.globals.server_is_ours(body.user_id.server_name()) { return Err(Error::BadRequest( ErrorKind::InvalidParam, "User does not belong to this server.", diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 7f79a1d9..4cd29795 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{Error, Result}; +use conduit::{pdu::gen_event_id_canonical_json, warn, Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_join_event}, events::{ @@ -13,9 +13,8 @@ use ruma::{ CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use service::{pdu::gen_event_id_canonical_json, user_is_local, Services}; +use service::Services; use tokio::sync::RwLock; -use tracing::warn; use crate::Ruma; @@ -126,7 +125,7 @@ async fn create_join_event( if content .join_authorized_via_users_server - .is_some_and(|user| user_is_local(&user)) + .is_some_and(|user| services.globals.user_is_local(&user)) && super::user_can_perform_restricted_join(services, &sender, room_id, &room_version_id).unwrap_or_default() { ruma::signatures::hash_and_sign_event( diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index f7484a33..b1b8fec8 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use axum::extract::State; +use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_leave_event}, events::{ @@ -15,8 +16,8 @@ use serde_json::value::RawValue as RawJsonValue; use tokio::sync::RwLock; use crate::{ - service::{pdu::gen_event_id_canonical_json, server_is_ours, Services}, - Error, Result, Ruma, + service::{pdu::gen_event_id_canonical_json, Services}, + Ruma, }; /// # `PUT /_matrix/federation/v1/send_leave/{roomId}/{eventId}` @@ -174,7 +175,7 @@ async fn create_leave_event( .state_cache .room_servers(room_id) .filter_map(Result::ok) - .filter(|server| !server_is_ours(server)); + .filter(|server| !services.globals.server_is_ours(server)); services.sending.send_pdu_servers(servers, &pdu_id)?; diff --git a/src/api/server/user.rs b/src/api/server/user.rs index 949e5f38..bd0372e6 100644 --- a/src/api/server/user.rs +++ b/src/api/server/user.rs @@ -1,4 +1,5 @@ use axum::extract::State; +use conduit::{Error, Result}; use ruma::api::{ client::error::ErrorKind, federation::{ @@ -9,8 +10,7 @@ use ruma::api::{ use crate::{ client::{claim_keys_helper, get_keys_helper}, - service::user_is_local, - Error, Result, Ruma, + Ruma, }; /// # `GET /_matrix/federation/v1/user/devices/{userId}` @@ -19,7 +19,7 @@ use crate::{ pub(crate) async fn get_devices_route( State(services): State, body: Ruma, ) -> Result { - if !user_is_local(&body.user_id) { + if !services.globals.user_is_local(&body.user_id) { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Tried to access user from other server.", @@ -72,7 +72,11 @@ pub(crate) async fn get_devices_route( pub(crate) async fn get_keys_route( State(services): State, body: Ruma, ) -> Result { - if body.device_keys.iter().any(|(u, _)| !user_is_local(u)) { + if body + .device_keys + .iter() + .any(|(u, _)| !services.globals.user_is_local(u)) + { return Err(Error::BadRequest( ErrorKind::InvalidParam, "User does not belong to this server.", @@ -101,7 +105,11 @@ pub(crate) async fn get_keys_route( pub(crate) async fn claim_keys_route( State(services): State, body: Ruma, ) -> Result { - if body.one_time_keys.iter().any(|(u, _)| !user_is_local(u)) { + if body + .one_time_keys + .iter() + .any(|(u, _)| !services.globals.user_is_local(u)) + { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Tried to access user from other server.", diff --git a/src/core/server.rs b/src/core/server.rs index bf0ab99d..89f1dea5 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -109,4 +109,7 @@ impl Server { #[inline] pub fn running(&self) -> bool { !self.stopping.load(Ordering::Acquire) } + + #[inline] + pub fn is_ours(&self, name: &str) -> bool { name == self.config.server_name } } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 8b9473a2..1d51bf38 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -22,7 +22,7 @@ use ruma::{ use serde_json::value::to_raw_value; use tokio::sync::{Mutex, RwLock}; -use crate::{globals, rooms, rooms::state::RoomMutexGuard, user_is_local, Dep}; +use crate::{globals, rooms, rooms::state::RoomMutexGuard, Dep}; pub struct Service { services: Services, @@ -301,7 +301,7 @@ impl Service { } // only allow public escaped commands by local admins - if is_public_escape && !user_is_local(&pdu.sender) { + if is_public_escape && !self.services.globals.user_is_local(&pdu.sender) { return false; } diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index eab156ee..c2c7d969 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -21,7 +21,7 @@ use ruma::{ use tokio::sync::Mutex; use url::Url; -use crate::{service, services}; +use crate::service; pub struct Service { pub db: Data, @@ -302,13 +302,11 @@ impl Service { true } + + /// checks if `user_id` is local to us via server_name comparison + #[inline] + pub fn user_is_local(&self, user_id: &UserId) -> bool { self.server_is_ours(user_id.server_name()) } + + #[inline] + pub fn server_is_ours(&self, server_name: &ServerName) -> bool { server_name == self.config.server_name } } - -#[inline] -#[must_use] -pub fn server_is_ours(server_name: &ServerName) -> bool { server_name == services().globals.config.server_name } - -/// checks if `user_id` is local to us via server_name comparison -#[inline] -#[must_use] -pub fn user_is_local(user_id: &UserId) -> bool { server_is_ours(user_id.server_name()) } diff --git a/src/service/mod.rs b/src/service/mod.rs index 6e749c99..ce106809 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -33,10 +33,7 @@ use conduit::{Result, Server}; use database::Database; pub(crate) use service::{Args, Dep, Service}; -pub use crate::{ - globals::{server_is_ours, user_is_local}, - services::Services, -}; +pub use crate::services::Services; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 705ac4ff..b8673304 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; use tokio::{sync::Mutex, time::sleep}; use self::data::Data; -use crate::{user_is_local, users, Dep}; +use crate::{globals, users, Dep}; /// Represents data required to be kept in order to implement the presence /// specification. @@ -80,6 +80,7 @@ pub struct Service { struct Services { server: Arc, + globals: Dep, users: Dep, } @@ -93,6 +94,7 @@ impl crate::Service for Service { Ok(Arc::new(Self { services: Services { server: args.server.clone(), + globals: args.depend::("globals"), users: args.depend::("users"), }, db: Data::new(&args), @@ -185,7 +187,7 @@ impl Service { self.db .set_presence(user_id, presence_state, currently_active, last_active_ago, status_msg)?; - if self.timeout_remote_users || user_is_local(user_id) { + if self.timeout_remote_users || self.services.globals.user_is_local(user_id) { let timeout = match presence_state { PresenceState::Online => self.services.server.config.presence_idle_timeout_s, _ => self.services.server.config.presence_offline_timeout_s, diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 344ab6d2..f2e01ab5 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -14,7 +14,7 @@ use ruma::{ }; use self::data::Data; -use crate::{admin, appservice, appservice::RegistrationInfo, globals, rooms, sending, server_is_ours, Dep}; +use crate::{admin, appservice, appservice::RegistrationInfo, globals, rooms, sending, Dep}; pub struct Service { db: Data, @@ -85,7 +85,10 @@ impl Service { pub async fn resolve_alias( &self, room_alias: &RoomAliasId, servers: Option<&Vec>, ) -> Result<(OwnedRoomId, Option>)> { - if !server_is_ours(room_alias.server_name()) + if !self + .services + .globals + .server_is_ours(room_alias.server_name()) && (!servers .as_ref() .is_some_and(|servers| servers.contains(&self.services.globals.server_name().to_owned())) @@ -195,7 +198,11 @@ impl Service { pub async fn appservice_checks( &self, room_alias: &RoomAliasId, appservice_info: &Option, ) -> Result<()> { - if !server_is_ours(room_alias.server_name()) { + if !self + .services + .globals + .server_is_ours(room_alias.server_name()) + { return Err(Error::BadRequest(ErrorKind::InvalidParam, "Alias is from another server.")); } diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index cbda73cf..19c73ea1 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -12,7 +12,7 @@ use ruma::{ OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; -use crate::{appservice::RegistrationInfo, globals, user_is_local, users, Dep}; +use crate::{appservice::RegistrationInfo, globals, users, Dep}; type StrippedStateEventIter<'a> = Box>)>> + 'a>; type AnySyncStateEventIter<'a> = Box>)>> + 'a>; @@ -355,7 +355,7 @@ impl Data { Box::new( self.room_members(room_id) .filter_map(Result::ok) - .filter(|user| user_is_local(user)), + .filter(|user| self.services.globals.user_is_local(user)), ) } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index ac2f688e..71899ceb 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -21,7 +21,7 @@ use ruma::{ OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; -use crate::{account_data, appservice::RegistrationInfo, rooms, user_is_local, users, Dep}; +use crate::{account_data, appservice::RegistrationInfo, globals, rooms, users, Dep}; pub struct Service { services: Services, @@ -30,6 +30,7 @@ pub struct Service { struct Services { account_data: Dep, + globals: Dep, state_accessor: Dep, users: Dep, } @@ -39,6 +40,7 @@ impl crate::Service for Service { Ok(Arc::new(Self { services: Services { account_data: args.depend::("account_data"), + globals: args.depend::("globals"), state_accessor: args.depend::("rooms::state_accessor"), users: args.depend::("users"), }, @@ -65,7 +67,7 @@ impl Service { // TODO: use futures to update remote profiles without blocking the membership // update #[allow(clippy::collapsible_if)] - if !user_is_local(user_id) { + if !self.services.globals.user_is_local(user_id) { if !self.services.users.exists(user_id)? { self.services.users.create(user_id, None)?; } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 50d29475..82b1cd0c 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -41,7 +41,7 @@ use tokio::sync::RwLock; use self::data::Data; use crate::{ account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms, - rooms::state_compressor::CompressedStateEvent, sending, server_is_ours, Dep, + rooms::state_compressor::CompressedStateEvent, sending, Dep, }; // Update Relationships @@ -846,7 +846,7 @@ impl Service { .state_cache .room_members(room_id) .filter_map(Result::ok) - .filter(|m| server_is_ours(m.server_name()) && m != target) + .filter(|m| self.services.globals.server_is_ours(m.server_name()) && m != target) .count(); if count < 2 { warn!("Last admin cannot leave from admins room"); @@ -871,7 +871,7 @@ impl Service { .state_cache .room_members(room_id) .filter_map(Result::ok) - .filter(|m| server_is_ours(m.server_name()) && m != target) + .filter(|m| self.services.globals.server_is_ours(m.server_name()) && m != target) .count(); if count < 2 { warn!("Last admin cannot be banned in admins room"); @@ -1092,7 +1092,7 @@ impl Service { .unwrap_or_default(); let room_mods = power_levels.users.iter().filter_map(|(user_id, level)| { - if level > &power_levels.users_default && !server_is_ours(user_id.server_name()) { + if level > &power_levels.users_default && !self.services.globals.user_is_local(user_id) { Some(user_id.server_name().to_owned()) } else { None @@ -1106,7 +1106,7 @@ impl Service { .filter_map(|alias| { alias .ok() - .filter(|alias| !server_is_ours(alias.server_name())) + .filter(|alias| !self.services.globals.server_is_ours(alias.server_name())) .map(|alias| alias.server_name().to_owned()) }); @@ -1114,7 +1114,7 @@ impl Service { .chain(room_alias_servers) .chain(self.services.server.config.trusted_servers.clone()) .filter(|server_name| { - if server_is_ours(server_name) { + if self.services.globals.server_is_ours(server_name) { return false; } diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index d863f217..3cf1cdd5 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -8,7 +8,7 @@ use ruma::{ }; use tokio::sync::{broadcast, RwLock}; -use crate::{globals, sending, user_is_local, Dep}; +use crate::{globals, sending, Dep}; pub struct Service { server: Arc, @@ -63,7 +63,7 @@ impl Service { } // update federation - if user_is_local(user_id) { + if self.services.globals.user_is_local(user_id) { self.federation_send(room_id, user_id, true)?; } @@ -89,7 +89,7 @@ impl Service { } // update federation - if user_is_local(user_id) { + if self.services.globals.user_is_local(user_id) { self.federation_send(room_id, user_id, false)?; } @@ -145,7 +145,7 @@ impl Service { // update federation for user in removable { - if user_is_local(&user) { + if self.services.globals.user_is_local(&user) { self.federation_send(room_id, &user, false)?; } } @@ -184,7 +184,11 @@ impl Service { } fn federation_send(&self, room_id: &RoomId, user_id: &UserId, typing: bool) -> Result<()> { - debug_assert!(user_is_local(user_id), "tried to broadcast typing status of remote user",); + debug_assert!( + self.services.globals.user_is_local(user_id), + "tried to broadcast typing status of remote user", + ); + if !self.server.config.allow_outgoing_typing { return Ok(()); } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 6f091b04..fc32d04f 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -13,7 +13,7 @@ use ruma::{ }; use tokio::sync::Mutex; -use crate::{account_data, client, globals, presence, pusher, resolver, rooms, server_is_ours, users, Dep}; +use crate::{account_data, client, globals, presence, pusher, resolver, rooms, users, Dep}; pub struct Service { server: Arc, @@ -136,7 +136,7 @@ impl Service { .state_cache .room_servers(room_id) .filter_map(Result::ok) - .filter(|server_name| !server_is_ours(server_name)); + .filter(|server_name| !self.services.globals.server_is_ours(server_name)); self.send_pdu_servers(servers, pdu_id) } @@ -185,7 +185,7 @@ impl Service { .state_cache .room_servers(room_id) .filter_map(Result::ok) - .filter(|server_name| !server_is_ours(server_name)); + .filter(|server_name| !self.services.globals.server_is_ours(server_name)); self.send_edu_servers(servers, serialized) } @@ -222,7 +222,7 @@ impl Service { .state_cache .room_servers(room_id) .filter_map(Result::ok) - .filter(|server_name| !server_is_ours(server_name)); + .filter(|server_name| !self.services.globals.server_is_ours(server_name)); self.flush_servers(servers) } diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 0668ce24..9fbc30ea 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -29,7 +29,6 @@ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tokio::time::sleep_until; use super::{appservice, Destination, Msg, SendingEvent, Service}; -use crate::user_is_local; #[derive(Debug)] enum TransactionStatus { @@ -264,7 +263,7 @@ impl Service { .users .keys_changed(room_id.as_ref(), since, None) .filter_map(Result::ok) - .filter(|user_id| user_is_local(user_id)), + .filter(|user_id| self.services.globals.user_is_local(user_id)), ); if self.server.config.allow_outgoing_read_receipts @@ -306,7 +305,7 @@ impl Service { for (user_id, count, presence_bytes) in self.services.presence.presence_since(since) { *max_edu_count = cmp::max(count, *max_edu_count); - if !user_is_local(&user_id) { + if !self.services.globals.user_is_local(&user_id) { continue; } @@ -358,7 +357,7 @@ impl Service { let (user_id, count, read_receipt) = r?; *max_edu_count = cmp::max(count, *max_edu_count); - if !user_is_local(&user_id) { + if !self.services.globals.user_is_local(&user_id) { continue; } From 2fb43dd38dfa1a86632921ebb65ca10f34093e33 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 22 Jul 2024 21:16:46 +0000 Subject: [PATCH 0565/2091] infra to synthesize program options with config options Signed-off-by: Jason Volk --- src/core/config/mod.rs | 2 +- src/main/clap.rs | 12 +++++++++++- src/main/main.rs | 2 +- src/main/server.rs | 5 +++-- 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index b40ebb65..a689701a 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -405,7 +405,7 @@ const DEPRECATED_KEYS: &[&str; 9] = &[ impl Config { /// Initialize config - pub fn new(path: Option) -> Result { + pub fn new(path: &Option) -> Result { let raw_config = if let Some(config_file_env) = Env::var("CONDUIT_CONFIG") { Figment::new() .merge(Toml::file(config_file_env).nested()) diff --git a/src/main/clap.rs b/src/main/clap.rs index 6ce164f5..e82fea16 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; use clap::Parser; +use conduit::{Config, Result}; /// Commandline arguments #[derive(Parser, Debug)] @@ -15,4 +16,13 @@ pub(crate) struct Args { /// Parse commandline arguments into structured data #[must_use] -pub(crate) fn parse() -> Args { Args::parse() } +pub(super) fn parse() -> Args { Args::parse() } + +/// Synthesize any command line options with configuration file options. +pub(crate) fn update(config: &mut Config, args: &Args) -> Result<()> { + // Indicate the admin console should be spawned automatically if the + // configuration file hasn't already. + config.admin_console_automatic |= args.console.unwrap_or(false); + + Ok(()) +} diff --git a/src/main/main.rs b/src/main/main.rs index 959e8610..b13e117d 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -33,7 +33,7 @@ fn main() -> Result<(), Error> { .build() .expect("built runtime"); - let server: Arc = Server::build(args, Some(runtime.handle()))?; + let server: Arc = Server::build(&args, Some(runtime.handle()))?; runtime.spawn(signal::signal(server.clone())); runtime.block_on(async_main(&server))?; diff --git a/src/main/server.rs b/src/main/server.rs index 73c06f0c..b1cd6936 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -21,8 +21,9 @@ pub(crate) struct Server { } impl Server { - pub(crate) fn build(args: Args, runtime: Option<&runtime::Handle>) -> Result, Error> { - let config = Config::new(args.config)?; + pub(crate) fn build(args: &Args, runtime: Option<&runtime::Handle>) -> Result, Error> { + let mut config = Config::new(&args.config)?; + crate::clap::update(&mut config, args)?; #[cfg(feature = "sentry_telemetry")] let sentry_guard = crate::sentry::init(&config); From 263e3380883ec2ae801f7be8fa0be7388b158377 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 22 Jul 2024 21:21:44 +0000 Subject: [PATCH 0566/2091] add --console program option to automatically spawn Signed-off-by: Jason Volk --- src/core/config/mod.rs | 6 ++++++ src/main/clap.rs | 6 +++++- src/service/admin/mod.rs | 22 +++++++++++++++++++--- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index a689701a..6cef402a 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -330,6 +330,8 @@ pub struct Config { pub block_non_admin_invites: bool, #[serde(default = "true_fn")] pub admin_escape_commands: bool, + #[serde(default)] + pub admin_console_automatic: bool, #[serde(default)] pub sentry: bool, @@ -579,6 +581,10 @@ impl fmt::Display for Config { &self.block_non_admin_invites.to_string(), ); line("Enable admin escape commands", &self.admin_escape_commands.to_string()); + line( + "Activate admin console after startup", + &self.admin_console_automatic.to_string(), + ); line("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()); line("Allow incoming federated typing", &self.allow_incoming_typing.to_string()); line( diff --git a/src/main/clap.rs b/src/main/clap.rs index e82fea16..16d88978 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -12,6 +12,10 @@ pub(crate) struct Args { #[arg(short, long)] /// Optional argument to the path of a conduwuit config TOML file pub(crate) config: Option, + + /// Activate admin command console automatically after startup. + #[arg(long, num_args(0))] + pub(crate) console: bool, } /// Parse commandline arguments into structured data @@ -22,7 +26,7 @@ pub(super) fn parse() -> Args { Args::parse() } pub(crate) fn update(config: &mut Config, args: &Args) -> Result<()> { // Indicate the admin console should be spawned automatically if the // configuration file hasn't already. - config.admin_console_automatic |= args.console.unwrap_or(false); + config.admin_console_automatic |= args.console; Ok(()) } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 1d51bf38..b3879deb 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -82,6 +82,8 @@ impl crate::Service for Service { async fn worker(self: Arc) -> Result<()> { let receiver = self.receiver.lock().await; let mut signals = self.services.server.signal.subscribe(); + self.console_auto_start().await; + loop { tokio::select! { command = receiver.recv_async() => match command { @@ -95,9 +97,7 @@ impl crate::Service for Service { } } - //TODO: not unwind safe - #[cfg(feature = "console")] - self.console.close().await; + self.console_auto_stop().await; //TODO: not unwind safe Ok(()) } @@ -340,4 +340,20 @@ impl Service { false } } + + /// Possibly spawn the terminal console at startup if configured. + async fn console_auto_start(&self) { + #[cfg(feature = "console")] + if self.services.server.config.admin_console_automatic { + // Allow more of the startup sequence to execute before spawning + tokio::task::yield_now().await; + self.console.start().await; + } + } + + /// Shutdown the console when the admin worker terminates. + async fn console_auto_stop(&self) { + #[cfg(feature = "console")] + self.console.close().await; + } } From ccfa939bd3995645868ddfc5fb9f39f3fa6eaabb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 22 Jul 2024 22:24:17 +0000 Subject: [PATCH 0567/2091] split admin command enum from handler Signed-off-by: Jason Volk --- src/admin/admin.rs | 66 ++++++++++++++++ src/admin/handler.rs | 91 ++++------------------- src/admin/mod.rs | 10 ++- src/admin/room/room_commands.rs | 2 +- src/admin/room/room_directory_commands.rs | 2 +- src/admin/tests.rs | 2 +- 6 files changed, 89 insertions(+), 84 deletions(-) create mode 100644 src/admin/admin.rs diff --git a/src/admin/admin.rs b/src/admin/admin.rs new file mode 100644 index 00000000..e55f6d61 --- /dev/null +++ b/src/admin/admin.rs @@ -0,0 +1,66 @@ +use clap::Parser; +use conduit::Result; +use ruma::events::room::message::RoomMessageEventContent; + +use crate::{ + appservice, appservice::AppserviceCommand, check, check::CheckCommand, debug, debug::DebugCommand, federation, + federation::FederationCommand, media, media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, + server, server::ServerCommand, user, user::UserCommand, +}; + +#[derive(Parser)] +#[command(name = "admin", version = env!("CARGO_PKG_VERSION"))] +pub(crate) enum AdminCommand { + #[command(subcommand)] + /// - Commands for managing appservices + Appservices(AppserviceCommand), + + #[command(subcommand)] + /// - Commands for managing local users + Users(UserCommand), + + #[command(subcommand)] + /// - Commands for managing rooms + Rooms(RoomCommand), + + #[command(subcommand)] + /// - Commands for managing federation + Federation(FederationCommand), + + #[command(subcommand)] + /// - Commands for managing the server + Server(ServerCommand), + + #[command(subcommand)] + /// - Commands for managing media + Media(MediaCommand), + + #[command(subcommand)] + /// - Commands for checking integrity + Check(CheckCommand), + + #[command(subcommand)] + /// - Commands for debugging things + Debug(DebugCommand), + + #[command(subcommand)] + /// - Low-level queries for database getters and iterators + Query(QueryCommand), +} + +#[tracing::instrument(skip_all, name = "command")] +pub(crate) async fn process(command: AdminCommand, body: Vec<&str>) -> Result { + let reply_message_content = match command { + AdminCommand::Appservices(command) => appservice::process(command, body).await?, + AdminCommand::Media(command) => media::process(command, body).await?, + AdminCommand::Users(command) => user::process(command, body).await?, + AdminCommand::Rooms(command) => room::process(command, body).await?, + AdminCommand::Federation(command) => federation::process(command, body).await?, + AdminCommand::Server(command) => server::process(command, body).await?, + AdminCommand::Debug(command) => debug::process(command, body).await?, + AdminCommand::Query(command) => query::process(command, body).await?, + AdminCommand::Check(command) => check::process(command, body).await?, + }; + + Ok(reply_message_content) +} diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 409abc18..46712077 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -11,64 +11,16 @@ use ruma::{ OwnedEventId, }; -extern crate conduit_service as service; - use conduit::{utils::string::common_prefix, Result}; -pub(crate) use service::admin::Command; -use service::admin::{CommandOutput, CommandResult, HandlerResult}; +use service::admin::{Command, CommandOutput, CommandResult, HandlerResult}; -use crate::{ - appservice, appservice::AppserviceCommand, check, check::CheckCommand, debug, debug::DebugCommand, federation, - federation::FederationCommand, media, media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, - server, server::ServerCommand, services, user, user::UserCommand, -}; -pub(crate) const PAGE_SIZE: usize = 100; - -#[derive(Parser)] -#[command(name = "admin", version = env!("CARGO_PKG_VERSION"))] -pub(crate) enum AdminCommand { - #[command(subcommand)] - /// - Commands for managing appservices - Appservices(AppserviceCommand), - - #[command(subcommand)] - /// - Commands for managing local users - Users(UserCommand), - - #[command(subcommand)] - /// - Commands for managing rooms - Rooms(RoomCommand), - - #[command(subcommand)] - /// - Commands for managing federation - Federation(FederationCommand), - - #[command(subcommand)] - /// - Commands for managing the server - Server(ServerCommand), - - #[command(subcommand)] - /// - Commands for managing media - Media(MediaCommand), - - #[command(subcommand)] - /// - Commands for checking integrity - Check(CheckCommand), - - #[command(subcommand)] - /// - Commands for debugging things - Debug(DebugCommand), - - #[command(subcommand)] - /// - Low-level queries for database getters and iterators - Query(QueryCommand), -} +use crate::{admin, admin::AdminCommand, services}; #[must_use] -pub(crate) fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(command)) } +pub(super) fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(command)) } #[must_use] -pub(crate) fn complete(line: &str) -> String { complete_admin_command(AdminCommand::command(), line) } +pub(super) fn complete(line: &str) -> String { complete_command(AdminCommand::command(), line) } #[tracing::instrument(skip_all, name = "admin")] async fn handle_command(command: Command) -> CommandResult { @@ -80,7 +32,7 @@ async fn handle_command(command: Command) -> CommandResult { } async fn process_command(command: &Command) -> CommandOutput { - process_admin_message(&command.command) + process(&command.command) .await .and_then(|content| reply(content, command.reply_id.clone())) } @@ -104,11 +56,11 @@ fn reply(mut content: RoomMessageEventContent, reply_id: Option) - } // Parse and process a message from the admin room -async fn process_admin_message(msg: &str) -> CommandOutput { +async fn process(msg: &str) -> CommandOutput { let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); let command = lines.next().expect("each string has at least one line"); let body = lines.collect::>(); - let parsed = match parse_admin_command(command) { + let parsed = match parse_command(command) { Ok(parsed) => parsed, Err(error) => { let server_name = services().globals.server_name(); @@ -118,7 +70,7 @@ async fn process_admin_message(msg: &str) -> CommandOutput { }; let timer = Instant::now(); - let result = process_admin_command(parsed, body).await; + let result = admin::process(parsed, body).await; let elapsed = timer.elapsed(); conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}"); match result { @@ -129,31 +81,14 @@ async fn process_admin_message(msg: &str) -> CommandOutput { } } -#[tracing::instrument(skip_all, name = "command")] -async fn process_admin_command(command: AdminCommand, body: Vec<&str>) -> Result { - let reply_message_content = match command { - AdminCommand::Appservices(command) => appservice::process(command, body).await?, - AdminCommand::Media(command) => media::process(command, body).await?, - AdminCommand::Users(command) => user::process(command, body).await?, - AdminCommand::Rooms(command) => room::process(command, body).await?, - AdminCommand::Federation(command) => federation::process(command, body).await?, - AdminCommand::Server(command) => server::process(command, body).await?, - AdminCommand::Debug(command) => debug::process(command, body).await?, - AdminCommand::Query(command) => query::process(command, body).await?, - AdminCommand::Check(command) => check::process(command, body).await?, - }; - - Ok(reply_message_content) -} - // Parse chat messages from the admin room into an AdminCommand object -fn parse_admin_command(command_line: &str) -> Result { - let argv = parse_command_line(command_line); +fn parse_command(command_line: &str) -> Result { + let argv = parse_line(command_line); AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) } -fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { - let argv = parse_command_line(line); +fn complete_command(mut cmd: clap::Command, line: &str) -> String { + let argv = parse_line(line); let mut ret = Vec::::with_capacity(argv.len().saturating_add(1)); 'token: for token in argv.into_iter().skip(1) { @@ -196,7 +131,7 @@ fn complete_admin_command(mut cmd: clap::Command, line: &str) -> String { } // Parse chat messages from the admin room into an AdminCommand object -fn parse_command_line(command_line: &str) -> Vec { +fn parse_line(command_line: &str) -> Vec { let mut argv = command_line .split_whitespace() .map(str::to_owned) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index b183f3f6..ff2aefd5 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1,18 +1,20 @@ #![recursion_limit = "168"] #![allow(clippy::wildcard_imports)] +pub(crate) mod admin; +pub(crate) mod handler; +mod tests; +pub(crate) mod utils; + pub(crate) mod appservice; pub(crate) mod check; pub(crate) mod debug; pub(crate) mod federation; -pub(crate) mod handler; pub(crate) mod media; pub(crate) mod query; pub(crate) mod room; pub(crate) mod server; -mod tests; pub(crate) mod user; -pub(crate) mod utils; extern crate conduit_api as api; extern crate conduit_core as conduit; @@ -23,6 +25,8 @@ pub(crate) use service::services; pub(crate) use crate::utils::{escape_html, get_room_info}; +pub(crate) const PAGE_SIZE: usize = 100; + mod_ctor! {} mod_dtor! {} diff --git a/src/admin/room/room_commands.rs b/src/admin/room/room_commands.rs index 1a387c7e..cf0f3ddb 100644 --- a/src/admin/room/room_commands.rs +++ b/src/admin/room/room_commands.rs @@ -2,7 +2,7 @@ use std::fmt::Write; use ruma::events::room::message::RoomMessageEventContent; -use crate::{escape_html, get_room_info, handler::PAGE_SIZE, services, Result}; +use crate::{escape_html, get_room_info, services, Result, PAGE_SIZE}; pub(super) async fn list( _body: Vec<&str>, page: Option, exclude_disabled: bool, exclude_banned: bool, diff --git a/src/admin/room/room_directory_commands.rs b/src/admin/room/room_directory_commands.rs index c9b4eb9e..912e970c 100644 --- a/src/admin/room/room_directory_commands.rs +++ b/src/admin/room/room_directory_commands.rs @@ -3,7 +3,7 @@ use std::fmt::Write; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId}; use super::RoomDirectoryCommand; -use crate::{escape_html, get_room_info, handler::PAGE_SIZE, services, Result}; +use crate::{escape_html, get_room_info, services, Result, PAGE_SIZE}; pub(super) async fn process(command: RoomDirectoryCommand, _body: Vec<&str>) -> Result { match command { diff --git a/src/admin/tests.rs b/src/admin/tests.rs index 69ccd896..296d4888 100644 --- a/src/admin/tests.rs +++ b/src/admin/tests.rs @@ -12,7 +12,7 @@ fn get_help_subcommand() { get_help_inner("help"); } fn get_help_inner(input: &str) { use clap::Parser; - use crate::handler::AdminCommand; + use crate::admin::AdminCommand; let Err(error) = AdminCommand::try_parse_from(["argv[0] doesn't matter", input]) else { panic!("no error!"); From 5ed95ea3572a9bd23942fdaaf900e18a40159aec Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 22 Jul 2024 22:45:25 +0000 Subject: [PATCH 0568/2091] contextualize handler in object Signed-off-by: Jason Volk --- src/admin/handler.rs | 245 +++++++++++++++++++++++-------------------- 1 file changed, 131 insertions(+), 114 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 46712077..ff04d378 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -12,16 +12,28 @@ use ruma::{ }; use conduit::{utils::string::common_prefix, Result}; -use service::admin::{Command, CommandOutput, CommandResult, HandlerResult}; +use service::{ + admin::{Command, CommandOutput, CommandResult, HandlerResult}, + Services, +}; -use crate::{admin, admin::AdminCommand, services}; +use crate::{admin, admin::AdminCommand}; + +struct Handler<'a> { + services: &'a Services, +} + +#[must_use] +pub(super) fn complete(line: &str) -> String { + Handler { + services: service::services(), + } + .complete_command(AdminCommand::command(), line) +} #[must_use] pub(super) fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(command)) } -#[must_use] -pub(super) fn complete(line: &str) -> String { complete_command(AdminCommand::command(), line) } - #[tracing::instrument(skip_all, name = "admin")] async fn handle_command(command: Command) -> CommandResult { AssertUnwindSafe(process_command(&command)) @@ -32,9 +44,12 @@ async fn handle_command(command: Command) -> CommandResult { } async fn process_command(command: &Command) -> CommandOutput { - process(&command.command) - .await - .and_then(|content| reply(content, command.reply_id.clone())) + Handler { + services: service::services(), + } + .process(&command.command) + .await + .and_then(|content| reply(content, command.reply_id.clone())) } fn handle_panic(error: &Error, command: Command) -> CommandResult { @@ -55,122 +70,124 @@ fn reply(mut content: RoomMessageEventContent, reply_id: Option) - Some(content) } -// Parse and process a message from the admin room -async fn process(msg: &str) -> CommandOutput { - let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); - let command = lines.next().expect("each string has at least one line"); - let body = lines.collect::>(); - let parsed = match parse_command(command) { - Ok(parsed) => parsed, - Err(error) => { - let server_name = services().globals.server_name(); - let message = error.replace("server.name", server_name.as_str()); - return Some(RoomMessageEventContent::notice_markdown(message)); - }, - }; +impl Handler<'_> { + // Parse and process a message from the admin room + async fn process(&self, msg: &str) -> CommandOutput { + let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); + let command = lines.next().expect("each string has at least one line"); + let body = lines.collect::>(); + let parsed = match self.parse_command(command) { + Ok(parsed) => parsed, + Err(error) => { + let server_name = self.services.globals.server_name(); + let message = error.replace("server.name", server_name.as_str()); + return Some(RoomMessageEventContent::notice_markdown(message)); + }, + }; - let timer = Instant::now(); - let result = admin::process(parsed, body).await; - let elapsed = timer.elapsed(); - conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}"); - match result { - Ok(reply) => Some(reply), - Err(error) => Some(RoomMessageEventContent::notice_markdown(format!( - "Encountered an error while handling the command:\n```\n{error:#?}\n```" - ))), + let timer = Instant::now(); + let result = Box::pin(admin::process(parsed, body)).await; + let elapsed = timer.elapsed(); + conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}"); + match result { + Ok(reply) => Some(reply), + Err(error) => Some(RoomMessageEventContent::notice_markdown(format!( + "Encountered an error while handling the command:\n```\n{error:#?}\n```" + ))), + } } -} -// Parse chat messages from the admin room into an AdminCommand object -fn parse_command(command_line: &str) -> Result { - let argv = parse_line(command_line); - AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) -} + // Parse chat messages from the admin room into an AdminCommand object + fn parse_command(&self, command_line: &str) -> Result { + let argv = self.parse_line(command_line); + AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) + } -fn complete_command(mut cmd: clap::Command, line: &str) -> String { - let argv = parse_line(line); - let mut ret = Vec::::with_capacity(argv.len().saturating_add(1)); + fn complete_command(&self, mut cmd: clap::Command, line: &str) -> String { + let argv = self.parse_line(line); + let mut ret = Vec::::with_capacity(argv.len().saturating_add(1)); - 'token: for token in argv.into_iter().skip(1) { - let cmd_ = cmd.clone(); - let mut choice = Vec::new(); + 'token: for token in argv.into_iter().skip(1) { + let cmd_ = cmd.clone(); + let mut choice = Vec::new(); - for sub in cmd_.get_subcommands() { - let name = sub.get_name(); - if *name == token { - // token already complete; recurse to subcommand - ret.push(token); - cmd.clone_from(sub); - continue 'token; - } else if name.starts_with(&token) { - // partial match; add to choices - choice.push(name); + for sub in cmd_.get_subcommands() { + let name = sub.get_name(); + if *name == token { + // token already complete; recurse to subcommand + ret.push(token); + cmd.clone_from(sub); + continue 'token; + } else if name.starts_with(&token) { + // partial match; add to choices + choice.push(name); + } } + + if choice.len() == 1 { + // One choice. Add extra space because it's complete + let choice = *choice.first().expect("only choice"); + ret.push(choice.to_owned()); + ret.push(String::new()); + } else if choice.is_empty() { + // Nothing found, return original string + ret.push(token); + } else { + // Find the common prefix + ret.push(common_prefix(&choice).into()); + } + + // Return from completion + return ret.join(" "); } - if choice.len() == 1 { - // One choice. Add extra space because it's complete - let choice = *choice.first().expect("only choice"); - ret.push(choice.to_owned()); - ret.push(String::new()); - } else if choice.is_empty() { - // Nothing found, return original string - ret.push(token); - } else { - // Find the common prefix - ret.push(common_prefix(&choice).into()); + // Return from no completion. Needs a space though. + ret.push(String::new()); + ret.join(" ") + } + + // Parse chat messages from the admin room into an AdminCommand object + fn parse_line(&self, command_line: &str) -> Vec { + let mut argv = command_line + .split_whitespace() + .map(str::to_owned) + .collect::>(); + + // Remove any escapes that came with a server-side escape command + if !argv.is_empty() && argv[0].ends_with("admin") { + argv[0] = argv[0].trim_start_matches('\\').into(); } - // Return from completion - return ret.join(" "); - } + // First indice has to be "admin" but for console convenience we add it here + let server_user = self.services.globals.server_user.as_str(); + if !argv.is_empty() && !argv[0].ends_with("admin") && !argv[0].starts_with(server_user) { + argv.insert(0, "admin".to_owned()); + } - // Return from no completion. Needs a space though. - ret.push(String::new()); - ret.join(" ") -} - -// Parse chat messages from the admin room into an AdminCommand object -fn parse_line(command_line: &str) -> Vec { - let mut argv = command_line - .split_whitespace() - .map(str::to_owned) - .collect::>(); - - // Remove any escapes that came with a server-side escape command - if !argv.is_empty() && argv[0].ends_with("admin") { - argv[0] = argv[0].trim_start_matches('\\').into(); - } - - // First indice has to be "admin" but for console convenience we add it here - let server_user = services().globals.server_user.as_str(); - if !argv.is_empty() && !argv[0].ends_with("admin") && !argv[0].starts_with(server_user) { - argv.insert(0, "admin".to_owned()); - } - - // Replace `help command` with `command --help` - // Clap has a help subcommand, but it omits the long help description. - if argv.len() > 1 && argv[1] == "help" { - argv.remove(1); - argv.push("--help".to_owned()); - } - - // Backwards compatibility with `register_appservice`-style commands - if argv.len() > 1 && argv[1].contains('_') { - argv[1] = argv[1].replace('_', "-"); - } - - // Backwards compatibility with `register_appservice`-style commands - if argv.len() > 2 && argv[2].contains('_') { - argv[2] = argv[2].replace('_', "-"); - } - - // if the user is using the `query` command (argv[1]), replace the database - // function/table calls with underscores to match the codebase - if argv.len() > 3 && argv[1].eq("query") { - argv[3] = argv[3].replace('_', "-"); - } - - trace!(?command_line, ?argv, "parse"); - argv + // Replace `help command` with `command --help` + // Clap has a help subcommand, but it omits the long help description. + if argv.len() > 1 && argv[1] == "help" { + argv.remove(1); + argv.push("--help".to_owned()); + } + + // Backwards compatibility with `register_appservice`-style commands + if argv.len() > 1 && argv[1].contains('_') { + argv[1] = argv[1].replace('_', "-"); + } + + // Backwards compatibility with `register_appservice`-style commands + if argv.len() > 2 && argv[2].contains('_') { + argv[2] = argv[2].replace('_', "-"); + } + + // if the user is using the `query` command (argv[1]), replace the database + // function/table calls with underscores to match the codebase + if argv.len() > 3 && argv[1].eq("query") { + argv[3] = argv[3].replace('_', "-"); + } + + trace!(?command_line, ?argv, "parse"); + argv + } } From 5a17fbccf56e395f9eea476442967275cfd544cf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 5 Jul 2024 08:40:02 +0000 Subject: [PATCH 0569/2091] add type_name debug tool Signed-off-by: Jason Volk --- src/core/debug.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/debug.rs b/src/core/debug.rs index 14d0be87..5e52f399 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -80,3 +80,8 @@ pub fn trap() { #[must_use] pub fn panic_str(p: &Box) -> &'static str { p.downcast_ref::<&str>().copied().unwrap_or_default() } + +#[cfg(debug_assertions)] +#[inline(always)] +#[must_use] +pub fn type_name(_: &T) -> &'static str { std::any::type_name::() } From 91b49a7786482e1bb7ffa3c7e8afe688e8f94afd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 23 Jul 2024 06:57:14 +0000 Subject: [PATCH 0570/2091] add basic exchange util Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index bbd52829..d7b6a72a 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -26,8 +26,16 @@ pub use string::{str_from_bytes, string_from_bytes}; pub use sys::available_parallelism; pub use time::now_millis as millis_since_unix_epoch; +#[inline] pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), max) } +#[inline] +pub fn exchange(state: &mut T, source: T) -> T { + let ret = state.clone(); + *state = source; + ret +} + #[must_use] pub fn generate_keypair() -> Vec { let mut value = rand::string(8).as_bytes().to_vec(); From 5c0bf2912205ab46989a5165080df8bce77686a2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 23 Jul 2024 07:03:33 +0000 Subject: [PATCH 0571/2091] add util for camel to snake case conversion Signed-off-by: Jason Volk --- src/core/utils/string.rs | 37 ++++++++++++++++++++++++++++++++++++- src/core/utils/tests.rs | 16 ++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 106d0cb7..fc3891df 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -1,4 +1,4 @@ -use crate::Result; +use crate::{utils::exchange, Result}; pub const EMPTY: &str = ""; @@ -26,6 +26,41 @@ macro_rules! is_format { }; } +#[inline] +#[must_use] +pub fn camel_to_snake_string(s: &str) -> String { + let est_len = s + .chars() + .fold(s.len(), |est, c| est.saturating_add(usize::from(c.is_ascii_uppercase()))); + + let mut ret = String::with_capacity(est_len); + camel_to_snake_case(&mut ret, s.as_bytes()).expect("string-to-string stream error"); + ret +} + +#[inline] +pub fn camel_to_snake_case(output: &mut O, input: I) -> Result<()> +where + I: std::io::Read, + O: std::fmt::Write, +{ + let mut state = false; + input + .bytes() + .take_while(Result::is_ok) + .map(Result::unwrap) + .map(char::from) + .try_for_each(|ch| { + let m = ch.is_ascii_uppercase(); + let s = exchange(&mut state, !m); + if m && s { + output.write_char('_')?; + } + output.write_char(ch.to_ascii_lowercase())?; + Result::<()>::Ok(()) + }) +} + /// Find the common prefix from a collection of strings and return a slice /// ``` /// use conduit_core::utils::string::common_prefix; diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index 43968947..e91accdf 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -134,3 +134,19 @@ async fn mutex_map_contend() { tokio::try_join!(join_b, join_a).expect("joined"); assert!(map.is_empty(), "Must be empty"); } + +#[test] +fn camel_to_snake_case_0() { + use utils::string::camel_to_snake_string; + + let res = camel_to_snake_string("CamelToSnakeCase"); + assert_eq!(res, "camel_to_snake_case"); +} + +#[test] +fn camel_to_snake_case_1() { + use utils::string::camel_to_snake_string; + + let res = camel_to_snake_string("CAmelTOSnakeCase"); + assert_eq!(res, "camel_tosnake_case"); +} From 4458efa2b2fdb850f68e9afa3ffd72d21155bed7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 07:21:51 +0000 Subject: [PATCH 0572/2091] rename signing_keys_for to verify_keys_for Signed-off-by: Jason Volk --- src/admin/query/globals.rs | 2 +- src/service/globals/data.rs | 2 +- src/service/globals/mod.rs | 4 ++-- src/service/rooms/event_handler/signing_keys.rs | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 9bdd38fc..38986071 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -46,7 +46,7 @@ pub(super) async fn globals(subcommand: Globals) -> Result { let timer = tokio::time::Instant::now(); - let results = services().globals.db.signing_keys_for(&origin); + let results = services().globals.db.verify_keys_for(&origin); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 5d6240cd..d73ddec6 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -289,7 +289,7 @@ impl Data { /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. - pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { + pub fn verify_keys_for(&self, origin: &ServerName) -> Result> { let signingkeys = self .server_signingkeys .get(origin.as_bytes())? diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index c2c7d969..d46eb79e 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -272,8 +272,8 @@ impl Service { /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. - pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { - let mut keys = self.db.signing_keys_for(origin)?; + pub fn verify_keys_for(&self, origin: &ServerName) -> Result> { + let mut keys = self.db.verify_keys_for(origin)?; if origin == self.server_name() { keys.insert( format!("ed25519:{}", self.keypair().version()) diff --git a/src/service/rooms/event_handler/signing_keys.rs b/src/service/rooms/event_handler/signing_keys.rs index 1ebcbefb..d7cb3a9d 100644 --- a/src/service/rooms/event_handler/signing_keys.rs +++ b/src/service/rooms/event_handler/signing_keys.rs @@ -148,7 +148,7 @@ impl super::Service { let result: BTreeMap<_, _> = self .services .globals - .signing_keys_for(origin)? + .verify_keys_for(origin)? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) .collect(); @@ -354,7 +354,7 @@ impl super::Service { let mut result: BTreeMap<_, _> = self .services .globals - .signing_keys_for(origin)? + .verify_keys_for(origin)? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) .collect(); From c64adbec0e267c8507d6f2dd3c566cdd614a99c0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 07:49:52 +0000 Subject: [PATCH 0573/2091] split signing_keys_for from verify_keys_for Signed-off-by: Jason Volk --- src/service/globals/data.rs | 13 ++++++++++--- src/service/globals/mod.rs | 9 ++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index d73ddec6..5b5d9f09 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -291,9 +291,7 @@ impl Data { /// for the server. pub fn verify_keys_for(&self, origin: &ServerName) -> Result> { let signingkeys = self - .server_signingkeys - .get(origin.as_bytes())? - .and_then(|bytes| serde_json::from_slice(&bytes).ok()) + .signing_keys_for(origin)? .map_or_else(BTreeMap::new, |keys: ServerSigningKeys| { let mut tree = keys.verify_keys; tree.extend( @@ -307,6 +305,15 @@ impl Data { Ok(signingkeys) } + pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { + let signingkeys = self + .server_signingkeys + .get(origin.as_bytes())? + .and_then(|bytes| serde_json::from_slice(&bytes).ok()); + + Ok(signingkeys) + } + pub fn database_version(&self) -> Result { self.global.get(b"version")?.map_or(Ok(0), |version| { utils::u64_from_bytes(&version).map_err(|_| Error::bad_database("Database version id is invalid.")) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index d46eb79e..2c588dce 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -13,7 +13,10 @@ use data::Data; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ - api::{client::discovery::discover_support::ContactRole, federation::discovery::VerifyKey}, + api::{ + client::discovery::discover_support::ContactRole, + federation::discovery::{ServerSigningKeys, VerifyKey}, + }, serde::Base64, DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, @@ -288,6 +291,10 @@ impl Service { Ok(keys) } + pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { + self.db.signing_keys_for(origin) + } + pub fn well_known_client(&self) -> &Option { &self.config.well_known.client } pub fn well_known_server(&self) -> &Option { &self.config.well_known.server } From f841c2356d402d2a7b59f4683ea02a3f14134873 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 17 Jun 2024 21:46:23 +0000 Subject: [PATCH 0574/2091] preliminary get-signing-keys command Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 45 +++++++++++++++++++++++++++++++++++++ src/admin/debug/mod.rs | 12 ++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index cbe52473..3a7d3544 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -1,5 +1,6 @@ use std::{ collections::{BTreeMap, HashMap}, + fmt::Write, sync::{Arc, Mutex}, time::Instant, }; @@ -605,6 +606,50 @@ pub(super) async fn force_set_room_state_from_server( )) } +pub(super) async fn get_signing_keys( + _body: Vec<&str>, server_name: Option>, _cached: bool, +) -> Result { + let server_name = server_name.unwrap_or_else(|| services().server.config.server_name.clone().into()); + let signing_keys = services().globals.signing_keys_for(&server_name)?; + + Ok(RoomMessageEventContent::notice_markdown(format!( + "```rs\n{signing_keys:#?}\n```" + ))) +} + +#[allow(dead_code)] +pub(super) async fn get_verify_keys( + _body: Vec<&str>, server_name: Option>, cached: bool, +) -> Result { + let server_name = server_name.unwrap_or_else(|| services().server.config.server_name.clone().into()); + let mut out = String::new(); + + if cached { + writeln!(out, "| Key ID | VerifyKey |")?; + writeln!(out, "| --- | --- |")?; + for (key_id, verify_key) in services().globals.verify_keys_for(&server_name)? { + writeln!(out, "| {key_id} | {verify_key:?} |")?; + } + + return Ok(RoomMessageEventContent::notice_markdown(out)); + } + + let signature_ids: Vec = Vec::new(); + let keys = services() + .rooms + .event_handler + .fetch_signing_keys_for_server(&server_name, signature_ids) + .await?; + + writeln!(out, "| Key ID | Public Key |")?; + writeln!(out, "| --- | --- |")?; + for (key_id, key) in keys { + writeln!(out, "| {key_id} | {key} |")?; + } + + Ok(RoomMessageEventContent::notice_markdown(out)) +} + pub(super) async fn resolve_true_destination( _body: Vec<&str>, server_name: Box, no_cache: bool, ) -> Result { diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 7d6cafa7..41527cf5 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -76,6 +76,14 @@ pub(super) enum DebugCommand { room_id: OwnedRoomOrAliasId, }, + /// - Get and display signing keys from local cache or remote server. + GetSigningKeys { + server_name: Option>, + + #[arg(short, long)] + cached: bool, + }, + /// - Sends a federation request to the remote server's /// `/_matrix/federation/v1/version` endpoint and measures the latency it /// took for the server to respond @@ -177,6 +185,10 @@ pub(super) async fn process(command: DebugCommand, body: Vec<&str>) -> Result echo(body, message).await?, + DebugCommand::GetSigningKeys { + server_name, + cached, + } => get_signing_keys(body, server_name, cached).await?, DebugCommand::GetAuthChain { event_id, } => get_auth_chain(body, event_id).await?, From 2468e0c3ded9a26cc4014e2d4e35e6d968f8f4cc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 00:13:03 +0000 Subject: [PATCH 0575/2091] unconditionally derive Debug on subcommand enums Signed-off-by: Jason Volk --- src/admin/admin.rs | 2 +- src/admin/appservice/mod.rs | 3 +-- src/admin/check/mod.rs | 3 +-- src/admin/debug/mod.rs | 3 +-- src/admin/debug/tester.rs | 3 +-- src/admin/federation/mod.rs | 3 +-- src/admin/media/mod.rs | 3 +-- src/admin/query/mod.rs | 30 ++++++++++-------------------- src/admin/room/mod.rs | 15 +++++---------- src/admin/server/mod.rs | 3 +-- src/admin/user/mod.rs | 3 +-- 11 files changed, 24 insertions(+), 47 deletions(-) diff --git a/src/admin/admin.rs b/src/admin/admin.rs index e55f6d61..f5fe5dc2 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -8,7 +8,7 @@ use crate::{ server, server::ServerCommand, user, user::UserCommand, }; -#[derive(Parser)] +#[derive(Debug, Parser)] #[command(name = "admin", version = env!("CARGO_PKG_VERSION"))] pub(crate) enum AdminCommand { #[command(subcommand)] diff --git a/src/admin/appservice/mod.rs b/src/admin/appservice/mod.rs index 87ab1b6d..81e04087 100644 --- a/src/admin/appservice/mod.rs +++ b/src/admin/appservice/mod.rs @@ -6,8 +6,7 @@ use ruma::events::room::message::RoomMessageEventContent; use self::commands::*; -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum AppserviceCommand { /// - Register an appservice using its registration YAML /// diff --git a/src/admin/check/mod.rs b/src/admin/check/mod.rs index fdbe3d0d..f1cfa2b9 100644 --- a/src/admin/check/mod.rs +++ b/src/admin/check/mod.rs @@ -6,8 +6,7 @@ use ruma::events::room::message::RoomMessageEventContent; use self::commands::*; -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum CheckCommand { AllUsers, } diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 41527cf5..0df47748 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -8,8 +8,7 @@ use tester::TesterCommand; use self::commands::*; -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum DebugCommand { /// - Echo input of admin command Echo { diff --git a/src/admin/debug/tester.rs b/src/admin/debug/tester.rs index 6982ef52..2765a344 100644 --- a/src/admin/debug/tester.rs +++ b/src/admin/debug/tester.rs @@ -2,8 +2,7 @@ use ruma::events::room::message::RoomMessageEventContent; use crate::Result; -#[derive(clap::Subcommand)] -#[cfg_attr(test, derive(Debug))] +#[derive(Debug, clap::Subcommand)] pub(crate) enum TesterCommand { Tester, Timer, diff --git a/src/admin/federation/mod.rs b/src/admin/federation/mod.rs index 46e323b2..d02b4295 100644 --- a/src/admin/federation/mod.rs +++ b/src/admin/federation/mod.rs @@ -6,8 +6,7 @@ use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, U use self::commands::*; -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum FederationCommand { /// - List all rooms we are currently handling an incoming pdu from IncomingFederation, diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 60f7a9bb..d30c55d0 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -6,8 +6,7 @@ use ruma::{events::room::message::RoomMessageEventContent, EventId, MxcUri}; use self::commands::*; -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum MediaCommand { /// - Deletes a single media file from our database and on the filesystem /// via a single MXC URL diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index ea7036d0..c86f4f53 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -21,8 +21,7 @@ use self::{ room_alias::room_alias, sending::sending, users::users, }; -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] /// Query tables from database pub(super) enum QueryCommand { /// - account_data.rs iterators and getters @@ -62,8 +61,7 @@ pub(super) enum QueryCommand { Resolver(Resolver), } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/account_data.rs pub(super) enum AccountData { /// - Returns all changes to the account data that happened after `since`. @@ -87,8 +85,7 @@ pub(super) enum AccountData { }, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/appservice.rs pub(super) enum Appservice { /// - Gets the appservice registration info/details from the ID as a string @@ -101,8 +98,7 @@ pub(super) enum Appservice { All, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/presence.rs pub(super) enum Presence { /// - Returns the latest presence event for the given user. @@ -119,8 +115,7 @@ pub(super) enum Presence { }, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/rooms/alias.rs pub(super) enum RoomAlias { ResolveLocalAlias { @@ -138,8 +133,7 @@ pub(super) enum RoomAlias { AllLocalAliases, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum RoomStateCache { ServerInRoom { server: Box, @@ -210,8 +204,7 @@ pub(super) enum RoomStateCache { }, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/globals.rs pub(super) enum Globals { DatabaseVersion, @@ -229,8 +222,7 @@ pub(super) enum Globals { }, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/sending.rs pub(super) enum Sending { /// - Queries database for all `servercurrentevent_data` @@ -285,15 +277,13 @@ pub(super) enum Sending { }, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/users.rs pub(super) enum Users { Iter, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] /// Resolver service and caches pub(super) enum Resolver { /// Query the destinations cache diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index b4fa15bd..8f125f0a 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -10,8 +10,7 @@ use ruma::{events::room::message::RoomMessageEventContent, RoomId, RoomOrAliasId use self::room_commands::list; -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum RoomCommand { /// - List all rooms the server knows about List { @@ -43,8 +42,7 @@ pub(super) enum RoomCommand { Directory(RoomDirectoryCommand), } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum RoomInfoCommand { /// - List joined members in a room ListJoinedMembers { @@ -60,8 +58,7 @@ pub(super) enum RoomInfoCommand { }, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum RoomAliasCommand { /// - Make an alias point to a room. Set { @@ -96,8 +93,7 @@ pub(super) enum RoomAliasCommand { }, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum RoomDirectoryCommand { /// - Publish a room to the room directory Publish { @@ -117,8 +113,7 @@ pub(super) enum RoomDirectoryCommand { }, } -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum RoomModerationCommand { /// - Bans a room from local users joining and evicts all our local users /// from the room. Also blocks any invites (local and remote) for the diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 41b10180..604dacc7 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -6,8 +6,7 @@ use ruma::events::room::message::RoomMessageEventContent; use self::commands::*; -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum ServerCommand { /// - Time elapsed since startup Uptime, diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index cdb5fa5e..1b92d668 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -6,8 +6,7 @@ use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomOrAliasId, R use self::commands::*; -#[cfg_attr(test, derive(Debug))] -#[derive(Subcommand)] +#[derive(Debug, Subcommand)] pub(super) enum UserCommand { /// - Create a new user Create { From 85f734ec74d98cda9d6557dd56e4f4b99a976ed0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 00:14:03 +0000 Subject: [PATCH 0576/2091] =?UTF-8?q?proc=20macro=20=E2=9C=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jason Volk --- Cargo.lock | 11 +++++++ Cargo.toml | 15 +++++++++ src/admin/Cargo.toml | 1 + src/admin/debug/commands.rs | 13 ++++---- src/admin/debug/mod.rs | 64 +++---------------------------------- src/admin/handler.rs | 6 ++-- src/admin/mod.rs | 1 + src/macros/Cargo.toml | 24 ++++++++++++++ src/macros/admin.rs | 50 +++++++++++++++++++++++++++++ src/macros/mod.rs | 8 +++++ 10 files changed, 123 insertions(+), 70 deletions(-) create mode 100644 src/macros/Cargo.toml create mode 100644 src/macros/admin.rs create mode 100644 src/macros/mod.rs diff --git a/Cargo.lock b/Cargo.lock index e9730053..0649f3be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,6 +612,7 @@ dependencies = [ "clap", "conduit_api", "conduit_core", + "conduit_macros", "conduit_service", "const-str", "futures-util", @@ -712,6 +713,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "conduit_macros" +version = "0.4.5" +dependencies = [ + "conduit_core", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "conduit_router" version = "0.4.6" diff --git a/Cargo.toml b/Cargo.toml index f5c2ad24..5fcb03ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -413,6 +413,16 @@ default-features = false [workspace.dependencies.checked_ops] version = "0.1" +[workspace.dependencies.syn] +version = "1.0" +features = ["full", "extra-traits"] + +[workspace.dependencies.quote] +version = "1.0" + +[workspace.dependencies.proc-macro2] +version = "1.0.86" + # # Patches @@ -480,6 +490,11 @@ package = "conduit_core" path = "src/core" default-features = false +[workspace.dependencies.conduit-macros] +package = "conduit_macros" +path = "src/macros" +default-features = false + ############################################################################### # # Release profiles diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 1e13fb7a..d756b3cb 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -29,6 +29,7 @@ release_max_log_level = [ clap.workspace = true conduit-api.workspace = true conduit-core.workspace = true +conduit-macros.workspace = true conduit-service.workspace = true const-str.workspace = true futures-util.workspace = true diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 3a7d3544..4efefce7 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -477,7 +477,7 @@ pub(super) async fn latest_pdu_in_room(_body: Vec<&str>, room_id: Box) - #[tracing::instrument(skip(_body))] pub(super) async fn force_set_room_state_from_server( - _body: Vec<&str>, server_name: Box, room_id: Box, + _body: Vec<&str>, room_id: Box, server_name: Box, ) -> Result { if !services() .rooms @@ -691,18 +691,19 @@ pub(super) async fn resolve_true_destination( Ok(RoomMessageEventContent::text_markdown(msg)) } -#[must_use] -pub(super) fn memory_stats() -> RoomMessageEventContent { +pub(super) async fn memory_stats(_body: Vec<&str>) -> Result { let html_body = conduit::alloc::memory_stats(); if html_body.is_none() { - return RoomMessageEventContent::text_plain("malloc stats are not supported on your compiled malloc."); + return Ok(RoomMessageEventContent::text_plain( + "malloc stats are not supported on your compiled malloc.", + )); } - RoomMessageEventContent::text_html( + Ok(RoomMessageEventContent::text_html( "This command's output can only be viewed by clients that render HTML.".to_owned(), html_body.expect("string result"), - ) + )) } #[cfg(tokio_unstable)] diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 0df47748..82b37c53 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -3,11 +3,12 @@ pub(crate) mod tester; use clap::Subcommand; use conduit::Result; +use conduit_macros::admin_command_dispatch; use ruma::{events::room::message::RoomMessageEventContent, EventId, OwnedRoomOrAliasId, RoomId, ServerName}; -use tester::TesterCommand; -use self::commands::*; +use self::{commands::*, tester::TesterCommand}; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] pub(super) enum DebugCommand { /// - Echo input of admin command @@ -176,63 +177,6 @@ pub(super) enum DebugCommand { /// - Developer test stubs #[command(subcommand)] + #[allow(non_snake_case)] Tester(TesterCommand), } - -pub(super) async fn process(command: DebugCommand, body: Vec<&str>) -> Result { - Ok(match command { - DebugCommand::Echo { - message, - } => echo(body, message).await?, - DebugCommand::GetSigningKeys { - server_name, - cached, - } => get_signing_keys(body, server_name, cached).await?, - DebugCommand::GetAuthChain { - event_id, - } => get_auth_chain(body, event_id).await?, - DebugCommand::ParsePdu => parse_pdu(body).await?, - DebugCommand::GetPdu { - event_id, - } => get_pdu(body, event_id).await?, - DebugCommand::GetRemotePdu { - event_id, - server, - } => get_remote_pdu(body, event_id, server).await?, - DebugCommand::GetRoomState { - room_id, - } => get_room_state(body, room_id).await?, - DebugCommand::Ping { - server, - } => ping(body, server).await?, - DebugCommand::ForceDeviceListUpdates => force_device_list_updates(body).await?, - DebugCommand::ChangeLogLevel { - filter, - reset, - } => change_log_level(body, filter, reset).await?, - DebugCommand::SignJson => sign_json(body).await?, - DebugCommand::VerifyJson => verify_json(body).await?, - DebugCommand::FirstPduInRoom { - room_id, - } => first_pdu_in_room(body, room_id).await?, - DebugCommand::LatestPduInRoom { - room_id, - } => latest_pdu_in_room(body, room_id).await?, - DebugCommand::GetRemotePduList { - server, - force, - } => get_remote_pdu_list(body, server, force).await?, - DebugCommand::ForceSetRoomStateFromServer { - room_id, - server_name, - } => force_set_room_state_from_server(body, server_name, room_id).await?, - DebugCommand::ResolveTrueDestination { - server_name, - no_cache, - } => resolve_true_destination(body, server_name, no_cache).await?, - DebugCommand::MemoryStats => memory_stats(), - DebugCommand::RuntimeMetrics => runtime_metrics(body).await?, - DebugCommand::RuntimeInterval => runtime_interval(body).await?, - DebugCommand::Tester(command) => tester::process(command, body).await?, - }) -} diff --git a/src/admin/handler.rs b/src/admin/handler.rs index ff04d378..6acb19bf 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,7 +1,7 @@ use std::{panic::AssertUnwindSafe, time::Instant}; use clap::{CommandFactory, Parser}; -use conduit::{error, trace, Error}; +use conduit::{error, trace, utils::string::common_prefix, Error, Result}; use futures_util::future::FutureExt; use ruma::{ events::{ @@ -10,8 +10,6 @@ use ruma::{ }, OwnedEventId, }; - -use conduit::{utils::string::common_prefix, Result}; use service::{ admin::{Command, CommandOutput, CommandResult, HandlerResult}, Services, @@ -36,7 +34,7 @@ pub(super) fn handle(command: Command) -> HandlerResult { Box::pin(handle_comman #[tracing::instrument(skip_all, name = "admin")] async fn handle_command(command: Command) -> CommandResult { - AssertUnwindSafe(process_command(&command)) + AssertUnwindSafe(Box::pin(process_command(&command))) .catch_unwind() .await .map_err(Error::from_panic) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index ff2aefd5..7d752ff8 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1,5 +1,6 @@ #![recursion_limit = "168"] #![allow(clippy::wildcard_imports)] +#![allow(clippy::enum_glob_use)] pub(crate) mod admin; pub(crate) mod handler; diff --git a/src/macros/Cargo.toml b/src/macros/Cargo.toml new file mode 100644 index 00000000..b9a35aab --- /dev/null +++ b/src/macros/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "conduit_macros" +categories.workspace = true +description.workspace = true +edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true + +[lib] +name = "conduit_macros" +path = "mod.rs" +proc-macro = true + +[dependencies] +syn.workspace = true +quote.workspace = true +proc-macro2.workspace = true +conduit-core.workspace = true + +[lints] +workspace = true diff --git a/src/macros/admin.rs b/src/macros/admin.rs new file mode 100644 index 00000000..e95e402a --- /dev/null +++ b/src/macros/admin.rs @@ -0,0 +1,50 @@ +use conduit_core::utils::string::camel_to_snake_string; +use proc_macro::{Span, TokenStream}; +use proc_macro2::TokenStream as TokenStream2; +use quote::quote; +use syn::{parse_macro_input, AttributeArgs, Fields, Ident, ItemEnum, Variant}; + +pub(super) fn command_dispatch(args: TokenStream, input_: TokenStream) -> TokenStream { + let input = input_.clone(); + let item = parse_macro_input!(input as ItemEnum); + let _args = parse_macro_input!(args as AttributeArgs); + let arm = item.variants.iter().map(dispatch_arm); + let name = item.ident; + let q = quote! { + pub(super) async fn process(command: #name, body: Vec<&str>) -> Result { + use #name::*; + #[allow(non_snake_case)] + Ok(match command { + #( #arm )* + }) + } + }; + + [input_, q.into()].into_iter().collect::() +} + +fn dispatch_arm(v: &Variant) -> TokenStream2 { + let name = &v.ident; + let target = camel_to_snake_string(&format!("{name}")); + let handler = Ident::new(&target, Span::call_site().into()); + match &v.fields { + Fields::Named(fields) => { + let field = fields.named.iter().filter_map(|f| f.ident.as_ref()); + let arg = field.clone(); + quote! { + #name { #( #field ),* } => Box::pin(#handler(body, #( #arg ),*)).await?, + } + }, + Fields::Unnamed(fields) => { + let field = &fields.unnamed.first().expect("one field"); + quote! { + #name ( #field ) => Box::pin(#handler::process(#field, body)).await?, + } + }, + Fields::Unit => { + quote! { + #name => Box::pin(#handler(body)).await?, + } + }, + } +} diff --git a/src/macros/mod.rs b/src/macros/mod.rs new file mode 100644 index 00000000..0aba7560 --- /dev/null +++ b/src/macros/mod.rs @@ -0,0 +1,8 @@ +mod admin; + +use proc_macro::TokenStream; + +#[proc_macro_attribute] +pub fn admin_command_dispatch(args: TokenStream, input: TokenStream) -> TokenStream { + admin::command_dispatch(args, input) +} From 111cbea6fc70b25fde58a33dc1e8594de79040e0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 00:15:03 +0000 Subject: [PATCH 0577/2091] add debug time command Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 9 +++++++-- src/admin/debug/mod.rs | 3 +++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 4efefce7..ff68641c 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -2,14 +2,14 @@ use std::{ collections::{BTreeMap, HashMap}, fmt::Write, sync::{Arc, Mutex}, - time::Instant, + time::{Instant, SystemTime}, }; use api::client::validate_and_add_event_id; use conduit::{ debug, info, log, log::{capture, Capture}, - warn, Error, PduEvent, Result, + utils, warn, Error, PduEvent, Result, }; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, @@ -739,3 +739,8 @@ pub(super) async fn runtime_interval(_body: Vec<&str>) -> Result) -> Result { + let now = SystemTime::now(); + Ok(RoomMessageEventContent::text_markdown(utils::time::format(now, "%+"))) +} diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 82b37c53..77bc36b8 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -175,6 +175,9 @@ pub(super) enum DebugCommand { /// invocation. RuntimeInterval, + /// - Print the current time + Time, + /// - Developer test stubs #[command(subcommand)] #[allow(non_snake_case)] From ee864bcd9e0133f4c002d68ebc373b54b183eb16 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 01:26:23 +0000 Subject: [PATCH 0578/2091] normalize admin debug command handlers Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 57 +++++++++++++++++++------------------ src/macros/admin.rs | 4 +-- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index ff68641c..32fba409 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -20,13 +20,13 @@ use service::services; use tokio::sync::RwLock; use tracing_subscriber::EnvFilter; -pub(super) async fn echo(_body: Vec<&str>, message: Vec) -> Result { +pub(super) async fn echo(_body: &[&str], message: Vec) -> Result { let message = message.join(" "); Ok(RoomMessageEventContent::notice_plain(message)) } -pub(super) async fn get_auth_chain(_body: Vec<&str>, event_id: Box) -> Result { +pub(super) async fn get_auth_chain(_body: &[&str], event_id: Box) -> Result { let event_id = Arc::::from(event_id); if let Some(event) = services().rooms.timeline.get_pdu_json(&event_id)? { let room_id_str = event @@ -52,7 +52,7 @@ pub(super) async fn get_auth_chain(_body: Vec<&str>, event_id: Box) -> } } -pub(super) async fn parse_pdu(body: Vec<&str>) -> Result { +pub(super) async fn parse_pdu(body: &[&str]) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -80,7 +80,7 @@ pub(super) async fn parse_pdu(body: Vec<&str>) -> Result, event_id: Box) -> Result { +pub(super) async fn get_pdu(_body: &[&str], event_id: Box) -> Result { let mut outlier = false; let mut pdu_json = services() .rooms @@ -108,7 +108,7 @@ pub(super) async fn get_pdu(_body: Vec<&str>, event_id: Box) -> Result< } pub(super) async fn get_remote_pdu_list( - body: Vec<&str>, server: Box, force: bool, + body: &[&str], server: Box, force: bool, ) -> Result { if !services().globals.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( @@ -130,14 +130,15 @@ pub(super) async fn get_remote_pdu_list( } let list = body - .clone() - .drain(1..body.len().checked_sub(1).unwrap()) + .iter() + .collect::>() + .drain(1..body.len().saturating_sub(1)) .filter_map(|pdu| EventId::parse(pdu).ok()) .collect::>(); for pdu in list { if force { - if let Err(e) = get_remote_pdu(Vec::new(), Box::from(pdu), server.clone()).await { + if let Err(e) = get_remote_pdu(&[], Box::from(pdu), server.clone()).await { services() .admin .send_message(RoomMessageEventContent::text_plain(format!( @@ -147,7 +148,7 @@ pub(super) async fn get_remote_pdu_list( warn!(%e, "Failed to get remote PDU, ignoring error"); } } else { - get_remote_pdu(Vec::new(), Box::from(pdu), server.clone()).await?; + get_remote_pdu(&[], Box::from(pdu), server.clone()).await?; } } @@ -155,7 +156,7 @@ pub(super) async fn get_remote_pdu_list( } pub(super) async fn get_remote_pdu( - _body: Vec<&str>, event_id: Box, server: Box, + _body: &[&str], event_id: Box, server: Box, ) -> Result { if !services().globals.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( @@ -240,7 +241,7 @@ pub(super) async fn get_remote_pdu( } } -pub(super) async fn get_room_state(_body: Vec<&str>, room: OwnedRoomOrAliasId) -> Result { +pub(super) async fn get_room_state(_body: &[&str], room: OwnedRoomOrAliasId) -> Result { let room_id = services().rooms.alias.resolve(&room).await?; let room_state = services() .rooms @@ -267,7 +268,7 @@ pub(super) async fn get_room_state(_body: Vec<&str>, room: OwnedRoomOrAliasId) - Ok(RoomMessageEventContent::notice_markdown(format!("```json\n{json}\n```"))) } -pub(super) async fn ping(_body: Vec<&str>, server: Box) -> Result { +pub(super) async fn ping(_body: &[&str], server: Box) -> Result { if server == services().globals.server_name() { return Ok(RoomMessageEventContent::text_plain( "Not allowed to send federation requests to ourselves.", @@ -305,7 +306,7 @@ pub(super) async fn ping(_body: Vec<&str>, server: Box) -> Result) -> Result { +pub(super) async fn force_device_list_updates(_body: &[&str]) -> Result { // Force E2EE device list updates for all users for user_id in services().users.iter().filter_map(Result::ok) { services().users.mark_device_key_update(&user_id)?; @@ -316,7 +317,7 @@ pub(super) async fn force_device_list_updates(_body: Vec<&str>) -> Result, filter: Option, reset: bool, + _body: &[&str], filter: Option, reset: bool, ) -> Result { let handles = &["console"]; @@ -380,7 +381,7 @@ pub(super) async fn change_log_level( Ok(RoomMessageEventContent::text_plain("No log level was specified.")) } -pub(super) async fn sign_json(body: Vec<&str>) -> Result { +pub(super) async fn sign_json(body: &[&str]) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -403,7 +404,7 @@ pub(super) async fn sign_json(body: Vec<&str>) -> Result) -> Result { +pub(super) async fn verify_json(body: &[&str]) -> Result { if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -434,7 +435,7 @@ pub(super) async fn verify_json(body: Vec<&str>) -> Result, room_id: Box) -> Result { +pub(super) async fn first_pdu_in_room(_body: &[&str], room_id: Box) -> Result { if !services() .rooms .state_cache @@ -455,7 +456,7 @@ pub(super) async fn first_pdu_in_room(_body: Vec<&str>, room_id: Box) -> } #[tracing::instrument(skip(_body))] -pub(super) async fn latest_pdu_in_room(_body: Vec<&str>, room_id: Box) -> Result { +pub(super) async fn latest_pdu_in_room(_body: &[&str], room_id: Box) -> Result { if !services() .rooms .state_cache @@ -477,7 +478,7 @@ pub(super) async fn latest_pdu_in_room(_body: Vec<&str>, room_id: Box) - #[tracing::instrument(skip(_body))] pub(super) async fn force_set_room_state_from_server( - _body: Vec<&str>, room_id: Box, server_name: Box, + _body: &[&str], room_id: Box, server_name: Box, ) -> Result { if !services() .rooms @@ -607,7 +608,7 @@ pub(super) async fn force_set_room_state_from_server( } pub(super) async fn get_signing_keys( - _body: Vec<&str>, server_name: Option>, _cached: bool, + _body: &[&str], server_name: Option>, _cached: bool, ) -> Result { let server_name = server_name.unwrap_or_else(|| services().server.config.server_name.clone().into()); let signing_keys = services().globals.signing_keys_for(&server_name)?; @@ -619,7 +620,7 @@ pub(super) async fn get_signing_keys( #[allow(dead_code)] pub(super) async fn get_verify_keys( - _body: Vec<&str>, server_name: Option>, cached: bool, + _body: &[&str], server_name: Option>, cached: bool, ) -> Result { let server_name = server_name.unwrap_or_else(|| services().server.config.server_name.clone().into()); let mut out = String::new(); @@ -651,7 +652,7 @@ pub(super) async fn get_verify_keys( } pub(super) async fn resolve_true_destination( - _body: Vec<&str>, server_name: Box, no_cache: bool, + _body: &[&str], server_name: Box, no_cache: bool, ) -> Result { if !services().globals.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( @@ -691,7 +692,7 @@ pub(super) async fn resolve_true_destination( Ok(RoomMessageEventContent::text_markdown(msg)) } -pub(super) async fn memory_stats(_body: Vec<&str>) -> Result { +pub(super) async fn memory_stats(_body: &[&str]) -> Result { let html_body = conduit::alloc::memory_stats(); if html_body.is_none() { @@ -707,7 +708,7 @@ pub(super) async fn memory_stats(_body: Vec<&str>) -> Result) -> Result { +pub(super) async fn runtime_metrics(_body: &[&str]) -> Result { let out = services().server.metrics.runtime_metrics().map_or_else( || "Runtime metrics are not available.".to_owned(), |metrics| format!("```rs\n{metrics:#?}\n```"), @@ -717,14 +718,14 @@ pub(super) async fn runtime_metrics(_body: Vec<&str>) -> Result) -> Result { +pub(super) async fn runtime_metrics(_body: &[&str]) -> Result { Ok(RoomMessageEventContent::text_markdown( "Runtime metrics require building with `tokio_unstable`.", )) } #[cfg(tokio_unstable)] -pub(super) async fn runtime_interval(_body: Vec<&str>) -> Result { +pub(super) async fn runtime_interval(_body: &[&str]) -> Result { let out = services().server.metrics.runtime_interval().map_or_else( || "Runtime metrics are not available.".to_owned(), |metrics| format!("```rs\n{metrics:#?}\n```"), @@ -734,13 +735,13 @@ pub(super) async fn runtime_interval(_body: Vec<&str>) -> Result) -> Result { +pub(super) async fn runtime_interval(_body: &[&str]) -> Result { Ok(RoomMessageEventContent::text_markdown( "Runtime metrics require building with `tokio_unstable`.", )) } -pub(super) async fn time(_body: Vec<&str>) -> Result { +pub(super) async fn time(_body: &[&str]) -> Result { let now = SystemTime::now(); Ok(RoomMessageEventContent::text_markdown(utils::time::format(now, "%+"))) } diff --git a/src/macros/admin.rs b/src/macros/admin.rs index e95e402a..a1a5b22a 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -32,7 +32,7 @@ fn dispatch_arm(v: &Variant) -> TokenStream2 { let field = fields.named.iter().filter_map(|f| f.ident.as_ref()); let arg = field.clone(); quote! { - #name { #( #field ),* } => Box::pin(#handler(body, #( #arg ),*)).await?, + #name { #( #field ),* } => Box::pin(#handler(&body, #( #arg ),*)).await?, } }, Fields::Unnamed(fields) => { @@ -43,7 +43,7 @@ fn dispatch_arm(v: &Variant) -> TokenStream2 { }, Fields::Unit => { quote! { - #name => Box::pin(#handler(body)).await?, + #name => Box::pin(#handler(&body)).await?, } }, } From d7d874f88d4d00204883bf3f773d76d75d0d54f1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 01:06:58 +0000 Subject: [PATCH 0579/2091] start core info module; move version to info Signed-off-by: Jason Volk --- src/core/info/mod.rs | 4 ++++ src/core/{ => info}/version.rs | 13 +++++++------ src/core/mod.rs | 4 ++-- 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 src/core/info/mod.rs rename src/core/{ => info}/version.rs (75%) diff --git a/src/core/info/mod.rs b/src/core/info/mod.rs new file mode 100644 index 00000000..42ec971e --- /dev/null +++ b/src/core/info/mod.rs @@ -0,0 +1,4 @@ +//! Information about the project. This module contains version, build, system, +//! etc information which can be queried by admins or used by developers. + +pub mod version; diff --git a/src/core/version.rs b/src/core/info/version.rs similarity index 75% rename from src/core/version.rs rename to src/core/info/version.rs index 2876cea8..fb71d4e1 100644 --- a/src/core/version.rs +++ b/src/core/info/version.rs @@ -1,9 +1,10 @@ -/// one true function for returning the conduwuit version with the necessary -/// CONDUWUIT_VERSION_EXTRA env variables used if specified -/// -/// Set the environment variable `CONDUWUIT_VERSION_EXTRA` to any UTF-8 string -/// to include it in parenthesis after the SemVer version. A common value are -/// git commit hashes. +//! one true function for returning the conduwuit version with the necessary +//! CONDUWUIT_VERSION_EXTRA env variables used if specified +//! +//! Set the environment variable `CONDUWUIT_VERSION_EXTRA` to any UTF-8 string +//! to include it in parenthesis after the SemVer version. A common value are +//! git commit hashes. + use std::sync::OnceLock; static BRANDING: &str = "Conduwuit"; diff --git a/src/core/mod.rs b/src/core/mod.rs index 9716b46e..749d7b08 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -2,19 +2,19 @@ pub mod alloc; pub mod config; pub mod debug; pub mod error; +pub mod info; pub mod log; pub mod metrics; pub mod mods; pub mod pdu; pub mod server; pub mod utils; -pub mod version; pub use config::Config; pub use error::Error; +pub use info::{version, version::version}; pub use pdu::{PduBuilder, PduCount, PduEvent}; pub use server::Server; -pub use version::version; pub type Result = std::result::Result; From 7d487d53d81d8095c4fd9128f848e0bde47427b5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 03:55:01 +0000 Subject: [PATCH 0580/2091] de-cycle conduit_macros from conduit_core. Signed-off-by: Jason Volk --- Cargo.lock | 1 - src/macros/Cargo.toml | 1 - src/macros/admin.rs | 3 ++- src/macros/mod.rs | 1 + src/macros/utils.rs | 25 +++++++++++++++++++++++++ 5 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 src/macros/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 0649f3be..71d79adb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -717,7 +717,6 @@ dependencies = [ name = "conduit_macros" version = "0.4.5" dependencies = [ - "conduit_core", "proc-macro2", "quote", "syn 1.0.109", diff --git a/src/macros/Cargo.toml b/src/macros/Cargo.toml index b9a35aab..ca98f169 100644 --- a/src/macros/Cargo.toml +++ b/src/macros/Cargo.toml @@ -18,7 +18,6 @@ proc-macro = true syn.workspace = true quote.workspace = true proc-macro2.workspace = true -conduit-core.workspace = true [lints] workspace = true diff --git a/src/macros/admin.rs b/src/macros/admin.rs index a1a5b22a..e1d294b9 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -1,9 +1,10 @@ -use conduit_core::utils::string::camel_to_snake_string; use proc_macro::{Span, TokenStream}; use proc_macro2::TokenStream as TokenStream2; use quote::quote; use syn::{parse_macro_input, AttributeArgs, Fields, Ident, ItemEnum, Variant}; +use crate::utils::camel_to_snake_string; + pub(super) fn command_dispatch(args: TokenStream, input_: TokenStream) -> TokenStream { let input = input_.clone(); let item = parse_macro_input!(input as ItemEnum); diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 0aba7560..6f286d66 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -1,4 +1,5 @@ mod admin; +mod utils; use proc_macro::TokenStream; diff --git a/src/macros/utils.rs b/src/macros/utils.rs new file mode 100644 index 00000000..f512c56c --- /dev/null +++ b/src/macros/utils.rs @@ -0,0 +1,25 @@ +#[must_use] +pub(crate) fn camel_to_snake_string(s: &str) -> String { + let mut output = String::with_capacity( + s.chars() + .fold(s.len(), |a, ch| a.saturating_add(usize::from(ch.is_ascii_uppercase()))), + ); + + let mut state = false; + s.chars().for_each(|ch| { + let m = ch.is_ascii_uppercase(); + let s = exchange(&mut state, !m); + if m && s { + output.push('_'); + } + output.push(ch.to_ascii_lowercase()); + }); + + output +} + +pub(crate) fn exchange(state: &mut T, source: T) -> T { + let ret = state.clone(); + *state = source; + ret +} From f01423164479345493c06c86121678113e27d58e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 09:04:16 +0000 Subject: [PATCH 0581/2091] add conf item to disable rocksdb compaction Signed-off-by: Jason Volk --- src/core/config/mod.rs | 3 +++ src/database/opts.rs | 1 + 2 files changed, 4 insertions(+) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 6cef402a..ae273545 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -236,6 +236,8 @@ pub struct Config { pub rocksdb_compaction_prio_idle: bool, #[serde(default = "true_fn")] pub rocksdb_compaction_ioprio_idle: bool, + #[serde(default = "true_fn")] + pub rocksdb_compaction: bool, pub emergency_password: Option, @@ -712,6 +714,7 @@ impl fmt::Display for Config { "RocksDB Compaction Idle IOPriority", &self.rocksdb_compaction_ioprio_idle.to_string(), ); + line("RocksDB Compaction enabled", &self.rocksdb_compaction.to_string()); line("Media integrity checks on startup", &self.media_startup_check.to_string()); line("Media compatibility filesystem links", &self.media_compat_file_link.to_string()); line("Prevent Media Downloads From", { diff --git a/src/database/opts.rs b/src/database/opts.rs index d2236454..3c3ad424 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -68,6 +68,7 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ set_compression_defaults(&mut opts, config); // Misc + opts.set_disable_auto_compactions(!config.rocksdb_compaction); opts.create_if_missing(true); // Default: https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes#ktoleratecorruptedtailrecords From 936d2915e2257be20213e3501e96be3df10c5f49 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 09:10:01 +0000 Subject: [PATCH 0582/2091] add cargo manifest reflection Signed-off-by: Jason Volk --- Cargo.lock | 12 ++++++++ Cargo.toml | 4 +++ src/core/Cargo.toml | 2 ++ src/core/error/mod.rs | 2 ++ src/core/info/cargo.rs | 66 ++++++++++++++++++++++++++++++++++++++++++ src/core/info/mod.rs | 1 + src/macros/cargo.rs | 51 ++++++++++++++++++++++++++++++++ src/macros/mod.rs | 4 +++ 8 files changed, 142 insertions(+) create mode 100644 src/core/info/cargo.rs create mode 100644 src/macros/cargo.rs diff --git a/Cargo.lock b/Cargo.lock index 71d79adb..d9af38d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -472,6 +472,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cargo_toml" +version = "0.20.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad639525b1c67b6a298f378417b060fbc04618bea559482a8484381cce27d965" +dependencies = [ + "serde", + "toml", +] + [[package]] name = "cc" version = "1.1.6" @@ -666,8 +676,10 @@ dependencies = [ "argon2", "axum 0.7.5", "bytes", + "cargo_toml", "checked_ops", "chrono", + "conduit_macros", "const-str", "either", "figment", diff --git a/Cargo.toml b/Cargo.toml index 5fcb03ef..acaed701 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,10 @@ name = "conduit" [workspace.dependencies.const-str] version = "0.5.7" +[workspace.dependencies.cargo_toml] +version = "0.20" +features = ["features"] + [workspace.dependencies.sanitize-filename] version = "0.5.0" diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 453d7b13..620aad02 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -53,8 +53,10 @@ sha256_media = [] argon2.workspace = true axum.workspace = true bytes.workspace = true +cargo_toml.workspace = true checked_ops.workspace = true chrono.workspace = true +conduit-macros.workspace = true const-str.workspace = true either.workspace = true figment.workspace = true diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 9439261e..8664d740 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -55,6 +55,8 @@ pub enum Error { Http(#[from] http::Error), #[error("{0}")] HttpHeader(#[from] http::header::InvalidHeaderValue), + #[error("{0}")] + CargoToml(#[from] cargo_toml::Error), // ruma #[error("{0}")] diff --git a/src/core/info/cargo.rs b/src/core/info/cargo.rs new file mode 100644 index 00000000..0d2db1ad --- /dev/null +++ b/src/core/info/cargo.rs @@ -0,0 +1,66 @@ +//! Information about the build related to Cargo. This is a frontend interface +//! informed by proc-macros that capture raw information at build time which is +//! further processed at runtime either during static initialization or as +//! necessary. + +use std::sync::OnceLock; + +use cargo_toml::Manifest; +use conduit_macros::cargo_manifest; + +use crate::Result; + +// Raw captures of the cargo manifest for each crate. This is provided by a +// proc-macro at build time since the source directory and the cargo toml's may +// not be present during execution. + +#[cargo_manifest] +const WORKSPACE_MANIFEST: &'static str = (); +#[cargo_manifest("macros")] +const MACROS_MANIFEST: &'static str = (); +#[cargo_manifest("core")] +const CORE_MANIFEST: &'static str = (); +#[cargo_manifest("database")] +const DATABASE_MANIFEST: &'static str = (); +#[cargo_manifest("service")] +const SERVICE_MANIFEST: &'static str = (); +#[cargo_manifest("admin")] +const ADMIN_MANIFEST: &'static str = (); +#[cargo_manifest("router")] +const ROUTER_MANIFEST: &'static str = (); +#[cargo_manifest("main")] +const MAIN_MANIFEST: &'static str = (); + +/// Processed list of features access all project crates. This is generated from +/// the data in the MANIFEST strings and contains all possible project features. +/// For *enabled* features see the info::rustc module instead. +static FEATURES: OnceLock> = OnceLock::new(); + +/// List of all possible features for the project. For *enabled* features in +/// this build see the companion function in info::rustc. +pub fn features() -> &'static Vec { + FEATURES.get_or_init(|| init_features().unwrap_or_else(|e| panic!("Failed initialize features: {e}"))) +} + +fn init_features() -> Result> { + let mut features = Vec::new(); + append_features(&mut features, WORKSPACE_MANIFEST)?; + append_features(&mut features, MACROS_MANIFEST)?; + append_features(&mut features, CORE_MANIFEST)?; + append_features(&mut features, DATABASE_MANIFEST)?; + append_features(&mut features, SERVICE_MANIFEST)?; + append_features(&mut features, ADMIN_MANIFEST)?; + append_features(&mut features, ROUTER_MANIFEST)?; + append_features(&mut features, MAIN_MANIFEST)?; + features.sort(); + features.dedup(); + + Ok(features) +} + +fn append_features(features: &mut Vec, manifest: &str) -> Result<()> { + let manifest = Manifest::from_str(manifest)?; + features.extend(manifest.features.keys().cloned()); + + Ok(()) +} diff --git a/src/core/info/mod.rs b/src/core/info/mod.rs index 42ec971e..7749bbdc 100644 --- a/src/core/info/mod.rs +++ b/src/core/info/mod.rs @@ -1,4 +1,5 @@ //! Information about the project. This module contains version, build, system, //! etc information which can be queried by admins or used by developers. +pub mod cargo; pub mod version; diff --git a/src/macros/cargo.rs b/src/macros/cargo.rs new file mode 100644 index 00000000..17132a6c --- /dev/null +++ b/src/macros/cargo.rs @@ -0,0 +1,51 @@ +use std::{fs::read_to_string, path::PathBuf}; + +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, AttributeArgs, ItemConst, Lit, NestedMeta}; + +pub(super) fn manifest(args: TokenStream, item: TokenStream) -> TokenStream { + let item = parse_macro_input!(item as ItemConst); + let args = parse_macro_input!(args as AttributeArgs); + let member = args.into_iter().find_map(|arg| { + let NestedMeta::Lit(arg) = arg else { + return None; + }; + let Lit::Str(arg) = arg else { + return None; + }; + Some(arg.value()) + }); + + let path = manifest_path(member.as_deref()); + let manifest = read_to_string(&path).unwrap_or_default(); + + let name = item.ident; + let val = manifest.as_str(); + let ret = quote! { + const #name: &'static str = #val; + }; + + ret.into() +} + +#[allow(clippy::option_env_unwrap)] +fn manifest_path(member: Option<&str>) -> PathBuf { + let mut path: PathBuf = option_env!("CARGO_MANIFEST_DIR") + .expect("missing CARGO_MANIFEST_DIR in environment") + .into(); + + // conduwuit/src/macros/ -> conduwuit/src/ + path.pop(); + + if let Some(member) = member { + // conduwuit/$member/Cargo.toml + path.push(member); + } else { + // conduwuit/src/ -> conduwuit/ + path.pop(); + } + + path.push("Cargo.toml"); + path +} diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 6f286d66..718583a4 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -1,4 +1,5 @@ mod admin; +mod cargo; mod utils; use proc_macro::TokenStream; @@ -7,3 +8,6 @@ use proc_macro::TokenStream; pub fn admin_command_dispatch(args: TokenStream, input: TokenStream) -> TokenStream { admin::command_dispatch(args, input) } + +#[proc_macro_attribute] +pub fn cargo_manifest(args: TokenStream, input: TokenStream) -> TokenStream { cargo::manifest(args, input) } From 2100618d47ae71ba0d272c65a0b5ddd5b3d7baf2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 23:01:00 +0000 Subject: [PATCH 0583/2091] add rustc build flags reflection Signed-off-by: Jason Volk --- Cargo.lock | 11 +++++++++ Cargo.toml | 3 +++ src/admin/mod.rs | 7 +++--- src/core/Cargo.toml | 1 + src/core/info/mod.rs | 3 +++ src/core/info/rustc.rs | 53 ++++++++++++++++++++++++++++++++++++++++++ src/core/mod.rs | 7 +++++- src/core/utils/mod.rs | 1 + src/database/mod.rs | 1 + src/macros/mod.rs | 4 ++++ src/macros/rustc.rs | 27 +++++++++++++++++++++ src/main/main.rs | 4 +++- src/router/mod.rs | 1 + src/service/mod.rs | 1 + 14 files changed, 119 insertions(+), 5 deletions(-) create mode 100644 src/core/info/rustc.rs create mode 100644 src/macros/rustc.rs diff --git a/Cargo.lock b/Cargo.lock index d9af38d2..52ea07c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -681,6 +681,7 @@ dependencies = [ "chrono", "conduit_macros", "const-str", + "ctor", "either", "figment", "hardened_malloc-rs", @@ -1018,6 +1019,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctor" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" +dependencies = [ + "quote", + "syn 2.0.71", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" diff --git a/Cargo.toml b/Cargo.toml index acaed701..b65ba7ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,9 @@ name = "conduit" [workspace.dependencies.const-str] version = "0.5.7" +[workspace.dependencies.ctor] +version = "0.2.8" + [workspace.dependencies.cargo_toml] version = "0.20" features = ["features"] diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 7d752ff8..cd1110ee 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -21,15 +21,16 @@ extern crate conduit_api as api; extern crate conduit_core as conduit; extern crate conduit_service as service; -pub(crate) use conduit::{mod_ctor, mod_dtor, Result}; +pub(crate) use conduit::Result; pub(crate) use service::services; pub(crate) use crate::utils::{escape_html, get_room_info}; pub(crate) const PAGE_SIZE: usize = 100; -mod_ctor! {} -mod_dtor! {} +conduit::mod_ctor! {} +conduit::mod_dtor! {} +conduit::rustc_flags_capture! {} /// Install the admin command handler pub async fn init() { diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 620aad02..ea772a83 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -58,6 +58,7 @@ checked_ops.workspace = true chrono.workspace = true conduit-macros.workspace = true const-str.workspace = true +ctor.workspace = true either.workspace = true figment.workspace = true http-body-util.workspace = true diff --git a/src/core/info/mod.rs b/src/core/info/mod.rs index 7749bbdc..e11a6021 100644 --- a/src/core/info/mod.rs +++ b/src/core/info/mod.rs @@ -2,4 +2,7 @@ //! etc information which can be queried by admins or used by developers. pub mod cargo; +pub mod rustc; pub mod version; + +pub use conduit_macros::rustc_flags_capture; diff --git a/src/core/info/rustc.rs b/src/core/info/rustc.rs new file mode 100644 index 00000000..048c0cd5 --- /dev/null +++ b/src/core/info/rustc.rs @@ -0,0 +1,53 @@ +//! Information about the build related to rustc. This is a frontend interface +//! informed by proc-macros at build time. Since the project is split into +//! several crates, lower-level information is supplied from each crate during +//! static initialization. + +use std::{ + collections::BTreeMap, + sync::{Mutex, OnceLock}, +}; + +use crate::utils::exchange; + +/// Raw capture of rustc flags used to build each crate in the project. Informed +/// by rustc_flags_capture macro (one in each crate's mod.rs). This is +/// done during static initialization which is why it's mutex-protected and pub. +/// Should not be written to by anything other than our macro. +pub static FLAGS: Mutex> = Mutex::new(BTreeMap::new()); + +/// Processed list of enabled features across all project crates. This is +/// generated from the data in FLAGS. +static FEATURES: OnceLock> = OnceLock::new(); + +/// List of features enabled for the project. +pub fn features() -> &'static Vec<&'static str> { FEATURES.get_or_init(init_features) } + +fn init_features() -> Vec<&'static str> { + let mut features = Vec::new(); + FLAGS + .lock() + .expect("locked") + .iter() + .for_each(|(_, flags)| append_features(&mut features, flags)); + + features.sort_unstable(); + features.dedup(); + features +} + +fn append_features(features: &mut Vec<&'static str>, flags: &[&'static str]) { + let mut next_is_cfg = false; + for flag in flags { + let is_cfg = *flag == "--cfg"; + let is_feature = flag.starts_with("feature="); + if exchange(&mut next_is_cfg, is_cfg) && is_feature { + if let Some(feature) = flag + .split_once('=') + .map(|(_, feature)| feature.trim_matches('"')) + { + features.push(feature); + } + } + } +} diff --git a/src/core/mod.rs b/src/core/mod.rs index 749d7b08..b302fdcc 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -12,12 +12,17 @@ pub mod utils; pub use config::Config; pub use error::Error; -pub use info::{version, version::version}; +pub use info::{rustc_flags_capture, version, version::version}; pub use pdu::{PduBuilder, PduCount, PduEvent}; pub use server::Server; +pub use utils::{ctor, dtor}; + +pub use crate as conduit_core; pub type Result = std::result::Result; +rustc_flags_capture! {} + #[cfg(not(conduit_mods))] pub mod mods { #[macro_export] diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index d7b6a72a..767b65a9 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -15,6 +15,7 @@ pub mod time; use std::cmp::{self, Ordering}; +pub use ::ctor::{ctor, dtor}; pub use bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}; pub use debug::slice_truncated as debug_slice_truncated; pub use hash::calculate_hash; diff --git a/src/database/mod.rs b/src/database/mod.rs index 283224f6..6446624c 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -23,3 +23,4 @@ pub(crate) use util::{or_else, result}; conduit::mod_ctor! {} conduit::mod_dtor! {} +conduit::rustc_flags_capture! {} diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 718583a4..94ea781e 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -1,5 +1,6 @@ mod admin; mod cargo; +mod rustc; mod utils; use proc_macro::TokenStream; @@ -11,3 +12,6 @@ pub fn admin_command_dispatch(args: TokenStream, input: TokenStream) -> TokenStr #[proc_macro_attribute] pub fn cargo_manifest(args: TokenStream, input: TokenStream) -> TokenStream { cargo::manifest(args, input) } + +#[proc_macro] +pub fn rustc_flags_capture(args: TokenStream) -> TokenStream { rustc::flags_capture(args) } diff --git a/src/macros/rustc.rs b/src/macros/rustc.rs new file mode 100644 index 00000000..a3bab26f --- /dev/null +++ b/src/macros/rustc.rs @@ -0,0 +1,27 @@ +use proc_macro::TokenStream; +use quote::quote; + +pub(super) fn flags_capture(args: TokenStream) -> TokenStream { + let cargo_crate_name = std::env::var("CARGO_CRATE_NAME"); + let crate_name = match cargo_crate_name.as_ref() { + Err(_) => return args, + Ok(crate_name) => crate_name.trim_start_matches("conduit_"), + }; + + let flag = std::env::args().collect::>(); + let ret = quote! { + #[conduit_core::ctor] + fn _set_rustc_flags() { + let flags = &[#( #flag ),*]; + conduit_core::info::rustc::FLAGS.lock().expect("locked").insert(#crate_name, flags); + } + + // static strings have to be yanked on module unload + #[conduit_core::dtor] + fn _unset_rustc_flags() { + conduit_core::info::rustc::FLAGS.lock().expect("locked").remove(#crate_name); + } + }; + + ret.into() +} diff --git a/src/main/main.rs b/src/main/main.rs index b13e117d..b8cb24ff 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -14,7 +14,7 @@ use std::{ time::Duration, }; -use conduit::{debug_info, error, utils::available_parallelism, Error, Result}; +use conduit::{debug_info, error, rustc_flags_capture, utils::available_parallelism, Error, Result}; use server::Server; use tokio::runtime; @@ -22,6 +22,8 @@ const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; const WORKER_KEEPALIVE: u64 = 36; +rustc_flags_capture! {} + fn main() -> Result<(), Error> { let args = clap::parse(); let runtime = runtime::Builder::new_multi_thread() diff --git a/src/router/mod.rs b/src/router/mod.rs index 03c70f6d..13fe3908 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -14,6 +14,7 @@ use conduit::{Result, Server}; conduit::mod_ctor! {} conduit::mod_dtor! {} +conduit::rustc_flags_capture! {} #[no_mangle] pub extern "Rust" fn start(server: &Arc) -> Pin> + Send>> { diff --git a/src/service/mod.rs b/src/service/mod.rs index ce106809..b6ec58b5 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -37,6 +37,7 @@ pub use crate::services::Services; conduit::mod_ctor! {} conduit::mod_dtor! {} +conduit::rustc_flags_capture! {} static SERVICES: RwLock> = RwLock::new(None); From 8bb69eb81ddf1fb90a47f02ee078da943438352c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 25 Jul 2024 01:41:31 +0000 Subject: [PATCH 0584/2091] add simple ast dimension diagnostic Signed-off-by: Jason Volk --- src/core/debug.rs | 3 +++ src/macros/debug.rs | 28 ++++++++++++++++++++++++++++ src/macros/mod.rs | 4 ++++ 3 files changed, 35 insertions(+) create mode 100644 src/macros/debug.rs diff --git a/src/core/debug.rs b/src/core/debug.rs index 5e52f399..63f25d47 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -1,5 +1,8 @@ use std::{any::Any, panic}; +/// Export debug proc_macros +pub use conduit_macros::recursion_depth; + /// Export all of the ancillary tools from here as well. pub use crate::utils::debug::*; diff --git a/src/macros/debug.rs b/src/macros/debug.rs new file mode 100644 index 00000000..5251fa17 --- /dev/null +++ b/src/macros/debug.rs @@ -0,0 +1,28 @@ +use std::cmp; + +use proc_macro::TokenStream; +use syn::{parse_macro_input, AttributeArgs, Item}; + +pub(super) fn recursion_depth(args: TokenStream, item_: TokenStream) -> TokenStream { + let item = item_.clone(); + let item = parse_macro_input!(item as Item); + let _args = parse_macro_input!(args as AttributeArgs); + + let mut best: usize = 0; + let mut count: usize = 0; + // think you'd find a fancy recursive ast visitor? think again + let tree = format!("{item:#?}"); + for line in tree.lines() { + let trim = line.trim_start_matches(' '); + let diff = line.len().saturating_sub(trim.len()); + let level = diff / 4; + best = cmp::max(level, best); + count = count.saturating_add(1); + } + + println!("--- Recursion Diagnostic ---"); + println!("DEPTH: {best}"); + println!("LENGTH: {count}"); + + item_ +} diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 94ea781e..b01e5275 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -1,5 +1,6 @@ mod admin; mod cargo; +mod debug; mod rustc; mod utils; @@ -13,5 +14,8 @@ pub fn admin_command_dispatch(args: TokenStream, input: TokenStream) -> TokenStr #[proc_macro_attribute] pub fn cargo_manifest(args: TokenStream, input: TokenStream) -> TokenStream { cargo::manifest(args, input) } +#[proc_macro_attribute] +pub fn recursion_depth(args: TokenStream, input: TokenStream) -> TokenStream { debug::recursion_depth(args, input) } + #[proc_macro] pub fn rustc_flags_capture(args: TokenStream) -> TokenStream { rustc::flags_capture(args) } From 4e975887cf2e2e7f81d28789fca2fc4a2fb32657 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 24 Jul 2024 23:53:48 +0000 Subject: [PATCH 0585/2091] add command to list features Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 45 +++++++++++++++++++++++++++++++++++- src/admin/server/mod.rs | 17 ++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index e4503736..c30f8f87 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,4 +1,6 @@ -use conduit::{utils::time, warn, Err, Result}; +use std::fmt::Write; + +use conduit::{info, utils::time, warn, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; use crate::services; @@ -19,6 +21,47 @@ pub(super) async fn show_config(_body: Vec<&str>) -> Result, available: bool, enabled: bool, comma: bool, +) -> Result { + let delim = if comma { + "," + } else { + " " + }; + if enabled && !available { + let features = info::rustc::features().join(delim); + let out = format!("```\n{features}\n```"); + return Ok(RoomMessageEventContent::text_markdown(out)); + } + + if available && !enabled { + let features = info::cargo::features().join(delim); + let out = format!("```\n{features}\n```"); + return Ok(RoomMessageEventContent::text_markdown(out)); + } + + let mut features = String::new(); + let enabled = info::rustc::features(); + let available = info::cargo::features(); + for feature in available { + let active = enabled.contains(&feature.as_str()); + let emoji = if active { + "✅" + } else { + "❌" + }; + let remark = if active { + "[enabled]" + } else { + "" + }; + writeln!(features, "{emoji} {feature} {remark}")?; + } + + Ok(RoomMessageEventContent::text_markdown(features)) +} + pub(super) async fn memory_usage(_body: Vec<&str>) -> Result { let services_usage = services().memory_usage().await?; let database_usage = services().db.db.memory_usage()?; diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 604dacc7..958cc54b 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -14,6 +14,18 @@ pub(super) enum ServerCommand { /// - Show configuration values ShowConfig, + /// - List the features built into the server + ListFeatures { + #[arg(short, long)] + available: bool, + + #[arg(short, long)] + enabled: bool, + + #[arg(short, long)] + comma: bool, + }, + /// - Print database memory usage statistics MemoryUsage, @@ -54,6 +66,11 @@ pub(super) async fn process(command: ServerCommand, body: Vec<&str>) -> Result uptime(body).await?, ServerCommand::ShowConfig => show_config(body).await?, + ServerCommand::ListFeatures { + available, + enabled, + comma, + } => list_features(body, available, enabled, comma).await?, ServerCommand::MemoryUsage => memory_usage(body).await?, ServerCommand::ClearCaches => clear_caches(body).await?, ServerCommand::ListBackups => list_backups(body).await?, From c423a8365692c6a02780d3f4224c4a0fdcd16e0e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 25 Jul 2024 02:59:54 +0000 Subject: [PATCH 0586/2091] add cli override for any configuration item Signed-off-by: Jason Volk --- Cargo.lock | 1 + Cargo.toml | 5 +++++ src/core/Cargo.toml | 1 + src/core/config/mod.rs | 17 +++++++++------- src/core/error/mod.rs | 6 ++++++ src/core/mod.rs | 1 + src/main/clap.rs | 46 +++++++++++++++++++++++++++++++++++++----- src/main/server.rs | 5 +++-- 8 files changed, 68 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52ea07c9..c73ef931 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -708,6 +708,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-metrics", + "toml", "tracing", "tracing-core", "tracing-subscriber", diff --git a/Cargo.toml b/Cargo.toml index b65ba7ad..66ba7385 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,11 @@ version = "0.2.8" version = "0.20" features = ["features"] +[workspace.dependencies.toml] +version = "0.8.14" +default-features = false +features = ["parse"] + [workspace.dependencies.sanitize-filename] version = "0.5.0" diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index ea772a83..f915c43e 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -86,6 +86,7 @@ tikv-jemalloc-sys.optional = true tikv-jemalloc-sys.workspace = true tokio.workspace = true tokio-metrics.workspace = true +toml.workspace = true tracing-core.workspace = true tracing-subscriber.workspace = true tracing.workspace = true diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index ae273545..a9443fd2 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -9,10 +9,8 @@ use either::{ Either, Either::{Left, Right}, }; -use figment::{ - providers::{Env, Format, Toml}, - Figment, -}; +use figment::providers::{Env, Format, Toml}; +pub use figment::{value::Value as FigmentValue, Figment}; use itertools::Itertools; use regex::RegexSet; use ruma::{ @@ -408,8 +406,8 @@ const DEPRECATED_KEYS: &[&str; 9] = &[ ]; impl Config { - /// Initialize config - pub fn new(path: &Option) -> Result { + /// Pre-initialize config + pub fn load(path: &Option) -> Result { let raw_config = if let Some(config_file_env) = Env::var("CONDUIT_CONFIG") { Figment::new() .merge(Toml::file(config_file_env).nested()) @@ -431,13 +429,18 @@ impl Config { .merge(Env::prefixed("CONDUWUIT_").global().split("__")) }; + Ok(raw_config) + } + + /// Finalize config + pub fn new(raw_config: &Figment) -> Result { let config = match raw_config.extract::() { Err(e) => return Err!("There was a problem with your configuration file: {e}"), Ok(config) => config, }; // don't start if we're listening on both UNIX sockets and TCP at same time - check::is_dual_listening(&raw_config)?; + check::is_dual_listening(raw_config)?; Ok(config) } diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 8664d740..5406d5ff 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -57,6 +57,12 @@ pub enum Error { HttpHeader(#[from] http::header::InvalidHeaderValue), #[error("{0}")] CargoToml(#[from] cargo_toml::Error), + #[error("{0}")] + FigmentError(#[from] figment::error::Error), + #[error("{0}")] + TomlSerError(#[from] toml::ser::Error), + #[error("{0}")] + TomlDeError(#[from] toml::de::Error), // ruma #[error("{0}")] diff --git a/src/core/mod.rs b/src/core/mod.rs index b302fdcc..5ed5ea15 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -10,6 +10,7 @@ pub mod pdu; pub mod server; pub mod utils; +pub use ::toml; pub use config::Config; pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; diff --git a/src/main/clap.rs b/src/main/clap.rs index 16d88978..92bd73c1 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -3,16 +3,24 @@ use std::path::PathBuf; use clap::Parser; -use conduit::{Config, Result}; +use conduit::{ + config::{Figment, FigmentValue}, + err, toml, Err, Result, +}; /// Commandline arguments #[derive(Parser, Debug)] #[clap(version = conduit::version(), about, long_about = None)] pub(crate) struct Args { #[arg(short, long)] - /// Optional argument to the path of a conduwuit config TOML file + /// Path to the config TOML file (optional) pub(crate) config: Option, + /// Override a configuration variable using TOML 'key=value' syntax + #[arg(long, short('O'))] + pub(crate) option: Vec, + + #[cfg(feature = "console")] /// Activate admin command console automatically after startup. #[arg(long, num_args(0))] pub(crate) console: bool, @@ -23,10 +31,38 @@ pub(crate) struct Args { pub(super) fn parse() -> Args { Args::parse() } /// Synthesize any command line options with configuration file options. -pub(crate) fn update(config: &mut Config, args: &Args) -> Result<()> { +pub(crate) fn update(mut config: Figment, args: &Args) -> Result { + #[cfg(feature = "console")] // Indicate the admin console should be spawned automatically if the // configuration file hasn't already. - config.admin_console_automatic |= args.console; + if args.console { + config = config.join(("admin_console_automatic", true)); + } - Ok(()) + // All other individual overrides can go last in case we have options which + // set multiple conf items at once and the user still needs granular overrides. + for option in &args.option { + let (key, val) = option + .split_once('=') + .ok_or_else(|| err!("Missing '=' in -O/--option: {option:?}"))?; + + if key.is_empty() { + return Err!("Missing key= in -O/--option: {option:?}"); + } + + if val.is_empty() { + return Err!("Missing =val in -O/--option: {option:?}"); + } + + // The value has to pass for what would appear as a line in the TOML file. + let val = toml::from_str::(option)?; + let FigmentValue::Dict(_, val) = val else { + panic!("Unexpected Figment Value: {val:#?}"); + }; + + // Figment::merge() overrides existing + config = config.merge((key, val[key].clone())); + } + + Ok(config) } diff --git a/src/main/server.rs b/src/main/server.rs index b1cd6936..71cdadce 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -22,8 +22,9 @@ pub(crate) struct Server { impl Server { pub(crate) fn build(args: &Args, runtime: Option<&runtime::Handle>) -> Result, Error> { - let mut config = Config::new(&args.config)?; - crate::clap::update(&mut config, args)?; + let raw_config = Config::load(&args.config)?; + let raw_config = crate::clap::update(raw_config, args)?; + let config = Config::new(&raw_config)?; #[cfg(feature = "sentry_telemetry")] let sentry_guard = crate::sentry::init(&config); From 271959ee271229e21594c7ddb2d2f52165ad8955 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 25 Jul 2024 21:29:37 +0000 Subject: [PATCH 0587/2091] add debug list-dependencies admin command Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 24 ++++++++++++++++++++++++ src/admin/debug/mod.rs | 6 ++++++ src/core/info/cargo.rs | 24 +++++++++++++++++++++++- 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 32fba409..307495d7 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -745,3 +745,27 @@ pub(super) async fn time(_body: &[&str]) -> Result { let now = SystemTime::now(); Ok(RoomMessageEventContent::text_markdown(utils::time::format(now, "%+"))) } + +pub(super) async fn list_dependencies(_body: &[&str], names: bool) -> Result { + if names { + let out = info::cargo::dependencies_names().join(" "); + return Ok(RoomMessageEventContent::notice_markdown(out)); + } + + let deps = info::cargo::dependencies(); + let mut out = String::new(); + writeln!(out, "| name | version | features |")?; + writeln!(out, "| ---- | ------- | -------- |")?; + for (name, dep) in deps { + let version = dep.try_req().unwrap_or("*"); + let feats = dep.req_features(); + let feats = if !feats.is_empty() { + feats.join(" ") + } else { + String::new() + }; + writeln!(out, "{name} | {version} | {feats}")?; + } + + Ok(RoomMessageEventContent::notice_markdown(out)) +} diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 77bc36b8..babd6047 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -178,6 +178,12 @@ pub(super) enum DebugCommand { /// - Print the current time Time, + /// - List dependencies + ListDependencies { + #[arg(short, long)] + names: bool, + }, + /// - Developer test stubs #[command(subcommand)] #[allow(non_snake_case)] diff --git a/src/core/info/cargo.rs b/src/core/info/cargo.rs index 0d2db1ad..544bbb8f 100644 --- a/src/core/info/cargo.rs +++ b/src/core/info/cargo.rs @@ -5,7 +5,7 @@ use std::sync::OnceLock; -use cargo_toml::Manifest; +use cargo_toml::{DepsSet, Manifest}; use conduit_macros::cargo_manifest; use crate::Result; @@ -36,6 +36,18 @@ const MAIN_MANIFEST: &'static str = (); /// For *enabled* features see the info::rustc module instead. static FEATURES: OnceLock> = OnceLock::new(); +/// Processed list of dependencies. This is generated from the datas captured in +/// the MANIFEST. +static DEPENDENCIES: OnceLock = OnceLock::new(); + +#[must_use] +pub fn dependencies_names() -> Vec<&'static str> { dependencies().keys().map(String::as_str).collect() } + +pub fn dependencies() -> &'static DepsSet { + DEPENDENCIES + .get_or_init(|| init_dependencies().unwrap_or_else(|e| panic!("Failed to initialize dependencies: {e}"))) +} + /// List of all possible features for the project. For *enabled* features in /// this build see the companion function in info::rustc. pub fn features() -> &'static Vec { @@ -64,3 +76,13 @@ fn append_features(features: &mut Vec, manifest: &str) -> Result<()> { Ok(()) } + +fn init_dependencies() -> Result { + let manifest = Manifest::from_str(WORKSPACE_MANIFEST)?; + Ok(manifest + .workspace + .as_ref() + .expect("manifest has workspace section") + .dependencies + .clone()) +} From 68f42baf736258b90924cce1aee2511765d1abc2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 25 Jul 2024 22:13:22 +0000 Subject: [PATCH 0588/2091] rename admin Command to CommandInput Signed-off-by: Jason Volk --- src/admin/handler.rs | 10 +++++----- src/service/admin/mod.rs | 18 +++++++++--------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 6acb19bf..26a5ea41 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -11,7 +11,7 @@ use ruma::{ OwnedEventId, }; use service::{ - admin::{Command, CommandOutput, CommandResult, HandlerResult}, + admin::{CommandInput, CommandOutput, CommandResult, HandlerResult}, Services, }; @@ -30,10 +30,10 @@ pub(super) fn complete(line: &str) -> String { } #[must_use] -pub(super) fn handle(command: Command) -> HandlerResult { Box::pin(handle_command(command)) } +pub(super) fn handle(command: CommandInput) -> HandlerResult { Box::pin(handle_command(command)) } #[tracing::instrument(skip_all, name = "admin")] -async fn handle_command(command: Command) -> CommandResult { +async fn handle_command(command: CommandInput) -> CommandResult { AssertUnwindSafe(Box::pin(process_command(&command))) .catch_unwind() .await @@ -41,7 +41,7 @@ async fn handle_command(command: Command) -> CommandResult { .or_else(|error| handle_panic(&error, command)) } -async fn process_command(command: &Command) -> CommandOutput { +async fn process_command(command: &CommandInput) -> CommandOutput { Handler { services: service::services(), } @@ -50,7 +50,7 @@ async fn process_command(command: &Command) -> CommandOutput { .and_then(|content| reply(content, command.reply_id.clone())) } -fn handle_panic(error: &Error, command: Command) -> CommandResult { +fn handle_panic(error: &Error, command: CommandInput) -> CommandResult { let link = "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}"); let content = RoomMessageEventContent::notice_markdown(msg); diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index b3879deb..6241c668 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -26,8 +26,8 @@ use crate::{globals, rooms, rooms::state::RoomMutexGuard, Dep}; pub struct Service { services: Services, - sender: Sender, - receiver: Mutex>, + sender: Sender, + receiver: Mutex>, pub handle: RwLock>, pub complete: StdRwLock>, #[cfg(feature = "console")] @@ -44,13 +44,13 @@ struct Services { } #[derive(Debug)] -pub struct Command { +pub struct CommandInput { pub command: String, pub reply_id: Option, } pub type Completer = fn(&str) -> String; -pub type Handler = fn(Command) -> HandlerResult; +pub type Handler = fn(CommandInput) -> HandlerResult; pub type HandlerResult = Pin + Send>>; pub type CommandResult = Result; pub type CommandOutput = Option; @@ -129,7 +129,7 @@ impl Service { } pub async fn command(&self, command: String, reply_id: Option) { - self.send(Command { + self.send(CommandInput { command, reply_id, }) @@ -139,7 +139,7 @@ impl Service { pub async fn command_in_place( &self, command: String, reply_id: Option, ) -> Result> { - self.process_command(Command { + self.process_command(CommandInput { command, reply_id, }) @@ -153,7 +153,7 @@ impl Service { .map(|complete| complete(command)) } - async fn send(&self, message: Command) { + async fn send(&self, message: CommandInput) { debug_assert!(!self.sender.is_closed(), "channel closed"); self.sender.send_async(message).await.expect("message sent"); } @@ -163,7 +163,7 @@ impl Service { self.console.handle_signal(sig).await; } - async fn handle_command(&self, command: Command) { + async fn handle_command(&self, command: CommandInput) { match self.process_command(command).await { Ok(Some(output)) => self.handle_response(output).await, Ok(None) => debug!("Command successful with no response"), @@ -171,7 +171,7 @@ impl Service { } } - async fn process_command(&self, command: Command) -> CommandResult { + async fn process_command(&self, command: CommandInput) -> CommandResult { if let Some(handle) = self.handle.read().await.as_ref() { handle(command).await } else { From 96f6a75bc82a9dabd25f84b917b5358244bd55a4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 26 Jul 2024 06:13:30 +0000 Subject: [PATCH 0589/2091] add refutable pattern function macro Signed-off-by: Jason Volk --- src/macros/mod.rs | 4 +++ src/macros/refutable.rs | 61 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 src/macros/refutable.rs diff --git a/src/macros/mod.rs b/src/macros/mod.rs index b01e5275..a0e61324 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -1,6 +1,7 @@ mod admin; mod cargo; mod debug; +mod refutable; mod rustc; mod utils; @@ -19,3 +20,6 @@ pub fn recursion_depth(args: TokenStream, input: TokenStream) -> TokenStream { d #[proc_macro] pub fn rustc_flags_capture(args: TokenStream) -> TokenStream { rustc::flags_capture(args) } + +#[proc_macro_attribute] +pub fn refutable(args: TokenStream, input: TokenStream) -> TokenStream { refutable::refutable(args, input) } diff --git a/src/macros/refutable.rs b/src/macros/refutable.rs new file mode 100644 index 00000000..6a6884e0 --- /dev/null +++ b/src/macros/refutable.rs @@ -0,0 +1,61 @@ +use proc_macro::{Span, TokenStream}; +use quote::{quote, ToTokens}; +use syn::{parse_macro_input, AttributeArgs, FnArg::Typed, Ident, ItemFn, Pat, PatIdent, PatType, Stmt}; + +pub(super) fn refutable(args: TokenStream, input: TokenStream) -> TokenStream { + let _args = parse_macro_input!(args as AttributeArgs); + let mut item = parse_macro_input!(input as ItemFn); + + let inputs = item.sig.inputs.clone(); + let stmt = &mut item.block.stmts; + let sig = &mut item.sig; + for (i, input) in inputs.iter().enumerate() { + let Typed(PatType { + pat, + .. + }) = input + else { + continue; + }; + + let Pat::Struct(ref pat) = **pat else { + continue; + }; + + let variant = &pat.path; + let fields = &pat.fields; + + // new versions of syn can replace this kronecker kludge with get_mut() + for (j, input) in sig.inputs.iter_mut().enumerate() { + if i != j { + continue; + } + + let Typed(PatType { + ref mut pat, + .. + }) = input + else { + continue; + }; + + let name = format!("_args_{i}"); + *pat = Box::new(Pat::Ident(PatIdent { + ident: Ident::new(&name, Span::call_site().into()), + attrs: Vec::new(), + by_ref: None, + mutability: None, + subpat: None, + })); + + let field = fields.iter(); + let refute = quote! { + let #variant { #( #field ),*, .. } = #name else { panic!("incorrect variant passed to function argument {i}"); }; + }; + + stmt.insert(0, syn::parse2::(refute).expect("syntax error")); + } + } + + item.into_token_stream().into() +} From 3b5607ecdc772f9f30d572b436f66c222862b8c2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 26 Jul 2024 06:41:26 +0000 Subject: [PATCH 0590/2091] add macro for out-of-line definitions Signed-off-by: Jason Volk --- src/macros/implement.rs | 23 +++++++++++++++++++++++ src/macros/mod.rs | 4 ++++ 2 files changed, 27 insertions(+) create mode 100644 src/macros/implement.rs diff --git a/src/macros/implement.rs b/src/macros/implement.rs new file mode 100644 index 00000000..1a06e588 --- /dev/null +++ b/src/macros/implement.rs @@ -0,0 +1,23 @@ +use proc_macro::TokenStream; +use quote::{quote, ToTokens}; +use syn::{parse_macro_input, AttributeArgs, ItemFn, Meta, NestedMeta}; + +pub(super) fn implement(args: TokenStream, input: TokenStream) -> TokenStream { + let args = parse_macro_input!(args as AttributeArgs); + let item = parse_macro_input!(input as ItemFn); + + let NestedMeta::Meta(Meta::Path(receiver)) = args + .first() + .expect("missing path to trait or item to implement") + else { + panic!("invalid path to item for implement"); + }; + + let out = quote! { + impl #receiver { + #item + } + }; + + out.into_token_stream().into() +} diff --git a/src/macros/mod.rs b/src/macros/mod.rs index a0e61324..8d7e2e5b 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -1,6 +1,7 @@ mod admin; mod cargo; mod debug; +mod implement; mod refutable; mod rustc; mod utils; @@ -23,3 +24,6 @@ pub fn rustc_flags_capture(args: TokenStream) -> TokenStream { rustc::flags_capt #[proc_macro_attribute] pub fn refutable(args: TokenStream, input: TokenStream) -> TokenStream { refutable::refutable(args, input) } + +#[proc_macro_attribute] +pub fn implement(args: TokenStream, input: TokenStream) -> TokenStream { implement::implement(args, input) } From ca82b59c6fad717ea8b0d3143cb2759464e48ddf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 26 Jul 2024 20:40:07 +0000 Subject: [PATCH 0591/2091] upgrade to syn 2.x Signed-off-by: Jason Volk --- Cargo.lock | 3 ++- Cargo.toml | 2 +- src/core/info/cargo.rs | 14 +++++------ src/macros/Cargo.toml | 1 + src/macros/admin.rs | 35 +++++++++++++++----------- src/macros/cargo.rs | 42 ++++++++++++++----------------- src/macros/debug.rs | 11 ++++---- src/macros/implement.rs | 21 +++++++++------- src/macros/mod.rs | 37 +++++++++++++++++++++++---- src/macros/refutable.rs | 56 ++++++++++++++++++----------------------- src/macros/utils.rs | 15 +++++++++++ 11 files changed, 138 insertions(+), 99 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c73ef931..ea215eec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -731,9 +731,10 @@ dependencies = [ name = "conduit_macros" version = "0.4.5" dependencies = [ + "itertools 0.13.0", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.71", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 66ba7385..17e7e712 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -426,7 +426,7 @@ default-features = false version = "0.1" [workspace.dependencies.syn] -version = "1.0" +version = "2.0" features = ["full", "extra-traits"] [workspace.dependencies.quote] diff --git a/src/core/info/cargo.rs b/src/core/info/cargo.rs index 544bbb8f..012a08e0 100644 --- a/src/core/info/cargo.rs +++ b/src/core/info/cargo.rs @@ -16,19 +16,19 @@ use crate::Result; #[cargo_manifest] const WORKSPACE_MANIFEST: &'static str = (); -#[cargo_manifest("macros")] +#[cargo_manifest(crate = "macros")] const MACROS_MANIFEST: &'static str = (); -#[cargo_manifest("core")] +#[cargo_manifest(crate = "core")] const CORE_MANIFEST: &'static str = (); -#[cargo_manifest("database")] +#[cargo_manifest(crate = "database")] const DATABASE_MANIFEST: &'static str = (); -#[cargo_manifest("service")] +#[cargo_manifest(crate = "service")] const SERVICE_MANIFEST: &'static str = (); -#[cargo_manifest("admin")] +#[cargo_manifest(crate = "admin")] const ADMIN_MANIFEST: &'static str = (); -#[cargo_manifest("router")] +#[cargo_manifest(crate = "router")] const ROUTER_MANIFEST: &'static str = (); -#[cargo_manifest("main")] +#[cargo_manifest(crate = "main")] const MAIN_MANIFEST: &'static str = (); /// Processed list of features access all project crates. This is generated from diff --git a/src/macros/Cargo.toml b/src/macros/Cargo.toml index ca98f169..9e866578 100644 --- a/src/macros/Cargo.toml +++ b/src/macros/Cargo.toml @@ -18,6 +18,7 @@ proc-macro = true syn.workspace = true quote.workspace = true proc-macro2.workspace = true +itertools.workspace = true [lints] workspace = true diff --git a/src/macros/admin.rs b/src/macros/admin.rs index e1d294b9..4189d64f 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -1,17 +1,15 @@ +use itertools::Itertools; use proc_macro::{Span, TokenStream}; use proc_macro2::TokenStream as TokenStream2; -use quote::quote; -use syn::{parse_macro_input, AttributeArgs, Fields, Ident, ItemEnum, Variant}; +use quote::{quote, ToTokens}; +use syn::{Error, Fields, Ident, ItemEnum, Meta, Variant}; -use crate::utils::camel_to_snake_string; +use crate::{utils::camel_to_snake_string, Result}; -pub(super) fn command_dispatch(args: TokenStream, input_: TokenStream) -> TokenStream { - let input = input_.clone(); - let item = parse_macro_input!(input as ItemEnum); - let _args = parse_macro_input!(args as AttributeArgs); - let arm = item.variants.iter().map(dispatch_arm); - let name = item.ident; - let q = quote! { +pub(super) fn command_dispatch(item: ItemEnum, _args: &[Meta]) -> Result { + let name = &item.ident; + let arm: Vec = item.variants.iter().map(dispatch_arm).try_collect()?; + let switch = quote! { pub(super) async fn process(command: #name, body: Vec<&str>) -> Result { use #name::*; #[allow(non_snake_case)] @@ -21,14 +19,17 @@ pub(super) fn command_dispatch(args: TokenStream, input_: TokenStream) -> TokenS } }; - [input_, q.into()].into_iter().collect::() + Ok([item.into_token_stream(), switch] + .into_iter() + .collect::() + .into()) } -fn dispatch_arm(v: &Variant) -> TokenStream2 { +fn dispatch_arm(v: &Variant) -> Result { let name = &v.ident; let target = camel_to_snake_string(&format!("{name}")); let handler = Ident::new(&target, Span::call_site().into()); - match &v.fields { + let res = match &v.fields { Fields::Named(fields) => { let field = fields.named.iter().filter_map(|f| f.ident.as_ref()); let arg = field.clone(); @@ -37,7 +38,9 @@ fn dispatch_arm(v: &Variant) -> TokenStream2 { } }, Fields::Unnamed(fields) => { - let field = &fields.unnamed.first().expect("one field"); + let Some(ref field) = fields.unnamed.first() else { + return Err(Error::new(Span::call_site().into(), "One unnamed field required")); + }; quote! { #name ( #field ) => Box::pin(#handler::process(#field, body)).await?, } @@ -47,5 +50,7 @@ fn dispatch_arm(v: &Variant) -> TokenStream2 { #name => Box::pin(#handler(&body)).await?, } }, - } + }; + + Ok(res) } diff --git a/src/macros/cargo.rs b/src/macros/cargo.rs index 17132a6c..cd36658e 100644 --- a/src/macros/cargo.rs +++ b/src/macros/cargo.rs @@ -1,39 +1,34 @@ use std::{fs::read_to_string, path::PathBuf}; -use proc_macro::TokenStream; +use proc_macro::{Span, TokenStream}; use quote::quote; -use syn::{parse_macro_input, AttributeArgs, ItemConst, Lit, NestedMeta}; +use syn::{Error, ItemConst, Meta}; -pub(super) fn manifest(args: TokenStream, item: TokenStream) -> TokenStream { - let item = parse_macro_input!(item as ItemConst); - let args = parse_macro_input!(args as AttributeArgs); - let member = args.into_iter().find_map(|arg| { - let NestedMeta::Lit(arg) = arg else { - return None; - }; - let Lit::Str(arg) = arg else { - return None; - }; - Some(arg.value()) - }); +use crate::{utils, Result}; - let path = manifest_path(member.as_deref()); +pub(super) fn manifest(item: ItemConst, args: &[Meta]) -> Result { + let member = utils::get_named_string(args, "crate"); + let path = manifest_path(member.as_deref())?; let manifest = read_to_string(&path).unwrap_or_default(); - - let name = item.ident; let val = manifest.as_str(); + let name = item.ident; let ret = quote! { const #name: &'static str = #val; }; - ret.into() + Ok(ret.into()) } #[allow(clippy::option_env_unwrap)] -fn manifest_path(member: Option<&str>) -> PathBuf { - let mut path: PathBuf = option_env!("CARGO_MANIFEST_DIR") - .expect("missing CARGO_MANIFEST_DIR in environment") - .into(); +fn manifest_path(member: Option<&str>) -> Result { + let Some(path) = option_env!("CARGO_MANIFEST_DIR") else { + return Err(Error::new( + Span::call_site().into(), + "missing CARGO_MANIFEST_DIR in environment", + )); + }; + + let mut path: PathBuf = path.into(); // conduwuit/src/macros/ -> conduwuit/src/ path.pop(); @@ -47,5 +42,6 @@ fn manifest_path(member: Option<&str>) -> PathBuf { } path.push("Cargo.toml"); - path + + Ok(path) } diff --git a/src/macros/debug.rs b/src/macros/debug.rs index 5251fa17..e83fd44e 100644 --- a/src/macros/debug.rs +++ b/src/macros/debug.rs @@ -1,13 +1,12 @@ use std::cmp; use proc_macro::TokenStream; -use syn::{parse_macro_input, AttributeArgs, Item}; +use quote::ToTokens; +use syn::{Item, Meta}; -pub(super) fn recursion_depth(args: TokenStream, item_: TokenStream) -> TokenStream { - let item = item_.clone(); - let item = parse_macro_input!(item as Item); - let _args = parse_macro_input!(args as AttributeArgs); +use crate::Result; +pub(super) fn recursion_depth(item: Item, _args: &[Meta]) -> Result { let mut best: usize = 0; let mut count: usize = 0; // think you'd find a fancy recursive ast visitor? think again @@ -24,5 +23,5 @@ pub(super) fn recursion_depth(args: TokenStream, item_: TokenStream) -> TokenStr println!("DEPTH: {best}"); println!("LENGTH: {count}"); - item_ + Ok(item.into_token_stream().into()) } diff --git a/src/macros/implement.rs b/src/macros/implement.rs index 1a06e588..b5c8d787 100644 --- a/src/macros/implement.rs +++ b/src/macros/implement.rs @@ -1,23 +1,26 @@ use proc_macro::TokenStream; -use quote::{quote, ToTokens}; -use syn::{parse_macro_input, AttributeArgs, ItemFn, Meta, NestedMeta}; +use quote::quote; +use syn::{ItemFn, Meta, MetaList}; -pub(super) fn implement(args: TokenStream, input: TokenStream) -> TokenStream { - let args = parse_macro_input!(args as AttributeArgs); - let item = parse_macro_input!(input as ItemFn); +use crate::Result; - let NestedMeta::Meta(Meta::Path(receiver)) = args +pub(super) fn implement(item: ItemFn, args: &[Meta]) -> Result { + let Meta::List(MetaList { + path, + .. + }) = &args .first() .expect("missing path to trait or item to implement") else { panic!("invalid path to item for implement"); }; + let input = item; let out = quote! { - impl #receiver { - #item + impl #path { + #input } }; - out.into_token_stream().into() + Ok(out.into()) } diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 8d7e2e5b..1a5494bb 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -7,23 +7,50 @@ mod rustc; mod utils; use proc_macro::TokenStream; +use syn::{ + parse::{Parse, Parser}, + parse_macro_input, Error, Item, ItemConst, ItemEnum, ItemFn, Meta, +}; + +pub(crate) type Result = std::result::Result; #[proc_macro_attribute] pub fn admin_command_dispatch(args: TokenStream, input: TokenStream) -> TokenStream { - admin::command_dispatch(args, input) + attribute_macro::(args, input, admin::command_dispatch) } #[proc_macro_attribute] -pub fn cargo_manifest(args: TokenStream, input: TokenStream) -> TokenStream { cargo::manifest(args, input) } +pub fn cargo_manifest(args: TokenStream, input: TokenStream) -> TokenStream { + attribute_macro::(args, input, cargo::manifest) +} #[proc_macro_attribute] -pub fn recursion_depth(args: TokenStream, input: TokenStream) -> TokenStream { debug::recursion_depth(args, input) } +pub fn recursion_depth(args: TokenStream, input: TokenStream) -> TokenStream { + attribute_macro::(args, input, debug::recursion_depth) +} #[proc_macro] pub fn rustc_flags_capture(args: TokenStream) -> TokenStream { rustc::flags_capture(args) } #[proc_macro_attribute] -pub fn refutable(args: TokenStream, input: TokenStream) -> TokenStream { refutable::refutable(args, input) } +pub fn refutable(args: TokenStream, input: TokenStream) -> TokenStream { + attribute_macro::(args, input, refutable::refutable) +} #[proc_macro_attribute] -pub fn implement(args: TokenStream, input: TokenStream) -> TokenStream { implement::implement(args, input) } +pub fn implement(args: TokenStream, input: TokenStream) -> TokenStream { + attribute_macro::(args, input, implement::implement) +} + +fn attribute_macro(args: TokenStream, input: TokenStream, func: F) -> TokenStream +where + F: Fn(I, &[Meta]) -> Result, + I: Parse, +{ + let item = parse_macro_input!(input as I); + syn::punctuated::Punctuated::::parse_terminated + .parse(args) + .map(|args| args.iter().cloned().collect::>()) + .and_then(|ref args| func(item, args)) + .unwrap_or_else(|e| e.to_compile_error().into()) +} diff --git a/src/macros/refutable.rs b/src/macros/refutable.rs index 6a6884e0..facb4729 100644 --- a/src/macros/refutable.rs +++ b/src/macros/refutable.rs @@ -1,11 +1,10 @@ use proc_macro::{Span, TokenStream}; use quote::{quote, ToTokens}; -use syn::{parse_macro_input, AttributeArgs, FnArg::Typed, Ident, ItemFn, Pat, PatIdent, PatType, Stmt}; +use syn::{FnArg::Typed, Ident, ItemFn, Meta, Pat, PatIdent, PatType, Stmt}; -pub(super) fn refutable(args: TokenStream, input: TokenStream) -> TokenStream { - let _args = parse_macro_input!(args as AttributeArgs); - let mut item = parse_macro_input!(input as ItemFn); +use crate::Result; +pub(super) fn refutable(mut item: ItemFn, _args: &[Meta]) -> Result { let inputs = item.sig.inputs.clone(); let stmt = &mut item.block.stmts; let sig = &mut item.sig; @@ -25,37 +24,30 @@ pub(super) fn refutable(args: TokenStream, input: TokenStream) -> TokenStream { let variant = &pat.path; let fields = &pat.fields; - // new versions of syn can replace this kronecker kludge with get_mut() - for (j, input) in sig.inputs.iter_mut().enumerate() { - if i != j { - continue; - } + let Some(Typed(PatType { + ref mut pat, + .. + })) = sig.inputs.get_mut(i) + else { + continue; + }; - let Typed(PatType { - ref mut pat, - .. - }) = input - else { - continue; - }; + let name = format!("_args_{i}"); + *pat = Box::new(Pat::Ident(PatIdent { + ident: Ident::new(&name, Span::call_site().into()), + attrs: Vec::new(), + by_ref: None, + mutability: None, + subpat: None, + })); - let name = format!("_args_{i}"); - *pat = Box::new(Pat::Ident(PatIdent { - ident: Ident::new(&name, Span::call_site().into()), - attrs: Vec::new(), - by_ref: None, - mutability: None, - subpat: None, - })); + let field = fields.iter(); + let refute = quote! { + let #variant { #( #field ),*, .. } = #name else { panic!("incorrect variant passed to function argument {i}"); }; + }; - let field = fields.iter(); - let refute = quote! { - let #variant { #( #field ),*, .. } = #name else { panic!("incorrect variant passed to function argument {i}"); }; - }; - - stmt.insert(0, syn::parse2::(refute).expect("syntax error")); - } + stmt.insert(0, syn::parse2::(refute)?); } - item.into_token_stream().into() + Ok(item.into_token_stream().into()) } diff --git a/src/macros/utils.rs b/src/macros/utils.rs index f512c56c..7ae55d39 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -1,3 +1,18 @@ +use syn::{Expr, Lit, Meta}; + +pub(crate) fn get_named_string(args: &[Meta], name: &str) -> Option { + args.iter().find_map(|arg| { + let value = arg.require_name_value().ok()?; + let Expr::Lit(ref lit) = value.value else { + return None; + }; + let Lit::Str(ref str) = lit.lit else { + return None; + }; + value.path.is_ident(name).then_some(str.value()) + }) +} + #[must_use] pub(crate) fn camel_to_snake_string(s: &str) -> String { let mut output = String::with_capacity( From 7a3cc3941e825a3d063ab365a51a9b8b1a4e2cbd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 26 Jul 2024 21:31:31 +0000 Subject: [PATCH 0592/2091] parse generics for implement macro Signed-off-by: Jason Volk --- src/macros/implement.rs | 34 ++++++++++++++++++++-------------- src/macros/utils.rs | 10 +++++++++- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/macros/implement.rs b/src/macros/implement.rs index b5c8d787..b798bae5 100644 --- a/src/macros/implement.rs +++ b/src/macros/implement.rs @@ -1,26 +1,32 @@ -use proc_macro::TokenStream; +use proc_macro::{Span, TokenStream}; use quote::quote; -use syn::{ItemFn, Meta, MetaList}; +use syn::{Error, ItemFn, Meta, Path}; +use utils::get_named_generics; -use crate::Result; +use crate::{utils, Result}; pub(super) fn implement(item: ItemFn, args: &[Meta]) -> Result { - let Meta::List(MetaList { - path, - .. - }) = &args - .first() - .expect("missing path to trait or item to implement") - else { - panic!("invalid path to item for implement"); - }; - + let generics = get_named_generics(args, "generics")?; + let receiver = get_receiver(args)?; + let params = get_named_generics(args, "params")?; let input = item; let out = quote! { - impl #path { + impl #generics #receiver #params { #input } }; Ok(out.into()) } + +fn get_receiver(args: &[Meta]) -> Result { + let receiver = &args + .first() + .ok_or_else(|| Error::new(Span::call_site().into(), "Missing required argument to receiver"))?; + + let Meta::Path(receiver) = receiver else { + return Err(Error::new(Span::call_site().into(), "First argument is not path to receiver")); + }; + + Ok(receiver.clone()) +} diff --git a/src/macros/utils.rs b/src/macros/utils.rs index 7ae55d39..58074e3a 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -1,4 +1,12 @@ -use syn::{Expr, Lit, Meta}; +use syn::{parse_str, Expr, Generics, Lit, Meta}; + +use crate::Result; + +pub(crate) fn get_named_generics(args: &[Meta], name: &str) -> Result { + const DEFAULT: &str = "<>"; + + parse_str::(&get_named_string(args, name).unwrap_or_else(|| DEFAULT.to_owned())) +} pub(crate) fn get_named_string(args: &[Meta], name: &str) -> Option { args.iter().find_map(|arg| { From 7e50db419362c6838d5eaeefccee24c7686b57e3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 27 Jul 2024 00:11:41 +0000 Subject: [PATCH 0593/2091] de-global services from admin Signed-off-by: Jason Volk --- src/admin/admin.rs | 34 +-- src/admin/appservice/commands.rs | 29 +- src/admin/appservice/mod.rs | 23 +- src/admin/check/commands.rs | 6 +- src/admin/check/mod.rs | 6 +- src/admin/command.rs | 6 + src/admin/debug/commands.rs | 258 +++++++++------- src/admin/debug/mod.rs | 6 +- src/admin/debug/tester.rs | 18 +- src/admin/federation/commands.rs | 39 ++- src/admin/federation/mod.rs | 23 +- src/admin/handler.rs | 23 +- src/admin/media/commands.rs | 37 +-- src/admin/media/mod.rs | 19 +- src/admin/mod.rs | 7 +- src/admin/query/account_data.rs | 46 ++- src/admin/query/appservice.rs | 30 +- src/admin/query/globals.rs | 49 ++- src/admin/query/mod.rs | 281 +----------------- src/admin/query/presence.rs | 36 ++- src/admin/query/resolver.rs | 44 ++- src/admin/query/room_alias.rs | 41 ++- src/admin/query/room_state_cache.rs | 170 +++++++---- src/admin/query/sending.rs | 91 +++++- src/admin/query/users.rs | 19 +- .../room/{room_alias_commands.rs => alias.rs} | 67 ++++- .../room/{room_commands.rs => commands.rs} | 19 +- ...oom_directory_commands.rs => directory.rs} | 38 ++- .../room/{room_info_commands.rs => info.rs} | 51 ++-- src/admin/room/mod.rs | 166 +---------- ...m_moderation_commands.rs => moderation.rs} | 195 +++++++----- src/admin/server/commands.rs | 83 +++--- src/admin/server/mod.rs | 35 +-- src/admin/user/commands.rs | 180 ++++++----- src/admin/user/mod.rs | 56 +--- src/macros/admin.rs | 22 +- src/macros/mod.rs | 5 + 37 files changed, 1131 insertions(+), 1127 deletions(-) create mode 100644 src/admin/command.rs rename src/admin/room/{room_alias_commands.rs => alias.rs} (73%) rename src/admin/room/{room_commands.rs => commands.rs} (82%) rename src/admin/room/{room_directory_commands.rs => directory.rs} (68%) rename src/admin/room/{room_info_commands.rs => info.rs} (54%) rename src/admin/room/{room_moderation_commands.rs => moderation.rs} (72%) diff --git a/src/admin/admin.rs b/src/admin/admin.rs index f5fe5dc2..fa497205 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -3,14 +3,14 @@ use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; use crate::{ - appservice, appservice::AppserviceCommand, check, check::CheckCommand, debug, debug::DebugCommand, federation, - federation::FederationCommand, media, media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, - server, server::ServerCommand, user, user::UserCommand, + appservice, appservice::AppserviceCommand, check, check::CheckCommand, command::Command, debug, + debug::DebugCommand, federation, federation::FederationCommand, media, media::MediaCommand, query, + query::QueryCommand, room, room::RoomCommand, server, server::ServerCommand, user, user::UserCommand, }; #[derive(Debug, Parser)] #[command(name = "admin", version = env!("CARGO_PKG_VERSION"))] -pub(crate) enum AdminCommand { +pub(super) enum AdminCommand { #[command(subcommand)] /// - Commands for managing appservices Appservices(AppserviceCommand), @@ -49,18 +49,18 @@ pub(crate) enum AdminCommand { } #[tracing::instrument(skip_all, name = "command")] -pub(crate) async fn process(command: AdminCommand, body: Vec<&str>) -> Result { - let reply_message_content = match command { - AdminCommand::Appservices(command) => appservice::process(command, body).await?, - AdminCommand::Media(command) => media::process(command, body).await?, - AdminCommand::Users(command) => user::process(command, body).await?, - AdminCommand::Rooms(command) => room::process(command, body).await?, - AdminCommand::Federation(command) => federation::process(command, body).await?, - AdminCommand::Server(command) => server::process(command, body).await?, - AdminCommand::Debug(command) => debug::process(command, body).await?, - AdminCommand::Query(command) => query::process(command, body).await?, - AdminCommand::Check(command) => check::process(command, body).await?, - }; +pub(super) async fn process(command: AdminCommand, context: &Command<'_>) -> Result { + use AdminCommand::*; - Ok(reply_message_content) + Ok(match command { + Appservices(command) => appservice::process(command, context).await?, + Media(command) => media::process(command, context).await?, + Users(command) => user::process(command, context).await?, + Rooms(command) => room::process(command, context).await?, + Federation(command) => federation::process(command, context).await?, + Server(command) => server::process(command, context).await?, + Debug(command) => debug::process(command, context).await?, + Query(command) => query::process(command, context).await?, + Check(command) => check::process(command, context).await?, + }) } diff --git a/src/admin/appservice/commands.rs b/src/admin/appservice/commands.rs index 5cce8351..7d6378f3 100644 --- a/src/admin/appservice/commands.rs +++ b/src/admin/appservice/commands.rs @@ -1,18 +1,20 @@ use ruma::{api::appservice::Registration, events::room::message::RoomMessageEventContent}; -use crate::{services, Result}; +use crate::{admin_command, Result}; -pub(super) async fn register(body: Vec<&str>) -> Result { - if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { +#[admin_command] +pub(super) async fn register(&self) -> Result { + if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", )); } - let appservice_config = body[1..body.len().checked_sub(1).unwrap()].join("\n"); + let appservice_config = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); let parsed_config = serde_yaml::from_str::(&appservice_config); match parsed_config { - Ok(yaml) => match services().appservice.register_appservice(yaml).await { + Ok(yaml) => match self.services.appservice.register_appservice(yaml).await { Ok(id) => Ok(RoomMessageEventContent::text_plain(format!( "Appservice registered with ID: {id}." ))), @@ -26,8 +28,10 @@ pub(super) async fn register(body: Vec<&str>) -> Result } } -pub(super) async fn unregister(_body: Vec<&str>, appservice_identifier: String) -> Result { - match services() +#[admin_command] +pub(super) async fn unregister(&self, appservice_identifier: String) -> Result { + match self + .services .appservice .unregister_appservice(&appservice_identifier) .await @@ -39,8 +43,10 @@ pub(super) async fn unregister(_body: Vec<&str>, appservice_identifier: String) } } -pub(super) async fn show(_body: Vec<&str>, appservice_identifier: String) -> Result { - match services() +#[admin_command] +pub(super) async fn show_appservice_config(&self, appservice_identifier: String) -> Result { + match self + .services .appservice .get_registration(&appservice_identifier) .await @@ -54,8 +60,9 @@ pub(super) async fn show(_body: Vec<&str>, appservice_identifier: String) -> Res } } -pub(super) async fn list(_body: Vec<&str>) -> Result { - let appservices = services().appservice.iter_ids().await; +#[admin_command] +pub(super) async fn list_registered(&self) -> Result { + let appservices = self.services.appservice.iter_ids().await; let output = format!("Appservices ({}): {}", appservices.len(), appservices.join(", ")); Ok(RoomMessageEventContent::text_plain(output)) } diff --git a/src/admin/appservice/mod.rs b/src/admin/appservice/mod.rs index 81e04087..ca5f46bb 100644 --- a/src/admin/appservice/mod.rs +++ b/src/admin/appservice/mod.rs @@ -2,11 +2,11 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::events::room::message::RoomMessageEventContent; -use self::commands::*; +use crate::admin_command_dispatch; #[derive(Debug, Subcommand)] +#[admin_command_dispatch] pub(super) enum AppserviceCommand { /// - Register an appservice using its registration YAML /// @@ -28,24 +28,13 @@ pub(super) enum AppserviceCommand { /// - Show an appservice's config using its ID /// /// You can find the ID using the `list-appservices` command. - Show { + #[clap(alias("show"))] + ShowAppserviceConfig { /// The appservice to show appservice_identifier: String, }, /// - List all the currently registered appservices - List, -} - -pub(super) async fn process(command: AppserviceCommand, body: Vec<&str>) -> Result { - Ok(match command { - AppserviceCommand::Register => register(body).await?, - AppserviceCommand::Unregister { - appservice_identifier, - } => unregister(body, appservice_identifier).await?, - AppserviceCommand::Show { - appservice_identifier, - } => show(body, appservice_identifier).await?, - AppserviceCommand::List => list(body).await?, - }) + #[clap(alias("list"))] + ListRegistered, } diff --git a/src/admin/check/commands.rs b/src/admin/check/commands.rs index 1fbea8f6..a757d504 100644 --- a/src/admin/check/commands.rs +++ b/src/admin/check/commands.rs @@ -1,12 +1,14 @@ use conduit::Result; +use conduit_macros::implement; use ruma::events::room::message::RoomMessageEventContent; -use crate::services; +use crate::{services, Command}; /// Uses the iterator in `src/database/key_value/users.rs` to iterator over /// every user in our database (remote and local). Reports total count, any /// errors if there were any, etc -pub(super) async fn check_all_users(_body: Vec<&str>) -> Result { +#[implement(Command, params = "<'_>")] +pub(super) async fn check_all_users(&self) -> Result { let timer = tokio::time::Instant::now(); let results = services().users.db.iter(); let query_time = timer.elapsed(); diff --git a/src/admin/check/mod.rs b/src/admin/check/mod.rs index f1cfa2b9..e543e5b5 100644 --- a/src/admin/check/mod.rs +++ b/src/admin/check/mod.rs @@ -4,15 +4,15 @@ use clap::Subcommand; use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; -use self::commands::*; +use crate::Command; #[derive(Debug, Subcommand)] pub(super) enum CheckCommand { AllUsers, } -pub(super) async fn process(command: CheckCommand, body: Vec<&str>) -> Result { +pub(super) async fn process(command: CheckCommand, context: &Command<'_>) -> Result { Ok(match command { - CheckCommand::AllUsers => check_all_users(body).await?, + CheckCommand::AllUsers => context.check_all_users().await?, }) } diff --git a/src/admin/command.rs b/src/admin/command.rs new file mode 100644 index 00000000..fbfdd2ba --- /dev/null +++ b/src/admin/command.rs @@ -0,0 +1,6 @@ +use service::Services; + +pub(crate) struct Command<'a> { + pub(crate) services: &'a Services, + pub(crate) body: &'a [&'a str], +} diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 307495d7..cf4ab31d 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -16,19 +16,22 @@ use ruma::{ events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, }; -use service::services; use tokio::sync::RwLock; use tracing_subscriber::EnvFilter; -pub(super) async fn echo(_body: &[&str], message: Vec) -> Result { +use crate::admin_command; + +#[admin_command] +pub(super) async fn echo(&self, message: Vec) -> Result { let message = message.join(" "); Ok(RoomMessageEventContent::notice_plain(message)) } -pub(super) async fn get_auth_chain(_body: &[&str], event_id: Box) -> Result { +#[admin_command] +pub(super) async fn get_auth_chain(&self, event_id: Box) -> Result { let event_id = Arc::::from(event_id); - if let Some(event) = services().rooms.timeline.get_pdu_json(&event_id)? { + if let Some(event) = self.services.rooms.timeline.get_pdu_json(&event_id)? { let room_id_str = event .get("room_id") .and_then(|val| val.as_str()) @@ -36,13 +39,16 @@ pub(super) async fn get_auth_chain(_body: &[&str], event_id: Box) -> Re let room_id = <&RoomId>::try_from(room_id_str) .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; + let start = Instant::now(); - let count = services() + let count = self + .services .rooms .auth_chain .event_ids_iter(room_id, vec![event_id]) .await? .count(); + let elapsed = start.elapsed(); Ok(RoomMessageEventContent::text_plain(format!( "Loaded auth chain with length {count} in {elapsed:?}" @@ -52,14 +58,16 @@ pub(super) async fn get_auth_chain(_body: &[&str], event_id: Box) -> Re } } -pub(super) async fn parse_pdu(body: &[&str]) -> Result { - if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { +#[admin_command] +pub(super) async fn parse_pdu(&self) -> Result { + if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", )); } - let string = body[1..body.len().saturating_sub(1)].join("\n"); + let string = self.body[1..self.body.len().saturating_sub(1)].join("\n"); match serde_json::from_str(&string) { Ok(value) => match ruma::signatures::reference_hash(&value, &RoomVersionId::V6) { Ok(hash) => { @@ -80,15 +88,17 @@ pub(super) async fn parse_pdu(body: &[&str]) -> Result } } -pub(super) async fn get_pdu(_body: &[&str], event_id: Box) -> Result { +#[admin_command] +pub(super) async fn get_pdu(&self, event_id: Box) -> Result { let mut outlier = false; - let mut pdu_json = services() + let mut pdu_json = self + .services .rooms .timeline .get_non_outlier_pdu_json(&event_id)?; if pdu_json.is_none() { outlier = true; - pdu_json = services().rooms.timeline.get_pdu_json(&event_id)?; + pdu_json = self.services.rooms.timeline.get_pdu_json(&event_id)?; } match pdu_json { Some(json) => { @@ -107,39 +117,42 @@ pub(super) async fn get_pdu(_body: &[&str], event_id: Box) -> Result, force: bool, + &self, server: Box, force: bool, ) -> Result { - if !services().globals.config.allow_federation { + if !self.services.globals.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( "Federation is disabled on this homeserver.", )); } - if server == services().globals.server_name() { + if server == self.services.globals.server_name() { return Ok(RoomMessageEventContent::text_plain( "Not allowed to send federation requests to ourselves. Please use `get-pdu` for fetching local PDUs from \ the database.", )); } - if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { + if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", )); } - let list = body + let list = self + .body .iter() .collect::>() - .drain(1..body.len().saturating_sub(1)) + .drain(1..self.body.len().saturating_sub(1)) .filter_map(|pdu| EventId::parse(pdu).ok()) .collect::>(); for pdu in list { if force { - if let Err(e) = get_remote_pdu(&[], Box::from(pdu), server.clone()).await { - services() + if let Err(e) = self.get_remote_pdu(Box::from(pdu), server.clone()).await { + self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( "Failed to get remote PDU, ignoring error: {e}" @@ -148,29 +161,31 @@ pub(super) async fn get_remote_pdu_list( warn!(%e, "Failed to get remote PDU, ignoring error"); } } else { - get_remote_pdu(&[], Box::from(pdu), server.clone()).await?; + self.get_remote_pdu(Box::from(pdu), server.clone()).await?; } } Ok(RoomMessageEventContent::text_plain("Fetched list of remote PDUs.")) } +#[admin_command] pub(super) async fn get_remote_pdu( - _body: &[&str], event_id: Box, server: Box, + &self, event_id: Box, server: Box, ) -> Result { - if !services().globals.config.allow_federation { + if !self.services.globals.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( "Federation is disabled on this homeserver.", )); } - if server == services().globals.server_name() { + if server == self.services.globals.server_name() { return Ok(RoomMessageEventContent::text_plain( "Not allowed to send federation requests to ourselves. Please use `get-pdu` for fetching local PDUs.", )); } - match services() + match self + .services .sending .send_federation_request( &server, @@ -191,7 +206,8 @@ pub(super) async fn get_remote_pdu( debug!("Attempting to parse PDU: {:?}", &response.pdu); let parsed_pdu = { - let parsed_result = services() + let parsed_result = self + .services .rooms .event_handler .parse_incoming_pdu(&response.pdu); @@ -212,7 +228,7 @@ pub(super) async fn get_remote_pdu( let pub_key_map = RwLock::new(BTreeMap::new()); debug!("Attempting to fetch homeserver signing keys for {server}"); - services() + self.services .rooms .event_handler .fetch_required_signing_keys(parsed_pdu.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) @@ -222,7 +238,7 @@ pub(super) async fn get_remote_pdu( }); info!("Attempting to handle event ID {event_id} as backfilled PDU"); - services() + self.services .rooms .timeline .backfill_pdu(&server, response.pdu, &pub_key_map) @@ -241,9 +257,11 @@ pub(super) async fn get_remote_pdu( } } -pub(super) async fn get_room_state(_body: &[&str], room: OwnedRoomOrAliasId) -> Result { - let room_id = services().rooms.alias.resolve(&room).await?; - let room_state = services() +#[admin_command] +pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result { + let room_id = self.services.rooms.alias.resolve(&room).await?; + let room_state = self + .services .rooms .state_accessor .room_state_full(&room_id) @@ -268,8 +286,9 @@ pub(super) async fn get_room_state(_body: &[&str], room: OwnedRoomOrAliasId) -> Ok(RoomMessageEventContent::notice_markdown(format!("```json\n{json}\n```"))) } -pub(super) async fn ping(_body: &[&str], server: Box) -> Result { - if server == services().globals.server_name() { +#[admin_command] +pub(super) async fn ping(&self, server: Box) -> Result { + if server == self.services.globals.server_name() { return Ok(RoomMessageEventContent::text_plain( "Not allowed to send federation requests to ourselves.", )); @@ -277,7 +296,8 @@ pub(super) async fn ping(_body: &[&str], server: Box) -> Result) -> Result Result { +#[admin_command] +pub(super) async fn force_device_list_updates(&self) -> Result { // Force E2EE device list updates for all users - for user_id in services().users.iter().filter_map(Result::ok) { - services().users.mark_device_key_update(&user_id)?; + for user_id in self.services.users.iter().filter_map(Result::ok) { + self.services.users.mark_device_key_update(&user_id)?; } Ok(RoomMessageEventContent::text_plain( "Marked all devices for all users as having new keys to update", )) } -pub(super) async fn change_log_level( - _body: &[&str], filter: Option, reset: bool, -) -> Result { +#[admin_command] +pub(super) async fn change_log_level(&self, filter: Option, reset: bool) -> Result { let handles = &["console"]; if reset { - let old_filter_layer = match EnvFilter::try_new(&services().globals.config.log) { + let old_filter_layer = match EnvFilter::try_new(&self.services.globals.config.log) { Ok(s) => s, Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( @@ -331,7 +351,8 @@ pub(super) async fn change_log_level( }, }; - match services() + match self + .services .server .log .reload @@ -340,7 +361,7 @@ pub(super) async fn change_log_level( Ok(()) => { return Ok(RoomMessageEventContent::text_plain(format!( "Successfully changed log level back to config value {}", - services().globals.config.log + self.services.globals.config.log ))); }, Err(e) => { @@ -361,7 +382,8 @@ pub(super) async fn change_log_level( }, }; - match services() + match self + .services .server .log .reload @@ -381,19 +403,21 @@ pub(super) async fn change_log_level( Ok(RoomMessageEventContent::text_plain("No log level was specified.")) } -pub(super) async fn sign_json(body: &[&str]) -> Result { - if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { +#[admin_command] +pub(super) async fn sign_json(&self) -> Result { + if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", )); } - let string = body[1..body.len().checked_sub(1).unwrap()].join("\n"); + let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); match serde_json::from_str(&string) { Ok(mut value) => { ruma::signatures::sign_json( - services().globals.server_name().as_str(), - services().globals.keypair(), + self.services.globals.server_name().as_str(), + self.services.globals.keypair(), &mut value, ) .expect("our request json is what ruma expects"); @@ -404,19 +428,21 @@ pub(super) async fn sign_json(body: &[&str]) -> Result } } -pub(super) async fn verify_json(body: &[&str]) -> Result { - if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { +#[admin_command] +pub(super) async fn verify_json(&self) -> Result { + if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", )); } - let string = body[1..body.len().checked_sub(1).unwrap()].join("\n"); + let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); match serde_json::from_str(&string) { Ok(value) => { let pub_key_map = RwLock::new(BTreeMap::new()); - services() + self.services .rooms .event_handler .fetch_required_signing_keys([&value], &pub_key_map) @@ -434,19 +460,22 @@ pub(super) async fn verify_json(body: &[&str]) -> Result) -> Result { - if !services() +#[admin_command] +#[tracing::instrument(skip(self))] +pub(super) async fn first_pdu_in_room(&self, room_id: Box) -> Result { + if !self + .services .rooms .state_cache - .server_in_room(&services().globals.config.server_name, &room_id)? + .server_in_room(&self.services.globals.config.server_name, &room_id)? { return Ok(RoomMessageEventContent::text_plain( "We are not participating in the room / we don't know about the room ID.", )); } - let first_pdu = services() + let first_pdu = self + .services .rooms .timeline .first_pdu_in_room(&room_id)? @@ -455,19 +484,22 @@ pub(super) async fn first_pdu_in_room(_body: &[&str], room_id: Box) -> R Ok(RoomMessageEventContent::text_plain(format!("{first_pdu:?}"))) } -#[tracing::instrument(skip(_body))] -pub(super) async fn latest_pdu_in_room(_body: &[&str], room_id: Box) -> Result { - if !services() +#[admin_command] +#[tracing::instrument(skip(self))] +pub(super) async fn latest_pdu_in_room(&self, room_id: Box) -> Result { + if !self + .services .rooms .state_cache - .server_in_room(&services().globals.config.server_name, &room_id)? + .server_in_room(&self.services.globals.config.server_name, &room_id)? { return Ok(RoomMessageEventContent::text_plain( "We are not participating in the room / we don't know about the room ID.", )); } - let latest_pdu = services() + let latest_pdu = self + .services .rooms .timeline .latest_pdu_in_room(&room_id)? @@ -476,32 +508,36 @@ pub(super) async fn latest_pdu_in_room(_body: &[&str], room_id: Box) -> Ok(RoomMessageEventContent::text_plain(format!("{latest_pdu:?}"))) } -#[tracing::instrument(skip(_body))] +#[admin_command] +#[tracing::instrument(skip(self))] pub(super) async fn force_set_room_state_from_server( - _body: &[&str], room_id: Box, server_name: Box, + &self, room_id: Box, server_name: Box, ) -> Result { - if !services() + if !self + .services .rooms .state_cache - .server_in_room(&services().globals.config.server_name, &room_id)? + .server_in_room(&self.services.globals.config.server_name, &room_id)? { return Ok(RoomMessageEventContent::text_plain( "We are not participating in the room / we don't know about the room ID.", )); } - let first_pdu = services() + let first_pdu = self + .services .rooms .timeline .latest_pdu_in_room(&room_id)? .ok_or_else(|| Error::bad_database("Failed to find the latest PDU in database"))?; - let room_version = services().rooms.state.get_room_version(&room_id)?; + let room_version = self.services.rooms.state.get_room_version(&room_id)?; let mut state: HashMap> = HashMap::new(); let pub_key_map = RwLock::new(BTreeMap::new()); - let remote_state_response = services() + let remote_state_response = self + .services .sending .send_federation_request( &server_name, @@ -515,7 +551,7 @@ pub(super) async fn force_set_room_state_from_server( let mut events = Vec::with_capacity(remote_state_response.pdus.len()); for pdu in remote_state_response.pdus.clone() { - events.push(match services().rooms.event_handler.parse_incoming_pdu(&pdu) { + events.push(match self.services.rooms.event_handler.parse_incoming_pdu(&pdu) { Ok(t) => t, Err(e) => { warn!("Could not parse PDU, ignoring: {e}"); @@ -525,7 +561,7 @@ pub(super) async fn force_set_room_state_from_server( } info!("Fetching required signing keys for all the state events we got"); - services() + self.services .rooms .event_handler .fetch_required_signing_keys(events.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) @@ -535,7 +571,7 @@ pub(super) async fn force_set_room_state_from_server( for result in remote_state_response .pdus .iter() - .map(|pdu| validate_and_add_event_id(services(), pdu, &room_version, &pub_key_map)) + .map(|pdu| validate_and_add_event_id(self.services, pdu, &room_version, &pub_key_map)) { let Ok((event_id, value)) = result.await else { continue; @@ -546,12 +582,13 @@ pub(super) async fn force_set_room_state_from_server( Error::BadServerResponse("Invalid PDU in send_join response.") })?; - services() + self.services .rooms .outlier .add_pdu_outlier(&event_id, &value)?; if let Some(state_key) = &pdu.state_key { - let shortstatekey = services() + let shortstatekey = self + .services .rooms .short .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key)?; @@ -563,32 +600,34 @@ pub(super) async fn force_set_room_state_from_server( for result in remote_state_response .auth_chain .iter() - .map(|pdu| validate_and_add_event_id(services(), pdu, &room_version, &pub_key_map)) + .map(|pdu| validate_and_add_event_id(self.services, pdu, &room_version, &pub_key_map)) { let Ok((event_id, value)) = result.await else { continue; }; - services() + self.services .rooms .outlier .add_pdu_outlier(&event_id, &value)?; } - let new_room_state = services() + let new_room_state = self + .services .rooms .event_handler .resolve_state(room_id.clone().as_ref(), &room_version, state) .await?; info!("Forcing new room state"); - let (short_state_hash, new, removed) = services() + let (short_state_hash, new, removed) = self + .services .rooms .state_compressor .save_state(room_id.clone().as_ref(), new_room_state)?; - let state_lock = services().rooms.state.mutex.lock(&room_id).await; - services() + let state_lock = self.services.rooms.state.mutex.lock(&room_id).await; + self.services .rooms .state .force_state(room_id.clone().as_ref(), short_state_hash, new, removed, &state_lock) @@ -598,7 +637,10 @@ pub(super) async fn force_set_room_state_from_server( "Updating joined counts for room just in case (e.g. we may have found a difference in the room's \ m.room.member state" ); - services().rooms.state_cache.update_joined_count(&room_id)?; + self.services + .rooms + .state_cache + .update_joined_count(&room_id)?; drop(state_lock); @@ -607,28 +649,30 @@ pub(super) async fn force_set_room_state_from_server( )) } +#[admin_command] pub(super) async fn get_signing_keys( - _body: &[&str], server_name: Option>, _cached: bool, + &self, server_name: Option>, _cached: bool, ) -> Result { - let server_name = server_name.unwrap_or_else(|| services().server.config.server_name.clone().into()); - let signing_keys = services().globals.signing_keys_for(&server_name)?; + let server_name = server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); + let signing_keys = self.services.globals.signing_keys_for(&server_name)?; Ok(RoomMessageEventContent::notice_markdown(format!( "```rs\n{signing_keys:#?}\n```" ))) } +#[admin_command] #[allow(dead_code)] pub(super) async fn get_verify_keys( - _body: &[&str], server_name: Option>, cached: bool, + &self, server_name: Option>, cached: bool, ) -> Result { - let server_name = server_name.unwrap_or_else(|| services().server.config.server_name.clone().into()); + let server_name = server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); let mut out = String::new(); if cached { writeln!(out, "| Key ID | VerifyKey |")?; writeln!(out, "| --- | --- |")?; - for (key_id, verify_key) in services().globals.verify_keys_for(&server_name)? { + for (key_id, verify_key) in self.services.globals.verify_keys_for(&server_name)? { writeln!(out, "| {key_id} | {verify_key:?} |")?; } @@ -636,7 +680,8 @@ pub(super) async fn get_verify_keys( } let signature_ids: Vec = Vec::new(); - let keys = services() + let keys = self + .services .rooms .event_handler .fetch_signing_keys_for_server(&server_name, signature_ids) @@ -651,16 +696,17 @@ pub(super) async fn get_verify_keys( Ok(RoomMessageEventContent::notice_markdown(out)) } +#[admin_command] pub(super) async fn resolve_true_destination( - _body: &[&str], server_name: Box, no_cache: bool, + &self, server_name: Box, no_cache: bool, ) -> Result { - if !services().globals.config.allow_federation { + if !self.services.globals.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( "Federation is disabled on this homeserver.", )); } - if server_name == services().globals.config.server_name { + if server_name == self.services.globals.config.server_name { return Ok(RoomMessageEventContent::text_plain( "Not allowed to send federation requests to ourselves. Please use `get-pdu` for fetching local PDUs.", )); @@ -672,12 +718,13 @@ pub(super) async fn resolve_true_destination( && matches!(data.span_name(), "actual" | "well-known" | "srv") }; - let state = &services().server.log.capture; + let state = &self.services.server.log.capture; let logs = Arc::new(Mutex::new(String::new())); let capture = Capture::new(state, Some(filter), capture::fmt_markdown(logs.clone())); let capture_scope = capture.start(); - let actual = services() + let actual = self + .services .resolver .resolve_actual_dest(&server_name, !no_cache) .await?; @@ -692,7 +739,8 @@ pub(super) async fn resolve_true_destination( Ok(RoomMessageEventContent::text_markdown(msg)) } -pub(super) async fn memory_stats(_body: &[&str]) -> Result { +#[admin_command] +pub(super) async fn memory_stats(&self) -> Result { let html_body = conduit::alloc::memory_stats(); if html_body.is_none() { @@ -708,8 +756,9 @@ pub(super) async fn memory_stats(_body: &[&str]) -> Result Result { - let out = services().server.metrics.runtime_metrics().map_or_else( +#[admin_command] +pub(super) async fn runtime_metrics(&self) -> Result { + let out = self.services.server.metrics.runtime_metrics().map_or_else( || "Runtime metrics are not available.".to_owned(), |metrics| format!("```rs\n{metrics:#?}\n```"), ); @@ -718,15 +767,17 @@ pub(super) async fn runtime_metrics(_body: &[&str]) -> Result Result { +#[admin_command] +pub(super) async fn runtime_metrics(&self) -> Result { Ok(RoomMessageEventContent::text_markdown( "Runtime metrics require building with `tokio_unstable`.", )) } #[cfg(tokio_unstable)] -pub(super) async fn runtime_interval(_body: &[&str]) -> Result { - let out = services().server.metrics.runtime_interval().map_or_else( +#[admin_command] +pub(super) async fn runtime_interval(&self) -> Result { + let out = self.services.server.metrics.runtime_interval().map_or_else( || "Runtime metrics are not available.".to_owned(), |metrics| format!("```rs\n{metrics:#?}\n```"), ); @@ -735,18 +786,21 @@ pub(super) async fn runtime_interval(_body: &[&str]) -> Result Result { +#[admin_command] +pub(super) async fn runtime_interval(&self) -> Result { Ok(RoomMessageEventContent::text_markdown( "Runtime metrics require building with `tokio_unstable`.", )) } -pub(super) async fn time(_body: &[&str]) -> Result { +#[admin_command] +pub(super) async fn time(&self) -> Result { let now = SystemTime::now(); Ok(RoomMessageEventContent::text_markdown(utils::time::format(now, "%+"))) } -pub(super) async fn list_dependencies(_body: &[&str], names: bool) -> Result { +#[admin_command] +pub(super) async fn list_dependencies(&self, names: bool) -> Result { if names { let out = info::cargo::dependencies_names().join(" "); return Ok(RoomMessageEventContent::notice_markdown(out)); diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index babd6047..fbe6fd26 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -3,10 +3,10 @@ pub(crate) mod tester; use clap::Subcommand; use conduit::Result; -use conduit_macros::admin_command_dispatch; -use ruma::{events::room::message::RoomMessageEventContent, EventId, OwnedRoomOrAliasId, RoomId, ServerName}; +use ruma::{EventId, OwnedRoomOrAliasId, RoomId, ServerName}; -use self::{commands::*, tester::TesterCommand}; +use self::tester::TesterCommand; +use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] diff --git a/src/admin/debug/tester.rs b/src/admin/debug/tester.rs index 2765a344..af4ea2dc 100644 --- a/src/admin/debug/tester.rs +++ b/src/admin/debug/tester.rs @@ -1,33 +1,29 @@ use ruma::events::room::message::RoomMessageEventContent; -use crate::Result; +use crate::{admin_command, admin_command_dispatch, Result}; +#[admin_command_dispatch] #[derive(Debug, clap::Subcommand)] pub(crate) enum TesterCommand { Tester, Timer, } -pub(super) async fn process(command: TesterCommand, body: Vec<&str>) -> Result { - match command { - TesterCommand::Tester => tester(body).await, - TesterCommand::Timer => timer(body).await, - } -} - #[inline(never)] #[rustfmt::skip] #[allow(unused_variables)] -async fn tester(body: Vec<&str>) -> Result { +#[admin_command] +async fn tester(&self) -> Result { Ok(RoomMessageEventContent::notice_plain("completed")) } #[inline(never)] #[rustfmt::skip] -async fn timer(body: Vec<&str>) -> Result { +#[admin_command] +async fn timer(&self) -> Result { let started = std::time::Instant::now(); - timed(&body); + timed(self.body); let elapsed = started.elapsed(); Ok(RoomMessageEventContent::notice_plain(format!("completed in {elapsed:#?}"))) diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index d6ecd3f7..8917a46b 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -1,21 +1,26 @@ use std::fmt::Write; +use conduit::Result; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId, ServerName, UserId}; -use crate::{escape_html, get_room_info, services, Result}; +use crate::{admin_command, escape_html, get_room_info}; -pub(super) async fn disable_room(_body: Vec<&str>, room_id: Box) -> Result { - services().rooms.metadata.disable_room(&room_id, true)?; +#[admin_command] +pub(super) async fn disable_room(&self, room_id: Box) -> Result { + self.services.rooms.metadata.disable_room(&room_id, true)?; Ok(RoomMessageEventContent::text_plain("Room disabled.")) } -pub(super) async fn enable_room(_body: Vec<&str>, room_id: Box) -> Result { - services().rooms.metadata.disable_room(&room_id, false)?; +#[admin_command] +pub(super) async fn enable_room(&self, room_id: Box) -> Result { + self.services.rooms.metadata.disable_room(&room_id, false)?; Ok(RoomMessageEventContent::text_plain("Room enabled.")) } -pub(super) async fn incoming_federation(_body: Vec<&str>) -> Result { - let map = services() +#[admin_command] +pub(super) async fn incoming_federation(&self) -> Result { + let map = self + .services .rooms .event_handler .federation_handletime @@ -31,10 +36,10 @@ pub(super) async fn incoming_federation(_body: Vec<&str>) -> Result, server_name: Box, -) -> Result { - let response = services() +#[admin_command] +pub(super) async fn fetch_support_well_known(&self, server_name: Box) -> Result { + let response = self + .services .client .default .get(format!("https://{server_name}/.well-known/matrix/support")) @@ -72,25 +77,27 @@ pub(super) async fn fetch_support_well_known( ))) } -pub(super) async fn remote_user_in_rooms(_body: Vec<&str>, user_id: Box) -> Result { - if user_id.server_name() == services().globals.config.server_name { +#[admin_command] +pub(super) async fn remote_user_in_rooms(&self, user_id: Box) -> Result { + if user_id.server_name() == self.services.globals.config.server_name { return Ok(RoomMessageEventContent::text_plain( "User belongs to our server, please use `list-joined-rooms` user admin command instead.", )); } - if !services().users.exists(&user_id)? { + if !self.services.users.exists(&user_id)? { return Ok(RoomMessageEventContent::text_plain( "Remote user does not exist in our database.", )); } - let mut rooms: Vec<(OwnedRoomId, u64, String)> = services() + let mut rooms: Vec<(OwnedRoomId, u64, String)> = self + .services .rooms .state_cache .rooms_joined(&user_id) .filter_map(Result::ok) - .map(|room_id| get_room_info(services(), &room_id)) + .map(|room_id| get_room_info(self.services, &room_id)) .collect(); if rooms.is_empty() { diff --git a/src/admin/federation/mod.rs b/src/admin/federation/mod.rs index d02b4295..8f5d3fae 100644 --- a/src/admin/federation/mod.rs +++ b/src/admin/federation/mod.rs @@ -2,10 +2,11 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId}; +use ruma::{RoomId, ServerName, UserId}; -use self::commands::*; +use crate::admin_command_dispatch; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] pub(super) enum FederationCommand { /// - List all rooms we are currently handling an incoming pdu from @@ -39,21 +40,3 @@ pub(super) enum FederationCommand { user_id: Box, }, } - -pub(super) async fn process(command: FederationCommand, body: Vec<&str>) -> Result { - Ok(match command { - FederationCommand::DisableRoom { - room_id, - } => disable_room(body, room_id).await?, - FederationCommand::EnableRoom { - room_id, - } => enable_room(body, room_id).await?, - FederationCommand::IncomingFederation => incoming_federation(body).await?, - FederationCommand::FetchSupportWellKnown { - server_name, - } => fetch_support_well_known(body, server_name).await?, - FederationCommand::RemoteUserInRooms { - user_id, - } => remote_user_in_rooms(body, user_id).await?, - }) -} diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 26a5ea41..32360c85 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -15,10 +15,10 @@ use service::{ Services, }; -use crate::{admin, admin::AdminCommand}; +use crate::{admin, admin::AdminCommand, Command}; -struct Handler<'a> { - services: &'a Services, +struct Handler { + services: &'static Services, } #[must_use] @@ -68,13 +68,12 @@ fn reply(mut content: RoomMessageEventContent, reply_id: Option) - Some(content) } -impl Handler<'_> { +impl Handler { // Parse and process a message from the admin room async fn process(&self, msg: &str) -> CommandOutput { let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); let command = lines.next().expect("each string has at least one line"); - let body = lines.collect::>(); - let parsed = match self.parse_command(command) { + let (parsed, body) = match self.parse_command(command) { Ok(parsed) => parsed, Err(error) => { let server_name = self.services.globals.server_name(); @@ -84,7 +83,12 @@ impl Handler<'_> { }; let timer = Instant::now(); - let result = Box::pin(admin::process(parsed, body)).await; + let body: Vec<&str> = body.iter().map(String::as_str).collect(); + let context = Command { + services: self.services, + body: &body, + }; + let result = Box::pin(admin::process(parsed, &context)).await; let elapsed = timer.elapsed(); conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}"); match result { @@ -96,9 +100,10 @@ impl Handler<'_> { } // Parse chat messages from the admin room into an AdminCommand object - fn parse_command(&self, command_line: &str) -> Result { + fn parse_command(&self, command_line: &str) -> Result<(AdminCommand, Vec), String> { let argv = self.parse_line(command_line); - AdminCommand::try_parse_from(argv).map_err(|error| error.to_string()) + let com = AdminCommand::try_parse_from(&argv).map_err(|error| error.to_string())?; + Ok((com, argv)) } fn complete_command(&self, mut cmd: clap::Command, line: &str) -> String { diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index d29d5f47..7906d951 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -1,11 +1,11 @@ -use conduit::Result; +use conduit::{debug, info, Result}; use ruma::{events::room::message::RoomMessageEventContent, EventId, MxcUri}; -use tracing::{debug, info}; -use crate::services; +use crate::admin_command; +#[admin_command] pub(super) async fn delete( - _body: Vec<&str>, mxc: Option>, event_id: Option>, + &self, mxc: Option>, event_id: Option>, ) -> Result { if event_id.is_some() && mxc.is_some() { return Ok(RoomMessageEventContent::text_plain( @@ -15,7 +15,7 @@ pub(super) async fn delete( if let Some(mxc) = mxc { debug!("Got MXC URL: {mxc}"); - services().media.delete(mxc.as_ref()).await?; + self.services.media.delete(mxc.as_ref()).await?; return Ok(RoomMessageEventContent::text_plain( "Deleted the MXC from our database and on our filesystem.", @@ -27,7 +27,7 @@ pub(super) async fn delete( let mut mxc_deletion_count: usize = 0; // parsing the PDU for any MXC URLs begins here - if let Some(event_json) = services().rooms.timeline.get_pdu_json(&event_id)? { + if let Some(event_json) = self.services.rooms.timeline.get_pdu_json(&event_id)? { if let Some(content_key) = event_json.get("content") { debug!("Event ID has \"content\"."); let content_obj = content_key.as_object(); @@ -123,7 +123,7 @@ pub(super) async fn delete( } for mxc_url in mxc_urls { - services().media.delete(&mxc_url).await?; + self.services.media.delete(&mxc_url).await?; mxc_deletion_count = mxc_deletion_count.saturating_add(1); } @@ -138,23 +138,26 @@ pub(super) async fn delete( )) } -pub(super) async fn delete_list(body: Vec<&str>) -> Result { - if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { +#[admin_command] +pub(super) async fn delete_list(&self) -> Result { + if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", )); } - let mxc_list = body - .clone() - .drain(1..body.len().checked_sub(1).unwrap()) + let mxc_list = self + .body + .to_vec() + .drain(1..self.body.len().checked_sub(1).unwrap()) .collect::>(); let mut mxc_deletion_count: usize = 0; for mxc in mxc_list { debug!("Deleting MXC {mxc} in bulk"); - services().media.delete(mxc).await?; + self.services.media.delete(mxc).await?; mxc_deletion_count = mxc_deletion_count .checked_add(1) .expect("mxc_deletion_count should not get this high"); @@ -165,10 +168,10 @@ pub(super) async fn delete_list(body: Vec<&str>) -> Result, duration: String, force: bool, -) -> Result { - let deleted_count = services() +#[admin_command] +pub(super) async fn delete_past_remote_media(&self, duration: String, force: bool) -> Result { + let deleted_count = self + .services .media .delete_all_remote_media_at_after_time(duration, force) .await?; diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index d30c55d0..31cbf810 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -2,10 +2,11 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{events::room::message::RoomMessageEventContent, EventId, MxcUri}; +use ruma::{EventId, MxcUri}; -use self::commands::*; +use crate::admin_command_dispatch; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] pub(super) enum MediaCommand { /// - Deletes a single media file from our database and on the filesystem @@ -36,17 +37,3 @@ pub(super) enum MediaCommand { force: bool, }, } - -pub(super) async fn process(command: MediaCommand, body: Vec<&str>) -> Result { - Ok(match command { - MediaCommand::Delete { - mxc, - event_id, - } => delete(body, mxc, event_id).await?, - MediaCommand::DeleteList => delete_list(body).await?, - MediaCommand::DeletePastRemoteMedia { - duration, - force, - } => delete_past_remote_media(body, duration, force).await?, - }) -} diff --git a/src/admin/mod.rs b/src/admin/mod.rs index cd1110ee..5d4c8f5e 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -3,6 +3,7 @@ #![allow(clippy::enum_glob_use)] pub(crate) mod admin; +pub(crate) mod command; pub(crate) mod handler; mod tests; pub(crate) mod utils; @@ -22,9 +23,13 @@ extern crate conduit_core as conduit; extern crate conduit_service as service; pub(crate) use conduit::Result; +pub(crate) use conduit_macros::{admin_command, admin_command_dispatch}; pub(crate) use service::services; -pub(crate) use crate::utils::{escape_html, get_room_info}; +pub(crate) use crate::{ + command::Command, + utils::{escape_html, get_room_info}, +}; pub(crate) const PAGE_SIZE: usize = 100; diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index ce9caedb..e18c298a 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -1,18 +1,48 @@ -use ruma::events::room::message::RoomMessageEventContent; +use clap::Subcommand; +use conduit::Result; +use ruma::{ + events::{room::message::RoomMessageEventContent, RoomAccountDataEventType}, + RoomId, UserId, +}; -use super::AccountData; -use crate::{services, Result}; +use crate::Command; + +#[derive(Debug, Subcommand)] +/// All the getters and iterators from src/database/key_value/account_data.rs +pub(crate) enum AccountDataCommand { + /// - Returns all changes to the account data that happened after `since`. + ChangesSince { + /// Full user ID + user_id: Box, + /// UNIX timestamp since (u64) + since: u64, + /// Optional room ID of the account data + room_id: Option>, + }, + + /// - Searches the account data for a specific kind. + Get { + /// Full user ID + user_id: Box, + /// Account data event type + kind: RoomAccountDataEventType, + /// Optional room ID of the account data + room_id: Option>, + }, +} /// All the getters and iterators from src/database/key_value/account_data.rs -pub(super) async fn account_data(subcommand: AccountData) -> Result { +pub(super) async fn process(subcommand: AccountDataCommand, context: &Command<'_>) -> Result { + let services = context.services; + match subcommand { - AccountData::ChangesSince { + AccountDataCommand::ChangesSince { user_id, since, room_id, } => { let timer = tokio::time::Instant::now(); - let results = services() + let results = services .account_data .changes_since(room_id.as_deref(), &user_id, since)?; let query_time = timer.elapsed(); @@ -21,13 +51,13 @@ pub(super) async fn account_data(subcommand: AccountData) -> Result { let timer = tokio::time::Instant::now(); - let results = services() + let results = services .account_data .get(room_id.as_deref(), &user_id, kind)?; let query_time = timer.elapsed(); diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 77ac9641..683c228f 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -1,16 +1,32 @@ +use clap::Subcommand; +use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; -use super::Appservice; -use crate::{services, Result}; +use crate::Command; + +#[derive(Debug, Subcommand)] +/// All the getters and iterators from src/database/key_value/appservice.rs +pub(crate) enum AppserviceCommand { + /// - Gets the appservice registration info/details from the ID as a string + GetRegistration { + /// Appservice registration ID + appservice_id: Box, + }, + + /// - Gets all appservice registrations with their ID and registration info + All, +} /// All the getters and iterators from src/database/key_value/appservice.rs -pub(super) async fn appservice(subcommand: Appservice) -> Result { +pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_>) -> Result { + let services = context.services; + match subcommand { - Appservice::GetRegistration { + AppserviceCommand::GetRegistration { appservice_id, } => { let timer = tokio::time::Instant::now(); - let results = services() + let results = services .appservice .db .get_registration(appservice_id.as_ref()); @@ -20,9 +36,9 @@ pub(super) async fn appservice(subcommand: Appservice) -> Result { + AppserviceCommand::All => { let timer = tokio::time::Instant::now(); - let results = services().appservice.all(); + let results = services.appservice.all(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 38986071..5f271c2c 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -1,52 +1,73 @@ -use ruma::events::room::message::RoomMessageEventContent; +use clap::Subcommand; +use conduit::Result; +use ruma::{events::room::message::RoomMessageEventContent, ServerName}; -use super::Globals; -use crate::{services, Result}; +use crate::Command; + +#[derive(Debug, Subcommand)] +/// All the getters and iterators from src/database/key_value/globals.rs +pub(crate) enum GlobalsCommand { + DatabaseVersion, + + CurrentCount, + + LastCheckForUpdatesId, + + LoadKeypair, + + /// - This returns an empty `Ok(BTreeMap<..>)` when there are no keys found + /// for the server. + SigningKeysFor { + origin: Box, + }, +} /// All the getters and iterators from src/database/key_value/globals.rs -pub(super) async fn globals(subcommand: Globals) -> Result { +pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) -> Result { + let services = context.services; + match subcommand { - Globals::DatabaseVersion => { + GlobalsCommand::DatabaseVersion => { let timer = tokio::time::Instant::now(); - let results = services().globals.db.database_version(); + let results = services.globals.db.database_version(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - Globals::CurrentCount => { + GlobalsCommand::CurrentCount => { let timer = tokio::time::Instant::now(); - let results = services().globals.db.current_count(); + let results = services.globals.db.current_count(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - Globals::LastCheckForUpdatesId => { + GlobalsCommand::LastCheckForUpdatesId => { let timer = tokio::time::Instant::now(); - let results = services().updates.last_check_for_updates_id(); + let results = services.updates.last_check_for_updates_id(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - Globals::LoadKeypair => { + GlobalsCommand::LoadKeypair => { let timer = tokio::time::Instant::now(); - let results = services().globals.db.load_keypair(); + let results = services.globals.db.load_keypair(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - Globals::SigningKeysFor { + GlobalsCommand::SigningKeysFor { origin, } => { let timer = tokio::time::Instant::now(); - let results = services().globals.db.verify_keys_for(&origin); + let results = services.globals.db.verify_keys_for(&origin); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index c86f4f53..1aa28c48 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -10,304 +10,51 @@ mod users; use clap::Subcommand; use conduit::Result; -use room_state_cache::room_state_cache; -use ruma::{ - events::{room::message::RoomMessageEventContent, RoomAccountDataEventType}, - OwnedServerName, RoomAliasId, RoomId, ServerName, UserId, -}; use self::{ - account_data::account_data, appservice::appservice, globals::globals, presence::presence, resolver::resolver, - room_alias::room_alias, sending::sending, users::users, + account_data::AccountDataCommand, appservice::AppserviceCommand, globals::GlobalsCommand, + presence::PresenceCommand, resolver::ResolverCommand, room_alias::RoomAliasCommand, + room_state_cache::RoomStateCacheCommand, sending::SendingCommand, users::UsersCommand, }; +use crate::admin_command_dispatch; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] /// Query tables from database pub(super) enum QueryCommand { /// - account_data.rs iterators and getters #[command(subcommand)] - AccountData(AccountData), + AccountData(AccountDataCommand), /// - appservice.rs iterators and getters #[command(subcommand)] - Appservice(Appservice), + Appservice(AppserviceCommand), /// - presence.rs iterators and getters #[command(subcommand)] - Presence(Presence), + Presence(PresenceCommand), /// - rooms/alias.rs iterators and getters #[command(subcommand)] - RoomAlias(RoomAlias), + RoomAlias(RoomAliasCommand), /// - rooms/state_cache iterators and getters #[command(subcommand)] - RoomStateCache(RoomStateCache), + RoomStateCache(RoomStateCacheCommand), /// - globals.rs iterators and getters #[command(subcommand)] - Globals(Globals), + Globals(GlobalsCommand), /// - sending.rs iterators and getters #[command(subcommand)] - Sending(Sending), + Sending(SendingCommand), /// - users.rs iterators and getters #[command(subcommand)] - Users(Users), + Users(UsersCommand), /// - resolver service #[command(subcommand)] - Resolver(Resolver), -} - -#[derive(Debug, Subcommand)] -/// All the getters and iterators from src/database/key_value/account_data.rs -pub(super) enum AccountData { - /// - Returns all changes to the account data that happened after `since`. - ChangesSince { - /// Full user ID - user_id: Box, - /// UNIX timestamp since (u64) - since: u64, - /// Optional room ID of the account data - room_id: Option>, - }, - - /// - Searches the account data for a specific kind. - Get { - /// Full user ID - user_id: Box, - /// Account data event type - kind: RoomAccountDataEventType, - /// Optional room ID of the account data - room_id: Option>, - }, -} - -#[derive(Debug, Subcommand)] -/// All the getters and iterators from src/database/key_value/appservice.rs -pub(super) enum Appservice { - /// - Gets the appservice registration info/details from the ID as a string - GetRegistration { - /// Appservice registration ID - appservice_id: Box, - }, - - /// - Gets all appservice registrations with their ID and registration info - All, -} - -#[derive(Debug, Subcommand)] -/// All the getters and iterators from src/database/key_value/presence.rs -pub(super) enum Presence { - /// - Returns the latest presence event for the given user. - GetPresence { - /// Full user ID - user_id: Box, - }, - - /// - Iterator of the most recent presence updates that happened after the - /// event with id `since`. - PresenceSince { - /// UNIX timestamp since (u64) - since: u64, - }, -} - -#[derive(Debug, Subcommand)] -/// All the getters and iterators from src/database/key_value/rooms/alias.rs -pub(super) enum RoomAlias { - ResolveLocalAlias { - /// Full room alias - alias: Box, - }, - - /// - Iterator of all our local room aliases for the room ID - LocalAliasesForRoom { - /// Full room ID - room_id: Box, - }, - - /// - Iterator of all our local aliases in our database with their room IDs - AllLocalAliases, -} - -#[derive(Debug, Subcommand)] -pub(super) enum RoomStateCache { - ServerInRoom { - server: Box, - room_id: Box, - }, - - RoomServers { - room_id: Box, - }, - - ServerRooms { - server: Box, - }, - - RoomMembers { - room_id: Box, - }, - - LocalUsersInRoom { - room_id: Box, - }, - - ActiveLocalUsersInRoom { - room_id: Box, - }, - - RoomJoinedCount { - room_id: Box, - }, - - RoomInvitedCount { - room_id: Box, - }, - - RoomUserOnceJoined { - room_id: Box, - }, - - RoomMembersInvited { - room_id: Box, - }, - - GetInviteCount { - room_id: Box, - user_id: Box, - }, - - GetLeftCount { - room_id: Box, - user_id: Box, - }, - - RoomsJoined { - user_id: Box, - }, - - RoomsLeft { - user_id: Box, - }, - - RoomsInvited { - user_id: Box, - }, - - InviteState { - user_id: Box, - room_id: Box, - }, -} - -#[derive(Debug, Subcommand)] -/// All the getters and iterators from src/database/key_value/globals.rs -pub(super) enum Globals { - DatabaseVersion, - - CurrentCount, - - LastCheckForUpdatesId, - - LoadKeypair, - - /// - This returns an empty `Ok(BTreeMap<..>)` when there are no keys found - /// for the server. - SigningKeysFor { - origin: Box, - }, -} - -#[derive(Debug, Subcommand)] -/// All the getters and iterators from src/database/key_value/sending.rs -pub(super) enum Sending { - /// - Queries database for all `servercurrentevent_data` - ActiveRequests, - - /// - Queries database for `servercurrentevent_data` but for a specific - /// destination - /// - /// This command takes only *one* format of these arguments: - /// - /// appservice_id - /// server_name - /// user_id AND push_key - /// - /// See src/service/sending/mod.rs for the definition of the `Destination` - /// enum - ActiveRequestsFor { - #[arg(short, long)] - appservice_id: Option, - #[arg(short, long)] - server_name: Option>, - #[arg(short, long)] - user_id: Option>, - #[arg(short, long)] - push_key: Option, - }, - - /// - Queries database for `servernameevent_data` which are the queued up - /// requests that will eventually be sent - /// - /// This command takes only *one* format of these arguments: - /// - /// appservice_id - /// server_name - /// user_id AND push_key - /// - /// See src/service/sending/mod.rs for the definition of the `Destination` - /// enum - QueuedRequests { - #[arg(short, long)] - appservice_id: Option, - #[arg(short, long)] - server_name: Option>, - #[arg(short, long)] - user_id: Option>, - #[arg(short, long)] - push_key: Option, - }, - - GetLatestEduCount { - server_name: Box, - }, -} - -#[derive(Debug, Subcommand)] -/// All the getters and iterators from src/database/key_value/users.rs -pub(super) enum Users { - Iter, -} - -#[derive(Debug, Subcommand)] -/// Resolver service and caches -pub(super) enum Resolver { - /// Query the destinations cache - DestinationsCache { - server_name: Option, - }, - - /// Query the overrides cache - OverridesCache { - name: Option, - }, -} - -/// Processes admin query commands -pub(super) async fn process(command: QueryCommand, _body: Vec<&str>) -> Result { - Ok(match command { - QueryCommand::AccountData(command) => account_data(command).await?, - QueryCommand::Appservice(command) => appservice(command).await?, - QueryCommand::Presence(command) => presence(command).await?, - QueryCommand::RoomAlias(command) => room_alias(command).await?, - QueryCommand::RoomStateCache(command) => room_state_cache(command).await?, - QueryCommand::Globals(command) => globals(command).await?, - QueryCommand::Sending(command) => sending(command).await?, - QueryCommand::Users(command) => users(command).await?, - QueryCommand::Resolver(command) => resolver(command).await?, - }) + Resolver(ResolverCommand), } diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index c47b7a51..145ecd9b 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -1,27 +1,47 @@ -use ruma::events::room::message::RoomMessageEventContent; +use clap::Subcommand; +use conduit::Result; +use ruma::{events::room::message::RoomMessageEventContent, UserId}; -use super::Presence; -use crate::{services, Result}; +use crate::Command; + +#[derive(Debug, Subcommand)] +/// All the getters and iterators from src/database/key_value/presence.rs +pub(crate) enum PresenceCommand { + /// - Returns the latest presence event for the given user. + GetPresence { + /// Full user ID + user_id: Box, + }, + + /// - Iterator of the most recent presence updates that happened after the + /// event with id `since`. + PresenceSince { + /// UNIX timestamp since (u64) + since: u64, + }, +} /// All the getters and iterators in key_value/presence.rs -pub(super) async fn presence(subcommand: Presence) -> Result { +pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) -> Result { + let services = context.services; + match subcommand { - Presence::GetPresence { + PresenceCommand::GetPresence { user_id, } => { let timer = tokio::time::Instant::now(); - let results = services().presence.db.get_presence(&user_id)?; + let results = services.presence.db.get_presence(&user_id)?; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - Presence::PresenceSince { + PresenceCommand::PresenceSince { since, } => { let timer = tokio::time::Instant::now(); - let results = services().presence.db.presence_since(since); + let results = services.presence.db.presence_since(since); let presence_since: Vec<(_, _, _)> = results.collect(); let query_time = timer.elapsed(); diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index 37d17960..e8340dad 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -1,24 +1,28 @@ use std::fmt::Write; +use clap::Subcommand; use conduit::{utils::time, Result}; use ruma::{events::room::message::RoomMessageEventContent, OwnedServerName}; -use super::Resolver; -use crate::services; +use crate::{admin_command, admin_command_dispatch}; -/// All the getters and iterators in key_value/users.rs -pub(super) async fn resolver(subcommand: Resolver) -> Result { - match subcommand { - Resolver::DestinationsCache { - server_name, - } => destinations_cache(server_name).await, - Resolver::OverridesCache { - name, - } => overrides_cache(name).await, - } +#[admin_command_dispatch] +#[derive(Debug, Subcommand)] +/// Resolver service and caches +pub(crate) enum ResolverCommand { + /// Query the destinations cache + DestinationsCache { + server_name: Option, + }, + + /// Query the overrides cache + OverridesCache { + name: Option, + }, } -async fn destinations_cache(server_name: Option) -> Result { +#[admin_command] +async fn destinations_cache(&self, server_name: Option) -> Result { use service::resolver::cache::CachedDest; let mut out = String::new(); @@ -36,7 +40,8 @@ async fn destinations_cache(server_name: Option) -> Result) -> Result) -> Result { +#[admin_command] +async fn overrides_cache(&self, server_name: Option) -> Result { use service::resolver::cache::CachedOverride; let mut out = String::new(); @@ -70,7 +76,13 @@ async fn overrides_cache(server_name: Option) -> Result, + }, + + /// - Iterator of all our local room aliases for the room ID + LocalAliasesForRoom { + /// Full room ID + room_id: Box, + }, + + /// - Iterator of all our local aliases in our database with their room IDs + AllLocalAliases, +} /// All the getters and iterators in src/database/key_value/rooms/alias.rs -pub(super) async fn room_alias(subcommand: RoomAlias) -> Result { +pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) -> Result { + let services = context.services; + match subcommand { - RoomAlias::ResolveLocalAlias { + RoomAliasCommand::ResolveLocalAlias { alias, } => { let timer = tokio::time::Instant::now(); - let results = services().rooms.alias.resolve_local_alias(&alias); + let results = services.rooms.alias.resolve_local_alias(&alias); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomAlias::LocalAliasesForRoom { + RoomAliasCommand::LocalAliasesForRoom { room_id, } => { let timer = tokio::time::Instant::now(); - let results = services().rooms.alias.local_aliases_for_room(&room_id); + let results = services.rooms.alias.local_aliases_for_room(&room_id); let aliases: Vec<_> = results.collect(); let query_time = timer.elapsed(); @@ -29,9 +50,9 @@ pub(super) async fn room_alias(subcommand: RoomAlias) -> Result { + RoomAliasCommand::AllLocalAliases => { let timer = tokio::time::Instant::now(); - let results = services().rooms.alias.all_local_aliases(); + let results = services.rooms.alias.all_local_aliases(); let aliases: Vec<_> = results.collect(); let query_time = timer.elapsed(); diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index aed2b4a2..4215cf8d 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -1,71 +1,136 @@ -use ruma::events::room::message::RoomMessageEventContent; +use clap::Subcommand; +use conduit::Result; +use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId}; -use super::RoomStateCache; -use crate::{services, Result}; +use crate::Command; + +#[derive(Debug, Subcommand)] +pub(crate) enum RoomStateCacheCommand { + ServerInRoom { + server: Box, + room_id: Box, + }, + + RoomServers { + room_id: Box, + }, + + ServerRooms { + server: Box, + }, + + RoomMembers { + room_id: Box, + }, + + LocalUsersInRoom { + room_id: Box, + }, + + ActiveLocalUsersInRoom { + room_id: Box, + }, + + RoomJoinedCount { + room_id: Box, + }, + + RoomInvitedCount { + room_id: Box, + }, + + RoomUserOnceJoined { + room_id: Box, + }, + + RoomMembersInvited { + room_id: Box, + }, + + GetInviteCount { + room_id: Box, + user_id: Box, + }, + + GetLeftCount { + room_id: Box, + user_id: Box, + }, + + RoomsJoined { + user_id: Box, + }, + + RoomsLeft { + user_id: Box, + }, + + RoomsInvited { + user_id: Box, + }, + + InviteState { + user_id: Box, + room_id: Box, + }, +} + +pub(super) async fn process( + subcommand: RoomStateCacheCommand, context: &Command<'_>, +) -> Result { + let services = context.services; -pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result { match subcommand { - RoomStateCache::ServerInRoom { + RoomStateCacheCommand::ServerInRoom { server, room_id, } => { let timer = tokio::time::Instant::now(); - let result = services() - .rooms - .state_cache - .server_in_room(&server, &room_id); + let result = services.rooms.state_cache.server_in_room(&server, &room_id); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" ))) }, - RoomStateCache::RoomServers { + RoomStateCacheCommand::RoomServers { room_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services() - .rooms - .state_cache - .room_servers(&room_id) - .collect(); + let results: Result> = services.rooms.state_cache.room_servers(&room_id).collect(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCache::ServerRooms { + RoomStateCacheCommand::ServerRooms { server, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services().rooms.state_cache.server_rooms(&server).collect(); + let results: Result> = services.rooms.state_cache.server_rooms(&server).collect(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCache::RoomMembers { + RoomStateCacheCommand::RoomMembers { room_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services() - .rooms - .state_cache - .room_members(&room_id) - .collect(); + let results: Result> = services.rooms.state_cache.room_members(&room_id).collect(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCache::LocalUsersInRoom { + RoomStateCacheCommand::LocalUsersInRoom { room_id, } => { let timer = tokio::time::Instant::now(); - let results: Vec<_> = services() + let results: Vec<_> = services .rooms .state_cache .local_users_in_room(&room_id) @@ -76,11 +141,11 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result { let timer = tokio::time::Instant::now(); - let results: Vec<_> = services() + let results: Vec<_> = services .rooms .state_cache .active_local_users_in_room(&room_id) @@ -91,33 +156,33 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result { let timer = tokio::time::Instant::now(); - let results = services().rooms.state_cache.room_joined_count(&room_id); + let results = services.rooms.state_cache.room_joined_count(&room_id); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCache::RoomInvitedCount { + RoomStateCacheCommand::RoomInvitedCount { room_id, } => { let timer = tokio::time::Instant::now(); - let results = services().rooms.state_cache.room_invited_count(&room_id); + let results = services.rooms.state_cache.room_invited_count(&room_id); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCache::RoomUserOnceJoined { + RoomStateCacheCommand::RoomUserOnceJoined { room_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services() + let results: Result> = services .rooms .state_cache .room_useroncejoined(&room_id) @@ -128,11 +193,11 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result { let timer = tokio::time::Instant::now(); - let results: Result> = services() + let results: Result> = services .rooms .state_cache .room_members_invited(&room_id) @@ -143,12 +208,12 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result { let timer = tokio::time::Instant::now(); - let results = services() + let results = services .rooms .state_cache .get_invite_count(&room_id, &user_id); @@ -158,12 +223,12 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result { let timer = tokio::time::Instant::now(); - let results = services() + let results = services .rooms .state_cache .get_left_count(&room_id, &user_id); @@ -173,56 +238,45 @@ pub(super) async fn room_state_cache(subcommand: RoomStateCache) -> Result { let timer = tokio::time::Instant::now(); - let results: Result> = services() - .rooms - .state_cache - .rooms_joined(&user_id) - .collect(); + let results: Result> = services.rooms.state_cache.rooms_joined(&user_id).collect(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCache::RoomsInvited { + RoomStateCacheCommand::RoomsInvited { user_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services() - .rooms - .state_cache - .rooms_invited(&user_id) - .collect(); + let results: Result> = services.rooms.state_cache.rooms_invited(&user_id).collect(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCache::RoomsLeft { + RoomStateCacheCommand::RoomsLeft { user_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services().rooms.state_cache.rooms_left(&user_id).collect(); + let results: Result> = services.rooms.state_cache.rooms_left(&user_id).collect(); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCache::InviteState { + RoomStateCacheCommand::InviteState { user_id, room_id, } => { let timer = tokio::time::Instant::now(); - let results = services() - .rooms - .state_cache - .invite_state(&user_id, &room_id); + let results = services.rooms.state_cache.invite_state(&user_id, &room_id); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 4e82695d..6d54bddf 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -1,14 +1,73 @@ -use ruma::events::room::message::RoomMessageEventContent; +use clap::Subcommand; +use conduit::Result; +use ruma::{events::room::message::RoomMessageEventContent, ServerName, UserId}; +use service::sending::Destination; -use super::Sending; -use crate::{service::sending::Destination, services, Result}; +use crate::Command; + +#[derive(Debug, Subcommand)] +/// All the getters and iterators from src/database/key_value/sending.rs +pub(crate) enum SendingCommand { + /// - Queries database for all `servercurrentevent_data` + ActiveRequests, + + /// - Queries database for `servercurrentevent_data` but for a specific + /// destination + /// + /// This command takes only *one* format of these arguments: + /// + /// appservice_id + /// server_name + /// user_id AND push_key + /// + /// See src/service/sending/mod.rs for the definition of the `Destination` + /// enum + ActiveRequestsFor { + #[arg(short, long)] + appservice_id: Option, + #[arg(short, long)] + server_name: Option>, + #[arg(short, long)] + user_id: Option>, + #[arg(short, long)] + push_key: Option, + }, + + /// - Queries database for `servernameevent_data` which are the queued up + /// requests that will eventually be sent + /// + /// This command takes only *one* format of these arguments: + /// + /// appservice_id + /// server_name + /// user_id AND push_key + /// + /// See src/service/sending/mod.rs for the definition of the `Destination` + /// enum + QueuedRequests { + #[arg(short, long)] + appservice_id: Option, + #[arg(short, long)] + server_name: Option>, + #[arg(short, long)] + user_id: Option>, + #[arg(short, long)] + push_key: Option, + }, + + GetLatestEduCount { + server_name: Box, + }, +} /// All the getters and iterators in key_value/sending.rs -pub(super) async fn sending(subcommand: Sending) -> Result { +pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -> Result { + let services = context.services; + match subcommand { - Sending::ActiveRequests => { + SendingCommand::ActiveRequests => { let timer = tokio::time::Instant::now(); - let results = services().sending.db.active_requests(); + let results = services.sending.db.active_requests(); let active_requests: Result> = results.collect(); let query_time = timer.elapsed(); @@ -16,7 +75,7 @@ pub(super) async fn sending(subcommand: Sending) -> Result Result services() + (None, Some(server_name), None, None) => services .sending .db .queued_requests(&Destination::Normal(server_name.into())), @@ -55,7 +114,7 @@ pub(super) async fn sending(subcommand: Sending) -> Result Result Result services() + (None, Some(server_name), None, None) => services .sending .db .active_requests_for(&Destination::Normal(server_name.into())), @@ -121,7 +180,7 @@ pub(super) async fn sending(subcommand: Sending) -> Result Result { let timer = tokio::time::Instant::now(); - let results = services().sending.db.get_latest_educount(&server_name); + let results = services.sending.db.get_latest_educount(&server_name); let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index 2e73bff3..fee12fbf 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -1,14 +1,23 @@ +use clap::Subcommand; +use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; -use super::Users; -use crate::{services, Result}; +use crate::Command; + +#[derive(Debug, Subcommand)] +/// All the getters and iterators from src/database/key_value/users.rs +pub(crate) enum UsersCommand { + Iter, +} /// All the getters and iterators in key_value/users.rs -pub(super) async fn users(subcommand: Users) -> Result { +pub(super) async fn process(subcommand: UsersCommand, context: &Command<'_>) -> Result { + let services = context.services; + match subcommand { - Users::Iter => { + UsersCommand::Iter => { let timer = tokio::time::Instant::now(); - let results = services().users.db.iter(); + let results = services.users.db.iter(); let users = results.collect::>(); let query_time = timer.elapsed(); diff --git a/src/admin/room/room_alias_commands.rs b/src/admin/room/alias.rs similarity index 73% rename from src/admin/room/room_alias_commands.rs rename to src/admin/room/alias.rs index 4f43ac6e..415b8a08 100644 --- a/src/admin/room/room_alias_commands.rs +++ b/src/admin/room/alias.rs @@ -1,12 +1,49 @@ use std::fmt::Write; -use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId}; +use clap::Subcommand; +use conduit::Result; +use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, RoomId}; -use super::RoomAliasCommand; -use crate::{escape_html, services, Result}; +use crate::{escape_html, Command}; -pub(super) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Result { - let server_user = &services().globals.server_user; +#[derive(Debug, Subcommand)] +pub(crate) enum RoomAliasCommand { + /// - Make an alias point to a room. + Set { + #[arg(short, long)] + /// Set the alias even if a room is already using it + force: bool, + + /// The room id to set the alias on + room_id: Box, + + /// The alias localpart to use (`alias`, not `#alias:servername.tld`) + room_alias_localpart: String, + }, + + /// - Remove a local alias + Remove { + /// The alias localpart to remove (`alias`, not `#alias:servername.tld`) + room_alias_localpart: String, + }, + + /// - Show which room is using an alias + Which { + /// The alias localpart to look up (`alias`, not + /// `#alias:servername.tld`) + room_alias_localpart: String, + }, + + /// - List aliases currently being used + List { + /// If set, only list the aliases for this room + room_id: Option>, + }, +} + +pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> Result { + let services = context.services; + let server_user = &services.globals.server_user; match command { RoomAliasCommand::Set { @@ -19,7 +56,7 @@ pub(super) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu | RoomAliasCommand::Which { ref room_alias_localpart, } => { - let room_alias_str = format!("#{}:{}", room_alias_localpart, services().globals.server_name()); + let room_alias_str = format!("#{}:{}", room_alias_localpart, services.globals.server_name()); let room_alias = match RoomAliasId::parse_box(room_alias_str) { Ok(alias) => alias, Err(err) => return Ok(RoomMessageEventContent::text_plain(format!("Failed to parse alias: {err}"))), @@ -29,8 +66,8 @@ pub(super) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu force, room_id, .. - } => match (force, services().rooms.alias.resolve_local_alias(&room_alias)) { - (true, Ok(Some(id))) => match services() + } => match (force, services.rooms.alias.resolve_local_alias(&room_alias)) { + (true, Ok(Some(id))) => match services .rooms .alias .set_alias(&room_alias, &room_id, server_user) @@ -43,7 +80,7 @@ pub(super) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu (false, Ok(Some(id))) => Ok(RoomMessageEventContent::text_plain(format!( "Refusing to overwrite in use alias for {id}, use -f or --force to overwrite" ))), - (_, Ok(None)) => match services() + (_, Ok(None)) => match services .rooms .alias .set_alias(&room_alias, &room_id, server_user) @@ -55,8 +92,8 @@ pub(super) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu }, RoomAliasCommand::Remove { .. - } => match services().rooms.alias.resolve_local_alias(&room_alias) { - Ok(Some(id)) => match services() + } => match services.rooms.alias.resolve_local_alias(&room_alias) { + Ok(Some(id)) => match services .rooms .alias .remove_alias(&room_alias, server_user) @@ -70,7 +107,7 @@ pub(super) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu }, RoomAliasCommand::Which { .. - } => match services().rooms.alias.resolve_local_alias(&room_alias) { + } => match services.rooms.alias.resolve_local_alias(&room_alias) { Ok(Some(id)) => Ok(RoomMessageEventContent::text_plain(format!("Alias resolves to {id}"))), Ok(None) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {err}"))), @@ -84,7 +121,7 @@ pub(super) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu room_id, } => { if let Some(room_id) = room_id { - let aliases = services() + let aliases = services .rooms .alias .local_aliases_for_room(&room_id) @@ -109,14 +146,14 @@ pub(super) async fn process(command: RoomAliasCommand, _body: Vec<&str>) -> Resu Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to list aliases: {err}"))), } } else { - let aliases = services() + let aliases = services .rooms .alias .all_local_aliases() .collect::, _>>(); match aliases { Ok(aliases) => { - let server_name = services().globals.server_name(); + let server_name = services.globals.server_name(); let plain_list = aliases .iter() .fold(String::new(), |mut output, (alias, id)| { diff --git a/src/admin/room/room_commands.rs b/src/admin/room/commands.rs similarity index 82% rename from src/admin/room/room_commands.rs rename to src/admin/room/commands.rs index cf0f3ddb..2e14a849 100644 --- a/src/admin/room/room_commands.rs +++ b/src/admin/room/commands.rs @@ -1,15 +1,18 @@ use std::fmt::Write; +use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; -use crate::{escape_html, get_room_info, services, Result, PAGE_SIZE}; +use crate::{admin_command, escape_html, get_room_info, PAGE_SIZE}; -pub(super) async fn list( - _body: Vec<&str>, page: Option, exclude_disabled: bool, exclude_banned: bool, +#[admin_command] +pub(super) async fn list_rooms( + &self, page: Option, exclude_disabled: bool, exclude_banned: bool, ) -> Result { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); - let mut rooms = services() + let mut rooms = self + .services .rooms .metadata .iter_ids() @@ -18,7 +21,8 @@ pub(super) async fn list( .ok() .filter(|room_id| { if exclude_disabled - && services() + && self + .services .rooms .metadata .is_disabled(room_id) @@ -28,7 +32,8 @@ pub(super) async fn list( } if exclude_banned - && services() + && self + .services .rooms .metadata .is_banned(room_id) @@ -39,7 +44,7 @@ pub(super) async fn list( true }) - .map(|room_id| get_room_info(services(), &room_id)) + .map(|room_id| get_room_info(self.services, &room_id)) }) .collect::>(); rooms.sort_by_key(|r| r.1); diff --git a/src/admin/room/room_directory_commands.rs b/src/admin/room/directory.rs similarity index 68% rename from src/admin/room/room_directory_commands.rs rename to src/admin/room/directory.rs index 912e970c..7bba2eb7 100644 --- a/src/admin/room/room_directory_commands.rs +++ b/src/admin/room/directory.rs @@ -1,21 +1,43 @@ use std::fmt::Write; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId}; +use clap::Subcommand; +use conduit::Result; +use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId}; -use super::RoomDirectoryCommand; -use crate::{escape_html, get_room_info, services, Result, PAGE_SIZE}; +use crate::{escape_html, get_room_info, Command, PAGE_SIZE}; -pub(super) async fn process(command: RoomDirectoryCommand, _body: Vec<&str>) -> Result { +#[derive(Debug, Subcommand)] +pub(crate) enum RoomDirectoryCommand { + /// - Publish a room to the room directory + Publish { + /// The room id of the room to publish + room_id: Box, + }, + + /// - Unpublish a room to the room directory + Unpublish { + /// The room id of the room to unpublish + room_id: Box, + }, + + /// - List rooms that are published + List { + page: Option, + }, +} + +pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_>) -> Result { + let services = context.services; match command { RoomDirectoryCommand::Publish { room_id, - } => match services().rooms.directory.set_public(&room_id) { + } => match services.rooms.directory.set_public(&room_id) { Ok(()) => Ok(RoomMessageEventContent::text_plain("Room published")), Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to update room: {err}"))), }, RoomDirectoryCommand::Unpublish { room_id, - } => match services().rooms.directory.set_not_public(&room_id) { + } => match services.rooms.directory.set_not_public(&room_id) { Ok(()) => Ok(RoomMessageEventContent::text_plain("Room unpublished")), Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to update room: {err}"))), }, @@ -24,12 +46,12 @@ pub(super) async fn process(command: RoomDirectoryCommand, _body: Vec<&str>) -> } => { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); - let mut rooms = services() + let mut rooms = services .rooms .directory .public_rooms() .filter_map(Result::ok) - .map(|id: OwnedRoomId| get_room_info(services(), &id)) + .map(|id: OwnedRoomId| get_room_info(services, &id)) .collect::>(); rooms.sort_by_key(|r| r.1); rooms.reverse(); diff --git a/src/admin/room/room_info_commands.rs b/src/admin/room/info.rs similarity index 54% rename from src/admin/room/room_info_commands.rs rename to src/admin/room/info.rs index 9182616f..8ba0a796 100644 --- a/src/admin/room/room_info_commands.rs +++ b/src/admin/room/info.rs @@ -1,22 +1,30 @@ +use clap::Subcommand; +use conduit::Result; use ruma::{events::room::message::RoomMessageEventContent, RoomId}; -use service::services; -use super::RoomInfoCommand; -use crate::Result; +use crate::{admin_command, admin_command_dispatch}; -pub(super) async fn process(command: RoomInfoCommand, body: Vec<&str>) -> Result { - match command { - RoomInfoCommand::ListJoinedMembers { - room_id, - } => list_joined_members(body, room_id).await, - RoomInfoCommand::ViewRoomTopic { - room_id, - } => view_room_topic(body, room_id).await, - } +#[admin_command_dispatch] +#[derive(Debug, Subcommand)] +pub(crate) enum RoomInfoCommand { + /// - List joined members in a room + ListJoinedMembers { + room_id: Box, + }, + + /// - Displays room topic + /// + /// Room topics can be huge, so this is in its + /// own separate command + ViewRoomTopic { + room_id: Box, + }, } -async fn list_joined_members(_body: Vec<&str>, room_id: Box) -> Result { - let room_name = services() +#[admin_command] +async fn list_joined_members(&self, room_id: Box) -> Result { + let room_name = self + .services .rooms .state_accessor .get_name(&room_id) @@ -24,7 +32,8 @@ async fn list_joined_members(_body: Vec<&str>, room_id: Box) -> Result, room_id: Box) -> Result, room_id: Box) -> Result, room_id: Box) -> Result { - let Some(room_topic) = services().rooms.state_accessor.get_room_topic(&room_id)? else { +#[admin_command] +async fn view_room_topic(&self, room_id: Box) -> Result { + let Some(room_topic) = self + .services + .rooms + .state_accessor + .get_room_topic(&room_id)? + else { return Ok(RoomMessageEventContent::text_plain("Room does not have a room topic set.")); }; diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index 8f125f0a..da7acb80 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -1,19 +1,23 @@ -mod room_alias_commands; -mod room_commands; -mod room_directory_commands; -mod room_info_commands; -mod room_moderation_commands; +mod alias; +mod commands; +mod directory; +mod info; +mod moderation; use clap::Subcommand; use conduit::Result; -use ruma::{events::room::message::RoomMessageEventContent, RoomId, RoomOrAliasId}; -use self::room_commands::list; +use self::{ + alias::RoomAliasCommand, directory::RoomDirectoryCommand, info::RoomInfoCommand, moderation::RoomModerationCommand, +}; +use crate::admin_command_dispatch; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] pub(super) enum RoomCommand { /// - List all rooms the server knows about - List { + #[clap(alias = "list")] + ListRooms { page: Option, /// Excludes rooms that we have federation disabled with @@ -41,149 +45,3 @@ pub(super) enum RoomCommand { /// - Manage the room directory Directory(RoomDirectoryCommand), } - -#[derive(Debug, Subcommand)] -pub(super) enum RoomInfoCommand { - /// - List joined members in a room - ListJoinedMembers { - room_id: Box, - }, - - /// - Displays room topic - /// - /// Room topics can be huge, so this is in its - /// own separate command - ViewRoomTopic { - room_id: Box, - }, -} - -#[derive(Debug, Subcommand)] -pub(super) enum RoomAliasCommand { - /// - Make an alias point to a room. - Set { - #[arg(short, long)] - /// Set the alias even if a room is already using it - force: bool, - - /// The room id to set the alias on - room_id: Box, - - /// The alias localpart to use (`alias`, not `#alias:servername.tld`) - room_alias_localpart: String, - }, - - /// - Remove a local alias - Remove { - /// The alias localpart to remove (`alias`, not `#alias:servername.tld`) - room_alias_localpart: String, - }, - - /// - Show which room is using an alias - Which { - /// The alias localpart to look up (`alias`, not - /// `#alias:servername.tld`) - room_alias_localpart: String, - }, - - /// - List aliases currently being used - List { - /// If set, only list the aliases for this room - room_id: Option>, - }, -} - -#[derive(Debug, Subcommand)] -pub(super) enum RoomDirectoryCommand { - /// - Publish a room to the room directory - Publish { - /// The room id of the room to publish - room_id: Box, - }, - - /// - Unpublish a room to the room directory - Unpublish { - /// The room id of the room to unpublish - room_id: Box, - }, - - /// - List rooms that are published - List { - page: Option, - }, -} - -#[derive(Debug, Subcommand)] -pub(super) enum RoomModerationCommand { - /// - Bans a room from local users joining and evicts all our local users - /// from the room. Also blocks any invites (local and remote) for the - /// banned room. - /// - /// Server admins (users in the conduwuit admin room) will not be evicted - /// and server admins can still join the room. To evict admins too, use - /// --force (also ignores errors) To disable incoming federation of the - /// room, use --disable-federation - BanRoom { - #[arg(short, long)] - /// Evicts admins out of the room and ignores any potential errors when - /// making our local users leave the room - force: bool, - - #[arg(long)] - /// Disables incoming federation of the room after banning and evicting - /// users - disable_federation: bool, - - /// The room in the format of `!roomid:example.com` or a room alias in - /// the format of `#roomalias:example.com` - room: Box, - }, - - /// - Bans a list of rooms (room IDs and room aliases) from a newline - /// delimited codeblock similar to `user deactivate-all` - BanListOfRooms { - #[arg(short, long)] - /// Evicts admins out of the room and ignores any potential errors when - /// making our local users leave the room - force: bool, - - #[arg(long)] - /// Disables incoming federation of the room after banning and evicting - /// users - disable_federation: bool, - }, - - /// - Unbans a room to allow local users to join again - /// - /// To re-enable incoming federation of the room, use --enable-federation - UnbanRoom { - #[arg(long)] - /// Enables incoming federation of the room after unbanning - enable_federation: bool, - - /// The room in the format of `!roomid:example.com` or a room alias in - /// the format of `#roomalias:example.com` - room: Box, - }, - - /// - List of all rooms we have banned - ListBannedRooms, -} - -pub(super) async fn process(command: RoomCommand, body: Vec<&str>) -> Result { - Ok(match command { - RoomCommand::Info(command) => room_info_commands::process(command, body).await?, - - RoomCommand::Alias(command) => room_alias_commands::process(command, body).await?, - - RoomCommand::Directory(command) => room_directory_commands::process(command, body).await?, - - RoomCommand::Moderation(command) => room_moderation_commands::process(command, body).await?, - - RoomCommand::List { - page, - exclude_disabled, - exclude_banned, - } => list(body, page, exclude_disabled, exclude_banned).await?, - }) -} diff --git a/src/admin/room/room_moderation_commands.rs b/src/admin/room/moderation.rs similarity index 72% rename from src/admin/room/room_moderation_commands.rs rename to src/admin/room/moderation.rs index 8ad8295b..86910379 100644 --- a/src/admin/room/room_moderation_commands.rs +++ b/src/admin/room/moderation.rs @@ -1,37 +1,77 @@ use api::client::leave_room; +use clap::Subcommand; use conduit::{debug, error, info, warn, Result}; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId}; -use super::RoomModerationCommand; -use crate::{get_room_info, services}; +use crate::{admin_command, admin_command_dispatch, get_room_info}; -pub(super) async fn process(command: RoomModerationCommand, body: Vec<&str>) -> Result { - match command { - RoomModerationCommand::BanRoom { - force, - room, - disable_federation, - } => ban_room(body, force, room, disable_federation).await, - RoomModerationCommand::BanListOfRooms { - force, - disable_federation, - } => ban_list_of_rooms(body, force, disable_federation).await, - RoomModerationCommand::UnbanRoom { - room, - enable_federation, - } => unban_room(body, room, enable_federation).await, - RoomModerationCommand::ListBannedRooms => list_banned_rooms(body).await, - } +#[admin_command_dispatch] +#[derive(Debug, Subcommand)] +pub(crate) enum RoomModerationCommand { + /// - Bans a room from local users joining and evicts all our local users + /// from the room. Also blocks any invites (local and remote) for the + /// banned room. + /// + /// Server admins (users in the conduwuit admin room) will not be evicted + /// and server admins can still join the room. To evict admins too, use + /// --force (also ignores errors) To disable incoming federation of the + /// room, use --disable-federation + BanRoom { + #[arg(short, long)] + /// Evicts admins out of the room and ignores any potential errors when + /// making our local users leave the room + force: bool, + + #[arg(long)] + /// Disables incoming federation of the room after banning and evicting + /// users + disable_federation: bool, + + /// The room in the format of `!roomid:example.com` or a room alias in + /// the format of `#roomalias:example.com` + room: Box, + }, + + /// - Bans a list of rooms (room IDs and room aliases) from a newline + /// delimited codeblock similar to `user deactivate-all` + BanListOfRooms { + #[arg(short, long)] + /// Evicts admins out of the room and ignores any potential errors when + /// making our local users leave the room + force: bool, + + #[arg(long)] + /// Disables incoming federation of the room after banning and evicting + /// users + disable_federation: bool, + }, + + /// - Unbans a room to allow local users to join again + /// + /// To re-enable incoming federation of the room, use --enable-federation + UnbanRoom { + #[arg(long)] + /// Enables incoming federation of the room after unbanning + enable_federation: bool, + + /// The room in the format of `!roomid:example.com` or a room alias in + /// the format of `#roomalias:example.com` + room: Box, + }, + + /// - List of all rooms we have banned + ListBannedRooms, } +#[admin_command] async fn ban_room( - _body: Vec<&str>, force: bool, room: Box, disable_federation: bool, + &self, force: bool, disable_federation: bool, room: Box, ) -> Result { debug!("Got room alias or ID: {}", room); - let admin_room_alias = &services().globals.admin_alias; + let admin_room_alias = &self.services.globals.admin_alias; - if let Some(admin_room_id) = services().admin.get_admin_room()? { + if let Some(admin_room_id) = self.services.admin.get_admin_room()? { if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) { return Ok(RoomMessageEventContent::text_plain("Not allowed to ban the admin room.")); } @@ -50,7 +90,7 @@ async fn ban_room( debug!("Room specified is a room ID, banning room ID"); - services().rooms.metadata.ban_room(&room_id, true)?; + self.services.rooms.metadata.ban_room(&room_id, true)?; room_id } else if room.is_room_alias_id() { @@ -69,12 +109,13 @@ async fn ban_room( get_alias_helper to fetch room ID remotely" ); - let room_id = if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { + let room_id = if let Some(room_id) = self.services.rooms.alias.resolve_local_alias(&room_alias)? { room_id } else { debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); - match services() + match self + .services .rooms .alias .resolve_alias(&room_alias, None) @@ -92,7 +133,7 @@ async fn ban_room( } }; - services().rooms.metadata.ban_room(&room_id, true)?; + self.services.rooms.metadata.ban_room(&room_id, true)?; room_id } else { @@ -104,20 +145,21 @@ async fn ban_room( debug!("Making all users leave the room {}", &room); if force { - for local_user in services() + for local_user in self + .services .rooms .state_cache .room_members(&room_id) .filter_map(|user| { user.ok().filter(|local_user| { - services().globals.user_is_local(local_user) + self.services.globals.user_is_local(local_user) // additional wrapped check here is to avoid adding remote users // who are in the admin room to the list of local users (would // fail auth check) - && (services().globals.user_is_local(local_user) + && (self.services.globals.user_is_local(local_user) // since this is a force operation, assume user is an admin // if somehow this fails - && services() + && self.services .users .is_admin(local_user) .unwrap_or(true)) @@ -128,30 +170,31 @@ async fn ban_room( &local_user, &room_id ); - if let Err(e) = leave_room(services(), &local_user, &room_id, None).await { + if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await { warn!(%e, "Failed to leave room"); } } } else { - for local_user in services() + for local_user in self + .services .rooms .state_cache .room_members(&room_id) .filter_map(|user| { user.ok().filter(|local_user| { - local_user.server_name() == services().globals.server_name() + local_user.server_name() == self.services.globals.server_name() // additional wrapped check here is to avoid adding remote users // who are in the admin room to the list of local users (would fail auth check) && (local_user.server_name() - == services().globals.server_name() - && !services() + == self.services.globals.server_name() + && !self.services .users .is_admin(local_user) .unwrap_or(false)) }) }) { debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(services(), &local_user, &room_id, None).await { + if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await { error!( "Error attempting to make local user {} leave room {} during room banning: {}", &local_user, &room_id, e @@ -166,7 +209,7 @@ async fn ban_room( } if disable_federation { - services().rooms.metadata.disable_room(&room_id, true)?; + self.services.rooms.metadata.disable_room(&room_id, true)?; return Ok(RoomMessageEventContent::text_plain( "Room banned, removed all our local users, and disabled incoming federation with room.", )); @@ -178,19 +221,22 @@ async fn ban_room( )) } -async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: bool) -> Result { - if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { +#[admin_command] +async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Result { + if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", )); } - let rooms_s = body - .clone() - .drain(1..body.len().saturating_sub(1)) + let rooms_s = self + .body + .to_vec() + .drain(1..self.body.len().saturating_sub(1)) .collect::>(); - let admin_room_alias = &services().globals.admin_alias; + let admin_room_alias = &self.services.globals.admin_alias; let mut room_ban_count: usize = 0; let mut room_ids: Vec = Vec::new(); @@ -198,7 +244,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo for &room in &rooms_s { match <&RoomOrAliasId>::try_from(room) { Ok(room_alias_or_id) => { - if let Some(admin_room_id) = services().admin.get_admin_room()? { + if let Some(admin_room_id) = self.services.admin.get_admin_room()? { if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias) { info!("User specified admin room in bulk ban list, ignoring"); continue; @@ -231,7 +277,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo match RoomAliasId::parse(room_alias_or_id) { Ok(room_alias) => { let room_id = - if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { + if let Some(room_id) = self.services.rooms.alias.resolve_local_alias(&room_alias)? { room_id } else { debug!( @@ -239,7 +285,8 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo ID over federation" ); - match services() + match self + .services .rooms .alias .resolve_alias(&room_alias, None) @@ -303,28 +350,35 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo } for room_id in room_ids { - if services().rooms.metadata.ban_room(&room_id, true).is_ok() { + if self + .services + .rooms + .metadata + .ban_room(&room_id, true) + .is_ok() + { debug!("Banned {room_id} successfully"); room_ban_count = room_ban_count.saturating_add(1); } debug!("Making all users leave the room {}", &room_id); if force { - for local_user in services() + for local_user in self + .services .rooms .state_cache .room_members(&room_id) .filter_map(|user| { user.ok().filter(|local_user| { - local_user.server_name() == services().globals.server_name() + local_user.server_name() == self.services.globals.server_name() // additional wrapped check here is to avoid adding remote // users who are in the admin room to the list of local // users (would fail auth check) && (local_user.server_name() - == services().globals.server_name() + == self.services.globals.server_name() // since this is a force operation, assume user is an // admin if somehow this fails - && services() + && self.services .users .is_admin(local_user) .unwrap_or(true)) @@ -334,31 +388,32 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", &local_user, room_id ); - if let Err(e) = leave_room(services(), &local_user, &room_id, None).await { + if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await { warn!(%e, "Failed to leave room"); } } } else { - for local_user in services() + for local_user in self + .services .rooms .state_cache .room_members(&room_id) .filter_map(|user| { user.ok().filter(|local_user| { - local_user.server_name() == services().globals.server_name() + local_user.server_name() == self.services.globals.server_name() // additional wrapped check here is to avoid adding remote // users who are in the admin room to the list of local // users (would fail auth check) && (local_user.server_name() - == services().globals.server_name() - && !services() + == self.services.globals.server_name() + && !self.services .users .is_admin(local_user) .unwrap_or(false)) }) }) { debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(services(), &local_user, &room_id, None).await { + if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await { error!( "Error attempting to make local user {} leave room {} during bulk room banning: {}", &local_user, &room_id, e @@ -374,7 +429,7 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo } if disable_federation { - services().rooms.metadata.disable_room(&room_id, true)?; + self.services.rooms.metadata.disable_room(&room_id, true)?; } } @@ -390,9 +445,8 @@ async fn ban_list_of_rooms(body: Vec<&str>, force: bool, disable_federation: boo } } -async fn unban_room( - _body: Vec<&str>, room: Box, enable_federation: bool, -) -> Result { +#[admin_command] +async fn unban_room(&self, enable_federation: bool, room: Box) -> Result { let room_id = if room.is_room_id() { let room_id = match RoomId::parse(&room) { Ok(room_id) => room_id, @@ -406,7 +460,7 @@ async fn unban_room( debug!("Room specified is a room ID, unbanning room ID"); - services().rooms.metadata.ban_room(&room_id, false)?; + self.services.rooms.metadata.ban_room(&room_id, false)?; room_id } else if room.is_room_alias_id() { @@ -425,12 +479,13 @@ async fn unban_room( get_alias_helper to fetch room ID remotely" ); - let room_id = if let Some(room_id) = services().rooms.alias.resolve_local_alias(&room_alias)? { + let room_id = if let Some(room_id) = self.services.rooms.alias.resolve_local_alias(&room_alias)? { room_id } else { debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); - match services() + match self + .services .rooms .alias .resolve_alias(&room_alias, None) @@ -448,7 +503,7 @@ async fn unban_room( } }; - services().rooms.metadata.ban_room(&room_id, false)?; + self.services.rooms.metadata.ban_room(&room_id, false)?; room_id } else { @@ -459,7 +514,7 @@ async fn unban_room( }; if enable_federation { - services().rooms.metadata.disable_room(&room_id, false)?; + self.services.rooms.metadata.disable_room(&room_id, false)?; return Ok(RoomMessageEventContent::text_plain("Room unbanned.")); } @@ -469,8 +524,10 @@ async fn unban_room( )) } -async fn list_banned_rooms(_body: Vec<&str>) -> Result { - let rooms = services() +#[admin_command] +async fn list_banned_rooms(&self) -> Result { + let rooms = self + .services .rooms .metadata .list_banned_rooms() @@ -484,7 +541,7 @@ async fn list_banned_rooms(_body: Vec<&str>) -> Result let mut rooms = room_ids .into_iter() - .map(|room_id| get_room_info(services(), &room_id)) + .map(|room_id| get_room_info(self.services, &room_id)) .collect::>(); rooms.sort_by_key(|r| r.1); rooms.reverse(); diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index c30f8f87..34b0f0a0 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,12 +1,14 @@ -use std::fmt::Write; +use std::{fmt::Write, sync::Arc}; use conduit::{info, utils::time, warn, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; -use crate::services; +use crate::admin_command; -pub(super) async fn uptime(_body: Vec<&str>) -> Result { - let elapsed = services() +#[admin_command] +pub(super) async fn uptime(&self) -> Result { + let elapsed = self + .services .server .started .elapsed() @@ -16,13 +18,15 @@ pub(super) async fn uptime(_body: Vec<&str>) -> Result Ok(RoomMessageEventContent::notice_plain(format!("{result}."))) } -pub(super) async fn show_config(_body: Vec<&str>) -> Result { +#[admin_command] +pub(super) async fn show_config(&self) -> Result { // Construct and send the response - Ok(RoomMessageEventContent::text_plain(format!("{}", services().globals.config))) + Ok(RoomMessageEventContent::text_plain(format!("{}", self.services.globals.config))) } +#[admin_command] pub(super) async fn list_features( - _body: Vec<&str>, available: bool, enabled: bool, comma: bool, + &self, available: bool, enabled: bool, comma: bool, ) -> Result { let delim = if comma { "," @@ -62,9 +66,10 @@ pub(super) async fn list_features( Ok(RoomMessageEventContent::text_markdown(features)) } -pub(super) async fn memory_usage(_body: Vec<&str>) -> Result { - let services_usage = services().memory_usage().await?; - let database_usage = services().db.db.memory_usage()?; +#[admin_command] +pub(super) async fn memory_usage(&self) -> Result { + let services_usage = self.services.memory_usage().await?; + let database_usage = self.services.db.db.memory_usage()?; let allocator_usage = conduit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}")); Ok(RoomMessageEventContent::text_plain(format!( @@ -72,14 +77,16 @@ pub(super) async fn memory_usage(_body: Vec<&str>) -> Result) -> Result { - services().clear_cache().await; +#[admin_command] +pub(super) async fn clear_caches(&self) -> Result { + self.services.clear_cache().await; Ok(RoomMessageEventContent::text_plain("Done.")) } -pub(super) async fn list_backups(_body: Vec<&str>) -> Result { - let result = services().globals.db.backup_list()?; +#[admin_command] +pub(super) async fn list_backups(&self) -> Result { + let result = self.services.globals.db.backup_list()?; if result.is_empty() { Ok(RoomMessageEventContent::text_plain("No backups found.")) @@ -88,46 +95,51 @@ pub(super) async fn list_backups(_body: Vec<&str>) -> Result) -> Result { - let mut result = services() +#[admin_command] +pub(super) async fn backup_database(&self) -> Result { + let globals = Arc::clone(&self.services.globals); + let mut result = self + .services .server .runtime() - .spawn_blocking(move || match services().globals.db.backup() { + .spawn_blocking(move || match globals.db.backup() { Ok(()) => String::new(), Err(e) => (*e).to_string(), }) - .await - .unwrap(); + .await?; if result.is_empty() { - result = services().globals.db.backup_list()?; + result = self.services.globals.db.backup_list()?; } - Ok(RoomMessageEventContent::text_plain(&result)) -} - -pub(super) async fn list_database_files(_body: Vec<&str>) -> Result { - let result = services().globals.db.file_list()?; - Ok(RoomMessageEventContent::notice_markdown(result)) } -pub(super) async fn admin_notice(_body: Vec<&str>, message: Vec) -> Result { +#[admin_command] +pub(super) async fn list_database_files(&self) -> Result { + let result = self.services.globals.db.file_list()?; + + Ok(RoomMessageEventContent::notice_markdown(result)) +} + +#[admin_command] +pub(super) async fn admin_notice(&self, message: Vec) -> Result { let message = message.join(" "); - services().admin.send_text(&message).await; + self.services.admin.send_text(&message).await; Ok(RoomMessageEventContent::notice_plain("Notice was sent to #admins")) } -#[cfg(conduit_mods)] -pub(super) async fn reload(_body: Vec<&str>) -> Result { - services().server.reload()?; +#[admin_command] +pub(super) async fn reload_mods(&self) -> Result { + self.services.server.reload()?; Ok(RoomMessageEventContent::notice_plain("Reloading server...")) } +#[admin_command] #[cfg(unix)] -pub(super) async fn restart(_body: Vec<&str>, force: bool) -> Result { +pub(super) async fn restart(&self, force: bool) -> Result { use conduit::utils::sys::current_exe_deleted; if !force && current_exe_deleted() { @@ -137,14 +149,15 @@ pub(super) async fn restart(_body: Vec<&str>, force: bool) -> Result) -> Result { +#[admin_command] +pub(super) async fn shutdown(&self) -> Result { warn!("shutdown command"); - services().server.shutdown()?; + self.services.server.shutdown()?; Ok(RoomMessageEventContent::notice_plain("Shutting down server...")) } diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 958cc54b..222c537a 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -2,10 +2,10 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::events::room::message::RoomMessageEventContent; -use self::commands::*; +use crate::admin_command_dispatch; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] pub(super) enum ServerCommand { /// - Time elapsed since startup @@ -47,9 +47,9 @@ pub(super) enum ServerCommand { message: Vec, }, - #[cfg(conduit_mods)] /// - Hot-reload the server - Reload, + #[clap(alias = "reload")] + ReloadMods, #[cfg(unix)] /// - Restart the server @@ -61,30 +61,3 @@ pub(super) enum ServerCommand { /// - Shutdown the server Shutdown, } - -pub(super) async fn process(command: ServerCommand, body: Vec<&str>) -> Result { - Ok(match command { - ServerCommand::Uptime => uptime(body).await?, - ServerCommand::ShowConfig => show_config(body).await?, - ServerCommand::ListFeatures { - available, - enabled, - comma, - } => list_features(body, available, enabled, comma).await?, - ServerCommand::MemoryUsage => memory_usage(body).await?, - ServerCommand::ClearCaches => clear_caches(body).await?, - ServerCommand::ListBackups => list_backups(body).await?, - ServerCommand::BackupDatabase => backup_database(body).await?, - ServerCommand::ListDatabaseFiles => list_database_files(body).await?, - ServerCommand::AdminNotice { - message, - } => admin_notice(body, message).await?, - #[cfg(conduit_mods)] - ServerCommand::Reload => reload(body).await?, - #[cfg(unix)] - ServerCommand::Restart { - force, - } => restart(body, force).await?, - ServerCommand::Shutdown => shutdown(body).await?, - }) -} diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 69019d79..bdd35d59 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{join_room_by_id_helper, leave_all_rooms, update_avatar_url, update_displayname}; -use conduit::{utils, Result}; +use conduit::{error, info, utils, warn, Result}; use ruma::{ events::{ room::message::RoomMessageEventContent, @@ -10,17 +10,17 @@ use ruma::{ }, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, }; -use tracing::{error, info, warn}; use crate::{ - escape_html, get_room_info, services, + admin_command, escape_html, get_room_info, utils::{parse_active_local_user_id, parse_local_user_id}, }; const AUTO_GEN_PASSWORD_LENGTH: usize = 25; -pub(super) async fn list(_body: Vec<&str>) -> Result { - match services().users.list_local_users() { +#[admin_command] +pub(super) async fn list_users(&self) -> Result { + match self.services.users.list_local_users() { Ok(users) => { let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); plain_msg += users.join("\n").as_str(); @@ -32,13 +32,12 @@ pub(super) async fn list(_body: Vec<&str>) -> Result { } } -pub(super) async fn create( - _body: Vec<&str>, username: String, password: Option, -) -> Result { +#[admin_command] +pub(super) async fn create_user(&self, username: String, password: Option) -> Result { // Validate user id - let user_id = parse_local_user_id(services(), &username)?; + let user_id = parse_local_user_id(self.services, &username)?; - if services().users.exists(&user_id)? { + if self.services.users.exists(&user_id)? { return Ok(RoomMessageEventContent::text_plain(format!("Userid {user_id} already exists"))); } @@ -51,30 +50,33 @@ pub(super) async fn create( let password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); // Create user - services().users.create(&user_id, Some(password.as_str()))?; + self.services + .users + .create(&user_id, Some(password.as_str()))?; // Default to pretty displayname let mut displayname = user_id.localpart().to_owned(); // If `new_user_displayname_suffix` is set, registration will push whatever // content is set to the user's display name with a space before it - if !services() + if !self + .services .globals .config .new_user_displayname_suffix .is_empty() { - write!(displayname, " {}", services().globals.config.new_user_displayname_suffix) + write!(displayname, " {}", self.services.globals.config.new_user_displayname_suffix) .expect("should be able to write to string buffer"); } - services() + self.services .users .set_displayname(&user_id, Some(displayname)) .await?; // Initial account data - services().account_data.update( + self.services.account_data.update( None, &user_id, ruma::events::GlobalAccountDataEventType::PushRules @@ -88,12 +90,13 @@ pub(super) async fn create( .expect("to json value always works"), )?; - if !services().globals.config.auto_join_rooms.is_empty() { - for room in &services().globals.config.auto_join_rooms { - if !services() + if !self.services.globals.config.auto_join_rooms.is_empty() { + for room in &self.services.globals.config.auto_join_rooms { + if !self + .services .rooms .state_cache - .server_in_room(services().globals.server_name(), room)? + .server_in_room(self.services.globals.server_name(), room)? { warn!("Skipping room {room} to automatically join as we have never joined before."); continue; @@ -101,11 +104,11 @@ pub(super) async fn create( if let Some(room_id_server_name) = room.server_name() { match join_room_by_id_helper( - services(), + self.services, &user_id, room, Some("Automatically joining this room upon registration".to_owned()), - &[room_id_server_name.to_owned(), services().globals.server_name().to_owned()], + &[room_id_server_name.to_owned(), self.services.globals.server_name().to_owned()], None, ) .await @@ -130,38 +133,38 @@ pub(super) async fn create( ))) } -pub(super) async fn deactivate( - _body: Vec<&str>, no_leave_rooms: bool, user_id: String, -) -> Result { +#[admin_command] +pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> Result { // Validate user id - let user_id = parse_local_user_id(services(), &user_id)?; + let user_id = parse_local_user_id(self.services, &user_id)?; // don't deactivate the server service account - if user_id == services().globals.server_user { + if user_id == self.services.globals.server_user { return Ok(RoomMessageEventContent::text_plain( "Not allowed to deactivate the server service account.", )); } - services().users.deactivate_account(&user_id)?; + self.services.users.deactivate_account(&user_id)?; if !no_leave_rooms { - services() + self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( "Making {user_id} leave all rooms after deactivation..." ))) .await; - let all_joined_rooms: Vec = services() + let all_joined_rooms: Vec = self + .services .rooms .state_cache .rooms_joined(&user_id) .filter_map(Result::ok) .collect(); - update_displayname(services(), user_id.clone(), None, all_joined_rooms.clone()).await?; - update_avatar_url(services(), user_id.clone(), None, None, all_joined_rooms).await?; - leave_all_rooms(services(), &user_id).await; + update_displayname(self.services, user_id.clone(), None, all_joined_rooms.clone()).await?; + update_avatar_url(self.services, user_id.clone(), None, None, all_joined_rooms).await?; + leave_all_rooms(self.services, &user_id).await; } Ok(RoomMessageEventContent::text_plain(format!( @@ -169,10 +172,11 @@ pub(super) async fn deactivate( ))) } -pub(super) async fn reset_password(_body: Vec<&str>, username: String) -> Result { - let user_id = parse_local_user_id(services(), &username)?; +#[admin_command] +pub(super) async fn reset_password(&self, username: String) -> Result { + let user_id = parse_local_user_id(self.services, &username)?; - if user_id == services().globals.server_user { + if user_id == self.services.globals.server_user { return Ok(RoomMessageEventContent::text_plain( "Not allowed to set the password for the server account. Please use the emergency password config option.", )); @@ -180,7 +184,8 @@ pub(super) async fn reset_password(_body: Vec<&str>, username: String) -> Result let new_password = utils::random_string(AUTO_GEN_PASSWORD_LENGTH); - match services() + match self + .services .users .set_password(&user_id, Some(new_password.as_str())) { @@ -193,28 +198,29 @@ pub(super) async fn reset_password(_body: Vec<&str>, username: String) -> Result } } -pub(super) async fn deactivate_all( - body: Vec<&str>, no_leave_rooms: bool, force: bool, -) -> Result { - if body.len() < 2 || !body[0].trim().starts_with("```") || body.last().unwrap_or(&"").trim() != "```" { +#[admin_command] +pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> Result { + if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", )); } - let usernames = body - .clone() - .drain(1..body.len().saturating_sub(1)) + let usernames = self + .body + .to_vec() + .drain(1..self.body.len().saturating_sub(1)) .collect::>(); let mut user_ids: Vec = Vec::with_capacity(usernames.len()); let mut admins = Vec::new(); for username in usernames { - match parse_active_local_user_id(services(), username) { + match parse_active_local_user_id(self.services, username) { Ok(user_id) => { - if services().users.is_admin(&user_id)? && !force { - services() + if self.services.users.is_admin(&user_id)? && !force { + self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( "{username} is an admin and --force is not set, skipping over" @@ -225,8 +231,8 @@ pub(super) async fn deactivate_all( } // don't deactivate the server service account - if user_id == services().globals.server_user { - services() + if user_id == self.services.globals.server_user { + self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( "{username} is the server service account, skipping over" @@ -238,7 +244,7 @@ pub(super) async fn deactivate_all( user_ids.push(user_id); }, Err(e) => { - services() + self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( "{username} is not a valid username, skipping over: {e}" @@ -252,24 +258,25 @@ pub(super) async fn deactivate_all( let mut deactivation_count: usize = 0; for user_id in user_ids { - match services().users.deactivate_account(&user_id) { + match self.services.users.deactivate_account(&user_id) { Ok(()) => { deactivation_count = deactivation_count.saturating_add(1); if !no_leave_rooms { info!("Forcing user {user_id} to leave all rooms apart of deactivate-all"); - let all_joined_rooms: Vec = services() + let all_joined_rooms: Vec = self + .services .rooms .state_cache .rooms_joined(&user_id) .filter_map(Result::ok) .collect(); - update_displayname(services(), user_id.clone(), None, all_joined_rooms.clone()).await?; - update_avatar_url(services(), user_id.clone(), None, None, all_joined_rooms).await?; - leave_all_rooms(services(), &user_id).await; + update_displayname(self.services, user_id.clone(), None, all_joined_rooms.clone()).await?; + update_avatar_url(self.services, user_id.clone(), None, None, all_joined_rooms).await?; + leave_all_rooms(self.services, &user_id).await; } }, Err(e) => { - services() + self.services .admin .send_message(RoomMessageEventContent::text_plain(format!("Failed deactivating user: {e}"))) .await; @@ -290,16 +297,18 @@ pub(super) async fn deactivate_all( } } -pub(super) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Result { +#[admin_command] +pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result { // Validate user id - let user_id = parse_local_user_id(services(), &user_id)?; + let user_id = parse_local_user_id(self.services, &user_id)?; - let mut rooms: Vec<(OwnedRoomId, u64, String)> = services() + let mut rooms: Vec<(OwnedRoomId, u64, String)> = self + .services .rooms .state_cache .rooms_joined(&user_id) .filter_map(Result::ok) - .map(|room_id| get_room_info(services(), &room_id)) + .map(|room_id| get_room_info(self.services, &room_id)) .collect(); if rooms.is_empty() { @@ -341,35 +350,38 @@ pub(super) async fn list_joined_rooms(_body: Vec<&str>, user_id: String) -> Resu Ok(RoomMessageEventContent::text_html(output_plain, output_html)) } +#[admin_command] pub(super) async fn force_join_room( - _body: Vec<&str>, user_id: String, room_id: OwnedRoomOrAliasId, + &self, user_id: String, room_id: OwnedRoomOrAliasId, ) -> Result { - let user_id = parse_local_user_id(services(), &user_id)?; - let room_id = services().rooms.alias.resolve(&room_id).await?; + let user_id = parse_local_user_id(self.services, &user_id)?; + let room_id = self.services.rooms.alias.resolve(&room_id).await?; assert!( - services().globals.user_is_local(&user_id), + self.services.globals.user_is_local(&user_id), "Parsed user_id must be a local user" ); - join_room_by_id_helper(services(), &user_id, &room_id, None, &[], None).await?; + join_room_by_id_helper(self.services, &user_id, &room_id, None, &[], None).await?; Ok(RoomMessageEventContent::notice_markdown(format!( "{user_id} has been joined to {room_id}.", ))) } -pub(super) async fn make_user_admin(_body: Vec<&str>, user_id: String) -> Result { - let user_id = parse_local_user_id(services(), &user_id)?; - let displayname = services() +#[admin_command] +pub(super) async fn make_user_admin(&self, user_id: String) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + let displayname = self + .services .users .displayname(&user_id)? .unwrap_or_else(|| user_id.to_string()); assert!( - services().globals.user_is_local(&user_id), + self.services.globals.user_is_local(&user_id), "Parsed user_id must be a local user" ); - services() + self.services .admin .make_user_admin(&user_id, displayname) .await?; @@ -379,12 +391,14 @@ pub(super) async fn make_user_admin(_body: Vec<&str>, user_id: String) -> Result ))) } +#[admin_command] pub(super) async fn put_room_tag( - _body: Vec<&str>, user_id: String, room_id: Box, tag: String, + &self, user_id: String, room_id: Box, tag: String, ) -> Result { - let user_id = parse_active_local_user_id(services(), &user_id)?; + let user_id = parse_active_local_user_id(self.services, &user_id)?; - let event = services() + let event = self + .services .account_data .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?; @@ -402,7 +416,7 @@ pub(super) async fn put_room_tag( .tags .insert(tag.clone().into(), TagInfo::new()); - services().account_data.update( + self.services.account_data.update( Some(&room_id), &user_id, RoomAccountDataEventType::Tag, @@ -414,12 +428,14 @@ pub(super) async fn put_room_tag( ))) } +#[admin_command] pub(super) async fn delete_room_tag( - _body: Vec<&str>, user_id: String, room_id: Box, tag: String, + &self, user_id: String, room_id: Box, tag: String, ) -> Result { - let user_id = parse_active_local_user_id(services(), &user_id)?; + let user_id = parse_active_local_user_id(self.services, &user_id)?; - let event = services() + let event = self + .services .account_data .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?; @@ -434,7 +450,7 @@ pub(super) async fn delete_room_tag( tags_event.content.tags.remove(&tag.clone().into()); - services().account_data.update( + self.services.account_data.update( Some(&room_id), &user_id, RoomAccountDataEventType::Tag, @@ -446,12 +462,12 @@ pub(super) async fn delete_room_tag( ))) } -pub(super) async fn get_room_tags( - _body: Vec<&str>, user_id: String, room_id: Box, -) -> Result { - let user_id = parse_active_local_user_id(services(), &user_id)?; +#[admin_command] +pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box) -> Result { + let user_id = parse_active_local_user_id(self.services, &user_id)?; - let event = services() + let event = self + .services .account_data .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?; diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 1b92d668..b0c0bd1e 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -2,14 +2,16 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomOrAliasId, RoomId}; +use ruma::{OwnedRoomOrAliasId, RoomId}; -use self::commands::*; +use crate::admin_command_dispatch; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] pub(super) enum UserCommand { /// - Create a new user - Create { + #[clap(alias = "create")] + CreateUser { /// Username of the new user username: String, /// Password of the new user, if unspecified one is generated @@ -56,7 +58,8 @@ pub(super) enum UserCommand { }, /// - List local users in the database - List, + #[clap(alias = "list")] + ListUsers, /// - Lists all the rooms (local and remote) that the specified user is /// joined in @@ -101,48 +104,3 @@ pub(super) enum UserCommand { room_id: Box, }, } - -pub(super) async fn process(command: UserCommand, body: Vec<&str>) -> Result { - Ok(match command { - UserCommand::List => list(body).await?, - UserCommand::Create { - username, - password, - } => create(body, username, password).await?, - UserCommand::Deactivate { - no_leave_rooms, - user_id, - } => deactivate(body, no_leave_rooms, user_id).await?, - UserCommand::ResetPassword { - username, - } => reset_password(body, username).await?, - UserCommand::DeactivateAll { - no_leave_rooms, - force, - } => deactivate_all(body, no_leave_rooms, force).await?, - UserCommand::ListJoinedRooms { - user_id, - } => list_joined_rooms(body, user_id).await?, - UserCommand::ForceJoinRoom { - user_id, - room_id, - } => force_join_room(body, user_id, room_id).await?, - UserCommand::MakeUserAdmin { - user_id, - } => make_user_admin(body, user_id).await?, - UserCommand::PutRoomTag { - user_id, - room_id, - tag, - } => put_room_tag(body, user_id, room_id, tag).await?, - UserCommand::DeleteRoomTag { - user_id, - room_id, - tag, - } => delete_room_tag(body, user_id, room_id, tag).await?, - UserCommand::GetRoomTags { - user_id, - room_id, - } => get_room_tags(body, user_id, room_id).await?, - }) -} diff --git a/src/macros/admin.rs b/src/macros/admin.rs index 4189d64f..d4ce7ad5 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -2,15 +2,27 @@ use itertools::Itertools; use proc_macro::{Span, TokenStream}; use proc_macro2::TokenStream as TokenStream2; use quote::{quote, ToTokens}; -use syn::{Error, Fields, Ident, ItemEnum, Meta, Variant}; +use syn::{parse_quote, Attribute, Error, Fields, Ident, ItemEnum, ItemFn, Meta, Variant}; use crate::{utils::camel_to_snake_string, Result}; +pub(super) fn command(mut item: ItemFn, _args: &[Meta]) -> Result { + let attr: Attribute = parse_quote! { + #[conduit_macros::implement(crate::Command, params = "<'_>")] + }; + + item.attrs.push(attr); + Ok(item.into_token_stream().into()) +} + pub(super) fn command_dispatch(item: ItemEnum, _args: &[Meta]) -> Result { let name = &item.ident; let arm: Vec = item.variants.iter().map(dispatch_arm).try_collect()?; let switch = quote! { - pub(super) async fn process(command: #name, body: Vec<&str>) -> Result { + pub(super) async fn process( + command: #name, + context: &crate::Command<'_> + ) -> Result { use #name::*; #[allow(non_snake_case)] Ok(match command { @@ -34,7 +46,7 @@ fn dispatch_arm(v: &Variant) -> Result { let field = fields.named.iter().filter_map(|f| f.ident.as_ref()); let arg = field.clone(); quote! { - #name { #( #field ),* } => Box::pin(#handler(&body, #( #arg ),*)).await?, + #name { #( #field ),* } => Box::pin(context.#handler(#( #arg ),*)).await?, } }, Fields::Unnamed(fields) => { @@ -42,12 +54,12 @@ fn dispatch_arm(v: &Variant) -> Result { return Err(Error::new(Span::call_site().into(), "One unnamed field required")); }; quote! { - #name ( #field ) => Box::pin(#handler::process(#field, body)).await?, + #name ( #field ) => Box::pin(#handler::process(#field, context)).await?, } }, Fields::Unit => { quote! { - #name => Box::pin(#handler(&body)).await?, + #name => Box::pin(context.#handler()).await?, } }, }; diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 1a5494bb..d32cda71 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -14,6 +14,11 @@ use syn::{ pub(crate) type Result = std::result::Result; +#[proc_macro_attribute] +pub fn admin_command(args: TokenStream, input: TokenStream) -> TokenStream { + attribute_macro::(args, input, admin::command) +} + #[proc_macro_attribute] pub fn admin_command_dispatch(args: TokenStream, input: TokenStream) -> TokenStream { attribute_macro::(args, input, admin::command_dispatch) From 2f85a5c1ac5fba02773004db7aa5e855c08364f4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 27 Jul 2024 07:17:07 +0000 Subject: [PATCH 0594/2091] de-global services Signed-off-by: Jason Volk --- src/admin/check/commands.rs | 4 +- src/admin/handler.rs | 275 ++++++++++++++++------------------ src/admin/mod.rs | 22 +-- src/api/client/account.rs | 8 +- src/api/client/alias.rs | 2 +- src/api/client/config.rs | 4 +- src/api/client/directory.rs | 6 +- src/api/client/keys.rs | 4 +- src/api/client/media.rs | 8 +- src/api/client/membership.rs | 16 +- src/api/client/message.rs | 4 +- src/api/client/profile.rs | 4 +- src/api/client/report.rs | 2 +- src/api/client/room.rs | 6 +- src/api/client/state.rs | 2 +- src/api/client/sync.rs | 13 +- src/api/mod.rs | 2 +- src/api/router.rs | 4 +- src/api/router/args.rs | 7 +- src/api/server/make_join.rs | 2 +- src/api/server/publicrooms.rs | 4 +- src/api/server/send.rs | 4 +- src/api/server/send_join.rs | 4 +- src/api/server/send_leave.rs | 4 +- src/api/server/user.rs | 4 +- src/database/database.rs | 6 +- src/main/main.rs | 35 ++++- src/main/server.rs | 6 +- src/router/layers.rs | 20 ++- src/router/mod.rs | 11 +- src/router/router.rs | 16 +- src/router/run.rs | 47 +++--- src/router/serve/mod.rs | 18 ++- src/service/admin/mod.rs | 20 ++- src/service/mod.rs | 53 +------ src/service/services.rs | 16 +- 36 files changed, 327 insertions(+), 336 deletions(-) diff --git a/src/admin/check/commands.rs b/src/admin/check/commands.rs index a757d504..0a983046 100644 --- a/src/admin/check/commands.rs +++ b/src/admin/check/commands.rs @@ -2,7 +2,7 @@ use conduit::Result; use conduit_macros::implement; use ruma::events::room::message::RoomMessageEventContent; -use crate::{services, Command}; +use crate::Command; /// Uses the iterator in `src/database/key_value/users.rs` to iterator over /// every user in our database (remote and local). Reports total count, any @@ -10,7 +10,7 @@ use crate::{services, Command}; #[implement(Command, params = "<'_>")] pub(super) async fn check_all_users(&self) -> Result { let timer = tokio::time::Instant::now(); - let results = services().users.db.iter(); + let results = self.services.users.db.iter(); let query_time = timer.elapsed(); let users = results.collect::>(); diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 32360c85..a7e4d79a 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,4 +1,4 @@ -use std::{panic::AssertUnwindSafe, time::Instant}; +use std::{panic::AssertUnwindSafe, sync::Arc, time::Instant}; use clap::{CommandFactory, Parser}; use conduit::{error, trace, utils::string::common_prefix, Error, Result}; @@ -17,37 +17,27 @@ use service::{ use crate::{admin, admin::AdminCommand, Command}; -struct Handler { - services: &'static Services, -} +#[must_use] +pub(super) fn complete(line: &str) -> String { complete_command(AdminCommand::command(), line) } #[must_use] -pub(super) fn complete(line: &str) -> String { - Handler { - services: service::services(), - } - .complete_command(AdminCommand::command(), line) +pub(super) fn handle(services: Arc, command: CommandInput) -> HandlerResult { + Box::pin(handle_command(services, command)) } -#[must_use] -pub(super) fn handle(command: CommandInput) -> HandlerResult { Box::pin(handle_command(command)) } - #[tracing::instrument(skip_all, name = "admin")] -async fn handle_command(command: CommandInput) -> CommandResult { - AssertUnwindSafe(Box::pin(process_command(&command))) +async fn handle_command(services: Arc, command: CommandInput) -> CommandResult { + AssertUnwindSafe(Box::pin(process_command(services, &command))) .catch_unwind() .await .map_err(Error::from_panic) .or_else(|error| handle_panic(&error, command)) } -async fn process_command(command: &CommandInput) -> CommandOutput { - Handler { - services: service::services(), - } - .process(&command.command) - .await - .and_then(|content| reply(content, command.reply_id.clone())) +async fn process_command(services: Arc, command: &CommandInput) -> CommandOutput { + process(services, &command.command) + .await + .and_then(|content| reply(content, command.reply_id.clone())) } fn handle_panic(error: &Error, command: CommandInput) -> CommandResult { @@ -68,129 +58,126 @@ fn reply(mut content: RoomMessageEventContent, reply_id: Option) - Some(content) } -impl Handler { - // Parse and process a message from the admin room - async fn process(&self, msg: &str) -> CommandOutput { - let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); - let command = lines.next().expect("each string has at least one line"); - let (parsed, body) = match self.parse_command(command) { - Ok(parsed) => parsed, - Err(error) => { - let server_name = self.services.globals.server_name(); - let message = error.replace("server.name", server_name.as_str()); - return Some(RoomMessageEventContent::notice_markdown(message)); - }, - }; +// Parse and process a message from the admin room +async fn process(services: Arc, msg: &str) -> CommandOutput { + let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); + let command = lines.next().expect("each string has at least one line"); + let (parsed, body) = match parse_command(command) { + Ok(parsed) => parsed, + Err(error) => { + let server_name = services.globals.server_name(); + let message = error.replace("server.name", server_name.as_str()); + return Some(RoomMessageEventContent::notice_markdown(message)); + }, + }; - let timer = Instant::now(); - let body: Vec<&str> = body.iter().map(String::as_str).collect(); - let context = Command { - services: self.services, - body: &body, - }; - let result = Box::pin(admin::process(parsed, &context)).await; - let elapsed = timer.elapsed(); - conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}"); - match result { - Ok(reply) => Some(reply), - Err(error) => Some(RoomMessageEventContent::notice_markdown(format!( - "Encountered an error while handling the command:\n```\n{error:#?}\n```" - ))), - } - } - - // Parse chat messages from the admin room into an AdminCommand object - fn parse_command(&self, command_line: &str) -> Result<(AdminCommand, Vec), String> { - let argv = self.parse_line(command_line); - let com = AdminCommand::try_parse_from(&argv).map_err(|error| error.to_string())?; - Ok((com, argv)) - } - - fn complete_command(&self, mut cmd: clap::Command, line: &str) -> String { - let argv = self.parse_line(line); - let mut ret = Vec::::with_capacity(argv.len().saturating_add(1)); - - 'token: for token in argv.into_iter().skip(1) { - let cmd_ = cmd.clone(); - let mut choice = Vec::new(); - - for sub in cmd_.get_subcommands() { - let name = sub.get_name(); - if *name == token { - // token already complete; recurse to subcommand - ret.push(token); - cmd.clone_from(sub); - continue 'token; - } else if name.starts_with(&token) { - // partial match; add to choices - choice.push(name); - } - } - - if choice.len() == 1 { - // One choice. Add extra space because it's complete - let choice = *choice.first().expect("only choice"); - ret.push(choice.to_owned()); - ret.push(String::new()); - } else if choice.is_empty() { - // Nothing found, return original string - ret.push(token); - } else { - // Find the common prefix - ret.push(common_prefix(&choice).into()); - } - - // Return from completion - return ret.join(" "); - } - - // Return from no completion. Needs a space though. - ret.push(String::new()); - ret.join(" ") - } - - // Parse chat messages from the admin room into an AdminCommand object - fn parse_line(&self, command_line: &str) -> Vec { - let mut argv = command_line - .split_whitespace() - .map(str::to_owned) - .collect::>(); - - // Remove any escapes that came with a server-side escape command - if !argv.is_empty() && argv[0].ends_with("admin") { - argv[0] = argv[0].trim_start_matches('\\').into(); - } - - // First indice has to be "admin" but for console convenience we add it here - let server_user = self.services.globals.server_user.as_str(); - if !argv.is_empty() && !argv[0].ends_with("admin") && !argv[0].starts_with(server_user) { - argv.insert(0, "admin".to_owned()); - } - - // Replace `help command` with `command --help` - // Clap has a help subcommand, but it omits the long help description. - if argv.len() > 1 && argv[1] == "help" { - argv.remove(1); - argv.push("--help".to_owned()); - } - - // Backwards compatibility with `register_appservice`-style commands - if argv.len() > 1 && argv[1].contains('_') { - argv[1] = argv[1].replace('_', "-"); - } - - // Backwards compatibility with `register_appservice`-style commands - if argv.len() > 2 && argv[2].contains('_') { - argv[2] = argv[2].replace('_', "-"); - } - - // if the user is using the `query` command (argv[1]), replace the database - // function/table calls with underscores to match the codebase - if argv.len() > 3 && argv[1].eq("query") { - argv[3] = argv[3].replace('_', "-"); - } - - trace!(?command_line, ?argv, "parse"); - argv + let timer = Instant::now(); + let body: Vec<&str> = body.iter().map(String::as_str).collect(); + let context = Command { + services: &services, + body: &body, + }; + let result = Box::pin(admin::process(parsed, &context)).await; + let elapsed = timer.elapsed(); + conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}"); + match result { + Ok(reply) => Some(reply), + Err(error) => Some(RoomMessageEventContent::notice_markdown(format!( + "Encountered an error while handling the command:\n```\n{error:#?}\n```" + ))), } } + +// Parse chat messages from the admin room into an AdminCommand object +fn parse_command(command_line: &str) -> Result<(AdminCommand, Vec), String> { + let argv = parse_line(command_line); + let com = AdminCommand::try_parse_from(&argv).map_err(|error| error.to_string())?; + Ok((com, argv)) +} + +fn complete_command(mut cmd: clap::Command, line: &str) -> String { + let argv = parse_line(line); + let mut ret = Vec::::with_capacity(argv.len().saturating_add(1)); + + 'token: for token in argv.into_iter().skip(1) { + let cmd_ = cmd.clone(); + let mut choice = Vec::new(); + + for sub in cmd_.get_subcommands() { + let name = sub.get_name(); + if *name == token { + // token already complete; recurse to subcommand + ret.push(token); + cmd.clone_from(sub); + continue 'token; + } else if name.starts_with(&token) { + // partial match; add to choices + choice.push(name); + } + } + + if choice.len() == 1 { + // One choice. Add extra space because it's complete + let choice = *choice.first().expect("only choice"); + ret.push(choice.to_owned()); + ret.push(String::new()); + } else if choice.is_empty() { + // Nothing found, return original string + ret.push(token); + } else { + // Find the common prefix + ret.push(common_prefix(&choice).into()); + } + + // Return from completion + return ret.join(" "); + } + + // Return from no completion. Needs a space though. + ret.push(String::new()); + ret.join(" ") +} + +// Parse chat messages from the admin room into an AdminCommand object +fn parse_line(command_line: &str) -> Vec { + let mut argv = command_line + .split_whitespace() + .map(str::to_owned) + .collect::>(); + + // Remove any escapes that came with a server-side escape command + if !argv.is_empty() && argv[0].ends_with("admin") { + argv[0] = argv[0].trim_start_matches('\\').into(); + } + + // First indice has to be "admin" but for console convenience we add it here + if !argv.is_empty() && !argv[0].ends_with("admin") && !argv[0].starts_with('@') { + argv.insert(0, "admin".to_owned()); + } + + // Replace `help command` with `command --help` + // Clap has a help subcommand, but it omits the long help description. + if argv.len() > 1 && argv[1] == "help" { + argv.remove(1); + argv.push("--help".to_owned()); + } + + // Backwards compatibility with `register_appservice`-style commands + if argv.len() > 1 && argv[1].contains('_') { + argv[1] = argv[1].replace('_', "-"); + } + + // Backwards compatibility with `register_appservice`-style commands + if argv.len() > 2 && argv[2].contains('_') { + argv[2] = argv[2].replace('_', "-"); + } + + // if the user is using the `query` command (argv[1]), replace the database + // function/table calls with underscores to match the codebase + if argv.len() > 3 && argv[1].eq("query") { + argv[3] = argv[3].replace('_', "-"); + } + + trace!(?command_line, ?argv, "parse"); + argv +} diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 5d4c8f5e..fb1c02be 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1,4 +1,4 @@ -#![recursion_limit = "168"] +#![recursion_limit = "192"] #![allow(clippy::wildcard_imports)] #![allow(clippy::enum_glob_use)] @@ -24,7 +24,6 @@ extern crate conduit_service as service; pub(crate) use conduit::Result; pub(crate) use conduit_macros::{admin_command, admin_command_dispatch}; -pub(crate) use service::services; pub(crate) use crate::{ command::Command, @@ -38,26 +37,19 @@ conduit::mod_dtor! {} conduit::rustc_flags_capture! {} /// Install the admin command handler -pub async fn init() { - _ = services() - .admin +pub async fn init(admin_service: &service::admin::Service) { + _ = admin_service .complete .write() .expect("locked for writing") .insert(handler::complete); - _ = services() - .admin - .handle - .write() - .await - .insert(handler::handle); + _ = admin_service.handle.write().await.insert(handler::handle); } /// Uninstall the admin command handler -pub async fn fini() { - _ = services().admin.handle.write().await.take(); - _ = services() - .admin +pub async fn fini(admin_service: &service::admin::Service) { + _ = admin_service.handle.write().await.take(); + _ = admin_service .complete .write() .expect("locked for writing") diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 7c2bb0b6..f093c459 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -370,7 +370,7 @@ pub(crate) async fn register_route( if let Some(room_id_server_name) = room.server_name() { if let Err(e) = join_room_by_id_helper( - services, + &services, &user_id, room, Some("Automatically joining this room upon registration".to_owned()), @@ -562,11 +562,11 @@ pub(crate) async fn deactivate_route( .rooms_joined(sender_user) .filter_map(Result::ok) .collect(); - super::update_displayname(services, sender_user.clone(), None, all_joined_rooms.clone()).await?; - super::update_avatar_url(services, sender_user.clone(), None, None, all_joined_rooms).await?; + super::update_displayname(&services, sender_user.clone(), None, all_joined_rooms.clone()).await?; + super::update_avatar_url(&services, sender_user.clone(), None, None, all_joined_rooms).await?; // Make the user leave all rooms before deactivation - super::leave_all_rooms(services, sender_user).await; + super::leave_all_rooms(&services, sender_user).await; info!("User {sender_user} deactivated their account."); services diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index dbc75e64..18d1c5b0 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -107,7 +107,7 @@ pub(crate) async fn get_alias_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Room with alias not found.")); }; - let servers = room_available_servers(services, &room_id, &room_alias, &pre_servers); + let servers = room_available_servers(&services, &room_id, &room_alias, &pre_servers); debug!(?room_alias, ?room_id, "available servers: {servers:?}"); Ok(get_alias::v3::Response::new(room_id, servers)) diff --git a/src/api/client/config.rs b/src/api/client/config.rs index 56d33ba7..61cc97ff 100644 --- a/src/api/client/config.rs +++ b/src/api/client/config.rs @@ -20,7 +20,7 @@ pub(crate) async fn set_global_account_data_route( State(services): State, body: Ruma, ) -> Result { set_account_data( - services, + &services, None, &body.sender_user, &body.event_type.to_string(), @@ -37,7 +37,7 @@ pub(crate) async fn set_room_account_data_route( State(services): State, body: Ruma, ) -> Result { set_account_data( - services, + &services, Some(&body.room_id), &body.sender_user, &body.event_type.to_string(), diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index cb30b60a..6054bd9c 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -48,7 +48,7 @@ pub(crate) async fn get_public_rooms_filtered_route( } let response = get_public_rooms_filtered_helper( - services, + &services, body.server.as_deref(), body.limit, body.since.as_deref(), @@ -88,7 +88,7 @@ pub(crate) async fn get_public_rooms_route( } let response = get_public_rooms_filtered_helper( - services, + &services, body.server.as_deref(), body.limit, body.since.as_deref(), @@ -124,7 +124,7 @@ pub(crate) async fn set_room_visibility_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } - if !user_can_publish_room(services, sender_user, &body.room_id)? { + if !user_can_publish_room(&services, sender_user, &body.room_id)? { return Err(Error::BadRequest( ErrorKind::forbidden(), "User is not allowed to publish this room", diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 8489dde3..6fa7a8ad 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -77,7 +77,7 @@ pub(crate) async fn get_keys_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); get_keys_helper( - services, + &services, Some(sender_user), &body.device_keys, |u| u == sender_user, @@ -92,7 +92,7 @@ pub(crate) async fn get_keys_route( pub(crate) async fn claim_keys_route( State(services): State, body: Ruma, ) -> Result { - claim_keys_helper(services, &body.one_time_keys).await + claim_keys_helper(&services, &body.one_time_keys).await } /// # `POST /_matrix/client/r0/keys/device_signing/upload` diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 78463fc6..f0afa290 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -76,12 +76,12 @@ pub(crate) async fn get_media_preview_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let url = &body.url; - if !url_preview_allowed(services, url) { + if !url_preview_allowed(&services, url) { warn!(%sender_user, "URL is not allowed to be previewed: {url}"); return Err(Error::BadRequest(ErrorKind::forbidden(), "URL is not allowed to be previewed")); } - match get_url_preview(services, url).await { + match get_url_preview(&services, url).await { Ok(preview) => { let res = serde_json::value::to_raw_value(&preview).map_err(|e| { error!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}"); @@ -221,7 +221,7 @@ pub(crate) async fn get_content_route( }) } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { let response = get_remote_content( - services, + &services, &mxc, &body.server_name, body.media_id.clone(), @@ -311,7 +311,7 @@ pub(crate) async fn get_content_as_filename_route( }) } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { match get_remote_content( - services, + &services, &mxc, &body.server_name, body.media_id.clone(), diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index d3b2d8f6..ca7e6b6f 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -167,7 +167,7 @@ pub(crate) async fn join_room_by_id_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); banned_room_check( - services, + &services, sender_user, Some(&body.room_id), body.room_id.server_name(), @@ -202,7 +202,7 @@ pub(crate) async fn join_room_by_id_route( } join_room_by_id_helper( - services, + &services, sender_user, &body.room_id, body.reason.clone(), @@ -231,7 +231,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { Ok(room_id) => { - banned_room_check(services, sender_user, Some(&room_id), room_id.server_name(), client).await?; + banned_room_check(&services, sender_user, Some(&room_id), room_id.server_name(), client).await?; let mut servers = body.server_name.clone(); servers.extend( @@ -270,7 +270,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( .await?; let (room_id, mut pre_servers) = response; - banned_room_check(services, sender_user, Some(&room_id), Some(room_alias.server_name()), client).await?; + banned_room_check(&services, sender_user, Some(&room_id), Some(room_alias.server_name()), client).await?; let mut servers = body.server_name; if let Some(pre_servers) = &mut pre_servers { @@ -303,7 +303,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( }; let join_room_response = join_room_by_id_helper( - services, + &services, sender_user, &room_id, body.reason.clone(), @@ -327,7 +327,7 @@ pub(crate) async fn leave_room_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - leave_room(services, sender_user, &body.room_id, body.reason.clone()).await?; + leave_room(&services, sender_user, &body.room_id, body.reason.clone()).await?; Ok(leave_room::v3::Response::new()) } @@ -353,13 +353,13 @@ pub(crate) async fn invite_user_route( )); } - banned_room_check(services, sender_user, Some(&body.room_id), body.room_id.server_name(), client).await?; + banned_room_check(&services, sender_user, Some(&body.room_id), body.room_id.server_name(), client).await?; if let invite_user::v3::InvitationRecipient::UserId { user_id, } = &body.recipient { - invite_helper(services, sender_user, user_id, &body.room_id, body.reason.clone(), false).await?; + invite_helper(&services, sender_user, user_id, &body.room_id, body.reason.clone(), false).await?; Ok(invite_user::v3::Response {}) } else { Err(Error::BadRequest(ErrorKind::NotFound, "User not found.")) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index c0b5cf0c..9aae4aaf 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -146,7 +146,7 @@ pub(crate) async fn get_message_events_route( .timeline .pdus_after(sender_user, &body.room_id, from)? .filter_map(Result::ok) // Filter out buggy events - .filter(|(_, pdu)| { contains_url_filter(pdu, &body.filter) && visibility_filter(services, pdu, sender_user, &body.room_id) + .filter(|(_, pdu)| { contains_url_filter(pdu, &body.filter) && visibility_filter(&services, pdu, sender_user, &body.room_id) }) .take_while(|&(k, _)| Some(k) != to) // Stop at `to` @@ -193,7 +193,7 @@ pub(crate) async fn get_message_events_route( .timeline .pdus_until(sender_user, &body.room_id, from)? .filter_map(Result::ok) // Filter out buggy events - .filter(|(_, pdu)| {contains_url_filter(pdu, &body.filter) && visibility_filter(services, pdu, sender_user, &body.room_id)}) + .filter(|(_, pdu)| {contains_url_filter(pdu, &body.filter) && visibility_filter(&services, pdu, sender_user, &body.room_id)}) .take_while(|&(k, _)| Some(k) != to) // Stop at `to` .take(limit) .collect(); diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 9e9bcf8e..8f6d9056 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -33,7 +33,7 @@ pub(crate) async fn set_displayname_route( .filter_map(Result::ok) .collect(); - update_displayname(services, sender_user.clone(), body.displayname.clone(), all_joined_rooms).await?; + update_displayname(&services, sender_user.clone(), body.displayname.clone(), all_joined_rooms).await?; if services.globals.allow_local_presence() { // Presence update @@ -118,7 +118,7 @@ pub(crate) async fn set_avatar_url_route( .collect(); update_avatar_url( - services, + &services, sender_user.clone(), body.avatar_url.clone(), body.blurhash.clone(), diff --git a/src/api/client/report.rs b/src/api/client/report.rs index a16df444..dc87fd21 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -40,7 +40,7 @@ pub(crate) async fn report_event_route( }; is_report_valid( - services, + &services, &pdu.event_id, &body.room_id, sender_user, diff --git a/src/api/client/room.rs b/src/api/client/room.rs index c4d82822..c78ba6ed 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -79,7 +79,7 @@ pub(crate) async fn create_room_route( } let room_id: OwnedRoomId = if let Some(custom_room_id) = &body.room_id { - custom_room_id_check(services, custom_room_id)? + custom_room_id_check(&services, custom_room_id)? } else { RoomId::new(&services.globals.config.server_name) }; @@ -96,7 +96,7 @@ pub(crate) async fn create_room_route( let state_lock = services.rooms.state.mutex.lock(&room_id).await; let alias: Option = if let Some(alias) = &body.room_alias_name { - Some(room_alias_check(services, alias, &body.appservice_info).await?) + Some(room_alias_check(&services, alias, &body.appservice_info).await?) } else { None }; @@ -438,7 +438,7 @@ pub(crate) async fn create_room_route( // 8. Events implied by invite (and TODO: invite_3pid) drop(state_lock); for user_id in &body.invite { - if let Err(e) = invite_helper(services, sender_user, user_id, &room_id, None, body.is_direct).await { + if let Err(e) = invite_helper(&services, sender_user, user_id, &room_id, None, body.is_direct).await { warn!(%e, "Failed to send invite"); } } diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 7af4f5f9..d0fb83d1 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -37,7 +37,7 @@ pub(crate) async fn send_state_event_for_key_route( Ok(send_state_event::v3::Response { event_id: send_state_event_for_key_helper( - services, + &services, sender_user, &body.room_id, &body.event_type, diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 6eeb8fff..34baf7c1 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -137,7 +137,7 @@ pub(crate) async fn sync_events_route( ); if services.globals.allow_local_presence() { - process_presence_updates(services, &mut presence_updates, since, &sender_user).await?; + process_presence_updates(&services, &mut presence_updates, since, &sender_user).await?; } let all_joined_rooms = services @@ -152,7 +152,7 @@ pub(crate) async fn sync_events_route( for room_id in all_joined_rooms { let room_id = room_id?; if let Ok(joined_room) = load_joined_room( - services, + &services, &sender_user, &sender_device, &room_id, @@ -182,7 +182,7 @@ pub(crate) async fn sync_events_route( .collect(); for result in all_left_rooms { handle_left_room( - services, + &services, since, &result?.0, &sender_user, @@ -1214,7 +1214,7 @@ pub(crate) async fn sync_events_v4_route( match new_membership { MembershipState::Join => { // A new user joined an encrypted room - if !share_encrypted_room(services, &sender_user, &user_id, room_id)? { + if !share_encrypted_room(&services, &sender_user, &user_id, room_id)? { device_list_changes.insert(user_id); } }, @@ -1243,7 +1243,7 @@ pub(crate) async fn sync_events_v4_route( .filter(|user_id| { // Only send keys if the sender doesn't share an encrypted room with the target // already - !share_encrypted_room(services, &sender_user, user_id, room_id).unwrap_or(false) + !share_encrypted_room(&services, &sender_user, user_id, room_id).unwrap_or(false) }), ); } @@ -1407,7 +1407,8 @@ pub(crate) async fn sync_events_v4_route( for (room_id, (required_state_request, timeline_limit, roomsince)) in &todo_rooms { let roomsincecount = PduCount::Normal(*roomsince); - let (timeline_pdus, limited) = load_timeline(services, &sender_user, room_id, roomsincecount, *timeline_limit)?; + let (timeline_pdus, limited) = + load_timeline(&services, &sender_user, room_id, roomsincecount, *timeline_limit)?; if roomsince != &0 && timeline_pdus.is_empty() { continue; diff --git a/src/api/mod.rs b/src/api/mod.rs index c7411b6c..7fa70873 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,4 +1,4 @@ -#![recursion_limit = "160"] +#![recursion_limit = "192"] pub mod client; pub mod router; diff --git a/src/api/router.rs b/src/api/router.rs index 761c173c..d624de32 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -4,6 +4,8 @@ mod handler; mod request; mod response; +use std::sync::Arc; + use axum::{ response::IntoResponse, routing::{any, get, post}, @@ -16,7 +18,7 @@ use self::handler::RouterExt; pub(super) use self::{args::Args as Ruma, response::RumaResponse}; use crate::{client, server}; -pub type State = &'static service::Services; +pub type State = Arc; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; diff --git a/src/api/router/args.rs b/src/api/router/args.rs index fa5b1e43..a3d09dff 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -7,7 +7,7 @@ use ruma::{api::IncomingRequest, CanonicalJsonValue, OwnedDeviceId, OwnedServerN use service::Services; use super::{auth, auth::Auth, request, request::Request}; -use crate::service::appservice::RegistrationInfo; +use crate::{service::appservice::RegistrationInfo, State}; /// Extractor for Ruma request structs pub(crate) struct Args { @@ -36,14 +36,13 @@ pub(crate) struct Args { } #[async_trait] -impl FromRequest for Args +impl FromRequest for Args where T: IncomingRequest, { type Rejection = Error; - async fn from_request(request: hyper::Request, _: &S) -> Result { - let services = service::services(); // ??? + async fn from_request(request: hyper::Request, services: &State) -> Result { let mut request = request::from(services, request).await?; let mut json_body = serde_json::from_slice::(&request.body).ok(); let auth = auth::auth(services, &mut request, &json_body, &T::METADATA).await?; diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index e1beaa33..e9b1a6c7 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -82,7 +82,7 @@ pub(crate) async fn create_join_event_template_route( .state_cache .is_left(&body.user_id, &body.room_id) .unwrap_or(true)) - && user_can_perform_restricted_join(services, &body.user_id, &body.room_id, &room_version_id)? + && user_can_perform_restricted_join(&services, &body.user_id, &body.room_id, &room_version_id)? { let auth_user = services .rooms diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index 1876dde1..af8a5846 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -26,7 +26,7 @@ pub(crate) async fn get_public_rooms_filtered_route( } let response = crate::client::get_public_rooms_filtered_helper( - services, + &services, None, body.limit, body.since.as_deref(), @@ -60,7 +60,7 @@ pub(crate) async fn get_public_rooms_route( } let response = crate::client::get_public_rooms_filtered_helper( - services, + &services, None, body.limit, body.since.as_deref(), diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 2f698d33..394289a6 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -62,8 +62,8 @@ pub(crate) async fn send_transaction_message_route( "Starting txn", ); - let resolved_map = handle_pdus(services, &client, &body, origin, &txn_start_time).await?; - handle_edus(services, &client, &body, origin).await?; + let resolved_map = handle_pdus(&services, &client, &body, origin, &txn_start_time).await?; + handle_edus(&services, &client, &body, origin).await?; debug!( pdus = ?body.pdus.len(), diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 4cd29795..17f56383 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -241,7 +241,7 @@ pub(crate) async fn create_join_event_v1_route( } } - let room_state = create_join_event(services, origin, &body.room_id, &body.pdu).await?; + let room_state = create_join_event(&services, origin, &body.room_id, &body.pdu).await?; Ok(create_join_event::v1::Response { room_state, @@ -286,7 +286,7 @@ pub(crate) async fn create_join_event_v2_route( auth_chain, state, event, - } = create_join_event(services, origin, &body.room_id, &body.pdu).await?; + } = create_join_event(&services, origin, &body.room_id, &body.pdu).await?; let room_state = create_join_event::v2::RoomState { members_omitted: false, auth_chain, diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index b1b8fec8..ef4c8c45 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -28,7 +28,7 @@ pub(crate) async fn create_leave_event_v1_route( ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - create_leave_event(services, origin, &body.room_id, &body.pdu).await?; + create_leave_event(&services, origin, &body.room_id, &body.pdu).await?; Ok(create_leave_event::v1::Response::new()) } @@ -41,7 +41,7 @@ pub(crate) async fn create_leave_event_v2_route( ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - create_leave_event(services, origin, &body.room_id, &body.pdu).await?; + create_leave_event(&services, origin, &body.room_id, &body.pdu).await?; Ok(create_leave_event::v2::Response::new()) } diff --git a/src/api/server/user.rs b/src/api/server/user.rs index bd0372e6..e9a400a7 100644 --- a/src/api/server/user.rs +++ b/src/api/server/user.rs @@ -84,7 +84,7 @@ pub(crate) async fn get_keys_route( } let result = get_keys_helper( - services, + &services, None, &body.device_keys, |u| Some(u.server_name()) == body.origin.as_deref(), @@ -116,7 +116,7 @@ pub(crate) async fn claim_keys_route( )); } - let result = claim_keys_helper(services, &body.one_time_keys).await?; + let result = claim_keys_helper(&services, &body.one_time_keys).await?; Ok(claim_keys::v1::Response { one_time_keys: result.one_time_keys, diff --git a/src/database/database.rs b/src/database/database.rs index 44bb655c..1d7bbc33 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -11,12 +11,12 @@ pub struct Database { impl Database { /// Load an existing database or create a new one. - pub async fn open(server: &Arc) -> Result { + pub async fn open(server: &Arc) -> Result> { let db = Engine::open(server)?; - Ok(Self { + Ok(Arc::new(Self { db: db.clone(), map: maps::open(&db)?, - }) + })) } #[inline] diff --git a/src/main/main.rs b/src/main/main.rs index b8cb24ff..8703eef2 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -1,3 +1,5 @@ +#![recursion_limit = "192"] + pub(crate) mod clap; mod mods; mod restart; @@ -57,17 +59,38 @@ fn main() -> Result<(), Error> { async fn async_main(server: &Arc) -> Result<(), Error> { extern crate conduit_router as router; - if let Err(error) = router::start(&server.server).await { - error!("Critical error starting server: {error}"); - return Err(error); - } + match router::start(&server.server).await { + Ok(services) => server.services.lock().await.insert(services), + Err(error) => { + error!("Critical error starting server: {error}"); + return Err(error); + }, + }; - if let Err(error) = router::run(&server.server).await { + if let Err(error) = router::run( + server + .services + .lock() + .await + .as_ref() + .expect("services initialized"), + ) + .await + { error!("Critical error running server: {error}"); return Err(error); } - if let Err(error) = router::stop(&server.server).await { + if let Err(error) = router::stop( + server + .services + .lock() + .await + .take() + .expect("services initialied"), + ) + .await + { error!("Critical error stopping server: {error}"); return Err(error); } diff --git a/src/main/server.rs b/src/main/server.rs index 71cdadce..e435b2f4 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use conduit::{config::Config, info, log::Log, utils::sys, Error, Result}; -use tokio::runtime; +use tokio::{runtime, sync::Mutex}; use crate::{clap::Args, tracing::TracingFlameGuard}; @@ -10,6 +10,8 @@ pub(crate) struct Server { /// Server runtime state; public portion pub(crate) server: Arc, + pub(crate) services: Mutex>>, + _tracing_flame_guard: TracingFlameGuard, #[cfg(feature = "sentry_telemetry")] @@ -54,6 +56,8 @@ impl Server { }, )), + services: None.into(), + _tracing_flame_guard: tracing_flame_guard, #[cfg(feature = "sentry_telemetry")] diff --git a/src/router/layers.rs b/src/router/layers.rs index 67342eb3..2b143666 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -6,6 +6,7 @@ use axum::{ }; use axum_client_ip::SecureClientIpSource; use conduit::{error, Result, Server}; +use conduit_service::Services; use http::{ header::{self, HeaderName}, HeaderValue, Method, StatusCode, @@ -34,7 +35,8 @@ const CONDUWUIT_CSP: &[&str] = &[ const CONDUWUIT_PERMISSIONS_POLICY: &[&str] = &["interest-cohort=()", "browsing-topics=()"]; -pub(crate) fn build(server: &Arc) -> Result { +pub(crate) fn build(services: &Arc) -> Result { + let server = &services.server; let layers = ServiceBuilder::new(); #[cfg(feature = "sentry_telemetry")] @@ -83,7 +85,7 @@ pub(crate) fn build(server: &Arc) -> Result { .layer(body_limit_layer(server)) .layer(CatchPanicLayer::custom(catch_panic)); - Ok(router::build(server).layer(layers)) + Ok(router::build(services).layer(layers)) } #[cfg(any(feature = "zstd_compression", feature = "gzip_compression", feature = "brotli_compression"))] @@ -151,12 +153,14 @@ fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max #[allow(clippy::needless_pass_by_value)] #[tracing::instrument(skip_all, name = "panic")] fn catch_panic(err: Box) -> http::Response> { - conduit_service::services() - .server - .metrics - .requests_panic - .fetch_add(1, std::sync::atomic::Ordering::Release); - + //TODO: XXX + /* + conduit_service::services() + .server + .metrics + .requests_panic + .fetch_add(1, std::sync::atomic::Ordering::Release); + */ let details = if let Some(s) = err.downcast_ref::() { s.clone() } else if let Some(s) = err.downcast_ref::<&str>() { diff --git a/src/router/mod.rs b/src/router/mod.rs index 13fe3908..67ebc0e3 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -11,22 +11,23 @@ extern crate conduit_core as conduit; use std::{future::Future, pin::Pin, sync::Arc}; use conduit::{Result, Server}; +use conduit_service::Services; conduit::mod_ctor! {} conduit::mod_dtor! {} conduit::rustc_flags_capture! {} #[no_mangle] -pub extern "Rust" fn start(server: &Arc) -> Pin> + Send>> { +pub extern "Rust" fn start(server: &Arc) -> Pin>> + Send>> { Box::pin(run::start(server.clone())) } #[no_mangle] -pub extern "Rust" fn stop(server: &Arc) -> Pin> + Send>> { - Box::pin(run::stop(server.clone())) +pub extern "Rust" fn stop(services: Arc) -> Pin> + Send>> { + Box::pin(run::stop(services)) } #[no_mangle] -pub extern "Rust" fn run(server: &Arc) -> Pin> + Send>> { - Box::pin(run::run(server.clone())) +pub extern "Rust" fn run(services: &Arc) -> Pin> + Send>> { + Box::pin(run::run(services.clone())) } diff --git a/src/router/router.rs b/src/router/router.rs index 7c374b47..3527f1e6 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -1,20 +1,20 @@ use std::sync::Arc; use axum::{response::IntoResponse, routing::get, Router}; -use conduit::{Error, Server}; +use conduit::Error; +use conduit_api::State; +use conduit_service::Services; use http::{StatusCode, Uri}; use ruma::api::client::error::ErrorKind; -extern crate conduit_api as api; -extern crate conduit_service as service; +pub(crate) fn build(services: &Arc) -> Router { + let router = Router::::new(); + let state = services.clone(); -pub(crate) fn build(server: &Arc) -> Router { - let router = Router::::new(); - - api::router::build(router, server) + conduit_api::router::build(router, &services.server) .route("/", get(it_works)) .fallback(not_found) - .with_state(service::services()) + .with_state(state) } async fn not_found(_uri: Uri) -> impl IntoResponse { diff --git a/src/router/run.rs b/src/router/run.rs index cb5d2abf..395aa8c4 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -1,28 +1,30 @@ -use std::{sync::Arc, time::Duration}; +extern crate conduit_admin as admin; +extern crate conduit_core as conduit; +extern crate conduit_service as service; + +use std::{ + sync::{atomic::Ordering, Arc}, + time::Duration, +}; use axum_server::Handle as ServerHandle; +use conduit::{debug, debug_error, debug_info, error, info, Error, Result, Server}; +use service::Services; use tokio::{ sync::broadcast::{self, Sender}, task::JoinHandle, }; -extern crate conduit_admin as admin; -extern crate conduit_core as conduit; -extern crate conduit_service as service; - -use std::sync::atomic::Ordering; - -use conduit::{debug, debug_info, error, info, Error, Result, Server}; - use crate::serve; /// Main loop base #[tracing::instrument(skip_all)] -pub(crate) async fn run(server: Arc) -> Result<()> { +pub(crate) async fn run(services: Arc) -> Result<()> { + let server = &services.server; debug!("Start"); // Install the admin room callback here for now - admin::init().await; + admin::init(&services.admin).await; // Setup shutdown/signal handling let handle = ServerHandle::new(); @@ -33,13 +35,13 @@ pub(crate) async fn run(server: Arc) -> Result<()> { let mut listener = server .runtime() - .spawn(serve::serve(server.clone(), handle.clone(), tx.subscribe())); + .spawn(serve::serve(services.clone(), handle.clone(), tx.subscribe())); // Focal point debug!("Running"); let res = tokio::select! { res = &mut listener => res.map_err(Error::from).unwrap_or_else(Err), - res = service::services().poll() => handle_services_poll(&server, res, listener).await, + res = services.poll() => handle_services_poll(server, res, listener).await, }; // Join the signal handler before we leave. @@ -47,7 +49,7 @@ pub(crate) async fn run(server: Arc) -> Result<()> { _ = sigs.await; // Remove the admin room callback - admin::fini().await; + admin::fini(&services.admin).await; debug_info!("Finish"); res @@ -55,26 +57,33 @@ pub(crate) async fn run(server: Arc) -> Result<()> { /// Async initializations #[tracing::instrument(skip_all)] -pub(crate) async fn start(server: Arc) -> Result<()> { +pub(crate) async fn start(server: Arc) -> Result> { debug!("Starting..."); - service::start(&server).await?; + let services = Services::build(server).await?.start().await?; #[cfg(feature = "systemd")] sd_notify::notify(true, &[sd_notify::NotifyState::Ready]).expect("failed to notify systemd of ready state"); debug!("Started"); - Ok(()) + Ok(services) } /// Async destructions #[tracing::instrument(skip_all)] -pub(crate) async fn stop(_server: Arc) -> Result<()> { +pub(crate) async fn stop(services: Arc) -> Result<()> { debug!("Shutting down..."); // Wait for all completions before dropping or we'll lose them to the module // unload and explode. - service::stop().await; + services.stop().await; + + if let Err(services) = Arc::try_unwrap(services) { + debug_error!( + "{} dangling references to Services after shutdown", + Arc::strong_count(&services) + ); + } debug!("Cleaning up..."); diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index 4e923444..58bf2de8 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -5,22 +5,26 @@ mod unix; use std::sync::Arc; use axum_server::Handle as ServerHandle; -use conduit::{Result, Server}; +use conduit::Result; +use conduit_service::Services; use tokio::sync::broadcast; -use crate::layers; +use super::layers; /// Serve clients -pub(super) async fn serve(server: Arc, handle: ServerHandle, shutdown: broadcast::Receiver<()>) -> Result<()> { +pub(super) async fn serve( + services: Arc, handle: ServerHandle, shutdown: broadcast::Receiver<()>, +) -> Result<()> { + let server = &services.server; let config = &server.config; let addrs = config.get_bind_addrs(); - let app = layers::build(&server)?; + let app = layers::build(&services)?; if cfg!(unix) && config.unix_socket_path.is_some() { - unix::serve(&server, app, shutdown).await + unix::serve(server, app, shutdown).await } else if config.tls.is_some() { - tls::serve(&server, app, handle, addrs).await + tls::serve(server, app, handle, addrs).await } else { - plain::serve(&server, app, handle, addrs).await + plain::serve(server, app, handle, addrs).await } } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 6241c668..71f3b73e 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,7 +9,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{debug, error, error::default_log, pdu::PduBuilder, Error, PduEvent, Result, Server}; +use conduit::{debug, error, error::default_log, pdu::PduBuilder, Err, Error, PduEvent, Result, Server}; pub use create::create_admin_room; use loole::{Receiver, Sender}; use ruma::{ @@ -41,6 +41,7 @@ struct Services { timeline: Dep, state: Dep, state_cache: Dep, + services: StdRwLock>>, } #[derive(Debug)] @@ -50,7 +51,7 @@ pub struct CommandInput { } pub type Completer = fn(&str) -> String; -pub type Handler = fn(CommandInput) -> HandlerResult; +pub type Handler = fn(Arc, CommandInput) -> HandlerResult; pub type HandlerResult = Pin + Send>>; pub type CommandResult = Result; pub type CommandOutput = Option; @@ -69,6 +70,7 @@ impl crate::Service for Service { timeline: args.depend::("rooms::timeline"), state: args.depend::("rooms::state"), state_cache: args.depend::("rooms::state_cache"), + services: None.into(), }, sender, receiver: Mutex::new(receiver), @@ -172,10 +174,14 @@ impl Service { } async fn process_command(&self, command: CommandInput) -> CommandResult { + let Some(services) = self.services.services.read().expect("locked").clone() else { + return Err!("Services self-reference not initialized."); + }; + if let Some(handle) = self.handle.read().await.as_ref() { - handle(command).await + handle(services, command).await } else { - Err(Error::Err("Admin module is not loaded.".into())) + Err!("Admin module is not loaded.") } } @@ -356,4 +362,10 @@ impl Service { #[cfg(feature = "console")] self.console.close().await; } + + /// Sets the self-reference to crate::Services which will provide context to + /// the admin commands. + pub(super) fn set_services(&self, services: Option>) { + *self.services.services.write().expect("locked for writing") = services; + } } diff --git a/src/service/mod.rs b/src/service/mod.rs index b6ec58b5..084bfac8 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,4 +1,4 @@ -#![recursion_limit = "160"] +#![recursion_limit = "192"] #![allow(refining_impl_trait)] mod manager; @@ -26,11 +26,7 @@ pub mod users; extern crate conduit_core as conduit; extern crate conduit_database as database; -use std::sync::{Arc, RwLock}; - pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; -use conduit::{Result, Server}; -use database::Database; pub(crate) use service::{Args, Dep, Service}; pub use crate::services::Services; @@ -38,50 +34,3 @@ pub use crate::services::Services; conduit::mod_ctor! {} conduit::mod_dtor! {} conduit::rustc_flags_capture! {} - -static SERVICES: RwLock> = RwLock::new(None); - -pub async fn start(server: &Arc) -> Result<()> { - let d = Arc::new(Database::open(server).await?); - let s = Box::new(Services::build(server.clone(), d)?); - _ = SERVICES.write().expect("write locked").insert(Box::leak(s)); - - services().start().await -} - -pub async fn stop() { - services().stop().await; - - // Deactivate services(). Any further use will panic the caller. - let s = SERVICES - .write() - .expect("write locked") - .take() - .expect("services initialized"); - - let s: *mut Services = std::ptr::from_ref(s).cast_mut(); - //SAFETY: Services was instantiated in init() and leaked into the SERVICES - // global perusing as 'static for the duration of service. Now we reclaim - // it to drop it before unloading the module. If this is not done there wil - // be multiple instances after module reload. - let s = unsafe { Box::from_raw(s) }; - - // Drop it so we encounter any trouble before the infolog message - drop(s); -} - -#[must_use] -pub fn services() -> &'static Services { - SERVICES - .read() - .expect("SERVICES locked for reading") - .expect("SERVICES initialized with Services instance") -} - -#[inline] -pub fn available() -> bool { - SERVICES - .read() - .expect("SERVICES locked for reading") - .is_some() -} diff --git a/src/service/services.rs b/src/service/services.rs index 59909f8c..b283db6c 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -44,7 +44,8 @@ pub struct Services { impl Services { #[allow(clippy::cognitive_complexity)] - pub fn build(server: Arc, db: Arc) -> Result { + pub async fn build(server: Arc) -> Result> { + let db = Database::open(&server).await?; let service: Arc = Arc::new(RwLock::new(BTreeMap::new())); macro_rules! build { ($tyname:ty) => {{ @@ -58,7 +59,7 @@ impl Services { }}; } - Ok(Self { + Ok(Arc::new(Self { account_data: build!(account_data::Service), admin: build!(admin::Service), appservice: build!(appservice::Service), @@ -102,12 +103,13 @@ impl Services { service, server, db, - }) + })) } - pub(super) async fn start(&self) -> Result<()> { + pub async fn start(self: &Arc) -> Result> { debug_info!("Starting services..."); + self.admin.set_services(Some(Arc::clone(self))); globals::migrations::migrations(self).await?; self.manager .lock() @@ -118,10 +120,10 @@ impl Services { .await?; debug_info!("Services startup complete."); - Ok(()) + Ok(Arc::clone(self)) } - pub(super) async fn stop(&self) { + pub async fn stop(&self) { info!("Shutting down services..."); self.interrupt(); @@ -129,6 +131,8 @@ impl Services { manager.stop().await; } + self.admin.set_services(None); + debug_info!("Services shutdown complete."); } From 954cfc6bb7f87c381e56f028d49f503134c732de Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 27 Jul 2024 08:28:35 +0000 Subject: [PATCH 0595/2091] bump cargo Signed-off-by: Jason Volk --- Cargo.lock | 86 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 4 +-- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ea215eec..83c443cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,9 +43,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" +checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anyhow" @@ -544,9 +544,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.9" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64acc1846d54c1fe936a78dc189c34e28d3f5afc348403f28ecf53660b9b8462" +checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" dependencies = [ "clap_builder", "clap_derive", @@ -554,9 +554,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.9" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8393d67ba2e7bfaf28a23458e4e2b543cc73a99595511eb207fdb8aede942" +checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" dependencies = [ "anstyle", "clap_lex", @@ -564,9 +564,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.8" +version = "4.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" +checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -576,9 +576,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" +checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "color_quant" @@ -729,12 +729,12 @@ dependencies = [ [[package]] name = "conduit_macros" -version = "0.4.5" +version = "0.4.6" dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1028,7 +1028,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.71", + "syn 2.0.72", ] [[package]] @@ -1717,7 +1717,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "hyper-util", - "rustls 0.23.11", + "rustls 0.23.12", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -1896,9 +1896,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jobserver" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -1967,9 +1967,9 @@ dependencies = [ [[package]] name = "lazy-regex" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d12be4595afdf58bd19e4a9f4e24187da2a66700786ff660a418e9059937a4c" +checksum = "576c8060ecfdf2e56995cf3274b4f2d71fa5e4fa3607c1c0b63c10180ee58741" dependencies = [ "lazy-regex-proc_macros", "once_cell", @@ -1978,9 +1978,9 @@ dependencies = [ [[package]] name = "lazy-regex-proc_macros" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44bcd58e6c97a7fcbaffcdc95728b393b8d98933bfadad49ed4097845b57ef0b" +checksum = "9efb9e65d4503df81c615dc33ff07042a9408ac7f26b45abee25566f7fbfd12c" dependencies = [ "proc-macro2", "quote", @@ -2329,9 +2329,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.1" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "081b846d1d56ddfc18fdf1a922e4f6e07a11768ea1b92dec44e42b72712ccfce" +checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" dependencies = [ "memchr", ] @@ -2765,7 +2765,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.11", + "rustls 0.23.12", "thiserror", "tokio", "tracing", @@ -2781,7 +2781,7 @@ dependencies = [ "rand", "ring", "rustc-hash", - "rustls 0.23.11", + "rustls 0.23.12", "slab", "thiserror", "tinyvec", @@ -2920,7 +2920,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.11", + "rustls 0.23.12", "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", @@ -3261,9 +3261,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.11" +version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4828ea528154ae444e5a642dbb7d5623354030dc9822b83fd9bb79683c7399d0" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ "log", "once_cell", @@ -3622,9 +3622,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" dependencies = [ "serde", ] @@ -4120,7 +4120,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.11", + "rustls 0.23.12", "rustls-pki-types", "tokio", ] @@ -4163,21 +4163,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.15" +version = "0.8.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28" +checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.16", + "toml_edit 0.22.17", ] [[package]] name = "toml_datetime" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db" dependencies = [ "serde", ] @@ -4195,15 +4195,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.16" +version = "0.22.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788" +checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16" dependencies = [ "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.15", + "winnow 0.6.16", ] [[package]] @@ -4482,7 +4482,7 @@ dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.23.11", + "rustls 0.23.12", "rustls-pki-types", "url", "webpki-roots", @@ -4536,9 +4536,9 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "want" @@ -4881,9 +4881,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.15" +version = "0.6.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "557404e450152cd6795bb558bca69e43c585055f4606e3bcae5894fc6dac9ba0" +checksum = "b480ae9340fc261e6be3e95a1ba86d54ae3f9171132a73ce8d4bbaf68339507c" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 17e7e712..48e6ac1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -426,11 +426,11 @@ default-features = false version = "0.1" [workspace.dependencies.syn] -version = "2.0" +version = "2.0.72" features = ["full", "extra-traits"] [workspace.dependencies.quote] -version = "1.0" +version = "1.0.36" [workspace.dependencies.proc-macro2] version = "1.0.86" From 4b765aa17cd0a0be33221539a3f28ec649c45635 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 27 Jul 2024 12:06:06 -0400 Subject: [PATCH 0596/2091] ci: only use sccache conditionally Signed-off-by: strawberry --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96719735..7d7fa560 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,10 +27,10 @@ concurrency: cancel-in-progress: true env: - # sccache - SCCACHE_GHA_ENABLED: "true" - RUSTC_WRAPPER: "sccache" - SCCACHE_BUCKET: "sccache" + # sccache only on main repo + SCCACHE_GHA_ENABLED: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'true' || 'false' }}" + RUSTC_WRAPPER: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' }}" + SCCACHE_BUCKET: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' }}" SCCACHE_S3_USE_SSL: ${{ vars.SCCACHE_S3_USE_SSL }} SCCACHE_REGION: ${{ vars.SCCACHE_REGION }} SCCACHE_ENDPOINT: ${{ vars.SCCACHE_ENDPOINT }} @@ -156,6 +156,7 @@ jobs: # use sccache for Rust - name: Run sccache-cache + if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') uses: mozilla-actions/sccache-action@main # use rust-cache @@ -288,6 +289,7 @@ jobs: # use sccache for Rust - name: Run sccache-cache + if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') uses: mozilla-actions/sccache-action@main # use rust-cache From c3e751542d15b8e8517136faec24f163aa5177a9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 27 Jul 2024 23:40:54 -0400 Subject: [PATCH 0597/2091] ci: default to empty string to "unset" variables Signed-off-by: strawberry --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d7fa560..5b56b66a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,8 +29,8 @@ concurrency: env: # sccache only on main repo SCCACHE_GHA_ENABLED: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'true' || 'false' }}" - RUSTC_WRAPPER: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' }}" - SCCACHE_BUCKET: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' }}" + RUSTC_WRAPPER: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}" + SCCACHE_BUCKET: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}" SCCACHE_S3_USE_SSL: ${{ vars.SCCACHE_S3_USE_SSL }} SCCACHE_REGION: ${{ vars.SCCACHE_REGION }} SCCACHE_ENDPOINT: ${{ vars.SCCACHE_ENDPOINT }} From ea95627dce897a7ffa249190dc0206cbdb43eebb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 27 Jul 2024 20:55:21 +0000 Subject: [PATCH 0598/2091] hoist rustc flag string array to global const Signed-off-by: Jason Volk --- src/macros/rustc.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/macros/rustc.rs b/src/macros/rustc.rs index a3bab26f..d286ed33 100644 --- a/src/macros/rustc.rs +++ b/src/macros/rustc.rs @@ -9,11 +9,13 @@ pub(super) fn flags_capture(args: TokenStream) -> TokenStream { }; let flag = std::env::args().collect::>(); + let flag_len = flag.len(); let ret = quote! { + pub static RUSTC_FLAGS: [&str; #flag_len] = [#( #flag ),*]; + #[conduit_core::ctor] fn _set_rustc_flags() { - let flags = &[#( #flag ),*]; - conduit_core::info::rustc::FLAGS.lock().expect("locked").insert(#crate_name, flags); + conduit_core::info::rustc::FLAGS.lock().expect("locked").insert(#crate_name, &RUSTC_FLAGS); } // static strings have to be yanked on module unload From ddc8c3b46cd4d5481a7c5e000caaa08bf37d5849 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 27 Jul 2024 23:25:13 +0000 Subject: [PATCH 0599/2091] split presence data object into file; improve service encapsulations Signed-off-by: Jason Volk --- Cargo.toml | 1 + src/service/presence/data.rs | 3 +- src/service/presence/mod.rs | 86 ++++++-------------------------- src/service/presence/presence.rs | 65 ++++++++++++++++++++++++ src/service/users/mod.rs | 28 ++++++----- 5 files changed, 99 insertions(+), 84 deletions(-) create mode 100644 src/service/presence/presence.rs diff --git a/Cargo.toml b/Cargo.toml index 48e6ac1a..82163b31 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -851,6 +851,7 @@ style = { level = "warn", priority = -1 } ## some sadness # trivial assertions are quite alright assertions_on_constants = { level = "allow", priority = 1 } +module_inception = { level = "allow", priority = 1 } ################### suspicious = { level = "warn", priority = -1 } diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 53f9d8c7..1a0293de 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -4,7 +4,8 @@ use conduit::{debug_warn, utils, Error, Result}; use database::Map; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; -use crate::{globals, presence::Presence, users, Dep}; +use super::Presence; +use crate::{globals, users, Dep}; pub struct Data { presenceid_presence: Arc, diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index b8673304..a54a6d7c 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -1,81 +1,25 @@ mod data; +mod presence; use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{checked, debug, error, utils, Error, Result, Server}; +use conduit::{checked, debug, error, Error, Result, Server}; use futures_util::{stream::FuturesUnordered, StreamExt}; -use ruma::{ - events::presence::{PresenceEvent, PresenceEventContent}, - presence::PresenceState, - OwnedUserId, UInt, UserId, -}; -use serde::{Deserialize, Serialize}; +use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; use tokio::{sync::Mutex, time::sleep}; -use self::data::Data; +use self::{data::Data, presence::Presence}; use crate::{globals, users, Dep}; -/// Represents data required to be kept in order to implement the presence -/// specification. -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct Presence { - state: PresenceState, - currently_active: bool, - last_active_ts: u64, - status_msg: Option, -} - -impl Presence { - #[must_use] - pub fn new(state: PresenceState, currently_active: bool, last_active_ts: u64, status_msg: Option) -> Self { - Self { - state, - currently_active, - last_active_ts, - status_msg, - } - } - - pub fn from_json_bytes(bytes: &[u8]) -> Result { - serde_json::from_slice(bytes).map_err(|_| Error::bad_database("Invalid presence data in database")) - } - - pub fn to_json_bytes(&self) -> Result> { - serde_json::to_vec(self).map_err(|_| Error::bad_database("Could not serialize Presence to JSON")) - } - - /// Creates a PresenceEvent from available data. - pub fn to_presence_event(&self, user_id: &UserId, users: &users::Service) -> Result { - let now = utils::millis_since_unix_epoch(); - let last_active_ago = if self.currently_active { - None - } else { - Some(UInt::new_saturating(now.saturating_sub(self.last_active_ts))) - }; - - Ok(PresenceEvent { - sender: user_id.to_owned(), - content: PresenceEventContent { - presence: self.state.clone(), - status_msg: self.status_msg.clone(), - currently_active: Some(self.currently_active), - last_active_ago, - displayname: users.displayname(user_id)?, - avatar_url: users.avatar_url(user_id)?, - }, - }) - } -} - pub struct Service { - services: Services, - pub db: Data, - pub timer_sender: loole::Sender<(OwnedUserId, Duration)>, - timer_receiver: Mutex>, + timer_sender: loole::Sender, + timer_receiver: Mutex>, timeout_remote_users: bool, idle_timeout: u64, offline_timeout: u64, + pub db: Data, + services: Services, } struct Services { @@ -84,6 +28,8 @@ struct Services { users: Dep, } +type TimerType = (OwnedUserId, Duration); + #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -92,17 +38,17 @@ impl crate::Service for Service { let offline_timeout_s = config.presence_offline_timeout_s; let (timer_sender, timer_receiver) = loole::unbounded(); Ok(Arc::new(Self { - services: Services { - server: args.server.clone(), - globals: args.depend::("globals"), - users: args.depend::("users"), - }, - db: Data::new(&args), timer_sender, timer_receiver: Mutex::new(timer_receiver), timeout_remote_users: config.presence_timeout_remote_users, idle_timeout: checked!(idle_timeout_s * 1_000)?, offline_timeout: checked!(offline_timeout_s * 1_000)?, + db: Data::new(&args), + services: Services { + server: args.server.clone(), + globals: args.depend::("globals"), + users: args.depend::("users"), + }, })) } diff --git a/src/service/presence/presence.rs b/src/service/presence/presence.rs new file mode 100644 index 00000000..570008f2 --- /dev/null +++ b/src/service/presence/presence.rs @@ -0,0 +1,65 @@ +use std::sync::Arc; + +use conduit::{utils, Error, Result}; +use ruma::{ + events::presence::{PresenceEvent, PresenceEventContent}, + presence::PresenceState, + UInt, UserId, +}; +use serde::{Deserialize, Serialize}; + +use crate::users; + +/// Represents data required to be kept in order to implement the presence +/// specification. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub(super) struct Presence { + state: PresenceState, + currently_active: bool, + last_active_ts: u64, + status_msg: Option, +} + +impl Presence { + #[must_use] + pub(super) fn new( + state: PresenceState, currently_active: bool, last_active_ts: u64, status_msg: Option, + ) -> Self { + Self { + state, + currently_active, + last_active_ts, + status_msg, + } + } + + pub(super) fn from_json_bytes(bytes: &[u8]) -> Result { + serde_json::from_slice(bytes).map_err(|_| Error::bad_database("Invalid presence data in database")) + } + + pub(super) fn to_json_bytes(&self) -> Result> { + serde_json::to_vec(self).map_err(|_| Error::bad_database("Could not serialize Presence to JSON")) + } + + /// Creates a PresenceEvent from available data. + pub(super) fn to_presence_event(&self, user_id: &UserId, users: &Arc) -> Result { + let now = utils::millis_since_unix_epoch(); + let last_active_ago = if self.currently_active { + None + } else { + Some(UInt::new_saturating(now.saturating_sub(self.last_active_ts))) + }; + + Ok(PresenceEvent { + sender: user_id.to_owned(), + content: PresenceEventContent { + presence: self.state.clone(), + status_msg: self.status_msg.clone(), + currently_active: Some(self.currently_active), + last_active_ago, + displayname: users.displayname(user_id)?, + avatar_url: users.avatar_url(user_id)?, + }, + }) + } +} diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 4c80d0d3..46084415 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -26,19 +26,10 @@ use ruma::{ use self::data::Data; use crate::{admin, rooms, Dep}; -pub struct SlidingSyncCache { - lists: BTreeMap, - subscriptions: BTreeMap, - known_rooms: BTreeMap>, // For every room, the roomsince number - extensions: ExtensionsConfig, -} - -type DbConnections = Mutex>>>; - pub struct Service { - services: Services, + connections: DbConnections, pub db: Data, - pub connections: DbConnections, + services: Services, } struct Services { @@ -49,18 +40,29 @@ struct Services { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + connections: StdMutex::new(BTreeMap::new()), + db: Data::new(&args), services: Services { admin: args.depend::("admin"), state_cache: args.depend::("rooms::state_cache"), }, - db: Data::new(&args), - connections: StdMutex::new(BTreeMap::new()), })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } +type DbConnections = Mutex>; +type DbConnectionsKey = (OwnedUserId, OwnedDeviceId, String); +type DbConnectionsVal = Arc>; + +struct SlidingSyncCache { + lists: BTreeMap, + subscriptions: BTreeMap, + known_rooms: BTreeMap>, // For every room, the roomsince number + extensions: ExtensionsConfig, +} + impl Service { /// Check if a user has an account on this homeserver. #[inline] From 49033db2b80f911efe3f10bd464eca62a4eb6245 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 28 Jul 2024 00:24:09 +0000 Subject: [PATCH 0600/2091] fix trailing body passed to commands Signed-off-by: Jason Volk --- src/admin/handler.rs | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index a7e4d79a..738d748c 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,7 +1,7 @@ use std::{panic::AssertUnwindSafe, sync::Arc, time::Instant}; use clap::{CommandFactory, Parser}; -use conduit::{error, trace, utils::string::common_prefix, Error, Result}; +use conduit::{checked, error, trace, utils::string::common_prefix, Error, Result}; use futures_util::future::FutureExt; use ruma::{ events::{ @@ -60,8 +60,11 @@ fn reply(mut content: RoomMessageEventContent, reply_id: Option) - // Parse and process a message from the admin room async fn process(services: Arc, msg: &str) -> CommandOutput { - let mut lines = msg.lines().filter(|l| !l.trim().is_empty()); - let command = lines.next().expect("each string has at least one line"); + let lines = msg.lines().filter(|l| !l.trim().is_empty()); + let command = lines + .clone() + .next() + .expect("each string has at least one line"); let (parsed, body) = match parse_command(command) { Ok(parsed) => parsed, Err(error) => { @@ -71,12 +74,12 @@ async fn process(services: Arc, msg: &str) -> CommandOutput { }, }; - let timer = Instant::now(); - let body: Vec<&str> = body.iter().map(String::as_str).collect(); + let body = parse_body(AdminCommand::command(), &body, lines.skip(1).collect()).expect("trailing body parsed"); let context = Command { services: &services, body: &body, }; + let timer = Instant::now(); let result = Box::pin(admin::process(parsed, &context)).await; let elapsed = timer.elapsed(); conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}"); @@ -95,6 +98,32 @@ fn parse_command(command_line: &str) -> Result<(AdminCommand, Vec), Stri Ok((com, argv)) } +fn parse_body<'a>(mut cmd: clap::Command, body: &'a [String], lines: Vec<&'a str>) -> Result> { + let mut start = 1; + 'token: for token in body.iter().skip(1) { + let cmd_ = cmd.clone(); + for sub in cmd_.get_subcommands() { + if sub.get_name() == *token { + start = checked!(start + 1)?; + cmd = sub.clone(); + continue 'token; + } + } + + // positional arguments have to be skipped too + let num_posargs = cmd_.get_positionals().count(); + start = checked!(start + num_posargs)?; + break; + } + + Ok(body + .iter() + .skip(start) + .map(String::as_str) + .chain(lines) + .collect::>()) +} + fn complete_command(mut cmd: clap::Command, line: &str) -> String { let argv = parse_line(line); let mut ret = Vec::::with_capacity(argv.len().saturating_add(1)); From e6884ef13a74167e2d0c71b8726d1d9b18bc6aad Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 28 Jul 2024 01:03:08 +0000 Subject: [PATCH 0601/2091] fix various nightly incidental lints Signed-off-by: Jason Volk --- Cargo.toml | 2 +- src/core/pdu/mod.rs | 2 +- src/service/resolver/actual.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 82163b31..d2d7dc2d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -801,7 +801,7 @@ perf = { level = "warn", priority = -1 } ################### #restriction = "warn" -allow_attributes = "warn" +#allow_attributes = "warn" # UNSTABLE arithmetic_side_effects = "warn" as_conversions = "warn" as_underscore = "warn" diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 32ec19c6..c7c2acf5 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -119,7 +119,7 @@ impl PduEvent { // deliberately allowing for the possibility of negative age let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into(); let then: i128 = self.origin_server_ts.into(); - let this_age: i128 = now.saturating_sub(then); + let this_age = now.saturating_sub(then); unsigned.insert("age".to_owned(), to_raw_value(&this_age).unwrap()); self.unsigned = Some(to_raw_value(&unsigned).expect("unsigned is valid")); diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 4d5c132f..d940c514 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -189,7 +189,7 @@ impl super::Service { .services .client .well_known - .get(&format!("https://{dest}/.well-known/matrix/server")) + .get(format!("https://{dest}/.well-known/matrix/server")) .send() .await; From 95121b6e050ed826bcfde25dc64a8d3eac8c5ec9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 28 Jul 2024 01:57:33 +0000 Subject: [PATCH 0602/2091] use single-line codeblock for specific features lists Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 34b0f0a0..de6ad98a 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -35,13 +35,13 @@ pub(super) async fn list_features( }; if enabled && !available { let features = info::rustc::features().join(delim); - let out = format!("```\n{features}\n```"); + let out = format!("`\n{features}\n`"); return Ok(RoomMessageEventContent::text_markdown(out)); } if available && !enabled { let features = info::cargo::features().join(delim); - let out = format!("```\n{features}\n```"); + let out = format!("`\n{features}\n`"); return Ok(RoomMessageEventContent::text_markdown(out)); } From a95ded167fea22be2f5b372367929a0e3726be59 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 28 Jul 2024 04:02:45 +0000 Subject: [PATCH 0603/2091] fix type complexity Signed-off-by: Jason Volk --- src/service/rooms/lazy_loading/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 64764198..0a9d4cf2 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -12,17 +12,19 @@ use ruma::{DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, RoomId, UserId}; use self::data::Data; pub struct Service { + pub lazy_load_waiting: Mutex, db: Data, - - #[allow(clippy::type_complexity)] - pub lazy_load_waiting: Mutex>>, } +type LazyLoadWaiting = HashMap; +type LazyLoadWaitingKey = (OwnedUserId, OwnedDeviceId, OwnedRoomId, PduCount); +type LazyLoadWaitingVal = HashSet; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), lazy_load_waiting: Mutex::new(HashMap::new()), + db: Data::new(args.db), })) } From a74613be9640e893690bce8e8dfde59682e09508 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 28 Jul 2024 06:17:43 +0000 Subject: [PATCH 0604/2091] improve database repair/shutdown log messages Signed-off-by: Jason Volk --- src/database/engine.rs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index d02ecf58..bf172551 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -1,10 +1,11 @@ use std::{ collections::{BTreeSet, HashMap}, fmt::Write, + path::PathBuf, sync::{atomic::AtomicU32, Arc, Mutex, RwLock}, }; -use conduit::{debug, error, info, utils::time::rfc2822_from_seconds, warn, Result, Server}; +use conduit::{debug, error, info, utils::time::rfc2822_from_seconds, warn, Err, Result, Server}; use rocksdb::{ backup::{BackupEngine, BackupEngineOptions}, perf::get_memory_usage_stats, @@ -50,10 +51,7 @@ impl Engine { let load_time = std::time::Instant::now(); if config.rocksdb_repair { - warn!("Starting database repair. This may take a long time..."); - if let Err(e) = Db::repair(&db_opts, &config.database_path) { - error!("Repair failed: {e:?}"); - } + repair(&db_opts, &config.database_path)?; } debug!("Listing column families in database"); @@ -77,6 +75,7 @@ impl Engine { let db = res.or_else(or_else)?; info!( + columns = cfs.len(), sequence = %db.latest_sequence_number(), time = ?load_time.elapsed(), "Opened database." @@ -243,6 +242,16 @@ impl Engine { } } +pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> { + warn!("Starting database repair. This may take a long time..."); + match Db::repair(db_opts, path) { + Ok(()) => info!("Database repair successful."), + Err(e) => return Err!("Repair failed: {e:?}"), + } + + Ok(()) +} + impl Drop for Engine { #[cold] fn drop(&mut self) { @@ -259,5 +268,10 @@ impl Drop for Engine { debug!("Joining background threads..."); self.env.join_all_threads(); + + info!( + sequence = %self.db.latest_sequence_number(), + "Closing database..." + ); } } From e37ac56dbaba3ee3bb579e63ffc22ef154ef6efe Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 28 Jul 2024 07:22:51 +0000 Subject: [PATCH 0605/2091] use weak references to break services dependency cycle Signed-off-by: Jason Volk --- src/service/service.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/service/service.rs b/src/service/service.rs index bf3b891b..d0d8d940 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -3,7 +3,7 @@ use std::{ collections::BTreeMap, fmt::Write, ops::Deref, - sync::{Arc, OnceLock, RwLock}, + sync::{Arc, OnceLock, RwLock, Weak}, }; use async_trait::async_trait; @@ -53,7 +53,7 @@ pub(crate) struct Args<'a> { /// Circular-dependencies between services require this indirection. pub(crate) struct Dep { dep: OnceLock>, - service: Arc, + service: Weak, name: &'static str, } @@ -65,8 +65,14 @@ impl Deref for Dep { /// Dereference a dependency. The dependency must be ready or panics. fn deref(&self) -> &Self::Target { - self.dep - .get_or_init(|| require::(&self.service, self.name)) + self.dep.get_or_init(|| { + let service = self + .service + .upgrade() + .expect("services map exists for dependency initialization."); + + require::(&service, self.name) + }) } } @@ -75,7 +81,7 @@ impl Args<'_> { pub(crate) fn depend(&self, name: &'static str) -> Dep { Dep:: { dep: OnceLock::new(), - service: self.service.clone(), + service: Arc::downgrade(self.service), name, } } From 15126ee1b2ee41014a84dae8d41a019ecdb80c5f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 28 Jul 2024 09:03:17 +0000 Subject: [PATCH 0606/2091] additional weak references where applicable Signed-off-by: Jason Volk --- src/service/admin/mod.rs | 19 ++++++++---- src/service/manager.rs | 3 +- src/service/service.rs | 63 +++++++++++++++++++++++----------------- src/service/services.rs | 29 +++++++++++------- 4 files changed, 70 insertions(+), 44 deletions(-) diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 71f3b73e..00d6597e 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -5,7 +5,7 @@ mod grant; use std::{ future::Future, pin::Pin, - sync::{Arc, RwLock as StdRwLock}, + sync::{Arc, RwLock as StdRwLock, Weak}, }; use async_trait::async_trait; @@ -41,7 +41,7 @@ struct Services { timeline: Dep, state: Dep, state_cache: Dep, - services: StdRwLock>>, + services: StdRwLock>>, } #[derive(Debug)] @@ -174,7 +174,14 @@ impl Service { } async fn process_command(&self, command: CommandInput) -> CommandResult { - let Some(services) = self.services.services.read().expect("locked").clone() else { + let Some(services) = self + .services + .services + .read() + .expect("locked") + .as_ref() + .and_then(Weak::upgrade) + else { return Err!("Services self-reference not initialized."); }; @@ -365,7 +372,9 @@ impl Service { /// Sets the self-reference to crate::Services which will provide context to /// the admin commands. - pub(super) fn set_services(&self, services: Option>) { - *self.services.services.write().expect("locked for writing") = services; + pub(super) fn set_services(&self, services: &Option>) { + let receiver = &mut *self.services.services.write().expect("locked for writing"); + let weak = services.as_ref().map(Arc::downgrade); + *receiver = weak; } } diff --git a/src/service/manager.rs b/src/service/manager.rs index 087fd3fa..42260bb3 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -60,7 +60,8 @@ impl Manager { .read() .expect("locked for reading") .values() - .map(|v| v.0.clone()) + .map(|val| val.0.upgrade()) + .map(|arc| arc.expect("services available for manager startup")) .collect(); debug!("Starting service workers..."); diff --git a/src/service/service.rs b/src/service/service.rs index d0d8d940..0b9bc76c 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -57,8 +57,10 @@ pub(crate) struct Dep { name: &'static str, } -pub(crate) type Map = RwLock>; -pub(crate) type MapVal = (Arc, Arc); +pub(crate) type Map = RwLock; +pub(crate) type MapType = BTreeMap; +pub(crate) type MapVal = (Weak, Weak); +pub(crate) type MapKey = String; impl Deref for Dep { type Target = Arc; @@ -76,9 +78,9 @@ impl Deref for Dep { } } -impl Args<'_> { +impl<'a> Args<'a> { /// Create a lazy-reference to a service when constructing another Service. - pub(crate) fn depend(&self, name: &'static str) -> Dep { + pub(crate) fn depend(&'a self, name: &'static str) -> Dep { Dep:: { dep: OnceLock::new(), service: Arc::downgrade(self.service), @@ -88,48 +90,55 @@ impl Args<'_> { /// Create a reference immediately to a service when constructing another /// Service. The other service must be constructed. - pub(crate) fn require(&self, name: &str) -> Arc { require::(self.service, name) } + pub(crate) fn require(&'a self, name: &'static str) -> Arc { + require::(self.service, name) + } } /// Reference a Service by name. Panics if the Service does not exist or was /// incorrectly cast. -pub(crate) fn require(map: &Map, name: &str) -> Arc { +pub(crate) fn require<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(map: &'b Map, name: &'a str) -> Arc { try_get::(map, name) .inspect_err(inspect_log) .expect("Failure to reference service required by another service.") } -/// Reference a Service by name. Returns Err if the Service does not exist or -/// was incorrectly cast. -pub(crate) fn try_get(map: &Map, name: &str) -> Result> { - map.read() - .expect("locked for reading") - .get(name) - .map_or_else( - || Err!("Service {name:?} does not exist or has not been built yet."), - |(_, s)| { - s.clone() - .downcast::() - .map_err(|_| err!("Service {name:?} must be correctly downcast.")) - }, - ) -} - /// Reference a Service by name. Returns None if the Service does not exist, but /// panics if incorrectly cast. /// /// # Panics /// Incorrect type is not a silent failure (None) as the type never has a reason /// to be incorrect. -pub(crate) fn get(map: &Map, name: &str) -> Option> { +pub(crate) fn get<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(map: &'b Map, name: &'a str) -> Option> { map.read() .expect("locked for reading") .get(name) .map(|(_, s)| { - s.clone() - .downcast::() - .expect("Service must be correctly downcast.") - }) + s.upgrade().map(|s| { + s.downcast::() + .expect("Service must be correctly downcast.") + }) + })? +} + +/// Reference a Service by name. Returns Err if the Service does not exist or +/// was incorrectly cast. +pub(crate) fn try_get<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(map: &'b Map, name: &'a str) -> Result> { + map.read() + .expect("locked for reading") + .get(name) + .map_or_else( + || Err!("Service {name:?} does not exist or has not been built yet."), + |(_, s)| { + s.upgrade().map_or_else( + || Err!("Service {name:?} no longer exists."), + |s| { + s.downcast::() + .map_err(|_| err!("Service {name:?} must be correctly downcast.")) + }, + ) + }, + ) } /// Utility for service implementations; see Service::name() in the trait. diff --git a/src/service/services.rs b/src/service/services.rs index b283db6c..74e11d0b 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -109,7 +109,7 @@ impl Services { pub async fn start(self: &Arc) -> Result> { debug_info!("Starting services..."); - self.admin.set_services(Some(Arc::clone(self))); + self.admin.set_services(&Some(Arc::clone(self))); globals::migrations::migrations(self).await?; self.manager .lock() @@ -131,7 +131,7 @@ impl Services { manager.stop().await; } - self.admin.set_services(None); + self.admin.set_services(&None); debug_info!("Services shutdown complete."); } @@ -146,7 +146,9 @@ impl Services { pub async fn clear_cache(&self) { for (service, ..) in self.service.read().expect("locked for reading").values() { - service.clear_cache(); + if let Some(service) = service.upgrade() { + service.clear_cache(); + } } //TODO @@ -161,7 +163,9 @@ impl Services { pub async fn memory_usage(&self) -> Result { let mut out = String::new(); for (service, ..) in self.service.read().expect("locked for reading").values() { - service.memory_usage(&mut out)?; + if let Some(service) = service.upgrade() { + service.memory_usage(&mut out)?; + } } //TODO @@ -179,27 +183,30 @@ impl Services { fn interrupt(&self) { debug!("Interrupting services..."); - for (name, (service, ..)) in self.service.read().expect("locked for reading").iter() { - trace!("Interrupting {name}"); - service.interrupt(); + if let Some(service) = service.upgrade() { + trace!("Interrupting {name}"); + service.interrupt(); + } } } - pub fn try_get(&self, name: &str) -> Result> { + pub fn try_get<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(&'b self, name: &'a str) -> Result> { service::try_get::(&self.service, name) } - pub fn get(&self, name: &str) -> Option> { service::get::(&self.service, name) } + pub fn get<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(&'b self, name: &'a str) -> Option> { + service::get::(&self.service, name) + } } +#[allow(clippy::needless_pass_by_value)] fn add_service(map: &Arc, s: Arc, a: Arc) { let name = s.name(); let len = map.read().expect("locked for reading").len(); trace!("built service #{len}: {name:?}"); - map.write() .expect("locked for writing") - .insert(name.to_owned(), (s, a)); + .insert(name.to_owned(), (Arc::downgrade(&s), Arc::downgrade(&a))); } From 5c258f41c886633d8ebdd5ddce72fe96af76fe2e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 28 Jul 2024 11:14:10 +0000 Subject: [PATCH 0607/2091] fixes for modules Signed-off-by: Jason Volk --- src/core/mods/new.rs | 4 ++-- src/main/mods.rs | 54 +++++++++++++++++++++++++++++++++----------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/src/core/mods/new.rs b/src/core/mods/new.rs index 99d7756a..77d89af4 100644 --- a/src/core/mods/new.rs +++ b/src/core/mods/new.rs @@ -1,7 +1,7 @@ use std::ffi::OsStr; use super::{path, Library}; -use crate::{Error, Result}; +use crate::{Err, Result}; const OPEN_FLAGS: i32 = libloading::os::unix::RTLD_LAZY | libloading::os::unix::RTLD_GLOBAL; @@ -16,7 +16,7 @@ pub fn from_path(path: &OsStr) -> Result { let lib = unsafe { Library::open(Some(path), OPEN_FLAGS) }; if let Err(e) = lib { let name = path::to_name(path)?; - return Err(Error::Err(format!("Loading module {name:?} failed: {e}"))); + return Err!("Loading module {name:?} failed: {e}"); } Ok(lib.expect("module loaded")) diff --git a/src/main/mods.rs b/src/main/mods.rs index f39527f7..e377e71c 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -1,18 +1,27 @@ #![cfg(conduit_mods)] +#[no_link] +extern crate conduit_service; + use std::{ future::Future, pin::Pin, sync::{atomic::Ordering, Arc}, }; -use conduit::{mods, Error, Result}; -use tracing::{debug, error}; +use conduit::{debug, error, mods, Error, Result}; +use conduit_service::Services; use crate::Server; -type RunFuncResult = Pin>>>; -type RunFuncProto = fn(&Arc) -> RunFuncResult; +type StartFuncResult = Pin>> + Send>>; +type StartFuncProto = fn(&Arc) -> StartFuncResult; + +type RunFuncResult = Pin> + Send>>; +type RunFuncProto = fn(&Arc) -> RunFuncResult; + +type StopFuncResult = Pin> + Send>>; +type StopFuncProto = fn(Arc) -> StopFuncResult; const RESTART_THRESH: &str = "conduit_service"; const MODULE_NAMES: &[&str] = &[ @@ -33,15 +42,25 @@ pub(crate) async fn run(server: &Arc, starts: bool) -> Result<(bool, boo let main_lock = server.mods.read().await; let main_mod = (*main_lock).last().expect("main module loaded"); if starts { - let start = main_mod.get::("start")?; - if let Err(error) = start(&server.server).await { - error!("Starting server: {error}"); - return Err(error); - } + let start = main_mod.get::("start")?; + match start(&server.server).await { + Ok(services) => server.services.lock().await.insert(services), + Err(error) => { + error!("Starting server: {error}"); + return Err(error); + }, + }; } server.server.stopping.store(false, Ordering::Release); let run = main_mod.get::("run")?; - if let Err(error) = run(&server.server).await { + if let Err(error) = run(server + .services + .lock() + .await + .as_ref() + .expect("services initialized")) + .await + { error!("Running server: {error}"); return Err(error); } @@ -49,8 +68,17 @@ pub(crate) async fn run(server: &Arc, starts: bool) -> Result<(bool, boo let stops = !reloads || stale(server).await? <= restart_thresh(); let starts = reloads && stops; if stops { - let stop = main_mod.get::("stop")?; - if let Err(error) = stop(&server.server).await { + let stop = main_mod.get::("stop")?; + if let Err(error) = stop( + server + .services + .lock() + .await + .take() + .expect("services initialized"), + ) + .await + { error!("Stopping server: {error}"); return Err(error); } @@ -103,7 +131,7 @@ pub(crate) async fn close(server: &Arc, force: bool) -> Result) -> usize { let watermark = stale(server).await.unwrap_or(available()); - available() - watermark + available().saturating_sub(watermark) } async fn stale(server: &Arc) -> Result { From ccef1a4c8b96549ab16acd6e2813f9d26ff2e526 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 28 Jul 2024 21:31:35 +0000 Subject: [PATCH 0608/2091] add formal wrapping for api state Signed-off-by: Jason Volk --- src/api/mod.rs | 4 ++-- src/api/router.rs | 6 ++---- src/api/router/state.rs | 22 ++++++++++++++++++++++ src/router/router.rs | 2 +- 4 files changed, 27 insertions(+), 7 deletions(-) create mode 100644 src/api/router/state.rs diff --git a/src/api/mod.rs b/src/api/mod.rs index 7fa70873..00ae35cd 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -10,8 +10,8 @@ extern crate conduit_service as service; pub(crate) use conduit::{debug_info, pdu::PduEvent, utils, Error, Result}; pub(crate) use service::services; -pub use crate::router::State; -pub(crate) use crate::router::{Ruma, RumaResponse}; +pub use self::router::State; +pub(crate) use self::router::{Ruma, RumaResponse}; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/api/router.rs b/src/api/router.rs index d624de32..cff58c32 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -3,8 +3,7 @@ mod auth; mod handler; mod request; mod response; - -use std::sync::Arc; +mod state; use axum::{ response::IntoResponse, @@ -15,11 +14,10 @@ use conduit::{err, Server}; use http::Uri; use self::handler::RouterExt; +pub use self::state::State; pub(super) use self::{args::Args as Ruma, response::RumaResponse}; use crate::{client, server}; -pub type State = Arc; - pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; let router = router diff --git a/src/api/router/state.rs b/src/api/router/state.rs new file mode 100644 index 00000000..b1149d89 --- /dev/null +++ b/src/api/router/state.rs @@ -0,0 +1,22 @@ +use std::{ops::Deref, sync::Arc}; + +use conduit_service::Services; + +#[derive(Clone)] +pub struct State { + services: Arc, +} + +impl State { + pub fn new(services: Arc) -> Self { + Self { + services, + } + } +} + +impl Deref for State { + type Target = Arc; + + fn deref(&self) -> &Self::Target { &self.services } +} diff --git a/src/router/router.rs b/src/router/router.rs index 3527f1e6..e59b088b 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -9,7 +9,7 @@ use ruma::api::client::error::ErrorKind; pub(crate) fn build(services: &Arc) -> Router { let router = Router::::new(); - let state = services.clone(); + let state = State::new(services.clone()); conduit_api::router::build(router, &services.server) .route("/", get(it_works)) From 1f88866612af280b54fb57aa96912742c3f3b11b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 30 Jul 2024 01:25:07 +0000 Subject: [PATCH 0609/2091] optimize api state extractor Signed-off-by: Jason Volk --- src/api/mod.rs | 3 +- src/api/router.rs | 5 ++- src/api/router/state.rs | 73 ++++++++++++++++++++++++++++++++++++----- src/router/layers.rs | 6 ++-- src/router/router.rs | 15 +++++---- src/router/serve/mod.rs | 2 +- 6 files changed, 81 insertions(+), 23 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 00ae35cd..82b857db 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -10,8 +10,7 @@ extern crate conduit_service as service; pub(crate) use conduit::{debug_info, pdu::PduEvent, utils, Error, Result}; pub(crate) use service::services; -pub use self::router::State; -pub(crate) use self::router::{Ruma, RumaResponse}; +pub(crate) use self::router::{Ruma, RumaResponse, State}; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/api/router.rs b/src/api/router.rs index cff58c32..7d6df16c 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -3,7 +3,7 @@ mod auth; mod handler; mod request; mod response; -mod state; +pub mod state; use axum::{ response::IntoResponse, @@ -14,8 +14,7 @@ use conduit::{err, Server}; use http::Uri; use self::handler::RouterExt; -pub use self::state::State; -pub(super) use self::{args::Args as Ruma, response::RumaResponse}; +pub(super) use self::{args::Args as Ruma, response::RumaResponse, state::State}; use crate::{client, server}; pub fn build(router: Router, server: &Server) -> Router { diff --git a/src/api/router/state.rs b/src/api/router/state.rs index b1149d89..24163b3c 100644 --- a/src/api/router/state.rs +++ b/src/api/router/state.rs @@ -2,21 +2,78 @@ use std::{ops::Deref, sync::Arc}; use conduit_service::Services; -#[derive(Clone)] +#[derive(Clone, Copy)] pub struct State { + services: *const Services, +} + +pub struct Guard { services: Arc, } -impl State { - pub fn new(services: Arc) -> Self { - Self { - services, - } +pub fn create(services: Arc) -> (State, Guard) { + let state = State { + services: Arc::into_raw(services.clone()), + }; + + let guard = Guard { + services, + }; + + (state, guard) +} + +impl Drop for Guard { + fn drop(&mut self) { + let ptr = Arc::as_ptr(&self.services); + // SAFETY: Parity with Arc::into_raw() called in create(). This revivifies the + // Arc lost to State so it can be dropped, otherwise Services will leak. + let arc = unsafe { Arc::from_raw(ptr) }; + debug_assert!( + Arc::strong_count(&arc) > 1, + "Services usually has more than one reference and is not dropped here" + ); } } impl Deref for State { - type Target = Arc; + type Target = Services; - fn deref(&self) -> &Self::Target { &self.services } + fn deref(&self) -> &Self::Target { + deref(&self.services).expect("dereferenced Services pointer in State must not be null") + } +} + +/// SAFETY: State is a thin wrapper containing a raw const pointer to Services +/// in lieu of an Arc. Services is internally threadsafe. If State contains +/// additional fields this notice should be reevaluated. +unsafe impl Send for State {} + +/// SAFETY: State is a thin wrapper containing a raw const pointer to Services +/// in lieu of an Arc. Services is internally threadsafe. If State contains +/// additional fields this notice should be reevaluated. +unsafe impl Sync for State {} + +fn deref(services: &*const Services) -> Option<&Services> { + // SAFETY: We replaced Arc with *const Services in State. This is + // worth about 10 clones (20 reference count updates) for each request handled. + // Though this is not an incredibly large quantity, it's woefully unnecessary + // given the context as explained below; though it is not currently known to be + // a performance bottleneck, the front-line position justifies preempting it. + // + // Services is created prior to the axum/tower stack and Router, and prior + // to serving any requests through the handlers in this crate. It is then + // dropped only after all requests have completed, the listening sockets + // have been closed, axum/tower has been dropped. Thus Services is + // expected to live at least as long as any instance of State, making the + // constant updates to the prior Arc unnecessary to keep Services alive. + // + // Nevertheless if it is possible to accomplish this by annotating State + // with a lifetime to hold a reference (and be aware I have made a + // significant effort trying to make this work) this unsafety may not be + // necessary. It is either very difficult or impossible to get a + // lifetime'ed reference through Router / RumaHandler; though it is + // possible to pass a reference through axum's `with_state()` in trivial + // configurations as the only requirement of a State is Clone. + unsafe { services.as_ref() } } diff --git a/src/router/layers.rs b/src/router/layers.rs index 2b143666..7779260d 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -6,6 +6,7 @@ use axum::{ }; use axum_client_ip::SecureClientIpSource; use conduit::{error, Result, Server}; +use conduit_api::router::state::Guard; use conduit_service::Services; use http::{ header::{self, HeaderName}, @@ -35,7 +36,7 @@ const CONDUWUIT_CSP: &[&str] = &[ const CONDUWUIT_PERMISSIONS_POLICY: &[&str] = &["interest-cohort=()", "browsing-topics=()"]; -pub(crate) fn build(services: &Arc) -> Result { +pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { let server = &services.server; let layers = ServiceBuilder::new(); @@ -85,7 +86,8 @@ pub(crate) fn build(services: &Arc) -> Result { .layer(body_limit_layer(server)) .layer(CatchPanicLayer::custom(catch_panic)); - Ok(router::build(services).layer(layers)) + let (router, guard) = router::build(services); + Ok((router.layer(layers), guard)) } #[cfg(any(feature = "zstd_compression", feature = "gzip_compression", feature = "brotli_compression"))] diff --git a/src/router/router.rs b/src/router/router.rs index e59b088b..31b3f3e4 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -2,19 +2,20 @@ use std::sync::Arc; use axum::{response::IntoResponse, routing::get, Router}; use conduit::Error; -use conduit_api::State; +use conduit_api::router::{state, state::Guard}; use conduit_service::Services; use http::{StatusCode, Uri}; use ruma::api::client::error::ErrorKind; -pub(crate) fn build(services: &Arc) -> Router { - let router = Router::::new(); - let state = State::new(services.clone()); - - conduit_api::router::build(router, &services.server) +pub(crate) fn build(services: &Arc) -> (Router, Guard) { + let router = Router::::new(); + let (state, guard) = state::create(services.clone()); + let router = conduit_api::router::build(router, &services.server) .route("/", get(it_works)) .fallback(not_found) - .with_state(state) + .with_state(state); + + (router, guard) } async fn not_found(_uri: Uri) -> impl IntoResponse { diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index 58bf2de8..9e171008 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -18,7 +18,7 @@ pub(super) async fn serve( let server = &services.server; let config = &server.config; let addrs = config.get_bind_addrs(); - let app = layers::build(&services)?; + let (app, _guard) = layers::build(&services)?; if cfg!(unix) && config.unix_socket_path.is_some() { unix::serve(server, app, shutdown).await From f632b06e6d33d0502e6ae67b2a6801d85a03b484 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 30 Jul 2024 02:28:25 +0000 Subject: [PATCH 0610/2091] use Services instead of Server for middleware function state Signed-off-by: Jason Volk --- src/router/layers.rs | 4 ++-- src/router/request.rs | 9 ++++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index 7779260d..2567fe08 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -48,7 +48,7 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { let layers = layers .sensitive_headers([header::AUTHORIZATION]) - .layer(axum::middleware::from_fn_with_state(Arc::clone(server), request::spawn)) + .layer(axum::middleware::from_fn_with_state(Arc::clone(services), request::spawn)) .layer( TraceLayer::new_for_http() .make_span_with(tracing_span::<_>) @@ -56,7 +56,7 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { .on_request(DefaultOnRequest::new().level(Level::TRACE)) .on_response(DefaultOnResponse::new().level(Level::DEBUG)), ) - .layer(axum::middleware::from_fn_with_state(Arc::clone(server), request::handle)) + .layer(axum::middleware::from_fn_with_state(Arc::clone(services), request::handle)) .layer(SecureClientIpSource::ConnectInfo.into_extension()) .layer(SetResponseHeaderLayer::if_not_present( HeaderName::from_static("origin-agent-cluster"), // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin-Agent-Cluster diff --git a/src/router/request.rs b/src/router/request.rs index 4d5c61e7..271d9573 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -4,13 +4,15 @@ use axum::{ extract::State, response::{IntoResponse, Response}, }; -use conduit::{debug, debug_error, debug_warn, defer, err, error, trace, Result, Server}; +use conduit::{debug, debug_error, debug_warn, defer, err, error, trace, Result}; +use conduit_service::Services; use http::{Method, StatusCode, Uri}; #[tracing::instrument(skip_all, level = "debug")] pub(crate) async fn spawn( - State(server): State>, req: http::Request, next: axum::middleware::Next, + State(services): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { + let server = &services.server; if !server.running() { debug_warn!("unavailable pending shutdown"); return Err(StatusCode::SERVICE_UNAVAILABLE); @@ -34,8 +36,9 @@ pub(crate) async fn spawn( #[tracing::instrument(skip_all, name = "handle")] pub(crate) async fn handle( - State(server): State>, req: http::Request, next: axum::middleware::Next, + State(services): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { + let server = &services.server; if !server.running() { debug_warn!( method = %req.method(), From a0a002b17fda2028afbc21585351d3399592a01f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 30 Jul 2024 02:06:15 +0000 Subject: [PATCH 0611/2091] disable legacy media compat by default; cleanse directory when disabled Signed-off-by: Jason Volk --- src/core/config/mod.rs | 2 +- src/service/globals/migrations.rs | 32 +++++++++++++++++++++++++++++-- src/service/media/mod.rs | 13 +++++++++---- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index a9443fd2..6ccb0559 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -287,7 +287,7 @@ pub struct Config { #[serde(default = "true_fn")] pub media_startup_check: bool, - #[serde(default = "true_fn")] + #[serde(default)] pub media_compat_file_link: bool, #[serde(default = "Vec::new")] pub prevent_media_downloads_from: Vec, diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 2fe22b0e..5e078595 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -743,6 +743,7 @@ async fn db_lt_13(services: &Services) -> Result<()> { /// again. async fn migrate_sha256_media(services: &Services) -> Result<()> { let db = &services.db; + let config = &services.server.config; warn!("Migrating legacy base64 file names to sha256 file names"); let mediaid_file = &db["mediaid_file"]; @@ -759,7 +760,9 @@ async fn migrate_sha256_media(services: &Services) -> Result<()> { for (old_path, path) in changes { if old_path.exists() { tokio::fs::rename(&old_path, &path).await?; - tokio::fs::symlink(&path, &old_path).await?; + if config.media_compat_file_link { + tokio::fs::symlink(&path, &old_path).await?; + } } } @@ -822,8 +825,14 @@ async fn handle_media_check( let (mediaid_file, mediaid_user) = dbs; - let old_exists = files.contains(old_path); let new_exists = files.contains(new_path); + let old_exists = files.contains(old_path); + let old_is_symlink = || async { + tokio::fs::symlink_metadata(old_path) + .await + .map_or(false, |md| md.is_symlink()) + }; + if !old_exists && !new_exists { error!( media_id = ?encode_key(key), ?new_path, ?old_path, @@ -849,10 +858,29 @@ async fn handle_media_check( "Legacy media found without sha256 migration. Fixing..." ); + debug_assert!( + old_is_symlink().await, + "Legacy media not expected to be a symlink without an existing sha256 migration." + ); + tokio::fs::rename(&old_path, &new_path).await?; tokio::fs::symlink(&new_path, &old_path).await?; } + if !config.media_compat_file_link && old_exists && old_is_symlink().await { + debug_warn!( + media_id = ?encode_key(key), ?new_path, ?old_path, + "Legacy link found but compat disabled. Cleansing symlink..." + ); + + debug_assert!( + new_exists, + "sha256 migration into new file expected prior to cleaning legacy symlink here." + ); + + tokio::fs::remove_file(&old_path).await?; + } + Ok(()) } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index d5d518dc..1faa9768 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -6,7 +6,7 @@ use std::{path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, debug_error, err, error, utils, utils::MutexMap, Err, Result, Server}; +use conduit::{debug, debug_error, err, error, trace, utils, utils::MutexMap, Err, Result, Server}; use data::{Data, Metadata}; use ruma::{OwnedMxcUri, OwnedUserId}; use serde::Serialize; @@ -99,10 +99,15 @@ impl Service { pub async fn delete(&self, mxc: &str) -> Result<()> { if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc) { for key in keys { - self.remove_media_file(&key).await?; + trace!(?mxc, ?key, "Deleting from filesystem"); + if let Err(e) = self.remove_media_file(&key).await { + error!(?mxc, ?key, "Failed to remove media file: {e}"); + } - debug!("Deleting MXC {mxc} from database"); - self.db.delete_file_mxc(mxc)?; + trace!(?mxc, ?key, "Deleting from database"); + if let Err(e) = self.db.delete_file_mxc(mxc) { + error!(?mxc, ?key, "Failed to remove media from database: {e}"); + } } Ok(()) From 24884c3b9b919c2d646ca894abfc67610d83941b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 30 Jul 2024 05:29:32 +0000 Subject: [PATCH 0612/2091] move media helpers from api to services Signed-off-by: Jason Volk --- Cargo.lock | 3 +- src/api/Cargo.toml | 2 - src/api/client/media.rs | 236 +----------------------------- src/core/mod.rs | 2 +- src/core/utils/mod.rs | 1 + src/service/Cargo.toml | 1 + src/service/media/data.rs | 2 +- src/service/media/mod.rs | 49 ++----- src/service/media/preview.rs | 275 +++++++++++++++++++++++++++++++++++ 9 files changed, 296 insertions(+), 275 deletions(-) create mode 100644 src/service/media/preview.rs diff --git a/Cargo.lock b/Cargo.lock index 83c443cc..a8da7d7e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -653,7 +653,6 @@ dependencies = [ "http 1.1.0", "http-body-util", "hyper 1.4.1", - "image", "ipaddress", "jsonwebtoken", "log", @@ -666,7 +665,6 @@ dependencies = [ "sha-1", "tokio", "tracing", - "webpage", ] [[package]] @@ -802,6 +800,7 @@ dependencies = [ "tokio", "tracing", "url", + "webpage", ] [[package]] diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 356adc1f..f537fd5f 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -47,7 +47,6 @@ hmac.workspace = true http.workspace = true http-body-util.workspace = true hyper.workspace = true -image.workspace = true ipaddress.workspace = true jsonwebtoken.workspace = true log.workspace = true @@ -60,7 +59,6 @@ serde.workspace = true sha-1.workspace = true tokio.workspace = true tracing.workspace = true -webpage.workspace = true [lints] workspace = true diff --git a/src/api/client/media.rs b/src/api/client/media.rs index f0afa290..5c83c17f 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -1,11 +1,11 @@ #![allow(deprecated)] -use std::{io::Cursor, time::Duration}; +use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - debug, debug_warn, error, + debug_warn, error, utils::{ self, content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename}, @@ -13,9 +13,6 @@ use conduit::{ }, warn, Error, Result, }; -use image::io::Reader as ImgReader; -use ipaddress::IPAddress; -use reqwest::Url; use ruma::api::client::{ error::{ErrorKind, RetryAfter}, media::{ @@ -24,16 +21,12 @@ use ruma::api::client::{ }, }; use service::{ - media::{FileMeta, UrlPreviewData}, + media::{FileMeta, MXC_LENGTH}, Services, }; -use webpage::HTML; use crate::{Ruma, RumaResponse}; -/// generated MXC ID (`media-id`) length -const MXC_LENGTH: usize = 32; - /// Cache control for immutable objects const CACHE_CONTROL_IMMUTABLE: &str = "public,max-age=31536000,immutable"; @@ -76,12 +69,12 @@ pub(crate) async fn get_media_preview_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let url = &body.url; - if !url_preview_allowed(&services, url) { + if !services.media.url_preview_allowed(url) { warn!(%sender_user, "URL is not allowed to be previewed: {url}"); return Err(Error::BadRequest(ErrorKind::forbidden(), "URL is not allowed to be previewed")); } - match get_url_preview(&services, url).await { + match services.media.get_url_preview(url).await { Ok(preview) => { let res = serde_json::value::to_raw_value(&preview).map_err(|e| { error!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}"); @@ -553,222 +546,3 @@ async fn get_remote_content( cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()), }) } - -async fn download_image(services: &Services, client: &reqwest::Client, url: &str) -> Result { - let image = client.get(url).send().await?.bytes().await?; - let mxc = format!("mxc://{}/{}", services.globals.server_name(), utils::random_string(MXC_LENGTH)); - - services - .media - .create(None, &mxc, None, None, &image) - .await?; - - let (width, height) = match ImgReader::new(Cursor::new(&image)).with_guessed_format() { - Err(_) => (None, None), - Ok(reader) => match reader.into_dimensions() { - Err(_) => (None, None), - Ok((width, height)) => (Some(width), Some(height)), - }, - }; - - Ok(UrlPreviewData { - image: Some(mxc), - image_size: Some(image.len()), - image_width: width, - image_height: height, - ..Default::default() - }) -} - -async fn download_html(services: &Services, client: &reqwest::Client, url: &str) -> Result { - let mut response = client.get(url).send().await?; - - let mut bytes: Vec = Vec::new(); - while let Some(chunk) = response.chunk().await? { - bytes.extend_from_slice(&chunk); - if bytes.len() > services.globals.url_preview_max_spider_size() { - debug!( - "Response body from URL {} exceeds url_preview_max_spider_size ({}), not processing the rest of the \ - response body and assuming our necessary data is in this range.", - url, - services.globals.url_preview_max_spider_size() - ); - break; - } - } - let body = String::from_utf8_lossy(&bytes); - let Ok(html) = HTML::from_string(body.to_string(), Some(url.to_owned())) else { - return Err(Error::BadRequest(ErrorKind::Unknown, "Failed to parse HTML")); - }; - - let mut data = match html.opengraph.images.first() { - None => UrlPreviewData::default(), - Some(obj) => download_image(services, client, &obj.url).await?, - }; - - let props = html.opengraph.properties; - - /* use OpenGraph title/description, but fall back to HTML if not available */ - data.title = props.get("title").cloned().or(html.title); - data.description = props.get("description").cloned().or(html.description); - - Ok(data) -} - -async fn request_url_preview(services: &Services, url: &str) -> Result { - if let Ok(ip) = IPAddress::parse(url) { - if !services.globals.valid_cidr_range(&ip) { - return Err(Error::BadServerResponse("Requesting from this address is forbidden")); - } - } - - let client = &services.client.url_preview; - let response = client.head(url).send().await?; - - if let Some(remote_addr) = response.remote_addr() { - if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { - if !services.globals.valid_cidr_range(&ip) { - return Err(Error::BadServerResponse("Requesting from this address is forbidden")); - } - } - } - - let Some(content_type) = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|x| x.to_str().ok()) - else { - return Err(Error::BadRequest(ErrorKind::Unknown, "Unknown Content-Type")); - }; - let data = match content_type { - html if html.starts_with("text/html") => download_html(services, client, url).await?, - img if img.starts_with("image/") => download_image(services, client, url).await?, - _ => return Err(Error::BadRequest(ErrorKind::Unknown, "Unsupported Content-Type")), - }; - - services.media.set_url_preview(url, &data).await?; - - Ok(data) -} - -async fn get_url_preview(services: &Services, url: &str) -> Result { - if let Some(preview) = services.media.get_url_preview(url).await { - return Ok(preview); - } - - // ensure that only one request is made per URL - let _request_lock = services.media.url_preview_mutex.lock(url).await; - - match services.media.get_url_preview(url).await { - Some(preview) => Ok(preview), - None => request_url_preview(services, url).await, - } -} - -fn url_preview_allowed(services: &Services, url_str: &str) -> bool { - let url: Url = match Url::parse(url_str) { - Ok(u) => u, - Err(e) => { - warn!("Failed to parse URL from a str: {}", e); - return false; - }, - }; - - if ["http", "https"] - .iter() - .all(|&scheme| scheme != url.scheme().to_lowercase()) - { - debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url); - return false; - } - - let host = match url.host_str() { - None => { - debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url); - return false; - }, - Some(h) => h.to_owned(), - }; - - let allowlist_domain_contains = services.globals.url_preview_domain_contains_allowlist(); - let allowlist_domain_explicit = services.globals.url_preview_domain_explicit_allowlist(); - let denylist_domain_explicit = services.globals.url_preview_domain_explicit_denylist(); - let allowlist_url_contains = services.globals.url_preview_url_contains_allowlist(); - - if allowlist_domain_contains.contains(&"*".to_owned()) - || allowlist_domain_explicit.contains(&"*".to_owned()) - || allowlist_url_contains.contains(&"*".to_owned()) - { - debug!("Config key contains * which is allowing all URL previews. Allowing URL {}", url); - return true; - } - - if !host.is_empty() { - if denylist_domain_explicit.contains(&host) { - debug!( - "Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)", - &host - ); - return false; - } - - if allowlist_domain_explicit.contains(&host) { - debug!("Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)", &host); - return true; - } - - if allowlist_domain_contains - .iter() - .any(|domain_s| domain_s.contains(&host.clone())) - { - debug!("Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)", &host); - return true; - } - - if allowlist_url_contains - .iter() - .any(|url_s| url.to_string().contains(&url_s.to_string())) - { - debug!("URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)", &host); - return true; - } - - // check root domain if available and if user has root domain checks - if services.globals.url_preview_check_root_domain() { - debug!("Checking root domain"); - match host.split_once('.') { - None => return false, - Some((_, root_domain)) => { - if denylist_domain_explicit.contains(&root_domain.to_owned()) { - debug!( - "Root domain {} is not allowed by url_preview_domain_explicit_denylist (check 1/3)", - &root_domain - ); - return true; - } - - if allowlist_domain_explicit.contains(&root_domain.to_owned()) { - debug!( - "Root domain {} is allowed by url_preview_domain_explicit_allowlist (check 2/3)", - &root_domain - ); - return true; - } - - if allowlist_domain_contains - .iter() - .any(|domain_s| domain_s.contains(&root_domain.to_owned())) - { - debug!( - "Root domain {} is allowed by url_preview_domain_contains_allowlist (check 3/3)", - &root_domain - ); - return true; - } - }, - } - } - } - - false -} diff --git a/src/core/mod.rs b/src/core/mod.rs index 5ed5ea15..9898243b 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -16,7 +16,7 @@ pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; pub use pdu::{PduBuilder, PduCount, PduEvent}; pub use server::Server; -pub use utils::{ctor, dtor}; +pub use utils::{ctor, dtor, implement}; pub use crate as conduit_core; diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 767b65a9..1556646e 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -17,6 +17,7 @@ use std::cmp::{self, Ordering}; pub use ::ctor::{ctor, dtor}; pub use bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}; +pub use conduit_macros::implement; pub use debug::slice_truncated as debug_slice_truncated; pub use hash::calculate_hash; pub use html::Escape as HtmlEscape; diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index c444e5f5..c1d9889e 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -69,6 +69,7 @@ termimad.optional = true tokio.workspace = true tracing.workspace = true url.workspace = true +webpage.workspace = true [lints] workspace = true diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 617ec526..70e010c2 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -4,7 +4,7 @@ use conduit::{debug, debug_info, utils::string_from_bytes, Error, Result}; use database::{Database, Map}; use ruma::api::client::error::ErrorKind; -use crate::media::UrlPreviewData; +use super::preview::UrlPreviewData; pub(crate) struct Data { mediaid_file: Arc, diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 1faa9768..ff3f3dc4 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,4 +1,5 @@ mod data; +mod preview; mod tests; mod thumbnail; @@ -9,13 +10,12 @@ use base64::{engine::general_purpose, Engine as _}; use conduit::{debug, debug_error, err, error, trace, utils, utils::MutexMap, Err, Result, Server}; use data::{Data, Metadata}; use ruma::{OwnedMxcUri, OwnedUserId}; -use serde::Serialize; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt, BufReader}, }; -use crate::{globals, Dep}; +use crate::{client, globals, Dep}; #[derive(Debug)] pub struct FileMeta { @@ -24,43 +24,32 @@ pub struct FileMeta { pub content_disposition: Option, } -#[derive(Serialize, Default)] -pub struct UrlPreviewData { - #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:title"))] - pub title: Option, - #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:description"))] - pub description: Option, - #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image"))] - pub image: Option, - #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:image:size"))] - pub image_size: Option, - #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:width"))] - pub image_width: Option, - #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:height"))] - pub image_height: Option, -} - pub struct Service { - services: Services, + url_preview_mutex: MutexMap, pub(crate) db: Data, - pub url_preview_mutex: MutexMap, + services: Services, } struct Services { server: Arc, + client: Dep, globals: Dep, } +/// generated MXC ID (`media-id`) length +pub const MXC_LENGTH: usize = 32; + #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + url_preview_mutex: MutexMap::new(), + db: Data::new(args.db), services: Services { server: args.server.clone(), + client: args.depend::("client"), globals: args.depend::("globals"), }, - db: Data::new(args.db), - url_preview_mutex: MutexMap::new(), })) } @@ -229,22 +218,6 @@ impl Service { Ok(deletion_count) } - pub async fn get_url_preview(&self, url: &str) -> Option { self.db.get_url_preview(url) } - - /// TODO: use this? - #[allow(dead_code)] - pub async fn remove_url_preview(&self, url: &str) -> Result<()> { - // TODO: also remove the downloaded image - self.db.remove_url_preview(url) - } - - pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> { - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("valid system time"); - self.db.set_url_preview(url, data, now) - } - pub async fn create_media_dir(&self) -> Result<()> { let dir = self.get_media_dir(); Ok(fs::create_dir_all(dir).await?) diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs new file mode 100644 index 00000000..ebfd22e7 --- /dev/null +++ b/src/service/media/preview.rs @@ -0,0 +1,275 @@ +use std::{io::Cursor, time::SystemTime}; + +use conduit::{debug, utils, warn, Error, Result}; +use conduit_core::implement; +use image::ImageReader as ImgReader; +use ipaddress::IPAddress; +use ruma::api::client::error::ErrorKind; +use serde::Serialize; +use url::Url; +use webpage::HTML; + +use super::{Service, MXC_LENGTH}; + +#[derive(Serialize, Default)] +pub struct UrlPreviewData { + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:title"))] + pub title: Option, + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:description"))] + pub description: Option, + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image"))] + pub image: Option, + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "matrix:image:size"))] + pub image_size: Option, + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:width"))] + pub image_width: Option, + #[serde(skip_serializing_if = "Option::is_none", rename(serialize = "og:image:height"))] + pub image_height: Option, +} + +#[implement(Service)] +pub async fn remove_url_preview(&self, url: &str) -> Result<()> { + // TODO: also remove the downloaded image + self.db.remove_url_preview(url) +} + +#[implement(Service)] +pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result<()> { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("valid system time"); + self.db.set_url_preview(url, data, now) +} + +#[implement(Service)] +pub async fn download_image(&self, url: &str) -> Result { + let client = &self.services.client.url_preview; + let image = client.get(url).send().await?.bytes().await?; + let mxc = format!( + "mxc://{}/{}", + self.services.globals.server_name(), + utils::random_string(MXC_LENGTH) + ); + + self.create(None, &mxc, None, None, &image).await?; + + let (width, height) = match ImgReader::new(Cursor::new(&image)).with_guessed_format() { + Err(_) => (None, None), + Ok(reader) => match reader.into_dimensions() { + Err(_) => (None, None), + Ok((width, height)) => (Some(width), Some(height)), + }, + }; + + Ok(UrlPreviewData { + image: Some(mxc), + image_size: Some(image.len()), + image_width: width, + image_height: height, + ..Default::default() + }) +} + +#[implement(Service)] +pub async fn get_url_preview(&self, url: &str) -> Result { + if let Some(preview) = self.db.get_url_preview(url) { + return Ok(preview); + } + + // ensure that only one request is made per URL + let _request_lock = self.url_preview_mutex.lock(url).await; + + match self.db.get_url_preview(url) { + Some(preview) => Ok(preview), + None => self.request_url_preview(url).await, + } +} + +#[implement(Service)] +async fn request_url_preview(&self, url: &str) -> Result { + if let Ok(ip) = IPAddress::parse(url) { + if !self.services.globals.valid_cidr_range(&ip) { + return Err(Error::BadServerResponse("Requesting from this address is forbidden")); + } + } + + let client = &self.services.client.url_preview; + let response = client.head(url).send().await?; + + if let Some(remote_addr) = response.remote_addr() { + if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { + if !self.services.globals.valid_cidr_range(&ip) { + return Err(Error::BadServerResponse("Requesting from this address is forbidden")); + } + } + } + + let Some(content_type) = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|x| x.to_str().ok()) + else { + return Err(Error::BadRequest(ErrorKind::Unknown, "Unknown Content-Type")); + }; + let data = match content_type { + html if html.starts_with("text/html") => self.download_html(url).await?, + img if img.starts_with("image/") => self.download_image(url).await?, + _ => return Err(Error::BadRequest(ErrorKind::Unknown, "Unsupported Content-Type")), + }; + + self.set_url_preview(url, &data).await?; + + Ok(data) +} + +#[implement(Service)] +async fn download_html(&self, url: &str) -> Result { + let client = &self.services.client.url_preview; + let mut response = client.get(url).send().await?; + + let mut bytes: Vec = Vec::new(); + while let Some(chunk) = response.chunk().await? { + bytes.extend_from_slice(&chunk); + if bytes.len() > self.services.globals.url_preview_max_spider_size() { + debug!( + "Response body from URL {} exceeds url_preview_max_spider_size ({}), not processing the rest of the \ + response body and assuming our necessary data is in this range.", + url, + self.services.globals.url_preview_max_spider_size() + ); + break; + } + } + let body = String::from_utf8_lossy(&bytes); + let Ok(html) = HTML::from_string(body.to_string(), Some(url.to_owned())) else { + return Err(Error::BadRequest(ErrorKind::Unknown, "Failed to parse HTML")); + }; + + let mut data = match html.opengraph.images.first() { + None => UrlPreviewData::default(), + Some(obj) => self.download_image(&obj.url).await?, + }; + + let props = html.opengraph.properties; + + /* use OpenGraph title/description, but fall back to HTML if not available */ + data.title = props.get("title").cloned().or(html.title); + data.description = props.get("description").cloned().or(html.description); + + Ok(data) +} + +#[implement(Service)] +pub fn url_preview_allowed(&self, url_str: &str) -> bool { + let url: Url = match Url::parse(url_str) { + Ok(u) => u, + Err(e) => { + warn!("Failed to parse URL from a str: {}", e); + return false; + }, + }; + + if ["http", "https"] + .iter() + .all(|&scheme| scheme != url.scheme().to_lowercase()) + { + debug!("Ignoring non-HTTP/HTTPS URL to preview: {}", url); + return false; + } + + let host = match url.host_str() { + None => { + debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url); + return false; + }, + Some(h) => h.to_owned(), + }; + + let allowlist_domain_contains = self + .services + .globals + .url_preview_domain_contains_allowlist(); + let allowlist_domain_explicit = self + .services + .globals + .url_preview_domain_explicit_allowlist(); + let denylist_domain_explicit = self.services.globals.url_preview_domain_explicit_denylist(); + let allowlist_url_contains = self.services.globals.url_preview_url_contains_allowlist(); + + if allowlist_domain_contains.contains(&"*".to_owned()) + || allowlist_domain_explicit.contains(&"*".to_owned()) + || allowlist_url_contains.contains(&"*".to_owned()) + { + debug!("Config key contains * which is allowing all URL previews. Allowing URL {}", url); + return true; + } + + if !host.is_empty() { + if denylist_domain_explicit.contains(&host) { + debug!( + "Host {} is not allowed by url_preview_domain_explicit_denylist (check 1/4)", + &host + ); + return false; + } + + if allowlist_domain_explicit.contains(&host) { + debug!("Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)", &host); + return true; + } + + if allowlist_domain_contains + .iter() + .any(|domain_s| domain_s.contains(&host.clone())) + { + debug!("Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)", &host); + return true; + } + + if allowlist_url_contains + .iter() + .any(|url_s| url.to_string().contains(&url_s.to_string())) + { + debug!("URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)", &host); + return true; + } + + // check root domain if available and if user has root domain checks + if self.services.globals.url_preview_check_root_domain() { + debug!("Checking root domain"); + match host.split_once('.') { + None => return false, + Some((_, root_domain)) => { + if denylist_domain_explicit.contains(&root_domain.to_owned()) { + debug!( + "Root domain {} is not allowed by url_preview_domain_explicit_denylist (check 1/3)", + &root_domain + ); + return true; + } + + if allowlist_domain_explicit.contains(&root_domain.to_owned()) { + debug!( + "Root domain {} is allowed by url_preview_domain_explicit_allowlist (check 2/3)", + &root_domain + ); + return true; + } + + if allowlist_domain_contains + .iter() + .any(|domain_s| domain_s.contains(&root_domain.to_owned())) + { + debug!( + "Root domain {} is allowed by url_preview_domain_contains_allowlist (check 3/3)", + &root_domain + ); + return true; + } + }, + } + } + } + + false +} From eb94aa70795fa148ec3d6201f0209975c048ce7b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 31 Jul 2024 21:07:21 +0000 Subject: [PATCH 0613/2091] remove dangling sending/resolve unit this was probably un-deleted during a rebase conflict resolution, but remained outside of the mod tree. Signed-off-by: Jason Volk --- src/service/sending/resolve.rs | 343 --------------------------------- 1 file changed, 343 deletions(-) delete mode 100644 src/service/sending/resolve.rs diff --git a/src/service/sending/resolve.rs b/src/service/sending/resolve.rs deleted file mode 100644 index f8d1c51c..00000000 --- a/src/service/sending/resolve.rs +++ /dev/null @@ -1,343 +0,0 @@ -use std::{ - fmt::Debug, - net::{IpAddr, SocketAddr}, -}; - -use conduit::{debug, debug_error, debug_info, debug_warn, trace, Err, Error, Result}; -use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; -use ipaddress::IPAddress; -use ruma::ServerName; - -use crate::{ - resolver::{add_port_to_hostname, get_ip_with_port, CachedDest, CachedOverride, FedDest}, - services, -}; - -#[derive(Clone, Debug)] -pub(crate) struct ActualDest { - pub(crate) dest: FedDest, - pub(crate) host: String, - pub(crate) string: String, - pub(crate) cached: bool, -} - -#[tracing::instrument(skip_all, name = "resolve")] -pub(crate) async fn get_actual_dest(server_name: &ServerName) -> Result { - let cached; - let cached_result = services().resolver.get_cached_destination(server_name); - - let CachedDest { - dest, - host, - .. - } = if let Some(result) = cached_result { - cached = true; - result - } else { - cached = false; - validate_dest(server_name)?; - resolve_actual_dest(server_name, true).await? - }; - - let string = dest.clone().into_https_string(); - Ok(ActualDest { - dest, - host, - string, - cached, - }) -} - -/// Returns: `actual_destination`, host header -/// Implemented according to the specification at -/// Numbers in comments below refer to bullet points in linked section of -/// specification -#[tracing::instrument(skip_all, name = "actual")] -pub async fn resolve_actual_dest(dest: &ServerName, cache: bool) -> Result { - trace!("Finding actual destination for {dest}"); - let mut host = dest.as_str().to_owned(); - let actual_dest = match get_ip_with_port(dest.as_str()) { - Some(host_port) => actual_dest_1(host_port)?, - None => { - if let Some(pos) = dest.as_str().find(':') { - actual_dest_2(dest, cache, pos).await? - } else if let Some(delegated) = request_well_known(dest.as_str()).await? { - actual_dest_3(&mut host, cache, delegated).await? - } else if let Some(overrider) = query_srv_record(dest.as_str()).await? { - actual_dest_4(&host, cache, overrider).await? - } else { - actual_dest_5(dest, cache).await? - } - }, - }; - - // Can't use get_ip_with_port here because we don't want to add a port - // to an IP address if it wasn't specified - let host = if let Ok(addr) = host.parse::() { - FedDest::Literal(addr) - } else if let Ok(addr) = host.parse::() { - FedDest::Named(addr.to_string(), ":8448".to_owned()) - } else if let Some(pos) = host.find(':') { - let (host, port) = host.split_at(pos); - FedDest::Named(host.to_owned(), port.to_owned()) - } else { - FedDest::Named(host, ":8448".to_owned()) - }; - - debug!("Actual destination: {actual_dest:?} hostname: {host:?}"); - Ok(CachedDest { - dest: actual_dest, - host: host.into_uri_string(), - expire: CachedDest::default_expire(), - }) -} - -fn actual_dest_1(host_port: FedDest) -> Result { - debug!("1: IP literal with provided or default port"); - Ok(host_port) -} - -async fn actual_dest_2(dest: &ServerName, cache: bool, pos: usize) -> Result { - debug!("2: Hostname with included port"); - let (host, port) = dest.as_str().split_at(pos); - conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache).await?; - Ok(FedDest::Named(host.to_owned(), port.to_owned())) -} - -async fn actual_dest_3(host: &mut String, cache: bool, delegated: String) -> Result { - debug!("3: A .well-known file is available"); - *host = add_port_to_hostname(&delegated).into_uri_string(); - match get_ip_with_port(&delegated) { - Some(host_and_port) => actual_dest_3_1(host_and_port), - None => { - if let Some(pos) = delegated.find(':') { - actual_dest_3_2(cache, delegated, pos).await - } else { - trace!("Delegated hostname has no port in this branch"); - if let Some(overrider) = query_srv_record(&delegated).await? { - actual_dest_3_3(cache, delegated, overrider).await - } else { - actual_dest_3_4(cache, delegated).await - } - } - }, - } -} - -fn actual_dest_3_1(host_and_port: FedDest) -> Result { - debug!("3.1: IP literal in .well-known file"); - Ok(host_and_port) -} - -async fn actual_dest_3_2(cache: bool, delegated: String, pos: usize) -> Result { - debug!("3.2: Hostname with port in .well-known file"); - let (host, port) = delegated.split_at(pos); - conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache).await?; - Ok(FedDest::Named(host.to_owned(), port.to_owned())) -} - -async fn actual_dest_3_3(cache: bool, delegated: String, overrider: FedDest) -> Result { - debug!("3.3: SRV lookup successful"); - let force_port = overrider.port(); - conditional_query_and_cache_override(&delegated, &overrider.hostname(), force_port.unwrap_or(8448), cache).await?; - if let Some(port) = force_port { - Ok(FedDest::Named(delegated, format!(":{port}"))) - } else { - Ok(add_port_to_hostname(&delegated)) - } -} - -async fn actual_dest_3_4(cache: bool, delegated: String) -> Result { - debug!("3.4: No SRV records, just use the hostname from .well-known"); - conditional_query_and_cache_override(&delegated, &delegated, 8448, cache).await?; - Ok(add_port_to_hostname(&delegated)) -} - -async fn actual_dest_4(host: &str, cache: bool, overrider: FedDest) -> Result { - debug!("4: No .well-known; SRV record found"); - let force_port = overrider.port(); - conditional_query_and_cache_override(host, &overrider.hostname(), force_port.unwrap_or(8448), cache).await?; - if let Some(port) = force_port { - Ok(FedDest::Named(host.to_owned(), format!(":{port}"))) - } else { - Ok(add_port_to_hostname(host)) - } -} - -async fn actual_dest_5(dest: &ServerName, cache: bool) -> Result { - debug!("5: No SRV record found"); - conditional_query_and_cache_override(dest.as_str(), dest.as_str(), 8448, cache).await?; - Ok(add_port_to_hostname(dest.as_str())) -} - -#[tracing::instrument(skip_all, name = "well-known")] -async fn request_well_known(dest: &str) -> Result> { - trace!("Requesting well known for {dest}"); - if !services().resolver.has_cached_override(dest) { - query_and_cache_override(dest, dest, 8448).await?; - } - - let response = services() - .client - .well_known - .get(&format!("https://{dest}/.well-known/matrix/server")) - .send() - .await; - - trace!("response: {:?}", response); - if let Err(e) = &response { - debug!("error: {e:?}"); - return Ok(None); - } - - let response = response?; - if !response.status().is_success() { - debug!("response not 2XX"); - return Ok(None); - } - - let text = response.text().await?; - trace!("response text: {:?}", text); - if text.len() >= 12288 { - debug_warn!("response contains junk"); - return Ok(None); - } - - let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); - - let m_server = body - .get("m.server") - .unwrap_or(&serde_json::Value::Null) - .as_str() - .unwrap_or_default(); - - if ruma::identifiers_validation::server_name::validate(m_server).is_err() { - debug_error!("response content missing or invalid"); - return Ok(None); - } - - debug_info!("{:?} found at {:?}", dest, m_server); - Ok(Some(m_server.to_owned())) -} - -#[inline] -async fn conditional_query_and_cache_override(overname: &str, hostname: &str, port: u16, cache: bool) -> Result<()> { - if cache { - query_and_cache_override(overname, hostname, port).await - } else { - Ok(()) - } -} - -#[tracing::instrument(skip_all, name = "ip")] -async fn query_and_cache_override(overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { - match services() - .resolver - .resolver - .lookup_ip(hostname.to_owned()) - .await - { - Err(e) => handle_resolve_error(&e), - Ok(override_ip) => { - if hostname != overname { - debug_info!("{overname:?} overriden by {hostname:?}"); - } - - services().resolver.set_cached_override( - overname.to_owned(), - CachedOverride { - ips: override_ip.iter().collect(), - port, - expire: CachedOverride::default_expire(), - }, - ); - - Ok(()) - }, - } -} - -#[tracing::instrument(skip_all, name = "srv")] -async fn query_srv_record(hostname: &'_ str) -> Result> { - fn handle_successful_srv(srv: &SrvLookup) -> Option { - srv.iter().next().map(|result| { - FedDest::Named( - result.target().to_string().trim_end_matches('.').to_owned(), - format!(":{}", result.port()), - ) - }) - } - - async fn lookup_srv(hostname: &str) -> Result { - debug!("querying SRV for {:?}", hostname); - let hostname = hostname.trim_end_matches('.'); - services() - .resolver - .resolver - .srv_lookup(hostname.to_owned()) - .await - } - - let hostnames = [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")]; - - for hostname in hostnames { - match lookup_srv(&hostname).await { - Ok(result) => return Ok(handle_successful_srv(&result)), - Err(e) => handle_resolve_error(&e)?, - } - } - - Ok(None) -} - -#[allow(clippy::single_match_else)] -fn handle_resolve_error(e: &ResolveError) -> Result<()> { - use hickory_resolver::error::ResolveErrorKind; - - match *e.kind() { - ResolveErrorKind::NoRecordsFound { - .. - } => { - // Raise to debug_warn if we can find out the result wasn't from cache - debug!("{e}"); - Ok(()) - }, - _ => Err!(error!("DNS {e}")), - } -} - -fn validate_dest(dest: &ServerName) -> Result<()> { - if dest == services().globals.server_name() { - return Err!("Won't send federation request to ourselves"); - } - - if dest.is_ip_literal() || IPAddress::is_valid(dest.host()) { - validate_dest_ip_literal(dest)?; - } - - Ok(()) -} - -fn validate_dest_ip_literal(dest: &ServerName) -> Result<()> { - trace!("Destination is an IP literal, checking against IP range denylist.",); - debug_assert!( - dest.is_ip_literal() || !IPAddress::is_valid(dest.host()), - "Destination is not an IP literal." - ); - let ip = IPAddress::parse(dest.host()).map_err(|e| { - debug_error!("Failed to parse IP literal from string: {}", e); - Error::BadServerResponse("Invalid IP address") - })?; - - validate_ip(&ip)?; - - Ok(()) -} - -pub(crate) fn validate_ip(ip: &IPAddress) -> Result<()> { - if !services().globals.valid_cidr_range(ip) { - return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); - } - - Ok(()) -} From 5b1642f64151d510b5690e8cfe61ee441d2bcf58 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 31 Jul 2024 21:05:15 +0000 Subject: [PATCH 0614/2091] split server_keys from event_handler into service Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 12 +++----- src/api/client/membership.rs | 9 ++---- src/api/router/auth.rs | 3 +- src/api/server/send.rs | 3 +- src/api/server/send_join.rs | 3 +- src/api/server/send_leave.rs | 3 +- src/service/mod.rs | 1 + src/service/rooms/event_handler/mod.rs | 29 ++++++++++--------- src/service/rooms/timeline/mod.rs | 6 ++-- .../signing_keys.rs => server_keys/mod.rs} | 27 ++++++++++++++++- src/service/services.rs | 4 ++- 11 files changed, 61 insertions(+), 39 deletions(-) rename src/service/{rooms/event_handler/signing_keys.rs => server_keys/mod.rs} (96%) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index cf4ab31d..4e04c94a 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -229,8 +229,7 @@ pub(super) async fn get_remote_pdu( debug!("Attempting to fetch homeserver signing keys for {server}"); self.services - .rooms - .event_handler + .server_keys .fetch_required_signing_keys(parsed_pdu.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) .await .unwrap_or_else(|e| { @@ -443,8 +442,7 @@ pub(super) async fn verify_json(&self) -> Result { let pub_key_map = RwLock::new(BTreeMap::new()); self.services - .rooms - .event_handler + .server_keys .fetch_required_signing_keys([&value], &pub_key_map) .await?; @@ -562,8 +560,7 @@ pub(super) async fn force_set_room_state_from_server( info!("Fetching required signing keys for all the state events we got"); self.services - .rooms - .event_handler + .server_keys .fetch_required_signing_keys(events.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) .await?; @@ -682,8 +679,7 @@ pub(super) async fn get_verify_keys( let signature_ids: Vec = Vec::new(); let keys = self .services - .rooms - .event_handler + .server_keys .fetch_signing_keys_for_server(&server_name, signature_ids) .await?; diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index ca7e6b6f..f0d2848f 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -883,8 +883,7 @@ async fn join_room_by_id_helper_remote( info!("Fetching join signing keys"); services - .rooms - .event_handler + .server_keys .fetch_join_signing_keys(&send_join_response, &room_version_id, &pub_key_map) .await?; @@ -1230,8 +1229,7 @@ async fn join_room_by_id_helper_local( drop(state_lock); let pub_key_map = RwLock::new(BTreeMap::new()); services - .rooms - .event_handler + .server_keys .fetch_required_signing_keys([&signed_value], &pub_key_map) .await?; services @@ -1474,8 +1472,7 @@ pub(crate) async fn invite_helper( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Origin field is invalid."))?; services - .rooms - .event_handler + .server_keys .fetch_required_signing_keys([&value], &pub_key_map) .await?; diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 3f08ddba..838c5e79 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -250,8 +250,7 @@ async fn auth_server( }; let keys_result = services - .rooms - .event_handler + .server_keys .fetch_signing_keys_for_server(origin, vec![x_matrix.key.to_string()]) .await; diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 394289a6..15f82faa 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -105,8 +105,7 @@ async fn handle_pdus( let pub_key_map = RwLock::new(BTreeMap::new()); if !parsed_pdus.is_empty() { services - .rooms - .event_handler + .server_keys .fetch_required_signing_keys(parsed_pdus.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) .await .unwrap_or_else(|e| warn!("Could not fetch all signatures for PDUs from {origin}: {e:?}")); diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 17f56383..c4d016f6 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -138,8 +138,7 @@ async fn create_join_event( } services - .rooms - .event_handler + .server_keys .fetch_required_signing_keys([&value], &pub_key_map) .await?; diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index ef4c8c45..e77c5d78 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -150,8 +150,7 @@ async fn create_leave_event( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; services - .rooms - .event_handler + .server_keys .fetch_required_signing_keys([&value], &pub_key_map) .await?; diff --git a/src/service/mod.rs b/src/service/mod.rs index 084bfac8..f588a542 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -18,6 +18,7 @@ pub mod pusher; pub mod resolver; pub mod rooms; pub mod sending; +pub mod server_keys; pub mod transaction_ids; pub mod uiaa; pub mod updates; diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index fd8e2185..bee986de 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1,5 +1,4 @@ mod parse_incoming_pdu; -mod signing_keys; use std::{ collections::{hash_map, BTreeMap, HashMap, HashSet}, @@ -35,7 +34,7 @@ use ruma::{ use tokio::sync::RwLock; use super::state_compressor::CompressedStateEvent; -use crate::{globals, rooms, sending, Dep}; +use crate::{globals, rooms, sending, server_keys, Dep}; pub struct Service { services: Services, @@ -50,6 +49,7 @@ struct Services { metadata: Dep, outlier: Dep, pdu_metadata: Dep, + server_keys: Dep, short: Dep, state: Dep, state_accessor: Dep, @@ -77,6 +77,7 @@ impl crate::Service for Service { auth_chain: args.depend::("rooms::auth_chain"), metadata: args.depend::("rooms::metadata"), outlier: args.depend::("rooms::outlier"), + server_keys: args.depend::("server_keys"), pdu_metadata: args.depend::("rooms::pdu_metadata"), short: args.depend::("rooms::short"), state: args.depend::("rooms::state"), @@ -1187,17 +1188,19 @@ impl Service { // We go through all the signatures we see on the PDUs and their unresolved // dependencies and fetch the corresponding signing keys - self.fetch_required_signing_keys( - events_with_auth_events - .iter() - .flat_map(|(_id, _local_pdu, events)| events) - .map(|(_event_id, event)| event), - pub_key_map, - ) - .await - .unwrap_or_else(|e| { - warn!("Could not fetch all signatures for PDUs from {}: {:?}", origin, e); - }); + self.services + .server_keys + .fetch_required_signing_keys( + events_with_auth_events + .iter() + .flat_map(|(_id, _local_pdu, events)| events) + .map(|(_event_id, event)| event), + pub_key_map, + ) + .await + .unwrap_or_else(|e| { + warn!("Could not fetch all signatures for PDUs from {}: {:?}", origin, e); + }); let mut pdus = Vec::with_capacity(events_with_auth_events.len()); for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 82b1cd0c..ac25b2a8 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -41,7 +41,7 @@ use tokio::sync::RwLock; use self::data::Data; use crate::{ account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms, - rooms::state_compressor::CompressedStateEvent, sending, Dep, + rooms::state_compressor::CompressedStateEvent, sending, server_keys, Dep, }; // Update Relationships @@ -86,6 +86,7 @@ struct Services { pdu_metadata: Dep, read_receipt: Dep, sending: Dep, + server_keys: Dep, user: Dep, pusher: Dep, threads: Dep, @@ -114,6 +115,7 @@ impl crate::Service for Service { pdu_metadata: args.depend::("rooms::pdu_metadata"), read_receipt: args.depend::("rooms::read_receipt"), sending: args.depend::("sending"), + server_keys: args.depend::("server_keys"), user: args.depend::("rooms::user"), pusher: args.depend::("pusher"), threads: args.depend::("rooms::threads"), @@ -1181,7 +1183,7 @@ impl Service { } self.services - .event_handler + .server_keys .fetch_required_signing_keys([&value], pub_key_map) .await?; diff --git a/src/service/rooms/event_handler/signing_keys.rs b/src/service/server_keys/mod.rs similarity index 96% rename from src/service/rooms/event_handler/signing_keys.rs rename to src/service/server_keys/mod.rs index d7cb3a9d..b27279af 100644 --- a/src/service/rooms/event_handler/signing_keys.rs +++ b/src/service/server_keys/mod.rs @@ -1,5 +1,6 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, + sync::Arc, time::{Duration, SystemTime}, }; @@ -21,7 +22,31 @@ use ruma::{ use serde_json::value::RawValue as RawJsonValue; use tokio::sync::{RwLock, RwLockWriteGuard}; -impl super::Service { +use crate::{globals, sending, Dep}; + +pub struct Service { + services: Services, +} + +struct Services { + globals: Dep, + sending: Dep, +} + +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + services: Services { + globals: args.depend::("globals"), + sending: args.depend::("sending"), + }, + })) + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { pub async fn fetch_required_signing_keys<'a, E>( &'a self, events: E, pub_key_map: &RwLock>>, ) -> Result<()> diff --git a/src/service/services.rs b/src/service/services.rs index 74e11d0b..2b9b93d4 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -12,7 +12,7 @@ use tokio::sync::Mutex; use crate::{ account_data, admin, appservice, client, emergency, globals, key_backups, manager::Manager, - media, presence, pusher, resolver, rooms, sending, service, + media, presence, pusher, resolver, rooms, sending, server_keys, service, service::{Args, Map, Service}, transaction_ids, uiaa, updates, users, }; @@ -31,6 +31,7 @@ pub struct Services { pub resolver: Arc, pub rooms: rooms::Service, pub sending: Arc, + pub server_keys: Arc, pub transaction_ids: Arc, pub uiaa: Arc, pub updates: Arc, @@ -94,6 +95,7 @@ impl Services { user: build!(rooms::user::Service), }, sending: build!(sending::Service), + server_keys: build!(server_keys::Service), transaction_ids: build!(transaction_ids::Service), uiaa: build!(uiaa::Service), updates: build!(updates::Service), From 5add9a8c345212670630dfd2144e4bc00b657629 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 1 Aug 2024 08:41:47 +0000 Subject: [PATCH 0615/2091] support field values in err! macro Signed-off-by: Jason Volk --- src/core/debug.rs | 6 +- src/core/error/err.rs | 139 ++++++++++++++++++++++++++++++++------- src/core/error/mod.rs | 3 +- src/core/utils/string.rs | 10 ++- 4 files changed, 127 insertions(+), 31 deletions(-) diff --git a/src/core/debug.rs b/src/core/debug.rs index 63f25d47..3ab9ed0d 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -14,7 +14,7 @@ pub use crate::utils::debug::*; #[macro_export] macro_rules! debug_event { ( $level:expr, $($x:tt)+ ) => { - if cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) { + if $crate::debug::logging() { ::tracing::event!( $level, $($x)+ ) } else { ::tracing::debug!( $($x)+ ) @@ -88,3 +88,7 @@ pub fn panic_str(p: &Box) -> &'static str { p.downcast_ref::<&st #[inline(always)] #[must_use] pub fn type_name(_: &T) -> &'static str { std::any::type_name::() } + +#[must_use] +#[inline] +pub const fn logging() -> bool { cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) } diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 55e38f18..b3d0240e 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -41,60 +41,149 @@ macro_rules! Err { #[macro_export] macro_rules! err { - (Config($item:literal, $($args:expr),*)) => {{ - $crate::error!(config = %$item, $($args),*); - $crate::error::Error::Config($item, $crate::format_maybe!($($args),*)) - }}; - - (Request(Forbidden($level:ident!($($args:expr),*)))) => {{ - $crate::$level!($($args),*); + (Request(Forbidden($level:ident!($($args:tt)+)))) => {{ + let mut buf = String::new(); $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::forbidden(), - $crate::format_maybe!($($args),*), + $crate::err_log!(buf, $level, $($args)+), ::http::StatusCode::BAD_REQUEST ) }}; - (Request(Forbidden($($args:expr),*))) => { + (Request(Forbidden($($args:tt)+))) => { $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::forbidden(), - $crate::format_maybe!($($args),*), + $crate::format_maybe!($($args)+), ::http::StatusCode::BAD_REQUEST ) }; - (Request($variant:ident($level:ident!($($args:expr),*)))) => {{ - $crate::$level!($($args),*); + (Request($variant:ident($level:ident!($($args:tt)+)))) => {{ + let mut buf = String::new(); $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::$variant, - $crate::format_maybe!($($args),*), + $crate::err_log!(buf, $level, $($args)+), ::http::StatusCode::BAD_REQUEST ) }}; - (Request($variant:ident($($args:expr),*))) => { + (Request($variant:ident($($args:tt)+))) => { $crate::error::Error::Request( ::ruma::api::client::error::ErrorKind::$variant, - $crate::format_maybe!($($args),*), + $crate::format_maybe!($($args)+), ::http::StatusCode::BAD_REQUEST ) }; - ($variant:ident($level:ident!($($args:expr),*))) => {{ - $crate::$level!($($args),*); - $crate::error::Error::$variant($crate::format_maybe!($($args),*)) + (Config($item:literal, $($args:tt)+)) => {{ + let mut buf = String::new(); + $crate::error::Error::Config($item, $crate::err_log!(buf, error, config = %$item, $($args)+)) }}; - ($variant:ident($($args:expr),*)) => { - $crate::error::Error::$variant($crate::format_maybe!($($args),*)) + ($variant:ident($level:ident!($($args:tt)+))) => {{ + let mut buf = String::new(); + $crate::error::Error::$variant($crate::err_log!(buf, $level, $($args)+)) + }}; + + ($variant:ident($($args:tt)+)) => { + $crate::error::Error::$variant($crate::format_maybe!($($args)+)) }; - ($level:ident!($($args:expr),*)) => {{ - $crate::$level!($($args),*); - $crate::error::Error::Err($crate::format_maybe!($($args),*)) + ($level:ident!($($args:tt)+)) => {{ + let mut buf = String::new(); + $crate::error::Error::Err($crate::err_log!(buf, $level, $($args)+)) }}; - ($($args:expr),*) => { - $crate::error::Error::Err($crate::format_maybe!($($args),*)) + ($($args:tt)+) => { + $crate::error::Error::Err($crate::format_maybe!($($args)+)) + }; +} + +/// A trinity of integration between tracing, logging, and Error. This is a +/// customization of tracing::event! with the primary purpose of sharing the +/// error string, fieldset parsing and formatting. An added benefit is that we +/// can share the same callsite metadata for the source of our Error and the +/// associated logging and tracing event dispatches. +#[macro_export] +macro_rules! err_log { + ($out:ident, $level:ident, $($fields:tt)+) => {{ + use std::{fmt, fmt::Write}; + + use ::tracing::{ + callsite, callsite2, level_enabled, metadata, valueset, Callsite, Event, __macro_support, + __tracing_log, + field::{Field, ValueSet, Visit}, + Level, + }; + + const LEVEL: Level = $crate::err_lev!($level); + static __CALLSITE: callsite::DefaultCallsite = callsite2! { + name: std::concat! { + "event ", + std::file!(), + ":", + std::line!(), + }, + kind: metadata::Kind::EVENT, + target: std::module_path!(), + level: LEVEL, + fields: $($fields)+, + }; + + let visit = &mut |vs: ValueSet<'_>| { + struct Visitor<'a>(&'a mut String); + impl Visit for Visitor<'_> { + fn record_debug(&mut self, field: &Field, val: &dyn fmt::Debug) { + if field.name() == "message" { + write!(self.0, "{:?}", val).expect("stream error"); + } else { + write!(self.0, " {}={:?}", field.name(), val).expect("stream error"); + } + } + } + + let meta = __CALLSITE.metadata(); + let enabled = level_enabled!(LEVEL) && { + let interest = __CALLSITE.interest(); + !interest.is_never() && __macro_support::__is_enabled(meta, interest) + }; + + if enabled { + Event::dispatch(meta, &vs); + } + + __tracing_log!(LEVEL, __CALLSITE, &vs); + vs.record(&mut Visitor(&mut $out)); + }; + + (visit)(valueset!(__CALLSITE.metadata().fields(), $($fields)+)); + ($out).into() + }} +} + +#[macro_export] +macro_rules! err_lev { + (debug_warn) => { + if $crate::debug::logging() { + ::tracing::Level::WARN + } else { + ::tracing::Level::DEBUG + } + }; + + (debug_error) => { + if $crate::debug::logging() { + ::tracing::Level::ERROR + } else { + ::tracing::Level::DEBUG + } + }; + + (warn) => { + ::tracing::Level::WARN + }; + + (error) => { + ::tracing::Level::ERROR }; } diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 5406d5ff..730220ac 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -5,8 +5,7 @@ mod response; use std::{any::Any, borrow::Cow, convert::Infallible, fmt}; -pub use log::*; - +pub use self::log::*; use crate::error; #[derive(thiserror::Error)] diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index fc3891df..a597d198 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -8,12 +8,12 @@ pub const EMPTY: &str = ""; /// arguments are provided the first is assumed to be a format string. #[macro_export] macro_rules! format_maybe { - ($s:literal) => { + ($s:literal $(,)?) => { if $crate::is_format!($s) { std::format!($s).into() } else { $s.into() } }; - ($($args:expr),*) => { - std::format!($($args),*).into() + ($s:literal, $($args:tt)+) => { + std::format!($s, $($args)+).into() }; } @@ -24,6 +24,10 @@ macro_rules! is_format { ($s:literal) => { ::const_str::contains!($s, "{") && ::const_str::contains!($s, "}") }; + + ($($s:tt)+) => { + false + }; } #[inline] From 3da22427303c323ea04346814d68047fcca0f69a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 28 Jul 2024 15:15:39 -0400 Subject: [PATCH 0616/2091] bump tokio, use `default-features = false` for more crates Signed-off-by: strawberry --- Cargo.lock | 4 ++-- Cargo.toml | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8da7d7e..723523b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4054,9 +4054,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.1" +version = "1.39.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a" +checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index d2d7dc2d..a8529e12 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ version = "0.2.8" [workspace.dependencies.cargo_toml] version = "0.20" +default-features = false features = ["features"] [workspace.dependencies.toml] @@ -96,6 +97,7 @@ features = ["typed-header", "tracing"] [workspace.dependencies.axum-server] version = "0.6.0" +default-features = false features = ["tls-rustls"] [workspace.dependencies.axum-client-ip] @@ -103,10 +105,12 @@ version = "0.6.0" [workspace.dependencies.tower] version = "0.4.13" +default-features = false features = ["util"] [workspace.dependencies.tower-http] version = "0.5.2" +default-features = false features = [ "add-extension", "cors", @@ -129,10 +133,12 @@ features = [ [workspace.dependencies.serde] version = "1.0.204" +default-features = false features = ["rc"] [workspace.dependencies.serde_json] version = "1.0.120" +default-features = false features = ["raw_value"] # Used for appservice registration files @@ -200,7 +206,8 @@ version = "0.3.30" default-features = false [workspace.dependencies.tokio] -version = "1.39.1" +version = "1.39.2" +default-features = false features = [ "fs", "net", @@ -221,6 +228,7 @@ version = "0.8.5" # Validating urls in config, was already a transitive dependency [workspace.dependencies.url] version = "2.5.0" +default-features = false features = ["serde"] # standard date and time tools @@ -231,6 +239,7 @@ default-features = false [workspace.dependencies.hyper] version = "1.4.1" +default-features = false features = [ "server", "http1", @@ -239,6 +248,7 @@ features = [ [workspace.dependencies.hyper-util] version = "0.1.6" +default-features = false features = [ "client", "server-auto", @@ -250,11 +260,13 @@ features = [ # to support multiple variations of setting a config option [workspace.dependencies.either] version = "1.11.0" +default-features = false features = ["serde"] # Used for reading the configuration from conduwuit.toml & environment variables [workspace.dependencies.figment] version = "0.10.18" +default-features = false features = ["env", "toml"] [workspace.dependencies.hickory-resolver] @@ -400,6 +412,7 @@ version = "0.3" [workspace.dependencies.nix] version = "0.29.0" +default-features = false features = ["resource"] [workspace.dependencies.sd-notify] @@ -427,6 +440,7 @@ version = "0.1" [workspace.dependencies.syn] version = "2.0.72" +default-features = false features = ["full", "extra-traits"] [workspace.dependencies.quote] From d8a87258d6518e029b548d33575cd1566bc11b1a Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 29 Jul 2024 19:18:56 -0400 Subject: [PATCH 0617/2091] rust: bump console-subscriber to 0.4 Signed-off-by: strawberry --- Cargo.lock | 273 ++++++++++++++++------------------------------------- Cargo.toml | 4 +- 2 files changed, 82 insertions(+), 195 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 723523b1..305c3e7f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,34 +153,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core 0.3.4", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.30", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper 0.1.2", - "tower", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.7.5" @@ -188,13 +160,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", - "axum-core 0.4.3", + "axum-core", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", - "hyper 1.4.1", + "hyper", "hyper-util", "itoa", "matchit", @@ -221,28 +193,11 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72188bed20deb981f3a4a9fe674e5980fd9e9c2bd880baa94715ad5d60d64c67" dependencies = [ - "axum 0.7.5", + "axum", "forwarded-header-value", "serde", ] -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 0.2.12", - "http-body 0.4.6", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.4.3" @@ -252,8 +207,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", @@ -270,13 +225,13 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" dependencies = [ - "axum 0.7.5", - "axum-core 0.4.3", + "axum", + "axum-core", "bytes", "futures-util", "headers", - "http 1.1.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "mime", "pin-project-lite", @@ -295,10 +250,10 @@ dependencies = [ "arc-swap", "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", - "hyper 1.4.1", + "hyper", "hyper-util", "pin-project-lite", "rustls 0.21.12", @@ -317,7 +272,7 @@ checksum = "7ea4cd08ae2a5f075d28fa31190163c8106a1d2d3189442494bae22b39040a0d" dependencies = [ "axum-server", "bytes", - "http 1.1.0", + "http", "http-body-util", "pin-project", "tokio", @@ -639,7 +594,7 @@ dependencies = [ name = "conduit_api" version = "0.4.6" dependencies = [ - "axum 0.7.5", + "axum", "axum-client-ip", "axum-extra", "base64 0.22.1", @@ -650,9 +605,10 @@ dependencies = [ "const-str", "futures-util", "hmac", - "http 1.1.0", + "http", "http-body-util", - "hyper 1.4.1", + "hyper", + "image", "ipaddress", "jsonwebtoken", "log", @@ -672,7 +628,7 @@ name = "conduit_core" version = "0.4.6" dependencies = [ "argon2", - "axum 0.7.5", + "axum", "bytes", "cargo_toml", "checked_ops", @@ -683,7 +639,7 @@ dependencies = [ "either", "figment", "hardened_malloc-rs", - "http 1.1.0", + "http", "http-body-util", "image", "ipaddress", @@ -739,7 +695,7 @@ dependencies = [ name = "conduit_router" version = "0.4.6" dependencies = [ - "axum 0.7.5", + "axum", "axum-client-ip", "axum-server", "axum-server-dual-protocol", @@ -749,9 +705,9 @@ dependencies = [ "conduit_core", "conduit_service", "const-str", - "http 1.1.0", + "http", "http-body-util", - "hyper 1.4.1", + "hyper", "hyper-util", "log", "ruma", @@ -779,7 +735,7 @@ dependencies = [ "cyborgtime", "futures-util", "hickory-resolver", - "http 1.1.0", + "http", "image", "ipaddress", "itertools 0.13.0", @@ -805,9 +761,9 @@ dependencies = [ [[package]] name = "console-api" -version = "0.7.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a257c22cd7e487dd4a13d413beabc512c5052f0bc048db0da6a84c3d8a6142fd" +checksum = "86ed14aa9c9f927213c6e4f3ef75faaad3406134efe84ba2cb7983431d5f0931" dependencies = [ "futures-core", "prost", @@ -818,9 +774,9 @@ dependencies = [ [[package]] name = "console-subscriber" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c4cc54bae66f7d9188996404abdf7fdfa23034ef8e43478c8810828abad758" +checksum = "e2e3a111a37f3333946ebf9da370ba5c5577b18eb342ec683eb488dd21980302" dependencies = [ "console-api", "crossbeam-channel", @@ -828,6 +784,7 @@ dependencies = [ "futures-task", "hdrhistogram", "humantime", + "hyper-util", "prost", "prost-types", "serde", @@ -1371,25 +1328,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.2.6", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.5" @@ -1401,7 +1339,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.1.0", + "http", "indexmap 2.2.6", "slab", "tokio", @@ -1449,7 +1387,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http 1.1.0", + "http", "httpdate", "mime", "sha1", @@ -1461,7 +1399,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.1.0", + "http", ] [[package]] @@ -1578,17 +1516,6 @@ dependencies = [ "syn 2.0.72", ] -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - [[package]] name = "http" version = "1.1.0" @@ -1609,17 +1536,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -1627,7 +1543,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.1.0", + "http", ] [[package]] @@ -1638,8 +1554,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" dependencies = [ "bytes", "futures-util", - "http 1.1.0", - "http-body 1.0.1", + "http", + "http-body", "pin-project-lite", ] @@ -1661,30 +1577,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" -[[package]] -name = "hyper" -version = "0.14.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.4.1" @@ -1694,9 +1586,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.5", - "http 1.1.0", - "http-body 1.0.1", + "h2", + "http", + "http-body", "httparse", "httpdate", "itoa", @@ -1713,8 +1605,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", - "http 1.1.0", - "hyper 1.4.1", + "http", + "hyper", "hyper-util", "rustls 0.23.12", "rustls-native-certs", @@ -1727,14 +1619,15 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" dependencies = [ - "hyper 0.14.30", + "hyper", + "hyper-util", "pin-project-lite", "tokio", - "tokio-io-timeout", + "tower-service", ] [[package]] @@ -1746,9 +1639,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.1.0", - "http-body 1.0.1", - "hyper 1.4.1", + "http", + "http-body", + "hyper", "pin-project-lite", "socket2", "tokio", @@ -2693,9 +2586,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" dependencies = [ "bytes", "prost-derive", @@ -2703,12 +2596,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.72", @@ -2716,9 +2609,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.12.6" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" dependencies = [ "prost", ] @@ -2903,12 +2796,12 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.4.5", + "h2", "hickory-resolver", - "http 1.1.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", - "hyper 1.4.1", + "hyper", "hyper-rustls", "hyper-util", "ipnet", @@ -3008,7 +2901,7 @@ dependencies = [ "assign", "bytes", "date_header", - "http 1.1.0", + "http", "js_int", "js_option", "maplit", @@ -3031,7 +2924,7 @@ dependencies = [ "base64 0.22.1", "bytes", "form_urlencoded", - "http 1.1.0", + "http", "indexmap 2.2.6", "js_int", "konst", @@ -3139,7 +3032,7 @@ version = "0.3.0" source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "headers", - "http 1.1.0", + "http", "http-auth", "ruma-common", "thiserror", @@ -3518,7 +3411,7 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c90802b38c899a2c9e557dff25ad186362eddf755d5f5244001b172dd03bead" dependencies = [ - "http 1.1.0", + "http", "pin-project", "sentry-core", "tower-layer", @@ -3590,11 +3483,12 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.120" +version = "1.0.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" +checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" dependencies = [ "itoa", + "memchr", "ryu", "serde", ] @@ -4070,16 +3964,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "2.4.0" @@ -4207,23 +4091,26 @@ dependencies = [ [[package]] name = "tonic" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +checksum = "38659f4a91aba8598d27821589f5db7dddd94601e7a01b1e485a50e5484c7401" dependencies = [ "async-stream", "async-trait", - "axum 0.6.20", - "base64 0.21.7", + "axum", + "base64 0.22.1", "bytes", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.30", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", "hyper-timeout", + "hyper-util", "percent-encoding", "pin-project", "prost", + "socket2", "tokio", "tokio-stream", "tower", @@ -4263,8 +4150,8 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http 1.1.0", - "http-body 1.0.1", + "http", + "http-body", "http-body-util", "pin-project-lite", "tokio", diff --git a/Cargo.toml b/Cargo.toml index a8529e12..7289849e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,7 +137,7 @@ default-features = false features = ["rc"] [workspace.dependencies.serde_json] -version = "1.0.120" +version = "1.0.121" default-features = false features = ["raw_value"] @@ -408,7 +408,7 @@ default-features = false features = ["use_std"] [workspace.dependencies.console-subscriber] -version = "0.3" +version = "0.4" [workspace.dependencies.nix] version = "0.29.0" From f51d4237c76053862c986bb3b536d58865016365 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 1 Aug 2024 08:48:28 +0000 Subject: [PATCH 0618/2091] bump cargo Signed-off-by: Jason Volk --- Cargo.lock | 107 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 65 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 305c3e7f..c6ae1f8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -394,9 +394,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.16.1" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" +checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83" [[package]] name = "byteorder" @@ -412,9 +412,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" +checksum = "fca2be1d5c43812bae364ee3f30b3afcb7877cf59f4aeb94c66f313a41d2fac9" [[package]] name = "bzip2-sys" @@ -439,9 +439,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" +checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" dependencies = [ "jobserver", "libc", @@ -499,9 +499,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" dependencies = [ "clap_builder", "clap_derive", @@ -509,9 +509,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" dependencies = [ "anstyle", "clap_lex", @@ -519,9 +519,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.11" +version = "4.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -608,7 +608,6 @@ dependencies = [ "http", "http-body-util", "hyper", - "image", "ipaddress", "jsonwebtoken", "log", @@ -1340,7 +1339,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.2.6", + "indexmap 2.3.0", "slab", "tokio", "tokio-util", @@ -1709,9 +1708,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -2248,7 +2247,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.2.6", + "indexmap 2.3.0", "js-sys", "once_cell", "pin-project-lite", @@ -2295,7 +2294,7 @@ dependencies = [ "glob", "once_cell", "opentelemetry", - "ordered-float 4.2.1", + "ordered-float 4.2.2", "percent-encoding", "rand", "thiserror", @@ -2314,9 +2313,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.1" +version = "4.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ff2cf528c6c03d9ed653d6c4ce1dc0582dc4af309790ad92f07c1cd551b0be" +checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" dependencies = [ "num-traits", ] @@ -2543,9 +2542,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f" +dependencies = [ + "zerocopy", +] [[package]] name = "precomputed-hash" @@ -2925,7 +2927,7 @@ dependencies = [ "bytes", "form_urlencoded", "http", - "indexmap 2.2.6", + "indexmap 2.3.0", "js_int", "konst", "percent-encoding", @@ -2951,7 +2953,7 @@ version = "0.28.1" source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" dependencies = [ "as_variant", - "indexmap 2.2.6", + "indexmap 2.3.0", "js_int", "js_option", "percent-encoding", @@ -3475,7 +3477,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" dependencies = [ "form_urlencoded", - "indexmap 2.2.6", + "indexmap 2.3.0", "itoa", "ryu", "serde", @@ -3540,7 +3542,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.3.0", "itoa", "ryu", "serde", @@ -3607,9 +3609,9 @@ dependencies = [ [[package]] name = "signal-hook-mio" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" dependencies = [ "libc", "mio 0.8.11", @@ -4010,9 +4012,9 @@ dependencies = [ [[package]] name = "tokio-socks" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51165dfa029d2a65969413a6cc96f354b86b464498702f174a4efa13608fd8c0" +checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" dependencies = [ "either", "futures-util", @@ -4046,21 +4048,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.17", + "toml_edit 0.22.20", ] [[package]] name = "toml_datetime" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] @@ -4071,22 +4073,22 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.3.0", "toml_datetime", "winnow 0.5.40", ] [[package]] name = "toml_edit" -version = "0.22.17" +version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16" +checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.3.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.16", + "winnow 0.6.18", ] [[package]] @@ -4767,9 +4769,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.16" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b480ae9340fc261e6be3e95a1ba86d54ae3f9171132a73ce8d4bbaf68339507c" +checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" dependencies = [ "memchr", ] @@ -4811,6 +4813,27 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "zerocopy" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "zeroize" version = "1.8.1" From 2e3e14b0459794e923970200f95766a854d8dfae Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 1 Aug 2024 10:58:27 +0000 Subject: [PATCH 0619/2091] refactor some additional errors Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 6 +- src/api/client/keys.rs | 5 +- src/api/client/media.rs | 39 ++++------ src/api/client/membership.rs | 44 ++++++----- src/core/error/mod.rs | 4 +- src/core/pdu/mod.rs | 8 +- src/service/media/preview.rs | 13 ++-- src/service/pusher/mod.rs | 33 ++++----- src/service/resolver/actual.rs | 10 +-- .../rooms/event_handler/parse_incoming_pdu.rs | 10 +-- src/service/sending/appservice.rs | 21 +++--- src/service/sending/send.rs | 9 +-- src/service/server_keys/mod.rs | 73 ++++++++++--------- 13 files changed, 127 insertions(+), 148 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 4e04c94a..1802357a 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -7,7 +7,7 @@ use std::{ use api::client::validate_and_add_event_id; use conduit::{ - debug, info, log, + debug, debug_error, err, info, log, log::{capture, Capture}, utils, warn, Error, PduEvent, Result, }; @@ -575,8 +575,8 @@ pub(super) async fn force_set_room_state_from_server( }; let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| { - warn!("Invalid PDU in fetching remote room state PDUs response: {} {:?}", e, value); - Error::BadServerResponse("Invalid PDU in send_join response.") + debug_error!("Invalid PDU in fetching remote room state PDUs response: {value:#?}"); + err!(BadServerResponse(debug_error!("Invalid PDU in send_join response: {e:?}"))) })?; self.services diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 6fa7a8ad..a426364a 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -4,7 +4,7 @@ use std::{ }; use axum::extract::State; -use conduit::{debug, utils, utils::math::continue_exponential_backoff_secs, Error, Result}; +use conduit::{utils, utils::math::continue_exponential_backoff_secs, Err, Error, Result}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::{ @@ -362,8 +362,7 @@ pub(crate) async fn get_keys_helper bool + Send>( const MIN: u64 = 5 * 60; const MAX: u64 = 60 * 60 * 24; if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) { - debug!("Backing off query from {server:?}"); - return (server, Err(Error::BadServerResponse("bad query, still backing off"))); + return (server, Err!(BadServerResponse("bad query from {server:?}, still backing off"))); } } diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 5c83c17f..bce9b2b5 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -5,13 +5,13 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - debug_warn, error, + debug_warn, err, error, utils::{ self, content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename}, math::ruma_from_usize, }, - warn, Error, Result, + warn, Err, Error, Result, }; use ruma::api::client::{ error::{ErrorKind, RetryAfter}, @@ -70,8 +70,9 @@ pub(crate) async fn get_media_preview_route( let url = &body.url; if !services.media.url_preview_allowed(url) { - warn!(%sender_user, "URL is not allowed to be previewed: {url}"); - return Err(Error::BadRequest(ErrorKind::forbidden(), "URL is not allowed to be previewed")); + return Err!(Request(Forbidden( + warn!(%sender_user, %url, "URL is not allowed to be previewed") + ))); } match services.media.get_url_preview(url).await { @@ -90,7 +91,6 @@ pub(crate) async fn get_media_preview_route( }, Err(e) => { warn!(%sender_user, "Failed to generate a URL preview: {e}"); - // there doesn't seem to be an agreed-upon error code in the spec. // the only response codes in the preview_url spec page are 200 and 429. Err(Error::BadRequest( @@ -222,10 +222,7 @@ pub(crate) async fn get_content_route( body.timeout_ms, ) .await - .map_err(|e| { - debug_warn!("Fetching media `{}` failed: {:?}", mxc, e); - Error::BadRequest(ErrorKind::NotFound, "Remote media error.") - })?; + .map_err(|e| err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))))?; let content_disposition = Some(make_content_disposition( &response.content_type, @@ -241,7 +238,7 @@ pub(crate) async fn get_content_route( cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()), }) } else { - Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) + Err!(Request(NotFound("Media not found."))) } } @@ -328,13 +325,10 @@ pub(crate) async fn get_content_as_filename_route( cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) }, - Err(e) => { - debug_warn!("Fetching media `{}` failed: {:?}", mxc, e); - Err(Error::BadRequest(ErrorKind::NotFound, "Remote media error.")) - }, + Err(e) => Err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))), } } else { - Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) + Err!(Request(NotFound("Media not found."))) } } @@ -385,10 +379,10 @@ pub(crate) async fn get_content_thumbnail_route( &mxc, body.width .try_into() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, + .map_err(|e| err!(Request(InvalidParam("Width is invalid: {e:?}"))))?, body.height .try_into() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Height is invalid."))?, + .map_err(|e| err!(Request(InvalidParam("Height is invalid: {e:?}"))))?, ) .await? { @@ -411,7 +405,7 @@ pub(crate) async fn get_content_thumbnail_route( // we'll lie to the client and say the blocked server's media was not found and // log. the client has no way of telling anyways so this is a security bonus. debug_warn!("Received request for media `{}` on blocklisted server", mxc); - return Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")); + return Err!(Request(NotFound("Media not found."))); } match services @@ -460,13 +454,10 @@ pub(crate) async fn get_content_thumbnail_route( content_disposition, }) }, - Err(e) => { - debug_warn!("Fetching media `{}` failed: {:?}", mxc, e); - Err(Error::BadRequest(ErrorKind::NotFound, "Remote media error.")) - }, + Err(e) => Err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))), } } else { - Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")) + Err!(Request(NotFound("Media not found."))) } } @@ -504,7 +495,7 @@ async fn get_remote_content( // we'll lie to the client and say the blocked server's media was not found and // log. the client has no way of telling anyways so this is a security bonus. debug_warn!("Received request for media `{mxc}` on blocklisted server"); - return Err(Error::BadRequest(ErrorKind::NotFound, "Media not found.")); + return Err!(Request(NotFound("Media not found."))); } let content_response = services diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index f0d2848f..4b27d8e3 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -8,11 +8,11 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - debug, debug_warn, error, info, + debug, debug_error, debug_warn, err, error, info, pdu::{gen_event_id_canonical_json, PduBuilder}, trace, utils, utils::math::continue_exponential_backoff_secs, - warn, Error, PduEvent, Result, + warn, Err, Error, PduEvent, Result, }; use ruma::{ api::{ @@ -705,11 +705,11 @@ async fn join_room_by_id_helper_remote( { room_version }, - _ => return Err(Error::BadServerResponse("Room version is not supported")), + _ => return Err!(BadServerResponse("Room version is not supported")), }; let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) - .map_err(|_| Error::BadServerResponse("Invalid make_join event json received from server."))?; + .map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?; let join_authorized_via_users_server = join_event_stub .get("content") @@ -876,7 +876,7 @@ async fn join_room_by_id_helper_remote( info!("Parsing join event"); let parsed_join_pdu = PduEvent::from_id_val(event_id, join_event.clone()) - .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; + .map_err(|e| err!(BadServerResponse("Invalid join event PDU: {e:?}")))?; let mut state = HashMap::new(); let pub_key_map = RwLock::new(BTreeMap::new()); @@ -899,8 +899,8 @@ async fn join_room_by_id_helper_remote( }; let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| { - warn!("Invalid PDU in send_join response: {} {:?}", e, value); - Error::BadServerResponse("Invalid PDU in send_join response.") + debug_warn!("Invalid PDU in send_join response: {value:#?}"); + err!(BadServerResponse("Invalid PDU in send_join response: {e:?}")) })?; services.rooms.outlier.add_pdu_outlier(&event_id, &value)?; @@ -1122,10 +1122,10 @@ async fn join_room_by_id_helper_local( { room_version_id }, - _ => return Err(Error::BadServerResponse("Room version is not supported")), + _ => return Err!(BadServerResponse("Room version is not supported")), }; let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) - .map_err(|_| Error::BadServerResponse("Invalid make_join event json received from server."))?; + .map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?; let join_authorized_via_users_server = join_event_stub .get("content") .map(|s| { @@ -1250,7 +1250,7 @@ async fn join_room_by_id_helper_local( async fn make_join_request( services: &Services, sender_user: &UserId, room_id: &RoomId, servers: &[OwnedServerName], ) -> Result<(federation::membership::prepare_join_event::v1::Response, OwnedServerName)> { - let mut make_join_response_and_server = Err(Error::BadServerResponse("No server available to assist in joining.")); + let mut make_join_response_and_server = Err!(BadServerResponse("No server available to assist in joining.")); let mut make_join_counter: u16 = 0; let mut incompatible_room_version_count: u8 = 0; @@ -1291,8 +1291,7 @@ async fn make_join_request( "15 servers have responded with M_INCOMPATIBLE_ROOM_VERSION or M_UNSUPPORTED_ROOM_VERSION, \ assuming that Conduwuit does not support the room {room_id}: {e}" ); - make_join_response_and_server = - Err(Error::BadServerResponse("Room version is not supported by Conduwuit")); + make_join_response_and_server = Err!(BadServerResponse("Room version is not supported by Conduwuit")); return make_join_response_and_server; } @@ -1300,8 +1299,7 @@ async fn make_join_request( warn!( "50 servers failed to provide valid make_join response, assuming no server can assist in joining." ); - make_join_response_and_server = - Err(Error::BadServerResponse("No server available to assist in joining.")); + make_join_response_and_server = Err!(BadServerResponse("No server available to assist in joining.")); return make_join_response_and_server; } } @@ -1321,8 +1319,8 @@ pub async fn validate_and_add_event_id( pub_key_map: &RwLock>>, ) -> Result<(OwnedEventId, CanonicalJsonObject)> { let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { - error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); - Error::BadServerResponse("Invalid PDU in server response") + debug_error!("Invalid PDU in server response: {pdu:#?}"); + err!(BadServerResponse("Invalid PDU in server response: {e:?}")) })?; let event_id = EventId::parse(format!( "${}", @@ -1358,15 +1356,15 @@ pub async fn validate_and_add_event_id( const MIN: u64 = 60 * 5; const MAX: u64 = 60 * 60 * 24; if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) { - debug!("Backing off from {event_id}"); - return Err(Error::BadServerResponse("bad event, still backing off")); + return Err!(BadServerResponse("bad event {event_id:?}, still backing off")); } } if let Err(e) = ruma::signatures::verify_event(&*pub_key_map.read().await, &value, room_version) { - warn!("Event {} failed verification {:?} {}", event_id, pdu, e); + debug_error!("Event {event_id} failed verification {pdu:#?}"); + let e = Err!(BadServerResponse(debug_error!("Event {event_id} failed verification: {e:?}"))); back_off(event_id).await; - return Err(Error::BadServerResponse("Event failed verification.")); + return e; } value.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); @@ -1648,7 +1646,7 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, } async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result<()> { - let mut make_leave_response_and_server = Err(Error::BadServerResponse("No server available to assist in leaving.")); + let mut make_leave_response_and_server = Err!(BadServerResponse("No server available to assist in leaving.")); let invite_state = services .rooms @@ -1705,11 +1703,11 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room { version }, - _ => return Err(Error::BadServerResponse("Room version is not supported")), + _ => return Err!(BadServerResponse("Room version is not supported")), }; let mut leave_event_stub = serde_json::from_str::(make_leave_response.event.get()) - .map_err(|_| Error::BadServerResponse("Invalid make_leave event json received from server."))?; + .map_err(|e| err!(BadServerResponse("Invalid make_leave event json received from server: {e:?}")))?; // TODO: Is origin needed? leave_event_stub.insert( diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 730220ac..89148d8c 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -86,13 +86,13 @@ pub enum Error { // conduwuit #[error("Arithmetic operation failed: {0}")] Arithmetic(Cow<'static, str>), + #[error("{0}")] + BadServerResponse(Cow<'static, str>), #[error("There was a problem with the '{0}' directive in your configuration: {1}")] Config(&'static str, Cow<'static, str>), #[error("{0}")] Database(Cow<'static, str>), #[error("{0}")] - BadServerResponse(&'static str), - #[error("{0}")] Conflict(&'static str), // This is only needed for when a room alias already exists // unique / untyped diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index c7c2acf5..439c831a 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -23,7 +23,7 @@ use serde_json::{ value::{to_raw_value, RawValue as RawJsonValue}, }; -use crate::{warn, Error}; +use crate::{err, warn, Error}; #[derive(Deserialize)] struct ExtractRedactedBecause { @@ -396,10 +396,8 @@ impl Ord for PduEvent { pub fn gen_event_id_canonical_json( pdu: &RawJsonValue, room_version_id: &RoomVersionId, ) -> crate::Result<(OwnedEventId, CanonicalJsonObject)> { - let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { - warn!("Error parsing incoming event {:?}: {:?}", pdu, e); - Error::BadServerResponse("Invalid PDU in server response") - })?; + let value: CanonicalJsonObject = serde_json::from_str(pdu.get()) + .map_err(|e| err!(BadServerResponse(warn!("Error parsing incoming event: {e:?}"))))?; let event_id = format!( "${}", diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index ebfd22e7..ac24d81a 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -1,10 +1,9 @@ use std::{io::Cursor, time::SystemTime}; -use conduit::{debug, utils, warn, Error, Result}; +use conduit::{debug, utils, warn, Err, Result}; use conduit_core::implement; use image::ImageReader as ImgReader; use ipaddress::IPAddress; -use ruma::api::client::error::ErrorKind; use serde::Serialize; use url::Url; use webpage::HTML; @@ -89,7 +88,7 @@ pub async fn get_url_preview(&self, url: &str) -> Result { async fn request_url_preview(&self, url: &str) -> Result { if let Ok(ip) = IPAddress::parse(url) { if !self.services.globals.valid_cidr_range(&ip) { - return Err(Error::BadServerResponse("Requesting from this address is forbidden")); + return Err!(BadServerResponse("Requesting from this address is forbidden")); } } @@ -99,7 +98,7 @@ async fn request_url_preview(&self, url: &str) -> Result { if let Some(remote_addr) = response.remote_addr() { if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { if !self.services.globals.valid_cidr_range(&ip) { - return Err(Error::BadServerResponse("Requesting from this address is forbidden")); + return Err!(BadServerResponse("Requesting from this address is forbidden")); } } } @@ -109,12 +108,12 @@ async fn request_url_preview(&self, url: &str) -> Result { .get(reqwest::header::CONTENT_TYPE) .and_then(|x| x.to_str().ok()) else { - return Err(Error::BadRequest(ErrorKind::Unknown, "Unknown Content-Type")); + return Err!(Request(Unknown("Unknown Content-Type"))); }; let data = match content_type { html if html.starts_with("text/html") => self.download_html(url).await?, img if img.starts_with("image/") => self.download_image(url).await?, - _ => return Err(Error::BadRequest(ErrorKind::Unknown, "Unsupported Content-Type")), + _ => return Err!(Request(Unknown("Unsupported Content-Type"))), }; self.set_url_preview(url, &data).await?; @@ -142,7 +141,7 @@ async fn download_html(&self, url: &str) -> Result { } let body = String::from_utf8_lossy(&bytes); let Ok(html) = HTML::from_string(body.to_string(), Some(url.to_owned())) else { - return Err(Error::BadRequest(ErrorKind::Unknown, "Failed to parse HTML")); + return Err!(Request(Unknown("Failed to parse HTML"))); }; let mut data = match html.opengraph.images.first() { diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 873f0f49..de87264c 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -3,7 +3,7 @@ mod data; use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; -use conduit::{debug_info, info, trace, utils::string_from_bytes, warn, Error, PduEvent, Result}; +use conduit::{debug_error, err, trace, utils::string_from_bytes, warn, Err, PduEvent, Result}; use ipaddress::IPAddress; use ruma::{ api::{ @@ -84,8 +84,9 @@ impl Service { let http_request = request .try_into_http_request::(&dest, SendAccessToken::IfRequired(""), &VERSIONS) .map_err(|e| { - warn!("Failed to find destination {dest} for push gateway: {e}"); - Error::BadServerResponse("Invalid push gateway destination") + err!(BadServerResponse(warn!( + "Failed to find destination {dest} for push gateway: {e}" + ))) })? .map(BytesMut::freeze); @@ -95,7 +96,7 @@ impl Service { trace!("Checking request URL for IP"); if let Ok(ip) = IPAddress::parse(url_host) { if !self.services.globals.valid_cidr_range(&ip) { - return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); + return Err!(BadServerResponse("Not allowed to send requests to this IP")); } } } @@ -110,7 +111,7 @@ impl Service { if let Some(remote_addr) = response.remote_addr() { if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { if !self.services.globals.valid_cidr_range(&ip) { - return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); + return Err!(BadServerResponse("Not allowed to send requests to this IP")); } } } @@ -129,10 +130,10 @@ impl Service { let body = response.bytes().await?; // TODO: handle timeout if !status.is_success() { - info!("Push gateway {dest} returned unsuccessful HTTP response ({status})"); - debug_info!("Push gateway response body: {:?}", string_from_bytes(&body)); - - return Err(Error::BadServerResponse("Push gateway returned unsuccessful response")); + debug_error!("Push gateway response body: {:?}", string_from_bytes(&body)); + return Err!(BadServerResponse(error!( + "Push gateway {dest} returned unsuccessful HTTP response: {status}" + ))); } let response = T::IncomingResponse::try_from_http_response( @@ -140,13 +141,11 @@ impl Service { .body(body) .expect("reqwest body is valid http body"), ); - response.map_err(|e| { - warn!("Push gateway {dest} returned invalid response bytes: {e}"); - Error::BadServerResponse("Push gateway returned bad/invalid response") - }) + response + .map_err(|e| err!(BadServerResponse(error!("Push gateway {dest} returned invalid response: {e}")))) }, Err(e) => { - warn!("Could not send request to pusher {dest}: {e}"); + debug_error!("Could not send request to pusher {dest}: {e}"); Err(e.into()) }, } @@ -165,7 +164,7 @@ impl Service { .room_state_get(&pdu.room_id, &StateEventType::RoomPowerLevels, "")? .map(|ev| { serde_json::from_str(ev.content.get()) - .map_err(|_| Error::bad_database("invalid m.room.power_levels event")) + .map_err(|e| err!(Database("invalid m.room.power_levels event: {e:?}"))) }) .transpose()? .unwrap_or_default(); @@ -181,8 +180,8 @@ impl Service { }; if notify.is_some() { - return Err(Error::bad_database( - r#"Malformed pushrule contains more than one of these actions: ["dont_notify", "notify", "coalesce"]"#, + return Err!(Database( + r#"Malformed pushrule contains more than one of these actions: ["dont_notify", "notify", "coalesce"]"# )); } diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index d940c514..623e3a96 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -4,7 +4,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, debug_error, debug_info, debug_warn, trace, Err, Error, Result}; +use conduit::{debug, debug_error, debug_info, debug_warn, err, trace, Err, Result}; use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; use ruma::ServerName; @@ -329,10 +329,8 @@ impl super::Service { dest.is_ip_literal() || !IPAddress::is_valid(dest.host()), "Destination is not an IP literal." ); - let ip = IPAddress::parse(dest.host()).map_err(|e| { - debug_error!("Failed to parse IP literal from string: {}", e); - Error::BadServerResponse("Invalid IP address") - })?; + let ip = IPAddress::parse(dest.host()) + .map_err(|e| err!(BadServerResponse(debug_error!("Failed to parse IP literal from string: {e}"))))?; self.validate_ip(&ip)?; @@ -341,7 +339,7 @@ impl super::Service { pub(crate) fn validate_ip(&self, ip: &IPAddress) -> Result<()> { if !self.services.globals.valid_cidr_range(ip) { - return Err(Error::BadServerResponse("Not allowed to send requests to this IP")); + return Err!(BadServerResponse("Not allowed to send requests to this IP")); } Ok(()) diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index a19862a5..a7ffe193 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,18 +1,18 @@ -use conduit::{pdu::gen_event_id_canonical_json, warn, Err, Error, Result}; -use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId}; +use conduit::{debug_warn, err, pdu::gen_event_id_canonical_json, Err, Result}; +use ruma::{CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId}; use serde_json::value::RawValue as RawJsonValue; impl super::Service { pub fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { - warn!("Error parsing incoming event {pdu:?}: {e:?}"); - Error::BadServerResponse("Invalid PDU in server response") + debug_warn!("Error parsing incoming event {pdu:#?}"); + err!(BadServerResponse("Error parsing incoming event {e:?}")) })?; let room_id: OwnedRoomId = value .get("room_id") .and_then(|id| RoomId::parse(id.as_str()?).ok()) - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid room id in pdu"))?; + .ok_or(err!(Request(InvalidParam("Invalid room id in pdu"))))?; let Ok(room_version_id) = self.services.state.get_room_version(&room_id) else { return Err!("Server is not in room {room_id}"); diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 5ed40ad9..32dbba4a 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, mem}; use bytes::BytesMut; -use conduit::{debug_error, trace, utils, warn, Error, Result}; +use conduit::{debug_error, err, trace, utils, warn, Err, Result}; use reqwest::Client; use ruma::api::{appservice::Registration, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken}; @@ -26,10 +26,7 @@ where let hs_token = registration.hs_token.as_str(); let mut http_request = request .try_into_http_request::(&dest, SendAccessToken::IfRequired(hs_token), &VERSIONS) - .map_err(|e| { - warn!("Failed to find destination {dest}: {e}"); - Error::BadServerResponse("Invalid appservice destination") - })? + .map_err(|e| err!(BadServerResponse(warn!("Failed to find destination {dest}: {e}"))))? .map(BytesMut::freeze); let mut parts = http_request.uri().clone().into_parts(); @@ -69,13 +66,11 @@ where let body = response.bytes().await?; // TODO: handle timeout if !status.is_success() { - warn!( + debug_error!("Appservice response bytes: {:?}", utils::string_from_bytes(&body)); + return Err!(BadServerResponse(error!( "Appservice \"{}\" returned unsuccessful HTTP response {status} at {dest}", registration.id - ); - debug_error!("Appservice response bytes: {:?}", utils::string_from_bytes(&body)); - - return Err(Error::BadServerResponse("Appservice returned unsuccessful HTTP response")); + ))); } let response = T::IncomingResponse::try_from_http_response( @@ -85,7 +80,9 @@ where ); response.map(Some).map_err(|e| { - warn!("Appservice \"{}\" returned invalid response bytes {dest}: {e}", registration.id); - Error::BadServerResponse("Appservice returned bad/invalid response") + err!(BadServerResponse(error!( + "Appservice \"{}\" returned invalid response bytes {dest}: {e}", + registration.id + ))) }) } diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index b3a84d62..8fd467f6 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, mem}; use conduit::{ - debug, debug_error, debug_warn, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error, Result, + debug, debug_error, debug_warn, err, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error, Result, }; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; @@ -62,7 +62,7 @@ impl super::Service { trace!("Preparing request"); let mut http_request = req .try_into_http_request::>(&actual.string, SATIR, &VERSIONS) - .map_err(|_| Error::BadServerResponse("Invalid destination"))?; + .map_err(|e| err!(BadServerResponse("Invalid destination: {e:?}")))?; sign_request::(&self.services.globals, dest, &mut http_request); @@ -139,10 +139,7 @@ where ); } - match response { - Err(_) => Err(Error::BadServerResponse("Server returned bad 200 response.")), - Ok(response) => Ok(response), - } + response.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}"))) } fn handle_error( diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index b27279af..a565e500 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -4,7 +4,7 @@ use std::{ time::{Duration, SystemTime}, }; -use conduit::{debug, error, info, trace, warn, Error, Result}; +use conduit::{debug, debug_error, debug_warn, err, error, info, trace, warn, Err, Result}; use futures_util::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::federation::{ @@ -57,13 +57,13 @@ impl Service { for event in events { for (signature_server, signature) in event .get("signatures") - .ok_or(Error::BadServerResponse("No signatures in server response pdu."))? + .ok_or(err!(BadServerResponse("No signatures in server response pdu.")))? .as_object() - .ok_or(Error::BadServerResponse("Invalid signatures object in server response pdu."))? + .ok_or(err!(BadServerResponse("Invalid signatures object in server response pdu.")))? { - let signature_object = signature.as_object().ok_or(Error::BadServerResponse( + let signature_object = signature.as_object().ok_or(err!(BadServerResponse( "Invalid signatures content object in server response pdu.", - ))?; + )))?; for signature_id in signature_object.keys() { server_key_ids @@ -94,10 +94,12 @@ impl Service { .map(|(signature_server, signature_ids)| async { let fetch_res = self .fetch_signing_keys_for_server( - signature_server.as_str().try_into().map_err(|_| { + signature_server.as_str().try_into().map_err(|e| { ( signature_server.clone(), - Error::BadServerResponse("Invalid servername in signatures of server response pdu."), + err!(BadServerResponse( + "Invalid servername in signatures of server response pdu: {e:?}" + )), ) })?, signature_ids.into_iter().collect(), // HashSet to Vec @@ -107,7 +109,9 @@ impl Service { match fetch_res { Ok(keys) => Ok((signature_server, keys)), Err(e) => { - warn!("Signature verification failed: Could not fetch signing key for {signature_server}: {e}",); + debug_error!( + "Signature verification failed: Could not fetch signing key for {signature_server}: {e}", + ); Err((signature_server, e)) }, } @@ -123,7 +127,7 @@ impl Service { .insert(signature_server.clone(), keys); }, Err((signature_server, e)) => { - warn!("Failed to fetch keys for {}: {:?}", signature_server, e); + debug_warn!("Failed to fetch keys for {signature_server}: {e:?}"); }, } } @@ -141,35 +145,37 @@ impl Service { pub_key_map: &mut RwLockWriteGuard<'_, BTreeMap>>, ) -> Result<()> { let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { - error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); - Error::BadServerResponse("Invalid PDU in server response") + debug_error!("Invalid PDU in server response: {pdu:#?}"); + err!(BadServerResponse(error!("Invalid PDU in server response: {e:?}"))) })?; let signatures = value .get("signatures") - .ok_or(Error::BadServerResponse("No signatures in server response pdu."))? + .ok_or(err!(BadServerResponse("No signatures in server response pdu.")))? .as_object() - .ok_or(Error::BadServerResponse("Invalid signatures object in server response pdu."))?; + .ok_or(err!(BadServerResponse("Invalid signatures object in server response pdu.")))?; for (signature_server, signature) in signatures { - let signature_object = signature.as_object().ok_or(Error::BadServerResponse( + let signature_object = signature.as_object().ok_or(err!(BadServerResponse( "Invalid signatures content object in server response pdu.", - ))?; + )))?; let signature_ids = signature_object.keys().cloned().collect::>(); let contains_all_ids = |keys: &BTreeMap| signature_ids.iter().all(|id| keys.contains_key(id)); - let origin = <&ServerName>::try_from(signature_server.as_str()) - .map_err(|_| Error::BadServerResponse("Invalid servername in signatures of server response pdu."))?; + let origin = <&ServerName>::try_from(signature_server.as_str()).map_err(|e| { + err!(BadServerResponse( + "Invalid servername in signatures of server response pdu: {e:?}" + )) + })?; if servers.contains_key(origin) || pub_key_map.contains_key(origin.as_str()) { continue; } - debug!("Loading signing keys for {}", origin); - + debug!("Loading signing keys for {origin}"); let result: BTreeMap<_, _> = self .services .globals @@ -179,7 +185,7 @@ impl Service { .collect(); if !contains_all_ids(&result) { - debug!("Signing key not loaded for {}", origin); + debug_warn!("Signing key not loaded for {origin}"); servers.insert(origin.to_owned(), BTreeMap::new()); } @@ -196,7 +202,7 @@ impl Service { pub_key_map: &RwLock>>, ) -> Result<()> { for server in self.services.globals.trusted_servers() { - debug!("Asking batch signing keys from trusted server {}", server); + debug!("Asking batch signing keys from trusted server {server}"); match self .services .sending @@ -209,14 +215,16 @@ impl Service { .await { Ok(keys) => { - debug!("Got signing keys: {:?}", keys); + debug!("Got signing keys: {keys:?}"); let mut pkm = pub_key_map.write().await; for k in keys.server_keys { let k = match k.deserialize() { Ok(key) => key, Err(e) => { - warn!("Received error {e} while fetching keys from trusted server {server}"); - warn!("{}", k.into_json()); + warn!( + "Received error {e} while fetching keys from trusted server {server}: {:#?}", + k.into_json() + ); continue; }, }; @@ -236,13 +244,10 @@ impl Service { pkm.insert(k.server_name.to_string(), result); } }, - Err(e) => { - warn!( - "Failed sending batched key request to trusted key server {server} for the remote servers \ - {:?}: {e}", - servers - ); - }, + Err(e) => error!( + "Failed sending batched key request to trusted key server {server} for the remote servers \ + {servers:?}: {e}" + ), } } @@ -478,7 +483,6 @@ impl Service { } } else { info!("query_trusted_key_servers_first is set to false, querying {origin} first"); - debug!("Asking {origin} for their signing keys over federation"); if let Some(server_key) = self .services @@ -536,7 +540,7 @@ impl Service { .filter_map(|e| e.deserialize().ok()) .collect::>() }) { - debug!("Got signing keys: {:?}", server_keys); + debug!("Got signing keys: {server_keys:?}"); for k in server_keys { self.services .globals @@ -561,7 +565,6 @@ impl Service { } } - warn!("Failed to find public key for server: {origin}"); - Err(Error::BadServerResponse("Failed to find public key for server")) + Err!(BadServerResponse(warn!("Failed to find public key for server {origin:?}"))) } } From fe0969e7ca89b9ddd7d64349c13b5b557e500a4d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 1 Aug 2024 12:25:22 +0000 Subject: [PATCH 0620/2091] fix release-max-perf profile [ci skip] Signed-off-by: Jason Volk --- Cargo.toml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 7289849e..09ba319e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -569,7 +569,17 @@ lto = "fat" [profile.release-max-perf.build-override] inherits = "release-max-perf" opt-level = 0 +codegen-units = 32 #rustflags = [ +# '-Crelocation-model=pic', +# '-Ctarget-feature=-crt-static', +# '-Clink-arg=-Wl,--no-gc-sections', +#] + +[profile.release-max-perf.package.conduit_macros] +inherits = "release-max-perf.build-override" +#rustflags = [ +# '-Crelocation-model=pic', # '-Ctarget-feature=-crt-static', #] From 69b4ec81b60f6b9b87b5a1161cd0fca0a26e9f76 Mon Sep 17 00:00:00 2001 From: skedastically <126093083+skedastically@users.noreply.github.com> Date: Mon, 5 Aug 2024 15:23:40 -0400 Subject: [PATCH 0621/2091] update container registry links (#549 / #548) * gitlab URL * Update docker.md --- docs/deploying/docker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index ffbad5b4..162d8a74 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -17,9 +17,9 @@ OCI images for conduwuit are available in the registries listed below. | GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:main][gl] | ![Image Size][shield-main] | Stable main branch. | | Docker Hub | [docker.io/girlbossceo/conduwuit:main][dh] | ![Image Size][shield-main] | Stable main branch. | -[dh]: https://hub.docker.com/repository/docker/girlbossceo/conduwuit +[dh]: https://hub.docker.com/r/girlbossceo/conduwuit [gh]: https://github.com/girlbossceo/conduwuit/pkgs/container/conduwuit -[gl]: https://gitlab.com/conduwuit/conduwuit/container_registry/6351657 +[gl]: https://gitlab.com/conduwuit/conduwuit/container_registry/6369729 [shield-latest]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/latest [shield-main]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/main From 71ee31a0146e644eceb6f55bb49728aadb2ba92a Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 30 Jul 2024 17:49:26 -0400 Subject: [PATCH 0622/2091] add missing newline in banned room list output Signed-off-by: strawberry --- src/admin/room/moderation.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index 86910379..ddde9146 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -547,7 +547,7 @@ async fn list_banned_rooms(&self) -> Result { rooms.reverse(); let output_plain = format!( - "Rooms Banned ({}):\n```\n{}```", + "Rooms Banned ({}):\n```\n{}\n```", rooms.len(), rooms .iter() From 82b973a8b12542b1413685d5359f8810fc3b502c Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 29 Jul 2024 19:11:02 -0400 Subject: [PATCH 0623/2091] nix: use cortex-a73 / ARMv8-a, strip and remove unnecessary bloat for some reason gcc is included in OCI images Signed-off-by: strawberry --- flake.nix | 4 ++++ nix/pkgs/main/default.nix | 18 +++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/flake.nix b/flake.nix index aadfef03..84ec86a2 100644 --- a/flake.nix +++ b/flake.nix @@ -137,7 +137,11 @@ # Useful for editing the book locally mdbook + # used for rust caching in CI to speed it up sccache + + # needed so we can get rid of gcc and other unused deps that bloat OCI images + removeReferencesTo ]) ++ scope.main.buildInputs ++ scope.main.propagatedBuildInputs diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 72d67465..66d60b77 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -7,6 +7,7 @@ , liburing , pkgsBuildHost , rocksdb +, removeReferencesTo , rust , rust-jemalloc-sys , stdenv @@ -94,8 +95,8 @@ buildDepsOnlyEnv = else if stdenv.targetPlatform.isAarch64 then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags ++ lib.optionals stdenv.targetPlatform.isAarch64 [ - # cortex-a55 == ARMv8.2-a - "-DPORTABLE=armv8.2-a" + # cortex-a73 == ARMv8-A + "-DPORTABLE=armv8-a" ] else old.cmakeFlags; }); @@ -128,7 +129,7 @@ buildPackageEnv = { + lib.optionalString stdenv.targetPlatform.isx86_64 " -Ctarget-cpu=x86-64-v2" + lib.optionalString stdenv.targetPlatform.isAarch64 - " -Ctarget-cpu=cortex-a55"; # cortex-a55 == ARMv8.2-a + " -Ctarget-cpu=cortex-a73"; # cortex-a73 == ARMv8-A }; @@ -154,6 +155,7 @@ commonAttrs = { }; dontStrip = profile == "dev" || profile == "test"; + dontPatchELF = profile == "dev" || profile == "test"; buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'; @@ -168,6 +170,9 @@ commonAttrs = { # differing values for `NIX_CFLAGS_COMPILE`, which contributes to spurious # rebuilds of bindgen and its depedents. jq + + # needed so we can get rid of gcc and other unused deps that bloat OCI images + removeReferencesTo ] ++ lib.optionals stdenv.isDarwin [ # https://github.com/NixOS/nixpkgs/issues/206242 @@ -177,6 +182,13 @@ commonAttrs = { # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612 pkgsBuildHost.darwin.apple_sdk.frameworks.Security ]; + + # for some reason gcc and other weird deps are added to OCI images and bloats it up + # + # + postInstall = with pkgsBuildHost; '' + find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${libgcc} -t ${linuxHeaders} -t ${libidn2} -t ${libunistring} '{}' + + ''; }; in From 12d9e454dbdc24494c6cd7c29cdc88e23013c68b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 28 Jul 2024 17:11:31 -0400 Subject: [PATCH 0624/2091] display success/failure count in get-remote-pdu-list Signed-off-by: strawberry --- src/admin/debug/commands.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 1802357a..53f4cf64 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -149,23 +149,32 @@ pub(super) async fn get_remote_pdu_list( .filter_map(|pdu| EventId::parse(pdu).ok()) .collect::>(); + let mut failed_count: usize = 0; + let mut success_count: usize = 0; + for pdu in list { if force { if let Err(e) = self.get_remote_pdu(Box::from(pdu), server.clone()).await { + failed_count = failed_count.saturating_add(1); self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( "Failed to get remote PDU, ignoring error: {e}" ))) .await; - warn!(%e, "Failed to get remote PDU, ignoring error"); + warn!("Failed to get remote PDU, ignoring error: {e}"); + } else { + success_count = success_count.saturating_add(1); } } else { self.get_remote_pdu(Box::from(pdu), server.clone()).await?; + success_count = success_count.saturating_add(1); } } - Ok(RoomMessageEventContent::text_plain("Fetched list of remote PDUs.")) + Ok(RoomMessageEventContent::text_plain(format!( + "Fetched {success_count} remote PDUs successfully with {failed_count} failures" + ))) } #[admin_command] From 1091c30845c8392ccaba038436aa21ad91e4354c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 28 Jul 2024 16:51:24 -0400 Subject: [PATCH 0625/2091] docs: mention FreeBSD specific build commandline for deployment Signed-off-by: strawberry --- docs/SUMMARY.md | 1 + docs/deploying/freebsd.md | 8 ++++++++ docs/deploying/generic.md | 6 ++---- 3 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 docs/deploying/freebsd.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index ca0feeba..5e4155c4 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -10,6 +10,7 @@ - [Docker](deploying/docker.md) - [Arch Linux](deploying/arch-linux.md) - [Debian](deploying/debian.md) + - [FreeBSD](deploying/freebsd.md) - [TURN](turn.md) - [Appservices](appservices.md) - [Maintenance](maintenance.md) diff --git a/docs/deploying/freebsd.md b/docs/deploying/freebsd.md new file mode 100644 index 00000000..44a77566 --- /dev/null +++ b/docs/deploying/freebsd.md @@ -0,0 +1,8 @@ +# conduwuit for FreeBSD + +conduwuit at the moment does not provide FreeBSD builds. Building conduwuit on FreeBSD requires a specific environment variable to use the +system prebuilt RocksDB library instead of rust-rocksdb / rust-librocksdb-sys which does *not* work and will cause a build error or coredump. + +Use the following environment variable: `ROCKSDB_LIB_DIR=/usr/local/lib` + +Such example commandline with it can be: `ROCKSDB_LIB_DIR=/usr/local/lib cargo build --release` diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 0cbd8ef8..18016028 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -9,14 +9,12 @@ You may simply download the binary that fits your machine. Run `uname -m` to see what you need. -Prebuilt binaries can be downloaded from the latest tagged release [here](https://github.com/girlbossceo/conduwuit/releases/latest). - -The latest tagged release also includes the Debian packages. +Prebuilt fully static musl binaries can be downloaded from the latest tagged release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or `main` CI branch workflow artifact output. These also include Debian packages. These binaries have jemalloc and io_uring statically linked and included with them. Alternatively, you may compile the binary yourself. We recommend using [Lix](https://lix.systems) to build conduwuit as this has the most guaranteed reproducibiltiy and easiest to get a build environment and output going. -Otherwise, follow standard Rust project build guides (installing git and cloning the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + libclang, installing liburing for io_uring and RocksDB, etc). +Otherwise, follow standard Rust project build guides (installing git and cloning the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + libclang for RocksDB, installing liburing for io_uring and RocksDB, etc). ## Adding a conduwuit user From 7f1bc490d6f9a154ed13cd0d7bb0b4c1cd21ec4d Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 28 Jul 2024 16:46:20 -0400 Subject: [PATCH 0626/2091] docs: update configuration docs for the new `-O` option Signed-off-by: strawberry --- docs/configuration.md | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 8b3fec8a..cf3ce00b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -4,15 +4,35 @@ This chapter describes various ways to configure conduwuit. ## Basics -Conduwuit uses a config file for the majority of the settings. Please refer to the -[example config file](./configuration/examples.md#example-configuration) for all of those settings. -The config file to use can either be specified on the command line when running conduwuit by specifying the +conduwuit uses a config file for the majority of the settings, but also supports setting individual config options via commandline. + +Please refer to the [example config file](./configuration/examples.md#example-configuration) for all of those settings. + +The config file to use can be specified on the commandline when running conduwuit by specifying the `-c`, `--config` flag. Alternatively, you can use the environment variable `CONDUWUIT_CONFIG` to specify the config -file to used. +file to used. Conduit's environment variables are supported for backwards compatibility. + +## Option commandline flag + +conduwuit supports setting individual config options in TOML format from the `-O` / `--option` flag. For example, you can set your server name via `-O server_name=\"example.com\"`. + +Note that the config is parsed as TOML, and shells like bash will remove quotes. So unfortunately it is required to escape quotes if the config option takes a string. +This does not apply to options that take booleans or numbers: +- `--option allow_registration=true` works ✅ +- `-O max_request_size=99999999` works ✅ +- `-O server_name=example.com` does not work ❌ +- `--option log=\"debug\"` works ✅ +- `--option server_name='"example.com'"` works ✅ + ## Environment variables All of the settings that are found in the config file can be specified by using environment variables. The environment variable names should be all caps and prefixed with `CONDUWUIT_`. + For example, if the setting you are changing is `max_request_size`, then the environment variable to set is `CONDUWUIT_MAX_REQUEST_SIZE`. + +To modify config options not in the `[global]` context such as `[global.well_known]`, use the `__` suffix split: `CONDUWUIT_WELL_KNOWN__SERVER` + +Conduit's environment variables are supported for backwards compatibility (e.g. `CONDUIT_SERVER_NAME`). From c896acb7298028e4c182c684f81daba90e25bf94 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 27 Jul 2024 19:09:31 -0400 Subject: [PATCH 0627/2091] dont consider presence status changes apart of presence spam Signed-off-by: strawberry --- src/service/presence/data.rs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 1a0293de..ec036b3d 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -61,6 +61,22 @@ impl Data { Some(ref presence) => presence.1.content.presence != *presence_state, }; + let status_msg_changed = match last_presence { + None => true, + Some(ref last_presence) => { + let old_msg = last_presence + .1 + .content + .status_msg + .clone() + .unwrap_or_default(); + + let new_msg = status_msg.clone().unwrap_or_default(); + + new_msg != old_msg + }, + }; + let now = utils::millis_since_unix_epoch(); let last_last_active_ts = match last_presence { None => 0, @@ -72,10 +88,10 @@ impl Data { Some(last_active_ago) => now.saturating_sub(last_active_ago.into()), }; - // tighten for state flicker? - if !state_changed && last_active_ts <= last_last_active_ts { + // TODO: tighten for state flicker? + if !status_msg_changed && !state_changed && last_active_ts < last_last_active_ts { debug_warn!( - "presence spam {:?} last_active_ts:{:?} <= {:?}", + "presence spam {:?} last_active_ts:{:?} < {:?}", user_id, last_active_ts, last_last_active_ts From 22f8c339fc0af4379b43fb5ed4736afc208c243f Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 27 Jul 2024 11:35:30 -0400 Subject: [PATCH 0628/2091] debian: remove --verbose from postinst Signed-off-by: strawberry --- debian/postinst | 1 - 1 file changed, 1 deletion(-) diff --git a/debian/postinst b/debian/postinst index edc99bd3..4eae4573 100644 --- a/debian/postinst +++ b/debian/postinst @@ -16,7 +16,6 @@ case "$1" in --home "$CONDUWUIT_DATABASE_PATH" \ --disabled-login \ --shell "/usr/sbin/nologin" \ - --verbose \ conduwuit fi From 57940f38eaef17f5453d7785823fdcee4b313159 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 25 Jul 2024 23:07:54 -0400 Subject: [PATCH 0629/2091] set last_seen_ip on new/initial device creation this is not automatically updating, but at least have something useful there instead of nothing Signed-off-by: strawberry --- src/api/client/account.rs | 10 +++++++--- src/api/client/session.rs | 24 +++++++++++++++++------- src/service/users/data.rs | 3 ++- src/service/users/mod.rs | 3 ++- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index f093c459..10e63d71 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -286,9 +286,13 @@ pub(crate) async fn register_route( let token = utils::random_string(TOKEN_LENGTH); // Create device for this account - services - .users - .create_device(&user_id, &device_id, &token, body.initial_device_display_name.clone())?; + services.users.create_device( + &user_id, + &device_id, + &token, + body.initial_device_display_name.clone(), + Some(client.to_string()), + )?; debug_info!(%user_id, %device_id, "User account was created"); diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 32f3ed29..7c2a9718 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -1,4 +1,5 @@ use axum::extract::State; +use axum_client_ip::InsecureClientIp; use ruma::{ api::client::{ error::ErrorKind, @@ -33,8 +34,9 @@ struct Claims { /// /// Get the supported login types of this server. One of these should be used as /// the `type` field when logging in. +#[tracing::instrument(skip_all, fields(%client), name = "register")] pub(crate) async fn get_login_types_route( - _body: Ruma, + InsecureClientIp(client): InsecureClientIp, _body: Ruma, ) -> Result { Ok(get_login_types::v3::Response::new(vec![ get_login_types::v3::LoginType::Password(PasswordLoginType::default()), @@ -56,8 +58,9 @@ pub(crate) async fn get_login_types_route( /// Note: You can use [`GET /// /_matrix/client/r0/login`](fn.get_supported_versions_route.html) to see /// supported login types. +#[tracing::instrument(skip_all, fields(%client), name = "register")] pub(crate) async fn login_route( - State(services): State, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { // Validate login method // TODO: Other login methods @@ -176,9 +179,13 @@ pub(crate) async fn login_route( if device_exists { services.users.set_token(&user_id, &device_id, &token)?; } else { - services - .users - .create_device(&user_id, &device_id, &token, body.initial_device_display_name.clone())?; + services.users.create_device( + &user_id, + &device_id, + &token, + body.initial_device_display_name.clone(), + Some(client.to_string()), + )?; } // send client well-known if specified so the client knows to reconfigure itself @@ -214,8 +221,9 @@ pub(crate) async fn login_route( /// last seen ts) /// - Forgets to-device events /// - Triggers device list updates +#[tracing::instrument(skip_all, fields(%client), name = "register")] pub(crate) async fn logout_route( - State(services): State, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -241,8 +249,10 @@ pub(crate) async fn logout_route( /// Note: This is equivalent to calling [`GET /// /_matrix/client/r0/logout`](fn.logout_route.html) from each device of this /// user. +#[tracing::instrument(skip_all, fields(%client), name = "register")] pub(crate) async fn logout_all_route( - State(services): State, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 2dcde7ce..e75a2e93 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -246,6 +246,7 @@ impl Data { /// Adds a new device to a user. pub(super) fn create_device( &self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option, + client_ip: Option, ) -> Result<()> { // This method should never be called for nonexistent users. We shouldn't assert // though... @@ -266,7 +267,7 @@ impl Data { &serde_json::to_vec(&Device { device_id: device_id.into(), display_name: initial_device_display_name, - last_seen_ip: None, // TODO + last_seen_ip: client_ip, last_seen_ts: Some(MilliSecondsSinceUnixEpoch::now()), }) .expect("Device::to_string never fails."), diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 46084415..bdb07310 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -328,9 +328,10 @@ impl Service { /// Adds a new device to a user. pub fn create_device( &self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option, + client_ip: Option, ) -> Result<()> { self.db - .create_device(user_id, device_id, token, initial_device_display_name) + .create_device(user_id, device_id, token, initial_device_display_name, client_ip) } /// Removes a device from a user. From 101a966922d015ef9a707fb24cf25ec51d5070cc Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 27 Jul 2024 19:44:11 -0400 Subject: [PATCH 0630/2091] docs: mention potential DNS issues when using docker Signed-off-by: strawberry --- docs/troubleshooting.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 81b09e81..61a22864 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -8,6 +8,17 @@ > > If there are things like Compose file issues or Dockerhub image issues, those can still be mentioned as long as they're something we can fix. +## General potential issues + +#### Potential DNS issues when using Docker + +Docker has issues with its default DNS setup that may cause DNS to not be properly functional when running conduwuit, resulting in federation issues. +The symptoms of this have shown in excessively long room joins (30+ minutes) from very long DNS timeouts, log entries of "mismatching responding nameservers", and/or partial or non-functional inbound/outbound federation. + +This is **not** a conduwuit issue, and is purely a Docker issue. It is not sustainable for heavy DNS activity which is normal for Matrix federation. The workarounds for this are: +- Use DNS over TCP via the config option `query_over_tcp_only = true` +- Don't use Docker's default DNS setup and instead allow the container to use and communicate with your host's DNS servers (host's `/etc/resolv.conf`) + ## Rocksdb / database issues #### Direct IO @@ -38,8 +49,6 @@ With this in mind: - Leave your conduwuit running in `PointInTime` for at least 30-60 minutes so as much possible corruption is restored - If all goes will, you should be able to restore back to using `TolerateCorruptedTailRecords` and you have successfully recovered your database -## Media - ## Debugging Note that users should not really be debugging things. If you find yourself debugging and find the issue, please let us know and/or how we can fix it. Various debug commands can be found in `!admin debug`. From dcaa96a79f7b1480b17e875025341007eba8455d Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 25 Jul 2024 23:06:14 -0400 Subject: [PATCH 0631/2091] sync: always include_redundant_members when using element_hacks feature see: - https://github.com/matrix-org/matrix-js-sdk/pull/4329 - https://github.com/element-hq/synapse/blob/568051c0f07393b786b9d813a1db53dd332c9fc2/synapse/handlers/pagination.py#L639 - https://matrix.to/#/!n8DKU1BeeJilOJXDPr:seirdy.one/$gvGF7GuPrLPR8nHg4UiHvP-ZVwuiMkYS8O6VlsDCW9w?via=awawawawawawawawawawawawawawawawawawawawawawawawawawawawawawaw.gay&via=transfem.dev&via=matrix.org Signed-off-by: strawberry --- src/api/client/context.rs | 4 +++- src/api/client/sync.rs | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 89a08153..f223d488 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -21,11 +21,13 @@ pub(crate) async fn get_context_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + // some clients, at least element, seem to require knowledge of redundant + // members for "inline" profiles on the timeline to work properly let (lazy_load_enabled, lazy_load_send_redundant) = match &body.filter.lazy_load_options { LazyLoadOptions::Enabled { include_redundant_members, } => (true, *include_redundant_members), - LazyLoadOptions::Disabled => (false, false), + LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")), }; let mut lazy_loaded = HashSet::new(); diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 34baf7c1..b25348d4 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -106,11 +106,13 @@ pub(crate) async fn sync_events_route( .unwrap_or_default(), }; + // some clients, at least element, seem to require knowledge of redundant + // members for "inline" profiles on the timeline to work properly let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options { LazyLoadOptions::Enabled { - include_redundant_members: redundant, - } => (true, redundant), - LazyLoadOptions::Disabled => (false, false), + include_redundant_members, + } => (true, include_redundant_members), + LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")), }; let full_state = body.full_state; From 1470331f7e40a3f04d2207cc046439f32c76c537 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 2 Aug 2024 01:05:31 +0000 Subject: [PATCH 0632/2091] add repair_mode documentation to example conf Signed-off-by: Jason Volk --- conduwuit-example.toml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 27e4cb0b..96b2f76f 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -514,6 +514,18 @@ allow_profile_lookup_federation_requests = true # Defaults to false as this uses more CPU when compressing. #rocksdb_bottommost_compression = false +# Database repair mode (for RocksDB SST corruption) +# +# Use this option when the server reports corruption while running or panics. If the server refuses +# to start use the recovery mode options first. Corruption errors containing the acronym 'SST' which +# occur after startup will likely require this option. +# +# - Backing up your database directory is recommended prior to running the repair. +# - Disabling repair mode and restarting the server is recommended after running the repair. +# +# Defaults to false +#rocksdb_repair = false + # Database recovery mode (for RocksDB WAL corruption) # # Use this option when the server reports corruption and refuses to start. Set mode 2 (PointInTime) From 849cfdcdfa5b343aed9073f761d3abe62833970e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 2 Aug 2024 01:21:01 +0000 Subject: [PATCH 0633/2091] add rocksdb stats level option with conf item Signed-off-by: Jason Volk --- conduwuit-example.toml | 13 +++++++++++++ src/core/config/mod.rs | 5 +++++ src/database/opts.rs | 21 ++++++++++++++++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 96b2f76f..fdf53783 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -514,6 +514,19 @@ allow_profile_lookup_federation_requests = true # Defaults to false as this uses more CPU when compressing. #rocksdb_bottommost_compression = false +# Level of statistics collection. Some admin commands to display database statistics may require +# this option to be set. Database performance may be impacted by higher settings. +# +# Option is a number ranging from 0 to 6: +# 0 = No statistics. +# 1 = No statistics in release mode (default). +# 2 to 3 = Statistics with no performance impact. +# 3 to 5 = Statistics with possible performance impact. +# 6 = All statistics. +# +# Defaults to 1 (No statistics, except in debug-mode) +#rocksdb_stats_level = 1 + # Database repair mode (for RocksDB SST corruption) # # Use this option when the server reports corruption while running or panics. If the server refuses diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 6ccb0559..bec7c4ea 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -236,6 +236,8 @@ pub struct Config { pub rocksdb_compaction_ioprio_idle: bool, #[serde(default = "true_fn")] pub rocksdb_compaction: bool, + #[serde(default = "default_rocksdb_stats_level")] + pub rocksdb_stats_level: u8, pub emergency_password: Option, @@ -718,6 +720,7 @@ impl fmt::Display for Config { &self.rocksdb_compaction_ioprio_idle.to_string(), ); line("RocksDB Compaction enabled", &self.rocksdb_compaction.to_string()); + line("RocksDB Statistics level", &self.rocksdb_stats_level.to_string()); line("Media integrity checks on startup", &self.media_startup_check.to_string()); line("Media compatibility filesystem links", &self.media_compat_file_link.to_string()); line("Prevent Media Downloads From", { @@ -1002,6 +1005,8 @@ fn default_rocksdb_compression_level() -> i32 { 32767 } #[allow(clippy::doc_markdown)] fn default_rocksdb_bottommost_compression_level() -> i32 { 32767 } +fn default_rocksdb_stats_level() -> u8 { 1 } + // I know, it's a great name #[must_use] pub fn default_default_room_version() -> RoomVersionId { RoomVersionId::V10 } diff --git a/src/database/opts.rs b/src/database/opts.rs index 3c3ad424..8791e778 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -2,8 +2,8 @@ use std::{cmp, collections::HashMap}; use conduit::{utils, Config}; use rocksdb::{ - BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, LogLevel, Options, - UniversalCompactOptions, UniversalCompactionStopStyle, + statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, + LogLevel, Options, UniversalCompactOptions, UniversalCompactionStopStyle, }; /// Create database-wide options suitable for opening the database. This also @@ -13,6 +13,11 @@ use rocksdb::{ /// through cf_options(). pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_cache: &Cache) -> Options { const MIN_PARALLELISM: usize = 2; + const DEFAULT_STATS_LEVEL: StatsLevel = if cfg!(debug_assertions) { + StatsLevel::ExceptDetailedTimers + } else { + StatsLevel::DisableAll + }; let mut opts = Options::default(); @@ -68,8 +73,18 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ set_compression_defaults(&mut opts, config); // Misc - opts.set_disable_auto_compactions(!config.rocksdb_compaction); opts.create_if_missing(true); + opts.set_disable_auto_compactions(!config.rocksdb_compaction); + + opts.set_statistics_level(match config.rocksdb_stats_level { + 0 => StatsLevel::DisableAll, + 1 => DEFAULT_STATS_LEVEL, + 2 => StatsLevel::ExceptHistogramOrTimers, + 3 => StatsLevel::ExceptTimers, + 4 => StatsLevel::ExceptDetailedTimers, + 5 => StatsLevel::ExceptTimeForMutex, + 6_u8..=u8::MAX => StatsLevel::All, + }); // Default: https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes#ktoleratecorruptedtailrecords // From f261d44edb97c63a80c73e1598ca154bff540e93 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 2 Aug 2024 01:21:51 +0000 Subject: [PATCH 0634/2091] add method to iterate opened database columns Signed-off-by: Jason Volk --- src/database/database.rs | 10 +++++++++- src/database/maps.rs | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/database/database.rs b/src/database/database.rs index 1d7bbc33..c357d50f 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -2,7 +2,12 @@ use std::{ops::Index, sync::Arc}; use conduit::{Result, Server}; -use crate::{cork::Cork, maps, maps::Maps, Engine, Map}; +use crate::{ + cork::Cork, + maps, + maps::{Maps, MapsKey, MapsVal}, + Engine, Map, +}; pub struct Database { pub db: Arc, @@ -30,6 +35,9 @@ impl Database { #[inline] #[must_use] pub fn cork_and_sync(&self) -> Cork { Cork::new(&self.db, true, true) } + + #[inline] + pub fn iter_maps(&self) -> impl Iterator + '_ { self.map.iter() } } impl Index<&str> for Database { diff --git a/src/database/maps.rs b/src/database/maps.rs index de78eaed..0207b4d4 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -4,7 +4,9 @@ use conduit::Result; use crate::{Engine, Map}; -pub type Maps = BTreeMap>; +pub type Maps = BTreeMap; +pub(crate) type MapsVal = Arc; +pub(crate) type MapsKey = String; pub(crate) fn open(db: &Arc) -> Result { open_list(db, MAPS) } From ac941a801ab0c1cf38202adfc651236b19120d12 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 2 Aug 2024 01:40:41 +0000 Subject: [PATCH 0635/2091] add interface to query rocksdb properties w/ admin cmd Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 21 +++++++++++++++++++++ src/admin/debug/mod.rs | 8 ++++++++ src/database/engine.rs | 18 +++++++++++++++++- src/database/map.rs | 6 +++++- 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 53f4cf64..014ba79b 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -828,3 +828,24 @@ pub(super) async fn list_dependencies(&self, names: bool) -> Result, map: Option, +) -> Result { + let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned()); + let map_name = map.as_ref().map_or(utils::string::EMPTY, String::as_str); + + let mut out = String::new(); + for (name, map) in self.services.db.iter_maps() { + if !map_name.is_empty() && *map_name != *name { + continue; + } + + let res = map.property(&property)?; + let res = res.trim(); + writeln!(out, "##### {name}:\n```\n{res}\n```")?; + } + + Ok(RoomMessageEventContent::notice_markdown(out)) +} diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index fbe6fd26..1f51a35e 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -184,6 +184,14 @@ pub(super) enum DebugCommand { names: bool, }, + /// - Get database statistics + DatabaseStats { + property: Option, + + #[arg(short, long, alias("column"))] + map: Option, + }, + /// - Developer test stubs #[command(subcommand)] #[allow(non_snake_case)] diff --git a/src/database/engine.rs b/src/database/engine.rs index bf172551..3975d3d9 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -1,5 +1,6 @@ use std::{ collections::{BTreeSet, HashMap}, + ffi::CStr, fmt::Write, path::PathBuf, sync::{atomic::AtomicU32, Arc, Mutex, RwLock}, @@ -9,7 +10,8 @@ use conduit::{debug, error, info, utils::time::rfc2822_from_seconds, warn, Err, use rocksdb::{ backup::{BackupEngine, BackupEngineOptions}, perf::get_memory_usage_stats, - BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode, Env, MultiThreaded, Options, + AsColumnFamilyRef, BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode, Env, + MultiThreaded, Options, }; use crate::{ @@ -240,6 +242,20 @@ impl Engine { }, } } + + /// Query for database property by null-terminated name which is expected to + /// have a result with an integer representation. This is intended for + /// low-overhead programmatic use. + pub(crate) fn property_integer(&self, cf: &impl AsColumnFamilyRef, name: &CStr) -> Result { + result(self.db.property_int_value_cf(cf, name)) + .and_then(|val| val.map_or_else(|| Err!("Property {name:?} not found."), Ok)) + } + + /// Query for database property by name receiving the result in a string. + pub(crate) fn property(&self, cf: &impl AsColumnFamilyRef, name: &str) -> Result { + result(self.db.property_value_cf(cf, name)) + .and_then(|val| val.map_or_else(|| Err!("Property {name:?} not found."), Ok)) + } } pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> { diff --git a/src/database/map.rs b/src/database/map.rs index 1b35a72a..ddae8c81 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -1,4 +1,4 @@ -use std::{future::Future, mem::size_of, pin::Pin, sync::Arc}; +use std::{ffi::CStr, future::Future, mem::size_of, pin::Pin, sync::Arc}; use conduit::{utils, Result}; use rocksdb::{ @@ -189,6 +189,10 @@ impl Map { self.watchers.watch(prefix) } + pub fn property_integer(&self, name: &CStr) -> Result { self.db.property_integer(&self.cf(), name) } + + pub fn property(&self, name: &str) -> Result { self.db.property(&self.cf(), name) } + #[inline] pub fn name(&self) -> &str { &self.name } From 13aa3537658e689c6a73f80bdfb288f275735a3a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 2 Aug 2024 22:09:30 +0000 Subject: [PATCH 0636/2091] add arrayvec as a dependency Signed-off-by: Jason Volk --- Cargo.lock | 7 +++++++ Cargo.toml | 3 +++ src/core/Cargo.toml | 1 + 3 files changed, 11 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index c6ae1f8e..2b6b129f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -71,6 +71,12 @@ dependencies = [ "password-hash", ] +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "as_variant" version = "1.2.0" @@ -627,6 +633,7 @@ name = "conduit_core" version = "0.4.6" dependencies = [ "argon2", + "arrayvec", "axum", "bytes", "cargo_toml", diff --git a/Cargo.toml b/Cargo.toml index 09ba319e..4f808d28 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,9 @@ version = "0.4.6" [workspace.metadata.crane] name = "conduit" +[workspace.dependencies.arrayvec] +version = "0.7.4" + [workspace.dependencies.const-str] version = "0.5.7" diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index f915c43e..75f0a8b6 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -51,6 +51,7 @@ sha256_media = [] [dependencies] argon2.workspace = true +arrayvec.workspace = true axum.workspace = true bytes.workspace = true cargo_toml.workspace = true From 0c6bbde25f5cba6f3878c8b933392a8615ffb5f5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 2 Aug 2024 22:18:16 +0000 Subject: [PATCH 0637/2091] fix markdown table formatter for log capture Signed-off-by: Jason Volk --- src/core/log/fmt.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/log/fmt.rs b/src/core/log/fmt.rs index dbaef890..62944977 100644 --- a/src/core/log/fmt.rs +++ b/src/core/log/fmt.rs @@ -32,7 +32,7 @@ where S: Write, { let level = level.as_str().to_uppercase(); - writeln!(out, "| `{level:>5}` | `{span:^12}` | `{msg} |")?; + writeln!(out, "| {level:>5} | {span:^12} | {msg} |")?; Ok(()) } @@ -41,7 +41,7 @@ pub fn markdown_table_head(out: &mut S) -> Result<()> where S: Write, { - write!(out, "| level | span | message |\n|------:|:----:|:--------|\n")?; + write!(out, "| level | span | message |\n| ------: | :-----: | :------- |\n")?; Ok(()) } From 15184d1a79aa702f1e48529395789d58ecf2e660 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 2 Aug 2024 22:59:14 +0000 Subject: [PATCH 0638/2091] Add span scope names array to capture filter data; optimize values visitor vec. Signed-off-by: Jason Volk --- src/core/log/capture/data.rs | 5 ++--- src/core/log/capture/layer.rs | 27 +++++++++++++++++++++------ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/core/log/capture/data.rs b/src/core/log/capture/data.rs index 3f92101d..63f09b6c 100644 --- a/src/core/log/capture/data.rs +++ b/src/core/log/capture/data.rs @@ -7,7 +7,8 @@ pub struct Data<'a> { pub layer: &'a Layer, pub event: &'a Event<'a>, pub current: &'a Current, - pub values: Option<&'a mut [Value]>, + pub values: &'a [Value], + pub scope: &'a [&'static str], } impl Data<'_> { @@ -23,8 +24,6 @@ impl Data<'_> { #[must_use] pub fn message(&self) -> &str { self.values - .as_ref() - .expect("values are not composed for a filter") .iter() .find(|(k, _)| *k == "message") .map_or("", |(_, v)| v.as_str()) diff --git a/src/core/log/capture/layer.rs b/src/core/log/capture/layer.rs index 57f22dc7..674bc379 100644 --- a/src/core/log/capture/layer.rs +++ b/src/core/log/capture/layer.rs @@ -1,21 +1,25 @@ use std::{fmt, sync::Arc}; +use arrayvec::ArrayVec; use tracing::field::{Field, Visit}; use tracing_core::{Event, Subscriber}; use tracing_subscriber::{layer::Context, registry::LookupSpan}; use super::{Capture, Data, State}; -pub type Value = (&'static str, String); - pub struct Layer { state: Arc, } struct Visitor { - values: Vec, + values: Values, } +type Values = ArrayVec; +pub type Value = (&'static str, String); + +type ScopeNames = ArrayVec<&'static str, 32>; + impl Layer { #[inline] pub fn new(state: &Arc) -> Self { @@ -51,8 +55,9 @@ fn handle(layer: &Layer, capture: &Capture, event: &Event<'_>, ctx: &Context< where S: Subscriber + for<'a> LookupSpan<'a>, { + let names = ScopeNames::new(); let mut visitor = Visitor { - values: Vec::new(), + values: Values::new(), }; event.record(&mut visitor); @@ -61,7 +66,8 @@ where layer, event, current: &ctx.current_span(), - values: Some(&mut visitor.values), + values: &visitor.values, + scope: &names, }); } @@ -69,12 +75,21 @@ fn filter(layer: &Layer, capture: &Capture, event: &Event<'_>, ctx: &Context< where S: Subscriber + for<'a> LookupSpan<'a>, { + let values = Values::new(); + let mut names = ScopeNames::new(); + if let Some(scope) = ctx.event_scope(event) { + for span in scope { + names.push(span.name()); + } + } + capture.filter.as_ref().map_or(true, |filter| { filter(Data { layer, event, current: &ctx.current_span(), - values: None, + values: &values, + scope: &names, }) }) } From e9ab54844563955e60331720ff183128b574552c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 3 Aug 2024 00:27:18 +0000 Subject: [PATCH 0639/2091] tweak some tracing log levels Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 014ba79b..720e6f63 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -213,7 +213,7 @@ pub(super) async fn get_remote_pdu( Error::BadRequest(ErrorKind::Unknown, "Received response from server but failed to parse PDU") })?; - debug!("Attempting to parse PDU: {:?}", &response.pdu); + trace!("Attempting to parse PDU: {:?}", &response.pdu); let parsed_pdu = { let parsed_result = self .services From f98bfbbdcd43e4f9f5c2448e93fd3584756e9014 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 3 Aug 2024 02:16:52 +0000 Subject: [PATCH 0640/2091] add string stream convenience util; add ?Sized bounds on log fmt functors Signed-off-by: Jason Volk --- src/core/log/fmt.rs | 8 ++++---- src/core/utils/string.rs | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/core/log/fmt.rs b/src/core/log/fmt.rs index 62944977..8766eb8d 100644 --- a/src/core/log/fmt.rs +++ b/src/core/log/fmt.rs @@ -5,7 +5,7 @@ use crate::Result; pub fn html(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()> where - S: Write, + S: Write + ?Sized, { let color = color::code_tag(level); let level = level.as_str().to_uppercase(); @@ -19,7 +19,7 @@ where pub fn markdown(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()> where - S: Write, + S: Write + ?Sized, { let level = level.as_str().to_uppercase(); writeln!(out, "`{level:>5}` `{span:^12}` `{msg}`")?; @@ -29,7 +29,7 @@ where pub fn markdown_table(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()> where - S: Write, + S: Write + ?Sized, { let level = level.as_str().to_uppercase(); writeln!(out, "| {level:>5} | {span:^12} | {msg} |")?; @@ -39,7 +39,7 @@ where pub fn markdown_table_head(out: &mut S) -> Result<()> where - S: Write, + S: Write + ?Sized, { write!(out, "| level | span | message |\n| ------: | :-----: | :------- |\n")?; diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index a597d198..85282b30 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -30,6 +30,16 @@ macro_rules! is_format { }; } +#[inline] +pub fn collect_stream(func: F) -> Result +where + F: FnOnce(&mut dyn std::fmt::Write) -> Result<()>, +{ + let mut out = String::new(); + func(&mut out)?; + Ok(out) +} + #[inline] #[must_use] pub fn camel_to_snake_string(s: &str) -> String { From a552321287b00e7fe7c3fe11a642f4bbae4c09a1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 3 Aug 2024 02:18:59 +0000 Subject: [PATCH 0641/2091] add Clap to Error Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/core/Cargo.toml | 1 + src/core/error/mod.rs | 2 ++ 3 files changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 2b6b129f..ac188fe3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -639,6 +639,7 @@ dependencies = [ "cargo_toml", "checked_ops", "chrono", + "clap", "conduit_macros", "const-str", "ctor", diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 75f0a8b6..a5c379e9 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -57,6 +57,7 @@ bytes.workspace = true cargo_toml.workspace = true checked_ops.workspace = true chrono.workspace = true +clap.workspace = true conduit-macros.workspace = true const-str.workspace = true ctor.workspace = true diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 89148d8c..b5721dee 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -62,6 +62,8 @@ pub enum Error { TomlSerError(#[from] toml::ser::Error), #[error("{0}")] TomlDeError(#[from] toml::de::Error), + #[error("{0}")] + Clap(#[from] clap::error::Error), // ruma #[error("{0}")] From eded585f79f7d0b75161fc6be04c3bb71b2e8fa0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 3 Aug 2024 02:20:06 +0000 Subject: [PATCH 0642/2091] rename some command types for consistency Signed-off-by: Jason Volk --- src/admin/handler.rs | 8 ++++---- src/service/admin/mod.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 738d748c..16518839 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -11,7 +11,7 @@ use ruma::{ OwnedEventId, }; use service::{ - admin::{CommandInput, CommandOutput, CommandResult, HandlerResult}, + admin::{CommandInput, CommandOutput, HandlerFuture, HandlerResult}, Services, }; @@ -21,12 +21,12 @@ use crate::{admin, admin::AdminCommand, Command}; pub(super) fn complete(line: &str) -> String { complete_command(AdminCommand::command(), line) } #[must_use] -pub(super) fn handle(services: Arc, command: CommandInput) -> HandlerResult { +pub(super) fn handle(services: Arc, command: CommandInput) -> HandlerFuture { Box::pin(handle_command(services, command)) } #[tracing::instrument(skip_all, name = "admin")] -async fn handle_command(services: Arc, command: CommandInput) -> CommandResult { +async fn handle_command(services: Arc, command: CommandInput) -> HandlerResult { AssertUnwindSafe(Box::pin(process_command(services, &command))) .catch_unwind() .await @@ -40,7 +40,7 @@ async fn process_command(services: Arc, command: &CommandInput) -> Com .and_then(|content| reply(content, command.reply_id.clone())) } -fn handle_panic(error: &Error, command: CommandInput) -> CommandResult { +fn handle_panic(error: &Error, command: CommandInput) -> HandlerResult { let link = "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}"); let content = RoomMessageEventContent::notice_markdown(msg); diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 00d6597e..0e8a65f1 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -51,9 +51,9 @@ pub struct CommandInput { } pub type Completer = fn(&str) -> String; -pub type Handler = fn(Arc, CommandInput) -> HandlerResult; -pub type HandlerResult = Pin + Send>>; -pub type CommandResult = Result; +pub type Handler = fn(Arc, CommandInput) -> HandlerFuture; +pub type HandlerFuture = Pin + Send>>; +pub type HandlerResult = Result; pub type CommandOutput = Option; const COMMAND_QUEUE_LIMIT: usize = 512; @@ -173,7 +173,7 @@ impl Service { } } - async fn process_command(&self, command: CommandInput) -> CommandResult { + async fn process_command(&self, command: CommandInput) -> HandlerResult { let Some(services) = self .services .services From 94b805de0bc785ed15baa1b8033235492d10e344 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 3 Aug 2024 05:34:38 +0000 Subject: [PATCH 0643/2091] generalize log capture to all admin commands; simplify handler Signed-off-by: Jason Volk --- src/admin/command.rs | 5 +- src/admin/debug/commands.rs | 28 ++------ src/admin/handler.rs | 140 ++++++++++++++++++++---------------- 3 files changed, 87 insertions(+), 86 deletions(-) diff --git a/src/admin/command.rs b/src/admin/command.rs index fbfdd2ba..a26564fc 100644 --- a/src/admin/command.rs +++ b/src/admin/command.rs @@ -1,6 +1,9 @@ -use service::Services; +use std::time::SystemTime; + +use conduit_service::Services; pub(crate) struct Command<'a> { pub(crate) services: &'a Services, pub(crate) body: &'a [&'a str], + pub(crate) timer: SystemTime, } diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 720e6f63..2d967006 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -1,16 +1,12 @@ use std::{ collections::{BTreeMap, HashMap}, fmt::Write, - sync::{Arc, Mutex}, + sync::Arc, time::{Instant, SystemTime}, }; use api::client::validate_and_add_event_id; -use conduit::{ - debug, debug_error, err, info, log, - log::{capture, Capture}, - utils, warn, Error, PduEvent, Result, -}; +use conduit::{debug, debug_error, err, info, trace, utils, warn, Error, PduEvent, Result}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, events::room::message::RoomMessageEventContent, @@ -717,30 +713,14 @@ pub(super) async fn resolve_true_destination( )); } - let filter: &capture::Filter = &|data| { - data.level() <= log::Level::DEBUG - && data.mod_name().starts_with("conduit") - && matches!(data.span_name(), "actual" | "well-known" | "srv") - }; - - let state = &self.services.server.log.capture; - let logs = Arc::new(Mutex::new(String::new())); - let capture = Capture::new(state, Some(filter), capture::fmt_markdown(logs.clone())); - - let capture_scope = capture.start(); let actual = self .services .resolver .resolve_actual_dest(&server_name, !no_cache) .await?; - drop(capture_scope); - let msg = format!( - "{}\nDestination: {}\nHostname URI: {}", - logs.lock().expect("locked"), - actual.dest, - actual.host, - ); + let msg = format!("Destination: {}\nHostname URI: {}", actual.dest, actual.host,); + Ok(RoomMessageEventContent::text_markdown(msg)) } diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 16518839..fd4fe839 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,7 +1,21 @@ -use std::{panic::AssertUnwindSafe, sync::Arc, time::Instant}; +use std::{ + panic::AssertUnwindSafe, + sync::{Arc, Mutex}, + time::SystemTime, +}; use clap::{CommandFactory, Parser}; -use conduit::{checked, error, trace, utils::string::common_prefix, Error, Result}; +use conduit::{ + debug, error, + log::{ + capture, + capture::Capture, + fmt::{markdown_table, markdown_table_head}, + }, + trace, + utils::string::{collect_stream, common_prefix}, + Error, Result, +}; use futures_util::future::FutureExt; use ruma::{ events::{ @@ -14,6 +28,7 @@ use service::{ admin::{CommandInput, CommandOutput, HandlerFuture, HandlerResult}, Services, }; +use tracing::Level; use crate::{admin, admin::AdminCommand, Command}; @@ -34,10 +49,21 @@ async fn handle_command(services: Arc, command: CommandInput) -> Handl .or_else(|error| handle_panic(&error, command)) } -async fn process_command(services: Arc, command: &CommandInput) -> CommandOutput { - process(services, &command.command) +async fn process_command(services: Arc, input: &CommandInput) -> CommandOutput { + let (command, args, body) = match parse(&services, input) { + Err(error) => return error, + Ok(parsed) => parsed, + }; + + let context = Command { + services: &services, + body: &body, + timer: SystemTime::now(), + }; + + process(&context, command, &args) .await - .and_then(|content| reply(content, command.reply_id.clone())) + .and_then(|content| reply(content, input.reply_id.clone())) } fn handle_panic(error: &Error, command: CommandInput) -> HandlerResult { @@ -59,69 +85,61 @@ fn reply(mut content: RoomMessageEventContent, reply_id: Option) - } // Parse and process a message from the admin room -async fn process(services: Arc, msg: &str) -> CommandOutput { - let lines = msg.lines().filter(|l| !l.trim().is_empty()); - let command = lines - .clone() - .next() - .expect("each string has at least one line"); - let (parsed, body) = match parse_command(command) { - Ok(parsed) => parsed, - Err(error) => { - let server_name = services.globals.server_name(); - let message = error.replace("server.name", server_name.as_str()); - return Some(RoomMessageEventContent::notice_markdown(message)); - }, +async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) -> CommandOutput { + let filter: &capture::Filter = + &|data| data.level() <= Level::DEBUG && data.mod_name().starts_with("conduit") && data.scope.contains(&"admin"); + let logs = Arc::new(Mutex::new( + collect_stream(|s| markdown_table_head(s)).expect("markdown table header"), + )); + + let capture = Capture::new( + &context.services.server.log.capture, + Some(filter), + capture::fmt(markdown_table, logs.clone()), + ); + + let capture_scope = capture.start(); + let result = Box::pin(admin::process(command, context)).await; + drop(capture_scope); + + debug!( + ok = result.is_ok(), + elapsed = ?context.timer.elapsed(), + command = ?args, + "command processed" + ); + + let logs = logs.lock().expect("locked"); + let output = match result { + Err(error) => format!("{logs}\nEncountered an error while handling the command:\n```\n{error:#?}\n```"), + Ok(reply) => format!("{logs}\n{}", reply.body()), //TODO: content is recreated to add logs }; - let body = parse_body(AdminCommand::command(), &body, lines.skip(1).collect()).expect("trailing body parsed"); - let context = Command { - services: &services, - body: &body, - }; - let timer = Instant::now(); - let result = Box::pin(admin::process(parsed, &context)).await; - let elapsed = timer.elapsed(); - conduit::debug!(?command, ok = result.is_ok(), "command processed in {elapsed:?}"); - match result { - Ok(reply) => Some(reply), - Err(error) => Some(RoomMessageEventContent::notice_markdown(format!( - "Encountered an error while handling the command:\n```\n{error:#?}\n```" - ))), - } + Some(RoomMessageEventContent::notice_markdown(output)) } // Parse chat messages from the admin room into an AdminCommand object -fn parse_command(command_line: &str) -> Result<(AdminCommand, Vec), String> { - let argv = parse_line(command_line); - let com = AdminCommand::try_parse_from(&argv).map_err(|error| error.to_string())?; - Ok((com, argv)) +fn parse<'a>( + services: &Arc, input: &'a CommandInput, +) -> Result<(AdminCommand, Vec, Vec<&'a str>), CommandOutput> { + let lines = input.command.lines().filter(|line| !line.trim().is_empty()); + let command_line = lines.clone().next().expect("command missing first line"); + let body = lines.skip(1).collect(); + match parse_command(command_line) { + Ok((command, args)) => Ok((command, args, body)), + Err(error) => { + let message = error + .to_string() + .replace("server.name", services.globals.server_name().as_str()); + Err(Some(RoomMessageEventContent::notice_markdown(message))) + }, + } } -fn parse_body<'a>(mut cmd: clap::Command, body: &'a [String], lines: Vec<&'a str>) -> Result> { - let mut start = 1; - 'token: for token in body.iter().skip(1) { - let cmd_ = cmd.clone(); - for sub in cmd_.get_subcommands() { - if sub.get_name() == *token { - start = checked!(start + 1)?; - cmd = sub.clone(); - continue 'token; - } - } - - // positional arguments have to be skipped too - let num_posargs = cmd_.get_positionals().count(); - start = checked!(start + num_posargs)?; - break; - } - - Ok(body - .iter() - .skip(start) - .map(String::as_str) - .chain(lines) - .collect::>()) +fn parse_command(line: &str) -> Result<(AdminCommand, Vec)> { + let argv = parse_line(line); + let command = AdminCommand::try_parse_from(&argv)?; + Ok((command, argv)) } fn complete_command(mut cmd: clap::Command, line: &str) -> String { From 9b9a91f6ef6016468461f806bdec6f7fb3dff00b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 4 Aug 2024 06:47:37 +0000 Subject: [PATCH 0644/2091] add our_modules() for log capture filter convenience Signed-off-by: Jason Volk --- src/admin/handler.rs | 2 +- src/core/info/mod.rs | 3 +++ src/core/log/capture/data.rs | 8 ++++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index fd4fe839..c25c380f 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -87,7 +87,7 @@ fn reply(mut content: RoomMessageEventContent, reply_id: Option) - // Parse and process a message from the admin room async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) -> CommandOutput { let filter: &capture::Filter = - &|data| data.level() <= Level::DEBUG && data.mod_name().starts_with("conduit") && data.scope.contains(&"admin"); + &|data| data.level() <= Level::DEBUG && data.our_modules() && data.scope.contains(&"admin"); let logs = Arc::new(Mutex::new( collect_stream(|s| markdown_table_head(s)).expect("markdown table header"), )); diff --git a/src/core/info/mod.rs b/src/core/info/mod.rs index e11a6021..e4a33b4e 100644 --- a/src/core/info/mod.rs +++ b/src/core/info/mod.rs @@ -6,3 +6,6 @@ pub mod rustc; pub mod version; pub use conduit_macros::rustc_flags_capture; + +pub const MODULE_ROOT: &str = const_str::split!(std::module_path!(), "::")[0]; +pub const CRATE_PREFIX: &str = const_str::split!(MODULE_ROOT, '_')[0]; diff --git a/src/core/log/capture/data.rs b/src/core/log/capture/data.rs index 63f09b6c..0ad7a6c2 100644 --- a/src/core/log/capture/data.rs +++ b/src/core/log/capture/data.rs @@ -2,6 +2,7 @@ use tracing::Level; use tracing_core::{span::Current, Event}; use super::{layer::Value, Layer}; +use crate::{info, utils::string::EMPTY}; pub struct Data<'a> { pub layer: &'a Layer, @@ -12,6 +13,9 @@ pub struct Data<'a> { } impl Data<'_> { + #[must_use] + pub fn our_modules(&self) -> bool { self.mod_name().starts_with(info::CRATE_PREFIX) } + #[must_use] pub fn level(&self) -> Level { *self.event.metadata().level() } @@ -19,13 +23,13 @@ impl Data<'_> { pub fn mod_name(&self) -> &str { self.event.metadata().module_path().unwrap_or_default() } #[must_use] - pub fn span_name(&self) -> &str { self.current.metadata().map_or("", |s| s.name()) } + pub fn span_name(&self) -> &str { self.current.metadata().map_or(EMPTY, |s| s.name()) } #[must_use] pub fn message(&self) -> &str { self.values .iter() .find(|(k, _)| *k == "message") - .map_or("", |(_, v)| v.as_str()) + .map_or(EMPTY, |(_, v)| v.as_str()) } } From 6662aa21ef4ab8e444fd3c62470fc090f23679fb Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 8 Aug 2024 14:41:46 -0400 Subject: [PATCH 0645/2091] bump complement to 6e4426a9e63233f9821a4d2382bfed145244183f MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'complement': 'github:matrix-org/complement/0d14432e010482ea9e13a6f7c47c1533c0c9d62f' (2024-07-10) → 'github:matrix-org/complement/6e4426a9e63233f9821a4d2382bfed145244183f' (2024-07-30) Signed-off-by: strawberry --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 9d7a162e..0688e378 100644 --- a/flake.lock +++ b/flake.lock @@ -81,11 +81,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1720637557, - "narHash": "sha256-oZz6nCmFmdJZpC+K1iOG2KkzTI6rlAmndxANPDVU7X0=", + "lastModified": 1722323564, + "narHash": "sha256-6w6/N8walz4Ayc9zu7iySqJRmGFukhkaICLn4dweAcA=", "owner": "matrix-org", "repo": "complement", - "rev": "0d14432e010482ea9e13a6f7c47c1533c0c9d62f", + "rev": "6e4426a9e63233f9821a4d2382bfed145244183f", "type": "github" }, "original": { From 28cd784972f9e6e78a77ee54ca07d998ca15a788 Mon Sep 17 00:00:00 2001 From: Matthew Scheffel Date: Fri, 16 Aug 2024 16:49:51 -0300 Subject: [PATCH 0646/2091] `paginate_relations_with_filter` DRY & flip ordering #504 --- src/service/rooms/pdu_metadata/mod.rs | 84 +++++++-------------------- 1 file changed, 20 insertions(+), 64 deletions(-) diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 7546dcb2..d9eaf324 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -85,8 +85,6 @@ impl Service { .unwrap_or(10) .min(100); - let next_token; - // Spec (v1.10) recommends depth of at least 3 let depth: u8 = if recurse { 3 @@ -94,10 +92,8 @@ impl Service { 1 }; - match dir { - Direction::Forward => { - let relations_until = &self.relations_until(sender_user, room_id, target, from, depth)?; - let events_after: Vec<_> = relations_until // TODO: should be relations_after + let relations_until = &self.relations_until(sender_user, room_id, target, from, depth)?; + let events: Vec<_> = relations_until // TODO: should be relations_after .iter() .filter(|(_, pdu)| { filter_event_type.as_ref().map_or(true, |t| &pdu.kind == t) @@ -121,70 +117,30 @@ impl Service { .take_while(|(k, _)| Some(k) != to.as_ref()) // Stop at `to` .collect(); - next_token = events_after.last().map(|(count, _)| count).copied(); + let next_token = events.last().map(|(count, _)| count).copied(); - let events_after: Vec<_> = events_after + let events_chunk: Vec<_> = match dir { + Direction::Forward => events + .into_iter() + .map(|(_, pdu)| pdu.to_message_like_event()) + .collect(), + Direction::Backward => events .into_iter() .rev() // relations are always most recent first .map(|(_, pdu)| pdu.to_message_like_event()) - .collect(); + .collect(), + }; - Ok(get_relating_events::v1::Response { - chunk: events_after, - next_batch: next_token.map(|t| t.stringify()), - prev_batch: Some(from.stringify()), - recursion_depth: if recurse { - Some(depth.into()) - } else { - None - }, - }) + Ok(get_relating_events::v1::Response { + chunk: events_chunk, + next_batch: next_token.map(|t| t.stringify()), + prev_batch: Some(from.stringify()), + recursion_depth: if recurse { + Some(depth.into()) + } else { + None }, - Direction::Backward => { - let relations_until = &self.relations_until(sender_user, room_id, target, from, depth)?; - let events_before: Vec<_> = relations_until - .iter() - .filter(|(_, pdu)| { - filter_event_type.as_ref().map_or(true, |t| &pdu.kind == t) - && if let Ok(content) = - serde_json::from_str::(pdu.content.get()) - { - filter_rel_type - .as_ref() - .map_or(true, |r| &content.relates_to.rel_type == r) - } else { - false - } - }) - .take(limit) - .filter(|(_, pdu)| { - self.services - .state_accessor - .user_can_see_event(sender_user, room_id, &pdu.event_id) - .unwrap_or(false) - }) - .take_while(|&(k, _)| Some(k) != to.as_ref()) // Stop at `to` - .collect(); - - next_token = events_before.last().map(|(count, _)| count).copied(); - - let events_before: Vec<_> = events_before - .into_iter() - .map(|(_, pdu)| pdu.to_message_like_event()) - .collect(); - - Ok(get_relating_events::v1::Response { - chunk: events_before, - next_batch: next_token.map(|t| t.stringify()), - prev_batch: Some(from.stringify()), - recursion_depth: if recurse { - Some(depth.into()) - } else { - None - }, - }) - }, - } + }) } pub fn relations_until<'a>( From d26493562528172595ea063c55f76b7eb0d6b7c4 Mon Sep 17 00:00:00 2001 From: Aurora Master <45430047+AuroraMaster@users.noreply.github.com> Date: Tue, 27 Aug 2024 15:40:38 +0800 Subject: [PATCH 0647/2091] Update ci.yml --- .gitea/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 6d063ee7..9ce7c993 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -50,7 +50,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Sync repository - uses: https://github.com/https://github.com/actions/checkout@v4 + uses: https://github.com/actions/checkout@v4 - name: Tag comparison check if: startsWith(gitea.ref, 'refs/tags/v') From aeb6fbe70f7c967e8831d5335d9bab7481358f60 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 29 Aug 2024 14:24:37 -0400 Subject: [PATCH 0648/2091] remove matrix.org code of conduct from conduwuit's foundational rules while their code of conduct has very great baseline points, this is not a code of conduct the foundation upholds themselves, so it would be a disserve to mislead folks into using this as a baseline. Signed-off-by: strawberry --- docs/conduwuit_coc.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/conduwuit_coc.md b/docs/conduwuit_coc.md index 2547e64f..1bd519c9 100644 --- a/docs/conduwuit_coc.md +++ b/docs/conduwuit_coc.md @@ -7,9 +7,8 @@ This space is dedicated to fostering a positive, supportive, and inclusive envir Conduct applies to all conduwuit spaces, including any further community rooms that reference this CoC. Here are our guidelines to help maintain the welcoming atmosphere that sets conduwuit apart. -For the foundational rules, please refer to the [Matrix.org Code of Conduct](https://matrix.org/legal/code-of-conduct/) -and the [Contributor's Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md). Below are -additional guidelines specific to the conduwuit community. +For the general foundational rules, please refer to the [Contributor's Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md). +Below are additional guidelines specific to the conduwuit community. ## Our Values and Guidelines From 982f9eb780cbfe6fe7b9316f05891e0fd6a14a0c Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 30 Jul 2024 00:05:22 -0400 Subject: [PATCH 0649/2091] implement MSC4166, returning M_NOT_FOUND if no TURN URIs are set Signed-off-by: strawberry --- src/api/client/voip.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index ed6971ee..a51e2657 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -2,7 +2,7 @@ use std::time::{Duration, SystemTime}; use axum::extract::State; use base64::{engine::general_purpose, Engine as _}; -use conduit::utils; +use conduit::{utils, Err}; use hmac::{Hmac, Mac}; use ruma::{api::client::voip::get_turn_server_info, SecondsSinceUnixEpoch, UserId}; use sha1::Sha1; @@ -19,6 +19,11 @@ type HmacSha1 = Hmac; pub(crate) async fn turn_server_route( State(services): State, body: Ruma, ) -> Result { + // MSC4166: return M_NOT_FOUND 404 if no TURN URIs are specified in any way + if services.server.config.turn_uris.is_empty() { + return Err!(Request(NotFound("Not Found"))); + } + let turn_secret = services.globals.turn_secret().clone(); let (username, password) = if !turn_secret.is_empty() { From fd96f597cd358b9cf83e7878f1f4a6ef01859065 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 9 Aug 2024 16:28:11 -0400 Subject: [PATCH 0650/2091] respect m.room.guest_access on room joins Signed-off-by: strawberry --- src/api/client/membership.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 4b27d8e3..9eb40b4a 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -662,6 +662,12 @@ pub async fn join_room_by_id_helper( ) -> Result { let state_lock = services.rooms.state.mutex.lock(room_id).await; + let user_is_guest = services.users.is_deactivated(sender_user).unwrap_or(false); + + if matches!(services.rooms.state_accessor.guest_can_join(room_id), Ok(false)) && user_is_guest { + return Err!(Request(Forbidden("Guests are not allowed to join this room"))); + } + if matches!(services.rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { debug_warn!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { From 8fe19a6ef0d02f9790dc01ff5226a514760e5b2b Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 9 Aug 2024 16:28:25 -0400 Subject: [PATCH 0651/2091] dont send duplicate membership update events if pfp and display name are same Signed-off-by: strawberry --- src/api/client/profile.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 8f6d9056..a72938b5 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -257,6 +257,12 @@ pub(crate) async fn get_profile_route( pub async fn update_displayname( services: &Services, user_id: OwnedUserId, displayname: Option, all_joined_rooms: Vec, ) -> Result<()> { + let current_display_name = services.users.displayname(&user_id).unwrap_or_default(); + + if displayname == current_display_name { + return Ok(()); + } + services .users .set_displayname(&user_id, displayname.clone()) @@ -305,6 +311,13 @@ pub async fn update_avatar_url( services: &Services, user_id: OwnedUserId, avatar_url: Option, blurhash: Option, all_joined_rooms: Vec, ) -> Result<()> { + let current_avatar_url = services.users.avatar_url(&user_id).unwrap_or_default(); + let current_blurhash = services.users.blurhash(&user_id).unwrap_or_default(); + + if current_avatar_url == avatar_url && current_blurhash == blurhash { + return Ok(()); + } + services .users .set_avatar_url(&user_id, avatar_url.clone()) From 52ccad04a667ad26a0200f63b37aac2ff376e777 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 26 Jul 2024 00:45:23 -0400 Subject: [PATCH 0652/2091] apply `forbidden_remote_server_names` to outbound sending and inbound federation handling Signed-off-by: strawberry --- conduwuit-example.toml | 8 +++++--- src/api/router/auth.rs | 15 +++++++++++++-- src/service/sending/send.rs | 13 ++++++++++++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index fdf53783..553f62a1 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -224,9 +224,11 @@ registration_token = "change this token for something specific to your server" # No default. # forbidden_alias_names = [] -# List of forbidden server names that we will block all client room joins, incoming federated room directory requests, incoming federated invites for, and incoming federated joins. This check is applied on the room ID, room alias, sender server name, and sender user's server name. -# Basically "global" ACLs. For our user (client) checks, admin users are allowed. -# No default. +# List of forbidden server names that we will block incoming AND outgoing federation with, and block client room joins / remote user invites. +# +# This check is applied on the room ID, room alias, sender server name, sender user's server name, inbound federation X-Matrix origin, and outbound federation handler. +# +# Basically "global" ACLs. No default. # forbidden_remote_server_names = [] # List of forbidden server names that we will block all outgoing federated room directory requests for. Useful for preventing our users from wandering into bad servers or spaces. diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 838c5e79..fe98e458 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -6,7 +6,7 @@ use axum_extra::{ typed_header::TypedHeaderRejectionReason, TypedHeader, }; -use conduit::{warn, Err, Error, Result}; +use conduit::{debug_info, warn, Err, Error, Result}; use http::uri::PathAndQuery; use ruma::{ api::{client::error::ErrorKind, AuthScheme, Metadata}, @@ -185,7 +185,7 @@ fn auth_appservice(services: &Services, request: &Request, info: Box, ) -> Result { - if !services.globals.allow_federation() { + if !services.server.config.allow_federation { return Err!(Config("allow_federation", "Federation is disabled.")); } @@ -206,6 +206,17 @@ async fn auth_server( })?; let origin = &x_matrix.origin; + + if services + .server + .config + .forbidden_remote_server_names + .contains(origin) + { + debug_info!("Refusing to accept inbound federation request to {origin}"); + return Err!(Request(Forbidden("Federation with this homeserver is not allowed."))); + } + let signatures = BTreeMap::from_iter([(x_matrix.key.clone(), CanonicalJsonValue::String(x_matrix.sig.to_string()))]); let signatures = BTreeMap::from_iter([( diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 8fd467f6..7feec613 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,7 +1,8 @@ use std::{fmt::Debug, mem}; use conduit::{ - debug, debug_error, debug_warn, err, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error, Result, + debug, debug_error, debug_info, debug_warn, err, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error, + Result, }; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; @@ -31,6 +32,16 @@ impl super::Service { return Err!(Config("allow_federation", "Federation is disabled.")); } + if self + .server + .config + .forbidden_remote_server_names + .contains(&dest.to_owned()) + { + debug_info!("Refusing to send outbound federation request to {dest}"); + return Err!(Request(Forbidden("Federation with this homeserver is not allowed."))); + } + let actual = self.services.resolver.get_actual_dest(dest).await?; let request = self.prepare::(dest, &actual, req).await?; self.execute::(dest, &actual, request, client).await From 83ef4eecc73c5902234fd6fe24ebda87a5e09bdf Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 11 Aug 2024 11:08:26 -0400 Subject: [PATCH 0653/2091] reduce unnecessary logging on URL preview and event, use sensible error code for URL previews Signed-off-by: strawberry --- src/api/client/media.rs | 35 +++++++++-------------------------- src/api/client/room.rs | 7 ++----- src/api/client/session.rs | 8 ++++---- 3 files changed, 15 insertions(+), 35 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index bce9b2b5..712c3fe2 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -5,7 +5,7 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - debug_warn, err, error, + debug_info, debug_warn, err, info, utils::{ self, content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename}, @@ -13,12 +13,8 @@ use conduit::{ }, warn, Err, Error, Result, }; -use ruma::api::client::{ - error::{ErrorKind, RetryAfter}, - media::{ - create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, - get_media_preview, - }, +use ruma::api::client::media::{ + create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, get_media_preview, }; use service::{ media::{FileMeta, MXC_LENGTH}, @@ -70,35 +66,22 @@ pub(crate) async fn get_media_preview_route( let url = &body.url; if !services.media.url_preview_allowed(url) { - return Err!(Request(Forbidden( - warn!(%sender_user, %url, "URL is not allowed to be previewed") - ))); + debug_info!(%sender_user, %url, "URL is not allowed to be previewed"); + return Err!(Request(Forbidden("URL is not allowed to be previewed"))); } match services.media.get_url_preview(url).await { Ok(preview) => { let res = serde_json::value::to_raw_value(&preview).map_err(|e| { - error!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}"); - Error::BadRequest( - ErrorKind::LimitExceeded { - retry_after: Some(RetryAfter::Delay(Duration::from_secs(5))), - }, - "Failed to generate a URL preview, try again later.", - ) + warn!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}"); + err!(Request(Unknown("Failed to generate a URL preview"))) })?; Ok(get_media_preview::v3::Response::from_raw_value(res)) }, Err(e) => { - warn!(%sender_user, "Failed to generate a URL preview: {e}"); - // there doesn't seem to be an agreed-upon error code in the spec. - // the only response codes in the preview_url spec page are 200 and 429. - Err(Error::BadRequest( - ErrorKind::LimitExceeded { - retry_after: Some(RetryAfter::Delay(Duration::from_secs(5))), - }, - "Failed to generate a URL preview, try again later.", - )) + info!(%sender_user, "Failed to generate a URL preview: {e}"); + Err!(Request(Unknown("Failed to generate a URL preview"))) }, } } diff --git a/src/api/client/room.rs b/src/api/client/room.rs index c78ba6ed..e894b878 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -1,7 +1,7 @@ use std::{cmp::max, collections::BTreeMap}; use axum::extract::State; -use conduit::{debug_info, debug_warn}; +use conduit::{debug_info, debug_warn, err}; use ruma::{ api::client::{ error::ErrorKind, @@ -475,10 +475,7 @@ pub(crate) async fn get_room_event_route( .rooms .timeline .get_pdu(&body.event_id)? - .ok_or_else(|| { - warn!("Event not found, event ID: {:?}", &body.event_id); - Error::BadRequest(ErrorKind::NotFound, "Event not found.") - })?; + .ok_or_else(|| err!(Request(NotFound("Event {} not found.", &body.event_id))))?; if !services .rooms diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 7c2a9718..4702b0ec 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -34,7 +34,7 @@ struct Claims { /// /// Get the supported login types of this server. One of these should be used as /// the `type` field when logging in. -#[tracing::instrument(skip_all, fields(%client), name = "register")] +#[tracing::instrument(skip_all, fields(%client), name = "login")] pub(crate) async fn get_login_types_route( InsecureClientIp(client): InsecureClientIp, _body: Ruma, ) -> Result { @@ -58,7 +58,7 @@ pub(crate) async fn get_login_types_route( /// Note: You can use [`GET /// /_matrix/client/r0/login`](fn.get_supported_versions_route.html) to see /// supported login types. -#[tracing::instrument(skip_all, fields(%client), name = "register")] +#[tracing::instrument(skip_all, fields(%client), name = "login")] pub(crate) async fn login_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { @@ -221,7 +221,7 @@ pub(crate) async fn login_route( /// last seen ts) /// - Forgets to-device events /// - Triggers device list updates -#[tracing::instrument(skip_all, fields(%client), name = "register")] +#[tracing::instrument(skip_all, fields(%client), name = "logout")] pub(crate) async fn logout_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { @@ -249,7 +249,7 @@ pub(crate) async fn logout_route( /// Note: This is equivalent to calling [`GET /// /_matrix/client/r0/logout`](fn.logout_route.html) from each device of this /// user. -#[tracing::instrument(skip_all, fields(%client), name = "register")] +#[tracing::instrument(skip_all, fields(%client), name = "logout")] pub(crate) async fn logout_all_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, From 7b8ef3b070a448399ec33109d85dd6e0f050439f Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 11 Aug 2024 12:05:02 -0400 Subject: [PATCH 0654/2091] dont apply new_user_displayname_suffix to appservice user registrations Signed-off-by: strawberry --- src/api/client/account.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 10e63d71..bf71bc0d 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -240,7 +240,7 @@ pub(crate) async fn register_route( // If `new_user_displayname_suffix` is set, registration will push whatever // content is set to the user's display name with a space before it - if !services.globals.new_user_displayname_suffix().is_empty() { + if !services.globals.new_user_displayname_suffix().is_empty() && body.appservice_info.is_none() { write!(displayname, " {}", services.globals.config.new_user_displayname_suffix) .expect("should be able to write to string buffer"); } From f86ab0cd03764e8c1d1ca14c8cda839748be5577 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 13 Aug 2024 19:23:32 -0400 Subject: [PATCH 0655/2091] bump rust toolchain to hotfix 1.80.1 Signed-off-by: strawberry --- Cargo.toml | 2 +- flake.nix | 2 +- rust-toolchain.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4f808d28..2f2a6793 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" -rust-version = "1.80.0" +rust-version = "1.80.1" version = "0.4.6" [workspace.metadata.crane] diff --git a/flake.nix b/flake.nix index 84ec86a2..9754b532 100644 --- a/flake.nix +++ b/flake.nix @@ -24,7 +24,7 @@ file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` - sha256 = "sha256-6eN/GKzjVSjEhGO9FhWObkRFaE1Jf+uqMSdQnb8lcB4="; + sha256 = "sha256-3jVIIf5XPnUU1CRaTyAiO0XHVbJl12MSx3eucTXCjtE="; }; mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 098a99e6..a21340c2 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -11,7 +11,7 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.80.0" +channel = "1.80.1" components = [ # For rust-analyzer "rust-src", From a2afdbe1fa88c395bfdc80b321f661cf090d1dd9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 13 Aug 2024 20:51:27 -0400 Subject: [PATCH 0656/2091] bump various cargo deps Signed-off-by: strawberry --- Cargo.lock | 495 ++++++++++++++++++++++++++++++++++++----------------- Cargo.toml | 26 ++- 2 files changed, 346 insertions(+), 175 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac188fe3..94c8411d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,7 +124,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -135,7 +135,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -159,6 +159,33 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "aws-lc-rs" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" +dependencies = [ + "aws-lc-sys", + "mirai-annotations", + "paste", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", + "libc", + "paste", +] + [[package]] name = "axum" version = "0.7.5" @@ -187,7 +214,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -242,7 +269,7 @@ dependencies = [ "mime", "pin-project-lite", "serde", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -251,7 +278,8 @@ dependencies = [ [[package]] name = "axum-server" version = "0.6.0" -source = "git+https://github.com/girlbossceo/axum-server?rev=8e3368d899079818934e61cc9c839abcbbcada8a#8e3368d899079818934e61cc9c839abcbbcada8a" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1ad46c3ec4e12f4a4b6835e173ba21c25e484c9d02b49770bf006ce5367c036" dependencies = [ "arc-swap", "bytes", @@ -266,7 +294,31 @@ dependencies = [ "rustls-pemfile", "tokio", "tokio-rustls 0.24.1", - "tower", + "tower 0.4.13", + "tower-service", +] + +[[package]] +name = "axum-server" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56bac90848f6a9393ac03c63c640925c4b7c8ca21654de40d53f55964667c7d8" +dependencies = [ + "arc-swap", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "rustls 0.23.12", + "rustls-pemfile", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.0", + "tower 0.4.13", "tower-service", ] @@ -276,7 +328,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ea4cd08ae2a5f075d28fa31190163c8106a1d2d3189442494bae22b39040a0d" dependencies = [ - "axum-server", + "axum-server 0.6.0", "bytes", "http", "http-body-util", @@ -333,12 +385,15 @@ dependencies = [ "itertools 0.12.1", "lazy_static", "lazycell", + "log", + "prettyplease", "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", - "syn 2.0.72", + "syn 2.0.74", + "which", ] [[package]] @@ -418,9 +473,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.7.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca2be1d5c43812bae364ee3f30b3afcb7877cf59f4aeb94c66f313a41d2fac9" +checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" [[package]] name = "bzip2-sys" @@ -445,9 +500,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.7" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" +checksum = "e9e8aabfac534be767c909e0690571677d49f41bd8465ae876fe043d52ba5292" dependencies = [ "jobserver", "libc", @@ -505,9 +560,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.13" +version = "4.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fbb260a053428790f3de475e304ff84cdbc4face759ea7a3e64c1edd938a7fc" +checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc" dependencies = [ "clap_builder", "clap_derive", @@ -515,9 +570,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.13" +version = "4.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64b17d7ea74e9f833c7dbf2cbe4fb12ff26783eda4782a8975b72f895c9b4d99" +checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" dependencies = [ "anstyle", "clap_lex", @@ -532,7 +587,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -541,6 +596,15 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +[[package]] +name = "cmake" +version = "0.1.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +dependencies = [ + "cc", +] + [[package]] name = "color_quant" version = "1.1.0" @@ -695,7 +759,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -704,7 +768,7 @@ version = "0.4.6" dependencies = [ "axum", "axum-client-ip", - "axum-server", + "axum-server 0.7.1", "axum-server-dual-protocol", "bytes", "conduit_admin", @@ -724,7 +788,7 @@ dependencies = [ "sentry-tracing", "serde_json", "tokio", - "tower", + "tower 0.5.0", "tower-http", "tracing", ] @@ -825,9 +889,9 @@ checksum = "6051f239ecec86fde3410901ab7860d458d160371533842974fc61f96d15879b" [[package]] name = "coolor" -version = "0.9.0" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37e93977247fb916abeee1ff8c6594c9b421fd9c26c9b720a3944acb2a7de27b" +checksum = "691defa50318376447a73ced869862baecfab35f6aabaa91a4cd726b315bfe1a" dependencies = [ "crossterm", ] @@ -844,15 +908,15 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" dependencies = [ "libc", ] @@ -868,9 +932,9 @@ dependencies = [ [[package]] name = "crokey" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b015414137feca6f4137550d60d233b580a09307e2de329412b8e8c661b6dfab" +checksum = "520e83558f4c008ac06fa6a86e5c1d4357be6f994cce7434463ebcdaadf47bb1" dependencies = [ "crokey-proc_macros", "crossterm", @@ -881,9 +945,9 @@ dependencies = [ [[package]] name = "crokey-proc_macros" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6085c6385034f0847d2b3120547a88da1e5e35727970a70f32b555eb3c47051c" +checksum = "370956e708a1ce65fe4ac5bb7185791e0ece7485087f17736d54a23a0895049f" dependencies = [ "crossterm", "proc-macro2", @@ -950,16 +1014,16 @@ checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crossterm" -version = "0.27.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ "bitflags 2.6.0", "crossterm_winapi", "futures-core", - "libc", - "mio 0.8.11", + "mio", "parking_lot", + "rustix", "signal-hook", "signal-hook-mio", "winapi", @@ -991,7 +1055,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1018,7 +1082,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1079,6 +1143,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "ed25519" version = "2.2.3" @@ -1122,7 +1192,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1131,6 +1201,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "fdeflate" version = "0.3.4" @@ -1174,9 +1254,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" dependencies = [ "crc32fast", "miniz_oxide", @@ -1207,6 +1287,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futf" version = "0.1.5" @@ -1258,7 +1344,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1347,7 +1433,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.3.0", + "indexmap 2.4.0", "slab", "tokio", "tokio-util", @@ -1487,6 +1573,15 @@ dependencies = [ "digest", ] +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "hostname" version = "0.3.1" @@ -1520,7 +1615,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1639,9 +1734,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" dependencies = [ "bytes", "futures-channel", @@ -1652,7 +1747,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower", + "tower 0.4.13", "tower-service", "tracing", ] @@ -1716,9 +1811,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0" +checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -1804,9 +1899,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -1884,7 +1979,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -1917,9 +2012,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.18" +version = "1.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +checksum = "fdc53a7799a7496ebc9fd29f31f7df80e83c9bda5299768af5f9e59eeea74647" dependencies = [ "cc", "pkg-config", @@ -1932,6 +2027,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "lock_api" version = "0.4.12" @@ -2071,28 +2172,23 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" -dependencies = [ - "libc", - "log", - "wasi", - "windows-sys 0.48.0", -] - -[[package]] -name = "mio" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ "hermit-abi", "libc", + "log", "wasi", "windows-sys 0.52.0", ] +[[package]] +name = "mirai-annotations" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2228,9 +2324,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.2" +version = "0.36.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" +checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" dependencies = [ "memchr", ] @@ -2255,7 +2351,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.3.0", + "indexmap 2.4.0", "js-sys", "once_cell", "pin-project-lite", @@ -2405,7 +2501,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -2498,7 +2594,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -2550,9 +2646,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.18" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dee4364d9f3b902ef14fab8a1ddffb783a1cb6b4bba3bfc1fa3922732c7de97f" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ "zerocopy", ] @@ -2563,6 +2659,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "prettyplease" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +dependencies = [ + "proc-macro2", + "syn 2.0.74", +] + [[package]] name = "proc-macro-crate" version = "3.1.0" @@ -2589,7 +2695,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", "version_check", "yansi", ] @@ -2614,7 +2720,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -2658,16 +2764,17 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quinn" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.0.0", "rustls 0.23.12", + "socket2", "thiserror", "tokio", "tracing", @@ -2675,14 +2782,14 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" dependencies = [ "bytes", "rand", "ring", - "rustc-hash", + "rustc-hash 2.0.0", "rustls 0.23.12", "slab", "thiserror", @@ -2699,6 +2806,7 @@ dependencies = [ "libc", "once_cell", "socket2", + "tracing", "windows-sys 0.52.0", ] @@ -2752,9 +2860,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", @@ -2871,7 +2979,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "assign", "js_int", @@ -2893,7 +3001,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "js_int", "ruma-common", @@ -2905,7 +3013,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "as_variant", "assign", @@ -2928,14 +3036,14 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "as_variant", "base64 0.22.1", "bytes", "form_urlencoded", "http", - "indexmap 2.3.0", + "indexmap 2.4.0", "js_int", "konst", "percent-encoding", @@ -2958,10 +3066,10 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "as_variant", - "indexmap 2.3.0", + "indexmap 2.4.0", "js_int", "js_option", "percent-encoding", @@ -2975,15 +3083,22 @@ dependencies = [ "thiserror", "tracing", "url", + "web-time 1.1.0", "wildmatch", ] [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ + "bytes", + "http", + "httparse", "js_int", + "memchr", + "mime", + "rand", "ruma-common", "ruma-events", "serde", @@ -2993,7 +3108,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "js_int", "thiserror", @@ -3002,7 +3117,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "js_int", "ruma-common", @@ -3012,7 +3127,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "once_cell", "proc-macro-crate", @@ -3020,14 +3135,14 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.72", + "syn 2.0.74", "toml", ] [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "js_int", "ruma-common", @@ -3039,7 +3154,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "headers", "http", @@ -3052,7 +3167,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3068,7 +3183,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c76e2873c1593a3308d4ba3e0e4a1db65acf8536#c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" dependencies = [ "itertools 0.12.1", "js_int", @@ -3126,6 +3241,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.4.0" @@ -3135,6 +3256,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustls" version = "0.21.12" @@ -3167,6 +3301,7 @@ version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ + "aws-lc-rs", "log", "once_cell", "ring", @@ -3191,9 +3326,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.2" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ "base64 0.22.1", "rustls-pki-types", @@ -3201,9 +3336,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" @@ -3221,6 +3356,7 @@ version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -3234,8 +3370,8 @@ checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "rustyline-async" -version = "0.4.2" -source = "git+https://github.com/girlbossceo/rustyline-async?rev=de26100b0db03e419a3d8e1dd26895d170d1fe50#de26100b0db03e419a3d8e1dd26895d170d1fe50" +version = "0.4.3" +source = "git+https://github.com/girlbossceo/rustyline-async?rev=9654cc84e19241f6e19021eb8e677892656f5071#9654cc84e19241f6e19021eb8e677892656f5071" dependencies = [ "crossterm", "futures-channel", @@ -3460,22 +3596,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.204" +version = "1.0.207" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "5665e14a49a4ea1b91029ba7d3bca9f299e1f7cfa194388ccc20f14743e784f2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.207" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "6aea2634c86b0e8ef2cfdc0c340baede54ec27b1e46febd7f80dffb2aa44a00e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -3485,7 +3621,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" dependencies = [ "form_urlencoded", - "indexmap 2.3.0", + "indexmap 2.4.0", "itoa", "ryu", "serde", @@ -3493,9 +3629,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.121" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" +checksum = "66ad62847a56b3dba58cc891acd13884b9c61138d330c0d7b6181713d4fce38d" dependencies = [ "itoa", "memchr", @@ -3550,7 +3686,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.3.0", + "indexmap 2.4.0", "itoa", "ryu", "serde", @@ -3622,7 +3758,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34db1a06d485c9142248b7a054f034b349b212551f3dfd19c94d45a754a217cd" dependencies = [ "libc", - "mio 0.8.11", + "mio", "signal-hook", ] @@ -3769,9 +3905,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" dependencies = [ "proc-macro2", "quote", @@ -3803,9 +3939,9 @@ dependencies = [ [[package]] name = "termimad" -version = "0.29.4" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff854b510d076163c3b8628ebfe6bfbe6f01c3254b8b8e77cac7ad2d8c72b06" +checksum = "920e7c4671e79f3d9df269da9c8edf0dbc580044fd727d3594f7bfba5eb6107a" dependencies = [ "coolor", "crokey", @@ -3844,7 +3980,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -3965,7 +4101,7 @@ dependencies = [ "backtrace", "bytes", "libc", - "mio 1.0.1", + "mio", "pin-project-lite", "signal-hook-registry", "socket2", @@ -3982,7 +4118,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -4081,7 +4217,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.3.0", + "indexmap 2.4.0", "toml_datetime", "winnow 0.5.40", ] @@ -4092,7 +4228,7 @@ version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.3.0", + "indexmap 2.4.0", "serde", "serde_spanned", "toml_datetime", @@ -4123,7 +4259,7 @@ dependencies = [ "socket2", "tokio", "tokio-stream", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -4149,6 +4285,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36b837f86b25d7c0d7988f00a54e74739be6477f2aac6201b8f429a7569991b7" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-http" version = "0.5.2" @@ -4166,7 +4316,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower", + "tower 0.4.13", "tower-layer", "tower-service", "tracing", @@ -4174,15 +4324,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -4202,7 +4352,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b47 dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -4371,9 +4521,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72139d247e5f97a3eff96229a7ae85ead5328a39efe76f8bf5a06313d505b6ea" +checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" dependencies = [ "base64 0.22.1", "log", @@ -4453,34 +4603,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -4490,9 +4641,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4500,28 +4651,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -4574,6 +4725,18 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "widestring" version = "1.1.0" @@ -4823,9 +4986,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zerocopy" -version = "0.6.6" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854e949ac82d619ee9a14c66a1b674ac730422372ccb759ce0c39cabcf2bf8e6" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", "zerocopy-derive", @@ -4833,13 +4996,13 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.6.6" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "125139de3f6b9d625c39e2efdd73d41bdac468ccd556556440e322be0e1bbd91" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.74", ] [[package]] @@ -4847,6 +5010,20 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.74", +] [[package]] name = "zstd" @@ -4859,18 +5036,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.0" +version = "7.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" +checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 2f2a6793..fee6e22e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ version = "0.8.5" # Used for the http request / response body type for Ruma endpoints used with reqwest [workspace.dependencies.bytes] -version = "1.6.1" +version = "1.7.1" [workspace.dependencies.http-body-util] version = "0.1.1" @@ -78,7 +78,7 @@ version = "0.1.1" version = "1.1.0" [workspace.dependencies.regex] -version = "1.10.4" +version = "1.10.6" [workspace.dependencies.axum] version = "0.7.5" @@ -99,7 +99,7 @@ default-features = false features = ["typed-header", "tracing"] [workspace.dependencies.axum-server] -version = "0.6.0" +version = "0.7.1" default-features = false features = ["tls-rustls"] @@ -107,7 +107,7 @@ features = ["tls-rustls"] version = "0.6.0" [workspace.dependencies.tower] -version = "0.4.13" +version = "0.5.0" default-features = false features = ["util"] @@ -140,7 +140,7 @@ default-features = false features = ["rc"] [workspace.dependencies.serde_json] -version = "1.0.121" +version = "1.0.124" default-features = false features = ["raw_value"] @@ -193,7 +193,7 @@ default-features = false # used for conduit's CLI and admin room command parsing [workspace.dependencies.clap] -version = "4.5.9" +version = "4.5.15" default-features = false features = [ "std", @@ -307,7 +307,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "c76e2873c1593a3308d4ba3e0e4a1db65acf8536" +rev = "11cbd52af1039f3d4a0b96333acd1d1687ba01aa" features = [ "compat", "rand", @@ -431,11 +431,11 @@ features = [ ] [workspace.dependencies.rustyline-async] -version = "0.4.2" +version = "0.4.3" default-features = false [workspace.dependencies.termimad] -version = "0.29.4" +version = "0.30.0" default-features = false [workspace.dependencies.checked_ops] @@ -473,17 +473,11 @@ rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" git = "https://github.com/girlbossceo/tracing" rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" -# fixes hyper graceful shutdowns [https://github.com/programatik29/axum-server/issues/114] -# https://github.com/girlbossceo/axum-server/commit/8e3368d899079818934e61cc9c839abcbbcada8a -[patch.crates-io.axum-server] -git = "https://github.com/girlbossceo/axum-server" -rev = "8e3368d899079818934e61cc9c839abcbbcada8a" - # adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50 # adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b [patch.crates-io.rustyline-async] git = "https://github.com/girlbossceo/rustyline-async" -rev = "de26100b0db03e419a3d8e1dd26895d170d1fe50" +rev = "9654cc84e19241f6e19021eb8e677892656f5071" # # Our crates From 66679c7522fc95e6b5e0e2b48399f80e1a4d8100 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 13 Aug 2024 20:54:28 -0400 Subject: [PATCH 0657/2091] nix: bump flake.lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'attic': 'github:zhaofengli/attic/6139576a3ce6bb992e0f6c3022528ec233e45f00' (2024-07-09) → 'github:zhaofengli/attic/26b9417bde03edc6280d1f7ce709cd619cdb72d4' (2024-08-13) • Updated input 'cachix': 'github:cachix/cachix/4e9e71f78b9500fa6210cf1eaa4d75bdbab777c3' (2024-07-02) → 'github:cachix/cachix/bd558fd32f55d6df1c3d496bade90335a1a6044d' (2024-08-10) • Updated input 'cachix/devenv': 'github:cachix/devenv/bb32aa986f2f695385e54428d0eaf7d05b31466e' (2024-06-30) → 'github:cachix/devenv/ff5eb4f2accbcda963af67f1a1159e3f6c7f5f91' (2024-08-08) • Updated input 'cachix/devenv/pre-commit-hooks': follows 'cachix/pre-commit-hooks' → follows 'cachix/git-hooks' • Added input 'cachix/git-hooks': 'github:cachix/git-hooks.nix/c7012d0c18567c889b948781bc74a501e92275d1' (2024-08-09) • Added input 'cachix/git-hooks/flake-compat': follows 'cachix/flake-compat' • Added input 'cachix/git-hooks/gitignore': 'github:hercules-ci/gitignore.nix/637db329424fd7e46cf4185293b9cc8c88c95394' (2024-02-28) • Added input 'cachix/git-hooks/gitignore/nixpkgs': follows 'cachix/git-hooks/nixpkgs' • Added input 'cachix/git-hooks/nixpkgs': follows 'cachix/nixpkgs' • Added input 'cachix/git-hooks/nixpkgs-stable': 'github:NixOS/nixpkgs/194846768975b7ad2c4988bdb82572c00222c0d7' (2024-07-07) • Updated input 'cachix/nixpkgs': 'github:NixOS/nixpkgs/00d80d13810dbfea8ab4ed1009b09100cca86ba8' (2024-07-01) → 'github:NixOS/nixpkgs/cb9a96f23c491c081b38eab96d22fa958043c9fa' (2024-08-04) • Removed input 'cachix/pre-commit-hooks' • Removed input 'cachix/pre-commit-hooks/flake-compat' • Removed input 'cachix/pre-commit-hooks/gitignore' • Removed input 'cachix/pre-commit-hooks/gitignore/nixpkgs' • Removed input 'cachix/pre-commit-hooks/nixpkgs' • Removed input 'cachix/pre-commit-hooks/nixpkgs-stable' • Updated input 'crane': 'github:ipetkov/crane/2d83156f23c43598cf44e152c33a59d3892f8b29' (2024-07-09) → 'github:ipetkov/crane/4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4' (2024-08-06) • Updated input 'fenix': 'github:nix-community/fenix/5087b12a595ee73131a944d922f24d81dae05725' (2024-07-13) → 'github:nix-community/fenix/296d44c440302980824c5f3b67e477cf0522e0c1' (2024-08-13) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/ffbc5ad993d5cd2f3b8bcf9a511165470944ab91' (2024-07-11) → 'github:rust-lang/rust-analyzer/32a86cb1dad2b208e8f36f1bb50c2e4806b0371f' (2024-08-12) • Updated input 'liburing': 'github:axboe/liburing/1d674f83b7d0f07553ac44d99a401b05853d9dbe' (2024-07-12) → 'github:axboe/liburing/401b3e4bde2316d35e93b548269c72bbb64adc02' (2024-08-13) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/7e7c39ea35c5cdd002cd4588b03a3fb9ece6fad9' (2024-07-12) → 'github:NixOS/nixpkgs/a58bc8ad779655e790115244571758e8de055e3d' (2024-08-11) Signed-off-by: strawberry --- flake.lock | 139 ++++++++++++++++++++++++----------------------------- 1 file changed, 63 insertions(+), 76 deletions(-) diff --git a/flake.lock b/flake.lock index 0688e378..66aae3d4 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1720542474, - "narHash": "sha256-aKjJ/4l2I9+wNGTaOGRsuS3M1+IoTibqgEMPDikXm04=", + "lastModified": 1723558887, + "narHash": "sha256-FBaRmikNnCX+HQJXAfNJyiOl3sM1NrlxvcFwQDqwCyk=", "owner": "zhaofengli", "repo": "attic", - "rev": "6139576a3ce6bb992e0f6c3022528ec233e45f00", + "rev": "26b9417bde03edc6280d1f7ce709cd619cdb72d4", "type": "github" }, "original": { @@ -27,15 +27,15 @@ "inputs": { "devenv": "devenv", "flake-compat": "flake-compat_3", - "nixpkgs": "nixpkgs_3", - "pre-commit-hooks": "pre-commit-hooks" + "git-hooks": "git-hooks", + "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1719923519, - "narHash": "sha256-7Rhljj2fsklFRsu+eq7N683Z9qukmreMEj5C1GqCrSA=", + "lastModified": 1723252655, + "narHash": "sha256-uOxcrNEMTOHvGHYyqDWnUpkhkY29l01KhjkgVQU2gXg=", "owner": "cachix", "repo": "cachix", - "rev": "4e9e71f78b9500fa6210cf1eaa4d75bdbab777c3", + "rev": "bd558fd32f55d6df1c3d496bade90335a1a6044d", "type": "github" }, "original": { @@ -123,11 +123,11 @@ ] }, "locked": { - "lastModified": 1720546058, - "narHash": "sha256-iU2yVaPIZm5vMGdlT0+57vdB/aPq/V5oZFBRwYw+HBM=", + "lastModified": 1722960479, + "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=", "owner": "ipetkov", "repo": "crane", - "rev": "2d83156f23c43598cf44e152c33a59d3892f8b29", + "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4", "type": "github" }, "original": { @@ -151,15 +151,15 @@ ], "pre-commit-hooks": [ "cachix", - "pre-commit-hooks" + "git-hooks" ] }, "locked": { - "lastModified": 1719759336, - "narHash": "sha256-3a34VL/QnHprl5gMy9xlx6d8J+iNp+W88Ex8smkgH9M=", + "lastModified": 1723156315, + "narHash": "sha256-0JrfahRMJ37Rf1i0iOOn+8Z4CLvbcGNwa2ChOAVrp/8=", "owner": "cachix", "repo": "devenv", - "rev": "bb32aa986f2f695385e54428d0eaf7d05b31466e", + "rev": "ff5eb4f2accbcda963af67f1a1159e3f6c7f5f91", "type": "github" }, "original": { @@ -209,11 +209,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1720852044, - "narHash": "sha256-3NBYz8VuXuKU+8ONd9NFafCNjPEGHIZQ2Mdoam1a4mY=", + "lastModified": 1723530607, + "narHash": "sha256-FaXZZLLDW1D+pj7UgrIslDS8XjMMG3Pus5gAvUYWQS0=", "owner": "nix-community", "repo": "fenix", - "rev": "5087b12a595ee73131a944d922f24d81dae05725", + "rev": "296d44c440302980824c5f3b67e477cf0522e0c1", "type": "github" }, "original": { @@ -272,22 +272,6 @@ } }, "flake-compat_4": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-compat_5": { "flake": false, "locked": { "lastModified": 1696426674, @@ -356,11 +340,38 @@ "type": "github" } }, + "git-hooks": { + "inputs": { + "flake-compat": [ + "cachix", + "flake-compat" + ], + "gitignore": "gitignore", + "nixpkgs": [ + "cachix", + "nixpkgs" + ], + "nixpkgs-stable": "nixpkgs-stable_2" + }, + "locked": { + "lastModified": 1723202784, + "narHash": "sha256-qbhjc/NEGaDbyy0ucycubq4N3//gDFFH3DOmp1D3u1Q=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "c7012d0c18567c889b948781bc74a501e92275d1", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "gitignore": { "inputs": { "nixpkgs": [ "cachix", - "pre-commit-hooks", + "git-hooks", "nixpkgs" ] }, @@ -381,11 +392,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1720798442, - "narHash": "sha256-gtPppAoksMLW4GuruQ36nf4EAqIA1Bs6V9Xcx8dBxrQ=", + "lastModified": 1723553052, + "narHash": "sha256-sjqmYP/tvQKmsTD6FOd8m7aEja2hhEyyB3aTxVXEbuE=", "owner": "axboe", "repo": "liburing", - "rev": "1d674f83b7d0f07553ac44d99a401b05853d9dbe", + "rev": "401b3e4bde2316d35e93b548269c72bbb64adc02", "type": "github" }, "original": { @@ -558,16 +569,16 @@ }, "nixpkgs-stable_2": { "locked": { - "lastModified": 1718811006, - "narHash": "sha256-0Y8IrGhRmBmT7HHXlxxepg2t8j1X90++qRN3lukGaIk=", + "lastModified": 1720386169, + "narHash": "sha256-NGKVY4PjzwAa4upkGtAMz1npHGoRzWotlSnVlqI40mo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "03d771e513ce90147b65fe922d87d3a0356fc125", + "rev": "194846768975b7ad2c4988bdb82572c00222c0d7", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } @@ -590,11 +601,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1719848872, - "narHash": "sha256-H3+EC5cYuq+gQW8y0lSrrDZfH71LB4DAf+TDFyvwCNA=", + "lastModified": 1722813957, + "narHash": "sha256-IAoYyYnED7P8zrBFMnmp7ydaJfwTnwcnqxUElC1I26Y=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "00d80d13810dbfea8ab4ed1009b09100cca86ba8", + "rev": "cb9a96f23c491c081b38eab96d22fa958043c9fa", "type": "github" }, "original": { @@ -606,11 +617,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1720768451, - "narHash": "sha256-EYekUHJE2gxeo2pM/zM9Wlqw1Uw2XTJXOSAO79ksc4Y=", + "lastModified": 1723362943, + "narHash": "sha256-dFZRVSgmJkyM0bkPpaYRtG/kRMRTorUIDj8BxoOt1T4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "7e7c39ea35c5cdd002cd4588b03a3fb9ece6fad9", + "rev": "a58bc8ad779655e790115244571758e8de055e3d", "type": "github" }, "original": { @@ -646,30 +657,6 @@ "type": "github" } }, - "pre-commit-hooks": { - "inputs": { - "flake-compat": "flake-compat_4", - "gitignore": "gitignore", - "nixpkgs": [ - "cachix", - "nixpkgs" - ], - "nixpkgs-stable": "nixpkgs-stable_2" - }, - "locked": { - "lastModified": 1719259945, - "narHash": "sha256-F1h+XIsGKT9TkGO3omxDLEb/9jOOsI6NnzsXFsZhry4=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "0ff4381bbb8f7a52ca4a851660fc7a437a4c6e07", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "type": "github" - } - }, "rocksdb": { "flake": false, "locked": { @@ -694,7 +681,7 @@ "complement": "complement", "crane": "crane_2", "fenix": "fenix", - "flake-compat": "flake-compat_5", + "flake-compat": "flake-compat_4", "flake-utils": "flake-utils_3", "liburing": "liburing", "nix-filter": "nix-filter", @@ -705,11 +692,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1720717809, - "narHash": "sha256-6I+fm+nTLF/iaj7ffiFGlSY7POmubwUaPA/Wq0Bm53M=", + "lastModified": 1723473250, + "narHash": "sha256-Ls0e6R4FmGUFXZlUcm6ZQaVNJ4Yj/nua4SSctXIopao=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "ffbc5ad993d5cd2f3b8bcf9a511165470944ab91", + "rev": "32a86cb1dad2b208e8f36f1bb50c2e4806b0371f", "type": "github" }, "original": { From 2d9aab2e79ee3426a3b447f1f300dd6e70e8edb9 Mon Sep 17 00:00:00 2001 From: Matthias Ahouansou Date: Wed, 14 Aug 2024 20:01:34 -0400 Subject: [PATCH 0658/2091] support timestamped appservice messaging Co-authored-by: strawberry Signed-off-by: strawberry --- src/api/client/membership.rs | 7 +++++++ src/api/client/message.rs | 5 +++++ src/api/client/profile.rs | 2 ++ src/api/client/redact.rs | 1 + src/api/client/room.rs | 14 ++++++++++++++ src/api/client/state.rs | 8 +++++++- src/api/server/make_join.rs | 1 + src/api/server/make_leave.rs | 1 + src/core/pdu/builder.rs | 6 +++++- src/service/admin/create.rs | 10 ++++++++++ src/service/admin/grant.rs | 4 ++++ src/service/admin/mod.rs | 2 ++ src/service/rooms/state_accessor/mod.rs | 1 + src/service/rooms/timeline/mod.rs | 12 +++++++++--- 14 files changed, 69 insertions(+), 5 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 9eb40b4a..8a2ab8b4 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -403,6 +403,7 @@ pub(crate) async fn kick_user_route( unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, + timestamp: None, }, sender_user, &body.room_id, @@ -465,6 +466,7 @@ pub(crate) async fn ban_user_route( unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, + timestamp: None, }, sender_user, &body.room_id, @@ -512,6 +514,7 @@ pub(crate) async fn unban_user_route( unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, + timestamp: None, }, sender_user, &body.room_id, @@ -1100,6 +1103,7 @@ async fn join_room_by_id_helper_local( unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, + timestamp: None, }, sender_user, room_id, @@ -1412,6 +1416,7 @@ pub(crate) async fn invite_helper( unsigned: None, state_key: Some(user_id.to_string()), redacts: None, + timestamp: None, }, sender_user, room_id, @@ -1523,6 +1528,7 @@ pub(crate) async fn invite_helper( unsigned: None, state_key: Some(user_id.to_string()), redacts: None, + timestamp: None, }, sender_user, room_id, @@ -1640,6 +1646,7 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, unsigned: None, state_key: Some(user_id.to_string()), redacts: None, + timestamp: None, }, user_id, room_id, diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 9aae4aaf..51aee8c1 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -84,6 +84,11 @@ pub(crate) async fn send_message_event_route( unsigned: Some(unsigned), state_key: None, redacts: None, + timestamp: if body.appservice_info.is_some() { + body.timestamp + } else { + None + }, }, sender_user, &body.room_id, diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index a72938b5..71d49cd8 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -295,6 +295,7 @@ pub async fn update_displayname( unsigned: None, state_key: Some(user_id.to_string()), redacts: None, + timestamp: None, }, room_id, )) @@ -355,6 +356,7 @@ pub async fn update_avatar_url( unsigned: None, state_key: Some(user_id.to_string()), redacts: None, + timestamp: None, }, room_id, )) diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index 89446754..2102f6cd 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -34,6 +34,7 @@ pub(crate) async fn redact_event_route( unsigned: None, state_key: None, redacts: Some(body.event_id.into()), + timestamp: None, }, sender_user, &body.room_id, diff --git a/src/api/client/room.rs b/src/api/client/room.rs index e894b878..a18ae206 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -197,6 +197,7 @@ pub(crate) async fn create_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &room_id, @@ -225,6 +226,7 @@ pub(crate) async fn create_room_route( unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, + timestamp: None, }, sender_user, &room_id, @@ -262,6 +264,7 @@ pub(crate) async fn create_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &room_id, @@ -285,6 +288,7 @@ pub(crate) async fn create_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &room_id, @@ -311,6 +315,7 @@ pub(crate) async fn create_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &room_id, @@ -330,6 +335,7 @@ pub(crate) async fn create_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &room_id, @@ -352,6 +358,7 @@ pub(crate) async fn create_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &room_id, @@ -405,6 +412,7 @@ pub(crate) async fn create_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &room_id, @@ -427,6 +435,7 @@ pub(crate) async fn create_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &room_id, @@ -581,6 +590,7 @@ pub(crate) async fn upgrade_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &body.room_id, @@ -672,6 +682,7 @@ pub(crate) async fn upgrade_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &replacement_room, @@ -700,6 +711,7 @@ pub(crate) async fn upgrade_room_route( unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, + timestamp: None, }, sender_user, &replacement_room, @@ -728,6 +740,7 @@ pub(crate) async fn upgrade_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &replacement_room, @@ -786,6 +799,7 @@ pub(crate) async fn upgrade_room_route( unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, sender_user, &body.room_id, diff --git a/src/api/client/state.rs b/src/api/client/state.rs index d0fb83d1..fd049663 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -43,6 +43,11 @@ pub(crate) async fn send_state_event_for_key_route( &body.event_type, &body.body.body, body.state_key.clone(), + if body.appservice_info.is_some() { + body.timestamp + } else { + None + }, ) .await? .into(), @@ -172,7 +177,7 @@ pub(crate) async fn get_state_events_for_empty_key_route( async fn send_state_event_for_key_helper( services: &Services, sender: &UserId, room_id: &RoomId, event_type: &StateEventType, - json: &Raw, state_key: String, + json: &Raw, state_key: String, timestamp: Option, ) -> Result> { allowed_to_send_state_event(services, room_id, event_type, json).await?; let state_lock = services.rooms.state.mutex.lock(room_id).await; @@ -186,6 +191,7 @@ async fn send_state_event_for_key_helper( unsigned: None, state_key: Some(state_key), redacts: None, + timestamp, }, sender, room_id, diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index e9b1a6c7..86e821d1 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -139,6 +139,7 @@ pub(crate) async fn create_join_event_template_route( unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, + timestamp: None, }, &body.user_id, &body.room_id, diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index ae7237ad..3eb0d77a 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -57,6 +57,7 @@ pub(crate) async fn create_leave_event_template_route( unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, + timestamp: None, }, &body.user_id, &body.room_id, diff --git a/src/core/pdu/builder.rs b/src/core/pdu/builder.rs index a8bad677..ba4c19e5 100644 --- a/src/core/pdu/builder.rs +++ b/src/core/pdu/builder.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use ruma::{events::TimelineEventType, EventId}; +use ruma::{events::TimelineEventType, EventId, MilliSecondsSinceUnixEpoch}; use serde::Deserialize; use serde_json::value::RawValue as RawJsonValue; @@ -13,4 +13,8 @@ pub struct PduBuilder { pub unsigned: Option>, pub state_key: Option, pub redacts: Option>, + /// For timestamped messaging, should only be used for appservices + /// + /// Will be set to current time if None + pub timestamp: Option, } diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 18cbe039..5cc85d70 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -71,6 +71,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -99,6 +100,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(server_user.to_string()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -124,6 +126,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -143,6 +146,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -162,6 +166,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -181,6 +186,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -201,6 +207,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -221,6 +228,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -245,6 +253,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -271,6 +280,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index c35f8c42..44e11a3e 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -50,6 +50,7 @@ impl super::Service { unsigned: None, state_key: Some(user_id.to_string()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -75,6 +76,7 @@ impl super::Service { unsigned: None, state_key: Some(user_id.to_string()), redacts: None, + timestamp: None, }, user_id, &room_id, @@ -100,6 +102,7 @@ impl super::Service { unsigned: None, state_key: Some(String::new()), redacts: None, + timestamp: None, }, server_user, &room_id, @@ -119,6 +122,7 @@ impl super::Service { unsigned: None, state_key: None, redacts: None, + timestamp: None, }, server_user, &room_id, diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 0e8a65f1..962d31dd 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -260,6 +260,7 @@ impl Service { unsigned: None, state_key: None, redacts: None, + timestamp: None, }; if let Err(e) = self @@ -289,6 +290,7 @@ impl Service { unsigned: None, state_key: None, redacts: None, + timestamp: None, }; self.services diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 2526f1bd..58fa31b3 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -350,6 +350,7 @@ impl Service { unsigned: None, state_key: Some(target_user.into()), redacts: None, + timestamp: None, }; Ok(self diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index ac25b2a8..22c27498 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -638,6 +638,7 @@ impl Service { unsigned, state_key, redacts, + timestamp, } = pdu_builder; let prev_events: Vec<_> = self @@ -705,9 +706,14 @@ impl Service { room_id: room_id.to_owned(), sender: sender.to_owned(), origin: None, - origin_server_ts: utils::millis_since_unix_epoch() - .try_into() - .expect("time is valid"), + origin_server_ts: timestamp.map_or_else( + || { + utils::millis_since_unix_epoch() + .try_into() + .expect("u64 fits into UInt") + }, + |ts| ts.get(), + ), kind: event_type, content, state_key, From c40ed0c98eb8b8564e8e4b719e456d9f45e9f809 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 17 Aug 2024 02:22:35 -0400 Subject: [PATCH 0659/2091] add missing remove aliases on room upgrade transferring room aliases Signed-off-by: strawberry --- src/api/client/room.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/api/client/room.rs b/src/api/client/room.rs index a18ae206..bdd36dd5 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -756,6 +756,11 @@ pub(crate) async fn upgrade_room_route( .local_aliases_for_room(&body.room_id) .filter_map(Result::ok) { + services + .rooms + .alias + .remove_alias(&alias, sender_user) + .await?; services .rooms .alias From 826b1d1d6234bbab17b5a54aa94c41c4e9c2ac2a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 17 Aug 2024 02:28:16 -0400 Subject: [PATCH 0660/2091] remove unnecessary "unsupported room version" error on PDU processing to allow custom room versions properly Signed-off-by: strawberry --- src/api/client/membership.rs | 12 +----------- src/api/client/room.rs | 27 ++++----------------------- src/api/server/make_join.rs | 9 +-------- src/service/admin/create.rs | 12 ++---------- src/service/rooms/timeline/mod.rs | 9 +-------- 5 files changed, 9 insertions(+), 60 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 8a2ab8b4..d3d80387 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -819,7 +819,7 @@ async fn join_room_by_id_helper_remote( }, // only room versions 8 and above using `join_authorized_via_users_server` (restricted joins) need to // validate and send signatures - V8 | V9 | V10 | V11 => { + _ => { if let Some(signed_raw) = &send_join_response.room_state.event { info!( "There is a signed event. This room is probably using restricted joins. Adding signature to \ @@ -868,16 +868,6 @@ async fn join_room_by_id_helper_remote( } } }, - _ => { - warn!( - "Unexpected or unsupported room version {} for room {}", - &room_version_id, room_id - ); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, } } diff --git a/src/api/client/room.rs b/src/api/client/room.rs index bdd36dd5..96c43758 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -139,13 +139,8 @@ pub(crate) async fn create_room_route( })?, ); }, - V11 => {}, // V11 removed the "creator" key _ => { - warn!("Unexpected or unsupported room version {room_version}"); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); + // V11+ removed the "creator" key }, } content.insert( @@ -161,14 +156,7 @@ pub(crate) async fn create_room_route( let content = match room_version { V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => RoomCreateEventContent::new_v1(sender_user.clone()), - V11 => RoomCreateEventContent::new_v11(), - _ => { - warn!("Unexpected or unsupported room version {room_version}"); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, + _ => RoomCreateEventContent::new_v11(), }; let mut content = serde_json::from_str::( to_raw_value(&content) @@ -634,16 +622,9 @@ pub(crate) async fn upgrade_room_route( })?, ); }, - V11 => { - // "creator" key no longer exists in V11 rooms - create_event_content.remove("creator"); - }, _ => { - warn!("Unexpected or unsupported room version {}", body.new_version); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); + // "creator" key no longer exists in V11+ rooms + create_event_content.remove("creator"); }, } } diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index 86e821d1..021016be 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -222,15 +222,8 @@ pub(crate) fn maybe_strip_event_id(pdu_json: &mut CanonicalJsonObject, room_vers match room_version_id { V1 | V2 => {}, - V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 | V11 => { - pdu_json.remove("event_id"); - }, _ => { - warn!("Unexpected or unsupported room version {room_version_id}"); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); + pdu_json.remove("event_id"); }, }; diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 5cc85d70..ef84d803 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -1,8 +1,7 @@ use std::collections::BTreeMap; -use conduit::{pdu::PduBuilder, warn, Error, Result}; +use conduit::{pdu::PduBuilder, Result}; use ruma::{ - api::client::error::ErrorKind, events::{ room::{ canonical_alias::RoomCanonicalAliasEventContent, @@ -45,14 +44,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { use RoomVersionId::*; match room_version { V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => RoomCreateEventContent::new_v1(server_user.clone()), - V11 => RoomCreateEventContent::new_v11(), - _ => { - warn!("Unexpected or unsupported room version {}", room_version); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, + _ => RoomCreateEventContent::new_v11(), } }; diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 22c27498..983bef70 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -449,7 +449,7 @@ impl Service { } } }, - V11 => { + _ => { let content = serde_json::from_str::(pdu.content.get()).map_err(|e| { warn!("Invalid content in redaction pdu: {e}"); @@ -467,13 +467,6 @@ impl Service { } } }, - _ => { - warn!("Unexpected or unsupported room version {room_version_id}"); - return Err(Error::BadRequest( - ErrorKind::BadJson, - "Unexpected or unsupported room version found", - )); - }, }; }, TimelineEventType::SpaceChild => { From f1b41ce6e3b8b2b76c140b293db34fe53e4ad6cf Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 17 Aug 2024 13:41:18 -0400 Subject: [PATCH 0661/2091] remove unnecessary alias check on deleting room aliases Signed-off-by: strawberry --- src/api/client/alias.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 18d1c5b0..12d6352c 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -69,15 +69,6 @@ pub(crate) async fn delete_alias_route( .appservice_checks(&body.room_alias, &body.appservice_info) .await?; - if services - .rooms - .alias - .resolve_local_alias(&body.room_alias)? - .is_none() - { - return Err(Error::BadRequest(ErrorKind::NotFound, "Alias does not exist.")); - } - services .rooms .alias From bc5ecb0c72e936048c7f4b8d122bc46617b8f5c3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 18 Aug 2024 16:34:48 -0400 Subject: [PATCH 0662/2091] attempt to recreate server default push rules more if broken/empty content/etc Signed-off-by: strawberry --- src/api/client/push.rs | 86 ++++++++++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 24 deletions(-) diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 26462e79..8723e676 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -1,4 +1,5 @@ use axum::extract::State; +use conduit::err; use ruma::{ api::client::{ error::ErrorKind, @@ -7,9 +8,14 @@ use ruma::{ set_pusher, set_pushrule, set_pushrule_actions, set_pushrule_enabled, RuleScope, }, }, - events::{push_rules::PushRulesEvent, GlobalAccountDataEventType}, + events::{ + push_rules::{PushRulesEvent, PushRulesEventContent}, + GlobalAccountDataEventType, + }, push::{InsertPushRuleError, RemovePushRuleError, Ruleset}, + CanonicalJsonObject, }; +use service::Services; use crate::{Error, Result, Ruma}; @@ -21,36 +27,46 @@ pub(crate) async fn get_pushrules_all_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = + let global_ruleset: Ruleset; + + let Ok(event) = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())?; + .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + else { + // push rules event doesn't exist, create it and return default + return recreate_push_rules_and_return(&services, sender_user); + }; if let Some(event) = event { - let account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))? - .content; + let value = serde_json::from_str::(event.get()) + .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; - Ok(get_pushrules_all::v3::Response { - global: account_data.global, - }) + let Some(content_value) = value.get("content") else { + // user somehow has a push rule event with no content key, recreate it and + // return server default silently + return recreate_push_rules_and_return(&services, sender_user); + }; + + if content_value.to_string().is_empty() { + // user somehow has a push rule event with empty content, recreate it and return + // server default silently + return recreate_push_rules_and_return(&services, sender_user); + } + + let account_data_content = serde_json::from_value::(content_value.clone().into()) + .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; + + global_ruleset = account_data_content.global; } else { - services.account_data.update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(PushRulesEvent { - content: ruma::events::push_rules::PushRulesEventContent { - global: Ruleset::server_default(sender_user), - }, - }) - .expect("to json always works"), - )?; - - Ok(get_pushrules_all::v3::Response { - global: Ruleset::server_default(sender_user), - }) + // user somehow has non-existent push rule event. recreate it and return server + // default silently + return recreate_push_rules_and_return(&services, sender_user); } + + Ok(get_pushrules_all::v3::Response { + global: global_ruleset, + }) } /// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}` @@ -378,3 +394,25 @@ pub(crate) async fn set_pushers_route( Ok(set_pusher::v3::Response::default()) } + +/// user somehow has bad push rules, these must always exist per spec. +/// so recreate it and return server default silently +fn recreate_push_rules_and_return( + services: &Services, sender_user: &ruma::UserId, +) -> Result { + services.account_data.update( + None, + sender_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(PushRulesEvent { + content: PushRulesEventContent { + global: Ruleset::server_default(sender_user), + }, + }) + .expect("to json always works"), + )?; + + Ok(get_pushrules_all::v3::Response { + global: Ruleset::server_default(sender_user), + }) +} From 4432c06c86a236cde4311ad78315b9e81656b3c8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 18 Aug 2024 16:46:19 -0400 Subject: [PATCH 0663/2091] remove local room aliases and unpublish room when banning a room Signed-off-by: strawberry --- src/admin/room/moderation.rs | 38 ++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index ddde9146..a2bc76f7 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -208,6 +208,25 @@ async fn ban_room( } } + // remove any local aliases, ignore errors + for ref local_alias in self + .services + .rooms + .alias + .local_aliases_for_room(&room_id) + .filter_map(Result::ok) + { + _ = self + .services + .rooms + .alias + .remove_alias(local_alias, &self.services.globals.server_user) + .await; + } + + // unpublish from room directory, ignore errors + _ = self.services.rooms.directory.set_not_public(&room_id); + if disable_federation { self.services.rooms.metadata.disable_room(&room_id, true)?; return Ok(RoomMessageEventContent::text_plain( @@ -428,6 +447,25 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu } } + // remove any local aliases, ignore errors + for ref local_alias in self + .services + .rooms + .alias + .local_aliases_for_room(&room_id) + .filter_map(Result::ok) + { + _ = self + .services + .rooms + .alias + .remove_alias(local_alias, &self.services.globals.server_user) + .await; + } + + // unpublish from room directory, ignore errors + _ = self.services.rooms.directory.set_not_public(&room_id); + if disable_federation { self.services.rooms.metadata.disable_room(&room_id, true)?; } From eb6e509ad8a4148e054fd5b58f3eeba5e058cd86 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 4 Aug 2024 06:32:19 +0000 Subject: [PATCH 0664/2091] use where clause for long lines Signed-off-by: Jason Volk --- src/service/service.rs | 15 ++++++++++++--- src/service/services.rs | 10 ++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/service/service.rs b/src/service/service.rs index 0b9bc76c..635f782e 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -97,7 +97,10 @@ impl<'a> Args<'a> { /// Reference a Service by name. Panics if the Service does not exist or was /// incorrectly cast. -pub(crate) fn require<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(map: &'b Map, name: &'a str) -> Arc { +pub(crate) fn require<'a, 'b, T>(map: &'b Map, name: &'a str) -> Arc +where + T: Send + Sync + 'a + 'b + 'static, +{ try_get::(map, name) .inspect_err(inspect_log) .expect("Failure to reference service required by another service.") @@ -109,7 +112,10 @@ pub(crate) fn require<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(map: &'b Map, /// # Panics /// Incorrect type is not a silent failure (None) as the type never has a reason /// to be incorrect. -pub(crate) fn get<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(map: &'b Map, name: &'a str) -> Option> { +pub(crate) fn get<'a, 'b, T>(map: &'b Map, name: &'a str) -> Option> +where + T: Send + Sync + 'a + 'b + 'static, +{ map.read() .expect("locked for reading") .get(name) @@ -123,7 +129,10 @@ pub(crate) fn get<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(map: &'b Map, name /// Reference a Service by name. Returns Err if the Service does not exist or /// was incorrectly cast. -pub(crate) fn try_get<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(map: &'b Map, name: &'a str) -> Result> { +pub(crate) fn try_get<'a, 'b, T>(map: &'b Map, name: &'a str) -> Result> +where + T: Send + Sync + 'a + 'b + 'static, +{ map.read() .expect("locked for reading") .get(name) diff --git a/src/service/services.rs b/src/service/services.rs index 2b9b93d4..8e69cdbb 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -193,11 +193,17 @@ impl Services { } } - pub fn try_get<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(&'b self, name: &'a str) -> Result> { + pub fn try_get<'a, 'b, T>(&'b self, name: &'a str) -> Result> + where + T: Send + Sync + 'a + 'b + 'static, + { service::try_get::(&self.service, name) } - pub fn get<'a, 'b, T: Send + Sync + 'a + 'b + 'static>(&'b self, name: &'a str) -> Option> { + pub fn get<'a, 'b, T>(&'b self, name: &'a str) -> Option> + where + T: Send + Sync + 'a + 'b + 'static, + { service::get::(&self.service, name) } } From 15ad7523aad98171ae9db15789f4d8798a5a687c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 3 Aug 2024 07:16:39 +0000 Subject: [PATCH 0665/2091] split Destination enum into unit Signed-off-by: Jason Volk --- src/service/sending/dest.rs | 56 +++++++++++++++++++++++++++++++++ src/service/sending/mod.rs | 62 ++++--------------------------------- 2 files changed, 62 insertions(+), 56 deletions(-) create mode 100644 src/service/sending/dest.rs diff --git a/src/service/sending/dest.rs b/src/service/sending/dest.rs new file mode 100644 index 00000000..9968acd7 --- /dev/null +++ b/src/service/sending/dest.rs @@ -0,0 +1,56 @@ +use std::fmt::Debug; + +use conduit::implement; +use ruma::{OwnedServerName, OwnedUserId}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum Destination { + Appservice(String), + Push(OwnedUserId, String), // user and pushkey + Normal(OwnedServerName), +} + +#[implement(Destination)] +#[must_use] +pub fn get_prefix(&self) -> Vec { + match self { + Self::Normal(server) => { + let len = server.as_bytes().len().saturating_add(1); + + let mut p = Vec::with_capacity(len); + p.extend_from_slice(server.as_bytes()); + p.push(0xFF); + p + }, + Self::Appservice(server) => { + let sigil = b"+"; + let len = sigil + .len() + .saturating_add(server.as_bytes().len()) + .saturating_add(1); + + let mut p = Vec::with_capacity(len); + p.extend_from_slice(sigil); + p.extend_from_slice(server.as_bytes()); + p.push(0xFF); + p + }, + Self::Push(user, pushkey) => { + let sigil = b"$"; + let len = sigil + .len() + .saturating_add(user.as_bytes().len()) + .saturating_add(1) + .saturating_add(pushkey.as_bytes().len()) + .saturating_add(1); + + let mut p = Vec::with_capacity(len); + p.extend_from_slice(sigil); + p.extend_from_slice(user.as_bytes()); + p.push(0xFF); + p.extend_from_slice(pushkey.as_bytes()); + p.push(0xFF); + p + }, + } +} diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index fc32d04f..b90ea361 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -1,5 +1,6 @@ mod appservice; mod data; +mod dest; mod send; mod sender; @@ -9,16 +10,18 @@ use async_trait::async_trait; use conduit::{err, warn, Result, Server}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, - OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, + OwnedServerName, RoomId, ServerName, UserId, }; use tokio::sync::Mutex; +use self::data::Data; +pub use self::dest::Destination; use crate::{account_data, client, globals, presence, pusher, resolver, rooms, users, Dep}; pub struct Service { server: Arc, services: Services, - pub db: data::Data, + pub db: Data, sender: loole::Sender, receiver: Mutex>, } @@ -46,13 +49,6 @@ struct Msg { queue_id: Vec, } -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub enum Destination { - Appservice(String), - Push(OwnedUserId, String), // user and pushkey - Normal(OwnedServerName), -} - #[allow(clippy::module_name_repetitions)] #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum SendingEvent { @@ -82,7 +78,7 @@ impl crate::Service for Service { appservice: args.depend::("appservice"), pusher: args.depend::("pusher"), }, - db: data::Data::new(&args), + db: Data::new(&args), sender, receiver: Mutex::new(receiver), })) @@ -280,49 +276,3 @@ impl Service { self.sender.send(msg).map_err(|e| err!("{e}")) } } - -impl Destination { - #[must_use] - pub fn get_prefix(&self) -> Vec { - match self { - Self::Normal(server) => { - let len = server.as_bytes().len().saturating_add(1); - - let mut p = Vec::with_capacity(len); - p.extend_from_slice(server.as_bytes()); - p.push(0xFF); - p - }, - Self::Appservice(server) => { - let sigil = b"+"; - let len = sigil - .len() - .saturating_add(server.as_bytes().len()) - .saturating_add(1); - - let mut p = Vec::with_capacity(len); - p.extend_from_slice(sigil); - p.extend_from_slice(server.as_bytes()); - p.push(0xFF); - p - }, - Self::Push(user, pushkey) => { - let sigil = b"$"; - let len = sigil - .len() - .saturating_add(user.as_bytes().len()) - .saturating_add(1) - .saturating_add(pushkey.as_bytes().len()) - .saturating_add(1); - - let mut p = Vec::with_capacity(len); - p.extend_from_slice(sigil); - p.extend_from_slice(user.as_bytes()); - p.push(0xFF); - p.extend_from_slice(pushkey.as_bytes()); - p.push(0xFF); - p - }, - } - } -} From b4d52821234bb8575523032624492985a09d2863 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 4 Aug 2024 06:03:58 +0000 Subject: [PATCH 0666/2091] add nullary type_name debug util Signed-off-by: Jason Volk --- src/core/debug.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/debug.rs b/src/core/debug.rs index 3ab9ed0d..f6c992b9 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -84,10 +84,13 @@ pub fn trap() { #[must_use] pub fn panic_str(p: &Box) -> &'static str { p.downcast_ref::<&str>().copied().unwrap_or_default() } -#[cfg(debug_assertions)] #[inline(always)] #[must_use] -pub fn type_name(_: &T) -> &'static str { std::any::type_name::() } +pub fn rttype_name(_: &T) -> &'static str { type_name::() } + +#[inline(always)] +#[must_use] +pub fn type_name() -> &'static str { std::any::type_name::() } #[must_use] #[inline] From 0e56d1c7a2be78d125a3a246ec2d15636a9eaddb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 4 Aug 2024 06:12:45 +0000 Subject: [PATCH 0667/2091] add math::try_into util Signed-off-by: Jason Volk --- src/core/utils/math.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index 96ac6dc2..f9d0de30 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -1,8 +1,8 @@ -use std::{cmp, time::Duration}; +use std::{cmp, convert::TryFrom, time::Duration}; pub use checked_ops::checked_ops; -use crate::{Err, Error, Result}; +use crate::{debug::type_name, err, Err, Error, Result}; /// Checked arithmetic expression. Returns a Result #[macro_export] @@ -86,3 +86,17 @@ pub fn ruma_from_usize(val: usize) -> ruma::UInt { #[must_use] #[allow(clippy::as_conversions, clippy::cast_possible_truncation)] pub fn usize_from_u64_truncated(val: u64) -> usize { val as usize } + +#[inline] +pub fn try_into, Src>(src: Src) -> Result { + Dst::try_from(src).map_err(try_into_err::) +} + +fn try_into_err, Src>(e: >::Error) -> Error { + drop(e); + err!(Arithmetic( + "failed to convert from {} to {}", + type_name::(), + type_name::() + )) +} From 992f725362698b2036f89a1aae32a9acd52a1cb5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 9 Aug 2024 02:58:10 +0000 Subject: [PATCH 0668/2091] allow expression to defer macro Signed-off-by: Jason Volk --- src/core/utils/defer.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/core/utils/defer.rs b/src/core/utils/defer.rs index 9d42e679..08477b6f 100644 --- a/src/core/utils/defer.rs +++ b/src/core/utils/defer.rs @@ -13,4 +13,10 @@ macro_rules! defer { closure: || $body, }; }; + + ($body:expr) => { + $crate::defer! {{ + $body + }} + }; } From f69f4c479ced451f0547dabb0615287e0c9c4328 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 4 Aug 2024 06:16:17 +0000 Subject: [PATCH 0669/2091] propagate error from database options building Signed-off-by: Jason Volk --- src/database/engine.rs | 12 +++++++++--- src/database/opts.rs | 38 +++++++++++++++++++++++--------------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 3975d3d9..6c5b5bd2 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -49,7 +49,7 @@ impl Engine { let mut db_env = Env::new().or_else(or_else)?; let row_cache = Cache::new_lru_cache(row_cache_capacity_bytes); - let db_opts = db_options(config, &mut db_env, &row_cache, col_cache.get("primary").expect("cache")); + let db_opts = db_options(config, &mut db_env, &row_cache, col_cache.get("primary").expect("cache"))?; let load_time = std::time::Instant::now(); if config.rocksdb_repair { @@ -63,9 +63,15 @@ impl Engine { .collect::>(); debug!("Opening {} column family descriptors in database", cfs.len()); + let cfopts = cfs + .iter() + .map(|name| cf_options(config, name, db_opts.clone(), &mut col_cache)) + .collect::>>()?; + let cfds = cfs .iter() - .map(|name| ColumnFamilyDescriptor::new(name, cf_options(config, name, db_opts.clone(), &mut col_cache))) + .zip(cfopts.into_iter()) + .map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts)) .collect::>(); debug!("Opening database..."); @@ -102,7 +108,7 @@ impl Engine { debug!("Creating new column family in database: {name}"); let mut col_cache = self.col_cache.write().expect("locked"); - let opts = cf_options(&self.server.config, name, self.opts.clone(), &mut col_cache); + let opts = cf_options(&self.server.config, name, self.opts.clone(), &mut col_cache)?; if let Err(e) = self.db.create_cf(name, &opts) { error!(?name, "Failed to create new column family: {e}"); return or_else(e); diff --git a/src/database/opts.rs b/src/database/opts.rs index 8791e778..890e0188 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -1,6 +1,6 @@ -use std::{cmp, collections::HashMap}; +use std::{cmp, collections::HashMap, convert::TryFrom}; -use conduit::{utils, Config}; +use conduit::{utils, Config, Result}; use rocksdb::{ statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, LogLevel, Options, UniversalCompactOptions, UniversalCompactionStopStyle, @@ -11,8 +11,7 @@ use rocksdb::{ /// resulting value. Note that we require special per-column options on some /// columns, therefor columns should only be opened after passing this result /// through cf_options(). -pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_cache: &Cache) -> Options { - const MIN_PARALLELISM: usize = 2; +pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_cache: &Cache) -> Result { const DEFAULT_STATS_LEVEL: StatsLevel = if cfg!(debug_assertions) { StatsLevel::ExceptDetailedTimers } else { @@ -25,14 +24,9 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ set_logging_defaults(&mut opts, config); // Processing - let threads = if config.rocksdb_parallelism_threads == 0 { - cmp::max(MIN_PARALLELISM, utils::available_parallelism()) - } else { - cmp::max(MIN_PARALLELISM, config.rocksdb_parallelism_threads) - }; - - opts.set_max_background_jobs(threads.try_into().unwrap()); - opts.set_max_subcompactions(threads.try_into().unwrap()); + opts.set_enable_pipelined_write(true); + opts.set_max_background_jobs(num_threads::(config)?); + opts.set_max_subcompactions(num_threads::(config)?); opts.set_max_file_opening_threads(0); if config.rocksdb_compaction_prio_idle { env.lower_thread_pool_cpu_priority(); @@ -100,13 +94,15 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ }); opts.set_env(env); - opts + Ok(opts) } /// Adjust options for the specific column by name. Provide the result of /// db_options() as the argument to this function and use the return value in /// the arguments to open the specific column. -pub(crate) fn cf_options(cfg: &Config, name: &str, mut opts: Options, cache: &mut HashMap) -> Options { +pub(crate) fn cf_options( + cfg: &Config, name: &str, mut opts: Options, cache: &mut HashMap, +) -> Result { // Columns with non-default compaction options match name { "backupid_algorithm" @@ -169,7 +165,7 @@ pub(crate) fn cf_options(cfg: &Config, name: &str, mut opts: Options, cache: &mu &_ => {}, } - opts + Ok(opts) } fn set_logging_defaults(opts: &mut Options, config: &Config) { @@ -346,3 +342,15 @@ fn table_options(_config: &Config) -> BlockBasedOptions { opts } + +fn num_threads>(config: &Config) -> Result { + const MIN_PARALLELISM: usize = 2; + + let requested = if config.rocksdb_parallelism_threads != 0 { + config.rocksdb_parallelism_threads + } else { + utils::available_parallelism() + }; + + utils::math::try_into::(cmp::max(MIN_PARALLELISM, requested)) +} From bac795de5fb542a526c7420bb2262ffd59c0c798 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 4 Aug 2024 08:42:03 +0000 Subject: [PATCH 0670/2091] use transparent thiserror attribute; cleanup error variants Signed-off-by: Jason Volk --- src/core/error/mod.rs | 118 +++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 60 deletions(-) diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index b5721dee..5520e865 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -16,86 +16,84 @@ pub enum Error { Panic(&'static str, Box), // std - #[error("{0}")] + #[error(transparent)] Fmt(#[from] fmt::Error), + #[error(transparent)] + FromUtf8(#[from] std::string::FromUtf8Error), #[error("I/O error: {0}")] Io(#[from] std::io::Error), - #[error("{0}")] - Utf8Error(#[from] std::str::Utf8Error), - #[error("{0}")] - FromUtf8Error(#[from] std::string::FromUtf8Error), - #[error("{0}")] - TryFromSliceError(#[from] std::array::TryFromSliceError), - #[error("{0}")] - TryFromIntError(#[from] std::num::TryFromIntError), - #[error("{0}")] - ParseIntError(#[from] std::num::ParseIntError), - #[error("{0}")] - ParseFloatError(#[from] std::num::ParseFloatError), + #[error(transparent)] + ParseFloat(#[from] std::num::ParseFloatError), + #[error(transparent)] + ParseInt(#[from] std::num::ParseIntError), + #[error(transparent)] + TryFromInt(#[from] std::num::TryFromIntError), + #[error(transparent)] + TryFromSlice(#[from] std::array::TryFromSliceError), + #[error(transparent)] + Utf8(#[from] std::str::Utf8Error), // third-party + #[error(transparent)] + CargoToml(#[from] cargo_toml::Error), + #[error(transparent)] + Clap(#[from] clap::error::Error), + #[error(transparent)] + Extension(#[from] axum::extract::rejection::ExtensionRejection), + #[error(transparent)] + Figment(#[from] figment::error::Error), + #[error(transparent)] + Http(#[from] http::Error), + #[error(transparent)] + HttpHeader(#[from] http::header::InvalidHeaderValue), + #[error("Image error: {0}")] + Image(#[from] image::error::ImageError), #[error("Join error: {0}")] JoinError(#[from] tokio::task::JoinError), + #[error(transparent)] + Path(#[from] axum::extract::rejection::PathRejection), #[error("Regex error: {0}")] Regex(#[from] regex::Error), + #[error("Request error: {0}")] + Reqwest(#[from] reqwest::Error), + #[error(transparent)] + TomlDe(#[from] toml::de::Error), + #[error(transparent)] + TomlSer(#[from] toml::ser::Error), #[error("Tracing filter error: {0}")] TracingFilter(#[from] tracing_subscriber::filter::ParseError), #[error("Tracing reload error: {0}")] TracingReload(#[from] tracing_subscriber::reload::Error), - #[error("Image error: {0}")] - Image(#[from] image::error::ImageError), - #[error("Request error: {0}")] - Reqwest(#[from] reqwest::Error), - #[error("{0}")] - Extension(#[from] axum::extract::rejection::ExtensionRejection), - #[error("{0}")] - Path(#[from] axum::extract::rejection::PathRejection), - #[error("{0}")] - Http(#[from] http::Error), - #[error("{0}")] - HttpHeader(#[from] http::header::InvalidHeaderValue), - #[error("{0}")] - CargoToml(#[from] cargo_toml::Error), - #[error("{0}")] - FigmentError(#[from] figment::error::Error), - #[error("{0}")] - TomlSerError(#[from] toml::ser::Error), - #[error("{0}")] - TomlDeError(#[from] toml::de::Error), - #[error("{0}")] - Clap(#[from] clap::error::Error), - // ruma - #[error("{0}")] - IntoHttpError(#[from] ruma::api::error::IntoHttpError), - #[error("{0}")] - RumaError(#[from] ruma::api::client::error::Error), - #[error("uiaa")] - Uiaa(ruma::api::client::uiaa::UiaaInfo), - #[error("{0}")] - Mxid(#[from] ruma::IdParseError), - #[error("{0}: {1}")] - BadRequest(ruma::api::client::error::ErrorKind, &'static str), //TODO: remove - #[error("{0}: {1}")] - Request(ruma::api::client::error::ErrorKind, Cow<'static, str>, http::StatusCode), - #[error("from {0}: {1}")] - Redaction(ruma::OwnedServerName, ruma::canonical_json::RedactionError), - #[error("Remote server {0} responded with: {1}")] - Federation(ruma::OwnedServerName, ruma::api::client::error::Error), - #[error("{0} in {1}")] - InconsistentRoomState(&'static str, ruma::OwnedRoomId), - - // conduwuit + // ruma/conduwuit #[error("Arithmetic operation failed: {0}")] Arithmetic(Cow<'static, str>), + #[error("{0}: {1}")] + BadRequest(ruma::api::client::error::ErrorKind, &'static str), //TODO: remove #[error("{0}")] BadServerResponse(Cow<'static, str>), #[error("There was a problem with the '{0}' directive in your configuration: {1}")] Config(&'static str, Cow<'static, str>), #[error("{0}")] - Database(Cow<'static, str>), - #[error("{0}")] Conflict(&'static str), // This is only needed for when a room alias already exists + #[error("{0}")] + Database(Cow<'static, str>), + #[error("Remote server {0} responded with: {1}")] + Federation(ruma::OwnedServerName, ruma::api::client::error::Error), + #[error("{0} in {1}")] + InconsistentRoomState(&'static str, ruma::OwnedRoomId), + #[error(transparent)] + IntoHttp(#[from] ruma::api::error::IntoHttpError), + #[error(transparent)] + Mxid(#[from] ruma::IdParseError), + #[error("from {0}: {1}")] + Redaction(ruma::OwnedServerName, ruma::canonical_json::RedactionError), + #[error("{0}: {1}")] + Request(ruma::api::client::error::ErrorKind, Cow<'static, str>, http::StatusCode), + #[error(transparent)] + Ruma(#[from] ruma::api::client::error::Error), + #[error("uiaa")] + Uiaa(ruma::api::client::uiaa::UiaaInfo), // unique / untyped #[error("{0}")] @@ -117,7 +115,7 @@ impl Error { pub fn message(&self) -> String { match self { Self::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), - Self::RumaError(ref error) => response::ruma_error_message(error), + Self::Ruma(ref error) => response::ruma_error_message(error), _ => format!("{self}"), } } @@ -136,7 +134,7 @@ impl Error { pub fn status_code(&self) -> http::StatusCode { match self { - Self::Federation(_, ref error) | Self::RumaError(ref error) => error.status_code, + Self::Federation(_, ref error) | Self::Ruma(ref error) => error.status_code, Self::Request(ref kind, _, code) => response::status_code(kind, *code), Self::BadRequest(ref kind, ..) => response::bad_request_code(kind), Self::Conflict(_) => http::StatusCode::CONFLICT, From efa70b9e78e32aaaa2a501cf54f2353d0a70e1af Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 4 Aug 2024 08:43:53 +0000 Subject: [PATCH 0671/2091] add serde de/ser impls for error Signed-off-by: Jason Volk --- src/core/error/mod.rs | 5 +++++ src/core/error/serde.rs | 13 +++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 src/core/error/serde.rs diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 5520e865..3bf3c883 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -2,6 +2,7 @@ mod err; mod log; mod panic; mod response; +mod serde; use std::{any::Any, borrow::Cow, convert::Infallible, fmt}; @@ -56,6 +57,10 @@ pub enum Error { Regex(#[from] regex::Error), #[error("Request error: {0}")] Reqwest(#[from] reqwest::Error), + #[error("{0}")] + SerdeDe(Cow<'static, str>), + #[error("{0}")] + SerdeSer(Cow<'static, str>), #[error(transparent)] TomlDe(#[from] toml::de::Error), #[error(transparent)] diff --git a/src/core/error/serde.rs b/src/core/error/serde.rs new file mode 100644 index 00000000..0c5a153b --- /dev/null +++ b/src/core/error/serde.rs @@ -0,0 +1,13 @@ +use std::fmt::Display; + +use serde::{de, ser}; + +use crate::Error; + +impl de::Error for Error { + fn custom(msg: T) -> Self { Self::SerdeDe(msg.to_string().into()) } +} + +impl ser::Error for Error { + fn custom(msg: T) -> Self { Self::SerdeSer(msg.to_string().into()) } +} From 5db816277b6d419d12a6a7e0208ca5b513fb1038 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 9 Aug 2024 18:10:00 +0000 Subject: [PATCH 0672/2091] add from serde_json error to enum Signed-off-by: Jason Volk --- src/core/error/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 3bf3c883..5047aa07 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -52,6 +52,8 @@ pub enum Error { #[error("Join error: {0}")] JoinError(#[from] tokio::task::JoinError), #[error(transparent)] + Json(#[from] serde_json::Error), + #[error(transparent)] Path(#[from] axum::extract::rejection::PathRejection), #[error("Regex error: {0}")] Regex(#[from] regex::Error), From 40d53563fcf6f85828fcbb3c61c68ed22a0bc5c8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 10 Aug 2024 19:18:22 +0000 Subject: [PATCH 0673/2091] integrate ruma js_int errors into enum Signed-off-by: Jason Volk --- Cargo.lock | 38 ++++++++++++++++---------------------- Cargo.toml | 2 +- src/core/error/mod.rs | 4 ++++ 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94c8411d..ef4ff325 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -500,12 +500,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e8aabfac534be767c909e0690571677d49f41bd8465ae876fe043d52ba5292" +checksum = "5fb8dd288a69fc53a1996d7ecfbf4a20d59065bff137ce7e56bbd620de191189" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -2979,7 +2980,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "assign", "js_int", @@ -3001,7 +3002,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "js_int", "ruma-common", @@ -3013,7 +3014,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "as_variant", "assign", @@ -3036,7 +3037,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "as_variant", "base64 0.22.1", @@ -3066,7 +3067,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3083,22 +3084,15 @@ dependencies = [ "thiserror", "tracing", "url", - "web-time 1.1.0", "wildmatch", ] [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ - "bytes", - "http", - "httparse", "js_int", - "memchr", - "mime", - "rand", "ruma-common", "ruma-events", "serde", @@ -3108,7 +3102,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "js_int", "thiserror", @@ -3117,7 +3111,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "js_int", "ruma-common", @@ -3127,7 +3121,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "once_cell", "proc-macro-crate", @@ -3142,7 +3136,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "js_int", "ruma-common", @@ -3154,7 +3148,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "headers", "http", @@ -3167,7 +3161,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3183,7 +3177,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11cbd52af1039f3d4a0b96333acd1d1687ba01aa#11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index fee6e22e..242654d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -307,7 +307,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "11cbd52af1039f3d4a0b96333acd1d1687ba01aa" +rev = "9508f16eb4150b276d1d744d68177db38ce70297" features = [ "compat", "rand", diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 5047aa07..824b4913 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -54,6 +54,10 @@ pub enum Error { #[error(transparent)] Json(#[from] serde_json::Error), #[error(transparent)] + JsParseInt(#[from] ruma::JsParseIntError), // js_int re-export + #[error(transparent)] + JsTryFromInt(#[from] ruma::JsTryFromIntError), // js_int re-export + #[error(transparent)] Path(#[from] axum::extract::rejection::PathRejection), #[error("Regex error: {0}")] Regex(#[from] regex::Error), From 025afb61cb360c846a14f51aa618e7970b0f808c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 11 Aug 2024 20:55:17 +0000 Subject: [PATCH 0674/2091] bump ruma again for borrowed identifier deserialization Signed-off-by: Jason Volk --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef4ff325..5bd94309 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2980,7 +2980,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "assign", "js_int", @@ -3002,7 +3002,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "js_int", "ruma-common", @@ -3014,7 +3014,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "as_variant", "assign", @@ -3037,7 +3037,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "as_variant", "base64 0.22.1", @@ -3067,7 +3067,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3090,7 +3090,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "js_int", "ruma-common", @@ -3102,7 +3102,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "js_int", "thiserror", @@ -3111,7 +3111,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "js_int", "ruma-common", @@ -3121,7 +3121,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "once_cell", "proc-macro-crate", @@ -3136,7 +3136,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "js_int", "ruma-common", @@ -3148,7 +3148,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "headers", "http", @@ -3161,7 +3161,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3177,7 +3177,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9508f16eb4150b276d1d744d68177db38ce70297#9508f16eb4150b276d1d744d68177db38ce70297" +source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 242654d8..0a242c51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -307,7 +307,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "9508f16eb4150b276d1d744d68177db38ce70297" +rev = "40bf9965ac30118d2f85547f2ce6b7463025841a" features = [ "compat", "rand", From b2d8da489c92be8d5560165ccd4f63c0ec4baa68 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 11 Aug 2024 23:07:58 +0000 Subject: [PATCH 0675/2091] add configurable automatic admin command execution after startup Signed-off-by: Jason Volk --- src/core/config/mod.rs | 3 +++ src/main/clap.rs | 9 ++++++++ src/service/admin/console.rs | 6 ++++++ src/service/admin/mod.rs | 40 ++++++++++++++++++++++++++++++++++-- 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index bec7c4ea..edca607e 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -334,6 +334,8 @@ pub struct Config { pub admin_escape_commands: bool, #[serde(default)] pub admin_console_automatic: bool, + #[serde(default)] + pub admin_execute: Vec, #[serde(default)] pub sentry: bool, @@ -592,6 +594,7 @@ impl fmt::Display for Config { "Activate admin console after startup", &self.admin_console_automatic.to_string(), ); + line("Execute admin commands after startup", &self.admin_execute.join(", ")); line("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()); line("Allow incoming federated typing", &self.allow_incoming_typing.to_string()); line( diff --git a/src/main/clap.rs b/src/main/clap.rs index 92bd73c1..56d614a4 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -24,6 +24,10 @@ pub(crate) struct Args { /// Activate admin command console automatically after startup. #[arg(long, num_args(0))] pub(crate) console: bool, + + /// Execute console command automatically after startup. + #[arg(long)] + pub(crate) execute: Vec, } /// Parse commandline arguments into structured data @@ -39,6 +43,11 @@ pub(crate) fn update(mut config: Figment, args: &Args) -> Result { config = config.join(("admin_console_automatic", true)); } + // Execute commands after any commands listed in configuration file + for command in &args.execute { + config = config.adjoin(("admin_execute", [command])); + } + // All other individual overrides can go last in case we have options which // set multiple conf items at once and the user still needs granular overrides. for option in &args.option { diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index c9a288d9..7f3a6fae 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -194,6 +194,12 @@ impl Console { } } +/// Standalone/static markdown printer. +pub fn print(markdown: &str) { + let output = configure_output(MadSkin::default_dark()); + output.print_text(markdown); +} + fn configure_output(mut output: MadSkin) -> MadSkin { use termimad::{crossterm::style::Color, Alignment, CompoundStyle, LineStyle}; diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 962d31dd..2719c142 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,7 +9,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{debug, error, error::default_log, pdu::PduBuilder, Err, Error, PduEvent, Result, Server}; +use conduit::{debug, error, error::default_log, info, pdu::PduBuilder, Err, Error, PduEvent, Result, Server}; pub use create::create_admin_room; use loole::{Receiver, Sender}; use ruma::{ @@ -20,7 +20,10 @@ use ruma::{ OwnedEventId, OwnedRoomId, RoomId, UserId, }; use serde_json::value::to_raw_value; -use tokio::sync::{Mutex, RwLock}; +use tokio::{ + sync::{Mutex, RwLock}, + time::{sleep, Duration}, +}; use crate::{globals, rooms, rooms::state::RoomMutexGuard, Dep}; @@ -84,6 +87,8 @@ impl crate::Service for Service { async fn worker(self: Arc) -> Result<()> { let receiver = self.receiver.lock().await; let mut signals = self.services.server.signal.subscribe(); + + self.startup_execute().await; self.console_auto_start().await; loop { @@ -356,6 +361,37 @@ impl Service { } } + /// Execute admin commands after startup + async fn startup_execute(&self) { + sleep(Duration::from_millis(500)).await; //TODO: remove this after run-states are broadcast + for (i, command) in self.services.server.config.admin_execute.iter().enumerate() { + self.startup_execute_command(i, command.clone()).await; + tokio::task::yield_now().await; + } + } + + /// Execute one admin command after startup + async fn startup_execute_command(&self, i: usize, command: String) { + debug!("Startup command #{i}: executing {command:?}"); + + match self.command_in_place(command, None).await { + Err(e) => error!("Startup command #{i} failed: {e:?}"), + Ok(None) => info!("Startup command #{i} completed (no output)."), + Ok(Some(output)) => Self::startup_command_output(i, &output), + } + } + + #[cfg(feature = "console")] + fn startup_command_output(i: usize, content: &RoomMessageEventContent) { + info!("Startup command #{i} completed:"); + console::print(content.body()); + } + + #[cfg(not(feature = "console"))] + fn startup_command_output(i: usize, content: &RoomMessageEventContent) { + info!("Startup command #{i} completed:\n{:#?}", content.body()); + } + /// Possibly spawn the terminal console at startup if configured. async fn console_auto_start(&self) { #[cfg(feature = "console")] From 2cf472a69b851e67f1b74018bdf5096e7426321d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 12 Aug 2024 00:05:18 +0000 Subject: [PATCH 0676/2091] elide header from command output when no logs captured Signed-off-by: Jason Volk --- src/admin/handler.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index c25c380f..423e3c90 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -1,4 +1,5 @@ use std::{ + fmt::Write, panic::AssertUnwindSafe, sync::{Arc, Mutex}, time::SystemTime, @@ -109,10 +110,21 @@ async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) "command processed" ); + let mut output = String::new(); + + // Prepend the logs only if any were captured let logs = logs.lock().expect("locked"); - let output = match result { - Err(error) => format!("{logs}\nEncountered an error while handling the command:\n```\n{error:#?}\n```"), - Ok(reply) => format!("{logs}\n{}", reply.body()), //TODO: content is recreated to add logs + if logs.lines().count() > 2 { + writeln!(&mut output, "{logs}").expect("failed to format logs to command output"); + } + drop(logs); + + match result { + Ok(content) => { + write!(&mut output, "{}", content.body()).expect("failed to format command result to output"); + }, + Err(error) => write!(&mut output, "Command failed with error:\n```\n{error:#?}\n```") + .expect("failed to format error to command output"), }; Some(RoomMessageEventContent::notice_markdown(output)) From f540bed61e4ae981f4fee15fcb3389d18937851b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 10 Aug 2024 20:14:26 +0000 Subject: [PATCH 0677/2091] propagate additional errors from db options Signed-off-by: Jason Volk --- src/database/engine.rs | 15 ++++++++++----- src/database/opts.rs | 24 +++++++++++++++--------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 6c5b5bd2..3850c1d3 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -49,7 +49,12 @@ impl Engine { let mut db_env = Env::new().or_else(or_else)?; let row_cache = Cache::new_lru_cache(row_cache_capacity_bytes); - let db_opts = db_options(config, &mut db_env, &row_cache, col_cache.get("primary").expect("cache"))?; + let db_opts = db_options( + config, + &mut db_env, + &row_cache, + col_cache.get("primary").expect("primary cache exists"), + )?; let load_time = std::time::Instant::now(); if config.rocksdb_repair { @@ -178,7 +183,7 @@ impl Engine { return Ok(()); } - let options = BackupEngineOptions::new(path.unwrap())?; + let options = BackupEngineOptions::new(path.expect("valid database backup path"))?; let mut engine = BackupEngine::open(&options, &self.env)?; if config.database_backups_to_keep > 0 { if let Err(e) = engine.create_new_backup_flush(&self.db, true) { @@ -186,7 +191,7 @@ impl Engine { } let engine_info = engine.get_backup_info(); - let info = &engine_info.last().unwrap(); + let info = &engine_info.last().expect("backup engine info is not empty"); info!( "Created database backup #{} using {} bytes in {} files", info.backup_id, info.size, info.num_files, @@ -196,7 +201,7 @@ impl Engine { if config.database_backups_to_keep >= 0 { let keep = u32::try_from(config.database_backups_to_keep)?; if let Err(e) = engine.purge_old_backups(keep.try_into()?) { - error!("Failed to purge old backup: {:?}", e.to_string()); + error!("Failed to purge old backup: {e:?}"); } } @@ -213,7 +218,7 @@ impl Engine { } let mut res = String::new(); - let options = BackupEngineOptions::new(path.expect("valid path")).or_else(or_else)?; + let options = BackupEngineOptions::new(path.expect("valid database backup path")).or_else(or_else)?; let engine = BackupEngine::open(&options, &self.env).or_else(or_else)?; for info in engine.get_backup_info() { writeln!( diff --git a/src/database/opts.rs b/src/database/opts.rs index 890e0188..d2ad4b95 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -1,6 +1,6 @@ use std::{cmp, collections::HashMap, convert::TryFrom}; -use conduit::{utils, Config, Result}; +use conduit::{err, utils, Config, Result}; use rocksdb::{ statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, LogLevel, Options, UniversalCompactOptions, UniversalCompactionStopStyle, @@ -125,7 +125,7 @@ pub(crate) fn cf_options( cfg, cache, name, - cache_size(cfg, cfg.shorteventid_cache_capacity, 64), + cache_size(cfg, cfg.shorteventid_cache_capacity, 64)?, ), "eventid_shorteventid" => set_table_with_new_cache( @@ -133,11 +133,17 @@ pub(crate) fn cf_options( cfg, cache, name, - cache_size(cfg, cfg.eventidshort_cache_capacity, 64), + cache_size(cfg, cfg.eventidshort_cache_capacity, 64)?, ), "shorteventid_authchain" => { - set_table_with_new_cache(&mut opts, cfg, cache, name, cache_size(cfg, cfg.auth_chain_cache_capacity, 192)); + set_table_with_new_cache( + &mut opts, + cfg, + cache, + name, + cache_size(cfg, cfg.auth_chain_cache_capacity, 192)?, + ); }, "shortstatekey_statekey" => set_table_with_new_cache( @@ -145,7 +151,7 @@ pub(crate) fn cf_options( cfg, cache, name, - cache_size(cfg, cfg.shortstatekey_cache_capacity, 1024), + cache_size(cfg, cfg.shortstatekey_cache_capacity, 1024)?, ), "statekey_shortstatekey" => set_table_with_new_cache( @@ -153,11 +159,11 @@ pub(crate) fn cf_options( cfg, cache, name, - cache_size(cfg, cfg.statekeyshort_cache_capacity, 1024), + cache_size(cfg, cfg.statekeyshort_cache_capacity, 1024)?, ), "eventid_outlierpdu" => { - set_table_with_new_cache(&mut opts, cfg, cache, name, cache_size(cfg, cfg.pdu_cache_capacity, 1536)); + set_table_with_new_cache(&mut opts, cfg, cache, name, cache_size(cfg, cfg.pdu_cache_capacity, 1536)?); }, "pduid_pdu" => set_table_with_shared_cache(&mut opts, cfg, cache, name, "eventid_outlierpdu"), @@ -321,13 +327,13 @@ fn set_table_with_shared_cache( opts.set_block_based_table_factory(&table); } -fn cache_size(config: &Config, base_size: u32, entity_size: usize) -> usize { +fn cache_size(config: &Config, base_size: u32, entity_size: usize) -> Result { let ents = f64::from(base_size) * config.cache_capacity_modifier; #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] (ents as usize) .checked_mul(entity_size) - .expect("cache capacity size is too large") + .ok_or_else(|| err!(Config("cache_capacity_modifier", "Cache size is too large."))) } fn table_options(_config: &Config) -> BlockBasedOptions { From 17a54bc4f8e7dcd96dce51f7e5316e75d679889e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 12 Aug 2024 22:53:07 +0000 Subject: [PATCH 0678/2091] refactor for ruma cow headers; update for ContentDisposition type Signed-off-by: Jason Volk --- src/api/client/media.rs | 105 +++++++++++--------------- src/core/error/mod.rs | 2 + src/core/utils/content_disposition.rs | 87 +++++++++++---------- src/service/media/data.rs | 17 +++-- src/service/media/mod.rs | 6 +- src/service/media/thumbnail.rs | 6 +- 6 files changed, 104 insertions(+), 119 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 712c3fe2..9a7d9eaa 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -6,11 +6,7 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ debug_info, debug_warn, err, info, - utils::{ - self, - content_disposition::{content_disposition_type, make_content_disposition, sanitise_filename}, - math::ruma_from_usize, - }, + utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize}, warn, Err, Error, Result, }; use ruma::api::client::media::{ @@ -118,21 +114,14 @@ pub(crate) async fn create_content_route( let mxc = format!("mxc://{}/{}", services.globals.server_name(), utils::random_string(MXC_LENGTH)); + let content_disposition = make_content_disposition(None, body.content_type.as_deref(), body.filename.as_deref()); + services .media .create( Some(sender_user.clone()), &mxc, - body.filename - .as_ref() - .map(|filename| { - format!( - "{}; filename={}", - content_disposition_type(&body.content_type), - sanitise_filename(filename.to_owned()) - ) - }) - .as_deref(), + Some(&content_disposition), body.content_type.as_deref(), &body.file, ) @@ -185,14 +174,14 @@ pub(crate) async fn get_content_route( content_disposition, }) = services.media.get(&mxc).await? { - let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); - let file = content.expect("content"); + let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + let file = content.expect("content"); Ok(get_content::v3::Response { file, - content_type, - content_disposition, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), + content_type: content_type.map(Into::into), + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { @@ -207,18 +196,15 @@ pub(crate) async fn get_content_route( .await .map_err(|e| err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))))?; - let content_disposition = Some(make_content_disposition( - &response.content_type, - response.content_disposition, - None, - )); + let content_disposition = + make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); Ok(get_content::v3::Response { file: response.file, content_type: response.content_type, - content_disposition, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()), + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) } else { Err!(Request(NotFound("Media not found."))) @@ -268,18 +254,15 @@ pub(crate) async fn get_content_as_filename_route( content_disposition, }) = services.media.get(&mxc).await? { - let content_disposition = Some(make_content_disposition( - &content_type, - content_disposition, - Some(body.filename.clone()), - )); + let content_disposition = + make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), Some(&body.filename)); let file = content.expect("content"); Ok(get_content_as_filename::v3::Response { file, - content_type, - content_disposition, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), + content_type: content_type.map(Into::into), + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { @@ -294,17 +277,17 @@ pub(crate) async fn get_content_as_filename_route( .await { Ok(remote_content_response) => { - let content_disposition = Some(make_content_disposition( - &remote_content_response.content_type, - remote_content_response.content_disposition, + let content_disposition = make_content_disposition( + remote_content_response.content_disposition.as_ref(), + remote_content_response.content_type.as_deref(), None, - )); + ); Ok(get_content_as_filename::v3::Response { - content_disposition, + content_disposition: Some(content_disposition), content_type: remote_content_response.content_type, file: remote_content_response.file, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) }, @@ -369,15 +352,15 @@ pub(crate) async fn get_content_thumbnail_route( ) .await? { - let content_disposition = Some(make_content_disposition(&content_type, content_disposition, None)); + let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); let file = content.expect("content"); Ok(get_content_thumbnail::v3::Response { file, - content_type, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), + content_type: content_type.map(Into::into), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - content_disposition, + content_disposition: Some(content_disposition), }) } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { if services @@ -423,18 +406,18 @@ pub(crate) async fn get_content_thumbnail_route( ) .await?; - let content_disposition = Some(make_content_disposition( - &get_thumbnail_response.content_type, - get_thumbnail_response.content_disposition, + let content_disposition = make_content_disposition( + get_thumbnail_response.content_disposition.as_ref(), + get_thumbnail_response.content_type.as_deref(), None, - )); + ); Ok(get_content_thumbnail::v3::Response { file: get_thumbnail_response.file, content_type: get_thumbnail_response.content_type, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()), - content_disposition, + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + content_disposition: Some(content_disposition), }) }, Err(e) => Err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))), @@ -495,18 +478,18 @@ async fn get_remote_content( ) .await?; - let content_disposition = Some(make_content_disposition( - &content_response.content_type, - content_response.content_disposition, + let content_disposition = make_content_disposition( + content_response.content_disposition.as_ref(), + content_response.content_type.as_deref(), None, - )); + ); services .media .create( None, mxc, - content_disposition.as_deref(), + Some(&content_disposition), content_response.content_type.as_deref(), &content_response.file, ) @@ -515,8 +498,8 @@ async fn get_remote_content( Ok(get_content::v3::Response { file: content_response.file, content_type: content_response.content_type, - content_disposition, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.to_owned()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.to_owned()), + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) } diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 824b4913..9797c741 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -87,6 +87,8 @@ pub enum Error { Config(&'static str, Cow<'static, str>), #[error("{0}")] Conflict(&'static str), // This is only needed for when a room alias already exists + #[error(transparent)] + ContentDisposition(#[from] ruma::http_headers::ContentDispositionParseError), #[error("{0}")] Database(Cow<'static, str>), #[error("Remote server {0} responded with: {1}")] diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index be17a731..a2fe923c 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -1,7 +1,8 @@ -use crate::debug_info; +use std::borrow::Cow; -const ATTACHMENT: &str = "attachment"; -const INLINE: &str = "inline"; +use ruma::http_headers::{ContentDisposition, ContentDispositionType}; + +use crate::debug_info; /// as defined by MSC2702 const ALLOWED_INLINE_CONTENT_TYPES: [&str; 26] = [ @@ -38,42 +39,44 @@ const ALLOWED_INLINE_CONTENT_TYPES: [&str; 26] = [ /// Content-Type against MSC2702 list of safe inline Content-Types /// (`ALLOWED_INLINE_CONTENT_TYPES`) #[must_use] -pub fn content_disposition_type(content_type: &Option) -> &'static str { +pub fn content_disposition_type(content_type: Option<&str>) -> ContentDispositionType { let Some(content_type) = content_type else { debug_info!("No Content-Type was given, assuming attachment for Content-Disposition"); - return ATTACHMENT; + return ContentDispositionType::Attachment; }; // is_sorted is unstable /* debug_assert!(ALLOWED_INLINE_CONTENT_TYPES.is_sorted(), * "ALLOWED_INLINE_CONTENT_TYPES is not sorted"); */ - let content_type = content_type + let content_type: Cow<'_, str> = content_type .split(';') .next() .unwrap_or(content_type) - .to_ascii_lowercase(); + .to_ascii_lowercase() + .into(); if ALLOWED_INLINE_CONTENT_TYPES - .binary_search(&content_type.as_str()) + .binary_search(&content_type.as_ref()) .is_ok() { - INLINE + ContentDispositionType::Inline } else { - ATTACHMENT + ContentDispositionType::Attachment } } /// sanitises the file name for the Content-Disposition using /// `sanitize_filename` crate #[tracing::instrument(level = "debug")] -pub fn sanitise_filename(filename: String) -> String { - let options = sanitize_filename::Options { - truncate: false, - ..Default::default() - }; - - sanitize_filename::sanitize_with_options(filename, options) +pub fn sanitise_filename(filename: &str) -> String { + sanitize_filename::sanitize_with_options( + filename, + sanitize_filename::Options { + truncate: false, + ..Default::default() + }, + ) } /// creates the final Content-Disposition based on whether the filename exists @@ -85,33 +88,13 @@ pub fn sanitise_filename(filename: String) -> String { /// /// else: `Content-Disposition: attachment/inline` pub fn make_content_disposition( - content_type: &Option, content_disposition: Option, req_filename: Option, -) -> String { - let filename: String; - - if let Some(req_filename) = req_filename { - filename = sanitise_filename(req_filename); - } else { - filename = content_disposition.map_or_else(String::new, |content_disposition| { - let (_, filename) = content_disposition - .split_once("filename=") - .unwrap_or(("", "")); - - if filename.is_empty() { - String::new() - } else { - sanitise_filename(filename.to_owned()) - } - }); - }; - - if !filename.is_empty() { - // Content-Disposition: attachment/inline; filename=filename.ext - format!("{}; filename={}", content_disposition_type(content_type), filename) - } else { - // Content-Disposition: attachment/inline - String::from(content_disposition_type(content_type)) - } + content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, filename: Option<&str>, +) -> ContentDisposition { + ContentDisposition::new(content_disposition_type(content_type)).with_filename( + filename + .or_else(|| content_disposition.and_then(|content_disposition| content_disposition.filename.as_deref())) + .map(sanitise_filename), + ) } #[cfg(test)] @@ -136,4 +119,20 @@ mod tests { assert_eq!(SANITISED, sanitize_filename::sanitize_with_options(SAMPLE, options.clone())); } + + #[test] + fn empty_sanitisation() { + use crate::utils::string::EMPTY; + + let result = sanitize_filename::sanitize_with_options( + EMPTY, + sanitize_filename::Options { + windows: true, + truncate: true, + replacement: "", + }, + ); + + assert_eq!(EMPTY, result); + } } diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 70e010c2..222d28b5 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use conduit::{debug, debug_info, utils::string_from_bytes, Error, Result}; use database::{Database, Map}; -use ruma::api::client::error::ErrorKind; +use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition}; use super::preview::UrlPreviewData; @@ -14,7 +14,7 @@ pub(crate) struct Data { #[derive(Debug)] pub(super) struct Metadata { - pub(super) content_disposition: Option, + pub(super) content_disposition: Option, pub(super) content_type: Option, pub(super) key: Vec, } @@ -29,8 +29,8 @@ impl Data { } pub(super) fn create_file_metadata( - &self, sender_user: Option<&str>, mxc: &str, width: u32, height: u32, content_disposition: Option<&str>, - content_type: Option<&str>, + &self, sender_user: Option<&str>, mxc: &str, width: u32, height: u32, + content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, ) -> Result> { let mut key = mxc.as_bytes().to_vec(); key.push(0xFF); @@ -39,9 +39,9 @@ impl Data { key.push(0xFF); key.extend_from_slice( content_disposition - .as_ref() - .map(|f| f.as_bytes()) - .unwrap_or_default(), + .map(ToString::to_string) + .unwrap_or_default() + .as_bytes(), ); key.push(0xFF); key.extend_from_slice( @@ -143,7 +143,8 @@ impl Data { } else { Some( string_from_bytes(content_disposition_bytes) - .map_err(|_| Error::bad_database("Content Disposition in mediaid_file is invalid unicode."))?, + .map_err(|_| Error::bad_database("Content Disposition in mediaid_file is invalid unicode."))? + .parse()?, ) }; diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index ff3f3dc4..62baf548 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; use conduit::{debug, debug_error, err, error, trace, utils, utils::MutexMap, Err, Result, Server}; use data::{Data, Metadata}; -use ruma::{OwnedMxcUri, OwnedUserId}; +use ruma::{http_headers::ContentDisposition, OwnedMxcUri, OwnedUserId}; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt, BufReader}, @@ -21,7 +21,7 @@ use crate::{client, globals, Dep}; pub struct FileMeta { pub content: Option>, pub content_type: Option, - pub content_disposition: Option, + pub content_disposition: Option, } pub struct Service { @@ -65,7 +65,7 @@ impl crate::Service for Service { impl Service { /// Uploads a file. pub async fn create( - &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, + &self, sender_user: Option, mxc: &str, content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, file: &[u8], ) -> Result<()> { // Width, Height = 0 if it's not a thumbnail diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 01bf73f6..7a1eef6c 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -2,7 +2,7 @@ use std::{cmp, io::Cursor, num::Saturating as Sat}; use conduit::{checked, Result}; use image::{imageops::FilterType, DynamicImage}; -use ruma::OwnedUserId; +use ruma::{http_headers::ContentDisposition, OwnedUserId}; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt}, @@ -14,7 +14,7 @@ impl super::Service { /// Uploads or replaces a file thumbnail. #[allow(clippy::too_many_arguments)] pub async fn upload_thumbnail( - &self, sender_user: Option, mxc: &str, content_disposition: Option<&str>, + &self, sender_user: Option, mxc: &str, content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, width: u32, height: u32, file: &[u8], ) -> Result<()> { let key = if let Some(user) = sender_user { @@ -104,7 +104,7 @@ impl super::Service { mxc, width, height, - data.content_disposition.as_deref(), + data.content_disposition.as_ref(), data.content_type.as_deref(), )?; From 43300ea41e3cb9680ffd305e5ff2e434080e7cc5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 15 Aug 2024 02:35:19 +0000 Subject: [PATCH 0679/2091] Make axum-dual-protocol a non-feature. Fix build issues. Signed-off-by: Jason Volk --- Cargo.lock | 147 ++++++++++++---------------------------- Cargo.toml | 10 +-- src/core/config/mod.rs | 2 - src/main/Cargo.toml | 3 - src/router/Cargo.toml | 4 -- src/router/layers.rs | 4 +- src/router/serve/tls.rs | 35 ++++------ 7 files changed, 66 insertions(+), 139 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5bd94309..78af3a1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -275,29 +275,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-server" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1ad46c3ec4e12f4a4b6835e173ba21c25e484c9d02b49770bf006ce5367c036" -dependencies = [ - "arc-swap", - "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-util", - "pin-project-lite", - "rustls 0.21.12", - "rustls-pemfile", - "tokio", - "tokio-rustls 0.24.1", - "tower 0.4.13", - "tower-service", -] - [[package]] name = "axum-server" version = "0.7.1" @@ -317,24 +294,25 @@ dependencies = [ "rustls-pemfile", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls", "tower 0.4.13", "tower-service", ] [[package]] name = "axum-server-dual-protocol" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ea4cd08ae2a5f075d28fa31190163c8106a1d2d3189442494bae22b39040a0d" +checksum = "2164551db024e87f20316d164eab9f5ad342d8188b08051ceb15ca92a60ea7b7" dependencies = [ - "axum-server 0.6.0", + "axum-server", "bytes", "http", "http-body-util", "pin-project", + "rustls 0.23.12", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tokio-util", "tower-layer", "tower-service", @@ -500,9 +478,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.11" +version = "1.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fb8dd288a69fc53a1996d7ecfbf4a20d59065bff137ce7e56bbd620de191189" +checksum = "68064e60dbf1f17005c2fde4d07c16d8baa506fd7ffed8ccab702d93617975c7" dependencies = [ "jobserver", "libc", @@ -561,9 +539,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.15" +version = "4.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d8838454fda655dafd3accb2b6e2bea645b9e4078abe84a22ceb947235c5cc" +checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" dependencies = [ "clap_builder", "clap_derive", @@ -599,9 +577,9 @@ checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" [[package]] name = "cmake" -version = "0.1.50" +version = "0.1.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31c789563b815f77f4250caee12365734369f942439b7defd71e18a48197130" +checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" dependencies = [ "cc", ] @@ -769,7 +747,7 @@ version = "0.4.6" dependencies = [ "axum", "axum-client-ip", - "axum-server 0.7.1", + "axum-server", "axum-server-dual-protocol", "bytes", "conduit_admin", @@ -1715,7 +1693,7 @@ dependencies = [ "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls", "tower-service", "webpki-roots", ] @@ -1997,9 +1975,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "a5f43f184355eefb8d17fc948dbecf6c13be3c141f20d834ae842193a448c72a" [[package]] name = "libloading" @@ -2940,7 +2918,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls", "tokio-socks", "tokio-util", "tower-service", @@ -2980,7 +2958,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "assign", "js_int", @@ -3002,7 +2980,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "js_int", "ruma-common", @@ -3014,7 +2992,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "as_variant", "assign", @@ -3037,7 +3015,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "as_variant", "base64 0.22.1", @@ -3067,7 +3045,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3084,15 +3062,22 @@ dependencies = [ "thiserror", "tracing", "url", + "web-time 1.1.0", "wildmatch", ] [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ + "bytes", + "http", + "httparse", "js_int", + "memchr", + "mime", + "rand", "ruma-common", "ruma-events", "serde", @@ -3102,7 +3087,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "js_int", "thiserror", @@ -3111,7 +3096,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "js_int", "ruma-common", @@ -3121,7 +3106,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "once_cell", "proc-macro-crate", @@ -3136,7 +3121,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "js_int", "ruma-common", @@ -3148,7 +3133,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "headers", "http", @@ -3161,7 +3146,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3177,7 +3162,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=40bf9965ac30118d2f85547f2ce6b7463025841a#40bf9965ac30118d2f85547f2ce6b7463025841a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" dependencies = [ "itertools 0.12.1", "js_int", @@ -3263,18 +3248,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "rustls" -version = "0.21.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" -dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", -] - [[package]] name = "rustls" version = "0.22.4" @@ -3284,7 +3257,7 @@ dependencies = [ "log", "ring", "rustls-pki-types", - "rustls-webpki 0.102.6", + "rustls-webpki", "subtle", "zeroize", ] @@ -3300,7 +3273,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.6", + "rustls-webpki", "subtle", "zeroize", ] @@ -3334,16 +3307,6 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "rustls-webpki" version = "0.102.6" @@ -3408,16 +3371,6 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sd-notify" version = "0.4.2" @@ -3590,18 +3543,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.207" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5665e14a49a4ea1b91029ba7d3bca9f299e1f7cfa194388ccc20f14743e784f2" +checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.207" +version = "1.0.208" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6aea2634c86b0e8ef2cfdc0c340baede54ec27b1e46febd7f80dffb2aa44a00e" +checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", @@ -3623,9 +3576,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.124" +version = "1.0.125" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66ad62847a56b3dba58cc891acd13884b9c61138d330c0d7b6181713d4fce38d" +checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" dependencies = [ "itoa", "memchr", @@ -4127,16 +4080,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "tokio-rustls" -version = "0.24.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" -dependencies = [ - "rustls 0.21.12", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.0" diff --git a/Cargo.toml b/Cargo.toml index 0a242c51..2826c440 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -103,6 +103,10 @@ version = "0.7.1" default-features = false features = ["tls-rustls"] +# to listen on both HTTP and HTTPS if listening on TLS dierctly from conduwuit for complement or sytest +[workspace.dependencies.axum-server-dual-protocol] +version = "0.7" + [workspace.dependencies.axum-client-ip] version = "0.6.0" @@ -307,7 +311,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "40bf9965ac30118d2f85547f2ce6b7463025841a" +rev = "d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" features = [ "compat", "rand", @@ -347,10 +351,6 @@ features = [ "bzip2", ] -# to listen on both HTTP and HTTPS if listening on TLS dierctly from conduwuit for complement or sytest -[workspace.dependencies.axum-server-dual-protocol] -version = "0.6" - # optional SHA256 media keys feature [workspace.dependencies.sha2] version = "0.10.8" diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index edca607e..8d6a32c6 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -368,8 +368,6 @@ pub struct TlsConfig { pub key: String, #[serde(default)] /// Whether to listen and allow for HTTP and HTTPS connections (insecure!) - /// Only works / does something if the `axum_dual_protocol` feature flag was - /// built pub dual_protocol: bool, } diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 8dc2a34d..c002c125 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -48,9 +48,6 @@ default = [ "zstd_compression", ] -axum_dual_protocol = [ - "conduit-router/axum_dual_protocol", -] brotli_compression = [ "conduit-api/brotli_compression", "conduit-core/brotli_compression", diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 38e6adc7..52535ecf 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -41,13 +41,9 @@ brotli_compression = [ systemd = [ "dep:sd-notify", ] -axum_dual_protocol = [ - "dep:axum-server-dual-protocol" -] [dependencies] axum-client-ip.workspace = true -axum-server-dual-protocol.optional = true axum-server-dual-protocol.workspace = true axum-server.workspace = true axum.workspace = true diff --git a/src/router/layers.rs b/src/router/layers.rs index 2567fe08..0dbf049f 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -16,9 +16,9 @@ use tower::ServiceBuilder; use tower_http::{ catch_panic::CatchPanicLayer, cors::{self, CorsLayer}, + sensitive_headers::SetSensitiveHeadersLayer, set_header::SetResponseHeaderLayer, trace::{DefaultOnFailure, DefaultOnRequest, DefaultOnResponse, TraceLayer}, - ServiceBuilderExt as _, }; use tracing::Level; @@ -47,7 +47,7 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { let layers = layers.layer(compression_layer(server)); let layers = layers - .sensitive_headers([header::AUTHORIZATION]) + .layer(SetSensitiveHeadersLayer::new([header::AUTHORIZATION])) .layer(axum::middleware::from_fn_with_state(Arc::clone(services), request::spawn)) .layer( TraceLayer::new_for_http() diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index 6f58ce82..109e14d8 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -1,9 +1,11 @@ use std::{net::SocketAddr, sync::Arc}; use axum::Router; -use axum_server::{bind_rustls, tls_rustls::RustlsConfig, Handle as ServerHandle}; -#[cfg(feature = "axum_dual_protocol")] -use axum_server_dual_protocol::ServerExt; +use axum_server::Handle as ServerHandle; +use axum_server_dual_protocol::{ + axum_server::{bind_rustls, tls_rustls::RustlsConfig}, + ServerExt, +}; use conduit::{Result, Server}; use tokio::task::JoinSet; use tracing::{debug, info, warn}; @@ -13,27 +15,18 @@ pub(super) async fn serve( ) -> Result<()> { let config = &server.config; let tls = config.tls.as_ref().expect("TLS configuration"); + let certs = &tls.certs; + let key = &tls.key; - debug!( - "Using direct TLS. Certificate path {} and certificate private key path {}", - &tls.certs, &tls.key - ); + debug!("Using direct TLS. Certificate path {certs} and certificate private key path {key}",); info!( "Note: It is strongly recommended that you use a reverse proxy instead of running conduwuit directly with TLS." ); - let conf = RustlsConfig::from_pem_file(&tls.certs, &tls.key).await?; - - if cfg!(feature = "axum_dual_protocol") { - info!( - "conduwuit was built with axum_dual_protocol feature to listen on both HTTP and HTTPS. This will only \ - take effect if `dual_protocol` is enabled in `[global.tls]`" - ); - } + let conf = RustlsConfig::from_pem_file(certs, key).await?; let mut join_set = JoinSet::new(); let app = app.into_make_service_with_connect_info::(); - if cfg!(feature = "axum_dual_protocol") && tls.dual_protocol { - #[cfg(feature = "axum_dual_protocol")] + if tls.dual_protocol { for addr in &addrs { join_set.spawn_on( axum_server_dual_protocol::bind_dual_protocol(*addr, conf.clone()) @@ -54,13 +47,13 @@ pub(super) async fn serve( } } - if cfg!(feature = "axum_dual_protocol") && tls.dual_protocol { + if tls.dual_protocol { warn!( - "Listening on {:?} with TLS certificate {} and supporting plain text (HTTP) connections too (insecure!)", - addrs, &tls.certs + "Listening on {addrs:?} with TLS certificate {certs} and supporting plain text (HTTP) connections too \ + (insecure!)", ); } else { - info!("Listening on {:?} with TLS certificate {}", addrs, &tls.certs); + info!("Listening on {addrs:?} with TLS certificate {certs}"); } while join_set.join_next().await.is_some() {} From c3f00f4d15905675e26a01a0952caeabd2e0be35 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 15 Aug 2024 19:50:01 +0000 Subject: [PATCH 0680/2091] remove redundant CSP directives which effectively default to the same value Signed-off-by: Jason Volk --- src/router/layers.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index 0dbf049f..a1a70bb8 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -27,8 +27,6 @@ use crate::{request, router}; const CONDUWUIT_CSP: &[&str] = &[ "sandbox", "default-src 'none'", - "font-src 'none'", - "script-src 'none'", "frame-ancestors 'none'", "form-action 'none'", "base-uri 'none'", From 54e6a41404f136b967a98664c739ed3d767a9ce0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 15 Aug 2024 20:08:53 +0000 Subject: [PATCH 0681/2091] move remote media fetchers into services minor error simplification Signed-off-by: Jason Volk --- src/api/client/media.rs | 242 ++++++++++-------------------------- src/service/globals/mod.rs | 2 - src/service/media/mod.rs | 5 +- src/service/media/remote.rs | 108 ++++++++++++++++ 4 files changed, 178 insertions(+), 179 deletions(-) create mode 100644 src/service/media/remote.rs diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 9a7d9eaa..0326568f 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -1,21 +1,16 @@ #![allow(deprecated)] -use std::time::Duration; - use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - debug_info, debug_warn, err, info, + err, utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize}, - warn, Err, Error, Result, + Err, Result, }; use ruma::api::client::media::{ create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, get_media_preview, }; -use service::{ - media::{FileMeta, MXC_LENGTH}, - Services, -}; +use service::media::{FileMeta, MXC_LENGTH}; use crate::{Ruma, RumaResponse}; @@ -62,24 +57,24 @@ pub(crate) async fn get_media_preview_route( let url = &body.url; if !services.media.url_preview_allowed(url) { - debug_info!(%sender_user, %url, "URL is not allowed to be previewed"); - return Err!(Request(Forbidden("URL is not allowed to be previewed"))); + return Err!(Request(Forbidden( + debug_warn!(%sender_user, %url, "URL is not allowed to be previewed") + ))); } - match services.media.get_url_preview(url).await { - Ok(preview) => { - let res = serde_json::value::to_raw_value(&preview).map_err(|e| { - warn!(%sender_user, "Failed to convert UrlPreviewData into a serde json value: {e}"); - err!(Request(Unknown("Failed to generate a URL preview"))) - })?; + let preview = services.media.get_url_preview(url).await.map_err(|e| { + err!(Request(Unknown( + debug_error!(%sender_user, %url, "Failed to fetch a URL preview: {e}") + ))) + })?; - Ok(get_media_preview::v3::Response::from_raw_value(res)) - }, - Err(e) => { - info!(%sender_user, "Failed to generate a URL preview: {e}"); - Err!(Request(Unknown("Failed to generate a URL preview"))) - }, - } + let res = serde_json::value::to_raw_value(&preview).map_err(|e| { + err!(Request(Unknown( + debug_error!(%sender_user, %url, "Failed to parse a URL preview: {e}") + ))) + })?; + + Ok(get_media_preview::v3::Response::from_raw_value(res)) } /// # `GET /_matrix/media/v1/preview_url` @@ -176,25 +171,25 @@ pub(crate) async fn get_content_route( { let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); - let file = content.expect("content"); Ok(get_content::v3::Response { - file, + file: content.expect("entire file contents"), content_type: content_type.map(Into::into), content_disposition: Some(content_disposition), cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { - let response = get_remote_content( - &services, - &mxc, - &body.server_name, - body.media_id.clone(), - body.allow_redirect, - body.timeout_ms, - ) - .await - .map_err(|e| err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))))?; + let response = services + .media + .fetch_remote_content( + &mxc, + &body.server_name, + body.media_id.clone(), + body.allow_redirect, + body.timeout_ms, + ) + .await + .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; let content_disposition = make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); @@ -257,42 +252,36 @@ pub(crate) async fn get_content_as_filename_route( let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), Some(&body.filename)); - let file = content.expect("content"); Ok(get_content_as_filename::v3::Response { - file, + file: content.expect("entire file contents"), content_type: content_type.map(Into::into), content_disposition: Some(content_disposition), cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), }) } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { - match get_remote_content( - &services, - &mxc, - &body.server_name, - body.media_id.clone(), - body.allow_redirect, - body.timeout_ms, - ) - .await - { - Ok(remote_content_response) => { - let content_disposition = make_content_disposition( - remote_content_response.content_disposition.as_ref(), - remote_content_response.content_type.as_deref(), - None, - ); + let response = services + .media + .fetch_remote_content( + &mxc, + &body.server_name, + body.media_id.clone(), + body.allow_redirect, + body.timeout_ms, + ) + .await + .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; - Ok(get_content_as_filename::v3::Response { - content_disposition: Some(content_disposition), - content_type: remote_content_response.content_type, - file: remote_content_response.file, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) - }, - Err(e) => Err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))), - } + let content_disposition = + make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); + + Ok(get_content_as_filename::v3::Response { + content_disposition: Some(content_disposition), + content_type: response.content_type, + file: response.file, + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + }) } else { Err!(Request(NotFound("Media not found."))) } @@ -353,75 +342,31 @@ pub(crate) async fn get_content_thumbnail_route( .await? { let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); - let file = content.expect("content"); Ok(get_content_thumbnail::v3::Response { - file, + file: content.expect("entire file contents"), content_type: content_type.map(Into::into), cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), content_disposition: Some(content_disposition), }) } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { - if services - .globals - .prevent_media_downloads_from() - .contains(&body.server_name) - { - // we'll lie to the client and say the blocked server's media was not found and - // log. the client has no way of telling anyways so this is a security bonus. - debug_warn!("Received request for media `{}` on blocklisted server", mxc); - return Err!(Request(NotFound("Media not found."))); - } - - match services - .sending - .send_federation_request( - &body.server_name, - get_content_thumbnail::v3::Request { - allow_remote: body.allow_remote, - height: body.height, - width: body.width, - method: body.method.clone(), - server_name: body.server_name.clone(), - media_id: body.media_id.clone(), - timeout_ms: body.timeout_ms, - allow_redirect: body.allow_redirect, - animated: body.animated, - }, - ) + let response = services + .media + .fetch_remote_thumbnail(&mxc, &body) .await - { - Ok(get_thumbnail_response) => { - services - .media - .upload_thumbnail( - None, - &mxc, - None, - get_thumbnail_response.content_type.as_deref(), - body.width.try_into().expect("all UInts are valid u32s"), - body.height.try_into().expect("all UInts are valid u32s"), - &get_thumbnail_response.file, - ) - .await?; + .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; - let content_disposition = make_content_disposition( - get_thumbnail_response.content_disposition.as_ref(), - get_thumbnail_response.content_type.as_deref(), - None, - ); + let content_disposition = + make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); - Ok(get_content_thumbnail::v3::Response { - file: get_thumbnail_response.file, - content_type: get_thumbnail_response.content_type, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - content_disposition: Some(content_disposition), - }) - }, - Err(e) => Err!(Request(NotFound(debug_warn!("Fetching media `{mxc}` failed: {e:?}")))), - } + Ok(get_content_thumbnail::v3::Response { + file: response.file, + content_type: response.content_type, + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + content_disposition: Some(content_disposition), + }) } else { Err!(Request(NotFound("Media not found."))) } @@ -448,58 +393,3 @@ pub(crate) async fn get_content_thumbnail_v1_route( .await .map(RumaResponse) } - -async fn get_remote_content( - services: &Services, mxc: &str, server_name: &ruma::ServerName, media_id: String, allow_redirect: bool, - timeout_ms: Duration, -) -> Result { - if services - .globals - .prevent_media_downloads_from() - .contains(&server_name.to_owned()) - { - // we'll lie to the client and say the blocked server's media was not found and - // log. the client has no way of telling anyways so this is a security bonus. - debug_warn!("Received request for media `{mxc}` on blocklisted server"); - return Err!(Request(NotFound("Media not found."))); - } - - let content_response = services - .sending - .send_federation_request( - server_name, - get_content::v3::Request { - allow_remote: true, - server_name: server_name.to_owned(), - media_id, - timeout_ms, - allow_redirect, - }, - ) - .await?; - - let content_disposition = make_content_disposition( - content_response.content_disposition.as_ref(), - content_response.content_type.as_deref(), - None, - ); - - services - .media - .create( - None, - mxc, - Some(&content_disposition), - content_response.content_type.as_deref(), - &content_response.file, - ) - .await?; - - Ok(get_content::v3::Response { - file: content_response.file, - content_type: content_response.content_type, - content_disposition: Some(content_disposition), - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) -} diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 2c588dce..05fe1a77 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -248,8 +248,6 @@ impl Service { pub fn allow_outgoing_read_receipts(&self) -> bool { self.config.allow_outgoing_read_receipts } - pub fn prevent_media_downloads_from(&self) -> &[OwnedServerName] { &self.config.prevent_media_downloads_from } - pub fn forbidden_remote_room_directory_server_names(&self) -> &[OwnedServerName] { &self.config.forbidden_remote_room_directory_server_names } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 62baf548..b63dc141 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,5 +1,6 @@ mod data; mod preview; +mod remote; mod tests; mod thumbnail; @@ -15,7 +16,7 @@ use tokio::{ io::{AsyncReadExt, AsyncWriteExt, BufReader}, }; -use crate::{client, globals, Dep}; +use crate::{client, globals, sending, Dep}; #[derive(Debug)] pub struct FileMeta { @@ -34,6 +35,7 @@ struct Services { server: Arc, client: Dep, globals: Dep, + sending: Dep, } /// generated MXC ID (`media-id`) length @@ -49,6 +51,7 @@ impl crate::Service for Service { server: args.server.clone(), client: args.depend::("client"), globals: args.depend::("globals"), + sending: args.depend::("sending"), }, })) } diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs new file mode 100644 index 00000000..c07ded76 --- /dev/null +++ b/src/service/media/remote.rs @@ -0,0 +1,108 @@ +use std::time::Duration; + +use conduit::{debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, Result}; +use ruma::{ + api::client::media::{get_content, get_content_thumbnail}, + ServerName, +}; + +#[implement(super::Service)] +#[allow(deprecated)] +pub async fn fetch_remote_thumbnail( + &self, mxc: &str, body: &get_content_thumbnail::v3::Request, +) -> Result { + let server_name = &body.server_name; + self.check_fetch_authorized(mxc, server_name)?; + + let reponse = self + .services + .sending + .send_federation_request( + server_name, + get_content_thumbnail::v3::Request { + allow_remote: body.allow_remote, + height: body.height, + width: body.width, + method: body.method.clone(), + server_name: body.server_name.clone(), + media_id: body.media_id.clone(), + timeout_ms: body.timeout_ms, + allow_redirect: body.allow_redirect, + animated: body.animated, + }, + ) + .await?; + + self.upload_thumbnail( + None, + mxc, + None, + reponse.content_type.as_deref(), + body.width + .try_into() + .map_err(|e| err!(Request(InvalidParam("Width is invalid: {e:?}"))))?, + body.height + .try_into() + .map_err(|e| err!(Request(InvalidParam("Height is invalid: {e:?}"))))?, + &reponse.file, + ) + .await?; + + Ok(reponse) +} + +#[implement(super::Service)] +#[allow(deprecated)] +pub async fn fetch_remote_content( + &self, mxc: &str, server_name: &ServerName, media_id: String, allow_redirect: bool, timeout_ms: Duration, +) -> Result { + self.check_fetch_authorized(mxc, server_name)?; + + let response = self + .services + .sending + .send_federation_request( + server_name, + get_content::v3::Request { + allow_remote: true, + server_name: server_name.to_owned(), + media_id, + timeout_ms, + allow_redirect, + }, + ) + .await?; + + let content_disposition = + make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); + + self.create( + None, + mxc, + Some(&content_disposition), + response.content_type.as_deref(), + &response.file, + ) + .await?; + + Ok(response) +} + +#[implement(super::Service)] +fn check_fetch_authorized(&self, mxc: &str, server_name: &ServerName) -> Result<()> { + if self + .services + .server + .config + .prevent_media_downloads_from + .iter() + .any(|entry| entry == server_name) + { + // we'll lie to the client and say the blocked server's media was not found and + // log. the client has no way of telling anyways so this is a security bonus. + debug_warn!(%mxc, "Received request for media on blocklisted server"); + return Err!(Request(NotFound("Media not found."))); + } + + Ok(()) +} From 152ae705a0518ed6fbfbbcfaaae85a0b58534d42 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 16 Aug 2024 01:13:20 +0000 Subject: [PATCH 0682/2091] refactor for structured Mxc type Signed-off-by: Jason Volk --- Cargo.lock | 26 +++++++-------- Cargo.toml | 2 +- src/admin/media/commands.rs | 12 +++++-- src/api/client/media.rs | 52 ++++++++++++++++-------------- src/core/error/mod.rs | 2 ++ src/service/media/data.rs | 59 ++++++++++++++++++++-------------- src/service/media/mod.rs | 19 +++++------ src/service/media/preview.rs | 14 ++++---- src/service/media/remote.rs | 47 +++++++++++++-------------- src/service/media/thumbnail.rs | 20 +++++------- 10 files changed, 133 insertions(+), 120 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 78af3a1d..277236b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2958,7 +2958,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "assign", "js_int", @@ -2980,7 +2980,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "js_int", "ruma-common", @@ -2992,7 +2992,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "as_variant", "assign", @@ -3015,7 +3015,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "as_variant", "base64 0.22.1", @@ -3045,7 +3045,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3069,7 +3069,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "bytes", "http", @@ -3087,7 +3087,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "js_int", "thiserror", @@ -3096,7 +3096,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "js_int", "ruma-common", @@ -3106,7 +3106,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "once_cell", "proc-macro-crate", @@ -3121,7 +3121,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "js_int", "ruma-common", @@ -3133,7 +3133,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "headers", "http", @@ -3146,7 +3146,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3162,7 +3162,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d23a8412bd8f875cf81bbd7e20cefa03263fcd0e#d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 2826c440..15007081 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -311,7 +311,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "d23a8412bd8f875cf81bbd7e20cefa03263fcd0e" +rev = "25fbd64b968c5d5088c07750aaa4873e072831b0" features = [ "compat", "rand", diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 7906d951..577d5aba 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -15,7 +15,10 @@ pub(super) async fn delete( if let Some(mxc) = mxc { debug!("Got MXC URL: {mxc}"); - self.services.media.delete(mxc.as_ref()).await?; + self.services + .media + .delete(&mxc.as_str().try_into()?) + .await?; return Ok(RoomMessageEventContent::text_plain( "Deleted the MXC from our database and on our filesystem.", @@ -123,7 +126,10 @@ pub(super) async fn delete( } for mxc_url in mxc_urls { - self.services.media.delete(&mxc_url).await?; + self.services + .media + .delete(&mxc_url.as_str().try_into()?) + .await?; mxc_deletion_count = mxc_deletion_count.saturating_add(1); } @@ -157,7 +163,7 @@ pub(super) async fn delete_list(&self) -> Result { for mxc in mxc_list { debug!("Deleting MXC {mxc} in bulk"); - self.services.media.delete(mxc).await?; + self.services.media.delete(&mxc.try_into()?).await?; mxc_deletion_count = mxc_deletion_count .checked_add(1) .expect("mxc_deletion_count should not get this high"); diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 0326568f..9553d425 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -7,8 +7,12 @@ use conduit::{ utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize}, Err, Result, }; -use ruma::api::client::media::{ - create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, get_media_preview, +use ruma::{ + api::client::media::{ + create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, + get_media_preview, + }, + Mxc, }; use service::media::{FileMeta, MXC_LENGTH}; @@ -106,16 +110,17 @@ pub(crate) async fn create_content_route( body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - - let mxc = format!("mxc://{}/{}", services.globals.server_name(), utils::random_string(MXC_LENGTH)); - let content_disposition = make_content_disposition(None, body.content_type.as_deref(), body.filename.as_deref()); + let mxc = Mxc { + server_name: services.globals.server_name(), + media_id: &utils::random_string(MXC_LENGTH), + }; services .media .create( - Some(sender_user.clone()), &mxc, + Some(sender_user), Some(&content_disposition), body.content_type.as_deref(), &body.file, @@ -123,7 +128,7 @@ pub(crate) async fn create_content_route( .await?; Ok(create_content::v3::Response { - content_uri: mxc.into(), + content_uri: mxc.to_string().into(), blurhash: None, }) } @@ -161,7 +166,10 @@ pub(crate) async fn get_content_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; if let Some(FileMeta { content, @@ -181,13 +189,7 @@ pub(crate) async fn get_content_route( } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { let response = services .media - .fetch_remote_content( - &mxc, - &body.server_name, - body.media_id.clone(), - body.allow_redirect, - body.timeout_ms, - ) + .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) .await .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; @@ -241,7 +243,10 @@ pub(crate) async fn get_content_as_filename_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; if let Some(FileMeta { content, @@ -262,13 +267,7 @@ pub(crate) async fn get_content_as_filename_route( } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { let response = services .media - .fetch_remote_content( - &mxc, - &body.server_name, - body.media_id.clone(), - body.allow_redirect, - body.timeout_ms, - ) + .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) .await .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; @@ -322,7 +321,10 @@ pub(crate) async fn get_content_thumbnail_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let mxc = format!("mxc://{}/{}", body.server_name, body.media_id); + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; if let Some(FileMeta { content, @@ -353,7 +355,7 @@ pub(crate) async fn get_content_thumbnail_route( } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { let response = services .media - .fetch_remote_thumbnail(&mxc, &body) + .fetch_remote_thumbnail_legacy(&body) .await .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 9797c741..92dbdfe3 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -98,6 +98,8 @@ pub enum Error { #[error(transparent)] IntoHttp(#[from] ruma::api::error::IntoHttpError), #[error(transparent)] + Mxc(#[from] ruma::MxcUriError), + #[error(transparent)] Mxid(#[from] ruma::IdParseError), #[error("from {0}: {1}")] Redaction(ruma::OwnedServerName, ruma::canonical_json::RedactionError), diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 222d28b5..f5de2bbe 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -1,8 +1,12 @@ use std::sync::Arc; -use conduit::{debug, debug_info, utils::string_from_bytes, Error, Result}; +use conduit::{ + debug, debug_info, trace, + utils::{str_from_bytes, string_from_bytes}, + Err, Error, Result, +}; use database::{Database, Map}; -use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition}; +use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition, Mxc, UserId}; use super::preview::UrlPreviewData; @@ -29,10 +33,12 @@ impl Data { } pub(super) fn create_file_metadata( - &self, sender_user: Option<&str>, mxc: &str, width: u32, height: u32, + &self, mxc: &Mxc<'_>, user: Option<&UserId>, width: u32, height: u32, content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, ) -> Result> { - let mut key = mxc.as_bytes().to_vec(); + let mut key: Vec = Vec::new(); + key.extend_from_slice(mxc.server_name.as_bytes()); + key.extend_from_slice(mxc.media_id.as_bytes()); key.push(0xFF); key.extend_from_slice(&width.to_be_bytes()); key.extend_from_slice(&height.to_be_bytes()); @@ -53,8 +59,10 @@ impl Data { self.mediaid_file.insert(&key, &[])?; - if let Some(user) = sender_user { - let key = mxc.as_bytes().to_vec(); + if let Some(user) = user { + let mut key: Vec = Vec::new(); + key.extend_from_slice(mxc.server_name.as_bytes()); + key.extend_from_slice(mxc.media_id.as_bytes()); let user = user.as_bytes().to_vec(); self.mediaid_user.insert(&key, &user)?; } @@ -62,22 +70,23 @@ impl Data { Ok(key) } - pub(super) fn delete_file_mxc(&self, mxc: &str) -> Result<()> { - debug!("MXC URI: {:?}", mxc); + pub(super) fn delete_file_mxc(&self, mxc: &Mxc<'_>) -> Result<()> { + debug!("MXC URI: {mxc}"); - let mut prefix = mxc.as_bytes().to_vec(); + let mut prefix: Vec = Vec::new(); + prefix.extend_from_slice(mxc.server_name.as_bytes()); + prefix.extend_from_slice(mxc.media_id.as_bytes()); prefix.push(0xFF); - debug!("MXC db prefix: {prefix:?}"); - - for (key, _) in self.mediaid_file.scan_prefix(prefix) { + trace!("MXC db prefix: {prefix:?}"); + for (key, _) in self.mediaid_file.scan_prefix(prefix.clone()) { debug!("Deleting key: {:?}", key); self.mediaid_file.remove(&key)?; } - for (key, value) in self.mediaid_user.scan_prefix(mxc.as_bytes().to_vec()) { - if key == mxc.as_bytes().to_vec() { - let user = string_from_bytes(&value).unwrap_or_default(); + for (key, value) in self.mediaid_user.scan_prefix(prefix.clone()) { + if key.starts_with(&prefix) { + let user = str_from_bytes(&value).unwrap_or_default(); debug_info!("Deleting key \"{key:?}\" which was uploaded by user {user}"); self.mediaid_user.remove(&key)?; @@ -88,10 +97,12 @@ impl Data { } /// Searches for all files with the given MXC - pub(super) fn search_mxc_metadata_prefix(&self, mxc: &str) -> Result>> { - debug!("MXC URI: {:?}", mxc); + pub(super) fn search_mxc_metadata_prefix(&self, mxc: &Mxc<'_>) -> Result>> { + debug!("MXC URI: {mxc}"); - let mut prefix = mxc.as_bytes().to_vec(); + let mut prefix: Vec = Vec::new(); + prefix.extend_from_slice(mxc.server_name.as_bytes()); + prefix.extend_from_slice(mxc.media_id.as_bytes()); prefix.push(0xFF); let keys: Vec> = self @@ -101,18 +112,18 @@ impl Data { .collect(); if keys.is_empty() { - return Err(Error::bad_database( - "Failed to find any keys in database with the provided MXC.", - )); + return Err!(Database("Failed to find any keys in database for `{mxc}`",)); } - debug!("Got the following keys: {:?}", keys); + debug!("Got the following keys: {keys:?}"); Ok(keys) } - pub(super) fn search_file_metadata(&self, mxc: &str, width: u32, height: u32) -> Result { - let mut prefix = mxc.as_bytes().to_vec(); + pub(super) fn search_file_metadata(&self, mxc: &Mxc<'_>, width: u32, height: u32) -> Result { + let mut prefix: Vec = Vec::new(); + prefix.extend_from_slice(mxc.server_name.as_bytes()); + prefix.extend_from_slice(mxc.media_id.as_bytes()); prefix.push(0xFF); prefix.extend_from_slice(&width.to_be_bytes()); prefix.extend_from_slice(&height.to_be_bytes()); diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index b63dc141..c8f8ad72 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -10,7 +10,7 @@ use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; use conduit::{debug, debug_error, err, error, trace, utils, utils::MutexMap, Err, Result, Server}; use data::{Data, Metadata}; -use ruma::{http_headers::ContentDisposition, OwnedMxcUri, OwnedUserId}; +use ruma::{http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt, BufReader}, @@ -68,17 +68,13 @@ impl crate::Service for Service { impl Service { /// Uploads a file. pub async fn create( - &self, sender_user: Option, mxc: &str, content_disposition: Option<&ContentDisposition>, + &self, mxc: &Mxc<'_>, user: Option<&UserId>, content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, file: &[u8], ) -> Result<()> { // Width, Height = 0 if it's not a thumbnail - let key = if let Some(user) = sender_user { - self.db - .create_file_metadata(Some(user.as_str()), mxc, 0, 0, content_disposition, content_type)? - } else { - self.db - .create_file_metadata(None, mxc, 0, 0, content_disposition, content_type)? - }; + let key = self + .db + .create_file_metadata(mxc, user, 0, 0, content_disposition, content_type)?; //TODO: Dangling metadata in database if creation fails let mut f = self.create_media_file(&key).await?; @@ -88,7 +84,7 @@ impl Service { } /// Deletes a file in the database and from the media directory via an MXC - pub async fn delete(&self, mxc: &str) -> Result<()> { + pub async fn delete(&self, mxc: &Mxc<'_>) -> Result<()> { if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc) { for key in keys { trace!(?mxc, ?key, "Deleting from filesystem"); @@ -111,7 +107,7 @@ impl Service { } /// Downloads a file. - pub async fn get(&self, mxc: &str) -> Result> { + pub async fn get(&self, mxc: &Mxc<'_>) -> Result> { if let Ok(Metadata { content_disposition, content_type, @@ -213,6 +209,7 @@ impl Service { debug!("Deleting media now in the past {user_duration:?}."); let mut deletion_count: usize = 0; for mxc in remote_mxcs { + let mxc: Mxc<'_> = mxc.as_str().try_into()?; debug!("Deleting MXC {mxc} from database and filesystem"); self.delete(&mxc).await?; deletion_count = deletion_count.saturating_add(1); diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index ac24d81a..5704075e 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -4,6 +4,7 @@ use conduit::{debug, utils, warn, Err, Result}; use conduit_core::implement; use image::ImageReader as ImgReader; use ipaddress::IPAddress; +use ruma::Mxc; use serde::Serialize; use url::Url; use webpage::HTML; @@ -44,13 +45,12 @@ pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result< pub async fn download_image(&self, url: &str) -> Result { let client = &self.services.client.url_preview; let image = client.get(url).send().await?.bytes().await?; - let mxc = format!( - "mxc://{}/{}", - self.services.globals.server_name(), - utils::random_string(MXC_LENGTH) - ); + let mxc = Mxc { + server_name: self.services.globals.server_name(), + media_id: &utils::random_string(MXC_LENGTH), + }; - self.create(None, &mxc, None, None, &image).await?; + self.create(&mxc, None, None, None, &image).await?; let (width, height) = match ImgReader::new(Cursor::new(&image)).with_guessed_format() { Err(_) => (None, None), @@ -61,7 +61,7 @@ pub async fn download_image(&self, url: &str) -> Result { }; Ok(UrlPreviewData { - image: Some(mxc), + image: Some(mxc.to_string()), image_size: Some(image.len()), image_width: width, image_height: height, diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index c07ded76..d4c458e2 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -1,25 +1,25 @@ use std::time::Duration; use conduit::{debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, Result}; -use ruma::{ - api::client::media::{get_content, get_content_thumbnail}, - ServerName, -}; +use ruma::{api::client::media, Mxc}; #[implement(super::Service)] #[allow(deprecated)] -pub async fn fetch_remote_thumbnail( - &self, mxc: &str, body: &get_content_thumbnail::v3::Request, -) -> Result { - let server_name = &body.server_name; - self.check_fetch_authorized(mxc, server_name)?; +pub async fn fetch_remote_thumbnail_legacy( + &self, body: &media::get_content_thumbnail::v3::Request, +) -> Result { + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; + self.check_fetch_authorized(&mxc)?; let reponse = self .services .sending .send_federation_request( - server_name, - get_content_thumbnail::v3::Request { + mxc.server_name, + media::get_content_thumbnail::v3::Request { allow_remote: body.allow_remote, height: body.height, width: body.width, @@ -34,8 +34,8 @@ pub async fn fetch_remote_thumbnail( .await?; self.upload_thumbnail( + &mxc, None, - mxc, None, reponse.content_type.as_deref(), body.width @@ -53,20 +53,19 @@ pub async fn fetch_remote_thumbnail( #[implement(super::Service)] #[allow(deprecated)] -pub async fn fetch_remote_content( - &self, mxc: &str, server_name: &ServerName, media_id: String, allow_redirect: bool, timeout_ms: Duration, -) -> Result { - self.check_fetch_authorized(mxc, server_name)?; - +pub async fn fetch_remote_content_legacy( + &self, mxc: &Mxc<'_>, allow_redirect: bool, timeout_ms: Duration, +) -> Result { + self.check_fetch_authorized(mxc)?; let response = self .services .sending .send_federation_request( - server_name, - get_content::v3::Request { + mxc.server_name, + media::get_content::v3::Request { allow_remote: true, - server_name: server_name.to_owned(), - media_id, + server_name: mxc.server_name.into(), + media_id: mxc.media_id.into(), timeout_ms, allow_redirect, }, @@ -77,8 +76,8 @@ pub async fn fetch_remote_content( make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); self.create( - None, mxc, + None, Some(&content_disposition), response.content_type.as_deref(), &response.file, @@ -89,14 +88,14 @@ pub async fn fetch_remote_content( } #[implement(super::Service)] -fn check_fetch_authorized(&self, mxc: &str, server_name: &ServerName) -> Result<()> { +fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> { if self .services .server .config .prevent_media_downloads_from .iter() - .any(|entry| entry == server_name) + .any(|entry| entry == mxc.server_name) { // we'll lie to the client and say the blocked server's media was not found and // log. the client has no way of telling anyways so this is a security bonus. diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 7a1eef6c..3aadcfc0 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -2,7 +2,7 @@ use std::{cmp, io::Cursor, num::Saturating as Sat}; use conduit::{checked, Result}; use image::{imageops::FilterType, DynamicImage}; -use ruma::{http_headers::ContentDisposition, OwnedUserId}; +use ruma::{http_headers::ContentDisposition, Mxc, UserId}; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt}, @@ -14,16 +14,12 @@ impl super::Service { /// Uploads or replaces a file thumbnail. #[allow(clippy::too_many_arguments)] pub async fn upload_thumbnail( - &self, sender_user: Option, mxc: &str, content_disposition: Option<&ContentDisposition>, + &self, mxc: &Mxc<'_>, user: Option<&UserId>, content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, width: u32, height: u32, file: &[u8], ) -> Result<()> { - let key = if let Some(user) = sender_user { - self.db - .create_file_metadata(Some(user.as_str()), mxc, width, height, content_disposition, content_type)? - } else { - self.db - .create_file_metadata(None, mxc, width, height, content_disposition, content_type)? - }; + let key = self + .db + .create_file_metadata(mxc, user, width, height, content_disposition, content_type)?; //TODO: Dangling metadata in database if creation fails let mut f = self.create_media_file(&key).await?; @@ -46,7 +42,7 @@ impl super::Service { /// For width,height <= 96 the server uses another thumbnailing algorithm /// which crops the image afterwards. #[tracing::instrument(skip(self), name = "thumbnail", level = "debug")] - pub async fn get_thumbnail(&self, mxc: &str, width: u32, height: u32) -> Result> { + pub async fn get_thumbnail(&self, mxc: &Mxc<'_>, width: u32, height: u32) -> Result> { // 0, 0 because that's the original file let (width, height, crop) = thumbnail_properties(width, height).unwrap_or((0, 0, false)); @@ -76,7 +72,7 @@ impl super::Service { /// Generate a thumbnail #[tracing::instrument(skip(self), name = "generate", level = "debug")] async fn get_thumbnail_generate( - &self, mxc: &str, width: u32, height: u32, crop: bool, data: Metadata, + &self, mxc: &Mxc<'_>, width: u32, height: u32, crop: bool, data: Metadata, ) -> Result> { let mut content = Vec::new(); let path = self.get_media_file(&data.key); @@ -100,8 +96,8 @@ impl super::Service { // Save thumbnail in database so we don't have to generate it again next time let thumbnail_key = self.db.create_file_metadata( - None, mxc, + None, width, height, data.content_disposition.as_ref(), From 4d0fc4122268f28689120e64cc35a0deb023a852 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 17 Aug 2024 03:39:09 +0000 Subject: [PATCH 0683/2091] add conf item for federation loopback prevention bypass Signed-off-by: Jason Volk --- src/core/config/mod.rs | 3 +++ src/service/resolver/actual.rs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 8d6a32c6..4dc9f877 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -144,6 +144,8 @@ pub struct Config { #[serde(default = "true_fn")] pub allow_federation: bool, #[serde(default)] + pub federation_loopback: bool, + #[serde(default)] pub allow_public_room_directory_over_federation: bool, #[serde(default)] pub allow_public_room_directory_without_auth: bool, @@ -563,6 +565,7 @@ impl fmt::Display for Config { line("New user display name suffix", &self.new_user_displayname_suffix); line("Allow encryption", &self.allow_encryption.to_string()); line("Allow federation", &self.allow_federation.to_string()); + line("Federation loopback", &self.federation_loopback.to_string()); line( "Allow incoming federated presence requests (updates)", &self.allow_incoming_presence.to_string(), diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 623e3a96..07d9a0fa 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -312,7 +312,8 @@ impl super::Service { } fn validate_dest(&self, dest: &ServerName) -> Result<()> { - if dest == self.services.server.config.server_name { + let config = &self.services.server.config; + if dest == config.server_name && !config.federation_loopback { return Err!("Won't send federation request to ourselves"); } From 50e66a29766524b1a0ae0f452b01c54b5e7e9d40 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 17 Aug 2024 03:40:05 +0000 Subject: [PATCH 0684/2091] add partial envfilter support for admin log capture; conf item Signed-off-by: Jason Volk --- src/admin/handler.rs | 37 ++++++++++++++++++++++++++----------- src/core/config/mod.rs | 5 +++++ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/admin/handler.rs b/src/admin/handler.rs index 423e3c90..631d17e1 100644 --- a/src/admin/handler.rs +++ b/src/admin/handler.rs @@ -30,6 +30,7 @@ use service::{ Services, }; use tracing::Level; +use tracing_subscriber::{filter::LevelFilter, EnvFilter}; use crate::{admin, admin::AdminCommand, Command}; @@ -87,17 +88,7 @@ fn reply(mut content: RoomMessageEventContent, reply_id: Option) - // Parse and process a message from the admin room async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) -> CommandOutput { - let filter: &capture::Filter = - &|data| data.level() <= Level::DEBUG && data.our_modules() && data.scope.contains(&"admin"); - let logs = Arc::new(Mutex::new( - collect_stream(|s| markdown_table_head(s)).expect("markdown table header"), - )); - - let capture = Capture::new( - &context.services.server.log.capture, - Some(filter), - capture::fmt(markdown_table, logs.clone()), - ); + let (capture, logs) = capture_create(context); let capture_scope = capture.start(); let result = Box::pin(admin::process(command, context)).await; @@ -130,6 +121,30 @@ async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) Some(RoomMessageEventContent::notice_markdown(output)) } +fn capture_create(context: &Command<'_>) -> (Arc, Arc>) { + let env_config = &context.services.server.config.admin_log_capture; + let env_filter = EnvFilter::try_new(env_config).unwrap_or_else(|_| "debug".into()); + let log_level = env_filter + .max_level_hint() + .and_then(LevelFilter::into_level) + .unwrap_or(Level::DEBUG); + + let filter = + move |data: capture::Data<'_>| data.level() <= log_level && data.our_modules() && data.scope.contains(&"admin"); + + let logs = Arc::new(Mutex::new( + collect_stream(|s| markdown_table_head(s)).expect("markdown table header"), + )); + + let capture = Capture::new( + &context.services.server.log.capture, + Some(filter), + capture::fmt(markdown_table, logs.clone()), + ); + + (capture, logs) +} + // Parse chat messages from the admin room into an AdminCommand object fn parse<'a>( services: &Arc, input: &'a CommandInput, diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 4dc9f877..b7708760 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -338,6 +338,8 @@ pub struct Config { pub admin_console_automatic: bool, #[serde(default)] pub admin_execute: Vec, + #[serde(default = "default_admin_log_capture")] + pub admin_log_capture: String, #[serde(default)] pub sentry: bool, @@ -596,6 +598,7 @@ impl fmt::Display for Config { &self.admin_console_automatic.to_string(), ); line("Execute admin commands after startup", &self.admin_execute.join(", ")); + line("Filter for admin command log capture", &self.admin_log_capture); line("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()); line("Allow incoming federated typing", &self.allow_incoming_typing.to_string()); line( @@ -1054,3 +1057,5 @@ fn default_sentry_traces_sample_rate() -> f32 { 0.15 } fn default_sentry_filter() -> String { "info".to_owned() } fn default_startup_netburst_keep() -> i64 { 50 } + +fn default_admin_log_capture() -> String { "debug".to_owned() } From 867050df9ebb6bcf9356e4a4691904a7cc0103fe Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 18 Aug 2024 16:52:53 -0400 Subject: [PATCH 0685/2091] use fork of rust-rocksdb and jemallocator Signed-off-by: strawberry --- Cargo.lock | 23 ++++++++++------------- Cargo.toml | 12 +++++++----- deps/rust-rocksdb/Cargo.toml | 4 ++-- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 277236b8..c58bb283 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3176,8 +3176,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.24.0+9.4.0" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=4056a3b0f823013fec49f6d0b3e5698856e6476a#4056a3b0f823013fec49f6d0b3e5698856e6476a" +version = "0.25.0+uwu" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=4fcec9354a018bb9667b01d306c103177bcfa790#4fcec9354a018bb9667b01d306c103177bcfa790" dependencies = [ "bindgen", "bzip2-sys", @@ -3193,8 +3193,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.28.0" -source = "git+https://github.com/zaidoon1/rust-rocksdb?rev=4056a3b0f823013fec49f6d0b3e5698856e6476a#4056a3b0f823013fec49f6d0b3e5698856e6476a" +version = "0.29.0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=4fcec9354a018bb9667b01d306c103177bcfa790#4fcec9354a018bb9667b01d306c103177bcfa790" dependencies = [ "libc", "rust-librocksdb-sys", @@ -3964,9 +3964,8 @@ dependencies = [ [[package]] name = "tikv-jemalloc-ctl" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "619bfed27d807b54f7f776b9430d4f8060e66ee138a28632ca898584d462c31c" +version = "0.6.0" +source = "git+https://github.com/girlbossceo/jemallocator?rev=c32af15f3b440ae5e46c3404f78b19093bbd5294#c32af15f3b440ae5e46c3404f78b19093bbd5294" dependencies = [ "libc", "paste", @@ -3975,9 +3974,8 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" -version = "0.5.4+5.3.0-patched" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9402443cb8fd499b6f327e40565234ff34dbda27460c5b47db0db77443dd85d1" +version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "git+https://github.com/girlbossceo/jemallocator?rev=c32af15f3b440ae5e46c3404f78b19093bbd5294#c32af15f3b440ae5e46c3404f78b19093bbd5294" dependencies = [ "cc", "libc", @@ -3985,9 +3983,8 @@ dependencies = [ [[package]] name = "tikv-jemallocator" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965fe0c26be5c56c94e38ba547249074803efd52adfb66de62107d95aab3eaca" +version = "0.6.0" +source = "git+https://github.com/girlbossceo/jemallocator?rev=c32af15f3b440ae5e46c3404f78b19093bbd5294#c32af15f3b440ae5e46c3404f78b19093bbd5294" dependencies = [ "libc", "tikv-jemalloc-sys", diff --git a/Cargo.toml b/Cargo.toml index 15007081..6d008896 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -396,19 +396,21 @@ version = "0.34.0" version = "0.34.0" # jemalloc usage -# locked to 0.5.4 due to static binary linking breakage [workspace.dependencies.tikv-jemalloc-sys] -version = "=0.5.4" +git = "https://github.com/girlbossceo/jemallocator" +rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294" default-features = false features = ["stats", "unprefixed_malloc_on_supported_platforms"] [workspace.dependencies.tikv-jemallocator] -version = "=0.5.4" +git = "https://github.com/girlbossceo/jemallocator" +rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294" default-features = false features = ["stats", "unprefixed_malloc_on_supported_platforms"] [workspace.dependencies.tikv-jemalloc-ctl] -version = "=0.5.4" +git = "https://github.com/girlbossceo/jemallocator" +rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294" default-features = false -features = ["use_std"] +features = ["stats", "use_std"] [workspace.dependencies.console-subscriber] version = "0.4" diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index cf49e1ad..ec262685 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -26,8 +26,8 @@ serde1 = ["rust-rocksdb/serde1"] malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] -git = "https://github.com/zaidoon1/rust-rocksdb" -rev = "4056a3b0f823013fec49f6d0b3e5698856e6476a" +git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" +rev = "4fcec9354a018bb9667b01d306c103177bcfa790" #branch = "master" default-features = false From 23690fd83788e113e801f6aee1cebb7d76e8346a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 18 Aug 2024 17:36:33 -0400 Subject: [PATCH 0686/2091] make jemalloc stats an optional feature Signed-off-by: strawberry --- Cargo.toml | 6 +++--- nix/pkgs/main/default.nix | 4 +++- src/core/Cargo.toml | 5 +++++ src/core/alloc/je.rs | 7 ++++++- src/main/Cargo.toml | 3 +++ 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6d008896..6f4ada7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -400,17 +400,17 @@ version = "0.34.0" git = "https://github.com/girlbossceo/jemallocator" rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294" default-features = false -features = ["stats", "unprefixed_malloc_on_supported_platforms"] +features = ["unprefixed_malloc_on_supported_platforms"] [workspace.dependencies.tikv-jemallocator] git = "https://github.com/girlbossceo/jemallocator" rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294" default-features = false -features = ["stats", "unprefixed_malloc_on_supported_platforms"] +features = ["unprefixed_malloc_on_supported_platforms"] [workspace.dependencies.tikv-jemalloc-ctl] git = "https://github.com/girlbossceo/jemallocator" rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294" default-features = false -features = ["stats", "use_std"] +features = ["use_std"] [workspace.dependencies.console-subscriber] version = "0.4" diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 66d60b77..47b98432 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -57,7 +57,9 @@ rust-jemalloc-sys' = (rust-jemalloc-sys.override { # we dont need cxx/C++ integration [ "--disable-cxx" ] ++ # tikv-jemalloc-sys/profiling feature - lib.optional (featureEnabled "jemalloc_prof") "--enable-prof"; + lib.optional (featureEnabled "jemalloc_prof") "--enable-prof" ++ + # tikv-jemalloc-sys/stats feature + (if (featureEnabled "jemalloc_stats") then [ "--enable-stats" ] else [ "--disable-stats" ]); }); buildDepsOnlyEnv = diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index a5c379e9..87943e60 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -32,6 +32,11 @@ jemalloc = [ jemalloc_prof = [ "tikv-jemalloc-sys/profiling", ] +jemalloc_stats = [ + "tikv-jemalloc-sys/stats", + "tikv-jemalloc-ctl/stats", + "tikv-jemallocator/stats", +] hardened_malloc = [ "dep:hardened_malloc-rs" ] diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 08bfc49a..7561eb95 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -2,7 +2,6 @@ use std::ffi::{c_char, c_void}; -use tikv_jemalloc_ctl as mallctl; use tikv_jemalloc_sys as ffi; use tikv_jemallocator as jemalloc; @@ -10,8 +9,10 @@ use tikv_jemallocator as jemalloc; static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc; #[must_use] +#[cfg(feature = "jemalloc_stats")] pub fn memory_usage() -> Option { use mallctl::stats; + use tikv_jemalloc_ctl as mallctl; let mibs = |input: Result| { let input = input.unwrap_or_default(); @@ -33,6 +34,10 @@ pub fn memory_usage() -> Option { )) } +#[must_use] +#[cfg(not(feature = "jemalloc_stats"))] +pub fn memory_usage() -> Option { None } + #[must_use] pub fn memory_stats() -> Option { const MAX_LENGTH: usize = 65536 - 4096; diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index c002c125..424bfad7 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -87,6 +87,9 @@ jemalloc = [ jemalloc_prof = [ "conduit-core/jemalloc_prof", ] +jemalloc_stats = [ + "conduit-core/jemalloc_stats", +] perf_measurements = [ "dep:opentelemetry", "dep:tracing-flame", From e0fd84149d69c48848ec38c711634948d4ddf358 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 20 Aug 2024 17:03:07 -0400 Subject: [PATCH 0687/2091] bump nix flake.lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'attic': 'github:zhaofengli/attic/26b9417bde03edc6280d1f7ce709cd619cdb72d4' (2024-08-13) → 'github:zhaofengli/attic/acf3c351f8de47c6857f31948ab253f9c7ce2a6f' (2024-08-19) • Updated input 'attic/crane': 'github:ipetkov/crane/480dff0be03dac0e51a8dfc26e882b0d123a450e' (2024-05-29) → 'github:ipetkov/crane/4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4' (2024-08-06) • Updated input 'attic/flake-compat': 'github:edolstra/flake-compat/35bb57c0c8d8b62bbfd284272c928ceb64ddbde9' (2023-01-17) → 'github:edolstra/flake-compat/0f9255e01c2351cc7d116c072cb317785dd33b33' (2023-10-04) • Updated input 'attic/flake-utils': 'github:numtide/flake-utils/5aed5285a952e0b949eb3ba02c12fa4fcfef535f' (2022-11-02) → 'github:numtide/flake-utils/b1d9ab70662946ef0850d488da1c9019f3a9752a' (2024-03-11) • Added input 'attic/flake-utils/systems': 'github:nix-systems/default/da67096a3b9bf56a91d16901293e51ba5b49a27e' (2023-04-09) • Updated input 'attic/nixpkgs': 'github:NixOS/nixpkgs/07262b18b97000d16a4bdb003418bd2fb067a932' (2024-03-25) → 'github:NixOS/nixpkgs/d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d' (2024-08-16) • Updated input 'attic/nixpkgs-stable': 'github:NixOS/nixpkgs/44733514b72e732bd49f5511bd0203dea9b9a434' (2024-03-26) → 'github:NixOS/nixpkgs/205fd4226592cc83fd4c0885a3e4c9c400efabb5' (2024-07-09) • Updated input 'cachix': 'github:cachix/cachix/bd558fd32f55d6df1c3d496bade90335a1a6044d' (2024-08-10) → 'github:cachix/cachix/72bb6b9987149f53f2458bb1596306c621d8b804' (2024-08-14) • Updated input 'crane': 'github:ipetkov/crane/4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4' (2024-08-06) → 'github:ipetkov/crane/7ce92819802bc583b7e82ebc08013a530f22209f' (2024-08-18) • Updated input 'fenix': 'github:nix-community/fenix/296d44c440302980824c5f3b67e477cf0522e0c1' (2024-08-13) → 'github:nix-community/fenix/b6a1c29a8f460af63f66cad2b5acf3b78867603e' (2024-08-20) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/32a86cb1dad2b208e8f36f1bb50c2e4806b0371f' (2024-08-12) → 'github:rust-lang/rust-analyzer/979e3b54f70f6f231c117a5d628b98106e5c7d31' (2024-08-19) • Updated input 'liburing': 'github:axboe/liburing/401b3e4bde2316d35e93b548269c72bbb64adc02' (2024-08-13) → 'github:axboe/liburing/1ecfda19594e68b760a09b91ef442f7e76a3514e' (2024-08-19) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/a58bc8ad779655e790115244571758e8de055e3d' (2024-08-11) → 'github:NixOS/nixpkgs/8a3354191c0d7144db9756a74755672387b702ba' (2024-08-18) • Updated input 'rocksdb': 'github:girlbossceo/rocksdb/911f4243e69c2e320a7a209bf1f5f3ff5f825495' (2024-07-13) → 'github:girlbossceo/rocksdb/60fa2881f3b60b0b77ad3ded0904d432a98f392c' (2024-08-18) Signed-off-by: strawberry --- flake.lock | 100 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/flake.lock b/flake.lock index 66aae3d4..9c4f3c98 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1723558887, - "narHash": "sha256-FBaRmikNnCX+HQJXAfNJyiOl3sM1NrlxvcFwQDqwCyk=", + "lastModified": 1724086414, + "narHash": "sha256-jcY81r8PdMQ9dCGhT0YLZzxPj3kQJXyWCmvQLXbR1EI=", "owner": "zhaofengli", "repo": "attic", - "rev": "26b9417bde03edc6280d1f7ce709cd619cdb72d4", + "rev": "acf3c351f8de47c6857f31948ab253f9c7ce2a6f", "type": "github" }, "original": { @@ -31,11 +31,11 @@ "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1723252655, - "narHash": "sha256-uOxcrNEMTOHvGHYyqDWnUpkhkY29l01KhjkgVQU2gXg=", + "lastModified": 1723641767, + "narHash": "sha256-r53P0B/anBz7GFoqA7YQSKtoeg8+QDqdYuJP8uTkfEY=", "owner": "cachix", "repo": "cachix", - "rev": "bd558fd32f55d6df1c3d496bade90335a1a6044d", + "rev": "72bb6b9987149f53f2458bb1596306c621d8b804", "type": "github" }, "original": { @@ -103,11 +103,11 @@ ] }, "locked": { - "lastModified": 1717025063, - "narHash": "sha256-dIubLa56W9sNNz0e8jGxrX3CAkPXsq7snuFA/Ie6dn8=", + "lastModified": 1722960479, + "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=", "owner": "ipetkov", "repo": "crane", - "rev": "480dff0be03dac0e51a8dfc26e882b0d123a450e", + "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4", "type": "github" }, "original": { @@ -123,11 +123,11 @@ ] }, "locked": { - "lastModified": 1722960479, - "narHash": "sha256-NhCkJJQhD5GUib8zN9JrmYGMwt4lCRp6ZVNzIiYCl0Y=", + "lastModified": 1724006180, + "narHash": "sha256-PVxPj0Ga2fMYMtcT9ARCthF+4U71YkOT7ZjgD/vf1Aw=", "owner": "ipetkov", "repo": "crane", - "rev": "4c6c77920b8d44cd6660c1621dea6b3fc4b4c4f4", + "rev": "7ce92819802bc583b7e82ebc08013a530f22209f", "type": "github" }, "original": { @@ -209,11 +209,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1723530607, - "narHash": "sha256-FaXZZLLDW1D+pj7UgrIslDS8XjMMG3Pus5gAvUYWQS0=", + "lastModified": 1724135364, + "narHash": "sha256-DuPnJXOeigXK8xQdP6BBc6MqJ9p4TzvNUqSrB2P1GQ4=", "owner": "nix-community", "repo": "fenix", - "rev": "296d44c440302980824c5f3b67e477cf0522e0c1", + "rev": "b6a1c29a8f460af63f66cad2b5acf3b78867603e", "type": "github" }, "original": { @@ -226,11 +226,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -289,12 +289,15 @@ } }, "flake-utils": { + "inputs": { + "systems": "systems" + }, "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", "owner": "numtide", "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", "type": "github" }, "original": { @@ -305,7 +308,7 @@ }, "flake-utils_2": { "inputs": { - "systems": "systems" + "systems": "systems_2" }, "locked": { "lastModified": 1689068808, @@ -323,7 +326,7 @@ }, "flake-utils_3": { "inputs": { - "systems": "systems_2" + "systems": "systems_3" }, "locked": { "lastModified": 1710146030, @@ -392,11 +395,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1723553052, - "narHash": "sha256-sjqmYP/tvQKmsTD6FOd8m7aEja2hhEyyB3aTxVXEbuE=", + "lastModified": 1724101565, + "narHash": "sha256-4fbiBVODg/CfRRkH0heOuNHmQnRGbWUKuc+U5AQAn64=", "owner": "axboe", "repo": "liburing", - "rev": "401b3e4bde2316d35e93b548269c72bbb64adc02", + "rev": "1ecfda19594e68b760a09b91ef442f7e76a3514e", "type": "github" }, "original": { @@ -505,11 +508,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1711401922, - "narHash": "sha256-QoQqXoj8ClGo0sqD/qWKFWezgEwUL0SUh37/vY2jNhc=", + "lastModified": 1723827930, + "narHash": "sha256-EU+W5F6y2CVNxGrGIMpY7nSVYq72WRChYxF4zpjx0y4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "07262b18b97000d16a4bdb003418bd2fb067a932", + "rev": "d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d", "type": "github" }, "original": { @@ -553,11 +556,11 @@ }, "nixpkgs-stable": { "locked": { - "lastModified": 1711460390, - "narHash": "sha256-akSgjDZL6pVHEfSE6sz1DNSXuYX6hq+P/1Z5IoYWs7E=", + "lastModified": 1720535198, + "narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "44733514b72e732bd49f5511bd0203dea9b9a434", + "rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5", "type": "github" }, "original": { @@ -617,11 +620,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1723362943, - "narHash": "sha256-dFZRVSgmJkyM0bkPpaYRtG/kRMRTorUIDj8BxoOt1T4=", + "lastModified": 1723991338, + "narHash": "sha256-Grh5PF0+gootJfOJFenTTxDTYPidA3V28dqJ/WV7iis=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "a58bc8ad779655e790115244571758e8de055e3d", + "rev": "8a3354191c0d7144db9756a74755672387b702ba", "type": "github" }, "original": { @@ -660,11 +663,11 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1720900786, - "narHash": "sha256-Vta9Um/RRuWwZ46BjXftV06iWLm/j/9MX39emXUvSAY=", + "lastModified": 1723945438, + "narHash": "sha256-cunZSzKxJJMZzGRtpzr05vV6k4RP5JZD46MGMgJ1x4s=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "911f4243e69c2e320a7a209bf1f5f3ff5f825495", + "rev": "60fa2881f3b60b0b77ad3ded0904d432a98f392c", "type": "github" }, "original": { @@ -692,11 +695,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1723473250, - "narHash": "sha256-Ls0e6R4FmGUFXZlUcm6ZQaVNJ4Yj/nua4SSctXIopao=", + "lastModified": 1724088625, + "narHash": "sha256-vgSBWgUKRyT0ZRZYj43GZkZzPEKgNodCuX/uofqlPsc=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "32a86cb1dad2b208e8f36f1bb50c2e4806b0371f", + "rev": "979e3b54f70f6f231c117a5d628b98106e5c7d31", "type": "github" }, "original": { @@ -735,6 +738,21 @@ "repo": "default", "type": "github" } + }, + "systems_3": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } } }, "root": "root", From edd42974e5e90e20612895c0f5a40ca45caf0138 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 20 Aug 2024 17:45:27 -0400 Subject: [PATCH 0688/2091] bump cargo.lock and deps, add zstd compression to reqwest client Signed-off-by: strawberry --- Cargo.lock | 51 ++++++++++++++++++++++++++++----------- Cargo.toml | 2 +- src/api/Cargo.toml | 3 +++ src/core/Cargo.toml | 4 ++- src/main/Cargo.toml | 1 + src/service/Cargo.toml | 3 +++ src/service/client/mod.rs | 14 +++++++++++ 7 files changed, 62 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c58bb283..e0adf0c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1834,7 +1834,7 @@ dependencies = [ "socket2", "widestring", "windows-sys 0.48.0", - "winreg 0.50.0", + "winreg", ] [[package]] @@ -2883,9 +2883,9 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" -version = "0.12.5" +version = "0.12.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" dependencies = [ "async-compression", "base64 0.22.1", @@ -2927,7 +2927,7 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "webpki-roots", - "winreg 0.52.0", + "windows-registry", ] [[package]] @@ -3872,6 +3872,9 @@ name = "sync_wrapper" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +dependencies = [ + "futures-core", +] [[package]] name = "tendril" @@ -4724,6 +4727,36 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-registry" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +dependencies = [ + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-result" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-strings" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +dependencies = [ + "windows-result", + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -4891,16 +4924,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "winreg" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "xml5ever" version = "0.18.1" diff --git a/Cargo.toml b/Cargo.toml index 6f4ada7c..30457205 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -129,7 +129,7 @@ features = [ ] [workspace.dependencies.reqwest] -version = "0.12.4" +version = "0.12.7" default-features = false features = [ "rustls-tls-native-roots", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index f537fd5f..2b89c3e8 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -25,6 +25,9 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] +zstd_compression = [ + "reqwest/zstd", +] gzip_compression = [ "reqwest/gzip", ] diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 87943e60..f6cd5171 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -46,7 +46,9 @@ gzip_compression = [ brotli_compression = [ "reqwest/brotli", ] -zstd_compression =[] +zstd_compression = [ + "reqwest/zstd", +] perf_measurements = [] sentry_telemetry = [] diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 424bfad7..fae39605 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -129,6 +129,7 @@ tokio_console = [ "tokio/tracing", ] zstd_compression = [ + "conduit-api/zstd_compression", "conduit-core/zstd_compression", "conduit-database/zstd_compression", "conduit-router/zstd_compression", diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index c1d9889e..92563934 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -29,6 +29,9 @@ element_hacks = [] gzip_compression = [ "reqwest/gzip", ] +zstd_compression = [ + "reqwest/zstd", +] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index 386bd33c..63f746db 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -123,6 +123,15 @@ fn base(config: &Config) -> Result { }; }; + #[cfg(feature = "zstd_compression")] + { + builder = if config.zstd_compression { + builder.zstd(true) + } else { + builder.zstd(false).no_brotli() + }; + }; + #[cfg(not(feature = "gzip_compression"))] { builder = builder.no_gzip(); @@ -133,6 +142,11 @@ fn base(config: &Config) -> Result { builder = builder.no_brotli(); }; + #[cfg(not(feature = "zstd_compression"))] + { + builder = builder.no_zstd(); + }; + if let Some(proxy) = config.proxy.to_proxy()? { Ok(builder.proxy(proxy)) } else { From fadb2de5bffb9e79ce026d55ab3c2fe5b31ac26b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 18 Aug 2024 17:54:05 -0400 Subject: [PATCH 0689/2091] docs: document list of projects conduwuit has forked Signed-off-by: strawberry --- docs/development.md | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/docs/development.md b/docs/development.md index a23fe859..62a03e2f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,9 +4,19 @@ Information about developing the project. If you are only interested in using it, you can safely ignore this section. If you plan on contributing, see the [contributor's guide](contributing.md). +## List of forked dependencies +During conduwuit development, we have had to fork some dependencies to support our use-cases in some areas. This ranges from things said upstream project won't accept for any reason, faster-paced development (unresponsive or slow upstream), conduwuit-specific usecases, or lack of time to upstream some things. + +- [ruma/ruma][1]: - various performance improvements, more features, faster-paced development, client/server interop hacks upstream won't accept, etc +- [facebook/rocksdb][2]: - liburing build fixes, GCC build fix, and logging callback C API for Rust tracing integration +- [tikv/jemallocator][3]: - musl builds seem to be broken on upstream +- [zyansheep/rustyline-async][4]: - tab completion callback and `CTRL+\` signal quit event for CLI +- [rust-rocksdb/rust-rocksdb][5]: - [`@zaidoon1`'s][8] fork has quicker updates, more up to date dependencies. Our changes fix musl build issues, Rust part of the logging callback C API, removes unnecessary `gtest` include, and uses our RocksDB and jemallocator +- [tokio-rs/tracing][6]: - Implements `Clone` for `EnvFilter` to support dynamically changing tracing envfilter's alongside other logging/metrics things + ## Debugging with `tokio-console` -[`tokio-console`][1] can be a useful tool for debugging and profiling. To make +[`tokio-console`][7] can be a useful tool for debugging and profiling. To make a `tokio-console`-enabled build of conduwuit, enable the `tokio_console` feature, disable the default `release_max_log_level` feature, and set the `--cfg tokio_unstable` flag to enable experimental tokio APIs. A build might @@ -19,4 +29,11 @@ RUSTFLAGS="--cfg tokio_unstable" cargo build \ --features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console ``` -[1]: https://docs.rs/tokio-console/latest/tokio_console/ +[1]: https://github.com/ruma/ruma/ +[2]: https://github.com/facebook/rocksdb/ +[3]: https://github.com/tikv/jemallocator/ +[4]: https://github.com/zyansheep/rustyline-async/ +[5]: https://github.com/rust-rocksdb/rust-rocksdb/ +[6]: https://github.com/tokio-rs/tracing/ +[7]: https://docs.rs/tokio-console/latest/tokio_console/ +[8]: https://github.com/zaidoon1/ From 1e4b1e516eb3c4804fab96595cb5de5ca2b445c0 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 21 Aug 2024 19:43:28 -0400 Subject: [PATCH 0690/2091] ci: allow olm vulnerable library in nixpkgs for ci Signed-off-by: strawberry --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b56b66a..6959ec90 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,6 +53,8 @@ env: show-trace = true extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= + # complement uses libolm + NIXPKGS_ALLOW_INSECURE: 1 permissions: packages: write @@ -130,9 +132,9 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow - nix develop .#all-features --command true + nix develop .#all-features --command true --impure - name: Cache CI dependencies run: | @@ -283,9 +285,9 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow - nix develop .#all-features --command true + nix develop .#all-features --command true --impure # use sccache for Rust - name: Run sccache-cache From ddacda9ecf687c49fda200d6c29c1d7e3a3bd751 Mon Sep 17 00:00:00 2001 From: morguldir Date: Wed, 21 Aug 2024 19:47:39 -0400 Subject: [PATCH 0691/2091] add heroes support to sliding sync responses Signed-off-by: strawberry --- src/api/client/sync.rs | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index b25348d4..b069a77e 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -20,7 +20,7 @@ use ruma::{ Ephemeral, Filter, GlobalAccountData, InviteState, InvitedRoom, JoinedRoom, LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms, State as RoomState, Timeline, ToDevice, }, - v4::SlidingOp, + v4::{SlidingOp, SlidingSyncRoomHero}, DeviceLists, UnreadNotificationsCount, }, uiaa::UiaaResponse, @@ -1466,13 +1466,10 @@ pub(crate) async fn sync_events_v4_route( .rooms .state_accessor .get_member(room_id, &member)? - .map(|memberevent| { - ( - memberevent - .displayname - .unwrap_or_else(|| member.to_string()), - memberevent.avatar_url, - ) + .map(|memberevent| SlidingSyncRoomHero { + user_id: member, + name: memberevent.displayname, + avatar: memberevent.avatar_url, }), ) }) @@ -1484,18 +1481,26 @@ pub(crate) async fn sync_events_v4_route( Ordering::Greater => { let firsts = heroes[1..] .iter() - .map(|h| h.0.clone()) + .map(|h| h.name.clone().unwrap_or_else(|| h.user_id.to_string())) .collect::>() .join(", "); - let last = heroes[0].0.clone(); + let last = heroes[0] + .name + .clone() + .unwrap_or_else(|| heroes[0].user_id.to_string()); Some(format!("{firsts} and {last}")) }, - Ordering::Equal => Some(heroes[0].0.clone()), + Ordering::Equal => Some( + heroes[0] + .name + .clone() + .unwrap_or_else(|| heroes[0].user_id.to_string()), + ), Ordering::Less => None, }; let heroes_avatar = if heroes.len() == 1 { - heroes[0].1.clone() + heroes[0].avatar.clone() } else { None }; @@ -1558,7 +1563,7 @@ pub(crate) async fn sync_events_v4_route( ), num_live: None, // Count events in timeline greater than global sync counter timestamp: None, - heroes: None, + heroes: Some(heroes), }, ); } From 08efebd5af2d8de1a1c9cb50a85a546b64ccd371 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 21 Aug 2024 20:22:19 -0400 Subject: [PATCH 0692/2091] bump rocksdb to v9.5.2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'rocksdb': 'github:girlbossceo/rocksdb/60fa2881f3b60b0b77ad3ded0904d432a98f392c' (2024-08-18) → 'github:girlbossceo/rocksdb/5a67ad7ce46328578ee5587fb0c23faa03d14e67' (2024-08-22) Signed-off-by: strawberry --- Cargo.lock | 6 +++--- deps/rust-rocksdb/Cargo.toml | 2 +- flake.lock | 8 ++++---- flake.nix | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e0adf0c7..60b0d0c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3176,8 +3176,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.25.0+uwu" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=4fcec9354a018bb9667b01d306c103177bcfa790#4fcec9354a018bb9667b01d306c103177bcfa790" +version = "0.25.0+9.5.2" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=5383ca8173299066b516406e3a2cf945ead891cb#5383ca8173299066b516406e3a2cf945ead891cb" dependencies = [ "bindgen", "bzip2-sys", @@ -3194,7 +3194,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.29.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=4fcec9354a018bb9667b01d306c103177bcfa790#4fcec9354a018bb9667b01d306c103177bcfa790" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=5383ca8173299066b516406e3a2cf945ead891cb#5383ca8173299066b516406e3a2cf945ead891cb" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index ec262685..cc87e99d 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "4fcec9354a018bb9667b01d306c103177bcfa790" +rev = "5383ca8173299066b516406e3a2cf945ead891cb" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index 9c4f3c98..99f10c86 100644 --- a/flake.lock +++ b/flake.lock @@ -663,16 +663,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1723945438, - "narHash": "sha256-cunZSzKxJJMZzGRtpzr05vV6k4RP5JZD46MGMgJ1x4s=", + "lastModified": 1724285323, + "narHash": "sha256-k60kreKQ0v+bQ16yBd2SfLYpuNjMw2qoRmZL/S3k6CU=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "60fa2881f3b60b0b77ad3ded0904d432a98f392c", + "rev": "5a67ad7ce46328578ee5587fb0c23faa03d14e67", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.4.0", + "ref": "v9.5.2", "repo": "rocksdb", "type": "github" } diff --git a/flake.nix b/flake.nix index 9754b532..5f156fec 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.4.0"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.5.2"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From c1bcdeaf0161d2c8578a64d76999ec80b076d750 Mon Sep 17 00:00:00 2001 From: morguldir Date: Thu, 22 Aug 2024 04:08:24 +0200 Subject: [PATCH 0693/2091] Add olm to permittedInsecurePackages for CI --- flake.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 5f156fec..a507ed81 100644 --- a/flake.nix +++ b/flake.nix @@ -16,7 +16,10 @@ outputs = inputs: inputs.flake-utils.lib.eachDefaultSystem (system: let - pkgsHost = inputs.nixpkgs.legacyPackages.${system}; + pkgsHost = import inputs.nixpkgs{ + inherit system; + config.permittedInsecurePackages = [ "olm-3.2.16" ]; + }; pkgsHostStatic = pkgsHost.pkgsStatic; # The Rust toolchain to use From dc3d9ebbf144299a86e08b5b4c43b3f80e56734b Mon Sep 17 00:00:00 2001 From: morguldir Date: Thu, 22 Aug 2024 04:08:57 +0200 Subject: [PATCH 0694/2091] Bump flake.lock to avoid https://github.com/NixOS/nixpkgs/issues/332957 for cargo-audit --- Cargo.lock | 111 ++++++++++++++++++++++++++++++----------------------- flake.lock | 36 ++++++++--------- 2 files changed, 81 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60b0d0c0..a0675640 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "aho-corasick" version = "1.1.3" @@ -73,9 +79,9 @@ dependencies = [ [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "as_variant" @@ -124,7 +130,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -135,7 +141,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -328,7 +334,7 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", ] @@ -370,7 +376,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.74", + "syn 2.0.75", "which", ] @@ -433,9 +439,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.16.3" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83" +checksum = "6fd4c6dcc3b0aea2f5c0b4b82c2b15fe39ddbc76041a310848f4706edf76bb31" [[package]] name = "byteorder" @@ -478,9 +484,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.12" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68064e60dbf1f17005c2fde4d07c16d8baa506fd7ffed8ccab702d93617975c7" +checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" dependencies = [ "jobserver", "libc", @@ -566,7 +572,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -738,7 +744,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1034,7 +1040,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1061,7 +1067,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1171,7 +1177,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1233,12 +1239,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.31" +version = "1.0.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920" +checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -1323,7 +1329,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1402,9 +1408,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" dependencies = [ "atomic-waker", "bytes", @@ -1594,7 +1600,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1958,7 +1964,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -1975,9 +1981,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.156" +version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5f43f184355eefb8d17fc948dbecf6c13be3c141f20d834ae842193a448c72a" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" [[package]] name = "libloading" @@ -2149,6 +2155,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.0.2" @@ -2480,7 +2495,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2573,7 +2588,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2614,7 +2629,7 @@ dependencies = [ "crc32fast", "fdeflate", "flate2", - "miniz_oxide", + "miniz_oxide 0.7.4", ] [[package]] @@ -2645,7 +2660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2674,7 +2689,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", "version_check", "yansi", ] @@ -2699,7 +2714,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -2713,9 +2728,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.11.0" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8746739f11d39ce5ad5c2520a9b75285310dbfe78c541ccf832d38615765aec0" +checksum = "cb4e75767fbc9d92b90e4d0c011f61358cde9513b31ef07ea3631b15ffc3b4fd" dependencies = [ "bitflags 2.6.0", "memchr", @@ -3114,7 +3129,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.74", + "syn 2.0.75", "toml", ] @@ -3280,9 +3295,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" +checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -3558,7 +3573,7 @@ checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -3852,9 +3867,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.74" +version = "2.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fceb41e3d546d0bd83421d3409b1460cc7444cd389341a4c880fe7a042cb3d7" +checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" dependencies = [ "proc-macro2", "quote", @@ -3930,7 +3945,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -4041,9 +4056,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.39.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" dependencies = [ "backtrace", "bytes", @@ -4065,7 +4080,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -4289,7 +4304,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b47 dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -4560,7 +4575,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", "wasm-bindgen-shared", ] @@ -4594,7 +4609,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4959,7 +4974,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] @@ -4979,7 +4994,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.74", + "syn 2.0.75", ] [[package]] diff --git a/flake.lock b/flake.lock index 99f10c86..c492e3be 100644 --- a/flake.lock +++ b/flake.lock @@ -9,11 +9,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1724086414, - "narHash": "sha256-jcY81r8PdMQ9dCGhT0YLZzxPj3kQJXyWCmvQLXbR1EI=", + "lastModified": 1724226964, + "narHash": "sha256-cltFh4su2vcFidxKp7LuEgX3ZGLfPy0DCdrQZ/QTe68=", "owner": "zhaofengli", "repo": "attic", - "rev": "acf3c351f8de47c6857f31948ab253f9c7ce2a6f", + "rev": "6d9aeaef0a067d664cb11bb7704f7ec373d47fb2", "type": "github" }, "original": { @@ -31,11 +31,11 @@ "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1723641767, - "narHash": "sha256-r53P0B/anBz7GFoqA7YQSKtoeg8+QDqdYuJP8uTkfEY=", + "lastModified": 1724232775, + "narHash": "sha256-6u2DycIEgrgNYlLxyGqdFVmBNiKIitnQKJ1pbRP5oko=", "owner": "cachix", "repo": "cachix", - "rev": "72bb6b9987149f53f2458bb1596306c621d8b804", + "rev": "03b6cb3f953097bff378fb8b9ea094bd091a4ec7", "type": "github" }, "original": { @@ -209,11 +209,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1724135364, - "narHash": "sha256-DuPnJXOeigXK8xQdP6BBc6MqJ9p4TzvNUqSrB2P1GQ4=", + "lastModified": 1724221791, + "narHash": "sha256-mKX67QPnUybOopVph/LhOV1G/H4EvPxDIfSmbufrVdA=", "owner": "nix-community", "repo": "fenix", - "rev": "b6a1c29a8f460af63f66cad2b5acf3b78867603e", + "rev": "e88b38a5a3834e039d413a88f8150a75ef6453ef", "type": "github" }, "original": { @@ -395,11 +395,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1724101565, - "narHash": "sha256-4fbiBVODg/CfRRkH0heOuNHmQnRGbWUKuc+U5AQAn64=", + "lastModified": 1724199144, + "narHash": "sha256-MVjnwO6EbKzzSrU51dSseLarZ1fRp+6SagAf/nE/XZU=", "owner": "axboe", "repo": "liburing", - "rev": "1ecfda19594e68b760a09b91ef442f7e76a3514e", + "rev": "2d4e799017d64cd2f8304503eef9064931bb3fbd", "type": "github" }, "original": { @@ -620,11 +620,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1723991338, - "narHash": "sha256-Grh5PF0+gootJfOJFenTTxDTYPidA3V28dqJ/WV7iis=", + "lastModified": 1724271409, + "narHash": "sha256-z4nw9HxkaXEn+5OT8ljLVL2oataHvAzUQ1LEi8Fp+SY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "8a3354191c0d7144db9756a74755672387b702ba", + "rev": "36a9aeaaa17a2d4348498275f9fe530cd4f9e519", "type": "github" }, "original": { @@ -695,11 +695,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1724088625, - "narHash": "sha256-vgSBWgUKRyT0ZRZYj43GZkZzPEKgNodCuX/uofqlPsc=", + "lastModified": 1724153119, + "narHash": "sha256-WxpvDJDttkINkXmUA/W5o11lwLPYhATAgu0QUAacZ2g=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "979e3b54f70f6f231c117a5d628b98106e5c7d31", + "rev": "3723e5910c14f0ffbd13de474b8a8fcc74db04ce", "type": "github" }, "original": { From 47ca835c207ce507c3ec53f9fc21af72c5f215ec Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 23 Aug 2024 19:29:36 -0400 Subject: [PATCH 0695/2091] initialise default TLS crypto provider with aws_lc_rs manually we use ring for hashing state and ruma, and reqwest/rustls defaults to aws_lc_rs, so we have to manually pick which one. there doesn't seem to be a way to just use one for some reason, so lets just use the new aws_lc_rs. Signed-off-by: strawberry --- Cargo.lock | 22 ++++++++++++---------- Cargo.toml | 3 +++ src/core/Cargo.toml | 1 + src/router/Cargo.toml | 1 + src/router/serve/tls.rs | 4 ++++ 5 files changed, 21 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a0675640..ef18182c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -484,9 +484,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.13" +version = "1.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72db2f7947ecee9b03b510377e8bb9077afa27176fdbff55c51027e976fdcc48" +checksum = "50d2eb3cd3d1bf4529e31c215ee6f93ec5a3d536d9f578f93d9d33ee19562932" dependencies = [ "jobserver", "libc", @@ -708,6 +708,7 @@ dependencies = [ "reqwest", "ring", "ruma", + "rustls 0.23.12", "sanitize-filename", "serde", "serde_json", @@ -767,6 +768,7 @@ dependencies = [ "hyper-util", "log", "ruma", + "rustls 0.23.12", "sd-notify", "sentry", "sentry-tower", @@ -1946,9 +1948,9 @@ dependencies = [ [[package]] name = "lazy-regex" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "576c8060ecfdf2e56995cf3274b4f2d71fa5e4fa3607c1c0b63c10180ee58741" +checksum = "8d8e41c97e6bc7ecb552016274b99fbb5d035e8de288c582d9b933af6677bfda" dependencies = [ "lazy-regex-proc_macros", "once_cell", @@ -1957,9 +1959,9 @@ dependencies = [ [[package]] name = "lazy-regex-proc_macros" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9efb9e65d4503df81c615dc33ff07042a9408ac7f26b45abee25566f7fbfd12c" +checksum = "76e1d8b05d672c53cb9c7b920bbba8783845ae4f0b076e02a3db1d02c81b4163" dependencies = [ "proc-macro2", "quote", @@ -2806,9 +2808,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -3591,9 +3593,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.125" +version = "1.0.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83c8e735a073ccf5be70aa8066aa984eaf2fa000db6c8d0100ae605b366d31ed" +checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" dependencies = [ "itoa", "memchr", diff --git a/Cargo.toml b/Cargo.toml index 30457205..a9269d08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,6 +128,9 @@ features = [ "catch-panic", ] +[workspace.dependencies.rustls] +version = "0.23.12" + [workspace.dependencies.reqwest] version = "0.12.7" default-features = false diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index f6cd5171..2e2b0581 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -82,6 +82,7 @@ regex.workspace = true reqwest.workspace = true ring.workspace = true ruma.workspace = true +rustls.workspace = true sanitize-filename.workspace = true serde_json.workspace = true serde_regex.workspace = true diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 52535ecf..2f85ffb7 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -62,6 +62,7 @@ http.workspace = true hyper.workspace = true hyper-util.workspace = true ruma.workspace = true +rustls.workspace = true sentry.optional = true sentry-tower.optional = true sentry-tower.workspace = true diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index 109e14d8..174a511f 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -18,6 +18,10 @@ pub(super) async fn serve( let certs = &tls.certs; let key = &tls.key; + // we use ring for ruma and hashing state, but aws-lc-rs is the new default. + // without this, TLS mode will panic. + _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + debug!("Using direct TLS. Certificate path {certs} and certificate private key path {key}",); info!( "Note: It is strongly recommended that you use a reverse proxy instead of running conduwuit directly with TLS." From c16e5b50eb202d3fddadf33c5dba43f6494a3cd5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 23 Aug 2024 21:11:20 -0400 Subject: [PATCH 0696/2091] update complement results Signed-off-by: strawberry --- tests/test_results/complement/test_results.jsonl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 873397ba..c5b7489b 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -145,19 +145,19 @@ {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'_over_/_matrix/client/v1/media/download"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'_over_/_matrix/client/v1/media/download"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'_over_/_matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name_over__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_upload_with_ASCII_file_name"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name_over__matrix/client/v1/media/download"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally_over__matrix/client/v1/media/download"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation"} {"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation_via__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_upload_with_Unicode_file_name"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline"} From 156199a2180e15ded5f0935a84550ffab78002eb Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 23 Aug 2024 22:16:13 -0400 Subject: [PATCH 0697/2091] add `--no_details` to list banned rooms command Signed-off-by: strawberry --- src/admin/room/moderation.rs | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index a2bc76f7..70d8486b 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -60,7 +60,12 @@ pub(crate) enum RoomModerationCommand { }, /// - List of all rooms we have banned - ListBannedRooms, + ListBannedRooms { + #[arg(long)] + /// Whether to only output room IDs without supplementary room + /// information + no_details: bool, + }, } #[admin_command] @@ -563,7 +568,7 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> } #[admin_command] -async fn list_banned_rooms(&self) -> Result { +async fn list_banned_rooms(&self, no_details: bool) -> Result { let rooms = self .services .rooms @@ -589,7 +594,11 @@ async fn list_banned_rooms(&self) -> Result { rooms.len(), rooms .iter() - .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) + .map(|(id, members, name)| if no_details { + format!("{id}") + } else { + format!("{id}\tMembers: {members}\tName: {name}") + }) .collect::>() .join("\n") ); @@ -597,7 +606,7 @@ async fn list_banned_rooms(&self) -> Result { Ok(RoomMessageEventContent::notice_markdown(output_plain)) }, Err(e) => { - error!("Failed to list banned rooms: {}", e); + error!("Failed to list banned rooms: {e}"); Ok(RoomMessageEventContent::text_plain(format!("Unable to list banned rooms: {e}"))) }, } From a79ae7d1a28c8afd8839180a91cd0ece67ebe989 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 23 Aug 2024 22:37:21 -0400 Subject: [PATCH 0698/2091] add missing MXC URI prefix bytes Signed-off-by: strawberry --- src/service/media/data.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/service/media/data.rs b/src/service/media/data.rs index f5de2bbe..6b1d9cf0 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -37,7 +37,9 @@ impl Data { content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, ) -> Result> { let mut key: Vec = Vec::new(); + key.extend_from_slice(b"mxc://"); key.extend_from_slice(mxc.server_name.as_bytes()); + key.extend_from_slice(b"/"); key.extend_from_slice(mxc.media_id.as_bytes()); key.push(0xFF); key.extend_from_slice(&width.to_be_bytes()); @@ -61,7 +63,9 @@ impl Data { if let Some(user) = user { let mut key: Vec = Vec::new(); + key.extend_from_slice(b"mxc://"); key.extend_from_slice(mxc.server_name.as_bytes()); + key.extend_from_slice(b"/"); key.extend_from_slice(mxc.media_id.as_bytes()); let user = user.as_bytes().to_vec(); self.mediaid_user.insert(&key, &user)?; @@ -74,7 +78,9 @@ impl Data { debug!("MXC URI: {mxc}"); let mut prefix: Vec = Vec::new(); + prefix.extend_from_slice(b"mxc://"); prefix.extend_from_slice(mxc.server_name.as_bytes()); + prefix.extend_from_slice(b"/"); prefix.extend_from_slice(mxc.media_id.as_bytes()); prefix.push(0xFF); @@ -101,7 +107,9 @@ impl Data { debug!("MXC URI: {mxc}"); let mut prefix: Vec = Vec::new(); + prefix.extend_from_slice(b"mxc://"); prefix.extend_from_slice(mxc.server_name.as_bytes()); + prefix.extend_from_slice(b"/"); prefix.extend_from_slice(mxc.media_id.as_bytes()); prefix.push(0xFF); @@ -122,7 +130,9 @@ impl Data { pub(super) fn search_file_metadata(&self, mxc: &Mxc<'_>, width: u32, height: u32) -> Result { let mut prefix: Vec = Vec::new(); + prefix.extend_from_slice(b"mxc://"); prefix.extend_from_slice(mxc.server_name.as_bytes()); + prefix.extend_from_slice(b"/"); prefix.extend_from_slice(mxc.media_id.as_bytes()); prefix.push(0xFF); prefix.extend_from_slice(&width.to_be_bytes()); From 6b65a8fc86019d2c06aa36d1ecc37b8f3e5069c8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 25 Aug 2024 11:46:47 -0400 Subject: [PATCH 0699/2091] add functions to delete media from specific local users Signed-off-by: strawberry --- src/service/media/data.rs | 19 ++++++++++++++++++- src/service/media/mod.rs | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 6b1d9cf0..06cd6cc7 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -6,7 +6,7 @@ use conduit::{ Err, Error, Result, }; use database::{Database, Map}; -use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition, Mxc, UserId}; +use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; use super::preview::UrlPreviewData; @@ -176,6 +176,23 @@ impl Data { }) } + /// Gets all the MXCs associated with a user + pub(super) fn get_all_user_mxcs(&self, user_id: &UserId) -> Vec { + let user_id = user_id.as_bytes().to_vec(); + + self.mediaid_user + .iter() + .filter_map(|(key, user)| { + if *user == user_id { + let mxc_s = string_from_bytes(&key).ok()?; + Some(OwnedMxcUri::from(mxc_s)) + } else { + None + } + }) + .collect() + } + /// Gets all the media keys in our database (this includes all the metadata /// associated with it such as width, height, content-type, etc) pub(crate) fn get_all_media_keys(&self) -> Vec> { self.mediaid_file.iter().map(|(key, _)| key).collect() } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index c8f8ad72..698f7dc2 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -8,7 +8,11 @@ use std::{path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{debug, debug_error, err, error, trace, utils, utils::MutexMap, Err, Result, Server}; +use conduit::{ + debug, debug_error, debug_info, err, error, trace, + utils::{self, MutexMap}, + warn, Err, Result, Server, +}; use data::{Data, Metadata}; use ruma::{http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; use tokio::{ @@ -106,6 +110,31 @@ impl Service { } } + /// Deletes all media by the specified user + /// + /// currently, this is only practical for local users + pub async fn delete_from_user(&self, user: &UserId, force: bool) -> Result { + let mxcs = self.db.get_all_user_mxcs(user); + let mut deletion_count: usize = 0; + + for mxc in mxcs { + let mxc: Mxc<'_> = mxc.as_str().try_into()?; + debug_info!("Deleting MXC {mxc} by user {user} from database and filesystem"); + if force { + _ = self + .delete(&mxc) + .await + .inspect_err(|e| warn!("Failed to delete {mxc} from user {user}, ignoring error: {e}")); + } else { + self.delete(&mxc).await?; + } + + deletion_count = deletion_count.saturating_add(1); + } + + Ok(deletion_count) + } + /// Downloads a file. pub async fn get(&self, mxc: &Mxc<'_>) -> Result> { if let Ok(Metadata { @@ -163,7 +192,7 @@ impl Service { return Err!(Database("Parsed MXC URL unicode bytes from database but still is None")); }; - debug!("Parsed MXC key to URL: {mxc_s}"); + debug_info!("Parsed MXC key to URL: {mxc_s}"); let mxc = OwnedMxcUri::from(mxc_s); if mxc.server_name() == Ok(self.services.globals.server_name()) { debug!("Ignoring local media MXC: {mxc}"); @@ -206,11 +235,11 @@ impl Service { return Err!(Database("Did not found any eligible MXCs to delete.")); } - debug!("Deleting media now in the past {user_duration:?}."); + debug_info!("Deleting media now in the past {user_duration:?}."); let mut deletion_count: usize = 0; for mxc in remote_mxcs { let mxc: Mxc<'_> = mxc.as_str().try_into()?; - debug!("Deleting MXC {mxc} from database and filesystem"); + debug_info!("Deleting MXC {mxc} from database and filesystem"); self.delete(&mxc).await?; deletion_count = deletion_count.saturating_add(1); } From e790785db823d3e7de64c29f5e8e66ed28cd88d8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 25 Aug 2024 11:49:38 -0400 Subject: [PATCH 0700/2091] fix: force argument for delete past remote media wasn't fully functional Signed-off-by: strawberry --- src/service/media/mod.rs | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 698f7dc2..ea225ae6 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -203,7 +203,20 @@ impl Service { let path = self.get_media_file(&key); debug!("MXC path: {path:?}"); - let file_metadata = fs::metadata(path.clone()).await?; + let file_metadata = match fs::metadata(path.clone()).await { + Ok(file_metadata) => file_metadata, + Err(e) => { + if force { + error!("Failed to obtain file metadata for MXC {mxc} at file path \"{path:?}\", skipping: {e}"); + continue; + } + + return Err!(Database( + "Failed to obtain file metadata for MXC {mxc} at file path \"{path:?}\": {e}" + )); + }, + }; + debug!("File metadata: {file_metadata:?}"); let file_created_at = match file_metadata.created() { @@ -214,9 +227,10 @@ impl Service { }, Err(err) => { if force { - error!("Could not delete MXC path {path:?}: {err:?}. Skipping..."); + error!("Could not delete MXC {mxc} at path {path:?}: {err:?}. Skipping..."); continue; } + return Err(err.into()); }, }; @@ -240,8 +254,20 @@ impl Service { for mxc in remote_mxcs { let mxc: Mxc<'_> = mxc.as_str().try_into()?; debug_info!("Deleting MXC {mxc} from database and filesystem"); - self.delete(&mxc).await?; - deletion_count = deletion_count.saturating_add(1); + + match self.delete(&mxc).await { + Ok(()) => { + deletion_count = deletion_count.saturating_add(1); + }, + Err(e) => { + if force { + warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); + continue; + } + + return Err!(Database(warn!("Failed to delete MXC {mxc}: {e}"))); + }, + } } Ok(deletion_count) From 2b7e0dcb80fb5482dd4eedbb1ca2be18f6685162 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 25 Aug 2024 12:24:08 -0400 Subject: [PATCH 0701/2091] add admin command to delete all local media by a local user Signed-off-by: strawberry --- src/admin/media/commands.rs | 17 ++++++++++++++++- src/admin/media/mod.rs | 10 ++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 577d5aba..4d875635 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -1,7 +1,7 @@ use conduit::{debug, info, Result}; use ruma::{events::room::message::RoomMessageEventContent, EventId, MxcUri}; -use crate::admin_command; +use crate::{admin_command, utils::parse_local_user_id}; #[admin_command] pub(super) async fn delete( @@ -186,3 +186,18 @@ pub(super) async fn delete_past_remote_media(&self, duration: String, force: boo "Deleted {deleted_count} total files.", ))) } + +#[admin_command] +pub(super) async fn delete_all_from_user(&self, username: String, force: bool) -> Result { + let user_id = parse_local_user_id(self.services, &username)?; + + let deleted_count = self + .services + .media + .delete_from_user(&user_id, force) + .await?; + + Ok(RoomMessageEventContent::text_plain(format!( + "Deleted {deleted_count} total files.", + ))) +} diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 31cbf810..3a1d4b11 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -32,8 +32,18 @@ pub(super) enum MediaCommand { /// - The duration (at or after), e.g. "5m" to delete all media in the /// past 5 minutes duration: String, + /// Continues deleting remote media if an undeletable object is found #[arg(short, long)] force: bool, }, + + /// - Deletes all the local media from a local user on our server + DeleteAllFromUser { + username: String, + + /// Continues deleting media if an undeletable object is found + #[arg(short, long)] + force: bool, + }, } From 8923c9a22729955eb4345aedb63036b3929b9b60 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 25 Aug 2024 14:32:47 -0400 Subject: [PATCH 0702/2091] add admin command to redact an individual event from a local user Signed-off-by: strawberry --- src/admin/user/commands.rs | 68 ++++++++++++++++++++++++++++-- src/admin/user/mod.rs | 10 ++++- src/service/rooms/timeline/data.rs | 2 +- 3 files changed, 74 insertions(+), 6 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index bdd35d59..c36a3c1d 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -1,15 +1,16 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{join_room_by_id_helper, leave_all_rooms, update_avatar_url, update_displayname}; -use conduit::{error, info, utils, warn, Result}; +use conduit::{error, info, utils, warn, PduBuilder, Result}; use ruma::{ events::{ - room::message::RoomMessageEventContent, + room::{message::RoomMessageEventContent, redaction::RoomRedactionEventContent}, tag::{TagEvent, TagEventContent, TagInfo}, - RoomAccountDataEventType, + RoomAccountDataEventType, TimelineEventType, }, - OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, + EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, }; +use serde_json::value::to_raw_value; use crate::{ admin_command, escape_html, get_room_info, @@ -485,3 +486,62 @@ pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box) tags_event.content.tags ))) } + +#[admin_command] +pub(super) async fn redact_event(&self, event_id: Box) -> Result { + let Some(event) = self + .services + .rooms + .timeline + .get_non_outlier_pdu(&event_id)? + else { + return Ok(RoomMessageEventContent::text_plain("Event does not exist in our database.")); + }; + + if event.is_redacted() { + return Ok(RoomMessageEventContent::text_plain("Event is already redacted.")); + } + + let room_id = event.room_id; + let sender_user = event.sender; + + if !self.services.globals.user_is_local(&sender_user) { + return Ok(RoomMessageEventContent::text_plain("This command only works on local users.")); + } + + let reason = format!( + "The administrator(s) of {} has redacted this user's message.", + self.services.globals.server_name() + ); + + let state_lock = self.services.rooms.state.mutex.lock(&room_id).await; + + let redaction_event_id = self + .services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomRedaction, + content: to_raw_value(&RoomRedactionEventContent { + redacts: Some(event.event_id.clone().into()), + reason: Some(reason), + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: None, + redacts: Some(event.event_id), + timestamp: None, + }, + &sender_user, + &room_id, + &state_lock, + ) + .await?; + + drop(state_lock); + + Ok(RoomMessageEventContent::text_plain(format!( + "Successfully redacted event. Redaction event ID: {redaction_event_id}" + ))) +} diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index b0c0bd1e..f6ecd5e8 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -2,7 +2,7 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{OwnedRoomOrAliasId, RoomId}; +use ruma::{EventId, OwnedRoomOrAliasId, RoomId}; use crate::admin_command_dispatch; @@ -103,4 +103,12 @@ pub(super) enum UserCommand { user_id: String, room_id: Box, }, + + /// - Attempts to forcefully redact the specified event ID from the sender + /// user + /// + /// This is only valid for local users + RedactEvent { + event_id: Box, + }, } diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 5917e96b..2f0c8f25 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -113,7 +113,7 @@ impl Data { self.eventid_pduid.get(event_id.as_bytes()) } - /// Returns the pdu. + /// Returns the pdu directly from `eventid_pduid` only. pub(super) fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { self.eventid_pduid .get(event_id.as_bytes())? From fb49e3706726cbf88d2a4cf310da325cd23992b9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 25 Aug 2024 16:10:07 -0400 Subject: [PATCH 0703/2091] add admin command to delete all remote media from a specific server Signed-off-by: strawberry --- src/admin/media/commands.rs | 60 +++++++++++++++++++++++++++++++++++-- src/admin/media/mod.rs | 11 ++++++- src/service/media/mod.rs | 40 +++++++++++++++++++++++-- 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 4d875635..476f048b 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -1,5 +1,5 @@ -use conduit::{debug, info, Result}; -use ruma::{events::room::message::RoomMessageEventContent, EventId, MxcUri}; +use conduit::{debug, info, trace, warn, Result}; +use ruma::{events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, ServerName}; use crate::{admin_command, utils::parse_local_user_id}; @@ -201,3 +201,59 @@ pub(super) async fn delete_all_from_user(&self, username: String, force: bool) - "Deleted {deleted_count} total files.", ))) } + +#[admin_command] +pub(super) async fn delete_all_from_server( + &self, server_name: Box, force: bool, +) -> Result { + if server_name == self.services.globals.server_name() { + return Ok(RoomMessageEventContent::text_plain("This command only works for remote media.")); + } + + let Ok(all_mxcs) = self.services.media.get_all_mxcs().await else { + return Ok(RoomMessageEventContent::text_plain("Failed to get MXC URIs from our database")); + }; + + let mut deleted_count: usize = 0; + + for mxc in all_mxcs { + let mxc_server_name = match mxc.server_name() { + Ok(server_name) => server_name, + Err(e) => { + if force { + warn!("Failed to parse MXC {mxc} server name from database, ignoring error and skipping: {e}"); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to parse MXC {mxc} server name from database: {e}", + ))); + }, + }; + + if mxc_server_name != server_name || self.services.globals.server_is_ours(mxc_server_name) { + trace!("skipping MXC URI {mxc}"); + continue; + } + + let mxc: Mxc<'_> = mxc.as_str().try_into()?; + + match self.services.media.delete(&mxc).await { + Ok(()) => { + deleted_count = deleted_count.saturating_add(1); + }, + Err(e) => { + if force { + warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); + continue; + } + + return Ok(RoomMessageEventContent::text_plain(format!("Failed to delete MXC {mxc}: {e}"))); + }, + } + } + + Ok(RoomMessageEventContent::text_plain(format!( + "Deleted {deleted_count} total files.", + ))) +} diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 3a1d4b11..1120a63d 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -2,7 +2,7 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{EventId, MxcUri}; +use ruma::{EventId, MxcUri, ServerName}; use crate::admin_command_dispatch; @@ -46,4 +46,13 @@ pub(super) enum MediaCommand { #[arg(short, long)] force: bool, }, + + /// - Deletes all remote media from the specified remote server + DeleteAllFromServer { + server_name: Box, + + /// Continues deleting media if an undeletable object is found + #[arg(short, long)] + force: bool, + }, } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index ea225ae6..f4a62666 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -159,11 +159,43 @@ impl Service { } } + /// Gets all the MXC URIs in our media database + pub async fn get_all_mxcs(&self) -> Result> { + let all_keys = self.db.get_all_media_keys(); + + let mut mxcs = Vec::with_capacity(all_keys.len()); + + for key in all_keys { + debug!("Full MXC key from database: {key:?}"); + + // we need to get the MXC URL from the first part of the key (the first 0xff / + // 255 push). this is all necessary because of conduit using magic keys for + // media + let mut parts = key.split(|&b| b == 0xFF); + let mxc = parts + .next() + .map(|bytes| { + utils::string_from_bytes(bytes) + .map_err(|e| err!(Database(error!("Failed to parse MXC unicode bytes from our database: {e}")))) + }) + .transpose()?; + + let Some(mxc_s) = mxc else { + return Err!(Database("Parsed MXC URL unicode bytes from database but still is None")); + }; + + debug_info!("Parsed MXC key to URL: {mxc_s}"); + let mxc = OwnedMxcUri::from(mxc_s); + + mxcs.push(mxc); + } + + Ok(mxcs) + } + /// Deletes all remote only media files in the given at or after /// time/duration. Returns a u32 with the amount of media files deleted. pub async fn delete_all_remote_media_at_after_time(&self, time: String, force: bool) -> Result { - let all_keys = self.db.get_all_media_keys(); - let user_duration: SystemTime = match cyborgtime::parse_duration(&time) { Err(e) => return Err!(Database(error!("Failed to parse specified time duration: {e}"))), Ok(duration) => SystemTime::now() @@ -171,7 +203,9 @@ impl Service { .ok_or(err!(Arithmetic("Duration {duration:?} is too large")))?, }; - let mut remote_mxcs: Vec = vec![]; + let all_keys = self.db.get_all_media_keys(); + + let mut remote_mxcs = Vec::with_capacity(all_keys.len()); for key in all_keys { debug!("Full MXC key from database: {key:?}"); From 1157140657d0631be225f5ebee910ebf5086f517 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 25 Aug 2024 17:47:02 -0400 Subject: [PATCH 0704/2091] mark sending ruma matrix version as 1_11 Signed-off-by: strawberry --- src/service/sending/send.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 7feec613..9a8f408b 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -67,7 +67,7 @@ impl super::Service { where T: OutgoingRequest + Debug + Send, { - const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_5]; + const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_11]; const SATIR: SendAccessToken<'_> = SendAccessToken::IfRequired(EMPTY); trace!("Preparing request"); From d5e7befaa971ffccd94f9414c5c0329732d89a9f Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 25 Aug 2024 18:36:34 -0400 Subject: [PATCH 0705/2091] move a few MXC parsing logs to trace Signed-off-by: strawberry --- src/service/media/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index f4a62666..fc86b215 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -166,7 +166,7 @@ impl Service { let mut mxcs = Vec::with_capacity(all_keys.len()); for key in all_keys { - debug!("Full MXC key from database: {key:?}"); + trace!("Full MXC key from database: {key:?}"); // we need to get the MXC URL from the first part of the key (the first 0xff / // 255 push). this is all necessary because of conduit using magic keys for @@ -184,7 +184,7 @@ impl Service { return Err!(Database("Parsed MXC URL unicode bytes from database but still is None")); }; - debug_info!("Parsed MXC key to URL: {mxc_s}"); + trace!("Parsed MXC key to URL: {mxc_s}"); let mxc = OwnedMxcUri::from(mxc_s); mxcs.push(mxc); @@ -208,7 +208,7 @@ impl Service { let mut remote_mxcs = Vec::with_capacity(all_keys.len()); for key in all_keys { - debug!("Full MXC key from database: {key:?}"); + trace!("Full MXC key from database: {key:?}"); // we need to get the MXC URL from the first part of the key (the first 0xff / // 255 push). this is all necessary because of conduit using magic keys for @@ -226,7 +226,7 @@ impl Service { return Err!(Database("Parsed MXC URL unicode bytes from database but still is None")); }; - debug_info!("Parsed MXC key to URL: {mxc_s}"); + trace!("Parsed MXC key to URL: {mxc_s}"); let mxc = OwnedMxcUri::from(mxc_s); if mxc.server_name() == Ok(self.services.globals.server_name()) { debug!("Ignoring local media MXC: {mxc}"); From e4ba34a6385648a589b136edeb8b1f3dde0bf685 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 08:07:38 +0000 Subject: [PATCH 0706/2091] remove zombie globals/client file Signed-off-by: Jason Volk --- src/service/globals/client.rs | 135 ---------------------------------- 1 file changed, 135 deletions(-) delete mode 100644 src/service/globals/client.rs diff --git a/src/service/globals/client.rs b/src/service/globals/client.rs deleted file mode 100644 index d8b84ded..00000000 --- a/src/service/globals/client.rs +++ /dev/null @@ -1,135 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use reqwest::redirect; - -use crate::{resolver, Config, Result}; - -pub struct Client { - pub default: reqwest::Client, - pub url_preview: reqwest::Client, - pub well_known: reqwest::Client, - pub federation: reqwest::Client, - pub sender: reqwest::Client, - pub appservice: reqwest::Client, - pub pusher: reqwest::Client, -} - -impl Client { - pub fn new(config: &Config, resolver: &Arc) -> Self { - Self { - default: Self::base(config) - .unwrap() - .dns_resolver(resolver.clone()) - .build() - .unwrap(), - - url_preview: Self::base(config) - .unwrap() - .dns_resolver(resolver.clone()) - .redirect(redirect::Policy::limited(3)) - .build() - .unwrap(), - - well_known: Self::base(config) - .unwrap() - .dns_resolver(resolver.hooked.clone()) - .connect_timeout(Duration::from_secs(config.well_known_conn_timeout)) - .read_timeout(Duration::from_secs(config.well_known_timeout)) - .timeout(Duration::from_secs(config.well_known_timeout)) - .pool_max_idle_per_host(0) - .redirect(redirect::Policy::limited(4)) - .build() - .unwrap(), - - federation: Self::base(config) - .unwrap() - .dns_resolver(resolver.hooked.clone()) - .read_timeout(Duration::from_secs(config.federation_timeout)) - .timeout(Duration::from_secs(config.federation_timeout)) - .pool_max_idle_per_host(config.federation_idle_per_host.into()) - .pool_idle_timeout(Duration::from_secs(config.federation_idle_timeout)) - .redirect(redirect::Policy::limited(3)) - .build() - .unwrap(), - - sender: Self::base(config) - .unwrap() - .dns_resolver(resolver.hooked.clone()) - .read_timeout(Duration::from_secs(config.sender_timeout)) - .timeout(Duration::from_secs(config.sender_timeout)) - .pool_max_idle_per_host(1) - .pool_idle_timeout(Duration::from_secs(config.sender_idle_timeout)) - .redirect(redirect::Policy::limited(2)) - .build() - .unwrap(), - - appservice: Self::base(config) - .unwrap() - .dns_resolver(resolver.clone()) - .connect_timeout(Duration::from_secs(5)) - .read_timeout(Duration::from_secs(config.appservice_timeout)) - .timeout(Duration::from_secs(config.appservice_timeout)) - .pool_max_idle_per_host(1) - .pool_idle_timeout(Duration::from_secs(config.appservice_idle_timeout)) - .redirect(redirect::Policy::limited(2)) - .build() - .unwrap(), - - pusher: Self::base(config) - .unwrap() - .dns_resolver(resolver.clone()) - .pool_max_idle_per_host(1) - .pool_idle_timeout(Duration::from_secs(config.pusher_idle_timeout)) - .redirect(redirect::Policy::limited(2)) - .build() - .unwrap(), - } - } - - fn base(config: &Config) -> Result { - let mut builder = reqwest::Client::builder() - .hickory_dns(true) - .connect_timeout(Duration::from_secs(config.request_conn_timeout)) - .read_timeout(Duration::from_secs(config.request_timeout)) - .timeout(Duration::from_secs(config.request_total_timeout)) - .pool_idle_timeout(Duration::from_secs(config.request_idle_timeout)) - .pool_max_idle_per_host(config.request_idle_per_host.into()) - .user_agent(conduit::version::user_agent()) - .redirect(redirect::Policy::limited(6)) - .connection_verbose(true); - - #[cfg(feature = "gzip_compression")] - { - builder = if config.gzip_compression { - builder.gzip(true) - } else { - builder.gzip(false).no_gzip() - }; - }; - - #[cfg(feature = "brotli_compression")] - { - builder = if config.brotli_compression { - builder.brotli(true) - } else { - builder.brotli(false).no_brotli() - }; - }; - - #[cfg(not(feature = "gzip_compression"))] - { - builder = builder.no_gzip(); - }; - - #[cfg(not(feature = "brotli_compression"))] - { - builder = builder.no_brotli(); - }; - - if let Some(proxy) = config.proxy.to_proxy()? { - Ok(builder.proxy(proxy)) - } else { - Ok(builder) - } - } -} From e4bcbb80885d6f756b0f7167e570e057b8214f84 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 08:10:04 +0000 Subject: [PATCH 0707/2091] propagate errors from reqwest client constructions Signed-off-by: Jason Volk --- src/service/client/mod.rs | 42 +++++++++++++-------------------------- 1 file changed, 14 insertions(+), 28 deletions(-) diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index 63f746db..7b653939 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -21,54 +21,43 @@ impl crate::Service for Service { let resolver = args.require::("resolver"); Ok(Arc::new(Self { - default: base(config) - .unwrap() + default: base(config)? .dns_resolver(resolver.resolver.clone()) - .build() - .unwrap(), + .build()?, - url_preview: base(config) - .unwrap() + url_preview: base(config)? .dns_resolver(resolver.resolver.clone()) .redirect(redirect::Policy::limited(3)) - .build() - .unwrap(), + .build()?, - well_known: base(config) - .unwrap() + well_known: base(config)? .dns_resolver(resolver.resolver.hooked.clone()) .connect_timeout(Duration::from_secs(config.well_known_conn_timeout)) .read_timeout(Duration::from_secs(config.well_known_timeout)) .timeout(Duration::from_secs(config.well_known_timeout)) .pool_max_idle_per_host(0) .redirect(redirect::Policy::limited(4)) - .build() - .unwrap(), + .build()?, - federation: base(config) - .unwrap() + federation: base(config)? .dns_resolver(resolver.resolver.hooked.clone()) .read_timeout(Duration::from_secs(config.federation_timeout)) .timeout(Duration::from_secs(config.federation_timeout)) .pool_max_idle_per_host(config.federation_idle_per_host.into()) .pool_idle_timeout(Duration::from_secs(config.federation_idle_timeout)) .redirect(redirect::Policy::limited(3)) - .build() - .unwrap(), + .build()?, - sender: base(config) - .unwrap() + sender: base(config)? .dns_resolver(resolver.resolver.hooked.clone()) .read_timeout(Duration::from_secs(config.sender_timeout)) .timeout(Duration::from_secs(config.sender_timeout)) .pool_max_idle_per_host(1) .pool_idle_timeout(Duration::from_secs(config.sender_idle_timeout)) .redirect(redirect::Policy::limited(2)) - .build() - .unwrap(), + .build()?, - appservice: base(config) - .unwrap() + appservice: base(config)? .dns_resolver(resolver.resolver.clone()) .connect_timeout(Duration::from_secs(5)) .read_timeout(Duration::from_secs(config.appservice_timeout)) @@ -76,17 +65,14 @@ impl crate::Service for Service { .pool_max_idle_per_host(1) .pool_idle_timeout(Duration::from_secs(config.appservice_idle_timeout)) .redirect(redirect::Policy::limited(2)) - .build() - .unwrap(), + .build()?, - pusher: base(config) - .unwrap() + pusher: base(config)? .dns_resolver(resolver.resolver.clone()) .pool_max_idle_per_host(1) .pool_idle_timeout(Duration::from_secs(config.pusher_idle_timeout)) .redirect(redirect::Policy::limited(2)) - .build() - .unwrap(), + .build()?, })) } From ffc41cb01f1009011074b947109c9520018e72d9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 26 Aug 2024 21:23:06 +0000 Subject: [PATCH 0708/2091] split admin service startup related into unit Signed-off-by: Jason Volk --- src/service/admin/mod.rs | 55 ++------------------------------ src/service/admin/startup.rs | 61 ++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 52 deletions(-) create mode 100644 src/service/admin/startup.rs diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 2719c142..21ded34d 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -1,6 +1,7 @@ pub mod console; mod create; mod grant; +mod startup; use std::{ future::Future, @@ -9,7 +10,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{debug, error, error::default_log, info, pdu::PduBuilder, Err, Error, PduEvent, Result, Server}; +use conduit::{debug, error, error::default_log, pdu::PduBuilder, Err, Error, PduEvent, Result, Server}; pub use create::create_admin_room; use loole::{Receiver, Sender}; use ruma::{ @@ -20,10 +21,7 @@ use ruma::{ OwnedEventId, OwnedRoomId, RoomId, UserId, }; use serde_json::value::to_raw_value; -use tokio::{ - sync::{Mutex, RwLock}, - time::{sleep, Duration}, -}; +use tokio::sync::{Mutex, RwLock}; use crate::{globals, rooms, rooms::state::RoomMutexGuard, Dep}; @@ -361,53 +359,6 @@ impl Service { } } - /// Execute admin commands after startup - async fn startup_execute(&self) { - sleep(Duration::from_millis(500)).await; //TODO: remove this after run-states are broadcast - for (i, command) in self.services.server.config.admin_execute.iter().enumerate() { - self.startup_execute_command(i, command.clone()).await; - tokio::task::yield_now().await; - } - } - - /// Execute one admin command after startup - async fn startup_execute_command(&self, i: usize, command: String) { - debug!("Startup command #{i}: executing {command:?}"); - - match self.command_in_place(command, None).await { - Err(e) => error!("Startup command #{i} failed: {e:?}"), - Ok(None) => info!("Startup command #{i} completed (no output)."), - Ok(Some(output)) => Self::startup_command_output(i, &output), - } - } - - #[cfg(feature = "console")] - fn startup_command_output(i: usize, content: &RoomMessageEventContent) { - info!("Startup command #{i} completed:"); - console::print(content.body()); - } - - #[cfg(not(feature = "console"))] - fn startup_command_output(i: usize, content: &RoomMessageEventContent) { - info!("Startup command #{i} completed:\n{:#?}", content.body()); - } - - /// Possibly spawn the terminal console at startup if configured. - async fn console_auto_start(&self) { - #[cfg(feature = "console")] - if self.services.server.config.admin_console_automatic { - // Allow more of the startup sequence to execute before spawning - tokio::task::yield_now().await; - self.console.start().await; - } - } - - /// Shutdown the console when the admin worker terminates. - async fn console_auto_stop(&self) { - #[cfg(feature = "console")] - self.console.close().await; - } - /// Sets the self-reference to crate::Services which will provide context to /// the admin commands. pub(super) fn set_services(&self, services: &Option>) { diff --git a/src/service/admin/startup.rs b/src/service/admin/startup.rs new file mode 100644 index 00000000..79dc70d0 --- /dev/null +++ b/src/service/admin/startup.rs @@ -0,0 +1,61 @@ + +use conduit::{ + debug, error, implement, info, +}; +use ruma::events::room::message::RoomMessageEventContent; +use tokio::time::{sleep, Duration}; + +use super::console; + +/// Possibly spawn the terminal console at startup if configured. +#[implement(super::Service)] +pub(super) async fn console_auto_start(&self) { + #[cfg(feature = "console")] + if self.services.server.config.admin_console_automatic { + // Allow more of the startup sequence to execute before spawning + tokio::task::yield_now().await; + self.console.start().await; + } +} + +/// Shutdown the console when the admin worker terminates. +#[implement(super::Service)] +pub(super) async fn console_auto_stop(&self) { + #[cfg(feature = "console")] + self.console.close().await; +} + +/// Execute admin commands after startup +#[implement(super::Service)] +pub(super) async fn startup_execute(&self) { + sleep(Duration::from_millis(500)).await; //TODO: remove this after run-states are broadcast + for (i, command) in self.services.server.config.admin_execute.iter().enumerate() { + self.startup_execute_command(i, command.clone()).await; + tokio::task::yield_now().await; + } +} + +/// Execute one admin command after startup +#[implement(super::Service)] +async fn startup_execute_command(&self, i: usize, command: String) { + debug!("Startup command #{i}: executing {command:?}"); + + match self.command_in_place(command, None).await { + Err(e) => error!("Startup command #{i} failed: {e:?}"), + Ok(None) => info!("Startup command #{i} completed (no output)."), + Ok(Some(output)) => Self::startup_command_output(i, &output), + } +} + +#[cfg(feature = "console")] +#[implement(super::Service)] +fn startup_command_output(i: usize, content: &RoomMessageEventContent) { + info!("Startup command #{i} completed:"); + console::print(content.body()); +} + +#[cfg(not(feature = "console"))] +#[implement(super::Service)] +fn startup_command_output(i: usize, content: &RoomMessageEventContent) { + info!("Startup command #{i} completed:\n{:#?}", content.body()); +} From fcb9d04d9e5039171b1758eab470bee482bd785a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 26 Aug 2024 21:29:00 +0000 Subject: [PATCH 0709/2091] add program argument for functional testing; simplify execute argument Signed-off-by: Jason Volk --- src/core/config/mod.rs | 5 ++++- src/main/clap.rs | 11 ++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index b7708760..6acb814a 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, BTreeSet}, fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, path::PathBuf, @@ -361,6 +361,9 @@ pub struct Config { #[serde(default)] pub tokio_console: bool, + #[serde(default)] + pub test: BTreeSet, + #[serde(flatten)] #[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime catchall: BTreeMap, diff --git a/src/main/clap.rs b/src/main/clap.rs index 56d614a4..3af0be02 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -28,6 +28,10 @@ pub(crate) struct Args { /// Execute console command automatically after startup. #[arg(long)] pub(crate) execute: Vec, + + /// Set functional testing modes if available. Ex '--test=smoke' + #[arg(long, hide(true))] + pub(crate) test: Vec, } /// Parse commandline arguments into structured data @@ -44,9 +48,10 @@ pub(crate) fn update(mut config: Figment, args: &Args) -> Result { } // Execute commands after any commands listed in configuration file - for command in &args.execute { - config = config.adjoin(("admin_execute", [command])); - } + config = config.adjoin(("admin_execute", &args.execute)); + + // Update config with names of any functional-tests + config = config.adjoin(("test", &args.test)); // All other individual overrides can go last in case we have options which // set multiple conf items at once and the user still needs granular overrides. From f4db6292b355c4fa75d8a6b8be0e2bf599b58c37 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 26 Aug 2024 21:30:37 +0000 Subject: [PATCH 0710/2091] add smoketest-assist execution mode Signed-off-by: Jason Volk --- src/service/admin/startup.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/service/admin/startup.rs b/src/service/admin/startup.rs index 79dc70d0..8a4a3d63 100644 --- a/src/service/admin/startup.rs +++ b/src/service/admin/startup.rs @@ -1,7 +1,4 @@ - -use conduit::{ - debug, error, implement, info, -}; +use conduit::{debug, debug_info, error, implement, info}; use ruma::events::room::message::RoomMessageEventContent; use tokio::time::{sleep, Duration}; @@ -33,6 +30,16 @@ pub(super) async fn startup_execute(&self) { self.startup_execute_command(i, command.clone()).await; tokio::task::yield_now().await; } + + // The smoketest functionality is placed here for now and simply initiates + // shutdown after all commands have executed. + if self.services.server.config.test.contains("smoke") { + debug_info!("Smoketest mode. All commands complete. Shutting down now..."); + self.services + .server + .shutdown() + .unwrap_or_else(error::default_log); + } } /// Execute one admin command after startup From f047675a63b2f0d15ee538a934b7fdc907e71d91 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 28 Aug 2024 01:33:58 +0000 Subject: [PATCH 0711/2091] rename admin/handler to admin/processor Handler is overused. Handler ought to mean the end-function handling the command. The command processor is the central dispatcher to the handler. Signed-off-by: Jason Volk --- src/admin/mod.rs | 12 ++++++++---- src/admin/{handler.rs => processor.rs} | 8 ++++---- src/service/admin/mod.rs | 14 ++++++-------- 3 files changed, 18 insertions(+), 16 deletions(-) rename src/admin/{handler.rs => processor.rs} (96%) diff --git a/src/admin/mod.rs b/src/admin/mod.rs index fb1c02be..83db18fa 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -4,7 +4,7 @@ pub(crate) mod admin; pub(crate) mod command; -pub(crate) mod handler; +pub(crate) mod processor; mod tests; pub(crate) mod utils; @@ -36,14 +36,18 @@ conduit::mod_ctor! {} conduit::mod_dtor! {} conduit::rustc_flags_capture! {} -/// Install the admin command handler +/// Install the admin command processor pub async fn init(admin_service: &service::admin::Service) { _ = admin_service .complete .write() .expect("locked for writing") - .insert(handler::complete); - _ = admin_service.handle.write().await.insert(handler::handle); + .insert(processor::complete); + _ = admin_service + .handle + .write() + .await + .insert(processor::dispatch); } /// Uninstall the admin command handler diff --git a/src/admin/handler.rs b/src/admin/processor.rs similarity index 96% rename from src/admin/handler.rs rename to src/admin/processor.rs index 631d17e1..2e4330c8 100644 --- a/src/admin/handler.rs +++ b/src/admin/processor.rs @@ -26,7 +26,7 @@ use ruma::{ OwnedEventId, }; use service::{ - admin::{CommandInput, CommandOutput, HandlerFuture, HandlerResult}, + admin::{CommandInput, CommandOutput, ProcessorFuture, ProcessorResult}, Services, }; use tracing::Level; @@ -38,12 +38,12 @@ use crate::{admin, admin::AdminCommand, Command}; pub(super) fn complete(line: &str) -> String { complete_command(AdminCommand::command(), line) } #[must_use] -pub(super) fn handle(services: Arc, command: CommandInput) -> HandlerFuture { +pub(super) fn dispatch(services: Arc, command: CommandInput) -> ProcessorFuture { Box::pin(handle_command(services, command)) } #[tracing::instrument(skip_all, name = "admin")] -async fn handle_command(services: Arc, command: CommandInput) -> HandlerResult { +async fn handle_command(services: Arc, command: CommandInput) -> ProcessorResult { AssertUnwindSafe(Box::pin(process_command(services, &command))) .catch_unwind() .await @@ -68,7 +68,7 @@ async fn process_command(services: Arc, input: &CommandInput) -> Comma .and_then(|content| reply(content, input.reply_id.clone())) } -fn handle_panic(error: &Error, command: CommandInput) -> HandlerResult { +fn handle_panic(error: &Error, command: CommandInput) -> ProcessorResult { let link = "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}"); let content = RoomMessageEventContent::notice_markdown(msg); diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 21ded34d..9e27cdf2 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -29,7 +29,7 @@ pub struct Service { services: Services, sender: Sender, receiver: Mutex>, - pub handle: RwLock>, + pub handle: RwLock>, pub complete: StdRwLock>, #[cfg(feature = "console")] pub console: Arc, @@ -52,9 +52,9 @@ pub struct CommandInput { } pub type Completer = fn(&str) -> String; -pub type Handler = fn(Arc, CommandInput) -> HandlerFuture; -pub type HandlerFuture = Pin + Send>>; -pub type HandlerResult = Result; +pub type Processor = fn(Arc, CommandInput) -> ProcessorFuture; +pub type ProcessorFuture = Pin + Send>>; +pub type ProcessorResult = Result; pub type CommandOutput = Option; const COMMAND_QUEUE_LIMIT: usize = 512; @@ -141,9 +141,7 @@ impl Service { .await; } - pub async fn command_in_place( - &self, command: String, reply_id: Option, - ) -> Result> { + pub async fn command_in_place(&self, command: String, reply_id: Option) -> ProcessorResult { self.process_command(CommandInput { command, reply_id, @@ -176,7 +174,7 @@ impl Service { } } - async fn process_command(&self, command: CommandInput) -> HandlerResult { + async fn process_command(&self, command: CommandInput) -> ProcessorResult { let Some(services) = self .services .services From bb5f2556c30782c8377388d5a83d0531bda9b199 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 28 Aug 2024 04:09:46 +0000 Subject: [PATCH 0712/2091] improve admin command error propagation Signed-off-by: Jason Volk --- src/admin/command.rs | 2 + src/admin/debug/mod.rs | 1 + src/admin/debug/tester.rs | 18 ++++++- src/admin/processor.rs | 58 ++++++++++++---------- src/core/config/mod.rs | 6 +++ src/service/admin/console.rs | 33 +++++++++++-- src/service/admin/mod.rs | 82 ++++++++++++++++++++----------- src/service/admin/startup.rs | 67 +++++++++++++++++++------ src/service/rooms/timeline/mod.rs | 3 +- 9 files changed, 192 insertions(+), 78 deletions(-) diff --git a/src/admin/command.rs b/src/admin/command.rs index a26564fc..c594736d 100644 --- a/src/admin/command.rs +++ b/src/admin/command.rs @@ -1,9 +1,11 @@ use std::time::SystemTime; use conduit_service::Services; +use ruma::EventId; pub(crate) struct Command<'a> { pub(crate) services: &'a Services, pub(crate) body: &'a [&'a str], pub(crate) timer: SystemTime, + pub(crate) reply_id: Option<&'a EventId>, } diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 1f51a35e..20ddbf2f 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -195,5 +195,6 @@ pub(super) enum DebugCommand { /// - Developer test stubs #[command(subcommand)] #[allow(non_snake_case)] + #[clap(hide(true))] Tester(TesterCommand), } diff --git a/src/admin/debug/tester.rs b/src/admin/debug/tester.rs index af4ea2dc..c11f893e 100644 --- a/src/admin/debug/tester.rs +++ b/src/admin/debug/tester.rs @@ -1,3 +1,4 @@ +use conduit::Err; use ruma::events::room::message::RoomMessageEventContent; use crate::{admin_command, admin_command_dispatch, Result}; @@ -5,13 +6,28 @@ use crate::{admin_command, admin_command_dispatch, Result}; #[admin_command_dispatch] #[derive(Debug, clap::Subcommand)] pub(crate) enum TesterCommand { + Panic, + Failure, Tester, Timer, } +#[rustfmt::skip] +#[admin_command] +async fn panic(&self) -> Result { + + panic!("panicked") +} + +#[rustfmt::skip] +#[admin_command] +async fn failure(&self) -> Result { + + Err!("failed") +} + #[inline(never)] #[rustfmt::skip] -#[allow(unused_variables)] #[admin_command] async fn tester(&self) -> Result { diff --git a/src/admin/processor.rs b/src/admin/processor.rs index 2e4330c8..67548f49 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -23,7 +23,7 @@ use ruma::{ relation::InReplyTo, room::message::{Relation::Reply, RoomMessageEventContent}, }, - OwnedEventId, + EventId, }; use service::{ admin::{CommandInput, CommandOutput, ProcessorFuture, ProcessorResult}, @@ -48,12 +48,12 @@ async fn handle_command(services: Arc, command: CommandInput) -> Proce .catch_unwind() .await .map_err(Error::from_panic) - .or_else(|error| handle_panic(&error, command)) + .unwrap_or_else(|error| handle_panic(&error, &command)) } -async fn process_command(services: Arc, input: &CommandInput) -> CommandOutput { +async fn process_command(services: Arc, input: &CommandInput) -> ProcessorResult { let (command, args, body) = match parse(&services, input) { - Err(error) => return error, + Err(error) => return Err(error), Ok(parsed) => parsed, }; @@ -61,33 +61,22 @@ async fn process_command(services: Arc, input: &CommandInput) -> Comma services: &services, body: &body, timer: SystemTime::now(), + reply_id: input.reply_id.as_deref(), }; - process(&context, command, &args) - .await - .and_then(|content| reply(content, input.reply_id.clone())) + process(&context, command, &args).await } -fn handle_panic(error: &Error, command: CommandInput) -> ProcessorResult { +fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult { let link = "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}"); let content = RoomMessageEventContent::notice_markdown(msg); error!("Panic while processing command: {error:?}"); - Ok(reply(content, command.reply_id)) -} - -fn reply(mut content: RoomMessageEventContent, reply_id: Option) -> Option { - content.relates_to = reply_id.map(|event_id| Reply { - in_reply_to: InReplyTo { - event_id, - }, - }); - - Some(content) + Err(reply(content, command.reply_id.as_deref())) } // Parse and process a message from the admin room -async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) -> CommandOutput { +async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) -> ProcessorResult { let (capture, logs) = capture_create(context); let capture_scope = capture.start(); @@ -112,13 +101,15 @@ async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) match result { Ok(content) => { - write!(&mut output, "{}", content.body()).expect("failed to format command result to output"); + write!(&mut output, "{0}", content.body()).expect("failed to format command result to output buffer"); + Ok(Some(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id))) }, - Err(error) => write!(&mut output, "Command failed with error:\n```\n{error:#?}\n```") - .expect("failed to format error to command output"), - }; - - Some(RoomMessageEventContent::notice_markdown(output)) + Err(error) => { + write!(&mut output, "Command failed with error:\n```\n{error:#?}\n```") + .expect("failed to format command result to output"); + Err(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id)) + }, + } } fn capture_create(context: &Command<'_>) -> (Arc, Arc>) { @@ -158,7 +149,10 @@ fn parse<'a>( let message = error .to_string() .replace("server.name", services.globals.server_name().as_str()); - Err(Some(RoomMessageEventContent::notice_markdown(message))) + Err(reply( + RoomMessageEventContent::notice_markdown(message), + input.reply_id.as_deref(), + )) }, } } @@ -255,3 +249,13 @@ fn parse_line(command_line: &str) -> Vec { trace!(?command_line, ?argv, "parse"); argv } + +fn reply(mut content: RoomMessageEventContent, reply_id: Option<&EventId>) -> RoomMessageEventContent { + content.relates_to = reply_id.map(|event_id| Reply { + in_reply_to: InReplyTo { + event_id: event_id.to_owned(), + }, + }); + + content +} diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 6acb814a..1f78dea6 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -338,6 +338,8 @@ pub struct Config { pub admin_console_automatic: bool, #[serde(default)] pub admin_execute: Vec, + #[serde(default)] + pub admin_execute_errors_ignore: bool, #[serde(default = "default_admin_log_capture")] pub admin_log_capture: String, @@ -601,6 +603,10 @@ impl fmt::Display for Config { &self.admin_console_automatic.to_string(), ); line("Execute admin commands after startup", &self.admin_execute.join(", ")); + line( + "Continue startup even if some commands fail", + &self.admin_execute_errors_ignore.to_string(), + ); line("Filter for admin command log capture", &self.admin_log_capture); line("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()); line("Allow incoming federated typing", &self.allow_incoming_typing.to_string()); diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 7f3a6fae..55bae365 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -158,13 +158,18 @@ impl Console { async fn process(self: Arc, line: String) { match self.admin.command_in_place(line, None).await { - Ok(Some(content)) => self.output(content).await, - Err(e) => error!("processing command: {e}"), - _ => (), + Ok(Some(ref content)) => self.output(content), + Err(ref content) => self.output_err(content), + _ => unreachable!(), } } - async fn output(self: Arc, output_content: RoomMessageEventContent) { + fn output_err(self: Arc, output_content: &RoomMessageEventContent) { + let output = configure_output_err(self.output.clone()); + output.print_text(output_content.body()); + } + + fn output(self: Arc, output_content: &RoomMessageEventContent) { self.output.print_text(output_content.body()); } @@ -194,12 +199,32 @@ impl Console { } } +/// Standalone/static markdown printer for errors. +pub fn print_err(markdown: &str) { + let output = configure_output_err(MadSkin::default_dark()); + output.print_text(markdown); +} /// Standalone/static markdown printer. pub fn print(markdown: &str) { let output = configure_output(MadSkin::default_dark()); output.print_text(markdown); } +fn configure_output_err(mut output: MadSkin) -> MadSkin { + use termimad::{crossterm::style::Color, Alignment, CompoundStyle, LineStyle}; + + let code_style = CompoundStyle::with_fgbg(Color::AnsiValue(196), Color::AnsiValue(234)); + output.inline_code = code_style.clone(); + output.code_block = LineStyle { + left_margin: 0, + right_margin: 0, + align: Alignment::Left, + compound_style: code_style, + }; + + output +} + fn configure_output(mut output: MadSkin) -> MadSkin { use termimad::{crossterm::style::Color, Alignment, CompoundStyle, LineStyle}; diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 9e27cdf2..bd89d5a3 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -10,7 +10,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{debug, error, error::default_log, pdu::PduBuilder, Err, Error, PduEvent, Result, Server}; +use conduit::{debug, err, error, error::default_log, pdu::PduBuilder, Error, PduEvent, Result, Server}; pub use create::create_admin_room; use loole::{Receiver, Sender}; use ruma::{ @@ -45,18 +45,34 @@ struct Services { services: StdRwLock>>, } +/// Inputs to a command are a multi-line string and optional reply_id. #[derive(Debug)] pub struct CommandInput { pub command: String, pub reply_id: Option, } +/// Prototype of the tab-completer. The input is buffered text when tab +/// asserted; the output will fully replace the input buffer. pub type Completer = fn(&str) -> String; -pub type Processor = fn(Arc, CommandInput) -> ProcessorFuture; -pub type ProcessorFuture = Pin + Send>>; -pub type ProcessorResult = Result; -pub type CommandOutput = Option; +/// Prototype of the command processor. This is a callback supplied by the +/// reloadable admin module. +pub type Processor = fn(Arc, CommandInput) -> ProcessorFuture; + +/// Return type of the processor +pub type ProcessorFuture = Pin + Send>>; + +/// Result wrapping of a command's handling. Both variants are complete message +/// events which have digested any prior errors. The wrapping preserves whether +/// the command failed without interpreting the text. Ok(None) outputs are +/// dropped to produce no response. +pub type ProcessorResult = Result, CommandOutput>; + +/// Alias for the output structure. +pub type CommandOutput = RoomMessageEventContent; + +/// Maximum number of commands which can be queued for dispatch. const COMMAND_QUEUE_LIMIT: usize = 512; #[async_trait] @@ -86,7 +102,7 @@ impl crate::Service for Service { let receiver = self.receiver.lock().await; let mut signals = self.services.server.signal.subscribe(); - self.startup_execute().await; + self.startup_execute().await?; self.console_auto_start().await; loop { @@ -120,11 +136,15 @@ impl crate::Service for Service { } impl Service { + /// Sends markdown message (not an m.notice for notification reasons) to the + /// admin room as the admin user. pub async fn send_text(&self, body: &str) { self.send_message(RoomMessageEventContent::text_markdown(body)) .await; } + /// Sends a message to the admin room as the admin user (see send_text() for + /// convenience). pub async fn send_message(&self, message_content: RoomMessageEventContent) { if let Ok(Some(room_id)) = self.get_admin_room() { let user_id = &self.services.globals.server_user; @@ -133,14 +153,20 @@ impl Service { } } - pub async fn command(&self, command: String, reply_id: Option) { - self.send(CommandInput { - command, - reply_id, - }) - .await; + /// Posts a command to the command processor queue and returns. Processing + /// will take place on the service worker's task asynchronously. Errors if + /// the queue is full. + pub fn command(&self, command: String, reply_id: Option) -> Result<()> { + self.sender + .send(CommandInput { + command, + reply_id, + }) + .map_err(|e| err!("Failed to enqueue admin command: {e:?}")) } + /// Dispatches a comamnd to the processor on the current task and waits for + /// completion. pub async fn command_in_place(&self, command: String, reply_id: Option) -> ProcessorResult { self.process_command(CommandInput { command, @@ -149,6 +175,8 @@ impl Service { .await } + /// Invokes the tab-completer to complete the command. When unavailable, + /// None is returned. pub fn complete_command(&self, command: &str) -> Option { self.complete .read() @@ -156,11 +184,6 @@ impl Service { .map(|complete| complete(command)) } - async fn send(&self, message: CommandInput) { - debug_assert!(!self.sender.is_closed(), "channel closed"); - self.sender.send_async(message).await.expect("message sent"); - } - async fn handle_signal(&self, #[allow(unused_variables)] sig: &'static str) { #[cfg(feature = "console")] self.console.handle_signal(sig).await; @@ -168,29 +191,28 @@ impl Service { async fn handle_command(&self, command: CommandInput) { match self.process_command(command).await { - Ok(Some(output)) => self.handle_response(output).await, + Ok(Some(output)) | Err(output) => self.handle_response(output).await, Ok(None) => debug!("Command successful with no response"), - Err(e) => error!("Command processing error: {e}"), } } async fn process_command(&self, command: CommandInput) -> ProcessorResult { - let Some(services) = self + let handle = &self + .handle + .read() + .await + .expect("Admin module is not loaded"); + + let services = self .services .services .read() .expect("locked") .as_ref() .and_then(Weak::upgrade) - else { - return Err!("Services self-reference not initialized."); - }; + .expect("Services self-reference not initialized."); - if let Some(handle) = self.handle.read().await.as_ref() { - handle(services, command).await - } else { - Err!("Admin module is not loaded.") - } + handle(services, command).await } /// Checks whether a given user is an admin of this server @@ -233,6 +255,10 @@ impl Service { }; let Ok(Some(pdu)) = self.services.timeline.get_pdu(&in_reply_to.event_id) else { + error!( + event_id = ?in_reply_to.event_id, + "Missing admin command in_reply_to event" + ); return; }; diff --git a/src/service/admin/startup.rs b/src/service/admin/startup.rs index 8a4a3d63..b7bc8d44 100644 --- a/src/service/admin/startup.rs +++ b/src/service/admin/startup.rs @@ -1,9 +1,7 @@ -use conduit::{debug, debug_info, error, implement, info}; +use conduit::{debug, debug_info, error, implement, info, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; use tokio::time::{sleep, Duration}; -use super::console; - /// Possibly spawn the terminal console at startup if configured. #[implement(super::Service)] pub(super) async fn console_auto_start(&self) { @@ -24,45 +22,82 @@ pub(super) async fn console_auto_stop(&self) { /// Execute admin commands after startup #[implement(super::Service)] -pub(super) async fn startup_execute(&self) { - sleep(Duration::from_millis(500)).await; //TODO: remove this after run-states are broadcast - for (i, command) in self.services.server.config.admin_execute.iter().enumerate() { - self.startup_execute_command(i, command.clone()).await; +pub(super) async fn startup_execute(&self) -> Result<()> { + // List of comamnds to execute + let commands = &self.services.server.config.admin_execute; + + // Determine if we're running in smoketest-mode which will change some behaviors + let smoketest = self.services.server.config.test.contains("smoke"); + + // When true, errors are ignored and startup continues. + let errors = !smoketest && self.services.server.config.admin_execute_errors_ignore; + + //TODO: remove this after run-states are broadcast + sleep(Duration::from_millis(500)).await; + + for (i, command) in commands.iter().enumerate() { + if let Err(e) = self.startup_execute_command(i, command.clone()).await { + if !errors { + return Err(e); + } + } + tokio::task::yield_now().await; } // The smoketest functionality is placed here for now and simply initiates // shutdown after all commands have executed. - if self.services.server.config.test.contains("smoke") { + if smoketest { debug_info!("Smoketest mode. All commands complete. Shutting down now..."); self.services .server .shutdown() - .unwrap_or_else(error::default_log); + .inspect_err(error::inspect_log) + .expect("Error shutting down from smoketest"); } + + Ok(()) } /// Execute one admin command after startup #[implement(super::Service)] -async fn startup_execute_command(&self, i: usize, command: String) { +async fn startup_execute_command(&self, i: usize, command: String) -> Result<()> { debug!("Startup command #{i}: executing {command:?}"); match self.command_in_place(command, None).await { - Err(e) => error!("Startup command #{i} failed: {e:?}"), - Ok(None) => info!("Startup command #{i} completed (no output)."), Ok(Some(output)) => Self::startup_command_output(i, &output), + Err(output) => Self::startup_command_error(i, &output), + Ok(None) => { + info!("Startup command #{i} completed (no output)."); + Ok(()) + }, } } #[cfg(feature = "console")] #[implement(super::Service)] -fn startup_command_output(i: usize, content: &RoomMessageEventContent) { - info!("Startup command #{i} completed:"); - console::print(content.body()); +fn startup_command_output(i: usize, content: &RoomMessageEventContent) -> Result<()> { + debug_info!("Startup command #{i} completed:"); + super::console::print(content.body()); + Ok(()) +} + +#[cfg(feature = "console")] +#[implement(super::Service)] +fn startup_command_error(i: usize, content: &RoomMessageEventContent) -> Result<()> { + super::console::print_err(content.body()); + Err!(debug_error!("Startup command #{i} failed.")) } #[cfg(not(feature = "console"))] #[implement(super::Service)] -fn startup_command_output(i: usize, content: &RoomMessageEventContent) { +fn startup_command_output(i: usize, content: &RoomMessageEventContent) -> Result<()> { info!("Startup command #{i} completed:\n{:#?}", content.body()); + Ok(()) +} + +#[cfg(not(feature = "console"))] +#[implement(super::Service)] +fn startup_command_error(i: usize, content: &RoomMessageEventContent) -> Result<()> { + Err!(error!("Startup command #{i} failed:\n{:#?}", content.body())) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 983bef70..4f2352f8 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -523,8 +523,7 @@ impl Service { if self.services.admin.is_admin_command(pdu, &body).await { self.services .admin - .command(body, Some((*pdu.event_id).into())) - .await; + .command(body, Some((*pdu.event_id).into()))?; } } }, From e7b2c1428011e0f32353bbd42b8765b21511abd3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 28 Aug 2024 07:05:13 +0000 Subject: [PATCH 0713/2091] tag admin room m.server_notice on grant Signed-off-by: Jason Volk --- src/core/config/mod.rs | 5 +++++ src/service/admin/grant.rs | 44 +++++++++++++++++++++++++++++++++++--- src/service/admin/mod.rs | 4 +++- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 1f78dea6..12b36a65 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -342,6 +342,8 @@ pub struct Config { pub admin_execute_errors_ignore: bool, #[serde(default = "default_admin_log_capture")] pub admin_log_capture: String, + #[serde(default = "default_admin_room_tag")] + pub admin_room_tag: String, #[serde(default)] pub sentry: bool, @@ -608,6 +610,7 @@ impl fmt::Display for Config { &self.admin_execute_errors_ignore.to_string(), ); line("Filter for admin command log capture", &self.admin_log_capture); + line("Admin room tag", &self.admin_room_tag); line("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()); line("Allow incoming federated typing", &self.allow_incoming_typing.to_string()); line( @@ -1068,3 +1071,5 @@ fn default_sentry_filter() -> String { "info".to_owned() } fn default_startup_netburst_keep() -> i64 { 50 } fn default_admin_log_capture() -> String { "debug".to_owned() } + +fn default_admin_room_tag() -> String { "m.server_notice".to_owned() } diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 44e11a3e..8cbee604 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use conduit::Result; +use conduit::{error, implement, Result}; use ruma::{ events::{ room::{ @@ -8,9 +8,10 @@ use ruma::{ message::RoomMessageEventContent, power_levels::RoomPowerLevelsEventContent, }, - TimelineEventType, + tag::{TagEvent, TagEventContent, TagInfo}, + RoomAccountDataEventType, TimelineEventType, }, - UserId, + RoomId, UserId, }; use serde_json::value::to_raw_value; @@ -110,6 +111,14 @@ impl super::Service { ) .await?; + // Set room tag + let room_tag = &self.services.server.config.admin_room_tag; + if !room_tag.is_empty() { + if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag) { + error!(?room_id, ?user_id, ?room_tag, ?e, "Failed to set tag for admin grant"); + } + } + // Send welcome message self.services.timeline.build_and_append_pdu( PduBuilder { @@ -132,3 +141,32 @@ impl super::Service { Ok(()) } } + +#[implement(super::Service)] +fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result<()> { + let mut event = self + .services + .account_data + .get(Some(room_id), user_id, RoomAccountDataEventType::Tag)? + .map(|event| serde_json::from_str(event.get())) + .and_then(Result::ok) + .unwrap_or_else(|| TagEvent { + content: TagEventContent { + tags: BTreeMap::new(), + }, + }); + + event + .content + .tags + .insert(tag.to_owned().into(), TagInfo::new()); + + self.services.account_data.update( + Some(room_id), + user_id, + RoomAccountDataEventType::Tag, + &serde_json::to_value(event)?, + )?; + + Ok(()) +} diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index bd89d5a3..3274249e 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -23,7 +23,7 @@ use ruma::{ use serde_json::value::to_raw_value; use tokio::sync::{Mutex, RwLock}; -use crate::{globals, rooms, rooms::state::RoomMutexGuard, Dep}; +use crate::{account_data, globals, rooms, rooms::state::RoomMutexGuard, Dep}; pub struct Service { services: Services, @@ -42,6 +42,7 @@ struct Services { timeline: Dep, state: Dep, state_cache: Dep, + account_data: Dep, services: StdRwLock>>, } @@ -87,6 +88,7 @@ impl crate::Service for Service { timeline: args.depend::("rooms::timeline"), state: args.depend::("rooms::state"), state_cache: args.depend::("rooms::state_cache"), + account_data: args.depend::("account_data"), services: None.into(), }, sender, From 7b0e830f4c65558ca7ed0b45dcb4a39637761781 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 26 Aug 2024 22:15:42 +0000 Subject: [PATCH 0714/2091] add interface to get file metadata w/ admin command Signed-off-by: Jason Volk --- src/admin/media/commands.rs | 8 ++++++++ src/admin/media/mod.rs | 7 ++++++- src/service/media/mod.rs | 14 +++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 476f048b..3894d5df 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -257,3 +257,11 @@ pub(super) async fn delete_all_from_server( "Deleted {deleted_count} total files.", ))) } + +#[admin_command] +pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result { + let mxc: Mxc<'_> = mxc.as_str().try_into()?; + let metadata = self.services.media.get_metadata(&mxc); + + Ok(RoomMessageEventContent::notice_markdown(format!("```\n{metadata:#?}\n```"))) +} diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 1120a63d..27d75f66 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -2,7 +2,7 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{EventId, MxcUri, ServerName}; +use ruma::{EventId, MxcUri, OwnedMxcUri, ServerName}; use crate::admin_command_dispatch; @@ -55,4 +55,9 @@ pub(super) enum MediaCommand { #[arg(short, long)] force: bool, }, + + GetFileInfo { + /// The MXC URL to lookup info for. + mxc: OwnedMxcUri, + }, } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index fc86b215..5b12a94c 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -31,7 +31,7 @@ pub struct FileMeta { pub struct Service { url_preview_mutex: MutexMap, - pub(crate) db: Data, + pub(super) db: Data, services: Services, } @@ -347,6 +347,18 @@ impl Service { Ok(file) } + #[inline] + pub fn get_metadata(&self, mxc: &Mxc<'_>) -> Option { + self.db + .search_file_metadata(mxc, 0, 0) + .map(|metadata| FileMeta { + content_disposition: metadata.content_disposition, + content_type: metadata.content_type, + content: None, + }) + .ok() + } + #[inline] #[must_use] pub fn get_media_file(&self, key: &[u8]) -> PathBuf { self.get_media_file_sha256(key) } From 4d42a29c51ef982d513e9d1db9841129c6e0e6d6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 01:15:31 +0000 Subject: [PATCH 0715/2091] add thumbnail dimension structure Signed-off-by: Jason Volk --- Cargo.lock | 26 ++--- Cargo.toml | 2 +- src/api/client/media.rs | 16 +-- src/service/media/data.rs | 16 +-- src/service/media/mod.rs | 9 +- src/service/media/remote.rs | 19 +--- src/service/media/thumbnail.rs | 181 ++++++++++++++++++++++----------- 7 files changed, 155 insertions(+), 114 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ef18182c..fabb9b09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "assign", "js_int", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "js_int", "ruma-common", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "as_variant", "assign", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "as_variant", "base64 0.22.1", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "bytes", "http", @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "js_int", "thiserror", @@ -3113,7 +3113,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "js_int", "ruma-common", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "once_cell", "proc-macro-crate", @@ -3138,7 +3138,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "js_int", "ruma-common", @@ -3150,7 +3150,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "headers", "http", @@ -3163,7 +3163,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3179,7 +3179,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=25fbd64b968c5d5088c07750aaa4873e072831b0#25fbd64b968c5d5088c07750aaa4873e072831b0" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index a9269d08..c6793c44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -314,7 +314,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "25fbd64b968c5d5088c07750aaa4873e072831b0" +rev = "a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" features = [ "compat", "rand", diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 9553d425..41a34d75 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -14,7 +14,7 @@ use ruma::{ }, Mxc, }; -use service::media::{FileMeta, MXC_LENGTH}; +use service::media::{Dim, FileMeta, MXC_LENGTH}; use crate::{Ruma, RumaResponse}; @@ -326,22 +326,12 @@ pub(crate) async fn get_content_thumbnail_route( media_id: &body.media_id, }; + let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; if let Some(FileMeta { content, content_type, content_disposition, - }) = services - .media - .get_thumbnail( - &mxc, - body.width - .try_into() - .map_err(|e| err!(Request(InvalidParam("Width is invalid: {e:?}"))))?, - body.height - .try_into() - .map_err(|e| err!(Request(InvalidParam("Height is invalid: {e:?}"))))?, - ) - .await? + }) = services.media.get_thumbnail(&mxc, &dim).await? { let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 06cd6cc7..e5d6d20b 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -8,7 +8,7 @@ use conduit::{ use database::{Database, Map}; use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; -use super::preview::UrlPreviewData; +use super::{preview::UrlPreviewData, thumbnail::Dim}; pub(crate) struct Data { mediaid_file: Arc, @@ -33,8 +33,8 @@ impl Data { } pub(super) fn create_file_metadata( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, width: u32, height: u32, - content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, + &self, mxc: &Mxc<'_>, user: Option<&UserId>, dim: &Dim, content_disposition: Option<&ContentDisposition>, + content_type: Option<&str>, ) -> Result> { let mut key: Vec = Vec::new(); key.extend_from_slice(b"mxc://"); @@ -42,8 +42,8 @@ impl Data { key.extend_from_slice(b"/"); key.extend_from_slice(mxc.media_id.as_bytes()); key.push(0xFF); - key.extend_from_slice(&width.to_be_bytes()); - key.extend_from_slice(&height.to_be_bytes()); + key.extend_from_slice(&dim.width.to_be_bytes()); + key.extend_from_slice(&dim.height.to_be_bytes()); key.push(0xFF); key.extend_from_slice( content_disposition @@ -128,15 +128,15 @@ impl Data { Ok(keys) } - pub(super) fn search_file_metadata(&self, mxc: &Mxc<'_>, width: u32, height: u32) -> Result { + pub(super) fn search_file_metadata(&self, mxc: &Mxc<'_>, dim: &Dim) -> Result { let mut prefix: Vec = Vec::new(); prefix.extend_from_slice(b"mxc://"); prefix.extend_from_slice(mxc.server_name.as_bytes()); prefix.extend_from_slice(b"/"); prefix.extend_from_slice(mxc.media_id.as_bytes()); prefix.push(0xFF); - prefix.extend_from_slice(&width.to_be_bytes()); - prefix.extend_from_slice(&height.to_be_bytes()); + prefix.extend_from_slice(&dim.width.to_be_bytes()); + prefix.extend_from_slice(&dim.height.to_be_bytes()); prefix.push(0xFF); let (key, _) = self diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 5b12a94c..5e3f82b3 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -13,13 +13,14 @@ use conduit::{ utils::{self, MutexMap}, warn, Err, Result, Server, }; -use data::{Data, Metadata}; use ruma::{http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt, BufReader}, }; +use self::data::{Data, Metadata}; +pub use self::thumbnail::Dim; use crate::{client, globals, sending, Dep}; #[derive(Debug)] @@ -78,7 +79,7 @@ impl Service { // Width, Height = 0 if it's not a thumbnail let key = self .db - .create_file_metadata(mxc, user, 0, 0, content_disposition, content_type)?; + .create_file_metadata(mxc, user, &Dim::default(), content_disposition, content_type)?; //TODO: Dangling metadata in database if creation fails let mut f = self.create_media_file(&key).await?; @@ -141,7 +142,7 @@ impl Service { content_disposition, content_type, key, - }) = self.db.search_file_metadata(mxc, 0, 0) + }) = self.db.search_file_metadata(mxc, &Dim::default()) { let mut content = Vec::new(); let path = self.get_media_file(&key); @@ -350,7 +351,7 @@ impl Service { #[inline] pub fn get_metadata(&self, mxc: &Mxc<'_>) -> Option { self.db - .search_file_metadata(mxc, 0, 0) + .search_file_metadata(mxc, &Dim::default()) .map(|metadata| FileMeta { content_disposition: metadata.content_disposition, content_type: metadata.content_type, diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index d4c458e2..41e87dbb 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -3,6 +3,8 @@ use std::time::Duration; use conduit::{debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, Result}; use ruma::{api::client::media, Mxc}; +use super::Dim; + #[implement(super::Service)] #[allow(deprecated)] pub async fn fetch_remote_thumbnail_legacy( @@ -33,20 +35,9 @@ pub async fn fetch_remote_thumbnail_legacy( ) .await?; - self.upload_thumbnail( - &mxc, - None, - None, - reponse.content_type.as_deref(), - body.width - .try_into() - .map_err(|e| err!(Request(InvalidParam("Width is invalid: {e:?}"))))?, - body.height - .try_into() - .map_err(|e| err!(Request(InvalidParam("Height is invalid: {e:?}"))))?, - &reponse.file, - ) - .await?; + let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; + self.upload_thumbnail(&mxc, None, None, reponse.content_type.as_deref(), &dim, &reponse.file) + .await?; Ok(reponse) } diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 3aadcfc0..630f7b3b 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -1,8 +1,8 @@ use std::{cmp, io::Cursor, num::Saturating as Sat}; -use conduit::{checked, Result}; +use conduit::{checked, err, Result}; use image::{imageops::FilterType, DynamicImage}; -use ruma::{http_headers::ContentDisposition, Mxc, UserId}; +use ruma::{http_headers::ContentDisposition, media::Method, Mxc, UInt, UserId}; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt}, @@ -10,16 +10,24 @@ use tokio::{ use super::{data::Metadata, FileMeta}; +/// Dimension specification for a thumbnail. +#[derive(Debug)] +pub struct Dim { + pub width: u32, + pub height: u32, + pub method: Method, +} + impl super::Service { /// Uploads or replaces a file thumbnail. #[allow(clippy::too_many_arguments)] pub async fn upload_thumbnail( &self, mxc: &Mxc<'_>, user: Option<&UserId>, content_disposition: Option<&ContentDisposition>, - content_type: Option<&str>, width: u32, height: u32, file: &[u8], + content_type: Option<&str>, dim: &Dim, file: &[u8], ) -> Result<()> { let key = self .db - .create_file_metadata(mxc, user, width, height, content_disposition, content_type)?; + .create_file_metadata(mxc, user, dim, content_disposition, content_type)?; //TODO: Dangling metadata in database if creation fails let mut f = self.create_media_file(&key).await?; @@ -42,15 +50,14 @@ impl super::Service { /// For width,height <= 96 the server uses another thumbnailing algorithm /// which crops the image afterwards. #[tracing::instrument(skip(self), name = "thumbnail", level = "debug")] - pub async fn get_thumbnail(&self, mxc: &Mxc<'_>, width: u32, height: u32) -> Result> { + pub async fn get_thumbnail(&self, mxc: &Mxc<'_>, dim: &Dim) -> Result> { // 0, 0 because that's the original file - let (width, height, crop) = thumbnail_properties(width, height).unwrap_or((0, 0, false)); + let dim = dim.normalized(); - if let Ok(metadata) = self.db.search_file_metadata(mxc, width, height) { + if let Ok(metadata) = self.db.search_file_metadata(mxc, &dim) { self.get_thumbnail_saved(metadata).await - } else if let Ok(metadata) = self.db.search_file_metadata(mxc, 0, 0) { - self.get_thumbnail_generate(mxc, width, height, crop, metadata) - .await + } else if let Ok(metadata) = self.db.search_file_metadata(mxc, &Dim::default()) { + self.get_thumbnail_generate(mxc, &dim, metadata).await } else { Ok(None) } @@ -71,9 +78,7 @@ impl super::Service { /// Generate a thumbnail #[tracing::instrument(skip(self), name = "generate", level = "debug")] - async fn get_thumbnail_generate( - &self, mxc: &Mxc<'_>, width: u32, height: u32, crop: bool, data: Metadata, - ) -> Result> { + async fn get_thumbnail_generate(&self, mxc: &Mxc<'_>, dim: &Dim, data: Metadata) -> Result> { let mut content = Vec::new(); let path = self.get_media_file(&data.key); fs::File::open(path) @@ -86,20 +91,19 @@ impl super::Service { return Ok(Some(into_filemeta(data, content))); }; - if width > image.width() || height > image.height() { + if dim.width > image.width() || dim.height > image.height() { return Ok(Some(into_filemeta(data, content))); } let mut thumbnail_bytes = Vec::new(); - let thumbnail = thumbnail_generate(&image, width, height, crop)?; + let thumbnail = thumbnail_generate(&image, dim)?; thumbnail.write_to(&mut Cursor::new(&mut thumbnail_bytes), image::ImageFormat::Png)?; // Save thumbnail in database so we don't have to generate it again next time let thumbnail_key = self.db.create_file_metadata( mxc, None, - width, - height, + dim, data.content_disposition.as_ref(), data.content_type.as_deref(), )?; @@ -111,56 +115,25 @@ impl super::Service { } } -fn thumbnail_generate(image: &DynamicImage, width: u32, height: u32, crop: bool) -> Result { - let thumbnail = if crop { - image.resize_to_fill(width, height, FilterType::CatmullRom) +fn thumbnail_generate(image: &DynamicImage, requested: &Dim) -> Result { + let thumbnail = if !requested.crop() { + let Dim { + width, + height, + .. + } = requested.scaled(&Dim { + width: image.width(), + height: image.height(), + ..Dim::default() + })?; + image.thumbnail_exact(width, height) } else { - let (exact_width, exact_height) = thumbnail_dimension(image, width, height)?; - image.thumbnail_exact(exact_width, exact_height) + image.resize_to_fill(requested.width, requested.height, FilterType::CatmullRom) }; Ok(thumbnail) } -fn thumbnail_dimension(image: &DynamicImage, width: u32, height: u32) -> Result<(u32, u32)> { - let image_width = image.width(); - let image_height = image.height(); - - let width = cmp::min(width, image_width); - let height = cmp::min(height, image_height); - - let use_width = Sat(width) * Sat(image_height) < Sat(height) * Sat(image_width); - - let x = if use_width { - let dividend = (Sat(height) * Sat(image_width)).0; - checked!(dividend / image_height)? - } else { - width - }; - - let y = if !use_width { - let dividend = (Sat(width) * Sat(image_height)).0; - checked!(dividend / image_width)? - } else { - height - }; - - Ok((x, y)) -} - -/// Returns width, height of the thumbnail and whether it should be cropped. -/// Returns None when the server should send the original file. -fn thumbnail_properties(width: u32, height: u32) -> Option<(u32, u32, bool)> { - match (width, height) { - (0..=32, 0..=32) => Some((32, 32, true)), - (0..=96, 0..=96) => Some((96, 96, true)), - (0..=320, 0..=240) => Some((320, 240, false)), - (0..=640, 0..=480) => Some((640, 480, false)), - (0..=800, 0..=600) => Some((800, 600, false)), - _ => None, - } -} - fn into_filemeta(data: Metadata, content: Vec) -> FileMeta { FileMeta { content: Some(content), @@ -168,3 +141,89 @@ fn into_filemeta(data: Metadata, content: Vec) -> FileMeta { content_disposition: data.content_disposition, } } + +impl Dim { + /// Instantiate a Dim from Ruma integers with optional method. + pub fn from_ruma(width: UInt, height: UInt, method: Option) -> Result { + let width = width + .try_into() + .map_err(|e| err!(Request(InvalidParam("Width is invalid: {e:?}"))))?; + let height = height + .try_into() + .map_err(|e| err!(Request(InvalidParam("Height is invalid: {e:?}"))))?; + + Ok(Self::new(width, height, method)) + } + + /// Instantiate a Dim with optional method + #[inline] + #[must_use] + pub fn new(width: u32, height: u32, method: Option) -> Self { + Self { + width, + height, + method: method.unwrap_or(Method::Scale), + } + } + + pub fn scaled(&self, image: &Self) -> Result { + let image_width = image.width; + let image_height = image.height; + + let width = cmp::min(self.width, image_width); + let height = cmp::min(self.height, image_height); + + let use_width = Sat(width) * Sat(image_height) < Sat(height) * Sat(image_width); + + let x = if use_width { + let dividend = (Sat(height) * Sat(image_width)).0; + checked!(dividend / image_height)? + } else { + width + }; + + let y = if !use_width { + let dividend = (Sat(width) * Sat(image_height)).0; + checked!(dividend / image_width)? + } else { + height + }; + + Ok(Self { + width: x, + height: y, + method: Method::Scale, + }) + } + + /// Returns width, height of the thumbnail and whether it should be cropped. + /// Returns None when the server should send the original file. + /// Ignores the input Method. + #[must_use] + pub fn normalized(&self) -> Self { + match (self.width, self.height) { + (0..=32, 0..=32) => Self::new(32, 32, Some(Method::Crop)), + (0..=96, 0..=96) => Self::new(96, 96, Some(Method::Crop)), + (0..=320, 0..=240) => Self::new(320, 240, Some(Method::Scale)), + (0..=640, 0..=480) => Self::new(640, 480, Some(Method::Scale)), + (0..=800, 0..=600) => Self::new(800, 600, Some(Method::Scale)), + _ => Self::default(), + } + } + + /// Returns true if the method is Crop. + #[inline] + #[must_use] + pub fn crop(&self) -> bool { self.method == Method::Crop } +} + +impl Default for Dim { + #[inline] + fn default() -> Self { + Self { + width: 0, + height: 0, + method: Method::Scale, + } + } +} From dea5fee6a38c5400d85d565fee1771633af3b180 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 01:23:44 +0000 Subject: [PATCH 0716/2091] abstract duration parsing into utils Signed-off-by: Jason Volk --- Cargo.lock | 2 +- src/admin/media/commands.rs | 1 + src/core/Cargo.toml | 1 + src/core/utils/time.rs | 18 ++++++++++++++++++ src/service/Cargo.toml | 1 - src/service/media/mod.rs | 13 +++---------- 6 files changed, 24 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fabb9b09..a2a70cda 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -692,6 +692,7 @@ dependencies = [ "conduit_macros", "const-str", "ctor", + "cyborgtime", "either", "figment", "hardened_malloc-rs", @@ -790,7 +791,6 @@ dependencies = [ "conduit_core", "conduit_database", "const-str", - "cyborgtime", "futures-util", "hickory-resolver", "http", diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 3894d5df..385419eb 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -176,6 +176,7 @@ pub(super) async fn delete_list(&self) -> Result { #[admin_command] pub(super) async fn delete_past_remote_media(&self, duration: String, force: bool) -> Result { + let duration = parse_timepoint_ago(&duration)?; let deleted_count = self .services .media diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 2e2b0581..cc5865c8 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -68,6 +68,7 @@ clap.workspace = true conduit-macros.workspace = true const-str.workspace = true ctor.workspace = true +cyborgtime.workspace = true either.workspace = true figment.workspace = true http-body-util.workspace = true diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 9a31632e..04f47ac3 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,5 +1,7 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use crate::{err, Result}; + #[inline] #[must_use] #[allow(clippy::as_conversions, clippy::cast_possible_truncation)] @@ -10,6 +12,22 @@ pub fn now_millis() -> u64 { .as_millis() as u64 } +#[inline] +pub fn parse_timepoint_ago(ago: &str) -> Result { timepoint_ago(parse_duration(ago)?) } + +#[inline] +pub fn timepoint_ago(duration: Duration) -> Result { + SystemTime::now() + .checked_sub(duration) + .ok_or_else(|| err!(Arithmetic("Duration {duration:?} is too large"))) +} + +#[inline] +pub fn parse_duration(duration: &str) -> Result { + cyborgtime::parse_duration(duration) + .map_err(|error| err!("'{duration:?}' is not a valid duration string: {error:?}")) +} + #[must_use] pub fn rfc2822_from_seconds(epoch: i64) -> String { use chrono::{DateTime, Utc}; diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 92563934..cfed5a0e 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -46,7 +46,6 @@ bytes.workspace = true conduit-core.workspace = true conduit-database.workspace = true const-str.workspace = true -cyborgtime.workspace = true futures-util.workspace = true hickory-resolver.workspace = true http.workspace = true diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 5e3f82b3..db0b8459 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -196,14 +196,7 @@ impl Service { /// Deletes all remote only media files in the given at or after /// time/duration. Returns a u32 with the amount of media files deleted. - pub async fn delete_all_remote_media_at_after_time(&self, time: String, force: bool) -> Result { - let user_duration: SystemTime = match cyborgtime::parse_duration(&time) { - Err(e) => return Err!(Database(error!("Failed to parse specified time duration: {e}"))), - Ok(duration) => SystemTime::now() - .checked_sub(duration) - .ok_or(err!(Arithmetic("Duration {duration:?} is too large")))?, - }; - + pub async fn delete_all_remote_media_at_after_time(&self, time: SystemTime, force: bool) -> Result { let all_keys = self.db.get_all_media_keys(); let mut remote_mxcs = Vec::with_capacity(all_keys.len()); @@ -271,7 +264,7 @@ impl Service { }; debug!("File created at: {file_created_at:?}"); - if file_created_at <= user_duration { + if file_created_at <= time { debug!("File is within user duration, pushing to list of file paths and keys to delete."); remote_mxcs.push(mxc.to_string()); } @@ -284,7 +277,7 @@ impl Service { return Err!(Database("Did not found any eligible MXCs to delete.")); } - debug_info!("Deleting media now in the past {user_duration:?}."); + debug_info!("Deleting media now in the past {time:?}."); let mut deletion_count: usize = 0; for mxc in remote_mxcs { let mxc: Mxc<'_> = mxc.as_str().try_into()?; From 9654dd9218daeb699bf6b7c920069b9837201cdf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 01:45:35 +0000 Subject: [PATCH 0717/2091] Fix stale example config for media_compat_file_link Signed-off-by: Jason Volk --- conduwuit-example.toml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 553f62a1..53e9acef 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -390,10 +390,13 @@ allow_profile_lookup_federation_requests = true # Enable backward-compatibility with Conduit's media directory by creating symlinks of media. This # option is only necessary if you plan on using Conduit again. Otherwise setting this to false -# reduces filesystem clutter and overhead for managing these symlinks in the directory. +# reduces filesystem clutter and overhead for managing these symlinks in the directory. This is now +# disabled by default. You may still return to upstream Conduit but you have to run Conduwuit at +# least once with this set to true and allow the media_startup_check to take place before shutting +# down to return to Conduit. # -# Enabled by default. -#media_compat_file_link = true +# Disabled by default. +#media_compat_file_link = false # Checks consistency of the media directory at startup: # 1. When `media_compat_file_link` is enbled, this check will upgrade media when switching back From 73af171830c94382ae3c1c7dbc83e36f77cd80a0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 01:46:09 +0000 Subject: [PATCH 0718/2091] add config option for pruning missing media Signed-off-by: Jason Volk --- conduwuit-example.toml | 8 ++++++++ src/core/config/mod.rs | 3 +++ src/service/globals/migrations.rs | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 53e9acef..a99aa05c 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -398,6 +398,14 @@ allow_profile_lookup_federation_requests = true # Disabled by default. #media_compat_file_link = false +# Prunes missing media from the database as part of the media startup checks. This means if you +# delete files from the media directory the corresponding entries will be removed from the +# database. This is disabled by default because if the media directory is accidentally moved or +# inaccessible the metadata entries in the database will be lost with sadness. +# +# Disabled by default. +#prune_missing_media = false + # Checks consistency of the media directory at startup: # 1. When `media_compat_file_link` is enbled, this check will upgrade media when switching back # and forth between Conduit and Conduwuit. Both options must be enabled to handle this. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 12b36a65..8836c2c9 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -293,6 +293,8 @@ pub struct Config { pub media_startup_check: bool, #[serde(default)] pub media_compat_file_link: bool, + #[serde(default)] + pub prune_missing_media: bool, #[serde(default = "Vec::new")] pub prevent_media_downloads_from: Vec, @@ -742,6 +744,7 @@ impl fmt::Display for Config { line("RocksDB Statistics level", &self.rocksdb_stats_level.to_string()); line("Media integrity checks on startup", &self.media_startup_check.to_string()); line("Media compatibility filesystem links", &self.media_compat_file_link.to_string()); + line("Prune missing media from database", &self.prune_missing_media.to_string()); line("Prevent Media Downloads From", { let mut lst = vec![]; for domain in &self.prevent_media_downloads_from { diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 5e078595..3e81340e 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -833,7 +833,7 @@ async fn handle_media_check( .map_or(false, |md| md.is_symlink()) }; - if !old_exists && !new_exists { + if config.prune_missing_media && !old_exists && !new_exists { error!( media_id = ?encode_key(key), ?new_path, ?old_path, "Media is missing at all paths. Removing from database..." From 47f9c69efffa1e9e8318e43cddc1a2d217ac502b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 01:55:09 +0000 Subject: [PATCH 0719/2091] split media related migrations into service dir Signed-off-by: Jason Volk --- src/service/globals/migrations.rs | 159 +----------------------------- src/service/media/migrations.rs | 158 +++++++++++++++++++++++++++++ src/service/media/mod.rs | 1 + 3 files changed, 164 insertions(+), 154 deletions(-) create mode 100644 src/service/media/migrations.rs diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 3e81340e..39d829b4 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -1,15 +1,12 @@ use std::{ collections::{HashMap, HashSet}, - ffi::{OsStr, OsString}, fs::{self}, io::Write, mem::size_of, - path::PathBuf, sync::Arc, - time::Instant, }; -use conduit::{debug, debug_info, debug_warn, error, info, utils, warn, Config, Error, Result}; +use conduit::{debug, debug_info, debug_warn, error, info, utils, warn, Error, Result}; use itertools::Itertools; use ruma::{ events::{push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType}, @@ -17,7 +14,7 @@ use ruma::{ EventId, OwnedRoomId, RoomId, UserId, }; -use crate::Services; +use crate::{media, Services}; /// The current schema version. /// - If database is opened at greater version we reject with error. The @@ -25,7 +22,7 @@ use crate::Services; /// - If database is opened at lesser version we apply migrations up to this. /// Note that named-feature migrations may also be performed when opening at /// equal or lesser version. These are expected to be backward-compatible. -const DATABASE_VERSION: u64 = 13; +pub(crate) const DATABASE_VERSION: u64 = 13; pub(crate) async fn migrations(services: &Services) -> Result<()> { // Matrix resource ownership is based on the server name; changing it @@ -131,9 +128,9 @@ async fn migrate(services: &Services) -> Result<()> { } if db["global"].get(b"feat_sha256_media")?.is_none() { - migrate_sha256_media(services).await?; + media::migrations::migrate_sha256_media(services).await?; } else if config.media_startup_check { - checkup_sha256_media(services).await?; + media::migrations::checkup_sha256_media(services).await?; } if db["global"] @@ -738,152 +735,6 @@ async fn db_lt_13(services: &Services) -> Result<()> { Ok(()) } -/// Migrates a media directory from legacy base64 file names to sha2 file names. -/// All errors are fatal. Upon success the database is keyed to not perform this -/// again. -async fn migrate_sha256_media(services: &Services) -> Result<()> { - let db = &services.db; - let config = &services.server.config; - - warn!("Migrating legacy base64 file names to sha256 file names"); - let mediaid_file = &db["mediaid_file"]; - - // Move old media files to new names - let mut changes = Vec::<(PathBuf, PathBuf)>::new(); - for (key, _) in mediaid_file.iter() { - let old = services.media.get_media_file_b64(&key); - let new = services.media.get_media_file_sha256(&key); - debug!(?key, ?old, ?new, num = changes.len(), "change"); - changes.push((old, new)); - } - // move the file to the new location - for (old_path, path) in changes { - if old_path.exists() { - tokio::fs::rename(&old_path, &path).await?; - if config.media_compat_file_link { - tokio::fs::symlink(&path, &old_path).await?; - } - } - } - - // Apply fix from when sha256_media was backward-incompat and bumped the schema - // version from 13 to 14. For users satisfying these conditions we can go back. - if services.globals.db.database_version()? == 14 && DATABASE_VERSION == 13 { - services.globals.db.bump_database_version(13)?; - } - - db["global"].insert(b"feat_sha256_media", &[])?; - info!("Finished applying sha256_media"); - Ok(()) -} - -/// Check is run on startup for prior-migrated media directories. This handles: -/// - Going back and forth to non-sha256 legacy binaries (e.g. upstream). -/// - Deletion of artifacts in the media directory which will then fall out of -/// sync with the database. -async fn checkup_sha256_media(services: &Services) -> Result<()> { - use crate::media::encode_key; - - debug!("Checking integrity of media directory"); - let db = &services.db; - let media = &services.media; - let config = &services.server.config; - let mediaid_file = &db["mediaid_file"]; - let mediaid_user = &db["mediaid_user"]; - let dbs = (mediaid_file, mediaid_user); - let timer = Instant::now(); - - let dir = media.get_media_dir(); - let files: HashSet = fs::read_dir(dir)? - .filter_map(|ent| ent.map_or(None, |ent| Some(ent.path().into_os_string()))) - .collect(); - - for key in media.db.get_all_media_keys() { - let new_path = media.get_media_file_sha256(&key).into_os_string(); - let old_path = media.get_media_file_b64(&key).into_os_string(); - if let Err(e) = handle_media_check(&dbs, config, &files, &key, &new_path, &old_path).await { - error!( - media_id = ?encode_key(&key), ?new_path, ?old_path, - "Failed to resolve media check failure: {e}" - ); - } - } - - debug_info!( - elapsed = ?timer.elapsed(), - "Finished checking media directory" - ); - - Ok(()) -} - -async fn handle_media_check( - dbs: &(&Arc, &Arc), config: &Config, files: &HashSet, key: &[u8], - new_path: &OsStr, old_path: &OsStr, -) -> Result<()> { - use crate::media::encode_key; - - let (mediaid_file, mediaid_user) = dbs; - - let new_exists = files.contains(new_path); - let old_exists = files.contains(old_path); - let old_is_symlink = || async { - tokio::fs::symlink_metadata(old_path) - .await - .map_or(false, |md| md.is_symlink()) - }; - - if config.prune_missing_media && !old_exists && !new_exists { - error!( - media_id = ?encode_key(key), ?new_path, ?old_path, - "Media is missing at all paths. Removing from database..." - ); - - mediaid_file.remove(key)?; - mediaid_user.remove(key)?; - } - - if config.media_compat_file_link && !old_exists && new_exists { - debug_warn!( - media_id = ?encode_key(key), ?new_path, ?old_path, - "Media found but missing legacy link. Fixing..." - ); - - tokio::fs::symlink(&new_path, &old_path).await?; - } - - if config.media_compat_file_link && !new_exists && old_exists { - debug_warn!( - media_id = ?encode_key(key), ?new_path, ?old_path, - "Legacy media found without sha256 migration. Fixing..." - ); - - debug_assert!( - old_is_symlink().await, - "Legacy media not expected to be a symlink without an existing sha256 migration." - ); - - tokio::fs::rename(&old_path, &new_path).await?; - tokio::fs::symlink(&new_path, &old_path).await?; - } - - if !config.media_compat_file_link && old_exists && old_is_symlink().await { - debug_warn!( - media_id = ?encode_key(key), ?new_path, ?old_path, - "Legacy link found but compat disabled. Cleansing symlink..." - ); - - debug_assert!( - new_exists, - "sha256 migration into new file expected prior to cleaning legacy symlink here." - ); - - tokio::fs::remove_file(&old_path).await?; - } - - Ok(()) -} - async fn fix_bad_double_separator_in_state_cache(services: &Services) -> Result<()> { warn!("Fixing bad double separator in state_cache roomuserid_joined"); diff --git a/src/service/media/migrations.rs b/src/service/media/migrations.rs new file mode 100644 index 00000000..9968d25b --- /dev/null +++ b/src/service/media/migrations.rs @@ -0,0 +1,158 @@ +use std::{ + collections::HashSet, + ffi::{OsStr, OsString}, + fs::{self}, + path::PathBuf, + sync::Arc, + time::Instant, +}; + +use conduit::{debug, debug_info, debug_warn, error, info, warn, Config, Result}; + +use crate::{globals, Services}; + +/// Migrates a media directory from legacy base64 file names to sha2 file names. +/// All errors are fatal. Upon success the database is keyed to not perform this +/// again. +pub(crate) async fn migrate_sha256_media(services: &Services) -> Result<()> { + let db = &services.db; + let config = &services.server.config; + + warn!("Migrating legacy base64 file names to sha256 file names"); + let mediaid_file = &db["mediaid_file"]; + + // Move old media files to new names + let mut changes = Vec::<(PathBuf, PathBuf)>::new(); + for (key, _) in mediaid_file.iter() { + let old = services.media.get_media_file_b64(&key); + let new = services.media.get_media_file_sha256(&key); + debug!(?key, ?old, ?new, num = changes.len(), "change"); + changes.push((old, new)); + } + // move the file to the new location + for (old_path, path) in changes { + if old_path.exists() { + tokio::fs::rename(&old_path, &path).await?; + if config.media_compat_file_link { + tokio::fs::symlink(&path, &old_path).await?; + } + } + } + + // Apply fix from when sha256_media was backward-incompat and bumped the schema + // version from 13 to 14. For users satisfying these conditions we can go back. + if services.globals.db.database_version()? == 14 && globals::migrations::DATABASE_VERSION == 13 { + services.globals.db.bump_database_version(13)?; + } + + db["global"].insert(b"feat_sha256_media", &[])?; + info!("Finished applying sha256_media"); + Ok(()) +} + +/// Check is run on startup for prior-migrated media directories. This handles: +/// - Going back and forth to non-sha256 legacy binaries (e.g. upstream). +/// - Deletion of artifacts in the media directory which will then fall out of +/// sync with the database. +pub(crate) async fn checkup_sha256_media(services: &Services) -> Result<()> { + use crate::media::encode_key; + + debug!("Checking integrity of media directory"); + let db = &services.db; + let media = &services.media; + let config = &services.server.config; + let mediaid_file = &db["mediaid_file"]; + let mediaid_user = &db["mediaid_user"]; + let dbs = (mediaid_file, mediaid_user); + let timer = Instant::now(); + + let dir = media.get_media_dir(); + let files: HashSet = fs::read_dir(dir)? + .filter_map(|ent| ent.map_or(None, |ent| Some(ent.path().into_os_string()))) + .collect(); + + for key in media.db.get_all_media_keys() { + let new_path = media.get_media_file_sha256(&key).into_os_string(); + let old_path = media.get_media_file_b64(&key).into_os_string(); + if let Err(e) = handle_media_check(&dbs, config, &files, &key, &new_path, &old_path).await { + error!( + media_id = ?encode_key(&key), ?new_path, ?old_path, + "Failed to resolve media check failure: {e}" + ); + } + } + + debug_info!( + elapsed = ?timer.elapsed(), + "Finished checking media directory" + ); + + Ok(()) +} + +async fn handle_media_check( + dbs: &(&Arc, &Arc), config: &Config, files: &HashSet, key: &[u8], + new_path: &OsStr, old_path: &OsStr, +) -> Result<()> { + use crate::media::encode_key; + + let (mediaid_file, mediaid_user) = dbs; + + let new_exists = files.contains(new_path); + let old_exists = files.contains(old_path); + let old_is_symlink = || async { + tokio::fs::symlink_metadata(old_path) + .await + .map_or(false, |md| md.is_symlink()) + }; + + if config.prune_missing_media && !old_exists && !new_exists { + error!( + media_id = ?encode_key(key), ?new_path, ?old_path, + "Media is missing at all paths. Removing from database..." + ); + + mediaid_file.remove(key)?; + mediaid_user.remove(key)?; + } + + if config.media_compat_file_link && !old_exists && new_exists { + debug_warn!( + media_id = ?encode_key(key), ?new_path, ?old_path, + "Media found but missing legacy link. Fixing..." + ); + + tokio::fs::symlink(&new_path, &old_path).await?; + } + + if config.media_compat_file_link && !new_exists && old_exists { + debug_warn!( + media_id = ?encode_key(key), ?new_path, ?old_path, + "Legacy media found without sha256 migration. Fixing..." + ); + + debug_assert!( + old_is_symlink().await, + "Legacy media not expected to be a symlink without an existing sha256 migration." + ); + + tokio::fs::rename(&old_path, &new_path).await?; + tokio::fs::symlink(&new_path, &old_path).await?; + } + + if !config.media_compat_file_link && old_exists && old_is_symlink().await { + debug_warn!( + media_id = ?encode_key(key), ?new_path, ?old_path, + "Legacy link found but compat disabled. Cleansing symlink..." + ); + + debug_assert!( + new_exists, + "sha256 migration into new file expected prior to cleaning legacy symlink here." + ); + + tokio::fs::remove_file(&old_path).await?; + } + + Ok(()) +} diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index db0b8459..04436f7c 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,4 +1,5 @@ mod data; +pub(super) mod migrations; mod preview; mod remote; mod tests; From 84a8e36120cfcd8ab1b79111a56ab7cc08e7f3ce Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 02:56:17 +0000 Subject: [PATCH 0720/2091] add federation endpoints for authenticated media Signed-off-by: Jason Volk --- Cargo.lock | 26 ++++++------- Cargo.toml | 2 +- src/api/router.rs | 2 + src/api/server/media.rs | 83 +++++++++++++++++++++++++++++++++++++++++ src/api/server/mod.rs | 2 + 5 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 src/api/server/media.rs diff --git a/Cargo.lock b/Cargo.lock index a2a70cda..40b88481 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "assign", "js_int", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "js_int", "ruma-common", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "as_variant", "assign", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "as_variant", "base64 0.22.1", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "bytes", "http", @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "js_int", "thiserror", @@ -3113,7 +3113,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "js_int", "ruma-common", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "once_cell", "proc-macro-crate", @@ -3138,7 +3138,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "js_int", "ruma-common", @@ -3150,7 +3150,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "headers", "http", @@ -3163,7 +3163,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3179,7 +3179,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a0cc9a80dd5da700fb9b992b6f92cb6be4c27487#a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index c6793c44..59284ecf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -314,7 +314,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "a0cc9a80dd5da700fb9b992b6f92cb6be4c27487" +rev = "89cee2d2d9f3776cb0a94fcea04fca456ed8a064" features = [ "compat", "rand", diff --git a/src/api/router.rs b/src/api/router.rs index 7d6df16c..a76cac32 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -230,6 +230,8 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(server::get_openid_userinfo_route) .ruma_route(server::get_hierarchy_route) .ruma_route(server::well_known_server) + .ruma_route(server::get_content_route) + .ruma_route(server::get_content_thumbnail_route) .route("/_conduwuit/local_user_count", get(client::conduwuit_local_user_count)) } else { router diff --git a/src/api/server/media.rs b/src/api/server/media.rs new file mode 100644 index 00000000..757cc9cc --- /dev/null +++ b/src/api/server/media.rs @@ -0,0 +1,83 @@ +use axum::extract::State; +use axum_client_ip::InsecureClientIp; +use conduit::{utils::content_disposition::make_content_disposition, Err, Result}; +use conduit_service::media::{Dim, FileMeta}; +use ruma::{ + api::federation::authenticated_media::{ + get_content, get_content_thumbnail, Content, ContentMetadata, FileOrLocation, + }, + Mxc, +}; + +use crate::Ruma; + +/// # `GET /_matrix/federation/v1/media/download/{mediaId}` +/// +/// Load media from our server. +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] +pub(crate) async fn get_content_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let mxc = Mxc { + server_name: services.globals.server_name(), + media_id: &body.media_id, + }; + + let Some(FileMeta { + content, + content_type, + content_disposition, + }) = services.media.get(&mxc).await? + else { + return Err!(Request(NotFound("Media not found."))); + }; + + let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + let content = Content { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + content_disposition: Some(content_disposition), + }; + + Ok(get_content::v1::Response { + content: FileOrLocation::File(content), + metadata: ContentMetadata::new(), + }) +} + +/// # `GET /_matrix/federation/v1/media/thumbnail/{mediaId}` +/// +/// Load media thumbnail from our server. +#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] +pub(crate) async fn get_content_thumbnail_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; + let mxc = Mxc { + server_name: services.globals.server_name(), + media_id: &body.media_id, + }; + + let Some(FileMeta { + content, + content_type, + content_disposition, + }) = services.media.get_thumbnail(&mxc, &dim).await? + else { + return Err!(Request(NotFound("Media not found."))); + }; + + let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + let content = Content { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + content_disposition: Some(content_disposition), + }; + + Ok(get_content_thumbnail::v1::Response { + content: FileOrLocation::File(content), + metadata: ContentMetadata::new(), + }) +} diff --git a/src/api/server/mod.rs b/src/api/server/mod.rs index 5d46a866..9a184f23 100644 --- a/src/api/server/mod.rs +++ b/src/api/server/mod.rs @@ -7,6 +7,7 @@ pub(super) mod invite; pub(super) mod key; pub(super) mod make_join; pub(super) mod make_leave; +pub(super) mod media; pub(super) mod openid; pub(super) mod publicrooms; pub(super) mod query; @@ -28,6 +29,7 @@ pub(super) use invite::*; pub(super) use key::*; pub(super) use make_join::*; pub(super) use make_leave::*; +pub(super) use media::*; pub(super) use openid::*; pub(super) use publicrooms::*; pub(super) use query::*; From fcfb323cd55ac22b166eb67dffc35241fb41f68d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 03:35:00 +0000 Subject: [PATCH 0721/2091] relegate legacy media handlers to separate file Signed-off-by: Jason Volk --- src/api/client/media.rs | 352 +-------------------------------- src/api/client/media_legacy.rs | 343 ++++++++++++++++++++++++++++++++ src/api/client/mod.rs | 2 + src/api/router.rs | 75 +++---- src/core/config/mod.rs | 3 + src/service/media/mod.rs | 6 + 6 files changed, 399 insertions(+), 382 deletions(-) create mode 100644 src/api/client/media_legacy.rs diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 41a34d75..b8140ff7 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -1,102 +1,13 @@ -#![allow(deprecated)] - use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - err, - utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize}, - Err, Result, + utils::{self, content_disposition::make_content_disposition}, + Result, }; -use ruma::{ - api::client::media::{ - create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, - get_media_preview, - }, - Mxc, -}; -use service::media::{Dim, FileMeta, MXC_LENGTH}; +use conduit_service::media::MXC_LENGTH; +use ruma::{api::client::media::create_content, Mxc}; -use crate::{Ruma, RumaResponse}; - -/// Cache control for immutable objects -const CACHE_CONTROL_IMMUTABLE: &str = "public,max-age=31536000,immutable"; - -const CORP_CROSS_ORIGIN: &str = "cross-origin"; - -/// # `GET /_matrix/media/v3/config` -/// -/// Returns max upload size. -pub(crate) async fn get_media_config_route( - State(services): State, _body: Ruma, -) -> Result { - Ok(get_media_config::v3::Response { - upload_size: ruma_from_usize(services.globals.config.max_request_size), - }) -} - -/// # `GET /_matrix/media/v1/config` -/// -/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or -/// clients may call. conduwuit adds these for compatibility purposes. -/// See -/// -/// Returns max upload size. -pub(crate) async fn get_media_config_v1_route( - State(services): State, body: Ruma, -) -> Result> { - get_media_config_route(State(services), body) - .await - .map(RumaResponse) -} - -/// # `GET /_matrix/media/v3/preview_url` -/// -/// Returns URL preview. -#[tracing::instrument(skip_all, fields(%client), name = "url_preview")] -pub(crate) async fn get_media_preview_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - - let url = &body.url; - if !services.media.url_preview_allowed(url) { - return Err!(Request(Forbidden( - debug_warn!(%sender_user, %url, "URL is not allowed to be previewed") - ))); - } - - let preview = services.media.get_url_preview(url).await.map_err(|e| { - err!(Request(Unknown( - debug_error!(%sender_user, %url, "Failed to fetch a URL preview: {e}") - ))) - })?; - - let res = serde_json::value::to_raw_value(&preview).map_err(|e| { - err!(Request(Unknown( - debug_error!(%sender_user, %url, "Failed to parse a URL preview: {e}") - ))) - })?; - - Ok(get_media_preview::v3::Response::from_raw_value(res)) -} - -/// # `GET /_matrix/media/v1/preview_url` -/// -/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or -/// clients may call. conduwuit adds these for compatibility purposes. -/// See -/// -/// Returns URL preview. -#[tracing::instrument(skip_all, fields(%client), name = "url_preview")] -pub(crate) async fn get_media_preview_v1_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result> { - get_media_preview_route(State(services), InsecureClientIp(client), body) - .await - .map(RumaResponse) -} +use crate::Ruma; /// # `POST /_matrix/media/v3/upload` /// @@ -132,256 +43,3 @@ pub(crate) async fn create_content_route( blurhash: None, }) } - -/// # `POST /_matrix/media/v1/upload` -/// -/// Permanently save media in the server. -/// -/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or -/// clients may call. conduwuit adds these for compatibility purposes. -/// See -/// -/// - Some metadata will be saved in the database -/// - Media will be saved in the media/ directory -#[tracing::instrument(skip_all, fields(%client), name = "media_upload")] -pub(crate) async fn create_content_v1_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result> { - create_content_route(State(services), InsecureClientIp(client), body) - .await - .map(RumaResponse) -} - -/// # `GET /_matrix/media/v3/download/{serverName}/{mediaId}` -/// -/// Load media from our server or over federation. -/// -/// - Only allows federation if `allow_remote` is true -/// - Only redirects if `allow_redirect` is true -/// - Uses client-provided `timeout_ms` if available, else defaults to 20 -/// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_get")] -pub(crate) async fn get_content_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result { - let mxc = Mxc { - server_name: &body.server_name, - media_id: &body.media_id, - }; - - if let Some(FileMeta { - content, - content_type, - content_disposition, - }) = services.media.get(&mxc).await? - { - let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); - - Ok(get_content::v3::Response { - file: content.expect("entire file contents"), - content_type: content_type.map(Into::into), - content_disposition: Some(content_disposition), - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) - } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { - let response = services - .media - .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) - .await - .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; - - let content_disposition = - make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); - - Ok(get_content::v3::Response { - file: response.file, - content_type: response.content_type, - content_disposition: Some(content_disposition), - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) - } else { - Err!(Request(NotFound("Media not found."))) - } -} - -/// # `GET /_matrix/media/v1/download/{serverName}/{mediaId}` -/// -/// Load media from our server or over federation. -/// -/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or -/// clients may call. conduwuit adds these for compatibility purposes. -/// See -/// -/// - Only allows federation if `allow_remote` is true -/// - Only redirects if `allow_redirect` is true -/// - Uses client-provided `timeout_ms` if available, else defaults to 20 -/// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_get")] -pub(crate) async fn get_content_v1_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result> { - get_content_route(State(services), InsecureClientIp(client), body) - .await - .map(RumaResponse) -} - -/// # `GET /_matrix/media/v3/download/{serverName}/{mediaId}/{fileName}` -/// -/// Load media from our server or over federation, permitting desired filename. -/// -/// - Only allows federation if `allow_remote` is true -/// - Only redirects if `allow_redirect` is true -/// - Uses client-provided `timeout_ms` if available, else defaults to 20 -/// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_get")] -pub(crate) async fn get_content_as_filename_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result { - let mxc = Mxc { - server_name: &body.server_name, - media_id: &body.media_id, - }; - - if let Some(FileMeta { - content, - content_type, - content_disposition, - }) = services.media.get(&mxc).await? - { - let content_disposition = - make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), Some(&body.filename)); - - Ok(get_content_as_filename::v3::Response { - file: content.expect("entire file contents"), - content_type: content_type.map(Into::into), - content_disposition: Some(content_disposition), - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) - } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { - let response = services - .media - .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) - .await - .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; - - let content_disposition = - make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); - - Ok(get_content_as_filename::v3::Response { - content_disposition: Some(content_disposition), - content_type: response.content_type, - file: response.file, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) - } else { - Err!(Request(NotFound("Media not found."))) - } -} - -/// # `GET /_matrix/media/v1/download/{serverName}/{mediaId}/{fileName}` -/// -/// Load media from our server or over federation, permitting desired filename. -/// -/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or -/// clients may call. conduwuit adds these for compatibility purposes. -/// See -/// -/// - Only allows federation if `allow_remote` is true -/// - Only redirects if `allow_redirect` is true -/// - Uses client-provided `timeout_ms` if available, else defaults to 20 -/// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_get")] -pub(crate) async fn get_content_as_filename_v1_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result> { - get_content_as_filename_route(State(services), InsecureClientIp(client), body) - .await - .map(RumaResponse) -} - -/// # `GET /_matrix/media/v3/thumbnail/{serverName}/{mediaId}` -/// -/// Load media thumbnail from our server or over federation. -/// -/// - Only allows federation if `allow_remote` is true -/// - Only redirects if `allow_redirect` is true -/// - Uses client-provided `timeout_ms` if available, else defaults to 20 -/// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] -pub(crate) async fn get_content_thumbnail_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result { - let mxc = Mxc { - server_name: &body.server_name, - media_id: &body.media_id, - }; - - let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; - if let Some(FileMeta { - content, - content_type, - content_disposition, - }) = services.media.get_thumbnail(&mxc, &dim).await? - { - let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); - - Ok(get_content_thumbnail::v3::Response { - file: content.expect("entire file contents"), - content_type: content_type.map(Into::into), - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - content_disposition: Some(content_disposition), - }) - } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { - let response = services - .media - .fetch_remote_thumbnail_legacy(&body) - .await - .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; - - let content_disposition = - make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); - - Ok(get_content_thumbnail::v3::Response { - file: response.file, - content_type: response.content_type, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - content_disposition: Some(content_disposition), - }) - } else { - Err!(Request(NotFound("Media not found."))) - } -} - -/// # `GET /_matrix/media/v1/thumbnail/{serverName}/{mediaId}` -/// -/// Load media thumbnail from our server or over federation. -/// -/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or -/// clients may call. conduwuit adds these for compatibility purposes. -/// See -/// -/// - Only allows federation if `allow_remote` is true -/// - Only redirects if `allow_redirect` is true -/// - Uses client-provided `timeout_ms` if available, else defaults to 20 -/// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] -pub(crate) async fn get_content_thumbnail_v1_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result> { - get_content_thumbnail_route(State(services), InsecureClientIp(client), body) - .await - .map(RumaResponse) -} diff --git a/src/api/client/media_legacy.rs b/src/api/client/media_legacy.rs new file mode 100644 index 00000000..e87b9a2b --- /dev/null +++ b/src/api/client/media_legacy.rs @@ -0,0 +1,343 @@ +#![allow(deprecated)] + +use axum::extract::State; +use axum_client_ip::InsecureClientIp; +use conduit::{ + err, + utils::{content_disposition::make_content_disposition, math::ruma_from_usize}, + Err, Result, +}; +use conduit_service::media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN}; +use ruma::{ + api::client::media::{ + create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, + get_media_preview, + }, + Mxc, +}; + +use crate::{client::create_content_route, Ruma, RumaResponse}; + +/// # `GET /_matrix/media/v3/config` +/// +/// Returns max upload size. +pub(crate) async fn get_media_config_legacy_route( + State(services): State, _body: Ruma, +) -> Result { + Ok(get_media_config::v3::Response { + upload_size: ruma_from_usize(services.globals.config.max_request_size), + }) +} + +/// # `GET /_matrix/media/v1/config` +/// +/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or +/// clients may call. conduwuit adds these for compatibility purposes. +/// See +/// +/// Returns max upload size. +pub(crate) async fn get_media_config_legacy_legacy_route( + State(services): State, body: Ruma, +) -> Result> { + get_media_config_legacy_route(State(services), body) + .await + .map(RumaResponse) +} + +/// # `GET /_matrix/media/v3/preview_url` +/// +/// Returns URL preview. +#[tracing::instrument(skip_all, fields(%client), name = "url_preview_legacy")] +pub(crate) async fn get_media_preview_legacy_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + let url = &body.url; + if !services.media.url_preview_allowed(url) { + return Err!(Request(Forbidden( + debug_warn!(%sender_user, %url, "URL is not allowed to be previewed") + ))); + } + + let preview = services.media.get_url_preview(url).await.map_err(|e| { + err!(Request(Unknown( + debug_error!(%sender_user, %url, "Failed to fetch a URL preview: {e}") + ))) + })?; + + let res = serde_json::value::to_raw_value(&preview).map_err(|e| { + err!(Request(Unknown( + debug_error!(%sender_user, %url, "Failed to parse a URL preview: {e}") + ))) + })?; + + Ok(get_media_preview::v3::Response::from_raw_value(res)) +} + +/// # `GET /_matrix/media/v1/preview_url` +/// +/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or +/// clients may call. conduwuit adds these for compatibility purposes. +/// See +/// +/// Returns URL preview. +pub(crate) async fn get_media_preview_legacy_legacy_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result> { + get_media_preview_legacy_route(State(services), InsecureClientIp(client), body) + .await + .map(RumaResponse) +} + +/// # `POST /_matrix/media/v1/upload` +/// +/// Permanently save media in the server. +/// +/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or +/// clients may call. conduwuit adds these for compatibility purposes. +/// See +/// +/// - Some metadata will be saved in the database +/// - Media will be saved in the media/ directory +pub(crate) async fn create_content_legacy_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result> { + create_content_route(State(services), InsecureClientIp(client), body) + .await + .map(RumaResponse) +} + +/// # `GET /_matrix/media/v3/download/{serverName}/{mediaId}` +/// +/// Load media from our server or over federation. +/// +/// - Only allows federation if `allow_remote` is true +/// - Only redirects if `allow_redirect` is true +/// - Uses client-provided `timeout_ms` if available, else defaults to 20 +/// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")] +pub(crate) async fn get_content_legacy_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; + + if let Some(FileMeta { + content, + content_type, + content_disposition, + }) = services.media.get(&mxc).await? + { + let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + + Ok(get_content::v3::Response { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + }) + } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { + let response = services + .media + .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) + .await + .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; + + let content_disposition = + make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); + + Ok(get_content::v3::Response { + file: response.file, + content_type: response.content_type, + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + }) + } else { + Err!(Request(NotFound("Media not found."))) + } +} + +/// # `GET /_matrix/media/v1/download/{serverName}/{mediaId}` +/// +/// Load media from our server or over federation. +/// +/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or +/// clients may call. conduwuit adds these for compatibility purposes. +/// See +/// +/// - Only allows federation if `allow_remote` is true +/// - Only redirects if `allow_redirect` is true +/// - Uses client-provided `timeout_ms` if available, else defaults to 20 +/// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")] +pub(crate) async fn get_content_legacy_legacy_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result> { + get_content_legacy_route(State(services), InsecureClientIp(client), body) + .await + .map(RumaResponse) +} + +/// # `GET /_matrix/media/v3/download/{serverName}/{mediaId}/{fileName}` +/// +/// Load media from our server or over federation, permitting desired filename. +/// +/// - Only allows federation if `allow_remote` is true +/// - Only redirects if `allow_redirect` is true +/// - Uses client-provided `timeout_ms` if available, else defaults to 20 +/// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")] +pub(crate) async fn get_content_as_filename_legacy_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; + + if let Some(FileMeta { + content, + content_type, + content_disposition, + }) = services.media.get(&mxc).await? + { + let content_disposition = + make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), Some(&body.filename)); + + Ok(get_content_as_filename::v3::Response { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + }) + } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { + let response = services + .media + .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) + .await + .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; + + let content_disposition = + make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); + + Ok(get_content_as_filename::v3::Response { + content_disposition: Some(content_disposition), + content_type: response.content_type, + file: response.file, + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + }) + } else { + Err!(Request(NotFound("Media not found."))) + } +} + +/// # `GET /_matrix/media/v1/download/{serverName}/{mediaId}/{fileName}` +/// +/// Load media from our server or over federation, permitting desired filename. +/// +/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or +/// clients may call. conduwuit adds these for compatibility purposes. +/// See +/// +/// - Only allows federation if `allow_remote` is true +/// - Only redirects if `allow_redirect` is true +/// - Uses client-provided `timeout_ms` if available, else defaults to 20 +/// seconds +pub(crate) async fn get_content_as_filename_legacy_legacy_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result> { + get_content_as_filename_legacy_route(State(services), InsecureClientIp(client), body) + .await + .map(RumaResponse) +} + +/// # `GET /_matrix/media/v3/thumbnail/{serverName}/{mediaId}` +/// +/// Load media thumbnail from our server or over federation. +/// +/// - Only allows federation if `allow_remote` is true +/// - Only redirects if `allow_redirect` is true +/// - Uses client-provided `timeout_ms` if available, else defaults to 20 +/// seconds +#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get_legacy")] +pub(crate) async fn get_content_thumbnail_legacy_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; + + let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; + if let Some(FileMeta { + content, + content_type, + content_disposition, + }) = services.media.get_thumbnail(&mxc, &dim).await? + { + let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + + Ok(get_content_thumbnail::v3::Response { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + content_disposition: Some(content_disposition), + }) + } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { + let response = services + .media + .fetch_remote_thumbnail_legacy(&body) + .await + .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; + + let content_disposition = + make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); + + Ok(get_content_thumbnail::v3::Response { + file: response.file, + content_type: response.content_type, + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + content_disposition: Some(content_disposition), + }) + } else { + Err!(Request(NotFound("Media not found."))) + } +} + +/// # `GET /_matrix/media/v1/thumbnail/{serverName}/{mediaId}` +/// +/// Load media thumbnail from our server or over federation. +/// +/// This is a legacy endpoint ("/v1/") that some very old homeservers and/or +/// clients may call. conduwuit adds these for compatibility purposes. +/// See +/// +/// - Only allows federation if `allow_remote` is true +/// - Only redirects if `allow_redirect` is true +/// - Uses client-provided `timeout_ms` if available, else defaults to 20 +/// seconds +pub(crate) async fn get_content_thumbnail_legacy_legacy_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result> { + get_content_thumbnail_legacy_route(State(services), InsecureClientIp(client), body) + .await + .map(RumaResponse) +} diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index cf13cf7e..03c87e5d 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -9,6 +9,7 @@ pub(super) mod directory; pub(super) mod filter; pub(super) mod keys; pub(super) mod media; +pub(super) mod media_legacy; pub(super) mod membership; pub(super) mod message; pub(super) mod openid; @@ -46,6 +47,7 @@ pub(super) use directory::*; pub(super) use filter::*; pub(super) use keys::*; pub(super) use media::*; +pub(super) use media_legacy::*; pub(super) use membership::*; pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room, validate_and_add_event_id}; pub(super) use message::*; diff --git a/src/api/router.rs b/src/api/router.rs index a76cac32..05dac3ad 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -19,7 +19,7 @@ use crate::{client, server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; - let router = router + let mut router = router .ruma_route(client::get_supported_versions_route) .ruma_route(client::get_register_available_route) .ruma_route(client::register_route) @@ -138,37 +138,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::search_events_route) .ruma_route(client::turn_server_route) .ruma_route(client::send_event_to_device_route) - .ruma_route(client::get_media_config_route) - .ruma_route(client::get_media_preview_route) .ruma_route(client::create_content_route) - // legacy v1 media routes - .route( - "/_matrix/media/v1/preview_url", - get(client::get_media_preview_v1_route) - ) - .route( - "/_matrix/media/v1/config", - get(client::get_media_config_v1_route) - ) - .route( - "/_matrix/media/v1/upload", - post(client::create_content_v1_route) - ) - .route( - "/_matrix/media/v1/download/:server_name/:media_id", - get(client::get_content_v1_route) - ) - .route( - "/_matrix/media/v1/download/:server_name/:media_id/:file_name", - get(client::get_content_as_filename_v1_route) - ) - .route( - "/_matrix/media/v1/thumbnail/:server_name/:media_id", - get(client::get_content_thumbnail_v1_route) - ) - .ruma_route(client::get_content_route) - .ruma_route(client::get_content_as_filename_route) - .ruma_route(client::get_content_thumbnail_route) .ruma_route(client::get_devices_route) .ruma_route(client::get_device_route) .ruma_route(client::update_device_route) @@ -202,7 +172,7 @@ pub fn build(router: Router, server: &Server) -> Router { .route("/client/server.json", get(client::syncv3_client_server_json)); if config.allow_federation { - router + router = router .ruma_route(server::get_server_version_route) .route("/_matrix/key/v2/server", get(server::get_server_keys_route)) .route("/_matrix/key/v2/server/:key_id", get(server::get_server_keys_deprecated_route)) @@ -232,14 +202,45 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(server::well_known_server) .ruma_route(server::get_content_route) .ruma_route(server::get_content_thumbnail_route) - .route("/_conduwuit/local_user_count", get(client::conduwuit_local_user_count)) + .route("/_conduwuit/local_user_count", get(client::conduwuit_local_user_count)); } else { - router + router = router .route("/_matrix/federation/*path", any(federation_disabled)) .route("/.well-known/matrix/server", any(federation_disabled)) .route("/_matrix/key/*path", any(federation_disabled)) - .route("/_conduwuit/local_user_count", any(federation_disabled)) + .route("/_conduwuit/local_user_count", any(federation_disabled)); } + + if config.allow_legacy_media { + router = router + .ruma_route(client::get_media_config_legacy_route) + .ruma_route(client::get_media_preview_legacy_route) + .ruma_route(client::get_content_legacy_route) + .ruma_route(client::get_content_as_filename_legacy_route) + .ruma_route(client::get_content_thumbnail_legacy_route) + .route("/_matrix/media/v1/config", get(client::get_media_config_legacy_legacy_route)) + .route("/_matrix/media/v1/upload", post(client::create_content_legacy_route)) + .route( + "/_matrix/media/v1/preview_url", + get(client::get_media_preview_legacy_legacy_route), + ) + .route( + "/_matrix/media/v1/download/:server_name/:media_id", + get(client::get_content_legacy_legacy_route), + ) + .route( + "/_matrix/media/v1/download/:server_name/:media_id/:file_name", + get(client::get_content_as_filename_legacy_legacy_route), + ) + .route( + "/_matrix/media/v1/thumbnail/:server_name/:media_id", + get(client::get_content_thumbnail_legacy_legacy_route), + ); + } else { + router = router.route("/_matrix/media/*path", any(legacy_media_disabled)); + } + + router } async fn initial_sync(_uri: Uri) -> impl IntoResponse { @@ -247,3 +248,7 @@ async fn initial_sync(_uri: Uri) -> impl IntoResponse { } async fn federation_disabled() -> impl IntoResponse { err!(Config("allow_federation", "Federation is disabled.")) } + +async fn legacy_media_disabled() -> impl IntoResponse { + err!(Config("allow_legacy_media", "Unauthenticated media is disabled.")) +} diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 8836c2c9..a9469e5b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -289,6 +289,8 @@ pub struct Config { #[serde(default)] pub allow_guests_auto_join_rooms: bool, + #[serde(default = "true_fn")] + pub allow_legacy_media: bool, #[serde(default = "true_fn")] pub media_startup_check: bool, #[serde(default)] @@ -745,6 +747,7 @@ impl fmt::Display for Config { line("Media integrity checks on startup", &self.media_startup_check.to_string()); line("Media compatibility filesystem links", &self.media_compat_file_link.to_string()); line("Prune missing media from database", &self.prune_missing_media.to_string()); + line("Allow legacy (unauthenticated) media", &self.allow_legacy_media.to_string()); line("Prevent Media Downloads From", { let mut lst = vec![]; for domain in &self.prevent_media_downloads_from { diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 04436f7c..2faa13d8 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -47,6 +47,12 @@ struct Services { /// generated MXC ID (`media-id`) length pub const MXC_LENGTH: usize = 32; +/// Cache control for immutable objects. +pub const CACHE_CONTROL_IMMUTABLE: &str = "public,max-age=31536000,immutable"; + +/// Default cross-origin resource policy. +pub const CORP_CROSS_ORIGIN: &str = "cross-origin"; + #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { From ea2343850fad0651b799f802596ebf785b069455 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 11:18:57 +0000 Subject: [PATCH 0722/2091] add remote fetch suite for authenticated media Signed-off-by: Jason Volk --- src/admin/media/commands.rs | 50 +++++- src/admin/media/mod.rs | 30 +++- src/service/client/mod.rs | 6 + src/service/media/remote.rs | 301 +++++++++++++++++++++++++++++++++++- 4 files changed, 381 insertions(+), 6 deletions(-) diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 385419eb..535ad31a 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -1,5 +1,10 @@ -use conduit::{debug, info, trace, warn, Result}; -use ruma::{events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, ServerName}; +use std::time::Duration; + +use conduit::{debug, info, trace, utils::time::parse_timepoint_ago, warn, Result}; +use conduit_service::media::Dim; +use ruma::{ + events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, OwnedMxcUri, OwnedServerName, ServerName, +}; use crate::{admin_command, utils::parse_local_user_id}; @@ -266,3 +271,44 @@ pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result, timeout: u32, +) -> Result { + let mxc: Mxc<'_> = mxc.as_str().try_into()?; + let timeout = Duration::from_millis(timeout.into()); + let mut result = self + .services + .media + .fetch_remote_content(&mxc, None, server.as_deref(), timeout) + .await?; + + // Grab the length of the content before clearing it to not flood the output + let len = result.content.as_ref().expect("content").len(); + result.content.as_mut().expect("content").clear(); + + let out = format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"); + Ok(RoomMessageEventContent::notice_markdown(out)) +} + +#[admin_command] +pub(super) async fn get_remote_thumbnail( + &self, mxc: OwnedMxcUri, server: Option, timeout: u32, width: u32, height: u32, +) -> Result { + let mxc: Mxc<'_> = mxc.as_str().try_into()?; + let timeout = Duration::from_millis(timeout.into()); + let dim = Dim::new(width, height, None); + let mut result = self + .services + .media + .fetch_remote_thumbnail(&mxc, None, server.as_deref(), timeout, &dim) + .await?; + + // Grab the length of the content before clearing it to not flood the output + let len = result.content.as_ref().expect("content").len(); + result.content.as_mut().expect("content").clear(); + + let out = format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"); + Ok(RoomMessageEventContent::notice_markdown(out)) +} diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 27d75f66..5977c0fa 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -2,7 +2,7 @@ mod commands; use clap::Subcommand; use conduit::Result; -use ruma::{EventId, MxcUri, OwnedMxcUri, ServerName}; +use ruma::{EventId, MxcUri, OwnedMxcUri, OwnedServerName, ServerName}; use crate::admin_command_dispatch; @@ -60,4 +60,32 @@ pub(super) enum MediaCommand { /// The MXC URL to lookup info for. mxc: OwnedMxcUri, }, + + GetRemoteFile { + /// The MXC URL to fetch + mxc: OwnedMxcUri, + + #[arg(short, long)] + server: Option, + + #[arg(short, long, default_value("10000"))] + timeout: u32, + }, + + GetRemoteThumbnail { + /// The MXC URL to fetch + mxc: OwnedMxcUri, + + #[arg(short, long)] + server: Option, + + #[arg(short, long, default_value("10000"))] + timeout: u32, + + #[arg(short, long)] + width: u32, + + #[arg(short, long)] + height: u32, + }, } diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index 7b653939..e0f43107 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -8,6 +8,7 @@ use crate::{resolver, service}; pub struct Service { pub default: reqwest::Client, pub url_preview: reqwest::Client, + pub extern_media: reqwest::Client, pub well_known: reqwest::Client, pub federation: reqwest::Client, pub sender: reqwest::Client, @@ -30,6 +31,11 @@ impl crate::Service for Service { .redirect(redirect::Policy::limited(3)) .build()?, + extern_media: base(config)? + .dns_resolver(resolver.resolver.clone()) + .redirect(redirect::Policy::limited(3)) + .build()?, + well_known: base(config)? .dns_resolver(resolver.resolver.hooked.clone()) .connect_timeout(Duration::from_secs(config.well_known_conn_timeout)) diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index 41e87dbb..9f622fe0 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -1,9 +1,304 @@ -use std::time::Duration; +use std::{fmt::Debug, time::Duration}; use conduit::{debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, Result}; -use ruma::{api::client::media, Mxc}; +use http::header::{HeaderValue, CONTENT_DISPOSITION, CONTENT_TYPE}; +use ruma::{ + api::{ + client::{ + error::ErrorKind::{NotFound, Unrecognized}, + media, + }, + federation, + federation::authenticated_media::{Content, FileOrLocation}, + OutgoingRequest, + }, + Mxc, ServerName, UserId, +}; -use super::Dim; +use super::{Dim, FileMeta}; + +#[implement(super::Service)] +pub async fn fetch_remote_thumbnail( + &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, dim: &Dim, +) -> Result { + self.check_fetch_authorized(mxc)?; + + let result = self + .fetch_thumbnail_unauthenticated(mxc, user, server, timeout_ms, dim) + .await; + + if let Err(Error::Request(NotFound, ..)) = &result { + return self + .fetch_thumbnail_authenticated(mxc, user, server, timeout_ms, dim) + .await; + } + + result +} + +#[implement(super::Service)] +pub async fn fetch_remote_content( + &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, +) -> Result { + self.check_fetch_authorized(mxc)?; + + let result = self + .fetch_content_unauthenticated(mxc, user, server, timeout_ms) + .await; + + if let Err(Error::Request(NotFound, ..)) = &result { + return self + .fetch_content_authenticated(mxc, user, server, timeout_ms) + .await; + } + + result +} + +#[implement(super::Service)] +async fn fetch_thumbnail_authenticated( + &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, dim: &Dim, +) -> Result { + use federation::authenticated_media::get_content_thumbnail::v1::{Request, Response}; + + let request = Request { + media_id: mxc.media_id.into(), + method: dim.method.clone().into(), + width: dim.width.into(), + height: dim.height.into(), + animated: true.into(), + timeout_ms, + }; + + let Response { + content, + .. + } = self.federation_request(mxc, user, server, request).await?; + + match content { + FileOrLocation::File(content) => self.handle_thumbnail_file(mxc, user, dim, content).await, + FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await, + } +} + +#[implement(super::Service)] +async fn fetch_content_authenticated( + &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, +) -> Result { + use federation::authenticated_media::get_content::v1::{Request, Response}; + + let request = Request { + media_id: mxc.media_id.into(), + timeout_ms, + }; + + let Response { + content, + .. + } = self.federation_request(mxc, user, server, request).await?; + + match content { + FileOrLocation::File(content) => self.handle_content_file(mxc, user, content).await, + FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await, + } +} + +#[allow(deprecated)] +#[implement(super::Service)] +async fn fetch_thumbnail_unauthenticated( + &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, dim: &Dim, +) -> Result { + use media::get_content_thumbnail::v3::{Request, Response}; + + let request = Request { + allow_remote: true, + allow_redirect: true, + animated: true.into(), + method: dim.method.clone().into(), + width: dim.width.into(), + height: dim.height.into(), + server_name: mxc.server_name.into(), + media_id: mxc.media_id.into(), + timeout_ms, + }; + + let Response { + file, + content_type, + content_disposition, + .. + } = self.federation_request(mxc, user, server, request).await?; + + let content = Content { + file, + content_type, + content_disposition, + }; + + self.handle_thumbnail_file(mxc, user, dim, content).await +} + +#[allow(deprecated)] +#[implement(super::Service)] +async fn fetch_content_unauthenticated( + &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, +) -> Result { + use media::get_content::v3::{Request, Response}; + + let request = Request { + allow_remote: true, + allow_redirect: true, + server_name: mxc.server_name.into(), + media_id: mxc.media_id.into(), + timeout_ms, + }; + + let Response { + file, + content_type, + content_disposition, + .. + } = self.federation_request(mxc, user, server, request).await?; + + let content = Content { + file, + content_type, + content_disposition, + }; + + self.handle_content_file(mxc, user, content).await +} + +#[implement(super::Service)] +async fn handle_thumbnail_file( + &self, mxc: &Mxc<'_>, user: Option<&UserId>, dim: &Dim, content: Content, +) -> Result { + let content_disposition = + make_content_disposition(content.content_disposition.as_ref(), content.content_type.as_deref(), None); + + self.upload_thumbnail( + mxc, + user, + Some(&content_disposition), + content.content_type.as_deref(), + dim, + &content.file, + ) + .await + .map(|()| FileMeta { + content: Some(content.file), + content_type: content.content_type.map(Into::into), + content_disposition: Some(content_disposition), + }) +} + +#[implement(super::Service)] +async fn handle_content_file(&self, mxc: &Mxc<'_>, user: Option<&UserId>, content: Content) -> Result { + let content_disposition = + make_content_disposition(content.content_disposition.as_ref(), content.content_type.as_deref(), None); + + self.create( + mxc, + user, + Some(&content_disposition), + content.content_type.as_deref(), + &content.file, + ) + .await + .map(|()| FileMeta { + content: Some(content.file), + content_type: content.content_type.map(Into::into), + content_disposition: Some(content_disposition), + }) +} + +#[implement(super::Service)] +async fn handle_location(&self, mxc: &Mxc<'_>, user: Option<&UserId>, location: &str) -> Result { + self.location_request(location).await.map_err(|error| { + err!(Request(NotFound( + debug_warn!(%mxc, ?user, ?location, ?error, "Fetching media from location failed") + ))) + }) +} + +#[implement(super::Service)] +async fn location_request(&self, location: &str) -> Result { + let response = self + .services + .client + .extern_media + .get(location) + .send() + .await?; + + let content_type = response + .headers() + .get(CONTENT_TYPE) + .map(HeaderValue::to_str) + .and_then(Result::ok) + .map(str::to_owned); + + let content_disposition = response + .headers() + .get(CONTENT_DISPOSITION) + .map(HeaderValue::as_bytes) + .map(TryFrom::try_from) + .and_then(Result::ok); + + response + .bytes() + .await + .map(Vec::from) + .map_err(Into::into) + .map(|content| FileMeta { + content: Some(content), + content_type: content_type.clone().map(Into::into), + content_disposition: Some(make_content_disposition( + content_disposition.as_ref(), + content_type.as_deref(), + None, + )), + }) +} + +#[implement(super::Service)] +async fn federation_request( + &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, request: Request, +) -> Result +where + Request: OutgoingRequest + Send + Debug, +{ + self.services + .sending + .send_federation_request(server.unwrap_or(mxc.server_name), request) + .await + .map_err(|error| handle_federation_error(mxc, user, server, error)) +} + +// Handles and adjusts the error for the caller to determine if they should +// request the fallback endpoint or give up. +fn handle_federation_error(mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, error: Error) -> Error { + let fallback = || { + err!(Request(NotFound( + debug_error!(%mxc, ?user, ?server, ?error, "Remote media not found") + ))) + }; + + // Matrix server responses for fallback always taken. + if error.kind() == NotFound || error.kind() == Unrecognized { + return fallback(); + } + + // If we get these from any middleware we'll try the other endpoint rather than + // giving up too early. + if error.status_code().is_client_error() || error.status_code().is_redirection() { + return fallback(); + } + + // Reached for 5xx errors. This is where we don't fallback given the likelyhood + // the other endpoint will also be a 5xx and we're wasting time. + error +} #[implement(super::Service)] #[allow(deprecated)] From 1638be0339e1636e2bad818d0ecce1c2f2b5fd55 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 27 Aug 2024 11:19:57 +0000 Subject: [PATCH 0723/2091] add authenticated media client api Signed-off-by: Jason Volk --- src/api/client/media.rs | 252 +++++++++++++++++++++++++++++++--- src/api/client/unversioned.rs | 1 + src/api/router.rs | 11 +- 3 files changed, 243 insertions(+), 21 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index b8140ff7..12012711 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -1,14 +1,37 @@ +use std::time::Duration; + use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - utils::{self, content_disposition::make_content_disposition}, - Result, + err, + utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize}, + Err, Result, +}; +use conduit_service::{ + media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN, MXC_LENGTH}, + Services, +}; +use ruma::{ + api::client::{ + authenticated_media::{ + get_content, get_content_as_filename, get_content_thumbnail, get_media_config, get_media_preview, + }, + media::create_content, + }, + Mxc, UserId, }; -use conduit_service::media::MXC_LENGTH; -use ruma::{api::client::media::create_content, Mxc}; use crate::Ruma; +/// # `GET /_matrix/client/v1/media/config` +pub(crate) async fn get_media_config_route( + State(services): State, _body: Ruma, +) -> Result { + Ok(get_media_config::v1::Response { + upload_size: ruma_from_usize(services.globals.config.max_request_size), + }) +} + /// # `POST /_matrix/media/v3/upload` /// /// Permanently save media in the server. @@ -20,8 +43,11 @@ pub(crate) async fn create_content_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let content_disposition = make_content_disposition(None, body.content_type.as_deref(), body.filename.as_deref()); + let user = body.sender_user.as_ref().expect("user is authenticated"); + + let filename = body.filename.as_deref(); + let content_type = body.content_type.as_deref(); + let content_disposition = make_content_disposition(None, content_type, filename); let mxc = Mxc { server_name: services.globals.server_name(), media_id: &utils::random_string(MXC_LENGTH), @@ -29,17 +55,209 @@ pub(crate) async fn create_content_route( services .media - .create( - &mxc, - Some(sender_user), - Some(&content_disposition), - body.content_type.as_deref(), - &body.file, - ) - .await?; + .create(&mxc, Some(user), Some(&content_disposition), content_type, &body.file) + .await + .map(|()| create_content::v3::Response { + content_uri: mxc.to_string().into(), + blurhash: None, + }) +} - Ok(create_content::v3::Response { - content_uri: mxc.to_string().into(), - blurhash: None, +/// # `GET /_matrix/client/v1/media/thumbnail/{serverName}/{mediaId}` +/// +/// Load media thumbnail from our server or over federation. +#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] +pub(crate) async fn get_content_thumbnail_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let user = body.sender_user.as_ref().expect("user is authenticated"); + + let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; + + let FileMeta { + content, + content_type, + content_disposition, + } = fetch_thumbnail(&services, &mxc, user, body.timeout_ms, &dim).await?; + + Ok(get_content_thumbnail::v1::Response { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + content_disposition, }) } + +/// # `GET /_matrix/client/v1/media/download/{serverName}/{mediaId}` +/// +/// Load media from our server or over federation. +#[tracing::instrument(skip_all, fields(%client), name = "media_get")] +pub(crate) async fn get_content_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let user = body.sender_user.as_ref().expect("user is authenticated"); + + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; + + let FileMeta { + content, + content_type, + content_disposition, + } = fetch_file(&services, &mxc, user, body.timeout_ms, None).await?; + + Ok(get_content::v1::Response { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + content_disposition, + }) +} + +/// # `GET /_matrix/client/v1/media/download/{serverName}/{mediaId}/{fileName}` +/// +/// Load media from our server or over federation as fileName. +#[tracing::instrument(skip_all, fields(%client), name = "media_get_af")] +pub(crate) async fn get_content_as_filename_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let user = body.sender_user.as_ref().expect("user is authenticated"); + + let mxc = Mxc { + server_name: &body.server_name, + media_id: &body.media_id, + }; + + let FileMeta { + content, + content_type, + content_disposition, + } = fetch_file(&services, &mxc, user, body.timeout_ms, Some(&body.filename)).await?; + + Ok(get_content_as_filename::v1::Response { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + content_disposition, + }) +} + +/// # `GET /_matrix/client/v1/media/preview_url` +/// +/// Returns URL preview. +#[tracing::instrument(skip_all, fields(%client), name = "url_preview")] +pub(crate) async fn get_media_preview_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + let url = &body.url; + if !services.media.url_preview_allowed(url) { + return Err!(Request(Forbidden( + debug_warn!(%sender_user, %url, "URL is not allowed to be previewed") + ))); + } + + let preview = services.media.get_url_preview(url).await.map_err(|error| { + err!(Request(Unknown( + debug_error!(%sender_user, %url, ?error, "Failed to fetch URL preview.") + ))) + })?; + + serde_json::value::to_raw_value(&preview) + .map(get_media_preview::v1::Response::from_raw_value) + .map_err(|error| { + err!(Request(Unknown( + debug_error!(%sender_user, %url, ?error, "Failed to parse URL preview.") + ))) + }) +} + +async fn fetch_thumbnail( + services: &Services, mxc: &Mxc<'_>, user: &UserId, timeout_ms: Duration, dim: &Dim, +) -> Result { + let FileMeta { + content, + content_type, + content_disposition, + } = fetch_thumbnail_meta(services, mxc, user, timeout_ms, dim).await?; + + let content_disposition = Some(make_content_disposition( + content_disposition.as_ref(), + content_type.as_deref(), + None, + )); + + Ok(FileMeta { + content, + content_type, + content_disposition, + }) +} + +async fn fetch_file( + services: &Services, mxc: &Mxc<'_>, user: &UserId, timeout_ms: Duration, filename: Option<&str>, +) -> Result { + let FileMeta { + content, + content_type, + content_disposition, + } = fetch_file_meta(services, mxc, user, timeout_ms).await?; + + let content_disposition = Some(make_content_disposition( + content_disposition.as_ref(), + content_type.as_deref(), + filename, + )); + + Ok(FileMeta { + content, + content_type, + content_disposition, + }) +} + +async fn fetch_thumbnail_meta( + services: &Services, mxc: &Mxc<'_>, user: &UserId, timeout_ms: Duration, dim: &Dim, +) -> Result { + if let Some(filemeta) = services.media.get_thumbnail(mxc, dim).await? { + return Ok(filemeta); + } + + if services.globals.server_is_ours(mxc.server_name) { + return Err!(Request(NotFound("Local thumbnail not found."))); + } + + services + .media + .fetch_remote_thumbnail(mxc, Some(user), None, timeout_ms, dim) + .await +} + +async fn fetch_file_meta(services: &Services, mxc: &Mxc<'_>, user: &UserId, timeout_ms: Duration) -> Result { + if let Some(filemeta) = services.media.get(mxc).await? { + return Ok(filemeta); + } + + if services.globals.server_is_ours(mxc.server_name) { + return Err!(Request(NotFound("Local media not found."))); + } + + services + .media + .fetch_remote_content(mxc, Some(user), None, timeout_ms) + .await +} diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 9a8f3220..82014cce 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -42,6 +42,7 @@ pub(crate) async fn get_supported_versions_route( "v1.3".to_owned(), "v1.4".to_owned(), "v1.5".to_owned(), + "v1.11".to_owned(), ], unstable_features: BTreeMap::from_iter([ ("org.matrix.e2e_cross_signing".to_owned(), true), diff --git a/src/api/router.rs b/src/api/router.rs index 05dac3ad..536ef6ea 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -139,6 +139,11 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(client::turn_server_route) .ruma_route(client::send_event_to_device_route) .ruma_route(client::create_content_route) + .ruma_route(client::get_content_thumbnail_route) + .ruma_route(client::get_content_route) + .ruma_route(client::get_content_as_filename_route) + .ruma_route(client::get_media_preview_route) + .ruma_route(client::get_media_config_route) .ruma_route(client::get_devices_route) .ruma_route(client::get_device_route) .ruma_route(client::update_device_route) @@ -247,8 +252,6 @@ async fn initial_sync(_uri: Uri) -> impl IntoResponse { err!(Request(GuestAccessForbidden("Guest access not implemented"))) } -async fn federation_disabled() -> impl IntoResponse { err!(Config("allow_federation", "Federation is disabled.")) } +async fn legacy_media_disabled() -> impl IntoResponse { err!(Request(Forbidden("Unauthenticated media is disabled."))) } -async fn legacy_media_disabled() -> impl IntoResponse { - err!(Config("allow_legacy_media", "Unauthenticated media is disabled.")) -} +async fn federation_disabled() -> impl IntoResponse { err!(Request(Forbidden("Federation is disabled."))) } From a934a7f687be0a3040c4bc7d663d04916f1bd7f1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 28 Aug 2024 09:16:41 +0000 Subject: [PATCH 0724/2091] freeze remote media via legacy endpoints Signed-off-by: Jason Volk --- src/core/config/mod.rs | 3 +++ src/service/media/remote.rs | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index a9469e5b..71ffd7f3 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -292,6 +292,8 @@ pub struct Config { #[serde(default = "true_fn")] pub allow_legacy_media: bool, #[serde(default = "true_fn")] + pub freeze_legacy_media: bool, + #[serde(default = "true_fn")] pub media_startup_check: bool, #[serde(default)] pub media_compat_file_link: bool, @@ -748,6 +750,7 @@ impl fmt::Display for Config { line("Media compatibility filesystem links", &self.media_compat_file_link.to_string()); line("Prune missing media from database", &self.prune_missing_media.to_string()); line("Allow legacy (unauthenticated) media", &self.allow_legacy_media.to_string()); + line("Freeze legacy (unauthenticated) media", &self.freeze_legacy_media.to_string()); line("Prevent Media Downloads From", { let mut lst = vec![]; for domain in &self.prevent_media_downloads_from { diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index 9f622fe0..59846b8e 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -310,6 +310,7 @@ pub async fn fetch_remote_thumbnail_legacy( media_id: &body.media_id, }; + self.check_legacy_freeze()?; self.check_fetch_authorized(&mxc)?; let reponse = self .services @@ -342,6 +343,7 @@ pub async fn fetch_remote_thumbnail_legacy( pub async fn fetch_remote_content_legacy( &self, mxc: &Mxc<'_>, allow_redirect: bool, timeout_ms: Duration, ) -> Result { + self.check_legacy_freeze()?; self.check_fetch_authorized(mxc)?; let response = self .services @@ -391,3 +393,13 @@ fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> { Ok(()) } + +#[implement(super::Service)] +fn check_legacy_freeze(&self) -> Result<()> { + self.services + .server + .config + .freeze_legacy_media + .then_some(()) + .ok_or(err!(Request(NotFound("Remote media is frozen.")))) +} From d8a1996e241d52a46cb2291f9735b3db9e8ba28c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 28 Aug 2024 23:29:27 +0000 Subject: [PATCH 0725/2091] bump ruma for deprecation warning suppression Signed-off-by: Jason Volk --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40b88481..7d9ddc53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "assign", "js_int", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "js_int", "ruma-common", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "as_variant", "assign", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "as_variant", "base64 0.22.1", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "bytes", "http", @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "js_int", "thiserror", @@ -3113,7 +3113,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "js_int", "ruma-common", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "once_cell", "proc-macro-crate", @@ -3138,7 +3138,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "js_int", "ruma-common", @@ -3150,7 +3150,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "headers", "http", @@ -3163,7 +3163,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3179,7 +3179,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=89cee2d2d9f3776cb0a94fcea04fca456ed8a064#89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 59284ecf..e780b314 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -314,7 +314,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "89cee2d2d9f3776cb0a94fcea04fca456ed8a064" +rev = "1de0f493e8aab7e65ea78e3a079a3de10167c777" features = [ "compat", "rand", From b17589217886d8c61ff8baeac3ce06e2ff0f8615 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 29 Aug 2024 19:17:33 +0000 Subject: [PATCH 0726/2091] support MSC4180 unstable stable flag for MSC3916 Signed-off-by: Jason Volk --- src/api/client/unversioned.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 82014cce..76574949 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -53,6 +53,8 @@ pub(crate) async fn get_supported_versions_route( ("org.matrix.msc3026.busy_presence".to_owned(), true), /* busy presence status (https://github.com/matrix-org/matrix-spec-proposals/pull/3026) */ ("org.matrix.msc3827".to_owned(), true), /* filtering of /publicRooms by room type (https://github.com/matrix-org/matrix-spec-proposals/pull/3827) */ ("org.matrix.msc3575".to_owned(), true), /* sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/3575/files#r1588877046) */ + ("org.matrix.msc3916.stable".to_owned(), true), /* authenticated media (https://github.com/matrix-org/matrix-spec-proposals/pull/3916) */ + ("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */ ]), }; From 3d8757c896fdfeba1e8d9f35d698d6225ebb83a4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 29 Aug 2024 05:31:50 +0000 Subject: [PATCH 0727/2091] update complement for new media Signed-off-by: Jason Volk --- .../complement/test_results.jsonl | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index c5b7489b..0f342cc4 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -42,7 +42,7 @@ {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/knock_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/non-state_membership_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/regular_event"} -{"Action":"fail","Test":"TestContentMediaV1"} +{"Action":"pass","Test":"TestContentMediaV1"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation/good_connectivity"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation/interrupted_connectivity"} @@ -138,43 +138,43 @@ {"Action":"fail","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} -{"Action":"fail","Test":"TestLocalPngThumbnail"} -{"Action":"fail","Test":"TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint"} -{"Action":"fail","Test":"TestMediaFilenames"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII"} +{"Action":"pass","Test":"TestLocalPngThumbnail"} +{"Action":"pass","Test":"TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint"} +{"Action":"pass","Test":"TestMediaFilenames"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'_over_/_matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'ascii'_over_/_matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'_over_/_matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name;with;semicolons'_over_/_matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'_over_/_matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_file_'name_with_spaces'_over_/_matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name_over__matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_download_specifying_a_different_ASCII_file_name_over__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII/Can_upload_with_ASCII_file_name"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name_over__matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_specifying_a_different_Unicode_file_name_over__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally_over__matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_locally_over__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation_via__matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_download_with_Unicode_file_name_over_federation_via__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Can_upload_with_Unicode_file_name"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline_via__matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_as_inline_via__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline_via__matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_safe_media_types_with_parameters_as_inline_via__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments"} -{"Action":"fail","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments_via__matrix/client/v1/media/download"} +{"Action":"pass","Test":"TestMediaFilenames/Parallel/Unicode/Will_serve_unsafe_media_types_as_attachments_via__matrix/client/v1/media/download"} {"Action":"pass","Test":"TestMediaWithoutFileName"} {"Action":"pass","Test":"TestMediaWithoutFileName/parallel"} {"Action":"pass","Test":"TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_locally"} {"Action":"pass","Test":"TestMediaWithoutFileName/parallel/Can_download_without_a_file_name_over_federation"} {"Action":"pass","Test":"TestMediaWithoutFileName/parallel/Can_upload_without_a_file_name"} -{"Action":"fail","Test":"TestMediaWithoutFileNameCSMediaV1"} -{"Action":"fail","Test":"TestMediaWithoutFileNameCSMediaV1/parallel"} -{"Action":"fail","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_locally"} -{"Action":"fail","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_over_federation"} +{"Action":"pass","Test":"TestMediaWithoutFileNameCSMediaV1"} +{"Action":"pass","Test":"TestMediaWithoutFileNameCSMediaV1/parallel"} +{"Action":"pass","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_locally"} +{"Action":"pass","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_over_federation"} {"Action":"pass","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_upload_without_a_file_name"} {"Action":"fail","Test":"TestNetworkPartitionOrdering"} {"Action":"fail","Test":"TestOutboundFederationIgnoresMissingEventWithBadJSONForRoomVersion6"} @@ -182,8 +182,8 @@ {"Action":"pass","Test":"TestOutboundFederationProfile/Outbound_federation_can_query_profile_data"} {"Action":"pass","Test":"TestOutboundFederationSend"} {"Action":"pass","Test":"TestRemoteAliasRequestsUnderstandUnicode"} -{"Action":"fail","Test":"TestRemotePngThumbnail"} -{"Action":"fail","Test":"TestRemotePngThumbnail/test_/_matrix/client/v1/media_endpoint"} +{"Action":"pass","Test":"TestRemotePngThumbnail"} +{"Action":"pass","Test":"TestRemotePngThumbnail/test_/_matrix/client/v1/media_endpoint"} {"Action":"fail","Test":"TestRemotePresence"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members"} From 2837151dbaccfc989f61921c02d9489408c8afe5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 30 Aug 2024 02:54:47 +0000 Subject: [PATCH 0728/2091] redirect legacy url preview Signed-off-by: Jason Volk --- src/api/router.rs | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/api/router.rs b/src/api/router.rs index 536ef6ea..1e4a7464 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -5,13 +5,15 @@ mod request; mod response; pub mod state; +use std::str::FromStr; + use axum::{ - response::IntoResponse, + response::{IntoResponse, Redirect}, routing::{any, get, post}, Router, }; use conduit::{err, Server}; -use http::Uri; +use http::{uri, Uri}; use self::handler::RouterExt; pub(super) use self::{args::Args as Ruma, response::RumaResponse, state::State}; @@ -243,11 +245,29 @@ pub fn build(router: Router, server: &Server) -> Router { ); } else { router = router.route("/_matrix/media/*path", any(legacy_media_disabled)); + router = router.route("/_matrix/media/v3/preview_url", any(redirect_legacy_preview)); } router } +async fn redirect_legacy_preview(uri: Uri) -> impl IntoResponse { + let path = "/_matrix/client/v1/media/preview_url"; + let query = uri.query().unwrap_or_default(); + + let path_and_query = format!("{path}?{query}"); + let path_and_query = uri::PathAndQuery::from_str(&path_and_query) + .expect("Failed to build PathAndQuery for media preview redirect URI"); + + let uri = uri::Builder::new() + .path_and_query(path_and_query) + .build() + .expect("Failed to build URI for redirect") + .to_string(); + + Redirect::temporary(&uri) +} + async fn initial_sync(_uri: Uri) -> impl IntoResponse { err!(Request(GuestAccessForbidden("Guest access not implemented"))) } From 0068624cd68d7c3f1413f429b6174d95e879be5e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 30 Aug 2024 08:12:30 +0000 Subject: [PATCH 0729/2091] add exception for upload when allow_legacy_media=false Signed-off-by: Jason Volk --- src/api/router.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/api/router.rs b/src/api/router.rs index 1e4a7464..94ec5571 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -244,8 +244,12 @@ pub fn build(router: Router, server: &Server) -> Router { get(client::get_content_thumbnail_legacy_legacy_route), ); } else { - router = router.route("/_matrix/media/*path", any(legacy_media_disabled)); - router = router.route("/_matrix/media/v3/preview_url", any(redirect_legacy_preview)); + router = router + .route("/_matrix/media/v1/*path", any(legacy_media_disabled)) + .route("/_matrix/media/v3/config", any(legacy_media_disabled)) + .route("/_matrix/media/v3/download/*path", any(legacy_media_disabled)) + .route("/_matrix/media/v3/thumbnail/*path", any(legacy_media_disabled)) + .route("/_matrix/media/v3/preview_url", any(redirect_legacy_preview)); } router From 16ce62302af404021f65905c932dd81c7eb78d5e Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 24 Aug 2024 05:11:38 +0200 Subject: [PATCH 0730/2091] .editorconfig: use tabs for rust files --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.editorconfig b/.editorconfig index cea657e2..b2455005 100644 --- a/.editorconfig +++ b/.editorconfig @@ -18,3 +18,6 @@ max_line_length = 80 [*.nix] indent_size = 2 + +[*.rs] +indent_style = tab From 0a5f18ed8352b7015d052a28146749a3942ef37f Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 24 Aug 2024 05:13:43 +0200 Subject: [PATCH 0731/2091] reformat: make markdown files wrap at 80 chars --- CONTRIBUTING.md | 89 +++++-- README.md | 49 ++-- docs/appservices.md | 8 +- docs/conduwuit_coc.md | 101 ++++---- docs/configuration.md | 38 +-- docs/deploying/arch-linux.md | 14 +- docs/deploying/docker.md | 122 ++++++---- docs/deploying/freebsd.md | 9 +- docs/deploying/generic.md | 118 +++++---- docs/deploying/nixos.md | 24 +- docs/development.md | 50 ++-- docs/development/hot_reload.md | 169 ++++++++++--- docs/development/testing.md | 17 +- docs/differences.md | 423 +++++++++++++++++++++++---------- docs/introduction.md | 3 +- docs/maintenance.md | 83 +++++-- docs/troubleshooting.md | 89 +++++-- docs/turn.md | 26 +- 18 files changed, 977 insertions(+), 455 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0cc2bfef..b5fa56e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,29 +1,46 @@ # Contributing guide -This page is for about contributing to conduwuit. The [development](development.md) page may be of interest for you as well. +This page is for about contributing to conduwuit. The +[development](development.md) page may be of interest for you as well. -If you would like to work on an [issue][issues] that is not assigned, preferably ask in the Matrix room first at [#conduwuit:puppygock.gay][conduwuit-matrix], and comment on it. +If you would like to work on an [issue][issues] that is not assigned, preferably +ask in the Matrix room first at [#conduwuit:puppygock.gay][conduwuit-matrix], +and comment on it. ### Linting and Formatting -It is mandatory all your changes satisfy the lints (clippy, rustc, rustdoc, etc) and your code is formatted via the **nightly** `cargo fmt`. A lot of the `rustfmt.toml` features depend on nightly toolchain. It would be ideal if they weren't nightly-exclusive features, but they currently still are. CI's rustfmt uses nightly. +It is mandatory all your changes satisfy the lints (clippy, rustc, rustdoc, etc) +and your code is formatted via the **nightly** `cargo fmt`. A lot of the +`rustfmt.toml` features depend on nightly toolchain. It would be ideal if they +weren't nightly-exclusive features, but they currently still are. CI's rustfmt +uses nightly. -If you need to allow a lint, please make sure it's either obvious as to why (e.g. clippy saying redundant clone but it's actually required) or it has a comment saying why. Do not write inefficient code for the sake of satisfying lints. If a lint is wrong and provides a more inefficient solution or suggestion, allow the lint and mention that in a comment. +If you need to allow a lint, please make sure it's either obvious as to why +(e.g. clippy saying redundant clone but it's actually required) or it has a +comment saying why. Do not write inefficient code for the sake of satisfying +lints. If a lint is wrong and provides a more inefficient solution or +suggestion, allow the lint and mention that in a comment. ### Running CI tests locally -conduwuit's CI for tests, linting, formatting, audit, etc use [`engage`][engage]. engage can be installed from nixpkgs or `cargo install engage`. conduwuit's Nix flake devshell has the nixpkgs engage with `direnv`. Use `engage --help` for more usage details. +conduwuit's CI for tests, linting, formatting, audit, etc use +[`engage`][engage]. engage can be installed from nixpkgs or `cargo install +engage`. conduwuit's Nix flake devshell has the nixpkgs engage with `direnv`. +Use `engage --help` for more usage details. -To test, format, lint, etc that CI would do, install engage, allow the `.envrc` file using `direnv allow`, and run `engage`. +To test, format, lint, etc that CI would do, install engage, allow the `.envrc` +file using `direnv allow`, and run `engage`. -All of the tasks are defined at the [engage.toml][engage.toml] file. You can view all of them neatly by running `engage list` +All of the tasks are defined at the [engage.toml][engage.toml] file. You can +view all of them neatly by running `engage list` If you would like to run only a specific engage task group, use `just`: - `engage just ` - Example: `engage just lints` -If you would like to run a specific engage task in a specific group, use `just [TASK]`: `engage just lints cargo-fmt` +If you would like to run a specific engage task in a specific group, use `just + [TASK]`: `engage just lints cargo-fmt` The following binaries are used in [`engage.toml`][engage.toml]: @@ -43,43 +60,75 @@ The following binaries are used in [`engage.toml`][engage.toml]: ### Matrix tests -CI runs [Complement][complement], but currently does not fail if results from the checked-in results differ with the new results. If your changes are done to fix Matrix tests, note that in your pull request. If more Complement tests start failing from your changes, please review the logs (they are uploaded as artifacts) and determine if they're intended or not. +CI runs [Complement][complement], but currently does not fail if results from +the checked-in results differ with the new results. If your changes are done to +fix Matrix tests, note that in your pull request. If more Complement tests start +failing from your changes, please review the logs (they are uploaded as +artifacts) and determine if they're intended or not. -If you'd like to run Complement locally using Nix, see the [testing](docs/development/testing.md) page. +If you'd like to run Complement locally using Nix, see the +[testing](docs/development/testing.md) page. [Sytest][sytest] support will come soon. ### Writing documentation -conduwuit's website uses [`mdbook`][mdbook] and deployed via CI using GitHub Pages in the [`documentation.yml`][documentation.yml] workflow file with Nix's mdbook in the devshell. All documentation is in the `docs/` directory at the top level. The compiled mdbook website is also uploaded as an artifact. +conduwuit's website uses [`mdbook`][mdbook] and deployed via CI using GitHub +Pages in the [`documentation.yml`][documentation.yml] workflow file with Nix's +mdbook in the devshell. All documentation is in the `docs/` directory at the top +level. The compiled mdbook website is also uploaded as an artifact. To build the documentation using Nix, run: `bin/nix-build-and-cache just .#book` -The output of the mdbook generation is in `result/`. mdbooks can be opened in your browser from the individual HTML files without any web server needed. +The output of the mdbook generation is in `result/`. mdbooks can be opened in +your browser from the individual HTML files without any web server needed. ### Inclusivity and Diversity -All **MUST** code and write with inclusivity and diversity in mind. See the [following page by Google on writing inclusive code and documentation](https://developers.google.com/style/inclusive-documentation). +All **MUST** code and write with inclusivity and diversity in mind. See the +[following page by Google on writing inclusive code and +documentation](https://developers.google.com/style/inclusive-documentation). -This **EXPLICITLY** forbids usage of terms like "blacklist"/"whitelist" and "master"/"slave", [forbids gender-specific words and phrases](https://developers.google.com/style/pronouns#gender-neutral-pronouns), forbids ableist language like "sanity-check", "cripple", or "insane", and forbids culture-specific language (e.g. US-only holidays or cultures). +This **EXPLICITLY** forbids usage of terms like "blacklist"/"whitelist" and +"master"/"slave", [forbids gender-specific words and +phrases](https://developers.google.com/style/pronouns#gender-neutral-pronouns), +forbids ableist language like "sanity-check", "cripple", or "insane", and +forbids culture-specific language (e.g. US-only holidays or cultures). -No exceptions are allowed. Dependencies that may use these terms are allowed but [do not replicate the name in your functions or variables](https://developers.google.com/style/inclusive-documentation#write-around). +No exceptions are allowed. Dependencies that may use these terms are allowed but +[do not replicate the name in your functions or +variables](https://developers.google.com/style/inclusive-documentation#write-around). -In addition to language, write and code with the user experience in mind. This is software that intends to be used by everyone, so make it easy and comfortable for everyone to use. 🏳️‍⚧️ +In addition to language, write and code with the user experience in mind. This +is software that intends to be used by everyone, so make it easy and comfortable +for everyone to use. 🏳️‍⚧️ ### Variable, comment, function, etc standards -Rust's default style and standards with regards to [function names, variable names, comments](https://rust-lang.github.io/api-guidelines/naming.html), etc applies here. +Rust's default style and standards with regards to [function names, variable +names, comments](https://rust-lang.github.io/api-guidelines/naming.html), etc +applies here. ### Creating pull requests -Please try to keep contributions to the GitHub. While the mirrors of conduwuit allow for pull/merge requests, there is no guarantee I will see them in a timely manner. Additionally, please mark WIP or unfinished or incomplete PRs as drafts. This prevents me from having to ping once in a while to double check the status of it, especially when the CI completed successfully and everything so it *looks* done. +Please try to keep contributions to the GitHub. While the mirrors of conduwuit +allow for pull/merge requests, there is no guarantee I will see them in a timely +manner. Additionally, please mark WIP or unfinished or incomplete PRs as drafts. +This prevents me from having to ping once in a while to double check the status +of it, especially when the CI completed successfully and everything so it +*looks* done. -If you open a pull request on one of the mirrors, it is your responsibility to inform me about its existence. In the future I may try to solve this with more repo bots in the conduwuit Matrix room. There is no mailing list or email-patch support on the sr.ht mirror, but if you'd like to email me a git patch you can do so at `strawberry@puppygock.gay`. +If you open a pull request on one of the mirrors, it is your responsibility to +inform me about its existence. In the future I may try to solve this with more +repo bots in the conduwuit Matrix room. There is no mailing list or email-patch +support on the sr.ht mirror, but if you'd like to email me a git patch you can +do so at `strawberry@puppygock.gay`. Direct all PRs/MRs to the `main` branch. -By sending a pull request or patch, you are agreeing that your changes are allowed to be licenced under the Apache-2.0 licence and all of your conduct is in line with the Contributor's Covenant. +By sending a pull request or patch, you are agreeing that your changes are +allowed to be licenced under the Apache-2.0 licence and all of your conduct is +in line with the Contributor's Covenant. [issues]: https://github.com/girlbossceo/conduwuit/issues [conduwuit-matrix]: https://matrix.to/#/#conduwuit:puppygock.gay diff --git a/README.md b/README.md index 7da24dc9..3e0ea593 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ # conduwuit -`main` / stable: [![CI and Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml) +`main` / stable: [![CI and +Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml) - -### a very cool, featureful fork of [Conduit](https://conduit.rs/) - + ### a very cool, featureful fork of +[Conduit](https://conduit.rs/) -Visit the [Conduwuit documentation](https://conduwuit.puppyirl.gay/) for more information. +Visit the [Conduwuit documentation](https://conduwuit.puppyirl.gay/) for more +information. - -#### What is Matrix? + #### What is Matrix? [Matrix](https://matrix.org) is an open network for secure and decentralized communication. Users from every Matrix homeserver can chat with users from all @@ -18,29 +18,32 @@ to communicate with users outside of Matrix, like a community on Discord. #### What is the goal? -An efficient Matrix homeserver that's easy to set up and just works. You can install -it on a mini-computer like the Raspberry Pi to host Matrix for your family, -friends or company. +An efficient Matrix homeserver that's easy to set up and just works. You can +install it on a mini-computer like the Raspberry Pi to host Matrix for your +family, friends or company. #### Can I try it out? -An official conduwuit server ran by me is available at transfem.dev ([element.transfem.dev](https://element.transfem.dev) / [cinny.transfem.dev](https://cinny.transfem.dev)) +An official conduwuit server ran by me is available at transfem.dev +([element.transfem.dev](https://element.transfem.dev) / +[cinny.transfem.dev](https://cinny.transfem.dev)) -transfem.dev is a public homeserver that can be used, it is not a "test only homeserver". This means there are rules, so please read the rules: [https://transfem.dev/homeserver_rules.txt](https://transfem.dev/homeserver_rules.txt) +transfem.dev is a public homeserver that can be used, it is not a "test only +homeserver". This means there are rules, so please read the rules: +[https://transfem.dev/homeserver_rules.txt](https://transfem.dev/homeserver_rules.txt) -transfem.dev is also listed at [servers.joinmatrix.org](https://servers.joinmatrix.org/) +transfem.dev is also listed at +[servers.joinmatrix.org](https://servers.joinmatrix.org/) #### What is the current status? -conduwuit is a hard fork of Conduit which is in beta, meaning you can join and participate in most -Matrix rooms, but not all features are supported and you might run into bugs -from time to time. +conduwuit is a hard fork of Conduit which is in beta, meaning you can join and +participate in most Matrix rooms, but not all features are supported and you +might run into bugs from time to time. - - -#### Contact + #### Contact If you run into any question, feel free to @@ -55,7 +58,9 @@ If you run into any question, feel free to #### Logo -Original repo and Matrix room picture was from bran (<3). Current banner image and logo is directly from [this cohost post](https://cohost.org/RatBaby/post/1028290-finally-a-flag-for). +Original repo and Matrix room picture was from bran (<3). Current banner image +and logo is directly from [this cohost +post](https://cohost.org/RatBaby/post/1028290-finally-a-flag-for). #### Is it conduwuit or Conduwuit? @@ -68,5 +73,5 @@ Both, but I prefer conduwuit. - git.girlcock.ceo: - git.gay: - Codeberg: -- sourcehut: - +- sourcehut: diff --git a/docs/appservices.md b/docs/appservices.md index 839001eb..28ea9717 100644 --- a/docs/appservices.md +++ b/docs/appservices.md @@ -2,13 +2,15 @@ ## Getting help -If you run into any problems while setting up an Appservice: ask us in [#conduwuit:puppygock.gay](https://matrix.to/#/#conduwuit:puppygock.gay) or [open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new). +If you run into any problems while setting up an Appservice: ask us in +[#conduwuit:puppygock.gay](https://matrix.to/#/#conduwuit:puppygock.gay) or +[open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new). ## Set up the appservice - general instructions Follow whatever instructions are given by the appservice. This usually includes -downloading, changing its config (setting domain, homeserver url, port etc.) -and later starting it. +downloading, changing its config (setting domain, homeserver url, port etc.) and +later starting it. At some point the appservice guide should ask you to add a registration yaml file to the homeserver. In Synapse you would do this by adding the path to the diff --git a/docs/conduwuit_coc.md b/docs/conduwuit_coc.md index 1bd519c9..5f71a257 100644 --- a/docs/conduwuit_coc.md +++ b/docs/conduwuit_coc.md @@ -1,75 +1,92 @@ # conduwuit Community Code of Conduct -Welcome to the conduwuit community! We’re excited to have you here. conduwuit is a hard-fork of the Conduit homeserver, -aimed at making Matrix more accessible and inclusive for everyone. +Welcome to the conduwuit community! We’re excited to have you here. conduwuit is +a hard-fork of the Conduit homeserver, aimed at making Matrix more accessible +and inclusive for everyone. -This space is dedicated to fostering a positive, supportive, and inclusive environment for everyone. This Code of -Conduct applies to all conduwuit spaces, including any further community rooms that reference this CoC. Here are our +This space is dedicated to fostering a positive, supportive, and inclusive +environment for everyone. This Code of Conduct applies to all conduwuit spaces, +including any further community rooms that reference this CoC. Here are our guidelines to help maintain the welcoming atmosphere that sets conduwuit apart. -For the general foundational rules, please refer to the [Contributor's Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md). +For the general foundational rules, please refer to the [Contributor's +Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md). Below are additional guidelines specific to the conduwuit community. ## Our Values and Guidelines -1. **Respect and Inclusivity**: We are committed to maintaining a community where everyone feels safe and respected. - Discrimination, harassment, or hate speech of any kind will not be tolerated. Recognise that each community member - experiences the world differently based on their past experiences, background, and identity. Share your own - experiences and be open to learning about others' diverse perspectives. +1. **Respect and Inclusivity**: We are committed to maintaining a community + where everyone feels safe and respected. Discrimination, harassment, or hate +speech of any kind will not be tolerated. Recognise that each community member +experiences the world differently based on their past experiences, background, +and identity. Share your own experiences and be open to learning about others' +diverse perspectives. -2. **Positivity and Constructiveness**: Engage in constructive discussions and support each other. If you feel angry, - negative, or aggressive, take a break until you can participate in a positive and constructive manner. Process - intense feelings with a friend or in a private setting before engaging in community conversations to help maintain - a supportive and focused environment. +2. **Positivity and Constructiveness**: Engage in constructive discussions and + support each other. If you feel angry, negative, or aggressive, take a break +until you can participate in a positive and constructive manner. Process intense +feelings with a friend or in a private setting before engaging in community +conversations to help maintain a supportive and focused environment. -3. **Clarity and Understanding**: Our community includes neurodivergent individuals and those who may not appreciate - sarcasm or subtlety. Communicate clearly and kindly, avoiding sarcasm and ensuring your messages are easily - understood by all. Additionally, avoid putting the burden of education on marginalized groups by doing your own - research before asking for explanations. +3. **Clarity and Understanding**: Our community includes neurodivergent + individuals and those who may not appreciate sarcasm or subtlety. Communicate +clearly and kindly, avoiding sarcasm and ensuring your messages are easily +understood by all. Additionally, avoid putting the burden of education on +marginalized groups by doing your own research before asking for explanations. -4. **Be Open to Inclusivity**: Actively engage in conversations about making our community more inclusive. Report - discriminatory behavior to the moderators and be open to constructive feedback that aims to improve our community. - Understand that discussing discrimination and negative experiences can be emotionally taxing, so focus on the - message rather than critiquing the tone used. +4. **Be Open to Inclusivity**: Actively engage in conversations about making our + community more inclusive. Report discriminatory behavior to the moderators +and be open to constructive feedback that aims to improve our community. +Understand that discussing discrimination and negative experiences can be +emotionally taxing, so focus on the message rather than critiquing the tone +used. -5. **Commit to Inclusivity**: Building an inclusive community requires time, energy, and resources. Recognise that - addressing discrimination and bias is an ongoing process that necessitates commitment and action from all community - members. +5. **Commit to Inclusivity**: Building an inclusive community requires time, + energy, and resources. Recognise that addressing discrimination and bias is +an ongoing process that necessitates commitment and action from all community +members. ## Matrix Community -This Code of Conduct applies to the entire [conduwuit Matrix Space](https://matrix.to/#/#conduwuit-space:puppygock.gay) -and its rooms, including: +This Code of Conduct applies to the entire [conduwuit Matrix +Space](https://matrix.to/#/#conduwuit-space:puppygock.gay) and its rooms, +including: ### [#conduwuit:puppygock.gay](https://matrix.to/#/#conduwuit:puppygock.gay) -This room is for support and discussions about conduwuit. Ask questions, share insights, and help each other out. +This room is for support and discussions about conduwuit. Ask questions, share +insights, and help each other out. ### [#conduwuit-offtopic:girlboss.ceo](https://matrix.to/#/#conduwuit-offtopic:girlboss.ceo) -For off-topic community conversations about any subject. While this room allows for a wide range of topics, the same -CoC applies. Keep discussions respectful and inclusive, and avoid divisive subjects like country/world politics. -General topics, such as world events, are welcome as long as they follow the CoC. +For off-topic community conversations about any subject. While this room allows +for a wide range of topics, the same CoC applies. Keep discussions respectful +and inclusive, and avoid divisive subjects like country/world politics. General +topics, such as world events, are welcome as long as they follow the CoC. ### [#conduwuit-dev:puppygock.gay](https://matrix.to/#/#conduwuit-dev:puppygock.gay) -This room is dedicated to discussing active development of conduwuit. Posting requires an elevated power level, which -can be requested in one of the other rooms. Use this space to collaborate and innovate. +This room is dedicated to discussing active development of conduwuit. Posting +requires an elevated power level, which can be requested in one of the other +rooms. Use this space to collaborate and innovate. ## Enforcement -We have a zero-tolerance policy for violations of this Code of Conduct. If someone’s behavior makes you uncomfortable, -please report it to the moderators. Actions we may take include: +We have a zero-tolerance policy for violations of this Code of Conduct. If +someone’s behavior makes you uncomfortable, please report it to the moderators. +Actions we may take include: -1. **Warning**: A warning given directly in the room or via a private message from the moderators, identifying - the violation and requesting corrective action. -2. **Temporary Mute**: Temporary restriction from participating in discussions for a specified period to allow for - reflection and cooling off. -3. **Kick or Ban**: Egregious behavior may result in an immediate kick or ban to protect other community members. - Bans are considered permanent and will only be reversed in exceptional circumstances after proven good behavior. +1. **Warning**: A warning given directly in the room or via a private message + from the moderators, identifying the violation and requesting corrective +action. +2. **Temporary Mute**: Temporary restriction from participating in discussions + for a specified period to allow for reflection and cooling off. +3. **Kick or Ban**: Egregious behavior may result in an immediate kick or ban to + protect other community members. Bans are considered permanent and will only +be reversed in exceptional circumstances after proven good behavior. -Please highlight issues directly in rooms when possible, but if you don't feel comfortable doing that, then please send -a DM to one of the moderators directly. +Please highlight issues directly in rooms when possible, but if you don't feel +comfortable doing that, then please send a DM to one of the moderators directly. Together, let’s build a community where everyone feels valued and respected. diff --git a/docs/configuration.md b/docs/configuration.md index cf3ce00b..78646ee3 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -4,20 +4,27 @@ This chapter describes various ways to configure conduwuit. ## Basics -conduwuit uses a config file for the majority of the settings, but also supports setting individual config options via commandline. +conduwuit uses a config file for the majority of the settings, but also supports +setting individual config options via commandline. -Please refer to the [example config file](./configuration/examples.md#example-configuration) for all of those settings. +Please refer to the [example config +file](./configuration/examples.md#example-configuration) for all of those +settings. -The config file to use can be specified on the commandline when running conduwuit by specifying the -`-c`, `--config` flag. Alternatively, you can use the environment variable `CONDUWUIT_CONFIG` to specify the config -file to used. Conduit's environment variables are supported for backwards compatibility. +The config file to use can be specified on the commandline when running +conduwuit by specifying the `-c`, `--config` flag. Alternatively, you can use +the environment variable `CONDUWUIT_CONFIG` to specify the config file to used. +Conduit's environment variables are supported for backwards compatibility. ## Option commandline flag -conduwuit supports setting individual config options in TOML format from the `-O` / `--option` flag. For example, you can set your server name via `-O server_name=\"example.com\"`. +conduwuit supports setting individual config options in TOML format from the +`-O` / `--option` flag. For example, you can set your server name via `-O +server_name=\"example.com\"`. -Note that the config is parsed as TOML, and shells like bash will remove quotes. So unfortunately it is required to escape quotes if the config option takes a string. -This does not apply to options that take booleans or numbers: +Note that the config is parsed as TOML, and shells like bash will remove quotes. +So unfortunately it is required to escape quotes if the config option takes a +string. This does not apply to options that take booleans or numbers: - `--option allow_registration=true` works ✅ - `-O max_request_size=99999999` works ✅ - `-O server_name=example.com` does not work ❌ @@ -27,12 +34,15 @@ This does not apply to options that take booleans or numbers: ## Environment variables -All of the settings that are found in the config file can be specified by using environment variables. -The environment variable names should be all caps and prefixed with `CONDUWUIT_`. +All of the settings that are found in the config file can be specified by using +environment variables. The environment variable names should be all caps and +prefixed with `CONDUWUIT_`. -For example, if the setting you are changing is `max_request_size`, then the environment variable to set is -`CONDUWUIT_MAX_REQUEST_SIZE`. +For example, if the setting you are changing is `max_request_size`, then the +environment variable to set is `CONDUWUIT_MAX_REQUEST_SIZE`. -To modify config options not in the `[global]` context such as `[global.well_known]`, use the `__` suffix split: `CONDUWUIT_WELL_KNOWN__SERVER` +To modify config options not in the `[global]` context such as +`[global.well_known]`, use the `__` suffix split: `CONDUWUIT_WELL_KNOWN__SERVER` -Conduit's environment variables are supported for backwards compatibility (e.g. `CONDUIT_SERVER_NAME`). +Conduit's environment variables are supported for backwards compatibility (e.g. +`CONDUIT_SERVER_NAME`). diff --git a/docs/deploying/arch-linux.md b/docs/deploying/arch-linux.md index e65c2c92..7436e5bf 100644 --- a/docs/deploying/arch-linux.md +++ b/docs/deploying/arch-linux.md @@ -2,8 +2,14 @@ Currently conduwuit is only on the Arch User Repository (AUR). -The conduwuit AUR packages are community maintained and are not maintained by conduwuit development team, but the AUR package maintainers are in the Matrix room. Please attempt to verify your AUR package's PKGBUILD file looks fine before asking for support. +The conduwuit AUR packages are community maintained and are not maintained by +conduwuit development team, but the AUR package maintainers are in the Matrix +room. Please attempt to verify your AUR package's PKGBUILD file looks fine +before asking for support. -- [conduwuit](https://aur.archlinux.org/packages/conduwuit) - latest tagged conduwuit -- [conduwuit-git](https://aur.archlinux.org/packages/conduwuit-git) - latest git conduwuit from `main` branch -- [conduwuit-bin](https://aur.archlinux.org/packages/conduwuit-bin) - latest tagged conduwuit static binary +- [conduwuit](https://aur.archlinux.org/packages/conduwuit) - latest tagged +conduwuit +- [conduwuit-git](https://aur.archlinux.org/packages/conduwuit-git) - latest git +conduwuit from `main` branch +- [conduwuit-bin](https://aur.archlinux.org/packages/conduwuit-bin) - latest +tagged conduwuit static binary diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index 162d8a74..9fc8dc2f 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -2,20 +2,29 @@ ## Docker -To run conduwuit with Docker you can either build the image yourself or pull it from a registry. +To run conduwuit with Docker you can either build the image yourself or pull it +from a registry. ### Use a registry OCI images for conduwuit are available in the registries listed below. -| Registry | Image | Size | Notes | -| --------------- | --------------------------------------------------------------- | ----------------------------- | ---------------------- | -| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:latest][gh] | ![Image Size][shield-latest] | Stable tagged image. | -| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:latest][gl] | ![Image Size][shield-latest] | Stable tagged image. | -| Docker Hub | [docker.io/girlbossceo/conduwuit:latest][dh] | ![Image Size][shield-latest] | Stable tagged image. | -| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:main][gh] | ![Image Size][shield-main] | Stable main branch. | -| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:main][gl] | ![Image Size][shield-main] | Stable main branch. | -| Docker Hub | [docker.io/girlbossceo/conduwuit:main][dh] | ![Image Size][shield-main] | Stable main branch. | +| Registry | Image +| Size | Notes | | --------------- | +--------------------------------------------------------------- | +----------------------------- | ---------------------- | | GitHub Registry | +[ghcr.io/girlbossceo/conduwuit:latest][gh] | ![Image Size][shield-latest] | +Stable tagged image. | | GitLab Registry | +[registry.gitlab.com/conduwuit/conduwuit:latest][gl] | ![Image +Size][shield-latest] | Stable tagged image. | | Docker Hub | +[docker.io/girlbossceo/conduwuit:latest][dh] | ![Image +Size][shield-latest] | Stable tagged image. | | GitHub Registry | +[ghcr.io/girlbossceo/conduwuit:main][gh] | ![Image Size][shield-main] | +Stable main branch. | | GitLab Registry | +[registry.gitlab.com/conduwuit/conduwuit:main][gl] | ![Image +Size][shield-main] | Stable main branch. | | Docker Hub | +[docker.io/girlbossceo/conduwuit:main][dh] | ![Image +Size][shield-main] | Stable main branch. | [dh]: https://hub.docker.com/r/girlbossceo/conduwuit [gh]: https://github.com/girlbossceo/conduwuit/pkgs/container/conduwuit @@ -25,9 +34,7 @@ OCI images for conduwuit are available in the registries listed below. Use -```bash -docker image pull -``` +```bash docker image pull ``` to pull it to your machine. @@ -35,61 +42,77 @@ to pull it to your machine. When you have the image you can simply run it with -```bash -docker run -d -p 8448:6167 \ - -v db:/var/lib/conduwuit/ \ - -e CONDUWUIT_SERVER_NAME="your.server.name" \ - -e CONDUWUIT_DATABASE_BACKEND="rocksdb" \ - -e CONDUWUIT_ALLOW_REGISTRATION=false \ - --name conduit +```bash +docker run -d -p 8448:6167 \ + -v db:/var/lib/conduwuit/ \ + -e CONDUWUIT_SERVER_NAME="your.server.name" \ + -e CONDUWUIT_DATABASE_BACKEND="rocksdb" \ + -e CONDUWUIT_ALLOW_REGISTRATION=false \ + --name conduit ``` or you can use [docker compose](#docker-compose). -The `-d` flag lets the container run in detached mode. You may supply an optional `conduwuit.toml` config file, the example config can be found [here](../configuration/examples.md). -You can pass in different env vars to change config values on the fly. You can even configure conduwuit completely by using env vars. For an overview of possible -values, please take a look at the [`docker-compose.yml`](docker-compose.yml) file. +The `-d` flag lets the container run in detached mode. You may supply an +optional `conduwuit.toml` config file, the example config can be found +[here](../configuration/examples.md). You can pass in different env vars to +change config values on the fly. You can even configure conduwuit completely by +using env vars. For an overview of possible values, please take a look at the +[`docker-compose.yml`](docker-compose.yml) file. -If you just want to test conduwuit for a short time, you can use the `--rm` flag, which will clean up everything related to your container after you stop it. +If you just want to test conduwuit for a short time, you can use the `--rm` +flag, which will clean up everything related to your container after you stop +it. ### Docker-compose -If the `docker run` command is not for you or your setup, you can also use one of the provided `docker-compose` files. +If the `docker run` command is not for you or your setup, you can also use one +of the provided `docker-compose` files. Depending on your proxy setup, you can use one of the following files; -- If you already have a `traefik` instance set up, use [`docker-compose.for-traefik.yml`](docker-compose.for-traefik.yml) -- If you don't have a `traefik` instance set up and would like to use it, use [`docker-compose.with-traefik.yml`](docker-compose.with-traefik.yml) -- If you want a setup that works out of the box with `caddy-docker-proxy`, use [`docker-compose.with-caddy.yml`](docker-compose.with-caddy.yml) and replace all `example.com` placeholders with your own domain +- If you already have a `traefik` instance set up, use +[`docker-compose.for-traefik.yml`](docker-compose.for-traefik.yml) +- If you don't have a `traefik` instance set up and would like to use it, use +[`docker-compose.with-traefik.yml`](docker-compose.with-traefik.yml) +- If you want a setup that works out of the box with `caddy-docker-proxy`, use +[`docker-compose.with-caddy.yml`](docker-compose.with-caddy.yml) and replace all +`example.com` placeholders with your own domain - For any other reverse proxy, use [`docker-compose.yml`](docker-compose.yml) -When picking the traefik-related compose file, rename it so it matches `docker-compose.yml`, and -rename the override file to `docker-compose.override.yml`. Edit the latter with the values you want -for your server. +When picking the traefik-related compose file, rename it so it matches +`docker-compose.yml`, and rename the override file to +`docker-compose.override.yml`. Edit the latter with the values you want for your +server. -When picking the `caddy-docker-proxy` compose file, it's important to first create the `caddy` network before spinning up the containers: +When picking the `caddy-docker-proxy` compose file, it's important to first +create the `caddy` network before spinning up the containers: -```bash -docker network create caddy -``` +```bash docker network create caddy ``` -After that, you can rename it so it matches `docker-compose.yml` and spin up the containers! +After that, you can rename it so it matches `docker-compose.yml` and spin up the +containers! Additional info about deploying conduwuit can be found [here](generic.md). ### Build -To build the conduwuit image with docker-compose, you first need to open and modify the `docker-compose.yml` file. There you need to comment the `image:` option and uncomment the `build:` option. Then call docker compose with: +To build the conduwuit image with docker-compose, you first need to open and +modify the `docker-compose.yml` file. There you need to comment the `image:` +option and uncomment the `build:` option. Then call docker compose with: -```bash +```bash docker compose up ``` -This will also start the container right afterwards, so if want it to run in detached mode, you also should use the `-d` flag. +This will also start the container right afterwards, so if want it to run in +detached mode, you also should use the `-d` flag. ### Run -If you already have built the image or want to use one from the registries, you can just start the container and everything else in the compose file in detached mode with: +If you already have built the image or want to use one from the registries, you +can just start the container and everything else in the compose file in detached +mode with: ```bash docker compose up -d @@ -99,18 +122,21 @@ docker compose up -d ### Use Traefik as Proxy -As a container user, you probably know about Traefik. It is a easy to use reverse proxy for making -containerized app and services available through the web. With the two provided files, +As a container user, you probably know about Traefik. It is a easy to use +reverse proxy for making containerized app and services available through the +web. With the two provided files, [`docker-compose.for-traefik.yml`](docker-compose.for-traefik.yml) (or [`docker-compose.with-traefik.yml`](docker-compose.with-traefik.yml)) and -[`docker-compose.override.yml`](docker-compose.override.yml), it is equally easy to deploy -and use conduwuit, with a little caveat. If you already took a look at the files, then you should have -seen the `well-known` service, and that is the little caveat. Traefik is simply a proxy and -loadbalancer and is not able to serve any kind of content, but for conduwuit to federate, we need to -either expose ports `443` and `8448` or serve two endpoints `.well-known/matrix/client` and -`.well-known/matrix/server`. +[`docker-compose.override.yml`](docker-compose.override.yml), it is equally easy +to deploy and use conduwuit, with a little caveat. If you already took a look at +the files, then you should have seen the `well-known` service, and that is the +little caveat. Traefik is simply a proxy and loadbalancer and is not able to +serve any kind of content, but for conduwuit to federate, we need to either +expose ports `443` and `8448` or serve two endpoints `.well-known/matrix/client` +and `.well-known/matrix/server`. -With the service `well-known` we use a single `nginx` container that will serve those two files. +With the service `well-known` we use a single `nginx` container that will serve +those two files. ## Voice communication diff --git a/docs/deploying/freebsd.md b/docs/deploying/freebsd.md index 44a77566..4ac83515 100644 --- a/docs/deploying/freebsd.md +++ b/docs/deploying/freebsd.md @@ -1,8 +1,11 @@ # conduwuit for FreeBSD -conduwuit at the moment does not provide FreeBSD builds. Building conduwuit on FreeBSD requires a specific environment variable to use the -system prebuilt RocksDB library instead of rust-rocksdb / rust-librocksdb-sys which does *not* work and will cause a build error or coredump. +conduwuit at the moment does not provide FreeBSD builds. Building conduwuit on +FreeBSD requires a specific environment variable to use the system prebuilt +RocksDB library instead of rust-rocksdb / rust-librocksdb-sys which does *not* +work and will cause a build error or coredump. Use the following environment variable: `ROCKSDB_LIB_DIR=/usr/local/lib` -Such example commandline with it can be: `ROCKSDB_LIB_DIR=/usr/local/lib cargo build --release` +Such example commandline with it can be: `ROCKSDB_LIB_DIR=/usr/local/lib cargo +build --release` diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 18016028..5a5eb5d1 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -2,125 +2,140 @@ > ## Getting help > -> If you run into any problems while setting up conduwuit, ask us -> in `#conduwuit:puppygock.gay` or [open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new). +> If you run into any problems while setting up conduwuit, ask us in +> `#conduwuit:puppygock.gay` or [open an issue on +> GitHub](https://github.com/girlbossceo/conduwuit/issues/new). ## Installing conduwuit -You may simply download the binary that fits your machine. Run `uname -m` to see what you need. +You may simply download the binary that fits your machine. Run `uname -m` to see +what you need. -Prebuilt fully static musl binaries can be downloaded from the latest tagged release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or `main` CI branch workflow artifact output. These also include Debian packages. These binaries have jemalloc and io_uring statically linked and included with them. +Prebuilt fully static musl binaries can be downloaded from the latest tagged +release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or +`main` CI branch workflow artifact output. These also include Debian packages. +These binaries have jemalloc and io_uring statically linked and included with +them. -Alternatively, you may compile the binary yourself. We recommend using [Lix](https://lix.systems) to build conduwuit as this has the most guaranteed +Alternatively, you may compile the binary yourself. We recommend using +[Lix](https://lix.systems) to build conduwuit as this has the most guaranteed reproducibiltiy and easiest to get a build environment and output going. -Otherwise, follow standard Rust project build guides (installing git and cloning the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + libclang for RocksDB, installing liburing for io_uring and RocksDB, etc). +Otherwise, follow standard Rust project build guides (installing git and cloning +the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + +libclang for RocksDB, installing liburing for io_uring and RocksDB, etc). ## Adding a conduwuit user -While conduwuit can run as any user it is better to use dedicated users for different services. This also allows -you to make sure that the file permissions are correctly set up. +While conduwuit can run as any user it is better to use dedicated users for +different services. This also allows you to make sure that the file permissions +are correctly set up. In Debian or Fedora/RHEL, you can use this command to create a conduwuit user: ```bash -sudo adduser --system conduwuit --group --disabled-login --no-create-home +sudo adduser --system conduwuit --group --disabled-login --no-create-home ``` For distros without `adduser`: -```bash -sudo useradd -r --shell /usr/bin/nologin --no-create-home conduwuit -``` +```bash sudo useradd -r --shell /usr/bin/nologin --no-create-home conduwuit ``` ## Forwarding ports in the firewall or the router -conduwuit uses the ports 443 and 8448 both of which need to be open in the firewall. +conduwuit uses the ports 443 and 8448 both of which need to be open in the +firewall. -If conduwuit runs behind a router or in a container and has a different public IP address than the host system these public ports need to be forwarded directly or indirectly to the port mentioned in the config. +If conduwuit runs behind a router or in a container and has a different public +IP address than the host system these public ports need to be forwarded directly +or indirectly to the port mentioned in the config. ## Setting up a systemd service -The systemd unit for conduwuit can be found [here](../configuration/examples.md#example-systemd-unit-file). You may need to change the `ExecStart=` path to where you placed the conduwuit binary. +The systemd unit for conduwuit can be found +[here](../configuration/examples.md#example-systemd-unit-file). You may need to +change the `ExecStart=` path to where you placed the conduwuit binary. ## Creating the conduwuit configuration file -Now we need to create the conduwuit's config file in `/etc/conduwuit/conduwuit.toml`. The example config can be found at [conduwuit-example.toml](../configuration/examples.md).**Please take a moment to read it. You need to change at least the server name.** +Now we need to create the conduwuit's config file in +`/etc/conduwuit/conduwuit.toml`. The example config can be found at +[conduwuit-example.toml](../configuration/examples.md).**Please take a moment to +read it. You need to change at least the server name.** RocksDB is the only supported database backend. ## Setting the correct file permissions -If you are using a dedicated user for conduwuit, you will need to allow it to read the config. To do that you can run this: +If you are using a dedicated user for conduwuit, you will need to allow it to +read the config. To do that you can run this: ```bash -sudo chown -R root:root /etc/conduwuit -sudo chmod -R 755 /etc/conduwuit +sudo chown -R root:root /etc/conduwuit sudo chmod -R 755 /etc/conduwuit ``` If you use the default database path you also need to run this: -```bash -sudo mkdir -p /var/lib/conduwuit/ -sudo chown -R conduwuit:conduwuit /var/lib/conduwuit/ -sudo chmod 700 /var/lib/conduwuit/ +```bash +sudo mkdir -p /var/lib/conduwuit/ sudo chown -R conduwuit:conduwuit +/var/lib/conduwuit/ sudo chmod 700 /var/lib/conduwuit/ ``` ## Setting up the Reverse Proxy -Refer to the documentation or various guides online of your chosen reverse proxy software. A [Caddy](https://caddyserver.com/) example will be provided as this is the recommended reverse proxy for new users and is very trivial to use (handles TLS, reverse proxy headers, etc transparently with proper defaults). +Refer to the documentation or various guides online of your chosen reverse proxy +software. A [Caddy](https://caddyserver.com/) example will be provided as this +is the recommended reverse proxy for new users and is very trivial to use +(handles TLS, reverse proxy headers, etc transparently with proper defaults). -Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization header, making federation non-functional. If using Apache, you need to use `nocanon` to prevent this. +Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization +header, making federation non-functional. If using Apache, you need to use +`nocanon` to prevent this. ### Caddy -Create `/etc/caddy/conf.d/conduwuit_caddyfile` and enter this (substitute for your server name). +Create `/etc/caddy/conf.d/conduwuit_caddyfile` and enter this (substitute for +your server name). ```caddy -your.server.name, your.server.name:8448 { - # TCP - reverse_proxy 127.0.0.1:6167 - - # UNIX socket - #reverse_proxy unix//run/conduwuit/conduwuit.sock +your.server.name, your.server.name:8448 { # TCP reverse_proxy + 127.0.0.1:6167 + # UNIX socket + #reverse_proxy unix//run/conduwuit/conduwuit.sock } ``` That's it! Just start and enable the service and you're set. -```bash -sudo systemctl enable --now caddy -``` +```bash sudo systemctl enable --now caddy ``` ## You're done Now you can start conduwuit with: -```bash -sudo systemctl start conduwuit -``` +```bash sudo systemctl start conduwuit ``` Set it to start automatically when your system boots with: -```bash -sudo systemctl enable conduwuit -``` +```bash sudo systemctl enable conduwuit ``` ## How do I know it works? -You can open [a Matrix client](https://matrix.org/ecosystem/clients), enter your homeserver and try to register. +You can open [a Matrix client](https://matrix.org/ecosystem/clients), enter your +homeserver and try to register. -You can also use these commands as a quick health check (replace `your.server.name`). +You can also use these commands as a quick health check (replace +`your.server.name`). -```bash -$ curl https://your.server.name/_conduwuit/server_version +```bash $ curl https://your.server.name/_conduwuit/server_version -# If using port 8448 -$ curl https://your.server.name:8448/_conduwuit/server_version -``` +# If using port 8448 $ curl +https://your.server.name:8448/_conduwuit/server_version ``` -- To check if your server can talk with other homeservers, you can use the [Matrix Federation Tester](https://federationtester.matrix.org/). - If you can register but cannot join federated rooms check your config again and also check if the port 8448 is open and forwarded correctly. +- To check if your server can talk with other homeservers, you can use the +[Matrix Federation Tester](https://federationtester.matrix.org/). If you can +register but cannot join federated rooms check your config again and also check +if the port 8448 is open and forwarded correctly. # What's next? @@ -130,4 +145,5 @@ For Audio/Video call functionality see the [TURN Guide](../turn.md). ## Appservices -If you want to set up an appservice, take a look at the [Appservice Guide](../appservices.md). +If you want to set up an appservice, take a look at the [Appservice +Guide](../appservices.md). diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index 9f247ac4..4a591315 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -9,31 +9,29 @@ conduwuit can be acquired by [Lix][lix] from various places: A binary cache for conduwuit that the CI/CD publishes to is available at the following places (both are the same just different names): -``` -https://attic.kennel.juneis.dog/conduit +``` https://attic.kennel.juneis.dog/conduit conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= https://attic.kennel.juneis.dog/conduwuit -conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= -``` +conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= ``` -The binary caches have been recreated recently due to attic issues. The old public keys were: +The binary caches have been recreated recently due to attic issues. The old +public keys were: -``` -conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= +``` conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= -conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= -``` +conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= ``` -If specifying a URL in your flake, please use the GitHub remote: `github:girlbossceo/conduwuit` +If specifying a URL in your flake, please use the GitHub remote: +`github:girlbossceo/conduwuit` The `flake.nix` and `default.nix` do not (currently) provide a NixOS module, so (for now) [`services.matrix-conduit`][module] from Nixpkgs should be used to configure conduwuit. -If you want to run the latest code, you should get conduwuit from the `flake.nix` -or `default.nix` and set [`services.matrix-conduit.package`][package] -appropriately. +If you want to run the latest code, you should get conduwuit from the +`flake.nix` or `default.nix` and set +[`services.matrix-conduit.package`][package] appropriately. [lix]: https://lix.systems/ [module]: https://search.nixos.org/options?channel=unstable&query=services.matrix-conduit diff --git a/docs/development.md b/docs/development.md index 62a03e2f..fbcfb433 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,29 +4,43 @@ Information about developing the project. If you are only interested in using it, you can safely ignore this section. If you plan on contributing, see the [contributor's guide](contributing.md). -## List of forked dependencies -During conduwuit development, we have had to fork some dependencies to support our use-cases in some areas. This ranges from things said upstream project won't accept for any reason, faster-paced development (unresponsive or slow upstream), conduwuit-specific usecases, or lack of time to upstream some things. +## List of forked dependencies During conduwuit development, we have had to fork +some dependencies to support our use-cases in some areas. This ranges from +things said upstream project won't accept for any reason, faster-paced +development (unresponsive or slow upstream), conduwuit-specific usecases, or +lack of time to upstream some things. -- [ruma/ruma][1]: - various performance improvements, more features, faster-paced development, client/server interop hacks upstream won't accept, etc -- [facebook/rocksdb][2]: - liburing build fixes, GCC build fix, and logging callback C API for Rust tracing integration -- [tikv/jemallocator][3]: - musl builds seem to be broken on upstream -- [zyansheep/rustyline-async][4]: - tab completion callback and `CTRL+\` signal quit event for CLI -- [rust-rocksdb/rust-rocksdb][5]: - [`@zaidoon1`'s][8] fork has quicker updates, more up to date dependencies. Our changes fix musl build issues, Rust part of the logging callback C API, removes unnecessary `gtest` include, and uses our RocksDB and jemallocator -- [tokio-rs/tracing][6]: - Implements `Clone` for `EnvFilter` to support dynamically changing tracing envfilter's alongside other logging/metrics things +- [ruma/ruma][1]: - various performance +improvements, more features, faster-paced development, client/server interop +hacks upstream won't accept, etc +- [facebook/rocksdb][2]: - liburing +build fixes, GCC build fix, and logging callback C API for Rust tracing +integration +- [tikv/jemallocator][3]: - musl +builds seem to be broken on upstream +- [zyansheep/rustyline-async][4]: + - tab completion callback and +`CTRL+\` signal quit event for CLI +- [rust-rocksdb/rust-rocksdb][5]: + - [`@zaidoon1`'s][8] fork +has quicker updates, more up to date dependencies. Our changes fix musl build +issues, Rust part of the logging callback C API, removes unnecessary `gtest` +include, and uses our RocksDB and jemallocator +- [tokio-rs/tracing][6]: - Implements +`Clone` for `EnvFilter` to support dynamically changing tracing envfilter's +alongside other logging/metrics things ## Debugging with `tokio-console` -[`tokio-console`][7] can be a useful tool for debugging and profiling. To make -a `tokio-console`-enabled build of conduwuit, enable the `tokio_console` feature, -disable the default `release_max_log_level` feature, and set the -`--cfg tokio_unstable` flag to enable experimental tokio APIs. A build might -look like this: +[`tokio-console`][7] can be a useful tool for debugging and profiling. To make a +`tokio-console`-enabled build of conduwuit, enable the `tokio_console` feature, +disable the default `release_max_log_level` feature, and set the `--cfg +tokio_unstable` flag to enable experimental tokio APIs. A build might look like +this: -```bash -RUSTFLAGS="--cfg tokio_unstable" cargo build \ - --release \ - --no-default-features \ - --features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console +```bash RUSTFLAGS="--cfg tokio_unstable" cargo build \ --release \ +--no-default-features \ +--features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console ``` [1]: https://github.com/ruma/ruma/ diff --git a/docs/development/hot_reload.md b/docs/development/hot_reload.md index 426e7c07..018eb4b3 100644 --- a/docs/development/hot_reload.md +++ b/docs/development/hot_reload.md @@ -2,89 +2,190 @@ ### Summary -When developing in debug-builds with the nightly toolchain, conduwuit is modular using dynamic libraries and various parts of the application are hot-reloadable while the server is running: http api handlers, admin commands, services, database, etc. These are all split up into individual workspace crates as seen in the `src/` directory. Changes to sourcecode in a crate rebuild that crate and subsequent crates depending on it. Reloading then occurs for the changed crates. +When developing in debug-builds with the nightly toolchain, conduwuit is modular +using dynamic libraries and various parts of the application are hot-reloadable +while the server is running: http api handlers, admin commands, services, +database, etc. These are all split up into individual workspace crates as seen +in the `src/` directory. Changes to sourcecode in a crate rebuild that crate and +subsequent crates depending on it. Reloading then occurs for the changed crates. -Release builds still produce static binaries which are unaffected. Rust's soundness guarantees are in full force. Thus you cannot hot-reload release binaries. +Release builds still produce static binaries which are unaffected. Rust's +soundness guarantees are in full force. Thus you cannot hot-reload release +binaries. ### Requirements -Currently, this development setup only works on x86_64 and aarch64 Linux glibc. [musl explicitly does not support hot reloadable libraries, and does not implement `dlclose`][2]. macOS does not fully support our usage of `RTLD_GLOBAL` possibly due to some thread-local issues. [This Rust issue][3] may be of relevance, specifically [this comment][4]. It may be possible to get it working on only very modern macOS versions such as at least Sonoma, as currently loading dylibs is supported, but not unloading them in our setup, and the cited comment mentions an Apple WWDC confirming there have been TLS changes to somewhat make this possible. +Currently, this development setup only works on x86_64 and aarch64 Linux glibc. +[musl explicitly does not support hot reloadable libraries, and does not +implement `dlclose`][2]. macOS does not fully support our usage of `RTLD_GLOBAL` +possibly due to some thread-local issues. [This Rust issue][3] may be of +relevance, specifically [this comment][4]. It may be possible to get it working +on only very modern macOS versions such as at least Sonoma, as currently loading +dylibs is supported, but not unloading them in our setup, and the cited comment +mentions an Apple WWDC confirming there have been TLS changes to somewhat make +this possible. -As mentioned above this requires the nightly toolchain. This is due to reliance on various Cargo.toml features that are only available on nightly, most specifically `RUSTFLAGS` in Cargo.toml. Some of the implementation could also be simpler based on other various nightly features. We hope lots of nightly features start making it out of nightly sooner as there have been dozens of very helpful features that have been stuck in nightly ("unstable") for at least 5+ years that would make this simpler. We encourage greater community consensus to move these features into stability. +As mentioned above this requires the nightly toolchain. This is due to reliance +on various Cargo.toml features that are only available on nightly, most +specifically `RUSTFLAGS` in Cargo.toml. Some of the implementation could also be +simpler based on other various nightly features. We hope lots of nightly +features start making it out of nightly sooner as there have been dozens of very +helpful features that have been stuck in nightly ("unstable") for at least 5+ +years that would make this simpler. We encourage greater community consensus to +move these features into stability. -This currently only works on x86_64/aarch64 Linux with a glibc C library. musl C library, macOS, and likely other host architectures are not supported (if other architectures work, feel free to let us know and/or make a PR updating this). This should work on GNU ld and lld (rust-lld) and gcc/clang, however if you happen to have linker issues it's recommended to try using `mold` or `gold` linkers, and please let us know in the [conduwuit Matrix room][7] the linker error and what linker solved this issue so we can figure out a solution. Ideally there should be minimal friction to using this, and in the future a build script (`build.rs`) may be suitable to making this easier to use if the capabilities allow us. +This currently only works on x86_64/aarch64 Linux with a glibc C library. musl C +library, macOS, and likely other host architectures are not supported (if other +architectures work, feel free to let us know and/or make a PR updating this). +This should work on GNU ld and lld (rust-lld) and gcc/clang, however if you +happen to have linker issues it's recommended to try using `mold` or `gold` +linkers, and please let us know in the [conduwuit Matrix room][7] the linker +error and what linker solved this issue so we can figure out a solution. Ideally +there should be minimal friction to using this, and in the future a build script +(`build.rs`) may be suitable to making this easier to use if the capabilities +allow us. ### Usage As of 19 May 2024, the instructions for using this are: -0. Have patience. Don't hesitate to join the [conduwuit Matrix room][7] to receive help using this. As indicated by the various rustflags used and some of the interesting issues linked at the bottom, this is definitely not something the Rust ecosystem or toolchain is used to doing. +0. Have patience. Don't hesitate to join the [conduwuit Matrix room][7] to + receive help using this. As indicated by the various rustflags used and some +of the interesting issues linked at the bottom, this is definitely not something +the Rust ecosystem or toolchain is used to doing. -1. Install the nightly toolchain using rustup. You may need to use `rustup override set nightly` in your local conduwuit directory, or use `cargo +nightly` for all actions. +1. Install the nightly toolchain using rustup. You may need to use `rustup + override set nightly` in your local conduwuit directory, or use `cargo ++nightly` for all actions. 2. Uncomment `cargo-features` at the top level / root Cargo.toml -3. Scroll down to the `# Developer profile` section and uncomment ALL the rustflags for each dev profile and their respective packages. +3. Scroll down to the `# Developer profile` section and uncomment ALL the + rustflags for each dev profile and their respective packages. -4. In each workspace crate's Cargo.toml (everything under `src/*` AND `deps/rust-rocksdb/Cargo.toml`), uncomment the `dylib` crate type under `[lib]`. +4. In each workspace crate's Cargo.toml (everything under `src/*` AND + `deps/rust-rocksdb/Cargo.toml`), uncomment the `dylib` crate type under +`[lib]`. -5. Due to [this rpath issue][5], you must export the `LD_LIBRARY_PATH` environment variable to your nightly Rust toolchain library directory. If using rustup (hopefully), use this: `export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/` +5. Due to [this rpath issue][5], you must export the `LD_LIBRARY_PATH` + environment variable to your nightly Rust toolchain library directory. If +using rustup (hopefully), use this: `export +LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.rustup/toolchains/nightly-x86_64-unknown-linux-gnu/lib/` -6. Start the server. You can use `cargo +nightly run` for this along with the standard. +6. Start the server. You can use `cargo +nightly run` for this along with the + standard. 7. Make some changes where you need to. -8. In a separate terminal window in the same directory (or using a terminal multiplexer like tmux), run the *build* Cargo command `cargo +nightly build`. Cargo should only rebuild what was changed / what's necessary, so it should not be rebuilding all the crates. +8. In a separate terminal window in the same directory (or using a terminal + multiplexer like tmux), run the *build* Cargo command `cargo +nightly build`. + Cargo should only rebuild what was changed / what's necessary, so it should + not be rebuilding all the crates. -9. In your conduwuit server terminal, hit/send `CTRL+C` signal. This will tell conduwuit to find which libraries need to be reloaded, and reloads them as necessary. +9. In your conduwuit server terminal, hit/send `CTRL+C` signal. This will tell + conduwuit to find which libraries need to be reloaded, and reloads them as + necessary. -10. If there were no errors, it will tell you it successfully reloaded `#` modules, and your changes should now be visible. Repeat 7 - 9 as needed. +10. If there were no errors, it will tell you it successfully reloaded `#` + modules, and your changes should now be visible. Repeat 7 - 9 as needed. -To shutdown conduwuit in this setup, hit/send `CTRL+\`. Normal builds still shutdown with `CTRL+C` as usual. +To shutdown conduwuit in this setup, hit/send `CTRL+\`. Normal builds still +shutdown with `CTRL+C` as usual. -Steps 1 - 5 are the initial first-time steps for using this. To remove the hot reload setup, revert/comment all the Cargo.toml changes. +Steps 1 - 5 are the initial first-time steps for using this. To remove the hot +reload setup, revert/comment all the Cargo.toml changes. -As mentioned in the requirements section, if you happen to have some linker issues, try using the `-fuse-ld=` rustflag and specify mold or gold in all the `rustflags` definitions in the top level Cargo.toml, and please let us know in the [conduwuit Matrix room][7] the problem. mold can be installed typically through your distro, and gold is provided by the binutils package. +As mentioned in the requirements section, if you happen to have some linker +issues, try using the `-fuse-ld=` rustflag and specify mold or gold in all the +`rustflags` definitions in the top level Cargo.toml, and please let us know in +the [conduwuit Matrix room][7] the problem. mold can be installed typically +through your distro, and gold is provided by the binutils package. -It's possible a helper script can be made to do all of this, or most preferably a specially made build script (build.rs). `cargo watch` support will be implemented soon which will eliminate the need to manually run `cargo build` all together. +It's possible a helper script can be made to do all of this, or most preferably +a specially made build script (build.rs). `cargo watch` support will be +implemented soon which will eliminate the need to manually run `cargo build` all +together. ### Addendum -Conduit was inherited as a single crate without modularity or reloading in its design. Reasonable partitioning and abstraction allowed a split into several crates, though many circular dependencies had to be corrected. The resulting crates now form a directed graph as depicted in figures below. The interfacing between these crates is still extremely broad which is not mitigable. +Conduit was inherited as a single crate without modularity or reloading in its +design. Reasonable partitioning and abstraction allowed a split into several +crates, though many circular dependencies had to be corrected. The resulting +crates now form a directed graph as depicted in figures below. The interfacing +between these crates is still extremely broad which is not mitigable. -Initially [hot_lib_reload][6] was investigated but found appropriate for a project designed with modularity through limited interfaces, not a large and complex existing codebase. Instead a bespoke solution built directly on [libloading][8] satisfied our constraints. This required relatively minimal modifications and zero maintenance burden compared to what would be required otherwise. The technical difference lies with relocation processing: we leverage global bindings (`RTLD_GLOBAL`) in a very intentional way. Most libraries and off-the-shelf module systems (such as [hot_lib_reload][6]) restrict themselves to local bindings (`RTLD_LOCAL`). This allows them to release software to multiple platforms with much greater consistency, but at the cost of burdening applications to explicitly manage these bindings. In our case with an optional feature for developers, we shrug any such requirement to enjoy the cost/benefit on platforms where global relocations are properly cooperative. +Initially [hot_lib_reload][6] was investigated but found appropriate for a +project designed with modularity through limited interfaces, not a large and +complex existing codebase. Instead a bespoke solution built directly on +[libloading][8] satisfied our constraints. This required relatively minimal +modifications and zero maintenance burden compared to what would be required +otherwise. The technical difference lies with relocation processing: we leverage +global bindings (`RTLD_GLOBAL`) in a very intentional way. Most libraries and +off-the-shelf module systems (such as [hot_lib_reload][6]) restrict themselves +to local bindings (`RTLD_LOCAL`). This allows them to release software to +multiple platforms with much greater consistency, but at the cost of burdening +applications to explicitly manage these bindings. In our case with an optional +feature for developers, we shrug any such requirement to enjoy the cost/benefit +on platforms where global relocations are properly cooperative. -To make use of `RTLD_GLOBAL` the application has to be oriented as a directed acyclic graph. The primary rule is simple and illustrated in the figure below: **no crate is allowed to call a function or use a variable from a crate below it.** +To make use of `RTLD_GLOBAL` the application has to be oriented as a directed +acyclic graph. The primary rule is simple and illustrated in the figure below: +**no crate is allowed to call a function or use a variable from a crate below +it.** -![conduwuit's dynamic library setup diagram - created by Jason Volk](assets/libraries.png) +![conduwuit's dynamic library setup diagram - created by Jason +Volk](assets/libraries.png) -When a symbol is referenced between crates they become bound: **crates cannot be unloaded until their calling crates are first unloaded.** Thus we start the reloading process from the crate which has no callers. There is a small problem though: the first crate is called by the base executable itself! This is solved by using an `RTLD_LOCAL` binding for just one link between the main executable and the first crate, freeing the executable from all modules as no global binding ever occurs between them. +When a symbol is referenced between crates they become bound: **crates cannot be +unloaded until their calling crates are first unloaded.** Thus we start the +reloading process from the crate which has no callers. There is a small problem +though: the first crate is called by the base executable itself! This is solved +by using an `RTLD_LOCAL` binding for just one link between the main executable +and the first crate, freeing the executable from all modules as no global +binding ever occurs between them. -![conduwuit's reload and load order diagram - created by Jason Volk](assets/reload_order.png) +![conduwuit's reload and load order diagram - created by Jason +Volk](assets/reload_order.png) -Proper resource management is essential for reliable reloading to occur. This is a very basic ask in RAII-idiomatic Rust and the exposure to reloading hazards is remarkably low, generally stemming from poor patterns and practices. Unfortunately static analysis doesn't enforce reload-safety programmatically (though it could one day), for now hazards can be avoided by knowing a few basic do's and dont's: +Proper resource management is essential for reliable reloading to occur. This is +a very basic ask in RAII-idiomatic Rust and the exposure to reloading hazards is +remarkably low, generally stemming from poor patterns and practices. +Unfortunately static analysis doesn't enforce reload-safety programmatically +(though it could one day), for now hazards can be avoided by knowing a few basic +do's and dont's: -1. Understand that code is memory. Just like one is forbidden from referencing free'd memory, one must not transfer control to free'd code. Exposure to this is primarily from two things: +1. Understand that code is memory. Just like one is forbidden from referencing + free'd memory, one must not transfer control to free'd code. Exposure to this +is primarily from two things: -- Callbacks, which this project makes very little use of. -- Async tasks, which are addressed below. + - Callbacks, which this project makes very little use of. + - Async tasks, which are addressed below. -2. Tie all resources to a scope or object lifetime with greatest possible symmetry (locality). For our purposes this applies to code resources, which means async blocks and tokio tasks. +2. Tie all resources to a scope or object lifetime with greatest possible +symmetry (locality). For our purposes this applies to code resources, which +means async blocks and tokio tasks. -- **Never spawn a task without receiving and storing its JoinHandle**. -- **Always wait on join handles** before leaving a scope or in another cleanup function called by an owning scope. + - **Never spawn a task without receiving and storing its JoinHandle**. + - **Always wait on join handles** before leaving a scope or in another cleanup + function called by an owning scope. 3. Know any minor specific quirks documented in code or here: -- Don't use `tokio::spawn`, instead use our `Handle` in `core/server.rs`, which is reachable in most of the codebase via `services()` or other state. This is due to some bugs or assumptions made in tokio, as it happens in `unsafe {}` blocks, which are mitigated by circumventing some thread-local variables. Using runtime handles is good practice in any case. + - Don't use `tokio::spawn`, instead use our `Handle` in `core/server.rs`, which + is reachable in most of the codebase via `services()` or other state. This is + due to some bugs or assumptions made in tokio, as it happens in `unsafe {}` + blocks, which are mitigated by circumventing some thread-local variables. Using + runtime handles is good practice in any case. The initial implementation PR is available [here][1]. ### Interesting related issues/bugs - [DT_RUNPATH produced in binary with rpath = true is wrong (cargo)][5] -- [Disabling MIR Optimization in Rust Compilation (cargo)](https://internals.rust-lang.org/t/disabling-mir-optimization-in-rust-compilation/19066/5) -- [Workspace-level metadata (cargo-deb)](https://github.com/kornelski/cargo-deb/issues/68) +- [Disabling MIR Optimization in Rust Compilation +(cargo)](https://internals.rust-lang.org/t/disabling-mir-optimization-in-rust-compilation/19066/5) +- [Workspace-level metadata +(cargo-deb)](https://github.com/kornelski/cargo-deb/issues/68) [1]: https://github.com/girlbossceo/conduwuit/pull/387 [2]: https://wiki.musl-libc.org/functional-differences-from-glibc.html#Unloading-libraries diff --git a/docs/development/testing.md b/docs/development/testing.md index f3fb7d9d..00f8e258 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -5,15 +5,18 @@ Have a look at [Complement's repository][complement] for an explanation of what it is. -To test against Complement, with [Lix][lix] and direnv installed and set up, you can: +To test against Complement, with [Lix][lix] and direnv installed and set up, you +can: -* Run `./bin/complement "$COMPLEMENT_SRC" ./path/to/logs.jsonl ./path/to/results.jsonl` - to build a Complement image, run the tests, and output the logs and results - to the specified paths. This will also output the OCI image at `result` +* Run `./bin/complement "$COMPLEMENT_SRC" ./path/to/logs.jsonl +./path/to/results.jsonl` to build a Complement image, run the tests, and output +the logs and results to the specified paths. This will also output the OCI image +at `result` * Run `nix build .#complement` from the root of the repository to just build a - Complement OCI image outputted to `result` (it's a `.tar.gz` file) -* Or download the latest Complement OCI image from the CI workflow artifacts output - from the commit/revision you want to test (e.g. from main) [here][ci-workflows] +Complement OCI image outputted to `result` (it's a `.tar.gz` file) +* Or download the latest Complement OCI image from the CI workflow artifacts +output from the commit/revision you want to test (e.g. from main) +[here][ci-workflows] [lix]: https://lix.systems/ [ci-workflows]: https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml?query=event%3Apush+is%3Asuccess+actor%3Agirlbossceo diff --git a/docs/differences.md b/docs/differences.md index 7dfbedc1..6815d248 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -1,4 +1,8 @@ -#### **Note: This list may not up to date. There are rapidly more and more improvements, fixes, changes, etc being made that it is becoming more difficult to maintain this list. I recommend that you give conduwuit a try and see the differences for yourself. If you have any concerns, feel free to join the conduwuit Matrix room and ask any pre-usage questions.** +#### **Note: This list may not up to date. There are rapidly more and more +improvements, fixes, changes, etc being made that it is becoming more difficult +to maintain this list. I recommend that you give conduwuit a try and see the +differences for yourself. If you have any concerns, feel free to join the +conduwuit Matrix room and ask any pre-usage questions.** ### list of features, bug fixes, etc that conduwuit does that Conduit does not @@ -6,192 +10,371 @@ Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! ## Performance -- Concurrency support for individual homeserver key fetching for faster remote room joins and room joins that will error less frequently -- Send `Cache-Control` response header with `immutable` and 1 year cache length for all media requests (download and thumbnail) to instruct clients to cache media, and reduce server load from media requests that could be otherwise cached -- Add feature flags and config options to enable/build with zstd, brotli, and/or gzip HTTP body compression (response and request) -- Eliminate all usage of the thread-blocking `getaddrinfo(3)` call upon DNS queries, significantly improving federation latency/ping and cache DNS results (NXDOMAINs, successful queries, etc) using hickory-dns / hickory-resolver +- Concurrency support for individual homeserver key fetching for faster remote +room joins and room joins that will error less frequently +- Send `Cache-Control` response header with `immutable` and 1 year cache length +for all media requests (download and thumbnail) to instruct clients to cache +media, and reduce server load from media requests that could be otherwise cached +- Add feature flags and config options to enable/build with zstd, brotli, and/or +gzip HTTP body compression (response and request) +- Eliminate all usage of the thread-blocking `getaddrinfo(3)` call upon DNS +queries, significantly improving federation latency/ping and cache DNS results +(NXDOMAINs, successful queries, etc) using hickory-dns / hickory-resolver - Enable HTTP/2 support on all requests -- Vastly improve RocksDB default settings to use new features that help with performance significantly, uses settings tailored to SSDs, various ways to tweak RocksDB, and a conduwuit setting to tell RocksDB to use settings that are tailored to HDDs or slow spinning rust storage or buggy filesystems. +- Vastly improve RocksDB default settings to use new features that help with +performance significantly, uses settings tailored to SSDs, various ways to tweak +RocksDB, and a conduwuit setting to tell RocksDB to use settings that are +tailored to HDDs or slow spinning rust storage or buggy filesystems. - Implement database flush and cleanup conduwuit operations when using RocksDB -- Implement RocksDB write buffer corking and coalescing in database write-heavy areas -- Perform connection pooling and keepalives where necessary to significantly improve federation performance and latency -- Various config options to tweak connection pooling, request timeouts, connection timeouts, DNS timeouts and settings, etc with good defaults which also help huge with performance via reusing connections and retrying where needed +- Implement RocksDB write buffer corking and coalescing in database write-heavy +areas +- Perform connection pooling and keepalives where necessary to significantly +improve federation performance and latency +- Various config options to tweak connection pooling, request timeouts, +connection timeouts, DNS timeouts and settings, etc with good defaults which +also help huge with performance via reusing connections and retrying where +needed - Properly get and use the amount of parallelism / tokio workers -- Implement building conduwuit with jemalloc (which extends to the RocksDB jemalloc feature for maximum gains) or hardened_malloc light variant, and io_uring support, and produce CI builds with jemalloc and io_uring by default for performance (Nix doesn't seem to build [hardened_malloc-rs](https://github.com/girlbossceo/hardened_malloc-rs) properly) -- Add support for caching DNS results with hickory-dns / hickory-resolver in conduwuit (not a replacement for a proper resolver cache, but still far better than nothing), also properly falls back on TCP for UDP errors or if a SRV response is too large -- Add config option for using DNS over TCP, and config option for controlling A/AAAA record lookup strategy (e.g. don't query AAAA records if you don't have IPv6 connectivity) -- Overall significant database, Client-Server, and federation performance and latency improvements (check out the ping room leaderboards if you don't believe me :>) -- Add config options for RocksDB compression and bottommost compression, including choosing the algorithm and compression level -- Use [loole](https://github.com/mahdi-shojaee/loole) MPSC channels instead of tokio MPSC channels for huge performance boosts in sending channels (mainly relevant for federation) and presence channels -- Use `tracing`/`log`'s `release_max_level_info` feature to improve performance, build speeds, binary size, and CPU usage in release builds by avoid compiling debug/trace log level macros that users will generally never use (can be disabled with a build-time feature flag) -- Remove some unnecessary checks on EDU handling for incoming transactions, effectively speeding them up -- Simplify, dedupe, etc huge chunks of the codebase, including some that were unnecessary overhead, binary bloats, or preventing compiler/linker optimisations -- Implement zero-copy RocksDB database accessors, substantially improving performance caused by unnecessary memory allocations +- Implement building conduwuit with jemalloc (which extends to the RocksDB +jemalloc feature for maximum gains) or hardened_malloc light variant, and +io_uring support, and produce CI builds with jemalloc and io_uring by default +for performance (Nix doesn't seem to build +[hardened_malloc-rs](https://github.com/girlbossceo/hardened_malloc-rs) +properly) +- Add support for caching DNS results with hickory-dns / hickory-resolver in +conduwuit (not a replacement for a proper resolver cache, but still far better +than nothing), also properly falls back on TCP for UDP errors or if a SRV +response is too large +- Add config option for using DNS over TCP, and config option for controlling +A/AAAA record lookup strategy (e.g. don't query AAAA records if you don't have +IPv6 connectivity) +- Overall significant database, Client-Server, and federation performance and +latency improvements (check out the ping room leaderboards if you don't believe +me :>) +- Add config options for RocksDB compression and bottommost compression, +including choosing the algorithm and compression level +- Use [loole](https://github.com/mahdi-shojaee/loole) MPSC channels instead of +tokio MPSC channels for huge performance boosts in sending channels (mainly +relevant for federation) and presence channels +- Use `tracing`/`log`'s `release_max_level_info` feature to improve performance, +build speeds, binary size, and CPU usage in release builds by avoid compiling +debug/trace log level macros that users will generally never use (can be +disabled with a build-time feature flag) +- Remove some unnecessary checks on EDU handling for incoming transactions, +effectively speeding them up +- Simplify, dedupe, etc huge chunks of the codebase, including some that were +unnecessary overhead, binary bloats, or preventing compiler/linker optimisations +- Implement zero-copy RocksDB database accessors, substantially improving +performance caused by unnecessary memory allocations ## General Fixes/Features -- Add legacy Element client hack fixing password changes and deactivations on legacy Element Android/iOS due to usage of an unspecced `user` field for UIAA -- Raise and improve all the various request timeouts making some things like room joins and client bugs error less or none at all than they should, and make them all user configurable +- Add legacy Element client hack fixing password changes and deactivations on +legacy Element Android/iOS due to usage of an unspecced `user` field for UIAA +- Raise and improve all the various request timeouts making some things like +room joins and client bugs error less or none at all than they should, and make +them all user configurable - Add missing `reason` field to user ban events (`/ban`) -- Safer and cleaner shutdowns across incoming/outgoing requests (graceful shutdown) and the database -- Stop sending `make_join` requests on room joins if 15 servers respond with `M_UNSUPPORTED_ROOM_VERSION` or `M_INVALID_ROOM_VERSION` -- Stop sending `make_join` requests if 50 servers cannot provide `make_join` for us -- Respect *most* client parameters for `/media/` requests (`allow_redirect` still needs work) -- Return joined member count of rooms for push rules/conditions instead of a hardcoded value of 10 -- Make `CONDUIT_CONFIG` optional, relevant for container users that configure only by environment variables and no longer need to set `CONDUIT_CONFIG` to an empty string. -- Allow HEAD and PATCH (MSC4138) HTTP requests in CORS for clients (despite not being explicity mentioned in Matrix spec, HTTP spec says all HEAD requests need to behave the same as GET requests, Synapse supports HEAD requests) +- Safer and cleaner shutdowns across incoming/outgoing requests (graceful +shutdown) and the database +- Stop sending `make_join` requests on room joins if 15 servers respond with +`M_UNSUPPORTED_ROOM_VERSION` or `M_INVALID_ROOM_VERSION` +- Stop sending `make_join` requests if 50 servers cannot provide `make_join` for +us +- Respect *most* client parameters for `/media/` requests (`allow_redirect` +still needs work) +- Return joined member count of rooms for push rules/conditions instead of a +hardcoded value of 10 +- Make `CONDUIT_CONFIG` optional, relevant for container users that configure +only by environment variables and no longer need to set `CONDUIT_CONFIG` to an +empty string. +- Allow HEAD and PATCH (MSC4138) HTTP requests in CORS for clients (despite not +being explicity mentioned in Matrix spec, HTTP spec says all HEAD requests need +to behave the same as GET requests, Synapse supports HEAD requests) - Fix using conduwuit with flake-compat on NixOS -- Resolve and remove some "features" from upstream that result in concurrency hazards, exponential backoff issues, or arbitrary performance limiters -- Find more servers for outbound federation `/hierarchy` requests instead of just the room ID server name -- Support for suggesting servers to join through at `/_matrix/client/v3/directory/room/{roomAlias}` -- Support for suggesting servers to join through us at `/_matrix/federation/v1/query/directory` +- Resolve and remove some "features" from upstream that result in concurrency +hazards, exponential backoff issues, or arbitrary performance limiters +- Find more servers for outbound federation `/hierarchy` requests instead of +just the room ID server name +- Support for suggesting servers to join through at +`/_matrix/client/v3/directory/room/{roomAlias}` +- Support for suggesting servers to join through us at +`/_matrix/federation/v1/query/directory` - Misc edge-case search fixes (e.g. potentially missing some events) -- Misc `/sync` fixes (e.g. returning unnecessary data or incorrect/invalid responses) -- Add `replaces_state` and `prev_sender` in `unsigned` for state event changes which primarily makes Element's "See history" button on a state event functional -- Fix Conduit not allowing incoming federation requests for various world readable rooms +- Misc `/sync` fixes (e.g. returning unnecessary data or incorrect/invalid +responses) +- Add `replaces_state` and `prev_sender` in `unsigned` for state event changes +which primarily makes Element's "See history" button on a state event functional +- Fix Conduit not allowing incoming federation requests for various world +readable rooms - Fix Conduit not respecting the client-requested file name on media requests -- Prevent sending junk / non-membership events to `/send_join` and `/send_leave` endpoints -- Only allow the requested membership type on `/send_join` and `/send_leave` endpoints (e.g. don't allow leave memberships on join endpoints) +- Prevent sending junk / non-membership events to `/send_join` and `/send_leave` +endpoints +- Only allow the requested membership type on `/send_join` and `/send_leave` +endpoints (e.g. don't allow leave memberships on join endpoints) - Prevent state key impersonation on `/send_join` and `/send_leave` endpoints -- Validate `X-Matrix` origin and request body `"origin"` field on incoming transactions -- Add `GET /_matrix/client/v1/register/m.login.registration_token/validity` endpoint -- Explicitly define support for sliding sync at `/_matrix/client/versions` (`org.matrix.msc3575`) +- Validate `X-Matrix` origin and request body `"origin"` field on incoming +transactions +- Add `GET /_matrix/client/v1/register/m.login.registration_token/validity` +endpoint +- Explicitly define support for sliding sync at `/_matrix/client/versions` +(`org.matrix.msc3575`) - Fix seeing empty status messages on user presences ## Moderation -- (Also see [Admin Room](#admin-room) for all the admin commands pertaining to moderation, there's a lot!) +- (Also see [Admin Room](#admin-room) for all the admin commands pertaining to +moderation, there's a lot!) - Add support for room banning/blocking by ID using admin command -- Add support for serving `support` well-known from `[global.well_known]` (MSC1929) (`/.well-known/matrix/support`) -- Config option to forbid publishing rooms to the room directory (`lockdown_public_room_directory`) except for admins -- Admin commands to delete room aliases and unpublish rooms from our room directory -- For all [`/report`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3roomsroomidreporteventid) requests: check if the reported event ID belongs to the reported room ID, raise report reasoning character limit to 750, fix broken formatting, make a small delayed random response per spec suggestion on privacy, and check if the sender user is in the reported room. +- Add support for serving `support` well-known from `[global.well_known]` +(MSC1929) (`/.well-known/matrix/support`) +- Config option to forbid publishing rooms to the room directory +(`lockdown_public_room_directory`) except for admins +- Admin commands to delete room aliases and unpublish rooms from our room +directory +- For all +[`/report`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3roomsroomidreporteventid) +requests: check if the reported event ID belongs to the reported room ID, raise +report reasoning character limit to 750, fix broken formatting, make a small +delayed random response per spec suggestion on privacy, and check if the sender +user is in the reported room. - Support blocking servers from downloading remote media from, returning a 404 -- Don't allow `m.call.invite` events to be sent in public rooms (prevents calling the entire room) -- On new public room creations, only allow moderators to send `m.call.invite`, `org.matrix.msc3401.call`, and `org.matrix.msc3401.call.member` events to prevent unprivileged users from calling the entire room -- Add support for a "global ACLs" feature (`forbidden_remote_server_names`) that blocks inbound remote room invites, room joins by room ID on server name, room joins by room alias on server name, incoming federated joins, and incoming federated room directory requests. This is very helpful for blocking servers that are purely toxic/bad and serve no value in allowing our users to suffer from things like room invite spam or such. Please note that this is not a substitute for room ACLs. -- Add support for a config option to forbid our local users from sending federated room directory requests for (`forbidden_remote_room_directory_server_names`). Similar to above, useful for blocking servers that help prevent our users from wandering into bad areas of Matrix via room directories of those malicious servers. -- Add config option for auto remediating/deactivating local non-admin users who attempt to join bad/forbidden rooms (`auto_deactivate_banned_room_attempts`) -- Deactivating users will remove their profile picture, blurhash, display name, and leave all rooms by default just like Synapse and for additional privacy +- Don't allow `m.call.invite` events to be sent in public rooms (prevents +calling the entire room) +- On new public room creations, only allow moderators to send `m.call.invite`, +`org.matrix.msc3401.call`, and `org.matrix.msc3401.call.member` events to +prevent unprivileged users from calling the entire room +- Add support for a "global ACLs" feature (`forbidden_remote_server_names`) that +blocks inbound remote room invites, room joins by room ID on server name, room +joins by room alias on server name, incoming federated joins, and incoming +federated room directory requests. This is very helpful for blocking servers +that are purely toxic/bad and serve no value in allowing our users to suffer +from things like room invite spam or such. Please note that this is not a +substitute for room ACLs. +- Add support for a config option to forbid our local users from sending +federated room directory requests for +(`forbidden_remote_room_directory_server_names`). Similar to above, useful for +blocking servers that help prevent our users from wandering into bad areas of +Matrix via room directories of those malicious servers. +- Add config option for auto remediating/deactivating local non-admin users who +attempt to join bad/forbidden rooms (`auto_deactivate_banned_room_attempts`) +- Deactivating users will remove their profile picture, blurhash, display name, +and leave all rooms by default just like Synapse and for additional privacy - Reject some EDUs from ACL'd users such as read receipts and typing indicators ## Privacy/Security -- Add config option for device name federation with a privacy-friendly default (disabled) -- Add config option for requiring authentication to the `/publicRooms` endpoint (room directory) with a default enabled for privacy -- Add config option for federating `/publicRooms` endpoint (room directory) to other servers with a default disabled for privacy -- Uses proper `argon2` crate by RustCrypto instead of questionable `rust-argon2` crate +- Add config option for device name federation with a privacy-friendly default +(disabled) +- Add config option for requiring authentication to the `/publicRooms` endpoint +(room directory) with a default enabled for privacy +- Add config option for federating `/publicRooms` endpoint (room directory) to +other servers with a default disabled for privacy +- Uses proper `argon2` crate by RustCrypto instead of questionable `rust-argon2` +crate - Generate passwords with 25 characters instead of 15 -- Config option `ip_range_denylist` to support refusing to send requests (typically federation) to specific IP ranges, typically RFC 1918, non-routable, testnet, etc addresses like Synapse for security (note: this is not a guaranteed protection, and you should be using a firewall with zones if you want guaranteed protection as doing this on the application level is prone to bypasses). -- Config option to block non-admin users from sending room invites or receiving remote room invites. Admin users are still allowed. +- Config option `ip_range_denylist` to support refusing to send requests +(typically federation) to specific IP ranges, typically RFC 1918, non-routable, +testnet, etc addresses like Synapse for security (note: this is not a guaranteed +protection, and you should be using a firewall with zones if you want guaranteed +protection as doing this on the application level is prone to bypasses). +- Config option to block non-admin users from sending room invites or receiving +remote room invites. Admin users are still allowed. - Config option to disable incoming and/or outgoing remote read receipts - Config option to disable incoming and/or outgoing remote typing indicators -- Config option to disable incoming, outgoing, and/or local presence and for timing out remote users -- Sanitise file names for the `Content-Disposition` header for all media requests (thumbnails, downloads, uploads) -- Media repository on handling `Content-Disposition` and `Content-Type` is fully spec compliant and secured -- Send secure default HTTP headers such as a strong restrictive CSP (see MSC4149), deny iframes, disable `X-XSS-Protection`, disable interest cohort in `Permission-Policy`, etc to mitigate any potential attack surface such as from untrusted media +- Config option to disable incoming, outgoing, and/or local presence and for +timing out remote users +- Sanitise file names for the `Content-Disposition` header for all media +requests (thumbnails, downloads, uploads) +- Media repository on handling `Content-Disposition` and `Content-Type` is fully +spec compliant and secured +- Send secure default HTTP headers such as a strong restrictive CSP (see +MSC4149), deny iframes, disable `X-XSS-Protection`, disable interest cohort in +`Permission-Policy`, etc to mitigate any potential attack surface such as from +untrusted media ## Administration/Logging -- Commandline argument to specify the path to a config file instead of relying on `CONDUIT_CONFIG` +- Commandline argument to specify the path to a config file instead of relying +on `CONDUIT_CONFIG` - Revamped admin room infrastructure and commands -- Substantially clean up, improve, and fix logging (less noisy dead server logging, registration attempts, more useful troubleshooting logging, proper error propagation, etc) -- Configurable RocksDB logging (`LOG` files) with proper defaults (rotate, max size, verbosity, etc) to stop LOG files from accumulating so much -- Explicit startup error if your configuration allows open registration without a token or such like Synapse with a way to bypass it if needed -- Replace the lightning bolt emoji option with support for setting any arbitrary text (e.g. another emoji) to suffix to all new user registrations, with a conduwuit default of "🏳️‍⚧️" +- Substantially clean up, improve, and fix logging (less noisy dead server +logging, registration attempts, more useful troubleshooting logging, proper +error propagation, etc) +- Configurable RocksDB logging (`LOG` files) with proper defaults (rotate, max +size, verbosity, etc) to stop LOG files from accumulating so much +- Explicit startup error if your configuration allows open registration without +a token or such like Synapse with a way to bypass it if needed +- Replace the lightning bolt emoji option with support for setting any arbitrary +text (e.g. another emoji) to suffix to all new user registrations, with a +conduwuit default of "🏳️‍⚧️" - Implement config option to auto join rooms upon registration - Warn on unknown config options specified -- Add `/_conduwuit/server_version` route to return the version of conduwuit without relying on the federation API `/_matrix/federation/v1/version` -- Add `/_conduwuit/local_user_count` route to return the amount of registered active local users on your homeserver *if federation is enabled* -- Add configurable RocksDB recovery modes to aid in recovering corrupted RocksDB databases -- Support config options via `CONDUWUIT_` prefix and accessing non-global struct config options with the `__` split (e.g. `CONDUWUIT_WELL_KNOWN__SERVER`) +- Add `/_conduwuit/server_version` route to return the version of conduwuit +without relying on the federation API `/_matrix/federation/v1/version` +- Add `/_conduwuit/local_user_count` route to return the amount of registered +active local users on your homeserver *if federation is enabled* +- Add configurable RocksDB recovery modes to aid in recovering corrupted RocksDB +databases +- Support config options via `CONDUWUIT_` prefix and accessing non-global struct +config options with the `__` split (e.g. `CONDUWUIT_WELL_KNOWN__SERVER`) - Add support for listening on multiple TCP ports and multiple addresses - **Opt-in** Sentry.io telemetry and metrics, mainly used for crash reporting -- Log the client IP on various requests such as registrations, banned room join attempts, logins, deactivations, federation transactions, etc +- Log the client IP on various requests such as registrations, banned room join +attempts, logins, deactivations, federation transactions, etc - Fix Conduit dropping some remote server federation response errors ## Maintenance/Stability - GitLab CI ported to GitHub Actions -- Add support for the Matrix spec compliance test suite [Complement](https://github.com/matrix-org/complement/) via the Nix flake and various other fixes for it -- Implement running and diff'ing Complement results in CI and error if any mismatch occurs to prevent large cases of conduwuit regressions -- Repo is (officially) mirrored to GitHub, GitLab, git.gay, git.girlcock.ceo, sourcehut, and Codeberg (see README.md for their links) -- Docker container images published to GitLab Container Registry, GitHub Container Registry, and Dockerhub -- Extensively revamp the example config to be extremely helpful and useful to both new users and power users -- Fixed every single clippy (default lints) and rustc warnings, including some that were performance related or potential safety issues / unsoundness +- Add support for the Matrix spec compliance test suite +[Complement](https://github.com/matrix-org/complement/) via the Nix flake and +various other fixes for it +- Implement running and diff'ing Complement results in CI and error if any +mismatch occurs to prevent large cases of conduwuit regressions +- Repo is (officially) mirrored to GitHub, GitLab, git.gay, git.girlcock.ceo, +sourcehut, and Codeberg (see README.md for their links) +- Docker container images published to GitLab Container Registry, GitHub +Container Registry, and Dockerhub +- Extensively revamp the example config to be extremely helpful and useful to +both new users and power users +- Fixed every single clippy (default lints) and rustc warnings, including some +that were performance related or potential safety issues / unsoundness - Add a **lot** of other clippy and rustc lints and a rustfmt.toml file -- Repo uses [Renovate](https://docs.renovatebot.com/), [Trivy](https://github.com/aquasecurity/trivy-action), and keeps ALL dependencies as up to date as possible -- Purge unmaintained/irrelevant/broken database backends (heed, sled, persy) and other unnecessary code or overhead +- Repo uses [Renovate](https://docs.renovatebot.com/), +[Trivy](https://github.com/aquasecurity/trivy-action), and keeps ALL +dependencies as up to date as possible +- Purge unmaintained/irrelevant/broken database backends (heed, sled, persy) and +other unnecessary code or overhead - webp support for images - Add cargo audit support to CI - Add documentation lints via lychee and markdownlint-cli to CI -- CI tests for all sorts of feature matrixes (jemalloc, non-defaullt, all features, etc) -- Add static and dynamic linking smoke tests in CI to prevent any potential linking regressions for Complement, static binaries, Nix devshells, etc -- Add timestamp by commit date when building OCI images for keeping image build reproducibility and still have a meaningful "last modified date" for OCI image +- CI tests for all sorts of feature matrixes (jemalloc, non-defaullt, all +features, etc) +- Add static and dynamic linking smoke tests in CI to prevent any potential +linking regressions for Complement, static binaries, Nix devshells, etc +- Add timestamp by commit date when building OCI images for keeping image build +reproducibility and still have a meaningful "last modified date" for OCI image - Add timestamp by commit date via `SOURCE_DATE_EPOCH` for Debian packages -- Startup check if conduwuit running in a container and is listening on 127.0.0.1 (generally containers are using NAT networking and 0.0.0.0 is the intended listening address) -- Add a panic catcher layer to return panic messages in HTTP responses if a panic occurs -- Add full compatibility support for SHA256 media file names instead of base64 file names to overcome filesystem file name length limitations (OS error file name too long) while still retaining upstream database compatibility -- Remove SQLite support due to being very poor performance, difficult to maintain against RocksDB, and is a blocker to significantly improved database code +- Startup check if conduwuit running in a container and is listening on +127.0.0.1 (generally containers are using NAT networking and 0.0.0.0 is the +intended listening address) +- Add a panic catcher layer to return panic messages in HTTP responses if a +panic occurs +- Add full compatibility support for SHA256 media file names instead of base64 +file names to overcome filesystem file name length limitations (OS error file +name too long) while still retaining upstream database compatibility +- Remove SQLite support due to being very poor performance, difficult to +maintain against RocksDB, and is a blocker to significantly improved database +code ## Admin Room -- Add support for a console CLI interface that can issue admin commands and output them in your terminal -- Add support for an admin-user-only commandline admin room interface that can be issued in any room with the `\\!admin` or `\!admin` prefix and returns the response as yourself in the same room -- Add admin commands for uptime, server startup, server shutdown, and server restart -- Fix admin room handler to not panic/crash if the admin room command response fails (e.g. too large message) -- Add command to dynamically change conduwuit's tracing log level filter on the fly +- Add support for a console CLI interface that can issue admin commands and +output them in your terminal +- Add support for an admin-user-only commandline admin room interface that can +be issued in any room with the `\\!admin` or `\!admin` prefix and returns the +response as yourself in the same room +- Add admin commands for uptime, server startup, server shutdown, and server +restart +- Fix admin room handler to not panic/crash if the admin room command response +fails (e.g. too large message) +- Add command to dynamically change conduwuit's tracing log level filter on the +fly - Add admin command to fetch a server's `/.well-known/matrix/support` file -- Add debug admin command to force update user device lists (could potentially resolve some E2EE flukes) -- Implement **RocksDB online backups**, listing RocksDB backups, and listing database file counts all via admin commands -- Add various database visibility commands such as being able to query the getters and iterators used in conduwuit, a very helpful online debugging utility +- Add debug admin command to force update user device lists (could potentially +resolve some E2EE flukes) +- Implement **RocksDB online backups**, listing RocksDB backups, and listing +database file counts all via admin commands +- Add various database visibility commands such as being able to query the +getters and iterators used in conduwuit, a very helpful online debugging utility - Forbid the admin room from being made public or world readable history - Add `!admin` as a way to call the admin bot -- Extend clear cache admin command to support clearing more caches such as DNS and TLS name overrides -- Admin debug command to send a federation request/ping to a server's `/_matrix/federation/v1/version` endpoint and measures the latency it took +- Extend clear cache admin command to support clearing more caches such as DNS +and TLS name overrides +- Admin debug command to send a federation request/ping to a server's +`/_matrix/federation/v1/version` endpoint and measures the latency it took - Add admin command to bulk delete media via a codeblock list of MXC URLs. -- Add admin command to delete both the thumbnail and media MXC URLs from an event ID (e.g. from an abuse report) +- Add admin command to delete both the thumbnail and media MXC URLs from an +event ID (e.g. from an abuse report) - Add admin command to list all the rooms a local user is joined in - Add admin command to list joined members in a room - Add admin command to view the room topic of a room -- Add admin command to delete all remote media in the past X minutes as a form of deleting media that you don't want on your server that a remote user posted in a room, a `--force` flag to ignore errors, and support for reading `last modified time` instead of `creation time` for filesystems that don't support file created metadata +- Add admin command to delete all remote media in the past X minutes as a form +of deleting media that you don't want on your server that a remote user posted +in a room, a `--force` flag to ignore errors, and support for reading `last +modified time` instead of `creation time` for filesystems that don't support +file created metadata - Add admin command to return a room's full/complete state -- Admin debug command to fetch a PDU from a remote server and inserts it into our database/timeline as backfill -- Add admin command to delete media via a specific MXC. This deletes the MXC from our database, and the file locally. -- Add admin commands for banning (blocking) room IDs from our local users joining (admins are always allowed) and evicts all our local users from that room, in addition to bulk room banning support, and blocks room invites (remote and local) to the banned room, as a moderation feature +- Admin debug command to fetch a PDU from a remote server and inserts it into +our database/timeline as backfill +- Add admin command to delete media via a specific MXC. This deletes the MXC +from our database, and the file locally. +- Add admin commands for banning (blocking) room IDs from our local users +joining (admins are always allowed) and evicts all our local users from that +room, in addition to bulk room banning support, and blocks room invites (remote +and local) to the banned room, as a moderation feature - Add admin commands to output jemalloc memory stats and memory usage - Add admin command to get rooms a *remote* user shares with us - Add debug admin commands to get the earliest and latest PDU in a room - Add debug admin command to echo a message -- Add admin command to insert rooms tags for a user, most useful for inserting the `m.server_notice` tag on your admin room to make it "persistent" in the "System Alerts" section of Element -- Add experimental admin debug command for Dendrite's `AdminDownloadState` (`/admin/downloadState/{serverName}/{roomID}`) admin API endpoint to download and use a remote server's room state in the room -- Disable URL previews by default in the admin room due to various command outputs having "URLs" in them that clients may needlessly render/request -- Extend memory usage admin server command to support showing memory allocator stats such as jemalloc's -- Add admin debug command to see memory allocator's full extended debug statistics such as jemalloc's +- Add admin command to insert rooms tags for a user, most useful for inserting +the `m.server_notice` tag on your admin room to make it "persistent" in the +"System Alerts" section of Element +- Add experimental admin debug command for Dendrite's `AdminDownloadState` +(`/admin/downloadState/{serverName}/{roomID}`) admin API endpoint to download +and use a remote server's room state in the room +- Disable URL previews by default in the admin room due to various command +outputs having "URLs" in them that clients may needlessly render/request +- Extend memory usage admin server command to support showing memory allocator +stats such as jemalloc's +- Add admin debug command to see memory allocator's full extended debug +statistics such as jemalloc's ## Misc -- Add guest support for accessing TURN servers via `turn_allow_guests` like Synapse -- Support for creating rooms with custom room IDs like Maunium Synapse (`room_id` request body field to `/createRoom`) -- Query parameter `?format=event|content` for returning either the room state event's content (default) for the full room state event on `/_matrix/client/v3/rooms/{roomId}/state/{eventType}[/{stateKey}]` requests (see ) +- Add guest support for accessing TURN servers via `turn_allow_guests` like +Synapse +- Support for creating rooms with custom room IDs like Maunium Synapse +(`room_id` request body field to `/createRoom`) +- Query parameter `?format=event|content` for returning either the room state +event's content (default) for the full room state event on +`/_matrix/client/v3/rooms/{roomId}/state/{eventType}[/{stateKey}]` requests (see +) - Send a User-Agent on all of our requests - Send `avatar_url` on invite room membership events/changes -- Support sending [`well_known` response to client login responses](https://spec.matrix.org/v1.10/client-server-api/#post_matrixclientv3login) if using config option `[well_known.client]` -- Implement `include_state` search criteria support for `/search` requests (response now can include room states) -- Declare various missing Matrix versions and features at `/_matrix/client/versions` -- Implement legacy Matrix `/v1/` media endpoints that some clients and servers may still call -- Config option to change Conduit's behaviour of homeserver key fetching (`query_trusted_key_servers_first`). This option sets whether conduwuit will query trusted notary key servers first before the individual homeserver(s), or vice versa which may help in joining certain rooms. +- Support sending [`well_known` response to client login +responses](https://spec.matrix.org/v1.10/client-server-api/#post_matrixclientv3login) +if using config option `[well_known.client]` +- Implement `include_state` search criteria support for `/search` requests +(response now can include room states) +- Declare various missing Matrix versions and features at +`/_matrix/client/versions` +- Implement legacy Matrix `/v1/` media endpoints that some clients and servers +may still call +- Config option to change Conduit's behaviour of homeserver key fetching +(`query_trusted_key_servers_first`). This option sets whether conduwuit will +query trusted notary key servers first before the individual homeserver(s), or +vice versa which may help in joining certain rooms. - Implement unstable MSC2666 support for querying mutual rooms with a user - Implement unstable MSC3266 room summary API support -- Implement unstable MSC4125 support for specifying servers to join via on federated invites +- Implement unstable MSC4125 support for specifying servers to join via on +federated invites - Make conduwuit build and be functional under Nix + macOS - Log out all sessions after unsetting the emergency password - Assume well-knowns are broken if they exceed past 12288 characters. -- Add support for listening on both HTTP and HTTPS if using direct TLS with conduwuit for usecases such as Complement +- Add support for listening on both HTTP and HTTPS if using direct TLS with +conduwuit for usecases such as Complement - Add config option for disabling RocksDB Direct IO if needed -- Add various documentation on maintaining conduwuit, using RocksDB online backups, some troubleshooting, using admin commands, moderation documentation, etc -- (Developers): Add support for [hot reloadable/"live" modular development](development/hot_reload.md) +- Add various documentation on maintaining conduwuit, using RocksDB online +backups, some troubleshooting, using admin commands, moderation documentation, +etc +- (Developers): Add support for [hot reloadable/"live" modular +development](development/hot_reload.md) - (Developers): Add support for tokio-console - (Developers): Add support for tracing flame graphs -- No cryptocurrency donations allowed, conduwuit is fully maintained by independent queer maintainers, and with a strong priority on inclusitivity and comfort for protected groups 🏳️‍⚧️ -- [Add a community Code of Conduct for all conduwuit community spaces, primarily the Matrix space](https://conduwuit.puppyirl.gay/conduwuit_coc.html) +- No cryptocurrency donations allowed, conduwuit is fully maintained by +independent queer maintainers, and with a strong priority on inclusitivity and +comfort for protected groups 🏳️‍⚧️ +- [Add a community Code of Conduct for all conduwuit community spaces, primarily +the Matrix space](https://conduwuit.puppyirl.gay/conduwuit_coc.html) diff --git a/docs/introduction.md b/docs/introduction.md index 6b1c35e3..9db76681 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -12,7 +12,8 @@ See the [differences](differences.md) page - [Deployment options](deploying.md) -If you want to connect an appservice to conduwuit, take a look at the [appservices documentation](appservices.md). +If you want to connect an appservice to conduwuit, take a look at the +[appservices documentation](appservices.md). #### How can I contribute? diff --git a/docs/maintenance.md b/docs/maintenance.md index 3cf2cf98..c8df95af 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -2,7 +2,11 @@ ## Moderation -conduwuit has moderation through admin room commands. "binary commands" (medium priority) and an admin API (low priority) is planned. Some moderation-related config options are available in the example config such as "global ACLs" and blocking media requests to certain servers. See the example config for the moderation config options under the "Moderation / Privacy / Security" section. +conduwuit has moderation through admin room commands. "binary commands" (medium +priority) and an admin API (low priority) is planned. Some moderation-related +config options are available in the example config such as "global ACLs" and +blocking media requests to certain servers. See the example config for the +moderation config options under the "Moderation / Privacy / Security" section. conduwuit has moderation admin commands for: @@ -11,56 +15,83 @@ conduwuit has moderation admin commands for: - managing room banning/blocking and user removal (`!admin rooms moderation`) - managing user accounts (`!admin users`) - fetching `/.well-known/matrix/support` from servers (`!admin federation`) -- blocking incoming federation for certain rooms (not the same as room banning) (`!admin federation`) +- blocking incoming federation for certain rooms (not the same as room banning) +(`!admin federation`) - deleting media (see [the media section](#media)) -Any commands with `-list` in them will require a codeblock in the message with each object being newline delimited. An example of doing this is: +Any commands with `-list` in them will require a codeblock in the message with +each object being newline delimited. An example of doing this is: -```` -!admin rooms moderation ban-list-of-rooms -``` -!roomid1:server.name -!roomid2:server.name -!roomid3:server.name -``` -```` +```` !admin rooms moderation ban-list-of-rooms ``` !roomid1:server.name +!roomid2:server.name !roomid3:server.name ``` ```` ## Database -If using RocksDB, there's very little you need to do. Compaction is ran automatically based on various defined thresholds tuned for conduwuit to be high performance with the least I/O amplifcation or overhead. Manually running compaction is not recommended, or compaction via a timer. RocksDB is built with io_uring support via liburing for async read I/O. +If using RocksDB, there's very little you need to do. Compaction is ran +automatically based on various defined thresholds tuned for conduwuit to be high +performance with the least I/O amplifcation or overhead. Manually running +compaction is not recommended, or compaction via a timer. RocksDB is built with +io_uring support via liburing for async read I/O. -Some RocksDB settings can be adjusted such as the compression method chosen. See the RocksDB section in the [example config](configuration/examples.md). btrfs users may benefit from disabling compression on RocksDB if CoW is in use. +Some RocksDB settings can be adjusted such as the compression method chosen. See +the RocksDB section in the [example config](configuration/examples.md). btrfs +users may benefit from disabling compression on RocksDB if CoW is in use. -RocksDB troubleshooting can be found [in the RocksDB section of troubleshooting](troubleshooting.md). +RocksDB troubleshooting can be found [in the RocksDB section of +troubleshooting](troubleshooting.md). ## Backups -Currently only RocksDB supports online backups. If you'd like to backup your database online without any downtime, see the `!admin server` command for the backup commands and the `database_backup_path` config options in the example config. Please note that the format of the database backup is not the exact same. This is unfortunately a bad design choice by Facebook as we are using the database backup engine API from RocksDB, however the data is still there and can still be joined together. +Currently only RocksDB supports online backups. If you'd like to backup your +database online without any downtime, see the `!admin server` command for the +backup commands and the `database_backup_path` config options in the example +config. Please note that the format of the database backup is not the exact +same. This is unfortunately a bad design choice by Facebook as we are using the +database backup engine API from RocksDB, however the data is still there and can +still be joined together. To restore a backup from an online RocksDB backup: - shutdown conduwuit - create a new directory for merging together the data -- in the online backup created, copy all `.sst` files in `$DATABASE_BACKUP_PATH/shared_checksum` to your new directory -- trim all the strings so instead of `######_sxxxxxxxxx.sst`, it reads `######.sst`. A way of doing this with sed and bash is `for file in *.sst; do mv "$file" "$(echo "$file" | sed 's/_s.*/.sst/')"; done` -- copy all the files in `$DATABASE_BACKUP_PATH/1` (or the latest backup number if you have multiple) to your new directory -- set your `database_path` config option to your new directory, or replace your old one with the new one you crafted +- in the online backup created, copy all `.sst` files in +`$DATABASE_BACKUP_PATH/shared_checksum` to your new directory +- trim all the strings so instead of `######_sxxxxxxxxx.sst`, it reads +`######.sst`. A way of doing this with sed and bash is `for file in *.sst; do mv +"$file" "$(echo "$file" | sed 's/_s.*/.sst/')"; done` +- copy all the files in `$DATABASE_BACKUP_PATH/1` (or the latest backup number +if you have multiple) to your new directory +- set your `database_path` config option to your new directory, or replace your +old one with the new one you crafted - start up conduwuit again and it should open as normal -If you'd like to do an offline backup, shutdown conduwuit and copy your `database_path` directory elsewhere. This can be restored with no modifications needed. +If you'd like to do an offline backup, shutdown conduwuit and copy your +`database_path` directory elsewhere. This can be restored with no modifications +needed. -Backing up media is also just copying the `media/` directory from your database directory. +Backing up media is also just copying the `media/` directory from your database +directory. ## Media Media still needs various work, however conduwuit implements media deletion via: -- MXC URI or Event ID (unencrypted and attempts to find the MXC URI in the event) +- MXC URI or Event ID (unencrypted and attempts to find the MXC URI in the +event) - Delete list of MXC URIs -- Delete remote media in the past `N` seconds/minutes via filesystem metadata on the file created time (`btime`) or file modified time (`mtime`) +- Delete remote media in the past `N` seconds/minutes via filesystem metadata on +the file created time (`btime`) or file modified time (`mtime`) -See the `!admin media` command for further information. All media in conduwuit is stored at `$DATABASE_DIR/media`. This will be configurable soon. +See the `!admin media` command for further information. All media in conduwuit +is stored at `$DATABASE_DIR/media`. This will be configurable soon. -If you are finding yourself needing extensive granular control over media, we recommend looking into [Matrix Media Repo](https://github.com/t2bot/matrix-media-repo). conduwuit intends to implement various utilities for media, but MMR is dedicated to extensive media management. +If you are finding yourself needing extensive granular control over media, we +recommend looking into [Matrix Media +Repo](https://github.com/t2bot/matrix-media-repo). conduwuit intends to +implement various utilities for media, but MMR is dedicated to extensive media +management. -Built-in S3 support is also planned, but for now using a "S3 filesystem" on `media/` works. conduwuit also sends a `Cache-Control` header of 1 year and immutable for all media requests (download and thumbnail) to reduce unnecessary media requests from browsers, reduce bandwidth usage, and reduce load. +Built-in S3 support is also planned, but for now using a "S3 filesystem" on +`media/` works. conduwuit also sends a `Cache-Control` header of 1 year and +immutable for all media requests (download and thumbnail) to reduce unnecessary +media requests from browsers, reduce bandwidth usage, and reduce load. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 61a22864..dd63d9bb 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2,34 +2,55 @@ > ## Docker users ⚠️ > -> Docker is extremely UX unfriendly. Because of this, a ton of issues or support is actually Docker support, not conduwuit support. We also cannot document the ever-growing list of Docker issues here. +> Docker is extremely UX unfriendly. Because of this, a ton of issues or support +> is actually Docker support, not conduwuit support. We also cannot document the +> ever-growing list of Docker issues here. > -> If you intend on asking for support and you are using Docker, **PLEASE** triple validate your issues are **NOT** because you have a misconfiguration in your Docker setup. +> If you intend on asking for support and you are using Docker, **PLEASE** +> triple validate your issues are **NOT** because you have a misconfiguration in +> your Docker setup. > -> If there are things like Compose file issues or Dockerhub image issues, those can still be mentioned as long as they're something we can fix. +> If there are things like Compose file issues or Dockerhub image issues, those +> can still be mentioned as long as they're something we can fix. ## General potential issues #### Potential DNS issues when using Docker -Docker has issues with its default DNS setup that may cause DNS to not be properly functional when running conduwuit, resulting in federation issues. -The symptoms of this have shown in excessively long room joins (30+ minutes) from very long DNS timeouts, log entries of "mismatching responding nameservers", and/or partial or non-functional inbound/outbound federation. +Docker has issues with its default DNS setup that may cause DNS to not be +properly functional when running conduwuit, resulting in federation issues. The +symptoms of this have shown in excessively long room joins (30+ minutes) from +very long DNS timeouts, log entries of "mismatching responding nameservers", +and/or partial or non-functional inbound/outbound federation. -This is **not** a conduwuit issue, and is purely a Docker issue. It is not sustainable for heavy DNS activity which is normal for Matrix federation. The workarounds for this are: +This is **not** a conduwuit issue, and is purely a Docker issue. It is not +sustainable for heavy DNS activity which is normal for Matrix federation. The +workarounds for this are: - Use DNS over TCP via the config option `query_over_tcp_only = true` -- Don't use Docker's default DNS setup and instead allow the container to use and communicate with your host's DNS servers (host's `/etc/resolv.conf`) +- Don't use Docker's default DNS setup and instead allow the container to use +and communicate with your host's DNS servers (host's `/etc/resolv.conf`) ## Rocksdb / database issues #### Direct IO -Some filesystems may not like RocksDB using [Direct IO](https://github.com/facebook/rocksdb/wiki/Direct-IO). Direct IO is for non-buffered I/O which improves conduwuit performance, but at least FUSE is a filesystem potentially known to not like this. See the [example config](configuration/examples.md) for disabling it if needed. Issues from Direct IO on unsupported filesystems are usually shown as startup errors. +Some filesystems may not like RocksDB using [Direct +IO](https://github.com/facebook/rocksdb/wiki/Direct-IO). Direct IO is for +non-buffered I/O which improves conduwuit performance, but at least FUSE is a +filesystem potentially known to not like this. See the [example +config](configuration/examples.md) for disabling it if needed. Issues from +Direct IO on unsupported filesystems are usually shown as startup errors. #### Database corruption -If your database is corrupted *and* is failing to start (e.g. checksum mismatch), it may be recoverable but careful steps must be taken, and there is no guarantee it may be recoverable. +If your database is corrupted *and* is failing to start (e.g. checksum +mismatch), it may be recoverable but careful steps must be taken, and there is +no guarantee it may be recoverable. -The first thing that can be done is launching conduwuit with the `rocksdb_repair` config option set to true. This will tell RocksDB to attempt to repair itself at launch. If this does not work, disable the option and continue reading. +The first thing that can be done is launching conduwuit with the +`rocksdb_repair` config option set to true. This will tell RocksDB to attempt to +repair itself at launch. If this does not work, disable the option and continue +reading. RocksDB has the following recovery modes: @@ -38,33 +59,59 @@ RocksDB has the following recovery modes: - `PointInTime` - `SkipAnyCorruptedRecord` -By default, conduwuit uses `TolerateCorruptedTailRecords` as generally these may be due to bad federation and we can re-fetch the correct data over federation. The RocksDB default is `PointInTime` which will attempt to restore a "snapshot" of the data when it was last known to be good. This data can be either a few seconds old, or multiple minutes prior. `PointInTime` may not be suitable for default usage due to clients and servers possibly not being able to handle sudden "backwards time travels", and `AbsoluteConsistency` may be too strict. +By default, conduwuit uses `TolerateCorruptedTailRecords` as generally these may +be due to bad federation and we can re-fetch the correct data over federation. +The RocksDB default is `PointInTime` which will attempt to restore a "snapshot" +of the data when it was last known to be good. This data can be either a few +seconds old, or multiple minutes prior. `PointInTime` may not be suitable for +default usage due to clients and servers possibly not being able to handle +sudden "backwards time travels", and `AbsoluteConsistency` may be too strict. -`AbsoluteConsistency` will fail to start the database if any sign of corruption is detected. `SkipAnyCorruptedRecord` will skip all forms of corruption unless it forbids the database from opening (e.g. too severe). Usage of `SkipAnyCorruptedRecord` voids any support as this may cause more damage and/or leave your database in a permanently inconsistent state, but it may do something if `PointInTime` does not work as a last ditch effort. +`AbsoluteConsistency` will fail to start the database if any sign of corruption +is detected. `SkipAnyCorruptedRecord` will skip all forms of corruption unless +it forbids the database from opening (e.g. too severe). Usage of +`SkipAnyCorruptedRecord` voids any support as this may cause more damage and/or +leave your database in a permanently inconsistent state, but it may do something +if `PointInTime` does not work as a last ditch effort. With this in mind: -- First start conduwuit with the `PointInTime` recovery method. See the [example config](configuration/examples.md) for how to do this using `rocksdb_recovery_mode` -- If your database successfully opens, clients are recommended to clear their client cache to account for the rollback -- Leave your conduwuit running in `PointInTime` for at least 30-60 minutes so as much possible corruption is restored -- If all goes will, you should be able to restore back to using `TolerateCorruptedTailRecords` and you have successfully recovered your database +- First start conduwuit with the `PointInTime` recovery method. See the [example +config](configuration/examples.md) for how to do this using +`rocksdb_recovery_mode` +- If your database successfully opens, clients are recommended to clear their +client cache to account for the rollback +- Leave your conduwuit running in `PointInTime` for at least 30-60 minutes so as +much possible corruption is restored +- If all goes will, you should be able to restore back to using +`TolerateCorruptedTailRecords` and you have successfully recovered your database ## Debugging -Note that users should not really be debugging things. If you find yourself debugging and find the issue, please let us know and/or how we can fix it. Various debug commands can be found in `!admin debug`. +Note that users should not really be debugging things. If you find yourself +debugging and find the issue, please let us know and/or how we can fix it. +Various debug commands can be found in `!admin debug`. #### Debug/Trace log level -conduwuit builds without debug or trace log levels by default for at least performance reasons. This may change in the future and/or binaries providing such configurations may be provided. If you need to access debug/trace log levels, you will need to build without the `release_max_log_level` feature. +conduwuit builds without debug or trace log levels by default for at least +performance reasons. This may change in the future and/or binaries providing +such configurations may be provided. If you need to access debug/trace log +levels, you will need to build without the `release_max_log_level` feature. #### Changing log level dynamically -conduwuit supports changing the tracing log environment filter on-the-fly using the admin command `!admin debug change-log-level`. This accepts a string **without quotes** the same format as the `log` config option. +conduwuit supports changing the tracing log environment filter on-the-fly using +the admin command `!admin debug change-log-level`. This accepts a string +**without quotes** the same format as the `log` config option. #### Pinging servers -conduwuit can ping other servers using `!admin debug ping`. This takes a server name and goes through the server discovery process and queries `/_matrix/federation/v1/version`. Errors are outputted. +conduwuit can ping other servers using `!admin debug ping`. This takes a server +name and goes through the server discovery process and queries +`/_matrix/federation/v1/version`. Errors are outputted. #### Allocator memory stats -When using jemalloc with jemallocator's `stats` feature, you can see conduwuit's jemalloc memory stats by using `!admin debug memory-stats` +When using jemalloc with jemallocator's `stats` feature, you can see conduwuit's +jemalloc memory stats by using `!admin debug memory-stats` diff --git a/docs/turn.md b/docs/turn.md index 29ddb79c..d54398ae 100644 --- a/docs/turn.md +++ b/docs/turn.md @@ -1,6 +1,8 @@ # Setting up TURN/STURN -In order to make or receive calls, a TURN server is required. conduwuit suggests using [Coturn](https://github.com/coturn/coturn) for this purpose, which is also available as a Docker image. +In order to make or receive calls, a TURN server is required. conduwuit suggests +using [Coturn](https://github.com/coturn/coturn) for this purpose, which is also +available as a Docker image. ### Configuration @@ -12,20 +14,25 @@ static-auth-secret=     realm= ``` -A common way to generate a suitable alphanumeric secret key is by using `pwgen -s 64 1`. +A common way to generate a suitable alphanumeric secret key is by using `pwgen +-s 64 1`. -These same values need to be set in conduwuit. See the [example config](configuration/examples.md) in the TURN section for configuring these and restart conduwuit after. +These same values need to be set in conduwuit. See the [example +config](configuration/examples.md) in the TURN section for configuring these and +restart conduwuit after. ### Run Run the [Coturn](https://hub.docker.com/r/coturn/coturn) image using ```bash -docker run -d --network=host -v $(pwd)/coturn.conf:/etc/coturn/turnserver.conf coturn/coturn +docker run -d --network=host -v +$(pwd)/coturn.conf:/etc/coturn/turnserver.conf coturn/coturn ``` -or docker-compose. For the latter, paste the following section into a file called `docker-compose.yml` -and run `docker compose up -d` in the same directory. +or docker-compose. For the latter, paste the following section into a file +called `docker-compose.yml` and run `docker compose up -d` in the same +directory. ```yml version: 3 @@ -39,6 +46,9 @@ services: - ./coturn.conf:/etc/coturn/turnserver.conf ``` -To understand why the host networking mode is used and explore alternative configuration options, please visit [Coturn's Docker documentation](https://github.com/coturn/coturn/blob/master/docker/coturn/README.md). +To understand why the host networking mode is used and explore alternative +configuration options, please visit [Coturn's Docker +documentation](https://github.com/coturn/coturn/blob/master/docker/coturn/README.md). -For security recommendations see Synapse's [Coturn documentation](https://element-hq.github.io/synapse/latest/turn-howto.html). +For security recommendations see Synapse's [Coturn +documentation](https://element-hq.github.io/synapse/latest/turn-howto.html). From 001523ec811a75f9d6d57dd8ffabc04af7803443 Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 24 Aug 2024 05:23:51 +0200 Subject: [PATCH 0732/2091] Avoid panicing during sliding sync if the user isn't in any rooms --- src/api/client/sync.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index b069a77e..a0a1749b 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1309,7 +1309,11 @@ pub(crate) async fn sync_events_v4_route( r.0, UInt::try_from(all_joined_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), ); - let room_ids = all_joined_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec(); + let room_ids = if !all_joined_rooms.is_empty() { + all_joined_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec() + } else { + Vec::new() + }; new_known_rooms.extend(room_ids.iter().cloned()); for room_id in &room_ids { let todo_room = todo_rooms From 351062eb828fe9712ea897d13cd6526a376a171b Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 24 Aug 2024 06:44:32 +0200 Subject: [PATCH 0733/2091] syncv3: add support for filtering invites and room types --- src/api/client/sync.rs | 88 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 17 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index a0a1749b..f41321ac 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -7,9 +7,7 @@ use std::{ use axum::extract::State; use conduit::{ - error, - utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, - Err, PduCount, + error, utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, warn, Err, PduCount }; use ruma::{ api::client::{ @@ -29,9 +27,7 @@ use ruma::{ presence::PresenceEvent, room::member::{MembershipState, RoomMemberEventContent}, StateEventType, TimelineEventType, - }, - serde::Raw, - uint, DeviceId, EventId, OwnedUserId, RoomId, UInt, UserId, + }, room::RoomType, serde::Raw, uint, DeviceId, EventId, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId }; use tracing::{Instrument as _, Span}; @@ -323,7 +319,7 @@ pub(crate) async fn sync_events_route( #[tracing::instrument(skip_all, fields(user_id = %sender_user, room_id = %room_id), name = "left_room")] async fn handle_left_room( services: &Services, since: u64, room_id: &RoomId, sender_user: &UserId, - left_rooms: &mut BTreeMap, next_batch_string: &str, full_state: bool, + left_rooms: &mut BTreeMap, next_batch_string: &str, full_state: bool, lazy_load_enabled: bool, ) -> Result<()> { // Get and drop the lock to wait for remaining operations to finish @@ -1114,6 +1110,20 @@ pub(crate) async fn sync_events_v4_route( .filter_map(Result::ok) .collect::>(); + let all_invited_rooms = services + .rooms + .state_cache + .rooms_invited(&sender_user) + .filter_map(Result::ok) + .map(|r| r.0) + .collect::>(); + + let all_rooms = all_joined_rooms + .iter() + .cloned() + .chain(all_invited_rooms.iter().cloned()) + .collect(); + if body.extensions.to_device.enabled.unwrap_or(false) { services .users @@ -1288,9 +1298,23 @@ pub(crate) async fn sync_events_v4_route( let mut todo_rooms = BTreeMap::new(); // and required state for (list_id, list) in body.lists { - if list.filters.and_then(|f| f.is_invite).unwrap_or(false) { - continue; - } + let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) { + Some(true) => &all_invited_rooms, + Some(false) => &all_joined_rooms, + None => &all_rooms, + }; + + let active_rooms = match list.filters.clone().map(|f| f.not_room_types) { + Some(filter) if filter.is_empty() => active_rooms.clone(), + Some(value) => filter_rooms(active_rooms, State(services), &value, true), + None => active_rooms.clone(), + }; + + let active_rooms = match list.filters.clone().map(|f| f.room_types) { + Some(filter) if filter.is_empty() => active_rooms.clone(), + Some(value) => filter_rooms(&active_rooms, State(services), &value, false), + None => active_rooms, + }; let mut new_known_rooms = BTreeSet::new(); @@ -1303,14 +1327,12 @@ pub(crate) async fn sync_events_v4_route( .map(|mut r| { r.0 = r.0.clamp( uint!(0), - UInt::try_from(all_joined_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), + UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), ); - r.1 = r.1.clamp( - r.0, - UInt::try_from(all_joined_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), - ); - let room_ids = if !all_joined_rooms.is_empty() { - all_joined_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec() + r.1 = + r.1.clamp(r.0, UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX)); + let room_ids = if !active_rooms.is_empty() { + active_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec() } else { Vec::new() }; @@ -1638,3 +1660,35 @@ pub(crate) async fn sync_events_v4_route( delta_token: None, }) } + +fn filter_rooms( + rooms: &[OwnedRoomId], State(services): State, filter: &[Option], negate: bool, +) -> Vec { + return rooms + .iter() + .filter(|r| { + match services.rooms.state_accessor.get_room_type(r) { + Err(e) => { + warn!("Requested room type for {}, but could not retrieve with error {}", r, e); + false + }, + Ok(None) => { + // For rooms which do not have a room type, use 'null' to include them + if negate { + !filter.contains(&None) + } else { + filter.contains(&None) + } + }, + Ok(Some(room_type)) => { + if negate { + !filter.contains(&Some(room_type)) + } else { + filter.is_empty() || filter.contains(&Some(room_type)) + } + }, + } + }) + .cloned() + .collect(); +} From 13334a88ca85155f762845769c99744033c389c8 Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 24 Aug 2024 06:45:33 +0200 Subject: [PATCH 0734/2091] syncv3: handle empty conn_id by mapping it to a predefined string --- src/api/client/sync.rs | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index f41321ac..9b0facc0 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -7,7 +7,9 @@ use std::{ use axum::extract::State; use conduit::{ - error, utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, warn, Err, PduCount + error, + utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, + warn, Err, PduCount, }; use ruma::{ api::client::{ @@ -27,7 +29,10 @@ use ruma::{ presence::PresenceEvent, room::member::{MembershipState, RoomMemberEventContent}, StateEventType, TimelineEventType, - }, room::RoomType, serde::Raw, uint, DeviceId, EventId, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId + }, + room::RoomType, + serde::Raw, + uint, DeviceId, EventId, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId, }; use tracing::{Instrument as _, Span}; @@ -36,6 +41,8 @@ use crate::{ utils, Error, PduEvent, Result, Ruma, RumaResponse, }; +const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; + /// # `GET /_matrix/client/r0/sync` /// /// Synchronize the client's state with the latest state on the server. @@ -1083,6 +1090,11 @@ pub(crate) async fn sync_events_v4_route( let next_batch = services.globals.next_count()?; + let conn_id = body + .conn_id + .clone() + .unwrap_or_else(|| SINGLE_CONNECTION_SYNC.to_owned()); + let globalsince = body .pos .as_ref() @@ -1090,11 +1102,9 @@ pub(crate) async fn sync_events_v4_route( .unwrap_or(0); if globalsince == 0 { - if let Some(conn_id) = &body.conn_id { - services - .users - .forget_sync_request_connection(sender_user.clone(), sender_device.clone(), conn_id.clone()); - } + services + .users + .forget_sync_request_connection(sender_user.clone(), sender_device.clone(), conn_id.clone()); } // Get sticky parameters from cache From 85400d15bc6fae56db58a1515de126c41bad0a64 Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 24 Aug 2024 06:57:31 +0200 Subject: [PATCH 0735/2091] syncv3: reset the connection after restarts --- src/api/client/sync.rs | 18 ++++++++++++++++-- src/service/users/mod.rs | 7 +++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 9b0facc0..544fe8b6 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -7,12 +7,13 @@ use std::{ use axum::extract::State; use conduit::{ - error, + debug, error, utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, warn, Err, PduCount, }; use ruma::{ api::client::{ + error::ErrorKind, filter::{FilterDefinition, LazyLoadOptions}, sync::sync_events::{ self, @@ -1081,7 +1082,7 @@ fn share_encrypted_room( /// Sliding Sync endpoint (future endpoint: `/_matrix/client/v4/sync`) pub(crate) async fn sync_events_v4_route( State(services): State, body: Ruma, -) -> Result> { +) -> Result { let sender_user = body.sender_user.expect("user is authenticated"); let sender_device = body.sender_device.expect("user is authenticated"); let mut body = body.body; @@ -1101,6 +1102,19 @@ pub(crate) async fn sync_events_v4_route( .and_then(|string| string.parse().ok()) .unwrap_or(0); + if globalsince != 0 + && !services + .users + .remembered(sender_user.clone(), sender_device.clone(), conn_id.clone()) + { + debug!("Restarting sync stream because it was gone from the database"); + return Err(Error::Request( + ErrorKind::UnknownPos, + "Connection data lost since last time".into(), + http::StatusCode::BAD_REQUEST, + )); + } + if globalsince == 0 { services .users diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index bdb07310..35f30a61 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -68,6 +68,13 @@ impl Service { #[inline] pub fn exists(&self, user_id: &UserId) -> Result { self.db.exists(user_id) } + pub fn remembered(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) -> bool { + self.connections + .lock() + .unwrap() + .contains_key(&(user_id, device_id, conn_id)) + } + pub fn forget_sync_request_connection(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) { self.connections .lock() From f4c8b773778e17aaa8fdd024f9881b95913e80ba Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 24 Aug 2024 07:33:07 +0200 Subject: [PATCH 0736/2091] syncv3: include the invite_state in the response --- src/api/client/sync.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 544fe8b6..0b1e557c 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1145,7 +1145,7 @@ pub(crate) async fn sync_events_v4_route( let all_rooms = all_joined_rooms .iter() .cloned() - .chain(all_invited_rooms.iter().cloned()) + .chain(all_invited_rooms.clone()) .collect(); if body.extensions.to_device.enabled.unwrap_or(false) { @@ -1490,6 +1490,16 @@ pub(crate) async fn sync_events_v4_route( .map(|(_, pdu)| pdu.to_sync_room_event()) .collect(); + let invite_state = if all_invited_rooms.contains(room_id) { + services + .rooms + .state_cache + .invite_state(&sender_user, room_id) + .unwrap_or(None) + } else { + None + }; + let required_state = required_state_request .iter() .map(|state| { @@ -1570,7 +1580,7 @@ pub(crate) async fn sync_events_v4_route( }, initial: Some(roomsince == &0), is_dm: None, - invite_state: None, + invite_state, unread_notifications: UnreadNotificationsCount { highlight_count: Some( services From 29a6065bae8f538d3d64ba0492a9395882e5a07d Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 24 Aug 2024 14:52:48 +0200 Subject: [PATCH 0737/2091] syncv3: include latest timestamps, calculated from a default set of types List borrowed from synapse --- Cargo.toml | 2 ++ src/api/client/sync.rs | 30 ++++++++++++++++++++---------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e780b314..66b38759 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -336,6 +336,8 @@ features = [ "unstable-msc3026", "unstable-msc3061", "unstable-msc3266", + "unstable-msc3381", # polls + "unstable-msc3489", # beacon / live location "unstable-msc3575", "unstable-msc4121", "unstable-msc4125", diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 0b1e557c..4acd4e93 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -25,15 +25,9 @@ use ruma::{ DeviceLists, UnreadNotificationsCount, }, uiaa::UiaaResponse, - }, - events::{ - presence::PresenceEvent, - room::member::{MembershipState, RoomMemberEventContent}, - StateEventType, TimelineEventType, - }, - room::RoomType, - serde::Raw, - uint, DeviceId, EventId, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId, + }, events::{ + presence::PresenceEvent, room::member::{MembershipState, RoomMemberEventContent}, StateEventType, TimelineEventType + }, room::RoomType, serde::Raw, state_res::Event, uint, DeviceId, EventId, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId }; use tracing::{Instrument as _, Span}; @@ -43,6 +37,14 @@ use crate::{ }; const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; +const DEFAULT_BUMP_TYPES: &[TimelineEventType] = &[ + TimelineEventType::Message, + TimelineEventType::Encrypted, + TimelineEventType::Sticker, + TimelineEventType::CallInvite, + TimelineEventType::PollStart, + TimelineEventType::Beacon, +]; /// # `GET /_matrix/client/r0/sync` /// @@ -1500,6 +1502,14 @@ pub(crate) async fn sync_events_v4_route( None }; + let mut timestamp: Option<_> = None; + for (_, pdu) in timeline_pdus { + timestamp = Some(MilliSecondsSinceUnixEpoch(pdu.origin_server_ts)); + if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) { + break; + } + } + let required_state = required_state_request .iter() .map(|state| { @@ -1622,7 +1632,7 @@ pub(crate) async fn sync_events_v4_route( .unwrap_or_else(|_| uint!(0)), ), num_live: None, // Count events in timeline greater than global sync counter - timestamp: None, + timestamp, heroes: Some(heroes), }, ); From 36b8de13391ef0c58fe3a18f2df9fb1301aa58b8 Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 30 Aug 2024 09:30:50 +0200 Subject: [PATCH 0738/2091] syncv3: include invited rooms in the count --- src/api/client/sync.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 4acd4e93..f9942f05 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1394,7 +1394,7 @@ pub(crate) async fn sync_events_v4_route( } }) .collect(), - count: ruma_from_usize(all_joined_rooms.len()), + count: ruma_from_usize(active_rooms.len()), }, ); From 77c0c13a83022d192c5543d8034ab69e93205138 Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 30 Aug 2024 10:31:08 +0200 Subject: [PATCH 0739/2091] syncv3: support per room account data --- Cargo.lock | 26 ++++----- Cargo.toml | 2 +- src/api/client/sync.rs | 93 +++++++++++++++++++++----------- src/service/account_data/data.rs | 30 +++++------ src/service/account_data/mod.rs | 7 ++- 5 files changed, 91 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7d9ddc53..d0e09b2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "assign", "js_int", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "js_int", "ruma-common", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "as_variant", "assign", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "as_variant", "base64 0.22.1", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "bytes", "http", @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "js_int", "thiserror", @@ -3113,7 +3113,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "js_int", "ruma-common", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "once_cell", "proc-macro-crate", @@ -3138,7 +3138,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "js_int", "ruma-common", @@ -3150,7 +3150,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "headers", "http", @@ -3163,7 +3163,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3179,7 +3179,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1de0f493e8aab7e65ea78e3a079a3de10167c777#1de0f493e8aab7e65ea78e3a079a3de10167c777" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 66b38759..92b1ff0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -314,7 +314,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "1de0f493e8aab7e65ea78e3a079a3de10167c777" +rev = "d7ddcd036f81edb257ab9371f9cadd46444e8a90" features = [ "compat", "rand", diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index f9942f05..46ac11c1 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -25,9 +25,16 @@ use ruma::{ DeviceLists, UnreadNotificationsCount, }, uiaa::UiaaResponse, - }, events::{ - presence::PresenceEvent, room::member::{MembershipState, RoomMemberEventContent}, StateEventType, TimelineEventType - }, room::RoomType, serde::Raw, state_res::Event, uint, DeviceId, EventId, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId + }, + events::{ + presence::PresenceEvent, + room::member::{MembershipState, RoomMemberEventContent}, + AnyRawAccountDataEvent, StateEventType, TimelineEventType, + }, + room::RoomType, + serde::Raw, + state_res::Event, + uint, DeviceId, EventId, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId, }; use tracing::{Instrument as _, Span}; @@ -46,6 +53,15 @@ const DEFAULT_BUMP_TYPES: &[TimelineEventType] = &[ TimelineEventType::Beacon, ]; +macro_rules! extract_variant { + ($e:expr, $variant:path) => { + match $e { + $variant(value) => Some(value), + _ => None, + } + }; +} + /// # `GET /_matrix/client/r0/sync` /// /// Synchronize the client's state with the latest state on the server. @@ -284,11 +300,7 @@ pub(crate) async fn sync_events_route( .account_data .changes_since(None, &sender_user, since)? .into_iter() - .filter_map(|(_, v)| { - serde_json::from_str(v.json().get()) - .map_err(|_| Error::bad_database("Invalid account event in database.")) - .ok() - }) + .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) .collect(), }, device_lists: DeviceLists { @@ -981,11 +993,7 @@ async fn load_joined_room( .account_data .changes_since(Some(room_id), sender_user, since)? .into_iter() - .filter_map(|(_, v)| { - serde_json::from_str(v.json().get()) - .map_err(|_| Error::bad_database("Invalid account event in database.")) - .ok() - }) + .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) .collect(), }, summary: RoomSummary { @@ -1160,6 +1168,33 @@ pub(crate) async fn sync_events_v4_route( let mut device_list_changes = HashSet::new(); let mut device_list_left = HashSet::new(); + let mut account_data = sync_events::v4::AccountData { + global: Vec::new(), + rooms: BTreeMap::new(), + }; + if body.extensions.account_data.enabled.unwrap_or(false) { + account_data.global = services + .account_data + .changes_since(None, &sender_user, globalsince)? + .into_iter() + .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) + .collect(); + + if let Some(rooms) = body.extensions.account_data.rooms { + for room in rooms { + account_data.rooms.insert( + room.clone(), + services + .account_data + .changes_since(Some(&room), &sender_user, globalsince)? + .into_iter() + .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect(), + ); + } + } + } + if body.extensions.e2ee.enabled.unwrap_or(false) { // Look for device list updates of this account device_list_changes.extend( @@ -1464,7 +1499,17 @@ pub(crate) async fn sync_events_v4_route( let (timeline_pdus, limited) = load_timeline(&services, &sender_user, room_id, roomsincecount, *timeline_limit)?; - if roomsince != &0 && timeline_pdus.is_empty() { + account_data.rooms.insert( + room_id.clone(), + services + .account_data + .changes_since(Some(room_id), &sender_user, *roomsince)? + .into_iter() + .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect(), + ); + + if roomsince != &0 && timeline_pdus.is_empty() && account_data.rooms.get(room_id).is_some_and(Vec::is_empty) { continue; } @@ -1504,7 +1549,7 @@ pub(crate) async fn sync_events_v4_route( let mut timestamp: Option<_> = None; for (_, pdu) in timeline_pdus { - timestamp = Some(MilliSecondsSinceUnixEpoch(pdu.origin_server_ts)); + timestamp = Some(MilliSecondsSinceUnixEpoch(pdu.origin_server_ts)); if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) { break; } @@ -1677,23 +1722,7 @@ pub(crate) async fn sync_events_v4_route( // Fallback keys are not yet supported device_unused_fallback_key_types: None, }, - account_data: sync_events::v4::AccountData { - global: if body.extensions.account_data.enabled.unwrap_or(false) { - services - .account_data - .changes_since(None, &sender_user, globalsince)? - .into_iter() - .filter_map(|(_, v)| { - serde_json::from_str(v.json().get()) - .map_err(|_| Error::bad_database("Invalid account event in database.")) - .ok() - }) - .collect() - } else { - Vec::new() - }, - rooms: BTreeMap::new(), - }, + account_data, receipts: sync_events::v4::Receipts { rooms: BTreeMap::new(), }, diff --git a/src/service/account_data/data.rs b/src/service/account_data/data.rs index 439603be..53a0e953 100644 --- a/src/service/account_data/data.rs +++ b/src/service/account_data/data.rs @@ -1,10 +1,10 @@ use std::{collections::HashMap, sync::Arc}; -use conduit::{utils, warn, Error, Result}; +use conduit::{Error, Result}; use database::Map; use ruma::{ api::client::error::ErrorKind, - events::{AnyEphemeralRoomEvent, RoomAccountDataEventType}, + events::{AnyGlobalAccountDataEvent, AnyRawAccountDataEvent, AnyRoomAccountDataEvent, RoomAccountDataEventType}, serde::Raw, RoomId, UserId, }; @@ -110,7 +110,7 @@ impl Data { /// Returns all changes to the account data that happened after `since`. pub(super) fn changes_since( &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, - ) -> Result>> { + ) -> Result> { let mut userdata = HashMap::new(); let mut prefix = room_id @@ -132,25 +132,21 @@ impl Data { .take_while(move |(k, _)| k.starts_with(&prefix)) .map(|(k, v)| { Ok::<_, Error>(( - RoomAccountDataEventType::from( - utils::string_from_bytes( - k.rsplit(|&b| b == 0xFF) - .next() - .ok_or_else(|| Error::bad_database("RoomUserData ID in db is invalid."))?, - ) - .map_err(|e| { - warn!("RoomUserData ID in database is invalid: {}", e); - Error::bad_database("RoomUserData ID in db is invalid.") - })?, - ), - serde_json::from_slice::>(&v) - .map_err(|_| Error::bad_database("Database contains invalid account data."))?, + k, + match room_id { + None => serde_json::from_slice::>(&v) + .map(AnyRawAccountDataEvent::Global) + .map_err(|_| Error::bad_database("Database contains invalid account data."))?, + Some(_) => serde_json::from_slice::>(&v) + .map(AnyRawAccountDataEvent::Room) + .map_err(|_| Error::bad_database("Database contains invalid account data."))?, + }, )) }) { let (kind, data) = r?; userdata.insert(kind, data); } - Ok(userdata) + Ok(userdata.into_values().collect()) } } diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index c569889e..eaa53641 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -1,12 +1,11 @@ mod data; -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; use conduit::Result; use data::Data; use ruma::{ - events::{AnyEphemeralRoomEvent, RoomAccountDataEventType}, - serde::Raw, + events::{AnyRawAccountDataEvent, RoomAccountDataEventType}, RoomId, UserId, }; @@ -47,7 +46,7 @@ impl Service { #[tracing::instrument(skip_all, name = "since", level = "debug")] pub fn changes_since( &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, - ) -> Result>> { + ) -> Result> { self.db.changes_since(room_id, user_id, since) } } From 26a6d349dcb6efb14130e972cf3201b4a9ec22fb Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 30 Aug 2024 10:35:05 +0200 Subject: [PATCH 0740/2091] nix: fix toolchain splicing warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit we load the toolchain from a file so not sure how we would create a toolchain based on pkgs 🤔 --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index a507ed81..e3bfe7fb 100644 --- a/flake.nix +++ b/flake.nix @@ -34,7 +34,7 @@ inherit pkgs; book = self.callPackage ./nix/pkgs/book {}; complement = self.callPackage ./nix/pkgs/complement {}; - craneLib = ((inputs.crane.mkLib pkgs).overrideToolchain toolchain); + craneLib = ((inputs.crane.mkLib pkgs).overrideToolchain (_: toolchain)); inherit inputs; main = self.callPackage ./nix/pkgs/main {}; oci-image = self.callPackage ./nix/pkgs/oci-image {}; From f8cf57c1fbdb0a34edc846c20d03078f874b3856 Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 30 Aug 2024 11:39:14 +0200 Subject: [PATCH 0741/2091] syncv3: fix bump types and only set timestamp if we find an event --- src/api/client/sync.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 46ac11c1..b99dc5e8 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -45,8 +45,8 @@ use crate::{ const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; const DEFAULT_BUMP_TYPES: &[TimelineEventType] = &[ - TimelineEventType::Message, - TimelineEventType::Encrypted, + TimelineEventType::RoomMessage, + TimelineEventType::RoomEncrypted, TimelineEventType::Sticker, TimelineEventType::CallInvite, TimelineEventType::PollStart, @@ -1549,8 +1549,8 @@ pub(crate) async fn sync_events_v4_route( let mut timestamp: Option<_> = None; for (_, pdu) in timeline_pdus { - timestamp = Some(MilliSecondsSinceUnixEpoch(pdu.origin_server_ts)); if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) { + timestamp = Some(MilliSecondsSinceUnixEpoch(pdu.origin_server_ts)); break; } } From bf9d49862100cc81a881310215f9dd40762c24a9 Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 30 Aug 2024 18:20:18 +0200 Subject: [PATCH 0742/2091] syncv3: check timestamps of all the rooms in the timeline --- src/api/client/sync.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index b99dc5e8..d95a88ff 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1549,9 +1549,9 @@ pub(crate) async fn sync_events_v4_route( let mut timestamp: Option<_> = None; for (_, pdu) in timeline_pdus { - if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) { - timestamp = Some(MilliSecondsSinceUnixEpoch(pdu.origin_server_ts)); - break; + let ts = MilliSecondsSinceUnixEpoch(pdu.origin_server_ts); + if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) && !timestamp.is_some_and(|time| time > ts) { + timestamp = Some(ts); } } From 9fde835673d3a227fd3f9740edf17076a5e01a28 Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 30 Aug 2024 18:23:42 +0200 Subject: [PATCH 0743/2091] syncv3: read receipts extension (MSC3960) --- src/api/client/sync.rs | 25 +++++++++++++--- src/service/rooms/read_receipt/data.rs | 10 ++----- src/service/rooms/read_receipt/mod.rs | 41 +++++++++++++++++++++++--- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index d95a88ff..236b85dd 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -36,6 +36,7 @@ use ruma::{ state_res::Event, uint, DeviceId, EventId, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId, }; +use service::rooms::read_receipt::pack_receipts; use tracing::{Instrument as _, Span}; use crate::{ @@ -1168,6 +1169,10 @@ pub(crate) async fn sync_events_v4_route( let mut device_list_changes = HashSet::new(); let mut device_list_left = HashSet::new(); + let mut receipts = sync_events::v4::Receipts { + rooms: BTreeMap::new(), + }; + let mut account_data = sync_events::v4::AccountData { global: Vec::new(), rooms: BTreeMap::new(), @@ -1509,7 +1514,21 @@ pub(crate) async fn sync_events_v4_route( .collect(), ); - if roomsince != &0 && timeline_pdus.is_empty() && account_data.rooms.get(room_id).is_some_and(Vec::is_empty) { + let room_receipts = services + .rooms + .read_receipt + .readreceipts_since(room_id, *roomsince); + let vector: Vec<_> = room_receipts.into_iter().collect(); + let receipt_size = vector.len(); + receipts + .rooms + .insert(room_id.clone(), pack_receipts(Box::new(vector.into_iter()))); + + if roomsince != &0 + && timeline_pdus.is_empty() + && account_data.rooms.get(room_id).is_some_and(Vec::is_empty) + && receipt_size == 0 + { continue; } @@ -1723,9 +1742,7 @@ pub(crate) async fn sync_events_v4_route( device_unused_fallback_key_types: None, }, account_data, - receipts: sync_events::v4::Receipts { - rooms: BTreeMap::new(), - }, + receipts, typing: sync_events::v4::Typing { rooms: BTreeMap::new(), }, diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 0f400ff3..0c156df3 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -2,17 +2,11 @@ use std::{mem::size_of, sync::Arc}; use conduit::{utils, Error, Result}; use database::Map; -use ruma::{ - events::{receipt::ReceiptEvent, AnySyncEphemeralRoomEvent}, - serde::Raw, - CanonicalJsonObject, OwnedUserId, RoomId, UserId, -}; +use ruma::{events::receipt::ReceiptEvent, serde::Raw, CanonicalJsonObject, RoomId, UserId}; +use super::AnySyncEphemeralRoomEventIter; use crate::{globals, Dep}; -type AnySyncEphemeralRoomEventIter<'a> = - Box)>> + 'a>; - pub(super) struct Data { roomuserid_privateread: Arc, roomuserid_lastprivatereadupdate: Arc, diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index d202d893..da11e2a0 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -1,10 +1,17 @@ mod data; -use std::sync::Arc; +use std::{collections::BTreeMap, sync::Arc}; -use conduit::Result; +use conduit::{debug, Result}; use data::Data; -use ruma::{events::receipt::ReceiptEvent, serde::Raw, OwnedUserId, RoomId, UserId}; +use ruma::{ + events::{ + receipt::{ReceiptEvent, ReceiptEventContent}, + AnySyncEphemeralRoomEvent, SyncEphemeralRoomEvent, + }, + serde::Raw, + OwnedUserId, RoomId, UserId, +}; use crate::{sending, Dep}; @@ -17,6 +24,9 @@ struct Services { sending: Dep, } +type AnySyncEphemeralRoomEventIter<'a> = + Box)>> + 'a>; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -44,7 +54,7 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub fn readreceipts_since<'a>( &'a self, room_id: &RoomId, since: u64, - ) -> impl Iterator)>> + 'a { + ) -> impl Iterator)>> + 'a { self.db.readreceipts_since(room_id, since) } @@ -65,3 +75,26 @@ impl Service { self.db.last_privateread_update(user_id, room_id) } } + +#[must_use] +pub fn pack_receipts(receipts: AnySyncEphemeralRoomEventIter<'_>) -> Raw> { + let mut json = BTreeMap::new(); + for (_user, _count, value) in receipts.flatten() { + let receipt = serde_json::from_str::>(value.json().get()); + if let Ok(value) = receipt { + for (event, receipt) in value.content { + json.insert(event, receipt); + } + } else { + debug!("failed to parse receipt: {:?}", receipt); + } + } + let content = ReceiptEventContent::from_iter(json); + + Raw::from_json( + serde_json::value::to_raw_value(&SyncEphemeralRoomEvent { + content, + }) + .expect("received valid json"), + ) +} From 119cc2eec0bfbc915ec35f0e2797da6b48547fa1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 12:26:47 -0400 Subject: [PATCH 0744/2091] fix typo with reqwest builder for disabling zstd Signed-off-by: strawberry --- src/service/client/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index e0f43107..b21f9dab 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -120,7 +120,7 @@ fn base(config: &Config) -> Result { builder = if config.zstd_compression { builder.zstd(true) } else { - builder.zstd(false).no_brotli() + builder.zstd(false).no_zstd() }; }; From 8d7e5ca2bbeccb14623a718ad8597f991aa01c73 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 12:27:36 -0400 Subject: [PATCH 0745/2091] redirect/handle r0 media paths too Signed-off-by: strawberry --- src/api/router.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/api/router.rs b/src/api/router.rs index 94ec5571..c2d64599 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -249,7 +249,11 @@ pub fn build(router: Router, server: &Server) -> Router { .route("/_matrix/media/v3/config", any(legacy_media_disabled)) .route("/_matrix/media/v3/download/*path", any(legacy_media_disabled)) .route("/_matrix/media/v3/thumbnail/*path", any(legacy_media_disabled)) - .route("/_matrix/media/v3/preview_url", any(redirect_legacy_preview)); + .route("/_matrix/media/v3/preview_url", any(redirect_legacy_preview)) + .route("/_matrix/media/r0/config", any(legacy_media_disabled)) + .route("/_matrix/media/r0/download/*path", any(legacy_media_disabled)) + .route("/_matrix/media/r0/thumbnail/*path", any(legacy_media_disabled)) + .route("/_matrix/media/r0/preview_url", any(redirect_legacy_preview)); } router From 42a42b24a91df2d0535eecbd6fd3ad4e703d42fb Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 12:38:25 -0400 Subject: [PATCH 0746/2091] renovate: exclude rust deps we forked / cant easily bump or upgrade Signed-off-by: strawberry --- renovate.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/renovate.json b/renovate.json index eafe1d16..eecf8532 100644 --- a/renovate.json +++ b/renovate.json @@ -12,5 +12,15 @@ "nix": { "enabled": true }, - "labels": ["dependencies", "github_actions"] + "labels": [ + "dependencies", + "github_actions" + ], + "ignoreDeps": [ + "tikv-jemllocator", + "tikv-jemalloc-sys", + "tikv-jemalloc-ctl", + "opentelemetry-rust", + "tracing-opentelemetry" + ] } From eaa899750656b29ca1b5d47ef0e8403389b80caa Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 16:03:47 +0000 Subject: [PATCH 0747/2091] Update nixos/nix Docker tag to v2.24.4 --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 78449e36..3c954e58 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ before_script: ci: stage: ci - image: nixos/nix:2.23.3 + image: nixos/nix:2.24.4 script: # Cache CI dependencies - ./bin/nix-build-and-cache ci @@ -83,7 +83,7 @@ ci: artifacts: stage: artifacts - image: nixos/nix:2.23.3 + image: nixos/nix:2.24.4 script: - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl - cp result/bin/conduit x86_64-unknown-linux-musl From b89d2ceccdafee23e37c001ae96d70dc0b9ed7db Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 12:43:32 -0400 Subject: [PATCH 0748/2091] bump syn, serde, and tokio Signed-off-by: strawberry --- Cargo.lock | 66 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 6 ++--- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0e09b2e..ca348f41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -130,7 +130,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -141,7 +141,7 @@ checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -376,7 +376,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.75", + "syn 2.0.76", "which", ] @@ -572,7 +572,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -746,7 +746,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1042,7 +1042,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1069,7 +1069,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1179,7 +1179,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1331,7 +1331,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1602,7 +1602,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -1966,7 +1966,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -2497,7 +2497,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -2590,7 +2590,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -2662,7 +2662,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" dependencies = [ "proc-macro2", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -2691,7 +2691,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "version_check", "yansi", ] @@ -2716,7 +2716,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3131,7 +3131,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.75", + "syn 2.0.76", "toml", ] @@ -3560,22 +3560,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.208" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff085d2cb684faa248efb494c39b68e522822ac0de72ccf08109abde717cfb2" +checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.208" +version = "1.0.209" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24008e81ff7613ed8e5ba0cfaf24e2c2f1e5b8a0495711e44fcd4882fca62bcf" +checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -3869,9 +3869,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.75" +version = "2.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6af063034fc1935ede7be0122941bafa9bacb949334d090b77ca98b5817c7d9" +checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" dependencies = [ "proc-macro2", "quote", @@ -3947,7 +3947,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4058,9 +4058,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.3" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", @@ -4082,7 +4082,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4306,7 +4306,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b47 dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4577,7 +4577,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "wasm-bindgen-shared", ] @@ -4611,7 +4611,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4976,7 +4976,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] @@ -4996,7 +4996,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.75", + "syn 2.0.76", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 92b1ff0f..691867b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,7 +142,7 @@ features = [ ] [workspace.dependencies.serde] -version = "1.0.204" +version = "1.0.209" default-features = false features = ["rc"] @@ -216,7 +216,7 @@ version = "0.3.30" default-features = false [workspace.dependencies.tokio] -version = "1.39.2" +version = "1.40.0" default-features = false features = [ "fs", @@ -449,7 +449,7 @@ default-features = false version = "0.1" [workspace.dependencies.syn] -version = "2.0.72" +version = "2.0.76" default-features = false features = ["full", "extra-traits"] From bceed3c829ef4a120597fa2f4f309485e4cb0dcc Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 15:01:54 -0400 Subject: [PATCH 0749/2091] dont debug print startup admin command content body Signed-off-by: strawberry --- src/service/admin/startup.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/admin/startup.rs b/src/service/admin/startup.rs index b7bc8d44..9c02c902 100644 --- a/src/service/admin/startup.rs +++ b/src/service/admin/startup.rs @@ -92,12 +92,12 @@ fn startup_command_error(i: usize, content: &RoomMessageEventContent) -> Result< #[cfg(not(feature = "console"))] #[implement(super::Service)] fn startup_command_output(i: usize, content: &RoomMessageEventContent) -> Result<()> { - info!("Startup command #{i} completed:\n{:#?}", content.body()); + info!("Startup command #{i} completed:\n{:#}", content.body()); Ok(()) } #[cfg(not(feature = "console"))] #[implement(super::Service)] fn startup_command_error(i: usize, content: &RoomMessageEventContent) -> Result<()> { - Err!(error!("Startup command #{i} failed:\n{:#?}", content.body())) + Err!(error!("Startup command #{i} failed:\n{:#}", content.body())) } From ae98610c50bdcbca516e424fd9d51bdf5d3c066b Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 15:06:28 -0400 Subject: [PATCH 0750/2091] docs: document new startup `--execute` admin cmd flag/argument Signed-off-by: strawberry --- docs/configuration.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/configuration.md b/docs/configuration.md index 78646ee3..f4f7f4c7 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -31,6 +31,22 @@ string. This does not apply to options that take booleans or numbers: - `--option log=\"debug\"` works ✅ - `--option server_name='"example.com'"` works ✅ +## Execute commandline flag + +conduwuit supports running admin commands on startup using the commandline +argument `--execute`. The most notable use for this is to create an admin user +on first startup. + +The syntax of this is a standard admin command without the prefix such as +`./conduwuit --execute "users create_user june"` + +An example output of a success is: +``` +INFO conduit_service::admin::startup: Startup command #0 completed: +Created user with user_id: @june:girlboss.ceo and password: `` +``` + +This commandline argument can be paired with the `--option` flag. ## Environment variables From 3a623dbdc3ae6f5f3b4d12e5257c6e4d6c76dfd1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 17:17:00 -0400 Subject: [PATCH 0751/2091] add force_leave_room admin command Signed-off-by: strawberry --- src/admin/user/commands.rs | 20 +++++++++++++++++++- src/admin/user/mod.rs | 6 ++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index c36a3c1d..b84fe873 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, fmt::Write as _}; -use api::client::{join_room_by_id_helper, leave_all_rooms, update_avatar_url, update_displayname}; +use api::client::{join_room_by_id_helper, leave_all_rooms, leave_room, update_avatar_url, update_displayname}; use conduit::{error, info, utils, warn, PduBuilder, Result}; use ruma::{ events::{ @@ -369,6 +369,24 @@ pub(super) async fn force_join_room( ))) } +#[admin_command] +pub(super) async fn force_leave_room( + &self, user_id: String, room_id: OwnedRoomOrAliasId, +) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + let room_id = self.services.rooms.alias.resolve(&room_id).await?; + + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + leave_room(self.services, &user_id, &room_id, None).await?; + + Ok(RoomMessageEventContent::notice_markdown(format!( + "{user_id} has left {room_id}.", + ))) +} + #[admin_command] pub(super) async fn make_user_admin(&self, user_id: String) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index f6ecd5e8..d8d740bf 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -73,6 +73,12 @@ pub(super) enum UserCommand { room_id: OwnedRoomOrAliasId, }, + /// - Manually leave a local user from a room. + ForceLeaveRoom { + user_id: String, + room_id: OwnedRoomOrAliasId, + }, + /// - Grant server-admin privileges to a user. MakeUserAdmin { user_id: String, From 922875477f309c0a6204770a77a41463c7c1a02c Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 18:54:41 -0400 Subject: [PATCH 0752/2091] docs: fix some borked codeblocks Signed-off-by: strawberry --- README.md | 2 +- docs/deploying/docker.md | 46 ++++++++++++++++++--------------------- docs/deploying/generic.md | 46 ++++++++++++++++++++++++--------------- docs/deploying/nixos.md | 13 ++++++----- docs/development.md | 8 ++++--- 5 files changed, 64 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index 3e0ea593..a29c3f84 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # conduwuit -`main` / stable: [![CI and +`main`: [![CI and Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml) ### a very cool, featureful fork of diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index 9fc8dc2f..7b8fd1a2 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -9,22 +9,14 @@ from a registry. OCI images for conduwuit are available in the registries listed below. -| Registry | Image -| Size | Notes | | --------------- | ---------------------------------------------------------------- | ------------------------------ | ---------------------- | | GitHub Registry | -[ghcr.io/girlbossceo/conduwuit:latest][gh] | ![Image Size][shield-latest] | -Stable tagged image. | | GitLab Registry | -[registry.gitlab.com/conduwuit/conduwuit:latest][gl] | ![Image -Size][shield-latest] | Stable tagged image. | | Docker Hub | -[docker.io/girlbossceo/conduwuit:latest][dh] | ![Image -Size][shield-latest] | Stable tagged image. | | GitHub Registry | -[ghcr.io/girlbossceo/conduwuit:main][gh] | ![Image Size][shield-main] | -Stable main branch. | | GitLab Registry | -[registry.gitlab.com/conduwuit/conduwuit:main][gl] | ![Image -Size][shield-main] | Stable main branch. | | Docker Hub | -[docker.io/girlbossceo/conduwuit:main][dh] | ![Image -Size][shield-main] | Stable main branch. | +| Registry | Image | Size | Notes | +| --------------- | --------------------------------------------------------------- | ----------------------------- | ---------------------- | +| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:latest][gh] | ![Image Size][shield-latest] | Stable tagged image. | +| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:latest][gl] | ![Image Size][shield-latest] | Stable tagged image. | +| Docker Hub | [docker.io/girlbossceo/conduwuit:latest][dh] | ![Image Size][shield-latest] | Stable tagged image. | +| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:main][gh] | ![Image Size][shield-main] | Stable main branch. | +| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:main][gl] | ![Image Size][shield-main] | Stable main branch. | +| Docker Hub | [docker.io/girlbossceo/conduwuit:main][dh] | ![Image Size][shield-main] | Stable main branch. | [dh]: https://hub.docker.com/r/girlbossceo/conduwuit [gh]: https://github.com/girlbossceo/conduwuit/pkgs/container/conduwuit @@ -34,7 +26,9 @@ Size][shield-main] | Stable main branch. | Use -```bash docker image pull ``` +```bash +docker image pull $LINK +``` to pull it to your machine. @@ -42,13 +36,13 @@ to pull it to your machine. When you have the image you can simply run it with -```bash -docker run -d -p 8448:6167 \ - -v db:/var/lib/conduwuit/ \ - -e CONDUWUIT_SERVER_NAME="your.server.name" \ - -e CONDUWUIT_DATABASE_BACKEND="rocksdb" \ +```bash +docker run -d -p 8448:6167 \ + -v db:/var/lib/conduwuit/ \ + -e CONDUWUIT_SERVER_NAME="your.server.name" \ + -e CONDUWUIT_DATABASE_BACKEND="rocksdb" \ -e CONDUWUIT_ALLOW_REGISTRATION=false \ - --name conduit + --name conduit $LINK ``` or you can use [docker compose](#docker-compose). @@ -88,7 +82,9 @@ server. When picking the `caddy-docker-proxy` compose file, it's important to first create the `caddy` network before spinning up the containers: -```bash docker network create caddy ``` +```bash +docker network create caddy +``` After that, you can rename it so it matches `docker-compose.yml` and spin up the containers! @@ -101,7 +97,7 @@ To build the conduwuit image with docker-compose, you first need to open and modify the `docker-compose.yml` file. There you need to comment the `image:` option and uncomment the `build:` option. Then call docker compose with: -```bash +```bash docker compose up ``` diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 5a5eb5d1..f0391fec 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -34,12 +34,14 @@ are correctly set up. In Debian or Fedora/RHEL, you can use this command to create a conduwuit user: ```bash -sudo adduser --system conduwuit --group --disabled-login --no-create-home +sudo adduser --system conduwuit --group --disabled-login --no-create-home ``` For distros without `adduser`: -```bash sudo useradd -r --shell /usr/bin/nologin --no-create-home conduwuit ``` +```bash +sudo useradd -r --shell /usr/bin/nologin --no-create-home conduwuit +``` ## Forwarding ports in the firewall or the router @@ -60,8 +62,9 @@ change the `ExecStart=` path to where you placed the conduwuit binary. Now we need to create the conduwuit's config file in `/etc/conduwuit/conduwuit.toml`. The example config can be found at -[conduwuit-example.toml](../configuration/examples.md).**Please take a moment to -read it. You need to change at least the server name.** +[conduwuit-example.toml](../configuration/examples.md). + +**Please take a moment to read the config. You need to change at least the server name.** RocksDB is the only supported database backend. @@ -76,9 +79,9 @@ sudo chown -R root:root /etc/conduwuit sudo chmod -R 755 /etc/conduwuit If you use the default database path you also need to run this: -```bash -sudo mkdir -p /var/lib/conduwuit/ sudo chown -R conduwuit:conduwuit -/var/lib/conduwuit/ sudo chmod 700 /var/lib/conduwuit/ +```bash +sudo mkdir -p /var/lib/conduwuit/ sudo chown -R conduwuit:conduwuit /var/lib/conduwuit/ +sudo chmod 700 /var/lib/conduwuit/ ``` ## Setting up the Reverse Proxy @@ -97,27 +100,34 @@ header, making federation non-functional. If using Apache, you need to use Create `/etc/caddy/conf.d/conduwuit_caddyfile` and enter this (substitute for your server name). -```caddy -your.server.name, your.server.name:8448 { # TCP reverse_proxy +```caddyfile +your.server.name, your.server.name:8448 { + # TCP reverse_proxy 127.0.0.1:6167 - # UNIX socket - #reverse_proxy unix//run/conduwuit/conduwuit.sock + # UNIX socket + #reverse_proxy unix//run/conduwuit/conduwuit.sock } ``` That's it! Just start and enable the service and you're set. -```bash sudo systemctl enable --now caddy ``` +```bash +sudo systemctl enable --now caddy +``` ## You're done Now you can start conduwuit with: -```bash sudo systemctl start conduwuit ``` +```bash +sudo systemctl start conduwuit +``` Set it to start automatically when your system boots with: -```bash sudo systemctl enable conduwuit ``` +```bash +sudo systemctl enable conduwuit +``` ## How do I know it works? @@ -127,10 +137,12 @@ homeserver and try to register. You can also use these commands as a quick health check (replace `your.server.name`). -```bash $ curl https://your.server.name/_conduwuit/server_version +```bash +curl https://your.server.name/_conduwuit/server_version -# If using port 8448 $ curl -https://your.server.name:8448/_conduwuit/server_version ``` +# If using port 8448 +curl https://your.server.name:8448/_conduwuit/server_version +``` - To check if your server can talk with other homeservers, you can use the [Matrix Federation Tester](https://federationtester.matrix.org/). If you can diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index 4a591315..271393bb 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -9,18 +9,21 @@ conduwuit can be acquired by [Lix][lix] from various places: A binary cache for conduwuit that the CI/CD publishes to is available at the following places (both are the same just different names): -``` https://attic.kennel.juneis.dog/conduit +``` +https://attic.kennel.juneis.dog/conduit conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= https://attic.kennel.juneis.dog/conduwuit -conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= ``` +conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= +``` The binary caches have been recreated recently due to attic issues. The old public keys were: -``` conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= - -conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= ``` +``` +conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= +conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= +``` If specifying a URL in your flake, please use the GitHub remote: `github:girlbossceo/conduwuit` diff --git a/docs/development.md b/docs/development.md index fbcfb433..eda9392a 100644 --- a/docs/development.md +++ b/docs/development.md @@ -38,9 +38,11 @@ disable the default `release_max_log_level` feature, and set the `--cfg tokio_unstable` flag to enable experimental tokio APIs. A build might look like this: -```bash RUSTFLAGS="--cfg tokio_unstable" cargo build \ --release \ ---no-default-features \ ---features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console +```bash +RUSTFLAGS="--cfg tokio_unstable" cargo build \ + --release \ + --no-default-features \ + --features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console ``` [1]: https://github.com/ruma/ruma/ From d68b71a0aa0791e3c981c4122392fd15ecafc607 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 19:37:21 -0400 Subject: [PATCH 0753/2091] add appservice ping client endpoint (MSC2659) Signed-off-by: strawberry --- src/api/client/appservice.rs | 47 ++++++++++++++++++++++++++++++++++++ src/api/client/mod.rs | 2 ++ src/api/router.rs | 1 + 3 files changed, 50 insertions(+) create mode 100644 src/api/client/appservice.rs diff --git a/src/api/client/appservice.rs b/src/api/client/appservice.rs new file mode 100644 index 00000000..d299185e --- /dev/null +++ b/src/api/client/appservice.rs @@ -0,0 +1,47 @@ +use axum::extract::State; +use conduit::{err, Err, Result}; +use ruma::api::{appservice::ping, client::appservice::request_ping}; + +use crate::Ruma; + +/// # `POST /_matrix/client/v1/appservice/{appserviceId}/ping` +/// +/// Ask the homeserver to ping the application service to ensure the connection +/// works. +pub(crate) async fn appservice_ping( + State(services): State, body: Ruma, +) -> Result { + let appservice_info = body + .appservice_info + .as_ref() + .ok_or_else(|| err!(Request(Forbidden("This endpoint can only be called by appservices."))))?; + + if body.appservice_id != appservice_info.registration.id { + return Err!(Request(Forbidden( + "Appservices can only ping themselves (wrong appservice ID)." + ))); + } + + if appservice_info.registration.url.is_none() { + return Err!(Request(UrlNotSet( + "Appservice does not have a URL set, there is nothing to ping." + ))); + } + + let timer = tokio::time::Instant::now(); + + let _response = services + .sending + .send_appservice_request( + appservice_info.registration.clone(), + ping::send_ping::v1::Request { + transaction_id: body.transaction_id.clone(), + }, + ) + .await? + .expect("We already validated if an appservice URL exists above"); + + Ok(request_ping::v1::Response { + duration: timer.elapsed(), + }) +} diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index 03c87e5d..5dcdc51a 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -1,5 +1,6 @@ pub(super) mod account; pub(super) mod alias; +pub(super) mod appservice; pub(super) mod backup; pub(super) mod capabilities; pub(super) mod config; @@ -38,6 +39,7 @@ pub(super) mod voip; pub(super) use account::*; pub(super) use alias::*; +pub(super) use appservice::*; pub(super) use backup::*; pub(super) use capabilities::*; pub(super) use config::*; diff --git a/src/api/router.rs b/src/api/router.rs index c2d64599..985f803b 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -22,6 +22,7 @@ use crate::{client, server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; let mut router = router + .ruma_route(client::appservice_ping) .ruma_route(client::get_supported_versions_route) .ruma_route(client::get_register_available_route) .ruma_route(client::register_route) From 4797183b43fd31c1203a09549e1413e0b471ebef Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 30 Aug 2024 19:52:55 -0400 Subject: [PATCH 0754/2091] remove unnecessary loop/allocations in `/joined_members` Signed-off-by: strawberry --- src/api/client/membership.rs | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index d3d80387..34e45a12 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -20,7 +20,8 @@ use ruma::{ error::ErrorKind, membership::{ ban_user, forget_room, get_member_events, invite_user, join_room_by_id, join_room_by_id_or_alias, - joined_members, joined_rooms, kick_user, leave_room, unban_user, ThirdPartySigned, + joined_members::{self, v3::RoomMember}, + joined_rooms, kick_user, leave_room, unban_user, ThirdPartySigned, }, }, federation::{self, membership::create_invite}, @@ -635,24 +636,22 @@ pub(crate) async fn joined_members_route( )); } - let mut joined = BTreeMap::new(); - for user_id in services + let joined: BTreeMap = services .rooms .state_cache .room_members(&body.room_id) - .filter_map(Result::ok) - { - let display_name = services.users.displayname(&user_id)?; - let avatar_url = services.users.avatar_url(&user_id)?; + .filter_map(|user| { + let user = user.ok()?; - joined.insert( - user_id, - joined_members::v3::RoomMember { - display_name, - avatar_url, - }, - ); - } + Some(( + user.clone(), + RoomMember { + display_name: services.users.displayname(&user).unwrap_or_default(), + avatar_url: services.users.avatar_url(&user).unwrap_or_default(), + }, + )) + }) + .collect(); Ok(joined_members::v3::Response { joined, From b45df5f7bd7a4653fbdb68a301c390fc2b14194f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 31 Aug 2024 09:55:11 +0000 Subject: [PATCH 0755/2091] bump appservice requests to v1.7 Signed-off-by: Jason Volk --- src/service/sending/appservice.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 32dbba4a..55092523 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -15,7 +15,7 @@ pub(crate) async fn send_request( where T: OutgoingRequest + Debug + Send, { - const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_0]; + const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_7]; let Some(dest) = registration.url else { return Ok(None); From 7b852352e51f51f1116279345e490eefaeea40ae Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 31 Aug 2024 14:08:31 +0200 Subject: [PATCH 0756/2091] deploying: make traefik config self-sufficient, include well known --- docs/deploying/docker-compose.for-traefik.yml | 82 +++++---- docs/deploying/docker-compose.override.yml | 61 +++--- .../deploying/docker-compose.with-traefik.yml | 174 ++++++++++++------ 3 files changed, 189 insertions(+), 128 deletions(-) diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index d10e5815..1c615673 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -1,40 +1,44 @@ # conduwuit - Behind Traefik Reverse Proxy services: - homeserver: - ### If you already built the conduduwit image with 'docker build' or want to use the Docker Hub image, - ### then you are ready to go. - image: girlbossceo/conduwuit:latest - restart: unless-stopped - volumes: - - db:/var/lib/conduwuit - #- ./conduwuit.toml:/etc/conduwuit.toml - networks: - - proxy - environment: - CONDUWUIT_SERVER_NAME: your.server.name # EDIT THIS - CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit - CONDUWUIT_DATABASE_BACKEND: rocksdb - CONDUWUIT_PORT: 6167 - CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB - CONDUWUIT_ALLOW_REGISTRATION: 'true' - CONDUWUIT_ALLOW_FEDERATION: 'true' - CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' - CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' - #CONDUWUIT_LOG: warn,state_res=warn - CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above - #cpuset: "0-4" # Uncomment to limit to specific CPU cores + homeserver: + ### If you already built the conduduwit image with 'docker build' or want to use the Docker Hub image, + ### then you are ready to go. + image: girlbossceo/conduwuit:latest + restart: unless-stopped + volumes: + - db:/var/lib/conduwuit + #- ./conduwuit.toml:/etc/conduwuit.toml + networks: + - proxy + environment: + CONDUWUIT_SERVER_NAME: your.server.name.example # EDIT THIS + CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit + CONDUWUIT_DATABASE_BACKEND: rocksdb + CONDUWUIT_PORT: 6167 # should match the loadbalancer traefik label + CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB + CONDUWUIT_ALLOW_REGISTRATION: 'true' + CONDUWUIT_ALLOW_FEDERATION: 'true' + CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' + CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' + #CONDUWUIT_LOG: warn,state_res=warn + CONDUWUIT_ADDRESS: 0.0.0.0 + #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above + + # We need some way to serve the client and server .well-known json. The simplest way is via the CONDUWUIT_WELL_KNOWN + # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a seperate + # see the override file for more information about delegation + CONDUWUIT_WELL_KNOWN: | + { + client=https://your.server.name.example, + server=your.server.name.example:443 + } + #cpuset: "0-4" # Uncomment to limit to specific CPU cores + ulimits: # conduwuit uses quite a few file descriptors, and on some systems it defaults to 1024, so you can tell docker to increase it + nofile: + soft: 1048567 + hard: 1048567 - # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container - # to serve those two as static files. If you want to use a different way, delete or comment the below service, here - # and in the docker compose override file. - well-known: - image: nginx:latest - restart: unless-stopped - volumes: - - ./nginx/matrix.conf:/etc/nginx/conf.d/matrix.conf # the config to serve the .well-known/matrix files - - ./nginx/www:/var/www/ # location of the client and server .well-known-files ### Uncomment if you want to use your own Element-Web App. ### Note: You need to provide a config.json for Element and you also need a second ### Domain or Subdomain for the communication between Element and conduwuit @@ -50,10 +54,12 @@ services: # - homeserver volumes: - db: + db: networks: - # This is the network Traefik listens to, if your network has a different - # name, don't forget to change it here and in the docker-compose.override.yml - proxy: - external: true + # This is the network Traefik listens to, if your network has a different + # name, don't forget to change it here and in the docker-compose.override.yml + proxy: + external: true + +# vim: ts=2:sw=2:expandtab diff --git a/docs/deploying/docker-compose.override.yml b/docs/deploying/docker-compose.override.yml index 23d6a90b..a343eeee 100644 --- a/docs/deploying/docker-compose.override.yml +++ b/docs/deploying/docker-compose.override.yml @@ -1,44 +1,37 @@ # conduwuit - Traefik Reverse Proxy Labels services: - homeserver: - labels: - - "traefik.enable=true" - - "traefik.docker.network=proxy" # Change this to the name of your Traefik docker proxy network + homeserver: + labels: + - "traefik.enable=true" + - "traefik.docker.network=proxy" # Change this to the name of your Traefik docker proxy network - - "traefik.http.routers.to-conduwuit.rule=Host(`.`)" # Change to the address on which conduwuit is hosted - - "traefik.http.routers.to-conduwuit.tls=true" - - "traefik.http.routers.to-conduwuit.tls.certresolver=letsencrypt" - - "traefik.http.routers.to-conduwuit.middlewares=cors-headers@docker" + - "traefik.http.routers.to-conduwuit.rule=Host(`.`)" # Change to the address on which conduwuit is hosted + - "traefik.http.routers.to-conduwuit.tls=true" + - "traefik.http.routers.to-conduwuit.tls.certresolver=letsencrypt" + - "traefik.http.routers.to-conduwuit.middlewares=cors-headers@docker" + - "traefik.http.services.to_conduwuit.loadbalancer.server.port=6167" - - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*" - - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization" - - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS" + - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*" + - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization" + - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS" - # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container - # to serve those two as static files. If you want to use a different way, delete or comment the below service, here - # and in the docker compose file. - well-known: - labels: - - "traefik.enable=true" - - "traefik.docker.network=proxy" + # If you want to have your account on , but host conduwuit on a subdomain, + # you can let it only handle the well known file on that domain instead + #- "traefik.http.routers.to-matrix-wellknown.rule=Host(``) && PathPrefix(`/.well-known/matrix`)" + #- "traefik.http.routers.to-matrix-wellknown.tls=true" + #- "traefik.http.routers.to-matrix-wellknown.tls.certresolver=letsencrypt" + #- "traefik.http.routers.to-matrix-wellknown.middlewares=cors-headers@docker" - - "traefik.http.routers.to-matrix-wellknown.rule=Host(`.`) && PathPrefix(`/.well-known/matrix`)" - - "traefik.http.routers.to-matrix-wellknown.tls=true" - - "traefik.http.routers.to-matrix-wellknown.tls.certresolver=letsencrypt" - - "traefik.http.routers.to-matrix-wellknown.middlewares=cors-headers@docker" + ### Uncomment this if you uncommented Element-Web App in the docker-compose.yml + # element-web: + # labels: + # - "traefik.enable=true" + # - "traefik.docker.network=proxy" # Change this to the name of your Traefik docker proxy network - - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*" - - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization" - - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS" + # - "traefik.http.routers.to-element-web.rule=Host(`.`)" # Change to the address on which Element-Web is hosted + # - "traefik.http.routers.to-element-web.tls=true" + # - "traefik.http.routers.to-element-web.tls.certresolver=letsencrypt" +# vim: ts=2:sw=2:expandtab - ### Uncomment this if you uncommented Element-Web App in the docker-compose.yml - # element-web: - # labels: - # - "traefik.enable=true" - # - "traefik.docker.network=proxy" # Change this to the name of your Traefik docker proxy network - - # - "traefik.http.routers.to-element-web.rule=Host(`.`)" # Change to the address on which Element-Web is hosted - # - "traefik.http.routers.to-element-web.tls=true" - # - "traefik.http.routers.to-element-web.tls.certresolver=letsencrypt" diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index 79d20051..f05006a5 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -1,42 +1,52 @@ # conduwuit - Behind Traefik Reverse Proxy services: - homeserver: - ### If you already built the conduwuit image with 'docker build' or want to use the Docker Hub image, - ### then you are ready to go. - image: girlbossceo/conduwuit:latest - restart: unless-stopped - volumes: - - db:/srv/conduwuit/.local/share/conduwuit - #- ./conduwuit.toml:/etc/conduwuit.toml - networks: - - proxy - environment: - CONDUWUIT_SERVER_NAME: your.server.name # EDIT THIS - CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' - CONDUWUIT_ALLOW_REGISTRATION : 'true' - #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above - ### Uncomment and change values as desired - # CONDUWUIT_ADDRESS: 0.0.0.0 - # CONDUWUIT_PORT: 6167 - # CONDUWUIT_LOG: info # default is: "warn,state_res=warn" - # CONDUWUIT_ALLOW_JAEGER: 'false' - # CONDUWUIT_ALLOW_ENCRYPTION: 'true' - # CONDUWUIT_ALLOW_FEDERATION: 'true' - # CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' - # CONDUWUIT_DATABASE_PATH: /srv/conduwuit/.local/share/conduwuit - # CONDUWUIT_WORKERS: 10 - # CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB + homeserver: + ### If you already built the conduwuit image with 'docker build' or want to use the Docker Hub image, + ### then you are ready to go. + image: girlbossceo/conduwuit:latest + restart: unless-stopped + volumes: + - db:/var/lib/conduwuit + #- ./conduwuit.toml:/etc/conduwuit.toml + networks: + - proxy + environment: + CONDUWUIT_SERVER_NAME: your.server.name.example # EDIT THIS + CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' + CONDUWUIT_ALLOW_REGISTRATION: 'false' # After setting a secure registration token, you can enable this + CONDUWUIT_REGISTRATION_TOKEN: # This is a token you can use to register on the server + CONDUWUIT_ADDRESS: 0.0.0.0 + CONDUWUIT_PORT: 6167 # you need to match this with the traefik load balancer label if you're want to change it + CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit + #CONDUWUIT_CONFIG: '/etc/conduit.toml' # Uncomment if you mapped config toml above + ### Uncomment and change values as desired, note that conduwuit has plenty of config options, so you should check out the example example config too + # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging + # CONDUWUIT_LOG: info # default is: "warn,state_res=warn" + # CONDUWUIT_ALLOW_JAEGER: 'false' + # CONDUWUIT_ALLOW_ENCRYPTION: 'true' + # CONDUWUIT_ALLOW_FEDERATION: 'true' + # CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' + # CONDUWUIT_ALLOW_INCOMING_PRESENCE: true + # CONDUWUIT_ALLOW_OUTGOING_PRESENCE: true + # CONDUWUIT_ALLOW_LOCAL_PRESENCE: true + # CONDUWUIT_WORKERS: 10 + # CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB + # CONDUWUIT_NEW_USER_DISPLAYNAME_SUFFIX = "🏳<200d>⚧" - # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container - # to serve those two as static files. If you want to use a different way, delete or comment the below service, here - # and in the docker compose override file. - well-known: - image: nginx:latest - restart: unless-stopped - volumes: - - ./nginx/matrix.conf:/etc/nginx/conf.d/matrix.conf # the config to serve the .well-known/matrix files - - ./nginx/www:/var/www/ # location of the client and server .well-known-files + # We need some way to serve the client and server .well-known json. The simplest way is via the CONDUWUIT_WELL_KNOWN + # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a seperate + # reverse proxy, but since you do not have a reverse proxy and following this guide, this example is included + CONDUWUIT_WELL_KNOWN: | + { + client=https://your.server.name.example, + server=your.server.name.example:443 + } + #cpuset: "0-4" # Uncomment to limit to specific CPU cores + ulimits: # conduwuit uses quite a few file descriptors, and on some systems it defaults to 1024, so you can tell docker to increase it + nofile: + soft: 1048567 + hard: 1048567 ### Uncomment if you want to use your own Element-Web App. ### Note: You need to provide a config.json for Element and you also need a second @@ -52,29 +62,79 @@ services: # depends_on: # - homeserver - traefik: - image: "traefik:latest" - container_name: "traefik" - restart: "unless-stopped" - ports: - - "80:80" - - "443:443" - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - # - "./traefik_config:/etc/traefik" - - "acme:/etc/traefik/acme" - labels: - - "traefik.enable=true" + traefik: + image: "traefik:latest" + container_name: "traefik" + restart: "unless-stopped" + ports: + - "80:80" + - "443:443" + volumes: + - "/var/run/docker.sock:/var/run/docker.sock:z" + - "acme:/etc/traefik/acme" + #- "./traefik_config:/etc/traefik:z" + labels: + - "traefik.enable=true" - # middleware redirect - - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" - # global redirect to https - - "traefik.http.routers.redirs.rule=hostregexp(`{host:.+}`)" - - "traefik.http.routers.redirs.entrypoints=http" - - "traefik.http.routers.redirs.middlewares=redirect-to-https" + # middleware redirect + - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https" + # global redirect to https + - "traefik.http.routers.redirs.rule=hostregexp(`{host:.+}`)" + - "traefik.http.routers.redirs.entrypoints=web" + - "traefik.http.routers.redirs.middlewares=redirect-to-https" - networks: - - proxy + configs: + - source: dynamic.yml + target: /etc/traefik/dynamic.yml + + environment: + TRAEFIK_LOG_LEVEL: DEBUG + TRAEFIK_ENTRYPOINTS_WEB: true + TRAEFIK_ENTRYPOINTS_WEB_ADDRESS: ":80" + TRAEFIK_ENTRYPOINTS_WEB_HTTP_REDIRECTIONS_ENTRYPOINT_TO: websecure + + TRAEFIK_ENTRYPOINTS_WEBSECURE: true + TRAEFIK_ENTRYPOINTS_WEBSECURE_ADDRESS: ":443" + TRAEFIK_ENTRYPOINTS_WEBSECURE_HTTP_TLS_CERTRESOLVER: letsencrypt + #TRAEFIK_ENTRYPOINTS_WEBSECURE_HTTP_MIDDLEWARES: secureHeaders@file # if you want to enabled STS + + TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT: true + TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_EMAIL: # Set this to the email you want to receive certificate expiration emails for + TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_KEYTYPE: EC384 + TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_HTTPCHALLENGE: true + TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_HTTPCHALLENGE_ENTRYPOINT: web + TRAEFIK_CERTIFICATESRESOLVERS_LETSENCRYPT_ACME_STORAGE: "/etc/traefik/acme/acme.json" + + TRAEFIK_PROVIDERS_DOCKER: true + TRAEFIK_PROVIDERS_DOCKER_ENDPOINT: "unix:///var/run/docker.sock" + TRAEFIK_PROVIDERS_DOCKER_EXPOSEDBYDEFAULT: false + + TRAEFIK_PROVIDERS_FILE: true + TRAEFIK_PROVIDERS_FILE_FILENAME: "/etc/traefik/dynamic.yml" + +configs: + dynamic.yml: + content: | + # Optionally set STS headers, like in https://hstspreload.org + # http: + # middlewares: + # secureHeaders: + # headers: + # forceSTSHeader: true + # stsIncludeSubdomains: true + # stsPreload: true + # stsSeconds: 31536000 + tls: + options: + default: + cipherSuites: + - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 + - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305 + minVersion: VersionTLS12 volumes: db: @@ -82,3 +142,5 @@ volumes: networks: proxy: + +# vim: ts=2:sw=2:expandtab From 14b9511d2effc79a9aa575dff2c435c4abdbee78 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 31 Aug 2024 11:45:36 +0000 Subject: [PATCH 0757/2091] fix default capture EnvFilter in release mode Signed-off-by: Jason Volk --- src/admin/processor.rs | 12 ++++++++++-- src/core/config/mod.rs | 7 ++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/src/admin/processor.rs b/src/admin/processor.rs index 67548f49..4f60f56e 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -15,7 +15,7 @@ use conduit::{ }, trace, utils::string::{collect_stream, common_prefix}, - Error, Result, + warn, Error, Result, }; use futures_util::future::FutureExt; use ruma::{ @@ -114,7 +114,15 @@ async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) fn capture_create(context: &Command<'_>) -> (Arc, Arc>) { let env_config = &context.services.server.config.admin_log_capture; - let env_filter = EnvFilter::try_new(env_config).unwrap_or_else(|_| "debug".into()); + let env_filter = EnvFilter::try_new(env_config).unwrap_or_else(|e| { + warn!("admin_log_capture filter invalid: {e:?}"); + cfg!(debug_assertions) + .then_some("debug") + .or(Some("info")) + .map(Into::into) + .expect("default capture EnvFilter") + }); + let log_level = env_filter .max_level_hint() .and_then(LevelFilter::into_level) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 71ffd7f3..547196e4 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1079,6 +1079,11 @@ fn default_sentry_filter() -> String { "info".to_owned() } fn default_startup_netburst_keep() -> i64 { 50 } -fn default_admin_log_capture() -> String { "debug".to_owned() } +fn default_admin_log_capture() -> String { + cfg!(debug_assertions) + .then_some("debug") + .unwrap_or("info") + .to_owned() +} fn default_admin_room_tag() -> String { "m.server_notice".to_owned() } From 5dbb868936bc1c96ff2671396389c3078d18a17c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 31 Aug 2024 12:18:21 -0400 Subject: [PATCH 0758/2091] remove unnecessary loops/allocations in client `/capabilities` Signed-off-by: strawberry --- src/api/client/capabilities.rs | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index c04347f8..3c278426 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -1,8 +1,11 @@ use std::collections::BTreeMap; use axum::extract::State; -use ruma::api::client::discovery::get_capabilities::{ - self, Capabilities, RoomVersionStability, RoomVersionsCapability, ThirdPartyIdChangesCapability, +use ruma::{ + api::client::discovery::get_capabilities::{ + self, Capabilities, RoomVersionStability, RoomVersionsCapability, ThirdPartyIdChangesCapability, + }, + RoomVersionId, }; use crate::{Result, Ruma}; @@ -14,13 +17,19 @@ use crate::{Result, Ruma}; pub(crate) async fn get_capabilities_route( State(services): State, _body: Ruma, ) -> Result { - let mut available = BTreeMap::new(); - for room_version in &services.globals.unstable_room_versions { - available.insert(room_version.clone(), RoomVersionStability::Unstable); - } - for room_version in &services.globals.stable_room_versions { - available.insert(room_version.clone(), RoomVersionStability::Stable); - } + let available: BTreeMap = services + .globals + .unstable_room_versions + .iter() + .map(|unstable_room_version| (unstable_room_version.clone(), RoomVersionStability::Unstable)) + .chain( + services + .globals + .stable_room_versions + .iter() + .map(|stable_room_version| (stable_room_version.clone(), RoomVersionStability::Stable)), + ) + .collect(); let mut capabilities = Capabilities::default(); capabilities.room_versions = RoomVersionsCapability { @@ -28,7 +37,7 @@ pub(crate) async fn get_capabilities_route( available, }; - // conduit does not implement 3PID stuff + // we do not implement 3PID stuff capabilities.thirdparty_id_changes = ThirdPartyIdChangesCapability { enabled: false, }; From bfb10cda26f17f021d6ba8d54152ac13c867cf42 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 31 Aug 2024 12:51:24 -0400 Subject: [PATCH 0759/2091] slightly cleanup and simplify client /report endpoint Signed-off-by: strawberry --- src/api/client/report.rs | 48 +++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/src/api/client/report.rs b/src/api/client/report.rs index dc87fd21..588bd368 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -31,6 +31,8 @@ pub(crate) async fn report_event_route( body.room_id, body.event_id ); + delay_response().await; + // check if we know about the reported event ID or if it's invalid let Some(pdu) = services.rooms.timeline.get_pdu(&body.event_id)? else { return Err(Error::BadRequest( @@ -81,21 +83,19 @@ pub(crate) async fn report_event_route( )) .await; - delay_response().await?; - Ok(report_content::v3::Response {}) } /// in the following order: /// /// check if the room ID from the URI matches the PDU's room ID -/// check if reporting user is in the reporting room /// check if score is in valid range /// check if report reasoning is less than or equal to 750 characters +/// check if reporting user is in the reporting room fn is_report_valid( services: &Services, event_id: &EventId, room_id: &RoomId, sender_user: &UserId, reason: &Option, score: Option, pdu: &std::sync::Arc, -) -> Result { +) -> Result<()> { debug_info!("Checking if report from user {sender_user} for event {event_id} in room {room_id} is valid"); if room_id != pdu.room_id { @@ -105,10 +105,24 @@ fn is_report_valid( )); } + if score.is_some_and(|s| s > int!(0) || s < int!(-100)) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid score, must be within 0 to -100", + )); + }; + + if reason.as_ref().is_some_and(|s| s.len() > 750) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Reason too long, should be 750 characters or fewer", + )); + }; + if !services .rooms .state_cache - .room_members(&pdu.room_id) + .room_members(room_id) .filter_map(Result::ok) .any(|user_id| user_id == *sender_user) { @@ -118,30 +132,14 @@ fn is_report_valid( )); } - if score.map(|s| s > int!(0) || s < int!(-100)) == Some(true) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid score, must be within 0 to -100", - )); - }; - - if reason.clone().map(|s| s.len() >= 750) == Some(true) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Reason too long, should be 750 characters or fewer", - )); - }; - - Ok(true) + Ok(()) } /// even though this is kinda security by obscurity, let's still make a small -/// random delay sending a successful response per spec suggestion regarding +/// random delay sending a response per spec suggestion regarding /// enumerating for potential events existing in our server. -async fn delay_response() -> Result<()> { - let time_to_wait = rand::thread_rng().gen_range(8..21); +async fn delay_response() { + let time_to_wait = rand::thread_rng().gen_range(3..10); debug_info!("Got successful /report request, waiting {time_to_wait} seconds before sending successful response."); sleep(Duration::from_secs(time_to_wait)).await; - - Ok(()) } From 5530e7434a1fe2cd7937a9a3458b0768facb6318 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 31 Aug 2024 13:01:41 -0400 Subject: [PATCH 0760/2091] notify admin room on new room directory publishes Signed-off-by: strawberry --- src/api/client/directory.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 6054bd9c..ca4c1814 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -139,6 +139,14 @@ pub(crate) async fn set_room_visibility_route( \"lockdown_public_room_directory\" is enabled", body.room_id ); + services + .admin + .send_text(&format!( + "Non-admin user {sender_user} tried to publish {0} to the room directory while \ + \"lockdown_public_room_directory\" is enabled", + body.room_id + )) + .await; return Err(Error::BadRequest( ErrorKind::forbidden(), @@ -147,7 +155,11 @@ pub(crate) async fn set_room_visibility_route( } services.rooms.directory.set_public(&body.room_id)?; - info!("{sender_user} made {0} public", body.room_id); + services + .admin + .send_text(&format!("{sender_user} made {} public to the room directory", body.room_id)) + .await; + info!("{sender_user} made {0} public to the room directory", body.room_id); }, room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id)?, _ => { From b362f0e0face205c3c9e8bdd73a4dbccf97469fc Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 31 Aug 2024 16:05:47 -0400 Subject: [PATCH 0761/2091] fix some other markdown formatting Signed-off-by: strawberry --- README.md | 16 +++++++++++----- docs/conduwuit_coc.md | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a29c3f84..c08f32ae 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,11 @@ `main`: [![CI and Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml) - ### a very cool, featureful fork of -[Conduit](https://conduit.rs/) + + +### a very cool, featureful fork of [Conduit](https://conduit.rs/) + + Visit the [Conduwuit documentation](https://conduwuit.puppyirl.gay/) for more information. @@ -43,7 +46,9 @@ might run into bugs from time to time. - #### Contact + + +#### Contact If you run into any question, feel free to @@ -73,5 +78,6 @@ Both, but I prefer conduwuit. - git.girlcock.ceo: - git.gay: - Codeberg: -- sourcehut: +- sourcehut: + + diff --git a/docs/conduwuit_coc.md b/docs/conduwuit_coc.md index 5f71a257..0fce2fe3 100644 --- a/docs/conduwuit_coc.md +++ b/docs/conduwuit_coc.md @@ -9,7 +9,7 @@ environment for everyone. This Code of Conduct applies to all conduwuit spaces, including any further community rooms that reference this CoC. Here are our guidelines to help maintain the welcoming atmosphere that sets conduwuit apart. -For the general foundational rules, please refer to the [Contributor's +For the general foundational rules, please refer to the [Contributor's Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md). Below are additional guidelines specific to the conduwuit community. @@ -90,4 +90,4 @@ comfortable doing that, then please send a DM to one of the moderators directly. Together, let’s build a community where everyone feels valued and respected. -- The conduwuit Moderation Team +— The conduwuit Moderation Team From 2fcedad2b12d50660bae45490ddb185737bc26a7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 1 Sep 2024 00:40:17 -0400 Subject: [PATCH 0762/2091] document ways to recovering admin room access Signed-off-by: strawberry --- docs/troubleshooting.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index dd63d9bb..c1499f3a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -13,6 +13,17 @@ > If there are things like Compose file issues or Dockerhub image issues, those > can still be mentioned as long as they're something we can fix. +## conduwuit and Matrix issues + +#### Lost access to admin room + +You can reinvite yourself to the admin room through the following methods: +- Use the `--execute "users make_user_admin "` conduwuit binary +argument once to invite yourslf to the admin room on startup +- Use the conduwuit console/CLI to run the `users make_user_admin` command +- Or specify the `emergency_password` config option to allow you to temporarily +log into the server account (`@conduit`) from a web client + ## General potential issues #### Potential DNS issues when using Docker @@ -30,7 +41,7 @@ workarounds for this are: - Don't use Docker's default DNS setup and instead allow the container to use and communicate with your host's DNS servers (host's `/etc/resolv.conf`) -## Rocksdb / database issues +## RocksDB / database issues #### Direct IO From fc1834d629620d430cbc49e4f4f5beaff5710d5e Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 1 Sep 2024 00:44:22 -0400 Subject: [PATCH 0763/2091] use codeblocks instead of HTML tables for some admin commands Signed-off-by: strawberry --- src/admin/room/commands.rs | 27 +++++---------------------- src/admin/user/commands.rs | 25 +++---------------------- 2 files changed, 8 insertions(+), 44 deletions(-) diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index 2e14a849..fff5e5b6 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -1,9 +1,7 @@ -use std::fmt::Write; - use conduit::Result; use ruma::events::room::message::RoomMessageEventContent; -use crate::{admin_command, escape_html, get_room_info, PAGE_SIZE}; +use crate::{admin_command, get_room_info, PAGE_SIZE}; #[admin_command] pub(super) async fn list_rooms( @@ -61,29 +59,14 @@ pub(super) async fn list_rooms( }; let output_plain = format!( - "Rooms:\n{}", + "Rooms ({}):\n```\n{}\n```", + rooms.len(), rooms .iter() .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) .collect::>() .join("\n") ); - let output_html = format!( - "\n\t\t\n{}
    Room list - page \ - {page}
    idmembersname
    ", - rooms - .iter() - .fold(String::new(), |mut output, (id, members, name)| { - writeln!( - output, - "{}\t{}\t{}", - escape_html(id.as_ref()), - members, - escape_html(name) - ) - .expect("should be able to write to string buffer"); - output - }) - ); - Ok(RoomMessageEventContent::text_html(output_plain, output_html)) + + Ok(RoomMessageEventContent::notice_markdown(output_plain)) } diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index b84fe873..2d5d1ead 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -13,7 +13,7 @@ use ruma::{ use serde_json::value::to_raw_value; use crate::{ - admin_command, escape_html, get_room_info, + admin_command, get_room_info, utils::{parse_active_local_user_id, parse_local_user_id}, }; @@ -320,7 +320,7 @@ pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result ResultRooms {user_id} Joined \ - ({})\nid\tmembers\tname\n{}", - rooms.len(), - rooms - .iter() - .fold(String::new(), |mut output, (id, members, name)| { - writeln!( - output, - "{}\t{}\t{}", - escape_html(id.as_ref()), - members, - escape_html(name) - ) - .unwrap(); - output - }) - ); - - Ok(RoomMessageEventContent::text_html(output_plain, output_html)) + Ok(RoomMessageEventContent::notice_markdown(output_plain)) } #[admin_command] From 27bfb67d753aa7f8408d00eee484348e95f433e8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 1 Sep 2024 00:56:49 -0400 Subject: [PATCH 0764/2091] add `--no-details` to admin rooms list command Signed-off-by: strawberry --- src/admin/room/commands.rs | 8 ++++++-- src/admin/room/mod.rs | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index fff5e5b6..2adfa7d7 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -5,7 +5,7 @@ use crate::{admin_command, get_room_info, PAGE_SIZE}; #[admin_command] pub(super) async fn list_rooms( - &self, page: Option, exclude_disabled: bool, exclude_banned: bool, + &self, page: Option, exclude_disabled: bool, exclude_banned: bool, no_details: bool, ) -> Result { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); @@ -63,7 +63,11 @@ pub(super) async fn list_rooms( rooms.len(), rooms .iter() - .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) + .map(|(id, members, name)| if no_details { + format!("{id}") + } else { + format!("{id}\tMembers: {members}\tName: {name}") + }) .collect::>() .join("\n") ); diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index da7acb80..64d2af45 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -27,6 +27,11 @@ pub(super) enum RoomCommand { /// Excludes rooms that we have banned #[arg(long)] exclude_banned: bool, + + #[arg(long)] + /// Whether to only output room IDs without supplementary room + /// information + no_details: bool, }, #[command(subcommand)] From 60605e95795caef5b692562791002526654009c9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 1 Sep 2024 00:59:43 -0400 Subject: [PATCH 0765/2091] remove unnecessary loop/allocations on CIDR range init Signed-off-by: strawberry --- src/service/globals/mod.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 05fe1a77..c82d3097 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -76,12 +76,15 @@ impl crate::Service for Service { // Experimental, partially supported room versions let unstable_room_versions = vec![RoomVersionId::V2, RoomVersionId::V3, RoomVersionId::V4, RoomVersionId::V5]; - let mut cidr_range_denylist = Vec::new(); - for cidr in config.ip_range_denylist.clone() { - let cidr = IPAddress::parse(cidr).expect("valid cidr range"); - trace!("Denied CIDR range: {:?}", cidr); - cidr_range_denylist.push(cidr); - } + let cidr_range_denylist: Vec<_> = config + .ip_range_denylist + .iter() + .map(|cidr| { + let cidr = IPAddress::parse(cidr).expect("valid cidr range"); + trace!("Denied CIDR range: {:?}", cidr); + cidr + }) + .collect(); let mut s = Self { db, From 4bac9b33cc4292f38e6357e933bba1f95b34f059 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 1 Sep 2024 11:09:36 +0000 Subject: [PATCH 0766/2091] propagate config error for cidr range Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index c82d3097..25e0f402 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -8,7 +8,7 @@ use std::{ time::Instant, }; -use conduit::{error, trace, Config, Result}; +use conduit::{err, error, trace, Config, Result}; use data::Data; use ipaddress::IPAddress; use regex::RegexSet; @@ -79,12 +79,10 @@ impl crate::Service for Service { let cidr_range_denylist: Vec<_> = config .ip_range_denylist .iter() - .map(|cidr| { - let cidr = IPAddress::parse(cidr).expect("valid cidr range"); - trace!("Denied CIDR range: {:?}", cidr); - cidr - }) - .collect(); + .map(IPAddress::parse) + .inspect(|cidr| trace!("Denied CIDR range: {cidr:?}")) + .collect::>() + .map_err(|e| err!(Config("ip_range_denylist", e)))?; let mut s = Self { db, From 393eef431b978228127534c65e00abac647d482d Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 31 Aug 2024 18:58:39 +0200 Subject: [PATCH 0767/2091] syncv3: avoid fetching timelines for invites --- src/api/client/sync.rs | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 236b85dd..4d080758 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1,6 +1,5 @@ use std::{ - cmp, - cmp::Ordering, + cmp::{self, Ordering}, collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}, time::Duration, }; @@ -1501,8 +1500,28 @@ pub(crate) async fn sync_events_v4_route( for (room_id, (required_state_request, timeline_limit, roomsince)) in &todo_rooms { let roomsincecount = PduCount::Normal(*roomsince); - let (timeline_pdus, limited) = - load_timeline(&services, &sender_user, room_id, roomsincecount, *timeline_limit)?; + let mut timestamp: Option<_> = None; + let mut invite_state = None; + let (timeline_pdus, limited); + if all_invited_rooms.contains(room_id) { + // TODO: figure out a timestamp we can use for remote invites + invite_state = services + .rooms + .state_cache + .invite_state(&sender_user, room_id) + .unwrap_or(None); + + (timeline_pdus, limited) = (Vec::new(), true); + } else { + (timeline_pdus, limited) = + match load_timeline(&services, &sender_user, room_id, roomsincecount, *timeline_limit) { + Ok(value) => value, + Err(err) => { + warn!("Encountered missing timeline in {}, error {}", room_id, err); + continue; + }, + }; + } account_data.rooms.insert( room_id.clone(), @@ -1556,17 +1575,6 @@ pub(crate) async fn sync_events_v4_route( .map(|(_, pdu)| pdu.to_sync_room_event()) .collect(); - let invite_state = if all_invited_rooms.contains(room_id) { - services - .rooms - .state_cache - .invite_state(&sender_user, room_id) - .unwrap_or(None) - } else { - None - }; - - let mut timestamp: Option<_> = None; for (_, pdu) in timeline_pdus { let ts = MilliSecondsSinceUnixEpoch(pdu.origin_server_ts); if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) && !timestamp.is_some_and(|time| time > ts) { From c13e9a7c2b79b59a2800c55a63726252633e9ae1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 1 Sep 2024 12:00:08 -0400 Subject: [PATCH 0768/2091] document `allow_legacy_media` config option Signed-off-by: strawberry --- conduwuit-example.toml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index a99aa05c..42baf249 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -160,6 +160,19 @@ ip_range_denylist = [ ### Moderation / Privacy / Security +# Config option to control whether the legacy unauthenticated Matrix media repository endpoints will be enabled. +# These endpoints consist of: +# - /_matrix/media/*/config +# - /_matrix/media/*/upload +# - /_matrix/media/*/preview_url +# - /_matrix/media/*/download/* +# - /_matrix/media/*/thumbnail/* +# +# The authenticated equivalent endpoints are always enabled. +# +# Defaults to true for now, but this is highly subject to change, likely in the next release. +#allow_legacy_media = true + # Set to true to allow user type "guest" registrations. Element attempts to register guest users automatically. # Defaults to false allow_guest_registration = false From 5dfda2d30014f1ee1b5a534d739d43cd741e4366 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 1 Sep 2024 12:08:38 -0400 Subject: [PATCH 0769/2091] fix one header in readme Signed-off-by: strawberry --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c08f32ae..cca299c9 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/bad Visit the [Conduwuit documentation](https://conduwuit.puppyirl.gay/) for more information. - #### What is Matrix? + + +#### What is Matrix? [Matrix](https://matrix.org) is an open network for secure and decentralized communication. Users from every Matrix homeserver can chat with users from all From 8849a100fd3069d2bc6107ff37c0580deaa5759f Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 1 Sep 2024 12:11:56 -0400 Subject: [PATCH 0770/2091] dont use HTML for initial welcome message Signed-off-by: strawberry --- src/service/admin/grant.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 8cbee604..e0d58600 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -123,9 +123,8 @@ impl super::Service { self.services.timeline.build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&RoomMessageEventContent::text_html( - format!("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `@conduit:{}: --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`", self.services.globals.server_name()), - format!("

    Thank you for trying out conduwuit!

    \n

    conduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

    \n

    Helpful links:

    \n
    \n

    Git and Documentation: https://github.com/girlbossceo/conduwuit
    Report issues: https://github.com/girlbossceo/conduwuit/issues

    \n
    \n

    For a list of available commands, send the following message in this room: @conduit:{}: --help

    \n

    Here are some rooms you can join (by typing the command):

    \n

    conduwuit room (Ask questions and get notified on updates):
    /join #conduwuit:puppygock.gay

    \n", self.services.globals.server_name()), + content: to_raw_value(&RoomMessageEventContent::text_markdown( + String::from("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `!admin --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`"), )) .expect("event is valid, we just created it"), unsigned: None, From 8f7ade4c22533a3177bfd8f175e178573ba6c1d4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 1 Sep 2024 12:44:24 -0400 Subject: [PATCH 0771/2091] document all the fancy admin room config options and arguments Signed-off-by: strawberry --- conduwuit-example.toml | 44 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 42baf249..76013b85 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -68,6 +68,10 @@ # only effective in release-mode; forced to false in debug-mode. #sentry_send_error = true +# Controls the tracing log level for Sentry to send things like breadcrumbs and transactions +# Defaults to "info" +#sentry_filter = "info" + ### Database configuration @@ -220,11 +224,6 @@ registration_token = "change this token for something specific to your server" # defaults to false # block_non_admin_invites = false -# Allows admins to enter commands in rooms other than #admins by prefixing with \!admin. The reply -# will be publicly visible to the room, originating from the sender. -# defaults to true -#admin_escape_commands = true - # List of forbidden username patterns/strings. Values in this list are matched as *contains*. # This is checked upon username availability check, registration, and startup as warnings if any local users in your database # have a forbidden username. @@ -318,6 +317,41 @@ allow_profile_lookup_federation_requests = true #auto_deactivate_banned_room_attempts = false +### Admin Room and Console + +# Controls whether the conduwuit admin room console / CLI will immediately activate on startup. +# This option can also be enabled with `--console` conduwuit argument +# +# Defaults to false +#admin_console_automatic = false + +# Controls what admin commands will be executed on startup. This is a vector list of strings of admin commands to run. +# +# An example of this can be: `admin_execute = ["debug ping puppygock.gay", "debug echo hi"]` +# +# This option can also be configured with the `--execute` conduwuit argument and can take standard shell commands and environment variables +# +# Such example could be: `./conduwuit --execute "server admin-notice conduwuit has started up at $(date)"` +# +# Defaults to nothing. +#admin_execute = [""] + +# Controls whether conduwuit should error and fail to start if an admin execute command (`--execute` / `admin_execute`) fails +# +# Defaults to false +#admin_execute_errors_ignore = false + +# Controls the max log level for admin command log captures (logs generated from running admin commands) +# +# Defaults to "info" on release builds, else "debug" on debug builds +#admin_log_capture = info + +# Allows admins to enter commands in rooms other than #admins by prefixing with \!admin. The reply +# will be publicly visible to the room, originating from the sender. +# defaults to true +#admin_escape_commands = true + + ### Misc # max log level for conduwuit. allows debug, info, warn, or error From f521f88daf2bd50686a8b525587cddc1b0b08f48 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 6 Sep 2024 19:00:38 -0400 Subject: [PATCH 0772/2091] docs: mildly update the NixOS page to ref https://github.com/NixOS/nixpkgs/pull/339260 Signed-off-by: strawberry --- docs/deploying/nixos.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index 271393bb..cbf56ae5 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -1,11 +1,13 @@ # conduwuit for NixOS -conduwuit can be acquired by [Lix][lix] from various places: +conduwuit can be acquired by Nix (or [Lix][lix]) from various places: * The `flake.nix` at the root of the repo * The `default.nix` at the root of the repo * From conduwuit's binary cache +A community maintained NixOS package is available at [`conduwuit`](https://search.nixos.org/packages?channel=unstable&show=conduwuit&from=0&size=50&sort=relevance&type=packages&query=conduwuit) + A binary cache for conduwuit that the CI/CD publishes to is available at the following places (both are the same just different names): @@ -17,24 +19,26 @@ https://attic.kennel.juneis.dog/conduwuit conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= ``` -The binary caches have been recreated recently due to attic issues. The old -public keys were: +The binary caches were recreated some months ago due to attic issues. The old public +keys were: ``` conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= ``` -If specifying a URL in your flake, please use the GitHub remote: -`github:girlbossceo/conduwuit` -The `flake.nix` and `default.nix` do not (currently) provide a NixOS module, so -(for now) [`services.matrix-conduit`][module] from Nixpkgs should be used to -configure conduwuit. +If specifying a Git remote URL in your flake, you can use any remotes that +are specified on the README (the mirrors), such as the GitHub: `github:girlbossceo/conduwuit` + +The `flake.nix` and `default.nix` do not currently provide a NixOS module (contributions +welcome!), so [`services.matrix-conduit`][module] from Nixpkgs can be used to configure +conduwuit. + +If you want to run the latest code, you should get conduwuit from the `flake.nix` +or `default.nix` and set [`services.matrix-conduit.package`][package] +appropriately to use conduwuit instead of Conduit. -If you want to run the latest code, you should get conduwuit from the -`flake.nix` or `default.nix` and set -[`services.matrix-conduit.package`][package] appropriately. [lix]: https://lix.systems/ [module]: https://search.nixos.org/options?channel=unstable&query=services.matrix-conduit From 909eeac5b06bdcb12c30d80958c076e8cf3d983f Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 6 Sep 2024 19:02:04 -0400 Subject: [PATCH 0773/2091] drop target CPU for aarch64 to cortex-a53 instead of cortex-a73 Signed-off-by: strawberry --- nix/pkgs/main/cross-compilation-env.nix | 2 +- nix/pkgs/main/default.nix | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index 75b5862f..3f4d593b 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -18,7 +18,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { [ "-C" "target-cpu=x86-64-v2" ] ++ lib.optionals stdenv.targetPlatform.isAarch64 - [ "-C" "target-cpu=cortex-a55" ] # cortex-a55 == ARMv8.2-a + [ "-C" "target-cpu=cortex-a53" ] # cortex-a53 == ARMv8-A # This disables PIE for static builds, which isn't great in terms # of security. Unfortunately, my hand is forced because nixpkgs' # `libstdc++.a` is built without `-fPIE`, which precludes us from diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 47b98432..31a2b7aa 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -97,7 +97,7 @@ buildDepsOnlyEnv = else if stdenv.targetPlatform.isAarch64 then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags ++ lib.optionals stdenv.targetPlatform.isAarch64 [ - # cortex-a73 == ARMv8-A + # cortex-a53 == ARMv8-A "-DPORTABLE=armv8-a" ] else old.cmakeFlags; @@ -131,7 +131,7 @@ buildPackageEnv = { + lib.optionalString stdenv.targetPlatform.isx86_64 " -Ctarget-cpu=x86-64-v2" + lib.optionalString stdenv.targetPlatform.isAarch64 - " -Ctarget-cpu=cortex-a73"; # cortex-a73 == ARMv8-A + " -Ctarget-cpu=cortex-a53"; # cortex-a53 == ARMv8-A }; From 80698c0b1736cbe4b5549f31b0317d868c874aa9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 6 Sep 2024 19:51:06 -0400 Subject: [PATCH 0774/2091] docs: add some more conduwuit development info Signed-off-by: strawberry --- CONTRIBUTING.md | 9 +++-- development.md | 1 + docs/contributing.md | 2 +- docs/development.md | 78 ++++++++++++++++++++++++++++++++++++++++++-- 4 files changed, 83 insertions(+), 7 deletions(-) create mode 120000 development.md mode change 100644 => 120000 docs/contributing.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b5fa56e4..0a11394e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ # Contributing guide This page is for about contributing to conduwuit. The -[development](development.md) page may be of interest for you as well. +[development](./development.md) page may be of interest for you as well. If you would like to work on an [issue][issues] that is not assigned, preferably ask in the Matrix room first at [#conduwuit:puppygock.gay][conduwuit-matrix], @@ -67,7 +67,7 @@ failing from your changes, please review the logs (they are uploaded as artifacts) and determine if they're intended or not. If you'd like to run Complement locally using Nix, see the -[testing](docs/development/testing.md) page. +[testing](development/testing.md) page. [Sytest][sytest] support will come soon. @@ -128,7 +128,10 @@ Direct all PRs/MRs to the `main` branch. By sending a pull request or patch, you are agreeing that your changes are allowed to be licenced under the Apache-2.0 licence and all of your conduct is -in line with the Contributor's Covenant. +in line with the Contributor's Covenant, and conduwuit's Code of Conduct. + +Contribution by users who violate either of these code of conducts will not have +their contributions accepted. [issues]: https://github.com/girlbossceo/conduwuit/issues [conduwuit-matrix]: https://matrix.to/#/#conduwuit:puppygock.gay diff --git a/development.md b/development.md new file mode 120000 index 00000000..35e9aab8 --- /dev/null +++ b/development.md @@ -0,0 +1 @@ +docs/development.md \ No newline at end of file diff --git a/docs/contributing.md b/docs/contributing.md deleted file mode 100644 index 81ddb505..00000000 --- a/docs/contributing.md +++ /dev/null @@ -1 +0,0 @@ -{{#include ../CONTRIBUTING.md}} diff --git a/docs/contributing.md b/docs/contributing.md new file mode 120000 index 00000000..44fcc634 --- /dev/null +++ b/docs/contributing.md @@ -0,0 +1 @@ +../CONTRIBUTING.md \ No newline at end of file diff --git a/docs/development.md b/docs/development.md index eda9392a..e1f36c0c 100644 --- a/docs/development.md +++ b/docs/development.md @@ -1,10 +1,74 @@ # Development Information about developing the project. If you are only interested in using -it, you can safely ignore this section. If you plan on contributing, see the -[contributor's guide](contributing.md). +it, you can safely ignore this page. If you plan on contributing, see the +[contributor's guide](./contributing.md). -## List of forked dependencies During conduwuit development, we have had to fork +## conduwuit project layout + +conduwuit uses a collection of sub-crates, packages, or workspace members +that indicate what each general area of code is for. All of the workspace +members are under `src/`. The workspace definition is at the top level / root +`Cargo.toml`. + +The crate names are generally self-explanatory: +- `admin` is the admin room +- `api` is the HTTP API, Matrix C-S and S-S endpoints, etc +- `core` is core conduwuit functionality like config loading, error definitions, +global utilities, logging infrastructure, etc +- `database` is RocksDB methods, helpers, RocksDB config, and general database definitions, +utilities, or functions +- `macros` are conduwuit Rust [macros][macros] like general helper macros, logging +and error handling macros, and [syn][syn] and [procedural macros][proc-macro] +used for admin room commands and others +- `main` is the "primary" sub-crate. This is where the `main()` function lives, +tokio worker and async initialisation, Sentry initialisation, [clap][clap] init, +and signal handling. If you are adding new [Rust features][features], they *must* +go here. +- `router` is the webserver and request handling bits, using axum, tower, tower-http, +hyper, etc, and the [global server state][state] to access `services`. +- `service` is the high-level database definitions and functions for data, +outbound/sending code, and other business logic such as media fetching. + +It is highly unlikely you will ever need to add a new workspace member, but +if you truly find yourself needing to, we recommend reaching out to us in +the Matrix room for discussions about it beforehand. + +The primary inspiration for this design was apart of hot reloadable development, +to support "conduwuit as a library" where specific parts can simply be swapped out. +There is evidence Conduit wanted to go this route too as `axum` is technically an +optional feature in Conduit, and can be compiled without the binary or axum library +for handling inbound web requests; but it was never completed or worked. + +See the Rust documentation on [Workspaces][workspaces] for general questions +and information on Cargo workspaces. + +## Adding compile-time [features][features] + +If you'd like to add a compile-time feature, you must first define it in +the `main` workspace crate located in `src/main/Cargo.toml`. The feature must +enable a feature in the other workspace crate(s) you intend to use it in. Then +the said workspace crate(s) must define the feature there in its `Cargo.toml`. + +So, if this is adding a feature to the API such as `woof`, you define the feature +in the `api` crate's `Cargo.toml` as `woof = []`. The feature definition in `main`'s +`Cargo.toml` will be `woof = ["conduit-api/woof"]`. + +The rationale for this is due to Rust / Cargo not supporting +["workspace level features"][9], we must make a choice of; either scattering +features all over the workspace crates, making it difficult for anyone to add +or remove default features; or define all the features in one central workspace +crate that propagate down/up to the other workspace crates. It is a Cargo pitfall, +and we'd like to see better developer UX in Rust's Workspaces. + +Additionally, the definition of one single place makes "feature collection" in our +Nix flake a million times easier instead of collecting and deduping them all from +searching in all the workspace crates' `Cargo.toml`s. Though we wouldn't need to +do this if Rust supported workspace-level features to begin with. + +## List of forked dependencies + +During conduwuit development, we have had to fork some dependencies to support our use-cases in some areas. This ranges from things said upstream project won't accept for any reason, faster-paced development (unresponsive or slow upstream), conduwuit-specific usecases, or @@ -53,3 +117,11 @@ RUSTFLAGS="--cfg tokio_unstable" cargo build \ [6]: https://github.com/tokio-rs/tracing/ [7]: https://docs.rs/tokio-console/latest/tokio_console/ [8]: https://github.com/zaidoon1/ +[9]: https://github.com/rust-lang/cargo/issues/12162 +[workspaces]: https://doc.rust-lang.org/cargo/reference/workspaces.html +[macros]: https://doc.rust-lang.org/book/ch19-06-macros.html +[syn]: https://docs.rs/syn/latest/syn/ +[proc-macro]: https://doc.rust-lang.org/reference/procedural-macros.html +[clap]: https://docs.rs/clap/latest/clap/ +[features]: https://doc.rust-lang.org/cargo/reference/features.html +[state]: https://docs.rs/axum/latest/axum/extract/struct.State.html From 6f643a4b068e88af26ee1f6ffac9fb74c4e65e0c Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 6 Sep 2024 21:08:21 -0400 Subject: [PATCH 0775/2091] bump rust to 1.81.0 Signed-off-by: strawberry --- Cargo.toml | 2 +- flake.nix | 2 +- rust-toolchain.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 691867b6..f6d2c7f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" -rust-version = "1.80.1" +rust-version = "1.81.0" version = "0.4.6" [workspace.metadata.crane] diff --git a/flake.nix b/flake.nix index e3bfe7fb..2439a7f4 100644 --- a/flake.nix +++ b/flake.nix @@ -27,7 +27,7 @@ file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` - sha256 = "sha256-3jVIIf5XPnUU1CRaTyAiO0XHVbJl12MSx3eucTXCjtE="; + sha256 = "sha256-VZZnlyP69+Y3crrLHQyJirqlHrTtGTsyiSnZB8jEvVo="; }; mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index a21340c2..892d9442 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -11,7 +11,7 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.80.1" +channel = "1.81.0" components = [ # For rust-analyzer "rust-src", From 5ae9a5ff318803ceda1e95cdc5e4366b6a088d67 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Sep 2024 08:27:03 -0400 Subject: [PATCH 0776/2091] bump nix lockfile, and ruma MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit • Updated input 'attic': 'github:zhaofengli/attic/6d9aeaef0a067d664cb11bb7704f7ec373d47fb2' (2024-08-21) → 'github:zhaofengli/attic/bea72d75b6165dfb529ba0c39cc6c7e9c7f0d234' (2024-09-02) • Added input 'attic/flake-parts': 'github:hercules-ci/flake-parts/8471fe90ad337a8074e957b69ca4d0089218391d' (2024-08-01) • Added input 'attic/flake-parts/nixpkgs-lib': follows 'attic/nixpkgs' • Updated input 'attic/nixpkgs': 'github:NixOS/nixpkgs/d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d' (2024-08-16) → 'github:NixOS/nixpkgs/b96f849e725333eb2b1c7f1cb84ff102062468ba' (2024-08-30) • Updated input 'attic/nixpkgs-stable': 'github:NixOS/nixpkgs/205fd4226592cc83fd4c0885a3e4c9c400efabb5' (2024-07-09) → 'github:NixOS/nixpkgs/797f7dc49e0bc7fab4b57c021cdf68f595e47841' (2024-08-22) • Updated input 'complement': 'github:matrix-org/complement/6e4426a9e63233f9821a4d2382bfed145244183f' (2024-07-30) → 'github:matrix-org/complement/39733c1b2f8314800776748cc7164f9a34650686' (2024-08-22) • Updated input 'crane': 'github:ipetkov/crane/7ce92819802bc583b7e82ebc08013a530f22209f' (2024-08-18) → 'github:ipetkov/crane/7e4586bad4e3f8f97a9271def747cf58c4b68f3c' (2024-09-04) • Removed input 'crane/nixpkgs' • Updated input 'fenix': 'github:nix-community/fenix/e88b38a5a3834e039d413a88f8150a75ef6453ef' (2024-08-21) → 'github:nix-community/fenix/d9afdb4465ba2f20bb73b0ff5d2c2837cafc2e14' (2024-09-06) • Updated input 'fenix/rust-analyzer-src': 'github:rust-lang/rust-analyzer/3723e5910c14f0ffbd13de474b8a8fcc74db04ce' (2024-08-20) → 'github:rust-lang/rust-analyzer/124c7482167ff6eea4f7663c0be87ea568ccd8c6' (2024-09-05) • Updated input 'liburing': 'github:axboe/liburing/2d4e799017d64cd2f8304503eef9064931bb3fbd' (2024-08-21) → 'github:axboe/liburing/0fe5c09195c0918f89582dd6ff098a58a0bdf62a' (2024-09-06) • Updated input 'nixpkgs': 'github:NixOS/nixpkgs/36a9aeaaa17a2d4348498275f9fe530cd4f9e519' (2024-08-21) → 'github:NixOS/nixpkgs/ad416d066ca1222956472ab7d0555a6946746a80' (2024-09-04) Signed-off-by: strawberry --- Cargo.lock | 30 ++++++++++---------- Cargo.toml | 2 +- flake.lock | 83 ++++++++++++++++++++++++++++++++---------------------- 3 files changed, 66 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca348f41..7a75e6fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2730,9 +2730,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.11.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4e75767fbc9d92b90e4d0c011f61358cde9513b31ef07ea3631b15ffc3b4fd" +checksum = "666f0f59e259aea2d72e6012290c09877a780935cc3c18b1ceded41f3890d59c" dependencies = [ "bitflags 2.6.0", "memchr", @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "assign", "js_int", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "js_int", "ruma-common", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "as_variant", "assign", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "as_variant", "base64 0.22.1", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "bytes", "http", @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "js_int", "thiserror", @@ -3113,7 +3113,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "js_int", "ruma-common", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "once_cell", "proc-macro-crate", @@ -3138,7 +3138,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "js_int", "ruma-common", @@ -3150,7 +3150,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "headers", "http", @@ -3163,7 +3163,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3179,7 +3179,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7ddcd036f81edb257ab9371f9cadd46444e8a90#d7ddcd036f81edb257ab9371f9cadd46444e8a90" +source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index f6d2c7f9..b6cc257c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -314,7 +314,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "d7ddcd036f81edb257ab9371f9cadd46444e8a90" +rev = "83a9bf73668aa459ffcf5ca6f95639835c0bac79" features = [ "compat", "rand", diff --git a/flake.lock b/flake.lock index c492e3be..b0508519 100644 --- a/flake.lock +++ b/flake.lock @@ -4,16 +4,17 @@ "inputs": { "crane": "crane", "flake-compat": "flake-compat", + "flake-parts": "flake-parts", "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1724226964, - "narHash": "sha256-cltFh4su2vcFidxKp7LuEgX3ZGLfPy0DCdrQZ/QTe68=", + "lastModified": 1725300620, + "narHash": "sha256-IdM+pZ6BnmD3o1fTJZ2BD43k7dwi1BbVfLDLpM1nE5s=", "owner": "zhaofengli", "repo": "attic", - "rev": "6d9aeaef0a067d664cb11bb7704f7ec373d47fb2", + "rev": "bea72d75b6165dfb529ba0c39cc6c7e9c7f0d234", "type": "github" }, "original": { @@ -81,11 +82,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1722323564, - "narHash": "sha256-6w6/N8walz4Ayc9zu7iySqJRmGFukhkaICLn4dweAcA=", + "lastModified": 1724347376, + "narHash": "sha256-y0e/ULDJ92IhNQZsS/06g0s+AYZ82aJfrIO9qEse94c=", "owner": "matrix-org", "repo": "complement", - "rev": "6e4426a9e63233f9821a4d2382bfed145244183f", + "rev": "39733c1b2f8314800776748cc7164f9a34650686", "type": "github" }, "original": { @@ -117,17 +118,12 @@ } }, "crane_2": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, "locked": { - "lastModified": 1724006180, - "narHash": "sha256-PVxPj0Ga2fMYMtcT9ARCthF+4U71YkOT7ZjgD/vf1Aw=", + "lastModified": 1725409566, + "narHash": "sha256-PrtLmqhM6UtJP7v7IGyzjBFhbG4eOAHT6LPYOFmYfbk=", "owner": "ipetkov", "repo": "crane", - "rev": "7ce92819802bc583b7e82ebc08013a530f22209f", + "rev": "7e4586bad4e3f8f97a9271def747cf58c4b68f3c", "type": "github" }, "original": { @@ -209,11 +205,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1724221791, - "narHash": "sha256-mKX67QPnUybOopVph/LhOV1G/H4EvPxDIfSmbufrVdA=", + "lastModified": 1725604324, + "narHash": "sha256-+VgeYuaCQn5vmoH1GTYQzvVTtWxirZmdDQJKr8uLgQI=", "owner": "nix-community", "repo": "fenix", - "rev": "e88b38a5a3834e039d413a88f8150a75ef6453ef", + "rev": "d9afdb4465ba2f20bb73b0ff5d2c2837cafc2e14", "type": "github" }, "original": { @@ -288,6 +284,27 @@ "type": "github" } }, + "flake-parts": { + "inputs": { + "nixpkgs-lib": [ + "attic", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1722555600, + "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "8471fe90ad337a8074e957b69ca4d0089218391d", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "flake-utils": { "inputs": { "systems": "systems" @@ -395,11 +412,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1724199144, - "narHash": "sha256-MVjnwO6EbKzzSrU51dSseLarZ1fRp+6SagAf/nE/XZU=", + "lastModified": 1725659644, + "narHash": "sha256-b7CciZaCayY6HcPQ/jjQk/sye/PINk0wP8R8Y3OMW0c=", "owner": "axboe", "repo": "liburing", - "rev": "2d4e799017d64cd2f8304503eef9064931bb3fbd", + "rev": "0fe5c09195c0918f89582dd6ff098a58a0bdf62a", "type": "github" }, "original": { @@ -508,11 +525,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1723827930, - "narHash": "sha256-EU+W5F6y2CVNxGrGIMpY7nSVYq72WRChYxF4zpjx0y4=", + "lastModified": 1724999960, + "narHash": "sha256-LB3jqSGW5u1ZcUcX6vO/qBOq5oXHlmOCxsTXGMEitp4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "d4a7a4d0e066278bfb0d77bd2a7adde1c0ec9e3d", + "rev": "b96f849e725333eb2b1c7f1cb84ff102062468ba", "type": "github" }, "original": { @@ -556,16 +573,16 @@ }, "nixpkgs-stable": { "locked": { - "lastModified": 1720535198, - "narHash": "sha256-zwVvxrdIzralnSbcpghA92tWu2DV2lwv89xZc8MTrbg=", + "lastModified": 1724316499, + "narHash": "sha256-Qb9MhKBUTCfWg/wqqaxt89Xfi6qTD3XpTzQ9eXi3JmE=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "205fd4226592cc83fd4c0885a3e4c9c400efabb5", + "rev": "797f7dc49e0bc7fab4b57c021cdf68f595e47841", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } @@ -620,11 +637,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1724271409, - "narHash": "sha256-z4nw9HxkaXEn+5OT8ljLVL2oataHvAzUQ1LEi8Fp+SY=", + "lastModified": 1725432240, + "narHash": "sha256-+yj+xgsfZaErbfYM3T+QvEE2hU7UuE+Jf0fJCJ8uPS0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "36a9aeaaa17a2d4348498275f9fe530cd4f9e519", + "rev": "ad416d066ca1222956472ab7d0555a6946746a80", "type": "github" }, "original": { @@ -695,11 +712,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1724153119, - "narHash": "sha256-WxpvDJDttkINkXmUA/W5o11lwLPYhATAgu0QUAacZ2g=", + "lastModified": 1725548942, + "narHash": "sha256-ZnF5MaOAeiiKIATYN4rrqNsnhSQOQ+Hvfg0mHLvN04Y=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "3723e5910c14f0ffbd13de474b8a8fcc74db04ce", + "rev": "124c7482167ff6eea4f7663c0be87ea568ccd8c6", "type": "github" }, "original": { From f163ebf3bbaa9656bbf1d78355220e1d859b17c7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Sep 2024 09:26:50 -0400 Subject: [PATCH 0777/2091] implement MSC4133 only with MSC4175 for GET/PUT/DELETE Signed-off-by: strawberry --- src/api/client/profile.rs | 32 +++++++--- src/api/client/unstable.rs | 128 ++++++++++++++++++++++++++++++++++++- src/api/router.rs | 3 + src/api/server/query.rs | 3 + src/database/maps.rs | 1 + src/service/users/data.rs | 53 +++++++++++++++ src/service/users/mod.rs | 7 ++ 7 files changed, 217 insertions(+), 10 deletions(-) diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 71d49cd8..0e1e0f0a 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{pdu::PduBuilder, warn, Error, Result}; +use conduit::{pdu::PduBuilder, warn, Err, Error, Result}; use ruma::{ api::{ client::{ @@ -26,20 +26,25 @@ pub(crate) async fn set_displayname_route( State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if *sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot update the profile of another user"))); + } + let all_joined_rooms: Vec = services .rooms .state_cache - .rooms_joined(sender_user) + .rooms_joined(&body.user_id) .filter_map(Result::ok) .collect(); - update_displayname(&services, sender_user.clone(), body.displayname.clone(), all_joined_rooms).await?; + update_displayname(&services, body.user_id.clone(), body.displayname.clone(), all_joined_rooms).await?; if services.globals.allow_local_presence() { // Presence update services .presence - .ping_presence(sender_user, &PresenceState::Online)?; + .ping_presence(&body.user_id, &PresenceState::Online)?; } Ok(set_display_name::v3::Response {}) @@ -110,16 +115,21 @@ pub(crate) async fn set_avatar_url_route( State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if *sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot update the profile of another user"))); + } + let all_joined_rooms: Vec = services .rooms .state_cache - .rooms_joined(sender_user) + .rooms_joined(&body.user_id) .filter_map(Result::ok) .collect(); update_avatar_url( &services, - sender_user.clone(), + body.user_id.clone(), body.avatar_url.clone(), body.blurhash.clone(), all_joined_rooms, @@ -130,7 +140,7 @@ pub(crate) async fn set_avatar_url_route( // Presence update services .presence - .ping_presence(sender_user, &PresenceState::Online)?; + .ping_presence(&body.user_id, &PresenceState::Online)?; } Ok(set_avatar_url::v3::Response {}) @@ -196,7 +206,7 @@ pub(crate) async fn get_avatar_url_route( /// # `GET /_matrix/client/v3/profile/{userId}` /// -/// Returns the displayname, avatar_url and blurhash of the user. +/// Returns the displayname, avatar_url, blurhash, and tz of the user. /// /// - If user is on another server and we do not have a local copy already, /// fetch profile over federation. @@ -232,11 +242,16 @@ pub(crate) async fn get_profile_route( .users .set_blurhash(&body.user_id, response.blurhash.clone()) .await?; + services + .users + .set_timezone(&body.user_id, response.tz.clone()) + .await?; return Ok(get_profile::v3::Response { displayname: response.displayname, avatar_url: response.avatar_url, blurhash: response.blurhash, + tz: response.tz, }); } } @@ -251,6 +266,7 @@ pub(crate) async fn get_profile_route( avatar_url: services.users.avatar_url(&body.user_id)?, blurhash: services.users.blurhash(&body.user_id)?, displayname: services.users.displayname(&body.user_id)?, + tz: services.users.timezone(&body.user_id)?, }) } diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 89650951..d672ad46 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -1,9 +1,18 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::warn; +use conduit::{warn, Err}; use ruma::{ - api::client::{error::ErrorKind, membership::mutual_rooms, room::get_summary}, + api::{ + client::{ + error::ErrorKind, + membership::mutual_rooms, + profile::{delete_timezone_key, get_timezone_key, set_timezone_key}, + room::get_summary, + }, + federation, + }, events::room::member::MembershipState, + presence::PresenceState, OwnedRoomId, }; @@ -161,3 +170,118 @@ pub(crate) async fn get_room_summary( .unwrap_or_else(|_e| None), }) } + +/// # `DELETE /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz` +/// +/// Deletes the `tz` (timezone) of a user, as per MSC4133 and MSC4175. +/// +/// - Also makes sure other users receive the update using presence EDUs +pub(crate) async fn delete_timezone_key_route( + State(services): State, body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if *sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot update the profile of another user"))); + } + + services.users.set_timezone(&body.user_id, None).await?; + + if services.globals.allow_local_presence() { + // Presence update + services + .presence + .ping_presence(&body.user_id, &PresenceState::Online)?; + } + + Ok(delete_timezone_key::unstable::Response {}) +} + +/// # `PUT /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz` +/// +/// Updates the `tz` (timezone) of a user, as per MSC4133 and MSC4175. +/// +/// - Also makes sure other users receive the update using presence EDUs +pub(crate) async fn set_timezone_key_route( + State(services): State, body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if *sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot update the profile of another user"))); + } + + services + .users + .set_timezone(&body.user_id, body.tz.clone()) + .await?; + + if services.globals.allow_local_presence() { + // Presence update + services + .presence + .ping_presence(&body.user_id, &PresenceState::Online)?; + } + + Ok(set_timezone_key::unstable::Response {}) +} + +/// # `GET /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz` +/// +/// Returns the `timezone` of the user as per MSC4133 and MSC4175. +/// +/// - If user is on another server and we do not have a local copy already fetch +/// `timezone` over federation +pub(crate) async fn get_timezone_key_route( + State(services): State, body: Ruma, +) -> Result { + if !services.globals.user_is_local(&body.user_id) { + // Create and update our local copy of the user + if let Ok(response) = services + .sending + .send_federation_request( + body.user_id.server_name(), + federation::query::get_profile_information::v1::Request { + user_id: body.user_id.clone(), + field: None, // we want the full user's profile to update locally as well + }, + ) + .await + { + if !services.users.exists(&body.user_id)? { + services.users.create(&body.user_id, None)?; + } + + services + .users + .set_displayname(&body.user_id, response.displayname.clone()) + .await?; + services + .users + .set_avatar_url(&body.user_id, response.avatar_url.clone()) + .await?; + services + .users + .set_blurhash(&body.user_id, response.blurhash.clone()) + .await?; + services + .users + .set_timezone(&body.user_id, response.tz.clone()) + .await?; + + return Ok(get_timezone_key::unstable::Response { + tz: response.tz, + }); + } + } + + if !services.users.exists(&body.user_id)? { + // Return 404 if this user doesn't exist and we couldn't fetch it over + // federation + return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); + } + + Ok(get_timezone_key::unstable::Response { + tz: services.users.timezone(&body.user_id)?, + }) +} diff --git a/src/api/router.rs b/src/api/router.rs index 985f803b..77e9a11a 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -22,6 +22,9 @@ use crate::{client, server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; let mut router = router + .ruma_route(client::get_timezone_key_route) + .ruma_route(client::set_timezone_key_route) + .ruma_route(client::delete_timezone_key_route) .ruma_route(client::appservice_ping) .ruma_route(client::get_supported_versions_route) .ruma_route(client::get_register_available_route) diff --git a/src/api/server/query.rs b/src/api/server/query.rs index 5712f46a..45830366 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -75,6 +75,7 @@ pub(crate) async fn get_profile_information_route( let mut displayname = None; let mut avatar_url = None; let mut blurhash = None; + let mut tz = None; match &body.field { Some(ProfileField::DisplayName) => { @@ -90,6 +91,7 @@ pub(crate) async fn get_profile_information_route( displayname = services.users.displayname(&body.user_id)?; avatar_url = services.users.avatar_url(&body.user_id)?; blurhash = services.users.blurhash(&body.user_id)?; + tz = services.users.timezone(&body.user_id)?; }, } @@ -97,5 +99,6 @@ pub(crate) async fn get_profile_information_route( displayname, avatar_url, blurhash, + tz, }) } diff --git a/src/database/maps.rs b/src/database/maps.rs index 0207b4d4..0e835abf 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -94,6 +94,7 @@ pub const MAPS: &[&str] = &[ "userid_presenceid", "userid_selfsigningkeyid", "userid_usersigningkeyid", + "useridprofilekey_value", "openidtoken_expiresatuserid", "userroomid_highlightcount", "userroomid_invitestate", diff --git a/src/service/users/data.rs b/src/service/users/data.rs index e75a2e93..70785d68 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -32,6 +32,7 @@ pub struct Data { userid_password: Arc, userid_selfsigningkeyid: Arc, userid_usersigningkeyid: Arc, + useridprofilekey_value: Arc, services: Services, } @@ -64,6 +65,7 @@ impl Data { userid_password: db["userid_password"].clone(), userid_selfsigningkeyid: db["userid_selfsigningkeyid"].clone(), userid_usersigningkeyid: db["userid_usersigningkeyid"].clone(), + useridprofilekey_value: db["useridprofilekey_value"].clone(), services: Services { server: args.server.clone(), globals: args.depend::("globals"), @@ -231,6 +233,57 @@ impl Data { .transpose() } + /// Get the timezone of a user. + pub(super) fn timezone(&self, user_id: &UserId) -> Result> { + // first check the unstable prefix + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(b"us.cloke.msc4175.tz"); + + let value = self + .useridprofilekey_value + .get(&key)? + .map(|bytes| utils::string_from_bytes(&bytes).map_err(|e| err!(Database("Timezone in db is invalid. {e}")))) + .transpose() + .unwrap(); + + // TODO: transparently migrate unstable key usage to the stable key once MSC4133 + // and MSC4175 are stable, likely a remove/insert in this block + if value.is_none() || value.as_ref().is_some_and(String::is_empty) { + // check the stable prefix + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(b"m.tz"); + + return self + .useridprofilekey_value + .get(&key)? + .map(|bytes| { + utils::string_from_bytes(&bytes).map_err(|e| err!(Database("Timezone in db is invalid. {e}"))) + }) + .transpose(); + } + + Ok(value) + } + + /// Sets a new timezone or removes it if timezone is None. + pub(super) fn set_timezone(&self, user_id: &UserId, timezone: Option) -> Result<()> { + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(b"us.cloke.msc4175.tz"); + + // TODO: insert to the stable MSC4175 key when it's stable + if let Some(timezone) = timezone { + self.useridprofilekey_value + .insert(&key, timezone.as_bytes())?; + } else { + self.useridprofilekey_value.remove(&key)?; + } + + Ok(()) + } + /// Sets a new avatar_url or removes it if avatar_url is None. pub(super) fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()> { if let Some(blurhash) = blurhash { diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 35f30a61..544614d4 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -327,6 +327,13 @@ impl Service { /// Get the blurhash of a user. pub fn blurhash(&self, user_id: &UserId) -> Result> { self.db.blurhash(user_id) } + pub fn timezone(&self, user_id: &UserId) -> Result> { self.db.timezone(user_id) } + + /// Sets a new tz or removes it if tz is None. + pub async fn set_timezone(&self, user_id: &UserId, tz: Option) -> Result<()> { + self.db.set_timezone(user_id, tz) + } + /// Sets a new blurhash or removes it if blurhash is None. pub async fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()> { self.db.set_blurhash(user_id, blurhash) From b3974c569dd1787e0217a6600dbbe47a9a0e4f0b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Sep 2024 10:10:46 -0400 Subject: [PATCH 0778/2091] log device display name on normal user registrations too Signed-off-by: strawberry --- src/api/client/account.rs | 59 +++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index bf71bc0d..e20e2fac 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -296,48 +296,47 @@ pub(crate) async fn register_route( debug_info!(%user_id, %device_id, "User account was created"); + let device_display_name = body.initial_device_display_name.clone().unwrap_or_default(); + // log in conduit admin channel if a non-guest user registered if body.appservice_info.is_none() && !is_guest { - info!("New user \"{user_id}\" registered on this server."); - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "New user \"{user_id}\" registered on this server from IP {client}." - ))) - .await; + if !device_display_name.is_empty() { + info!("New user \"{user_id}\" registered on this server with device display name: {device_display_name}"); + services + .admin + .send_message(RoomMessageEventContent::notice_plain(format!( + "New user \"{user_id}\" registered on this server from IP {client} and device display name \ + \"{device_display_name}\"" + ))) + .await; + } else { + info!("New user \"{user_id}\" registered on this server."); + services + .admin + .send_message(RoomMessageEventContent::notice_plain(format!( + "New user \"{user_id}\" registered on this server from IP {client}" + ))) + .await; + } } // log in conduit admin channel if a guest registered if body.appservice_info.is_none() && is_guest && services.globals.log_guest_registrations() { info!("New guest user \"{user_id}\" registered on this server."); - if let Some(device_display_name) = &body.initial_device_display_name { - if body - .initial_device_display_name - .as_ref() - .is_some_and(|device_display_name| !device_display_name.is_empty()) - { - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with device display name `{device_display_name}` registered on this \ - server from IP {client}." - ))) - .await; - } else { - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with no device display name registered on this server from IP \ - {client}.", - ))) - .await; - } + if !device_display_name.is_empty() { + services + .admin + .send_message(RoomMessageEventContent::notice_plain(format!( + "Guest user \"{user_id}\" with device display name \"{device_display_name}\" registered on this \ + server from IP {client}" + ))) + .await; } else { services .admin .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with no device display name registered on this server from IP {client}.", + "Guest user \"{user_id}\" with no device display name registered on this server from IP {client}", ))) .await; } From f14a2536646fc620f1408d15341695043f556efe Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Sep 2024 12:46:59 -0400 Subject: [PATCH 0779/2091] add local_only arg to list joined members in room admin cmd Signed-off-by: strawberry --- src/admin/room/info.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index 8ba0a796..d17a2924 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -10,6 +10,10 @@ pub(crate) enum RoomInfoCommand { /// - List joined members in a room ListJoinedMembers { room_id: Box, + + /// Lists only our local users in the specified room + #[arg(long)] + local_only: bool, }, /// - Displays room topic @@ -22,7 +26,7 @@ pub(crate) enum RoomInfoCommand { } #[admin_command] -async fn list_joined_members(&self, room_id: Box) -> Result { +async fn list_joined_members(&self, room_id: Box, local_only: bool) -> Result { let room_name = self .services .rooms @@ -37,7 +41,15 @@ async fn list_joined_members(&self, room_id: Box) -> Result Date: Sun, 8 Sep 2024 05:03:30 +0200 Subject: [PATCH 0780/2091] nix: update flake.lock, skip building tests for liburing Signed-off-by: morguldir --- flake.lock | 22 +++++++++++----------- flake.nix | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/flake.lock b/flake.lock index b0508519..15ce8e43 100644 --- a/flake.lock +++ b/flake.lock @@ -205,11 +205,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1725604324, - "narHash": "sha256-+VgeYuaCQn5vmoH1GTYQzvVTtWxirZmdDQJKr8uLgQI=", + "lastModified": 1725690497, + "narHash": "sha256-5fT+96rV7Hx29HG+4/oBbr3V+yExKuLN2vcBcPbVBlU=", "owner": "nix-community", "repo": "fenix", - "rev": "d9afdb4465ba2f20bb73b0ff5d2c2837cafc2e14", + "rev": "4b8d964df93d1f918ee6c4f003b3548c432cc866", "type": "github" }, "original": { @@ -413,7 +413,7 @@ "flake": false, "locked": { "lastModified": 1725659644, - "narHash": "sha256-b7CciZaCayY6HcPQ/jjQk/sye/PINk0wP8R8Y3OMW0c=", + "narHash": "sha256-WjnpmopfvFoUbubIu9bki+Y6P4YXDfvnW4+72hniq3g=", "owner": "axboe", "repo": "liburing", "rev": "0fe5c09195c0918f89582dd6ff098a58a0bdf62a", @@ -637,16 +637,16 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1725432240, - "narHash": "sha256-+yj+xgsfZaErbfYM3T+QvEE2hU7UuE+Jf0fJCJ8uPS0=", + "lastModified": 1725534445, + "narHash": "sha256-Yd0FK9SkWy+ZPuNqUgmVPXokxDgMJoGuNpMEtkfcf84=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ad416d066ca1222956472ab7d0555a6946746a80", + "rev": "9bb1e7571aadf31ddb4af77fc64b2d59580f9a39", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-unstable", + "ref": "nixpkgs-unstable", "repo": "nixpkgs", "type": "github" } @@ -712,11 +712,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1725548942, - "narHash": "sha256-ZnF5MaOAeiiKIATYN4rrqNsnhSQOQ+Hvfg0mHLvN04Y=", + "lastModified": 1725630423, + "narHash": "sha256-gNCLk3Zg7JlAwmWbVHTH6f3+iqdeQ4fheOotCZy8x5M=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "124c7482167ff6eea4f7663c0be87ea568ccd8c6", + "rev": "08c7bbc2dbe4dcc8968484f1a0e1e6fe7a1d4f6d", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 2439a7f4..1351cd3c 100644 --- a/flake.nix +++ b/flake.nix @@ -3,12 +3,12 @@ attic.url = "github:zhaofengli/attic?ref=main"; cachix.url = "github:cachix/cachix?ref=master"; complement = { url = "github:matrix-org/complement?ref=main"; flake = false; }; - crane = { url = "github:ipetkov/crane?ref=master"; inputs.nixpkgs.follows = "nixpkgs"; }; + crane = { url = "github:ipetkov/crane?ref=master"; }; fenix = { url = "github:nix-community/fenix?ref=main"; inputs.nixpkgs.follows = "nixpkgs"; }; flake-compat = { url = "github:edolstra/flake-compat?ref=master"; flake = false; }; flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; - nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-unstable"; + nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.5.2"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; @@ -38,7 +38,15 @@ inherit inputs; main = self.callPackage ./nix/pkgs/main {}; oci-image = self.callPackage ./nix/pkgs/oci-image {}; - rocksdb = pkgs.rocksdb.overrideAttrs (old: { + liburing = pkgs.liburing.overrideAttrs { + # Tests weren't building + outputs = [ "out" "dev" "man" ]; + buildFlags = [ "library"]; + src = inputs.liburing; + }; + rocksdb = (pkgs.rocksdb.override { + liburing = self.liburing; + }).overrideAttrs (old: { src = inputs.rocksdb; version = pkgs.lib.removePrefix "v" @@ -76,14 +84,6 @@ # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use preInstall = ""; }); - # TODO: remove once https://github.com/NixOS/nixpkgs/pull/314945 is available - liburing = pkgs.liburing.overrideAttrs (old: { - # the configure script doesn't support these, and unconditionally - # builds both static and dynamic libraries. - configureFlags = pkgs.lib.subtractLists - [ "--enable-static" "--disable-shared" ] - old.configureFlags; - }); }); scopeHost = mkScope pkgsHost; From 3e0d404fb4b6b1506824f8d5dfeea7672dea3448 Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 7 Sep 2024 18:09:21 +0200 Subject: [PATCH 0781/2091] syncv3: use RoomTypeFilter struct instead of Option Signed-off-by: morguldir Signed-off-by: strawberry --- Cargo.lock | 27 ++++++++++++++------------- Cargo.toml | 4 ++-- src/api/client/sync.rs | 39 +++++++++++++++------------------------ 3 files changed, 31 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7a75e6fd..539a9a2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "assign", "js_int", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "js_int", "ruma-common", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "as_variant", "assign", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "as_variant", "base64 0.22.1", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "as_variant", "indexmap 2.4.0", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "bytes", "http", @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "js_int", "thiserror", @@ -3113,7 +3113,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "js_int", "ruma-common", @@ -3123,8 +3123,9 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ + "cfg-if", "once_cell", "proc-macro-crate", "proc-macro2", @@ -3138,7 +3139,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "js_int", "ruma-common", @@ -3150,7 +3151,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "headers", "http", @@ -3163,7 +3164,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3179,7 +3180,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=83a9bf73668aa459ffcf5ca6f95639835c0bac79#83a9bf73668aa459ffcf5ca6f95639835c0bac79" +source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index b6cc257c..3419983a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -314,7 +314,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "83a9bf73668aa459ffcf5ca6f95639835c0bac79" +rev = "11155e576a1382783c0bcf5ad4458708777ec36e" features = [ "compat", "rand", @@ -608,7 +608,7 @@ inherits = "release" # and can be raised if build times are tolerable. [profile.dev] -debug = 1 +debug = "full" opt-level = 0 panic = "unwind" debug-assertions = true diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 4d080758..eb534205 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -25,12 +25,12 @@ use ruma::{ }, uiaa::UiaaResponse, }, + directory::RoomTypeFilter, events::{ presence::PresenceEvent, room::member::{MembershipState, RoomMemberEventContent}, AnyRawAccountDataEvent, StateEventType, TimelineEventType, }, - room::RoomType, serde::Raw, state_res::Event, uint, DeviceId, EventId, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId, @@ -1760,32 +1760,23 @@ pub(crate) async fn sync_events_v4_route( } fn filter_rooms( - rooms: &[OwnedRoomId], State(services): State, filter: &[Option], negate: bool, + rooms: &[OwnedRoomId], State(services): State, filter: &[RoomTypeFilter], negate: bool, ) -> Vec { return rooms .iter() - .filter(|r| { - match services.rooms.state_accessor.get_room_type(r) { - Err(e) => { - warn!("Requested room type for {}, but could not retrieve with error {}", r, e); - false - }, - Ok(None) => { - // For rooms which do not have a room type, use 'null' to include them - if negate { - !filter.contains(&None) - } else { - filter.contains(&None) - } - }, - Ok(Some(room_type)) => { - if negate { - !filter.contains(&Some(room_type)) - } else { - filter.is_empty() || filter.contains(&Some(room_type)) - } - }, - } + .filter(|r| match services.rooms.state_accessor.get_room_type(r) { + Err(e) => { + warn!("Requested room type for {}, but could not retrieve with error {}", r, e); + false + }, + Ok(result) => { + let result = RoomTypeFilter::from(result); + if negate { + !filter.contains(&result) + } else { + filter.is_empty() || filter.contains(&result) + } + }, }) .cloned() .collect(); From c47337f3dbe9289df11fa4b20ce1b2d825cb2aa8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 8 Sep 2024 10:24:38 -0400 Subject: [PATCH 0782/2091] docs: ignore development.md and contributing.md from lychee and update 2 other nix references Signed-off-by: strawberry Signed-off-by: morguldir --- docs/deploying/generic.md | 2 +- docs/development/testing.md | 5 ++--- engage.toml | 2 +- nix/pkgs/book/default.nix | 2 ++ 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index f0391fec..cae57986 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -18,7 +18,7 @@ These binaries have jemalloc and io_uring statically linked and included with them. Alternatively, you may compile the binary yourself. We recommend using -[Lix](https://lix.systems) to build conduwuit as this has the most guaranteed +Nix (or [Lix](https://lix.systems) to build conduwuit as this has the most guaranteed reproducibiltiy and easiest to get a build environment and output going. Otherwise, follow standard Rust project build guides (installing git and cloning diff --git a/docs/development/testing.md b/docs/development/testing.md index 00f8e258..06720dd8 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -5,8 +5,8 @@ Have a look at [Complement's repository][complement] for an explanation of what it is. -To test against Complement, with [Lix][lix] and direnv installed and set up, you -can: +To test against Complement, with Nix (or [Lix](https://lix.systems) and direnv installed +and set up, you can: * Run `./bin/complement "$COMPLEMENT_SRC" ./path/to/logs.jsonl ./path/to/results.jsonl` to build a Complement image, run the tests, and output @@ -18,6 +18,5 @@ Complement OCI image outputted to `result` (it's a `.tar.gz` file) output from the commit/revision you want to test (e.g. from main) [here][ci-workflows] -[lix]: https://lix.systems/ [ci-workflows]: https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml?query=event%3Apush+is%3Asuccess+actor%3Agirlbossceo [complement]: https://github.com/matrix-org/complement diff --git a/engage.toml b/engage.toml index e2428aea..633cb95d 100644 --- a/engage.toml +++ b/engage.toml @@ -152,7 +152,7 @@ cargo clippy \ [[task]] name = "lychee" group = "lints" -script = "lychee --verbose --offline docs *.md --exclude development.md" +script = "lychee --verbose --offline docs *.md --exclude development.md --exclude contributing.md --exclude testing.md" [[task]] name = "markdownlint" diff --git a/nix/pkgs/book/default.nix b/nix/pkgs/book/default.nix index cafd162f..3995ab79 100644 --- a/nix/pkgs/book/default.nix +++ b/nix/pkgs/book/default.nix @@ -14,8 +14,10 @@ stdenv.mkDerivation { include = [ "book.toml" "conduwuit-example.toml" + "CODE_OF_CONDUCT.md" "CONTRIBUTING.md" "README.md" + "development.md" "debian/conduwuit.service" "debian/README.md" "arch/conduwuit.service" From 96ab59b5b0c90f41d4748d10f5263dd7c546d157 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 8 Sep 2024 10:28:21 -0400 Subject: [PATCH 0783/2091] bump cargo.lock Signed-off-by: strawberry --- Cargo.lock | 284 +++++++++++++++++++++++++---------------------------- 1 file changed, 136 insertions(+), 148 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 539a9a2e..ce5ba829 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,9 +55,9 @@ checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" [[package]] name = "arc-swap" @@ -130,18 +130,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -167,9 +167,9 @@ checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "aws-lc-rs" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae74d9bd0a7530e8afd1770739ad34b36838829d6ad61818f9230f683f5ad77" +checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -179,9 +179,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f0e249228c6ad2d240c2dc94b714d711629d52bad946075d8e9b2f5391f0703" +checksum = "5055edc4a9a1b2a917a818258cdfb86a535947feebd9981adc99667a062c6f85" dependencies = [ "bindgen", "cc", @@ -376,7 +376,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.76", + "syn 2.0.77", "which", ] @@ -439,9 +439,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fd4c6dcc3b0aea2f5c0b4b82c2b15fe39ddbc76041a310848f4706edf76bb31" +checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" [[package]] name = "byteorder" @@ -484,9 +484,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.14" +version = "1.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d2eb3cd3d1bf4529e31c215ee6f93ec5a3d536d9f578f93d9d33ee19562932" +checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" dependencies = [ "jobserver", "libc", @@ -545,9 +545,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.16" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6719fffa43d0d87e5fd8caeab59be1554fb028cd30edc88fc4369b17971019" +checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" dependencies = [ "clap_builder", "clap_derive", @@ -555,9 +555,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.15" +version = "4.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216aec2b177652e3846684cbfe25c9964d18ec45234f0f5da5157b207ed1aab6" +checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" dependencies = [ "anstyle", "clap_lex", @@ -572,7 +572,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -746,7 +746,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -870,9 +870,9 @@ checksum = "3618cccc083bb987a415d85c02ca6c9994ea5b44731ec28b9ecf09658655fba9" [[package]] name = "const_panic" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6051f239ecec86fde3410901ab7860d458d160371533842974fc61f96d15879b" +checksum = "7782af8f90fe69a4bb41e460abe1727d493403d8b2cc43201a3a3e906b24379f" [[package]] name = "coolor" @@ -901,9 +901,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e852e6dc9a5bed1fae92dd2375037bf2b768725bf3be87811edee3249d09ad" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -1042,7 +1042,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1069,7 +1069,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1179,7 +1179,7 @@ dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1241,9 +1241,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.32" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" +checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", "miniz_oxide 0.8.0", @@ -1331,7 +1331,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1420,7 +1420,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.4.0", + "indexmap 2.5.0", "slab", "tokio", "tokio-util", @@ -1602,7 +1602,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1618,9 +1618,9 @@ dependencies = [ [[package]] name = "http-auth" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643c9bbf6a4ea8a656d6b4cd53d34f79e3f841ad5203c1a55fb7d761923bc255" +checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" dependencies = [ "memchr", ] @@ -1689,16 +1689,16 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.2" +version = "0.27.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" +checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" dependencies = [ "futures-util", "http", "hyper", "hyper-util", "rustls 0.23.12", - "rustls-native-certs", + "rustls-native-certs 0.8.0", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1798,9 +1798,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ead53efc7ea8ed3cfb0c79fc8023fbb782a5432b52830b6518941cebe6505c" +checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" dependencies = [ "equivalent", "hashbrown 0.14.5", @@ -1847,9 +1847,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" [[package]] name = "itertools" @@ -1966,7 +1966,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -1999,9 +1999,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.19" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc53a7799a7496ebc9fd29f31f7df80e83c9bda5299768af5f9e59eeea74647" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "pkg-config", @@ -2320,9 +2320,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.3" +version = "0.36.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9" +checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" dependencies = [ "memchr", ] @@ -2347,7 +2347,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.4.0", + "indexmap 2.5.0", "js-sys", "once_cell", "pin-project-lite", @@ -2497,7 +2497,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -2590,7 +2590,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -2657,21 +2657,21 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.20" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "proc-macro-crate" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.21.1", + "toml_edit", ] [[package]] @@ -2691,16 +2691,16 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "version_check", "yansi", ] [[package]] name = "prost" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" +checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995" dependencies = [ "bytes", "prost-derive", @@ -2708,22 +2708,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" +checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" dependencies = [ "anyhow", "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] name = "prost-types" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" +checksum = "60caa6738c7369b940c3d49246a8d1749323674c65cb13010134f5c9bad5b519" dependencies = [ "prost", ] @@ -2760,9 +2760,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quinn" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", @@ -2778,9 +2778,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand", @@ -2795,15 +2795,15 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ "libc", "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2927,7 +2927,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.12", - "rustls-native-certs", + "rustls-native-certs 0.7.3", "rustls-pemfile", "rustls-pki-types", "serde", @@ -3039,7 +3039,7 @@ dependencies = [ "bytes", "form_urlencoded", "http", - "indexmap 2.4.0", + "indexmap 2.5.0", "js_int", "konst", "percent-encoding", @@ -3065,7 +3065,7 @@ version = "0.28.1" source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" dependencies = [ "as_variant", - "indexmap 2.4.0", + "indexmap 2.5.0", "js_int", "js_option", "percent-encoding", @@ -3132,7 +3132,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.76", + "syn 2.0.77", "toml", ] @@ -3246,18 +3246,18 @@ checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver", ] [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" dependencies = [ "bitflags 2.6.0", "errno", @@ -3298,9 +3298,22 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04182dffc9091a404e0fc069ea5cd60e5b866c3adf881eff99a32d048242dffa" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" dependencies = [ "openssl-probe", "rustls-pemfile", @@ -3327,9 +3340,9 @@ checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" dependencies = [ "aws-lc-rs", "ring", @@ -3376,11 +3389,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3561,22 +3574,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99fce0ffe7310761ca6bf9faf5115afbc19688edd00171d81b1bb1b116c63e09" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.209" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5831b979fd7b5439637af1752d535ff49f4860c0f341d1baeb6faf0f4242170" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -3586,7 +3599,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" dependencies = [ "form_urlencoded", - "indexmap 2.4.0", + "indexmap 2.5.0", "itoa", "ryu", "serde", @@ -3594,9 +3607,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.127" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8043c06d9f82bd7271361ed64f415fe5e12a77fdb52e573e7f06a516dea329ad" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", "memchr", @@ -3651,7 +3664,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "itoa", "ryu", "serde", @@ -3870,9 +3883,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.76" +version = "2.0.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" +checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" dependencies = [ "proc-macro2", "quote", @@ -3948,7 +3961,7 @@ checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4083,7 +4096,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4123,9 +4136,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite", @@ -4134,9 +4147,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -4154,7 +4167,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.20", + "toml_edit", ] [[package]] @@ -4166,35 +4179,24 @@ dependencies = [ "serde", ] -[[package]] -name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.4.0", - "toml_datetime", - "winnow 0.5.40", -] - [[package]] name = "toml_edit" version = "0.22.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" dependencies = [ - "indexmap 2.4.0", + "indexmap 2.5.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.18", + "winnow", ] [[package]] name = "tonic" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38659f4a91aba8598d27821589f5db7dddd94601e7a01b1e485a50e5484c7401" +checksum = "c6f6ba989e4b2c58ae83d862d3a3e27690b6e3ae630d0deb59f3697f32aa88ad" dependencies = [ "async-stream", "async-trait", @@ -4307,7 +4309,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b47 dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4578,7 +4580,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wasm-bindgen-shared", ] @@ -4612,7 +4614,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4667,9 +4669,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.3" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd7c23921eeb1713a4e851530e9b9756e4fb0e89978582942612524cf09f01cd" +checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" dependencies = [ "rustls-pki-types", ] @@ -4793,6 +4795,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -4914,15 +4925,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "winnow" -version = "0.5.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" -dependencies = [ - "memchr", -] - [[package]] name = "winnow" version = "0.6.18" @@ -4977,7 +4979,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.76", + "syn 2.0.77", ] [[package]] @@ -4985,20 +4987,6 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" -dependencies = [ - "zeroize_derive", -] - -[[package]] -name = "zeroize_derive" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.76", -] [[package]] name = "zstd" From 8dccc04b40df175f322e094526fd6f0fa6eee00d Mon Sep 17 00:00:00 2001 From: morguldir Date: Sun, 8 Sep 2024 17:40:02 +0200 Subject: [PATCH 0784/2091] nix: explicitly include liburing in the devshell Signed-off-by: morguldir --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 1351cd3c..7f2bfeb9 100644 --- a/flake.nix +++ b/flake.nix @@ -118,6 +118,7 @@ ++ (with pkgsHost.pkgs; [ engage cargo-audit + liburing # Needed for producing Debian packages cargo-deb From 1c1f300efea6fb6d2848fe1d4872e5b4813917fc Mon Sep 17 00:00:00 2001 From: morguldir Date: Sun, 8 Sep 2024 17:41:02 +0200 Subject: [PATCH 0785/2091] ci: avoid propagating bash errors immidiately Signed-off-by: morguldir --- .github/workflows/ci.yml | 5 +++++ .github/workflows/documentation.yml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6959ec90..05dc7e6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,7 @@ jobs: - name: Cache CI dependencies run: | # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + set +e # Don't exit on errors ATTEMPTS=3 SUCCESS=false while (( ATTEMPTS-- > 0 )) @@ -305,6 +306,7 @@ jobs: SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + set +e # Don't exit on errors ATTEMPTS=3 SUCCESS=false while (( ATTEMPTS-- > 0 )) @@ -345,6 +347,7 @@ jobs: SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + set +e # Don't exit on errors ATTEMPTS=3 SUCCESS=false while (( ATTEMPTS-- > 0 )) @@ -424,6 +427,7 @@ jobs: - name: Build OCI image ${{ matrix.target }} run: | # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + set +e # Don't exit on errors ATTEMPTS=3 SUCCESS=false while (( ATTEMPTS-- > 0 )) @@ -446,6 +450,7 @@ jobs: - name: Build debug OCI image ${{ matrix.target }} run: | # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + set +e # Don't exit on errors ATTEMPTS=3 SUCCESS=false while (( ATTEMPTS-- > 0 )) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 37f2c0f0..68fe3557 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -111,6 +111,7 @@ jobs: - name: Cache CI dependencies run: | # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + set +e # Don't exit on errors ATTEMPTS=3 SUCCESS=false while (( ATTEMPTS-- > 0 )) @@ -136,6 +137,7 @@ jobs: - name: Build documentation (book) run: | # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here + set +e # Don't exit on errors ATTEMPTS=3 SUCCESS=false while (( ATTEMPTS-- > 0 )) From 24c408f4c6b780399f47632f0bfa99739d9515a6 Mon Sep 17 00:00:00 2001 From: morguldir Date: Sun, 8 Sep 2024 19:21:23 +0200 Subject: [PATCH 0786/2091] bump rocksdb to 9.6.1, add binutils to devshell --- Cargo.lock | 8 ++++---- deps/rust-rocksdb/Cargo.toml | 2 +- flake.lock | 8 ++++---- flake.nix | 5 ++++- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ce5ba829..e95de4b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3194,8 +3194,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.25.0+9.5.2" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=5383ca8173299066b516406e3a2cf945ead891cb#5383ca8173299066b516406e3a2cf945ead891cb" +version = "0.26.0+9.6.1" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=22f942609921ecf837399e1c2fe6fcb8dbb1095e#22f942609921ecf837399e1c2fe6fcb8dbb1095e" dependencies = [ "bindgen", "bzip2-sys", @@ -3211,8 +3211,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.29.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=5383ca8173299066b516406e3a2cf945ead891cb#5383ca8173299066b516406e3a2cf945ead891cb" +version = "0.30.0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=22f942609921ecf837399e1c2fe6fcb8dbb1095e#22f942609921ecf837399e1c2fe6fcb8dbb1095e" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index cc87e99d..e86f6e77 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "5383ca8173299066b516406e3a2cf945ead891cb" +rev = "22f942609921ecf837399e1c2fe6fcb8dbb1095e" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index 15ce8e43..bd82f81f 100644 --- a/flake.lock +++ b/flake.lock @@ -680,16 +680,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1724285323, - "narHash": "sha256-k60kreKQ0v+bQ16yBd2SfLYpuNjMw2qoRmZL/S3k6CU=", + "lastModified": 1725811807, + "narHash": "sha256-k6aubvLSJwW7CtIxBRSlM6Z8UiJDCdFUgDWPYLV47Qk=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "5a67ad7ce46328578ee5587fb0c23faa03d14e67", + "rev": "347d50e212b24d98b5ad9841404ff80c1bb873f0", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.5.2", + "ref": "v9.6.1", "repo": "rocksdb", "type": "github" } diff --git a/flake.nix b/flake.nix index 7f2bfeb9..74d15408 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.5.2"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.6.1"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; @@ -120,6 +120,9 @@ cargo-audit liburing + # Required by hardened-malloc.rs dep + binutils + # Needed for producing Debian packages cargo-deb From 02aee2f1745b0af1d38f27907d349696ade3f1c7 Mon Sep 17 00:00:00 2001 From: morguldir Date: Sun, 8 Sep 2024 20:51:34 +0200 Subject: [PATCH 0787/2091] ci: retry attic push 3 times, and continue on errors Signed-off-by: morguldir --- .github/workflows/ci.yml | 95 ++--------------------------- .github/workflows/documentation.yml | 38 +----------- bin/nix-build-and-cache | 9 +-- 3 files changed, 12 insertions(+), 130 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05dc7e6f..ffa3dbff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,24 +138,7 @@ jobs: - name: Cache CI dependencies run: | - # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here - set +e # Don't exit on errors - ATTEMPTS=3 - SUCCESS=false - while (( ATTEMPTS-- > 0 )) - do - bin/nix-build-and-cache ci - if [[ $? == 0 ]]; then - SUCCESS=true - break - else - sleep 3 - fi - done - - if [[ $SUCCESS == "false" ]]; then - exit 1 - fi + bin/nix-build-and-cache ci # use sccache for Rust - name: Run sccache-cache @@ -305,24 +288,7 @@ jobs: CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here - set +e # Don't exit on errors - ATTEMPTS=3 - SUCCESS=false - while (( ATTEMPTS-- > 0 )) - do - bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features - if [[ $? == 0 ]]; then - SUCCESS=true - break - else - sleep 3 - fi - done - - if [[ $SUCCESS == "false" ]]; then - exit 1 - fi + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features mkdir -v -p target/release/ mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ @@ -346,24 +312,7 @@ jobs: CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here - set +e # Don't exit on errors - ATTEMPTS=3 - SUCCESS=false - while (( ATTEMPTS-- > 0 )) - do - bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug - if [[ $? == 0 ]]; then - SUCCESS=true - break - else - sleep 3 - fi - done - - if [[ $SUCCESS == "false" ]]; then - exit 1 - fi + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles. # so we need to coerce cargo-deb into thinking this is a release binary @@ -426,47 +375,13 @@ jobs: - name: Build OCI image ${{ matrix.target }} run: | - # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here - set +e # Don't exit on errors - ATTEMPTS=3 - SUCCESS=false - while (( ATTEMPTS-- > 0 )) - do - bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features - if [[ $? == 0 ]]; then - SUCCESS=true - break - else - sleep 3 - fi - done - - if [[ $SUCCESS == "false" ]]; then - exit 1 - fi + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features cp -v -f result oci-image-${{ matrix.target }}.tar.gz - name: Build debug OCI image ${{ matrix.target }} run: | - # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here - set +e # Don't exit on errors - ATTEMPTS=3 - SUCCESS=false - while (( ATTEMPTS-- > 0 )) - do - bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug - if [[ $? == 0 ]]; then - SUCCESS=true - break - else - sleep 3 - fi - done - - if [[ $SUCCESS == "false" ]]; then - exit 1 - fi + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 68fe3557..58f1dd9c 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -110,24 +110,7 @@ jobs: - name: Cache CI dependencies run: | - # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here - set +e # Don't exit on errors - ATTEMPTS=3 - SUCCESS=false - while (( ATTEMPTS-- > 0 )) - do - bin/nix-build-and-cache ci - if [[ $? == 0 ]]; then - SUCCESS=true - break - else - sleep 3 - fi - done - - if [[ $SUCCESS == "false" ]]; then - exit 1 - fi + bin/nix-build-and-cache ci - name: Run lychee and markdownlint run: | @@ -136,24 +119,7 @@ jobs: - name: Build documentation (book) run: | - # attic nix binary cache server is very, very terribly flakey. nothing i can do to fix it other than retry multiple times here - set +e # Don't exit on errors - ATTEMPTS=3 - SUCCESS=false - while (( ATTEMPTS-- > 0 )) - do - bin/nix-build-and-cache just .#book - if [[ $? == 0 ]]; then - SUCCESS=true - break - else - sleep 3 - fi - done - - if [[ $SUCCESS == "false" ]]; then - exit 1 - fi + bin/nix-build-and-cache just .#book cp -r --dereference result public diff --git a/bin/nix-build-and-cache b/bin/nix-build-and-cache index 3e39e91f..f6b471a6 100755 --- a/bin/nix-build-and-cache +++ b/bin/nix-build-and-cache @@ -34,6 +34,9 @@ just() { ) done + withattic() { + nix shell --inputs-from "$toplevel" attic --command xargs attic push "$@" <<< "${cache[*]}" + } # Upload them to Attic (conduit store) # # Use `xargs` and a here-string because something would probably explode if @@ -41,8 +44,7 @@ just() { # store paths include a newline in them. ( IFS=$'\n' - nix shell --inputs-from "$toplevel" attic -c xargs \ - attic push conduit <<< "${cache[*]}" + withattic conduit || withattic conduit || withattic conduit || true ) # main "conduwuit" store @@ -59,8 +61,7 @@ just() { # store paths include a newline in them. ( IFS=$'\n' - nix shell --inputs-from "$toplevel" attic -c xargs \ - attic push conduwuit <<< "${cache[*]}" + withattic conduwuit || withattic conduwuit || withattic conduwuit || true # push to cachix if available if [ "$CACHIX_AUTH_TOKEN" ]; then From e888810e671816c7880348b3014fed11930dad60 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 8 Sep 2024 16:22:02 -0400 Subject: [PATCH 0788/2091] update complement results for TestFederationThumbnail Signed-off-by: strawberry --- tests/test_results/complement/test_results.jsonl | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 0f342cc4..ff695bb7 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -64,6 +64,7 @@ {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_several_times"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_has_'is_direct'_flag_in_prev_content_after_joining"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_see_room_metadata"} +{"Action":"pass","Test":"TestFederationThumbnail"} {"Action":"fail","Test":"TestGetMissingEventsGapFilling"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility"} From b44f7f54762b267160f28b288115843054f14f8b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 8 Sep 2024 19:21:53 -0400 Subject: [PATCH 0789/2091] remove MSC4175 timezone on account deactivation Signed-off-by: strawberry --- src/admin/user/commands.rs | 1 + src/api/client/account.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 2d5d1ead..c51545f7 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -165,6 +165,7 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> .collect(); update_displayname(self.services, user_id.clone(), None, all_joined_rooms.clone()).await?; update_avatar_url(self.services, user_id.clone(), None, None, all_joined_rooms).await?; + self.services.users.set_timezone(&user_id, None).await?; leave_all_rooms(self.services, &user_id).await; } diff --git a/src/api/client/account.rs b/src/api/client/account.rs index e20e2fac..01e0a501 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -558,6 +558,9 @@ pub(crate) async fn deactivate_route( // Remove devices and mark account as deactivated services.users.deactivate_account(sender_user)?; + // Remove timezone profile field + services.users.set_timezone(sender_user, None).await?; + // Remove profile pictures and display name let all_joined_rooms: Vec = services .rooms From 38cd88e1e83f88be69a6eb660a4668374139ed8d Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 8 Sep 2024 19:33:21 -0400 Subject: [PATCH 0790/2091] remove unnecessary cloning on account deactivation profile updates Signed-off-by: strawberry --- src/admin/user/commands.rs | 9 +++++---- src/api/client/account.rs | 5 +++-- src/api/client/membership.rs | 10 ++++++---- src/api/client/profile.rs | 26 +++++++++++++------------- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index c51545f7..7b303dc6 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -163,8 +163,8 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> .rooms_joined(&user_id) .filter_map(Result::ok) .collect(); - update_displayname(self.services, user_id.clone(), None, all_joined_rooms.clone()).await?; - update_avatar_url(self.services, user_id.clone(), None, None, all_joined_rooms).await?; + update_displayname(self.services, &user_id, None, all_joined_rooms.clone()).await?; + update_avatar_url(self.services, &user_id, None, None, all_joined_rooms).await?; self.services.users.set_timezone(&user_id, None).await?; leave_all_rooms(self.services, &user_id).await; } @@ -272,8 +272,9 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> .rooms_joined(&user_id) .filter_map(Result::ok) .collect(); - update_displayname(self.services, user_id.clone(), None, all_joined_rooms.clone()).await?; - update_avatar_url(self.services, user_id.clone(), None, None, all_joined_rooms).await?; + update_displayname(self.services, &user_id, None, all_joined_rooms.clone()).await?; + update_avatar_url(self.services, &user_id, None, None, all_joined_rooms).await?; + self.services.users.set_timezone(&user_id, None).await?; leave_all_rooms(self.services, &user_id).await; } }, diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 01e0a501..f1f330f6 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -568,8 +568,9 @@ pub(crate) async fn deactivate_route( .rooms_joined(sender_user) .filter_map(Result::ok) .collect(); - super::update_displayname(&services, sender_user.clone(), None, all_joined_rooms.clone()).await?; - super::update_avatar_url(&services, sender_user.clone(), None, None, all_joined_rooms).await?; + super::update_displayname(&services, sender_user, None, all_joined_rooms.clone()).await?; + super::update_avatar_url(&services, sender_user, None, None, all_joined_rooms).await?; + services.users.set_timezone(sender_user, None).await?; // Make the user leave all rooms before deactivation super::leave_all_rooms(&services, sender_user).await; diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 34e45a12..c7d37c45 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -93,8 +93,9 @@ async fn banned_room_check( .filter_map(Result::ok) .collect(); - update_displayname(services, user_id.into(), None, all_joined_rooms.clone()).await?; - update_avatar_url(services, user_id.into(), None, None, all_joined_rooms).await?; + update_displayname(services, user_id, None, all_joined_rooms.clone()).await?; + update_avatar_url(services, user_id, None, None, all_joined_rooms).await?; + services.users.set_timezone(user_id, None).await?; leave_all_rooms(services, user_id).await; } @@ -136,8 +137,9 @@ async fn banned_room_check( .filter_map(Result::ok) .collect(); - update_displayname(services, user_id.into(), None, all_joined_rooms.clone()).await?; - update_avatar_url(services, user_id.into(), None, None, all_joined_rooms).await?; + update_displayname(services, user_id, None, all_joined_rooms.clone()).await?; + update_avatar_url(services, user_id, None, None, all_joined_rooms).await?; + services.users.set_timezone(user_id, None).await?; leave_all_rooms(services, user_id).await; } diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 0e1e0f0a..789af643 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -10,7 +10,7 @@ use ruma::{ }, events::{room::member::RoomMemberEventContent, StateEventType, TimelineEventType}, presence::PresenceState, - OwnedMxcUri, OwnedRoomId, OwnedUserId, + OwnedMxcUri, OwnedRoomId, UserId, }; use serde_json::value::to_raw_value; use service::Services; @@ -38,7 +38,7 @@ pub(crate) async fn set_displayname_route( .filter_map(Result::ok) .collect(); - update_displayname(&services, body.user_id.clone(), body.displayname.clone(), all_joined_rooms).await?; + update_displayname(&services, &body.user_id, body.displayname.clone(), all_joined_rooms).await?; if services.globals.allow_local_presence() { // Presence update @@ -129,7 +129,7 @@ pub(crate) async fn set_avatar_url_route( update_avatar_url( &services, - body.user_id.clone(), + &body.user_id, body.avatar_url.clone(), body.blurhash.clone(), all_joined_rooms, @@ -271,9 +271,9 @@ pub(crate) async fn get_profile_route( } pub async fn update_displayname( - services: &Services, user_id: OwnedUserId, displayname: Option, all_joined_rooms: Vec, + services: &Services, user_id: &UserId, displayname: Option, all_joined_rooms: Vec, ) -> Result<()> { - let current_display_name = services.users.displayname(&user_id).unwrap_or_default(); + let current_display_name = services.users.displayname(user_id).unwrap_or_default(); if displayname == current_display_name { return Ok(()); @@ -281,7 +281,7 @@ pub async fn update_displayname( services .users - .set_displayname(&user_id, displayname.clone()) + .set_displayname(user_id, displayname.clone()) .await?; // Send a new join membership event into all joined rooms @@ -325,11 +325,11 @@ pub async fn update_displayname( } pub async fn update_avatar_url( - services: &Services, user_id: OwnedUserId, avatar_url: Option, blurhash: Option, + services: &Services, user_id: &UserId, avatar_url: Option, blurhash: Option, all_joined_rooms: Vec, ) -> Result<()> { - let current_avatar_url = services.users.avatar_url(&user_id).unwrap_or_default(); - let current_blurhash = services.users.blurhash(&user_id).unwrap_or_default(); + let current_avatar_url = services.users.avatar_url(user_id).unwrap_or_default(); + let current_blurhash = services.users.blurhash(user_id).unwrap_or_default(); if current_avatar_url == avatar_url && current_blurhash == blurhash { return Ok(()); @@ -337,11 +337,11 @@ pub async fn update_avatar_url( services .users - .set_avatar_url(&user_id, avatar_url.clone()) + .set_avatar_url(user_id, avatar_url.clone()) .await?; services .users - .set_blurhash(&user_id, blurhash.clone()) + .set_blurhash(user_id, blurhash.clone()) .await?; // Send a new join membership event into all joined rooms @@ -386,14 +386,14 @@ pub async fn update_avatar_url( } pub async fn update_all_rooms( - services: &Services, all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: OwnedUserId, + services: &Services, all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: &UserId, ) { for (pdu_builder, room_id) in all_joined_rooms { let state_lock = services.rooms.state.mutex.lock(room_id).await; if let Err(e) = services .rooms .timeline - .build_and_append_pdu(pdu_builder, &user_id, room_id, &state_lock) + .build_and_append_pdu(pdu_builder, user_id, room_id, &state_lock) .await { warn!(%user_id, %room_id, %e, "Failed to update/send new profile join membership update in room"); From 61347bee06af43d8d318e9bdee6e25435bf9d527 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 8 Sep 2024 19:53:33 -0400 Subject: [PATCH 0791/2091] advertise support for MSC4133 and MSC4175 Signed-off-by: strawberry --- src/api/client/capabilities.rs | 6 ++++++ src/api/client/unversioned.rs | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 3c278426..83e1dc7e 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -7,6 +7,7 @@ use ruma::{ }, RoomVersionId, }; +use serde_json::json; use crate::{Result, Ruma}; @@ -42,6 +43,11 @@ pub(crate) async fn get_capabilities_route( enabled: false, }; + // MSC4133 capability + capabilities + .set("uk.tcpip.msc4133.profile_fields", json!({"enabled": true})) + .expect("this is valid JSON we created"); + Ok(get_capabilities::v3::Response { capabilities, }) diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 76574949..d714fda5 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -55,6 +55,8 @@ pub(crate) async fn get_supported_versions_route( ("org.matrix.msc3575".to_owned(), true), /* sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/3575/files#r1588877046) */ ("org.matrix.msc3916.stable".to_owned(), true), /* authenticated media (https://github.com/matrix-org/matrix-spec-proposals/pull/3916) */ ("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */ + ("uk.tcpip.msc4133".to_owned(), true), /* Extending User Profile API with Key:Value Pairs (https://github.com/matrix-org/matrix-spec-proposals/pull/4133) */ + ("us.cloke.msc4175".to_owned(), true), /* Profile field for user time zone (https://github.com/matrix-org/matrix-spec-proposals/pull/4175) */ ]), }; From 55f71d3912ab38a508d7f4e1d2e8318e2745556e Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 8 Sep 2024 19:56:53 -0400 Subject: [PATCH 0792/2091] bump conduwuit to 0.4.7 Signed-off-by: strawberry --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e95de4b9..3dcbb678 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -598,7 +598,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" -version = "0.4.6" +version = "0.4.7" dependencies = [ "clap", "conduit_admin", @@ -627,7 +627,7 @@ dependencies = [ [[package]] name = "conduit_admin" -version = "0.4.6" +version = "0.4.7" dependencies = [ "clap", "conduit_api", @@ -647,7 +647,7 @@ dependencies = [ [[package]] name = "conduit_api" -version = "0.4.6" +version = "0.4.7" dependencies = [ "axum", "axum-client-ip", @@ -679,7 +679,7 @@ dependencies = [ [[package]] name = "conduit_core" -version = "0.4.6" +version = "0.4.7" dependencies = [ "argon2", "arrayvec", @@ -729,7 +729,7 @@ dependencies = [ [[package]] name = "conduit_database" -version = "0.4.6" +version = "0.4.7" dependencies = [ "conduit_core", "const-str", @@ -741,7 +741,7 @@ dependencies = [ [[package]] name = "conduit_macros" -version = "0.4.6" +version = "0.4.7" dependencies = [ "itertools 0.13.0", "proc-macro2", @@ -751,7 +751,7 @@ dependencies = [ [[package]] name = "conduit_router" -version = "0.4.6" +version = "0.4.7" dependencies = [ "axum", "axum-client-ip", @@ -783,7 +783,7 @@ dependencies = [ [[package]] name = "conduit_service" -version = "0.4.6" +version = "0.4.7" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 3419983a..9f274abf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" rust-version = "1.81.0" -version = "0.4.6" +version = "0.4.7" [workspace.metadata.crane] name = "conduit" From 9de780b56c7e7e9221677e73b43d65c0395feb32 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 9 Sep 2024 21:10:56 -0400 Subject: [PATCH 0793/2091] remove unnecessary displayname requirement on making user admin Signed-off-by: strawberry --- src/admin/user/commands.rs | 10 +--------- src/api/client/account.rs | 5 +---- src/service/admin/grant.rs | 4 ++-- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 7b303dc6..b7df11f4 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -373,20 +373,12 @@ pub(super) async fn force_leave_room( #[admin_command] pub(super) async fn make_user_admin(&self, user_id: String) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; - let displayname = self - .services - .users - .displayname(&user_id)? - .unwrap_or_else(|| user_id.to_string()); assert!( self.services.globals.user_is_local(&user_id), "Parsed user_id must be a local user" ); - self.services - .admin - .make_user_admin(&user_id, displayname) - .await?; + self.services.admin.make_user_admin(&user_id).await?; Ok(RoomMessageEventContent::notice_markdown(format!( "{user_id} has been granted admin privileges.", diff --git a/src/api/client/account.rs b/src/api/client/account.rs index f1f330f6..fb010007 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -347,10 +347,7 @@ pub(crate) async fn register_route( if !is_guest { if let Some(admin_room) = services.admin.get_admin_room()? { if services.rooms.state_cache.room_joined_count(&admin_room)? == Some(1) { - services - .admin - .make_user_admin(&user_id, displayname) - .await?; + services.admin.make_user_admin(&user_id).await?; warn!("Granting {user_id} admin privileges as the first user"); } diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index e0d58600..3fe2441f 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -21,7 +21,7 @@ impl super::Service { /// Invite the user to the conduit admin room. /// /// In conduit, this is equivalent to granting admin privileges. - pub async fn make_user_admin(&self, user_id: &UserId, displayname: String) -> Result<()> { + pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { let Some(room_id) = self.get_admin_room()? else { return Ok(()); }; @@ -65,7 +65,7 @@ impl super::Service { event_type: TimelineEventType::RoomMember, content: to_raw_value(&RoomMemberEventContent { membership: MembershipState::Join, - displayname: Some(displayname), + displayname: None, avatar_url: None, is_direct: None, third_party_invite: None, From 38552b36e95326391168f2d70e9724825e1ef65c Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 9 Sep 2024 21:13:37 -0400 Subject: [PATCH 0794/2091] make the first user admin if created from CLI / --execute Signed-off-by: strawberry --- src/admin/user/commands.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index b7df11f4..cb571d77 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -128,6 +128,22 @@ pub(super) async fn create_user(&self, username: String, password: Option Date: Mon, 9 Sep 2024 21:41:57 -0400 Subject: [PATCH 0795/2091] remove a few unnecessary muts Signed-off-by: strawberry --- src/api/client/room.rs | 3 +-- src/service/admin/create.rs | 3 +-- src/service/admin/grant.rs | 4 +--- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 96c43758..442fe266 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -230,8 +230,7 @@ pub(crate) async fn create_room_route( _ => RoomPreset::PrivateChat, // Room visibility should not be custom }); - let mut users = BTreeMap::new(); - users.insert(sender_user.clone(), int!(100)); + let mut users = BTreeMap::from_iter([(sender_user.clone(), int!(100))]); if preset == RoomPreset::TrustedPrivateChat { for invite_ in &body.invite { diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index ef84d803..4e2b831c 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -101,8 +101,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .await?; // 3. Power levels - let mut users = BTreeMap::new(); - users.insert(server_user.clone(), 100.into()); + let users = BTreeMap::from_iter([(server_user.clone(), 100.into())]); services .rooms diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 3fe2441f..b4589ebc 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -86,9 +86,7 @@ impl super::Service { .await?; // Set power level - let mut users = BTreeMap::new(); - users.insert(server_user.clone(), 100.into()); - users.insert(user_id.to_owned(), 100.into()); + let users = BTreeMap::from_iter([(server_user.clone(), 100.into()), (user_id.to_owned(), 100.into())]); self.services .timeline From a41e63b40e83a82991eac4f48cbf2e3fea711c19 Mon Sep 17 00:00:00 2001 From: morguldir Date: Thu, 15 Aug 2024 18:39:23 +0200 Subject: [PATCH 0796/2091] Add back allow_check_for_updates Signed-off-by: morguldir --- src/service/updates/mod.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index f89471cd..3c69b243 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -1,13 +1,13 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{err, info, utils, warn, Error, Result}; +use conduit::{debug, err, info, utils, warn, Error, Result}; use database::Map; use ruma::events::room::message::RoomMessageEventContent; use serde::Deserialize; use tokio::{sync::Notify, time::interval}; -use crate::{admin, client, Dep}; +use crate::{admin, client, globals, Dep}; pub struct Service { services: Services, @@ -19,6 +19,7 @@ pub struct Service { struct Services { admin: Dep, client: Dep, + globals: Dep, } #[derive(Deserialize)] @@ -42,6 +43,7 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { services: Services { + globals: args.depend::("globals"), admin: args.depend::("admin"), client: args.depend::("client"), }, @@ -52,6 +54,10 @@ impl crate::Service for Service { } async fn worker(self: Arc) -> Result<()> { + if !self.services.globals.allow_check_for_updates() { + debug!("Disabling update check"); + return Ok(()); + } let mut i = interval(self.interval); loop { tokio::select! { From 80b72637e2f91223c77fa110465f0c15f2bf2603 Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 13 Sep 2024 05:09:10 +0200 Subject: [PATCH 0797/2091] ci: enable cachix after restoring the cache Signed-off-by: morguldir --- .github/workflows/ci.yml | 24 ++++++++++++------------ .github/workflows/documentation.yml | 12 ++++++------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffa3dbff..3e024c48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -87,12 +87,6 @@ jobs: - uses: nixbuild/nix-quick-install-action@v28 - - name: Enable Cachix binary cache - run: | - nix profile install nixpkgs#cachix - cachix use crane - cachix use nix-community - - name: Restore and cache Nix store uses: nix-community/cache-nix-action@v5.1.0 with: @@ -114,6 +108,12 @@ jobs: # always save the cache save-always: true + - name: Enable Cachix binary cache + run: | + nix profile install nixpkgs#cachix + cachix use crane + cachix use nix-community + - name: Apply Nix binary cache configuration run: | sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null < /dev/null < /dev/null < Date: Sat, 14 Sep 2024 11:16:19 -0400 Subject: [PATCH 0798/2091] implement generic K-V support for MSC4133, GET/PUT/DELETE no PATCH still yet Signed-off-by: strawberry --- Cargo.lock | 26 ++--- Cargo.toml | 3 +- src/api/client/membership.rs | 6 +- src/api/client/profile.rs | 12 ++ src/api/client/unstable.rs | 217 ++++++++++++++++++++++++++++++++++- src/api/router.rs | 3 + src/api/server/query.rs | 19 ++- src/service/users/data.rs | 54 +++++++++ src/service/users/mod.rs | 20 ++++ 9 files changed, 340 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3dcbb678..67ba00f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "assign", "js_int", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "js_int", "ruma-common", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "as_variant", "assign", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "as_variant", "base64 0.22.1", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "as_variant", "indexmap 2.5.0", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "bytes", "http", @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "js_int", "thiserror", @@ -3113,7 +3113,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "js_int", "ruma-common", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "cfg-if", "once_cell", @@ -3139,7 +3139,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "js_int", "ruma-common", @@ -3151,7 +3151,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "headers", "http", @@ -3164,7 +3164,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3180,7 +3180,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=11155e576a1382783c0bcf5ad4458708777ec36e#11155e576a1382783c0bcf5ad4458708777ec36e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "itertools 0.12.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 9f274abf..86c2bcc3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -314,7 +314,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "11155e576a1382783c0bcf5ad4458708777ec36e" +rev = "b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" features = [ "compat", "rand", @@ -341,6 +341,7 @@ features = [ "unstable-msc3575", "unstable-msc4121", "unstable-msc4125", + "unstable-msc4186", "unstable-extensible-events", ] diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index c7d37c45..234f3571 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -236,7 +236,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( Ok(room_id) => { banned_room_check(&services, sender_user, Some(&room_id), room_id.server_name(), client).await?; - let mut servers = body.server_name.clone(); + let mut servers = body.via.clone(); servers.extend( services .rooms @@ -269,13 +269,13 @@ pub(crate) async fn join_room_by_id_or_alias_route( let response = services .rooms .alias - .resolve_alias(&room_alias, Some(&body.server_name.clone())) + .resolve_alias(&room_alias, Some(&body.via.clone())) .await?; let (room_id, mut pre_servers) = response; banned_room_check(&services, sender_user, Some(&room_id), Some(room_alias.server_name()), client).await?; - let mut servers = body.server_name; + let mut servers = body.via; if let Some(pre_servers) = &mut pre_servers { servers.append(pre_servers); } diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 789af643..bf47a3f8 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -247,11 +247,18 @@ pub(crate) async fn get_profile_route( .set_timezone(&body.user_id, response.tz.clone()) .await?; + for (profile_key, profile_key_value) in &response.custom_profile_fields { + services + .users + .set_profile_key(&body.user_id, profile_key, Some(profile_key_value.clone()))?; + } + return Ok(get_profile::v3::Response { displayname: response.displayname, avatar_url: response.avatar_url, blurhash: response.blurhash, tz: response.tz, + custom_profile_fields: response.custom_profile_fields, }); } } @@ -267,6 +274,11 @@ pub(crate) async fn get_profile_route( blurhash: services.users.blurhash(&body.user_id)?, displayname: services.users.displayname(&body.user_id)?, tz: services.users.timezone(&body.user_id)?, + custom_profile_fields: services + .users + .all_profile_keys(&body.user_id) + .filter_map(Result::ok) + .collect(), }) } diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index d672ad46..ab4703fd 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{warn, Err}; @@ -6,7 +8,10 @@ use ruma::{ client::{ error::ErrorKind, membership::mutual_rooms, - profile::{delete_timezone_key, get_timezone_key, set_timezone_key}, + profile::{ + delete_profile_key, delete_timezone_key, get_profile_key, get_timezone_key, set_profile_key, + set_timezone_key, + }, room::get_summary, }, federation, @@ -16,6 +21,7 @@ use ruma::{ OwnedRoomId, }; +use super::{update_avatar_url, update_displayname}; use crate::{Error, Result, Ruma, RumaResponse}; /// # `GET /_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms` @@ -226,6 +232,138 @@ pub(crate) async fn set_timezone_key_route( Ok(set_timezone_key::unstable::Response {}) } +/// # `PUT /_matrix/client/unstable/uk.tcpip.msc4133/profile/{user_id}/{field}` +/// +/// Updates the profile key-value field of a user, as per MSC4133. +/// +/// This also handles the avatar_url and displayname being updated. +pub(crate) async fn set_profile_key_route( + State(services): State, body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if *sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot update the profile of another user"))); + } + + if body.kv_pair.is_empty() { + return Err!(Request(BadJson( + "The key-value pair JSON body is empty. Use DELETE to delete a key" + ))); + } + + if body.kv_pair.len() > 1 { + // TODO: support PATCH or "recursively" adding keys in some sort + return Err!(Request(BadJson("This endpoint can only take one key-value pair at a time"))); + } + + let Some(profile_key_value) = body.kv_pair.get(&body.key) else { + return Err!(Request(BadJson( + "The key does not match the URL field key, or JSON body is empty (use DELETE)" + ))); + }; + + if body + .kv_pair + .keys() + .any(|key| key.starts_with("u.") && !profile_key_value.is_string()) + { + return Err!(Request(BadJson("u.* profile key fields must be strings"))); + } + + if body.kv_pair.keys().any(|key| key.len() > 128) { + return Err!(Request(BadJson("Key names cannot be longer than 128 bytes"))); + } + + if body.key == "displayname" { + let all_joined_rooms: Vec = services + .rooms + .state_cache + .rooms_joined(&body.user_id) + .filter_map(Result::ok) + .collect(); + + update_displayname(&services, &body.user_id, Some(profile_key_value.to_string()), all_joined_rooms).await?; + } else if body.key == "avatar_url" { + let mxc = ruma::OwnedMxcUri::from(profile_key_value.to_string()); + + let all_joined_rooms: Vec = services + .rooms + .state_cache + .rooms_joined(&body.user_id) + .filter_map(Result::ok) + .collect(); + + update_avatar_url(&services, &body.user_id, Some(mxc), None, all_joined_rooms).await?; + } else { + services + .users + .set_profile_key(&body.user_id, &body.key, Some(profile_key_value.clone()))?; + } + + if services.globals.allow_local_presence() { + // Presence update + services + .presence + .ping_presence(&body.user_id, &PresenceState::Online)?; + } + + Ok(set_profile_key::unstable::Response {}) +} + +/// # `DELETE /_matrix/client/unstable/uk.tcpip.msc4133/profile/{user_id}/{field}` +/// +/// Deletes the profile key-value field of a user, as per MSC4133. +/// +/// This also handles the avatar_url and displayname being updated. +pub(crate) async fn delete_profile_key_route( + State(services): State, body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if *sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot update the profile of another user"))); + } + + if body.kv_pair.len() > 1 { + // TODO: support PATCH or "recursively" adding keys in some sort + return Err!(Request(BadJson("This endpoint can only take one key-value pair at a time"))); + } + + if body.key == "displayname" { + let all_joined_rooms: Vec = services + .rooms + .state_cache + .rooms_joined(&body.user_id) + .filter_map(Result::ok) + .collect(); + + update_displayname(&services, &body.user_id, None, all_joined_rooms).await?; + } else if body.key == "avatar_url" { + let all_joined_rooms: Vec = services + .rooms + .state_cache + .rooms_joined(&body.user_id) + .filter_map(Result::ok) + .collect(); + + update_avatar_url(&services, &body.user_id, None, None, all_joined_rooms).await?; + } else { + services + .users + .set_profile_key(&body.user_id, &body.key, None)?; + } + + if services.globals.allow_local_presence() { + // Presence update + services + .presence + .ping_presence(&body.user_id, &PresenceState::Online)?; + } + + Ok(delete_profile_key::unstable::Response {}) +} + /// # `GET /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz` /// /// Returns the `timezone` of the user as per MSC4133 and MSC4175. @@ -285,3 +423,80 @@ pub(crate) async fn get_timezone_key_route( tz: services.users.timezone(&body.user_id)?, }) } + +/// # `GET /_matrix/client/unstable/uk.tcpip.msc4133/profile/{userId}/{field}}` +/// +/// Gets the profile key-value field of a user, as per MSC4133. +/// +/// - If user is on another server and we do not have a local copy already fetch +/// `timezone` over federation +pub(crate) async fn get_profile_key_route( + State(services): State, body: Ruma, +) -> Result { + let mut profile_key_value: BTreeMap = BTreeMap::new(); + + if !services.globals.user_is_local(&body.user_id) { + // Create and update our local copy of the user + if let Ok(response) = services + .sending + .send_federation_request( + body.user_id.server_name(), + federation::query::get_profile_information::v1::Request { + user_id: body.user_id.clone(), + field: None, // we want the full user's profile to update locally as well + }, + ) + .await + { + if !services.users.exists(&body.user_id)? { + services.users.create(&body.user_id, None)?; + } + + services + .users + .set_displayname(&body.user_id, response.displayname.clone()) + .await?; + services + .users + .set_avatar_url(&body.user_id, response.avatar_url.clone()) + .await?; + services + .users + .set_blurhash(&body.user_id, response.blurhash.clone()) + .await?; + services + .users + .set_timezone(&body.user_id, response.tz.clone()) + .await?; + + if let Some(value) = response.custom_profile_fields.get(&body.key) { + profile_key_value.insert(body.key.clone(), value.clone()); + services + .users + .set_profile_key(&body.user_id, &body.key, Some(value.clone()))?; + } else { + return Err!(Request(NotFound("The requested profile key does not exist."))); + } + + return Ok(get_profile_key::unstable::Response { + value: profile_key_value, + }); + } + } + + if !services.users.exists(&body.user_id)? { + // Return 404 if this user doesn't exist and we couldn't fetch it over + // federation + return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); + } + + if let Some(value) = services.users.profile_key(&body.user_id, &body.key)? { + profile_key_value.insert(body.key.clone(), value); + } else { + return Err!(Request(NotFound("The requested profile key does not exist."))); + } + + Ok(get_profile_key::unstable::Response { + value: profile_key_value, + }) +} diff --git a/src/api/router.rs b/src/api/router.rs index 77e9a11a..4264e01d 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -23,6 +23,9 @@ pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; let mut router = router .ruma_route(client::get_timezone_key_route) + .ruma_route(client::get_profile_key_route) + .ruma_route(client::set_profile_key_route) + .ruma_route(client::delete_profile_key_route) .ruma_route(client::set_timezone_key_route) .ruma_route(client::delete_timezone_key_route) .ruma_route(client::appservice_ping) diff --git a/src/api/server/query.rs b/src/api/server/query.rs index 45830366..c2b78bde 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -1,3 +1,5 @@ +use std::collections::BTreeMap; + use axum::extract::State; use conduit::{Error, Result}; use get_profile_information::v1::ProfileField; @@ -76,6 +78,7 @@ pub(crate) async fn get_profile_information_route( let mut avatar_url = None; let mut blurhash = None; let mut tz = None; + let mut custom_profile_fields = BTreeMap::new(); match &body.field { Some(ProfileField::DisplayName) => { @@ -85,13 +88,24 @@ pub(crate) async fn get_profile_information_route( avatar_url = services.users.avatar_url(&body.user_id)?; blurhash = services.users.blurhash(&body.user_id)?; }, - // TODO: what to do with custom - Some(_) => {}, + Some(custom_field) => { + if let Some(value) = services + .users + .profile_key(&body.user_id, custom_field.as_str())? + { + custom_profile_fields.insert(custom_field.to_string(), value); + } + }, None => { displayname = services.users.displayname(&body.user_id)?; avatar_url = services.users.avatar_url(&body.user_id)?; blurhash = services.users.blurhash(&body.user_id)?; tz = services.users.timezone(&body.user_id)?; + custom_profile_fields = services + .users + .all_profile_keys(&body.user_id) + .filter_map(Result::ok) + .collect(); }, } @@ -100,5 +114,6 @@ pub(crate) async fn get_profile_information_route( avatar_url, blurhash, tz, + custom_profile_fields, }) } diff --git a/src/service/users/data.rs b/src/service/users/data.rs index 70785d68..70ff12e3 100644 --- a/src/service/users/data.rs +++ b/src/service/users/data.rs @@ -233,6 +233,60 @@ impl Data { .transpose() } + /// Gets a specific user profile key + pub(super) fn profile_key(&self, user_id: &UserId, profile_key: &str) -> Result> { + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(profile_key.as_bytes()); + + self.useridprofilekey_value + .get(&key)? + .map_or(Ok(None), |bytes| Ok(Some(serde_json::from_slice(&bytes).unwrap()))) + } + + /// Gets all the user's profile keys and values in an iterator + pub(super) fn all_profile_keys<'a>( + &'a self, user_id: &UserId, + ) -> Box> + 'a + Send> { + let prefix = user_id.as_bytes().to_vec(); + + Box::new( + self.useridprofilekey_value + .scan_prefix(prefix) + .map(|(key, value)| { + let profile_key_name = utils::string_from_bytes( + key.rsplit(|&b| b == 0xFF) + .next() + .ok_or_else(|| err!(Database("Profile key in db is invalid")))?, + ) + .map_err(|e| err!(Database("Profile key in db is invalid. {e}")))?; + + let profile_key_value = serde_json::from_slice(&value) + .map_err(|e| err!(Database("Profile key in db is invalid. {e}")))?; + + Ok((profile_key_name, profile_key_value)) + }), + ) + } + + /// Sets a new profile key value, removes the key if value is None + pub(super) fn set_profile_key( + &self, user_id: &UserId, profile_key: &str, profile_key_value: Option, + ) -> Result<()> { + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(profile_key.as_bytes()); + + // TODO: insert to the stable MSC4175 key when it's stable + if let Some(value) = profile_key_value { + let value = serde_json::to_vec(&value).unwrap(); + + self.useridprofilekey_value.insert(&key, &value) + } else { + self.useridprofilekey_value.remove(&key) + } + } + /// Get the timezone of a user. pub(super) fn timezone(&self, user_id: &UserId) -> Result> { // first check the unstable prefix diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 544614d4..80897b5f 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -329,6 +329,26 @@ impl Service { pub fn timezone(&self, user_id: &UserId) -> Result> { self.db.timezone(user_id) } + /// Gets a specific user profile key + pub fn profile_key(&self, user_id: &UserId, profile_key: &str) -> Result> { + self.db.profile_key(user_id, profile_key) + } + + /// Gets all the user's profile keys and values in an iterator + pub fn all_profile_keys<'a>( + &'a self, user_id: &UserId, + ) -> Box> + 'a + Send> { + self.db.all_profile_keys(user_id) + } + + /// Sets a new profile key value, removes the key if value is None + pub fn set_profile_key( + &self, user_id: &UserId, profile_key: &str, profile_key_value: Option, + ) -> Result<()> { + self.db + .set_profile_key(user_id, profile_key, profile_key_value) + } + /// Sets a new tz or removes it if tz is None. pub async fn set_timezone(&self, user_id: &UserId, tz: Option) -> Result<()> { self.db.set_timezone(user_id, tz) From e146c752790efe34f05d2803a753718a70a3fc3b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 14 Sep 2024 12:27:22 -0400 Subject: [PATCH 0799/2091] dont include appservices in room guest access enforcement check Signed-off-by: strawberry --- src/admin/user/commands.rs | 3 ++- src/api/client/account.rs | 1 + src/api/client/membership.rs | 9 ++++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index cb571d77..551ce20b 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -111,6 +111,7 @@ pub(super) async fn create_user(&self, username: String, password: Option, ) -> Result { let sender_user = body.sender_user.as_deref().expect("user is authenticated"); + let appservice_info = &body.appservice_info; let body = body.body; let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { @@ -312,6 +314,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( body.reason.clone(), &servers, body.third_party_signed.as_ref(), + appservice_info, ) .await?; @@ -662,11 +665,11 @@ pub(crate) async fn joined_members_route( pub async fn join_room_by_id_helper( services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - third_party_signed: Option<&ThirdPartySigned>, + third_party_signed: Option<&ThirdPartySigned>, appservice_info: &Option, ) -> Result { let state_lock = services.rooms.state.mutex.lock(room_id).await; - let user_is_guest = services.users.is_deactivated(sender_user).unwrap_or(false); + let user_is_guest = services.users.is_deactivated(sender_user).unwrap_or(false) && appservice_info.is_none(); if matches!(services.rooms.state_accessor.guest_can_join(room_id), Ok(false)) && user_is_guest { return Err!(Request(Forbidden("Guests are not allowed to join this room"))); From a65dd6dfb3e3d109bd90fe8b93cd2a1fbac44a16 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 14 Sep 2024 12:37:29 -0400 Subject: [PATCH 0800/2091] dont allow guests to publish to room directories Signed-off-by: strawberry --- src/api/client/directory.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index ca4c1814..1a3e717d 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,6 +1,6 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{err, info, warn, Error, Result}; +use conduit::{err, info, warn, Err, Error, Result}; use ruma::{ api::{ client::{ @@ -124,6 +124,10 @@ pub(crate) async fn set_room_visibility_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } + if services.users.is_deactivated(sender_user).unwrap_or(false) && body.appservice_info.is_none() { + return Err!(Request(Forbidden("Guests cannot publish to room directories"))); + } + if !user_can_publish_room(&services, sender_user, &body.room_id)? { return Err(Error::BadRequest( ErrorKind::forbidden(), From 895b178720a4a40c9adc36d06a24d9f682ec7334 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Sep 2024 11:36:47 -0400 Subject: [PATCH 0801/2091] add admin command to force demote a local user from a room Signed-off-by: strawberry --- src/admin/user/commands.rs | 76 +++++++++++++++++++++++++++++++++++++- src/admin/user/mod.rs | 7 ++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 551ce20b..b51f32ef 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -4,9 +4,13 @@ use api::client::{join_room_by_id_helper, leave_all_rooms, leave_room, update_av use conduit::{error, info, utils, warn, PduBuilder, Result}; use ruma::{ events::{ - room::{message::RoomMessageEventContent, redaction::RoomRedactionEventContent}, + room::{ + message::RoomMessageEventContent, + power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, + redaction::RoomRedactionEventContent, + }, tag::{TagEvent, TagEventContent, TagInfo}, - RoomAccountDataEventType, TimelineEventType, + RoomAccountDataEventType, StateEventType, TimelineEventType, }, EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, }; @@ -387,6 +391,74 @@ pub(super) async fn force_leave_room( ))) } +#[admin_command] +pub(super) async fn force_demote( + &self, user_id: String, room_id: OwnedRoomOrAliasId, +) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + let room_id = self.services.rooms.alias.resolve(&room_id).await?; + + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + + let state_lock = self.services.rooms.state.mutex.lock(&room_id).await; + + let room_power_levels = self + .services + .rooms + .state_accessor + .room_state_get(&room_id, &StateEventType::RoomPowerLevels, "")? + .as_ref() + .and_then(|event| serde_json::from_str(event.content.get()).ok()?) + .and_then(|content: RoomPowerLevelsEventContent| content.into()); + + let user_can_demote_self = room_power_levels + .as_ref() + .is_some_and(|power_levels_content| { + RoomPowerLevels::from(power_levels_content.clone()).user_can_change_user_power_level(&user_id, &user_id) + }) || self + .services + .rooms + .state_accessor + .room_state_get(&room_id, &StateEventType::RoomCreate, "")? + .as_ref() + .is_some_and(|event| event.sender == user_id); + + if !user_can_demote_self { + return Ok(RoomMessageEventContent::notice_markdown( + "User is not allowed to modify their own power levels in the room.", + )); + } + + let mut power_levels_content = room_power_levels.unwrap_or_default(); + power_levels_content.users.remove(&user_id); + + let event_id = self + .services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomPowerLevels, + content: to_raw_value(&power_levels_content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + timestamp: None, + }, + &user_id, + &room_id, + &state_lock, + ) + .await?; + + Ok(RoomMessageEventContent::notice_markdown(format!( + "User {user_id} demoted themselves to the room default power level in {room_id} - {event_id}" + ))) +} + #[admin_command] pub(super) async fn make_user_admin(&self, user_id: String) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index d8d740bf..e7bb5c73 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -79,6 +79,13 @@ pub(super) enum UserCommand { room_id: OwnedRoomOrAliasId, }, + /// - Forces the specified user to drop their power levels to the room + /// default, if their permissions allow and the auth check permits + ForceDemote { + user_id: String, + room_id: OwnedRoomOrAliasId, + }, + /// - Grant server-admin privileges to a user. MakeUserAdmin { user_id: String, From 17fd34eb12e3dbbe3922787ad59c8cbaeddc8534 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Sep 2024 12:08:07 -0400 Subject: [PATCH 0802/2091] dedupe some account deactivation steps, remove all profile fields on deactivation Signed-off-by: strawberry --- src/admin/user/commands.rs | 13 ++++------- src/api/client/account.rs | 43 +++++++++++++++++++++++++++--------- src/api/client/membership.rs | 23 +++---------------- src/api/client/mod.rs | 1 + 4 files changed, 40 insertions(+), 40 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index b51f32ef..20691f1a 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, fmt::Write as _}; -use api::client::{join_room_by_id_helper, leave_all_rooms, leave_room, update_avatar_url, update_displayname}; +use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room}; use conduit::{error, info, utils, warn, PduBuilder, Result}; use ruma::{ events::{ @@ -184,10 +184,8 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> .rooms_joined(&user_id) .filter_map(Result::ok) .collect(); - update_displayname(self.services, &user_id, None, all_joined_rooms.clone()).await?; - update_avatar_url(self.services, &user_id, None, None, all_joined_rooms).await?; - self.services.users.set_timezone(&user_id, None).await?; - leave_all_rooms(self.services, &user_id).await; + + full_user_deactivate(self.services, &user_id, all_joined_rooms).await?; } Ok(RoomMessageEventContent::text_plain(format!( @@ -293,10 +291,7 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> .rooms_joined(&user_id) .filter_map(Result::ok) .collect(); - update_displayname(self.services, &user_id, None, all_joined_rooms.clone()).await?; - update_avatar_url(self.services, &user_id, None, None, all_joined_rooms).await?; - self.services.users.set_timezone(&user_id, None).await?; - leave_all_rooms(self.services, &user_id).await; + full_user_deactivate(self.services, &user_id, all_joined_rooms).await?; } }, Err(e) => { diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 605ab20b..27e6523b 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -18,6 +18,7 @@ use ruma::{ events::{room::message::RoomMessageEventContent, GlobalAccountDataEventType}, push, OwnedRoomId, UserId, }; +use service::Services; use super::{join_room_by_id_helper, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; use crate::Ruma; @@ -553,12 +554,6 @@ pub(crate) async fn deactivate_route( return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } - // Remove devices and mark account as deactivated - services.users.deactivate_account(sender_user)?; - - // Remove timezone profile field - services.users.set_timezone(sender_user, None).await?; - // Remove profile pictures and display name let all_joined_rooms: Vec = services .rooms @@ -566,12 +561,8 @@ pub(crate) async fn deactivate_route( .rooms_joined(sender_user) .filter_map(Result::ok) .collect(); - super::update_displayname(&services, sender_user, None, all_joined_rooms.clone()).await?; - super::update_avatar_url(&services, sender_user, None, None, all_joined_rooms).await?; - services.users.set_timezone(sender_user, None).await?; - // Make the user leave all rooms before deactivation - super::leave_all_rooms(&services, sender_user).await; + full_user_deactivate(&services, sender_user, all_joined_rooms).await?; info!("User {sender_user} deactivated their account."); services @@ -649,3 +640,33 @@ pub(crate) async fn check_registration_token_validity( valid: reg_token == body.token, }) } + +/// Runs through all the deactivation steps: +/// +/// - Mark as deactivated +/// - Removing display name +/// - Removing avatar URL and blurhash +/// - Removing all profile data +/// - Leaving all rooms +pub async fn full_user_deactivate( + services: &Services, user_id: &UserId, all_joined_rooms: Vec, +) -> Result<()> { + services.users.deactivate_account(user_id)?; + + super::update_displayname(services, user_id, None, all_joined_rooms.clone()).await?; + super::update_avatar_url(services, user_id, None, None, all_joined_rooms).await?; + super::leave_all_rooms(services, user_id).await; + + let all_profile_keys = services + .users + .all_profile_keys(user_id) + .filter_map(Result::ok); + + for (profile_key, _profile_value) in all_profile_keys { + services + .users + .set_profile_key(user_id, &profile_key, None)?; + } + + Ok(()) +} diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 514403e7..b3b3c80e 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -43,10 +43,7 @@ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::{appservice::RegistrationInfo, rooms::state::RoomMutexGuard, Services}; use tokio::sync::RwLock; -use crate::{ - client::{update_avatar_url, update_displayname}, - Ruma, -}; +use crate::{client::full_user_deactivate, Ruma}; /// Checks if the room is banned in any way possible and the sender user is not /// an admin. @@ -82,10 +79,6 @@ async fn banned_room_check( ))) .await; - if let Err(e) = services.users.deactivate_account(user_id) { - warn!(%user_id, %e, "Failed to deactivate account"); - } - let all_joined_rooms: Vec = services .rooms .state_cache @@ -93,10 +86,7 @@ async fn banned_room_check( .filter_map(Result::ok) .collect(); - update_displayname(services, user_id, None, all_joined_rooms.clone()).await?; - update_avatar_url(services, user_id, None, None, all_joined_rooms).await?; - services.users.set_timezone(user_id, None).await?; - leave_all_rooms(services, user_id).await; + full_user_deactivate(services, user_id, all_joined_rooms).await?; } return Err(Error::BadRequest( @@ -126,10 +116,6 @@ async fn banned_room_check( ))) .await; - if let Err(e) = services.users.deactivate_account(user_id) { - warn!(%user_id, %e, "Failed to deactivate account"); - } - let all_joined_rooms: Vec = services .rooms .state_cache @@ -137,10 +123,7 @@ async fn banned_room_check( .filter_map(Result::ok) .collect(); - update_displayname(services, user_id, None, all_joined_rooms.clone()).await?; - update_avatar_url(services, user_id, None, None, all_joined_rooms).await?; - services.users.set_timezone(user_id, None).await?; - leave_all_rooms(services, user_id).await; + full_user_deactivate(services, user_id, all_joined_rooms).await?; } return Err(Error::BadRequest( diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index 5dcdc51a..4b7b64b9 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -37,6 +37,7 @@ pub(super) mod unversioned; pub(super) mod user_directory; pub(super) mod voip; +pub use account::full_user_deactivate; pub(super) use account::*; pub(super) use alias::*; pub(super) use appservice::*; From e9e5fe2176e8be611db91bbd38e3f8d4e92a3e3b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Sep 2024 12:22:46 -0400 Subject: [PATCH 0803/2091] implement MSC4165, removing own power levels on deactivation Signed-off-by: strawberry --- src/api/client/account.rs | 76 ++++++++++++++++++++++++++++++++++----- 1 file changed, 68 insertions(+), 8 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 27e6523b..b87eb400 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -2,7 +2,7 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{debug_info, error, info, utils, warn, Error, Result}; +use conduit::{debug_info, error, info, utils, warn, Error, PduBuilder, Result}; use register::RegistrationKind; use ruma::{ api::client::{ @@ -15,9 +15,16 @@ use ruma::{ error::ErrorKind, uiaa::{AuthFlow, AuthType, UiaaInfo}, }, - events::{room::message::RoomMessageEventContent, GlobalAccountDataEventType}, + events::{ + room::{ + message::RoomMessageEventContent, + power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, + }, + GlobalAccountDataEventType, StateEventType, TimelineEventType, + }, push, OwnedRoomId, UserId, }; +use serde_json::value::to_raw_value; use service::Services; use super::{join_room_by_id_helper, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; @@ -647,15 +654,14 @@ pub(crate) async fn check_registration_token_validity( /// - Removing display name /// - Removing avatar URL and blurhash /// - Removing all profile data -/// - Leaving all rooms +/// - Leaving all rooms (and forgets all of them) pub async fn full_user_deactivate( services: &Services, user_id: &UserId, all_joined_rooms: Vec, ) -> Result<()> { services.users.deactivate_account(user_id)?; super::update_displayname(services, user_id, None, all_joined_rooms.clone()).await?; - super::update_avatar_url(services, user_id, None, None, all_joined_rooms).await?; - super::leave_all_rooms(services, user_id).await; + super::update_avatar_url(services, user_id, None, None, all_joined_rooms.clone()).await?; let all_profile_keys = services .users @@ -663,10 +669,64 @@ pub async fn full_user_deactivate( .filter_map(Result::ok); for (profile_key, _profile_value) in all_profile_keys { - services - .users - .set_profile_key(user_id, &profile_key, None)?; + if let Err(e) = services.users.set_profile_key(user_id, &profile_key, None) { + warn!("Failed removing {user_id} profile key {profile_key}: {e}"); + } } + for room_id in all_joined_rooms { + let state_lock = services.rooms.state.mutex.lock(&room_id).await; + + let room_power_levels = services + .rooms + .state_accessor + .room_state_get(&room_id, &StateEventType::RoomPowerLevels, "")? + .as_ref() + .and_then(|event| serde_json::from_str(event.content.get()).ok()?) + .and_then(|content: RoomPowerLevelsEventContent| content.into()); + + let user_can_demote_self = room_power_levels + .as_ref() + .is_some_and(|power_levels_content| { + RoomPowerLevels::from(power_levels_content.clone()).user_can_change_user_power_level(user_id, user_id) + }) || services + .rooms + .state_accessor + .room_state_get(&room_id, &StateEventType::RoomCreate, "")? + .as_ref() + .is_some_and(|event| event.sender == user_id); + + if user_can_demote_self { + let mut power_levels_content = room_power_levels.unwrap_or_default(); + power_levels_content.users.remove(user_id); + + // ignore errors so deactivation doesn't fail + if let Err(e) = services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomPowerLevels, + content: to_raw_value(&power_levels_content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + timestamp: None, + }, + user_id, + &room_id, + &state_lock, + ) + .await + { + warn!(%room_id, %user_id, "Failed to demote user's own power level: {e}"); + } else { + info!("Demoted {user_id} in {room_id} as part of account deactivation"); + } + } + } + + super::leave_all_rooms(services, user_id).await; + Ok(()) } From 032b199129f8648a77bde285f755a78e9ec349a7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Sep 2024 19:56:29 -0400 Subject: [PATCH 0804/2091] add db query command to get all pushers for a user Signed-off-by: strawberry --- src/admin/query/mod.rs | 7 ++++++- src/admin/query/pusher.rs | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 src/admin/query/pusher.rs diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index 1aa28c48..1f0f5505 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -2,6 +2,7 @@ mod account_data; mod appservice; mod globals; mod presence; +mod pusher; mod resolver; mod room_alias; mod room_state_cache; @@ -13,7 +14,7 @@ use conduit::Result; use self::{ account_data::AccountDataCommand, appservice::AppserviceCommand, globals::GlobalsCommand, - presence::PresenceCommand, resolver::ResolverCommand, room_alias::RoomAliasCommand, + presence::PresenceCommand, pusher::PusherCommand, resolver::ResolverCommand, room_alias::RoomAliasCommand, room_state_cache::RoomStateCacheCommand, sending::SendingCommand, users::UsersCommand, }; use crate::admin_command_dispatch; @@ -57,4 +58,8 @@ pub(super) enum QueryCommand { /// - resolver service #[command(subcommand)] Resolver(ResolverCommand), + + /// - pusher service + #[command(subcommand)] + Pusher(PusherCommand), } diff --git a/src/admin/query/pusher.rs b/src/admin/query/pusher.rs new file mode 100644 index 00000000..637c57b6 --- /dev/null +++ b/src/admin/query/pusher.rs @@ -0,0 +1,32 @@ +use clap::Subcommand; +use conduit::Result; +use ruma::{events::room::message::RoomMessageEventContent, UserId}; + +use crate::Command; + +#[derive(Debug, Subcommand)] +pub(crate) enum PusherCommand { + /// - Returns all the pushers for the user. + GetPushers { + /// Full user ID + user_id: Box, + }, +} + +pub(super) async fn process(subcommand: PusherCommand, context: &Command<'_>) -> Result { + let services = context.services; + + match subcommand { + PusherCommand::GetPushers { + user_id, + } => { + let timer = tokio::time::Instant::now(); + let results = services.pusher.get_pushers(&user_id)?; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) + }, + } +} From b9a8f8e6c72f6f1079b458b3f6a813b7b3953a4e Mon Sep 17 00:00:00 2001 From: Jacob Taylor Date: Wed, 4 Sep 2024 08:06:18 -0700 Subject: [PATCH 0805/2091] automatically scale conduwuit caches by CPU-core-count --- conduwuit-example.toml | 3 ++- src/core/config/mod.rs | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 76013b85..7cf9716d 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -481,6 +481,7 @@ allow_profile_lookup_federation_requests = true ### Generic database options # Set this to any float value to multiply conduwuit's in-memory LRU caches with. +# By default, the caches scale automatically with cpu-core-count. # May be useful if you have significant memory to spare to increase performance. # # This was previously called `conduit_cache_capacity_modifier` @@ -490,7 +491,7 @@ allow_profile_lookup_federation_requests = true # Set this to any float value in megabytes for conduwuit to tell the database engine that this much memory is available for database-related caches. # May be useful if you have significant memory to spare to increase performance. -# Defaults to 256.0 +# Defaults to 128.0 + (64.0 * CPU core count). #db_cache_capacity_mb = 256.0 diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 547196e4..8f857505 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -887,29 +887,29 @@ fn default_database_backups_to_keep() -> i16 { 1 } fn default_database_backend() -> String { "rocksdb".to_owned() } -fn default_db_cache_capacity_mb() -> f64 { 256.0 } +fn default_db_cache_capacity_mb() -> f64 { 128.0 + (64.0 * (crate::utils::available_parallelism() as f64)) } -fn default_pdu_cache_capacity() -> u32 { 150_000 } +fn default_pdu_cache_capacity() -> u32 { 100_000 + (10_000 * crate::utils::available_parallelism() as u32) } fn default_cache_capacity_modifier() -> f64 { 1.0 } -fn default_auth_chain_cache_capacity() -> u32 { 100_000 } +fn default_auth_chain_cache_capacity() -> u32 { 100_000 + (10_000 * crate::utils::available_parallelism() as u32) } -fn default_shorteventid_cache_capacity() -> u32 { 500_000 } +fn default_shorteventid_cache_capacity() -> u32 { 100_000 + (50_000 * crate::utils::available_parallelism() as u32) } -fn default_eventidshort_cache_capacity() -> u32 { 100_000 } +fn default_eventidshort_cache_capacity() -> u32 { 100_000 + (25_000 * crate::utils::available_parallelism() as u32) } -fn default_shortstatekey_cache_capacity() -> u32 { 100_000 } +fn default_shortstatekey_cache_capacity() -> u32 { 100_000 + (10_000 * crate::utils::available_parallelism() as u32) } -fn default_statekeyshort_cache_capacity() -> u32 { 100_000 } +fn default_statekeyshort_cache_capacity() -> u32 { 100_000 + (10_000 * crate::utils::available_parallelism() as u32) } -fn default_server_visibility_cache_capacity() -> u32 { 100 } +fn default_server_visibility_cache_capacity() -> u32 { 500 * crate::utils::available_parallelism() as u32 } -fn default_user_visibility_cache_capacity() -> u32 { 100 } +fn default_user_visibility_cache_capacity() -> u32 { 1000 * crate::utils::available_parallelism() as u32 } -fn default_stateinfo_cache_capacity() -> u32 { 100 } +fn default_stateinfo_cache_capacity() -> u32 { 1000 * crate::utils::available_parallelism() as u32 } -fn default_roomid_spacehierarchy_cache_capacity() -> u32 { 100 } +fn default_roomid_spacehierarchy_cache_capacity() -> u32 { 1000 * crate::utils::available_parallelism() as u32 } fn default_dns_cache_entries() -> u32 { 32768 } From 04130372466133975ff29cf445792df67d145e29 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 5 Oct 2024 22:54:51 +0000 Subject: [PATCH 0806/2091] fix lints Signed-off-by: Jason Volk --- src/core/config/mod.rs | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 8f857505..47ed0d60 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -21,7 +21,7 @@ use url::Url; pub use self::check::check; use self::proxy::ProxyConfig; -use crate::{error::Error, Err, Result}; +use crate::{error::Error, utils::sys, Err, Result}; pub mod check; pub mod proxy; @@ -887,29 +887,29 @@ fn default_database_backups_to_keep() -> i16 { 1 } fn default_database_backend() -> String { "rocksdb".to_owned() } -fn default_db_cache_capacity_mb() -> f64 { 128.0 + (64.0 * (crate::utils::available_parallelism() as f64)) } +fn default_db_cache_capacity_mb() -> f64 { 128.0 + parallelism_scaled_f64(64.0) } -fn default_pdu_cache_capacity() -> u32 { 100_000 + (10_000 * crate::utils::available_parallelism() as u32) } +fn default_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } fn default_cache_capacity_modifier() -> f64 { 1.0 } -fn default_auth_chain_cache_capacity() -> u32 { 100_000 + (10_000 * crate::utils::available_parallelism() as u32) } +fn default_auth_chain_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } -fn default_shorteventid_cache_capacity() -> u32 { 100_000 + (50_000 * crate::utils::available_parallelism() as u32) } +fn default_shorteventid_cache_capacity() -> u32 { parallelism_scaled_u32(50_000).saturating_add(100_000) } -fn default_eventidshort_cache_capacity() -> u32 { 100_000 + (25_000 * crate::utils::available_parallelism() as u32) } +fn default_eventidshort_cache_capacity() -> u32 { parallelism_scaled_u32(25_000).saturating_add(100_000) } -fn default_shortstatekey_cache_capacity() -> u32 { 100_000 + (10_000 * crate::utils::available_parallelism() as u32) } +fn default_shortstatekey_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } -fn default_statekeyshort_cache_capacity() -> u32 { 100_000 + (10_000 * crate::utils::available_parallelism() as u32) } +fn default_statekeyshort_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } -fn default_server_visibility_cache_capacity() -> u32 { 500 * crate::utils::available_parallelism() as u32 } +fn default_server_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(500) } -fn default_user_visibility_cache_capacity() -> u32 { 1000 * crate::utils::available_parallelism() as u32 } +fn default_user_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(1000) } -fn default_stateinfo_cache_capacity() -> u32 { 1000 * crate::utils::available_parallelism() as u32 } +fn default_stateinfo_cache_capacity() -> u32 { parallelism_scaled_u32(1000) } -fn default_roomid_spacehierarchy_cache_capacity() -> u32 { 1000 * crate::utils::available_parallelism() as u32 } +fn default_roomid_spacehierarchy_cache_capacity() -> u32 { parallelism_scaled_u32(1000) } fn default_dns_cache_entries() -> u32 { 32768 } @@ -1087,3 +1087,13 @@ fn default_admin_log_capture() -> String { } fn default_admin_room_tag() -> String { "m.server_notice".to_owned() } + +#[allow(clippy::as_conversions, clippy::cast_precision_loss)] +fn parallelism_scaled_f64(val: f64) -> f64 { val * (sys::available_parallelism() as f64) } + +fn parallelism_scaled_u32(val: u32) -> u32 { + let val = val.try_into().expect("failed to cast u32 to usize"); + parallelism_scaled(val).try_into().unwrap_or(u32::MAX) +} + +fn parallelism_scaled(val: usize) -> usize { val.saturating_mul(sys::available_parallelism()) } From e589464954ff88bf817b8bdff4c803cc3bf18a28 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 5 Oct 2024 22:29:55 -0400 Subject: [PATCH 0807/2091] bump cargo.lock and deps Signed-off-by: strawberry --- .gitlab-ci.yml | 4 +- Cargo.lock | 468 +++++++++++++++++++++++-------------------------- Cargo.toml | 13 +- 3 files changed, 226 insertions(+), 259 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c954e58..83a69144 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,7 +58,7 @@ before_script: ci: stage: ci - image: nixos/nix:2.24.4 + image: nixos/nix:2.24.9 script: # Cache CI dependencies - ./bin/nix-build-and-cache ci @@ -83,7 +83,7 @@ ci: artifacts: stage: artifacts - image: nixos/nix:2.24.4 + image: nixos/nix:2.24.9 script: - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl - cp result/bin/conduit x86_64-unknown-linux-musl diff --git a/Cargo.lock b/Cargo.lock index 67ba00f2..869ce36e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,19 +4,13 @@ version = 3 [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ "gimli", ] -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - [[package]] name = "adler2" version = "2.0.0" @@ -55,9 +49,9 @@ checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anyhow" -version = "1.0.87" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10f00e1f6e58a40e807377c75c6a7f97bf9044fab57816f2414e6f5f4499d7b8" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "arc-swap" @@ -97,9 +91,9 @@ checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" [[package]] name = "async-compression" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec134f64e2bc57411226dfc4e52dec859ddfc7e711fc5e07b612584f000e4aa" +checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429" dependencies = [ "brotli", "flate2", @@ -113,9 +107,9 @@ dependencies = [ [[package]] name = "async-stream" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" dependencies = [ "async-stream-impl", "futures-core", @@ -124,24 +118,24 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "async-trait" -version = "0.1.82" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a27b8a3a6e1a44fa4c8baf1f653e4172e81486d4941f2237e20dc2d0cf4ddff1" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -161,15 +155,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f95446d919226d587817a7d21379e6eb099b97b45110a7f272a444ca5c54070" +checksum = "cdd82dba44d209fddb11c190e0a94b78651f95299598e472215667417a03ff1d" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -179,9 +173,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5055edc4a9a1b2a917a818258cdfb86a535947feebd9981adc99667a062c6f85" +checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972" dependencies = [ "bindgen", "cc", @@ -194,9 +188,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" dependencies = [ "async-trait", "axum-core", @@ -220,7 +214,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower 0.4.13", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -239,9 +233,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" dependencies = [ "async-trait", "bytes", @@ -252,7 +246,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower-layer", "tower-service", "tracing", @@ -260,9 +254,9 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0be6ea09c9b96cb5076af0de2e383bd2bc0c18f827cf1967bdd353e0b910d733" +checksum = "73c3220b188aea709cf1b6c5f9b01c3bd936bb08bd2b5184a12b35ac8131b1f9" dependencies = [ "axum", "axum-core", @@ -275,7 +269,7 @@ dependencies = [ "mime", "pin-project-lite", "serde", - "tower 0.4.13", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -296,7 +290,7 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls 0.23.12", + "rustls 0.23.14", "rustls-pemfile", "rustls-pki-types", "tokio", @@ -316,7 +310,7 @@ dependencies = [ "http", "http-body-util", "pin-project", - "rustls 0.23.12", + "rustls 0.23.14", "tokio", "tokio-rustls", "tokio-util", @@ -326,17 +320,17 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cc", "cfg-if", "libc", - "miniz_oxide 0.7.4", + "miniz_oxide", "object", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -376,7 +370,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.77", + "syn 2.0.79", "which", ] @@ -412,9 +406,9 @@ dependencies = [ [[package]] name = "brotli" -version = "6.0.0" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -457,9 +451,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.7.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "bzip2-sys" @@ -474,9 +468,9 @@ dependencies = [ [[package]] name = "cargo_toml" -version = "0.20.4" +version = "0.20.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad639525b1c67b6a298f378417b060fbc04618bea559482a8484381cce27d965" +checksum = "88da5a13c620b4ca0078845707ea9c3faf11edbc3ffd8497d11d686211cd1ac0" dependencies = [ "serde", "toml", @@ -484,9 +478,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.18" +version = "1.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" +checksum = "e8d9e0b4957f635b8d3da819d0db5603620467ecf1f692d22a8c2717ce27e6d8" dependencies = [ "jobserver", "libc", @@ -545,9 +539,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" dependencies = [ "clap_builder", "clap_derive", @@ -555,9 +549,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" dependencies = [ "anstyle", "clap_lex", @@ -565,14 +559,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -709,7 +703,7 @@ dependencies = [ "reqwest", "ring", "ruma", - "rustls 0.23.12", + "rustls 0.23.14", "sanitize-filename", "serde", "serde_json", @@ -746,7 +740,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -769,14 +763,14 @@ dependencies = [ "hyper-util", "log", "ruma", - "rustls 0.23.12", + "rustls 0.23.14", "sd-notify", "sentry", "sentry-tower", "sentry-tracing", "serde_json", "tokio", - "tower 0.5.0", + "tower 0.5.1", "tower-http", "tracing", ] @@ -870,9 +864,9 @@ checksum = "3618cccc083bb987a415d85c02ca6c9994ea5b44731ec28b9ecf09658655fba9" [[package]] name = "const_panic" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7782af8f90fe69a4bb41e460abe1727d493403d8b2cc43201a3a3e906b24379f" +checksum = "013b6c2c3a14d678f38cd23994b02da3a1a1b6a5d1eedddfe63a5a5f11b13a81" [[package]] name = "coolor" @@ -1042,7 +1036,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1069,7 +1063,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1172,14 +1166,14 @@ dependencies = [ [[package]] name = "enum-as-inner" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" dependencies = [ - "heck 0.4.1", + "heck", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1200,9 +1194,9 @@ dependencies = [ [[package]] name = "fdeflate" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f9bfee30e4dedf0ab8b422f03af778d9612b63f502710fc500a334ebe2de645" +checksum = "d8090f921a24b04994d9929e204f50b498a33ea6ba559ffaa05e04f7ee7fb5ab" dependencies = [ "simd-adler32", ] @@ -1241,12 +1235,12 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.33" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", - "miniz_oxide 0.8.0", + "miniz_oxide", ] [[package]] @@ -1292,9 +1286,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -1302,15 +1296,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -1319,38 +1313,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", "futures-io", @@ -1398,9 +1392,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -1420,7 +1414,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -1441,9 +1435,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" [[package]] name = "hdrhistogram" @@ -1482,12 +1476,6 @@ dependencies = [ "http", ] -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" @@ -1602,7 +1590,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1650,9 +1638,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -1697,8 +1685,8 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.12", - "rustls-native-certs 0.8.0", + "rustls 0.23.14", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -1721,9 +1709,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -1798,12 +1786,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -1847,9 +1835,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "itertools" @@ -1966,7 +1954,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -1983,9 +1971,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.158" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libloading" @@ -2053,9 +2041,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.10.0" +version = "1.11.1+lz4-1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109de74d5d2353660401699a4174a4ff23fcc649caf553df71933c7fb45ad868" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" dependencies = [ "cc", "libc", @@ -2147,16 +2135,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" -[[package]] -name = "miniz_oxide" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" -dependencies = [ - "adler", - "simd-adler32", -] - [[package]] name = "miniz_oxide" version = "0.8.0" @@ -2164,6 +2142,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -2320,18 +2299,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.4" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "084f1a5821ac4c651660a94a7153d27ac9d8a53736203f58b31945ded098070a" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "openssl-probe" @@ -2347,7 +2326,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.5.0", + "indexmap 2.6.0", "js-sys", "once_cell", "pin-project-lite", @@ -2394,7 +2373,7 @@ dependencies = [ "glob", "once_cell", "opentelemetry", - "ordered-float 4.2.2", + "ordered-float 4.3.0", "percent-encoding", "rand", "thiserror", @@ -2413,9 +2392,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.2.2" +version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a91171844676f8c7990ce64959210cd2eaef32c2612c50f9fae9f8aaa6065a6" +checksum = "44d501f1a72f71d3c063a6bbc8f7271fa73aa09fe5d6283b6571e2ed176a2537" dependencies = [ "num-traits", ] @@ -2497,7 +2476,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2575,22 +2554,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2617,21 +2596,21 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "png" -version = "0.17.13" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06e4b0d3d1312775e782c86c91a111aa1f910cbb65e1337f9975b5f9a554b5e1" +checksum = "52f9d46a34a05a6a57566bc2bfae066ef07585a6e3fa30fbbdff5936380623f0" dependencies = [ "bitflags 1.3.2", "crc32fast", "fdeflate", "flate2", - "miniz_oxide 0.7.4", + "miniz_oxide", ] [[package]] @@ -2662,7 +2641,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -2691,16 +2670,16 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "version_check", "yansi", ] [[package]] name = "prost" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2ecbe40f08db5c006b5764a2645f7f3f141ce756412ac9e1dd6087e6d32995" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" dependencies = [ "bytes", "prost-derive", @@ -2708,22 +2687,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf0c195eebb4af52c752bec4f52f645da98b6e92077a04110c7f349477ae5ac" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" dependencies = [ "anyhow", "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] name = "prost-types" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60caa6738c7369b940c3d49246a8d1749323674c65cb13010134f5c9bad5b519" +checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" dependencies = [ "prost", ] @@ -2769,7 +2748,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.12", + "rustls 0.23.14", "socket2", "thiserror", "tokio", @@ -2786,7 +2765,7 @@ dependencies = [ "rand", "ring", "rustc-hash 2.0.0", - "rustls 0.23.12", + "rustls 0.23.14", "slab", "thiserror", "tinyvec", @@ -2847,23 +2826,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -2877,13 +2856,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -2894,15 +2873,15 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "async-compression", "base64 0.22.1", @@ -2926,8 +2905,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.12", - "rustls-native-certs 0.7.3", + "rustls 0.23.14", + "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", "serde", @@ -3039,7 +3018,7 @@ dependencies = [ "bytes", "form_urlencoded", "http", - "indexmap 2.5.0", + "indexmap 2.6.0", "js_int", "konst", "percent-encoding", @@ -3065,7 +3044,7 @@ version = "0.28.1" source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" dependencies = [ "as_variant", - "indexmap 2.5.0", + "indexmap 2.6.0", "js_int", "js_option", "percent-encoding", @@ -3132,7 +3111,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.77", + "syn 2.0.79", "toml", ] @@ -3255,9 +3234,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.36" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f55e80d50763938498dd5ebb18647174e0c76dc38c5505294bb224624f30f36" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -3282,9 +3261,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" dependencies = [ "aws-lc-rs", "log", @@ -3296,19 +3275,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" -dependencies = [ - "openssl-probe", - "rustls-pemfile", - "rustls-pki-types", - "schannel", - "security-framework", -] - [[package]] name = "rustls-native-certs" version = "0.8.0" @@ -3324,25 +3290,24 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" -version = "0.102.7" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "aws-lc-rs", "ring", @@ -3404,9 +3369,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sd-notify" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4646d6f919800cd25c50edb49438a1381e2cd4833c027e75e8897981c50b8b5e" +checksum = "1be20c5f7f393ee700f8b2f28ea35812e4e212f40774b550cd2a93ea91684451" [[package]] name = "security-framework" @@ -3423,9 +3388,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -3589,7 +3554,7 @@ checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -3599,7 +3564,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" dependencies = [ "form_urlencoded", - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -3639,9 +3604,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -3664,7 +3629,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -3883,9 +3848,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.77" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -3946,22 +3911,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4096,7 +4061,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4117,7 +4082,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.12", + "rustls 0.23.14", "rustls-pki-types", "tokio", ] @@ -4181,11 +4146,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.20" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -4194,9 +4159,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6f6ba989e4b2c58ae83d862d3a3e27690b6e3ae630d0deb59f3697f32aa88ad" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ "async-stream", "async-trait", @@ -4244,23 +4209,24 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b837f86b25d7c0d7988f00a54e74739be6477f2aac6201b8f429a7569991b7" +checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" dependencies = [ "futures-core", "futures-util", "pin-project-lite", "sync_wrapper 0.1.2", + "tokio", "tower-layer", "tower-service", ] [[package]] name = "tower-http" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" dependencies = [ "async-compression", "bitflags 2.6.0", @@ -4273,7 +4239,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower 0.4.13", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -4309,7 +4275,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b47 dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] @@ -4433,36 +4399,36 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unsafe-libyaml" @@ -4485,7 +4451,7 @@ dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.23.12", + "rustls 0.23.14", "rustls-pki-types", "url", "webpki-roots", @@ -4580,7 +4546,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-shared", ] @@ -4614,7 +4580,7 @@ checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4669,9 +4635,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.5" +version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" dependencies = [ "rustls-pki-types", ] @@ -4702,9 +4668,9 @@ checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" [[package]] name = "wildmatch" -version = "2.3.4" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3928939971918220fed093266b809d1ee4ec6c1a2d72692ff6876898f3b16c19" +checksum = "68ce1ab1f8c62655ebe1350f589c61e505cf94d385bc6a12899442d9081e71fd" [[package]] name = "winapi" @@ -4927,9 +4893,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.18" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -4979,7 +4945,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.77", + "syn 2.0.79", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 86c2bcc3..c1ebe2ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ version = "0.8.5" # Used for the http request / response body type for Ruma endpoints used with reqwest [workspace.dependencies.bytes] -version = "1.7.1" +version = "1.7.2" [workspace.dependencies.http-body-util] version = "0.1.1" @@ -111,12 +111,12 @@ version = "0.7" version = "0.6.0" [workspace.dependencies.tower] -version = "0.5.0" +version = "0.5.1" default-features = false features = ["util"] [workspace.dependencies.tower-http] -version = "0.5.2" +version = "0.6.0" default-features = false features = [ "add-extension", @@ -129,10 +129,10 @@ features = [ ] [workspace.dependencies.rustls] -version = "0.23.12" +version = "0.23.13" [workspace.dependencies.reqwest] -version = "0.12.7" +version = "0.12.8" default-features = false features = [ "rustls-tls-native-roots", @@ -257,7 +257,8 @@ features = [ ] [workspace.dependencies.hyper-util] -version = "0.1.6" +# 0.1.9 causes DNS issues +version = "=0.1.8" default-features = false features = [ "client", From 8428e7cdf739fc1ce2dd34b96b6ff95b22d0d4b7 Mon Sep 17 00:00:00 2001 From: nisbet-hubbard <87453615+nisbet-hubbard@users.noreply.github.com> Date: Sat, 19 Oct 2024 11:12:41 +0800 Subject: [PATCH 0808/2091] Update generic.md --- docs/deploying/generic.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index cae57986..2ea32e3a 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -58,6 +58,8 @@ The systemd unit for conduwuit can be found [here](../configuration/examples.md#example-systemd-unit-file). You may need to change the `ExecStart=` path to where you placed the conduwuit binary. +On systems where rsyslog is used alongside journald (i.e. Red Hat-based distros and OpenSUSE), put `$EscapeControlCharactersOnReceive off` inside `/etc/rsyslog.conf` to allow color in logs. + ## Creating the conduwuit configuration file Now we need to create the conduwuit's config file in From 6c1c7b35a57669b5974ad6fa26c42bf2632dbb61 Mon Sep 17 00:00:00 2001 From: Toby Murray Date: Wed, 23 Oct 2024 19:59:56 -0400 Subject: [PATCH 0809/2091] Separate command lines I don't think this works if the commands are invoked on the same line with no thing joining them, so separate them on to separate lines. --- docs/deploying/generic.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 2ea32e3a..6c478d19 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -76,13 +76,15 @@ If you are using a dedicated user for conduwuit, you will need to allow it to read the config. To do that you can run this: ```bash -sudo chown -R root:root /etc/conduwuit sudo chmod -R 755 /etc/conduwuit +sudo chown -R root:root /etc/conduwuit +sudo chmod -R 755 /etc/conduwuit ``` If you use the default database path you also need to run this: ```bash -sudo mkdir -p /var/lib/conduwuit/ sudo chown -R conduwuit:conduwuit /var/lib/conduwuit/ +sudo mkdir -p /var/lib/conduwuit/ +sudo chown -R conduwuit:conduwuit /var/lib/conduwuit/ sudo chmod 700 /var/lib/conduwuit/ ``` From 2d54264fbe1379a3238e128bb63035a021edc9e4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 20:47:51 -0400 Subject: [PATCH 0810/2091] bump rust to 1.82.0, rocksdb v9.7.3, ruwuma, and a few nix pkgs Signed-off-by: strawberry --- Cargo.lock | 285 +++++++++++++------------- Cargo.toml | 28 ++- deps/rust-rocksdb/Cargo.toml | 2 +- flake.lock | 351 ++++++++++++++++++++++++++------ flake.nix | 6 +- rust-toolchain.toml | 11 +- src/api/client/room.rs | 1 + src/api/router/auth.rs | 1 + src/core/Cargo.toml | 4 - src/core/config/check.rs | 16 -- src/core/debug.rs | 2 +- src/main/Cargo.toml | 1 + src/service/rooms/short/data.rs | 1 + src/service/sending/sender.rs | 2 + 14 files changed, 467 insertions(+), 244 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 869ce36e..1103dc8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,9 +49,9 @@ checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" [[package]] name = "anyhow" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" [[package]] name = "arc-swap" @@ -91,9 +91,9 @@ checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" [[package]] name = "async-compression" -version = "0.4.13" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e614738943d3f68c628ae3dbce7c3daffb196665f82f8c8ea6b65de73c79429" +checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" dependencies = [ "brotli", "flate2", @@ -124,7 +124,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -135,7 +135,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -222,9 +222,9 @@ dependencies = [ [[package]] name = "axum-client-ip" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72188bed20deb981f3a4a9fe674e5980fd9e9c2bd880baa94715ad5d60d64c67" +checksum = "9eefda7e2b27e1bda4d6fa8a06b50803b8793769045918bc37ad062d48a6efac" dependencies = [ "axum", "forwarded-header-value", @@ -290,7 +290,7 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls 0.23.14", + "rustls 0.23.15", "rustls-pemfile", "rustls-pki-types", "tokio", @@ -310,7 +310,7 @@ dependencies = [ "http", "http-body-util", "pin-project", - "rustls 0.23.14", + "rustls 0.23.15", "tokio", "tokio-rustls", "tokio-util", @@ -353,9 +353,9 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bindgen" -version = "0.69.4" +version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ "bitflags 2.6.0", "cexpr", @@ -370,7 +370,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.79", + "syn 2.0.85", "which", ] @@ -433,9 +433,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" [[package]] name = "byteorder" @@ -451,9 +451,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" [[package]] name = "bzip2-sys" @@ -478,9 +478,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.25" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d9e0b4957f635b8d3da819d0db5603620467ecf1f692d22a8c2717ce27e6d8" +checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" dependencies = [ "jobserver", "libc", @@ -539,9 +539,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -549,9 +549,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.19" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstyle", "clap_lex", @@ -566,7 +566,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -703,7 +703,7 @@ dependencies = [ "reqwest", "ring", "ruma", - "rustls 0.23.14", + "rustls 0.23.15", "sanitize-filename", "serde", "serde_json", @@ -740,7 +740,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -763,7 +763,7 @@ dependencies = [ "hyper-util", "log", "ruma", - "rustls 0.23.14", + "rustls 0.23.15", "sd-notify", "sentry", "sentry-tower", @@ -1036,7 +1036,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -1063,7 +1063,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -1173,7 +1173,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -1325,7 +1325,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -1590,7 +1590,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -1656,9 +1656,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" dependencies = [ "bytes", "futures-channel", @@ -1685,7 +1685,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.14", + "rustls 0.23.15", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -1749,9 +1749,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.2" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10" +checksum = "bc144d44a31d753b02ce64093d532f55ff8dc4ebf2ffb8a63c0dda691385acae" dependencies = [ "bytemuck", "byteorder-lite", @@ -1766,9 +1766,9 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904" +checksum = "e031e8e3d94711a9ccb5d6ea357439ef3dcbed361798bd4071dc4d9793fbe22f" dependencies = [ "byteorder-lite", "quick-error 2.0.1", @@ -1874,9 +1874,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" dependencies = [ "wasm-bindgen", ] @@ -1954,7 +1954,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -1971,9 +1971,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.159" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "libloading" @@ -2373,7 +2373,7 @@ dependencies = [ "glob", "once_cell", "opentelemetry", - "ordered-float 4.3.0", + "ordered-float 4.4.0", "percent-encoding", "rand", "thiserror", @@ -2392,9 +2392,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d501f1a72f71d3c063a6bbc8f7271fa73aa09fe5d6283b6571e2ed176a2537" +checksum = "83e7ccb95e240b7c9506a3d544f10d935e142cc90b0a1d56954fb44d89ad6b97" dependencies = [ "num-traits", ] @@ -2476,7 +2476,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -2554,29 +2554,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf123a161dde1e524adf36f90bc5d8d3462824a9c43553ad07a8183161189ec" +checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4502d8515ca9f32f1fb543d987f63d95a14934883db45bdb48060b6b69257f8" +checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] name = "pin-project-lite" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" [[package]] name = "pin-utils" @@ -2636,12 +2636,12 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.22" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -2655,9 +2655,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" dependencies = [ "unicode-ident", ] @@ -2670,7 +2670,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", "version_check", "yansi", ] @@ -2695,7 +2695,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -2709,9 +2709,9 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "666f0f59e259aea2d72e6012290c09877a780935cc3c18b1ceded41f3890d59c" +checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ "bitflags 2.6.0", "memchr", @@ -2748,7 +2748,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.14", + "rustls 0.23.15", "socket2", "thiserror", "tokio", @@ -2765,7 +2765,7 @@ dependencies = [ "rand", "ring", "rustc-hash 2.0.0", - "rustls 0.23.14", + "rustls 0.23.15", "slab", "thiserror", "tinyvec", @@ -2835,9 +2835,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", @@ -2905,7 +2905,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.14", + "rustls 0.23.15", "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", @@ -2954,7 +2954,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "assign", "js_int", @@ -2976,7 +2976,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "js_int", "ruma-common", @@ -2988,7 +2988,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "as_variant", "assign", @@ -3011,7 +3011,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "as_variant", "base64 0.22.1", @@ -3041,7 +3041,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3065,7 +3065,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "bytes", "http", @@ -3083,7 +3083,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "js_int", "thiserror", @@ -3092,7 +3092,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "js_int", "ruma-common", @@ -3102,7 +3102,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "cfg-if", "once_cell", @@ -3111,14 +3111,14 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.79", + "syn 2.0.85", "toml", ] [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "js_int", "ruma-common", @@ -3130,7 +3130,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "headers", "http", @@ -3143,7 +3143,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3159,7 +3159,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad#b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" dependencies = [ "itertools 0.12.1", "js_int", @@ -3173,8 +3173,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.26.0+9.6.1" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=22f942609921ecf837399e1c2fe6fcb8dbb1095e#22f942609921ecf837399e1c2fe6fcb8dbb1095e" +version = "0.28.0+9.7.3" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=c1e5523eae095a893deaf9056128c7dbc2d5fd73#c1e5523eae095a893deaf9056128c7dbc2d5fd73" dependencies = [ "bindgen", "bzip2-sys", @@ -3190,8 +3190,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.30.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=22f942609921ecf837399e1c2fe6fcb8dbb1095e#22f942609921ecf837399e1c2fe6fcb8dbb1095e" +version = "0.31.0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=c1e5523eae095a893deaf9056128c7dbc2d5fd73#c1e5523eae095a893deaf9056128c7dbc2d5fd73" dependencies = [ "libc", "rust-librocksdb-sys", @@ -3261,9 +3261,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.14" +version = "0.23.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415d9944693cb90382053259f89fbb077ea730ad7273047ec63b19bc9b160ba8" +checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" dependencies = [ "aws-lc-rs", "log", @@ -3299,9 +3299,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" +checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" [[package]] name = "rustls-webpki" @@ -3317,9 +3317,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" +checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "rustyline-async" @@ -3354,9 +3354,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.24" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" +checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" dependencies = [ "windows-sys 0.59.0", ] @@ -3539,22 +3539,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.213" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -3572,9 +3572,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" dependencies = [ "itoa", "memchr", @@ -3848,9 +3848,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.79" +version = "2.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" dependencies = [ "proc-macro2", "quote", @@ -3885,9 +3885,9 @@ dependencies = [ [[package]] name = "termimad" -version = "0.30.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "920e7c4671e79f3d9df269da9c8edf0dbc580044fd727d3594f7bfba5eb6107a" +checksum = "22117210909e9dfff30a558f554c7fb3edb198ef614e7691386785fb7679677c" dependencies = [ "coolor", "crokey", @@ -3911,22 +3911,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" +checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.64" +version = "1.0.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" +checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -3964,7 +3964,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-ctl" version = "0.6.0" -source = "git+https://github.com/girlbossceo/jemallocator?rev=c32af15f3b440ae5e46c3404f78b19093bbd5294#c32af15f3b440ae5e46c3404f78b19093bbd5294" +source = "git+https://github.com/girlbossceo/jemallocator?rev=d87938bfddc26377dd7fdf14bbcd345f3ab19442#d87938bfddc26377dd7fdf14bbcd345f3ab19442" dependencies = [ "libc", "paste", @@ -3974,7 +3974,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" -source = "git+https://github.com/girlbossceo/jemallocator?rev=c32af15f3b440ae5e46c3404f78b19093bbd5294#c32af15f3b440ae5e46c3404f78b19093bbd5294" +source = "git+https://github.com/girlbossceo/jemallocator?rev=d87938bfddc26377dd7fdf14bbcd345f3ab19442#d87938bfddc26377dd7fdf14bbcd345f3ab19442" dependencies = [ "cc", "libc", @@ -3983,7 +3983,7 @@ dependencies = [ [[package]] name = "tikv-jemallocator" version = "0.6.0" -source = "git+https://github.com/girlbossceo/jemallocator?rev=c32af15f3b440ae5e46c3404f78b19093bbd5294#c32af15f3b440ae5e46c3404f78b19093bbd5294" +source = "git+https://github.com/girlbossceo/jemallocator?rev=d87938bfddc26377dd7fdf14bbcd345f3ab19442#d87938bfddc26377dd7fdf14bbcd345f3ab19442" dependencies = [ "libc", "tikv-jemalloc-sys", @@ -4037,9 +4037,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.40.0" +version = "1.41.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" +checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" dependencies = [ "backtrace", "bytes", @@ -4061,7 +4061,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -4082,7 +4082,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.14", + "rustls 0.23.15", "rustls-pki-types", "tokio", ] @@ -4275,7 +4275,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b47 dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] @@ -4390,12 +4390,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" [[package]] name = "unicode-bidi" @@ -4451,7 +4448,7 @@ dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.23.14", + "rustls 0.23.15", "rustls-pki-types", "url", "webpki-roots", @@ -4483,9 +4480,9 @@ checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] name = "uuid" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", "serde", @@ -4526,9 +4523,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" dependencies = [ "cfg-if", "once_cell", @@ -4537,24 +4534,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.43" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" +checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" dependencies = [ "cfg-if", "js-sys", @@ -4564,9 +4561,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4574,28 +4571,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.93" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" +checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" [[package]] name = "web-sys" -version = "0.3.70" +version = "0.3.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" +checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" dependencies = [ "js-sys", "wasm-bindgen", @@ -4945,7 +4942,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.79", + "syn 2.0.85", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index c1ebe2ec..fd124512 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" -rust-version = "1.81.0" +rust-version = "1.82.0" version = "0.4.7" [workspace.metadata.crane] @@ -108,7 +108,7 @@ features = ["tls-rustls"] version = "0.7" [workspace.dependencies.axum-client-ip] -version = "0.6.0" +version = "0.6.1" [workspace.dependencies.tower] version = "0.5.1" @@ -200,7 +200,7 @@ default-features = false # used for conduit's CLI and admin room command parsing [workspace.dependencies.clap] -version = "4.5.15" +version = "4.5.20" default-features = false features = [ "std", @@ -248,7 +248,7 @@ features = ["alloc", "std"] default-features = false [workspace.dependencies.hyper] -version = "1.4.1" +version = "1.5.0" default-features = false features = [ "server", @@ -315,7 +315,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "b6f82a72b6c0899d8ac8e53206d375c2c6f0a2ad" +rev = "9900d0676564883cfade556d6e8da2a2c9061efd" features = [ "compat", "rand", @@ -330,16 +330,19 @@ features = [ "ring-compat", "identifiers-validation", "unstable-unspecified", + "unstable-msc2409", "unstable-msc2448", "unstable-msc2666", "unstable-msc2867", "unstable-msc2870", "unstable-msc3026", "unstable-msc3061", + "unstable-msc3245", "unstable-msc3266", "unstable-msc3381", # polls "unstable-msc3489", # beacon / live location "unstable-msc3575", + "unstable-msc4075", "unstable-msc4121", "unstable-msc4125", "unstable-msc4186", @@ -405,17 +408,17 @@ version = "0.34.0" # jemalloc usage [workspace.dependencies.tikv-jemalloc-sys] git = "https://github.com/girlbossceo/jemallocator" -rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294" +rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442" default-features = false features = ["unprefixed_malloc_on_supported_platforms"] [workspace.dependencies.tikv-jemallocator] git = "https://github.com/girlbossceo/jemallocator" -rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294" +rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442" default-features = false features = ["unprefixed_malloc_on_supported_platforms"] [workspace.dependencies.tikv-jemalloc-ctl] git = "https://github.com/girlbossceo/jemallocator" -rev = "c32af15f3b440ae5e46c3404f78b19093bbd5294" +rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442" default-features = false features = ["use_std"] @@ -444,7 +447,7 @@ version = "0.4.3" default-features = false [workspace.dependencies.termimad] -version = "0.30.0" +version = "0.30.1" default-features = false [workspace.dependencies.checked_ops] @@ -459,7 +462,7 @@ features = ["full", "extra-traits"] version = "1.0.36" [workspace.dependencies.proc-macro2] -version = "1.0.86" +version = "1.0.89" # @@ -717,12 +720,16 @@ opt-level = 'z' # primarily used for CI [profile.test] inherits = "dev" +strip = false +opt-level = 0 codegen-units = 16 incremental = false [profile.test.package.'*'] inherits = "dev" debug = 0 +strip = false +opt-level = 0 codegen-units = 16 incremental = false @@ -810,6 +817,7 @@ significant_drop_tightening = { level = "allow", priority = 1 } # TODO pedantic = { level = "warn", priority = -1 } ## some sadness +too_long_first_doc_paragraph = { level = "allow", priority = 1 } doc_markdown = { level = "allow", priority = 1 } enum_glob_use = { level = "allow", priority = 1 } if_not_else = { level = "allow", priority = 1 } diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index e86f6e77..8c168b24 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "22f942609921ecf837399e1c2fe6fcb8dbb1095e" +rev = "c1e5523eae095a893deaf9056128c7dbc2d5fd73" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index bd82f81f..271a2154 100644 --- a/flake.lock +++ b/flake.lock @@ -5,16 +5,15 @@ "crane": "crane", "flake-compat": "flake-compat", "flake-parts": "flake-parts", - "flake-utils": "flake-utils", "nixpkgs": "nixpkgs", "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1725300620, - "narHash": "sha256-IdM+pZ6BnmD3o1fTJZ2BD43k7dwi1BbVfLDLpM1nE5s=", + "lastModified": 1729116596, + "narHash": "sha256-NnLMLIXGZtAscUF4dCShksuQ1nOGF6Y2dEeyj0rBbUg=", "owner": "zhaofengli", "repo": "attic", - "rev": "bea72d75b6165dfb529ba0c39cc6c7e9c7f0d234", + "rev": "2b05b7d986cf6009b1c1ef7daa4961cd1a658782", "type": "github" }, "original": { @@ -29,14 +28,14 @@ "devenv": "devenv", "flake-compat": "flake-compat_3", "git-hooks": "git-hooks", - "nixpkgs": "nixpkgs_3" + "nixpkgs": "nixpkgs_4" }, "locked": { - "lastModified": 1724232775, - "narHash": "sha256-6u2DycIEgrgNYlLxyGqdFVmBNiKIitnQKJ1pbRP5oko=", + "lastModified": 1728672398, + "narHash": "sha256-KxuGSoVUFnQLB2ZcYODW7AVPAh9JqRlD5BrfsC/Q4qs=", "owner": "cachix", "repo": "cachix", - "rev": "03b6cb3f953097bff378fb8b9ea094bd091a4ec7", + "rev": "aac51f698309fd0f381149214b7eee213c66ef0a", "type": "github" }, "original": { @@ -54,12 +53,51 @@ "devenv", "flake-compat" ], + "git-hooks": [ + "cachix", + "devenv", + "pre-commit-hooks" + ], "nixpkgs": [ "cachix", "devenv", "nixpkgs" + ] + }, + "locked": { + "lastModified": 1726520618, + "narHash": "sha256-jOsaBmJ/EtX5t/vbylCdS7pWYcKGmWOKg4QKUzKr6dA=", + "owner": "cachix", + "repo": "cachix", + "rev": "695525f9086542dfb09fde0871dbf4174abbf634", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "cachix", + "type": "github" + } + }, + "cachix_3": { + "inputs": { + "devenv": "devenv_3", + "flake-compat": [ + "cachix", + "devenv", + "cachix", + "devenv", + "flake-compat" + ], + "nixpkgs": [ + "cachix", + "devenv", + "cachix", + "devenv", + "nixpkgs" ], "pre-commit-hooks": [ + "cachix", + "devenv", "cachix", "devenv", "pre-commit-hooks" @@ -119,11 +157,11 @@ }, "crane_2": { "locked": { - "lastModified": 1725409566, - "narHash": "sha256-PrtLmqhM6UtJP7v7IGyzjBFhbG4eOAHT6LPYOFmYfbk=", + "lastModified": 1729741221, + "narHash": "sha256-8AHZZXs1lFkERfBY0C8cZGElSo33D/et7NKEpLRmvzo=", "owner": "ipetkov", "repo": "crane", - "rev": "7e4586bad4e3f8f97a9271def747cf58c4b68f3c", + "rev": "f235b656ee5b2bfd6d94c3bfd67896a575d4a6ed", "type": "github" }, "original": { @@ -140,7 +178,7 @@ "cachix", "flake-compat" ], - "nix": "nix_2", + "nix": "nix_3", "nixpkgs": [ "cachix", "nixpkgs" @@ -150,6 +188,43 @@ "git-hooks" ] }, + "locked": { + "lastModified": 1727963652, + "narHash": "sha256-os0EDjn7QVXL6RtHNb9TrZLXVm2Tc5/nZKk3KpbTzd8=", + "owner": "cachix", + "repo": "devenv", + "rev": "cb0052e25dbcc8267b3026160dc73cddaac7d5fd", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "devenv", + "type": "github" + } + }, + "devenv_2": { + "inputs": { + "cachix": "cachix_3", + "flake-compat": [ + "cachix", + "devenv", + "cachix", + "flake-compat" + ], + "nix": "nix_2", + "nixpkgs": [ + "cachix", + "devenv", + "cachix", + "nixpkgs" + ], + "pre-commit-hooks": [ + "cachix", + "devenv", + "cachix", + "git-hooks" + ] + }, "locked": { "lastModified": 1723156315, "narHash": "sha256-0JrfahRMJ37Rf1i0iOOn+8Z4CLvbcGNwa2ChOAVrp/8=", @@ -164,9 +239,11 @@ "type": "github" } }, - "devenv_2": { + "devenv_3": { "inputs": { "flake-compat": [ + "cachix", + "devenv", "cachix", "devenv", "cachix", @@ -176,6 +253,8 @@ "nixpkgs": "nixpkgs_2", "poetry2nix": "poetry2nix", "pre-commit-hooks": [ + "cachix", + "devenv", "cachix", "devenv", "cachix", @@ -205,11 +284,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1725690497, - "narHash": "sha256-5fT+96rV7Hx29HG+4/oBbr3V+yExKuLN2vcBcPbVBlU=", + "lastModified": 1729751566, + "narHash": "sha256-99u/hrgBdi8bxSXZc9ZbNkR5EL1htrkbd3lsbKzS60g=", "owner": "nix-community", "repo": "fenix", - "rev": "4b8d964df93d1f918ee6c4f003b3548c432cc866", + "rev": "f32a2d484091a6dc98220b1f4a2c2d60b7c97c64", "type": "github" }, "original": { @@ -305,27 +384,32 @@ "type": "github" } }, - "flake-utils": { + "flake-parts_2": { "inputs": { - "systems": "systems" + "nixpkgs-lib": [ + "cachix", + "devenv", + "nix", + "nixpkgs" + ] }, "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "lastModified": 1712014858, + "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", "type": "github" }, "original": { - "owner": "numtide", - "repo": "flake-utils", + "owner": "hercules-ci", + "repo": "flake-parts", "type": "github" } }, - "flake-utils_2": { + "flake-utils": { "inputs": { - "systems": "systems_2" + "systems": "systems" }, "locked": { "lastModified": 1689068808, @@ -341,9 +425,24 @@ "type": "github" } }, + "flake-utils_2": { + "locked": { + "lastModified": 1667395993, + "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, "flake-utils_3": { "inputs": { - "systems": "systems_3" + "systems": "systems_2" }, "locked": { "lastModified": 1710146030, @@ -374,11 +473,11 @@ "nixpkgs-stable": "nixpkgs-stable_2" }, "locked": { - "lastModified": 1723202784, - "narHash": "sha256-qbhjc/NEGaDbyy0ucycubq4N3//gDFFH3DOmp1D3u1Q=", + "lastModified": 1727854478, + "narHash": "sha256-/odH2nUMAwkMgOS2nG2z0exLQNJS4S2LfMW0teqU7co=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "c7012d0c18567c889b948781bc74a501e92275d1", + "rev": "5f58871c9657b5fc0a7f65670fe2ba99c26c1d79", "type": "github" }, "original": { @@ -409,6 +508,22 @@ "type": "github" } }, + "libgit2": { + "flake": false, + "locked": { + "lastModified": 1697646580, + "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "owner": "libgit2", + "repo": "libgit2", + "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "type": "github" + }, + "original": { + "owner": "libgit2", + "repo": "libgit2", + "type": "github" + } + }, "liburing": { "flake": false, "locked": { @@ -434,6 +549,8 @@ "devenv", "cachix", "devenv", + "cachix", + "devenv", "nixpkgs" ], "nixpkgs-regression": "nixpkgs-regression" @@ -476,6 +593,8 @@ "devenv", "cachix", "devenv", + "cachix", + "devenv", "poetry2nix", "nixpkgs" ] @@ -497,11 +616,15 @@ "nix_2": { "inputs": { "flake-compat": [ + "cachix", + "devenv", "cachix", "devenv", "flake-compat" ], "nixpkgs": [ + "cachix", + "devenv", "cachix", "devenv", "nixpkgs" @@ -523,13 +646,42 @@ "type": "github" } }, + "nix_3": { + "inputs": { + "flake-compat": [ + "cachix", + "devenv", + "flake-compat" + ], + "flake-parts": "flake-parts_2", + "libgit2": "libgit2", + "nixpkgs": "nixpkgs_3", + "nixpkgs-23-11": "nixpkgs-23-11", + "nixpkgs-regression": "nixpkgs-regression_3", + "pre-commit-hooks": "pre-commit-hooks" + }, + "locked": { + "lastModified": 1727438425, + "narHash": "sha256-X8ES7I1cfNhR9oKp06F6ir4Np70WGZU5sfCOuNBEwMg=", + "owner": "domenkozar", + "repo": "nix", + "rev": "f6c5ae4c1b2e411e6b1e6a8181cc84363d6a7546", + "type": "github" + }, + "original": { + "owner": "domenkozar", + "ref": "devenv-2.24", + "repo": "nix", + "type": "github" + } + }, "nixpkgs": { "locked": { - "lastModified": 1724999960, - "narHash": "sha256-LB3jqSGW5u1ZcUcX6vO/qBOq5oXHlmOCxsTXGMEitp4=", + "lastModified": 1726042813, + "narHash": "sha256-LnNKCCxnwgF+575y0pxUdlGZBO/ru1CtGHIqQVfvjlA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b96f849e725333eb2b1c7f1cb84ff102062468ba", + "rev": "159be5db480d1df880a0135ca0bfed84c2f88353", "type": "github" }, "original": { @@ -539,6 +691,22 @@ "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -571,6 +739,22 @@ "type": "github" } }, + "nixpkgs-regression_3": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, "nixpkgs-stable": { "locked": { "lastModified": 1724316499, @@ -621,11 +805,27 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1722813957, - "narHash": "sha256-IAoYyYnED7P8zrBFMnmp7ydaJfwTnwcnqxUElC1I26Y=", + "lastModified": 1717432640, + "narHash": "sha256-+f9c4/ZX5MWDOuB1rKoWj+lBNm0z0rs4CK47HBLxy1o=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "cb9a96f23c491c081b38eab96d22fa958043c9fa", + "rev": "88269ab3044128b7c2f4c7d68448b2fb50456870", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "release-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_4": { + "locked": { + "lastModified": 1727802920, + "narHash": "sha256-HP89HZOT0ReIbI7IJZJQoJgxvB2Tn28V6XS3MNKnfLs=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "27e30d177e57d912d614c88c622dcfdb2e6e6515", "type": "github" }, "original": { @@ -635,7 +835,7 @@ "type": "github" } }, - "nixpkgs_4": { + "nixpkgs_5": { "locked": { "lastModified": 1725534445, "narHash": "sha256-Yd0FK9SkWy+ZPuNqUgmVPXokxDgMJoGuNpMEtkfcf84=", @@ -653,13 +853,15 @@ }, "poetry2nix": { "inputs": { - "flake-utils": "flake-utils_2", + "flake-utils": "flake-utils", "nix-github-actions": "nix-github-actions", "nixpkgs": [ "cachix", "devenv", "cachix", "devenv", + "cachix", + "devenv", "nixpkgs" ] }, @@ -677,19 +879,59 @@ "type": "github" } }, + "pre-commit-hooks": { + "inputs": { + "flake-compat": [ + "cachix", + "devenv", + "nix" + ], + "flake-utils": "flake-utils_2", + "gitignore": [ + "cachix", + "devenv", + "nix" + ], + "nixpkgs": [ + "cachix", + "devenv", + "nix", + "nixpkgs" + ], + "nixpkgs-stable": [ + "cachix", + "devenv", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1712897695, + "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=", + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "pre-commit-hooks.nix", + "type": "github" + } + }, "rocksdb": { "flake": false, "locked": { - "lastModified": 1725811807, - "narHash": "sha256-k6aubvLSJwW7CtIxBRSlM6Z8UiJDCdFUgDWPYLV47Qk=", + "lastModified": 1729712930, + "narHash": "sha256-jlp4kPkRTpoJaUdobEoHd8rCGAQNBy4ZHZ6y5zL/ibw=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "347d50e212b24d98b5ad9841404ff80c1bb873f0", + "rev": "871eda6953c3f399aae39808dcfccdd014885beb", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.6.1", + "ref": "v9.7.3", "repo": "rocksdb", "type": "github" } @@ -705,18 +947,18 @@ "flake-utils": "flake-utils_3", "liburing": "liburing", "nix-filter": "nix-filter", - "nixpkgs": "nixpkgs_4", + "nixpkgs": "nixpkgs_5", "rocksdb": "rocksdb" } }, "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1725630423, - "narHash": "sha256-gNCLk3Zg7JlAwmWbVHTH6f3+iqdeQ4fheOotCZy8x5M=", + "lastModified": 1729715509, + "narHash": "sha256-jUDN4e1kObbksb4sc+57NEeujBEDRdLCOu9wiE3RZdM=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "08c7bbc2dbe4dcc8968484f1a0e1e6fe7a1d4f6d", + "rev": "40492e15d49b89cf409e2c5536444131fac49429", "type": "github" }, "original": { @@ -755,21 +997,6 @@ "repo": "default", "type": "github" } - }, - "systems_3": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 74d15408..7c870cce 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.6.1"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.7.3"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; @@ -27,7 +27,7 @@ file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` - sha256 = "sha256-VZZnlyP69+Y3crrLHQyJirqlHrTtGTsyiSnZB8jEvVo="; + sha256 = "sha256-yMuSb5eQPO/bHv+Bcf/US8LVMbf/G/0MSfiPwBhiPpk="; }; mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { @@ -41,7 +41,7 @@ liburing = pkgs.liburing.overrideAttrs { # Tests weren't building outputs = [ "out" "dev" "man" ]; - buildFlags = [ "library"]; + buildFlags = [ "library" ]; src = inputs.liburing; }; rocksdb = (pkgs.rocksdb.override { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 892d9442..086f7ba0 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -2,8 +2,6 @@ # # Other files that need upkeep when this changes: # -# * `.gitlab-ci.yml` -# * `.github/workflows/ci.yml` # * `Cargo.toml` # * `flake.nix` # @@ -11,13 +9,20 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.81.0" +channel = "1.82.0" +profile = "minimal" components = [ # For rust-analyzer "rust-src", + "rust-analyzer", + # For CI and editors + "rustfmt", + "clippy", ] targets = [ + #"x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl", + #"aarch64-apple-darwin", ] diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 442fe266..923f077a 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -119,6 +119,7 @@ pub(crate) async fn create_room_route( None => services.globals.default_room_version(), }; + #[allow(clippy::single_match_else)] let content = match &body.creation_content { Some(content) => { use RoomVersionId::*; diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index fe98e458..670f72ba 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -236,6 +236,7 @@ async fn auth_server( } } + #[allow(clippy::or_fun_call)] let signature_uri = CanonicalJsonValue::String( request .parts diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index cc5865c8..5e28aab0 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -52,10 +52,6 @@ zstd_compression = [ perf_measurements = [] sentry_telemetry = [] -# these do nothing, these are purely for informing users to update their build scripts if they use one -rocksdb = [] -sha256_media = [] - [dependencies] argon2.workspace = true arrayvec.workspace = true diff --git a/src/core/config/check.rs b/src/core/config/check.rs index b36b9c5e..8dea55d8 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -9,22 +9,6 @@ pub fn check(config: &Config) -> Result<()> { info!("Note: conduwuit was built without optimisations (i.e. debug build)"); } - // prevents catching this in `--all-features` - if cfg!(all(feature = "rocksdb", not(feature = "sha256_media"))) { - warn!( - "Note the rocksdb feature was deleted from conduwuit. SQLite support was removed and RocksDB is the only \ - supported backend now. Please update your build script to remove this feature." - ); - } - - // prevents catching this in `--all-features` - if cfg!(all(feature = "sha256_media", not(feature = "rocksdb"))) { - warn!( - "Note the sha256_media feature was deleted from conduwuit, it is now fully integrated in a \ - forwards-compatible way. Please update your build script to remove this feature." - ); - } - warn_deprecated(config); warn_unknown_key(config); diff --git a/src/core/debug.rs b/src/core/debug.rs index f6c992b9..844445d5 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -61,7 +61,7 @@ pub fn set_panic_trap() { #[inline(always)] #[allow(deprecated_in_future)] -fn panic_handler(info: &panic::PanicInfo<'_>, next: &dyn Fn(&panic::PanicInfo<'_>)) { +fn panic_handler(info: &panic::PanicHookInfo<'_>, next: &dyn Fn(&panic::PanicHookInfo<'_>)) { trap(); next(info); } diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index fae39605..b090618c 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -42,6 +42,7 @@ default = [ "gzip_compression", "io_uring", "jemalloc", + "jemalloc_stats", "release_max_log_level", "sentry_telemetry", "systemd", diff --git a/src/service/rooms/short/data.rs b/src/service/rooms/short/data.rs index 963bd927..17fbb64e 100644 --- a/src/service/rooms/short/data.rs +++ b/src/service/rooms/short/data.rs @@ -63,6 +63,7 @@ impl Data { .iter() .enumerate() { + #[allow(clippy::single_match_else)] match short { Some(short) => ret.push( utils::u64_from_bytes(short).map_err(|_| Error::bad_database("Invalid shorteventid in db."))?, diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 9fbc30ea..7354b73d 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -479,6 +479,8 @@ impl Service { .collect::>(), ))) .into(), + ephemeral: Vec::new(), + to_device: Vec::new(), }, ) .await From 45254638b1be280cef46a359a6a263ce7ff226a5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 28 Sep 2024 17:38:37 -0400 Subject: [PATCH 0811/2091] drop target-cpu optimised builds this seems too broken. Signed-off-by: strawberry --- nix/pkgs/main/cross-compilation-env.nix | 6 ------ nix/pkgs/main/default.nix | 24 +----------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index 3f4d593b..83fe6ed6 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -13,12 +13,6 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { lib.concatStringsSep " " ([] - ++ lib.optionals - stdenv.targetPlatform.isx86_64 - [ "-C" "target-cpu=x86-64-v2" ] - ++ lib.optionals - stdenv.targetPlatform.isAarch64 - [ "-C" "target-cpu=cortex-a53" ] # cortex-a53 == ARMv8-A # This disables PIE for static builds, which isn't great in terms # of security. Unfortunately, my hand is forced because nixpkgs' # `libstdc++.a` is built without `-fPIE`, which precludes us from diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 31a2b7aa..8559e878 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -83,24 +83,6 @@ buildDepsOnlyEnv = meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin; enableLiburing = enableLiburing; - - sse42Support = stdenv.targetPlatform.isx86_64; - - cmakeFlags = if stdenv.targetPlatform.isx86_64 - then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags - ++ lib.optionals stdenv.targetPlatform.isx86_64 [ - "-DPORTABLE=x86-64-v2" - "-DUSE_SSE=1" - "-DHAVE_SSE=1" - "-DHAVE_SSE42=1" - ] - else if stdenv.targetPlatform.isAarch64 - then lib.subtractLists [ "-DPORTABLE=1" ] old.cmakeFlags - ++ lib.optionals stdenv.targetPlatform.isAarch64 [ - # cortex-a53 == ARMv8-A - "-DPORTABLE=armv8-a" - ] - else old.cmakeFlags; }); in { @@ -127,11 +109,7 @@ buildPackageEnv = { # Only needed in static stdenv because these are transitive dependencies of rocksdb CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic) - " -L${lib.getLib liburing}/lib -luring" - + lib.optionalString stdenv.targetPlatform.isx86_64 - " -Ctarget-cpu=x86-64-v2" - + lib.optionalString stdenv.targetPlatform.isAarch64 - " -Ctarget-cpu=cortex-a53"; # cortex-a53 == ARMv8-A + " -L${lib.getLib liburing}/lib -luring"; }; From e38c37d9e78e254411f1df9f91fd8b2596503004 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 17 Sep 2024 18:04:59 -0400 Subject: [PATCH 0812/2091] allow users to respond to polls by default (org.matrix.msc3381.poll.response) Signed-off-by: strawberry --- src/api/client/room.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 923f077a..86e14f76 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -821,6 +821,12 @@ fn default_power_levels_content( power_levels_content["events"]["m.room.history_visibility"] = serde_json::to_value(100).expect("100 is valid Value"); + // always allow users to respond (not post new) to polls. this is primarily + // useful in read-only announcement rooms that post a public poll. + power_levels_content["events"]["org.matrix.msc3381.poll.response"] = + serde_json::to_value(0).expect("0 is valid Value"); + power_levels_content["events"]["m.poll.response"] = serde_json::to_value(0).expect("0 is valid Value"); + // synapse does this too. clients do not expose these permissions. it prevents // default users from calling public rooms, for obvious reasons. if *visibility == room::Visibility::Public { From 6acdd0d947295dedf2d0e650e3bc7baf0617c1df Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 17 Sep 2024 18:55:08 -0400 Subject: [PATCH 0813/2091] improve some general documentation Signed-off-by: strawberry --- README.md | 37 ++++++++++++++++++++++----- docs/deploying/generic.md | 54 ++++++++++++++++++++++++++++++++++----- docs/turn.md | 16 ++++++++++++ 3 files changed, 94 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index cca299c9..d4d76e6c 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/bad -Visit the [Conduwuit documentation](https://conduwuit.puppyirl.gay/) for more +Visit the [conduwuit documentation](https://conduwuit.puppyirl.gay/) for more information. @@ -23,9 +23,9 @@ to communicate with users outside of Matrix, like a community on Discord. #### What is the goal? -An efficient Matrix homeserver that's easy to set up and just works. You can -install it on a mini-computer like the Raspberry Pi to host Matrix for your -family, friends or company. +A high-performance and efficient Matrix homeserver that's easy to set up and +just works. You can install it on a mini-computer like the Raspberry Pi to +host Matrix for your family, friends or company. #### Can I try it out? @@ -42,9 +42,28 @@ transfem.dev is also listed at #### What is the current status? -conduwuit is a hard fork of Conduit which is in beta, meaning you can join and -participate in most Matrix rooms, but not all features are supported and you -might run into bugs from time to time. +conduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status +initially was inherited from Conduit, however overtime this Beta status is rapidly +becoming less and less relevant as our codebase significantly diverges more and more. + +conduwuit is quite stable and very usable as a daily driver and for a low-medium +sized homeserver. There is still a lot of more work to be done, but it is in a far +better place than the project was in early 2024. + +#### How is conduwuit funded? Is conduwuit sustainable? + +conduwuit has no external funding. This is made possible purely in my freetime with +contributors, also in their free time, and only by user-curated donations. + +conduwuit has existed since around November 2023, but [only became more publicly known +in March/April 2024](https://matrix.org/blog/2024/04/26/this-week-in-matrix-2024-04-26/#conduwuit-website) +and we have no plans in stopping or slowing down any time soon! + +#### Can I migrate or switch from Conduit? + +conduwuit is a complete drop-in replacement for Conduit. As long as you are using RocksDB, +the only "migration" you need to do is replace the binary or container image. There +is no harm or additional steps required for using conduwuit. @@ -59,6 +78,10 @@ If you run into any question, feel free to #### Donate +conduwuit development is purely made possible by myself and contributors. I do +not get paid to work on this, and I work on it in my free time. Donations are +heavily appreciated! 💜🥺 + - Liberapay: - Ko-fi: - GitHub Sponsors: diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 6c478d19..562ebd7e 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -13,18 +13,38 @@ what you need. Prebuilt fully static musl binaries can be downloaded from the latest tagged release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or -`main` CI branch workflow artifact output. These also include Debian packages. +`main` CI branch workflow artifact output. These also include Debian/Ubuntu packages. + These binaries have jemalloc and io_uring statically linked and included with -them. +them, so no additional dynamic dependencies need to be installed. Alternatively, you may compile the binary yourself. We recommend using -Nix (or [Lix](https://lix.systems) to build conduwuit as this has the most guaranteed -reproducibiltiy and easiest to get a build environment and output going. +Nix (or [Lix](https://lix.systems)) to build conduwuit as this has the most guaranteed +reproducibiltiy and easiest to get a build environment and output going. This also +allows easy cross-compilation. + +You can run the `nix build -L .#static-x86_64-unknown-linux-musl-all-features` or +`nix build -L .#static-aarch64-unknown-linux-musl-all-features` commands based +on architecture to cross-compile the necessary static binary located at +`result/bin/conduit`. This is reproducible with the static binaries produced in our CI. Otherwise, follow standard Rust project build guides (installing git and cloning the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + libclang for RocksDB, installing liburing for io_uring and RocksDB, etc). +## Migrating from Conduit + +As mentioned in the README, there is little to no steps needed to migrate +from Conduit. As long as you are using the RocksDB database backend, just +replace the binary / container image / etc. + +**Note**: If you are relying on Conduit's "automatic delegation" feature, +this will **NOT** work on conduwuit and you must configure delegation manually. +This is not a mistake and no support for this feature will be added. + +See the `[global.well_known]` config section, or configure your web server +appropriately to send the delegation responses. + ## Adding a conduwuit user While conduwuit can run as any user it is better to use dedicated users for @@ -91,13 +111,32 @@ sudo chmod 700 /var/lib/conduwuit/ ## Setting up the Reverse Proxy Refer to the documentation or various guides online of your chosen reverse proxy -software. A [Caddy](https://caddyserver.com/) example will be provided as this +software. There are many examples of basic Apache/Nginx reverse proxy setups +out there. + +A [Caddy](https://caddyserver.com/) example will be provided as this is the recommended reverse proxy for new users and is very trivial to use (handles TLS, reverse proxy headers, etc transparently with proper defaults). Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization header, making federation non-functional. If using Apache, you need to use -`nocanon` to prevent this. +`nocanon` in your `ProxyPass` directive to prevent this (note that Apache +isn't very good as a general reverse proxy). + +Nginx users may need to set `proxy_buffering off;` if there are issues with +uploading media like images. + +You will need to reverse proxy everything under following routes: +- `/_matrix/` - core Matrix C-S and S-S APIs +- `/_conduwuit/` - ad-hoc conduwuit routes such as `/local_user_count` and +`/server_version` + +You can optionally reverse proxy the following individual routes: +- `/.well-known/matrix/client` and `/.well-known/matrix/server` if using +conduwuit to perform delegation +- `/.well-known/matrix/support` if using conduwuit to send the homeserver admin +contact and support page (formerly known as MSC1929) +- `/` if you would like to see `hewwo from conduwuit woof!` at the root ### Caddy @@ -146,6 +185,9 @@ curl https://your.server.name/_conduwuit/server_version # If using port 8448 curl https://your.server.name:8448/_conduwuit/server_version + +# If federation is enabled +curl https://your.server.name:8448/_matrix/federation/v1/version ``` - To check if your server can talk with other homeservers, you can use the diff --git a/docs/turn.md b/docs/turn.md index d54398ae..12e7e3ae 100644 --- a/docs/turn.md +++ b/docs/turn.md @@ -21,6 +21,22 @@ These same values need to be set in conduwuit. See the [example config](configuration/examples.md) in the TURN section for configuring these and restart conduwuit after. +`turn_secret` must be set to your coturn `static-auth-secret`, or use +`turn_username` and `turn_password` if using legacy username:password +TURN authentication (not preferred). + +`turn_uris` must be the list of TURN URIs you would like to send to the client. +Typically you will just replace the example domain `example.turn.uri` with the +`realm` you set from the example config. + +If you are using TURN over TLS, you can replace `turn:` with `turns:` in the +`turn_uris` config option to instruct clients to attempt to connect to +TURN over TLS. This is highly recommended. + +If you need unauthenticated access to the TURN URIs, or some clients may be +having trouble, you can enable `turn_guest_access` in conduwuit which disables +authentication for the TURN URI endpoint `/_matrix/client/v3/voip/turnServer` + ### Run Run the [Coturn](https://hub.docker.com/r/coturn/coturn) image using From 73afc1fd8f40ce9f546d7c7d097b54b1fc14ed82 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 20 Sep 2024 16:22:29 -0400 Subject: [PATCH 0814/2091] allow taking multiple `--config` arguments to "include"/merge more config files Signed-off-by: strawberry --- src/core/config/mod.rs | 31 ++++++++++++++----------------- src/main/clap.rs | 2 +- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 47ed0d60..9b5405b9 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -426,29 +426,26 @@ const DEPRECATED_KEYS: &[&str; 9] = &[ impl Config { /// Pre-initialize config - pub fn load(path: &Option) -> Result { + pub fn load(paths: &Option>) -> Result { let raw_config = if let Some(config_file_env) = Env::var("CONDUIT_CONFIG") { - Figment::new() - .merge(Toml::file(config_file_env).nested()) - .merge(Env::prefixed("CONDUIT_").global().split("__")) - .merge(Env::prefixed("CONDUWUIT_").global().split("__")) + Figment::new().merge(Toml::file(config_file_env).nested()) } else if let Some(config_file_arg) = Env::var("CONDUWUIT_CONFIG") { - Figment::new() - .merge(Toml::file(config_file_arg).nested()) - .merge(Env::prefixed("CONDUIT_").global().split("__")) - .merge(Env::prefixed("CONDUWUIT_").global().split("__")) - } else if let Some(config_file_arg) = path { - Figment::new() - .merge(Toml::file(config_file_arg).nested()) - .merge(Env::prefixed("CONDUIT_").global().split("__")) - .merge(Env::prefixed("CONDUWUIT_").global().split("__")) + Figment::new().merge(Toml::file(config_file_arg).nested()) + } else if let Some(config_file_args) = paths { + let mut figment = Figment::new(); + + for config in config_file_args { + figment = figment.merge(Toml::file(config).nested()); + } + + figment } else { Figment::new() - .merge(Env::prefixed("CONDUIT_").global().split("__")) - .merge(Env::prefixed("CONDUWUIT_").global().split("__")) }; - Ok(raw_config) + Ok(raw_config + .merge(Env::prefixed("CONDUIT_").global().split("__")) + .merge(Env::prefixed("CONDUWUIT_").global().split("__"))) } /// Finalize config diff --git a/src/main/clap.rs b/src/main/clap.rs index 3af0be02..86b9fbd6 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -14,7 +14,7 @@ use conduit::{ pub(crate) struct Args { #[arg(short, long)] /// Path to the config TOML file (optional) - pub(crate) config: Option, + pub(crate) config: Option>, /// Override a configuration variable using TOML 'key=value' syntax #[arg(long, short('O'))] From e0b2595905c293e9a4340f53940b03f00fe27612 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 20 Sep 2024 22:57:04 -0400 Subject: [PATCH 0815/2091] support reading TURN secret from a file (turn_secret_file) Signed-off-by: strawberry --- conduwuit-example.toml | 11 +++++++++++ docs/turn.md | 6 +++--- src/api/client/voip.rs | 2 +- src/core/config/mod.rs | 8 +++++++- src/service/globals/mod.rs | 15 +++++++++++++-- 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 7cf9716d..2792f792 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -845,9 +845,20 @@ allow_profile_lookup_federation_requests = true # vector list of TURN URIs/servers to use # +# replace "example.turn.uri" with your TURN domain, such as the coturn "realm". +# if using TURN over TLS, replace "turn:" with "turns:" +# # No default #turn_uris = ["turn:example.turn.uri?transport=udp", "turn:example.turn.uri?transport=tcp"] +# TURN secret to use that's read from the file path specified +# +# this takes priority over "turn_secret" first, and falls back to "turn_secret" if invalid or +# failed to open. +# +# no default +#turn_secret_file = "/path/to/secret.txt" + # TURN secret to use for generating the HMAC-SHA1 hash apart of username and password generation # # this is more secure, but if needed you can use traditional username/password below. diff --git a/docs/turn.md b/docs/turn.md index 12e7e3ae..287f2545 100644 --- a/docs/turn.md +++ b/docs/turn.md @@ -21,9 +21,9 @@ These same values need to be set in conduwuit. See the [example config](configuration/examples.md) in the TURN section for configuring these and restart conduwuit after. -`turn_secret` must be set to your coturn `static-auth-secret`, or use -`turn_username` and `turn_password` if using legacy username:password -TURN authentication (not preferred). +`turn_secret` or a path to `turn_secret_file` must have a value of your +coturn `static-auth-secret`, or use `turn_username` and `turn_password` +if using legacy username:password TURN authentication (not preferred). `turn_uris` must be the list of TURN URIs you would like to send to the client. Typically you will just replace the example domain `example.turn.uri` with the diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index a51e2657..7ee4571f 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -24,7 +24,7 @@ pub(crate) async fn turn_server_route( return Err!(Request(NotFound("Not Found"))); } - let turn_secret = services.globals.turn_secret().clone(); + let turn_secret = services.globals.turn_secret.clone(); let (username, password) = if !turn_secret.is_empty() { let expiry = SecondsSinceUnixEpoch::from_system_time( diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 9b5405b9..f48c78c9 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -194,6 +194,7 @@ pub struct Config { pub turn_uris: Vec, #[serde(default)] pub turn_secret: String, + pub turn_secret_file: Option, #[serde(default = "default_turn_ttl")] pub turn_ttl: u64, @@ -681,12 +682,17 @@ impl fmt::Display for Config { } }); line("TURN secret", { - if self.turn_secret.is_empty() { + if self.turn_secret.is_empty() && self.turn_secret_file.is_none() { "not set" } else { "set" } }); + line("TURN secret file path", { + self.turn_secret_file + .as_ref() + .map_or("", |path| path.to_str().unwrap_or_default()) + }); line("Turn TTL", &self.turn_ttl.to_string()); line("Turn URIs", { let mut lst = vec![]; diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 25e0f402..a044d30a 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -40,6 +40,7 @@ pub struct Service { pub stateres_mutex: Arc>, pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, + pub turn_secret: String, } type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries @@ -84,6 +85,17 @@ impl crate::Service for Service { .collect::>() .map_err(|e| err!(Config("ip_range_denylist", e)))?; + let turn_secret = config + .turn_secret_file + .as_ref() + .map_or(config.turn_secret.clone(), |path| { + std::fs::read_to_string(path).unwrap_or_else(|e| { + error!("Failed to read the TURN secret file: {e}"); + + config.turn_secret.clone() + }) + }); + let mut s = Self { db, config: config.clone(), @@ -99,6 +111,7 @@ impl crate::Service for Service { .expect("#admins:server_name is valid alias name"), server_user: UserId::parse_with_server_name(String::from("conduit"), &config.server_name) .expect("@conduit:server_name is valid"), + turn_secret, }; if !s @@ -207,8 +220,6 @@ impl Service { pub fn turn_username(&self) -> &String { &self.config.turn_username } - pub fn turn_secret(&self) -> &String { &self.config.turn_secret } - pub fn allow_profile_lookup_federation_requests(&self) -> bool { self.config.allow_profile_lookup_federation_requests } From 2eee454a183d3390e335bbb5a01f16847b10a04c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 28 Sep 2024 21:55:02 -0400 Subject: [PATCH 0816/2091] docs: nixos and unix socket fail, jemalloc and hardened.nix Signed-off-by: strawberry --- docs/deploying/nixos.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index cbf56ae5..9147db7f 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -8,6 +8,8 @@ conduwuit can be acquired by Nix (or [Lix][lix]) from various places: A community maintained NixOS package is available at [`conduwuit`](https://search.nixos.org/packages?channel=unstable&show=conduwuit&from=0&size=50&sort=relevance&type=packages&query=conduwuit) +### Binary cache + A binary cache for conduwuit that the CI/CD publishes to is available at the following places (both are the same just different names): @@ -31,6 +33,8 @@ conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= If specifying a Git remote URL in your flake, you can use any remotes that are specified on the README (the mirrors), such as the GitHub: `github:girlbossceo/conduwuit` +### NixOS module + The `flake.nix` and `default.nix` do not currently provide a NixOS module (contributions welcome!), so [`services.matrix-conduit`][module] from Nixpkgs can be used to configure conduwuit. @@ -39,7 +43,35 @@ If you want to run the latest code, you should get conduwuit from the `flake.nix or `default.nix` and set [`services.matrix-conduit.package`][package] appropriately to use conduwuit instead of Conduit. +### UNIX sockets + +Due to the lack of a conduwuit NixOS module, when using the `services.matrix-conduit` module +it is not possible to use UNIX sockets. This is because the UNIX socket option does not exist +in Conduit, and their module forces listening on `[::1]:6167` by default if unspecified. + +Additionally, the [`matrix-conduit` systemd unit][systemd-unit] in the module does not allow +the `AF_UNIX` socket address family in their systemd unit's `RestrictAddressFamilies=` which +disallows the namespace from accessing or creating UNIX sockets. + +There is no known workaround these. A conduwuit NixOS configuration module must be developed and +published by the community. + +### jemalloc and hardened profile + +conduwuit uses jemalloc by default. This may interfere with the [`hardened.nix` profile][hardened.nix] +due to them using `scudo` by default. You must either disable/hide `scudo` from conduwuit, or +disable jemalloc like so: + +```nix +let + conduwuit = pkgs.unstable.conduwuit.override { + enableJemalloc = false; + }; +in +``` [lix]: https://lix.systems/ [module]: https://search.nixos.org/options?channel=unstable&query=services.matrix-conduit [package]: https://search.nixos.org/options?channel=unstable&query=services.matrix-conduit.package +[hardened.nix]: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/profiles/hardened.nix#L22 +[systemd-unit]: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/services/matrix/conduit.nix#L132 From acb9eae707046d89a1cb180c01c0136240ffc926 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 29 Sep 2024 03:09:20 -0400 Subject: [PATCH 0817/2091] add back server name to error sending PDU to remote server Signed-off-by: strawberry --- src/service/sending/sender.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 7354b73d..206bf92b 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -638,7 +638,9 @@ impl Service { .pdus .iter() .filter(|(_, res)| res.is_err()) - .for_each(|(pdu_id, res)| warn!("error for {pdu_id} from remote: {res:?}")); + .for_each( + |(pdu_id, res)| warn!(%transaction_id, %server, "error sending PDU {pdu_id} to remote server: {res:?}"), + ); }) .map(|_| dest.clone()) .map_err(|e| (dest.clone(), e)) From 83becf013c00e233cebb385ee0e1fa365b85e2e5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 29 Sep 2024 12:27:42 -0400 Subject: [PATCH 0818/2091] add config option to disable ANSI log colours Signed-off-by: strawberry --- conduwuit-example.toml | 5 +++++ src/core/config/mod.rs | 2 ++ src/main/tracing.rs | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 2792f792..6142705d 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -363,6 +363,11 @@ allow_profile_lookup_federation_requests = true # Defaults to "info" #log = "info" +# controls whether logs will be outputted with ANSI colours +# +# defaults to true +#log_colors = true + # controls whether encrypted rooms and events are allowed (default true) #allow_encryption = false diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index f48c78c9..bb520d06 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -184,6 +184,8 @@ pub struct Config { pub query_trusted_key_servers_first: bool, #[serde(default = "default_log")] pub log: String, + #[serde(default = "true_fn", alias = "log_colours")] + pub log_colors: bool, #[serde(default = "default_openid_token_ttl")] pub openid_token_ttl: u64, #[serde(default)] diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 0217f38a..9b4ad659 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -18,7 +18,7 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla let reload_handles = LogLevelReloadHandles::default(); let console_filter = EnvFilter::try_new(&config.log).map_err(|e| err!(Config("log", "{e}.")))?; - let console_layer = tracing_subscriber::fmt::Layer::new(); + let console_layer = tracing_subscriber::fmt::Layer::new().with_ansi(config.log_colors); let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); reload_handles.add("console", Box::new(console_reload_handle)); From a7c4a7933d290c533659e227c4eefa9ee25a9530 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 29 Sep 2024 13:02:18 -0400 Subject: [PATCH 0819/2091] disable log colours in the complement config Signed-off-by: strawberry --- nix/pkgs/complement/config.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index 08c538bb..7aaed2f4 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -16,6 +16,7 @@ url_preview_domain_contains_allowlist = ["*"] media_compat_file_link = false media_startup_check = false rocksdb_direct_io = false +log_colors = false [global.tls] certs = "/certificate.crt" From 87734a074fe6813f48e2846ca9454c11b9498d11 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 2 Oct 2024 20:45:15 -0400 Subject: [PATCH 0820/2091] add m.call and m.call.member to list of permissions to set on public rooms Signed-off-by: strawberry --- src/api/client/room.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 86e14f76..609f3b40 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -831,6 +831,8 @@ fn default_power_levels_content( // default users from calling public rooms, for obvious reasons. if *visibility == room::Visibility::Public { power_levels_content["events"]["m.call.invite"] = serde_json::to_value(50).expect("50 is valid Value"); + power_levels_content["events"]["m.call"] = serde_json::to_value(50).expect("50 is valid Value"); + power_levels_content["events"]["m.call.member"] = serde_json::to_value(50).expect("50 is valid Value"); power_levels_content["events"]["org.matrix.msc3401.call"] = serde_json::to_value(50).expect("50 is valid Value"); power_levels_content["events"]["org.matrix.msc3401.call.member"] = From e5efd5583855f6eca5b1550b61ac4489304b3303 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 10 Oct 2024 16:23:38 -0400 Subject: [PATCH 0821/2091] feature-gate direct TLS mode to make rustls/aws-lc-rs optional Signed-off-by: strawberry --- Cargo.lock | 13 ++++++------- Cargo.toml | 1 - src/core/Cargo.toml | 1 - src/main/Cargo.toml | 3 +++ src/router/Cargo.toml | 8 ++++++++ src/router/serve/mod.rs | 10 +++++++++- src/router/serve/tls.rs | 4 +++- 7 files changed, 29 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1103dc8a..6386f968 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,9 +43,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1" +checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" [[package]] name = "anyhow" @@ -703,7 +703,6 @@ dependencies = [ "reqwest", "ring", "ruma", - "rustls 0.23.15", "sanitize-filename", "serde", "serde_json", @@ -813,9 +812,9 @@ dependencies = [ [[package]] name = "console-api" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ed14aa9c9f927213c6e4f3ef75faaad3406134efe84ba2cb7983431d5f0931" +checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" dependencies = [ "futures-core", "prost", @@ -826,9 +825,9 @@ dependencies = [ [[package]] name = "console-subscriber" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e3a111a37f3333946ebf9da370ba5c5577b18eb342ec683eb488dd21980302" +checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01" dependencies = [ "console-api", "crossbeam-channel", diff --git a/Cargo.toml b/Cargo.toml index fd124512..b75c4975 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,7 +101,6 @@ features = ["typed-header", "tracing"] [workspace.dependencies.axum-server] version = "0.7.1" default-features = false -features = ["tls-rustls"] # to listen on both HTTP and HTTPS if listening on TLS dierctly from conduwuit for complement or sytest [workspace.dependencies.axum-server-dual-protocol] diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 5e28aab0..71364734 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -79,7 +79,6 @@ regex.workspace = true reqwest.workspace = true ring.workspace = true ruma.workspace = true -rustls.workspace = true sanitize-filename.workspace = true serde_json.workspace = true serde_regex.workspace = true diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index b090618c..b3390bfb 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -66,6 +66,9 @@ console = [ # "conduit-router/dev_release_log_level", # "conduit-service/dev_release_log_level", #] +direct_tls = [ + "conduit-router/direct_tls" +] element_hacks = [ "conduit-api/element_hacks", "conduit-service/element_hacks", diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 2f85ffb7..62690194 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -42,9 +42,16 @@ systemd = [ "dep:sd-notify", ] +direct_tls = [ + "axum-server/tls-rustls", + "dep:rustls", + "dep:axum-server-dual-protocol", +] + [dependencies] axum-client-ip.workspace = true axum-server-dual-protocol.workspace = true +axum-server-dual-protocol.optional = true axum-server.workspace = true axum.workspace = true conduit-admin.workspace = true @@ -63,6 +70,7 @@ hyper.workspace = true hyper-util.workspace = true ruma.workspace = true rustls.workspace = true +rustls.optional = true sentry.optional = true sentry-tower.optional = true sentry-tower.workspace = true diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index 9e171008..858d3455 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -1,4 +1,5 @@ mod plain; +#[cfg(feature = "direct_tls")] mod tls; mod unix; @@ -23,7 +24,14 @@ pub(super) async fn serve( if cfg!(unix) && config.unix_socket_path.is_some() { unix::serve(server, app, shutdown).await } else if config.tls.is_some() { - tls::serve(server, app, handle, addrs).await + #[cfg(feature = "direct_tls")] + return tls::serve(server, app, handle, addrs).await; + + #[cfg(not(feature = "direct_tls"))] + return conduit::Err!(Config( + "tls", + "conduwuit was not built with direct TLS support (\"direct_tls\")" + )); } else { plain::serve(server, app, handle, addrs).await } diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index 174a511f..08c5e7b6 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -20,7 +20,9 @@ pub(super) async fn serve( // we use ring for ruma and hashing state, but aws-lc-rs is the new default. // without this, TLS mode will panic. - _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .expect("failed to initialise aws-lc-rs rustls crypto provider"); debug!("Using direct TLS. Certificate path {certs} and certificate private key path {key}",); info!( From 30e3e45f9fa2b8950cc508a7363be54011998b70 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 21:29:17 -0400 Subject: [PATCH 0822/2091] misc CI improvements, build macOS binaries, flake improvements/fixes Signed-off-by: strawberry --- .github/workflows/ci.yml | 110 +++++++++++++++++++++--- .github/workflows/documentation.yml | 10 ++- .github/workflows/trivy.yml | 4 +- .gitlab-ci.yml | 12 +++ bin/complement | 2 +- bin/nix-build-and-cache | 11 ++- flake.nix | 58 ++++++++++--- nix/pkgs/complement/default.nix | 13 ++- nix/pkgs/main/cross-compilation-env.nix | 21 +++-- nix/pkgs/main/default.nix | 27 ++++-- nix/pkgs/oci-image/default.nix | 2 +- 11 files changed, 219 insertions(+), 51 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e024c48..911737aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,6 @@ on: - 'docker/**' branches: - main - - change-ci-cache tags: - '*' # Allows you to run this workflow manually from the Actions tab @@ -24,7 +23,7 @@ on: concurrency: group: ${{ github.head_ref || github.ref_name }} - cancel-in-progress: true + cancel-in-progress: false env: # sccache only on main repo @@ -51,8 +50,11 @@ env: # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps NIX_CONFIG: | show-trace = true - extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org + extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= + experimental-features = nix-command flakes + extra-experimental-features = nix-command flakes + accept-flake-config = true # complement uses libolm NIXPKGS_ALLOW_INSECURE: 1 @@ -64,12 +66,23 @@ jobs: tests: name: Test runs-on: ubuntu-latest - env: - CARGO_PROFILE: "test" steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main + - name: Free up more runner space + run: | + set +o pipefail + # large docker images + sudo docker image prune --all --force || true + # large packages + sudo apt-get purge -y '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true + sudo apt-get autoremove -y + sudo apt-get clean + # large folders + sudo rm -rf /var/lib/apt/lists/* /usr/local/games /usr/local/sqlpackage /usr/local/.ghcup /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/local/share/vcpkg /usr/local/lib/python* /usr/local/lib/node_modules /usr/local/julia* /opt/mssql-tools /etc/skel /usr/share/vim /usr/share/postgresql /usr/share/man /usr/share/apache-maven-* /usr/share/R /usr/share/alsa /usr/share/miniconda /usr/share/grub /usr/share/gradle-* /usr/share/locale /usr/share/texinfo /usr/share/kotlinc /usr/share/swift /usr/share/doc /usr/share/az_9.3.0 /usr/share/sbt /usr/share/ri /usr/share/icons /usr/share/java /usr/share/fonts /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/R /usr/lib/postgresql /usr/lib/heroku /usr/lib/gcc + set -o pipefail + - name: Sync repository uses: actions/checkout@v4 @@ -85,7 +98,7 @@ jobs: exit 1 fi - - uses: nixbuild/nix-quick-install-action@v28 + - uses: nixbuild/nix-quick-install-action@master - name: Restore and cache Nix store uses: nix-community/cache-nix-action@v5.1.0 @@ -117,8 +130,11 @@ jobs: - name: Apply Nix binary cache configuration run: | sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null < "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow - nix develop .#all-features --command true --impure + nix develop .#all-features --command true - name: Cache CI dependencies run: | @@ -151,10 +167,14 @@ jobs: cache-all-crates: "true" - name: Run CI tests + env: + CARGO_PROFILE: "test" run: | direnv exec . engage > >(tee -a test_output.log) - name: Run Complement tests + env: + CARGO_PROFILE: "test" run: | # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) @@ -202,7 +222,7 @@ jobs: echo '```' >> $GITHUB_STEP_SUMMARY fi - - name: Run cargo clean test artifacts + - name: Run cargo clean test artifacts to free up space run: | cargo clean --profile test @@ -254,8 +274,11 @@ jobs: - name: Apply Nix binary cache configuration run: | sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <> $GITHUB_STEP_SUMMARY + exit 1 + fi + # use sccache for Rust + - name: Run sccache-cache + if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') + uses: mozilla-actions/sccache-action@main + # use rust-cache + - uses: Swatinem/rust-cache@v2 + with: + cache-all-crates: "true" + # Nix can't do portable macOS builds yet + - name: Build macOS x86_64 binary + if: ${{ matrix.os == 'macos-13' }} + run: | + CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short HEAD)" cargo build --release + cp -v -f target/release/conduit conduwuit-macos-x86_64 + otool -L conduwuit-macos-x86_64 + # quick smoke test of the x86_64 macOS binary + - name: Run x86_64 macOS release binary + if: ${{ matrix.os == 'macos-13' }} + run: | + ./conduwuit-macos-x86_64 --version + - name: Build macOS arm64 binary + if: ${{ matrix.os == 'macos-latest' }} + run: | + CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short HEAD)" cargo build --release + cp -v -f target/release/conduit conduwuit-macos-arm64 + otool -L conduwuit-macos-arm64 + # quick smoke test of the arm64 macOS binary + - name: Run arm64 macOS release binary + if: ${{ matrix.os == 'macos-latest' }} + run: | + ./conduwuit-macos-arm64 --version + - name: Upload macOS x86_64 binary + if: ${{ matrix.os == 'macos-13' }} + uses: actions/upload-artifact@v4 + with: + name: conduwuit-macos-x86_64 + path: conduwuit-macos-x86_64 + if-no-files-found: error + - name: Upload macOS arm64 binary + if: ${{ matrix.os == 'macos-latest' }} + uses: actions/upload-artifact@v4 + with: + name: conduwuit-macos-arm64 + path: conduwuit-macos-arm64 + if-no-files-found: error + docker: name: Docker publish runs-on: ubuntu-latest diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ccd144d2..45e52b22 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -24,8 +24,11 @@ env: # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps NIX_CONFIG: | show-trace = true - extra-substituters = https://attic.kennel.juneis.dog/conduit https://attic.kennel.juneis.dog/conduwuit https://cache.lix.systems https://conduwuit.cachix.org + extra-substituters = extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://cache.lix.systems https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= + experimental-features = nix-command flakes + extra-experimental-features = nix-command flakes + accept-flake-config = true # Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. # However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. @@ -89,8 +92,11 @@ jobs: - name: Apply Nix binary cache configuration run: | sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null < /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi + # Accept flake config from "untrusted" users + - if command -v nix > /dev/null; then echo "accept-flake-config = true" >> /etc/nix/nix.conf; fi # Add conduwuit binary cache - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduwuit" >> /etc/nix/nix.conf; fi @@ -47,6 +57,8 @@ before_script: - if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-substituters = https://aseipp-nix-cache.freetls.fastly.net" >> /etc/nix/nix.conf; fi + # Install direnv and nix-direnv - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv; fi diff --git a/bin/complement b/bin/complement index a715b14b..094cf248 100755 --- a/bin/complement +++ b/bin/complement @@ -15,7 +15,7 @@ LOG_FILE="$2" # A `.jsonl` file to write test results to RESULTS_FILE="$3" -OCI_IMAGE="complement-conduit:main" +OCI_IMAGE="complement-conduwuit:main" # Complement tests that are skipped due to flakiness/reliability issues SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*' diff --git a/bin/nix-build-and-cache b/bin/nix-build-and-cache index f6b471a6..ac64ff23 100755 --- a/bin/nix-build-and-cache +++ b/bin/nix-build-and-cache @@ -26,7 +26,12 @@ just() { "$ATTIC_TOKEN" # Find all output paths of the installables and their build dependencies - readarray -t derivations < <(nix path-info --derivation "$@") + #readarray -t derivations < <(nix path-info --derivation "$@") + derivations=() + while IFS=$'\n' read derivation; do + derivations+=("$derivation") + done < <(nix path-info --derivation "$@") + cache=() for derivation in "${derivations[@]}"; do cache+=( @@ -77,8 +82,8 @@ ci() { --inputs-from "$toplevel" # Keep sorted - "$toplevel#devShells.x86_64-linux.default" - "$toplevel#devShells.x86_64-linux.all-features" + #"$toplevel#devShells.x86_64-linux.default" + #"$toplevel#devShells.x86_64-linux.all-features" attic#default cachix#default nixpkgs#direnv diff --git a/flake.nix b/flake.nix index 7c870cce..85b7baa0 100644 --- a/flake.nix +++ b/flake.nix @@ -38,6 +38,14 @@ inherit inputs; main = self.callPackage ./nix/pkgs/main {}; oci-image = self.callPackage ./nix/pkgs/oci-image {}; + tini = pkgs.tini.overrideAttrs { + # newer clang/gcc is unhappy with tini-static: + patches = [ (pkgs.fetchpatch { + url = "https://patch-diff.githubusercontent.com/raw/krallin/tini/pull/224.patch"; + hash = "sha256-4bTfAhRyIT71VALhHY13hUgbjLEUyvgkIJMt3w9ag3k="; + }) + ]; + }; liburing = pkgs.liburing.overrideAttrs { # Tests weren't building outputs = [ "out" "dev" "man" ]; @@ -88,6 +96,16 @@ scopeHost = mkScope pkgsHost; scopeHostStatic = mkScope pkgsHostStatic; + scopeCrossLinux = mkScope pkgsHost.pkgsLinux.pkgsStatic; + mkCrossScope = crossSystem: + let pkgsCrossStatic = (import inputs.nixpkgs { + inherit system; + crossSystem = { + config = crossSystem; + }; + }).pkgsStatic; + in + mkScope pkgsCrossStatic; mkDevShell = scope: scope.pkgs.mkShell { env = scope.main.env // { @@ -118,7 +136,6 @@ ++ (with pkgsHost.pkgs; [ engage cargo-audit - liburing # Required by hardened-malloc.rs dep binutils @@ -149,12 +166,21 @@ # needed so we can get rid of gcc and other unused deps that bloat OCI images removeReferencesTo - ]) + ] + # liburing is Linux-exclusive + ++ lib.optional stdenv.hostPlatform.isLinux liburing + # needed to build Rust applications on macOS + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + # https://github.com/NixOS/nixpkgs/issues/206242 + # ld: library not found for -liconv + libiconv + # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell + # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612 + pkgsBuildHost.darwin.apple_sdk.frameworks.Security + ]) ++ scope.main.buildInputs ++ scope.main.propagatedBuildInputs ++ scope.main.nativeBuildInputs; - - meta.broken = scope.main.meta.broken; }; in { @@ -228,6 +254,8 @@ complement = scopeHost.complement; static-complement = scopeHostStatic.complement; + # macOS containers don't exist, so the complement images must be forced to linux + linux-complement = (mkCrossScope "${pkgsHost.hostPlatform.qemuArch}-linux-musl").complement; } // builtins.listToAttrs @@ -236,14 +264,7 @@ (crossSystem: let binaryName = "static-${crossSystem}"; - pkgsCrossStatic = - (import inputs.nixpkgs { - inherit system; - crossSystem = { - config = crossSystem; - }; - }).pkgsStatic; - scopeCrossStatic = mkScope pkgsCrossStatic; + scopeCrossStatic = mkCrossScope crossSystem; in [ # An output for a statically-linked binary @@ -373,11 +394,20 @@ }; }; } + + # An output for a complement OCI image for the specified platform + { + name = "complement-${crossSystem}"; + value = scopeCrossStatic.complement; + } ] ) [ - "x86_64-unknown-linux-musl" - "aarch64-unknown-linux-musl" + #"x86_64-apple-darwin" + #"aarch64-apple-darwin" + "x86_64-linux-gnu" + "x86_64-linux-musl" + "aarch64-linux-musl" ] ) ); diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index 5ebe9eb3..80e9ce27 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -18,6 +18,15 @@ let all_features = true; disable_release_max_log_level = true; disable_features = [ + # no reason to use jemalloc for complement, just has compatibility/build issues + "jemalloc" + # console/CLI stuff isn't used or relevant for complement + "console" + "tokio_console" + # sentry telemetry isn't useful for complement, disabled by default anyways + "sentry_telemetry" + # the containers don't use or need systemd signal support + "systemd" # this is non-functional on nix for some reason "hardened_malloc" # dont include experimental features @@ -57,7 +66,7 @@ let in dockerTools.buildImage { - name = "complement-${main.pname}"; + name = "complement-conduwuit"; tag = "main"; copyToRoot = buildEnv { @@ -78,7 +87,7 @@ dockerTools.buildImage { "${lib.getExe start}" ]; - Entrypoint = if !stdenv.isDarwin + Entrypoint = if !stdenv.hostPlatform.isDarwin # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT) # are handled as expected then [ "${lib.getExe' tini "tini"}" "--" ] diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index 83fe6ed6..c189d57c 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -1,5 +1,6 @@ { lib , pkgsBuildHost +, pkgsBuildTarget , rust , stdenv }: @@ -35,7 +36,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { # including it here. Linkers are weird. (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64) && stdenv.hostPlatform.isStatic - && !stdenv.isDarwin + && !stdenv.hostPlatform.isDarwin && !stdenv.cc.bintools.isLLVM ) [ @@ -52,11 +53,12 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { # even covers the case of build scripts that need native code compiled and # run on the build platform (I think). # -# [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L57-L80 +# [0]: https://github.com/NixOS/nixpkgs/blob/nixpkgs-unstable/pkgs/build-support/rust/lib/default.nix#L48-L68 // ( let inherit (rust.lib) envVars; + shouldUseLLD = platform: platform.isAarch64 && platform.isStatic && !stdenv.hostPlatform.isDarwin; in lib.optionalAttrs (stdenv.targetPlatform.rust.rustcTarget @@ -64,23 +66,30 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { ( let inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget; + linkerForTarget = if shouldUseLLD stdenv.targetPlatform + && !stdenv.cc.bintools.isLLVM # whether stdenv's linker is lld already + then "${pkgsBuildTarget.llvmPackages.bintools}/bin/${stdenv.cc.targetPrefix}ld.lld" + else envVars.ccForTarget; in { "CC_${cargoEnvVarTarget}" = envVars.ccForTarget; "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = - envVars.linkerForTarget; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = linkerForTarget; } ) // ( let inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget; + linkerForHost = if shouldUseLLD stdenv.targetPlatform + && !stdenv.cc.bintools.isLLVM + then "${pkgsBuildHost.llvmPackages.bintools}/bin/${stdenv.cc.targetPrefix}ld.lld" + else envVars.ccForHost; in { "CC_${cargoEnvVarTarget}" = envVars.ccForHost; "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForHost; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = linkerForHost; CARGO_BUILD_TARGET = rustcTarget; } ) @@ -92,7 +101,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { { "CC_${cargoEnvVarTarget}" = envVars.ccForBuild; "CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.linkerForBuild; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForBuild; HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc"; HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++"; } diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 8559e878..1088b03c 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -6,6 +6,7 @@ , libiconv , liburing , pkgsBuildHost +, pkgsBuildTarget , rocksdb , removeReferencesTo , rust @@ -40,7 +41,7 @@ features'' = lib.subtractLists disable_features' features'; featureEnabled = feature : builtins.elem feature features''; -enableLiburing = featureEnabled "io_uring" && !stdenv.isDarwin; +enableLiburing = featureEnabled "io_uring" && !stdenv.hostPlatform.isDarwin; # This derivation will set the JEMALLOC_OVERRIDE variable, causing the # tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's @@ -72,16 +73,12 @@ buildDepsOnlyEnv = # jemalloc symbols are prefixed. # # [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17 - enableJemalloc = featureEnabled "jemalloc" && !stdenv.isDarwin; + enableJemalloc = featureEnabled "jemalloc" && !stdenv.hostPlatform.isDarwin; # for some reason enableLiburing in nixpkgs rocksdb is default true # which breaks Darwin entirely enableLiburing = enableLiburing; }).overrideAttrs (old: { - # TODO: static rocksdb fails to build on darwin, also see - # build log at - meta.broken = stdenv.hostPlatform.isStatic && stdenv.isDarwin; - enableLiburing = enableLiburing; }); in @@ -99,6 +96,7 @@ buildDepsOnlyEnv = inherit lib pkgsBuildHost + pkgsBuildTarget rust stdenv; }); @@ -137,7 +135,16 @@ commonAttrs = { dontStrip = profile == "dev" || profile == "test"; dontPatchELF = profile == "dev" || profile == "test"; - buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'; + buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys' + # needed to build Rust applications on macOS + ++ lib.optionals stdenv.hostPlatform.isDarwin [ + # https://github.com/NixOS/nixpkgs/issues/206242 + # ld: library not found for -liconv + libiconv + # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell + # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612 + pkgsBuildHost.darwin.apple_sdk.frameworks.Security + ]; nativeBuildInputs = [ # bindgen needs the build platform's libclang. Apparently due to "splicing @@ -154,8 +161,10 @@ commonAttrs = { # needed so we can get rid of gcc and other unused deps that bloat OCI images removeReferencesTo ] - ++ lib.optionals stdenv.isDarwin [ + # needed to build Rust applications on macOS + ++ lib.optionals stdenv.hostPlatform.isDarwin [ # https://github.com/NixOS/nixpkgs/issues/206242 + # ld: library not found for -liconv libiconv # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell @@ -167,7 +176,7 @@ commonAttrs = { # # postInstall = with pkgsBuildHost; '' - find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${libgcc} -t ${linuxHeaders} -t ${libidn2} -t ${libunistring} '{}' + + find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${rustc.unwrapped} -t ${rustc} -t ${libidn2} -t ${libunistring} '{}' + ''; }; in diff --git a/nix/pkgs/oci-image/default.nix b/nix/pkgs/oci-image/default.nix index ed2ec19a..5078523b 100644 --- a/nix/pkgs/oci-image/default.nix +++ b/nix/pkgs/oci-image/default.nix @@ -16,7 +16,7 @@ dockerTools.buildLayeredImage { dockerTools.caCertificates ]; config = { - Entrypoint = if !stdenv.isDarwin + Entrypoint = if !stdenv.hostPlatform.isDarwin # Use the `tini` init system so that signals (e.g. ctrl+c/SIGINT) # are handled as expected then [ "${lib.getExe' tini "tini"}" "--" ] From 9761e2f10ce853922014dca319103821c81b7e9a Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 21:38:26 -0400 Subject: [PATCH 0823/2091] fix lockdown_public_room_directory bypass, add appservice exclusion Signed-off-by: strawberry --- src/api/client/directory.rs | 5 ++++- src/api/client/room.rs | 30 +++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 1a3e717d..423f2bde 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -137,7 +137,10 @@ pub(crate) async fn set_room_visibility_route( match &body.visibility { room::Visibility::Public => { - if services.globals.config.lockdown_public_room_directory && !services.users.is_admin(sender_user)? { + if services.globals.config.lockdown_public_room_directory + && !services.users.is_admin(sender_user)? + && body.appservice_info.is_none() + { info!( "Non-admin user {sender_user} tried to publish {0} to the room directory while \ \"lockdown_public_room_directory\" is enabled", diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 609f3b40..0a75787d 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -1,7 +1,7 @@ use std::{cmp::max, collections::BTreeMap}; use axum::extract::State; -use conduit::{debug_info, debug_warn, err}; +use conduit::{debug_info, debug_warn, err, Err}; use ruma::{ api::client::{ error::ErrorKind, @@ -64,6 +64,7 @@ const TRANSFERABLE_STATE_EVENTS: &[StateEventType; 9] = &[ /// - Send events listed in initial state /// - Send events implied by `name` and `topic` /// - Send invite events +#[allow(clippy::large_stack_frames)] pub(crate) async fn create_room_route( State(services): State, body: Ruma, ) -> Result { @@ -92,6 +93,28 @@ pub(crate) async fn create_room_route( )); } + if body.visibility == room::Visibility::Public + && services.globals.config.lockdown_public_room_directory + && !services.users.is_admin(sender_user)? + && body.appservice_info.is_none() + { + info!( + "Non-admin user {sender_user} tried to publish {0} to the room directory while \ + \"lockdown_public_room_directory\" is enabled", + &room_id + ); + services + .admin + .send_text(&format!( + "Non-admin user {sender_user} tried to publish {0} to the room directory while \ + \"lockdown_public_room_directory\" is enabled", + &room_id + )) + .await; + + return Err!(Request(Forbidden("Publishing rooms to the room directory is not allowed"))); + } + let _short_id = services.rooms.short.get_or_create_shortroomid(&room_id)?; let state_lock = services.rooms.state.mutex.lock(&room_id).await; @@ -450,6 +473,11 @@ pub(crate) async fn create_room_route( if body.visibility == room::Visibility::Public { services.rooms.directory.set_public(&room_id)?; + services + .admin + .send_text(&format!("{sender_user} made {} public to the room directory", &room_id)) + .await; + info!("{sender_user} made {0} public to the room directory", &room_id); } info!("{sender_user} created a room with room ID {room_id}"); From 359fb25262cfe55173d012166d4cf5b472d357d5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 21:45:13 -0400 Subject: [PATCH 0824/2091] add missing feat_sha256_media to fresh database creations Signed-off-by: strawberry --- src/service/globals/migrations.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 39d829b4..a52c6f00 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -54,6 +54,7 @@ async fn fresh(services: &Services) -> Result<()> { .db .bump_database_version(DATABASE_VERSION)?; + db["global"].insert(b"feat_sha256_media", &[])?; db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[])?; db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; From 724711218af79a1b4c64ac5f3c95cb0d3f72c24e Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 21:46:23 -0400 Subject: [PATCH 0825/2091] add note that ko-fi takes a fee Signed-off-by: strawberry --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d4d76e6c..962139d6 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ not get paid to work on this, and I work on it in my free time. Donations are heavily appreciated! 💜🥺 - Liberapay: -- Ko-fi: +- Ko-fi (note they take a fee): - GitHub Sponsors: #### Logo From 37a2ba59d0af70eaacf61afe004a3fb765110b93 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 1 Oct 2024 01:27:45 -0400 Subject: [PATCH 0826/2091] improve UX of admin media deletion commands, ignore errors by default, support deleting local media too Signed-off-by: strawberry --- src/admin/media/commands.rs | 133 ++++++++++++++++++++++-------------- src/admin/media/mod.rs | 47 +++++++------ src/service/media/mod.rs | 128 +++++++++++++++++----------------- 3 files changed, 171 insertions(+), 137 deletions(-) diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 535ad31a..3c4bf2ef 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use conduit::{debug, info, trace, utils::time::parse_timepoint_ago, warn, Result}; +use conduit::{debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, Result}; use conduit_service::media::Dim; use ruma::{ events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, OwnedMxcUri, OwnedServerName, ServerName, @@ -19,7 +19,7 @@ pub(super) async fn delete( } if let Some(mxc) = mxc { - debug!("Got MXC URL: {mxc}"); + trace!("Got MXC URL: {mxc}"); self.services .media .delete(&mxc.as_str().try_into()?) @@ -28,11 +28,12 @@ pub(super) async fn delete( return Ok(RoomMessageEventContent::text_plain( "Deleted the MXC from our database and on our filesystem.", )); - } else if let Some(event_id) = event_id { - debug!("Got event ID to delete media from: {event_id}"); + } - let mut mxc_urls = vec![]; - let mut mxc_deletion_count: usize = 0; + if let Some(event_id) = event_id { + trace!("Got event ID to delete media from: {event_id}"); + + let mut mxc_urls = Vec::with_capacity(4); // parsing the PDU for any MXC URLs begins here if let Some(event_json) = self.services.rooms.timeline.get_pdu_json(&event_id)? { @@ -124,18 +125,28 @@ pub(super) async fn delete( } if mxc_urls.is_empty() { - // we shouldn't get here (should have errored earlier) but just in case for - // whatever reason we do... info!("Parsed event ID {event_id} but did not contain any MXC URLs."); return Ok(RoomMessageEventContent::text_plain("Parsed event ID but found no MXC URLs.")); } + let mut mxc_deletion_count: usize = 0; + for mxc_url in mxc_urls { - self.services + match self + .services .media .delete(&mxc_url.as_str().try_into()?) - .await?; - mxc_deletion_count = mxc_deletion_count.saturating_add(1); + .await + { + Ok(()) => { + debug_info!("Successfully deleted {mxc_url} from filesystem and database"); + mxc_deletion_count = mxc_deletion_count.saturating_add(1); + }, + Err(e) => { + debug_warn!("Failed to delete {mxc_url}, ignoring error and skipping: {e}"); + continue; + }, + } } return Ok(RoomMessageEventContent::text_plain(format!( @@ -158,34 +169,62 @@ pub(super) async fn delete_list(&self) -> Result { )); } + let mut failed_parsed_mxcs: usize = 0; + let mxc_list = self .body .to_vec() .drain(1..self.body.len().checked_sub(1).unwrap()) - .collect::>(); + .filter_map(|mxc_s| { + mxc_s + .try_into() + .inspect_err(|e| { + debug_warn!("Failed to parse user-provided MXC URI: {e}"); + + failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1); + }) + .ok() + }) + .collect::>>(); let mut mxc_deletion_count: usize = 0; - for mxc in mxc_list { - debug!("Deleting MXC {mxc} in bulk"); - self.services.media.delete(&mxc.try_into()?).await?; - mxc_deletion_count = mxc_deletion_count - .checked_add(1) - .expect("mxc_deletion_count should not get this high"); + for mxc in &mxc_list { + trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk"); + match self.services.media.delete(mxc).await { + Ok(()) => { + debug_info!("Successfully deleted {mxc} from filesystem and database"); + mxc_deletion_count = mxc_deletion_count.saturating_add(1); + }, + Err(e) => { + debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); + continue; + }, + } } Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database and the filesystem.", + "Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database and the filesystem. \ + {failed_parsed_mxcs} MXCs failed to be parsed from the database.", ))) } #[admin_command] -pub(super) async fn delete_past_remote_media(&self, duration: String, force: bool) -> Result { +pub(super) async fn delete_past_remote_media( + &self, duration: String, before: bool, after: bool, yes_i_want_to_delete_local_media: bool, +) -> Result { + if before && after { + return Ok(RoomMessageEventContent::text_plain( + "Please only pick one argument, --before or --after.", + )); + } + assert!(!(before && after), "--before and --after should not be specified together"); + let duration = parse_timepoint_ago(&duration)?; let deleted_count = self .services .media - .delete_all_remote_media_at_after_time(duration, force) + .delete_all_remote_media_at_after_time(duration, before, after, yes_i_want_to_delete_local_media) .await?; Ok(RoomMessageEventContent::text_plain(format!( @@ -194,14 +233,10 @@ pub(super) async fn delete_past_remote_media(&self, duration: String, force: boo } #[admin_command] -pub(super) async fn delete_all_from_user(&self, username: String, force: bool) -> Result { +pub(super) async fn delete_all_from_user(&self, username: String) -> Result { let user_id = parse_local_user_id(self.services, &username)?; - let deleted_count = self - .services - .media - .delete_from_user(&user_id, force) - .await?; + let deleted_count = self.services.media.delete_from_user(&user_id).await?; Ok(RoomMessageEventContent::text_plain(format!( "Deleted {deleted_count} total files.", @@ -210,34 +245,36 @@ pub(super) async fn delete_all_from_user(&self, username: String, force: bool) - #[admin_command] pub(super) async fn delete_all_from_server( - &self, server_name: Box, force: bool, + &self, server_name: Box, yes_i_want_to_delete_local_media: bool, ) -> Result { - if server_name == self.services.globals.server_name() { - return Ok(RoomMessageEventContent::text_plain("This command only works for remote media.")); + if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media { + return Ok(RoomMessageEventContent::text_plain( + "This command only works for remote media by default.", + )); } - let Ok(all_mxcs) = self.services.media.get_all_mxcs().await else { + let Ok(all_mxcs) = self + .services + .media + .get_all_mxcs() + .await + .inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}")) + else { return Ok(RoomMessageEventContent::text_plain("Failed to get MXC URIs from our database")); }; let mut deleted_count: usize = 0; for mxc in all_mxcs { - let mxc_server_name = match mxc.server_name() { - Ok(server_name) => server_name, - Err(e) => { - if force { - warn!("Failed to parse MXC {mxc} server name from database, ignoring error and skipping: {e}"); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse MXC {mxc} server name from database: {e}", - ))); - }, + let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| { + debug_warn!("Failed to parse MXC {mxc} server name from database, ignoring error and skipping: {e}"); + }) else { + continue; }; - if mxc_server_name != server_name || self.services.globals.server_is_ours(mxc_server_name) { + if mxc_server_name != server_name + || (self.services.globals.server_is_ours(mxc_server_name) && !yes_i_want_to_delete_local_media) + { trace!("skipping MXC URI {mxc}"); continue; } @@ -249,12 +286,8 @@ pub(super) async fn delete_all_from_server( deleted_count = deleted_count.saturating_add(1); }, Err(e) => { - if force { - warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!("Failed to delete MXC {mxc}: {e}"))); + debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); + continue; }, } } diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 5977c0fa..fbf6532b 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -10,7 +10,7 @@ use crate::admin_command_dispatch; #[derive(Debug, Subcommand)] pub(super) enum MediaCommand { /// - Deletes a single media file from our database and on the filesystem - /// via a single MXC URL + /// via a single MXC URL or event ID (not redacted) Delete { /// The MXC URL to delete #[arg(long)] @@ -23,37 +23,44 @@ pub(super) enum MediaCommand { }, /// - Deletes a codeblock list of MXC URLs from our database and on the - /// filesystem + /// filesystem. This will always ignore errors. DeleteList, - /// - Deletes all remote media in the last X amount of time using filesystem - /// metadata first created at date. + /// - Deletes all remote media in the last/after "X" time using filesystem + /// metadata first created at date, or fallback to last modified date. + /// This will always ignore errors by default. + /// + /// Synapse DeletePastRemoteMedia { - /// - The duration (at or after), e.g. "5m" to delete all media in the - /// past 5 minutes + /// - The duration (at or after/before), e.g. "5m" to delete all media + /// in the past or up to 5 minutes duration: String, - /// Continues deleting remote media if an undeletable object is found - #[arg(short, long)] - force: bool, + #[arg(long, short)] + before: bool, + + #[arg(long, short)] + after: bool, + + /// Long argument to delete local media + #[arg(long)] + yes_i_want_to_delete_local_media: bool, }, - /// - Deletes all the local media from a local user on our server + /// - Deletes all the local media from a local user on our server. This will + /// always ignore errors by default. DeleteAllFromUser { username: String, - - /// Continues deleting media if an undeletable object is found - #[arg(short, long)] - force: bool, }, - /// - Deletes all remote media from the specified remote server + /// - Deletes all remote media from the specified remote server. This will + /// always ignore errors by default. DeleteAllFromServer { server_name: Box, - /// Continues deleting media if an undeletable object is found - #[arg(short, long)] - force: bool, + /// Long argument to delete local media + #[arg(long)] + yes_i_want_to_delete_local_media: bool, }, GetFileInfo { @@ -82,10 +89,10 @@ pub(super) enum MediaCommand { #[arg(short, long, default_value("10000"))] timeout: u32, - #[arg(short, long)] + #[arg(short, long, default_value("800"))] width: u32, - #[arg(short, long)] + #[arg(short, long, default_value("800"))] height: u32, }, } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 2faa13d8..d3765a17 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -10,7 +10,7 @@ use std::{path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; use conduit::{ - debug, debug_error, debug_info, err, error, trace, + debug, debug_error, debug_info, debug_warn, err, error, trace, utils::{self, MutexMap}, warn, Err, Result, Server, }; @@ -99,45 +99,46 @@ impl Service { pub async fn delete(&self, mxc: &Mxc<'_>) -> Result<()> { if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc) { for key in keys { - trace!(?mxc, ?key, "Deleting from filesystem"); + trace!(?mxc, "MXC Key: {key:?}"); + debug_info!(?mxc, "Deleting from filesystem"); + if let Err(e) = self.remove_media_file(&key).await { - error!(?mxc, ?key, "Failed to remove media file: {e}"); + debug_error!(?mxc, "Failed to remove media file: {e}"); } - trace!(?mxc, ?key, "Deleting from database"); - if let Err(e) = self.db.delete_file_mxc(mxc) { - error!(?mxc, ?key, "Failed to remove media from database: {e}"); - } + debug_info!(?mxc, "Deleting from database"); + _ = self.db.delete_file_mxc(mxc); } Ok(()) } else { - Err!(Database(error!( - "Failed to find any media keys for MXC {mxc:?} in our database." - ))) + Err!(Database(error!("Failed to find any media keys for MXC {mxc} in our database."))) } } /// Deletes all media by the specified user /// /// currently, this is only practical for local users - pub async fn delete_from_user(&self, user: &UserId, force: bool) -> Result { + pub async fn delete_from_user(&self, user: &UserId) -> Result { let mxcs = self.db.get_all_user_mxcs(user); let mut deletion_count: usize = 0; for mxc in mxcs { - let mxc: Mxc<'_> = mxc.as_str().try_into()?; - debug_info!("Deleting MXC {mxc} by user {user} from database and filesystem"); - if force { - _ = self - .delete(&mxc) - .await - .inspect_err(|e| warn!("Failed to delete {mxc} from user {user}, ignoring error: {e}")); - } else { - self.delete(&mxc).await?; - } + let Ok(mxc) = mxc.as_str().try_into().inspect_err(|e| { + debug_error!(?mxc, "Failed to parse MXC URI from database: {e}"); + }) else { + continue; + }; - deletion_count = deletion_count.saturating_add(1); + debug_info!(%deletion_count, "Deleting MXC {mxc} by user {user} from database and filesystem"); + match self.delete(&mxc).await { + Ok(()) => { + deletion_count = deletion_count.saturating_add(1); + }, + Err(e) => { + debug_error!(%deletion_count, "Failed to delete {mxc} from user {user}, ignoring error: {e}"); + }, + } } Ok(deletion_count) @@ -176,9 +177,6 @@ impl Service { for key in all_keys { trace!("Full MXC key from database: {key:?}"); - // we need to get the MXC URL from the first part of the key (the first 0xff / - // 255 push). this is all necessary because of conduit using magic keys for - // media let mut parts = key.split(|&b| b == 0xFF); let mxc = parts .next() @@ -189,31 +187,33 @@ impl Service { .transpose()?; let Some(mxc_s) = mxc else { - return Err!(Database("Parsed MXC URL unicode bytes from database but still is None")); + debug_warn!(?mxc, "Parsed MXC URL unicode bytes from database but is still invalid"); + continue; }; trace!("Parsed MXC key to URL: {mxc_s}"); let mxc = OwnedMxcUri::from(mxc_s); - mxcs.push(mxc); + if mxc.is_valid() { + mxcs.push(mxc); + } else { + debug_warn!("{mxc:?} from database was found to not be valid"); + } } Ok(mxcs) } /// Deletes all remote only media files in the given at or after - /// time/duration. Returns a u32 with the amount of media files deleted. - pub async fn delete_all_remote_media_at_after_time(&self, time: SystemTime, force: bool) -> Result { + /// time/duration. Returns a usize with the amount of media files deleted. + pub async fn delete_all_remote_media_at_after_time( + &self, time: SystemTime, before: bool, after: bool, yes_i_want_to_delete_local_media: bool, + ) -> Result { let all_keys = self.db.get_all_media_keys(); - let mut remote_mxcs = Vec::with_capacity(all_keys.len()); for key in all_keys { trace!("Full MXC key from database: {key:?}"); - - // we need to get the MXC URL from the first part of the key (the first 0xff / - // 255 push). this is all necessary because of conduit using magic keys for - // media let mut parts = key.split(|&b| b == 0xFF); let mxc = parts .next() @@ -224,35 +224,30 @@ impl Service { .transpose()?; let Some(mxc_s) = mxc else { - return Err!(Database("Parsed MXC URL unicode bytes from database but still is None")); + debug_warn!(?mxc, "Parsed MXC URL unicode bytes from database but is still invalid"); + continue; }; trace!("Parsed MXC key to URL: {mxc_s}"); let mxc = OwnedMxcUri::from(mxc_s); - if mxc.server_name() == Ok(self.services.globals.server_name()) { - debug!("Ignoring local media MXC: {mxc}"); - // ignore our own MXC URLs as this would be local media. + if (mxc.server_name() == Ok(self.services.globals.server_name()) && !yes_i_want_to_delete_local_media) + || !mxc.is_valid() + { + debug!("Ignoring local or broken media MXC: {mxc}"); continue; } let path = self.get_media_file(&key); - debug!("MXC path: {path:?}"); let file_metadata = match fs::metadata(path.clone()).await { Ok(file_metadata) => file_metadata, Err(e) => { - if force { - error!("Failed to obtain file metadata for MXC {mxc} at file path \"{path:?}\", skipping: {e}"); - continue; - } - - return Err!(Database( - "Failed to obtain file metadata for MXC {mxc} at file path \"{path:?}\": {e}" - )); + error!("Failed to obtain file metadata for MXC {mxc} at file path \"{path:?}\", skipping: {e}"); + continue; }, }; - debug!("File metadata: {file_metadata:?}"); + trace!(%mxc, ?path, "File metadata: {file_metadata:?}"); let file_created_at = match file_metadata.created() { Ok(value) => value, @@ -261,33 +256,36 @@ impl Service { file_metadata.modified()? }, Err(err) => { - if force { - error!("Could not delete MXC {mxc} at path {path:?}: {err:?}. Skipping..."); - continue; - } - - return Err(err.into()); + error!("Could not delete MXC {mxc} at path {path:?}: {err:?}. Skipping..."); + continue; }, }; debug!("File created at: {file_created_at:?}"); - if file_created_at <= time { - debug!("File is within user duration, pushing to list of file paths and keys to delete."); + + if file_created_at >= time && before { + debug!("File is within (before) user duration, pushing to list of file paths and keys to delete."); + remote_mxcs.push(mxc.to_string()); + } else if file_created_at <= time && after { + debug!("File is not within (after) user duration, pushing to list of file paths and keys to delete."); remote_mxcs.push(mxc.to_string()); } } - debug!( - "Finished going through all our media in database for eligible keys to delete, checking if these are empty" - ); if remote_mxcs.is_empty() { return Err!(Database("Did not found any eligible MXCs to delete.")); } - debug_info!("Deleting media now in the past {time:?}."); + debug_info!("Deleting media now in the past {time:?}"); + let mut deletion_count: usize = 0; + for mxc in remote_mxcs { - let mxc: Mxc<'_> = mxc.as_str().try_into()?; + let Ok(mxc) = mxc.as_str().try_into() else { + debug_warn!("Invalid MXC in database, skipping"); + continue; + }; + debug_info!("Deleting MXC {mxc} from database and filesystem"); match self.delete(&mxc).await { @@ -295,12 +293,8 @@ impl Service { deletion_count = deletion_count.saturating_add(1); }, Err(e) => { - if force { - warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); - continue; - } - - return Err!(Database(warn!("Failed to delete MXC {mxc}: {e}"))); + warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); + continue; }, } } From 0760150822bc5b4858e76720cbad2f604039b69b Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 22:08:23 -0400 Subject: [PATCH 0827/2091] cache all 3 x86_64 nix devshells in CI Signed-off-by: strawberry --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 911737aa..09b5cd52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,9 @@ jobs: - name: Cache CI dependencies run: | bin/nix-build-and-cache ci + bin/nix-build-and-cache just '.#devShells.x86_64-linux.default' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.all-features' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.dynamic' # use sccache for Rust - name: Run sccache-cache From 75be68fa618d2f450c19e4180f160e069bfcb533 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 22:19:18 -0400 Subject: [PATCH 0828/2091] add config option to control sending admin notices of alerts Signed-off-by: strawberry --- conduwuit-example.toml | 8 +++ src/api/client/account.rs | 94 +++++++++++++++++++++--------------- src/api/client/directory.rs | 30 +++++++----- src/api/client/membership.rs | 34 +++++++------ src/api/client/room.rs | 30 +++++++----- src/core/config/mod.rs | 8 +++ 6 files changed, 128 insertions(+), 76 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 6142705d..b532d381 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -351,6 +351,14 @@ allow_profile_lookup_federation_requests = true # defaults to true #admin_escape_commands = true +# Controls whether admin room notices like account registrations, password changes, account deactivations, +# room directory publications, etc will be sent to the admin room. +# +# Update notices and normal admin command responses will still be sent. +# +# defaults to true +#admin_room_notices = true + ### Misc diff --git a/src/api/client/account.rs b/src/api/client/account.rs index b87eb400..cee86f80 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -310,21 +310,27 @@ pub(crate) async fn register_route( if body.appservice_info.is_none() && !is_guest { if !device_display_name.is_empty() { info!("New user \"{user_id}\" registered on this server with device display name: {device_display_name}"); - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "New user \"{user_id}\" registered on this server from IP {client} and device display name \ - \"{device_display_name}\"" - ))) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::notice_plain(format!( + "New user \"{user_id}\" registered on this server from IP {client} and device display name \ + \"{device_display_name}\"" + ))) + .await; + } } else { info!("New user \"{user_id}\" registered on this server."); - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "New user \"{user_id}\" registered on this server from IP {client}" - ))) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::notice_plain(format!( + "New user \"{user_id}\" registered on this server from IP {client}" + ))) + .await; + } } } @@ -333,20 +339,26 @@ pub(crate) async fn register_route( info!("New guest user \"{user_id}\" registered on this server."); if !device_display_name.is_empty() { - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with device display name \"{device_display_name}\" registered on this \ - server from IP {client}" - ))) - .await; + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::notice_plain(format!( + "Guest user \"{user_id}\" with device display name \"{device_display_name}\" registered on \ + this server from IP {client}" + ))) + .await; + } } else { - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with no device display name registered on this server from IP {client}", - ))) - .await; + #[allow(clippy::collapsible_else_if)] + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::notice_plain(format!( + "Guest user \"{user_id}\" with no device display name registered on this server from IP \ + {client}", + ))) + .await; + } } } @@ -481,12 +493,15 @@ pub(crate) async fn change_password_route( } info!("User {sender_user} changed their password."); - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "User {sender_user} changed their password." - ))) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::notice_plain(format!( + "User {sender_user} changed their password." + ))) + .await; + } Ok(change_password::v3::Response {}) } @@ -572,12 +587,15 @@ pub(crate) async fn deactivate_route( full_user_deactivate(&services, sender_user, all_joined_rooms).await?; info!("User {sender_user} deactivated their account."); - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "User {sender_user} deactivated their account." - ))) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::notice_plain(format!( + "User {sender_user} deactivated their account." + ))) + .await; + } Ok(deactivate::v3::Response { id_server_unbind_result: ThirdPartyIdRemovalStatus::NoSupport, diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 423f2bde..602f876a 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -146,14 +146,17 @@ pub(crate) async fn set_room_visibility_route( \"lockdown_public_room_directory\" is enabled", body.room_id ); - services - .admin - .send_text(&format!( - "Non-admin user {sender_user} tried to publish {0} to the room directory while \ - \"lockdown_public_room_directory\" is enabled", - body.room_id - )) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_text(&format!( + "Non-admin user {sender_user} tried to publish {0} to the room directory while \ + \"lockdown_public_room_directory\" is enabled", + body.room_id + )) + .await; + } return Err(Error::BadRequest( ErrorKind::forbidden(), @@ -162,10 +165,13 @@ pub(crate) async fn set_room_visibility_route( } services.rooms.directory.set_public(&body.room_id)?; - services - .admin - .send_text(&format!("{sender_user} made {} public to the room directory", body.room_id)) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_text(&format!("{sender_user} made {} public to the room directory", body.room_id)) + .await; + } info!("{sender_user} made {0} public to the room directory", body.room_id); }, room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id)?, diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index b3b3c80e..470db669 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -71,13 +71,16 @@ async fn banned_room_check( if services.globals.config.auto_deactivate_banned_room_attempts { warn!("Automatically deactivating user {user_id} due to attempted banned room join"); - services - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "Automatically deactivating user {user_id} due to attempted banned room join from IP \ - {client_ip}" - ))) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "Automatically deactivating user {user_id} due to attempted banned room join from IP \ + {client_ip}" + ))) + .await; + } let all_joined_rooms: Vec = services .rooms @@ -108,13 +111,16 @@ async fn banned_room_check( if services.globals.config.auto_deactivate_banned_room_attempts { warn!("Automatically deactivating user {user_id} due to attempted banned room join"); - services - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "Automatically deactivating user {user_id} due to attempted banned room join from IP \ - {client_ip}" - ))) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "Automatically deactivating user {user_id} due to attempted banned room join from IP \ + {client_ip}" + ))) + .await; + } let all_joined_rooms: Vec = services .rooms diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 0a75787d..0112e76d 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -103,14 +103,17 @@ pub(crate) async fn create_room_route( \"lockdown_public_room_directory\" is enabled", &room_id ); - services - .admin - .send_text(&format!( - "Non-admin user {sender_user} tried to publish {0} to the room directory while \ - \"lockdown_public_room_directory\" is enabled", - &room_id - )) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_text(&format!( + "Non-admin user {sender_user} tried to publish {0} to the room directory while \ + \"lockdown_public_room_directory\" is enabled", + &room_id + )) + .await; + } return Err!(Request(Forbidden("Publishing rooms to the room directory is not allowed"))); } @@ -473,10 +476,13 @@ pub(crate) async fn create_room_route( if body.visibility == room::Visibility::Public { services.rooms.directory.set_public(&room_id)?; - services - .admin - .send_text(&format!("{sender_user} made {} public to the room directory", &room_id)) - .await; + + if services.globals.config.admin_room_notices { + services + .admin + .send_text(&format!("{sender_user} made {} public to the room directory", &room_id)) + .await; + } info!("{sender_user} made {0} public to the room directory", &room_id); } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index bb520d06..d5ed845b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -377,6 +377,13 @@ pub struct Config { #[serde(default)] pub test: BTreeSet, + /// Controls whether admin room notices like account registrations, password + /// changes, account deactivations, room directory publications, etc will + /// be sent to the admin room. Update notices and normal admin command + /// responses will still be sent. + #[serde(default = "true_fn")] + pub admin_room_notices: bool, + #[serde(flatten)] #[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime catchall: BTreeMap, @@ -867,6 +874,7 @@ impl fmt::Display for Config { .map_or("", |url| url.as_str()), ); line("Enable the tokio-console", &self.tokio_console.to_string()); + line("Admin room notices", &self.admin_room_notices.to_string()); Ok(()) } From 4d7bbe9fb463014ae9ab7d1df15e5a059d5d8800 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 20 Sep 2024 23:34:49 -0400 Subject: [PATCH 0829/2091] tiny micro-optimisations in some config stuff Signed-off-by: strawberry --- src/core/config/mod.rs | 20 ++++++++++++-------- src/service/globals/mod.rs | 15 +++++++++------ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index d5ed845b..d2d583a8 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -473,7 +473,11 @@ impl Config { #[must_use] pub fn get_bind_addrs(&self) -> Vec { - let mut addrs = Vec::new(); + let mut addrs = Vec::with_capacity( + self.get_bind_hosts() + .len() + .saturating_add(self.get_bind_ports().len()), + ); for host in &self.get_bind_hosts() { for port in &self.get_bind_ports() { addrs.push(SocketAddr::new(*host, *port)); @@ -704,7 +708,7 @@ impl fmt::Display for Config { }); line("Turn TTL", &self.turn_ttl.to_string()); line("Turn URIs", { - let mut lst = vec![]; + let mut lst = Vec::with_capacity(self.turn_uris.len()); for item in self.turn_uris.iter().cloned().enumerate() { let (_, uri): (usize, String) = item; lst.push(uri); @@ -712,7 +716,7 @@ impl fmt::Display for Config { &lst.join(", ") }); line("Auto Join Rooms", { - let mut lst = vec![]; + let mut lst = Vec::with_capacity(self.auto_join_rooms.len()); for room in &self.auto_join_rooms { lst.push(room); } @@ -764,28 +768,28 @@ impl fmt::Display for Config { line("Allow legacy (unauthenticated) media", &self.allow_legacy_media.to_string()); line("Freeze legacy (unauthenticated) media", &self.freeze_legacy_media.to_string()); line("Prevent Media Downloads From", { - let mut lst = vec![]; + let mut lst = Vec::with_capacity(self.prevent_media_downloads_from.len()); for domain in &self.prevent_media_downloads_from { lst.push(domain.host()); } &lst.join(", ") }); line("Forbidden Remote Server Names (\"Global\" ACLs)", { - let mut lst = vec![]; + let mut lst = Vec::with_capacity(self.forbidden_remote_server_names.len()); for domain in &self.forbidden_remote_server_names { lst.push(domain.host()); } &lst.join(", ") }); line("Forbidden Remote Room Directory Server Names", { - let mut lst = vec![]; + let mut lst = Vec::with_capacity(self.forbidden_remote_room_directory_server_names.len()); for domain in &self.forbidden_remote_room_directory_server_names { lst.push(domain.host()); } &lst.join(", ") }); - line("Outbound Request IP Range Denylist", { - let mut lst = vec![]; + line("Outbound Request IP Range (CIDR) Denylist", { + let mut lst = Vec::with_capacity(self.ip_range_denylist.len()); for item in self.ip_range_denylist.iter().cloned().enumerate() { let (_, ip): (usize, String) = item; lst.push(ip); diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index a044d30a..87f8f492 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -275,12 +275,15 @@ impl Service { pub fn block_non_admin_invites(&self) -> bool { self.config.block_non_admin_invites } pub fn supported_room_versions(&self) -> Vec { - let mut room_versions: Vec = Vec::with_capacity(self.stable_room_versions.len()); - room_versions.extend(self.stable_room_versions.clone()); - if self.allow_unstable_room_versions() { - room_versions.extend(self.unstable_room_versions.clone()); - }; - room_versions + if self.config.allow_unstable_room_versions { + self.stable_room_versions + .clone() + .into_iter() + .chain(self.unstable_room_versions.clone()) + .collect() + } else { + self.stable_room_versions.clone() + } } /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found From f6918833d76757aad9d1daa66fbb4e259cd1e573 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 23:03:36 -0400 Subject: [PATCH 0830/2091] remove `-unknown-` from nix flake Signed-off-by: strawberry --- .github/workflows/ci.yml | 30 +++++++++++++++++++++-------- .github/workflows/documentation.yml | 2 +- .gitlab-ci.yml | 20 +++++++++---------- bin/complement | 2 +- docs/deploying/generic.md | 4 ++-- 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09b5cd52..4d299514 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -236,8 +236,8 @@ jobs: strategy: matrix: include: - - target: aarch64-unknown-linux-musl - - target: x86_64-unknown-linux-musl + - target: aarch64-linux-musl + - target: x86_64-linux-musl steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -311,7 +311,14 @@ jobs: - name: Build static ${{ matrix.target }} run: | - CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features @@ -335,7 +342,14 @@ jobs: - name: Build static debug ${{ matrix.target }} run: | - CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug @@ -542,10 +556,10 @@ jobs: - name: Move OCI images into position run: | - mv -v oci-image-x86_64-unknown-linux-musl/*.tar.gz oci-image-amd64.tar.gz - mv -v oci-image-aarch64-unknown-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz - mv -v oci-image-x86_64-unknown-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz - mv -v oci-image-aarch64-unknown-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz + mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz + mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz + mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz + mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz - name: Load and push amd64 image if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 45e52b22..506a87d9 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -60,7 +60,7 @@ jobs: if: github.event_name != 'pull_request' uses: actions/configure-pages@v5 - - uses: nixbuild/nix-quick-install-action@v28 + - uses: nixbuild/nix-quick-install-action@master - name: Restore and cache Nix store uses: nix-community/cache-nix-action@v5.1.0 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 895b794b..c9f239d6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -97,29 +97,29 @@ artifacts: stage: artifacts image: nixos/nix:2.24.9 script: - - ./bin/nix-build-and-cache just .#static-x86_64-unknown-linux-musl - - cp result/bin/conduit x86_64-unknown-linux-musl + - ./bin/nix-build-and-cache just .#static-x86_64-linux-musl + - cp result/bin/conduit x86_64-linux-musl - mkdir -p target/release - cp result/bin/conduit target/release - direnv exec . cargo deb --no-build --no-strip - - mv target/debian/*.deb x86_64-unknown-linux-musl.deb + - mv target/debian/*.deb x86_64-linux-musl.deb # Since the OCI image package is based on the binary package, this has the # fun side effect of uploading the normal binary too. Conduit users who are # deploying with Nix can leverage this fact by adding our binary cache to # their systems. # - # Note that although we have an `oci-image-x86_64-unknown-linux-musl` + # Note that although we have an `oci-image-x86_64-linux-musl` # output, we don't build it because it would be largely redundant to this # one since it's all containerized anyway. - ./bin/nix-build-and-cache just .#oci-image - cp result oci-image-amd64.tar.gz - - ./bin/nix-build-and-cache just .#static-aarch64-unknown-linux-musl - - cp result/bin/conduit aarch64-unknown-linux-musl + - ./bin/nix-build-and-cache just .#static-aarch64-linux-musl + - cp result/bin/conduit aarch64-linux-musl - - ./bin/nix-build-and-cache just .#oci-image-aarch64-unknown-linux-musl + - ./bin/nix-build-and-cache just .#oci-image-aarch64-linux-musl - cp result oci-image-arm64v8.tar.gz - ./bin/nix-build-and-cache just .#book @@ -127,9 +127,9 @@ artifacts: - cp -r --dereference result public artifacts: paths: - - x86_64-unknown-linux-musl - - aarch64-unknown-linux-musl - - x86_64-unknown-linux-musl.deb + - x86_64-linux-musl + - aarch64-linux-musl + - x86_64-linux-musl.deb - oci-image-amd64.tar.gz - oci-image-arm64v8.tar.gz - public diff --git a/bin/complement b/bin/complement index 094cf248..601edb5a 100755 --- a/bin/complement +++ b/bin/complement @@ -34,7 +34,7 @@ toplevel="$(git rev-parse --show-toplevel)" pushd "$toplevel" > /dev/null -bin/nix-build-and-cache just .#static-complement +bin/nix-build-and-cache just .#linux-complement docker load < result popd > /dev/null diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 562ebd7e..1e44ab54 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -23,8 +23,8 @@ Nix (or [Lix](https://lix.systems)) to build conduwuit as this has the most guar reproducibiltiy and easiest to get a build environment and output going. This also allows easy cross-compilation. -You can run the `nix build -L .#static-x86_64-unknown-linux-musl-all-features` or -`nix build -L .#static-aarch64-unknown-linux-musl-all-features` commands based +You can run the `nix build -L .#static-x86_64-linux-musl-all-features` or +`nix build -L .#static-aarch64-linux-musl-all-features` commands based on architecture to cross-compile the necessary static binary located at `result/bin/conduit`. This is reproducible with the static binaries produced in our CI. From 7e828440f948ce38005105dd498f0e1f9048c02b Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 23:44:15 -0400 Subject: [PATCH 0831/2091] allow conduit database version 16 Conduit bumped the database version to 16, but did not introduce any breaking changes. Their database migrations are extremely fragile and risky, and also do not really apply to us, so just to retain Conduit -> conduwuit compatibility we'll check for both versions. Signed-off-by: strawberry --- src/service/globals/migrations.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index a52c6f00..66917520 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -24,6 +24,14 @@ use crate::{media, Services}; /// equal or lesser version. These are expected to be backward-compatible. pub(crate) const DATABASE_VERSION: u64 = 13; +/// Conduit's database version. +/// +/// Conduit bumped the database version to 16, but did not introduce any +/// breaking changes. Their database migrations are extremely fragile and risky, +/// and also do not really apply to us, so just to retain Conduit -> conduwuit +/// compatibility we'll check for both versions. +pub(crate) const CONDUIT_DATABASE_VERSION: u64 = 16; + pub(crate) async fn migrations(services: &Services) -> Result<()> { // Matrix resource ownership is based on the server name; changing it // requires recreating the database from scratch. @@ -148,9 +156,11 @@ async fn migrate(services: &Services) -> Result<()> { retroactively_fix_bad_data_from_roomuserid_joined(services).await?; } - assert_eq!( - services.globals.db.database_version().unwrap(), - DATABASE_VERSION, + let version_match = services.globals.db.database_version().unwrap() == DATABASE_VERSION + || services.globals.db.database_version().unwrap() == CONDUIT_DATABASE_VERSION; + + assert!( + version_match, "Failed asserting local database version {} is equal to known latest conduwuit database version {}", services.globals.db.database_version().unwrap(), DATABASE_VERSION, From 16014e15946132160a40fa057387aadbea1550e4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 24 Oct 2024 23:51:02 -0400 Subject: [PATCH 0832/2091] remove `ProcSubset=pid` from systemd units for now they appear to cause strange rust malloc issues on Debian systems Signed-off-by: strawberry --- arch/conduwuit.service | 2 +- debian/conduwuit.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index e2040a18..b343c91a 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -15,7 +15,7 @@ DevicePolicy=closed LockPersonality=yes MemoryDenyWriteExecute=yes NoNewPrivileges=yes -ProcSubset=pid +#ProcSubset=pid ProtectClock=yes ProtectControlGroups=yes ProtectHome=yes diff --git a/debian/conduwuit.service b/debian/conduwuit.service index ef59f098..d1b0d5d6 100644 --- a/debian/conduwuit.service +++ b/debian/conduwuit.service @@ -22,7 +22,7 @@ DevicePolicy=closed LockPersonality=yes MemoryDenyWriteExecute=yes NoNewPrivileges=yes -ProcSubset=pid +#ProcSubset=pid ProtectClock=yes ProtectControlGroups=yes ProtectHome=yes From 2db017af37dfedd49b854fb431ef629b8157eefc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 30 Aug 2024 00:26:23 +0000 Subject: [PATCH 0833/2091] simplify service trait bounds and lifetimes Signed-off-by: Jason Volk --- src/service/service.rs | 23 +++++++++-------------- src/service/services.rs | 8 ++++---- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/service/service.rs b/src/service/service.rs index 635f782e..065f78a0 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -51,7 +51,7 @@ pub(crate) struct Args<'a> { /// Dep is a reference to a service used within another service. /// Circular-dependencies between services require this indirection. -pub(crate) struct Dep { +pub(crate) struct Dep { dep: OnceLock>, service: Weak, name: &'static str, @@ -62,7 +62,7 @@ pub(crate) type MapType = BTreeMap; pub(crate) type MapVal = (Weak, Weak); pub(crate) type MapKey = String; -impl Deref for Dep { +impl Deref for Dep { type Target = Arc; /// Dereference a dependency. The dependency must be ready or panics. @@ -80,7 +80,7 @@ impl Deref for Dep { impl<'a> Args<'a> { /// Create a lazy-reference to a service when constructing another Service. - pub(crate) fn depend(&'a self, name: &'static str) -> Dep { + pub(crate) fn depend(&'a self, name: &'static str) -> Dep { Dep:: { dep: OnceLock::new(), service: Arc::downgrade(self.service), @@ -90,17 +90,12 @@ impl<'a> Args<'a> { /// Create a reference immediately to a service when constructing another /// Service. The other service must be constructed. - pub(crate) fn require(&'a self, name: &'static str) -> Arc { - require::(self.service, name) - } + pub(crate) fn require(&'a self, name: &str) -> Arc { require::(self.service, name) } } /// Reference a Service by name. Panics if the Service does not exist or was /// incorrectly cast. -pub(crate) fn require<'a, 'b, T>(map: &'b Map, name: &'a str) -> Arc -where - T: Send + Sync + 'a + 'b + 'static, -{ +pub(crate) fn require(map: &Map, name: &str) -> Arc { try_get::(map, name) .inspect_err(inspect_log) .expect("Failure to reference service required by another service.") @@ -112,9 +107,9 @@ where /// # Panics /// Incorrect type is not a silent failure (None) as the type never has a reason /// to be incorrect. -pub(crate) fn get<'a, 'b, T>(map: &'b Map, name: &'a str) -> Option> +pub(crate) fn get(map: &Map, name: &str) -> Option> where - T: Send + Sync + 'a + 'b + 'static, + T: Any + Send + Sync + Sized, { map.read() .expect("locked for reading") @@ -129,9 +124,9 @@ where /// Reference a Service by name. Returns Err if the Service does not exist or /// was incorrectly cast. -pub(crate) fn try_get<'a, 'b, T>(map: &'b Map, name: &'a str) -> Result> +pub(crate) fn try_get(map: &Map, name: &str) -> Result> where - T: Send + Sync + 'a + 'b + 'static, + T: Any + Send + Sync + Sized, { map.read() .expect("locked for reading") diff --git a/src/service/services.rs b/src/service/services.rs index 8e69cdbb..3aa095b8 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -193,16 +193,16 @@ impl Services { } } - pub fn try_get<'a, 'b, T>(&'b self, name: &'a str) -> Result> + pub fn try_get(&self, name: &str) -> Result> where - T: Send + Sync + 'a + 'b + 'static, + T: Any + Send + Sync + Sized, { service::try_get::(&self.service, name) } - pub fn get<'a, 'b, T>(&'b self, name: &'a str) -> Option> + pub fn get(&self, name: &str) -> Option> where - T: Send + Sync + 'a + 'b + 'static, + T: Any + Send + Sync + Sized, { service::get::(&self.service, name) } From 99ad404ea9f72b3a4d7aabb55a17127d82f39d12 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 31 Aug 2024 02:13:23 +0000 Subject: [PATCH 0834/2091] add str traits for split, between, unquote; consolidate tests Signed-off-by: Jason Volk --- src/core/utils/string.rs | 13 +++--- src/core/utils/string/between.rs | 26 ++++++++++++ src/core/utils/string/split.rs | 22 ++++++++++ src/core/utils/string/tests.rs | 70 +++++++++++++++++++++++++++++++ src/core/utils/string/unquote.rs | 33 +++++++++++++++ src/core/utils/string/unquoted.rs | 52 +++++++++++++++++++++++ src/core/utils/tests.rs | 43 ------------------- src/service/service.rs | 4 +- 8 files changed, 212 insertions(+), 51 deletions(-) create mode 100644 src/core/utils/string/between.rs create mode 100644 src/core/utils/string/split.rs create mode 100644 src/core/utils/string/tests.rs create mode 100644 src/core/utils/string/unquote.rs create mode 100644 src/core/utils/string/unquoted.rs diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 85282b30..e65a3369 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -1,3 +1,10 @@ +mod between; +mod split; +mod tests; +mod unquote; +mod unquoted; + +pub use self::{between::Between, split::SplitInfallible, unquote::Unquote, unquoted::Unquoted}; use crate::{utils::exchange, Result}; pub const EMPTY: &str = ""; @@ -95,12 +102,6 @@ pub fn common_prefix<'a>(choice: &'a [&str]) -> &'a str { }) } -#[inline] -#[must_use] -pub fn split_once_infallible<'a>(input: &'a str, delim: &'_ str) -> (&'a str, &'a str) { - input.split_once(delim).unwrap_or((input, EMPTY)) -} - /// Parses the bytes into a string. pub fn string_from_bytes(bytes: &[u8]) -> Result { let str: &str = str_from_bytes(bytes)?; diff --git a/src/core/utils/string/between.rs b/src/core/utils/string/between.rs new file mode 100644 index 00000000..209a9dab --- /dev/null +++ b/src/core/utils/string/between.rs @@ -0,0 +1,26 @@ +type Delim<'a> = (&'a str, &'a str); + +/// Slice a string between a pair of delimeters. +pub trait Between<'a> { + /// Extract a string between the delimeters. If the delimeters were not + /// found None is returned, otherwise the first extraction is returned. + fn between(&self, delim: Delim<'_>) -> Option<&'a str>; + + /// Extract a string between the delimeters. If the delimeters were not + /// found the original string is returned; take note of this behavior, + /// if an empty slice is desired for this case use the fallible version and + /// unwrap to EMPTY. + fn between_infallible(&self, delim: Delim<'_>) -> &'a str; +} + +impl<'a> Between<'a> for &'a str { + #[inline] + fn between_infallible(&self, delim: Delim<'_>) -> &'a str { self.between(delim).unwrap_or(self) } + + #[inline] + fn between(&self, delim: Delim<'_>) -> Option<&'a str> { + self.split_once(delim.0) + .and_then(|(_, b)| b.rsplit_once(delim.1)) + .map(|(a, _)| a) + } +} diff --git a/src/core/utils/string/split.rs b/src/core/utils/string/split.rs new file mode 100644 index 00000000..96de28df --- /dev/null +++ b/src/core/utils/string/split.rs @@ -0,0 +1,22 @@ +use super::EMPTY; + +type Pair<'a> = (&'a str, &'a str); + +/// Split a string with default behaviors on non-match. +pub trait SplitInfallible<'a> { + /// Split a string at the first occurrence of delim. If not found, the + /// entire string is returned in \[0\], while \[1\] is empty. + fn split_once_infallible(&self, delim: &str) -> Pair<'a>; + + /// Split a string from the last occurrence of delim. If not found, the + /// entire string is returned in \[0\], while \[1\] is empty. + fn rsplit_once_infallible(&self, delim: &str) -> Pair<'a>; +} + +impl<'a> SplitInfallible<'a> for &'a str { + #[inline] + fn rsplit_once_infallible(&self, delim: &str) -> Pair<'a> { self.rsplit_once(delim).unwrap_or((self, EMPTY)) } + + #[inline] + fn split_once_infallible(&self, delim: &str) -> Pair<'a> { self.split_once(delim).unwrap_or((self, EMPTY)) } +} diff --git a/src/core/utils/string/tests.rs b/src/core/utils/string/tests.rs new file mode 100644 index 00000000..e8c17de6 --- /dev/null +++ b/src/core/utils/string/tests.rs @@ -0,0 +1,70 @@ +#![cfg(test)] + +#[test] +fn common_prefix() { + let input = ["conduwuit", "conduit", "construct"]; + let output = super::common_prefix(&input); + assert_eq!(output, "con"); +} + +#[test] +fn common_prefix_empty() { + let input = ["abcdefg", "hijklmn", "opqrstu"]; + let output = super::common_prefix(&input); + assert_eq!(output, ""); +} + +#[test] +fn common_prefix_none() { + let input = []; + let output = super::common_prefix(&input); + assert_eq!(output, ""); +} + +#[test] +fn camel_to_snake_case_0() { + let res = super::camel_to_snake_string("CamelToSnakeCase"); + assert_eq!(res, "camel_to_snake_case"); +} + +#[test] +fn camel_to_snake_case_1() { + let res = super::camel_to_snake_string("CAmelTOSnakeCase"); + assert_eq!(res, "camel_tosnake_case"); +} + +#[test] +fn unquote() { + use super::Unquote; + + assert_eq!("\"foo\"".unquote(), Some("foo")); + assert_eq!("\"foo".unquote(), None); + assert_eq!("foo".unquote(), None); +} + +#[test] +fn unquote_infallible() { + use super::Unquote; + + assert_eq!("\"foo\"".unquote_infallible(), "foo"); + assert_eq!("\"foo".unquote_infallible(), "\"foo"); + assert_eq!("foo".unquote_infallible(), "foo"); +} + +#[test] +fn between() { + use super::Between; + + assert_eq!("\"foo\"".between(("\"", "\"")), Some("foo")); + assert_eq!("\"foo".between(("\"", "\"")), None); + assert_eq!("foo".between(("\"", "\"")), None); +} + +#[test] +fn between_infallible() { + use super::Between; + + assert_eq!("\"foo\"".between_infallible(("\"", "\"")), "foo"); + assert_eq!("\"foo".between_infallible(("\"", "\"")), "\"foo"); + assert_eq!("foo".between_infallible(("\"", "\"")), "foo"); +} diff --git a/src/core/utils/string/unquote.rs b/src/core/utils/string/unquote.rs new file mode 100644 index 00000000..eeded610 --- /dev/null +++ b/src/core/utils/string/unquote.rs @@ -0,0 +1,33 @@ +const QUOTE: char = '"'; + +/// Slice a string between quotes +pub trait Unquote<'a> { + /// Whether the input is quoted. If this is false the fallible methods of + /// this interface will fail. + fn is_quoted(&self) -> bool; + + /// Unquotes a string. If the input is not quoted it is simply returned + /// as-is. If the input is partially quoted on either end that quote is not + /// removed. + fn unquote(&self) -> Option<&'a str>; + + /// Unquotes a string. The input must be quoted on each side for Some to be + /// returned + fn unquote_infallible(&self) -> &'a str; +} + +impl<'a> Unquote<'a> for &'a str { + #[inline] + fn unquote_infallible(&self) -> &'a str { + self.strip_prefix(QUOTE) + .unwrap_or(self) + .strip_suffix(QUOTE) + .unwrap_or(self) + } + + #[inline] + fn unquote(&self) -> Option<&'a str> { self.strip_prefix(QUOTE).and_then(|s| s.strip_suffix(QUOTE)) } + + #[inline] + fn is_quoted(&self) -> bool { self.starts_with(QUOTE) && self.ends_with(QUOTE) } +} diff --git a/src/core/utils/string/unquoted.rs b/src/core/utils/string/unquoted.rs new file mode 100644 index 00000000..5b002d99 --- /dev/null +++ b/src/core/utils/string/unquoted.rs @@ -0,0 +1,52 @@ +use std::ops::Deref; + +use serde::{de, Deserialize, Deserializer}; + +use super::Unquote; +use crate::{err, Result}; + +/// Unquoted string which deserialized from a quoted string. Construction from a +/// &str is infallible such that the input can already be unquoted. Construction +/// from serde deserialization is fallible and the input must be quoted. +#[repr(transparent)] +pub struct Unquoted(str); + +impl<'a> Unquoted { + #[inline] + #[must_use] + pub fn as_str(&'a self) -> &'a str { &self.0 } +} + +impl<'a, 'de: 'a> Deserialize<'de> for &'a Unquoted { + fn deserialize>(deserializer: D) -> Result { + let s = <&'a str>::deserialize(deserializer)?; + s.is_quoted() + .then_some(s) + .ok_or(err!(SerdeDe("expected quoted string"))) + .map_err(de::Error::custom) + .map(Into::into) + } +} + +impl<'a> From<&'a str> for &'a Unquoted { + fn from(s: &'a str) -> &'a Unquoted { + let s: &'a str = s.unquote_infallible(); + + //SAFETY: This is a pattern I lifted from ruma-identifiers for strong-type strs + // by wrapping in a tuple-struct. + #[allow(clippy::transmute_ptr_to_ptr)] + unsafe { + std::mem::transmute(s) + } + } +} + +impl Deref for Unquoted { + type Target = str; + + fn deref(&self) -> &Self::Target { &self.0 } +} + +impl<'a> AsRef for &'a Unquoted { + fn as_ref(&self) -> &'a str { &self.0 } +} diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index e91accdf..5880470a 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -36,33 +36,6 @@ fn increment_wrap() { assert_eq!(res, 0); } -#[test] -fn common_prefix() { - use utils::string; - - let input = ["conduwuit", "conduit", "construct"]; - let output = string::common_prefix(&input); - assert_eq!(output, "con"); -} - -#[test] -fn common_prefix_empty() { - use utils::string; - - let input = ["abcdefg", "hijklmn", "opqrstu"]; - let output = string::common_prefix(&input); - assert_eq!(output, ""); -} - -#[test] -fn common_prefix_none() { - use utils::string; - - let input = []; - let output = string::common_prefix(&input); - assert_eq!(output, ""); -} - #[test] fn checked_add() { use crate::checked; @@ -134,19 +107,3 @@ async fn mutex_map_contend() { tokio::try_join!(join_b, join_a).expect("joined"); assert!(map.is_empty(), "Must be empty"); } - -#[test] -fn camel_to_snake_case_0() { - use utils::string::camel_to_snake_string; - - let res = camel_to_snake_string("CamelToSnakeCase"); - assert_eq!(res, "camel_to_snake_case"); -} - -#[test] -fn camel_to_snake_case_1() { - use utils::string::camel_to_snake_string; - - let res = camel_to_snake_string("CAmelTOSnakeCase"); - assert_eq!(res, "camel_tosnake_case"); -} diff --git a/src/service/service.rs b/src/service/service.rs index 065f78a0..03165050 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -7,7 +7,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{err, error::inspect_log, utils::string::split_once_infallible, Err, Result, Server}; +use conduit::{err, error::inspect_log, utils::string::SplitInfallible, Err, Result, Server}; use database::Database; /// Abstract interface for a Service @@ -147,4 +147,4 @@ where /// Utility for service implementations; see Service::name() in the trait. #[inline] -pub(crate) fn make_name(module_path: &str) -> &str { split_once_infallible(module_path, "::").1 } +pub(crate) fn make_name(module_path: &str) -> &str { module_path.split_once_infallible("::").1 } From 2709995f84cfa9dcaab14ac9ee856aafe06b22c0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 1 Sep 2024 01:53:22 +0000 Subject: [PATCH 0835/2091] add MapExpect to Result add DebugInspect to Result move Result typedef into unit Signed-off-by: Jason Volk --- src/core/debug.rs | 6 ++-- src/core/mod.rs | 4 +-- src/core/result.rs | 6 ++++ src/core/result/debug_inspect.rs | 52 ++++++++++++++++++++++++++++++++ src/core/result/map_expect.rs | 15 +++++++++ 5 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 src/core/result.rs create mode 100644 src/core/result/debug_inspect.rs create mode 100644 src/core/result/map_expect.rs diff --git a/src/core/debug.rs b/src/core/debug.rs index 844445d5..1e36ca8e 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -1,10 +1,10 @@ use std::{any::Any, panic}; -/// Export debug proc_macros +// Export debug proc_macros pub use conduit_macros::recursion_depth; -/// Export all of the ancillary tools from here as well. -pub use crate::utils::debug::*; +// Export all of the ancillary tools from here as well. +pub use crate::{result::DebugInspect, utils::debug::*}; /// Log event at given level in debug-mode (when debug-assertions are enabled). /// In release-mode it becomes DEBUG level, and possibly subject to elision. diff --git a/src/core/mod.rs b/src/core/mod.rs index 9898243b..31851f4f 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -7,6 +7,7 @@ pub mod log; pub mod metrics; pub mod mods; pub mod pdu; +pub mod result; pub mod server; pub mod utils; @@ -15,13 +16,12 @@ pub use config::Config; pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; pub use pdu::{PduBuilder, PduCount, PduEvent}; +pub use result::Result; pub use server::Server; pub use utils::{ctor, dtor, implement}; pub use crate as conduit_core; -pub type Result = std::result::Result; - rustc_flags_capture! {} #[cfg(not(conduit_mods))] diff --git a/src/core/result.rs b/src/core/result.rs new file mode 100644 index 00000000..d58467cf --- /dev/null +++ b/src/core/result.rs @@ -0,0 +1,6 @@ +mod debug_inspect; +mod map_expect; + +pub use self::{debug_inspect::DebugInspect, map_expect::MapExpect}; + +pub type Result = std::result::Result; diff --git a/src/core/result/debug_inspect.rs b/src/core/result/debug_inspect.rs new file mode 100644 index 00000000..ef80979d --- /dev/null +++ b/src/core/result/debug_inspect.rs @@ -0,0 +1,52 @@ +use super::Result; + +/// Inspect Result values with release-mode elision. +pub trait DebugInspect { + /// Inspects an Err contained value in debug-mode. In release-mode closure F + /// is elided. + #[must_use] + fn debug_inspect_err(self, f: F) -> Self; + + /// Inspects an Ok contained value in debug-mode. In release-mode closure F + /// is elided. + #[must_use] + fn debug_inspect(self, f: F) -> Self; +} + +#[cfg(debug_assertions)] +impl DebugInspect for Result { + #[inline] + fn debug_inspect(self, f: F) -> Self + where + F: FnOnce(&T), + { + self.inspect(f) + } + + #[inline] + fn debug_inspect_err(self, f: F) -> Self + where + F: FnOnce(&E), + { + self.inspect_err(f) + } +} + +#[cfg(not(debug_assertions))] +impl DebugInspect for Result { + #[inline] + fn debug_inspect(self, _: F) -> Self + where + F: FnOnce(&T), + { + self + } + + #[inline] + fn debug_inspect_err(self, _: F) -> Self + where + F: FnOnce(&E), + { + self + } +} diff --git a/src/core/result/map_expect.rs b/src/core/result/map_expect.rs new file mode 100644 index 00000000..8ce9195f --- /dev/null +++ b/src/core/result/map_expect.rs @@ -0,0 +1,15 @@ +use std::fmt::Debug; + +use super::Result; + +pub trait MapExpect { + /// Calls expect(msg) on the mapped Result value. This is similar to + /// map(Result::unwrap) but composes an expect call and message without + /// requiring a closure. + fn map_expect(self, msg: &str) -> Option; +} + +impl MapExpect for Option> { + #[inline] + fn map_expect(self, msg: &str) -> Option { self.map(|result| result.expect(msg)) } +} From 3d4b0f10a59008a5cc785fd299937cdbee0dacd1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 7 Sep 2024 22:04:28 +0000 Subject: [PATCH 0836/2091] add expected! macro to checked math expression suite Signed-off-by: Jason Volk --- src/core/utils/math.rs | 36 +++++++++++++++++++++-------- src/service/rooms/auth_chain/mod.rs | 2 +- src/service/rooms/timeline/mod.rs | 2 +- 3 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index f9d0de30..8c4b01be 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -7,32 +7,50 @@ use crate::{debug::type_name, err, Err, Error, Result}; /// Checked arithmetic expression. Returns a Result #[macro_export] macro_rules! checked { - ($($input:tt)*) => { - $crate::utils::math::checked_ops!($($input)*) + ($($input:tt)+) => { + $crate::utils::math::checked_ops!($($input)+) .ok_or_else(|| $crate::err!(Arithmetic("operation overflowed or result invalid"))) - } + }; } -/// in release-mode. Use for performance when the expression is obviously safe. -/// The check remains in debug-mode for regression analysis. +/// Checked arithmetic expression which panics on failure. This is for +/// expressions which do not meet the threshold for validated! but the caller +/// has no realistic expectation for error and no interest in cluttering the +/// callsite with result handling from checked!. +#[macro_export] +macro_rules! expected { + ($msg:literal, $($input:tt)+) => { + $crate::checked!($($input)+).expect($msg) + }; + + ($($input:tt)+) => { + $crate::expected!("arithmetic expression expectation failure", $($input)+) + }; +} + +/// Unchecked arithmetic expression in release-mode. Use for performance when +/// the expression is obviously safe. The check remains in debug-mode for +/// regression analysis. #[cfg(not(debug_assertions))] #[macro_export] macro_rules! validated { - ($($input:tt)*) => { + ($($input:tt)+) => { //#[allow(clippy::arithmetic_side_effects)] { //Some($($input)*) // .ok_or_else(|| $crate::err!(Arithmetic("this error should never been seen"))) //} //NOTE: remove me when stmt_expr_attributes is stable - $crate::checked!($($input)*) - } + $crate::expected!("validated arithmetic expression failed", $($input)+) + }; } +/// Checked arithmetic expression in debug-mode. Use for performance when +/// the expression is obviously safe. The check is elided in release-mode. #[cfg(debug_assertions)] #[macro_export] macro_rules! validated { - ($($input:tt)*) => { $crate::checked!($($input)*) } + ($($input:tt)+) => { $crate::expected!($($input)+) } } /// Returns false if the exponential backoff has expired based on the inputs diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 9a1e7e67..d0bc425f 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -66,7 +66,7 @@ impl Service { .enumerate() { let bucket: usize = short.try_into()?; - let bucket: usize = validated!(bucket % NUM_BUCKETS)?; + let bucket: usize = validated!(bucket % NUM_BUCKETS); buckets[bucket].insert((short, starting_events[i])); } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 4f2352f8..04d9559d 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1205,7 +1205,7 @@ impl Service { let count = self.services.globals.next_count()?; let mut pdu_id = shortroomid.to_be_bytes().to_vec(); pdu_id.extend_from_slice(&0_u64.to_be_bytes()); - pdu_id.extend_from_slice(&(validated!(max - count)?).to_be_bytes()); + pdu_id.extend_from_slice(&(validated!(max - count)).to_be_bytes()); // Insert pdu self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value)?; From aa265f7ca4ee5f6b15cce83a235cf5f9c4317cfc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Sep 2024 04:39:27 +0000 Subject: [PATCH 0837/2091] add err log trait to Result Signed-off-by: Jason Volk --- src/core/error/log.rs | 33 +++++++++++++++--- src/core/result.rs | 4 ++- src/core/result/inspect_log.rs | 60 ++++++++++++++++++++++++++++++++ src/core/result/log_debug_err.rs | 36 +++++++++++++++++++ src/core/result/log_err.rs | 36 +++++++++++++++++++ 5 files changed, 163 insertions(+), 6 deletions(-) create mode 100644 src/core/result/inspect_log.rs create mode 100644 src/core/result/log_debug_err.rs create mode 100644 src/core/result/log_err.rs diff --git a/src/core/error/log.rs b/src/core/error/log.rs index c272bf73..60bd7014 100644 --- a/src/core/error/log.rs +++ b/src/core/error/log.rs @@ -1,7 +1,8 @@ use std::{convert::Infallible, fmt}; +use tracing::Level; + use super::Error; -use crate::{debug_error, error}; #[inline] pub fn else_log(error: E) -> Result @@ -64,11 +65,33 @@ where } #[inline] -pub fn inspect_log(error: &E) { - error!("{error}"); +pub fn inspect_log(error: &E) { inspect_log_level(error, Level::ERROR); } + +#[inline] +pub fn inspect_debug_log(error: &E) { inspect_debug_log_level(error, Level::ERROR); } + +#[inline] +pub fn inspect_log_level(error: &E, level: Level) { + use crate::{debug, error, info, trace, warn}; + + match level { + Level::ERROR => error!("{error}"), + Level::WARN => warn!("{error}"), + Level::INFO => info!("{error}"), + Level::DEBUG => debug!("{error}"), + Level::TRACE => trace!("{error}"), + } } #[inline] -pub fn inspect_debug_log(error: &E) { - debug_error!("{error:?}"); +pub fn inspect_debug_log_level(error: &E, level: Level) { + use crate::{debug, debug_error, debug_info, debug_warn, trace}; + + match level { + Level::ERROR => debug_error!("{error:?}"), + Level::WARN => debug_warn!("{error:?}"), + Level::INFO => debug_info!("{error:?}"), + Level::DEBUG => debug!("{error:?}"), + Level::TRACE => trace!("{error:?}"), + } } diff --git a/src/core/result.rs b/src/core/result.rs index d58467cf..c3eaf95b 100644 --- a/src/core/result.rs +++ b/src/core/result.rs @@ -1,6 +1,8 @@ mod debug_inspect; +mod log_debug_err; +mod log_err; mod map_expect; -pub use self::{debug_inspect::DebugInspect, map_expect::MapExpect}; +pub use self::{debug_inspect::DebugInspect, log_debug_err::LogDebugErr, log_err::LogErr, map_expect::MapExpect}; pub type Result = std::result::Result; diff --git a/src/core/result/inspect_log.rs b/src/core/result/inspect_log.rs new file mode 100644 index 00000000..577761c5 --- /dev/null +++ b/src/core/result/inspect_log.rs @@ -0,0 +1,60 @@ +use std::fmt; + +use tracing::Level; + +use super::Result; +use crate::error; + +pub trait ErrLog +where + E: fmt::Display, +{ + fn log_err(self, level: Level) -> Self; + + fn err_log(self) -> Self + where + Self: Sized, + { + self.log_err(Level::ERROR) + } +} + +pub trait ErrDebugLog +where + E: fmt::Debug, +{ + fn log_err_debug(self, level: Level) -> Self; + + fn err_debug_log(self) -> Self + where + Self: Sized, + { + self.log_err_debug(Level::ERROR) + } +} + +impl ErrLog for Result +where + E: fmt::Display, +{ + #[inline] + fn log_err(self, level: Level) -> Self + where + Self: Sized, + { + self.inspect_err(|error| error::inspect_log_level(&error, level)) + } +} + +impl ErrDebugLog for Result +where + E: fmt::Debug, +{ + #[inline] + fn log_err_debug(self, level: Level) -> Self + where + Self: Sized, + { + self.inspect_err(|error| error::inspect_debug_log_level(&error, level)) + } +} diff --git a/src/core/result/log_debug_err.rs b/src/core/result/log_debug_err.rs new file mode 100644 index 00000000..be2000ae --- /dev/null +++ b/src/core/result/log_debug_err.rs @@ -0,0 +1,36 @@ +use std::fmt; + +use tracing::Level; + +use super::{DebugInspect, Result}; +use crate::error; + +pub trait LogDebugErr +where + E: fmt::Debug, +{ + #[must_use] + fn err_debug_log(self, level: Level) -> Self; + + #[inline] + #[must_use] + fn log_debug_err(self) -> Self + where + Self: Sized, + { + self.err_debug_log(Level::ERROR) + } +} + +impl LogDebugErr for Result +where + E: fmt::Debug, +{ + #[inline] + fn err_debug_log(self, level: Level) -> Self + where + Self: Sized, + { + self.debug_inspect_err(|error| error::inspect_debug_log_level(&error, level)) + } +} diff --git a/src/core/result/log_err.rs b/src/core/result/log_err.rs new file mode 100644 index 00000000..079571f5 --- /dev/null +++ b/src/core/result/log_err.rs @@ -0,0 +1,36 @@ +use std::fmt; + +use tracing::Level; + +use super::Result; +use crate::error; + +pub trait LogErr +where + E: fmt::Display, +{ + #[must_use] + fn err_log(self, level: Level) -> Self; + + #[inline] + #[must_use] + fn log_err(self) -> Self + where + Self: Sized, + { + self.err_log(Level::ERROR) + } +} + +impl LogErr for Result +where + E: fmt::Display, +{ + #[inline] + fn err_log(self, level: Level) -> Self + where + Self: Sized, + { + self.inspect_err(|error| error::inspect_log_level(&error, level)) + } +} From bd75ff65c96427429a0334b6d899f2b630fe5f8a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Sep 2024 06:53:15 +0000 Subject: [PATCH 0838/2091] move common_elements util into unit Signed-off-by: Jason Volk --- src/core/utils/algorithm.rs | 25 +++++++++++++++++++++++++ src/core/utils/mod.rs | 28 +++------------------------- 2 files changed, 28 insertions(+), 25 deletions(-) create mode 100644 src/core/utils/algorithm.rs diff --git a/src/core/utils/algorithm.rs b/src/core/utils/algorithm.rs new file mode 100644 index 00000000..9bc1bc8a --- /dev/null +++ b/src/core/utils/algorithm.rs @@ -0,0 +1,25 @@ +use std::cmp::Ordering; + +#[allow(clippy::impl_trait_in_params)] +pub fn common_elements( + mut iterators: impl Iterator>>, check_order: impl Fn(&[u8], &[u8]) -> Ordering, +) -> Option>> { + let first_iterator = iterators.next()?; + let mut other_iterators = iterators.map(Iterator::peekable).collect::>(); + + Some(first_iterator.filter(move |target| { + other_iterators.iter_mut().all(|it| { + while let Some(element) = it.peek() { + match check_order(element, target) { + Ordering::Greater => return false, // We went too far + Ordering::Equal => return true, // Element is in both iters + Ordering::Less => { + // Keep searching + it.next(); + }, + } + } + false + }) + })) +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 1556646e..29d0b87b 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -1,3 +1,4 @@ +pub mod algorithm; pub mod bytes; pub mod content_disposition; pub mod debug; @@ -13,9 +14,10 @@ pub mod sys; mod tests; pub mod time; -use std::cmp::{self, Ordering}; +use std::cmp; pub use ::ctor::{ctor, dtor}; +pub use algorithm::common_elements; pub use bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}; pub use conduit_macros::implement; pub use debug::slice_truncated as debug_slice_truncated; @@ -47,27 +49,3 @@ pub fn generate_keypair() -> Vec { ); value } - -#[allow(clippy::impl_trait_in_params)] -pub fn common_elements( - mut iterators: impl Iterator>>, check_order: impl Fn(&[u8], &[u8]) -> Ordering, -) -> Option>> { - let first_iterator = iterators.next()?; - let mut other_iterators = iterators.map(Iterator::peekable).collect::>(); - - Some(first_iterator.filter(move |target| { - other_iterators.iter_mut().all(|it| { - while let Some(element) = it.peek() { - match check_order(element, target) { - Ordering::Greater => return false, // We went too far - Ordering::Equal => return true, // Element is in both iters - Ordering::Less => { - // Keep searching - it.next(); - }, - } - } - false - }) - })) -} From 63053640f1e5789719b3f88f92b3006ae3caecf4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 13 Sep 2024 18:24:33 +0000 Subject: [PATCH 0839/2091] add util functors for is_zero/is_equal; move clamp to math utils Signed-off-by: Jason Volk --- src/core/utils/math.rs | 35 +++++++++++++++++++++++++++++++++++ src/core/utils/mod.rs | 12 ++---------- src/macros/utils.rs | 7 ++----- 3 files changed, 39 insertions(+), 15 deletions(-) diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index 8c4b01be..215de339 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -53,6 +53,38 @@ macro_rules! validated { ($($input:tt)+) => { $crate::expected!($($input)+) } } +/// Functor for equality to zero +#[macro_export] +macro_rules! is_zero { + () => { + $crate::is_matching!(0) + }; +} + +/// Functor for equality i.e. .is_some_and(is_equal!(2)) +#[macro_export] +macro_rules! is_equal_to { + ($val:expr) => { + |x| (x == $val) + }; +} + +/// Functor for less i.e. .is_some_and(is_less_than!(2)) +#[macro_export] +macro_rules! is_less_than { + ($val:expr) => { + |x| (x < $val) + }; +} + +/// Functor for matches! i.e. .is_some_and(is_matching!('A'..='Z')) +#[macro_export] +macro_rules! is_matching { + ($val:expr) => { + |x| matches!(x, $val) + }; +} + /// Returns false if the exponential backoff has expired based on the inputs #[inline] #[must_use] @@ -118,3 +150,6 @@ fn try_into_err, Src>(e: >::Error) -> Erro type_name::() )) } + +#[inline] +pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), max) } diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 29d0b87b..03b755e9 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -14,8 +14,6 @@ pub mod sys; mod tests; pub mod time; -use std::cmp; - pub use ::ctor::{ctor, dtor}; pub use algorithm::common_elements; pub use bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}; @@ -24,6 +22,7 @@ pub use debug::slice_truncated as debug_slice_truncated; pub use hash::calculate_hash; pub use html::Escape as HtmlEscape; pub use json::{deserialize_from_str, to_canonical_object}; +pub use math::clamp; pub use mutex_map::{Guard as MutexMapGuard, MutexMap}; pub use rand::string as random_string; pub use string::{str_from_bytes, string_from_bytes}; @@ -31,14 +30,7 @@ pub use sys::available_parallelism; pub use time::now_millis as millis_since_unix_epoch; #[inline] -pub fn clamp(val: T, min: T, max: T) -> T { cmp::min(cmp::max(val, min), max) } - -#[inline] -pub fn exchange(state: &mut T, source: T) -> T { - let ret = state.clone(); - *state = source; - ret -} +pub fn exchange(state: &mut T, source: T) -> T { std::mem::replace(state, source) } #[must_use] pub fn generate_keypair() -> Vec { diff --git a/src/macros/utils.rs b/src/macros/utils.rs index 58074e3a..197dd90e 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -41,8 +41,5 @@ pub(crate) fn camel_to_snake_string(s: &str) -> String { output } -pub(crate) fn exchange(state: &mut T, source: T) -> T { - let ret = state.clone(); - *state = source; - ret -} +#[inline] +pub(crate) fn exchange(state: &mut T, source: T) -> T { std::mem::replace(state, source) } From a5822ebc274ddf92ba75f4f62d4e527078447baf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 13 Sep 2024 18:55:56 +0000 Subject: [PATCH 0840/2091] add missing err! case Signed-off-by: Jason Volk --- src/core/error/err.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/error/err.rs b/src/core/error/err.rs index b3d0240e..4972e92a 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -85,6 +85,10 @@ macro_rules! err { $crate::error::Error::$variant($crate::err_log!(buf, $level, $($args)+)) }}; + ($variant:ident($($args:ident),+)) => { + $crate::error::Error::$variant($($args),+) + }; + ($variant:ident($($args:tt)+)) => { $crate::error::Error::$variant($crate::format_maybe!($($args)+)) }; From f7ce4db0b00bc24be6c127895e16ba29a248c8a4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 12 Sep 2024 00:01:25 +0000 Subject: [PATCH 0841/2091] add is_not_found functor to error; tweak status code matcher Signed-off-by: Jason Volk --- src/core/error/mod.rs | 19 +++++++++++++------ src/core/result.rs | 6 +++++- src/core/result/not_found.rs | 12 ++++++++++++ 3 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 src/core/result/not_found.rs diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 92dbdfe3..48b9b58f 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -141,19 +141,22 @@ impl Error { use ruma::api::client::error::ErrorKind::Unknown; match self { - Self::Federation(_, error) => response::ruma_error_kind(error).clone(), + Self::Federation(_, error) | Self::Ruma(error) => response::ruma_error_kind(error).clone(), Self::BadRequest(kind, ..) | Self::Request(kind, ..) => kind.clone(), _ => Unknown, } } pub fn status_code(&self) -> http::StatusCode { + use http::StatusCode; + match self { - Self::Federation(_, ref error) | Self::Ruma(ref error) => error.status_code, - Self::Request(ref kind, _, code) => response::status_code(kind, *code), - Self::BadRequest(ref kind, ..) => response::bad_request_code(kind), - Self::Conflict(_) => http::StatusCode::CONFLICT, - _ => http::StatusCode::INTERNAL_SERVER_ERROR, + Self::Federation(_, error) | Self::Ruma(error) => error.status_code, + Self::Request(kind, _, code) => response::status_code(kind, *code), + Self::BadRequest(kind, ..) => response::bad_request_code(kind), + Self::Reqwest(error) => error.status().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + Self::Conflict(_) => StatusCode::CONFLICT, + _ => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -176,3 +179,7 @@ impl From for Error { pub fn infallible(_e: &Infallible) { panic!("infallible error should never exist"); } + +#[inline] +#[must_use] +pub fn is_not_found(e: &Error) -> bool { e.status_code() == http::StatusCode::NOT_FOUND } diff --git a/src/core/result.rs b/src/core/result.rs index c3eaf95b..41d1d66c 100644 --- a/src/core/result.rs +++ b/src/core/result.rs @@ -2,7 +2,11 @@ mod debug_inspect; mod log_debug_err; mod log_err; mod map_expect; +mod not_found; -pub use self::{debug_inspect::DebugInspect, log_debug_err::LogDebugErr, log_err::LogErr, map_expect::MapExpect}; +pub use self::{ + debug_inspect::DebugInspect, log_debug_err::LogDebugErr, log_err::LogErr, map_expect::MapExpect, + not_found::NotFound, +}; pub type Result = std::result::Result; diff --git a/src/core/result/not_found.rs b/src/core/result/not_found.rs new file mode 100644 index 00000000..69ce821b --- /dev/null +++ b/src/core/result/not_found.rs @@ -0,0 +1,12 @@ +use super::Result; +use crate::{error, Error}; + +pub trait NotFound { + #[must_use] + fn is_not_found(&self) -> bool; +} + +impl NotFound for Result { + #[inline] + fn is_not_found(&self) -> bool { self.as_ref().is_err_and(error::is_not_found) } +} From a5de27442a0c68a3ff2b86d15e82e5065884b787 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 12 Sep 2024 00:59:08 +0000 Subject: [PATCH 0842/2091] re-export crates used by error macros Signed-off-by: Jason Volk --- src/core/error/err.rs | 30 +++++++++++++++--------------- src/core/mod.rs | 3 +++ 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 4972e92a..82bb40b0 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -44,34 +44,34 @@ macro_rules! err { (Request(Forbidden($level:ident!($($args:tt)+)))) => {{ let mut buf = String::new(); $crate::error::Error::Request( - ::ruma::api::client::error::ErrorKind::forbidden(), + $crate::ruma::api::client::error::ErrorKind::forbidden(), $crate::err_log!(buf, $level, $($args)+), - ::http::StatusCode::BAD_REQUEST + $crate::http::StatusCode::BAD_REQUEST ) }}; (Request(Forbidden($($args:tt)+))) => { $crate::error::Error::Request( - ::ruma::api::client::error::ErrorKind::forbidden(), + $crate::ruma::api::client::error::ErrorKind::forbidden(), $crate::format_maybe!($($args)+), - ::http::StatusCode::BAD_REQUEST + $crate::http::StatusCode::BAD_REQUEST ) }; (Request($variant:ident($level:ident!($($args:tt)+)))) => {{ let mut buf = String::new(); $crate::error::Error::Request( - ::ruma::api::client::error::ErrorKind::$variant, + $crate::ruma::api::client::error::ErrorKind::$variant, $crate::err_log!(buf, $level, $($args)+), - ::http::StatusCode::BAD_REQUEST + $crate::http::StatusCode::BAD_REQUEST ) }}; (Request($variant:ident($($args:tt)+))) => { $crate::error::Error::Request( - ::ruma::api::client::error::ErrorKind::$variant, + $crate::ruma::api::client::error::ErrorKind::$variant, $crate::format_maybe!($($args)+), - ::http::StatusCode::BAD_REQUEST + $crate::http::StatusCode::BAD_REQUEST ) }; @@ -113,7 +113,7 @@ macro_rules! err_log { ($out:ident, $level:ident, $($fields:tt)+) => {{ use std::{fmt, fmt::Write}; - use ::tracing::{ + use $crate::tracing::{ callsite, callsite2, level_enabled, metadata, valueset, Callsite, Event, __macro_support, __tracing_log, field::{Field, ValueSet, Visit}, @@ -169,25 +169,25 @@ macro_rules! err_log { macro_rules! err_lev { (debug_warn) => { if $crate::debug::logging() { - ::tracing::Level::WARN + $crate::tracing::Level::WARN } else { - ::tracing::Level::DEBUG + $crate::tracing::Level::DEBUG } }; (debug_error) => { if $crate::debug::logging() { - ::tracing::Level::ERROR + $crate::tracing::Level::ERROR } else { - ::tracing::Level::DEBUG + $crate::tracing::Level::DEBUG } }; (warn) => { - ::tracing::Level::WARN + $crate::tracing::Level::WARN }; (error) => { - ::tracing::Level::ERROR + $crate::tracing::Level::ERROR }; } diff --git a/src/core/mod.rs b/src/core/mod.rs index 31851f4f..e4553186 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -11,7 +11,10 @@ pub mod result; pub mod server; pub mod utils; +pub use ::http; +pub use ::ruma; pub use ::toml; +pub use ::tracing; pub use config::Config; pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; From 60010140784a286e53aa560cad6604caded4257e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 13 Sep 2024 07:40:22 +0000 Subject: [PATCH 0843/2091] add UnwrapInfallible to Result Signed-off-by: Jason Volk --- src/core/result.rs | 3 ++- src/core/result/unwrap_infallible.rs | 17 +++++++++++++++++ src/router/serve/unix.rs | 8 ++------ 3 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 src/core/result/unwrap_infallible.rs diff --git a/src/core/result.rs b/src/core/result.rs index 41d1d66c..96a34b8a 100644 --- a/src/core/result.rs +++ b/src/core/result.rs @@ -3,10 +3,11 @@ mod log_debug_err; mod log_err; mod map_expect; mod not_found; +mod unwrap_infallible; pub use self::{ debug_inspect::DebugInspect, log_debug_err::LogDebugErr, log_err::LogErr, map_expect::MapExpect, - not_found::NotFound, + not_found::NotFound, unwrap_infallible::UnwrapInfallible, }; pub type Result = std::result::Result; diff --git a/src/core/result/unwrap_infallible.rs b/src/core/result/unwrap_infallible.rs new file mode 100644 index 00000000..99309e02 --- /dev/null +++ b/src/core/result/unwrap_infallible.rs @@ -0,0 +1,17 @@ +use std::convert::Infallible; + +use super::{DebugInspect, Result}; +use crate::error; + +pub trait UnwrapInfallible { + fn unwrap_infallible(self) -> T; +} + +impl UnwrapInfallible for Result { + #[inline] + fn unwrap_infallible(self) -> T { + // SAFETY: Branchless unwrap for errors that can never happen. In debug + // mode this is asserted. + unsafe { self.debug_inspect_err(error::infallible).unwrap_unchecked() } + } +} diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index fb011f18..5df41b61 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -10,7 +10,7 @@ use axum::{ extract::{connect_info::IntoMakeServiceWithConnectInfo, Request}, Router, }; -use conduit::{debug, debug_error, error::infallible, info, trace, warn, Err, Result, Server}; +use conduit::{debug, debug_error, info, result::UnwrapInfallible, trace, warn, Err, Result, Server}; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, @@ -62,11 +62,7 @@ async fn accept( let socket = TokioIo::new(socket); trace!(?listener, ?socket, ?remote, "accepted"); - let called = app - .call(NULL_ADDR) - .await - .inspect_err(infallible) - .expect("infallible"); + let called = app.call(NULL_ADDR).await.unwrap_infallible(); let service = move |req: Request| called.clone().oneshot(req); let handler = service_fn(service); From 946ca364e032be8ca2529099b415990262c977fd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 8 Aug 2024 17:18:30 +0000 Subject: [PATCH 0844/2091] Database Refactor combine service/users data w/ mod unit split sliding sync related out of service/users instrument database entry points remove increment crap from database interface de-wrap all database get() calls de-wrap all database insert() calls de-wrap all database remove() calls refactor database interface for async streaming add query key serializer for database implement Debug for result handle add query deserializer for database add deserialization trait for option handle start a stream utils suite de-wrap/asyncify/type-query count_one_time_keys() de-wrap/asyncify users count add admin query users command suite de-wrap/asyncify users exists de-wrap/partially asyncify user filter related asyncify/de-wrap users device/keys related asyncify/de-wrap user auth/misc related asyncify/de-wrap users blurhash asyncify/de-wrap account_data get; merge Data into Service partial asyncify/de-wrap uiaa; merge Data into Service partially asyncify/de-wrap transaction_ids get; merge Data into Service partially asyncify/de-wrap key_backups; merge Data into Service asyncify/de-wrap pusher service getters; merge Data into Service asyncify/de-wrap rooms alias getters/some iterators asyncify/de-wrap rooms directory getters/iterator partially asyncify/de-wrap rooms lazy-loading partially asyncify/de-wrap rooms metadata asyncify/dewrap rooms outlier asyncify/dewrap rooms pdu_metadata dewrap/partially asyncify rooms read receipt de-wrap rooms search service de-wrap/partially asyncify rooms user service partial de-wrap rooms state_compressor de-wrap rooms state_cache de-wrap room state et al de-wrap rooms timeline service additional users device/keys related de-wrap/asyncify sender asyncify services refactor database to TryFuture/TryStream refactor services for TryFuture/TryStream asyncify api handlers additional asyncification for admin module abstract stream related; support reverse streams additional stream conversions asyncify state-res related Signed-off-by: Jason Volk --- Cargo.lock | 53 +- Cargo.toml | 7 +- clippy.toml | 2 +- src/admin/Cargo.toml | 3 +- src/admin/check/commands.rs | 9 +- src/admin/debug/commands.rs | 100 +- src/admin/federation/commands.rs | 13 +- src/admin/media/commands.rs | 4 +- src/admin/processor.rs | 2 +- src/admin/query/account_data.rs | 6 +- src/admin/query/appservice.rs | 6 +- src/admin/query/globals.rs | 6 +- src/admin/query/presence.rs | 5 +- src/admin/query/pusher.rs | 2 +- src/admin/query/room_alias.rs | 13 +- src/admin/query/room_state_cache.rs | 93 +- src/admin/query/sending.rs | 9 +- src/admin/query/users.rs | 351 +++- src/admin/room/alias.rs | 118 +- src/admin/room/commands.rs | 47 +- src/admin/room/directory.rs | 22 +- src/admin/room/info.rs | 48 +- src/admin/room/mod.rs | 6 + src/admin/room/moderation.rs | 343 ++-- src/admin/user/commands.rs | 192 ++- src/admin/utils.rs | 22 +- src/api/Cargo.toml | 2 +- src/api/client/account.rs | 182 ++- src/api/client/alias.rs | 28 +- src/api/client/backup.rs | 230 +-- src/api/client/config.rs | 40 +- src/api/client/context.rs | 108 +- src/api/client/device.rs | 46 +- src/api/client/directory.rs | 188 +-- src/api/client/filter.rs | 25 +- src/api/client/keys.rs | 186 ++- src/api/client/membership.rs | 486 +++--- src/api/client/message.rs | 196 +-- src/api/client/presence.rs | 14 +- src/api/client/profile.rs | 200 ++- src/api/client/push.rs | 182 ++- src/api/client/read_marker.rs | 90 +- src/api/client/relations.rs | 90 +- src/api/client/report.rs | 20 +- src/api/client/room.rs | 105 +- src/api/client/search.rs | 108 +- src/api/client/session.rs | 75 +- src/api/client/state.rs | 63 +- src/api/client/sync.rs | 1091 +++++++------ src/api/client/tag.rs | 45 +- src/api/client/threads.rs | 19 +- src/api/client/to_device.rs | 52 +- src/api/client/typing.rs | 3 +- src/api/client/unstable.rs | 168 +- src/api/client/unversioned.rs | 3 +- src/api/client/user_directory.rs | 47 +- src/api/router.rs | 322 ++-- src/api/router/args.rs | 26 +- src/api/router/auth.rs | 10 +- src/api/router/handler.rs | 38 +- src/api/router/response.rs | 9 +- src/api/server/backfill.rs | 84 +- src/api/server/event.rs | 39 +- src/api/server/event_auth.rs | 33 +- src/api/server/get_missing_events.rs | 31 +- src/api/server/hierarchy.rs | 2 +- src/api/server/invite.rs | 38 +- src/api/server/make_join.rs | 87 +- src/api/server/make_leave.rs | 37 +- src/api/server/openid.rs | 5 +- src/api/server/query.rs | 36 +- src/api/server/send.rs | 196 +-- src/api/server/send_join.rs | 71 +- src/api/server/send_leave.rs | 20 +- src/api/server/state.rs | 65 +- src/api/server/state_ids.rs | 37 +- src/api/server/user.rs | 51 +- src/core/Cargo.toml | 1 + src/core/error/mod.rs | 4 +- src/core/pdu/mod.rs | 43 +- src/core/result/log_debug_err.rs | 18 +- src/core/result/log_err.rs | 20 +- src/core/utils/algorithm.rs | 25 - src/core/utils/mod.rs | 32 +- src/core/utils/set.rs | 47 + src/core/utils/stream/cloned.rs | 20 + src/core/utils/stream/expect.rs | 17 + src/core/utils/stream/ignore.rs | 21 + src/core/utils/stream/iter_stream.rs | 27 + src/core/utils/stream/mod.rs | 13 + src/core/utils/stream/ready.rs | 109 ++ src/core/utils/stream/try_ready.rs | 35 + src/core/utils/tests.rs | 130 ++ src/database/Cargo.toml | 3 + src/database/database.rs | 2 +- src/database/de.rs | 261 +++ src/database/deserialized.rs | 34 + src/database/engine.rs | 2 +- src/database/handle.rs | 89 +- src/database/iter.rs | 110 -- src/database/keyval.rs | 83 + src/database/map.rs | 304 ++-- src/database/map/count.rs | 36 + src/database/map/keys.rs | 21 + src/database/map/keys_from.rs | 49 + src/database/map/keys_prefix.rs | 54 + src/database/map/rev_keys.rs | 21 + src/database/map/rev_keys_from.rs | 49 + src/database/map/rev_keys_prefix.rs | 54 + src/database/map/rev_stream.rs | 29 + src/database/map/rev_stream_from.rs | 68 + src/database/map/rev_stream_prefix.rs | 74 + src/database/map/stream.rs | 28 + src/database/map/stream_from.rs | 68 + src/database/map/stream_prefix.rs | 74 + src/database/mod.rs | 28 +- src/database/ser.rs | 315 ++++ src/database/slice.rs | 57 - src/database/stream.rs | 122 ++ src/database/stream/items.rs | 44 + src/database/stream/items_rev.rs | 44 + src/database/stream/keys.rs | 44 + src/database/stream/keys_rev.rs | 44 + src/database/util.rs | 12 + src/service/Cargo.toml | 2 +- src/service/account_data/data.rs | 152 -- src/service/account_data/mod.rs | 164 +- src/service/admin/console.rs | 2 +- src/service/admin/create.rs | 2 +- src/service/admin/grant.rs | 216 +-- src/service/admin/mod.rs | 104 +- src/service/appservice/data.rs | 28 +- src/service/appservice/mod.rs | 49 +- src/service/emergency/mod.rs | 30 +- src/service/globals/data.rs | 121 +- src/service/globals/migrations.rs | 741 ++------- src/service/globals/mod.rs | 8 +- src/service/key_backups/data.rs | 346 ---- src/service/key_backups/mod.rs | 360 ++++- src/service/manager.rs | 2 +- src/service/media/data.rs | 100 +- src/service/media/migrations.rs | 33 +- src/service/media/mod.rs | 15 +- src/service/media/preview.rs | 8 +- src/service/media/thumbnail.rs | 4 +- src/service/mod.rs | 1 + src/service/presence/data.rs | 111 +- src/service/presence/mod.rs | 63 +- src/service/presence/presence.rs | 12 +- src/service/pusher/data.rs | 77 - src/service/pusher/mod.rs | 124 +- src/service/resolver/actual.rs | 6 +- src/service/rooms/alias/data.rs | 125 -- src/service/rooms/alias/mod.rs | 147 +- src/service/rooms/auth_chain/data.rs | 21 +- src/service/rooms/auth_chain/mod.rs | 45 +- src/service/rooms/directory/data.rs | 39 - src/service/rooms/directory/mod.rs | 40 +- src/service/rooms/event_handler/mod.rs | 1220 +++++++------- .../rooms/event_handler/parse_incoming_pdu.rs | 6 +- src/service/rooms/lazy_loading/data.rs | 65 - src/service/rooms/lazy_loading/mod.rs | 112 +- src/service/rooms/metadata/data.rs | 110 -- src/service/rooms/metadata/mod.rs | 99 +- src/service/rooms/outlier/data.rs | 42 - src/service/rooms/outlier/mod.rs | 59 +- src/service/rooms/pdu_metadata/data.rs | 76 +- src/service/rooms/pdu_metadata/mod.rs | 171 +- src/service/rooms/read_receipt/data.rs | 148 +- src/service/rooms/read_receipt/mod.rs | 49 +- src/service/rooms/search/data.rs | 73 +- src/service/rooms/search/mod.rs | 17 +- src/service/rooms/short/data.rs | 212 ++- src/service/rooms/short/mod.rs | 36 +- src/service/rooms/spaces/mod.rs | 174 +- src/service/rooms/state/data.rs | 71 +- src/service/rooms/state/mod.rs | 276 ++-- src/service/rooms/state_accessor/data.rs | 156 +- src/service/rooms/state_accessor/mod.rs | 356 +++-- src/service/rooms/state_cache/data.rs | 646 +------- src/service/rooms/state_cache/mod.rs | 475 ++++-- src/service/rooms/state_compressor/data.rs | 20 +- src/service/rooms/state_compressor/mod.rs | 85 +- src/service/rooms/threads/data.rs | 78 +- src/service/rooms/threads/mod.rs | 38 +- src/service/rooms/timeline/data.rs | 329 ++-- src/service/rooms/timeline/mod.rs | 656 ++++---- src/service/rooms/typing/mod.rs | 33 +- src/service/rooms/user/data.rs | 146 +- src/service/rooms/user/mod.rs | 46 +- src/service/sending/data.rs | 194 +-- src/service/sending/mod.rs | 113 +- src/service/sending/sender.rs | 278 ++-- src/service/server_keys/mod.rs | 26 +- src/service/services.rs | 4 +- src/service/sync/mod.rs | 233 +++ src/service/transaction_ids/data.rs | 44 - src/service/transaction_ids/mod.rs | 44 +- src/service/uiaa/data.rs | 87 - src/service/uiaa/mod.rs | 313 ++-- src/service/updates/mod.rs | 90 +- src/service/users/data.rs | 1098 ------------- src/service/users/mod.rs | 1413 +++++++++++------ 203 files changed, 12202 insertions(+), 10709 deletions(-) delete mode 100644 src/core/utils/algorithm.rs create mode 100644 src/core/utils/set.rs create mode 100644 src/core/utils/stream/cloned.rs create mode 100644 src/core/utils/stream/expect.rs create mode 100644 src/core/utils/stream/ignore.rs create mode 100644 src/core/utils/stream/iter_stream.rs create mode 100644 src/core/utils/stream/mod.rs create mode 100644 src/core/utils/stream/ready.rs create mode 100644 src/core/utils/stream/try_ready.rs create mode 100644 src/database/de.rs create mode 100644 src/database/deserialized.rs delete mode 100644 src/database/iter.rs create mode 100644 src/database/keyval.rs create mode 100644 src/database/map/count.rs create mode 100644 src/database/map/keys.rs create mode 100644 src/database/map/keys_from.rs create mode 100644 src/database/map/keys_prefix.rs create mode 100644 src/database/map/rev_keys.rs create mode 100644 src/database/map/rev_keys_from.rs create mode 100644 src/database/map/rev_keys_prefix.rs create mode 100644 src/database/map/rev_stream.rs create mode 100644 src/database/map/rev_stream_from.rs create mode 100644 src/database/map/rev_stream_prefix.rs create mode 100644 src/database/map/stream.rs create mode 100644 src/database/map/stream_from.rs create mode 100644 src/database/map/stream_prefix.rs create mode 100644 src/database/ser.rs delete mode 100644 src/database/slice.rs create mode 100644 src/database/stream.rs create mode 100644 src/database/stream/items.rs create mode 100644 src/database/stream/items_rev.rs create mode 100644 src/database/stream/keys.rs create mode 100644 src/database/stream/keys_rev.rs delete mode 100644 src/service/account_data/data.rs delete mode 100644 src/service/key_backups/data.rs delete mode 100644 src/service/pusher/data.rs delete mode 100644 src/service/rooms/alias/data.rs delete mode 100644 src/service/rooms/directory/data.rs delete mode 100644 src/service/rooms/lazy_loading/data.rs delete mode 100644 src/service/rooms/metadata/data.rs delete mode 100644 src/service/rooms/outlier/data.rs create mode 100644 src/service/sync/mod.rs delete mode 100644 src/service/transaction_ids/data.rs delete mode 100644 src/service/uiaa/data.rs delete mode 100644 src/service/users/data.rs diff --git a/Cargo.lock b/Cargo.lock index 6386f968..08e0498a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -626,10 +626,11 @@ dependencies = [ "clap", "conduit_api", "conduit_core", + "conduit_database", "conduit_macros", "conduit_service", "const-str", - "futures-util", + "futures", "log", "ruma", "serde_json", @@ -652,7 +653,7 @@ dependencies = [ "conduit_database", "conduit_service", "const-str", - "futures-util", + "futures", "hmac", "http", "http-body-util", @@ -689,6 +690,7 @@ dependencies = [ "cyborgtime", "either", "figment", + "futures", "hardened_malloc-rs", "http", "http-body-util", @@ -726,8 +728,11 @@ version = "0.4.7" dependencies = [ "conduit_core", "const-str", + "futures", "log", "rust-rocksdb-uwu", + "serde", + "serde_json", "tokio", "tracing", ] @@ -784,7 +789,7 @@ dependencies = [ "conduit_core", "conduit_database", "const-str", - "futures-util", + "futures", "hickory-resolver", "http", "image", @@ -1283,6 +1288,20 @@ dependencies = [ "new_debug_unreachable", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -1345,6 +1364,7 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -2953,7 +2973,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "assign", "js_int", @@ -2975,7 +2995,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "js_int", "ruma-common", @@ -2987,7 +3007,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "as_variant", "assign", @@ -3010,7 +3030,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "as_variant", "base64 0.22.1", @@ -3040,7 +3060,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3064,7 +3084,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "bytes", "http", @@ -3082,7 +3102,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "js_int", "thiserror", @@ -3091,7 +3111,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "js_int", "ruma-common", @@ -3101,7 +3121,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "cfg-if", "once_cell", @@ -3117,7 +3137,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "js_int", "ruma-common", @@ -3129,7 +3149,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "headers", "http", @@ -3142,7 +3162,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3158,8 +3178,9 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=9900d0676564883cfade556d6e8da2a2c9061efd#9900d0676564883cfade556d6e8da2a2c9061efd" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" dependencies = [ + "futures-util", "itertools 0.12.1", "js_int", "ruma-common", diff --git a/Cargo.toml b/Cargo.toml index b75c4975..3bfb3bc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -210,9 +210,10 @@ features = [ "string", ] -[workspace.dependencies.futures-util] +[workspace.dependencies.futures] version = "0.3.30" default-features = false +features = ["std"] [workspace.dependencies.tokio] version = "1.40.0" @@ -314,7 +315,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "9900d0676564883cfade556d6e8da2a2c9061efd" +rev = "e7db44989d68406393270d3a91815597385d3acb" features = [ "compat", "rand", @@ -463,7 +464,6 @@ version = "1.0.36" [workspace.dependencies.proc-macro2] version = "1.0.89" - # # Patches # @@ -828,6 +828,7 @@ missing_panics_doc = { level = "allow", priority = 1 } module_name_repetitions = { level = "allow", priority = 1 } no_effect_underscore_binding = { level = "allow", priority = 1 } similar_names = { level = "allow", priority = 1 } +single_match_else = { level = "allow", priority = 1 } struct_field_names = { level = "allow", priority = 1 } unnecessary_wraps = { level = "allow", priority = 1 } unused_async = { level = "allow", priority = 1 } diff --git a/clippy.toml b/clippy.toml index c942b93c..08641fcc 100644 --- a/clippy.toml +++ b/clippy.toml @@ -2,6 +2,6 @@ array-size-threshold = 4096 cognitive-complexity-threshold = 94 # TODO reduce me ALARA excessive-nesting-threshold = 11 # TODO reduce me to 4 or 5 future-size-threshold = 7745 # TODO reduce me ALARA -stack-size-threshold = 144000 # reduce me ALARA +stack-size-threshold = 196608 # reduce me ALARA too-many-lines-threshold = 700 # TODO reduce me to <= 100 type-complexity-threshold = 250 # reduce me to ~200 diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index d756b3cb..f5cab449 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -29,10 +29,11 @@ release_max_log_level = [ clap.workspace = true conduit-api.workspace = true conduit-core.workspace = true +conduit-database.workspace = true conduit-macros.workspace = true conduit-service.workspace = true const-str.workspace = true -futures-util.workspace = true +futures.workspace = true log.workspace = true ruma.workspace = true serde_json.workspace = true diff --git a/src/admin/check/commands.rs b/src/admin/check/commands.rs index 0a983046..88fca462 100644 --- a/src/admin/check/commands.rs +++ b/src/admin/check/commands.rs @@ -1,5 +1,6 @@ use conduit::Result; use conduit_macros::implement; +use futures::StreamExt; use ruma::events::room::message::RoomMessageEventContent; use crate::Command; @@ -10,14 +11,12 @@ use crate::Command; #[implement(Command, params = "<'_>")] pub(super) async fn check_all_users(&self) -> Result { let timer = tokio::time::Instant::now(); - let results = self.services.users.db.iter(); + let users = self.services.users.iter().collect::>().await; let query_time = timer.elapsed(); - let users = results.collect::>(); - let total = users.len(); - let err_count = users.iter().filter(|user| user.is_err()).count(); - let ok_count = users.iter().filter(|user| user.is_ok()).count(); + let err_count = users.iter().filter(|_user| false).count(); + let ok_count = users.iter().filter(|_user| true).count(); let message = format!( "Database query completed in {query_time:?}:\n\n```\nTotal entries: {total:?}\nFailure/Invalid user count: \ diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 2d967006..65c9bc71 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -7,6 +7,7 @@ use std::{ use api::client::validate_and_add_event_id; use conduit::{debug, debug_error, err, info, trace, utils, warn, Error, PduEvent, Result}; +use futures::StreamExt; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, events::room::message::RoomMessageEventContent, @@ -27,7 +28,7 @@ pub(super) async fn echo(&self, message: Vec) -> Result) -> Result { let event_id = Arc::::from(event_id); - if let Some(event) = self.services.rooms.timeline.get_pdu_json(&event_id)? { + if let Ok(event) = self.services.rooms.timeline.get_pdu_json(&event_id).await { let room_id_str = event .get("room_id") .and_then(|val| val.as_str()) @@ -43,7 +44,8 @@ pub(super) async fn get_auth_chain(&self, event_id: Box) -> Result) -> Result { + Ok(json) => { let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json"); Ok(RoomMessageEventContent::notice_markdown(format!( "{}\n```json\n{}\n```", @@ -109,7 +114,7 @@ pub(super) async fn get_pdu(&self, event_id: Box) -> Result Ok(RoomMessageEventContent::text_plain("PDU not found locally.")), + Err(_) => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")), } } @@ -157,7 +162,8 @@ pub(super) async fn get_remote_pdu_list( .send_message(RoomMessageEventContent::text_plain(format!( "Failed to get remote PDU, ignoring error: {e}" ))) - .await; + .await + .ok(); warn!("Failed to get remote PDU, ignoring error: {e}"); } else { success_count = success_count.saturating_add(1); @@ -215,7 +221,9 @@ pub(super) async fn get_remote_pdu( .services .rooms .event_handler - .parse_incoming_pdu(&response.pdu); + .parse_incoming_pdu(&response.pdu) + .await; + let (event_id, value, room_id) = match parsed_result { Ok(t) => t, Err(e) => { @@ -333,9 +341,12 @@ pub(super) async fn ping(&self, server: Box) -> Result Result { // Force E2EE device list updates for all users - for user_id in self.services.users.iter().filter_map(Result::ok) { - self.services.users.mark_device_key_update(&user_id)?; - } + self.services + .users + .stream() + .for_each(|user_id| self.services.users.mark_device_key_update(user_id)) + .await; + Ok(RoomMessageEventContent::text_plain( "Marked all devices for all users as having new keys to update", )) @@ -470,7 +481,8 @@ pub(super) async fn first_pdu_in_room(&self, room_id: Box) -> Result) -> Result) -> Result) -> Result> = HashMap::new(); let pub_key_map = RwLock::new(BTreeMap::new()); @@ -554,13 +571,21 @@ pub(super) async fn force_set_room_state_from_server( let mut events = Vec::with_capacity(remote_state_response.pdus.len()); for pdu in remote_state_response.pdus.clone() { - events.push(match self.services.rooms.event_handler.parse_incoming_pdu(&pdu) { - Ok(t) => t, - Err(e) => { - warn!("Could not parse PDU, ignoring: {e}"); - continue; + events.push( + match self + .services + .rooms + .event_handler + .parse_incoming_pdu(&pdu) + .await + { + Ok(t) => t, + Err(e) => { + warn!("Could not parse PDU, ignoring: {e}"); + continue; + }, }, - }); + ); } info!("Fetching required signing keys for all the state events we got"); @@ -587,13 +612,16 @@ pub(super) async fn force_set_room_state_from_server( self.services .rooms .outlier - .add_pdu_outlier(&event_id, &value)?; + .add_pdu_outlier(&event_id, &value); + if let Some(state_key) = &pdu.state_key { let shortstatekey = self .services .rooms .short - .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key)?; + .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key) + .await; + state.insert(shortstatekey, pdu.event_id.clone()); } } @@ -611,7 +639,7 @@ pub(super) async fn force_set_room_state_from_server( self.services .rooms .outlier - .add_pdu_outlier(&event_id, &value)?; + .add_pdu_outlier(&event_id, &value); } let new_room_state = self @@ -626,7 +654,8 @@ pub(super) async fn force_set_room_state_from_server( .services .rooms .state_compressor - .save_state(room_id.clone().as_ref(), new_room_state)?; + .save_state(room_id.clone().as_ref(), new_room_state) + .await?; let state_lock = self.services.rooms.state.mutex.lock(&room_id).await; self.services @@ -642,7 +671,8 @@ pub(super) async fn force_set_room_state_from_server( self.services .rooms .state_cache - .update_joined_count(&room_id)?; + .update_joined_count(&room_id) + .await; drop(state_lock); @@ -656,7 +686,7 @@ pub(super) async fn get_signing_keys( &self, server_name: Option>, _cached: bool, ) -> Result { let server_name = server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); - let signing_keys = self.services.globals.signing_keys_for(&server_name)?; + let signing_keys = self.services.globals.signing_keys_for(&server_name).await?; Ok(RoomMessageEventContent::notice_markdown(format!( "```rs\n{signing_keys:#?}\n```" @@ -674,7 +704,7 @@ pub(super) async fn get_verify_keys( if cached { writeln!(out, "| Key ID | VerifyKey |")?; writeln!(out, "| --- | --- |")?; - for (key_id, verify_key) in self.services.globals.verify_keys_for(&server_name)? { + for (key_id, verify_key) in self.services.globals.verify_keys_for(&server_name).await? { writeln!(out, "| {key_id} | {verify_key:?} |")?; } diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 8917a46b..ce95ac01 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -1,19 +1,20 @@ use std::fmt::Write; use conduit::Result; +use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId, ServerName, UserId}; use crate::{admin_command, escape_html, get_room_info}; #[admin_command] pub(super) async fn disable_room(&self, room_id: Box) -> Result { - self.services.rooms.metadata.disable_room(&room_id, true)?; + self.services.rooms.metadata.disable_room(&room_id, true); Ok(RoomMessageEventContent::text_plain("Room disabled.")) } #[admin_command] pub(super) async fn enable_room(&self, room_id: Box) -> Result { - self.services.rooms.metadata.disable_room(&room_id, false)?; + self.services.rooms.metadata.disable_room(&room_id, false); Ok(RoomMessageEventContent::text_plain("Room enabled.")) } @@ -85,7 +86,7 @@ pub(super) async fn remote_user_in_rooms(&self, user_id: Box) -> Result< )); } - if !self.services.users.exists(&user_id)? { + if !self.services.users.exists(&user_id).await { return Ok(RoomMessageEventContent::text_plain( "Remote user does not exist in our database.", )); @@ -96,9 +97,9 @@ pub(super) async fn remote_user_in_rooms(&self, user_id: Box) -> Result< .rooms .state_cache .rooms_joined(&user_id) - .filter_map(Result::ok) - .map(|room_id| get_room_info(self.services, &room_id)) - .collect(); + .then(|room_id| get_room_info(self.services, room_id)) + .collect() + .await; if rooms.is_empty() { return Ok(RoomMessageEventContent::text_plain("User is not in any rooms.")); diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 3c4bf2ef..82ac162e 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -36,7 +36,7 @@ pub(super) async fn delete( let mut mxc_urls = Vec::with_capacity(4); // parsing the PDU for any MXC URLs begins here - if let Some(event_json) = self.services.rooms.timeline.get_pdu_json(&event_id)? { + if let Ok(event_json) = self.services.rooms.timeline.get_pdu_json(&event_id).await { if let Some(content_key) = event_json.get("content") { debug!("Event ID has \"content\"."); let content_obj = content_key.as_object(); @@ -300,7 +300,7 @@ pub(super) async fn delete_all_from_server( #[admin_command] pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result { let mxc: Mxc<'_> = mxc.as_str().try_into()?; - let metadata = self.services.media.get_metadata(&mxc); + let metadata = self.services.media.get_metadata(&mxc).await; Ok(RoomMessageEventContent::notice_markdown(format!("```\n{metadata:#?}\n```"))) } diff --git a/src/admin/processor.rs b/src/admin/processor.rs index 4f60f56e..3c1895ff 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -17,7 +17,7 @@ use conduit::{ utils::string::{collect_stream, common_prefix}, warn, Error, Result, }; -use futures_util::future::FutureExt; +use futures::future::FutureExt; use ruma::{ events::{ relation::InReplyTo, diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index e18c298a..896bf95c 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -44,7 +44,8 @@ pub(super) async fn process(subcommand: AccountDataCommand, context: &Command<'_ let timer = tokio::time::Instant::now(); let results = services .account_data - .changes_since(room_id.as_deref(), &user_id, since)?; + .changes_since(room_id.as_deref(), &user_id, since) + .await?; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -59,7 +60,8 @@ pub(super) async fn process(subcommand: AccountDataCommand, context: &Command<'_ let timer = tokio::time::Instant::now(); let results = services .account_data - .get(room_id.as_deref(), &user_id, kind)?; + .get(room_id.as_deref(), &user_id, kind) + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 683c228f..4b97ef4e 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -29,7 +29,9 @@ pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_> let results = services .appservice .db - .get_registration(appservice_id.as_ref()); + .get_registration(appservice_id.as_ref()) + .await; + let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -38,7 +40,7 @@ pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_> }, AppserviceCommand::All => { let timer = tokio::time::Instant::now(); - let results = services.appservice.all(); + let results = services.appservice.all().await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 5f271c2c..150a213c 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -29,7 +29,7 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) - match subcommand { GlobalsCommand::DatabaseVersion => { let timer = tokio::time::Instant::now(); - let results = services.globals.db.database_version(); + let results = services.globals.db.database_version().await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -47,7 +47,7 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) - }, GlobalsCommand::LastCheckForUpdatesId => { let timer = tokio::time::Instant::now(); - let results = services.updates.last_check_for_updates_id(); + let results = services.updates.last_check_for_updates_id().await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -67,7 +67,7 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) - origin, } => { let timer = tokio::time::Instant::now(); - let results = services.globals.db.verify_keys_for(&origin); + let results = services.globals.db.verify_keys_for(&origin).await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index 145ecd9b..6189270c 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -1,5 +1,6 @@ use clap::Subcommand; use conduit::Result; +use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, UserId}; use crate::Command; @@ -30,7 +31,7 @@ pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) user_id, } => { let timer = tokio::time::Instant::now(); - let results = services.presence.db.get_presence(&user_id)?; + let results = services.presence.db.get_presence(&user_id).await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -42,7 +43,7 @@ pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) } => { let timer = tokio::time::Instant::now(); let results = services.presence.db.presence_since(since); - let presence_since: Vec<(_, _, _)> = results.collect(); + let presence_since: Vec<(_, _, _)> = results.collect().await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/pusher.rs b/src/admin/query/pusher.rs index 637c57b6..a1bd32f9 100644 --- a/src/admin/query/pusher.rs +++ b/src/admin/query/pusher.rs @@ -21,7 +21,7 @@ pub(super) async fn process(subcommand: PusherCommand, context: &Command<'_>) -> user_id, } => { let timer = tokio::time::Instant::now(); - let results = services.pusher.get_pushers(&user_id)?; + let results = services.pusher.get_pushers(&user_id).await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index 1809e26a..05fac42c 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -1,5 +1,6 @@ use clap::Subcommand; use conduit::Result; +use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, RoomId}; use crate::Command; @@ -31,7 +32,7 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) alias, } => { let timer = tokio::time::Instant::now(); - let results = services.rooms.alias.resolve_local_alias(&alias); + let results = services.rooms.alias.resolve_local_alias(&alias).await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -43,7 +44,7 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) } => { let timer = tokio::time::Instant::now(); let results = services.rooms.alias.local_aliases_for_room(&room_id); - let aliases: Vec<_> = results.collect(); + let aliases: Vec<_> = results.collect().await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -52,8 +53,12 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) }, RoomAliasCommand::AllLocalAliases => { let timer = tokio::time::Instant::now(); - let results = services.rooms.alias.all_local_aliases(); - let aliases: Vec<_> = results.collect(); + let aliases = services + .rooms + .alias + .all_local_aliases() + .collect::>() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index 4215cf8d..e32517fb 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -1,5 +1,6 @@ use clap::Subcommand; use conduit::Result; +use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId}; use crate::Command; @@ -86,7 +87,11 @@ pub(super) async fn process( room_id, } => { let timer = tokio::time::Instant::now(); - let result = services.rooms.state_cache.server_in_room(&server, &room_id); + let result = services + .rooms + .state_cache + .server_in_room(&server, &room_id) + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -97,7 +102,13 @@ pub(super) async fn process( room_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services.rooms.state_cache.room_servers(&room_id).collect(); + let results: Vec<_> = services + .rooms + .state_cache + .room_servers(&room_id) + .map(ToOwned::to_owned) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -108,7 +119,13 @@ pub(super) async fn process( server, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services.rooms.state_cache.server_rooms(&server).collect(); + let results: Vec<_> = services + .rooms + .state_cache + .server_rooms(&server) + .map(ToOwned::to_owned) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -119,7 +136,13 @@ pub(super) async fn process( room_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services.rooms.state_cache.room_members(&room_id).collect(); + let results: Vec<_> = services + .rooms + .state_cache + .room_members(&room_id) + .map(ToOwned::to_owned) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -134,7 +157,9 @@ pub(super) async fn process( .rooms .state_cache .local_users_in_room(&room_id) - .collect(); + .map(ToOwned::to_owned) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -149,7 +174,9 @@ pub(super) async fn process( .rooms .state_cache .active_local_users_in_room(&room_id) - .collect(); + .map(ToOwned::to_owned) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -160,7 +187,7 @@ pub(super) async fn process( room_id, } => { let timer = tokio::time::Instant::now(); - let results = services.rooms.state_cache.room_joined_count(&room_id); + let results = services.rooms.state_cache.room_joined_count(&room_id).await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -171,7 +198,11 @@ pub(super) async fn process( room_id, } => { let timer = tokio::time::Instant::now(); - let results = services.rooms.state_cache.room_invited_count(&room_id); + let results = services + .rooms + .state_cache + .room_invited_count(&room_id) + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -182,11 +213,13 @@ pub(super) async fn process( room_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services + let results: Vec<_> = services .rooms .state_cache .room_useroncejoined(&room_id) - .collect(); + .map(ToOwned::to_owned) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -197,11 +230,13 @@ pub(super) async fn process( room_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services + let results: Vec<_> = services .rooms .state_cache .room_members_invited(&room_id) - .collect(); + .map(ToOwned::to_owned) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -216,7 +251,8 @@ pub(super) async fn process( let results = services .rooms .state_cache - .get_invite_count(&room_id, &user_id); + .get_invite_count(&room_id, &user_id) + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -231,7 +267,8 @@ pub(super) async fn process( let results = services .rooms .state_cache - .get_left_count(&room_id, &user_id); + .get_left_count(&room_id, &user_id) + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -242,7 +279,13 @@ pub(super) async fn process( user_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services.rooms.state_cache.rooms_joined(&user_id).collect(); + let results: Vec<_> = services + .rooms + .state_cache + .rooms_joined(&user_id) + .map(ToOwned::to_owned) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -253,7 +296,12 @@ pub(super) async fn process( user_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services.rooms.state_cache.rooms_invited(&user_id).collect(); + let results: Vec<_> = services + .rooms + .state_cache + .rooms_invited(&user_id) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -264,7 +312,12 @@ pub(super) async fn process( user_id, } => { let timer = tokio::time::Instant::now(); - let results: Result> = services.rooms.state_cache.rooms_left(&user_id).collect(); + let results: Vec<_> = services + .rooms + .state_cache + .rooms_left(&user_id) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -276,7 +329,11 @@ pub(super) async fn process( room_id, } => { let timer = tokio::time::Instant::now(); - let results = services.rooms.state_cache.invite_state(&user_id, &room_id); + let results = services + .rooms + .state_cache + .invite_state(&user_id, &room_id) + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 6d54bddf..eaab1f5e 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -1,5 +1,6 @@ use clap::Subcommand; use conduit::Result; +use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, ServerName, UserId}; use service::sending::Destination; @@ -68,7 +69,7 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - SendingCommand::ActiveRequests => { let timer = tokio::time::Instant::now(); let results = services.sending.db.active_requests(); - let active_requests: Result> = results.collect(); + let active_requests = results.collect::>().await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -133,7 +134,7 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - }, }; - let queued_requests = results.collect::>>(); + let queued_requests = results.collect::>().await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -199,7 +200,7 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - }, }; - let active_requests = results.collect::>>(); + let active_requests = results.collect::>().await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -210,7 +211,7 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - server_name, } => { let timer = tokio::time::Instant::now(); - let results = services.sending.db.get_latest_educount(&server_name); + let results = services.sending.db.get_latest_educount(&server_name).await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index fee12fbf..0792e484 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -1,29 +1,344 @@ use clap::Subcommand; use conduit::Result; -use ruma::events::room::message::RoomMessageEventContent; +use futures::stream::StreamExt; +use ruma::{events::room::message::RoomMessageEventContent, OwnedDeviceId, OwnedRoomId, OwnedUserId}; -use crate::Command; +use crate::{admin_command, admin_command_dispatch}; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/users.rs pub(crate) enum UsersCommand { - Iter, + CountUsers, + + IterUsers, + + PasswordHash { + user_id: OwnedUserId, + }, + + ListDevices { + user_id: OwnedUserId, + }, + + ListDevicesMetadata { + user_id: OwnedUserId, + }, + + GetDeviceMetadata { + user_id: OwnedUserId, + device_id: OwnedDeviceId, + }, + + GetDevicesVersion { + user_id: OwnedUserId, + }, + + CountOneTimeKeys { + user_id: OwnedUserId, + device_id: OwnedDeviceId, + }, + + GetDeviceKeys { + user_id: OwnedUserId, + device_id: OwnedDeviceId, + }, + + GetUserSigningKey { + user_id: OwnedUserId, + }, + + GetMasterKey { + user_id: OwnedUserId, + }, + + GetToDeviceEvents { + user_id: OwnedUserId, + device_id: OwnedDeviceId, + }, + + GetLatestBackup { + user_id: OwnedUserId, + }, + + GetLatestBackupVersion { + user_id: OwnedUserId, + }, + + GetBackupAlgorithm { + user_id: OwnedUserId, + version: String, + }, + + GetAllBackups { + user_id: OwnedUserId, + version: String, + }, + + GetRoomBackups { + user_id: OwnedUserId, + version: String, + room_id: OwnedRoomId, + }, + + GetBackupSession { + user_id: OwnedUserId, + version: String, + room_id: OwnedRoomId, + session_id: String, + }, } -/// All the getters and iterators in key_value/users.rs -pub(super) async fn process(subcommand: UsersCommand, context: &Command<'_>) -> Result { - let services = context.services; +#[admin_command] +async fn get_backup_session( + &self, user_id: OwnedUserId, version: String, room_id: OwnedRoomId, session_id: String, +) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .key_backups + .get_session(&user_id, &version, &room_id, &session_id) + .await; + let query_time = timer.elapsed(); - match subcommand { - UsersCommand::Iter => { - let timer = tokio::time::Instant::now(); - let results = services.users.db.iter(); - let users = results.collect::>(); - let query_time = timer.elapsed(); - - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{users:#?}\n```" - ))) - }, - } + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn get_room_backups( + &self, user_id: OwnedUserId, version: String, room_id: OwnedRoomId, +) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .key_backups + .get_room(&user_id, &version, &room_id) + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result { + let timer = tokio::time::Instant::now(); + let result = self.services.key_backups.get_all(&user_id, &version).await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .key_backups + .get_backup(&user_id, &version) + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .key_backups + .get_latest_backup_version(&user_id) + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self.services.key_backups.get_latest_backup(&user_id).await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn iter_users(&self) -> Result { + let timer = tokio::time::Instant::now(); + let result: Vec = self.services.users.stream().map(Into::into).collect().await; + + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn count_users(&self) -> Result { + let timer = tokio::time::Instant::now(); + let result = self.services.users.count().await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn password_hash(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self.services.users.password_hash(&user_id).await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn list_devices(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let devices = self + .services + .users + .all_device_ids(&user_id) + .map(ToOwned::to_owned) + .collect::>() + .await; + + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```" + ))) +} + +#[admin_command] +async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let devices = self + .services + .users + .all_devices_metadata(&user_id) + .collect::>() + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{devices:#?}\n```" + ))) +} + +#[admin_command] +async fn get_device_metadata(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { + let timer = tokio::time::Instant::now(); + let device = self + .services + .users + .get_device_metadata(&user_id, &device_id) + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```" + ))) +} + +#[admin_command] +async fn get_devices_version(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let device = self.services.users.get_devicelist_version(&user_id).await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```" + ))) +} + +#[admin_command] +async fn count_one_time_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .users + .count_one_time_keys(&user_id, &device_id) + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .users + .get_device_keys(&user_id, &device_id) + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self.services.users.get_user_signing_key(&user_id).await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn get_master_key(&self, user_id: OwnedUserId) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .users + .get_master_key(None, &user_id, &|_| true) + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +async fn get_to_device_events( + &self, user_id: OwnedUserId, device_id: OwnedDeviceId, +) -> Result { + let timer = tokio::time::Instant::now(); + let result = self + .services + .users + .get_to_device_events(&user_id, &device_id) + .collect::>() + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) } diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 415b8a08..34b6c42e 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -2,7 +2,8 @@ use std::fmt::Write; use clap::Subcommand; use conduit::Result; -use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, RoomId}; +use futures::StreamExt; +use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId}; use crate::{escape_html, Command}; @@ -66,8 +67,8 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> force, room_id, .. - } => match (force, services.rooms.alias.resolve_local_alias(&room_alias)) { - (true, Ok(Some(id))) => match services + } => match (force, services.rooms.alias.resolve_local_alias(&room_alias).await) { + (true, Ok(id)) => match services .rooms .alias .set_alias(&room_alias, &room_id, server_user) @@ -77,10 +78,10 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> ))), Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), }, - (false, Ok(Some(id))) => Ok(RoomMessageEventContent::text_plain(format!( + (false, Ok(id)) => Ok(RoomMessageEventContent::text_plain(format!( "Refusing to overwrite in use alias for {id}, use -f or --force to overwrite" ))), - (_, Ok(None)) => match services + (_, Err(_)) => match services .rooms .alias .set_alias(&room_alias, &room_id, server_user) @@ -88,12 +89,11 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> Ok(()) => Ok(RoomMessageEventContent::text_plain("Successfully set alias")), Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), }, - (_, Err(err)) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {err}"))), }, RoomAliasCommand::Remove { .. - } => match services.rooms.alias.resolve_local_alias(&room_alias) { - Ok(Some(id)) => match services + } => match services.rooms.alias.resolve_local_alias(&room_alias).await { + Ok(id) => match services .rooms .alias .remove_alias(&room_alias, server_user) @@ -102,15 +102,13 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> Ok(()) => Ok(RoomMessageEventContent::text_plain(format!("Removed alias from {id}"))), Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), }, - Ok(None) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {err}"))), + Err(_) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), }, RoomAliasCommand::Which { .. - } => match services.rooms.alias.resolve_local_alias(&room_alias) { - Ok(Some(id)) => Ok(RoomMessageEventContent::text_plain(format!("Alias resolves to {id}"))), - Ok(None) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to lookup alias: {err}"))), + } => match services.rooms.alias.resolve_local_alias(&room_alias).await { + Ok(id) => Ok(RoomMessageEventContent::text_plain(format!("Alias resolves to {id}"))), + Err(_) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), }, RoomAliasCommand::List { .. @@ -125,63 +123,59 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> .rooms .alias .local_aliases_for_room(&room_id) - .collect::, _>>(); - match aliases { - Ok(aliases) => { - let plain_list = aliases.iter().fold(String::new(), |mut output, alias| { - writeln!(output, "- {alias}").expect("should be able to write to string buffer"); - output - }); + .map(Into::into) + .collect::>() + .await; - let html_list = aliases.iter().fold(String::new(), |mut output, alias| { - writeln!(output, "
  • {}
    • ", escape_html(alias.as_ref())) - .expect("should be able to write to string buffer"); - output - }); + let plain_list = aliases.iter().fold(String::new(), |mut output, alias| { + writeln!(output, "- {alias}").expect("should be able to write to string buffer"); + output + }); - let plain = format!("Aliases for {room_id}:\n{plain_list}"); - let html = format!("Aliases for {room_id}:\n
      {html_list}
    "); - Ok(RoomMessageEventContent::text_html(plain, html)) - }, - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to list aliases: {err}"))), - } + let html_list = aliases.iter().fold(String::new(), |mut output, alias| { + writeln!(output, "
  • {}
    • ", escape_html(alias.as_ref())) + .expect("should be able to write to string buffer"); + output + }); + + let plain = format!("Aliases for {room_id}:\n{plain_list}"); + let html = format!("Aliases for {room_id}:\n
      {html_list}
    "); + Ok(RoomMessageEventContent::text_html(plain, html)) } else { let aliases = services .rooms .alias .all_local_aliases() - .collect::, _>>(); - match aliases { - Ok(aliases) => { - let server_name = services.globals.server_name(); - let plain_list = aliases - .iter() - .fold(String::new(), |mut output, (alias, id)| { - writeln!(output, "- `{alias}` -> #{id}:{server_name}") - .expect("should be able to write to string buffer"); - output - }); + .map(|(room_id, localpart)| (room_id.into(), localpart.into())) + .collect::>() + .await; - let html_list = aliases - .iter() - .fold(String::new(), |mut output, (alias, id)| { - writeln!( - output, - "
  • {} -> #{}:{}
    • ", - escape_html(alias.as_ref()), - escape_html(id.as_ref()), - server_name - ) - .expect("should be able to write to string buffer"); - output - }); + let server_name = services.globals.server_name(); + let plain_list = aliases + .iter() + .fold(String::new(), |mut output, (alias, id)| { + writeln!(output, "- `{alias}` -> #{id}:{server_name}") + .expect("should be able to write to string buffer"); + output + }); - let plain = format!("Aliases:\n{plain_list}"); - let html = format!("Aliases:\n
      {html_list}
    "); - Ok(RoomMessageEventContent::text_html(plain, html)) - }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Unable to list room aliases: {e}"))), - } + let html_list = aliases + .iter() + .fold(String::new(), |mut output, (alias, id)| { + writeln!( + output, + "
  • {} -> #{}:{}
    • ", + escape_html(alias.as_ref()), + escape_html(id), + server_name + ) + .expect("should be able to write to string buffer"); + output + }); + + let plain = format!("Aliases:\n{plain_list}"); + let html = format!("Aliases:\n
      {html_list}
    "); + Ok(RoomMessageEventContent::text_html(plain, html)) } }, } diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index 2adfa7d7..1c90a998 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -1,11 +1,12 @@ use conduit::Result; -use ruma::events::room::message::RoomMessageEventContent; +use futures::StreamExt; +use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId}; use crate::{admin_command, get_room_info, PAGE_SIZE}; #[admin_command] pub(super) async fn list_rooms( - &self, page: Option, exclude_disabled: bool, exclude_banned: bool, no_details: bool, + &self, page: Option, _exclude_disabled: bool, _exclude_banned: bool, no_details: bool, ) -> Result { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); @@ -14,37 +15,12 @@ pub(super) async fn list_rooms( .rooms .metadata .iter_ids() - .filter_map(|room_id| { - room_id - .ok() - .filter(|room_id| { - if exclude_disabled - && self - .services - .rooms - .metadata - .is_disabled(room_id) - .unwrap_or(false) - { - return false; - } + //.filter(|room_id| async { !exclude_disabled || !self.services.rooms.metadata.is_disabled(room_id).await }) + //.filter(|room_id| async { !exclude_banned || !self.services.rooms.metadata.is_banned(room_id).await }) + .then(|room_id| get_room_info(self.services, room_id)) + .collect::>() + .await; - if exclude_banned - && self - .services - .rooms - .metadata - .is_banned(room_id) - .unwrap_or(false) - { - return false; - } - - true - }) - .map(|room_id| get_room_info(self.services, &room_id)) - }) - .collect::>(); rooms.sort_by_key(|r| r.1); rooms.reverse(); @@ -74,3 +50,10 @@ pub(super) async fn list_rooms( Ok(RoomMessageEventContent::notice_markdown(output_plain)) } + +#[admin_command] +pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result { + let result = self.services.rooms.metadata.exists(&room_id).await; + + Ok(RoomMessageEventContent::notice_markdown(format!("{result}"))) +} diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index 7bba2eb7..7ccdea6f 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -2,7 +2,8 @@ use std::fmt::Write; use clap::Subcommand; use conduit::Result; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId}; +use futures::StreamExt; +use ruma::{events::room::message::RoomMessageEventContent, RoomId}; use crate::{escape_html, get_room_info, Command, PAGE_SIZE}; @@ -31,15 +32,15 @@ pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_> match command { RoomDirectoryCommand::Publish { room_id, - } => match services.rooms.directory.set_public(&room_id) { - Ok(()) => Ok(RoomMessageEventContent::text_plain("Room published")), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to update room: {err}"))), + } => { + services.rooms.directory.set_public(&room_id); + Ok(RoomMessageEventContent::notice_plain("Room published")) }, RoomDirectoryCommand::Unpublish { room_id, - } => match services.rooms.directory.set_not_public(&room_id) { - Ok(()) => Ok(RoomMessageEventContent::text_plain("Room unpublished")), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Unable to update room: {err}"))), + } => { + services.rooms.directory.set_not_public(&room_id); + Ok(RoomMessageEventContent::notice_plain("Room unpublished")) }, RoomDirectoryCommand::List { page, @@ -50,9 +51,10 @@ pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_> .rooms .directory .public_rooms() - .filter_map(Result::ok) - .map(|id: OwnedRoomId| get_room_info(services, &id)) - .collect::>(); + .then(|room_id| get_room_info(services, room_id)) + .collect::>() + .await; + rooms.sort_by_key(|r| r.1); rooms.reverse(); diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index d17a2924..fc0619e3 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -1,5 +1,6 @@ use clap::Subcommand; -use conduit::Result; +use conduit::{utils::ReadyExt, Result}; +use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId}; use crate::{admin_command, admin_command_dispatch}; @@ -32,46 +33,42 @@ async fn list_joined_members(&self, room_id: Box, local_only: bool) -> R .rooms .state_accessor .get_name(&room_id) - .ok() - .flatten() - .unwrap_or_else(|| room_id.to_string()); + .await + .unwrap_or_else(|_| room_id.to_string()); - let members = self + let member_info: Vec<_> = self .services .rooms .state_cache .room_members(&room_id) - .filter_map(|member| { + .ready_filter(|user_id| { if local_only { - member - .ok() - .filter(|user| self.services.globals.user_is_local(user)) + self.services.globals.user_is_local(user_id) } else { - member.ok() + true } - }); - - let member_info = members - .into_iter() - .map(|user_id| { - ( - user_id.clone(), + }) + .filter_map(|user_id| async move { + let user_id = user_id.to_owned(); + Some(( self.services .users .displayname(&user_id) - .unwrap_or(None) - .unwrap_or_else(|| user_id.to_string()), - ) + .await + .unwrap_or_else(|_| user_id.to_string()), + user_id, + )) }) - .collect::>(); + .collect() + .await; let output_plain = format!( "{} Members in Room \"{}\":\n```\n{}\n```", member_info.len(), room_name, member_info - .iter() - .map(|(mxid, displayname)| format!("{mxid} | {displayname}")) + .into_iter() + .map(|(displayname, mxid)| format!("{mxid} | {displayname}")) .collect::>() .join("\n") ); @@ -81,11 +78,12 @@ async fn list_joined_members(&self, room_id: Box, local_only: bool) -> R #[admin_command] async fn view_room_topic(&self, room_id: Box) -> Result { - let Some(room_topic) = self + let Ok(room_topic) = self .services .rooms .state_accessor - .get_room_topic(&room_id)? + .get_room_topic(&room_id) + .await else { return Ok(RoomMessageEventContent::text_plain("Room does not have a room topic set.")); }; diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index 64d2af45..8c6cbeaa 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -6,6 +6,7 @@ mod moderation; use clap::Subcommand; use conduit::Result; +use ruma::OwnedRoomId; use self::{ alias::RoomAliasCommand, directory::RoomDirectoryCommand, info::RoomInfoCommand, moderation::RoomModerationCommand, @@ -49,4 +50,9 @@ pub(super) enum RoomCommand { #[command(subcommand)] /// - Manage the room directory Directory(RoomDirectoryCommand), + + /// - Check if we know about a room + Exists { + room_id: OwnedRoomId, + }, } diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index 70d8486b..9a772da4 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -1,6 +1,11 @@ use api::client::leave_room; use clap::Subcommand; -use conduit::{debug, error, info, warn, Result}; +use conduit::{ + debug, error, info, + utils::{IterStream, ReadyExt}, + warn, Result, +}; +use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId}; use crate::{admin_command, admin_command_dispatch, get_room_info}; @@ -76,7 +81,7 @@ async fn ban_room( let admin_room_alias = &self.services.globals.admin_alias; - if let Some(admin_room_id) = self.services.admin.get_admin_room()? { + if let Ok(admin_room_id) = self.services.admin.get_admin_room().await { if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) { return Ok(RoomMessageEventContent::text_plain("Not allowed to ban the admin room.")); } @@ -95,7 +100,7 @@ async fn ban_room( debug!("Room specified is a room ID, banning room ID"); - self.services.rooms.metadata.ban_room(&room_id, true)?; + self.services.rooms.metadata.ban_room(&room_id, true); room_id } else if room.is_room_alias_id() { @@ -114,7 +119,13 @@ async fn ban_room( get_alias_helper to fetch room ID remotely" ); - let room_id = if let Some(room_id) = self.services.rooms.alias.resolve_local_alias(&room_alias)? { + let room_id = if let Ok(room_id) = self + .services + .rooms + .alias + .resolve_local_alias(&room_alias) + .await + { room_id } else { debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); @@ -138,7 +149,7 @@ async fn ban_room( } }; - self.services.rooms.metadata.ban_room(&room_id, true)?; + self.services.rooms.metadata.ban_room(&room_id, true); room_id } else { @@ -150,56 +161,40 @@ async fn ban_room( debug!("Making all users leave the room {}", &room); if force { - for local_user in self + let mut users = self .services .rooms .state_cache .room_members(&room_id) - .filter_map(|user| { - user.ok().filter(|local_user| { - self.services.globals.user_is_local(local_user) - // additional wrapped check here is to avoid adding remote users - // who are in the admin room to the list of local users (would - // fail auth check) - && (self.services.globals.user_is_local(local_user) - // since this is a force operation, assume user is an admin - // if somehow this fails - && self.services - .users - .is_admin(local_user) - .unwrap_or(true)) - }) - }) { + .ready_filter(|user| self.services.globals.user_is_local(user)) + .boxed(); + + while let Some(local_user) = users.next().await { debug!( - "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", - &local_user, &room_id + "Attempting leave for user {local_user} in room {room_id} (forced, ignoring all errors, evicting \ + admins too)", ); - if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await { + if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { warn!(%e, "Failed to leave room"); } } } else { - for local_user in self + let mut users = self .services .rooms .state_cache .room_members(&room_id) - .filter_map(|user| { - user.ok().filter(|local_user| { - local_user.server_name() == self.services.globals.server_name() - // additional wrapped check here is to avoid adding remote users - // who are in the admin room to the list of local users (would fail auth check) - && (local_user.server_name() - == self.services.globals.server_name() - && !self.services - .users - .is_admin(local_user) - .unwrap_or(false)) - }) - }) { + .ready_filter(|user| self.services.globals.user_is_local(user)) + .boxed(); + + while let Some(local_user) = users.next().await { + if self.services.users.is_admin(local_user).await { + continue; + } + debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await { + if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { error!( "Error attempting to make local user {} leave room {} during room banning: {}", &local_user, &room_id, e @@ -214,12 +209,14 @@ async fn ban_room( } // remove any local aliases, ignore errors - for ref local_alias in self + for local_alias in &self .services .rooms .alias .local_aliases_for_room(&room_id) - .filter_map(Result::ok) + .map(ToOwned::to_owned) + .collect::>() + .await { _ = self .services @@ -230,10 +227,10 @@ async fn ban_room( } // unpublish from room directory, ignore errors - _ = self.services.rooms.directory.set_not_public(&room_id); + self.services.rooms.directory.set_not_public(&room_id); if disable_federation { - self.services.rooms.metadata.disable_room(&room_id, true)?; + self.services.rooms.metadata.disable_room(&room_id, true); return Ok(RoomMessageEventContent::text_plain( "Room banned, removed all our local users, and disabled incoming federation with room.", )); @@ -268,7 +265,7 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu for &room in &rooms_s { match <&RoomOrAliasId>::try_from(room) { Ok(room_alias_or_id) => { - if let Some(admin_room_id) = self.services.admin.get_admin_room()? { + if let Ok(admin_room_id) = self.services.admin.get_admin_room().await { if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias) { info!("User specified admin room in bulk ban list, ignoring"); continue; @@ -300,43 +297,48 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu if room_alias_or_id.is_room_alias_id() { match RoomAliasId::parse(room_alias_or_id) { Ok(room_alias) => { - let room_id = - if let Some(room_id) = self.services.rooms.alias.resolve_local_alias(&room_alias)? { - room_id - } else { - debug!( - "We don't have this room alias to a room ID locally, attempting to fetch room \ - ID over federation" - ); + let room_id = if let Ok(room_id) = self + .services + .rooms + .alias + .resolve_local_alias(&room_alias) + .await + { + room_id + } else { + debug!( + "We don't have this room alias to a room ID locally, attempting to fetch room ID \ + over federation" + ); - match self - .services - .rooms - .alias - .resolve_alias(&room_alias, None) - .await - { - Ok((room_id, servers)) => { - debug!( - ?room_id, - ?servers, - "Got federation response fetching room ID for {room}", - ); - room_id - }, - Err(e) => { - // don't fail if force blocking - if force { - warn!("Failed to resolve room alias {room} to a room ID: {e}"); - continue; - } + match self + .services + .rooms + .alias + .resolve_alias(&room_alias, None) + .await + { + Ok((room_id, servers)) => { + debug!( + ?room_id, + ?servers, + "Got federation response fetching room ID for {room}", + ); + room_id + }, + Err(e) => { + // don't fail if force blocking + if force { + warn!("Failed to resolve room alias {room} to a room ID: {e}"); + continue; + } - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to resolve room alias {room} to a room ID: {e}" - ))); - }, - } - }; + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to resolve room alias {room} to a room ID: {e}" + ))); + }, + } + }; room_ids.push(room_id); }, @@ -374,74 +376,52 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu } for room_id in room_ids { - if self - .services - .rooms - .metadata - .ban_room(&room_id, true) - .is_ok() - { - debug!("Banned {room_id} successfully"); - room_ban_count = room_ban_count.saturating_add(1); - } + self.services.rooms.metadata.ban_room(&room_id, true); + + debug!("Banned {room_id} successfully"); + room_ban_count = room_ban_count.saturating_add(1); debug!("Making all users leave the room {}", &room_id); if force { - for local_user in self + let mut users = self .services .rooms .state_cache .room_members(&room_id) - .filter_map(|user| { - user.ok().filter(|local_user| { - local_user.server_name() == self.services.globals.server_name() - // additional wrapped check here is to avoid adding remote - // users who are in the admin room to the list of local - // users (would fail auth check) - && (local_user.server_name() - == self.services.globals.server_name() - // since this is a force operation, assume user is an - // admin if somehow this fails - && self.services - .users - .is_admin(local_user) - .unwrap_or(true)) - }) - }) { + .ready_filter(|user| self.services.globals.user_is_local(user)) + .boxed(); + + while let Some(local_user) = users.next().await { debug!( - "Attempting leave for user {} in room {} (forced, ignoring all errors, evicting admins too)", - &local_user, room_id + "Attempting leave for user {local_user} in room {room_id} (forced, ignoring all errors, evicting \ + admins too)", ); - if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await { + + if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { warn!(%e, "Failed to leave room"); } } } else { - for local_user in self + let mut users = self .services .rooms .state_cache .room_members(&room_id) - .filter_map(|user| { - user.ok().filter(|local_user| { - local_user.server_name() == self.services.globals.server_name() - // additional wrapped check here is to avoid adding remote - // users who are in the admin room to the list of local - // users (would fail auth check) - && (local_user.server_name() - == self.services.globals.server_name() - && !self.services - .users - .is_admin(local_user) - .unwrap_or(false)) - }) - }) { - debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(self.services, &local_user, &room_id, None).await { + .ready_filter(|user| self.services.globals.user_is_local(user)) + .boxed(); + + while let Some(local_user) = users.next().await { + if self.services.users.is_admin(local_user).await { + continue; + } + + debug!("Attempting leave for user {local_user} in room {room_id}"); + if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { error!( - "Error attempting to make local user {} leave room {} during bulk room banning: {}", - &local_user, &room_id, e + "Error attempting to make local user {local_user} leave room {room_id} during bulk room \ + banning: {e}", ); + return Ok(RoomMessageEventContent::text_plain(format!( "Error attempting to make local user {} leave room {} during room banning (room is still \ banned but not removing any more users and not banning any more rooms): {}\nIf you would \ @@ -453,26 +433,26 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu } // remove any local aliases, ignore errors - for ref local_alias in self - .services + self.services .rooms .alias .local_aliases_for_room(&room_id) - .filter_map(Result::ok) - { - _ = self - .services - .rooms - .alias - .remove_alias(local_alias, &self.services.globals.server_user) - .await; - } + .map(ToOwned::to_owned) + .for_each(|local_alias| async move { + self.services + .rooms + .alias + .remove_alias(&local_alias, &self.services.globals.server_user) + .await + .ok(); + }) + .await; // unpublish from room directory, ignore errors - _ = self.services.rooms.directory.set_not_public(&room_id); + self.services.rooms.directory.set_not_public(&room_id); if disable_federation { - self.services.rooms.metadata.disable_room(&room_id, true)?; + self.services.rooms.metadata.disable_room(&room_id, true); } } @@ -503,7 +483,7 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> debug!("Room specified is a room ID, unbanning room ID"); - self.services.rooms.metadata.ban_room(&room_id, false)?; + self.services.rooms.metadata.ban_room(&room_id, false); room_id } else if room.is_room_alias_id() { @@ -522,7 +502,13 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> get_alias_helper to fetch room ID remotely" ); - let room_id = if let Some(room_id) = self.services.rooms.alias.resolve_local_alias(&room_alias)? { + let room_id = if let Ok(room_id) = self + .services + .rooms + .alias + .resolve_local_alias(&room_alias) + .await + { room_id } else { debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); @@ -546,7 +532,7 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> } }; - self.services.rooms.metadata.ban_room(&room_id, false)?; + self.services.rooms.metadata.ban_room(&room_id, false); room_id } else { @@ -557,7 +543,7 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> }; if enable_federation { - self.services.rooms.metadata.disable_room(&room_id, false)?; + self.services.rooms.metadata.disable_room(&room_id, false); return Ok(RoomMessageEventContent::text_plain("Room unbanned.")); } @@ -569,45 +555,42 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> #[admin_command] async fn list_banned_rooms(&self, no_details: bool) -> Result { - let rooms = self + let room_ids = self .services .rooms .metadata .list_banned_rooms() - .collect::, _>>(); + .map(Into::into) + .collect::>() + .await; - match rooms { - Ok(room_ids) => { - if room_ids.is_empty() { - return Ok(RoomMessageEventContent::text_plain("No rooms are banned.")); - } - - let mut rooms = room_ids - .into_iter() - .map(|room_id| get_room_info(self.services, &room_id)) - .collect::>(); - rooms.sort_by_key(|r| r.1); - rooms.reverse(); - - let output_plain = format!( - "Rooms Banned ({}):\n```\n{}\n```", - rooms.len(), - rooms - .iter() - .map(|(id, members, name)| if no_details { - format!("{id}") - } else { - format!("{id}\tMembers: {members}\tName: {name}") - }) - .collect::>() - .join("\n") - ); - - Ok(RoomMessageEventContent::notice_markdown(output_plain)) - }, - Err(e) => { - error!("Failed to list banned rooms: {e}"); - Ok(RoomMessageEventContent::text_plain(format!("Unable to list banned rooms: {e}"))) - }, + if room_ids.is_empty() { + return Ok(RoomMessageEventContent::text_plain("No rooms are banned.")); } + + let mut rooms = room_ids + .iter() + .stream() + .then(|room_id| get_room_info(self.services, room_id)) + .collect::>() + .await; + + rooms.sort_by_key(|r| r.1); + rooms.reverse(); + + let output_plain = format!( + "Rooms Banned ({}):\n```\n{}\n```", + rooms.len(), + rooms + .iter() + .map(|(id, members, name)| if no_details { + format!("{id}") + } else { + format!("{id}\tMembers: {members}\tName: {name}") + }) + .collect::>() + .join("\n") + ); + + Ok(RoomMessageEventContent::notice_markdown(output_plain)) } diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 20691f1a..1b086856 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -1,7 +1,9 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room}; -use conduit::{error, info, utils, warn, PduBuilder, Result}; +use conduit::{error, info, is_equal_to, utils, warn, PduBuilder, Result}; +use conduit_api::client::{leave_all_rooms, update_avatar_url, update_displayname}; +use futures::StreamExt; use ruma::{ events::{ room::{ @@ -25,16 +27,19 @@ const AUTO_GEN_PASSWORD_LENGTH: usize = 25; #[admin_command] pub(super) async fn list_users(&self) -> Result { - match self.services.users.list_local_users() { - Ok(users) => { - let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); - plain_msg += users.join("\n").as_str(); - plain_msg += "\n```"; + let users = self + .services + .users + .list_local_users() + .map(ToString::to_string) + .collect::>() + .await; - Ok(RoomMessageEventContent::notice_markdown(plain_msg)) - }, - Err(e) => Ok(RoomMessageEventContent::text_plain(e.to_string())), - } + let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); + plain_msg += users.join("\n").as_str(); + plain_msg += "\n```"; + + Ok(RoomMessageEventContent::notice_markdown(plain_msg)) } #[admin_command] @@ -42,7 +47,7 @@ pub(super) async fn create_user(&self, username: String, password: Option )); } - self.services.users.deactivate_account(&user_id)?; + self.services.users.deactivate_account(&user_id).await?; if !no_leave_rooms { self.services @@ -175,17 +184,22 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> .send_message(RoomMessageEventContent::text_plain(format!( "Making {user_id} leave all rooms after deactivation..." ))) - .await; + .await + .ok(); let all_joined_rooms: Vec = self .services .rooms .state_cache .rooms_joined(&user_id) - .filter_map(Result::ok) - .collect(); + .map(Into::into) + .collect() + .await; - full_user_deactivate(self.services, &user_id, all_joined_rooms).await?; + full_user_deactivate(self.services, &user_id, &all_joined_rooms).await?; + update_displayname(self.services, &user_id, None, &all_joined_rooms).await?; + update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms).await?; + leave_all_rooms(self.services, &user_id).await; } Ok(RoomMessageEventContent::text_plain(format!( @@ -238,15 +252,16 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> let mut admins = Vec::new(); for username in usernames { - match parse_active_local_user_id(self.services, username) { + match parse_active_local_user_id(self.services, username).await { Ok(user_id) => { - if self.services.users.is_admin(&user_id)? && !force { + if self.services.users.is_admin(&user_id).await && !force { self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( "{username} is an admin and --force is not set, skipping over" ))) - .await; + .await + .ok(); admins.push(username); continue; } @@ -258,7 +273,8 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> .send_message(RoomMessageEventContent::text_plain(format!( "{username} is the server service account, skipping over" ))) - .await; + .await + .ok(); continue; } @@ -270,7 +286,8 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> .send_message(RoomMessageEventContent::text_plain(format!( "{username} is not a valid username, skipping over: {e}" ))) - .await; + .await + .ok(); continue; }, } @@ -279,7 +296,7 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> let mut deactivation_count: usize = 0; for user_id in user_ids { - match self.services.users.deactivate_account(&user_id) { + match self.services.users.deactivate_account(&user_id).await { Ok(()) => { deactivation_count = deactivation_count.saturating_add(1); if !no_leave_rooms { @@ -289,16 +306,26 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> .rooms .state_cache .rooms_joined(&user_id) - .filter_map(Result::ok) - .collect(); - full_user_deactivate(self.services, &user_id, all_joined_rooms).await?; + .map(Into::into) + .collect() + .await; + + full_user_deactivate(self.services, &user_id, &all_joined_rooms).await?; + update_displayname(self.services, &user_id, None, &all_joined_rooms) + .await + .ok(); + update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms) + .await + .ok(); + leave_all_rooms(self.services, &user_id).await; } }, Err(e) => { self.services .admin .send_message(RoomMessageEventContent::text_plain(format!("Failed deactivating user: {e}"))) - .await; + .await + .ok(); }, } } @@ -326,9 +353,9 @@ pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result(&room_id, &StateEventType::RoomPowerLevels, "") + .await + .ok(); let user_can_demote_self = room_power_levels .as_ref() @@ -417,9 +443,9 @@ pub(super) async fn force_demote( .services .rooms .state_accessor - .room_state_get(&room_id, &StateEventType::RoomCreate, "")? - .as_ref() - .is_some_and(|event| event.sender == user_id); + .room_state_get(&room_id, &StateEventType::RoomCreate, "") + .await + .is_ok_and(|event| event.sender == user_id); if !user_can_demote_self { return Ok(RoomMessageEventContent::notice_markdown( @@ -473,15 +499,16 @@ pub(super) async fn make_user_admin(&self, user_id: String) -> Result, tag: String, ) -> Result { - let user_id = parse_active_local_user_id(self.services, &user_id)?; + let user_id = parse_active_local_user_id(self.services, &user_id).await?; let event = self .services .account_data - .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?; + .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag) + .await; let mut tags_event = event.map_or_else( - || TagEvent { + |_| TagEvent { content: TagEventContent { tags: BTreeMap::new(), }, @@ -494,12 +521,15 @@ pub(super) async fn put_room_tag( .tags .insert(tag.clone().into(), TagInfo::new()); - self.services.account_data.update( - Some(&room_id), - &user_id, - RoomAccountDataEventType::Tag, - &serde_json::to_value(tags_event).expect("to json value always works"), - )?; + self.services + .account_data + .update( + Some(&room_id), + &user_id, + RoomAccountDataEventType::Tag, + &serde_json::to_value(tags_event).expect("to json value always works"), + ) + .await?; Ok(RoomMessageEventContent::text_plain(format!( "Successfully updated room account data for {user_id} and room {room_id} with tag {tag}" @@ -510,15 +540,16 @@ pub(super) async fn put_room_tag( pub(super) async fn delete_room_tag( &self, user_id: String, room_id: Box, tag: String, ) -> Result { - let user_id = parse_active_local_user_id(self.services, &user_id)?; + let user_id = parse_active_local_user_id(self.services, &user_id).await?; let event = self .services .account_data - .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?; + .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag) + .await; let mut tags_event = event.map_or_else( - || TagEvent { + |_| TagEvent { content: TagEventContent { tags: BTreeMap::new(), }, @@ -528,12 +559,15 @@ pub(super) async fn delete_room_tag( tags_event.content.tags.remove(&tag.clone().into()); - self.services.account_data.update( - Some(&room_id), - &user_id, - RoomAccountDataEventType::Tag, - &serde_json::to_value(tags_event).expect("to json value always works"), - )?; + self.services + .account_data + .update( + Some(&room_id), + &user_id, + RoomAccountDataEventType::Tag, + &serde_json::to_value(tags_event).expect("to json value always works"), + ) + .await?; Ok(RoomMessageEventContent::text_plain(format!( "Successfully updated room account data for {user_id} and room {room_id}, deleting room tag {tag}" @@ -542,15 +576,16 @@ pub(super) async fn delete_room_tag( #[admin_command] pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box) -> Result { - let user_id = parse_active_local_user_id(self.services, &user_id)?; + let user_id = parse_active_local_user_id(self.services, &user_id).await?; let event = self .services .account_data - .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag)?; + .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag) + .await; let tags_event = event.map_or_else( - || TagEvent { + |_| TagEvent { content: TagEventContent { tags: BTreeMap::new(), }, @@ -566,11 +601,12 @@ pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box) #[admin_command] pub(super) async fn redact_event(&self, event_id: Box) -> Result { - let Some(event) = self + let Ok(event) = self .services .rooms .timeline - .get_non_outlier_pdu(&event_id)? + .get_non_outlier_pdu(&event_id) + .await else { return Ok(RoomMessageEventContent::text_plain("Event does not exist in our database.")); }; diff --git a/src/admin/utils.rs b/src/admin/utils.rs index 8d3d15ae..ba98bbea 100644 --- a/src/admin/utils.rs +++ b/src/admin/utils.rs @@ -8,23 +8,21 @@ pub(crate) fn escape_html(s: &str) -> String { .replace('>', ">") } -pub(crate) fn get_room_info(services: &Services, id: &RoomId) -> (OwnedRoomId, u64, String) { +pub(crate) async fn get_room_info(services: &Services, room_id: &RoomId) -> (OwnedRoomId, u64, String) { ( - id.into(), + room_id.into(), services .rooms .state_cache - .room_joined_count(id) - .ok() - .flatten() + .room_joined_count(room_id) + .await .unwrap_or(0), services .rooms .state_accessor - .get_name(id) - .ok() - .flatten() - .unwrap_or_else(|| id.to_string()), + .get_name(room_id) + .await + .unwrap_or_else(|_| room_id.to_string()), ) } @@ -46,14 +44,14 @@ pub(crate) fn parse_local_user_id(services: &Services, user_id: &str) -> Result< } /// Parses user ID that is an active (not guest or deactivated) local user -pub(crate) fn parse_active_local_user_id(services: &Services, user_id: &str) -> Result { +pub(crate) async fn parse_active_local_user_id(services: &Services, user_id: &str) -> Result { let user_id = parse_local_user_id(services, user_id)?; - if !services.users.exists(&user_id)? { + if !services.users.exists(&user_id).await { return Err!("User {user_id:?} does not exist on this server."); } - if services.users.is_deactivated(&user_id)? { + if services.users.is_deactivated(&user_id).await? { return Err!("User {user_id:?} is deactivated."); } diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 2b89c3e8..6e37cb40 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -45,7 +45,7 @@ conduit-core.workspace = true conduit-database.workspace = true conduit-service.workspace = true const-str.workspace = true -futures-util.workspace = true +futures.workspace = true hmac.workspace = true http.workspace = true http-body-util.workspace = true diff --git a/src/api/client/account.rs b/src/api/client/account.rs index cee86f80..63d02f8f 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -2,7 +2,8 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{debug_info, error, info, utils, warn, Error, PduBuilder, Result}; +use conduit::{debug_info, error, info, is_equal_to, utils, utils::ReadyExt, warn, Error, PduBuilder, Result}; +use futures::{FutureExt, StreamExt}; use register::RegistrationKind; use ruma::{ api::client::{ @@ -55,7 +56,7 @@ pub(crate) async fn get_register_available_route( .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; // Check if username is creative enough - if services.users.exists(&user_id)? { + if services.users.exists(&user_id).await { return Err(Error::BadRequest(ErrorKind::UserInUse, "Desired user ID is already taken.")); } @@ -125,7 +126,7 @@ pub(crate) async fn register_route( // forbid guests from registering if there is not a real admin user yet. give // generic user error. - if is_guest && services.users.count()? < 2 { + if is_guest && services.users.count().await < 2 { warn!( "Guest account attempted to register before a real admin user has been registered, rejecting \ registration. Guest's initial device name: {:?}", @@ -142,7 +143,7 @@ pub(crate) async fn register_route( .filter(|user_id| !user_id.is_historical() && services.globals.user_is_local(user_id)) .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; - if services.users.exists(&proposed_user_id)? { + if services.users.exists(&proposed_user_id).await { return Err(Error::BadRequest(ErrorKind::UserInUse, "Desired user ID is already taken.")); } @@ -162,7 +163,7 @@ pub(crate) async fn register_route( services.globals.server_name(), ) .unwrap(); - if !services.users.exists(&proposed_user_id)? { + if !services.users.exists(&proposed_user_id).await { break proposed_user_id; } }, @@ -210,12 +211,15 @@ pub(crate) async fn register_route( if !skip_auth { if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services.uiaa.try_auth( - &UserId::parse_with_server_name("", services.globals.server_name()).expect("we know this is valid"), - "".into(), - auth, - &uiaainfo, - )?; + let (worked, uiaainfo) = services + .uiaa + .try_auth( + &UserId::parse_with_server_name("", services.globals.server_name()).expect("we know this is valid"), + "".into(), + auth, + &uiaainfo, + ) + .await?; if !worked { return Err(Error::Uiaa(uiaainfo)); } @@ -227,7 +231,7 @@ pub(crate) async fn register_route( "".into(), &uiaainfo, &json, - )?; + ); return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); @@ -255,21 +259,23 @@ pub(crate) async fn register_route( services .users - .set_displayname(&user_id, Some(displayname.clone())) - .await?; + .set_displayname(&user_id, Some(displayname.clone())); // Initial account data - services.account_data.update( - None, - &user_id, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(ruma::events::push_rules::PushRulesEvent { - content: ruma::events::push_rules::PushRulesEventContent { - global: push::Ruleset::server_default(&user_id), - }, - }) - .expect("to json always works"), - )?; + services + .account_data + .update( + None, + &user_id, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(ruma::events::push_rules::PushRulesEvent { + content: ruma::events::push_rules::PushRulesEventContent { + global: push::Ruleset::server_default(&user_id), + }, + }) + .expect("to json always works"), + ) + .await?; // Inhibit login does not work for guests if !is_guest && body.inhibit_login { @@ -294,13 +300,16 @@ pub(crate) async fn register_route( let token = utils::random_string(TOKEN_LENGTH); // Create device for this account - services.users.create_device( - &user_id, - &device_id, - &token, - body.initial_device_display_name.clone(), - Some(client.to_string()), - )?; + services + .users + .create_device( + &user_id, + &device_id, + &token, + body.initial_device_display_name.clone(), + Some(client.to_string()), + ) + .await?; debug_info!(%user_id, %device_id, "User account was created"); @@ -318,7 +327,8 @@ pub(crate) async fn register_route( "New user \"{user_id}\" registered on this server from IP {client} and device display name \ \"{device_display_name}\"" ))) - .await; + .await + .ok(); } } else { info!("New user \"{user_id}\" registered on this server."); @@ -329,7 +339,8 @@ pub(crate) async fn register_route( .send_message(RoomMessageEventContent::notice_plain(format!( "New user \"{user_id}\" registered on this server from IP {client}" ))) - .await; + .await + .ok(); } } } @@ -346,7 +357,8 @@ pub(crate) async fn register_route( "Guest user \"{user_id}\" with device display name \"{device_display_name}\" registered on \ this server from IP {client}" ))) - .await; + .await + .ok(); } } else { #[allow(clippy::collapsible_else_if)] @@ -357,7 +369,8 @@ pub(crate) async fn register_route( "Guest user \"{user_id}\" with no device display name registered on this server from IP \ {client}", ))) - .await; + .await + .ok(); } } } @@ -365,10 +378,15 @@ pub(crate) async fn register_route( // If this is the first real user, grant them admin privileges except for guest // users Note: the server user, @conduit:servername, is generated first if !is_guest { - if let Some(admin_room) = services.admin.get_admin_room()? { - if services.rooms.state_cache.room_joined_count(&admin_room)? == Some(1) { + if let Ok(admin_room) = services.admin.get_admin_room().await { + if services + .rooms + .state_cache + .room_joined_count(&admin_room) + .await + .is_ok_and(is_equal_to!(1)) + { services.admin.make_user_admin(&user_id).await?; - warn!("Granting {user_id} admin privileges as the first user"); } } @@ -382,7 +400,8 @@ pub(crate) async fn register_route( if !services .rooms .state_cache - .server_in_room(services.globals.server_name(), room)? + .server_in_room(services.globals.server_name(), room) + .await { warn!("Skipping room {room} to automatically join as we have never joined before."); continue; @@ -398,6 +417,7 @@ pub(crate) async fn register_route( None, &body.appservice_info, ) + .boxed() .await { // don't return this error so we don't fail registrations @@ -461,16 +481,20 @@ pub(crate) async fn change_password_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = services .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo)?; + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; + if !worked { return Err(Error::Uiaa(uiaainfo)); } - // Success! + + // Success! } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json)?; + .create(sender_user, sender_device, &uiaainfo, &json); + return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); @@ -482,14 +506,12 @@ pub(crate) async fn change_password_route( if body.logout_devices { // Logout all devices except the current one - for id in services + services .users .all_device_ids(sender_user) - .filter_map(Result::ok) - .filter(|id| id != sender_device) - { - services.users.remove_device(sender_user, &id)?; - } + .ready_filter(|id| id != sender_device) + .for_each(|id| services.users.remove_device(sender_user, id)) + .await; } info!("User {sender_user} changed their password."); @@ -500,7 +522,8 @@ pub(crate) async fn change_password_route( .send_message(RoomMessageEventContent::notice_plain(format!( "User {sender_user} changed their password." ))) - .await; + .await + .ok(); } Ok(change_password::v3::Response {}) @@ -520,7 +543,7 @@ pub(crate) async fn whoami_route( Ok(whoami::v3::Response { user_id: sender_user.clone(), device_id, - is_guest: services.users.is_deactivated(sender_user)? && body.appservice_info.is_none(), + is_guest: services.users.is_deactivated(sender_user).await? && body.appservice_info.is_none(), }) } @@ -561,7 +584,9 @@ pub(crate) async fn deactivate_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = services .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo)?; + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; + if !worked { return Err(Error::Uiaa(uiaainfo)); } @@ -570,7 +595,8 @@ pub(crate) async fn deactivate_route( uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json)?; + .create(sender_user, sender_device, &uiaainfo, &json); + return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); @@ -581,10 +607,14 @@ pub(crate) async fn deactivate_route( .rooms .state_cache .rooms_joined(sender_user) - .filter_map(Result::ok) - .collect(); + .map(Into::into) + .collect() + .await; - full_user_deactivate(&services, sender_user, all_joined_rooms).await?; + super::update_displayname(&services, sender_user, None, &all_joined_rooms).await?; + super::update_avatar_url(&services, sender_user, None, None, &all_joined_rooms).await?; + + full_user_deactivate(&services, sender_user, &all_joined_rooms).await?; info!("User {sender_user} deactivated their account."); @@ -594,7 +624,8 @@ pub(crate) async fn deactivate_route( .send_message(RoomMessageEventContent::notice_plain(format!( "User {sender_user} deactivated their account." ))) - .await; + .await + .ok(); } Ok(deactivate::v3::Response { @@ -674,34 +705,27 @@ pub(crate) async fn check_registration_token_validity( /// - Removing all profile data /// - Leaving all rooms (and forgets all of them) pub async fn full_user_deactivate( - services: &Services, user_id: &UserId, all_joined_rooms: Vec, + services: &Services, user_id: &UserId, all_joined_rooms: &[OwnedRoomId], ) -> Result<()> { - services.users.deactivate_account(user_id)?; + services.users.deactivate_account(user_id).await?; + super::update_displayname(services, user_id, None, all_joined_rooms).await?; + super::update_avatar_url(services, user_id, None, None, all_joined_rooms).await?; - super::update_displayname(services, user_id, None, all_joined_rooms.clone()).await?; - super::update_avatar_url(services, user_id, None, None, all_joined_rooms.clone()).await?; - - let all_profile_keys = services + services .users .all_profile_keys(user_id) - .filter_map(Result::ok); - - for (profile_key, _profile_value) in all_profile_keys { - if let Err(e) = services.users.set_profile_key(user_id, &profile_key, None) { - warn!("Failed removing {user_id} profile key {profile_key}: {e}"); - } - } + .ready_for_each(|(profile_key, _)| services.users.set_profile_key(user_id, &profile_key, None)) + .await; for room_id in all_joined_rooms { - let state_lock = services.rooms.state.mutex.lock(&room_id).await; + let state_lock = services.rooms.state.mutex.lock(room_id).await; let room_power_levels = services .rooms .state_accessor - .room_state_get(&room_id, &StateEventType::RoomPowerLevels, "")? - .as_ref() - .and_then(|event| serde_json::from_str(event.content.get()).ok()?) - .and_then(|content: RoomPowerLevelsEventContent| content.into()); + .room_state_get_content::(room_id, &StateEventType::RoomPowerLevels, "") + .await + .ok(); let user_can_demote_self = room_power_levels .as_ref() @@ -710,9 +734,9 @@ pub async fn full_user_deactivate( }) || services .rooms .state_accessor - .room_state_get(&room_id, &StateEventType::RoomCreate, "")? - .as_ref() - .is_some_and(|event| event.sender == user_id); + .room_state_get(room_id, &StateEventType::RoomCreate, "") + .await + .is_ok_and(|event| event.sender == user_id); if user_can_demote_self { let mut power_levels_content = room_power_levels.unwrap_or_default(); @@ -732,7 +756,7 @@ pub async fn full_user_deactivate( timestamp: None, }, user_id, - &room_id, + room_id, &state_lock, ) .await diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 12d6352c..2399a355 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -1,11 +1,9 @@ use axum::extract::State; -use conduit::{debug, Error, Result}; +use conduit::{debug, Err, Result}; +use futures::StreamExt; use rand::seq::SliceRandom; use ruma::{ - api::client::{ - alias::{create_alias, delete_alias, get_alias}, - error::ErrorKind, - }, + api::client::alias::{create_alias, delete_alias, get_alias}, OwnedServerName, RoomAliasId, RoomId, }; use service::Services; @@ -33,16 +31,17 @@ pub(crate) async fn create_alias_route( .forbidden_alias_names() .is_match(body.room_alias.alias()) { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Room alias is forbidden.")); + return Err!(Request(Forbidden("Room alias is forbidden."))); } if services .rooms .alias - .resolve_local_alias(&body.room_alias)? - .is_some() + .resolve_local_alias(&body.room_alias) + .await + .is_ok() { - return Err(Error::Conflict("Alias already exists.")); + return Err!(Conflict("Alias already exists.")); } services @@ -95,16 +94,16 @@ pub(crate) async fn get_alias_route( .resolve_alias(&room_alias, servers.as_ref()) .await else { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room with alias not found.")); + return Err!(Request(NotFound("Room with alias not found."))); }; - let servers = room_available_servers(&services, &room_id, &room_alias, &pre_servers); + let servers = room_available_servers(&services, &room_id, &room_alias, &pre_servers).await; debug!(?room_alias, ?room_id, "available servers: {servers:?}"); Ok(get_alias::v3::Response::new(room_id, servers)) } -fn room_available_servers( +async fn room_available_servers( services: &Services, room_id: &RoomId, room_alias: &RoomAliasId, pre_servers: &Option>, ) -> Vec { // find active servers in room state cache to suggest @@ -112,8 +111,9 @@ fn room_available_servers( .rooms .state_cache .room_servers(room_id) - .filter_map(Result::ok) - .collect(); + .map(ToOwned::to_owned) + .collect() + .await; // push any servers we want in the list already (e.g. responded remote alias // servers, room alias server itself) diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index 4ead8777..d52da80a 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -1,18 +1,16 @@ use axum::extract::State; +use conduit::{err, Err}; use ruma::{ - api::client::{ - backup::{ - add_backup_keys, add_backup_keys_for_room, add_backup_keys_for_session, create_backup_version, - delete_backup_keys, delete_backup_keys_for_room, delete_backup_keys_for_session, delete_backup_version, - get_backup_info, get_backup_keys, get_backup_keys_for_room, get_backup_keys_for_session, - get_latest_backup_info, update_backup_version, - }, - error::ErrorKind, + api::client::backup::{ + add_backup_keys, add_backup_keys_for_room, add_backup_keys_for_session, create_backup_version, + delete_backup_keys, delete_backup_keys_for_room, delete_backup_keys_for_session, delete_backup_version, + get_backup_info, get_backup_keys, get_backup_keys_for_room, get_backup_keys_for_session, + get_latest_backup_info, update_backup_version, }, UInt, }; -use crate::{Error, Result, Ruma}; +use crate::{Result, Ruma}; /// # `POST /_matrix/client/r0/room_keys/version` /// @@ -40,7 +38,8 @@ pub(crate) async fn update_backup_version_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); services .key_backups - .update_backup(sender_user, &body.version, &body.algorithm)?; + .update_backup(sender_user, &body.version, &body.algorithm) + .await?; Ok(update_backup_version::v3::Response {}) } @@ -55,14 +54,15 @@ pub(crate) async fn get_latest_backup_info_route( let (version, algorithm) = services .key_backups - .get_latest_backup(sender_user)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Key backup does not exist."))?; + .get_latest_backup(sender_user) + .await + .map_err(|_| err!(Request(NotFound("Key backup does not exist."))))?; Ok(get_latest_backup_info::v3::Response { algorithm, - count: (UInt::try_from(services.key_backups.count_keys(sender_user, &version)?) + count: (UInt::try_from(services.key_backups.count_keys(sender_user, &version).await) .expect("user backup keys count should not be that high")), - etag: services.key_backups.get_etag(sender_user, &version)?, + etag: services.key_backups.get_etag(sender_user, &version).await, version, }) } @@ -76,18 +76,21 @@ pub(crate) async fn get_backup_info_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let algorithm = services .key_backups - .get_backup(sender_user, &body.version)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Key backup does not exist."))?; + .get_backup(sender_user, &body.version) + .await + .map_err(|_| err!(Request(NotFound("Key backup does not exist at version {:?}", body.version))))?; Ok(get_backup_info::v3::Response { algorithm, - count: (UInt::try_from( - services - .key_backups - .count_keys(sender_user, &body.version)?, - ) - .expect("user backup keys count should not be that high")), - etag: services.key_backups.get_etag(sender_user, &body.version)?, + count: services + .key_backups + .count_keys(sender_user, &body.version) + .await + .try_into()?, + etag: services + .key_backups + .get_etag(sender_user, &body.version) + .await, version: body.version.clone(), }) } @@ -105,7 +108,8 @@ pub(crate) async fn delete_backup_version_route( services .key_backups - .delete_backup(sender_user, &body.version)?; + .delete_backup(sender_user, &body.version) + .await; Ok(delete_backup_version::v3::Response {}) } @@ -123,34 +127,36 @@ pub(crate) async fn add_backup_keys_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if Some(&body.version) - != services - .key_backups - .get_latest_backup_version(sender_user)? - .as_ref() + if services + .key_backups + .get_latest_backup_version(sender_user) + .await + .is_ok_and(|version| version != body.version) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "You may only manipulate the most recently created version of the backup.", - )); + return Err!(Request(InvalidParam( + "You may only manipulate the most recently created version of the backup." + ))); } for (room_id, room) in &body.rooms { for (session_id, key_data) in &room.sessions { services .key_backups - .add_key(sender_user, &body.version, room_id, session_id, key_data)?; + .add_key(sender_user, &body.version, room_id, session_id, key_data) + .await?; } } Ok(add_backup_keys::v3::Response { - count: (UInt::try_from( - services - .key_backups - .count_keys(sender_user, &body.version)?, - ) - .expect("user backup keys count should not be that high")), - etag: services.key_backups.get_etag(sender_user, &body.version)?, + count: services + .key_backups + .count_keys(sender_user, &body.version) + .await + .try_into()?, + etag: services + .key_backups + .get_etag(sender_user, &body.version) + .await, }) } @@ -167,32 +173,34 @@ pub(crate) async fn add_backup_keys_for_room_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if Some(&body.version) - != services - .key_backups - .get_latest_backup_version(sender_user)? - .as_ref() + if services + .key_backups + .get_latest_backup_version(sender_user) + .await + .is_ok_and(|version| version != body.version) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "You may only manipulate the most recently created version of the backup.", - )); + return Err!(Request(InvalidParam( + "You may only manipulate the most recently created version of the backup." + ))); } for (session_id, key_data) in &body.sessions { services .key_backups - .add_key(sender_user, &body.version, &body.room_id, session_id, key_data)?; + .add_key(sender_user, &body.version, &body.room_id, session_id, key_data) + .await?; } Ok(add_backup_keys_for_room::v3::Response { - count: (UInt::try_from( - services - .key_backups - .count_keys(sender_user, &body.version)?, - ) - .expect("user backup keys count should not be that high")), - etag: services.key_backups.get_etag(sender_user, &body.version)?, + count: services + .key_backups + .count_keys(sender_user, &body.version) + .await + .try_into()?, + etag: services + .key_backups + .get_etag(sender_user, &body.version) + .await, }) } @@ -209,30 +217,32 @@ pub(crate) async fn add_backup_keys_for_session_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if Some(&body.version) - != services - .key_backups - .get_latest_backup_version(sender_user)? - .as_ref() + if services + .key_backups + .get_latest_backup_version(sender_user) + .await + .is_ok_and(|version| version != body.version) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "You may only manipulate the most recently created version of the backup.", - )); + return Err!(Request(InvalidParam( + "You may only manipulate the most recently created version of the backup." + ))); } services .key_backups - .add_key(sender_user, &body.version, &body.room_id, &body.session_id, &body.session_data)?; + .add_key(sender_user, &body.version, &body.room_id, &body.session_id, &body.session_data) + .await?; Ok(add_backup_keys_for_session::v3::Response { - count: (UInt::try_from( - services - .key_backups - .count_keys(sender_user, &body.version)?, - ) - .expect("user backup keys count should not be that high")), - etag: services.key_backups.get_etag(sender_user, &body.version)?, + count: services + .key_backups + .count_keys(sender_user, &body.version) + .await + .try_into()?, + etag: services + .key_backups + .get_etag(sender_user, &body.version) + .await, }) } @@ -244,7 +254,10 @@ pub(crate) async fn get_backup_keys_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let rooms = services.key_backups.get_all(sender_user, &body.version)?; + let rooms = services + .key_backups + .get_all(sender_user, &body.version) + .await; Ok(get_backup_keys::v3::Response { rooms, @@ -261,7 +274,8 @@ pub(crate) async fn get_backup_keys_for_room_route( let sessions = services .key_backups - .get_room(sender_user, &body.version, &body.room_id)?; + .get_room(sender_user, &body.version, &body.room_id) + .await; Ok(get_backup_keys_for_room::v3::Response { sessions, @@ -278,8 +292,9 @@ pub(crate) async fn get_backup_keys_for_session_route( let key_data = services .key_backups - .get_session(sender_user, &body.version, &body.room_id, &body.session_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Backup key not found for this user's session."))?; + .get_session(sender_user, &body.version, &body.room_id, &body.session_id) + .await + .map_err(|_| err!(Request(NotFound(debug_error!("Backup key not found for this user's session.")))))?; Ok(get_backup_keys_for_session::v3::Response { key_data, @@ -296,16 +311,19 @@ pub(crate) async fn delete_backup_keys_route( services .key_backups - .delete_all_keys(sender_user, &body.version)?; + .delete_all_keys(sender_user, &body.version) + .await; Ok(delete_backup_keys::v3::Response { - count: (UInt::try_from( - services - .key_backups - .count_keys(sender_user, &body.version)?, - ) - .expect("user backup keys count should not be that high")), - etag: services.key_backups.get_etag(sender_user, &body.version)?, + count: services + .key_backups + .count_keys(sender_user, &body.version) + .await + .try_into()?, + etag: services + .key_backups + .get_etag(sender_user, &body.version) + .await, }) } @@ -319,16 +337,19 @@ pub(crate) async fn delete_backup_keys_for_room_route( services .key_backups - .delete_room_keys(sender_user, &body.version, &body.room_id)?; + .delete_room_keys(sender_user, &body.version, &body.room_id) + .await; Ok(delete_backup_keys_for_room::v3::Response { - count: (UInt::try_from( - services - .key_backups - .count_keys(sender_user, &body.version)?, - ) - .expect("user backup keys count should not be that high")), - etag: services.key_backups.get_etag(sender_user, &body.version)?, + count: services + .key_backups + .count_keys(sender_user, &body.version) + .await + .try_into()?, + etag: services + .key_backups + .get_etag(sender_user, &body.version) + .await, }) } @@ -342,15 +363,18 @@ pub(crate) async fn delete_backup_keys_for_session_route( services .key_backups - .delete_room_key(sender_user, &body.version, &body.room_id, &body.session_id)?; + .delete_room_key(sender_user, &body.version, &body.room_id, &body.session_id) + .await; Ok(delete_backup_keys_for_session::v3::Response { - count: (UInt::try_from( - services - .key_backups - .count_keys(sender_user, &body.version)?, - ) - .expect("user backup keys count should not be that high")), - etag: services.key_backups.get_etag(sender_user, &body.version)?, + count: services + .key_backups + .count_keys(sender_user, &body.version) + .await + .try_into()?, + etag: services + .key_backups + .get_etag(sender_user, &body.version) + .await, }) } diff --git a/src/api/client/config.rs b/src/api/client/config.rs index 61cc97ff..33b85136 100644 --- a/src/api/client/config.rs +++ b/src/api/client/config.rs @@ -1,4 +1,5 @@ use axum::extract::State; +use conduit::err; use ruma::{ api::client::{ config::{get_global_account_data, get_room_account_data, set_global_account_data, set_room_account_data}, @@ -25,7 +26,8 @@ pub(crate) async fn set_global_account_data_route( &body.sender_user, &body.event_type.to_string(), body.data.json(), - )?; + ) + .await?; Ok(set_global_account_data::v3::Response {}) } @@ -42,7 +44,8 @@ pub(crate) async fn set_room_account_data_route( &body.sender_user, &body.event_type.to_string(), body.data.json(), - )?; + ) + .await?; Ok(set_room_account_data::v3::Response {}) } @@ -57,8 +60,9 @@ pub(crate) async fn get_global_account_data_route( let event: Box = services .account_data - .get(None, sender_user, body.event_type.to_string().into())? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Data not found."))?; + .get(None, sender_user, body.event_type.to_string().into()) + .await + .map_err(|_| err!(Request(NotFound("Data not found."))))?; let account_data = serde_json::from_str::(event.get()) .map_err(|_| Error::bad_database("Invalid account data event in db."))? @@ -79,8 +83,9 @@ pub(crate) async fn get_room_account_data_route( let event: Box = services .account_data - .get(Some(&body.room_id), sender_user, body.event_type.clone())? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Data not found."))?; + .get(Some(&body.room_id), sender_user, body.event_type.clone()) + .await + .map_err(|_| err!(Request(NotFound("Data not found."))))?; let account_data = serde_json::from_str::(event.get()) .map_err(|_| Error::bad_database("Invalid account data event in db."))? @@ -91,7 +96,7 @@ pub(crate) async fn get_room_account_data_route( }) } -fn set_account_data( +async fn set_account_data( services: &Services, room_id: Option<&RoomId>, sender_user: &Option, event_type: &str, data: &RawJsonValue, ) -> Result<()> { @@ -100,15 +105,18 @@ fn set_account_data( let data: serde_json::Value = serde_json::from_str(data.get()).map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?; - services.account_data.update( - room_id, - sender_user, - event_type.into(), - &json!({ - "type": event_type, - "content": data, - }), - )?; + services + .account_data + .update( + room_id, + sender_user, + event_type.into(), + &json!({ + "type": event_type, + "content": data, + }), + ) + .await?; Ok(()) } diff --git a/src/api/client/context.rs b/src/api/client/context.rs index f223d488..cc49b763 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,13 +1,14 @@ use std::collections::HashSet; use axum::extract::State; +use conduit::{err, error, Err}; +use futures::StreamExt; use ruma::{ - api::client::{context::get_context, error::ErrorKind, filter::LazyLoadOptions}, + api::client::{context::get_context, filter::LazyLoadOptions}, events::StateEventType, }; -use tracing::error; -use crate::{Error, Result, Ruma}; +use crate::{Result, Ruma}; /// # `GET /_matrix/client/r0/rooms/{roomId}/context` /// @@ -35,34 +36,33 @@ pub(crate) async fn get_context_route( let base_token = services .rooms .timeline - .get_pdu_count(&body.event_id)? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "Base event id not found."))?; + .get_pdu_count(&body.event_id) + .await + .map_err(|_| err!(Request(NotFound("Base event id not found."))))?; let base_event = services .rooms .timeline - .get_pdu(&body.event_id)? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "Base event not found."))?; + .get_pdu(&body.event_id) + .await + .map_err(|_| err!(Request(NotFound("Base event not found."))))?; - let room_id = base_event.room_id.clone(); + let room_id = &base_event.room_id; if !services .rooms .state_accessor - .user_can_see_event(sender_user, &room_id, &body.event_id)? + .user_can_see_event(sender_user, room_id, &body.event_id) + .await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this event.", - )); + return Err!(Request(Forbidden("You don't have permission to view this event."))); } - if !services.rooms.lazy_loading.lazy_load_was_sent_before( - sender_user, - sender_device, - &room_id, - &base_event.sender, - )? || lazy_load_send_redundant + if !services + .rooms + .lazy_loading + .lazy_load_was_sent_before(sender_user, sender_device, room_id, &base_event.sender) + .await || lazy_load_send_redundant { lazy_loaded.insert(base_event.sender.as_str().to_owned()); } @@ -75,25 +75,26 @@ pub(crate) async fn get_context_route( let events_before: Vec<_> = services .rooms .timeline - .pdus_until(sender_user, &room_id, base_token)? + .pdus_until(sender_user, room_id, base_token) + .await? .take(limit / 2) - .filter_map(Result::ok) // Remove buggy events - .filter(|(_, pdu)| { + .filter_map(|(count, pdu)| async move { services .rooms .state_accessor - .user_can_see_event(sender_user, &room_id, &pdu.event_id) - .unwrap_or(false) + .user_can_see_event(sender_user, room_id, &pdu.event_id) + .await + .then_some((count, pdu)) }) - .collect(); + .collect() + .await; for (_, event) in &events_before { - if !services.rooms.lazy_loading.lazy_load_was_sent_before( - sender_user, - sender_device, - &room_id, - &event.sender, - )? || lazy_load_send_redundant + if !services + .rooms + .lazy_loading + .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) + .await || lazy_load_send_redundant { lazy_loaded.insert(event.sender.as_str().to_owned()); } @@ -111,25 +112,26 @@ pub(crate) async fn get_context_route( let events_after: Vec<_> = services .rooms .timeline - .pdus_after(sender_user, &room_id, base_token)? + .pdus_after(sender_user, room_id, base_token) + .await? .take(limit / 2) - .filter_map(Result::ok) // Remove buggy events - .filter(|(_, pdu)| { + .filter_map(|(count, pdu)| async move { services .rooms .state_accessor - .user_can_see_event(sender_user, &room_id, &pdu.event_id) - .unwrap_or(false) + .user_can_see_event(sender_user, room_id, &pdu.event_id) + .await + .then_some((count, pdu)) }) - .collect(); + .collect() + .await; for (_, event) in &events_after { - if !services.rooms.lazy_loading.lazy_load_was_sent_before( - sender_user, - sender_device, - &room_id, - &event.sender, - )? || lazy_load_send_redundant + if !services + .rooms + .lazy_loading + .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) + .await || lazy_load_send_redundant { lazy_loaded.insert(event.sender.as_str().to_owned()); } @@ -142,12 +144,14 @@ pub(crate) async fn get_context_route( events_after .last() .map_or(&*body.event_id, |(_, e)| &*e.event_id), - )? + ) + .await .map_or( services .rooms .state - .get_room_shortstatehash(&room_id)? + .get_room_shortstatehash(room_id) + .await .expect("All rooms have state"), |hash| hash, ); @@ -156,7 +160,8 @@ pub(crate) async fn get_context_route( .rooms .state_accessor .state_full_ids(shortstatehash) - .await?; + .await + .map_err(|e| err!(Database("State not found: {e}")))?; let end_token = events_after .last() @@ -173,18 +178,19 @@ pub(crate) async fn get_context_route( let (event_type, state_key) = services .rooms .short - .get_statekey_from_short(shortstatekey)?; + .get_statekey_from_short(shortstatekey) + .await?; if event_type != StateEventType::RoomMember { - let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { - error!("Pdu in state not found: {}", id); + let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { + error!("Pdu in state not found: {id}"); continue; }; state.push(pdu.to_state_event()); } else if !lazy_load_enabled || lazy_loaded.contains(&state_key) { - let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { - error!("Pdu in state not found: {}", id); + let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { + error!("Pdu in state not found: {id}"); continue; }; diff --git a/src/api/client/device.rs b/src/api/client/device.rs index bad7f284..93eaa393 100644 --- a/src/api/client/device.rs +++ b/src/api/client/device.rs @@ -1,4 +1,6 @@ use axum::extract::State; +use conduit::{err, Err}; +use futures::StreamExt; use ruma::api::client::{ device::{self, delete_device, delete_devices, get_device, get_devices, update_device}, error::ErrorKind, @@ -19,8 +21,8 @@ pub(crate) async fn get_devices_route( let devices: Vec = services .users .all_devices_metadata(sender_user) - .filter_map(Result::ok) // Filter out buggy devices - .collect(); + .collect() + .await; Ok(get_devices::v3::Response { devices, @@ -37,8 +39,9 @@ pub(crate) async fn get_device_route( let device = services .users - .get_device_metadata(sender_user, &body.body.device_id)? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?; + .get_device_metadata(sender_user, &body.body.device_id) + .await + .map_err(|_| err!(Request(NotFound("Device not found."))))?; Ok(get_device::v3::Response { device, @@ -55,14 +58,16 @@ pub(crate) async fn update_device_route( let mut device = services .users - .get_device_metadata(sender_user, &body.device_id)? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?; + .get_device_metadata(sender_user, &body.device_id) + .await + .map_err(|_| err!(Request(NotFound("Device not found."))))?; device.display_name.clone_from(&body.display_name); services .users - .update_device_metadata(sender_user, &body.device_id, &device)?; + .update_device_metadata(sender_user, &body.device_id, &device) + .await?; Ok(update_device::v3::Response {}) } @@ -97,22 +102,28 @@ pub(crate) async fn delete_device_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = services .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo)?; + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; + if !worked { - return Err(Error::Uiaa(uiaainfo)); + return Err!(Uiaa(uiaainfo)); } // Success! } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json)?; - return Err(Error::Uiaa(uiaainfo)); + .create(sender_user, sender_device, &uiaainfo, &json); + + return Err!(Uiaa(uiaainfo)); } else { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + return Err!(Request(NotJson("Not json."))); } - services.users.remove_device(sender_user, &body.device_id)?; + services + .users + .remove_device(sender_user, &body.device_id) + .await; Ok(delete_device::v3::Response {}) } @@ -149,7 +160,9 @@ pub(crate) async fn delete_devices_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = services .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo)?; + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; + if !worked { return Err(Error::Uiaa(uiaainfo)); } @@ -158,14 +171,15 @@ pub(crate) async fn delete_devices_route( uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json)?; + .create(sender_user, sender_device, &uiaainfo, &json); + return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } for device_id in &body.devices { - services.users.remove_device(sender_user, device_id)?; + services.users.remove_device(sender_user, device_id).await; } Ok(delete_devices::v3::Response {}) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 602f876a..ea499545 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,6 +1,7 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{err, info, warn, Err, Error, Result}; +use conduit::{info, warn, Err, Error, Result}; +use futures::{StreamExt, TryFutureExt}; use ruma::{ api::{ client::{ @@ -18,7 +19,7 @@ use ruma::{ }, StateEventType, }, - uint, RoomId, ServerName, UInt, UserId, + uint, OwnedRoomId, RoomId, ServerName, UInt, UserId, }; use service::Services; @@ -119,16 +120,22 @@ pub(crate) async fn set_room_visibility_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services.rooms.metadata.exists(&body.room_id)? { + if !services.rooms.metadata.exists(&body.room_id).await { // Return 404 if the room doesn't exist return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } - if services.users.is_deactivated(sender_user).unwrap_or(false) && body.appservice_info.is_none() { + if services + .users + .is_deactivated(sender_user) + .await + .unwrap_or(false) + && body.appservice_info.is_none() + { return Err!(Request(Forbidden("Guests cannot publish to room directories"))); } - if !user_can_publish_room(&services, sender_user, &body.room_id)? { + if !user_can_publish_room(&services, sender_user, &body.room_id).await? { return Err(Error::BadRequest( ErrorKind::forbidden(), "User is not allowed to publish this room", @@ -138,7 +145,7 @@ pub(crate) async fn set_room_visibility_route( match &body.visibility { room::Visibility::Public => { if services.globals.config.lockdown_public_room_directory - && !services.users.is_admin(sender_user)? + && !services.users.is_admin(sender_user).await && body.appservice_info.is_none() { info!( @@ -164,7 +171,7 @@ pub(crate) async fn set_room_visibility_route( )); } - services.rooms.directory.set_public(&body.room_id)?; + services.rooms.directory.set_public(&body.room_id); if services.globals.config.admin_room_notices { services @@ -174,7 +181,7 @@ pub(crate) async fn set_room_visibility_route( } info!("{sender_user} made {0} public to the room directory", body.room_id); }, - room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id)?, + room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id), _ => { return Err(Error::BadRequest( ErrorKind::InvalidParam, @@ -192,13 +199,13 @@ pub(crate) async fn set_room_visibility_route( pub(crate) async fn get_room_visibility_route( State(services): State, body: Ruma, ) -> Result { - if !services.rooms.metadata.exists(&body.room_id)? { + if !services.rooms.metadata.exists(&body.room_id).await { // Return 404 if the room doesn't exist return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); } Ok(get_room_visibility::v3::Response { - visibility: if services.rooms.directory.is_public_room(&body.room_id)? { + visibility: if services.rooms.directory.is_public_room(&body.room_id).await { room::Visibility::Public } else { room::Visibility::Private @@ -257,101 +264,41 @@ pub(crate) async fn get_public_rooms_filtered_helper( } } - let mut all_rooms: Vec<_> = services + let mut all_rooms: Vec = services .rooms .directory .public_rooms() - .map(|room_id| { - let room_id = room_id?; - - let chunk = PublicRoomsChunk { - canonical_alias: services - .rooms - .state_accessor - .get_canonical_alias(&room_id)?, - name: services.rooms.state_accessor.get_name(&room_id)?, - num_joined_members: services - .rooms - .state_cache - .room_joined_count(&room_id)? - .unwrap_or_else(|| { - warn!("Room {} has no member count", room_id); - 0 - }) - .try_into() - .expect("user count should not be that big"), - topic: services - .rooms - .state_accessor - .get_room_topic(&room_id) - .unwrap_or(None), - world_readable: services.rooms.state_accessor.is_world_readable(&room_id)?, - guest_can_join: services - .rooms - .state_accessor - .guest_can_join(&room_id)?, - avatar_url: services - .rooms - .state_accessor - .get_avatar(&room_id)? - .into_option() - .unwrap_or_default() - .url, - join_rule: services - .rooms - .state_accessor - .room_state_get(&room_id, &StateEventType::RoomJoinRules, "")? - .map(|s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomJoinRulesEventContent| match c.join_rule { - JoinRule::Public => Some(PublicRoomJoinRule::Public), - JoinRule::Knock => Some(PublicRoomJoinRule::Knock), - _ => None, - }) - .map_err(|e| { - err!(Database(error!("Invalid room join rule event in database: {e}"))) - }) - }) - .transpose()? - .flatten() - .ok_or_else(|| Error::bad_database("Missing room join rule event for room."))?, - room_type: services - .rooms - .state_accessor - .get_room_type(&room_id)?, - room_id, - }; - Ok(chunk) - }) - .filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms - .filter(|chunk| { + .map(ToOwned::to_owned) + .then(|room_id| public_rooms_chunk(services, room_id)) + .filter_map(|chunk| async move { if let Some(query) = filter.generic_search_term.as_ref().map(|q| q.to_lowercase()) { if let Some(name) = &chunk.name { if name.as_str().to_lowercase().contains(&query) { - return true; + return Some(chunk); } } if let Some(topic) = &chunk.topic { if topic.to_lowercase().contains(&query) { - return true; + return Some(chunk); } } if let Some(canonical_alias) = &chunk.canonical_alias { if canonical_alias.as_str().to_lowercase().contains(&query) { - return true; + return Some(chunk); } } - false - } else { - // No search term - true + return None; } + + // No search term + Some(chunk) }) // We need to collect all, so we can sort by member count - .collect(); + .collect() + .await; all_rooms.sort_by(|l, r| r.num_joined_members.cmp(&l.num_joined_members)); @@ -394,22 +341,23 @@ pub(crate) async fn get_public_rooms_filtered_helper( /// Check whether the user can publish to the room directory via power levels of /// room history visibility event or room creator -fn user_can_publish_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result { - if let Some(event) = services +async fn user_can_publish_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result { + if let Ok(event) = services .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? + .room_state_get(room_id, &StateEventType::RoomPowerLevels, "") + .await { serde_json::from_str(event.content.get()) .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) .map(|content: RoomPowerLevelsEventContent| { RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomHistoryVisibility) }) - } else if let Some(event) = - services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "")? + } else if let Ok(event) = services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomCreate, "") + .await { Ok(event.sender == user_id) } else { @@ -419,3 +367,61 @@ fn user_can_publish_room(services: &Services, user_id: &UserId, room_id: &RoomId )); } } + +async fn public_rooms_chunk(services: &Services, room_id: OwnedRoomId) -> PublicRoomsChunk { + PublicRoomsChunk { + canonical_alias: services + .rooms + .state_accessor + .get_canonical_alias(&room_id) + .await + .ok(), + name: services.rooms.state_accessor.get_name(&room_id).await.ok(), + num_joined_members: services + .rooms + .state_cache + .room_joined_count(&room_id) + .await + .unwrap_or(0) + .try_into() + .expect("joined count overflows ruma UInt"), + topic: services + .rooms + .state_accessor + .get_room_topic(&room_id) + .await + .ok(), + world_readable: services + .rooms + .state_accessor + .is_world_readable(&room_id) + .await, + guest_can_join: services.rooms.state_accessor.guest_can_join(&room_id).await, + avatar_url: services + .rooms + .state_accessor + .get_avatar(&room_id) + .await + .into_option() + .unwrap_or_default() + .url, + join_rule: services + .rooms + .state_accessor + .room_state_get_content(&room_id, &StateEventType::RoomJoinRules, "") + .map_ok(|c: RoomJoinRulesEventContent| match c.join_rule { + JoinRule::Public => PublicRoomJoinRule::Public, + JoinRule::Knock => PublicRoomJoinRule::Knock, + _ => "invite".into(), + }) + .await + .unwrap_or_default(), + room_type: services + .rooms + .state_accessor + .get_room_type(&room_id) + .await + .ok(), + room_id, + } +} diff --git a/src/api/client/filter.rs b/src/api/client/filter.rs index 8b2690c6..2a8ebb9c 100644 --- a/src/api/client/filter.rs +++ b/src/api/client/filter.rs @@ -1,10 +1,8 @@ use axum::extract::State; -use ruma::api::client::{ - error::ErrorKind, - filter::{create_filter, get_filter}, -}; +use conduit::err; +use ruma::api::client::filter::{create_filter, get_filter}; -use crate::{Error, Result, Ruma}; +use crate::{Result, Ruma}; /// # `GET /_matrix/client/r0/user/{userId}/filter/{filterId}` /// @@ -15,11 +13,13 @@ pub(crate) async fn get_filter_route( State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let Some(filter) = services.users.get_filter(sender_user, &body.filter_id)? else { - return Err(Error::BadRequest(ErrorKind::NotFound, "Filter not found.")); - }; - Ok(get_filter::v3::Response::new(filter)) + services + .users + .get_filter(sender_user, &body.filter_id) + .await + .map(get_filter::v3::Response::new) + .map_err(|_| err!(Request(NotFound("Filter not found.")))) } /// # `PUT /_matrix/client/r0/user/{userId}/filter` @@ -29,7 +29,8 @@ pub(crate) async fn create_filter_route( State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - Ok(create_filter::v3::Response::new( - services.users.create_filter(sender_user, &body.filter)?, - )) + + let filter_id = services.users.create_filter(sender_user, &body.filter); + + Ok(create_filter::v3::Response::new(filter_id)) } diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index a426364a..abf2a22f 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -4,8 +4,8 @@ use std::{ }; use axum::extract::State; -use conduit::{utils, utils::math::continue_exponential_backoff_secs, Err, Error, Result}; -use futures_util::{stream::FuturesUnordered, StreamExt}; +use conduit::{err, utils, utils::math::continue_exponential_backoff_secs, Err, Error, Result}; +use futures::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::{ client::{ @@ -21,7 +21,10 @@ use ruma::{ use serde_json::json; use super::SESSION_ID_LENGTH; -use crate::{service::Services, Ruma}; +use crate::{ + service::{users::parse_master_key, Services}, + Ruma, +}; /// # `POST /_matrix/client/r0/keys/upload` /// @@ -39,7 +42,8 @@ pub(crate) async fn upload_keys_route( for (key_key, key_value) in &body.one_time_keys { services .users - .add_one_time_key(sender_user, sender_device, key_key, key_value)?; + .add_one_time_key(sender_user, sender_device, key_key, key_value) + .await?; } if let Some(device_keys) = &body.device_keys { @@ -47,19 +51,22 @@ pub(crate) async fn upload_keys_route( // This check is needed to assure that signatures are kept if services .users - .get_device_keys(sender_user, sender_device)? - .is_none() + .get_device_keys(sender_user, sender_device) + .await + .is_err() { services .users - .add_device_keys(sender_user, sender_device, device_keys)?; + .add_device_keys(sender_user, sender_device, device_keys) + .await; } } Ok(upload_keys::v3::Response { one_time_key_counts: services .users - .count_one_time_keys(sender_user, sender_device)?, + .count_one_time_keys(sender_user, sender_device) + .await, }) } @@ -120,7 +127,9 @@ pub(crate) async fn upload_signing_keys_route( if let Some(auth) = &body.auth { let (worked, uiaainfo) = services .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo)?; + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; + if !worked { return Err(Error::Uiaa(uiaainfo)); } @@ -129,20 +138,24 @@ pub(crate) async fn upload_signing_keys_route( uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json)?; + .create(sender_user, sender_device, &uiaainfo, &json); + return Err(Error::Uiaa(uiaainfo)); } else { return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } if let Some(master_key) = &body.master_key { - services.users.add_cross_signing_keys( - sender_user, - master_key, - &body.self_signing_key, - &body.user_signing_key, - true, // notify so that other users see the new keys - )?; + services + .users + .add_cross_signing_keys( + sender_user, + master_key, + &body.self_signing_key, + &body.user_signing_key, + true, // notify so that other users see the new keys + ) + .await?; } Ok(upload_signing_keys::v3::Response {}) @@ -179,9 +192,11 @@ pub(crate) async fn upload_signatures_route( .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid signature value."))? .to_owned(), ); + services .users - .sign_key(user_id, key_id, signature, sender_user)?; + .sign_key(user_id, key_id, signature, sender_user) + .await?; } } } @@ -204,56 +219,51 @@ pub(crate) async fn get_key_changes_route( let mut device_list_updates = HashSet::new(); + let from = body + .from + .parse() + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from`."))?; + + let to = body + .to + .parse() + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `to`."))?; + device_list_updates.extend( services .users - .keys_changed( - sender_user.as_str(), - body.from - .parse() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from`."))?, - Some( - body.to - .parse() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `to`."))?, - ), - ) - .filter_map(Result::ok), + .keys_changed(sender_user.as_str(), from, Some(to)) + .map(ToOwned::to_owned) + .collect::>() + .await, ); - for room_id in services - .rooms - .state_cache - .rooms_joined(sender_user) - .filter_map(Result::ok) - { + let mut rooms_joined = services.rooms.state_cache.rooms_joined(sender_user).boxed(); + + while let Some(room_id) = rooms_joined.next().await { device_list_updates.extend( services .users - .keys_changed( - room_id.as_ref(), - body.from - .parse() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `from`."))?, - Some( - body.to - .parse() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `to`."))?, - ), - ) - .filter_map(Result::ok), + .keys_changed(room_id.as_ref(), from, Some(to)) + .map(ToOwned::to_owned) + .collect::>() + .await, ); } + Ok(get_key_changes::v3::Response { changed: device_list_updates.into_iter().collect(), left: Vec::new(), // TODO }) } -pub(crate) async fn get_keys_helper bool + Send>( +pub(crate) async fn get_keys_helper( services: &Services, sender_user: Option<&UserId>, device_keys_input: &BTreeMap>, allowed_signatures: F, include_display_names: bool, -) -> Result { +) -> Result +where + F: Fn(&UserId) -> bool + Send + Sync, +{ let mut master_keys = BTreeMap::new(); let mut self_signing_keys = BTreeMap::new(); let mut user_signing_keys = BTreeMap::new(); @@ -274,56 +284,60 @@ pub(crate) async fn get_keys_helper bool + Send>( if device_ids.is_empty() { let mut container = BTreeMap::new(); - for device_id in services.users.all_device_ids(user_id) { - let device_id = device_id?; - if let Some(mut keys) = services.users.get_device_keys(user_id, &device_id)? { + let mut devices = services.users.all_device_ids(user_id).boxed(); + + while let Some(device_id) = devices.next().await { + if let Ok(mut keys) = services.users.get_device_keys(user_id, device_id).await { let metadata = services .users - .get_device_metadata(user_id, &device_id)? - .ok_or_else(|| Error::bad_database("all_device_keys contained nonexistent device."))?; + .get_device_metadata(user_id, device_id) + .await + .map_err(|_| err!(Database("all_device_keys contained nonexistent device.")))?; add_unsigned_device_display_name(&mut keys, metadata, include_display_names) - .map_err(|_| Error::bad_database("invalid device keys in database"))?; + .map_err(|_| err!(Database("invalid device keys in database")))?; - container.insert(device_id, keys); + container.insert(device_id.to_owned(), keys); } } + device_keys.insert(user_id.to_owned(), container); } else { for device_id in device_ids { let mut container = BTreeMap::new(); - if let Some(mut keys) = services.users.get_device_keys(user_id, device_id)? { + if let Ok(mut keys) = services.users.get_device_keys(user_id, device_id).await { let metadata = services .users - .get_device_metadata(user_id, device_id)? - .ok_or(Error::BadRequest( - ErrorKind::InvalidParam, - "Tried to get keys for nonexistent device.", - ))?; + .get_device_metadata(user_id, device_id) + .await + .map_err(|_| err!(Request(InvalidParam("Tried to get keys for nonexistent device."))))?; add_unsigned_device_display_name(&mut keys, metadata, include_display_names) - .map_err(|_| Error::bad_database("invalid device keys in database"))?; + .map_err(|_| err!(Database("invalid device keys in database")))?; + container.insert(device_id.to_owned(), keys); } + device_keys.insert(user_id.to_owned(), container); } } - if let Some(master_key) = services + if let Ok(master_key) = services .users - .get_master_key(sender_user, user_id, &allowed_signatures)? + .get_master_key(sender_user, user_id, &allowed_signatures) + .await { master_keys.insert(user_id.to_owned(), master_key); } - if let Some(self_signing_key) = - services - .users - .get_self_signing_key(sender_user, user_id, &allowed_signatures)? + if let Ok(self_signing_key) = services + .users + .get_self_signing_key(sender_user, user_id, &allowed_signatures) + .await { self_signing_keys.insert(user_id.to_owned(), self_signing_key); } if Some(user_id) == sender_user { - if let Some(user_signing_key) = services.users.get_user_signing_key(user_id)? { + if let Ok(user_signing_key) = services.users.get_user_signing_key(user_id).await { user_signing_keys.insert(user_id.to_owned(), user_signing_key); } } @@ -386,23 +400,26 @@ pub(crate) async fn get_keys_helper bool + Send>( while let Some((server, response)) = futures.next().await { if let Ok(Ok(response)) = response { for (user, masterkey) in response.master_keys { - let (master_key_id, mut master_key) = services.users.parse_master_key(&user, &masterkey)?; + let (master_key_id, mut master_key) = parse_master_key(&user, &masterkey)?; - if let Some(our_master_key) = - services - .users - .get_key(&master_key_id, sender_user, &user, &allowed_signatures)? + if let Ok(our_master_key) = services + .users + .get_key(&master_key_id, sender_user, &user, &allowed_signatures) + .await { - let (_, our_master_key) = services.users.parse_master_key(&user, &our_master_key)?; + let (_, our_master_key) = parse_master_key(&user, &our_master_key)?; master_key.signatures.extend(our_master_key.signatures); } let json = serde_json::to_value(master_key).expect("to_value always works"); let raw = serde_json::from_value(json).expect("Raw::from_value always works"); - services.users.add_cross_signing_keys( - &user, &raw, &None, &None, - false, /* Dont notify. A notification would trigger another key request resulting in an - * endless loop */ - )?; + services + .users + .add_cross_signing_keys( + &user, &raw, &None, &None, + false, /* Dont notify. A notification would trigger another key request resulting in an + * endless loop */ + ) + .await?; master_keys.insert(user.clone(), raw); } @@ -465,9 +482,10 @@ pub(crate) async fn claim_keys_helper( let mut container = BTreeMap::new(); for (device_id, key_algorithm) in map { - if let Some(one_time_keys) = services + if let Ok(one_time_keys) = services .users - .take_one_time_key(user_id, device_id, key_algorithm)? + .take_one_time_key(user_id, device_id, key_algorithm) + .await { let mut c = BTreeMap::new(); c.insert(one_time_keys.0, one_time_keys.1); diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 470db669..5a5d436f 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -11,9 +11,10 @@ use conduit::{ debug, debug_error, debug_warn, err, error, info, pdu::{gen_event_id_canonical_json, PduBuilder}, trace, utils, - utils::math::continue_exponential_backoff_secs, + utils::{math::continue_exponential_backoff_secs, IterStream, ReadyExt}, warn, Err, Error, PduEvent, Result, }; +use futures::{FutureExt, StreamExt}; use ruma::{ api::{ client::{ @@ -55,9 +56,9 @@ async fn banned_room_check( services: &Services, user_id: &UserId, room_id: Option<&RoomId>, server_name: Option<&ServerName>, client_ip: IpAddr, ) -> Result<()> { - if !services.users.is_admin(user_id)? { + if !services.users.is_admin(user_id).await { if let Some(room_id) = room_id { - if services.rooms.metadata.is_banned(room_id)? + if services.rooms.metadata.is_banned(room_id).await || services .globals .config @@ -79,23 +80,22 @@ async fn banned_room_check( "Automatically deactivating user {user_id} due to attempted banned room join from IP \ {client_ip}" ))) - .await; + .await + .ok(); } let all_joined_rooms: Vec = services .rooms .state_cache .rooms_joined(user_id) - .filter_map(Result::ok) - .collect(); + .map(Into::into) + .collect() + .await; - full_user_deactivate(services, user_id, all_joined_rooms).await?; + full_user_deactivate(services, user_id, &all_joined_rooms).await?; } - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "This room is banned on this homeserver.", - )); + return Err!(Request(Forbidden("This room is banned on this homeserver."))); } } else if let Some(server_name) = server_name { if services @@ -119,23 +119,22 @@ async fn banned_room_check( "Automatically deactivating user {user_id} due to attempted banned room join from IP \ {client_ip}" ))) - .await; + .await + .ok(); } let all_joined_rooms: Vec = services .rooms .state_cache .rooms_joined(user_id) - .filter_map(Result::ok) - .collect(); + .map(Into::into) + .collect() + .await; - full_user_deactivate(services, user_id, all_joined_rooms).await?; + full_user_deactivate(services, user_id, &all_joined_rooms).await?; } - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "This remote server is banned on this homeserver.", - )); + return Err!(Request(Forbidden("This remote server is banned on this homeserver."))); } } } @@ -172,14 +171,16 @@ pub(crate) async fn join_room_by_id_route( .rooms .state_cache .servers_invite_via(&body.room_id) - .filter_map(Result::ok) - .collect::>(); + .map(ToOwned::to_owned) + .collect::>() + .await; servers.extend( services .rooms .state_cache - .invite_state(sender_user, &body.room_id)? + .invite_state(sender_user, &body.room_id) + .await .unwrap_or_default() .iter() .filter_map(|event| serde_json::from_str(event.json().get()).ok()) @@ -202,6 +203,7 @@ pub(crate) async fn join_room_by_id_route( body.third_party_signed.as_ref(), &body.appservice_info, ) + .boxed() .await } @@ -233,14 +235,17 @@ pub(crate) async fn join_room_by_id_or_alias_route( .rooms .state_cache .servers_invite_via(&room_id) - .filter_map(Result::ok), + .map(ToOwned::to_owned) + .collect::>() + .await, ); servers.extend( services .rooms .state_cache - .invite_state(sender_user, &room_id)? + .invite_state(sender_user, &room_id) + .await .unwrap_or_default() .iter() .filter_map(|event| serde_json::from_str(event.json().get()).ok()) @@ -270,19 +275,23 @@ pub(crate) async fn join_room_by_id_or_alias_route( if let Some(pre_servers) = &mut pre_servers { servers.append(pre_servers); } + servers.extend( services .rooms .state_cache .servers_invite_via(&room_id) - .filter_map(Result::ok), + .map(ToOwned::to_owned) + .collect::>() + .await, ); servers.extend( services .rooms .state_cache - .invite_state(sender_user, &room_id)? + .invite_state(sender_user, &room_id) + .await .unwrap_or_default() .iter() .filter_map(|event| serde_json::from_str(event.json().get()).ok()) @@ -305,6 +314,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( body.third_party_signed.as_ref(), appservice_info, ) + .boxed() .await?; Ok(join_room_by_id_or_alias::v3::Response { @@ -337,7 +347,7 @@ pub(crate) async fn invite_user_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services.users.is_admin(sender_user)? && services.globals.block_non_admin_invites() { + if !services.users.is_admin(sender_user).await && services.globals.block_non_admin_invites() { info!( "User {sender_user} is not an admin and attempted to send an invite to room {}", &body.room_id @@ -375,15 +385,13 @@ pub(crate) async fn kick_user_route( services .rooms .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref())? - .ok_or(Error::BadRequest( - ErrorKind::BadState, - "Cannot kick member that's not in the room.", - ))? + .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) + .await + .map_err(|_| err!(Request(BadState("Cannot kick member that's not in the room."))))? .content .get(), ) - .map_err(|_| Error::bad_database("Invalid member event in database."))?; + .map_err(|_| err!(Database("Invalid member event in database.")))?; event.membership = MembershipState::Leave; event.reason.clone_from(&body.reason); @@ -421,10 +429,13 @@ pub(crate) async fn ban_user_route( let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + let blurhash = services.users.blurhash(&body.user_id).await.ok(); + let event = services .rooms .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref())? + .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) + .await .map_or( Ok(RoomMemberEventContent { membership: MembershipState::Ban, @@ -432,7 +443,7 @@ pub(crate) async fn ban_user_route( avatar_url: None, is_direct: None, third_party_invite: None, - blurhash: services.users.blurhash(&body.user_id).unwrap_or_default(), + blurhash: blurhash.clone(), reason: body.reason.clone(), join_authorized_via_users_server: None, }), @@ -442,12 +453,12 @@ pub(crate) async fn ban_user_route( membership: MembershipState::Ban, displayname: None, avatar_url: None, - blurhash: services.users.blurhash(&body.user_id).unwrap_or_default(), + blurhash: blurhash.clone(), reason: body.reason.clone(), join_authorized_via_users_server: None, ..event }) - .map_err(|_| Error::bad_database("Invalid member event in database.")) + .map_err(|e| err!(Database("Invalid member event in database: {e:?}"))) }, )?; @@ -488,12 +499,13 @@ pub(crate) async fn unban_user_route( services .rooms .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref())? - .ok_or(Error::BadRequest(ErrorKind::BadState, "Cannot unban a user who is not banned."))? + .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) + .await + .map_err(|_| err!(Request(BadState("Cannot unban a user who is not banned."))))? .content .get(), ) - .map_err(|_| Error::bad_database("Invalid member event in database."))?; + .map_err(|e| err!(Database("Invalid member event in database: {e:?}")))?; event.membership = MembershipState::Leave; event.reason.clone_from(&body.reason); @@ -539,18 +551,16 @@ pub(crate) async fn forget_room_route( if services .rooms .state_cache - .is_joined(sender_user, &body.room_id)? + .is_joined(sender_user, &body.room_id) + .await { - return Err(Error::BadRequest( - ErrorKind::Unknown, - "You must leave the room before forgetting it", - )); + return Err!(Request(Unknown("You must leave the room before forgetting it"))); } services .rooms .state_cache - .forget(&body.room_id, sender_user)?; + .forget(&body.room_id, sender_user); Ok(forget_room::v3::Response::new()) } @@ -568,8 +578,9 @@ pub(crate) async fn joined_rooms_route( .rooms .state_cache .rooms_joined(sender_user) - .filter_map(Result::ok) - .collect(), + .map(ToOwned::to_owned) + .collect() + .await, }) } @@ -587,12 +598,10 @@ pub(crate) async fn get_member_events_route( if !services .rooms .state_accessor - .user_can_see_state_events(sender_user, &body.room_id)? + .user_can_see_state_events(sender_user, &body.room_id) + .await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this room.", - )); + return Err!(Request(Forbidden("You don't have permission to view this room."))); } Ok(get_member_events::v3::Response { @@ -622,30 +631,27 @@ pub(crate) async fn joined_members_route( if !services .rooms .state_accessor - .user_can_see_state_events(sender_user, &body.room_id)? + .user_can_see_state_events(sender_user, &body.room_id) + .await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this room.", - )); + return Err!(Request(Forbidden("You don't have permission to view this room."))); } let joined: BTreeMap = services .rooms .state_cache .room_members(&body.room_id) - .filter_map(|user| { - let user = user.ok()?; - - Some(( - user.clone(), + .then(|user| async move { + ( + user.to_owned(), RoomMember { - display_name: services.users.displayname(&user).unwrap_or_default(), - avatar_url: services.users.avatar_url(&user).unwrap_or_default(), + display_name: services.users.displayname(user).await.ok(), + avatar_url: services.users.avatar_url(user).await.ok(), }, - )) + ) }) - .collect(); + .collect() + .await; Ok(joined_members::v3::Response { joined, @@ -658,13 +664,23 @@ pub async fn join_room_by_id_helper( ) -> Result { let state_lock = services.rooms.state.mutex.lock(room_id).await; - let user_is_guest = services.users.is_deactivated(sender_user).unwrap_or(false) && appservice_info.is_none(); + let user_is_guest = services + .users + .is_deactivated(sender_user) + .await + .unwrap_or(false) + && appservice_info.is_none(); - if matches!(services.rooms.state_accessor.guest_can_join(room_id), Ok(false)) && user_is_guest { + if user_is_guest && !services.rooms.state_accessor.guest_can_join(room_id).await { return Err!(Request(Forbidden("Guests are not allowed to join this room"))); } - if matches!(services.rooms.state_cache.is_joined(sender_user, room_id), Ok(true)) { + if services + .rooms + .state_cache + .is_joined(sender_user, room_id) + .await + { debug_warn!("{sender_user} is already joined in {room_id}"); return Ok(join_room_by_id::v3::Response { room_id: room_id.into(), @@ -674,15 +690,17 @@ pub async fn join_room_by_id_helper( if services .rooms .state_cache - .server_in_room(services.globals.server_name(), room_id)? - || servers.is_empty() + .server_in_room(services.globals.server_name(), room_id) + .await || servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])) { join_room_by_id_helper_local(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) + .boxed() .await } else { // Ask a remote server if we are not participating in this room join_room_by_id_helper_remote(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) + .boxed() .await } } @@ -739,11 +757,11 @@ async fn join_room_by_id_helper_remote( "content".to_owned(), to_canonical_value(RoomMemberEventContent { membership: MembershipState::Join, - displayname: services.users.displayname(sender_user)?, - avatar_url: services.users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), is_direct: None, third_party_invite: None, - blurhash: services.users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user).await.ok(), reason, join_authorized_via_users_server: join_authorized_via_users_server.clone(), }) @@ -791,10 +809,11 @@ async fn join_room_by_id_helper_remote( federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), event_id: event_id.to_owned(), + omit_members: false, pdu: services .sending - .convert_to_outgoing_federation_event(join_event.clone()), - omit_members: false, + .convert_to_outgoing_federation_event(join_event.clone()) + .await, }, ) .await?; @@ -864,7 +883,11 @@ async fn join_room_by_id_helper_remote( } } - services.rooms.short.get_or_create_shortroomid(room_id)?; + services + .rooms + .short + .get_or_create_shortroomid(room_id) + .await; info!("Parsing join event"); let parsed_join_pdu = PduEvent::from_id_val(event_id, join_event.clone()) @@ -895,12 +918,13 @@ async fn join_room_by_id_helper_remote( err!(BadServerResponse("Invalid PDU in send_join response: {e:?}")) })?; - services.rooms.outlier.add_pdu_outlier(&event_id, &value)?; + services.rooms.outlier.add_pdu_outlier(&event_id, &value); if let Some(state_key) = &pdu.state_key { let shortstatekey = services .rooms .short - .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key)?; + .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key) + .await; state.insert(shortstatekey, pdu.event_id.clone()); } } @@ -916,50 +940,53 @@ async fn join_room_by_id_helper_remote( continue; }; - services.rooms.outlier.add_pdu_outlier(&event_id, &value)?; + services.rooms.outlier.add_pdu_outlier(&event_id, &value); } debug!("Running send_join auth check"); + let fetch_state = &state; + let state_fetch = |k: &'static StateEventType, s: String| async move { + let shortstatekey = services.rooms.short.get_shortstatekey(k, &s).await.ok()?; + + let event_id = fetch_state.get(&shortstatekey)?; + services.rooms.timeline.get_pdu(event_id).await.ok() + }; let auth_check = state_res::event_auth::auth_check( &state_res::RoomVersion::new(&room_version_id).expect("room version is supported"), &parsed_join_pdu, - None::, // TODO: third party invite - |k, s| { - services - .rooms - .timeline - .get_pdu( - state.get( - &services - .rooms - .short - .get_or_create_shortstatekey(&k.to_string().into(), s) - .ok()?, - )?, - ) - .ok()? - }, + None, // TODO: third party invite + |k, s| state_fetch(k, s.to_owned()), ) - .map_err(|e| { - warn!("Auth check failed: {e}"); - Error::BadRequest(ErrorKind::forbidden(), "Auth check failed") - })?; + .await + .map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?; if !auth_check { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Auth check failed")); + return Err!(Request(Forbidden("Auth check failed"))); } info!("Saving state from send_join"); - let (statehash_before_join, new, removed) = services.rooms.state_compressor.save_state( - room_id, - Arc::new( - state - .into_iter() - .map(|(k, id)| services.rooms.state_compressor.compress_state_event(k, &id)) - .collect::>()?, - ), - )?; + let (statehash_before_join, new, removed) = services + .rooms + .state_compressor + .save_state( + room_id, + Arc::new( + state + .into_iter() + .stream() + .then(|(k, id)| async move { + services + .rooms + .state_compressor + .compress_state_event(k, &id) + .await + }) + .collect() + .await, + ), + ) + .await?; services .rooms @@ -968,12 +995,20 @@ async fn join_room_by_id_helper_remote( .await?; info!("Updating joined counts for new room"); - services.rooms.state_cache.update_joined_count(room_id)?; + services + .rooms + .state_cache + .update_joined_count(room_id) + .await; // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't // fail. - let statehash_after_join = services.rooms.state.append_to_state(&parsed_join_pdu)?; + let statehash_after_join = services + .rooms + .state + .append_to_state(&parsed_join_pdu) + .await?; info!("Appending new room join event"); services @@ -993,7 +1028,7 @@ async fn join_room_by_id_helper_remote( services .rooms .state - .set_room_state(room_id, statehash_after_join, &state_lock)?; + .set_room_state(room_id, statehash_after_join, &state_lock); Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) } @@ -1005,23 +1040,15 @@ async fn join_room_by_id_helper_local( ) -> Result { debug!("We can join locally"); - let join_rules_event = services + let join_rules_event_content = services .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomJoinRules, "")?; - - let join_rules_event_content: Option = join_rules_event - .as_ref() - .map(|join_rules_event| { - serde_json::from_str(join_rules_event.content.get()).map_err(|e| { - warn!("Invalid join rules event: {}", e); - Error::bad_database("Invalid join rules event in db.") - }) - }) - .transpose()?; + .room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") + .await + .map(|content: RoomJoinRulesEventContent| content); let restriction_rooms = match join_rules_event_content { - Some(RoomJoinRulesEventContent { + Ok(RoomJoinRulesEventContent { join_rule: JoinRule::Restricted(restricted) | JoinRule::KnockRestricted(restricted), }) => restricted .allow @@ -1034,29 +1061,34 @@ async fn join_room_by_id_helper_local( _ => Vec::new(), }; - let local_members = services + let local_members: Vec<_> = services .rooms .state_cache .room_members(room_id) - .filter_map(Result::ok) - .filter(|user| services.globals.user_is_local(user)) - .collect::>(); + .ready_filter(|user| services.globals.user_is_local(user)) + .map(ToOwned::to_owned) + .collect() + .await; let mut join_authorized_via_users_server: Option = None; - if restriction_rooms.iter().any(|restriction_room_id| { - services - .rooms - .state_cache - .is_joined(sender_user, restriction_room_id) - .unwrap_or(false) - }) { + if restriction_rooms + .iter() + .stream() + .any(|restriction_room_id| { + services + .rooms + .state_cache + .is_joined(sender_user, restriction_room_id) + }) + .await + { for user in local_members { if services .rooms .state_accessor .user_can_invite(room_id, &user, sender_user, &state_lock) - .unwrap_or(false) + .await { join_authorized_via_users_server = Some(user); break; @@ -1066,11 +1098,11 @@ async fn join_room_by_id_helper_local( let event = RoomMemberEventContent { membership: MembershipState::Join, - displayname: services.users.displayname(sender_user)?, - avatar_url: services.users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), is_direct: None, third_party_invite: None, - blurhash: services.users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user).await.ok(), reason: reason.clone(), join_authorized_via_users_server, }; @@ -1144,11 +1176,11 @@ async fn join_room_by_id_helper_local( "content".to_owned(), to_canonical_value(RoomMemberEventContent { membership: MembershipState::Join, - displayname: services.users.displayname(sender_user)?, - avatar_url: services.users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), is_direct: None, third_party_invite: None, - blurhash: services.users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user).await.ok(), reason, join_authorized_via_users_server, }) @@ -1195,10 +1227,11 @@ async fn join_room_by_id_helper_local( federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), event_id: event_id.to_owned(), + omit_members: false, pdu: services .sending - .convert_to_outgoing_federation_event(join_event.clone()), - omit_members: false, + .convert_to_outgoing_federation_event(join_event.clone()) + .await, }, ) .await?; @@ -1369,7 +1402,7 @@ pub(crate) async fn invite_helper( services: &Services, sender_user: &UserId, user_id: &UserId, room_id: &RoomId, reason: Option, is_direct: bool, ) -> Result<()> { - if !services.users.is_admin(user_id)? && services.globals.block_non_admin_invites() { + if !services.users.is_admin(user_id).await && services.globals.block_non_admin_invites() { info!("User {sender_user} is not an admin and attempted to send an invite to room {room_id}"); return Err(Error::BadRequest( ErrorKind::forbidden(), @@ -1381,7 +1414,7 @@ pub(crate) async fn invite_helper( let (pdu, pdu_json, invite_room_state) = { let state_lock = services.rooms.state.mutex.lock(room_id).await; let content = to_raw_value(&RoomMemberEventContent { - avatar_url: services.users.avatar_url(user_id)?, + avatar_url: services.users.avatar_url(user_id).await.ok(), displayname: None, is_direct: Some(is_direct), membership: MembershipState::Invite, @@ -1392,28 +1425,32 @@ pub(crate) async fn invite_helper( }) .expect("member event is valid value"); - let (pdu, pdu_json) = services.rooms.timeline.create_hash_and_sign_event( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content, - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }, - sender_user, - room_id, - &state_lock, - )?; + let (pdu, pdu_json) = services + .rooms + .timeline + .create_hash_and_sign_event( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content, + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + timestamp: None, + }, + sender_user, + room_id, + &state_lock, + ) + .await?; - let invite_room_state = services.rooms.state.calculate_invite_state(&pdu)?; + let invite_room_state = services.rooms.state.calculate_invite_state(&pdu).await?; drop(state_lock); (pdu, pdu_json, invite_room_state) }; - let room_version_id = services.rooms.state.get_room_version(room_id)?; + let room_version_id = services.rooms.state.get_room_version(room_id).await?; let response = services .sending @@ -1425,9 +1462,15 @@ pub(crate) async fn invite_helper( room_version: room_version_id.clone(), event: services .sending - .convert_to_outgoing_federation_event(pdu_json.clone()), + .convert_to_outgoing_federation_event(pdu_json.clone()) + .await, invite_room_state, - via: services.rooms.state_cache.servers_route_via(room_id).ok(), + via: services + .rooms + .state_cache + .servers_route_via(room_id) + .await + .ok(), }, ) .await?; @@ -1478,11 +1521,16 @@ pub(crate) async fn invite_helper( "Could not accept incoming PDU as timeline event.", ))?; - services.sending.send_pdu_room(room_id, &pdu_id)?; + services.sending.send_pdu_room(room_id, &pdu_id).await?; return Ok(()); } - if !services.rooms.state_cache.is_joined(sender_user, room_id)? { + if !services + .rooms + .state_cache + .is_joined(sender_user, room_id) + .await + { return Err(Error::BadRequest( ErrorKind::forbidden(), "You don't have permission to view this room.", @@ -1499,11 +1547,11 @@ pub(crate) async fn invite_helper( event_type: TimelineEventType::RoomMember, content: to_raw_value(&RoomMemberEventContent { membership: MembershipState::Invite, - displayname: services.users.displayname(user_id)?, - avatar_url: services.users.avatar_url(user_id)?, + displayname: services.users.displayname(user_id).await.ok(), + avatar_url: services.users.avatar_url(user_id).await.ok(), is_direct: Some(is_direct), third_party_invite: None, - blurhash: services.users.blurhash(user_id)?, + blurhash: services.users.blurhash(user_id).await.ok(), reason, join_authorized_via_users_server: None, }) @@ -1531,36 +1579,37 @@ pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { .rooms .state_cache .rooms_joined(user_id) + .map(ToOwned::to_owned) .chain( services .rooms .state_cache .rooms_invited(user_id) - .map(|t| t.map(|(r, _)| r)), + .map(|(r, _)| r), ) - .collect::>(); + .collect::>() + .await; for room_id in all_rooms { - let Ok(room_id) = room_id else { - continue; - }; - // ignore errors if let Err(e) = leave_room(services, user_id, &room_id, None).await { warn!(%room_id, %user_id, %e, "Failed to leave room"); } - if let Err(e) = services.rooms.state_cache.forget(&room_id, user_id) { - warn!(%room_id, %user_id, %e, "Failed to forget room"); - } + + services.rooms.state_cache.forget(&room_id, user_id); } } pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, reason: Option) -> Result<()> { + //use conduit::utils::stream::OptionStream; + use futures::TryFutureExt; + // Ask a remote server if we don't have this room if !services .rooms .state_cache - .server_in_room(services.globals.server_name(), room_id)? + .server_in_room(services.globals.server_name(), room_id) + .await { if let Err(e) = remote_leave_room(services, user_id, room_id).await { warn!("Failed to leave room {} remotely: {}", user_id, e); @@ -1570,34 +1619,42 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, let last_state = services .rooms .state_cache - .invite_state(user_id, room_id)? - .map_or_else(|| services.rooms.state_cache.left_state(user_id, room_id), |s| Ok(Some(s)))?; + .invite_state(user_id, room_id) + .map_err(|_| services.rooms.state_cache.left_state(user_id, room_id)) + .await + .ok(); // We always drop the invite, we can't rely on other servers - services.rooms.state_cache.update_membership( - room_id, - user_id, - RoomMemberEventContent::new(MembershipState::Leave), - user_id, - last_state, - None, - true, - )?; + services + .rooms + .state_cache + .update_membership( + room_id, + user_id, + RoomMemberEventContent::new(MembershipState::Leave), + user_id, + last_state, + None, + true, + ) + .await?; } else { let state_lock = services.rooms.state.mutex.lock(room_id).await; - let member_event = - services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())?; + let member_event = services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str()) + .await; // Fix for broken rooms - let member_event = match member_event { - None => { - error!("Trying to leave a room you are not a member of."); + let Ok(member_event) = member_event else { + error!("Trying to leave a room you are not a member of."); - services.rooms.state_cache.update_membership( + services + .rooms + .state_cache + .update_membership( room_id, user_id, RoomMemberEventContent::new(MembershipState::Leave), @@ -1605,16 +1662,14 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, None, None, true, - )?; - return Ok(()); - }, - Some(e) => e, + ) + .await?; + + return Ok(()); }; - let mut event: RoomMemberEventContent = serde_json::from_str(member_event.content.get()).map_err(|e| { - error!("Invalid room member event in database: {}", e); - Error::bad_database("Invalid member event in database.") - })?; + let mut event: RoomMemberEventContent = serde_json::from_str(member_event.content.get()) + .map_err(|e| err!(Database(error!("Invalid room member event in database: {e}"))))?; event.membership = MembershipState::Leave; event.reason = reason; @@ -1647,15 +1702,17 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room let invite_state = services .rooms .state_cache - .invite_state(user_id, room_id)? - .ok_or(Error::BadRequest(ErrorKind::BadState, "User is not invited."))?; + .invite_state(user_id, room_id) + .await + .map_err(|_| err!(Request(BadState("User is not invited."))))?; let mut servers: HashSet = services .rooms .state_cache .servers_invite_via(room_id) - .filter_map(Result::ok) - .collect(); + .map(ToOwned::to_owned) + .collect() + .await; servers.extend( invite_state @@ -1760,7 +1817,8 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room event_id, pdu: services .sending - .convert_to_outgoing_federation_event(leave_event.clone()), + .convert_to_outgoing_federation_event(leave_event.clone()) + .await, }, ) .await?; diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 51aee8c1..bab5fa54 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,7 +1,8 @@ use std::collections::{BTreeMap, HashSet}; use axum::extract::State; -use conduit::PduCount; +use conduit::{err, utils::ReadyExt, Err, PduCount}; +use futures::{FutureExt, StreamExt}; use ruma::{ api::client::{ error::ErrorKind, @@ -9,13 +10,14 @@ use ruma::{ message::{get_message_events, send_message_event}, }, events::{MessageLikeEventType, StateEventType}, - RoomId, UserId, + UserId, }; use serde_json::{from_str, Value}; +use service::rooms::timeline::PdusIterItem; use crate::{ service::{pdu::PduBuilder, Services}, - utils, Error, PduEvent, Result, Ruma, + utils, Error, Result, Ruma, }; /// # `PUT /_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}` @@ -30,79 +32,78 @@ use crate::{ pub(crate) async fn send_message_event_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user.as_deref().expect("user is authenticated"); let sender_device = body.sender_device.as_deref(); - - let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + let appservice_info = body.appservice_info.as_ref(); // Forbid m.room.encrypted if encryption is disabled if MessageLikeEventType::RoomEncrypted == body.event_type && !services.globals.allow_encryption() { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Encryption has been disabled")); + return Err!(Request(Forbidden("Encryption has been disabled"))); } - if body.event_type == MessageLikeEventType::CallInvite && services.rooms.directory.is_public_room(&body.room_id)? { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Room call invites are not allowed in public rooms", - )); + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + + if body.event_type == MessageLikeEventType::CallInvite + && services.rooms.directory.is_public_room(&body.room_id).await + { + return Err!(Request(Forbidden("Room call invites are not allowed in public rooms"))); } // Check if this is a new transaction id - if let Some(response) = services + if let Ok(response) = services .transaction_ids - .existing_txnid(sender_user, sender_device, &body.txn_id)? + .existing_txnid(sender_user, sender_device, &body.txn_id) + .await { // The client might have sent a txnid of the /sendToDevice endpoint // This txnid has no response associated with it if response.is_empty() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Tried to use txn id already used for an incompatible endpoint.", - )); + return Err!(Request(InvalidParam( + "Tried to use txn id already used for an incompatible endpoint." + ))); } - let event_id = utils::string_from_bytes(&response) - .map_err(|_| Error::bad_database("Invalid txnid bytes in database."))? - .try_into() - .map_err(|_| Error::bad_database("Invalid event id in txnid data."))?; return Ok(send_message_event::v3::Response { - event_id, + event_id: utils::string_from_bytes(&response) + .map(TryInto::try_into) + .map_err(|e| err!(Database("Invalid event_id in txnid data: {e:?}")))??, }); } let mut unsigned = BTreeMap::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.to_string().into()); + let content = from_str(body.body.body.json().get()) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; + let event_id = services .rooms .timeline .build_and_append_pdu( PduBuilder { event_type: body.event_type.to_string().into(), - content: from_str(body.body.body.json().get()) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?, + content, unsigned: Some(unsigned), state_key: None, redacts: None, - timestamp: if body.appservice_info.is_some() { - body.timestamp - } else { - None - }, + timestamp: appservice_info.and(body.timestamp), }, sender_user, &body.room_id, &state_lock, ) - .await?; + .await + .map(|event_id| (*event_id).to_owned())?; services .transaction_ids - .add_txnid(sender_user, sender_device, &body.txn_id, event_id.as_bytes())?; + .add_txnid(sender_user, sender_device, &body.txn_id, event_id.as_bytes()); drop(state_lock); - Ok(send_message_event::v3::Response::new((*event_id).to_owned())) + Ok(send_message_event::v3::Response { + event_id, + }) } /// # `GET /_matrix/client/r0/rooms/{roomId}/messages` @@ -117,8 +118,12 @@ pub(crate) async fn get_message_events_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); - let from = match body.from.clone() { - Some(from) => PduCount::try_from_string(&from)?, + let room_id = &body.room_id; + let filter = &body.filter; + + let limit = usize::try_from(body.limit).unwrap_or(10).min(100); + let from = match body.from.as_ref() { + Some(from) => PduCount::try_from_string(from)?, None => match body.dir { ruma::api::Direction::Forward => PduCount::min(), ruma::api::Direction::Backward => PduCount::max(), @@ -133,30 +138,25 @@ pub(crate) async fn get_message_events_route( services .rooms .lazy_loading - .lazy_load_confirm_delivery(sender_user, sender_device, &body.room_id, from) - .await?; - - let limit = usize::try_from(body.limit).unwrap_or(10).min(100); - - let next_token; + .lazy_load_confirm_delivery(sender_user, sender_device, room_id, from); let mut resp = get_message_events::v3::Response::new(); - let mut lazy_loaded = HashSet::new(); - + let next_token; match body.dir { ruma::api::Direction::Forward => { - let events_after: Vec<_> = services + let events_after: Vec = services .rooms .timeline - .pdus_after(sender_user, &body.room_id, from)? - .filter_map(Result::ok) // Filter out buggy events - .filter(|(_, pdu)| { contains_url_filter(pdu, &body.filter) && visibility_filter(&services, pdu, sender_user, &body.room_id) - - }) - .take_while(|&(k, _)| Some(k) != to) // Stop at `to` + .pdus_after(sender_user, room_id, from) + .await? + .ready_filter_map(|item| contains_url_filter(item, filter)) + .filter_map(|item| visibility_filter(&services, item, sender_user)) + .ready_take_while(|(count, _)| Some(*count) != to) // Stop at `to` .take(limit) - .collect(); + .collect() + .boxed() + .await; for (_, event) in &events_after { /* TODO: Remove the not "element_hacks" check when these are resolved: @@ -164,16 +164,18 @@ pub(crate) async fn get_message_events_route( * https://github.com/vector-im/element-web/issues/21034 */ if !cfg!(feature = "element_hacks") - && !services.rooms.lazy_loading.lazy_load_was_sent_before( - sender_user, - sender_device, - &body.room_id, - &event.sender, - )? { + && !services + .rooms + .lazy_loading + .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) + .await + { lazy_loaded.insert(event.sender.clone()); } - lazy_loaded.insert(event.sender.clone()); + if cfg!(features = "element_hacks") { + lazy_loaded.insert(event.sender.clone()); + } } next_token = events_after.last().map(|(count, _)| count).copied(); @@ -191,17 +193,22 @@ pub(crate) async fn get_message_events_route( services .rooms .timeline - .backfill_if_required(&body.room_id, from) + .backfill_if_required(room_id, from) + .boxed() .await?; - let events_before: Vec<_> = services + + let events_before: Vec = services .rooms .timeline - .pdus_until(sender_user, &body.room_id, from)? - .filter_map(Result::ok) // Filter out buggy events - .filter(|(_, pdu)| {contains_url_filter(pdu, &body.filter) && visibility_filter(&services, pdu, sender_user, &body.room_id)}) - .take_while(|&(k, _)| Some(k) != to) // Stop at `to` + .pdus_until(sender_user, room_id, from) + .await? + .ready_filter_map(|item| contains_url_filter(item, filter)) + .filter_map(|item| visibility_filter(&services, item, sender_user)) + .ready_take_while(|(count, _)| Some(*count) != to) // Stop at `to` .take(limit) - .collect(); + .collect() + .boxed() + .await; for (_, event) in &events_before { /* TODO: Remove the not "element_hacks" check when these are resolved: @@ -209,16 +216,18 @@ pub(crate) async fn get_message_events_route( * https://github.com/vector-im/element-web/issues/21034 */ if !cfg!(feature = "element_hacks") - && !services.rooms.lazy_loading.lazy_load_was_sent_before( - sender_user, - sender_device, - &body.room_id, - &event.sender, - )? { + && !services + .rooms + .lazy_loading + .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) + .await + { lazy_loaded.insert(event.sender.clone()); } - lazy_loaded.insert(event.sender.clone()); + if cfg!(features = "element_hacks") { + lazy_loaded.insert(event.sender.clone()); + } } next_token = events_before.last().map(|(count, _)| count).copied(); @@ -236,11 +245,11 @@ pub(crate) async fn get_message_events_route( resp.state = Vec::new(); for ll_id in &lazy_loaded { - if let Some(member_event) = - services - .rooms - .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomMember, ll_id.as_str())? + if let Ok(member_event) = services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomMember, ll_id.as_str()) + .await { resp.state.push(member_event.to_state_event()); } @@ -249,34 +258,43 @@ pub(crate) async fn get_message_events_route( // remove the feature check when we are sure clients like element can handle it if !cfg!(feature = "element_hacks") { if let Some(next_token) = next_token { - services - .rooms - .lazy_loading - .lazy_load_mark_sent(sender_user, sender_device, &body.room_id, lazy_loaded, next_token) - .await; + services.rooms.lazy_loading.lazy_load_mark_sent( + sender_user, + sender_device, + room_id, + lazy_loaded, + next_token, + ); } } Ok(resp) } -fn visibility_filter(services: &Services, pdu: &PduEvent, user_id: &UserId, room_id: &RoomId) -> bool { +async fn visibility_filter(services: &Services, item: PdusIterItem, user_id: &UserId) -> Option { + let (_, pdu) = &item; + services .rooms .state_accessor - .user_can_see_event(user_id, room_id, &pdu.event_id) - .unwrap_or(false) + .user_can_see_event(user_id, &pdu.room_id, &pdu.event_id) + .await + .then_some(item) } -fn contains_url_filter(pdu: &PduEvent, filter: &RoomEventFilter) -> bool { +fn contains_url_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Option { + let (_, pdu) = &item; + if filter.url_filter.is_none() { - return true; + return Some(item); } let content: Value = from_str(pdu.content.get()).unwrap(); - match filter.url_filter { + let res = match filter.url_filter { Some(UrlFilter::EventsWithoutUrl) => !content["url"].is_string(), Some(UrlFilter::EventsWithUrl) => content["url"].is_string(), None => true, - } + }; + + res.then_some(item) } diff --git a/src/api/client/presence.rs b/src/api/client/presence.rs index 8384d5ac..ba48808b 100644 --- a/src/api/client/presence.rs +++ b/src/api/client/presence.rs @@ -28,7 +28,8 @@ pub(crate) async fn set_presence_route( services .presence - .set_presence(sender_user, &body.presence, None, None, body.status_msg.clone())?; + .set_presence(sender_user, &body.presence, None, None, body.status_msg.clone()) + .await?; Ok(set_presence::v3::Response {}) } @@ -49,14 +50,15 @@ pub(crate) async fn get_presence_route( let mut presence_event = None; - for _room_id in services + let has_shared_rooms = services .rooms .user - .get_shared_rooms(vec![sender_user.clone(), body.user_id.clone()])? - { - if let Some(presence) = services.presence.get_presence(&body.user_id)? { + .has_shared_rooms(sender_user, &body.user_id) + .await; + + if has_shared_rooms { + if let Ok(presence) = services.presence.get_presence(&body.user_id).await { presence_event = Some(presence); - break; } } diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index bf47a3f8..495bc8ec 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -1,5 +1,10 @@ use axum::extract::State; -use conduit::{pdu::PduBuilder, warn, Err, Error, Result}; +use conduit::{ + pdu::PduBuilder, + utils::{stream::TryIgnore, IterStream}, + warn, Err, Error, Result, +}; +use futures::{StreamExt, TryStreamExt}; use ruma::{ api::{ client::{ @@ -35,16 +40,18 @@ pub(crate) async fn set_displayname_route( .rooms .state_cache .rooms_joined(&body.user_id) - .filter_map(Result::ok) - .collect(); + .map(ToOwned::to_owned) + .collect() + .await; - update_displayname(&services, &body.user_id, body.displayname.clone(), all_joined_rooms).await?; + update_displayname(&services, &body.user_id, body.displayname.clone(), &all_joined_rooms).await?; if services.globals.allow_local_presence() { // Presence update services .presence - .ping_presence(&body.user_id, &PresenceState::Online)?; + .ping_presence(&body.user_id, &PresenceState::Online) + .await?; } Ok(set_display_name::v3::Response {}) @@ -72,22 +79,19 @@ pub(crate) async fn get_displayname_route( ) .await { - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { services.users.create(&body.user_id, None)?; } services .users - .set_displayname(&body.user_id, response.displayname.clone()) - .await?; + .set_displayname(&body.user_id, response.displayname.clone()); services .users - .set_avatar_url(&body.user_id, response.avatar_url.clone()) - .await?; + .set_avatar_url(&body.user_id, response.avatar_url.clone()); services .users - .set_blurhash(&body.user_id, response.blurhash.clone()) - .await?; + .set_blurhash(&body.user_id, response.blurhash.clone()); return Ok(get_display_name::v3::Response { displayname: response.displayname, @@ -95,14 +99,14 @@ pub(crate) async fn get_displayname_route( } } - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); } Ok(get_display_name::v3::Response { - displayname: services.users.displayname(&body.user_id)?, + displayname: services.users.displayname(&body.user_id).await.ok(), }) } @@ -124,15 +128,16 @@ pub(crate) async fn set_avatar_url_route( .rooms .state_cache .rooms_joined(&body.user_id) - .filter_map(Result::ok) - .collect(); + .map(ToOwned::to_owned) + .collect() + .await; update_avatar_url( &services, &body.user_id, body.avatar_url.clone(), body.blurhash.clone(), - all_joined_rooms, + &all_joined_rooms, ) .await?; @@ -140,7 +145,9 @@ pub(crate) async fn set_avatar_url_route( // Presence update services .presence - .ping_presence(&body.user_id, &PresenceState::Online)?; + .ping_presence(&body.user_id, &PresenceState::Online) + .await + .ok(); } Ok(set_avatar_url::v3::Response {}) @@ -168,22 +175,21 @@ pub(crate) async fn get_avatar_url_route( ) .await { - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { services.users.create(&body.user_id, None)?; } services .users - .set_displayname(&body.user_id, response.displayname.clone()) - .await?; + .set_displayname(&body.user_id, response.displayname.clone()); + services .users - .set_avatar_url(&body.user_id, response.avatar_url.clone()) - .await?; + .set_avatar_url(&body.user_id, response.avatar_url.clone()); + services .users - .set_blurhash(&body.user_id, response.blurhash.clone()) - .await?; + .set_blurhash(&body.user_id, response.blurhash.clone()); return Ok(get_avatar_url::v3::Response { avatar_url: response.avatar_url, @@ -192,15 +198,15 @@ pub(crate) async fn get_avatar_url_route( } } - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); } Ok(get_avatar_url::v3::Response { - avatar_url: services.users.avatar_url(&body.user_id)?, - blurhash: services.users.blurhash(&body.user_id)?, + avatar_url: services.users.avatar_url(&body.user_id).await.ok(), + blurhash: services.users.blurhash(&body.user_id).await.ok(), }) } @@ -226,31 +232,30 @@ pub(crate) async fn get_profile_route( ) .await { - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { services.users.create(&body.user_id, None)?; } services .users - .set_displayname(&body.user_id, response.displayname.clone()) - .await?; + .set_displayname(&body.user_id, response.displayname.clone()); + services .users - .set_avatar_url(&body.user_id, response.avatar_url.clone()) - .await?; + .set_avatar_url(&body.user_id, response.avatar_url.clone()); + services .users - .set_blurhash(&body.user_id, response.blurhash.clone()) - .await?; + .set_blurhash(&body.user_id, response.blurhash.clone()); + services .users - .set_timezone(&body.user_id, response.tz.clone()) - .await?; + .set_timezone(&body.user_id, response.tz.clone()); for (profile_key, profile_key_value) in &response.custom_profile_fields { services .users - .set_profile_key(&body.user_id, profile_key, Some(profile_key_value.clone()))?; + .set_profile_key(&body.user_id, profile_key, Some(profile_key_value.clone())); } return Ok(get_profile::v3::Response { @@ -263,104 +268,93 @@ pub(crate) async fn get_profile_route( } } - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); } Ok(get_profile::v3::Response { - avatar_url: services.users.avatar_url(&body.user_id)?, - blurhash: services.users.blurhash(&body.user_id)?, - displayname: services.users.displayname(&body.user_id)?, - tz: services.users.timezone(&body.user_id)?, + avatar_url: services.users.avatar_url(&body.user_id).await.ok(), + blurhash: services.users.blurhash(&body.user_id).await.ok(), + displayname: services.users.displayname(&body.user_id).await.ok(), + tz: services.users.timezone(&body.user_id).await.ok(), custom_profile_fields: services .users .all_profile_keys(&body.user_id) - .filter_map(Result::ok) - .collect(), + .collect() + .await, }) } pub async fn update_displayname( - services: &Services, user_id: &UserId, displayname: Option, all_joined_rooms: Vec, + services: &Services, user_id: &UserId, displayname: Option, all_joined_rooms: &[OwnedRoomId], ) -> Result<()> { - let current_display_name = services.users.displayname(user_id).unwrap_or_default(); + let current_display_name = services.users.displayname(user_id).await.ok(); if displayname == current_display_name { return Ok(()); } - services - .users - .set_displayname(user_id, displayname.clone()) - .await?; + services.users.set_displayname(user_id, displayname.clone()); // Send a new join membership event into all joined rooms - let all_joined_rooms: Vec<_> = all_joined_rooms - .iter() - .map(|room_id| { - Ok::<_, Error>(( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - displayname: displayname.clone(), - join_authorized_via_users_server: None, - ..serde_json::from_str( - services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())? - .ok_or_else(|| { - Error::bad_database("Tried to send display name update for user not in the room.") - })? - .content - .get(), - ) - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }, - room_id, - )) - }) - .filter_map(Result::ok) - .collect(); + let mut joined_rooms = Vec::new(); + for room_id in all_joined_rooms { + let Ok(event) = services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str()) + .await + else { + continue; + }; - update_all_rooms(services, all_joined_rooms, user_id).await; + let pdu = PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + displayname: displayname.clone(), + join_authorized_via_users_server: None, + ..serde_json::from_str(event.content.get()).expect("Database contains invalid PDU.") + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + timestamp: None, + }; + + joined_rooms.push((pdu, room_id)); + } + + update_all_rooms(services, joined_rooms, user_id).await; Ok(()) } pub async fn update_avatar_url( services: &Services, user_id: &UserId, avatar_url: Option, blurhash: Option, - all_joined_rooms: Vec, + all_joined_rooms: &[OwnedRoomId], ) -> Result<()> { - let current_avatar_url = services.users.avatar_url(user_id).unwrap_or_default(); - let current_blurhash = services.users.blurhash(user_id).unwrap_or_default(); + let current_avatar_url = services.users.avatar_url(user_id).await.ok(); + let current_blurhash = services.users.blurhash(user_id).await.ok(); if current_avatar_url == avatar_url && current_blurhash == blurhash { return Ok(()); } - services - .users - .set_avatar_url(user_id, avatar_url.clone()) - .await?; - services - .users - .set_blurhash(user_id, blurhash.clone()) - .await?; + services.users.set_avatar_url(user_id, avatar_url.clone()); + + services.users.set_blurhash(user_id, blurhash.clone()); // Send a new join membership event into all joined rooms + let avatar_url = &avatar_url; + let blurhash = &blurhash; let all_joined_rooms: Vec<_> = all_joined_rooms .iter() - .map(|room_id| { - Ok::<_, Error>(( + .try_stream() + .and_then(|room_id: &OwnedRoomId| async move { + Ok(( PduBuilder { event_type: TimelineEventType::RoomMember, content: to_raw_value(&RoomMemberEventContent { @@ -371,8 +365,9 @@ pub async fn update_avatar_url( services .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())? - .ok_or_else(|| { + .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str()) + .await + .map_err(|_| { Error::bad_database("Tried to send avatar URL update for user not in the room.") })? .content @@ -389,8 +384,9 @@ pub async fn update_avatar_url( room_id, )) }) - .filter_map(Result::ok) - .collect(); + .ignore_err() + .collect() + .await; update_all_rooms(services, all_joined_rooms, user_id).await; diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 8723e676..39095199 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -29,41 +29,37 @@ pub(crate) async fn get_pushrules_all_route( let global_ruleset: Ruleset; - let Ok(event) = - services - .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) - else { - // push rules event doesn't exist, create it and return default - return recreate_push_rules_and_return(&services, sender_user); - }; + let event = services + .account_data + .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .await; - if let Some(event) = event { - let value = serde_json::from_str::(event.get()) - .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; - - let Some(content_value) = value.get("content") else { - // user somehow has a push rule event with no content key, recreate it and - // return server default silently - return recreate_push_rules_and_return(&services, sender_user); - }; - - if content_value.to_string().is_empty() { - // user somehow has a push rule event with empty content, recreate it and return - // server default silently - return recreate_push_rules_and_return(&services, sender_user); - } - - let account_data_content = serde_json::from_value::(content_value.clone().into()) - .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; - - global_ruleset = account_data_content.global; - } else { + let Ok(event) = event else { // user somehow has non-existent push rule event. recreate it and return server // default silently - return recreate_push_rules_and_return(&services, sender_user); + return recreate_push_rules_and_return(&services, sender_user).await; + }; + + let value = serde_json::from_str::(event.get()) + .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; + + let Some(content_value) = value.get("content") else { + // user somehow has a push rule event with no content key, recreate it and + // return server default silently + return recreate_push_rules_and_return(&services, sender_user).await; + }; + + if content_value.to_string().is_empty() { + // user somehow has a push rule event with empty content, recreate it and return + // server default silently + return recreate_push_rules_and_return(&services, sender_user).await; } + let account_data_content = serde_json::from_value::(content_value.clone().into()) + .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; + + global_ruleset = account_data_content.global; + Ok(get_pushrules_all::v3::Response { global: global_ruleset, }) @@ -79,8 +75,9 @@ pub(crate) async fn get_pushrule_route( let event = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .await + .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; let account_data = serde_json::from_str::(event.get()) .map_err(|_| Error::bad_database("Invalid account data event in db."))? @@ -118,8 +115,9 @@ pub(crate) async fn set_pushrule_route( let event = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .await + .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; let mut account_data = serde_json::from_str::(event.get()) .map_err(|_| Error::bad_database("Invalid account data event in db."))?; @@ -155,12 +153,15 @@ pub(crate) async fn set_pushrule_route( return Err(err); } - services.account_data.update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - )?; + services + .account_data + .update( + None, + sender_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(account_data).expect("to json value always works"), + ) + .await?; Ok(set_pushrule::v3::Response {}) } @@ -182,8 +183,9 @@ pub(crate) async fn get_pushrule_actions_route( let event = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .await + .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; let account_data = serde_json::from_str::(event.get()) .map_err(|_| Error::bad_database("Invalid account data event in db."))? @@ -217,8 +219,9 @@ pub(crate) async fn set_pushrule_actions_route( let event = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .await + .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; let mut account_data = serde_json::from_str::(event.get()) .map_err(|_| Error::bad_database("Invalid account data event in db."))?; @@ -232,12 +235,15 @@ pub(crate) async fn set_pushrule_actions_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")); } - services.account_data.update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - )?; + services + .account_data + .update( + None, + sender_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(account_data).expect("to json value always works"), + ) + .await?; Ok(set_pushrule_actions::v3::Response {}) } @@ -259,8 +265,9 @@ pub(crate) async fn get_pushrule_enabled_route( let event = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .await + .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; let account_data = serde_json::from_str::(event.get()) .map_err(|_| Error::bad_database("Invalid account data event in db."))?; @@ -293,8 +300,9 @@ pub(crate) async fn set_pushrule_enabled_route( let event = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .await + .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; let mut account_data = serde_json::from_str::(event.get()) .map_err(|_| Error::bad_database("Invalid account data event in db."))?; @@ -308,12 +316,15 @@ pub(crate) async fn set_pushrule_enabled_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")); } - services.account_data.update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - )?; + services + .account_data + .update( + None, + sender_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(account_data).expect("to json value always works"), + ) + .await?; Ok(set_pushrule_enabled::v3::Response {}) } @@ -335,8 +346,9 @@ pub(crate) async fn delete_pushrule_route( let event = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into())? - .ok_or(Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .await + .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; let mut account_data = serde_json::from_str::(event.get()) .map_err(|_| Error::bad_database("Invalid account data event in db."))?; @@ -357,12 +369,15 @@ pub(crate) async fn delete_pushrule_route( return Err(err); } - services.account_data.update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - )?; + services + .account_data + .update( + None, + sender_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(account_data).expect("to json value always works"), + ) + .await?; Ok(delete_pushrule::v3::Response {}) } @@ -376,7 +391,7 @@ pub(crate) async fn get_pushers_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); Ok(get_pushers::v3::Response { - pushers: services.pusher.get_pushers(sender_user)?, + pushers: services.pusher.get_pushers(sender_user).await, }) } @@ -390,27 +405,30 @@ pub(crate) async fn set_pushers_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services.pusher.set_pusher(sender_user, &body.action)?; + services.pusher.set_pusher(sender_user, &body.action); Ok(set_pusher::v3::Response::default()) } /// user somehow has bad push rules, these must always exist per spec. /// so recreate it and return server default silently -fn recreate_push_rules_and_return( +async fn recreate_push_rules_and_return( services: &Services, sender_user: &ruma::UserId, ) -> Result { - services.account_data.update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(PushRulesEvent { - content: PushRulesEventContent { - global: Ruleset::server_default(sender_user), - }, - }) - .expect("to json always works"), - )?; + services + .account_data + .update( + None, + sender_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(PushRulesEvent { + content: PushRulesEventContent { + global: Ruleset::server_default(sender_user), + }, + }) + .expect("to json always works"), + ) + .await?; Ok(get_pushrules_all::v3::Response { global: Ruleset::server_default(sender_user), diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index f40f2493..f28b2aec 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -31,27 +31,32 @@ pub(crate) async fn set_read_marker_route( event_id: fully_read.clone(), }, }; - services.account_data.update( - Some(&body.room_id), - sender_user, - RoomAccountDataEventType::FullyRead, - &serde_json::to_value(fully_read_event).expect("to json value always works"), - )?; + services + .account_data + .update( + Some(&body.room_id), + sender_user, + RoomAccountDataEventType::FullyRead, + &serde_json::to_value(fully_read_event).expect("to json value always works"), + ) + .await?; } if body.private_read_receipt.is_some() || body.read_receipt.is_some() { services .rooms .user - .reset_notification_counts(sender_user, &body.room_id)?; + .reset_notification_counts(sender_user, &body.room_id); } if let Some(event) = &body.private_read_receipt { let count = services .rooms .timeline - .get_pdu_count(event)? - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?; + .get_pdu_count(event) + .await + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?; + let count = match count { PduCount::Backfilled(_) => { return Err(Error::BadRequest( @@ -64,7 +69,7 @@ pub(crate) async fn set_read_marker_route( services .rooms .read_receipt - .private_read_set(&body.room_id, sender_user, count)?; + .private_read_set(&body.room_id, sender_user, count); } if let Some(event) = &body.read_receipt { @@ -83,14 +88,18 @@ pub(crate) async fn set_read_marker_route( let mut receipt_content = BTreeMap::new(); receipt_content.insert(event.to_owned(), receipts); - services.rooms.read_receipt.readreceipt_update( - sender_user, - &body.room_id, - &ruma::events::receipt::ReceiptEvent { - content: ruma::events::receipt::ReceiptEventContent(receipt_content), - room_id: body.room_id.clone(), - }, - )?; + services + .rooms + .read_receipt + .readreceipt_update( + sender_user, + &body.room_id, + &ruma::events::receipt::ReceiptEvent { + content: ruma::events::receipt::ReceiptEventContent(receipt_content), + room_id: body.room_id.clone(), + }, + ) + .await; } Ok(set_read_marker::v3::Response {}) @@ -111,7 +120,7 @@ pub(crate) async fn create_receipt_route( services .rooms .user - .reset_notification_counts(sender_user, &body.room_id)?; + .reset_notification_counts(sender_user, &body.room_id); } match body.receipt_type { @@ -121,12 +130,15 @@ pub(crate) async fn create_receipt_route( event_id: body.event_id.clone(), }, }; - services.account_data.update( - Some(&body.room_id), - sender_user, - RoomAccountDataEventType::FullyRead, - &serde_json::to_value(fully_read_event).expect("to json value always works"), - )?; + services + .account_data + .update( + Some(&body.room_id), + sender_user, + RoomAccountDataEventType::FullyRead, + &serde_json::to_value(fully_read_event).expect("to json value always works"), + ) + .await?; }, create_receipt::v3::ReceiptType::Read => { let mut user_receipts = BTreeMap::new(); @@ -143,21 +155,27 @@ pub(crate) async fn create_receipt_route( let mut receipt_content = BTreeMap::new(); receipt_content.insert(body.event_id.clone(), receipts); - services.rooms.read_receipt.readreceipt_update( - sender_user, - &body.room_id, - &ruma::events::receipt::ReceiptEvent { - content: ruma::events::receipt::ReceiptEventContent(receipt_content), - room_id: body.room_id.clone(), - }, - )?; + services + .rooms + .read_receipt + .readreceipt_update( + sender_user, + &body.room_id, + &ruma::events::receipt::ReceiptEvent { + content: ruma::events::receipt::ReceiptEventContent(receipt_content), + room_id: body.room_id.clone(), + }, + ) + .await; }, create_receipt::v3::ReceiptType::ReadPrivate => { let count = services .rooms .timeline - .get_pdu_count(&body.event_id)? - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?; + .get_pdu_count(&body.event_id) + .await + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?; + let count = match count { PduCount::Backfilled(_) => { return Err(Error::BadRequest( @@ -170,7 +188,7 @@ pub(crate) async fn create_receipt_route( services .rooms .read_receipt - .private_read_set(&body.room_id, sender_user, count)?; + .private_read_set(&body.room_id, sender_user, count); }, _ => return Err(Error::bad_database("Unsupported receipt type")), } diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index ae645940..d4384730 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -9,20 +9,24 @@ use crate::{Result, Ruma}; pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user.as_deref().expect("user is authenticated"); - let res = services.rooms.pdu_metadata.paginate_relations_with_filter( - sender_user, - &body.room_id, - &body.event_id, - &Some(body.event_type.clone()), - &Some(body.rel_type.clone()), - &body.from, - &body.to, - &body.limit, - body.recurse, - body.dir, - )?; + let res = services + .rooms + .pdu_metadata + .paginate_relations_with_filter( + sender_user, + &body.room_id, + &body.event_id, + body.event_type.clone().into(), + body.rel_type.clone().into(), + body.from.as_ref(), + body.to.as_ref(), + body.limit, + body.recurse, + body.dir, + ) + .await?; Ok(get_relating_events_with_rel_type_and_event_type::v1::Response { chunk: res.chunk, @@ -36,20 +40,24 @@ pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( pub(crate) async fn get_relating_events_with_rel_type_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user.as_deref().expect("user is authenticated"); - let res = services.rooms.pdu_metadata.paginate_relations_with_filter( - sender_user, - &body.room_id, - &body.event_id, - &None, - &Some(body.rel_type.clone()), - &body.from, - &body.to, - &body.limit, - body.recurse, - body.dir, - )?; + let res = services + .rooms + .pdu_metadata + .paginate_relations_with_filter( + sender_user, + &body.room_id, + &body.event_id, + None, + body.rel_type.clone().into(), + body.from.as_ref(), + body.to.as_ref(), + body.limit, + body.recurse, + body.dir, + ) + .await?; Ok(get_relating_events_with_rel_type::v1::Response { chunk: res.chunk, @@ -63,18 +71,22 @@ pub(crate) async fn get_relating_events_with_rel_type_route( pub(crate) async fn get_relating_events_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user.as_deref().expect("user is authenticated"); - services.rooms.pdu_metadata.paginate_relations_with_filter( - sender_user, - &body.room_id, - &body.event_id, - &None, - &None, - &body.from, - &body.to, - &body.limit, - body.recurse, - body.dir, - ) + services + .rooms + .pdu_metadata + .paginate_relations_with_filter( + sender_user, + &body.room_id, + &body.event_id, + None, + None, + body.from.as_ref(), + body.to.as_ref(), + body.limit, + body.recurse, + body.dir, + ) + .await } diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 588bd368..a40c35a2 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -1,6 +1,7 @@ use std::time::Duration; use axum::extract::State; +use conduit::{utils::ReadyExt, Err}; use rand::Rng; use ruma::{ api::client::{error::ErrorKind, room::report_content}, @@ -34,11 +35,8 @@ pub(crate) async fn report_event_route( delay_response().await; // check if we know about the reported event ID or if it's invalid - let Some(pdu) = services.rooms.timeline.get_pdu(&body.event_id)? else { - return Err(Error::BadRequest( - ErrorKind::NotFound, - "Event ID is not known to us or Event ID is invalid", - )); + let Ok(pdu) = services.rooms.timeline.get_pdu(&body.event_id).await else { + return Err!(Request(NotFound("Event ID is not known to us or Event ID is invalid"))); }; is_report_valid( @@ -49,7 +47,8 @@ pub(crate) async fn report_event_route( &body.reason, body.score, &pdu, - )?; + ) + .await?; // send admin room message that we received the report with an @room ping for // urgency @@ -81,7 +80,8 @@ pub(crate) async fn report_event_route( HtmlEscape(body.reason.as_deref().unwrap_or("")) ), )) - .await; + .await + .ok(); Ok(report_content::v3::Response {}) } @@ -92,7 +92,7 @@ pub(crate) async fn report_event_route( /// check if score is in valid range /// check if report reasoning is less than or equal to 750 characters /// check if reporting user is in the reporting room -fn is_report_valid( +async fn is_report_valid( services: &Services, event_id: &EventId, room_id: &RoomId, sender_user: &UserId, reason: &Option, score: Option, pdu: &std::sync::Arc, ) -> Result<()> { @@ -123,8 +123,8 @@ fn is_report_valid( .rooms .state_cache .room_members(room_id) - .filter_map(Result::ok) - .any(|user_id| user_id == *sender_user) + .ready_any(|user_id| user_id == sender_user) + .await { return Err(Error::BadRequest( ErrorKind::NotFound, diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 0112e76d..1edf85d8 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -2,6 +2,7 @@ use std::{cmp::max, collections::BTreeMap}; use axum::extract::State; use conduit::{debug_info, debug_warn, err, Err}; +use futures::{FutureExt, StreamExt}; use ruma::{ api::client::{ error::ErrorKind, @@ -74,7 +75,7 @@ pub(crate) async fn create_room_route( if !services.globals.allow_room_creation() && body.appservice_info.is_none() - && !services.users.is_admin(sender_user)? + && !services.users.is_admin(sender_user).await { return Err(Error::BadRequest(ErrorKind::forbidden(), "Room creation has been disabled.")); } @@ -86,7 +87,7 @@ pub(crate) async fn create_room_route( }; // check if room ID doesn't already exist instead of erroring on auth check - if services.rooms.short.get_shortroomid(&room_id)?.is_some() { + if services.rooms.short.get_shortroomid(&room_id).await.is_ok() { return Err(Error::BadRequest( ErrorKind::RoomInUse, "Room with that custom room ID already exists", @@ -95,7 +96,7 @@ pub(crate) async fn create_room_route( if body.visibility == room::Visibility::Public && services.globals.config.lockdown_public_room_directory - && !services.users.is_admin(sender_user)? + && !services.users.is_admin(sender_user).await && body.appservice_info.is_none() { info!( @@ -118,7 +119,11 @@ pub(crate) async fn create_room_route( return Err!(Request(Forbidden("Publishing rooms to the room directory is not allowed"))); } - let _short_id = services.rooms.short.get_or_create_shortroomid(&room_id)?; + let _short_id = services + .rooms + .short + .get_or_create_shortroomid(&room_id) + .await; let state_lock = services.rooms.state.mutex.lock(&room_id).await; let alias: Option = if let Some(alias) = &body.room_alias_name { @@ -218,6 +223,7 @@ pub(crate) async fn create_room_route( &room_id, &state_lock, ) + .boxed() .await?; // 2. Let the room creator join @@ -229,11 +235,11 @@ pub(crate) async fn create_room_route( event_type: TimelineEventType::RoomMember, content: to_raw_value(&RoomMemberEventContent { membership: MembershipState::Join, - displayname: services.users.displayname(sender_user)?, - avatar_url: services.users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), is_direct: Some(body.is_direct), third_party_invite: None, - blurhash: services.users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user).await.ok(), reason: None, join_authorized_via_users_server: None, }) @@ -247,6 +253,7 @@ pub(crate) async fn create_room_route( &room_id, &state_lock, ) + .boxed() .await?; // 3. Power levels @@ -284,6 +291,7 @@ pub(crate) async fn create_room_route( &room_id, &state_lock, ) + .boxed() .await?; // 4. Canonical room alias @@ -308,6 +316,7 @@ pub(crate) async fn create_room_route( &room_id, &state_lock, ) + .boxed() .await?; } @@ -335,6 +344,7 @@ pub(crate) async fn create_room_route( &room_id, &state_lock, ) + .boxed() .await?; // 5.2 History Visibility @@ -355,6 +365,7 @@ pub(crate) async fn create_room_route( &room_id, &state_lock, ) + .boxed() .await?; // 5.3 Guest Access @@ -378,6 +389,7 @@ pub(crate) async fn create_room_route( &room_id, &state_lock, ) + .boxed() .await?; // 6. Events listed in initial_state @@ -410,6 +422,7 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu(pdu_builder, sender_user, &room_id, &state_lock) + .boxed() .await?; } @@ -432,6 +445,7 @@ pub(crate) async fn create_room_route( &room_id, &state_lock, ) + .boxed() .await?; } @@ -455,13 +469,17 @@ pub(crate) async fn create_room_route( &room_id, &state_lock, ) + .boxed() .await?; } // 8. Events implied by invite (and TODO: invite_3pid) drop(state_lock); for user_id in &body.invite { - if let Err(e) = invite_helper(&services, sender_user, user_id, &room_id, None, body.is_direct).await { + if let Err(e) = invite_helper(&services, sender_user, user_id, &room_id, None, body.is_direct) + .boxed() + .await + { warn!(%e, "Failed to send invite"); } } @@ -475,7 +493,7 @@ pub(crate) async fn create_room_route( } if body.visibility == room::Visibility::Public { - services.rooms.directory.set_public(&room_id)?; + services.rooms.directory.set_public(&room_id); if services.globals.config.admin_room_notices { services @@ -505,13 +523,15 @@ pub(crate) async fn get_room_event_route( let event = services .rooms .timeline - .get_pdu(&body.event_id)? - .ok_or_else(|| err!(Request(NotFound("Event {} not found.", &body.event_id))))?; + .get_pdu(&body.event_id) + .await + .map_err(|_| err!(Request(NotFound("Event {} not found.", &body.event_id))))?; if !services .rooms .state_accessor - .user_can_see_event(sender_user, &event.room_id, &body.event_id)? + .user_can_see_event(sender_user, &event.room_id, &body.event_id) + .await { return Err(Error::BadRequest( ErrorKind::forbidden(), @@ -541,7 +561,8 @@ pub(crate) async fn get_room_aliases_route( if !services .rooms .state_accessor - .user_can_see_state_events(sender_user, &body.room_id)? + .user_can_see_state_events(sender_user, &body.room_id) + .await { return Err(Error::BadRequest( ErrorKind::forbidden(), @@ -554,8 +575,9 @@ pub(crate) async fn get_room_aliases_route( .rooms .alias .local_aliases_for_room(&body.room_id) - .filter_map(Result::ok) - .collect(), + .map(ToOwned::to_owned) + .collect() + .await, }) } @@ -591,7 +613,8 @@ pub(crate) async fn upgrade_room_route( let _short_id = services .rooms .short - .get_or_create_shortroomid(&replacement_room)?; + .get_or_create_shortroomid(&replacement_room) + .await; let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; @@ -629,12 +652,12 @@ pub(crate) async fn upgrade_room_route( services .rooms .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomCreate, "")? - .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))? + .room_state_get(&body.room_id, &StateEventType::RoomCreate, "") + .await + .map_err(|_| err!(Database("Found room without m.room.create event.")))? .content .get(), - ) - .map_err(|_| Error::bad_database("Invalid room event in database."))?; + )?; // Use the m.room.tombstone event as the predecessor let predecessor = Some(ruma::events::room::create::PreviousRoom::new( @@ -714,11 +737,11 @@ pub(crate) async fn upgrade_room_route( event_type: TimelineEventType::RoomMember, content: to_raw_value(&RoomMemberEventContent { membership: MembershipState::Join, - displayname: services.users.displayname(sender_user)?, - avatar_url: services.users.avatar_url(sender_user)?, + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), is_direct: None, third_party_invite: None, - blurhash: services.users.blurhash(sender_user)?, + blurhash: services.users.blurhash(sender_user).await.ok(), reason: None, join_authorized_via_users_server: None, }) @@ -739,10 +762,11 @@ pub(crate) async fn upgrade_room_route( let event_content = match services .rooms .state_accessor - .room_state_get(&body.room_id, event_type, "")? + .room_state_get(&body.room_id, event_type, "") + .await { - Some(v) => v.content.clone(), - None => continue, // Skipping missing events. + Ok(v) => v.content.clone(), + Err(_) => continue, // Skipping missing events. }; services @@ -765,21 +789,23 @@ pub(crate) async fn upgrade_room_route( } // Moves any local aliases to the new room - for alias in services + let mut local_aliases = services .rooms .alias .local_aliases_for_room(&body.room_id) - .filter_map(Result::ok) - { + .boxed(); + + while let Some(alias) = local_aliases.next().await { services .rooms .alias - .remove_alias(&alias, sender_user) + .remove_alias(alias, sender_user) .await?; + services .rooms .alias - .set_alias(&alias, &replacement_room, sender_user)?; + .set_alias(alias, &replacement_room, sender_user)?; } // Get the old room power levels @@ -787,12 +813,12 @@ pub(crate) async fn upgrade_room_route( services .rooms .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomPowerLevels, "")? - .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))? + .room_state_get(&body.room_id, &StateEventType::RoomPowerLevels, "") + .await + .map_err(|_| err!(Database("Found room without m.room.create event.")))? .content .get(), - ) - .map_err(|_| Error::bad_database("Invalid room event in database."))?; + )?; // Setting events_default and invite to the greater of 50 and users_default + 1 let new_level = max( @@ -800,9 +826,7 @@ pub(crate) async fn upgrade_room_route( power_levels_event_content .users_default .checked_add(int!(1)) - .ok_or_else(|| { - Error::BadRequest(ErrorKind::BadJson, "users_default power levels event content is not valid") - })?, + .ok_or_else(|| err!(Request(BadJson("users_default power levels event content is not valid"))))?, ); power_levels_event_content.events_default = new_level; power_levels_event_content.invite = new_level; @@ -921,8 +945,9 @@ async fn room_alias_check( if services .rooms .alias - .resolve_local_alias(&full_room_alias)? - .is_some() + .resolve_local_alias(&full_room_alias) + .await + .is_ok() { return Err(Error::BadRequest(ErrorKind::RoomInUse, "Room alias already exists.")); } diff --git a/src/api/client/search.rs b/src/api/client/search.rs index b143bd2c..7a061d49 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -1,6 +1,12 @@ use std::collections::BTreeMap; use axum::extract::State; +use conduit::{ + debug, + utils::{IterStream, ReadyExt}, + Err, +}; +use futures::{FutureExt, StreamExt}; use ruma::{ api::client::{ error::ErrorKind, @@ -13,7 +19,6 @@ use ruma::{ serde::Raw, uint, OwnedRoomId, }; -use tracing::debug; use crate::{Error, Result, Ruma}; @@ -32,14 +37,17 @@ pub(crate) async fn search_events_route( let filter = &search_criteria.filter; let include_state = &search_criteria.include_state; - let room_ids = filter.rooms.clone().unwrap_or_else(|| { + let room_ids = if let Some(room_ids) = &filter.rooms { + room_ids.clone() + } else { services .rooms .state_cache .rooms_joined(sender_user) - .filter_map(Result::ok) + .map(ToOwned::to_owned) .collect() - }); + .await + }; // Use limit or else 10, with maximum 100 let limit: usize = filter @@ -53,18 +61,21 @@ pub(crate) async fn search_events_route( if include_state.is_some_and(|include_state| include_state) { for room_id in &room_ids { - if !services.rooms.state_cache.is_joined(sender_user, room_id)? { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this room.", - )); + if !services + .rooms + .state_cache + .is_joined(sender_user, room_id) + .await + { + return Err!(Request(Forbidden("You don't have permission to view this room."))); } // check if sender_user can see state events if services .rooms .state_accessor - .user_can_see_state_events(sender_user, room_id)? + .user_can_see_state_events(sender_user, room_id) + .await { let room_state = services .rooms @@ -87,10 +98,15 @@ pub(crate) async fn search_events_route( } } - let mut searches = Vec::new(); + let mut search_vecs = Vec::new(); for room_id in &room_ids { - if !services.rooms.state_cache.is_joined(sender_user, room_id)? { + if !services + .rooms + .state_cache + .is_joined(sender_user, room_id) + .await + { return Err(Error::BadRequest( ErrorKind::forbidden(), "You don't have permission to view this room.", @@ -100,12 +116,18 @@ pub(crate) async fn search_events_route( if let Some(search) = services .rooms .search - .search_pdus(room_id, &search_criteria.search_term)? + .search_pdus(room_id, &search_criteria.search_term) + .await { - searches.push(search.0.peekable()); + search_vecs.push(search.0); } } + let mut searches: Vec<_> = search_vecs + .iter() + .map(|vec| vec.iter().peekable()) + .collect(); + let skip: usize = match body.next_batch.as_ref().map(|s| s.parse()) { Some(Ok(s)) => s, Some(Err(_)) => return Err(Error::BadRequest(ErrorKind::InvalidParam, "Invalid next_batch token.")), @@ -118,8 +140,8 @@ pub(crate) async fn search_events_route( for _ in 0..next_batch { if let Some(s) = searches .iter_mut() - .map(|s| (s.peek().cloned(), s)) - .max_by_key(|(peek, _)| peek.clone()) + .map(|s| (s.peek().copied(), s)) + .max_by_key(|(peek, _)| *peek) .and_then(|(_, i)| i.next()) { results.push(s); @@ -127,42 +149,38 @@ pub(crate) async fn search_events_route( } let results: Vec<_> = results - .iter() + .into_iter() .skip(skip) - .filter_map(|result| { + .stream() + .filter_map(|id| services.rooms.timeline.get_pdu_from_id(id).map(Result::ok)) + .ready_filter(|pdu| !pdu.is_redacted()) + .filter_map(|pdu| async move { services .rooms - .timeline - .get_pdu_from_id(result) - .ok()? - .filter(|pdu| { - !pdu.is_redacted() - && services - .rooms - .state_accessor - .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) - .unwrap_or(false) - }) - .map(|pdu| pdu.to_room_event()) + .state_accessor + .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) + .await + .then_some(pdu) }) - .map(|result| { - Ok::<_, Error>(SearchResult { - context: EventContextResult { - end: None, - events_after: Vec::new(), - events_before: Vec::new(), - profile_info: BTreeMap::new(), - start: None, - }, - rank: None, - result: Some(result), - }) - }) - .filter_map(Result::ok) .take(limit) - .collect(); + .map(|pdu| pdu.to_room_event()) + .map(|result| SearchResult { + context: EventContextResult { + end: None, + events_after: Vec::new(), + events_before: Vec::new(), + profile_info: BTreeMap::new(), + start: None, + }, + rank: None, + result: Some(result), + }) + .collect() + .boxed() + .await; let more_unloaded_results = searches.iter_mut().any(|s| s.peek().is_some()); + let next_batch = more_unloaded_results.then(|| next_batch.to_string()); Ok(search_events::v3::Response::new(ResultCategories { diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 4702b0ec..6347a2c9 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -1,5 +1,7 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; +use conduit::{debug, err, info, utils::ReadyExt, warn, Err}; +use futures::StreamExt; use ruma::{ api::client::{ error::ErrorKind, @@ -19,7 +21,6 @@ use ruma::{ UserId, }; use serde::Deserialize; -use tracing::{debug, info, warn}; use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; use crate::{utils, utils::hash, Error, Result, Ruma}; @@ -79,21 +80,22 @@ pub(crate) async fn login_route( UserId::parse(user) } else { warn!("Bad login type: {:?}", &body.login_info); - return Err(Error::BadRequest(ErrorKind::forbidden(), "Bad login type.")); + return Err!(Request(Forbidden("Bad login type."))); } .map_err(|_| Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; let hash = services .users - .password_hash(&user_id)? - .ok_or(Error::BadRequest(ErrorKind::forbidden(), "Wrong username or password."))?; + .password_hash(&user_id) + .await + .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; if hash.is_empty() { - return Err(Error::BadRequest(ErrorKind::UserDeactivated, "The user has been deactivated")); + return Err!(Request(UserDeactivated("The user has been deactivated"))); } if hash::verify_password(password, &hash).is_err() { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Wrong username or password.")); + return Err!(Request(Forbidden("Wrong username or password."))); } user_id @@ -112,15 +114,12 @@ pub(crate) async fn login_route( let username = token.claims.sub.to_lowercase(); - UserId::parse_with_server_name(username, services.globals.server_name()).map_err(|e| { - warn!("Failed to parse username from user logging in: {e}"); - Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.") - })? + UserId::parse_with_server_name(username, services.globals.server_name()) + .map_err(|e| err!(Request(InvalidUsername(debug_error!(?e, "Failed to parse login username")))))? } else { - return Err(Error::BadRequest( - ErrorKind::Unknown, - "Token login is not supported (server has no jwt decoding key).", - )); + return Err!(Request(Unknown( + "Token login is not supported (server has no jwt decoding key)." + ))); } }, #[allow(deprecated)] @@ -169,23 +168,32 @@ pub(crate) async fn login_route( let token = utils::random_string(TOKEN_LENGTH); // Determine if device_id was provided and exists in the db for this user - let device_exists = body.device_id.as_ref().map_or(false, |device_id| { + let device_exists = if body.device_id.is_some() { services .users .all_device_ids(&user_id) - .any(|x| x.as_ref().map_or(false, |v| v == device_id)) - }); + .ready_any(|v| v == device_id) + .await + } else { + false + }; if device_exists { - services.users.set_token(&user_id, &device_id, &token)?; + services + .users + .set_token(&user_id, &device_id, &token) + .await?; } else { - services.users.create_device( - &user_id, - &device_id, - &token, - body.initial_device_display_name.clone(), - Some(client.to_string()), - )?; + services + .users + .create_device( + &user_id, + &device_id, + &token, + body.initial_device_display_name.clone(), + Some(client.to_string()), + ) + .await?; } // send client well-known if specified so the client knows to reconfigure itself @@ -228,10 +236,13 @@ pub(crate) async fn logout_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); - services.users.remove_device(sender_user, sender_device)?; + services + .users + .remove_device(sender_user, sender_device) + .await; // send device list update for user after logout - services.users.mark_device_key_update(sender_user)?; + services.users.mark_device_key_update(sender_user).await; Ok(logout::v3::Response::new()) } @@ -256,12 +267,14 @@ pub(crate) async fn logout_all_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - for device_id in services.users.all_device_ids(sender_user).flatten() { - services.users.remove_device(sender_user, &device_id)?; - } + services + .users + .all_device_ids(sender_user) + .for_each(|device_id| services.users.remove_device(sender_user, device_id)) + .await; // send device list update for user after logout - services.users.mark_device_key_update(sender_user)?; + services.users.mark_device_key_update(sender_user).await; Ok(logout_all::v3::Response::new()) } diff --git a/src/api/client/state.rs b/src/api/client/state.rs index fd049663..f9a4a763 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use axum::extract::State; -use conduit::{debug_info, error, pdu::PduBuilder, Error, Result}; +use conduit::{err, error, pdu::PduBuilder, Err, Error, Result}; use ruma::{ api::client::{ error::ErrorKind, @@ -84,12 +84,10 @@ pub(crate) async fn get_state_events_route( if !services .rooms .state_accessor - .user_can_see_state_events(sender_user, &body.room_id)? + .user_can_see_state_events(sender_user, &body.room_id) + .await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view the room state.", - )); + return Err!(Request(Forbidden("You don't have permission to view the room state."))); } Ok(get_state_events::v3::Response { @@ -120,22 +118,25 @@ pub(crate) async fn get_state_events_for_key_route( if !services .rooms .state_accessor - .user_can_see_state_events(sender_user, &body.room_id)? + .user_can_see_state_events(sender_user, &body.room_id) + .await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view the room state.", - )); + return Err!(Request(Forbidden("You don't have permission to view the room state."))); } let event = services .rooms .state_accessor - .room_state_get(&body.room_id, &body.event_type, &body.state_key)? - .ok_or_else(|| { - debug_info!("State event {:?} not found in room {:?}", &body.event_type, &body.room_id); - Error::BadRequest(ErrorKind::NotFound, "State event not found.") + .room_state_get(&body.room_id, &body.event_type, &body.state_key) + .await + .map_err(|_| { + err!(Request(NotFound(error!( + room_id = ?body.room_id, + event_type = ?body.event_type, + "State event not found in room.", + )))) })?; + if body .format .as_ref() @@ -204,7 +205,7 @@ async fn send_state_event_for_key_helper( async fn allowed_to_send_state_event( services: &Services, room_id: &RoomId, event_type: &StateEventType, json: &Raw, -) -> Result<()> { +) -> Result { match event_type { // Forbid m.room.encryption if encryption is disabled StateEventType::RoomEncryption => { @@ -214,7 +215,7 @@ async fn allowed_to_send_state_event( }, // admin room is a sensitive room, it should not ever be made public StateEventType::RoomJoinRules => { - if let Some(admin_room_id) = services.admin.get_admin_room()? { + if let Ok(admin_room_id) = services.admin.get_admin_room().await { if admin_room_id == room_id { if let Ok(join_rule) = serde_json::from_str::(json.json().get()) { if join_rule.join_rule == JoinRule::Public { @@ -229,7 +230,7 @@ async fn allowed_to_send_state_event( }, // admin room is a sensitive room, it should not ever be made world readable StateEventType::RoomHistoryVisibility => { - if let Some(admin_room_id) = services.admin.get_admin_room()? { + if let Ok(admin_room_id) = services.admin.get_admin_room().await { if admin_room_id == room_id { if let Ok(visibility_content) = serde_json::from_str::(json.json().get()) @@ -254,23 +255,27 @@ async fn allowed_to_send_state_event( } for alias in aliases { - if !services.globals.server_is_ours(alias.server_name()) - || services - .rooms - .alias - .resolve_local_alias(&alias)? - .filter(|room| room == room_id) // Make sure it's the right room - .is_none() + if !services.globals.server_is_ours(alias.server_name()) { + return Err!(Request(Forbidden("canonical_alias must be for this server"))); + } + + if !services + .rooms + .alias + .resolve_local_alias(&alias) + .await + .is_ok_and(|room| room == room_id) + // Make sure it's the right room { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You are only allowed to send canonical_alias events when its aliases already exist", - )); + return Err!(Request(Forbidden( + "You are only allowed to send canonical_alias events when its aliases already exist" + ))); } } } }, _ => (), } + Ok(()) } diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index eb534205..53d4f3c3 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -6,10 +6,14 @@ use std::{ use axum::extract::State; use conduit::{ - debug, error, - utils::math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, - warn, Err, PduCount, + debug, err, error, is_equal_to, + utils::{ + math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, + IterStream, ReadyExt, + }, + warn, PduCount, }; +use futures::{pin_mut, StreamExt}; use ruma::{ api::client::{ error::ErrorKind, @@ -108,7 +112,8 @@ pub(crate) async fn sync_events_route( if services.globals.allow_local_presence() { services .presence - .ping_presence(&sender_user, &body.set_presence)?; + .ping_presence(&sender_user, &body.set_presence) + .await?; } // Setup watchers, so if there's no response, we can wait for them @@ -124,7 +129,8 @@ pub(crate) async fn sync_events_route( Some(Filter::FilterDefinition(filter)) => filter, Some(Filter::FilterId(filter_id)) => services .users - .get_filter(&sender_user, &filter_id)? + .get_filter(&sender_user, &filter_id) + .await .unwrap_or_default(), }; @@ -157,7 +163,9 @@ pub(crate) async fn sync_events_route( services .users .keys_changed(sender_user.as_ref(), since, None) - .filter_map(Result::ok), + .map(ToOwned::to_owned) + .collect::>() + .await, ); if services.globals.allow_local_presence() { @@ -168,13 +176,14 @@ pub(crate) async fn sync_events_route( .rooms .state_cache .rooms_joined(&sender_user) - .collect::>(); + .map(ToOwned::to_owned) + .collect::>() + .await; // Coalesce database writes for the remainder of this scope. let _cork = services.db.cork_and_flush(); for room_id in all_joined_rooms { - let room_id = room_id?; if let Ok(joined_room) = load_joined_room( &services, &sender_user, @@ -203,12 +212,14 @@ pub(crate) async fn sync_events_route( .rooms .state_cache .rooms_left(&sender_user) - .collect(); + .collect() + .await; + for result in all_left_rooms { handle_left_room( &services, since, - &result?.0, + &result.0, &sender_user, &mut left_rooms, &next_batch_string, @@ -224,10 +235,10 @@ pub(crate) async fn sync_events_route( .rooms .state_cache .rooms_invited(&sender_user) - .collect(); - for result in all_invited_rooms { - let (room_id, invite_state_events) = result?; + .collect() + .await; + for (room_id, invite_state_events) in all_invited_rooms { // Get and drop the lock to wait for remaining operations to finish let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; drop(insert_lock); @@ -235,7 +246,9 @@ pub(crate) async fn sync_events_route( let invite_count = services .rooms .state_cache - .get_invite_count(&room_id, &sender_user)?; + .get_invite_count(&room_id, &sender_user) + .await + .ok(); // Invited before last sync if Some(since) >= invite_count { @@ -253,22 +266,8 @@ pub(crate) async fn sync_events_route( } for user_id in left_encrypted_users { - let dont_share_encrypted_room = services - .rooms - .user - .get_shared_rooms(vec![sender_user.clone(), user_id.clone()])? - .filter_map(Result::ok) - .filter_map(|other_room_id| { - Some( - services - .rooms - .state_accessor - .room_state_get(&other_room_id, &StateEventType::RoomEncryption, "") - .ok()? - .is_some(), - ) - }) - .all(|encrypted| !encrypted); + let dont_share_encrypted_room = !share_encrypted_room(&services, &sender_user, &user_id, None).await; + // If the user doesn't share an encrypted room with the target anymore, we need // to tell them if dont_share_encrypted_room { @@ -279,7 +278,8 @@ pub(crate) async fn sync_events_route( // Remove all to-device events the device received *last time* services .users - .remove_to_device_events(&sender_user, &sender_device, since)?; + .remove_to_device_events(&sender_user, &sender_device, since) + .await; let response = sync_events::v3::Response { next_batch: next_batch_string, @@ -298,7 +298,8 @@ pub(crate) async fn sync_events_route( account_data: GlobalAccountData { events: services .account_data - .changes_since(None, &sender_user, since)? + .changes_since(None, &sender_user, since) + .await? .into_iter() .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) .collect(), @@ -309,11 +310,14 @@ pub(crate) async fn sync_events_route( }, device_one_time_keys_count: services .users - .count_one_time_keys(&sender_user, &sender_device)?, + .count_one_time_keys(&sender_user, &sender_device) + .await, to_device: ToDevice { events: services .users - .get_to_device_events(&sender_user, &sender_device)?, + .get_to_device_events(&sender_user, &sender_device) + .collect() + .await, }, // Fallback keys are not yet supported device_unused_fallback_key_types: None, @@ -351,14 +355,16 @@ async fn handle_left_room( let left_count = services .rooms .state_cache - .get_left_count(room_id, sender_user)?; + .get_left_count(room_id, sender_user) + .await + .ok(); // Left before last sync if Some(since) >= left_count { return Ok(()); } - if !services.rooms.metadata.exists(room_id)? { + if !services.rooms.metadata.exists(room_id).await { // This is just a rejected invite, not a room we know // Insert a leave event anyways let event = PduEvent { @@ -408,27 +414,29 @@ async fn handle_left_room( let since_shortstatehash = services .rooms .user - .get_token_shortstatehash(room_id, since)?; + .get_token_shortstatehash(room_id, since) + .await; let since_state_ids = match since_shortstatehash { - Some(s) => services.rooms.state_accessor.state_full_ids(s).await?, - None => HashMap::new(), + Ok(s) => services.rooms.state_accessor.state_full_ids(s).await?, + Err(_) => HashMap::new(), }; - let Some(left_event_id) = - services - .rooms - .state_accessor - .room_state_get_id(room_id, &StateEventType::RoomMember, sender_user.as_str())? + let Ok(left_event_id) = services + .rooms + .state_accessor + .room_state_get_id(room_id, &StateEventType::RoomMember, sender_user.as_str()) + .await else { error!("Left room but no left state event"); return Ok(()); }; - let Some(left_shortstatehash) = services + let Ok(left_shortstatehash) = services .rooms .state_accessor - .pdu_shortstatehash(&left_event_id)? + .pdu_shortstatehash(&left_event_id) + .await else { error!(event_id = %left_event_id, "Leave event has no state"); return Ok(()); @@ -443,14 +451,15 @@ async fn handle_left_room( let leave_shortstatekey = services .rooms .short - .get_or_create_shortstatekey(&StateEventType::RoomMember, sender_user.as_str())?; + .get_or_create_shortstatekey(&StateEventType::RoomMember, sender_user.as_str()) + .await; left_state_ids.insert(leave_shortstatekey, left_event_id); let mut i: u8 = 0; for (key, id) in left_state_ids { if full_state || since_state_ids.get(&key) != Some(&id) { - let (event_type, state_key) = services.rooms.short.get_statekey_from_short(key)?; + let (event_type, state_key) = services.rooms.short.get_statekey_from_short(key).await?; if !lazy_load_enabled || event_type != StateEventType::RoomMember @@ -458,7 +467,7 @@ async fn handle_left_room( // TODO: Delete the following line when this is resolved: https://github.com/vector-im/element-web/issues/22565 || (cfg!(feature = "element_hacks") && *sender_user == state_key) { - let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { + let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { error!("Pdu in state not found: {}", id); continue; }; @@ -495,19 +504,25 @@ async fn handle_left_room( async fn process_presence_updates( services: &Services, presence_updates: &mut HashMap, since: u64, syncing_user: &UserId, ) -> Result<()> { + let presence_since = services.presence.presence_since(since); + // Take presence updates - for (user_id, _, presence_bytes) in services.presence.presence_since(since) { + pin_mut!(presence_since); + while let Some((user_id, _, presence_bytes)) = presence_since.next().await { if !services .rooms .state_cache - .user_sees_user(syncing_user, &user_id)? + .user_sees_user(syncing_user, &user_id) + .await { continue; } let presence_event = services .presence - .from_json_bytes_to_event(&presence_bytes, &user_id)?; + .from_json_bytes_to_event(&presence_bytes, &user_id) + .await?; + match presence_updates.entry(user_id) { Entry::Vacant(slot) => { slot.insert(presence_event); @@ -551,14 +566,14 @@ async fn load_joined_room( let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); - let (timeline_pdus, limited) = load_timeline(services, sender_user, room_id, sincecount, 10)?; + let (timeline_pdus, limited) = load_timeline(services, sender_user, room_id, sincecount, 10).await?; let send_notification_counts = !timeline_pdus.is_empty() || services .rooms .user - .last_notification_read(sender_user, room_id)? - > since; + .last_notification_read(sender_user, room_id) + .await > since; let mut timeline_users = HashSet::new(); for (_, event) in &timeline_pdus { @@ -568,355 +583,384 @@ async fn load_joined_room( services .rooms .lazy_loading - .lazy_load_confirm_delivery(sender_user, sender_device, room_id, sincecount) - .await?; + .lazy_load_confirm_delivery(sender_user, sender_device, room_id, sincecount); // Database queries: - let Some(current_shortstatehash) = services.rooms.state.get_room_shortstatehash(room_id)? else { - return Err!(Database(error!("Room {room_id} has no state"))); - }; + let current_shortstatehash = services + .rooms + .state + .get_room_shortstatehash(room_id) + .await + .map_err(|_| err!(Database(error!("Room {room_id} has no state"))))?; let since_shortstatehash = services .rooms .user - .get_token_shortstatehash(room_id, since)?; + .get_token_shortstatehash(room_id, since) + .await + .ok(); - let (heroes, joined_member_count, invited_member_count, joined_since_last_sync, state_events) = - if timeline_pdus.is_empty() && since_shortstatehash == Some(current_shortstatehash) { - // No state changes - (Vec::new(), None, None, false, Vec::new()) - } else { - // Calculates joined_member_count, invited_member_count and heroes - let calculate_counts = || { - let joined_member_count = services - .rooms - .state_cache - .room_joined_count(room_id)? - .unwrap_or(0); - let invited_member_count = services - .rooms - .state_cache - .room_invited_count(room_id)? - .unwrap_or(0); + let (heroes, joined_member_count, invited_member_count, joined_since_last_sync, state_events) = if timeline_pdus + .is_empty() + && (since_shortstatehash.is_none() || since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash))) + { + // No state changes + (Vec::new(), None, None, false, Vec::new()) + } else { + // Calculates joined_member_count, invited_member_count and heroes + let calculate_counts = || async { + let joined_member_count = services + .rooms + .state_cache + .room_joined_count(room_id) + .await + .unwrap_or(0); - // Recalculate heroes (first 5 members) - let mut heroes: Vec = Vec::with_capacity(5); + let invited_member_count = services + .rooms + .state_cache + .room_invited_count(room_id) + .await + .unwrap_or(0); - if joined_member_count.saturating_add(invited_member_count) <= 5 { - // Go through all PDUs and for each member event, check if the user is still - // joined or invited until we have 5 or we reach the end + if joined_member_count.saturating_add(invited_member_count) > 5 { + return Ok::<_, Error>((Some(joined_member_count), Some(invited_member_count), Vec::new())); + } - for hero in services - .rooms - .timeline - .all_pdus(sender_user, room_id)? - .filter_map(Result::ok) // Ignore all broken pdus - .filter(|(_, pdu)| pdu.kind == TimelineEventType::RoomMember) - .map(|(_, pdu)| { - let content: RoomMemberEventContent = serde_json::from_str(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid member event in database."))?; + // Go through all PDUs and for each member event, check if the user is still + // joined or invited until we have 5 or we reach the end - if let Some(state_key) = &pdu.state_key { - let user_id = UserId::parse(state_key.clone()) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; + // Recalculate heroes (first 5 members) + let heroes = services + .rooms + .timeline + .all_pdus(sender_user, room_id) + .await? + .ready_filter(|(_, pdu)| pdu.kind == TimelineEventType::RoomMember) + .filter_map(|(_, pdu)| async move { + let Ok(content) = serde_json::from_str::(pdu.content.get()) else { + return None; + }; - // The membership was and still is invite or join - if matches!(content.membership, MembershipState::Join | MembershipState::Invite) - && (services.rooms.state_cache.is_joined(&user_id, room_id)? - || services.rooms.state_cache.is_invited(&user_id, room_id)?) - { - Ok::<_, Error>(Some(user_id)) - } else { - Ok(None) - } - } else { - Ok(None) - } - }) - .filter_map(Result::ok) - // Filter for possible heroes - .flatten() + let Some(state_key) = &pdu.state_key else { + return None; + }; + + let Ok(user_id) = UserId::parse(state_key) else { + return None; + }; + + if user_id == sender_user { + return None; + } + + // The membership was and still is invite or join + if !matches!(content.membership, MembershipState::Join | MembershipState::Invite) { + return None; + } + + if !services + .rooms + .state_cache + .is_joined(&user_id, room_id) + .await && services + .rooms + .state_cache + .is_invited(&user_id, room_id) + .await { - if heroes.contains(&hero) || hero == sender_user { - continue; - } + return None; + } - heroes.push(hero); + Some(user_id) + }) + .collect::>() + .await; + + Ok::<_, Error>(( + Some(joined_member_count), + Some(invited_member_count), + heroes.into_iter().collect::>(), + )) + }; + + let since_sender_member: Option = if let Some(short) = since_shortstatehash { + services + .rooms + .state_accessor + .state_get(short, &StateEventType::RoomMember, sender_user.as_str()) + .await + .and_then(|pdu| serde_json::from_str(pdu.content.get()).map_err(Into::into)) + .ok() + } else { + None + }; + + let joined_since_last_sync = + since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + + if since_shortstatehash.is_none() || joined_since_last_sync { + // Probably since = 0, we will do an initial sync + + let (joined_member_count, invited_member_count, heroes) = calculate_counts().await?; + + let current_state_ids = services + .rooms + .state_accessor + .state_full_ids(current_shortstatehash) + .await?; + + let mut state_events = Vec::new(); + let mut lazy_loaded = HashSet::new(); + + let mut i: u8 = 0; + for (shortstatekey, id) in current_state_ids { + let (event_type, state_key) = services + .rooms + .short + .get_statekey_from_short(shortstatekey) + .await?; + + if event_type != StateEventType::RoomMember { + let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { + error!("Pdu in state not found: {id}"); + continue; + }; + state_events.push(pdu); + + i = i.wrapping_add(1); + if i % 100 == 0 { + tokio::task::yield_now().await; + } + } else if !lazy_load_enabled + || full_state + || timeline_users.contains(&state_key) + // TODO: Delete the following line when this is resolved: https://github.com/vector-im/element-web/issues/22565 + || (cfg!(feature = "element_hacks") && *sender_user == state_key) + { + let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { + error!("Pdu in state not found: {id}"); + continue; + }; + + // This check is in case a bad user ID made it into the database + if let Ok(uid) = UserId::parse(&state_key) { + lazy_loaded.insert(uid); + } + state_events.push(pdu); + + i = i.wrapping_add(1); + if i % 100 == 0 { + tokio::task::yield_now().await; } } + } - Ok::<_, Error>((Some(joined_member_count), Some(invited_member_count), heroes)) - }; + // Reset lazy loading because this is an initial sync + services + .rooms + .lazy_loading + .lazy_load_reset(sender_user, sender_device, room_id) + .await; - let since_sender_member: Option = since_shortstatehash - .and_then(|shortstatehash| { - services - .rooms - .state_accessor - .state_get(shortstatehash, &StateEventType::RoomMember, sender_user.as_str()) - .transpose() - }) - .transpose()? - .and_then(|pdu| { - serde_json::from_str(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() - }); + // The state_events above should contain all timeline_users, let's mark them as + // lazy loaded. + services.rooms.lazy_loading.lazy_load_mark_sent( + sender_user, + sender_device, + room_id, + lazy_loaded, + next_batchcount, + ); - let joined_since_last_sync = - since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + (heroes, joined_member_count, invited_member_count, true, state_events) + } else { + // Incremental /sync + let since_shortstatehash = since_shortstatehash.expect("missing since_shortstatehash on incremental sync"); - if since_shortstatehash.is_none() || joined_since_last_sync { - // Probably since = 0, we will do an initial sync - - let (joined_member_count, invited_member_count, heroes) = calculate_counts()?; + let mut delta_state_events = Vec::new(); + if since_shortstatehash != current_shortstatehash { let current_state_ids = services .rooms .state_accessor .state_full_ids(current_shortstatehash) .await?; - let mut state_events = Vec::new(); - let mut lazy_loaded = HashSet::new(); - - let mut i: u8 = 0; - for (shortstatekey, id) in current_state_ids { - let (event_type, state_key) = services - .rooms - .short - .get_statekey_from_short(shortstatekey)?; - - if event_type != StateEventType::RoomMember { - let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { - error!("Pdu in state not found: {}", id); - continue; - }; - state_events.push(pdu); - - i = i.wrapping_add(1); - if i % 100 == 0 { - tokio::task::yield_now().await; - } - } else if !lazy_load_enabled - || full_state - || timeline_users.contains(&state_key) - // TODO: Delete the following line when this is resolved: https://github.com/vector-im/element-web/issues/22565 - || (cfg!(feature = "element_hacks") && *sender_user == state_key) - { - let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { - error!("Pdu in state not found: {}", id); - continue; - }; - - // This check is in case a bad user ID made it into the database - if let Ok(uid) = UserId::parse(&state_key) { - lazy_loaded.insert(uid); - } - state_events.push(pdu); - - i = i.wrapping_add(1); - if i % 100 == 0 { - tokio::task::yield_now().await; - } - } - } - - // Reset lazy loading because this is an initial sync - services - .rooms - .lazy_loading - .lazy_load_reset(sender_user, sender_device, room_id)?; - - // The state_events above should contain all timeline_users, let's mark them as - // lazy loaded. - services - .rooms - .lazy_loading - .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy_loaded, next_batchcount) - .await; - - (heroes, joined_member_count, invited_member_count, true, state_events) - } else { - // Incremental /sync - let since_shortstatehash = since_shortstatehash.unwrap(); - - let mut delta_state_events = Vec::new(); - - if since_shortstatehash != current_shortstatehash { - let current_state_ids = services - .rooms - .state_accessor - .state_full_ids(current_shortstatehash) - .await?; - let since_state_ids = services - .rooms - .state_accessor - .state_full_ids(since_shortstatehash) - .await?; - - for (key, id) in current_state_ids { - if full_state || since_state_ids.get(&key) != Some(&id) { - let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { - error!("Pdu in state not found: {}", id); - continue; - }; - - delta_state_events.push(pdu); - tokio::task::yield_now().await; - } - } - } - - let encrypted_room = services + let since_state_ids = services .rooms .state_accessor - .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "")? - .is_some(); + .state_full_ids(since_shortstatehash) + .await?; - let since_encryption = services.rooms.state_accessor.state_get( - since_shortstatehash, - &StateEventType::RoomEncryption, - "", - )?; - - // Calculations: - let new_encrypted_room = encrypted_room && since_encryption.is_none(); - - let send_member_count = delta_state_events - .iter() - .any(|event| event.kind == TimelineEventType::RoomMember); - - if encrypted_room { - for state_event in &delta_state_events { - if state_event.kind != TimelineEventType::RoomMember { + for (key, id) in current_state_ids { + if full_state || since_state_ids.get(&key) != Some(&id) { + let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { + error!("Pdu in state not found: {id}"); continue; - } + }; - if let Some(state_key) = &state_event.state_key { - let user_id = UserId::parse(state_key.clone()) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; - - if user_id == sender_user { - continue; - } - - let new_membership = - serde_json::from_str::(state_event.content.get()) - .map_err(|_| Error::bad_database("Invalid PDU in database."))? - .membership; - - match new_membership { - MembershipState::Join => { - // A new user joined an encrypted room - if !share_encrypted_room(services, sender_user, &user_id, room_id)? { - device_list_updates.insert(user_id); - } - }, - MembershipState::Leave => { - // Write down users that have left encrypted rooms we are in - left_encrypted_users.insert(user_id); - }, - _ => {}, - } - } + delta_state_events.push(pdu); + tokio::task::yield_now().await; } } + } - if joined_since_last_sync && encrypted_room || new_encrypted_room { - // If the user is in a new encrypted room, give them all joined users - device_list_updates.extend( - services - .rooms - .state_cache - .room_members(room_id) - .flatten() - .filter(|user_id| { - // Don't send key updates from the sender to the sender - sender_user != user_id - }) - .filter(|user_id| { - // Only send keys if the sender doesn't share an encrypted room with the target - // already - !share_encrypted_room(services, sender_user, user_id, room_id).unwrap_or(false) - }), - ); - } + let encrypted_room = services + .rooms + .state_accessor + .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") + .await + .is_ok(); - let (joined_member_count, invited_member_count, heroes) = if send_member_count { - calculate_counts()? - } else { - (None, None, Vec::new()) - }; + let since_encryption = services + .rooms + .state_accessor + .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") + .await; - let mut state_events = delta_state_events; - let mut lazy_loaded = HashSet::new(); + // Calculations: + let new_encrypted_room = encrypted_room && since_encryption.is_err(); - // Mark all member events we're returning as lazy-loaded - for pdu in &state_events { - if pdu.kind == TimelineEventType::RoomMember { - match UserId::parse( - pdu.state_key - .as_ref() - .expect("State event has state key") - .clone(), - ) { - Ok(state_key_userid) => { - lazy_loaded.insert(state_key_userid); - }, - Err(e) => error!("Invalid state key for member event: {}", e), - } - } - } + let send_member_count = delta_state_events + .iter() + .any(|event| event.kind == TimelineEventType::RoomMember); - // Fetch contextual member state events for events from the timeline, and - // mark them as lazy-loaded as well. - for (_, event) in &timeline_pdus { - if lazy_loaded.contains(&event.sender) { + if encrypted_room { + for state_event in &delta_state_events { + if state_event.kind != TimelineEventType::RoomMember { continue; } - if !services.rooms.lazy_loading.lazy_load_was_sent_before( - sender_user, - sender_device, - room_id, - &event.sender, - )? || lazy_load_send_redundant - { - if let Some(member_event) = services.rooms.state_accessor.room_state_get( - room_id, - &StateEventType::RoomMember, - event.sender.as_str(), - )? { - lazy_loaded.insert(event.sender.clone()); - state_events.push(member_event); + if let Some(state_key) = &state_event.state_key { + let user_id = UserId::parse(state_key.clone()) + .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; + + if user_id == sender_user { + continue; + } + + let new_membership = serde_json::from_str::(state_event.content.get()) + .map_err(|_| Error::bad_database("Invalid PDU in database."))? + .membership; + + match new_membership { + MembershipState::Join => { + // A new user joined an encrypted room + if !share_encrypted_room(services, sender_user, &user_id, Some(room_id)).await { + device_list_updates.insert(user_id); + } + }, + MembershipState::Leave => { + // Write down users that have left encrypted rooms we are in + left_encrypted_users.insert(user_id); + }, + _ => {}, } } } + } - services + if joined_since_last_sync && encrypted_room || new_encrypted_room { + // If the user is in a new encrypted room, give them all joined users + device_list_updates.extend( + services + .rooms + .state_cache + .room_members(room_id) + .ready_filter(|user_id| { + // Don't send key updates from the sender to the sender + sender_user != *user_id + }) + .filter_map(|user_id| async move { + // Only send keys if the sender doesn't share an encrypted room with the target + // already + (!share_encrypted_room(services, sender_user, user_id, Some(room_id)).await) + .then_some(user_id.to_owned()) + }) + .collect::>() + .await, + ); + } + + let (joined_member_count, invited_member_count, heroes) = if send_member_count { + calculate_counts().await? + } else { + (None, None, Vec::new()) + }; + + let mut state_events = delta_state_events; + let mut lazy_loaded = HashSet::new(); + + // Mark all member events we're returning as lazy-loaded + for pdu in &state_events { + if pdu.kind == TimelineEventType::RoomMember { + match UserId::parse( + pdu.state_key + .as_ref() + .expect("State event has state key") + .clone(), + ) { + Ok(state_key_userid) => { + lazy_loaded.insert(state_key_userid); + }, + Err(e) => error!("Invalid state key for member event: {}", e), + } + } + } + + // Fetch contextual member state events for events from the timeline, and + // mark them as lazy-loaded as well. + for (_, event) in &timeline_pdus { + if lazy_loaded.contains(&event.sender) { + continue; + } + + if !services .rooms .lazy_loading - .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy_loaded, next_batchcount) - .await; - - ( - heroes, - joined_member_count, - invited_member_count, - joined_since_last_sync, - state_events, - ) + .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) + .await || lazy_load_send_redundant + { + if let Ok(member_event) = services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomMember, event.sender.as_str()) + .await + { + lazy_loaded.insert(event.sender.clone()); + state_events.push(member_event); + } + } } - }; + + services.rooms.lazy_loading.lazy_load_mark_sent( + sender_user, + sender_device, + room_id, + lazy_loaded, + next_batchcount, + ); + + ( + heroes, + joined_member_count, + invited_member_count, + joined_since_last_sync, + state_events, + ) + } + }; // Look for device list updates in this room device_list_updates.extend( services .users .keys_changed(room_id.as_ref(), since, None) - .filter_map(Result::ok), + .map(ToOwned::to_owned) + .collect::>() + .await, ); let notification_count = if send_notification_counts { @@ -924,7 +968,8 @@ async fn load_joined_room( services .rooms .user - .notification_count(sender_user, room_id)? + .notification_count(sender_user, room_id) + .await .try_into() .expect("notification count can't go that high"), ) @@ -937,7 +982,8 @@ async fn load_joined_room( services .rooms .user - .highlight_count(sender_user, room_id)? + .highlight_count(sender_user, room_id) + .await .try_into() .expect("highlight count can't go that high"), ) @@ -966,9 +1012,9 @@ async fn load_joined_room( .rooms .read_receipt .readreceipts_since(room_id, since) - .filter_map(Result::ok) // Filter out buggy events .map(|(_, _, v)| v) - .collect(); + .collect() + .await; if services.rooms.typing.last_typing_update(room_id).await? > since { edus.push( @@ -985,13 +1031,15 @@ async fn load_joined_room( services .rooms .user - .associate_token_shortstatehash(room_id, next_batch, current_shortstatehash)?; + .associate_token_shortstatehash(room_id, next_batch, current_shortstatehash) + .await; Ok(JoinedRoom { account_data: RoomAccountData { events: services .account_data - .changes_since(Some(room_id), sender_user, since)? + .changes_since(Some(room_id), sender_user, since) + .await? .into_iter() .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) .collect(), @@ -1023,41 +1071,37 @@ async fn load_joined_room( }) } -fn load_timeline( +async fn load_timeline( services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: u64, ) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> { let timeline_pdus; let limited = if services .rooms .timeline - .last_timeline_count(sender_user, room_id)? + .last_timeline_count(sender_user, room_id) + .await? > roomsincecount { let mut non_timeline_pdus = services .rooms .timeline - .pdus_until(sender_user, room_id, PduCount::max())? - .filter_map(|r| { - // Filter out buggy events - if r.is_err() { - error!("Bad pdu in pdus_since: {:?}", r); - } - r.ok() - }) - .take_while(|(pducount, _)| pducount > &roomsincecount); + .pdus_until(sender_user, room_id, PduCount::max()) + .await? + .ready_take_while(|(pducount, _)| pducount > &roomsincecount); // Take the last events for the timeline timeline_pdus = non_timeline_pdus .by_ref() .take(usize_from_u64_truncated(limit)) .collect::>() + .await .into_iter() .rev() .collect::>(); // They /sync response doesn't always return all messages, so we say the output // is limited unless there are events in non_timeline_pdus - non_timeline_pdus.next().is_some() + non_timeline_pdus.next().await.is_some() } else { timeline_pdus = Vec::new(); false @@ -1065,26 +1109,23 @@ fn load_timeline( Ok((timeline_pdus, limited)) } -fn share_encrypted_room( - services: &Services, sender_user: &UserId, user_id: &UserId, ignore_room: &RoomId, -) -> Result { - Ok(services +async fn share_encrypted_room( + services: &Services, sender_user: &UserId, user_id: &UserId, ignore_room: Option<&RoomId>, +) -> bool { + services .rooms .user - .get_shared_rooms(vec![sender_user.to_owned(), user_id.to_owned()])? - .filter_map(Result::ok) - .filter(|room_id| room_id != ignore_room) - .filter_map(|other_room_id| { - Some( - services - .rooms - .state_accessor - .room_state_get(&other_room_id, &StateEventType::RoomEncryption, "") - .ok()? - .is_some(), - ) + .get_shared_rooms(sender_user, user_id) + .ready_filter(|&room_id| Some(room_id) != ignore_room) + .any(|other_room_id| async move { + services + .rooms + .state_accessor + .room_state_get(other_room_id, &StateEventType::RoomEncryption, "") + .await + .is_ok() }) - .any(|encrypted| encrypted)) + .await } /// POST `/_matrix/client/unstable/org.matrix.msc3575/sync` @@ -1114,7 +1155,7 @@ pub(crate) async fn sync_events_v4_route( if globalsince != 0 && !services - .users + .sync .remembered(sender_user.clone(), sender_device.clone(), conn_id.clone()) { debug!("Restarting sync stream because it was gone from the database"); @@ -1127,41 +1168,43 @@ pub(crate) async fn sync_events_v4_route( if globalsince == 0 { services - .users + .sync .forget_sync_request_connection(sender_user.clone(), sender_device.clone(), conn_id.clone()); } // Get sticky parameters from cache let known_rooms = services - .users + .sync .update_sync_request_with_cache(sender_user.clone(), sender_device.clone(), &mut body); let all_joined_rooms = services .rooms .state_cache .rooms_joined(&sender_user) - .filter_map(Result::ok) - .collect::>(); + .map(ToOwned::to_owned) + .collect::>() + .await; let all_invited_rooms = services .rooms .state_cache .rooms_invited(&sender_user) - .filter_map(Result::ok) .map(|r| r.0) - .collect::>(); + .collect::>() + .await; let all_rooms = all_joined_rooms .iter() - .cloned() - .chain(all_invited_rooms.clone()) + .chain(all_invited_rooms.iter()) + .map(Clone::clone) .collect(); if body.extensions.to_device.enabled.unwrap_or(false) { services .users - .remove_to_device_events(&sender_user, &sender_device, globalsince)?; + .remove_to_device_events(&sender_user, &sender_device, globalsince) + .await; } let mut left_encrypted_users = HashSet::new(); // Users that have left any encrypted rooms the sender was in @@ -1179,7 +1222,8 @@ pub(crate) async fn sync_events_v4_route( if body.extensions.account_data.enabled.unwrap_or(false) { account_data.global = services .account_data - .changes_since(None, &sender_user, globalsince)? + .changes_since(None, &sender_user, globalsince) + .await? .into_iter() .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) .collect(); @@ -1190,7 +1234,8 @@ pub(crate) async fn sync_events_v4_route( room.clone(), services .account_data - .changes_since(Some(&room), &sender_user, globalsince)? + .changes_since(Some(&room), &sender_user, globalsince) + .await? .into_iter() .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) .collect(), @@ -1205,40 +1250,42 @@ pub(crate) async fn sync_events_v4_route( services .users .keys_changed(sender_user.as_ref(), globalsince, None) - .filter_map(Result::ok), + .map(ToOwned::to_owned) + .collect::>() + .await, ); for room_id in &all_joined_rooms { - let Some(current_shortstatehash) = services.rooms.state.get_room_shortstatehash(room_id)? else { - error!("Room {} has no state", room_id); + let Ok(current_shortstatehash) = services.rooms.state.get_room_shortstatehash(room_id).await else { + error!("Room {room_id} has no state"); continue; }; let since_shortstatehash = services .rooms .user - .get_token_shortstatehash(room_id, globalsince)?; + .get_token_shortstatehash(room_id, globalsince) + .await + .ok(); - let since_sender_member: Option = since_shortstatehash - .and_then(|shortstatehash| { - services - .rooms - .state_accessor - .state_get(shortstatehash, &StateEventType::RoomMember, sender_user.as_str()) - .transpose() - }) - .transpose()? - .and_then(|pdu| { - serde_json::from_str(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid PDU in database.")) - .ok() - }); + let since_sender_member: Option = if let Some(short) = since_shortstatehash { + services + .rooms + .state_accessor + .state_get(short, &StateEventType::RoomMember, sender_user.as_str()) + .await + .and_then(|pdu| serde_json::from_str(pdu.content.get()).map_err(Into::into)) + .ok() + } else { + None + }; let encrypted_room = services .rooms .state_accessor - .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "")? - .is_some(); + .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") + .await + .is_ok(); if let Some(since_shortstatehash) = since_shortstatehash { // Skip if there are only timeline changes @@ -1246,22 +1293,24 @@ pub(crate) async fn sync_events_v4_route( continue; } - let since_encryption = services.rooms.state_accessor.state_get( - since_shortstatehash, - &StateEventType::RoomEncryption, - "", - )?; + let since_encryption = services + .rooms + .state_accessor + .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") + .await; let joined_since_last_sync = since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); - let new_encrypted_room = encrypted_room && since_encryption.is_none(); + let new_encrypted_room = encrypted_room && since_encryption.is_err(); + if encrypted_room { let current_state_ids = services .rooms .state_accessor .state_full_ids(current_shortstatehash) .await?; + let since_state_ids = services .rooms .state_accessor @@ -1270,8 +1319,8 @@ pub(crate) async fn sync_events_v4_route( for (key, id) in current_state_ids { if since_state_ids.get(&key) != Some(&id) { - let Some(pdu) = services.rooms.timeline.get_pdu(&id)? else { - error!("Pdu in state not found: {}", id); + let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { + error!("Pdu in state not found: {id}"); continue; }; if pdu.kind == TimelineEventType::RoomMember { @@ -1291,7 +1340,9 @@ pub(crate) async fn sync_events_v4_route( match new_membership { MembershipState::Join => { // A new user joined an encrypted room - if !share_encrypted_room(&services, &sender_user, &user_id, room_id)? { + if !share_encrypted_room(&services, &sender_user, &user_id, Some(room_id)) + .await + { device_list_changes.insert(user_id); } }, @@ -1306,22 +1357,25 @@ pub(crate) async fn sync_events_v4_route( } } if joined_since_last_sync || new_encrypted_room { + let sender_user = &sender_user; // If the user is in a new encrypted room, give them all joined users device_list_changes.extend( services .rooms .state_cache .room_members(room_id) - .flatten() - .filter(|user_id| { + .ready_filter(|user_id| { // Don't send key updates from the sender to the sender - &sender_user != user_id + sender_user != user_id }) - .filter(|user_id| { + .filter_map(|user_id| async move { // Only send keys if the sender doesn't share an encrypted room with the target // already - !share_encrypted_room(&services, &sender_user, user_id, room_id).unwrap_or(false) - }), + (!share_encrypted_room(&services, sender_user, user_id, Some(room_id)).await) + .then_some(user_id.to_owned()) + }) + .collect::>() + .await, ); } } @@ -1331,26 +1385,15 @@ pub(crate) async fn sync_events_v4_route( services .users .keys_changed(room_id.as_ref(), globalsince, None) - .filter_map(Result::ok), + .map(ToOwned::to_owned) + .collect::>() + .await, ); } + for user_id in left_encrypted_users { - let dont_share_encrypted_room = services - .rooms - .user - .get_shared_rooms(vec![sender_user.clone(), user_id.clone()])? - .filter_map(Result::ok) - .filter_map(|other_room_id| { - Some( - services - .rooms - .state_accessor - .room_state_get(&other_room_id, &StateEventType::RoomEncryption, "") - .ok()? - .is_some(), - ) - }) - .all(|encrypted| !encrypted); + let dont_share_encrypted_room = !share_encrypted_room(&services, &sender_user, &user_id, None).await; + // If the user doesn't share an encrypted room with the target anymore, we need // to tell them if dont_share_encrypted_room { @@ -1362,7 +1405,7 @@ pub(crate) async fn sync_events_v4_route( let mut lists = BTreeMap::new(); let mut todo_rooms = BTreeMap::new(); // and required state - for (list_id, list) in body.lists { + for (list_id, list) in &body.lists { let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) { Some(true) => &all_invited_rooms, Some(false) => &all_joined_rooms, @@ -1371,23 +1414,23 @@ pub(crate) async fn sync_events_v4_route( let active_rooms = match list.filters.clone().map(|f| f.not_room_types) { Some(filter) if filter.is_empty() => active_rooms.clone(), - Some(value) => filter_rooms(active_rooms, State(services), &value, true), + Some(value) => filter_rooms(active_rooms, State(services), &value, true).await, None => active_rooms.clone(), }; let active_rooms = match list.filters.clone().map(|f| f.room_types) { Some(filter) if filter.is_empty() => active_rooms.clone(), - Some(value) => filter_rooms(&active_rooms, State(services), &value, false), + Some(value) => filter_rooms(&active_rooms, State(services), &value, false).await, None => active_rooms, }; let mut new_known_rooms = BTreeSet::new(); + let ranges = list.ranges.clone(); lists.insert( list_id.clone(), sync_events::v4::SyncList { - ops: list - .ranges + ops: ranges .into_iter() .map(|mut r| { r.0 = r.0.clamp( @@ -1396,29 +1439,34 @@ pub(crate) async fn sync_events_v4_route( ); r.1 = r.1.clamp(r.0, UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX)); + let room_ids = if !active_rooms.is_empty() { active_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec() } else { Vec::new() }; + new_known_rooms.extend(room_ids.iter().cloned()); for room_id in &room_ids { let todo_room = todo_rooms .entry(room_id.clone()) .or_insert((BTreeSet::new(), 0, u64::MAX)); + let limit = list .room_details .timeline_limit .map_or(10, u64::from) .min(100); + todo_room .0 .extend(list.room_details.required_state.iter().cloned()); + todo_room.1 = todo_room.1.max(limit); // 0 means unknown because it got out of date todo_room.2 = todo_room.2.min( known_rooms - .get(&list_id) + .get(list_id.as_str()) .and_then(|k| k.get(room_id)) .copied() .unwrap_or(0), @@ -1438,11 +1486,11 @@ pub(crate) async fn sync_events_v4_route( ); if let Some(conn_id) = &body.conn_id { - services.users.update_sync_known_rooms( + services.sync.update_sync_known_rooms( sender_user.clone(), sender_device.clone(), conn_id.clone(), - list_id, + list_id.clone(), new_known_rooms, globalsince, ); @@ -1451,7 +1499,7 @@ pub(crate) async fn sync_events_v4_route( let mut known_subscription_rooms = BTreeSet::new(); for (room_id, room) in &body.room_subscriptions { - if !services.rooms.metadata.exists(room_id)? { + if !services.rooms.metadata.exists(room_id).await { continue; } let todo_room = todo_rooms @@ -1477,7 +1525,7 @@ pub(crate) async fn sync_events_v4_route( } if let Some(conn_id) = &body.conn_id { - services.users.update_sync_known_rooms( + services.sync.update_sync_known_rooms( sender_user.clone(), sender_device.clone(), conn_id.clone(), @@ -1488,7 +1536,7 @@ pub(crate) async fn sync_events_v4_route( } if let Some(conn_id) = &body.conn_id { - services.users.update_sync_subscriptions( + services.sync.update_sync_subscriptions( sender_user.clone(), sender_device.clone(), conn_id.clone(), @@ -1509,12 +1557,13 @@ pub(crate) async fn sync_events_v4_route( .rooms .state_cache .invite_state(&sender_user, room_id) - .unwrap_or(None); + .await + .ok(); (timeline_pdus, limited) = (Vec::new(), true); } else { (timeline_pdus, limited) = - match load_timeline(&services, &sender_user, room_id, roomsincecount, *timeline_limit) { + match load_timeline(&services, &sender_user, room_id, roomsincecount, *timeline_limit).await { Ok(value) => value, Err(err) => { warn!("Encountered missing timeline in {}, error {}", room_id, err); @@ -1527,17 +1576,20 @@ pub(crate) async fn sync_events_v4_route( room_id.clone(), services .account_data - .changes_since(Some(room_id), &sender_user, *roomsince)? + .changes_since(Some(room_id), &sender_user, *roomsince) + .await? .into_iter() .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) .collect(), ); - let room_receipts = services + let vector: Vec<_> = services .rooms .read_receipt - .readreceipts_since(room_id, *roomsince); - let vector: Vec<_> = room_receipts.into_iter().collect(); + .readreceipts_since(room_id, *roomsince) + .collect() + .await; + let receipt_size = vector.len(); receipts .rooms @@ -1584,41 +1636,42 @@ pub(crate) async fn sync_events_v4_route( let required_state = required_state_request .iter() - .map(|state| { + .stream() + .filter_map(|state| async move { services .rooms .state_accessor .room_state_get(room_id, &state.0, &state.1) + .await + .map(|s| s.to_sync_state_event()) + .ok() }) - .filter_map(Result::ok) - .flatten() - .map(|state| state.to_sync_state_event()) - .collect(); + .collect() + .await; // Heroes let heroes = services .rooms .state_cache .room_members(room_id) - .filter_map(Result::ok) - .filter(|member| member != &sender_user) - .map(|member| { - Ok::<_, Error>( - services - .rooms - .state_accessor - .get_member(room_id, &member)? - .map(|memberevent| SlidingSyncRoomHero { - user_id: member, - name: memberevent.displayname, - avatar: memberevent.avatar_url, - }), - ) + .ready_filter(|member| member != &sender_user) + .filter_map(|member| async move { + services + .rooms + .state_accessor + .get_member(room_id, member) + .await + .map(|memberevent| SlidingSyncRoomHero { + user_id: member.to_owned(), + name: memberevent.displayname, + avatar: memberevent.avatar_url, + }) + .ok() }) - .filter_map(Result::ok) - .flatten() .take(5) - .collect::>(); + .collect::>() + .await; + let name = match heroes.len().cmp(&(1_usize)) { Ordering::Greater => { let firsts = heroes[1..] @@ -1626,10 +1679,12 @@ pub(crate) async fn sync_events_v4_route( .map(|h| h.name.clone().unwrap_or_else(|| h.user_id.to_string())) .collect::>() .join(", "); + let last = heroes[0] .name .clone() .unwrap_or_else(|| heroes[0].user_id.to_string()); + Some(format!("{firsts} and {last}")) }, Ordering::Equal => Some( @@ -1650,11 +1705,17 @@ pub(crate) async fn sync_events_v4_route( rooms.insert( room_id.clone(), sync_events::v4::SlidingSyncRoom { - name: services.rooms.state_accessor.get_name(room_id)?.or(name), + name: services + .rooms + .state_accessor + .get_name(room_id) + .await + .ok() + .or(name), avatar: if let Some(heroes_avatar) = heroes_avatar { ruma::JsOption::Some(heroes_avatar) } else { - match services.rooms.state_accessor.get_avatar(room_id)? { + match services.rooms.state_accessor.get_avatar(room_id).await { ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url), ruma::JsOption::Null => ruma::JsOption::Null, ruma::JsOption::Undefined => ruma::JsOption::Undefined, @@ -1668,7 +1729,8 @@ pub(crate) async fn sync_events_v4_route( services .rooms .user - .highlight_count(&sender_user, room_id)? + .highlight_count(&sender_user, room_id) + .await .try_into() .expect("notification count can't go that high"), ), @@ -1676,7 +1738,8 @@ pub(crate) async fn sync_events_v4_route( services .rooms .user - .notification_count(&sender_user, room_id)? + .notification_count(&sender_user, room_id) + .await .try_into() .expect("notification count can't go that high"), ), @@ -1689,7 +1752,8 @@ pub(crate) async fn sync_events_v4_route( services .rooms .state_cache - .room_joined_count(room_id)? + .room_joined_count(room_id) + .await .unwrap_or(0) .try_into() .unwrap_or_else(|_| uint!(0)), @@ -1698,7 +1762,8 @@ pub(crate) async fn sync_events_v4_route( services .rooms .state_cache - .room_invited_count(room_id)? + .room_invited_count(room_id) + .await .unwrap_or(0) .try_into() .unwrap_or_else(|_| uint!(0)), @@ -1732,7 +1797,9 @@ pub(crate) async fn sync_events_v4_route( Some(sync_events::v4::ToDevice { events: services .users - .get_to_device_events(&sender_user, &sender_device)?, + .get_to_device_events(&sender_user, &sender_device) + .collect() + .await, next_batch: next_batch.to_string(), }) } else { @@ -1745,7 +1812,8 @@ pub(crate) async fn sync_events_v4_route( }, device_one_time_keys_count: services .users - .count_one_time_keys(&sender_user, &sender_device)?, + .count_one_time_keys(&sender_user, &sender_device) + .await, // Fallback keys are not yet supported device_unused_fallback_key_types: None, }, @@ -1759,25 +1827,26 @@ pub(crate) async fn sync_events_v4_route( }) } -fn filter_rooms( +async fn filter_rooms( rooms: &[OwnedRoomId], State(services): State, filter: &[RoomTypeFilter], negate: bool, ) -> Vec { - return rooms + rooms .iter() - .filter(|r| match services.rooms.state_accessor.get_room_type(r) { - Err(e) => { - warn!("Requested room type for {}, but could not retrieve with error {}", r, e); - false - }, - Ok(result) => { - let result = RoomTypeFilter::from(result); - if negate { - !filter.contains(&result) - } else { - filter.is_empty() || filter.contains(&result) - } - }, + .stream() + .filter_map(|r| async move { + match services.rooms.state_accessor.get_room_type(r).await { + Err(_) => false, + Ok(result) => { + let result = RoomTypeFilter::from(Some(result)); + if negate { + !filter.contains(&result) + } else { + filter.is_empty() || filter.contains(&result) + } + }, + } + .then_some(r.to_owned()) }) - .cloned() - .collect(); + .collect() + .await } diff --git a/src/api/client/tag.rs b/src/api/client/tag.rs index 301568e5..bcd0f817 100644 --- a/src/api/client/tag.rs +++ b/src/api/client/tag.rs @@ -23,10 +23,11 @@ pub(crate) async fn update_tag_route( let event = services .account_data - .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag)?; + .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag) + .await; let mut tags_event = event.map_or_else( - || { + |_| { Ok(TagEvent { content: TagEventContent { tags: BTreeMap::new(), @@ -41,12 +42,15 @@ pub(crate) async fn update_tag_route( .tags .insert(body.tag.clone().into(), body.tag_info.clone()); - services.account_data.update( - Some(&body.room_id), - sender_user, - RoomAccountDataEventType::Tag, - &serde_json::to_value(tags_event).expect("to json value always works"), - )?; + services + .account_data + .update( + Some(&body.room_id), + sender_user, + RoomAccountDataEventType::Tag, + &serde_json::to_value(tags_event).expect("to json value always works"), + ) + .await?; Ok(create_tag::v3::Response {}) } @@ -63,10 +67,11 @@ pub(crate) async fn delete_tag_route( let event = services .account_data - .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag)?; + .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag) + .await; let mut tags_event = event.map_or_else( - || { + |_| { Ok(TagEvent { content: TagEventContent { tags: BTreeMap::new(), @@ -78,12 +83,15 @@ pub(crate) async fn delete_tag_route( tags_event.content.tags.remove(&body.tag.clone().into()); - services.account_data.update( - Some(&body.room_id), - sender_user, - RoomAccountDataEventType::Tag, - &serde_json::to_value(tags_event).expect("to json value always works"), - )?; + services + .account_data + .update( + Some(&body.room_id), + sender_user, + RoomAccountDataEventType::Tag, + &serde_json::to_value(tags_event).expect("to json value always works"), + ) + .await?; Ok(delete_tag::v3::Response {}) } @@ -100,10 +108,11 @@ pub(crate) async fn get_tags_route( let event = services .account_data - .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag)?; + .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag) + .await; let tags_event = event.map_or_else( - || { + |_| { Ok(TagEvent { content: TagEventContent { tags: BTreeMap::new(), diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index 8100f0e6..50f6cdfb 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -1,4 +1,6 @@ use axum::extract::State; +use conduit::PduEvent; +use futures::StreamExt; use ruma::{ api::client::{error::ErrorKind, threads::get_threads}, uint, @@ -27,20 +29,23 @@ pub(crate) async fn get_threads_route( u64::MAX }; - let threads = services + let room_id = &body.room_id; + let threads: Vec<(u64, PduEvent)> = services .rooms .threads - .threads_until(sender_user, &body.room_id, from, &body.include)? + .threads_until(sender_user, &body.room_id, from, &body.include) + .await? .take(limit) - .filter_map(Result::ok) - .filter(|(_, pdu)| { + .filter_map(|(count, pdu)| async move { services .rooms .state_accessor - .user_can_see_event(sender_user, &body.room_id, &pdu.event_id) - .unwrap_or(false) + .user_can_see_event(sender_user, room_id, &pdu.event_id) + .await + .then_some((count, pdu)) }) - .collect::>(); + .collect() + .await; let next_batch = threads.last().map(|(count, _)| count.to_string()); diff --git a/src/api/client/to_device.rs b/src/api/client/to_device.rs index 1f557ad7..2b37a9ec 100644 --- a/src/api/client/to_device.rs +++ b/src/api/client/to_device.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use axum::extract::State; use conduit::{Error, Result}; +use futures::StreamExt; use ruma::{ api::{ client::{error::ErrorKind, to_device::send_event_to_device}, @@ -24,8 +25,9 @@ pub(crate) async fn send_event_to_device_route( // Check if this is a new transaction id if services .transaction_ids - .existing_txnid(sender_user, sender_device, &body.txn_id)? - .is_some() + .existing_txnid(sender_user, sender_device, &body.txn_id) + .await + .is_ok() { return Ok(send_event_to_device::v3::Response {}); } @@ -53,31 +55,35 @@ pub(crate) async fn send_event_to_device_route( continue; } + let event_type = &body.event_type.to_string(); + + let event = event + .deserialize_as() + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid"))?; + match target_device_id_maybe { DeviceIdOrAllDevices::DeviceId(target_device_id) => { - services.users.add_to_device_event( - sender_user, - target_user_id, - target_device_id, - &body.event_type.to_string(), - event - .deserialize_as() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid"))?, - )?; + services + .users + .add_to_device_event(sender_user, target_user_id, target_device_id, event_type, event) + .await; }, DeviceIdOrAllDevices::AllDevices => { - for target_device_id in services.users.all_device_ids(target_user_id) { - services.users.add_to_device_event( - sender_user, - target_user_id, - &target_device_id?, - &body.event_type.to_string(), - event - .deserialize_as() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid"))?, - )?; - } + let (event_type, event) = (&event_type, &event); + services + .users + .all_device_ids(target_user_id) + .for_each(|target_device_id| { + services.users.add_to_device_event( + sender_user, + target_user_id, + target_device_id, + event_type, + event.clone(), + ) + }) + .await; }, } } @@ -86,7 +92,7 @@ pub(crate) async fn send_event_to_device_route( // Save transaction id with empty data services .transaction_ids - .add_txnid(sender_user, sender_device, &body.txn_id, &[])?; + .add_txnid(sender_user, sender_device, &body.txn_id, &[]); Ok(send_event_to_device::v3::Response {}) } diff --git a/src/api/client/typing.rs b/src/api/client/typing.rs index a06648e0..932d221e 100644 --- a/src/api/client/typing.rs +++ b/src/api/client/typing.rs @@ -16,7 +16,8 @@ pub(crate) async fn create_typing_event_route( if !services .rooms .state_cache - .is_joined(sender_user, &body.room_id)? + .is_joined(sender_user, &body.room_id) + .await { return Err(Error::BadRequest(ErrorKind::forbidden(), "You are not in this room.")); } diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index ab4703fd..dc570295 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -2,7 +2,8 @@ use std::collections::BTreeMap; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{warn, Err}; +use conduit::Err; +use futures::StreamExt; use ruma::{ api::{ client::{ @@ -45,7 +46,7 @@ pub(crate) async fn get_mutual_rooms_route( )); } - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { return Ok(mutual_rooms::unstable::Response { joined: vec![], next_batch_token: None, @@ -55,9 +56,10 @@ pub(crate) async fn get_mutual_rooms_route( let mutual_rooms: Vec = services .rooms .user - .get_shared_rooms(vec![sender_user.clone(), body.user_id.clone()])? - .filter_map(Result::ok) - .collect(); + .get_shared_rooms(sender_user, &body.user_id) + .map(ToOwned::to_owned) + .collect() + .await; Ok(mutual_rooms::unstable::Response { joined: mutual_rooms, @@ -99,7 +101,7 @@ pub(crate) async fn get_room_summary( let room_id = services.rooms.alias.resolve(&body.room_id_or_alias).await?; - if !services.rooms.metadata.exists(&room_id)? { + if !services.rooms.metadata.exists(&room_id).await { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server")); } @@ -108,7 +110,7 @@ pub(crate) async fn get_room_summary( .rooms .state_accessor .is_world_readable(&room_id) - .unwrap_or(false) + .await { return Err(Error::BadRequest( ErrorKind::forbidden(), @@ -122,50 +124,58 @@ pub(crate) async fn get_room_summary( .rooms .state_accessor .get_canonical_alias(&room_id) - .unwrap_or(None), + .await + .ok(), avatar_url: services .rooms .state_accessor - .get_avatar(&room_id)? + .get_avatar(&room_id) + .await .into_option() .unwrap_or_default() .url, - guest_can_join: services.rooms.state_accessor.guest_can_join(&room_id)?, - name: services - .rooms - .state_accessor - .get_name(&room_id) - .unwrap_or(None), + guest_can_join: services.rooms.state_accessor.guest_can_join(&room_id).await, + name: services.rooms.state_accessor.get_name(&room_id).await.ok(), num_joined_members: services .rooms .state_cache .room_joined_count(&room_id) - .unwrap_or_default() - .unwrap_or_else(|| { - warn!("Room {room_id} has no member count"); - 0 - }) - .try_into() - .expect("user count should not be that big"), + .await + .unwrap_or(0) + .try_into()?, topic: services .rooms .state_accessor .get_room_topic(&room_id) - .unwrap_or(None), + .await + .ok(), world_readable: services .rooms .state_accessor .is_world_readable(&room_id) - .unwrap_or(false), - join_rule: services.rooms.state_accessor.get_join_rule(&room_id)?.0, - room_type: services.rooms.state_accessor.get_room_type(&room_id)?, - room_version: Some(services.rooms.state.get_room_version(&room_id)?), + .await, + join_rule: services + .rooms + .state_accessor + .get_join_rule(&room_id) + .await + .unwrap_or_default() + .0, + room_type: services + .rooms + .state_accessor + .get_room_type(&room_id) + .await + .ok(), + room_version: services.rooms.state.get_room_version(&room_id).await.ok(), membership: if let Some(sender_user) = sender_user { services .rooms .state_accessor - .get_member(&room_id, sender_user)? - .map_or_else(|| Some(MembershipState::Leave), |content| Some(content.membership)) + .get_member(&room_id, sender_user) + .await + .map_or_else(|_| MembershipState::Leave, |content| content.membership) + .into() } else { None }, @@ -173,7 +183,8 @@ pub(crate) async fn get_room_summary( .rooms .state_accessor .get_room_encryption(&room_id) - .unwrap_or_else(|_e| None), + .await + .ok(), }) } @@ -191,13 +202,14 @@ pub(crate) async fn delete_timezone_key_route( return Err!(Request(Forbidden("You cannot update the profile of another user"))); } - services.users.set_timezone(&body.user_id, None).await?; + services.users.set_timezone(&body.user_id, None); if services.globals.allow_local_presence() { // Presence update services .presence - .ping_presence(&body.user_id, &PresenceState::Online)?; + .ping_presence(&body.user_id, &PresenceState::Online) + .await?; } Ok(delete_timezone_key::unstable::Response {}) @@ -217,16 +229,14 @@ pub(crate) async fn set_timezone_key_route( return Err!(Request(Forbidden("You cannot update the profile of another user"))); } - services - .users - .set_timezone(&body.user_id, body.tz.clone()) - .await?; + services.users.set_timezone(&body.user_id, body.tz.clone()); if services.globals.allow_local_presence() { // Presence update services .presence - .ping_presence(&body.user_id, &PresenceState::Online)?; + .ping_presence(&body.user_id, &PresenceState::Online) + .await?; } Ok(set_timezone_key::unstable::Response {}) @@ -280,10 +290,11 @@ pub(crate) async fn set_profile_key_route( .rooms .state_cache .rooms_joined(&body.user_id) - .filter_map(Result::ok) - .collect(); + .map(Into::into) + .collect() + .await; - update_displayname(&services, &body.user_id, Some(profile_key_value.to_string()), all_joined_rooms).await?; + update_displayname(&services, &body.user_id, Some(profile_key_value.to_string()), &all_joined_rooms).await?; } else if body.key == "avatar_url" { let mxc = ruma::OwnedMxcUri::from(profile_key_value.to_string()); @@ -291,21 +302,23 @@ pub(crate) async fn set_profile_key_route( .rooms .state_cache .rooms_joined(&body.user_id) - .filter_map(Result::ok) - .collect(); + .map(Into::into) + .collect() + .await; - update_avatar_url(&services, &body.user_id, Some(mxc), None, all_joined_rooms).await?; + update_avatar_url(&services, &body.user_id, Some(mxc), None, &all_joined_rooms).await?; } else { services .users - .set_profile_key(&body.user_id, &body.key, Some(profile_key_value.clone()))?; + .set_profile_key(&body.user_id, &body.key, Some(profile_key_value.clone())); } if services.globals.allow_local_presence() { // Presence update services .presence - .ping_presence(&body.user_id, &PresenceState::Online)?; + .ping_presence(&body.user_id, &PresenceState::Online) + .await?; } Ok(set_profile_key::unstable::Response {}) @@ -335,30 +348,33 @@ pub(crate) async fn delete_profile_key_route( .rooms .state_cache .rooms_joined(&body.user_id) - .filter_map(Result::ok) - .collect(); + .map(Into::into) + .collect() + .await; - update_displayname(&services, &body.user_id, None, all_joined_rooms).await?; + update_displayname(&services, &body.user_id, None, &all_joined_rooms).await?; } else if body.key == "avatar_url" { let all_joined_rooms: Vec = services .rooms .state_cache .rooms_joined(&body.user_id) - .filter_map(Result::ok) - .collect(); + .map(Into::into) + .collect() + .await; - update_avatar_url(&services, &body.user_id, None, None, all_joined_rooms).await?; + update_avatar_url(&services, &body.user_id, None, None, &all_joined_rooms).await?; } else { services .users - .set_profile_key(&body.user_id, &body.key, None)?; + .set_profile_key(&body.user_id, &body.key, None); } if services.globals.allow_local_presence() { // Presence update services .presence - .ping_presence(&body.user_id, &PresenceState::Online)?; + .ping_presence(&body.user_id, &PresenceState::Online) + .await?; } Ok(delete_profile_key::unstable::Response {}) @@ -386,26 +402,25 @@ pub(crate) async fn get_timezone_key_route( ) .await { - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { services.users.create(&body.user_id, None)?; } services .users - .set_displayname(&body.user_id, response.displayname.clone()) - .await?; + .set_displayname(&body.user_id, response.displayname.clone()); + services .users - .set_avatar_url(&body.user_id, response.avatar_url.clone()) - .await?; + .set_avatar_url(&body.user_id, response.avatar_url.clone()); + services .users - .set_blurhash(&body.user_id, response.blurhash.clone()) - .await?; + .set_blurhash(&body.user_id, response.blurhash.clone()); + services .users - .set_timezone(&body.user_id, response.tz.clone()) - .await?; + .set_timezone(&body.user_id, response.tz.clone()); return Ok(get_timezone_key::unstable::Response { tz: response.tz, @@ -413,14 +428,14 @@ pub(crate) async fn get_timezone_key_route( } } - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); } Ok(get_timezone_key::unstable::Response { - tz: services.users.timezone(&body.user_id)?, + tz: services.users.timezone(&body.user_id).await.ok(), }) } @@ -448,32 +463,31 @@ pub(crate) async fn get_profile_key_route( ) .await { - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { services.users.create(&body.user_id, None)?; } services .users - .set_displayname(&body.user_id, response.displayname.clone()) - .await?; + .set_displayname(&body.user_id, response.displayname.clone()); + services .users - .set_avatar_url(&body.user_id, response.avatar_url.clone()) - .await?; + .set_avatar_url(&body.user_id, response.avatar_url.clone()); + services .users - .set_blurhash(&body.user_id, response.blurhash.clone()) - .await?; + .set_blurhash(&body.user_id, response.blurhash.clone()); + services .users - .set_timezone(&body.user_id, response.tz.clone()) - .await?; + .set_timezone(&body.user_id, response.tz.clone()); if let Some(value) = response.custom_profile_fields.get(&body.key) { profile_key_value.insert(body.key.clone(), value.clone()); services .users - .set_profile_key(&body.user_id, &body.key, Some(value.clone()))?; + .set_profile_key(&body.user_id, &body.key, Some(value.clone())); } else { return Err!(Request(NotFound("The requested profile key does not exist."))); } @@ -484,13 +498,13 @@ pub(crate) async fn get_profile_key_route( } } - if !services.users.exists(&body.user_id)? { + if !services.users.exists(&body.user_id).await { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation - return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); + return Err!(Request(NotFound("Profile was not found."))); } - if let Some(value) = services.users.profile_key(&body.user_id, &body.key)? { + if let Ok(value) = services.users.profile_key(&body.user_id, &body.key).await { profile_key_value.insert(body.key.clone(), value); } else { return Err!(Request(NotFound("The requested profile key does not exist."))); diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index d714fda5..d5bb14e5 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use axum::{extract::State, response::IntoResponse, Json}; +use futures::StreamExt; use ruma::api::client::{ discovery::{ discover_homeserver::{self, HomeserverInfo, SlidingSyncProxyInfo}, @@ -173,7 +174,7 @@ pub(crate) async fn conduwuit_server_version() -> Result { /// homeserver. Endpoint is disabled if federation is disabled for privacy. This /// only includes active users (not deactivated, no guests, etc) pub(crate) async fn conduwuit_local_user_count(State(services): State) -> Result { - let user_count = services.users.list_local_users()?.len(); + let user_count = services.users.list_local_users().count().await; Ok(Json(serde_json::json!({ "count": user_count diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index 87d4062c..8ea7f1b8 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -1,4 +1,5 @@ use axum::extract::State; +use futures::{pin_mut, StreamExt}; use ruma::{ api::client::user_directory::search_users, events::{ @@ -21,14 +22,12 @@ pub(crate) async fn search_users_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let limit = usize::try_from(body.limit).unwrap_or(10); // default limit is 10 - let mut users = services.users.iter().filter_map(|user_id| { + let users = services.users.stream().filter_map(|user_id| async { // Filter out buggy users (they should not exist, but you never know...) - let user_id = user_id.ok()?; - let user = search_users::v3::User { - user_id: user_id.clone(), - display_name: services.users.displayname(&user_id).ok()?, - avatar_url: services.users.avatar_url(&user_id).ok()?, + user_id: user_id.to_owned(), + display_name: services.users.displayname(user_id).await.ok(), + avatar_url: services.users.avatar_url(user_id).await.ok(), }; let user_id_matches = user @@ -56,20 +55,19 @@ pub(crate) async fn search_users_route( let user_is_in_public_rooms = services .rooms .state_cache - .rooms_joined(&user_id) - .filter_map(Result::ok) - .any(|room| { + .rooms_joined(&user.user_id) + .any(|room| async move { services .rooms .state_accessor - .room_state_get(&room, &StateEventType::RoomJoinRules, "") + .room_state_get(room, &StateEventType::RoomJoinRules, "") + .await .map_or(false, |event| { - event.map_or(false, |event| { - serde_json::from_str(event.content.get()) - .map_or(false, |r: RoomJoinRulesEventContent| r.join_rule == JoinRule::Public) - }) + serde_json::from_str(event.content.get()) + .map_or(false, |r: RoomJoinRulesEventContent| r.join_rule == JoinRule::Public) }) - }); + }) + .await; if user_is_in_public_rooms { user_visible = true; @@ -77,25 +75,22 @@ pub(crate) async fn search_users_route( let user_is_in_shared_rooms = services .rooms .user - .get_shared_rooms(vec![sender_user.clone(), user_id]) - .ok()? - .next() - .is_some(); + .has_shared_rooms(sender_user, &user.user_id) + .await; if user_is_in_shared_rooms { user_visible = true; } } - if !user_visible { - return None; - } - - Some(user) + user_visible.then_some(user) }); - let results = users.by_ref().take(limit).collect(); - let limited = users.next().is_some(); + pin_mut!(users); + + let limited = users.by_ref().next().await.is_some(); + + let results = users.take(limit).collect().await; Ok(search_users::v3::Response { results, diff --git a/src/api/router.rs b/src/api/router.rs index 4264e01d..c4275f05 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -22,101 +22,101 @@ use crate::{client, server}; pub fn build(router: Router, server: &Server) -> Router { let config = &server.config; let mut router = router - .ruma_route(client::get_timezone_key_route) - .ruma_route(client::get_profile_key_route) - .ruma_route(client::set_profile_key_route) - .ruma_route(client::delete_profile_key_route) - .ruma_route(client::set_timezone_key_route) - .ruma_route(client::delete_timezone_key_route) - .ruma_route(client::appservice_ping) - .ruma_route(client::get_supported_versions_route) - .ruma_route(client::get_register_available_route) - .ruma_route(client::register_route) - .ruma_route(client::get_login_types_route) - .ruma_route(client::login_route) - .ruma_route(client::whoami_route) - .ruma_route(client::logout_route) - .ruma_route(client::logout_all_route) - .ruma_route(client::change_password_route) - .ruma_route(client::deactivate_route) - .ruma_route(client::third_party_route) - .ruma_route(client::request_3pid_management_token_via_email_route) - .ruma_route(client::request_3pid_management_token_via_msisdn_route) - .ruma_route(client::check_registration_token_validity) - .ruma_route(client::get_capabilities_route) - .ruma_route(client::get_pushrules_all_route) - .ruma_route(client::set_pushrule_route) - .ruma_route(client::get_pushrule_route) - .ruma_route(client::set_pushrule_enabled_route) - .ruma_route(client::get_pushrule_enabled_route) - .ruma_route(client::get_pushrule_actions_route) - .ruma_route(client::set_pushrule_actions_route) - .ruma_route(client::delete_pushrule_route) - .ruma_route(client::get_room_event_route) - .ruma_route(client::get_room_aliases_route) - .ruma_route(client::get_filter_route) - .ruma_route(client::create_filter_route) - .ruma_route(client::create_openid_token_route) - .ruma_route(client::set_global_account_data_route) - .ruma_route(client::set_room_account_data_route) - .ruma_route(client::get_global_account_data_route) - .ruma_route(client::get_room_account_data_route) - .ruma_route(client::set_displayname_route) - .ruma_route(client::get_displayname_route) - .ruma_route(client::set_avatar_url_route) - .ruma_route(client::get_avatar_url_route) - .ruma_route(client::get_profile_route) - .ruma_route(client::set_presence_route) - .ruma_route(client::get_presence_route) - .ruma_route(client::upload_keys_route) - .ruma_route(client::get_keys_route) - .ruma_route(client::claim_keys_route) - .ruma_route(client::create_backup_version_route) - .ruma_route(client::update_backup_version_route) - .ruma_route(client::delete_backup_version_route) - .ruma_route(client::get_latest_backup_info_route) - .ruma_route(client::get_backup_info_route) - .ruma_route(client::add_backup_keys_route) - .ruma_route(client::add_backup_keys_for_room_route) - .ruma_route(client::add_backup_keys_for_session_route) - .ruma_route(client::delete_backup_keys_for_room_route) - .ruma_route(client::delete_backup_keys_for_session_route) - .ruma_route(client::delete_backup_keys_route) - .ruma_route(client::get_backup_keys_for_room_route) - .ruma_route(client::get_backup_keys_for_session_route) - .ruma_route(client::get_backup_keys_route) - .ruma_route(client::set_read_marker_route) - .ruma_route(client::create_receipt_route) - .ruma_route(client::create_typing_event_route) - .ruma_route(client::create_room_route) - .ruma_route(client::redact_event_route) - .ruma_route(client::report_event_route) - .ruma_route(client::create_alias_route) - .ruma_route(client::delete_alias_route) - .ruma_route(client::get_alias_route) - .ruma_route(client::join_room_by_id_route) - .ruma_route(client::join_room_by_id_or_alias_route) - .ruma_route(client::joined_members_route) - .ruma_route(client::leave_room_route) - .ruma_route(client::forget_room_route) - .ruma_route(client::joined_rooms_route) - .ruma_route(client::kick_user_route) - .ruma_route(client::ban_user_route) - .ruma_route(client::unban_user_route) - .ruma_route(client::invite_user_route) - .ruma_route(client::set_room_visibility_route) - .ruma_route(client::get_room_visibility_route) - .ruma_route(client::get_public_rooms_route) - .ruma_route(client::get_public_rooms_filtered_route) - .ruma_route(client::search_users_route) - .ruma_route(client::get_member_events_route) - .ruma_route(client::get_protocols_route) + .ruma_route(&client::get_timezone_key_route) + .ruma_route(&client::get_profile_key_route) + .ruma_route(&client::set_profile_key_route) + .ruma_route(&client::delete_profile_key_route) + .ruma_route(&client::set_timezone_key_route) + .ruma_route(&client::delete_timezone_key_route) + .ruma_route(&client::appservice_ping) + .ruma_route(&client::get_supported_versions_route) + .ruma_route(&client::get_register_available_route) + .ruma_route(&client::register_route) + .ruma_route(&client::get_login_types_route) + .ruma_route(&client::login_route) + .ruma_route(&client::whoami_route) + .ruma_route(&client::logout_route) + .ruma_route(&client::logout_all_route) + .ruma_route(&client::change_password_route) + .ruma_route(&client::deactivate_route) + .ruma_route(&client::third_party_route) + .ruma_route(&client::request_3pid_management_token_via_email_route) + .ruma_route(&client::request_3pid_management_token_via_msisdn_route) + .ruma_route(&client::check_registration_token_validity) + .ruma_route(&client::get_capabilities_route) + .ruma_route(&client::get_pushrules_all_route) + .ruma_route(&client::set_pushrule_route) + .ruma_route(&client::get_pushrule_route) + .ruma_route(&client::set_pushrule_enabled_route) + .ruma_route(&client::get_pushrule_enabled_route) + .ruma_route(&client::get_pushrule_actions_route) + .ruma_route(&client::set_pushrule_actions_route) + .ruma_route(&client::delete_pushrule_route) + .ruma_route(&client::get_room_event_route) + .ruma_route(&client::get_room_aliases_route) + .ruma_route(&client::get_filter_route) + .ruma_route(&client::create_filter_route) + .ruma_route(&client::create_openid_token_route) + .ruma_route(&client::set_global_account_data_route) + .ruma_route(&client::set_room_account_data_route) + .ruma_route(&client::get_global_account_data_route) + .ruma_route(&client::get_room_account_data_route) + .ruma_route(&client::set_displayname_route) + .ruma_route(&client::get_displayname_route) + .ruma_route(&client::set_avatar_url_route) + .ruma_route(&client::get_avatar_url_route) + .ruma_route(&client::get_profile_route) + .ruma_route(&client::set_presence_route) + .ruma_route(&client::get_presence_route) + .ruma_route(&client::upload_keys_route) + .ruma_route(&client::get_keys_route) + .ruma_route(&client::claim_keys_route) + .ruma_route(&client::create_backup_version_route) + .ruma_route(&client::update_backup_version_route) + .ruma_route(&client::delete_backup_version_route) + .ruma_route(&client::get_latest_backup_info_route) + .ruma_route(&client::get_backup_info_route) + .ruma_route(&client::add_backup_keys_route) + .ruma_route(&client::add_backup_keys_for_room_route) + .ruma_route(&client::add_backup_keys_for_session_route) + .ruma_route(&client::delete_backup_keys_for_room_route) + .ruma_route(&client::delete_backup_keys_for_session_route) + .ruma_route(&client::delete_backup_keys_route) + .ruma_route(&client::get_backup_keys_for_room_route) + .ruma_route(&client::get_backup_keys_for_session_route) + .ruma_route(&client::get_backup_keys_route) + .ruma_route(&client::set_read_marker_route) + .ruma_route(&client::create_receipt_route) + .ruma_route(&client::create_typing_event_route) + .ruma_route(&client::create_room_route) + .ruma_route(&client::redact_event_route) + .ruma_route(&client::report_event_route) + .ruma_route(&client::create_alias_route) + .ruma_route(&client::delete_alias_route) + .ruma_route(&client::get_alias_route) + .ruma_route(&client::join_room_by_id_route) + .ruma_route(&client::join_room_by_id_or_alias_route) + .ruma_route(&client::joined_members_route) + .ruma_route(&client::leave_room_route) + .ruma_route(&client::forget_room_route) + .ruma_route(&client::joined_rooms_route) + .ruma_route(&client::kick_user_route) + .ruma_route(&client::ban_user_route) + .ruma_route(&client::unban_user_route) + .ruma_route(&client::invite_user_route) + .ruma_route(&client::set_room_visibility_route) + .ruma_route(&client::get_room_visibility_route) + .ruma_route(&client::get_public_rooms_route) + .ruma_route(&client::get_public_rooms_filtered_route) + .ruma_route(&client::search_users_route) + .ruma_route(&client::get_member_events_route) + .ruma_route(&client::get_protocols_route) .route("/_matrix/client/unstable/thirdparty/protocols", get(client::get_protocols_route_unstable)) - .ruma_route(client::send_message_event_route) - .ruma_route(client::send_state_event_for_key_route) - .ruma_route(client::get_state_events_route) - .ruma_route(client::get_state_events_for_key_route) + .ruma_route(&client::send_message_event_route) + .ruma_route(&client::send_state_event_for_key_route) + .ruma_route(&client::get_state_events_route) + .ruma_route(&client::get_state_events_for_key_route) // Ruma doesn't have support for multiple paths for a single endpoint yet, and these routes // share one Ruma request / response type pair with {get,send}_state_event_for_key_route .route( @@ -140,46 +140,46 @@ pub fn build(router: Router, server: &Server) -> Router { get(client::get_state_events_for_empty_key_route) .put(client::send_state_event_for_empty_key_route), ) - .ruma_route(client::sync_events_route) - .ruma_route(client::sync_events_v4_route) - .ruma_route(client::get_context_route) - .ruma_route(client::get_message_events_route) - .ruma_route(client::search_events_route) - .ruma_route(client::turn_server_route) - .ruma_route(client::send_event_to_device_route) - .ruma_route(client::create_content_route) - .ruma_route(client::get_content_thumbnail_route) - .ruma_route(client::get_content_route) - .ruma_route(client::get_content_as_filename_route) - .ruma_route(client::get_media_preview_route) - .ruma_route(client::get_media_config_route) - .ruma_route(client::get_devices_route) - .ruma_route(client::get_device_route) - .ruma_route(client::update_device_route) - .ruma_route(client::delete_device_route) - .ruma_route(client::delete_devices_route) - .ruma_route(client::get_tags_route) - .ruma_route(client::update_tag_route) - .ruma_route(client::delete_tag_route) - .ruma_route(client::upload_signing_keys_route) - .ruma_route(client::upload_signatures_route) - .ruma_route(client::get_key_changes_route) - .ruma_route(client::get_pushers_route) - .ruma_route(client::set_pushers_route) - .ruma_route(client::upgrade_room_route) - .ruma_route(client::get_threads_route) - .ruma_route(client::get_relating_events_with_rel_type_and_event_type_route) - .ruma_route(client::get_relating_events_with_rel_type_route) - .ruma_route(client::get_relating_events_route) - .ruma_route(client::get_hierarchy_route) - .ruma_route(client::get_mutual_rooms_route) - .ruma_route(client::get_room_summary) + .ruma_route(&client::sync_events_route) + .ruma_route(&client::sync_events_v4_route) + .ruma_route(&client::get_context_route) + .ruma_route(&client::get_message_events_route) + .ruma_route(&client::search_events_route) + .ruma_route(&client::turn_server_route) + .ruma_route(&client::send_event_to_device_route) + .ruma_route(&client::create_content_route) + .ruma_route(&client::get_content_thumbnail_route) + .ruma_route(&client::get_content_route) + .ruma_route(&client::get_content_as_filename_route) + .ruma_route(&client::get_media_preview_route) + .ruma_route(&client::get_media_config_route) + .ruma_route(&client::get_devices_route) + .ruma_route(&client::get_device_route) + .ruma_route(&client::update_device_route) + .ruma_route(&client::delete_device_route) + .ruma_route(&client::delete_devices_route) + .ruma_route(&client::get_tags_route) + .ruma_route(&client::update_tag_route) + .ruma_route(&client::delete_tag_route) + .ruma_route(&client::upload_signing_keys_route) + .ruma_route(&client::upload_signatures_route) + .ruma_route(&client::get_key_changes_route) + .ruma_route(&client::get_pushers_route) + .ruma_route(&client::set_pushers_route) + .ruma_route(&client::upgrade_room_route) + .ruma_route(&client::get_threads_route) + .ruma_route(&client::get_relating_events_with_rel_type_and_event_type_route) + .ruma_route(&client::get_relating_events_with_rel_type_route) + .ruma_route(&client::get_relating_events_route) + .ruma_route(&client::get_hierarchy_route) + .ruma_route(&client::get_mutual_rooms_route) + .ruma_route(&client::get_room_summary) .route( "/_matrix/client/unstable/im.nheko.summary/rooms/:room_id_or_alias/summary", get(client::get_room_summary_legacy) ) - .ruma_route(client::well_known_support) - .ruma_route(client::well_known_client) + .ruma_route(&client::well_known_support) + .ruma_route(&client::well_known_client) .route("/_conduwuit/server_version", get(client::conduwuit_server_version)) .route("/_matrix/client/r0/rooms/:room_id/initialSync", get(initial_sync)) .route("/_matrix/client/v3/rooms/:room_id/initialSync", get(initial_sync)) @@ -187,35 +187,35 @@ pub fn build(router: Router, server: &Server) -> Router { if config.allow_federation { router = router - .ruma_route(server::get_server_version_route) + .ruma_route(&server::get_server_version_route) .route("/_matrix/key/v2/server", get(server::get_server_keys_route)) .route("/_matrix/key/v2/server/:key_id", get(server::get_server_keys_deprecated_route)) - .ruma_route(server::get_public_rooms_route) - .ruma_route(server::get_public_rooms_filtered_route) - .ruma_route(server::send_transaction_message_route) - .ruma_route(server::get_event_route) - .ruma_route(server::get_backfill_route) - .ruma_route(server::get_missing_events_route) - .ruma_route(server::get_event_authorization_route) - .ruma_route(server::get_room_state_route) - .ruma_route(server::get_room_state_ids_route) - .ruma_route(server::create_leave_event_template_route) - .ruma_route(server::create_leave_event_v1_route) - .ruma_route(server::create_leave_event_v2_route) - .ruma_route(server::create_join_event_template_route) - .ruma_route(server::create_join_event_v1_route) - .ruma_route(server::create_join_event_v2_route) - .ruma_route(server::create_invite_route) - .ruma_route(server::get_devices_route) - .ruma_route(server::get_room_information_route) - .ruma_route(server::get_profile_information_route) - .ruma_route(server::get_keys_route) - .ruma_route(server::claim_keys_route) - .ruma_route(server::get_openid_userinfo_route) - .ruma_route(server::get_hierarchy_route) - .ruma_route(server::well_known_server) - .ruma_route(server::get_content_route) - .ruma_route(server::get_content_thumbnail_route) + .ruma_route(&server::get_public_rooms_route) + .ruma_route(&server::get_public_rooms_filtered_route) + .ruma_route(&server::send_transaction_message_route) + .ruma_route(&server::get_event_route) + .ruma_route(&server::get_backfill_route) + .ruma_route(&server::get_missing_events_route) + .ruma_route(&server::get_event_authorization_route) + .ruma_route(&server::get_room_state_route) + .ruma_route(&server::get_room_state_ids_route) + .ruma_route(&server::create_leave_event_template_route) + .ruma_route(&server::create_leave_event_v1_route) + .ruma_route(&server::create_leave_event_v2_route) + .ruma_route(&server::create_join_event_template_route) + .ruma_route(&server::create_join_event_v1_route) + .ruma_route(&server::create_join_event_v2_route) + .ruma_route(&server::create_invite_route) + .ruma_route(&server::get_devices_route) + .ruma_route(&server::get_room_information_route) + .ruma_route(&server::get_profile_information_route) + .ruma_route(&server::get_keys_route) + .ruma_route(&server::claim_keys_route) + .ruma_route(&server::get_openid_userinfo_route) + .ruma_route(&server::get_hierarchy_route) + .ruma_route(&server::well_known_server) + .ruma_route(&server::get_content_route) + .ruma_route(&server::get_content_thumbnail_route) .route("/_conduwuit/local_user_count", get(client::conduwuit_local_user_count)); } else { router = router @@ -227,11 +227,11 @@ pub fn build(router: Router, server: &Server) -> Router { if config.allow_legacy_media { router = router - .ruma_route(client::get_media_config_legacy_route) - .ruma_route(client::get_media_preview_legacy_route) - .ruma_route(client::get_content_legacy_route) - .ruma_route(client::get_content_as_filename_legacy_route) - .ruma_route(client::get_content_thumbnail_legacy_route) + .ruma_route(&client::get_media_config_legacy_route) + .ruma_route(&client::get_media_preview_legacy_route) + .ruma_route(&client::get_content_legacy_route) + .ruma_route(&client::get_content_as_filename_legacy_route) + .ruma_route(&client::get_content_thumbnail_legacy_route) .route("/_matrix/media/v1/config", get(client::get_media_config_legacy_legacy_route)) .route("/_matrix/media/v1/upload", post(client::create_content_legacy_route)) .route( diff --git a/src/api/router/args.rs b/src/api/router/args.rs index a3d09dff..7381a55f 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -10,7 +10,10 @@ use super::{auth, auth::Auth, request, request::Request}; use crate::{service::appservice::RegistrationInfo, State}; /// Extractor for Ruma request structs -pub(crate) struct Args { +pub(crate) struct Args +where + T: IncomingRequest + Send + Sync + 'static, +{ /// Request struct body pub(crate) body: T, @@ -38,7 +41,7 @@ pub(crate) struct Args { #[async_trait] impl FromRequest for Args where - T: IncomingRequest, + T: IncomingRequest + Send + Sync + 'static, { type Rejection = Error; @@ -57,7 +60,10 @@ where } } -impl Deref for Args { +impl Deref for Args +where + T: IncomingRequest + Send + Sync + 'static, +{ type Target = T; fn deref(&self) -> &Self::Target { &self.body } @@ -67,7 +73,7 @@ fn make_body( services: &Services, request: &mut Request, json_body: &mut Option, auth: &Auth, ) -> Result where - T: IncomingRequest, + T: IncomingRequest + Send + Sync + 'static, { let body = if let Some(CanonicalJsonValue::Object(json_body)) = json_body { let user_id = auth.sender_user.clone().unwrap_or_else(|| { @@ -77,15 +83,13 @@ where let uiaa_request = json_body .get("auth") - .and_then(|auth| auth.as_object()) + .and_then(CanonicalJsonValue::as_object) .and_then(|auth| auth.get("session")) - .and_then(|session| session.as_str()) + .and_then(CanonicalJsonValue::as_str) .and_then(|session| { - services.uiaa.get_uiaa_request( - &user_id, - &auth.sender_device.clone().unwrap_or_else(|| EMPTY.into()), - session, - ) + services + .uiaa + .get_uiaa_request(&user_id, auth.sender_device.as_deref(), session) }); if let Some(CanonicalJsonValue::Object(initial_request)) = uiaa_request { diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 670f72ba..8d76b4be 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -44,8 +44,8 @@ pub(super) async fn auth( let token = if let Some(token) = token { if let Some(reg_info) = services.appservice.find_from_token(token).await { Token::Appservice(Box::new(reg_info)) - } else if let Some((user_id, device_id)) = services.users.find_from_token(token)? { - Token::User((user_id, OwnedDeviceId::from(device_id))) + } else if let Ok((user_id, device_id)) = services.users.find_from_token(token).await { + Token::User((user_id, device_id)) } else { Token::Invalid } @@ -98,7 +98,7 @@ pub(super) async fn auth( )) } }, - (AuthScheme::AccessToken, Token::Appservice(info)) => Ok(auth_appservice(services, request, info)?), + (AuthScheme::AccessToken, Token::Appservice(info)) => Ok(auth_appservice(services, request, info).await?), (AuthScheme::None | AuthScheme::AccessTokenOptional | AuthScheme::AppserviceToken, Token::Appservice(info)) => { Ok(Auth { origin: None, @@ -150,7 +150,7 @@ pub(super) async fn auth( } } -fn auth_appservice(services: &Services, request: &Request, info: Box) -> Result { +async fn auth_appservice(services: &Services, request: &Request, info: Box) -> Result { let user_id = request .query .user_id @@ -170,7 +170,7 @@ fn auth_appservice(services: &Services, request: &Request, info: Box { + fn add_route(&'static self, router: Router, path: &str) -> Router; + fn add_routes(&'static self, router: Router) -> Router; +} + pub(in super::super) trait RouterExt { - fn ruma_route(self, handler: H) -> Self + fn ruma_route(self, handler: &'static H) -> Self where H: RumaHandler; } impl RouterExt for Router { - fn ruma_route(self, handler: H) -> Self + fn ruma_route(self, handler: &'static H) -> Self where H: RumaHandler, { @@ -27,34 +31,28 @@ impl RouterExt for Router { } } -pub(in super::super) trait RumaHandler { - fn add_routes(&self, router: Router) -> Router; - - fn add_route(&self, router: Router, path: &str) -> Router; -} - macro_rules! ruma_handler { ( $($tx:ident),* $(,)? ) => { #[allow(non_snake_case)] - impl RumaHandler<($($tx,)* Ruma,)> for Fun + impl RumaHandler<($($tx,)* Ruma,)> for Fun where - Req: IncomingRequest + Send + 'static, - Ret: IntoResponse, - Fut: Future> + Send, - Fun: FnOnce($($tx,)* Ruma,) -> Fut + Clone + Send + Sync + 'static, - $( $tx: FromRequestParts + Send + 'static, )* + Fun: Fn($($tx,)* Ruma,) -> Fut + Send + Sync + 'static, + Fut: Future> + Send, + Req: IncomingRequest + Send + Sync, + Err: IntoResponse + Send, + ::OutgoingResponse: Send, + $( $tx: FromRequestParts + Send + Sync + 'static, )* { - fn add_routes(&self, router: Router) -> Router { + fn add_routes(&'static self, router: Router) -> Router { Req::METADATA .history .all_paths() .fold(router, |router, path| self.add_route(router, path)) } - fn add_route(&self, router: Router, path: &str) -> Router { - let handle = self.clone(); + fn add_route(&'static self, router: Router, path: &str) -> Router { + let action = |$($tx,)* req| self($($tx,)* req).map_ok(RumaResponse); let method = method_to_filter(&Req::METADATA.method); - let action = |$($tx,)* req| async { handle($($tx,)* req).await.map(RumaResponse) }; router.route(path, on(method, action)) } } diff --git a/src/api/router/response.rs b/src/api/router/response.rs index 2aaa79fa..70bbb936 100644 --- a/src/api/router/response.rs +++ b/src/api/router/response.rs @@ -5,13 +5,18 @@ use http::StatusCode; use http_body_util::Full; use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; -pub(crate) struct RumaResponse(pub(crate) T); +pub(crate) struct RumaResponse(pub(crate) T) +where + T: OutgoingResponse; impl From for RumaResponse { fn from(t: Error) -> Self { Self(t.into()) } } -impl IntoResponse for RumaResponse { +impl IntoResponse for RumaResponse +where + T: OutgoingResponse, +{ fn into_response(self) -> Response { self.0 .try_into_http_response::() diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 1b665c19..2bbc95ca 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -1,9 +1,13 @@ +use std::cmp; + use axum::extract::State; -use conduit::{Error, Result}; -use ruma::{ - api::{client::error::ErrorKind, federation::backfill::get_backfill}, - uint, user_id, MilliSecondsSinceUnixEpoch, +use conduit::{ + is_equal_to, + utils::{IterStream, ReadyExt}, + Err, PduCount, Result, }; +use futures::{FutureExt, StreamExt}; +use ruma::{api::federation::backfill::get_backfill, uint, user_id, MilliSecondsSinceUnixEpoch}; use crate::Ruma; @@ -19,27 +23,35 @@ pub(crate) async fn get_backfill_route( services .rooms .event_handler - .acl_check(origin, &body.room_id)?; + .acl_check(origin, &body.room_id) + .await?; if !services .rooms .state_accessor - .is_world_readable(&body.room_id)? - && !services - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? + .is_world_readable(&body.room_id) + .await && !services + .rooms + .state_cache + .server_in_room(origin, &body.room_id) + .await { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + return Err!(Request(Forbidden("Server is not in room."))); } let until = body .v .iter() - .map(|event_id| services.rooms.timeline.get_pdu_count(event_id)) - .filter_map(|r| r.ok().flatten()) - .max() - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event not found."))?; + .stream() + .filter_map(|event_id| { + services + .rooms + .timeline + .get_pdu_count(event_id) + .map(Result::ok) + }) + .ready_fold(PduCount::Backfilled(0), cmp::max) + .await; let limit = body .limit @@ -47,31 +59,37 @@ pub(crate) async fn get_backfill_route( .try_into() .expect("UInt could not be converted to usize"); - let all_events = services + let pdus = services .rooms .timeline - .pdus_until(user_id!("@doesntmatter:conduit.rs"), &body.room_id, until)? - .take(limit); + .pdus_until(user_id!("@doesntmatter:conduit.rs"), &body.room_id, until) + .await? + .take(limit) + .filter_map(|(_, pdu)| async move { + if !services + .rooms + .state_accessor + .server_can_see_event(origin, &pdu.room_id, &pdu.event_id) + .await + .is_ok_and(is_equal_to!(true)) + { + return None; + } - let events = all_events - .filter_map(Result::ok) - .filter(|(_, e)| { - matches!( - services - .rooms - .state_accessor - .server_can_see_event(origin, &e.room_id, &e.event_id,), - Ok(true), - ) + services + .rooms + .timeline + .get_pdu_json(&pdu.event_id) + .await + .ok() }) - .map(|(_, pdu)| services.rooms.timeline.get_pdu_json(&pdu.event_id)) - .filter_map(|r| r.ok().flatten()) - .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) - .collect(); + .then(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) + .collect() + .await; Ok(get_backfill::v1::Response { origin: services.globals.server_name().to_owned(), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - pdus: events, + pdus, }) } diff --git a/src/api/server/event.rs b/src/api/server/event.rs index e11a01a2..e4eac794 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -1,9 +1,6 @@ use axum::extract::State; -use conduit::{Error, Result}; -use ruma::{ - api::{client::error::ErrorKind, federation::event::get_event}, - MilliSecondsSinceUnixEpoch, RoomId, -}; +use conduit::{err, Err, Result}; +use ruma::{api::federation::event::get_event, MilliSecondsSinceUnixEpoch, RoomId}; use crate::Ruma; @@ -21,34 +18,46 @@ pub(crate) async fn get_event_route( let event = services .rooms .timeline - .get_pdu_json(&body.event_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Event not found."))?; + .get_pdu_json(&body.event_id) + .await + .map_err(|_| err!(Request(NotFound("Event not found."))))?; let room_id_str = event .get("room_id") .and_then(|val| val.as_str()) - .ok_or_else(|| Error::bad_database("Invalid event in database."))?; + .ok_or_else(|| err!(Database("Invalid event in database.")))?; let room_id = - <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; + <&RoomId>::try_from(room_id_str).map_err(|_| err!(Database("Invalid room_id in event in database.")))?; - if !services.rooms.state_accessor.is_world_readable(room_id)? - && !services.rooms.state_cache.server_in_room(origin, room_id)? + if !services + .rooms + .state_accessor + .is_world_readable(room_id) + .await && !services + .rooms + .state_cache + .server_in_room(origin, room_id) + .await { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + return Err!(Request(Forbidden("Server is not in room."))); } if !services .rooms .state_accessor - .server_can_see_event(origin, room_id, &body.event_id)? + .server_can_see_event(origin, room_id, &body.event_id) + .await? { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not allowed to see event.")); + return Err!(Request(Forbidden("Server is not allowed to see event."))); } Ok(get_event::v1::Response { origin: services.globals.server_name().to_owned(), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - pdu: services.sending.convert_to_outgoing_federation_event(event), + pdu: services + .sending + .convert_to_outgoing_federation_event(event) + .await, }) } diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 4b0f6bc0..6ec00b50 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use axum::extract::State; use conduit::{Error, Result}; +use futures::StreamExt; use ruma::{ api::{client::error::ErrorKind, federation::authorization::get_event_authorization}, RoomId, @@ -22,16 +23,18 @@ pub(crate) async fn get_event_authorization_route( services .rooms .event_handler - .acl_check(origin, &body.room_id)?; + .acl_check(origin, &body.room_id) + .await?; if !services .rooms .state_accessor - .is_world_readable(&body.room_id)? - && !services - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? + .is_world_readable(&body.room_id) + .await && !services + .rooms + .state_cache + .server_in_room(origin, &body.room_id) + .await { return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); } @@ -39,8 +42,9 @@ pub(crate) async fn get_event_authorization_route( let event = services .rooms .timeline - .get_pdu_json(&body.event_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Event not found."))?; + .get_pdu_json(&body.event_id) + .await + .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "Event not found."))?; let room_id_str = event .get("room_id") @@ -50,16 +54,17 @@ pub(crate) async fn get_event_authorization_route( let room_id = <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; - let auth_chain_ids = services + let auth_chain = services .rooms .auth_chain .event_ids_iter(room_id, vec![Arc::from(&*body.event_id)]) - .await?; + .await? + .filter_map(|id| async move { services.rooms.timeline.get_pdu_json(&id).await.ok() }) + .then(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) + .collect() + .await; Ok(get_event_authorization::v1::Response { - auth_chain: auth_chain_ids - .filter_map(|id| services.rooms.timeline.get_pdu_json(&id).ok()?) - .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) - .collect(), + auth_chain, }) } diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index e2c3c93c..7ae0ff60 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -18,16 +18,18 @@ pub(crate) async fn get_missing_events_route( services .rooms .event_handler - .acl_check(origin, &body.room_id)?; + .acl_check(origin, &body.room_id) + .await?; if !services .rooms .state_accessor - .is_world_readable(&body.room_id)? - && !services - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? + .is_world_readable(&body.room_id) + .await && !services + .rooms + .state_cache + .server_in_room(origin, &body.room_id) + .await { return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room")); } @@ -43,7 +45,12 @@ pub(crate) async fn get_missing_events_route( let mut i: usize = 0; while i < queued_events.len() && events.len() < limit { - if let Some(pdu) = services.rooms.timeline.get_pdu_json(&queued_events[i])? { + if let Ok(pdu) = services + .rooms + .timeline + .get_pdu_json(&queued_events[i]) + .await + { let room_id_str = pdu .get("room_id") .and_then(|val| val.as_str()) @@ -64,7 +71,8 @@ pub(crate) async fn get_missing_events_route( if !services .rooms .state_accessor - .server_can_see_event(origin, &body.room_id, &queued_events[i])? + .server_can_see_event(origin, &body.room_id, &queued_events[i]) + .await? { i = i.saturating_add(1); continue; @@ -81,7 +89,12 @@ pub(crate) async fn get_missing_events_route( ) .map_err(|_| Error::bad_database("Invalid prev_events in event in database."))?, ); - events.push(services.sending.convert_to_outgoing_federation_event(pdu)); + events.push( + services + .sending + .convert_to_outgoing_federation_event(pdu) + .await, + ); } i = i.saturating_add(1); } diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs index 530ed145..002bd763 100644 --- a/src/api/server/hierarchy.rs +++ b/src/api/server/hierarchy.rs @@ -12,7 +12,7 @@ pub(crate) async fn get_hierarchy_route( ) -> Result { let origin = body.origin.as_ref().expect("server is authenticated"); - if services.rooms.metadata.exists(&body.room_id)? { + if services.rooms.metadata.exists(&body.room_id).await { services .rooms .spaces diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 688e026c..9968bdf7 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -24,7 +24,8 @@ pub(crate) async fn create_invite_route( services .rooms .event_handler - .acl_check(origin, &body.room_id)?; + .acl_check(origin, &body.room_id) + .await?; if !services .globals @@ -98,7 +99,8 @@ pub(crate) async fn create_invite_route( services .rooms .event_handler - .acl_check(invited_user.server_name(), &body.room_id)?; + .acl_check(invited_user.server_name(), &body.room_id) + .await?; ruma::signatures::hash_and_sign_event( services.globals.server_name().as_str(), @@ -128,14 +130,14 @@ pub(crate) async fn create_invite_route( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "sender is not a user ID."))?; - if services.rooms.metadata.is_banned(&body.room_id)? && !services.users.is_admin(&invited_user)? { + if services.rooms.metadata.is_banned(&body.room_id).await && !services.users.is_admin(&invited_user).await { return Err(Error::BadRequest( ErrorKind::forbidden(), "This room is banned on this homeserver.", )); } - if services.globals.block_non_admin_invites() && !services.users.is_admin(&invited_user)? { + if services.globals.block_non_admin_invites() && !services.users.is_admin(&invited_user).await { return Err(Error::BadRequest( ErrorKind::forbidden(), "This server does not allow room invites.", @@ -159,22 +161,28 @@ pub(crate) async fn create_invite_route( if !services .rooms .state_cache - .server_in_room(services.globals.server_name(), &body.room_id)? + .server_in_room(services.globals.server_name(), &body.room_id) + .await { - services.rooms.state_cache.update_membership( - &body.room_id, - &invited_user, - RoomMemberEventContent::new(MembershipState::Invite), - &sender, - Some(invite_state), - body.via.clone(), - true, - )?; + services + .rooms + .state_cache + .update_membership( + &body.room_id, + &invited_user, + RoomMemberEventContent::new(MembershipState::Invite), + &sender, + Some(invite_state), + body.via.clone(), + true, + ) + .await?; } Ok(create_invite::v2::Response { event: services .sending - .convert_to_outgoing_federation_event(signed_event), + .convert_to_outgoing_federation_event(signed_event) + .await, }) } diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index 021016be..ba081aad 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -1,4 +1,6 @@ use axum::extract::State; +use conduit::utils::{IterStream, ReadyExt}; +use futures::StreamExt; use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_join_event}, events::{ @@ -24,7 +26,7 @@ use crate::{ pub(crate) async fn create_join_event_template_route( State(services): State, body: Ruma, ) -> Result { - if !services.rooms.metadata.exists(&body.room_id)? { + if !services.rooms.metadata.exists(&body.room_id).await { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } @@ -40,7 +42,8 @@ pub(crate) async fn create_join_event_template_route( services .rooms .event_handler - .acl_check(origin, &body.room_id)?; + .acl_check(origin, &body.room_id) + .await?; if services .globals @@ -73,7 +76,7 @@ pub(crate) async fn create_join_event_template_route( } } - let room_version_id = services.rooms.state.get_room_version(&body.room_id)?; + let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; @@ -81,22 +84,24 @@ pub(crate) async fn create_join_event_template_route( .rooms .state_cache .is_left(&body.user_id, &body.room_id) - .unwrap_or(true)) - && user_can_perform_restricted_join(&services, &body.user_id, &body.room_id, &room_version_id)? + .await) + && user_can_perform_restricted_join(&services, &body.user_id, &body.room_id, &room_version_id).await? { let auth_user = services .rooms .state_cache .room_members(&body.room_id) - .filter_map(Result::ok) - .filter(|user| user.server_name() == services.globals.server_name()) - .find(|user| { + .ready_filter(|user| user.server_name() == services.globals.server_name()) + .filter(|user| { services .rooms .state_accessor .user_can_invite(&body.room_id, user, &body.user_id, &state_lock) - .unwrap_or(false) - }); + }) + .boxed() + .next() + .await + .map(ToOwned::to_owned); if auth_user.is_some() { auth_user @@ -110,7 +115,7 @@ pub(crate) async fn create_join_event_template_route( None }; - let room_version_id = services.rooms.state.get_room_version(&body.room_id)?; + let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; if !body.ver.contains(&room_version_id) { return Err(Error::BadRequest( ErrorKind::IncompatibleRoomVersion { @@ -132,19 +137,23 @@ pub(crate) async fn create_join_event_template_route( }) .expect("member event is valid value"); - let (_pdu, mut pdu_json) = services.rooms.timeline.create_hash_and_sign_event( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content, - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - timestamp: None, - }, - &body.user_id, - &body.room_id, - &state_lock, - )?; + let (_pdu, mut pdu_json) = services + .rooms + .timeline + .create_hash_and_sign_event( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content, + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + timestamp: None, + }, + &body.user_id, + &body.room_id, + &state_lock, + ) + .await?; drop(state_lock); @@ -161,7 +170,7 @@ pub(crate) async fn create_join_event_template_route( /// This doesn't check the current user's membership. This should be done /// externally, either by using the state cache or attempting to authorize the /// event. -pub(crate) fn user_can_perform_restricted_join( +pub(crate) async fn user_can_perform_restricted_join( services: &Services, user_id: &UserId, room_id: &RoomId, room_version_id: &RoomVersionId, ) -> Result { use RoomVersionId::*; @@ -169,18 +178,15 @@ pub(crate) fn user_can_perform_restricted_join( let join_rules_event = services .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomJoinRules, "")?; + .room_state_get(room_id, &StateEventType::RoomJoinRules, "") + .await; - let Some(join_rules_event_content) = join_rules_event - .as_ref() - .map(|join_rules_event| { - serde_json::from_str::(join_rules_event.content.get()).map_err(|e| { - warn!("Invalid join rules event in database: {e}"); - Error::bad_database("Invalid join rules event in database") - }) + let Ok(Ok(join_rules_event_content)) = join_rules_event.as_ref().map(|join_rules_event| { + serde_json::from_str::(join_rules_event.content.get()).map_err(|e| { + warn!("Invalid join rules event in database: {e}"); + Error::bad_database("Invalid join rules event in database") }) - .transpose()? - else { + }) else { return Ok(false); }; @@ -201,13 +207,10 @@ pub(crate) fn user_can_perform_restricted_join( None } }) - .any(|m| { - services - .rooms - .state_cache - .is_joined(user_id, &m.room_id) - .unwrap_or(false) - }) { + .stream() + .any(|m| services.rooms.state_cache.is_joined(user_id, &m.room_id)) + .await + { Ok(true) } else { Err(Error::BadRequest( diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 3eb0d77a..41ea1c80 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -18,7 +18,7 @@ use crate::{service::pdu::PduBuilder, Ruma}; pub(crate) async fn create_leave_event_template_route( State(services): State, body: Ruma, ) -> Result { - if !services.rooms.metadata.exists(&body.room_id)? { + if !services.rooms.metadata.exists(&body.room_id).await { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } @@ -34,9 +34,10 @@ pub(crate) async fn create_leave_event_template_route( services .rooms .event_handler - .acl_check(origin, &body.room_id)?; + .acl_check(origin, &body.room_id) + .await?; - let room_version_id = services.rooms.state.get_room_version(&body.room_id)?; + let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; let content = to_raw_value(&RoomMemberEventContent { avatar_url: None, @@ -50,19 +51,23 @@ pub(crate) async fn create_leave_event_template_route( }) .expect("member event is valid value"); - let (_pdu, mut pdu_json) = services.rooms.timeline.create_hash_and_sign_event( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content, - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - timestamp: None, - }, - &body.user_id, - &body.room_id, - &state_lock, - )?; + let (_pdu, mut pdu_json) = services + .rooms + .timeline + .create_hash_and_sign_event( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content, + unsigned: None, + state_key: Some(body.user_id.to_string()), + redacts: None, + timestamp: None, + }, + &body.user_id, + &body.room_id, + &state_lock, + ) + .await?; drop(state_lock); diff --git a/src/api/server/openid.rs b/src/api/server/openid.rs index 6a1b99b7..9b54807a 100644 --- a/src/api/server/openid.rs +++ b/src/api/server/openid.rs @@ -10,6 +10,9 @@ pub(crate) async fn get_openid_userinfo_route( State(services): State, body: Ruma, ) -> Result { Ok(get_openid_userinfo::v1::Response::new( - services.users.find_from_openid_token(&body.access_token)?, + services + .users + .find_from_openid_token(&body.access_token) + .await?, )) } diff --git a/src/api/server/query.rs b/src/api/server/query.rs index c2b78bde..348b8c6e 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -1,7 +1,8 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{Error, Result}; +use conduit::{err, Error, Result}; +use futures::StreamExt; use get_profile_information::v1::ProfileField; use rand::seq::SliceRandom; use ruma::{ @@ -23,15 +24,17 @@ pub(crate) async fn get_room_information_route( let room_id = services .rooms .alias - .resolve_local_alias(&body.room_alias)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Room alias not found."))?; + .resolve_local_alias(&body.room_alias) + .await + .map_err(|_| err!(Request(NotFound("Room alias not found."))))?; let mut servers: Vec = services .rooms .state_cache .room_servers(&room_id) - .filter_map(Result::ok) - .collect(); + .map(ToOwned::to_owned) + .collect() + .await; servers.sort_unstable(); servers.dedup(); @@ -82,30 +85,31 @@ pub(crate) async fn get_profile_information_route( match &body.field { Some(ProfileField::DisplayName) => { - displayname = services.users.displayname(&body.user_id)?; + displayname = services.users.displayname(&body.user_id).await.ok(); }, Some(ProfileField::AvatarUrl) => { - avatar_url = services.users.avatar_url(&body.user_id)?; - blurhash = services.users.blurhash(&body.user_id)?; + avatar_url = services.users.avatar_url(&body.user_id).await.ok(); + blurhash = services.users.blurhash(&body.user_id).await.ok(); }, Some(custom_field) => { - if let Some(value) = services + if let Ok(value) = services .users - .profile_key(&body.user_id, custom_field.as_str())? + .profile_key(&body.user_id, custom_field.as_str()) + .await { custom_profile_fields.insert(custom_field.to_string(), value); } }, None => { - displayname = services.users.displayname(&body.user_id)?; - avatar_url = services.users.avatar_url(&body.user_id)?; - blurhash = services.users.blurhash(&body.user_id)?; - tz = services.users.timezone(&body.user_id)?; + displayname = services.users.displayname(&body.user_id).await.ok(); + avatar_url = services.users.avatar_url(&body.user_id).await.ok(); + blurhash = services.users.blurhash(&body.user_id).await.ok(); + tz = services.users.timezone(&body.user_id).await.ok(); custom_profile_fields = services .users .all_profile_keys(&body.user_id) - .filter_map(Result::ok) - .collect(); + .collect() + .await; }, } diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 15f82faa..bb424988 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -2,7 +2,8 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{debug, debug_warn, err, trace, warn, Err}; +use conduit::{debug, debug_warn, err, result::LogErr, trace, utils::ReadyExt, warn, Err, Error, Result}; +use futures::StreamExt; use ruma::{ api::{ client::error::ErrorKind, @@ -23,10 +24,13 @@ use tokio::sync::RwLock; use crate::{ services::Services, utils::{self}, - Error, Result, Ruma, + Ruma, }; -type ResolvedMap = BTreeMap>; +const PDU_LIMIT: usize = 50; +const EDU_LIMIT: usize = 100; + +type ResolvedMap = BTreeMap>; /// # `PUT /_matrix/federation/v1/send/{txnId}` /// @@ -44,12 +48,16 @@ pub(crate) async fn send_transaction_message_route( ))); } - if body.pdus.len() > 50_usize { - return Err!(Request(Forbidden("Not allowed to send more than 50 PDUs in one transaction"))); + if body.pdus.len() > PDU_LIMIT { + return Err!(Request(Forbidden( + "Not allowed to send more than {PDU_LIMIT} PDUs in one transaction" + ))); } - if body.edus.len() > 100_usize { - return Err!(Request(Forbidden("Not allowed to send more than 100 EDUs in one transaction"))); + if body.edus.len() > EDU_LIMIT { + return Err!(Request(Forbidden( + "Not allowed to send more than {EDU_LIMIT} EDUs in one transaction" + ))); } let txn_start_time = Instant::now(); @@ -62,8 +70,8 @@ pub(crate) async fn send_transaction_message_route( "Starting txn", ); - let resolved_map = handle_pdus(&services, &client, &body, origin, &txn_start_time).await?; - handle_edus(&services, &client, &body, origin).await?; + let resolved_map = handle_pdus(&services, &client, &body, origin, &txn_start_time).await; + handle_edus(&services, &client, &body, origin).await; debug!( pdus = ?body.pdus.len(), @@ -85,10 +93,10 @@ pub(crate) async fn send_transaction_message_route( async fn handle_pdus( services: &Services, _client: &IpAddr, body: &Ruma, origin: &ServerName, txn_start_time: &Instant, -) -> Result { +) -> ResolvedMap { let mut parsed_pdus = Vec::with_capacity(body.pdus.len()); for pdu in &body.pdus { - parsed_pdus.push(match services.rooms.event_handler.parse_incoming_pdu(pdu) { + parsed_pdus.push(match services.rooms.event_handler.parse_incoming_pdu(pdu).await { Ok(t) => t, Err(e) => { debug_warn!("Could not parse PDU: {e}"); @@ -151,38 +159,34 @@ async fn handle_pdus( } } - Ok(resolved_map) + resolved_map } async fn handle_edus( services: &Services, client: &IpAddr, body: &Ruma, origin: &ServerName, -) -> Result<()> { +) { for edu in body .edus .iter() .filter_map(|edu| serde_json::from_str::(edu.json().get()).ok()) { match edu { - Edu::Presence(presence) => handle_edu_presence(services, client, origin, presence).await?, - Edu::Receipt(receipt) => handle_edu_receipt(services, client, origin, receipt).await?, - Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await?, - Edu::DeviceListUpdate(content) => handle_edu_device_list_update(services, client, origin, content).await?, - Edu::DirectToDevice(content) => handle_edu_direct_to_device(services, client, origin, content).await?, - Edu::SigningKeyUpdate(content) => handle_edu_signing_key_update(services, client, origin, content).await?, + Edu::Presence(presence) => handle_edu_presence(services, client, origin, presence).await, + Edu::Receipt(receipt) => handle_edu_receipt(services, client, origin, receipt).await, + Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await, + Edu::DeviceListUpdate(content) => handle_edu_device_list_update(services, client, origin, content).await, + Edu::DirectToDevice(content) => handle_edu_direct_to_device(services, client, origin, content).await, + Edu::SigningKeyUpdate(content) => handle_edu_signing_key_update(services, client, origin, content).await, Edu::_Custom(ref _custom) => { debug_warn!(?body.edus, "received custom/unknown EDU"); }, } } - - Ok(()) } -async fn handle_edu_presence( - services: &Services, _client: &IpAddr, origin: &ServerName, presence: PresenceContent, -) -> Result<()> { +async fn handle_edu_presence(services: &Services, _client: &IpAddr, origin: &ServerName, presence: PresenceContent) { if !services.globals.allow_incoming_presence() { - return Ok(()); + return; } for update in presence.push { @@ -194,23 +198,24 @@ async fn handle_edu_presence( continue; } - services.presence.set_presence( - &update.user_id, - &update.presence, - Some(update.currently_active), - Some(update.last_active_ago), - update.status_msg.clone(), - )?; + services + .presence + .set_presence( + &update.user_id, + &update.presence, + Some(update.currently_active), + Some(update.last_active_ago), + update.status_msg.clone(), + ) + .await + .log_err() + .ok(); } - - Ok(()) } -async fn handle_edu_receipt( - services: &Services, _client: &IpAddr, origin: &ServerName, receipt: ReceiptContent, -) -> Result<()> { +async fn handle_edu_receipt(services: &Services, _client: &IpAddr, origin: &ServerName, receipt: ReceiptContent) { if !services.globals.allow_incoming_read_receipts() { - return Ok(()); + return; } for (room_id, room_updates) in receipt.receipts { @@ -218,6 +223,7 @@ async fn handle_edu_receipt( .rooms .event_handler .acl_check(origin, &room_id) + .await .is_err() { debug_warn!( @@ -240,8 +246,8 @@ async fn handle_edu_receipt( .rooms .state_cache .room_members(&room_id) - .filter_map(Result::ok) - .any(|member| member.server_name() == user_id.server_name()) + .ready_any(|member| member.server_name() == user_id.server_name()) + .await { for event_id in &user_updates.event_ids { let user_receipts = BTreeMap::from([(user_id.clone(), user_updates.data.clone())]); @@ -255,7 +261,8 @@ async fn handle_edu_receipt( services .rooms .read_receipt - .readreceipt_update(&user_id, &room_id, &event)?; + .readreceipt_update(&user_id, &room_id, &event) + .await; } } else { debug_warn!( @@ -266,15 +273,11 @@ async fn handle_edu_receipt( } } } - - Ok(()) } -async fn handle_edu_typing( - services: &Services, _client: &IpAddr, origin: &ServerName, typing: TypingContent, -) -> Result<()> { +async fn handle_edu_typing(services: &Services, _client: &IpAddr, origin: &ServerName, typing: TypingContent) { if !services.globals.config.allow_incoming_typing { - return Ok(()); + return; } if typing.user_id.server_name() != origin { @@ -282,26 +285,28 @@ async fn handle_edu_typing( %typing.user_id, %origin, "received typing EDU for user not belonging to origin" ); - return Ok(()); + return; } if services .rooms .event_handler .acl_check(typing.user_id.server_name(), &typing.room_id) + .await .is_err() { debug_warn!( %typing.user_id, %typing.room_id, %origin, "received typing EDU for ACL'd user's server" ); - return Ok(()); + return; } if services .rooms .state_cache - .is_joined(&typing.user_id, &typing.room_id)? + .is_joined(&typing.user_id, &typing.room_id) + .await { if typing.typing { let timeout = utils::millis_since_unix_epoch().saturating_add( @@ -315,28 +320,29 @@ async fn handle_edu_typing( .rooms .typing .typing_add(&typing.user_id, &typing.room_id, timeout) - .await?; + .await + .log_err() + .ok(); } else { services .rooms .typing .typing_remove(&typing.user_id, &typing.room_id) - .await?; + .await + .log_err() + .ok(); } } else { debug_warn!( %typing.user_id, %typing.room_id, %origin, "received typing EDU for user not in room" ); - return Ok(()); } - - Ok(()) } async fn handle_edu_device_list_update( services: &Services, _client: &IpAddr, origin: &ServerName, content: DeviceListUpdateContent, -) -> Result<()> { +) { let DeviceListUpdateContent { user_id, .. @@ -347,17 +353,15 @@ async fn handle_edu_device_list_update( %user_id, %origin, "received device list update EDU for user not belonging to origin" ); - return Ok(()); + return; } - services.users.mark_device_key_update(&user_id)?; - - Ok(()) + services.users.mark_device_key_update(&user_id).await; } async fn handle_edu_direct_to_device( services: &Services, _client: &IpAddr, origin: &ServerName, content: DirectDeviceContent, -) -> Result<()> { +) { let DirectDeviceContent { sender, ev_type, @@ -370,45 +374,52 @@ async fn handle_edu_direct_to_device( %sender, %origin, "received direct to device EDU for user not belonging to origin" ); - return Ok(()); + return; } // Check if this is a new transaction id if services .transaction_ids - .existing_txnid(&sender, None, &message_id)? - .is_some() + .existing_txnid(&sender, None, &message_id) + .await + .is_ok() { - return Ok(()); + return; } for (target_user_id, map) in &messages { for (target_device_id_maybe, event) in map { + let Ok(event) = event + .deserialize_as() + .map_err(|e| err!(Request(InvalidParam(error!("To-Device event is invalid: {e}"))))) + else { + continue; + }; + + let ev_type = ev_type.to_string(); match target_device_id_maybe { DeviceIdOrAllDevices::DeviceId(target_device_id) => { - services.users.add_to_device_event( - &sender, - target_user_id, - target_device_id, - &ev_type.to_string(), - event - .deserialize_as() - .map_err(|e| err!(Request(InvalidParam(error!("To-Device event is invalid: {e}")))))?, - )?; + services + .users + .add_to_device_event(&sender, target_user_id, target_device_id, &ev_type, event) + .await; }, DeviceIdOrAllDevices::AllDevices => { - for target_device_id in services.users.all_device_ids(target_user_id) { - services.users.add_to_device_event( - &sender, - target_user_id, - &target_device_id?, - &ev_type.to_string(), - event - .deserialize_as() - .map_err(|e| err!(Request(InvalidParam("Event is invalid: {e}"))))?, - )?; - } + let (sender, ev_type, event) = (&sender, &ev_type, &event); + services + .users + .all_device_ids(target_user_id) + .for_each(|target_device_id| { + services.users.add_to_device_event( + sender, + target_user_id, + target_device_id, + ev_type, + event.clone(), + ) + }) + .await; }, } } @@ -417,14 +428,12 @@ async fn handle_edu_direct_to_device( // Save transaction id with empty data services .transaction_ids - .add_txnid(&sender, None, &message_id, &[])?; - - Ok(()) + .add_txnid(&sender, None, &message_id, &[]); } async fn handle_edu_signing_key_update( services: &Services, _client: &IpAddr, origin: &ServerName, content: SigningKeyUpdateContent, -) -> Result<()> { +) { let SigningKeyUpdateContent { user_id, master_key, @@ -436,14 +445,15 @@ async fn handle_edu_signing_key_update( %user_id, %origin, "received signing key update EDU from server that does not belong to user's server" ); - return Ok(()); + return; } if let Some(master_key) = master_key { services .users - .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true)?; + .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true) + .await + .log_err() + .ok(); } - - Ok(()) } diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index c4d016f6..639fcafd 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -3,7 +3,8 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{pdu::gen_event_id_canonical_json, warn, Error, Result}; +use conduit::{err, pdu::gen_event_id_canonical_json, utils::IterStream, warn, Error, Result}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_join_event}, events::{ @@ -22,27 +23,32 @@ use crate::Ruma; async fn create_join_event( services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, ) -> Result { - if !services.rooms.metadata.exists(room_id)? { + if !services.rooms.metadata.exists(room_id).await { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } // ACL check origin server - services.rooms.event_handler.acl_check(origin, room_id)?; + services + .rooms + .event_handler + .acl_check(origin, room_id) + .await?; // We need to return the state prior to joining, let's keep a reference to that // here let shortstatehash = services .rooms .state - .get_room_shortstatehash(room_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Event state not found."))?; + .get_room_shortstatehash(room_id) + .await + .map_err(|_| err!(Request(NotFound("Event state not found."))))?; let pub_key_map = RwLock::new(BTreeMap::new()); // let mut auth_cache = EventMap::new(); // We do not add the event_id field to the pdu here because of signature and // hashes checks - let room_version_id = services.rooms.state.get_room_version(room_id)?; + let room_version_id = services.rooms.state.get_room_version(room_id).await?; let Ok((event_id, mut value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json @@ -97,7 +103,8 @@ async fn create_join_event( services .rooms .event_handler - .acl_check(sender.server_name(), room_id)?; + .acl_check(sender.server_name(), room_id) + .await?; // check if origin server is trying to send for another server if sender.server_name() != origin { @@ -126,7 +133,9 @@ async fn create_join_event( if content .join_authorized_via_users_server .is_some_and(|user| services.globals.user_is_local(&user)) - && super::user_can_perform_restricted_join(services, &sender, room_id, &room_version_id).unwrap_or_default() + && super::user_can_perform_restricted_join(services, &sender, room_id, &room_version_id) + .await + .unwrap_or_default() { ruma::signatures::hash_and_sign_event( services.globals.server_name().as_str(), @@ -158,12 +167,14 @@ async fn create_join_event( .mutex_federation .lock(room_id) .await; + let pdu_id: Vec = services .rooms .event_handler .handle_incoming_pdu(&origin, room_id, &event_id, value.clone(), true, &pub_key_map) .await? .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; + drop(mutex_lock); let state_ids = services @@ -171,29 +182,43 @@ async fn create_join_event( .state_accessor .state_full_ids(shortstatehash) .await?; - let auth_chain_ids = services + + let state = state_ids + .iter() + .try_stream() + .and_then(|(_, event_id)| services.rooms.timeline.get_pdu_json(event_id)) + .and_then(|pdu| { + services + .sending + .convert_to_outgoing_federation_event(pdu) + .map(Ok) + }) + .try_collect() + .await?; + + let auth_chain = services .rooms .auth_chain .event_ids_iter(room_id, state_ids.values().cloned().collect()) + .await? + .map(Ok) + .and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) + .and_then(|pdu| { + services + .sending + .convert_to_outgoing_federation_event(pdu) + .map(Ok) + }) + .try_collect() .await?; - services.sending.send_pdu_room(room_id, &pdu_id)?; + services.sending.send_pdu_room(room_id, &pdu_id).await?; Ok(create_join_event::v1::RoomState { - auth_chain: auth_chain_ids - .filter_map(|id| services.rooms.timeline.get_pdu_json(&id).ok().flatten()) - .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) - .collect(), - state: state_ids - .iter() - .filter_map(|(_, id)| services.rooms.timeline.get_pdu_json(id).ok().flatten()) - .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) - .collect(), + auth_chain, + state, // Event field is required if the room version supports restricted join rules. - event: Some( - to_raw_value(&CanonicalJsonValue::Object(value)) - .expect("To raw json should not fail since only change was adding signature"), - ), + event: to_raw_value(&CanonicalJsonValue::Object(value)).ok(), }) } diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index e77c5d78..81f41af0 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{Error, Result}; +use conduit::{utils::ReadyExt, Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_leave_event}, events::{ @@ -49,18 +49,22 @@ pub(crate) async fn create_leave_event_v2_route( async fn create_leave_event( services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, ) -> Result<()> { - if !services.rooms.metadata.exists(room_id)? { + if !services.rooms.metadata.exists(room_id).await { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } // ACL check origin - services.rooms.event_handler.acl_check(origin, room_id)?; + services + .rooms + .event_handler + .acl_check(origin, room_id) + .await?; let pub_key_map = RwLock::new(BTreeMap::new()); // We do not add the event_id field to the pdu here because of signature and // hashes checks - let room_version_id = services.rooms.state.get_room_version(room_id)?; + let room_version_id = services.rooms.state.get_room_version(room_id).await?; let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json return Err(Error::BadRequest( @@ -114,7 +118,8 @@ async fn create_leave_event( services .rooms .event_handler - .acl_check(sender.server_name(), room_id)?; + .acl_check(sender.server_name(), room_id) + .await?; if sender.server_name() != origin { return Err(Error::BadRequest( @@ -173,10 +178,9 @@ async fn create_leave_event( .rooms .state_cache .room_servers(room_id) - .filter_map(Result::ok) - .filter(|server| !services.globals.server_is_ours(server)); + .ready_filter(|server| !services.globals.server_is_ours(server)); - services.sending.send_pdu_servers(servers, &pdu_id)?; + services.sending.send_pdu_servers(servers, &pdu_id).await?; Ok(()) } diff --git a/src/api/server/state.rs b/src/api/server/state.rs index d215236a..37a14a3f 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -1,8 +1,9 @@ use std::sync::Arc; use axum::extract::State; -use conduit::{Error, Result}; -use ruma::api::{client::error::ErrorKind, federation::event::get_room_state}; +use conduit::{err, result::LogErr, utils::IterStream, Err, Result}; +use futures::{FutureExt, StreamExt, TryStreamExt}; +use ruma::api::federation::event::get_room_state; use crate::Ruma; @@ -17,56 +18,66 @@ pub(crate) async fn get_room_state_route( services .rooms .event_handler - .acl_check(origin, &body.room_id)?; + .acl_check(origin, &body.room_id) + .await?; if !services .rooms .state_accessor - .is_world_readable(&body.room_id)? - && !services - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? + .is_world_readable(&body.room_id) + .await && !services + .rooms + .state_cache + .server_in_room(origin, &body.room_id) + .await { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + return Err!(Request(Forbidden("Server is not in room."))); } let shortstatehash = services .rooms .state_accessor - .pdu_shortstatehash(&body.event_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Pdu state not found."))?; + .pdu_shortstatehash(&body.event_id) + .await + .map_err(|_| err!(Request(NotFound("PDU state not found."))))?; let pdus = services .rooms .state_accessor .state_full_ids(shortstatehash) - .await? - .into_values() - .map(|id| { + .await + .log_err() + .map_err(|_| err!(Request(NotFound("PDU state IDs not found."))))? + .values() + .try_stream() + .and_then(|id| services.rooms.timeline.get_pdu_json(id)) + .and_then(|pdu| { services .sending - .convert_to_outgoing_federation_event(services.rooms.timeline.get_pdu_json(&id).unwrap().unwrap()) + .convert_to_outgoing_federation_event(pdu) + .map(Ok) }) - .collect(); + .try_collect() + .await?; - let auth_chain_ids = services + let auth_chain = services .rooms .auth_chain .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) + .await? + .map(Ok) + .and_then(|id| async move { services.rooms.timeline.get_pdu_json(&id).await }) + .and_then(|pdu| { + services + .sending + .convert_to_outgoing_federation_event(pdu) + .map(Ok) + }) + .try_collect() .await?; Ok(get_room_state::v1::Response { - auth_chain: auth_chain_ids - .filter_map(|id| { - services - .rooms - .timeline - .get_pdu_json(&id) - .ok()? - .map(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) - }) - .collect(), + auth_chain, pdus, }) } diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index d22f2df4..95ca65aa 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -1,9 +1,11 @@ use std::sync::Arc; use axum::extract::State; -use ruma::api::{client::error::ErrorKind, federation::event::get_room_state_ids}; +use conduit::{err, Err}; +use futures::StreamExt; +use ruma::api::federation::event::get_room_state_ids; -use crate::{Error, Result, Ruma}; +use crate::{Result, Ruma}; /// # `GET /_matrix/federation/v1/state_ids/{roomId}` /// @@ -17,31 +19,35 @@ pub(crate) async fn get_room_state_ids_route( services .rooms .event_handler - .acl_check(origin, &body.room_id)?; + .acl_check(origin, &body.room_id) + .await?; if !services .rooms .state_accessor - .is_world_readable(&body.room_id)? - && !services - .rooms - .state_cache - .server_in_room(origin, &body.room_id)? + .is_world_readable(&body.room_id) + .await && !services + .rooms + .state_cache + .server_in_room(origin, &body.room_id) + .await { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + return Err!(Request(Forbidden("Server is not in room."))); } let shortstatehash = services .rooms .state_accessor - .pdu_shortstatehash(&body.event_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Pdu state not found."))?; + .pdu_shortstatehash(&body.event_id) + .await + .map_err(|_| err!(Request(NotFound("Pdu state not found."))))?; let pdu_ids = services .rooms .state_accessor .state_full_ids(shortstatehash) - .await? + .await + .map_err(|_| err!(Request(NotFound("State ids not found"))))? .into_values() .map(|id| (*id).to_owned()) .collect(); @@ -50,10 +56,13 @@ pub(crate) async fn get_room_state_ids_route( .rooms .auth_chain .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) - .await?; + .await? + .map(|id| (*id).to_owned()) + .collect() + .await; Ok(get_room_state_ids::v1::Response { - auth_chain_ids: auth_chain_ids.map(|id| (*id).to_owned()).collect(), + auth_chain_ids, pdu_ids, }) } diff --git a/src/api/server/user.rs b/src/api/server/user.rs index e9a400a7..0718da58 100644 --- a/src/api/server/user.rs +++ b/src/api/server/user.rs @@ -1,5 +1,6 @@ use axum::extract::State; use conduit::{Error, Result}; +use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::api::{ client::error::ErrorKind, federation::{ @@ -28,41 +29,51 @@ pub(crate) async fn get_devices_route( let origin = body.origin.as_ref().expect("server is authenticated"); + let user_id = &body.user_id; Ok(get_devices::v1::Response { - user_id: body.user_id.clone(), + user_id: user_id.clone(), stream_id: services .users - .get_devicelist_version(&body.user_id)? + .get_devicelist_version(user_id) + .await .unwrap_or(0) - .try_into() - .expect("version will not grow that large"), + .try_into()?, devices: services .users - .all_devices_metadata(&body.user_id) - .filter_map(Result::ok) - .filter_map(|metadata| { - let device_id_string = metadata.device_id.as_str().to_owned(); + .all_devices_metadata(user_id) + .filter_map(|metadata| async move { + let device_id = metadata.device_id.clone(); + let device_id_clone = device_id.clone(); + let device_id_string = device_id.as_str().to_owned(); let device_display_name = if services.globals.allow_device_name_federation() { - metadata.display_name + metadata.display_name.clone() } else { Some(device_id_string) }; - Some(UserDevice { - keys: services - .users - .get_device_keys(&body.user_id, &metadata.device_id) - .ok()??, - device_id: metadata.device_id, - device_display_name, - }) + + services + .users + .get_device_keys(user_id, &device_id_clone) + .map_ok(|keys| UserDevice { + device_id, + keys, + device_display_name, + }) + .map(Result::ok) + .await }) - .collect(), + .collect() + .await, master_key: services .users - .get_master_key(None, &body.user_id, &|u| u.server_name() == origin)?, + .get_master_key(None, &body.user_id, &|u| u.server_name() == origin) + .await + .ok(), self_signing_key: services .users - .get_self_signing_key(None, &body.user_id, &|u| u.server_name() == origin)?, + .get_self_signing_key(None, &body.user_id, &|u| u.server_name() == origin) + .await + .ok(), }) } diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 71364734..cb957bc9 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -67,6 +67,7 @@ ctor.workspace = true cyborgtime.workspace = true either.workspace = true figment.workspace = true +futures.workspace = true http-body-util.workspace = true http.workspace = true image.workspace = true diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 48b9b58f..79e3d5b4 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -86,7 +86,7 @@ pub enum Error { #[error("There was a problem with the '{0}' directive in your configuration: {1}")] Config(&'static str, Cow<'static, str>), #[error("{0}")] - Conflict(&'static str), // This is only needed for when a room alias already exists + Conflict(Cow<'static, str>), // This is only needed for when a room alias already exists #[error(transparent)] ContentDisposition(#[from] ruma::http_headers::ContentDispositionParseError), #[error("{0}")] @@ -107,6 +107,8 @@ pub enum Error { Request(ruma::api::client::error::ErrorKind, Cow<'static, str>, http::StatusCode), #[error(transparent)] Ruma(#[from] ruma::api::client::error::Error), + #[error(transparent)] + StateRes(#[from] ruma::state_res::Error), #[error("uiaa")] Uiaa(ruma::api::client::uiaa::UiaaInfo), diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 439c831a..cf9ffe64 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -3,8 +3,6 @@ mod count; use std::{cmp::Ordering, collections::BTreeMap, sync::Arc}; -pub use builder::PduBuilder; -pub use count::PduCount; use ruma::{ canonical_json::redact_content_in_place, events::{ @@ -23,7 +21,8 @@ use serde_json::{ value::{to_raw_value, RawValue as RawJsonValue}, }; -use crate::{err, warn, Error}; +pub use self::{builder::PduBuilder, count::PduCount}; +use crate::{err, warn, Error, Result}; #[derive(Deserialize)] struct ExtractRedactedBecause { @@ -65,11 +64,12 @@ pub struct PduEvent { impl PduEvent { #[tracing::instrument(skip(self), level = "debug")] - pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> crate::Result<()> { + pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> Result<()> { self.unsigned = None; let mut content = serde_json::from_str(self.content.get()) .map_err(|_| Error::bad_database("PDU in db has invalid content."))?; + redact_content_in_place(&mut content, &room_version_id, self.kind.to_string()) .map_err(|e| Error::Redaction(self.sender.server_name().to_owned(), e))?; @@ -98,31 +98,38 @@ impl PduEvent { unsigned.redacted_because.is_some() } - pub fn remove_transaction_id(&mut self) -> crate::Result<()> { - if let Some(unsigned) = &self.unsigned { - let mut unsigned: BTreeMap> = serde_json::from_str(unsigned.get()) - .map_err(|_| Error::bad_database("Invalid unsigned in pdu event"))?; - unsigned.remove("transaction_id"); - self.unsigned = Some(to_raw_value(&unsigned).expect("unsigned is valid")); - } + pub fn remove_transaction_id(&mut self) -> Result<()> { + let Some(unsigned) = &self.unsigned else { + return Ok(()); + }; + + let mut unsigned: BTreeMap> = + serde_json::from_str(unsigned.get()).map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; + + unsigned.remove("transaction_id"); + self.unsigned = to_raw_value(&unsigned) + .map(Some) + .expect("unsigned is valid"); Ok(()) } - pub fn add_age(&mut self) -> crate::Result<()> { + pub fn add_age(&mut self) -> Result<()> { let mut unsigned: BTreeMap> = self .unsigned .as_ref() .map_or_else(|| Ok(BTreeMap::new()), |u| serde_json::from_str(u.get())) - .map_err(|_| Error::bad_database("Invalid unsigned in pdu event"))?; + .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; // deliberately allowing for the possibility of negative age let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into(); let then: i128 = self.origin_server_ts.into(); let this_age = now.saturating_sub(then); - unsigned.insert("age".to_owned(), to_raw_value(&this_age).unwrap()); - self.unsigned = Some(to_raw_value(&unsigned).expect("unsigned is valid")); + unsigned.insert("age".to_owned(), to_raw_value(&this_age).expect("age is valid")); + self.unsigned = to_raw_value(&unsigned) + .map(Some) + .expect("unsigned is valid"); Ok(()) } @@ -369,9 +376,9 @@ impl state_res::Event for PduEvent { fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } - fn prev_events(&self) -> Box + '_> { Box::new(self.prev_events.iter()) } + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { self.prev_events.iter() } - fn auth_events(&self) -> Box + '_> { Box::new(self.auth_events.iter()) } + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { self.auth_events.iter() } fn redacts(&self) -> Option<&Self::Id> { self.redacts.as_ref() } } @@ -395,7 +402,7 @@ impl Ord for PduEvent { /// CanonicalJsonValue>`. pub fn gen_event_id_canonical_json( pdu: &RawJsonValue, room_version_id: &RoomVersionId, -) -> crate::Result<(OwnedEventId, CanonicalJsonObject)> { +) -> Result<(OwnedEventId, CanonicalJsonObject)> { let value: CanonicalJsonObject = serde_json::from_str(pdu.get()) .map_err(|e| err!(BadServerResponse(warn!("Error parsing incoming event: {e:?}"))))?; diff --git a/src/core/result/log_debug_err.rs b/src/core/result/log_debug_err.rs index be2000ae..8835afd1 100644 --- a/src/core/result/log_debug_err.rs +++ b/src/core/result/log_debug_err.rs @@ -1,18 +1,14 @@ -use std::fmt; +use std::fmt::Debug; use tracing::Level; use super::{DebugInspect, Result}; use crate::error; -pub trait LogDebugErr -where - E: fmt::Debug, -{ +pub trait LogDebugErr { #[must_use] fn err_debug_log(self, level: Level) -> Self; - #[inline] #[must_use] fn log_debug_err(self) -> Self where @@ -22,15 +18,9 @@ where } } -impl LogDebugErr for Result -where - E: fmt::Debug, -{ +impl LogDebugErr for Result { #[inline] - fn err_debug_log(self, level: Level) -> Self - where - Self: Sized, - { + fn err_debug_log(self, level: Level) -> Self { self.debug_inspect_err(|error| error::inspect_debug_log_level(&error, level)) } } diff --git a/src/core/result/log_err.rs b/src/core/result/log_err.rs index 079571f5..374a5e59 100644 --- a/src/core/result/log_err.rs +++ b/src/core/result/log_err.rs @@ -1,18 +1,14 @@ -use std::fmt; +use std::fmt::Display; use tracing::Level; use super::Result; use crate::error; -pub trait LogErr -where - E: fmt::Display, -{ +pub trait LogErr { #[must_use] fn err_log(self, level: Level) -> Self; - #[inline] #[must_use] fn log_err(self) -> Self where @@ -22,15 +18,7 @@ where } } -impl LogErr for Result -where - E: fmt::Display, -{ +impl LogErr for Result { #[inline] - fn err_log(self, level: Level) -> Self - where - Self: Sized, - { - self.inspect_err(|error| error::inspect_log_level(&error, level)) - } + fn err_log(self, level: Level) -> Self { self.inspect_err(|error| error::inspect_log_level(&error, level)) } } diff --git a/src/core/utils/algorithm.rs b/src/core/utils/algorithm.rs deleted file mode 100644 index 9bc1bc8a..00000000 --- a/src/core/utils/algorithm.rs +++ /dev/null @@ -1,25 +0,0 @@ -use std::cmp::Ordering; - -#[allow(clippy::impl_trait_in_params)] -pub fn common_elements( - mut iterators: impl Iterator>>, check_order: impl Fn(&[u8], &[u8]) -> Ordering, -) -> Option>> { - let first_iterator = iterators.next()?; - let mut other_iterators = iterators.map(Iterator::peekable).collect::>(); - - Some(first_iterator.filter(move |target| { - other_iterators.iter_mut().all(|it| { - while let Some(element) = it.peek() { - match check_order(element, target) { - Ordering::Greater => return false, // We went too far - Ordering::Equal => return true, // Element is in both iters - Ordering::Less => { - // Keep searching - it.next(); - }, - } - } - false - }) - })) -} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 03b755e9..b1ea3709 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -1,4 +1,3 @@ -pub mod algorithm; pub mod bytes; pub mod content_disposition; pub mod debug; @@ -9,25 +8,30 @@ pub mod json; pub mod math; pub mod mutex_map; pub mod rand; +pub mod set; +pub mod stream; pub mod string; pub mod sys; mod tests; pub mod time; +pub use ::conduit_macros::implement; pub use ::ctor::{ctor, dtor}; -pub use algorithm::common_elements; -pub use bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}; -pub use conduit_macros::implement; -pub use debug::slice_truncated as debug_slice_truncated; -pub use hash::calculate_hash; -pub use html::Escape as HtmlEscape; -pub use json::{deserialize_from_str, to_canonical_object}; -pub use math::clamp; -pub use mutex_map::{Guard as MutexMapGuard, MutexMap}; -pub use rand::string as random_string; -pub use string::{str_from_bytes, string_from_bytes}; -pub use sys::available_parallelism; -pub use time::now_millis as millis_since_unix_epoch; + +pub use self::{ + bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}, + debug::slice_truncated as debug_slice_truncated, + hash::calculate_hash, + html::Escape as HtmlEscape, + json::{deserialize_from_str, to_canonical_object}, + math::clamp, + mutex_map::{Guard as MutexMapGuard, MutexMap}, + rand::string as random_string, + stream::{IterStream, ReadyExt, TryReadyExt}, + string::{str_from_bytes, string_from_bytes}, + sys::available_parallelism, + time::now_millis as millis_since_unix_epoch, +}; #[inline] pub fn exchange(state: &mut T, source: T) -> T { std::mem::replace(state, source) } diff --git a/src/core/utils/set.rs b/src/core/utils/set.rs new file mode 100644 index 00000000..563f9df5 --- /dev/null +++ b/src/core/utils/set.rs @@ -0,0 +1,47 @@ +use std::cmp::{Eq, Ord}; + +use crate::{is_equal_to, is_less_than}; + +/// Intersection of sets +/// +/// Outputs the set of elements common to all input sets. Inputs do not have to +/// be sorted. If inputs are sorted a more optimized function is available in +/// this suite and should be used. +pub fn intersection(mut input: Iters) -> impl Iterator + Send +where + Iters: Iterator + Clone + Send, + Iter: Iterator + Send, + Item: Eq + Send, +{ + input.next().into_iter().flat_map(move |first| { + let input = input.clone(); + first.filter(move |targ| { + input + .clone() + .all(|mut other| other.any(is_equal_to!(*targ))) + }) + }) +} + +/// Intersection of sets +/// +/// Outputs the set of elements common to all input sets. Inputs must be sorted. +pub fn intersection_sorted(mut input: Iters) -> impl Iterator + Send +where + Iters: Iterator + Clone + Send, + Iter: Iterator + Send, + Item: Eq + Ord + Send, +{ + input.next().into_iter().flat_map(move |first| { + let mut input = input.clone().collect::>(); + first.filter(move |targ| { + input.iter_mut().all(|it| { + it.by_ref() + .skip_while(is_less_than!(targ)) + .peekable() + .peek() + .is_some_and(is_equal_to!(targ)) + }) + }) + }) +} diff --git a/src/core/utils/stream/cloned.rs b/src/core/utils/stream/cloned.rs new file mode 100644 index 00000000..d6a0e647 --- /dev/null +++ b/src/core/utils/stream/cloned.rs @@ -0,0 +1,20 @@ +use std::clone::Clone; + +use futures::{stream::Map, Stream, StreamExt}; + +pub trait Cloned<'a, T, S> +where + S: Stream, + T: Clone + 'a, +{ + fn cloned(self) -> Map T>; +} + +impl<'a, T, S> Cloned<'a, T, S> for S +where + S: Stream, + T: Clone + 'a, +{ + #[inline] + fn cloned(self) -> Map T> { self.map(Clone::clone) } +} diff --git a/src/core/utils/stream/expect.rs b/src/core/utils/stream/expect.rs new file mode 100644 index 00000000..3ab7181a --- /dev/null +++ b/src/core/utils/stream/expect.rs @@ -0,0 +1,17 @@ +use futures::{Stream, StreamExt, TryStream}; + +use crate::Result; + +pub trait TryExpect<'a, Item> { + fn expect_ok(self) -> impl Stream + Send + 'a; +} + +impl<'a, T, Item> TryExpect<'a, Item> for T +where + T: Stream> + TryStream + Send + 'a, +{ + #[inline] + fn expect_ok(self: T) -> impl Stream + Send + 'a { + self.map(|res| res.expect("stream expectation failure")) + } +} diff --git a/src/core/utils/stream/ignore.rs b/src/core/utils/stream/ignore.rs new file mode 100644 index 00000000..997aa4ba --- /dev/null +++ b/src/core/utils/stream/ignore.rs @@ -0,0 +1,21 @@ +use futures::{future::ready, Stream, StreamExt, TryStream}; + +use crate::{Error, Result}; + +pub trait TryIgnore<'a, Item> { + fn ignore_err(self) -> impl Stream + Send + 'a; + + fn ignore_ok(self) -> impl Stream + Send + 'a; +} + +impl<'a, T, Item> TryIgnore<'a, Item> for T +where + T: Stream> + TryStream + Send + 'a, + Item: Send + 'a, +{ + #[inline] + fn ignore_err(self: T) -> impl Stream + Send + 'a { self.filter_map(|res| ready(res.ok())) } + + #[inline] + fn ignore_ok(self: T) -> impl Stream + Send + 'a { self.filter_map(|res| ready(res.err())) } +} diff --git a/src/core/utils/stream/iter_stream.rs b/src/core/utils/stream/iter_stream.rs new file mode 100644 index 00000000..69edf64f --- /dev/null +++ b/src/core/utils/stream/iter_stream.rs @@ -0,0 +1,27 @@ +use futures::{ + stream, + stream::{Stream, TryStream}, + StreamExt, +}; + +pub trait IterStream { + /// Convert an Iterator into a Stream + fn stream(self) -> impl Stream::Item> + Send; + + /// Convert an Iterator into a TryStream + fn try_stream(self) -> impl TryStream::Item, Error = crate::Error> + Send; +} + +impl IterStream for I +where + I: IntoIterator + Send, + ::IntoIter: Send, +{ + #[inline] + fn stream(self) -> impl Stream::Item> + Send { stream::iter(self) } + + #[inline] + fn try_stream(self) -> impl TryStream::Item, Error = crate::Error> + Send { + self.stream().map(Ok) + } +} diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs new file mode 100644 index 00000000..781bd522 --- /dev/null +++ b/src/core/utils/stream/mod.rs @@ -0,0 +1,13 @@ +mod cloned; +mod expect; +mod ignore; +mod iter_stream; +mod ready; +mod try_ready; + +pub use cloned::Cloned; +pub use expect::TryExpect; +pub use ignore::TryIgnore; +pub use iter_stream::IterStream; +pub use ready::ReadyExt; +pub use try_ready::TryReadyExt; diff --git a/src/core/utils/stream/ready.rs b/src/core/utils/stream/ready.rs new file mode 100644 index 00000000..13f730a7 --- /dev/null +++ b/src/core/utils/stream/ready.rs @@ -0,0 +1,109 @@ +//! Synchronous combinator extensions to futures::Stream + +use futures::{ + future::{ready, Ready}, + stream::{Any, Filter, FilterMap, Fold, ForEach, SkipWhile, Stream, StreamExt, TakeWhile}, +}; + +/// Synchronous combinators to augment futures::StreamExt. Most Stream +/// combinators take asynchronous arguments, but often only simple predicates +/// are required to steer a Stream like an Iterator. This suite provides a +/// convenience to reduce boilerplate by de-cluttering non-async predicates. +/// +/// This interface is not necessarily complete; feel free to add as-needed. +pub trait ReadyExt +where + S: Stream + Send + ?Sized, + Self: Stream + Send + Sized, +{ + fn ready_any(self, f: F) -> Any, impl FnMut(S::Item) -> Ready> + where + F: Fn(S::Item) -> bool; + + fn ready_filter<'a, F>(self, f: F) -> Filter, impl FnMut(&S::Item) -> Ready + 'a> + where + F: Fn(&S::Item) -> bool + 'a; + + fn ready_filter_map(self, f: F) -> FilterMap>, impl FnMut(S::Item) -> Ready>> + where + F: Fn(S::Item) -> Option; + + fn ready_fold(self, init: T, f: F) -> Fold, T, impl FnMut(T, S::Item) -> Ready> + where + F: Fn(T, S::Item) -> T; + + fn ready_for_each(self, f: F) -> ForEach, impl FnMut(S::Item) -> Ready<()>> + where + F: FnMut(S::Item); + + fn ready_take_while<'a, F>(self, f: F) -> TakeWhile, impl FnMut(&S::Item) -> Ready + 'a> + where + F: Fn(&S::Item) -> bool + 'a; + + fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile, impl FnMut(&S::Item) -> Ready + 'a> + where + F: Fn(&S::Item) -> bool + 'a; +} + +impl ReadyExt for S +where + S: Stream + Send + ?Sized, + Self: Stream + Send + Sized, +{ + #[inline] + fn ready_any(self, f: F) -> Any, impl FnMut(S::Item) -> Ready> + where + F: Fn(S::Item) -> bool, + { + self.any(move |t| ready(f(t))) + } + + #[inline] + fn ready_filter<'a, F>(self, f: F) -> Filter, impl FnMut(&S::Item) -> Ready + 'a> + where + F: Fn(&S::Item) -> bool + 'a, + { + self.filter(move |t| ready(f(t))) + } + + #[inline] + fn ready_filter_map(self, f: F) -> FilterMap>, impl FnMut(S::Item) -> Ready>> + where + F: Fn(S::Item) -> Option, + { + self.filter_map(move |t| ready(f(t))) + } + + #[inline] + fn ready_fold(self, init: T, f: F) -> Fold, T, impl FnMut(T, S::Item) -> Ready> + where + F: Fn(T, S::Item) -> T, + { + self.fold(init, move |a, t| ready(f(a, t))) + } + + #[inline] + #[allow(clippy::unit_arg)] + fn ready_for_each(self, mut f: F) -> ForEach, impl FnMut(S::Item) -> Ready<()>> + where + F: FnMut(S::Item), + { + self.for_each(move |t| ready(f(t))) + } + + #[inline] + fn ready_take_while<'a, F>(self, f: F) -> TakeWhile, impl FnMut(&S::Item) -> Ready + 'a> + where + F: Fn(&S::Item) -> bool + 'a, + { + self.take_while(move |t| ready(f(t))) + } + + #[inline] + fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile, impl FnMut(&S::Item) -> Ready + 'a> + where + F: Fn(&S::Item) -> bool + 'a, + { + self.skip_while(move |t| ready(f(t))) + } +} diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs new file mode 100644 index 00000000..ab37d9b3 --- /dev/null +++ b/src/core/utils/stream/try_ready.rs @@ -0,0 +1,35 @@ +//! Synchronous combinator extensions to futures::TryStream + +use futures::{ + future::{ready, Ready}, + stream::{AndThen, TryStream, TryStreamExt}, +}; + +use crate::Result; + +/// Synchronous combinators to augment futures::TryStreamExt. +/// +/// This interface is not necessarily complete; feel free to add as-needed. +pub trait TryReadyExt +where + S: TryStream> + Send + ?Sized, + Self: TryStream + Send + Sized, +{ + fn ready_and_then(self, f: F) -> AndThen>, impl FnMut(S::Ok) -> Ready>> + where + F: Fn(S::Ok) -> Result; +} + +impl TryReadyExt for S +where + S: TryStream> + Send + ?Sized, + Self: TryStream + Send + Sized, +{ + #[inline] + fn ready_and_then(self, f: F) -> AndThen>, impl FnMut(S::Ok) -> Ready>> + where + F: Fn(S::Ok) -> Result, + { + self.and_then(move |t| ready(f(t))) + } +} diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index 5880470a..84d35936 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -107,3 +107,133 @@ async fn mutex_map_contend() { tokio::try_join!(join_b, join_a).expect("joined"); assert!(map.is_empty(), "Must be empty"); } + +#[test] +#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)] +fn set_intersection_none() { + use utils::set::intersection; + + let a: [&str; 0] = []; + let b: [&str; 0] = []; + let i = [a.iter(), b.iter()]; + let r = intersection(i.into_iter()); + assert_eq!(r.count(), 0); + + let a: [&str; 0] = []; + let b = ["abc", "def"]; + let i = [a.iter(), b.iter()]; + let r = intersection(i.into_iter()); + assert_eq!(r.count(), 0); + let i = [b.iter(), a.iter()]; + let r = intersection(i.into_iter()); + assert_eq!(r.count(), 0); + let i = [a.iter()]; + let r = intersection(i.into_iter()); + assert_eq!(r.count(), 0); + + let a = ["foo", "bar", "baz"]; + let b = ["def", "hij", "klm", "nop"]; + let i = [a.iter(), b.iter()]; + let r = intersection(i.into_iter()); + assert_eq!(r.count(), 0); +} + +#[test] +#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)] +fn set_intersection_all() { + use utils::set::intersection; + + let a = ["foo"]; + let b = ["foo"]; + let i = [a.iter(), b.iter()]; + let r = intersection(i.into_iter()); + assert!(r.eq(["foo"].iter())); + + let a = ["foo", "bar"]; + let b = ["bar", "foo"]; + let i = [a.iter(), b.iter()]; + let r = intersection(i.into_iter()); + assert!(r.eq(["foo", "bar"].iter())); + let i = [b.iter()]; + let r = intersection(i.into_iter()); + assert!(r.eq(["bar", "foo"].iter())); + + let a = ["foo", "bar", "baz"]; + let b = ["baz", "foo", "bar"]; + let c = ["bar", "baz", "foo"]; + let i = [a.iter(), b.iter(), c.iter()]; + let r = intersection(i.into_iter()); + assert!(r.eq(["foo", "bar", "baz"].iter())); +} + +#[test] +#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)] +fn set_intersection_some() { + use utils::set::intersection; + + let a = ["foo"]; + let b = ["bar", "foo"]; + let i = [a.iter(), b.iter()]; + let r = intersection(i.into_iter()); + assert!(r.eq(["foo"].iter())); + let i = [b.iter(), a.iter()]; + let r = intersection(i.into_iter()); + assert!(r.eq(["foo"].iter())); + + let a = ["abcdef", "foo", "hijkl", "abc"]; + let b = ["hij", "bar", "baz", "abc", "foo"]; + let c = ["abc", "xyz", "foo", "ghi"]; + let i = [a.iter(), b.iter(), c.iter()]; + let r = intersection(i.into_iter()); + assert!(r.eq(["foo", "abc"].iter())); +} + +#[test] +#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)] +fn set_intersection_sorted_some() { + use utils::set::intersection_sorted; + + let a = ["bar"]; + let b = ["bar", "foo"]; + let i = [a.iter(), b.iter()]; + let r = intersection_sorted(i.into_iter()); + assert!(r.eq(["bar"].iter())); + let i = [b.iter(), a.iter()]; + let r = intersection_sorted(i.into_iter()); + assert!(r.eq(["bar"].iter())); + + let a = ["aaa", "ccc", "eee", "ggg"]; + let b = ["aaa", "bbb", "ccc", "ddd", "eee"]; + let c = ["bbb", "ccc", "eee", "fff"]; + let i = [a.iter(), b.iter(), c.iter()]; + let r = intersection_sorted(i.into_iter()); + assert!(r.eq(["ccc", "eee"].iter())); +} + +#[test] +#[allow(clippy::iter_on_single_items, clippy::many_single_char_names)] +fn set_intersection_sorted_all() { + use utils::set::intersection_sorted; + + let a = ["foo"]; + let b = ["foo"]; + let i = [a.iter(), b.iter()]; + let r = intersection_sorted(i.into_iter()); + assert!(r.eq(["foo"].iter())); + + let a = ["bar", "foo"]; + let b = ["bar", "foo"]; + let i = [a.iter(), b.iter()]; + let r = intersection_sorted(i.into_iter()); + assert!(r.eq(["bar", "foo"].iter())); + let i = [b.iter()]; + let r = intersection_sorted(i.into_iter()); + assert!(r.eq(["bar", "foo"].iter())); + + let a = ["bar", "baz", "foo"]; + let b = ["bar", "baz", "foo"]; + let c = ["bar", "baz", "foo"]; + let i = [a.iter(), b.iter(), c.iter()]; + let r = intersection_sorted(i.into_iter()); + assert!(r.eq(["bar", "baz", "foo"].iter())); +} diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 34d98416..b5eb7612 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -37,8 +37,11 @@ zstd_compression = [ [dependencies] conduit-core.workspace = true const-str.workspace = true +futures.workspace = true log.workspace = true rust-rocksdb.workspace = true +serde.workspace = true +serde_json.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/src/database/database.rs b/src/database/database.rs index c357d50f..ac6f62e9 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -37,7 +37,7 @@ impl Database { pub fn cork_and_sync(&self) -> Cork { Cork::new(&self.db, true, true) } #[inline] - pub fn iter_maps(&self) -> impl Iterator + '_ { self.map.iter() } + pub fn iter_maps(&self) -> impl Iterator + Send + '_ { self.map.iter() } } impl Index<&str> for Database { diff --git a/src/database/de.rs b/src/database/de.rs new file mode 100644 index 00000000..8ce25aa3 --- /dev/null +++ b/src/database/de.rs @@ -0,0 +1,261 @@ +use conduit::{checked, debug::DebugInspect, err, utils::string, Error, Result}; +use serde::{ + de, + de::{DeserializeSeed, Visitor}, + Deserialize, +}; + +pub(crate) fn from_slice<'a, T>(buf: &'a [u8]) -> Result +where + T: Deserialize<'a>, +{ + let mut deserializer = Deserializer { + buf, + pos: 0, + }; + + T::deserialize(&mut deserializer).debug_inspect(|_| { + deserializer + .finished() + .expect("deserialization failed to consume trailing bytes"); + }) +} + +pub(crate) struct Deserializer<'de> { + buf: &'de [u8], + pos: usize, +} + +/// Directive to ignore a record. This type can be used to skip deserialization +/// until the next separator is found. +#[derive(Debug, Deserialize)] +pub struct Ignore; + +impl<'de> Deserializer<'de> { + const SEP: u8 = b'\xFF'; + + fn finished(&self) -> Result<()> { + let pos = self.pos; + let len = self.buf.len(); + let parsed = &self.buf[0..pos]; + let unparsed = &self.buf[pos..]; + let remain = checked!(len - pos)?; + let trailing_sep = remain == 1 && unparsed[0] == Self::SEP; + (remain == 0 || trailing_sep) + .then_some(()) + .ok_or(err!(SerdeDe( + "{remain} trailing of {len} bytes not deserialized.\n{parsed:?}\n{unparsed:?}", + ))) + } + + #[inline] + fn record_next(&mut self) -> &'de [u8] { + self.buf[self.pos..] + .split(|b| *b == Deserializer::SEP) + .inspect(|record| self.inc_pos(record.len())) + .next() + .expect("remainder of buf even if SEP was not found") + } + + #[inline] + fn record_trail(&mut self) -> &'de [u8] { + let record = &self.buf[self.pos..]; + self.inc_pos(record.len()); + record + } + + #[inline] + fn record_start(&mut self) { + let started = self.pos != 0; + debug_assert!( + !started || self.buf[self.pos] == Self::SEP, + "Missing expected record separator at current position" + ); + + self.inc_pos(started.into()); + } + + #[inline] + fn inc_pos(&mut self, n: usize) { + self.pos = self.pos.saturating_add(n); + debug_assert!(self.pos <= self.buf.len(), "pos out of range"); + } +} + +impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { + type Error = Error; + + fn deserialize_map(self, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize Map not implemented") + } + + fn deserialize_seq(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_seq(self) + } + + fn deserialize_tuple(self, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_seq(self) + } + + fn deserialize_tuple_struct(self, _name: &'static str, _len: usize, visitor: V) -> Result + where + V: Visitor<'de>, + { + visitor.visit_seq(self) + } + + fn deserialize_struct( + self, _name: &'static str, _fields: &'static [&'static str], _visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize Struct not implemented") + } + + fn deserialize_unit_struct(self, name: &'static str, visitor: V) -> Result + where + V: Visitor<'de>, + { + match name { + "Ignore" => self.record_next(), + _ => unimplemented!("Unrecognized deserialization Directive {name:?}"), + }; + + visitor.visit_unit() + } + + fn deserialize_newtype_struct(self, _name: &'static str, _visitor: V) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize Newtype Struct not implemented") + } + + fn deserialize_enum( + self, _name: &'static str, _variants: &'static [&'static str], _visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + unimplemented!("deserialize Enum not implemented") + } + + fn deserialize_option>(self, _visitor: V) -> Result { + unimplemented!("deserialize Option not implemented") + } + + fn deserialize_bool>(self, _visitor: V) -> Result { + unimplemented!("deserialize bool not implemented") + } + + fn deserialize_i8>(self, _visitor: V) -> Result { + unimplemented!("deserialize i8 not implemented") + } + + fn deserialize_i16>(self, _visitor: V) -> Result { + unimplemented!("deserialize i16 not implemented") + } + + fn deserialize_i32>(self, _visitor: V) -> Result { + unimplemented!("deserialize i32 not implemented") + } + + fn deserialize_i64>(self, visitor: V) -> Result { + let bytes: [u8; size_of::()] = self.buf[self.pos..].try_into()?; + self.pos = self.pos.saturating_add(size_of::()); + visitor.visit_i64(i64::from_be_bytes(bytes)) + } + + fn deserialize_u8>(self, _visitor: V) -> Result { + unimplemented!("deserialize u8 not implemented") + } + + fn deserialize_u16>(self, _visitor: V) -> Result { + unimplemented!("deserialize u16 not implemented") + } + + fn deserialize_u32>(self, _visitor: V) -> Result { + unimplemented!("deserialize u32 not implemented") + } + + fn deserialize_u64>(self, visitor: V) -> Result { + let bytes: [u8; size_of::()] = self.buf[self.pos..].try_into()?; + self.pos = self.pos.saturating_add(size_of::()); + visitor.visit_u64(u64::from_be_bytes(bytes)) + } + + fn deserialize_f32>(self, _visitor: V) -> Result { + unimplemented!("deserialize f32 not implemented") + } + + fn deserialize_f64>(self, _visitor: V) -> Result { + unimplemented!("deserialize f64 not implemented") + } + + fn deserialize_char>(self, _visitor: V) -> Result { + unimplemented!("deserialize char not implemented") + } + + fn deserialize_str>(self, visitor: V) -> Result { + let input = self.record_next(); + let out = string::str_from_bytes(input)?; + visitor.visit_borrowed_str(out) + } + + fn deserialize_string>(self, visitor: V) -> Result { + let input = self.record_next(); + let out = string::string_from_bytes(input)?; + visitor.visit_string(out) + } + + fn deserialize_bytes>(self, visitor: V) -> Result { + let input = self.record_trail(); + visitor.visit_borrowed_bytes(input) + } + + fn deserialize_byte_buf>(self, _visitor: V) -> Result { + unimplemented!("deserialize Byte Buf not implemented") + } + + fn deserialize_unit>(self, _visitor: V) -> Result { + unimplemented!("deserialize Unit Struct not implemented") + } + + fn deserialize_identifier>(self, _visitor: V) -> Result { + unimplemented!("deserialize Identifier not implemented") + } + + fn deserialize_ignored_any>(self, _visitor: V) -> Result { + unimplemented!("deserialize Ignored Any not implemented") + } + + fn deserialize_any>(self, _visitor: V) -> Result { + unimplemented!("deserialize any not implemented") + } +} + +impl<'a, 'de: 'a> de::SeqAccess<'de> for &'a mut Deserializer<'de> { + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: DeserializeSeed<'de>, + { + if self.pos >= self.buf.len() { + return Ok(None); + } + + self.record_start(); + seed.deserialize(&mut **self).map(Some) + } +} diff --git a/src/database/deserialized.rs b/src/database/deserialized.rs new file mode 100644 index 00000000..7da112d5 --- /dev/null +++ b/src/database/deserialized.rs @@ -0,0 +1,34 @@ +use std::convert::identity; + +use conduit::Result; +use serde::Deserialize; + +pub trait Deserialized { + fn map_de(self, f: F) -> Result + where + F: FnOnce(T) -> U, + T: for<'de> Deserialize<'de>; + + fn map_json(self, f: F) -> Result + where + F: FnOnce(T) -> U, + T: for<'de> Deserialize<'de>; + + #[inline] + fn deserialized(self) -> Result + where + T: for<'de> Deserialize<'de>, + Self: Sized, + { + self.map_de(identity::) + } + + #[inline] + fn deserialized_json(self) -> Result + where + T: for<'de> Deserialize<'de>, + Self: Sized, + { + self.map_json(identity::) + } +} diff --git a/src/database/engine.rs b/src/database/engine.rs index 3850c1d3..067232e6 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -106,7 +106,7 @@ impl Engine { })) } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "trace")] pub(crate) fn open_cf(&self, name: &str) -> Result>> { let mut cfs = self.cfs.lock().expect("locked"); if !cfs.contains(name) { diff --git a/src/database/handle.rs b/src/database/handle.rs index 0b45a75f..89d87137 100644 --- a/src/database/handle.rs +++ b/src/database/handle.rs @@ -1,6 +1,10 @@ -use std::ops::Deref; +use std::{fmt, fmt::Debug, ops::Deref}; +use conduit::Result; use rocksdb::DBPinnableSlice; +use serde::{Deserialize, Serialize, Serializer}; + +use crate::{keyval::deserialize_val, Deserialized, Slice}; pub struct Handle<'a> { val: DBPinnableSlice<'a>, @@ -14,14 +18,91 @@ impl<'a> From> for Handle<'a> { } } +impl Debug for Handle<'_> { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + let val: &Slice = self; + let ptr = val.as_ptr(); + let len = val.len(); + write!(out, "Handle {{val: {{ptr: {ptr:?}, len: {len}}}}}") + } +} + +impl Serialize for Handle<'_> { + #[inline] + fn serialize(&self, serializer: S) -> Result { + let bytes: &Slice = self; + serializer.serialize_bytes(bytes) + } +} + impl Deref for Handle<'_> { - type Target = [u8]; + type Target = Slice; #[inline] fn deref(&self) -> &Self::Target { &self.val } } -impl AsRef<[u8]> for Handle<'_> { +impl AsRef for Handle<'_> { #[inline] - fn as_ref(&self) -> &[u8] { &self.val } + fn as_ref(&self) -> &Slice { &self.val } +} + +impl Deserialized for Result> { + #[inline] + fn map_json(self, f: F) -> Result + where + F: FnOnce(T) -> U, + T: for<'de> Deserialize<'de>, + { + self?.map_json(f) + } + + #[inline] + fn map_de(self, f: F) -> Result + where + F: FnOnce(T) -> U, + T: for<'de> Deserialize<'de>, + { + self?.map_de(f) + } +} + +impl<'a> Deserialized for Result<&'a Handle<'a>> { + #[inline] + fn map_json(self, f: F) -> Result + where + F: FnOnce(T) -> U, + T: for<'de> Deserialize<'de>, + { + self.and_then(|handle| handle.map_json(f)) + } + + #[inline] + fn map_de(self, f: F) -> Result + where + F: FnOnce(T) -> U, + T: for<'de> Deserialize<'de>, + { + self.and_then(|handle| handle.map_de(f)) + } +} + +impl<'a> Deserialized for &'a Handle<'a> { + fn map_json(self, f: F) -> Result + where + F: FnOnce(T) -> U, + T: for<'de> Deserialize<'de>, + { + serde_json::from_slice::(self.as_ref()) + .map_err(Into::into) + .map(f) + } + + fn map_de(self, f: F) -> Result + where + F: FnOnce(T) -> U, + T: for<'de> Deserialize<'de>, + { + deserialize_val(self.as_ref()).map(f) + } } diff --git a/src/database/iter.rs b/src/database/iter.rs deleted file mode 100644 index 4845e977..00000000 --- a/src/database/iter.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::{iter::FusedIterator, sync::Arc}; - -use conduit::Result; -use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode, Direction, IteratorMode, ReadOptions}; - -use crate::{ - engine::Db, - result, - slice::{OwnedKeyVal, OwnedKeyValPair}, - Engine, -}; - -type Cursor<'cursor> = DBRawIteratorWithThreadMode<'cursor, Db>; - -struct State<'cursor> { - cursor: Cursor<'cursor>, - direction: Direction, - valid: bool, - init: bool, -} - -impl<'cursor> State<'cursor> { - pub(crate) fn new( - db: &'cursor Arc, cf: &'cursor Arc, opts: ReadOptions, mode: &IteratorMode<'_>, - ) -> Self { - let mut cursor = db.db.raw_iterator_cf_opt(&**cf, opts); - let direction = into_direction(mode); - let valid = seek_init(&mut cursor, mode); - Self { - cursor, - direction, - valid, - init: true, - } - } -} - -pub struct Iter<'cursor> { - state: State<'cursor>, -} - -impl<'cursor> Iter<'cursor> { - pub(crate) fn new( - db: &'cursor Arc, cf: &'cursor Arc, opts: ReadOptions, mode: &IteratorMode<'_>, - ) -> Self { - Self { - state: State::new(db, cf, opts, mode), - } - } -} - -impl Iterator for Iter<'_> { - type Item = OwnedKeyValPair; - - fn next(&mut self) -> Option { - if !self.state.init && self.state.valid { - seek_next(&mut self.state.cursor, self.state.direction); - } else if self.state.init { - self.state.init = false; - } - - self.state - .cursor - .item() - .map(OwnedKeyVal::from) - .map(OwnedKeyVal::to_tuple) - .or_else(|| { - when_invalid(&mut self.state).expect("iterator invalidated due to error"); - None - }) - } -} - -impl FusedIterator for Iter<'_> {} - -fn when_invalid(state: &mut State<'_>) -> Result<()> { - state.valid = false; - result(state.cursor.status()) -} - -fn seek_next(cursor: &mut Cursor<'_>, direction: Direction) { - match direction { - Direction::Forward => cursor.next(), - Direction::Reverse => cursor.prev(), - } -} - -fn seek_init(cursor: &mut Cursor<'_>, mode: &IteratorMode<'_>) -> bool { - use Direction::{Forward, Reverse}; - use IteratorMode::{End, From, Start}; - - match mode { - Start => cursor.seek_to_first(), - End => cursor.seek_to_last(), - From(key, Forward) => cursor.seek(key), - From(key, Reverse) => cursor.seek_for_prev(key), - }; - - cursor.valid() -} - -fn into_direction(mode: &IteratorMode<'_>) -> Direction { - use Direction::{Forward, Reverse}; - use IteratorMode::{End, From, Start}; - - match mode { - Start | From(_, Forward) => Forward, - End | From(_, Reverse) => Reverse, - } -} diff --git a/src/database/keyval.rs b/src/database/keyval.rs new file mode 100644 index 00000000..c9d25977 --- /dev/null +++ b/src/database/keyval.rs @@ -0,0 +1,83 @@ +use conduit::Result; +use serde::Deserialize; + +use crate::de; + +pub(crate) type OwnedKeyVal = (Vec, Vec); +pub(crate) type OwnedKey = Vec; +pub(crate) type OwnedVal = Vec; + +pub type KeyVal<'a, K = &'a Slice, V = &'a Slice> = (Key<'a, K>, Val<'a, V>); +pub type Key<'a, T = &'a Slice> = T; +pub type Val<'a, T = &'a Slice> = T; + +pub type Slice = [u8]; + +#[inline] +pub(crate) fn _expect_deserialize<'a, K, V>(kv: Result>) -> KeyVal<'a, K, V> +where + K: Deserialize<'a>, + V: Deserialize<'a>, +{ + result_deserialize(kv).expect("failed to deserialize result key/val") +} + +#[inline] +pub(crate) fn _expect_deserialize_key<'a, K>(key: Result>) -> Key<'a, K> +where + K: Deserialize<'a>, +{ + result_deserialize_key(key).expect("failed to deserialize result key") +} + +#[inline] +pub(crate) fn result_deserialize<'a, K, V>(kv: Result>) -> Result> +where + K: Deserialize<'a>, + V: Deserialize<'a>, +{ + deserialize(kv?) +} + +#[inline] +pub(crate) fn result_deserialize_key<'a, K>(key: Result>) -> Result> +where + K: Deserialize<'a>, +{ + deserialize_key(key?) +} + +#[inline] +pub(crate) fn deserialize<'a, K, V>(kv: KeyVal<'a>) -> Result> +where + K: Deserialize<'a>, + V: Deserialize<'a>, +{ + Ok((deserialize_key::(kv.0)?, deserialize_val::(kv.1)?)) +} + +#[inline] +pub(crate) fn deserialize_key<'a, K>(key: Key<'a>) -> Result> +where + K: Deserialize<'a>, +{ + de::from_slice::(key) +} + +#[inline] +pub(crate) fn deserialize_val<'a, V>(val: Val<'a>) -> Result> +where + V: Deserialize<'a>, +{ + de::from_slice::(val) +} + +#[inline] +#[must_use] +pub fn to_owned(kv: KeyVal<'_>) -> OwnedKeyVal { (kv.0.to_owned(), kv.1.to_owned()) } + +#[inline] +pub fn key(kv: KeyVal<'_, K, V>) -> Key<'_, K> { kv.0 } + +#[inline] +pub fn val(kv: KeyVal<'_, K, V>) -> Val<'_, V> { kv.1 } diff --git a/src/database/map.rs b/src/database/map.rs index ddae8c81..a3cf32d4 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -1,15 +1,39 @@ -use std::{ffi::CStr, future::Future, mem::size_of, pin::Pin, sync::Arc}; +mod count; +mod keys; +mod keys_from; +mod keys_prefix; +mod rev_keys; +mod rev_keys_from; +mod rev_keys_prefix; +mod rev_stream; +mod rev_stream_from; +mod rev_stream_prefix; +mod stream; +mod stream_from; +mod stream_prefix; -use conduit::{utils, Result}; -use rocksdb::{ - AsColumnFamilyRef, ColumnFamily, Direction, IteratorMode, ReadOptions, WriteBatchWithTransaction, WriteOptions, +use std::{ + convert::AsRef, + ffi::CStr, + fmt, + fmt::{Debug, Display}, + future::Future, + io::Write, + pin::Pin, + sync::Arc, }; +use conduit::{err, Result}; +use futures::future; +use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, WriteBatchWithTransaction, WriteOptions}; +use serde::Serialize; + use crate::{ - or_else, result, - slice::{Byte, Key, KeyVal, OwnedKey, OwnedKeyValPair, OwnedVal, Val}, + keyval::{OwnedKey, OwnedVal}, + ser, + util::{map_err, or_else}, watchers::Watchers, - Engine, Handle, Iter, + Engine, Handle, }; pub struct Map { @@ -21,8 +45,6 @@ pub struct Map { read_options: ReadOptions, } -type OwnedKeyValPairIter<'a> = Box + Send + 'a>; - impl Map { pub(crate) fn open(db: &Arc, name: &str) -> Result> { Ok(Arc::new(Self { @@ -35,14 +57,125 @@ impl Map { })) } - pub fn get(&self, key: &Key) -> Result>> { - let read_options = &self.read_options; - let res = self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options); - - Ok(result(res)?.map(Handle::from)) + #[tracing::instrument(skip(self), fields(%self), level = "trace")] + pub fn del(&self, key: &K) + where + K: Serialize + ?Sized + Debug, + { + let mut buf = Vec::::with_capacity(64); + self.bdel(key, &mut buf); } - pub fn multi_get(&self, keys: &[&Key]) -> Result>> { + #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] + pub fn bdel(&self, key: &K, buf: &mut B) + where + K: Serialize + ?Sized + Debug, + B: Write + AsRef<[u8]>, + { + let key = ser::serialize(buf, key).expect("failed to serialize deletion key"); + self.remove(&key); + } + + #[tracing::instrument(level = "trace")] + pub fn remove(&self, key: &K) + where + K: AsRef<[u8]> + ?Sized + Debug, + { + let write_options = &self.write_options; + self.db + .db + .delete_cf_opt(&self.cf(), key, write_options) + .or_else(or_else) + .expect("database remove error"); + + if !self.db.corked() { + self.db.flush().expect("database flush error"); + } + } + + #[tracing::instrument(skip(self, value), fields(%self), level = "trace")] + pub fn insert(&self, key: &K, value: &V) + where + K: AsRef<[u8]> + ?Sized + Debug, + V: AsRef<[u8]> + ?Sized, + { + let write_options = &self.write_options; + self.db + .db + .put_cf_opt(&self.cf(), key, value, write_options) + .or_else(or_else) + .expect("database insert error"); + + if !self.db.corked() { + self.db.flush().expect("database flush error"); + } + + self.watchers.wake(key.as_ref()); + } + + #[tracing::instrument(skip(self), fields(%self), level = "trace")] + pub fn insert_batch<'a, I, K, V>(&'a self, iter: I) + where + I: Iterator + Send + Debug, + K: AsRef<[u8]> + Sized + Debug + 'a, + V: AsRef<[u8]> + Sized + 'a, + { + let mut batch = WriteBatchWithTransaction::::default(); + for (key, val) in iter { + batch.put_cf(&self.cf(), key.as_ref(), val.as_ref()); + } + + let write_options = &self.write_options; + self.db + .db + .write_opt(batch, write_options) + .or_else(or_else) + .expect("database insert batch error"); + + if !self.db.corked() { + self.db.flush().expect("database flush error"); + } + } + + #[tracing::instrument(skip(self), fields(%self), level = "trace")] + pub fn qry(&self, key: &K) -> impl Future>> + Send + where + K: Serialize + ?Sized + Debug, + { + let mut buf = Vec::::with_capacity(64); + self.bqry(key, &mut buf) + } + + #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] + pub fn bqry(&self, key: &K, buf: &mut B) -> impl Future>> + Send + where + K: Serialize + ?Sized + Debug, + B: Write + AsRef<[u8]>, + { + let key = ser::serialize(buf, key).expect("failed to serialize query key"); + let val = self.get(key); + future::ready(val) + } + + #[tracing::instrument(skip(self), fields(%self), level = "trace")] + pub fn get(&self, key: &K) -> Result> + where + K: AsRef<[u8]> + ?Sized + Debug, + { + self.db + .db + .get_pinned_cf_opt(&self.cf(), key, &self.read_options) + .map_err(map_err)? + .map(Handle::from) + .ok_or(err!(Request(NotFound("Not found in database")))) + } + + #[tracing::instrument(skip(self), fields(%self), level = "trace")] + pub fn multi_get<'a, I, K>(&self, keys: I) -> Vec> + where + I: Iterator + ExactSizeIterator + Send + Debug, + K: AsRef<[u8]> + Sized + Debug + 'a, + { // Optimization can be `true` if key vector is pre-sorted **by the column // comparator**. const SORTED: bool = false; @@ -57,140 +190,25 @@ impl Map { match res { Ok(Some(res)) => ret.push(Some((*res).to_vec())), Ok(None) => ret.push(None), - Err(e) => return or_else(e), + Err(e) => or_else(e).expect("database multiget error"), } } - Ok(ret) + ret } - pub fn insert(&self, key: &Key, value: &Val) -> Result<()> { - let write_options = &self.write_options; - self.db - .db - .put_cf_opt(&self.cf(), key, value, write_options) - .or_else(or_else)?; - - if !self.db.corked() { - self.db.flush()?; - } - - self.watchers.wake(key); - - Ok(()) - } - - pub fn insert_batch<'a, I>(&'a self, iter: I) -> Result<()> + #[inline] + pub fn watch_prefix<'a, K>(&'a self, prefix: &K) -> Pin + Send + 'a>> where - I: Iterator>, + K: AsRef<[u8]> + ?Sized + Debug, { - let mut batch = WriteBatchWithTransaction::::default(); - for KeyVal(key, value) in iter { - batch.put_cf(&self.cf(), key, value); - } - - let write_options = &self.write_options; - let res = self.db.db.write_opt(batch, write_options); - - if !self.db.corked() { - self.db.flush()?; - } - - result(res) - } - - pub fn remove(&self, key: &Key) -> Result<()> { - let write_options = &self.write_options; - let res = self.db.db.delete_cf_opt(&self.cf(), key, write_options); - - if !self.db.corked() { - self.db.flush()?; - } - - result(res) - } - - pub fn remove_batch<'a, I>(&'a self, iter: I) -> Result<()> - where - I: Iterator, - { - let mut batch = WriteBatchWithTransaction::::default(); - for key in iter { - batch.delete_cf(&self.cf(), key); - } - - let write_options = &self.write_options; - let res = self.db.db.write_opt(batch, write_options); - - if !self.db.corked() { - self.db.flush()?; - } - - result(res) - } - - pub fn iter(&self) -> OwnedKeyValPairIter<'_> { - let mode = IteratorMode::Start; - let read_options = read_options_default(); - Box::new(Iter::new(&self.db, &self.cf, read_options, &mode)) - } - - pub fn iter_from(&self, from: &Key, reverse: bool) -> OwnedKeyValPairIter<'_> { - let direction = if reverse { - Direction::Reverse - } else { - Direction::Forward - }; - let mode = IteratorMode::From(from, direction); - let read_options = read_options_default(); - Box::new(Iter::new(&self.db, &self.cf, read_options, &mode)) - } - - pub fn scan_prefix(&self, prefix: OwnedKey) -> OwnedKeyValPairIter<'_> { - let mode = IteratorMode::From(&prefix, Direction::Forward); - let read_options = read_options_default(); - Box::new(Iter::new(&self.db, &self.cf, read_options, &mode).take_while(move |(k, _)| k.starts_with(&prefix))) - } - - pub fn increment(&self, key: &Key) -> Result<[Byte; size_of::()]> { - let old = self.get(key)?; - let new = utils::increment(old.as_deref()); - self.insert(key, &new)?; - - if !self.db.corked() { - self.db.flush()?; - } - - Ok(new) - } - - pub fn increment_batch<'a, I>(&'a self, iter: I) -> Result<()> - where - I: Iterator, - { - let mut batch = WriteBatchWithTransaction::::default(); - for key in iter { - let old = self.get(key)?; - let new = utils::increment(old.as_deref()); - batch.put_cf(&self.cf(), key, new); - } - - let write_options = &self.write_options; - let res = self.db.db.write_opt(batch, write_options); - - if !self.db.corked() { - self.db.flush()?; - } - - result(res) - } - - pub fn watch_prefix<'a>(&'a self, prefix: &Key) -> Pin + Send + 'a>> { - self.watchers.watch(prefix) + self.watchers.watch(prefix.as_ref()) } + #[inline] pub fn property_integer(&self, name: &CStr) -> Result { self.db.property_integer(&self.cf(), name) } + #[inline] pub fn property(&self, name: &str) -> Result { self.db.property(&self.cf(), name) } #[inline] @@ -199,12 +217,12 @@ impl Map { fn cf(&self) -> impl AsColumnFamilyRef + '_ { &*self.cf } } -impl<'a> IntoIterator for &'a Map { - type IntoIter = Box + Send + 'a>; - type Item = OwnedKeyValPair; +impl Debug for Map { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { write!(out, "Map {{name: {0}}}", self.name) } +} - #[inline] - fn into_iter(self) -> Self::IntoIter { self.iter() } +impl Display for Map { + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { write!(out, "{0}", self.name) } } fn open(db: &Arc, name: &str) -> Result> { diff --git a/src/database/map/count.rs b/src/database/map/count.rs new file mode 100644 index 00000000..4356b71f --- /dev/null +++ b/src/database/map/count.rs @@ -0,0 +1,36 @@ +use std::{fmt::Debug, future::Future}; + +use conduit::implement; +use futures::stream::StreamExt; +use serde::Serialize; + +use crate::de::Ignore; + +/// Count the total number of entries in the map. +#[implement(super::Map)] +#[inline] +pub fn count(&self) -> impl Future + Send + '_ { self.keys::().count() } + +/// Count the number of entries in the map starting from a lower-bound. +/// +/// - From is a structured key +#[implement(super::Map)] +#[inline] +pub fn count_from<'a, P>(&'a self, from: &P) -> impl Future + Send + 'a +where + P: Serialize + ?Sized + Debug + 'a, +{ + self.keys_from::(from).count() +} + +/// Count the number of entries in the map matching a prefix. +/// +/// - Prefix is structured key +#[implement(super::Map)] +#[inline] +pub fn count_prefix<'a, P>(&'a self, prefix: &P) -> impl Future + Send + 'a +where + P: Serialize + ?Sized + Debug + 'a, +{ + self.keys_prefix::(prefix).count() +} diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs new file mode 100644 index 00000000..2396494c --- /dev/null +++ b/src/database/map/keys.rs @@ -0,0 +1,21 @@ +use conduit::{implement, Result}; +use futures::{Stream, StreamExt}; +use serde::Deserialize; + +use crate::{keyval, keyval::Key, stream}; + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn keys<'a, K>(&'a self) -> impl Stream>> + Send +where + K: Deserialize<'a> + Send, +{ + self.raw_keys().map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn raw_keys(&self) -> impl Stream>> + Send { + let opts = super::read_options_default(); + stream::Keys::new(&self.db, &self.cf, opts, None) +} diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs new file mode 100644 index 00000000..1993750a --- /dev/null +++ b/src/database/map/keys_from.rs @@ -0,0 +1,49 @@ +use std::{convert::AsRef, fmt::Debug}; + +use conduit::{implement, Result}; +use futures::{Stream, StreamExt}; +use serde::{Deserialize, Serialize}; + +use crate::{keyval, keyval::Key, ser, stream}; + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn keys_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, +{ + self.keys_raw_from(from) + .map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn keys_raw_from

    (&self, from: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, +{ + let key = ser::serialize_to_vec(from).expect("failed to serialize query key"); + self.raw_keys_from(&key) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn keys_from_raw<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +where + P: AsRef<[u8]> + ?Sized + Debug + Sync, + K: Deserialize<'a> + Send, +{ + self.raw_keys_from(from) + .map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn raw_keys_from

    (&self, from: &P) -> impl Stream>> + Send +where + P: AsRef<[u8]> + ?Sized + Debug, +{ + let opts = super::read_options_default(); + stream::Keys::new(&self.db, &self.cf, opts, Some(from.as_ref())) +} diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs new file mode 100644 index 00000000..d6c0927b --- /dev/null +++ b/src/database/map/keys_prefix.rs @@ -0,0 +1,54 @@ +use std::{convert::AsRef, fmt::Debug}; + +use conduit::{implement, Result}; +use futures::{ + future, + stream::{Stream, StreamExt}, + TryStreamExt, +}; +use serde::{Deserialize, Serialize}; + +use crate::{keyval, keyval::Key, ser}; + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn keys_prefix<'a, K, P>(&'a self, prefix: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, +{ + self.keys_raw_prefix(prefix) + .map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn keys_raw_prefix

    (&self, prefix: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, +{ + let key = ser::serialize_to_vec(prefix).expect("failed to serialize query key"); + self.raw_keys_from(&key) + .try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key))) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn keys_prefix_raw<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + K: Deserialize<'a> + Send + 'a, +{ + self.raw_keys_prefix(prefix) + .map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn raw_keys_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, +{ + self.raw_keys_from(prefix) + .try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref()))) +} diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs new file mode 100644 index 00000000..449ccfff --- /dev/null +++ b/src/database/map/rev_keys.rs @@ -0,0 +1,21 @@ +use conduit::{implement, Result}; +use futures::{Stream, StreamExt}; +use serde::Deserialize; + +use crate::{keyval, keyval::Key, stream}; + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_keys<'a, K>(&'a self) -> impl Stream>> + Send +where + K: Deserialize<'a> + Send, +{ + self.rev_raw_keys().map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_raw_keys(&self) -> impl Stream>> + Send { + let opts = super::read_options_default(); + stream::KeysRev::new(&self.db, &self.cf, opts, None) +} diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs new file mode 100644 index 00000000..e012e60a --- /dev/null +++ b/src/database/map/rev_keys_from.rs @@ -0,0 +1,49 @@ +use std::{convert::AsRef, fmt::Debug}; + +use conduit::{implement, Result}; +use futures::{Stream, StreamExt}; +use serde::{Deserialize, Serialize}; + +use crate::{keyval, keyval::Key, ser, stream}; + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_keys_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, +{ + self.rev_keys_raw_from(from) + .map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_keys_raw_from

    (&self, from: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, +{ + let key = ser::serialize_to_vec(from).expect("failed to serialize query key"); + self.rev_raw_keys_from(&key) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_keys_from_raw<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +where + P: AsRef<[u8]> + ?Sized + Debug + Sync, + K: Deserialize<'a> + Send, +{ + self.rev_raw_keys_from(from) + .map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_raw_keys_from

    (&self, from: &P) -> impl Stream>> + Send +where + P: AsRef<[u8]> + ?Sized + Debug, +{ + let opts = super::read_options_default(); + stream::KeysRev::new(&self.db, &self.cf, opts, Some(from.as_ref())) +} diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs new file mode 100644 index 00000000..162c4f9b --- /dev/null +++ b/src/database/map/rev_keys_prefix.rs @@ -0,0 +1,54 @@ +use std::{convert::AsRef, fmt::Debug}; + +use conduit::{implement, Result}; +use futures::{ + future, + stream::{Stream, StreamExt}, + TryStreamExt, +}; +use serde::{Deserialize, Serialize}; + +use crate::{keyval, keyval::Key, ser}; + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_keys_prefix<'a, K, P>(&'a self, prefix: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, +{ + self.rev_keys_raw_prefix(prefix) + .map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_keys_raw_prefix

    (&self, prefix: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, +{ + let key = ser::serialize_to_vec(prefix).expect("failed to serialize query key"); + self.rev_raw_keys_from(&key) + .try_take_while(move |k: &Key<'_>| future::ok(k.starts_with(&key))) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_keys_prefix_raw<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + K: Deserialize<'a> + Send + 'a, +{ + self.rev_raw_keys_prefix(prefix) + .map(keyval::result_deserialize_key::) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_raw_keys_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, +{ + self.rev_raw_keys_from(prefix) + .try_take_while(|k: &Key<'_>| future::ok(k.starts_with(prefix.as_ref()))) +} diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs new file mode 100644 index 00000000..de22fd5c --- /dev/null +++ b/src/database/map/rev_stream.rs @@ -0,0 +1,29 @@ +use conduit::{implement, Result}; +use futures::stream::{Stream, StreamExt}; +use serde::Deserialize; + +use crate::{keyval, keyval::KeyVal, stream}; + +/// Iterate key-value entries in the map from the end. +/// +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_stream<'a, K, V>(&'a self) -> impl Stream>> + Send +where + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, +{ + self.rev_raw_stream() + .map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map from the end. +/// +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_raw_stream(&self) -> impl Stream>> + Send { + let opts = super::read_options_default(); + stream::ItemsRev::new(&self.db, &self.cf, opts, None) +} diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs new file mode 100644 index 00000000..650cf038 --- /dev/null +++ b/src/database/map/rev_stream_from.rs @@ -0,0 +1,68 @@ +use std::{convert::AsRef, fmt::Debug}; + +use conduit::{implement, Result}; +use futures::stream::{Stream, StreamExt}; +use serde::{Deserialize, Serialize}; + +use crate::{keyval, keyval::KeyVal, ser, stream}; + +/// Iterate key-value entries in the map starting from upper-bound. +/// +/// - Query is serialized +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_stream_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, +{ + let key = ser::serialize_to_vec(from).expect("failed to serialize query key"); + self.rev_stream_raw_from(&key) + .map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map starting from upper-bound. +/// +/// - Query is serialized +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_stream_raw_from

    (&self, from: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, +{ + let key = ser::serialize_to_vec(from).expect("failed to serialize query key"); + self.rev_raw_stream_from(&key) +} + +/// Iterate key-value entries in the map starting from upper-bound. +/// +/// - Query is raw +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_stream_from_raw<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +where + P: AsRef<[u8]> + ?Sized + Debug + Sync, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, +{ + self.rev_raw_stream_from(from) + .map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map starting from upper-bound. +/// +/// - Query is raw +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_raw_stream_from

    (&self, from: &P) -> impl Stream>> + Send +where + P: AsRef<[u8]> + ?Sized + Debug, +{ + let opts = super::read_options_default(); + stream::ItemsRev::new(&self.db, &self.cf, opts, Some(from.as_ref())) +} diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs new file mode 100644 index 00000000..9ef89e9c --- /dev/null +++ b/src/database/map/rev_stream_prefix.rs @@ -0,0 +1,74 @@ +use std::{convert::AsRef, fmt::Debug}; + +use conduit::{implement, Result}; +use futures::{ + future, + stream::{Stream, StreamExt}, + TryStreamExt, +}; +use serde::{Deserialize, Serialize}; + +use crate::{keyval, keyval::KeyVal, ser}; + +/// Iterate key-value entries in the map where the key matches a prefix. +/// +/// - Query is serialized +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_stream_prefix<'a, K, V, P>(&'a self, prefix: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, +{ + self.rev_stream_raw_prefix(prefix) + .map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map where the key matches a prefix. +/// +/// - Query is serialized +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_stream_raw_prefix

    (&self, prefix: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, +{ + let key = ser::serialize_to_vec(prefix).expect("failed to serialize query key"); + self.rev_raw_stream_from(&key) + .try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key))) +} + +/// Iterate key-value entries in the map where the key matches a prefix. +/// +/// - Query is raw +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_stream_prefix_raw<'a, K, V, P>( + &'a self, prefix: &'a P, +) -> impl Stream>> + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + K: Deserialize<'a> + Send + 'a, + V: Deserialize<'a> + Send + 'a, +{ + self.rev_raw_stream_prefix(prefix) + .map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map where the key matches a prefix. +/// +/// - Query is raw +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn rev_raw_stream_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, +{ + self.rev_raw_stream_from(prefix) + .try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref()))) +} diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs new file mode 100644 index 00000000..dfbea072 --- /dev/null +++ b/src/database/map/stream.rs @@ -0,0 +1,28 @@ +use conduit::{implement, Result}; +use futures::stream::{Stream, StreamExt}; +use serde::Deserialize; + +use crate::{keyval, keyval::KeyVal, stream}; + +/// Iterate key-value entries in the map from the beginning. +/// +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn stream<'a, K, V>(&'a self) -> impl Stream>> + Send +where + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, +{ + self.raw_stream().map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map from the beginning. +/// +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn raw_stream(&self) -> impl Stream>> + Send { + let opts = super::read_options_default(); + stream::Items::new(&self.db, &self.cf, opts, None) +} diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs new file mode 100644 index 00000000..153d5bb6 --- /dev/null +++ b/src/database/map/stream_from.rs @@ -0,0 +1,68 @@ +use std::{convert::AsRef, fmt::Debug}; + +use conduit::{implement, Result}; +use futures::stream::{Stream, StreamExt}; +use serde::{Deserialize, Serialize}; + +use crate::{keyval, keyval::KeyVal, ser, stream}; + +/// Iterate key-value entries in the map starting from lower-bound. +/// +/// - Query is serialized +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn stream_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, +{ + let key = ser::serialize_to_vec(from).expect("failed to serialize query key"); + self.stream_raw_from(&key) + .map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map starting from lower-bound. +/// +/// - Query is serialized +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn stream_raw_from

    (&self, from: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, +{ + let key = ser::serialize_to_vec(from).expect("failed to serialize query key"); + self.raw_stream_from(&key) +} + +/// Iterate key-value entries in the map starting from lower-bound. +/// +/// - Query is raw +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn stream_from_raw<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +where + P: AsRef<[u8]> + ?Sized + Debug + Sync, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, +{ + self.raw_stream_from(from) + .map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map starting from lower-bound. +/// +/// - Query is raw +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn raw_stream_from

    (&self, from: &P) -> impl Stream>> + Send +where + P: AsRef<[u8]> + ?Sized + Debug, +{ + let opts = super::read_options_default(); + stream::Items::new(&self.db, &self.cf, opts, Some(from.as_ref())) +} diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs new file mode 100644 index 00000000..56154a8b --- /dev/null +++ b/src/database/map/stream_prefix.rs @@ -0,0 +1,74 @@ +use std::{convert::AsRef, fmt::Debug}; + +use conduit::{implement, Result}; +use futures::{ + future, + stream::{Stream, StreamExt}, + TryStreamExt, +}; +use serde::{Deserialize, Serialize}; + +use crate::{keyval, keyval::KeyVal, ser}; + +/// Iterate key-value entries in the map where the key matches a prefix. +/// +/// - Query is serialized +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn stream_prefix<'a, K, V, P>(&'a self, prefix: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, + K: Deserialize<'a> + Send, + V: Deserialize<'a> + Send, +{ + self.stream_raw_prefix(prefix) + .map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map where the key matches a prefix. +/// +/// - Query is serialized +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn stream_raw_prefix

    (&self, prefix: &P) -> impl Stream>> + Send +where + P: Serialize + ?Sized + Debug, +{ + let key = ser::serialize_to_vec(prefix).expect("failed to serialize query key"); + self.raw_stream_from(&key) + .try_take_while(move |(k, _): &KeyVal<'_>| future::ok(k.starts_with(&key))) +} + +/// Iterate key-value entries in the map where the key matches a prefix. +/// +/// - Query is raw +/// - Result is deserialized +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn stream_prefix_raw<'a, K, V, P>( + &'a self, prefix: &'a P, +) -> impl Stream>> + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, + K: Deserialize<'a> + Send + 'a, + V: Deserialize<'a> + Send + 'a, +{ + self.raw_stream_prefix(prefix) + .map(keyval::result_deserialize::) +} + +/// Iterate key-value entries in the map where the key matches a prefix. +/// +/// - Query is raw +/// - Result is raw +#[implement(super::Map)] +#[tracing::instrument(skip(self), fields(%self), level = "trace")] +pub fn raw_stream_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, +{ + self.raw_stream_from(prefix) + .try_take_while(|(k, _): &KeyVal<'_>| future::ok(k.starts_with(prefix.as_ref()))) +} diff --git a/src/database/mod.rs b/src/database/mod.rs index 6446624c..e66abf68 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,25 +1,35 @@ mod cork; mod database; +mod de; +mod deserialized; mod engine; mod handle; -mod iter; +pub mod keyval; mod map; pub mod maps; mod opts; -mod slice; +mod ser; +mod stream; mod util; mod watchers; +pub(crate) use self::{ + engine::Engine, + util::{or_else, result}, +}; + extern crate conduit_core as conduit; extern crate rust_rocksdb as rocksdb; -pub use database::Database; -pub(crate) use engine::Engine; -pub use handle::Handle; -pub use iter::Iter; -pub use map::Map; -pub use slice::{Key, KeyVal, OwnedKey, OwnedKeyVal, OwnedVal, Val}; -pub(crate) use util::{or_else, result}; +pub use self::{ + database::Database, + de::Ignore, + deserialized::Deserialized, + handle::Handle, + keyval::{KeyVal, Slice}, + map::Map, + ser::{Interfix, Separator}, +}; conduit::mod_ctor! {} conduit::mod_dtor! {} diff --git a/src/database/ser.rs b/src/database/ser.rs new file mode 100644 index 00000000..bd4bbd9a --- /dev/null +++ b/src/database/ser.rs @@ -0,0 +1,315 @@ +use std::io::Write; + +use conduit::{err, result::DebugInspect, utils::exchange, Error, Result}; +use serde::{ser, Serialize}; + +#[inline] +pub(crate) fn serialize_to_vec(val: &T) -> Result> +where + T: Serialize + ?Sized, +{ + let mut buf = Vec::with_capacity(64); + serialize(&mut buf, val)?; + + Ok(buf) +} + +#[inline] +pub(crate) fn serialize<'a, W, T>(out: &'a mut W, val: &'a T) -> Result<&'a [u8]> +where + W: Write + AsRef<[u8]>, + T: Serialize + ?Sized, +{ + let mut serializer = Serializer { + out, + depth: 0, + sep: false, + fin: false, + }; + + val.serialize(&mut serializer) + .map_err(|error| err!(SerdeSer("{error}"))) + .debug_inspect(|()| { + debug_assert_eq!(serializer.depth, 0, "Serialization completed at non-zero recursion level"); + })?; + + Ok((*out).as_ref()) +} + +pub(crate) struct Serializer<'a, W: Write> { + out: &'a mut W, + depth: u32, + sep: bool, + fin: bool, +} + +/// Directive to force separator serialization specifically for prefix keying +/// use. This is a quirk of the database schema and prefix iterations. +#[derive(Debug, Serialize)] +pub struct Interfix; + +/// Directive to force separator serialization. Separators are usually +/// serialized automatically. +#[derive(Debug, Serialize)] +pub struct Separator; + +impl Serializer<'_, W> { + const SEP: &'static [u8] = b"\xFF"; + + fn sequence_start(&mut self) { + debug_assert!(!self.is_finalized(), "Sequence start with finalization set"); + debug_assert!(!self.sep, "Sequence start with separator set"); + if cfg!(debug_assertions) { + self.depth = self.depth.saturating_add(1); + } + } + + fn sequence_end(&mut self) { + self.sep = false; + if cfg!(debug_assertions) { + self.depth = self.depth.saturating_sub(1); + } + } + + fn record_start(&mut self) -> Result<()> { + debug_assert!(!self.is_finalized(), "Starting a record after serialization finalized"); + exchange(&mut self.sep, true) + .then(|| self.separator()) + .unwrap_or(Ok(())) + } + + fn separator(&mut self) -> Result<()> { + debug_assert!(!self.is_finalized(), "Writing a separator after serialization finalized"); + self.out.write_all(Self::SEP).map_err(Into::into) + } + + fn set_finalized(&mut self) { + debug_assert!(!self.is_finalized(), "Finalization already set"); + if cfg!(debug_assertions) { + self.fin = true; + } + } + + fn is_finalized(&self) -> bool { self.fin } +} + +impl ser::Serializer for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + type SerializeMap = Self; + type SerializeSeq = Self; + type SerializeStruct = Self; + type SerializeStructVariant = Self; + type SerializeTuple = Self; + type SerializeTupleStruct = Self; + type SerializeTupleVariant = Self; + + fn serialize_map(self, _len: Option) -> Result { + unimplemented!("serialize Map not implemented") + } + + fn serialize_seq(self, _len: Option) -> Result { + self.sequence_start(); + self.record_start()?; + Ok(self) + } + + fn serialize_tuple(self, _len: usize) -> Result { + self.sequence_start(); + Ok(self) + } + + fn serialize_tuple_struct(self, _name: &'static str, _len: usize) -> Result { + self.sequence_start(); + Ok(self) + } + + fn serialize_tuple_variant( + self, _name: &'static str, _idx: u32, _var: &'static str, _len: usize, + ) -> Result { + self.sequence_start(); + Ok(self) + } + + fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { + self.sequence_start(); + Ok(self) + } + + fn serialize_struct_variant( + self, _name: &'static str, _idx: u32, _var: &'static str, _len: usize, + ) -> Result { + self.sequence_start(); + Ok(self) + } + + fn serialize_newtype_struct(self, _name: &'static str, _value: &T) -> Result { + unimplemented!("serialize New Type Struct not implemented") + } + + fn serialize_newtype_variant( + self, _name: &'static str, _idx: u32, _var: &'static str, _value: &T, + ) -> Result { + unimplemented!("serialize New Type Variant not implemented") + } + + fn serialize_unit_struct(self, name: &'static str) -> Result { + match name { + "Interfix" => { + self.set_finalized(); + }, + "Separator" => { + self.separator()?; + }, + _ => unimplemented!("Unrecognized serialization directive: {name:?}"), + }; + + Ok(()) + } + + fn serialize_unit_variant(self, _name: &'static str, _idx: u32, _var: &'static str) -> Result { + unimplemented!("serialize Unit Variant not implemented") + } + + fn serialize_some(self, val: &T) -> Result { val.serialize(self) } + + fn serialize_none(self) -> Result { Ok(()) } + + fn serialize_char(self, v: char) -> Result { + let mut buf: [u8; 4] = [0; 4]; + self.serialize_str(v.encode_utf8(&mut buf)) + } + + fn serialize_str(self, v: &str) -> Result { self.serialize_bytes(v.as_bytes()) } + + fn serialize_bytes(self, v: &[u8]) -> Result { self.out.write_all(v).map_err(Error::Io) } + + fn serialize_f64(self, _v: f64) -> Result { unimplemented!("serialize f64 not implemented") } + + fn serialize_f32(self, _v: f32) -> Result { unimplemented!("serialize f32 not implemented") } + + fn serialize_i64(self, v: i64) -> Result { self.out.write_all(&v.to_be_bytes()).map_err(Error::Io) } + + fn serialize_i32(self, _v: i32) -> Result { unimplemented!("serialize i32 not implemented") } + + fn serialize_i16(self, _v: i16) -> Result { unimplemented!("serialize i16 not implemented") } + + fn serialize_i8(self, _v: i8) -> Result { unimplemented!("serialize i8 not implemented") } + + fn serialize_u64(self, v: u64) -> Result { self.out.write_all(&v.to_be_bytes()).map_err(Error::Io) } + + fn serialize_u32(self, _v: u32) -> Result { unimplemented!("serialize u32 not implemented") } + + fn serialize_u16(self, _v: u16) -> Result { unimplemented!("serialize u16 not implemented") } + + fn serialize_u8(self, v: u8) -> Result { self.out.write_all(&[v]).map_err(Error::Io) } + + fn serialize_bool(self, _v: bool) -> Result { unimplemented!("serialize bool not implemented") } + + fn serialize_unit(self) -> Result { unimplemented!("serialize unit not implemented") } +} + +impl ser::SerializeMap for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_key(&mut self, _key: &T) -> Result { + unimplemented!("serialize Map Key not implemented") + } + + fn serialize_value(&mut self, _val: &T) -> Result { + unimplemented!("serialize Map Val not implemented") + } + + fn end(self) -> Result { + self.sequence_end(); + Ok(()) + } +} + +impl ser::SerializeSeq for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_element(&mut self, val: &T) -> Result { val.serialize(&mut **self) } + + fn end(self) -> Result { + self.sequence_end(); + Ok(()) + } +} + +impl ser::SerializeStruct for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_field(&mut self, _key: &'static str, val: &T) -> Result { + self.record_start()?; + val.serialize(&mut **self) + } + + fn end(self) -> Result { + self.sequence_end(); + Ok(()) + } +} + +impl ser::SerializeStructVariant for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_field(&mut self, _key: &'static str, val: &T) -> Result { + self.record_start()?; + val.serialize(&mut **self) + } + + fn end(self) -> Result { + self.sequence_end(); + Ok(()) + } +} + +impl ser::SerializeTuple for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_element(&mut self, val: &T) -> Result { + self.record_start()?; + val.serialize(&mut **self) + } + + fn end(self) -> Result { + self.sequence_end(); + Ok(()) + } +} + +impl ser::SerializeTupleStruct for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_field(&mut self, val: &T) -> Result { + self.record_start()?; + val.serialize(&mut **self) + } + + fn end(self) -> Result { + self.sequence_end(); + Ok(()) + } +} + +impl ser::SerializeTupleVariant for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_field(&mut self, val: &T) -> Result { + self.record_start()?; + val.serialize(&mut **self) + } + + fn end(self) -> Result { + self.sequence_end(); + Ok(()) + } +} diff --git a/src/database/slice.rs b/src/database/slice.rs deleted file mode 100644 index 448d969d..00000000 --- a/src/database/slice.rs +++ /dev/null @@ -1,57 +0,0 @@ -pub struct OwnedKeyVal(pub OwnedKey, pub OwnedVal); -pub(crate) type OwnedKeyValPair = (OwnedKey, OwnedVal); -pub type OwnedVal = Vec; -pub type OwnedKey = Vec; - -pub struct KeyVal<'item>(pub &'item Key, pub &'item Val); -pub(crate) type KeyValPair<'item> = (&'item Key, &'item Val); -pub type Val = [Byte]; -pub type Key = [Byte]; - -pub(crate) type Byte = u8; - -impl OwnedKeyVal { - #[must_use] - pub fn as_slice(&self) -> KeyVal<'_> { KeyVal(&self.0, &self.1) } - - #[must_use] - pub fn to_tuple(self) -> OwnedKeyValPair { (self.0, self.1) } -} - -impl From for OwnedKeyVal { - fn from((key, val): OwnedKeyValPair) -> Self { Self(key, val) } -} - -impl From<&KeyVal<'_>> for OwnedKeyVal { - #[inline] - fn from(slice: &KeyVal<'_>) -> Self { slice.to_owned() } -} - -impl From> for OwnedKeyVal { - fn from((key, val): KeyValPair<'_>) -> Self { Self(Vec::from(key), Vec::from(val)) } -} - -impl From for OwnedKeyValPair { - fn from(val: OwnedKeyVal) -> Self { val.to_tuple() } -} - -impl KeyVal<'_> { - #[inline] - #[must_use] - pub fn to_owned(&self) -> OwnedKeyVal { OwnedKeyVal::from(self) } - - #[must_use] - pub fn as_tuple(&self) -> KeyValPair<'_> { (self.0, self.1) } -} - -impl<'a> From<&'a OwnedKeyVal> for KeyVal<'a> { - fn from(owned: &'a OwnedKeyVal) -> Self { owned.as_slice() } -} - -impl<'a> From<&'a OwnedKeyValPair> for KeyVal<'a> { - fn from((key, val): &'a OwnedKeyValPair) -> Self { KeyVal(key.as_slice(), val.as_slice()) } -} - -impl<'a> From> for KeyVal<'a> { - fn from((key, val): KeyValPair<'a>) -> Self { KeyVal(key, val) } -} diff --git a/src/database/stream.rs b/src/database/stream.rs new file mode 100644 index 00000000..d9b74215 --- /dev/null +++ b/src/database/stream.rs @@ -0,0 +1,122 @@ +mod items; +mod items_rev; +mod keys; +mod keys_rev; + +use std::sync::Arc; + +use conduit::{utils::exchange, Error, Result}; +use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode, ReadOptions}; + +pub(crate) use self::{items::Items, items_rev::ItemsRev, keys::Keys, keys_rev::KeysRev}; +use crate::{ + engine::Db, + keyval::{Key, KeyVal, Val}, + util::map_err, + Engine, Slice, +}; + +struct State<'a> { + inner: Inner<'a>, + seek: bool, + init: bool, +} + +trait Cursor<'a, T> { + fn state(&self) -> &State<'a>; + + fn fetch(&self) -> Option; + + fn seek(&mut self); + + fn get(&self) -> Option> { + self.fetch() + .map(Ok) + .or_else(|| self.state().status().map(Err)) + } + + fn seek_and_get(&mut self) -> Option> { + self.seek(); + self.get() + } +} + +type Inner<'a> = DBRawIteratorWithThreadMode<'a, Db>; +type From<'a> = Option>; + +impl<'a> State<'a> { + fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { + Self { + inner: db.db.raw_iterator_cf_opt(&**cf, opts), + init: true, + seek: false, + } + } + + fn init_fwd(mut self, from: From<'_>) -> Self { + if let Some(key) = from { + self.inner.seek(key); + self.seek = true; + } + + self + } + + fn init_rev(mut self, from: From<'_>) -> Self { + if let Some(key) = from { + self.inner.seek_for_prev(key); + self.seek = true; + } + + self + } + + fn seek_fwd(&mut self) { + if !exchange(&mut self.init, false) { + self.inner.next(); + } else if !self.seek { + self.inner.seek_to_first(); + } + } + + fn seek_rev(&mut self) { + if !exchange(&mut self.init, false) { + self.inner.prev(); + } else if !self.seek { + self.inner.seek_to_last(); + } + } + + fn fetch_key(&self) -> Option> { self.inner.key().map(Key::from) } + + fn _fetch_val(&self) -> Option> { self.inner.value().map(Val::from) } + + fn fetch(&self) -> Option> { self.inner.item().map(KeyVal::from) } + + fn status(&self) -> Option { self.inner.status().map_err(map_err).err() } + + fn valid(&self) -> bool { self.inner.valid() } +} + +fn keyval_longevity<'a, 'b: 'a>(item: KeyVal<'a>) -> KeyVal<'b> { + (slice_longevity::<'a, 'b>(item.0), slice_longevity::<'a, 'b>(item.1)) +} + +fn slice_longevity<'a, 'b: 'a>(item: &'a Slice) -> &'b Slice { + // SAFETY: The lifetime of the data returned by the rocksdb cursor is only valid + // between each movement of the cursor. It is hereby unsafely extended to match + // the lifetime of the cursor itself. This is due to the limitation of the + // Stream trait where the Item is incapable of conveying a lifetime; this is due + // to GAT's being unstable during its development. This unsafety can be removed + // as soon as this limitation is addressed by an upcoming version. + // + // We have done our best to mitigate the implications of this in conjunction + // with the deserialization API such that borrows being held across movements of + // the cursor do not happen accidentally. The compiler will still error when + // values herein produced try to leave a closure passed to a StreamExt API. But + // escapes can happen if you explicitly and intentionally attempt it, and there + // will be no compiler error or warning. This is primarily the case with + // calling collect() without a preceding map(ToOwned::to_owned). A collection + // of references here is illegal, but this will not be enforced by the compiler. + unsafe { std::mem::transmute(item) } +} diff --git a/src/database/stream/items.rs b/src/database/stream/items.rs new file mode 100644 index 00000000..31d5e9e8 --- /dev/null +++ b/src/database/stream/items.rs @@ -0,0 +1,44 @@ +use std::{pin::Pin, sync::Arc}; + +use conduit::Result; +use futures::{ + stream::FusedStream, + task::{Context, Poll}, + Stream, +}; +use rocksdb::{ColumnFamily, ReadOptions}; + +use super::{keyval_longevity, Cursor, From, State}; +use crate::{keyval::KeyVal, Engine}; + +pub(crate) struct Items<'a> { + state: State<'a>, +} + +impl<'a> Items<'a> { + pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions, from: From<'_>) -> Self { + Self { + state: State::new(db, cf, opts).init_fwd(from), + } + } +} + +impl<'a> Cursor<'a, KeyVal<'a>> for Items<'a> { + fn state(&self) -> &State<'a> { &self.state } + + fn fetch(&self) -> Option> { self.state.fetch().map(keyval_longevity) } + + fn seek(&mut self) { self.state.seek_fwd(); } +} + +impl<'a> Stream for Items<'a> { + type Item = Result>; + + fn poll_next(mut self: Pin<&mut Self>, _ctx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.seek_and_get()) + } +} + +impl FusedStream for Items<'_> { + fn is_terminated(&self) -> bool { !self.state.init && !self.state.valid() } +} diff --git a/src/database/stream/items_rev.rs b/src/database/stream/items_rev.rs new file mode 100644 index 00000000..ab57a250 --- /dev/null +++ b/src/database/stream/items_rev.rs @@ -0,0 +1,44 @@ +use std::{pin::Pin, sync::Arc}; + +use conduit::Result; +use futures::{ + stream::FusedStream, + task::{Context, Poll}, + Stream, +}; +use rocksdb::{ColumnFamily, ReadOptions}; + +use super::{keyval_longevity, Cursor, From, State}; +use crate::{keyval::KeyVal, Engine}; + +pub(crate) struct ItemsRev<'a> { + state: State<'a>, +} + +impl<'a> ItemsRev<'a> { + pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions, from: From<'_>) -> Self { + Self { + state: State::new(db, cf, opts).init_rev(from), + } + } +} + +impl<'a> Cursor<'a, KeyVal<'a>> for ItemsRev<'a> { + fn state(&self) -> &State<'a> { &self.state } + + fn fetch(&self) -> Option> { self.state.fetch().map(keyval_longevity) } + + fn seek(&mut self) { self.state.seek_rev(); } +} + +impl<'a> Stream for ItemsRev<'a> { + type Item = Result>; + + fn poll_next(mut self: Pin<&mut Self>, _ctx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.seek_and_get()) + } +} + +impl FusedStream for ItemsRev<'_> { + fn is_terminated(&self) -> bool { !self.state.init && !self.state.valid() } +} diff --git a/src/database/stream/keys.rs b/src/database/stream/keys.rs new file mode 100644 index 00000000..1c5d12e3 --- /dev/null +++ b/src/database/stream/keys.rs @@ -0,0 +1,44 @@ +use std::{pin::Pin, sync::Arc}; + +use conduit::Result; +use futures::{ + stream::FusedStream, + task::{Context, Poll}, + Stream, +}; +use rocksdb::{ColumnFamily, ReadOptions}; + +use super::{slice_longevity, Cursor, From, State}; +use crate::{keyval::Key, Engine}; + +pub(crate) struct Keys<'a> { + state: State<'a>, +} + +impl<'a> Keys<'a> { + pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions, from: From<'_>) -> Self { + Self { + state: State::new(db, cf, opts).init_fwd(from), + } + } +} + +impl<'a> Cursor<'a, Key<'a>> for Keys<'a> { + fn state(&self) -> &State<'a> { &self.state } + + fn fetch(&self) -> Option> { self.state.fetch_key().map(slice_longevity) } + + fn seek(&mut self) { self.state.seek_fwd(); } +} + +impl<'a> Stream for Keys<'a> { + type Item = Result>; + + fn poll_next(mut self: Pin<&mut Self>, _ctx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.seek_and_get()) + } +} + +impl FusedStream for Keys<'_> { + fn is_terminated(&self) -> bool { !self.state.init && !self.state.valid() } +} diff --git a/src/database/stream/keys_rev.rs b/src/database/stream/keys_rev.rs new file mode 100644 index 00000000..26707483 --- /dev/null +++ b/src/database/stream/keys_rev.rs @@ -0,0 +1,44 @@ +use std::{pin::Pin, sync::Arc}; + +use conduit::Result; +use futures::{ + stream::FusedStream, + task::{Context, Poll}, + Stream, +}; +use rocksdb::{ColumnFamily, ReadOptions}; + +use super::{slice_longevity, Cursor, From, State}; +use crate::{keyval::Key, Engine}; + +pub(crate) struct KeysRev<'a> { + state: State<'a>, +} + +impl<'a> KeysRev<'a> { + pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions, from: From<'_>) -> Self { + Self { + state: State::new(db, cf, opts).init_rev(from), + } + } +} + +impl<'a> Cursor<'a, Key<'a>> for KeysRev<'a> { + fn state(&self) -> &State<'a> { &self.state } + + fn fetch(&self) -> Option> { self.state.fetch_key().map(slice_longevity) } + + fn seek(&mut self) { self.state.seek_rev(); } +} + +impl<'a> Stream for KeysRev<'a> { + type Item = Result>; + + fn poll_next(mut self: Pin<&mut Self>, _ctx: &mut Context<'_>) -> Poll> { + Poll::Ready(self.seek_and_get()) + } +} + +impl FusedStream for KeysRev<'_> { + fn is_terminated(&self) -> bool { !self.state.init && !self.state.valid() } +} diff --git a/src/database/util.rs b/src/database/util.rs index f0ccbcbe..d36e183f 100644 --- a/src/database/util.rs +++ b/src/database/util.rs @@ -1,4 +1,16 @@ use conduit::{err, Result}; +use rocksdb::{Direction, IteratorMode}; + +#[inline] +pub(crate) fn _into_direction(mode: &IteratorMode<'_>) -> Direction { + use Direction::{Forward, Reverse}; + use IteratorMode::{End, From, Start}; + + match mode { + Start | From(_, Forward) => Forward, + End | From(_, Reverse) => Reverse, + } +} #[inline] pub(crate) fn result(r: std::result::Result) -> Result { diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index cfed5a0e..737a7039 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -46,7 +46,7 @@ bytes.workspace = true conduit-core.workspace = true conduit-database.workspace = true const-str.workspace = true -futures-util.workspace = true +futures.workspace = true hickory-resolver.workspace = true http.workspace = true image.workspace = true diff --git a/src/service/account_data/data.rs b/src/service/account_data/data.rs deleted file mode 100644 index 53a0e953..00000000 --- a/src/service/account_data/data.rs +++ /dev/null @@ -1,152 +0,0 @@ -use std::{collections::HashMap, sync::Arc}; - -use conduit::{Error, Result}; -use database::Map; -use ruma::{ - api::client::error::ErrorKind, - events::{AnyGlobalAccountDataEvent, AnyRawAccountDataEvent, AnyRoomAccountDataEvent, RoomAccountDataEventType}, - serde::Raw, - RoomId, UserId, -}; - -use crate::{globals, Dep}; - -pub(super) struct Data { - roomuserdataid_accountdata: Arc, - roomusertype_roomuserdataid: Arc, - services: Services, -} - -struct Services { - globals: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - roomuserdataid_accountdata: db["roomuserdataid_accountdata"].clone(), - roomusertype_roomuserdataid: db["roomusertype_roomuserdataid"].clone(), - services: Services { - globals: args.depend::("globals"), - }, - } - } - - /// Places one event in the account data of the user and removes the - /// previous entry. - pub(super) fn update( - &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: &RoomAccountDataEventType, - data: &serde_json::Value, - ) -> Result<()> { - let mut prefix = room_id - .map(ToString::to_string) - .unwrap_or_default() - .as_bytes() - .to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(user_id.as_bytes()); - prefix.push(0xFF); - - let mut roomuserdataid = prefix.clone(); - roomuserdataid.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); - roomuserdataid.push(0xFF); - roomuserdataid.extend_from_slice(event_type.to_string().as_bytes()); - - let mut key = prefix; - key.extend_from_slice(event_type.to_string().as_bytes()); - - if data.get("type").is_none() || data.get("content").is_none() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Account data doesn't have all required fields.", - )); - } - - self.roomuserdataid_accountdata.insert( - &roomuserdataid, - &serde_json::to_vec(&data).expect("to_vec always works on json values"), - )?; - - let prev = self.roomusertype_roomuserdataid.get(&key)?; - - self.roomusertype_roomuserdataid - .insert(&key, &roomuserdataid)?; - - // Remove old entry - if let Some(prev) = prev { - self.roomuserdataid_accountdata.remove(&prev)?; - } - - Ok(()) - } - - /// Searches the account data for a specific kind. - pub(super) fn get( - &self, room_id: Option<&RoomId>, user_id: &UserId, kind: &RoomAccountDataEventType, - ) -> Result>> { - let mut key = room_id - .map(ToString::to_string) - .unwrap_or_default() - .as_bytes() - .to_vec(); - key.push(0xFF); - key.extend_from_slice(user_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(kind.to_string().as_bytes()); - - self.roomusertype_roomuserdataid - .get(&key)? - .and_then(|roomuserdataid| { - self.roomuserdataid_accountdata - .get(&roomuserdataid) - .transpose() - }) - .transpose()? - .map(|data| serde_json::from_slice(&data).map_err(|_| Error::bad_database("could not deserialize"))) - .transpose() - } - - /// Returns all changes to the account data that happened after `since`. - pub(super) fn changes_since( - &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, - ) -> Result> { - let mut userdata = HashMap::new(); - - let mut prefix = room_id - .map(ToString::to_string) - .unwrap_or_default() - .as_bytes() - .to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(user_id.as_bytes()); - prefix.push(0xFF); - - // Skip the data that's exactly at since, because we sent that last time - let mut first_possible = prefix.clone(); - first_possible.extend_from_slice(&(since.saturating_add(1)).to_be_bytes()); - - for r in self - .roomuserdataid_accountdata - .iter_from(&first_possible, false) - .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(|(k, v)| { - Ok::<_, Error>(( - k, - match room_id { - None => serde_json::from_slice::>(&v) - .map(AnyRawAccountDataEvent::Global) - .map_err(|_| Error::bad_database("Database contains invalid account data."))?, - Some(_) => serde_json::from_slice::>(&v) - .map(AnyRawAccountDataEvent::Room) - .map_err(|_| Error::bad_database("Database contains invalid account data."))?, - }, - )) - }) { - let (kind, data) = r?; - userdata.insert(kind, data); - } - - Ok(userdata.into_values().collect()) - } -} diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index eaa53641..b4eb143d 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -1,52 +1,158 @@ -mod data; +use std::{collections::HashMap, sync::Arc}; -use std::sync::Arc; - -use conduit::Result; -use data::Data; +use conduit::{ + implement, + utils::{stream::TryIgnore, ReadyExt}, + Err, Error, Result, +}; +use database::{Deserialized, Map}; +use futures::{StreamExt, TryFutureExt}; use ruma::{ - events::{AnyRawAccountDataEvent, RoomAccountDataEventType}, + events::{AnyGlobalAccountDataEvent, AnyRawAccountDataEvent, AnyRoomAccountDataEvent, RoomAccountDataEventType}, + serde::Raw, RoomId, UserId, }; +use serde_json::value::RawValue; + +use crate::{globals, Dep}; pub struct Service { + services: Services, db: Data, } +struct Data { + roomuserdataid_accountdata: Arc, + roomusertype_roomuserdataid: Arc, +} + +struct Services { + globals: Dep, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(&args), + services: Services { + globals: args.depend::("globals"), + }, + db: Data { + roomuserdataid_accountdata: args.db["roomuserdataid_accountdata"].clone(), + roomusertype_roomuserdataid: args.db["roomusertype_roomuserdataid"].clone(), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - /// Places one event in the account data of the user and removes the - /// previous entry. - #[allow(clippy::needless_pass_by_value)] - pub fn update( - &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: RoomAccountDataEventType, - data: &serde_json::Value, - ) -> Result<()> { - self.db.update(room_id, user_id, &event_type, data) +/// Places one event in the account data of the user and removes the +/// previous entry. +#[allow(clippy::needless_pass_by_value)] +#[implement(Service)] +pub async fn update( + &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: RoomAccountDataEventType, data: &serde_json::Value, +) -> Result<()> { + let event_type = event_type.to_string(); + let count = self.services.globals.next_count()?; + + let mut prefix = room_id + .map(ToString::to_string) + .unwrap_or_default() + .as_bytes() + .to_vec(); + prefix.push(0xFF); + prefix.extend_from_slice(user_id.as_bytes()); + prefix.push(0xFF); + + let mut roomuserdataid = prefix.clone(); + roomuserdataid.extend_from_slice(&count.to_be_bytes()); + roomuserdataid.push(0xFF); + roomuserdataid.extend_from_slice(event_type.as_bytes()); + + let mut key = prefix; + key.extend_from_slice(event_type.as_bytes()); + + if data.get("type").is_none() || data.get("content").is_none() { + return Err!(Request(InvalidParam("Account data doesn't have all required fields."))); } - /// Searches the account data for a specific kind. - #[allow(clippy::needless_pass_by_value)] - pub fn get( - &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: RoomAccountDataEventType, - ) -> Result>> { - self.db.get(room_id, user_id, &event_type) + self.db.roomuserdataid_accountdata.insert( + &roomuserdataid, + &serde_json::to_vec(&data).expect("to_vec always works on json values"), + ); + + let prev_key = (room_id, user_id, &event_type); + let prev = self.db.roomusertype_roomuserdataid.qry(&prev_key).await; + + self.db + .roomusertype_roomuserdataid + .insert(&key, &roomuserdataid); + + // Remove old entry + if let Ok(prev) = prev { + self.db.roomuserdataid_accountdata.remove(&prev); } - /// Returns all changes to the account data that happened after `since`. - #[tracing::instrument(skip_all, name = "since", level = "debug")] - pub fn changes_since( - &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, - ) -> Result> { - self.db.changes_since(room_id, user_id, since) - } + Ok(()) +} + +/// Searches the account data for a specific kind. +#[implement(Service)] +pub async fn get( + &self, room_id: Option<&RoomId>, user_id: &UserId, kind: RoomAccountDataEventType, +) -> Result> { + let key = (room_id, user_id, kind.to_string()); + self.db + .roomusertype_roomuserdataid + .qry(&key) + .and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.qry(&roomuserdataid)) + .await + .deserialized_json() +} + +/// Returns all changes to the account data that happened after `since`. +#[implement(Service)] +pub async fn changes_since( + &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, +) -> Result> { + let mut userdata = HashMap::new(); + + let mut prefix = room_id + .map(ToString::to_string) + .unwrap_or_default() + .as_bytes() + .to_vec(); + prefix.push(0xFF); + prefix.extend_from_slice(user_id.as_bytes()); + prefix.push(0xFF); + + // Skip the data that's exactly at since, because we sent that last time + let mut first_possible = prefix.clone(); + first_possible.extend_from_slice(&(since.saturating_add(1)).to_be_bytes()); + + self.db + .roomuserdataid_accountdata + .raw_stream_from(&first_possible) + .ignore_err() + .ready_take_while(move |(k, _)| k.starts_with(&prefix)) + .map(|(k, v)| { + let v = match room_id { + None => serde_json::from_slice::>(v) + .map(AnyRawAccountDataEvent::Global) + .map_err(|_| Error::bad_database("Database contains invalid account data."))?, + Some(_) => serde_json::from_slice::>(v) + .map(AnyRawAccountDataEvent::Room) + .map_err(|_| Error::bad_database("Database contains invalid account data."))?, + }; + + Ok((k.to_owned(), v)) + }) + .ignore_err() + .ready_for_each(|(kind, data)| { + userdata.insert(kind, data); + }) + .await; + + Ok(userdata.into_values().collect()) } diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 55bae365..0f5016e1 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -5,7 +5,7 @@ use std::{ }; use conduit::{debug, defer, error, log, Server}; -use futures_util::future::{AbortHandle, Abortable}; +use futures::future::{AbortHandle, Abortable}; use ruma::events::room::message::RoomMessageEventContent; use rustyline_async::{Readline, ReadlineError, ReadlineEvent}; use termimad::MadSkin; diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 4e2b831c..7b090aa0 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -30,7 +30,7 @@ use crate::Services; pub async fn create_admin_room(services: &Services) -> Result<()> { let room_id = RoomId::new(services.globals.server_name()); - let _short_id = services.rooms.short.get_or_create_shortroomid(&room_id)?; + let _short_id = services.rooms.short.get_or_create_shortroomid(&room_id); let state_lock = services.rooms.state.mutex.lock(&room_id).await; diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index b4589ebc..4b3ebb88 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -17,108 +17,108 @@ use serde_json::value::to_raw_value; use crate::pdu::PduBuilder; -impl super::Service { - /// Invite the user to the conduit admin room. - /// - /// In conduit, this is equivalent to granting admin privileges. - pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { - let Some(room_id) = self.get_admin_room()? else { - return Ok(()); - }; +/// Invite the user to the conduit admin room. +/// +/// In conduit, this is equivalent to granting admin privileges. +#[implement(super::Service)] +pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { + let Ok(room_id) = self.get_admin_room().await else { + return Ok(()); + }; - let state_lock = self.services.state.mutex.lock(&room_id).await; + let state_lock = self.services.state.mutex.lock(&room_id).await; - // Use the server user to grant the new admin's power level - let server_user = &self.services.globals.server_user; + // Use the server user to grant the new admin's power level + let server_user = &self.services.globals.server_user; - // Invite and join the real user - self.services - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Invite, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; - self.services - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Join, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }, - user_id, - &room_id, - &state_lock, - ) - .await?; + // Invite and join the real user + self.services + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Invite, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + join_authorized_via_users_server: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + timestamp: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; + self.services + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + join_authorized_via_users_server: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + timestamp: None, + }, + user_id, + &room_id, + &state_lock, + ) + .await?; - // Set power level - let users = BTreeMap::from_iter([(server_user.clone(), 100.into()), (user_id.to_owned(), 100.into())]); + // Set power level + let users = BTreeMap::from_iter([(server_user.clone(), 100.into()), (user_id.to_owned(), 100.into())]); - self.services - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&RoomPowerLevelsEventContent { - users, - ..Default::default() - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, - server_user, - &room_id, - &state_lock, - ) - .await?; + self.services + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomPowerLevels, + content: to_raw_value(&RoomPowerLevelsEventContent { + users, + ..Default::default() + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + timestamp: None, + }, + server_user, + &room_id, + &state_lock, + ) + .await?; - // Set room tag - let room_tag = &self.services.server.config.admin_room_tag; - if !room_tag.is_empty() { - if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag) { - error!(?room_id, ?user_id, ?room_tag, ?e, "Failed to set tag for admin grant"); - } + // Set room tag + let room_tag = &self.services.server.config.admin_room_tag; + if !room_tag.is_empty() { + if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag).await { + error!(?room_id, ?user_id, ?room_tag, ?e, "Failed to set tag for admin grant"); } + } - // Send welcome message - self.services.timeline.build_and_append_pdu( + // Send welcome message + self.services.timeline.build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomMessage, content: to_raw_value(&RoomMessageEventContent::text_markdown( @@ -135,19 +135,18 @@ impl super::Service { &state_lock, ).await?; - Ok(()) - } + Ok(()) } #[implement(super::Service)] -fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result<()> { +async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result<()> { let mut event = self .services .account_data - .get(Some(room_id), user_id, RoomAccountDataEventType::Tag)? - .map(|event| serde_json::from_str(event.get())) - .and_then(Result::ok) - .unwrap_or_else(|| TagEvent { + .get(Some(room_id), user_id, RoomAccountDataEventType::Tag) + .await + .and_then(|event| serde_json::from_str(event.get()).map_err(Into::into)) + .unwrap_or_else(|_| TagEvent { content: TagEventContent { tags: BTreeMap::new(), }, @@ -158,12 +157,15 @@ fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result< .tags .insert(tag.to_owned().into(), TagInfo::new()); - self.services.account_data.update( - Some(room_id), - user_id, - RoomAccountDataEventType::Tag, - &serde_json::to_value(event)?, - )?; + self.services + .account_data + .update( + Some(room_id), + user_id, + RoomAccountDataEventType::Tag, + &serde_json::to_value(event)?, + ) + .await?; Ok(()) } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 3274249e..12eacc8f 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -12,6 +12,7 @@ use std::{ use async_trait::async_trait; use conduit::{debug, err, error, error::default_log, pdu::PduBuilder, Error, PduEvent, Result, Server}; pub use create::create_admin_room; +use futures::{FutureExt, TryFutureExt}; use loole::{Receiver, Sender}; use ruma::{ events::{ @@ -142,17 +143,18 @@ impl Service { /// admin room as the admin user. pub async fn send_text(&self, body: &str) { self.send_message(RoomMessageEventContent::text_markdown(body)) - .await; + .await + .ok(); } /// Sends a message to the admin room as the admin user (see send_text() for /// convenience). - pub async fn send_message(&self, message_content: RoomMessageEventContent) { - if let Ok(Some(room_id)) = self.get_admin_room() { - let user_id = &self.services.globals.server_user; - self.respond_to_room(message_content, &room_id, user_id) - .await; - } + pub async fn send_message(&self, message_content: RoomMessageEventContent) -> Result<()> { + let user_id = &self.services.globals.server_user; + let room_id = self.get_admin_room().await?; + self.respond_to_room(message_content, &room_id, user_id) + .boxed() + .await } /// Posts a command to the command processor queue and returns. Processing @@ -193,8 +195,12 @@ impl Service { async fn handle_command(&self, command: CommandInput) { match self.process_command(command).await { - Ok(Some(output)) | Err(output) => self.handle_response(output).await, Ok(None) => debug!("Command successful with no response"), + Ok(Some(output)) | Err(output) => self + .handle_response(output) + .boxed() + .await + .unwrap_or_else(default_log), } } @@ -218,71 +224,67 @@ impl Service { } /// Checks whether a given user is an admin of this server - pub async fn user_is_admin(&self, user_id: &UserId) -> Result { - if let Ok(Some(admin_room)) = self.get_admin_room() { - self.services.state_cache.is_joined(user_id, &admin_room) - } else { - Ok(false) - } + pub async fn user_is_admin(&self, user_id: &UserId) -> bool { + let Ok(admin_room) = self.get_admin_room().await else { + return false; + }; + + self.services + .state_cache + .is_joined(user_id, &admin_room) + .await } /// Gets the room ID of the admin room /// /// Errors are propagated from the database, and will have None if there is /// no admin room - pub fn get_admin_room(&self) -> Result> { - if let Some(room_id) = self + pub async fn get_admin_room(&self) -> Result { + let room_id = self .services .alias - .resolve_local_alias(&self.services.globals.admin_alias)? - { - if self - .services - .state_cache - .is_joined(&self.services.globals.server_user, &room_id)? - { - return Ok(Some(room_id)); - } - } + .resolve_local_alias(&self.services.globals.admin_alias) + .await?; - Ok(None) + self.services + .state_cache + .is_joined(&self.services.globals.server_user, &room_id) + .await + .then_some(room_id) + .ok_or_else(|| err!(Request(NotFound("Admin user not joined to admin room")))) } - async fn handle_response(&self, content: RoomMessageEventContent) { + async fn handle_response(&self, content: RoomMessageEventContent) -> Result<()> { let Some(Relation::Reply { in_reply_to, }) = content.relates_to.as_ref() else { - return; + return Ok(()); }; - let Ok(Some(pdu)) = self.services.timeline.get_pdu(&in_reply_to.event_id) else { + let Ok(pdu) = self.services.timeline.get_pdu(&in_reply_to.event_id).await else { error!( event_id = ?in_reply_to.event_id, "Missing admin command in_reply_to event" ); - return; + return Ok(()); }; - let response_sender = if self.is_admin_room(&pdu.room_id) { + let response_sender = if self.is_admin_room(&pdu.room_id).await { &self.services.globals.server_user } else { &pdu.sender }; self.respond_to_room(content, &pdu.room_id, response_sender) - .await; + .await } - async fn respond_to_room(&self, content: RoomMessageEventContent, room_id: &RoomId, user_id: &UserId) { - assert!( - self.user_is_admin(user_id) - .await - .expect("checked user is admin"), - "sender is not admin" - ); + async fn respond_to_room( + &self, content: RoomMessageEventContent, room_id: &RoomId, user_id: &UserId, + ) -> Result<()> { + assert!(self.user_is_admin(user_id).await, "sender is not admin"); - let state_lock = self.services.state.mutex.lock(room_id).await; let response_pdu = PduBuilder { event_type: TimelineEventType::RoomMessage, content: to_raw_value(&content).expect("event is valid, we just created it"), @@ -292,6 +294,7 @@ impl Service { timestamp: None, }; + let state_lock = self.services.state.mutex.lock(room_id).await; if let Err(e) = self .services .timeline @@ -302,6 +305,8 @@ impl Service { .await .unwrap_or_else(default_log); } + + Ok(()) } async fn handle_response_error( @@ -355,12 +360,12 @@ impl Service { } // Prevent unescaped !admin from being used outside of the admin room - if is_public_prefix && !self.is_admin_room(&pdu.room_id) { + if is_public_prefix && !self.is_admin_room(&pdu.room_id).await { return false; } // Only senders who are admin can proceed - if !self.user_is_admin(&pdu.sender).await.unwrap_or(false) { + if !self.user_is_admin(&pdu.sender).await { return false; } @@ -368,7 +373,7 @@ impl Service { // the administrator can execute commands as conduit let emergency_password_set = self.services.globals.emergency_password().is_some(); let from_server = pdu.sender == *server_user && !emergency_password_set; - if from_server && self.is_admin_room(&pdu.room_id) { + if from_server && self.is_admin_room(&pdu.room_id).await { return false; } @@ -377,12 +382,11 @@ impl Service { } #[must_use] - pub fn is_admin_room(&self, room_id: &RoomId) -> bool { - if let Ok(Some(admin_room_id)) = self.get_admin_room() { - admin_room_id == room_id - } else { - false - } + pub async fn is_admin_room(&self, room_id_: &RoomId) -> bool { + self.get_admin_room() + .map_ok(|room_id| room_id == room_id_) + .await + .unwrap_or(false) } /// Sets the self-reference to crate::Services which will provide context to diff --git a/src/service/appservice/data.rs b/src/service/appservice/data.rs index 40e641a1..d5fa5476 100644 --- a/src/service/appservice/data.rs +++ b/src/service/appservice/data.rs @@ -1,7 +1,8 @@ use std::sync::Arc; -use conduit::{utils, Error, Result}; -use database::{Database, Map}; +use conduit::{err, utils::stream::TryIgnore, Result}; +use database::{Database, Deserialized, Map}; +use futures::Stream; use ruma::api::appservice::Registration; pub struct Data { @@ -19,7 +20,7 @@ impl Data { pub(super) fn register_appservice(&self, yaml: &Registration) -> Result { let id = yaml.id.as_str(); self.id_appserviceregistrations - .insert(id.as_bytes(), serde_yaml::to_string(&yaml).unwrap().as_bytes())?; + .insert(id.as_bytes(), serde_yaml::to_string(&yaml).unwrap().as_bytes()); Ok(id.to_owned()) } @@ -31,24 +32,19 @@ impl Data { /// * `service_name` - the name you send to register the service previously pub(super) fn unregister_appservice(&self, service_name: &str) -> Result<()> { self.id_appserviceregistrations - .remove(service_name.as_bytes())?; + .remove(service_name.as_bytes()); Ok(()) } - pub fn get_registration(&self, id: &str) -> Result> { + pub async fn get_registration(&self, id: &str) -> Result { self.id_appserviceregistrations - .get(id.as_bytes())? - .map(|bytes| { - serde_yaml::from_slice(&bytes) - .map_err(|_| Error::bad_database("Invalid registration bytes in id_appserviceregistrations.")) - }) - .transpose() + .qry(id) + .await + .deserialized_json() + .map_err(|e| err!(Database("Invalid appservice {id:?} registration: {e:?}"))) } - pub(super) fn iter_ids<'a>(&'a self) -> Result> + 'a>> { - Ok(Box::new(self.id_appserviceregistrations.iter().map(|(id, _)| { - utils::string_from_bytes(&id) - .map_err(|_| Error::bad_database("Invalid id bytes in id_appserviceregistrations.")) - }))) + pub(super) fn iter_ids(&self) -> impl Stream + Send + '_ { + self.id_appserviceregistrations.keys().ignore_err() } } diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index c0752d56..7e2dc738 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -2,9 +2,10 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; +use async_trait::async_trait; use conduit::{err, Result}; use data::Data; -use futures_util::Future; +use futures::{Future, StreamExt, TryStreamExt}; use regex::RegexSet; use ruma::{ api::appservice::{Namespace, Registration}, @@ -126,13 +127,22 @@ struct Services { sending: Dep, } +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { - let mut registration_info = BTreeMap::new(); - let db = Data::new(args.db); + Ok(Arc::new(Self { + db: Data::new(args.db), + services: Services { + sending: args.depend::("sending"), + }, + registration_info: RwLock::new(BTreeMap::new()), + })) + } + + async fn worker(self: Arc) -> Result<()> { // Inserting registrations into cache - for appservice in iter_ids(&db)? { - registration_info.insert( + for appservice in iter_ids(&self.db).await? { + self.registration_info.write().await.insert( appservice.0, appservice .1 @@ -141,13 +151,7 @@ impl crate::Service for Service { ); } - Ok(Arc::new(Self { - db, - services: Services { - sending: args.depend::("sending"), - }, - registration_info: RwLock::new(registration_info), - })) + Ok(()) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } @@ -155,7 +159,7 @@ impl crate::Service for Service { impl Service { #[inline] - pub fn all(&self) -> Result> { iter_ids(&self.db) } + pub async fn all(&self) -> Result> { iter_ids(&self.db).await } /// Registers an appservice and returns the ID to the caller pub async fn register_appservice(&self, yaml: Registration) -> Result { @@ -188,7 +192,8 @@ impl Service { // sending to the URL self.services .sending - .cleanup_events(service_name.to_owned())?; + .cleanup_events(service_name.to_owned()) + .await; Ok(()) } @@ -251,15 +256,9 @@ impl Service { } } -fn iter_ids(db: &Data) -> Result> { - db.iter_ids()? - .filter_map(Result::ok) - .map(move |id| { - Ok(( - id.clone(), - db.get_registration(&id)? - .expect("iter_ids only returns appservices that exist"), - )) - }) - .collect() +async fn iter_ids(db: &Data) -> Result> { + db.iter_ids() + .then(|id| async move { Ok((id.clone(), db.get_registration(&id).await?)) }) + .try_collect() + .await } diff --git a/src/service/emergency/mod.rs b/src/service/emergency/mod.rs index 1bb0843d..98020bc2 100644 --- a/src/service/emergency/mod.rs +++ b/src/service/emergency/mod.rs @@ -33,6 +33,7 @@ impl crate::Service for Service { async fn worker(self: Arc) -> Result<()> { self.set_emergency_access() + .await .inspect_err(|e| error!("Could not set the configured emergency password for the conduit user: {e}"))?; Ok(()) @@ -44,7 +45,7 @@ impl crate::Service for Service { impl Service { /// Sets the emergency password and push rules for the @conduit account in /// case emergency password is set - fn set_emergency_access(&self) -> Result { + async fn set_emergency_access(&self) -> Result { let conduit_user = &self.services.globals.server_user; self.services @@ -56,17 +57,20 @@ impl Service { None => (Ruleset::new(), false), }; - self.services.account_data.update( - None, - conduit_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(&GlobalAccountDataEvent { - content: PushRulesEventContent { - global: ruleset, - }, - }) - .expect("to json value always works"), - )?; + self.services + .account_data + .update( + None, + conduit_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(&GlobalAccountDataEvent { + content: PushRulesEventContent { + global: ruleset, + }, + }) + .expect("to json value always works"), + ) + .await?; if pwd_set { warn!( @@ -75,7 +79,7 @@ impl Service { ); } else { // logs out any users still in the server service account and removes sessions - self.services.users.deactivate_account(conduit_user)?; + self.services.users.deactivate_account(conduit_user).await?; } Ok(pwd_set) diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 5b5d9f09..3286e40c 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -4,8 +4,8 @@ use std::{ }; use conduit::{trace, utils, Error, Result, Server}; -use database::{Database, Map}; -use futures_util::{stream::FuturesUnordered, StreamExt}; +use database::{Database, Deserialized, Map}; +use futures::{pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; use ruma::{ api::federation::discovery::{ServerSigningKeys, VerifyKey}, signatures::Ed25519KeyPair, @@ -83,7 +83,7 @@ impl Data { .checked_add(1) .expect("counter must not overflow u64"); - self.global.insert(COUNTER, &counter.to_be_bytes())?; + self.global.insert(COUNTER, &counter.to_be_bytes()); Ok(*counter) } @@ -102,7 +102,7 @@ impl Data { fn stored_count(global: &Arc) -> Result { global - .get(COUNTER)? + .get(COUNTER) .as_deref() .map_or(Ok(0_u64), utils::u64_from_bytes) } @@ -133,36 +133,18 @@ impl Data { futures.push(self.userroomid_highlightcount.watch_prefix(&userid_prefix)); // Events for rooms we are in - for room_id in self - .services - .state_cache - .rooms_joined(user_id) - .filter_map(Result::ok) - { - let short_roomid = self - .services - .short - .get_shortroomid(&room_id) - .ok() - .flatten() - .expect("room exists") - .to_be_bytes() - .to_vec(); + let rooms_joined = self.services.state_cache.rooms_joined(user_id); + + pin_mut!(rooms_joined); + while let Some(room_id) = rooms_joined.next().await { + let Ok(short_roomid) = self.services.short.get_shortroomid(room_id).await else { + continue; + }; let roomid_bytes = room_id.as_bytes().to_vec(); let mut roomid_prefix = roomid_bytes.clone(); roomid_prefix.push(0xFF); - // PDUs - futures.push(self.pduid_pdu.watch_prefix(&short_roomid)); - - // EDUs - futures.push(Box::pin(async move { - let _result = self.services.typing.wait_for_update(&room_id).await; - })); - - futures.push(self.readreceiptid_readreceipt.watch_prefix(&roomid_prefix)); - // Key changes futures.push(self.keychangeid_userid.watch_prefix(&roomid_prefix)); @@ -174,6 +156,19 @@ impl Data { self.roomusertype_roomuserdataid .watch_prefix(&roomuser_prefix), ); + + // PDUs + let short_roomid = short_roomid.to_be_bytes().to_vec(); + futures.push(self.pduid_pdu.watch_prefix(&short_roomid)); + + // EDUs + let typing_room_id = room_id.to_owned(); + let typing_wait_for_update = async move { + self.services.typing.wait_for_update(&typing_room_id).await; + }; + + futures.push(typing_wait_for_update.boxed()); + futures.push(self.readreceiptid_readreceipt.watch_prefix(&roomid_prefix)); } let mut globaluserdata_prefix = vec![0xFF]; @@ -190,12 +185,14 @@ impl Data { // One time keys futures.push(self.userid_lastonetimekeyupdate.watch_prefix(&userid_bytes)); - futures.push(Box::pin(async move { + // Server shutdown + let server_shutdown = async move { while self.services.server.running() { - let _result = self.services.server.signal.subscribe().recv().await; + self.services.server.signal.subscribe().recv().await.ok(); } - })); + }; + futures.push(server_shutdown.boxed()); if !self.services.server.running() { return Ok(()); } @@ -209,10 +206,10 @@ impl Data { } pub fn load_keypair(&self) -> Result { - let keypair_bytes = self.global.get(b"keypair")?.map_or_else( - || { + let keypair_bytes = self.global.get(b"keypair").map_or_else( + |_| { let keypair = utils::generate_keypair(); - self.global.insert(b"keypair", &keypair)?; + self.global.insert(b"keypair", &keypair); Ok::<_, Error>(keypair) }, |val| Ok(val.to_vec()), @@ -241,7 +238,10 @@ impl Data { } #[inline] - pub fn remove_keypair(&self) -> Result<()> { self.global.remove(b"keypair") } + pub fn remove_keypair(&self) -> Result<()> { + self.global.remove(b"keypair"); + Ok(()) + } /// TODO: the key valid until timestamp (`valid_until_ts`) is only honored /// in room version > 4 @@ -250,15 +250,15 @@ impl Data { /// /// This doesn't actually check that the keys provided are newer than the /// old set. - pub fn add_signing_key( + pub async fn add_signing_key( &self, origin: &ServerName, new_keys: ServerSigningKeys, - ) -> Result> { + ) -> BTreeMap { // Not atomic, but this is not critical - let signingkeys = self.server_signingkeys.get(origin.as_bytes())?; + let signingkeys = self.server_signingkeys.qry(origin).await; let mut keys = signingkeys - .and_then(|keys| serde_json::from_slice(&keys).ok()) - .unwrap_or_else(|| { + .and_then(|keys| serde_json::from_slice(&keys).map_err(Into::into)) + .unwrap_or_else(|_| { // Just insert "now", it doesn't matter ServerSigningKeys::new(origin.to_owned(), MilliSecondsSinceUnixEpoch::now()) }); @@ -275,7 +275,7 @@ impl Data { self.server_signingkeys.insert( origin.as_bytes(), &serde_json::to_vec(&keys).expect("serversigningkeys can be serialized"), - )?; + ); let mut tree = keys.verify_keys; tree.extend( @@ -284,45 +284,38 @@ impl Data { .map(|old| (old.0, VerifyKey::new(old.1.key))), ); - Ok(tree) + tree } /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. - pub fn verify_keys_for(&self, origin: &ServerName) -> Result> { - let signingkeys = self - .signing_keys_for(origin)? - .map_or_else(BTreeMap::new, |keys: ServerSigningKeys| { + pub async fn verify_keys_for(&self, origin: &ServerName) -> Result> { + self.signing_keys_for(origin).await.map_or_else( + |_| Ok(BTreeMap::new()), + |keys: ServerSigningKeys| { let mut tree = keys.verify_keys; tree.extend( keys.old_verify_keys .into_iter() .map(|old| (old.0, VerifyKey::new(old.1.key))), ); - tree - }); - - Ok(signingkeys) + Ok(tree) + }, + ) } - pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { - let signingkeys = self - .server_signingkeys - .get(origin.as_bytes())? - .and_then(|bytes| serde_json::from_slice(&bytes).ok()); - - Ok(signingkeys) + pub async fn signing_keys_for(&self, origin: &ServerName) -> Result { + self.server_signingkeys + .qry(origin) + .await + .deserialized_json() } - pub fn database_version(&self) -> Result { - self.global.get(b"version")?.map_or(Ok(0), |version| { - utils::u64_from_bytes(&version).map_err(|_| Error::bad_database("Database version id is invalid.")) - }) - } + pub async fn database_version(&self) -> u64 { self.global.qry("version").await.deserialized().unwrap_or(0) } #[inline] pub fn bump_database_version(&self, new_version: u64) -> Result<()> { - self.global.insert(b"version", &new_version.to_be_bytes())?; + self.global.insert(b"version", &new_version.to_be_bytes()); Ok(()) } diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 66917520..c7a73230 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -1,17 +1,15 @@ -use std::{ - collections::{HashMap, HashSet}, - fs::{self}, - io::Write, - mem::size_of, - sync::Arc, +use conduit::{ + debug_info, debug_warn, error, info, + result::NotFound, + utils::{stream::TryIgnore, IterStream, ReadyExt}, + warn, Err, Error, Result, }; - -use conduit::{debug, debug_info, debug_warn, error, info, utils, warn, Error, Result}; +use futures::{FutureExt, StreamExt}; use itertools::Itertools; use ruma::{ events::{push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType}, push::Ruleset, - EventId, OwnedRoomId, RoomId, UserId, + UserId, }; use crate::{media, Services}; @@ -33,12 +31,14 @@ pub(crate) const DATABASE_VERSION: u64 = 13; pub(crate) const CONDUIT_DATABASE_VERSION: u64 = 16; pub(crate) async fn migrations(services: &Services) -> Result<()> { + let users_count = services.users.count().await; + // Matrix resource ownership is based on the server name; changing it // requires recreating the database from scratch. - if services.users.count()? > 0 { + if users_count > 0 { let conduit_user = &services.globals.server_user; - if !services.users.exists(conduit_user)? { + if !services.users.exists(conduit_user).await { error!("The {} server user does not exist, and the database is not new.", conduit_user); return Err(Error::bad_database( "Cannot reuse an existing database after changing the server name, please delete the old one first.", @@ -46,7 +46,7 @@ pub(crate) async fn migrations(services: &Services) -> Result<()> { } } - if services.users.count()? > 0 { + if users_count > 0 { migrate(services).await } else { fresh(services).await @@ -62,9 +62,9 @@ async fn fresh(services: &Services) -> Result<()> { .db .bump_database_version(DATABASE_VERSION)?; - db["global"].insert(b"feat_sha256_media", &[])?; - db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[])?; - db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; + db["global"].insert(b"feat_sha256_media", &[]); + db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[]); + db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[]); // Create the admin room and server user on first run crate::admin::create_admin_room(services).await?; @@ -82,136 +82,109 @@ async fn migrate(services: &Services) -> Result<()> { let db = &services.db; let config = &services.server.config; - if services.globals.db.database_version()? < 1 { - db_lt_1(services).await?; + if services.globals.db.database_version().await < 11 { + return Err!(Database( + "Database schema version {} is no longer supported", + services.globals.db.database_version().await + )); } - if services.globals.db.database_version()? < 2 { - db_lt_2(services).await?; - } - - if services.globals.db.database_version()? < 3 { - db_lt_3(services).await?; - } - - if services.globals.db.database_version()? < 4 { - db_lt_4(services).await?; - } - - if services.globals.db.database_version()? < 5 { - db_lt_5(services).await?; - } - - if services.globals.db.database_version()? < 6 { - db_lt_6(services).await?; - } - - if services.globals.db.database_version()? < 7 { - db_lt_7(services).await?; - } - - if services.globals.db.database_version()? < 8 { - db_lt_8(services).await?; - } - - if services.globals.db.database_version()? < 9 { - db_lt_9(services).await?; - } - - if services.globals.db.database_version()? < 10 { - db_lt_10(services).await?; - } - - if services.globals.db.database_version()? < 11 { - db_lt_11(services).await?; - } - - if services.globals.db.database_version()? < 12 { + if services.globals.db.database_version().await < 12 { db_lt_12(services).await?; } // This migration can be reused as-is anytime the server-default rules are // updated. - if services.globals.db.database_version()? < 13 { + if services.globals.db.database_version().await < 13 { db_lt_13(services).await?; } - if db["global"].get(b"feat_sha256_media")?.is_none() { + if db["global"].qry("feat_sha256_media").await.is_not_found() { media::migrations::migrate_sha256_media(services).await?; } else if config.media_startup_check { media::migrations::checkup_sha256_media(services).await?; } if db["global"] - .get(b"fix_bad_double_separator_in_state_cache")? - .is_none() + .qry("fix_bad_double_separator_in_state_cache") + .await + .is_not_found() { fix_bad_double_separator_in_state_cache(services).await?; } if db["global"] - .get(b"retroactively_fix_bad_data_from_roomuserid_joined")? - .is_none() + .qry("retroactively_fix_bad_data_from_roomuserid_joined") + .await + .is_not_found() { retroactively_fix_bad_data_from_roomuserid_joined(services).await?; } - let version_match = services.globals.db.database_version().unwrap() == DATABASE_VERSION - || services.globals.db.database_version().unwrap() == CONDUIT_DATABASE_VERSION; + let version_match = services.globals.db.database_version().await == DATABASE_VERSION + || services.globals.db.database_version().await == CONDUIT_DATABASE_VERSION; assert!( version_match, "Failed asserting local database version {} is equal to known latest conduwuit database version {}", - services.globals.db.database_version().unwrap(), + services.globals.db.database_version().await, DATABASE_VERSION, ); { let patterns = services.globals.forbidden_usernames(); if !patterns.is_empty() { - for user_id in services + services .users - .iter() - .filter_map(Result::ok) - .filter(|user| !services.users.is_deactivated(user).unwrap_or(true)) - .filter(|user| user.server_name() == config.server_name) - { - let matches = patterns.matches(user_id.localpart()); - if matches.matched_any() { - warn!( - "User {} matches the following forbidden username patterns: {}", - user_id.to_string(), - matches - .into_iter() - .map(|x| &patterns.patterns()[x]) - .join(", ") - ); - } - } - } - } - - { - let patterns = services.globals.forbidden_alias_names(); - if !patterns.is_empty() { - for address in services.rooms.metadata.iter_ids() { - let room_id = address?; - let room_aliases = services.rooms.alias.local_aliases_for_room(&room_id); - for room_alias_result in room_aliases { - let room_alias = room_alias_result?; - let matches = patterns.matches(room_alias.alias()); + .stream() + .filter(|user_id| services.users.is_active_local(user_id)) + .ready_for_each(|user_id| { + let matches = patterns.matches(user_id.localpart()); if matches.matched_any() { warn!( - "Room with alias {} ({}) matches the following forbidden room name patterns: {}", - room_alias, - &room_id, + "User {} matches the following forbidden username patterns: {}", + user_id.to_string(), matches .into_iter() .map(|x| &patterns.patterns()[x]) .join(", ") ); } - } + }) + .await; + } + } + + { + let patterns = services.globals.forbidden_alias_names(); + if !patterns.is_empty() { + for room_id in services + .rooms + .metadata + .iter_ids() + .map(ToOwned::to_owned) + .collect::>() + .await + { + services + .rooms + .alias + .local_aliases_for_room(&room_id) + .ready_for_each(|room_alias| { + let matches = patterns.matches(room_alias.alias()); + if matches.matched_any() { + warn!( + "Room with alias {} ({}) matches the following forbidden room name patterns: {}", + room_alias, + &room_id, + matches + .into_iter() + .map(|x| &patterns.patterns()[x]) + .join(", ") + ); + } + }) + .await; } } } @@ -224,424 +197,17 @@ async fn migrate(services: &Services) -> Result<()> { Ok(()) } -async fn db_lt_1(services: &Services) -> Result<()> { - let db = &services.db; - - let roomserverids = &db["roomserverids"]; - let serverroomids = &db["serverroomids"]; - for (roomserverid, _) in roomserverids.iter() { - let mut parts = roomserverid.split(|&b| b == 0xFF); - let room_id = parts.next().expect("split always returns one element"); - let Some(servername) = parts.next() else { - error!("Migration: Invalid roomserverid in db."); - continue; - }; - let mut serverroomid = servername.to_vec(); - serverroomid.push(0xFF); - serverroomid.extend_from_slice(room_id); - - serverroomids.insert(&serverroomid, &[])?; - } - - services.globals.db.bump_database_version(1)?; - info!("Migration: 0 -> 1 finished"); - Ok(()) -} - -async fn db_lt_2(services: &Services) -> Result<()> { - let db = &services.db; - - // We accidentally inserted hashed versions of "" into the db instead of just "" - let userid_password = &db["roomserverids"]; - for (userid, password) in userid_password.iter() { - let empty_pass = utils::hash::password("").expect("our own password to be properly hashed"); - let password = std::str::from_utf8(&password).expect("password is valid utf-8"); - let empty_hashed_password = utils::hash::verify_password(password, &empty_pass).is_ok(); - if empty_hashed_password { - userid_password.insert(&userid, b"")?; - } - } - - services.globals.db.bump_database_version(2)?; - info!("Migration: 1 -> 2 finished"); - Ok(()) -} - -async fn db_lt_3(services: &Services) -> Result<()> { - let db = &services.db; - - // Move media to filesystem - let mediaid_file = &db["mediaid_file"]; - for (key, content) in mediaid_file.iter() { - if content.is_empty() { - continue; - } - - #[allow(deprecated)] - let path = services.media.get_media_file(&key); - let mut file = fs::File::create(path)?; - file.write_all(&content)?; - mediaid_file.insert(&key, &[])?; - } - - services.globals.db.bump_database_version(3)?; - info!("Migration: 2 -> 3 finished"); - Ok(()) -} - -async fn db_lt_4(services: &Services) -> Result<()> { - let config = &services.server.config; - - // Add federated users to services as deactivated - for our_user in services.users.iter() { - let our_user = our_user?; - if services.users.is_deactivated(&our_user)? { - continue; - } - for room in services.rooms.state_cache.rooms_joined(&our_user) { - for user in services.rooms.state_cache.room_members(&room?) { - let user = user?; - if user.server_name() != config.server_name { - info!(?user, "Migration: creating user"); - services.users.create(&user, None)?; - } - } - } - } - - services.globals.db.bump_database_version(4)?; - info!("Migration: 3 -> 4 finished"); - Ok(()) -} - -async fn db_lt_5(services: &Services) -> Result<()> { - let db = &services.db; - - // Upgrade user data store - let roomuserdataid_accountdata = &db["roomuserdataid_accountdata"]; - let roomusertype_roomuserdataid = &db["roomusertype_roomuserdataid"]; - for (roomuserdataid, _) in roomuserdataid_accountdata.iter() { - let mut parts = roomuserdataid.split(|&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let user_id = parts.next().unwrap(); - let event_type = roomuserdataid.rsplit(|&b| b == 0xFF).next().unwrap(); - - let mut key = room_id.to_vec(); - key.push(0xFF); - key.extend_from_slice(user_id); - key.push(0xFF); - key.extend_from_slice(event_type); - - roomusertype_roomuserdataid.insert(&key, &roomuserdataid)?; - } - - services.globals.db.bump_database_version(5)?; - info!("Migration: 4 -> 5 finished"); - Ok(()) -} - -async fn db_lt_6(services: &Services) -> Result<()> { - let db = &services.db; - - // Set room member count - let roomid_shortstatehash = &db["roomid_shortstatehash"]; - for (roomid, _) in roomid_shortstatehash.iter() { - let string = utils::string_from_bytes(&roomid).unwrap(); - let room_id = <&RoomId>::try_from(string.as_str()).unwrap(); - services.rooms.state_cache.update_joined_count(room_id)?; - } - - services.globals.db.bump_database_version(6)?; - info!("Migration: 5 -> 6 finished"); - Ok(()) -} - -async fn db_lt_7(services: &Services) -> Result<()> { - let db = &services.db; - - // Upgrade state store - let mut last_roomstates: HashMap = HashMap::new(); - let mut current_sstatehash: Option = None; - let mut current_room = None; - let mut current_state = HashSet::new(); - - let handle_state = |current_sstatehash: u64, - current_room: &RoomId, - current_state: HashSet<_>, - last_roomstates: &mut HashMap<_, _>| { - let last_roomsstatehash = last_roomstates.get(current_room); - - let states_parents = last_roomsstatehash.map_or_else( - || Ok(Vec::new()), - |&last_roomsstatehash| { - services - .rooms - .state_compressor - .load_shortstatehash_info(last_roomsstatehash) - }, - )?; - - let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew = current_state - .difference(&parent_stateinfo.1) - .copied() - .collect::>(); - - let statediffremoved = parent_stateinfo - .1 - .difference(¤t_state) - .copied() - .collect::>(); - - (statediffnew, statediffremoved) - } else { - (current_state, HashSet::new()) - }; - - services.rooms.state_compressor.save_state_from_diff( - current_sstatehash, - Arc::new(statediffnew), - Arc::new(statediffremoved), - 2, // every state change is 2 event changes on average - states_parents, - )?; - - /* - let mut tmp = services.rooms.load_shortstatehash_info(¤t_sstatehash)?; - let state = tmp.pop().unwrap(); - println!( - "{}\t{}{:?}: {:?} + {:?} - {:?}", - current_room, - " ".repeat(tmp.len()), - utils::u64_from_bytes(¤t_sstatehash).unwrap(), - tmp.last().map(|b| utils::u64_from_bytes(&b.0).unwrap()), - state - .2 - .iter() - .map(|b| utils::u64_from_bytes(&b[size_of::()..]).unwrap()) - .collect::>(), - state - .3 - .iter() - .map(|b| utils::u64_from_bytes(&b[size_of::()..]).unwrap()) - .collect::>() - ); - */ - - Ok::<_, Error>(()) - }; - - let stateid_shorteventid = &db["stateid_shorteventid"]; - let shorteventid_eventid = &db["shorteventid_eventid"]; - for (k, seventid) in stateid_shorteventid.iter() { - let sstatehash = utils::u64_from_bytes(&k[0..size_of::()]).expect("number of bytes is correct"); - let sstatekey = k[size_of::()..].to_vec(); - if Some(sstatehash) != current_sstatehash { - if let Some(current_sstatehash) = current_sstatehash { - handle_state( - current_sstatehash, - current_room.as_deref().unwrap(), - current_state, - &mut last_roomstates, - )?; - last_roomstates.insert(current_room.clone().unwrap(), current_sstatehash); - } - current_state = HashSet::new(); - current_sstatehash = Some(sstatehash); - - let event_id = shorteventid_eventid.get(&seventid).unwrap().unwrap(); - let string = utils::string_from_bytes(&event_id).unwrap(); - let event_id = <&EventId>::try_from(string.as_str()).unwrap(); - let pdu = services.rooms.timeline.get_pdu(event_id).unwrap().unwrap(); - - if Some(&pdu.room_id) != current_room.as_ref() { - current_room = Some(pdu.room_id.clone()); - } - } - - let mut val = sstatekey; - val.extend_from_slice(&seventid); - current_state.insert(val.try_into().expect("size is correct")); - } - - if let Some(current_sstatehash) = current_sstatehash { - handle_state( - current_sstatehash, - current_room.as_deref().unwrap(), - current_state, - &mut last_roomstates, - )?; - } - - services.globals.db.bump_database_version(7)?; - info!("Migration: 6 -> 7 finished"); - Ok(()) -} - -async fn db_lt_8(services: &Services) -> Result<()> { - let db = &services.db; - - let roomid_shortstatehash = &db["roomid_shortstatehash"]; - let roomid_shortroomid = &db["roomid_shortroomid"]; - let pduid_pdu = &db["pduid_pdu"]; - let eventid_pduid = &db["eventid_pduid"]; - - // Generate short room ids for all rooms - for (room_id, _) in roomid_shortstatehash.iter() { - let shortroomid = services.globals.next_count()?.to_be_bytes(); - roomid_shortroomid.insert(&room_id, &shortroomid)?; - info!("Migration: 8"); - } - // Update pduids db layout - let batch = pduid_pdu - .iter() - .filter_map(|(key, v)| { - if !key.starts_with(b"!") { - return None; - } - let mut parts = key.splitn(2, |&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let count = parts.next().unwrap(); - - let short_room_id = roomid_shortroomid - .get(room_id) - .unwrap() - .expect("shortroomid should exist"); - - let mut new_key = short_room_id.to_vec(); - new_key.extend_from_slice(count); - - Some(database::OwnedKeyVal(new_key, v)) - }) - .collect::>(); - - pduid_pdu.insert_batch(batch.iter().map(database::KeyVal::from))?; - - let batch2 = eventid_pduid - .iter() - .filter_map(|(k, value)| { - if !value.starts_with(b"!") { - return None; - } - let mut parts = value.splitn(2, |&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let count = parts.next().unwrap(); - - let short_room_id = roomid_shortroomid - .get(room_id) - .unwrap() - .expect("shortroomid should exist"); - - let mut new_value = short_room_id.to_vec(); - new_value.extend_from_slice(count); - - Some(database::OwnedKeyVal(k, new_value)) - }) - .collect::>(); - - eventid_pduid.insert_batch(batch2.iter().map(database::KeyVal::from))?; - - services.globals.db.bump_database_version(8)?; - info!("Migration: 7 -> 8 finished"); - Ok(()) -} - -async fn db_lt_9(services: &Services) -> Result<()> { - let db = &services.db; - - let tokenids = &db["tokenids"]; - let roomid_shortroomid = &db["roomid_shortroomid"]; - - // Update tokenids db layout - let mut iter = tokenids - .iter() - .filter_map(|(key, _)| { - if !key.starts_with(b"!") { - return None; - } - let mut parts = key.splitn(4, |&b| b == 0xFF); - let room_id = parts.next().unwrap(); - let word = parts.next().unwrap(); - let _pdu_id_room = parts.next().unwrap(); - let pdu_id_count = parts.next().unwrap(); - - let short_room_id = roomid_shortroomid - .get(room_id) - .unwrap() - .expect("shortroomid should exist"); - let mut new_key = short_room_id.to_vec(); - new_key.extend_from_slice(word); - new_key.push(0xFF); - new_key.extend_from_slice(pdu_id_count); - Some(database::OwnedKeyVal(new_key, Vec::::new())) - }) - .peekable(); - - while iter.peek().is_some() { - let batch = iter.by_ref().take(1000).collect::>(); - tokenids.insert_batch(batch.iter().map(database::KeyVal::from))?; - debug!("Inserted smaller batch"); - } - - info!("Deleting starts"); - - let batch2: Vec<_> = tokenids - .iter() - .filter_map(|(key, _)| { - if key.starts_with(b"!") { - Some(key) - } else { - None - } - }) - .collect(); - - for key in batch2 { - tokenids.remove(&key)?; - } - - services.globals.db.bump_database_version(9)?; - info!("Migration: 8 -> 9 finished"); - Ok(()) -} - -async fn db_lt_10(services: &Services) -> Result<()> { - let db = &services.db; - - let statekey_shortstatekey = &db["statekey_shortstatekey"]; - let shortstatekey_statekey = &db["shortstatekey_statekey"]; - - // Add other direction for shortstatekeys - for (statekey, shortstatekey) in statekey_shortstatekey.iter() { - shortstatekey_statekey.insert(&shortstatekey, &statekey)?; - } - - // Force E2EE device list updates so we can send them over federation - for user_id in services.users.iter().filter_map(Result::ok) { - services.users.mark_device_key_update(&user_id)?; - } - - services.globals.db.bump_database_version(10)?; - info!("Migration: 9 -> 10 finished"); - Ok(()) -} - -#[allow(unreachable_code)] -async fn db_lt_11(services: &Services) -> Result<()> { - error!("Dropping a column to clear data is not implemented yet."); - //let userdevicesessionid_uiaarequest = &db["userdevicesessionid_uiaarequest"]; - //userdevicesessionid_uiaarequest.clear()?; - - services.globals.db.bump_database_version(11)?; - info!("Migration: 10 -> 11 finished"); - Ok(()) -} - async fn db_lt_12(services: &Services) -> Result<()> { let config = &services.server.config; - for username in services.users.list_local_users()? { - let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { + for username in &services + .users + .list_local_users() + .map(UserId::to_owned) + .collect::>() + .await + { + let user = match UserId::parse_with_server_name(username.as_str(), &config.server_name) { Ok(u) => u, Err(e) => { warn!("Invalid username {username}: {e}"); @@ -652,7 +218,7 @@ async fn db_lt_12(services: &Services) -> Result<()> { let raw_rules_list = services .account_data .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) - .unwrap() + .await .expect("Username is invalid"); let mut account_data = serde_json::from_str::(raw_rules_list.get()).unwrap(); @@ -694,12 +260,15 @@ async fn db_lt_12(services: &Services) -> Result<()> { } } - services.account_data.update( - None, - &user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - )?; + services + .account_data + .update( + None, + &user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(account_data).expect("to json value always works"), + ) + .await?; } services.globals.db.bump_database_version(12)?; @@ -710,8 +279,14 @@ async fn db_lt_12(services: &Services) -> Result<()> { async fn db_lt_13(services: &Services) -> Result<()> { let config = &services.server.config; - for username in services.users.list_local_users()? { - let user = match UserId::parse_with_server_name(username.clone(), &config.server_name) { + for username in &services + .users + .list_local_users() + .map(UserId::to_owned) + .collect::>() + .await + { + let user = match UserId::parse_with_server_name(username.as_str(), &config.server_name) { Ok(u) => u, Err(e) => { warn!("Invalid username {username}: {e}"); @@ -722,7 +297,7 @@ async fn db_lt_13(services: &Services) -> Result<()> { let raw_rules_list = services .account_data .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) - .unwrap() + .await .expect("Username is invalid"); let mut account_data = serde_json::from_str::(raw_rules_list.get()).unwrap(); @@ -733,12 +308,15 @@ async fn db_lt_13(services: &Services) -> Result<()> { .global .update_with_server_default(user_default_rules); - services.account_data.update( - None, - &user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - )?; + services + .account_data + .update( + None, + &user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(account_data).expect("to json value always works"), + ) + .await?; } services.globals.db.bump_database_version(13)?; @@ -754,32 +332,37 @@ async fn fix_bad_double_separator_in_state_cache(services: &Services) -> Result< let _cork = db.cork_and_sync(); let mut iter_count: usize = 0; - for (mut key, value) in roomuserid_joined.iter() { - iter_count = iter_count.saturating_add(1); - debug_info!(%iter_count); - let first_sep_index = key - .iter() - .position(|&i| i == 0xFF) - .expect("found 0xFF delim"); + roomuserid_joined + .raw_stream() + .ignore_err() + .ready_for_each(|(key, value)| { + let mut key = key.to_vec(); + iter_count = iter_count.saturating_add(1); + debug_info!(%iter_count); + let first_sep_index = key + .iter() + .position(|&i| i == 0xFF) + .expect("found 0xFF delim"); - if key - .iter() - .get(first_sep_index..=first_sep_index.saturating_add(1)) - .copied() - .collect_vec() - == vec![0xFF, 0xFF] - { - debug_warn!("Found bad key: {key:?}"); - roomuserid_joined.remove(&key)?; + if key + .iter() + .get(first_sep_index..=first_sep_index.saturating_add(1)) + .copied() + .collect_vec() + == vec![0xFF, 0xFF] + { + debug_warn!("Found bad key: {key:?}"); + roomuserid_joined.remove(&key); - key.remove(first_sep_index); - debug_warn!("Fixed key: {key:?}"); - roomuserid_joined.insert(&key, &value)?; - } - } + key.remove(first_sep_index); + debug_warn!("Fixed key: {key:?}"); + roomuserid_joined.insert(&key, value); + } + }) + .await; db.db.cleanup()?; - db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[])?; + db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[]); info!("Finished fixing"); Ok(()) @@ -795,69 +378,71 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) .rooms .metadata .iter_ids() - .filter_map(Result::ok) - .collect_vec(); + .map(ToOwned::to_owned) + .collect::>() + .await; - for room_id in room_ids.clone() { + for room_id in &room_ids { debug_info!("Fixing room {room_id}"); let users_in_room = services .rooms .state_cache - .room_members(&room_id) - .filter_map(Result::ok) - .collect_vec(); + .room_members(room_id) + .collect::>() + .await; let joined_members = users_in_room .iter() + .stream() .filter(|user_id| { services .rooms .state_accessor - .get_member(&room_id, user_id) - .unwrap_or(None) - .map_or(false, |membership| membership.membership == MembershipState::Join) + .get_member(room_id, user_id) + .map(|member| member.map_or(false, |member| member.membership == MembershipState::Join)) }) - .collect_vec(); + .collect::>() + .await; let non_joined_members = users_in_room .iter() + .stream() .filter(|user_id| { services .rooms .state_accessor - .get_member(&room_id, user_id) - .unwrap_or(None) - .map_or(false, |membership| { - membership.membership == MembershipState::Leave || membership.membership == MembershipState::Ban - }) + .get_member(room_id, user_id) + .map(|member| member.map_or(false, |member| member.membership == MembershipState::Join)) }) - .collect_vec(); + .collect::>() + .await; for user_id in joined_members { debug_info!("User is joined, marking as joined"); - services - .rooms - .state_cache - .mark_as_joined(user_id, &room_id)?; + services.rooms.state_cache.mark_as_joined(user_id, room_id); } for user_id in non_joined_members { debug_info!("User is left or banned, marking as left"); - services.rooms.state_cache.mark_as_left(user_id, &room_id)?; + services.rooms.state_cache.mark_as_left(user_id, room_id); } } - for room_id in room_ids { + for room_id in &room_ids { debug_info!( "Updating joined count for room {room_id} to fix servers in room after correcting membership states" ); - services.rooms.state_cache.update_joined_count(&room_id)?; + services + .rooms + .state_cache + .update_joined_count(room_id) + .await; } db.db.cleanup()?; - db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[])?; + db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[]); info!("Finished fixing"); Ok(()) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 87f8f492..f777901f 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -288,8 +288,8 @@ impl Service { /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. - pub fn verify_keys_for(&self, origin: &ServerName) -> Result> { - let mut keys = self.db.verify_keys_for(origin)?; + pub async fn verify_keys_for(&self, origin: &ServerName) -> Result> { + let mut keys = self.db.verify_keys_for(origin).await?; if origin == self.server_name() { keys.insert( format!("ed25519:{}", self.keypair().version()) @@ -304,8 +304,8 @@ impl Service { Ok(keys) } - pub fn signing_keys_for(&self, origin: &ServerName) -> Result> { - self.db.signing_keys_for(origin) + pub async fn signing_keys_for(&self, origin: &ServerName) -> Result { + self.db.signing_keys_for(origin).await } pub fn well_known_client(&self) -> &Option { &self.config.well_known.client } diff --git a/src/service/key_backups/data.rs b/src/service/key_backups/data.rs deleted file mode 100644 index 30ac593b..00000000 --- a/src/service/key_backups/data.rs +++ /dev/null @@ -1,346 +0,0 @@ -use std::{collections::BTreeMap, sync::Arc}; - -use conduit::{utils, Error, Result}; -use database::Map; -use ruma::{ - api::client::{ - backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, - error::ErrorKind, - }, - serde::Raw, - OwnedRoomId, RoomId, UserId, -}; - -use crate::{globals, Dep}; - -pub(super) struct Data { - backupid_algorithm: Arc, - backupid_etag: Arc, - backupkeyid_backup: Arc, - services: Services, -} - -struct Services { - globals: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - backupid_algorithm: db["backupid_algorithm"].clone(), - backupid_etag: db["backupid_etag"].clone(), - backupkeyid_backup: db["backupkeyid_backup"].clone(), - services: Services { - globals: args.depend::("globals"), - }, - } - } - - pub(super) fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { - let version = self.services.globals.next_count()?.to_string(); - - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - - self.backupid_algorithm.insert( - &key, - &serde_json::to_vec(backup_metadata).expect("BackupAlgorithm::to_vec always works"), - )?; - self.backupid_etag - .insert(&key, &self.services.globals.next_count()?.to_be_bytes())?; - Ok(version) - } - - pub(super) fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - - self.backupid_algorithm.remove(&key)?; - self.backupid_etag.remove(&key)?; - - key.push(0xFF); - - for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) { - self.backupkeyid_backup.remove(&outdated_key)?; - } - - Ok(()) - } - - pub(super) fn update_backup( - &self, user_id: &UserId, version: &str, backup_metadata: &Raw, - ) -> Result { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - - if self.backupid_algorithm.get(&key)?.is_none() { - return Err(Error::BadRequest(ErrorKind::NotFound, "Tried to update nonexistent backup.")); - } - - self.backupid_algorithm - .insert(&key, backup_metadata.json().get().as_bytes())?; - self.backupid_etag - .insert(&key, &self.services.globals.next_count()?.to_be_bytes())?; - Ok(version.to_owned()) - } - - pub(super) fn get_latest_backup_version(&self, user_id: &UserId) -> Result> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - let mut last_possible_key = prefix.clone(); - last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes()); - - self.backupid_algorithm - .iter_from(&last_possible_key, true) - .take_while(move |(k, _)| k.starts_with(&prefix)) - .next() - .map(|(key, _)| { - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("backupid_algorithm key is invalid.")) - }) - .transpose() - } - - pub(super) fn get_latest_backup(&self, user_id: &UserId) -> Result)>> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - let mut last_possible_key = prefix.clone(); - last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes()); - - self.backupid_algorithm - .iter_from(&last_possible_key, true) - .take_while(move |(k, _)| k.starts_with(&prefix)) - .next() - .map(|(key, value)| { - let version = utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))?; - - Ok(( - version, - serde_json::from_slice(&value) - .map_err(|_| Error::bad_database("Algorithm in backupid_algorithm is invalid."))?, - )) - }) - .transpose() - } - - pub(super) fn get_backup(&self, user_id: &UserId, version: &str) -> Result>> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - - self.backupid_algorithm - .get(&key)? - .map_or(Ok(None), |bytes| { - serde_json::from_slice(&bytes) - .map_err(|_| Error::bad_database("Algorithm in backupid_algorithm is invalid.")) - }) - } - - pub(super) fn add_key( - &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, key_data: &Raw, - ) -> Result<()> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - - if self.backupid_algorithm.get(&key)?.is_none() { - return Err(Error::BadRequest(ErrorKind::NotFound, "Tried to update nonexistent backup.")); - } - - self.backupid_etag - .insert(&key, &self.services.globals.next_count()?.to_be_bytes())?; - - key.push(0xFF); - key.extend_from_slice(room_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(session_id.as_bytes()); - - self.backupkeyid_backup - .insert(&key, key_data.json().get().as_bytes())?; - - Ok(()) - } - - pub(super) fn count_keys(&self, user_id: &UserId, version: &str) -> Result { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(version.as_bytes()); - - Ok(self.backupkeyid_backup.scan_prefix(prefix).count()) - } - - pub(super) fn get_etag(&self, user_id: &UserId, version: &str) -> Result { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - - Ok(utils::u64_from_bytes( - &self - .backupid_etag - .get(&key)? - .ok_or_else(|| Error::bad_database("Backup has no etag."))?, - ) - .map_err(|_| Error::bad_database("etag in backupid_etag invalid."))? - .to_string()) - } - - pub(super) fn get_all(&self, user_id: &UserId, version: &str) -> Result> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(version.as_bytes()); - prefix.push(0xFF); - - let mut rooms = BTreeMap::::new(); - - for result in self - .backupkeyid_backup - .scan_prefix(prefix) - .map(|(key, value)| { - let mut parts = key.rsplit(|&b| b == 0xFF); - - let session_id = utils::string_from_bytes( - parts - .next() - .ok_or_else(|| Error::bad_database("backupkeyid_backup key is invalid."))?, - ) - .map_err(|_| Error::bad_database("backupkeyid_backup session_id is invalid."))?; - - let room_id = RoomId::parse( - utils::string_from_bytes( - parts - .next() - .ok_or_else(|| Error::bad_database("backupkeyid_backup key is invalid."))?, - ) - .map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid."))?, - ) - .map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid room id."))?; - - let key_data = serde_json::from_slice(&value) - .map_err(|_| Error::bad_database("KeyBackupData in backupkeyid_backup is invalid."))?; - - Ok::<_, Error>((room_id, session_id, key_data)) - }) { - let (room_id, session_id, key_data) = result?; - rooms - .entry(room_id) - .or_insert_with(|| RoomKeyBackup { - sessions: BTreeMap::new(), - }) - .sessions - .insert(session_id, key_data); - } - - Ok(rooms) - } - - pub(super) fn get_room( - &self, user_id: &UserId, version: &str, room_id: &RoomId, - ) -> Result>> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(version.as_bytes()); - prefix.push(0xFF); - prefix.extend_from_slice(room_id.as_bytes()); - prefix.push(0xFF); - - Ok(self - .backupkeyid_backup - .scan_prefix(prefix) - .map(|(key, value)| { - let mut parts = key.rsplit(|&b| b == 0xFF); - - let session_id = utils::string_from_bytes( - parts - .next() - .ok_or_else(|| Error::bad_database("backupkeyid_backup key is invalid."))?, - ) - .map_err(|_| Error::bad_database("backupkeyid_backup session_id is invalid."))?; - - let key_data = serde_json::from_slice(&value) - .map_err(|_| Error::bad_database("KeyBackupData in backupkeyid_backup is invalid."))?; - - Ok::<_, Error>((session_id, key_data)) - }) - .filter_map(Result::ok) - .collect()) - } - - pub(super) fn get_session( - &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, - ) -> Result>> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - key.push(0xFF); - key.extend_from_slice(room_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(session_id.as_bytes()); - - self.backupkeyid_backup - .get(&key)? - .map(|value| { - serde_json::from_slice(&value) - .map_err(|_| Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")) - }) - .transpose() - } - - pub(super) fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - key.push(0xFF); - - for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) { - self.backupkeyid_backup.remove(&outdated_key)?; - } - - Ok(()) - } - - pub(super) fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) -> Result<()> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - key.push(0xFF); - key.extend_from_slice(room_id.as_bytes()); - key.push(0xFF); - - for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) { - self.backupkeyid_backup.remove(&outdated_key)?; - } - - Ok(()) - } - - pub(super) fn delete_room_key( - &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, - ) -> Result<()> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - key.push(0xFF); - key.extend_from_slice(room_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(session_id.as_bytes()); - - for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) { - self.backupkeyid_backup.remove(&outdated_key)?; - } - - Ok(()) - } -} diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 65d3c065..12712e79 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -1,93 +1,319 @@ -mod data; - use std::{collections::BTreeMap, sync::Arc}; -use conduit::Result; -use data::Data; +use conduit::{ + err, implement, utils, + utils::stream::{ReadyExt, TryIgnore}, + Err, Error, Result, +}; +use database::{Deserialized, Ignore, Interfix, Map}; +use futures::StreamExt; use ruma::{ api::client::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, serde::Raw, OwnedRoomId, RoomId, UserId, }; +use crate::{globals, Dep}; + pub struct Service { db: Data, + services: Services, +} + +struct Data { + backupid_algorithm: Arc, + backupid_etag: Arc, + backupkeyid_backup: Arc, +} + +struct Services { + globals: Dep, } impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(&args), + db: Data { + backupid_algorithm: args.db["backupid_algorithm"].clone(), + backupid_etag: args.db["backupid_etag"].clone(), + backupkeyid_backup: args.db["backupkeyid_backup"].clone(), + }, + services: Services { + globals: args.depend::("globals"), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - pub fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { - self.db.create_backup(user_id, backup_metadata) - } +#[implement(Service)] +pub fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { + let version = self.services.globals.next_count()?.to_string(); - pub fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> { - self.db.delete_backup(user_id, version) - } + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(version.as_bytes()); - pub fn update_backup( - &self, user_id: &UserId, version: &str, backup_metadata: &Raw, - ) -> Result { - self.db.update_backup(user_id, version, backup_metadata) - } + self.db.backupid_algorithm.insert( + &key, + &serde_json::to_vec(backup_metadata).expect("BackupAlgorithm::to_vec always works"), + ); - pub fn get_latest_backup_version(&self, user_id: &UserId) -> Result> { - self.db.get_latest_backup_version(user_id) - } + self.db + .backupid_etag + .insert(&key, &self.services.globals.next_count()?.to_be_bytes()); - pub fn get_latest_backup(&self, user_id: &UserId) -> Result)>> { - self.db.get_latest_backup(user_id) - } - - pub fn get_backup(&self, user_id: &UserId, version: &str) -> Result>> { - self.db.get_backup(user_id, version) - } - - pub fn add_key( - &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, key_data: &Raw, - ) -> Result<()> { - self.db - .add_key(user_id, version, room_id, session_id, key_data) - } - - pub fn count_keys(&self, user_id: &UserId, version: &str) -> Result { self.db.count_keys(user_id, version) } - - pub fn get_etag(&self, user_id: &UserId, version: &str) -> Result { self.db.get_etag(user_id, version) } - - pub fn get_all(&self, user_id: &UserId, version: &str) -> Result> { - self.db.get_all(user_id, version) - } - - pub fn get_room( - &self, user_id: &UserId, version: &str, room_id: &RoomId, - ) -> Result>> { - self.db.get_room(user_id, version, room_id) - } - - pub fn get_session( - &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, - ) -> Result>> { - self.db.get_session(user_id, version, room_id, session_id) - } - - pub fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> { - self.db.delete_all_keys(user_id, version) - } - - pub fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) -> Result<()> { - self.db.delete_room_keys(user_id, version, room_id) - } - - pub fn delete_room_key(&self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str) -> Result<()> { - self.db - .delete_room_key(user_id, version, room_id, session_id) - } + Ok(version) +} + +#[implement(Service)] +pub async fn delete_backup(&self, user_id: &UserId, version: &str) { + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(version.as_bytes()); + + self.db.backupid_algorithm.remove(&key); + self.db.backupid_etag.remove(&key); + + let key = (user_id, version, Interfix); + self.db + .backupkeyid_backup + .keys_raw_prefix(&key) + .ignore_err() + .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) + .await; +} + +#[implement(Service)] +pub async fn update_backup( + &self, user_id: &UserId, version: &str, backup_metadata: &Raw, +) -> Result { + let key = (user_id, version); + if self.db.backupid_algorithm.qry(&key).await.is_err() { + return Err!(Request(NotFound("Tried to update nonexistent backup."))); + } + + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(version.as_bytes()); + + self.db + .backupid_algorithm + .insert(&key, backup_metadata.json().get().as_bytes()); + self.db + .backupid_etag + .insert(&key, &self.services.globals.next_count()?.to_be_bytes()); + + Ok(version.to_owned()) +} + +#[implement(Service)] +pub async fn get_latest_backup_version(&self, user_id: &UserId) -> Result { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xFF); + let mut last_possible_key = prefix.clone(); + last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes()); + + self.db + .backupid_algorithm + .rev_raw_keys_from(&last_possible_key) + .ignore_err() + .ready_take_while(move |key| key.starts_with(&prefix)) + .next() + .await + .ok_or_else(|| err!(Request(NotFound("No backup versions found")))) + .and_then(|key| { + utils::string_from_bytes( + key.rsplit(|&b| b == 0xFF) + .next() + .expect("rsplit always returns an element"), + ) + .map_err(|_| Error::bad_database("backupid_algorithm key is invalid.")) + }) +} + +#[implement(Service)] +pub async fn get_latest_backup(&self, user_id: &UserId) -> Result<(String, Raw)> { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xFF); + let mut last_possible_key = prefix.clone(); + last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes()); + + self.db + .backupid_algorithm + .rev_raw_stream_from(&last_possible_key) + .ignore_err() + .ready_take_while(move |(key, _)| key.starts_with(&prefix)) + .next() + .await + .ok_or_else(|| err!(Request(NotFound("No backup found")))) + .and_then(|(key, val)| { + let version = utils::string_from_bytes( + key.rsplit(|&b| b == 0xFF) + .next() + .expect("rsplit always returns an element"), + ) + .map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))?; + + let algorithm = serde_json::from_slice(val) + .map_err(|_| Error::bad_database("Algorithm in backupid_algorithm is invalid."))?; + + Ok((version, algorithm)) + }) +} + +#[implement(Service)] +pub async fn get_backup(&self, user_id: &UserId, version: &str) -> Result> { + let key = (user_id, version); + self.db + .backupid_algorithm + .qry(&key) + .await + .deserialized_json() +} + +#[implement(Service)] +pub async fn add_key( + &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, key_data: &Raw, +) -> Result<()> { + let key = (user_id, version); + if self.db.backupid_algorithm.qry(&key).await.is_err() { + return Err!(Request(NotFound("Tried to update nonexistent backup."))); + } + + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(version.as_bytes()); + + self.db + .backupid_etag + .insert(&key, &self.services.globals.next_count()?.to_be_bytes()); + + key.push(0xFF); + key.extend_from_slice(room_id.as_bytes()); + key.push(0xFF); + key.extend_from_slice(session_id.as_bytes()); + + self.db + .backupkeyid_backup + .insert(&key, key_data.json().get().as_bytes()); + + Ok(()) +} + +#[implement(Service)] +pub async fn count_keys(&self, user_id: &UserId, version: &str) -> usize { + let prefix = (user_id, version); + self.db + .backupkeyid_backup + .keys_raw_prefix(&prefix) + .count() + .await +} + +#[implement(Service)] +pub async fn get_etag(&self, user_id: &UserId, version: &str) -> String { + let key = (user_id, version); + self.db + .backupid_etag + .qry(&key) + .await + .deserialized::() + .as_ref() + .map(ToString::to_string) + .expect("Backup has no etag.") +} + +#[implement(Service)] +pub async fn get_all(&self, user_id: &UserId, version: &str) -> BTreeMap { + type KeyVal<'a> = ((Ignore, Ignore, &'a RoomId, &'a str), &'a [u8]); + + let mut rooms = BTreeMap::::new(); + let default = || RoomKeyBackup { + sessions: BTreeMap::new(), + }; + + let prefix = (user_id, version, Interfix); + self.db + .backupkeyid_backup + .stream_prefix(&prefix) + .ignore_err() + .ready_for_each(|((_, _, room_id, session_id), value): KeyVal<'_>| { + let key_data = serde_json::from_slice(value).expect("Invalid KeyBackupData JSON"); + rooms + .entry(room_id.into()) + .or_insert_with(default) + .sessions + .insert(session_id.into(), key_data); + }) + .await; + + rooms +} + +#[implement(Service)] +pub async fn get_room( + &self, user_id: &UserId, version: &str, room_id: &RoomId, +) -> BTreeMap> { + type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), &'a [u8]); + + let prefix = (user_id, version, room_id, Interfix); + self.db + .backupkeyid_backup + .stream_prefix(&prefix) + .ignore_err() + .map(|((.., session_id), value): KeyVal<'_>| { + let session_id = session_id.to_owned(); + let key_backup_data = serde_json::from_slice(value).expect("Invalid KeyBackupData JSON"); + (session_id, key_backup_data) + }) + .collect() + .await +} + +#[implement(Service)] +pub async fn get_session( + &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, +) -> Result> { + let key = (user_id, version, room_id, session_id); + + self.db + .backupkeyid_backup + .qry(&key) + .await + .deserialized_json() +} + +#[implement(Service)] +pub async fn delete_all_keys(&self, user_id: &UserId, version: &str) { + let key = (user_id, version, Interfix); + self.db + .backupkeyid_backup + .keys_raw_prefix(&key) + .ignore_err() + .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) + .await; +} + +#[implement(Service)] +pub async fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: &RoomId) { + let key = (user_id, version, room_id, Interfix); + self.db + .backupkeyid_backup + .keys_raw_prefix(&key) + .ignore_err() + .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) + .await; +} + +#[implement(Service)] +pub async fn delete_room_key(&self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str) { + let key = (user_id, version, room_id, session_id); + self.db + .backupkeyid_backup + .keys_raw_prefix(&key) + .ignore_err() + .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) + .await; } diff --git a/src/service/manager.rs b/src/service/manager.rs index 42260bb3..21e0ed7c 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -1,7 +1,7 @@ use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; use conduit::{debug, debug_warn, error, trace, utils::time, warn, Err, Error, Result, Server}; -use futures_util::FutureExt; +use futures::FutureExt; use tokio::{ sync::{Mutex, MutexGuard}, task::{JoinHandle, JoinSet}, diff --git a/src/service/media/data.rs b/src/service/media/data.rs index e5d6d20b..29d562cc 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -2,10 +2,11 @@ use std::sync::Arc; use conduit::{ debug, debug_info, trace, - utils::{str_from_bytes, string_from_bytes}, + utils::{str_from_bytes, stream::TryIgnore, string_from_bytes, ReadyExt}, Err, Error, Result, }; use database::{Database, Map}; +use futures::StreamExt; use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; use super::{preview::UrlPreviewData, thumbnail::Dim}; @@ -59,7 +60,7 @@ impl Data { .unwrap_or_default(), ); - self.mediaid_file.insert(&key, &[])?; + self.mediaid_file.insert(&key, &[]); if let Some(user) = user { let mut key: Vec = Vec::new(); @@ -68,13 +69,13 @@ impl Data { key.extend_from_slice(b"/"); key.extend_from_slice(mxc.media_id.as_bytes()); let user = user.as_bytes().to_vec(); - self.mediaid_user.insert(&key, &user)?; + self.mediaid_user.insert(&key, &user); } Ok(key) } - pub(super) fn delete_file_mxc(&self, mxc: &Mxc<'_>) -> Result<()> { + pub(super) async fn delete_file_mxc(&self, mxc: &Mxc<'_>) { debug!("MXC URI: {mxc}"); let mut prefix: Vec = Vec::new(); @@ -85,25 +86,31 @@ impl Data { prefix.push(0xFF); trace!("MXC db prefix: {prefix:?}"); - for (key, _) in self.mediaid_file.scan_prefix(prefix.clone()) { - debug!("Deleting key: {:?}", key); - self.mediaid_file.remove(&key)?; - } + self.mediaid_file + .raw_keys_prefix(&prefix) + .ignore_err() + .ready_for_each(|key| { + debug!("Deleting key: {:?}", key); + self.mediaid_file.remove(key); + }) + .await; - for (key, value) in self.mediaid_user.scan_prefix(prefix.clone()) { - if key.starts_with(&prefix) { - let user = str_from_bytes(&value).unwrap_or_default(); + self.mediaid_user + .raw_stream_prefix(&prefix) + .ignore_err() + .ready_for_each(|(key, val)| { + if key.starts_with(&prefix) { + let user = str_from_bytes(val).unwrap_or_default(); + debug_info!("Deleting key {key:?} which was uploaded by user {user}"); - debug_info!("Deleting key \"{key:?}\" which was uploaded by user {user}"); - self.mediaid_user.remove(&key)?; - } - } - - Ok(()) + self.mediaid_user.remove(key); + } + }) + .await; } /// Searches for all files with the given MXC - pub(super) fn search_mxc_metadata_prefix(&self, mxc: &Mxc<'_>) -> Result>> { + pub(super) async fn search_mxc_metadata_prefix(&self, mxc: &Mxc<'_>) -> Result>> { debug!("MXC URI: {mxc}"); let mut prefix: Vec = Vec::new(); @@ -115,9 +122,10 @@ impl Data { let keys: Vec> = self .mediaid_file - .scan_prefix(prefix) - .map(|(key, _)| key) - .collect(); + .keys_prefix_raw(&prefix) + .ignore_err() + .collect() + .await; if keys.is_empty() { return Err!(Database("Failed to find any keys in database for `{mxc}`",)); @@ -128,7 +136,7 @@ impl Data { Ok(keys) } - pub(super) fn search_file_metadata(&self, mxc: &Mxc<'_>, dim: &Dim) -> Result { + pub(super) async fn search_file_metadata(&self, mxc: &Mxc<'_>, dim: &Dim) -> Result { let mut prefix: Vec = Vec::new(); prefix.extend_from_slice(b"mxc://"); prefix.extend_from_slice(mxc.server_name.as_bytes()); @@ -139,10 +147,13 @@ impl Data { prefix.extend_from_slice(&dim.height.to_be_bytes()); prefix.push(0xFF); - let (key, _) = self + let key = self .mediaid_file - .scan_prefix(prefix) + .raw_keys_prefix(&prefix) + .ignore_err() + .map(ToOwned::to_owned) .next() + .await .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found"))?; let mut parts = key.rsplit(|&b| b == 0xFF); @@ -177,28 +188,31 @@ impl Data { } /// Gets all the MXCs associated with a user - pub(super) fn get_all_user_mxcs(&self, user_id: &UserId) -> Vec { - let user_id = user_id.as_bytes().to_vec(); - + pub(super) async fn get_all_user_mxcs(&self, user_id: &UserId) -> Vec { self.mediaid_user - .iter() - .filter_map(|(key, user)| { - if *user == user_id { - let mxc_s = string_from_bytes(&key).ok()?; - Some(OwnedMxcUri::from(mxc_s)) - } else { - None - } - }) + .stream() + .ignore_err() + .ready_filter_map(|(key, user): (&str, &UserId)| (user == user_id).then(|| key.into())) .collect() + .await } /// Gets all the media keys in our database (this includes all the metadata /// associated with it such as width, height, content-type, etc) - pub(crate) fn get_all_media_keys(&self) -> Vec> { self.mediaid_file.iter().map(|(key, _)| key).collect() } + pub(crate) async fn get_all_media_keys(&self) -> Vec> { + self.mediaid_file + .raw_keys() + .ignore_err() + .map(<[u8]>::to_vec) + .collect() + .await + } #[inline] - pub(super) fn remove_url_preview(&self, url: &str) -> Result<()> { self.url_previews.remove(url.as_bytes()) } + pub(super) fn remove_url_preview(&self, url: &str) -> Result<()> { + self.url_previews.remove(url.as_bytes()); + Ok(()) + } pub(super) fn set_url_preview( &self, url: &str, data: &UrlPreviewData, timestamp: std::time::Duration, @@ -233,11 +247,13 @@ impl Data { value.push(0xFF); value.extend_from_slice(&data.image_height.unwrap_or(0).to_be_bytes()); - self.url_previews.insert(url.as_bytes(), &value) + self.url_previews.insert(url.as_bytes(), &value); + + Ok(()) } - pub(super) fn get_url_preview(&self, url: &str) -> Option { - let values = self.url_previews.get(url.as_bytes()).ok()??; + pub(super) async fn get_url_preview(&self, url: &str) -> Result { + let values = self.url_previews.qry(url).await?; let mut values = values.split(|&b| b == 0xFF); @@ -291,7 +307,7 @@ impl Data { x => x, }; - Some(UrlPreviewData { + Ok(UrlPreviewData { title, description, image, diff --git a/src/service/media/migrations.rs b/src/service/media/migrations.rs index 9968d25b..2d1b39f9 100644 --- a/src/service/media/migrations.rs +++ b/src/service/media/migrations.rs @@ -7,7 +7,11 @@ use std::{ time::Instant, }; -use conduit::{debug, debug_info, debug_warn, error, info, warn, Config, Result}; +use conduit::{ + debug, debug_info, debug_warn, error, info, + utils::{stream::TryIgnore, ReadyExt}, + warn, Config, Result, +}; use crate::{globals, Services}; @@ -23,12 +27,17 @@ pub(crate) async fn migrate_sha256_media(services: &Services) -> Result<()> { // Move old media files to new names let mut changes = Vec::<(PathBuf, PathBuf)>::new(); - for (key, _) in mediaid_file.iter() { - let old = services.media.get_media_file_b64(&key); - let new = services.media.get_media_file_sha256(&key); - debug!(?key, ?old, ?new, num = changes.len(), "change"); - changes.push((old, new)); - } + mediaid_file + .raw_keys() + .ignore_err() + .ready_for_each(|key| { + let old = services.media.get_media_file_b64(key); + let new = services.media.get_media_file_sha256(key); + debug!(?key, ?old, ?new, num = changes.len(), "change"); + changes.push((old, new)); + }) + .await; + // move the file to the new location for (old_path, path) in changes { if old_path.exists() { @@ -41,11 +50,11 @@ pub(crate) async fn migrate_sha256_media(services: &Services) -> Result<()> { // Apply fix from when sha256_media was backward-incompat and bumped the schema // version from 13 to 14. For users satisfying these conditions we can go back. - if services.globals.db.database_version()? == 14 && globals::migrations::DATABASE_VERSION == 13 { + if services.globals.db.database_version().await == 14 && globals::migrations::DATABASE_VERSION == 13 { services.globals.db.bump_database_version(13)?; } - db["global"].insert(b"feat_sha256_media", &[])?; + db["global"].insert(b"feat_sha256_media", &[]); info!("Finished applying sha256_media"); Ok(()) } @@ -71,7 +80,7 @@ pub(crate) async fn checkup_sha256_media(services: &Services) -> Result<()> { .filter_map(|ent| ent.map_or(None, |ent| Some(ent.path().into_os_string()))) .collect(); - for key in media.db.get_all_media_keys() { + for key in media.db.get_all_media_keys().await { let new_path = media.get_media_file_sha256(&key).into_os_string(); let old_path = media.get_media_file_b64(&key).into_os_string(); if let Err(e) = handle_media_check(&dbs, config, &files, &key, &new_path, &old_path).await { @@ -112,8 +121,8 @@ async fn handle_media_check( "Media is missing at all paths. Removing from database..." ); - mediaid_file.remove(key)?; - mediaid_user.remove(key)?; + mediaid_file.remove(key); + mediaid_user.remove(key); } if config.media_compat_file_link && !old_exists && new_exists { diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index d3765a17..c0b15726 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -97,7 +97,7 @@ impl Service { /// Deletes a file in the database and from the media directory via an MXC pub async fn delete(&self, mxc: &Mxc<'_>) -> Result<()> { - if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc) { + if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc).await { for key in keys { trace!(?mxc, "MXC Key: {key:?}"); debug_info!(?mxc, "Deleting from filesystem"); @@ -107,7 +107,7 @@ impl Service { } debug_info!(?mxc, "Deleting from database"); - _ = self.db.delete_file_mxc(mxc); + self.db.delete_file_mxc(mxc).await; } Ok(()) @@ -120,7 +120,7 @@ impl Service { /// /// currently, this is only practical for local users pub async fn delete_from_user(&self, user: &UserId) -> Result { - let mxcs = self.db.get_all_user_mxcs(user); + let mxcs = self.db.get_all_user_mxcs(user).await; let mut deletion_count: usize = 0; for mxc in mxcs { @@ -150,7 +150,7 @@ impl Service { content_disposition, content_type, key, - }) = self.db.search_file_metadata(mxc, &Dim::default()) + }) = self.db.search_file_metadata(mxc, &Dim::default()).await { let mut content = Vec::new(); let path = self.get_media_file(&key); @@ -170,7 +170,7 @@ impl Service { /// Gets all the MXC URIs in our media database pub async fn get_all_mxcs(&self) -> Result> { - let all_keys = self.db.get_all_media_keys(); + let all_keys = self.db.get_all_media_keys().await; let mut mxcs = Vec::with_capacity(all_keys.len()); @@ -209,7 +209,7 @@ impl Service { pub async fn delete_all_remote_media_at_after_time( &self, time: SystemTime, before: bool, after: bool, yes_i_want_to_delete_local_media: bool, ) -> Result { - let all_keys = self.db.get_all_media_keys(); + let all_keys = self.db.get_all_media_keys().await; let mut remote_mxcs = Vec::with_capacity(all_keys.len()); for key in all_keys { @@ -343,9 +343,10 @@ impl Service { } #[inline] - pub fn get_metadata(&self, mxc: &Mxc<'_>) -> Option { + pub async fn get_metadata(&self, mxc: &Mxc<'_>) -> Option { self.db .search_file_metadata(mxc, &Dim::default()) + .await .map(|metadata| FileMeta { content_disposition: metadata.content_disposition, content_type: metadata.content_type, diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index 5704075e..6b147383 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -71,16 +71,16 @@ pub async fn download_image(&self, url: &str) -> Result { #[implement(Service)] pub async fn get_url_preview(&self, url: &str) -> Result { - if let Some(preview) = self.db.get_url_preview(url) { + if let Ok(preview) = self.db.get_url_preview(url).await { return Ok(preview); } // ensure that only one request is made per URL let _request_lock = self.url_preview_mutex.lock(url).await; - match self.db.get_url_preview(url) { - Some(preview) => Ok(preview), - None => self.request_url_preview(url).await, + match self.db.get_url_preview(url).await { + Ok(preview) => Ok(preview), + Err(_) => self.request_url_preview(url).await, } } diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 630f7b3b..04ec0303 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -54,9 +54,9 @@ impl super::Service { // 0, 0 because that's the original file let dim = dim.normalized(); - if let Ok(metadata) = self.db.search_file_metadata(mxc, &dim) { + if let Ok(metadata) = self.db.search_file_metadata(mxc, &dim).await { self.get_thumbnail_saved(metadata).await - } else if let Ok(metadata) = self.db.search_file_metadata(mxc, &Dim::default()) { + } else if let Ok(metadata) = self.db.search_file_metadata(mxc, &Dim::default()).await { self.get_thumbnail_generate(mxc, &dim, metadata).await } else { Ok(None) diff --git a/src/service/mod.rs b/src/service/mod.rs index f588a542..cb8bfcd9 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -19,6 +19,7 @@ pub mod resolver; pub mod rooms; pub mod sending; pub mod server_keys; +pub mod sync; pub mod transaction_ids; pub mod uiaa; pub mod updates; diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index ec036b3d..0c3f3d31 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -1,7 +1,12 @@ use std::sync::Arc; -use conduit::{debug_warn, utils, Error, Result}; -use database::Map; +use conduit::{ + debug_warn, utils, + utils::{stream::TryIgnore, ReadyExt}, + Result, +}; +use database::{Deserialized, Map}; +use futures::Stream; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; use super::Presence; @@ -31,39 +36,35 @@ impl Data { } } - pub fn get_presence(&self, user_id: &UserId) -> Result> { - if let Some(count_bytes) = self.userid_presenceid.get(user_id.as_bytes())? { - let count = utils::u64_from_bytes(&count_bytes) - .map_err(|_e| Error::bad_database("No 'count' bytes in presence key"))?; + pub async fn get_presence(&self, user_id: &UserId) -> Result<(u64, PresenceEvent)> { + let count = self + .userid_presenceid + .qry(user_id) + .await + .deserialized::()?; - let key = presenceid_key(count, user_id); - self.presenceid_presence - .get(&key)? - .map(|presence_bytes| -> Result<(u64, PresenceEvent)> { - Ok(( - count, - Presence::from_json_bytes(&presence_bytes)?.to_presence_event(user_id, &self.services.users)?, - )) - }) - .transpose() - } else { - Ok(None) - } + let key = presenceid_key(count, user_id); + let bytes = self.presenceid_presence.qry(&key).await?; + let event = Presence::from_json_bytes(&bytes)? + .to_presence_event(user_id, &self.services.users) + .await; + + Ok((count, event)) } - pub(super) fn set_presence( + pub(super) async fn set_presence( &self, user_id: &UserId, presence_state: &PresenceState, currently_active: Option, last_active_ago: Option, status_msg: Option, ) -> Result<()> { - let last_presence = self.get_presence(user_id)?; + let last_presence = self.get_presence(user_id).await; let state_changed = match last_presence { - None => true, - Some(ref presence) => presence.1.content.presence != *presence_state, + Err(_) => true, + Ok(ref presence) => presence.1.content.presence != *presence_state, }; let status_msg_changed = match last_presence { - None => true, - Some(ref last_presence) => { + Err(_) => true, + Ok(ref last_presence) => { let old_msg = last_presence .1 .content @@ -79,8 +80,8 @@ impl Data { let now = utils::millis_since_unix_epoch(); let last_last_active_ts = match last_presence { - None => 0, - Some((_, ref presence)) => now.saturating_sub(presence.content.last_active_ago.unwrap_or_default().into()), + Err(_) => 0, + Ok((_, ref presence)) => now.saturating_sub(presence.content.last_active_ago.unwrap_or_default().into()), }; let last_active_ts = match last_active_ago { @@ -90,12 +91,7 @@ impl Data { // TODO: tighten for state flicker? if !status_msg_changed && !state_changed && last_active_ts < last_last_active_ts { - debug_warn!( - "presence spam {:?} last_active_ts:{:?} < {:?}", - user_id, - last_active_ts, - last_last_active_ts - ); + debug_warn!("presence spam {user_id:?} last_active_ts:{last_active_ts:?} < {last_last_active_ts:?}",); return Ok(()); } @@ -115,41 +111,42 @@ impl Data { let key = presenceid_key(count, user_id); self.presenceid_presence - .insert(&key, &presence.to_json_bytes()?)?; + .insert(&key, &presence.to_json_bytes()?); self.userid_presenceid - .insert(user_id.as_bytes(), &count.to_be_bytes())?; + .insert(user_id.as_bytes(), &count.to_be_bytes()); - if let Some((last_count, _)) = last_presence { + if let Ok((last_count, _)) = last_presence { let key = presenceid_key(last_count, user_id); - self.presenceid_presence.remove(&key)?; + self.presenceid_presence.remove(&key); } Ok(()) } - pub(super) fn remove_presence(&self, user_id: &UserId) -> Result<()> { - if let Some(count_bytes) = self.userid_presenceid.get(user_id.as_bytes())? { - let count = utils::u64_from_bytes(&count_bytes) - .map_err(|_e| Error::bad_database("No 'count' bytes in presence key"))?; - let key = presenceid_key(count, user_id); - self.presenceid_presence.remove(&key)?; - self.userid_presenceid.remove(user_id.as_bytes())?; - } + pub(super) async fn remove_presence(&self, user_id: &UserId) { + let Ok(count) = self + .userid_presenceid + .qry(user_id) + .await + .deserialized::() + else { + return; + }; - Ok(()) + let key = presenceid_key(count, user_id); + self.presenceid_presence.remove(&key); + self.userid_presenceid.remove(user_id.as_bytes()); } - pub fn presence_since<'a>(&'a self, since: u64) -> Box)> + 'a> { - Box::new( - self.presenceid_presence - .iter() - .flat_map(|(key, presence_bytes)| -> Result<(OwnedUserId, u64, Vec)> { - let (count, user_id) = presenceid_parse(&key)?; - Ok((user_id.to_owned(), count, presence_bytes)) - }) - .filter(move |(_, count, _)| *count > since), - ) + pub fn presence_since(&self, since: u64) -> impl Stream)> + Send + '_ { + self.presenceid_presence + .raw_stream() + .ignore_err() + .ready_filter_map(move |(key, presence_bytes)| { + let (count, user_id) = presenceid_parse(key).expect("invalid presenceid_parse"); + (count > since).then(|| (user_id.to_owned(), count, presence_bytes.to_vec())) + }) } } @@ -162,7 +159,7 @@ fn presenceid_key(count: u64, user_id: &UserId) -> Vec { fn presenceid_parse(key: &[u8]) -> Result<(u64, &UserId)> { let (count, user_id) = key.split_at(8); let user_id = user_id_from_bytes(user_id)?; - let count = utils::u64_from_bytes(count).unwrap(); + let count = utils::u64_from_u8(count); Ok((count, user_id)) } diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index a54a6d7c..3b5c4caf 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -4,8 +4,8 @@ mod presence; use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{checked, debug, error, Error, Result, Server}; -use futures_util::{stream::FuturesUnordered, StreamExt}; +use conduit::{checked, debug, error, result::LogErr, Error, Result, Server}; +use futures::{stream::FuturesUnordered, Stream, StreamExt, TryFutureExt}; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; use tokio::{sync::Mutex, time::sleep}; @@ -58,7 +58,9 @@ impl crate::Service for Service { loop { debug_assert!(!receiver.is_closed(), "channel error"); tokio::select! { - Some(user_id) = presence_timers.next() => self.process_presence_timer(&user_id)?, + Some(user_id) = presence_timers.next() => { + self.process_presence_timer(&user_id).await.log_err().ok(); + }, event = receiver.recv_async() => match event { Err(_e) => return Ok(()), Ok((user_id, timeout)) => { @@ -82,28 +84,27 @@ impl crate::Service for Service { impl Service { /// Returns the latest presence event for the given user. #[inline] - pub fn get_presence(&self, user_id: &UserId) -> Result> { - if let Some((_, presence)) = self.db.get_presence(user_id)? { - Ok(Some(presence)) - } else { - Ok(None) - } + pub async fn get_presence(&self, user_id: &UserId) -> Result { + self.db + .get_presence(user_id) + .map_ok(|(_, presence)| presence) + .await } /// Pings the presence of the given user in the given room, setting the /// specified state. - pub fn ping_presence(&self, user_id: &UserId, new_state: &PresenceState) -> Result<()> { + pub async fn ping_presence(&self, user_id: &UserId, new_state: &PresenceState) -> Result<()> { const REFRESH_TIMEOUT: u64 = 60 * 25 * 1000; - let last_presence = self.db.get_presence(user_id)?; + let last_presence = self.db.get_presence(user_id).await; let state_changed = match last_presence { - None => true, - Some((_, ref presence)) => presence.content.presence != *new_state, + Err(_) => true, + Ok((_, ref presence)) => presence.content.presence != *new_state, }; let last_last_active_ago = match last_presence { - None => 0_u64, - Some((_, ref presence)) => presence.content.last_active_ago.unwrap_or_default().into(), + Err(_) => 0_u64, + Ok((_, ref presence)) => presence.content.last_active_ago.unwrap_or_default().into(), }; if !state_changed && last_last_active_ago < REFRESH_TIMEOUT { @@ -111,17 +112,18 @@ impl Service { } let status_msg = match last_presence { - Some((_, ref presence)) => presence.content.status_msg.clone(), - None => Some(String::new()), + Ok((_, ref presence)) => presence.content.status_msg.clone(), + Err(_) => Some(String::new()), }; let last_active_ago = UInt::new(0); let currently_active = *new_state == PresenceState::Online; self.set_presence(user_id, new_state, Some(currently_active), last_active_ago, status_msg) + .await } /// Adds a presence event which will be saved until a new event replaces it. - pub fn set_presence( + pub async fn set_presence( &self, user_id: &UserId, state: &PresenceState, currently_active: Option, last_active_ago: Option, status_msg: Option, ) -> Result<()> { @@ -131,7 +133,8 @@ impl Service { }; self.db - .set_presence(user_id, presence_state, currently_active, last_active_ago, status_msg)?; + .set_presence(user_id, presence_state, currently_active, last_active_ago, status_msg) + .await?; if self.timeout_remote_users || self.services.globals.user_is_local(user_id) { let timeout = match presence_state { @@ -154,28 +157,33 @@ impl Service { /// /// TODO: Why is this not used? #[allow(dead_code)] - pub fn remove_presence(&self, user_id: &UserId) -> Result<()> { self.db.remove_presence(user_id) } + pub async fn remove_presence(&self, user_id: &UserId) { self.db.remove_presence(user_id).await } /// Returns the most recent presence updates that happened after the event /// with id `since`. #[inline] - pub fn presence_since(&self, since: u64) -> Box)> + '_> { + pub fn presence_since(&self, since: u64) -> impl Stream)> + Send + '_ { self.db.presence_since(since) } - pub fn from_json_bytes_to_event(&self, bytes: &[u8], user_id: &UserId) -> Result { + #[inline] + pub async fn from_json_bytes_to_event(&self, bytes: &[u8], user_id: &UserId) -> Result { let presence = Presence::from_json_bytes(bytes)?; - presence.to_presence_event(user_id, &self.services.users) + let event = presence + .to_presence_event(user_id, &self.services.users) + .await; + + Ok(event) } - fn process_presence_timer(&self, user_id: &OwnedUserId) -> Result<()> { + async fn process_presence_timer(&self, user_id: &OwnedUserId) -> Result<()> { let mut presence_state = PresenceState::Offline; let mut last_active_ago = None; let mut status_msg = None; - let presence_event = self.get_presence(user_id)?; + let presence_event = self.get_presence(user_id).await; - if let Some(presence_event) = presence_event { + if let Ok(presence_event) = presence_event { presence_state = presence_event.content.presence; last_active_ago = presence_event.content.last_active_ago; status_msg = presence_event.content.status_msg; @@ -192,7 +200,8 @@ impl Service { ); if let Some(new_state) = new_state { - self.set_presence(user_id, &new_state, Some(false), last_active_ago, status_msg)?; + self.set_presence(user_id, &new_state, Some(false), last_active_ago, status_msg) + .await?; } Ok(()) diff --git a/src/service/presence/presence.rs b/src/service/presence/presence.rs index 570008f2..0d5c226b 100644 --- a/src/service/presence/presence.rs +++ b/src/service/presence/presence.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use conduit::{utils, Error, Result}; use ruma::{ events::presence::{PresenceEvent, PresenceEventContent}, @@ -42,7 +40,7 @@ impl Presence { } /// Creates a PresenceEvent from available data. - pub(super) fn to_presence_event(&self, user_id: &UserId, users: &Arc) -> Result { + pub(super) async fn to_presence_event(&self, user_id: &UserId, users: &users::Service) -> PresenceEvent { let now = utils::millis_since_unix_epoch(); let last_active_ago = if self.currently_active { None @@ -50,16 +48,16 @@ impl Presence { Some(UInt::new_saturating(now.saturating_sub(self.last_active_ts))) }; - Ok(PresenceEvent { + PresenceEvent { sender: user_id.to_owned(), content: PresenceEventContent { presence: self.state.clone(), status_msg: self.status_msg.clone(), currently_active: Some(self.currently_active), last_active_ago, - displayname: users.displayname(user_id)?, - avatar_url: users.avatar_url(user_id)?, + displayname: users.displayname(user_id).await.ok(), + avatar_url: users.avatar_url(user_id).await.ok(), }, - }) + } } } diff --git a/src/service/pusher/data.rs b/src/service/pusher/data.rs deleted file mode 100644 index f9734334..00000000 --- a/src/service/pusher/data.rs +++ /dev/null @@ -1,77 +0,0 @@ -use std::sync::Arc; - -use conduit::{utils, Error, Result}; -use database::{Database, Map}; -use ruma::{ - api::client::push::{set_pusher, Pusher}, - UserId, -}; - -pub(super) struct Data { - senderkey_pusher: Arc, -} - -impl Data { - pub(super) fn new(db: &Arc) -> Self { - Self { - senderkey_pusher: db["senderkey_pusher"].clone(), - } - } - - pub(super) fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result<()> { - match pusher { - set_pusher::v3::PusherAction::Post(data) => { - let mut key = sender.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(data.pusher.ids.pushkey.as_bytes()); - self.senderkey_pusher - .insert(&key, &serde_json::to_vec(pusher).expect("Pusher is valid JSON value"))?; - Ok(()) - }, - set_pusher::v3::PusherAction::Delete(ids) => { - let mut key = sender.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(ids.pushkey.as_bytes()); - self.senderkey_pusher.remove(&key).map_err(Into::into) - }, - } - } - - pub(super) fn get_pusher(&self, sender: &UserId, pushkey: &str) -> Result> { - let mut senderkey = sender.as_bytes().to_vec(); - senderkey.push(0xFF); - senderkey.extend_from_slice(pushkey.as_bytes()); - - self.senderkey_pusher - .get(&senderkey)? - .map(|push| serde_json::from_slice(&push).map_err(|_| Error::bad_database("Invalid Pusher in db."))) - .transpose() - } - - pub(super) fn get_pushers(&self, sender: &UserId) -> Result> { - let mut prefix = sender.as_bytes().to_vec(); - prefix.push(0xFF); - - self.senderkey_pusher - .scan_prefix(prefix) - .map(|(_, push)| serde_json::from_slice(&push).map_err(|_| Error::bad_database("Invalid Pusher in db."))) - .collect() - } - - pub(super) fn get_pushkeys<'a>(&'a self, sender: &UserId) -> Box> + 'a> { - let mut prefix = sender.as_bytes().to_vec(); - prefix.push(0xFF); - - Box::new(self.senderkey_pusher.scan_prefix(prefix).map(|(k, _)| { - let mut parts = k.splitn(2, |&b| b == 0xFF); - let _senderkey = parts.next(); - let push_key = parts - .next() - .ok_or_else(|| Error::bad_database("Invalid senderkey_pusher in db"))?; - let push_key_string = utils::string_from_bytes(push_key) - .map_err(|_| Error::bad_database("Invalid pusher bytes in senderkey_pusher"))?; - - Ok(push_key_string) - })) - } -} diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index de87264c..44ff1945 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -1,9 +1,13 @@ -mod data; - use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; -use conduit::{debug_error, err, trace, utils::string_from_bytes, warn, Err, PduEvent, Result}; +use conduit::{ + debug_error, err, trace, + utils::{stream::TryIgnore, string_from_bytes}, + Err, PduEvent, Result, +}; +use database::{Deserialized, Ignore, Interfix, Map}; +use futures::{Stream, StreamExt}; use ipaddress::IPAddress; use ruma::{ api::{ @@ -22,12 +26,11 @@ use ruma::{ uint, RoomId, UInt, UserId, }; -use self::data::Data; use crate::{client, globals, rooms, users, Dep}; pub struct Service { - services: Services, db: Data, + services: Services, } struct Services { @@ -38,9 +41,16 @@ struct Services { users: Dep, } +struct Data { + senderkey_pusher: Arc, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + db: Data { + senderkey_pusher: args.db["senderkey_pusher"].clone(), + }, services: Services { globals: args.depend::("globals"), client: args.depend::("client"), @@ -48,7 +58,6 @@ impl crate::Service for Service { state_cache: args.depend::("rooms::state_cache"), users: args.depend::("users"), }, - db: Data::new(args.db), })) } @@ -56,19 +65,52 @@ impl crate::Service for Service { } impl Service { - pub fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result<()> { - self.db.set_pusher(sender, pusher) + pub fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) { + match pusher { + set_pusher::v3::PusherAction::Post(data) => { + let mut key = sender.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(data.pusher.ids.pushkey.as_bytes()); + self.db + .senderkey_pusher + .insert(&key, &serde_json::to_vec(pusher).expect("Pusher is valid JSON value")); + }, + set_pusher::v3::PusherAction::Delete(ids) => { + let mut key = sender.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(ids.pushkey.as_bytes()); + self.db.senderkey_pusher.remove(&key); + }, + } } - pub fn get_pusher(&self, sender: &UserId, pushkey: &str) -> Result> { - self.db.get_pusher(sender, pushkey) + pub async fn get_pusher(&self, sender: &UserId, pushkey: &str) -> Result { + let senderkey = (sender, pushkey); + self.db + .senderkey_pusher + .qry(&senderkey) + .await + .deserialized_json() } - pub fn get_pushers(&self, sender: &UserId) -> Result> { self.db.get_pushers(sender) } + pub async fn get_pushers(&self, sender: &UserId) -> Vec { + let prefix = (sender, Interfix); + self.db + .senderkey_pusher + .stream_prefix(&prefix) + .ignore_err() + .map(|(_, val): (Ignore, &[u8])| serde_json::from_slice(val).expect("Invalid Pusher in db.")) + .collect() + .await + } - #[must_use] - pub fn get_pushkeys(&self, sender: &UserId) -> Box> + '_> { - self.db.get_pushkeys(sender) + pub fn get_pushkeys<'a>(&'a self, sender: &'a UserId) -> impl Stream + Send + 'a { + let prefix = (sender, Interfix); + self.db + .senderkey_pusher + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, pushkey): (Ignore, &str)| pushkey) } #[tracing::instrument(skip(self, dest, request))] @@ -161,15 +203,18 @@ impl Service { let power_levels: RoomPowerLevelsEventContent = self .services .state_accessor - .room_state_get(&pdu.room_id, &StateEventType::RoomPowerLevels, "")? - .map(|ev| { + .room_state_get(&pdu.room_id, &StateEventType::RoomPowerLevels, "") + .await + .and_then(|ev| { serde_json::from_str(ev.content.get()) - .map_err(|e| err!(Database("invalid m.room.power_levels event: {e:?}"))) + .map_err(|e| err!(Database(error!("invalid m.room.power_levels event: {e:?}")))) }) - .transpose()? .unwrap_or_default(); - for action in self.get_actions(user, &ruleset, &power_levels, &pdu.to_sync_room_event(), &pdu.room_id)? { + for action in self + .get_actions(user, &ruleset, &power_levels, &pdu.to_sync_room_event(), &pdu.room_id) + .await? + { let n = match action { Action::Notify => true, Action::SetTweak(tweak) => { @@ -197,7 +242,7 @@ impl Service { } #[tracing::instrument(skip(self, user, ruleset, pdu), level = "debug")] - pub fn get_actions<'a>( + pub async fn get_actions<'a>( &self, user: &UserId, ruleset: &'a Ruleset, power_levels: &RoomPowerLevelsEventContent, pdu: &Raw, room_id: &RoomId, ) -> Result<&'a [Action]> { @@ -207,21 +252,27 @@ impl Service { notifications: power_levels.notifications.clone(), }; + let room_joined_count = self + .services + .state_cache + .room_joined_count(room_id) + .await + .unwrap_or(1) + .try_into() + .unwrap_or_else(|_| uint!(0)); + + let user_display_name = self + .services + .users + .displayname(user) + .await + .unwrap_or_else(|_| user.localpart().to_owned()); + let ctx = PushConditionRoomCtx { room_id: room_id.to_owned(), - member_count: UInt::try_from( - self.services - .state_cache - .room_joined_count(room_id)? - .unwrap_or(1), - ) - .unwrap_or_else(|_| uint!(0)), + member_count: room_joined_count, user_id: user.to_owned(), - user_display_name: self - .services - .users - .displayname(user)? - .unwrap_or_else(|| user.localpart().to_owned()), + user_display_name, power_levels: Some(power_levels), }; @@ -278,9 +329,14 @@ impl Service { notifi.user_is_target = event.state_key.as_deref() == Some(event.sender.as_str()); } - notifi.sender_display_name = self.services.users.displayname(&event.sender)?; + notifi.sender_display_name = self.services.users.displayname(&event.sender).await.ok(); - notifi.room_name = self.services.state_accessor.get_name(&event.room_id)?; + notifi.room_name = self + .services + .state_accessor + .get_name(&event.room_id) + .await + .ok(); self.send_request(&http.url, send_event_notification::v1::Request::new(notifi)) .await?; diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 07d9a0fa..ea4b1100 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -193,7 +193,7 @@ impl super::Service { .send() .await; - trace!("response: {:?}", response); + trace!("response: {response:?}"); if let Err(e) = &response { debug!("error: {e:?}"); return Ok(None); @@ -206,7 +206,7 @@ impl super::Service { } let text = response.text().await?; - trace!("response text: {:?}", text); + trace!("response text: {text:?}"); if text.len() >= 12288 { debug_warn!("response contains junk"); return Ok(None); @@ -225,7 +225,7 @@ impl super::Service { return Ok(None); } - debug_info!("{:?} found at {:?}", dest, m_server); + debug_info!("{dest:?} found at {m_server:?}"); Ok(Some(m_server.to_owned())) } diff --git a/src/service/rooms/alias/data.rs b/src/service/rooms/alias/data.rs deleted file mode 100644 index efd2b5b7..00000000 --- a/src/service/rooms/alias/data.rs +++ /dev/null @@ -1,125 +0,0 @@ -use std::sync::Arc; - -use conduit::{utils, Error, Result}; -use database::Map; -use ruma::{api::client::error::ErrorKind, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, UserId}; - -use crate::{globals, Dep}; - -pub(super) struct Data { - alias_userid: Arc, - alias_roomid: Arc, - aliasid_alias: Arc, - services: Services, -} - -struct Services { - globals: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - alias_userid: db["alias_userid"].clone(), - alias_roomid: db["alias_roomid"].clone(), - aliasid_alias: db["aliasid_alias"].clone(), - services: Services { - globals: args.depend::("globals"), - }, - } - } - - pub(super) fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { - // Comes first as we don't want a stuck alias - self.alias_userid - .insert(alias.alias().as_bytes(), user_id.as_bytes())?; - - self.alias_roomid - .insert(alias.alias().as_bytes(), room_id.as_bytes())?; - - let mut aliasid = room_id.as_bytes().to_vec(); - aliasid.push(0xFF); - aliasid.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); - self.aliasid_alias.insert(&aliasid, alias.as_bytes())?; - - Ok(()) - } - - pub(super) fn remove_alias(&self, alias: &RoomAliasId) -> Result<()> { - if let Some(room_id) = self.alias_roomid.get(alias.alias().as_bytes())? { - let mut prefix = room_id.to_vec(); - prefix.push(0xFF); - - for (key, _) in self.aliasid_alias.scan_prefix(prefix) { - self.aliasid_alias.remove(&key)?; - } - - self.alias_roomid.remove(alias.alias().as_bytes())?; - - self.alias_userid.remove(alias.alias().as_bytes())?; - } else { - return Err(Error::BadRequest(ErrorKind::NotFound, "Alias does not exist or is invalid.")); - } - - Ok(()) - } - - pub(super) fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { - self.alias_roomid - .get(alias.alias().as_bytes())? - .map(|bytes| { - RoomId::parse( - utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Room ID in alias_roomid is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("Room ID in alias_roomid is invalid.")) - }) - .transpose() - } - - pub(super) fn who_created_alias(&self, alias: &RoomAliasId) -> Result> { - self.alias_userid - .get(alias.alias().as_bytes())? - .map(|bytes| { - UserId::parse( - utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("User ID in alias_userid is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("User ID in alias_roomid is invalid.")) - }) - .transpose() - } - - pub(super) fn local_aliases_for_room<'a>( - &'a self, room_id: &RoomId, - ) -> Box> + 'a + Send> { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xFF); - - Box::new(self.aliasid_alias.scan_prefix(prefix).map(|(_, bytes)| { - utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Invalid alias bytes in aliasid_alias."))? - .try_into() - .map_err(|_| Error::bad_database("Invalid alias in aliasid_alias.")) - })) - } - - pub(super) fn all_local_aliases<'a>(&'a self) -> Box> + 'a> { - Box::new( - self.alias_roomid - .iter() - .map(|(room_alias_bytes, room_id_bytes)| { - let room_alias_localpart = utils::string_from_bytes(&room_alias_bytes) - .map_err(|_| Error::bad_database("Invalid alias bytes in aliasid_alias."))?; - - let room_id = utils::string_from_bytes(&room_id_bytes) - .map_err(|_| Error::bad_database("Invalid room_id bytes in aliasid_alias."))? - .try_into() - .map_err(|_| Error::bad_database("Invalid room_id in aliasid_alias."))?; - - Ok((room_id, room_alias_localpart)) - }), - ) - } -} diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index f2e01ab5..6b81a221 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -1,19 +1,23 @@ -mod data; mod remote; use std::sync::Arc; -use conduit::{err, Error, Result}; +use conduit::{ + err, + utils::{stream::TryIgnore, ReadyExt}, + Err, Error, Result, +}; +use database::{Deserialized, Ignore, Interfix, Map}; +use futures::{Stream, StreamExt}; use ruma::{ api::client::error::ErrorKind, events::{ room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, StateEventType, }, - OwnedRoomAliasId, OwnedRoomId, OwnedServerName, RoomAliasId, RoomId, RoomOrAliasId, UserId, + OwnedRoomId, OwnedServerName, OwnedUserId, RoomAliasId, RoomId, RoomOrAliasId, UserId, }; -use self::data::Data; use crate::{admin, appservice, appservice::RegistrationInfo, globals, rooms, sending, Dep}; pub struct Service { @@ -21,6 +25,12 @@ pub struct Service { services: Services, } +struct Data { + alias_userid: Arc, + alias_roomid: Arc, + aliasid_alias: Arc, +} + struct Services { admin: Dep, appservice: Dep, @@ -32,7 +42,11 @@ struct Services { impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(&args), + db: Data { + alias_userid: args.db["alias_userid"].clone(), + alias_roomid: args.db["alias_roomid"].clone(), + aliasid_alias: args.db["aliasid_alias"].clone(), + }, services: Services { admin: args.depend::("admin"), appservice: args.depend::("appservice"), @@ -50,25 +64,52 @@ impl Service { #[tracing::instrument(skip(self))] pub fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { if alias == self.services.globals.admin_alias && user_id != self.services.globals.server_user { - Err(Error::BadRequest( + return Err(Error::BadRequest( ErrorKind::forbidden(), "Only the server user can set this alias", - )) - } else { - self.db.set_alias(alias, room_id, user_id) + )); } + + // Comes first as we don't want a stuck alias + self.db + .alias_userid + .insert(alias.alias().as_bytes(), user_id.as_bytes()); + + self.db + .alias_roomid + .insert(alias.alias().as_bytes(), room_id.as_bytes()); + + let mut aliasid = room_id.as_bytes().to_vec(); + aliasid.push(0xFF); + aliasid.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); + self.db.aliasid_alias.insert(&aliasid, alias.as_bytes()); + + Ok(()) } #[tracing::instrument(skip(self))] pub async fn remove_alias(&self, alias: &RoomAliasId, user_id: &UserId) -> Result<()> { - if self.user_can_remove_alias(alias, user_id).await? { - self.db.remove_alias(alias) - } else { - Err(Error::BadRequest( - ErrorKind::forbidden(), - "User is not permitted to remove this alias.", - )) + if !self.user_can_remove_alias(alias, user_id).await? { + return Err!(Request(Forbidden("User is not permitted to remove this alias."))); } + + let alias = alias.alias(); + let Ok(room_id) = self.db.alias_roomid.qry(&alias).await else { + return Err!(Request(NotFound("Alias does not exist or is invalid."))); + }; + + let prefix = (&room_id, Interfix); + self.db + .aliasid_alias + .keys_prefix(&prefix) + .ignore_err() + .ready_for_each(|key: &[u8]| self.db.aliasid_alias.remove(&key)) + .await; + + self.db.alias_roomid.remove(alias.as_bytes()); + self.db.alias_userid.remove(alias.as_bytes()); + + Ok(()) } pub async fn resolve(&self, room: &RoomOrAliasId) -> Result { @@ -97,9 +138,9 @@ impl Service { return self.remote_resolve(room_alias, servers).await; } - let room_id: Option = match self.resolve_local_alias(room_alias)? { - Some(r) => Some(r), - None => self.resolve_appservice_alias(room_alias).await?, + let room_id: Option = match self.resolve_local_alias(room_alias).await { + Ok(r) => Some(r), + Err(_) => self.resolve_appservice_alias(room_alias).await?, }; room_id.map_or_else( @@ -109,46 +150,54 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result> { - self.db.resolve_local_alias(alias) + pub async fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result { + self.db.alias_roomid.qry(alias.alias()).await.deserialized() } #[tracing::instrument(skip(self), level = "debug")] - pub fn local_aliases_for_room<'a>( - &'a self, room_id: &RoomId, - ) -> Box> + 'a + Send> { - self.db.local_aliases_for_room(room_id) + pub fn local_aliases_for_room<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .aliasid_alias + .stream_prefix(&prefix) + .ignore_err() + .map(|((Ignore, Ignore), alias): ((Ignore, Ignore), &RoomAliasId)| alias) } #[tracing::instrument(skip(self), level = "debug")] - pub fn all_local_aliases<'a>(&'a self) -> Box> + 'a> { - self.db.all_local_aliases() + pub fn all_local_aliases<'a>(&'a self) -> impl Stream + Send + 'a { + self.db + .alias_roomid + .stream() + .ignore_err() + .map(|(alias_localpart, room_id): (&str, &RoomId)| (room_id, alias_localpart)) } async fn user_can_remove_alias(&self, alias: &RoomAliasId, user_id: &UserId) -> Result { - let Some(room_id) = self.resolve_local_alias(alias)? else { - return Err(Error::BadRequest(ErrorKind::NotFound, "Alias not found.")); - }; + let room_id = self + .resolve_local_alias(alias) + .await + .map_err(|_| err!(Request(NotFound("Alias not found."))))?; let server_user = &self.services.globals.server_user; // The creator of an alias can remove it if self - .db - .who_created_alias(alias)? - .is_some_and(|user| user == user_id) + .who_created_alias(alias).await + .is_ok_and(|user| user == user_id) // Server admins can remove any local alias - || self.services.admin.user_is_admin(user_id).await? + || self.services.admin.user_is_admin(user_id).await // Always allow the server service account to remove the alias, since there may not be an admin room || server_user == user_id { Ok(true) // Checking whether the user is able to change canonical aliases of the // room - } else if let Some(event) = - self.services - .state_accessor - .room_state_get(&room_id, &StateEventType::RoomPowerLevels, "")? + } else if let Ok(event) = self + .services + .state_accessor + .room_state_get(&room_id, &StateEventType::RoomPowerLevels, "") + .await { serde_json::from_str(event.content.get()) .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) @@ -157,10 +206,11 @@ impl Service { }) // If there is no power levels event, only the room creator can change // canonical aliases - } else if let Some(event) = - self.services - .state_accessor - .room_state_get(&room_id, &StateEventType::RoomCreate, "")? + } else if let Ok(event) = self + .services + .state_accessor + .room_state_get(&room_id, &StateEventType::RoomCreate, "") + .await { Ok(event.sender == user_id) } else { @@ -168,6 +218,10 @@ impl Service { } } + async fn who_created_alias(&self, alias: &RoomAliasId) -> Result { + self.db.alias_userid.qry(alias.alias()).await.deserialized() + } + async fn resolve_appservice_alias(&self, room_alias: &RoomAliasId) -> Result> { use ruma::api::appservice::query::query_room_alias; @@ -185,10 +239,11 @@ impl Service { .await, Ok(Some(_opt_result)) ) { - return Ok(Some( - self.resolve_local_alias(room_alias)? - .ok_or_else(|| err!(Request(NotFound("Room does not exist."))))?, - )); + return self + .resolve_local_alias(room_alias) + .await + .map_err(|_| err!(Request(NotFound("Room does not exist.")))) + .map(Some); } } diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index 6e7c7835..3d00374e 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -24,7 +24,7 @@ impl Data { } } - pub(super) fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result>> { + pub(super) async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result>> { // Check RAM cache if let Some(result) = self.auth_chain_cache.lock().unwrap().get_mut(key) { return Ok(Some(Arc::clone(result))); @@ -33,17 +33,14 @@ impl Data { // We only save auth chains for single events in the db if key.len() == 1 { // Check DB cache - let chain = self - .shorteventid_authchain - .get(&key[0].to_be_bytes())? - .map(|chain| { - chain - .chunks_exact(size_of::()) - .map(utils::u64_from_u8) - .collect::>() - }); + let chain = self.shorteventid_authchain.qry(&key[0]).await.map(|chain| { + chain + .chunks_exact(size_of::()) + .map(utils::u64_from_u8) + .collect::>() + }); - if let Some(chain) = chain { + if let Ok(chain) = chain { // Cache in RAM self.auth_chain_cache .lock() @@ -66,7 +63,7 @@ impl Data { .iter() .flat_map(|s| s.to_be_bytes().to_vec()) .collect::>(), - )?; + ); } // Cache in RAM diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index d0bc425f..7bc239d7 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -5,7 +5,8 @@ use std::{ sync::Arc, }; -use conduit::{debug, error, trace, validated, warn, Err, Result}; +use conduit::{debug, debug_error, trace, utils::IterStream, validated, warn, Err, Result}; +use futures::{FutureExt, Stream, StreamExt}; use ruma::{EventId, RoomId}; use self::data::Data; @@ -38,7 +39,7 @@ impl crate::Service for Service { impl Service { pub async fn event_ids_iter<'a>( &'a self, room_id: &RoomId, starting_events_: Vec>, - ) -> Result> + 'a> { + ) -> Result> + Send + 'a> { let mut starting_events: Vec<&EventId> = Vec::with_capacity(starting_events_.len()); for starting_event in &starting_events_ { starting_events.push(starting_event); @@ -48,7 +49,13 @@ impl Service { .get_auth_chain(room_id, &starting_events) .await? .into_iter() - .filter_map(move |sid| self.services.short.get_eventid_from_short(sid).ok())) + .stream() + .filter_map(|sid| { + self.services + .short + .get_eventid_from_short(sid) + .map(Result::ok) + })) } #[tracing::instrument(skip_all, name = "auth_chain")] @@ -61,7 +68,8 @@ impl Service { for (i, &short) in self .services .short - .multi_get_or_create_shorteventid(starting_events)? + .multi_get_or_create_shorteventid(starting_events) + .await .iter() .enumerate() { @@ -85,7 +93,7 @@ impl Service { } let chunk_key: Vec = chunk.iter().map(|(short, _)| short).copied().collect(); - if let Some(cached) = self.get_cached_eventid_authchain(&chunk_key)? { + if let Some(cached) = self.get_cached_eventid_authchain(&chunk_key).await? { trace!("Found cache entry for whole chunk"); full_auth_chain.extend(cached.iter().copied()); hits = hits.saturating_add(1); @@ -96,12 +104,12 @@ impl Service { let mut misses2: usize = 0; let mut chunk_cache = Vec::with_capacity(chunk.len()); for (sevent_id, event_id) in chunk { - if let Some(cached) = self.get_cached_eventid_authchain(&[sevent_id])? { + if let Some(cached) = self.get_cached_eventid_authchain(&[sevent_id]).await? { trace!(?event_id, "Found cache entry for event"); chunk_cache.extend(cached.iter().copied()); hits2 = hits2.saturating_add(1); } else { - let auth_chain = self.get_auth_chain_inner(room_id, event_id)?; + let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; self.cache_auth_chain(vec![sevent_id], &auth_chain)?; chunk_cache.extend(auth_chain.iter()); misses2 = misses2.saturating_add(1); @@ -143,15 +151,16 @@ impl Service { } #[tracing::instrument(skip(self, room_id))] - fn get_auth_chain_inner(&self, room_id: &RoomId, event_id: &EventId) -> Result> { + async fn get_auth_chain_inner(&self, room_id: &RoomId, event_id: &EventId) -> Result> { let mut todo = vec![Arc::from(event_id)]; let mut found = HashSet::new(); while let Some(event_id) = todo.pop() { trace!(?event_id, "processing auth event"); - match self.services.timeline.get_pdu(&event_id) { - Ok(Some(pdu)) => { + match self.services.timeline.get_pdu(&event_id).await { + Err(e) => debug_error!(?event_id, ?e, "Could not find pdu mentioned in auth events"), + Ok(pdu) => { if pdu.room_id != room_id { return Err!(Request(Forbidden( "auth event {event_id:?} for incorrect room {} which is not {}", @@ -160,7 +169,11 @@ impl Service { ))); } for auth_event in &pdu.auth_events { - let sauthevent = self.services.short.get_or_create_shorteventid(auth_event)?; + let sauthevent = self + .services + .short + .get_or_create_shorteventid(auth_event) + .await; if found.insert(sauthevent) { trace!(?event_id, ?auth_event, "adding auth event to processing queue"); @@ -168,20 +181,14 @@ impl Service { } } }, - Ok(None) => { - warn!(?event_id, "Could not find pdu mentioned in auth events"); - }, - Err(error) => { - error!(?event_id, ?error, "Could not load event in auth chain"); - }, } } Ok(found) } - pub fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result>> { - self.db.get_cached_eventid_authchain(key) + pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result>> { + self.db.get_cached_eventid_authchain(key).await } #[tracing::instrument(skip(self), level = "debug")] diff --git a/src/service/rooms/directory/data.rs b/src/service/rooms/directory/data.rs deleted file mode 100644 index 713ee057..00000000 --- a/src/service/rooms/directory/data.rs +++ /dev/null @@ -1,39 +0,0 @@ -use std::sync::Arc; - -use conduit::{utils, Error, Result}; -use database::{Database, Map}; -use ruma::{OwnedRoomId, RoomId}; - -pub(super) struct Data { - publicroomids: Arc, -} - -impl Data { - pub(super) fn new(db: &Arc) -> Self { - Self { - publicroomids: db["publicroomids"].clone(), - } - } - - pub(super) fn set_public(&self, room_id: &RoomId) -> Result<()> { - self.publicroomids.insert(room_id.as_bytes(), &[]) - } - - pub(super) fn set_not_public(&self, room_id: &RoomId) -> Result<()> { - self.publicroomids.remove(room_id.as_bytes()) - } - - pub(super) fn is_public_room(&self, room_id: &RoomId) -> Result { - Ok(self.publicroomids.get(room_id.as_bytes())?.is_some()) - } - - pub(super) fn public_rooms<'a>(&'a self) -> Box> + 'a> { - Box::new(self.publicroomids.iter().map(|(bytes, _)| { - RoomId::parse( - utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Room ID in publicroomids is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("Room ID in publicroomids is invalid.")) - })) - } -} diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 706e6c2e..3585205d 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -1,36 +1,44 @@ -mod data; - use std::sync::Arc; -use conduit::Result; -use ruma::{OwnedRoomId, RoomId}; - -use self::data::Data; +use conduit::{implement, utils::stream::TryIgnore, Result}; +use database::{Ignore, Map}; +use futures::{Stream, StreamExt}; +use ruma::RoomId; pub struct Service { db: Data, } +struct Data { + publicroomids: Arc, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data { + publicroomids: args.db["publicroomids"].clone(), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - #[tracing::instrument(skip(self), level = "debug")] - pub fn set_public(&self, room_id: &RoomId) -> Result<()> { self.db.set_public(room_id) } +#[implement(Service)] +pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_id.as_bytes(), &[]); } - #[tracing::instrument(skip(self), level = "debug")] - pub fn set_not_public(&self, room_id: &RoomId) -> Result<()> { self.db.set_not_public(room_id) } +#[implement(Service)] +pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id.as_bytes()); } - #[tracing::instrument(skip(self), level = "debug")] - pub fn is_public_room(&self, room_id: &RoomId) -> Result { self.db.is_public_room(room_id) } +#[implement(Service)] +pub async fn is_public_room(&self, room_id: &RoomId) -> bool { self.db.publicroomids.qry(room_id).await.is_ok() } - #[tracing::instrument(skip(self), level = "debug")] - pub fn public_rooms(&self) -> impl Iterator> + '_ { self.db.public_rooms() } +#[implement(Service)] +pub fn public_rooms(&self) -> impl Stream + Send { + self.db + .publicroomids + .keys() + .ignore_err() + .map(|(room_id, _): (&RoomId, Ignore)| room_id) } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index bee986de..07d6e4db 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -3,17 +3,18 @@ mod parse_incoming_pdu; use std::{ collections::{hash_map, BTreeMap, HashMap, HashSet}, fmt::Write, - pin::Pin, sync::{Arc, RwLock as StdRwLock}, time::Instant, }; use conduit::{ - debug, debug_error, debug_info, err, error, info, pdu, trace, - utils::{math::continue_exponential_backoff_secs, MutexMap}, - warn, Error, PduEvent, Result, + debug, debug_error, debug_info, debug_warn, err, info, pdu, + result::LogErr, + trace, + utils::{math::continue_exponential_backoff_secs, IterStream, MutexMap}, + warn, Err, Error, PduEvent, Result, }; -use futures_util::Future; +use futures::{future, future::ready, FutureExt, StreamExt, TryFutureExt}; use ruma::{ api::{ client::error::ErrorKind, @@ -27,7 +28,7 @@ use ruma::{ }, int, serde::Base64, - state_res::{self, RoomVersion, StateMap}, + state_res::{self, EventTypeExt, RoomVersion, StateMap}, uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId, ServerName, }; @@ -60,14 +61,6 @@ struct Services { type RoomMutexMap = MutexMap; type HandleTimeMap = HashMap; -// We use some AsyncRecursiveType hacks here so we can call async funtion -// recursively. -type AsyncRecursiveType<'a, T> = Pin + 'a + Send>>; -type AsyncRecursiveCanonicalJsonVec<'a> = - AsyncRecursiveType<'a, Vec<(Arc, Option>)>>; -type AsyncRecursiveCanonicalJsonResult<'a> = - AsyncRecursiveType<'a, Result<(Arc, BTreeMap)>>; - impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -142,17 +135,17 @@ impl Service { pub_key_map: &'a RwLock>>, ) -> Result>> { // 1. Skip the PDU if we already have it as a timeline event - if let Some(pdu_id) = self.services.timeline.get_pdu_id(event_id)? { + if let Ok(pdu_id) = self.services.timeline.get_pdu_id(event_id).await { return Ok(Some(pdu_id.to_vec())); } // 1.1 Check the server is in the room - if !self.services.metadata.exists(room_id)? { + if !self.services.metadata.exists(room_id).await { return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server")); } // 1.2 Check if the room is disabled - if self.services.metadata.is_disabled(room_id)? { + if self.services.metadata.is_disabled(room_id).await { return Err(Error::BadRequest( ErrorKind::forbidden(), "Federation of this room is currently disabled on this server.", @@ -160,7 +153,7 @@ impl Service { } // 1.3.1 Check room ACL on origin field/server - self.acl_check(origin, room_id)?; + self.acl_check(origin, room_id).await?; // 1.3.2 Check room ACL on sender's server name let sender: OwnedUserId = serde_json::from_value( @@ -172,26 +165,23 @@ impl Service { ) .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "User ID in sender is invalid"))?; - self.acl_check(sender.server_name(), room_id)?; + self.acl_check(sender.server_name(), room_id).await?; // Fetch create event let create_event = self .services .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "")? - .ok_or_else(|| Error::bad_database("Failed to find create event in db."))?; + .room_state_get(room_id, &StateEventType::RoomCreate, "") + .await?; // Procure the room version let room_version_id = Self::get_room_version_id(&create_event)?; - let first_pdu_in_room = self - .services - .timeline - .first_pdu_in_room(room_id)? - .ok_or_else(|| Error::bad_database("Failed to find first pdu in db."))?; + let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; let (incoming_pdu, val) = self .handle_outlier_pdu(origin, &create_event, event_id, room_id, value, false, pub_key_map) + .boxed() .await?; Self::check_room_id(room_id, &incoming_pdu)?; @@ -235,7 +225,7 @@ impl Service { { Ok(()) => continue, Err(e) => { - warn!("Prev event {} failed: {}", prev_id, e); + warn!("Prev event {prev_id} failed: {e}"); match self .services .globals @@ -287,7 +277,7 @@ impl Service { create_event: &Arc, first_pdu_in_room: &Arc, prev_id: &EventId, ) -> Result<()> { // Check for disabled again because it might have changed - if self.services.metadata.is_disabled(room_id)? { + if self.services.metadata.is_disabled(room_id).await { debug!( "Federaton of room {room_id} is currently disabled on this server. Request by origin {origin} and \ event ID {event_id}" @@ -349,149 +339,153 @@ impl Service { } #[allow(clippy::too_many_arguments)] - fn handle_outlier_pdu<'a>( - &'a self, origin: &'a ServerName, create_event: &'a PduEvent, event_id: &'a EventId, room_id: &'a RoomId, + async fn handle_outlier_pdu<'a>( + &self, origin: &'a ServerName, create_event: &'a PduEvent, event_id: &'a EventId, room_id: &'a RoomId, mut value: BTreeMap, auth_events_known: bool, pub_key_map: &'a RwLock>>, - ) -> AsyncRecursiveCanonicalJsonResult<'a> { - Box::pin(async move { - // 1. Remove unsigned field - value.remove("unsigned"); + ) -> Result<(Arc, BTreeMap)> { + // 1. Remove unsigned field + value.remove("unsigned"); - // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we anywhere?: https://matrix.org/docs/spec/rooms/v6#canonical-json + // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we anywhere?: https://matrix.org/docs/spec/rooms/v6#canonical-json - // 2. Check signatures, otherwise drop - // 3. check content hash, redact if doesn't match - let room_version_id = Self::get_room_version_id(create_event)?; + // 2. Check signatures, otherwise drop + // 3. check content hash, redact if doesn't match + let room_version_id = Self::get_room_version_id(create_event)?; - let guard = pub_key_map.read().await; - let mut val = match ruma::signatures::verify_event(&guard, &value, &room_version_id) { - Err(e) => { - // Drop - warn!("Dropping bad event {}: {}", event_id, e,); - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Signature verification failed")); - }, - Ok(ruma::signatures::Verified::Signatures) => { - // Redact - debug_info!("Calculated hash does not match (redaction): {event_id}"); - let Ok(obj) = ruma::canonical_json::redact(value, &room_version_id, None) else { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Redaction failed")); - }; + let guard = pub_key_map.read().await; + let mut val = match ruma::signatures::verify_event(&guard, &value, &room_version_id) { + Err(e) => { + // Drop + warn!("Dropping bad event {event_id}: {e}"); + return Err!(Request(InvalidParam("Signature verification failed"))); + }, + Ok(ruma::signatures::Verified::Signatures) => { + // Redact + debug_info!("Calculated hash does not match (redaction): {event_id}"); + let Ok(obj) = ruma::canonical_json::redact(value, &room_version_id, None) else { + return Err!(Request(InvalidParam("Redaction failed"))); + }; - // Skip the PDU if it is redacted and we already have it as an outlier event - if self.services.timeline.get_pdu_json(event_id)?.is_some() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Event was redacted and we already knew about it", - )); - } + // Skip the PDU if it is redacted and we already have it as an outlier event + if self.services.timeline.get_pdu_json(event_id).await.is_ok() { + return Err!(Request(InvalidParam("Event was redacted and we already knew about it"))); + } - obj - }, - Ok(ruma::signatures::Verified::All) => value, - }; + obj + }, + Ok(ruma::signatures::Verified::All) => value, + }; - drop(guard); + drop(guard); - // Now that we have checked the signature and hashes we can add the eventID and - // convert to our PduEvent type - val.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); - let incoming_pdu = serde_json::from_value::( - serde_json::to_value(&val).expect("CanonicalJsonObj is a valid JsonValue"), - ) - .map_err(|_| Error::bad_database("Event is not a valid PDU."))?; + // Now that we have checked the signature and hashes we can add the eventID and + // convert to our PduEvent type + val.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); + let incoming_pdu = serde_json::from_value::( + serde_json::to_value(&val).expect("CanonicalJsonObj is a valid JsonValue"), + ) + .map_err(|_| Error::bad_database("Event is not a valid PDU."))?; - Self::check_room_id(room_id, &incoming_pdu)?; + Self::check_room_id(room_id, &incoming_pdu)?; - if !auth_events_known { - // 4. fetch any missing auth events doing all checks listed here starting at 1. - // These are not timeline events - // 5. Reject "due to auth events" if can't get all the auth events or some of - // the auth events are also rejected "due to auth events" - // NOTE: Step 5 is not applied anymore because it failed too often - debug!("Fetching auth events"); + if !auth_events_known { + // 4. fetch any missing auth events doing all checks listed here starting at 1. + // These are not timeline events + // 5. Reject "due to auth events" if can't get all the auth events or some of + // the auth events are also rejected "due to auth events" + // NOTE: Step 5 is not applied anymore because it failed too often + debug!("Fetching auth events"); + Box::pin( self.fetch_and_handle_outliers( origin, &incoming_pdu .auth_events .iter() .map(|x| Arc::from(&**x)) - .collect::>(), + .collect::>>(), create_event, room_id, &room_version_id, pub_key_map, - ) - .await; - } - - // 6. Reject "due to auth events" if the event doesn't pass auth based on the - // auth events - debug!("Checking based on auth events"); - // Build map of auth events - let mut auth_events = HashMap::with_capacity(incoming_pdu.auth_events.len()); - for id in &incoming_pdu.auth_events { - let Some(auth_event) = self.services.timeline.get_pdu(id)? else { - warn!("Could not find auth event {}", id); - continue; - }; - - Self::check_room_id(room_id, &auth_event)?; - - match auth_events.entry(( - auth_event.kind.to_string().into(), - auth_event - .state_key - .clone() - .expect("all auth events have state keys"), - )) { - hash_map::Entry::Vacant(v) => { - v.insert(auth_event); - }, - hash_map::Entry::Occupied(_) => { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Auth event's type and state_key combination exists multiple times.", - )); - }, - } - } - - // The original create event must be in the auth events - if !matches!( - auth_events - .get(&(StateEventType::RoomCreate, String::new())) - .map(AsRef::as_ref), - Some(_) | None - ) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Incoming event refers to wrong create event.", - )); - } - - if !state_res::event_auth::auth_check( - &Self::to_room_version(&room_version_id), - &incoming_pdu, - None::, // TODO: third party invite - |k, s| auth_events.get(&(k.to_string().into(), s.to_owned())), + ), ) - .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed"))? - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Auth check failed")); + .await; + } + + // 6. Reject "due to auth events" if the event doesn't pass auth based on the + // auth events + debug!("Checking based on auth events"); + // Build map of auth events + let mut auth_events = HashMap::with_capacity(incoming_pdu.auth_events.len()); + for id in &incoming_pdu.auth_events { + let Ok(auth_event) = self.services.timeline.get_pdu(id).await else { + warn!("Could not find auth event {id}"); + continue; + }; + + Self::check_room_id(room_id, &auth_event)?; + + match auth_events.entry(( + auth_event.kind.to_string().into(), + auth_event + .state_key + .clone() + .expect("all auth events have state keys"), + )) { + hash_map::Entry::Vacant(v) => { + v.insert(auth_event); + }, + hash_map::Entry::Occupied(_) => { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Auth event's type and state_key combination exists multiple times.", + )); + }, } + } - trace!("Validation successful."); + // The original create event must be in the auth events + if !matches!( + auth_events + .get(&(StateEventType::RoomCreate, String::new())) + .map(AsRef::as_ref), + Some(_) | None + ) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Incoming event refers to wrong create event.", + )); + } - // 7. Persist the event as an outlier. - self.services - .outlier - .add_pdu_outlier(&incoming_pdu.event_id, &val)?; + let state_fetch = |ty: &'static StateEventType, sk: &str| { + let key = ty.with_state_key(sk); + ready(auth_events.get(&key)) + }; - trace!("Added pdu as outlier."); + let auth_check = state_res::event_auth::auth_check( + &Self::to_room_version(&room_version_id), + &incoming_pdu, + None, // TODO: third party invite + state_fetch, + ) + .await + .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; - Ok((Arc::new(incoming_pdu), val)) - }) + if !auth_check { + return Err!(Request(Forbidden("Auth check failed"))); + } + + trace!("Validation successful."); + + // 7. Persist the event as an outlier. + self.services + .outlier + .add_pdu_outlier(&incoming_pdu.event_id, &val); + + trace!("Added pdu as outlier."); + + Ok((Arc::new(incoming_pdu), val)) } pub async fn upgrade_outlier_to_timeline_pdu( @@ -499,16 +493,22 @@ impl Service { origin: &ServerName, room_id: &RoomId, pub_key_map: &RwLock>>, ) -> Result>> { // Skip the PDU if we already have it as a timeline event - if let Ok(Some(pduid)) = self.services.timeline.get_pdu_id(&incoming_pdu.event_id) { + if let Ok(pduid) = self + .services + .timeline + .get_pdu_id(&incoming_pdu.event_id) + .await + { return Ok(Some(pduid.to_vec())); } if self .services .pdu_metadata - .is_event_soft_failed(&incoming_pdu.event_id)? + .is_event_soft_failed(&incoming_pdu.event_id) + .await { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event has been soft failed")); + return Err!(Request(InvalidParam("Event has been soft failed"))); } debug!("Upgrading to timeline pdu"); @@ -545,57 +545,69 @@ impl Service { debug!("Performing auth check"); // 11. Check the auth of the event passes based on the state of the event - let check_result = state_res::event_auth::auth_check( + let state_fetch_state = &state_at_incoming_event; + let state_fetch = |k: &'static StateEventType, s: String| async move { + let shortstatekey = self.services.short.get_shortstatekey(k, &s).await.ok()?; + + let event_id = state_fetch_state.get(&shortstatekey)?; + self.services.timeline.get_pdu(event_id).await.ok() + }; + + let auth_check = state_res::event_auth::auth_check( &room_version, &incoming_pdu, - None::, // TODO: third party invite - |k, s| { - self.services - .short - .get_shortstatekey(&k.to_string().into(), s) - .ok() - .flatten() - .and_then(|shortstatekey| state_at_incoming_event.get(&shortstatekey)) - .and_then(|event_id| self.services.timeline.get_pdu(event_id).ok().flatten()) - }, + None, // TODO: third party invite + |k, s| state_fetch(k, s.to_owned()), ) - .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))?; + .await + .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; - if !check_result { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Event has failed auth check with state at the event.", - )); + if !auth_check { + return Err!(Request(Forbidden("Event has failed auth check with state at the event."))); } debug!("Gathering auth events"); - let auth_events = self.services.state.get_auth_events( - room_id, - &incoming_pdu.kind, - &incoming_pdu.sender, - incoming_pdu.state_key.as_deref(), - &incoming_pdu.content, - )?; + let auth_events = self + .services + .state + .get_auth_events( + room_id, + &incoming_pdu.kind, + &incoming_pdu.sender, + incoming_pdu.state_key.as_deref(), + &incoming_pdu.content, + ) + .await?; + + let state_fetch = |k: &'static StateEventType, s: &str| { + let key = k.with_state_key(s); + ready(auth_events.get(&key).cloned()) + }; + + let auth_check = state_res::event_auth::auth_check( + &room_version, + &incoming_pdu, + None, // third-party invite + state_fetch, + ) + .await + .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; // Soft fail check before doing state res debug!("Performing soft-fail check"); let soft_fail = { use RoomVersionId::*; - !state_res::event_auth::auth_check(&room_version, &incoming_pdu, None::, |k, s| { - auth_events.get(&(k.clone(), s.to_owned())) - }) - .map_err(|_e| Error::BadRequest(ErrorKind::forbidden(), "Auth check failed."))? + !auth_check || incoming_pdu.kind == TimelineEventType::RoomRedaction && match room_version_id { V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &incoming_pdu.redacts { - !self.services.state_accessor.user_can_redact( - redact_id, - &incoming_pdu.sender, - &incoming_pdu.room_id, - true, - )? + !self + .services + .state_accessor + .user_can_redact(redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) + .await? } else { false } @@ -605,12 +617,11 @@ impl Service { .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; if let Some(redact_id) = &content.redacts { - !self.services.state_accessor.user_can_redact( - redact_id, - &incoming_pdu.sender, - &incoming_pdu.room_id, - true, - )? + !self + .services + .state_accessor + .user_can_redact(redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) + .await? } else { false } @@ -627,28 +638,52 @@ impl Service { // Now we calculate the set of extremities this room has after the incoming // event has been applied. We start with the previous extremities (aka leaves) trace!("Calculating extremities"); - let mut extremities = self.services.state.get_forward_extremities(room_id)?; - trace!("Calculated {} extremities", extremities.len()); + let mut extremities: HashSet<_> = self + .services + .state + .get_forward_extremities(room_id) + .map(ToOwned::to_owned) + .collect() + .await; // Remove any forward extremities that are referenced by this incoming event's // prev_events + trace!( + "Calculated {} extremities; checking against {} prev_events", + extremities.len(), + incoming_pdu.prev_events.len() + ); for prev_event in &incoming_pdu.prev_events { - extremities.remove(prev_event); + extremities.remove(&(**prev_event)); } // Only keep those extremities were not referenced yet - extremities.retain(|id| !matches!(self.services.pdu_metadata.is_event_referenced(room_id, id), Ok(true))); + let mut retained = HashSet::new(); + for id in &extremities { + if !self + .services + .pdu_metadata + .is_event_referenced(room_id, id) + .await + { + retained.insert(id.clone()); + } + } + + extremities.retain(|id| retained.contains(id)); debug!("Retained {} extremities. Compressing state", extremities.len()); - let state_ids_compressed = Arc::new( - state_at_incoming_event - .iter() - .map(|(shortstatekey, id)| { - self.services - .state_compressor - .compress_state_event(*shortstatekey, id) - }) - .collect::>()?, - ); + + let mut state_ids_compressed = HashSet::new(); + for (shortstatekey, id) in &state_at_incoming_event { + state_ids_compressed.insert( + self.services + .state_compressor + .compress_state_event(*shortstatekey, id) + .await, + ); + } + + let state_ids_compressed = Arc::new(state_ids_compressed); if incoming_pdu.state_key.is_some() { debug!("Event is a state-event. Deriving new room state"); @@ -659,9 +694,11 @@ impl Service { let shortstatekey = self .services .short - .get_or_create_shortstatekey(&incoming_pdu.kind.to_string().into(), state_key)?; + .get_or_create_shortstatekey(&incoming_pdu.kind.to_string().into(), state_key) + .await; - state_after.insert(shortstatekey, Arc::from(&*incoming_pdu.event_id)); + let event_id = &incoming_pdu.event_id; + state_after.insert(shortstatekey, event_id.clone()); } let new_room_state = self @@ -673,7 +710,8 @@ impl Service { let (sstatehash, new, removed) = self .services .state_compressor - .save_state(room_id, new_room_state)?; + .save_state(room_id, new_room_state) + .await?; self.services .state @@ -698,16 +736,16 @@ impl Service { .await?; // Soft fail, we keep the event as an outlier but don't add it to the timeline - warn!("Event was soft failed: {:?}", incoming_pdu); + warn!("Event was soft failed: {incoming_pdu:?}"); self.services .pdu_metadata - .mark_event_soft_failed(&incoming_pdu.event_id)?; + .mark_event_soft_failed(&incoming_pdu.event_id); return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event has been soft failed")); } trace!("Appending pdu to timeline"); - extremities.insert(incoming_pdu.event_id.clone()); + extremities.insert(incoming_pdu.event_id.clone().into()); // Now that the event has passed all auth it is added into the timeline. // We use the `state_at_event` instead of `state_after` so we accurately @@ -718,7 +756,7 @@ impl Service { .append_incoming_pdu( &incoming_pdu, val, - extremities.iter().map(|e| (**e).to_owned()).collect(), + extremities.into_iter().collect(), state_ids_compressed, soft_fail, &state_lock, @@ -742,8 +780,9 @@ impl Service { let current_sstatehash = self .services .state - .get_room_shortstatehash(room_id)? - .expect("every room has state"); + .get_room_shortstatehash(room_id) + .await + .map_err(|e| err!(Database(error!("No state for {room_id:?}: {e:?}"))))?; let current_state_ids = self .services @@ -752,7 +791,6 @@ impl Service { .await?; let fork_states = [current_state_ids, incoming_state]; - let mut auth_chain_sets = Vec::with_capacity(fork_states.len()); for state in &fork_states { auth_chain_sets.push( @@ -760,62 +798,59 @@ impl Service { .auth_chain .event_ids_iter(room_id, state.iter().map(|(_, id)| id.clone()).collect()) .await? - .collect(), + .collect::>>() + .await, ); } debug!("Loading fork states"); - let fork_states: Vec<_> = fork_states + let fork_states: Vec>> = fork_states .into_iter() - .map(|map| { - map.into_iter() + .stream() + .then(|fork_state| { + fork_state + .into_iter() + .stream() .filter_map(|(k, id)| { self.services .short .get_statekey_from_short(k) - .map(|(ty, st_key)| ((ty.to_string().into(), st_key), id)) - .ok() + .map_ok_or_else(|_| None, move |(ty, st_key)| Some(((ty, st_key), id))) }) - .collect::>() + .collect() }) - .collect(); - - let lock = self.services.globals.stateres_mutex.lock(); + .collect() + .boxed() + .await; debug!("Resolving state"); - let state_resolve = state_res::resolve(room_version_id, &fork_states, auth_chain_sets, |id| { - let res = self.services.timeline.get_pdu(id); - if let Err(e) = &res { - error!("Failed to fetch event: {}", e); - } - res.ok().flatten() - }); + let lock = self.services.globals.stateres_mutex.lock(); - let state = match state_resolve { - Ok(new_state) => new_state, - Err(e) => { - error!("State resolution failed: {}", e); - return Err(Error::bad_database( - "State resolution failed, either an event could not be found or deserialization", - )); - }, - }; + let event_fetch = |event_id| self.event_fetch(event_id); + let event_exists = |event_id| self.event_exists(event_id); + let state = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) + .await + .map_err(|e| err!(Database(error!("State resolution failed: {e:?}"))))?; drop(lock); debug!("State resolution done. Compressing state"); - let new_room_state = state - .into_iter() - .map(|((event_type, state_key), event_id)| { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&event_type.to_string().into(), &state_key)?; - self.services - .state_compressor - .compress_state_event(shortstatekey, &event_id) - }) - .collect::>()?; + let mut new_room_state = HashSet::new(); + for ((event_type, state_key), event_id) in state { + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&event_type.to_string().into(), &state_key) + .await; + + let compressed = self + .services + .state_compressor + .compress_state_event(shortstatekey, &event_id) + .await; + + new_room_state.insert(compressed); + } Ok(Arc::new(new_room_state)) } @@ -827,46 +862,47 @@ impl Service { &self, incoming_pdu: &Arc, ) -> Result>>> { let prev_event = &*incoming_pdu.prev_events[0]; - let prev_event_sstatehash = self + let Ok(prev_event_sstatehash) = self .services .state_accessor - .pdu_shortstatehash(prev_event)?; - - let state = if let Some(shortstatehash) = prev_event_sstatehash { - Some( - self.services - .state_accessor - .state_full_ids(shortstatehash) - .await, - ) - } else { - None + .pdu_shortstatehash(prev_event) + .await + else { + return Ok(None); }; - if let Some(Ok(mut state)) = state { - debug!("Using cached state"); - let prev_pdu = self + let Ok(mut state) = self + .services + .state_accessor + .state_full_ids(prev_event_sstatehash) + .await + .log_err() + else { + return Ok(None); + }; + + debug!("Using cached state"); + let prev_pdu = self + .services + .timeline + .get_pdu(prev_event) + .await + .map_err(|e| err!(Database("Could not find prev event, but we know the state: {e:?}")))?; + + if let Some(state_key) = &prev_pdu.state_key { + let shortstatekey = self .services - .timeline - .get_pdu(prev_event) - .ok() - .flatten() - .ok_or_else(|| Error::bad_database("Could not find prev event, but we know the state."))?; + .short + .get_or_create_shortstatekey(&prev_pdu.kind.to_string().into(), state_key) + .await; - if let Some(state_key) = &prev_pdu.state_key { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&prev_pdu.kind.to_string().into(), state_key)?; - - state.insert(shortstatekey, Arc::from(prev_event)); - // Now it's the state after the pdu - } - - return Ok(Some(state)); + state.insert(shortstatekey, Arc::from(prev_event)); + // Now it's the state after the pdu } - Ok(None) + debug_assert!(!state.is_empty(), "should be returning None for empty HashMap result"); + + Ok(Some(state)) } #[tracing::instrument(skip_all, name = "state")] @@ -878,15 +914,16 @@ impl Service { let mut okay = true; for prev_eventid in &incoming_pdu.prev_events { - let Ok(Some(prev_event)) = self.services.timeline.get_pdu(prev_eventid) else { + let Ok(prev_event) = self.services.timeline.get_pdu(prev_eventid).await else { okay = false; break; }; - let Ok(Some(sstatehash)) = self + let Ok(sstatehash) = self .services .state_accessor .pdu_shortstatehash(prev_eventid) + .await else { okay = false; break; @@ -901,20 +938,25 @@ impl Service { let mut fork_states = Vec::with_capacity(extremity_sstatehashes.len()); let mut auth_chain_sets = Vec::with_capacity(extremity_sstatehashes.len()); - for (sstatehash, prev_event) in extremity_sstatehashes { - let mut leaf_state: HashMap<_, _> = self + let Ok(mut leaf_state) = self .services .state_accessor .state_full_ids(sstatehash) - .await?; + .await + else { + continue; + }; if let Some(state_key) = &prev_event.state_key { let shortstatekey = self .services .short - .get_or_create_shortstatekey(&prev_event.kind.to_string().into(), state_key)?; - leaf_state.insert(shortstatekey, Arc::from(&*prev_event.event_id)); + .get_or_create_shortstatekey(&prev_event.kind.to_string().into(), state_key) + .await; + + let event_id = &prev_event.event_id; + leaf_state.insert(shortstatekey, event_id.clone()); // Now it's the state after the pdu } @@ -922,13 +964,18 @@ impl Service { let mut starting_events = Vec::with_capacity(leaf_state.len()); for (k, id) in leaf_state { - if let Ok((ty, st_key)) = self.services.short.get_statekey_from_short(k) { + if let Ok((ty, st_key)) = self + .services + .short + .get_statekey_from_short(k) + .await + .log_err() + { // FIXME: Undo .to_string().into() when StateMap // is updated to use StateEventType state.insert((ty.to_string().into(), st_key), id.clone()); - } else { - warn!("Failed to get_statekey_from_short."); } + starting_events.push(id); } @@ -937,43 +984,40 @@ impl Service { .auth_chain .event_ids_iter(room_id, starting_events) .await? - .collect(), + .collect() + .await, ); fork_states.push(state); } let lock = self.services.globals.stateres_mutex.lock(); - let result = state_res::resolve(room_version_id, &fork_states, auth_chain_sets, |id| { - let res = self.services.timeline.get_pdu(id); - if let Err(e) = &res { - error!("Failed to fetch event: {}", e); - } - res.ok().flatten() - }); + + let event_fetch = |event_id| self.event_fetch(event_id); + let event_exists = |event_id| self.event_exists(event_id); + let result = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) + .await + .map_err(|e| err!(Database(warn!(?e, "State resolution on prev events failed.")))); + drop(lock); - Ok(match result { - Ok(new_state) => Some( - new_state - .into_iter() - .map(|((event_type, state_key), event_id)| { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&event_type.to_string().into(), &state_key)?; - Ok((shortstatekey, event_id)) - }) - .collect::>()?, - ), - Err(e) => { - warn!( - "State resolution on prev events failed, either an event could not be found or deserialization: {}", - e - ); - None - }, - }) + let Ok(new_state) = result else { + return Ok(None); + }; + + new_state + .iter() + .stream() + .then(|((event_type, state_key), event_id)| { + self.services + .short + .get_or_create_shortstatekey(event_type, state_key) + .map(move |shortstatekey| (shortstatekey, event_id.clone())) + }) + .collect() + .map(Some) + .map(Ok) + .await } /// Call /state_ids to find out what the state at this pdu is. We trust the @@ -985,7 +1029,7 @@ impl Service { pub_key_map: &RwLock>>, event_id: &EventId, ) -> Result>>> { debug!("Fetching state ids"); - match self + let res = self .services .sending .send_federation_request( @@ -996,61 +1040,57 @@ impl Service { }, ) .await - { - Ok(res) => { - debug!("Fetching state events"); - let collect = res - .pdu_ids - .iter() - .map(|x| Arc::from(&**x)) - .collect::>(); + .inspect_err(|e| warn!("Fetching state for event failed: {e}"))?; - let state_vec = self - .fetch_and_handle_outliers(origin, &collect, create_event, room_id, room_version_id, pub_key_map) - .await; + debug!("Fetching state events"); + let collect = res + .pdu_ids + .iter() + .map(|x| Arc::from(&**x)) + .collect::>(); - let mut state: HashMap<_, Arc> = HashMap::with_capacity(state_vec.len()); - for (pdu, _) in state_vec { - let state_key = pdu - .state_key - .clone() - .ok_or_else(|| Error::bad_database("Found non-state pdu in state events."))?; + let state_vec = self + .fetch_and_handle_outliers(origin, &collect, create_event, room_id, room_version_id, pub_key_map) + .boxed() + .await; - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&pdu.kind.to_string().into(), &state_key)?; + let mut state: HashMap<_, Arc> = HashMap::with_capacity(state_vec.len()); + for (pdu, _) in state_vec { + let state_key = pdu + .state_key + .clone() + .ok_or_else(|| Error::bad_database("Found non-state pdu in state events."))?; - match state.entry(shortstatekey) { - hash_map::Entry::Vacant(v) => { - v.insert(Arc::from(&*pdu.event_id)); - }, - hash_map::Entry::Occupied(_) => { - return Err(Error::bad_database( - "State event's type and state_key combination exists multiple times.", - )) - }, - } - } + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&pdu.kind.to_string().into(), &state_key) + .await; - // The original create event must still be in the state - let create_shortstatekey = self - .services - .short - .get_shortstatekey(&StateEventType::RoomCreate, "")? - .expect("Room exists"); - - if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(&create_event.event_id) { - return Err(Error::bad_database("Incoming event refers to wrong create event.")); - } - - Ok(Some(state)) - }, - Err(e) => { - warn!("Fetching state for event failed: {}", e); - Err(e) - }, + match state.entry(shortstatekey) { + hash_map::Entry::Vacant(v) => { + v.insert(Arc::from(&*pdu.event_id)); + }, + hash_map::Entry::Occupied(_) => { + return Err(Error::bad_database( + "State event's type and state_key combination exists multiple times.", + )) + }, + } } + + // The original create event must still be in the state + let create_shortstatekey = self + .services + .short + .get_shortstatekey(&StateEventType::RoomCreate, "") + .await?; + + if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(&create_event.event_id) { + return Err!(Database("Incoming event refers to wrong create event.")); + } + + Ok(Some(state)) } /// Find the event and auth it. Once the event is validated (steps 1 - 8) @@ -1062,191 +1102,196 @@ impl Service { /// b. Look at outlier pdu tree /// c. Ask origin server over federation /// d. TODO: Ask other servers over federation? - pub fn fetch_and_handle_outliers<'a>( - &'a self, origin: &'a ServerName, events: &'a [Arc], create_event: &'a PduEvent, room_id: &'a RoomId, + pub async fn fetch_and_handle_outliers<'a>( + &self, origin: &'a ServerName, events: &'a [Arc], create_event: &'a PduEvent, room_id: &'a RoomId, room_version_id: &'a RoomVersionId, pub_key_map: &'a RwLock>>, - ) -> AsyncRecursiveCanonicalJsonVec<'a> { - Box::pin(async move { - let back_off = |id| async { - match self + ) -> Vec<(Arc, Option>)> { + let back_off = |id| match self + .services + .globals + .bad_event_ratelimiter + .write() + .expect("locked") + .entry(id) + { + hash_map::Entry::Vacant(e) => { + e.insert((Instant::now(), 1)); + }, + hash_map::Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)), + }; + + let mut events_with_auth_events = Vec::with_capacity(events.len()); + for id in events { + // a. Look in the main timeline (pduid_pdu tree) + // b. Look at outlier pdu tree + // (get_pdu_json checks both) + if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { + trace!("Found {id} in db"); + events_with_auth_events.push((id, Some(local_pdu), vec![])); + continue; + } + + // c. Ask origin server over federation + // We also handle its auth chain here so we don't get a stack overflow in + // handle_outlier_pdu. + let mut todo_auth_events = vec![Arc::clone(id)]; + let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); + let mut events_all = HashSet::with_capacity(todo_auth_events.len()); + let mut i: u64 = 0; + while let Some(next_id) = todo_auth_events.pop() { + if let Some((time, tries)) = self .services .globals .bad_event_ratelimiter - .write() + .read() .expect("locked") - .entry(id) + .get(&*next_id) { - hash_map::Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - hash_map::Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)), + // Exponential backoff + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + info!("Backing off from {next_id}"); + continue; + } } - }; - let mut events_with_auth_events = Vec::with_capacity(events.len()); - for id in events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Ok(Some(local_pdu)) = self.services.timeline.get_pdu(id) { - trace!("Found {} in db", id); - events_with_auth_events.push((id, Some(local_pdu), vec![])); + if events_all.contains(&next_id) { continue; } - // c. Ask origin server over federation - // We also handle its auth chain here so we don't get a stack overflow in - // handle_outlier_pdu. - let mut todo_auth_events = vec![Arc::clone(id)]; - let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); - let mut events_all = HashSet::with_capacity(todo_auth_events.len()); - let mut i: u64 = 0; - while let Some(next_id) = todo_auth_events.pop() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .expect("locked") - .get(&*next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { - info!("Backing off from {next_id}"); + i = i.saturating_add(1); + if i % 100 == 0 { + tokio::task::yield_now().await; + } + + if self.services.timeline.get_pdu(&next_id).await.is_ok() { + trace!("Found {next_id} in db"); + continue; + } + + debug!("Fetching {next_id} over federation."); + match self + .services + .sending + .send_federation_request( + origin, + get_event::v1::Request { + event_id: (*next_id).to_owned(), + }, + ) + .await + { + Ok(res) => { + debug!("Got {next_id} over federation"); + let Ok((calculated_event_id, value)) = + pdu::gen_event_id_canonical_json(&res.pdu, room_version_id) + else { + back_off((*next_id).to_owned()); continue; + }; + + if calculated_event_id != *next_id { + warn!( + "Server didn't return event id we requested: requested: {next_id}, we got \ + {calculated_event_id}. Event: {:?}", + &res.pdu + ); } - } - if events_all.contains(&next_id) { - continue; - } - - i = i.saturating_add(1); - if i % 100 == 0 { - tokio::task::yield_now().await; - } - - if let Ok(Some(_)) = self.services.timeline.get_pdu(&next_id) { - trace!("Found {} in db", next_id); - continue; - } - - debug!("Fetching {} over federation.", next_id); - match self - .services - .sending - .send_federation_request( - origin, - get_event::v1::Request { - event_id: (*next_id).to_owned(), - }, - ) - .await - { - Ok(res) => { - debug!("Got {} over federation", next_id); - let Ok((calculated_event_id, value)) = - pdu::gen_event_id_canonical_json(&res.pdu, room_version_id) - else { - back_off((*next_id).to_owned()).await; - continue; - }; - - if calculated_event_id != *next_id { - warn!( - "Server didn't return event id we requested: requested: {}, we got {}. Event: {:?}", - next_id, calculated_event_id, &res.pdu - ); - } - - if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) { - for auth_event in auth_events { - if let Ok(auth_event) = serde_json::from_value(auth_event.clone().into()) { - let a: Arc = auth_event; - todo_auth_events.push(a); - } else { - warn!("Auth event id is not valid"); - } + if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) { + for auth_event in auth_events { + if let Ok(auth_event) = serde_json::from_value(auth_event.clone().into()) { + let a: Arc = auth_event; + todo_auth_events.push(a); + } else { + warn!("Auth event id is not valid"); } - } else { - warn!("Auth event list invalid"); } - - events_in_reverse_order.push((next_id.clone(), value)); - events_all.insert(next_id); - }, - Err(e) => { - debug_error!("Failed to fetch event {next_id}: {e}"); - back_off((*next_id).to_owned()).await; - }, - } - } - events_with_auth_events.push((id, None, events_in_reverse_order)); - } - - // We go through all the signatures we see on the PDUs and their unresolved - // dependencies and fetch the corresponding signing keys - self.services - .server_keys - .fetch_required_signing_keys( - events_with_auth_events - .iter() - .flat_map(|(_id, _local_pdu, events)| events) - .map(|(_event_id, event)| event), - pub_key_map, - ) - .await - .unwrap_or_else(|e| { - warn!("Could not fetch all signatures for PDUs from {}: {:?}", origin, e); - }); - - let mut pdus = Vec::with_capacity(events_with_auth_events.len()); - for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Some(local_pdu) = local_pdu { - trace!("Found {} in db", id); - pdus.push((local_pdu, None)); - } - for (next_id, value) in events_in_reverse_order.iter().rev() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .expect("locked") - .get(&**next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { - debug!("Backing off from {next_id}"); - continue; + } else { + warn!("Auth event list invalid"); } - } - match self - .handle_outlier_pdu(origin, create_event, next_id, room_id, value.clone(), true, pub_key_map) - .await - { - Ok((pdu, json)) => { - if next_id == id { - pdus.push((pdu, Some(json))); - } - }, - Err(e) => { - warn!("Authentication of event {} failed: {:?}", next_id, e); - back_off((**next_id).to_owned()).await; - }, - } + events_in_reverse_order.push((next_id.clone(), value)); + events_all.insert(next_id); + }, + Err(e) => { + debug_error!("Failed to fetch event {next_id}: {e}"); + back_off((*next_id).to_owned()); + }, } } - pdus - }) + events_with_auth_events.push((id, None, events_in_reverse_order)); + } + + // We go through all the signatures we see on the PDUs and their unresolved + // dependencies and fetch the corresponding signing keys + self.services + .server_keys + .fetch_required_signing_keys( + events_with_auth_events + .iter() + .flat_map(|(_id, _local_pdu, events)| events) + .map(|(_event_id, event)| event), + pub_key_map, + ) + .await + .unwrap_or_else(|e| { + warn!("Could not fetch all signatures for PDUs from {origin}: {e:?}"); + }); + + let mut pdus = Vec::with_capacity(events_with_auth_events.len()); + for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { + // a. Look in the main timeline (pduid_pdu tree) + // b. Look at outlier pdu tree + // (get_pdu_json checks both) + if let Some(local_pdu) = local_pdu { + trace!("Found {id} in db"); + pdus.push((local_pdu.clone(), None)); + } + + for (next_id, value) in events_in_reverse_order.into_iter().rev() { + if let Some((time, tries)) = self + .services + .globals + .bad_event_ratelimiter + .read() + .expect("locked") + .get(&*next_id) + { + // Exponential backoff + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + debug!("Backing off from {next_id}"); + continue; + } + } + + match Box::pin(self.handle_outlier_pdu( + origin, + create_event, + &next_id, + room_id, + value.clone(), + true, + pub_key_map, + )) + .await + { + Ok((pdu, json)) => { + if next_id == *id { + pdus.push((pdu, Some(json))); + } + }, + Err(e) => { + warn!("Authentication of event {next_id} failed: {e:?}"); + back_off(next_id.into()); + }, + } + } + } + pdus } #[allow(clippy::type_complexity)] @@ -1262,16 +1307,12 @@ impl Service { let mut eventid_info = HashMap::new(); let mut todo_outlier_stack: Vec> = initial_set; - let first_pdu_in_room = self - .services - .timeline - .first_pdu_in_room(room_id)? - .ok_or_else(|| Error::bad_database("Failed to find first pdu in db."))?; + let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; let mut amount = 0; while let Some(prev_event_id) = todo_outlier_stack.pop() { - if let Some((pdu, json_opt)) = self + if let Some((pdu, mut json_opt)) = self .fetch_and_handle_outliers( origin, &[prev_event_id.clone()], @@ -1280,28 +1321,29 @@ impl Service { room_version_id, pub_key_map, ) + .boxed() .await .pop() { Self::check_room_id(room_id, &pdu)?; - if amount > self.services.globals.max_fetch_prev_events() { - // Max limit reached - debug!( - "Max prev event limit reached! Limit: {}", - self.services.globals.max_fetch_prev_events() - ); + let limit = self.services.globals.max_fetch_prev_events(); + if amount > limit { + debug_warn!("Max prev event limit reached! Limit: {limit}"); graph.insert(prev_event_id.clone(), HashSet::new()); continue; } - if let Some(json) = json_opt.or_else(|| { - self.services + if json_opt.is_none() { + json_opt = self + .services .outlier .get_outlier_pdu_json(&prev_event_id) - .ok() - .flatten() - }) { + .await + .ok(); + } + + if let Some(json) = json_opt { if pdu.origin_server_ts > first_pdu_in_room.origin_server_ts { amount = amount.saturating_add(1); for prev_prev in &pdu.prev_events { @@ -1327,56 +1369,42 @@ impl Service { } } - let sorted = state_res::lexicographical_topological_sort(&graph, |event_id| { + let event_fetch = |event_id| { + let origin_server_ts = eventid_info + .get(&event_id) + .cloned() + .map_or_else(|| uint!(0), |info| info.0.origin_server_ts); + // This return value is the key used for sorting events, // events are then sorted by power level, time, // and lexically by event_id. - Ok(( - int!(0), - MilliSecondsSinceUnixEpoch( - eventid_info - .get(event_id) - .map_or_else(|| uint!(0), |info| info.0.origin_server_ts), - ), - )) - }) - .map_err(|e| { - error!("Error sorting prev events: {e}"); - Error::bad_database("Error sorting prev events") - })?; + future::ok((int!(0), MilliSecondsSinceUnixEpoch(origin_server_ts))) + }; + + let sorted = state_res::lexicographical_topological_sort(&graph, &event_fetch) + .await + .map_err(|e| err!(Database(error!("Error sorting prev events: {e}"))))?; Ok((sorted, eventid_info)) } /// Returns Ok if the acl allows the server #[tracing::instrument(skip_all)] - pub fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result<()> { - let acl_event = if let Some(acl) = - self.services - .state_accessor - .room_state_get(room_id, &StateEventType::RoomServerAcl, "")? - { - trace!("ACL event found: {acl:?}"); - acl - } else { - trace!("No ACL event found"); + pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result<()> { + let Ok(acl_event_content) = self + .services + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomServerAcl, "") + .await + .map(|c: RoomServerAclEventContent| c) + .inspect(|acl| trace!("ACL content found: {acl:?}")) + .inspect_err(|e| trace!("No ACL content found: {e:?}")) + else { return Ok(()); }; - let acl_event_content: RoomServerAclEventContent = match serde_json::from_str(acl_event.content.get()) { - Ok(content) => { - trace!("Found ACL event contents: {content:?}"); - content - }, - Err(e) => { - warn!("Invalid ACL event: {e}"); - return Ok(()); - }, - }; - if acl_event_content.allow.is_empty() { warn!("Ignoring broken ACL event (allow key is empty)"); - // Ignore broken acl events return Ok(()); } @@ -1384,16 +1412,18 @@ impl Service { trace!("server {server_name} is allowed by ACL"); Ok(()) } else { - debug!("Server {} was denied by room ACL in {}", server_name, room_id); - Err(Error::BadRequest(ErrorKind::forbidden(), "Server was denied by room ACL")) + debug!("Server {server_name} was denied by room ACL in {room_id}"); + Err!(Request(Forbidden("Server was denied by room ACL"))) } } fn check_room_id(room_id: &RoomId, pdu: &PduEvent) -> Result<()> { if pdu.room_id != room_id { - warn!("Found event from room {} in room {}", pdu.room_id, room_id); - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event has wrong room id")); + return Err!(Request(InvalidParam( + warn!(pdu_event_id = ?pdu.event_id, pdu_room_id = ?pdu.room_id, ?room_id, "Found event from room in room") + ))); } + Ok(()) } @@ -1408,4 +1438,10 @@ impl Service { fn to_room_version(room_version_id: &RoomVersionId) -> RoomVersion { RoomVersion::new(room_version_id).expect("room version is supported") } + + async fn event_exists(&self, event_id: Arc) -> bool { self.services.timeline.pdu_exists(&event_id).await } + + async fn event_fetch(&self, event_id: Arc) -> Option> { + self.services.timeline.get_pdu(&event_id).await.ok() + } } diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index a7ffe193..2de3e28e 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -3,7 +3,9 @@ use ruma::{CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId}; use serde_json::value::RawValue as RawJsonValue; impl super::Service { - pub fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { + pub async fn parse_incoming_pdu( + &self, pdu: &RawJsonValue, + ) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { debug_warn!("Error parsing incoming event {pdu:#?}"); err!(BadServerResponse("Error parsing incoming event {e:?}")) @@ -14,7 +16,7 @@ impl super::Service { .and_then(|id| RoomId::parse(id.as_str()?).ok()) .ok_or(err!(Request(InvalidParam("Invalid room id in pdu"))))?; - let Ok(room_version_id) = self.services.state.get_room_version(&room_id) else { + let Ok(room_version_id) = self.services.state.get_room_version(&room_id).await else { return Err!("Server is not in room {room_id}"); }; diff --git a/src/service/rooms/lazy_loading/data.rs b/src/service/rooms/lazy_loading/data.rs deleted file mode 100644 index 073d45f5..00000000 --- a/src/service/rooms/lazy_loading/data.rs +++ /dev/null @@ -1,65 +0,0 @@ -use std::sync::Arc; - -use conduit::Result; -use database::{Database, Map}; -use ruma::{DeviceId, RoomId, UserId}; - -pub(super) struct Data { - lazyloadedids: Arc, -} - -impl Data { - pub(super) fn new(db: &Arc) -> Self { - Self { - lazyloadedids: db["lazyloadedids"].clone(), - } - } - - pub(super) fn lazy_load_was_sent_before( - &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, - ) -> Result { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(device_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(room_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(ll_user.as_bytes()); - Ok(self.lazyloadedids.get(&key)?.is_some()) - } - - pub(super) fn lazy_load_confirm_delivery( - &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, - confirmed_user_ids: &mut dyn Iterator, - ) -> Result<()> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(device_id.as_bytes()); - prefix.push(0xFF); - prefix.extend_from_slice(room_id.as_bytes()); - prefix.push(0xFF); - - for ll_id in confirmed_user_ids { - let mut key = prefix.clone(); - key.extend_from_slice(ll_id.as_bytes()); - self.lazyloadedids.insert(&key, &[])?; - } - - Ok(()) - } - - pub(super) fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId) -> Result<()> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(device_id.as_bytes()); - prefix.push(0xFF); - prefix.extend_from_slice(room_id.as_bytes()); - prefix.push(0xFF); - - for (key, _) in self.lazyloadedids.scan_prefix(prefix) { - self.lazyloadedids.remove(&key)?; - } - - Ok(()) - } -} diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 0a9d4cf2..e0816d3f 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -1,21 +1,26 @@ -mod data; - use std::{ collections::{HashMap, HashSet}, fmt::Write, sync::{Arc, Mutex}, }; -use conduit::{PduCount, Result}; +use conduit::{ + implement, + utils::{stream::TryIgnore, ReadyExt}, + PduCount, Result, +}; +use database::{Interfix, Map}; use ruma::{DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, RoomId, UserId}; -use self::data::Data; - pub struct Service { - pub lazy_load_waiting: Mutex, + lazy_load_waiting: Mutex, db: Data, } +struct Data { + lazyloadedids: Arc, +} + type LazyLoadWaiting = HashMap; type LazyLoadWaitingKey = (OwnedUserId, OwnedDeviceId, OwnedRoomId, PduCount); type LazyLoadWaitingVal = HashSet; @@ -23,8 +28,10 @@ type LazyLoadWaitingVal = HashSet; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - lazy_load_waiting: Mutex::new(HashMap::new()), - db: Data::new(args.db), + lazy_load_waiting: LazyLoadWaiting::new().into(), + db: Data { + lazyloadedids: args.db["lazyloadedids"].clone(), + }, })) } @@ -40,47 +47,60 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - #[tracing::instrument(skip(self), level = "debug")] - pub fn lazy_load_was_sent_before( - &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, - ) -> Result { - self.db - .lazy_load_was_sent_before(user_id, device_id, room_id, ll_user) - } +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +#[inline] +pub async fn lazy_load_was_sent_before( + &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, +) -> bool { + let key = (user_id, device_id, room_id, ll_user); + self.db.lazyloadedids.qry(&key).await.is_ok() +} - #[tracing::instrument(skip(self), level = "debug")] - pub async fn lazy_load_mark_sent( - &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, lazy_load: HashSet, - count: PduCount, - ) { - self.lazy_load_waiting - .lock() - .expect("locked") - .insert((user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), count), lazy_load); - } +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn lazy_load_mark_sent( + &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, lazy_load: HashSet, count: PduCount, +) { + let key = (user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), count); - #[tracing::instrument(skip(self), level = "debug")] - pub async fn lazy_load_confirm_delivery( - &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, since: PduCount, - ) -> Result<()> { - if let Some(user_ids) = self.lazy_load_waiting.lock().expect("locked").remove(&( - user_id.to_owned(), - device_id.to_owned(), - room_id.to_owned(), - since, - )) { - self.db - .lazy_load_confirm_delivery(user_id, device_id, room_id, &mut user_ids.iter().map(|u| &**u))?; - } else { - // Ignore - } + self.lazy_load_waiting + .lock() + .expect("locked") + .insert(key, lazy_load); +} - Ok(()) - } +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn lazy_load_confirm_delivery(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, since: PduCount) { + let key = (user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), since); - #[tracing::instrument(skip(self), level = "debug")] - pub fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId) -> Result<()> { - self.db.lazy_load_reset(user_id, device_id, room_id) + let Some(user_ids) = self.lazy_load_waiting.lock().expect("locked").remove(&key) else { + return; + }; + + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xFF); + prefix.extend_from_slice(device_id.as_bytes()); + prefix.push(0xFF); + prefix.extend_from_slice(room_id.as_bytes()); + prefix.push(0xFF); + + for ll_id in &user_ids { + let mut key = prefix.clone(); + key.extend_from_slice(ll_id.as_bytes()); + self.db.lazyloadedids.insert(&key, &[]); } } + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub async fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId) { + let prefix = (user_id, device_id, room_id, Interfix); + self.db + .lazyloadedids + .keys_raw_prefix(&prefix) + .ignore_err() + .ready_for_each(|key| self.db.lazyloadedids.remove(key)) + .await; +} diff --git a/src/service/rooms/metadata/data.rs b/src/service/rooms/metadata/data.rs deleted file mode 100644 index efe681b1..00000000 --- a/src/service/rooms/metadata/data.rs +++ /dev/null @@ -1,110 +0,0 @@ -use std::sync::Arc; - -use conduit::{error, utils, Error, Result}; -use database::Map; -use ruma::{OwnedRoomId, RoomId}; - -use crate::{rooms, Dep}; - -pub(super) struct Data { - disabledroomids: Arc, - bannedroomids: Arc, - roomid_shortroomid: Arc, - pduid_pdu: Arc, - services: Services, -} - -struct Services { - short: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - disabledroomids: db["disabledroomids"].clone(), - bannedroomids: db["bannedroomids"].clone(), - roomid_shortroomid: db["roomid_shortroomid"].clone(), - pduid_pdu: db["pduid_pdu"].clone(), - services: Services { - short: args.depend::("rooms::short"), - }, - } - } - - pub(super) fn exists(&self, room_id: &RoomId) -> Result { - let prefix = match self.services.short.get_shortroomid(room_id)? { - Some(b) => b.to_be_bytes().to_vec(), - None => return Ok(false), - }; - - // Look for PDUs in that room. - Ok(self - .pduid_pdu - .iter_from(&prefix, false) - .next() - .filter(|(k, _)| k.starts_with(&prefix)) - .is_some()) - } - - pub(super) fn iter_ids<'a>(&'a self) -> Box> + 'a> { - Box::new(self.roomid_shortroomid.iter().map(|(bytes, _)| { - RoomId::parse( - utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Room ID in publicroomids is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("Room ID in roomid_shortroomid is invalid.")) - })) - } - - #[inline] - pub(super) fn is_disabled(&self, room_id: &RoomId) -> Result { - Ok(self.disabledroomids.get(room_id.as_bytes())?.is_some()) - } - - #[inline] - pub(super) fn disable_room(&self, room_id: &RoomId, disabled: bool) -> Result<()> { - if disabled { - self.disabledroomids.insert(room_id.as_bytes(), &[])?; - } else { - self.disabledroomids.remove(room_id.as_bytes())?; - } - - Ok(()) - } - - #[inline] - pub(super) fn is_banned(&self, room_id: &RoomId) -> Result { - Ok(self.bannedroomids.get(room_id.as_bytes())?.is_some()) - } - - #[inline] - pub(super) fn ban_room(&self, room_id: &RoomId, banned: bool) -> Result<()> { - if banned { - self.bannedroomids.insert(room_id.as_bytes(), &[])?; - } else { - self.bannedroomids.remove(room_id.as_bytes())?; - } - - Ok(()) - } - - pub(super) fn list_banned_rooms<'a>(&'a self) -> Box> + 'a> { - Box::new(self.bannedroomids.iter().map( - |(room_id_bytes, _ /* non-banned rooms should not be in this table */)| { - let room_id = utils::string_from_bytes(&room_id_bytes) - .map_err(|e| { - error!("Invalid room_id bytes in bannedroomids: {e}"); - Error::bad_database("Invalid room_id in bannedroomids.") - })? - .try_into() - .map_err(|e| { - error!("Invalid room_id in bannedroomids: {e}"); - Error::bad_database("Invalid room_id in bannedroomids") - })?; - - Ok(room_id) - }, - )) - } -} diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 7415c53b..5d4a47c7 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -1,51 +1,92 @@ -mod data; - use std::sync::Arc; -use conduit::Result; -use ruma::{OwnedRoomId, RoomId}; +use conduit::{implement, utils::stream::TryIgnore, Result}; +use database::Map; +use futures::{Stream, StreamExt}; +use ruma::RoomId; -use self::data::Data; +use crate::{rooms, Dep}; pub struct Service { db: Data, + services: Services, +} + +struct Data { + disabledroomids: Arc, + bannedroomids: Arc, + roomid_shortroomid: Arc, + pduid_pdu: Arc, +} + +struct Services { + short: Dep, } impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(&args), + db: Data { + disabledroomids: args.db["disabledroomids"].clone(), + bannedroomids: args.db["bannedroomids"].clone(), + roomid_shortroomid: args.db["roomid_shortroomid"].clone(), + pduid_pdu: args.db["pduid_pdu"].clone(), + }, + services: Services { + short: args.depend::("rooms::short"), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - /// Checks if a room exists. - #[inline] - pub fn exists(&self, room_id: &RoomId) -> Result { self.db.exists(room_id) } +#[implement(Service)] +pub async fn exists(&self, room_id: &RoomId) -> bool { + let Ok(prefix) = self.services.short.get_shortroomid(room_id).await else { + return false; + }; - #[must_use] - pub fn iter_ids<'a>(&'a self) -> Box> + 'a> { self.db.iter_ids() } + // Look for PDUs in that room. + self.db + .pduid_pdu + .keys_raw_prefix(&prefix) + .ignore_err() + .next() + .await + .is_some() +} - #[inline] - pub fn is_disabled(&self, room_id: &RoomId) -> Result { self.db.is_disabled(room_id) } +#[implement(Service)] +pub fn iter_ids(&self) -> impl Stream + Send + '_ { self.db.roomid_shortroomid.keys().ignore_err() } - #[inline] - pub fn disable_room(&self, room_id: &RoomId, disabled: bool) -> Result<()> { - self.db.disable_room(room_id, disabled) - } - - #[inline] - pub fn is_banned(&self, room_id: &RoomId) -> Result { self.db.is_banned(room_id) } - - #[inline] - pub fn ban_room(&self, room_id: &RoomId, banned: bool) -> Result<()> { self.db.ban_room(room_id, banned) } - - #[inline] - #[must_use] - pub fn list_banned_rooms<'a>(&'a self) -> Box> + 'a> { - self.db.list_banned_rooms() +#[implement(Service)] +#[inline] +pub fn disable_room(&self, room_id: &RoomId, disabled: bool) { + if disabled { + self.db.disabledroomids.insert(room_id.as_bytes(), &[]); + } else { + self.db.disabledroomids.remove(room_id.as_bytes()); } } + +#[implement(Service)] +#[inline] +pub fn ban_room(&self, room_id: &RoomId, banned: bool) { + if banned { + self.db.bannedroomids.insert(room_id.as_bytes(), &[]); + } else { + self.db.bannedroomids.remove(room_id.as_bytes()); + } +} + +#[implement(Service)] +pub fn list_banned_rooms(&self) -> impl Stream + Send + '_ { self.db.bannedroomids.keys().ignore_err() } + +#[implement(Service)] +#[inline] +pub async fn is_disabled(&self, room_id: &RoomId) -> bool { self.db.disabledroomids.qry(room_id).await.is_ok() } + +#[implement(Service)] +#[inline] +pub async fn is_banned(&self, room_id: &RoomId) -> bool { self.db.bannedroomids.qry(room_id).await.is_ok() } diff --git a/src/service/rooms/outlier/data.rs b/src/service/rooms/outlier/data.rs deleted file mode 100644 index aa804721..00000000 --- a/src/service/rooms/outlier/data.rs +++ /dev/null @@ -1,42 +0,0 @@ -use std::sync::Arc; - -use conduit::{Error, Result}; -use database::{Database, Map}; -use ruma::{CanonicalJsonObject, EventId}; - -use crate::PduEvent; - -pub(super) struct Data { - eventid_outlierpdu: Arc, -} - -impl Data { - pub(super) fn new(db: &Arc) -> Self { - Self { - eventid_outlierpdu: db["eventid_outlierpdu"].clone(), - } - } - - pub(super) fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result> { - self.eventid_outlierpdu - .get(event_id.as_bytes())? - .map_or(Ok(None), |pdu| { - serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db.")) - }) - } - - pub(super) fn get_outlier_pdu(&self, event_id: &EventId) -> Result> { - self.eventid_outlierpdu - .get(event_id.as_bytes())? - .map_or(Ok(None), |pdu| { - serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db.")) - }) - } - - pub(super) fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) -> Result<()> { - self.eventid_outlierpdu.insert( - event_id.as_bytes(), - &serde_json::to_vec(&pdu).expect("CanonicalJsonObject is valid"), - ) - } -} diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 22bd2092..277b5982 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -1,9 +1,7 @@ -mod data; - use std::sync::Arc; -use conduit::Result; -use data::Data; +use conduit::{implement, Result}; +use database::{Deserialized, Map}; use ruma::{CanonicalJsonObject, EventId}; use crate::PduEvent; @@ -12,31 +10,48 @@ pub struct Service { db: Data, } +struct Data { + eventid_outlierpdu: Arc, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data { + eventid_outlierpdu: args.db["eventid_outlierpdu"].clone(), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - /// Returns the pdu from the outlier tree. - pub fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result> { - self.db.get_outlier_pdu_json(event_id) - } - - /// Returns the pdu from the outlier tree. - /// - /// TODO: use this? - #[allow(dead_code)] - pub fn get_pdu_outlier(&self, event_id: &EventId) -> Result> { self.db.get_outlier_pdu(event_id) } - - /// Append the PDU as an outlier. - #[tracing::instrument(skip(self, pdu), level = "debug")] - pub fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) -> Result<()> { - self.db.add_pdu_outlier(event_id, pdu) - } +/// Returns the pdu from the outlier tree. +#[implement(Service)] +pub async fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result { + self.db + .eventid_outlierpdu + .qry(event_id) + .await + .deserialized_json() +} + +/// Returns the pdu from the outlier tree. +#[implement(Service)] +pub async fn get_pdu_outlier(&self, event_id: &EventId) -> Result { + self.db + .eventid_outlierpdu + .qry(event_id) + .await + .deserialized_json() +} + +/// Append the PDU as an outlier. +#[implement(Service)] +#[tracing::instrument(skip(self, pdu), level = "debug")] +pub fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) { + self.db.eventid_outlierpdu.insert( + event_id.as_bytes(), + &serde_json::to_vec(&pdu).expect("CanonicalJsonObject is valid"), + ); } diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index d1649da8..f2323475 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,7 +1,13 @@ use std::{mem::size_of, sync::Arc}; -use conduit::{utils, Error, PduCount, PduEvent, Result}; +use conduit::{ + result::LogErr, + utils, + utils::{stream::TryIgnore, ReadyExt}, + PduCount, PduEvent, +}; use database::Map; +use futures::{Stream, StreamExt}; use ruma::{EventId, RoomId, UserId}; use crate::{rooms, Dep}; @@ -17,8 +23,7 @@ struct Services { timeline: Dep, } -type PdusIterItem = Result<(PduCount, PduEvent)>; -type PdusIterator<'a> = Box + 'a>; +pub(super) type PdusIterItem = (PduCount, PduEvent); impl Data { pub(super) fn new(args: &crate::Args<'_>) -> Self { @@ -33,19 +38,17 @@ impl Data { } } - pub(super) fn add_relation(&self, from: u64, to: u64) -> Result<()> { + pub(super) fn add_relation(&self, from: u64, to: u64) { let mut key = to.to_be_bytes().to_vec(); key.extend_from_slice(&from.to_be_bytes()); - self.tofrom_relation.insert(&key, &[])?; - Ok(()) + self.tofrom_relation.insert(&key, &[]); } pub(super) fn relations_until<'a>( &'a self, user_id: &'a UserId, shortroomid: u64, target: u64, until: PduCount, - ) -> Result> { + ) -> impl Stream + Send + 'a + '_ { let prefix = target.to_be_bytes().to_vec(); let mut current = prefix.clone(); - let count_raw = match until { PduCount::Normal(x) => x.saturating_sub(1), PduCount::Backfilled(x) => { @@ -55,53 +58,42 @@ impl Data { }; current.extend_from_slice(&count_raw.to_be_bytes()); - Ok(Box::new( - self.tofrom_relation - .iter_from(¤t, true) - .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(tofrom, _data)| { - let from = utils::u64_from_bytes(&tofrom[(size_of::())..]) - .map_err(|_| Error::bad_database("Invalid count in tofrom_relation."))?; + self.tofrom_relation + .rev_raw_keys_from(¤t) + .ignore_err() + .ready_take_while(move |key| key.starts_with(&prefix)) + .map(|to_from| utils::u64_from_u8(&to_from[(size_of::())..])) + .filter_map(move |from| async move { + let mut pduid = shortroomid.to_be_bytes().to_vec(); + pduid.extend_from_slice(&from.to_be_bytes()); + let mut pdu = self.services.timeline.get_pdu_from_id(&pduid).await.ok()?; - let mut pduid = shortroomid.to_be_bytes().to_vec(); - pduid.extend_from_slice(&from.to_be_bytes()); + if pdu.sender != user_id { + pdu.remove_transaction_id().log_err().ok(); + } - let mut pdu = self - .services - .timeline - .get_pdu_from_id(&pduid)? - .ok_or_else(|| Error::bad_database("Pdu in tofrom_relation is invalid."))?; - if pdu.sender != user_id { - pdu.remove_transaction_id()?; - } - Ok((PduCount::Normal(from), pdu)) - }), - )) + Some((PduCount::Normal(from), pdu)) + }) } - pub(super) fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) -> Result<()> { + pub(super) fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) { for prev in event_ids { let mut key = room_id.as_bytes().to_vec(); key.extend_from_slice(prev.as_bytes()); - self.referencedevents.insert(&key, &[])?; + self.referencedevents.insert(&key, &[]); } - - Ok(()) } - pub(super) fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> Result { - let mut key = room_id.as_bytes().to_vec(); - key.extend_from_slice(event_id.as_bytes()); - Ok(self.referencedevents.get(&key)?.is_some()) + pub(super) async fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> bool { + let key = (room_id, event_id); + self.referencedevents.qry(&key).await.is_ok() } - pub(super) fn mark_event_soft_failed(&self, event_id: &EventId) -> Result<()> { - self.softfailedeventids.insert(event_id.as_bytes(), &[]) + pub(super) fn mark_event_soft_failed(&self, event_id: &EventId) { + self.softfailedeventids.insert(event_id.as_bytes(), &[]); } - pub(super) fn is_event_soft_failed(&self, event_id: &EventId) -> Result { - self.softfailedeventids - .get(event_id.as_bytes()) - .map(|o| o.is_some()) + pub(super) async fn is_event_soft_failed(&self, event_id: &EventId) -> bool { + self.softfailedeventids.qry(event_id).await.is_ok() } } diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index d9eaf324..dbaebfbf 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -1,8 +1,8 @@ mod data; - use std::sync::Arc; -use conduit::{PduCount, PduEvent, Result}; +use conduit::{utils::stream::IterStream, PduCount, Result}; +use futures::StreamExt; use ruma::{ api::{client::relations::get_relating_events, Direction}, events::{relation::RelationType, TimelineEventType}, @@ -10,7 +10,7 @@ use ruma::{ }; use serde::Deserialize; -use self::data::Data; +use self::data::{Data, PdusIterItem}; use crate::{rooms, Dep}; pub struct Service { @@ -51,21 +51,19 @@ impl crate::Service for Service { impl Service { #[tracing::instrument(skip(self, from, to), level = "debug")] - pub fn add_relation(&self, from: PduCount, to: PduCount) -> Result<()> { + pub fn add_relation(&self, from: PduCount, to: PduCount) { match (from, to) { (PduCount::Normal(f), PduCount::Normal(t)) => self.db.add_relation(f, t), _ => { // TODO: Relations with backfilled pdus - - Ok(()) }, } } #[allow(clippy::too_many_arguments)] - pub fn paginate_relations_with_filter( - &self, sender_user: &UserId, room_id: &RoomId, target: &EventId, filter_event_type: &Option, - filter_rel_type: &Option, from: &Option, to: &Option, limit: &Option, + pub async fn paginate_relations_with_filter( + &self, sender_user: &UserId, room_id: &RoomId, target: &EventId, filter_event_type: Option, + filter_rel_type: Option, from: Option<&String>, to: Option<&String>, limit: Option, recurse: bool, dir: Direction, ) -> Result { let from = match from { @@ -76,7 +74,7 @@ impl Service { }, }; - let to = to.as_ref().and_then(|t| PduCount::try_from_string(t).ok()); + let to = to.and_then(|t| PduCount::try_from_string(t).ok()); // Use limit or else 10, with maximum 100 let limit = limit @@ -92,30 +90,32 @@ impl Service { 1 }; - let relations_until = &self.relations_until(sender_user, room_id, target, from, depth)?; - let events: Vec<_> = relations_until // TODO: should be relations_after - .iter() - .filter(|(_, pdu)| { - filter_event_type.as_ref().map_or(true, |t| &pdu.kind == t) - && if let Ok(content) = - serde_json::from_str::(pdu.content.get()) - { - filter_rel_type - .as_ref() - .map_or(true, |r| &content.relates_to.rel_type == r) - } else { - false - } - }) - .take(limit) - .filter(|(_, pdu)| { - self.services - .state_accessor - .user_can_see_event(sender_user, room_id, &pdu.event_id) - .unwrap_or(false) - }) - .take_while(|(k, _)| Some(k) != to.as_ref()) // Stop at `to` - .collect(); + let relations_until: Vec = self + .relations_until(sender_user, room_id, target, from, depth) + .await?; + + // TODO: should be relations_after + let events: Vec<_> = relations_until + .into_iter() + .filter(move |(_, pdu): &PdusIterItem| { + if !filter_event_type.as_ref().map_or(true, |t| pdu.kind == *t) { + return false; + } + + let Ok(content) = serde_json::from_str::(pdu.content.get()) else { + return false; + }; + + filter_rel_type + .as_ref() + .map_or(true, |r| *r == content.relates_to.rel_type) + }) + .take(limit) + .take_while(|(k, _)| Some(*k) != to) + .stream() + .filter_map(|item| self.visibility_filter(sender_user, item)) + .collect() + .await; let next_token = events.last().map(|(count, _)| count).copied(); @@ -125,9 +125,9 @@ impl Service { .map(|(_, pdu)| pdu.to_message_like_event()) .collect(), Direction::Backward => events - .into_iter() - .rev() // relations are always most recent first - .map(|(_, pdu)| pdu.to_message_like_event()) + .into_iter() + .rev() // relations are always most recent first + .map(|(_, pdu)| pdu.to_message_like_event()) .collect(), }; @@ -135,68 +135,85 @@ impl Service { chunk: events_chunk, next_batch: next_token.map(|t| t.stringify()), prev_batch: Some(from.stringify()), - recursion_depth: if recurse { - Some(depth.into()) - } else { - None - }, + recursion_depth: recurse.then_some(depth.into()), }) } - pub fn relations_until<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, target: &'a EventId, until: PduCount, max_depth: u8, - ) -> Result> { - let room_id = self.services.short.get_or_create_shortroomid(room_id)?; - #[allow(unknown_lints)] - #[allow(clippy::manual_unwrap_or_default)] - let target = match self.services.timeline.get_pdu_count(target)? { - Some(PduCount::Normal(c)) => c, + async fn visibility_filter(&self, sender_user: &UserId, item: PdusIterItem) -> Option { + let (_, pdu) = &item; + + self.services + .state_accessor + .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) + .await + .then_some(item) + } + + pub async fn relations_until( + &self, user_id: &UserId, room_id: &RoomId, target: &EventId, until: PduCount, max_depth: u8, + ) -> Result> { + let room_id = self.services.short.get_or_create_shortroomid(room_id).await; + + let target = match self.services.timeline.get_pdu_count(target).await { + Ok(PduCount::Normal(c)) => c, // TODO: Support backfilled relations _ => 0, // This will result in an empty iterator }; - self.db + let mut pdus: Vec = self + .db .relations_until(user_id, room_id, target, until) - .map(|mut relations| { - let mut pdus: Vec<_> = (*relations).into_iter().filter_map(Result::ok).collect(); - let mut stack: Vec<_> = pdus.clone().iter().map(|pdu| (pdu.to_owned(), 1)).collect(); + .collect() + .await; - while let Some(stack_pdu) = stack.pop() { - let target = match stack_pdu.0 .0 { - PduCount::Normal(c) => c, - // TODO: Support backfilled relations - PduCount::Backfilled(_) => 0, // This will result in an empty iterator - }; + let mut stack: Vec<_> = pdus.clone().into_iter().map(|pdu| (pdu, 1)).collect(); - if let Ok(relations) = self.db.relations_until(user_id, room_id, target, until) { - for relation in relations.flatten() { - if stack_pdu.1 < max_depth { - stack.push((relation.clone(), stack_pdu.1.saturating_add(1))); - } + while let Some(stack_pdu) = stack.pop() { + let target = match stack_pdu.0 .0 { + PduCount::Normal(c) => c, + // TODO: Support backfilled relations + PduCount::Backfilled(_) => 0, // This will result in an empty iterator + }; - pdus.push(relation); - } - } + let relations: Vec = self + .db + .relations_until(user_id, room_id, target, until) + .collect() + .await; + + for relation in relations { + if stack_pdu.1 < max_depth { + stack.push((relation.clone(), stack_pdu.1.saturating_add(1))); } - pdus.sort_by(|a, b| a.0.cmp(&b.0)); - pdus - }) + pdus.push(relation); + } + } + + pdus.sort_by(|a, b| a.0.cmp(&b.0)); + + Ok(pdus) } + #[inline] #[tracing::instrument(skip_all, level = "debug")] - pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) -> Result<()> { - self.db.mark_as_referenced(room_id, event_ids) + pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) { + self.db.mark_as_referenced(room_id, event_ids); } + #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> Result { - self.db.is_event_referenced(room_id, event_id) + pub async fn is_event_referenced(&self, room_id: &RoomId, event_id: &EventId) -> bool { + self.db.is_event_referenced(room_id, event_id).await } + #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub fn mark_event_soft_failed(&self, event_id: &EventId) -> Result<()> { self.db.mark_event_soft_failed(event_id) } + pub fn mark_event_soft_failed(&self, event_id: &EventId) { self.db.mark_event_soft_failed(event_id) } + #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub fn is_event_soft_failed(&self, event_id: &EventId) -> Result { self.db.is_event_soft_failed(event_id) } + pub async fn is_event_soft_failed(&self, event_id: &EventId) -> bool { + self.db.is_event_soft_failed(event_id).await + } } diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 0c156df3..a2c0fabc 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -1,10 +1,18 @@ use std::{mem::size_of, sync::Arc}; -use conduit::{utils, Error, Result}; -use database::Map; -use ruma::{events::receipt::ReceiptEvent, serde::Raw, CanonicalJsonObject, RoomId, UserId}; +use conduit::{ + utils, + utils::{stream::TryIgnore, ReadyExt}, + Error, Result, +}; +use database::{Deserialized, Map}; +use futures::{Stream, StreamExt}; +use ruma::{ + events::{receipt::ReceiptEvent, AnySyncEphemeralRoomEvent}, + serde::Raw, + CanonicalJsonObject, OwnedUserId, RoomId, UserId, +}; -use super::AnySyncEphemeralRoomEventIter; use crate::{globals, Dep}; pub(super) struct Data { @@ -18,6 +26,8 @@ struct Services { globals: Dep, } +pub(super) type ReceiptItem = (OwnedUserId, u64, Raw); + impl Data { pub(super) fn new(args: &crate::Args<'_>) -> Self { let db = &args.db; @@ -31,7 +41,9 @@ impl Data { } } - pub(super) fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) -> Result<()> { + pub(super) async fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) { + type KeyVal<'a> = (&'a RoomId, u64, &'a UserId); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -39,108 +51,90 @@ impl Data { last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes()); // Remove old entry - if let Some((old, _)) = self - .readreceiptid_readreceipt - .iter_from(&last_possible_key, true) - .take_while(|(key, _)| key.starts_with(&prefix)) - .find(|(key, _)| { - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element") - == user_id.as_bytes() - }) { - // This is the old room_latest - self.readreceiptid_readreceipt.remove(&old)?; - } + self.readreceiptid_readreceipt + .rev_keys_from_raw(&last_possible_key) + .ignore_err() + .ready_take_while(|(r, ..): &KeyVal<'_>| *r == room_id) + .ready_filter_map(|(r, c, u): KeyVal<'_>| (u == user_id).then_some((r, c, u))) + .ready_for_each(|old: KeyVal<'_>| { + // This is the old room_latest + self.readreceiptid_readreceipt.del(&old); + }) + .await; let mut room_latest_id = prefix; - room_latest_id.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); + room_latest_id.extend_from_slice(&self.services.globals.next_count().unwrap().to_be_bytes()); room_latest_id.push(0xFF); room_latest_id.extend_from_slice(user_id.as_bytes()); self.readreceiptid_readreceipt.insert( &room_latest_id, &serde_json::to_vec(event).expect("EduEvent::to_string always works"), - )?; - - Ok(()) + ); } - pub(super) fn readreceipts_since<'a>(&'a self, room_id: &RoomId, since: u64) -> AnySyncEphemeralRoomEventIter<'a> { + pub(super) fn readreceipts_since<'a>( + &'a self, room_id: &'a RoomId, since: u64, + ) -> impl Stream + Send + 'a { + let after_since = since.saturating_add(1); // +1 so we don't send the event at since + let first_possible_edu = (room_id, after_since); + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); let prefix2 = prefix.clone(); - let mut first_possible_edu = prefix.clone(); - first_possible_edu.extend_from_slice(&(since.saturating_add(1)).to_be_bytes()); // +1 so we don't send the event at since + self.readreceiptid_readreceipt + .stream_raw_from(&first_possible_edu) + .ignore_err() + .ready_take_while(move |(k, _)| k.starts_with(&prefix2)) + .map(move |(k, v)| { + let count_offset = prefix.len().saturating_add(size_of::()); + let user_id_offset = count_offset.saturating_add(1); - Box::new( - self.readreceiptid_readreceipt - .iter_from(&first_possible_edu, false) - .take_while(move |(k, _)| k.starts_with(&prefix2)) - .map(move |(k, v)| { - let count_offset = prefix.len().saturating_add(size_of::()); - let count = utils::u64_from_bytes(&k[prefix.len()..count_offset]) - .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?; - let user_id_offset = count_offset.saturating_add(1); - let user_id = UserId::parse( - utils::string_from_bytes(&k[user_id_offset..]) - .map_err(|_| Error::bad_database("Invalid readreceiptid userid bytes in db."))?, - ) + let count = utils::u64_from_bytes(&k[prefix.len()..count_offset]) + .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?; + + let user_id_str = utils::string_from_bytes(&k[user_id_offset..]) + .map_err(|_| Error::bad_database("Invalid readreceiptid userid bytes in db."))?; + + let user_id = UserId::parse(user_id_str) .map_err(|_| Error::bad_database("Invalid readreceiptid userid in db."))?; - let mut json = serde_json::from_slice::(&v) - .map_err(|_| Error::bad_database("Read receipt in roomlatestid_roomlatest is invalid json."))?; - json.remove("room_id"); + let mut json = serde_json::from_slice::(v) + .map_err(|_| Error::bad_database("Read receipt in roomlatestid_roomlatest is invalid json."))?; - Ok(( - user_id, - count, - Raw::from_json(serde_json::value::to_raw_value(&json).expect("json is valid raw value")), - )) - }), - ) + json.remove("room_id"); + + let event = Raw::from_json(serde_json::value::to_raw_value(&json)?); + + Ok((user_id, count, event)) + }) + .ignore_err() } - pub(super) fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) -> Result<()> { + pub(super) fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) { let mut key = room_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(user_id.as_bytes()); self.roomuserid_privateread - .insert(&key, &count.to_be_bytes())?; + .insert(&key, &count.to_be_bytes()); self.roomuserid_lastprivatereadupdate - .insert(&key, &self.services.globals.next_count()?.to_be_bytes()) + .insert(&key, &self.services.globals.next_count().unwrap().to_be_bytes()); } - pub(super) fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(user_id.as_bytes()); - - self.roomuserid_privateread - .get(&key)? - .map_or(Ok(None), |v| { - Ok(Some( - utils::u64_from_bytes(&v).map_err(|_| Error::bad_database("Invalid private read marker bytes"))?, - )) - }) + pub(super) async fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.roomuserid_privateread.qry(&key).await.deserialized() } - pub(super) fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(user_id.as_bytes()); - - Ok(self - .roomuserid_lastprivatereadupdate - .get(&key)? - .map(|bytes| { - utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Count in roomuserid_lastprivatereadupdate is invalid.")) - }) - .transpose()? - .unwrap_or(0)) + pub(super) async fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (room_id, user_id); + self.roomuserid_lastprivatereadupdate + .qry(&key) + .await + .deserialized() + .unwrap_or(0) } } diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index da11e2a0..ec34361e 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -3,16 +3,17 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; use conduit::{debug, Result}; -use data::Data; +use futures::Stream; use ruma::{ events::{ receipt::{ReceiptEvent, ReceiptEventContent}, - AnySyncEphemeralRoomEvent, SyncEphemeralRoomEvent, + SyncEphemeralRoomEvent, }, serde::Raw, - OwnedUserId, RoomId, UserId, + RoomId, UserId, }; +use self::data::{Data, ReceiptItem}; use crate::{sending, Dep}; pub struct Service { @@ -24,9 +25,6 @@ struct Services { sending: Dep, } -type AnySyncEphemeralRoomEventIter<'a> = - Box)>> + 'a>; - impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -42,44 +40,53 @@ impl crate::Service for Service { impl Service { /// Replaces the previous read receipt. - pub fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) -> Result<()> { - self.db.readreceipt_update(user_id, room_id, event)?; - self.services.sending.flush_room(room_id)?; - - Ok(()) + pub async fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) { + self.db.readreceipt_update(user_id, room_id, event).await; + self.services + .sending + .flush_room(room_id) + .await + .expect("room flush failed"); } /// Returns an iterator over the most recent read_receipts in a room that /// happened after the event with id `since`. + #[inline] #[tracing::instrument(skip(self), level = "debug")] pub fn readreceipts_since<'a>( - &'a self, room_id: &RoomId, since: u64, - ) -> impl Iterator)>> + 'a { + &'a self, room_id: &'a RoomId, since: u64, + ) -> impl Stream + Send + 'a { self.db.readreceipts_since(room_id, since) } /// Sets a private read marker at `count`. + #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) -> Result<()> { - self.db.private_read_set(room_id, user_id, count) + pub fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) { + self.db.private_read_set(room_id, user_id, count); } /// Returns the private read marker. + #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { - self.db.private_read_get(room_id, user_id) + pub async fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result { + self.db.private_read_get(room_id, user_id).await } /// Returns the count of the last typing update in this room. - pub fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result { - self.db.last_privateread_update(user_id, room_id) + #[inline] + pub async fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + self.db.last_privateread_update(user_id, room_id).await } } #[must_use] -pub fn pack_receipts(receipts: AnySyncEphemeralRoomEventIter<'_>) -> Raw> { +pub fn pack_receipts(receipts: I) -> Raw> +where + I: Iterator, +{ let mut json = BTreeMap::new(); - for (_user, _count, value) in receipts.flatten() { + for (_, _, value) in receipts { let receipt = serde_json::from_str::>(value.json().get()); if let Ok(value) = receipt { for (event, receipt) in value.content { diff --git a/src/service/rooms/search/data.rs b/src/service/rooms/search/data.rs index a0086095..de98beee 100644 --- a/src/service/rooms/search/data.rs +++ b/src/service/rooms/search/data.rs @@ -1,13 +1,12 @@ use std::sync::Arc; -use conduit::{utils, Result}; +use conduit::utils::{set, stream::TryIgnore, IterStream, ReadyExt}; use database::Map; +use futures::StreamExt; use ruma::RoomId; use crate::{rooms, Dep}; -type SearchPdusResult<'a> = Result> + 'a>, Vec)>>; - pub(super) struct Data { tokenids: Arc, services: Services, @@ -28,7 +27,7 @@ impl Data { } } - pub(super) fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { + pub(super) fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { let batch = tokenize(message_body) .map(|word| { let mut key = shortroomid.to_be_bytes().to_vec(); @@ -39,11 +38,10 @@ impl Data { }) .collect::>(); - self.tokenids - .insert_batch(batch.iter().map(database::KeyVal::from)) + self.tokenids.insert_batch(batch.iter()); } - pub(super) fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { + pub(super) fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { let batch = tokenize(message_body).map(|word| { let mut key = shortroomid.to_be_bytes().to_vec(); key.extend_from_slice(word.as_bytes()); @@ -53,46 +51,53 @@ impl Data { }); for token in batch { - self.tokenids.remove(&token)?; + self.tokenids.remove(&token); } - - Ok(()) } - pub(super) fn search_pdus<'a>(&'a self, room_id: &RoomId, search_string: &str) -> SearchPdusResult<'a> { + pub(super) async fn search_pdus( + &self, room_id: &RoomId, search_string: &str, + ) -> Option<(Vec>, Vec)> { let prefix = self .services .short - .get_shortroomid(room_id)? - .expect("room exists") + .get_shortroomid(room_id) + .await + .ok()? .to_be_bytes() .to_vec(); let words: Vec<_> = tokenize(search_string).collect(); - let iterators = words.clone().into_iter().map(move |word| { - let mut prefix2 = prefix.clone(); - prefix2.extend_from_slice(word.as_bytes()); - prefix2.push(0xFF); - let prefix3 = prefix2.clone(); + let bufs: Vec<_> = words + .clone() + .into_iter() + .stream() + .then(move |word| { + let mut prefix2 = prefix.clone(); + prefix2.extend_from_slice(word.as_bytes()); + prefix2.push(0xFF); + let prefix3 = prefix2.clone(); - let mut last_possible_id = prefix2.clone(); - last_possible_id.extend_from_slice(&u64::MAX.to_be_bytes()); + let mut last_possible_id = prefix2.clone(); + last_possible_id.extend_from_slice(&u64::MAX.to_be_bytes()); - self.tokenids - .iter_from(&last_possible_id, true) // Newest pdus first - .take_while(move |(k, _)| k.starts_with(&prefix2)) - .map(move |(key, _)| key[prefix3.len()..].to_vec()) - }); + self.tokenids + .rev_raw_keys_from(&last_possible_id) // Newest pdus first + .ignore_err() + .ready_take_while(move |key| key.starts_with(&prefix2)) + .map(move |key| key[prefix3.len()..].to_vec()) + .collect::>() + }) + .collect() + .await; - let Some(common_elements) = utils::common_elements(iterators, |a, b| { - // We compare b with a because we reversed the iterator earlier - b.cmp(a) - }) else { - return Ok(None); - }; - - Ok(Some((Box::new(common_elements), words))) + Some(( + set::intersection(bufs.iter().map(|buf| buf.iter())) + .cloned() + .collect(), + words, + )) } } @@ -100,7 +105,7 @@ impl Data { /// /// This may be used to tokenize both message bodies (for indexing) or search /// queries (for querying). -fn tokenize(body: &str) -> impl Iterator + '_ { +fn tokenize(body: &str) -> impl Iterator + Send + '_ { body.split_terminator(|c: char| !c.is_alphanumeric()) .filter(|s| !s.is_empty()) .filter(|word| word.len() <= 50) diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 8caa0ce3..80b58804 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -21,20 +21,21 @@ impl crate::Service for Service { } impl Service { + #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { - self.db.index_pdu(shortroomid, pdu_id, message_body) + pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { + self.db.index_pdu(shortroomid, pdu_id, message_body); } + #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) -> Result<()> { - self.db.deindex_pdu(shortroomid, pdu_id, message_body) + pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { + self.db.deindex_pdu(shortroomid, pdu_id, message_body); } + #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub fn search_pdus<'a>( - &'a self, room_id: &RoomId, search_string: &str, - ) -> Result> + 'a, Vec)>> { - self.db.search_pdus(room_id, search_string) + pub async fn search_pdus(&self, room_id: &RoomId, search_string: &str) -> Option<(Vec>, Vec)> { + self.db.search_pdus(room_id, search_string).await } } diff --git a/src/service/rooms/short/data.rs b/src/service/rooms/short/data.rs index 17fbb64e..f6a82488 100644 --- a/src/service/rooms/short/data.rs +++ b/src/service/rooms/short/data.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use conduit::{utils, warn, Error, Result}; -use database::Map; +use conduit::{err, utils, Error, Result}; +use database::{Deserialized, Map}; use ruma::{events::StateEventType, EventId, RoomId}; use crate::{globals, Dep}; @@ -36,44 +36,46 @@ impl Data { } } - pub(super) fn get_or_create_shorteventid(&self, event_id: &EventId) -> Result { - let short = if let Some(shorteventid) = self.eventid_shorteventid.get(event_id.as_bytes())? { - utils::u64_from_bytes(&shorteventid).map_err(|_| Error::bad_database("Invalid shorteventid in db."))? - } else { - let shorteventid = self.services.globals.next_count()?; - self.eventid_shorteventid - .insert(event_id.as_bytes(), &shorteventid.to_be_bytes())?; - self.shorteventid_eventid - .insert(&shorteventid.to_be_bytes(), event_id.as_bytes())?; - shorteventid - }; + pub(super) async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { + if let Ok(shorteventid) = self.eventid_shorteventid.qry(event_id).await.deserialized() { + return shorteventid; + } - Ok(short) + let shorteventid = self.services.globals.next_count().unwrap(); + self.eventid_shorteventid + .insert(event_id.as_bytes(), &shorteventid.to_be_bytes()); + self.shorteventid_eventid + .insert(&shorteventid.to_be_bytes(), event_id.as_bytes()); + + shorteventid } - pub(super) fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Result> { + pub(super) async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Vec { let mut ret: Vec = Vec::with_capacity(event_ids.len()); let keys = event_ids .iter() .map(|id| id.as_bytes()) .collect::>(); + for (i, short) in self .eventid_shorteventid - .multi_get(&keys)? + .multi_get(keys.iter()) .iter() .enumerate() { #[allow(clippy::single_match_else)] match short { Some(short) => ret.push( - utils::u64_from_bytes(short).map_err(|_| Error::bad_database("Invalid shorteventid in db."))?, + utils::u64_from_bytes(short) + .map_err(|_| Error::bad_database("Invalid shorteventid in db.")) + .unwrap(), ), None => { - let short = self.services.globals.next_count()?; + let short = self.services.globals.next_count().unwrap(); self.eventid_shorteventid - .insert(keys[i], &short.to_be_bytes())?; + .insert(keys[i], &short.to_be_bytes()); self.shorteventid_eventid - .insert(&short.to_be_bytes(), keys[i])?; + .insert(&short.to_be_bytes(), keys[i]); debug_assert!(ret.len() == i, "position of result must match input"); ret.push(short); @@ -81,115 +83,85 @@ impl Data { } } - Ok(ret) + ret } - pub(super) fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result> { - let mut statekey_vec = event_type.to_string().as_bytes().to_vec(); - statekey_vec.push(0xFF); - statekey_vec.extend_from_slice(state_key.as_bytes()); + pub(super) async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { + let key = (event_type, state_key); + self.statekey_shortstatekey.qry(&key).await.deserialized() + } - let short = self - .statekey_shortstatekey - .get(&statekey_vec)? - .map(|shortstatekey| { - utils::u64_from_bytes(&shortstatekey).map_err(|_| Error::bad_database("Invalid shortstatekey in db.")) + pub(super) async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> u64 { + let key = (event_type.to_string(), state_key); + if let Ok(shortstatekey) = self.statekey_shortstatekey.qry(&key).await.deserialized() { + return shortstatekey; + } + + let mut key = event_type.to_string().as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(state_key.as_bytes()); + + let shortstatekey = self.services.globals.next_count().unwrap(); + self.statekey_shortstatekey + .insert(&key, &shortstatekey.to_be_bytes()); + self.shortstatekey_statekey + .insert(&shortstatekey.to_be_bytes(), &key); + + shortstatekey + } + + pub(super) async fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { + self.shorteventid_eventid + .qry(&shorteventid) + .await + .deserialized() + .map_err(|e| err!(Database("Failed to find EventId from short {shorteventid:?}: {e:?}"))) + } + + pub(super) async fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { + self.shortstatekey_statekey + .qry(&shortstatekey) + .await + .deserialized() + .map_err(|e| { + err!(Database( + "Failed to find (StateEventType, state_key) from short {shortstatekey:?}: {e:?}" + )) }) - .transpose()?; - - Ok(short) - } - - pub(super) fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { - let mut statekey_vec = event_type.to_string().as_bytes().to_vec(); - statekey_vec.push(0xFF); - statekey_vec.extend_from_slice(state_key.as_bytes()); - - let short = if let Some(shortstatekey) = self.statekey_shortstatekey.get(&statekey_vec)? { - utils::u64_from_bytes(&shortstatekey).map_err(|_| Error::bad_database("Invalid shortstatekey in db."))? - } else { - let shortstatekey = self.services.globals.next_count()?; - self.statekey_shortstatekey - .insert(&statekey_vec, &shortstatekey.to_be_bytes())?; - self.shortstatekey_statekey - .insert(&shortstatekey.to_be_bytes(), &statekey_vec)?; - shortstatekey - }; - - Ok(short) - } - - pub(super) fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { - let bytes = self - .shorteventid_eventid - .get(&shorteventid.to_be_bytes())? - .ok_or_else(|| Error::bad_database("Shorteventid does not exist"))?; - - let event_id = EventId::parse_arc( - utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("EventID in shorteventid_eventid is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("EventId in shorteventid_eventid is invalid."))?; - - Ok(event_id) - } - - pub(super) fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { - let bytes = self - .shortstatekey_statekey - .get(&shortstatekey.to_be_bytes())? - .ok_or_else(|| Error::bad_database("Shortstatekey does not exist"))?; - - let mut parts = bytes.splitn(2, |&b| b == 0xFF); - let eventtype_bytes = parts.next().expect("split always returns one entry"); - let statekey_bytes = parts - .next() - .ok_or_else(|| Error::bad_database("Invalid statekey in shortstatekey_statekey."))?; - - let event_type = StateEventType::from(utils::string_from_bytes(eventtype_bytes).map_err(|e| { - warn!("Event type in shortstatekey_statekey is invalid: {}", e); - Error::bad_database("Event type in shortstatekey_statekey is invalid.") - })?); - - let state_key = utils::string_from_bytes(statekey_bytes) - .map_err(|_| Error::bad_database("Statekey in shortstatekey_statekey is invalid unicode."))?; - - let result = (event_type, state_key); - - Ok(result) } /// Returns (shortstatehash, already_existed) - pub(super) fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> Result<(u64, bool)> { - Ok(if let Some(shortstatehash) = self.statehash_shortstatehash.get(state_hash)? { - ( - utils::u64_from_bytes(&shortstatehash) - .map_err(|_| Error::bad_database("Invalid shortstatehash in db."))?, - true, - ) - } else { - let shortstatehash = self.services.globals.next_count()?; - self.statehash_shortstatehash - .insert(state_hash, &shortstatehash.to_be_bytes())?; - (shortstatehash, false) - }) + pub(super) async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, bool) { + if let Ok(shortstatehash) = self + .statehash_shortstatehash + .qry(state_hash) + .await + .deserialized() + { + return (shortstatehash, true); + } + + let shortstatehash = self.services.globals.next_count().unwrap(); + self.statehash_shortstatehash + .insert(state_hash, &shortstatehash.to_be_bytes()); + + (shortstatehash, false) } - pub(super) fn get_shortroomid(&self, room_id: &RoomId) -> Result> { + pub(super) async fn get_shortroomid(&self, room_id: &RoomId) -> Result { + self.roomid_shortroomid.qry(room_id).await.deserialized() + } + + pub(super) async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> u64 { self.roomid_shortroomid - .get(room_id.as_bytes())? - .map(|bytes| utils::u64_from_bytes(&bytes).map_err(|_| Error::bad_database("Invalid shortroomid in db."))) - .transpose() - } - - pub(super) fn get_or_create_shortroomid(&self, room_id: &RoomId) -> Result { - Ok(if let Some(short) = self.roomid_shortroomid.get(room_id.as_bytes())? { - utils::u64_from_bytes(&short).map_err(|_| Error::bad_database("Invalid shortroomid in db."))? - } else { - let short = self.services.globals.next_count()?; - self.roomid_shortroomid - .insert(room_id.as_bytes(), &short.to_be_bytes())?; - short - }) + .qry(room_id) + .await + .deserialized() + .unwrap_or_else(|_| { + let short = self.services.globals.next_count().unwrap(); + self.roomid_shortroomid + .insert(room_id.as_bytes(), &short.to_be_bytes()); + short + }) } } diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index bfe0e9a0..00bb7cb1 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -22,38 +22,40 @@ impl crate::Service for Service { } impl Service { - pub fn get_or_create_shorteventid(&self, event_id: &EventId) -> Result { - self.db.get_or_create_shorteventid(event_id) + pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { + self.db.get_or_create_shorteventid(event_id).await } - pub fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Result> { - self.db.multi_get_or_create_shorteventid(event_ids) + pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Vec { + self.db.multi_get_or_create_shorteventid(event_ids).await } - pub fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result> { - self.db.get_shortstatekey(event_type, state_key) + pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { + self.db.get_shortstatekey(event_type, state_key).await } - pub fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { - self.db.get_or_create_shortstatekey(event_type, state_key) + pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> u64 { + self.db + .get_or_create_shortstatekey(event_type, state_key) + .await } - pub fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { - self.db.get_eventid_from_short(shorteventid) + pub async fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { + self.db.get_eventid_from_short(shorteventid).await } - pub fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { - self.db.get_statekey_from_short(shortstatekey) + pub async fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { + self.db.get_statekey_from_short(shortstatekey).await } /// Returns (shortstatehash, already_existed) - pub fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> Result<(u64, bool)> { - self.db.get_or_create_shortstatehash(state_hash) + pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, bool) { + self.db.get_or_create_shortstatehash(state_hash).await } - pub fn get_shortroomid(&self, room_id: &RoomId) -> Result> { self.db.get_shortroomid(room_id) } + pub async fn get_shortroomid(&self, room_id: &RoomId) -> Result { self.db.get_shortroomid(room_id).await } - pub fn get_or_create_shortroomid(&self, room_id: &RoomId) -> Result { - self.db.get_or_create_shortroomid(room_id) + pub async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> u64 { + self.db.get_or_create_shortroomid(room_id).await } } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 24d612d8..17fbf0ef 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,7 +7,12 @@ use std::{ sync::Arc, }; -use conduit::{checked, debug, debug_info, err, utils::math::usize_from_f64, warn, Error, Result}; +use conduit::{ + checked, debug, debug_info, err, + utils::{math::usize_from_f64, IterStream}, + Error, Result, +}; +use futures::{StreamExt, TryFutureExt}; use lru_cache::LruCache; use ruma::{ api::{ @@ -211,12 +216,15 @@ impl Service { .as_ref() { return Ok(if let Some(cached) = cached { - if self.is_accessible_child( - current_room, - &cached.summary.join_rule, - &identifier, - &cached.summary.allowed_room_ids, - ) { + if self + .is_accessible_child( + current_room, + &cached.summary.join_rule, + &identifier, + &cached.summary.allowed_room_ids, + ) + .await + { Some(SummaryAccessibility::Accessible(Box::new(cached.summary.clone()))) } else { Some(SummaryAccessibility::Inaccessible) @@ -228,7 +236,9 @@ impl Service { Ok( if let Some(children_pdus) = self.get_stripped_space_child_events(current_room).await? { - let summary = self.get_room_summary(current_room, children_pdus, &identifier); + let summary = self + .get_room_summary(current_room, children_pdus, &identifier) + .await; if let Ok(summary) = summary { self.roomid_spacehierarchy_cache.lock().await.insert( current_room.clone(), @@ -322,12 +332,15 @@ impl Service { ); } } - if self.is_accessible_child( - current_room, - &response.room.join_rule, - &Identifier::UserId(user_id), - &response.room.allowed_room_ids, - ) { + if self + .is_accessible_child( + current_room, + &response.room.join_rule, + &Identifier::UserId(user_id), + &response.room.allowed_room_ids, + ) + .await + { return Ok(Some(SummaryAccessibility::Accessible(Box::new(summary.clone())))); } @@ -358,7 +371,7 @@ impl Service { } } - fn get_room_summary( + async fn get_room_summary( &self, current_room: &OwnedRoomId, children_state: Vec>, identifier: &Identifier<'_>, ) -> Result { @@ -367,48 +380,43 @@ impl Service { let join_rule = self .services .state_accessor - .room_state_get(room_id, &StateEventType::RoomJoinRules, "")? - .map(|s| { + .room_state_get(room_id, &StateEventType::RoomJoinRules, "") + .await + .map_or(JoinRule::Invite, |s| { serde_json::from_str(s.content.get()) .map(|c: RoomJoinRulesEventContent| c.join_rule) .map_err(|e| err!(Database(error!("Invalid room join rule event in database: {e}")))) - }) - .transpose()? - .unwrap_or(JoinRule::Invite); + .unwrap() + }); let allowed_room_ids = self .services .state_accessor .allowed_room_ids(join_rule.clone()); - if !self.is_accessible_child(current_room, &join_rule.clone().into(), identifier, &allowed_room_ids) { + if !self + .is_accessible_child(current_room, &join_rule.clone().into(), identifier, &allowed_room_ids) + .await + { debug!("User is not allowed to see room {room_id}"); // This error will be caught later return Err(Error::BadRequest(ErrorKind::forbidden(), "User is not allowed to see the room")); } - let join_rule = join_rule.into(); - Ok(SpaceHierarchyParentSummary { canonical_alias: self .services .state_accessor .get_canonical_alias(room_id) - .unwrap_or(None), - name: self - .services - .state_accessor - .get_name(room_id) - .unwrap_or(None), + .await + .ok(), + name: self.services.state_accessor.get_name(room_id).await.ok(), num_joined_members: self .services .state_cache .room_joined_count(room_id) - .unwrap_or_default() - .unwrap_or_else(|| { - warn!("Room {room_id} has no member count"); - 0 - }) + .await + .unwrap_or(0) .try_into() .expect("user count should not be that big"), room_id: room_id.to_owned(), @@ -416,18 +424,29 @@ impl Service { .services .state_accessor .get_room_topic(room_id) - .unwrap_or(None), - world_readable: self.services.state_accessor.is_world_readable(room_id)?, - guest_can_join: self.services.state_accessor.guest_can_join(room_id)?, + .await + .ok(), + world_readable: self + .services + .state_accessor + .is_world_readable(room_id) + .await, + guest_can_join: self.services.state_accessor.guest_can_join(room_id).await, avatar_url: self .services .state_accessor - .get_avatar(room_id)? + .get_avatar(room_id) + .await .into_option() .unwrap_or_default() .url, - join_rule, - room_type: self.services.state_accessor.get_room_type(room_id)?, + join_rule: join_rule.into(), + room_type: self + .services + .state_accessor + .get_room_type(room_id) + .await + .ok(), children_state, allowed_room_ids, }) @@ -474,21 +493,22 @@ impl Service { results.push(summary_to_chunk(*summary.clone())); } else { children = children - .into_iter() - .rev() - .skip_while(|(room, _)| { - if let Ok(short) = self.services.short.get_shortroomid(room) - { - short.as_ref() != short_room_ids.get(parents.len()) - } else { - false - } - }) - .collect::>() - // skip_while doesn't implement DoubleEndedIterator, which is needed for rev - .into_iter() - .rev() - .collect(); + .iter() + .rev() + .stream() + .skip_while(|(room, _)| { + self.services + .short + .get_shortroomid(room) + .map_ok(|short| Some(&short) != short_room_ids.get(parents.len())) + .unwrap_or_else(|_| false) + }) + .map(Clone::clone) + .collect::)>>() + .await + .into_iter() + .rev() + .collect(); if children.is_empty() { return Err(Error::BadRequest( @@ -531,7 +551,7 @@ impl Service { let mut short_room_ids = vec![]; for room in parents { - short_room_ids.push(self.services.short.get_or_create_shortroomid(&room)?); + short_room_ids.push(self.services.short.get_or_create_shortroomid(&room).await); } Some( @@ -554,7 +574,7 @@ impl Service { async fn get_stripped_space_child_events( &self, room_id: &RoomId, ) -> Result>>, Error> { - let Some(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id)? else { + let Ok(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id).await else { return Ok(None); }; @@ -562,10 +582,13 @@ impl Service { .services .state_accessor .state_full_ids(current_shortstatehash) - .await?; + .await + .map_err(|e| err!(Database("State in space not found: {e}")))?; + let mut children_pdus = Vec::new(); for (key, id) in state { - let (event_type, state_key) = self.services.short.get_statekey_from_short(key)?; + let (event_type, state_key) = self.services.short.get_statekey_from_short(key).await?; + if event_type != StateEventType::SpaceChild { continue; } @@ -573,8 +596,9 @@ impl Service { let pdu = self .services .timeline - .get_pdu(&id)? - .ok_or_else(|| Error::bad_database("Event in space state not found"))?; + .get_pdu(&id) + .await + .map_err(|e| err!(Database("Event {id:?} in space state not found: {e:?}")))?; if serde_json::from_str::(pdu.content.get()) .ok() @@ -593,7 +617,7 @@ impl Service { } /// With the given identifier, checks if a room is accessable - fn is_accessible_child( + async fn is_accessible_child( &self, current_room: &OwnedRoomId, join_rule: &SpaceRoomJoinRule, identifier: &Identifier<'_>, allowed_room_ids: &Vec, ) -> bool { @@ -607,6 +631,7 @@ impl Service { .services .event_handler .acl_check(server_name, room_id) + .await .is_err() { return false; @@ -617,12 +642,11 @@ impl Service { .services .state_cache .is_joined(user_id, current_room) - .unwrap_or_default() - || self - .services - .state_cache - .is_invited(user_id, current_room) - .unwrap_or_default() + .await || self + .services + .state_cache + .is_invited(user_id, current_room) + .await { return true; } @@ -633,22 +657,12 @@ impl Service { for room in allowed_room_ids { match identifier { Identifier::UserId(user) => { - if self - .services - .state_cache - .is_joined(user, room) - .unwrap_or_default() - { + if self.services.state_cache.is_joined(user, room).await { return true; } }, Identifier::ServerName(server) => { - if self - .services - .state_cache - .server_in_room(server, room) - .unwrap_or_default() - { + if self.services.state_cache.server_in_room(server, room).await { return true; } }, diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index 3c110afc..ccf7509a 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -1,34 +1,31 @@ -use std::{collections::HashSet, sync::Arc}; +use std::sync::Arc; -use conduit::{utils, Error, Result}; -use database::{Database, Map}; -use ruma::{EventId, OwnedEventId, RoomId}; +use conduit::{ + utils::{stream::TryIgnore, ReadyExt}, + Result, +}; +use database::{Database, Deserialized, Interfix, Map}; +use ruma::{OwnedEventId, RoomId}; use super::RoomMutexGuard; pub(super) struct Data { shorteventid_shortstatehash: Arc, - roomid_pduleaves: Arc, roomid_shortstatehash: Arc, + pub(super) roomid_pduleaves: Arc, } impl Data { pub(super) fn new(db: &Arc) -> Self { Self { shorteventid_shortstatehash: db["shorteventid_shortstatehash"].clone(), - roomid_pduleaves: db["roomid_pduleaves"].clone(), roomid_shortstatehash: db["roomid_shortstatehash"].clone(), + roomid_pduleaves: db["roomid_pduleaves"].clone(), } } - pub(super) fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result> { - self.roomid_shortstatehash - .get(room_id.as_bytes())? - .map_or(Ok(None), |bytes| { - Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| { - Error::bad_database("Invalid shortstatehash in roomid_shortstatehash") - })?)) - }) + pub(super) async fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result { + self.roomid_shortstatehash.qry(room_id).await.deserialized() } #[inline] @@ -37,53 +34,35 @@ impl Data { room_id: &RoomId, new_shortstatehash: u64, _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex - ) -> Result<()> { + ) { self.roomid_shortstatehash - .insert(room_id.as_bytes(), &new_shortstatehash.to_be_bytes())?; - Ok(()) + .insert(room_id.as_bytes(), &new_shortstatehash.to_be_bytes()); } - pub(super) fn set_event_state(&self, shorteventid: u64, shortstatehash: u64) -> Result<()> { + pub(super) fn set_event_state(&self, shorteventid: u64, shortstatehash: u64) { self.shorteventid_shortstatehash - .insert(&shorteventid.to_be_bytes(), &shortstatehash.to_be_bytes())?; - Ok(()) + .insert(&shorteventid.to_be_bytes(), &shortstatehash.to_be_bytes()); } - pub(super) fn get_forward_extremities(&self, room_id: &RoomId) -> Result>> { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xFF); - - self.roomid_pduleaves - .scan_prefix(prefix) - .map(|(_, bytes)| { - EventId::parse_arc( - utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("EventID in roomid_pduleaves is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("EventId in roomid_pduleaves is invalid.")) - }) - .collect() - } - - pub(super) fn set_forward_extremities( + pub(super) async fn set_forward_extremities( &self, room_id: &RoomId, event_ids: Vec, _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex - ) -> Result<()> { + ) { + let prefix = (room_id, Interfix); + self.roomid_pduleaves + .keys_raw_prefix(&prefix) + .ignore_err() + .ready_for_each(|key| self.roomid_pduleaves.remove(key)) + .await; + let mut prefix = room_id.as_bytes().to_vec(); prefix.push(0xFF); - - for (key, _) in self.roomid_pduleaves.scan_prefix(prefix.clone()) { - self.roomid_pduleaves.remove(&key)?; - } - for event_id in event_ids { let mut key = prefix.clone(); key.extend_from_slice(event_id.as_bytes()); - self.roomid_pduleaves.insert(&key, event_id.as_bytes())?; + self.roomid_pduleaves.insert(&key, event_id.as_bytes()); } - - Ok(()) } } diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index cb219bc0..c7f6605c 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -7,12 +7,14 @@ use std::{ }; use conduit::{ - utils::{calculate_hash, MutexMap, MutexMapGuard}, - warn, Error, PduEvent, Result, + err, + utils::{calculate_hash, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard}, + warn, PduEvent, Result, }; use data::Data; +use database::{Ignore, Interfix}; +use futures::{pin_mut, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{ - api::client::error::ErrorKind, events::{ room::{create::RoomCreateEventContent, member::RoomMemberEventContent}, AnyStrippedStateEvent, StateEventType, TimelineEventType, @@ -81,14 +83,16 @@ impl Service { _statediffremoved: Arc>, state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result<()> { - for event_id in statediffnew.iter().filter_map(|new| { + let event_ids = statediffnew.iter().stream().filter_map(|new| { self.services .state_compressor .parse_compressed_state_event(new) - .ok() - .map(|(_, id)| id) - }) { - let Some(pdu) = self.services.timeline.get_pdu_json(&event_id)? else { + .map_ok_or_else(|_| None, |(_, event_id)| Some(event_id)) + }); + + pin_mut!(event_ids); + while let Some(event_id) = event_ids.next().await { + let Ok(pdu) = self.services.timeline.get_pdu_json(&event_id).await else { continue; }; @@ -113,15 +117,10 @@ impl Service { continue; }; - self.services.state_cache.update_membership( - room_id, - &user_id, - membership_event, - &pdu.sender, - None, - None, - false, - )?; + self.services + .state_cache + .update_membership(room_id, &user_id, membership_event, &pdu.sender, None, None, false) + .await?; }, TimelineEventType::SpaceChild => { self.services @@ -135,10 +134,9 @@ impl Service { } } - self.services.state_cache.update_joined_count(room_id)?; + self.services.state_cache.update_joined_count(room_id).await; - self.db - .set_room_state(room_id, shortstatehash, state_lock)?; + self.db.set_room_state(room_id, shortstatehash, state_lock); Ok(()) } @@ -148,12 +146,16 @@ impl Service { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `eventid_statehash`. #[tracing::instrument(skip(self, state_ids_compressed), level = "debug")] - pub fn set_event_state( + pub async fn set_event_state( &self, event_id: &EventId, room_id: &RoomId, state_ids_compressed: Arc>, ) -> Result { - let shorteventid = self.services.short.get_or_create_shorteventid(event_id)?; + let shorteventid = self + .services + .short + .get_or_create_shorteventid(event_id) + .await; - let previous_shortstatehash = self.db.get_room_shortstatehash(room_id)?; + let previous_shortstatehash = self.db.get_room_shortstatehash(room_id).await; let state_hash = calculate_hash( &state_ids_compressed @@ -165,13 +167,18 @@ impl Service { let (shortstatehash, already_existed) = self .services .short - .get_or_create_shortstatehash(&state_hash)?; + .get_or_create_shortstatehash(&state_hash) + .await; if !already_existed { - let states_parents = previous_shortstatehash.map_or_else( - || Ok(Vec::new()), - |p| self.services.state_compressor.load_shortstatehash_info(p), - )?; + let states_parents = if let Ok(p) = previous_shortstatehash { + self.services + .state_compressor + .load_shortstatehash_info(p) + .await? + } else { + Vec::new() + }; let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { let statediffnew: HashSet<_> = state_ids_compressed @@ -198,7 +205,7 @@ impl Service { )?; } - self.db.set_event_state(shorteventid, shortstatehash)?; + self.db.set_event_state(shorteventid, shortstatehash); Ok(shortstatehash) } @@ -208,34 +215,40 @@ impl Service { /// This adds all current state events (not including the incoming event) /// to `stateid_pduid` and adds the incoming event to `eventid_statehash`. #[tracing::instrument(skip(self, new_pdu), level = "debug")] - pub fn append_to_state(&self, new_pdu: &PduEvent) -> Result { + pub async fn append_to_state(&self, new_pdu: &PduEvent) -> Result { let shorteventid = self .services .short - .get_or_create_shorteventid(&new_pdu.event_id)?; + .get_or_create_shorteventid(&new_pdu.event_id) + .await; - let previous_shortstatehash = self.get_room_shortstatehash(&new_pdu.room_id)?; + let previous_shortstatehash = self.get_room_shortstatehash(&new_pdu.room_id).await; - if let Some(p) = previous_shortstatehash { - self.db.set_event_state(shorteventid, p)?; + if let Ok(p) = previous_shortstatehash { + self.db.set_event_state(shorteventid, p); } if let Some(state_key) = &new_pdu.state_key { - let states_parents = previous_shortstatehash.map_or_else( - || Ok(Vec::new()), - #[inline] - |p| self.services.state_compressor.load_shortstatehash_info(p), - )?; + let states_parents = if let Ok(p) = previous_shortstatehash { + self.services + .state_compressor + .load_shortstatehash_info(p) + .await? + } else { + Vec::new() + }; let shortstatekey = self .services .short - .get_or_create_shortstatekey(&new_pdu.kind.to_string().into(), state_key)?; + .get_or_create_shortstatekey(&new_pdu.kind.to_string().into(), state_key) + .await; let new = self .services .state_compressor - .compress_state_event(shortstatekey, &new_pdu.event_id)?; + .compress_state_event(shortstatekey, &new_pdu.event_id) + .await; let replaces = states_parents .last() @@ -276,49 +289,55 @@ impl Service { } #[tracing::instrument(skip(self, invite_event), level = "debug")] - pub fn calculate_invite_state(&self, invite_event: &PduEvent) -> Result>> { + pub async fn calculate_invite_state(&self, invite_event: &PduEvent) -> Result>> { let mut state = Vec::new(); // Add recommended events - if let Some(e) = - self.services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomCreate, "")? + if let Ok(e) = self + .services + .state_accessor + .room_state_get(&invite_event.room_id, &StateEventType::RoomCreate, "") + .await { state.push(e.to_stripped_state_event()); } - if let Some(e) = - self.services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomJoinRules, "")? + if let Ok(e) = self + .services + .state_accessor + .room_state_get(&invite_event.room_id, &StateEventType::RoomJoinRules, "") + .await { state.push(e.to_stripped_state_event()); } - if let Some(e) = self.services.state_accessor.room_state_get( - &invite_event.room_id, - &StateEventType::RoomCanonicalAlias, - "", - )? { - state.push(e.to_stripped_state_event()); - } - if let Some(e) = - self.services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomAvatar, "")? + if let Ok(e) = self + .services + .state_accessor + .room_state_get(&invite_event.room_id, &StateEventType::RoomCanonicalAlias, "") + .await { state.push(e.to_stripped_state_event()); } - if let Some(e) = - self.services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomName, "")? + if let Ok(e) = self + .services + .state_accessor + .room_state_get(&invite_event.room_id, &StateEventType::RoomAvatar, "") + .await { state.push(e.to_stripped_state_event()); } - if let Some(e) = self.services.state_accessor.room_state_get( - &invite_event.room_id, - &StateEventType::RoomMember, - invite_event.sender.as_str(), - )? { + if let Ok(e) = self + .services + .state_accessor + .room_state_get(&invite_event.room_id, &StateEventType::RoomName, "") + .await + { + state.push(e.to_stripped_state_event()); + } + if let Ok(e) = self + .services + .state_accessor + .room_state_get(&invite_event.room_id, &StateEventType::RoomMember, invite_event.sender.as_str()) + .await + { state.push(e.to_stripped_state_event()); } @@ -333,101 +352,108 @@ impl Service { room_id: &RoomId, shortstatehash: u64, mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex - ) -> Result<()> { - self.db.set_room_state(room_id, shortstatehash, mutex_lock) + ) { + self.db.set_room_state(room_id, shortstatehash, mutex_lock); } /// Returns the room's version. #[tracing::instrument(skip(self), level = "debug")] - pub fn get_room_version(&self, room_id: &RoomId) -> Result { - let create_event = self - .services + pub async fn get_room_version(&self, room_id: &RoomId) -> Result { + self.services .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "")?; - - let create_event_content: RoomCreateEventContent = create_event - .as_ref() - .map(|create_event| { - serde_json::from_str(create_event.content.get()).map_err(|e| { - warn!("Invalid create event: {}", e); - Error::bad_database("Invalid create event in db.") - }) - }) - .transpose()? - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "No create event found"))?; - - Ok(create_event_content.room_version) + .room_state_get_content(room_id, &StateEventType::RoomCreate, "") + .await + .map(|content: RoomCreateEventContent| content.room_version) + .map_err(|e| err!(Request(NotFound("No create event found: {e:?}")))) } #[inline] - pub fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result> { - self.db.get_room_shortstatehash(room_id) + pub async fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result { + self.db.get_room_shortstatehash(room_id).await } - pub fn get_forward_extremities(&self, room_id: &RoomId) -> Result>> { - self.db.get_forward_extremities(room_id) + pub fn get_forward_extremities<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + '_ { + let prefix = (room_id, Interfix); + + self.db + .roomid_pduleaves + .keys_prefix(&prefix) + .map_ok(|(_, event_id): (Ignore, &EventId)| event_id) + .ignore_err() } - pub fn set_forward_extremities( + pub async fn set_forward_extremities( &self, room_id: &RoomId, event_ids: Vec, state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex - ) -> Result<()> { + ) { self.db .set_forward_extremities(room_id, event_ids, state_lock) + .await; } /// This fetches auth events from the current state. #[tracing::instrument(skip(self), level = "debug")] - pub fn get_auth_events( + pub async fn get_auth_events( &self, room_id: &RoomId, kind: &TimelineEventType, sender: &UserId, state_key: Option<&str>, content: &serde_json::value::RawValue, ) -> Result>> { - let Some(shortstatehash) = self.get_room_shortstatehash(room_id)? else { + let Ok(shortstatehash) = self.get_room_shortstatehash(room_id).await else { return Ok(HashMap::new()); }; - let auth_events = - state_res::auth_types_for_event(kind, sender, state_key, content).expect("content is a valid JSON object"); + let auth_events = state_res::auth_types_for_event(kind, sender, state_key, content)?; - let mut sauthevents = auth_events - .into_iter() + let mut sauthevents: HashMap<_, _> = auth_events + .iter() + .stream() .filter_map(|(event_type, state_key)| { self.services .short - .get_shortstatekey(&event_type.to_string().into(), &state_key) - .ok() - .flatten() - .map(|s| (s, (event_type, state_key))) + .get_shortstatekey(event_type, state_key) + .map_ok(move |s| (s, (event_type, state_key))) + .map(Result::ok) }) - .collect::>(); + .collect() + .await; let full_state = self .services .state_compressor - .load_shortstatehash_info(shortstatehash)? + .load_shortstatehash_info(shortstatehash) + .await + .map_err(|e| { + err!(Database( + "Missing shortstatehash info for {room_id:?} at {shortstatehash:?}: {e:?}" + )) + })? .pop() .expect("there is always one layer") .1; - Ok(full_state - .iter() - .filter_map(|compressed| { - self.services - .state_compressor - .parse_compressed_state_event(compressed) - .ok() - }) - .filter_map(|(shortstatekey, event_id)| sauthevents.remove(&shortstatekey).map(|k| (k, event_id))) - .filter_map(|(k, event_id)| { - self.services - .timeline - .get_pdu(&event_id) - .ok() - .flatten() - .map(|pdu| (k, pdu)) - }) - .collect()) + let mut ret = HashMap::new(); + for compressed in full_state.iter() { + let Ok((shortstatekey, event_id)) = self + .services + .state_compressor + .parse_compressed_state_event(compressed) + .await + else { + continue; + }; + + let Some((ty, state_key)) = sauthevents.remove(&shortstatekey) else { + continue; + }; + + let Ok(pdu) = self.services.timeline.get_pdu(&event_id).await else { + continue; + }; + + ret.insert((ty.to_owned(), state_key.to_owned()), pdu); + } + + Ok(ret) } } diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 4c85148d..79a98325 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,7 +1,8 @@ use std::{collections::HashMap, sync::Arc}; -use conduit::{utils, Error, PduEvent, Result}; -use database::Map; +use conduit::{err, PduEvent, Result}; +use database::{Deserialized, Map}; +use futures::TryFutureExt; use ruma::{events::StateEventType, EventId, RoomId}; use crate::{rooms, Dep}; @@ -39,17 +40,22 @@ impl Data { let full_state = self .services .state_compressor - .load_shortstatehash_info(shortstatehash)? + .load_shortstatehash_info(shortstatehash) + .await + .map_err(|e| err!(Database("Missing state IDs: {e}")))? .pop() .expect("there is always one layer") .1; + let mut result = HashMap::new(); let mut i: u8 = 0; for compressed in full_state.iter() { let parsed = self .services .state_compressor - .parse_compressed_state_event(compressed)?; + .parse_compressed_state_event(compressed) + .await?; + result.insert(parsed.0, parsed.1); i = i.wrapping_add(1); @@ -57,6 +63,7 @@ impl Data { tokio::task::yield_now().await; } } + Ok(result) } @@ -67,7 +74,8 @@ impl Data { let full_state = self .services .state_compressor - .load_shortstatehash_info(shortstatehash)? + .load_shortstatehash_info(shortstatehash) + .await? .pop() .expect("there is always one layer") .1; @@ -78,18 +86,13 @@ impl Data { let (_, eventid) = self .services .state_compressor - .parse_compressed_state_event(compressed)?; - if let Some(pdu) = self.services.timeline.get_pdu(&eventid)? { - result.insert( - ( - pdu.kind.to_string().into(), - pdu.state_key - .as_ref() - .ok_or_else(|| Error::bad_database("State event has no state key."))? - .clone(), - ), - pdu, - ); + .parse_compressed_state_event(compressed) + .await?; + + if let Ok(pdu) = self.services.timeline.get_pdu(&eventid).await { + if let Some(state_key) = pdu.state_key.as_ref() { + result.insert((pdu.kind.to_string().into(), state_key.clone()), pdu); + } } i = i.wrapping_add(1); @@ -101,61 +104,63 @@ impl Data { Ok(result) } - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). #[allow(clippy::unused_self)] - pub(super) fn state_get_id( + pub(super) async fn state_get_id( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, - ) -> Result>> { - let Some(shortstatekey) = self + ) -> Result> { + let shortstatekey = self .services .short - .get_shortstatekey(event_type, state_key)? - else { - return Ok(None); - }; + .get_shortstatekey(event_type, state_key) + .await?; + let full_state = self .services .state_compressor - .load_shortstatehash_info(shortstatehash)? + .load_shortstatehash_info(shortstatehash) + .await + .map_err(|e| err!(Database(error!(?event_type, ?state_key, "Missing state: {e:?}"))))? .pop() .expect("there is always one layer") .1; - Ok(full_state + + let compressed = full_state .iter() .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) - .and_then(|compressed| { - self.services - .state_compressor - .parse_compressed_state_event(compressed) - .ok() - .map(|(_, id)| id) - })) + .ok_or(err!(Database("No shortstatekey in compressed state")))?; + + self.services + .state_compressor + .parse_compressed_state_event(compressed) + .map_ok(|(_, id)| id) + .map_err(|e| { + err!(Database(error!( + ?event_type, + ?state_key, + ?shortstatekey, + "Failed to parse compressed: {e:?}" + ))) + }) + .await } - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - pub(super) fn state_get( + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). + pub(super) async fn state_get( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, - ) -> Result>> { - self.state_get_id(shortstatehash, event_type, state_key)? - .map_or(Ok(None), |event_id| self.services.timeline.get_pdu(&event_id)) + ) -> Result> { + self.state_get_id(shortstatehash, event_type, state_key) + .and_then(|event_id| async move { self.services.timeline.get_pdu(&event_id).await }) + .await } /// Returns the state hash for this pdu. - pub(super) fn pdu_shortstatehash(&self, event_id: &EventId) -> Result> { + pub(super) async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { self.eventid_shorteventid - .get(event_id.as_bytes())? - .map_or(Ok(None), |shorteventid| { - self.shorteventid_shortstatehash - .get(&shorteventid)? - .map(|bytes| { - utils::u64_from_bytes(&bytes).map_err(|_| { - Error::bad_database("Invalid shortstatehash bytes in shorteventid_shortstatehash") - }) - }) - .transpose() - }) + .qry(event_id) + .and_then(|shorteventid| self.shorteventid_shortstatehash.qry(&shorteventid)) + .await + .deserialized() } /// Returns the full room state. @@ -163,34 +168,33 @@ impl Data { pub(super) async fn room_state_full( &self, room_id: &RoomId, ) -> Result>> { - if let Some(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id)? { - self.state_full(current_shortstatehash).await - } else { - Ok(HashMap::new()) - } + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_full(shortstatehash)) + .map_err(|e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .await } - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - pub(super) fn room_state_get_id( + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). + pub(super) async fn room_state_get_id( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result>> { - if let Some(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id)? { - self.state_get_id(current_shortstatehash, event_type, state_key) - } else { - Ok(None) - } + ) -> Result> { + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get_id(shortstatehash, event_type, state_key)) + .await } - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - pub(super) fn room_state_get( + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). + pub(super) async fn room_state_get( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result>> { - if let Some(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id)? { - self.state_get(current_shortstatehash, event_type, state_key) - } else { - Ok(None) - } + ) -> Result> { + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get(shortstatehash, event_type, state_key)) + .await } } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 58fa31b3..4c28483c 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -6,8 +6,13 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{err, error, pdu::PduBuilder, utils::math::usize_from_f64, warn, Error, PduEvent, Result}; -use data::Data; +use conduit::{ + err, error, + pdu::PduBuilder, + utils::{math::usize_from_f64, ReadyExt}, + Error, PduEvent, Result, +}; +use futures::StreamExt; use lru_cache::LruCache; use ruma::{ events::{ @@ -31,8 +36,10 @@ use ruma::{ EventEncryptionAlgorithm, EventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; +use serde::Deserialize; use serde_json::value::to_raw_value; +use self::data::Data; use crate::{rooms, rooms::state::RoomMutexGuard, Dep}; pub struct Service { @@ -99,54 +106,58 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] - pub fn state_get_id( + pub async fn state_get_id( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, - ) -> Result>> { - self.db.state_get_id(shortstatehash, event_type, state_key) + ) -> Result> { + self.db + .state_get_id(shortstatehash, event_type, state_key) + .await } /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). #[inline] - pub fn state_get( + pub async fn state_get( &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, - ) -> Result>> { - self.db.state_get(shortstatehash, event_type, state_key) + ) -> Result> { + self.db + .state_get(shortstatehash, event_type, state_key) + .await } /// Get membership for given user in state - fn user_membership(&self, shortstatehash: u64, user_id: &UserId) -> Result { - self.state_get(shortstatehash, &StateEventType::RoomMember, user_id.as_str())? - .map_or(Ok(MembershipState::Leave), |s| { + async fn user_membership(&self, shortstatehash: u64, user_id: &UserId) -> MembershipState { + self.state_get(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) + .await + .map_or(MembershipState::Leave, |s| { serde_json::from_str(s.content.get()) .map(|c: RoomMemberEventContent| c.membership) .map_err(|_| Error::bad_database("Invalid room membership event in database.")) + .unwrap() }) } /// The user was a joined member at this state (potentially in the past) #[inline] - fn user_was_joined(&self, shortstatehash: u64, user_id: &UserId) -> bool { - self.user_membership(shortstatehash, user_id) - .is_ok_and(|s| s == MembershipState::Join) - // Return sensible default, i.e. - // false + async fn user_was_joined(&self, shortstatehash: u64, user_id: &UserId) -> bool { + self.user_membership(shortstatehash, user_id).await == MembershipState::Join } /// The user was an invited or joined room member at this state (potentially /// in the past) #[inline] - fn user_was_invited(&self, shortstatehash: u64, user_id: &UserId) -> bool { - self.user_membership(shortstatehash, user_id) - .is_ok_and(|s| s == MembershipState::Join || s == MembershipState::Invite) - // Return sensible default, i.e. false + async fn user_was_invited(&self, shortstatehash: u64, user_id: &UserId) -> bool { + let s = self.user_membership(shortstatehash, user_id).await; + s == MembershipState::Join || s == MembershipState::Invite } /// Whether a server is allowed to see an event through federation, based on /// the room's history_visibility at that event's state. #[tracing::instrument(skip(self, origin, room_id, event_id))] - pub fn server_can_see_event(&self, origin: &ServerName, room_id: &RoomId, event_id: &EventId) -> Result { - let Some(shortstatehash) = self.pdu_shortstatehash(event_id)? else { + pub async fn server_can_see_event( + &self, origin: &ServerName, room_id: &RoomId, event_id: &EventId, + ) -> Result { + let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { return Ok(true); }; @@ -160,8 +171,9 @@ impl Service { } let history_visibility = self - .state_get(shortstatehash, &StateEventType::RoomHistoryVisibility, "")? - .map_or(Ok(HistoryVisibility::Shared), |s| { + .state_get(shortstatehash, &StateEventType::RoomHistoryVisibility, "") + .await + .map_or(HistoryVisibility::Shared, |s| { serde_json::from_str(s.content.get()) .map(|c: RoomHistoryVisibilityEventContent| c.history_visibility) .map_err(|e| { @@ -171,25 +183,28 @@ impl Service { ); Error::bad_database("Invalid history visibility event in database.") }) - }) - .unwrap_or(HistoryVisibility::Shared); + .unwrap() + }); - let mut current_server_members = self + let current_server_members = self .services .state_cache .room_members(room_id) - .filter_map(Result::ok) - .filter(|member| member.server_name() == origin); + .ready_filter(|member| member.server_name() == origin); let visibility = match history_visibility { HistoryVisibility::WorldReadable | HistoryVisibility::Shared => true, HistoryVisibility::Invited => { // Allow if any member on requesting server was AT LEAST invited, else deny - current_server_members.any(|member| self.user_was_invited(shortstatehash, &member)) + current_server_members + .any(|member| self.user_was_invited(shortstatehash, member)) + .await }, HistoryVisibility::Joined => { // Allow if any member on requested server was joined, else deny - current_server_members.any(|member| self.user_was_joined(shortstatehash, &member)) + current_server_members + .any(|member| self.user_was_joined(shortstatehash, member)) + .await }, _ => { error!("Unknown history visibility {history_visibility}"); @@ -208,9 +223,9 @@ impl Service { /// Whether a user is allowed to see an event, based on /// the room's history_visibility at that event's state. #[tracing::instrument(skip(self, user_id, room_id, event_id))] - pub fn user_can_see_event(&self, user_id: &UserId, room_id: &RoomId, event_id: &EventId) -> Result { - let Some(shortstatehash) = self.pdu_shortstatehash(event_id)? else { - return Ok(true); + pub async fn user_can_see_event(&self, user_id: &UserId, room_id: &RoomId, event_id: &EventId) -> bool { + let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { + return true; }; if let Some(visibility) = self @@ -219,14 +234,15 @@ impl Service { .unwrap() .get_mut(&(user_id.to_owned(), shortstatehash)) { - return Ok(*visibility); + return *visibility; } - let currently_member = self.services.state_cache.is_joined(user_id, room_id)?; + let currently_member = self.services.state_cache.is_joined(user_id, room_id).await; let history_visibility = self - .state_get(shortstatehash, &StateEventType::RoomHistoryVisibility, "")? - .map_or(Ok(HistoryVisibility::Shared), |s| { + .state_get(shortstatehash, &StateEventType::RoomHistoryVisibility, "") + .await + .map_or(HistoryVisibility::Shared, |s| { serde_json::from_str(s.content.get()) .map(|c: RoomHistoryVisibilityEventContent| c.history_visibility) .map_err(|e| { @@ -236,19 +252,19 @@ impl Service { ); Error::bad_database("Invalid history visibility event in database.") }) - }) - .unwrap_or(HistoryVisibility::Shared); + .unwrap() + }); let visibility = match history_visibility { HistoryVisibility::WorldReadable => true, HistoryVisibility::Shared => currently_member, HistoryVisibility::Invited => { // Allow if any member on requesting server was AT LEAST invited, else deny - self.user_was_invited(shortstatehash, user_id) + self.user_was_invited(shortstatehash, user_id).await }, HistoryVisibility::Joined => { // Allow if any member on requested server was joined, else deny - self.user_was_joined(shortstatehash, user_id) + self.user_was_joined(shortstatehash, user_id).await }, _ => { error!("Unknown history visibility {history_visibility}"); @@ -261,17 +277,18 @@ impl Service { .unwrap() .insert((user_id.to_owned(), shortstatehash), visibility); - Ok(visibility) + visibility } /// Whether a user is allowed to see an event, based on /// the room's history_visibility at that event's state. #[tracing::instrument(skip(self, user_id, room_id))] - pub fn user_can_see_state_events(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let currently_member = self.services.state_cache.is_joined(user_id, room_id)?; + pub async fn user_can_see_state_events(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let currently_member = self.services.state_cache.is_joined(user_id, room_id).await; let history_visibility = self - .room_state_get(room_id, &StateEventType::RoomHistoryVisibility, "")? + .room_state_get(room_id, &StateEventType::RoomHistoryVisibility, "") + .await .map_or(Ok(HistoryVisibility::Shared), |s| { serde_json::from_str(s.content.get()) .map(|c: RoomHistoryVisibilityEventContent| c.history_visibility) @@ -285,11 +302,13 @@ impl Service { }) .unwrap_or(HistoryVisibility::Shared); - Ok(currently_member || history_visibility == HistoryVisibility::WorldReadable) + currently_member || history_visibility == HistoryVisibility::WorldReadable } /// Returns the state hash for this pdu. - pub fn pdu_shortstatehash(&self, event_id: &EventId) -> Result> { self.db.pdu_shortstatehash(event_id) } + pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { + self.db.pdu_shortstatehash(event_id).await + } /// Returns the full room state. #[tracing::instrument(skip(self), level = "debug")] @@ -300,47 +319,61 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] - pub fn room_state_get_id( + pub async fn room_state_get_id( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result>> { - self.db.room_state_get_id(room_id, event_type, state_key) + ) -> Result> { + self.db + .room_state_get_id(room_id, event_type, state_key) + .await } /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] - pub fn room_state_get( + pub async fn room_state_get( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result>> { - self.db.room_state_get(room_id, event_type, state_key) + ) -> Result> { + self.db.room_state_get(room_id, event_type, state_key).await } - pub fn get_name(&self, room_id: &RoomId) -> Result> { - self.room_state_get(room_id, &StateEventType::RoomName, "")? - .map_or(Ok(None), |s| { - Ok(serde_json::from_str(s.content.get()).map_or_else(|_| None, |c: RoomNameEventContent| Some(c.name))) - }) + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). + pub async fn room_state_get_content( + &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, + ) -> Result + where + T: for<'de> Deserialize<'de> + Send, + { + use serde_json::from_str; + + self.room_state_get(room_id, event_type, state_key) + .await + .and_then(|event| from_str::(event.content.get()).map_err(Into::into)) } - pub fn get_avatar(&self, room_id: &RoomId) -> Result> { - self.room_state_get(room_id, &StateEventType::RoomAvatar, "")? - .map_or(Ok(ruma::JsOption::Undefined), |s| { + pub async fn get_name(&self, room_id: &RoomId) -> Result { + self.room_state_get_content(room_id, &StateEventType::RoomName, "") + .await + .map(|c: RoomNameEventContent| c.name) + } + + pub async fn get_avatar(&self, room_id: &RoomId) -> ruma::JsOption { + self.room_state_get(room_id, &StateEventType::RoomAvatar, "") + .await + .map_or(ruma::JsOption::Undefined, |s| { serde_json::from_str(s.content.get()) .map_err(|_| Error::bad_database("Invalid room avatar event in database.")) + .unwrap() }) } - pub fn get_member(&self, room_id: &RoomId, user_id: &UserId) -> Result> { - self.room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str())? - .map_or(Ok(None), |s| { - serde_json::from_str(s.content.get()) - .map_err(|_| Error::bad_database("Invalid room member event in database.")) - }) + pub async fn get_member(&self, room_id: &RoomId, user_id: &UserId) -> Result { + self.room_state_get_content(room_id, &StateEventType::RoomMember, user_id.as_str()) + .await } - pub fn user_can_invite( + pub async fn user_can_invite( &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &RoomMutexGuard, - ) -> Result { + ) -> bool { let content = to_raw_value(&RoomMemberEventContent::new(MembershipState::Invite)) .expect("Event content always serializes"); @@ -353,122 +386,101 @@ impl Service { timestamp: None, }; - Ok(self - .services + self.services .timeline .create_hash_and_sign_event(new_event, sender, room_id, state_lock) - .is_ok()) + .await + .is_ok() } /// Checks if guests are able to view room content without joining - pub fn is_world_readable(&self, room_id: &RoomId) -> Result { - self.room_state_get(room_id, &StateEventType::RoomHistoryVisibility, "")? - .map_or(Ok(false), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomHistoryVisibilityEventContent| { - c.history_visibility == HistoryVisibility::WorldReadable - }) - .map_err(|e| { - error!( - "Invalid room history visibility event in database for room {room_id}, assuming not world \ - readable: {e} " - ); - Error::bad_database("Invalid room history visibility event in database.") - }) - }) + pub async fn is_world_readable(&self, room_id: &RoomId) -> bool { + self.room_state_get_content(room_id, &StateEventType::RoomHistoryVisibility, "") + .await + .map(|c: RoomHistoryVisibilityEventContent| c.history_visibility == HistoryVisibility::WorldReadable) + .unwrap_or(false) } /// Checks if guests are able to join a given room - pub fn guest_can_join(&self, room_id: &RoomId) -> Result { - self.room_state_get(room_id, &StateEventType::RoomGuestAccess, "")? - .map_or(Ok(false), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomGuestAccessEventContent| c.guest_access == GuestAccess::CanJoin) - .map_err(|_| Error::bad_database("Invalid room guest access event in database.")) - }) + pub async fn guest_can_join(&self, room_id: &RoomId) -> bool { + self.room_state_get_content(room_id, &StateEventType::RoomGuestAccess, "") + .await + .map(|c: RoomGuestAccessEventContent| c.guest_access == GuestAccess::CanJoin) + .unwrap_or(false) } /// Gets the primary alias from canonical alias event - pub fn get_canonical_alias(&self, room_id: &RoomId) -> Result, Error> { - self.room_state_get(room_id, &StateEventType::RoomCanonicalAlias, "")? - .map_or(Ok(None), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomCanonicalAliasEventContent| c.alias) - .map_err(|_| Error::bad_database("Invalid canonical alias event in database.")) + pub async fn get_canonical_alias(&self, room_id: &RoomId) -> Result { + self.room_state_get_content(room_id, &StateEventType::RoomCanonicalAlias, "") + .await + .and_then(|c: RoomCanonicalAliasEventContent| { + c.alias + .ok_or_else(|| err!(Request(NotFound("No alias found in event content.")))) }) } /// Gets the room topic - pub fn get_room_topic(&self, room_id: &RoomId) -> Result, Error> { - self.room_state_get(room_id, &StateEventType::RoomTopic, "")? - .map_or(Ok(None), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomTopicEventContent| Some(c.topic)) - .map_err(|e| { - error!("Invalid room topic event in database for room {room_id}: {e}"); - Error::bad_database("Invalid room topic event in database.") - }) - }) + pub async fn get_room_topic(&self, room_id: &RoomId) -> Result { + self.room_state_get_content(room_id, &StateEventType::RoomTopic, "") + .await + .map(|c: RoomTopicEventContent| c.topic) } /// Checks if a given user can redact a given event /// /// If federation is true, it allows redaction events from any user of the /// same server as the original event sender - pub fn user_can_redact( + pub async fn user_can_redact( &self, redacts: &EventId, sender: &UserId, room_id: &RoomId, federation: bool, ) -> Result { - self.room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? - .map_or_else( - || { - // Falling back on m.room.create to judge power level - if let Some(pdu) = self.room_state_get(room_id, &StateEventType::RoomCreate, "")? { - Ok(pdu.sender == sender - || if let Ok(Some(pdu)) = self.services.timeline.get_pdu(redacts) { - pdu.sender == sender - } else { - false - }) + if let Ok(event) = self + .room_state_get(room_id, &StateEventType::RoomPowerLevels, "") + .await + { + let Ok(event) = serde_json::from_str(event.content.get()) + .map(|content: RoomPowerLevelsEventContent| content.into()) + .map(|event: RoomPowerLevels| event) + else { + return Ok(false); + }; + + Ok(event.user_can_redact_event_of_other(sender) + || event.user_can_redact_own_event(sender) + && if let Ok(pdu) = self.services.timeline.get_pdu(redacts).await { + if federation { + pdu.sender.server_name() == sender.server_name() + } else { + pdu.sender == sender + } } else { - Err(Error::bad_database( - "No m.room.power_levels or m.room.create events in database for room", - )) - } - }, - |event| { - serde_json::from_str(event.content.get()) - .map(|content: RoomPowerLevelsEventContent| content.into()) - .map(|event: RoomPowerLevels| { - event.user_can_redact_event_of_other(sender) - || event.user_can_redact_own_event(sender) - && if let Ok(Some(pdu)) = self.services.timeline.get_pdu(redacts) { - if federation { - pdu.sender.server_name() == sender.server_name() - } else { - pdu.sender == sender - } - } else { - false - } - }) - .map_err(|_| Error::bad_database("Invalid m.room.power_levels event in database")) - }, - ) + false + }) + } else { + // Falling back on m.room.create to judge power level + if let Ok(pdu) = self + .room_state_get(room_id, &StateEventType::RoomCreate, "") + .await + { + Ok(pdu.sender == sender + || if let Ok(pdu) = self.services.timeline.get_pdu(redacts).await { + pdu.sender == sender + } else { + false + }) + } else { + Err(Error::bad_database( + "No m.room.power_levels or m.room.create events in database for room", + )) + } + } } /// Returns the join rule (`SpaceRoomJoinRule`) for a given room - pub fn get_join_rule(&self, room_id: &RoomId) -> Result<(SpaceRoomJoinRule, Vec), Error> { - Ok(self - .room_state_get(room_id, &StateEventType::RoomJoinRules, "")? - .map(|s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomJoinRulesEventContent| { - (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule)) - }) - .map_err(|e| err!(Database(error!("Invalid room join rule event in database: {e}")))) - }) - .transpose()? - .unwrap_or((SpaceRoomJoinRule::Invite, vec![]))) + pub async fn get_join_rule(&self, room_id: &RoomId) -> Result<(SpaceRoomJoinRule, Vec)> { + self.room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") + .await + .map(|c: RoomJoinRulesEventContent| (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule))) + .or_else(|_| Ok((SpaceRoomJoinRule::Invite, vec![]))) } /// Returns an empty vec if not a restricted room @@ -487,25 +499,21 @@ impl Service { room_ids } - pub fn get_room_type(&self, room_id: &RoomId) -> Result> { - Ok(self - .room_state_get(room_id, &StateEventType::RoomCreate, "")? - .map(|s| { - serde_json::from_str::(s.content.get()) - .map_err(|e| err!(Database(error!("Invalid room create event in database: {e}")))) + pub async fn get_room_type(&self, room_id: &RoomId) -> Result { + self.room_state_get_content(room_id, &StateEventType::RoomCreate, "") + .await + .and_then(|content: RoomCreateEventContent| { + content + .room_type + .ok_or_else(|| err!(Request(NotFound("No type found in event content")))) }) - .transpose()? - .and_then(|e| e.room_type)) } /// Gets the room's encryption algorithm if `m.room.encryption` state event /// is found - pub fn get_room_encryption(&self, room_id: &RoomId) -> Result> { - self.room_state_get(room_id, &StateEventType::RoomEncryption, "")? - .map_or(Ok(None), |s| { - serde_json::from_str::(s.content.get()) - .map(|content| Some(content.algorithm)) - .map_err(|e| err!(Database(error!("Invalid room encryption event in database: {e}")))) - }) + pub async fn get_room_encryption(&self, room_id: &RoomId) -> Result { + self.room_state_get_content(room_id, &StateEventType::RoomEncryption, "") + .await + .map(|content: RoomEncryptionEventContent| content.algorithm) } } diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 19c73ea1..38e504f6 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -1,43 +1,42 @@ use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, sync::{Arc, RwLock}, }; -use conduit::{utils, Error, Result}; -use database::Map; -use itertools::Itertools; +use conduit::{utils, utils::stream::TryIgnore, Error, Result}; +use database::{Deserialized, Interfix, Map}; +use futures::{Stream, StreamExt}; use ruma::{ events::{AnyStrippedStateEvent, AnySyncStateEvent}, serde::Raw, - OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, + OwnedRoomId, RoomId, UserId, }; -use crate::{appservice::RegistrationInfo, globals, users, Dep}; +use crate::{globals, Dep}; -type StrippedStateEventIter<'a> = Box>)>> + 'a>; -type AnySyncStateEventIter<'a> = Box>)>> + 'a>; type AppServiceInRoomCache = RwLock>>; +type StrippedStateEventItem = (OwnedRoomId, Vec>); +type SyncStateEventItem = (OwnedRoomId, Vec>); pub(super) struct Data { pub(super) appservice_in_room_cache: AppServiceInRoomCache, - roomid_invitedcount: Arc, - roomid_inviteviaservers: Arc, - roomid_joinedcount: Arc, - roomserverids: Arc, - roomuserid_invitecount: Arc, - roomuserid_joined: Arc, - roomuserid_leftcount: Arc, - roomuseroncejoinedids: Arc, - serverroomids: Arc, - userroomid_invitestate: Arc, - userroomid_joined: Arc, - userroomid_leftstate: Arc, + pub(super) roomid_invitedcount: Arc, + pub(super) roomid_inviteviaservers: Arc, + pub(super) roomid_joinedcount: Arc, + pub(super) roomserverids: Arc, + pub(super) roomuserid_invitecount: Arc, + pub(super) roomuserid_joined: Arc, + pub(super) roomuserid_leftcount: Arc, + pub(super) roomuseroncejoinedids: Arc, + pub(super) serverroomids: Arc, + pub(super) userroomid_invitestate: Arc, + pub(super) userroomid_joined: Arc, + pub(super) userroomid_leftstate: Arc, services: Services, } struct Services { globals: Dep, - users: Dep, } impl Data { @@ -59,19 +58,18 @@ impl Data { userroomid_leftstate: db["userroomid_leftstate"].clone(), services: Services { globals: args.depend::("globals"), - users: args.depend::("users"), }, } } - pub(super) fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { + pub(super) fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); - self.roomuseroncejoinedids.insert(&userroom_id, &[]) + self.roomuseroncejoinedids.insert(&userroom_id, &[]); } - pub(super) fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { + pub(super) fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { let roomid = room_id.as_bytes().to_vec(); let mut roomuser_id = roomid.clone(); @@ -82,64 +80,17 @@ impl Data { userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); - self.userroomid_joined.insert(&userroom_id, &[])?; - self.roomuserid_joined.insert(&roomuser_id, &[])?; - self.userroomid_invitestate.remove(&userroom_id)?; - self.roomuserid_invitecount.remove(&roomuser_id)?; - self.userroomid_leftstate.remove(&userroom_id)?; - self.roomuserid_leftcount.remove(&roomuser_id)?; + self.userroomid_joined.insert(&userroom_id, &[]); + self.roomuserid_joined.insert(&roomuser_id, &[]); + self.userroomid_invitestate.remove(&userroom_id); + self.roomuserid_invitecount.remove(&roomuser_id); + self.userroomid_leftstate.remove(&userroom_id); + self.roomuserid_leftcount.remove(&roomuser_id); - self.roomid_inviteviaservers.remove(&roomid)?; - - Ok(()) + self.roomid_inviteviaservers.remove(&roomid); } - pub(super) fn mark_as_invited( - &self, user_id: &UserId, room_id: &RoomId, last_state: Option>>, - invite_via: Option>, - ) -> Result<()> { - let mut roomuser_id = room_id.as_bytes().to_vec(); - roomuser_id.push(0xFF); - roomuser_id.extend_from_slice(user_id.as_bytes()); - - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); - - self.userroomid_invitestate.insert( - &userroom_id, - &serde_json::to_vec(&last_state.unwrap_or_default()).expect("state to bytes always works"), - )?; - self.roomuserid_invitecount - .insert(&roomuser_id, &self.services.globals.next_count()?.to_be_bytes())?; - self.userroomid_joined.remove(&userroom_id)?; - self.roomuserid_joined.remove(&roomuser_id)?; - self.userroomid_leftstate.remove(&userroom_id)?; - self.roomuserid_leftcount.remove(&roomuser_id)?; - - if let Some(servers) = invite_via { - let mut prev_servers = self - .servers_invite_via(room_id) - .filter_map(Result::ok) - .collect_vec(); - #[allow(clippy::redundant_clone)] // this is a necessary clone? - prev_servers.append(servers.clone().as_mut()); - let servers = prev_servers.iter().rev().unique().rev().collect_vec(); - - let servers = servers - .iter() - .map(|server| server.as_bytes()) - .collect_vec() - .join(&[0xFF][..]); - - self.roomid_inviteviaservers - .insert(room_id.as_bytes(), &servers)?; - } - - Ok(()) - } - - pub(super) fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { + pub(super) fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) { let roomid = room_id.as_bytes().to_vec(); let mut roomuser_id = roomid.clone(); @@ -153,115 +104,20 @@ impl Data { self.userroomid_leftstate.insert( &userroom_id, &serde_json::to_vec(&Vec::>::new()).unwrap(), - )?; // TODO + ); // TODO self.roomuserid_leftcount - .insert(&roomuser_id, &self.services.globals.next_count()?.to_be_bytes())?; - self.userroomid_joined.remove(&userroom_id)?; - self.roomuserid_joined.remove(&roomuser_id)?; - self.userroomid_invitestate.remove(&userroom_id)?; - self.roomuserid_invitecount.remove(&roomuser_id)?; + .insert(&roomuser_id, &self.services.globals.next_count().unwrap().to_be_bytes()); + self.userroomid_joined.remove(&userroom_id); + self.roomuserid_joined.remove(&roomuser_id); + self.userroomid_invitestate.remove(&userroom_id); + self.roomuserid_invitecount.remove(&roomuser_id); - self.roomid_inviteviaservers.remove(&roomid)?; - - Ok(()) - } - - pub(super) fn update_joined_count(&self, room_id: &RoomId) -> Result<()> { - let mut joinedcount = 0_u64; - let mut invitedcount = 0_u64; - let mut joined_servers = HashSet::new(); - - for joined in self.room_members(room_id).filter_map(Result::ok) { - joined_servers.insert(joined.server_name().to_owned()); - joinedcount = joinedcount.saturating_add(1); - } - - for _invited in self.room_members_invited(room_id).filter_map(Result::ok) { - invitedcount = invitedcount.saturating_add(1); - } - - self.roomid_joinedcount - .insert(room_id.as_bytes(), &joinedcount.to_be_bytes())?; - - self.roomid_invitedcount - .insert(room_id.as_bytes(), &invitedcount.to_be_bytes())?; - - for old_joined_server in self.room_servers(room_id).filter_map(Result::ok) { - if !joined_servers.remove(&old_joined_server) { - // Server not in room anymore - let mut roomserver_id = room_id.as_bytes().to_vec(); - roomserver_id.push(0xFF); - roomserver_id.extend_from_slice(old_joined_server.as_bytes()); - - let mut serverroom_id = old_joined_server.as_bytes().to_vec(); - serverroom_id.push(0xFF); - serverroom_id.extend_from_slice(room_id.as_bytes()); - - self.roomserverids.remove(&roomserver_id)?; - self.serverroomids.remove(&serverroom_id)?; - } - } - - // Now only new servers are in joined_servers anymore - for server in joined_servers { - let mut roomserver_id = room_id.as_bytes().to_vec(); - roomserver_id.push(0xFF); - roomserver_id.extend_from_slice(server.as_bytes()); - - let mut serverroom_id = server.as_bytes().to_vec(); - serverroom_id.push(0xFF); - serverroom_id.extend_from_slice(room_id.as_bytes()); - - self.roomserverids.insert(&roomserver_id, &[])?; - self.serverroomids.insert(&serverroom_id, &[])?; - } - - self.appservice_in_room_cache - .write() - .unwrap() - .remove(room_id); - - Ok(()) - } - - #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] - pub(super) fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { - let maybe = self - .appservice_in_room_cache - .read() - .unwrap() - .get(room_id) - .and_then(|map| map.get(&appservice.registration.id)) - .copied(); - - if let Some(b) = maybe { - Ok(b) - } else { - let bridge_user_id = UserId::parse_with_server_name( - appservice.registration.sender_localpart.as_str(), - self.services.globals.server_name(), - ) - .ok(); - - let in_room = bridge_user_id.map_or(false, |id| self.is_joined(&id, room_id).unwrap_or(false)) - || self - .room_members(room_id) - .any(|userid| userid.map_or(false, |userid| appservice.users.is_match(userid.as_str()))); - - self.appservice_in_room_cache - .write() - .unwrap() - .entry(room_id.to_owned()) - .or_default() - .insert(appservice.registration.id.clone(), in_room); - - Ok(in_room) - } + self.roomid_inviteviaservers.remove(&roomid); } /// Makes a user forget a room. #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { + pub(super) fn forget(&self, room_id: &RoomId, user_id: &UserId) { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -270,397 +126,69 @@ impl Data { roomuser_id.push(0xFF); roomuser_id.extend_from_slice(user_id.as_bytes()); - self.userroomid_leftstate.remove(&userroom_id)?; - self.roomuserid_leftcount.remove(&roomuser_id)?; - - Ok(()) - } - - /// Returns an iterator of all servers participating in this room. - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn room_servers<'a>( - &'a self, room_id: &RoomId, - ) -> Box> + 'a> { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xFF); - - Box::new(self.roomserverids.scan_prefix(prefix).map(|(key, _)| { - ServerName::parse( - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("Server name in roomserverids is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("Server name in roomserverids is invalid.")) - })) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result { - let mut key = server.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(room_id.as_bytes()); - - self.serverroomids.get(&key).map(|o| o.is_some()) - } - - /// Returns an iterator of all rooms a server participates in (as far as we - /// know). - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn server_rooms<'a>( - &'a self, server: &ServerName, - ) -> Box> + 'a> { - let mut prefix = server.as_bytes().to_vec(); - prefix.push(0xFF); - - Box::new(self.serverroomids.scan_prefix(prefix).map(|(key, _)| { - RoomId::parse( - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("RoomId in serverroomids is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("RoomId in serverroomids is invalid.")) - })) - } - - /// Returns an iterator of all joined members of a room. - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn room_members<'a>( - &'a self, room_id: &RoomId, - ) -> Box> + Send + 'a> { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xFF); - - Box::new(self.roomuserid_joined.scan_prefix(prefix).map(|(key, _)| { - UserId::parse( - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("User ID in roomuserid_joined is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("User ID in roomuserid_joined is invalid.")) - })) - } - - /// Returns an iterator of all our local users in the room, even if they're - /// deactivated/guests - pub(super) fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> Box + 'a> { - Box::new( - self.room_members(room_id) - .filter_map(Result::ok) - .filter(|user| self.services.globals.user_is_local(user)), - ) - } - - /// Returns an iterator of all our local joined users in a room who are - /// active (not deactivated, not guest) - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn active_local_users_in_room<'a>( - &'a self, room_id: &RoomId, - ) -> Box + 'a> { - Box::new( - self.local_users_in_room(room_id) - .filter(|user| !self.services.users.is_deactivated(user).unwrap_or(true)), - ) - } - - /// Returns the number of users which are currently in a room - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn room_joined_count(&self, room_id: &RoomId) -> Result> { - self.roomid_joinedcount - .get(room_id.as_bytes())? - .map(|b| utils::u64_from_bytes(&b).map_err(|_| Error::bad_database("Invalid joinedcount in db."))) - .transpose() - } - - /// Returns the number of users which are currently invited to a room - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn room_invited_count(&self, room_id: &RoomId) -> Result> { - self.roomid_invitedcount - .get(room_id.as_bytes())? - .map(|b| utils::u64_from_bytes(&b).map_err(|_| Error::bad_database("Invalid joinedcount in db."))) - .transpose() - } - - /// Returns an iterator over all User IDs who ever joined a room. - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn room_useroncejoined<'a>( - &'a self, room_id: &RoomId, - ) -> Box> + 'a> { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xFF); - - Box::new( - self.roomuseroncejoinedids - .scan_prefix(prefix) - .map(|(key, _)| { - UserId::parse( - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("User ID in room_useroncejoined is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("User ID in room_useroncejoined is invalid.")) - }), - ) - } - - /// Returns an iterator over all invited members of a room. - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn room_members_invited<'a>( - &'a self, room_id: &RoomId, - ) -> Box> + 'a> { - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xFF); - - Box::new( - self.roomuserid_invitecount - .scan_prefix(prefix) - .map(|(key, _)| { - UserId::parse( - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("User ID in roomuserid_invited is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("User ID in roomuserid_invited is invalid.")) - }), - ) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(user_id.as_bytes()); - - self.roomuserid_invitecount - .get(&key)? - .map_or(Ok(None), |bytes| { - Ok(Some( - utils::u64_from_bytes(&bytes).map_err(|_| Error::bad_database("Invalid invitecount in db."))?, - )) - }) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(user_id.as_bytes()); - - self.roomuserid_leftcount - .get(&key)? - .map(|bytes| utils::u64_from_bytes(&bytes).map_err(|_| Error::bad_database("Invalid leftcount in db."))) - .transpose() - } - - /// Returns an iterator over all rooms this user joined. - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn rooms_joined(&self, user_id: &UserId) -> Box> + '_> { - Box::new( - self.userroomid_joined - .scan_prefix(user_id.as_bytes().to_vec()) - .map(|(key, _)| { - RoomId::parse( - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("Room ID in userroomid_joined is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("Room ID in userroomid_joined is invalid.")) - }), - ) + self.userroomid_leftstate.remove(&userroom_id); + self.roomuserid_leftcount.remove(&roomuser_id); } /// Returns an iterator over all rooms a user was invited to. - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn rooms_invited<'a>(&'a self, user_id: &UserId) -> StrippedStateEventIter<'a> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - - Box::new( - self.userroomid_invitestate - .scan_prefix(prefix) - .map(|(key, state)| { - let room_id = RoomId::parse( - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("Room ID in userroomid_invited is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("Room ID in userroomid_invited is invalid."))?; - - let state = serde_json::from_slice(&state) - .map_err(|_| Error::bad_database("Invalid state in userroomid_invitestate."))?; - - Ok((room_id, state)) - }), - ) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn invite_state( - &self, user_id: &UserId, room_id: &RoomId, - ) -> Result>>> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(room_id.as_bytes()); - + #[inline] + pub(super) fn rooms_invited<'a>( + &'a self, user_id: &'a UserId, + ) -> impl Stream + Send + 'a { + let prefix = (user_id, Interfix); self.userroomid_invitestate - .get(&key)? - .map(|state| { - let state = serde_json::from_slice(&state) - .map_err(|_| Error::bad_database("Invalid state in userroomid_invitestate."))?; + .stream_raw_prefix(&prefix) + .ignore_err() + .map(|(key, val)| { + let room_id = key.rsplit(|&b| b == 0xFF).next().unwrap(); + let room_id = utils::string_from_bytes(room_id).unwrap(); + let room_id = RoomId::parse(room_id).unwrap(); + let state = serde_json::from_slice(val) + .map_err(|_| Error::bad_database("Invalid state in userroomid_invitestate.")) + .unwrap(); - Ok(state) + (room_id, state) }) - .transpose() } #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn left_state( + pub(super) async fn invite_state( &self, user_id: &UserId, room_id: &RoomId, - ) -> Result>>> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(room_id.as_bytes()); + ) -> Result>> { + let key = (user_id, room_id); + self.userroomid_invitestate + .qry(&key) + .await + .deserialized_json() + } + #[tracing::instrument(skip(self), level = "debug")] + pub(super) async fn left_state( + &self, user_id: &UserId, room_id: &RoomId, + ) -> Result>> { + let key = (user_id, room_id); self.userroomid_leftstate - .get(&key)? - .map(|state| { - let state = serde_json::from_slice(&state) - .map_err(|_| Error::bad_database("Invalid state in userroomid_leftstate."))?; - - Ok(state) - }) - .transpose() + .qry(&key) + .await + .deserialized_json() } /// Returns an iterator over all rooms a user left. - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn rooms_left<'a>(&'a self, user_id: &UserId) -> AnySyncStateEventIter<'a> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); + #[inline] + pub(super) fn rooms_left<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + let prefix = (user_id, Interfix); + self.userroomid_leftstate + .stream_raw_prefix(&prefix) + .ignore_err() + .map(|(key, val)| { + let room_id = key.rsplit(|&b| b == 0xFF).next().unwrap(); + let room_id = utils::string_from_bytes(room_id).unwrap(); + let room_id = RoomId::parse(room_id).unwrap(); + let state = serde_json::from_slice(val) + .map_err(|_| Error::bad_database("Invalid state in userroomid_leftstate.")) + .unwrap(); - Box::new( - self.userroomid_leftstate - .scan_prefix(prefix) - .map(|(key, state)| { - let room_id = RoomId::parse( - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("Room ID in userroomid_invited is invalid unicode."))?, - ) - .map_err(|_| Error::bad_database("Room ID in userroomid_invited is invalid."))?; - - let state = serde_json::from_slice(&state) - .map_err(|_| Error::bad_database("Invalid state in userroomid_leftstate."))?; - - Ok((room_id, state)) - }), - ) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); - - Ok(self.roomuseroncejoinedids.get(&userroom_id)?.is_some()) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); - - Ok(self.userroomid_joined.get(&userroom_id)?.is_some()) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); - - Ok(self.userroomid_invitestate.get(&userroom_id)?.is_some()) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); - - Ok(self.userroomid_leftstate.get(&userroom_id)?.is_some()) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn servers_invite_via<'a>( - &'a self, room_id: &RoomId, - ) -> Box> + 'a> { - let key = room_id.as_bytes().to_vec(); - - Box::new( - self.roomid_inviteviaservers - .scan_prefix(key) - .map(|(_, servers)| { - ServerName::parse( - utils::string_from_bytes( - servers - .rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| { - Error::bad_database("Server name in roomid_inviteviaservers is invalid unicode.") - })?, - ) - .map_err(|_| Error::bad_database("Server name in roomid_inviteviaservers is invalid.")) - }), - ) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) -> Result<()> { - let mut prev_servers = self - .servers_invite_via(room_id) - .filter_map(Result::ok) - .collect_vec(); - prev_servers.extend(servers.to_owned()); - prev_servers.sort_unstable(); - prev_servers.dedup(); - - let servers = prev_servers - .iter() - .map(|server| server.as_bytes()) - .collect_vec() - .join(&[0xFF][..]); - - self.roomid_inviteviaservers - .insert(room_id.as_bytes(), &servers)?; - - Ok(()) + (room_id, state) + }) } } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 71899ceb..ce5b024b 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -1,9 +1,15 @@ mod data; -use std::sync::Arc; +use std::{collections::HashSet, sync::Arc}; -use conduit::{err, error, warn, Error, Result}; +use conduit::{ + err, + utils::{stream::TryIgnore, ReadyExt}, + warn, Result, +}; use data::Data; +use database::{Deserialized, Ignore, Interfix}; +use futures::{Stream, StreamExt}; use itertools::Itertools; use ruma::{ events::{ @@ -18,7 +24,7 @@ use ruma::{ }, int, serde::Raw, - OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, + OwnedRoomId, OwnedServerName, RoomId, ServerName, UserId, }; use crate::{account_data, appservice::RegistrationInfo, globals, rooms, users, Dep}; @@ -55,7 +61,7 @@ impl Service { /// Update current membership data. #[tracing::instrument(skip(self, last_state))] #[allow(clippy::too_many_arguments)] - pub fn update_membership( + pub async fn update_membership( &self, room_id: &RoomId, user_id: &UserId, membership_event: RoomMemberEventContent, sender: &UserId, last_state: Option>>, invite_via: Option>, update_joined_count: bool, @@ -68,7 +74,7 @@ impl Service { // update #[allow(clippy::collapsible_if)] if !self.services.globals.user_is_local(user_id) { - if !self.services.users.exists(user_id)? { + if !self.services.users.exists(user_id).await { self.services.users.create(user_id, None)?; } @@ -100,17 +106,17 @@ impl Service { match &membership { MembershipState::Join => { // Check if the user never joined this room - if !self.once_joined(user_id, room_id)? { + if !self.once_joined(user_id, room_id).await { // Add the user ID to the join list then - self.db.mark_as_once_joined(user_id, room_id)?; + self.db.mark_as_once_joined(user_id, room_id); // Check if the room has a predecessor - if let Some(predecessor) = self + if let Ok(Some(predecessor)) = self .services .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "")? - .and_then(|create| serde_json::from_str(create.content.get()).ok()) - .and_then(|content: RoomCreateEventContent| content.predecessor) + .room_state_get_content(room_id, &StateEventType::RoomCreate, "") + .await + .map(|content: RoomCreateEventContent| content.predecessor) { // Copy user settings from predecessor to the current room: // - Push rules @@ -138,32 +144,33 @@ impl Service { // .ok(); // Copy old tags to new room - if let Some(tag_event) = self + if let Ok(tag_event) = self .services .account_data - .get(Some(&predecessor.room_id), user_id, RoomAccountDataEventType::Tag)? - .map(|event| { + .get(Some(&predecessor.room_id), user_id, RoomAccountDataEventType::Tag) + .await + .and_then(|event| { serde_json::from_str(event.get()) .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) { self.services .account_data - .update(Some(room_id), user_id, RoomAccountDataEventType::Tag, &tag_event?) + .update(Some(room_id), user_id, RoomAccountDataEventType::Tag, &tag_event) + .await .ok(); }; // Copy direct chat flag - if let Some(direct_event) = self + if let Ok(mut direct_event) = self .services .account_data - .get(None, user_id, GlobalAccountDataEventType::Direct.to_string().into())? - .map(|event| { + .get(None, user_id, GlobalAccountDataEventType::Direct.to_string().into()) + .await + .and_then(|event| { serde_json::from_str::(event.get()) .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) { - let mut direct_event = direct_event?; let mut room_ids_updated = false; - for room_ids in direct_event.content.0.values_mut() { if room_ids.iter().any(|r| r == &predecessor.room_id) { room_ids.push(room_id.to_owned()); @@ -172,18 +179,21 @@ impl Service { } if room_ids_updated { - self.services.account_data.update( - None, - user_id, - GlobalAccountDataEventType::Direct.to_string().into(), - &serde_json::to_value(&direct_event).expect("to json always works"), - )?; + self.services + .account_data + .update( + None, + user_id, + GlobalAccountDataEventType::Direct.to_string().into(), + &serde_json::to_value(&direct_event).expect("to json always works"), + ) + .await?; } }; } } - self.db.mark_as_joined(user_id, room_id)?; + self.db.mark_as_joined(user_id, room_id); }, MembershipState::Invite => { // We want to know if the sender is ignored by the receiver @@ -196,12 +206,12 @@ impl Service { GlobalAccountDataEventType::IgnoredUserList .to_string() .into(), - )? - .map(|event| { + ) + .await + .and_then(|event| { serde_json::from_str::(event.get()) .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) }) - .transpose()? .map_or(false, |ignored| { ignored .content @@ -214,194 +224,282 @@ impl Service { return Ok(()); } - self.db - .mark_as_invited(user_id, room_id, last_state, invite_via)?; + self.mark_as_invited(user_id, room_id, last_state, invite_via) + .await; }, MembershipState::Leave | MembershipState::Ban => { - self.db.mark_as_left(user_id, room_id)?; + self.db.mark_as_left(user_id, room_id); }, _ => {}, } if update_joined_count { - self.update_joined_count(room_id)?; + self.update_joined_count(room_id).await; } Ok(()) } - #[tracing::instrument(skip(self, room_id), level = "debug")] - pub fn update_joined_count(&self, room_id: &RoomId) -> Result<()> { self.db.update_joined_count(room_id) } - #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] - pub fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> Result { - self.db.appservice_in_room(room_id, appservice) + pub async fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> bool { + let maybe = self + .db + .appservice_in_room_cache + .read() + .unwrap() + .get(room_id) + .and_then(|map| map.get(&appservice.registration.id)) + .copied(); + + if let Some(b) = maybe { + b + } else { + let bridge_user_id = UserId::parse_with_server_name( + appservice.registration.sender_localpart.as_str(), + self.services.globals.server_name(), + ) + .ok(); + + let in_room = if let Some(id) = &bridge_user_id { + self.is_joined(id, room_id).await + } else { + false + }; + + let in_room = in_room + || self + .room_members(room_id) + .ready_any(|userid| appservice.users.is_match(userid.as_str())) + .await; + + self.db + .appservice_in_room_cache + .write() + .unwrap() + .entry(room_id.to_owned()) + .or_default() + .insert(appservice.registration.id.clone(), in_room); + + in_room + } } /// Direct DB function to directly mark a user as left. It is not /// recommended to use this directly. You most likely should use /// `update_membership` instead #[tracing::instrument(skip(self), level = "debug")] - pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { - self.db.mark_as_left(user_id, room_id) - } + pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) { self.db.mark_as_left(user_id, room_id); } /// Direct DB function to directly mark a user as joined. It is not /// recommended to use this directly. You most likely should use /// `update_membership` instead #[tracing::instrument(skip(self), level = "debug")] - pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { - self.db.mark_as_joined(user_id, room_id) - } + pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { self.db.mark_as_joined(user_id, room_id); } /// Makes a user forget a room. #[tracing::instrument(skip(self), level = "debug")] - pub fn forget(&self, room_id: &RoomId, user_id: &UserId) -> Result<()> { self.db.forget(room_id, user_id) } + pub fn forget(&self, room_id: &RoomId, user_id: &UserId) { self.db.forget(room_id, user_id); } /// Returns an iterator of all servers participating in this room. #[tracing::instrument(skip(self), level = "debug")] - pub fn room_servers(&self, room_id: &RoomId) -> impl Iterator> + '_ { - self.db.room_servers(room_id) + pub fn room_servers<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomserverids + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, server): (Ignore, &ServerName)| server) } #[tracing::instrument(skip(self), level = "debug")] - pub fn server_in_room(&self, server: &ServerName, room_id: &RoomId) -> Result { - self.db.server_in_room(server, room_id) + pub async fn server_in_room<'a>(&'a self, server: &'a ServerName, room_id: &'a RoomId) -> bool { + let key = (server, room_id); + self.db.serverroomids.qry(&key).await.is_ok() } /// Returns an iterator of all rooms a server participates in (as far as we /// know). #[tracing::instrument(skip(self), level = "debug")] - pub fn server_rooms(&self, server: &ServerName) -> impl Iterator> + '_ { - self.db.server_rooms(server) + pub fn server_rooms<'a>(&'a self, server: &'a ServerName) -> impl Stream + Send + 'a { + let prefix = (server, Interfix); + self.db + .serverroomids + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, room_id): (Ignore, &RoomId)| room_id) } /// Returns true if server can see user by sharing at least one room. #[tracing::instrument(skip(self), level = "debug")] - pub fn server_sees_user(&self, server: &ServerName, user_id: &UserId) -> Result { - Ok(self - .server_rooms(server) - .filter_map(Result::ok) - .any(|room_id: OwnedRoomId| self.is_joined(user_id, &room_id).unwrap_or(false))) + pub async fn server_sees_user(&self, server: &ServerName, user_id: &UserId) -> bool { + self.server_rooms(server) + .any(|room_id| self.is_joined(user_id, room_id)) + .await } /// Returns true if user_a and user_b share at least one room. #[tracing::instrument(skip(self), level = "debug")] - pub fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> Result { + pub async fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> bool { // Minimize number of point-queries by iterating user with least nr rooms - let (a, b) = if self.rooms_joined(user_a).count() < self.rooms_joined(user_b).count() { + let (a, b) = if self.rooms_joined(user_a).count().await < self.rooms_joined(user_b).count().await { (user_a, user_b) } else { (user_b, user_a) }; - Ok(self - .rooms_joined(a) - .filter_map(Result::ok) - .any(|room_id| self.is_joined(b, &room_id).unwrap_or(false))) + self.rooms_joined(a) + .any(|room_id| self.is_joined(b, room_id)) + .await } - /// Returns an iterator over all joined members of a room. + /// Returns an iterator of all joined members of a room. #[tracing::instrument(skip(self), level = "debug")] - pub fn room_members(&self, room_id: &RoomId) -> impl Iterator> + Send + '_ { - self.db.room_members(room_id) + pub fn room_members<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuserid_joined + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, &UserId)| user_id) } /// Returns the number of users which are currently in a room #[tracing::instrument(skip(self), level = "debug")] - pub fn room_joined_count(&self, room_id: &RoomId) -> Result> { self.db.room_joined_count(room_id) } + pub async fn room_joined_count(&self, room_id: &RoomId) -> Result { + self.db.roomid_joinedcount.qry(room_id).await.deserialized() + } #[tracing::instrument(skip(self), level = "debug")] /// Returns an iterator of all our local users in the room, even if they're /// deactivated/guests - pub fn local_users_in_room<'a>(&'a self, room_id: &RoomId) -> impl Iterator + 'a { - self.db.local_users_in_room(room_id) + pub fn local_users_in_room<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + self.room_members(room_id) + .ready_filter(|user| self.services.globals.user_is_local(user)) } #[tracing::instrument(skip(self), level = "debug")] /// Returns an iterator of all our local joined users in a room who are /// active (not deactivated, not guest) - pub fn active_local_users_in_room<'a>(&'a self, room_id: &RoomId) -> impl Iterator + 'a { - self.db.active_local_users_in_room(room_id) + pub fn active_local_users_in_room<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + self.local_users_in_room(room_id) + .filter(|user| self.services.users.is_active(user)) } /// Returns the number of users which are currently invited to a room #[tracing::instrument(skip(self), level = "debug")] - pub fn room_invited_count(&self, room_id: &RoomId) -> Result> { self.db.room_invited_count(room_id) } + pub async fn room_invited_count(&self, room_id: &RoomId) -> Result { + self.db + .roomid_invitedcount + .qry(room_id) + .await + .deserialized() + } /// Returns an iterator over all User IDs who ever joined a room. #[tracing::instrument(skip(self), level = "debug")] - pub fn room_useroncejoined(&self, room_id: &RoomId) -> impl Iterator> + '_ { - self.db.room_useroncejoined(room_id) + pub fn room_useroncejoined<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuseroncejoinedids + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, &UserId)| user_id) } /// Returns an iterator over all invited members of a room. #[tracing::instrument(skip(self), level = "debug")] - pub fn room_members_invited(&self, room_id: &RoomId) -> impl Iterator> + '_ { - self.db.room_members_invited(room_id) + pub fn room_members_invited<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuserid_invitecount + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, &UserId)| user_id) } #[tracing::instrument(skip(self), level = "debug")] - pub fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { - self.db.get_invite_count(room_id, user_id) + pub async fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.db + .roomuserid_invitecount + .qry(&key) + .await + .deserialized() } #[tracing::instrument(skip(self), level = "debug")] - pub fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result> { - self.db.get_left_count(room_id, user_id) + pub async fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.db.roomuserid_leftcount.qry(&key).await.deserialized() } /// Returns an iterator over all rooms this user joined. #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_joined(&self, user_id: &UserId) -> impl Iterator> + '_ { - self.db.rooms_joined(user_id) + pub fn rooms_joined(&self, user_id: &UserId) -> impl Stream + Send { + self.db + .userroomid_joined + .keys_prefix(user_id) + .ignore_err() + .map(|(_, room_id): (Ignore, &RoomId)| room_id) } /// Returns an iterator over all rooms a user was invited to. #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_invited( - &self, user_id: &UserId, - ) -> impl Iterator>)>> + '_ { + pub fn rooms_invited<'a>( + &'a self, user_id: &'a UserId, + ) -> impl Stream>)> + Send + 'a { self.db.rooms_invited(user_id) } #[tracing::instrument(skip(self), level = "debug")] - pub fn invite_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>> { - self.db.invite_state(user_id, room_id) + pub async fn invite_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>> { + self.db.invite_state(user_id, room_id).await } #[tracing::instrument(skip(self), level = "debug")] - pub fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>>> { - self.db.left_state(user_id, room_id) + pub async fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>> { + self.db.left_state(user_id, room_id).await } /// Returns an iterator over all rooms a user left. #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_left( - &self, user_id: &UserId, - ) -> impl Iterator>)>> + '_ { + pub fn rooms_left<'a>( + &'a self, user_id: &'a UserId, + ) -> impl Stream>)> + Send + 'a { self.db.rooms_left(user_id) } #[tracing::instrument(skip(self), level = "debug")] - pub fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { - self.db.once_joined(user_id, room_id) + pub async fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let key = (user_id, room_id); + self.db.roomuseroncejoinedids.qry(&key).await.is_ok() } #[tracing::instrument(skip(self), level = "debug")] - pub fn is_joined(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.is_joined(user_id, room_id) } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> Result { - self.db.is_invited(user_id, room_id) + pub async fn is_joined<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_joined.qry(&key).await.is_ok() } #[tracing::instrument(skip(self), level = "debug")] - pub fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> Result { self.db.is_left(user_id, room_id) } + pub async fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_invitestate.qry(&key).await.is_ok() + } #[tracing::instrument(skip(self), level = "debug")] - pub fn servers_invite_via(&self, room_id: &RoomId) -> impl Iterator> + '_ { - self.db.servers_invite_via(room_id) + pub async fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_leftstate.qry(&key).await.is_ok() + } + + #[tracing::instrument(skip(self), level = "debug")] + pub fn servers_invite_via<'a>(&'a self, room_id: &RoomId) -> impl Stream + Send + 'a { + self.db + .roomid_inviteviaservers + .stream_prefix(room_id) + .ignore_err() + .map(|(_, servers): (Ignore, Vec<&ServerName>)| &**(servers.last().expect("at least one servername"))) } /// Gets up to three servers that are likely to be in the room in the @@ -409,37 +507,27 @@ impl Service { /// /// See #[tracing::instrument(skip(self))] - pub fn servers_route_via(&self, room_id: &RoomId) -> Result> { + pub async fn servers_route_via(&self, room_id: &RoomId) -> Result> { let most_powerful_user_server = self .services .state_accessor - .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? - .map(|pdu| { - serde_json::from_str(pdu.content.get()).map(|conent: RoomPowerLevelsEventContent| { - conent - .users - .iter() - .max_by_key(|(_, power)| *power) - .and_then(|x| { - if x.1 >= &int!(50) { - Some(x) - } else { - None - } - }) - .map(|(user, _power)| user.server_name().to_owned()) - }) + .room_state_get_content(room_id, &StateEventType::RoomPowerLevels, "") + .await + .map(|content: RoomPowerLevelsEventContent| { + content + .users + .iter() + .max_by_key(|(_, power)| *power) + .and_then(|x| (x.1 >= &int!(50)).then_some(x)) + .map(|(user, _power)| user.server_name().to_owned()) }) - .transpose() - .map_err(|e| { - error!("Invalid power levels event content in database: {e}"); - Error::bad_database("Invalid power levels event content in database") - })? - .flatten(); + .map_err(|e| err!(Database(error!(?e, "Invalid power levels event content in database."))))?; let mut servers: Vec = self .room_members(room_id) - .filter_map(Result::ok) + .collect::>() + .await + .iter() .counts_by(|user| user.server_name().to_owned()) .iter() .sorted_by_key(|(_, users)| *users) @@ -468,4 +556,139 @@ impl Service { .expect("locked") .clear(); } + + pub async fn update_joined_count(&self, room_id: &RoomId) { + let mut joinedcount = 0_u64; + let mut invitedcount = 0_u64; + let mut joined_servers = HashSet::new(); + + self.room_members(room_id) + .ready_for_each(|joined| { + joined_servers.insert(joined.server_name().to_owned()); + joinedcount = joinedcount.saturating_add(1); + }) + .await; + + invitedcount = invitedcount.saturating_add( + self.room_members_invited(room_id) + .count() + .await + .try_into() + .unwrap_or(0), + ); + + self.db + .roomid_joinedcount + .insert(room_id.as_bytes(), &joinedcount.to_be_bytes()); + + self.db + .roomid_invitedcount + .insert(room_id.as_bytes(), &invitedcount.to_be_bytes()); + + self.room_servers(room_id) + .ready_for_each(|old_joined_server| { + if !joined_servers.remove(old_joined_server) { + // Server not in room anymore + let mut roomserver_id = room_id.as_bytes().to_vec(); + roomserver_id.push(0xFF); + roomserver_id.extend_from_slice(old_joined_server.as_bytes()); + + let mut serverroom_id = old_joined_server.as_bytes().to_vec(); + serverroom_id.push(0xFF); + serverroom_id.extend_from_slice(room_id.as_bytes()); + + self.db.roomserverids.remove(&roomserver_id); + self.db.serverroomids.remove(&serverroom_id); + } + }) + .await; + + // Now only new servers are in joined_servers anymore + for server in joined_servers { + let mut roomserver_id = room_id.as_bytes().to_vec(); + roomserver_id.push(0xFF); + roomserver_id.extend_from_slice(server.as_bytes()); + + let mut serverroom_id = server.as_bytes().to_vec(); + serverroom_id.push(0xFF); + serverroom_id.extend_from_slice(room_id.as_bytes()); + + self.db.roomserverids.insert(&roomserver_id, &[]); + self.db.serverroomids.insert(&serverroom_id, &[]); + } + + self.db + .appservice_in_room_cache + .write() + .unwrap() + .remove(room_id); + } + + pub async fn mark_as_invited( + &self, user_id: &UserId, room_id: &RoomId, last_state: Option>>, + invite_via: Option>, + ) { + let mut roomuser_id = room_id.as_bytes().to_vec(); + roomuser_id.push(0xFF); + roomuser_id.extend_from_slice(user_id.as_bytes()); + + let mut userroom_id = user_id.as_bytes().to_vec(); + userroom_id.push(0xFF); + userroom_id.extend_from_slice(room_id.as_bytes()); + + self.db.userroomid_invitestate.insert( + &userroom_id, + &serde_json::to_vec(&last_state.unwrap_or_default()).expect("state to bytes always works"), + ); + self.db + .roomuserid_invitecount + .insert(&roomuser_id, &self.services.globals.next_count().unwrap().to_be_bytes()); + self.db.userroomid_joined.remove(&userroom_id); + self.db.roomuserid_joined.remove(&roomuser_id); + self.db.userroomid_leftstate.remove(&userroom_id); + self.db.roomuserid_leftcount.remove(&roomuser_id); + + if let Some(servers) = invite_via { + let mut prev_servers = self + .servers_invite_via(room_id) + .map(ToOwned::to_owned) + .collect::>() + .await; + #[allow(clippy::redundant_clone)] // this is a necessary clone? + prev_servers.append(servers.clone().as_mut()); + let servers = prev_servers.iter().rev().unique().rev().collect_vec(); + + let servers = servers + .iter() + .map(|server| server.as_bytes()) + .collect_vec() + .join(&[0xFF][..]); + + self.db + .roomid_inviteviaservers + .insert(room_id.as_bytes(), &servers); + } + } + + #[tracing::instrument(skip(self), level = "debug")] + pub async fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) { + let mut prev_servers = self + .servers_invite_via(room_id) + .map(ToOwned::to_owned) + .collect::>() + .await; + prev_servers.extend(servers.to_owned()); + prev_servers.sort_unstable(); + prev_servers.dedup(); + + let servers = prev_servers + .iter() + .map(|server| server.as_bytes()) + .collect_vec() + .join(&[0xFF][..]); + + self.db + .roomid_inviteviaservers + .insert(room_id.as_bytes(), &servers); + } } diff --git a/src/service/rooms/state_compressor/data.rs b/src/service/rooms/state_compressor/data.rs index 33773001..9a9f70a2 100644 --- a/src/service/rooms/state_compressor/data.rs +++ b/src/service/rooms/state_compressor/data.rs @@ -1,6 +1,6 @@ use std::{collections::HashSet, mem::size_of, sync::Arc}; -use conduit::{checked, utils, Error, Result}; +use conduit::{err, expected, utils, Result}; use database::{Database, Map}; use super::CompressedStateEvent; @@ -22,11 +22,13 @@ impl Data { } } - pub(super) fn get_statediff(&self, shortstatehash: u64) -> Result { + pub(super) async fn get_statediff(&self, shortstatehash: u64) -> Result { let value = self .shortstatehash_statediff - .get(&shortstatehash.to_be_bytes())? - .ok_or_else(|| Error::bad_database("State hash does not exist"))?; + .qry(&shortstatehash) + .await + .map_err(|e| err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")))?; + let parent = utils::u64_from_bytes(&value[0..size_of::()]).expect("bytes have right length"); let parent = if parent != 0 { Some(parent) @@ -40,10 +42,10 @@ impl Data { let stride = size_of::(); let mut i = stride; - while let Some(v) = value.get(i..checked!(i + 2 * stride)?) { + while let Some(v) = value.get(i..expected!(i + 2 * stride)) { if add_mode && v.starts_with(&0_u64.to_be_bytes()) { add_mode = false; - i = checked!(i + stride)?; + i = expected!(i + stride); continue; } if add_mode { @@ -51,7 +53,7 @@ impl Data { } else { removed.insert(v.try_into().expect("we checked the size above")); } - i = checked!(i + 2 * stride)?; + i = expected!(i + 2 * stride); } Ok(StateDiff { @@ -61,7 +63,7 @@ impl Data { }) } - pub(super) fn save_statediff(&self, shortstatehash: u64, diff: &StateDiff) -> Result<()> { + pub(super) fn save_statediff(&self, shortstatehash: u64, diff: &StateDiff) { let mut value = diff.parent.unwrap_or(0).to_be_bytes().to_vec(); for new in diff.added.iter() { value.extend_from_slice(&new[..]); @@ -75,6 +77,6 @@ impl Data { } self.shortstatehash_statediff - .insert(&shortstatehash.to_be_bytes(), &value) + .insert(&shortstatehash.to_be_bytes(), &value); } } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 2550774e..cd3f2f73 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -27,14 +27,12 @@ type StateInfoLruCache = Mutex< >, >; -type ShortStateInfoResult = Result< - Vec<( - u64, // sstatehash - Arc>, // full state - Arc>, // added - Arc>, // removed - )>, ->; +type ShortStateInfoResult = Vec<( + u64, // sstatehash + Arc>, // full state + Arc>, // added + Arc>, // removed +)>; type ParentStatesVec = Vec<( u64, // sstatehash @@ -43,7 +41,7 @@ type ParentStatesVec = Vec<( Arc>, // removed )>; -type HashSetCompressStateEvent = Result<(u64, Arc>, Arc>)>; +type HashSetCompressStateEvent = (u64, Arc>, Arc>); pub type CompressedStateEvent = [u8; 2 * size_of::()]; pub struct Service { @@ -86,12 +84,11 @@ impl crate::Service for Service { impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. - #[tracing::instrument(skip(self), level = "debug")] - pub fn load_shortstatehash_info(&self, shortstatehash: u64) -> ShortStateInfoResult { + pub async fn load_shortstatehash_info(&self, shortstatehash: u64) -> Result { if let Some(r) = self .stateinfo_cache .lock() - .unwrap() + .expect("locked") .get_mut(&shortstatehash) { return Ok(r.clone()); @@ -101,11 +98,11 @@ impl Service { parent, added, removed, - } = self.db.get_statediff(shortstatehash)?; + } = self.db.get_statediff(shortstatehash).await?; if let Some(parent) = parent { - let mut response = self.load_shortstatehash_info(parent)?; - let mut state = (*response.last().unwrap().1).clone(); + let mut response = Box::pin(self.load_shortstatehash_info(parent)).await?; + let mut state = (*response.last().expect("at least one response").1).clone(); state.extend(added.iter().copied()); let removed = (*removed).clone(); for r in &removed { @@ -116,7 +113,7 @@ impl Service { self.stateinfo_cache .lock() - .unwrap() + .expect("locked") .insert(shortstatehash, response.clone()); Ok(response) @@ -124,33 +121,42 @@ impl Service { let response = vec![(shortstatehash, added.clone(), added, removed)]; self.stateinfo_cache .lock() - .unwrap() + .expect("locked") .insert(shortstatehash, response.clone()); + Ok(response) } } - pub fn compress_state_event(&self, shortstatekey: u64, event_id: &EventId) -> Result { + pub async fn compress_state_event(&self, shortstatekey: u64, event_id: &EventId) -> CompressedStateEvent { let mut v = shortstatekey.to_be_bytes().to_vec(); v.extend_from_slice( &self .services .short - .get_or_create_shorteventid(event_id)? + .get_or_create_shorteventid(event_id) + .await .to_be_bytes(), ); - Ok(v.try_into().expect("we checked the size above")) + + v.try_into().expect("we checked the size above") } /// Returns shortstatekey, event id #[inline] - pub fn parse_compressed_state_event(&self, compressed_event: &CompressedStateEvent) -> Result<(u64, Arc)> { - Ok(( - utils::u64_from_bytes(&compressed_event[0..size_of::()]).expect("bytes have right length"), - self.services.short.get_eventid_from_short( - utils::u64_from_bytes(&compressed_event[size_of::()..]).expect("bytes have right length"), - )?, - )) + pub async fn parse_compressed_state_event( + &self, compressed_event: &CompressedStateEvent, + ) -> Result<(u64, Arc)> { + use utils::u64_from_u8; + + let shortstatekey = u64_from_u8(&compressed_event[0..size_of::()]); + let event_id = self + .services + .short + .get_eventid_from_short(u64_from_u8(&compressed_event[size_of::()..])) + .await?; + + Ok((shortstatekey, event_id)) } /// Creates a new shortstatehash that often is just a diff to an already @@ -227,7 +233,7 @@ impl Service { added: statediffnew, removed: statediffremoved, }, - )?; + ); return Ok(()); }; @@ -280,7 +286,7 @@ impl Service { added: statediffnew, removed: statediffremoved, }, - )?; + ); } Ok(()) @@ -288,10 +294,15 @@ impl Service { /// Returns the new shortstatehash, and the state diff from the previous /// room state - pub fn save_state( + pub async fn save_state( &self, room_id: &RoomId, new_state_ids_compressed: Arc>, - ) -> HashSetCompressStateEvent { - let previous_shortstatehash = self.services.state.get_room_shortstatehash(room_id)?; + ) -> Result { + let previous_shortstatehash = self + .services + .state + .get_room_shortstatehash(room_id) + .await + .ok(); let state_hash = utils::calculate_hash( &new_state_ids_compressed @@ -303,14 +314,18 @@ impl Service { let (new_shortstatehash, already_existed) = self .services .short - .get_or_create_shortstatehash(&state_hash)?; + .get_or_create_shortstatehash(&state_hash) + .await; if Some(new_shortstatehash) == previous_shortstatehash { return Ok((new_shortstatehash, Arc::new(HashSet::new()), Arc::new(HashSet::new()))); } - let states_parents = - previous_shortstatehash.map_or_else(|| Ok(Vec::new()), |p| self.load_shortstatehash_info(p))?; + let states_parents = if let Some(p) = previous_shortstatehash { + self.load_shortstatehash_info(p).await.unwrap_or_default() + } else { + ShortStateInfoResult::new() + }; let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { let statediffnew: HashSet<_> = new_state_ids_compressed diff --git a/src/service/rooms/threads/data.rs b/src/service/rooms/threads/data.rs index fb279a00..f50b812c 100644 --- a/src/service/rooms/threads/data.rs +++ b/src/service/rooms/threads/data.rs @@ -1,13 +1,18 @@ use std::{mem::size_of, sync::Arc}; -use conduit::{checked, utils, Error, PduEvent, Result}; -use database::Map; +use conduit::{ + checked, + result::LogErr, + utils, + utils::{stream::TryIgnore, ReadyExt}, + PduEvent, Result, +}; +use database::{Deserialized, Map}; +use futures::{Stream, StreamExt}; use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId}; use crate::{rooms, Dep}; -type PduEventIterResult<'a> = Result> + 'a>>; - pub(super) struct Data { threadid_userids: Arc, services: Services, @@ -30,38 +35,37 @@ impl Data { } } - pub(super) fn threads_until<'a>( + pub(super) async fn threads_until<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, _include: &'a IncludeThreads, - ) -> PduEventIterResult<'a> { + ) -> Result + Send + 'a> { let prefix = self .services .short - .get_shortroomid(room_id)? - .expect("room exists") + .get_shortroomid(room_id) + .await? .to_be_bytes() .to_vec(); let mut current = prefix.clone(); current.extend_from_slice(&(checked!(until - 1)?).to_be_bytes()); - Ok(Box::new( - self.threadid_userids - .iter_from(¤t, true) - .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(pduid, _users)| { - let count = utils::u64_from_bytes(&pduid[(size_of::())..]) - .map_err(|_| Error::bad_database("Invalid pduid in threadid_userids."))?; - let mut pdu = self - .services - .timeline - .get_pdu_from_id(&pduid)? - .ok_or_else(|| Error::bad_database("Invalid pduid reference in threadid_userids"))?; - if pdu.sender != user_id { - pdu.remove_transaction_id()?; - } - Ok((count, pdu)) - }), - )) + let stream = self + .threadid_userids + .rev_raw_keys_from(¤t) + .ignore_err() + .ready_take_while(move |key| key.starts_with(&prefix)) + .map(|pduid| (utils::u64_from_u8(&pduid[(size_of::())..]), pduid)) + .filter_map(move |(count, pduid)| async move { + let mut pdu = self.services.timeline.get_pdu_from_id(pduid).await.ok()?; + + if pdu.sender != user_id { + pdu.remove_transaction_id().log_err().ok(); + } + + Some((count, pdu)) + }); + + Ok(stream) } pub(super) fn update_participants(&self, root_id: &[u8], participants: &[OwnedUserId]) -> Result<()> { @@ -71,28 +75,12 @@ impl Data { .collect::>() .join(&[0xFF][..]); - self.threadid_userids.insert(root_id, &users)?; + self.threadid_userids.insert(root_id, &users); Ok(()) } - pub(super) fn get_participants(&self, root_id: &[u8]) -> Result>> { - if let Some(users) = self.threadid_userids.get(root_id)? { - Ok(Some( - users - .split(|b| *b == 0xFF) - .map(|bytes| { - UserId::parse( - utils::string_from_bytes(bytes) - .map_err(|_| Error::bad_database("Invalid UserId bytes in threadid_userids."))?, - ) - .map_err(|_| Error::bad_database("Invalid UserId in threadid_userids.")) - }) - .filter_map(Result::ok) - .collect(), - )) - } else { - Ok(None) - } + pub(super) async fn get_participants(&self, root_id: &[u8]) -> Result> { + self.threadid_userids.qry(root_id).await.deserialized() } } diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index ae51cd0f..2eafe5d5 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -2,12 +2,12 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{Error, PduEvent, Result}; +use conduit::{err, PduEvent, Result}; use data::Data; +use futures::Stream; use ruma::{ - api::client::{error::ErrorKind, threads::get_threads::v1::IncludeThreads}, - events::relation::BundledThread, - uint, CanonicalJsonValue, EventId, RoomId, UserId, + api::client::threads::get_threads::v1::IncludeThreads, events::relation::BundledThread, uint, CanonicalJsonValue, + EventId, RoomId, UserId, }; use serde_json::json; @@ -36,30 +36,35 @@ impl crate::Service for Service { } impl Service { - pub fn threads_until<'a>( + pub async fn threads_until<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, include: &'a IncludeThreads, - ) -> Result> + 'a> { - self.db.threads_until(user_id, room_id, until, include) + ) -> Result + Send + 'a> { + self.db + .threads_until(user_id, room_id, until, include) + .await } - pub fn add_to_thread(&self, root_event_id: &EventId, pdu: &PduEvent) -> Result<()> { + pub async fn add_to_thread(&self, root_event_id: &EventId, pdu: &PduEvent) -> Result<()> { let root_id = self .services .timeline - .get_pdu_id(root_event_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Invalid event id in thread message"))?; + .get_pdu_id(root_event_id) + .await + .map_err(|e| err!(Request(InvalidParam("Invalid event_id in thread message: {e:?}"))))?; let root_pdu = self .services .timeline - .get_pdu_from_id(&root_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Thread root pdu not found"))?; + .get_pdu_from_id(&root_id) + .await + .map_err(|e| err!(Request(InvalidParam("Thread root not found: {e:?}"))))?; let mut root_pdu_json = self .services .timeline - .get_pdu_json_from_id(&root_id)? - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Thread root pdu not found"))?; + .get_pdu_json_from_id(&root_id) + .await + .map_err(|e| err!(Request(InvalidParam("Thread root pdu not found: {e:?}"))))?; if let CanonicalJsonValue::Object(unsigned) = root_pdu_json .entry("unsigned".to_owned()) @@ -103,11 +108,12 @@ impl Service { self.services .timeline - .replace_pdu(&root_id, &root_pdu_json, &root_pdu)?; + .replace_pdu(&root_id, &root_pdu_json, &root_pdu) + .await?; } let mut users = Vec::new(); - if let Some(userids) = self.db.get_participants(&root_id)? { + if let Ok(userids) = self.db.get_participants(&root_id).await { users.extend_from_slice(&userids); } else { users.push(root_pdu.sender); diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 2f0c8f25..cd746be4 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -1,12 +1,20 @@ use std::{ collections::{hash_map, HashMap}, mem::size_of, - sync::{Arc, Mutex}, + sync::Arc, }; -use conduit::{checked, error, utils, Error, PduCount, PduEvent, Result}; -use database::{Database, Map}; -use ruma::{api::client::error::ErrorKind, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; +use conduit::{ + err, expected, + result::{LogErr, NotFound}, + utils, + utils::{stream::TryIgnore, u64_from_u8, ReadyExt}, + Err, PduCount, PduEvent, Result, +}; +use database::{Database, Deserialized, KeyVal, Map}; +use futures::{FutureExt, Stream, StreamExt}; +use ruma::{CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; +use tokio::sync::Mutex; use crate::{rooms, Dep}; @@ -25,8 +33,7 @@ struct Services { short: Dep, } -type PdusIterItem = Result<(PduCount, PduEvent)>; -type PdusIterator<'a> = Box + 'a>; +pub type PdusIterItem = (PduCount, PduEvent); type LastTimelineCountCache = Mutex>; impl Data { @@ -46,23 +53,20 @@ impl Data { } } - pub(super) fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { + pub(super) async fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { match self .lasttimelinecount_cache .lock() - .expect("locked") + .await .entry(room_id.to_owned()) { hash_map::Entry::Vacant(v) => { if let Some(last_count) = self - .pdus_until(sender_user, room_id, PduCount::max())? - .find_map(|r| { - // Filter out buggy events - if r.is_err() { - error!("Bad pdu in pdus_since: {:?}", r); - } - r.ok() - }) { + .pdus_until(sender_user, room_id, PduCount::max()) + .await? + .next() + .await + { Ok(*v.insert(last_count.0)) } else { Ok(PduCount::Normal(0)) @@ -73,232 +77,215 @@ impl Data { } /// Returns the `count` of this pdu's id. - pub(super) fn get_pdu_count(&self, event_id: &EventId) -> Result> { + pub(super) async fn get_pdu_count(&self, event_id: &EventId) -> Result { self.eventid_pduid - .get(event_id.as_bytes())? + .qry(event_id) + .await .map(|pdu_id| pdu_count(&pdu_id)) - .transpose() } /// Returns the json of a pdu. - pub(super) fn get_pdu_json(&self, event_id: &EventId) -> Result> { - self.get_non_outlier_pdu_json(event_id)?.map_or_else( - || { - self.eventid_outlierpdu - .get(event_id.as_bytes())? - .map(|pdu| serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))) - .transpose() - }, - |x| Ok(Some(x)), - ) + pub(super) async fn get_pdu_json(&self, event_id: &EventId) -> Result { + if let Ok(pdu) = self.get_non_outlier_pdu_json(event_id).await { + return Ok(pdu); + } + + self.eventid_outlierpdu + .qry(event_id) + .await + .deserialized_json() } /// Returns the json of a pdu. - pub(super) fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result> { - self.eventid_pduid - .get(event_id.as_bytes())? - .map(|pduid| { - self.pduid_pdu - .get(&pduid)? - .ok_or_else(|| Error::bad_database("Invalid pduid in eventid_pduid.")) - }) - .transpose()? - .map(|pdu| serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))) - .transpose() + pub(super) async fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result { + let pduid = self.get_pdu_id(event_id).await?; + + self.pduid_pdu.qry(&pduid).await.deserialized_json() } /// Returns the pdu's id. #[inline] - pub(super) fn get_pdu_id(&self, event_id: &EventId) -> Result>> { - self.eventid_pduid.get(event_id.as_bytes()) + pub(super) async fn get_pdu_id(&self, event_id: &EventId) -> Result> { + self.eventid_pduid.qry(event_id).await } /// Returns the pdu directly from `eventid_pduid` only. - pub(super) fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { - self.eventid_pduid - .get(event_id.as_bytes())? - .map(|pduid| { - self.pduid_pdu - .get(&pduid)? - .ok_or_else(|| Error::bad_database("Invalid pduid in eventid_pduid.")) - }) - .transpose()? - .map(|pdu| serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))) - .transpose() + pub(super) async fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result { + let pduid = self.get_pdu_id(event_id).await?; + + self.pduid_pdu.qry(&pduid).await.deserialized_json() + } + + /// Like get_non_outlier_pdu(), but without the expense of fetching and + /// parsing the PduEvent + pub(super) async fn non_outlier_pdu_exists(&self, event_id: &EventId) -> Result<()> { + let pduid = self.get_pdu_id(event_id).await?; + + self.pduid_pdu.qry(&pduid).await?; + + Ok(()) } /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - pub(super) fn get_pdu(&self, event_id: &EventId) -> Result>> { - if let Some(pdu) = self - .get_non_outlier_pdu(event_id)? - .map_or_else( - || { - self.eventid_outlierpdu - .get(event_id.as_bytes())? - .map(|pdu| serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))) - .transpose() - }, - |x| Ok(Some(x)), - )? - .map(Arc::new) - { - Ok(Some(pdu)) - } else { - Ok(None) + pub(super) async fn get_pdu(&self, event_id: &EventId) -> Result> { + if let Ok(pdu) = self.get_non_outlier_pdu(event_id).await { + return Ok(Arc::new(pdu)); } + + self.eventid_outlierpdu + .qry(event_id) + .await + .deserialized_json() + .map(Arc::new) + } + + /// Like get_non_outlier_pdu(), but without the expense of fetching and + /// parsing the PduEvent + pub(super) async fn outlier_pdu_exists(&self, event_id: &EventId) -> Result<()> { + self.eventid_outlierpdu.qry(event_id).await?; + + Ok(()) + } + + /// Like get_pdu(), but without the expense of fetching and parsing the data + pub(super) async fn pdu_exists(&self, event_id: &EventId) -> bool { + let non_outlier = self.non_outlier_pdu_exists(event_id).map(|res| res.is_ok()); + let outlier = self.outlier_pdu_exists(event_id).map(|res| res.is_ok()); + + //TODO: parallelize + non_outlier.await || outlier.await } /// Returns the pdu. /// /// This does __NOT__ check the outliers `Tree`. - pub(super) fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result> { - self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| { - Ok(Some( - serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))?, - )) - }) + pub(super) async fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result { + self.pduid_pdu.qry(pdu_id).await.deserialized_json() } /// Returns the pdu as a `BTreeMap`. - pub(super) fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result> { - self.pduid_pdu.get(pdu_id)?.map_or(Ok(None), |pdu| { - Ok(Some( - serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db."))?, - )) - }) + pub(super) async fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result { + self.pduid_pdu.qry(pdu_id).await.deserialized_json() } - pub(super) fn append_pdu( - &self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64, - ) -> Result<()> { + pub(super) async fn append_pdu(&self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64) { self.pduid_pdu.insert( pdu_id, &serde_json::to_vec(json).expect("CanonicalJsonObject is always a valid"), - )?; + ); self.lasttimelinecount_cache .lock() - .expect("locked") + .await .insert(pdu.room_id.clone(), PduCount::Normal(count)); - self.eventid_pduid.insert(pdu.event_id.as_bytes(), pdu_id)?; - self.eventid_outlierpdu.remove(pdu.event_id.as_bytes())?; - - Ok(()) + self.eventid_pduid.insert(pdu.event_id.as_bytes(), pdu_id); + self.eventid_outlierpdu.remove(pdu.event_id.as_bytes()); } - pub(super) fn prepend_backfill_pdu( - &self, pdu_id: &[u8], event_id: &EventId, json: &CanonicalJsonObject, - ) -> Result<()> { + pub(super) fn prepend_backfill_pdu(&self, pdu_id: &[u8], event_id: &EventId, json: &CanonicalJsonObject) { self.pduid_pdu.insert( pdu_id, &serde_json::to_vec(json).expect("CanonicalJsonObject is always a valid"), - )?; + ); - self.eventid_pduid.insert(event_id.as_bytes(), pdu_id)?; - self.eventid_outlierpdu.remove(event_id.as_bytes())?; - - Ok(()) + self.eventid_pduid.insert(event_id.as_bytes(), pdu_id); + self.eventid_outlierpdu.remove(event_id.as_bytes()); } /// Removes a pdu and creates a new one with the same id. - pub(super) fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, _pdu: &PduEvent) -> Result<()> { - if self.pduid_pdu.get(pdu_id)?.is_some() { - self.pduid_pdu.insert( - pdu_id, - &serde_json::to_vec(pdu_json).expect("CanonicalJsonObject is always a valid"), - )?; - } else { - return Err(Error::BadRequest(ErrorKind::NotFound, "PDU does not exist.")); + pub(super) async fn replace_pdu( + &self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, _pdu: &PduEvent, + ) -> Result<()> { + if self.pduid_pdu.qry(pdu_id).await.is_not_found() { + return Err!(Request(NotFound("PDU does not exist."))); } + let pdu = serde_json::to_vec(pdu_json)?; + self.pduid_pdu.insert(pdu_id, &pdu); + Ok(()) } /// Returns an iterator over all events and their tokens in a room that /// happened before the event with id `until` in reverse-chronological /// order. - pub(super) fn pdus_until(&self, user_id: &UserId, room_id: &RoomId, until: PduCount) -> Result> { - let (prefix, current) = self.count_to_id(room_id, until, 1, true)?; + pub(super) async fn pdus_until<'a>( + &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, + ) -> Result + Send + 'a> { + let (prefix, current) = self.count_to_id(room_id, until, 1, true).await?; + let stream = self + .pduid_pdu + .rev_raw_stream_from(¤t) + .ignore_err() + .ready_take_while(move |(key, _)| key.starts_with(&prefix)) + .map(move |item| Self::each_pdu(item, user_id)); - let user_id = user_id.to_owned(); - - Ok(Box::new( - self.pduid_pdu - .iter_from(¤t, true) - .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(pdu_id, v)| { - let mut pdu = serde_json::from_slice::(&v) - .map_err(|_| Error::bad_database("PDU in db is invalid."))?; - if pdu.sender != user_id { - pdu.remove_transaction_id()?; - } - pdu.add_age()?; - let count = pdu_count(&pdu_id)?; - Ok((count, pdu)) - }), - )) + Ok(stream) } - pub(super) fn pdus_after(&self, user_id: &UserId, room_id: &RoomId, from: PduCount) -> Result> { - let (prefix, current) = self.count_to_id(room_id, from, 1, false)?; + pub(super) async fn pdus_after<'a>( + &'a self, user_id: &'a UserId, room_id: &'a RoomId, from: PduCount, + ) -> Result + Send + 'a> { + let (prefix, current) = self.count_to_id(room_id, from, 1, false).await?; + let stream = self + .pduid_pdu + .raw_stream_from(¤t) + .ignore_err() + .ready_take_while(move |(key, _)| key.starts_with(&prefix)) + .map(move |item| Self::each_pdu(item, user_id)); - let user_id = user_id.to_owned(); + Ok(stream) + } - Ok(Box::new( - self.pduid_pdu - .iter_from(¤t, false) - .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(move |(pdu_id, v)| { - let mut pdu = serde_json::from_slice::(&v) - .map_err(|_| Error::bad_database("PDU in db is invalid."))?; - if pdu.sender != user_id { - pdu.remove_transaction_id()?; - } - pdu.add_age()?; - let count = pdu_count(&pdu_id)?; - Ok((count, pdu)) - }), - )) + fn each_pdu((pdu_id, pdu): KeyVal<'_>, user_id: &UserId) -> PdusIterItem { + let mut pdu = + serde_json::from_slice::(pdu).expect("PduEvent in pduid_pdu database column is invalid JSON"); + + if pdu.sender != user_id { + pdu.remove_transaction_id().log_err().ok(); + } + + pdu.add_age().log_err().ok(); + let count = pdu_count(pdu_id); + + (count, pdu) } pub(super) fn increment_notification_counts( &self, room_id: &RoomId, notifies: Vec, highlights: Vec, - ) -> Result<()> { - let mut notifies_batch = Vec::new(); - let mut highlights_batch = Vec::new(); + ) { + let _cork = self.db.cork(); + for user in notifies { let mut userroom_id = user.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); - notifies_batch.push(userroom_id); + increment(&self.userroomid_notificationcount, &userroom_id); } + for user in highlights { let mut userroom_id = user.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); - highlights_batch.push(userroom_id); + increment(&self.userroomid_highlightcount, &userroom_id); } - - self.userroomid_notificationcount - .increment_batch(notifies_batch.iter().map(Vec::as_slice))?; - self.userroomid_highlightcount - .increment_batch(highlights_batch.iter().map(Vec::as_slice))?; - Ok(()) } - pub(super) fn count_to_id( + pub(super) async fn count_to_id( &self, room_id: &RoomId, count: PduCount, offset: u64, subtract: bool, ) -> Result<(Vec, Vec)> { let prefix = self .services .short - .get_shortroomid(room_id)? - .ok_or_else(|| Error::bad_database("Looked for bad shortroomid in timeline"))? + .get_shortroomid(room_id) + .await + .map_err(|e| err!(Request(NotFound("Room {room_id:?} not found: {e:?}"))))? .to_be_bytes() .to_vec(); + let mut pdu_id = prefix.clone(); // +1 so we don't send the base event let count_raw = match count { @@ -326,17 +313,23 @@ impl Data { } /// Returns the `count` of this pdu's id. -pub(super) fn pdu_count(pdu_id: &[u8]) -> Result { - let stride = size_of::(); - let pdu_id_len = pdu_id.len(); - let last_u64 = utils::u64_from_bytes(&pdu_id[checked!(pdu_id_len - stride)?..]) - .map_err(|_| Error::bad_database("PDU has invalid count bytes."))?; - let second_last_u64 = - utils::u64_from_bytes(&pdu_id[checked!(pdu_id_len - 2 * stride)?..checked!(pdu_id_len - stride)?]); +pub(super) fn pdu_count(pdu_id: &[u8]) -> PduCount { + const STRIDE: usize = size_of::(); - if matches!(second_last_u64, Ok(0)) { - Ok(PduCount::Backfilled(u64::MAX.saturating_sub(last_u64))) + let pdu_id_len = pdu_id.len(); + let last_u64 = u64_from_u8(&pdu_id[expected!(pdu_id_len - STRIDE)..]); + let second_last_u64 = u64_from_u8(&pdu_id[expected!(pdu_id_len - 2 * STRIDE)..expected!(pdu_id_len - STRIDE)]); + + if second_last_u64 == 0 { + PduCount::Backfilled(u64::MAX.saturating_sub(last_u64)) } else { - Ok(PduCount::Normal(last_u64)) + PduCount::Normal(last_u64) } } + +//TODO: this is an ABA +fn increment(db: &Arc, key: &[u8]) { + let old = db.get(key); + let new = utils::increment(old.ok().as_deref()); + db.insert(key, &new); +} diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 04d9559d..5360d2c9 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1,19 +1,20 @@ mod data; use std::{ + cmp, collections::{BTreeMap, HashSet}, fmt::Write, sync::Arc, }; use conduit::{ - debug, error, info, + debug, err, error, info, pdu::{EventHash, PduBuilder, PduCount, PduEvent}, utils, - utils::{MutexMap, MutexMapGuard}, - validated, warn, Error, Result, Server, + utils::{stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, + validated, warn, Err, Error, Result, Server, }; -use itertools::Itertools; +use futures::{future, future::ready, Future, Stream, StreamExt, TryStreamExt}; use ruma::{ api::{client::error::ErrorKind, federation}, canonical_json::to_canonical_value, @@ -39,6 +40,7 @@ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tokio::sync::RwLock; use self::data::Data; +pub use self::data::PdusIterItem; use crate::{ account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms, rooms::state_compressor::CompressedStateEvent, sending, server_keys, Dep, @@ -129,6 +131,7 @@ impl crate::Service for Service { } fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + /* let lasttimelinecount_cache = self .db .lasttimelinecount_cache @@ -136,6 +139,7 @@ impl crate::Service for Service { .expect("locked") .len(); writeln!(out, "lasttimelinecount_cache: {lasttimelinecount_cache}")?; + */ let mutex_insert = self.mutex_insert.len(); writeln!(out, "insert_mutex: {mutex_insert}")?; @@ -144,11 +148,13 @@ impl crate::Service for Service { } fn clear_cache(&self) { + /* self.db .lasttimelinecount_cache .lock() .expect("locked") .clear(); + */ } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } @@ -156,28 +162,32 @@ impl crate::Service for Service { impl Service { #[tracing::instrument(skip(self), level = "debug")] - pub fn first_pdu_in_room(&self, room_id: &RoomId) -> Result>> { - self.all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id)? + pub async fn first_pdu_in_room(&self, room_id: &RoomId) -> Result> { + self.all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id) + .await? .next() - .map(|o| o.map(|(_, p)| Arc::new(p))) - .transpose() + .await + .map(|(_, p)| Arc::new(p)) + .ok_or_else(|| err!(Request(NotFound("No PDU found in room")))) } #[tracing::instrument(skip(self), level = "debug")] - pub fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result>> { - self.all_pdus(user_id!("@placeholder:conduwuit.placeholder"), room_id)? - .last() - .map(|o| o.map(|(_, p)| Arc::new(p))) - .transpose() + pub async fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result> { + self.pdus_until(user_id!("@placeholder:conduwuit.placeholder"), room_id, PduCount::max()) + .await? + .next() + .await + .map(|(_, p)| Arc::new(p)) + .ok_or_else(|| err!(Request(NotFound("No PDU found in room")))) } #[tracing::instrument(skip(self), level = "debug")] - pub fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { - self.db.last_timeline_count(sender_user, room_id) + pub async fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { + self.db.last_timeline_count(sender_user, room_id).await } /// Returns the `count` of this pdu's id. - pub fn get_pdu_count(&self, event_id: &EventId) -> Result> { self.db.get_pdu_count(event_id) } + pub async fn get_pdu_count(&self, event_id: &EventId) -> Result { self.db.get_pdu_count(event_id).await } // TODO Is this the same as the function above? /* @@ -203,49 +213,56 @@ impl Service { */ /// Returns the json of a pdu. - pub fn get_pdu_json(&self, event_id: &EventId) -> Result> { - self.db.get_pdu_json(event_id) + pub async fn get_pdu_json(&self, event_id: &EventId) -> Result { + self.db.get_pdu_json(event_id).await } /// Returns the json of a pdu. #[inline] - pub fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result> { - self.db.get_non_outlier_pdu_json(event_id) + pub async fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result { + self.db.get_non_outlier_pdu_json(event_id).await } /// Returns the pdu's id. #[inline] - pub fn get_pdu_id(&self, event_id: &EventId) -> Result>> { - self.db.get_pdu_id(event_id) + pub async fn get_pdu_id(&self, event_id: &EventId) -> Result> { + self.db.get_pdu_id(event_id).await } /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. #[inline] - pub fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result> { - self.db.get_non_outlier_pdu(event_id) + pub async fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result { + self.db.get_non_outlier_pdu(event_id).await } /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - pub fn get_pdu(&self, event_id: &EventId) -> Result>> { self.db.get_pdu(event_id) } + pub async fn get_pdu(&self, event_id: &EventId) -> Result> { self.db.get_pdu(event_id).await } + + /// Checks if pdu exists + /// + /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. + pub fn pdu_exists<'a>(&'a self, event_id: &'a EventId) -> impl Future + Send + 'a { + self.db.pdu_exists(event_id) + } /// Returns the pdu. /// /// This does __NOT__ check the outliers `Tree`. - pub fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result> { self.db.get_pdu_from_id(pdu_id) } + pub async fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result { self.db.get_pdu_from_id(pdu_id).await } /// Returns the pdu as a `BTreeMap`. - pub fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result> { - self.db.get_pdu_json_from_id(pdu_id) + pub async fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result { + self.db.get_pdu_json_from_id(pdu_id).await } /// Removes a pdu and creates a new one with the same id. #[tracing::instrument(skip(self), level = "debug")] - pub fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, pdu: &PduEvent) -> Result<()> { - self.db.replace_pdu(pdu_id, pdu_json, pdu) + pub async fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, pdu: &PduEvent) -> Result<()> { + self.db.replace_pdu(pdu_id, pdu_json, pdu).await } /// Creates a new persisted data unit and adds it to a room. @@ -268,8 +285,9 @@ impl Service { let shortroomid = self .services .short - .get_shortroomid(&pdu.room_id)? - .expect("room exists"); + .get_shortroomid(&pdu.room_id) + .await + .map_err(|_| err!(Database("Room does not exist")))?; // Make unsigned fields correct. This is not properly documented in the spec, // but state events need to have previous content in the unsigned field, so @@ -279,17 +297,17 @@ impl Service { .entry("unsigned".to_owned()) .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::default())) { - if let Some(shortstatehash) = self + if let Ok(shortstatehash) = self .services .state_accessor .pdu_shortstatehash(&pdu.event_id) - .unwrap() + .await { - if let Some(prev_state) = self + if let Ok(prev_state) = self .services .state_accessor .state_get(shortstatehash, &pdu.kind.to_string().into(), state_key) - .unwrap() + .await { unsigned.insert( "prev_content".to_owned(), @@ -318,10 +336,12 @@ impl Service { // We must keep track of all events that have been referenced. self.services .pdu_metadata - .mark_as_referenced(&pdu.room_id, &pdu.prev_events)?; + .mark_as_referenced(&pdu.room_id, &pdu.prev_events); + self.services .state - .set_forward_extremities(&pdu.room_id, leaves, state_lock)?; + .set_forward_extremities(&pdu.room_id, leaves, state_lock) + .await; let insert_lock = self.mutex_insert.lock(&pdu.room_id).await; @@ -330,17 +350,17 @@ impl Service { // appending fails self.services .read_receipt - .private_read_set(&pdu.room_id, &pdu.sender, count1)?; + .private_read_set(&pdu.room_id, &pdu.sender, count1); self.services .user - .reset_notification_counts(&pdu.sender, &pdu.room_id)?; + .reset_notification_counts(&pdu.sender, &pdu.room_id); - let count2 = self.services.globals.next_count()?; + let count2 = self.services.globals.next_count().unwrap(); let mut pdu_id = shortroomid.to_be_bytes().to_vec(); pdu_id.extend_from_slice(&count2.to_be_bytes()); // Insert pdu - self.db.append_pdu(&pdu_id, pdu, &pdu_json, count2)?; + self.db.append_pdu(&pdu_id, pdu, &pdu_json, count2).await; drop(insert_lock); @@ -348,12 +368,9 @@ impl Service { let power_levels: RoomPowerLevelsEventContent = self .services .state_accessor - .room_state_get(&pdu.room_id, &StateEventType::RoomPowerLevels, "")? - .map(|ev| { - serde_json::from_str(ev.content.get()) - .map_err(|_| Error::bad_database("invalid m.room.power_levels event")) - }) - .transpose()? + .room_state_get_content(&pdu.room_id, &StateEventType::RoomPowerLevels, "") + .await + .map_err(|_| err!(Database("invalid m.room.power_levels event"))) .unwrap_or_default(); let sync_pdu = pdu.to_sync_room_event(); @@ -365,7 +382,9 @@ impl Service { .services .state_cache .active_local_users_in_room(&pdu.room_id) - .collect_vec(); + .map(ToOwned::to_owned) + .collect::>() + .await; if pdu.kind == TimelineEventType::RoomMember { if let Some(state_key) = &pdu.state_key { @@ -386,23 +405,20 @@ impl Service { let rules_for_user = self .services .account_data - .get(None, user, GlobalAccountDataEventType::PushRules.to_string().into())? - .map(|event| { - serde_json::from_str::(event.get()).map_err(|e| { - warn!("Invalid push rules event in db for user ID {user}: {e}"); - Error::bad_database("Invalid push rules event in db.") - }) - }) - .transpose()? - .map_or_else(|| Ruleset::server_default(user), |ev: PushRulesEvent| ev.content.global); + .get(None, user, GlobalAccountDataEventType::PushRules.to_string().into()) + .await + .and_then(|event| serde_json::from_str::(event.get()).map_err(Into::into)) + .map_err(|e| err!(Database(warn!(?user, ?e, "Invalid push rules event in db for user")))) + .map_or_else(|_| Ruleset::server_default(user), |ev: PushRulesEvent| ev.content.global); let mut highlight = false; let mut notify = false; - for action in - self.services - .pusher - .get_actions(user, &rules_for_user, &power_levels, &sync_pdu, &pdu.room_id)? + for action in self + .services + .pusher + .get_actions(user, &rules_for_user, &power_levels, &sync_pdu, &pdu.room_id) + .await? { match action { Action::Notify => notify = true, @@ -421,31 +437,36 @@ impl Service { highlights.push(user.clone()); } - for push_key in self.services.pusher.get_pushkeys(user) { - self.services - .sending - .send_pdu_push(&pdu_id, user, push_key?)?; - } + self.services + .pusher + .get_pushkeys(user) + .ready_for_each(|push_key| { + self.services + .sending + .send_pdu_push(&pdu_id, user, push_key.to_owned()) + .expect("TODO: replace with future"); + }) + .await; } self.db - .increment_notification_counts(&pdu.room_id, notifies, highlights)?; + .increment_notification_counts(&pdu.room_id, notifies, highlights); match pdu.kind { TimelineEventType::RoomRedaction => { use RoomVersionId::*; - let room_version_id = self.services.state.get_room_version(&pdu.room_id)?; + let room_version_id = self.services.state.get_room_version(&pdu.room_id).await?; match room_version_id { V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { - if self.services.state_accessor.user_can_redact( - redact_id, - &pdu.sender, - &pdu.room_id, - false, - )? { - self.redact_pdu(redact_id, pdu, shortroomid)?; + if self + .services + .state_accessor + .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false) + .await? + { + self.redact_pdu(redact_id, pdu, shortroomid).await?; } } }, @@ -457,13 +478,13 @@ impl Service { })?; if let Some(redact_id) = &content.redacts { - if self.services.state_accessor.user_can_redact( - redact_id, - &pdu.sender, - &pdu.room_id, - false, - )? { - self.redact_pdu(redact_id, pdu, shortroomid)?; + if self + .services + .state_accessor + .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false) + .await? + { + self.redact_pdu(redact_id, pdu, shortroomid).await?; } } }, @@ -492,7 +513,7 @@ impl Service { let invite_state = match content.membership { MembershipState::Invite => { - let state = self.services.state.calculate_invite_state(pdu)?; + let state = self.services.state.calculate_invite_state(pdu).await?; Some(state) }, _ => None, @@ -500,15 +521,18 @@ impl Service { // Update our membership info, we do this here incase a user is invited // and immediately leaves we need the DB to record the invite event for auth - self.services.state_cache.update_membership( - &pdu.room_id, - &target_user_id, - content, - &pdu.sender, - invite_state, - None, - true, - )?; + self.services + .state_cache + .update_membership( + &pdu.room_id, + &target_user_id, + content, + &pdu.sender, + invite_state, + None, + true, + ) + .await?; } }, TimelineEventType::RoomMessage => { @@ -516,9 +540,7 @@ impl Service { .map_err(|_| Error::bad_database("Invalid content in pdu."))?; if let Some(body) = content.body { - self.services - .search - .index_pdu(shortroomid, &pdu_id, &body)?; + self.services.search.index_pdu(shortroomid, &pdu_id, &body); if self.services.admin.is_admin_command(pdu, &body).await { self.services @@ -531,10 +553,10 @@ impl Service { } if let Ok(content) = serde_json::from_str::(pdu.content.get()) { - if let Some(related_pducount) = self.get_pdu_count(&content.relates_to.event_id)? { + if let Ok(related_pducount) = self.get_pdu_count(&content.relates_to.event_id).await { self.services .pdu_metadata - .add_relation(PduCount::Normal(count2), related_pducount)?; + .add_relation(PduCount::Normal(count2), related_pducount); } } @@ -545,14 +567,17 @@ impl Service { } => { // We need to do it again here, because replies don't have // event_id as a top level field - if let Some(related_pducount) = self.get_pdu_count(&in_reply_to.event_id)? { + if let Ok(related_pducount) = self.get_pdu_count(&in_reply_to.event_id).await { self.services .pdu_metadata - .add_relation(PduCount::Normal(count2), related_pducount)?; + .add_relation(PduCount::Normal(count2), related_pducount); } }, Relation::Thread(thread) => { - self.services.threads.add_to_thread(&thread.event_id, pdu)?; + self.services + .threads + .add_to_thread(&thread.event_id, pdu) + .await?; }, _ => {}, // TODO: Aggregate other types } @@ -562,7 +587,8 @@ impl Service { if self .services .state_cache - .appservice_in_room(&pdu.room_id, appservice)? + .appservice_in_room(&pdu.room_id, appservice) + .await { self.services .sending @@ -596,15 +622,14 @@ impl Service { .as_ref() .map_or(false, |state_key| users.is_match(state_key)) }; - let matching_aliases = |aliases: &NamespaceRegex| { + let matching_aliases = |aliases: NamespaceRegex| { self.services .alias .local_aliases_for_room(&pdu.room_id) - .filter_map(Result::ok) - .any(|room_alias| aliases.is_match(room_alias.as_str())) + .ready_any(move |room_alias| aliases.is_match(room_alias.as_str())) }; - if matching_aliases(&appservice.aliases) + if matching_aliases(appservice.aliases.clone()).await || appservice.rooms.is_match(pdu.room_id.as_str()) || matching_users(&appservice.users) { @@ -617,7 +642,7 @@ impl Service { Ok(pdu_id) } - pub fn create_hash_and_sign_event( + pub async fn create_hash_and_sign_event( &self, pdu_builder: PduBuilder, sender: &UserId, @@ -636,47 +661,59 @@ impl Service { let prev_events: Vec<_> = self .services .state - .get_forward_extremities(room_id)? - .into_iter() + .get_forward_extremities(room_id) .take(20) - .collect(); + .map(Arc::from) + .collect() + .await; // If there was no create event yet, assume we are creating a room - let room_version_id = self.services.state.get_room_version(room_id).or_else(|_| { - if event_type == TimelineEventType::RoomCreate { - let content = serde_json::from_str::(content.get()) - .expect("Invalid content in RoomCreate pdu."); - Ok(content.room_version) - } else { - Err(Error::InconsistentRoomState( - "non-create event for room of unknown version", - room_id.to_owned(), - )) - } - })?; + let room_version_id = self + .services + .state + .get_room_version(room_id) + .await + .or_else(|_| { + if event_type == TimelineEventType::RoomCreate { + let content = serde_json::from_str::(content.get()) + .expect("Invalid content in RoomCreate pdu."); + Ok(content.room_version) + } else { + Err(Error::InconsistentRoomState( + "non-create event for room of unknown version", + room_id.to_owned(), + )) + } + })?; let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); - let auth_events = - self.services - .state - .get_auth_events(room_id, &event_type, sender, state_key.as_deref(), &content)?; + let auth_events = self + .services + .state + .get_auth_events(room_id, &event_type, sender, state_key.as_deref(), &content) + .await?; // Our depth is the maximum depth of prev_events + 1 let depth = prev_events .iter() - .filter_map(|event_id| Some(self.get_pdu(event_id).ok()??.depth)) - .max() - .unwrap_or_else(|| uint!(0)) + .stream() + .map(Ok) + .and_then(|event_id| self.get_pdu(event_id)) + .and_then(|pdu| future::ok(pdu.depth)) + .ignore_err() + .ready_fold(uint!(0), cmp::max) + .await .saturating_add(uint!(1)); let mut unsigned = unsigned.unwrap_or_default(); if let Some(state_key) = &state_key { - if let Some(prev_pdu) = - self.services - .state_accessor - .room_state_get(room_id, &event_type.to_string().into(), state_key)? + if let Ok(prev_pdu) = self + .services + .state_accessor + .room_state_get(room_id, &event_type.to_string().into(), state_key) + .await { unsigned.insert( "prev_content".to_owned(), @@ -727,19 +764,22 @@ impl Service { signatures: None, }; + let auth_fetch = |k: &StateEventType, s: &str| { + let key = (k.clone(), s.to_owned()); + ready(auth_events.get(&key)) + }; + let auth_check = state_res::auth_check( &room_version, &pdu, - None::, // TODO: third_party_invite - |k, s| auth_events.get(&(k.clone(), s.to_owned())), + None, // TODO: third_party_invite + auth_fetch, ) - .map_err(|e| { - error!("Auth check failed: {:?}", e); - Error::BadRequest(ErrorKind::forbidden(), "Auth check failed.") - })?; + .await + .map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?; if !auth_check { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Event is not authorized.")); + return Err!(Request(Forbidden("Event is not authorized."))); } // Hash and sign @@ -795,7 +835,8 @@ impl Service { let _shorteventid = self .services .short - .get_or_create_shorteventid(&pdu.event_id)?; + .get_or_create_shorteventid(&pdu.event_id) + .await; Ok((pdu, pdu_json)) } @@ -811,108 +852,117 @@ impl Service { room_id: &RoomId, state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result> { - let (pdu, pdu_json) = self.create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock)?; - if let Some(admin_room) = self.services.admin.get_admin_room()? { - if admin_room == room_id { - match pdu.event_type() { - TimelineEventType::RoomEncryption => { - warn!("Encryption is not allowed in the admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Encryption is not allowed in the admins room", - )); - }, - TimelineEventType::RoomMember => { - let target = pdu - .state_key() - .filter(|v| v.starts_with('@')) - .unwrap_or(sender.as_str()); - let server_user = &self.services.globals.server_user.to_string(); + let (pdu, pdu_json) = self + .create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock) + .await?; - let content = serde_json::from_str::(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid content in pdu"))?; + if self.services.admin.is_admin_room(&pdu.room_id).await { + match pdu.event_type() { + TimelineEventType::RoomEncryption => { + warn!("Encryption is not allowed in the admins room"); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Encryption is not allowed in the admins room", + )); + }, + TimelineEventType::RoomMember => { + let target = pdu + .state_key() + .filter(|v| v.starts_with('@')) + .unwrap_or(sender.as_str()); + let server_user = &self.services.globals.server_user.to_string(); - if content.membership == MembershipState::Leave { - if target == server_user { - warn!("Server user cannot leave from admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server user cannot leave from admins room.", - )); - } + let content = serde_json::from_str::(pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid content in pdu"))?; - let count = self - .services - .state_cache - .room_members(room_id) - .filter_map(Result::ok) - .filter(|m| self.services.globals.server_is_ours(m.server_name()) && m != target) - .count(); - if count < 2 { - warn!("Last admin cannot leave from admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Last admin cannot leave from admins room.", - )); - } + if content.membership == MembershipState::Leave { + if target == server_user { + warn!("Server user cannot leave from admins room"); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server user cannot leave from admins room.", + )); } - if content.membership == MembershipState::Ban && pdu.state_key().is_some() { - if target == server_user { - warn!("Server user cannot be banned in admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server user cannot be banned in admins room.", - )); - } + let count = self + .services + .state_cache + .room_members(&pdu.room_id) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .ready_filter(|user| *user != target) + .boxed() + .count() + .await; - let count = self - .services - .state_cache - .room_members(room_id) - .filter_map(Result::ok) - .filter(|m| self.services.globals.server_is_ours(m.server_name()) && m != target) - .count(); - if count < 2 { - warn!("Last admin cannot be banned in admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Last admin cannot be banned in admins room.", - )); - } + if count < 2 { + warn!("Last admin cannot leave from admins room"); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Last admin cannot leave from admins room.", + )); } - }, - _ => {}, - } + } + + if content.membership == MembershipState::Ban && pdu.state_key().is_some() { + if target == server_user { + warn!("Server user cannot be banned in admins room"); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Server user cannot be banned in admins room.", + )); + } + + let count = self + .services + .state_cache + .room_members(&pdu.room_id) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .ready_filter(|user| *user != target) + .boxed() + .count() + .await; + + if count < 2 { + warn!("Last admin cannot be banned in admins room"); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Last admin cannot be banned in admins room.", + )); + } + } + }, + _ => {}, } } // If redaction event is not authorized, do not append it to the timeline if pdu.kind == TimelineEventType::RoomRedaction { use RoomVersionId::*; - match self.services.state.get_room_version(&pdu.room_id)? { + match self.services.state.get_room_version(&pdu.room_id).await? { V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { if !self .services .state_accessor - .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false)? + .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false) + .await? { - return Err(Error::BadRequest(ErrorKind::forbidden(), "User cannot redact this event.")); + return Err!(Request(Forbidden("User cannot redact this event."))); } }; }, _ => { let content = serde_json::from_str::(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; + .map_err(|e| err!(Database("Invalid content in redaction pdu: {e:?}")))?; if let Some(redact_id) = &content.redacts { if !self .services .state_accessor - .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false)? + .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false) + .await? { - return Err(Error::BadRequest(ErrorKind::forbidden(), "User cannot redact this event.")); + return Err!(Request(Forbidden("User cannot redact this event."))); } } }, @@ -922,7 +972,7 @@ impl Service { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't // fail. - let statehashid = self.services.state.append_to_state(&pdu)?; + let statehashid = self.services.state.append_to_state(&pdu).await?; let pdu_id = self .append_pdu( @@ -939,14 +989,15 @@ impl Service { // in time where events in the current room state do not exist self.services .state - .set_room_state(room_id, statehashid, state_lock)?; + .set_room_state(&pdu.room_id, statehashid, state_lock); let mut servers: HashSet = self .services .state_cache - .room_servers(room_id) - .filter_map(Result::ok) - .collect(); + .room_servers(&pdu.room_id) + .map(ToOwned::to_owned) + .collect() + .await; // In case we are kicking or banning a user, we need to inform their server of // the change @@ -966,7 +1017,8 @@ impl Service { self.services .sending - .send_pdu_servers(servers.into_iter(), &pdu_id)?; + .send_pdu_servers(servers.iter().map(AsRef::as_ref).stream(), &pdu_id) + .await?; Ok(pdu.event_id) } @@ -988,15 +1040,19 @@ impl Service { // fail. self.services .state - .set_event_state(&pdu.event_id, &pdu.room_id, state_ids_compressed)?; + .set_event_state(&pdu.event_id, &pdu.room_id, state_ids_compressed) + .await?; if soft_fail { self.services .pdu_metadata - .mark_as_referenced(&pdu.room_id, &pdu.prev_events)?; + .mark_as_referenced(&pdu.room_id, &pdu.prev_events); + self.services .state - .set_forward_extremities(&pdu.room_id, new_room_leaves, state_lock)?; + .set_forward_extremities(&pdu.room_id, new_room_leaves, state_lock) + .await; + return Ok(None); } @@ -1009,71 +1065,71 @@ impl Service { /// Returns an iterator over all PDUs in a room. #[inline] - pub fn all_pdus<'a>( - &'a self, user_id: &UserId, room_id: &RoomId, - ) -> Result> + 'a> { - self.pdus_after(user_id, room_id, PduCount::min()) + pub async fn all_pdus<'a>( + &'a self, user_id: &'a UserId, room_id: &'a RoomId, + ) -> Result + Send + 'a> { + self.pdus_after(user_id, room_id, PduCount::min()).await } /// Returns an iterator over all events and their tokens in a room that /// happened before the event with id `until` in reverse-chronological /// order. #[tracing::instrument(skip(self), level = "debug")] - pub fn pdus_until<'a>( - &'a self, user_id: &UserId, room_id: &RoomId, until: PduCount, - ) -> Result> + 'a> { - self.db.pdus_until(user_id, room_id, until) + pub async fn pdus_until<'a>( + &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, + ) -> Result + Send + 'a> { + self.db.pdus_until(user_id, room_id, until).await } /// Returns an iterator over all events and their token in a room that /// happened after the event with id `from` in chronological order. #[tracing::instrument(skip(self), level = "debug")] - pub fn pdus_after<'a>( - &'a self, user_id: &UserId, room_id: &RoomId, from: PduCount, - ) -> Result> + 'a> { - self.db.pdus_after(user_id, room_id, from) + pub async fn pdus_after<'a>( + &'a self, user_id: &'a UserId, room_id: &'a RoomId, from: PduCount, + ) -> Result + Send + 'a> { + self.db.pdus_after(user_id, room_id, from).await } /// Replace a PDU with the redacted form. #[tracing::instrument(skip(self, reason))] - pub fn redact_pdu(&self, event_id: &EventId, reason: &PduEvent, shortroomid: u64) -> Result<()> { + pub async fn redact_pdu(&self, event_id: &EventId, reason: &PduEvent, shortroomid: u64) -> Result<()> { // TODO: Don't reserialize, keep original json - if let Some(pdu_id) = self.get_pdu_id(event_id)? { - let mut pdu = self - .get_pdu_from_id(&pdu_id)? - .ok_or_else(|| Error::bad_database("PDU ID points to invalid PDU."))?; + let Ok(pdu_id) = self.get_pdu_id(event_id).await else { + // If event does not exist, just noop + return Ok(()); + }; - if let Ok(content) = serde_json::from_str::(pdu.content.get()) { - if let Some(body) = content.body { - self.services - .search - .deindex_pdu(shortroomid, &pdu_id, &body)?; - } + let mut pdu = self + .get_pdu_from_id(&pdu_id) + .await + .map_err(|e| err!(Database(error!(?pdu_id, ?event_id, ?e, "PDU ID points to invalid PDU."))))?; + + if let Ok(content) = serde_json::from_str::(pdu.content.get()) { + if let Some(body) = content.body { + self.services + .search + .deindex_pdu(shortroomid, &pdu_id, &body); } - - let room_version_id = self.services.state.get_room_version(&pdu.room_id)?; - - pdu.redact(room_version_id, reason)?; - - self.replace_pdu( - &pdu_id, - &utils::to_canonical_object(&pdu).map_err(|e| { - error!("Failed to convert PDU to canonical JSON: {}", e); - Error::bad_database("Failed to convert PDU to canonical JSON.") - })?, - &pdu, - )?; } - // If event does not exist, just noop - Ok(()) + + let room_version_id = self.services.state.get_room_version(&pdu.room_id).await?; + + pdu.redact(room_version_id, reason)?; + + let obj = utils::to_canonical_object(&pdu) + .map_err(|e| err!(Database(error!(?event_id, ?e, "Failed to convert PDU to canonical JSON"))))?; + + self.replace_pdu(&pdu_id, &obj, &pdu).await } #[tracing::instrument(skip(self))] pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { let first_pdu = self - .all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id)? + .all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id) + .await? .next() - .expect("Room is not empty")?; + .await + .expect("Room is not empty"); if first_pdu.0 < from { // No backfill required, there are still events between them @@ -1083,17 +1139,18 @@ impl Service { let power_levels: RoomPowerLevelsEventContent = self .services .state_accessor - .room_state_get(room_id, &StateEventType::RoomPowerLevels, "")? + .room_state_get(room_id, &StateEventType::RoomPowerLevels, "") + .await .map(|ev| { serde_json::from_str(ev.content.get()) .map_err(|_| Error::bad_database("invalid m.room.power_levels event")) + .unwrap() }) - .transpose()? .unwrap_or_default(); let room_mods = power_levels.users.iter().filter_map(|(user_id, level)| { if level > &power_levels.users_default && !self.services.globals.user_is_local(user_id) { - Some(user_id.server_name().to_owned()) + Some(user_id.server_name()) } else { None } @@ -1103,34 +1160,43 @@ impl Service { .services .alias .local_aliases_for_room(room_id) - .filter_map(|alias| { - alias - .ok() - .filter(|alias| !self.services.globals.server_is_ours(alias.server_name())) - .map(|alias| alias.server_name().to_owned()) + .ready_filter_map(|alias| { + self.services + .globals + .server_is_ours(alias.server_name()) + .then_some(alias.server_name()) }); - let servers = room_mods + let mut servers = room_mods + .stream() .chain(room_alias_servers) - .chain(self.services.server.config.trusted_servers.clone()) - .filter(|server_name| { - if self.services.globals.server_is_ours(server_name) { - return false; - } - + .map(ToOwned::to_owned) + .chain( + self.services + .server + .config + .trusted_servers + .iter() + .map(ToOwned::to_owned) + .stream(), + ) + .ready_filter(|server_name| !self.services.globals.server_is_ours(server_name)) + .filter_map(|server_name| async move { self.services .state_cache - .server_in_room(server_name, room_id) - .unwrap_or(false) - }); + .server_in_room(&server_name, room_id) + .await + .then_some(server_name) + }) + .boxed(); - for backfill_server in servers { + while let Some(ref backfill_server) = servers.next().await { info!("Asking {backfill_server} for backfill"); let response = self .services .sending .send_federation_request( - &backfill_server, + backfill_server, federation::backfill::get_backfill::v1::Request { room_id: room_id.to_owned(), v: vec![first_pdu.1.event_id.as_ref().to_owned()], @@ -1142,7 +1208,7 @@ impl Service { Ok(response) => { let pub_key_map = RwLock::new(BTreeMap::new()); for pdu in response.pdus { - if let Err(e) = self.backfill_pdu(&backfill_server, pdu, &pub_key_map).await { + if let Err(e) = self.backfill_pdu(backfill_server, pdu, &pub_key_map).await { warn!("Failed to add backfilled pdu in room {room_id}: {e}"); } } @@ -1163,7 +1229,7 @@ impl Service { &self, origin: &ServerName, pdu: Box, pub_key_map: &RwLock>>, ) -> Result<()> { - let (event_id, value, room_id) = self.services.event_handler.parse_incoming_pdu(&pdu)?; + let (event_id, value, room_id) = self.services.event_handler.parse_incoming_pdu(&pdu).await?; // Lock so we cannot backfill the same pdu twice at the same time let mutex_lock = self @@ -1174,7 +1240,7 @@ impl Service { .await; // Skip the PDU if we already have it as a timeline event - if let Some(pdu_id) = self.get_pdu_id(&event_id)? { + if let Ok(pdu_id) = self.get_pdu_id(&event_id).await { let pdu_id = pdu_id.to_vec(); debug!("We already know {event_id} at {pdu_id:?}"); return Ok(()); @@ -1190,36 +1256,38 @@ impl Service { .handle_incoming_pdu(origin, &room_id, &event_id, value, false, pub_key_map) .await?; - let value = self.get_pdu_json(&event_id)?.expect("We just created it"); - let pdu = self.get_pdu(&event_id)?.expect("We just created it"); + let value = self + .get_pdu_json(&event_id) + .await + .expect("We just created it"); + let pdu = self.get_pdu(&event_id).await.expect("We just created it"); let shortroomid = self .services .short - .get_shortroomid(&room_id)? + .get_shortroomid(&room_id) + .await .expect("room exists"); let insert_lock = self.mutex_insert.lock(&room_id).await; let max = u64::MAX; - let count = self.services.globals.next_count()?; + let count = self.services.globals.next_count().unwrap(); let mut pdu_id = shortroomid.to_be_bytes().to_vec(); pdu_id.extend_from_slice(&0_u64.to_be_bytes()); pdu_id.extend_from_slice(&(validated!(max - count)).to_be_bytes()); // Insert pdu - self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value)?; + self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value); drop(insert_lock); if pdu.kind == TimelineEventType::RoomMessage { let content = serde_json::from_str::(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid content in pdu."))?; + .map_err(|e| err!(Database("Invalid content in pdu: {e:?}")))?; if let Some(body) = content.body { - self.services - .search - .index_pdu(shortroomid, &pdu_id, &body)?; + self.services.search.index_pdu(shortroomid, &pdu_id, &body); } } drop(mutex_lock); diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 3cf1cdd5..bcfce616 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -46,7 +46,7 @@ impl Service { /// Sets a user as typing until the timeout timestamp is reached or /// roomtyping_remove is called. pub async fn typing_add(&self, user_id: &UserId, room_id: &RoomId, timeout: u64) -> Result<()> { - debug_info!("typing started {:?} in {:?} timeout:{:?}", user_id, room_id, timeout); + debug_info!("typing started {user_id:?} in {room_id:?} timeout:{timeout:?}"); // update clients self.typing .write() @@ -54,17 +54,19 @@ impl Service { .entry(room_id.to_owned()) .or_default() .insert(user_id.to_owned(), timeout); + self.last_typing_update .write() .await .insert(room_id.to_owned(), self.services.globals.next_count()?); + if self.typing_update_sender.send(room_id.to_owned()).is_err() { trace!("receiver found what it was looking for and is no longer interested"); } // update federation if self.services.globals.user_is_local(user_id) { - self.federation_send(room_id, user_id, true)?; + self.federation_send(room_id, user_id, true).await?; } Ok(()) @@ -72,7 +74,7 @@ impl Service { /// Removes a user from typing before the timeout is reached. pub async fn typing_remove(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { - debug_info!("typing stopped {:?} in {:?}", user_id, room_id); + debug_info!("typing stopped {user_id:?} in {room_id:?}"); // update clients self.typing .write() @@ -80,31 +82,31 @@ impl Service { .entry(room_id.to_owned()) .or_default() .remove(user_id); + self.last_typing_update .write() .await .insert(room_id.to_owned(), self.services.globals.next_count()?); + if self.typing_update_sender.send(room_id.to_owned()).is_err() { trace!("receiver found what it was looking for and is no longer interested"); } // update federation if self.services.globals.user_is_local(user_id) { - self.federation_send(room_id, user_id, false)?; + self.federation_send(room_id, user_id, false).await?; } Ok(()) } - pub async fn wait_for_update(&self, room_id: &RoomId) -> Result<()> { + pub async fn wait_for_update(&self, room_id: &RoomId) { let mut receiver = self.typing_update_sender.subscribe(); while let Ok(next) = receiver.recv().await { if next == room_id { break; } } - - Ok(()) } /// Makes sure that typing events with old timestamps get removed. @@ -123,30 +125,30 @@ impl Service { removable.push(user.clone()); } } - - drop(typing); }; if !removable.is_empty() { let typing = &mut self.typing.write().await; let room = typing.entry(room_id.to_owned()).or_default(); for user in &removable { - debug_info!("typing timeout {:?} in {:?}", &user, room_id); + debug_info!("typing timeout {user:?} in {room_id:?}"); room.remove(user); } + // update clients self.last_typing_update .write() .await .insert(room_id.to_owned(), self.services.globals.next_count()?); + if self.typing_update_sender.send(room_id.to_owned()).is_err() { trace!("receiver found what it was looking for and is no longer interested"); } // update federation - for user in removable { - if self.services.globals.user_is_local(&user) { - self.federation_send(room_id, &user, false)?; + for user in &removable { + if self.services.globals.user_is_local(user) { + self.federation_send(room_id, user, false).await?; } } } @@ -183,7 +185,7 @@ impl Service { }) } - fn federation_send(&self, room_id: &RoomId, user_id: &UserId, typing: bool) -> Result<()> { + async fn federation_send(&self, room_id: &RoomId, user_id: &UserId, typing: bool) -> Result<()> { debug_assert!( self.services.globals.user_is_local(user_id), "tried to broadcast typing status of remote user", @@ -197,7 +199,8 @@ impl Service { self.services .sending - .send_edu_room(room_id, serde_json::to_vec(&edu).expect("Serialized Edu::Typing"))?; + .send_edu_room(room_id, serde_json::to_vec(&edu).expect("Serialized Edu::Typing")) + .await?; Ok(()) } diff --git a/src/service/rooms/user/data.rs b/src/service/rooms/user/data.rs index c7131615..d4d9874c 100644 --- a/src/service/rooms/user/data.rs +++ b/src/service/rooms/user/data.rs @@ -1,8 +1,9 @@ use std::sync::Arc; -use conduit::{utils, Error, Result}; -use database::Map; -use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; +use conduit::Result; +use database::{Deserialized, Map}; +use futures::{Stream, StreamExt}; +use ruma::{RoomId, UserId}; use crate::{globals, rooms, Dep}; @@ -11,13 +12,13 @@ pub(super) struct Data { userroomid_highlightcount: Arc, roomuserid_lastnotificationread: Arc, roomsynctoken_shortstatehash: Arc, - userroomid_joined: Arc, services: Services, } struct Services { globals: Dep, short: Dep, + state_cache: Dep, } impl Data { @@ -28,15 +29,15 @@ impl Data { userroomid_highlightcount: db["userroomid_highlightcount"].clone(), roomuserid_lastnotificationread: db["userroomid_highlightcount"].clone(), //< NOTE: known bug from conduit roomsynctoken_shortstatehash: db["roomsynctoken_shortstatehash"].clone(), - userroomid_joined: db["userroomid_joined"].clone(), services: Services { globals: args.depend::("globals"), short: args.depend::("rooms::short"), + state_cache: args.depend::("rooms::state_cache"), }, } } - pub(super) fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { + pub(super) fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) { let mut userroom_id = user_id.as_bytes().to_vec(); userroom_id.push(0xFF); userroom_id.extend_from_slice(room_id.as_bytes()); @@ -45,128 +46,73 @@ impl Data { roomuser_id.extend_from_slice(user_id.as_bytes()); self.userroomid_notificationcount - .insert(&userroom_id, &0_u64.to_be_bytes())?; + .insert(&userroom_id, &0_u64.to_be_bytes()); self.userroomid_highlightcount - .insert(&userroom_id, &0_u64.to_be_bytes())?; + .insert(&userroom_id, &0_u64.to_be_bytes()); self.roomuserid_lastnotificationread - .insert(&roomuser_id, &self.services.globals.next_count()?.to_be_bytes())?; - - Ok(()) + .insert(&roomuser_id, &self.services.globals.next_count().unwrap().to_be_bytes()); } - pub(super) fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); - + pub(super) async fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (user_id, room_id); self.userroomid_notificationcount - .get(&userroom_id)? - .map_or(Ok(0), |bytes| { - utils::u64_from_bytes(&bytes).map_err(|_| Error::bad_database("Invalid notification count in db.")) - }) + .qry(&key) + .await + .deserialized() + .unwrap_or(0) } - pub(super) fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); - + pub(super) async fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (user_id, room_id); self.userroomid_highlightcount - .get(&userroom_id)? - .map_or(Ok(0), |bytes| { - utils::u64_from_bytes(&bytes).map_err(|_| Error::bad_database("Invalid highlight count in db.")) - }) + .qry(&key) + .await + .deserialized() + .unwrap_or(0) } - pub(super) fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> Result { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(user_id.as_bytes()); - - Ok(self - .roomuserid_lastnotificationread - .get(&key)? - .map(|bytes| { - utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Count in roomuserid_lastprivatereadupdate is invalid.")) - }) - .transpose()? - .unwrap_or(0)) + pub(super) async fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (room_id, user_id); + self.roomuserid_lastnotificationread + .qry(&key) + .await + .deserialized() + .unwrap_or(0) } - pub(super) fn associate_token_shortstatehash( - &self, room_id: &RoomId, token: u64, shortstatehash: u64, - ) -> Result<()> { + pub(super) async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: u64) { let shortroomid = self .services .short - .get_shortroomid(room_id)? + .get_shortroomid(room_id) + .await .expect("room exists"); let mut key = shortroomid.to_be_bytes().to_vec(); key.extend_from_slice(&token.to_be_bytes()); self.roomsynctoken_shortstatehash - .insert(&key, &shortstatehash.to_be_bytes()) + .insert(&key, &shortstatehash.to_be_bytes()); } - pub(super) fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result> { - let shortroomid = self - .services - .short - .get_shortroomid(room_id)? - .expect("room exists"); - - let mut key = shortroomid.to_be_bytes().to_vec(); - key.extend_from_slice(&token.to_be_bytes()); + pub(super) async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result { + let shortroomid = self.services.short.get_shortroomid(room_id).await?; + let key: &[u64] = &[shortroomid, token]; self.roomsynctoken_shortstatehash - .get(&key)? - .map(|bytes| { - utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Invalid shortstatehash in roomsynctoken_shortstatehash")) - }) - .transpose() + .qry(key) + .await + .deserialized() } + //TODO: optimize; replace point-queries with dual iteration pub(super) fn get_shared_rooms<'a>( - &'a self, users: Vec, - ) -> Result> + 'a>> { - let iterators = users.into_iter().map(move |user_id| { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - - self.userroomid_joined - .scan_prefix(prefix) - .map(|(key, _)| { - let roomid_index = key - .iter() - .enumerate() - .find(|(_, &b)| b == 0xFF) - .ok_or_else(|| Error::bad_database("Invalid userroomid_joined in db."))? - .0 - .saturating_add(1); // +1 because the room id starts AFTER the separator - - let room_id = key[roomid_index..].to_vec(); - - Ok::<_, Error>(room_id) - }) - .filter_map(Result::ok) - }); - - // We use the default compare function because keys are sorted correctly (not - // reversed) - Ok(Box::new( - utils::common_elements(iterators, Ord::cmp) - .expect("users is not empty") - .map(|bytes| { - RoomId::parse( - utils::string_from_bytes(&bytes) - .map_err(|_| Error::bad_database("Invalid RoomId bytes in userroomid_joined"))?, - ) - .map_err(|_| Error::bad_database("Invalid RoomId in userroomid_joined.")) - }), - )) + &'a self, user_a: &'a UserId, user_b: &'a UserId, + ) -> impl Stream + Send + 'a { + self.services + .state_cache + .rooms_joined(user_a) + .filter(|room_id| self.services.state_cache.is_joined(user_b, room_id)) } } diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 93d38470..d9d90ecf 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -3,7 +3,8 @@ mod data; use std::sync::Arc; use conduit::Result; -use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; +use futures::{pin_mut, Stream, StreamExt}; +use ruma::{RoomId, UserId}; use self::data::Data; @@ -22,32 +23,49 @@ impl crate::Service for Service { } impl Service { - pub fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) -> Result<()> { - self.db.reset_notification_counts(user_id, room_id) + #[inline] + pub fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) { + self.db.reset_notification_counts(user_id, room_id); } - pub fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> Result { - self.db.notification_count(user_id, room_id) + #[inline] + pub async fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + self.db.notification_count(user_id, room_id).await } - pub fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> Result { - self.db.highlight_count(user_id, room_id) + #[inline] + pub async fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + self.db.highlight_count(user_id, room_id).await } - pub fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> Result { - self.db.last_notification_read(user_id, room_id) + #[inline] + pub async fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + self.db.last_notification_read(user_id, room_id).await } - pub fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: u64) -> Result<()> { + #[inline] + pub async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: u64) { self.db .associate_token_shortstatehash(room_id, token, shortstatehash) + .await; } - pub fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result> { - self.db.get_token_shortstatehash(room_id, token) + #[inline] + pub async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result { + self.db.get_token_shortstatehash(room_id, token).await } - pub fn get_shared_rooms(&self, users: Vec) -> Result> + '_> { - self.db.get_shared_rooms(users) + #[inline] + pub fn get_shared_rooms<'a>( + &'a self, user_a: &'a UserId, user_b: &'a UserId, + ) -> impl Stream + Send + 'a { + self.db.get_shared_rooms(user_a, user_b) + } + + pub async fn has_shared_rooms<'a>(&'a self, user_a: &'a UserId, user_b: &'a UserId) -> bool { + let get_shared_rooms = self.get_shared_rooms(user_a, user_b); + + pin_mut!(get_shared_rooms); + get_shared_rooms.next().await.is_some() } } diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 6c8e2544..b96f9a03 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -1,14 +1,21 @@ use std::sync::Arc; -use conduit::{utils, Error, Result}; -use database::{Database, Map}; +use conduit::{ + utils, + utils::{stream::TryIgnore, ReadyExt}, + Error, Result, +}; +use database::{Database, Deserialized, Map}; +use futures::{Stream, StreamExt}; use ruma::{ServerName, UserId}; use super::{Destination, SendingEvent}; use crate::{globals, Dep}; -type OutgoingSendingIter<'a> = Box, Destination, SendingEvent)>> + 'a>; -type SendingEventIter<'a> = Box, SendingEvent)>> + 'a>; +pub(super) type OutgoingItem = (Key, SendingEvent, Destination); +pub(super) type SendingItem = (Key, SendingEvent); +pub(super) type QueueItem = (Key, SendingEvent); +pub(super) type Key = Vec; pub struct Data { servercurrentevent_data: Arc, @@ -36,85 +43,34 @@ impl Data { } } - #[inline] - pub fn active_requests(&self) -> OutgoingSendingIter<'_> { - Box::new( - self.servercurrentevent_data - .iter() - .map(|(key, v)| parse_servercurrentevent(&key, v).map(|(k, e)| (key, k, e))), - ) - } + pub(super) fn delete_active_request(&self, key: &[u8]) { self.servercurrentevent_data.remove(key); } - #[inline] - pub fn active_requests_for<'a>(&'a self, destination: &Destination) -> SendingEventIter<'a> { + pub(super) async fn delete_all_active_requests_for(&self, destination: &Destination) { let prefix = destination.get_prefix(); - Box::new( - self.servercurrentevent_data - .scan_prefix(prefix) - .map(|(key, v)| parse_servercurrentevent(&key, v).map(|(_, e)| (key, e))), - ) + self.servercurrentevent_data + .raw_keys_prefix(&prefix) + .ignore_err() + .ready_for_each(|key| self.servercurrentevent_data.remove(key)) + .await; } - pub(super) fn delete_active_request(&self, key: &[u8]) -> Result<()> { self.servercurrentevent_data.remove(key) } - - pub(super) fn delete_all_active_requests_for(&self, destination: &Destination) -> Result<()> { + pub(super) async fn delete_all_requests_for(&self, destination: &Destination) { let prefix = destination.get_prefix(); - for (key, _) in self.servercurrentevent_data.scan_prefix(prefix) { - self.servercurrentevent_data.remove(&key)?; - } + self.servercurrentevent_data + .raw_keys_prefix(&prefix) + .ignore_err() + .ready_for_each(|key| self.servercurrentevent_data.remove(key)) + .await; - Ok(()) - } - - pub(super) fn delete_all_requests_for(&self, destination: &Destination) -> Result<()> { - let prefix = destination.get_prefix(); - for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) { - self.servercurrentevent_data.remove(&key).unwrap(); - } - - for (key, _) in self.servernameevent_data.scan_prefix(prefix) { - self.servernameevent_data.remove(&key).unwrap(); - } - - Ok(()) - } - - pub(super) fn queue_requests(&self, requests: &[(&Destination, SendingEvent)]) -> Result>> { - let mut batch = Vec::new(); - let mut keys = Vec::new(); - for (destination, event) in requests { - let mut key = destination.get_prefix(); - if let SendingEvent::Pdu(value) = &event { - key.extend_from_slice(value); - } else { - key.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); - } - let value = if let SendingEvent::Edu(value) = &event { - &**value - } else { - &[] - }; - batch.push((key.clone(), value.to_owned())); - keys.push(key); - } self.servernameevent_data - .insert_batch(batch.iter().map(database::KeyVal::from))?; - Ok(keys) + .raw_keys_prefix(&prefix) + .ignore_err() + .ready_for_each(|key| self.servernameevent_data.remove(key)) + .await; } - pub fn queued_requests<'a>( - &'a self, destination: &Destination, - ) -> Box)>> + 'a> { - let prefix = destination.get_prefix(); - return Box::new( - self.servernameevent_data - .scan_prefix(prefix) - .map(|(k, v)| parse_servercurrentevent(&k, v).map(|(_, ev)| (ev, k))), - ); - } - - pub(super) fn mark_as_active(&self, events: &[(SendingEvent, Vec)]) -> Result<()> { - for (e, key) in events { + pub(super) fn mark_as_active(&self, events: &[QueueItem]) { + for (key, e) in events { if key.is_empty() { continue; } @@ -124,29 +80,87 @@ impl Data { } else { &[] }; - self.servercurrentevent_data.insert(key, value)?; - self.servernameevent_data.remove(key)?; + self.servercurrentevent_data.insert(key, value); + self.servernameevent_data.remove(key); + } + } + + #[inline] + pub fn active_requests(&self) -> impl Stream + Send + '_ { + self.servercurrentevent_data + .raw_stream() + .ignore_err() + .map(|(key, val)| { + let (dest, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); + + (key.to_vec(), event, dest) + }) + } + + #[inline] + pub fn active_requests_for<'a>(&'a self, destination: &Destination) -> impl Stream + Send + 'a { + let prefix = destination.get_prefix(); + self.servercurrentevent_data + .stream_raw_prefix(&prefix) + .ignore_err() + .map(|(key, val)| { + let (_, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); + + (key.to_vec(), event) + }) + } + + pub(super) fn queue_requests(&self, requests: &[(&SendingEvent, &Destination)]) -> Vec> { + let mut batch = Vec::new(); + let mut keys = Vec::new(); + for (event, destination) in requests { + let mut key = destination.get_prefix(); + if let SendingEvent::Pdu(value) = &event { + key.extend_from_slice(value); + } else { + key.extend_from_slice(&self.services.globals.next_count().unwrap().to_be_bytes()); + } + let value = if let SendingEvent::Edu(value) = &event { + &**value + } else { + &[] + }; + batch.push((key.clone(), value.to_owned())); + keys.push(key); } - Ok(()) + self.servernameevent_data.insert_batch(batch.iter()); + keys } - pub(super) fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) -> Result<()> { - self.servername_educount - .insert(server_name.as_bytes(), &last_count.to_be_bytes()) - } + pub fn queued_requests<'a>(&'a self, destination: &Destination) -> impl Stream + Send + 'a { + let prefix = destination.get_prefix(); + self.servernameevent_data + .stream_raw_prefix(&prefix) + .ignore_err() + .map(|(key, val)| { + let (_, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); - pub fn get_latest_educount(&self, server_name: &ServerName) -> Result { - self.servername_educount - .get(server_name.as_bytes())? - .map_or(Ok(0), |bytes| { - utils::u64_from_bytes(&bytes).map_err(|_| Error::bad_database("Invalid u64 in servername_educount.")) + (key.to_vec(), event) }) } + + pub(super) fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) { + self.servername_educount + .insert(server_name.as_bytes(), &last_count.to_be_bytes()); + } + + pub async fn get_latest_educount(&self, server_name: &ServerName) -> u64 { + self.servername_educount + .qry(server_name) + .await + .deserialized() + .unwrap_or(0) + } } #[tracing::instrument(skip(key), level = "debug")] -fn parse_servercurrentevent(key: &[u8], value: Vec) -> Result<(Destination, SendingEvent)> { +fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, SendingEvent)> { // Appservices start with a plus Ok::<_, Error>(if key.starts_with(b"+") { let mut parts = key[1..].splitn(2, |&b| b == 0xFF); @@ -164,7 +178,7 @@ fn parse_servercurrentevent(key: &[u8], value: Vec) -> Result<(Destination, if value.is_empty() { SendingEvent::Pdu(event.to_vec()) } else { - SendingEvent::Edu(value) + SendingEvent::Edu(value.to_vec()) }, ) } else if key.starts_with(b"$") { @@ -192,7 +206,7 @@ fn parse_servercurrentevent(key: &[u8], value: Vec) -> Result<(Destination, SendingEvent::Pdu(event.to_vec()) } else { // I'm pretty sure this should never be called - SendingEvent::Edu(value) + SendingEvent::Edu(value.to_vec()) }, ) } else { @@ -214,7 +228,7 @@ fn parse_servercurrentevent(key: &[u8], value: Vec) -> Result<(Destination, if value.is_empty() { SendingEvent::Pdu(event.to_vec()) } else { - SendingEvent::Edu(value) + SendingEvent::Edu(value.to_vec()) }, ) }) diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index b90ea361..e3582f2e 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -7,10 +7,11 @@ mod sender; use std::{fmt::Debug, sync::Arc}; use async_trait::async_trait; -use conduit::{err, warn, Result, Server}; +use conduit::{err, utils::ReadyExt, warn, Result, Server}; +use futures::{future::ready, Stream, StreamExt, TryStreamExt}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, - OwnedServerName, RoomId, ServerName, UserId, + RoomId, ServerName, UserId, }; use tokio::sync::Mutex; @@ -104,7 +105,7 @@ impl Service { let dest = Destination::Push(user.to_owned(), pushkey); let event = SendingEvent::Pdu(pdu_id.to_owned()); let _cork = self.db.db.cork(); - let keys = self.db.queue_requests(&[(&dest, event.clone())])?; + let keys = self.db.queue_requests(&[(&event, &dest)]); self.dispatch(Msg { dest, event, @@ -117,7 +118,7 @@ impl Service { let dest = Destination::Appservice(appservice_id); let event = SendingEvent::Pdu(pdu_id); let _cork = self.db.db.cork(); - let keys = self.db.queue_requests(&[(&dest, event.clone())])?; + let keys = self.db.queue_requests(&[(&event, &dest)]); self.dispatch(Msg { dest, event, @@ -126,30 +127,31 @@ impl Service { } #[tracing::instrument(skip(self, room_id, pdu_id), level = "debug")] - pub fn send_pdu_room(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result<()> { + pub async fn send_pdu_room(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result<()> { let servers = self .services .state_cache .room_servers(room_id) - .filter_map(Result::ok) - .filter(|server_name| !self.services.globals.server_is_ours(server_name)); + .ready_filter(|server_name| !self.services.globals.server_is_ours(server_name)); - self.send_pdu_servers(servers, pdu_id) + self.send_pdu_servers(servers, pdu_id).await } #[tracing::instrument(skip(self, servers, pdu_id), level = "debug")] - pub fn send_pdu_servers>(&self, servers: I, pdu_id: &[u8]) -> Result<()> { - let requests = servers - .into_iter() - .map(|server| (Destination::Normal(server), SendingEvent::Pdu(pdu_id.to_owned()))) - .collect::>(); + pub async fn send_pdu_servers<'a, S>(&self, servers: S, pdu_id: &[u8]) -> Result<()> + where + S: Stream + Send + 'a, + { let _cork = self.db.db.cork(); - let keys = self.db.queue_requests( - &requests - .iter() - .map(|(o, e)| (o, e.clone())) - .collect::>(), - )?; + let requests = servers + .map(|server| (Destination::Normal(server.into()), SendingEvent::Pdu(pdu_id.into()))) + .collect::>() + .await; + + let keys = self + .db + .queue_requests(&requests.iter().map(|(o, e)| (e, o)).collect::>()); + for ((dest, event), queue_id) in requests.into_iter().zip(keys) { self.dispatch(Msg { dest, @@ -166,7 +168,7 @@ impl Service { let dest = Destination::Normal(server.to_owned()); let event = SendingEvent::Edu(serialized); let _cork = self.db.db.cork(); - let keys = self.db.queue_requests(&[(&dest, event.clone())])?; + let keys = self.db.queue_requests(&[(&event, &dest)]); self.dispatch(Msg { dest, event, @@ -175,30 +177,30 @@ impl Service { } #[tracing::instrument(skip(self, room_id, serialized), level = "debug")] - pub fn send_edu_room(&self, room_id: &RoomId, serialized: Vec) -> Result<()> { + pub async fn send_edu_room(&self, room_id: &RoomId, serialized: Vec) -> Result<()> { let servers = self .services .state_cache .room_servers(room_id) - .filter_map(Result::ok) - .filter(|server_name| !self.services.globals.server_is_ours(server_name)); + .ready_filter(|server_name| !self.services.globals.server_is_ours(server_name)); - self.send_edu_servers(servers, serialized) + self.send_edu_servers(servers, serialized).await } #[tracing::instrument(skip(self, servers, serialized), level = "debug")] - pub fn send_edu_servers>(&self, servers: I, serialized: Vec) -> Result<()> { - let requests = servers - .into_iter() - .map(|server| (Destination::Normal(server), SendingEvent::Edu(serialized.clone()))) - .collect::>(); + pub async fn send_edu_servers<'a, S>(&self, servers: S, serialized: Vec) -> Result<()> + where + S: Stream + Send + 'a, + { let _cork = self.db.db.cork(); - let keys = self.db.queue_requests( - &requests - .iter() - .map(|(o, e)| (o, e.clone())) - .collect::>(), - )?; + let requests = servers + .map(|server| (Destination::Normal(server.to_owned()), SendingEvent::Edu(serialized.clone()))) + .collect::>() + .await; + + let keys = self + .db + .queue_requests(&requests.iter().map(|(o, e)| (e, o)).collect::>()); for ((dest, event), queue_id) in requests.into_iter().zip(keys) { self.dispatch(Msg { @@ -212,29 +214,33 @@ impl Service { } #[tracing::instrument(skip(self, room_id), level = "debug")] - pub fn flush_room(&self, room_id: &RoomId) -> Result<()> { + pub async fn flush_room(&self, room_id: &RoomId) -> Result<()> { let servers = self .services .state_cache .room_servers(room_id) - .filter_map(Result::ok) - .filter(|server_name| !self.services.globals.server_is_ours(server_name)); + .ready_filter(|server_name| !self.services.globals.server_is_ours(server_name)); - self.flush_servers(servers) + self.flush_servers(servers).await } #[tracing::instrument(skip(self, servers), level = "debug")] - pub fn flush_servers>(&self, servers: I) -> Result<()> { - let requests = servers.into_iter().map(Destination::Normal); - for dest in requests { - self.dispatch(Msg { - dest, - event: SendingEvent::Flush, - queue_id: Vec::::new(), - })?; - } - - Ok(()) + pub async fn flush_servers<'a, S>(&self, servers: S) -> Result<()> + where + S: Stream + Send + 'a, + { + servers + .map(ToOwned::to_owned) + .map(Destination::Normal) + .map(Ok) + .try_for_each(|dest| { + ready(self.dispatch(Msg { + dest, + event: SendingEvent::Flush, + queue_id: Vec::::new(), + })) + }) + .await } #[tracing::instrument(skip_all, name = "request")] @@ -263,11 +269,10 @@ impl Service { /// Cleanup event data /// Used for instance after we remove an appservice registration #[tracing::instrument(skip(self), level = "debug")] - pub fn cleanup_events(&self, appservice_id: String) -> Result<()> { + pub async fn cleanup_events(&self, appservice_id: String) { self.db - .delete_all_requests_for(&Destination::Appservice(appservice_id))?; - - Ok(()) + .delete_all_requests_for(&Destination::Appservice(appservice_id)) + .await; } fn dispatch(&self, msg: Msg) -> Result<()> { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 206bf92b..4db9922a 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -7,18 +7,15 @@ use std::{ use base64::{engine::general_purpose, Engine as _}; use conduit::{ - debug, debug_warn, error, trace, - utils::{calculate_hash, math::continue_exponential_backoff_secs}, + debug, debug_warn, err, trace, + utils::{calculate_hash, math::continue_exponential_backoff_secs, ReadyExt}, warn, Error, Result, }; -use federation::transactions::send_transaction_message; -use futures_util::{future::BoxFuture, stream::FuturesUnordered, StreamExt}; +use futures::{future::BoxFuture, pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; use ruma::{ - api::federation::{ - self, - transactions::edu::{ - DeviceListUpdateContent, Edu, PresenceContent, PresenceUpdate, ReceiptContent, ReceiptData, ReceiptMap, - }, + api::federation::transactions::{ + edu::{DeviceListUpdateContent, Edu, PresenceContent, PresenceUpdate, ReceiptContent, ReceiptData, ReceiptMap}, + send_transaction_message, }, device_id, events::{push_rules::PushRulesEvent, receipt::ReceiptType, AnySyncEphemeralRoomEvent, GlobalAccountDataEventType}, @@ -28,7 +25,7 @@ use ruma::{ use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tokio::time::sleep_until; -use super::{appservice, Destination, Msg, SendingEvent, Service}; +use super::{appservice, data::QueueItem, Destination, Msg, SendingEvent, Service}; #[derive(Debug)] enum TransactionStatus { @@ -50,20 +47,20 @@ const CLEANUP_TIMEOUT_MS: u64 = 3500; impl Service { #[tracing::instrument(skip_all, name = "sender")] pub(super) async fn sender(&self) -> Result<()> { - let receiver = self.receiver.lock().await; - let mut futures: SendingFutures<'_> = FuturesUnordered::new(); let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); + let mut futures: SendingFutures<'_> = FuturesUnordered::new(); + let receiver = self.receiver.lock().await; - self.initial_requests(&futures, &mut statuses); + self.initial_requests(&mut futures, &mut statuses).await; loop { debug_assert!(!receiver.is_closed(), "channel error"); tokio::select! { request = receiver.recv_async() => match request { - Ok(request) => self.handle_request(request, &futures, &mut statuses), + Ok(request) => self.handle_request(request, &mut futures, &mut statuses).await, Err(_) => break, }, Some(response) = futures.next() => { - self.handle_response(response, &futures, &mut statuses); + self.handle_response(response, &mut futures, &mut statuses).await; }, } } @@ -72,18 +69,16 @@ impl Service { Ok(()) } - fn handle_response<'a>( - &'a self, response: SendingResult, futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus, + async fn handle_response<'a>( + &'a self, response: SendingResult, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { match response { - Ok(dest) => self.handle_response_ok(&dest, futures, statuses), - Err((dest, e)) => Self::handle_response_err(dest, futures, statuses, &e), + Ok(dest) => self.handle_response_ok(&dest, futures, statuses).await, + Err((dest, e)) => Self::handle_response_err(dest, statuses, &e), }; } - fn handle_response_err( - dest: Destination, _futures: &SendingFutures<'_>, statuses: &mut CurTransactionStatus, e: &Error, - ) { + fn handle_response_err(dest: Destination, statuses: &mut CurTransactionStatus, e: &Error) { debug!(dest = ?dest, "{e:?}"); statuses.entry(dest).and_modify(|e| { *e = match e { @@ -94,39 +89,40 @@ impl Service { }); } - fn handle_response_ok<'a>( - &'a self, dest: &Destination, futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus, + #[allow(clippy::needless_pass_by_ref_mut)] + async fn handle_response_ok<'a>( + &'a self, dest: &Destination, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { let _cork = self.db.db.cork(); - self.db - .delete_all_active_requests_for(dest) - .expect("all active requests deleted"); + self.db.delete_all_active_requests_for(dest).await; // Find events that have been added since starting the last request let new_events = self .db .queued_requests(dest) - .filter_map(Result::ok) .take(DEQUEUE_LIMIT) - .collect::>(); + .collect::>() + .await; // Insert any pdus we found if !new_events.is_empty() { - self.db - .mark_as_active(&new_events) - .expect("marked as active"); - let new_events_vec = new_events.into_iter().map(|(event, _)| event).collect(); - futures.push(Box::pin(self.send_events(dest.clone(), new_events_vec))); + self.db.mark_as_active(&new_events); + + let new_events_vec = new_events.into_iter().map(|(_, event)| event).collect(); + futures.push(self.send_events(dest.clone(), new_events_vec).boxed()); } else { statuses.remove(dest); } } - fn handle_request<'a>(&'a self, msg: Msg, futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus) { - let iv = vec![(msg.event, msg.queue_id)]; - if let Ok(Some(events)) = self.select_events(&msg.dest, iv, statuses) { + #[allow(clippy::needless_pass_by_ref_mut)] + async fn handle_request<'a>( + &'a self, msg: Msg, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, + ) { + let iv = vec![(msg.queue_id, msg.event)]; + if let Ok(Some(events)) = self.select_events(&msg.dest, iv, statuses).await { if !events.is_empty() { - futures.push(Box::pin(self.send_events(msg.dest, events))); + futures.push(self.send_events(msg.dest, events).boxed()); } else { statuses.remove(&msg.dest); } @@ -142,7 +138,7 @@ impl Service { tokio::select! { () = sleep_until(deadline.into()) => break, response = futures.next() => match response { - Some(response) => self.handle_response(response, futures, statuses), + Some(response) => self.handle_response(response, futures, statuses).await, None => return, } } @@ -151,16 +147,17 @@ impl Service { debug_warn!("Leaving with {} unfinished requests...", futures.len()); } - fn initial_requests<'a>(&'a self, futures: &SendingFutures<'a>, statuses: &mut CurTransactionStatus) { + #[allow(clippy::needless_pass_by_ref_mut)] + async fn initial_requests<'a>(&'a self, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus) { let keep = usize::try_from(self.server.config.startup_netburst_keep).unwrap_or(usize::MAX); let mut txns = HashMap::>::new(); - for (key, dest, event) in self.db.active_requests().filter_map(Result::ok) { + let mut active = self.db.active_requests().boxed(); + + while let Some((key, event, dest)) = active.next().await { let entry = txns.entry(dest.clone()).or_default(); if self.server.config.startup_netburst_keep >= 0 && entry.len() >= keep { - warn!("Dropping unsent event {:?} {:?}", dest, String::from_utf8_lossy(&key)); - self.db - .delete_active_request(&key) - .expect("active request deleted"); + warn!("Dropping unsent event {dest:?} {:?}", String::from_utf8_lossy(&key)); + self.db.delete_active_request(&key); } else { entry.push(event); } @@ -169,16 +166,16 @@ impl Service { for (dest, events) in txns { if self.server.config.startup_netburst && !events.is_empty() { statuses.insert(dest.clone(), TransactionStatus::Running); - futures.push(Box::pin(self.send_events(dest.clone(), events))); + futures.push(self.send_events(dest.clone(), events).boxed()); } } } #[tracing::instrument(skip_all, level = "debug")] - fn select_events( + async fn select_events( &self, dest: &Destination, - new_events: Vec<(SendingEvent, Vec)>, // Events we want to send: event and full key + new_events: Vec, // Events we want to send: event and full key statuses: &mut CurTransactionStatus, ) -> Result>> { let (allow, retry) = self.select_events_current(dest.clone(), statuses)?; @@ -195,8 +192,8 @@ impl Service { if retry { self.db .active_requests_for(dest) - .filter_map(Result::ok) - .for_each(|(_, e)| events.push(e)); + .ready_for_each(|(_, e)| events.push(e)) + .await; return Ok(Some(events)); } @@ -204,17 +201,17 @@ impl Service { // Compose the next transaction let _cork = self.db.db.cork(); if !new_events.is_empty() { - self.db.mark_as_active(&new_events)?; - for (e, _) in new_events { + self.db.mark_as_active(&new_events); + for (_, e) in new_events { events.push(e); } } // Add EDU's into the transaction if let Destination::Normal(server_name) = dest { - if let Ok((select_edus, last_count)) = self.select_edus(server_name) { + if let Ok((select_edus, last_count)) = self.select_edus(server_name).await { events.extend(select_edus.into_iter().map(SendingEvent::Edu)); - self.db.set_latest_educount(server_name, last_count)?; + self.db.set_latest_educount(server_name, last_count); } } @@ -248,26 +245,32 @@ impl Service { } #[tracing::instrument(skip_all, level = "debug")] - fn select_edus(&self, server_name: &ServerName) -> Result<(Vec>, u64)> { + async fn select_edus(&self, server_name: &ServerName) -> Result<(Vec>, u64)> { // u64: count of last edu - let since = self.db.get_latest_educount(server_name)?; + let since = self.db.get_latest_educount(server_name).await; let mut events = Vec::new(); let mut max_edu_count = since; let mut device_list_changes = HashSet::new(); - for room_id in self.services.state_cache.server_rooms(server_name) { - let room_id = room_id?; + let server_rooms = self.services.state_cache.server_rooms(server_name); + + pin_mut!(server_rooms); + while let Some(room_id) = server_rooms.next().await { // Look for device list updates in this room device_list_changes.extend( self.services .users - .keys_changed(room_id.as_ref(), since, None) - .filter_map(Result::ok) - .filter(|user_id| self.services.globals.user_is_local(user_id)), + .keys_changed(room_id.as_str(), since, None) + .ready_filter(|user_id| self.services.globals.user_is_local(user_id)) + .map(ToOwned::to_owned) + .collect::>() + .await, ); if self.server.config.allow_outgoing_read_receipts - && !self.select_edus_receipts(&room_id, since, &mut max_edu_count, &mut events)? + && !self + .select_edus_receipts(room_id, since, &mut max_edu_count, &mut events) + .await? { break; } @@ -290,19 +293,22 @@ impl Service { } if self.server.config.allow_outgoing_presence { - self.select_edus_presence(server_name, since, &mut max_edu_count, &mut events)?; + self.select_edus_presence(server_name, since, &mut max_edu_count, &mut events) + .await?; } Ok((events, max_edu_count)) } /// Look for presence - fn select_edus_presence( + async fn select_edus_presence( &self, server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, ) -> Result { - // Look for presence updates for this server + let presence_since = self.services.presence.presence_since(since); + + pin_mut!(presence_since); let mut presence_updates = Vec::new(); - for (user_id, count, presence_bytes) in self.services.presence.presence_since(since) { + while let Some((user_id, count, presence_bytes)) = presence_since.next().await { *max_edu_count = cmp::max(count, *max_edu_count); if !self.services.globals.user_is_local(&user_id) { @@ -312,7 +318,8 @@ impl Service { if !self .services .state_cache - .server_sees_user(server_name, &user_id)? + .server_sees_user(server_name, &user_id) + .await { continue; } @@ -320,7 +327,9 @@ impl Service { let presence_event = self .services .presence - .from_json_bytes_to_event(&presence_bytes, &user_id)?; + .from_json_bytes_to_event(&presence_bytes, &user_id) + .await?; + presence_updates.push(PresenceUpdate { user_id, presence: presence_event.content.presence, @@ -346,32 +355,33 @@ impl Service { } /// Look for read receipts in this room - fn select_edus_receipts( + async fn select_edus_receipts( &self, room_id: &RoomId, since: u64, max_edu_count: &mut u64, events: &mut Vec>, ) -> Result { - for r in self + let receipts = self .services .read_receipt - .readreceipts_since(room_id, since) - { - let (user_id, count, read_receipt) = r?; - *max_edu_count = cmp::max(count, *max_edu_count); + .readreceipts_since(room_id, since); + pin_mut!(receipts); + while let Some((user_id, count, read_receipt)) = receipts.next().await { + *max_edu_count = cmp::max(count, *max_edu_count); if !self.services.globals.user_is_local(&user_id) { continue; } let event = serde_json::from_str(read_receipt.json().get()) .map_err(|_| Error::bad_database("Invalid edu event in read_receipts."))?; + let federation_event = if let AnySyncEphemeralRoomEvent::Receipt(r) = event { let mut read = BTreeMap::new(); - let (event_id, mut receipt) = r .content .0 .into_iter() .next() .expect("we only use one event per read receipt"); + let receipt = receipt .remove(&ReceiptType::Read) .expect("our read receipts always set this") @@ -427,24 +437,17 @@ impl Service { async fn send_events_dest_appservice( &self, dest: &Destination, id: &str, events: Vec, ) -> SendingResult { - let mut pdu_jsons = Vec::new(); + let Some(appservice) = self.services.appservice.get_registration(id).await else { + return Err((dest.clone(), err!(Database(warn!(?id, "Missing appservice registration"))))); + }; + let mut pdu_jsons = Vec::new(); for event in &events { match event { SendingEvent::Pdu(pdu_id) => { - pdu_jsons.push( - self.services - .timeline - .get_pdu_from_id(pdu_id) - .map_err(|e| (dest.clone(), e))? - .ok_or_else(|| { - ( - dest.clone(), - Error::bad_database("[Appservice] Event in servernameevent_data not found in db."), - ) - })? - .to_room_event(), - ); + if let Ok(pdu) = self.services.timeline.get_pdu_from_id(pdu_id).await { + pdu_jsons.push(pdu.to_room_event()); + } }, SendingEvent::Edu(_) | SendingEvent::Flush => { // Appservices don't need EDUs (?) and flush only; @@ -453,32 +456,24 @@ impl Service { } } + let txn_id = &*general_purpose::URL_SAFE_NO_PAD.encode(calculate_hash( + &events + .iter() + .map(|e| match e { + SendingEvent::Edu(b) | SendingEvent::Pdu(b) => &**b, + SendingEvent::Flush => &[], + }) + .collect::>(), + )); + //debug_assert!(!pdu_jsons.is_empty(), "sending empty transaction"); let client = &self.services.client.appservice; match appservice::send_request( client, - self.services - .appservice - .get_registration(id) - .await - .ok_or_else(|| { - ( - dest.clone(), - Error::bad_database("[Appservice] Could not load registration from db."), - ) - })?, + appservice, ruma::api::appservice::event::push_events::v1::Request { events: pdu_jsons, - txn_id: (&*general_purpose::URL_SAFE_NO_PAD.encode(calculate_hash( - &events - .iter() - .map(|e| match e { - SendingEvent::Edu(b) | SendingEvent::Pdu(b) => &**b, - SendingEvent::Flush => &[], - }) - .collect::>(), - ))) - .into(), + txn_id: txn_id.into(), ephemeral: Vec::new(), to_device: Vec::new(), }, @@ -494,23 +489,17 @@ impl Service { async fn send_events_dest_push( &self, dest: &Destination, userid: &OwnedUserId, pushkey: &str, events: Vec, ) -> SendingResult { - let mut pdus = Vec::new(); + let Ok(pusher) = self.services.pusher.get_pusher(userid, pushkey).await else { + return Err((dest.clone(), err!(Database(error!(?userid, ?pushkey, "Missing pusher"))))); + }; + let mut pdus = Vec::new(); for event in &events { match event { SendingEvent::Pdu(pdu_id) => { - pdus.push( - self.services - .timeline - .get_pdu_from_id(pdu_id) - .map_err(|e| (dest.clone(), e))? - .ok_or_else(|| { - ( - dest.clone(), - Error::bad_database("[Push] Event in servernameevent_data not found in db."), - ) - })?, - ); + if let Ok(pdu) = self.services.timeline.get_pdu_from_id(pdu_id).await { + pdus.push(pdu); + } }, SendingEvent::Edu(_) | SendingEvent::Flush => { // Push gateways don't need EDUs (?) and flush only; @@ -529,28 +518,22 @@ impl Service { } } - let Some(pusher) = self - .services - .pusher - .get_pusher(userid, pushkey) - .map_err(|e| (dest.clone(), e))? - else { - continue; - }; - let rules_for_user = self .services .account_data .get(None, userid, GlobalAccountDataEventType::PushRules.to_string().into()) - .unwrap_or_default() - .and_then(|event| serde_json::from_str::(event.get()).ok()) - .map_or_else(|| push::Ruleset::server_default(userid), |ev: PushRulesEvent| ev.content.global); + .await + .and_then(|event| serde_json::from_str::(event.get()).map_err(Into::into)) + .map_or_else( + |_| push::Ruleset::server_default(userid), + |ev: PushRulesEvent| ev.content.global, + ); let unread: UInt = self .services .user .notification_count(userid, &pdu.room_id) - .map_err(|e| (dest.clone(), e))? + .await .try_into() .expect("notification count can't go that high"); @@ -559,7 +542,6 @@ impl Service { .pusher .send_push_notice(userid, unread, &pusher, rules_for_user, &pdu) .await - .map(|_response| dest.clone()) .map_err(|e| (dest.clone(), e)); } @@ -586,21 +568,11 @@ impl Service { for event in &events { match event { // TODO: check room version and remove event_id if needed - SendingEvent::Pdu(pdu_id) => pdu_jsons.push( - self.convert_to_outgoing_federation_event( - self.services - .timeline - .get_pdu_json_from_id(pdu_id) - .map_err(|e| (dest.clone(), e))? - .ok_or_else(|| { - error!(?dest, ?server, ?pdu_id, "event not found"); - ( - dest.clone(), - Error::bad_database("[Normal] Event in servernameevent_data not found in db."), - ) - })?, - ), - ), + SendingEvent::Pdu(pdu_id) => { + if let Ok(pdu) = self.services.timeline.get_pdu_json_from_id(pdu_id).await { + pdu_jsons.push(self.convert_to_outgoing_federation_event(pdu).await); + } + }, SendingEvent::Edu(edu) => { if let Ok(raw) = serde_json::from_slice(edu) { edu_jsons.push(raw); @@ -647,7 +619,7 @@ impl Service { } /// This does not return a full `Pdu` it is only to satisfy ruma's types. - pub fn convert_to_outgoing_federation_event(&self, mut pdu_json: CanonicalJsonObject) -> Box { + pub async fn convert_to_outgoing_federation_event(&self, mut pdu_json: CanonicalJsonObject) -> Box { if let Some(unsigned) = pdu_json .get_mut("unsigned") .and_then(|val| val.as_object_mut()) @@ -660,7 +632,7 @@ impl Service { .get("room_id") .and_then(|val| RoomId::parse(val.as_str()?).ok()) { - match self.services.state.get_room_version(&room_id) { + match self.services.state.get_room_version(&room_id).await { Ok(room_version_id) => match room_version_id { RoomVersionId::V1 | RoomVersionId::V2 => {}, _ => _ = pdu_json.remove("event_id"), diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index a565e500..ae2b8c3c 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -5,7 +5,7 @@ use std::{ }; use conduit::{debug, debug_error, debug_warn, err, error, info, trace, warn, Err, Result}; -use futures_util::{stream::FuturesUnordered, StreamExt}; +use futures::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::federation::{ discovery::{ @@ -179,7 +179,8 @@ impl Service { let result: BTreeMap<_, _> = self .services .globals - .verify_keys_for(origin)? + .verify_keys_for(origin) + .await? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) .collect(); @@ -236,7 +237,8 @@ impl Service { .services .globals .db - .add_signing_key(&k.server_name, k.clone())? + .add_signing_key(&k.server_name, k.clone()) + .await .into_iter() .map(|(k, v)| (k.to_string(), v.key)) .collect::>(); @@ -283,7 +285,8 @@ impl Service { .services .globals .db - .add_signing_key(&origin, key)? + .add_signing_key(&origin, key) + .await .into_iter() .map(|(k, v)| (k.to_string(), v.key)) .collect(); @@ -384,7 +387,8 @@ impl Service { let mut result: BTreeMap<_, _> = self .services .globals - .verify_keys_for(origin)? + .verify_keys_for(origin) + .await? .into_iter() .map(|(k, v)| (k.to_string(), v.key)) .collect(); @@ -431,7 +435,8 @@ impl Service { self.services .globals .db - .add_signing_key(origin, k.clone())?; + .add_signing_key(origin, k.clone()) + .await; result.extend( k.verify_keys .into_iter() @@ -462,7 +467,8 @@ impl Service { self.services .globals .db - .add_signing_key(origin, server_key.clone())?; + .add_signing_key(origin, server_key.clone()) + .await; result.extend( server_key @@ -495,7 +501,8 @@ impl Service { self.services .globals .db - .add_signing_key(origin, server_key.clone())?; + .add_signing_key(origin, server_key.clone()) + .await; result.extend( server_key @@ -545,7 +552,8 @@ impl Service { self.services .globals .db - .add_signing_key(origin, k.clone())?; + .add_signing_key(origin, k.clone()) + .await; result.extend( k.verify_keys .into_iter() diff --git a/src/service/services.rs b/src/service/services.rs index 3aa095b8..da22fb2d 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -14,7 +14,7 @@ use crate::{ manager::Manager, media, presence, pusher, resolver, rooms, sending, server_keys, service, service::{Args, Map, Service}, - transaction_ids, uiaa, updates, users, + sync, transaction_ids, uiaa, updates, users, }; pub struct Services { @@ -32,6 +32,7 @@ pub struct Services { pub rooms: rooms::Service, pub sending: Arc, pub server_keys: Arc, + pub sync: Arc, pub transaction_ids: Arc, pub uiaa: Arc, pub updates: Arc, @@ -96,6 +97,7 @@ impl Services { }, sending: build!(sending::Service), server_keys: build!(server_keys::Service), + sync: build!(sync::Service), transaction_ids: build!(transaction_ids::Service), uiaa: build!(uiaa::Service), updates: build!(updates::Service), diff --git a/src/service/sync/mod.rs b/src/service/sync/mod.rs new file mode 100644 index 00000000..1bf4610f --- /dev/null +++ b/src/service/sync/mod.rs @@ -0,0 +1,233 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{Arc, Mutex, Mutex as StdMutex}, +}; + +use conduit::Result; +use ruma::{ + api::client::sync::sync_events::{ + self, + v4::{ExtensionsConfig, SyncRequestList}, + }, + OwnedDeviceId, OwnedRoomId, OwnedUserId, +}; + +pub struct Service { + connections: DbConnections, +} + +struct SlidingSyncCache { + lists: BTreeMap, + subscriptions: BTreeMap, + known_rooms: BTreeMap>, // For every room, the roomsince number + extensions: ExtensionsConfig, +} + +type DbConnections = Mutex>; +type DbConnectionsKey = (OwnedUserId, OwnedDeviceId, String); +type DbConnectionsVal = Arc>; + +impl crate::Service for Service { + fn build(_args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + connections: StdMutex::new(BTreeMap::new()), + })) + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { + pub fn remembered(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) -> bool { + self.connections + .lock() + .unwrap() + .contains_key(&(user_id, device_id, conn_id)) + } + + pub fn forget_sync_request_connection(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) { + self.connections + .lock() + .expect("locked") + .remove(&(user_id, device_id, conn_id)); + } + + pub fn update_sync_request_with_cache( + &self, user_id: OwnedUserId, device_id: OwnedDeviceId, request: &mut sync_events::v4::Request, + ) -> BTreeMap> { + let Some(conn_id) = request.conn_id.clone() else { + return BTreeMap::new(); + }; + + let mut cache = self.connections.lock().expect("locked"); + let cached = Arc::clone( + cache + .entry((user_id, device_id, conn_id)) + .or_insert_with(|| { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + }), + ); + let cached = &mut cached.lock().expect("locked"); + drop(cache); + + for (list_id, list) in &mut request.lists { + if let Some(cached_list) = cached.lists.get(list_id) { + if list.sort.is_empty() { + list.sort.clone_from(&cached_list.sort); + }; + if list.room_details.required_state.is_empty() { + list.room_details + .required_state + .clone_from(&cached_list.room_details.required_state); + }; + list.room_details.timeline_limit = list + .room_details + .timeline_limit + .or(cached_list.room_details.timeline_limit); + list.include_old_rooms = list + .include_old_rooms + .clone() + .or_else(|| cached_list.include_old_rooms.clone()); + match (&mut list.filters, cached_list.filters.clone()) { + (Some(list_filters), Some(cached_filters)) => { + list_filters.is_dm = list_filters.is_dm.or(cached_filters.is_dm); + if list_filters.spaces.is_empty() { + list_filters.spaces = cached_filters.spaces; + } + list_filters.is_encrypted = list_filters.is_encrypted.or(cached_filters.is_encrypted); + list_filters.is_invite = list_filters.is_invite.or(cached_filters.is_invite); + if list_filters.room_types.is_empty() { + list_filters.room_types = cached_filters.room_types; + } + if list_filters.not_room_types.is_empty() { + list_filters.not_room_types = cached_filters.not_room_types; + } + list_filters.room_name_like = list_filters + .room_name_like + .clone() + .or(cached_filters.room_name_like); + if list_filters.tags.is_empty() { + list_filters.tags = cached_filters.tags; + } + if list_filters.not_tags.is_empty() { + list_filters.not_tags = cached_filters.not_tags; + } + }, + (_, Some(cached_filters)) => list.filters = Some(cached_filters), + (Some(list_filters), _) => list.filters = Some(list_filters.clone()), + (..) => {}, + } + if list.bump_event_types.is_empty() { + list.bump_event_types + .clone_from(&cached_list.bump_event_types); + }; + } + cached.lists.insert(list_id.clone(), list.clone()); + } + + cached + .subscriptions + .extend(request.room_subscriptions.clone()); + request + .room_subscriptions + .extend(cached.subscriptions.clone()); + + request.extensions.e2ee.enabled = request + .extensions + .e2ee + .enabled + .or(cached.extensions.e2ee.enabled); + + request.extensions.to_device.enabled = request + .extensions + .to_device + .enabled + .or(cached.extensions.to_device.enabled); + + request.extensions.account_data.enabled = request + .extensions + .account_data + .enabled + .or(cached.extensions.account_data.enabled); + request.extensions.account_data.lists = request + .extensions + .account_data + .lists + .clone() + .or_else(|| cached.extensions.account_data.lists.clone()); + request.extensions.account_data.rooms = request + .extensions + .account_data + .rooms + .clone() + .or_else(|| cached.extensions.account_data.rooms.clone()); + + cached.extensions = request.extensions.clone(); + + cached.known_rooms.clone() + } + + pub fn update_sync_subscriptions( + &self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String, + subscriptions: BTreeMap, + ) { + let mut cache = self.connections.lock().expect("locked"); + let cached = Arc::clone( + cache + .entry((user_id, device_id, conn_id)) + .or_insert_with(|| { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + }), + ); + let cached = &mut cached.lock().expect("locked"); + drop(cache); + + cached.subscriptions = subscriptions; + } + + pub fn update_sync_known_rooms( + &self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String, list_id: String, + new_cached_rooms: BTreeSet, globalsince: u64, + ) { + let mut cache = self.connections.lock().expect("locked"); + let cached = Arc::clone( + cache + .entry((user_id, device_id, conn_id)) + .or_insert_with(|| { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + }), + ); + let cached = &mut cached.lock().expect("locked"); + drop(cache); + + for (roomid, lastsince) in cached + .known_rooms + .entry(list_id.clone()) + .or_default() + .iter_mut() + { + if !new_cached_rooms.contains(roomid) { + *lastsince = 0; + } + } + let list = cached.known_rooms.entry(list_id).or_default(); + for roomid in new_cached_rooms { + list.insert(roomid, globalsince); + } + } +} diff --git a/src/service/transaction_ids/data.rs b/src/service/transaction_ids/data.rs deleted file mode 100644 index 791b46f0..00000000 --- a/src/service/transaction_ids/data.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::sync::Arc; - -use conduit::Result; -use database::{Database, Map}; -use ruma::{DeviceId, TransactionId, UserId}; - -pub struct Data { - userdevicetxnid_response: Arc, -} - -impl Data { - pub(super) fn new(db: &Arc) -> Self { - Self { - userdevicetxnid_response: db["userdevicetxnid_response"].clone(), - } - } - - pub(super) fn add_txnid( - &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, data: &[u8], - ) -> Result<()> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(device_id.map(DeviceId::as_bytes).unwrap_or_default()); - key.push(0xFF); - key.extend_from_slice(txn_id.as_bytes()); - - self.userdevicetxnid_response.insert(&key, data)?; - - Ok(()) - } - - pub(super) fn existing_txnid( - &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, - ) -> Result>> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(device_id.map(DeviceId::as_bytes).unwrap_or_default()); - key.push(0xFF); - key.extend_from_slice(txn_id.as_bytes()); - - // If there's no entry, this is a new transaction - self.userdevicetxnid_response.get(&key) - } -} diff --git a/src/service/transaction_ids/mod.rs b/src/service/transaction_ids/mod.rs index 78e6337f..72f60adb 100644 --- a/src/service/transaction_ids/mod.rs +++ b/src/service/transaction_ids/mod.rs @@ -1,35 +1,45 @@ -mod data; - use std::sync::Arc; -use conduit::Result; -use data::Data; +use conduit::{implement, Result}; +use database::{Handle, Map}; use ruma::{DeviceId, TransactionId, UserId}; pub struct Service { - pub db: Data, + db: Data, +} + +struct Data { + userdevicetxnid_response: Arc, } impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + db: Data { + userdevicetxnid_response: args.db["userdevicetxnid_response"].clone(), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - pub fn add_txnid( - &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, data: &[u8], - ) -> Result<()> { - self.db.add_txnid(user_id, device_id, txn_id, data) - } +#[implement(Service)] +pub fn add_txnid(&self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, data: &[u8]) { + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(device_id.map(DeviceId::as_bytes).unwrap_or_default()); + key.push(0xFF); + key.extend_from_slice(txn_id.as_bytes()); - pub fn existing_txnid( - &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, - ) -> Result>> { - self.db.existing_txnid(user_id, device_id, txn_id) - } + self.db.userdevicetxnid_response.insert(&key, data); +} + +// If there's no entry, this is a new transaction +#[implement(Service)] +pub async fn existing_txnid( + &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, +) -> Result> { + let key = (user_id, device_id, txn_id); + self.db.userdevicetxnid_response.qry(&key).await } diff --git a/src/service/uiaa/data.rs b/src/service/uiaa/data.rs deleted file mode 100644 index ce071da0..00000000 --- a/src/service/uiaa/data.rs +++ /dev/null @@ -1,87 +0,0 @@ -use std::{ - collections::BTreeMap, - sync::{Arc, RwLock}, -}; - -use conduit::{Error, Result}; -use database::{Database, Map}; -use ruma::{ - api::client::{error::ErrorKind, uiaa::UiaaInfo}, - CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedUserId, UserId, -}; - -pub struct Data { - userdevicesessionid_uiaarequest: RwLock>, - userdevicesessionid_uiaainfo: Arc, -} - -impl Data { - pub(super) fn new(db: &Arc) -> Self { - Self { - userdevicesessionid_uiaarequest: RwLock::new(BTreeMap::new()), - userdevicesessionid_uiaainfo: db["userdevicesessionid_uiaainfo"].clone(), - } - } - - pub(super) fn set_uiaa_request( - &self, user_id: &UserId, device_id: &DeviceId, session: &str, request: &CanonicalJsonValue, - ) -> Result<()> { - self.userdevicesessionid_uiaarequest - .write() - .unwrap() - .insert( - (user_id.to_owned(), device_id.to_owned(), session.to_owned()), - request.to_owned(), - ); - - Ok(()) - } - - pub(super) fn get_uiaa_request( - &self, user_id: &UserId, device_id: &DeviceId, session: &str, - ) -> Option { - self.userdevicesessionid_uiaarequest - .read() - .unwrap() - .get(&(user_id.to_owned(), device_id.to_owned(), session.to_owned())) - .map(ToOwned::to_owned) - } - - pub(super) fn update_uiaa_session( - &self, user_id: &UserId, device_id: &DeviceId, session: &str, uiaainfo: Option<&UiaaInfo>, - ) -> Result<()> { - let mut userdevicesessionid = user_id.as_bytes().to_vec(); - userdevicesessionid.push(0xFF); - userdevicesessionid.extend_from_slice(device_id.as_bytes()); - userdevicesessionid.push(0xFF); - userdevicesessionid.extend_from_slice(session.as_bytes()); - - if let Some(uiaainfo) = uiaainfo { - self.userdevicesessionid_uiaainfo.insert( - &userdevicesessionid, - &serde_json::to_vec(&uiaainfo).expect("UiaaInfo::to_vec always works"), - )?; - } else { - self.userdevicesessionid_uiaainfo - .remove(&userdevicesessionid)?; - } - - Ok(()) - } - - pub(super) fn get_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: &str) -> Result { - let mut userdevicesessionid = user_id.as_bytes().to_vec(); - userdevicesessionid.push(0xFF); - userdevicesessionid.extend_from_slice(device_id.as_bytes()); - userdevicesessionid.push(0xFF); - userdevicesessionid.extend_from_slice(session.as_bytes()); - - serde_json::from_slice( - &self - .userdevicesessionid_uiaainfo - .get(&userdevicesessionid)? - .ok_or(Error::BadRequest(ErrorKind::forbidden(), "UIAA session does not exist."))?, - ) - .map_err(|_| Error::bad_database("UiaaInfo in userdeviceid_uiaainfo is invalid.")) - } -} diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 6041bbd3..7e231514 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -1,174 +1,243 @@ -mod data; +use std::{ + collections::BTreeMap, + sync::{Arc, RwLock}, +}; -use std::sync::Arc; - -use conduit::{error, utils, utils::hash, Error, Result, Server}; -use data::Data; +use conduit::{ + err, error, implement, utils, + utils::{hash, string::EMPTY}, + Error, Result, Server, +}; +use database::{Deserialized, Map}; use ruma::{ api::client::{ error::ErrorKind, uiaa::{AuthData, AuthType, Password, UiaaInfo, UserIdentifier}, }, - CanonicalJsonValue, DeviceId, UserId, + CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedUserId, UserId, }; use crate::{globals, users, Dep}; -pub const SESSION_ID_LENGTH: usize = 32; - pub struct Service { - server: Arc, + userdevicesessionid_uiaarequest: RwLock, + db: Data, services: Services, - pub db: Data, } struct Services { + server: Arc, globals: Dep, users: Dep, } +struct Data { + userdevicesessionid_uiaainfo: Arc, +} + +type RequestMap = BTreeMap; +type RequestKey = (OwnedUserId, OwnedDeviceId, String); + +pub const SESSION_ID_LENGTH: usize = 32; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - server: args.server.clone(), + userdevicesessionid_uiaarequest: RwLock::new(RequestMap::new()), + db: Data { + userdevicesessionid_uiaainfo: args.db["userdevicesessionid_uiaainfo"].clone(), + }, services: Services { + server: args.server.clone(), globals: args.depend::("globals"), users: args.depend::("users"), }, - db: Data::new(args.db), })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - /// Creates a new Uiaa session. Make sure the session token is unique. - pub fn create( - &self, user_id: &UserId, device_id: &DeviceId, uiaainfo: &UiaaInfo, json_body: &CanonicalJsonValue, - ) -> Result<()> { - self.db.set_uiaa_request( - user_id, - device_id, - uiaainfo.session.as_ref().expect("session should be set"), /* TODO: better session error handling (why - * is it optional in ruma?) */ - json_body, - )?; - self.db.update_uiaa_session( - user_id, - device_id, - uiaainfo.session.as_ref().expect("session should be set"), - Some(uiaainfo), - ) +/// Creates a new Uiaa session. Make sure the session token is unique. +#[implement(Service)] +pub fn create(&self, user_id: &UserId, device_id: &DeviceId, uiaainfo: &UiaaInfo, json_body: &CanonicalJsonValue) { + // TODO: better session error handling (why is uiaainfo.session optional in + // ruma?) + self.set_uiaa_request( + user_id, + device_id, + uiaainfo.session.as_ref().expect("session should be set"), + json_body, + ); + + self.update_uiaa_session( + user_id, + device_id, + uiaainfo.session.as_ref().expect("session should be set"), + Some(uiaainfo), + ); +} + +#[implement(Service)] +pub async fn try_auth( + &self, user_id: &UserId, device_id: &DeviceId, auth: &AuthData, uiaainfo: &UiaaInfo, +) -> Result<(bool, UiaaInfo)> { + let mut uiaainfo = if let Some(session) = auth.session() { + self.get_uiaa_session(user_id, device_id, session).await? + } else { + uiaainfo.clone() + }; + + if uiaainfo.session.is_none() { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); } - pub fn try_auth( - &self, user_id: &UserId, device_id: &DeviceId, auth: &AuthData, uiaainfo: &UiaaInfo, - ) -> Result<(bool, UiaaInfo)> { - let mut uiaainfo = auth.session().map_or_else( - || Ok(uiaainfo.clone()), - |session| self.db.get_uiaa_session(user_id, device_id, session), - )?; + match auth { + // Find out what the user completed + AuthData::Password(Password { + identifier, + password, + #[cfg(feature = "element_hacks")] + user, + .. + }) => { + #[cfg(feature = "element_hacks")] + let username = if let Some(UserIdentifier::UserIdOrLocalpart(username)) = identifier { + username + } else if let Some(username) = user { + username + } else { + return Err(Error::BadRequest(ErrorKind::Unrecognized, "Identifier type not recognized.")); + }; - if uiaainfo.session.is_none() { - uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - } + #[cfg(not(feature = "element_hacks"))] + let Some(UserIdentifier::UserIdOrLocalpart(username)) = identifier + else { + return Err(Error::BadRequest(ErrorKind::Unrecognized, "Identifier type not recognized.")); + }; - match auth { - // Find out what the user completed - AuthData::Password(Password { - identifier, - password, - #[cfg(feature = "element_hacks")] - user, - .. - }) => { - #[cfg(feature = "element_hacks")] - let username = if let Some(UserIdentifier::UserIdOrLocalpart(username)) = identifier { - username - } else if let Some(username) = user { - username - } else { - return Err(Error::BadRequest(ErrorKind::Unrecognized, "Identifier type not recognized.")); - }; + let user_id = UserId::parse_with_server_name(username.clone(), self.services.globals.server_name()) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid."))?; - #[cfg(not(feature = "element_hacks"))] - let Some(UserIdentifier::UserIdOrLocalpart(username)) = identifier - else { - return Err(Error::BadRequest(ErrorKind::Unrecognized, "Identifier type not recognized.")); - }; - - let user_id = UserId::parse_with_server_name(username.clone(), self.services.globals.server_name()) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid."))?; - - // Check if password is correct - if let Some(hash) = self.services.users.password_hash(&user_id)? { - let hash_matches = hash::verify_password(password, &hash).is_ok(); - if !hash_matches { - uiaainfo.auth_error = Some(ruma::api::client::error::StandardErrorBody { - kind: ErrorKind::forbidden(), - message: "Invalid username or password.".to_owned(), - }); - return Ok((false, uiaainfo)); - } - } - - // Password was correct! Let's add it to `completed` - uiaainfo.completed.push(AuthType::Password); - }, - AuthData::RegistrationToken(t) => { - if Some(t.token.trim()) == self.server.config.registration_token.as_deref() { - uiaainfo.completed.push(AuthType::RegistrationToken); - } else { + // Check if password is correct + if let Ok(hash) = self.services.users.password_hash(&user_id).await { + let hash_matches = hash::verify_password(password, &hash).is_ok(); + if !hash_matches { uiaainfo.auth_error = Some(ruma::api::client::error::StandardErrorBody { kind: ErrorKind::forbidden(), - message: "Invalid registration token.".to_owned(), + message: "Invalid username or password.".to_owned(), }); return Ok((false, uiaainfo)); } - }, - AuthData::Dummy(_) => { - uiaainfo.completed.push(AuthType::Dummy); - }, - k => error!("type not supported: {:?}", k), - } - - // Check if a flow now succeeds - let mut completed = false; - 'flows: for flow in &mut uiaainfo.flows { - for stage in &flow.stages { - if !uiaainfo.completed.contains(stage) { - continue 'flows; - } } - // We didn't break, so this flow succeeded! - completed = true; - } - if !completed { - self.db.update_uiaa_session( - user_id, - device_id, - uiaainfo.session.as_ref().expect("session is always set"), - Some(&uiaainfo), - )?; - return Ok((false, uiaainfo)); - } + // Password was correct! Let's add it to `completed` + uiaainfo.completed.push(AuthType::Password); + }, + AuthData::RegistrationToken(t) => { + if Some(t.token.trim()) == self.services.server.config.registration_token.as_deref() { + uiaainfo.completed.push(AuthType::RegistrationToken); + } else { + uiaainfo.auth_error = Some(ruma::api::client::error::StandardErrorBody { + kind: ErrorKind::forbidden(), + message: "Invalid registration token.".to_owned(), + }); + return Ok((false, uiaainfo)); + } + }, + AuthData::Dummy(_) => { + uiaainfo.completed.push(AuthType::Dummy); + }, + k => error!("type not supported: {:?}", k), + } - // UIAA was successful! Remove this session and return true - self.db.update_uiaa_session( + // Check if a flow now succeeds + let mut completed = false; + 'flows: for flow in &mut uiaainfo.flows { + for stage in &flow.stages { + if !uiaainfo.completed.contains(stage) { + continue 'flows; + } + } + // We didn't break, so this flow succeeded! + completed = true; + } + + if !completed { + self.update_uiaa_session( user_id, device_id, uiaainfo.session.as_ref().expect("session is always set"), - None, - )?; - Ok((true, uiaainfo)) + Some(&uiaainfo), + ); + + return Ok((false, uiaainfo)); } - #[must_use] - pub fn get_uiaa_request( - &self, user_id: &UserId, device_id: &DeviceId, session: &str, - ) -> Option { - self.db.get_uiaa_request(user_id, device_id, session) + // UIAA was successful! Remove this session and return true + self.update_uiaa_session( + user_id, + device_id, + uiaainfo.session.as_ref().expect("session is always set"), + None, + ); + + Ok((true, uiaainfo)) +} + +#[implement(Service)] +fn set_uiaa_request(&self, user_id: &UserId, device_id: &DeviceId, session: &str, request: &CanonicalJsonValue) { + let key = (user_id.to_owned(), device_id.to_owned(), session.to_owned()); + self.userdevicesessionid_uiaarequest + .write() + .expect("locked for writing") + .insert(key, request.to_owned()); +} + +#[implement(Service)] +pub fn get_uiaa_request( + &self, user_id: &UserId, device_id: Option<&DeviceId>, session: &str, +) -> Option { + let key = ( + user_id.to_owned(), + device_id.unwrap_or_else(|| EMPTY.into()).to_owned(), + session.to_owned(), + ); + + self.userdevicesessionid_uiaarequest + .read() + .expect("locked for reading") + .get(&key) + .cloned() +} + +#[implement(Service)] +fn update_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: &str, uiaainfo: Option<&UiaaInfo>) { + let mut userdevicesessionid = user_id.as_bytes().to_vec(); + userdevicesessionid.push(0xFF); + userdevicesessionid.extend_from_slice(device_id.as_bytes()); + userdevicesessionid.push(0xFF); + userdevicesessionid.extend_from_slice(session.as_bytes()); + + if let Some(uiaainfo) = uiaainfo { + self.db.userdevicesessionid_uiaainfo.insert( + &userdevicesessionid, + &serde_json::to_vec(&uiaainfo).expect("UiaaInfo::to_vec always works"), + ); + } else { + self.db + .userdevicesessionid_uiaainfo + .remove(&userdevicesessionid); } } + +#[implement(Service)] +async fn get_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: &str) -> Result { + let key = (user_id, device_id, session); + self.db + .userdevicesessionid_uiaainfo + .qry(&key) + .await + .deserialized_json() + .map_err(|_| err!(Request(Forbidden("UIAA session does not exist.")))) +} diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index 3c69b243..4e16e22b 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -1,19 +1,22 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{debug, err, info, utils, warn, Error, Result}; -use database::Map; +use conduit::{debug, info, warn, Result}; +use database::{Deserialized, Map}; use ruma::events::room::message::RoomMessageEventContent; use serde::Deserialize; -use tokio::{sync::Notify, time::interval}; +use tokio::{ + sync::Notify, + time::{interval, MissedTickBehavior}, +}; use crate::{admin, client, globals, Dep}; pub struct Service { - services: Services, - db: Arc, - interrupt: Notify, interval: Duration, + interrupt: Notify, + db: Arc, + services: Services, } struct Services { @@ -22,12 +25,12 @@ struct Services { globals: Dep, } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] struct CheckForUpdatesResponse { updates: Vec, } -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] struct CheckForUpdatesResponseEntry { id: u64, date: String, @@ -42,33 +45,38 @@ const LAST_CHECK_FOR_UPDATES_COUNT: &[u8] = b"u"; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + interval: Duration::from_secs(CHECK_FOR_UPDATES_INTERVAL), + interrupt: Notify::new(), + db: args.db["global"].clone(), services: Services { globals: args.depend::("globals"), admin: args.depend::("admin"), client: args.depend::("client"), }, - db: args.db["global"].clone(), - interrupt: Notify::new(), - interval: Duration::from_secs(CHECK_FOR_UPDATES_INTERVAL), })) } + #[tracing::instrument(skip_all, name = "updates", level = "trace")] async fn worker(self: Arc) -> Result<()> { if !self.services.globals.allow_check_for_updates() { debug!("Disabling update check"); return Ok(()); } + let mut i = interval(self.interval); + i.set_missed_tick_behavior(MissedTickBehavior::Delay); loop { tokio::select! { - () = self.interrupt.notified() => return Ok(()), + () = self.interrupt.notified() => break, _ = i.tick() => (), } - if let Err(e) = self.handle_updates().await { + if let Err(e) = self.check().await { warn!(%e, "Failed to check for updates"); } } + + Ok(()) } fn interrupt(&self) { self.interrupt.notify_waiters(); } @@ -77,52 +85,52 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip_all)] - async fn handle_updates(&self) -> Result<()> { + #[tracing::instrument(skip_all, level = "trace")] + async fn check(&self) -> Result<()> { let response = self .services .client .default .get(CHECK_FOR_UPDATES_URL) .send() + .await? + .text() .await?; - let response = serde_json::from_str::(&response.text().await?) - .map_err(|e| err!("Bad check for updates response: {e}"))?; - - let mut last_update_id = self.last_check_for_updates_id()?; - for update in response.updates { - last_update_id = last_update_id.max(update.id); - if update.id > self.last_check_for_updates_id()? { - info!("{:#}", update.message); - self.services - .admin - .send_message(RoomMessageEventContent::text_markdown(format!( - "### the following is a message from the conduwuit puppy\n\nit was sent on `{}`:\n\n@room: {}", - update.date, update.message - ))) - .await; + let response = serde_json::from_str::(&response)?; + for update in &response.updates { + if update.id > self.last_check_for_updates_id().await { + self.handle(update).await; + self.update_check_for_updates_id(update.id); } } - self.update_check_for_updates_id(last_update_id)?; Ok(()) } + async fn handle(&self, update: &CheckForUpdatesResponseEntry) { + info!("{} {:#}", update.date, update.message); + self.services + .admin + .send_message(RoomMessageEventContent::text_markdown(format!( + "### the following is a message from the conduwuit puppy\n\nit was sent on `{}`:\n\n@room: {}", + update.date, update.message + ))) + .await + .ok(); + } + #[inline] - pub fn update_check_for_updates_id(&self, id: u64) -> Result<()> { + pub fn update_check_for_updates_id(&self, id: u64) { self.db - .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes())?; - - Ok(()) + .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes()); } - pub fn last_check_for_updates_id(&self) -> Result { + pub async fn last_check_for_updates_id(&self) -> u64 { self.db - .get(LAST_CHECK_FOR_UPDATES_COUNT)? - .map_or(Ok(0_u64), |bytes| { - utils::u64_from_bytes(&bytes) - .map_err(|_| Error::bad_database("last check for updates count has invalid bytes.")) - }) + .qry(LAST_CHECK_FOR_UPDATES_COUNT) + .await + .deserialized() + .unwrap_or(0_u64) } } diff --git a/src/service/users/data.rs b/src/service/users/data.rs deleted file mode 100644 index 70ff12e3..00000000 --- a/src/service/users/data.rs +++ /dev/null @@ -1,1098 +0,0 @@ -use std::{collections::BTreeMap, mem::size_of, sync::Arc}; - -use conduit::{debug_info, err, utils, warn, Err, Error, Result, Server}; -use database::Map; -use ruma::{ - api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, - encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, - events::{AnyToDeviceEvent, StateEventType}, - serde::Raw, - uint, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedDeviceKeyId, - OwnedMxcUri, OwnedUserId, UInt, UserId, -}; - -use crate::{globals, rooms, users::clean_signatures, Dep}; - -pub struct Data { - keychangeid_userid: Arc, - keyid_key: Arc, - onetimekeyid_onetimekeys: Arc, - openidtoken_expiresatuserid: Arc, - todeviceid_events: Arc, - token_userdeviceid: Arc, - userdeviceid_metadata: Arc, - userdeviceid_token: Arc, - userfilterid_filter: Arc, - userid_avatarurl: Arc, - userid_blurhash: Arc, - userid_devicelistversion: Arc, - userid_displayname: Arc, - userid_lastonetimekeyupdate: Arc, - userid_masterkeyid: Arc, - userid_password: Arc, - userid_selfsigningkeyid: Arc, - userid_usersigningkeyid: Arc, - useridprofilekey_value: Arc, - services: Services, -} - -struct Services { - server: Arc, - globals: Dep, - state_cache: Dep, - state_accessor: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - keychangeid_userid: db["keychangeid_userid"].clone(), - keyid_key: db["keyid_key"].clone(), - onetimekeyid_onetimekeys: db["onetimekeyid_onetimekeys"].clone(), - openidtoken_expiresatuserid: db["openidtoken_expiresatuserid"].clone(), - todeviceid_events: db["todeviceid_events"].clone(), - token_userdeviceid: db["token_userdeviceid"].clone(), - userdeviceid_metadata: db["userdeviceid_metadata"].clone(), - userdeviceid_token: db["userdeviceid_token"].clone(), - userfilterid_filter: db["userfilterid_filter"].clone(), - userid_avatarurl: db["userid_avatarurl"].clone(), - userid_blurhash: db["userid_blurhash"].clone(), - userid_devicelistversion: db["userid_devicelistversion"].clone(), - userid_displayname: db["userid_displayname"].clone(), - userid_lastonetimekeyupdate: db["userid_lastonetimekeyupdate"].clone(), - userid_masterkeyid: db["userid_masterkeyid"].clone(), - userid_password: db["userid_password"].clone(), - userid_selfsigningkeyid: db["userid_selfsigningkeyid"].clone(), - userid_usersigningkeyid: db["userid_usersigningkeyid"].clone(), - useridprofilekey_value: db["useridprofilekey_value"].clone(), - services: Services { - server: args.server.clone(), - globals: args.depend::("globals"), - state_cache: args.depend::("rooms::state_cache"), - state_accessor: args.depend::("rooms::state_accessor"), - }, - } - } - - /// Check if a user has an account on this homeserver. - #[inline] - pub(super) fn exists(&self, user_id: &UserId) -> Result { - Ok(self.userid_password.get(user_id.as_bytes())?.is_some()) - } - - /// Check if account is deactivated - pub(super) fn is_deactivated(&self, user_id: &UserId) -> Result { - Ok(self - .userid_password - .get(user_id.as_bytes())? - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "User does not exist."))? - .is_empty()) - } - - /// Returns the number of users registered on this server. - #[inline] - pub(super) fn count(&self) -> Result { Ok(self.userid_password.iter().count()) } - - /// Find out which user an access token belongs to. - pub(super) fn find_from_token(&self, token: &str) -> Result> { - self.token_userdeviceid - .get(token.as_bytes())? - .map_or(Ok(None), |bytes| { - let mut parts = bytes.split(|&b| b == 0xFF); - let user_bytes = parts - .next() - .ok_or_else(|| err!(Database("User ID in token_userdeviceid is invalid.")))?; - let device_bytes = parts - .next() - .ok_or_else(|| err!(Database("Device ID in token_userdeviceid is invalid.")))?; - - Ok(Some(( - UserId::parse( - utils::string_from_bytes(user_bytes) - .map_err(|e| err!(Database("User ID in token_userdeviceid is invalid unicode. {e}")))?, - ) - .map_err(|e| err!(Database("User ID in token_userdeviceid is invalid. {e}")))?, - utils::string_from_bytes(device_bytes) - .map_err(|e| err!(Database("Device ID in token_userdeviceid is invalid. {e}")))?, - ))) - }) - } - - /// Returns an iterator over all users on this homeserver. - pub fn iter<'a>(&'a self) -> Box> + 'a> { - Box::new(self.userid_password.iter().map(|(bytes, _)| { - UserId::parse( - utils::string_from_bytes(&bytes) - .map_err(|e| err!(Database("User ID in userid_password is invalid unicode. {e}")))?, - ) - .map_err(|e| err!(Database("User ID in userid_password is invalid. {e}"))) - })) - } - - /// Returns a list of local users as list of usernames. - /// - /// A user account is considered `local` if the length of it's password is - /// greater then zero. - pub(super) fn list_local_users(&self) -> Result> { - let users: Vec = self - .userid_password - .iter() - .filter_map(|(username, pw)| get_username_with_valid_password(&username, &pw)) - .collect(); - Ok(users) - } - - /// Returns the password hash for the given user. - pub(super) fn password_hash(&self, user_id: &UserId) -> Result> { - self.userid_password - .get(user_id.as_bytes())? - .map_or(Ok(None), |bytes| { - Ok(Some(utils::string_from_bytes(&bytes).map_err(|_| { - Error::bad_database("Password hash in db is not valid string.") - })?)) - }) - } - - /// Hash and set the user's password to the Argon2 hash - pub(super) fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { - if let Some(password) = password { - if let Ok(hash) = utils::hash::password(password) { - self.userid_password - .insert(user_id.as_bytes(), hash.as_bytes())?; - Ok(()) - } else { - Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Password does not meet the requirements.", - )) - } - } else { - self.userid_password.insert(user_id.as_bytes(), b"")?; - Ok(()) - } - } - - /// Returns the displayname of a user on this homeserver. - pub(super) fn displayname(&self, user_id: &UserId) -> Result> { - self.userid_displayname - .get(user_id.as_bytes())? - .map_or(Ok(None), |bytes| { - Ok(Some( - utils::string_from_bytes(&bytes) - .map_err(|e| err!(Database("Displayname in db is invalid. {e}")))?, - )) - }) - } - - /// Sets a new displayname or removes it if displayname is None. You still - /// need to nofify all rooms of this change. - pub(super) fn set_displayname(&self, user_id: &UserId, displayname: Option) -> Result<()> { - if let Some(displayname) = displayname { - self.userid_displayname - .insert(user_id.as_bytes(), displayname.as_bytes())?; - } else { - self.userid_displayname.remove(user_id.as_bytes())?; - } - - Ok(()) - } - - /// Get the `avatar_url` of a user. - pub(super) fn avatar_url(&self, user_id: &UserId) -> Result> { - self.userid_avatarurl - .get(user_id.as_bytes())? - .map(|bytes| { - let s_bytes = utils::string_from_bytes(&bytes) - .map_err(|e| err!(Database(warn!("Avatar URL in db is invalid: {e}"))))?; - let mxc_uri: OwnedMxcUri = s_bytes.into(); - Ok(mxc_uri) - }) - .transpose() - } - - /// Sets a new avatar_url or removes it if avatar_url is None. - pub(super) fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) -> Result<()> { - if let Some(avatar_url) = avatar_url { - self.userid_avatarurl - .insert(user_id.as_bytes(), avatar_url.to_string().as_bytes())?; - } else { - self.userid_avatarurl.remove(user_id.as_bytes())?; - } - - Ok(()) - } - - /// Get the blurhash of a user. - pub(super) fn blurhash(&self, user_id: &UserId) -> Result> { - self.userid_blurhash - .get(user_id.as_bytes())? - .map(|bytes| { - utils::string_from_bytes(&bytes).map_err(|e| err!(Database("Avatar URL in db is invalid. {e}"))) - }) - .transpose() - } - - /// Gets a specific user profile key - pub(super) fn profile_key(&self, user_id: &UserId, profile_key: &str) -> Result> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(profile_key.as_bytes()); - - self.useridprofilekey_value - .get(&key)? - .map_or(Ok(None), |bytes| Ok(Some(serde_json::from_slice(&bytes).unwrap()))) - } - - /// Gets all the user's profile keys and values in an iterator - pub(super) fn all_profile_keys<'a>( - &'a self, user_id: &UserId, - ) -> Box> + 'a + Send> { - let prefix = user_id.as_bytes().to_vec(); - - Box::new( - self.useridprofilekey_value - .scan_prefix(prefix) - .map(|(key, value)| { - let profile_key_name = utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .ok_or_else(|| err!(Database("Profile key in db is invalid")))?, - ) - .map_err(|e| err!(Database("Profile key in db is invalid. {e}")))?; - - let profile_key_value = serde_json::from_slice(&value) - .map_err(|e| err!(Database("Profile key in db is invalid. {e}")))?; - - Ok((profile_key_name, profile_key_value)) - }), - ) - } - - /// Sets a new profile key value, removes the key if value is None - pub(super) fn set_profile_key( - &self, user_id: &UserId, profile_key: &str, profile_key_value: Option, - ) -> Result<()> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(profile_key.as_bytes()); - - // TODO: insert to the stable MSC4175 key when it's stable - if let Some(value) = profile_key_value { - let value = serde_json::to_vec(&value).unwrap(); - - self.useridprofilekey_value.insert(&key, &value) - } else { - self.useridprofilekey_value.remove(&key) - } - } - - /// Get the timezone of a user. - pub(super) fn timezone(&self, user_id: &UserId) -> Result> { - // first check the unstable prefix - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(b"us.cloke.msc4175.tz"); - - let value = self - .useridprofilekey_value - .get(&key)? - .map(|bytes| utils::string_from_bytes(&bytes).map_err(|e| err!(Database("Timezone in db is invalid. {e}")))) - .transpose() - .unwrap(); - - // TODO: transparently migrate unstable key usage to the stable key once MSC4133 - // and MSC4175 are stable, likely a remove/insert in this block - if value.is_none() || value.as_ref().is_some_and(String::is_empty) { - // check the stable prefix - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(b"m.tz"); - - return self - .useridprofilekey_value - .get(&key)? - .map(|bytes| { - utils::string_from_bytes(&bytes).map_err(|e| err!(Database("Timezone in db is invalid. {e}"))) - }) - .transpose(); - } - - Ok(value) - } - - /// Sets a new timezone or removes it if timezone is None. - pub(super) fn set_timezone(&self, user_id: &UserId, timezone: Option) -> Result<()> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(b"us.cloke.msc4175.tz"); - - // TODO: insert to the stable MSC4175 key when it's stable - if let Some(timezone) = timezone { - self.useridprofilekey_value - .insert(&key, timezone.as_bytes())?; - } else { - self.useridprofilekey_value.remove(&key)?; - } - - Ok(()) - } - - /// Sets a new avatar_url or removes it if avatar_url is None. - pub(super) fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()> { - if let Some(blurhash) = blurhash { - self.userid_blurhash - .insert(user_id.as_bytes(), blurhash.as_bytes())?; - } else { - self.userid_blurhash.remove(user_id.as_bytes())?; - } - - Ok(()) - } - - /// Adds a new device to a user. - pub(super) fn create_device( - &self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option, - client_ip: Option, - ) -> Result<()> { - // This method should never be called for nonexistent users. We shouldn't assert - // though... - if !self.exists(user_id)? { - warn!("Called create_device for non-existent user {} in database", user_id); - return Err(Error::BadRequest(ErrorKind::InvalidParam, "User does not exist.")); - } - - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - - self.userid_devicelistversion - .increment(user_id.as_bytes())?; - - self.userdeviceid_metadata.insert( - &userdeviceid, - &serde_json::to_vec(&Device { - device_id: device_id.into(), - display_name: initial_device_display_name, - last_seen_ip: client_ip, - last_seen_ts: Some(MilliSecondsSinceUnixEpoch::now()), - }) - .expect("Device::to_string never fails."), - )?; - - self.set_token(user_id, device_id, token)?; - - Ok(()) - } - - /// Removes a device from a user. - pub(super) fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - - // Remove tokens - if let Some(old_token) = self.userdeviceid_token.get(&userdeviceid)? { - self.userdeviceid_token.remove(&userdeviceid)?; - self.token_userdeviceid.remove(&old_token)?; - } - - // Remove todevice events - let mut prefix = userdeviceid.clone(); - prefix.push(0xFF); - - for (key, _) in self.todeviceid_events.scan_prefix(prefix) { - self.todeviceid_events.remove(&key)?; - } - - // TODO: Remove onetimekeys - - self.userid_devicelistversion - .increment(user_id.as_bytes())?; - - self.userdeviceid_metadata.remove(&userdeviceid)?; - - Ok(()) - } - - /// Returns an iterator over all device ids of this user. - pub(super) fn all_device_ids<'a>( - &'a self, user_id: &UserId, - ) -> Box> + 'a> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - // All devices have metadata - Box::new( - self.userdeviceid_metadata - .scan_prefix(prefix) - .map(|(bytes, _)| { - Ok(utils::string_from_bytes( - bytes - .rsplit(|&b| b == 0xFF) - .next() - .ok_or_else(|| err!(Database("UserDevice ID in db is invalid.")))?, - ) - .map_err(|e| err!(Database("Device ID in userdeviceid_metadata is invalid. {e}")))? - .into()) - }), - ) - } - - /// Replaces the access token of one device. - pub(super) fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> { - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - - // should not be None, but we shouldn't assert either lol... - if self.userdeviceid_metadata.get(&userdeviceid)?.is_none() { - return Err!(Database(error!( - "User {user_id:?} does not exist or device ID {device_id:?} has no metadata." - ))); - } - - // Remove old token - if let Some(old_token) = self.userdeviceid_token.get(&userdeviceid)? { - self.token_userdeviceid.remove(&old_token)?; - // It will be removed from userdeviceid_token by the insert later - } - - // Assign token to user device combination - self.userdeviceid_token - .insert(&userdeviceid, token.as_bytes())?; - self.token_userdeviceid - .insert(token.as_bytes(), &userdeviceid)?; - - Ok(()) - } - - pub(super) fn add_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, one_time_key_key: &DeviceKeyId, - one_time_key_value: &Raw, - ) -> Result<()> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(device_id.as_bytes()); - - // All devices have metadata - // Only existing devices should be able to call this, but we shouldn't assert - // either... - if self.userdeviceid_metadata.get(&key)?.is_none() { - return Err!(Database(error!( - "User {user_id:?} does not exist or device ID {device_id:?} has no metadata." - ))); - } - - key.push(0xFF); - // TODO: Use DeviceKeyId::to_string when it's available (and update everything, - // because there are no wrapping quotation marks anymore) - key.extend_from_slice( - serde_json::to_string(one_time_key_key) - .expect("DeviceKeyId::to_string always works") - .as_bytes(), - ); - - self.onetimekeyid_onetimekeys.insert( - &key, - &serde_json::to_vec(&one_time_key_value).expect("OneTimeKey::to_vec always works"), - )?; - - self.userid_lastonetimekeyupdate - .insert(user_id.as_bytes(), &self.services.globals.next_count()?.to_be_bytes())?; - - Ok(()) - } - - pub(super) fn last_one_time_keys_update(&self, user_id: &UserId) -> Result { - self.userid_lastonetimekeyupdate - .get(user_id.as_bytes())? - .map_or(Ok(0), |bytes| { - utils::u64_from_bytes(&bytes) - .map_err(|e| err!(Database("Count in roomid_lastroomactiveupdate is invalid. {e}"))) - }) - } - - pub(super) fn take_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, key_algorithm: &DeviceKeyAlgorithm, - ) -> Result)>> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(device_id.as_bytes()); - prefix.push(0xFF); - prefix.push(b'"'); // Annoying quotation mark - prefix.extend_from_slice(key_algorithm.as_ref().as_bytes()); - prefix.push(b':'); - - self.userid_lastonetimekeyupdate - .insert(user_id.as_bytes(), &self.services.globals.next_count()?.to_be_bytes())?; - - self.onetimekeyid_onetimekeys - .scan_prefix(prefix) - .next() - .map(|(key, value)| { - self.onetimekeyid_onetimekeys.remove(&key)?; - - Ok(( - serde_json::from_slice( - key.rsplit(|&b| b == 0xFF) - .next() - .ok_or_else(|| err!(Database("OneTimeKeyId in db is invalid.")))?, - ) - .map_err(|e| err!(Database("OneTimeKeyId in db is invalid. {e}")))?, - serde_json::from_slice(&value).map_err(|e| err!(Database("OneTimeKeys in db are invalid. {e}")))?, - )) - }) - .transpose() - } - - pub(super) fn count_one_time_keys( - &self, user_id: &UserId, device_id: &DeviceId, - ) -> Result> { - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - - let mut counts = BTreeMap::new(); - - for algorithm in self - .onetimekeyid_onetimekeys - .scan_prefix(userdeviceid) - .map(|(bytes, _)| { - Ok::<_, Error>( - serde_json::from_slice::( - bytes - .rsplit(|&b| b == 0xFF) - .next() - .ok_or_else(|| err!(Database("OneTimeKey ID in db is invalid.")))?, - ) - .map_err(|e| err!(Database("DeviceKeyId in db is invalid. {e}")))? - .algorithm(), - ) - }) { - let count: &mut UInt = counts.entry(algorithm?).or_default(); - *count = count.saturating_add(uint!(1)); - } - - Ok(counts) - } - - pub(super) fn add_device_keys( - &self, user_id: &UserId, device_id: &DeviceId, device_keys: &Raw, - ) -> Result<()> { - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - - self.keyid_key.insert( - &userdeviceid, - &serde_json::to_vec(&device_keys).expect("DeviceKeys::to_vec always works"), - )?; - - self.mark_device_key_update(user_id)?; - - Ok(()) - } - - pub(super) fn add_cross_signing_keys( - &self, user_id: &UserId, master_key: &Raw, self_signing_key: &Option>, - user_signing_key: &Option>, notify: bool, - ) -> Result<()> { - // TODO: Check signatures - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - - let (master_key_key, _) = Self::parse_master_key(user_id, master_key)?; - - self.keyid_key - .insert(&master_key_key, master_key.json().get().as_bytes())?; - - self.userid_masterkeyid - .insert(user_id.as_bytes(), &master_key_key)?; - - // Self-signing key - if let Some(self_signing_key) = self_signing_key { - let mut self_signing_key_ids = self_signing_key - .deserialize() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid self signing key"))? - .keys - .into_values(); - - let self_signing_key_id = self_signing_key_ids - .next() - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Self signing key contained no key."))?; - - if self_signing_key_ids.next().is_some() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Self signing key contained more than one key.", - )); - } - - let mut self_signing_key_key = prefix.clone(); - self_signing_key_key.extend_from_slice(self_signing_key_id.as_bytes()); - - self.keyid_key - .insert(&self_signing_key_key, self_signing_key.json().get().as_bytes())?; - - self.userid_selfsigningkeyid - .insert(user_id.as_bytes(), &self_signing_key_key)?; - } - - // User-signing key - if let Some(user_signing_key) = user_signing_key { - let mut user_signing_key_ids = user_signing_key - .deserialize() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid user signing key"))? - .keys - .into_values(); - - let user_signing_key_id = user_signing_key_ids - .next() - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "User signing key contained no key."))?; - - if user_signing_key_ids.next().is_some() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "User signing key contained more than one key.", - )); - } - - let mut user_signing_key_key = prefix; - user_signing_key_key.extend_from_slice(user_signing_key_id.as_bytes()); - - self.keyid_key - .insert(&user_signing_key_key, user_signing_key.json().get().as_bytes())?; - - self.userid_usersigningkeyid - .insert(user_id.as_bytes(), &user_signing_key_key)?; - } - - if notify { - self.mark_device_key_update(user_id)?; - } - - Ok(()) - } - - pub(super) fn sign_key( - &self, target_id: &UserId, key_id: &str, signature: (String, String), sender_id: &UserId, - ) -> Result<()> { - let mut key = target_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(key_id.as_bytes()); - - let mut cross_signing_key: serde_json::Value = serde_json::from_slice( - &self - .keyid_key - .get(&key)? - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Tried to sign nonexistent key."))?, - ) - .map_err(|e| err!(Database("key in keyid_key is invalid. {e}")))?; - - let signatures = cross_signing_key - .get_mut("signatures") - .ok_or_else(|| err!(Database("key in keyid_key has no signatures field.")))? - .as_object_mut() - .ok_or_else(|| err!(Database("key in keyid_key has invalid signatures field.")))? - .entry(sender_id.to_string()) - .or_insert_with(|| serde_json::Map::new().into()); - - signatures - .as_object_mut() - .ok_or_else(|| err!(Database("signatures in keyid_key for a user is invalid.")))? - .insert(signature.0, signature.1.into()); - - self.keyid_key.insert( - &key, - &serde_json::to_vec(&cross_signing_key).expect("CrossSigningKey::to_vec always works"), - )?; - - self.mark_device_key_update(target_id)?; - - Ok(()) - } - - pub(super) fn keys_changed<'a>( - &'a self, user_or_room_id: &str, from: u64, to: Option, - ) -> Box> + 'a> { - let mut prefix = user_or_room_id.as_bytes().to_vec(); - prefix.push(0xFF); - - let mut start = prefix.clone(); - start.extend_from_slice(&(from.saturating_add(1)).to_be_bytes()); - - let to = to.unwrap_or(u64::MAX); - - Box::new( - self.keychangeid_userid - .iter_from(&start, false) - .take_while(move |(k, _)| { - k.starts_with(&prefix) - && if let Some(current) = k.splitn(2, |&b| b == 0xFF).nth(1) { - if let Ok(c) = utils::u64_from_bytes(current) { - c <= to - } else { - warn!("BadDatabase: Could not parse keychangeid_userid bytes"); - false - } - } else { - warn!("BadDatabase: Could not parse keychangeid_userid"); - false - } - }) - .map(|(_, bytes)| { - UserId::parse( - utils::string_from_bytes(&bytes).map_err(|_| { - Error::bad_database("User ID in devicekeychangeid_userid is invalid unicode.") - })?, - ) - .map_err(|e| err!(Database("User ID in devicekeychangeid_userid is invalid. {e}"))) - }), - ) - } - - pub(super) fn mark_device_key_update(&self, user_id: &UserId) -> Result<()> { - let count = self.services.globals.next_count()?.to_be_bytes(); - for room_id in self - .services - .state_cache - .rooms_joined(user_id) - .filter_map(Result::ok) - { - // Don't send key updates to unencrypted rooms - if self - .services - .state_accessor - .room_state_get(&room_id, &StateEventType::RoomEncryption, "")? - .is_none() - { - continue; - } - - let mut key = room_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(&count); - - self.keychangeid_userid.insert(&key, user_id.as_bytes())?; - } - - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(&count); - self.keychangeid_userid.insert(&key, user_id.as_bytes())?; - - Ok(()) - } - - pub(super) fn get_device_keys(&self, user_id: &UserId, device_id: &DeviceId) -> Result>> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(device_id.as_bytes()); - - self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| { - Ok(Some( - serde_json::from_slice(&bytes).map_err(|e| err!(Database("DeviceKeys in db are invalid. {e}")))?, - )) - }) - } - - pub(super) fn parse_master_key( - user_id: &UserId, master_key: &Raw, - ) -> Result<(Vec, CrossSigningKey)> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - - let master_key = master_key - .deserialize() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid master key"))?; - let mut master_key_ids = master_key.keys.values(); - let master_key_id = master_key_ids - .next() - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Master key contained no key."))?; - if master_key_ids.next().is_some() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Master key contained more than one key.", - )); - } - let mut master_key_key = prefix.clone(); - master_key_key.extend_from_slice(master_key_id.as_bytes()); - Ok((master_key_key, master_key)) - } - - pub(super) fn get_key( - &self, key: &[u8], sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, - ) -> Result>> { - self.keyid_key.get(key)?.map_or(Ok(None), |bytes| { - let mut cross_signing_key = serde_json::from_slice::(&bytes) - .map_err(|e| err!(Database("CrossSigningKey in db is invalid. {e}")))?; - clean_signatures(&mut cross_signing_key, sender_user, user_id, allowed_signatures)?; - - Ok(Some(Raw::from_json( - serde_json::value::to_raw_value(&cross_signing_key).expect("Value to RawValue serialization"), - ))) - }) - } - - pub(super) fn get_master_key( - &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, - ) -> Result>> { - self.userid_masterkeyid - .get(user_id.as_bytes())? - .map_or(Ok(None), |key| self.get_key(&key, sender_user, user_id, allowed_signatures)) - } - - pub(super) fn get_self_signing_key( - &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, - ) -> Result>> { - self.userid_selfsigningkeyid - .get(user_id.as_bytes())? - .map_or(Ok(None), |key| self.get_key(&key, sender_user, user_id, allowed_signatures)) - } - - pub(super) fn get_user_signing_key(&self, user_id: &UserId) -> Result>> { - self.userid_usersigningkeyid - .get(user_id.as_bytes())? - .map_or(Ok(None), |key| { - self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| { - Ok(Some( - serde_json::from_slice(&bytes) - .map_err(|e| err!(Database("CrossSigningKey in db is invalid. {e}")))?, - )) - }) - }) - } - - pub(super) fn add_to_device_event( - &self, sender: &UserId, target_user_id: &UserId, target_device_id: &DeviceId, event_type: &str, - content: serde_json::Value, - ) -> Result<()> { - let mut key = target_user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(target_device_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(&self.services.globals.next_count()?.to_be_bytes()); - - let mut json = serde_json::Map::new(); - json.insert("type".to_owned(), event_type.to_owned().into()); - json.insert("sender".to_owned(), sender.to_string().into()); - json.insert("content".to_owned(), content); - - let value = serde_json::to_vec(&json).expect("Map::to_vec always works"); - - self.todeviceid_events.insert(&key, &value)?; - - Ok(()) - } - - pub(super) fn get_to_device_events( - &self, user_id: &UserId, device_id: &DeviceId, - ) -> Result>> { - let mut events = Vec::new(); - - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(device_id.as_bytes()); - prefix.push(0xFF); - - for (_, value) in self.todeviceid_events.scan_prefix(prefix) { - events.push( - serde_json::from_slice(&value) - .map_err(|e| err!(Database("Event in todeviceid_events is invalid. {e}")))?, - ); - } - - Ok(events) - } - - pub(super) fn remove_to_device_events(&self, user_id: &UserId, device_id: &DeviceId, until: u64) -> Result<()> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(device_id.as_bytes()); - prefix.push(0xFF); - - let mut last = prefix.clone(); - last.extend_from_slice(&until.to_be_bytes()); - - for (key, _) in self - .todeviceid_events - .iter_from(&last, true) // this includes last - .take_while(move |(k, _)| k.starts_with(&prefix)) - .map(|(key, _)| { - Ok::<_, Error>(( - key.clone(), - utils::u64_from_bytes(&key[key.len().saturating_sub(size_of::())..key.len()]) - .map_err(|e| err!(Database("ToDeviceId has invalid count bytes. {e}")))?, - )) - }) - .filter_map(Result::ok) - .take_while(|&(_, count)| count <= until) - { - self.todeviceid_events.remove(&key)?; - } - - Ok(()) - } - - pub(super) fn update_device_metadata(&self, user_id: &UserId, device_id: &DeviceId, device: &Device) -> Result<()> { - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - - // Only existing devices should be able to call this, but we shouldn't assert - // either... - if self.userdeviceid_metadata.get(&userdeviceid)?.is_none() { - warn!( - "Called update_device_metadata for a non-existent user \"{}\" and/or device ID \"{}\" with no \ - metadata in database", - user_id, device_id - ); - return Err(Error::bad_database( - "User does not exist or device ID has no metadata in database.", - )); - } - - self.userid_devicelistversion - .increment(user_id.as_bytes())?; - - self.userdeviceid_metadata.insert( - &userdeviceid, - &serde_json::to_vec(device).expect("Device::to_string always works"), - )?; - - Ok(()) - } - - /// Get device metadata. - pub(super) fn get_device_metadata(&self, user_id: &UserId, device_id: &DeviceId) -> Result> { - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - - self.userdeviceid_metadata - .get(&userdeviceid)? - .map_or(Ok(None), |bytes| { - Ok(Some(serde_json::from_slice(&bytes).map_err(|_| { - Error::bad_database("Metadata in userdeviceid_metadata is invalid.") - })?)) - }) - } - - pub(super) fn get_devicelist_version(&self, user_id: &UserId) -> Result> { - self.userid_devicelistversion - .get(user_id.as_bytes())? - .map_or(Ok(None), |bytes| { - utils::u64_from_bytes(&bytes) - .map_err(|e| err!(Database("Invalid devicelistversion in db. {e}"))) - .map(Some) - }) - } - - pub(super) fn all_devices_metadata<'a>( - &'a self, user_id: &UserId, - ) -> Box> + 'a> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - - Box::new( - self.userdeviceid_metadata - .scan_prefix(key) - .map(|(_, bytes)| { - serde_json::from_slice::(&bytes) - .map_err(|e| err!(Database("Device in userdeviceid_metadata is invalid. {e}"))) - }), - ) - } - - /// Creates a new sync filter. Returns the filter id. - pub(super) fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> Result { - let filter_id = utils::random_string(4); - - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(filter_id.as_bytes()); - - self.userfilterid_filter - .insert(&key, &serde_json::to_vec(&filter).expect("filter is valid json"))?; - - Ok(filter_id) - } - - pub(super) fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result> { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(filter_id.as_bytes()); - - let raw = self.userfilterid_filter.get(&key)?; - - if let Some(raw) = raw { - serde_json::from_slice(&raw).map_err(|e| err!(Database("Invalid filter event in db. {e}"))) - } else { - Ok(None) - } - } - - /// Creates an OpenID token, which can be used to prove that a user has - /// access to an account (primarily for integrations) - pub(super) fn create_openid_token(&self, user_id: &UserId, token: &str) -> Result { - use std::num::Saturating as Sat; - - let expires_in = self.services.server.config.openid_token_ttl; - let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in) * Sat(1000); - - let mut value = expires_at.0.to_be_bytes().to_vec(); - value.extend_from_slice(user_id.as_bytes()); - - self.openidtoken_expiresatuserid - .insert(token.as_bytes(), value.as_slice())?; - - Ok(expires_in) - } - - /// Find out which user an OpenID access token belongs to. - pub(super) fn find_from_openid_token(&self, token: &str) -> Result { - let Some(value) = self.openidtoken_expiresatuserid.get(token.as_bytes())? else { - return Err(Error::BadRequest(ErrorKind::Unauthorized, "OpenID token is unrecognised")); - }; - - let (expires_at_bytes, user_bytes) = value.split_at(0_u64.to_be_bytes().len()); - - let expires_at = u64::from_be_bytes( - expires_at_bytes - .try_into() - .map_err(|e| err!(Database("expires_at in openid_userid is invalid u64. {e}")))?, - ); - - if expires_at < utils::millis_since_unix_epoch() { - debug_info!("OpenID token is expired, removing"); - self.openidtoken_expiresatuserid.remove(token.as_bytes())?; - - return Err(Error::BadRequest(ErrorKind::Unauthorized, "OpenID token is expired")); - } - - UserId::parse( - utils::string_from_bytes(user_bytes) - .map_err(|e| err!(Database("User ID in openid_userid is invalid unicode. {e}")))?, - ) - .map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) - } -} - -/// Will only return with Some(username) if the password was not empty and the -/// username could be successfully parsed. -/// If `utils::string_from_bytes`(...) returns an error that username will be -/// skipped and the error will be logged. -pub(super) fn get_username_with_valid_password(username: &[u8], password: &[u8]) -> Option { - // A valid password is not empty - if password.is_empty() { - None - } else { - match utils::string_from_bytes(username) { - Ok(u) => Some(u), - Err(e) => { - warn!("Failed to parse username while calling get_local_users(): {}", e.to_string()); - None - }, - } - } -} diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 80897b5f..9a058ba9 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,552 +1,984 @@ -mod data; +use std::{collections::BTreeMap, mem, mem::size_of, sync::Arc}; -use std::{ - collections::{BTreeMap, BTreeSet}, - mem, - sync::{Arc, Mutex, Mutex as StdMutex}, +use conduit::{ + debug_warn, err, utils, + utils::{stream::TryIgnore, string::Unquoted, ReadyExt, TryReadyExt}, + warn, Err, Error, Result, Server, }; - -use conduit::{Error, Result}; +use database::{Deserialized, Ignore, Interfix, Map}; +use futures::{pin_mut, FutureExt, Stream, StreamExt, TryFutureExt}; use ruma::{ - api::client::{ - device::Device, - filter::FilterDefinition, - sync::sync_events::{ - self, - v4::{ExtensionsConfig, SyncRequestList}, - }, - }, + api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, - events::AnyToDeviceEvent, + events::{AnyToDeviceEvent, StateEventType}, serde::Raw, - DeviceId, DeviceKeyAlgorithm, DeviceKeyId, OwnedDeviceId, OwnedDeviceKeyId, OwnedMxcUri, OwnedRoomId, OwnedUserId, - UInt, UserId, + DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedDeviceKeyId, + OwnedMxcUri, OwnedUserId, UInt, UserId, }; -use self::data::Data; -use crate::{admin, rooms, Dep}; +use crate::{admin, globals, rooms, Dep}; pub struct Service { - connections: DbConnections, - pub db: Data, services: Services, + db: Data, } struct Services { + server: Arc, admin: Dep, + globals: Dep, + state_accessor: Dep, state_cache: Dep, } +struct Data { + keychangeid_userid: Arc, + keyid_key: Arc, + onetimekeyid_onetimekeys: Arc, + openidtoken_expiresatuserid: Arc, + todeviceid_events: Arc, + token_userdeviceid: Arc, + userdeviceid_metadata: Arc, + userdeviceid_token: Arc, + userfilterid_filter: Arc, + userid_avatarurl: Arc, + userid_blurhash: Arc, + userid_devicelistversion: Arc, + userid_displayname: Arc, + userid_lastonetimekeyupdate: Arc, + userid_masterkeyid: Arc, + userid_password: Arc, + userid_selfsigningkeyid: Arc, + userid_usersigningkeyid: Arc, + useridprofilekey_value: Arc, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - connections: StdMutex::new(BTreeMap::new()), - db: Data::new(&args), services: Services { + server: args.server.clone(), admin: args.depend::("admin"), + globals: args.depend::("globals"), + state_accessor: args.depend::("rooms::state_accessor"), state_cache: args.depend::("rooms::state_cache"), }, + db: Data { + keychangeid_userid: args.db["keychangeid_userid"].clone(), + keyid_key: args.db["keyid_key"].clone(), + onetimekeyid_onetimekeys: args.db["onetimekeyid_onetimekeys"].clone(), + openidtoken_expiresatuserid: args.db["openidtoken_expiresatuserid"].clone(), + todeviceid_events: args.db["todeviceid_events"].clone(), + token_userdeviceid: args.db["token_userdeviceid"].clone(), + userdeviceid_metadata: args.db["userdeviceid_metadata"].clone(), + userdeviceid_token: args.db["userdeviceid_token"].clone(), + userfilterid_filter: args.db["userfilterid_filter"].clone(), + userid_avatarurl: args.db["userid_avatarurl"].clone(), + userid_blurhash: args.db["userid_blurhash"].clone(), + userid_devicelistversion: args.db["userid_devicelistversion"].clone(), + userid_displayname: args.db["userid_displayname"].clone(), + userid_lastonetimekeyupdate: args.db["userid_lastonetimekeyupdate"].clone(), + userid_masterkeyid: args.db["userid_masterkeyid"].clone(), + userid_password: args.db["userid_password"].clone(), + userid_selfsigningkeyid: args.db["userid_selfsigningkeyid"].clone(), + userid_usersigningkeyid: args.db["userid_usersigningkeyid"].clone(), + useridprofilekey_value: args.db["useridprofilekey_value"].clone(), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -type DbConnections = Mutex>; -type DbConnectionsKey = (OwnedUserId, OwnedDeviceId, String); -type DbConnectionsVal = Arc>; - -struct SlidingSyncCache { - lists: BTreeMap, - subscriptions: BTreeMap, - known_rooms: BTreeMap>, // For every room, the roomsince number - extensions: ExtensionsConfig, -} - impl Service { - /// Check if a user has an account on this homeserver. - #[inline] - pub fn exists(&self, user_id: &UserId) -> Result { self.db.exists(user_id) } - - pub fn remembered(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) -> bool { - self.connections - .lock() - .unwrap() - .contains_key(&(user_id, device_id, conn_id)) - } - - pub fn forget_sync_request_connection(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) { - self.connections - .lock() - .unwrap() - .remove(&(user_id, device_id, conn_id)); - } - - pub fn update_sync_request_with_cache( - &self, user_id: OwnedUserId, device_id: OwnedDeviceId, request: &mut sync_events::v4::Request, - ) -> BTreeMap> { - let Some(conn_id) = request.conn_id.clone() else { - return BTreeMap::new(); - }; - - let mut cache = self.connections.lock().unwrap(); - let cached = Arc::clone( - cache - .entry((user_id, device_id, conn_id)) - .or_insert_with(|| { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }), - ); - let cached = &mut cached.lock().unwrap(); - drop(cache); - - for (list_id, list) in &mut request.lists { - if let Some(cached_list) = cached.lists.get(list_id) { - if list.sort.is_empty() { - list.sort.clone_from(&cached_list.sort); - }; - if list.room_details.required_state.is_empty() { - list.room_details - .required_state - .clone_from(&cached_list.room_details.required_state); - }; - list.room_details.timeline_limit = list - .room_details - .timeline_limit - .or(cached_list.room_details.timeline_limit); - list.include_old_rooms = list - .include_old_rooms - .clone() - .or_else(|| cached_list.include_old_rooms.clone()); - match (&mut list.filters, cached_list.filters.clone()) { - (Some(list_filters), Some(cached_filters)) => { - list_filters.is_dm = list_filters.is_dm.or(cached_filters.is_dm); - if list_filters.spaces.is_empty() { - list_filters.spaces = cached_filters.spaces; - } - list_filters.is_encrypted = list_filters.is_encrypted.or(cached_filters.is_encrypted); - list_filters.is_invite = list_filters.is_invite.or(cached_filters.is_invite); - if list_filters.room_types.is_empty() { - list_filters.room_types = cached_filters.room_types; - } - if list_filters.not_room_types.is_empty() { - list_filters.not_room_types = cached_filters.not_room_types; - } - list_filters.room_name_like = list_filters - .room_name_like - .clone() - .or(cached_filters.room_name_like); - if list_filters.tags.is_empty() { - list_filters.tags = cached_filters.tags; - } - if list_filters.not_tags.is_empty() { - list_filters.not_tags = cached_filters.not_tags; - } - }, - (_, Some(cached_filters)) => list.filters = Some(cached_filters), - (Some(list_filters), _) => list.filters = Some(list_filters.clone()), - (..) => {}, - } - if list.bump_event_types.is_empty() { - list.bump_event_types - .clone_from(&cached_list.bump_event_types); - }; - } - cached.lists.insert(list_id.clone(), list.clone()); - } - - cached - .subscriptions - .extend(request.room_subscriptions.clone()); - request - .room_subscriptions - .extend(cached.subscriptions.clone()); - - request.extensions.e2ee.enabled = request - .extensions - .e2ee - .enabled - .or(cached.extensions.e2ee.enabled); - - request.extensions.to_device.enabled = request - .extensions - .to_device - .enabled - .or(cached.extensions.to_device.enabled); - - request.extensions.account_data.enabled = request - .extensions - .account_data - .enabled - .or(cached.extensions.account_data.enabled); - request.extensions.account_data.lists = request - .extensions - .account_data - .lists - .clone() - .or_else(|| cached.extensions.account_data.lists.clone()); - request.extensions.account_data.rooms = request - .extensions - .account_data - .rooms - .clone() - .or_else(|| cached.extensions.account_data.rooms.clone()); - - cached.extensions = request.extensions.clone(); - - cached.known_rooms.clone() - } - - pub fn update_sync_subscriptions( - &self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String, - subscriptions: BTreeMap, - ) { - let mut cache = self.connections.lock().unwrap(); - let cached = Arc::clone( - cache - .entry((user_id, device_id, conn_id)) - .or_insert_with(|| { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }), - ); - let cached = &mut cached.lock().unwrap(); - drop(cache); - - cached.subscriptions = subscriptions; - } - - pub fn update_sync_known_rooms( - &self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String, list_id: String, - new_cached_rooms: BTreeSet, globalsince: u64, - ) { - let mut cache = self.connections.lock().unwrap(); - let cached = Arc::clone( - cache - .entry((user_id, device_id, conn_id)) - .or_insert_with(|| { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }), - ); - let cached = &mut cached.lock().unwrap(); - drop(cache); - - for (roomid, lastsince) in cached - .known_rooms - .entry(list_id.clone()) - .or_default() - .iter_mut() - { - if !new_cached_rooms.contains(roomid) { - *lastsince = 0; - } - } - let list = cached.known_rooms.entry(list_id).or_default(); - for roomid in new_cached_rooms { - list.insert(roomid, globalsince); - } - } - - /// Check if account is deactivated - pub fn is_deactivated(&self, user_id: &UserId) -> Result { self.db.is_deactivated(user_id) } - /// Check if a user is an admin - pub fn is_admin(&self, user_id: &UserId) -> Result { - if let Some(admin_room_id) = self.services.admin.get_admin_room()? { - self.services.state_cache.is_joined(user_id, &admin_room_id) - } else { - Ok(false) - } - } + #[inline] + pub async fn is_admin(&self, user_id: &UserId) -> bool { self.services.admin.user_is_admin(user_id).await } /// Create a new user account on this homeserver. #[inline] pub fn create(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { - self.db.set_password(user_id, password)?; - Ok(()) - } - - /// Returns the number of users registered on this server. - #[inline] - pub fn count(&self) -> Result { self.db.count() } - - /// Find out which user an access token belongs to. - pub fn find_from_token(&self, token: &str) -> Result> { - self.db.find_from_token(token) - } - - /// Returns an iterator over all users on this homeserver. - pub fn iter(&self) -> impl Iterator> + '_ { self.db.iter() } - - /// Returns a list of local users as list of usernames. - /// - /// A user account is considered `local` if the length of it's password is - /// greater then zero. - pub fn list_local_users(&self) -> Result> { self.db.list_local_users() } - - /// Returns the password hash for the given user. - pub fn password_hash(&self, user_id: &UserId) -> Result> { self.db.password_hash(user_id) } - - /// Hash and set the user's password to the Argon2 hash - #[inline] - pub fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { - self.db.set_password(user_id, password) - } - - /// Returns the displayname of a user on this homeserver. - pub fn displayname(&self, user_id: &UserId) -> Result> { self.db.displayname(user_id) } - - /// Sets a new displayname or removes it if displayname is None. You still - /// need to nofify all rooms of this change. - pub async fn set_displayname(&self, user_id: &UserId, displayname: Option) -> Result<()> { - self.db.set_displayname(user_id, displayname) - } - - /// Get the avatar_url of a user. - pub fn avatar_url(&self, user_id: &UserId) -> Result> { self.db.avatar_url(user_id) } - - /// Sets a new avatar_url or removes it if avatar_url is None. - pub async fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) -> Result<()> { - self.db.set_avatar_url(user_id, avatar_url) - } - - /// Get the blurhash of a user. - pub fn blurhash(&self, user_id: &UserId) -> Result> { self.db.blurhash(user_id) } - - pub fn timezone(&self, user_id: &UserId) -> Result> { self.db.timezone(user_id) } - - /// Gets a specific user profile key - pub fn profile_key(&self, user_id: &UserId, profile_key: &str) -> Result> { - self.db.profile_key(user_id, profile_key) - } - - /// Gets all the user's profile keys and values in an iterator - pub fn all_profile_keys<'a>( - &'a self, user_id: &UserId, - ) -> Box> + 'a + Send> { - self.db.all_profile_keys(user_id) - } - - /// Sets a new profile key value, removes the key if value is None - pub fn set_profile_key( - &self, user_id: &UserId, profile_key: &str, profile_key_value: Option, - ) -> Result<()> { - self.db - .set_profile_key(user_id, profile_key, profile_key_value) - } - - /// Sets a new tz or removes it if tz is None. - pub async fn set_timezone(&self, user_id: &UserId, tz: Option) -> Result<()> { - self.db.set_timezone(user_id, tz) - } - - /// Sets a new blurhash or removes it if blurhash is None. - pub async fn set_blurhash(&self, user_id: &UserId, blurhash: Option) -> Result<()> { - self.db.set_blurhash(user_id, blurhash) - } - - /// Adds a new device to a user. - pub fn create_device( - &self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option, - client_ip: Option, - ) -> Result<()> { - self.db - .create_device(user_id, device_id, token, initial_device_display_name, client_ip) - } - - /// Removes a device from a user. - pub fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { - self.db.remove_device(user_id, device_id) - } - - /// Returns an iterator over all device ids of this user. - pub fn all_device_ids<'a>(&'a self, user_id: &UserId) -> impl Iterator> + 'a { - self.db.all_device_ids(user_id) - } - - /// Replaces the access token of one device. - #[inline] - pub fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> { - self.db.set_token(user_id, device_id, token) - } - - pub fn add_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, one_time_key_key: &DeviceKeyId, - one_time_key_value: &Raw, - ) -> Result<()> { - self.db - .add_one_time_key(user_id, device_id, one_time_key_key, one_time_key_value) - } - - // TODO: use this ? - #[allow(dead_code)] - pub fn last_one_time_keys_update(&self, user_id: &UserId) -> Result { - self.db.last_one_time_keys_update(user_id) - } - - pub fn take_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, key_algorithm: &DeviceKeyAlgorithm, - ) -> Result)>> { - self.db.take_one_time_key(user_id, device_id, key_algorithm) - } - - pub fn count_one_time_keys( - &self, user_id: &UserId, device_id: &DeviceId, - ) -> Result> { - self.db.count_one_time_keys(user_id, device_id) - } - - pub fn add_device_keys(&self, user_id: &UserId, device_id: &DeviceId, device_keys: &Raw) -> Result<()> { - self.db.add_device_keys(user_id, device_id, device_keys) - } - - pub fn add_cross_signing_keys( - &self, user_id: &UserId, master_key: &Raw, self_signing_key: &Option>, - user_signing_key: &Option>, notify: bool, - ) -> Result<()> { - self.db - .add_cross_signing_keys(user_id, master_key, self_signing_key, user_signing_key, notify) - } - - pub fn sign_key( - &self, target_id: &UserId, key_id: &str, signature: (String, String), sender_id: &UserId, - ) -> Result<()> { - self.db.sign_key(target_id, key_id, signature, sender_id) - } - - pub fn keys_changed<'a>( - &'a self, user_or_room_id: &str, from: u64, to: Option, - ) -> impl Iterator> + 'a { - self.db.keys_changed(user_or_room_id, from, to) - } - - #[inline] - pub fn mark_device_key_update(&self, user_id: &UserId) -> Result<()> { self.db.mark_device_key_update(user_id) } - - pub fn get_device_keys(&self, user_id: &UserId, device_id: &DeviceId) -> Result>> { - self.db.get_device_keys(user_id, device_id) - } - - #[inline] - pub fn parse_master_key( - &self, user_id: &UserId, master_key: &Raw, - ) -> Result<(Vec, CrossSigningKey)> { - Data::parse_master_key(user_id, master_key) - } - - #[inline] - pub fn get_key( - &self, key: &[u8], sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, - ) -> Result>> { - self.db - .get_key(key, sender_user, user_id, allowed_signatures) - } - - pub fn get_master_key( - &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, - ) -> Result>> { - self.db - .get_master_key(sender_user, user_id, allowed_signatures) - } - - pub fn get_self_signing_key( - &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &dyn Fn(&UserId) -> bool, - ) -> Result>> { - self.db - .get_self_signing_key(sender_user, user_id, allowed_signatures) - } - - pub fn get_user_signing_key(&self, user_id: &UserId) -> Result>> { - self.db.get_user_signing_key(user_id) - } - - pub fn add_to_device_event( - &self, sender: &UserId, target_user_id: &UserId, target_device_id: &DeviceId, event_type: &str, - content: serde_json::Value, - ) -> Result<()> { - self.db - .add_to_device_event(sender, target_user_id, target_device_id, event_type, content) - } - - pub fn get_to_device_events(&self, user_id: &UserId, device_id: &DeviceId) -> Result>> { - self.db.get_to_device_events(user_id, device_id) - } - - pub fn remove_to_device_events(&self, user_id: &UserId, device_id: &DeviceId, until: u64) -> Result<()> { - self.db.remove_to_device_events(user_id, device_id, until) - } - - pub fn update_device_metadata(&self, user_id: &UserId, device_id: &DeviceId, device: &Device) -> Result<()> { - self.db.update_device_metadata(user_id, device_id, device) - } - - /// Get device metadata. - pub fn get_device_metadata(&self, user_id: &UserId, device_id: &DeviceId) -> Result> { - self.db.get_device_metadata(user_id, device_id) - } - - pub fn get_devicelist_version(&self, user_id: &UserId) -> Result> { - self.db.get_devicelist_version(user_id) - } - - pub fn all_devices_metadata<'a>(&'a self, user_id: &UserId) -> impl Iterator> + 'a { - self.db.all_devices_metadata(user_id) + self.set_password(user_id, password) } /// Deactivate account - pub fn deactivate_account(&self, user_id: &UserId) -> Result<()> { + pub async fn deactivate_account(&self, user_id: &UserId) -> Result<()> { // Remove all associated devices - for device_id in self.all_device_ids(user_id) { - self.remove_device(user_id, &device_id?)?; - } + self.all_device_ids(user_id) + .for_each(|device_id| self.remove_device(user_id, device_id)) + .await; // Set the password to "" to indicate a deactivated account. Hashes will never // result in an empty string, so the user will not be able to log in again. // Systems like changing the password without logging in should check if the // account is deactivated. - self.db.set_password(user_id, None)?; + self.set_password(user_id, None)?; // TODO: Unhook 3PID Ok(()) } - /// Creates a new sync filter. Returns the filter id. - pub fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> Result { - self.db.create_filter(user_id, filter) + /// Check if a user has an account on this homeserver. + #[inline] + pub async fn exists(&self, user_id: &UserId) -> bool { self.db.userid_password.qry(user_id).await.is_ok() } + + /// Check if account is deactivated + pub async fn is_deactivated(&self, user_id: &UserId) -> Result { + self.db + .userid_password + .qry(user_id) + .map_ok(|val| val.is_empty()) + .map_err(|_| err!(Request(NotFound("User does not exist.")))) + .await } - pub fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result> { - self.db.get_filter(user_id, filter_id) + /// Check if account is active, infallible + pub async fn is_active(&self, user_id: &UserId) -> bool { !self.is_deactivated(user_id).await.unwrap_or(true) } + + /// Check if account is active, infallible + pub async fn is_active_local(&self, user_id: &UserId) -> bool { + self.services.globals.user_is_local(user_id) && self.is_active(user_id).await + } + + /// Returns the number of users registered on this server. + #[inline] + pub async fn count(&self) -> usize { self.db.userid_password.count().await } + + /// Find out which user an access token belongs to. + pub async fn find_from_token(&self, token: &str) -> Result<(OwnedUserId, OwnedDeviceId)> { + self.db.token_userdeviceid.qry(token).await.deserialized() + } + + /// Returns an iterator over all users on this homeserver (offered for + /// compatibility) + #[allow(clippy::iter_without_into_iter, clippy::iter_not_returning_iterator)] + pub fn iter(&self) -> impl Stream + Send + '_ { self.stream().map(ToOwned::to_owned) } + + /// Returns an iterator over all users on this homeserver. + pub fn stream(&self) -> impl Stream + Send { self.db.userid_password.keys().ignore_err() } + + /// Returns a list of local users as list of usernames. + /// + /// A user account is considered `local` if the length of it's password is + /// greater then zero. + pub fn list_local_users(&self) -> impl Stream + Send + '_ { + self.db + .userid_password + .stream() + .ignore_err() + .ready_filter_map(|(u, p): (&UserId, &[u8])| (!p.is_empty()).then_some(u)) + } + + /// Returns the password hash for the given user. + pub async fn password_hash(&self, user_id: &UserId) -> Result { + self.db.userid_password.qry(user_id).await.deserialized() + } + + /// Hash and set the user's password to the Argon2 hash + pub fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { + if let Some(password) = password { + if let Ok(hash) = utils::hash::password(password) { + self.db + .userid_password + .insert(user_id.as_bytes(), hash.as_bytes()); + Ok(()) + } else { + Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Password does not meet the requirements.", + )) + } + } else { + self.db.userid_password.insert(user_id.as_bytes(), b""); + Ok(()) + } + } + + /// Returns the displayname of a user on this homeserver. + pub async fn displayname(&self, user_id: &UserId) -> Result { + self.db.userid_displayname.qry(user_id).await.deserialized() + } + + /// Sets a new displayname or removes it if displayname is None. You still + /// need to nofify all rooms of this change. + pub fn set_displayname(&self, user_id: &UserId, displayname: Option) { + if let Some(displayname) = displayname { + self.db + .userid_displayname + .insert(user_id.as_bytes(), displayname.as_bytes()); + } else { + self.db.userid_displayname.remove(user_id.as_bytes()); + } + } + + /// Get the `avatar_url` of a user. + pub async fn avatar_url(&self, user_id: &UserId) -> Result { + self.db.userid_avatarurl.qry(user_id).await.deserialized() + } + + /// Sets a new avatar_url or removes it if avatar_url is None. + pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) { + if let Some(avatar_url) = avatar_url { + self.db + .userid_avatarurl + .insert(user_id.as_bytes(), avatar_url.to_string().as_bytes()); + } else { + self.db.userid_avatarurl.remove(user_id.as_bytes()); + } + } + + /// Get the blurhash of a user. + pub async fn blurhash(&self, user_id: &UserId) -> Result { + self.db.userid_blurhash.qry(user_id).await.deserialized() + } + + /// Sets a new avatar_url or removes it if avatar_url is None. + pub fn set_blurhash(&self, user_id: &UserId, blurhash: Option) { + if let Some(blurhash) = blurhash { + self.db + .userid_blurhash + .insert(user_id.as_bytes(), blurhash.as_bytes()); + } else { + self.db.userid_blurhash.remove(user_id.as_bytes()); + } + } + + /// Adds a new device to a user. + pub async fn create_device( + &self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option, + client_ip: Option, + ) -> Result<()> { + // This method should never be called for nonexistent users. We shouldn't assert + // though... + if !self.exists(user_id).await { + warn!("Called create_device for non-existent user {} in database", user_id); + return Err(Error::BadRequest(ErrorKind::InvalidParam, "User does not exist.")); + } + + let mut userdeviceid = user_id.as_bytes().to_vec(); + userdeviceid.push(0xFF); + userdeviceid.extend_from_slice(device_id.as_bytes()); + + increment(&self.db.userid_devicelistversion, user_id.as_bytes()); + + self.db.userdeviceid_metadata.insert( + &userdeviceid, + &serde_json::to_vec(&Device { + device_id: device_id.into(), + display_name: initial_device_display_name, + last_seen_ip: client_ip, + last_seen_ts: Some(MilliSecondsSinceUnixEpoch::now()), + }) + .expect("Device::to_string never fails."), + ); + + self.set_token(user_id, device_id, token).await?; + + Ok(()) + } + + /// Removes a device from a user. + pub async fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) { + let mut userdeviceid = user_id.as_bytes().to_vec(); + userdeviceid.push(0xFF); + userdeviceid.extend_from_slice(device_id.as_bytes()); + + // Remove tokens + if let Ok(old_token) = self.db.userdeviceid_token.qry(&userdeviceid).await { + self.db.userdeviceid_token.remove(&userdeviceid); + self.db.token_userdeviceid.remove(&old_token); + } + + // Remove todevice events + let prefix = (user_id, device_id, Interfix); + self.db + .todeviceid_events + .keys_raw_prefix(&prefix) + .ignore_err() + .ready_for_each(|key| self.db.todeviceid_events.remove(key)) + .await; + + // TODO: Remove onetimekeys + + increment(&self.db.userid_devicelistversion, user_id.as_bytes()); + + self.db.userdeviceid_metadata.remove(&userdeviceid); + } + + /// Returns an iterator over all device ids of this user. + pub fn all_device_ids<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + let prefix = (user_id, Interfix); + self.db + .userdeviceid_metadata + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, device_id): (Ignore, &DeviceId)| device_id) + } + + /// Replaces the access token of one device. + pub async fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> { + let key = (user_id, device_id); + // should not be None, but we shouldn't assert either lol... + if self.db.userdeviceid_metadata.qry(&key).await.is_err() { + return Err!(Database(error!( + ?user_id, + ?device_id, + "User does not exist or device has no metadata." + ))); + } + + // Remove old token + if let Ok(old_token) = self.db.userdeviceid_token.qry(&key).await { + self.db.token_userdeviceid.remove(&old_token); + // It will be removed from userdeviceid_token by the insert later + } + + // Assign token to user device combination + let mut userdeviceid = user_id.as_bytes().to_vec(); + userdeviceid.push(0xFF); + userdeviceid.extend_from_slice(device_id.as_bytes()); + self.db + .userdeviceid_token + .insert(&userdeviceid, token.as_bytes()); + self.db + .token_userdeviceid + .insert(token.as_bytes(), &userdeviceid); + + Ok(()) + } + + pub async fn add_one_time_key( + &self, user_id: &UserId, device_id: &DeviceId, one_time_key_key: &DeviceKeyId, + one_time_key_value: &Raw, + ) -> Result<()> { + // All devices have metadata + // Only existing devices should be able to call this, but we shouldn't assert + // either... + let key = (user_id, device_id); + if self.db.userdeviceid_metadata.qry(&key).await.is_err() { + return Err!(Database(error!( + ?user_id, + ?device_id, + "User does not exist or device has no metadata." + ))); + } + + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(device_id.as_bytes()); + key.push(0xFF); + // TODO: Use DeviceKeyId::to_string when it's available (and update everything, + // because there are no wrapping quotation marks anymore) + key.extend_from_slice( + serde_json::to_string(one_time_key_key) + .expect("DeviceKeyId::to_string always works") + .as_bytes(), + ); + + self.db.onetimekeyid_onetimekeys.insert( + &key, + &serde_json::to_vec(&one_time_key_value).expect("OneTimeKey::to_vec always works"), + ); + + self.db + .userid_lastonetimekeyupdate + .insert(user_id.as_bytes(), &self.services.globals.next_count()?.to_be_bytes()); + + Ok(()) + } + + pub async fn last_one_time_keys_update(&self, user_id: &UserId) -> u64 { + self.db + .userid_lastonetimekeyupdate + .qry(user_id) + .await + .deserialized() + .unwrap_or(0) + } + + pub async fn take_one_time_key( + &self, user_id: &UserId, device_id: &DeviceId, key_algorithm: &DeviceKeyAlgorithm, + ) -> Result<(OwnedDeviceKeyId, Raw)> { + self.db + .userid_lastonetimekeyupdate + .insert(user_id.as_bytes(), &self.services.globals.next_count()?.to_be_bytes()); + + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xFF); + prefix.extend_from_slice(device_id.as_bytes()); + prefix.push(0xFF); + prefix.push(b'"'); // Annoying quotation mark + prefix.extend_from_slice(key_algorithm.as_ref().as_bytes()); + prefix.push(b':'); + + let one_time_key = self + .db + .onetimekeyid_onetimekeys + .raw_stream_prefix(&prefix) + .ignore_err() + .map(|(key, val)| { + self.db.onetimekeyid_onetimekeys.remove(key); + + let key = key + .rsplit(|&b| b == 0xFF) + .next() + .ok_or_else(|| err!(Database("OneTimeKeyId in db is invalid."))) + .unwrap(); + + let key = serde_json::from_slice(key) + .map_err(|e| err!(Database("OneTimeKeyId in db is invalid. {e}"))) + .unwrap(); + + let val = serde_json::from_slice(val) + .map_err(|e| err!(Database("OneTimeKeys in db are invalid. {e}"))) + .unwrap(); + + (key, val) + }) + .next() + .await; + + one_time_key.ok_or_else(|| err!(Request(NotFound("No one-time-key found")))) + } + + pub async fn count_one_time_keys( + &self, user_id: &UserId, device_id: &DeviceId, + ) -> BTreeMap { + type KeyVal<'a> = ((Ignore, Ignore, &'a Unquoted), Ignore); + + let mut algorithm_counts = BTreeMap::::new(); + let query = (user_id, device_id); + self.db + .onetimekeyid_onetimekeys + .stream_prefix(&query) + .ignore_err() + .ready_for_each(|((Ignore, Ignore, device_key_id), Ignore): KeyVal<'_>| { + let device_key_id: &DeviceKeyId = device_key_id + .as_str() + .try_into() + .expect("Invalid DeviceKeyID in database"); + + let count: &mut UInt = algorithm_counts + .entry(device_key_id.algorithm()) + .or_default(); + + *count = count.saturating_add(1_u32.into()); + }) + .await; + + algorithm_counts + } + + pub async fn add_device_keys(&self, user_id: &UserId, device_id: &DeviceId, device_keys: &Raw) { + let mut userdeviceid = user_id.as_bytes().to_vec(); + userdeviceid.push(0xFF); + userdeviceid.extend_from_slice(device_id.as_bytes()); + + self.db.keyid_key.insert( + &userdeviceid, + &serde_json::to_vec(&device_keys).expect("DeviceKeys::to_vec always works"), + ); + + self.mark_device_key_update(user_id).await; + } + + pub async fn add_cross_signing_keys( + &self, user_id: &UserId, master_key: &Raw, self_signing_key: &Option>, + user_signing_key: &Option>, notify: bool, + ) -> Result<()> { + // TODO: Check signatures + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xFF); + + let (master_key_key, _) = parse_master_key(user_id, master_key)?; + + self.db + .keyid_key + .insert(&master_key_key, master_key.json().get().as_bytes()); + + self.db + .userid_masterkeyid + .insert(user_id.as_bytes(), &master_key_key); + + // Self-signing key + if let Some(self_signing_key) = self_signing_key { + let mut self_signing_key_ids = self_signing_key + .deserialize() + .map_err(|e| err!(Request(InvalidParam("Invalid self signing key: {e:?}"))))? + .keys + .into_values(); + + let self_signing_key_id = self_signing_key_ids + .next() + .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Self signing key contained no key."))?; + + if self_signing_key_ids.next().is_some() { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Self signing key contained more than one key.", + )); + } + + let mut self_signing_key_key = prefix.clone(); + self_signing_key_key.extend_from_slice(self_signing_key_id.as_bytes()); + + self.db + .keyid_key + .insert(&self_signing_key_key, self_signing_key.json().get().as_bytes()); + + self.db + .userid_selfsigningkeyid + .insert(user_id.as_bytes(), &self_signing_key_key); + } + + // User-signing key + if let Some(user_signing_key) = user_signing_key { + let mut user_signing_key_ids = user_signing_key + .deserialize() + .map_err(|_| err!(Request(InvalidParam("Invalid user signing key"))))? + .keys + .into_values(); + + let user_signing_key_id = user_signing_key_ids + .next() + .ok_or(err!(Request(InvalidParam("User signing key contained no key."))))?; + + if user_signing_key_ids.next().is_some() { + return Err!(Request(InvalidParam("User signing key contained more than one key."))); + } + + let mut user_signing_key_key = prefix; + user_signing_key_key.extend_from_slice(user_signing_key_id.as_bytes()); + + self.db + .keyid_key + .insert(&user_signing_key_key, user_signing_key.json().get().as_bytes()); + + self.db + .userid_usersigningkeyid + .insert(user_id.as_bytes(), &user_signing_key_key); + } + + if notify { + self.mark_device_key_update(user_id).await; + } + + Ok(()) + } + + pub async fn sign_key( + &self, target_id: &UserId, key_id: &str, signature: (String, String), sender_id: &UserId, + ) -> Result<()> { + let key = (target_id, key_id); + + let mut cross_signing_key: serde_json::Value = self + .db + .keyid_key + .qry(&key) + .await + .map_err(|_| err!(Request(InvalidParam("Tried to sign nonexistent key."))))? + .deserialized_json() + .map_err(|e| err!(Database("key in keyid_key is invalid. {e:?}")))?; + + let signatures = cross_signing_key + .get_mut("signatures") + .ok_or_else(|| err!(Database("key in keyid_key has no signatures field.")))? + .as_object_mut() + .ok_or_else(|| err!(Database("key in keyid_key has invalid signatures field.")))? + .entry(sender_id.to_string()) + .or_insert_with(|| serde_json::Map::new().into()); + + signatures + .as_object_mut() + .ok_or_else(|| err!(Database("signatures in keyid_key for a user is invalid.")))? + .insert(signature.0, signature.1.into()); + + let mut key = target_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(key_id.as_bytes()); + self.db.keyid_key.insert( + &key, + &serde_json::to_vec(&cross_signing_key).expect("CrossSigningKey::to_vec always works"), + ); + + self.mark_device_key_update(target_id).await; + + Ok(()) + } + + pub fn keys_changed<'a>( + &'a self, user_or_room_id: &'a str, from: u64, to: Option, + ) -> impl Stream + Send + 'a { + type KeyVal<'a> = ((&'a str, u64), &'a UserId); + + let to = to.unwrap_or(u64::MAX); + let start = (user_or_room_id, from.saturating_add(1)); + self.db + .keychangeid_userid + .stream_from(&start) + .ignore_err() + .ready_take_while(move |((prefix, count), _): &KeyVal<'_>| *prefix == user_or_room_id && *count <= to) + .map(|((..), user_id): KeyVal<'_>| user_id) + } + + pub async fn mark_device_key_update(&self, user_id: &UserId) { + let count = self.services.globals.next_count().unwrap().to_be_bytes(); + let rooms_joined = self.services.state_cache.rooms_joined(user_id); + pin_mut!(rooms_joined); + while let Some(room_id) = rooms_joined.next().await { + // Don't send key updates to unencrypted rooms + if self + .services + .state_accessor + .room_state_get(room_id, &StateEventType::RoomEncryption, "") + .await + .is_err() + { + continue; + } + + let mut key = room_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(&count); + + self.db.keychangeid_userid.insert(&key, user_id.as_bytes()); + } + + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(&count); + self.db.keychangeid_userid.insert(&key, user_id.as_bytes()); + } + + pub async fn get_device_keys<'a>(&'a self, user_id: &'a UserId, device_id: &DeviceId) -> Result> { + let key_id = (user_id, device_id); + self.db.keyid_key.qry(&key_id).await.deserialized_json() + } + + pub async fn get_key( + &self, key_id: &[u8], sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &F, + ) -> Result> + where + F: Fn(&UserId) -> bool + Send + Sync, + { + let key = self + .db + .keyid_key + .qry(key_id) + .await + .deserialized_json::()?; + + let cleaned = clean_signatures(key, sender_user, user_id, allowed_signatures)?; + let raw_value = serde_json::value::to_raw_value(&cleaned)?; + Ok(Raw::from_json(raw_value)) + } + + pub async fn get_master_key( + &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &F, + ) -> Result> + where + F: Fn(&UserId) -> bool + Send + Sync, + { + let key_id = self.db.userid_masterkeyid.qry(user_id).await?; + + self.get_key(&key_id, sender_user, user_id, allowed_signatures) + .await + } + + pub async fn get_self_signing_key( + &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &F, + ) -> Result> + where + F: Fn(&UserId) -> bool + Send + Sync, + { + let key_id = self.db.userid_selfsigningkeyid.qry(user_id).await?; + + self.get_key(&key_id, sender_user, user_id, allowed_signatures) + .await + } + + pub async fn get_user_signing_key(&self, user_id: &UserId) -> Result> { + let key_id = self.db.userid_usersigningkeyid.qry(user_id).await?; + + self.db.keyid_key.qry(&*key_id).await.deserialized_json() + } + + pub async fn add_to_device_event( + &self, sender: &UserId, target_user_id: &UserId, target_device_id: &DeviceId, event_type: &str, + content: serde_json::Value, + ) { + let mut key = target_user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(target_device_id.as_bytes()); + key.push(0xFF); + key.extend_from_slice(&self.services.globals.next_count().unwrap().to_be_bytes()); + + let mut json = serde_json::Map::new(); + json.insert("type".to_owned(), event_type.to_owned().into()); + json.insert("sender".to_owned(), sender.to_string().into()); + json.insert("content".to_owned(), content); + + let value = serde_json::to_vec(&json).expect("Map::to_vec always works"); + + self.db.todeviceid_events.insert(&key, &value); + } + + pub fn get_to_device_events<'a>( + &'a self, user_id: &'a UserId, device_id: &'a DeviceId, + ) -> impl Stream> + Send + 'a { + let prefix = (user_id, device_id, Interfix); + self.db + .todeviceid_events + .stream_raw_prefix(&prefix) + .ready_and_then(|(_, val)| serde_json::from_slice(val).map_err(Into::into)) + .ignore_err() + } + + pub async fn remove_to_device_events(&self, user_id: &UserId, device_id: &DeviceId, until: u64) { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xFF); + prefix.extend_from_slice(device_id.as_bytes()); + prefix.push(0xFF); + + let mut last = prefix.clone(); + last.extend_from_slice(&until.to_be_bytes()); + + self.db + .todeviceid_events + .rev_raw_keys_from(&last) // this includes last + .ignore_err() + .ready_take_while(move |key| key.starts_with(&prefix)) + .map(|key| { + let len = key.len(); + let start = len.saturating_sub(size_of::()); + let count = utils::u64_from_u8(&key[start..len]); + (key, count) + }) + .ready_take_while(move |(_, count)| *count <= until) + .ready_for_each(|(key, _)| self.db.todeviceid_events.remove(&key)) + .boxed() + .await; + } + + pub async fn update_device_metadata(&self, user_id: &UserId, device_id: &DeviceId, device: &Device) -> Result<()> { + // Only existing devices should be able to call this, but we shouldn't assert + // either... + let key = (user_id, device_id); + if self.db.userdeviceid_metadata.qry(&key).await.is_err() { + return Err!(Database(error!( + ?user_id, + ?device_id, + "Called update_device_metadata for a non-existent user and/or device" + ))); + } + + increment(&self.db.userid_devicelistversion, user_id.as_bytes()); + + let mut userdeviceid = user_id.as_bytes().to_vec(); + userdeviceid.push(0xFF); + userdeviceid.extend_from_slice(device_id.as_bytes()); + self.db.userdeviceid_metadata.insert( + &userdeviceid, + &serde_json::to_vec(device).expect("Device::to_string always works"), + ); + + Ok(()) + } + + /// Get device metadata. + pub async fn get_device_metadata(&self, user_id: &UserId, device_id: &DeviceId) -> Result { + self.db + .userdeviceid_metadata + .qry(&(user_id, device_id)) + .await + .deserialized_json() + } + + pub async fn get_devicelist_version(&self, user_id: &UserId) -> Result { + self.db + .userid_devicelistversion + .qry(user_id) + .await + .deserialized() + } + + pub fn all_devices_metadata<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + self.db + .userdeviceid_metadata + .stream_raw_prefix(&(user_id, Interfix)) + .ready_and_then(|(_, val)| serde_json::from_slice::(val).map_err(Into::into)) + .ignore_err() + } + + /// Creates a new sync filter. Returns the filter id. + pub fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> String { + let filter_id = utils::random_string(4); + + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(filter_id.as_bytes()); + + self.db + .userfilterid_filter + .insert(&key, &serde_json::to_vec(&filter).expect("filter is valid json")); + + filter_id + } + + pub async fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result { + self.db + .userfilterid_filter + .qry(&(user_id, filter_id)) + .await + .deserialized_json() } /// Creates an OpenID token, which can be used to prove that a user has /// access to an account (primarily for integrations) pub fn create_openid_token(&self, user_id: &UserId, token: &str) -> Result { - self.db.create_openid_token(user_id, token) + use std::num::Saturating as Sat; + + let expires_in = self.services.server.config.openid_token_ttl; + let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in) * Sat(1000); + + let mut value = expires_at.0.to_be_bytes().to_vec(); + value.extend_from_slice(user_id.as_bytes()); + + self.db + .openidtoken_expiresatuserid + .insert(token.as_bytes(), value.as_slice()); + + Ok(expires_in) } /// Find out which user an OpenID access token belongs to. - pub fn find_from_openid_token(&self, token: &str) -> Result { self.db.find_from_openid_token(token) } + pub async fn find_from_openid_token(&self, token: &str) -> Result { + let Ok(value) = self.db.openidtoken_expiresatuserid.qry(token).await else { + return Err!(Request(Unauthorized("OpenID token is unrecognised"))); + }; + + let (expires_at_bytes, user_bytes) = value.split_at(0_u64.to_be_bytes().len()); + let expires_at = u64::from_be_bytes( + expires_at_bytes + .try_into() + .map_err(|e| err!(Database("expires_at in openid_userid is invalid u64. {e}")))?, + ); + + if expires_at < utils::millis_since_unix_epoch() { + debug_warn!("OpenID token is expired, removing"); + self.db.openidtoken_expiresatuserid.remove(token.as_bytes()); + + return Err!(Request(Unauthorized("OpenID token is expired"))); + } + + let user_string = utils::string_from_bytes(user_bytes) + .map_err(|e| err!(Database("User ID in openid_userid is invalid unicode. {e}")))?; + + UserId::parse(user_string).map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) + } + + /// Gets a specific user profile key + pub async fn profile_key(&self, user_id: &UserId, profile_key: &str) -> Result { + let key = (user_id, profile_key); + self.db + .useridprofilekey_value + .qry(&key) + .await + .deserialized() + } + + /// Gets all the user's profile keys and values in an iterator + pub fn all_profile_keys<'a>( + &'a self, user_id: &'a UserId, + ) -> impl Stream + 'a + Send { + type KeyVal = ((Ignore, String), serde_json::Value); + + let prefix = (user_id, Interfix); + self.db + .useridprofilekey_value + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, key), val): KeyVal| (key, val)) + } + + /// Sets a new profile key value, removes the key if value is None + pub fn set_profile_key(&self, user_id: &UserId, profile_key: &str, profile_key_value: Option) { + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(profile_key.as_bytes()); + + // TODO: insert to the stable MSC4175 key when it's stable + if let Some(value) = profile_key_value { + let value = serde_json::to_vec(&value).unwrap(); + + self.db.useridprofilekey_value.insert(&key, &value); + } else { + self.db.useridprofilekey_value.remove(&key); + } + } + + /// Get the timezone of a user. + pub async fn timezone(&self, user_id: &UserId) -> Result { + // TODO: transparently migrate unstable key usage to the stable key once MSC4133 + // and MSC4175 are stable, likely a remove/insert in this block. + + // first check the unstable prefix then check the stable prefix + let unstable_key = (user_id, "us.cloke.msc4175.tz"); + let stable_key = (user_id, "m.tz"); + self.db + .useridprofilekey_value + .qry(&unstable_key) + .or_else(|_| self.db.useridprofilekey_value.qry(&stable_key)) + .await + .deserialized() + } + + /// Sets a new timezone or removes it if timezone is None. + pub fn set_timezone(&self, user_id: &UserId, timezone: Option) { + let mut key = user_id.as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(b"us.cloke.msc4175.tz"); + + // TODO: insert to the stable MSC4175 key when it's stable + if let Some(timezone) = timezone { + self.db + .useridprofilekey_value + .insert(&key, timezone.as_bytes()); + } else { + self.db.useridprofilekey_value.remove(&key); + } + } +} + +pub fn parse_master_key(user_id: &UserId, master_key: &Raw) -> Result<(Vec, CrossSigningKey)> { + let mut prefix = user_id.as_bytes().to_vec(); + prefix.push(0xFF); + + let master_key = master_key + .deserialize() + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid master key"))?; + let mut master_key_ids = master_key.keys.values(); + let master_key_id = master_key_ids + .next() + .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Master key contained no key."))?; + if master_key_ids.next().is_some() { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Master key contained more than one key.", + )); + } + let mut master_key_key = prefix.clone(); + master_key_key.extend_from_slice(master_key_id.as_bytes()); + Ok((master_key_key, master_key)) } /// Ensure that a user only sees signatures from themselves and the target user -pub fn clean_signatures bool>( - cross_signing_key: &mut serde_json::Value, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: F, -) -> Result<(), Error> { +fn clean_signatures( + mut cross_signing_key: serde_json::Value, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &F, +) -> Result +where + F: Fn(&UserId) -> bool + Send + Sync, +{ if let Some(signatures) = cross_signing_key .get_mut("signatures") .and_then(|v| v.as_object_mut()) @@ -563,5 +995,12 @@ pub fn clean_signatures bool>( } } - Ok(()) + Ok(cross_signing_key) +} + +//TODO: this is an ABA +fn increment(db: &Arc, key: &[u8]) { + let old = db.get(key); + let new = utils::increment(old.ok().as_deref()); + db.insert(key, &new); } From 4776fe66c4a9d5cbb0153e8ff23009d21ed5010e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 28 Sep 2024 15:14:48 +0000 Subject: [PATCH 0845/2091] handle serde_json for deserialized() Signed-off-by: Jason Volk --- src/database/de.rs | 94 ++++++++++++++++++++------- src/database/deserialized.rs | 14 ---- src/database/handle.rs | 28 -------- src/service/account_data/mod.rs | 2 +- src/service/appservice/data.rs | 2 +- src/service/globals/data.rs | 5 +- src/service/key_backups/mod.rs | 12 +--- src/service/pusher/mod.rs | 2 +- src/service/rooms/outlier/mod.rs | 4 +- src/service/rooms/state_cache/data.rs | 10 +-- src/service/rooms/timeline/data.rs | 15 ++--- src/service/uiaa/mod.rs | 2 +- src/service/users/mod.rs | 12 ++-- 13 files changed, 95 insertions(+), 107 deletions(-) diff --git a/src/database/de.rs b/src/database/de.rs index 8ce25aa3..a5d2c127 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -58,10 +58,15 @@ impl<'de> Deserializer<'de> { } #[inline] - fn record_trail(&mut self) -> &'de [u8] { - let record = &self.buf[self.pos..]; - self.inc_pos(record.len()); - record + fn record_next_peek_byte(&self) -> Option { + let started = self.pos != 0; + let buf = &self.buf[self.pos..]; + debug_assert!( + !started || buf[0] == Self::SEP, + "Missing expected record separator at current position" + ); + + buf.get::(started.into()).copied() } #[inline] @@ -75,6 +80,13 @@ impl<'de> Deserializer<'de> { self.inc_pos(started.into()); } + #[inline] + fn record_trail(&mut self) -> &'de [u8] { + let record = &self.buf[self.pos..]; + self.inc_pos(record.len()); + record + } + #[inline] fn inc_pos(&mut self, n: usize) { self.pos = self.pos.saturating_add(n); @@ -85,13 +97,6 @@ impl<'de> Deserializer<'de> { impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { type Error = Error; - fn deserialize_map(self, _visitor: V) -> Result - where - V: Visitor<'de>, - { - unimplemented!("deserialize Map not implemented") - } - fn deserialize_seq(self, visitor: V) -> Result where V: Visitor<'de>, @@ -113,13 +118,23 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { visitor.visit_seq(self) } - fn deserialize_struct( - self, _name: &'static str, _fields: &'static [&'static str], _visitor: V, - ) -> Result + fn deserialize_map(self, visitor: V) -> Result where V: Visitor<'de>, { - unimplemented!("deserialize Struct not implemented") + let input = self.record_next(); + let mut d = serde_json::Deserializer::from_slice(input); + d.deserialize_map(visitor).map_err(Into::into) + } + + fn deserialize_struct(self, name: &'static str, fields: &'static [&'static str], visitor: V) -> Result + where + V: Visitor<'de>, + { + let input = self.record_next(); + let mut d = serde_json::Deserializer::from_slice(input); + d.deserialize_struct(name, fields, visitor) + .map_err(Into::into) } fn deserialize_unit_struct(self, name: &'static str, visitor: V) -> Result @@ -134,11 +149,14 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { visitor.visit_unit() } - fn deserialize_newtype_struct(self, _name: &'static str, _visitor: V) -> Result + fn deserialize_newtype_struct(self, name: &'static str, visitor: V) -> Result where V: Visitor<'de>, { - unimplemented!("deserialize Newtype Struct not implemented") + match name { + "$serde_json::private::RawValue" => visitor.visit_map(self), + _ => visitor.visit_newtype_struct(self), + } } fn deserialize_enum( @@ -228,19 +246,31 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } fn deserialize_unit>(self, _visitor: V) -> Result { - unimplemented!("deserialize Unit Struct not implemented") + unimplemented!("deserialize Unit not implemented") } - fn deserialize_identifier>(self, _visitor: V) -> Result { - unimplemented!("deserialize Identifier not implemented") + // this only used for $serde_json::private::RawValue at this time; see MapAccess + fn deserialize_identifier>(self, visitor: V) -> Result { + let input = "$serde_json::private::RawValue"; + visitor.visit_borrowed_str(input) } fn deserialize_ignored_any>(self, _visitor: V) -> Result { unimplemented!("deserialize Ignored Any not implemented") } - fn deserialize_any>(self, _visitor: V) -> Result { - unimplemented!("deserialize any not implemented") + fn deserialize_any>(self, visitor: V) -> Result { + debug_assert_eq!( + conduit::debug::type_name::(), + "serde_json::value::de::::deserialize::ValueVisitor", + "deserialize_any: type not expected" + ); + + match self.record_next_peek_byte() { + Some(b'{') => self.deserialize_map(visitor), + _ => self.deserialize_str(visitor), + } } } @@ -259,3 +289,23 @@ impl<'a, 'de: 'a> de::SeqAccess<'de> for &'a mut Deserializer<'de> { seed.deserialize(&mut **self).map(Some) } } + +// this only used for $serde_json::private::RawValue at this time. our db +// schema doesn't have its own map format; we use json for that anyway +impl<'a, 'de: 'a> de::MapAccess<'de> for &'a mut Deserializer<'de> { + type Error = Error; + + fn next_key_seed(&mut self, seed: K) -> Result> + where + K: DeserializeSeed<'de>, + { + seed.deserialize(&mut **self).map(Some) + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: DeserializeSeed<'de>, + { + seed.deserialize(&mut **self) + } +} diff --git a/src/database/deserialized.rs b/src/database/deserialized.rs index 7da112d5..a59b2ce5 100644 --- a/src/database/deserialized.rs +++ b/src/database/deserialized.rs @@ -9,11 +9,6 @@ pub trait Deserialized { F: FnOnce(T) -> U, T: for<'de> Deserialize<'de>; - fn map_json(self, f: F) -> Result - where - F: FnOnce(T) -> U, - T: for<'de> Deserialize<'de>; - #[inline] fn deserialized(self) -> Result where @@ -22,13 +17,4 @@ pub trait Deserialized { { self.map_de(identity::) } - - #[inline] - fn deserialized_json(self) -> Result - where - T: for<'de> Deserialize<'de>, - Self: Sized, - { - self.map_json(identity::) - } } diff --git a/src/database/handle.rs b/src/database/handle.rs index 89d87137..0d4bd02e 100644 --- a/src/database/handle.rs +++ b/src/database/handle.rs @@ -48,15 +48,6 @@ impl AsRef for Handle<'_> { } impl Deserialized for Result> { - #[inline] - fn map_json(self, f: F) -> Result - where - F: FnOnce(T) -> U, - T: for<'de> Deserialize<'de>, - { - self?.map_json(f) - } - #[inline] fn map_de(self, f: F) -> Result where @@ -68,15 +59,6 @@ impl Deserialized for Result> { } impl<'a> Deserialized for Result<&'a Handle<'a>> { - #[inline] - fn map_json(self, f: F) -> Result - where - F: FnOnce(T) -> U, - T: for<'de> Deserialize<'de>, - { - self.and_then(|handle| handle.map_json(f)) - } - #[inline] fn map_de(self, f: F) -> Result where @@ -88,16 +70,6 @@ impl<'a> Deserialized for Result<&'a Handle<'a>> { } impl<'a> Deserialized for &'a Handle<'a> { - fn map_json(self, f: F) -> Result - where - F: FnOnce(T) -> U, - T: for<'de> Deserialize<'de>, - { - serde_json::from_slice::(self.as_ref()) - .map_err(Into::into) - .map(f) - } - fn map_de(self, f: F) -> Result where F: FnOnce(T) -> U, diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index b4eb143d..4f00cff1 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -108,7 +108,7 @@ pub async fn get( .qry(&key) .and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.qry(&roomuserdataid)) .await - .deserialized_json() + .deserialized() } /// Returns all changes to the account data that happened after `since`. diff --git a/src/service/appservice/data.rs b/src/service/appservice/data.rs index d5fa5476..f31c5e63 100644 --- a/src/service/appservice/data.rs +++ b/src/service/appservice/data.rs @@ -40,7 +40,7 @@ impl Data { self.id_appserviceregistrations .qry(id) .await - .deserialized_json() + .deserialized() .map_err(|e| err!(Database("Invalid appservice {id:?} registration: {e:?}"))) } diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 3286e40c..76f97944 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -305,10 +305,7 @@ impl Data { } pub async fn signing_keys_for(&self, origin: &ServerName) -> Result { - self.server_signingkeys - .qry(origin) - .await - .deserialized_json() + self.server_signingkeys.qry(origin).await.deserialized() } pub async fn database_version(&self) -> u64 { self.global.qry("version").await.deserialized().unwrap_or(0) } diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 12712e79..decf32f7 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -166,11 +166,7 @@ pub async fn get_latest_backup(&self, user_id: &UserId) -> Result<(String, Raw Result> { let key = (user_id, version); - self.db - .backupid_algorithm - .qry(&key) - .await - .deserialized_json() + self.db.backupid_algorithm.qry(&key).await.deserialized() } #[implement(Service)] @@ -278,11 +274,7 @@ pub async fn get_session( ) -> Result> { let key = (user_id, version, room_id, session_id); - self.db - .backupkeyid_backup - .qry(&key) - .await - .deserialized_json() + self.db.backupkeyid_backup.qry(&key).await.deserialized() } #[implement(Service)] diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 44ff1945..8d8b553f 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -90,7 +90,7 @@ impl Service { .senderkey_pusher .qry(&senderkey) .await - .deserialized_json() + .deserialized() } pub async fn get_pushers(&self, sender: &UserId) -> Vec { diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 277b5982..4c9225ae 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -33,7 +33,7 @@ pub async fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result Result { .eventid_outlierpdu .qry(event_id) .await - .deserialized_json() + .deserialized() } /// Append the PDU as an outlier. diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 38e504f6..f3ccaf10 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -156,10 +156,7 @@ impl Data { &self, user_id: &UserId, room_id: &RoomId, ) -> Result>> { let key = (user_id, room_id); - self.userroomid_invitestate - .qry(&key) - .await - .deserialized_json() + self.userroomid_invitestate.qry(&key).await.deserialized() } #[tracing::instrument(skip(self), level = "debug")] @@ -167,10 +164,7 @@ impl Data { &self, user_id: &UserId, room_id: &RoomId, ) -> Result>> { let key = (user_id, room_id); - self.userroomid_leftstate - .qry(&key) - .await - .deserialized_json() + self.userroomid_leftstate.qry(&key).await.deserialized() } /// Returns an iterator over all rooms a user left. diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index cd746be4..314dcb9f 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -90,17 +90,14 @@ impl Data { return Ok(pdu); } - self.eventid_outlierpdu - .qry(event_id) - .await - .deserialized_json() + self.eventid_outlierpdu.qry(event_id).await.deserialized() } /// Returns the json of a pdu. pub(super) async fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result { let pduid = self.get_pdu_id(event_id).await?; - self.pduid_pdu.qry(&pduid).await.deserialized_json() + self.pduid_pdu.qry(&pduid).await.deserialized() } /// Returns the pdu's id. @@ -113,7 +110,7 @@ impl Data { pub(super) async fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result { let pduid = self.get_pdu_id(event_id).await?; - self.pduid_pdu.qry(&pduid).await.deserialized_json() + self.pduid_pdu.qry(&pduid).await.deserialized() } /// Like get_non_outlier_pdu(), but without the expense of fetching and @@ -137,7 +134,7 @@ impl Data { self.eventid_outlierpdu .qry(event_id) .await - .deserialized_json() + .deserialized() .map(Arc::new) } @@ -162,12 +159,12 @@ impl Data { /// /// This does __NOT__ check the outliers `Tree`. pub(super) async fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result { - self.pduid_pdu.qry(pdu_id).await.deserialized_json() + self.pduid_pdu.qry(pdu_id).await.deserialized() } /// Returns the pdu as a `BTreeMap`. pub(super) async fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result { - self.pduid_pdu.qry(pdu_id).await.deserialized_json() + self.pduid_pdu.qry(pdu_id).await.deserialized() } pub(super) async fn append_pdu(&self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64) { diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 7e231514..0415bfc2 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -238,6 +238,6 @@ async fn get_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session .userdevicesessionid_uiaainfo .qry(&key) .await - .deserialized_json() + .deserialized() .map_err(|_| err!(Request(Forbidden("UIAA session does not exist.")))) } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 9a058ba9..ca37ed9d 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -577,7 +577,7 @@ impl Service { .qry(&key) .await .map_err(|_| err!(Request(InvalidParam("Tried to sign nonexistent key."))))? - .deserialized_json() + .deserialized() .map_err(|e| err!(Database("key in keyid_key is invalid. {e:?}")))?; let signatures = cross_signing_key @@ -652,7 +652,7 @@ impl Service { pub async fn get_device_keys<'a>(&'a self, user_id: &'a UserId, device_id: &DeviceId) -> Result> { let key_id = (user_id, device_id); - self.db.keyid_key.qry(&key_id).await.deserialized_json() + self.db.keyid_key.qry(&key_id).await.deserialized() } pub async fn get_key( @@ -666,7 +666,7 @@ impl Service { .keyid_key .qry(key_id) .await - .deserialized_json::()?; + .deserialized::()?; let cleaned = clean_signatures(key, sender_user, user_id, allowed_signatures)?; let raw_value = serde_json::value::to_raw_value(&cleaned)?; @@ -700,7 +700,7 @@ impl Service { pub async fn get_user_signing_key(&self, user_id: &UserId) -> Result> { let key_id = self.db.userid_usersigningkeyid.qry(user_id).await?; - self.db.keyid_key.qry(&*key_id).await.deserialized_json() + self.db.keyid_key.qry(&*key_id).await.deserialized() } pub async fn add_to_device_event( @@ -791,7 +791,7 @@ impl Service { .userdeviceid_metadata .qry(&(user_id, device_id)) .await - .deserialized_json() + .deserialized() } pub async fn get_devicelist_version(&self, user_id: &UserId) -> Result { @@ -830,7 +830,7 @@ impl Service { .userfilterid_filter .qry(&(user_id, filter_id)) .await - .deserialized_json() + .deserialized() } /// Creates an OpenID token, which can be used to prove that a user has From 3f7ec4221d89767e5bf0ff3e2a64c847a8dce264 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 25 Sep 2024 03:52:28 +0000 Subject: [PATCH 0846/2091] minor auth_chain optimizations/cleanup Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 47 ++++++++-------- src/api/server/event_auth.rs | 4 +- src/api/server/send_join.rs | 7 ++- src/api/server/state.rs | 4 +- src/api/server/state_ids.rs | 4 +- src/service/rooms/auth_chain/data.rs | 77 ++++++++++++++------------ src/service/rooms/auth_chain/mod.rs | 56 +++++++++---------- src/service/rooms/event_handler/mod.rs | 44 ++++++++------- 8 files changed, 125 insertions(+), 118 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 65c9bc71..350e08c6 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -27,33 +27,32 @@ pub(super) async fn echo(&self, message: Vec) -> Result) -> Result { - let event_id = Arc::::from(event_id); - if let Ok(event) = self.services.rooms.timeline.get_pdu_json(&event_id).await { - let room_id_str = event - .get("room_id") - .and_then(|val| val.as_str()) - .ok_or_else(|| Error::bad_database("Invalid event in database"))?; + let Ok(event) = self.services.rooms.timeline.get_pdu_json(&event_id).await else { + return Ok(RoomMessageEventContent::notice_plain("Event not found.")); + }; - let room_id = <&RoomId>::try_from(room_id_str) - .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; + let room_id_str = event + .get("room_id") + .and_then(|val| val.as_str()) + .ok_or_else(|| Error::bad_database("Invalid event in database"))?; - let start = Instant::now(); - let count = self - .services - .rooms - .auth_chain - .event_ids_iter(room_id, vec![event_id]) - .await? - .count() - .await; + let room_id = <&RoomId>::try_from(room_id_str) + .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; - let elapsed = start.elapsed(); - Ok(RoomMessageEventContent::text_plain(format!( - "Loaded auth chain with length {count} in {elapsed:?}" - ))) - } else { - Ok(RoomMessageEventContent::text_plain("Event not found.")) - } + let start = Instant::now(); + let count = self + .services + .rooms + .auth_chain + .event_ids_iter(room_id, &[&event_id]) + .await? + .count() + .await; + + let elapsed = start.elapsed(); + Ok(RoomMessageEventContent::text_plain(format!( + "Loaded auth chain with length {count} in {elapsed:?}" + ))) } #[admin_command] diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 6ec00b50..8307a4ad 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::borrow::Borrow; use axum::extract::State; use conduit::{Error, Result}; @@ -57,7 +57,7 @@ pub(crate) async fn get_event_authorization_route( let auth_chain = services .rooms .auth_chain - .event_ids_iter(room_id, vec![Arc::from(&*body.event_id)]) + .event_ids_iter(room_id, &[body.event_id.borrow()]) .await? .filter_map(|id| async move { services.rooms.timeline.get_pdu_json(&id).await.ok() }) .then(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 639fcafd..f9257690 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -1,6 +1,6 @@ #![allow(deprecated)] -use std::collections::BTreeMap; +use std::{borrow::Borrow, collections::BTreeMap}; use axum::extract::State; use conduit::{err, pdu::gen_event_id_canonical_json, utils::IterStream, warn, Error, Result}; @@ -11,7 +11,7 @@ use ruma::{ room::member::{MembershipState, RoomMemberEventContent}, StateEventType, }, - CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, + CanonicalJsonValue, EventId, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::Services; @@ -196,10 +196,11 @@ async fn create_join_event( .try_collect() .await?; + let starting_events: Vec<&EventId> = state_ids.values().map(Borrow::borrow).collect(); let auth_chain = services .rooms .auth_chain - .event_ids_iter(room_id, state_ids.values().cloned().collect()) + .event_ids_iter(room_id, &starting_events) .await? .map(Ok) .and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) diff --git a/src/api/server/state.rs b/src/api/server/state.rs index 37a14a3f..3a27cd0a 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::borrow::Borrow; use axum::extract::State; use conduit::{err, result::LogErr, utils::IterStream, Err, Result}; @@ -63,7 +63,7 @@ pub(crate) async fn get_room_state_route( let auth_chain = services .rooms .auth_chain - .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) + .event_ids_iter(&body.room_id, &[body.event_id.borrow()]) .await? .map(Ok) .and_then(|id| async move { services.rooms.timeline.get_pdu_json(&id).await }) diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index 95ca65aa..b026abf1 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::borrow::Borrow; use axum::extract::State; use conduit::{err, Err}; @@ -55,7 +55,7 @@ pub(crate) async fn get_room_state_ids_route( let auth_chain_ids = services .rooms .auth_chain - .event_ids_iter(&body.room_id, vec![Arc::from(&*body.event_id)]) + .event_ids_iter(&body.room_id, &[body.event_id.borrow()]) .await? .map(|id| (*id).to_owned()) .collect() diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index 3d00374e..5c9dbda8 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{utils, utils::math::usize_from_f64, Result}; +use conduit::{err, utils, utils::math::usize_from_f64, Err, Result}; use database::Map; use lru_cache::LruCache; @@ -24,54 +24,63 @@ impl Data { } } - pub(super) async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result>> { + pub(super) async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { + debug_assert!(!key.is_empty(), "auth_chain key must not be empty"); + // Check RAM cache - if let Some(result) = self.auth_chain_cache.lock().unwrap().get_mut(key) { - return Ok(Some(Arc::clone(result))); + if let Some(result) = self + .auth_chain_cache + .lock() + .expect("cache locked") + .get_mut(key) + { + return Ok(Arc::clone(result)); } // We only save auth chains for single events in the db - if key.len() == 1 { - // Check DB cache - let chain = self.shorteventid_authchain.qry(&key[0]).await.map(|chain| { - chain - .chunks_exact(size_of::()) - .map(utils::u64_from_u8) - .collect::>() - }); - - if let Ok(chain) = chain { - // Cache in RAM - self.auth_chain_cache - .lock() - .expect("locked") - .insert(vec![key[0]], Arc::clone(&chain)); - - return Ok(Some(chain)); - } + if key.len() != 1 { + return Err!(Request(NotFound("auth_chain not cached"))); } - Ok(None) + // Check database + let chain = self + .shorteventid_authchain + .qry(&key[0]) + .await + .map_err(|_| err!(Request(NotFound("auth_chain not found"))))?; + + let chain = chain + .chunks_exact(size_of::()) + .map(utils::u64_from_u8) + .collect::>(); + + // Cache in RAM + self.auth_chain_cache + .lock() + .expect("cache locked") + .insert(vec![key[0]], Arc::clone(&chain)); + + Ok(chain) } - pub(super) fn cache_auth_chain(&self, key: Vec, auth_chain: Arc<[u64]>) -> Result<()> { + pub(super) fn cache_auth_chain(&self, key: Vec, auth_chain: Arc<[u64]>) { + debug_assert!(!key.is_empty(), "auth_chain key must not be empty"); + // Only persist single events in db if key.len() == 1 { - self.shorteventid_authchain.insert( - &key[0].to_be_bytes(), - &auth_chain - .iter() - .flat_map(|s| s.to_be_bytes().to_vec()) - .collect::>(), - ); + let key = key[0].to_be_bytes(); + let val = auth_chain + .iter() + .flat_map(|s| s.to_be_bytes().to_vec()) + .collect::>(); + + self.shorteventid_authchain.insert(&key, &val); } // Cache in RAM self.auth_chain_cache .lock() - .expect("locked") + .expect("cache locked") .insert(key, auth_chain); - - Ok(()) } } diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 7bc239d7..eae13b74 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -37,25 +37,18 @@ impl crate::Service for Service { } impl Service { - pub async fn event_ids_iter<'a>( - &'a self, room_id: &RoomId, starting_events_: Vec>, - ) -> Result> + Send + 'a> { - let mut starting_events: Vec<&EventId> = Vec::with_capacity(starting_events_.len()); - for starting_event in &starting_events_ { - starting_events.push(starting_event); - } + pub async fn event_ids_iter( + &self, room_id: &RoomId, starting_events: &[&EventId], + ) -> Result> + Send + '_> { + let chain = self.get_auth_chain(room_id, starting_events).await?; + let iter = chain.into_iter().stream().filter_map(|sid| { + self.services + .short + .get_eventid_from_short(sid) + .map(Result::ok) + }); - Ok(self - .get_auth_chain(room_id, &starting_events) - .await? - .into_iter() - .stream() - .filter_map(|sid| { - self.services - .short - .get_eventid_from_short(sid) - .map(Result::ok) - })) + Ok(iter) } #[tracing::instrument(skip_all, name = "auth_chain")] @@ -93,7 +86,7 @@ impl Service { } let chunk_key: Vec = chunk.iter().map(|(short, _)| short).copied().collect(); - if let Some(cached) = self.get_cached_eventid_authchain(&chunk_key).await? { + if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { trace!("Found cache entry for whole chunk"); full_auth_chain.extend(cached.iter().copied()); hits = hits.saturating_add(1); @@ -104,13 +97,13 @@ impl Service { let mut misses2: usize = 0; let mut chunk_cache = Vec::with_capacity(chunk.len()); for (sevent_id, event_id) in chunk { - if let Some(cached) = self.get_cached_eventid_authchain(&[sevent_id]).await? { + if let Ok(cached) = self.get_cached_eventid_authchain(&[sevent_id]).await { trace!(?event_id, "Found cache entry for event"); chunk_cache.extend(cached.iter().copied()); hits2 = hits2.saturating_add(1); } else { let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; - self.cache_auth_chain(vec![sevent_id], &auth_chain)?; + self.cache_auth_chain(vec![sevent_id], &auth_chain); chunk_cache.extend(auth_chain.iter()); misses2 = misses2.saturating_add(1); debug!( @@ -125,7 +118,7 @@ impl Service { chunk_cache.sort_unstable(); chunk_cache.dedup(); - self.cache_auth_chain_vec(chunk_key, &chunk_cache)?; + self.cache_auth_chain_vec(chunk_key, &chunk_cache); full_auth_chain.extend(chunk_cache.iter()); misses = misses.saturating_add(1); debug!( @@ -163,11 +156,11 @@ impl Service { Ok(pdu) => { if pdu.room_id != room_id { return Err!(Request(Forbidden( - "auth event {event_id:?} for incorrect room {} which is not {}", + "auth event {event_id:?} for incorrect room {} which is not {room_id}", pdu.room_id, - room_id ))); } + for auth_event in &pdu.auth_events { let sauthevent = self .services @@ -187,20 +180,21 @@ impl Service { Ok(found) } - pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result>> { + #[inline] + pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { self.db.get_cached_eventid_authchain(key).await } #[tracing::instrument(skip(self), level = "debug")] - pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) -> Result<()> { - self.db - .cache_auth_chain(key, auth_chain.iter().copied().collect::>()) + pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) { + let val = auth_chain.iter().copied().collect::>(); + self.db.cache_auth_chain(key, val); } #[tracing::instrument(skip(self), level = "debug")] - pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &Vec) -> Result<()> { - self.db - .cache_auth_chain(key, auth_chain.iter().copied().collect::>()) + pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &Vec) { + let val = auth_chain.iter().copied().collect::>(); + self.db.cache_auth_chain(key, val); } pub fn get_cache_usage(&self) -> (usize, usize) { diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 07d6e4db..57b87706 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1,6 +1,7 @@ mod parse_incoming_pdu; use std::{ + borrow::Borrow, collections::{hash_map, BTreeMap, HashMap, HashSet}, fmt::Write, sync::{Arc, RwLock as StdRwLock}, @@ -773,6 +774,7 @@ impl Service { Ok(pdu_id) } + #[tracing::instrument(skip_all, name = "resolve")] pub async fn resolve_state( &self, room_id: &RoomId, room_version_id: &RoomVersionId, incoming_state: HashMap>, ) -> Result>> { @@ -793,14 +795,17 @@ impl Service { let fork_states = [current_state_ids, incoming_state]; let mut auth_chain_sets = Vec::with_capacity(fork_states.len()); for state in &fork_states { - auth_chain_sets.push( - self.services - .auth_chain - .event_ids_iter(room_id, state.iter().map(|(_, id)| id.clone()).collect()) - .await? - .collect::>>() - .await, - ); + let starting_events: Vec<&EventId> = state.values().map(Borrow::borrow).collect(); + + let auth_chain = self + .services + .auth_chain + .event_ids_iter(room_id, &starting_events) + .await? + .collect::>>() + .await; + + auth_chain_sets.push(auth_chain); } debug!("Loading fork states"); @@ -962,12 +967,11 @@ impl Service { let mut state = StateMap::with_capacity(leaf_state.len()); let mut starting_events = Vec::with_capacity(leaf_state.len()); - - for (k, id) in leaf_state { + for (k, id) in &leaf_state { if let Ok((ty, st_key)) = self .services .short - .get_statekey_from_short(k) + .get_statekey_from_short(*k) .await .log_err() { @@ -976,18 +980,18 @@ impl Service { state.insert((ty.to_string().into(), st_key), id.clone()); } - starting_events.push(id); + starting_events.push(id.borrow()); } - auth_chain_sets.push( - self.services - .auth_chain - .event_ids_iter(room_id, starting_events) - .await? - .collect() - .await, - ); + let auth_chain = self + .services + .auth_chain + .event_ids_iter(room_id, &starting_events) + .await? + .collect() + .await; + auth_chain_sets.push(auth_chain); fork_states.push(state); } From 4496cf2d5b08780fd2d2b32c31c2c0b38bf010e7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 21 Sep 2024 16:28:46 -0400 Subject: [PATCH 0847/2091] add missing await to first admin room creation Signed-off-by: strawberry --- src/service/admin/create.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 7b090aa0..3dd5aea3 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -30,7 +30,11 @@ use crate::Services; pub async fn create_admin_room(services: &Services) -> Result<()> { let room_id = RoomId::new(services.globals.server_name()); - let _short_id = services.rooms.short.get_or_create_shortroomid(&room_id); + let _short_id = services + .rooms + .short + .get_or_create_shortroomid(&room_id) + .await; let state_lock = services.rooms.state.mutex.lock(&room_id).await; From 5192927a5342cffd9a7284bad3eb2c4b4819c674 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 29 Sep 2024 07:37:43 +0000 Subject: [PATCH 0848/2091] split remaining map suites Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 10 -- src/database/map.rs | 163 ++--------------------------- src/database/map/get.rs | 82 +++++++++++++++ src/database/map/insert.rs | 52 +++++++++ src/database/map/remove.rs | 44 ++++++++ src/service/globals/data.rs | 28 +++-- src/service/rooms/short/data.rs | 2 +- src/service/rooms/timeline/data.rs | 2 +- src/service/users/mod.rs | 2 +- 9 files changed, 205 insertions(+), 180 deletions(-) create mode 100644 src/database/map/get.rs create mode 100644 src/database/map/insert.rs create mode 100644 src/database/map/remove.rs diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index b1ea3709..fef83395 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -35,13 +35,3 @@ pub use self::{ #[inline] pub fn exchange(state: &mut T, source: T) -> T { std::mem::replace(state, source) } - -#[must_use] -pub fn generate_keypair() -> Vec { - let mut value = rand::string(8).as_bytes().to_vec(); - value.push(0xFF); - value.extend_from_slice( - &ruma::signatures::Ed25519KeyPair::generate().expect("Ed25519KeyPair generation always works (?)"), - ); - value -} diff --git a/src/database/map.rs b/src/database/map.rs index a3cf32d4..cac20d6a 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -1,7 +1,10 @@ mod count; +mod get; +mod insert; mod keys; mod keys_from; mod keys_prefix; +mod remove; mod rev_keys; mod rev_keys_from; mod rev_keys_prefix; @@ -18,23 +21,14 @@ use std::{ fmt, fmt::{Debug, Display}, future::Future, - io::Write, pin::Pin, sync::Arc, }; -use conduit::{err, Result}; -use futures::future; -use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, WriteBatchWithTransaction, WriteOptions}; -use serde::Serialize; +use conduit::Result; +use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, WriteOptions}; -use crate::{ - keyval::{OwnedKey, OwnedVal}, - ser, - util::{map_err, or_else}, - watchers::Watchers, - Engine, Handle, -}; +use crate::{watchers::Watchers, Engine}; pub struct Map { name: String, @@ -57,146 +51,6 @@ impl Map { })) } - #[tracing::instrument(skip(self), fields(%self), level = "trace")] - pub fn del(&self, key: &K) - where - K: Serialize + ?Sized + Debug, - { - let mut buf = Vec::::with_capacity(64); - self.bdel(key, &mut buf); - } - - #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] - pub fn bdel(&self, key: &K, buf: &mut B) - where - K: Serialize + ?Sized + Debug, - B: Write + AsRef<[u8]>, - { - let key = ser::serialize(buf, key).expect("failed to serialize deletion key"); - self.remove(&key); - } - - #[tracing::instrument(level = "trace")] - pub fn remove(&self, key: &K) - where - K: AsRef<[u8]> + ?Sized + Debug, - { - let write_options = &self.write_options; - self.db - .db - .delete_cf_opt(&self.cf(), key, write_options) - .or_else(or_else) - .expect("database remove error"); - - if !self.db.corked() { - self.db.flush().expect("database flush error"); - } - } - - #[tracing::instrument(skip(self, value), fields(%self), level = "trace")] - pub fn insert(&self, key: &K, value: &V) - where - K: AsRef<[u8]> + ?Sized + Debug, - V: AsRef<[u8]> + ?Sized, - { - let write_options = &self.write_options; - self.db - .db - .put_cf_opt(&self.cf(), key, value, write_options) - .or_else(or_else) - .expect("database insert error"); - - if !self.db.corked() { - self.db.flush().expect("database flush error"); - } - - self.watchers.wake(key.as_ref()); - } - - #[tracing::instrument(skip(self), fields(%self), level = "trace")] - pub fn insert_batch<'a, I, K, V>(&'a self, iter: I) - where - I: Iterator + Send + Debug, - K: AsRef<[u8]> + Sized + Debug + 'a, - V: AsRef<[u8]> + Sized + 'a, - { - let mut batch = WriteBatchWithTransaction::::default(); - for (key, val) in iter { - batch.put_cf(&self.cf(), key.as_ref(), val.as_ref()); - } - - let write_options = &self.write_options; - self.db - .db - .write_opt(batch, write_options) - .or_else(or_else) - .expect("database insert batch error"); - - if !self.db.corked() { - self.db.flush().expect("database flush error"); - } - } - - #[tracing::instrument(skip(self), fields(%self), level = "trace")] - pub fn qry(&self, key: &K) -> impl Future>> + Send - where - K: Serialize + ?Sized + Debug, - { - let mut buf = Vec::::with_capacity(64); - self.bqry(key, &mut buf) - } - - #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] - pub fn bqry(&self, key: &K, buf: &mut B) -> impl Future>> + Send - where - K: Serialize + ?Sized + Debug, - B: Write + AsRef<[u8]>, - { - let key = ser::serialize(buf, key).expect("failed to serialize query key"); - let val = self.get(key); - future::ready(val) - } - - #[tracing::instrument(skip(self), fields(%self), level = "trace")] - pub fn get(&self, key: &K) -> Result> - where - K: AsRef<[u8]> + ?Sized + Debug, - { - self.db - .db - .get_pinned_cf_opt(&self.cf(), key, &self.read_options) - .map_err(map_err)? - .map(Handle::from) - .ok_or(err!(Request(NotFound("Not found in database")))) - } - - #[tracing::instrument(skip(self), fields(%self), level = "trace")] - pub fn multi_get<'a, I, K>(&self, keys: I) -> Vec> - where - I: Iterator + ExactSizeIterator + Send + Debug, - K: AsRef<[u8]> + Sized + Debug + 'a, - { - // Optimization can be `true` if key vector is pre-sorted **by the column - // comparator**. - const SORTED: bool = false; - - let mut ret: Vec> = Vec::with_capacity(keys.len()); - let read_options = &self.read_options; - for res in self - .db - .db - .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) - { - match res { - Ok(Some(res)) => ret.push(Some((*res).to_vec())), - Ok(None) => ret.push(None), - Err(e) => or_else(e).expect("database multiget error"), - } - } - - ret - } - #[inline] pub fn watch_prefix<'a, K>(&'a self, prefix: &K) -> Pin + Send + 'a>> where @@ -230,10 +84,7 @@ fn open(db: &Arc, name: &str) -> Result> { let bounded_ptr = Arc::into_raw(bounded_arc); let cf_ptr = bounded_ptr.cast::(); - // SAFETY: After thorough contemplation this appears to be the best solution, - // even by a significant margin. - // - // BACKGROUND: Column family handles out of RocksDB are basic pointers and can + // SAFETY: Column family handles out of RocksDB are basic pointers and can // be invalidated: 1. when the database closes. 2. when the column is dropped or // closed. rust_rocksdb wraps this for us by storing handles in their own // `RwLock` map and returning an Arc>` to diff --git a/src/database/map/get.rs b/src/database/map/get.rs new file mode 100644 index 00000000..b4d6a6ea --- /dev/null +++ b/src/database/map/get.rs @@ -0,0 +1,82 @@ +use std::{convert::AsRef, fmt::Debug, future::Future, io::Write}; + +use conduit::{err, implement, Result}; +use futures::future::ready; +use serde::Serialize; + +use crate::{ + keyval::{OwnedKey, OwnedVal}, + ser, + util::{map_err, or_else}, + Handle, +}; + +#[implement(super::Map)] +pub fn qry(&self, key: &K) -> impl Future>> + Send +where + K: Serialize + ?Sized + Debug, +{ + let mut buf = Vec::::with_capacity(64); + self.bqry(key, &mut buf) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] +pub fn bqry(&self, key: &K, buf: &mut B) -> impl Future>> + Send +where + K: Serialize + ?Sized + Debug, + B: Write + AsRef<[u8]>, +{ + let key = ser::serialize(buf, key).expect("failed to serialize query key"); + self.get(key) +} + +#[implement(super::Map)] +pub fn get(&self, key: &K) -> impl Future>> + Send +where + K: AsRef<[u8]> + ?Sized + Debug, +{ + ready(self.get_blocking(key)) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] +pub fn get_blocking(&self, key: &K) -> Result> +where + K: AsRef<[u8]> + ?Sized + Debug, +{ + self.db + .db + .get_pinned_cf_opt(&self.cf(), key, &self.read_options) + .map_err(map_err)? + .map(Handle::from) + .ok_or(err!(Request(NotFound("Not found in database")))) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self, keys), fields(%self), level = "trace")] +pub fn get_batch_blocking<'a, I, K>(&self, keys: I) -> Vec> +where + I: Iterator + ExactSizeIterator + Send + Debug, + K: AsRef<[u8]> + Sized + Debug + 'a, +{ + // Optimization can be `true` if key vector is pre-sorted **by the column + // comparator**. + const SORTED: bool = false; + + let mut ret: Vec> = Vec::with_capacity(keys.len()); + let read_options = &self.read_options; + for res in self + .db + .db + .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) + { + match res { + Ok(Some(res)) => ret.push(Some((*res).to_vec())), + Ok(None) => ret.push(None), + Err(e) => or_else(e).expect("database multiget error"), + } + } + + ret +} diff --git a/src/database/map/insert.rs b/src/database/map/insert.rs new file mode 100644 index 00000000..953c9c94 --- /dev/null +++ b/src/database/map/insert.rs @@ -0,0 +1,52 @@ +use std::{convert::AsRef, fmt::Debug}; + +use conduit::implement; +use rocksdb::WriteBatchWithTransaction; + +use crate::util::or_else; + +#[implement(super::Map)] +#[tracing::instrument(skip(self, value), fields(%self), level = "trace")] +pub fn insert(&self, key: &K, value: &V) +where + K: AsRef<[u8]> + ?Sized + Debug, + V: AsRef<[u8]> + ?Sized, +{ + let write_options = &self.write_options; + self.db + .db + .put_cf_opt(&self.cf(), key, value, write_options) + .or_else(or_else) + .expect("database insert error"); + + if !self.db.corked() { + self.db.flush().expect("database flush error"); + } + + self.watchers.wake(key.as_ref()); +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self, iter), fields(%self), level = "trace")] +pub fn insert_batch<'a, I, K, V>(&'a self, iter: I) +where + I: Iterator + Send + Debug, + K: AsRef<[u8]> + Sized + Debug + 'a, + V: AsRef<[u8]> + Sized + 'a, +{ + let mut batch = WriteBatchWithTransaction::::default(); + for (key, val) in iter { + batch.put_cf(&self.cf(), key.as_ref(), val.as_ref()); + } + + let write_options = &self.write_options; + self.db + .db + .write_opt(batch, write_options) + .or_else(or_else) + .expect("database insert batch error"); + + if !self.db.corked() { + self.db.flush().expect("database flush error"); + } +} diff --git a/src/database/map/remove.rs b/src/database/map/remove.rs new file mode 100644 index 00000000..fcf7587e --- /dev/null +++ b/src/database/map/remove.rs @@ -0,0 +1,44 @@ +use std::{convert::AsRef, fmt::Debug, io::Write}; + +use conduit::implement; +use serde::Serialize; + +use crate::{ser, util::or_else}; + +#[implement(super::Map)] +pub fn del(&self, key: &K) +where + K: Serialize + ?Sized + Debug, +{ + let mut buf = Vec::::with_capacity(64); + self.bdel(key, &mut buf); +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] +pub fn bdel(&self, key: &K, buf: &mut B) +where + K: Serialize + ?Sized + Debug, + B: Write + AsRef<[u8]>, +{ + let key = ser::serialize(buf, key).expect("failed to serialize deletion key"); + self.remove(key); +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] +pub fn remove(&self, key: &K) +where + K: AsRef<[u8]> + ?Sized + Debug, +{ + let write_options = &self.write_options; + self.db + .db + .delete_cf_opt(&self.cf(), key, write_options) + .or_else(or_else) + .expect("database remove error"); + + if !self.db.corked() { + self.db.flush().expect("database flush error"); + } +} diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 76f97944..5332f07d 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, RwLock}, }; -use conduit::{trace, utils, Error, Result, Server}; +use conduit::{trace, utils, utils::rand, Error, Result, Server}; use database::{Database, Deserialized, Map}; use futures::{pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; use ruma::{ @@ -102,7 +102,7 @@ impl Data { fn stored_count(global: &Arc) -> Result { global - .get(COUNTER) + .get_blocking(COUNTER) .as_deref() .map_or(Ok(0_u64), utils::u64_from_bytes) } @@ -206,17 +206,23 @@ impl Data { } pub fn load_keypair(&self) -> Result { - let keypair_bytes = self.global.get(b"keypair").map_or_else( - |_| { - let keypair = utils::generate_keypair(); - self.global.insert(b"keypair", &keypair); - Ok::<_, Error>(keypair) - }, - |val| Ok(val.to_vec()), - )?; + let generate = |_| { + let keypair = Ed25519KeyPair::generate().expect("Ed25519KeyPair generation always works (?)"); + + let mut value = rand::string(8).as_bytes().to_vec(); + value.push(0xFF); + value.extend_from_slice(&keypair); + + self.global.insert(b"keypair", &value); + value + }; + + let keypair_bytes: Vec = self + .global + .get_blocking(b"keypair") + .map_or_else(generate, Into::into); let mut parts = keypair_bytes.splitn(2, |&b| b == 0xFF); - utils::string_from_bytes( // 1. version parts diff --git a/src/service/rooms/short/data.rs b/src/service/rooms/short/data.rs index f6a82488..fff3f2d6 100644 --- a/src/service/rooms/short/data.rs +++ b/src/service/rooms/short/data.rs @@ -59,7 +59,7 @@ impl Data { for (i, short) in self .eventid_shorteventid - .multi_get(keys.iter()) + .get_batch_blocking(keys.iter()) .iter() .enumerate() { diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 314dcb9f..1f9dad1d 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -326,7 +326,7 @@ pub(super) fn pdu_count(pdu_id: &[u8]) -> PduCount { //TODO: this is an ABA fn increment(db: &Arc, key: &[u8]) { - let old = db.get(key); + let old = db.get_blocking(key); let new = utils::increment(old.ok().as_deref()); db.insert(key, &new); } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index ca37ed9d..fa8c41b6 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1000,7 +1000,7 @@ where //TODO: this is an ABA fn increment(db: &Arc, key: &[u8]) { - let old = db.get(key); + let old = db.get_blocking(key); let new = utils::increment(old.ok().as_deref()); db.insert(key, &new); } From 0e8ae1e13e601c572336e38d1b020eb6a6aafe0d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 29 Sep 2024 12:49:24 +0000 Subject: [PATCH 0849/2091] add ArrayVec-backed serialized query overload; doc comments Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/database/Cargo.toml | 1 + src/database/de.rs | 2 +- src/database/handle.rs | 28 ++++++++++++++++------------ src/database/map/get.rs | 23 +++++++++++++++++++++++ src/database/map/remove.rs | 10 ++++++++++ 6 files changed, 52 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08e0498a..043d9704 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -726,6 +726,7 @@ dependencies = [ name = "conduit_database" version = "0.4.7" dependencies = [ + "arrayvec", "conduit_core", "const-str", "futures", diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index b5eb7612..0e718aa7 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -35,6 +35,7 @@ zstd_compression = [ ] [dependencies] +arrayvec.workspace = true conduit-core.workspace = true const-str.workspace = true futures.workspace = true diff --git a/src/database/de.rs b/src/database/de.rs index a5d2c127..fc36560d 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -195,7 +195,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } fn deserialize_u8>(self, _visitor: V) -> Result { - unimplemented!("deserialize u8 not implemented") + unimplemented!("deserialize u8 not implemented; try dereferencing the Handle for [u8] access instead") } fn deserialize_u16>(self, _visitor: V) -> Result { diff --git a/src/database/handle.rs b/src/database/handle.rs index 0d4bd02e..daee224d 100644 --- a/src/database/handle.rs +++ b/src/database/handle.rs @@ -35,18 +35,6 @@ impl Serialize for Handle<'_> { } } -impl Deref for Handle<'_> { - type Target = Slice; - - #[inline] - fn deref(&self) -> &Self::Target { &self.val } -} - -impl AsRef for Handle<'_> { - #[inline] - fn as_ref(&self) -> &Slice { &self.val } -} - impl Deserialized for Result> { #[inline] fn map_de(self, f: F) -> Result @@ -78,3 +66,19 @@ impl<'a> Deserialized for &'a Handle<'a> { deserialize_val(self.as_ref()).map(f) } } + +impl From> for Vec { + fn from(handle: Handle<'_>) -> Self { handle.deref().to_vec() } +} + +impl Deref for Handle<'_> { + type Target = Slice; + + #[inline] + fn deref(&self) -> &Self::Target { &self.val } +} + +impl AsRef for Handle<'_> { + #[inline] + fn as_ref(&self) -> &Slice { &self.val } +} diff --git a/src/database/map/get.rs b/src/database/map/get.rs index b4d6a6ea..71489402 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -1,5 +1,6 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write}; +use arrayvec::ArrayVec; use conduit::{err, implement, Result}; use futures::future::ready; use serde::Serialize; @@ -11,6 +12,9 @@ use crate::{ Handle, }; +/// Fetch a value from the database into cache, returning a reference-handle +/// asynchronously. The key is serialized into an allocated buffer to perform +/// the query. #[implement(super::Map)] pub fn qry(&self, key: &K) -> impl Future>> + Send where @@ -20,6 +24,20 @@ where self.bqry(key, &mut buf) } +/// Fetch a value from the database into cache, returning a reference-handle +/// asynchronously. The key is serialized into a fixed-sized buffer to perform +/// the query. The maximum size is supplied as const generic parameter. +#[implement(super::Map)] +pub fn aqry(&self, key: &K) -> impl Future>> + Send +where + K: Serialize + ?Sized + Debug, +{ + let mut buf = ArrayVec::::new(); + self.bqry(key, &mut buf) +} + +/// Fetch a value from the database into cache, returning a reference-handle +/// asynchronously. The key is serialized into a user-supplied Writer. #[implement(super::Map)] #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] pub fn bqry(&self, key: &K, buf: &mut B) -> impl Future>> + Send @@ -31,6 +49,8 @@ where self.get(key) } +/// Fetch a value from the database into cache, returning a reference-handle +/// asynchronously. The key is referenced directly to perform the query. #[implement(super::Map)] pub fn get(&self, key: &K) -> impl Future>> + Send where @@ -39,6 +59,9 @@ where ready(self.get_blocking(key)) } +/// Fetch a value from the database into cache, returning a reference-handle. +/// The key is referenced directly to perform the query. This is a thread- +/// blocking call. #[implement(super::Map)] #[tracing::instrument(skip(self, key), fields(%self), level = "trace")] pub fn get_blocking(&self, key: &K) -> Result> diff --git a/src/database/map/remove.rs b/src/database/map/remove.rs index fcf7587e..10bb2ff0 100644 --- a/src/database/map/remove.rs +++ b/src/database/map/remove.rs @@ -1,5 +1,6 @@ use std::{convert::AsRef, fmt::Debug, io::Write}; +use arrayvec::ArrayVec; use conduit::implement; use serde::Serialize; @@ -14,6 +15,15 @@ where self.bdel(key, &mut buf); } +#[implement(super::Map)] +pub fn adel(&self, key: &K) +where + K: Serialize + ?Sized + Debug, +{ + let mut buf = ArrayVec::::new(); + self.bdel(key, &mut buf); +} + #[implement(super::Map)] #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] pub fn bdel(&self, key: &K, buf: &mut B) From c569881b0853245dea0f8704342d6cfa6c465edb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 29 Sep 2024 13:13:09 +0000 Subject: [PATCH 0850/2091] merge rooms/short Data w/ Service; optimize queries Signed-off-by: Jason Volk --- src/service/account_data/mod.rs | 2 +- src/service/appservice/data.rs | 2 +- src/service/globals/data.rs | 12 +- src/service/globals/migrations.rs | 6 +- src/service/media/data.rs | 2 +- src/service/presence/data.rs | 6 +- src/service/rooms/alias/mod.rs | 6 +- src/service/rooms/directory/mod.rs | 2 +- src/service/rooms/outlier/mod.rs | 4 +- src/service/rooms/short/data.rs | 167 --------------- src/service/rooms/short/mod.rs | 232 +++++++++++++++++---- src/service/rooms/state/data.rs | 2 +- src/service/rooms/state_accessor/data.rs | 4 +- src/service/rooms/state_cache/mod.rs | 8 +- src/service/rooms/state_compressor/data.rs | 4 +- src/service/rooms/timeline/data.rs | 22 +- src/service/sending/data.rs | 6 +- src/service/users/mod.rs | 32 +-- 18 files changed, 257 insertions(+), 262 deletions(-) delete mode 100644 src/service/rooms/short/data.rs diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 4f00cff1..482229e7 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -106,7 +106,7 @@ pub async fn get( self.db .roomusertype_roomuserdataid .qry(&key) - .and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.qry(&roomuserdataid)) + .and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.get(&roomuserdataid)) .await .deserialized() } diff --git a/src/service/appservice/data.rs b/src/service/appservice/data.rs index f31c5e63..4eb9d09e 100644 --- a/src/service/appservice/data.rs +++ b/src/service/appservice/data.rs @@ -38,7 +38,7 @@ impl Data { pub async fn get_registration(&self, id: &str) -> Result { self.id_appserviceregistrations - .qry(id) + .get(id) .await .deserialized() .map_err(|e| err!(Database("Invalid appservice {id:?} registration: {e:?}"))) diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 5332f07d..57a295d9 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -260,7 +260,7 @@ impl Data { &self, origin: &ServerName, new_keys: ServerSigningKeys, ) -> BTreeMap { // Not atomic, but this is not critical - let signingkeys = self.server_signingkeys.qry(origin).await; + let signingkeys = self.server_signingkeys.get(origin).await; let mut keys = signingkeys .and_then(|keys| serde_json::from_slice(&keys).map_err(Into::into)) @@ -311,10 +311,16 @@ impl Data { } pub async fn signing_keys_for(&self, origin: &ServerName) -> Result { - self.server_signingkeys.qry(origin).await.deserialized() + self.server_signingkeys.get(origin).await.deserialized() } - pub async fn database_version(&self) -> u64 { self.global.qry("version").await.deserialized().unwrap_or(0) } + pub async fn database_version(&self) -> u64 { + self.global + .get(b"version") + .await + .deserialized() + .unwrap_or(0) + } #[inline] pub fn bump_database_version(&self, new_version: u64) -> Result<()> { diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index c7a73230..469159fc 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -99,14 +99,14 @@ async fn migrate(services: &Services) -> Result<()> { db_lt_13(services).await?; } - if db["global"].qry("feat_sha256_media").await.is_not_found() { + if db["global"].get(b"feat_sha256_media").await.is_not_found() { media::migrations::migrate_sha256_media(services).await?; } else if config.media_startup_check { media::migrations::checkup_sha256_media(services).await?; } if db["global"] - .qry("fix_bad_double_separator_in_state_cache") + .get(b"fix_bad_double_separator_in_state_cache") .await .is_not_found() { @@ -114,7 +114,7 @@ async fn migrate(services: &Services) -> Result<()> { } if db["global"] - .qry("retroactively_fix_bad_data_from_roomuserid_joined") + .get(b"retroactively_fix_bad_data_from_roomuserid_joined") .await .is_not_found() { diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 29d562cc..248e9e1d 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -253,7 +253,7 @@ impl Data { } pub(super) async fn get_url_preview(&self, url: &str) -> Result { - let values = self.url_previews.qry(url).await?; + let values = self.url_previews.get(url).await?; let mut values = values.split(|&b| b == 0xFF); diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 0c3f3d31..9c9d0ae3 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -39,12 +39,12 @@ impl Data { pub async fn get_presence(&self, user_id: &UserId) -> Result<(u64, PresenceEvent)> { let count = self .userid_presenceid - .qry(user_id) + .get(user_id) .await .deserialized::()?; let key = presenceid_key(count, user_id); - let bytes = self.presenceid_presence.qry(&key).await?; + let bytes = self.presenceid_presence.get(&key).await?; let event = Presence::from_json_bytes(&bytes)? .to_presence_event(user_id, &self.services.users) .await; @@ -127,7 +127,7 @@ impl Data { pub(super) async fn remove_presence(&self, user_id: &UserId) { let Ok(count) = self .userid_presenceid - .qry(user_id) + .get(user_id) .await .deserialized::() else { diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 6b81a221..1d44cd2d 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -94,7 +94,7 @@ impl Service { } let alias = alias.alias(); - let Ok(room_id) = self.db.alias_roomid.qry(&alias).await else { + let Ok(room_id) = self.db.alias_roomid.get(&alias).await else { return Err!(Request(NotFound("Alias does not exist or is invalid."))); }; @@ -151,7 +151,7 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub async fn resolve_local_alias(&self, alias: &RoomAliasId) -> Result { - self.db.alias_roomid.qry(alias.alias()).await.deserialized() + self.db.alias_roomid.get(alias.alias()).await.deserialized() } #[tracing::instrument(skip(self), level = "debug")] @@ -219,7 +219,7 @@ impl Service { } async fn who_created_alias(&self, alias: &RoomAliasId) -> Result { - self.db.alias_userid.qry(alias.alias()).await.deserialized() + self.db.alias_userid.get(alias.alias()).await.deserialized() } async fn resolve_appservice_alias(&self, room_alias: &RoomAliasId) -> Result> { diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 3585205d..5666a91a 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -32,7 +32,7 @@ pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_i pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id.as_bytes()); } #[implement(Service)] -pub async fn is_public_room(&self, room_id: &RoomId) -> bool { self.db.publicroomids.qry(room_id).await.is_ok() } +pub async fn is_public_room(&self, room_id: &RoomId) -> bool { self.db.publicroomids.get(room_id).await.is_ok() } #[implement(Service)] pub fn public_rooms(&self) -> impl Stream + Send { diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 4c9225ae..b9d04263 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -31,7 +31,7 @@ impl crate::Service for Service { pub async fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result { self.db .eventid_outlierpdu - .qry(event_id) + .get(event_id) .await .deserialized() } @@ -41,7 +41,7 @@ pub async fn get_outlier_pdu_json(&self, event_id: &EventId) -> Result Result { self.db .eventid_outlierpdu - .qry(event_id) + .get(event_id) .await .deserialized() } diff --git a/src/service/rooms/short/data.rs b/src/service/rooms/short/data.rs deleted file mode 100644 index fff3f2d6..00000000 --- a/src/service/rooms/short/data.rs +++ /dev/null @@ -1,167 +0,0 @@ -use std::sync::Arc; - -use conduit::{err, utils, Error, Result}; -use database::{Deserialized, Map}; -use ruma::{events::StateEventType, EventId, RoomId}; - -use crate::{globals, Dep}; - -pub(super) struct Data { - eventid_shorteventid: Arc, - shorteventid_eventid: Arc, - statekey_shortstatekey: Arc, - shortstatekey_statekey: Arc, - roomid_shortroomid: Arc, - statehash_shortstatehash: Arc, - services: Services, -} - -struct Services { - globals: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - eventid_shorteventid: db["eventid_shorteventid"].clone(), - shorteventid_eventid: db["shorteventid_eventid"].clone(), - statekey_shortstatekey: db["statekey_shortstatekey"].clone(), - shortstatekey_statekey: db["shortstatekey_statekey"].clone(), - roomid_shortroomid: db["roomid_shortroomid"].clone(), - statehash_shortstatehash: db["statehash_shortstatehash"].clone(), - services: Services { - globals: args.depend::("globals"), - }, - } - } - - pub(super) async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { - if let Ok(shorteventid) = self.eventid_shorteventid.qry(event_id).await.deserialized() { - return shorteventid; - } - - let shorteventid = self.services.globals.next_count().unwrap(); - self.eventid_shorteventid - .insert(event_id.as_bytes(), &shorteventid.to_be_bytes()); - self.shorteventid_eventid - .insert(&shorteventid.to_be_bytes(), event_id.as_bytes()); - - shorteventid - } - - pub(super) async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Vec { - let mut ret: Vec = Vec::with_capacity(event_ids.len()); - let keys = event_ids - .iter() - .map(|id| id.as_bytes()) - .collect::>(); - - for (i, short) in self - .eventid_shorteventid - .get_batch_blocking(keys.iter()) - .iter() - .enumerate() - { - #[allow(clippy::single_match_else)] - match short { - Some(short) => ret.push( - utils::u64_from_bytes(short) - .map_err(|_| Error::bad_database("Invalid shorteventid in db.")) - .unwrap(), - ), - None => { - let short = self.services.globals.next_count().unwrap(); - self.eventid_shorteventid - .insert(keys[i], &short.to_be_bytes()); - self.shorteventid_eventid - .insert(&short.to_be_bytes(), keys[i]); - - debug_assert!(ret.len() == i, "position of result must match input"); - ret.push(short); - }, - } - } - - ret - } - - pub(super) async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { - let key = (event_type, state_key); - self.statekey_shortstatekey.qry(&key).await.deserialized() - } - - pub(super) async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> u64 { - let key = (event_type.to_string(), state_key); - if let Ok(shortstatekey) = self.statekey_shortstatekey.qry(&key).await.deserialized() { - return shortstatekey; - } - - let mut key = event_type.to_string().as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(state_key.as_bytes()); - - let shortstatekey = self.services.globals.next_count().unwrap(); - self.statekey_shortstatekey - .insert(&key, &shortstatekey.to_be_bytes()); - self.shortstatekey_statekey - .insert(&shortstatekey.to_be_bytes(), &key); - - shortstatekey - } - - pub(super) async fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { - self.shorteventid_eventid - .qry(&shorteventid) - .await - .deserialized() - .map_err(|e| err!(Database("Failed to find EventId from short {shorteventid:?}: {e:?}"))) - } - - pub(super) async fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { - self.shortstatekey_statekey - .qry(&shortstatekey) - .await - .deserialized() - .map_err(|e| { - err!(Database( - "Failed to find (StateEventType, state_key) from short {shortstatekey:?}: {e:?}" - )) - }) - } - - /// Returns (shortstatehash, already_existed) - pub(super) async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, bool) { - if let Ok(shortstatehash) = self - .statehash_shortstatehash - .qry(state_hash) - .await - .deserialized() - { - return (shortstatehash, true); - } - - let shortstatehash = self.services.globals.next_count().unwrap(); - self.statehash_shortstatehash - .insert(state_hash, &shortstatehash.to_be_bytes()); - - (shortstatehash, false) - } - - pub(super) async fn get_shortroomid(&self, room_id: &RoomId) -> Result { - self.roomid_shortroomid.qry(room_id).await.deserialized() - } - - pub(super) async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> u64 { - self.roomid_shortroomid - .qry(room_id) - .await - .deserialized() - .unwrap_or_else(|_| { - let short = self.services.globals.next_count().unwrap(); - self.roomid_shortroomid - .insert(room_id.as_bytes(), &short.to_be_bytes()); - short - }) - } -} diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 00bb7cb1..66da3948 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,61 +1,215 @@ -mod data; - use std::sync::Arc; -use conduit::Result; +use conduit::{err, implement, utils, Error, Result}; +use database::{Deserialized, Map}; use ruma::{events::StateEventType, EventId, RoomId}; -use self::data::Data; +use crate::{globals, Dep}; pub struct Service { db: Data, + services: Services, +} + +struct Data { + eventid_shorteventid: Arc, + shorteventid_eventid: Arc, + statekey_shortstatekey: Arc, + shortstatekey_statekey: Arc, + roomid_shortroomid: Arc, + statehash_shortstatehash: Arc, +} + +struct Services { + globals: Dep, } impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(&args), + db: Data { + eventid_shorteventid: args.db["eventid_shorteventid"].clone(), + shorteventid_eventid: args.db["shorteventid_eventid"].clone(), + statekey_shortstatekey: args.db["statekey_shortstatekey"].clone(), + shortstatekey_statekey: args.db["shortstatekey_statekey"].clone(), + roomid_shortroomid: args.db["roomid_shortroomid"].clone(), + statehash_shortstatehash: args.db["statehash_shortstatehash"].clone(), + }, + services: Services { + globals: args.depend::("globals"), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { - self.db.get_or_create_shorteventid(event_id).await +#[implement(Service)] +pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { + if let Ok(shorteventid) = self + .db + .eventid_shorteventid + .get(event_id) + .await + .deserialized() + { + return shorteventid; } - pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Vec { - self.db.multi_get_or_create_shorteventid(event_ids).await - } + let shorteventid = self.services.globals.next_count().unwrap(); + self.db + .eventid_shorteventid + .insert(event_id.as_bytes(), &shorteventid.to_be_bytes()); + self.db + .shorteventid_eventid + .insert(&shorteventid.to_be_bytes(), event_id.as_bytes()); - pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { - self.db.get_shortstatekey(event_type, state_key).await - } - - pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> u64 { - self.db - .get_or_create_shortstatekey(event_type, state_key) - .await - } - - pub async fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { - self.db.get_eventid_from_short(shorteventid).await - } - - pub async fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { - self.db.get_statekey_from_short(shortstatekey).await - } - - /// Returns (shortstatehash, already_existed) - pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, bool) { - self.db.get_or_create_shortstatehash(state_hash).await - } - - pub async fn get_shortroomid(&self, room_id: &RoomId) -> Result { self.db.get_shortroomid(room_id).await } - - pub async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> u64 { - self.db.get_or_create_shortroomid(room_id).await - } + shorteventid +} + +#[implement(Service)] +pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Vec { + let mut ret: Vec = Vec::with_capacity(event_ids.len()); + let keys = event_ids + .iter() + .map(|id| id.as_bytes()) + .collect::>(); + + for (i, short) in self + .db + .eventid_shorteventid + .get_batch_blocking(keys.iter()) + .iter() + .enumerate() + { + match short { + Some(short) => ret.push( + utils::u64_from_bytes(short) + .map_err(|_| Error::bad_database("Invalid shorteventid in db.")) + .unwrap(), + ), + None => { + let short = self.services.globals.next_count().unwrap(); + self.db + .eventid_shorteventid + .insert(keys[i], &short.to_be_bytes()); + self.db + .shorteventid_eventid + .insert(&short.to_be_bytes(), keys[i]); + + debug_assert!(ret.len() == i, "position of result must match input"); + ret.push(short); + }, + } + } + + ret +} + +#[implement(Service)] +pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { + let key = (event_type, state_key); + self.db + .statekey_shortstatekey + .qry(&key) + .await + .deserialized() +} + +#[implement(Service)] +pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> u64 { + let key = (event_type.to_string(), state_key); + if let Ok(shortstatekey) = self + .db + .statekey_shortstatekey + .qry(&key) + .await + .deserialized() + { + return shortstatekey; + } + + let mut key = event_type.to_string().as_bytes().to_vec(); + key.push(0xFF); + key.extend_from_slice(state_key.as_bytes()); + + let shortstatekey = self.services.globals.next_count().unwrap(); + self.db + .statekey_shortstatekey + .insert(&key, &shortstatekey.to_be_bytes()); + self.db + .shortstatekey_statekey + .insert(&shortstatekey.to_be_bytes(), &key); + + shortstatekey +} + +#[implement(Service)] +pub async fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { + const BUFSIZE: usize = size_of::(); + + self.db + .shorteventid_eventid + .aqry::(&shorteventid) + .await + .deserialized() + .map_err(|e| err!(Database("Failed to find EventId from short {shorteventid:?}: {e:?}"))) +} + +#[implement(Service)] +pub async fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { + const BUFSIZE: usize = size_of::(); + + self.db + .shortstatekey_statekey + .aqry::(&shortstatekey) + .await + .deserialized() + .map_err(|e| { + err!(Database( + "Failed to find (StateEventType, state_key) from short {shortstatekey:?}: {e:?}" + )) + }) +} + +/// Returns (shortstatehash, already_existed) +#[implement(Service)] +pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, bool) { + if let Ok(shortstatehash) = self + .db + .statehash_shortstatehash + .get(state_hash) + .await + .deserialized() + { + return (shortstatehash, true); + } + + let shortstatehash = self.services.globals.next_count().unwrap(); + self.db + .statehash_shortstatehash + .insert(state_hash, &shortstatehash.to_be_bytes()); + + (shortstatehash, false) +} + +#[implement(Service)] +pub async fn get_shortroomid(&self, room_id: &RoomId) -> Result { + self.db.roomid_shortroomid.qry(room_id).await.deserialized() +} + +#[implement(Service)] +pub async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> u64 { + self.db + .roomid_shortroomid + .get(room_id) + .await + .deserialized() + .unwrap_or_else(|_| { + let short = self.services.globals.next_count().unwrap(); + self.db + .roomid_shortroomid + .insert(room_id.as_bytes(), &short.to_be_bytes()); + short + }) } diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index ccf7509a..3072e3c6 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -25,7 +25,7 @@ impl Data { } pub(super) async fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result { - self.roomid_shortstatehash.qry(room_id).await.deserialized() + self.roomid_shortstatehash.get(room_id).await.deserialized() } #[inline] diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 79a98325..adc26f00 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -157,8 +157,8 @@ impl Data { /// Returns the state hash for this pdu. pub(super) async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { self.eventid_shorteventid - .qry(event_id) - .and_then(|shorteventid| self.shorteventid_shortstatehash.qry(&shorteventid)) + .get(event_id) + .and_then(|shorteventid| self.shorteventid_shortstatehash.get(&shorteventid)) .await .deserialized() } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index ce5b024b..eedff861 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -435,10 +435,10 @@ impl Service { /// Returns an iterator over all rooms this user joined. #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_joined(&self, user_id: &UserId) -> impl Stream + Send { + pub fn rooms_joined<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { self.db .userroomid_joined - .keys_prefix(user_id) + .keys_prefix_raw(user_id) .ignore_err() .map(|(_, room_id): (Ignore, &RoomId)| room_id) } @@ -494,10 +494,10 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub fn servers_invite_via<'a>(&'a self, room_id: &RoomId) -> impl Stream + Send + 'a { + pub fn servers_invite_via<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { self.db .roomid_inviteviaservers - .stream_prefix(room_id) + .stream_prefix_raw(room_id) .ignore_err() .map(|(_, servers): (Ignore, Vec<&ServerName>)| &**(servers.last().expect("at least one servername"))) } diff --git a/src/service/rooms/state_compressor/data.rs b/src/service/rooms/state_compressor/data.rs index 9a9f70a2..cb020470 100644 --- a/src/service/rooms/state_compressor/data.rs +++ b/src/service/rooms/state_compressor/data.rs @@ -23,9 +23,11 @@ impl Data { } pub(super) async fn get_statediff(&self, shortstatehash: u64) -> Result { + const BUFSIZE: usize = size_of::(); + let value = self .shortstatehash_statediff - .qry(&shortstatehash) + .aqry::(&shortstatehash) .await .map_err(|e| err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")))?; diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 1f9dad1d..cb85cf19 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -79,7 +79,7 @@ impl Data { /// Returns the `count` of this pdu's id. pub(super) async fn get_pdu_count(&self, event_id: &EventId) -> Result { self.eventid_pduid - .qry(event_id) + .get(event_id) .await .map(|pdu_id| pdu_count(&pdu_id)) } @@ -90,27 +90,27 @@ impl Data { return Ok(pdu); } - self.eventid_outlierpdu.qry(event_id).await.deserialized() + self.eventid_outlierpdu.get(event_id).await.deserialized() } /// Returns the json of a pdu. pub(super) async fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result { let pduid = self.get_pdu_id(event_id).await?; - self.pduid_pdu.qry(&pduid).await.deserialized() + self.pduid_pdu.get(&pduid).await.deserialized() } /// Returns the pdu's id. #[inline] pub(super) async fn get_pdu_id(&self, event_id: &EventId) -> Result> { - self.eventid_pduid.qry(event_id).await + self.eventid_pduid.get(event_id).await } /// Returns the pdu directly from `eventid_pduid` only. pub(super) async fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result { let pduid = self.get_pdu_id(event_id).await?; - self.pduid_pdu.qry(&pduid).await.deserialized() + self.pduid_pdu.get(&pduid).await.deserialized() } /// Like get_non_outlier_pdu(), but without the expense of fetching and @@ -118,7 +118,7 @@ impl Data { pub(super) async fn non_outlier_pdu_exists(&self, event_id: &EventId) -> Result<()> { let pduid = self.get_pdu_id(event_id).await?; - self.pduid_pdu.qry(&pduid).await?; + self.pduid_pdu.get(&pduid).await?; Ok(()) } @@ -132,7 +132,7 @@ impl Data { } self.eventid_outlierpdu - .qry(event_id) + .get(event_id) .await .deserialized() .map(Arc::new) @@ -141,7 +141,7 @@ impl Data { /// Like get_non_outlier_pdu(), but without the expense of fetching and /// parsing the PduEvent pub(super) async fn outlier_pdu_exists(&self, event_id: &EventId) -> Result<()> { - self.eventid_outlierpdu.qry(event_id).await?; + self.eventid_outlierpdu.get(event_id).await?; Ok(()) } @@ -159,12 +159,12 @@ impl Data { /// /// This does __NOT__ check the outliers `Tree`. pub(super) async fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result { - self.pduid_pdu.qry(pdu_id).await.deserialized() + self.pduid_pdu.get(pdu_id).await.deserialized() } /// Returns the pdu as a `BTreeMap`. pub(super) async fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result { - self.pduid_pdu.qry(pdu_id).await.deserialized() + self.pduid_pdu.get(pdu_id).await.deserialized() } pub(super) async fn append_pdu(&self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64) { @@ -196,7 +196,7 @@ impl Data { pub(super) async fn replace_pdu( &self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, _pdu: &PduEvent, ) -> Result<()> { - if self.pduid_pdu.qry(pdu_id).await.is_not_found() { + if self.pduid_pdu.get(pdu_id).await.is_not_found() { return Err!(Request(NotFound("PDU does not exist."))); } diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index b96f9a03..6f4b5b97 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -98,7 +98,7 @@ impl Data { } #[inline] - pub fn active_requests_for<'a>(&'a self, destination: &Destination) -> impl Stream + Send + 'a { + pub fn active_requests_for(&self, destination: &Destination) -> impl Stream + Send + '_ { let prefix = destination.get_prefix(); self.servercurrentevent_data .stream_raw_prefix(&prefix) @@ -133,7 +133,7 @@ impl Data { keys } - pub fn queued_requests<'a>(&'a self, destination: &Destination) -> impl Stream + Send + 'a { + pub fn queued_requests(&self, destination: &Destination) -> impl Stream + Send + '_ { let prefix = destination.get_prefix(); self.servernameevent_data .stream_raw_prefix(&prefix) @@ -152,7 +152,7 @@ impl Data { pub async fn get_latest_educount(&self, server_name: &ServerName) -> u64 { self.servername_educount - .qry(server_name) + .get(server_name) .await .deserialized() .unwrap_or(0) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index fa8c41b6..eb77ef35 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -120,13 +120,13 @@ impl Service { /// Check if a user has an account on this homeserver. #[inline] - pub async fn exists(&self, user_id: &UserId) -> bool { self.db.userid_password.qry(user_id).await.is_ok() } + pub async fn exists(&self, user_id: &UserId) -> bool { self.db.userid_password.get(user_id).await.is_ok() } /// Check if account is deactivated pub async fn is_deactivated(&self, user_id: &UserId) -> Result { self.db .userid_password - .qry(user_id) + .get(user_id) .map_ok(|val| val.is_empty()) .map_err(|_| err!(Request(NotFound("User does not exist.")))) .await @@ -146,7 +146,7 @@ impl Service { /// Find out which user an access token belongs to. pub async fn find_from_token(&self, token: &str) -> Result<(OwnedUserId, OwnedDeviceId)> { - self.db.token_userdeviceid.qry(token).await.deserialized() + self.db.token_userdeviceid.get(token).await.deserialized() } /// Returns an iterator over all users on this homeserver (offered for @@ -171,7 +171,7 @@ impl Service { /// Returns the password hash for the given user. pub async fn password_hash(&self, user_id: &UserId) -> Result { - self.db.userid_password.qry(user_id).await.deserialized() + self.db.userid_password.get(user_id).await.deserialized() } /// Hash and set the user's password to the Argon2 hash @@ -196,7 +196,7 @@ impl Service { /// Returns the displayname of a user on this homeserver. pub async fn displayname(&self, user_id: &UserId) -> Result { - self.db.userid_displayname.qry(user_id).await.deserialized() + self.db.userid_displayname.get(user_id).await.deserialized() } /// Sets a new displayname or removes it if displayname is None. You still @@ -213,7 +213,7 @@ impl Service { /// Get the `avatar_url` of a user. pub async fn avatar_url(&self, user_id: &UserId) -> Result { - self.db.userid_avatarurl.qry(user_id).await.deserialized() + self.db.userid_avatarurl.get(user_id).await.deserialized() } /// Sets a new avatar_url or removes it if avatar_url is None. @@ -229,7 +229,7 @@ impl Service { /// Get the blurhash of a user. pub async fn blurhash(&self, user_id: &UserId) -> Result { - self.db.userid_blurhash.qry(user_id).await.deserialized() + self.db.userid_blurhash.get(user_id).await.deserialized() } /// Sets a new avatar_url or removes it if avatar_url is None. @@ -284,7 +284,7 @@ impl Service { userdeviceid.extend_from_slice(device_id.as_bytes()); // Remove tokens - if let Ok(old_token) = self.db.userdeviceid_token.qry(&userdeviceid).await { + if let Ok(old_token) = self.db.userdeviceid_token.get(&userdeviceid).await { self.db.userdeviceid_token.remove(&userdeviceid); self.db.token_userdeviceid.remove(&old_token); } @@ -390,7 +390,7 @@ impl Service { pub async fn last_one_time_keys_update(&self, user_id: &UserId) -> u64 { self.db .userid_lastonetimekeyupdate - .qry(user_id) + .get(user_id) .await .deserialized() .unwrap_or(0) @@ -664,7 +664,7 @@ impl Service { let key = self .db .keyid_key - .qry(key_id) + .get(key_id) .await .deserialized::()?; @@ -679,7 +679,7 @@ impl Service { where F: Fn(&UserId) -> bool + Send + Sync, { - let key_id = self.db.userid_masterkeyid.qry(user_id).await?; + let key_id = self.db.userid_masterkeyid.get(user_id).await?; self.get_key(&key_id, sender_user, user_id, allowed_signatures) .await @@ -691,16 +691,16 @@ impl Service { where F: Fn(&UserId) -> bool + Send + Sync, { - let key_id = self.db.userid_selfsigningkeyid.qry(user_id).await?; + let key_id = self.db.userid_selfsigningkeyid.get(user_id).await?; self.get_key(&key_id, sender_user, user_id, allowed_signatures) .await } pub async fn get_user_signing_key(&self, user_id: &UserId) -> Result> { - let key_id = self.db.userid_usersigningkeyid.qry(user_id).await?; + let key_id = self.db.userid_usersigningkeyid.get(user_id).await?; - self.db.keyid_key.qry(&*key_id).await.deserialized() + self.db.keyid_key.get(&*key_id).await.deserialized() } pub async fn add_to_device_event( @@ -797,7 +797,7 @@ impl Service { pub async fn get_devicelist_version(&self, user_id: &UserId) -> Result { self.db .userid_devicelistversion - .qry(user_id) + .get(user_id) .await .deserialized() } @@ -853,7 +853,7 @@ impl Service { /// Find out which user an OpenID access token belongs to. pub async fn find_from_openid_token(&self, token: &str) -> Result { - let Ok(value) = self.db.openidtoken_expiresatuserid.qry(token).await else { + let Ok(value) = self.db.openidtoken_expiresatuserid.get(token).await else { return Err!(Request(Unauthorized("OpenID token is unrecognised"))); }; From a8d5cf96517d706cff5fe73650405edaff4e0779 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 23 Sep 2024 21:38:56 +0000 Subject: [PATCH 0851/2091] Add rocksdb logging integration with tracing. Signed-off-by: Jason Volk --- src/database/engine.rs | 17 ++++++++++++++++- src/database/opts.rs | 2 ++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 067232e6..edf077fc 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -10,7 +10,7 @@ use conduit::{debug, error, info, utils::time::rfc2822_from_seconds, warn, Err, use rocksdb::{ backup::{BackupEngine, BackupEngineOptions}, perf::get_memory_usage_stats, - AsColumnFamilyRef, BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode, Env, + AsColumnFamilyRef, BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode, Env, LogLevel, MultiThreaded, Options, }; @@ -279,6 +279,21 @@ pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> { Ok(()) } +#[tracing::instrument(skip_all, name = "rocksdb")] +pub(crate) fn handle_log(level: LogLevel, msg: &str) { + let msg = msg.trim(); + if msg.starts_with("Options") { + return; + } + + match level { + LogLevel::Header | LogLevel::Debug => debug!("{msg}"), + LogLevel::Error | LogLevel::Fatal => error!("{msg}"), + LogLevel::Info => debug!("{msg}"), + LogLevel::Warn => warn!("{msg}"), + }; +} + impl Drop for Engine { #[cold] fn drop(&mut self) { diff --git a/src/database/opts.rs b/src/database/opts.rs index d2ad4b95..46fb4c54 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -191,6 +191,8 @@ fn set_logging_defaults(opts: &mut Options, config: &Config) { if config.rocksdb_log_stderr { opts.set_stderr_logger(rocksdb_log_level, "rocksdb"); + } else { + opts.set_callback_logger(rocksdb_log_level, &super::engine::handle_log); } } From 6b80361c31fc8b2eeeafbcfbf14a463c3423ee7c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 30 Sep 2024 06:46:54 +0000 Subject: [PATCH 0852/2091] additional stream tools Signed-off-by: Jason Volk --- src/core/result.rs | 5 +- src/core/result/into_is_ok.rs | 10 +++ src/core/utils/bool.rs | 16 +++++ src/core/utils/future/mod.rs | 3 + src/core/utils/future/try_ext_ext.rs | 48 +++++++++++++ src/core/utils/mod.rs | 6 +- src/core/utils/stream/mod.rs | 2 + src/core/utils/stream/ready.rs | 102 ++++++++++++++++++--------- src/core/utils/stream/tools.rs | 80 +++++++++++++++++++++ src/service/rooms/state_cache/mod.rs | 16 ++--- 10 files changed, 242 insertions(+), 46 deletions(-) create mode 100644 src/core/result/into_is_ok.rs create mode 100644 src/core/utils/bool.rs create mode 100644 src/core/utils/future/mod.rs create mode 100644 src/core/utils/future/try_ext_ext.rs create mode 100644 src/core/utils/stream/tools.rs diff --git a/src/core/result.rs b/src/core/result.rs index 96a34b8a..82d67a9c 100644 --- a/src/core/result.rs +++ b/src/core/result.rs @@ -1,4 +1,5 @@ mod debug_inspect; +mod into_is_ok; mod log_debug_err; mod log_err; mod map_expect; @@ -6,8 +7,8 @@ mod not_found; mod unwrap_infallible; pub use self::{ - debug_inspect::DebugInspect, log_debug_err::LogDebugErr, log_err::LogErr, map_expect::MapExpect, - not_found::NotFound, unwrap_infallible::UnwrapInfallible, + debug_inspect::DebugInspect, into_is_ok::IntoIsOk, log_debug_err::LogDebugErr, log_err::LogErr, + map_expect::MapExpect, not_found::NotFound, unwrap_infallible::UnwrapInfallible, }; pub type Result = std::result::Result; diff --git a/src/core/result/into_is_ok.rs b/src/core/result/into_is_ok.rs new file mode 100644 index 00000000..220ce010 --- /dev/null +++ b/src/core/result/into_is_ok.rs @@ -0,0 +1,10 @@ +use super::Result; + +pub trait IntoIsOk { + fn into_is_ok(self) -> bool; +} + +impl IntoIsOk for Result { + #[inline] + fn into_is_ok(self) -> bool { self.is_ok() } +} diff --git a/src/core/utils/bool.rs b/src/core/utils/bool.rs new file mode 100644 index 00000000..d7ce78fe --- /dev/null +++ b/src/core/utils/bool.rs @@ -0,0 +1,16 @@ +//! Trait BoolExt + +/// Boolean extensions and chain.starters +pub trait BoolExt { + fn or T>(self, f: F) -> Option; + + fn or_some(self, t: T) -> Option; +} + +impl BoolExt for bool { + #[inline] + fn or T>(self, f: F) -> Option { (!self).then(f) } + + #[inline] + fn or_some(self, t: T) -> Option { (!self).then_some(t) } +} diff --git a/src/core/utils/future/mod.rs b/src/core/utils/future/mod.rs new file mode 100644 index 00000000..6d45b656 --- /dev/null +++ b/src/core/utils/future/mod.rs @@ -0,0 +1,3 @@ +mod try_ext_ext; + +pub use try_ext_ext::TryExtExt; diff --git a/src/core/utils/future/try_ext_ext.rs b/src/core/utils/future/try_ext_ext.rs new file mode 100644 index 00000000..e444ad94 --- /dev/null +++ b/src/core/utils/future/try_ext_ext.rs @@ -0,0 +1,48 @@ +//! Extended external extensions to futures::TryFutureExt + +use futures::{future::MapOkOrElse, TryFuture, TryFutureExt}; + +/// This interface is not necessarily complete; feel free to add as-needed. +pub trait TryExtExt +where + Self: TryFuture + Send, +{ + fn map_ok_or( + self, default: U, f: F, + ) -> MapOkOrElse U, impl FnOnce(Self::Error) -> U> + where + F: FnOnce(Self::Ok) -> U, + Self: Send + Sized; + + fn ok( + self, + ) -> MapOkOrElse Option, impl FnOnce(Self::Error) -> Option> + where + Self: Sized; +} + +impl TryExtExt for Fut +where + Fut: TryFuture + Send, +{ + #[inline] + fn map_ok_or( + self, default: U, f: F, + ) -> MapOkOrElse U, impl FnOnce(Self::Error) -> U> + where + F: FnOnce(Self::Ok) -> U, + Self: Send + Sized, + { + self.map_ok_or_else(|_| default, f) + } + + #[inline] + fn ok( + self, + ) -> MapOkOrElse Option, impl FnOnce(Self::Error) -> Option> + where + Self: Sized, + { + self.map_ok_or(None, Some) + } +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index fef83395..c34691d2 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -1,7 +1,9 @@ +pub mod bool; pub mod bytes; pub mod content_disposition; pub mod debug; pub mod defer; +pub mod future; pub mod hash; pub mod html; pub mod json; @@ -19,15 +21,17 @@ pub use ::conduit_macros::implement; pub use ::ctor::{ctor, dtor}; pub use self::{ + bool::BoolExt, bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}, debug::slice_truncated as debug_slice_truncated, + future::TryExtExt as TryFutureExtExt, hash::calculate_hash, html::Escape as HtmlEscape, json::{deserialize_from_str, to_canonical_object}, math::clamp, mutex_map::{Guard as MutexMapGuard, MutexMap}, rand::string as random_string, - stream::{IterStream, ReadyExt, TryReadyExt}, + stream::{IterStream, ReadyExt, Tools as StreamTools, TryReadyExt}, string::{str_from_bytes, string_from_bytes}, sys::available_parallelism, time::now_millis as millis_since_unix_epoch, diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index 781bd522..1111915b 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -3,6 +3,7 @@ mod expect; mod ignore; mod iter_stream; mod ready; +mod tools; mod try_ready; pub use cloned::Cloned; @@ -10,4 +11,5 @@ pub use expect::TryExpect; pub use ignore::TryIgnore; pub use iter_stream::IterStream; pub use ready::ReadyExt; +pub use tools::Tools; pub use try_ready::TryReadyExt; diff --git a/src/core/utils/stream/ready.rs b/src/core/utils/stream/ready.rs index 13f730a7..da5aec5a 100644 --- a/src/core/utils/stream/ready.rs +++ b/src/core/utils/stream/ready.rs @@ -2,7 +2,7 @@ use futures::{ future::{ready, Ready}, - stream::{Any, Filter, FilterMap, Fold, ForEach, SkipWhile, Stream, StreamExt, TakeWhile}, + stream::{Any, Filter, FilterMap, Fold, ForEach, Scan, SkipWhile, Stream, StreamExt, TakeWhile}, }; /// Synchronous combinators to augment futures::StreamExt. Most Stream @@ -11,98 +11,130 @@ use futures::{ /// convenience to reduce boilerplate by de-cluttering non-async predicates. /// /// This interface is not necessarily complete; feel free to add as-needed. -pub trait ReadyExt +pub trait ReadyExt where - S: Stream + Send + ?Sized, - Self: Stream + Send + Sized, + Self: Stream + Send + Sized, { - fn ready_any(self, f: F) -> Any, impl FnMut(S::Item) -> Ready> + fn ready_any(self, f: F) -> Any, impl FnMut(Item) -> Ready> where - F: Fn(S::Item) -> bool; + F: Fn(Item) -> bool; - fn ready_filter<'a, F>(self, f: F) -> Filter, impl FnMut(&S::Item) -> Ready + 'a> + fn ready_filter<'a, F>(self, f: F) -> Filter, impl FnMut(&Item) -> Ready + 'a> where - F: Fn(&S::Item) -> bool + 'a; + F: Fn(&Item) -> bool + 'a; - fn ready_filter_map(self, f: F) -> FilterMap>, impl FnMut(S::Item) -> Ready>> + fn ready_filter_map(self, f: F) -> FilterMap>, impl FnMut(Item) -> Ready>> where - F: Fn(S::Item) -> Option; + F: Fn(Item) -> Option; - fn ready_fold(self, init: T, f: F) -> Fold, T, impl FnMut(T, S::Item) -> Ready> + fn ready_fold(self, init: T, f: F) -> Fold, T, impl FnMut(T, Item) -> Ready> where - F: Fn(T, S::Item) -> T; + F: Fn(T, Item) -> T; - fn ready_for_each(self, f: F) -> ForEach, impl FnMut(S::Item) -> Ready<()>> + fn ready_for_each(self, f: F) -> ForEach, impl FnMut(Item) -> Ready<()>> where - F: FnMut(S::Item); + F: FnMut(Item); - fn ready_take_while<'a, F>(self, f: F) -> TakeWhile, impl FnMut(&S::Item) -> Ready + 'a> + fn ready_take_while<'a, F>(self, f: F) -> TakeWhile, impl FnMut(&Item) -> Ready + 'a> where - F: Fn(&S::Item) -> bool + 'a; + F: Fn(&Item) -> bool + 'a; - fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile, impl FnMut(&S::Item) -> Ready + 'a> + fn ready_scan( + self, init: T, f: F, + ) -> Scan>, impl FnMut(&mut T, Item) -> Ready>> where - F: Fn(&S::Item) -> bool + 'a; + F: Fn(&mut T, Item) -> Option; + + fn ready_scan_each( + self, init: T, f: F, + ) -> Scan>, impl FnMut(&mut T, Item) -> Ready>> + where + F: Fn(&mut T, &Item); + + fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile, impl FnMut(&Item) -> Ready + 'a> + where + F: Fn(&Item) -> bool + 'a; } -impl ReadyExt for S +impl ReadyExt for S where - S: Stream + Send + ?Sized, - Self: Stream + Send + Sized, + S: Stream + Send + Sized, { #[inline] - fn ready_any(self, f: F) -> Any, impl FnMut(S::Item) -> Ready> + fn ready_any(self, f: F) -> Any, impl FnMut(Item) -> Ready> where - F: Fn(S::Item) -> bool, + F: Fn(Item) -> bool, { self.any(move |t| ready(f(t))) } #[inline] - fn ready_filter<'a, F>(self, f: F) -> Filter, impl FnMut(&S::Item) -> Ready + 'a> + fn ready_filter<'a, F>(self, f: F) -> Filter, impl FnMut(&Item) -> Ready + 'a> where - F: Fn(&S::Item) -> bool + 'a, + F: Fn(&Item) -> bool + 'a, { self.filter(move |t| ready(f(t))) } #[inline] - fn ready_filter_map(self, f: F) -> FilterMap>, impl FnMut(S::Item) -> Ready>> + fn ready_filter_map(self, f: F) -> FilterMap>, impl FnMut(Item) -> Ready>> where - F: Fn(S::Item) -> Option, + F: Fn(Item) -> Option, { self.filter_map(move |t| ready(f(t))) } #[inline] - fn ready_fold(self, init: T, f: F) -> Fold, T, impl FnMut(T, S::Item) -> Ready> + fn ready_fold(self, init: T, f: F) -> Fold, T, impl FnMut(T, Item) -> Ready> where - F: Fn(T, S::Item) -> T, + F: Fn(T, Item) -> T, { self.fold(init, move |a, t| ready(f(a, t))) } #[inline] #[allow(clippy::unit_arg)] - fn ready_for_each(self, mut f: F) -> ForEach, impl FnMut(S::Item) -> Ready<()>> + fn ready_for_each(self, mut f: F) -> ForEach, impl FnMut(Item) -> Ready<()>> where - F: FnMut(S::Item), + F: FnMut(Item), { self.for_each(move |t| ready(f(t))) } #[inline] - fn ready_take_while<'a, F>(self, f: F) -> TakeWhile, impl FnMut(&S::Item) -> Ready + 'a> + fn ready_take_while<'a, F>(self, f: F) -> TakeWhile, impl FnMut(&Item) -> Ready + 'a> where - F: Fn(&S::Item) -> bool + 'a, + F: Fn(&Item) -> bool + 'a, { self.take_while(move |t| ready(f(t))) } #[inline] - fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile, impl FnMut(&S::Item) -> Ready + 'a> + fn ready_scan( + self, init: T, f: F, + ) -> Scan>, impl FnMut(&mut T, Item) -> Ready>> where - F: Fn(&S::Item) -> bool + 'a, + F: Fn(&mut T, Item) -> Option, + { + self.scan(init, move |s, t| ready(f(s, t))) + } + + fn ready_scan_each( + self, init: T, f: F, + ) -> Scan>, impl FnMut(&mut T, Item) -> Ready>> + where + F: Fn(&mut T, &Item), + { + self.ready_scan(init, move |s, t| { + f(s, &t); + Some(t) + }) + } + + #[inline] + fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile, impl FnMut(&Item) -> Ready + 'a> + where + F: Fn(&Item) -> bool + 'a, { self.skip_while(move |t| ready(f(t))) } diff --git a/src/core/utils/stream/tools.rs b/src/core/utils/stream/tools.rs new file mode 100644 index 00000000..cc6b7ca9 --- /dev/null +++ b/src/core/utils/stream/tools.rs @@ -0,0 +1,80 @@ +//! StreamTools for futures::Stream + +use std::{collections::HashMap, hash::Hash}; + +use futures::{Future, Stream, StreamExt}; + +use super::ReadyExt; +use crate::expected; + +/// StreamTools +/// +/// This interface is not necessarily complete; feel free to add as-needed. +pub trait Tools +where + Self: Stream + Send + Sized, + ::Item: Send, +{ + fn counts(self) -> impl Future> + Send + where + ::Item: Eq + Hash; + + fn counts_by(self, f: F) -> impl Future> + Send + where + F: Fn(Item) -> K + Send, + K: Eq + Hash + Send; + + fn counts_by_with_cap(self, f: F) -> impl Future> + Send + where + F: Fn(Item) -> K + Send, + K: Eq + Hash + Send; + + fn counts_with_cap(self) -> impl Future> + Send + where + ::Item: Eq + Hash; +} + +impl Tools for S +where + S: Stream + Send + Sized, + ::Item: Send, +{ + #[inline] + fn counts(self) -> impl Future> + Send + where + ::Item: Eq + Hash, + { + self.counts_with_cap::<0>() + } + + #[inline] + fn counts_by(self, f: F) -> impl Future> + Send + where + F: Fn(Item) -> K + Send, + K: Eq + Hash + Send, + { + self.counts_by_with_cap::<0, K, F>(f) + } + + #[inline] + fn counts_by_with_cap(self, f: F) -> impl Future> + Send + where + F: Fn(Item) -> K + Send, + K: Eq + Hash + Send, + { + self.map(f).counts_with_cap::() + } + + #[inline] + fn counts_with_cap(self) -> impl Future> + Send + where + ::Item: Eq + Hash, + { + self.ready_fold(HashMap::with_capacity(CAP), |mut counts, item| { + let entry = counts.entry(item).or_default(); + let value = *entry; + *entry = expected!(value + 1); + counts + }) + } +} diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index eedff861..25388084 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -4,7 +4,7 @@ use std::{collections::HashSet, sync::Arc}; use conduit::{ err, - utils::{stream::TryIgnore, ReadyExt}, + utils::{stream::TryIgnore, ReadyExt, StreamTools}, warn, Result, }; use data::Data; @@ -495,11 +495,13 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub fn servers_invite_via<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + type KeyVal<'a> = (Ignore, Vec<&'a ServerName>); + self.db .roomid_inviteviaservers .stream_prefix_raw(room_id) .ignore_err() - .map(|(_, servers): (Ignore, Vec<&ServerName>)| &**(servers.last().expect("at least one servername"))) + .map(|(_, servers): KeyVal<'_>| *servers.last().expect("at least one server")) } /// Gets up to three servers that are likely to be in the room in the @@ -525,16 +527,14 @@ impl Service { let mut servers: Vec = self .room_members(room_id) - .collect::>() - .await - .iter() .counts_by(|user| user.server_name().to_owned()) - .iter() + .await + .into_iter() .sorted_by_key(|(_, users)| *users) - .map(|(server, _)| server.to_owned()) + .map(|(server, _)| server) .rev() .take(3) - .collect_vec(); + .collect(); if let Some(server) = most_powerful_user_server { servers.insert(0, server); From 96fcf7f94d65e93fdcb23acd3f52945813dbc18e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 1 Oct 2024 04:20:31 +0000 Subject: [PATCH 0853/2091] add rocksdb secondary; fix read_only mode. Signed-off-by: Jason Volk --- src/core/config/mod.rs | 3 +++ src/database/database.rs | 8 ++++++++ src/database/engine.rs | 11 +++++++++-- src/service/emergency/mod.rs | 4 ++++ src/service/globals/mod.rs | 3 +++ 5 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index d2d583a8..d8e1c7d9 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -236,6 +236,8 @@ pub struct Config { #[serde(default)] pub rocksdb_read_only: bool, #[serde(default)] + pub rocksdb_secondary: bool, + #[serde(default)] pub rocksdb_compaction_prio_idle: bool, #[serde(default = "true_fn")] pub rocksdb_compaction_ioprio_idle: bool, @@ -752,6 +754,7 @@ impl fmt::Display for Config { line("RocksDB Recovery Mode", &self.rocksdb_recovery_mode.to_string()); line("RocksDB Repair Mode", &self.rocksdb_repair.to_string()); line("RocksDB Read-only Mode", &self.rocksdb_read_only.to_string()); + line("RocksDB Secondary Mode", &self.rocksdb_secondary.to_string()); line( "RocksDB Compaction Idle Priority", &self.rocksdb_compaction_prio_idle.to_string(), diff --git a/src/database/database.rs b/src/database/database.rs index ac6f62e9..4c29c840 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -38,6 +38,14 @@ impl Database { #[inline] pub fn iter_maps(&self) -> impl Iterator + Send + '_ { self.map.iter() } + + #[inline] + #[must_use] + pub fn is_read_only(&self) -> bool { self.db.secondary || self.db.read_only } + + #[inline] + #[must_use] + pub fn is_secondary(&self) -> bool { self.db.secondary } } impl Index<&str> for Database { diff --git a/src/database/engine.rs b/src/database/engine.rs index edf077fc..99d971ed 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -28,6 +28,8 @@ pub struct Engine { cfs: Mutex>, pub(crate) db: Db, corks: AtomicU32, + pub(super) read_only: bool, + pub(super) secondary: bool, } pub(crate) type Db = DBWithThreadMode; @@ -80,10 +82,13 @@ impl Engine { .collect::>(); debug!("Opening database..."); + let path = &config.database_path; let res = if config.rocksdb_read_only { - Db::open_cf_for_read_only(&db_opts, &config.database_path, cfs.clone(), false) + Db::open_cf_descriptors_read_only(&db_opts, path, cfds, false) + } else if config.rocksdb_secondary { + Db::open_cf_descriptors_as_secondary(&db_opts, path, path, cfds) } else { - Db::open_cf_descriptors(&db_opts, &config.database_path, cfds) + Db::open_cf_descriptors(&db_opts, path, cfds) }; let db = res.or_else(or_else)?; @@ -103,6 +108,8 @@ impl Engine { cfs: Mutex::new(cfs), db, corks: AtomicU32::new(0), + read_only: config.rocksdb_read_only, + secondary: config.rocksdb_secondary, })) } diff --git a/src/service/emergency/mod.rs b/src/service/emergency/mod.rs index 98020bc2..c99a0891 100644 --- a/src/service/emergency/mod.rs +++ b/src/service/emergency/mod.rs @@ -32,6 +32,10 @@ impl crate::Service for Service { } async fn worker(self: Arc) -> Result<()> { + if self.services.globals.is_read_only() { + return Ok(()); + } + self.set_emergency_access() .await .inspect_err(|e| error!("Could not set the configured emergency password for the conduit user: {e}"))?; diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index f777901f..f24e8a27 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -329,4 +329,7 @@ impl Service { #[inline] pub fn server_is_ours(&self, server_name: &ServerName) -> bool { server_name == self.config.server_name } + + #[inline] + pub fn is_read_only(&self) -> bool { self.db.db.is_read_only() } } From 26dcab272d04eff968997a94f90636df389ffda6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 1 Oct 2024 02:47:39 +0000 Subject: [PATCH 0854/2091] various cleanup tweaks/fixes Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/admin/query/room_alias.rs | 10 ++++- src/admin/room/alias.rs | 4 +- src/admin/room/directory.rs | 8 ++-- src/admin/room/info.rs | 10 ++--- src/admin/room/moderation.rs | 4 +- src/api/client/keys.rs | 2 +- src/api/client/membership.rs | 15 ++++--- src/api/client/search.rs | 4 +- src/api/client/sync.rs | 67 +++++++++++++--------------- src/api/client/user_directory.rs | 11 ++--- src/core/Cargo.toml | 1 + src/core/error/mod.rs | 2 + src/service/appservice/data.rs | 4 +- src/service/globals/migrations.rs | 11 ++--- src/service/rooms/state_cache/mod.rs | 28 +++--------- src/service/rooms/timeline/mod.rs | 1 - src/service/users/mod.rs | 2 + 18 files changed, 86 insertions(+), 99 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 043d9704..065aa1e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -709,6 +709,7 @@ dependencies = [ "serde", "serde_json", "serde_regex", + "serde_yaml", "thiserror", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index 05fac42c..382e4a78 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -43,8 +43,13 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) room_id, } => { let timer = tokio::time::Instant::now(); - let results = services.rooms.alias.local_aliases_for_room(&room_id); - let aliases: Vec<_> = results.collect().await; + let aliases: Vec<_> = services + .rooms + .alias + .local_aliases_for_room(&room_id) + .map(ToOwned::to_owned) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -57,6 +62,7 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) .rooms .alias .all_local_aliases() + .map(|(room_id, alias)| (room_id.to_owned(), alias.to_owned())) .collect::>() .await; let query_time = timer.elapsed(); diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 34b6c42e..1ccde47d 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -119,12 +119,12 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> room_id, } => { if let Some(room_id) = room_id { - let aliases = services + let aliases: Vec = services .rooms .alias .local_aliases_for_room(&room_id) .map(Into::into) - .collect::>() + .collect() .await; let plain_list = aliases.iter().fold(String::new(), |mut output, alias| { diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index 7ccdea6f..1080356a 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -47,22 +47,22 @@ pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_> } => { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); - let mut rooms = services + let mut rooms: Vec<_> = services .rooms .directory .public_rooms() .then(|room_id| get_room_info(services, room_id)) - .collect::>() + .collect() .await; rooms.sort_by_key(|r| r.1); rooms.reverse(); - let rooms = rooms + let rooms: Vec<_> = rooms .into_iter() .skip(page.saturating_sub(1).saturating_mul(PAGE_SIZE)) .take(PAGE_SIZE) - .collect::>(); + .collect(); if rooms.is_empty() { return Ok(RoomMessageEventContent::text_plain("No more rooms.")); diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index fc0619e3..13a74a9d 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -42,14 +42,12 @@ async fn list_joined_members(&self, room_id: Box, local_only: bool) -> R .state_cache .room_members(&room_id) .ready_filter(|user_id| { - if local_only { - self.services.globals.user_is_local(user_id) - } else { - true - } + local_only + .then(|| self.services.globals.user_is_local(user_id)) + .unwrap_or(true) }) + .map(ToOwned::to_owned) .filter_map(|user_id| async move { - let user_id = user_id.to_owned(); Some(( self.services .users diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index 9a772da4..cfc048bd 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -555,13 +555,13 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> #[admin_command] async fn list_banned_rooms(&self, no_details: bool) -> Result { - let room_ids = self + let room_ids: Vec = self .services .rooms .metadata .list_banned_rooms() .map(Into::into) - .collect::>() + .collect() .await; if room_ids.is_empty() { diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index abf2a22f..254d92cc 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -244,7 +244,7 @@ pub(crate) async fn get_key_changes_route( device_list_updates.extend( services .users - .keys_changed(room_id.as_ref(), from, Some(to)) + .keys_changed(room_id.as_str(), from, Some(to)) .map(ToOwned::to_owned) .collect::>() .await, diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 5a5d436f..6e3bc894 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -167,12 +167,12 @@ pub(crate) async fn join_room_by_id_route( .await?; // There is no body.server_name for /roomId/join - let mut servers = services + let mut servers: Vec<_> = services .rooms .state_cache .servers_invite_via(&body.room_id) .map(ToOwned::to_owned) - .collect::>() + .collect() .await; servers.extend( @@ -641,12 +641,13 @@ pub(crate) async fn joined_members_route( .rooms .state_cache .room_members(&body.room_id) + .map(ToOwned::to_owned) .then(|user| async move { ( - user.to_owned(), + user.clone(), RoomMember { - display_name: services.users.displayname(user).await.ok(), - avatar_url: services.users.avatar_url(user).await.ok(), + display_name: services.users.displayname(&user).await.ok(), + avatar_url: services.users.avatar_url(&user).await.ok(), }, ) }) @@ -1575,7 +1576,7 @@ pub(crate) async fn invite_helper( // Make a user leave all their joined rooms, forgets all rooms, and ignores // errors pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { - let all_rooms = services + let all_rooms: Vec<_> = services .rooms .state_cache .rooms_joined(user_id) @@ -1587,7 +1588,7 @@ pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { .rooms_invited(user_id) .map(|(r, _)| r), ) - .collect::>() + .collect() .await; for room_id in all_rooms { diff --git a/src/api/client/search.rs b/src/api/client/search.rs index 7a061d49..b073640e 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -77,14 +77,14 @@ pub(crate) async fn search_events_route( .user_can_see_state_events(sender_user, room_id) .await { - let room_state = services + let room_state: Vec<_> = services .rooms .state_accessor .room_state_full(room_id) .await? .values() .map(|pdu| pdu.to_state_event()) - .collect::>(); + .collect(); debug!("Room state: {:?}", room_state); diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 53d4f3c3..adb4d8da 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -7,13 +7,14 @@ use std::{ use axum::extract::State; use conduit::{ debug, err, error, is_equal_to, + result::IntoIsOk, utils::{ math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, - IterStream, ReadyExt, + BoolExt, IterStream, ReadyExt, TryFutureExtExt, }, warn, PduCount, }; -use futures::{pin_mut, StreamExt}; +use futures::{pin_mut, FutureExt, StreamExt, TryFutureExt}; use ruma::{ api::client::{ error::ErrorKind, @@ -172,12 +173,12 @@ pub(crate) async fn sync_events_route( process_presence_updates(&services, &mut presence_updates, since, &sender_user).await?; } - let all_joined_rooms = services + let all_joined_rooms: Vec<_> = services .rooms .state_cache .rooms_joined(&sender_user) .map(ToOwned::to_owned) - .collect::>() + .collect() .await; // Coalesce database writes for the remainder of this scope. @@ -869,15 +870,13 @@ async fn load_joined_room( .rooms .state_cache .room_members(room_id) - .ready_filter(|user_id| { - // Don't send key updates from the sender to the sender - sender_user != *user_id - }) - .filter_map(|user_id| async move { - // Only send keys if the sender doesn't share an encrypted room with the target - // already - (!share_encrypted_room(services, sender_user, user_id, Some(room_id)).await) - .then_some(user_id.to_owned()) + // Don't send key updates from the sender to the sender + .ready_filter(|user_id| sender_user != *user_id) + // Only send keys if the sender doesn't share an encrypted room with the target + // already + .filter_map(|user_id| { + share_encrypted_room(services, sender_user, user_id, Some(room_id)) + .map(|res| res.or_some(user_id.to_owned())) }) .collect::>() .await, @@ -1117,13 +1116,12 @@ async fn share_encrypted_room( .user .get_shared_rooms(sender_user, user_id) .ready_filter(|&room_id| Some(room_id) != ignore_room) - .any(|other_room_id| async move { + .any(|other_room_id| { services .rooms .state_accessor .room_state_get(other_room_id, &StateEventType::RoomEncryption, "") - .await - .is_ok() + .map(Result::into_is_ok) }) .await } @@ -1178,20 +1176,20 @@ pub(crate) async fn sync_events_v4_route( .sync .update_sync_request_with_cache(sender_user.clone(), sender_device.clone(), &mut body); - let all_joined_rooms = services + let all_joined_rooms: Vec<_> = services .rooms .state_cache .rooms_joined(&sender_user) .map(ToOwned::to_owned) - .collect::>() + .collect() .await; - let all_invited_rooms = services + let all_invited_rooms: Vec<_> = services .rooms .state_cache .rooms_invited(&sender_user) .map(|r| r.0) - .collect::>() + .collect() .await; let all_rooms = all_joined_rooms @@ -1364,15 +1362,13 @@ pub(crate) async fn sync_events_v4_route( .rooms .state_cache .room_members(room_id) - .ready_filter(|user_id| { - // Don't send key updates from the sender to the sender - sender_user != user_id - }) - .filter_map(|user_id| async move { - // Only send keys if the sender doesn't share an encrypted room with the target - // already - (!share_encrypted_room(&services, sender_user, user_id, Some(room_id)).await) - .then_some(user_id.to_owned()) + // Don't send key updates from the sender to the sender + .ready_filter(|user_id| sender_user != user_id) + // Only send keys if the sender doesn't share an encrypted room with the target + // already + .filter_map(|user_id| { + share_encrypted_room(&services, sender_user, user_id, Some(room_id)) + .map(|res| res.or_some(user_id.to_owned())) }) .collect::>() .await, @@ -1650,26 +1646,25 @@ pub(crate) async fn sync_events_v4_route( .await; // Heroes - let heroes = services + let heroes: Vec<_> = services .rooms .state_cache .room_members(room_id) .ready_filter(|member| member != &sender_user) - .filter_map(|member| async move { + .filter_map(|user_id| { services .rooms .state_accessor - .get_member(room_id, member) - .await - .map(|memberevent| SlidingSyncRoomHero { - user_id: member.to_owned(), + .get_member(room_id, user_id) + .map_ok(|memberevent| SlidingSyncRoomHero { + user_id: user_id.into(), name: memberevent.displayname, avatar: memberevent.avatar_url, }) .ok() }) .take(5) - .collect::>() + .collect() .await; let name = match heroes.len().cmp(&(1_usize)) { diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index 8ea7f1b8..868811a3 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -1,4 +1,5 @@ use axum::extract::State; +use conduit::utils::TryFutureExtExt; use futures::{pin_mut, StreamExt}; use ruma::{ api::client::user_directory::search_users, @@ -56,16 +57,12 @@ pub(crate) async fn search_users_route( .rooms .state_cache .rooms_joined(&user.user_id) - .any(|room| async move { + .any(|room| { services .rooms .state_accessor - .room_state_get(room, &StateEventType::RoomJoinRules, "") - .await - .map_or(false, |event| { - serde_json::from_str(event.content.get()) - .map_or(false, |r: RoomJoinRulesEventContent| r.join_rule == JoinRule::Public) - }) + .room_state_get_content::(room, &StateEventType::RoomJoinRules, "") + .map_ok_or(false, |content| content.join_rule == JoinRule::Public) }) .await; diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index cb957bc9..4fe413e9 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -83,6 +83,7 @@ ruma.workspace = true sanitize-filename.workspace = true serde_json.workspace = true serde_regex.workspace = true +serde_yaml.workspace = true serde.workspace = true thiserror.workspace = true tikv-jemallocator.optional = true diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 79e3d5b4..ad7f9f3c 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -75,6 +75,8 @@ pub enum Error { TracingFilter(#[from] tracing_subscriber::filter::ParseError), #[error("Tracing reload error: {0}")] TracingReload(#[from] tracing_subscriber::reload::Error), + #[error(transparent)] + Yaml(#[from] serde_yaml::Error), // ruma/conduwuit #[error("Arithmetic operation failed: {0}")] diff --git a/src/service/appservice/data.rs b/src/service/appservice/data.rs index 4eb9d09e..8fb7d958 100644 --- a/src/service/appservice/data.rs +++ b/src/service/appservice/data.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use conduit::{err, utils::stream::TryIgnore, Result}; -use database::{Database, Deserialized, Map}; +use database::{Database, Map}; use futures::Stream; use ruma::api::appservice::Registration; @@ -40,7 +40,7 @@ impl Data { self.id_appserviceregistrations .get(id) .await - .deserialized() + .and_then(|ref bytes| serde_yaml::from_slice(bytes).map_err(Into::into)) .map_err(|e| err!(Database("Invalid appservice {id:?} registration: {e:?}"))) } diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 469159fc..fc6e477b 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -9,7 +9,7 @@ use itertools::Itertools; use ruma::{ events::{push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType}, push::Ruleset, - UserId, + OwnedUserId, UserId, }; use crate::{media, Services}; @@ -385,11 +385,12 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) for room_id in &room_ids { debug_info!("Fixing room {room_id}"); - let users_in_room = services + let users_in_room: Vec = services .rooms .state_cache .room_members(room_id) - .collect::>() + .map(ToOwned::to_owned) + .collect() .await; let joined_members = users_in_room @@ -418,12 +419,12 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) .collect::>() .await; - for user_id in joined_members { + for user_id in &joined_members { debug_info!("User is joined, marking as joined"); services.rooms.state_cache.mark_as_joined(user_id, room_id); } - for user_id in non_joined_members { + for user_id in &non_joined_members { debug_info!("User is left or banned, marking as left"); services.rooms.state_cache.mark_as_left(user_id, room_id); } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 25388084..dbe38561 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -648,35 +648,19 @@ impl Service { self.db.userroomid_leftstate.remove(&userroom_id); self.db.roomuserid_leftcount.remove(&roomuser_id); - if let Some(servers) = invite_via { - let mut prev_servers = self - .servers_invite_via(room_id) - .map(ToOwned::to_owned) - .collect::>() - .await; - #[allow(clippy::redundant_clone)] // this is a necessary clone? - prev_servers.append(servers.clone().as_mut()); - let servers = prev_servers.iter().rev().unique().rev().collect_vec(); - - let servers = servers - .iter() - .map(|server| server.as_bytes()) - .collect_vec() - .join(&[0xFF][..]); - - self.db - .roomid_inviteviaservers - .insert(room_id.as_bytes(), &servers); + if let Some(servers) = invite_via.as_deref() { + self.add_servers_invite_via(room_id, servers).await; } } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self, servers), level = "debug")] pub async fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) { - let mut prev_servers = self + let mut prev_servers: Vec<_> = self .servers_invite_via(room_id) .map(ToOwned::to_owned) - .collect::>() + .collect() .await; + prev_servers.extend(servers.to_owned()); prev_servers.sort_unstable(); prev_servers.dedup(); diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 5360d2c9..6a26a1d5 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -408,7 +408,6 @@ impl Service { .get(None, user, GlobalAccountDataEventType::PushRules.to_string().into()) .await .and_then(|event| serde_json::from_str::(event.get()).map_err(Into::into)) - .map_err(|e| err!(Database(warn!(?user, ?e, "Invalid push rules event in db for user")))) .map_or_else(|_| Ruleset::server_default(user), |ev: PushRulesEvent| ev.content.global); let mut highlight = false; diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index eb77ef35..438c220b 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -623,7 +623,9 @@ impl Service { pub async fn mark_device_key_update(&self, user_id: &UserId) { let count = self.services.globals.next_count().unwrap().to_be_bytes(); + let rooms_joined = self.services.state_cache.rooms_joined(user_id); + pin_mut!(rooms_joined); while let Some(room_id) = rooms_joined.next().await { // Don't send key updates to unencrypted rooms From ab06701ed08862bef04bf06800dbf021bd317497 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 1 Oct 2024 22:37:01 +0000 Subject: [PATCH 0855/2091] refactor multi-get to handle result type Signed-off-by: Jason Volk --- src/database/keyval.rs | 8 ------ src/database/map/get.rs | 45 ++++++++++++++++------------------ src/service/rooms/short/mod.rs | 39 +++++++++-------------------- 3 files changed, 33 insertions(+), 59 deletions(-) diff --git a/src/database/keyval.rs b/src/database/keyval.rs index c9d25977..a288f184 100644 --- a/src/database/keyval.rs +++ b/src/database/keyval.rs @@ -3,10 +3,6 @@ use serde::Deserialize; use crate::de; -pub(crate) type OwnedKeyVal = (Vec, Vec); -pub(crate) type OwnedKey = Vec; -pub(crate) type OwnedVal = Vec; - pub type KeyVal<'a, K = &'a Slice, V = &'a Slice> = (Key<'a, K>, Val<'a, V>); pub type Key<'a, T = &'a Slice> = T; pub type Val<'a, T = &'a Slice> = T; @@ -72,10 +68,6 @@ where de::from_slice::(val) } -#[inline] -#[must_use] -pub fn to_owned(kv: KeyVal<'_>) -> OwnedKeyVal { (kv.0.to_owned(), kv.1.to_owned()) } - #[inline] pub fn key(kv: KeyVal<'_, K, V>) -> Key<'_, K> { kv.0 } diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 71489402..72382e36 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -3,14 +3,12 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write}; use arrayvec::ArrayVec; use conduit::{err, implement, Result}; use futures::future::ready; +use rocksdb::DBPinnableSlice; use serde::Serialize; -use crate::{ - keyval::{OwnedKey, OwnedVal}, - ser, - util::{map_err, or_else}, - Handle, -}; +use crate::{ser, util, Handle}; + +type RocksdbResult<'a> = Result>, rocksdb::Error>; /// Fetch a value from the database into cache, returning a reference-handle /// asynchronously. The key is serialized into an allocated buffer to perform @@ -68,17 +66,17 @@ pub fn get_blocking(&self, key: &K) -> Result> where K: AsRef<[u8]> + ?Sized + Debug, { - self.db + let res = self .db - .get_pinned_cf_opt(&self.cf(), key, &self.read_options) - .map_err(map_err)? - .map(Handle::from) - .ok_or(err!(Request(NotFound("Not found in database")))) + .db + .get_pinned_cf_opt(&self.cf(), key, &self.read_options); + + into_result_handle(res) } #[implement(super::Map)] #[tracing::instrument(skip(self, keys), fields(%self), level = "trace")] -pub fn get_batch_blocking<'a, I, K>(&self, keys: I) -> Vec> +pub fn get_batch_blocking<'a, I, K>(&self, keys: I) -> Vec>> where I: Iterator + ExactSizeIterator + Send + Debug, K: AsRef<[u8]> + Sized + Debug + 'a, @@ -87,19 +85,18 @@ where // comparator**. const SORTED: bool = false; - let mut ret: Vec> = Vec::with_capacity(keys.len()); let read_options = &self.read_options; - for res in self - .db + self.db .db .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) - { - match res { - Ok(Some(res)) => ret.push(Some((*res).to_vec())), - Ok(None) => ret.push(None), - Err(e) => or_else(e).expect("database multiget error"), - } - } - - ret + .into_iter() + .map(into_result_handle) + .collect() +} + +fn into_result_handle(result: RocksdbResult<'_>) -> Result> { + result + .map_err(util::map_err)? + .map(Handle::from) + .ok_or(err!(Request(NotFound("Not found in database")))) } diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 66da3948..825ee109 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{err, implement, utils, Error, Result}; +use conduit::{err, implement, utils, Result}; use database::{Deserialized, Map}; use ruma::{events::StateEventType, EventId, RoomId}; @@ -69,41 +69,26 @@ pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { #[implement(Service)] pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Vec { - let mut ret: Vec = Vec::with_capacity(event_ids.len()); - let keys = event_ids - .iter() - .map(|id| id.as_bytes()) - .collect::>(); - - for (i, short) in self - .db + self.db .eventid_shorteventid - .get_batch_blocking(keys.iter()) - .iter() + .get_batch_blocking(event_ids.iter()) + .into_iter() .enumerate() - { - match short { - Some(short) => ret.push( - utils::u64_from_bytes(short) - .map_err(|_| Error::bad_database("Invalid shorteventid in db.")) - .unwrap(), - ), - None => { + .map(|(i, result)| match result { + Ok(ref short) => utils::u64_from_u8(short), + Err(_) => { let short = self.services.globals.next_count().unwrap(); self.db .eventid_shorteventid - .insert(keys[i], &short.to_be_bytes()); + .insert(event_ids[i], &short.to_be_bytes()); self.db .shorteventid_eventid - .insert(&short.to_be_bytes(), keys[i]); + .insert(&short.to_be_bytes(), event_ids[i]); - debug_assert!(ret.len() == i, "position of result must match input"); - ret.push(short); + short }, - } - } - - ret + }) + .collect() } #[implement(Service)] From 36677bb9828038294d06f2292eef755139216c40 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 1 Oct 2024 23:19:47 +0000 Subject: [PATCH 0856/2091] optimize auth_chain short_id to event_id translation step Signed-off-by: Jason Volk --- src/service/rooms/auth_chain/mod.rs | 30 ++++++++++++++++++-------- src/service/rooms/event_handler/mod.rs | 16 +++++++------- src/service/rooms/short/mod.rs | 17 +++++++++++++++ 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index eae13b74..f3861ca3 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use conduit::{debug, debug_error, trace, utils::IterStream, validated, warn, Err, Result}; -use futures::{FutureExt, Stream, StreamExt}; +use futures::Stream; use ruma::{EventId, RoomId}; use self::data::Data; @@ -40,15 +40,27 @@ impl Service { pub async fn event_ids_iter( &self, room_id: &RoomId, starting_events: &[&EventId], ) -> Result> + Send + '_> { - let chain = self.get_auth_chain(room_id, starting_events).await?; - let iter = chain.into_iter().stream().filter_map(|sid| { - self.services - .short - .get_eventid_from_short(sid) - .map(Result::ok) - }); + let stream = self + .get_event_ids(room_id, starting_events) + .await? + .into_iter() + .stream(); - Ok(iter) + Ok(stream) + } + + pub async fn get_event_ids(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result>> { + let chain = self.get_auth_chain(room_id, starting_events).await?; + let event_ids = self + .services + .short + .multi_get_eventid_from_short(&chain) + .await + .into_iter() + .filter_map(Result::ok) + .collect(); + + Ok(event_ids) } #[tracing::instrument(skip_all, name = "auth_chain")] diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 57b87706..4708a86c 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -797,13 +797,13 @@ impl Service { for state in &fork_states { let starting_events: Vec<&EventId> = state.values().map(Borrow::borrow).collect(); - let auth_chain = self + let auth_chain: HashSet> = self .services .auth_chain - .event_ids_iter(room_id, &starting_events) + .get_event_ids(room_id, &starting_events) .await? - .collect::>>() - .await; + .into_iter() + .collect(); auth_chain_sets.push(auth_chain); } @@ -983,13 +983,13 @@ impl Service { starting_events.push(id.borrow()); } - let auth_chain = self + let auth_chain: HashSet> = self .services .auth_chain - .event_ids_iter(room_id, &starting_events) + .get_event_ids(room_id, &starting_events) .await? - .collect() - .await; + .into_iter() + .collect(); auth_chain_sets.push(auth_chain); fork_states.push(state); diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 825ee109..20082da2 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -141,6 +141,23 @@ pub async fn get_eventid_from_short(&self, shorteventid: u64) -> Result Vec>> { + const BUFSIZE: usize = size_of::(); + + let keys: Vec<[u8; BUFSIZE]> = shorteventid + .iter() + .map(|short| short.to_be_bytes()) + .collect(); + + self.db + .shorteventid_eventid + .get_batch_blocking(keys.iter()) + .into_iter() + .map(Deserialized::deserialized) + .collect() +} + #[implement(Service)] pub async fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(StateEventType, String)> { const BUFSIZE: usize = size_of::(); From 83119526291f25a78d67a15f638eeaedc0b10b2d Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 28 Sep 2024 18:30:40 -0400 Subject: [PATCH 0857/2091] bump ruma, cargo.lock, and deps Signed-off-by: strawberry --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- src/api/client/capabilities.rs | 8 +++++++- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 065aa1e4..4d40c458 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "assign", "js_int", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "js_int", "ruma-common", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "as_variant", "assign", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "as_variant", "base64 0.22.1", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "bytes", "http", @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "js_int", "thiserror", @@ -3113,7 +3113,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "js_int", "ruma-common", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "cfg-if", "once_cell", @@ -3139,7 +3139,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "js_int", "ruma-common", @@ -3151,7 +3151,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "headers", "http", @@ -3164,7 +3164,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3180,7 +3180,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7db44989d68406393270d3a91815597385d3acb#e7db44989d68406393270d3a91815597385d3acb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" dependencies = [ "futures-util", "itertools 0.12.1", diff --git a/Cargo.toml b/Cargo.toml index 3bfb3bc8..28e280cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "e7db44989d68406393270d3a91815597385d3acb" +rev = "ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" features = [ "compat", "rand", diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 83e1dc7e..89157e47 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -3,7 +3,8 @@ use std::collections::BTreeMap; use axum::extract::State; use ruma::{ api::client::discovery::get_capabilities::{ - self, Capabilities, RoomVersionStability, RoomVersionsCapability, ThirdPartyIdChangesCapability, + self, Capabilities, GetLoginTokenCapability, RoomVersionStability, RoomVersionsCapability, + ThirdPartyIdChangesCapability, }, RoomVersionId, }; @@ -43,6 +44,11 @@ pub(crate) async fn get_capabilities_route( enabled: false, }; + // we dont support generating tokens yet + capabilities.get_login_token = GetLoginTokenCapability { + enabled: false, + }; + // MSC4133 capability capabilities .set("uk.tcpip.msc4133.profile_fields", json!({"enabled": true})) From fafe32089980eefc5bda1cb8991a0be762c30e6b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 28 Sep 2024 19:38:35 -0400 Subject: [PATCH 0858/2091] send EDUs to appservices if in events to_device is not supported yet Signed-off-by: strawberry --- src/service/sending/sender.rs | 39 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 4db9922a..3a401995 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -13,9 +13,14 @@ use conduit::{ }; use futures::{future::BoxFuture, pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; use ruma::{ - api::federation::transactions::{ - edu::{DeviceListUpdateContent, Edu, PresenceContent, PresenceUpdate, ReceiptContent, ReceiptData, ReceiptMap}, - send_transaction_message, + api::{ + appservice::event::push_events::v1::Edu as RumaEdu, + federation::transactions::{ + edu::{ + DeviceListUpdateContent, Edu, PresenceContent, PresenceUpdate, ReceiptContent, ReceiptData, ReceiptMap, + }, + send_transaction_message, + }, }, device_id, events::{push_rules::PushRulesEvent, receipt::ReceiptType, AnySyncEphemeralRoomEvent, GlobalAccountDataEventType}, @@ -441,7 +446,18 @@ impl Service { return Err((dest.clone(), err!(Database(warn!(?id, "Missing appservice registration"))))); }; - let mut pdu_jsons = Vec::new(); + let mut pdu_jsons = Vec::with_capacity( + events + .iter() + .filter(|event| matches!(event, SendingEvent::Pdu(_))) + .count(), + ); + let mut edu_jsons: Vec = Vec::with_capacity( + events + .iter() + .filter(|event| matches!(event, SendingEvent::Edu(_))) + .count(), + ); for event in &events { match event { SendingEvent::Pdu(pdu_id) => { @@ -449,10 +465,12 @@ impl Service { pdu_jsons.push(pdu.to_room_event()); } }, - SendingEvent::Edu(_) | SendingEvent::Flush => { - // Appservices don't need EDUs (?) and flush only; - // no new content + SendingEvent::Edu(edu) => { + if let Ok(edu) = serde_json::from_slice(edu) { + edu_jsons.push(edu); + } }, + SendingEvent::Flush => {}, // flush only; no new content } } @@ -466,7 +484,8 @@ impl Service { .collect::>(), )); - //debug_assert!(!pdu_jsons.is_empty(), "sending empty transaction"); + //debug_assert!(pdu_jsons.len() + edu_jsons.len() > 0, "sending empty + // transaction"); let client = &self.services.client.appservice; match appservice::send_request( client, @@ -474,8 +493,8 @@ impl Service { ruma::api::appservice::event::push_events::v1::Request { events: pdu_jsons, txn_id: txn_id.into(), - ephemeral: Vec::new(), - to_device: Vec::new(), + ephemeral: edu_jsons, + to_device: Vec::new(), // TODO }, ) .await From 890ee84f713c3f4905247934d5bfc277f61959cd Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 28 Sep 2024 21:44:38 -0400 Subject: [PATCH 0859/2091] dont send read receipts and typing indicators from ignored users Signed-off-by: strawberry --- src/api/client/sync.rs | 25 ++++++++++++++-- src/service/rooms/state_cache/mod.rs | 26 +--------------- src/service/rooms/typing/mod.rs | 45 +++++++++++++++++++++------- src/service/users/mod.rs | 32 ++++++++++++++++++-- 4 files changed, 88 insertions(+), 40 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index adb4d8da..1383f902 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -1011,15 +1011,27 @@ async fn load_joined_room( .rooms .read_receipt .readreceipts_since(room_id, since) - .map(|(_, _, v)| v) + .filter_map(|(read_user, _, v)| async move { + (!services + .users + .user_is_ignored(&read_user, sender_user) + .await) + .then_some(v) + }) .collect() .await; if services.rooms.typing.last_typing_update(room_id).await? > since { edus.push( serde_json::from_str( - &serde_json::to_string(&services.rooms.typing.typings_all(room_id).await?) - .expect("event is valid, we just created it"), + &serde_json::to_string( + &services + .rooms + .typing + .typings_all(room_id, sender_user) + .await?, + ) + .expect("event is valid, we just created it"), ) .expect("event is valid, we just created it"), ); @@ -1583,6 +1595,13 @@ pub(crate) async fn sync_events_v4_route( .rooms .read_receipt .readreceipts_since(room_id, *roomsince) + .filter_map(|(read_user, ts, v)| async move { + (!services + .users + .user_is_ignored(&read_user, sender_user) + .await) + .then_some((read_user, ts, v)) + }) .collect() .await; diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index dbe38561..b1a71caf 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -14,7 +14,6 @@ use itertools::Itertools; use ruma::{ events::{ direct::DirectEvent, - ignored_user_list::IgnoredUserListEvent, room::{ create::RoomCreateEventContent, member::{MembershipState, RoomMemberEventContent}, @@ -197,30 +196,7 @@ impl Service { }, MembershipState::Invite => { // We want to know if the sender is ignored by the receiver - let is_ignored = self - .services - .account_data - .get( - None, // Ignored users are in global account data - user_id, // Receiver - GlobalAccountDataEventType::IgnoredUserList - .to_string() - .into(), - ) - .await - .and_then(|event| { - serde_json::from_str::(event.get()) - .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) - }) - .map_or(false, |ignored| { - ignored - .content - .ignored_users - .iter() - .any(|(user, _details)| user == sender) - }); - - if is_ignored { + if self.services.users.user_is_ignored(sender, user_id).await { return Ok(()); } diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index bcfce616..8ee34f44 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -1,6 +1,11 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduit::{debug_info, trace, utils, Result, Server}; +use conduit::{ + debug_info, trace, + utils::{self, IterStream}, + Result, Server, +}; +use futures::StreamExt; use ruma::{ api::federation::transactions::edu::{Edu, TypingContent}, events::SyncEphemeralRoomEvent, @@ -8,7 +13,7 @@ use ruma::{ }; use tokio::sync::{broadcast, RwLock}; -use crate::{globals, sending, Dep}; +use crate::{globals, sending, users, Dep}; pub struct Service { server: Arc, @@ -23,6 +28,7 @@ pub struct Service { struct Services { globals: Dep, sending: Dep, + users: Dep, } impl crate::Service for Service { @@ -32,6 +38,7 @@ impl crate::Service for Service { services: Services { globals: args.depend::("globals"), sending: args.depend::("sending"), + users: args.depend::("users"), }, typing: RwLock::new(BTreeMap::new()), last_typing_update: RwLock::new(BTreeMap::new()), @@ -170,17 +177,35 @@ impl Service { /// Returns a new typing EDU. pub async fn typings_all( - &self, room_id: &RoomId, + &self, room_id: &RoomId, sender_user: &UserId, ) -> Result> { + let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); + + let Some(typing_indicators) = room_typing_indicators else { + return Ok(SyncEphemeralRoomEvent { + content: ruma::events::typing::TypingEventContent { + user_ids: Vec::new(), + }, + }); + }; + + let user_ids: Vec<_> = typing_indicators + .into_keys() + .stream() + .filter_map(|typing_user_id| async move { + (!self + .services + .users + .user_is_ignored(&typing_user_id, sender_user) + .await) + .then_some(typing_user_id) + }) + .collect() + .await; + Ok(SyncEphemeralRoomEvent { content: ruma::events::typing::TypingEventContent { - user_ids: self - .typing - .read() - .await - .get(room_id) - .map(|m| m.keys().cloned().collect()) - .unwrap_or_default(), + user_ids, }, }) } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 438c220b..1c079085 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -10,13 +10,13 @@ use futures::{pin_mut, FutureExt, Stream, StreamExt, TryFutureExt}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, - events::{AnyToDeviceEvent, StateEventType}, + events::{ignored_user_list::IgnoredUserListEvent, AnyToDeviceEvent, GlobalAccountDataEventType, StateEventType}, serde::Raw, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedDeviceKeyId, OwnedMxcUri, OwnedUserId, UInt, UserId, }; -use crate::{admin, globals, rooms, Dep}; +use crate::{account_data, admin, globals, rooms, Dep}; pub struct Service { services: Services, @@ -25,6 +25,7 @@ pub struct Service { struct Services { server: Arc, + account_data: Dep, admin: Dep, globals: Dep, state_accessor: Dep, @@ -58,6 +59,7 @@ impl crate::Service for Service { Ok(Arc::new(Self { services: Services { server: args.server.clone(), + account_data: args.depend::("account_data"), admin: args.depend::("admin"), globals: args.depend::("globals"), state_accessor: args.depend::("rooms::state_accessor"), @@ -91,6 +93,32 @@ impl crate::Service for Service { } impl Service { + /// Returns true/false based on whether the recipient/receiving user has + /// blocked the sender + pub async fn user_is_ignored(&self, sender_user: &UserId, recipient_user: &UserId) -> bool { + self.services + .account_data + .get( + None, + recipient_user, + GlobalAccountDataEventType::IgnoredUserList + .to_string() + .into(), + ) + .await + .and_then(|event| { + serde_json::from_str::(event.get()) + .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) + }) + .map_or(false, |ignored| { + ignored + .content + .ignored_users + .keys() + .any(|blocked_user| blocked_user == sender_user) + }) + } + /// Check if a user is an admin #[inline] pub async fn is_admin(&self, user_id: &UserId) -> bool { self.services.admin.user_is_admin(user_id).await } From 2083c38c764d5d144ff6355ce0688e8fe98d7d49 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 28 Sep 2024 22:12:17 -0400 Subject: [PATCH 0860/2091] dont send non-state events from ignored users over sync Signed-off-by: strawberry --- src/api/client/sync.rs | 91 +++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 1383f902..51df88a3 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -35,6 +35,7 @@ use ruma::{ presence::PresenceEvent, room::member::{MembershipState, RoomMemberEventContent}, AnyRawAccountDataEvent, StateEventType, TimelineEventType, + TimelineEventType::*, }, serde::Raw, state_res::Event, @@ -1004,8 +1005,31 @@ async fn load_joined_room( let room_events: Vec<_> = timeline_pdus .iter() - .map(|(_, pdu)| pdu.to_sync_room_event()) - .collect(); + .stream() + .filter_map(|(_, pdu)| async move { + // list of safe and common non-state events to ignore + if matches!( + &pdu.kind, + RoomMessage + | Sticker | CallInvite + | CallNotify | RoomEncrypted + | Image | File | Audio + | Voice | Video | UnstablePollStart + | PollStart | KeyVerificationStart + | Reaction | Emote + | Location + ) && services + .users + .user_is_ignored(&pdu.sender, sender_user) + .await + { + return None; + } + + Some(pdu.to_sync_room_event()) + }) + .collect() + .await; let mut edus: Vec<_> = services .rooms @@ -1144,11 +1168,11 @@ async fn share_encrypted_room( pub(crate) async fn sync_events_v4_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.expect("user is authenticated"); + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.expect("user is authenticated"); let mut body = body.body; // Setup watchers, so if there's no response, we can wait for them - let watcher = services.globals.watch(&sender_user, &sender_device); + let watcher = services.globals.watch(sender_user, &sender_device); let next_batch = services.globals.next_count()?; @@ -1191,7 +1215,7 @@ pub(crate) async fn sync_events_v4_route( let all_joined_rooms: Vec<_> = services .rooms .state_cache - .rooms_joined(&sender_user) + .rooms_joined(sender_user) .map(ToOwned::to_owned) .collect() .await; @@ -1199,7 +1223,7 @@ pub(crate) async fn sync_events_v4_route( let all_invited_rooms: Vec<_> = services .rooms .state_cache - .rooms_invited(&sender_user) + .rooms_invited(sender_user) .map(|r| r.0) .collect() .await; @@ -1213,7 +1237,7 @@ pub(crate) async fn sync_events_v4_route( if body.extensions.to_device.enabled.unwrap_or(false) { services .users - .remove_to_device_events(&sender_user, &sender_device, globalsince) + .remove_to_device_events(sender_user, &sender_device, globalsince) .await; } @@ -1232,7 +1256,7 @@ pub(crate) async fn sync_events_v4_route( if body.extensions.account_data.enabled.unwrap_or(false) { account_data.global = services .account_data - .changes_since(None, &sender_user, globalsince) + .changes_since(None, sender_user, globalsince) .await? .into_iter() .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) @@ -1244,7 +1268,7 @@ pub(crate) async fn sync_events_v4_route( room.clone(), services .account_data - .changes_since(Some(&room), &sender_user, globalsince) + .changes_since(Some(&room), sender_user, globalsince) .await? .into_iter() .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) @@ -1338,7 +1362,7 @@ pub(crate) async fn sync_events_v4_route( let user_id = UserId::parse(state_key.clone()) .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; - if user_id == sender_user { + if user_id == *sender_user { continue; } @@ -1350,7 +1374,7 @@ pub(crate) async fn sync_events_v4_route( match new_membership { MembershipState::Join => { // A new user joined an encrypted room - if !share_encrypted_room(&services, &sender_user, &user_id, Some(room_id)) + if !share_encrypted_room(&services, sender_user, &user_id, Some(room_id)) .await { device_list_changes.insert(user_id); @@ -1367,7 +1391,6 @@ pub(crate) async fn sync_events_v4_route( } } if joined_since_last_sync || new_encrypted_room { - let sender_user = &sender_user; // If the user is in a new encrypted room, give them all joined users device_list_changes.extend( services @@ -1400,7 +1423,7 @@ pub(crate) async fn sync_events_v4_route( } for user_id in left_encrypted_users { - let dont_share_encrypted_room = !share_encrypted_room(&services, &sender_user, &user_id, None).await; + let dont_share_encrypted_room = !share_encrypted_room(&services, sender_user, &user_id, None).await; // If the user doesn't share an encrypted room with the target anymore, we need // to tell them @@ -1564,14 +1587,14 @@ pub(crate) async fn sync_events_v4_route( invite_state = services .rooms .state_cache - .invite_state(&sender_user, room_id) + .invite_state(sender_user, room_id) .await .ok(); (timeline_pdus, limited) = (Vec::new(), true); } else { (timeline_pdus, limited) = - match load_timeline(&services, &sender_user, room_id, roomsincecount, *timeline_limit).await { + match load_timeline(&services, sender_user, room_id, roomsincecount, *timeline_limit).await { Ok(value) => value, Err(err) => { warn!("Encountered missing timeline in {}, error {}", room_id, err); @@ -1584,7 +1607,7 @@ pub(crate) async fn sync_events_v4_route( room_id.clone(), services .account_data - .changes_since(Some(room_id), &sender_user, *roomsince) + .changes_since(Some(room_id), sender_user, *roomsince) .await? .into_iter() .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) @@ -1639,8 +1662,30 @@ pub(crate) async fn sync_events_v4_route( let room_events: Vec<_> = timeline_pdus .iter() - .map(|(_, pdu)| pdu.to_sync_room_event()) - .collect(); + .stream() + .filter_map(|(_, pdu)| async move { + // list of safe and common non-state events to ignore + if matches!( + &pdu.kind, + RoomMessage + | Sticker | CallInvite + | CallNotify | RoomEncrypted + | Image | File | Audio + | Voice | Video | UnstablePollStart + | PollStart | KeyVerificationStart + | Reaction | Emote | Location + ) && services + .users + .user_is_ignored(&pdu.sender, sender_user) + .await + { + return None; + } + + Some(pdu.to_sync_room_event()) + }) + .collect() + .await; for (_, pdu) in timeline_pdus { let ts = MilliSecondsSinceUnixEpoch(pdu.origin_server_ts); @@ -1669,7 +1714,7 @@ pub(crate) async fn sync_events_v4_route( .rooms .state_cache .room_members(room_id) - .ready_filter(|member| member != &sender_user) + .ready_filter(|member| member != sender_user) .filter_map(|user_id| { services .rooms @@ -1743,7 +1788,7 @@ pub(crate) async fn sync_events_v4_route( services .rooms .user - .highlight_count(&sender_user, room_id) + .highlight_count(sender_user, room_id) .await .try_into() .expect("notification count can't go that high"), @@ -1752,7 +1797,7 @@ pub(crate) async fn sync_events_v4_route( services .rooms .user - .notification_count(&sender_user, room_id) + .notification_count(sender_user, room_id) .await .try_into() .expect("notification count can't go that high"), @@ -1811,7 +1856,7 @@ pub(crate) async fn sync_events_v4_route( Some(sync_events::v4::ToDevice { events: services .users - .get_to_device_events(&sender_user, &sender_device) + .get_to_device_events(sender_user, &sender_device) .collect() .await, next_batch: next_batch.to_string(), @@ -1826,7 +1871,7 @@ pub(crate) async fn sync_events_v4_route( }, device_one_time_keys_count: services .users - .count_one_time_keys(&sender_user, &sender_device) + .count_one_time_keys(sender_user, &sender_device) .await, // Fallback keys are not yet supported device_unused_fallback_key_types: None, From 4413793f7e302c9e5b0880ba3eb3f20f8558e6b3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 28 Sep 2024 23:15:43 -0400 Subject: [PATCH 0861/2091] dont allow sending/receiving room invites with ignored users Signed-off-by: strawberry --- src/api/client/membership.rs | 8 ++++++++ src/api/client/room.rs | 20 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 6e3bc894..f89903b4 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -364,6 +364,14 @@ pub(crate) async fn invite_user_route( user_id, } = &body.recipient { + if services.users.user_is_ignored(sender_user, user_id).await { + return Err!(Request(Forbidden("You cannot invite users you have ignored to rooms."))); + } else if services.users.user_is_ignored(user_id, sender_user).await { + // silently drop the invite to the recipient if they've been ignored by the + // sender, pretend it worked + return Ok(invite_user::v3::Response {}); + } + invite_helper(&services, sender_user, user_id, &body.room_id, body.reason.clone(), false).await?; Ok(invite_user::v3::Response {}) } else { diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 1edf85d8..0d8e12a2 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -267,8 +267,16 @@ pub(crate) async fn create_room_route( let mut users = BTreeMap::from_iter([(sender_user.clone(), int!(100))]); if preset == RoomPreset::TrustedPrivateChat { - for invite_ in &body.invite { - users.insert(invite_.clone(), int!(100)); + for invite in &body.invite { + if services.users.user_is_ignored(sender_user, invite).await { + return Err!(Request(Forbidden("You cannot invite users you have ignored to rooms."))); + } else if services.users.user_is_ignored(invite, sender_user).await { + // silently drop the invite to the recipient if they've been ignored by the + // sender, pretend it worked + continue; + } + + users.insert(invite.clone(), int!(100)); } } @@ -476,6 +484,14 @@ pub(crate) async fn create_room_route( // 8. Events implied by invite (and TODO: invite_3pid) drop(state_lock); for user_id in &body.invite { + if services.users.user_is_ignored(sender_user, user_id).await { + return Err!(Request(Forbidden("You cannot invite users you have ignored to rooms."))); + } else if services.users.user_is_ignored(user_id, sender_user).await { + // silently drop the invite to the recipient if they've been ignored by the + // sender, pretend it worked + continue; + } + if let Err(e) = invite_helper(&services, sender_user, user_id, &room_id, None, body.is_direct) .boxed() .await From b64a23516520fe764e5d463a28d6b341942642c3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 29 Sep 2024 00:28:05 -0400 Subject: [PATCH 0862/2091] use ok_or_else for a rare error Signed-off-by: strawberry --- src/service/rooms/event_handler/parse_incoming_pdu.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 2de3e28e..9081fcbc 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -14,7 +14,7 @@ impl super::Service { let room_id: OwnedRoomId = value .get("room_id") .and_then(|id| RoomId::parse(id.as_str()?).ok()) - .ok_or(err!(Request(InvalidParam("Invalid room id in pdu"))))?; + .ok_or_else(|| err!(Request(InvalidParam("Invalid room id in pdu"))))?; let Ok(room_version_id) = self.services.state.get_room_version(&room_id).await else { return Err!("Server is not in room {room_id}"); From ee1580e4800f254cee39707b8e0c3d0a9339bb23 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 29 Sep 2024 00:50:12 -0400 Subject: [PATCH 0863/2091] fix list_rooms admin command filters Signed-off-by: strawberry --- src/admin/room/commands.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index 1c90a998..35e40c8b 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -6,7 +6,7 @@ use crate::{admin_command, get_room_info, PAGE_SIZE}; #[admin_command] pub(super) async fn list_rooms( - &self, page: Option, _exclude_disabled: bool, _exclude_banned: bool, no_details: bool, + &self, page: Option, exclude_disabled: bool, exclude_banned: bool, no_details: bool, ) -> Result { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); @@ -15,8 +15,12 @@ pub(super) async fn list_rooms( .rooms .metadata .iter_ids() - //.filter(|room_id| async { !exclude_disabled || !self.services.rooms.metadata.is_disabled(room_id).await }) - //.filter(|room_id| async { !exclude_banned || !self.services.rooms.metadata.is_banned(room_id).await }) + .filter_map(|room_id| async move { + (!exclude_disabled || !self.services.rooms.metadata.is_disabled(room_id).await).then_some(room_id) + }) + .filter_map(|room_id| async move { + (!exclude_banned || !self.services.rooms.metadata.is_banned(room_id).await).then_some(room_id) + }) .then(|room_id| get_room_info(self.services, room_id)) .collect::>() .await; From 7a59add8f1bc8d4697580823e8651f4b72e4b9d5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 29 Sep 2024 01:54:07 -0400 Subject: [PATCH 0864/2091] add support for reading a registration token from a file Signed-off-by: strawberry --- conduwuit-example.toml | 18 +++++++++++++---- docs/deploying/docker-compose.for-traefik.yml | 2 +- docs/deploying/docker-compose.with-caddy.yml | 2 +- .../deploying/docker-compose.with-traefik.yml | 6 +++--- docs/deploying/docker-compose.yml | 2 +- src/api/client/account.rs | 6 +++--- src/core/config/check.rs | 20 ++++++++++++++++++- src/core/config/mod.rs | 15 +++++++++++--- src/service/globals/mod.rs | 16 +++++++++++++++ src/service/uiaa/mod.rs | 12 +++++++---- 10 files changed, 78 insertions(+), 21 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index b532d381..11735616 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -195,11 +195,14 @@ allow_guests_auto_join_rooms = false # Enables registration. If set to false, no users can register on this # server. +# # If set to true without a token configured, users can register with no form of 2nd- # step only if you set # `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` to -# true in your config. If you would like -# registration only via token reg, please configure the `registration_token` key. +# true in your config. +# +# If you would like registration only via token reg, please configure +# `registration_token` or `registration_token_file`. allow_registration = false # Please note that an open registration homeserver with no second-step verification # is highly prone to abuse and potential defederation by homeservers, including @@ -208,7 +211,14 @@ allow_registration = false # A static registration token that new users will have to provide when creating # an account. If unset and `allow_registration` is true, registration is open # without any condition. YOU NEED TO EDIT THIS. -registration_token = "change this token for something specific to your server" +registration_token = "change this token/string here or set registration_token_file" + +# Path to a file on the system that gets read for the registration token +# +# conduwuit must be able to access the file, and it must not be empty +# +# no default +#registration_token_file = "/etc/conduwuit/.reg_token" # controls whether federation is allowed or not # defaults to true @@ -344,7 +354,7 @@ allow_profile_lookup_federation_requests = true # Controls the max log level for admin command log captures (logs generated from running admin commands) # # Defaults to "info" on release builds, else "debug" on debug builds -#admin_log_capture = info +#admin_log_capture = "info" # Allows admins to enter commands in rooms other than #admins by prefixing with \!admin. The reply # will be publicly visible to the room, originating from the sender. diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index 1c615673..ae93d52f 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -16,7 +16,7 @@ services: CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit CONDUWUIT_DATABASE_BACKEND: rocksdb CONDUWUIT_PORT: 6167 # should match the loadbalancer traefik label - CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB + CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB CONDUWUIT_ALLOW_REGISTRATION: 'true' CONDUWUIT_ALLOW_FEDERATION: 'true' CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index 899f4d67..36924212 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -32,7 +32,7 @@ services: CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit CONDUWUIT_DATABASE_BACKEND: rocksdb CONDUWUIT_PORT: 6167 - CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB + CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB CONDUWUIT_ALLOW_REGISTRATION: 'true' CONDUWUIT_ALLOW_FEDERATION: 'true' CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index f05006a5..89118c74 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -15,7 +15,8 @@ services: CONDUWUIT_SERVER_NAME: your.server.name.example # EDIT THIS CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' CONDUWUIT_ALLOW_REGISTRATION: 'false' # After setting a secure registration token, you can enable this - CONDUWUIT_REGISTRATION_TOKEN: # This is a token you can use to register on the server + CONDUWUIT_REGISTRATION_TOKEN: "" # This is a token you can use to register on the server + #CONDUWUIT_REGISTRATION_TOKEN_FILE: "" # Alternatively you can configure a path to a token file to read CONDUWUIT_ADDRESS: 0.0.0.0 CONDUWUIT_PORT: 6167 # you need to match this with the traefik load balancer label if you're want to change it CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit @@ -23,7 +24,6 @@ services: ### Uncomment and change values as desired, note that conduwuit has plenty of config options, so you should check out the example example config too # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging # CONDUWUIT_LOG: info # default is: "warn,state_res=warn" - # CONDUWUIT_ALLOW_JAEGER: 'false' # CONDUWUIT_ALLOW_ENCRYPTION: 'true' # CONDUWUIT_ALLOW_FEDERATION: 'true' # CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' @@ -31,7 +31,7 @@ services: # CONDUWUIT_ALLOW_OUTGOING_PRESENCE: true # CONDUWUIT_ALLOW_LOCAL_PRESENCE: true # CONDUWUIT_WORKERS: 10 - # CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB + # CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB # CONDUWUIT_NEW_USER_DISPLAYNAME_SUFFIX = "🏳<200d>⚧" # We need some way to serve the client and server .well-known json. The simplest way is via the CONDUWUIT_WELL_KNOWN diff --git a/docs/deploying/docker-compose.yml b/docs/deploying/docker-compose.yml index bc9f2477..26145c5a 100644 --- a/docs/deploying/docker-compose.yml +++ b/docs/deploying/docker-compose.yml @@ -16,7 +16,7 @@ services: CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit CONDUWUIT_DATABASE_BACKEND: rocksdb CONDUWUIT_PORT: 6167 - CONDUWUIT_MAX_REQUEST_SIZE: 20_000_000 # in bytes, ~20 MB + CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB CONDUWUIT_ALLOW_REGISTRATION: 'true' CONDUWUIT_ALLOW_FEDERATION: 'true' CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 63d02f8f..1ededa36 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -111,7 +111,7 @@ pub(crate) async fn register_route( if is_guest && (!services.globals.allow_guest_registration() - || (services.globals.allow_registration() && services.globals.config.registration_token.is_some())) + || (services.globals.allow_registration() && services.globals.registration_token.is_some())) { info!( "Guest registration disabled / registration enabled with token configured, rejecting guest registration \ @@ -183,7 +183,7 @@ pub(crate) async fn register_route( // UIAA let mut uiaainfo; - let skip_auth = if services.globals.config.registration_token.is_some() { + let skip_auth = if services.globals.registration_token.is_some() { // Registration token required uiaainfo = UiaaInfo { flows: vec![AuthFlow { @@ -685,7 +685,7 @@ pub(crate) async fn request_3pid_management_token_via_msisdn_route( pub(crate) async fn check_registration_token_validity( State(services): State, body: Ruma, ) -> Result { - let Some(reg_token) = services.globals.config.registration_token.clone() else { + let Some(reg_token) = services.globals.registration_token.clone() else { return Err(Error::BadRequest( ErrorKind::forbidden(), "Server does not allow token registration.", diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 8dea55d8..c0d05533 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -94,6 +94,22 @@ pub fn check(config: &Config) -> Result<()> { )); } + // check if we can read the token file path, and check if the file is empty + if config.registration_token_file.as_ref().is_some_and(|path| { + let Ok(token) = std::fs::read_to_string(path).inspect_err(|e| { + error!("Failed to read the registration token file: {e}"); + }) else { + return true; + }; + + token == String::new() + }) { + return Err!(Config( + "registration_token_file", + "Registration token file was specified but is empty or failed to be read" + )); + } + if config.max_request_size < 5_120_000 { return Err!(Config( "max_request_size", @@ -111,12 +127,13 @@ pub fn check(config: &Config) -> Result<()> { if config.allow_registration && !config.yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse && config.registration_token.is_none() + && config.registration_token_file.is_none() { return Err!(Config( "registration_token", "!! You have `allow_registration` enabled without a token configured in your config which means you are \ allowing ANYONE to register on your conduwuit instance without any 2nd-step (e.g. registration token).\n -If this is not the intended behaviour, please set a registration token with the `registration_token` config option.\n +If this is not the intended behaviour, please set a registration token.\n For security and safety reasons, conduwuit will shut down. If you are extra sure this is the desired behaviour you \ want, please set the following config option to true: `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`" @@ -126,6 +143,7 @@ For security and safety reasons, conduwuit will shut down. If you are extra sure if config.allow_registration && config.yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse && config.registration_token.is_none() + && config.registration_token_file.is_none() { warn!( "Open registration is enabled via setting \ diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index d8e1c7d9..126b3123 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -139,6 +139,7 @@ pub struct Config { #[serde(default)] pub yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse: bool, pub registration_token: Option, + pub registration_token_file: Option, #[serde(default = "true_fn")] pub allow_encryption: bool, #[serde(default = "true_fn")] @@ -572,12 +573,20 @@ impl fmt::Display for Config { line("Allow registration", &self.allow_registration.to_string()); line( "Registration token", - if self.registration_token.is_some() { - "set" + if self.registration_token.is_none() && self.registration_token_file.is_none() && self.allow_registration { + "not set (⚠️ open registration!)" + } else if self.registration_token.is_none() && self.registration_token_file.is_none() { + "not set" } else { - "not set (open registration!)" + "set" }, ); + line( + "Registration token file path", + self.registration_token_file + .as_ref() + .map_or("", |path| path.to_str().unwrap_or_default()), + ); line( "Allow guest registration (inherently false if allow registration is false)", &self.allow_guest_registration.to_string(), diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index f24e8a27..fb970f07 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -41,6 +41,7 @@ pub struct Service { pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, pub turn_secret: String, + pub registration_token: Option, } type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries @@ -96,6 +97,20 @@ impl crate::Service for Service { }) }); + let registration_token = + config + .registration_token_file + .as_ref() + .map_or(config.registration_token.clone(), |path| { + let Ok(token) = std::fs::read_to_string(path).inspect_err(|e| { + error!("Failed to read the registration token file: {e}"); + }) else { + return config.registration_token.clone(); + }; + + Some(token) + }); + let mut s = Self { db, config: config.clone(), @@ -112,6 +127,7 @@ impl crate::Service for Service { server_user: UserId::parse_with_server_name(String::from("conduit"), &config.server_name) .expect("@conduit:server_name is valid"), turn_secret, + registration_token, }; if !s diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 0415bfc2..f75f1bcd 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -6,7 +6,7 @@ use std::{ use conduit::{ err, error, implement, utils, utils::{hash, string::EMPTY}, - Error, Result, Server, + Error, Result, }; use database::{Deserialized, Map}; use ruma::{ @@ -26,7 +26,6 @@ pub struct Service { } struct Services { - server: Arc, globals: Dep, users: Dep, } @@ -48,7 +47,6 @@ impl crate::Service for Service { userdevicesessionid_uiaainfo: args.db["userdevicesessionid_uiaainfo"].clone(), }, services: Services { - server: args.server.clone(), globals: args.depend::("globals"), users: args.depend::("users"), }, @@ -135,7 +133,13 @@ pub async fn try_auth( uiaainfo.completed.push(AuthType::Password); }, AuthData::RegistrationToken(t) => { - if Some(t.token.trim()) == self.services.server.config.registration_token.as_deref() { + if self + .services + .globals + .registration_token + .as_ref() + .is_some_and(|reg_token| t.token.trim() == reg_token) + { uiaainfo.completed.push(AuthType::RegistrationToken); } else { uiaainfo.auth_error = Some(ruma::api::client::error::StandardErrorBody { From 6a81bf23dec75e97be30c44f248ef9bd1493835e Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 29 Sep 2024 20:13:37 -0400 Subject: [PATCH 0865/2091] dont send events from ignored users over /messages Signed-off-by: strawberry --- src/api/client/message.rs | 80 +++++++++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 15 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index bab5fa54..d577e3c8 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,7 +1,11 @@ use std::collections::{BTreeMap, HashSet}; use axum::extract::State; -use conduit::{err, utils::ReadyExt, Err, PduCount}; +use conduit::{ + err, + utils::{IterStream, ReadyExt}, + Err, PduCount, +}; use futures::{FutureExt, StreamExt}; use ruma::{ api::client::{ @@ -9,7 +13,7 @@ use ruma::{ filter::{RoomEventFilter, UrlFilter}, message::{get_message_events, send_message_event}, }, - events::{MessageLikeEventType, StateEventType}, + events::{MessageLikeEventType, StateEventType, TimelineEventType::*}, UserId, }; use serde_json::{from_str, Value}; @@ -182,8 +186,30 @@ pub(crate) async fn get_message_events_route( let events_after: Vec<_> = events_after .into_iter() - .map(|(_, pdu)| pdu.to_room_event()) - .collect(); + .stream() + .filter_map(|(_, pdu)| async move { + // list of safe and common non-state events to ignore + if matches!( + &pdu.kind, + RoomMessage + | Sticker | CallInvite + | CallNotify | RoomEncrypted + | Image | File | Audio + | Voice | Video | UnstablePollStart + | PollStart | KeyVerificationStart + | Reaction | Emote | Location + ) && services + .users + .user_is_ignored(&pdu.sender, sender_user) + .await + { + return None; + } + + Some(pdu.to_room_event()) + }) + .collect() + .await; resp.start = from.stringify(); resp.end = next_token.map(|count| count.stringify()); @@ -203,6 +229,27 @@ pub(crate) async fn get_message_events_route( .pdus_until(sender_user, room_id, from) .await? .ready_filter_map(|item| contains_url_filter(item, filter)) + .filter_map(|(count, pdu)| async move { + // list of safe and common non-state events to ignore + if matches!( + &pdu.kind, + RoomMessage + | Sticker | CallInvite + | CallNotify | RoomEncrypted + | Image | File | Audio + | Voice | Video | UnstablePollStart + | PollStart | KeyVerificationStart + | Reaction | Emote | Location + ) && services + .users + .user_is_ignored(&pdu.sender, sender_user) + .await + { + return None; + } + + Some((count, pdu)) + }) .filter_map(|item| visibility_filter(&services, item, sender_user)) .ready_take_while(|(count, _)| Some(*count) != to) // Stop at `to` .take(limit) @@ -243,17 +290,20 @@ pub(crate) async fn get_message_events_route( }, } - resp.state = Vec::new(); - for ll_id in &lazy_loaded { - if let Ok(member_event) = services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, ll_id.as_str()) - .await - { - resp.state.push(member_event.to_state_event()); - } - } + resp.state = lazy_loaded + .iter() + .stream() + .filter_map(|ll_user_id| async move { + services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomMember, ll_user_id.as_str()) + .await + .map(|member_event| member_event.to_state_event()) + .ok() + }) + .collect() + .await; // remove the feature check when we are sure clients like element can handle it if !cfg!(feature = "element_hacks") { From a9e3e8f77ad38549f7bb21c9447e5e7549ac31fe Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 29 Sep 2024 20:40:54 -0400 Subject: [PATCH 0866/2091] dont send non-state events from ignored users over /context/{eventId} Signed-off-by: strawberry --- src/api/client/context.rs | 64 ++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index cc49b763..9a5c4e82 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -5,12 +5,12 @@ use conduit::{err, error, Err}; use futures::StreamExt; use ruma::{ api::client::{context::get_context, filter::LazyLoadOptions}, - events::StateEventType, + events::{StateEventType, TimelineEventType::*}, }; use crate::{Result, Ruma}; -/// # `GET /_matrix/client/r0/rooms/{roomId}/context` +/// # `GET /_matrix/client/r0/rooms/{roomId}/context/{eventId}` /// /// Allows loading room history around an event. /// @@ -31,7 +31,7 @@ pub(crate) async fn get_context_route( LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")), }; - let mut lazy_loaded = HashSet::new(); + let mut lazy_loaded = HashSet::with_capacity(100); let base_token = services .rooms @@ -79,6 +79,25 @@ pub(crate) async fn get_context_route( .await? .take(limit / 2) .filter_map(|(count, pdu)| async move { + // list of safe and common non-state events to ignore + if matches!( + &pdu.kind, + RoomMessage + | Sticker | CallInvite + | CallNotify | RoomEncrypted + | Image | File | Audio + | Voice | Video | UnstablePollStart + | PollStart | KeyVerificationStart + | Reaction | Emote + | Location + ) && services + .users + .user_is_ignored(&pdu.sender, sender_user) + .await + { + return None; + } + services .rooms .state_accessor @@ -104,11 +123,6 @@ pub(crate) async fn get_context_route( .last() .map_or_else(|| base_token.stringify(), |(count, _)| count.stringify()); - let events_before: Vec<_> = events_before - .into_iter() - .map(|(_, pdu)| pdu.to_room_event()) - .collect(); - let events_after: Vec<_> = services .rooms .timeline @@ -116,6 +130,25 @@ pub(crate) async fn get_context_route( .await? .take(limit / 2) .filter_map(|(count, pdu)| async move { + // list of safe and common non-state events to ignore + if matches!( + &pdu.kind, + RoomMessage + | Sticker | CallInvite + | CallNotify | RoomEncrypted + | Image | File | Audio + | Voice | Video | UnstablePollStart + | PollStart | KeyVerificationStart + | Reaction | Emote + | Location + ) && services + .users + .user_is_ignored(&pdu.sender, sender_user) + .await + { + return None; + } + services .rooms .state_accessor @@ -167,11 +200,6 @@ pub(crate) async fn get_context_route( .last() .map_or_else(|| base_token.stringify(), |(count, _)| count.stringify()); - let events_after: Vec<_> = events_after - .into_iter() - .map(|(_, pdu)| pdu.to_room_event()) - .collect(); - let mut state = Vec::with_capacity(state_ids.len()); for (shortstatekey, id) in state_ids { @@ -201,9 +229,15 @@ pub(crate) async fn get_context_route( Ok(get_context::v3::Response { start: Some(start_token), end: Some(end_token), - events_before, + events_before: events_before + .iter() + .map(|(_, pdu)| pdu.to_room_event()) + .collect(), event: Some(base_event), - events_after, + events_after: events_after + .iter() + .map(|(_, pdu)| pdu.to_room_event()) + .collect(), state, }) } From 115ea03edfc1cf785ad280abcc850bf14a2b76cc Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 29 Sep 2024 20:57:33 -0400 Subject: [PATCH 0867/2091] remove unnecessary full type annos Signed-off-by: strawberry --- src/api/client/sync.rs | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 51df88a3..c4ff1eeb 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -34,8 +34,8 @@ use ruma::{ events::{ presence::PresenceEvent, room::member::{MembershipState, RoomMemberEventContent}, - AnyRawAccountDataEvent, StateEventType, TimelineEventType, - TimelineEventType::*, + AnyRawAccountDataEvent, StateEventType, + TimelineEventType::{self, *}, }, serde::Raw, state_res::Event, @@ -50,14 +50,8 @@ use crate::{ }; const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; -const DEFAULT_BUMP_TYPES: &[TimelineEventType] = &[ - TimelineEventType::RoomMessage, - TimelineEventType::RoomEncrypted, - TimelineEventType::Sticker, - TimelineEventType::CallInvite, - TimelineEventType::PollStart, - TimelineEventType::Beacon, -]; +const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = + &[RoomMessage, RoomEncrypted, Sticker, CallInvite, PollStart, Beacon]; macro_rules! extract_variant { ($e:expr, $variant:path) => { @@ -376,7 +370,7 @@ async fn handle_left_room( origin_server_ts: utils::millis_since_unix_epoch() .try_into() .expect("Timestamp is valid js_int value"), - kind: TimelineEventType::RoomMember, + kind: RoomMember, content: serde_json::from_str(r#"{"membership":"leave"}"#).expect("this is valid JSON"), state_key: Some(sender_user.to_string()), unsigned: None, @@ -639,7 +633,7 @@ async fn load_joined_room( .timeline .all_pdus(sender_user, room_id) .await? - .ready_filter(|(_, pdu)| pdu.kind == TimelineEventType::RoomMember) + .ready_filter(|(_, pdu)| pdu.kind == RoomMember) .filter_map(|(_, pdu)| async move { let Ok(content) = serde_json::from_str::(pdu.content.get()) else { return None; @@ -827,11 +821,11 @@ async fn load_joined_room( let send_member_count = delta_state_events .iter() - .any(|event| event.kind == TimelineEventType::RoomMember); + .any(|event| event.kind == RoomMember); if encrypted_room { for state_event in &delta_state_events { - if state_event.kind != TimelineEventType::RoomMember { + if state_event.kind != RoomMember { continue; } @@ -895,7 +889,7 @@ async fn load_joined_room( // Mark all member events we're returning as lazy-loaded for pdu in &state_events { - if pdu.kind == TimelineEventType::RoomMember { + if pdu.kind == RoomMember { match UserId::parse( pdu.state_key .as_ref() @@ -1357,7 +1351,7 @@ pub(crate) async fn sync_events_v4_route( error!("Pdu in state not found: {id}"); continue; }; - if pdu.kind == TimelineEventType::RoomMember { + if pdu.kind == RoomMember { if let Some(state_key) = &pdu.state_key { let user_id = UserId::parse(state_key.clone()) .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; From 4eb7ad79d18c1d89784d2756a8652a1c0ce2d347 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 1 Oct 2024 01:59:24 -0400 Subject: [PATCH 0868/2091] update last_seen_ip and last_seen_ts on updating device metadata Signed-off-by: strawberry --- src/api/client/device.rs | 20 +++++++++++++++----- src/service/users/mod.rs | 11 ----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/api/client/device.rs b/src/api/client/device.rs index 93eaa393..7e56f27e 100644 --- a/src/api/client/device.rs +++ b/src/api/client/device.rs @@ -1,10 +1,14 @@ use axum::extract::State; +use axum_client_ip::InsecureClientIp; use conduit::{err, Err}; use futures::StreamExt; -use ruma::api::client::{ - device::{self, delete_device, delete_devices, get_device, get_devices, update_device}, - error::ErrorKind, - uiaa::{AuthFlow, AuthType, UiaaInfo}, +use ruma::{ + api::client::{ + device::{self, delete_device, delete_devices, get_device, get_devices, update_device}, + error::ErrorKind, + uiaa::{AuthFlow, AuthType, UiaaInfo}, + }, + MilliSecondsSinceUnixEpoch, }; use super::SESSION_ID_LENGTH; @@ -51,8 +55,10 @@ pub(crate) async fn get_device_route( /// # `PUT /_matrix/client/r0/devices/{deviceId}` /// /// Updates the metadata on a given device of the sender user. +#[tracing::instrument(skip_all, fields(%client), name = "update_device")] pub(crate) async fn update_device_route( - State(services): State, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -63,6 +69,10 @@ pub(crate) async fn update_device_route( .map_err(|_| err!(Request(NotFound("Device not found."))))?; device.display_name.clone_from(&body.display_name); + device.last_seen_ip.clone_from(&Some(client.to_string())); + device + .last_seen_ts + .clone_from(&Some(MilliSecondsSinceUnixEpoch::now())); services .users diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 1c079085..44d169dd 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -791,17 +791,6 @@ impl Service { } pub async fn update_device_metadata(&self, user_id: &UserId, device_id: &DeviceId, device: &Device) -> Result<()> { - // Only existing devices should be able to call this, but we shouldn't assert - // either... - let key = (user_id, device_id); - if self.db.userdeviceid_metadata.qry(&key).await.is_err() { - return Err!(Database(error!( - ?user_id, - ?device_id, - "Called update_device_metadata for a non-existent user and/or device" - ))); - } - increment(&self.db.userid_devicelistversion, user_id.as_bytes()); let mut userdeviceid = user_id.as_bytes().to_vec(); From 98363852b18c4f2cc1525c671b770a6fbf4a7f3a Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 2 Oct 2024 00:56:09 -0400 Subject: [PATCH 0869/2091] fix: dont add remote users for push targets, use hashset instead of vec Signed-off-by: strawberry --- src/service/rooms/timeline/mod.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 6a26a1d5..f8f770bc 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -43,7 +43,7 @@ use self::data::Data; pub use self::data::PdusIterItem; use crate::{ account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms, - rooms::state_compressor::CompressedStateEvent, sending, server_keys, Dep, + rooms::state_compressor::CompressedStateEvent, sending, server_keys, users, Dep, }; // Update Relationships @@ -90,6 +90,7 @@ struct Services { sending: Dep, server_keys: Dep, user: Dep, + users: Dep, pusher: Dep, threads: Dep, search: Dep, @@ -119,6 +120,7 @@ impl crate::Service for Service { sending: args.depend::("sending"), server_keys: args.depend::("server_keys"), user: args.depend::("rooms::user"), + users: args.depend::("users"), pusher: args.depend::("pusher"), threads: args.depend::("rooms::threads"), search: args.depend::("rooms::search"), @@ -378,20 +380,20 @@ impl Service { let mut notifies = Vec::new(); let mut highlights = Vec::new(); - let mut push_target = self + let mut push_target: HashSet<_> = self .services .state_cache .active_local_users_in_room(&pdu.room_id) .map(ToOwned::to_owned) - .collect::>() + .collect() .await; if pdu.kind == TimelineEventType::RoomMember { if let Some(state_key) = &pdu.state_key { - let target_user_id = UserId::parse(state_key.clone()).expect("This state_key was previously validated"); + let target_user_id = UserId::parse(state_key.clone())?; - if !push_target.contains(&target_user_id) { - push_target.push(target_user_id); + if self.services.users.is_active_local(&target_user_id).await { + push_target.insert(target_user_id); } } } From 54a107c3c473049f5049e96f807d4c505f5a13db Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 2 Oct 2024 01:47:19 -0400 Subject: [PATCH 0870/2091] drop unnecessary error to debug_warn Signed-off-by: strawberry --- src/api/client/state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index f9a4a763..d89c23e8 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -130,7 +130,7 @@ pub(crate) async fn get_state_events_for_key_route( .room_state_get(&body.room_id, &body.event_type, &body.state_key) .await .map_err(|_| { - err!(Request(NotFound(error!( + err!(Request(NotFound(debug_warn!( room_id = ?body.room_id, event_type = ?body.event_type, "State event not found in room.", From ab9a65db5d8501c60c523bc69d704032459db482 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 2 Oct 2024 01:47:53 -0400 Subject: [PATCH 0871/2091] add MSC4151 room reporting support Signed-off-by: strawberry --- Cargo.lock | 28 +++++----- Cargo.toml | 2 +- src/api/client/report.rs | 107 +++++++++++++++++++++++++++------------ src/api/router.rs | 1 + 4 files changed, 90 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d40c458..e72c7e80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2975,7 +2975,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "assign", "js_int", @@ -2997,7 +2997,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "js_int", "ruma-common", @@ -3009,7 +3009,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "as_variant", "assign", @@ -3032,7 +3032,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "as_variant", "base64 0.22.1", @@ -3062,7 +3062,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3086,7 +3086,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "bytes", "http", @@ -3104,7 +3104,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "js_int", "thiserror", @@ -3113,7 +3113,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "js_int", "ruma-common", @@ -3123,7 +3123,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "cfg-if", "once_cell", @@ -3139,7 +3139,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "js_int", "ruma-common", @@ -3151,7 +3151,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "headers", "http", @@ -3164,7 +3164,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3180,10 +3180,10 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e#ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" dependencies = [ "futures-util", - "itertools 0.12.1", + "itertools 0.13.0", "js_int", "ruma-common", "ruma-events", diff --git a/Cargo.toml b/Cargo.toml index 28e280cf..18f33375 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "ade2f1daf0b1d9e8f7de81a24dca8925406e4d8e" +rev = "e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" features = [ "compat", "rand", diff --git a/src/api/client/report.rs b/src/api/client/report.rs index a40c35a2..cf789246 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -1,10 +1,14 @@ use std::time::Duration; use axum::extract::State; +use axum_client_ip::InsecureClientIp; use conduit::{utils::ReadyExt, Err}; use rand::Rng; use ruma::{ - api::client::{error::ErrorKind, room::report_content}, + api::client::{ + error::ErrorKind, + room::{report_content, report_room}, + }, events::room::message, int, EventId, RoomId, UserId, }; @@ -14,22 +18,75 @@ use tracing::info; use crate::{ debug_info, service::{pdu::PduEvent, Services}, - utils::HtmlEscape, Error, Result, Ruma, }; +/// # `POST /_matrix/client/v3/rooms/{roomId}/report` +/// +/// Reports an abusive room to homeserver admins +#[tracing::instrument(skip_all, fields(%client), name = "report_room")] +pub(crate) async fn report_room_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + // user authentication + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + info!( + "Received room report by user {sender_user} for room {} with reason: {:?}", + body.room_id, body.reason + ); + + delay_response().await; + + if !services + .rooms + .state_cache + .server_in_room(&services.globals.config.server_name, &body.room_id) + .await + { + return Err!(Request(NotFound( + "Room does not exist to us, no local users have joined at all" + ))); + } + + if body.reason.as_ref().is_some_and(|s| s.len() > 750) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Reason too long, should be 750 characters or fewer", + )); + }; + + // send admin room message that we received the report with an @room ping for + // urgency + services + .admin + .send_message(message::RoomMessageEventContent::text_markdown(format!( + "@room Room report received from {} -\n\nRoom ID: {}\n\nReport Reason: {}", + sender_user.to_owned(), + body.room_id, + body.reason.as_deref().unwrap_or("") + ))) + .await + .ok(); + + Ok(report_room::v3::Response {}) +} + /// # `POST /_matrix/client/v3/rooms/{roomId}/report/{eventId}` /// /// Reports an inappropriate event to homeserver admins +#[tracing::instrument(skip_all, fields(%client), name = "report_event")] pub(crate) async fn report_event_route( - State(services): State, body: Ruma, + State(services): State, InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { // user authentication let sender_user = body.sender_user.as_ref().expect("user is authenticated"); info!( - "Received /report request by user {sender_user} for room {} and event ID {}", - body.room_id, body.event_id + "Received event report by user {sender_user} for room {} and event ID {}, with reason: {:?}", + body.room_id, body.event_id, body.reason ); delay_response().await; @@ -39,7 +96,7 @@ pub(crate) async fn report_event_route( return Err!(Request(NotFound("Event ID is not known to us or Event ID is invalid"))); }; - is_report_valid( + is_event_report_valid( &services, &pdu.event_id, &body.room_id, @@ -54,32 +111,16 @@ pub(crate) async fn report_event_route( // urgency services .admin - .send_message(message::RoomMessageEventContent::text_html( - format!( - "@room Report received from: {}\n\nEvent ID: {}\nRoom ID: {}\nSent By: {}\n\nReport Score: {}\nReport \ - Reason: {}", - sender_user.to_owned(), - pdu.event_id, - pdu.room_id, - pdu.sender.clone(), - body.score.unwrap_or_else(|| ruma::Int::from(0)), - body.reason.as_deref().unwrap_or("") - ), - format!( - "

    ", - sender_user.to_owned(), - pdu.event_id.clone(), - pdu.room_id.clone(), - pdu.sender.clone(), - body.score.unwrap_or_else(|| ruma::Int::from(0)), - HtmlEscape(body.reason.as_deref().unwrap_or("")) - ), - )) + .send_message(message::RoomMessageEventContent::text_markdown(format!( + "@room Event report received from {} -\n\nEvent ID: {}\nRoom ID: {}\nSent By: {}\n\nReport Score: \ + {}\nReport Reason: {}", + sender_user.to_owned(), + pdu.event_id, + pdu.room_id, + pdu.sender, + body.score.unwrap_or_else(|| ruma::Int::from(0)), + body.reason.as_deref().unwrap_or("") + ))) .await .ok(); @@ -92,7 +133,7 @@ pub(crate) async fn report_event_route( /// check if score is in valid range /// check if report reasoning is less than or equal to 750 characters /// check if reporting user is in the reporting room -async fn is_report_valid( +async fn is_event_report_valid( services: &Services, event_id: &EventId, room_id: &RoomId, sender_user: &UserId, reason: &Option, score: Option, pdu: &std::sync::Arc, ) -> Result<()> { diff --git a/src/api/router.rs b/src/api/router.rs index c4275f05..ddd91d11 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -91,6 +91,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(&client::create_room_route) .ruma_route(&client::redact_event_route) .ruma_route(&client::report_event_route) + .ruma_route(&client::report_room_route) .ruma_route(&client::create_alias_route) .ruma_route(&client::delete_alias_route) .ruma_route(&client::get_alias_route) From bd56d8304561bae45f8578a61e89e020a8387888 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 2 Oct 2024 09:26:28 -0400 Subject: [PATCH 0872/2091] fix room directory regression Signed-off-by: strawberry --- src/service/rooms/directory/mod.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 5666a91a..2112ecef 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -1,8 +1,8 @@ use std::sync::Arc; use conduit::{implement, utils::stream::TryIgnore, Result}; -use database::{Ignore, Map}; -use futures::{Stream, StreamExt}; +use database::Map; +use futures::Stream; use ruma::RoomId; pub struct Service { @@ -35,10 +35,4 @@ pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(ro pub async fn is_public_room(&self, room_id: &RoomId) -> bool { self.db.publicroomids.get(room_id).await.is_ok() } #[implement(Service)] -pub fn public_rooms(&self) -> impl Stream + Send { - self.db - .publicroomids - .keys() - .ignore_err() - .map(|(room_id, _): (&RoomId, Ignore)| room_id) -} +pub fn public_rooms(&self) -> impl Stream + Send { self.db.publicroomids.keys().ignore_err() } From fa7c1200b55a1d90df57dc31c3d54c92fb89fff0 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 2 Oct 2024 21:38:52 -0400 Subject: [PATCH 0873/2091] miniscule spaces code optimisations still terrible though Signed-off-by: strawberry --- src/service/rooms/spaces/mod.rs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 17fbf0ef..920424a4 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -62,11 +62,11 @@ impl FromStr for PaginationToken { let mut values = value.split('_'); let mut pag_tok = || { - let mut rooms = vec![]; - - for room in values.next()?.split(',') { - rooms.push(u64::from_str(room).ok()?); - } + let rooms = values + .next()? + .split(',') + .filter_map(|room_s| u64::from_str(room_s).ok()) + .collect(); Some(Self { short_room_ids: rooms, @@ -469,7 +469,7 @@ impl Service { }, )]]; - let mut results = Vec::new(); + let mut results = Vec::with_capacity(limit); while let Some((current_room, via)) = { next_room_to_traverse(&mut stack, &mut parents) } { if results.len() >= limit { @@ -548,11 +548,12 @@ impl Service { parents.pop_front(); parents.push_back(room); - let mut short_room_ids = vec![]; - - for room in parents { - short_room_ids.push(self.services.short.get_or_create_shortroomid(&room).await); - } + let short_room_ids: Vec<_> = parents + .iter() + .stream() + .filter_map(|room_id| async move { self.services.short.get_shortroomid(room_id).await.ok() }) + .collect() + .await; Some( PaginationToken { @@ -585,7 +586,7 @@ impl Service { .await .map_err(|e| err!(Database("State in space not found: {e}")))?; - let mut children_pdus = Vec::new(); + let mut children_pdus = Vec::with_capacity(state.len()); for (key, id) in state { let (event_type, state_key) = self.services.short.get_statekey_from_short(key).await?; From c6b7c24e99891a8374a7444048c493240f7dbca5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 21:42:25 +0000 Subject: [PATCH 0874/2091] consume all bytes for top-level Ignore; add comments/tweaks Signed-off-by: Jason Volk --- src/database/de.rs | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/database/de.rs b/src/database/de.rs index fc36560d..9ee52267 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -12,6 +12,7 @@ where let mut deserializer = Deserializer { buf, pos: 0, + seq: false, }; T::deserialize(&mut deserializer).debug_inspect(|_| { @@ -24,6 +25,7 @@ where pub(crate) struct Deserializer<'de> { buf: &'de [u8], pos: usize, + seq: bool, } /// Directive to ignore a record. This type can be used to skip deserialization @@ -32,8 +34,11 @@ pub(crate) struct Deserializer<'de> { pub struct Ignore; impl<'de> Deserializer<'de> { + /// Record separator; an intentionally invalid-utf8 byte. const SEP: u8 = b'\xFF'; + /// Determine if the input was fully consumed and error if bytes remaining. + /// This is intended for debug assertions; not optimized for parsing logic. fn finished(&self) -> Result<()> { let pos = self.pos; let len = self.buf.len(); @@ -48,6 +53,20 @@ impl<'de> Deserializer<'de> { ))) } + /// Consume the current record to ignore it. Inside a sequence the next + /// record is skipped but at the top-level all records are skipped such that + /// deserialization completes with self.finished() == Ok. + #[inline] + fn record_ignore(&mut self) { + if self.seq { + self.record_next(); + } else { + self.record_trail(); + } + } + + /// Consume the current record. The position pointer is moved to the start + /// of the next record. Slice of the current record is returned. #[inline] fn record_next(&mut self) -> &'de [u8] { self.buf[self.pos..] @@ -57,8 +76,10 @@ impl<'de> Deserializer<'de> { .expect("remainder of buf even if SEP was not found") } + /// Peek at the first byte of the current record. If all records were + /// consumed None is returned instead. #[inline] - fn record_next_peek_byte(&self) -> Option { + fn record_peek_byte(&self) -> Option { let started = self.pos != 0; let buf = &self.buf[self.pos..]; debug_assert!( @@ -69,6 +90,8 @@ impl<'de> Deserializer<'de> { buf.get::(started.into()).copied() } + /// Consume the record separator such that the position cleanly points to + /// the start of the next record. (Case for some sequences) #[inline] fn record_start(&mut self) { let started = self.pos != 0; @@ -78,8 +101,11 @@ impl<'de> Deserializer<'de> { ); self.inc_pos(started.into()); + self.seq = true; } + /// Consume all remaining bytes, which may include record separators, + /// returning a raw slice. #[inline] fn record_trail(&mut self) -> &'de [u8] { let record = &self.buf[self.pos..]; @@ -87,6 +113,7 @@ impl<'de> Deserializer<'de> { record } + /// Increment the position pointer. #[inline] fn inc_pos(&mut self, n: usize) { self.pos = self.pos.saturating_add(n); @@ -142,7 +169,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { V: Visitor<'de>, { match name { - "Ignore" => self.record_next(), + "Ignore" => self.record_ignore(), _ => unimplemented!("Unrecognized deserialization Directive {name:?}"), }; @@ -190,7 +217,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { fn deserialize_i64>(self, visitor: V) -> Result { let bytes: [u8; size_of::()] = self.buf[self.pos..].try_into()?; - self.pos = self.pos.saturating_add(size_of::()); + self.inc_pos(size_of::()); visitor.visit_i64(i64::from_be_bytes(bytes)) } @@ -208,7 +235,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { fn deserialize_u64>(self, visitor: V) -> Result { let bytes: [u8; size_of::()] = self.buf[self.pos..].try_into()?; - self.pos = self.pos.saturating_add(size_of::()); + self.inc_pos(size_of::()); visitor.visit_u64(u64::from_be_bytes(bytes)) } @@ -267,7 +294,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { "deserialize_any: type not expected" ); - match self.record_next_peek_byte() { + match self.record_peek_byte() { Some(b'{') => self.deserialize_map(visitor), _ => self.deserialize_str(visitor), } From 2d049dacc37a7f3c7265006a8ebc39516ce7ee55 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Oct 2024 02:23:28 +0000 Subject: [PATCH 0875/2091] fix get_all_media_keys deserialization Signed-off-by: Jason Volk --- src/service/media/data.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 248e9e1d..b2271883 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -122,8 +122,9 @@ impl Data { let keys: Vec> = self .mediaid_file - .keys_prefix_raw(&prefix) + .raw_keys_prefix(&prefix) .ignore_err() + .map(<[u8]>::to_vec) .collect() .await; From bd9a9cc5f84066d7131ded8263a46b1ab57667b8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 22:03:39 +0000 Subject: [PATCH 0876/2091] fix trait-solver issue requiring recursion_limit increase Signed-off-by: Jason Volk --- src/api/mod.rs | 2 -- src/main/main.rs | 2 -- src/router/mod.rs | 2 -- src/service/mod.rs | 1 - src/service/service.rs | 45 ++++++++++++++++++++++++++++++++--------- src/service/services.rs | 2 ++ 6 files changed, 37 insertions(+), 17 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 82b857db..96837470 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,5 +1,3 @@ -#![recursion_limit = "192"] - pub mod client; pub mod router; pub mod server; diff --git a/src/main/main.rs b/src/main/main.rs index 8703eef2..8e644a15 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -1,5 +1,3 @@ -#![recursion_limit = "192"] - pub(crate) mod clap; mod mods; mod restart; diff --git a/src/router/mod.rs b/src/router/mod.rs index 67ebc0e3..e123442c 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -1,5 +1,3 @@ -#![recursion_limit = "160"] - mod layers; mod request; mod router; diff --git a/src/service/mod.rs b/src/service/mod.rs index cb8bfcd9..604e3404 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,4 +1,3 @@ -#![recursion_limit = "192"] #![allow(refining_impl_trait)] mod manager; diff --git a/src/service/service.rs b/src/service/service.rs index 03165050..7ec2ea0f 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -51,7 +51,7 @@ pub(crate) struct Args<'a> { /// Dep is a reference to a service used within another service. /// Circular-dependencies between services require this indirection. -pub(crate) struct Dep { +pub(crate) struct Dep { dep: OnceLock>, service: Weak, name: &'static str, @@ -62,24 +62,47 @@ pub(crate) type MapType = BTreeMap; pub(crate) type MapVal = (Weak, Weak); pub(crate) type MapKey = String; -impl Deref for Dep { +/// SAFETY: Workaround for a compiler limitation (or bug) where it is Hard to +/// prove the Sync'ness of Dep because services contain circular references +/// to other services through Dep's. The Sync'ness of Dep can still be +/// proved without unsafety by declaring the crate-attribute #![recursion_limit +/// = "192"] but this may take a while. Re-evaluate this when a new trait-solver +/// (such as Chalk) becomes available. +unsafe impl Sync for Dep {} + +/// SAFETY: Ancillary to unsafe impl Sync; while this is not needed to prevent +/// violating the recursion_limit, the trait-solver still spends an inordinate +/// amount of time to prove this. +unsafe impl Send for Dep {} + +impl Deref for Dep { type Target = Arc; /// Dereference a dependency. The dependency must be ready or panics. + #[inline] fn deref(&self) -> &Self::Target { - self.dep.get_or_init(|| { - let service = self - .service - .upgrade() - .expect("services map exists for dependency initialization."); + self.dep.get_or_init( + #[inline(never)] + || self.init(), + ) + } +} - require::(&service, self.name) - }) +impl Dep { + #[inline] + fn init(&self) -> Arc { + let service = self + .service + .upgrade() + .expect("services map exists for dependency initialization."); + + require::(&service, self.name) } } impl<'a> Args<'a> { /// Create a lazy-reference to a service when constructing another Service. + #[inline] pub(crate) fn depend(&'a self, name: &'static str) -> Dep { Dep:: { dep: OnceLock::new(), @@ -90,12 +113,14 @@ impl<'a> Args<'a> { /// Create a reference immediately to a service when constructing another /// Service. The other service must be constructed. + #[inline] pub(crate) fn require(&'a self, name: &str) -> Arc { require::(self.service, name) } } /// Reference a Service by name. Panics if the Service does not exist or was /// incorrectly cast. -pub(crate) fn require(map: &Map, name: &str) -> Arc { +#[inline] +fn require(map: &Map, name: &str) -> Arc { try_get::(map, name) .inspect_err(inspect_log) .expect("Failure to reference service required by another service.") diff --git a/src/service/services.rs b/src/service/services.rs index da22fb2d..0b63a5ca 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -195,6 +195,7 @@ impl Services { } } + #[inline] pub fn try_get(&self, name: &str) -> Result> where T: Any + Send + Sync + Sized, @@ -202,6 +203,7 @@ impl Services { service::try_get::(&self.service, name) } + #[inline] pub fn get(&self, name: &str) -> Option> where T: Any + Send + Sync + Sized, From ba683cf5340ff4321b8e8789b101d923b07bd9d4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Oct 2024 17:17:10 +0000 Subject: [PATCH 0877/2091] fix aliasid_alias key deserialization Signed-off-by: Jason Volk --- src/service/rooms/alias/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 1d44cd2d..f50cc46c 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -101,9 +101,9 @@ impl Service { let prefix = (&room_id, Interfix); self.db .aliasid_alias - .keys_prefix(&prefix) + .keys_raw_prefix(&prefix) .ignore_err() - .ready_for_each(|key: &[u8]| self.db.aliasid_alias.remove(&key)) + .ready_for_each(|key| self.db.aliasid_alias.remove(key)) .await; self.db.alias_roomid.remove(alias.as_bytes()); @@ -161,7 +161,7 @@ impl Service { .aliasid_alias .stream_prefix(&prefix) .ignore_err() - .map(|((Ignore, Ignore), alias): ((Ignore, Ignore), &RoomAliasId)| alias) + .map(|(_, alias): (Ignore, &RoomAliasId)| alias) } #[tracing::instrument(skip(self), level = "debug")] From 9eace1fbbb8eaaf819d12649e0a777dd5c7f4cf6 Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 5 Oct 2024 12:30:05 -0400 Subject: [PATCH 0878/2091] fix sliding sync room type filter regression Signed-off-by: strawberry --- src/api/client/sync.rs | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index c4ff1eeb..5940d7cf 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -6,7 +6,9 @@ use std::{ use axum::extract::State; use conduit::{ - debug, err, error, is_equal_to, + debug, err, error, + error::is_not_found, + is_equal_to, result::IntoIsOk, utils::{ math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, @@ -1887,18 +1889,21 @@ async fn filter_rooms( .iter() .stream() .filter_map(|r| async move { - match services.rooms.state_accessor.get_room_type(r).await { - Err(_) => false, - Ok(result) => { - let result = RoomTypeFilter::from(Some(result)); - if negate { - !filter.contains(&result) - } else { - filter.is_empty() || filter.contains(&result) - } - }, + let room_type = services.rooms.state_accessor.get_room_type(r).await; + + if room_type.as_ref().is_err_and(|e| !is_not_found(e)) { + return None; } - .then_some(r.to_owned()) + + let room_type_filter = RoomTypeFilter::from(room_type.ok()); + + let include = if negate { + !filter.contains(&room_type_filter) + } else { + filter.is_empty() || filter.contains(&room_type_filter) + }; + + include.then_some(r.to_owned()) }) .collect() .await From 8eec78e9e0e5076289e98b82b2bb9b4a139be70d Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 5 Oct 2024 12:51:15 -0400 Subject: [PATCH 0879/2091] mark the server user bot as online/offline on shutdown/startup Signed-off-by: strawberry --- src/service/services.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/service/services.rs b/src/service/services.rs index 0b63a5ca..ea81f434 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -123,6 +123,14 @@ impl Services { .start() .await?; + // set the server user as online + if self.server.config.allow_local_presence { + _ = self + .presence + .ping_presence(&self.globals.server_user, &ruma::presence::PresenceState::Online) + .await; + } + debug_info!("Services startup complete."); Ok(Arc::clone(self)) } @@ -130,6 +138,14 @@ impl Services { pub async fn stop(&self) { info!("Shutting down services..."); + // set the server user as offline + if self.server.config.allow_local_presence { + _ = self + .presence + .ping_presence(&self.globals.server_user, &ruma::presence::PresenceState::Offline) + .await; + } + self.interrupt(); if let Some(manager) = self.manager.lock().await.as_ref() { manager.stop().await; From 814b9e28b68dbe6af91d3a397f6a631f1f4e9113 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 9 Oct 2024 03:37:13 +0000 Subject: [PATCH 0880/2091] fix unnecessary re-serializations Signed-off-by: Jason Volk --- src/database/map/rev_stream_from.rs | 3 +-- src/database/map/stream_from.rs | 3 +-- src/service/rooms/metadata/mod.rs | 4 ++-- src/service/rooms/pdu_metadata/data.rs | 2 +- src/service/rooms/short/mod.rs | 2 +- src/service/rooms/state_cache/mod.rs | 4 ++-- src/service/updates/mod.rs | 2 +- 7 files changed, 9 insertions(+), 11 deletions(-) diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 650cf038..c48f406b 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -18,8 +18,7 @@ where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, { - let key = ser::serialize_to_vec(from).expect("failed to serialize query key"); - self.rev_stream_raw_from(&key) + self.rev_stream_raw_from(from) .map(keyval::result_deserialize::) } diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 153d5bb6..db828125 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -18,8 +18,7 @@ where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, { - let key = ser::serialize_to_vec(from).expect("failed to serialize query key"); - self.stream_raw_from(&key) + self.stream_raw_from(from) .map(keyval::result_deserialize::) } diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 5d4a47c7..d8be6aab 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -85,8 +85,8 @@ pub fn list_banned_rooms(&self) -> impl Stream + Send + '_ { sel #[implement(Service)] #[inline] -pub async fn is_disabled(&self, room_id: &RoomId) -> bool { self.db.disabledroomids.qry(room_id).await.is_ok() } +pub async fn is_disabled(&self, room_id: &RoomId) -> bool { self.db.disabledroomids.get(room_id).await.is_ok() } #[implement(Service)] #[inline] -pub async fn is_banned(&self, room_id: &RoomId) -> bool { self.db.bannedroomids.qry(room_id).await.is_ok() } +pub async fn is_banned(&self, room_id: &RoomId) -> bool { self.db.bannedroomids.get(room_id).await.is_ok() } diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index f2323475..8e045658 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -94,6 +94,6 @@ impl Data { } pub(super) async fn is_event_soft_failed(&self, event_id: &EventId) -> bool { - self.softfailedeventids.qry(event_id).await.is_ok() + self.softfailedeventids.get(event_id).await.is_ok() } } diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 20082da2..bd8fdcc9 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -197,7 +197,7 @@ pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, boo #[implement(Service)] pub async fn get_shortroomid(&self, room_id: &RoomId) -> Result { - self.db.roomid_shortroomid.qry(room_id).await.deserialized() + self.db.roomid_shortroomid.get(room_id).await.deserialized() } #[implement(Service)] diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index b1a71caf..a6c468f5 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -342,7 +342,7 @@ impl Service { /// Returns the number of users which are currently in a room #[tracing::instrument(skip(self), level = "debug")] pub async fn room_joined_count(&self, room_id: &RoomId) -> Result { - self.db.roomid_joinedcount.qry(room_id).await.deserialized() + self.db.roomid_joinedcount.get(room_id).await.deserialized() } #[tracing::instrument(skip(self), level = "debug")] @@ -366,7 +366,7 @@ impl Service { pub async fn room_invited_count(&self, room_id: &RoomId) -> Result { self.db .roomid_invitedcount - .qry(room_id) + .get(room_id) .await .deserialized() } diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index 4e16e22b..fca63725 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -128,7 +128,7 @@ impl Service { pub async fn last_check_for_updates_id(&self) -> u64 { self.db - .qry(LAST_CHECK_FOR_UPDATES_COUNT) + .get(LAST_CHECK_FOR_UPDATES_COUNT) .await .deserialized() .unwrap_or(0_u64) From 56dd0f51392cd3f21f62cc054838d6f01160f6de Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Oct 2024 22:08:55 +0000 Subject: [PATCH 0881/2091] use loop condition to account for loole channel close Signed-off-by: Jason Volk --- src/service/presence/mod.rs | 7 ++++--- src/service/sending/sender.rs | 3 +-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 3b5c4caf..82a99bd5 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -55,14 +55,13 @@ impl crate::Service for Service { async fn worker(self: Arc) -> Result<()> { let mut presence_timers = FuturesUnordered::new(); let receiver = self.timer_receiver.lock().await; - loop { - debug_assert!(!receiver.is_closed(), "channel error"); + while !receiver.is_closed() { tokio::select! { Some(user_id) = presence_timers.next() => { self.process_presence_timer(&user_id).await.log_err().ok(); }, event = receiver.recv_async() => match event { - Err(_e) => return Ok(()), + Err(_) => break, Ok((user_id, timeout)) => { debug!("Adding timer {}: {user_id} timeout:{timeout:?}", presence_timers.len()); presence_timers.push(presence_timer(user_id, timeout)); @@ -70,6 +69,8 @@ impl crate::Service for Service { }, } } + + Ok(()) } fn interrupt(&self) { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 3a401995..19205a65 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -57,8 +57,7 @@ impl Service { let receiver = self.receiver.lock().await; self.initial_requests(&mut futures, &mut statuses).await; - loop { - debug_assert!(!receiver.is_closed(), "channel error"); + while !receiver.is_closed() { tokio::select! { request = receiver.recv_async() => match request { Ok(request) => self.handle_request(request, &mut futures, &mut statuses).await, From 89a3c807002ea7f6278e30541df7f3249b8fc681 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 2 Oct 2024 21:18:32 +0000 Subject: [PATCH 0882/2091] split admin-room branch from build_and_append_pdu (fixes large stack warning) Signed-off-by: Jason Volk --- src/service/rooms/timeline/mod.rs | 146 ++++++++++++++---------------- 1 file changed, 68 insertions(+), 78 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index f8f770bc..b49e9fad 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -8,13 +8,13 @@ use std::{ }; use conduit::{ - debug, err, error, info, + debug, err, error, implement, info, pdu::{EventHash, PduBuilder, PduCount, PduEvent}, utils, utils::{stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, validated, warn, Err, Error, Result, Server, }; -use futures::{future, future::ready, Future, Stream, StreamExt, TryStreamExt}; +use futures::{future, future::ready, Future, FutureExt, Stream, StreamExt, TryStreamExt}; use ruma::{ api::{client::error::ErrorKind, federation}, canonical_json::to_canonical_value, @@ -858,82 +858,7 @@ impl Service { .await?; if self.services.admin.is_admin_room(&pdu.room_id).await { - match pdu.event_type() { - TimelineEventType::RoomEncryption => { - warn!("Encryption is not allowed in the admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Encryption is not allowed in the admins room", - )); - }, - TimelineEventType::RoomMember => { - let target = pdu - .state_key() - .filter(|v| v.starts_with('@')) - .unwrap_or(sender.as_str()); - let server_user = &self.services.globals.server_user.to_string(); - - let content = serde_json::from_str::(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid content in pdu"))?; - - if content.membership == MembershipState::Leave { - if target == server_user { - warn!("Server user cannot leave from admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server user cannot leave from admins room.", - )); - } - - let count = self - .services - .state_cache - .room_members(&pdu.room_id) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .ready_filter(|user| *user != target) - .boxed() - .count() - .await; - - if count < 2 { - warn!("Last admin cannot leave from admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Last admin cannot leave from admins room.", - )); - } - } - - if content.membership == MembershipState::Ban && pdu.state_key().is_some() { - if target == server_user { - warn!("Server user cannot be banned in admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server user cannot be banned in admins room.", - )); - } - - let count = self - .services - .state_cache - .room_members(&pdu.room_id) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .ready_filter(|user| *user != target) - .boxed() - .count() - .await; - - if count < 2 { - warn!("Last admin cannot be banned in admins room"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Last admin cannot be banned in admins room.", - )); - } - } - }, - _ => {}, - } + self.check_pdu_for_admin_room(&pdu, sender).boxed().await?; } // If redaction event is not authorized, do not append it to the timeline @@ -1298,6 +1223,71 @@ impl Service { } } +#[implement(Service)] +#[tracing::instrument(skip_all, level = "debug")] +async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Result<()> { + match pdu.event_type() { + TimelineEventType::RoomEncryption => { + return Err!(Request(Forbidden(error!("Encryption not supported in admins room.")))); + }, + TimelineEventType::RoomMember => { + let target = pdu + .state_key() + .filter(|v| v.starts_with('@')) + .unwrap_or(sender.as_str()); + + let server_user = &self.services.globals.server_user.to_string(); + + let content: RoomMemberEventContent = pdu.get_content()?; + match content.membership { + MembershipState::Leave => { + if target == server_user { + return Err!(Request(Forbidden(error!("Server user cannot leave the admins room.")))); + } + + let count = self + .services + .state_cache + .room_members(&pdu.room_id) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .ready_filter(|user| *user != target) + .boxed() + .count() + .await; + + if count < 2 { + return Err!(Request(Forbidden(error!("Last admin cannot leave the admins room.")))); + } + }, + + MembershipState::Ban if pdu.state_key().is_some() => { + if target == server_user { + return Err!(Request(Forbidden(error!("Server cannot be banned from admins room.")))); + } + + let count = self + .services + .state_cache + .room_members(&pdu.room_id) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .ready_filter(|user| *user != target) + .boxed() + .count() + .await; + + if count < 2 { + return Err!(Request(Forbidden(error!("Last admin cannot be banned from admins room.")))); + } + }, + _ => {}, + }; + }, + _ => {}, + }; + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; From 08a2fecc0ed2e0404446d16e40bc136dfff7b7c7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 26 Sep 2024 04:59:16 +0000 Subject: [PATCH 0883/2091] catch panics at base functions to integrate with other fatal errors. Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/router/Cargo.toml | 13 +++++++------ src/router/mod.rs | 23 ++++++++++++++++++----- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e72c7e80..b9f366e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -763,6 +763,7 @@ dependencies = [ "conduit_core", "conduit_service", "const-str", + "futures", "http", "http-body-util", "hyper", diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 62690194..e1535868 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -54,20 +54,18 @@ axum-server-dual-protocol.workspace = true axum-server-dual-protocol.optional = true axum-server.workspace = true axum.workspace = true +bytes.workspace = true conduit-admin.workspace = true conduit-api.workspace = true conduit-core.workspace = true conduit-service.workspace = true const-str.workspace = true -log.workspace = true -tokio.workspace = true -tower.workspace = true -tracing.workspace = true -bytes.workspace = true -http-body-util.workspace = true +futures.workspace = true http.workspace = true +http-body-util.workspace = true hyper.workspace = true hyper-util.workspace = true +log.workspace = true ruma.workspace = true rustls.workspace = true rustls.optional = true @@ -78,7 +76,10 @@ sentry-tracing.optional = true sentry-tracing.workspace = true sentry.workspace = true serde_json.workspace = true +tokio.workspace = true +tower.workspace = true tower-http.workspace = true +tracing.workspace = true [target.'cfg(unix)'.dependencies] sd-notify.workspace = true diff --git a/src/router/mod.rs b/src/router/mod.rs index e123442c..1580f605 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -6,10 +6,11 @@ mod serve; extern crate conduit_core as conduit; -use std::{future::Future, pin::Pin, sync::Arc}; +use std::{panic::AssertUnwindSafe, pin::Pin, sync::Arc}; -use conduit::{Result, Server}; +use conduit::{Error, Result, Server}; use conduit_service::Services; +use futures::{Future, FutureExt, TryFutureExt}; conduit::mod_ctor! {} conduit::mod_dtor! {} @@ -17,15 +18,27 @@ conduit::rustc_flags_capture! {} #[no_mangle] pub extern "Rust" fn start(server: &Arc) -> Pin>> + Send>> { - Box::pin(run::start(server.clone())) + AssertUnwindSafe(run::start(server.clone())) + .catch_unwind() + .map_err(Error::from_panic) + .unwrap_or_else(Err) + .boxed() } #[no_mangle] pub extern "Rust" fn stop(services: Arc) -> Pin> + Send>> { - Box::pin(run::stop(services)) + AssertUnwindSafe(run::stop(services)) + .catch_unwind() + .map_err(Error::from_panic) + .unwrap_or_else(Err) + .boxed() } #[no_mangle] pub extern "Rust" fn run(services: &Arc) -> Pin> + Send>> { - Box::pin(run::run(services.clone())) + AssertUnwindSafe(run::run(services.clone())) + .catch_unwind() + .map_err(Error::from_panic) + .unwrap_or_else(Err) + .boxed() } From a2e5c3d5d3bc9253fb634bf8b1b30ec7087e886f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 00:30:16 +0000 Subject: [PATCH 0884/2091] add FlatOk trait to Result/Option suite Signed-off-by: Jason Volk --- src/core/result.rs | 3 ++- src/core/result/flat_ok.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 src/core/result/flat_ok.rs diff --git a/src/core/result.rs b/src/core/result.rs index 82d67a9c..9a60d19e 100644 --- a/src/core/result.rs +++ b/src/core/result.rs @@ -1,4 +1,5 @@ mod debug_inspect; +mod flat_ok; mod into_is_ok; mod log_debug_err; mod log_err; @@ -7,7 +8,7 @@ mod not_found; mod unwrap_infallible; pub use self::{ - debug_inspect::DebugInspect, into_is_ok::IntoIsOk, log_debug_err::LogDebugErr, log_err::LogErr, + debug_inspect::DebugInspect, flat_ok::FlatOk, into_is_ok::IntoIsOk, log_debug_err::LogDebugErr, log_err::LogErr, map_expect::MapExpect, not_found::NotFound, unwrap_infallible::UnwrapInfallible, }; diff --git a/src/core/result/flat_ok.rs b/src/core/result/flat_ok.rs new file mode 100644 index 00000000..e378e5d0 --- /dev/null +++ b/src/core/result/flat_ok.rs @@ -0,0 +1,34 @@ +use super::Result; + +pub trait FlatOk { + /// Equivalent to .transpose().ok().flatten() + fn flat_ok(self) -> Option; + + /// Equivalent to .transpose().ok().flatten().ok_or(...) + fn flat_ok_or(self, err: E) -> Result; + + /// Equivalent to .transpose().ok().flatten().ok_or_else(...) + fn flat_ok_or_else E>(self, err: F) -> Result; +} + +impl FlatOk for Option> { + #[inline] + fn flat_ok(self) -> Option { self.transpose().ok().flatten() } + + #[inline] + fn flat_ok_or(self, err: Ep) -> Result { self.flat_ok().ok_or(err) } + + #[inline] + fn flat_ok_or_else Ep>(self, err: F) -> Result { self.flat_ok().ok_or_else(err) } +} + +impl FlatOk for Result, E> { + #[inline] + fn flat_ok(self) -> Option { self.ok().flatten() } + + #[inline] + fn flat_ok_or(self, err: Ep) -> Result { self.flat_ok().ok_or(err) } + + #[inline] + fn flat_ok_or_else Ep>(self, err: F) -> Result { self.flat_ok().ok_or_else(err) } +} From 4485f36e34d9da010b37d0db832ac6e38c794e7e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 01:00:00 +0000 Subject: [PATCH 0885/2091] add mactors for true/false Signed-off-by: Jason Volk --- src/core/mod.rs | 56 ++++++++++++++++++++++++++++++++++++++++++ src/core/utils/math.rs | 32 ------------------------ 2 files changed, 56 insertions(+), 32 deletions(-) diff --git a/src/core/mod.rs b/src/core/mod.rs index e4553186..491d8b4c 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -38,3 +38,59 @@ pub mod mods { () => {}; } } + +/// Functor for falsy +#[macro_export] +macro_rules! is_false { + () => { + |x| !x + }; +} + +/// Functor for truthy +#[macro_export] +macro_rules! is_true { + () => { + |x| !!x + }; +} + +/// Functor for equality to zero +#[macro_export] +macro_rules! is_zero { + () => { + $crate::is_matching!(0) + }; +} + +/// Functor for equality i.e. .is_some_and(is_equal!(2)) +#[macro_export] +macro_rules! is_equal_to { + ($val:expr) => { + |x| x == $val + }; +} + +/// Functor for less i.e. .is_some_and(is_less_than!(2)) +#[macro_export] +macro_rules! is_less_than { + ($val:expr) => { + |x| x < $val + }; +} + +/// Functor for matches! i.e. .is_some_and(is_matching!('A'..='Z')) +#[macro_export] +macro_rules! is_matching { + ($val:expr) => { + |x| matches!(x, $val) + }; +} + +/// Functor for !is_empty() +#[macro_export] +macro_rules! is_not_empty { + () => { + |x| !x.is_empty() + }; +} diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index 215de339..ccff6400 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -53,38 +53,6 @@ macro_rules! validated { ($($input:tt)+) => { $crate::expected!($($input)+) } } -/// Functor for equality to zero -#[macro_export] -macro_rules! is_zero { - () => { - $crate::is_matching!(0) - }; -} - -/// Functor for equality i.e. .is_some_and(is_equal!(2)) -#[macro_export] -macro_rules! is_equal_to { - ($val:expr) => { - |x| (x == $val) - }; -} - -/// Functor for less i.e. .is_some_and(is_less_than!(2)) -#[macro_export] -macro_rules! is_less_than { - ($val:expr) => { - |x| (x < $val) - }; -} - -/// Functor for matches! i.e. .is_some_and(is_matching!('A'..='Z')) -#[macro_export] -macro_rules! is_matching { - ($val:expr) => { - |x| matches!(x, $val) - }; -} - /// Returns false if the exponential backoff has expired based on the inputs #[inline] #[must_use] From dd9f53080acb354905a610dc235c8720c72d742c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 08:04:16 +0000 Subject: [PATCH 0886/2091] add unwrap_or to TryFutureExtExt Signed-off-by: Jason Volk --- src/core/utils/future/try_ext_ext.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/core/utils/future/try_ext_ext.rs b/src/core/utils/future/try_ext_ext.rs index e444ad94..d30d2cac 100644 --- a/src/core/utils/future/try_ext_ext.rs +++ b/src/core/utils/future/try_ext_ext.rs @@ -1,6 +1,9 @@ //! Extended external extensions to futures::TryFutureExt -use futures::{future::MapOkOrElse, TryFuture, TryFutureExt}; +use futures::{ + future::{MapOkOrElse, UnwrapOrElse}, + TryFuture, TryFutureExt, +}; /// This interface is not necessarily complete; feel free to add as-needed. pub trait TryExtExt @@ -19,6 +22,10 @@ where ) -> MapOkOrElse Option, impl FnOnce(Self::Error) -> Option> where Self: Sized; + + fn unwrap_or(self, default: Self::Ok) -> UnwrapOrElse Self::Ok> + where + Self: Sized; } impl TryExtExt for Fut @@ -45,4 +52,12 @@ where { self.map_ok_or(None, Some) } + + #[inline] + fn unwrap_or(self, default: Self::Ok) -> UnwrapOrElse Self::Ok> + where + Self: Sized, + { + self.unwrap_or_else(move |_| default) + } } From 685eadb1713d0d09a48025b74f05407cd6f65742 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Oct 2024 17:07:31 +0000 Subject: [PATCH 0887/2091] add is_not_found as Error member function; tweak interface; add doc comments Signed-off-by: Jason Volk --- src/api/client/sync.rs | 8 +++----- src/api/server/send.rs | 4 ++-- src/core/error/mod.rs | 21 +++++++++++++++++---- src/core/result/not_found.rs | 4 ++-- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 5940d7cf..f0b26e80 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -6,10 +6,8 @@ use std::{ use axum::extract::State; use conduit::{ - debug, err, error, - error::is_not_found, - is_equal_to, - result::IntoIsOk, + debug, err, error, is_equal_to, + result::{FlatOk, IntoIsOk}, utils::{ math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, BoolExt, IterStream, ReadyExt, TryFutureExtExt, @@ -1891,7 +1889,7 @@ async fn filter_rooms( .filter_map(|r| async move { let room_type = services.rooms.state_accessor.get_room_type(r).await; - if room_type.as_ref().is_err_and(|e| !is_not_found(e)) { + if room_type.as_ref().is_err_and(|e| !e.is_not_found()) { return None; } diff --git a/src/api/server/send.rs b/src/api/server/send.rs index bb424988..50a79e00 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{debug, debug_warn, err, result::LogErr, trace, utils::ReadyExt, warn, Err, Error, Result}; +use conduit::{debug, debug_warn, err, error, result::LogErr, trace, utils::ReadyExt, warn, Err, Error, Result}; use futures::StreamExt; use ruma::{ api::{ @@ -85,7 +85,7 @@ pub(crate) async fn send_transaction_message_route( Ok(send_transaction_message::v1::Response { pdus: resolved_map .into_iter() - .map(|(e, r)| (e, r.map_err(|e| e.sanitized_string()))) + .map(|(e, r)| (e, r.map_err(error::sanitized_message))) .collect(), }) } diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index ad7f9f3c..39fa4340 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -120,17 +120,19 @@ pub enum Error { } impl Error { + //#[deprecated] pub fn bad_database(message: &'static str) -> Self { crate::err!(Database(error!("{message}"))) } /// Sanitizes public-facing errors that can leak sensitive information. - pub fn sanitized_string(&self) -> String { + pub fn sanitized_message(&self) -> String { match self { Self::Database(..) => String::from("Database error occurred."), Self::Io(..) => String::from("I/O error occurred."), - _ => self.to_string(), + _ => self.message(), } } + /// Generate the error message string. pub fn message(&self) -> String { match self { Self::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), @@ -151,6 +153,8 @@ impl Error { } } + /// Returns the HTTP error code or closest approximation based on error + /// variant. pub fn status_code(&self) -> http::StatusCode { use http::StatusCode; @@ -163,10 +167,17 @@ impl Error { _ => StatusCode::INTERNAL_SERVER_ERROR, } } + + /// Returns true for "not found" errors. This means anything that qualifies + /// as a "not found" from any variant's contained error type. This call is + /// often used as a special case to eliminate a contained Option with a + /// Result where Ok(None) is instead Err(e) if e.is_not_found(). + #[inline] + pub fn is_not_found(&self) -> bool { self.status_code() == http::StatusCode::NOT_FOUND } } impl fmt::Debug for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{self}") } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.message()) } } #[allow(clippy::fallible_impl_from)] @@ -184,6 +195,8 @@ pub fn infallible(_e: &Infallible) { panic!("infallible error should never exist"); } +/// Convenience functor for fundamental Error::sanitized_message(); see member. #[inline] #[must_use] -pub fn is_not_found(e: &Error) -> bool { e.status_code() == http::StatusCode::NOT_FOUND } +#[allow(clippy::needless_pass_by_value)] +pub fn sanitized_message(e: Error) -> String { e.sanitized_message() } diff --git a/src/core/result/not_found.rs b/src/core/result/not_found.rs index 69ce821b..d61825af 100644 --- a/src/core/result/not_found.rs +++ b/src/core/result/not_found.rs @@ -1,5 +1,5 @@ use super::Result; -use crate::{error, Error}; +use crate::Error; pub trait NotFound { #[must_use] @@ -8,5 +8,5 @@ pub trait NotFound { impl NotFound for Result { #[inline] - fn is_not_found(&self) -> bool { self.as_ref().is_err_and(error::is_not_found) } + fn is_not_found(&self) -> bool { self.as_ref().is_err_and(Error::is_not_found) } } From 2b2055fe8a47ba9dd1981237ee130923766938f4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Oct 2024 03:40:00 +0000 Subject: [PATCH 0888/2091] parallelize calculate_invite_state Signed-off-by: Jason Volk --- src/api/client/membership.rs | 2 +- src/service/rooms/state/mod.rs | 80 ++++++++++--------------------- src/service/rooms/timeline/mod.rs | 5 +- 3 files changed, 27 insertions(+), 60 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index f89903b4..ae56094c 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1452,7 +1452,7 @@ pub(crate) async fn invite_helper( ) .await?; - let invite_room_state = services.rooms.state.calculate_invite_state(&pdu).await?; + let invite_room_state = services.rooms.state.summary_stripped(&pdu).await; drop(state_lock); diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index c7f6605c..177b7e9b 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -3,6 +3,7 @@ mod data; use std::{ collections::{HashMap, HashSet}, fmt::Write, + iter::once, sync::Arc, }; @@ -13,7 +14,7 @@ use conduit::{ }; use data::Data; use database::{Ignore, Interfix}; -use futures::{pin_mut, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{future::join_all, pin_mut, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{ events::{ room::{create::RoomCreateEventContent, member::RoomMemberEventContent}, @@ -288,61 +289,30 @@ impl Service { } } - #[tracing::instrument(skip(self, invite_event), level = "debug")] - pub async fn calculate_invite_state(&self, invite_event: &PduEvent) -> Result>> { - let mut state = Vec::new(); - // Add recommended events - if let Ok(e) = self - .services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomCreate, "") - .await - { - state.push(e.to_stripped_state_event()); - } - if let Ok(e) = self - .services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomJoinRules, "") - .await - { - state.push(e.to_stripped_state_event()); - } - if let Ok(e) = self - .services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomCanonicalAlias, "") - .await - { - state.push(e.to_stripped_state_event()); - } - if let Ok(e) = self - .services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomAvatar, "") - .await - { - state.push(e.to_stripped_state_event()); - } - if let Ok(e) = self - .services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomName, "") - .await - { - state.push(e.to_stripped_state_event()); - } - if let Ok(e) = self - .services - .state_accessor - .room_state_get(&invite_event.room_id, &StateEventType::RoomMember, invite_event.sender.as_str()) - .await - { - state.push(e.to_stripped_state_event()); - } + #[tracing::instrument(skip_all, level = "debug")] + pub async fn summary_stripped(&self, invite: &PduEvent) -> Vec> { + let cells = [ + (&StateEventType::RoomCreate, ""), + (&StateEventType::RoomJoinRules, ""), + (&StateEventType::RoomCanonicalAlias, ""), + (&StateEventType::RoomName, ""), + (&StateEventType::RoomAvatar, ""), + (&StateEventType::RoomMember, invite.sender.as_str()), // Add recommended events + ]; - state.push(invite_event.to_stripped_state_event()); - Ok(state) + let fetches = cells.iter().map(|(event_type, state_key)| { + self.services + .state_accessor + .room_state_get(&invite.room_id, event_type, state_key) + }); + + join_all(fetches) + .await + .into_iter() + .filter_map(Result::ok) + .map(|e| e.to_stripped_state_event()) + .chain(once(invite.to_stripped_state_event())) + .collect() } /// Set the state hash to a new version, but does not update state_cache. diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index b49e9fad..84f29c86 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -513,10 +513,7 @@ impl Service { })?; let invite_state = match content.membership { - MembershipState::Invite => { - let state = self.services.state.calculate_invite_state(pdu).await?; - Some(state) - }, + MembershipState::Invite => self.services.state.summary_stripped(pdu).await.into(), _ => None, }; From 48a767d52c3d24c6a460a6defdddaa9c7c707387 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 01:01:25 +0000 Subject: [PATCH 0889/2091] abstract common patterns as core pdu memberfns Signed-off-by: Jason Volk --- src/core/pdu/mod.rs | 67 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index cf9ffe64..a94e2bdc 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -18,11 +18,11 @@ use ruma::{ use serde::{Deserialize, Serialize}; use serde_json::{ json, - value::{to_raw_value, RawValue as RawJsonValue}, + value::{to_raw_value, RawValue as RawJsonValue, Value as JsonValue}, }; pub use self::{builder::PduBuilder, count::PduCount}; -use crate::{err, warn, Error, Result}; +use crate::{err, is_true, warn, Error, Result}; #[derive(Deserialize)] struct ExtractRedactedBecause { @@ -58,8 +58,8 @@ pub struct PduEvent { pub unsigned: Option>, pub hashes: EventHash, #[serde(default, skip_serializing_if = "Option::is_none")] - pub signatures: Option>, /* BTreeMap, BTreeMap> */ + // BTreeMap, BTreeMap> + pub signatures: Option>, } impl PduEvent { @@ -170,6 +170,54 @@ impl PduEvent { (self.redacts.clone(), self.content.clone()) } + #[must_use] + pub fn get_content_as_value(&self) -> JsonValue { + self.get_content() + .expect("pdu content must be a valid JSON value") + } + + pub fn get_content(&self) -> Result + where + T: for<'de> Deserialize<'de>, + { + serde_json::from_str(self.content.get()) + .map_err(|e| err!(Database("Failed to deserialize pdu content into type: {e}"))) + } + + pub fn contains_unsigned_property(&self, property: &str, is_type: F) -> bool + where + F: FnOnce(&JsonValue) -> bool, + { + self.get_unsigned_as_value() + .get(property) + .map(is_type) + .is_some_and(is_true!()) + } + + pub fn get_unsigned_property(&self, property: &str) -> Result + where + T: for<'de> Deserialize<'de>, + { + self.get_unsigned_as_value() + .get_mut(property) + .map(JsonValue::take) + .map(serde_json::from_value) + .ok_or(err!(Request(NotFound("property not found in unsigned object"))))? + .map_err(|e| err!(Database("Failed to deserialize unsigned.{property} into type: {e}"))) + } + + #[must_use] + pub fn get_unsigned_as_value(&self) -> JsonValue { self.get_unsigned::().unwrap_or_default() } + + pub fn get_unsigned(&self) -> Result { + self.unsigned + .as_ref() + .map(|raw| raw.get()) + .map(serde_json::from_str) + .ok_or(err!(Request(NotFound("\"unsigned\" property not found in pdu"))))? + .map_err(|e| err!(Database("Failed to deserialize \"unsigned\" into value: {e}"))) + } + #[tracing::instrument(skip(self), level = "debug")] pub fn to_sync_room_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); @@ -270,8 +318,8 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_state_event(&self) -> Raw { + #[must_use] + pub fn to_state_event_value(&self) -> JsonValue { let mut json = json!({ "content": self.content, "type": self.kind, @@ -286,7 +334,12 @@ impl PduEvent { json["unsigned"] = json!(unsigned); } - serde_json::from_value(json).expect("Raw::from_value always works") + json + } + + #[tracing::instrument(skip(self), level = "debug")] + pub fn to_state_event(&self) -> Raw { + serde_json::from_value(self.to_state_event_value()).expect("Raw::from_value always works") } #[tracing::instrument(skip(self), level = "debug")] From da34b43302d8e0d66dc218a1612e5a6eb18cb710 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 2 Oct 2024 07:57:18 +0000 Subject: [PATCH 0890/2091] abstract account-data deserializations for serde_json::from_elim Signed-off-by: Jason Volk --- src/admin/query/account_data.rs | 9 +-- src/admin/user/commands.rs | 42 ++++------ src/api/client/config.rs | 20 ++--- src/api/client/push.rs | 117 ++++++++++----------------- src/api/client/tag.rs | 68 ++++++---------- src/service/account_data/mod.rs | 39 ++++++--- src/service/admin/grant.rs | 3 +- src/service/globals/migrations.rs | 11 +-- src/service/rooms/state_cache/mod.rs | 14 +--- src/service/rooms/timeline/mod.rs | 3 +- src/service/sending/sender.rs | 3 +- src/service/users/mod.rs | 14 +--- 12 files changed, 133 insertions(+), 210 deletions(-) diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index 896bf95c..ea45eb16 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -1,9 +1,6 @@ use clap::Subcommand; use conduit::Result; -use ruma::{ - events::{room::message::RoomMessageEventContent, RoomAccountDataEventType}, - RoomId, UserId, -}; +use ruma::{events::room::message::RoomMessageEventContent, RoomId, UserId}; use crate::Command; @@ -25,7 +22,7 @@ pub(crate) enum AccountDataCommand { /// Full user ID user_id: Box, /// Account data event type - kind: RoomAccountDataEventType, + kind: String, /// Optional room ID of the account data room_id: Option>, }, @@ -60,7 +57,7 @@ pub(super) async fn process(subcommand: AccountDataCommand, context: &Command<'_ let timer = tokio::time::Instant::now(); let results = services .account_data - .get(room_id.as_deref(), &user_id, kind) + .get_raw(room_id.as_deref(), &user_id, &kind) .await; let query_time = timer.elapsed(); diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 1b086856..562bb9c7 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -501,20 +501,16 @@ pub(super) async fn put_room_tag( ) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; - let event = self + let mut tags_event = self .services .account_data - .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag) - .await; - - let mut tags_event = event.map_or_else( - |_| TagEvent { + .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) + .await + .unwrap_or(TagEvent { content: TagEventContent { tags: BTreeMap::new(), }, - }, - |e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"), - ); + }); tags_event .content @@ -542,20 +538,16 @@ pub(super) async fn delete_room_tag( ) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; - let event = self + let mut tags_event = self .services .account_data - .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag) - .await; - - let mut tags_event = event.map_or_else( - |_| TagEvent { + .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) + .await + .unwrap_or(TagEvent { content: TagEventContent { tags: BTreeMap::new(), }, - }, - |e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"), - ); + }); tags_event.content.tags.remove(&tag.clone().into()); @@ -578,20 +570,16 @@ pub(super) async fn delete_room_tag( pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; - let event = self + let tags_event = self .services .account_data - .get(Some(&room_id), &user_id, RoomAccountDataEventType::Tag) - .await; - - let tags_event = event.map_or_else( - |_| TagEvent { + .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) + .await + .unwrap_or(TagEvent { content: TagEventContent { tags: BTreeMap::new(), }, - }, - |e| serde_json::from_str(e.get()).expect("Bad account data in database for user {user_id}"), - ); + }); Ok(RoomMessageEventContent::notice_markdown(format!( "```\n{:#?}\n```", diff --git a/src/api/client/config.rs b/src/api/client/config.rs index 33b85136..d06cc072 100644 --- a/src/api/client/config.rs +++ b/src/api/client/config.rs @@ -58,18 +58,14 @@ pub(crate) async fn get_global_account_data_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event: Box = services + let account_data: ExtractGlobalEventContent = services .account_data - .get(None, sender_user, body.event_type.to_string().into()) + .get_global(sender_user, body.event_type.clone()) .await .map_err(|_| err!(Request(NotFound("Data not found."))))?; - let account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))? - .content; - Ok(get_global_account_data::v3::Response { - account_data, + account_data: account_data.content, }) } @@ -81,18 +77,14 @@ pub(crate) async fn get_room_account_data_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event: Box = services + let account_data: ExtractRoomEventContent = services .account_data - .get(Some(&body.room_id), sender_user, body.event_type.clone()) + .get_room(&body.room_id, sender_user, body.event_type.clone()) .await .map_err(|_| err!(Request(NotFound("Data not found."))))?; - let account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))? - .content; - Ok(get_room_account_data::v3::Response { - account_data, + account_data: account_data.content, }) } diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 39095199..103c0c5e 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -13,7 +13,7 @@ use ruma::{ GlobalAccountDataEventType, }, push::{InsertPushRuleError, RemovePushRuleError, Ruleset}, - CanonicalJsonObject, + CanonicalJsonObject, CanonicalJsonValue, }; use service::Services; @@ -27,38 +27,23 @@ pub(crate) async fn get_pushrules_all_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let global_ruleset: Ruleset; - - let event = services + let Some(content_value) = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) - .await; - - let Ok(event) = event else { + .get_global::(sender_user, GlobalAccountDataEventType::PushRules) + .await + .ok() + .and_then(|event| event.get("content").cloned()) + .filter(CanonicalJsonValue::is_object) + else { // user somehow has non-existent push rule event. recreate it and return server // default silently return recreate_push_rules_and_return(&services, sender_user).await; }; - let value = serde_json::from_str::(event.get()) + let account_data_content = serde_json::from_value::(content_value.into()) .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; - let Some(content_value) = value.get("content") else { - // user somehow has a push rule event with no content key, recreate it and - // return server default silently - return recreate_push_rules_and_return(&services, sender_user).await; - }; - - if content_value.to_string().is_empty() { - // user somehow has a push rule event with empty content, recreate it and return - // server default silently - return recreate_push_rules_and_return(&services, sender_user).await; - } - - let account_data_content = serde_json::from_value::(content_value.clone().into()) - .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; - - global_ruleset = account_data_content.global; + let global_ruleset: Ruleset = account_data_content.global; Ok(get_pushrules_all::v3::Response { global: global_ruleset, @@ -73,17 +58,14 @@ pub(crate) async fn get_pushrule_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = services + let event: PushRulesEvent = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(sender_user, GlobalAccountDataEventType::PushRules) .await - .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .map_err(|_| err!(Request(NotFound("PushRules event not found."))))?; - let account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))? - .content; - - let rule = account_data + let rule = event + .content .global .get(body.kind.clone(), &body.rule_id) .map(Into::into); @@ -113,14 +95,11 @@ pub(crate) async fn set_pushrule_route( )); } - let event = services + let mut account_data: PushRulesEvent = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(sender_user, GlobalAccountDataEventType::PushRules) .await - .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; - - let mut account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))?; + .map_err(|_| err!(Request(NotFound("PushRules event not found."))))?; if let Err(error) = account_data @@ -181,21 +160,18 @@ pub(crate) async fn get_pushrule_actions_route( )); } - let event = services + let event: PushRulesEvent = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(sender_user, GlobalAccountDataEventType::PushRules) .await - .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .map_err(|_| err!(Request(NotFound("PushRules event not found."))))?; - let account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))? - .content; - - let global = account_data.global; - let actions = global + let actions = event + .content + .global .get(body.kind.clone(), &body.rule_id) .map(|rule| rule.actions().to_owned()) - .ok_or(Error::BadRequest(ErrorKind::NotFound, "Push rule not found."))?; + .ok_or(err!(Request(NotFound("Push rule not found."))))?; Ok(get_pushrule_actions::v3::Response { actions, @@ -217,14 +193,11 @@ pub(crate) async fn set_pushrule_actions_route( )); } - let event = services + let mut account_data: PushRulesEvent = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(sender_user, GlobalAccountDataEventType::PushRules) .await - .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; - - let mut account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))?; + .map_err(|_| err!(Request(NotFound("PushRules event not found."))))?; if account_data .content @@ -263,20 +236,18 @@ pub(crate) async fn get_pushrule_enabled_route( )); } - let event = services + let event: PushRulesEvent = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(sender_user, GlobalAccountDataEventType::PushRules) .await - .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; + .map_err(|_| err!(Request(NotFound("PushRules event not found."))))?; - let account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))?; - - let global = account_data.content.global; - let enabled = global + let enabled = event + .content + .global .get(body.kind.clone(), &body.rule_id) .map(ruma::push::AnyPushRuleRef::enabled) - .ok_or(Error::BadRequest(ErrorKind::NotFound, "Push rule not found."))?; + .ok_or(err!(Request(NotFound("Push rule not found."))))?; Ok(get_pushrule_enabled::v3::Response { enabled, @@ -298,14 +269,11 @@ pub(crate) async fn set_pushrule_enabled_route( )); } - let event = services + let mut account_data: PushRulesEvent = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(sender_user, GlobalAccountDataEventType::PushRules) .await - .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; - - let mut account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))?; + .map_err(|_| err!(Request(NotFound("PushRules event not found."))))?; if account_data .content @@ -344,14 +312,11 @@ pub(crate) async fn delete_pushrule_route( )); } - let event = services + let mut account_data: PushRulesEvent = services .account_data - .get(None, sender_user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(sender_user, GlobalAccountDataEventType::PushRules) .await - .map_err(|_| Error::BadRequest(ErrorKind::NotFound, "PushRules event not found."))?; - - let mut account_data = serde_json::from_str::(event.get()) - .map_err(|_| Error::bad_database("Invalid account data event in db."))?; + .map_err(|_| err!(Request(NotFound("PushRules event not found."))))?; if let Err(error) = account_data .content diff --git a/src/api/client/tag.rs b/src/api/client/tag.rs index bcd0f817..b5fa19e3 100644 --- a/src/api/client/tag.rs +++ b/src/api/client/tag.rs @@ -9,7 +9,7 @@ use ruma::{ }, }; -use crate::{Error, Result, Ruma}; +use crate::{Result, Ruma}; /// # `PUT /_matrix/client/r0/user/{userId}/rooms/{roomId}/tags/{tag}` /// @@ -21,21 +21,15 @@ pub(crate) async fn update_tag_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = services + let mut tags_event = services .account_data - .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag) - .await; - - let mut tags_event = event.map_or_else( - |_| { - Ok(TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, - }) - }, - |e| serde_json::from_str(e.get()).map_err(|_| Error::bad_database("Invalid account data event in db.")), - )?; + .get_room(&body.room_id, sender_user, RoomAccountDataEventType::Tag) + .await + .unwrap_or(TagEvent { + content: TagEventContent { + tags: BTreeMap::new(), + }, + }); tags_event .content @@ -65,21 +59,15 @@ pub(crate) async fn delete_tag_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = services + let mut tags_event = services .account_data - .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag) - .await; - - let mut tags_event = event.map_or_else( - |_| { - Ok(TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, - }) - }, - |e| serde_json::from_str(e.get()).map_err(|_| Error::bad_database("Invalid account data event in db.")), - )?; + .get_room(&body.room_id, sender_user, RoomAccountDataEventType::Tag) + .await + .unwrap_or(TagEvent { + content: TagEventContent { + tags: BTreeMap::new(), + }, + }); tags_event.content.tags.remove(&body.tag.clone().into()); @@ -106,21 +94,15 @@ pub(crate) async fn get_tags_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let event = services + let tags_event = services .account_data - .get(Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag) - .await; - - let tags_event = event.map_or_else( - |_| { - Ok(TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, - }) - }, - |e| serde_json::from_str(e.get()).map_err(|_| Error::bad_database("Invalid account data event in db.")), - )?; + .get_room(&body.room_id, sender_user, RoomAccountDataEventType::Tag) + .await + .unwrap_or(TagEvent { + content: TagEventContent { + tags: BTreeMap::new(), + }, + }); Ok(get_tags::v3::Response { tags: tags_event.content.tags, diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 482229e7..8065ac55 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -5,14 +5,17 @@ use conduit::{ utils::{stream::TryIgnore, ReadyExt}, Err, Error, Result, }; -use database::{Deserialized, Map}; +use database::{Deserialized, Handle, Map}; use futures::{StreamExt, TryFutureExt}; use ruma::{ - events::{AnyGlobalAccountDataEvent, AnyRawAccountDataEvent, AnyRoomAccountDataEvent, RoomAccountDataEventType}, + events::{ + AnyGlobalAccountDataEvent, AnyRawAccountDataEvent, AnyRoomAccountDataEvent, GlobalAccountDataEventType, + RoomAccountDataEventType, + }, serde::Raw, RoomId, UserId, }; -use serde_json::value::RawValue; +use serde::Deserialize; use crate::{globals, Dep}; @@ -97,18 +100,36 @@ pub async fn update( Ok(()) } -/// Searches the account data for a specific kind. +/// Searches the room account data for a specific kind. #[implement(Service)] -pub async fn get( - &self, room_id: Option<&RoomId>, user_id: &UserId, kind: RoomAccountDataEventType, -) -> Result> { - let key = (room_id, user_id, kind.to_string()); +pub async fn get_global(&self, user_id: &UserId, kind: GlobalAccountDataEventType) -> Result +where + T: for<'de> Deserialize<'de>, +{ + self.get_raw(None, user_id, &kind.to_string()) + .await + .deserialized() +} + +/// Searches the global account data for a specific kind. +#[implement(Service)] +pub async fn get_room(&self, room_id: &RoomId, user_id: &UserId, kind: RoomAccountDataEventType) -> Result +where + T: for<'de> Deserialize<'de>, +{ + self.get_raw(Some(room_id), user_id, &kind.to_string()) + .await + .deserialized() +} + +#[implement(Service)] +pub async fn get_raw(&self, room_id: Option<&RoomId>, user_id: &UserId, kind: &str) -> Result> { + let key = (room_id, user_id, kind.to_owned()); self.db .roomusertype_roomuserdataid .qry(&key) .and_then(|roomuserdataid| self.db.roomuserdataid_accountdata.get(&roomuserdataid)) .await - .deserialized() } /// Returns all changes to the account data that happened after `since`. diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 4b3ebb88..6e266ca9 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -143,9 +143,8 @@ async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> R let mut event = self .services .account_data - .get(Some(room_id), user_id, RoomAccountDataEventType::Tag) + .get_room(room_id, user_id, RoomAccountDataEventType::Tag) .await - .and_then(|event| serde_json::from_str(event.get()).map_err(Into::into)) .unwrap_or_else(|_| TagEvent { content: TagEventContent { tags: BTreeMap::new(), diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index fc6e477b..334e71c6 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -215,13 +215,12 @@ async fn db_lt_12(services: &Services) -> Result<()> { }, }; - let raw_rules_list = services + let mut account_data: PushRulesEvent = services .account_data - .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(&user, GlobalAccountDataEventType::PushRules) .await .expect("Username is invalid"); - let mut account_data = serde_json::from_str::(raw_rules_list.get()).unwrap(); let rules_list = &mut account_data.content.global; //content rule @@ -294,14 +293,12 @@ async fn db_lt_13(services: &Services) -> Result<()> { }, }; - let raw_rules_list = services + let mut account_data: PushRulesEvent = services .account_data - .get(None, &user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(&user, GlobalAccountDataEventType::PushRules) .await .expect("Username is invalid"); - let mut account_data = serde_json::from_str::(raw_rules_list.get()).unwrap(); - let user_default_rules = Ruleset::server_default(&user); account_data .content diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index a6c468f5..8539c940 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -146,12 +146,9 @@ impl Service { if let Ok(tag_event) = self .services .account_data - .get(Some(&predecessor.room_id), user_id, RoomAccountDataEventType::Tag) + .get_room(&predecessor.room_id, user_id, RoomAccountDataEventType::Tag) .await - .and_then(|event| { - serde_json::from_str(event.get()) - .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) - }) { + { self.services .account_data .update(Some(room_id), user_id, RoomAccountDataEventType::Tag, &tag_event) @@ -163,12 +160,9 @@ impl Service { if let Ok(mut direct_event) = self .services .account_data - .get(None, user_id, GlobalAccountDataEventType::Direct.to_string().into()) + .get_global::(user_id, GlobalAccountDataEventType::Direct) .await - .and_then(|event| { - serde_json::from_str::(event.get()) - .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) - }) { + { let mut room_ids_updated = false; for room_ids in direct_event.content.0.values_mut() { if room_ids.iter().any(|r| r == &predecessor.room_id) { diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 84f29c86..7cf06522 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -407,9 +407,8 @@ impl Service { let rules_for_user = self .services .account_data - .get(None, user, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(user, GlobalAccountDataEventType::PushRules) .await - .and_then(|event| serde_json::from_str::(event.get()).map_err(Into::into)) .map_or_else(|_| Ruleset::server_default(user), |ev: PushRulesEvent| ev.content.global); let mut highlight = false; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 19205a65..90977abe 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -539,9 +539,8 @@ impl Service { let rules_for_user = self .services .account_data - .get(None, userid, GlobalAccountDataEventType::PushRules.to_string().into()) + .get_global(userid, GlobalAccountDataEventType::PushRules) .await - .and_then(|event| serde_json::from_str::(event.get()).map_err(Into::into)) .map_or_else( |_| push::Ruleset::server_default(userid), |ev: PushRulesEvent| ev.content.global, diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 44d169dd..3ab6b3c3 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -98,19 +98,9 @@ impl Service { pub async fn user_is_ignored(&self, sender_user: &UserId, recipient_user: &UserId) -> bool { self.services .account_data - .get( - None, - recipient_user, - GlobalAccountDataEventType::IgnoredUserList - .to_string() - .into(), - ) + .get_global(recipient_user, GlobalAccountDataEventType::IgnoredUserList) .await - .and_then(|event| { - serde_json::from_str::(event.get()) - .map_err(|e| err!(Database(warn!("Invalid account data event in db: {e:?}")))) - }) - .map_or(false, |ignored| { + .map_or(false, |ignored: IgnoredUserListEvent| { ignored .content .ignored_users From 68315ac1128196c46802216054b31ec517dbfcb2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 09:38:10 +0000 Subject: [PATCH 0891/2091] Add state_get_content(shortid) for serde_json::from elim Signed-off-by: Jason Volk --- src/api/client/sync.rs | 39 ++++++------- src/service/rooms/state_accessor/mod.rs | 78 ++++++++++--------------- 2 files changed, 48 insertions(+), 69 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index f0b26e80..65d62a78 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -14,7 +14,7 @@ use conduit::{ }, warn, PduCount, }; -use futures::{pin_mut, FutureExt, StreamExt, TryFutureExt}; +use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt, TryFutureExt}; use ruma::{ api::client::{ error::ErrorKind, @@ -681,20 +681,22 @@ async fn load_joined_room( )) }; - let since_sender_member: Option = if let Some(short) = since_shortstatehash { + let get_sender_member_content = |short| { services .rooms .state_accessor - .state_get(short, &StateEventType::RoomMember, sender_user.as_str()) - .await - .and_then(|pdu| serde_json::from_str(pdu.content.get()).map_err(Into::into)) + .state_get_content(short, &StateEventType::RoomMember, sender_user.as_str()) .ok() - } else { - None }; - let joined_since_last_sync = - since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + let since_sender_member: OptionFuture<_> = since_shortstatehash.map(get_sender_member_content).into(); + + let joined_since_last_sync = since_sender_member + .await + .flatten() + .map_or(true, |content: RoomMemberEventContent| { + content.membership != MembershipState::Join + }); if since_shortstatehash.is_none() || joined_since_last_sync { // Probably since = 0, we will do an initial sync @@ -1296,18 +1298,6 @@ pub(crate) async fn sync_events_v4_route( .await .ok(); - let since_sender_member: Option = if let Some(short) = since_shortstatehash { - services - .rooms - .state_accessor - .state_get(short, &StateEventType::RoomMember, sender_user.as_str()) - .await - .and_then(|pdu| serde_json::from_str(pdu.content.get()).map_err(Into::into)) - .ok() - } else { - None - }; - let encrypted_room = services .rooms .state_accessor @@ -1327,6 +1317,13 @@ pub(crate) async fn sync_events_v4_route( .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") .await; + let since_sender_member: Option = services + .rooms + .state_accessor + .state_get_content(since_shortstatehash, &StateEventType::RoomMember, sender_user.as_str()) + .ok() + .await; + let joined_since_last_sync = since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 4c28483c..ece8679d 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -33,8 +33,8 @@ use ruma::{ }, room::RoomType, space::SpaceRoomJoinRule, - EventEncryptionAlgorithm, EventId, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, - UserId, + EventEncryptionAlgorithm, EventId, JsOption, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, + ServerName, UserId, }; use serde::Deserialize; use serde_json::value::to_raw_value; @@ -125,16 +125,23 @@ impl Service { .await } + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). + pub async fn state_get_content( + &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, + ) -> Result + where + T: for<'de> Deserialize<'de> + Send, + { + self.state_get(shortstatehash, event_type, state_key) + .await + .and_then(|event| event.get_content()) + } + /// Get membership for given user in state async fn user_membership(&self, shortstatehash: u64, user_id: &UserId) -> MembershipState { - self.state_get(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) + self.state_get_content(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) .await - .map_or(MembershipState::Leave, |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomMemberEventContent| c.membership) - .map_err(|_| Error::bad_database("Invalid room membership event in database.")) - .unwrap() - }) + .map_or(MembershipState::Leave, |c: RoomMemberEventContent| c.membership) } /// The user was a joined member at this state (potentially in the past) @@ -171,19 +178,10 @@ impl Service { } let history_visibility = self - .state_get(shortstatehash, &StateEventType::RoomHistoryVisibility, "") + .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") .await - .map_or(HistoryVisibility::Shared, |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomHistoryVisibilityEventContent| c.history_visibility) - .map_err(|e| { - error!( - "Invalid history visibility event in database for room {room_id}, assuming is \"shared\": \ - {e}" - ); - Error::bad_database("Invalid history visibility event in database.") - }) - .unwrap() + .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { + c.history_visibility }); let current_server_members = self @@ -240,19 +238,10 @@ impl Service { let currently_member = self.services.state_cache.is_joined(user_id, room_id).await; let history_visibility = self - .state_get(shortstatehash, &StateEventType::RoomHistoryVisibility, "") + .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") .await - .map_or(HistoryVisibility::Shared, |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomHistoryVisibilityEventContent| c.history_visibility) - .map_err(|e| { - error!( - "Invalid history visibility event in database for room {room_id}, assuming is \"shared\": \ - {e}" - ); - Error::bad_database("Invalid history visibility event in database.") - }) - .unwrap() + .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { + c.history_visibility }); let visibility = match history_visibility { @@ -284,25 +273,18 @@ impl Service { /// the room's history_visibility at that event's state. #[tracing::instrument(skip(self, user_id, room_id))] pub async fn user_can_see_state_events(&self, user_id: &UserId, room_id: &RoomId) -> bool { - let currently_member = self.services.state_cache.is_joined(user_id, room_id).await; + if self.services.state_cache.is_joined(user_id, room_id).await { + return true; + } let history_visibility = self - .room_state_get(room_id, &StateEventType::RoomHistoryVisibility, "") + .room_state_get_content(room_id, &StateEventType::RoomHistoryVisibility, "") .await - .map_or(Ok(HistoryVisibility::Shared), |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomHistoryVisibilityEventContent| c.history_visibility) - .map_err(|e| { - error!( - "Invalid history visibility event in database for room {room_id}, assuming is \"shared\": \ - {e}" - ); - Error::bad_database("Invalid history visibility event in database.") - }) - }) - .unwrap_or(HistoryVisibility::Shared); + .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { + c.history_visibility + }); - currently_member || history_visibility == HistoryVisibility::WorldReadable + history_visibility == HistoryVisibility::WorldReadable } /// Returns the state hash for this pdu. From f7af6966b7fbfac3de141f7101bfe8b5e3904c85 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 09:44:43 +0000 Subject: [PATCH 0892/2091] refactor to room_state_get_content() for serde_json::from_ elim Signed-off-by: Jason Volk --- src/api/client/membership.rs | 78 ++++++++++--------------- src/api/client/profile.rs | 57 ++++++++---------- src/api/client/room.rs | 32 ++++------ src/service/rooms/alias/mod.rs | 27 +++++---- src/service/rooms/spaces/mod.rs | 9 +-- src/service/rooms/state_accessor/mod.rs | 23 +++----- src/service/rooms/timeline/mod.rs | 7 +-- 7 files changed, 91 insertions(+), 142 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index ae56094c..a260b8c5 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -389,17 +389,12 @@ pub(crate) async fn kick_user_route( let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let mut event: RoomMemberEventContent = serde_json::from_str( - services - .rooms - .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) - .await - .map_err(|_| err!(Request(BadState("Cannot kick member that's not in the room."))))? - .content - .get(), - ) - .map_err(|_| err!(Database("Invalid member event in database.")))?; + let mut event: RoomMemberEventContent = services + .rooms + .state_accessor + .room_state_get_content(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) + .await + .map_err(|_| err!(Request(BadState("Cannot kick member that's not in the room."))))?; event.membership = MembershipState::Leave; event.reason.clone_from(&body.reason); @@ -442,10 +437,10 @@ pub(crate) async fn ban_user_route( let event = services .rooms .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) + .room_state_get_content(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) .await - .map_or( - Ok(RoomMemberEventContent { + .map_or_else( + |_| RoomMemberEventContent { membership: MembershipState::Ban, displayname: None, avatar_url: None, @@ -454,21 +449,17 @@ pub(crate) async fn ban_user_route( blurhash: blurhash.clone(), reason: body.reason.clone(), join_authorized_via_users_server: None, - }), - |event| { - serde_json::from_str(event.content.get()) - .map(|event: RoomMemberEventContent| RoomMemberEventContent { - membership: MembershipState::Ban, - displayname: None, - avatar_url: None, - blurhash: blurhash.clone(), - reason: body.reason.clone(), - join_authorized_via_users_server: None, - ..event - }) - .map_err(|e| err!(Database("Invalid member event in database: {e:?}"))) }, - )?; + |event| RoomMemberEventContent { + membership: MembershipState::Ban, + displayname: None, + avatar_url: None, + blurhash: blurhash.clone(), + reason: body.reason.clone(), + join_authorized_via_users_server: None, + ..event + }, + ); services .rooms @@ -503,17 +494,12 @@ pub(crate) async fn unban_user_route( let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let mut event: RoomMemberEventContent = serde_json::from_str( - services - .rooms - .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) - .await - .map_err(|_| err!(Request(BadState("Cannot unban a user who is not banned."))))? - .content - .get(), - ) - .map_err(|e| err!(Database("Invalid member event in database: {e:?}")))?; + let mut event: RoomMemberEventContent = services + .rooms + .state_accessor + .room_state_get_content(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) + .await + .map_err(|_| err!(Request(BadState("Cannot unban a user who is not banned."))))?; event.membership = MembershipState::Leave; event.reason.clone_from(&body.reason); @@ -1650,14 +1636,13 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, } else { let state_lock = services.rooms.state.mutex.lock(room_id).await; - let member_event = services + let Ok(mut event) = services .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str()) - .await; - - // Fix for broken rooms - let Ok(member_event) = member_event else { + .room_state_get_content::(room_id, &StateEventType::RoomMember, user_id.as_str()) + .await + else { + // Fix for broken rooms error!("Trying to leave a room you are not a member of."); services @@ -1677,9 +1662,6 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, return Ok(()); }; - let mut event: RoomMemberEventContent = serde_json::from_str(member_event.content.get()) - .map_err(|e| err!(Database(error!("Invalid room member event in database: {e}"))))?; - event.membership = MembershipState::Leave; event.reason = reason; diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 495bc8ec..cdc047f0 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -301,10 +301,10 @@ pub async fn update_displayname( // Send a new join membership event into all joined rooms let mut joined_rooms = Vec::new(); for room_id in all_joined_rooms { - let Ok(event) = services + let Ok(content) = services .rooms .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str()) + .room_state_get_content(room_id, &StateEventType::RoomMember, user_id.as_str()) .await else { continue; @@ -315,7 +315,7 @@ pub async fn update_displayname( content: to_raw_value(&RoomMemberEventContent { displayname: displayname.clone(), join_authorized_via_users_server: None, - ..serde_json::from_str(event.content.get()).expect("Database contains invalid PDU.") + ..content }) .expect("event is valid, we just created it"), unsigned: None, @@ -354,35 +354,28 @@ pub async fn update_avatar_url( .iter() .try_stream() .and_then(|room_id: &OwnedRoomId| async move { - Ok(( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - avatar_url: avatar_url.clone(), - blurhash: blurhash.clone(), - join_authorized_via_users_server: None, - ..serde_json::from_str( - services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str()) - .await - .map_err(|_| { - Error::bad_database("Tried to send avatar URL update for user not in the room.") - })? - .content - .get(), - ) - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }, - room_id, - )) + let content = services + .rooms + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomMember, user_id.as_str()) + .await?; + + let pdu = PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + avatar_url: avatar_url.clone(), + blurhash: blurhash.clone(), + join_authorized_via_users_server: None, + ..content + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(user_id.to_string()), + redacts: None, + timestamp: None, + }; + + Ok((pdu, room_id)) }) .ignore_err() .collect() diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 0d8e12a2..e22ad796 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -664,16 +664,12 @@ pub(crate) async fn upgrade_room_route( let state_lock = services.rooms.state.mutex.lock(&replacement_room).await; // Get the old room creation event - let mut create_event_content = serde_json::from_str::( - services - .rooms - .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomCreate, "") - .await - .map_err(|_| err!(Database("Found room without m.room.create event.")))? - .content - .get(), - )?; + let mut create_event_content: CanonicalJsonObject = services + .rooms + .state_accessor + .room_state_get_content(&body.room_id, &StateEventType::RoomCreate, "") + .await + .map_err(|_| err!(Database("Found room without m.room.create event.")))?; // Use the m.room.tombstone event as the predecessor let predecessor = Some(ruma::events::room::create::PreviousRoom::new( @@ -825,16 +821,12 @@ pub(crate) async fn upgrade_room_route( } // Get the old room power levels - let mut power_levels_event_content: RoomPowerLevelsEventContent = serde_json::from_str( - services - .rooms - .state_accessor - .room_state_get(&body.room_id, &StateEventType::RoomPowerLevels, "") - .await - .map_err(|_| err!(Database("Found room without m.room.create event.")))? - .content - .get(), - )?; + let mut power_levels_event_content: RoomPowerLevelsEventContent = services + .rooms + .state_accessor + .room_state_get_content(&body.room_id, &StateEventType::RoomPowerLevels, "") + .await + .map_err(|_| err!(Database("Found room without m.room.power_levels event.")))?; // Setting events_default and invite to the greater of 50 and users_default + 1 let new_level = max( diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index f50cc46c..7fac6be6 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -190,32 +190,31 @@ impl Service { // Always allow the server service account to remove the alias, since there may not be an admin room || server_user == user_id { - Ok(true) - // Checking whether the user is able to change canonical aliases of the - // room - } else if let Ok(event) = self + return Ok(true); + } + + // Checking whether the user is able to change canonical aliases of the room + if let Ok(content) = self .services .state_accessor - .room_state_get(&room_id, &StateEventType::RoomPowerLevels, "") + .room_state_get_content::(&room_id, &StateEventType::RoomPowerLevels, "") .await { - serde_json::from_str(event.content.get()) - .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) - .map(|content: RoomPowerLevelsEventContent| { - RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomCanonicalAlias) - }) + return Ok(RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomCanonicalAlias)); + } + // If there is no power levels event, only the room creator can change // canonical aliases - } else if let Ok(event) = self + if let Ok(event) = self .services .state_accessor .room_state_get(&room_id, &StateEventType::RoomCreate, "") .await { - Ok(event.sender == user_id) - } else { - Err(Error::bad_database("Room has no m.room.create event")) + return Ok(event.sender == user_id); } + + Err!(Database("Room has no m.room.create event")) } async fn who_created_alias(&self, alias: &RoomAliasId) -> Result { diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 920424a4..a30c2cfc 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -380,14 +380,9 @@ impl Service { let join_rule = self .services .state_accessor - .room_state_get(room_id, &StateEventType::RoomJoinRules, "") + .room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") .await - .map_or(JoinRule::Invite, |s| { - serde_json::from_str(s.content.get()) - .map(|c: RoomJoinRulesEventContent| c.join_rule) - .map_err(|e| err!(Database(error!("Invalid room join rule event in database: {e}")))) - .unwrap() - }); + .map_or(JoinRule::Invite, |c: RoomJoinRulesEventContent| c.join_rule); let allowed_room_ids = self .services diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index ece8679d..3b2c2931 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -338,14 +338,13 @@ impl Service { .map(|c: RoomNameEventContent| c.name) } - pub async fn get_avatar(&self, room_id: &RoomId) -> ruma::JsOption { - self.room_state_get(room_id, &StateEventType::RoomAvatar, "") + pub async fn get_avatar(&self, room_id: &RoomId) -> JsOption { + let content = self + .room_state_get_content(room_id, &StateEventType::RoomAvatar, "") .await - .map_or(ruma::JsOption::Undefined, |s| { - serde_json::from_str(s.content.get()) - .map_err(|_| Error::bad_database("Invalid room avatar event in database.")) - .unwrap() - }) + .ok(); + + JsOption::from_option(content) } pub async fn get_member(&self, room_id: &RoomId, user_id: &UserId) -> Result { @@ -416,16 +415,10 @@ impl Service { &self, redacts: &EventId, sender: &UserId, room_id: &RoomId, federation: bool, ) -> Result { if let Ok(event) = self - .room_state_get(room_id, &StateEventType::RoomPowerLevels, "") + .room_state_get_content::(room_id, &StateEventType::RoomPowerLevels, "") .await { - let Ok(event) = serde_json::from_str(event.content.get()) - .map(|content: RoomPowerLevelsEventContent| content.into()) - .map(|event: RoomPowerLevels| event) - else { - return Ok(false); - }; - + let event: RoomPowerLevels = event.into(); Ok(event.user_can_redact_event_of_other(sender) || event.user_can_redact_own_event(sender) && if let Ok(pdu) = self.services.timeline.get_pdu(redacts).await { diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 7cf06522..cc5940e6 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1061,13 +1061,8 @@ impl Service { let power_levels: RoomPowerLevelsEventContent = self .services .state_accessor - .room_state_get(room_id, &StateEventType::RoomPowerLevels, "") + .room_state_get_content(room_id, &StateEventType::RoomPowerLevels, "") .await - .map(|ev| { - serde_json::from_str(ev.content.get()) - .map_err(|_| Error::bad_database("invalid m.room.power_levels event")) - .unwrap() - }) .unwrap_or_default(); let room_mods = power_levels.users.iter().filter_map(|(user_id, level)| { From 55c85f685177eb22f126fdd7382e99959e32e3d8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 09:49:28 +0000 Subject: [PATCH 0893/2091] refactor to pdu.get_content() for serde_json::from_ elim Signed-off-by: Jason Volk --- src/api/client/sync.rs | 46 +++++++------------------ src/service/rooms/event_handler/mod.rs | 10 +++--- src/service/rooms/pdu_metadata/mod.rs | 2 +- src/service/rooms/spaces/mod.rs | 10 +++--- src/service/rooms/state/mod.rs | 17 ++------- src/service/rooms/state_accessor/mod.rs | 4 +-- src/service/rooms/timeline/mod.rs | 36 +++++-------------- 7 files changed, 35 insertions(+), 90 deletions(-) diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 65d62a78..8c4c6a44 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -635,17 +635,8 @@ async fn load_joined_room( .await? .ready_filter(|(_, pdu)| pdu.kind == RoomMember) .filter_map(|(_, pdu)| async move { - let Ok(content) = serde_json::from_str::(pdu.content.get()) else { - return None; - }; - - let Some(state_key) = &pdu.state_key else { - return None; - }; - - let Ok(user_id) = UserId::parse(state_key) else { - return None; - }; + let content: RoomMemberEventContent = pdu.get_content().ok()?; + let user_id: &UserId = pdu.state_key.as_deref().map(TryInto::try_into).flat_ok()?; if user_id == sender_user { return None; @@ -656,22 +647,17 @@ async fn load_joined_room( return None; } - if !services - .rooms - .state_cache - .is_joined(&user_id, room_id) - .await && services - .rooms - .state_cache - .is_invited(&user_id, room_id) - .await - { + let is_invited = services.rooms.state_cache.is_invited(user_id, room_id); + + let is_joined = services.rooms.state_cache.is_joined(user_id, room_id); + + if !is_joined.await && is_invited.await { return None; } - Some(user_id) + Some(user_id.to_owned()) }) - .collect::>() + .collect::>() .await; Ok::<_, Error>(( @@ -839,11 +825,9 @@ async fn load_joined_room( continue; } - let new_membership = serde_json::from_str::(state_event.content.get()) - .map_err(|_| Error::bad_database("Invalid PDU in database."))? - .membership; + let content: RoomMemberEventContent = state_event.get_content()?; - match new_membership { + match content.membership { MembershipState::Join => { // A new user joined an encrypted room if !share_encrypted_room(services, sender_user, &user_id, Some(room_id)).await { @@ -1357,12 +1341,8 @@ pub(crate) async fn sync_events_v4_route( continue; } - let new_membership = - serde_json::from_str::(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid PDU in database."))? - .membership; - - match new_membership { + let content: RoomMemberEventContent = pdu.get_content()?; + match content.membership { MembershipState::Join => { // A new user joined an encrypted room if !share_encrypted_room(&services, sender_user, &user_id, Some(room_id)) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 4708a86c..05f9a27a 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -614,9 +614,7 @@ impl Service { } }, _ => { - let content = serde_json::from_str::(incoming_pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid content in redaction pdu."))?; - + let content: RoomRedactionEventContent = incoming_pdu.get_content()?; if let Some(redact_id) = &content.redacts { !self .services @@ -1432,10 +1430,10 @@ impl Service { } fn get_room_version_id(create_event: &PduEvent) -> Result { - let create_event_content: RoomCreateEventContent = serde_json::from_str(create_event.content.get()) - .map_err(|e| err!(Database("Invalid create event: {e}")))?; + let content: RoomCreateEventContent = create_event.get_content()?; + let room_version = content.room_version; - Ok(create_event_content.room_version) + Ok(room_version) } #[inline] diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index dbaebfbf..fb85d031 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -102,7 +102,7 @@ impl Service { return false; } - let Ok(content) = serde_json::from_str::(pdu.content.get()) else { + let Ok(content) = pdu.get_content::() else { return false; }; diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index a30c2cfc..5aea5f6a 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -596,12 +596,10 @@ impl Service { .await .map_err(|e| err!(Database("Event {id:?} in space state not found: {e:?}")))?; - if serde_json::from_str::(pdu.content.get()) - .ok() - .map(|c| c.via) - .map_or(true, |v| v.is_empty()) - { - continue; + if let Ok(content) = pdu.get_content::() { + if content.via.is_empty() { + continue; + } } if OwnedRoomId::try_from(state_key).is_ok() { diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 177b7e9b..81760b36 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -93,28 +93,17 @@ impl Service { pin_mut!(event_ids); while let Some(event_id) = event_ids.next().await { - let Ok(pdu) = self.services.timeline.get_pdu_json(&event_id).await else { + let Ok(pdu) = self.services.timeline.get_pdu(&event_id).await else { continue; }; - let pdu: PduEvent = match serde_json::from_str( - &serde_json::to_string(&pdu).expect("CanonicalJsonObj can be serialized to JSON"), - ) { - Ok(pdu) => pdu, - Err(_) => continue, - }; - match pdu.kind { TimelineEventType::RoomMember => { - let Ok(membership_event) = serde_json::from_str::(pdu.content.get()) else { + let Some(user_id) = pdu.state_key.as_ref().map(UserId::parse).flat_ok() else { continue; }; - let Some(state_key) = pdu.state_key else { - continue; - }; - - let Ok(user_id) = UserId::parse(state_key) else { + let Ok(membership_event) = pdu.get_content::() else { continue; }; diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 3b2c2931..3855d92a 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -325,11 +325,9 @@ impl Service { where T: for<'de> Deserialize<'de> + Send, { - use serde_json::from_str; - self.room_state_get(room_id, event_type, state_key) .await - .and_then(|event| from_str::(event.content.get()).map_err(Into::into)) + .and_then(|event| event.get_content()) } pub async fn get_name(&self, room_id: &RoomId) -> Result { diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index cc5940e6..487262e6 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -471,12 +471,7 @@ impl Service { } }, _ => { - let content = - serde_json::from_str::(pdu.content.get()).map_err(|e| { - warn!("Invalid content in redaction pdu: {e}"); - Error::bad_database("Invalid content in redaction pdu") - })?; - + let content: RoomRedactionEventContent = pdu.get_content()?; if let Some(redact_id) = &content.redacts { if self .services @@ -506,11 +501,7 @@ impl Service { let target_user_id = UserId::parse(state_key.clone()).expect("This state_key was previously validated"); - let content = serde_json::from_str::(pdu.content.get()).map_err(|e| { - error!("Invalid room member event content in pdu: {e}"); - Error::bad_database("Invalid room member event content in pdu.") - })?; - + let content: RoomMemberEventContent = pdu.get_content()?; let invite_state = match content.membership { MembershipState::Invite => self.services.state.summary_stripped(pdu).await.into(), _ => None, @@ -533,9 +524,7 @@ impl Service { } }, TimelineEventType::RoomMessage => { - let content = serde_json::from_str::(pdu.content.get()) - .map_err(|_| Error::bad_database("Invalid content in pdu."))?; - + let content: ExtractBody = pdu.get_content()?; if let Some(body) = content.body { self.services.search.index_pdu(shortroomid, &pdu_id, &body); @@ -549,7 +538,7 @@ impl Service { _ => {}, } - if let Ok(content) = serde_json::from_str::(pdu.content.get()) { + if let Ok(content) = pdu.get_content::() { if let Ok(related_pducount) = self.get_pdu_count(&content.relates_to.event_id).await { self.services .pdu_metadata @@ -557,7 +546,7 @@ impl Service { } } - if let Ok(content) = serde_json::from_str::(pdu.content.get()) { + if let Ok(content) = pdu.get_content::() { match content.relates_to { Relation::Reply { in_reply_to, @@ -712,10 +701,7 @@ impl Service { .room_state_get(room_id, &event_type.to_string().into(), state_key) .await { - unsigned.insert( - "prev_content".to_owned(), - serde_json::from_str(prev_pdu.content.get()).expect("string is valid json"), - ); + unsigned.insert("prev_content".to_owned(), prev_pdu.get_content_as_value()); unsigned.insert( "prev_sender".to_owned(), serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"), @@ -874,9 +860,7 @@ impl Service { }; }, _ => { - let content = serde_json::from_str::(pdu.content.get()) - .map_err(|e| err!(Database("Invalid content in redaction pdu: {e:?}")))?; - + let content: RoomRedactionEventContent = pdu.get_content()?; if let Some(redact_id) = &content.redacts { if !self .services @@ -1026,7 +1010,7 @@ impl Service { .await .map_err(|e| err!(Database(error!(?pdu_id, ?event_id, ?e, "PDU ID points to invalid PDU."))))?; - if let Ok(content) = serde_json::from_str::(pdu.content.get()) { + if let Ok(content) = pdu.get_content::() { if let Some(body) = content.body { self.services .search @@ -1200,9 +1184,7 @@ impl Service { drop(insert_lock); if pdu.kind == TimelineEventType::RoomMessage { - let content = serde_json::from_str::(pdu.content.get()) - .map_err(|e| err!(Database("Invalid content in pdu: {e:?}")))?; - + let content: ExtractBody = pdu.get_content()?; if let Some(body) = content.body { self.services.search.index_pdu(shortroomid, &pdu_id, &body); } From d526db681f045f28519a3757f761090599d2a14e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 09:57:43 +0000 Subject: [PATCH 0894/2091] refactor various patterns for serde_json::from_ elim bump ruma Signed-off-by: Jason Volk --- Cargo.lock | 26 ++++++++++----------- Cargo.toml | 2 +- src/api/client/membership.rs | 24 +++++++------------- src/api/client/state.rs | 28 +++++++---------------- src/api/server/get_missing_events.rs | 24 +++++++++++--------- src/api/server/invite.rs | 31 ++++++++++---------------- src/service/rooms/event_handler/mod.rs | 16 +++++-------- src/service/rooms/state/mod.rs | 1 + src/service/sending/sender.rs | 8 ++----- 9 files changed, 64 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9f366e7..cae6994c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2976,7 +2976,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "assign", "js_int", @@ -2998,7 +2998,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "js_int", "ruma-common", @@ -3010,7 +3010,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "as_variant", "assign", @@ -3033,7 +3033,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "as_variant", "base64 0.22.1", @@ -3063,7 +3063,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3087,7 +3087,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "bytes", "http", @@ -3105,7 +3105,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "js_int", "thiserror", @@ -3114,7 +3114,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "js_int", "ruma-common", @@ -3124,7 +3124,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "cfg-if", "once_cell", @@ -3140,7 +3140,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "js_int", "ruma-common", @@ -3152,7 +3152,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "headers", "http", @@ -3165,7 +3165,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3181,7 +3181,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37#e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" dependencies = [ "futures-util", "itertools 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index 18f33375..25d1001d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "e81ed2741b4ebe98fe41cabdfee2ac28a52a8e37" +rev = "f485a0265c67a59df75fc6686787538172fa4cac" features = [ "compat", "rand", diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index a260b8c5..fde6099a 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -183,10 +183,8 @@ pub(crate) async fn join_room_by_id_route( .await .unwrap_or_default() .iter() - .filter_map(|event| serde_json::from_str(event.json().get()).ok()) - .filter_map(|event: serde_json::Value| event.get("sender").cloned()) - .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) - .filter_map(|sender| UserId::parse(sender).ok()) + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) .map(|user| user.server_name().to_owned()), ); @@ -248,10 +246,8 @@ pub(crate) async fn join_room_by_id_or_alias_route( .await .unwrap_or_default() .iter() - .filter_map(|event| serde_json::from_str(event.json().get()).ok()) - .filter_map(|event: serde_json::Value| event.get("sender").cloned()) - .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) - .filter_map(|sender| UserId::parse(sender).ok()) + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) .map(|user| user.server_name().to_owned()), ); @@ -294,10 +290,8 @@ pub(crate) async fn join_room_by_id_or_alias_route( .await .unwrap_or_default() .iter() - .filter_map(|event| serde_json::from_str(event.json().get()).ok()) - .filter_map(|event: serde_json::Value| event.get("sender").cloned()) - .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) - .filter_map(|sender| UserId::parse(sender).ok()) + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) .map(|user| user.server_name().to_owned()), ); @@ -1708,10 +1702,8 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room servers.extend( invite_state .iter() - .filter_map(|event| serde_json::from_str(event.json().get()).ok()) - .filter_map(|event: serde_json::Value| event.get("sender").cloned()) - .filter_map(|sender| sender.as_str().map(ToOwned::to_owned)) - .filter_map(|sender| UserId::parse(sender).ok()) + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) .map(|user| user.server_name().to_owned()), ); diff --git a/src/api/client/state.rs b/src/api/client/state.rs index d89c23e8..2a13ba1f 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use axum::extract::State; -use conduit::{err, error, pdu::PduBuilder, Err, Error, Result}; +use conduit::{err, pdu::PduBuilder, utils::BoolExt, Err, Error, Result}; use ruma::{ api::client::{ error::ErrorKind, @@ -137,27 +137,15 @@ pub(crate) async fn get_state_events_for_key_route( )))) })?; - if body + let event_format = body .format .as_ref() - .is_some_and(|f| f.to_lowercase().eq("event")) - { - Ok(get_state_events_for_key::v3::Response { - content: None, - event: serde_json::from_str(event.to_state_event().json().get()).map_err(|e| { - error!("Invalid room state event in database: {}", e); - Error::bad_database("Invalid room state event in database") - })?, - }) - } else { - Ok(get_state_events_for_key::v3::Response { - content: Some(serde_json::from_str(event.content.get()).map_err(|e| { - error!("Invalid room state event content in database: {}", e); - Error::bad_database("Invalid room state event content in database") - })?), - event: None, - }) - } + .is_some_and(|f| f.to_lowercase().eq("event")); + + Ok(get_state_events_for_key::v3::Response { + content: event_format.or(|| event.get_content_as_value()), + event: event_format.then(|| event.to_state_event_value()), + }) } /// # `GET /_matrix/client/v3/rooms/{roomid}/state/{eventType}` diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index 7ae0ff60..e267898f 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -2,7 +2,7 @@ use axum::extract::State; use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_missing_events}, - OwnedEventId, RoomId, + CanonicalJsonValue, EventId, RoomId, }; use crate::Ruma; @@ -78,17 +78,19 @@ pub(crate) async fn get_missing_events_route( continue; } - queued_events.extend_from_slice( - &serde_json::from_value::>( - serde_json::to_value( - pdu.get("prev_events") - .cloned() - .ok_or_else(|| Error::bad_database("Event in db has no prev_events property."))?, - ) - .expect("canonical json is valid json value"), - ) - .map_err(|_| Error::bad_database("Invalid prev_events in event in database."))?, + let prev_events = pdu + .get("prev_events") + .and_then(CanonicalJsonValue::as_array) + .unwrap_or_default(); + + queued_events.extend( + prev_events + .iter() + .map(<&EventId>::try_from) + .filter_map(Result::ok) + .map(ToOwned::to_owned), ); + events.push( services .sending diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 9968bdf7..dd2374b6 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -1,11 +1,11 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{utils, warn, Error, PduEvent, Result}; +use conduit::{err, utils, warn, Err, Error, PduEvent, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_invite}, events::room::member::{MembershipState, RoomMemberEventContent}, serde::JsonObject, - CanonicalJsonValue, EventId, OwnedUserId, + CanonicalJsonValue, EventId, OwnedUserId, UserId, }; use crate::Ruma; @@ -79,14 +79,11 @@ pub(crate) async fn create_invite_route( let mut signed_event = utils::to_canonical_object(&body.event) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invite event is invalid."))?; - let invited_user: OwnedUserId = serde_json::from_value( - signed_event - .get("state_key") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event has no state_key property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "state_key is not a user ID."))?; + let invited_user: OwnedUserId = signed_event + .get("state_key") + .try_into() + .map(UserId::to_owned) + .map_err(|e| err!(Request(InvalidParam("Invalid state_key property: {e}"))))?; if !services.globals.server_is_ours(invited_user.server_name()) { return Err(Error::BadRequest( @@ -121,14 +118,10 @@ pub(crate) async fn create_invite_route( // Add event_id back signed_event.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.to_string())); - let sender: OwnedUserId = serde_json::from_value( - signed_event - .get("sender") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event had no sender property."))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "sender is not a user ID."))?; + let sender: &UserId = signed_event + .get("sender") + .try_into() + .map_err(|e| err!(Request(InvalidParam("Invalid sender property: {e}"))))?; if services.rooms.metadata.is_banned(&body.room_id).await && !services.users.is_admin(&invited_user).await { return Err(Error::BadRequest( @@ -171,7 +164,7 @@ pub(crate) async fn create_invite_route( &body.room_id, &invited_user, RoomMemberEventContent::new(MembershipState::Invite), - &sender, + sender, Some(invite_state), body.via.clone(), true, diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 05f9a27a..f8042b67 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -30,8 +30,8 @@ use ruma::{ int, serde::Base64, state_res::{self, EventTypeExt, RoomVersion, StateMap}, - uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, - RoomVersionId, ServerName, + uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, + ServerName, UserId, }; use tokio::sync::RwLock; @@ -157,14 +157,10 @@ impl Service { self.acl_check(origin, room_id).await?; // 1.3.2 Check room ACL on sender's server name - let sender: OwnedUserId = serde_json::from_value( - value - .get("sender") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "PDU does not have a sender key"))? - .clone() - .into(), - ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "User ID in sender is invalid"))?; + let sender: &UserId = value + .get("sender") + .try_into() + .map_err(|e| err!(Request(InvalidParam("PDU does not have a valid sender key: {e}"))))?; self.acl_check(sender.server_name(), room_id).await?; diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 81760b36..cfcb2da6 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -9,6 +9,7 @@ use std::{ use conduit::{ err, + result::FlatOk, utils::{calculate_hash, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard}, warn, PduEvent, Result, }; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 90977abe..5c0a324b 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -528,12 +528,8 @@ impl Service { for pdu in pdus { // Redacted events are not notification targets (we don't send push for them) - if let Some(unsigned) = &pdu.unsigned { - if let Ok(unsigned) = serde_json::from_str::(unsigned.get()) { - if unsigned.get("redacted_because").is_some() { - continue; - } - } + if pdu.contains_unsigned_property("redacted_because", serde_json::Value::is_string) { + continue; } let rules_for_user = self From 57e0a5f65dce2be514d0bc45dbfb26b5c5b0cd00 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 10:02:24 +0000 Subject: [PATCH 0895/2091] additional database stream deserializations for serde_json::from_ elim Signed-off-by: Jason Volk --- src/service/key_backups/mod.rs | 64 ++++++++------------------- src/service/pusher/mod.rs | 2 +- src/service/rooms/state_cache/data.rs | 62 +++++++++++++------------- src/service/users/mod.rs | 11 ++--- 4 files changed, 57 insertions(+), 82 deletions(-) diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index decf32f7..55263eeb 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -1,9 +1,9 @@ use std::{collections::BTreeMap, sync::Arc}; use conduit::{ - err, implement, utils, + err, implement, utils::stream::{ReadyExt, TryIgnore}, - Err, Error, Result, + Err, Result, }; use database::{Deserialized, Ignore, Interfix, Map}; use futures::StreamExt; @@ -110,57 +110,35 @@ pub async fn update_backup( #[implement(Service)] pub async fn get_latest_backup_version(&self, user_id: &UserId) -> Result { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - let mut last_possible_key = prefix.clone(); - last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes()); + type Key<'a> = (&'a UserId, &'a str); + let last_possible_key = (user_id, u64::MAX); self.db .backupid_algorithm - .rev_raw_keys_from(&last_possible_key) + .rev_keys_from(&last_possible_key) .ignore_err() - .ready_take_while(move |key| key.starts_with(&prefix)) + .ready_take_while(|(user_id_, _): &Key<'_>| *user_id_ == user_id) + .map(|(_, version): Key<'_>| version.to_owned()) .next() .await .ok_or_else(|| err!(Request(NotFound("No backup versions found")))) - .and_then(|key| { - utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("backupid_algorithm key is invalid.")) - }) } #[implement(Service)] pub async fn get_latest_backup(&self, user_id: &UserId) -> Result<(String, Raw)> { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - let mut last_possible_key = prefix.clone(); - last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes()); + type Key<'a> = (&'a UserId, &'a str); + type KeyVal<'a> = (Key<'a>, Raw); + let last_possible_key = (user_id, u64::MAX); self.db .backupid_algorithm - .rev_raw_stream_from(&last_possible_key) + .rev_stream_from(&last_possible_key) .ignore_err() - .ready_take_while(move |(key, _)| key.starts_with(&prefix)) + .ready_take_while(|((user_id_, _), _): &KeyVal<'_>| *user_id_ == user_id) + .map(|((_, version), algorithm): KeyVal<'_>| (version.to_owned(), algorithm)) .next() .await .ok_or_else(|| err!(Request(NotFound("No backup found")))) - .and_then(|(key, val)| { - let version = utils::string_from_bytes( - key.rsplit(|&b| b == 0xFF) - .next() - .expect("rsplit always returns an element"), - ) - .map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))?; - - let algorithm = serde_json::from_slice(val) - .map_err(|_| Error::bad_database("Algorithm in backupid_algorithm is invalid."))?; - - Ok((version, algorithm)) - }) } #[implement(Service)] @@ -223,7 +201,8 @@ pub async fn get_etag(&self, user_id: &UserId, version: &str) -> String { #[implement(Service)] pub async fn get_all(&self, user_id: &UserId, version: &str) -> BTreeMap { - type KeyVal<'a> = ((Ignore, Ignore, &'a RoomId, &'a str), &'a [u8]); + type Key<'a> = (Ignore, Ignore, &'a RoomId, &'a str); + type KeyVal<'a> = (Key<'a>, Raw); let mut rooms = BTreeMap::::new(); let default = || RoomKeyBackup { @@ -235,13 +214,12 @@ pub async fn get_all(&self, user_id: &UserId, version: &str) -> BTreeMap| { - let key_data = serde_json::from_slice(value).expect("Invalid KeyBackupData JSON"); + .ready_for_each(|((_, _, room_id, session_id), key_backup_data): KeyVal<'_>| { rooms .entry(room_id.into()) .or_insert_with(default) .sessions - .insert(session_id.into(), key_data); + .insert(session_id.into(), key_backup_data); }) .await; @@ -252,18 +230,14 @@ pub async fn get_all(&self, user_id: &UserId, version: &str) -> BTreeMap BTreeMap> { - type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), &'a [u8]); + type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), Raw); let prefix = (user_id, version, room_id, Interfix); self.db .backupkeyid_backup .stream_prefix(&prefix) .ignore_err() - .map(|((.., session_id), value): KeyVal<'_>| { - let session_id = session_id.to_owned(); - let key_backup_data = serde_json::from_slice(value).expect("Invalid KeyBackupData JSON"); - (session_id, key_backup_data) - }) + .map(|((.., session_id), key_backup_data): KeyVal<'_>| (session_id.to_owned(), key_backup_data)) .collect() .await } diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 8d8b553f..e7b1824a 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -99,7 +99,7 @@ impl Service { .senderkey_pusher .stream_prefix(&prefix) .ignore_err() - .map(|(_, val): (Ignore, &[u8])| serde_json::from_slice(val).expect("Invalid Pusher in db.")) + .map(|(_, pusher): (Ignore, Pusher)| pusher) .collect() .await } diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index f3ccaf10..6e01e49d 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, RwLock}, }; -use conduit::{utils, utils::stream::TryIgnore, Error, Result}; +use conduit::{utils::stream::TryIgnore, Result}; use database::{Deserialized, Interfix, Map}; use futures::{Stream, StreamExt}; use ruma::{ @@ -135,20 +135,31 @@ impl Data { pub(super) fn rooms_invited<'a>( &'a self, user_id: &'a UserId, ) -> impl Stream + Send + 'a { + type Key<'a> = (&'a UserId, &'a RoomId); + type KeyVal<'a> = (Key<'a>, Raw>); + let prefix = (user_id, Interfix); self.userroomid_invitestate - .stream_raw_prefix(&prefix) + .stream_prefix(&prefix) .ignore_err() - .map(|(key, val)| { - let room_id = key.rsplit(|&b| b == 0xFF).next().unwrap(); - let room_id = utils::string_from_bytes(room_id).unwrap(); - let room_id = RoomId::parse(room_id).unwrap(); - let state = serde_json::from_slice(val) - .map_err(|_| Error::bad_database("Invalid state in userroomid_invitestate.")) - .unwrap(); + .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) + .ignore_err() + } - (room_id, state) - }) + /// Returns an iterator over all rooms a user left. + #[inline] + pub(super) fn rooms_left<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + type Key<'a> = (&'a UserId, &'a RoomId); + type KeyVal<'a> = (Key<'a>, Raw>>); + + let prefix = (user_id, Interfix); + self.userroomid_leftstate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) + .ignore_err() } #[tracing::instrument(skip(self), level = "debug")] @@ -156,7 +167,11 @@ impl Data { &self, user_id: &UserId, room_id: &RoomId, ) -> Result>> { let key = (user_id, room_id); - self.userroomid_invitestate.qry(&key).await.deserialized() + self.userroomid_invitestate + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) } #[tracing::instrument(skip(self), level = "debug")] @@ -164,25 +179,10 @@ impl Data { &self, user_id: &UserId, room_id: &RoomId, ) -> Result>> { let key = (user_id, room_id); - self.userroomid_leftstate.qry(&key).await.deserialized() - } - - /// Returns an iterator over all rooms a user left. - #[inline] - pub(super) fn rooms_left<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { - let prefix = (user_id, Interfix); self.userroomid_leftstate - .stream_raw_prefix(&prefix) - .ignore_err() - .map(|(key, val)| { - let room_id = key.rsplit(|&b| b == 0xFF).next().unwrap(); - let room_id = utils::string_from_bytes(room_id).unwrap(); - let room_id = RoomId::parse(room_id).unwrap(); - let state = serde_json::from_slice(val) - .map_err(|_| Error::bad_database("Invalid state in userroomid_leftstate.")) - .unwrap(); - - (room_id, state) - }) + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) } } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 3ab6b3c3..71a93666 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeMap, mem, mem::size_of, sync::Arc}; use conduit::{ debug_warn, err, utils, - utils::{stream::TryIgnore, string::Unquoted, ReadyExt, TryReadyExt}, + utils::{stream::TryIgnore, string::Unquoted, ReadyExt}, warn, Err, Error, Result, Server, }; use database::{Deserialized, Ignore, Interfix, Map}; @@ -749,9 +749,9 @@ impl Service { let prefix = (user_id, device_id, Interfix); self.db .todeviceid_events - .stream_raw_prefix(&prefix) - .ready_and_then(|(_, val)| serde_json::from_slice(val).map_err(Into::into)) + .stream_prefix(&prefix) .ignore_err() + .map(|(_, val): (Ignore, Raw)| val) } pub async fn remove_to_device_events(&self, user_id: &UserId, device_id: &DeviceId, until: u64) { @@ -812,11 +812,12 @@ impl Service { } pub fn all_devices_metadata<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + let key = (user_id, Interfix); self.db .userdeviceid_metadata - .stream_raw_prefix(&(user_id, Interfix)) - .ready_and_then(|(_, val)| serde_json::from_slice::(val).map_err(Into::into)) + .stream_prefix(&key) .ignore_err() + .map(|(_, val): (Ignore, Device)| val) } /// Creates a new sync filter. Returns the filter id. From f503ed918c90720c28f978c2851d252e21920a29 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Oct 2024 10:03:31 +0000 Subject: [PATCH 0896/2091] misc cleanup Signed-off-by: Jason Volk --- src/api/client/membership.rs | 33 ++++++++----------- src/api/client/state.rs | 2 +- src/api/server/invite.rs | 28 ++++------------ src/api/server/send.rs | 22 ++++++------- src/service/admin/mod.rs | 2 +- .../rooms/event_handler/parse_incoming_pdu.rs | 30 ++++++++--------- src/service/rooms/timeline/mod.rs | 3 +- 7 files changed, 49 insertions(+), 71 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index fde6099a..f21f3d7d 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1333,10 +1333,8 @@ pub async fn validate_and_add_event_id( services: &Services, pdu: &RawJsonValue, room_version: &RoomVersionId, pub_key_map: &RwLock>>, ) -> Result<(OwnedEventId, CanonicalJsonObject)> { - let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { - debug_error!("Invalid PDU in server response: {pdu:#?}"); - err!(BadServerResponse("Invalid PDU in server response: {e:?}")) - })?; + let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()) + .map_err(|e| err!(BadServerResponse(debug_error!("Invalid PDU in server response: {e:?}"))))?; let event_id = EventId::parse(format!( "${}", ruma::signatures::reference_hash(&value, room_version).expect("ruma can calculate reference hashes") @@ -1478,10 +1476,8 @@ pub(crate) async fn invite_helper( if *pdu.event_id != *event_id { warn!( - "Server {} changed invite event, that's not allowed in the spec: ours: {:?}, theirs: {:?}", + "Server {} changed invite event, that's not allowed in the spec: ours: {pdu_json:?}, theirs: {value:?}", user_id.server_name(), - pdu_json, - value ); } @@ -1564,20 +1560,19 @@ pub(crate) async fn invite_helper( // Make a user leave all their joined rooms, forgets all rooms, and ignores // errors pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { - let all_rooms: Vec<_> = services + let rooms_joined = services .rooms .state_cache .rooms_joined(user_id) - .map(ToOwned::to_owned) - .chain( - services - .rooms - .state_cache - .rooms_invited(user_id) - .map(|(r, _)| r), - ) - .collect() - .await; + .map(ToOwned::to_owned); + + let rooms_invited = services + .rooms + .state_cache + .rooms_invited(user_id) + .map(|(r, _)| r); + + let all_rooms: Vec<_> = rooms_joined.chain(rooms_invited).collect().await; for room_id in all_rooms { // ignore errors @@ -1601,7 +1596,7 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, .await { if let Err(e) = remote_leave_room(services, user_id, room_id).await { - warn!("Failed to leave room {} remotely: {}", user_id, e); + warn!("Failed to leave room {user_id} remotely: {e}"); // Don't tell the client about this error } diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 2a13ba1f..1396ae77 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -176,7 +176,7 @@ async fn send_state_event_for_key_helper( .build_and_append_pdu( PduBuilder { event_type: event_type.to_string().into(), - content: serde_json::from_str(json.json().get()).expect("content is valid json"), + content: serde_json::from_str(json.json().get())?, unsigned: None, state_key: Some(state_key), redacts: None, diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index dd2374b6..447e54be 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -47,10 +47,7 @@ pub(crate) async fn create_invite_route( .forbidden_remote_server_names .contains(&server.to_owned()) { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + return Err!(Request(Forbidden("Server is banned on this homeserver."))); } } @@ -64,15 +61,13 @@ pub(crate) async fn create_invite_route( "Received federated/remote invite from banned server {origin} for room ID {}. Rejecting.", body.room_id ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + + return Err!(Request(Forbidden("Server is banned on this homeserver."))); } if let Some(via) = &body.via { if via.is_empty() { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "via field must not be empty.")); + return Err!(Request(InvalidParam("via field must not be empty."))); } } @@ -86,10 +81,7 @@ pub(crate) async fn create_invite_route( .map_err(|e| err!(Request(InvalidParam("Invalid state_key property: {e}"))))?; if !services.globals.server_is_ours(invited_user.server_name()) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "User does not belong to this homeserver.", - )); + return Err!(Request(InvalidParam("User does not belong to this homeserver."))); } // Make sure we're not ACL'ed from their room. @@ -124,17 +116,11 @@ pub(crate) async fn create_invite_route( .map_err(|e| err!(Request(InvalidParam("Invalid sender property: {e}"))))?; if services.rooms.metadata.is_banned(&body.room_id).await && !services.users.is_admin(&invited_user).await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "This room is banned on this homeserver.", - )); + return Err!(Request(Forbidden("This room is banned on this homeserver."))); } if services.globals.block_non_admin_invites() && !services.users.is_admin(&invited_user).await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "This server does not allow room invites.", - )); + return Err!(Request(Forbidden("This server does not allow room invites."))); } let mut invite_state = body.invite_room_state.clone(); diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 50a79e00..f6916ccf 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -16,9 +16,11 @@ use ruma::{ }, }, events::receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType}, + serde::Raw, to_device::DeviceIdOrAllDevices, OwnedEventId, ServerName, }; +use serde_json::value::RawValue as RawJsonValue; use tokio::sync::RwLock; use crate::{ @@ -70,8 +72,8 @@ pub(crate) async fn send_transaction_message_route( "Starting txn", ); - let resolved_map = handle_pdus(&services, &client, &body, origin, &txn_start_time).await; - handle_edus(&services, &client, &body, origin).await; + let resolved_map = handle_pdus(&services, &client, &body.pdus, origin, &txn_start_time).await; + handle_edus(&services, &client, &body.edus, origin).await; debug!( pdus = ?body.pdus.len(), @@ -91,11 +93,10 @@ pub(crate) async fn send_transaction_message_route( } async fn handle_pdus( - services: &Services, _client: &IpAddr, body: &Ruma, origin: &ServerName, - txn_start_time: &Instant, + services: &Services, _client: &IpAddr, pdus: &[Box], origin: &ServerName, txn_start_time: &Instant, ) -> ResolvedMap { - let mut parsed_pdus = Vec::with_capacity(body.pdus.len()); - for pdu in &body.pdus { + let mut parsed_pdus = Vec::with_capacity(pdus.len()); + for pdu in pdus { parsed_pdus.push(match services.rooms.event_handler.parse_incoming_pdu(pdu).await { Ok(t) => t, Err(e) => { @@ -162,11 +163,8 @@ async fn handle_pdus( resolved_map } -async fn handle_edus( - services: &Services, client: &IpAddr, body: &Ruma, origin: &ServerName, -) { - for edu in body - .edus +async fn handle_edus(services: &Services, client: &IpAddr, edus: &[Raw], origin: &ServerName) { + for edu in edus .iter() .filter_map(|edu| serde_json::from_str::(edu.json().get()).ok()) { @@ -178,7 +176,7 @@ async fn handle_edus( Edu::DirectToDevice(content) => handle_edu_direct_to_device(services, client, origin, content).await, Edu::SigningKeyUpdate(content) => handle_edu_signing_key_update(services, client, origin, content).await, Edu::_Custom(ref _custom) => { - debug_warn!(?body.edus, "received custom/unknown EDU"); + debug_warn!(?edus, "received custom/unknown EDU"); }, } } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 12eacc8f..da7f3cf4 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -198,7 +198,6 @@ impl Service { Ok(None) => debug!("Command successful with no response"), Ok(Some(output)) | Err(output) => self .handle_response(output) - .boxed() .await .unwrap_or_else(default_log), } @@ -277,6 +276,7 @@ impl Service { }; self.respond_to_room(content, &pdu.room_id, response_sender) + .boxed() .await } diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 9081fcbc..39920219 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,29 +1,29 @@ -use conduit::{debug_warn, err, pdu::gen_event_id_canonical_json, Err, Result}; -use ruma::{CanonicalJsonObject, OwnedEventId, OwnedRoomId, RoomId}; +use conduit::{err, pdu::gen_event_id_canonical_json, result::FlatOk, Result}; +use ruma::{CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, RoomId}; use serde_json::value::RawValue as RawJsonValue; impl super::Service { pub async fn parse_incoming_pdu( &self, pdu: &RawJsonValue, ) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { - let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { - debug_warn!("Error parsing incoming event {pdu:#?}"); - err!(BadServerResponse("Error parsing incoming event {e:?}")) - })?; + let value = serde_json::from_str::(pdu.get()) + .map_err(|e| err!(BadServerResponse(debug_warn!("Error parsing incoming event {e:?}"))))?; let room_id: OwnedRoomId = value .get("room_id") - .and_then(|id| RoomId::parse(id.as_str()?).ok()) - .ok_or_else(|| err!(Request(InvalidParam("Invalid room id in pdu"))))?; + .and_then(CanonicalJsonValue::as_str) + .map(RoomId::parse) + .flat_ok_or(err!(Request(InvalidParam("Invalid room_id in pdu"))))?; - let Ok(room_version_id) = self.services.state.get_room_version(&room_id).await else { - return Err!("Server is not in room {room_id}"); - }; + let room_version_id = self + .services + .state + .get_room_version(&room_id) + .await + .map_err(|_| err!("Server is not in room {room_id}"))?; - let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { - // Event could not be converted to canonical json - return Err!(Request(InvalidParam("Could not convert event to canonical json."))); - }; + let (event_id, value) = gen_event_id_canonical_json(pdu, &room_version_id) + .map_err(|e| err!(Request(InvalidParam("Could not convert event to canonical json: {e}"))))?; Ok((event_id, value, room_id)) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 487262e6..21e5395d 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -661,8 +661,7 @@ impl Service { .await .or_else(|_| { if event_type == TimelineEventType::RoomCreate { - let content = serde_json::from_str::(content.get()) - .expect("Invalid content in RoomCreate pdu."); + let content: RoomCreateEventContent = serde_json::from_str(content.get())?; Ok(content.room_version) } else { Err(Error::InconsistentRoomState( From e482c0646f58ae0fe58abc12dff4be7cb1fd8e8f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Oct 2024 20:25:32 +0000 Subject: [PATCH 0897/2091] Add constructions and Default for PduBuilder simplify various RoomMemberEventContent constructions Signed-off-by: Jason Volk --- src/admin/user/commands.rs | 21 +-- src/api/client/account.rs | 12 +- src/api/client/membership.rs | 157 +++++++--------------- src/api/client/message.rs | 17 +-- src/api/client/profile.rs | 33 ++--- src/api/client/redact.rs | 18 +-- src/api/client/room.rs | 159 +++++++--------------- src/api/client/state.rs | 3 +- src/api/server/make_join.rs | 29 ++-- src/api/server/make_leave.rs | 25 +--- src/core/pdu/builder.rs | 59 +++++++- src/core/pdu/mod.rs | 5 +- src/service/admin/create.rs | 171 +++++++----------------- src/service/admin/grant.rs | 83 +++--------- src/service/admin/mod.rs | 31 +---- src/service/rooms/state_accessor/mod.rs | 20 +-- 16 files changed, 279 insertions(+), 564 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 562bb9c7..df393833 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -12,11 +12,10 @@ use ruma::{ redaction::RoomRedactionEventContent, }, tag::{TagEvent, TagEventContent, TagInfo}, - RoomAccountDataEventType, StateEventType, TimelineEventType, + RoomAccountDataEventType, StateEventType, }, EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, }; -use serde_json::value::to_raw_value; use crate::{ admin_command, get_room_info, @@ -461,14 +460,7 @@ pub(super) async fn force_demote( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&power_levels_content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + PduBuilder::state(String::new(), &power_levels_content), &user_id, &room_id, &state_lock, @@ -623,16 +615,11 @@ pub(super) async fn redact_event(&self, event_id: Box) -> Result(room_id, &StateEventType::RoomMember, user_id.as_str()) @@ -1651,21 +1601,18 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, return Ok(()); }; - event.membership = MembershipState::Leave; - event.reason = reason; - services .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&event).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }, + PduBuilder::state( + user_id.to_string(), + &RoomMemberEventContent { + membership: MembershipState::Leave, + reason, + ..event + }, + ), user_id, room_id, &state_lock, diff --git a/src/api/client/message.rs b/src/api/client/message.rs index d577e3c8..578b675b 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -9,7 +9,6 @@ use conduit::{ use futures::{FutureExt, StreamExt}; use ruma::{ api::client::{ - error::ErrorKind, filter::{RoomEventFilter, UrlFilter}, message::{get_message_events, send_message_event}, }, @@ -21,7 +20,7 @@ use service::rooms::timeline::PdusIterItem; use crate::{ service::{pdu::PduBuilder, Services}, - utils, Error, Result, Ruma, + utils, Result, Ruma, }; /// # `PUT /_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}` @@ -77,27 +76,25 @@ pub(crate) async fn send_message_event_route( let mut unsigned = BTreeMap::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.to_string().into()); - let content = from_str(body.body.body.json().get()) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?; + let content = + from_str(body.body.body.json().get()).map_err(|e| err!(Request(BadJson("Invalid JSON body: {e}"))))?; let event_id = services .rooms .timeline .build_and_append_pdu( PduBuilder { - event_type: body.event_type.to_string().into(), + event_type: body.event_type.clone().into(), content, unsigned: Some(unsigned), - state_key: None, - redacts: None, timestamp: appservice_info.and(body.timestamp), + ..Default::default() }, sender_user, &body.room_id, &state_lock, ) - .await - .map(|event_id| (*event_id).to_owned())?; + .await?; services .transaction_ids @@ -106,7 +103,7 @@ pub(crate) async fn send_message_event_route( drop(state_lock); Ok(send_message_event::v3::Response { - event_id, + event_id: event_id.into(), }) } diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index cdc047f0..32f7a723 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -13,11 +13,10 @@ use ruma::{ }, federation, }, - events::{room::member::RoomMemberEventContent, StateEventType, TimelineEventType}, + events::{room::member::RoomMemberEventContent, StateEventType}, presence::PresenceState, OwnedMxcUri, OwnedRoomId, UserId, }; -use serde_json::value::to_raw_value; use service::Services; use crate::Ruma; @@ -310,19 +309,14 @@ pub async fn update_displayname( continue; }; - let pdu = PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { + let pdu = PduBuilder::state( + user_id.to_string(), + &RoomMemberEventContent { displayname: displayname.clone(), join_authorized_via_users_server: None, ..content - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }; + }, + ); joined_rooms.push((pdu, room_id)); } @@ -360,20 +354,15 @@ pub async fn update_avatar_url( .room_state_get_content(room_id, &StateEventType::RoomMember, user_id.as_str()) .await?; - let pdu = PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { + let pdu = PduBuilder::state( + user_id.to_string(), + &RoomMemberEventContent { avatar_url: avatar_url.clone(), blurhash: blurhash.clone(), join_authorized_via_users_server: None, ..content - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }; + }, + ); Ok((pdu, room_id)) }) diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index 2102f6cd..a986dc18 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -1,9 +1,5 @@ use axum::extract::State; -use ruma::{ - api::client::redact::redact_event, - events::{room::redaction::RoomRedactionEventContent, TimelineEventType}, -}; -use serde_json::value::to_raw_value; +use ruma::{api::client::redact::redact_event, events::room::redaction::RoomRedactionEventContent}; use crate::{service::pdu::PduBuilder, Result, Ruma}; @@ -25,16 +21,11 @@ pub(crate) async fn redact_event_route( .timeline .build_and_append_pdu( PduBuilder { - event_type: TimelineEventType::RoomRedaction, - content: to_raw_value(&RoomRedactionEventContent { + redacts: Some(body.event_id.clone().into()), + ..PduBuilder::timeline(&RoomRedactionEventContent { redacts: Some(body.event_id.clone()), reason: body.reason.clone(), }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: Some(body.event_id.into()), - timestamp: None, }, sender_user, &body.room_id, @@ -44,8 +35,7 @@ pub(crate) async fn redact_event_route( drop(state_lock); - let event_id = (*event_id).to_owned(); Ok(redact_event::v3::Response { - event_id, + event_id: event_id.into(), }) } diff --git a/src/api/client/room.rs b/src/api/client/room.rs index e22ad796..daadb724 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -150,8 +150,7 @@ pub(crate) async fn create_room_route( None => services.globals.default_room_version(), }; - #[allow(clippy::single_match_else)] - let content = match &body.creation_content { + let create_content = match &body.creation_content { Some(content) => { use RoomVersionId::*; @@ -213,11 +212,9 @@ pub(crate) async fn create_room_route( .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomCreate, - content: to_raw_value(&content).expect("event is valid, we just created it"), - unsigned: None, + content: to_raw_value(&create_content).expect("create event content serialization"), state_key: Some(String::new()), - redacts: None, - timestamp: None, + ..Default::default() }, sender_user, &room_id, @@ -231,24 +228,16 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Join, + PduBuilder::state( + sender_user.to_string(), + &RoomMemberEventContent { displayname: services.users.displayname(sender_user).await.ok(), avatar_url: services.users.avatar_url(sender_user).await.ok(), - is_direct: Some(body.is_direct), - third_party_invite: None, blurhash: services.users.blurhash(sender_user).await.ok(), - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_user.to_string()), - redacts: None, - timestamp: None, - }, + is_direct: Some(body.is_direct), + ..RoomMemberEventContent::new(MembershipState::Join) + }, + ), sender_user, &room_id, &state_lock, @@ -289,11 +278,9 @@ pub(crate) async fn create_room_route( .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&power_levels_content).expect("to_raw_value always works on serde_json::Value"), - unsigned: None, + content: to_raw_value(&power_levels_content).expect("serialized power_levels event content"), state_key: Some(String::new()), - redacts: None, - timestamp: None, + ..Default::default() }, sender_user, &room_id, @@ -308,18 +295,13 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomCanonicalAlias, - content: to_raw_value(&RoomCanonicalAliasEventContent { + PduBuilder::state( + String::new(), + &RoomCanonicalAliasEventContent { alias: Some(room_alias_id.to_owned()), alt_aliases: vec![], - }) - .expect("We checked that alias earlier, it must be fine"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }, + ), sender_user, &room_id, &state_lock, @@ -335,19 +317,14 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomJoinRules, - content: to_raw_value(&RoomJoinRulesEventContent::new(match preset { + PduBuilder::state( + String::new(), + &RoomJoinRulesEventContent::new(match preset { RoomPreset::PublicChat => JoinRule::Public, // according to spec "invite" is the default _ => JoinRule::Invite, - })) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }), + ), sender_user, &room_id, &state_lock, @@ -360,15 +337,10 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomHistoryVisibility, - content: to_raw_value(&RoomHistoryVisibilityEventContent::new(HistoryVisibility::Shared)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + PduBuilder::state( + String::new(), + &RoomHistoryVisibilityEventContent::new(HistoryVisibility::Shared), + ), sender_user, &room_id, &state_lock, @@ -381,18 +353,13 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomGuestAccess, - content: to_raw_value(&RoomGuestAccessEventContent::new(match preset { + PduBuilder::state( + String::new(), + &RoomGuestAccessEventContent::new(match preset { RoomPreset::PublicChat => GuestAccess::Forbidden, _ => GuestAccess::CanJoin, - })) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }), + ), sender_user, &room_id, &state_lock, @@ -440,15 +407,7 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomName, - content: to_raw_value(&RoomNameEventContent::new(name.clone())) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + PduBuilder::state(String::new(), &RoomNameEventContent::new(name.clone())), sender_user, &room_id, &state_lock, @@ -462,17 +421,12 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomTopic, - content: to_raw_value(&RoomTopicEventContent { + PduBuilder::state( + String::new(), + &RoomTopicEventContent { topic: topic.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }, + ), sender_user, &room_id, &state_lock, @@ -641,18 +595,13 @@ pub(crate) async fn upgrade_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomTombstone, - content: to_raw_value(&RoomTombstoneEventContent { + PduBuilder::state( + String::new(), + &RoomTombstoneEventContent { body: "This room has been replaced".to_owned(), replacement_room: replacement_room.clone(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }, + ), sender_user, &body.room_id, &state_lock, @@ -788,10 +737,8 @@ pub(crate) async fn upgrade_room_route( PduBuilder { event_type: event_type.to_string().into(), content: event_content, - unsigned: None, state_key: Some(String::new()), - redacts: None, - timestamp: None, + ..Default::default() }, sender_user, &replacement_room, @@ -821,7 +768,7 @@ pub(crate) async fn upgrade_room_route( } // Get the old room power levels - let mut power_levels_event_content: RoomPowerLevelsEventContent = services + let power_levels_event_content: RoomPowerLevelsEventContent = services .rooms .state_accessor .room_state_get_content(&body.room_id, &StateEventType::RoomPowerLevels, "") @@ -836,8 +783,6 @@ pub(crate) async fn upgrade_room_route( .checked_add(int!(1)) .ok_or_else(|| err!(Request(BadJson("users_default power levels event content is not valid"))))?, ); - power_levels_event_content.events_default = new_level; - power_levels_event_content.invite = new_level; // Modify the power levels in the old room to prevent sending of events and // inviting new users @@ -845,14 +790,14 @@ pub(crate) async fn upgrade_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&power_levels_event_content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + PduBuilder::state( + String::new(), + &RoomPowerLevelsEventContent { + events_default: new_level, + invite: new_level, + ..power_levels_event_content + }, + ), sender_user, &body.room_id, &state_lock, diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 1396ae77..5090d557 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -177,10 +177,9 @@ async fn send_state_event_for_key_helper( PduBuilder { event_type: event_type.to_string().into(), content: serde_json::from_str(json.json().get())?, - unsigned: None, state_key: Some(state_key), - redacts: None, timestamp, + ..Default::default() }, sender, room_id, diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index ba081aad..85668038 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -8,7 +8,7 @@ use ruma::{ join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent}, member::{MembershipState, RoomMemberEventContent}, }, - StateEventType, TimelineEventType, + StateEventType, }, CanonicalJsonObject, RoomId, RoomVersionId, UserId, }; @@ -125,30 +125,17 @@ pub(crate) async fn create_join_event_template_route( )); } - let content = to_raw_value(&RoomMemberEventContent { - avatar_url: None, - blurhash: None, - displayname: None, - is_direct: None, - membership: MembershipState::Join, - third_party_invite: None, - reason: None, - join_authorized_via_users_server, - }) - .expect("member event is valid value"); - let (_pdu, mut pdu_json) = services .rooms .timeline .create_hash_and_sign_event( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content, - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - timestamp: None, - }, + PduBuilder::state( + body.user_id.to_string(), + &RoomMemberEventContent { + join_authorized_via_users_server, + ..RoomMemberEventContent::new(MembershipState::Join) + }, + ), &body.user_id, &body.room_id, &state_lock, diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 41ea1c80..81a32c86 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -2,10 +2,7 @@ use axum::extract::State; use conduit::{Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_leave_event}, - events::{ - room::member::{MembershipState, RoomMemberEventContent}, - TimelineEventType, - }, + events::room::member::{MembershipState, RoomMemberEventContent}, }; use serde_json::value::to_raw_value; @@ -39,30 +36,12 @@ pub(crate) async fn create_leave_event_template_route( let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let content = to_raw_value(&RoomMemberEventContent { - avatar_url: None, - blurhash: None, - displayname: None, - is_direct: None, - membership: MembershipState::Leave, - third_party_invite: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("member event is valid value"); let (_pdu, mut pdu_json) = services .rooms .timeline .create_hash_and_sign_event( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content, - unsigned: None, - state_key: Some(body.user_id.to_string()), - redacts: None, - timestamp: None, - }, + PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Leave)), &body.user_id, &body.room_id, &state_lock, diff --git a/src/core/pdu/builder.rs b/src/core/pdu/builder.rs index ba4c19e5..80ff0713 100644 --- a/src/core/pdu/builder.rs +++ b/src/core/pdu/builder.rs @@ -1,20 +1,67 @@ use std::{collections::BTreeMap, sync::Arc}; -use ruma::{events::TimelineEventType, EventId, MilliSecondsSinceUnixEpoch}; +use ruma::{ + events::{EventContent, MessageLikeEventType, StateEventType, TimelineEventType}, + EventId, MilliSecondsSinceUnixEpoch, +}; use serde::Deserialize; -use serde_json::value::RawValue as RawJsonValue; +use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; /// Build the start of a PDU in order to add it to the Database. #[derive(Debug, Deserialize)] -pub struct PduBuilder { +pub struct Builder { #[serde(rename = "type")] pub event_type: TimelineEventType, + pub content: Box, - pub unsigned: Option>, + + pub unsigned: Option, + pub state_key: Option, + pub redacts: Option>, - /// For timestamped messaging, should only be used for appservices - /// + + /// For timestamped messaging, should only be used for appservices. /// Will be set to current time if None pub timestamp: Option, } + +type Unsigned = BTreeMap; + +impl Builder { + pub fn state(state_key: String, content: &T) -> Self + where + T: EventContent, + { + Self { + event_type: content.event_type().into(), + content: to_raw_value(content).expect("Builder failed to serialize state event content to RawValue"), + state_key: Some(state_key), + ..Self::default() + } + } + + pub fn timeline(content: &T) -> Self + where + T: EventContent, + { + Self { + event_type: content.event_type().into(), + content: to_raw_value(content).expect("Builder failed to serialize timeline event content to RawValue"), + ..Self::default() + } + } +} + +impl Default for Builder { + fn default() -> Self { + Self { + event_type: "m.room.message".into(), + content: Box::::default(), + unsigned: None, + state_key: None, + redacts: None, + timestamp: None, + } + } +} diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index a94e2bdc..5f50fe5b 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -21,7 +21,10 @@ use serde_json::{ value::{to_raw_value, RawValue as RawJsonValue, Value as JsonValue}, }; -pub use self::{builder::PduBuilder, count::PduCount}; +pub use self::{ + builder::{Builder, Builder as PduBuilder}, + count::PduCount, +}; use crate::{err, is_true, warn, Error, Result}; #[derive(Deserialize)] diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 3dd5aea3..1631f1cb 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -2,24 +2,20 @@ use std::collections::BTreeMap; use conduit::{pdu::PduBuilder, Result}; use ruma::{ - events::{ - room::{ - canonical_alias::RoomCanonicalAliasEventContent, - create::RoomCreateEventContent, - guest_access::{GuestAccess, RoomGuestAccessEventContent}, - history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, - join_rules::{JoinRule, RoomJoinRulesEventContent}, - member::{MembershipState, RoomMemberEventContent}, - name::RoomNameEventContent, - power_levels::RoomPowerLevelsEventContent, - preview_url::RoomPreviewUrlsEventContent, - topic::RoomTopicEventContent, - }, - TimelineEventType, + events::room::{ + canonical_alias::RoomCanonicalAliasEventContent, + create::RoomCreateEventContent, + guest_access::{GuestAccess, RoomGuestAccessEventContent}, + history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + name::RoomNameEventContent, + power_levels::RoomPowerLevelsEventContent, + preview_url::RoomPreviewUrlsEventContent, + topic::RoomTopicEventContent, }, RoomId, RoomVersionId, }; -use serde_json::value::to_raw_value; use crate::Services; @@ -44,7 +40,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { let room_version = services.globals.default_room_version(); - let mut content = { + let create_content = { use RoomVersionId::*; match room_version { V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => RoomCreateEventContent::new_v1(server_user.clone()), @@ -52,23 +48,20 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { } }; - content.federate = true; - content.predecessor = None; - content.room_version = room_version; - // 1. The room create event services .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomCreate, - content: to_raw_value(&content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + PduBuilder::state( + String::new(), + &RoomCreateEventContent { + federate: true, + predecessor: None, + room_version, + ..create_content + }, + ), server_user, &room_id, &state_lock, @@ -80,24 +73,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Join, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(server_user.to_string()), - redacts: None, - timestamp: None, - }, + PduBuilder::state(server_user.to_string(), &RoomMemberEventContent::new(MembershipState::Join)), server_user, &room_id, &state_lock, @@ -111,18 +87,13 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&RoomPowerLevelsEventContent { + PduBuilder::state( + String::new(), + &RoomPowerLevelsEventContent { users, ..Default::default() - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }, + ), server_user, &room_id, &state_lock, @@ -134,15 +105,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomJoinRules, - content: to_raw_value(&RoomJoinRulesEventContent::new(JoinRule::Invite)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + PduBuilder::state(String::new(), &RoomJoinRulesEventContent::new(JoinRule::Invite)), server_user, &room_id, &state_lock, @@ -154,15 +117,10 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomHistoryVisibility, - content: to_raw_value(&RoomHistoryVisibilityEventContent::new(HistoryVisibility::Shared)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + PduBuilder::state( + String::new(), + &RoomHistoryVisibilityEventContent::new(HistoryVisibility::Shared), + ), server_user, &room_id, &state_lock, @@ -174,15 +132,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomGuestAccess, - content: to_raw_value(&RoomGuestAccessEventContent::new(GuestAccess::Forbidden)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + PduBuilder::state(String::new(), &RoomGuestAccessEventContent::new(GuestAccess::Forbidden)), server_user, &room_id, &state_lock, @@ -195,15 +145,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomName, - content: to_raw_value(&RoomNameEventContent::new(room_name)) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + PduBuilder::state(String::new(), &RoomNameEventContent::new(room_name)), server_user, &room_id, &state_lock, @@ -214,17 +156,12 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomTopic, - content: to_raw_value(&RoomTopicEventContent { + PduBuilder::state( + String::new(), + &RoomTopicEventContent { topic: format!("Manage {}", services.globals.server_name()), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }, + ), server_user, &room_id, &state_lock, @@ -238,18 +175,13 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomCanonicalAlias, - content: to_raw_value(&RoomCanonicalAliasEventContent { + PduBuilder::state( + String::new(), + &RoomCanonicalAliasEventContent { alias: Some(alias.clone()), alt_aliases: Vec::new(), - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }, + ), server_user, &room_id, &state_lock, @@ -266,17 +198,12 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomPreviewUrls, - content: to_raw_value(&RoomPreviewUrlsEventContent { + PduBuilder::state( + String::new(), + &RoomPreviewUrlsEventContent { disabled: true, - }) - .expect("event is valid we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }, + ), server_user, &room_id, &state_lock, diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 6e266ca9..405da982 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -9,11 +9,10 @@ use ruma::{ power_levels::RoomPowerLevelsEventContent, }, tag::{TagEvent, TagEventContent, TagInfo}, - RoomAccountDataEventType, TimelineEventType, + RoomAccountDataEventType, }, RoomId, UserId, }; -use serde_json::value::to_raw_value; use crate::pdu::PduBuilder; @@ -35,24 +34,7 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { self.services .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Invite, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }, + PduBuilder::state(user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Invite)), server_user, &room_id, &state_lock, @@ -61,24 +43,7 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { self.services .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Join, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(user_id.to_string()), - redacts: None, - timestamp: None, - }, + PduBuilder::state(user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Join)), user_id, &room_id, &state_lock, @@ -91,18 +56,13 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { self.services .timeline .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&RoomPowerLevelsEventContent { + PduBuilder::state( + String::new(), + &RoomPowerLevelsEventContent { users, ..Default::default() - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, + }, + ), server_user, &room_id, &state_lock, @@ -117,23 +77,18 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { } } + let welcome_message = String::from("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `!admin --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`"); + // Send welcome message - self.services.timeline.build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&RoomMessageEventContent::text_markdown( - String::from("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `!admin --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`"), - )) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - timestamp: None, - }, - server_user, - &room_id, - &state_lock, - ).await?; + self.services + .timeline + .build_and_append_pdu( + PduBuilder::timeline(&RoomMessageEventContent::text_markdown(welcome_message)), + server_user, + &room_id, + &state_lock, + ) + .await?; Ok(()) } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index da7f3cf4..58cc012c 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -15,13 +15,9 @@ pub use create::create_admin_room; use futures::{FutureExt, TryFutureExt}; use loole::{Receiver, Sender}; use ruma::{ - events::{ - room::message::{Relation, RoomMessageEventContent}, - TimelineEventType, - }, + events::room::message::{Relation, RoomMessageEventContent}, OwnedEventId, OwnedRoomId, RoomId, UserId, }; -use serde_json::value::to_raw_value; use tokio::sync::{Mutex, RwLock}; use crate::{account_data, globals, rooms, rooms::state::RoomMutexGuard, Dep}; @@ -285,20 +281,12 @@ impl Service { ) -> Result<()> { assert!(self.user_is_admin(user_id).await, "sender is not admin"); - let response_pdu = PduBuilder { - event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&content).expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - timestamp: None, - }; - let state_lock = self.services.state.mutex.lock(room_id).await; + if let Err(e) = self .services .timeline - .build_and_append_pdu(response_pdu, user_id, room_id, &state_lock) + .build_and_append_pdu(PduBuilder::timeline(&content), user_id, room_id, &state_lock) .await { self.handle_response_error(e, room_id, user_id, &state_lock) @@ -313,23 +301,14 @@ impl Service { &self, e: Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, ) -> Result<()> { error!("Failed to build and append admin room response PDU: \"{e}\""); - let error_room_message = RoomMessageEventContent::text_plain(format!( + let content = RoomMessageEventContent::text_plain(format!( "Failed to build and append admin room PDU: \"{e}\"\n\nThe original admin command may have finished \ successfully, but we could not return the output." )); - let response_pdu = PduBuilder { - event_type: TimelineEventType::RoomMessage, - content: to_raw_value(&error_room_message).expect("event is valid, we just created it"), - unsigned: None, - state_key: None, - redacts: None, - timestamp: None, - }; - self.services .timeline - .build_and_append_pdu(response_pdu, user_id, room_id, state_lock) + .build_and_append_pdu(PduBuilder::timeline(&content), user_id, room_id, state_lock) .await?; Ok(()) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 3855d92a..19f1f141 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -37,7 +37,6 @@ use ruma::{ ServerName, UserId, }; use serde::Deserialize; -use serde_json::value::to_raw_value; use self::data::Data; use crate::{rooms, rooms::state::RoomMutexGuard, Dep}; @@ -353,21 +352,14 @@ impl Service { pub async fn user_can_invite( &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &RoomMutexGuard, ) -> bool { - let content = to_raw_value(&RoomMemberEventContent::new(MembershipState::Invite)) - .expect("Event content always serializes"); - - let new_event = PduBuilder { - event_type: ruma::events::TimelineEventType::RoomMember, - content, - unsigned: None, - state_key: Some(target_user.into()), - redacts: None, - timestamp: None, - }; - self.services .timeline - .create_hash_and_sign_event(new_event, sender, room_id, state_lock) + .create_hash_and_sign_event( + PduBuilder::state(target_user.into(), &RoomMemberEventContent::new(MembershipState::Invite)), + sender, + room_id, + state_lock, + ) .await .is_ok() } From 8ea2dccc9ad72df70555c8dc04ee85d6ed49f1a7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 9 Jul 2024 17:23:00 +0000 Subject: [PATCH 0898/2091] sort rustfmt Signed-off-by: Jason Volk --- rustfmt.toml | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/rustfmt.toml b/rustfmt.toml index 114677d4..fd912a19 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -1,28 +1,27 @@ -edition = "2021" - +array_width = 80 +chain_width = 60 +comment_width = 80 condense_wildcard_suffixes = true +edition = "2021" +fn_call_width = 80 +fn_params_layout = "Compressed" +fn_single_line = true format_code_in_doc_comments = true format_macro_bodies = true format_macro_matchers = true format_strings = true -hex_literal_case = "Upper" -max_width = 120 -tab_spaces = 4 -array_width = 80 -comment_width = 80 -wrap_comments = true -fn_params_layout = "Compressed" -fn_call_width = 80 -fn_single_line = true +group_imports = "StdExternalCrate" hard_tabs = true -match_block_trailing_comma = true +hex_literal_case = "Upper" imports_granularity = "Crate" +match_block_trailing_comma = true +max_width = 120 +newline_style = "Unix" normalize_comments = false reorder_impl_items = true reorder_imports = true -group_imports = "StdExternalCrate" -newline_style = "Unix" +tab_spaces = 4 use_field_init_shorthand = true use_small_heuristics = "Off" use_try_shorthand = true -chain_width = 60 +wrap_comments = true From c9c405facfcfd30c76e3b830929a4e4c90b930c2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 29 Sep 2024 09:20:17 +0000 Subject: [PATCH 0899/2091] relax Sized bound for debug::type_name Signed-off-by: Jason Volk --- src/core/debug.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/debug.rs b/src/core/debug.rs index 1e36ca8e..85574a2f 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -86,11 +86,11 @@ pub fn panic_str(p: &Box) -> &'static str { p.downcast_ref::<&st #[inline(always)] #[must_use] -pub fn rttype_name(_: &T) -> &'static str { type_name::() } +pub fn rttype_name(_: &T) -> &'static str { type_name::() } #[inline(always)] #[must_use] -pub fn type_name() -> &'static str { std::any::type_name::() } +pub fn type_name() -> &'static str { std::any::type_name::() } #[must_use] #[inline] From 16f82b02a07110ae3f4133758d3a7e20ca2401ea Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 29 Sep 2024 04:18:47 +0000 Subject: [PATCH 0900/2091] add util to restore state on scope exit Signed-off-by: Jason Volk --- src/core/utils/defer.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/core/utils/defer.rs b/src/core/utils/defer.rs index 08477b6f..29199700 100644 --- a/src/core/utils/defer.rs +++ b/src/core/utils/defer.rs @@ -15,8 +15,14 @@ macro_rules! defer { }; ($body:expr) => { - $crate::defer! {{ - $body - }} + $crate::defer! {{ $body }} + }; +} + +#[macro_export] +macro_rules! scope_restore { + ($val:ident, $ours:expr) => { + let theirs = $crate::utils::exchange($val, $ours); + $crate::defer! {{ *$val = theirs; }}; }; } From a5e85727b5d1447a67e6ef970f5cc9d54f866f87 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 29 Sep 2024 09:01:57 +0000 Subject: [PATCH 0901/2091] add tuple access functor-macro Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index c34691d2..4dbecf91 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -39,3 +39,10 @@ pub use self::{ #[inline] pub fn exchange(state: &mut T, source: T) -> T { std::mem::replace(state, source) } + +#[macro_export] +macro_rules! at { + ($idx:tt) => { + |t| t.$idx + }; +} From 43b0bb6a5e62a9262abcad63431bf9ac0c2d60cc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 7 Oct 2024 19:19:53 +0000 Subject: [PATCH 0902/2091] add non-allocating fixed-size random string generator Signed-off-by: Jason Volk --- src/core/utils/rand.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/core/utils/rand.rs b/src/core/utils/rand.rs index b80671eb..d717c4bd 100644 --- a/src/core/utils/rand.rs +++ b/src/core/utils/rand.rs @@ -3,6 +3,7 @@ use std::{ time::{Duration, SystemTime}, }; +use arrayvec::ArrayString; use rand::{thread_rng, Rng}; pub fn string(length: usize) -> String { @@ -13,6 +14,18 @@ pub fn string(length: usize) -> String { .collect() } +#[inline] +pub fn string_array() -> ArrayString { + let mut ret = ArrayString::::new(); + thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(LENGTH) + .map(char::from) + .for_each(|c| ret.push(c)); + + ret +} + #[inline] #[must_use] pub fn timepoint_secs(range: Range) -> SystemTime { From c40d20cb95283c1e03c72fec437c48b8debee678 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 25 Sep 2024 05:04:25 +0000 Subject: [PATCH 0903/2091] add macro util to determine if cargo build or check/clippy. Signed-off-by: Jason Volk --- src/macros/utils.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/macros/utils.rs b/src/macros/utils.rs index 197dd90e..e4ffc622 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -2,6 +2,16 @@ use syn::{parse_str, Expr, Generics, Lit, Meta}; use crate::Result; +pub(crate) fn is_cargo_build() -> bool { + std::env::args() + .find(|flag| flag.starts_with("--emit")) + .as_ref() + .and_then(|flag| flag.split_once('=')) + .map(|val| val.1.split(',')) + .and_then(|mut vals| vals.find(|elem| *elem == "link")) + .is_some() +} + pub(crate) fn get_named_generics(args: &[Meta], name: &str) -> Result { const DEFAULT: &str = "<>"; From 2a59a56eaa6d63c7db6634b1c1662d7f34dd7598 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Sep 2024 22:17:02 +0000 Subject: [PATCH 0904/2091] initial example-config generator Signed-off-by: Jason Volk --- src/core/config/mod.rs | 2 + src/macros/config.rs | 98 ++++++++++++++++++++++++++++++++++++++++++ src/macros/mod.rs | 8 +++- 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 src/macros/config.rs diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 126b3123..64e1c9ba 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -5,6 +5,7 @@ use std::{ path::PathBuf, }; +use conduit_macros::config_example_generator; use either::{ Either, Either::{Left, Right}, @@ -27,6 +28,7 @@ pub mod check; pub mod proxy; /// all the config options for conduwuit +#[config_example_generator] #[derive(Clone, Debug, Deserialize)] #[allow(clippy::struct_excessive_bools)] pub struct Config { diff --git a/src/macros/config.rs b/src/macros/config.rs new file mode 100644 index 00000000..6d29c21f --- /dev/null +++ b/src/macros/config.rs @@ -0,0 +1,98 @@ +use std::fmt::Write; + +use proc_macro::TokenStream; +use quote::ToTokens; +use syn::{Expr, ExprLit, Field, Fields, FieldsNamed, ItemStruct, Lit, Meta, MetaNameValue, Type, TypePath}; + +use crate::{utils::is_cargo_build, Result}; + +#[allow(clippy::needless_pass_by_value)] +pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result { + if is_cargo_build() { + generate_example(&input, args)?; + } + + Ok(input.to_token_stream().into()) +} + +#[allow(clippy::needless_pass_by_value)] +#[allow(unused_variables)] +fn generate_example(input: &ItemStruct, _args: &[Meta]) -> Result<()> { + if let Fields::Named(FieldsNamed { + named, + .. + }) = &input.fields + { + for field in named { + let Some(ident) = &field.ident else { + continue; + }; + + let Some(doc) = get_doc_comment(field) else { + continue; + }; + + let Some(type_name) = get_type_name(field) else { + continue; + }; + + //println!("{:?} {type_name:?}\n{doc}", ident.to_string()); + } + } + + Ok(()) +} + +fn get_doc_comment(field: &Field) -> Option { + let mut out = String::new(); + for attr in &field.attrs { + let Meta::NameValue(MetaNameValue { + path, + value, + .. + }) = &attr.meta + else { + continue; + }; + + if !path + .segments + .iter() + .next() + .is_some_and(|s| s.ident == "doc") + { + continue; + } + + let Expr::Lit(ExprLit { + lit, + .. + }) = &value + else { + continue; + }; + + let Lit::Str(token) = &lit else { + continue; + }; + + writeln!(&mut out, "# {}", token.value()).expect("wrote to output string buffer"); + } + + (!out.is_empty()).then_some(out) +} + +fn get_type_name(field: &Field) -> Option { + let Type::Path(TypePath { + path, + .. + }) = &field.ty + else { + return None; + }; + + path.segments + .iter() + .next() + .map(|segment| segment.ident.to_string()) +} diff --git a/src/macros/mod.rs b/src/macros/mod.rs index d32cda71..1aa1e24f 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -1,5 +1,6 @@ mod admin; mod cargo; +mod config; mod debug; mod implement; mod refutable; @@ -9,7 +10,7 @@ mod utils; use proc_macro::TokenStream; use syn::{ parse::{Parse, Parser}, - parse_macro_input, Error, Item, ItemConst, ItemEnum, ItemFn, Meta, + parse_macro_input, Error, Item, ItemConst, ItemEnum, ItemFn, ItemStruct, Meta, }; pub(crate) type Result = std::result::Result; @@ -47,6 +48,11 @@ pub fn implement(args: TokenStream, input: TokenStream) -> TokenStream { attribute_macro::(args, input, implement::implement) } +#[proc_macro_attribute] +pub fn config_example_generator(args: TokenStream, input: TokenStream) -> TokenStream { + attribute_macro::(args, input, config::example_generator) +} + fn attribute_macro(args: TokenStream, input: TokenStream, func: F) -> TokenStream where F: Fn(I, &[Meta]) -> Result, From f67cfcd5353bf112760f89a9451aafc2ba2d9fde Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Oct 2024 19:10:20 +0000 Subject: [PATCH 0905/2091] cleanup Config::load() Signed-off-by: Jason Volk --- src/core/config/mod.rs | 42 +++++++++++++++++------------------------- src/main/server.rs | 2 +- 2 files changed, 18 insertions(+), 26 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 64e1c9ba..40c900e5 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1,3 +1,6 @@ +pub mod check; +pub mod proxy; + use std::{ collections::{BTreeMap, BTreeSet}, fmt, @@ -22,10 +25,7 @@ use url::Url; pub use self::check::check; use self::proxy::ProxyConfig; -use crate::{error::Error, utils::sys, Err, Result}; - -pub mod check; -pub mod proxy; +use crate::{err, error::Error, utils::sys, Result}; /// all the config options for conduwuit #[config_example_generator] @@ -441,34 +441,26 @@ const DEPRECATED_KEYS: &[&str; 9] = &[ impl Config { /// Pre-initialize config - pub fn load(paths: &Option>) -> Result { - let raw_config = if let Some(config_file_env) = Env::var("CONDUIT_CONFIG") { - Figment::new().merge(Toml::file(config_file_env).nested()) - } else if let Some(config_file_arg) = Env::var("CONDUWUIT_CONFIG") { - Figment::new().merge(Toml::file(config_file_arg).nested()) - } else if let Some(config_file_args) = paths { - let mut figment = Figment::new(); + pub fn load(paths: Option<&[PathBuf]>) -> Result { + let paths_files = paths.into_iter().flatten().map(Toml::file); - for config in config_file_args { - figment = figment.merge(Toml::file(config).nested()); - } + let envs = [Env::var("CONDUIT_CONFIG"), Env::var("CONDUWUIT_CONFIG")]; + let envs_files = envs.into_iter().flatten().map(Toml::file); - figment - } else { - Figment::new() - }; - - Ok(raw_config + let config = envs_files + .chain(paths_files) + .fold(Figment::new(), |config, file| config.merge(file.nested())) .merge(Env::prefixed("CONDUIT_").global().split("__")) - .merge(Env::prefixed("CONDUWUIT_").global().split("__"))) + .merge(Env::prefixed("CONDUWUIT_").global().split("__")); + + Ok(config) } /// Finalize config pub fn new(raw_config: &Figment) -> Result { - let config = match raw_config.extract::() { - Err(e) => return Err!("There was a problem with your configuration file: {e}"), - Ok(config) => config, - }; + let config = raw_config + .extract::() + .map_err(|e| err!("There was a problem with your configuration file: {e}"))?; // don't start if we're listening on both UNIX sockets and TCP at same time check::is_dual_listening(raw_config)?; diff --git a/src/main/server.rs b/src/main/server.rs index e435b2f4..4813d586 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -24,7 +24,7 @@ pub(crate) struct Server { impl Server { pub(crate) fn build(args: &Args, runtime: Option<&runtime::Handle>) -> Result, Error> { - let raw_config = Config::load(&args.config)?; + let raw_config = Config::load(args.config.as_deref())?; let raw_config = crate::clap::update(raw_config, args)?; let config = Config::new(&raw_config)?; From fc4d109f35d2cfb54ae3a463cb66e318e5947510 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 5 Oct 2024 19:39:27 +0000 Subject: [PATCH 0906/2091] add document comments to config items Signed-off-by: Jason Volk --- src/core/config/mod.rs | 845 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 842 insertions(+), 3 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 40c900e5..b5e07da2 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -31,221 +31,768 @@ use crate::{err, error::Error, utils::sys, Result}; #[config_example_generator] #[derive(Clone, Debug, Deserialize)] #[allow(clippy::struct_excessive_bools)] +#[allow(rustdoc::broken_intra_doc_links, rustdoc::bare_urls)] pub struct Config { - /// [`IpAddr`] conduwuit will listen on (can be IPv4 or IPv6) + /// The server_name is the pretty name of this server. It is used as a + /// suffix for user and room ids. Examples: matrix.org, conduit.rs + /// + /// The Conduit server needs all /_matrix/ requests to be reachable at + /// https://your.server.name/ on port 443 (client-server) and 8448 (federation). + /// + /// If that's not possible for you, you can create /.well-known files to + /// redirect requests (delegation). See + /// https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixclient + /// and + /// https://spec.matrix.org/v1.9/server-server-api/#getwell-knownmatrixserver + /// for more information. + /// + /// YOU NEED TO EDIT THIS + pub server_name: OwnedServerName, + + /// Database backend: Only rocksdb is supported. + /// default address (IPv4 or IPv6) conduwuit will listen on. Generally you + /// want this to be localhost (127.0.0.1 / ::1). If you are using Docker or + /// a container NAT networking setup, you likely need this to be 0.0.0.0. + /// To listen multiple addresses, specify a vector e.g. ["127.0.0.1", "::1"] + /// + /// default if unspecified is both IPv4 and IPv6 localhost: ["127.0.0.1", + /// "::1"] #[serde(default = "default_address")] address: ListeningAddr, - /// default TCP port(s) conduwuit will listen on + + /// The port(s) conduwuit will be running on. You need to set up a reverse + /// proxy such as Caddy or Nginx so all requests to /_matrix on port 443 + /// and 8448 will be forwarded to the conduwuit instance running on this + /// port Docker users: Don't change this, you'll need to map an external + /// port to this. To listen on multiple ports, specify a vector e.g. [8080, + /// 8448] + /// + /// default if unspecified is 8008 #[serde(default = "default_port")] port: ListeningPort, + pub tls: Option, + + /// Uncomment unix_socket_path to listen on a UNIX socket at the specified + /// path. If listening on a UNIX socket, you must remove/comment the + /// 'address' key if defined and add your reverse proxy to the 'conduwuit' + /// group, unless world RW permissions are specified with unix_socket_perms + /// (666 minimum). pub unix_socket_path: Option, + #[serde(default = "default_unix_socket_perms")] pub unix_socket_perms: u32, - pub server_name: OwnedServerName, + #[serde(default = "default_database_backend")] pub database_backend: String, + + /// This is the only directory where conduwuit will save its data, including + /// media. Note: this was previously "/var/lib/matrix-conduit" pub database_path: PathBuf, + pub database_backup_path: Option, + #[serde(default = "default_database_backups_to_keep")] pub database_backups_to_keep: i16, + + /// Set this to any float value in megabytes for conduwuit to tell the + /// database engine that this much memory is available for database-related + /// caches. May be useful if you have significant memory to spare to + /// increase performance. Defaults to 256.0 #[serde(default = "default_db_cache_capacity_mb")] pub db_cache_capacity_mb: f64, + + /// Option to control adding arbitrary text to the end of the user's + /// displayname upon registration with a space before the text. This was the + /// lightning bolt emoji option, just replaced with support for adding your + /// own custom text or emojis. To disable, set this to "" (an empty string) + /// Defaults to "🏳️⚧️" (trans pride flag) #[serde(default = "default_new_user_displayname_suffix")] pub new_user_displayname_suffix: String, + + /// If enabled, conduwuit will send a simple GET request periodically to + /// `https://pupbrain.dev/check-for-updates/stable` for any new + /// announcements made. Despite the name, this is not an update check + /// endpoint, it is simply an announcement check endpoint. Defaults to + /// false. #[serde(default)] pub allow_check_for_updates: bool, #[serde(default = "default_pdu_cache_capacity")] pub pdu_cache_capacity: u32, + + /// Set this to any float value to multiply conduwuit's in-memory LRU caches + /// with. May be useful if you have significant memory to spare to increase + /// performance. + /// + /// This was previously called `conduit_cache_capacity_modifier` + /// + /// Defaults to 1.0. #[serde(default = "default_cache_capacity_modifier", alias = "conduit_cache_capacity_modifier")] pub cache_capacity_modifier: f64, + #[serde(default = "default_auth_chain_cache_capacity")] pub auth_chain_cache_capacity: u32, + #[serde(default = "default_shorteventid_cache_capacity")] pub shorteventid_cache_capacity: u32, + #[serde(default = "default_eventidshort_cache_capacity")] pub eventidshort_cache_capacity: u32, + #[serde(default = "default_shortstatekey_cache_capacity")] pub shortstatekey_cache_capacity: u32, + #[serde(default = "default_statekeyshort_cache_capacity")] pub statekeyshort_cache_capacity: u32, + #[serde(default = "default_server_visibility_cache_capacity")] pub server_visibility_cache_capacity: u32, + #[serde(default = "default_user_visibility_cache_capacity")] pub user_visibility_cache_capacity: u32, + #[serde(default = "default_stateinfo_cache_capacity")] pub stateinfo_cache_capacity: u32, + #[serde(default = "default_roomid_spacehierarchy_cache_capacity")] pub roomid_spacehierarchy_cache_capacity: u32, + /// Maximum entries stored in DNS memory-cache. The size of an entry may + /// vary so please take care if raising this value excessively. Only + /// decrease this when using an external DNS cache. Please note + /// that systemd does *not* count as an external cache, even when configured + /// to do so. #[serde(default = "default_dns_cache_entries")] pub dns_cache_entries: u32, + + /// Minimum time-to-live in seconds for entries in the DNS cache. The + /// default may appear high to most administrators; this is by design. Only + /// decrease this if you are using an external DNS cache. #[serde(default = "default_dns_min_ttl")] pub dns_min_ttl: u64, + + /// Minimum time-to-live in seconds for NXDOMAIN entries in the DNS cache. + /// This value is critical for the server to federate efficiently. + /// NXDOMAIN's are assumed to not be returning to the federation + /// and aggressively cached rather than constantly rechecked. + /// + /// Defaults to 3 days as these are *very rarely* false negatives. #[serde(default = "default_dns_min_ttl_nxdomain")] pub dns_min_ttl_nxdomain: u64, + + /// Number of retries after a timeout. #[serde(default = "default_dns_attempts")] pub dns_attempts: u16, + + /// The number of seconds to wait for a reply to a DNS query. Please note + /// that recursive queries can take up to several seconds for some domains, + /// so this value should not be too low. #[serde(default = "default_dns_timeout")] pub dns_timeout: u64, + + /// Fallback to TCP on DNS errors. Set this to false if unsupported by + /// nameserver. #[serde(default = "true_fn")] pub dns_tcp_fallback: bool, + + /// Enable to query all nameservers until the domain is found. Referred to + /// as "trust_negative_responses" in hickory_reso> This can avoid useless + /// DNS queries if the first nameserver responds with NXDOMAIN or an empty + /// NOERROR response. + /// + /// The default is to query one nameserver and stop (false). #[serde(default = "true_fn")] pub query_all_nameservers: bool, + + /// Enables using *only* TCP for querying your specified nameservers instead + /// of UDP. + /// + /// You very likely do *not* want this. hickory-resolver already falls back + /// to TCP on UDP errors. Defaults to false #[serde(default)] pub query_over_tcp_only: bool, + + /// DNS A/AAAA record lookup strategy + /// + /// Takes a number of one of the following options: + /// 1 - Ipv4Only (Only query for A records, no AAAA/IPv6) + /// 2 - Ipv6Only (Only query for AAAA records, no A/IPv4) + /// 3 - Ipv4AndIpv6 (Query for A and AAAA records in parallel, uses whatever + /// returns a successful response first) 4 - Ipv6thenIpv4 (Query for AAAA + /// record, if that fails then query the A record) 5 - Ipv4thenIpv6 (Query + /// for A record, if that fails then query the AAAA record) + /// + /// If you don't have IPv6 networking, then for better performance it may be + /// suitable to set this to Ipv4Only (1) as you will never ever use the + /// AAAA record contents even if the AAAA record is successful instead of + /// the A record. + /// + /// Defaults to 5 - Ipv4ThenIpv6 as this is the most compatible and IPv4 + /// networking is currently the most prevalent. #[serde(default = "default_ip_lookup_strategy")] pub ip_lookup_strategy: u8, + /// Max request size for file uploads #[serde(default = "default_max_request_size")] pub max_request_size: usize, + #[serde(default = "default_max_fetch_prev_events")] pub max_fetch_prev_events: u16, + /// Default/base connection timeout. + /// This is used only by URL previews and update/news endpoint checks + /// + /// Defaults to 10 seconds #[serde(default = "default_request_conn_timeout")] pub request_conn_timeout: u64, + + /// Default/base request timeout. The time waiting to receive more data from + /// another server. This is used only by URL previews, update/news, and + /// misc endpoint checks + /// + /// Defaults to 35 seconds #[serde(default = "default_request_timeout")] pub request_timeout: u64, + + /// Default/base request total timeout. The time limit for a whole request. + /// This is set very high to not cancel healthy requests while serving as a + /// backstop. This is used only by URL previews and update/news endpoint + /// checks + /// + /// Defaults to 320 seconds #[serde(default = "default_request_total_timeout")] pub request_total_timeout: u64, + + /// Default/base idle connection pool timeout + /// This is used only by URL previews and update/news endpoint checks + /// + /// Defaults to 5 seconds #[serde(default = "default_request_idle_timeout")] pub request_idle_timeout: u64, + + /// Default/base max idle connections per host + /// This is used only by URL previews and update/news endpoint checks + /// + /// Defaults to 1 as generally the same open connection can be re-used #[serde(default = "default_request_idle_per_host")] pub request_idle_per_host: u16, + + /// Federation well-known resolution connection timeout + /// + /// Defaults to 6 seconds #[serde(default = "default_well_known_conn_timeout")] pub well_known_conn_timeout: u64, + + /// Federation HTTP well-known resolution request timeout + /// + /// Defaults to 10 seconds #[serde(default = "default_well_known_timeout")] pub well_known_timeout: u64, + + /// Federation client request timeout + /// You most definitely want this to be high to account for extremely large + /// room joins, slow homeservers, your own resources etc. + /// + /// Defaults to 300 seconds #[serde(default = "default_federation_timeout")] pub federation_timeout: u64, + + /// Federation client idle connection pool timeout + /// + /// Defaults to 25 seconds #[serde(default = "default_federation_idle_timeout")] pub federation_idle_timeout: u64, + + /// Federation client max idle connections per host + /// + /// Defaults to 1 as generally the same open connection can be re-used #[serde(default = "default_federation_idle_per_host")] pub federation_idle_per_host: u16, + + /// Federation sender request timeout + /// The time it takes for the remote server to process sent transactions can + /// take a while. + /// + /// Defaults to 180 seconds #[serde(default = "default_sender_timeout")] pub sender_timeout: u64, + + /// Federation sender idle connection pool timeout + /// + /// Defaults to 180 seconds #[serde(default = "default_sender_idle_timeout")] pub sender_idle_timeout: u64, + + /// Federation sender transaction retry backoff limit + /// + /// Defaults to 86400 seconds #[serde(default = "default_sender_retry_backoff_limit")] pub sender_retry_backoff_limit: u64, + + /// Appservice URL request connection timeout + /// + /// Defaults to 35 seconds as generally appservices are hosted within the + /// same network #[serde(default = "default_appservice_timeout")] pub appservice_timeout: u64, + + /// Appservice URL idle connection pool timeout + /// + /// Defaults to 300 seconds #[serde(default = "default_appservice_idle_timeout")] pub appservice_idle_timeout: u64, + + /// Notification gateway pusher idle connection pool timeout + /// + /// Defaults to 15 seconds #[serde(default = "default_pusher_idle_timeout")] pub pusher_idle_timeout: u64, + /// Enables registration. If set to false, no users can register on this + /// server. + /// + /// If set to true without a token configured, users can register with no + /// form of 2nd- step only if you set + /// `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` to + /// true in your config. + /// + /// If you would like registration only via token reg, please configure + /// `registration_token` or `registration_token_file`. #[serde(default)] pub allow_registration: bool, + #[serde(default)] pub yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse: bool, + + /// A static registration token that new users will have to provide when + /// creating an account. If unset and `allow_registration` is true, + /// registration is open without any condition. YOU NEED TO EDIT THIS. pub registration_token: Option, + + /// Path to a file on the system that gets read for the registration token + /// + /// conduwuit must be able to access the file, and it must not be empty + /// + /// no default pub registration_token_file: Option, + + /// controls whether encrypted rooms and events are allowed (default true) #[serde(default = "true_fn")] pub allow_encryption: bool, + + /// controls whether federation is allowed or not + /// defaults to true #[serde(default = "true_fn")] pub allow_federation: bool, + #[serde(default)] pub federation_loopback: bool, + + /// Set this to true to allow your server's public room directory to be + /// federated. Set this to false to protect against /publicRooms spiders, + /// but will forbid external users from viewing your server's public room + /// directory. If federation is disabled entirely (`allow_federation`), + /// this is inherently false. #[serde(default)] pub allow_public_room_directory_over_federation: bool, + + /// Set this to true to allow your server's public room directory to be + /// queried without client authentication (access token) through the Client + /// APIs. Set this to false to protect against /publicRooms spiders. #[serde(default)] pub allow_public_room_directory_without_auth: bool, + + /// allow guests/unauthenticated users to access TURN credentials + /// + /// this is the equivalent of Synapse's `turn_allow_guests` config option. + /// this allows any unauthenticated user to call + /// `/_matrix/client/v3/voip/turnServer`. + /// + /// defaults to false #[serde(default)] pub turn_allow_guests: bool, + + /// Set this to true to lock down your server's public room directory and + /// only allow admins to publish rooms to the room directory. Unpublishing + /// is still allowed by all users with this enabled. + /// + /// Defaults to false #[serde(default)] pub lockdown_public_room_directory: bool, + + /// Set this to true to allow federating device display names / allow + /// external users to see your device display name. If federation is + /// disabled entirely (`allow_federation`), this is inherently false. For + /// privacy, this is best disabled. #[serde(default)] pub allow_device_name_federation: bool, + + /// Config option to allow or disallow incoming federation requests that + /// obtain the profiles of our local users from + /// `/_matrix/federation/v1/query/profile` + /// + /// This is inherently false if `allow_federation` is disabled + /// + /// Defaults to true #[serde(default = "true_fn")] pub allow_profile_lookup_federation_requests: bool, + + /// controls whether users are allowed to create rooms. + /// appservices and admins are always allowed to create rooms + /// defaults to true #[serde(default = "true_fn")] pub allow_room_creation: bool, + + /// Set to false to disable users from joining or creating room versions + /// that aren't 100% officially supported by conduwuit. + /// conduwuit officially supports room versions 6 - 10. conduwuit has + /// experimental/unstable support for 3 - 5, and 11. Defaults to true. #[serde(default = "true_fn")] pub allow_unstable_room_versions: bool, + #[serde(default = "default_default_room_version")] pub default_room_version: RoomVersionId, + #[serde(default)] pub well_known: WellKnownConfig, + #[serde(default)] pub allow_jaeger: bool, + #[serde(default = "default_jaeger_filter")] pub jaeger_filter: String, + + /// If the 'perf_measurements' feature is enabled, enables collecting folded + /// stack trace profile of tracing spans using tracing_flame. The resulting + /// profile can be visualized with inferno[1], speedscope[2], or a number of + /// other tools. [1]: https://github.com/jonhoo/inferno + /// [2]: www.speedscope.app #[serde(default)] pub tracing_flame: bool, + #[serde(default = "default_tracing_flame_filter")] pub tracing_flame_filter: String, + #[serde(default = "default_tracing_flame_output_path")] pub tracing_flame_output_path: String, + #[serde(default)] pub proxy: ProxyConfig, + pub jwt_secret: Option, + + /// Servers listed here will be used to gather public keys of other servers + /// (notary trusted key servers). + /// + /// (Currently, conduwuit doesn't support batched key requests, so this list + /// should only contain other Synapse servers) Defaults to `matrix.org` #[serde(default = "default_trusted_servers")] pub trusted_servers: Vec, + + /// Option to control whether conduwuit will query your list of trusted + /// notary key servers (`trusted_servers`) for remote homeserver signing + /// keys it doesn't know *first*, or query the individual servers first + /// before falling back to the trusted key servers. + /// + /// The former/default behaviour makes federated/remote rooms joins + /// generally faster because we're querying a single (or list of) server + /// that we know works, is reasonably fast, and is reliable for just about + /// all the homeserver signing keys in the room. Querying individual + /// servers may take longer depending on the general infrastructure of + /// everyone in there, how many dead servers there are, etc. + /// + /// However, this does create an increased reliance on one single or + /// multiple large entities as `trusted_servers` should generally + /// contain long-term and large servers who know a very large number of + /// homeservers. + /// + /// If you don't know what any of this means, leave this and + /// `trusted_servers` alone to their defaults. + /// + /// Defaults to true as this is the fastest option for federation. #[serde(default = "true_fn")] pub query_trusted_key_servers_first: bool, + + /// max log level for conduwuit. allows debug, info, warn, or error + /// see also: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives + /// **Caveat**: + /// For release builds, the tracing crate is configured to only implement + /// levels higher than error to avoid unnecessary overhead in the compiled + /// binary from trace macros. For debug builds, this restriction is not + /// applied. + /// + /// Defaults to "info" #[serde(default = "default_log")] pub log: String, + + /// controls whether logs will be outputted with ANSI colours + /// + /// defaults to true #[serde(default = "true_fn", alias = "log_colours")] pub log_colors: bool, + + /// OpenID token expiration/TTL in seconds + /// + /// These are the OpenID tokens that are primarily used for Matrix account + /// integrations, *not* OIDC/OpenID Connect/etc + /// + /// Defaults to 3600 (1 hour) #[serde(default = "default_openid_token_ttl")] pub openid_token_ttl: u64, + + /// TURN username to provide the client + /// + /// no default #[serde(default)] pub turn_username: String, + + /// TURN password to provide the client + /// + /// no default #[serde(default)] pub turn_password: String, + + /// vector list of TURN URIs/servers to use + /// + /// replace "example.turn.uri" with your TURN domain, such as the coturn + /// "realm". if using TURN over TLS, replace "turn:" with "turns:" + /// + /// No default #[serde(default = "Vec::new")] pub turn_uris: Vec, + + /// TURN secret to use for generating the HMAC-SHA1 hash apart of username + /// and password generation + /// + /// this is more secure, but if needed you can use traditional + /// username/password below. + /// + /// no default #[serde(default)] pub turn_secret: String, + + /// TURN secret to use that's read from the file path specified + /// + /// this takes priority over "turn_secret" first, and falls back to + /// "turn_secret" if invalid or failed to open. + /// + /// no default pub turn_secret_file: Option, + + /// TURN TTL + /// + /// Default is 86400 seconds #[serde(default = "default_turn_ttl")] pub turn_ttl: u64, + /// List/vector of room **IDs** that conduwuit will make newly registered + /// users join. The room IDs specified must be rooms that you have joined + /// at least once on the server, and must be public. + /// + /// No default. #[serde(default = "Vec::new")] pub auto_join_rooms: Vec, + + /// Config option to automatically deactivate the account of any user who + /// attempts to join a: + /// - banned room + /// - forbidden room alias + /// - room alias or ID with a forbidden server name + /// + /// This may be useful if all your banned lists consist of toxic rooms or + /// servers that no good faith user would ever attempt to join, and + /// to automatically remediate the problem without any admin user + /// intervention. + /// + /// This will also make the user leave all rooms. Federation (e.g. remote + /// room invites) are ignored here. + /// + /// Defaults to false as rooms can be banned for non-moderation-related + /// reasons #[serde(default)] pub auto_deactivate_banned_room_attempts: bool, + /// RocksDB log level. This is not the same as conduwuit's log level. This + /// is the log level for the RocksDB engine/library which show up in your + /// database folder/path as `LOG` files. Defaults to error. conduwuit will + /// typically log RocksDB errors as normal. #[serde(default = "default_rocksdb_log_level")] pub rocksdb_log_level: String, + #[serde(default)] pub rocksdb_log_stderr: bool, + + /// Max RocksDB `LOG` file size before rotating in bytes. Defaults to 4MB. #[serde(default = "default_rocksdb_max_log_file_size")] pub rocksdb_max_log_file_size: usize, + + /// Time in seconds before RocksDB will forcibly rotate logs. Defaults to 0. #[serde(default = "default_rocksdb_log_time_to_roll")] pub rocksdb_log_time_to_roll: usize, + + /// Set this to true to use RocksDB config options that are tailored to HDDs + /// (slower device storage) + /// + /// It is worth noting that by default, conduwuit will use RocksDB with + /// Direct IO enabled. *Generally* speaking this improves performance as it + /// bypasses buffered I/O (system page cache). However there is a potential + /// chance that Direct IO may cause issues with database operations if your + /// setup is uncommon. This has been observed with FUSE filesystems, and + /// possibly ZFS filesystem. RocksDB generally deals/corrects these issues + /// but it cannot account for all setups. If you experience any weird + /// RocksDB issues, try enabling this option as it turns off Direct IO and + /// feel free to report in the conduwuit Matrix room if this option fixes + /// your DB issues. See https://github.com/facebook/rocksdb/wiki/Direct-IO for more information. + /// + /// Defaults to false #[serde(default)] pub rocksdb_optimize_for_spinning_disks: bool, + + /// Enables direct-io to increase database performance. This is enabled by + /// default. Set this option to false if the database resides on a + /// filesystem which does not support direct-io. #[serde(default = "true_fn")] pub rocksdb_direct_io: bool, + + /// Amount of threads that RocksDB will use for parallelism on database + /// operatons such as cleanup, sync, flush, compaction, etc. Set to 0 to use + /// all your logical threads. + /// + /// Defaults to your CPU logical thread count. #[serde(default = "default_rocksdb_parallelism_threads")] pub rocksdb_parallelism_threads: usize, + + /// Maximum number of LOG files RocksDB will keep. This must *not* be set to + /// 0. It must be at least 1. Defaults to 3 as these are not very useful. #[serde(default = "default_rocksdb_max_log_files")] pub rocksdb_max_log_files: usize, + + /// Type of RocksDB database compression to use. + /// Available options are "zstd", "zlib", "bz2", "lz4", or "none" + /// It is best to use ZSTD as an overall good balance between + /// speed/performance, storage, IO amplification, and CPU usage. + /// For more performance but less compression (more storage used) and less + /// CPU usage, use LZ4. See https://github.com/facebook/rocksdb/wiki/Compression for more details. + /// + /// "none" will disable compression. + /// + /// Defaults to "zstd" #[serde(default = "default_rocksdb_compression_algo")] pub rocksdb_compression_algo: String, + + /// Level of compression the specified compression algorithm for RocksDB to + /// use. Default is 32767, which is internally read by RocksDB as the + /// default magic number and translated to the library's default + /// compression level as they all differ. + /// See their `kDefaultCompressionLevel`. #[serde(default = "default_rocksdb_compression_level")] pub rocksdb_compression_level: i32, + + /// Level of compression the specified compression algorithm for the + /// bottommost level/data for RocksDB to use. Default is 32767, which is + /// internally read by RocksDB as the default magic number and translated + /// to the library's default compression level as they all differ. + /// See their `kDefaultCompressionLevel`. + /// + /// Since this is the bottommost level (generally old and least used data), + /// it may be desirable to have a very high compression level here as it's + /// lesss likely for this data to be used. Research your chosen compression + /// algorithm. #[serde(default = "default_rocksdb_bottommost_compression_level")] pub rocksdb_bottommost_compression_level: i32, + + /// Whether to enable RocksDB "bottommost_compression". + /// At the expense of more CPU usage, this will further compress the + /// database to reduce more storage. It is recommended to use ZSTD + /// compression with this for best compression results. See https://github.com/facebook/rocksdb/wiki/Compression for more details. + /// + /// Defaults to false as this uses more CPU when compressing. #[serde(default)] pub rocksdb_bottommost_compression: bool, + + /// Database recovery mode (for RocksDB WAL corruption) + /// + /// Use this option when the server reports corruption and refuses to start. + /// Set mode 2 (PointInTime) to cleanly recover from this corruption. The + /// server will continue from the last good state, several seconds or + /// minutes prior to the crash. Clients may have to run "clear-cache & + /// reload" to account for the rollback. Upon success, you may reset the + /// mode back to default and restart again. Please note in some cases the + /// corruption error may not be cleared for at least 30 minutes of + /// operation in PointInTime mode. + /// + /// As a very last ditch effort, if PointInTime does not fix or resolve + /// anything, you can try mode 3 (SkipAnyCorruptedRecord) but this will + /// leave the server in a potentially inconsistent state. + /// + /// The default mode 1 (TolerateCorruptedTailRecords) will automatically + /// drop the last entry in the database if corrupted during shutdown, but + /// nothing more. It is extraordinarily unlikely this will desynchronize + /// clients. To disable any form of silent rollback set mode 0 + /// (AbsoluteConsistency). + /// + /// The options are: + /// 0 = AbsoluteConsistency + /// 1 = TolerateCorruptedTailRecords (default) + /// 2 = PointInTime (use me if trying to recover) + /// 3 = SkipAnyCorruptedRecord (you now voided your Conduwuit warranty) + /// + /// See https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes for more information + /// + /// Defaults to 1 (TolerateCorruptedTailRecords) #[serde(default = "default_rocksdb_recovery_mode")] pub rocksdb_recovery_mode: u8, + + /// Database repair mode (for RocksDB SST corruption) + /// + /// Use this option when the server reports corruption while running or + /// panics. If the server refuses to start use the recovery mode options + /// first. Corruption errors containing the acronym 'SST' which occur after + /// startup will likely require this option. + /// + /// - Backing up your database directory is recommended prior to running the + /// repair. + /// - Disabling repair mode and restarting the server is recommended after + /// running the repair. + /// + /// Defaults to false #[serde(default)] pub rocksdb_repair: bool, + #[serde(default)] pub rocksdb_read_only: bool, + #[serde(default)] pub rocksdb_secondary: bool, + + /// Enables idle CPU priority for compaction thread. This is not enabled by + /// default to prevent compaction from falling too far behind on busy + /// systems. #[serde(default)] pub rocksdb_compaction_prio_idle: bool, + + /// Enables idle IO priority for compaction thread. This prevents any + /// unexpected lag in the server's operation and is usually a good idea. + /// Enabled by default. #[serde(default = "true_fn")] pub rocksdb_compaction_ioprio_idle: bool, + #[serde(default = "true_fn")] pub rocksdb_compaction: bool, + + /// Level of statistics collection. Some admin commands to display database + /// statistics may require this option to be set. Database performance may + /// be impacted by higher settings. + /// + /// Option is a number ranging from 0 to 6: + /// 0 = No statistics. + /// 1 = No statistics in release mode (default). + /// 2 to 3 = Statistics with no performance impact. + /// 3 to 5 = Statistics with possible performance impact. + /// 6 = All statistics. + /// + /// Defaults to 1 (No statistics, except in debug-mode) #[serde(default = "default_rocksdb_stats_level")] pub rocksdb_stats_level: u8, @@ -254,128 +801,420 @@ pub struct Config { #[serde(default = "default_notification_push_path")] pub notification_push_path: String, + /// Config option to control local (your server only) presence + /// updates/requests. Defaults to true. Note that presence on conduwuit is + /// very fast unlike Synapse's. If using outgoing presence, this MUST be + /// enabled. #[serde(default = "true_fn")] pub allow_local_presence: bool, + + /// Config option to control incoming federated presence updates/requests. + /// Defaults to true. This option receives presence updates from other + /// servers, but does not send any unless `allow_outgoing_presence` is true. + /// Note that presence on conduwuit is very fast unlike Synapse's. #[serde(default = "true_fn")] pub allow_incoming_presence: bool, + + /// Config option to control outgoing presence updates/requests. Defaults to + /// true. This option sends presence updates to other servers, but does not + /// receive any unless `allow_incoming_presence` is true. + /// Note that presence on conduwuit is very fast unlike Synapse's. + /// If using outgoing presence, you MUST enable `allow_local_presence` as + /// well. #[serde(default = "true_fn")] pub allow_outgoing_presence: bool, + + /// Config option to control how many seconds before presence updates that + /// you are idle. Defaults to 5 minutes. #[serde(default = "default_presence_idle_timeout_s")] pub presence_idle_timeout_s: u64, + + /// Config option to control how many seconds before presence updates that + /// you are offline. Defaults to 30 minutes. #[serde(default = "default_presence_offline_timeout_s")] pub presence_offline_timeout_s: u64, + + /// Config option to enable the presence idle timer for remote users. + /// Disabling is offered as an optimization for servers participating in + /// many large rooms or when resources are limited. Disabling it may cause + /// incorrect presence states (i.e. stuck online) to be seen for some + /// remote users. Defaults to true. #[serde(default = "true_fn")] pub presence_timeout_remote_users: bool, + /// Config option to control whether we should receive remote incoming read + /// receipts. Defaults to true. #[serde(default = "true_fn")] pub allow_incoming_read_receipts: bool, + + /// Config option to control whether we should send read receipts to remote + /// servers. Defaults to true. #[serde(default = "true_fn")] pub allow_outgoing_read_receipts: bool, + /// Config option to control outgoing typing updates to federation. Defaults + /// to true. #[serde(default = "true_fn")] pub allow_outgoing_typing: bool, + + /// Config option to control incoming typing updates from federation. + /// Defaults to true. #[serde(default = "true_fn")] pub allow_incoming_typing: bool, + + /// Config option to control maximum time federation user can indicate + /// typing. #[serde(default = "default_typing_federation_timeout_s")] pub typing_federation_timeout_s: u64, + + /// Config option to control minimum time local client can indicate typing. + /// This does not override a client's request to stop typing. It only + /// enforces a minimum value in case of no stop request. #[serde(default = "default_typing_client_timeout_min_s")] pub typing_client_timeout_min_s: u64, + + /// Config option to control maximum time local client can indicate typing. #[serde(default = "default_typing_client_timeout_max_s")] pub typing_client_timeout_max_s: u64, + /// Set this to true for conduwuit to compress HTTP response bodies using + /// zstd. This option does nothing if conduwuit was not built with + /// `zstd_compression` feature. Please be aware that enabling HTTP + /// compression may weaken TLS. Most users should not need to enable this. + /// See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH + /// before deciding to enable this. #[serde(default)] pub zstd_compression: bool, + + /// Set this to true for conduwuit to compress HTTP response bodies using + /// gzip. This option does nothing if conduwuit was not built with + /// `gzip_compression` feature. Please be aware that enabling HTTP + /// compression may weaken TLS. Most users should not need to enable this. + /// See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before + /// deciding to enable this. #[serde(default)] pub gzip_compression: bool, + + /// Set this to true for conduwuit to compress HTTP response bodies using + /// brotli. This option does nothing if conduwuit was not built with + /// `brotli_compression` feature. Please be aware that enabling HTTP + /// compression may weaken TLS. Most users should not need to enable this. + /// See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before + /// deciding to enable this. #[serde(default)] pub brotli_compression: bool, + /// Set to true to allow user type "guest" registrations. Element attempts + /// to register guest users automatically. Defaults to false #[serde(default)] pub allow_guest_registration: bool, + + /// Set to true to log guest registrations in the admin room. + /// Defaults to false as it may be noisy or unnecessary. #[serde(default)] pub log_guest_registrations: bool, + + /// Set to true to allow guest registrations/users to auto join any rooms + /// specified in `auto_join_rooms` Defaults to false #[serde(default)] pub allow_guests_auto_join_rooms: bool, + /// Config option to control whether the legacy unauthenticated Matrix media + /// repository endpoints will be enabled. These endpoints consist of: + /// - /_matrix/media/*/config + /// - /_matrix/media/*/upload + /// - /_matrix/media/*/preview_url + /// - /_matrix/media/*/download/* + /// - /_matrix/media/*/thumbnail/* + /// + /// The authenticated equivalent endpoints are always enabled. + /// + /// Defaults to true for now, but this is highly subject to change, likely + /// in the next release. #[serde(default = "true_fn")] pub allow_legacy_media: bool, + #[serde(default = "true_fn")] pub freeze_legacy_media: bool, + + /// Checks consistency of the media directory at startup: + /// 1. When `media_compat_file_link` is enbled, this check will upgrade + /// media when switching back and forth between Conduit and Conduwuit. + /// Both options must be enabled to handle this. + /// 2. When media is deleted from the directory, this check will also delete + /// its database entry. + /// + /// If none of these checks apply to your use cases, and your media + /// directory is significantly large setting this to false may reduce + /// startup time. + /// + /// Enabled by default. #[serde(default = "true_fn")] pub media_startup_check: bool, + + /// Enable backward-compatibility with Conduit's media directory by creating + /// symlinks of media. This option is only necessary if you plan on using + /// Conduit again. Otherwise setting this to false reduces filesystem + /// clutter and overhead for managing these symlinks in the directory. This + /// is now disabled by default. You may still return to upstream Conduit + /// but you have to run Conduwuit at least once with this set to true and + /// allow the media_startup_check to take place before shutting + /// down to return to Conduit. + /// + /// Disabled by default. #[serde(default)] pub media_compat_file_link: bool, + + /// Prunes missing media from the database as part of the media startup + /// checks. This means if you delete files from the media directory the + /// corresponding entries will be removed from the database. This is + /// disabled by default because if the media directory is accidentally moved + /// or inaccessible the metadata entries in the database will be lost with + /// sadness. + /// + /// Disabled by default. #[serde(default)] pub prune_missing_media: bool, + + /// Vector list of servers that conduwuit will refuse to download remote + /// media from. No default. #[serde(default = "Vec::new")] pub prevent_media_downloads_from: Vec, + /// List of forbidden server names that we will block incoming AND outgoing + /// federation with, and block client room joins / remote user invites. + /// + /// This check is applied on the room ID, room alias, sender server name, + /// sender user's server name, inbound federation X-Matrix origin, and + /// outbound federation handler. + /// + /// Basically "global" ACLs. No default. #[serde(default = "Vec::new")] pub forbidden_remote_server_names: Vec, + + /// List of forbidden server names that we will block all outgoing federated + /// room directory requests for. Useful for preventing our users from + /// wandering into bad servers or spaces. No default. #[serde(default = "Vec::new")] pub forbidden_remote_room_directory_server_names: Vec, + /// Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you + /// do not want conduwuit to send outbound requests to. Defaults to + /// RFC1918, unroutable, loopback, multicast, and testnet addresses for + /// security. + /// + /// To disable, set this to be an empty vector (`[]`). + /// Please be aware that this is *not* a guarantee. You should be using a + /// firewall with zones as doing this on the application layer may have + /// bypasses. + /// + /// Currently this does not account for proxies in use like Synapse does. #[serde(default = "default_ip_range_denylist")] pub ip_range_denylist: Vec, + /// Vector list of domains allowed to send requests to for URL previews. + /// Defaults to none. Note: this is a *contains* match, not an explicit + /// match. Putting "google.com" will match "https://google.com" and + /// "http://mymaliciousdomainexamplegoogle.com" Setting this to "*" will + /// allow all URL previews. Please note that this opens up significant + /// attack surface to your server, you are expected to be aware of the + /// risks by doing so. #[serde(default = "Vec::new")] pub url_preview_domain_contains_allowlist: Vec, + + /// Vector list of explicit domains allowed to send requests to for URL + /// previews. Defaults to none. Note: This is an *explicit* match, not a + /// contains match. Putting "google.com" will match "https://google.com", + /// "http://google.com", but not + /// "https://mymaliciousdomainexamplegoogle.com". Setting this to "*" will + /// allow all URL previews. Please note that this opens up significant + /// attack surface to your server, you are expected to be aware of the + /// risks by doing so. #[serde(default = "Vec::new")] pub url_preview_domain_explicit_allowlist: Vec, + + /// Vector list of explicit domains not allowed to send requests to for URL + /// previews. Defaults to none. Note: This is an *explicit* match, not a + /// contains match. Putting "google.com" will match "https://google.com", + /// "http://google.com", but not + /// "https://mymaliciousdomainexamplegoogle.com". The denylist is checked + /// first before allowlist. Setting this to "*" will not do anything. #[serde(default = "Vec::new")] pub url_preview_domain_explicit_denylist: Vec, + + /// Vector list of URLs allowed to send requests to for URL previews. + /// Defaults to none. Note that this is a *contains* match, not an + /// explicit match. Putting "google.com" will match + /// "https://google.com/", + /// "https://google.com/url?q=https://mymaliciousdomainexample.com", and + /// "https://mymaliciousdomainexample.com/hi/google.com" Setting this to + /// "*" will allow all URL previews. Please note that this opens up + /// significant attack surface to your server, you are expected to be + /// aware of the risks by doing so. #[serde(default = "Vec::new")] pub url_preview_url_contains_allowlist: Vec, + + /// Maximum amount of bytes allowed in a URL preview body size when + /// spidering. Defaults to 384KB (384_000 bytes) #[serde(default = "default_url_preview_max_spider_size")] pub url_preview_max_spider_size: usize, + + /// Option to decide whether you would like to run the domain allowlist + /// checks (contains and explicit) on the root domain or not. Does not apply + /// to URL contains allowlist. Defaults to false. Example: If this is + /// enabled and you have "wikipedia.org" allowed in the explicit and/or + /// contains domain allowlist, it will allow all subdomains under + /// "wikipedia.org" such as "en.m.wikipedia.org" as the root domain is + /// checked and matched. Useful if the domain contains allowlist is still + /// too broad for you but you still want to allow all the subdomains under a + /// root domain. #[serde(default)] pub url_preview_check_root_domain: bool, + /// List of forbidden room aliases and room IDs as patterns/strings. Values + /// in this list are matched as *contains*. This is checked upon room alias + /// creation, custom room ID creation if used, and startup as warnings if + /// any room aliases in your database have a forbidden room alias/ID. + /// No default. #[serde(default = "RegexSet::empty")] #[serde(with = "serde_regex")] pub forbidden_alias_names: RegexSet, + /// List of forbidden username patterns/strings. Values in this list are + /// matched as *contains*. This is checked upon username availability + /// check, registration, and startup as warnings if any local users in your + /// database have a forbidden username. + /// No default. #[serde(default = "RegexSet::empty")] #[serde(with = "serde_regex")] pub forbidden_usernames: RegexSet, + /// Retry failed and incomplete messages to remote servers immediately upon + /// startup. This is called bursting. If this is disabled, said messages + /// may not be delivered until more messages are queued for that server. Do + /// not change this option unless server resources are extremely limited or + /// the scale of the server's deployment is huge. Do not disable this + /// unless you know what you are doing. #[serde(default = "true_fn")] pub startup_netburst: bool, + + /// messages are dropped and not reattempted. The `startup_netburst` option + /// must be enabled for this value to have any effect. Do not change this + /// value unless you know what you are doing. Set this value to -1 to + /// reattempt every message without trimming the queues; this may consume + /// significant disk. Set this value to 0 to drop all messages without any + /// attempt at redelivery. #[serde(default = "default_startup_netburst_keep")] pub startup_netburst_keep: i64, + /// controls whether non-admin local users are forbidden from sending room + /// invites (local and remote), and if non-admin users can receive remote + /// room invites. admins are always allowed to send and receive all room + /// invites. defaults to false #[serde(default)] pub block_non_admin_invites: bool, + + /// Allows admins to enter commands in rooms other than #admins by prefixing + /// with \!admin. The reply will be publicly visible to the room, + /// originating from the sender. defaults to true #[serde(default = "true_fn")] pub admin_escape_commands: bool, + + /// Controls whether the conduwuit admin room console / CLI will immediately + /// activate on startup. This option can also be enabled with `--console` + /// conduwuit argument + /// + /// Defaults to false #[serde(default)] pub admin_console_automatic: bool, + + /// Controls what admin commands will be executed on startup. This is a + /// vector list of strings of admin commands to run. + /// + /// An example of this can be: `admin_execute = ["debug ping puppygock.gay", + /// "debug echo hi"]` + /// + /// This option can also be configured with the `--execute` conduwuit + /// argument and can take standard shell commands and environment variables + /// + /// Such example could be: `./conduwuit --execute "server admin-notice + /// conduwuit has started up at $(date)"` + /// + /// Defaults to nothing. #[serde(default)] pub admin_execute: Vec, + + /// Controls whether conduwuit should error and fail to start if an admin + /// execute command (`--execute` / `admin_execute`) fails + /// + /// Defaults to false #[serde(default)] pub admin_execute_errors_ignore: bool, + + /// Controls the max log level for admin command log captures (logs + /// generated from running admin commands) + /// + /// Defaults to "info" on release builds, else "debug" on debug builds #[serde(default = "default_admin_log_capture")] pub admin_log_capture: String, + #[serde(default = "default_admin_room_tag")] pub admin_room_tag: String, + /// Sentry.io crash/panic reporting, performance monitoring/metrics, etc. + /// This is NOT enabled by default. conduwuit's default Sentry reporting + /// endpoint is o4506996327251968.ingest.us.sentry.io + /// + /// Defaults to *false* #[serde(default)] pub sentry: bool, + + /// Sentry reporting URL if a custom one is desired + /// + /// Defaults to conduwuit's default Sentry endpoint: + /// "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" #[serde(default = "default_sentry_endpoint")] pub sentry_endpoint: Option, + + /// Report your Conduwuit server_name in Sentry.io crash reports and metrics + /// + /// Defaults to false #[serde(default)] pub sentry_send_server_name: bool, + + /// Performance monitoring/tracing sample rate for Sentry.io + /// + /// Note that too high values may impact performance, and can be disabled by + /// setting it to 0.0 (0%) This value is read as a percentage to Sentry, + /// represented as a decimal + /// + /// Defaults to 15% of traces (0.15) #[serde(default = "default_sentry_traces_sample_rate")] pub sentry_traces_sample_rate: f32, + + /// Whether to attach a stacktrace to Sentry reports. #[serde(default)] pub sentry_attach_stacktrace: bool, + + /// Send panics to sentry. This is true by default, but sentry has to be + /// enabled. #[serde(default = "true_fn")] pub sentry_send_panic: bool, + + /// Send errors to sentry. This is true by default, but sentry has to be + /// enabled. This option is only effective in release-mode; forced to false + /// in debug-mode. #[serde(default = "true_fn")] pub sentry_send_error: bool, + + /// Controls the tracing log level for Sentry to send things like + /// breadcrumbs and transactions Defaults to "info" #[serde(default = "default_sentry_filter")] pub sentry_filter: String, + /// Enable the tokio-console. This option is only relevant to developers. + /// See: docs/development.md#debugging-with-tokio-console for more + /// information. #[serde(default)] pub tokio_console: bool, From 2f24d7117a4f493bb90ea20b2b780486ef40272c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Oct 2024 05:15:24 +0000 Subject: [PATCH 0907/2091] further develop serializer for insertions add JSON delegator to db serializer consolidate writes through memfun; simplifications Signed-off-by: Jason Volk --- src/database/mod.rs | 3 +- src/database/ser.rs | 260 +++++++++++++++++++++++------------------- src/database/tests.rs | 232 +++++++++++++++++++++++++++++++++++++ 3 files changed, 376 insertions(+), 119 deletions(-) create mode 100644 src/database/tests.rs diff --git a/src/database/mod.rs b/src/database/mod.rs index e66abf68..c39b2b2f 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -10,6 +10,7 @@ pub mod maps; mod opts; mod ser; mod stream; +mod tests; mod util; mod watchers; @@ -28,7 +29,7 @@ pub use self::{ handle::Handle, keyval::{KeyVal, Slice}, map::Map, - ser::{Interfix, Separator}, + ser::{serialize, serialize_to_array, serialize_to_vec, Interfix, Json, Separator}, }; conduit::mod_ctor! {} diff --git a/src/database/ser.rs b/src/database/ser.rs index bd4bbd9a..742f1e34 100644 --- a/src/database/ser.rs +++ b/src/database/ser.rs @@ -1,12 +1,24 @@ use std::io::Write; -use conduit::{err, result::DebugInspect, utils::exchange, Error, Result}; +use arrayvec::ArrayVec; +use conduit::{debug::type_name, err, result::DebugInspect, utils::exchange, Error, Result}; use serde::{ser, Serialize}; #[inline] -pub(crate) fn serialize_to_vec(val: &T) -> Result> +pub fn serialize_to_array(val: T) -> Result> where - T: Serialize + ?Sized, + T: Serialize, +{ + let mut buf = ArrayVec::::new(); + serialize(&mut buf, val)?; + + Ok(buf) +} + +#[inline] +pub fn serialize_to_vec(val: T) -> Result> +where + T: Serialize, { let mut buf = Vec::with_capacity(64); serialize(&mut buf, val)?; @@ -15,10 +27,10 @@ where } #[inline] -pub(crate) fn serialize<'a, W, T>(out: &'a mut W, val: &'a T) -> Result<&'a [u8]> +pub fn serialize<'a, W, T>(out: &'a mut W, val: T) -> Result<&'a [u8]> where - W: Write + AsRef<[u8]>, - T: Serialize + ?Sized, + W: Write + AsRef<[u8]> + 'a, + T: Serialize, { let mut serializer = Serializer { out, @@ -43,6 +55,10 @@ pub(crate) struct Serializer<'a, W: Write> { fin: bool, } +/// Newtype for JSON serialization. +#[derive(Debug, Serialize)] +pub struct Json(pub T); + /// Directive to force separator serialization specifically for prefix keying /// use. This is a quirk of the database schema and prefix iterations. #[derive(Debug, Serialize)] @@ -56,38 +72,43 @@ pub struct Separator; impl Serializer<'_, W> { const SEP: &'static [u8] = b"\xFF"; + fn tuple_start(&mut self) { + debug_assert!(!self.sep, "Tuple start with separator set"); + self.sequence_start(); + } + + fn tuple_end(&mut self) -> Result { + self.sequence_end()?; + Ok(()) + } + fn sequence_start(&mut self) { debug_assert!(!self.is_finalized(), "Sequence start with finalization set"); - debug_assert!(!self.sep, "Sequence start with separator set"); - if cfg!(debug_assertions) { - self.depth = self.depth.saturating_add(1); - } + cfg!(debug_assertions).then(|| self.depth = self.depth.saturating_add(1)); } - fn sequence_end(&mut self) { - self.sep = false; - if cfg!(debug_assertions) { - self.depth = self.depth.saturating_sub(1); - } + fn sequence_end(&mut self) -> Result { + cfg!(debug_assertions).then(|| self.depth = self.depth.saturating_sub(1)); + Ok(()) } - fn record_start(&mut self) -> Result<()> { + fn record_start(&mut self) -> Result { debug_assert!(!self.is_finalized(), "Starting a record after serialization finalized"); exchange(&mut self.sep, true) .then(|| self.separator()) .unwrap_or(Ok(())) } - fn separator(&mut self) -> Result<()> { + fn separator(&mut self) -> Result { debug_assert!(!self.is_finalized(), "Writing a separator after serialization finalized"); self.out.write_all(Self::SEP).map_err(Into::into) } + fn write(&mut self, buf: &[u8]) -> Result { self.out.write_all(buf).map_err(Into::into) } + fn set_finalized(&mut self) { debug_assert!(!self.is_finalized(), "Finalization already set"); - if cfg!(debug_assertions) { - self.fin = true; - } + cfg!(debug_assertions).then(|| self.fin = true); } fn is_finalized(&self) -> bool { self.fin } @@ -104,53 +125,65 @@ impl ser::Serializer for &mut Serializer<'_, W> { type SerializeTupleStruct = Self; type SerializeTupleVariant = Self; - fn serialize_map(self, _len: Option) -> Result { - unimplemented!("serialize Map not implemented") - } - fn serialize_seq(self, _len: Option) -> Result { self.sequence_start(); - self.record_start()?; Ok(self) } fn serialize_tuple(self, _len: usize) -> Result { - self.sequence_start(); + self.tuple_start(); Ok(self) } fn serialize_tuple_struct(self, _name: &'static str, _len: usize) -> Result { - self.sequence_start(); + self.tuple_start(); Ok(self) } fn serialize_tuple_variant( self, _name: &'static str, _idx: u32, _var: &'static str, _len: usize, ) -> Result { - self.sequence_start(); - Ok(self) + unimplemented!("serialize Tuple Variant not implemented") + } + + fn serialize_map(self, _len: Option) -> Result { + unimplemented!( + "serialize Map not implemented; did you mean to use database::Json() around your serde_json::Value?" + ) } fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { - self.sequence_start(); - Ok(self) + unimplemented!( + "serialize Struct not implemented at this time; did you mean to use database::Json() around your struct?" + ) } fn serialize_struct_variant( self, _name: &'static str, _idx: u32, _var: &'static str, _len: usize, ) -> Result { - self.sequence_start(); - Ok(self) + unimplemented!("serialize Struct Variant not implemented") } - fn serialize_newtype_struct(self, _name: &'static str, _value: &T) -> Result { - unimplemented!("serialize New Type Struct not implemented") + #[allow(clippy::needless_borrows_for_generic_args)] // buggy + fn serialize_newtype_struct(self, name: &'static str, value: &T) -> Result + where + T: Serialize + ?Sized, + { + debug_assert!( + name != "Json" || type_name::() != "alloc::boxed::Box", + "serializing a Json(RawValue); you can skip serialization instead" + ); + + match name { + "Json" => serde_json::to_writer(&mut self.out, value).map_err(Into::into), + _ => unimplemented!("Unrecognized serialization Newtype {name:?}"), + } } fn serialize_newtype_variant( self, _name: &'static str, _idx: u32, _var: &'static str, _value: &T, ) -> Result { - unimplemented!("serialize New Type Variant not implemented") + unimplemented!("serialize Newtype Variant not implemented") } fn serialize_unit_struct(self, name: &'static str) -> Result { @@ -180,35 +213,94 @@ impl ser::Serializer for &mut Serializer<'_, W> { self.serialize_str(v.encode_utf8(&mut buf)) } - fn serialize_str(self, v: &str) -> Result { self.serialize_bytes(v.as_bytes()) } + fn serialize_str(self, v: &str) -> Result { + debug_assert!( + self.depth > 0, + "serializing string at the top-level; you can skip serialization instead" + ); - fn serialize_bytes(self, v: &[u8]) -> Result { self.out.write_all(v).map_err(Error::Io) } + self.serialize_bytes(v.as_bytes()) + } + + fn serialize_bytes(self, v: &[u8]) -> Result { + debug_assert!( + self.depth > 0, + "serializing byte array at the top-level; you can skip serialization instead" + ); + + self.write(v) + } fn serialize_f64(self, _v: f64) -> Result { unimplemented!("serialize f64 not implemented") } fn serialize_f32(self, _v: f32) -> Result { unimplemented!("serialize f32 not implemented") } - fn serialize_i64(self, v: i64) -> Result { self.out.write_all(&v.to_be_bytes()).map_err(Error::Io) } + fn serialize_i64(self, v: i64) -> Result { self.write(&v.to_be_bytes()) } - fn serialize_i32(self, _v: i32) -> Result { unimplemented!("serialize i32 not implemented") } + fn serialize_i32(self, v: i32) -> Result { self.write(&v.to_be_bytes()) } fn serialize_i16(self, _v: i16) -> Result { unimplemented!("serialize i16 not implemented") } fn serialize_i8(self, _v: i8) -> Result { unimplemented!("serialize i8 not implemented") } - fn serialize_u64(self, v: u64) -> Result { self.out.write_all(&v.to_be_bytes()).map_err(Error::Io) } + fn serialize_u64(self, v: u64) -> Result { self.write(&v.to_be_bytes()) } - fn serialize_u32(self, _v: u32) -> Result { unimplemented!("serialize u32 not implemented") } + fn serialize_u32(self, v: u32) -> Result { self.write(&v.to_be_bytes()) } fn serialize_u16(self, _v: u16) -> Result { unimplemented!("serialize u16 not implemented") } - fn serialize_u8(self, v: u8) -> Result { self.out.write_all(&[v]).map_err(Error::Io) } + fn serialize_u8(self, v: u8) -> Result { self.write(&[v]) } fn serialize_bool(self, _v: bool) -> Result { unimplemented!("serialize bool not implemented") } fn serialize_unit(self) -> Result { unimplemented!("serialize unit not implemented") } } +impl ser::SerializeSeq for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_element(&mut self, val: &T) -> Result { val.serialize(&mut **self) } + + fn end(self) -> Result { self.sequence_end() } +} + +impl ser::SerializeTuple for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_element(&mut self, val: &T) -> Result { + self.record_start()?; + val.serialize(&mut **self) + } + + fn end(self) -> Result { self.tuple_end() } +} + +impl ser::SerializeTupleStruct for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_field(&mut self, val: &T) -> Result { + self.record_start()?; + val.serialize(&mut **self) + } + + fn end(self) -> Result { self.tuple_end() } +} + +impl ser::SerializeTupleVariant for &mut Serializer<'_, W> { + type Error = Error; + type Ok = (); + + fn serialize_field(&mut self, val: &T) -> Result { + self.record_start()?; + val.serialize(&mut **self) + } + + fn end(self) -> Result { self.tuple_end() } +} + impl ser::SerializeMap for &mut Serializer<'_, W> { type Error = Error; type Ok = (); @@ -221,95 +313,27 @@ impl ser::SerializeMap for &mut Serializer<'_, W> { unimplemented!("serialize Map Val not implemented") } - fn end(self) -> Result { - self.sequence_end(); - Ok(()) - } -} - -impl ser::SerializeSeq for &mut Serializer<'_, W> { - type Error = Error; - type Ok = (); - - fn serialize_element(&mut self, val: &T) -> Result { val.serialize(&mut **self) } - - fn end(self) -> Result { - self.sequence_end(); - Ok(()) - } + fn end(self) -> Result { unimplemented!("serialize Map End not implemented") } } impl ser::SerializeStruct for &mut Serializer<'_, W> { type Error = Error; type Ok = (); - fn serialize_field(&mut self, _key: &'static str, val: &T) -> Result { - self.record_start()?; - val.serialize(&mut **self) + fn serialize_field(&mut self, _key: &'static str, _val: &T) -> Result { + unimplemented!("serialize Struct Field not implemented") } - fn end(self) -> Result { - self.sequence_end(); - Ok(()) - } + fn end(self) -> Result { unimplemented!("serialize Struct End not implemented") } } impl ser::SerializeStructVariant for &mut Serializer<'_, W> { type Error = Error; type Ok = (); - fn serialize_field(&mut self, _key: &'static str, val: &T) -> Result { - self.record_start()?; - val.serialize(&mut **self) + fn serialize_field(&mut self, _key: &'static str, _val: &T) -> Result { + unimplemented!("serialize Struct Variant Field not implemented") } - fn end(self) -> Result { - self.sequence_end(); - Ok(()) - } -} - -impl ser::SerializeTuple for &mut Serializer<'_, W> { - type Error = Error; - type Ok = (); - - fn serialize_element(&mut self, val: &T) -> Result { - self.record_start()?; - val.serialize(&mut **self) - } - - fn end(self) -> Result { - self.sequence_end(); - Ok(()) - } -} - -impl ser::SerializeTupleStruct for &mut Serializer<'_, W> { - type Error = Error; - type Ok = (); - - fn serialize_field(&mut self, val: &T) -> Result { - self.record_start()?; - val.serialize(&mut **self) - } - - fn end(self) -> Result { - self.sequence_end(); - Ok(()) - } -} - -impl ser::SerializeTupleVariant for &mut Serializer<'_, W> { - type Error = Error; - type Ok = (); - - fn serialize_field(&mut self, val: &T) -> Result { - self.record_start()?; - val.serialize(&mut **self) - } - - fn end(self) -> Result { - self.sequence_end(); - Ok(()) - } + fn end(self) -> Result { unimplemented!("serialize Struct Variant End not implemented") } } diff --git a/src/database/tests.rs b/src/database/tests.rs new file mode 100644 index 00000000..47dfb32c --- /dev/null +++ b/src/database/tests.rs @@ -0,0 +1,232 @@ +#![cfg(test)] +#![allow(clippy::needless_borrows_for_generic_args)] + +use std::fmt::Debug; + +use arrayvec::ArrayVec; +use conduit::ruma::{serde::Raw, RoomId, UserId}; +use serde::Serialize; + +use crate::{ + de, ser, + ser::{serialize_to_vec, Json}, + Interfix, +}; + +#[test] +#[should_panic(expected = "serializing string at the top-level")] +fn ser_str() { + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + let s = serialize_to_vec(&user_id).expect("failed to serialize user_id"); + assert_eq!(&s, user_id.as_bytes()); +} + +#[test] +fn ser_tuple() { + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + + let mut a = user_id.as_bytes().to_vec(); + a.push(0xFF); + a.extend_from_slice(room_id.as_bytes()); + + let b = (user_id, room_id); + let b = serialize_to_vec(&b).expect("failed to serialize tuple"); + + assert_eq!(a, b); +} + +#[test] +#[should_panic(expected = "I/O error: failed to write whole buffer")] +fn ser_overflow() { + const BUFSIZE: usize = 10; + + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + + assert!(BUFSIZE < user_id.as_str().len() + room_id.as_str().len()); + let mut buf = ArrayVec::::new(); + + let val = (user_id, room_id); + _ = ser::serialize(&mut buf, val).unwrap(); +} + +#[test] +fn ser_complex() { + use conduit::ruma::Mxc; + + #[derive(Debug, Serialize)] + struct Dim { + width: u32, + height: u32, + } + + let mxc = Mxc { + server_name: "example.com".try_into().unwrap(), + media_id: "AbCdEfGhIjK", + }; + + let dim = Dim { + width: 123, + height: 456, + }; + + let mut a = Vec::new(); + a.extend_from_slice(b"mxc://"); + a.extend_from_slice(mxc.server_name.as_bytes()); + a.extend_from_slice(b"/"); + a.extend_from_slice(mxc.media_id.as_bytes()); + a.push(0xFF); + a.extend_from_slice(&dim.width.to_be_bytes()); + a.extend_from_slice(&dim.height.to_be_bytes()); + a.push(0xFF); + + let d: &[u32] = &[dim.width, dim.height]; + let b = (mxc, d, Interfix); + let b = serialize_to_vec(b).expect("failed to serialize complex"); + + assert_eq!(a, b); +} + +#[test] +fn ser_json() { + use conduit::ruma::api::client::filter::FilterDefinition; + + let filter = FilterDefinition { + event_fields: Some(vec!["content.body".to_owned()]), + ..Default::default() + }; + + let serialized = serialize_to_vec(Json(&filter)).expect("failed to serialize value"); + + let s = String::from_utf8_lossy(&serialized); + assert_eq!(&s, r#"{"event_fields":["content.body"]}"#); +} + +#[test] +fn ser_json_value() { + use conduit::ruma::api::client::filter::FilterDefinition; + + let filter = FilterDefinition { + event_fields: Some(vec!["content.body".to_owned()]), + ..Default::default() + }; + + let value = serde_json::to_value(filter).expect("failed to serialize to serde_json::value"); + let serialized = serialize_to_vec(Json(value)).expect("failed to serialize value"); + + let s = String::from_utf8_lossy(&serialized); + assert_eq!(&s, r#"{"event_fields":["content.body"]}"#); +} + +#[test] +fn ser_json_macro() { + use serde_json::json; + + #[derive(Serialize)] + struct Foo { + foo: String, + } + + let content = Foo { + foo: "bar".to_owned(), + }; + let content = serde_json::to_value(content).expect("failed to serialize content"); + let sender: &UserId = "@foo:example.com".try_into().unwrap(); + let serialized = serialize_to_vec(Json(json!({ + "sender": sender, + "content": content, + }))) + .expect("failed to serialize value"); + + let s = String::from_utf8_lossy(&serialized); + assert_eq!(&s, r#"{"content":{"foo":"bar"},"sender":"@foo:example.com"}"#); +} + +#[test] +#[should_panic(expected = "serializing string at the top-level")] +fn ser_json_raw() { + use conduit::ruma::api::client::filter::FilterDefinition; + + let filter = FilterDefinition { + event_fields: Some(vec!["content.body".to_owned()]), + ..Default::default() + }; + + let value = serde_json::value::to_raw_value(&filter).expect("failed to serialize to raw value"); + let a = serialize_to_vec(value.get()).expect("failed to serialize raw value"); + let s = String::from_utf8_lossy(&a); + assert_eq!(&s, r#"{"event_fields":["content.body"]}"#); +} + +#[test] +#[should_panic(expected = "you can skip serialization instead")] +fn ser_json_raw_json() { + use conduit::ruma::api::client::filter::FilterDefinition; + + let filter = FilterDefinition { + event_fields: Some(vec!["content.body".to_owned()]), + ..Default::default() + }; + + let value = serde_json::value::to_raw_value(&filter).expect("failed to serialize to raw value"); + let a = serialize_to_vec(Json(value)).expect("failed to serialize json value"); + let s = String::from_utf8_lossy(&a); + assert_eq!(&s, r#"{"event_fields":["content.body"]}"#); +} + +#[test] +fn de_tuple() { + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + + let raw: &[u8] = b"@user:example.com\xFF!room:example.com"; + let (a, b): (&UserId, &RoomId) = de::from_slice(raw).expect("failed to deserialize"); + + assert_eq!(a, user_id, "deserialized user_id does not match"); + assert_eq!(b, room_id, "deserialized room_id does not match"); +} + +#[test] +fn de_json_array() { + let a = &["foo", "bar", "baz"]; + let s = serde_json::to_vec(a).expect("failed to serialize to JSON array"); + + let b: Raw>> = de::from_slice(&s).expect("failed to deserialize"); + + let d: Vec = serde_json::from_str(b.json().get()).expect("failed to deserialize JSON"); + + for (i, a) in a.iter().enumerate() { + assert_eq!(*a, d[i]); + } +} + +#[test] +fn de_json_raw_array() { + let a = &["foo", "bar", "baz"]; + let s = serde_json::to_vec(a).expect("failed to serialize to JSON array"); + + let b: Raw>> = de::from_slice(&s).expect("failed to deserialize"); + + let c: Vec> = serde_json::from_str(b.json().get()).expect("failed to deserialize JSON"); + + for (i, a) in a.iter().enumerate() { + let c = serde_json::to_value(c[i].json()).expect("failed to deserialize JSON to string"); + assert_eq!(*a, c); + } +} + +#[test] +fn ser_array() { + let a: u64 = 123_456; + let b: u64 = 987_654; + + let arr: &[u64] = &[a, b]; + + let mut v = Vec::new(); + v.extend_from_slice(&a.to_be_bytes()); + v.extend_from_slice(&b.to_be_bytes()); + + let s = serialize_to_vec(arr).expect("failed to serialize"); + assert_eq!(&s, &v, "serialization does not match"); +} From d3d11356ee59858dcf26fa66ad2b6c9c4ac13a61 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Oct 2024 22:15:19 +0000 Subject: [PATCH 0908/2091] add serialized insert interface Signed-off-by: Jason Volk --- src/database/map/insert.rs | 187 +++++++++++++++++++++++++++++++++++-- src/database/map/remove.rs | 12 +-- 2 files changed, 186 insertions(+), 13 deletions(-) diff --git a/src/database/map/insert.rs b/src/database/map/insert.rs index 953c9c94..39a0c422 100644 --- a/src/database/map/insert.rs +++ b/src/database/map/insert.rs @@ -1,21 +1,194 @@ -use std::{convert::AsRef, fmt::Debug}; +//! Insert a Key+Value into the database. +//! +//! Overloads are provided for the user to choose the most efficient +//! serialization or bypass for pre=serialized (raw) inputs. +use std::{convert::AsRef, fmt::Debug, io::Write}; + +use arrayvec::ArrayVec; use conduit::implement; use rocksdb::WriteBatchWithTransaction; +use serde::Serialize; -use crate::util::or_else; +use crate::{ser, util::or_else}; +/// Insert Key/Value +/// +/// - Key is serialized +/// - Val is serialized #[implement(super::Map)] -#[tracing::instrument(skip(self, value), fields(%self), level = "trace")] -pub fn insert(&self, key: &K, value: &V) +pub fn put(&self, key: K, val: V) where - K: AsRef<[u8]> + ?Sized + Debug, - V: AsRef<[u8]> + ?Sized, + K: Serialize + Debug, + V: Serialize, +{ + let mut key_buf = Vec::new(); + let mut val_buf = Vec::new(); + self.bput(key, val, (&mut key_buf, &mut val_buf)); +} + +/// Insert Key/Value +/// +/// - Key is serialized +/// - Val is raw +#[implement(super::Map)] +pub fn put_raw(&self, key: K, val: V) +where + K: Serialize + Debug, + V: AsRef<[u8]>, +{ + let mut key_buf = Vec::new(); + self.bput_raw(key, val, &mut key_buf); +} + +/// Insert Key/Value +/// +/// - Key is raw +/// - Val is serialized +#[implement(super::Map)] +pub fn raw_put(&self, key: K, val: V) +where + K: AsRef<[u8]>, + V: Serialize, +{ + let mut val_buf = Vec::new(); + self.raw_bput(key, val, &mut val_buf); +} + +/// Insert Key/Value +/// +/// - Key is serialized +/// - Val is serialized to stack-buffer +#[implement(super::Map)] +pub fn put_aput(&self, key: K, val: V) +where + K: Serialize + Debug, + V: Serialize, +{ + let mut key_buf = Vec::new(); + let mut val_buf = ArrayVec::::new(); + self.bput(key, val, (&mut key_buf, &mut val_buf)); +} + +/// Insert Key/Value +/// +/// - Key is serialized to stack-buffer +/// - Val is serialized +#[implement(super::Map)] +pub fn aput_put(&self, key: K, val: V) +where + K: Serialize + Debug, + V: Serialize, +{ + let mut key_buf = ArrayVec::::new(); + let mut val_buf = Vec::new(); + self.bput(key, val, (&mut key_buf, &mut val_buf)); +} + +/// Insert Key/Value +/// +/// - Key is serialized to stack-buffer +/// - Val is serialized to stack-buffer +#[implement(super::Map)] +pub fn aput(&self, key: K, val: V) +where + K: Serialize + Debug, + V: Serialize, +{ + let mut key_buf = ArrayVec::::new(); + let mut val_buf = ArrayVec::::new(); + self.bput(key, val, (&mut key_buf, &mut val_buf)); +} + +/// Insert Key/Value +/// +/// - Key is serialized to stack-buffer +/// - Val is raw +#[implement(super::Map)] +pub fn aput_raw(&self, key: K, val: V) +where + K: Serialize + Debug, + V: AsRef<[u8]>, +{ + let mut key_buf = ArrayVec::::new(); + self.bput_raw(key, val, &mut key_buf); +} + +/// Insert Key/Value +/// +/// - Key is raw +/// - Val is serialized to stack-buffer +#[implement(super::Map)] +pub fn raw_aput(&self, key: K, val: V) +where + K: AsRef<[u8]>, + V: Serialize, +{ + let mut val_buf = ArrayVec::::new(); + self.raw_bput(key, val, &mut val_buf); +} + +/// Insert Key/Value +/// +/// - Key is serialized to supplied buffer +/// - Val is serialized to supplied buffer +#[implement(super::Map)] +pub fn bput(&self, key: K, val: V, mut buf: (Bk, Bv)) +where + K: Serialize + Debug, + V: Serialize, + Bk: Write + AsRef<[u8]>, + Bv: Write + AsRef<[u8]>, +{ + let val = ser::serialize(&mut buf.1, val).expect("failed to serialize insertion val"); + self.bput_raw(key, val, &mut buf.0); +} + +/// Insert Key/Value +/// +/// - Key is serialized to supplied buffer +/// - Val is raw +#[implement(super::Map)] +pub fn bput_raw(&self, key: K, val: V, mut buf: Bk) +where + K: Serialize + Debug, + V: AsRef<[u8]>, + Bk: Write + AsRef<[u8]>, +{ + let key = ser::serialize(&mut buf, key).expect("failed to serialize insertion key"); + self.insert(&key, val); +} + +/// Insert Key/Value +/// +/// - Key is raw +/// - Val is serialized to supplied buffer +#[implement(super::Map)] +pub fn raw_bput(&self, key: K, val: V, mut buf: Bv) +where + K: AsRef<[u8]>, + V: Serialize, + Bv: Write + AsRef<[u8]>, +{ + let val = ser::serialize(&mut buf, val).expect("failed to serialize insertion val"); + self.insert(&key, val); +} + +/// Insert Key/Value +/// +/// - Key is raw +/// - Val is raw +#[implement(super::Map)] +#[tracing::instrument(skip_all, fields(%self), level = "trace")] +pub fn insert(&self, key: &K, val: V) +where + K: AsRef<[u8]> + ?Sized, + V: AsRef<[u8]>, { let write_options = &self.write_options; self.db .db - .put_cf_opt(&self.cf(), key, value, write_options) + .put_cf_opt(&self.cf(), key, val, write_options) .or_else(or_else) .expect("database insert error"); diff --git a/src/database/map/remove.rs b/src/database/map/remove.rs index 10bb2ff0..42eaa477 100644 --- a/src/database/map/remove.rs +++ b/src/database/map/remove.rs @@ -7,18 +7,18 @@ use serde::Serialize; use crate::{ser, util::or_else}; #[implement(super::Map)] -pub fn del(&self, key: &K) +pub fn del(&self, key: K) where - K: Serialize + ?Sized + Debug, + K: Serialize + Debug, { let mut buf = Vec::::with_capacity(64); self.bdel(key, &mut buf); } #[implement(super::Map)] -pub fn adel(&self, key: &K) +pub fn adel(&self, key: K) where - K: Serialize + ?Sized + Debug, + K: Serialize + Debug, { let mut buf = ArrayVec::::new(); self.bdel(key, &mut buf); @@ -26,9 +26,9 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] -pub fn bdel(&self, key: &K, buf: &mut B) +pub fn bdel(&self, key: K, buf: &mut B) where - K: Serialize + ?Sized + Debug, + K: Serialize + Debug, B: Write + AsRef<[u8]>, { let key = ser::serialize(buf, key).expect("failed to serialize deletion key"); From 19880ce12bf3bf79bcfa8cb21223de48ab268686 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 9 Oct 2024 19:41:52 +0000 Subject: [PATCH 0909/2091] add IgnoreAll directive to deserializer Signed-off-by: Jason Volk --- src/database/de.rs | 27 +++++++++++++++++++++++++-- src/database/mod.rs | 2 +- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/database/de.rs b/src/database/de.rs index 9ee52267..e5fdf7cb 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -5,6 +5,7 @@ use serde::{ Deserialize, }; +/// Deserialize into T from buffer. pub(crate) fn from_slice<'a, T>(buf: &'a [u8]) -> Result where T: Deserialize<'a>, @@ -22,6 +23,7 @@ where }) } +/// Deserialization state. pub(crate) struct Deserializer<'de> { buf: &'de [u8], pos: usize, @@ -33,6 +35,11 @@ pub(crate) struct Deserializer<'de> { #[derive(Debug, Deserialize)] pub struct Ignore; +/// Directive to ignore all remaining records. This can be used in a sequence to +/// ignore the rest of the sequence. +#[derive(Debug, Deserialize)] +pub struct IgnoreAll; + impl<'de> Deserializer<'de> { /// Record separator; an intentionally invalid-utf8 byte. const SEP: u8 = b'\xFF'; @@ -53,6 +60,13 @@ impl<'de> Deserializer<'de> { ))) } + /// Called at the start of arrays and tuples + #[inline] + fn sequence_start(&mut self) { + debug_assert!(!self.seq, "Nested sequences are not handled at this time"); + self.seq = true; + } + /// Consume the current record to ignore it. Inside a sequence the next /// record is skipped but at the top-level all records are skipped such that /// deserialization completes with self.finished() == Ok. @@ -61,10 +75,16 @@ impl<'de> Deserializer<'de> { if self.seq { self.record_next(); } else { - self.record_trail(); + self.record_ignore_all(); } } + /// Consume the current and all remaining records to ignore them. Similar to + /// Ignore at the top-level, but it can be provided in a sequence to Ignore + /// all remaining elements. + #[inline] + fn record_ignore_all(&mut self) { self.record_trail(); } + /// Consume the current record. The position pointer is moved to the start /// of the next record. Slice of the current record is returned. #[inline] @@ -101,7 +121,6 @@ impl<'de> Deserializer<'de> { ); self.inc_pos(started.into()); - self.seq = true; } /// Consume all remaining bytes, which may include record separators, @@ -128,6 +147,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { where V: Visitor<'de>, { + self.sequence_start(); visitor.visit_seq(self) } @@ -135,6 +155,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { where V: Visitor<'de>, { + self.sequence_start(); visitor.visit_seq(self) } @@ -142,6 +163,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { where V: Visitor<'de>, { + self.sequence_start(); visitor.visit_seq(self) } @@ -170,6 +192,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { { match name { "Ignore" => self.record_ignore(), + "IgnoreAll" => self.record_ignore_all(), _ => unimplemented!("Unrecognized deserialization Directive {name:?}"), }; diff --git a/src/database/mod.rs b/src/database/mod.rs index c39b2b2f..6d3b2079 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -24,7 +24,7 @@ extern crate rust_rocksdb as rocksdb; pub use self::{ database::Database, - de::Ignore, + de::{Ignore, IgnoreAll}, deserialized::Deserialized, handle::Handle, keyval::{KeyVal, Slice}, From 8258d16a94855dae3df68ae8dcdea1bde0601f4e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 9 Oct 2024 05:08:22 +0000 Subject: [PATCH 0910/2091] re-scheme naming of stream iterator overloads Signed-off-by: Jason Volk --- src/database/map/count.rs | 8 +++----- src/database/map/keys_from.rs | 6 +++--- src/database/map/keys_prefix.rs | 6 +++--- src/database/map/rev_keys_from.rs | 6 +++--- src/database/map/rev_keys_prefix.rs | 6 +++--- src/database/map/rev_stream_from.rs | 6 +++--- src/database/map/rev_stream_prefix.rs | 6 +++--- src/database/map/stream_from.rs | 6 +++--- src/database/map/stream_prefix.rs | 6 +++--- src/service/key_backups/mod.rs | 10 +++++----- src/service/rooms/alias/mod.rs | 2 +- src/service/rooms/lazy_loading/mod.rs | 2 +- src/service/rooms/metadata/mod.rs | 2 +- src/service/rooms/read_receipt/data.rs | 2 +- src/service/rooms/state/data.rs | 2 +- src/service/rooms/state_cache/mod.rs | 4 ++-- src/service/sending/data.rs | 4 ++-- src/service/users/mod.rs | 2 +- 18 files changed, 42 insertions(+), 44 deletions(-) diff --git a/src/database/map/count.rs b/src/database/map/count.rs index 4356b71f..dab45b7a 100644 --- a/src/database/map/count.rs +++ b/src/database/map/count.rs @@ -4,12 +4,10 @@ use conduit::implement; use futures::stream::StreamExt; use serde::Serialize; -use crate::de::Ignore; - /// Count the total number of entries in the map. #[implement(super::Map)] #[inline] -pub fn count(&self) -> impl Future + Send + '_ { self.keys::().count() } +pub fn count(&self) -> impl Future + Send + '_ { self.raw_keys().count() } /// Count the number of entries in the map starting from a lower-bound. /// @@ -20,7 +18,7 @@ pub fn count_from<'a, P>(&'a self, from: &P) -> impl Future + Se where P: Serialize + ?Sized + Debug + 'a, { - self.keys_from::(from).count() + self.keys_from_raw(from).count() } /// Count the number of entries in the map matching a prefix. @@ -32,5 +30,5 @@ pub fn count_prefix<'a, P>(&'a self, prefix: &P) -> impl Future where P: Serialize + ?Sized + Debug + 'a, { - self.keys_prefix::(prefix).count() + self.keys_prefix_raw(prefix).count() } diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 1993750a..4eb3b12e 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -13,13 +13,13 @@ where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, { - self.keys_raw_from(from) + self.keys_from_raw(from) .map(keyval::result_deserialize_key::) } #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn keys_raw_from

    (&self, from: &P) -> impl Stream>> + Send +pub fn keys_from_raw

    (&self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -29,7 +29,7 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn keys_from_raw<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn keys_raw_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index d6c0927b..0ff755f3 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -17,13 +17,13 @@ where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, { - self.keys_raw_prefix(prefix) + self.keys_prefix_raw(prefix) .map(keyval::result_deserialize_key::) } #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn keys_raw_prefix

    (&self, prefix: &P) -> impl Stream>> + Send +pub fn keys_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -34,7 +34,7 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn keys_prefix_raw<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn keys_raw_prefix<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, K: Deserialize<'a> + Send + 'a, diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index e012e60a..b142718c 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -13,13 +13,13 @@ where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, { - self.rev_keys_raw_from(from) + self.rev_keys_from_raw(from) .map(keyval::result_deserialize_key::) } #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_keys_raw_from

    (&self, from: &P) -> impl Stream>> + Send +pub fn rev_keys_from_raw

    (&self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -29,7 +29,7 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_keys_from_raw<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn rev_keys_raw_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index 162c4f9b..5297cecf 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -17,13 +17,13 @@ where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, { - self.rev_keys_raw_prefix(prefix) + self.rev_keys_prefix_raw(prefix) .map(keyval::result_deserialize_key::) } #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_keys_raw_prefix

    (&self, prefix: &P) -> impl Stream>> + Send +pub fn rev_keys_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -34,7 +34,7 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_keys_prefix_raw<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn rev_keys_raw_prefix<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, K: Deserialize<'a> + Send + 'a, diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index c48f406b..78318a7f 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -18,7 +18,7 @@ where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, { - self.rev_stream_raw_from(from) + self.rev_stream_from_raw(from) .map(keyval::result_deserialize::) } @@ -28,7 +28,7 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_stream_raw_from

    (&self, from: &P) -> impl Stream>> + Send +pub fn rev_stream_from_raw

    (&self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -42,7 +42,7 @@ where /// - Result is deserialized #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_stream_from_raw<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn rev_stream_raw_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index 9ef89e9c..601c3298 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -22,7 +22,7 @@ where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, { - self.rev_stream_raw_prefix(prefix) + self.rev_stream_prefix_raw(prefix) .map(keyval::result_deserialize::) } @@ -32,7 +32,7 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_stream_raw_prefix

    (&self, prefix: &P) -> impl Stream>> + Send +pub fn rev_stream_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -47,7 +47,7 @@ where /// - Result is deserialized #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_stream_prefix_raw<'a, K, V, P>( +pub fn rev_stream_raw_prefix<'a, K, V, P>( &'a self, prefix: &'a P, ) -> impl Stream>> + Send + 'a where diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index db828125..0d3bb1e1 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -18,7 +18,7 @@ where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, { - self.stream_raw_from(from) + self.stream_from_raw(from) .map(keyval::result_deserialize::) } @@ -28,7 +28,7 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn stream_raw_from

    (&self, from: &P) -> impl Stream>> + Send +pub fn stream_from_raw

    (&self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -42,7 +42,7 @@ where /// - Result is deserialized #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn stream_from_raw<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn stream_raw_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index 56154a8b..cab3dd09 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -22,7 +22,7 @@ where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, { - self.stream_raw_prefix(prefix) + self.stream_prefix_raw(prefix) .map(keyval::result_deserialize::) } @@ -32,7 +32,7 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn stream_raw_prefix

    (&self, prefix: &P) -> impl Stream>> + Send +pub fn stream_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -47,7 +47,7 @@ where /// - Result is deserialized #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn stream_prefix_raw<'a, K, V, P>( +pub fn stream_raw_prefix<'a, K, V, P>( &'a self, prefix: &'a P, ) -> impl Stream>> + Send + 'a where diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 55263eeb..4c303757 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -79,7 +79,7 @@ pub async fn delete_backup(&self, user_id: &UserId, version: &str) { let key = (user_id, version, Interfix); self.db .backupkeyid_backup - .keys_raw_prefix(&key) + .keys_prefix_raw(&key) .ignore_err() .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) .await; @@ -181,7 +181,7 @@ pub async fn count_keys(&self, user_id: &UserId, version: &str) -> usize { let prefix = (user_id, version); self.db .backupkeyid_backup - .keys_raw_prefix(&prefix) + .keys_prefix_raw(&prefix) .count() .await } @@ -256,7 +256,7 @@ pub async fn delete_all_keys(&self, user_id: &UserId, version: &str) { let key = (user_id, version, Interfix); self.db .backupkeyid_backup - .keys_raw_prefix(&key) + .keys_prefix_raw(&key) .ignore_err() .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) .await; @@ -267,7 +267,7 @@ pub async fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: & let key = (user_id, version, room_id, Interfix); self.db .backupkeyid_backup - .keys_raw_prefix(&key) + .keys_prefix_raw(&key) .ignore_err() .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) .await; @@ -278,7 +278,7 @@ pub async fn delete_room_key(&self, user_id: &UserId, version: &str, room_id: &R let key = (user_id, version, room_id, session_id); self.db .backupkeyid_backup - .keys_raw_prefix(&key) + .keys_prefix_raw(&key) .ignore_err() .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) .await; diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 7fac6be6..3f944729 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -101,7 +101,7 @@ impl Service { let prefix = (&room_id, Interfix); self.db .aliasid_alias - .keys_raw_prefix(&prefix) + .keys_prefix_raw(&prefix) .ignore_err() .ready_for_each(|key| self.db.aliasid_alias.remove(key)) .await; diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index e0816d3f..9493dcc4 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -99,7 +99,7 @@ pub async fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room let prefix = (user_id, device_id, room_id, Interfix); self.db .lazyloadedids - .keys_raw_prefix(&prefix) + .keys_prefix_raw(&prefix) .ignore_err() .ready_for_each(|key| self.db.lazyloadedids.remove(key)) .await; diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index d8be6aab..8367eb72 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -50,7 +50,7 @@ pub async fn exists(&self, room_id: &RoomId) -> bool { // Look for PDUs in that room. self.db .pduid_pdu - .keys_raw_prefix(&prefix) + .keys_prefix_raw(&prefix) .ignore_err() .next() .await diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index a2c0fabc..74b649ef 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -84,7 +84,7 @@ impl Data { let prefix2 = prefix.clone(); self.readreceiptid_readreceipt - .stream_raw_from(&first_possible_edu) + .stream_from_raw(&first_possible_edu) .ignore_err() .ready_take_while(move |(k, _)| k.starts_with(&prefix2)) .map(move |(k, v)| { diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index 3072e3c6..7265038f 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -52,7 +52,7 @@ impl Data { ) { let prefix = (room_id, Interfix); self.roomid_pduleaves - .keys_raw_prefix(&prefix) + .keys_prefix_raw(&prefix) .ignore_err() .ready_for_each(|key| self.roomid_pduleaves.remove(key)) .await; diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 8539c940..edfae529 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -408,7 +408,7 @@ impl Service { pub fn rooms_joined<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { self.db .userroomid_joined - .keys_prefix_raw(user_id) + .keys_raw_prefix(user_id) .ignore_err() .map(|(_, room_id): (Ignore, &RoomId)| room_id) } @@ -469,7 +469,7 @@ impl Service { self.db .roomid_inviteviaservers - .stream_prefix_raw(room_id) + .stream_raw_prefix(room_id) .ignore_err() .map(|(_, servers): KeyVal<'_>| *servers.last().expect("at least one server")) } diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 6f4b5b97..96d4a6a9 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -101,7 +101,7 @@ impl Data { pub fn active_requests_for(&self, destination: &Destination) -> impl Stream + Send + '_ { let prefix = destination.get_prefix(); self.servercurrentevent_data - .stream_raw_prefix(&prefix) + .stream_prefix_raw(&prefix) .ignore_err() .map(|(key, val)| { let (_, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); @@ -136,7 +136,7 @@ impl Data { pub fn queued_requests(&self, destination: &Destination) -> impl Stream + Send + '_ { let prefix = destination.get_prefix(); self.servernameevent_data - .stream_raw_prefix(&prefix) + .stream_prefix_raw(&prefix) .ignore_err() .map(|(key, val)| { let (_, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 71a93666..a99a7df4 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -311,7 +311,7 @@ impl Service { let prefix = (user_id, device_id, Interfix); self.db .todeviceid_events - .keys_raw_prefix(&prefix) + .keys_prefix_raw(&prefix) .ignore_err() .ready_for_each(|key| self.db.todeviceid_events.remove(key)) .await; From 2ed0c267eb698c33befc4daa482811f0ae45707a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 7 Oct 2024 17:54:27 +0000 Subject: [PATCH 0911/2091] Refactor for structured insertions Signed-off-by: Jason Volk --- Cargo.lock | 26 +-- Cargo.toml | 2 +- src/api/client/sync.rs | 5 +- src/api/server/invite.rs | 6 - src/service/account_data/mod.rs | 40 +--- src/service/globals/data.rs | 32 ++- src/service/globals/migrations.rs | 17 +- src/service/key_backups/mod.rs | 62 ++---- src/service/media/data.rs | 103 +++------ src/service/media/migrations.rs | 2 +- src/service/presence/data.rs | 18 +- src/service/presence/presence.rs | 4 - src/service/pusher/mod.rs | 16 +- src/service/rooms/directory/mod.rs | 4 +- src/service/rooms/lazy_loading/mod.rs | 12 +- src/service/rooms/metadata/mod.rs | 8 +- src/service/rooms/outlier/mod.rs | 7 +- src/service/rooms/pdu_metadata/data.rs | 16 +- src/service/rooms/read_receipt/data.rs | 40 +--- src/service/rooms/short/mod.rs | 47 ++-- src/service/rooms/state/data.rs | 13 +- src/service/rooms/state_accessor/mod.rs | 6 + src/service/rooms/state_cache/data.rs | 67 +++--- src/service/rooms/state_cache/mod.rs | 89 ++++---- src/service/rooms/timeline/data.rs | 18 +- src/service/rooms/user/data.rs | 26 +-- src/service/sending/data.rs | 3 +- src/service/sending/dest.rs | 2 +- src/service/uiaa/mod.rs | 17 +- src/service/updates/mod.rs | 5 +- src/service/users/mod.rs | 272 +++++++++--------------- 31 files changed, 364 insertions(+), 621 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cae6994c..db1394ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2976,7 +2976,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "assign", "js_int", @@ -2998,7 +2998,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "js_int", "ruma-common", @@ -3010,7 +3010,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "as_variant", "assign", @@ -3033,7 +3033,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "as_variant", "base64 0.22.1", @@ -3063,7 +3063,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3087,7 +3087,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "bytes", "http", @@ -3105,7 +3105,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "js_int", "thiserror", @@ -3114,7 +3114,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "js_int", "ruma-common", @@ -3124,7 +3124,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "cfg-if", "once_cell", @@ -3140,7 +3140,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "js_int", "ruma-common", @@ -3152,7 +3152,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "headers", "http", @@ -3165,7 +3165,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3181,7 +3181,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f485a0265c67a59df75fc6686787538172fa4cac#f485a0265c67a59df75fc6686787538172fa4cac" +source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" dependencies = [ "futures-util", "itertools 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index 25d1001d..0a98befd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "f485a0265c67a59df75fc6686787538172fa4cac" +rev = "3109496a1f91357c89cbb57cf86f179e2cb013e7" features = [ "compat", "rand", diff --git a/src/api/client/sync.rs b/src/api/client/sync.rs index 8c4c6a44..65af775d 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync.rs @@ -7,7 +7,7 @@ use std::{ use axum::extract::State; use conduit::{ debug, err, error, is_equal_to, - result::{FlatOk, IntoIsOk}, + result::FlatOk, utils::{ math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, BoolExt, IterStream, ReadyExt, TryFutureExtExt, @@ -1136,8 +1136,7 @@ async fn share_encrypted_room( services .rooms .state_accessor - .room_state_get(other_room_id, &StateEventType::RoomEncryption, "") - .map(Result::into_is_ok) + .is_encrypted_room(other_room_id) }) .await } diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 447e54be..f02655e6 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -65,12 +65,6 @@ pub(crate) async fn create_invite_route( return Err!(Request(Forbidden("Server is banned on this homeserver."))); } - if let Some(via) = &body.via { - if via.is_empty() { - return Err!(Request(InvalidParam("via field must not be empty."))); - } - } - let mut signed_event = utils::to_canonical_object(&body.event) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invite event is invalid."))?; diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 8065ac55..ac3f5f83 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -5,7 +5,7 @@ use conduit::{ utils::{stream::TryIgnore, ReadyExt}, Err, Error, Result, }; -use database::{Deserialized, Handle, Map}; +use database::{Deserialized, Handle, Json, Map}; use futures::{StreamExt, TryFutureExt}; use ruma::{ events::{ @@ -56,41 +56,19 @@ impl crate::Service for Service { pub async fn update( &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: RoomAccountDataEventType, data: &serde_json::Value, ) -> Result<()> { - let event_type = event_type.to_string(); - let count = self.services.globals.next_count()?; - - let mut prefix = room_id - .map(ToString::to_string) - .unwrap_or_default() - .as_bytes() - .to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(user_id.as_bytes()); - prefix.push(0xFF); - - let mut roomuserdataid = prefix.clone(); - roomuserdataid.extend_from_slice(&count.to_be_bytes()); - roomuserdataid.push(0xFF); - roomuserdataid.extend_from_slice(event_type.as_bytes()); - - let mut key = prefix; - key.extend_from_slice(event_type.as_bytes()); - if data.get("type").is_none() || data.get("content").is_none() { return Err!(Request(InvalidParam("Account data doesn't have all required fields."))); } - self.db.roomuserdataid_accountdata.insert( - &roomuserdataid, - &serde_json::to_vec(&data).expect("to_vec always works on json values"), - ); - - let prev_key = (room_id, user_id, &event_type); - let prev = self.db.roomusertype_roomuserdataid.qry(&prev_key).await; - + let count = self.services.globals.next_count().unwrap(); + let roomuserdataid = (room_id, user_id, count, &event_type); self.db - .roomusertype_roomuserdataid - .insert(&key, &roomuserdataid); + .roomuserdataid_accountdata + .put(roomuserdataid, Json(data)); + + let key = (room_id, user_id, &event_type); + let prev = self.db.roomusertype_roomuserdataid.qry(&key).await; + self.db.roomusertype_roomuserdataid.put(key, roomuserdataid); // Remove old entry if let Ok(prev) = prev { diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 57a295d9..3638cb56 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -4,7 +4,7 @@ use std::{ }; use conduit::{trace, utils, utils::rand, Error, Result, Server}; -use database::{Database, Deserialized, Map}; +use database::{Database, Deserialized, Json, Map}; use futures::{pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; use ruma::{ api::federation::discovery::{ServerSigningKeys, VerifyKey}, @@ -83,7 +83,7 @@ impl Data { .checked_add(1) .expect("counter must not overflow u64"); - self.global.insert(COUNTER, &counter.to_be_bytes()); + self.global.insert(COUNTER, counter.to_be_bytes()); Ok(*counter) } @@ -259,29 +259,21 @@ impl Data { pub async fn add_signing_key( &self, origin: &ServerName, new_keys: ServerSigningKeys, ) -> BTreeMap { - // Not atomic, but this is not critical - let signingkeys = self.server_signingkeys.get(origin).await; - - let mut keys = signingkeys - .and_then(|keys| serde_json::from_slice(&keys).map_err(Into::into)) + // (timo) Not atomic, but this is not critical + let mut keys: ServerSigningKeys = self + .server_signingkeys + .get(origin) + .await + .deserialized() .unwrap_or_else(|_| { // Just insert "now", it doesn't matter ServerSigningKeys::new(origin.to_owned(), MilliSecondsSinceUnixEpoch::now()) }); - let ServerSigningKeys { - verify_keys, - old_verify_keys, - .. - } = new_keys; + keys.verify_keys.extend(new_keys.verify_keys); + keys.old_verify_keys.extend(new_keys.old_verify_keys); - keys.verify_keys.extend(verify_keys); - keys.old_verify_keys.extend(old_verify_keys); - - self.server_signingkeys.insert( - origin.as_bytes(), - &serde_json::to_vec(&keys).expect("serversigningkeys can be serialized"), - ); + self.server_signingkeys.raw_put(origin, Json(&keys)); let mut tree = keys.verify_keys; tree.extend( @@ -324,7 +316,7 @@ impl Data { #[inline] pub fn bump_database_version(&self, new_version: u64) -> Result<()> { - self.global.insert(b"version", &new_version.to_be_bytes()); + self.global.raw_put(b"version", new_version); Ok(()) } diff --git a/src/service/globals/migrations.rs b/src/service/globals/migrations.rs index 334e71c6..c953e7b1 100644 --- a/src/service/globals/migrations.rs +++ b/src/service/globals/migrations.rs @@ -2,7 +2,7 @@ use conduit::{ debug_info, debug_warn, error, info, result::NotFound, utils::{stream::TryIgnore, IterStream, ReadyExt}, - warn, Err, Error, Result, + warn, Err, Result, }; use futures::{FutureExt, StreamExt}; use itertools::Itertools; @@ -37,10 +37,9 @@ pub(crate) async fn migrations(services: &Services) -> Result<()> { // requires recreating the database from scratch. if users_count > 0 { let conduit_user = &services.globals.server_user; - if !services.users.exists(conduit_user).await { - error!("The {} server user does not exist, and the database is not new.", conduit_user); - return Err(Error::bad_database( + error!("The {conduit_user} server user does not exist, and the database is not new."); + return Err!(Database( "Cannot reuse an existing database after changing the server name, please delete the old one first.", )); } @@ -62,9 +61,9 @@ async fn fresh(services: &Services) -> Result<()> { .db .bump_database_version(DATABASE_VERSION)?; - db["global"].insert(b"feat_sha256_media", &[]); - db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[]); - db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[]); + db["global"].insert(b"feat_sha256_media", []); + db["global"].insert(b"fix_bad_double_separator_in_state_cache", []); + db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", []); // Create the admin room and server user on first run crate::admin::create_admin_room(services).await?; @@ -359,7 +358,7 @@ async fn fix_bad_double_separator_in_state_cache(services: &Services) -> Result< .await; db.db.cleanup()?; - db["global"].insert(b"fix_bad_double_separator_in_state_cache", &[]); + db["global"].insert(b"fix_bad_double_separator_in_state_cache", []); info!("Finished fixing"); Ok(()) @@ -440,7 +439,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) } db.db.cleanup()?; - db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", &[]); + db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", []); info!("Finished fixing"); Ok(()) diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 4c303757..bae6f214 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -5,7 +5,7 @@ use conduit::{ utils::stream::{ReadyExt, TryIgnore}, Err, Result, }; -use database::{Deserialized, Ignore, Interfix, Map}; +use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::StreamExt; use ruma::{ api::client::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, @@ -50,31 +50,21 @@ impl crate::Service for Service { #[implement(Service)] pub fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { let version = self.services.globals.next_count()?.to_string(); + let count = self.services.globals.next_count()?; - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); + let key = (user_id, &version); + self.db.backupid_algorithm.put(key, Json(backup_metadata)); - self.db.backupid_algorithm.insert( - &key, - &serde_json::to_vec(backup_metadata).expect("BackupAlgorithm::to_vec always works"), - ); - - self.db - .backupid_etag - .insert(&key, &self.services.globals.next_count()?.to_be_bytes()); + self.db.backupid_etag.put(key, count); Ok(version) } #[implement(Service)] pub async fn delete_backup(&self, user_id: &UserId, version: &str) { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - - self.db.backupid_algorithm.remove(&key); - self.db.backupid_etag.remove(&key); + let key = (user_id, version); + self.db.backupid_algorithm.del(key); + self.db.backupid_etag.del(key); let key = (user_id, version, Interfix); self.db @@ -86,26 +76,21 @@ pub async fn delete_backup(&self, user_id: &UserId, version: &str) { } #[implement(Service)] -pub async fn update_backup( - &self, user_id: &UserId, version: &str, backup_metadata: &Raw, -) -> Result { +pub async fn update_backup<'a>( + &self, user_id: &UserId, version: &'a str, backup_metadata: &Raw, +) -> Result<&'a str> { let key = (user_id, version); if self.db.backupid_algorithm.qry(&key).await.is_err() { return Err!(Request(NotFound("Tried to update nonexistent backup."))); } - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - + let count = self.services.globals.next_count().unwrap(); + self.db.backupid_etag.put(key, count); self.db .backupid_algorithm - .insert(&key, backup_metadata.json().get().as_bytes()); - self.db - .backupid_etag - .insert(&key, &self.services.globals.next_count()?.to_be_bytes()); + .put_raw(key, backup_metadata.json().get()); - Ok(version.to_owned()) + Ok(version) } #[implement(Service)] @@ -156,22 +141,13 @@ pub async fn add_key( return Err!(Request(NotFound("Tried to update nonexistent backup."))); } - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(version.as_bytes()); - - self.db - .backupid_etag - .insert(&key, &self.services.globals.next_count()?.to_be_bytes()); - - key.push(0xFF); - key.extend_from_slice(room_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(session_id.as_bytes()); + let count = self.services.globals.next_count().unwrap(); + self.db.backupid_etag.put(key, count); + let key = (user_id, version, room_id, session_id); self.db .backupkeyid_backup - .insert(&key, key_data.json().get().as_bytes()); + .put_raw(key, key_data.json().get()); Ok(()) } diff --git a/src/service/media/data.rs b/src/service/media/data.rs index b2271883..9afbd708 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -1,13 +1,13 @@ -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use conduit::{ - debug, debug_info, trace, + debug, debug_info, err, utils::{str_from_bytes, stream::TryIgnore, string_from_bytes, ReadyExt}, Err, Error, Result, }; -use database::{Database, Map}; +use database::{Database, Interfix, Map}; use futures::StreamExt; -use ruma::{api::client::error::ErrorKind, http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; +use ruma::{http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; use super::{preview::UrlPreviewData, thumbnail::Dim}; @@ -37,39 +37,13 @@ impl Data { &self, mxc: &Mxc<'_>, user: Option<&UserId>, dim: &Dim, content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, ) -> Result> { - let mut key: Vec = Vec::new(); - key.extend_from_slice(b"mxc://"); - key.extend_from_slice(mxc.server_name.as_bytes()); - key.extend_from_slice(b"/"); - key.extend_from_slice(mxc.media_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(&dim.width.to_be_bytes()); - key.extend_from_slice(&dim.height.to_be_bytes()); - key.push(0xFF); - key.extend_from_slice( - content_disposition - .map(ToString::to_string) - .unwrap_or_default() - .as_bytes(), - ); - key.push(0xFF); - key.extend_from_slice( - content_type - .as_ref() - .map(|c| c.as_bytes()) - .unwrap_or_default(), - ); - - self.mediaid_file.insert(&key, &[]); - + let dim: &[u32] = &[dim.width, dim.height]; + let key = (mxc, dim, content_disposition, content_type); + let key = database::serialize_to_vec(key)?; + self.mediaid_file.insert(&key, []); if let Some(user) = user { - let mut key: Vec = Vec::new(); - key.extend_from_slice(b"mxc://"); - key.extend_from_slice(mxc.server_name.as_bytes()); - key.extend_from_slice(b"/"); - key.extend_from_slice(mxc.media_id.as_bytes()); - let user = user.as_bytes().to_vec(); - self.mediaid_user.insert(&key, &user); + let key = (mxc, user); + self.mediaid_user.put_raw(key, user); } Ok(key) @@ -78,33 +52,23 @@ impl Data { pub(super) async fn delete_file_mxc(&self, mxc: &Mxc<'_>) { debug!("MXC URI: {mxc}"); - let mut prefix: Vec = Vec::new(); - prefix.extend_from_slice(b"mxc://"); - prefix.extend_from_slice(mxc.server_name.as_bytes()); - prefix.extend_from_slice(b"/"); - prefix.extend_from_slice(mxc.media_id.as_bytes()); - prefix.push(0xFF); - - trace!("MXC db prefix: {prefix:?}"); + let prefix = (mxc, Interfix); self.mediaid_file - .raw_keys_prefix(&prefix) + .keys_prefix_raw(&prefix) .ignore_err() - .ready_for_each(|key| { - debug!("Deleting key: {:?}", key); - self.mediaid_file.remove(key); - }) + .ready_for_each(|key| self.mediaid_file.remove(key)) .await; self.mediaid_user - .raw_stream_prefix(&prefix) + .stream_prefix_raw(&prefix) .ignore_err() .ready_for_each(|(key, val)| { - if key.starts_with(&prefix) { - let user = str_from_bytes(val).unwrap_or_default(); - debug_info!("Deleting key {key:?} which was uploaded by user {user}"); + debug_assert!(key.starts_with(mxc.to_string().as_bytes()), "key should start with the mxc"); - self.mediaid_user.remove(key); - } + let user = str_from_bytes(val).unwrap_or_default(); + debug_info!("Deleting key {key:?} which was uploaded by user {user}"); + + self.mediaid_user.remove(key); }) .await; } @@ -113,16 +77,10 @@ impl Data { pub(super) async fn search_mxc_metadata_prefix(&self, mxc: &Mxc<'_>) -> Result>> { debug!("MXC URI: {mxc}"); - let mut prefix: Vec = Vec::new(); - prefix.extend_from_slice(b"mxc://"); - prefix.extend_from_slice(mxc.server_name.as_bytes()); - prefix.extend_from_slice(b"/"); - prefix.extend_from_slice(mxc.media_id.as_bytes()); - prefix.push(0xFF); - + let prefix = (mxc, Interfix); let keys: Vec> = self .mediaid_file - .raw_keys_prefix(&prefix) + .keys_prefix_raw(&prefix) .ignore_err() .map(<[u8]>::to_vec) .collect() @@ -138,24 +96,17 @@ impl Data { } pub(super) async fn search_file_metadata(&self, mxc: &Mxc<'_>, dim: &Dim) -> Result { - let mut prefix: Vec = Vec::new(); - prefix.extend_from_slice(b"mxc://"); - prefix.extend_from_slice(mxc.server_name.as_bytes()); - prefix.extend_from_slice(b"/"); - prefix.extend_from_slice(mxc.media_id.as_bytes()); - prefix.push(0xFF); - prefix.extend_from_slice(&dim.width.to_be_bytes()); - prefix.extend_from_slice(&dim.height.to_be_bytes()); - prefix.push(0xFF); + let dim: &[u32] = &[dim.width, dim.height]; + let prefix = (mxc, dim, Interfix); let key = self .mediaid_file - .raw_keys_prefix(&prefix) + .keys_prefix_raw(&prefix) .ignore_err() .map(ToOwned::to_owned) .next() .await - .ok_or_else(|| Error::BadRequest(ErrorKind::NotFound, "Media not found"))?; + .ok_or_else(|| err!(Request(NotFound("Media not found"))))?; let mut parts = key.rsplit(|&b| b == 0xFF); @@ -215,9 +166,7 @@ impl Data { Ok(()) } - pub(super) fn set_url_preview( - &self, url: &str, data: &UrlPreviewData, timestamp: std::time::Duration, - ) -> Result<()> { + pub(super) fn set_url_preview(&self, url: &str, data: &UrlPreviewData, timestamp: Duration) -> Result<()> { let mut value = Vec::::new(); value.extend_from_slice(×tamp.as_secs().to_be_bytes()); value.push(0xFF); diff --git a/src/service/media/migrations.rs b/src/service/media/migrations.rs index 2d1b39f9..0e358d44 100644 --- a/src/service/media/migrations.rs +++ b/src/service/media/migrations.rs @@ -54,7 +54,7 @@ pub(crate) async fn migrate_sha256_media(services: &Services) -> Result<()> { services.globals.db.bump_database_version(13)?; } - db["global"].insert(b"feat_sha256_media", &[]); + db["global"].insert(b"feat_sha256_media", []); info!("Finished applying sha256_media"); Ok(()) } diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 9c9d0ae3..8522746f 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -5,7 +5,7 @@ use conduit::{ utils::{stream::TryIgnore, ReadyExt}, Result, }; -use database::{Deserialized, Map}; +use database::{Deserialized, Json, Map}; use futures::Stream; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; @@ -107,14 +107,12 @@ impl Data { last_active_ts, status_msg, ); + let count = self.services.globals.next_count()?; let key = presenceid_key(count, user_id); - self.presenceid_presence - .insert(&key, &presence.to_json_bytes()?); - - self.userid_presenceid - .insert(user_id.as_bytes(), &count.to_be_bytes()); + self.presenceid_presence.raw_put(key, Json(presence)); + self.userid_presenceid.raw_put(user_id, count); if let Ok((last_count, _)) = last_presence { let key = presenceid_key(last_count, user_id); @@ -136,7 +134,7 @@ impl Data { let key = presenceid_key(count, user_id); self.presenceid_presence.remove(&key); - self.userid_presenceid.remove(user_id.as_bytes()); + self.userid_presenceid.remove(user_id); } pub fn presence_since(&self, since: u64) -> impl Stream)> + Send + '_ { @@ -152,7 +150,11 @@ impl Data { #[inline] fn presenceid_key(count: u64, user_id: &UserId) -> Vec { - [count.to_be_bytes().to_vec(), user_id.as_bytes().to_vec()].concat() + let cap = size_of::().saturating_add(user_id.as_bytes().len()); + let mut key = Vec::with_capacity(cap); + key.extend_from_slice(&count.to_be_bytes()); + key.extend_from_slice(user_id.as_bytes()); + key } #[inline] diff --git a/src/service/presence/presence.rs b/src/service/presence/presence.rs index 0d5c226b..c4372003 100644 --- a/src/service/presence/presence.rs +++ b/src/service/presence/presence.rs @@ -35,10 +35,6 @@ impl Presence { serde_json::from_slice(bytes).map_err(|_| Error::bad_database("Invalid presence data in database")) } - pub(super) fn to_json_bytes(&self) -> Result> { - serde_json::to_vec(self).map_err(|_| Error::bad_database("Could not serialize Presence to JSON")) - } - /// Creates a PresenceEvent from available data. pub(super) async fn to_presence_event(&self, user_id: &UserId, users: &users::Service) -> PresenceEvent { let now = utils::millis_since_unix_epoch(); diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index e7b1824a..af15e332 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -6,7 +6,7 @@ use conduit::{ utils::{stream::TryIgnore, string_from_bytes}, Err, PduEvent, Result, }; -use database::{Deserialized, Ignore, Interfix, Map}; +use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::{Stream, StreamExt}; use ipaddress::IPAddress; use ruma::{ @@ -68,18 +68,12 @@ impl Service { pub fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) { match pusher { set_pusher::v3::PusherAction::Post(data) => { - let mut key = sender.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(data.pusher.ids.pushkey.as_bytes()); - self.db - .senderkey_pusher - .insert(&key, &serde_json::to_vec(pusher).expect("Pusher is valid JSON value")); + let key = (sender, &data.pusher.ids.pushkey); + self.db.senderkey_pusher.put(key, Json(pusher)); }, set_pusher::v3::PusherAction::Delete(ids) => { - let mut key = sender.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(ids.pushkey.as_bytes()); - self.db.senderkey_pusher.remove(&key); + let key = (sender, &ids.pushkey); + self.db.senderkey_pusher.del(key); }, } } diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 2112ecef..f366ffe2 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -26,10 +26,10 @@ impl crate::Service for Service { } #[implement(Service)] -pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_id.as_bytes(), &[]); } +pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_id, []); } #[implement(Service)] -pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id.as_bytes()); } +pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id); } #[implement(Service)] pub async fn is_public_room(&self, room_id: &RoomId) -> bool { self.db.publicroomids.get(room_id).await.is_ok() } diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 9493dcc4..7a4da2a6 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -79,17 +79,9 @@ pub fn lazy_load_confirm_delivery(&self, user_id: &UserId, device_id: &DeviceId, return; }; - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(device_id.as_bytes()); - prefix.push(0xFF); - prefix.extend_from_slice(room_id.as_bytes()); - prefix.push(0xFF); - for ll_id in &user_ids { - let mut key = prefix.clone(); - key.extend_from_slice(ll_id.as_bytes()); - self.db.lazyloadedids.insert(&key, &[]); + let key = (user_id, device_id, room_id, ll_id); + self.db.lazyloadedids.put_raw(key, []); } } diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 8367eb72..4ee390a5 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -64,9 +64,9 @@ pub fn iter_ids(&self) -> impl Stream + Send + '_ { self.db.room #[inline] pub fn disable_room(&self, room_id: &RoomId, disabled: bool) { if disabled { - self.db.disabledroomids.insert(room_id.as_bytes(), &[]); + self.db.disabledroomids.insert(room_id, []); } else { - self.db.disabledroomids.remove(room_id.as_bytes()); + self.db.disabledroomids.remove(room_id); } } @@ -74,9 +74,9 @@ pub fn disable_room(&self, room_id: &RoomId, disabled: bool) { #[inline] pub fn ban_room(&self, room_id: &RoomId, banned: bool) { if banned { - self.db.bannedroomids.insert(room_id.as_bytes(), &[]); + self.db.bannedroomids.insert(room_id, []); } else { - self.db.bannedroomids.remove(room_id.as_bytes()); + self.db.bannedroomids.remove(room_id); } } diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index b9d04263..03e77838 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use conduit::{implement, Result}; -use database::{Deserialized, Map}; +use database::{Deserialized, Json, Map}; use ruma::{CanonicalJsonObject, EventId}; use crate::PduEvent; @@ -50,8 +50,5 @@ pub async fn get_pdu_outlier(&self, event_id: &EventId) -> Result { #[implement(Service)] #[tracing::instrument(skip(self, pdu), level = "debug")] pub fn add_pdu_outlier(&self, event_id: &EventId, pdu: &CanonicalJsonObject) { - self.db.eventid_outlierpdu.insert( - event_id.as_bytes(), - &serde_json::to_vec(&pdu).expect("CanonicalJsonObject is valid"), - ); + self.db.eventid_outlierpdu.raw_put(event_id, Json(pdu)); } diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 8e045658..4d570e6d 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -39,9 +39,10 @@ impl Data { } pub(super) fn add_relation(&self, from: u64, to: u64) { - let mut key = to.to_be_bytes().to_vec(); - key.extend_from_slice(&from.to_be_bytes()); - self.tofrom_relation.insert(&key, &[]); + const BUFSIZE: usize = size_of::() * 2; + + let key: &[u64] = &[to, from]; + self.tofrom_relation.aput_raw::(key, []); } pub(super) fn relations_until<'a>( @@ -78,9 +79,8 @@ impl Data { pub(super) fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) { for prev in event_ids { - let mut key = room_id.as_bytes().to_vec(); - key.extend_from_slice(prev.as_bytes()); - self.referencedevents.insert(&key, &[]); + let key = (room_id, prev); + self.referencedevents.put_raw(key, []); } } @@ -89,9 +89,7 @@ impl Data { self.referencedevents.qry(&key).await.is_ok() } - pub(super) fn mark_event_soft_failed(&self, event_id: &EventId) { - self.softfailedeventids.insert(event_id.as_bytes(), &[]); - } + pub(super) fn mark_event_soft_failed(&self, event_id: &EventId) { self.softfailedeventids.insert(event_id, []); } pub(super) async fn is_event_soft_failed(&self, event_id: &EventId) -> bool { self.softfailedeventids.get(event_id).await.is_ok() diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 74b649ef..80a35e88 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -5,7 +5,7 @@ use conduit::{ utils::{stream::TryIgnore, ReadyExt}, Error, Result, }; -use database::{Deserialized, Map}; +use database::{Deserialized, Json, Map}; use futures::{Stream, StreamExt}; use ruma::{ events::{receipt::ReceiptEvent, AnySyncEphemeralRoomEvent}, @@ -44,33 +44,19 @@ impl Data { pub(super) async fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) { type KeyVal<'a> = (&'a RoomId, u64, &'a UserId); - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xFF); - - let mut last_possible_key = prefix.clone(); - last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes()); - // Remove old entry + let last_possible_key = (room_id, u64::MAX); self.readreceiptid_readreceipt - .rev_keys_from_raw(&last_possible_key) + .rev_keys_from(&last_possible_key) .ignore_err() .ready_take_while(|(r, ..): &KeyVal<'_>| *r == room_id) .ready_filter_map(|(r, c, u): KeyVal<'_>| (u == user_id).then_some((r, c, u))) - .ready_for_each(|old: KeyVal<'_>| { - // This is the old room_latest - self.readreceiptid_readreceipt.del(&old); - }) + .ready_for_each(|old: KeyVal<'_>| self.readreceiptid_readreceipt.del(old)) .await; - let mut room_latest_id = prefix; - room_latest_id.extend_from_slice(&self.services.globals.next_count().unwrap().to_be_bytes()); - room_latest_id.push(0xFF); - room_latest_id.extend_from_slice(user_id.as_bytes()); - - self.readreceiptid_readreceipt.insert( - &room_latest_id, - &serde_json::to_vec(event).expect("EduEvent::to_string always works"), - ); + let count = self.services.globals.next_count().unwrap(); + let latest_id = (room_id, count, user_id); + self.readreceiptid_readreceipt.put(latest_id, Json(event)); } pub(super) fn readreceipts_since<'a>( @@ -113,15 +99,11 @@ impl Data { } pub(super) fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) { - let mut key = room_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(user_id.as_bytes()); + let key = (room_id, user_id); + let next_count = self.services.globals.next_count().unwrap(); - self.roomuserid_privateread - .insert(&key, &count.to_be_bytes()); - - self.roomuserid_lastprivatereadupdate - .insert(&key, &self.services.globals.next_count().unwrap().to_be_bytes()); + self.roomuserid_privateread.put(key, count); + self.roomuserid_lastprivatereadupdate.put(key, next_count); } pub(super) async fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result { diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index bd8fdcc9..609c0e07 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{mem::size_of_val, sync::Arc}; use conduit::{err, implement, utils, Result}; use database::{Deserialized, Map}; @@ -46,6 +46,8 @@ impl crate::Service for Service { #[implement(Service)] pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { + const BUFSIZE: usize = size_of::(); + if let Ok(shorteventid) = self .db .eventid_shorteventid @@ -57,12 +59,15 @@ pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { } let shorteventid = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&shorteventid) == BUFSIZE, "buffer requirement changed"); + self.db .eventid_shorteventid - .insert(event_id.as_bytes(), &shorteventid.to_be_bytes()); + .raw_aput::(event_id, shorteventid); + self.db .shorteventid_eventid - .insert(&shorteventid.to_be_bytes(), event_id.as_bytes()); + .aput_raw::(shorteventid, event_id); shorteventid } @@ -77,13 +82,17 @@ pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> .map(|(i, result)| match result { Ok(ref short) => utils::u64_from_u8(short), Err(_) => { + const BUFSIZE: usize = size_of::(); + let short = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); + self.db .eventid_shorteventid - .insert(event_ids[i], &short.to_be_bytes()); + .raw_aput::(event_ids[i], short); self.db .shorteventid_eventid - .insert(&short.to_be_bytes(), event_ids[i]); + .aput_raw::(short, event_ids[i]); short }, @@ -103,7 +112,9 @@ pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &s #[implement(Service)] pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> u64 { - let key = (event_type.to_string(), state_key); + const BUFSIZE: usize = size_of::(); + + let key = (event_type, state_key); if let Ok(shortstatekey) = self .db .statekey_shortstatekey @@ -114,17 +125,16 @@ pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, sta return shortstatekey; } - let mut key = event_type.to_string().as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(state_key.as_bytes()); - let shortstatekey = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&shortstatekey) == BUFSIZE, "buffer requirement changed"); + self.db .statekey_shortstatekey - .insert(&key, &shortstatekey.to_be_bytes()); + .put_aput::(key, shortstatekey); + self.db .shortstatekey_statekey - .insert(&shortstatekey.to_be_bytes(), &key); + .aput_put::(shortstatekey, key); shortstatekey } @@ -177,6 +187,8 @@ pub async fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(State /// Returns (shortstatehash, already_existed) #[implement(Service)] pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, bool) { + const BUFSIZE: usize = size_of::(); + if let Ok(shortstatehash) = self .db .statehash_shortstatehash @@ -188,9 +200,11 @@ pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, boo } let shortstatehash = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&shortstatehash) == BUFSIZE, "buffer requirement changed"); + self.db .statehash_shortstatehash - .insert(state_hash, &shortstatehash.to_be_bytes()); + .raw_aput::(state_hash, shortstatehash); (shortstatehash, false) } @@ -208,10 +222,15 @@ pub async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> u64 { .await .deserialized() .unwrap_or_else(|_| { + const BUFSIZE: usize = size_of::(); + let short = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); + self.db .roomid_shortroomid - .insert(room_id.as_bytes(), &short.to_be_bytes()); + .raw_aput::(room_id, short); + short }) } diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs index 7265038f..813f48ae 100644 --- a/src/service/rooms/state/data.rs +++ b/src/service/rooms/state/data.rs @@ -36,12 +36,12 @@ impl Data { _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) { self.roomid_shortstatehash - .insert(room_id.as_bytes(), &new_shortstatehash.to_be_bytes()); + .raw_put(room_id, new_shortstatehash); } pub(super) fn set_event_state(&self, shorteventid: u64, shortstatehash: u64) { self.shorteventid_shortstatehash - .insert(&shorteventid.to_be_bytes(), &shortstatehash.to_be_bytes()); + .put(shorteventid, shortstatehash); } pub(super) async fn set_forward_extremities( @@ -57,12 +57,9 @@ impl Data { .ready_for_each(|key| self.roomid_pduleaves.remove(key)) .await; - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xFF); - for event_id in event_ids { - let mut key = prefix.clone(); - key.extend_from_slice(event_id.as_bytes()); - self.roomid_pduleaves.insert(&key, event_id.as_bytes()); + for event_id in &event_ids { + let key = (room_id, event_id); + self.roomid_pduleaves.put_raw(key, event_id); } } } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 19f1f141..561db18a 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -481,4 +481,10 @@ impl Service { .await .map(|content: RoomEncryptionEventContent| content.algorithm) } + + pub async fn is_encrypted_room(&self, room_id: &RoomId) -> bool { + self.room_state_get(room_id, &StateEventType::RoomEncryption, "") + .await + .is_ok() + } } diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs index 6e01e49d..c06c8107 100644 --- a/src/service/rooms/state_cache/data.rs +++ b/src/service/rooms/state_cache/data.rs @@ -4,7 +4,7 @@ use std::{ }; use conduit::{utils::stream::TryIgnore, Result}; -use database::{Deserialized, Interfix, Map}; +use database::{serialize_to_vec, Deserialized, Interfix, Json, Map}; use futures::{Stream, StreamExt}; use ruma::{ events::{AnyStrippedStateEvent, AnySyncStateEvent}, @@ -63,71 +63,62 @@ impl Data { } pub(super) fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) { - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); - self.roomuseroncejoinedids.insert(&userroom_id, &[]); + let key = (user_id, room_id); + + self.roomuseroncejoinedids.put_raw(key, []); } pub(super) fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { - let roomid = room_id.as_bytes().to_vec(); + let userroom_id = (user_id, room_id); + let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); - let mut roomuser_id = roomid.clone(); - roomuser_id.push(0xFF); - roomuser_id.extend_from_slice(user_id.as_bytes()); + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); + self.userroomid_joined.insert(&userroom_id, []); + self.roomuserid_joined.insert(&roomuser_id, []); - self.userroomid_joined.insert(&userroom_id, &[]); - self.roomuserid_joined.insert(&roomuser_id, &[]); self.userroomid_invitestate.remove(&userroom_id); self.roomuserid_invitecount.remove(&roomuser_id); + self.userroomid_leftstate.remove(&userroom_id); self.roomuserid_leftcount.remove(&roomuser_id); - self.roomid_inviteviaservers.remove(&roomid); + self.roomid_inviteviaservers.remove(room_id); } pub(super) fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) { - let roomid = room_id.as_bytes().to_vec(); + let userroom_id = (user_id, room_id); + let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); - let mut roomuser_id = roomid.clone(); - roomuser_id.push(0xFF); - roomuser_id.extend_from_slice(user_id.as_bytes()); + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); + // (timo) TODO + let leftstate = Vec::>::new(); + let count = self.services.globals.next_count().unwrap(); + + self.userroomid_leftstate + .raw_put(&userroom_id, Json(leftstate)); + self.roomuserid_leftcount.raw_put(&roomuser_id, count); - self.userroomid_leftstate.insert( - &userroom_id, - &serde_json::to_vec(&Vec::>::new()).unwrap(), - ); // TODO - self.roomuserid_leftcount - .insert(&roomuser_id, &self.services.globals.next_count().unwrap().to_be_bytes()); self.userroomid_joined.remove(&userroom_id); self.roomuserid_joined.remove(&roomuser_id); + self.userroomid_invitestate.remove(&userroom_id); self.roomuserid_invitecount.remove(&roomuser_id); - self.roomid_inviteviaservers.remove(&roomid); + self.roomid_inviteviaservers.remove(room_id); } /// Makes a user forget a room. #[tracing::instrument(skip(self), level = "debug")] pub(super) fn forget(&self, room_id: &RoomId, user_id: &UserId) { - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); + let userroom_id = (user_id, room_id); + let roomuser_id = (room_id, user_id); - let mut roomuser_id = room_id.as_bytes().to_vec(); - roomuser_id.push(0xFF); - roomuser_id.extend_from_slice(user_id.as_bytes()); - - self.userroomid_leftstate.remove(&userroom_id); - self.roomuserid_leftcount.remove(&roomuser_id); + self.userroomid_leftstate.del(userroom_id); + self.roomuserid_leftcount.del(roomuser_id); } /// Returns an iterator over all rooms a user was invited to. diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index edfae529..077eee10 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -3,13 +3,13 @@ mod data; use std::{collections::HashSet, sync::Arc}; use conduit::{ - err, + err, is_not_empty, utils::{stream::TryIgnore, ReadyExt, StreamTools}, warn, Result, }; use data::Data; -use database::{Deserialized, Ignore, Interfix}; -use futures::{Stream, StreamExt}; +use database::{serialize_to_vec, Deserialized, Ignore, Interfix, Json}; +use futures::{stream::iter, Stream, StreamExt}; use itertools::Itertools; use ruma::{ events::{ @@ -547,50 +547,37 @@ impl Service { .unwrap_or(0), ); - self.db - .roomid_joinedcount - .insert(room_id.as_bytes(), &joinedcount.to_be_bytes()); - - self.db - .roomid_invitedcount - .insert(room_id.as_bytes(), &invitedcount.to_be_bytes()); + self.db.roomid_joinedcount.raw_put(room_id, joinedcount); + self.db.roomid_invitedcount.raw_put(room_id, invitedcount); self.room_servers(room_id) .ready_for_each(|old_joined_server| { - if !joined_servers.remove(old_joined_server) { - // Server not in room anymore - let mut roomserver_id = room_id.as_bytes().to_vec(); - roomserver_id.push(0xFF); - roomserver_id.extend_from_slice(old_joined_server.as_bytes()); - - let mut serverroom_id = old_joined_server.as_bytes().to_vec(); - serverroom_id.push(0xFF); - serverroom_id.extend_from_slice(room_id.as_bytes()); - - self.db.roomserverids.remove(&roomserver_id); - self.db.serverroomids.remove(&serverroom_id); + if joined_servers.remove(old_joined_server) { + return; } + + // Server not in room anymore + let roomserver_id = (room_id, old_joined_server); + let serverroom_id = (old_joined_server, room_id); + + self.db.roomserverids.del(roomserver_id); + self.db.serverroomids.del(serverroom_id); }) .await; // Now only new servers are in joined_servers anymore - for server in joined_servers { - let mut roomserver_id = room_id.as_bytes().to_vec(); - roomserver_id.push(0xFF); - roomserver_id.extend_from_slice(server.as_bytes()); + for server in &joined_servers { + let roomserver_id = (room_id, server); + let serverroom_id = (server, room_id); - let mut serverroom_id = server.as_bytes().to_vec(); - serverroom_id.push(0xFF); - serverroom_id.extend_from_slice(room_id.as_bytes()); - - self.db.roomserverids.insert(&roomserver_id, &[]); - self.db.serverroomids.insert(&serverroom_id, &[]); + self.db.roomserverids.put_raw(roomserver_id, []); + self.db.serverroomids.put_raw(serverroom_id, []); } self.db .appservice_in_room_cache .write() - .unwrap() + .expect("locked") .remove(room_id); } @@ -598,44 +585,44 @@ impl Service { &self, user_id: &UserId, room_id: &RoomId, last_state: Option>>, invite_via: Option>, ) { - let mut roomuser_id = room_id.as_bytes().to_vec(); - roomuser_id.push(0xFF); - roomuser_id.extend_from_slice(user_id.as_bytes()); + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); + let userroom_id = (user_id, room_id); + let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); + + self.db + .userroomid_invitestate + .raw_put(&userroom_id, Json(last_state.unwrap_or_default())); - self.db.userroomid_invitestate.insert( - &userroom_id, - &serde_json::to_vec(&last_state.unwrap_or_default()).expect("state to bytes always works"), - ); self.db .roomuserid_invitecount - .insert(&roomuser_id, &self.services.globals.next_count().unwrap().to_be_bytes()); + .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); + self.db.userroomid_joined.remove(&userroom_id); self.db.roomuserid_joined.remove(&roomuser_id); + self.db.userroomid_leftstate.remove(&userroom_id); self.db.roomuserid_leftcount.remove(&roomuser_id); - if let Some(servers) = invite_via.as_deref() { + if let Some(servers) = invite_via.filter(is_not_empty!()) { self.add_servers_invite_via(room_id, servers).await; } } #[tracing::instrument(skip(self, servers), level = "debug")] - pub async fn add_servers_invite_via(&self, room_id: &RoomId, servers: &[OwnedServerName]) { - let mut prev_servers: Vec<_> = self + pub async fn add_servers_invite_via(&self, room_id: &RoomId, servers: Vec) { + let mut servers: Vec<_> = self .servers_invite_via(room_id) .map(ToOwned::to_owned) + .chain(iter(servers.into_iter())) .collect() .await; - prev_servers.extend(servers.to_owned()); - prev_servers.sort_unstable(); - prev_servers.dedup(); + servers.sort_unstable(); + servers.dedup(); - let servers = prev_servers + let servers = servers .iter() .map(|server| server.as_bytes()) .collect_vec() diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index cb85cf19..c51b7856 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -11,7 +11,7 @@ use conduit::{ utils::{stream::TryIgnore, u64_from_u8, ReadyExt}, Err, PduCount, PduEvent, Result, }; -use database::{Database, Deserialized, KeyVal, Map}; +use database::{Database, Deserialized, Json, KeyVal, Map}; use futures::{FutureExt, Stream, StreamExt}; use ruma::{CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; use tokio::sync::Mutex; @@ -168,10 +168,7 @@ impl Data { } pub(super) async fn append_pdu(&self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64) { - self.pduid_pdu.insert( - pdu_id, - &serde_json::to_vec(json).expect("CanonicalJsonObject is always a valid"), - ); + self.pduid_pdu.raw_put(pdu_id, Json(json)); self.lasttimelinecount_cache .lock() @@ -183,13 +180,10 @@ impl Data { } pub(super) fn prepend_backfill_pdu(&self, pdu_id: &[u8], event_id: &EventId, json: &CanonicalJsonObject) { - self.pduid_pdu.insert( - pdu_id, - &serde_json::to_vec(json).expect("CanonicalJsonObject is always a valid"), - ); + self.pduid_pdu.raw_put(pdu_id, Json(json)); - self.eventid_pduid.insert(event_id.as_bytes(), pdu_id); - self.eventid_outlierpdu.remove(event_id.as_bytes()); + self.eventid_pduid.insert(event_id, pdu_id); + self.eventid_outlierpdu.remove(event_id); } /// Removes a pdu and creates a new one with the same id. @@ -328,5 +322,5 @@ pub(super) fn pdu_count(pdu_id: &[u8]) -> PduCount { fn increment(db: &Arc, key: &[u8]) { let old = db.get_blocking(key); let new = utils::increment(old.ok().as_deref()); - db.insert(key, &new); + db.insert(key, new); } diff --git a/src/service/rooms/user/data.rs b/src/service/rooms/user/data.rs index d4d9874c..96b009f8 100644 --- a/src/service/rooms/user/data.rs +++ b/src/service/rooms/user/data.rs @@ -38,20 +38,13 @@ impl Data { } pub(super) fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) { - let mut userroom_id = user_id.as_bytes().to_vec(); - userroom_id.push(0xFF); - userroom_id.extend_from_slice(room_id.as_bytes()); - let mut roomuser_id = room_id.as_bytes().to_vec(); - roomuser_id.push(0xFF); - roomuser_id.extend_from_slice(user_id.as_bytes()); + let userroom_id = (user_id, room_id); + self.userroomid_highlightcount.put(userroom_id, 0_u64); + self.userroomid_notificationcount.put(userroom_id, 0_u64); - self.userroomid_notificationcount - .insert(&userroom_id, &0_u64.to_be_bytes()); - self.userroomid_highlightcount - .insert(&userroom_id, &0_u64.to_be_bytes()); - - self.roomuserid_lastnotificationread - .insert(&roomuser_id, &self.services.globals.next_count().unwrap().to_be_bytes()); + let roomuser_id = (room_id, user_id); + let count = self.services.globals.next_count().unwrap(); + self.roomuserid_lastnotificationread.put(roomuser_id, count); } pub(super) async fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { @@ -89,11 +82,8 @@ impl Data { .await .expect("room exists"); - let mut key = shortroomid.to_be_bytes().to_vec(); - key.extend_from_slice(&token.to_be_bytes()); - - self.roomsynctoken_shortstatehash - .insert(&key, &shortstatehash.to_be_bytes()); + let key: &[u64] = &[shortroomid, token]; + self.roomsynctoken_shortstatehash.put(key, shortstatehash); } pub(super) async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result { diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 96d4a6a9..f75a212c 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -146,8 +146,7 @@ impl Data { } pub(super) fn set_latest_educount(&self, server_name: &ServerName, last_count: u64) { - self.servername_educount - .insert(server_name.as_bytes(), &last_count.to_be_bytes()); + self.servername_educount.raw_put(server_name, last_count); } pub async fn get_latest_educount(&self, server_name: &ServerName) -> u64 { diff --git a/src/service/sending/dest.rs b/src/service/sending/dest.rs index 9968acd7..234a0b90 100644 --- a/src/service/sending/dest.rs +++ b/src/service/sending/dest.rs @@ -12,7 +12,7 @@ pub enum Destination { #[implement(Destination)] #[must_use] -pub fn get_prefix(&self) -> Vec { +pub(super) fn get_prefix(&self) -> Vec { match self { Self::Normal(server) => { let len = server.as_bytes().len().saturating_add(1); diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index f75f1bcd..d2865d88 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -8,7 +8,7 @@ use conduit::{ utils::{hash, string::EMPTY}, Error, Result, }; -use database::{Deserialized, Map}; +use database::{Deserialized, Json, Map}; use ruma::{ api::client::{ error::ErrorKind, @@ -217,21 +217,14 @@ pub fn get_uiaa_request( #[implement(Service)] fn update_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: &str, uiaainfo: Option<&UiaaInfo>) { - let mut userdevicesessionid = user_id.as_bytes().to_vec(); - userdevicesessionid.push(0xFF); - userdevicesessionid.extend_from_slice(device_id.as_bytes()); - userdevicesessionid.push(0xFF); - userdevicesessionid.extend_from_slice(session.as_bytes()); + let key = (user_id, device_id, session); if let Some(uiaainfo) = uiaainfo { - self.db.userdevicesessionid_uiaainfo.insert( - &userdevicesessionid, - &serde_json::to_vec(&uiaainfo).expect("UiaaInfo::to_vec always works"), - ); - } else { self.db .userdevicesessionid_uiaainfo - .remove(&userdevicesessionid); + .put(key, Json(uiaainfo)); + } else { + self.db.userdevicesessionid_uiaainfo.del(key); } } diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index fca63725..adc85fe6 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -121,10 +121,7 @@ impl Service { } #[inline] - pub fn update_check_for_updates_id(&self, id: u64) { - self.db - .insert(LAST_CHECK_FOR_UPDATES_COUNT, &id.to_be_bytes()); - } + pub fn update_check_for_updates_id(&self, id: u64) { self.db.raw_put(LAST_CHECK_FOR_UPDATES_COUNT, id); } pub async fn last_check_for_updates_id(&self) -> u64 { self.db diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index a99a7df4..589aee8a 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -3,18 +3,19 @@ use std::{collections::BTreeMap, mem, mem::size_of, sync::Arc}; use conduit::{ debug_warn, err, utils, utils::{stream::TryIgnore, string::Unquoted, ReadyExt}, - warn, Err, Error, Result, Server, + Err, Error, Result, Server, }; -use database::{Deserialized, Ignore, Interfix, Map}; -use futures::{pin_mut, FutureExt, Stream, StreamExt, TryFutureExt}; +use database::{Deserialized, Ignore, Interfix, Json, Map}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, - events::{ignored_user_list::IgnoredUserListEvent, AnyToDeviceEvent, GlobalAccountDataEventType, StateEventType}, + events::{ignored_user_list::IgnoredUserListEvent, AnyToDeviceEvent, GlobalAccountDataEventType}, serde::Raw, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedDeviceKeyId, OwnedMxcUri, OwnedUserId, UInt, UserId, }; +use serde_json::json; use crate::{account_data, admin, globals, rooms, Dep}; @@ -194,22 +195,16 @@ impl Service { /// Hash and set the user's password to the Argon2 hash pub fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { - if let Some(password) = password { - if let Ok(hash) = utils::hash::password(password) { - self.db - .userid_password - .insert(user_id.as_bytes(), hash.as_bytes()); - Ok(()) - } else { - Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Password does not meet the requirements.", - )) - } - } else { - self.db.userid_password.insert(user_id.as_bytes(), b""); - Ok(()) - } + password + .map(utils::hash::password) + .transpose() + .map_err(|e| err!(Request(InvalidParam("Password does not meet the requirements: {e}"))))? + .map_or_else( + || self.db.userid_password.insert(user_id, b""), + |hash| self.db.userid_password.insert(user_id, hash), + ); + + Ok(()) } /// Returns the displayname of a user on this homeserver. @@ -221,11 +216,9 @@ impl Service { /// need to nofify all rooms of this change. pub fn set_displayname(&self, user_id: &UserId, displayname: Option) { if let Some(displayname) = displayname { - self.db - .userid_displayname - .insert(user_id.as_bytes(), displayname.as_bytes()); + self.db.userid_displayname.insert(user_id, displayname); } else { - self.db.userid_displayname.remove(user_id.as_bytes()); + self.db.userid_displayname.remove(user_id); } } @@ -237,11 +230,9 @@ impl Service { /// Sets a new avatar_url or removes it if avatar_url is None. pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) { if let Some(avatar_url) = avatar_url { - self.db - .userid_avatarurl - .insert(user_id.as_bytes(), avatar_url.to_string().as_bytes()); + self.db.userid_avatarurl.insert(user_id, &avatar_url); } else { - self.db.userid_avatarurl.remove(user_id.as_bytes()); + self.db.userid_avatarurl.remove(user_id); } } @@ -253,11 +244,9 @@ impl Service { /// Sets a new avatar_url or removes it if avatar_url is None. pub fn set_blurhash(&self, user_id: &UserId, blurhash: Option) { if let Some(blurhash) = blurhash { - self.db - .userid_blurhash - .insert(user_id.as_bytes(), blurhash.as_bytes()); + self.db.userid_blurhash.insert(user_id, blurhash); } else { - self.db.userid_blurhash.remove(user_id.as_bytes()); + self.db.userid_blurhash.remove(user_id); } } @@ -269,41 +258,29 @@ impl Service { // This method should never be called for nonexistent users. We shouldn't assert // though... if !self.exists(user_id).await { - warn!("Called create_device for non-existent user {} in database", user_id); - return Err(Error::BadRequest(ErrorKind::InvalidParam, "User does not exist.")); + return Err!(Request(InvalidParam(error!("Called create_device for non-existent {user_id}")))); } - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); + let key = (user_id, device_id); + let val = Device { + device_id: device_id.into(), + display_name: initial_device_display_name, + last_seen_ip: client_ip, + last_seen_ts: Some(MilliSecondsSinceUnixEpoch::now()), + }; increment(&self.db.userid_devicelistversion, user_id.as_bytes()); - - self.db.userdeviceid_metadata.insert( - &userdeviceid, - &serde_json::to_vec(&Device { - device_id: device_id.into(), - display_name: initial_device_display_name, - last_seen_ip: client_ip, - last_seen_ts: Some(MilliSecondsSinceUnixEpoch::now()), - }) - .expect("Device::to_string never fails."), - ); - - self.set_token(user_id, device_id, token).await?; - - Ok(()) + self.db.userdeviceid_metadata.put(key, Json(val)); + self.set_token(user_id, device_id, token).await } /// Removes a device from a user. pub async fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) { - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); + let userdeviceid = (user_id, device_id); // Remove tokens - if let Ok(old_token) = self.db.userdeviceid_token.get(&userdeviceid).await { - self.db.userdeviceid_token.remove(&userdeviceid); + if let Ok(old_token) = self.db.userdeviceid_token.qry(&userdeviceid).await { + self.db.userdeviceid_token.del(userdeviceid); self.db.token_userdeviceid.remove(&old_token); } @@ -320,7 +297,7 @@ impl Service { increment(&self.db.userid_devicelistversion, user_id.as_bytes()); - self.db.userdeviceid_metadata.remove(&userdeviceid); + self.db.userdeviceid_metadata.del(userdeviceid); } /// Returns an iterator over all device ids of this user. @@ -333,6 +310,11 @@ impl Service { .map(|(_, device_id): (Ignore, &DeviceId)| device_id) } + pub async fn get_token(&self, user_id: &UserId, device_id: &DeviceId) -> Result { + let key = (user_id, device_id); + self.db.userdeviceid_token.qry(&key).await.deserialized() + } + /// Replaces the access token of one device. pub async fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> { let key = (user_id, device_id); @@ -352,15 +334,8 @@ impl Service { } // Assign token to user device combination - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - self.db - .userdeviceid_token - .insert(&userdeviceid, token.as_bytes()); - self.db - .token_userdeviceid - .insert(token.as_bytes(), &userdeviceid); + self.db.userdeviceid_token.put_raw(key, token); + self.db.token_userdeviceid.raw_put(token, key); Ok(()) } @@ -393,14 +368,12 @@ impl Service { .as_bytes(), ); - self.db.onetimekeyid_onetimekeys.insert( - &key, - &serde_json::to_vec(&one_time_key_value).expect("OneTimeKey::to_vec always works"), - ); - self.db - .userid_lastonetimekeyupdate - .insert(user_id.as_bytes(), &self.services.globals.next_count()?.to_be_bytes()); + .onetimekeyid_onetimekeys + .raw_put(key, Json(one_time_key_value)); + + let count = self.services.globals.next_count().unwrap(); + self.db.userid_lastonetimekeyupdate.raw_put(user_id, count); Ok(()) } @@ -417,9 +390,8 @@ impl Service { pub async fn take_one_time_key( &self, user_id: &UserId, device_id: &DeviceId, key_algorithm: &DeviceKeyAlgorithm, ) -> Result<(OwnedDeviceKeyId, Raw)> { - self.db - .userid_lastonetimekeyupdate - .insert(user_id.as_bytes(), &self.services.globals.next_count()?.to_be_bytes()); + let count = self.services.globals.next_count()?.to_be_bytes(); + self.db.userid_lastonetimekeyupdate.insert(user_id, count); let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -488,15 +460,9 @@ impl Service { } pub async fn add_device_keys(&self, user_id: &UserId, device_id: &DeviceId, device_keys: &Raw) { - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - - self.db.keyid_key.insert( - &userdeviceid, - &serde_json::to_vec(&device_keys).expect("DeviceKeys::to_vec always works"), - ); + let key = (user_id, device_id); + self.db.keyid_key.put(key, Json(device_keys)); self.mark_device_key_update(user_id).await; } @@ -611,13 +577,8 @@ impl Service { .ok_or_else(|| err!(Database("signatures in keyid_key for a user is invalid.")))? .insert(signature.0, signature.1.into()); - let mut key = target_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(key_id.as_bytes()); - self.db.keyid_key.insert( - &key, - &serde_json::to_vec(&cross_signing_key).expect("CrossSigningKey::to_vec always works"), - ); + let key = (target_id, key_id); + self.db.keyid_key.put(key, Json(cross_signing_key)); self.mark_device_key_update(target_id).await; @@ -640,34 +601,21 @@ impl Service { } pub async fn mark_device_key_update(&self, user_id: &UserId) { - let count = self.services.globals.next_count().unwrap().to_be_bytes(); + let count = self.services.globals.next_count().unwrap(); - let rooms_joined = self.services.state_cache.rooms_joined(user_id); - - pin_mut!(rooms_joined); - while let Some(room_id) = rooms_joined.next().await { + self.services + .state_cache + .rooms_joined(user_id) // Don't send key updates to unencrypted rooms - if self - .services - .state_accessor - .room_state_get(room_id, &StateEventType::RoomEncryption, "") - .await - .is_err() - { - continue; - } + .filter(|room_id| self.services.state_accessor.is_encrypted_room(room_id)) + .ready_for_each(|room_id| { + let key = (room_id, count); + self.db.keychangeid_userid.put_raw(key, user_id); + }) + .await; - let mut key = room_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(&count); - - self.db.keychangeid_userid.insert(&key, user_id.as_bytes()); - } - - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(&count); - self.db.keychangeid_userid.insert(&key, user_id.as_bytes()); + let key = (user_id, count); + self.db.keychangeid_userid.put_raw(key, user_id); } pub async fn get_device_keys<'a>(&'a self, user_id: &'a UserId, device_id: &DeviceId) -> Result> { @@ -681,12 +629,7 @@ impl Service { where F: Fn(&UserId) -> bool + Send + Sync, { - let key = self - .db - .keyid_key - .get(key_id) - .await - .deserialized::()?; + let key: serde_json::Value = self.db.keyid_key.get(key_id).await.deserialized()?; let cleaned = clean_signatures(key, sender_user, user_id, allowed_signatures)?; let raw_value = serde_json::value::to_raw_value(&cleaned)?; @@ -718,29 +661,29 @@ impl Service { } pub async fn get_user_signing_key(&self, user_id: &UserId) -> Result> { - let key_id = self.db.userid_usersigningkeyid.get(user_id).await?; - - self.db.keyid_key.get(&*key_id).await.deserialized() + self.db + .userid_usersigningkeyid + .get(user_id) + .and_then(|key_id| self.db.keyid_key.get(&*key_id)) + .await + .deserialized() } pub async fn add_to_device_event( &self, sender: &UserId, target_user_id: &UserId, target_device_id: &DeviceId, event_type: &str, content: serde_json::Value, ) { - let mut key = target_user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(target_device_id.as_bytes()); - key.push(0xFF); - key.extend_from_slice(&self.services.globals.next_count().unwrap().to_be_bytes()); + let count = self.services.globals.next_count().unwrap(); - let mut json = serde_json::Map::new(); - json.insert("type".to_owned(), event_type.to_owned().into()); - json.insert("sender".to_owned(), sender.to_string().into()); - json.insert("content".to_owned(), content); - - let value = serde_json::to_vec(&json).expect("Map::to_vec always works"); - - self.db.todeviceid_events.insert(&key, &value); + let key = (target_user_id, target_device_id, count); + self.db.todeviceid_events.put( + key, + Json(json!({ + "type": event_type, + "sender": sender, + "content": content, + })), + ); } pub fn get_to_device_events<'a>( @@ -783,13 +726,8 @@ impl Service { pub async fn update_device_metadata(&self, user_id: &UserId, device_id: &DeviceId, device: &Device) -> Result<()> { increment(&self.db.userid_devicelistversion, user_id.as_bytes()); - let mut userdeviceid = user_id.as_bytes().to_vec(); - userdeviceid.push(0xFF); - userdeviceid.extend_from_slice(device_id.as_bytes()); - self.db.userdeviceid_metadata.insert( - &userdeviceid, - &serde_json::to_vec(device).expect("Device::to_string always works"), - ); + let key = (user_id, device_id); + self.db.userdeviceid_metadata.put(key, Json(device)); Ok(()) } @@ -824,23 +762,15 @@ impl Service { pub fn create_filter(&self, user_id: &UserId, filter: &FilterDefinition) -> String { let filter_id = utils::random_string(4); - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(filter_id.as_bytes()); - - self.db - .userfilterid_filter - .insert(&key, &serde_json::to_vec(&filter).expect("filter is valid json")); + let key = (user_id, &filter_id); + self.db.userfilterid_filter.put(key, Json(filter)); filter_id } pub async fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result { - self.db - .userfilterid_filter - .qry(&(user_id, filter_id)) - .await - .deserialized() + let key = (user_id, filter_id); + self.db.userfilterid_filter.qry(&key).await.deserialized() } /// Creates an OpenID token, which can be used to prove that a user has @@ -913,17 +843,13 @@ impl Service { /// Sets a new profile key value, removes the key if value is None pub fn set_profile_key(&self, user_id: &UserId, profile_key: &str, profile_key_value: Option) { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(profile_key.as_bytes()); - // TODO: insert to the stable MSC4175 key when it's stable - if let Some(value) = profile_key_value { - let value = serde_json::to_vec(&value).unwrap(); + let key = (user_id, profile_key); - self.db.useridprofilekey_value.insert(&key, &value); + if let Some(value) = profile_key_value { + self.db.useridprofilekey_value.put(key, value); } else { - self.db.useridprofilekey_value.remove(&key); + self.db.useridprofilekey_value.del(key); } } @@ -945,17 +871,13 @@ impl Service { /// Sets a new timezone or removes it if timezone is None. pub fn set_timezone(&self, user_id: &UserId, timezone: Option) { - let mut key = user_id.as_bytes().to_vec(); - key.push(0xFF); - key.extend_from_slice(b"us.cloke.msc4175.tz"); - // TODO: insert to the stable MSC4175 key when it's stable + let key = (user_id, "us.cloke.msc4175.tz"); + if let Some(timezone) = timezone { - self.db - .useridprofilekey_value - .insert(&key, timezone.as_bytes()); + self.db.useridprofilekey_value.put_raw(key, &timezone); } else { - self.db.useridprofilekey_value.remove(&key); + self.db.useridprofilekey_value.del(key); } } } @@ -1012,5 +934,5 @@ where fn increment(db: &Arc, key: &[u8]) { let old = db.get_blocking(key); let new = utils::increment(old.ok().as_deref()); - db.insert(key, &new); + db.insert(key, new); } From 89b5c4ee1c6cfd662a60d704b1ffec736d7a3600 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 11 Oct 2024 20:32:32 +0000 Subject: [PATCH 0912/2091] add timepoint_from_now to complement timepoint_ago in utils Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 2 +- src/core/utils/time.rs | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 4dbecf91..13221341 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -34,7 +34,7 @@ pub use self::{ stream::{IterStream, ReadyExt, Tools as StreamTools, TryReadyExt}, string::{str_from_bytes, string_from_bytes}, sys::available_parallelism, - time::now_millis as millis_since_unix_epoch, + time::{now_millis as millis_since_unix_epoch, timepoint_ago, timepoint_from_now}, }; #[inline] diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 04f47ac3..f96a27d0 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -22,6 +22,13 @@ pub fn timepoint_ago(duration: Duration) -> Result { .ok_or_else(|| err!(Arithmetic("Duration {duration:?} is too large"))) } +#[inline] +pub fn timepoint_from_now(duration: Duration) -> Result { + SystemTime::now() + .checked_add(duration) + .ok_or_else(|| err!(Arithmetic("Duration {duration:?} is too large"))) +} + #[inline] pub fn parse_duration(duration: &str) -> Result { cyborgtime::parse_duration(duration) From 1a09eb0f0235a1dfe7c51f525f656cffad62b60d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 13 Oct 2024 00:57:08 +0000 Subject: [PATCH 0913/2091] use string::EMPTY; minor formatting and misc cleanups Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 12 ++++++++---- src/api/client/membership.rs | 11 +++++++---- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 350e08c6..fd8c39f7 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,7 +6,7 @@ use std::{ }; use api::client::validate_and_add_event_id; -use conduit::{debug, debug_error, err, info, trace, utils, warn, Error, PduEvent, Result}; +use conduit::{debug, debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result}; use futures::StreamExt; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, @@ -57,7 +57,9 @@ pub(super) async fn get_auth_chain(&self, event_id: Box) -> Result Result { - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&EMPTY).trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -134,7 +136,9 @@ pub(super) async fn get_remote_pdu_list( )); } - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&EMPTY).trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -843,7 +847,7 @@ pub(super) async fn database_stats( &self, property: Option, map: Option, ) -> Result { let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned()); - let map_name = map.as_ref().map_or(utils::string::EMPTY, String::as_str); + let map_name = map.as_ref().map_or(EMPTY, String::as_str); let mut out = String::new(); for (name, map) in self.services.db.iter_maps() { diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 06035592..a7a5b166 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -658,13 +658,16 @@ pub async fn join_room_by_id_helper( }); } - if services + let server_in_room = services .rooms .state_cache .server_in_room(services.globals.server_name(), room_id) - .await || servers.is_empty() - || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])) - { + .await; + + let local_join = + server_in_room || servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); + + if local_join { join_room_by_id_helper_local(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) .boxed() .await From d82ea331cfdcd51c2c746618deb26e1fd220abc0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 14 Oct 2024 05:16:18 +0000 Subject: [PATCH 0914/2091] add random shuffle util Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 2 +- src/core/utils/rand.rs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 13221341..3adecc6c 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -30,7 +30,7 @@ pub use self::{ json::{deserialize_from_str, to_canonical_object}, math::clamp, mutex_map::{Guard as MutexMapGuard, MutexMap}, - rand::string as random_string, + rand::{shuffle, string as random_string}, stream::{IterStream, ReadyExt, Tools as StreamTools, TryReadyExt}, string::{str_from_bytes, string_from_bytes}, sys::available_parallelism, diff --git a/src/core/utils/rand.rs b/src/core/utils/rand.rs index d717c4bd..9e6fc7a8 100644 --- a/src/core/utils/rand.rs +++ b/src/core/utils/rand.rs @@ -4,7 +4,12 @@ use std::{ }; use arrayvec::ArrayString; -use rand::{thread_rng, Rng}; +use rand::{seq::SliceRandom, thread_rng, Rng}; + +pub fn shuffle(vec: &mut [T]) { + let mut rng = thread_rng(); + vec.shuffle(&mut rng); +} pub fn string(length: usize) -> String { thread_rng() From c0939c3e9a9d7c193e8092333cd9289499540463 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 11 Oct 2024 18:57:59 +0000 Subject: [PATCH 0915/2091] Refactor server_keys service/interface and related callsites Signed-off-by: Jason Volk Signed-off-by: strawberry --- Cargo.lock | 26 +- Cargo.toml | 2 +- src/admin/debug/commands.rs | 173 +++---- src/admin/debug/mod.rs | 17 +- src/admin/query/globals.rs | 13 +- src/api/client/membership.rs | 252 +++------ src/api/client/mod.rs | 2 +- src/api/router/args.rs | 2 +- src/api/router/auth.rs | 222 ++++---- src/api/server/invite.rs | 11 +- src/api/server/key.rs | 70 +-- src/api/server/send.rs | 22 +- src/api/server/send_join.rs | 24 +- src/api/server/send_leave.rs | 13 +- src/core/config/mod.rs | 28 - src/core/error/mod.rs | 4 + src/core/pdu/mod.rs | 25 +- src/service/globals/data.rs | 118 +---- src/service/globals/mod.rs | 50 +- src/service/rooms/event_handler/mod.rs | 105 ++-- src/service/rooms/timeline/mod.rs | 45 +- src/service/sending/mod.rs | 4 +- src/service/sending/send.rs | 24 +- src/service/server_keys/acquire.rs | 175 +++++++ src/service/server_keys/get.rs | 86 ++++ src/service/server_keys/keypair.rs | 64 +++ src/service/server_keys/mod.rs | 678 +++++-------------------- src/service/server_keys/request.rs | 97 ++++ src/service/server_keys/sign.rs | 18 + src/service/server_keys/verify.rs | 33 ++ 30 files changed, 1025 insertions(+), 1378 deletions(-) create mode 100644 src/service/server_keys/acquire.rs create mode 100644 src/service/server_keys/get.rs create mode 100644 src/service/server_keys/keypair.rs create mode 100644 src/service/server_keys/request.rs create mode 100644 src/service/server_keys/sign.rs create mode 100644 src/service/server_keys/verify.rs diff --git a/Cargo.lock b/Cargo.lock index db1394ce..4ac7cc35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2976,7 +2976,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "assign", "js_int", @@ -2998,7 +2998,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "js_int", "ruma-common", @@ -3010,7 +3010,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "as_variant", "assign", @@ -3033,7 +3033,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "as_variant", "base64 0.22.1", @@ -3063,7 +3063,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3087,7 +3087,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "bytes", "http", @@ -3105,7 +3105,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "js_int", "thiserror", @@ -3114,7 +3114,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "js_int", "ruma-common", @@ -3124,7 +3124,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "cfg-if", "once_cell", @@ -3140,7 +3140,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "js_int", "ruma-common", @@ -3152,7 +3152,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "headers", "http", @@ -3165,7 +3165,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3181,7 +3181,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=3109496a1f91357c89cbb57cf86f179e2cb013e7#3109496a1f91357c89cbb57cf86f179e2cb013e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" dependencies = [ "futures-util", "itertools 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index 0a98befd..966c2818 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "3109496a1f91357c89cbb57cf86f179e2cb013e7" +rev = "d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" features = [ "compat", "rand", diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index fd8c39f7..7fe8addf 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -1,19 +1,17 @@ use std::{ - collections::{BTreeMap, HashMap}, + collections::HashMap, fmt::Write, sync::Arc, time::{Instant, SystemTime}, }; -use api::client::validate_and_add_event_id; -use conduit::{debug, debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result}; +use conduit::{debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result}; use futures::StreamExt; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, }; -use tokio::sync::RwLock; use tracing_subscriber::EnvFilter; use crate::admin_command; @@ -219,7 +217,7 @@ pub(super) async fn get_remote_pdu( })?; trace!("Attempting to parse PDU: {:?}", &response.pdu); - let parsed_pdu = { + let _parsed_pdu = { let parsed_result = self .services .rooms @@ -241,22 +239,11 @@ pub(super) async fn get_remote_pdu( vec![(event_id, value, room_id)] }; - let pub_key_map = RwLock::new(BTreeMap::new()); - - debug!("Attempting to fetch homeserver signing keys for {server}"); - self.services - .server_keys - .fetch_required_signing_keys(parsed_pdu.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) - .await - .unwrap_or_else(|e| { - warn!("Could not fetch all signatures for PDUs from {server}: {e:?}"); - }); - info!("Attempting to handle event ID {event_id} as backfilled PDU"); self.services .rooms .timeline - .backfill_pdu(&server, response.pdu, &pub_key_map) + .backfill_pdu(&server, response.pdu) .await?; let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json"); @@ -433,12 +420,10 @@ pub(super) async fn sign_json(&self) -> Result { let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); match serde_json::from_str(&string) { Ok(mut value) => { - ruma::signatures::sign_json( - self.services.globals.server_name().as_str(), - self.services.globals.keypair(), - &mut value, - ) - .expect("our request json is what ruma expects"); + self.services + .server_keys + .sign_json(&mut value) + .expect("our request json is what ruma expects"); let json_text = serde_json::to_string_pretty(&value).expect("canonical json is valid json"); Ok(RoomMessageEventContent::text_plain(json_text)) }, @@ -456,27 +441,31 @@ pub(super) async fn verify_json(&self) -> Result { } let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); - match serde_json::from_str(&string) { - Ok(value) => { - let pub_key_map = RwLock::new(BTreeMap::new()); - - self.services - .server_keys - .fetch_required_signing_keys([&value], &pub_key_map) - .await?; - - let pub_key_map = pub_key_map.read().await; - match ruma::signatures::verify_json(&pub_key_map, &value) { - Ok(()) => Ok(RoomMessageEventContent::text_plain("Signature correct")), - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Signature verification failed: {e}" - ))), - } + match serde_json::from_str::(&string) { + Ok(value) => match self.services.server_keys.verify_json(&value, None).await { + Ok(()) => Ok(RoomMessageEventContent::text_plain("Signature correct")), + Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + "Signature verification failed: {e}" + ))), }, Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), } } +#[admin_command] +pub(super) async fn verify_pdu(&self, event_id: Box) -> Result { + let mut event = self.services.rooms.timeline.get_pdu_json(&event_id).await?; + + event.remove("event_id"); + let msg = match self.services.server_keys.verify_event(&event, None).await { + Ok(ruma::signatures::Verified::Signatures) => "signatures OK, but content hash failed (redaction).", + Ok(ruma::signatures::Verified::All) => "signatures and hashes OK.", + Err(e) => return Err(e), + }; + + Ok(RoomMessageEventContent::notice_plain(msg)) +} + #[admin_command] #[tracing::instrument(skip(self))] pub(super) async fn first_pdu_in_room(&self, room_id: Box) -> Result { @@ -557,7 +546,6 @@ pub(super) async fn force_set_room_state_from_server( let room_version = self.services.rooms.state.get_room_version(&room_id).await?; let mut state: HashMap> = HashMap::new(); - let pub_key_map = RwLock::new(BTreeMap::new()); let remote_state_response = self .services @@ -571,38 +559,28 @@ pub(super) async fn force_set_room_state_from_server( ) .await?; - let mut events = Vec::with_capacity(remote_state_response.pdus.len()); - for pdu in remote_state_response.pdus.clone() { - events.push( - match self - .services - .rooms - .event_handler - .parse_incoming_pdu(&pdu) - .await - { - Ok(t) => t, - Err(e) => { - warn!("Could not parse PDU, ignoring: {e}"); - continue; - }, + match self + .services + .rooms + .event_handler + .parse_incoming_pdu(&pdu) + .await + { + Ok(t) => t, + Err(e) => { + warn!("Could not parse PDU, ignoring: {e}"); + continue; }, - ); + }; } - info!("Fetching required signing keys for all the state events we got"); - self.services - .server_keys - .fetch_required_signing_keys(events.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) - .await?; - info!("Going through room_state response PDUs"); - for result in remote_state_response - .pdus - .iter() - .map(|pdu| validate_and_add_event_id(self.services, pdu, &room_version, &pub_key_map)) - { + for result in remote_state_response.pdus.iter().map(|pdu| { + self.services + .server_keys + .validate_and_add_event_id(pdu, &room_version) + }) { let Ok((event_id, value)) = result.await else { continue; }; @@ -630,11 +608,11 @@ pub(super) async fn force_set_room_state_from_server( } info!("Going through auth_chain response"); - for result in remote_state_response - .auth_chain - .iter() - .map(|pdu| validate_and_add_event_id(self.services, pdu, &room_version, &pub_key_map)) - { + for result in remote_state_response.auth_chain.iter().map(|pdu| { + self.services + .server_keys + .validate_and_add_event_id(pdu, &room_version) + }) { let Ok((event_id, value)) = result.await else { continue; }; @@ -686,10 +664,33 @@ pub(super) async fn force_set_room_state_from_server( #[admin_command] pub(super) async fn get_signing_keys( - &self, server_name: Option>, _cached: bool, + &self, server_name: Option>, notary: Option>, query: bool, ) -> Result { let server_name = server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); - let signing_keys = self.services.globals.signing_keys_for(&server_name).await?; + + if let Some(notary) = notary { + let signing_keys = self + .services + .server_keys + .notary_request(¬ary, &server_name) + .await?; + + return Ok(RoomMessageEventContent::notice_markdown(format!( + "```rs\n{signing_keys:#?}\n```" + ))); + } + + let signing_keys = if query { + self.services + .server_keys + .server_request(&server_name) + .await? + } else { + self.services + .server_keys + .signing_keys_for(&server_name) + .await? + }; Ok(RoomMessageEventContent::notice_markdown(format!( "```rs\n{signing_keys:#?}\n```" @@ -697,34 +698,20 @@ pub(super) async fn get_signing_keys( } #[admin_command] -#[allow(dead_code)] -pub(super) async fn get_verify_keys( - &self, server_name: Option>, cached: bool, -) -> Result { +pub(super) async fn get_verify_keys(&self, server_name: Option>) -> Result { let server_name = server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); - let mut out = String::new(); - if cached { - writeln!(out, "| Key ID | VerifyKey |")?; - writeln!(out, "| --- | --- |")?; - for (key_id, verify_key) in self.services.globals.verify_keys_for(&server_name).await? { - writeln!(out, "| {key_id} | {verify_key:?} |")?; - } - - return Ok(RoomMessageEventContent::notice_markdown(out)); - } - - let signature_ids: Vec = Vec::new(); let keys = self .services .server_keys - .fetch_signing_keys_for_server(&server_name, signature_ids) - .await?; + .verify_keys_for(&server_name) + .await; + let mut out = String::new(); writeln!(out, "| Key ID | Public Key |")?; writeln!(out, "| --- | --- |")?; for (key_id, key) in keys { - writeln!(out, "| {key_id} | {key} |")?; + writeln!(out, "| {key_id} | {key:?} |")?; } Ok(RoomMessageEventContent::notice_markdown(out)) diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 20ddbf2f..b74e9c36 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -80,8 +80,16 @@ pub(super) enum DebugCommand { GetSigningKeys { server_name: Option>, + #[arg(long)] + notary: Option>, + #[arg(short, long)] - cached: bool, + query: bool, + }, + + /// - Get and display signing keys from local cache or remote server. + GetVerifyKeys { + server_name: Option>, }, /// - Sends a federation request to the remote server's @@ -119,6 +127,13 @@ pub(super) enum DebugCommand { /// the command. VerifyJson, + /// - Verify PDU + /// + /// This re-verifies a PDU existing in the database found by ID. + VerifyPdu { + event_id: Box, + }, + /// - Prints the very first PDU in the specified room (typically /// m.room.create) FirstPduInRoom { diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 150a213c..837d34e6 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -13,8 +13,6 @@ pub(crate) enum GlobalsCommand { LastCheckForUpdatesId, - LoadKeypair, - /// - This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. SigningKeysFor { @@ -54,20 +52,11 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - GlobalsCommand::LoadKeypair => { - let timer = tokio::time::Instant::now(); - let results = services.globals.db.load_keypair(); - let query_time = timer.elapsed(); - - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) - }, GlobalsCommand::SigningKeysFor { origin, } => { let timer = tokio::time::Instant::now(); - let results = services.globals.db.verify_keys_for(&origin).await; + let results = services.server_keys.verify_keys_for(&origin).await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index a7a5b166..2fa34ff7 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1,17 +1,16 @@ use std::{ - collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, net::IpAddr, sync::Arc, - time::Instant, }; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - debug, debug_error, debug_warn, err, error, info, + debug, debug_info, debug_warn, err, error, info, pdu, pdu::{gen_event_id_canonical_json, PduBuilder}, trace, utils, - utils::{math::continue_exponential_backoff_secs, IterStream, ReadyExt}, + utils::{IterStream, ReadyExt}, warn, Err, Error, PduEvent, Result, }; use futures::{FutureExt, StreamExt}; @@ -36,13 +35,10 @@ use ruma::{ }, StateEventType, }, - serde::Base64, - state_res, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, - OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, + state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, + RoomVersionId, ServerName, UserId, }; -use serde_json::value::RawValue as RawJsonValue; use service::{appservice::RegistrationInfo, rooms::state::RoomMutexGuard, Services}; -use tokio::sync::RwLock; use crate::{client::full_user_deactivate, Ruma}; @@ -670,20 +666,22 @@ pub async fn join_room_by_id_helper( if local_join { join_room_by_id_helper_local(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) .boxed() - .await + .await?; } else { // Ask a remote server if we are not participating in this room join_room_by_id_helper_remote(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) .boxed() - .await + .await?; } + + Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) } #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote")] async fn join_room_by_id_helper_remote( services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, -) -> Result { +) -> Result { info!("Joining {room_id} over federation."); let (make_join_response, remote_server) = make_join_request(services, sender_user, room_id, servers).await?; @@ -751,43 +749,33 @@ async fn join_room_by_id_helper_remote( // In order to create a compatible ref hash (EventID) the `hashes` field needs // to be present - ruma::signatures::hash_and_sign_event( - services.globals.server_name().as_str(), - services.globals.keypair(), - &mut join_event_stub, - &room_version_id, - ) - .expect("event is valid, we just created it"); + services + .server_keys + .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; // Generate event id - let event_id = format!( - "${}", - ruma::signatures::reference_hash(&join_event_stub, &room_version_id) - .expect("ruma can calculate reference hashes") - ); - let event_id = <&EventId>::try_from(event_id.as_str()).expect("ruma's reference hashes are valid event ids"); + let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?; // Add event_id back - join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); + join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); // It has enough fields to be called a proper event now let mut join_event = join_event_stub; info!("Asking {remote_server} for send_join in room {room_id}"); + let send_join_request = federation::membership::create_join_event::v2::Request { + room_id: room_id.to_owned(), + event_id: event_id.clone(), + omit_members: false, + pdu: services + .sending + .convert_to_outgoing_federation_event(join_event.clone()) + .await, + }; + let send_join_response = services .sending - .send_federation_request( - &remote_server, - federation::membership::create_join_event::v2::Request { - room_id: room_id.to_owned(), - event_id: event_id.to_owned(), - omit_members: false, - pdu: services - .sending - .convert_to_outgoing_federation_event(join_event.clone()) - .await, - }, - ) + .send_federation_request(&remote_server, send_join_request) .await?; info!("send_join finished"); @@ -805,7 +793,7 @@ async fn join_room_by_id_helper_remote( // validate and send signatures _ => { if let Some(signed_raw) = &send_join_response.room_state.event { - info!( + debug_info!( "There is a signed event. This room is probably using restricted joins. Adding signature to \ our event" ); @@ -862,25 +850,25 @@ async fn join_room_by_id_helper_remote( .await; info!("Parsing join event"); - let parsed_join_pdu = PduEvent::from_id_val(event_id, join_event.clone()) + let parsed_join_pdu = PduEvent::from_id_val(&event_id, join_event.clone()) .map_err(|e| err!(BadServerResponse("Invalid join event PDU: {e:?}")))?; - let mut state = HashMap::new(); - let pub_key_map = RwLock::new(BTreeMap::new()); - - info!("Fetching join signing keys"); + info!("Acquiring server signing keys for response events"); + let resp_events = &send_join_response.room_state; + let resp_state = &resp_events.state; + let resp_auth = &resp_events.auth_chain; services .server_keys - .fetch_join_signing_keys(&send_join_response, &room_version_id, &pub_key_map) - .await?; + .acquire_events_pubkeys(resp_auth.iter().chain(resp_state.iter())) + .await; info!("Going through send_join response room_state"); - for result in send_join_response - .room_state - .state - .iter() - .map(|pdu| validate_and_add_event_id(services, pdu, &room_version_id, &pub_key_map)) - { + let mut state = HashMap::new(); + for result in send_join_response.room_state.state.iter().map(|pdu| { + services + .server_keys + .validate_and_add_event_id(pdu, &room_version_id) + }) { let Ok((event_id, value)) = result.await else { continue; }; @@ -902,12 +890,11 @@ async fn join_room_by_id_helper_remote( } info!("Going through send_join response auth_chain"); - for result in send_join_response - .room_state - .auth_chain - .iter() - .map(|pdu| validate_and_add_event_id(services, pdu, &room_version_id, &pub_key_map)) - { + for result in send_join_response.room_state.auth_chain.iter().map(|pdu| { + services + .server_keys + .validate_and_add_event_id(pdu, &room_version_id) + }) { let Ok((event_id, value)) = result.await else { continue; }; @@ -937,29 +924,22 @@ async fn join_room_by_id_helper_remote( return Err!(Request(Forbidden("Auth check failed"))); } - info!("Saving state from send_join"); + info!("Compressing state from send_join"); + let compressed = state + .iter() + .stream() + .then(|(&k, id)| services.rooms.state_compressor.compress_state_event(k, id)) + .collect() + .await; + + debug!("Saving compressed state"); let (statehash_before_join, new, removed) = services .rooms .state_compressor - .save_state( - room_id, - Arc::new( - state - .into_iter() - .stream() - .then(|(k, id)| async move { - services - .rooms - .state_compressor - .compress_state_event(k, &id) - .await - }) - .collect() - .await, - ), - ) + .save_state(room_id, Arc::new(compressed)) .await?; + debug!("Forcing state for new room"); services .rooms .state @@ -1002,14 +982,14 @@ async fn join_room_by_id_helper_remote( .state .set_room_state(room_id, statehash_after_join, &state_lock); - Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) + Ok(()) } #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_local")] async fn join_room_by_id_helper_local( services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, -) -> Result { +) -> Result { debug!("We can join locally"); let join_rules_event_content = services @@ -1089,7 +1069,7 @@ async fn join_room_by_id_helper_local( ) .await { - Ok(_event_id) => return Ok(join_room_by_id::v3::Response::new(room_id.to_owned())), + Ok(_) => return Ok(()), Err(e) => e, }; @@ -1159,24 +1139,15 @@ async fn join_room_by_id_helper_local( // In order to create a compatible ref hash (EventID) the `hashes` field needs // to be present - ruma::signatures::hash_and_sign_event( - services.globals.server_name().as_str(), - services.globals.keypair(), - &mut join_event_stub, - &room_version_id, - ) - .expect("event is valid, we just created it"); + services + .server_keys + .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; // Generate event id - let event_id = format!( - "${}", - ruma::signatures::reference_hash(&join_event_stub, &room_version_id) - .expect("ruma can calculate reference hashes") - ); - let event_id = <&EventId>::try_from(event_id.as_str()).expect("ruma's reference hashes are valid event ids"); + let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?; // Add event_id back - join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); + join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); // It has enough fields to be called a proper event now let join_event = join_event_stub; @@ -1187,7 +1158,7 @@ async fn join_room_by_id_helper_local( &remote_server, federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), - event_id: event_id.to_owned(), + event_id: event_id.clone(), omit_members: false, pdu: services .sending @@ -1214,15 +1185,10 @@ async fn join_room_by_id_helper_local( } drop(state_lock); - let pub_key_map = RwLock::new(BTreeMap::new()); - services - .server_keys - .fetch_required_signing_keys([&signed_value], &pub_key_map) - .await?; services .rooms .event_handler - .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true, &pub_key_map) + .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true) .await?; } else { return Err(error); @@ -1231,7 +1197,7 @@ async fn join_room_by_id_helper_local( return Err(error); } - Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) + Ok(()) } async fn make_join_request( @@ -1301,62 +1267,6 @@ async fn make_join_request( make_join_response_and_server } -pub async fn validate_and_add_event_id( - services: &Services, pdu: &RawJsonValue, room_version: &RoomVersionId, - pub_key_map: &RwLock>>, -) -> Result<(OwnedEventId, CanonicalJsonObject)> { - let mut value: CanonicalJsonObject = serde_json::from_str(pdu.get()) - .map_err(|e| err!(BadServerResponse(debug_error!("Invalid PDU in server response: {e:?}"))))?; - let event_id = EventId::parse(format!( - "${}", - ruma::signatures::reference_hash(&value, room_version).expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - let back_off = |id| async { - match services - .globals - .bad_event_ratelimiter - .write() - .expect("locked") - .entry(id) - { - Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - Entry::Occupied(mut e) => { - *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); - }, - } - }; - - if let Some((time, tries)) = services - .globals - .bad_event_ratelimiter - .read() - .expect("locked") - .get(&event_id) - { - // Exponential backoff - const MIN: u64 = 60 * 5; - const MAX: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) { - return Err!(BadServerResponse("bad event {event_id:?}, still backing off")); - } - } - - if let Err(e) = ruma::signatures::verify_event(&*pub_key_map.read().await, &value, room_version) { - debug_error!("Event {event_id} failed verification {pdu:#?}"); - let e = Err!(BadServerResponse(debug_error!("Event {event_id} failed verification: {e:?}"))); - back_off(event_id).await; - return e; - } - - value.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); - - Ok((event_id, value)) -} - pub(crate) async fn invite_helper( services: &Services, sender_user: &UserId, user_id: &UserId, room_id: &RoomId, reason: Option, is_direct: bool, @@ -1423,8 +1333,6 @@ pub(crate) async fn invite_helper( ) .await?; - let pub_key_map = RwLock::new(BTreeMap::new()); - // We do not add the event_id field to the pdu here because of signature and // hashes checks let Ok((event_id, value)) = gen_event_id_canonical_json(&response.event, &room_version_id) else { @@ -1452,15 +1360,10 @@ pub(crate) async fn invite_helper( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Origin field is invalid."))?; - services - .server_keys - .fetch_required_signing_keys([&value], &pub_key_map) - .await?; - let pdu_id: Vec = services .rooms .event_handler - .handle_incoming_pdu(&origin, room_id, &event_id, value, true, &pub_key_map) + .handle_incoming_pdu(&origin, room_id, &event_id, value, true) .await? .ok_or(Error::BadRequest( ErrorKind::InvalidParam, @@ -1714,24 +1617,15 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room // In order to create a compatible ref hash (EventID) the `hashes` field needs // to be present - ruma::signatures::hash_and_sign_event( - services.globals.server_name().as_str(), - services.globals.keypair(), - &mut leave_event_stub, - &room_version_id, - ) - .expect("event is valid, we just created it"); + services + .server_keys + .hash_and_sign_event(&mut leave_event_stub, &room_version_id)?; // Generate event id - let event_id = EventId::parse(format!( - "${}", - ruma::signatures::reference_hash(&leave_event_stub, &room_version_id) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); + let event_id = pdu::gen_event_id(&leave_event_stub, &room_version_id)?; // Add event_id back - leave_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); + leave_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); // It has enough fields to be called a proper event now let leave_event = leave_event_stub; diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index 4b7b64b9..2928be87 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -52,7 +52,7 @@ pub(super) use keys::*; pub(super) use media::*; pub(super) use media_legacy::*; pub(super) use membership::*; -pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room, validate_and_add_event_id}; +pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room}; pub(super) use message::*; pub(super) use openid::*; pub(super) use presence::*; diff --git a/src/api/router/args.rs b/src/api/router/args.rs index 7381a55f..746e1cfc 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -48,7 +48,7 @@ where async fn from_request(request: hyper::Request, services: &State) -> Result { let mut request = request::from(services, request).await?; let mut json_body = serde_json::from_slice::(&request.body).ok(); - let auth = auth::auth(services, &mut request, &json_body, &T::METADATA).await?; + let auth = auth::auth(services, &mut request, json_body.as_ref(), &T::METADATA).await?; Ok(Self { body: make_body::(services, &mut request, &mut json_body, &auth)?, origin: auth.origin, diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 8d76b4be..6b90c5ff 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -1,19 +1,20 @@ -use std::collections::BTreeMap; - use axum::RequestPartsExt; use axum_extra::{ headers::{authorization::Bearer, Authorization}, typed_header::TypedHeaderRejectionReason, TypedHeader, }; -use conduit::{debug_info, warn, Err, Error, Result}; +use conduit::{debug_error, err, warn, Err, Error, Result}; use http::uri::PathAndQuery; use ruma::{ api::{client::error::ErrorKind, AuthScheme, Metadata}, server_util::authorization::XMatrix, - CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, + CanonicalJsonObject, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, +}; +use service::{ + server_keys::{PubKeyMap, PubKeys}, + Services, }; -use service::Services; use super::request::Request; use crate::service::appservice::RegistrationInfo; @@ -33,7 +34,7 @@ pub(super) struct Auth { } pub(super) async fn auth( - services: &Services, request: &mut Request, json_body: &Option, metadata: &Metadata, + services: &Services, request: &mut Request, json_body: Option<&CanonicalJsonValue>, metadata: &Metadata, ) -> Result { let bearer: Option>> = request.parts.extract().await?; let token = match &bearer { @@ -151,27 +152,24 @@ pub(super) async fn auth( } async fn auth_appservice(services: &Services, request: &Request, info: Box) -> Result { - let user_id = request + let user_id_default = + || UserId::parse_with_server_name(info.registration.sender_localpart.as_str(), services.globals.server_name()); + + let Ok(user_id) = request .query .user_id .clone() - .map_or_else( - || { - UserId::parse_with_server_name( - info.registration.sender_localpart.as_str(), - services.globals.server_name(), - ) - }, - UserId::parse, - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; + .map_or_else(user_id_default, UserId::parse) + else { + return Err!(Request(InvalidUsername("Username is invalid."))); + }; if !info.is_user_match(&user_id) { - return Err(Error::BadRequest(ErrorKind::Exclusive, "User is not in namespace.")); + return Err!(Request(Exclusive("User is not in namespace."))); } if !services.users.exists(&user_id).await { - return Err(Error::BadRequest(ErrorKind::forbidden(), "User does not exist.")); + return Err!(Request(Forbidden("User does not exist."))); } Ok(Auth { @@ -182,118 +180,104 @@ async fn auth_appservice(services: &Services, request: &Request, info: Box, -) -> Result { +async fn auth_server(services: &Services, request: &mut Request, body: Option<&CanonicalJsonValue>) -> Result { + type Member = (String, CanonicalJsonValue); + type Object = CanonicalJsonObject; + type Value = CanonicalJsonValue; + + let x_matrix = parse_x_matrix(request).await?; + auth_server_checks(services, &x_matrix)?; + + let destination = services.globals.server_name(); + let origin = &x_matrix.origin; + #[allow(clippy::or_fun_call)] + let signature_uri = request + .parts + .uri + .path_and_query() + .unwrap_or(&PathAndQuery::from_static("/")) + .to_string(); + + let signature: [Member; 1] = [(x_matrix.key.to_string(), Value::String(x_matrix.sig.to_string()))]; + let signatures: [Member; 1] = [(origin.to_string(), Value::Object(signature.into()))]; + let authorization: [Member; 5] = [ + ("destination".into(), Value::String(destination.into())), + ("method".into(), Value::String(request.parts.method.to_string())), + ("origin".into(), Value::String(origin.to_string())), + ("signatures".into(), Value::Object(signatures.into())), + ("uri".into(), Value::String(signature_uri)), + ]; + + let mut authorization: Object = authorization.into(); + if let Some(body) = body { + authorization.insert("content".to_owned(), body.clone()); + } + + let key = services + .server_keys + .get_verify_key(origin, &x_matrix.key) + .await + .map_err(|e| err!(Request(Forbidden(warn!("Failed to fetch signing keys: {e}")))))?; + + let keys: PubKeys = [(x_matrix.key.to_string(), key.key)].into(); + let keys: PubKeyMap = [(origin.to_string(), keys)].into(); + if let Err(e) = ruma::signatures::verify_json(&keys, authorization) { + debug_error!("Failed to verify federation request from {origin}: {e}"); + if request.parts.uri.to_string().contains('@') { + warn!( + "Request uri contained '@' character. Make sure your reverse proxy gives Conduit the raw uri (apache: \ + use nocanon)" + ); + } + + return Err!(Request(Forbidden("Failed to verify X-Matrix signatures."))); + } + + Ok(Auth { + origin: origin.to_owned().into(), + sender_user: None, + sender_device: None, + appservice_info: None, + }) +} + +fn auth_server_checks(services: &Services, x_matrix: &XMatrix) -> Result<()> { if !services.server.config.allow_federation { return Err!(Config("allow_federation", "Federation is disabled.")); } - let TypedHeader(Authorization(x_matrix)) = request - .parts - .extract::>>() - .await - .map_err(|e| { - warn!("Missing or invalid Authorization header: {e}"); - - let msg = match e.reason() { - TypedHeaderRejectionReason::Missing => "Missing Authorization header.", - TypedHeaderRejectionReason::Error(_) => "Invalid X-Matrix signatures.", - _ => "Unknown header-related error", - }; - - Error::BadRequest(ErrorKind::forbidden(), msg) - })?; + let destination = services.globals.server_name(); + if x_matrix.destination.as_deref() != Some(destination) { + return Err!(Request(Forbidden("Invalid destination."))); + } let origin = &x_matrix.origin; - if services .server .config .forbidden_remote_server_names .contains(origin) { - debug_info!("Refusing to accept inbound federation request to {origin}"); - return Err!(Request(Forbidden("Federation with this homeserver is not allowed."))); + return Err!(Request(Forbidden(debug_warn!("Federation requests from {origin} denied.")))); } - let signatures = - BTreeMap::from_iter([(x_matrix.key.clone(), CanonicalJsonValue::String(x_matrix.sig.to_string()))]); - let signatures = BTreeMap::from_iter([( - origin.as_str().to_owned(), - CanonicalJsonValue::Object( - signatures - .into_iter() - .map(|(k, v)| (k.to_string(), v)) - .collect(), - ), - )]); - - let server_destination = services.globals.server_name().as_str().to_owned(); - if let Some(destination) = x_matrix.destination.as_ref() { - if destination != &server_destination { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Invalid authorization.")); - } - } - - #[allow(clippy::or_fun_call)] - let signature_uri = CanonicalJsonValue::String( - request - .parts - .uri - .path_and_query() - .unwrap_or(&PathAndQuery::from_static("/")) - .to_string(), - ); - - let mut request_map = BTreeMap::from_iter([ - ( - "method".to_owned(), - CanonicalJsonValue::String(request.parts.method.to_string()), - ), - ("uri".to_owned(), signature_uri), - ("origin".to_owned(), CanonicalJsonValue::String(origin.as_str().to_owned())), - ("destination".to_owned(), CanonicalJsonValue::String(server_destination)), - ("signatures".to_owned(), CanonicalJsonValue::Object(signatures)), - ]); - - if let Some(json_body) = json_body { - request_map.insert("content".to_owned(), json_body.clone()); - }; - - let keys_result = services - .server_keys - .fetch_signing_keys_for_server(origin, vec![x_matrix.key.to_string()]) - .await; - - let keys = keys_result.map_err(|e| { - warn!("Failed to fetch signing keys: {e}"); - Error::BadRequest(ErrorKind::forbidden(), "Failed to fetch signing keys.") - })?; - - let pub_key_map = BTreeMap::from_iter([(origin.as_str().to_owned(), keys)]); - - match ruma::signatures::verify_json(&pub_key_map, &request_map) { - Ok(()) => Ok(Auth { - origin: Some(origin.clone()), - sender_user: None, - sender_device: None, - appservice_info: None, - }), - Err(e) => { - warn!("Failed to verify json request from {origin}: {e}\n{request_map:?}"); - - if request.parts.uri.to_string().contains('@') { - warn!( - "Request uri contained '@' character. Make sure your reverse proxy gives Conduit the raw uri \ - (apache: use nocanon)" - ); - } - - Err(Error::BadRequest( - ErrorKind::forbidden(), - "Failed to verify X-Matrix signatures.", - )) - }, - } + Ok(()) +} + +async fn parse_x_matrix(request: &mut Request) -> Result { + let TypedHeader(Authorization(x_matrix)) = request + .parts + .extract::>>() + .await + .map_err(|e| { + let msg = match e.reason() { + TypedHeaderRejectionReason::Missing => "Missing Authorization header.", + TypedHeaderRejectionReason::Error(_) => "Invalid X-Matrix signatures.", + _ => "Unknown header-related error", + }; + + err!(Request(Forbidden(warn!("{msg}: {e}")))) + })?; + + Ok(x_matrix) } diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index f02655e6..a9e404c5 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -85,13 +85,10 @@ pub(crate) async fn create_invite_route( .acl_check(invited_user.server_name(), &body.room_id) .await?; - ruma::signatures::hash_and_sign_event( - services.globals.server_name().as_str(), - services.globals.keypair(), - &mut signed_event, - &body.room_version, - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; + services + .server_keys + .hash_and_sign_event(&mut signed_event, &body.room_version) + .map_err(|e| err!(Request(InvalidParam("Failed to sign event: {e}"))))?; // Generate event id let event_id = EventId::parse(format!( diff --git a/src/api/server/key.rs b/src/api/server/key.rs index 686e4424..3913ce43 100644 --- a/src/api/server/key.rs +++ b/src/api/server/key.rs @@ -1,20 +1,16 @@ -use std::{ - collections::BTreeMap, - time::{Duration, SystemTime}, -}; +use std::{collections::BTreeMap, time::Duration}; use axum::{extract::State, response::IntoResponse, Json}; +use conduit::{utils::timepoint_from_now, Result}; use ruma::{ api::{ - federation::discovery::{get_server_keys, ServerSigningKeys, VerifyKey}, + federation::discovery::{get_server_keys, ServerSigningKeys}, OutgoingResponse, }, - serde::{Base64, Raw}, - MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, + serde::Raw, + MilliSecondsSinceUnixEpoch, }; -use crate::Result; - /// # `GET /_matrix/key/v2/server` /// /// Gets the public signing keys of this server. @@ -24,47 +20,33 @@ use crate::Result; // Response type for this endpoint is Json because we need to calculate a // signature for the response pub(crate) async fn get_server_keys_route(State(services): State) -> Result { - let verify_keys: BTreeMap = BTreeMap::from([( - format!("ed25519:{}", services.globals.keypair().version()) - .try_into() - .expect("found invalid server signing keys in DB"), - VerifyKey { - key: Base64::new(services.globals.keypair().public_key().to_vec()), - }, - )]); + let server_name = services.globals.server_name(); + let verify_keys = services.server_keys.verify_keys_for(server_name).await; + let server_key = ServerSigningKeys { + verify_keys, + server_name: server_name.to_owned(), + valid_until_ts: valid_until_ts(), + old_verify_keys: BTreeMap::new(), + signatures: BTreeMap::new(), + }; - let mut response = serde_json::from_slice( - get_server_keys::v2::Response { - server_key: Raw::new(&ServerSigningKeys { - server_name: services.globals.server_name().to_owned(), - verify_keys, - old_verify_keys: BTreeMap::new(), - signatures: BTreeMap::new(), - valid_until_ts: MilliSecondsSinceUnixEpoch::from_system_time( - SystemTime::now() - .checked_add(Duration::from_secs(86400 * 7)) - .expect("valid_until_ts should not get this high"), - ) - .expect("time is valid"), - }) - .expect("static conversion, no errors"), - } - .try_into_http_response::>() - .unwrap() - .body(), - ) - .unwrap(); + let response = get_server_keys::v2::Response { + server_key: Raw::new(&server_key)?, + } + .try_into_http_response::>()?; - ruma::signatures::sign_json( - services.globals.server_name().as_str(), - services.globals.keypair(), - &mut response, - ) - .unwrap(); + let mut response = serde_json::from_slice(response.body())?; + services.server_keys.sign_json(&mut response)?; Ok(Json(response)) } +fn valid_until_ts() -> MilliSecondsSinceUnixEpoch { + let dur = Duration::from_secs(86400 * 7); + let timepoint = timepoint_from_now(dur).expect("SystemTime should not overflow"); + MilliSecondsSinceUnixEpoch::from_system_time(timepoint).expect("UInt should not overflow") +} + /// # `GET /_matrix/key/v2/server/{keyId}` /// /// Gets the public signing keys of this server. diff --git a/src/api/server/send.rs b/src/api/server/send.rs index f6916ccf..40f9403b 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -21,7 +21,6 @@ use ruma::{ OwnedEventId, ServerName, }; use serde_json::value::RawValue as RawJsonValue; -use tokio::sync::RwLock; use crate::{ services::Services, @@ -109,22 +108,6 @@ async fn handle_pdus( // and hashes checks } - // We go through all the signatures we see on the PDUs and fetch the - // corresponding signing keys - let pub_key_map = RwLock::new(BTreeMap::new()); - if !parsed_pdus.is_empty() { - services - .server_keys - .fetch_required_signing_keys(parsed_pdus.iter().map(|(_event_id, event, _room_id)| event), &pub_key_map) - .await - .unwrap_or_else(|e| warn!("Could not fetch all signatures for PDUs from {origin}: {e:?}")); - - debug!( - elapsed = ?txn_start_time.elapsed(), - "Fetched signing keys" - ); - } - let mut resolved_map = BTreeMap::new(); for (event_id, value, room_id) in parsed_pdus { let pdu_start_time = Instant::now(); @@ -134,17 +117,18 @@ async fn handle_pdus( .mutex_federation .lock(&room_id) .await; + resolved_map.insert( event_id.clone(), services .rooms .event_handler - .handle_incoming_pdu(origin, &room_id, &event_id, value, true, &pub_key_map) + .handle_incoming_pdu(origin, &room_id, &event_id, value, true) .await .map(|_| ()), ); - drop(mutex_lock); + drop(mutex_lock); debug!( pdu_elapsed = ?pdu_start_time.elapsed(), txn_elapsed = ?txn_start_time.elapsed(), diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index f9257690..d888d75e 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -1,6 +1,6 @@ #![allow(deprecated)] -use std::{borrow::Borrow, collections::BTreeMap}; +use std::borrow::Borrow; use axum::extract::State; use conduit::{err, pdu::gen_event_id_canonical_json, utils::IterStream, warn, Error, Result}; @@ -15,7 +15,6 @@ use ruma::{ }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::Services; -use tokio::sync::RwLock; use crate::Ruma; @@ -43,9 +42,6 @@ async fn create_join_event( .await .map_err(|_| err!(Request(NotFound("Event state not found."))))?; - let pub_key_map = RwLock::new(BTreeMap::new()); - // let mut auth_cache = EventMap::new(); - // We do not add the event_id field to the pdu here because of signature and // hashes checks let room_version_id = services.rooms.state.get_room_version(room_id).await?; @@ -137,20 +133,12 @@ async fn create_join_event( .await .unwrap_or_default() { - ruma::signatures::hash_and_sign_event( - services.globals.server_name().as_str(), - services.globals.keypair(), - &mut value, - &room_version_id, - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Failed to sign event."))?; + services + .server_keys + .hash_and_sign_event(&mut value, &room_version_id) + .map_err(|e| err!(Request(InvalidParam("Failed to sign event: {e}"))))?; } - services - .server_keys - .fetch_required_signing_keys([&value], &pub_key_map) - .await?; - let origin: OwnedServerName = serde_json::from_value( serde_json::to_value( value @@ -171,7 +159,7 @@ async fn create_join_event( let pdu_id: Vec = services .rooms .event_handler - .handle_incoming_pdu(&origin, room_id, &event_id, value.clone(), true, &pub_key_map) + .handle_incoming_pdu(&origin, room_id, &event_id, value.clone(), true) .await? .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index 81f41af0..0530f9dd 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -1,7 +1,5 @@ #![allow(deprecated)] -use std::collections::BTreeMap; - use axum::extract::State; use conduit::{utils::ReadyExt, Error, Result}; use ruma::{ @@ -13,7 +11,6 @@ use ruma::{ OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::RawValue as RawJsonValue; -use tokio::sync::RwLock; use crate::{ service::{pdu::gen_event_id_canonical_json, Services}, @@ -60,8 +57,6 @@ async fn create_leave_event( .acl_check(origin, room_id) .await?; - let pub_key_map = RwLock::new(BTreeMap::new()); - // We do not add the event_id field to the pdu here because of signature and // hashes checks let room_version_id = services.rooms.state.get_room_version(room_id).await?; @@ -154,21 +149,17 @@ async fn create_leave_event( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; - services - .server_keys - .fetch_required_signing_keys([&value], &pub_key_map) - .await?; - let mutex_lock = services .rooms .event_handler .mutex_federation .lock(room_id) .await; + let pdu_id: Vec = services .rooms .event_handler - .handle_incoming_pdu(&origin, room_id, &event_id, value, true, &pub_key_map) + .handle_incoming_pdu(&origin, room_id, &event_id, value, true) .await? .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index b5e07da2..114c6e76 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -490,30 +490,6 @@ pub struct Config { #[serde(default = "default_trusted_servers")] pub trusted_servers: Vec, - /// Option to control whether conduwuit will query your list of trusted - /// notary key servers (`trusted_servers`) for remote homeserver signing - /// keys it doesn't know *first*, or query the individual servers first - /// before falling back to the trusted key servers. - /// - /// The former/default behaviour makes federated/remote rooms joins - /// generally faster because we're querying a single (or list of) server - /// that we know works, is reasonably fast, and is reliable for just about - /// all the homeserver signing keys in the room. Querying individual - /// servers may take longer depending on the general infrastructure of - /// everyone in there, how many dead servers there are, etc. - /// - /// However, this does create an increased reliance on one single or - /// multiple large entities as `trusted_servers` should generally - /// contain long-term and large servers who know a very large number of - /// homeservers. - /// - /// If you don't know what any of this means, leave this and - /// `trusted_servers` alone to their defaults. - /// - /// Defaults to true as this is the fastest option for federation. - #[serde(default = "true_fn")] - pub query_trusted_key_servers_first: bool, - /// max log level for conduwuit. allows debug, info, warn, or error /// see also: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives /// **Caveat**: @@ -1518,10 +1494,6 @@ impl fmt::Display for Config { .map(|server| server.host()) .join(", "), ); - line( - "Query Trusted Key Servers First", - &self.query_trusted_key_servers_first.to_string(), - ); line("OpenID Token TTL", &self.openid_token_ttl.to_string()); line( "TURN username", diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 39fa4340..42250a0c 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -85,6 +85,8 @@ pub enum Error { BadRequest(ruma::api::client::error::ErrorKind, &'static str), //TODO: remove #[error("{0}")] BadServerResponse(Cow<'static, str>), + #[error(transparent)] + CanonicalJson(#[from] ruma::CanonicalJsonError), #[error("There was a problem with the '{0}' directive in your configuration: {1}")] Config(&'static str, Cow<'static, str>), #[error("{0}")] @@ -110,6 +112,8 @@ pub enum Error { #[error(transparent)] Ruma(#[from] ruma::api::client::error::Error), #[error(transparent)] + Signatures(#[from] ruma::signatures::Error), + #[error(transparent)] StateRes(#[from] ruma::state_res::Error), #[error("uiaa")] Uiaa(ruma::api::client::uiaa::UiaaInfo), diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 5f50fe5b..274b96bd 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -408,10 +408,13 @@ impl PduEvent { serde_json::from_value(json).expect("Raw::from_value always works") } - pub fn from_id_val(event_id: &EventId, mut json: CanonicalJsonObject) -> Result { - json.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); + pub fn from_id_val(event_id: &EventId, mut json: CanonicalJsonObject) -> Result { + json.insert("event_id".into(), CanonicalJsonValue::String(event_id.into())); - serde_json::from_value(serde_json::to_value(json).expect("valid JSON")) + let value = serde_json::to_value(json)?; + let pdu = serde_json::from_value(value)?; + + Ok(pdu) } } @@ -462,13 +465,15 @@ pub fn gen_event_id_canonical_json( let value: CanonicalJsonObject = serde_json::from_str(pdu.get()) .map_err(|e| err!(BadServerResponse(warn!("Error parsing incoming event: {e:?}"))))?; - let event_id = format!( - "${}", - // Anything higher than version3 behaves the same - ruma::signatures::reference_hash(&value, room_version_id).expect("ruma can calculate reference hashes") - ) - .try_into() - .expect("ruma's reference hashes are valid event ids"); + let event_id = gen_event_id(&value, room_version_id)?; Ok((event_id, value)) } + +/// Generates a correct eventId for the incoming pdu. +pub fn gen_event_id(value: &CanonicalJsonObject, room_version_id: &RoomVersionId) -> Result { + let reference_hash = ruma::signatures::reference_hash(value, room_version_id)?; + let event_id: OwnedEventId = format!("${reference_hash}").try_into()?; + + Ok(event_id) +} diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 3638cb56..eea7597a 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -1,16 +1,9 @@ -use std::{ - collections::BTreeMap, - sync::{Arc, RwLock}, -}; +use std::sync::{Arc, RwLock}; -use conduit::{trace, utils, utils::rand, Error, Result, Server}; -use database::{Database, Deserialized, Json, Map}; +use conduit::{trace, utils, Result, Server}; +use database::{Database, Deserialized, Map}; use futures::{pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; -use ruma::{ - api::federation::discovery::{ServerSigningKeys, VerifyKey}, - signatures::Ed25519KeyPair, - DeviceId, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, ServerName, UserId, -}; +use ruma::{DeviceId, UserId}; use crate::{rooms, Dep}; @@ -25,7 +18,6 @@ pub struct Data { pduid_pdu: Arc, keychangeid_userid: Arc, roomusertype_roomuserdataid: Arc, - server_signingkeys: Arc, readreceiptid_readreceipt: Arc, userid_lastonetimekeyupdate: Arc, counter: RwLock, @@ -56,7 +48,6 @@ impl Data { pduid_pdu: db["pduid_pdu"].clone(), keychangeid_userid: db["keychangeid_userid"].clone(), roomusertype_roomuserdataid: db["roomusertype_roomuserdataid"].clone(), - server_signingkeys: db["server_signingkeys"].clone(), readreceiptid_readreceipt: db["readreceiptid_readreceipt"].clone(), userid_lastonetimekeyupdate: db["userid_lastonetimekeyupdate"].clone(), counter: RwLock::new(Self::stored_count(&db["global"]).expect("initialized global counter")), @@ -205,107 +196,6 @@ impl Data { Ok(()) } - pub fn load_keypair(&self) -> Result { - let generate = |_| { - let keypair = Ed25519KeyPair::generate().expect("Ed25519KeyPair generation always works (?)"); - - let mut value = rand::string(8).as_bytes().to_vec(); - value.push(0xFF); - value.extend_from_slice(&keypair); - - self.global.insert(b"keypair", &value); - value - }; - - let keypair_bytes: Vec = self - .global - .get_blocking(b"keypair") - .map_or_else(generate, Into::into); - - let mut parts = keypair_bytes.splitn(2, |&b| b == 0xFF); - utils::string_from_bytes( - // 1. version - parts - .next() - .expect("splitn always returns at least one element"), - ) - .map_err(|_| Error::bad_database("Invalid version bytes in keypair.")) - .and_then(|version| { - // 2. key - parts - .next() - .ok_or_else(|| Error::bad_database("Invalid keypair format in database.")) - .map(|key| (version, key)) - }) - .and_then(|(version, key)| { - Ed25519KeyPair::from_der(key, version) - .map_err(|_| Error::bad_database("Private or public keys are invalid.")) - }) - } - - #[inline] - pub fn remove_keypair(&self) -> Result<()> { - self.global.remove(b"keypair"); - Ok(()) - } - - /// TODO: the key valid until timestamp (`valid_until_ts`) is only honored - /// in room version > 4 - /// - /// Remove the outdated keys and insert the new ones. - /// - /// This doesn't actually check that the keys provided are newer than the - /// old set. - pub async fn add_signing_key( - &self, origin: &ServerName, new_keys: ServerSigningKeys, - ) -> BTreeMap { - // (timo) Not atomic, but this is not critical - let mut keys: ServerSigningKeys = self - .server_signingkeys - .get(origin) - .await - .deserialized() - .unwrap_or_else(|_| { - // Just insert "now", it doesn't matter - ServerSigningKeys::new(origin.to_owned(), MilliSecondsSinceUnixEpoch::now()) - }); - - keys.verify_keys.extend(new_keys.verify_keys); - keys.old_verify_keys.extend(new_keys.old_verify_keys); - - self.server_signingkeys.raw_put(origin, Json(&keys)); - - let mut tree = keys.verify_keys; - tree.extend( - keys.old_verify_keys - .into_iter() - .map(|old| (old.0, VerifyKey::new(old.1.key))), - ); - - tree - } - - /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found - /// for the server. - pub async fn verify_keys_for(&self, origin: &ServerName) -> Result> { - self.signing_keys_for(origin).await.map_or_else( - |_| Ok(BTreeMap::new()), - |keys: ServerSigningKeys| { - let mut tree = keys.verify_keys; - tree.extend( - keys.old_verify_keys - .into_iter() - .map(|old| (old.0, VerifyKey::new(old.1.key))), - ); - Ok(tree) - }, - ) - } - - pub async fn signing_keys_for(&self, origin: &ServerName) -> Result { - self.server_signingkeys.get(origin).await.deserialized() - } - pub async fn database_version(&self) -> u64 { self.global .get(b"version") diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index fb970f07..7680007d 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -2,7 +2,7 @@ mod data; pub(super) mod migrations; use std::{ - collections::{BTreeMap, HashMap}, + collections::HashMap, fmt::Write, sync::{Arc, RwLock}, time::Instant, @@ -13,13 +13,8 @@ use data::Data; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ - api::{ - client::discovery::discover_support::ContactRole, - federation::discovery::{ServerSigningKeys, VerifyKey}, - }, - serde::Base64, - DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomAliasId, - RoomVersionId, ServerName, UserId, + api::client::discovery::discover_support::ContactRole, DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedServerName, + OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, }; use tokio::sync::Mutex; use url::Url; @@ -31,7 +26,6 @@ pub struct Service { pub config: Config, pub cidr_range_denylist: Vec, - keypair: Arc, jwt_decoding_key: Option, pub stable_room_versions: Vec, pub unstable_room_versions: Vec, @@ -50,16 +44,6 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let db = Data::new(&args); let config = &args.server.config; - let keypair = db.load_keypair(); - - let keypair = match keypair { - Ok(k) => k, - Err(e) => { - error!("Keypair invalid. Deleting..."); - db.remove_keypair()?; - return Err(e); - }, - }; let jwt_decoding_key = config .jwt_secret @@ -115,7 +99,6 @@ impl crate::Service for Service { db, config: config.clone(), cidr_range_denylist, - keypair: Arc::new(keypair), jwt_decoding_key, stable_room_versions, unstable_room_versions, @@ -175,9 +158,6 @@ impl crate::Service for Service { } impl Service { - /// Returns this server's keypair. - pub fn keypair(&self) -> &ruma::signatures::Ed25519KeyPair { &self.keypair } - #[inline] pub fn next_count(&self) -> Result { self.db.next_count() } @@ -224,8 +204,6 @@ impl Service { pub fn trusted_servers(&self) -> &[OwnedServerName] { &self.config.trusted_servers } - pub fn query_trusted_key_servers_first(&self) -> bool { self.config.query_trusted_key_servers_first } - pub fn jwt_decoding_key(&self) -> Option<&jsonwebtoken::DecodingKey> { self.jwt_decoding_key.as_ref() } pub fn turn_password(&self) -> &String { &self.config.turn_password } @@ -302,28 +280,6 @@ impl Service { } } - /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found - /// for the server. - pub async fn verify_keys_for(&self, origin: &ServerName) -> Result> { - let mut keys = self.db.verify_keys_for(origin).await?; - if origin == self.server_name() { - keys.insert( - format!("ed25519:{}", self.keypair().version()) - .try_into() - .expect("found invalid server signing keys in DB"), - VerifyKey { - key: Base64::new(self.keypair.public_key().to_vec()), - }, - ); - } - - Ok(keys) - } - - pub async fn signing_keys_for(&self, origin: &ServerName) -> Result { - self.db.signing_keys_for(origin).await - } - pub fn well_known_client(&self) -> &Option { &self.config.well_known.client } pub fn well_known_server(&self) -> &Option { &self.config.well_known.server } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index f8042b67..8448404b 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -28,12 +28,10 @@ use ruma::{ StateEventType, TimelineEventType, }, int, - serde::Base64, state_res::{self, EventTypeExt, RoomVersion, StateMap}, - uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, - ServerName, UserId, + uint, CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, + RoomId, RoomVersionId, ServerName, UserId, }; -use tokio::sync::RwLock; use super::state_compressor::CompressedStateEvent; use crate::{globals, rooms, sending, server_keys, Dep}; @@ -129,11 +127,10 @@ impl Service { /// 13. Use state resolution to find new room state /// 14. Check if the event passes auth based on the "current state" of the /// room, if not soft fail it - #[tracing::instrument(skip(self, origin, value, is_timeline_event, pub_key_map), name = "pdu")] + #[tracing::instrument(skip(self, origin, value, is_timeline_event), name = "pdu")] pub async fn handle_incoming_pdu<'a>( &self, origin: &'a ServerName, room_id: &'a RoomId, event_id: &'a EventId, value: BTreeMap, is_timeline_event: bool, - pub_key_map: &'a RwLock>>, ) -> Result>> { // 1. Skip the PDU if we already have it as a timeline event if let Ok(pdu_id) = self.services.timeline.get_pdu_id(event_id).await { @@ -177,7 +174,7 @@ impl Service { let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; let (incoming_pdu, val) = self - .handle_outlier_pdu(origin, &create_event, event_id, room_id, value, false, pub_key_map) + .handle_outlier_pdu(origin, &create_event, event_id, room_id, value, false) .boxed() .await?; @@ -200,7 +197,6 @@ impl Service { &create_event, room_id, &room_version_id, - pub_key_map, incoming_pdu.prev_events.clone(), ) .await?; @@ -212,7 +208,6 @@ impl Service { origin, event_id, room_id, - pub_key_map, &mut eventid_info, &create_event, &first_pdu_in_room, @@ -250,7 +245,7 @@ impl Service { .insert(room_id.to_owned(), (event_id.to_owned(), start_time)); let r = self - .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, &create_event, origin, room_id, pub_key_map) + .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, &create_event, origin, room_id) .await; self.federation_handletime @@ -264,12 +259,11 @@ impl Service { #[allow(clippy::type_complexity)] #[allow(clippy::too_many_arguments)] #[tracing::instrument( - skip(self, origin, event_id, room_id, pub_key_map, eventid_info, create_event, first_pdu_in_room), + skip(self, origin, event_id, room_id, eventid_info, create_event, first_pdu_in_room), name = "prev" )] pub async fn handle_prev_pdu<'a>( &self, origin: &'a ServerName, event_id: &'a EventId, room_id: &'a RoomId, - pub_key_map: &'a RwLock>>, eventid_info: &mut HashMap, (Arc, BTreeMap)>, create_event: &Arc, first_pdu_in_room: &Arc, prev_id: &EventId, ) -> Result<()> { @@ -318,7 +312,7 @@ impl Service { .expect("locked") .insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time)); - self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id, pub_key_map) + self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id) .await?; self.federation_handletime @@ -338,8 +332,7 @@ impl Service { #[allow(clippy::too_many_arguments)] async fn handle_outlier_pdu<'a>( &self, origin: &'a ServerName, create_event: &'a PduEvent, event_id: &'a EventId, room_id: &'a RoomId, - mut value: BTreeMap, auth_events_known: bool, - pub_key_map: &'a RwLock>>, + mut value: CanonicalJsonObject, auth_events_known: bool, ) -> Result<(Arc, BTreeMap)> { // 1. Remove unsigned field value.remove("unsigned"); @@ -349,14 +342,13 @@ impl Service { // 2. Check signatures, otherwise drop // 3. check content hash, redact if doesn't match let room_version_id = Self::get_room_version_id(create_event)?; - - let guard = pub_key_map.read().await; - let mut val = match ruma::signatures::verify_event(&guard, &value, &room_version_id) { - Err(e) => { - // Drop - warn!("Dropping bad event {event_id}: {e}"); - return Err!(Request(InvalidParam("Signature verification failed"))); - }, + let mut val = match self + .services + .server_keys + .verify_event(&value, Some(&room_version_id)) + .await + { + Ok(ruma::signatures::Verified::All) => value, Ok(ruma::signatures::Verified::Signatures) => { // Redact debug_info!("Calculated hash does not match (redaction): {event_id}"); @@ -371,11 +363,13 @@ impl Service { obj }, - Ok(ruma::signatures::Verified::All) => value, + Err(e) => { + return Err!(Request(InvalidParam(debug_error!( + "Signature verification failed for {event_id}: {e}" + )))) + }, }; - drop(guard); - // Now that we have checked the signature and hashes we can add the eventID and // convert to our PduEvent type val.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); @@ -404,7 +398,6 @@ impl Service { create_event, room_id, &room_version_id, - pub_key_map, ), ) .await; @@ -487,7 +480,7 @@ impl Service { pub async fn upgrade_outlier_to_timeline_pdu( &self, incoming_pdu: Arc, val: BTreeMap, create_event: &PduEvent, - origin: &ServerName, room_id: &RoomId, pub_key_map: &RwLock>>, + origin: &ServerName, room_id: &RoomId, ) -> Result>> { // Skip the PDU if we already have it as a timeline event if let Ok(pduid) = self @@ -526,14 +519,7 @@ impl Service { if state_at_incoming_event.is_none() { state_at_incoming_event = self - .fetch_state( - origin, - create_event, - room_id, - &room_version_id, - pub_key_map, - &incoming_pdu.event_id, - ) + .fetch_state(origin, create_event, room_id, &room_version_id, &incoming_pdu.event_id) .await?; } @@ -1021,10 +1007,10 @@ impl Service { /// Call /state_ids to find out what the state at this pdu is. We trust the /// server's response to some extend (sic), but we still do a lot of checks /// on the events - #[tracing::instrument(skip(self, pub_key_map, create_event, room_version_id))] + #[tracing::instrument(skip(self, create_event, room_version_id))] async fn fetch_state( &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, - pub_key_map: &RwLock>>, event_id: &EventId, + event_id: &EventId, ) -> Result>>> { debug!("Fetching state ids"); let res = self @@ -1048,7 +1034,7 @@ impl Service { .collect::>(); let state_vec = self - .fetch_and_handle_outliers(origin, &collect, create_event, room_id, room_version_id, pub_key_map) + .fetch_and_handle_outliers(origin, &collect, create_event, room_id, room_version_id) .boxed() .await; @@ -1102,7 +1088,7 @@ impl Service { /// d. TODO: Ask other servers over federation? pub async fn fetch_and_handle_outliers<'a>( &self, origin: &'a ServerName, events: &'a [Arc], create_event: &'a PduEvent, room_id: &'a RoomId, - room_version_id: &'a RoomVersionId, pub_key_map: &'a RwLock>>, + room_version_id: &'a RoomVersionId, ) -> Vec<(Arc, Option>)> { let back_off = |id| match self .services @@ -1222,22 +1208,6 @@ impl Service { events_with_auth_events.push((id, None, events_in_reverse_order)); } - // We go through all the signatures we see on the PDUs and their unresolved - // dependencies and fetch the corresponding signing keys - self.services - .server_keys - .fetch_required_signing_keys( - events_with_auth_events - .iter() - .flat_map(|(_id, _local_pdu, events)| events) - .map(|(_event_id, event)| event), - pub_key_map, - ) - .await - .unwrap_or_else(|e| { - warn!("Could not fetch all signatures for PDUs from {origin}: {e:?}"); - }); - let mut pdus = Vec::with_capacity(events_with_auth_events.len()); for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { // a. Look in the main timeline (pduid_pdu tree) @@ -1266,16 +1236,8 @@ impl Service { } } - match Box::pin(self.handle_outlier_pdu( - origin, - create_event, - &next_id, - room_id, - value.clone(), - true, - pub_key_map, - )) - .await + match Box::pin(self.handle_outlier_pdu(origin, create_event, &next_id, room_id, value.clone(), true)) + .await { Ok((pdu, json)) => { if next_id == *id { @@ -1296,7 +1258,7 @@ impl Service { #[tracing::instrument(skip_all)] async fn fetch_prev( &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, - pub_key_map: &RwLock>>, initial_set: Vec>, + initial_set: Vec>, ) -> Result<( Vec>, HashMap, (Arc, BTreeMap)>, @@ -1311,14 +1273,7 @@ impl Service { while let Some(prev_event_id) = todo_outlier_stack.pop() { if let Some((pdu, mut json_opt)) = self - .fetch_and_handle_outliers( - origin, - &[prev_event_id.clone()], - create_event, - room_id, - room_version_id, - pub_key_map, - ) + .fetch_and_handle_outliers(origin, &[prev_event_id.clone()], create_event, room_id, room_version_id) .boxed() .await .pop() diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 21e5395d..902e50ff 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -16,7 +16,7 @@ use conduit::{ }; use futures::{future, future::ready, Future, FutureExt, Stream, StreamExt, TryStreamExt}; use ruma::{ - api::{client::error::ErrorKind, federation}, + api::federation, canonical_json::to_canonical_value, events::{ push_rules::PushRulesEvent, @@ -30,14 +30,12 @@ use ruma::{ GlobalAccountDataEventType, StateEventType, TimelineEventType, }, push::{Action, Ruleset, Tweak}, - serde::Base64, state_res::{self, Event, RoomVersion}, uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, RoomId, RoomVersionId, ServerName, UserId, }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::sync::RwLock; use self::data::Data; pub use self::data::PdusIterItem; @@ -784,21 +782,15 @@ impl Service { to_canonical_value(self.services.globals.server_name()).expect("server name is a valid CanonicalJsonValue"), ); - match ruma::signatures::hash_and_sign_event( - self.services.globals.server_name().as_str(), - self.services.globals.keypair(), - &mut pdu_json, - &room_version_id, - ) { - Ok(()) => {}, - Err(e) => { - return match e { - ruma::signatures::Error::PduSize => { - Err(Error::BadRequest(ErrorKind::TooLarge, "Message is too long")) - }, - _ => Err(Error::BadRequest(ErrorKind::Unknown, "Signing event failed")), - } - }, + if let Err(e) = self + .services + .server_keys + .hash_and_sign_event(&mut pdu_json, &room_version_id) + { + return match e { + Error::Signatures(ruma::signatures::Error::PduSize) => Err!(Request(TooLarge("Message is too long"))), + _ => Err!(Request(Unknown("Signing event failed"))), + }; } // Generate event id @@ -1106,9 +1098,8 @@ impl Service { .await; match response { Ok(response) => { - let pub_key_map = RwLock::new(BTreeMap::new()); for pdu in response.pdus { - if let Err(e) = self.backfill_pdu(backfill_server, pdu, &pub_key_map).await { + if let Err(e) = self.backfill_pdu(backfill_server, pdu).await { warn!("Failed to add backfilled pdu in room {room_id}: {e}"); } } @@ -1124,11 +1115,8 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, pdu, pub_key_map))] - pub async fn backfill_pdu( - &self, origin: &ServerName, pdu: Box, - pub_key_map: &RwLock>>, - ) -> Result<()> { + #[tracing::instrument(skip(self, pdu))] + pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> Result<()> { let (event_id, value, room_id) = self.services.event_handler.parse_incoming_pdu(&pdu).await?; // Lock so we cannot backfill the same pdu twice at the same time @@ -1146,14 +1134,9 @@ impl Service { return Ok(()); } - self.services - .server_keys - .fetch_required_signing_keys([&value], pub_key_map) - .await?; - self.services .event_handler - .handle_incoming_pdu(origin, &room_id, &event_id, value, false, pub_key_map) + .handle_incoming_pdu(origin, &room_id, &event_id, value, false) .await?; let value = self diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index e3582f2e..5970c383 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -17,7 +17,7 @@ use tokio::sync::Mutex; use self::data::Data; pub use self::dest::Destination; -use crate::{account_data, client, globals, presence, pusher, resolver, rooms, users, Dep}; +use crate::{account_data, client, globals, presence, pusher, resolver, rooms, server_keys, users, Dep}; pub struct Service { server: Arc, @@ -41,6 +41,7 @@ struct Services { account_data: Dep, appservice: Dep, pusher: Dep, + server_keys: Dep, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -78,6 +79,7 @@ impl crate::Service for Service { account_data: args.depend::("account_data"), appservice: args.depend::("appservice"), pusher: args.depend::("pusher"), + server_keys: args.depend::("server_keys"), }, db: Data::new(&args), sender, diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 9a8f408b..73b6a468 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,8 +1,8 @@ use std::{fmt::Debug, mem}; use conduit::{ - debug, debug_error, debug_info, debug_warn, err, error::inspect_debug_log, trace, utils::string::EMPTY, Err, Error, - Result, + debug, debug_error, debug_info, debug_warn, err, error::inspect_debug_log, implement, trace, utils::string::EMPTY, + Err, Error, Result, }; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; @@ -18,7 +18,7 @@ use ruma::{ }; use crate::{ - globals, resolver, + resolver, resolver::{actual::ActualDest, cache::CachedDest}, }; @@ -75,7 +75,7 @@ impl super::Service { .try_into_http_request::>(&actual.string, SATIR, &VERSIONS) .map_err(|e| err!(BadServerResponse("Invalid destination: {e:?}")))?; - sign_request::(&self.services.globals, dest, &mut http_request); + self.sign_request::(dest, &mut http_request); let request = Request::try_from(http_request)?; self.validate_url(request.url())?; @@ -178,7 +178,8 @@ where Err(e.into()) } -fn sign_request(globals: &globals::Service, dest: &ServerName, http_request: &mut http::Request>) +#[implement(super::Service)] +fn sign_request(&self, dest: &ServerName, http_request: &mut http::Request>) where T: OutgoingRequest + Debug + Send, { @@ -200,11 +201,13 @@ where .to_string() .into(), ); - req_map.insert("origin".to_owned(), globals.server_name().as_str().into()); + req_map.insert("origin".to_owned(), self.services.globals.server_name().to_string().into()); req_map.insert("destination".to_owned(), dest.as_str().into()); let mut req_json = serde_json::from_value(req_map.into()).expect("valid JSON is valid BTreeMap"); - ruma::signatures::sign_json(globals.server_name().as_str(), globals.keypair(), &mut req_json) + self.services + .server_keys + .sign_json(&mut req_json) .expect("our request json is what ruma expects"); let req_json: serde_json::Map = @@ -231,7 +234,12 @@ where http_request.headers_mut().insert( AUTHORIZATION, - HeaderValue::from(&XMatrix::new(globals.config.server_name.clone(), dest.to_owned(), key, sig)), + HeaderValue::from(&XMatrix::new( + self.services.globals.server_name().to_owned(), + dest.to_owned(), + key, + sig, + )), ); } } diff --git a/src/service/server_keys/acquire.rs b/src/service/server_keys/acquire.rs new file mode 100644 index 00000000..2b170040 --- /dev/null +++ b/src/service/server_keys/acquire.rs @@ -0,0 +1,175 @@ +use std::{ + borrow::Borrow, + collections::{BTreeMap, BTreeSet}, +}; + +use conduit::{debug, debug_warn, error, implement, result::FlatOk, warn}; +use futures::{stream::FuturesUnordered, StreamExt}; +use ruma::{ + api::federation::discovery::ServerSigningKeys, serde::Raw, CanonicalJsonObject, OwnedServerName, + OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, +}; +use serde_json::value::RawValue as RawJsonValue; + +use super::key_exists; + +type Batch = BTreeMap>; + +#[implement(super::Service)] +pub async fn acquire_events_pubkeys<'a, I>(&self, events: I) +where + I: Iterator> + Send, +{ + type Batch = BTreeMap>; + type Signatures = BTreeMap>; + + let mut batch = Batch::new(); + events + .cloned() + .map(Raw::::from_json) + .map(|event| event.get_field::("signatures")) + .filter_map(FlatOk::flat_ok) + .flat_map(IntoIterator::into_iter) + .for_each(|(server, sigs)| { + batch.entry(server).or_default().extend(sigs.into_keys()); + }); + + let batch = batch + .iter() + .map(|(server, keys)| (server.borrow(), keys.iter().map(Borrow::borrow))); + + self.acquire_pubkeys(batch).await; +} + +#[implement(super::Service)] +pub async fn acquire_pubkeys<'a, S, K>(&self, batch: S) +where + S: Iterator + Send + Clone, + K: Iterator + Send + Clone, +{ + let requested_servers = batch.clone().count(); + let requested_keys = batch.clone().flat_map(|(_, key_ids)| key_ids).count(); + + debug!("acquire {requested_keys} keys from {requested_servers}"); + + let missing = self.acquire_locals(batch).await; + let missing_keys = keys_count(&missing); + let missing_servers = missing.len(); + if missing_servers == 0 { + return; + } + + debug!("missing {missing_keys} keys for {missing_servers} servers locally"); + + let missing = self.acquire_origins(missing.into_iter()).await; + let missing_keys = keys_count(&missing); + let missing_servers = missing.len(); + if missing_servers == 0 { + return; + } + + debug_warn!("missing {missing_keys} keys for {missing_servers} servers unreachable"); + + let missing = self.acquire_notary(missing.into_iter()).await; + let missing_keys = keys_count(&missing); + let missing_servers = missing.len(); + if missing_keys > 0 { + debug_warn!("still missing {missing_keys} keys for {missing_servers} servers from all notaries"); + warn!("did not obtain {missing_keys} of {requested_keys} keys; some events may not be accepted"); + } +} + +#[implement(super::Service)] +async fn acquire_locals<'a, S, K>(&self, batch: S) -> Batch +where + S: Iterator + Send, + K: Iterator + Send, +{ + let mut missing = Batch::new(); + for (server, key_ids) in batch { + for key_id in key_ids { + if !self.verify_key_exists(server, key_id).await { + missing + .entry(server.into()) + .or_default() + .push(key_id.into()); + } + } + } + + missing +} + +#[implement(super::Service)] +async fn acquire_origins(&self, batch: I) -> Batch +where + I: Iterator)> + Send, +{ + let mut requests: FuturesUnordered<_> = batch + .map(|(origin, key_ids)| self.acquire_origin(origin, key_ids)) + .collect(); + + let mut missing = Batch::new(); + while let Some((origin, key_ids)) = requests.next().await { + if !key_ids.is_empty() { + missing.insert(origin, key_ids); + } + } + + missing +} + +#[implement(super::Service)] +async fn acquire_origin( + &self, origin: OwnedServerName, mut key_ids: Vec, +) -> (OwnedServerName, Vec) { + if let Ok(server_keys) = self.server_request(&origin).await { + self.add_signing_keys(server_keys.clone()).await; + key_ids.retain(|key_id| !key_exists(&server_keys, key_id)); + } + + (origin, key_ids) +} + +#[implement(super::Service)] +async fn acquire_notary(&self, batch: I) -> Batch +where + I: Iterator)> + Send, +{ + let mut missing: Batch = batch.collect(); + for notary in self.services.globals.trusted_servers() { + let missing_keys = keys_count(&missing); + let missing_servers = missing.len(); + debug!("Asking notary {notary} for {missing_keys} missing keys from {missing_servers} servers"); + + let batch = missing + .iter() + .map(|(server, keys)| (server.borrow(), keys.iter().map(Borrow::borrow))); + + match self.batch_notary_request(notary, batch).await { + Err(e) => error!("Failed to contact notary {notary:?}: {e}"), + Ok(results) => { + for server_keys in results { + self.acquire_notary_result(&mut missing, server_keys).await; + } + }, + } + } + + missing +} + +#[implement(super::Service)] +async fn acquire_notary_result(&self, missing: &mut Batch, server_keys: ServerSigningKeys) { + let server = &server_keys.server_name; + self.add_signing_keys(server_keys.clone()).await; + + if let Some(key_ids) = missing.get_mut(server) { + key_ids.retain(|key_id| key_exists(&server_keys, key_id)); + if key_ids.is_empty() { + missing.remove(server); + } + } +} + +fn keys_count(batch: &Batch) -> usize { batch.iter().flat_map(|(_, key_ids)| key_ids.iter()).count() } diff --git a/src/service/server_keys/get.rs b/src/service/server_keys/get.rs new file mode 100644 index 00000000..0f449b46 --- /dev/null +++ b/src/service/server_keys/get.rs @@ -0,0 +1,86 @@ +use std::borrow::Borrow; + +use conduit::{implement, Err, Result}; +use ruma::{api::federation::discovery::VerifyKey, CanonicalJsonObject, RoomVersionId, ServerName, ServerSigningKeyId}; + +use super::{extract_key, PubKeyMap, PubKeys}; + +#[implement(super::Service)] +pub async fn get_event_keys(&self, object: &CanonicalJsonObject, version: &RoomVersionId) -> Result { + use ruma::signatures::required_keys; + + let required = match required_keys(object, version) { + Ok(required) => required, + Err(e) => return Err!(BadServerResponse("Failed to determine keys required to verify: {e}")), + }; + + let batch = required + .iter() + .map(|(s, ids)| (s.borrow(), ids.iter().map(Borrow::borrow))); + + Ok(self.get_pubkeys(batch).await) +} + +#[implement(super::Service)] +pub async fn get_pubkeys<'a, S, K>(&self, batch: S) -> PubKeyMap +where + S: Iterator + Send, + K: Iterator + Send, +{ + let mut keys = PubKeyMap::new(); + for (server, key_ids) in batch { + let pubkeys = self.get_pubkeys_for(server, key_ids).await; + keys.insert(server.into(), pubkeys); + } + + keys +} + +#[implement(super::Service)] +pub async fn get_pubkeys_for<'a, I>(&self, origin: &ServerName, key_ids: I) -> PubKeys +where + I: Iterator + Send, +{ + let mut keys = PubKeys::new(); + for key_id in key_ids { + if let Ok(verify_key) = self.get_verify_key(origin, key_id).await { + keys.insert(key_id.into(), verify_key.key); + } + } + + keys +} + +#[implement(super::Service)] +pub async fn get_verify_key(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> Result { + if let Some(result) = self.verify_keys_for(origin).await.remove(key_id) { + return Ok(result); + } + + if let Ok(server_key) = self.server_request(origin).await { + self.add_signing_keys(server_key.clone()).await; + if let Some(result) = extract_key(server_key, key_id) { + return Ok(result); + } + } + + for notary in self.services.globals.trusted_servers() { + if let Ok(server_keys) = self.notary_request(notary, origin).await { + for server_key in &server_keys { + self.add_signing_keys(server_key.clone()).await; + } + + for server_key in server_keys { + if let Some(result) = extract_key(server_key, key_id) { + return Ok(result); + } + } + } + } + + Err!(BadServerResponse(debug_error!( + ?key_id, + ?origin, + "Failed to fetch federation signing-key" + ))) +} diff --git a/src/service/server_keys/keypair.rs b/src/service/server_keys/keypair.rs new file mode 100644 index 00000000..31a24cdf --- /dev/null +++ b/src/service/server_keys/keypair.rs @@ -0,0 +1,64 @@ +use std::sync::Arc; + +use conduit::{debug, debug_info, err, error, utils, utils::string_from_bytes, Result}; +use database::Database; +use ruma::{api::federation::discovery::VerifyKey, serde::Base64, signatures::Ed25519KeyPair}; + +use super::VerifyKeys; + +pub(super) fn init(db: &Arc) -> Result<(Box, VerifyKeys)> { + let keypair = load(db).inspect_err(|_e| { + error!("Keypair invalid. Deleting..."); + remove(db); + })?; + + let verify_key = VerifyKey { + key: Base64::new(keypair.public_key().to_vec()), + }; + + let id = format!("ed25519:{}", keypair.version()); + let verify_keys: VerifyKeys = [(id.try_into()?, verify_key)].into(); + + Ok((keypair, verify_keys)) +} + +fn load(db: &Arc) -> Result> { + let (version, key) = db["global"] + .get_blocking(b"keypair") + .map(|ref val| { + // database deserializer is having trouble with this so it's manual for now + let mut elems = val.split(|&b| b == b'\xFF'); + let vlen = elems.next().expect("invalid keypair entry").len(); + let ver = string_from_bytes(&val[..vlen]).expect("invalid keypair version"); + let der = val[vlen.saturating_add(1)..].to_vec(); + debug!("Found existing Ed25519 keypair: {ver:?}"); + (ver, der) + }) + .or_else(|e| { + assert!(e.is_not_found(), "unexpected error fetching keypair"); + create(db) + })?; + + let key = + Ed25519KeyPair::from_der(&key, version).map_err(|e| err!("Failed to load ed25519 keypair from der: {e:?}"))?; + + Ok(Box::new(key)) +} + +fn create(db: &Arc) -> Result<(String, Vec)> { + let keypair = Ed25519KeyPair::generate().map_err(|e| err!("Failed to generate new ed25519 keypair: {e:?}"))?; + + let id = utils::rand::string(8); + debug_info!("Generated new Ed25519 keypair: {id:?}"); + + let value: (String, Vec) = (id, keypair.to_vec()); + db["global"].raw_put(b"keypair", &value); + + Ok(value) +} + +#[inline] +fn remove(db: &Arc) { + let global = &db["global"]; + global.remove(b"keypair"); +} diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index ae2b8c3c..c3b84cb3 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -1,31 +1,30 @@ -use std::{ - collections::{BTreeMap, HashMap, HashSet}, - sync::Arc, - time::{Duration, SystemTime}, -}; +mod acquire; +mod get; +mod keypair; +mod request; +mod sign; +mod verify; -use conduit::{debug, debug_error, debug_warn, err, error, info, trace, warn, Err, Result}; -use futures::{stream::FuturesUnordered, StreamExt}; +use std::{collections::BTreeMap, sync::Arc, time::Duration}; + +use conduit::{implement, utils::time::timepoint_from_now, Result}; +use database::{Deserialized, Json, Map}; use ruma::{ - api::federation::{ - discovery::{ - get_remote_server_keys, - get_remote_server_keys_batch::{self, v2::QueryCriteria}, - get_server_keys, - }, - membership::create_join_event, - }, - serde::Base64, - CanonicalJsonObject, CanonicalJsonValue, MilliSecondsSinceUnixEpoch, OwnedServerName, OwnedServerSigningKeyId, - RoomVersionId, ServerName, + api::federation::discovery::{ServerSigningKeys, VerifyKey}, + serde::Raw, + signatures::{Ed25519KeyPair, PublicKeyMap, PublicKeySet}, + MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, }; use serde_json::value::RawValue as RawJsonValue; -use tokio::sync::{RwLock, RwLockWriteGuard}; use crate::{globals, sending, Dep}; pub struct Service { + keypair: Box, + verify_keys: VerifyKeys, + minimum_valid: Duration, services: Services, + db: Data, } struct Services { @@ -33,546 +32,135 @@ struct Services { sending: Dep, } +struct Data { + server_signingkeys: Arc, +} + +pub type VerifyKeys = BTreeMap; +pub type PubKeyMap = PublicKeyMap; +pub type PubKeys = PublicKeySet; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { + let minimum_valid = Duration::from_secs(3600); + let (keypair, verify_keys) = keypair::init(args.db)?; + Ok(Arc::new(Self { + keypair, + verify_keys, + minimum_valid, services: Services { globals: args.depend::("globals"), sending: args.depend::("sending"), }, + db: Data { + server_signingkeys: args.db["server_signingkeys"].clone(), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - pub async fn fetch_required_signing_keys<'a, E>( - &'a self, events: E, pub_key_map: &RwLock>>, - ) -> Result<()> - where - E: IntoIterator> + Send, - { - let mut server_key_ids = HashMap::new(); - for event in events { - for (signature_server, signature) in event - .get("signatures") - .ok_or(err!(BadServerResponse("No signatures in server response pdu.")))? - .as_object() - .ok_or(err!(BadServerResponse("Invalid signatures object in server response pdu.")))? - { - let signature_object = signature.as_object().ok_or(err!(BadServerResponse( - "Invalid signatures content object in server response pdu.", - )))?; +#[implement(Service)] +#[inline] +pub fn keypair(&self) -> &Ed25519KeyPair { &self.keypair } - for signature_id in signature_object.keys() { - server_key_ids - .entry(signature_server.clone()) - .or_insert_with(HashSet::new) - .insert(signature_id.clone()); - } - } - } +#[implement(Service)] +async fn add_signing_keys(&self, new_keys: ServerSigningKeys) { + let origin = &new_keys.server_name; - if server_key_ids.is_empty() { - // Nothing to do, can exit early - trace!("server_key_ids is empty, not fetching any keys"); - return Ok(()); - } + // (timo) Not atomic, but this is not critical + let mut keys: ServerSigningKeys = self + .db + .server_signingkeys + .get(origin) + .await + .deserialized() + .unwrap_or_else(|_| { + // Just insert "now", it doesn't matter + ServerSigningKeys::new(origin.to_owned(), MilliSecondsSinceUnixEpoch::now()) + }); - trace!( - "Fetch keys for {}", - server_key_ids - .keys() - .cloned() - .collect::>() - .join(", ") - ); - - let mut server_keys: FuturesUnordered<_> = server_key_ids - .into_iter() - .map(|(signature_server, signature_ids)| async { - let fetch_res = self - .fetch_signing_keys_for_server( - signature_server.as_str().try_into().map_err(|e| { - ( - signature_server.clone(), - err!(BadServerResponse( - "Invalid servername in signatures of server response pdu: {e:?}" - )), - ) - })?, - signature_ids.into_iter().collect(), // HashSet to Vec - ) - .await; - - match fetch_res { - Ok(keys) => Ok((signature_server, keys)), - Err(e) => { - debug_error!( - "Signature verification failed: Could not fetch signing key for {signature_server}: {e}", - ); - Err((signature_server, e)) - }, - } - }) - .collect(); - - while let Some(fetch_res) = server_keys.next().await { - match fetch_res { - Ok((signature_server, keys)) => { - pub_key_map - .write() - .await - .insert(signature_server.clone(), keys); - }, - Err((signature_server, e)) => { - debug_warn!("Failed to fetch keys for {signature_server}: {e:?}"); - }, - } - } - - Ok(()) - } - - // Gets a list of servers for which we don't have the signing key yet. We go - // over the PDUs and either cache the key or add it to the list that needs to be - // retrieved. - async fn get_server_keys_from_cache( - &self, pdu: &RawJsonValue, - servers: &mut BTreeMap>, - _room_version: &RoomVersionId, - pub_key_map: &mut RwLockWriteGuard<'_, BTreeMap>>, - ) -> Result<()> { - let value: CanonicalJsonObject = serde_json::from_str(pdu.get()).map_err(|e| { - debug_error!("Invalid PDU in server response: {pdu:#?}"); - err!(BadServerResponse(error!("Invalid PDU in server response: {e:?}"))) - })?; - - let signatures = value - .get("signatures") - .ok_or(err!(BadServerResponse("No signatures in server response pdu.")))? - .as_object() - .ok_or(err!(BadServerResponse("Invalid signatures object in server response pdu.")))?; - - for (signature_server, signature) in signatures { - let signature_object = signature.as_object().ok_or(err!(BadServerResponse( - "Invalid signatures content object in server response pdu.", - )))?; - - let signature_ids = signature_object.keys().cloned().collect::>(); - - let contains_all_ids = - |keys: &BTreeMap| signature_ids.iter().all(|id| keys.contains_key(id)); - - let origin = <&ServerName>::try_from(signature_server.as_str()).map_err(|e| { - err!(BadServerResponse( - "Invalid servername in signatures of server response pdu: {e:?}" - )) - })?; - - if servers.contains_key(origin) || pub_key_map.contains_key(origin.as_str()) { - continue; - } - - debug!("Loading signing keys for {origin}"); - let result: BTreeMap<_, _> = self - .services - .globals - .verify_keys_for(origin) - .await? - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)) - .collect(); - - if !contains_all_ids(&result) { - debug_warn!("Signing key not loaded for {origin}"); - servers.insert(origin.to_owned(), BTreeMap::new()); - } - - pub_key_map.insert(origin.to_string(), result); - } - - Ok(()) - } - - /// Batch requests homeserver signing keys from trusted notary key servers - /// (`trusted_servers` config option) - async fn batch_request_signing_keys( - &self, mut servers: BTreeMap>, - pub_key_map: &RwLock>>, - ) -> Result<()> { - for server in self.services.globals.trusted_servers() { - debug!("Asking batch signing keys from trusted server {server}"); - match self - .services - .sending - .send_federation_request( - server, - get_remote_server_keys_batch::v2::Request { - server_keys: servers.clone(), - }, - ) - .await - { - Ok(keys) => { - debug!("Got signing keys: {keys:?}"); - let mut pkm = pub_key_map.write().await; - for k in keys.server_keys { - let k = match k.deserialize() { - Ok(key) => key, - Err(e) => { - warn!( - "Received error {e} while fetching keys from trusted server {server}: {:#?}", - k.into_json() - ); - continue; - }, - }; - - // TODO: Check signature from trusted server? - servers.remove(&k.server_name); - - let result = self - .services - .globals - .db - .add_signing_key(&k.server_name, k.clone()) - .await - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)) - .collect::>(); - - pkm.insert(k.server_name.to_string(), result); - } - }, - Err(e) => error!( - "Failed sending batched key request to trusted key server {server} for the remote servers \ - {servers:?}: {e}" - ), - } - } - - Ok(()) - } - - /// Requests multiple homeserver signing keys from individual servers (not - /// trused notary servers) - async fn request_signing_keys( - &self, servers: BTreeMap>, - pub_key_map: &RwLock>>, - ) -> Result<()> { - debug!("Asking individual servers for signing keys: {servers:?}"); - let mut futures: FuturesUnordered<_> = servers - .into_keys() - .map(|server| async move { - ( - self.services - .sending - .send_federation_request(&server, get_server_keys::v2::Request::new()) - .await, - server, - ) - }) - .collect(); - - while let Some(result) = futures.next().await { - debug!("Received new Future result"); - if let (Ok(get_keys_response), origin) = result { - debug!("Result is from {origin}"); - if let Ok(key) = get_keys_response.server_key.deserialize() { - let result: BTreeMap<_, _> = self - .services - .globals - .db - .add_signing_key(&origin, key) - .await - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)) - .collect(); - pub_key_map.write().await.insert(origin.to_string(), result); - } - } - debug!("Done handling Future result"); - } - - Ok(()) - } - - pub async fn fetch_join_signing_keys( - &self, event: &create_join_event::v2::Response, room_version: &RoomVersionId, - pub_key_map: &RwLock>>, - ) -> Result<()> { - let mut servers: BTreeMap> = BTreeMap::new(); - - { - let mut pkm = pub_key_map.write().await; - - // Try to fetch keys, failure is okay. Servers we couldn't find in the cache - // will be added to `servers` - for pdu in event - .room_state - .state - .iter() - .chain(&event.room_state.auth_chain) - { - if let Err(error) = self - .get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm) - .await - { - debug!(%error, "failed to get server keys from cache"); - }; - } - - drop(pkm); - }; - - if servers.is_empty() { - trace!("We had all keys cached locally, not fetching any keys from remote servers"); - return Ok(()); - } - - if self.services.globals.query_trusted_key_servers_first() { - info!( - "query_trusted_key_servers_first is set to true, querying notary trusted key servers first for \ - homeserver signing keys." - ); - - self.batch_request_signing_keys(servers.clone(), pub_key_map) - .await?; - - if servers.is_empty() { - debug!("Trusted server supplied all signing keys, no more keys to fetch"); - return Ok(()); - } - - debug!("Remaining servers left that the notary/trusted servers did not provide: {servers:?}"); - - self.request_signing_keys(servers.clone(), pub_key_map) - .await?; - } else { - debug!("query_trusted_key_servers_first is set to false, querying individual homeservers first"); - - self.request_signing_keys(servers.clone(), pub_key_map) - .await?; - - if servers.is_empty() { - debug!("Individual homeservers supplied all signing keys, no more keys to fetch"); - return Ok(()); - } - - debug!("Remaining servers left the individual homeservers did not provide: {servers:?}"); - - self.batch_request_signing_keys(servers.clone(), pub_key_map) - .await?; - } - - debug!("Search for signing keys done"); - - /*if servers.is_empty() { - warn!("Failed to find homeserver signing keys for the remaining servers: {servers:?}"); - }*/ - - Ok(()) - } - - /// Search the DB for the signing keys of the given server, if we don't have - /// them fetch them from the server and save to our DB. - #[tracing::instrument(skip_all)] - pub async fn fetch_signing_keys_for_server( - &self, origin: &ServerName, signature_ids: Vec, - ) -> Result> { - let contains_all_ids = |keys: &BTreeMap| signature_ids.iter().all(|id| keys.contains_key(id)); - - let mut result: BTreeMap<_, _> = self - .services - .globals - .verify_keys_for(origin) - .await? - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)) - .collect(); - - if contains_all_ids(&result) { - trace!("We have all homeserver signing keys locally for {origin}, not fetching any remotely"); - return Ok(result); - } - - // i didnt split this out into their own functions because it's relatively small - if self.services.globals.query_trusted_key_servers_first() { - info!( - "query_trusted_key_servers_first is set to true, querying notary trusted servers first for {origin} \ - keys" - ); - - for server in self.services.globals.trusted_servers() { - debug!("Asking notary server {server} for {origin}'s signing key"); - if let Some(server_keys) = self - .services - .sending - .send_federation_request( - server, - get_remote_server_keys::v2::Request::new( - origin.to_owned(), - MilliSecondsSinceUnixEpoch::from_system_time( - SystemTime::now() - .checked_add(Duration::from_secs(3600)) - .expect("SystemTime too large"), - ) - .expect("time is valid"), - ), - ) - .await - .ok() - .map(|resp| { - resp.server_keys - .into_iter() - .filter_map(|e| e.deserialize().ok()) - .collect::>() - }) { - debug!("Got signing keys: {:?}", server_keys); - for k in server_keys { - self.services - .globals - .db - .add_signing_key(origin, k.clone()) - .await; - result.extend( - k.verify_keys - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)), - ); - result.extend( - k.old_verify_keys - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)), - ); - } - - if contains_all_ids(&result) { - return Ok(result); - } - } - } - - debug!("Asking {origin} for their signing keys over federation"); - if let Some(server_key) = self - .services - .sending - .send_federation_request(origin, get_server_keys::v2::Request::new()) - .await - .ok() - .and_then(|resp| resp.server_key.deserialize().ok()) - { - self.services - .globals - .db - .add_signing_key(origin, server_key.clone()) - .await; - - result.extend( - server_key - .verify_keys - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)), - ); - result.extend( - server_key - .old_verify_keys - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)), - ); - - if contains_all_ids(&result) { - return Ok(result); - } - } - } else { - info!("query_trusted_key_servers_first is set to false, querying {origin} first"); - debug!("Asking {origin} for their signing keys over federation"); - if let Some(server_key) = self - .services - .sending - .send_federation_request(origin, get_server_keys::v2::Request::new()) - .await - .ok() - .and_then(|resp| resp.server_key.deserialize().ok()) - { - self.services - .globals - .db - .add_signing_key(origin, server_key.clone()) - .await; - - result.extend( - server_key - .verify_keys - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)), - ); - result.extend( - server_key - .old_verify_keys - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)), - ); - - if contains_all_ids(&result) { - return Ok(result); - } - } - - for server in self.services.globals.trusted_servers() { - debug!("Asking notary server {server} for {origin}'s signing key"); - if let Some(server_keys) = self - .services - .sending - .send_federation_request( - server, - get_remote_server_keys::v2::Request::new( - origin.to_owned(), - MilliSecondsSinceUnixEpoch::from_system_time( - SystemTime::now() - .checked_add(Duration::from_secs(3600)) - .expect("SystemTime too large"), - ) - .expect("time is valid"), - ), - ) - .await - .ok() - .map(|resp| { - resp.server_keys - .into_iter() - .filter_map(|e| e.deserialize().ok()) - .collect::>() - }) { - debug!("Got signing keys: {server_keys:?}"); - for k in server_keys { - self.services - .globals - .db - .add_signing_key(origin, k.clone()) - .await; - result.extend( - k.verify_keys - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)), - ); - result.extend( - k.old_verify_keys - .into_iter() - .map(|(k, v)| (k.to_string(), v.key)), - ); - } - - if contains_all_ids(&result) { - return Ok(result); - } - } - } - } - - Err!(BadServerResponse(warn!("Failed to find public key for server {origin:?}"))) - } + keys.verify_keys.extend(new_keys.verify_keys); + keys.old_verify_keys.extend(new_keys.old_verify_keys); + self.db.server_signingkeys.raw_put(origin, Json(&keys)); +} + +#[implement(Service)] +async fn verify_key_exists(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> bool { + type KeysMap<'a> = BTreeMap<&'a ServerSigningKeyId, &'a RawJsonValue>; + + let Ok(keys) = self + .db + .server_signingkeys + .get(origin) + .await + .deserialized::>() + else { + return false; + }; + + if let Ok(Some(verify_keys)) = keys.get_field::>("verify_keys") { + if verify_keys.contains_key(key_id) { + return true; + } + } + + if let Ok(Some(old_verify_keys)) = keys.get_field::>("old_verify_keys") { + if old_verify_keys.contains_key(key_id) { + return true; + } + } + + false +} + +#[implement(Service)] +pub async fn verify_keys_for(&self, origin: &ServerName) -> VerifyKeys { + let mut keys = self + .signing_keys_for(origin) + .await + .map(|keys| merge_old_keys(keys).verify_keys) + .unwrap_or(BTreeMap::new()); + + if self.services.globals.server_is_ours(origin) { + keys.extend(self.verify_keys.clone().into_iter()); + } + + keys +} + +#[implement(Service)] +pub async fn signing_keys_for(&self, origin: &ServerName) -> Result { + self.db.server_signingkeys.get(origin).await.deserialized() +} + +#[implement(Service)] +fn minimum_valid_ts(&self) -> MilliSecondsSinceUnixEpoch { + let timepoint = timepoint_from_now(self.minimum_valid).expect("SystemTime should not overflow"); + MilliSecondsSinceUnixEpoch::from_system_time(timepoint).expect("UInt should not overflow") +} + +fn merge_old_keys(mut keys: ServerSigningKeys) -> ServerSigningKeys { + keys.verify_keys.extend( + keys.old_verify_keys + .clone() + .into_iter() + .map(|(key_id, old)| (key_id, VerifyKey::new(old.key))), + ); + + keys +} + +fn extract_key(mut keys: ServerSigningKeys, key_id: &ServerSigningKeyId) -> Option { + keys.verify_keys.remove(key_id).or_else(|| { + keys.old_verify_keys + .remove(key_id) + .map(|old| VerifyKey::new(old.key)) + }) +} + +fn key_exists(keys: &ServerSigningKeys, key_id: &ServerSigningKeyId) -> bool { + keys.verify_keys.contains_key(key_id) || keys.old_verify_keys.contains_key(key_id) } diff --git a/src/service/server_keys/request.rs b/src/service/server_keys/request.rs new file mode 100644 index 00000000..84dd2871 --- /dev/null +++ b/src/service/server_keys/request.rs @@ -0,0 +1,97 @@ +use std::collections::BTreeMap; + +use conduit::{implement, Err, Result}; +use ruma::{ + api::federation::discovery::{ + get_remote_server_keys, + get_remote_server_keys_batch::{self, v2::QueryCriteria}, + get_server_keys, ServerSigningKeys, + }, + OwnedServerName, OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, +}; + +#[implement(super::Service)] +pub(super) async fn batch_notary_request<'a, S, K>( + &self, notary: &ServerName, batch: S, +) -> Result> +where + S: Iterator + Send, + K: Iterator + Send, +{ + use get_remote_server_keys_batch::v2::Request; + type RumaBatch = BTreeMap>; + + let criteria = QueryCriteria { + minimum_valid_until_ts: Some(self.minimum_valid_ts()), + }; + + let mut server_keys = RumaBatch::new(); + for (server, key_ids) in batch { + let entry = server_keys.entry(server.into()).or_default(); + for key_id in key_ids { + entry.insert(key_id.into(), criteria.clone()); + } + } + + debug_assert!(!server_keys.is_empty(), "empty batch request to notary"); + let request = Request { + server_keys, + }; + + self.services + .sending + .send_federation_request(notary, request) + .await + .map(|response| response.server_keys) + .map(|keys| { + keys.into_iter() + .map(|key| key.deserialize()) + .filter_map(Result::ok) + .collect() + }) +} + +#[implement(super::Service)] +pub async fn notary_request(&self, notary: &ServerName, target: &ServerName) -> Result> { + use get_remote_server_keys::v2::Request; + + let request = Request { + server_name: target.into(), + minimum_valid_until_ts: self.minimum_valid_ts(), + }; + + self.services + .sending + .send_federation_request(notary, request) + .await + .map(|response| response.server_keys) + .map(|keys| { + keys.into_iter() + .map(|key| key.deserialize()) + .filter_map(Result::ok) + .collect() + }) +} + +#[implement(super::Service)] +pub async fn server_request(&self, target: &ServerName) -> Result { + use get_server_keys::v2::Request; + + let server_signing_key = self + .services + .sending + .send_federation_request(target, Request::new()) + .await + .map(|response| response.server_key) + .and_then(|key| key.deserialize().map_err(Into::into))?; + + if server_signing_key.server_name != target { + return Err!(BadServerResponse(debug_warn!( + requested = ?target, + response = ?server_signing_key.server_name, + "Server responded with bogus server_name" + ))); + } + + Ok(server_signing_key) +} diff --git a/src/service/server_keys/sign.rs b/src/service/server_keys/sign.rs new file mode 100644 index 00000000..28fd7e80 --- /dev/null +++ b/src/service/server_keys/sign.rs @@ -0,0 +1,18 @@ +use conduit::{implement, Result}; +use ruma::{CanonicalJsonObject, RoomVersionId}; + +#[implement(super::Service)] +pub fn sign_json(&self, object: &mut CanonicalJsonObject) -> Result { + use ruma::signatures::sign_json; + + let server_name = self.services.globals.server_name().as_str(); + sign_json(server_name, self.keypair(), object).map_err(Into::into) +} + +#[implement(super::Service)] +pub fn hash_and_sign_event(&self, object: &mut CanonicalJsonObject, room_version: &RoomVersionId) -> Result { + use ruma::signatures::hash_and_sign_event; + + let server_name = self.services.globals.server_name().as_str(); + hash_and_sign_event(server_name, self.keypair(), object, room_version).map_err(Into::into) +} diff --git a/src/service/server_keys/verify.rs b/src/service/server_keys/verify.rs new file mode 100644 index 00000000..ad20fec7 --- /dev/null +++ b/src/service/server_keys/verify.rs @@ -0,0 +1,33 @@ +use conduit::{implement, pdu::gen_event_id_canonical_json, Err, Result}; +use ruma::{signatures::Verified, CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, RoomVersionId}; +use serde_json::value::RawValue as RawJsonValue; + +#[implement(super::Service)] +pub async fn validate_and_add_event_id( + &self, pdu: &RawJsonValue, room_version: &RoomVersionId, +) -> Result<(OwnedEventId, CanonicalJsonObject)> { + let (event_id, mut value) = gen_event_id_canonical_json(pdu, room_version)?; + if let Err(e) = self.verify_event(&value, Some(room_version)).await { + return Err!(BadServerResponse(debug_error!("Event {event_id} failed verification: {e:?}"))); + } + + value.insert("event_id".into(), CanonicalJsonValue::String(event_id.as_str().into())); + + Ok((event_id, value)) +} + +#[implement(super::Service)] +pub async fn verify_event( + &self, event: &CanonicalJsonObject, room_version: Option<&RoomVersionId>, +) -> Result { + let room_version = room_version.unwrap_or(&RoomVersionId::V11); + let keys = self.get_event_keys(event, room_version).await?; + ruma::signatures::verify_event(&keys, event, room_version).map_err(Into::into) +} + +#[implement(super::Service)] +pub async fn verify_json(&self, event: &CanonicalJsonObject, room_version: Option<&RoomVersionId>) -> Result { + let room_version = room_version.unwrap_or(&RoomVersionId::V11); + let keys = self.get_event_keys(event, room_version).await?; + ruma::signatures::verify_json(&keys, event.clone()).map_err(Into::into) +} From b4ec1e9d3cbc58f68c3733061c11c55700ff3018 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 14 Oct 2024 03:58:25 +0000 Subject: [PATCH 0916/2091] add federation client for select high-timeout requests Signed-off-by: Jason Volk --- src/api/client/membership.rs | 4 ++-- src/core/config/mod.rs | 2 +- src/service/client/mod.rs | 9 ++++++++- src/service/rooms/event_handler/mod.rs | 2 +- src/service/sending/mod.rs | 11 +++++++++++ 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 2fa34ff7..31fd9076 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -775,7 +775,7 @@ async fn join_room_by_id_helper_remote( let send_join_response = services .sending - .send_federation_request(&remote_server, send_join_request) + .send_synapse_request(&remote_server, send_join_request) .await?; info!("send_join finished"); @@ -1154,7 +1154,7 @@ async fn join_room_by_id_helper_local( let send_join_response = services .sending - .send_federation_request( + .send_synapse_request( &remote_server, federation::membership::create_join_event::v2::Request { room_id: room_id.to_owned(), diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 114c6e76..02b277d0 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1775,7 +1775,7 @@ fn default_well_known_conn_timeout() -> u64 { 6 } fn default_well_known_timeout() -> u64 { 10 } -fn default_federation_timeout() -> u64 { 300 } +fn default_federation_timeout() -> u64 { 25 } fn default_federation_idle_timeout() -> u64 { 25 } diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index b21f9dab..f9a89e99 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -11,6 +11,7 @@ pub struct Service { pub extern_media: reqwest::Client, pub well_known: reqwest::Client, pub federation: reqwest::Client, + pub synapse: reqwest::Client, pub sender: reqwest::Client, pub appservice: reqwest::Client, pub pusher: reqwest::Client, @@ -48,12 +49,18 @@ impl crate::Service for Service { federation: base(config)? .dns_resolver(resolver.resolver.hooked.clone()) .read_timeout(Duration::from_secs(config.federation_timeout)) - .timeout(Duration::from_secs(config.federation_timeout)) .pool_max_idle_per_host(config.federation_idle_per_host.into()) .pool_idle_timeout(Duration::from_secs(config.federation_idle_timeout)) .redirect(redirect::Policy::limited(3)) .build()?, + synapse: base(config)? + .dns_resolver(resolver.resolver.hooked.clone()) + .read_timeout(Duration::from_secs(305)) + .pool_max_idle_per_host(0) + .redirect(redirect::Policy::limited(3)) + .build()?, + sender: base(config)? .dns_resolver(resolver.resolver.hooked.clone()) .read_timeout(Duration::from_secs(config.sender_timeout)) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 8448404b..0ffd9659 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1016,7 +1016,7 @@ impl Service { let res = self .services .sending - .send_federation_request( + .send_synapse_request( origin, get_room_state_ids::v1::Request { room_id: room_id.to_owned(), diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 5970c383..63c5e655 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -245,6 +245,7 @@ impl Service { .await } + /// Sends a request to a federation server #[tracing::instrument(skip_all, name = "request")] pub async fn send_federation_request(&self, dest: &ServerName, request: T) -> Result where @@ -254,6 +255,16 @@ impl Service { self.send(client, dest, request).await } + /// Like send_federation_request() but with a very large timeout + #[tracing::instrument(skip_all, name = "synapse")] + pub async fn send_synapse_request(&self, dest: &ServerName, request: T) -> Result + where + T: OutgoingRequest + Debug + Send, + { + let client = &self.services.client.synapse; + self.send(client, dest, request).await + } + /// Sends a request to an appservice /// /// Only returns None if there is no url specified in the appservice From d0ee4b6d253079ceef7ee9094db9e3f70f1ed048 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 14 Oct 2024 01:01:12 +0000 Subject: [PATCH 0917/2091] add resolve_with_servers() to alias service; simplify api Signed-off-by: Jason Volk --- src/admin/user/commands.rs | 9 ++- src/api/client/alias.rs | 16 ++--- src/api/client/membership.rs | 66 +++++++++-------- src/service/rooms/alias/mod.rs | 47 +++++++----- src/service/rooms/alias/remote.rs | 116 ++++++++++++++---------------- 5 files changed, 131 insertions(+), 123 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index df393833..fb6d2bf1 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -381,13 +381,18 @@ pub(super) async fn force_join_room( &self, user_id: String, room_id: OwnedRoomOrAliasId, ) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; - let room_id = self.services.rooms.alias.resolve(&room_id).await?; + let (room_id, servers) = self + .services + .rooms + .alias + .resolve_with_servers(&room_id, None) + .await?; assert!( self.services.globals.user_is_local(&user_id), "Parsed user_id must be a local user" ); - join_room_by_id_helper(self.services, &user_id, &room_id, None, &[], None, &None).await?; + join_room_by_id_helper(self.services, &user_id, &room_id, None, &servers, None, &None).await?; Ok(RoomMessageEventContent::notice_markdown(format!( "{user_id} has been joined to {room_id}.", diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 2399a355..83f3291d 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -86,25 +86,19 @@ pub(crate) async fn get_alias_route( State(services): State, body: Ruma, ) -> Result { let room_alias = body.body.room_alias; - let servers = None; - let Ok((room_id, pre_servers)) = services - .rooms - .alias - .resolve_alias(&room_alias, servers.as_ref()) - .await - else { + let Ok((room_id, servers)) = services.rooms.alias.resolve_alias(&room_alias, None).await else { return Err!(Request(NotFound("Room with alias not found."))); }; - let servers = room_available_servers(&services, &room_id, &room_alias, &pre_servers).await; + let servers = room_available_servers(&services, &room_id, &room_alias, servers).await; debug!(?room_alias, ?room_id, "available servers: {servers:?}"); Ok(get_alias::v3::Response::new(room_id, servers)) } async fn room_available_servers( - services: &Services, room_id: &RoomId, room_alias: &RoomAliasId, pre_servers: &Option>, + services: &Services, room_id: &RoomId, room_alias: &RoomAliasId, pre_servers: Vec, ) -> Vec { // find active servers in room state cache to suggest let mut servers: Vec = services @@ -117,9 +111,7 @@ async fn room_available_servers( // push any servers we want in the list already (e.g. responded remote alias // servers, room alias server itself) - if let Some(pre_servers) = pre_servers { - servers.extend(pre_servers.clone()); - }; + servers.extend(pre_servers); servers.sort_unstable(); servers.dedup(); diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 31fd9076..27de60c6 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -9,8 +9,9 @@ use axum_client_ip::InsecureClientIp; use conduit::{ debug, debug_info, debug_warn, err, error, info, pdu, pdu::{gen_event_id_canonical_json, PduBuilder}, + result::FlatOk, trace, utils, - utils::{IterStream, ReadyExt}, + utils::{shuffle, IterStream, ReadyExt}, warn, Err, Error, PduEvent, Result, }; use futures::{FutureExt, StreamExt}; @@ -188,6 +189,10 @@ pub(crate) async fn join_room_by_id_route( servers.push(server.into()); } + servers.sort_unstable(); + servers.dedup(); + shuffle(&mut servers); + join_room_by_id_helper( &services, sender_user, @@ -251,45 +256,48 @@ pub(crate) async fn join_room_by_id_or_alias_route( servers.push(server.to_owned()); } + servers.sort_unstable(); + servers.dedup(); + shuffle(&mut servers); + (servers, room_id) }, Err(room_alias) => { - let response = services + let (room_id, mut servers) = services .rooms .alias - .resolve_alias(&room_alias, Some(&body.via.clone())) + .resolve_alias(&room_alias, Some(body.via.clone())) .await?; - let (room_id, mut pre_servers) = response; banned_room_check(&services, sender_user, Some(&room_id), Some(room_alias.server_name()), client).await?; - let mut servers = body.via; - if let Some(pre_servers) = &mut pre_servers { - servers.append(pre_servers); - } + let addl_via_servers = services + .rooms + .state_cache + .servers_invite_via(&room_id) + .map(ToOwned::to_owned); - servers.extend( - services - .rooms - .state_cache - .servers_invite_via(&room_id) - .map(ToOwned::to_owned) - .collect::>() - .await, - ); + let addl_state_servers = services + .rooms + .state_cache + .invite_state(sender_user, &room_id) + .await + .unwrap_or_default(); - servers.extend( - services - .rooms - .state_cache - .invite_state(sender_user, &room_id) - .await - .unwrap_or_default() - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()), - ); + let mut addl_servers: Vec<_> = addl_state_servers + .iter() + .map(|event| event.get_field("sender")) + .filter_map(FlatOk::flat_ok) + .map(|user: &UserId| user.server_name().to_owned()) + .stream() + .chain(addl_via_servers) + .collect() + .await; + + addl_servers.sort_unstable(); + addl_servers.dedup(); + shuffle(&mut addl_servers); + servers.append(&mut addl_servers); (servers, room_id) }, diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 3f944729..0cdec8ee 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -112,40 +112,51 @@ impl Service { Ok(()) } + #[inline] pub async fn resolve(&self, room: &RoomOrAliasId) -> Result { + self.resolve_with_servers(room, None) + .await + .map(|(room_id, _)| room_id) + } + + pub async fn resolve_with_servers( + &self, room: &RoomOrAliasId, servers: Option>, + ) -> Result<(OwnedRoomId, Vec)> { if room.is_room_id() { - let room_id: &RoomId = &RoomId::parse(room).expect("valid RoomId"); - Ok(room_id.to_owned()) + let room_id = RoomId::parse(room).expect("valid RoomId"); + Ok((room_id, servers.unwrap_or_default())) } else { - let alias: &RoomAliasId = &RoomAliasId::parse(room).expect("valid RoomAliasId"); - Ok(self.resolve_alias(alias, None).await?.0) + let alias = &RoomAliasId::parse(room).expect("valid RoomAliasId"); + self.resolve_alias(alias, servers).await } } #[tracing::instrument(skip(self), name = "resolve")] pub async fn resolve_alias( - &self, room_alias: &RoomAliasId, servers: Option<&Vec>, - ) -> Result<(OwnedRoomId, Option>)> { - if !self - .services - .globals - .server_is_ours(room_alias.server_name()) - && (!servers + &self, room_alias: &RoomAliasId, servers: Option>, + ) -> Result<(OwnedRoomId, Vec)> { + let server_name = room_alias.server_name(); + let server_is_ours = self.services.globals.server_is_ours(server_name); + let servers_contains_ours = || { + servers .as_ref() - .is_some_and(|servers| servers.contains(&self.services.globals.server_name().to_owned())) - || servers.as_ref().is_none()) - { - return self.remote_resolve(room_alias, servers).await; + .is_some_and(|servers| servers.contains(&self.services.globals.config.server_name)) + }; + + if !server_is_ours && !servers_contains_ours() { + return self + .remote_resolve(room_alias, servers.unwrap_or_default()) + .await; } - let room_id: Option = match self.resolve_local_alias(room_alias).await { + let room_id = match self.resolve_local_alias(room_alias).await { Ok(r) => Some(r), Err(_) => self.resolve_appservice_alias(room_alias).await?, }; room_id.map_or_else( - || Err(Error::BadRequest(ErrorKind::NotFound, "Room with alias not found.")), - |room_id| Ok((room_id, None)), + || Err!(Request(NotFound("Room with alias not found."))), + |room_id| Ok((room_id, Vec::new())), ) } diff --git a/src/service/rooms/alias/remote.rs b/src/service/rooms/alias/remote.rs index 5d835240..d9acccc9 100644 --- a/src/service/rooms/alias/remote.rs +++ b/src/service/rooms/alias/remote.rs @@ -1,75 +1,67 @@ -use conduit::{debug, debug_warn, Error, Result}; -use ruma::{ - api::{client::error::ErrorKind, federation}, - OwnedRoomId, OwnedServerName, RoomAliasId, -}; +use std::iter::once; -impl super::Service { - pub(super) async fn remote_resolve( - &self, room_alias: &RoomAliasId, servers: Option<&Vec>, - ) -> Result<(OwnedRoomId, Option>)> { - debug!(?room_alias, ?servers, "resolve"); +use conduit::{debug, debug_error, err, implement, Result}; +use federation::query::get_room_information::v1::Response; +use ruma::{api::federation, OwnedRoomId, OwnedServerName, RoomAliasId, ServerName}; - let mut response = self - .services - .sending - .send_federation_request( - room_alias.server_name(), - federation::query::get_room_information::v1::Request { - room_alias: room_alias.to_owned(), - }, - ) - .await; +#[implement(super::Service)] +pub(super) async fn remote_resolve( + &self, room_alias: &RoomAliasId, servers: Vec, +) -> Result<(OwnedRoomId, Vec)> { + debug!(?room_alias, servers = ?servers, "resolve"); + let servers = once(room_alias.server_name()) + .map(ToOwned::to_owned) + .chain(servers.into_iter()); - debug!("room alias server_name get_alias_helper response: {response:?}"); + let mut resolved_servers = Vec::new(); + let mut resolved_room_id: Option = None; + for server in servers { + match self.remote_request(room_alias, &server).await { + Err(e) => debug_error!("Failed to query for {room_alias:?} from {server}: {e}"), + Ok(Response { + room_id, + servers, + }) => { + debug!("Server {server} answered with {room_id:?} for {room_alias:?} servers: {servers:?}"); - if let Err(ref e) = response { - debug_warn!( - "Server {} of the original room alias failed to assist in resolving room alias: {e}", - room_alias.server_name(), - ); - } + resolved_room_id.get_or_insert(room_id); + add_server(&mut resolved_servers, server); - if response.as_ref().is_ok_and(|resp| resp.servers.is_empty()) || response.as_ref().is_err() { - if let Some(servers) = servers { - for server in servers { - response = self - .services - .sending - .send_federation_request( - server, - federation::query::get_room_information::v1::Request { - room_alias: room_alias.to_owned(), - }, - ) - .await; - debug!("Got response from server {server} for room aliases: {response:?}"); - - if let Ok(ref response) = response { - if !response.servers.is_empty() { - break; - } - debug_warn!( - "Server {server} responded with room aliases, but was empty? Response: {response:?}" - ); - } + if !servers.is_empty() { + add_servers(&mut resolved_servers, servers); + break; } - } + }, } + } - if let Ok(response) = response { - let room_id = response.room_id; + resolved_room_id + .map(|room_id| (room_id, resolved_servers)) + .ok_or_else(|| err!(Request(NotFound("No servers could assist in resolving the room alias")))) +} - let mut pre_servers = response.servers; - // since the room alis server responded, insert it into the list - pre_servers.push(room_alias.server_name().into()); +#[implement(super::Service)] +async fn remote_request(&self, room_alias: &RoomAliasId, server: &ServerName) -> Result { + use federation::query::get_room_information::v1::Request; - return Ok((room_id, Some(pre_servers))); - } + let request = Request { + room_alias: room_alias.to_owned(), + }; - Err(Error::BadRequest( - ErrorKind::NotFound, - "No servers could assist in resolving the room alias", - )) + self.services + .sending + .send_federation_request(server, request) + .await +} + +fn add_servers(servers: &mut Vec, new: Vec) { + for server in new { + add_server(servers, server); + } +} + +fn add_server(servers: &mut Vec, server: OwnedServerName) { + if !servers.contains(&server) { + servers.push(server); } } From ed5b5d7877996f0ca4862ee3b08cfebd35744904 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 15 Oct 2024 09:34:43 +0000 Subject: [PATCH 0918/2091] merge rooms state service and data Signed-off-by: Jason Volk --- src/service/rooms/state/data.rs | 65 ----------------------------- src/service/rooms/state/mod.rs | 73 +++++++++++++++++++++++---------- 2 files changed, 52 insertions(+), 86 deletions(-) delete mode 100644 src/service/rooms/state/data.rs diff --git a/src/service/rooms/state/data.rs b/src/service/rooms/state/data.rs deleted file mode 100644 index 813f48ae..00000000 --- a/src/service/rooms/state/data.rs +++ /dev/null @@ -1,65 +0,0 @@ -use std::sync::Arc; - -use conduit::{ - utils::{stream::TryIgnore, ReadyExt}, - Result, -}; -use database::{Database, Deserialized, Interfix, Map}; -use ruma::{OwnedEventId, RoomId}; - -use super::RoomMutexGuard; - -pub(super) struct Data { - shorteventid_shortstatehash: Arc, - roomid_shortstatehash: Arc, - pub(super) roomid_pduleaves: Arc, -} - -impl Data { - pub(super) fn new(db: &Arc) -> Self { - Self { - shorteventid_shortstatehash: db["shorteventid_shortstatehash"].clone(), - roomid_shortstatehash: db["roomid_shortstatehash"].clone(), - roomid_pduleaves: db["roomid_pduleaves"].clone(), - } - } - - pub(super) async fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result { - self.roomid_shortstatehash.get(room_id).await.deserialized() - } - - #[inline] - pub(super) fn set_room_state( - &self, - room_id: &RoomId, - new_shortstatehash: u64, - _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex - ) { - self.roomid_shortstatehash - .raw_put(room_id, new_shortstatehash); - } - - pub(super) fn set_event_state(&self, shorteventid: u64, shortstatehash: u64) { - self.shorteventid_shortstatehash - .put(shorteventid, shortstatehash); - } - - pub(super) async fn set_forward_extremities( - &self, - room_id: &RoomId, - event_ids: Vec, - _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex - ) { - let prefix = (room_id, Interfix); - self.roomid_pduleaves - .keys_prefix_raw(&prefix) - .ignore_err() - .ready_for_each(|key| self.roomid_pduleaves.remove(key)) - .await; - - for event_id in &event_ids { - let key = (room_id, event_id); - self.roomid_pduleaves.put_raw(key, event_id); - } - } -} diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index cfcb2da6..6abaa198 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -1,5 +1,3 @@ -mod data; - use std::{ collections::{HashMap, HashSet}, fmt::Write, @@ -10,11 +8,10 @@ use std::{ use conduit::{ err, result::FlatOk, - utils::{calculate_hash, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard}, + utils::{calculate_hash, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, warn, PduEvent, Result, }; -use data::Data; -use database::{Ignore, Interfix}; +use database::{Deserialized, Ignore, Interfix, Map}; use futures::{future::join_all, pin_mut, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{ events::{ @@ -30,9 +27,9 @@ use super::state_compressor::CompressedStateEvent; use crate::{globals, rooms, Dep}; pub struct Service { + pub mutex: RoomMutexMap, services: Services, db: Data, - pub mutex: RoomMutexMap, } struct Services { @@ -45,12 +42,19 @@ struct Services { timeline: Dep, } +struct Data { + shorteventid_shortstatehash: Arc, + roomid_shortstatehash: Arc, + roomid_pduleaves: Arc, +} + type RoomMutexMap = MutexMap; pub type RoomMutexGuard = MutexMapGuard; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + mutex: RoomMutexMap::new(), services: Services { globals: args.depend::("globals"), short: args.depend::("rooms::short"), @@ -60,12 +64,15 @@ impl crate::Service for Service { state_compressor: args.depend::("rooms::state_compressor"), timeline: args.depend::("rooms::timeline"), }, - db: Data::new(args.db), - mutex: RoomMutexMap::new(), + db: Data { + shorteventid_shortstatehash: args.db["shorteventid_shortstatehash"].clone(), + roomid_shortstatehash: args.db["roomid_shortstatehash"].clone(), + roomid_pduleaves: args.db["roomid_pduleaves"].clone(), + }, })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + fn memory_usage(&self, out: &mut dyn Write) -> Result { let mutex = self.mutex.len(); writeln!(out, "state_mutex: {mutex}")?; @@ -84,7 +91,7 @@ impl Service { statediffnew: Arc>, _statediffremoved: Arc>, state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex - ) -> Result<()> { + ) -> Result { let event_ids = statediffnew.iter().stream().filter_map(|new| { self.services .state_compressor @@ -127,7 +134,7 @@ impl Service { self.services.state_cache.update_joined_count(room_id).await; - self.db.set_room_state(room_id, shortstatehash, state_lock); + self.set_room_state(room_id, shortstatehash, state_lock); Ok(()) } @@ -140,13 +147,15 @@ impl Service { pub async fn set_event_state( &self, event_id: &EventId, room_id: &RoomId, state_ids_compressed: Arc>, ) -> Result { + const BUFSIZE: usize = size_of::(); + let shorteventid = self .services .short .get_or_create_shorteventid(event_id) .await; - let previous_shortstatehash = self.db.get_room_shortstatehash(room_id).await; + let previous_shortstatehash = self.get_room_shortstatehash(room_id).await; let state_hash = calculate_hash( &state_ids_compressed @@ -196,7 +205,9 @@ impl Service { )?; } - self.db.set_event_state(shorteventid, shortstatehash); + self.db + .shorteventid_shortstatehash + .aput::(shorteventid, shortstatehash); Ok(shortstatehash) } @@ -207,6 +218,8 @@ impl Service { /// to `stateid_pduid` and adds the incoming event to `eventid_statehash`. #[tracing::instrument(skip(self, new_pdu), level = "debug")] pub async fn append_to_state(&self, new_pdu: &PduEvent) -> Result { + const BUFSIZE: usize = size_of::(); + let shorteventid = self .services .short @@ -216,7 +229,9 @@ impl Service { let previous_shortstatehash = self.get_room_shortstatehash(&new_pdu.room_id).await; if let Ok(p) = previous_shortstatehash { - self.db.set_event_state(shorteventid, p); + self.db + .shorteventid_shortstatehash + .aput::(shorteventid, p); } if let Some(state_key) = &new_pdu.state_key { @@ -306,14 +321,18 @@ impl Service { } /// Set the state hash to a new version, but does not update state_cache. - #[tracing::instrument(skip(self, mutex_lock), level = "debug")] + #[tracing::instrument(skip(self, _mutex_lock), level = "debug")] pub fn set_room_state( &self, room_id: &RoomId, shortstatehash: u64, - mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) { - self.db.set_room_state(room_id, shortstatehash, mutex_lock); + const BUFSIZE: usize = size_of::(); + + self.db + .roomid_shortstatehash + .raw_aput::(room_id, shortstatehash); } /// Returns the room's version. @@ -327,9 +346,12 @@ impl Service { .map_err(|e| err!(Request(NotFound("No create event found: {e:?}")))) } - #[inline] pub async fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result { - self.db.get_room_shortstatehash(room_id).await + self.db + .roomid_shortstatehash + .get(room_id) + .await + .deserialized() } pub fn get_forward_extremities<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + '_ { @@ -346,11 +368,20 @@ impl Service { &self, room_id: &RoomId, event_ids: Vec, - state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex + _state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) { + let prefix = (room_id, Interfix); self.db - .set_forward_extremities(room_id, event_ids, state_lock) + .roomid_pduleaves + .keys_prefix_raw(&prefix) + .ignore_err() + .ready_for_each(|key| self.db.roomid_pduleaves.remove(key)) .await; + + for event_id in &event_ids { + let key = (room_id, event_id); + self.db.roomid_pduleaves.put_raw(key, event_id); + } } /// This fetches auth events from the current state. From 4576313a7c4a2f89fc2c2b1f04ad739dbb546b0a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 15 Oct 2024 09:54:20 +0000 Subject: [PATCH 0919/2091] merge rooms user service and data Signed-off-by: Jason Volk --- src/service/rooms/user/data.rs | 108 --------------------- src/service/rooms/user/mod.rs | 168 +++++++++++++++++++++++---------- 2 files changed, 118 insertions(+), 158 deletions(-) delete mode 100644 src/service/rooms/user/data.rs diff --git a/src/service/rooms/user/data.rs b/src/service/rooms/user/data.rs deleted file mode 100644 index 96b009f8..00000000 --- a/src/service/rooms/user/data.rs +++ /dev/null @@ -1,108 +0,0 @@ -use std::sync::Arc; - -use conduit::Result; -use database::{Deserialized, Map}; -use futures::{Stream, StreamExt}; -use ruma::{RoomId, UserId}; - -use crate::{globals, rooms, Dep}; - -pub(super) struct Data { - userroomid_notificationcount: Arc, - userroomid_highlightcount: Arc, - roomuserid_lastnotificationread: Arc, - roomsynctoken_shortstatehash: Arc, - services: Services, -} - -struct Services { - globals: Dep, - short: Dep, - state_cache: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - userroomid_notificationcount: db["userroomid_notificationcount"].clone(), - userroomid_highlightcount: db["userroomid_highlightcount"].clone(), - roomuserid_lastnotificationread: db["userroomid_highlightcount"].clone(), //< NOTE: known bug from conduit - roomsynctoken_shortstatehash: db["roomsynctoken_shortstatehash"].clone(), - services: Services { - globals: args.depend::("globals"), - short: args.depend::("rooms::short"), - state_cache: args.depend::("rooms::state_cache"), - }, - } - } - - pub(super) fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) { - let userroom_id = (user_id, room_id); - self.userroomid_highlightcount.put(userroom_id, 0_u64); - self.userroomid_notificationcount.put(userroom_id, 0_u64); - - let roomuser_id = (room_id, user_id); - let count = self.services.globals.next_count().unwrap(); - self.roomuserid_lastnotificationread.put(roomuser_id, count); - } - - pub(super) async fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { - let key = (user_id, room_id); - self.userroomid_notificationcount - .qry(&key) - .await - .deserialized() - .unwrap_or(0) - } - - pub(super) async fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { - let key = (user_id, room_id); - self.userroomid_highlightcount - .qry(&key) - .await - .deserialized() - .unwrap_or(0) - } - - pub(super) async fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> u64 { - let key = (room_id, user_id); - self.roomuserid_lastnotificationread - .qry(&key) - .await - .deserialized() - .unwrap_or(0) - } - - pub(super) async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: u64) { - let shortroomid = self - .services - .short - .get_shortroomid(room_id) - .await - .expect("room exists"); - - let key: &[u64] = &[shortroomid, token]; - self.roomsynctoken_shortstatehash.put(key, shortstatehash); - } - - pub(super) async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result { - let shortroomid = self.services.short.get_shortroomid(room_id).await?; - - let key: &[u64] = &[shortroomid, token]; - self.roomsynctoken_shortstatehash - .qry(key) - .await - .deserialized() - } - - //TODO: optimize; replace point-queries with dual iteration - pub(super) fn get_shared_rooms<'a>( - &'a self, user_a: &'a UserId, user_b: &'a UserId, - ) -> impl Stream + Send + 'a { - self.services - .state_cache - .rooms_joined(user_a) - .filter(|room_id| self.services.state_cache.is_joined(user_b, room_id)) - } -} diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index d9d90ecf..e484203d 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -1,71 +1,139 @@ -mod data; - use std::sync::Arc; -use conduit::Result; +use conduit::{implement, Result}; +use database::{Deserialized, Map}; use futures::{pin_mut, Stream, StreamExt}; use ruma::{RoomId, UserId}; -use self::data::Data; +use crate::{globals, rooms, Dep}; pub struct Service { db: Data, + services: Services, +} + +struct Data { + userroomid_notificationcount: Arc, + userroomid_highlightcount: Arc, + roomuserid_lastnotificationread: Arc, + roomsynctoken_shortstatehash: Arc, +} + +struct Services { + globals: Dep, + short: Dep, + state_cache: Dep, } impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(&args), + db: Data { + userroomid_notificationcount: args.db["userroomid_notificationcount"].clone(), + userroomid_highlightcount: args.db["userroomid_highlightcount"].clone(), + roomuserid_lastnotificationread: args.db["userroomid_highlightcount"].clone(), //< NOTE: known bug from conduit + roomsynctoken_shortstatehash: args.db["roomsynctoken_shortstatehash"].clone(), + }, + + services: Services { + globals: args.depend::("globals"), + short: args.depend::("rooms::short"), + state_cache: args.depend::("rooms::state_cache"), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - #[inline] - pub fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) { - self.db.reset_notification_counts(user_id, room_id); - } +#[implement(Service)] +pub fn reset_notification_counts(&self, user_id: &UserId, room_id: &RoomId) { + let userroom_id = (user_id, room_id); + self.db.userroomid_highlightcount.put(userroom_id, 0_u64); + self.db.userroomid_notificationcount.put(userroom_id, 0_u64); - #[inline] - pub async fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { - self.db.notification_count(user_id, room_id).await - } - - #[inline] - pub async fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { - self.db.highlight_count(user_id, room_id).await - } - - #[inline] - pub async fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> u64 { - self.db.last_notification_read(user_id, room_id).await - } - - #[inline] - pub async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: u64) { - self.db - .associate_token_shortstatehash(room_id, token, shortstatehash) - .await; - } - - #[inline] - pub async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result { - self.db.get_token_shortstatehash(room_id, token).await - } - - #[inline] - pub fn get_shared_rooms<'a>( - &'a self, user_a: &'a UserId, user_b: &'a UserId, - ) -> impl Stream + Send + 'a { - self.db.get_shared_rooms(user_a, user_b) - } - - pub async fn has_shared_rooms<'a>(&'a self, user_a: &'a UserId, user_b: &'a UserId) -> bool { - let get_shared_rooms = self.get_shared_rooms(user_a, user_b); - - pin_mut!(get_shared_rooms); - get_shared_rooms.next().await.is_some() - } + let roomuser_id = (room_id, user_id); + let count = self.services.globals.next_count().unwrap(); + self.db + .roomuserid_lastnotificationread + .put(roomuser_id, count); +} + +#[implement(Service)] +pub async fn notification_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (user_id, room_id); + self.db + .userroomid_notificationcount + .qry(&key) + .await + .deserialized() + .unwrap_or(0) +} + +#[implement(Service)] +pub async fn highlight_count(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (user_id, room_id); + self.db + .userroomid_highlightcount + .qry(&key) + .await + .deserialized() + .unwrap_or(0) +} + +#[implement(Service)] +pub async fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + let key = (room_id, user_id); + self.db + .roomuserid_lastnotificationread + .qry(&key) + .await + .deserialized() + .unwrap_or(0) +} + +#[implement(Service)] +pub async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: u64) { + let shortroomid = self + .services + .short + .get_shortroomid(room_id) + .await + .expect("room exists"); + + let key: &[u64] = &[shortroomid, token]; + self.db + .roomsynctoken_shortstatehash + .put(key, shortstatehash); +} + +#[implement(Service)] +pub async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result { + let shortroomid = self.services.short.get_shortroomid(room_id).await?; + + let key: &[u64] = &[shortroomid, token]; + self.db + .roomsynctoken_shortstatehash + .qry(key) + .await + .deserialized() +} + +#[implement(Service)] +pub async fn has_shared_rooms<'a>(&'a self, user_a: &'a UserId, user_b: &'a UserId) -> bool { + let get_shared_rooms = self.get_shared_rooms(user_a, user_b); + + pin_mut!(get_shared_rooms); + get_shared_rooms.next().await.is_some() +} + +//TODO: optimize; replace point-queries with dual iteration +#[implement(Service)] +pub fn get_shared_rooms<'a>( + &'a self, user_a: &'a UserId, user_b: &'a UserId, +) -> impl Stream + Send + 'a { + self.services + .state_cache + .rooms_joined(user_a) + .filter(|room_id| self.services.state_cache.is_joined(user_b, room_id)) } From 0b085ea84fae3125d956e874cf082532e9f57ca8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 15 Oct 2024 10:34:37 +0000 Subject: [PATCH 0920/2091] merge remaining rooms state_cache data and service Signed-off-by: Jason Volk --- src/service/rooms/state_cache/data.rs | 179 ------------------------- src/service/rooms/state_cache/mod.rs | 184 ++++++++++++++++++++------ 2 files changed, 147 insertions(+), 216 deletions(-) delete mode 100644 src/service/rooms/state_cache/data.rs diff --git a/src/service/rooms/state_cache/data.rs b/src/service/rooms/state_cache/data.rs deleted file mode 100644 index c06c8107..00000000 --- a/src/service/rooms/state_cache/data.rs +++ /dev/null @@ -1,179 +0,0 @@ -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; - -use conduit::{utils::stream::TryIgnore, Result}; -use database::{serialize_to_vec, Deserialized, Interfix, Json, Map}; -use futures::{Stream, StreamExt}; -use ruma::{ - events::{AnyStrippedStateEvent, AnySyncStateEvent}, - serde::Raw, - OwnedRoomId, RoomId, UserId, -}; - -use crate::{globals, Dep}; - -type AppServiceInRoomCache = RwLock>>; -type StrippedStateEventItem = (OwnedRoomId, Vec>); -type SyncStateEventItem = (OwnedRoomId, Vec>); - -pub(super) struct Data { - pub(super) appservice_in_room_cache: AppServiceInRoomCache, - pub(super) roomid_invitedcount: Arc, - pub(super) roomid_inviteviaservers: Arc, - pub(super) roomid_joinedcount: Arc, - pub(super) roomserverids: Arc, - pub(super) roomuserid_invitecount: Arc, - pub(super) roomuserid_joined: Arc, - pub(super) roomuserid_leftcount: Arc, - pub(super) roomuseroncejoinedids: Arc, - pub(super) serverroomids: Arc, - pub(super) userroomid_invitestate: Arc, - pub(super) userroomid_joined: Arc, - pub(super) userroomid_leftstate: Arc, - services: Services, -} - -struct Services { - globals: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - appservice_in_room_cache: RwLock::new(HashMap::new()), - roomid_invitedcount: db["roomid_invitedcount"].clone(), - roomid_inviteviaservers: db["roomid_inviteviaservers"].clone(), - roomid_joinedcount: db["roomid_joinedcount"].clone(), - roomserverids: db["roomserverids"].clone(), - roomuserid_invitecount: db["roomuserid_invitecount"].clone(), - roomuserid_joined: db["roomuserid_joined"].clone(), - roomuserid_leftcount: db["roomuserid_leftcount"].clone(), - roomuseroncejoinedids: db["roomuseroncejoinedids"].clone(), - serverroomids: db["serverroomids"].clone(), - userroomid_invitestate: db["userroomid_invitestate"].clone(), - userroomid_joined: db["userroomid_joined"].clone(), - userroomid_leftstate: db["userroomid_leftstate"].clone(), - services: Services { - globals: args.depend::("globals"), - }, - } - } - - pub(super) fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) { - let key = (user_id, room_id); - - self.roomuseroncejoinedids.put_raw(key, []); - } - - pub(super) fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { - let userroom_id = (user_id, room_id); - let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); - - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); - - self.userroomid_joined.insert(&userroom_id, []); - self.roomuserid_joined.insert(&roomuser_id, []); - - self.userroomid_invitestate.remove(&userroom_id); - self.roomuserid_invitecount.remove(&roomuser_id); - - self.userroomid_leftstate.remove(&userroom_id); - self.roomuserid_leftcount.remove(&roomuser_id); - - self.roomid_inviteviaservers.remove(room_id); - } - - pub(super) fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) { - let userroom_id = (user_id, room_id); - let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); - - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); - - // (timo) TODO - let leftstate = Vec::>::new(); - let count = self.services.globals.next_count().unwrap(); - - self.userroomid_leftstate - .raw_put(&userroom_id, Json(leftstate)); - self.roomuserid_leftcount.raw_put(&roomuser_id, count); - - self.userroomid_joined.remove(&userroom_id); - self.roomuserid_joined.remove(&roomuser_id); - - self.userroomid_invitestate.remove(&userroom_id); - self.roomuserid_invitecount.remove(&roomuser_id); - - self.roomid_inviteviaservers.remove(room_id); - } - - /// Makes a user forget a room. - #[tracing::instrument(skip(self), level = "debug")] - pub(super) fn forget(&self, room_id: &RoomId, user_id: &UserId) { - let userroom_id = (user_id, room_id); - let roomuser_id = (room_id, user_id); - - self.userroomid_leftstate.del(userroom_id); - self.roomuserid_leftcount.del(roomuser_id); - } - - /// Returns an iterator over all rooms a user was invited to. - #[inline] - pub(super) fn rooms_invited<'a>( - &'a self, user_id: &'a UserId, - ) -> impl Stream + Send + 'a { - type Key<'a> = (&'a UserId, &'a RoomId); - type KeyVal<'a> = (Key<'a>, Raw>); - - let prefix = (user_id, Interfix); - self.userroomid_invitestate - .stream_prefix(&prefix) - .ignore_err() - .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) - .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) - .ignore_err() - } - - /// Returns an iterator over all rooms a user left. - #[inline] - pub(super) fn rooms_left<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { - type Key<'a> = (&'a UserId, &'a RoomId); - type KeyVal<'a> = (Key<'a>, Raw>>); - - let prefix = (user_id, Interfix); - self.userroomid_leftstate - .stream_prefix(&prefix) - .ignore_err() - .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) - .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) - .ignore_err() - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) async fn invite_state( - &self, user_id: &UserId, room_id: &RoomId, - ) -> Result>> { - let key = (user_id, room_id); - self.userroomid_invitestate - .qry(&key) - .await - .deserialized() - .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub(super) async fn left_state( - &self, user_id: &UserId, room_id: &RoomId, - ) -> Result>> { - let key = (user_id, room_id); - self.userroomid_leftstate - .qry(&key) - .await - .deserialized() - .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) - } -} diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 077eee10..4f4ff264 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -1,14 +1,14 @@ -mod data; - -use std::{collections::HashSet, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::{Arc, RwLock}, +}; use conduit::{ err, is_not_empty, utils::{stream::TryIgnore, ReadyExt, StreamTools}, warn, Result, }; -use data::Data; -use database::{serialize_to_vec, Deserialized, Ignore, Interfix, Json}; +use database::{serialize_to_vec, Deserialized, Ignore, Interfix, Json, Map}; use futures::{stream::iter, Stream, StreamExt}; use itertools::Itertools; use ruma::{ @@ -29,6 +29,7 @@ use ruma::{ use crate::{account_data, appservice::RegistrationInfo, globals, rooms, users, Dep}; pub struct Service { + appservice_in_room_cache: AppServiceInRoomCache, services: Services, db: Data, } @@ -40,16 +41,49 @@ struct Services { users: Dep, } +struct Data { + roomid_invitedcount: Arc, + roomid_inviteviaservers: Arc, + roomid_joinedcount: Arc, + roomserverids: Arc, + roomuserid_invitecount: Arc, + roomuserid_joined: Arc, + roomuserid_leftcount: Arc, + roomuseroncejoinedids: Arc, + serverroomids: Arc, + userroomid_invitestate: Arc, + userroomid_joined: Arc, + userroomid_leftstate: Arc, +} + +type AppServiceInRoomCache = RwLock>>; +type StrippedStateEventItem = (OwnedRoomId, Vec>); +type SyncStateEventItem = (OwnedRoomId, Vec>); + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + appservice_in_room_cache: RwLock::new(HashMap::new()), services: Services { account_data: args.depend::("account_data"), globals: args.depend::("globals"), state_accessor: args.depend::("rooms::state_accessor"), users: args.depend::("users"), }, - db: Data::new(&args), + db: Data { + roomid_invitedcount: args.db["roomid_invitedcount"].clone(), + roomid_inviteviaservers: args.db["roomid_inviteviaservers"].clone(), + roomid_joinedcount: args.db["roomid_joinedcount"].clone(), + roomserverids: args.db["roomserverids"].clone(), + roomuserid_invitecount: args.db["roomuserid_invitecount"].clone(), + roomuserid_joined: args.db["roomuserid_joined"].clone(), + roomuserid_leftcount: args.db["roomuserid_leftcount"].clone(), + roomuseroncejoinedids: args.db["roomuseroncejoinedids"].clone(), + serverroomids: args.db["serverroomids"].clone(), + userroomid_invitestate: args.db["userroomid_invitestate"].clone(), + userroomid_joined: args.db["userroomid_joined"].clone(), + userroomid_leftstate: args.db["userroomid_leftstate"].clone(), + }, })) } @@ -107,7 +141,7 @@ impl Service { // Check if the user never joined this room if !self.once_joined(user_id, room_id).await { // Add the user ID to the join list then - self.db.mark_as_once_joined(user_id, room_id); + self.mark_as_once_joined(user_id, room_id); // Check if the room has a predecessor if let Ok(Some(predecessor)) = self @@ -186,7 +220,7 @@ impl Service { } } - self.db.mark_as_joined(user_id, room_id); + self.mark_as_joined(user_id, room_id); }, MembershipState::Invite => { // We want to know if the sender is ignored by the receiver @@ -198,7 +232,7 @@ impl Service { .await; }, MembershipState::Leave | MembershipState::Ban => { - self.db.mark_as_left(user_id, room_id); + self.mark_as_left(user_id, room_id); }, _ => {}, } @@ -213,10 +247,9 @@ impl Service { #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] pub async fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> bool { let maybe = self - .db .appservice_in_room_cache .read() - .unwrap() + .expect("locked") .get(room_id) .and_then(|map| map.get(&appservice.registration.id)) .copied(); @@ -242,10 +275,9 @@ impl Service { .ready_any(|userid| appservice.users.is_match(userid.as_str())) .await; - self.db - .appservice_in_room_cache + self.appservice_in_room_cache .write() - .unwrap() + .expect("locked") .entry(room_id.to_owned()) .or_default() .insert(appservice.registration.id.clone(), in_room); @@ -254,21 +286,67 @@ impl Service { } } - /// Direct DB function to directly mark a user as left. It is not - /// recommended to use this directly. You most likely should use - /// `update_membership` instead - #[tracing::instrument(skip(self), level = "debug")] - pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) { self.db.mark_as_left(user_id, room_id); } - /// Direct DB function to directly mark a user as joined. It is not /// recommended to use this directly. You most likely should use /// `update_membership` instead #[tracing::instrument(skip(self), level = "debug")] - pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { self.db.mark_as_joined(user_id, room_id); } + pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { + let userroom_id = (user_id, room_id); + let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); + + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); + + self.db.userroomid_joined.insert(&userroom_id, []); + self.db.roomuserid_joined.insert(&roomuser_id, []); + + self.db.userroomid_invitestate.remove(&userroom_id); + self.db.roomuserid_invitecount.remove(&roomuser_id); + + self.db.userroomid_leftstate.remove(&userroom_id); + self.db.roomuserid_leftcount.remove(&roomuser_id); + + self.db.roomid_inviteviaservers.remove(room_id); + } + + /// Direct DB function to directly mark a user as left. It is not + /// recommended to use this directly. You most likely should use + /// `update_membership` instead + #[tracing::instrument(skip(self), level = "debug")] + pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) { + let userroom_id = (user_id, room_id); + let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); + + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); + + // (timo) TODO + let leftstate = Vec::>::new(); + let count = self.services.globals.next_count().unwrap(); + + self.db + .userroomid_leftstate + .raw_put(&userroom_id, Json(leftstate)); + self.db.roomuserid_leftcount.raw_put(&roomuser_id, count); + + self.db.userroomid_joined.remove(&userroom_id); + self.db.roomuserid_joined.remove(&roomuser_id); + + self.db.userroomid_invitestate.remove(&userroom_id); + self.db.roomuserid_invitecount.remove(&roomuser_id); + + self.db.roomid_inviteviaservers.remove(room_id); + } /// Makes a user forget a room. #[tracing::instrument(skip(self), level = "debug")] - pub fn forget(&self, room_id: &RoomId, user_id: &UserId) { self.db.forget(room_id, user_id); } + pub fn forget(&self, room_id: &RoomId, user_id: &UserId) { + let userroom_id = (user_id, room_id); + let roomuser_id = (room_id, user_id); + + self.db.userroomid_leftstate.del(userroom_id); + self.db.roomuserid_leftcount.del(roomuser_id); + } /// Returns an iterator of all servers participating in this room. #[tracing::instrument(skip(self), level = "debug")] @@ -415,28 +493,56 @@ impl Service { /// Returns an iterator over all rooms a user was invited to. #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_invited<'a>( - &'a self, user_id: &'a UserId, - ) -> impl Stream>)> + Send + 'a { - self.db.rooms_invited(user_id) + pub fn rooms_invited<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + type KeyVal<'a> = (Key<'a>, Raw>); + type Key<'a> = (&'a UserId, &'a RoomId); + + let prefix = (user_id, Interfix); + self.db + .userroomid_invitestate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) + .ignore_err() } #[tracing::instrument(skip(self), level = "debug")] pub async fn invite_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>> { - self.db.invite_state(user_id, room_id).await + let key = (user_id, room_id); + self.db + .userroomid_invitestate + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) } #[tracing::instrument(skip(self), level = "debug")] pub async fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>> { - self.db.left_state(user_id, room_id).await + let key = (user_id, room_id); + self.db + .userroomid_leftstate + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) } /// Returns an iterator over all rooms a user left. #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_left<'a>( - &'a self, user_id: &'a UserId, - ) -> impl Stream>)> + Send + 'a { - self.db.rooms_left(user_id) + pub fn rooms_left<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + type KeyVal<'a> = (Key<'a>, Raw>>); + type Key<'a> = (&'a UserId, &'a RoomId); + + let prefix = (user_id, Interfix); + self.db + .userroomid_leftstate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) + .ignore_err() } #[tracing::instrument(skip(self), level = "debug")] @@ -515,13 +621,13 @@ impl Service { } pub fn get_appservice_in_room_cache_usage(&self) -> (usize, usize) { - let cache = self.db.appservice_in_room_cache.read().expect("locked"); + let cache = self.appservice_in_room_cache.read().expect("locked"); + (cache.len(), cache.capacity()) } pub fn clear_appservice_in_room_cache(&self) { - self.db - .appservice_in_room_cache + self.appservice_in_room_cache .write() .expect("locked") .clear(); @@ -574,13 +680,17 @@ impl Service { self.db.serverroomids.put_raw(serverroom_id, []); } - self.db - .appservice_in_room_cache + self.appservice_in_room_cache .write() .expect("locked") .remove(room_id); } + fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) { + let key = (user_id, room_id); + self.db.roomuseroncejoinedids.put_raw(key, []); + } + pub async fn mark_as_invited( &self, user_id: &UserId, room_id: &RoomId, last_state: Option>>, invite_via: Option>, From 84191656fb1e8340afb1ace819e48537a77c053e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 16 Oct 2024 02:31:36 +0000 Subject: [PATCH 0921/2091] slightly cleanup appservice_in_room Signed-off-by: Jason Volk --- src/service/rooms/state_cache/mod.rs | 61 +++++++++++++--------------- 1 file changed, 29 insertions(+), 32 deletions(-) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 4f4ff264..11684eab 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -5,6 +5,7 @@ use std::{ use conduit::{ err, is_not_empty, + result::LogErr, utils::{stream::TryIgnore, ReadyExt, StreamTools}, warn, Result, }; @@ -246,44 +247,40 @@ impl Service { #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] pub async fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> bool { - let maybe = self + if let Some(cached) = self .appservice_in_room_cache .read() .expect("locked") .get(room_id) .and_then(|map| map.get(&appservice.registration.id)) - .copied(); - - if let Some(b) = maybe { - b - } else { - let bridge_user_id = UserId::parse_with_server_name( - appservice.registration.sender_localpart.as_str(), - self.services.globals.server_name(), - ) - .ok(); - - let in_room = if let Some(id) = &bridge_user_id { - self.is_joined(id, room_id).await - } else { - false - }; - - let in_room = in_room - || self - .room_members(room_id) - .ready_any(|userid| appservice.users.is_match(userid.as_str())) - .await; - - self.appservice_in_room_cache - .write() - .expect("locked") - .entry(room_id.to_owned()) - .or_default() - .insert(appservice.registration.id.clone(), in_room); - - in_room + .copied() + { + return cached; } + + let bridge_user_id = UserId::parse_with_server_name( + appservice.registration.sender_localpart.as_str(), + self.services.globals.server_name(), + ); + + let Ok(bridge_user_id) = bridge_user_id.log_err() else { + return false; + }; + + let in_room = self.is_joined(&bridge_user_id, room_id).await + || self + .room_members(room_id) + .ready_any(|user_id| appservice.users.is_match(user_id.as_str())) + .await; + + self.appservice_in_room_cache + .write() + .expect("locked") + .entry(room_id.into()) + .or_default() + .insert(appservice.registration.id.clone(), in_room); + + in_room } /// Direct DB function to directly mark a user as joined. It is not From 55b8908894303f1e784a881e879d5f8d6773abaf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 16 Oct 2024 03:12:30 +0000 Subject: [PATCH 0922/2091] merge rooms state_compressor service and data Signed-off-by: Jason Volk --- src/service/rooms/state_compressor/data.rs | 84 ------------ src/service/rooms/state_compressor/mod.rs | 142 ++++++++++++++------- 2 files changed, 97 insertions(+), 129 deletions(-) delete mode 100644 src/service/rooms/state_compressor/data.rs diff --git a/src/service/rooms/state_compressor/data.rs b/src/service/rooms/state_compressor/data.rs deleted file mode 100644 index cb020470..00000000 --- a/src/service/rooms/state_compressor/data.rs +++ /dev/null @@ -1,84 +0,0 @@ -use std::{collections::HashSet, mem::size_of, sync::Arc}; - -use conduit::{err, expected, utils, Result}; -use database::{Database, Map}; - -use super::CompressedStateEvent; - -pub(super) struct StateDiff { - pub(super) parent: Option, - pub(super) added: Arc>, - pub(super) removed: Arc>, -} - -pub(super) struct Data { - shortstatehash_statediff: Arc, -} - -impl Data { - pub(super) fn new(db: &Arc) -> Self { - Self { - shortstatehash_statediff: db["shortstatehash_statediff"].clone(), - } - } - - pub(super) async fn get_statediff(&self, shortstatehash: u64) -> Result { - const BUFSIZE: usize = size_of::(); - - let value = self - .shortstatehash_statediff - .aqry::(&shortstatehash) - .await - .map_err(|e| err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")))?; - - let parent = utils::u64_from_bytes(&value[0..size_of::()]).expect("bytes have right length"); - let parent = if parent != 0 { - Some(parent) - } else { - None - }; - - let mut add_mode = true; - let mut added = HashSet::new(); - let mut removed = HashSet::new(); - - let stride = size_of::(); - let mut i = stride; - while let Some(v) = value.get(i..expected!(i + 2 * stride)) { - if add_mode && v.starts_with(&0_u64.to_be_bytes()) { - add_mode = false; - i = expected!(i + stride); - continue; - } - if add_mode { - added.insert(v.try_into().expect("we checked the size above")); - } else { - removed.insert(v.try_into().expect("we checked the size above")); - } - i = expected!(i + 2 * stride); - } - - Ok(StateDiff { - parent, - added: Arc::new(added), - removed: Arc::new(removed), - }) - } - - pub(super) fn save_statediff(&self, shortstatehash: u64, diff: &StateDiff) { - let mut value = diff.parent.unwrap_or(0).to_be_bytes().to_vec(); - for new in diff.added.iter() { - value.extend_from_slice(&new[..]); - } - - if !diff.removed.is_empty() { - value.extend_from_slice(&0_u64.to_be_bytes()); - for removed in diff.removed.iter() { - value.extend_from_slice(&removed[..]); - } - } - - self.shortstatehash_statediff - .insert(&shortstatehash.to_be_bytes(), &value); - } -} diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index cd3f2f73..be66c597 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -1,53 +1,21 @@ -mod data; - use std::{ collections::HashSet, fmt::Write, mem::size_of, - sync::{Arc, Mutex as StdMutex, Mutex}, + sync::{Arc, Mutex}, }; -use conduit::{checked, utils, utils::math::usize_from_f64, Result}; -use data::Data; +use conduit::{checked, err, expected, utils, utils::math::usize_from_f64, Result}; +use database::Map; use lru_cache::LruCache; use ruma::{EventId, RoomId}; -use self::data::StateDiff; use crate::{rooms, Dep}; -type StateInfoLruCache = Mutex< - LruCache< - u64, - Vec<( - u64, // sstatehash - Arc>, // full state - Arc>, // added - Arc>, // removed - )>, - >, ->; - -type ShortStateInfoResult = Vec<( - u64, // sstatehash - Arc>, // full state - Arc>, // added - Arc>, // removed -)>; - -type ParentStatesVec = Vec<( - u64, // sstatehash - Arc>, // full state - Arc>, // added - Arc>, // removed -)>; - -type HashSetCompressStateEvent = (u64, Arc>, Arc>); -pub type CompressedStateEvent = [u8; 2 * size_of::()]; - pub struct Service { + pub stateinfo_cache: Mutex, db: Data, services: Services, - pub stateinfo_cache: StateInfoLruCache, } struct Services { @@ -55,17 +23,42 @@ struct Services { state: Dep, } +struct Data { + shortstatehash_statediff: Arc, +} + +struct StateDiff { + parent: Option, + added: Arc>, + removed: Arc>, +} + +type StateInfoLruCache = LruCache; +type ShortStateInfoVec = Vec; +type ParentStatesVec = Vec; +type ShortStateInfo = ( + u64, // sstatehash + Arc>, // full state + Arc>, // added + Arc>, // removed +); + +type HashSetCompressStateEvent = (u64, Arc>, Arc>); +pub type CompressedStateEvent = [u8; 2 * size_of::()]; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; let cache_capacity = f64::from(config.stateinfo_cache_capacity) * config.cache_capacity_modifier; Ok(Arc::new(Self { - db: Data::new(args.db), + stateinfo_cache: LruCache::new(usize_from_f64(cache_capacity)?).into(), + db: Data { + shortstatehash_statediff: args.db["shortstatehash_statediff"].clone(), + }, services: Services { short: args.depend::("rooms::short"), state: args.depend::("rooms::state"), }, - stateinfo_cache: StdMutex::new(LruCache::new(usize_from_f64(cache_capacity)?)), })) } @@ -84,7 +77,7 @@ impl crate::Service for Service { impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. - pub async fn load_shortstatehash_info(&self, shortstatehash: u64) -> Result { + pub async fn load_shortstatehash_info(&self, shortstatehash: u64) -> Result { if let Some(r) = self .stateinfo_cache .lock() @@ -98,7 +91,7 @@ impl Service { parent, added, removed, - } = self.db.get_statediff(shortstatehash).await?; + } = self.get_statediff(shortstatehash).await?; if let Some(parent) = parent { let mut response = Box::pin(self.load_shortstatehash_info(parent)).await?; @@ -177,12 +170,12 @@ impl Service { /// for this layer /// * `parent_states` - A stack with info on shortstatehash, full state, /// added diff and removed diff for each parent layer - #[tracing::instrument(skip(self, statediffnew, statediffremoved, diff_to_sibling, parent_states), level = "debug")] + #[tracing::instrument(skip_all, level = "debug")] pub fn save_state_from_diff( &self, shortstatehash: u64, statediffnew: Arc>, statediffremoved: Arc>, diff_to_sibling: usize, mut parent_states: ParentStatesVec, - ) -> Result<()> { + ) -> Result { let statediffnew_len = statediffnew.len(); let statediffremoved_len = statediffremoved.len(); let diffsum = checked!(statediffnew_len + statediffremoved_len)?; @@ -226,7 +219,7 @@ impl Service { if parent_states.is_empty() { // There is no parent layer, create a new state - self.db.save_statediff( + self.save_statediff( shortstatehash, &StateDiff { parent: None, @@ -279,7 +272,7 @@ impl Service { )?; } else { // Diff small enough, we add diff as layer on top of parent - self.db.save_statediff( + self.save_statediff( shortstatehash, &StateDiff { parent: Some(parent.0), @@ -324,7 +317,7 @@ impl Service { let states_parents = if let Some(p) = previous_shortstatehash { self.load_shortstatehash_info(p).await.unwrap_or_default() } else { - ShortStateInfoResult::new() + ShortStateInfoVec::new() }; let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { @@ -356,4 +349,63 @@ impl Service { Ok((new_shortstatehash, statediffnew, statediffremoved)) } + + async fn get_statediff(&self, shortstatehash: u64) -> Result { + const BUFSIZE: usize = size_of::(); + const STRIDE: usize = size_of::(); + + let value = self + .db + .shortstatehash_statediff + .aqry::(&shortstatehash) + .await + .map_err(|e| err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")))?; + + let parent = utils::u64_from_bytes(&value[0..size_of::()]) + .ok() + .take_if(|parent| *parent != 0); + + let mut add_mode = true; + let mut added = HashSet::new(); + let mut removed = HashSet::new(); + + let mut i = STRIDE; + while let Some(v) = value.get(i..expected!(i + 2 * STRIDE)) { + if add_mode && v.starts_with(&0_u64.to_be_bytes()) { + add_mode = false; + i = expected!(i + STRIDE); + continue; + } + if add_mode { + added.insert(v.try_into()?); + } else { + removed.insert(v.try_into()?); + } + i = expected!(i + 2 * STRIDE); + } + + Ok(StateDiff { + parent, + added: Arc::new(added), + removed: Arc::new(removed), + }) + } + + fn save_statediff(&self, shortstatehash: u64, diff: &StateDiff) { + let mut value = diff.parent.unwrap_or(0).to_be_bytes().to_vec(); + for new in diff.added.iter() { + value.extend_from_slice(&new[..]); + } + + if !diff.removed.is_empty() { + value.extend_from_slice(&0_u64.to_be_bytes()); + for removed in diff.removed.iter() { + value.extend_from_slice(&removed[..]); + } + } + + self.db + .shortstatehash_statediff + .insert(&shortstatehash.to_be_bytes(), &value); + } } From 828cb96ba9dd8e323a37c0e33d8a6e1c23b84ac5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 16 Oct 2024 05:32:27 +0000 Subject: [PATCH 0923/2091] split client/sync Signed-off-by: Jason Volk --- src/api/client/sync/mod.rs | 67 ++ src/api/client/{sync.rs => sync/v3.rs} | 835 +------------------------ src/api/client/sync/v4.rs | 784 +++++++++++++++++++++++ src/core/utils/mod.rs | 10 + 4 files changed, 870 insertions(+), 826 deletions(-) create mode 100644 src/api/client/sync/mod.rs rename src/api/client/{sync.rs => sync/v3.rs} (56%) create mode 100644 src/api/client/sync/v4.rs diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs new file mode 100644 index 00000000..0cfc7b8b --- /dev/null +++ b/src/api/client/sync/mod.rs @@ -0,0 +1,67 @@ +mod v3; +mod v4; + +use conduit::{ + utils::{math::usize_from_u64_truncated, ReadyExt}, + PduCount, +}; +use futures::StreamExt; +use ruma::{RoomId, UserId}; + +pub(crate) use self::{v3::sync_events_route, v4::sync_events_v4_route}; +use crate::{service::Services, Error, PduEvent, Result}; + +async fn load_timeline( + services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: u64, +) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> { + let timeline_pdus; + let limited = if services + .rooms + .timeline + .last_timeline_count(sender_user, room_id) + .await? + > roomsincecount + { + let mut non_timeline_pdus = services + .rooms + .timeline + .pdus_until(sender_user, room_id, PduCount::max()) + .await? + .ready_take_while(|(pducount, _)| pducount > &roomsincecount); + + // Take the last events for the timeline + timeline_pdus = non_timeline_pdus + .by_ref() + .take(usize_from_u64_truncated(limit)) + .collect::>() + .await + .into_iter() + .rev() + .collect::>(); + + // They /sync response doesn't always return all messages, so we say the output + // is limited unless there are events in non_timeline_pdus + non_timeline_pdus.next().await.is_some() + } else { + timeline_pdus = Vec::new(); + false + }; + Ok((timeline_pdus, limited)) +} + +async fn share_encrypted_room( + services: &Services, sender_user: &UserId, user_id: &UserId, ignore_room: Option<&RoomId>, +) -> bool { + services + .rooms + .user + .get_shared_rooms(sender_user, user_id) + .ready_filter(|&room_id| Some(room_id) != ignore_room) + .any(|other_room_id| { + services + .rooms + .state_accessor + .is_encrypted_room(other_room_id) + }) + .await +} diff --git a/src/api/client/sync.rs b/src/api/client/sync/v3.rs similarity index 56% rename from src/api/client/sync.rs rename to src/api/client/sync/v3.rs index 65af775d..f29fe220 100644 --- a/src/api/client/sync.rs +++ b/src/api/client/sync/v3.rs @@ -1,23 +1,19 @@ use std::{ - cmp::{self, Ordering}, - collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet}, + cmp::{self}, + collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, time::Duration, }; use axum::extract::State; use conduit::{ - debug, err, error, is_equal_to, + err, error, extract_variant, is_equal_to, result::FlatOk, - utils::{ - math::{ruma_from_u64, ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, - BoolExt, IterStream, ReadyExt, TryFutureExtExt, - }, - warn, PduCount, + utils::{math::ruma_from_u64, BoolExt, IterStream, ReadyExt, TryFutureExtExt}, + PduCount, }; -use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt, TryFutureExt}; +use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt}; use ruma::{ api::client::{ - error::ErrorKind, filter::{FilterDefinition, LazyLoadOptions}, sync::sync_events::{ self, @@ -25,43 +21,27 @@ use ruma::{ Ephemeral, Filter, GlobalAccountData, InviteState, InvitedRoom, JoinedRoom, LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms, State as RoomState, Timeline, ToDevice, }, - v4::{SlidingOp, SlidingSyncRoomHero}, DeviceLists, UnreadNotificationsCount, }, uiaa::UiaaResponse, }, - directory::RoomTypeFilter, events::{ presence::PresenceEvent, room::member::{MembershipState, RoomMemberEventContent}, AnyRawAccountDataEvent, StateEventType, - TimelineEventType::{self, *}, + TimelineEventType::*, }, serde::Raw, - state_res::Event, - uint, DeviceId, EventId, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId, + uint, DeviceId, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId, }; -use service::rooms::read_receipt::pack_receipts; use tracing::{Instrument as _, Span}; +use super::{load_timeline, share_encrypted_room}; use crate::{ service::{pdu::EventHash, Services}, utils, Error, PduEvent, Result, Ruma, RumaResponse, }; -const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; -const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = - &[RoomMessage, RoomEncrypted, Sticker, CallInvite, PollStart, Beacon]; - -macro_rules! extract_variant { - ($e:expr, $variant:path) => { - match $e { - $variant(value) => Some(value), - _ => None, - } - }; -} - /// # `GET /_matrix/client/r0/sync` /// /// Synchronize the client's state with the latest state on the server. @@ -1085,800 +1065,3 @@ async fn load_joined_room( unread_thread_notifications: BTreeMap::new(), }) } - -async fn load_timeline( - services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: u64, -) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> { - let timeline_pdus; - let limited = if services - .rooms - .timeline - .last_timeline_count(sender_user, room_id) - .await? - > roomsincecount - { - let mut non_timeline_pdus = services - .rooms - .timeline - .pdus_until(sender_user, room_id, PduCount::max()) - .await? - .ready_take_while(|(pducount, _)| pducount > &roomsincecount); - - // Take the last events for the timeline - timeline_pdus = non_timeline_pdus - .by_ref() - .take(usize_from_u64_truncated(limit)) - .collect::>() - .await - .into_iter() - .rev() - .collect::>(); - - // They /sync response doesn't always return all messages, so we say the output - // is limited unless there are events in non_timeline_pdus - non_timeline_pdus.next().await.is_some() - } else { - timeline_pdus = Vec::new(); - false - }; - Ok((timeline_pdus, limited)) -} - -async fn share_encrypted_room( - services: &Services, sender_user: &UserId, user_id: &UserId, ignore_room: Option<&RoomId>, -) -> bool { - services - .rooms - .user - .get_shared_rooms(sender_user, user_id) - .ready_filter(|&room_id| Some(room_id) != ignore_room) - .any(|other_room_id| { - services - .rooms - .state_accessor - .is_encrypted_room(other_room_id) - }) - .await -} - -/// POST `/_matrix/client/unstable/org.matrix.msc3575/sync` -/// -/// Sliding Sync endpoint (future endpoint: `/_matrix/client/v4/sync`) -pub(crate) async fn sync_events_v4_route( - State(services): State, body: Ruma, -) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.expect("user is authenticated"); - let mut body = body.body; - // Setup watchers, so if there's no response, we can wait for them - let watcher = services.globals.watch(sender_user, &sender_device); - - let next_batch = services.globals.next_count()?; - - let conn_id = body - .conn_id - .clone() - .unwrap_or_else(|| SINGLE_CONNECTION_SYNC.to_owned()); - - let globalsince = body - .pos - .as_ref() - .and_then(|string| string.parse().ok()) - .unwrap_or(0); - - if globalsince != 0 - && !services - .sync - .remembered(sender_user.clone(), sender_device.clone(), conn_id.clone()) - { - debug!("Restarting sync stream because it was gone from the database"); - return Err(Error::Request( - ErrorKind::UnknownPos, - "Connection data lost since last time".into(), - http::StatusCode::BAD_REQUEST, - )); - } - - if globalsince == 0 { - services - .sync - .forget_sync_request_connection(sender_user.clone(), sender_device.clone(), conn_id.clone()); - } - - // Get sticky parameters from cache - let known_rooms = - services - .sync - .update_sync_request_with_cache(sender_user.clone(), sender_device.clone(), &mut body); - - let all_joined_rooms: Vec<_> = services - .rooms - .state_cache - .rooms_joined(sender_user) - .map(ToOwned::to_owned) - .collect() - .await; - - let all_invited_rooms: Vec<_> = services - .rooms - .state_cache - .rooms_invited(sender_user) - .map(|r| r.0) - .collect() - .await; - - let all_rooms = all_joined_rooms - .iter() - .chain(all_invited_rooms.iter()) - .map(Clone::clone) - .collect(); - - if body.extensions.to_device.enabled.unwrap_or(false) { - services - .users - .remove_to_device_events(sender_user, &sender_device, globalsince) - .await; - } - - let mut left_encrypted_users = HashSet::new(); // Users that have left any encrypted rooms the sender was in - let mut device_list_changes = HashSet::new(); - let mut device_list_left = HashSet::new(); - - let mut receipts = sync_events::v4::Receipts { - rooms: BTreeMap::new(), - }; - - let mut account_data = sync_events::v4::AccountData { - global: Vec::new(), - rooms: BTreeMap::new(), - }; - if body.extensions.account_data.enabled.unwrap_or(false) { - account_data.global = services - .account_data - .changes_since(None, sender_user, globalsince) - .await? - .into_iter() - .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) - .collect(); - - if let Some(rooms) = body.extensions.account_data.rooms { - for room in rooms { - account_data.rooms.insert( - room.clone(), - services - .account_data - .changes_since(Some(&room), sender_user, globalsince) - .await? - .into_iter() - .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) - .collect(), - ); - } - } - } - - if body.extensions.e2ee.enabled.unwrap_or(false) { - // Look for device list updates of this account - device_list_changes.extend( - services - .users - .keys_changed(sender_user.as_ref(), globalsince, None) - .map(ToOwned::to_owned) - .collect::>() - .await, - ); - - for room_id in &all_joined_rooms { - let Ok(current_shortstatehash) = services.rooms.state.get_room_shortstatehash(room_id).await else { - error!("Room {room_id} has no state"); - continue; - }; - - let since_shortstatehash = services - .rooms - .user - .get_token_shortstatehash(room_id, globalsince) - .await - .ok(); - - let encrypted_room = services - .rooms - .state_accessor - .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") - .await - .is_ok(); - - if let Some(since_shortstatehash) = since_shortstatehash { - // Skip if there are only timeline changes - if since_shortstatehash == current_shortstatehash { - continue; - } - - let since_encryption = services - .rooms - .state_accessor - .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") - .await; - - let since_sender_member: Option = services - .rooms - .state_accessor - .state_get_content(since_shortstatehash, &StateEventType::RoomMember, sender_user.as_str()) - .ok() - .await; - - let joined_since_last_sync = - since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); - - let new_encrypted_room = encrypted_room && since_encryption.is_err(); - - if encrypted_room { - let current_state_ids = services - .rooms - .state_accessor - .state_full_ids(current_shortstatehash) - .await?; - - let since_state_ids = services - .rooms - .state_accessor - .state_full_ids(since_shortstatehash) - .await?; - - for (key, id) in current_state_ids { - if since_state_ids.get(&key) != Some(&id) { - let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { - error!("Pdu in state not found: {id}"); - continue; - }; - if pdu.kind == RoomMember { - if let Some(state_key) = &pdu.state_key { - let user_id = UserId::parse(state_key.clone()) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; - - if user_id == *sender_user { - continue; - } - - let content: RoomMemberEventContent = pdu.get_content()?; - match content.membership { - MembershipState::Join => { - // A new user joined an encrypted room - if !share_encrypted_room(&services, sender_user, &user_id, Some(room_id)) - .await - { - device_list_changes.insert(user_id); - } - }, - MembershipState::Leave => { - // Write down users that have left encrypted rooms we are in - left_encrypted_users.insert(user_id); - }, - _ => {}, - } - } - } - } - } - if joined_since_last_sync || new_encrypted_room { - // If the user is in a new encrypted room, give them all joined users - device_list_changes.extend( - services - .rooms - .state_cache - .room_members(room_id) - // Don't send key updates from the sender to the sender - .ready_filter(|user_id| sender_user != user_id) - // Only send keys if the sender doesn't share an encrypted room with the target - // already - .filter_map(|user_id| { - share_encrypted_room(&services, sender_user, user_id, Some(room_id)) - .map(|res| res.or_some(user_id.to_owned())) - }) - .collect::>() - .await, - ); - } - } - } - // Look for device list updates in this room - device_list_changes.extend( - services - .users - .keys_changed(room_id.as_ref(), globalsince, None) - .map(ToOwned::to_owned) - .collect::>() - .await, - ); - } - - for user_id in left_encrypted_users { - let dont_share_encrypted_room = !share_encrypted_room(&services, sender_user, &user_id, None).await; - - // If the user doesn't share an encrypted room with the target anymore, we need - // to tell them - if dont_share_encrypted_room { - device_list_left.insert(user_id); - } - } - } - - let mut lists = BTreeMap::new(); - let mut todo_rooms = BTreeMap::new(); // and required state - - for (list_id, list) in &body.lists { - let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) { - Some(true) => &all_invited_rooms, - Some(false) => &all_joined_rooms, - None => &all_rooms, - }; - - let active_rooms = match list.filters.clone().map(|f| f.not_room_types) { - Some(filter) if filter.is_empty() => active_rooms.clone(), - Some(value) => filter_rooms(active_rooms, State(services), &value, true).await, - None => active_rooms.clone(), - }; - - let active_rooms = match list.filters.clone().map(|f| f.room_types) { - Some(filter) if filter.is_empty() => active_rooms.clone(), - Some(value) => filter_rooms(&active_rooms, State(services), &value, false).await, - None => active_rooms, - }; - - let mut new_known_rooms = BTreeSet::new(); - - let ranges = list.ranges.clone(); - lists.insert( - list_id.clone(), - sync_events::v4::SyncList { - ops: ranges - .into_iter() - .map(|mut r| { - r.0 = r.0.clamp( - uint!(0), - UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), - ); - r.1 = - r.1.clamp(r.0, UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX)); - - let room_ids = if !active_rooms.is_empty() { - active_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec() - } else { - Vec::new() - }; - - new_known_rooms.extend(room_ids.iter().cloned()); - for room_id in &room_ids { - let todo_room = todo_rooms - .entry(room_id.clone()) - .or_insert((BTreeSet::new(), 0, u64::MAX)); - - let limit = list - .room_details - .timeline_limit - .map_or(10, u64::from) - .min(100); - - todo_room - .0 - .extend(list.room_details.required_state.iter().cloned()); - - todo_room.1 = todo_room.1.max(limit); - // 0 means unknown because it got out of date - todo_room.2 = todo_room.2.min( - known_rooms - .get(list_id.as_str()) - .and_then(|k| k.get(room_id)) - .copied() - .unwrap_or(0), - ); - } - sync_events::v4::SyncOp { - op: SlidingOp::Sync, - range: Some(r), - index: None, - room_ids, - room_id: None, - } - }) - .collect(), - count: ruma_from_usize(active_rooms.len()), - }, - ); - - if let Some(conn_id) = &body.conn_id { - services.sync.update_sync_known_rooms( - sender_user.clone(), - sender_device.clone(), - conn_id.clone(), - list_id.clone(), - new_known_rooms, - globalsince, - ); - } - } - - let mut known_subscription_rooms = BTreeSet::new(); - for (room_id, room) in &body.room_subscriptions { - if !services.rooms.metadata.exists(room_id).await { - continue; - } - let todo_room = todo_rooms - .entry(room_id.clone()) - .or_insert((BTreeSet::new(), 0, u64::MAX)); - let limit = room.timeline_limit.map_or(10, u64::from).min(100); - todo_room.0.extend(room.required_state.iter().cloned()); - todo_room.1 = todo_room.1.max(limit); - // 0 means unknown because it got out of date - todo_room.2 = todo_room.2.min( - known_rooms - .get("subscriptions") - .and_then(|k| k.get(room_id)) - .copied() - .unwrap_or(0), - ); - known_subscription_rooms.insert(room_id.clone()); - } - - for r in body.unsubscribe_rooms { - known_subscription_rooms.remove(&r); - body.room_subscriptions.remove(&r); - } - - if let Some(conn_id) = &body.conn_id { - services.sync.update_sync_known_rooms( - sender_user.clone(), - sender_device.clone(), - conn_id.clone(), - "subscriptions".to_owned(), - known_subscription_rooms, - globalsince, - ); - } - - if let Some(conn_id) = &body.conn_id { - services.sync.update_sync_subscriptions( - sender_user.clone(), - sender_device.clone(), - conn_id.clone(), - body.room_subscriptions, - ); - } - - let mut rooms = BTreeMap::new(); - for (room_id, (required_state_request, timeline_limit, roomsince)) in &todo_rooms { - let roomsincecount = PduCount::Normal(*roomsince); - - let mut timestamp: Option<_> = None; - let mut invite_state = None; - let (timeline_pdus, limited); - if all_invited_rooms.contains(room_id) { - // TODO: figure out a timestamp we can use for remote invites - invite_state = services - .rooms - .state_cache - .invite_state(sender_user, room_id) - .await - .ok(); - - (timeline_pdus, limited) = (Vec::new(), true); - } else { - (timeline_pdus, limited) = - match load_timeline(&services, sender_user, room_id, roomsincecount, *timeline_limit).await { - Ok(value) => value, - Err(err) => { - warn!("Encountered missing timeline in {}, error {}", room_id, err); - continue; - }, - }; - } - - account_data.rooms.insert( - room_id.clone(), - services - .account_data - .changes_since(Some(room_id), sender_user, *roomsince) - .await? - .into_iter() - .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) - .collect(), - ); - - let vector: Vec<_> = services - .rooms - .read_receipt - .readreceipts_since(room_id, *roomsince) - .filter_map(|(read_user, ts, v)| async move { - (!services - .users - .user_is_ignored(&read_user, sender_user) - .await) - .then_some((read_user, ts, v)) - }) - .collect() - .await; - - let receipt_size = vector.len(); - receipts - .rooms - .insert(room_id.clone(), pack_receipts(Box::new(vector.into_iter()))); - - if roomsince != &0 - && timeline_pdus.is_empty() - && account_data.rooms.get(room_id).is_some_and(Vec::is_empty) - && receipt_size == 0 - { - continue; - } - - let prev_batch = timeline_pdus - .first() - .map_or(Ok::<_, Error>(None), |(pdu_count, _)| { - Ok(Some(match pdu_count { - PduCount::Backfilled(_) => { - error!("timeline in backfill state?!"); - "0".to_owned() - }, - PduCount::Normal(c) => c.to_string(), - })) - })? - .or_else(|| { - if roomsince != &0 { - Some(roomsince.to_string()) - } else { - None - } - }); - - let room_events: Vec<_> = timeline_pdus - .iter() - .stream() - .filter_map(|(_, pdu)| async move { - // list of safe and common non-state events to ignore - if matches!( - &pdu.kind, - RoomMessage - | Sticker | CallInvite - | CallNotify | RoomEncrypted - | Image | File | Audio - | Voice | Video | UnstablePollStart - | PollStart | KeyVerificationStart - | Reaction | Emote | Location - ) && services - .users - .user_is_ignored(&pdu.sender, sender_user) - .await - { - return None; - } - - Some(pdu.to_sync_room_event()) - }) - .collect() - .await; - - for (_, pdu) in timeline_pdus { - let ts = MilliSecondsSinceUnixEpoch(pdu.origin_server_ts); - if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) && !timestamp.is_some_and(|time| time > ts) { - timestamp = Some(ts); - } - } - - let required_state = required_state_request - .iter() - .stream() - .filter_map(|state| async move { - services - .rooms - .state_accessor - .room_state_get(room_id, &state.0, &state.1) - .await - .map(|s| s.to_sync_state_event()) - .ok() - }) - .collect() - .await; - - // Heroes - let heroes: Vec<_> = services - .rooms - .state_cache - .room_members(room_id) - .ready_filter(|member| member != sender_user) - .filter_map(|user_id| { - services - .rooms - .state_accessor - .get_member(room_id, user_id) - .map_ok(|memberevent| SlidingSyncRoomHero { - user_id: user_id.into(), - name: memberevent.displayname, - avatar: memberevent.avatar_url, - }) - .ok() - }) - .take(5) - .collect() - .await; - - let name = match heroes.len().cmp(&(1_usize)) { - Ordering::Greater => { - let firsts = heroes[1..] - .iter() - .map(|h| h.name.clone().unwrap_or_else(|| h.user_id.to_string())) - .collect::>() - .join(", "); - - let last = heroes[0] - .name - .clone() - .unwrap_or_else(|| heroes[0].user_id.to_string()); - - Some(format!("{firsts} and {last}")) - }, - Ordering::Equal => Some( - heroes[0] - .name - .clone() - .unwrap_or_else(|| heroes[0].user_id.to_string()), - ), - Ordering::Less => None, - }; - - let heroes_avatar = if heroes.len() == 1 { - heroes[0].avatar.clone() - } else { - None - }; - - rooms.insert( - room_id.clone(), - sync_events::v4::SlidingSyncRoom { - name: services - .rooms - .state_accessor - .get_name(room_id) - .await - .ok() - .or(name), - avatar: if let Some(heroes_avatar) = heroes_avatar { - ruma::JsOption::Some(heroes_avatar) - } else { - match services.rooms.state_accessor.get_avatar(room_id).await { - ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url), - ruma::JsOption::Null => ruma::JsOption::Null, - ruma::JsOption::Undefined => ruma::JsOption::Undefined, - } - }, - initial: Some(roomsince == &0), - is_dm: None, - invite_state, - unread_notifications: UnreadNotificationsCount { - highlight_count: Some( - services - .rooms - .user - .highlight_count(sender_user, room_id) - .await - .try_into() - .expect("notification count can't go that high"), - ), - notification_count: Some( - services - .rooms - .user - .notification_count(sender_user, room_id) - .await - .try_into() - .expect("notification count can't go that high"), - ), - }, - timeline: room_events, - required_state, - prev_batch, - limited, - joined_count: Some( - services - .rooms - .state_cache - .room_joined_count(room_id) - .await - .unwrap_or(0) - .try_into() - .unwrap_or_else(|_| uint!(0)), - ), - invited_count: Some( - services - .rooms - .state_cache - .room_invited_count(room_id) - .await - .unwrap_or(0) - .try_into() - .unwrap_or_else(|_| uint!(0)), - ), - num_live: None, // Count events in timeline greater than global sync counter - timestamp, - heroes: Some(heroes), - }, - ); - } - - if rooms - .iter() - .all(|(_, r)| r.timeline.is_empty() && r.required_state.is_empty()) - { - // Hang a few seconds so requests are not spammed - // Stop hanging if new info arrives - let default = Duration::from_secs(30); - let duration = cmp::min(body.timeout.unwrap_or(default), default); - _ = tokio::time::timeout(duration, watcher).await; - } - - Ok(sync_events::v4::Response { - initial: globalsince == 0, - txn_id: body.txn_id.clone(), - pos: next_batch.to_string(), - lists, - rooms, - extensions: sync_events::v4::Extensions { - to_device: if body.extensions.to_device.enabled.unwrap_or(false) { - Some(sync_events::v4::ToDevice { - events: services - .users - .get_to_device_events(sender_user, &sender_device) - .collect() - .await, - next_batch: next_batch.to_string(), - }) - } else { - None - }, - e2ee: sync_events::v4::E2EE { - device_lists: DeviceLists { - changed: device_list_changes.into_iter().collect(), - left: device_list_left.into_iter().collect(), - }, - device_one_time_keys_count: services - .users - .count_one_time_keys(sender_user, &sender_device) - .await, - // Fallback keys are not yet supported - device_unused_fallback_key_types: None, - }, - account_data, - receipts, - typing: sync_events::v4::Typing { - rooms: BTreeMap::new(), - }, - }, - delta_token: None, - }) -} - -async fn filter_rooms( - rooms: &[OwnedRoomId], State(services): State, filter: &[RoomTypeFilter], negate: bool, -) -> Vec { - rooms - .iter() - .stream() - .filter_map(|r| async move { - let room_type = services.rooms.state_accessor.get_room_type(r).await; - - if room_type.as_ref().is_err_and(|e| !e.is_not_found()) { - return None; - } - - let room_type_filter = RoomTypeFilter::from(room_type.ok()); - - let include = if negate { - !filter.contains(&room_type_filter) - } else { - filter.is_empty() || filter.contains(&room_type_filter) - }; - - include.then_some(r.to_owned()) - }) - .collect() - .await -} diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs new file mode 100644 index 00000000..2adb3b71 --- /dev/null +++ b/src/api/client/sync/v4.rs @@ -0,0 +1,784 @@ +use std::{ + cmp::{self, Ordering}, + collections::{BTreeMap, BTreeSet, HashSet}, + time::Duration, +}; + +use axum::extract::State; +use conduit::{ + debug, error, extract_variant, + utils::{ + math::{ruma_from_usize, usize_from_ruma}, + BoolExt, IterStream, ReadyExt, TryFutureExtExt, + }, + warn, Error, PduCount, Result, +}; +use futures::{FutureExt, StreamExt, TryFutureExt}; +use ruma::{ + api::client::{ + error::ErrorKind, + sync::sync_events::{ + self, + v4::{SlidingOp, SlidingSyncRoomHero}, + DeviceLists, UnreadNotificationsCount, + }, + }, + directory::RoomTypeFilter, + events::{ + room::member::{MembershipState, RoomMemberEventContent}, + AnyRawAccountDataEvent, StateEventType, + TimelineEventType::{self, *}, + }, + state_res::Event, + uint, MilliSecondsSinceUnixEpoch, OwnedRoomId, UInt, UserId, +}; +use service::{rooms::read_receipt::pack_receipts, Services}; + +use super::{load_timeline, share_encrypted_room}; +use crate::Ruma; + +const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; +const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = + &[RoomMessage, RoomEncrypted, Sticker, CallInvite, PollStart, Beacon]; + +/// POST `/_matrix/client/unstable/org.matrix.msc3575/sync` +/// +/// Sliding Sync endpoint (future endpoint: `/_matrix/client/v4/sync`) +pub(crate) async fn sync_events_v4_route( + State(services): State, body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_device = body.sender_device.expect("user is authenticated"); + let mut body = body.body; + // Setup watchers, so if there's no response, we can wait for them + let watcher = services.globals.watch(sender_user, &sender_device); + + let next_batch = services.globals.next_count()?; + + let conn_id = body + .conn_id + .clone() + .unwrap_or_else(|| SINGLE_CONNECTION_SYNC.to_owned()); + + let globalsince = body + .pos + .as_ref() + .and_then(|string| string.parse().ok()) + .unwrap_or(0); + + if globalsince != 0 + && !services + .sync + .remembered(sender_user.clone(), sender_device.clone(), conn_id.clone()) + { + debug!("Restarting sync stream because it was gone from the database"); + return Err(Error::Request( + ErrorKind::UnknownPos, + "Connection data lost since last time".into(), + http::StatusCode::BAD_REQUEST, + )); + } + + if globalsince == 0 { + services + .sync + .forget_sync_request_connection(sender_user.clone(), sender_device.clone(), conn_id.clone()); + } + + // Get sticky parameters from cache + let known_rooms = + services + .sync + .update_sync_request_with_cache(sender_user.clone(), sender_device.clone(), &mut body); + + let all_joined_rooms: Vec<_> = services + .rooms + .state_cache + .rooms_joined(sender_user) + .map(ToOwned::to_owned) + .collect() + .await; + + let all_invited_rooms: Vec<_> = services + .rooms + .state_cache + .rooms_invited(sender_user) + .map(|r| r.0) + .collect() + .await; + + let all_rooms = all_joined_rooms + .iter() + .chain(all_invited_rooms.iter()) + .map(Clone::clone) + .collect(); + + if body.extensions.to_device.enabled.unwrap_or(false) { + services + .users + .remove_to_device_events(sender_user, &sender_device, globalsince) + .await; + } + + let mut left_encrypted_users = HashSet::new(); // Users that have left any encrypted rooms the sender was in + let mut device_list_changes = HashSet::new(); + let mut device_list_left = HashSet::new(); + + let mut receipts = sync_events::v4::Receipts { + rooms: BTreeMap::new(), + }; + + let mut account_data = sync_events::v4::AccountData { + global: Vec::new(), + rooms: BTreeMap::new(), + }; + if body.extensions.account_data.enabled.unwrap_or(false) { + account_data.global = services + .account_data + .changes_since(None, sender_user, globalsince) + .await? + .into_iter() + .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) + .collect(); + + if let Some(rooms) = body.extensions.account_data.rooms { + for room in rooms { + account_data.rooms.insert( + room.clone(), + services + .account_data + .changes_since(Some(&room), sender_user, globalsince) + .await? + .into_iter() + .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect(), + ); + } + } + } + + if body.extensions.e2ee.enabled.unwrap_or(false) { + // Look for device list updates of this account + device_list_changes.extend( + services + .users + .keys_changed(sender_user.as_ref(), globalsince, None) + .map(ToOwned::to_owned) + .collect::>() + .await, + ); + + for room_id in &all_joined_rooms { + let Ok(current_shortstatehash) = services.rooms.state.get_room_shortstatehash(room_id).await else { + error!("Room {room_id} has no state"); + continue; + }; + + let since_shortstatehash = services + .rooms + .user + .get_token_shortstatehash(room_id, globalsince) + .await + .ok(); + + let encrypted_room = services + .rooms + .state_accessor + .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") + .await + .is_ok(); + + if let Some(since_shortstatehash) = since_shortstatehash { + // Skip if there are only timeline changes + if since_shortstatehash == current_shortstatehash { + continue; + } + + let since_encryption = services + .rooms + .state_accessor + .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") + .await; + + let since_sender_member: Option = services + .rooms + .state_accessor + .state_get_content(since_shortstatehash, &StateEventType::RoomMember, sender_user.as_str()) + .ok() + .await; + + let joined_since_last_sync = + since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + + let new_encrypted_room = encrypted_room && since_encryption.is_err(); + + if encrypted_room { + let current_state_ids = services + .rooms + .state_accessor + .state_full_ids(current_shortstatehash) + .await?; + + let since_state_ids = services + .rooms + .state_accessor + .state_full_ids(since_shortstatehash) + .await?; + + for (key, id) in current_state_ids { + if since_state_ids.get(&key) != Some(&id) { + let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { + error!("Pdu in state not found: {id}"); + continue; + }; + if pdu.kind == RoomMember { + if let Some(state_key) = &pdu.state_key { + let user_id = UserId::parse(state_key.clone()) + .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; + + if user_id == *sender_user { + continue; + } + + let content: RoomMemberEventContent = pdu.get_content()?; + match content.membership { + MembershipState::Join => { + // A new user joined an encrypted room + if !share_encrypted_room(&services, sender_user, &user_id, Some(room_id)) + .await + { + device_list_changes.insert(user_id); + } + }, + MembershipState::Leave => { + // Write down users that have left encrypted rooms we are in + left_encrypted_users.insert(user_id); + }, + _ => {}, + } + } + } + } + } + if joined_since_last_sync || new_encrypted_room { + // If the user is in a new encrypted room, give them all joined users + device_list_changes.extend( + services + .rooms + .state_cache + .room_members(room_id) + // Don't send key updates from the sender to the sender + .ready_filter(|user_id| sender_user != user_id) + // Only send keys if the sender doesn't share an encrypted room with the target + // already + .filter_map(|user_id| { + share_encrypted_room(&services, sender_user, user_id, Some(room_id)) + .map(|res| res.or_some(user_id.to_owned())) + }) + .collect::>() + .await, + ); + } + } + } + // Look for device list updates in this room + device_list_changes.extend( + services + .users + .keys_changed(room_id.as_ref(), globalsince, None) + .map(ToOwned::to_owned) + .collect::>() + .await, + ); + } + + for user_id in left_encrypted_users { + let dont_share_encrypted_room = !share_encrypted_room(&services, sender_user, &user_id, None).await; + + // If the user doesn't share an encrypted room with the target anymore, we need + // to tell them + if dont_share_encrypted_room { + device_list_left.insert(user_id); + } + } + } + + let mut lists = BTreeMap::new(); + let mut todo_rooms = BTreeMap::new(); // and required state + + for (list_id, list) in &body.lists { + let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) { + Some(true) => &all_invited_rooms, + Some(false) => &all_joined_rooms, + None => &all_rooms, + }; + + let active_rooms = match list.filters.clone().map(|f| f.not_room_types) { + Some(filter) if filter.is_empty() => active_rooms.clone(), + Some(value) => filter_rooms(&services, active_rooms, &value, true).await, + None => active_rooms.clone(), + }; + + let active_rooms = match list.filters.clone().map(|f| f.room_types) { + Some(filter) if filter.is_empty() => active_rooms.clone(), + Some(value) => filter_rooms(&services, &active_rooms, &value, false).await, + None => active_rooms, + }; + + let mut new_known_rooms = BTreeSet::new(); + + let ranges = list.ranges.clone(); + lists.insert( + list_id.clone(), + sync_events::v4::SyncList { + ops: ranges + .into_iter() + .map(|mut r| { + r.0 = r.0.clamp( + uint!(0), + UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), + ); + r.1 = + r.1.clamp(r.0, UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX)); + + let room_ids = if !active_rooms.is_empty() { + active_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec() + } else { + Vec::new() + }; + + new_known_rooms.extend(room_ids.iter().cloned()); + for room_id in &room_ids { + let todo_room = todo_rooms + .entry(room_id.clone()) + .or_insert((BTreeSet::new(), 0, u64::MAX)); + + let limit = list + .room_details + .timeline_limit + .map_or(10, u64::from) + .min(100); + + todo_room + .0 + .extend(list.room_details.required_state.iter().cloned()); + + todo_room.1 = todo_room.1.max(limit); + // 0 means unknown because it got out of date + todo_room.2 = todo_room.2.min( + known_rooms + .get(list_id.as_str()) + .and_then(|k| k.get(room_id)) + .copied() + .unwrap_or(0), + ); + } + sync_events::v4::SyncOp { + op: SlidingOp::Sync, + range: Some(r), + index: None, + room_ids, + room_id: None, + } + }) + .collect(), + count: ruma_from_usize(active_rooms.len()), + }, + ); + + if let Some(conn_id) = &body.conn_id { + services.sync.update_sync_known_rooms( + sender_user.clone(), + sender_device.clone(), + conn_id.clone(), + list_id.clone(), + new_known_rooms, + globalsince, + ); + } + } + + let mut known_subscription_rooms = BTreeSet::new(); + for (room_id, room) in &body.room_subscriptions { + if !services.rooms.metadata.exists(room_id).await { + continue; + } + let todo_room = todo_rooms + .entry(room_id.clone()) + .or_insert((BTreeSet::new(), 0, u64::MAX)); + let limit = room.timeline_limit.map_or(10, u64::from).min(100); + todo_room.0.extend(room.required_state.iter().cloned()); + todo_room.1 = todo_room.1.max(limit); + // 0 means unknown because it got out of date + todo_room.2 = todo_room.2.min( + known_rooms + .get("subscriptions") + .and_then(|k| k.get(room_id)) + .copied() + .unwrap_or(0), + ); + known_subscription_rooms.insert(room_id.clone()); + } + + for r in body.unsubscribe_rooms { + known_subscription_rooms.remove(&r); + body.room_subscriptions.remove(&r); + } + + if let Some(conn_id) = &body.conn_id { + services.sync.update_sync_known_rooms( + sender_user.clone(), + sender_device.clone(), + conn_id.clone(), + "subscriptions".to_owned(), + known_subscription_rooms, + globalsince, + ); + } + + if let Some(conn_id) = &body.conn_id { + services.sync.update_sync_subscriptions( + sender_user.clone(), + sender_device.clone(), + conn_id.clone(), + body.room_subscriptions, + ); + } + + let mut rooms = BTreeMap::new(); + for (room_id, (required_state_request, timeline_limit, roomsince)) in &todo_rooms { + let roomsincecount = PduCount::Normal(*roomsince); + + let mut timestamp: Option<_> = None; + let mut invite_state = None; + let (timeline_pdus, limited); + if all_invited_rooms.contains(room_id) { + // TODO: figure out a timestamp we can use for remote invites + invite_state = services + .rooms + .state_cache + .invite_state(sender_user, room_id) + .await + .ok(); + + (timeline_pdus, limited) = (Vec::new(), true); + } else { + (timeline_pdus, limited) = + match load_timeline(&services, sender_user, room_id, roomsincecount, *timeline_limit).await { + Ok(value) => value, + Err(err) => { + warn!("Encountered missing timeline in {}, error {}", room_id, err); + continue; + }, + }; + } + + account_data.rooms.insert( + room_id.clone(), + services + .account_data + .changes_since(Some(room_id), sender_user, *roomsince) + .await? + .into_iter() + .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect(), + ); + + let vector: Vec<_> = services + .rooms + .read_receipt + .readreceipts_since(room_id, *roomsince) + .filter_map(|(read_user, ts, v)| async move { + (!services + .users + .user_is_ignored(&read_user, sender_user) + .await) + .then_some((read_user, ts, v)) + }) + .collect() + .await; + + let receipt_size = vector.len(); + receipts + .rooms + .insert(room_id.clone(), pack_receipts(Box::new(vector.into_iter()))); + + if roomsince != &0 + && timeline_pdus.is_empty() + && account_data.rooms.get(room_id).is_some_and(Vec::is_empty) + && receipt_size == 0 + { + continue; + } + + let prev_batch = timeline_pdus + .first() + .map_or(Ok::<_, Error>(None), |(pdu_count, _)| { + Ok(Some(match pdu_count { + PduCount::Backfilled(_) => { + error!("timeline in backfill state?!"); + "0".to_owned() + }, + PduCount::Normal(c) => c.to_string(), + })) + })? + .or_else(|| { + if roomsince != &0 { + Some(roomsince.to_string()) + } else { + None + } + }); + + let room_events: Vec<_> = timeline_pdus + .iter() + .stream() + .filter_map(|(_, pdu)| async move { + // list of safe and common non-state events to ignore + if matches!( + &pdu.kind, + RoomMessage + | Sticker | CallInvite + | CallNotify | RoomEncrypted + | Image | File | Audio + | Voice | Video | UnstablePollStart + | PollStart | KeyVerificationStart + | Reaction | Emote | Location + ) && services + .users + .user_is_ignored(&pdu.sender, sender_user) + .await + { + return None; + } + + Some(pdu.to_sync_room_event()) + }) + .collect() + .await; + + for (_, pdu) in timeline_pdus { + let ts = MilliSecondsSinceUnixEpoch(pdu.origin_server_ts); + if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) && !timestamp.is_some_and(|time| time > ts) { + timestamp = Some(ts); + } + } + + let required_state = required_state_request + .iter() + .stream() + .filter_map(|state| async move { + services + .rooms + .state_accessor + .room_state_get(room_id, &state.0, &state.1) + .await + .map(|s| s.to_sync_state_event()) + .ok() + }) + .collect() + .await; + + // Heroes + let heroes: Vec<_> = services + .rooms + .state_cache + .room_members(room_id) + .ready_filter(|member| member != sender_user) + .filter_map(|user_id| { + services + .rooms + .state_accessor + .get_member(room_id, user_id) + .map_ok(|memberevent| SlidingSyncRoomHero { + user_id: user_id.into(), + name: memberevent.displayname, + avatar: memberevent.avatar_url, + }) + .ok() + }) + .take(5) + .collect() + .await; + + let name = match heroes.len().cmp(&(1_usize)) { + Ordering::Greater => { + let firsts = heroes[1..] + .iter() + .map(|h| h.name.clone().unwrap_or_else(|| h.user_id.to_string())) + .collect::>() + .join(", "); + + let last = heroes[0] + .name + .clone() + .unwrap_or_else(|| heroes[0].user_id.to_string()); + + Some(format!("{firsts} and {last}")) + }, + Ordering::Equal => Some( + heroes[0] + .name + .clone() + .unwrap_or_else(|| heroes[0].user_id.to_string()), + ), + Ordering::Less => None, + }; + + let heroes_avatar = if heroes.len() == 1 { + heroes[0].avatar.clone() + } else { + None + }; + + rooms.insert( + room_id.clone(), + sync_events::v4::SlidingSyncRoom { + name: services + .rooms + .state_accessor + .get_name(room_id) + .await + .ok() + .or(name), + avatar: if let Some(heroes_avatar) = heroes_avatar { + ruma::JsOption::Some(heroes_avatar) + } else { + match services.rooms.state_accessor.get_avatar(room_id).await { + ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url), + ruma::JsOption::Null => ruma::JsOption::Null, + ruma::JsOption::Undefined => ruma::JsOption::Undefined, + } + }, + initial: Some(roomsince == &0), + is_dm: None, + invite_state, + unread_notifications: UnreadNotificationsCount { + highlight_count: Some( + services + .rooms + .user + .highlight_count(sender_user, room_id) + .await + .try_into() + .expect("notification count can't go that high"), + ), + notification_count: Some( + services + .rooms + .user + .notification_count(sender_user, room_id) + .await + .try_into() + .expect("notification count can't go that high"), + ), + }, + timeline: room_events, + required_state, + prev_batch, + limited, + joined_count: Some( + services + .rooms + .state_cache + .room_joined_count(room_id) + .await + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), + ), + invited_count: Some( + services + .rooms + .state_cache + .room_invited_count(room_id) + .await + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), + ), + num_live: None, // Count events in timeline greater than global sync counter + timestamp, + heroes: Some(heroes), + }, + ); + } + + if rooms + .iter() + .all(|(_, r)| r.timeline.is_empty() && r.required_state.is_empty()) + { + // Hang a few seconds so requests are not spammed + // Stop hanging if new info arrives + let default = Duration::from_secs(30); + let duration = cmp::min(body.timeout.unwrap_or(default), default); + _ = tokio::time::timeout(duration, watcher).await; + } + + Ok(sync_events::v4::Response { + initial: globalsince == 0, + txn_id: body.txn_id.clone(), + pos: next_batch.to_string(), + lists, + rooms, + extensions: sync_events::v4::Extensions { + to_device: if body.extensions.to_device.enabled.unwrap_or(false) { + Some(sync_events::v4::ToDevice { + events: services + .users + .get_to_device_events(sender_user, &sender_device) + .collect() + .await, + next_batch: next_batch.to_string(), + }) + } else { + None + }, + e2ee: sync_events::v4::E2EE { + device_lists: DeviceLists { + changed: device_list_changes.into_iter().collect(), + left: device_list_left.into_iter().collect(), + }, + device_one_time_keys_count: services + .users + .count_one_time_keys(sender_user, &sender_device) + .await, + // Fallback keys are not yet supported + device_unused_fallback_key_types: None, + }, + account_data, + receipts, + typing: sync_events::v4::Typing { + rooms: BTreeMap::new(), + }, + }, + delta_token: None, + }) +} + +async fn filter_rooms( + services: &Services, rooms: &[OwnedRoomId], filter: &[RoomTypeFilter], negate: bool, +) -> Vec { + rooms + .iter() + .stream() + .filter_map(|r| async move { + let room_type = services.rooms.state_accessor.get_room_type(r).await; + + if room_type.as_ref().is_err_and(|e| !e.is_not_found()) { + return None; + } + + let room_type_filter = RoomTypeFilter::from(room_type.ok()); + + let include = if negate { + !filter.contains(&room_type_filter) + } else { + filter.is_empty() || filter.contains(&room_type_filter) + }; + + include.then_some(r.to_owned()) + }) + .collect() + .await +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 3adecc6c..96a98537 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -40,6 +40,16 @@ pub use self::{ #[inline] pub fn exchange(state: &mut T, source: T) -> T { std::mem::replace(state, source) } +#[macro_export] +macro_rules! extract_variant { + ($e:expr, $variant:path) => { + match $e { + $variant(value) => Some(value), + _ => None, + } + }; +} + #[macro_export] macro_rules! at { ($idx:tt) => { From 1fdcab0319f2461ae000a0876823848f9e5af921 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 16 Oct 2024 06:58:37 +0000 Subject: [PATCH 0924/2091] additional sync cleanup Signed-off-by: Jason Volk --- src/api/client/sync/mod.rs | 51 +++++++++++++++++++------------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 0cfc7b8b..ed22010c 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -14,38 +14,37 @@ use crate::{service::Services, Error, PduEvent, Result}; async fn load_timeline( services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: u64, ) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> { - let timeline_pdus; - let limited = if services + let last_timeline_count = services .rooms .timeline .last_timeline_count(sender_user, room_id) + .await?; + + if last_timeline_count <= roomsincecount { + return Ok((Vec::new(), false)); + } + + let mut non_timeline_pdus = services + .rooms + .timeline + .pdus_until(sender_user, room_id, PduCount::max()) .await? - > roomsincecount - { - let mut non_timeline_pdus = services - .rooms - .timeline - .pdus_until(sender_user, room_id, PduCount::max()) - .await? - .ready_take_while(|(pducount, _)| pducount > &roomsincecount); + .ready_take_while(|(pducount, _)| pducount > &roomsincecount); - // Take the last events for the timeline - timeline_pdus = non_timeline_pdus - .by_ref() - .take(usize_from_u64_truncated(limit)) - .collect::>() - .await - .into_iter() - .rev() - .collect::>(); + // Take the last events for the timeline + let timeline_pdus: Vec<_> = non_timeline_pdus + .by_ref() + .take(usize_from_u64_truncated(limit)) + .collect::>() + .await + .into_iter() + .rev() + .collect(); + + // They /sync response doesn't always return all messages, so we say the output + // is limited unless there are events in non_timeline_pdus + let limited = non_timeline_pdus.next().await.is_some(); - // They /sync response doesn't always return all messages, so we say the output - // is limited unless there are events in non_timeline_pdus - non_timeline_pdus.next().await.is_some() - } else { - timeline_pdus = Vec::new(); - false - }; Ok((timeline_pdus, limited)) } From 93130fbb85e4f5645d1b98aeddd8e1828b1b936c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 21 Oct 2024 20:21:00 +0000 Subject: [PATCH 0925/2091] add is_ok to futures TryExtExt utils Signed-off-by: Jason Volk --- src/core/utils/future/try_ext_ext.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/core/utils/future/try_ext_ext.rs b/src/core/utils/future/try_ext_ext.rs index d30d2cac..7c0b36a2 100644 --- a/src/core/utils/future/try_ext_ext.rs +++ b/src/core/utils/future/try_ext_ext.rs @@ -10,6 +10,17 @@ pub trait TryExtExt where Self: TryFuture + Send, { + /// Resolves to a bool for whether the TryFuture (Future of a Result) + /// resolved to Ok or Err. + /// + /// is_ok() has to consume *self rather than borrow. The intent of this + /// extension is therefor for a caller only ever caring about result status + /// while discarding all contents. + #[allow(clippy::wrong_self_convention)] + fn is_ok(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> + where + Self: Sized; + fn map_ok_or( self, default: U, f: F, ) -> MapOkOrElse U, impl FnOnce(Self::Error) -> U> @@ -32,6 +43,14 @@ impl TryExtExt for Fut where Fut: TryFuture + Send, { + #[inline] + fn is_ok(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> + where + Self: Sized, + { + self.map_ok_or(false, |_| true) + } + #[inline] fn map_ok_or( self, default: U, f: F, From ac75ebee8afd9874d02a2c33b86ab388e7ab289b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 16 Oct 2024 11:33:24 +0000 Subject: [PATCH 0926/2091] event_handler/timeline service cleanups Signed-off-by: Jason Volk --- src/service/rooms/event_handler/mod.rs | 67 ++++++++++++++------------ src/service/rooms/timeline/data.rs | 29 ++++------- 2 files changed, 45 insertions(+), 51 deletions(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 0ffd9659..41ab79f1 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -169,7 +169,7 @@ impl Service { .await?; // Procure the room version - let room_version_id = Self::get_room_version_id(&create_event)?; + let room_version_id = get_room_version_id(&create_event)?; let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; @@ -178,7 +178,7 @@ impl Service { .boxed() .await?; - Self::check_room_id(room_id, &incoming_pdu)?; + check_room_id(room_id, &incoming_pdu)?; // 8. if not timeline event: stop if !is_timeline_event { @@ -341,7 +341,7 @@ impl Service { // 2. Check signatures, otherwise drop // 3. check content hash, redact if doesn't match - let room_version_id = Self::get_room_version_id(create_event)?; + let room_version_id = get_room_version_id(create_event)?; let mut val = match self .services .server_keys @@ -378,7 +378,7 @@ impl Service { ) .map_err(|_| Error::bad_database("Event is not a valid PDU."))?; - Self::check_room_id(room_id, &incoming_pdu)?; + check_room_id(room_id, &incoming_pdu)?; if !auth_events_known { // 4. fetch any missing auth events doing all checks listed here starting at 1. @@ -414,7 +414,7 @@ impl Service { continue; }; - Self::check_room_id(room_id, &auth_event)?; + check_room_id(room_id, &auth_event)?; match auth_events.entry(( auth_event.kind.to_string().into(), @@ -454,7 +454,7 @@ impl Service { }; let auth_check = state_res::event_auth::auth_check( - &Self::to_room_version(&room_version_id), + &to_room_version(&room_version_id), &incoming_pdu, None, // TODO: third party invite state_fetch, @@ -502,8 +502,8 @@ impl Service { } debug!("Upgrading to timeline pdu"); - let timer = tokio::time::Instant::now(); - let room_version_id = Self::get_room_version_id(create_event)?; + let timer = Instant::now(); + let room_version_id = get_room_version_id(create_event)?; // 10. Fetch missing state and auth chain events by calling /state_ids at // backwards extremities doing all the checks in this list starting at 1. @@ -524,7 +524,7 @@ impl Service { } let state_at_incoming_event = state_at_incoming_event.expect("we always set this to some above"); - let room_version = Self::to_room_version(&room_version_id); + let room_version = to_room_version(&room_version_id); debug!("Performing auth check"); // 11. Check the auth of the event passes based on the state of the event @@ -1278,7 +1278,7 @@ impl Service { .await .pop() { - Self::check_room_id(room_id, &pdu)?; + check_room_id(room_id, &pdu)?; let limit = self.services.globals.max_fetch_prev_events(); if amount > limit { @@ -1370,31 +1370,34 @@ impl Service { } } - fn check_room_id(room_id: &RoomId, pdu: &PduEvent) -> Result<()> { - if pdu.room_id != room_id { - return Err!(Request(InvalidParam( - warn!(pdu_event_id = ?pdu.event_id, pdu_room_id = ?pdu.room_id, ?room_id, "Found event from room in room") - ))); - } - - Ok(()) - } - - fn get_room_version_id(create_event: &PduEvent) -> Result { - let content: RoomCreateEventContent = create_event.get_content()?; - let room_version = content.room_version; - - Ok(room_version) - } - - #[inline] - fn to_room_version(room_version_id: &RoomVersionId) -> RoomVersion { - RoomVersion::new(room_version_id).expect("room version is supported") - } - async fn event_exists(&self, event_id: Arc) -> bool { self.services.timeline.pdu_exists(&event_id).await } async fn event_fetch(&self, event_id: Arc) -> Option> { self.services.timeline.get_pdu(&event_id).await.ok() } } + +fn check_room_id(room_id: &RoomId, pdu: &PduEvent) -> Result { + if pdu.room_id != room_id { + return Err!(Request(InvalidParam(error!( + pdu_event_id = ?pdu.event_id, + pdu_room_id = ?pdu.room_id, + ?room_id, + "Found event from room in room", + )))); + } + + Ok(()) +} + +fn get_room_version_id(create_event: &PduEvent) -> Result { + let content: RoomCreateEventContent = create_event.get_content()?; + let room_version = content.room_version; + + Ok(room_version) +} + +#[inline] +fn to_room_version(room_version_id: &RoomVersionId) -> RoomVersion { + RoomVersion::new(room_version_id).expect("room version is supported") +} diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index c51b7856..5428a3b9 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -8,11 +8,11 @@ use conduit::{ err, expected, result::{LogErr, NotFound}, utils, - utils::{stream::TryIgnore, u64_from_u8, ReadyExt}, + utils::{future::TryExtExt, stream::TryIgnore, u64_from_u8, ReadyExt}, Err, PduCount, PduEvent, Result, }; use database::{Database, Deserialized, Json, KeyVal, Map}; -use futures::{FutureExt, Stream, StreamExt}; +use futures::{Stream, StreamExt}; use ruma::{CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; use tokio::sync::Mutex; @@ -115,12 +115,10 @@ impl Data { /// Like get_non_outlier_pdu(), but without the expense of fetching and /// parsing the PduEvent - pub(super) async fn non_outlier_pdu_exists(&self, event_id: &EventId) -> Result<()> { + pub(super) async fn non_outlier_pdu_exists(&self, event_id: &EventId) -> Result { let pduid = self.get_pdu_id(event_id).await?; - self.pduid_pdu.get(&pduid).await?; - - Ok(()) + self.pduid_pdu.get(&pduid).await.map(|_| ()) } /// Returns the pdu. @@ -140,16 +138,14 @@ impl Data { /// Like get_non_outlier_pdu(), but without the expense of fetching and /// parsing the PduEvent - pub(super) async fn outlier_pdu_exists(&self, event_id: &EventId) -> Result<()> { - self.eventid_outlierpdu.get(event_id).await?; - - Ok(()) + pub(super) async fn outlier_pdu_exists(&self, event_id: &EventId) -> Result { + self.eventid_outlierpdu.get(event_id).await.map(|_| ()) } /// Like get_pdu(), but without the expense of fetching and parsing the data pub(super) async fn pdu_exists(&self, event_id: &EventId) -> bool { - let non_outlier = self.non_outlier_pdu_exists(event_id).map(|res| res.is_ok()); - let outlier = self.outlier_pdu_exists(event_id).map(|res| res.is_ok()); + let non_outlier = self.non_outlier_pdu_exists(event_id).is_ok(); + let outlier = self.outlier_pdu_exists(event_id).is_ok(); //TODO: parallelize non_outlier.await || outlier.await @@ -169,7 +165,6 @@ impl Data { pub(super) async fn append_pdu(&self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64) { self.pduid_pdu.raw_put(pdu_id, Json(json)); - self.lasttimelinecount_cache .lock() .await @@ -181,21 +176,17 @@ impl Data { pub(super) fn prepend_backfill_pdu(&self, pdu_id: &[u8], event_id: &EventId, json: &CanonicalJsonObject) { self.pduid_pdu.raw_put(pdu_id, Json(json)); - self.eventid_pduid.insert(event_id, pdu_id); self.eventid_outlierpdu.remove(event_id); } /// Removes a pdu and creates a new one with the same id. - pub(super) async fn replace_pdu( - &self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, _pdu: &PduEvent, - ) -> Result<()> { + pub(super) async fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, _pdu: &PduEvent) -> Result { if self.pduid_pdu.get(pdu_id).await.is_not_found() { return Err!(Request(NotFound("PDU does not exist."))); } - let pdu = serde_json::to_vec(pdu_json)?; - self.pduid_pdu.insert(pdu_id, &pdu); + self.pduid_pdu.raw_put(pdu_id, Json(pdu_json)); Ok(()) } From b505f0d0d7a8ec2accc4b38dfe3391c9f780ba25 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 21 Oct 2024 22:00:39 +0000 Subject: [PATCH 0927/2091] add (back) query_trusted_key_servers_first w/ additional configuration detail Signed-off-by: Jason Volk --- src/core/config/mod.rs | 29 +++++++++++++++ src/service/server_keys/acquire.rs | 59 +++++++++++++++++++++++------- src/service/server_keys/get.rs | 47 ++++++++++++++++++++---- src/service/server_keys/mod.rs | 4 +- 4 files changed, 116 insertions(+), 23 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 02b277d0..52ce8a01 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -490,6 +490,35 @@ pub struct Config { #[serde(default = "default_trusted_servers")] pub trusted_servers: Vec, + /// Whether to query the servers listed in trusted_servers first or query + /// the origin server first. For best security, querying the origin server + /// first is advised to minimize the exposure to a compromised trusted + /// server. For maximum performance this can be set to true, however other + /// options exist to query trusted servers first under specific high-load + /// circumstances and should be evaluated before setting this to true. + #[serde(default)] + pub query_trusted_key_servers_first: bool, + + /// Whether to query the servers listed in trusted_servers first + /// specifically on room joins. This option limits the exposure to a + /// compromised trusted server to room joins only. The join operation + /// requires gathering keys from many origin servers which can cause + /// significant delays. Therefor this defaults to true to mitigate + /// unexpected delays out-of-the-box. The security-paranoid or those + /// willing to tolerate delays are advised to set this to false. Note that + /// setting query_trusted_key_servers_first to true causes this option to + /// be ignored. + #[serde(default = "true_fn")] + pub query_trusted_key_servers_first_on_join: bool, + + /// Only query trusted servers for keys and never the origin server. This is + /// intended for clusters or custom deployments using their trusted_servers + /// as forwarding-agents to cache and deduplicate requests. Notary servers + /// do not act as forwarding-agents by default, therefor do not enable this + /// unless you know exactly what you are doing. + #[serde(default)] + pub only_query_trusted_key_servers: bool, + /// max log level for conduwuit. allows debug, info, warn, or error /// see also: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives /// **Caveat**: diff --git a/src/service/server_keys/acquire.rs b/src/service/server_keys/acquire.rs index 2b170040..25b676b8 100644 --- a/src/service/server_keys/acquire.rs +++ b/src/service/server_keys/acquire.rs @@ -47,35 +47,66 @@ where S: Iterator + Send + Clone, K: Iterator + Send + Clone, { + let notary_only = self.services.server.config.only_query_trusted_key_servers; + let notary_first_always = self.services.server.config.query_trusted_key_servers_first; + let notary_first_on_join = self + .services + .server + .config + .query_trusted_key_servers_first_on_join; + let requested_servers = batch.clone().count(); let requested_keys = batch.clone().flat_map(|(_, key_ids)| key_ids).count(); debug!("acquire {requested_keys} keys from {requested_servers}"); - let missing = self.acquire_locals(batch).await; - let missing_keys = keys_count(&missing); - let missing_servers = missing.len(); + let mut missing = self.acquire_locals(batch).await; + let mut missing_keys = keys_count(&missing); + let mut missing_servers = missing.len(); if missing_servers == 0 { return; } debug!("missing {missing_keys} keys for {missing_servers} servers locally"); - let missing = self.acquire_origins(missing.into_iter()).await; - let missing_keys = keys_count(&missing); - let missing_servers = missing.len(); - if missing_servers == 0 { - return; + if notary_first_always || notary_first_on_join { + missing = self.acquire_notary(missing.into_iter()).await; + missing_keys = keys_count(&missing); + missing_servers = missing.len(); + if missing_keys == 0 { + return; + } + + debug_warn!("missing {missing_keys} keys for {missing_servers} servers from all notaries first"); } - debug_warn!("missing {missing_keys} keys for {missing_servers} servers unreachable"); + if !notary_only { + missing = self.acquire_origins(missing.into_iter()).await; + missing_keys = keys_count(&missing); + missing_servers = missing.len(); + if missing_keys == 0 { + return; + } + + debug_warn!("missing {missing_keys} keys for {missing_servers} servers unreachable"); + } + + if !notary_first_always && !notary_first_on_join { + missing = self.acquire_notary(missing.into_iter()).await; + missing_keys = keys_count(&missing); + missing_servers = missing.len(); + if missing_keys == 0 { + return; + } + + debug_warn!("still missing {missing_keys} keys for {missing_servers} servers from all notaries."); + } - let missing = self.acquire_notary(missing.into_iter()).await; - let missing_keys = keys_count(&missing); - let missing_servers = missing.len(); if missing_keys > 0 { - debug_warn!("still missing {missing_keys} keys for {missing_servers} servers from all notaries"); - warn!("did not obtain {missing_keys} of {requested_keys} keys; some events may not be accepted"); + warn!( + "did not obtain {missing_keys} keys for {missing_servers} servers out of {requested_keys} total keys for \ + {requested_servers} total servers; some events may not be verifiable" + ); } } diff --git a/src/service/server_keys/get.rs b/src/service/server_keys/get.rs index 0f449b46..441e33d4 100644 --- a/src/service/server_keys/get.rs +++ b/src/service/server_keys/get.rs @@ -53,17 +53,40 @@ where #[implement(super::Service)] pub async fn get_verify_key(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> Result { + let notary_first = self.services.server.config.query_trusted_key_servers_first; + let notary_only = self.services.server.config.only_query_trusted_key_servers; + if let Some(result) = self.verify_keys_for(origin).await.remove(key_id) { return Ok(result); } - if let Ok(server_key) = self.server_request(origin).await { - self.add_signing_keys(server_key.clone()).await; - if let Some(result) = extract_key(server_key, key_id) { + if notary_first { + if let Ok(result) = self.get_verify_key_from_notaries(origin, key_id).await { return Ok(result); } } + if !notary_only { + if let Ok(result) = self.get_verify_key_from_origin(origin, key_id).await { + return Ok(result); + } + } + + if !notary_first { + if let Ok(result) = self.get_verify_key_from_notaries(origin, key_id).await { + return Ok(result); + } + } + + Err!(BadServerResponse(debug_error!( + ?key_id, + ?origin, + "Failed to fetch federation signing-key" + ))) +} + +#[implement(super::Service)] +async fn get_verify_key_from_notaries(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> Result { for notary in self.services.globals.trusted_servers() { if let Ok(server_keys) = self.notary_request(notary, origin).await { for server_key in &server_keys { @@ -78,9 +101,17 @@ pub async fn get_verify_key(&self, origin: &ServerName, key_id: &ServerSigningKe } } - Err!(BadServerResponse(debug_error!( - ?key_id, - ?origin, - "Failed to fetch federation signing-key" - ))) + Err!(Request(NotFound("Failed to fetch signing-key from notaries"))) +} + +#[implement(super::Service)] +async fn get_verify_key_from_origin(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> Result { + if let Ok(server_key) = self.server_request(origin).await { + self.add_signing_keys(server_key.clone()).await; + if let Some(result) = extract_key(server_key, key_id) { + return Ok(result); + } + } + + Err!(Request(NotFound("Failed to fetch signing-key from origin"))) } diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index c3b84cb3..dc09703c 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -7,7 +7,7 @@ mod verify; use std::{collections::BTreeMap, sync::Arc, time::Duration}; -use conduit::{implement, utils::time::timepoint_from_now, Result}; +use conduit::{implement, utils::time::timepoint_from_now, Result, Server}; use database::{Deserialized, Json, Map}; use ruma::{ api::federation::discovery::{ServerSigningKeys, VerifyKey}, @@ -30,6 +30,7 @@ pub struct Service { struct Services { globals: Dep, sending: Dep, + server: Arc, } struct Data { @@ -52,6 +53,7 @@ impl crate::Service for Service { services: Services { globals: args.depend::("globals"), sending: args.depend::("sending"), + server: args.server.clone(), }, db: Data { server_signingkeys: args.db["server_signingkeys"].clone(), From 0e55fa2de24a945e55469ec496f85e29e5f10d5b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 21 Oct 2024 23:54:54 +0000 Subject: [PATCH 0928/2091] add ready_try_for_each to TryReadyExt extension utils Signed-off-by: Jason Volk --- src/core/utils/stream/try_ready.rs | 18 +++++++++++++++++- src/service/sending/mod.rs | 14 +++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs index ab37d9b3..df356456 100644 --- a/src/core/utils/stream/try_ready.rs +++ b/src/core/utils/stream/try_ready.rs @@ -2,7 +2,7 @@ use futures::{ future::{ready, Ready}, - stream::{AndThen, TryStream, TryStreamExt}, + stream::{AndThen, TryForEach, TryStream, TryStreamExt}, }; use crate::Result; @@ -18,6 +18,12 @@ where fn ready_and_then(self, f: F) -> AndThen>, impl FnMut(S::Ok) -> Ready>> where F: Fn(S::Ok) -> Result; + + fn ready_try_for_each( + self, f: F, + ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> + where + F: Fn(S::Ok) -> Result<(), E>; } impl TryReadyExt for S @@ -32,4 +38,14 @@ where { self.and_then(move |t| ready(f(t))) } + + #[inline] + fn ready_try_for_each( + self, f: F, + ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> + where + F: Fn(S::Ok) -> Result<(), E>, + { + self.try_for_each(move |t| ready(f(t))) + } } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 63c5e655..a1d5f692 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -7,8 +7,12 @@ mod sender; use std::{fmt::Debug, sync::Arc}; use async_trait::async_trait; -use conduit::{err, utils::ReadyExt, warn, Result, Server}; -use futures::{future::ready, Stream, StreamExt, TryStreamExt}; +use conduit::{ + err, + utils::{ReadyExt, TryReadyExt}, + warn, Result, Server, +}; +use futures::{Stream, StreamExt}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, RoomId, ServerName, UserId, @@ -235,12 +239,12 @@ impl Service { .map(ToOwned::to_owned) .map(Destination::Normal) .map(Ok) - .try_for_each(|dest| { - ready(self.dispatch(Msg { + .ready_try_for_each(|dest| { + self.dispatch(Msg { dest, event: SendingEvent::Flush, queue_id: Vec::::new(), - })) + }) }) .await } From 167807e0a6e333a4a8f7be9b8ed0da46831ce234 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 00:09:55 +0000 Subject: [PATCH 0929/2091] de-wrapper max_fetch_prev_event; increase default config Signed-off-by: Jason Volk --- src/core/config/mod.rs | 2 +- src/service/globals/mod.rs | 2 -- src/service/rooms/event_handler/mod.rs | 6 ++++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 52ce8a01..23d35424 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1822,7 +1822,7 @@ fn default_appservice_idle_timeout() -> u64 { 300 } fn default_pusher_idle_timeout() -> u64 { 15 } -fn default_max_fetch_prev_events() -> u16 { 100_u16 } +fn default_max_fetch_prev_events() -> u16 { 192_u16 } fn default_tracing_flame_filter() -> String { cfg!(debug_assertions) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 7680007d..329a6583 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -171,8 +171,6 @@ impl Service { #[inline] pub fn server_name(&self) -> &ServerName { self.config.server_name.as_ref() } - pub fn max_fetch_prev_events(&self) -> u16 { self.config.max_fetch_prev_events } - pub fn allow_registration(&self) -> bool { self.config.allow_registration } pub fn allow_guest_registration(&self) -> bool { self.config.allow_guest_registration } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 41ab79f1..8f96f68e 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -13,7 +13,7 @@ use conduit::{ result::LogErr, trace, utils::{math::continue_exponential_backoff_secs, IterStream, MutexMap}, - warn, Err, Error, PduEvent, Result, + warn, Err, Error, PduEvent, Result, Server, }; use futures::{future, future::ready, FutureExt, StreamExt, TryFutureExt}; use ruma::{ @@ -55,6 +55,7 @@ struct Services { state_accessor: Dep, state_compressor: Dep, timeline: Dep, + server: Arc, } type RoomMutexMap = MutexMap; @@ -76,6 +77,7 @@ impl crate::Service for Service { state_accessor: args.depend::("rooms::state_accessor"), state_compressor: args.depend::("rooms::state_compressor"), timeline: args.depend::("rooms::timeline"), + server: args.server.clone(), }, federation_handletime: HandleTimeMap::new().into(), mutex_federation: RoomMutexMap::new(), @@ -1280,7 +1282,7 @@ impl Service { { check_room_id(room_id, &pdu)?; - let limit = self.services.globals.max_fetch_prev_events(); + let limit = self.services.server.config.max_fetch_prev_events; if amount > limit { debug_warn!("Max prev event limit reached! Limit: {limit}"); graph.insert(prev_event_id.clone(), HashSet::new()); From c06f560913ce637419f9a825ca8b6ffaca698bc8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 03:21:56 +0000 Subject: [PATCH 0930/2091] add some additional database::de test cases Signed-off-by: Jason Volk --- src/database/tests.rs | 62 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/src/database/tests.rs b/src/database/tests.rs index 47dfb32c..bfab99ef 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -10,7 +10,7 @@ use serde::Serialize; use crate::{ de, ser, ser::{serialize_to_vec, Json}, - Interfix, + Ignore, Interfix, }; #[test] @@ -187,6 +187,66 @@ fn de_tuple() { assert_eq!(b, room_id, "deserialized room_id does not match"); } +#[test] +#[should_panic(expected = "failed to deserialize")] +fn de_tuple_invalid() { + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + + let raw: &[u8] = b"@user:example.com\xFF@user:example.com"; + let (a, b): (&UserId, &RoomId) = de::from_slice(raw).expect("failed to deserialize"); + + assert_eq!(a, user_id, "deserialized user_id does not match"); + assert_eq!(b, room_id, "deserialized room_id does not match"); +} + +#[test] +#[should_panic(expected = "failed to deserialize")] +fn de_tuple_incomplete() { + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + + let raw: &[u8] = b"@user:example.com"; + let (a, _): (&UserId, &RoomId) = de::from_slice(raw).expect("failed to deserialize"); + + assert_eq!(a, user_id, "deserialized user_id does not match"); +} + +#[test] +#[should_panic(expected = "failed to deserialize")] +fn de_tuple_incomplete_with_sep() { + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + + let raw: &[u8] = b"@user:example.com\xFF"; + let (a, _): (&UserId, &RoomId) = de::from_slice(raw).expect("failed to deserialize"); + + assert_eq!(a, user_id, "deserialized user_id does not match"); +} + +#[test] +#[should_panic(expected = "deserialization failed to consume trailing bytes")] +fn de_tuple_unfinished() { + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + + let raw: &[u8] = b"@user:example.com\xFF!room:example.com\xFF@user:example.com"; + let (a, b): (&UserId, &RoomId) = de::from_slice(raw).expect("failed to deserialize"); + + assert_eq!(a, user_id, "deserialized user_id does not match"); + assert_eq!(b, room_id, "deserialized room_id does not match"); +} + +#[test] +fn de_tuple_ignore() { + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + + let raw: &[u8] = b"@user:example.com\xFF@user2:example.net\xFF!room:example.com"; + let (a, _, c): (&UserId, Ignore, &RoomId) = de::from_slice(raw).expect("failed to deserialize"); + + assert_eq!(a, user_id, "deserialized user_id does not match"); + assert_eq!(c, room_id, "deserialized room_id does not match"); +} + #[test] fn de_json_array() { let a = &["foo", "bar", "baz"]; From 0e0438e1f9b49a3fa1b8fc0dece769d91c2bafbf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 03:28:45 +0000 Subject: [PATCH 0931/2091] further optimize presence_since iteration Signed-off-by: Jason Volk --- src/admin/query/presence.rs | 10 +++++++--- src/api/client/sync/v3.rs | 8 ++++---- src/service/presence/data.rs | 11 ++++++----- src/service/presence/mod.rs | 3 +-- src/service/sending/sender.rs | 8 +++++--- 5 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index 6189270c..0963429e 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -42,12 +42,16 @@ pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) since, } => { let timer = tokio::time::Instant::now(); - let results = services.presence.db.presence_since(since); - let presence_since: Vec<(_, _, _)> = results.collect().await; + let results: Vec<(_, _, _)> = services + .presence + .presence_since(since) + .map(|(user_id, count, bytes)| (user_id.to_owned(), count, bytes.to_vec())) + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{presence_since:#?}\n```" + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, } diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index f29fe220..2bd318df 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -488,7 +488,7 @@ async fn process_presence_updates( if !services .rooms .state_cache - .user_sees_user(syncing_user, &user_id) + .user_sees_user(syncing_user, user_id) .await { continue; @@ -496,10 +496,10 @@ async fn process_presence_updates( let presence_event = services .presence - .from_json_bytes_to_event(&presence_bytes, &user_id) + .from_json_bytes_to_event(presence_bytes, user_id) .await?; - match presence_updates.entry(user_id) { + match presence_updates.entry(user_id.into()) { Entry::Vacant(slot) => { slot.insert(presence_event); }, @@ -524,7 +524,7 @@ async fn process_presence_updates( .currently_active .or(curr_content.currently_active); }, - } + }; } Ok(()) diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 8522746f..68b2c3fe 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -7,7 +7,7 @@ use conduit::{ }; use database::{Deserialized, Json, Map}; use futures::Stream; -use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; +use ruma::{events::presence::PresenceEvent, presence::PresenceState, UInt, UserId}; use super::Presence; use crate::{globals, users, Dep}; @@ -137,13 +137,14 @@ impl Data { self.userid_presenceid.remove(user_id); } - pub fn presence_since(&self, since: u64) -> impl Stream)> + Send + '_ { + #[inline] + pub(super) fn presence_since(&self, since: u64) -> impl Stream + Send + '_ { self.presenceid_presence .raw_stream() .ignore_err() - .ready_filter_map(move |(key, presence_bytes)| { - let (count, user_id) = presenceid_parse(key).expect("invalid presenceid_parse"); - (count > since).then(|| (user_id.to_owned(), count, presence_bytes.to_vec())) + .ready_filter_map(move |(key, presence)| { + let (count, user_id) = presenceid_parse(key).ok()?; + (count > since).then_some((user_id, count, presence)) }) } } diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 82a99bd5..b2106f3f 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -162,8 +162,7 @@ impl Service { /// Returns the most recent presence updates that happened after the event /// with id `since`. - #[inline] - pub fn presence_since(&self, since: u64) -> impl Stream)> + Send + '_ { + pub fn presence_since(&self, since: u64) -> impl Stream + Send + '_ { self.db.presence_since(since) } diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 5c0a324b..a57d4aea 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -7,7 +7,9 @@ use std::{ use base64::{engine::general_purpose, Engine as _}; use conduit::{ - debug, debug_warn, err, trace, + debug, debug_warn, err, + result::LogErr, + trace, utils::{calculate_hash, math::continue_exponential_backoff_secs, ReadyExt}, warn, Error, Result, }; @@ -315,14 +317,14 @@ impl Service { while let Some((user_id, count, presence_bytes)) = presence_since.next().await { *max_edu_count = cmp::max(count, *max_edu_count); - if !self.services.globals.user_is_local(&user_id) { + if !self.services.globals.user_is_local(user_id) { continue; } if !self .services .state_cache - .server_sees_user(server_name, &user_id) + .server_sees_user(server_name, user_id) .await { continue; From a74461fc9a9bd8f5a237662b399431cacc3f29e6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 04:03:07 +0000 Subject: [PATCH 0932/2091] split keys_changed for stronger-type overloads Signed-off-by: Jason Volk --- src/api/client/keys.rs | 5 +++-- src/api/client/sync/v3.rs | 5 +++-- src/api/client/sync/v4.rs | 5 +++-- src/api/mod.rs | 1 - src/service/users/mod.rs | 21 ++++++++++++++++++--- 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 254d92cc..44d9164c 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -232,7 +232,7 @@ pub(crate) async fn get_key_changes_route( device_list_updates.extend( services .users - .keys_changed(sender_user.as_str(), from, Some(to)) + .keys_changed(sender_user, from, Some(to)) .map(ToOwned::to_owned) .collect::>() .await, @@ -244,7 +244,8 @@ pub(crate) async fn get_key_changes_route( device_list_updates.extend( services .users - .keys_changed(room_id.as_str(), from, Some(to)) + .room_keys_changed(room_id, from, Some(to)) + .map(|(user_id, _)| user_id) .map(ToOwned::to_owned) .collect::>() .await, diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 2bd318df..ccca1f85 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -138,7 +138,7 @@ pub(crate) async fn sync_events_route( device_list_updates.extend( services .users - .keys_changed(sender_user.as_ref(), since, None) + .keys_changed(&sender_user, since, None) .map(ToOwned::to_owned) .collect::>() .await, @@ -917,7 +917,8 @@ async fn load_joined_room( device_list_updates.extend( services .users - .keys_changed(room_id.as_ref(), since, None) + .room_keys_changed(room_id, since, None) + .map(|(user_id, _)| user_id) .map(ToOwned::to_owned) .collect::>() .await, diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 2adb3b71..4f8323e6 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -162,7 +162,7 @@ pub(crate) async fn sync_events_v4_route( device_list_changes.extend( services .users - .keys_changed(sender_user.as_ref(), globalsince, None) + .keys_changed(sender_user, globalsince, None) .map(ToOwned::to_owned) .collect::>() .await, @@ -285,7 +285,8 @@ pub(crate) async fn sync_events_v4_route( device_list_changes.extend( services .users - .keys_changed(room_id.as_ref(), globalsince, None) + .room_keys_changed(room_id, globalsince, None) + .map(|(user_id, _)| user_id) .map(ToOwned::to_owned) .collect::>() .await, diff --git a/src/api/mod.rs b/src/api/mod.rs index 96837470..ed8aacf2 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -6,7 +6,6 @@ extern crate conduit_core as conduit; extern crate conduit_service as service; pub(crate) use conduit::{debug_info, pdu::PduEvent, utils, Error, Result}; -pub(crate) use service::services; pub(crate) use self::router::{Ruma, RumaResponse, State}; diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 589aee8a..b9183e12 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -13,7 +13,7 @@ use ruma::{ events::{ignored_user_list::IgnoredUserListEvent, AnyToDeviceEvent, GlobalAccountDataEventType}, serde::Raw, DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedDeviceKeyId, - OwnedMxcUri, OwnedUserId, UInt, UserId, + OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, }; use serde_json::json; @@ -585,9 +585,24 @@ impl Service { Ok(()) } + #[inline] pub fn keys_changed<'a>( - &'a self, user_or_room_id: &'a str, from: u64, to: Option, + &'a self, user_id: &'a UserId, from: u64, to: Option, ) -> impl Stream + Send + 'a { + self.keys_changed_user_or_room(user_id.as_str(), from, to) + .map(|(user_id, ..)| user_id) + } + + #[inline] + pub fn room_keys_changed<'a>( + &'a self, room_id: &'a RoomId, from: u64, to: Option, + ) -> impl Stream + Send + 'a { + self.keys_changed_user_or_room(room_id.as_str(), from, to) + } + + fn keys_changed_user_or_room<'a>( + &'a self, user_or_room_id: &'a str, from: u64, to: Option, + ) -> impl Stream + Send + 'a { type KeyVal<'a> = ((&'a str, u64), &'a UserId); let to = to.unwrap_or(u64::MAX); @@ -597,7 +612,7 @@ impl Service { .stream_from(&start) .ignore_err() .ready_take_while(move |((prefix, count), _): &KeyVal<'_>| *prefix == user_or_room_id && *count <= to) - .map(|((..), user_id): KeyVal<'_>| user_id) + .map(|((_, count), user_id): KeyVal<'_>| (user_id, count)) } pub async fn mark_device_key_update(&self, user_id: &UserId) { From d35376a90cb521b578aff75a1441699e10695bac Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 05:30:28 +0000 Subject: [PATCH 0933/2091] aggregate receipts into single edu; dedup presence; refactor selection limits etc Signed-off-by: Jason Volk --- src/api/server/send.rs | 8 +- src/service/sending/mod.rs | 5 +- src/service/sending/sender.rs | 307 ++++++++++++++++++++-------------- 3 files changed, 190 insertions(+), 130 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 40f9403b..e2100a0f 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -21,16 +21,16 @@ use ruma::{ OwnedEventId, ServerName, }; use serde_json::value::RawValue as RawJsonValue; +use service::{ + sending::{EDU_LIMIT, PDU_LIMIT}, + Services, +}; use crate::{ - services::Services, utils::{self}, Ruma, }; -const PDU_LIMIT: usize = 50; -const EDU_LIMIT: usize = 100; - type ResolvedMap = BTreeMap>; /// # `PUT /_matrix/federation/v1/send/{txnId}` diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index a1d5f692..ea266883 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -20,7 +20,10 @@ use ruma::{ use tokio::sync::Mutex; use self::data::Data; -pub use self::dest::Destination; +pub use self::{ + dest::Destination, + sender::{EDU_LIMIT, PDU_LIMIT}, +}; use crate::{account_data, client, globals, presence, pusher, resolver, rooms, server_keys, users, Dep}; pub struct Service { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index a57d4aea..d9087d44 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -7,7 +7,7 @@ use std::{ use base64::{engine::general_purpose, Engine as _}; use conduit::{ - debug, debug_warn, err, + debug, debug_warn, err, error, result::LogErr, trace, utils::{calculate_hash, math::continue_exponential_backoff_secs, ReadyExt}, @@ -26,8 +26,8 @@ use ruma::{ }, device_id, events::{push_rules::PushRulesEvent, receipt::ReceiptType, AnySyncEphemeralRoomEvent, GlobalAccountDataEventType}, - push, uint, CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, - ServerName, UInt, + push, uint, CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, + RoomVersionId, ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use tokio::time::sleep_until; @@ -47,10 +47,16 @@ type SendingFuture<'a> = BoxFuture<'a, SendingResult>; type SendingFutures<'a> = FuturesUnordered>; type CurTransactionStatus = HashMap; -const DEQUEUE_LIMIT: usize = 48; -const SELECT_EDU_LIMIT: usize = 16; const CLEANUP_TIMEOUT_MS: u64 = 3500; +const SELECT_PRESENCE_LIMIT: usize = 256; +const SELECT_RECEIPT_LIMIT: usize = 256; +const SELECT_EDU_LIMIT: usize = EDU_LIMIT - 2; +const DEQUEUE_LIMIT: usize = 48; + +pub const PDU_LIMIT: usize = 50; +pub const EDU_LIMIT: usize = 100; + impl Service { #[tracing::instrument(skip_all, name = "sender")] pub(super) async fn sender(&self) -> Result<()> { @@ -216,6 +222,7 @@ impl Service { // Add EDU's into the transaction if let Destination::Normal(server_name) = dest { if let Ok((select_edus, last_count)) = self.select_edus(server_name).await { + debug_assert!(select_edus.len() <= EDU_LIMIT, "exceeded edus limit"); events.extend(select_edus.into_iter().map(SendingEvent::Edu)); self.db.set_latest_educount(server_name, last_count); } @@ -254,69 +261,176 @@ impl Service { async fn select_edus(&self, server_name: &ServerName) -> Result<(Vec>, u64)> { // u64: count of last edu let since = self.db.get_latest_educount(server_name).await; - let mut events = Vec::new(); let mut max_edu_count = since; - let mut device_list_changes = HashSet::new(); + let mut events = Vec::new(); - let server_rooms = self.services.state_cache.server_rooms(server_name); + self.select_edus_device_changes(server_name, since, &mut max_edu_count, &mut events) + .await; - pin_mut!(server_rooms); - while let Some(room_id) = server_rooms.next().await { - // Look for device list updates in this room - device_list_changes.extend( - self.services - .users - .keys_changed(room_id.as_str(), since, None) - .ready_filter(|user_id| self.services.globals.user_is_local(user_id)) - .map(ToOwned::to_owned) - .collect::>() - .await, - ); - - if self.server.config.allow_outgoing_read_receipts - && !self - .select_edus_receipts(room_id, since, &mut max_edu_count, &mut events) - .await? - { - break; - } - } - - for user_id in device_list_changes { - // Empty prev id forces synapse to resync; because synapse resyncs, - // we can just insert placeholder data - let edu = Edu::DeviceListUpdate(DeviceListUpdateContent { - user_id, - device_id: device_id!("placeholder").to_owned(), - device_display_name: Some("Placeholder".to_owned()), - stream_id: uint!(1), - prev_id: Vec::new(), - deleted: None, - keys: None, - }); - - events.push(serde_json::to_vec(&edu).expect("json can be serialized")); + if self.server.config.allow_outgoing_read_receipts { + self.select_edus_receipts(server_name, since, &mut max_edu_count, &mut events) + .await; } if self.server.config.allow_outgoing_presence { self.select_edus_presence(server_name, since, &mut max_edu_count, &mut events) - .await?; + .await; } Ok((events, max_edu_count)) } + /// Look for presence + async fn select_edus_device_changes( + &self, server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, + ) { + debug_assert!(events.len() < SELECT_EDU_LIMIT, "called when edu limit reached"); + + let server_rooms = self.services.state_cache.server_rooms(server_name); + + pin_mut!(server_rooms); + let mut device_list_changes = HashSet::::new(); + while let Some(room_id) = server_rooms.next().await { + let keys_changed = self + .services + .users + .room_keys_changed(room_id, since, None) + .ready_filter(|(user_id, _)| self.services.globals.user_is_local(user_id)); + + pin_mut!(keys_changed); + while let Some((user_id, count)) = keys_changed.next().await { + *max_edu_count = cmp::max(count, *max_edu_count); + if !device_list_changes.insert(user_id.into()) { + continue; + } + + // Empty prev id forces synapse to resync; because synapse resyncs, + // we can just insert placeholder data + let edu = Edu::DeviceListUpdate(DeviceListUpdateContent { + user_id: user_id.into(), + device_id: device_id!("placeholder").to_owned(), + device_display_name: Some("Placeholder".to_owned()), + stream_id: uint!(1), + prev_id: Vec::new(), + deleted: None, + keys: None, + }); + + let edu = serde_json::to_vec(&edu).expect("failed to serialize device list update to JSON"); + + events.push(edu); + if events.len() >= SELECT_EDU_LIMIT { + return; + } + } + } + } + + /// Look for read receipts in this room + async fn select_edus_receipts( + &self, server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, + ) { + debug_assert!(events.len() < EDU_LIMIT, "called when edu limit reached"); + + let server_rooms = self.services.state_cache.server_rooms(server_name); + + pin_mut!(server_rooms); + let mut num = 0; + let mut receipts = BTreeMap::::new(); + while let Some(room_id) = server_rooms.next().await { + let receipt_map = self + .select_edus_receipts_room(room_id, since, max_edu_count, &mut num) + .await; + + if !receipt_map.read.is_empty() { + receipts.insert(room_id.into(), receipt_map); + } + } + + if receipts.is_empty() { + return; + } + + let receipt_content = Edu::Receipt(ReceiptContent { + receipts, + }); + + let receipt_content = + serde_json::to_vec(&receipt_content).expect("Failed to serialize Receipt EDU to JSON vec"); + + events.push(receipt_content); + } + + /// Look for read receipts in this room + async fn select_edus_receipts_room( + &self, room_id: &RoomId, since: u64, max_edu_count: &mut u64, num: &mut usize, + ) -> ReceiptMap { + let receipts = self + .services + .read_receipt + .readreceipts_since(room_id, since); + + pin_mut!(receipts); + let mut read = BTreeMap::::new(); + while let Some((user_id, count, read_receipt)) = receipts.next().await { + *max_edu_count = cmp::max(count, *max_edu_count); + if !self.services.globals.user_is_local(&user_id) { + continue; + } + + let Ok(event) = serde_json::from_str(read_receipt.json().get()) else { + error!(?user_id, ?count, ?read_receipt, "Invalid edu event in read_receipts."); + continue; + }; + + let AnySyncEphemeralRoomEvent::Receipt(r) = event else { + error!(?user_id, ?count, ?event, "Invalid event type in read_receipts"); + continue; + }; + + let (event_id, mut receipt) = r + .content + .0 + .into_iter() + .next() + .expect("we only use one event per read receipt"); + + let receipt = receipt + .remove(&ReceiptType::Read) + .expect("our read receipts always set this") + .remove(&user_id) + .expect("our read receipts always have the user here"); + + let receipt_data = ReceiptData { + data: receipt, + event_ids: vec![event_id.clone()], + }; + + if read.insert(user_id, receipt_data).is_none() { + *num = num.saturating_add(1); + if *num >= SELECT_RECEIPT_LIMIT { + break; + } + } + } + + ReceiptMap { + read, + } + } + /// Look for presence async fn select_edus_presence( &self, server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, - ) -> Result { + ) { + debug_assert!(events.len() < EDU_LIMIT, "called when edu limit reached"); + let presence_since = self.services.presence.presence_since(since); pin_mut!(presence_since); - let mut presence_updates = Vec::new(); + let mut presence_updates = HashMap::::new(); while let Some((user_id, count, presence_bytes)) = presence_since.next().await { *max_edu_count = cmp::max(count, *max_edu_count); - if !self.services.globals.user_is_local(user_id) { continue; } @@ -330,101 +444,44 @@ impl Service { continue; } - let presence_event = self + let Ok(presence_event) = self .services .presence - .from_json_bytes_to_event(&presence_bytes, &user_id) - .await?; + .from_json_bytes_to_event(presence_bytes, user_id) + .await + .log_err() + else { + continue; + }; - presence_updates.push(PresenceUpdate { - user_id, + let update = PresenceUpdate { + user_id: user_id.into(), presence: presence_event.content.presence, currently_active: presence_event.content.currently_active.unwrap_or(false), + status_msg: presence_event.content.status_msg, last_active_ago: presence_event .content .last_active_ago .unwrap_or_else(|| uint!(0)), - status_msg: presence_event.content.status_msg, - }); + }; - if presence_updates.len() >= SELECT_EDU_LIMIT { + presence_updates.insert(user_id.into(), update); + if presence_updates.len() >= SELECT_PRESENCE_LIMIT { break; } } - if !presence_updates.is_empty() { - let presence_content = Edu::Presence(PresenceContent::new(presence_updates)); - events.push(serde_json::to_vec(&presence_content).expect("PresenceEvent can be serialized")); + if presence_updates.is_empty() { + return; } - Ok(true) - } + let presence_content = Edu::Presence(PresenceContent { + push: presence_updates.into_values().collect(), + }); - /// Look for read receipts in this room - async fn select_edus_receipts( - &self, room_id: &RoomId, since: u64, max_edu_count: &mut u64, events: &mut Vec>, - ) -> Result { - let receipts = self - .services - .read_receipt - .readreceipts_since(room_id, since); + let presence_content = serde_json::to_vec(&presence_content).expect("failed to serialize Presence EDU to JSON"); - pin_mut!(receipts); - while let Some((user_id, count, read_receipt)) = receipts.next().await { - *max_edu_count = cmp::max(count, *max_edu_count); - if !self.services.globals.user_is_local(&user_id) { - continue; - } - - let event = serde_json::from_str(read_receipt.json().get()) - .map_err(|_| Error::bad_database("Invalid edu event in read_receipts."))?; - - let federation_event = if let AnySyncEphemeralRoomEvent::Receipt(r) = event { - let mut read = BTreeMap::new(); - let (event_id, mut receipt) = r - .content - .0 - .into_iter() - .next() - .expect("we only use one event per read receipt"); - - let receipt = receipt - .remove(&ReceiptType::Read) - .expect("our read receipts always set this") - .remove(&user_id) - .expect("our read receipts always have the user here"); - - read.insert( - user_id, - ReceiptData { - data: receipt.clone(), - event_ids: vec![event_id.clone()], - }, - ); - - let receipt_map = ReceiptMap { - read, - }; - - let mut receipts = BTreeMap::new(); - receipts.insert(room_id.to_owned(), receipt_map); - - Edu::Receipt(ReceiptContent { - receipts, - }) - } else { - Error::bad_database("Invalid event type in read_receipts"); - continue; - }; - - events.push(serde_json::to_vec(&federation_event).expect("json can be serialized")); - - if events.len() >= SELECT_EDU_LIMIT { - return Ok(false); - } - } - - Ok(true) + events.push(presence_content); } async fn send_events(&self, dest: Destination, events: Vec) -> SendingResult { From ca57dc79288e563e5e090f6699b115bc49b9d27f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 06:21:57 +0000 Subject: [PATCH 0934/2091] optimize config denylists Signed-off-by: Jason Volk --- src/api/client/directory.rs | 20 ++++++++------------ src/core/config/mod.rs | 14 +++++++------- src/service/globals/mod.rs | 4 ---- src/service/media/remote.rs | 3 +-- src/service/sending/send.rs | 9 ++++----- 5 files changed, 20 insertions(+), 30 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index ea499545..6cf7b13f 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -37,14 +37,12 @@ pub(crate) async fn get_public_rooms_filtered_route( ) -> Result { if let Some(server) = &body.server { if services - .globals - .forbidden_remote_room_directory_server_names() + .server + .config + .forbidden_remote_room_directory_server_names .contains(server) { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + return Err!(Request(Forbidden("Server is banned on this homeserver."))); } } @@ -77,14 +75,12 @@ pub(crate) async fn get_public_rooms_route( ) -> Result { if let Some(server) = &body.server { if services - .globals - .forbidden_remote_room_directory_server_names() + .server + .config + .forbidden_remote_room_directory_server_names .contains(server) { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + return Err!(Request(Forbidden("Server is banned on this homeserver."))); } } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 23d35424..59ddd7c7 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -2,7 +2,7 @@ pub mod check; pub mod proxy; use std::{ - collections::{BTreeMap, BTreeSet}, + collections::{BTreeMap, BTreeSet, HashSet}, fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, path::PathBuf, @@ -983,8 +983,8 @@ pub struct Config { /// Vector list of servers that conduwuit will refuse to download remote /// media from. No default. - #[serde(default = "Vec::new")] - pub prevent_media_downloads_from: Vec, + #[serde(default = "HashSet::new")] + pub prevent_media_downloads_from: HashSet, /// List of forbidden server names that we will block incoming AND outgoing /// federation with, and block client room joins / remote user invites. @@ -994,14 +994,14 @@ pub struct Config { /// outbound federation handler. /// /// Basically "global" ACLs. No default. - #[serde(default = "Vec::new")] - pub forbidden_remote_server_names: Vec, + #[serde(default = "HashSet::new")] + pub forbidden_remote_server_names: HashSet, /// List of forbidden server names that we will block all outgoing federated /// room directory requests for. Useful for preventing our users from /// wandering into bad servers or spaces. No default. - #[serde(default = "Vec::new")] - pub forbidden_remote_room_directory_server_names: Vec, + #[serde(default = "HashSet::new")] + pub forbidden_remote_room_directory_server_names: HashSet, /// Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you /// do not want conduwuit to send outbound requests to. Defaults to diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 329a6583..157c3944 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -252,10 +252,6 @@ impl Service { pub fn allow_outgoing_read_receipts(&self) -> bool { self.config.allow_outgoing_read_receipts } - pub fn forbidden_remote_room_directory_server_names(&self) -> &[OwnedServerName] { - &self.config.forbidden_remote_room_directory_server_names - } - pub fn well_known_support_page(&self) -> &Option { &self.config.well_known.support_page } pub fn well_known_support_role(&self) -> &Option { &self.config.well_known.support_role } diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index 59846b8e..1c6c9ca0 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -382,8 +382,7 @@ fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> { .server .config .prevent_media_downloads_from - .iter() - .any(|entry| entry == mxc.server_name) + .contains(mxc.server_name) { // we'll lie to the client and say the blocked server's media was not found and // log. the client has no way of telling anyways so this is a security bonus. diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 73b6a468..62da59ef 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,8 +1,8 @@ use std::{fmt::Debug, mem}; use conduit::{ - debug, debug_error, debug_info, debug_warn, err, error::inspect_debug_log, implement, trace, utils::string::EMPTY, - Err, Error, Result, + debug, debug_error, debug_warn, err, error::inspect_debug_log, implement, trace, utils::string::EMPTY, Err, Error, + Result, }; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; @@ -36,10 +36,9 @@ impl super::Service { .server .config .forbidden_remote_server_names - .contains(&dest.to_owned()) + .contains(dest) { - debug_info!("Refusing to send outbound federation request to {dest}"); - return Err!(Request(Forbidden("Federation with this homeserver is not allowed."))); + return Err!(Request(Forbidden(debug_warn!("Federation with this {dest} is not allowed.")))); } let actual = self.services.resolver.get_actual_dest(dest).await?; From b8260e0104860eee3b8dfcb6f9091e3ad87ae2de Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 06:37:09 +0000 Subject: [PATCH 0935/2091] optimize for pdu_exists; remove a yield thing Signed-off-by: Jason Volk --- src/service/rooms/event_handler/mod.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 8f96f68e..24c2692d 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -359,7 +359,7 @@ impl Service { }; // Skip the PDU if it is redacted and we already have it as an outlier event - if self.services.timeline.get_pdu_json(event_id).await.is_ok() { + if self.services.timeline.pdu_exists(event_id).await { return Err!(Request(InvalidParam("Event was redacted and we already knew about it"))); } @@ -1123,7 +1123,6 @@ impl Service { let mut todo_auth_events = vec![Arc::clone(id)]; let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); let mut events_all = HashSet::with_capacity(todo_auth_events.len()); - let mut i: u64 = 0; while let Some(next_id) = todo_auth_events.pop() { if let Some((time, tries)) = self .services @@ -1146,12 +1145,7 @@ impl Service { continue; } - i = i.saturating_add(1); - if i % 100 == 0 { - tokio::task::yield_now().await; - } - - if self.services.timeline.get_pdu(&next_id).await.is_ok() { + if self.services.timeline.pdu_exists(&next_id).await { trace!("Found {next_id} in db"); continue; } From dd6621a720b03ca18a7e0fca6881923c373e3cac Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 07:07:42 +0000 Subject: [PATCH 0936/2091] reduce unnecessary clone in pdu handler Signed-off-by: Jason Volk --- src/api/server/send.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index e2100a0f..4f526052 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -118,15 +118,12 @@ async fn handle_pdus( .lock(&room_id) .await; - resolved_map.insert( - event_id.clone(), - services - .rooms - .event_handler - .handle_incoming_pdu(origin, &room_id, &event_id, value, true) - .await - .map(|_| ()), - ); + let result = services + .rooms + .event_handler + .handle_incoming_pdu(origin, &room_id, &event_id, value, true) + .await + .map(|_| ()); drop(mutex_lock); debug!( @@ -134,12 +131,14 @@ async fn handle_pdus( txn_elapsed = ?txn_start_time.elapsed(), "Finished PDU {event_id}", ); + + resolved_map.insert(event_id, result); } - for pdu in &resolved_map { - if let Err(e) = pdu.1 { + for (id, result) in &resolved_map { + if let Err(e) = result { if matches!(e, Error::BadRequest(ErrorKind::NotFound, _)) { - warn!("Incoming PDU failed {pdu:?}"); + warn!("Incoming PDU failed {id}: {e:?}"); } } } From b08c1241a89514046f666fc21f817af2feb8bce2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 07:15:28 +0000 Subject: [PATCH 0937/2091] add some interruption points in recursive event handling to prevent shutdown hangs Signed-off-by: Jason Volk --- src/api/server/send.rs | 7 ++++--- src/core/server.rs | 9 ++++++++- src/service/rooms/event_handler/mod.rs | 3 +++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 4f526052..d5d3ffbb 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -71,7 +71,7 @@ pub(crate) async fn send_transaction_message_route( "Starting txn", ); - let resolved_map = handle_pdus(&services, &client, &body.pdus, origin, &txn_start_time).await; + let resolved_map = handle_pdus(&services, &client, &body.pdus, origin, &txn_start_time).await?; handle_edus(&services, &client, &body.edus, origin).await; debug!( @@ -93,7 +93,7 @@ pub(crate) async fn send_transaction_message_route( async fn handle_pdus( services: &Services, _client: &IpAddr, pdus: &[Box], origin: &ServerName, txn_start_time: &Instant, -) -> ResolvedMap { +) -> Result { let mut parsed_pdus = Vec::with_capacity(pdus.len()); for pdu in pdus { parsed_pdus.push(match services.rooms.event_handler.parse_incoming_pdu(pdu).await { @@ -110,6 +110,7 @@ async fn handle_pdus( let mut resolved_map = BTreeMap::new(); for (event_id, value, room_id) in parsed_pdus { + services.server.check_running()?; let pdu_start_time = Instant::now(); let mutex_lock = services .rooms @@ -143,7 +144,7 @@ async fn handle_pdus( } } - resolved_map + Ok(resolved_map) } async fn handle_edus(services: &Services, client: &IpAddr, edus: &[Raw], origin: &ServerName) { diff --git a/src/core/server.rs b/src/core/server.rs index 89f1dea5..627e125d 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -5,7 +5,7 @@ use std::{ use tokio::{runtime, sync::broadcast}; -use crate::{config::Config, log::Log, metrics::Metrics, Err, Result}; +use crate::{config::Config, err, log::Log, metrics::Metrics, Err, Result}; /// Server runtime state; public portion pub struct Server { @@ -107,6 +107,13 @@ impl Server { .expect("runtime handle available in Server") } + #[inline] + pub fn check_running(&self) -> Result { + self.running() + .then_some(()) + .ok_or_else(|| err!(debug_warn!("Server is shutting down."))) + } + #[inline] pub fn running(&self) -> bool { !self.stopping.load(Ordering::Acquire) } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 24c2692d..0b2bbf73 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -205,6 +205,7 @@ impl Service { debug!(events = ?sorted_prev_events, "Got previous events"); for prev_id in sorted_prev_events { + self.services.server.check_running()?; match self .handle_prev_pdu( origin, @@ -1268,6 +1269,8 @@ impl Service { let mut amount = 0; while let Some(prev_event_id) = todo_outlier_stack.pop() { + self.services.server.check_running()?; + if let Some((pdu, mut json_opt)) = self .fetch_and_handle_outliers(origin, &[prev_event_id.clone()], create_event, room_id, room_version_id) .boxed() From 339654216857dc5caa492f0b9a0aa442af56c0f9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 09:09:20 +0000 Subject: [PATCH 0938/2091] complete the example-config generator macro Signed-off-by: Jason Volk --- src/macros/config.rs | 157 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 149 insertions(+), 8 deletions(-) diff --git a/src/macros/config.rs b/src/macros/config.rs index 6d29c21f..3c93bd08 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -1,11 +1,19 @@ -use std::fmt::Write; +use std::{fmt::Write as _, fs::File, io::Write as _}; use proc_macro::TokenStream; +use proc_macro2::Span; use quote::ToTokens; -use syn::{Expr, ExprLit, Field, Fields, FieldsNamed, ItemStruct, Lit, Meta, MetaNameValue, Type, TypePath}; +use syn::{ + parse::Parser, punctuated::Punctuated, Error, Expr, ExprLit, Field, Fields, FieldsNamed, ItemStruct, Lit, Meta, + MetaList, MetaNameValue, Type, TypePath, +}; use crate::{utils::is_cargo_build, Result}; +const UNDOCUMENTED: &str = "# This item is undocumented. Please contribute documentation for it."; +const HEADER: &str = "## Conduwuit Configuration\n##\n## THIS FILE IS GENERATED. Changes to documentation and \ + defaults must\n## be made within the code found at src/core/config/\n"; + #[allow(clippy::needless_pass_by_value)] pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result { if is_cargo_build() { @@ -18,6 +26,12 @@ pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result Result<()> { + let mut file = File::create("conduwuit-example.toml") + .map_err(|e| Error::new(Span::call_site(), format!("Failed to open config file for generation: {e}")))?; + + file.write_all(HEADER.as_bytes()) + .expect("written to config file"); + if let Fields::Named(FieldsNamed { named, .. @@ -28,21 +42,143 @@ fn generate_example(input: &ItemStruct, _args: &[Meta]) -> Result<()> { continue; }; - let Some(doc) = get_doc_comment(field) else { - continue; - }; - let Some(type_name) = get_type_name(field) else { continue; }; - //println!("{:?} {type_name:?}\n{doc}", ident.to_string()); + let doc = get_doc_comment(field) + .unwrap_or_else(|| UNDOCUMENTED.into()) + .trim_end() + .to_owned(); + + let doc = if doc.ends_with('#') { + format!("{doc}\n") + } else { + format!("{doc}\n#\n") + }; + + let default = get_doc_default(field) + .or_else(|| get_default(field)) + .unwrap_or_default(); + + let default = if !default.is_empty() { + format!(" {default}") + } else { + default + }; + + file.write_fmt(format_args!("\n{doc}")) + .expect("written to config file"); + + file.write_fmt(format_args!("#{ident} ={default}\n")) + .expect("written to config file"); } } Ok(()) } +fn get_default(field: &Field) -> Option { + for attr in &field.attrs { + let Meta::List(MetaList { + path, + tokens, + .. + }) = &attr.meta + else { + continue; + }; + + if !path + .segments + .iter() + .next() + .is_some_and(|s| s.ident == "serde") + { + continue; + } + + let Some(arg) = Punctuated::::parse_terminated + .parse(tokens.clone().into()) + .ok()? + .iter() + .next() + .cloned() + else { + continue; + }; + + match arg { + Meta::NameValue(MetaNameValue { + value: Expr::Lit(ExprLit { + lit: Lit::Str(str), + .. + }), + .. + }) => { + match str.value().as_str() { + "HashSet::new" | "Vec::new" | "RegexSet::empty" => Some("[]".to_owned()), + "true_fn" => return Some("true".to_owned()), + _ => return None, + }; + }, + Meta::Path { + .. + } => return Some("false".to_owned()), + _ => return None, + }; + } + + None +} + +fn get_doc_default(field: &Field) -> Option { + for attr in &field.attrs { + let Meta::NameValue(MetaNameValue { + path, + value, + .. + }) = &attr.meta + else { + continue; + }; + + if !path + .segments + .iter() + .next() + .is_some_and(|s| s.ident == "doc") + { + continue; + } + + let Expr::Lit(ExprLit { + lit, + .. + }) = &value + else { + continue; + }; + + let Lit::Str(token) = &lit else { + continue; + }; + + let value = token.value(); + if !value.trim().starts_with("default:") { + continue; + } + + return value + .split_once(':') + .map(|(_, v)| v) + .map(str::trim) + .map(ToOwned::to_owned); + } + + None +} + fn get_doc_comment(field: &Field) -> Option { let mut out = String::new(); for attr in &field.attrs { @@ -76,7 +212,12 @@ fn get_doc_comment(field: &Field) -> Option { continue; }; - writeln!(&mut out, "# {}", token.value()).expect("wrote to output string buffer"); + let value = token.value(); + if value.trim().starts_with("default:") { + continue; + } + + writeln!(&mut out, "#{value}").expect("wrote to output string buffer"); } (!out.is_empty()).then_some(out) From 367d1533801d8d8c0b53aa07992cbac5f267db5c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 10:09:14 +0000 Subject: [PATCH 0939/2091] add default-directives to config document comments Signed-off-by: Jason Volk --- src/core/config/mod.rs | 249 ++++++++++++++++++++++++----------------- 1 file changed, 144 insertions(+), 105 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 59ddd7c7..919bb486 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -54,9 +54,9 @@ pub struct Config { /// want this to be localhost (127.0.0.1 / ::1). If you are using Docker or /// a container NAT networking setup, you likely need this to be 0.0.0.0. /// To listen multiple addresses, specify a vector e.g. ["127.0.0.1", "::1"] + /// Default if unspecified is both IPv4 and IPv6 localhost. /// - /// default if unspecified is both IPv4 and IPv6 localhost: ["127.0.0.1", - /// "::1"] + /// default: ["127.0.0.1", "::1"] #[serde(default = "default_address")] address: ListeningAddr, @@ -67,7 +67,7 @@ pub struct Config { /// port to this. To listen on multiple ports, specify a vector e.g. [8080, /// 8448] /// - /// default if unspecified is 8008 + /// default: 8008 #[serde(default = "default_port")] port: ListeningPort, @@ -80,9 +80,11 @@ pub struct Config { /// (666 minimum). pub unix_socket_path: Option, + /// default: 660 #[serde(default = "default_unix_socket_perms")] pub unix_socket_perms: u32, + /// default: rocksdb #[serde(default = "default_database_backend")] pub database_backend: String, @@ -98,7 +100,9 @@ pub struct Config { /// Set this to any float value in megabytes for conduwuit to tell the /// database engine that this much memory is available for database-related /// caches. May be useful if you have significant memory to spare to - /// increase performance. Defaults to 256.0 + /// increase performance. + /// + /// default: 256.0 #[serde(default = "default_db_cache_capacity_mb")] pub db_cache_capacity_mb: f64, @@ -107,6 +111,8 @@ pub struct Config { /// lightning bolt emoji option, just replaced with support for adding your /// own custom text or emojis. To disable, set this to "" (an empty string) /// Defaults to "🏳️⚧️" (trans pride flag) + /// + /// default: 🏳️⚧️ #[serde(default = "default_new_user_displayname_suffix")] pub new_user_displayname_suffix: String, @@ -123,11 +129,10 @@ pub struct Config { /// Set this to any float value to multiply conduwuit's in-memory LRU caches /// with. May be useful if you have significant memory to spare to increase - /// performance. + /// performance. This was previously called + /// `conduit_cache_capacity_modifier`. /// - /// This was previously called `conduit_cache_capacity_modifier` - /// - /// Defaults to 1.0. + /// default: 1.0. #[serde(default = "default_cache_capacity_modifier", alias = "conduit_cache_capacity_modifier")] pub cache_capacity_modifier: f64, @@ -197,11 +202,9 @@ pub struct Config { pub dns_tcp_fallback: bool, /// Enable to query all nameservers until the domain is found. Referred to - /// as "trust_negative_responses" in hickory_reso> This can avoid useless - /// DNS queries if the first nameserver responds with NXDOMAIN or an empty - /// NOERROR response. - /// - /// The default is to query one nameserver and stop (false). + /// as "trust_negative_responses" in hickory_resolver. This can avoid + /// useless DNS queries if the first nameserver responds with NXDOMAIN or + /// an empty NOERROR response. #[serde(default = "true_fn")] pub query_all_nameservers: bool, @@ -230,116 +233,121 @@ pub struct Config { /// /// Defaults to 5 - Ipv4ThenIpv6 as this is the most compatible and IPv4 /// networking is currently the most prevalent. + /// + /// default: 5 #[serde(default = "default_ip_lookup_strategy")] pub ip_lookup_strategy: u8, /// Max request size for file uploads + /// + /// default: 20971520 #[serde(default = "default_max_request_size")] pub max_request_size: usize, #[serde(default = "default_max_fetch_prev_events")] pub max_fetch_prev_events: u16, - /// Default/base connection timeout. - /// This is used only by URL previews and update/news endpoint checks + /// Default/base connection timeout (seconds). This is used only by URL + /// previews and update/news endpoint checks. /// - /// Defaults to 10 seconds + /// default: 10 #[serde(default = "default_request_conn_timeout")] pub request_conn_timeout: u64, - /// Default/base request timeout. The time waiting to receive more data from - /// another server. This is used only by URL previews, update/news, and - /// misc endpoint checks + /// Default/base request timeout (seconds). The time waiting to receive more + /// data from another server. This is used only by URL previews, + /// update/news, and misc endpoint checks. /// - /// Defaults to 35 seconds + /// default: 35 #[serde(default = "default_request_timeout")] pub request_timeout: u64, - /// Default/base request total timeout. The time limit for a whole request. - /// This is set very high to not cancel healthy requests while serving as a - /// backstop. This is used only by URL previews and update/news endpoint - /// checks + /// Default/base request total timeout (seconds). The time limit for a whole + /// request. This is set very high to not cancel healthy requests while + /// serving as a backstop. This is used only by URL previews and + /// update/news endpoint checks. /// - /// Defaults to 320 seconds + /// default: 320 #[serde(default = "default_request_total_timeout")] pub request_total_timeout: u64, - /// Default/base idle connection pool timeout - /// This is used only by URL previews and update/news endpoint checks + /// Default/base idle connection pool timeout (seconds). This is used only + /// by URL previews and update/news endpoint checks. /// - /// Defaults to 5 seconds + /// default: 5 #[serde(default = "default_request_idle_timeout")] pub request_idle_timeout: u64, - /// Default/base max idle connections per host - /// This is used only by URL previews and update/news endpoint checks + /// Default/base max idle connections per host. This is used only by URL + /// previews and update/news endpoint checks. Defaults to 1 as generally the + /// same open connection can be re-used. /// - /// Defaults to 1 as generally the same open connection can be re-used + /// default: 1 #[serde(default = "default_request_idle_per_host")] pub request_idle_per_host: u16, - /// Federation well-known resolution connection timeout + /// Federation well-known resolution connection timeout (seconds) /// - /// Defaults to 6 seconds + /// default: 6 #[serde(default = "default_well_known_conn_timeout")] pub well_known_conn_timeout: u64, - /// Federation HTTP well-known resolution request timeout + /// Federation HTTP well-known resolution request timeout (seconds) /// - /// Defaults to 10 seconds + /// default: 10 #[serde(default = "default_well_known_timeout")] pub well_known_timeout: u64, - /// Federation client request timeout - /// You most definitely want this to be high to account for extremely large - /// room joins, slow homeservers, your own resources etc. + /// Federation client request timeout (seconds). You most definitely want + /// this to be high to account for extremely large room joins, slow + /// homeservers, your own resources etc. /// - /// Defaults to 300 seconds + /// default: 300 #[serde(default = "default_federation_timeout")] pub federation_timeout: u64, - /// Federation client idle connection pool timeout + /// Federation client idle connection pool timeout (seconds) /// - /// Defaults to 25 seconds + /// default: 25 #[serde(default = "default_federation_idle_timeout")] pub federation_idle_timeout: u64, - /// Federation client max idle connections per host + /// Federation client max idle connections per host. Defaults to 1 as + /// generally the same open connection can be re-used /// - /// Defaults to 1 as generally the same open connection can be re-used + /// default: 1 #[serde(default = "default_federation_idle_per_host")] pub federation_idle_per_host: u16, - /// Federation sender request timeout - /// The time it takes for the remote server to process sent transactions can - /// take a while. + /// Federation sender request timeout (seconds). The time it takes for the + /// remote server to process sent transactions can take a while. /// - /// Defaults to 180 seconds + /// default: 180 #[serde(default = "default_sender_timeout")] pub sender_timeout: u64, - /// Federation sender idle connection pool timeout + /// Federation sender idle connection pool timeout (seconds) /// - /// Defaults to 180 seconds + /// default: 180 #[serde(default = "default_sender_idle_timeout")] pub sender_idle_timeout: u64, - /// Federation sender transaction retry backoff limit + /// Federation sender transaction retry backoff limit (seconds) /// - /// Defaults to 86400 seconds + /// default: 86400 #[serde(default = "default_sender_retry_backoff_limit")] pub sender_retry_backoff_limit: u64, - /// Appservice URL request connection timeout + /// Appservice URL request connection timeout. Defaults to 35 seconds as + /// generally appservices are hosted within the same network. /// - /// Defaults to 35 seconds as generally appservices are hosted within the - /// same network + /// default: 35 #[serde(default = "default_appservice_timeout")] pub appservice_timeout: u64, - /// Appservice URL idle connection pool timeout + /// Appservice URL idle connection pool timeout (seconds) /// - /// Defaults to 300 seconds + /// default: 300 #[serde(default = "default_appservice_idle_timeout")] pub appservice_idle_timeout: u64, @@ -377,12 +385,11 @@ pub struct Config { /// no default pub registration_token_file: Option, - /// controls whether encrypted rooms and events are allowed (default true) + /// Controls whether encrypted rooms and events are allowed. #[serde(default = "true_fn")] pub allow_encryption: bool, - /// controls whether federation is allowed or not - /// defaults to true + /// Controls whether federation is allowed or not. #[serde(default = "true_fn")] pub allow_federation: bool, @@ -487,6 +494,8 @@ pub struct Config { /// /// (Currently, conduwuit doesn't support batched key requests, so this list /// should only contain other Synapse servers) Defaults to `matrix.org` + /// + /// default: ["matrix.org"] #[serde(default = "default_trusted_servers")] pub trusted_servers: Vec, @@ -527,13 +536,13 @@ pub struct Config { /// binary from trace macros. For debug builds, this restriction is not /// applied. /// - /// Defaults to "info" + /// default: "info" #[serde(default = "default_log")] pub log: String, /// controls whether logs will be outputted with ANSI colours /// - /// defaults to true + /// default: true #[serde(default = "true_fn", alias = "log_colours")] pub log_colors: bool, @@ -542,7 +551,7 @@ pub struct Config { /// These are the OpenID tokens that are primarily used for Matrix account /// integrations, *not* OIDC/OpenID Connect/etc /// - /// Defaults to 3600 (1 hour) + /// default: 3600 #[serde(default = "default_openid_token_ttl")] pub openid_token_ttl: u64, @@ -585,9 +594,9 @@ pub struct Config { /// no default pub turn_secret_file: Option, - /// TURN TTL + /// TURN TTL in seconds /// - /// Default is 86400 seconds + /// default: 86400 #[serde(default = "default_turn_ttl")] pub turn_ttl: u64, @@ -629,10 +638,14 @@ pub struct Config { pub rocksdb_log_stderr: bool, /// Max RocksDB `LOG` file size before rotating in bytes. Defaults to 4MB. + /// + /// default: 4194304 #[serde(default = "default_rocksdb_max_log_file_size")] pub rocksdb_max_log_file_size: usize, - /// Time in seconds before RocksDB will forcibly rotate logs. Defaults to 0. + /// Time in seconds before RocksDB will forcibly rotate logs. + /// + /// default: 0 #[serde(default = "default_rocksdb_log_time_to_roll")] pub rocksdb_log_time_to_roll: usize, @@ -649,8 +662,6 @@ pub struct Config { /// RocksDB issues, try enabling this option as it turns off Direct IO and /// feel free to report in the conduwuit Matrix room if this option fixes /// your DB issues. See https://github.com/facebook/rocksdb/wiki/Direct-IO for more information. - /// - /// Defaults to false #[serde(default)] pub rocksdb_optimize_for_spinning_disks: bool, @@ -662,14 +673,16 @@ pub struct Config { /// Amount of threads that RocksDB will use for parallelism on database /// operatons such as cleanup, sync, flush, compaction, etc. Set to 0 to use - /// all your logical threads. + /// all your logical threads. Defaults to your CPU logical thread count. /// - /// Defaults to your CPU logical thread count. + /// default: 0 #[serde(default = "default_rocksdb_parallelism_threads")] pub rocksdb_parallelism_threads: usize, /// Maximum number of LOG files RocksDB will keep. This must *not* be set to /// 0. It must be at least 1. Defaults to 3 as these are not very useful. + /// + /// default: 3 #[serde(default = "default_rocksdb_max_log_files")] pub rocksdb_max_log_files: usize, @@ -682,7 +695,7 @@ pub struct Config { /// /// "none" will disable compression. /// - /// Defaults to "zstd" + /// default: "zstd" #[serde(default = "default_rocksdb_compression_algo")] pub rocksdb_compression_algo: String, @@ -746,6 +759,8 @@ pub struct Config { /// See https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes for more information /// /// Defaults to 1 (TolerateCorruptedTailRecords) + /// + /// default: 1 #[serde(default = "default_rocksdb_recovery_mode")] pub rocksdb_recovery_mode: u8, @@ -760,8 +775,6 @@ pub struct Config { /// repair. /// - Disabling repair mode and restarting the server is recommended after /// running the repair. - /// - /// Defaults to false #[serde(default)] pub rocksdb_repair: bool, @@ -798,6 +811,8 @@ pub struct Config { /// 6 = All statistics. /// /// Defaults to 1 (No statistics, except in debug-mode) + /// + /// default: 1 #[serde(default = "default_rocksdb_stats_level")] pub rocksdb_stats_level: u8, @@ -831,11 +846,15 @@ pub struct Config { /// Config option to control how many seconds before presence updates that /// you are idle. Defaults to 5 minutes. + /// + /// default: 300 #[serde(default = "default_presence_idle_timeout_s")] pub presence_idle_timeout_s: u64, /// Config option to control how many seconds before presence updates that /// you are offline. Defaults to 30 minutes. + /// + /// default: 1800 #[serde(default = "default_presence_offline_timeout_s")] pub presence_offline_timeout_s: u64, @@ -843,42 +862,46 @@ pub struct Config { /// Disabling is offered as an optimization for servers participating in /// many large rooms or when resources are limited. Disabling it may cause /// incorrect presence states (i.e. stuck online) to be seen for some - /// remote users. Defaults to true. + /// remote users. #[serde(default = "true_fn")] pub presence_timeout_remote_users: bool, /// Config option to control whether we should receive remote incoming read - /// receipts. Defaults to true. + /// receipts. #[serde(default = "true_fn")] pub allow_incoming_read_receipts: bool, /// Config option to control whether we should send read receipts to remote - /// servers. Defaults to true. + /// servers. #[serde(default = "true_fn")] pub allow_outgoing_read_receipts: bool, - /// Config option to control outgoing typing updates to federation. Defaults - /// to true. + /// Config option to control outgoing typing updates to federation. #[serde(default = "true_fn")] pub allow_outgoing_typing: bool, /// Config option to control incoming typing updates from federation. - /// Defaults to true. #[serde(default = "true_fn")] pub allow_incoming_typing: bool, /// Config option to control maximum time federation user can indicate /// typing. + /// + /// default: 30 #[serde(default = "default_typing_federation_timeout_s")] pub typing_federation_timeout_s: u64, /// Config option to control minimum time local client can indicate typing. /// This does not override a client's request to stop typing. It only /// enforces a minimum value in case of no stop request. + /// + /// default: 15 #[serde(default = "default_typing_client_timeout_min_s")] pub typing_client_timeout_min_s: u64, /// Config option to control maximum time local client can indicate typing. + /// + /// default: 45 #[serde(default = "default_typing_client_timeout_max_s")] pub typing_client_timeout_max_s: u64, @@ -910,7 +933,7 @@ pub struct Config { pub brotli_compression: bool, /// Set to true to allow user type "guest" registrations. Element attempts - /// to register guest users automatically. Defaults to false + /// to register guest users automatically. Defaults to false. #[serde(default)] pub allow_guest_registration: bool, @@ -920,7 +943,7 @@ pub struct Config { pub log_guest_registrations: bool, /// Set to true to allow guest registrations/users to auto join any rooms - /// specified in `auto_join_rooms` Defaults to false + /// specified in `auto_join_rooms` Defaults to false. #[serde(default)] pub allow_guests_auto_join_rooms: bool, @@ -964,9 +987,7 @@ pub struct Config { /// is now disabled by default. You may still return to upstream Conduit /// but you have to run Conduwuit at least once with this set to true and /// allow the media_startup_check to take place before shutting - /// down to return to Conduit. - /// - /// Disabled by default. + /// down to return to Conduit. Disabled by default. #[serde(default)] pub media_compat_file_link: bool, @@ -975,9 +996,7 @@ pub struct Config { /// corresponding entries will be removed from the database. This is /// disabled by default because if the media directory is accidentally moved /// or inaccessible the metadata entries in the database will be lost with - /// sadness. - /// - /// Disabled by default. + /// sadness. Disabled by default. #[serde(default)] pub prune_missing_media: bool, @@ -1008,12 +1027,35 @@ pub struct Config { /// RFC1918, unroutable, loopback, multicast, and testnet addresses for /// security. /// - /// To disable, set this to be an empty vector (`[]`). /// Please be aware that this is *not* a guarantee. You should be using a /// firewall with zones as doing this on the application layer may have /// bypasses. /// /// Currently this does not account for proxies in use like Synapse does. + /// + /// To disable, set this to be an empty vector (`[]`). + /// The default is: + /// [ + /// "127.0.0.0/8", + /// "10.0.0.0/8", + /// "172.16.0.0/12", + /// "192.168.0.0/16", + /// "100.64.0.0/10", + /// "192.0.0.0/24", + /// "169.254.0.0/16", + /// "192.88.99.0/24", + /// "198.18.0.0/15", + /// "192.0.2.0/24", + /// "198.51.100.0/24", + /// "203.0.113.0/24", + /// "224.0.0.0/4", + /// "::1/128", + /// "fe80::/10", + /// "fc00::/7", + /// "2001:db8::/32", + /// "ff00::/8", + /// "fec0::/10", + /// ] #[serde(default = "default_ip_range_denylist")] pub ip_range_denylist: Vec, @@ -1060,7 +1102,9 @@ pub struct Config { pub url_preview_url_contains_allowlist: Vec, /// Maximum amount of bytes allowed in a URL preview body size when - /// spidering. Defaults to 384KB (384_000 bytes) + /// spidering. Defaults to 384KB. + /// + /// defaukt: 384000 #[serde(default = "default_url_preview_max_spider_size")] pub url_preview_max_spider_size: usize, @@ -1109,27 +1153,27 @@ pub struct Config { /// reattempt every message without trimming the queues; this may consume /// significant disk. Set this value to 0 to drop all messages without any /// attempt at redelivery. + /// + /// default: 50 #[serde(default = "default_startup_netburst_keep")] pub startup_netburst_keep: i64, /// controls whether non-admin local users are forbidden from sending room /// invites (local and remote), and if non-admin users can receive remote /// room invites. admins are always allowed to send and receive all room - /// invites. defaults to false + /// invites. #[serde(default)] pub block_non_admin_invites: bool, /// Allows admins to enter commands in rooms other than #admins by prefixing /// with \!admin. The reply will be publicly visible to the room, - /// originating from the sender. defaults to true + /// originating from the sender. #[serde(default = "true_fn")] pub admin_escape_commands: bool, /// Controls whether the conduwuit admin room console / CLI will immediately /// activate on startup. This option can also be enabled with `--console` - /// conduwuit argument - /// - /// Defaults to false + /// conduwuit argument. #[serde(default)] pub admin_console_automatic: bool, @@ -1145,21 +1189,20 @@ pub struct Config { /// Such example could be: `./conduwuit --execute "server admin-notice /// conduwuit has started up at $(date)"` /// - /// Defaults to nothing. + /// default: [] #[serde(default)] pub admin_execute: Vec, /// Controls whether conduwuit should error and fail to start if an admin - /// execute command (`--execute` / `admin_execute`) fails - /// - /// Defaults to false + /// execute command (`--execute` / `admin_execute`) fails. #[serde(default)] pub admin_execute_errors_ignore: bool, /// Controls the max log level for admin command log captures (logs - /// generated from running admin commands) + /// generated from running admin commands). Defaults to "info" on release + /// builds, else "debug" on debug builds. /// - /// Defaults to "info" on release builds, else "debug" on debug builds + /// default: "info" #[serde(default = "default_admin_log_capture")] pub admin_log_capture: String, @@ -1169,8 +1212,6 @@ pub struct Config { /// Sentry.io crash/panic reporting, performance monitoring/metrics, etc. /// This is NOT enabled by default. conduwuit's default Sentry reporting /// endpoint is o4506996327251968.ingest.us.sentry.io - /// - /// Defaults to *false* #[serde(default)] pub sentry: bool, @@ -1182,8 +1223,6 @@ pub struct Config { pub sentry_endpoint: Option, /// Report your Conduwuit server_name in Sentry.io crash reports and metrics - /// - /// Defaults to false #[serde(default)] pub sentry_send_server_name: bool, @@ -1191,9 +1230,9 @@ pub struct Config { /// /// Note that too high values may impact performance, and can be disabled by /// setting it to 0.0 (0%) This value is read as a percentage to Sentry, - /// represented as a decimal + /// represented as a decimal. Defaults to 15% of traces (0.15) /// - /// Defaults to 15% of traces (0.15) + /// default: 0.15 #[serde(default = "default_sentry_traces_sample_rate")] pub sentry_traces_sample_rate: f32, From 5cb0a5f67668828b7c47b8a8efc3f8c834c1d7f2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Oct 2024 22:16:59 +0000 Subject: [PATCH 0940/2091] add config generator controls via attribute metadatas Signed-off-by: Jason Volk --- src/core/config/mod.rs | 45 +++++++++++++++++++- src/macros/config.rs | 93 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 127 insertions(+), 11 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 919bb486..ff214420 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -28,10 +28,19 @@ use self::proxy::ProxyConfig; use crate::{err, error::Error, utils::sys, Result}; /// all the config options for conduwuit -#[config_example_generator] -#[derive(Clone, Debug, Deserialize)] #[allow(clippy::struct_excessive_bools)] #[allow(rustdoc::broken_intra_doc_links, rustdoc::bare_urls)] +#[derive(Clone, Debug, Deserialize)] +#[config_example_generator( + filename = "conduwuit-example.toml", + section = "global", + undocumented = "# This item is undocumented. Please contribute documentation for it.", + header = "### Conduwuit Configuration\n###\n### THIS FILE IS GENERATED. YOUR CHANGES WILL BE OVERWRITTEN!\n### \ + You should rename this file before configuring your server. Changes\n### to documentation and defaults \ + can be contributed in sourcecode at\n### src/core/config/mod.rs. This file is generated when \ + building.\n###\n", + ignore = "catchall well_known tls" +)] pub struct Config { /// The server_name is the pretty name of this server. It is used as a /// suffix for user and room ids. Examples: matrix.org, conduit.rs @@ -71,6 +80,7 @@ pub struct Config { #[serde(default = "default_port")] port: ListeningPort, + // external structure; separate section pub tls: Option, /// Uncomment unix_socket_path to listen on a UNIX socket at the specified @@ -458,15 +468,18 @@ pub struct Config { #[serde(default = "true_fn")] pub allow_unstable_room_versions: bool, + /// default: 10 #[serde(default = "default_default_room_version")] pub default_room_version: RoomVersionId, + // external structure; separate section #[serde(default)] pub well_known: WellKnownConfig, #[serde(default)] pub allow_jaeger: bool, + /// default: "info" #[serde(default = "default_jaeger_filter")] pub jaeger_filter: String, @@ -478,12 +491,38 @@ pub struct Config { #[serde(default)] pub tracing_flame: bool, + /// default: "info" #[serde(default = "default_tracing_flame_filter")] pub tracing_flame_filter: String, + /// default: "./tracing.folded" #[serde(default = "default_tracing_flame_output_path")] pub tracing_flame_output_path: String, + /// Examples: + /// - No proxy (default): + /// proxy ="none" + /// + /// - For global proxy, create the section at the bottom of this file: + /// [global.proxy] + /// global = { url = "socks5h://localhost:9050" } + /// + /// - To proxy some domains: + /// [global.proxy] + /// [[global.proxy.by_domain]] + /// url = "socks5h://localhost:9050" + /// include = ["*.onion", "matrix.myspecial.onion"] + /// exclude = ["*.myspecial.onion"] + /// + /// Include vs. Exclude: + /// - If include is an empty list, it is assumed to be `["*"]`. + /// - If a domain matches both the exclude and include list, the proxy will + /// only be used if it was included because of a more specific rule than + /// it was excluded. In the above example, the proxy would be used for + /// `ordinary.onion`, `matrix.myspecial.onion`, but not + /// `hello.myspecial.onion`. + /// + /// default: "none" #[serde(default)] pub proxy: ProxyConfig, @@ -1278,6 +1317,7 @@ pub struct Config { } #[derive(Clone, Debug, Deserialize)] +#[config_example_generator(filename = "conduwuit-example.toml", section = "global.tls")] pub struct TlsConfig { pub certs: String, pub key: String, @@ -1287,6 +1327,7 @@ pub struct TlsConfig { } #[derive(Clone, Debug, Deserialize, Default)] +#[config_example_generator(filename = "conduwuit-example.toml", section = "global.well_known")] pub struct WellKnownConfig { pub client: Option, pub server: Option, diff --git a/src/macros/config.rs b/src/macros/config.rs index 3c93bd08..f8616352 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -1,18 +1,21 @@ -use std::{fmt::Write as _, fs::File, io::Write as _}; +use std::{ + collections::{HashMap, HashSet}, + fmt::Write as _, + fs::OpenOptions, + io::Write as _, +}; use proc_macro::TokenStream; use proc_macro2::Span; use quote::ToTokens; use syn::{ - parse::Parser, punctuated::Punctuated, Error, Expr, ExprLit, Field, Fields, FieldsNamed, ItemStruct, Lit, Meta, - MetaList, MetaNameValue, Type, TypePath, + parse::Parser, punctuated::Punctuated, spanned::Spanned, Error, Expr, ExprLit, Field, Fields, FieldsNamed, + ItemStruct, Lit, Meta, MetaList, MetaNameValue, Type, TypePath, }; use crate::{utils::is_cargo_build, Result}; const UNDOCUMENTED: &str = "# This item is undocumented. Please contribute documentation for it."; -const HEADER: &str = "## Conduwuit Configuration\n##\n## THIS FILE IS GENERATED. Changes to documentation and \ - defaults must\n## be made within the code found at src/core/config/\n"; #[allow(clippy::needless_pass_by_value)] pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result { @@ -25,11 +28,41 @@ pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result Result<()> { - let mut file = File::create("conduwuit-example.toml") +fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { + let settings = get_settings(args); + + let filename = settings + .get("filename") + .ok_or_else(|| Error::new(args[0].span(), "missing required 'filename' attribute argument"))?; + + let undocumented = settings + .get("undocumented") + .map_or(UNDOCUMENTED, String::as_str); + + let ignore: HashSet<&str> = settings + .get("ignore") + .map_or("", String::as_str) + .split(' ') + .collect(); + + let section = settings + .get("section") + .ok_or_else(|| Error::new(args[0].span(), "missing required 'section' attribute argument"))?; + + let mut file = OpenOptions::new() + .write(true) + .create(section == "global") + .truncate(section == "global") + .append(section != "global") + .open(filename) .map_err(|e| Error::new(Span::call_site(), format!("Failed to open config file for generation: {e}")))?; - file.write_all(HEADER.as_bytes()) + if let Some(header) = settings.get("header") { + file.write_all(header.as_bytes()) + .expect("written to config file"); + } + + file.write_fmt(format_args!("\n[{section}]\n")) .expect("written to config file"); if let Fields::Named(FieldsNamed { @@ -42,12 +75,16 @@ fn generate_example(input: &ItemStruct, _args: &[Meta]) -> Result<()> { continue; }; + if ignore.contains(ident.to_string().as_str()) { + continue; + } + let Some(type_name) = get_type_name(field) else { continue; }; let doc = get_doc_comment(field) - .unwrap_or_else(|| UNDOCUMENTED.into()) + .unwrap_or_else(|| undocumented.into()) .trim_end() .to_owned(); @@ -75,9 +112,47 @@ fn generate_example(input: &ItemStruct, _args: &[Meta]) -> Result<()> { } } + if let Some(footer) = settings.get("footer") { + file.write_all(footer.as_bytes()) + .expect("written to config file"); + } + Ok(()) } +fn get_settings(args: &[Meta]) -> HashMap { + let mut map = HashMap::new(); + for arg in args { + let Meta::NameValue(MetaNameValue { + path, + value, + .. + }) = arg + else { + continue; + }; + + let Expr::Lit( + ExprLit { + lit: Lit::Str(str), + .. + }, + .., + ) = value + else { + continue; + }; + + let Some(key) = path.segments.iter().next().map(|s| s.ident.clone()) else { + continue; + }; + + map.insert(key.to_string(), str.value()); + } + + map +} + fn get_default(field: &Field) -> Option { for attr in &field.attrs { let Meta::List(MetaList { From c769fcc3471dcc4c976569be57aea65109105f92 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 24 Oct 2024 01:31:30 +0000 Subject: [PATCH 0941/2091] move core result into core utils Signed-off-by: Jason Volk --- src/core/mod.rs | 4 +--- src/core/utils/mod.rs | 1 + src/core/{ => utils}/result.rs | 0 src/core/{ => utils}/result/debug_inspect.rs | 0 src/core/{ => utils}/result/flat_ok.rs | 0 src/core/{ => utils}/result/inspect_log.rs | 0 src/core/{ => utils}/result/into_is_ok.rs | 0 src/core/{ => utils}/result/log_debug_err.rs | 0 src/core/{ => utils}/result/log_err.rs | 0 src/core/{ => utils}/result/map_expect.rs | 0 src/core/{ => utils}/result/not_found.rs | 0 src/core/{ => utils}/result/unwrap_infallible.rs | 0 12 files changed, 2 insertions(+), 3 deletions(-) rename src/core/{ => utils}/result.rs (100%) rename src/core/{ => utils}/result/debug_inspect.rs (100%) rename src/core/{ => utils}/result/flat_ok.rs (100%) rename src/core/{ => utils}/result/inspect_log.rs (100%) rename src/core/{ => utils}/result/into_is_ok.rs (100%) rename src/core/{ => utils}/result/log_debug_err.rs (100%) rename src/core/{ => utils}/result/log_err.rs (100%) rename src/core/{ => utils}/result/map_expect.rs (100%) rename src/core/{ => utils}/result/not_found.rs (100%) rename src/core/{ => utils}/result/unwrap_infallible.rs (100%) diff --git a/src/core/mod.rs b/src/core/mod.rs index 491d8b4c..79052554 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -7,7 +7,6 @@ pub mod log; pub mod metrics; pub mod mods; pub mod pdu; -pub mod result; pub mod server; pub mod utils; @@ -19,9 +18,8 @@ pub use config::Config; pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; pub use pdu::{PduBuilder, PduCount, PduEvent}; -pub use result::Result; pub use server::Server; -pub use utils::{ctor, dtor, implement}; +pub use utils::{ctor, dtor, implement, result, result::Result}; pub use crate as conduit_core; diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 96a98537..3943a8da 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -10,6 +10,7 @@ pub mod json; pub mod math; pub mod mutex_map; pub mod rand; +pub mod result; pub mod set; pub mod stream; pub mod string; diff --git a/src/core/result.rs b/src/core/utils/result.rs similarity index 100% rename from src/core/result.rs rename to src/core/utils/result.rs diff --git a/src/core/result/debug_inspect.rs b/src/core/utils/result/debug_inspect.rs similarity index 100% rename from src/core/result/debug_inspect.rs rename to src/core/utils/result/debug_inspect.rs diff --git a/src/core/result/flat_ok.rs b/src/core/utils/result/flat_ok.rs similarity index 100% rename from src/core/result/flat_ok.rs rename to src/core/utils/result/flat_ok.rs diff --git a/src/core/result/inspect_log.rs b/src/core/utils/result/inspect_log.rs similarity index 100% rename from src/core/result/inspect_log.rs rename to src/core/utils/result/inspect_log.rs diff --git a/src/core/result/into_is_ok.rs b/src/core/utils/result/into_is_ok.rs similarity index 100% rename from src/core/result/into_is_ok.rs rename to src/core/utils/result/into_is_ok.rs diff --git a/src/core/result/log_debug_err.rs b/src/core/utils/result/log_debug_err.rs similarity index 100% rename from src/core/result/log_debug_err.rs rename to src/core/utils/result/log_debug_err.rs diff --git a/src/core/result/log_err.rs b/src/core/utils/result/log_err.rs similarity index 100% rename from src/core/result/log_err.rs rename to src/core/utils/result/log_err.rs diff --git a/src/core/result/map_expect.rs b/src/core/utils/result/map_expect.rs similarity index 100% rename from src/core/result/map_expect.rs rename to src/core/utils/result/map_expect.rs diff --git a/src/core/result/not_found.rs b/src/core/utils/result/not_found.rs similarity index 100% rename from src/core/result/not_found.rs rename to src/core/utils/result/not_found.rs diff --git a/src/core/result/unwrap_infallible.rs b/src/core/utils/result/unwrap_infallible.rs similarity index 100% rename from src/core/result/unwrap_infallible.rs rename to src/core/utils/result/unwrap_infallible.rs From aa768b5dec1338a6dbdd1b3a9a00bc6ec9d53090 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 24 Oct 2024 05:03:31 +0000 Subject: [PATCH 0942/2091] distill active and old keys for federation key/server response Signed-off-by: Jason Volk --- src/api/server/key.rs | 40 +++++++++++++++++++++++++--------- src/service/server_keys/mod.rs | 17 +++++++++++++++ 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/api/server/key.rs b/src/api/server/key.rs index 3913ce43..5284593d 100644 --- a/src/api/server/key.rs +++ b/src/api/server/key.rs @@ -1,10 +1,14 @@ -use std::{collections::BTreeMap, time::Duration}; +use std::{ + collections::BTreeMap, + mem::take, + time::{Duration, SystemTime}, +}; use axum::{extract::State, response::IntoResponse, Json}; use conduit::{utils::timepoint_from_now, Result}; use ruma::{ api::{ - federation::discovery::{get_server_keys, ServerSigningKeys}, + federation::discovery::{get_server_keys, OldVerifyKey, ServerSigningKeys}, OutgoingResponse, }, serde::Raw, @@ -21,21 +25,32 @@ use ruma::{ // signature for the response pub(crate) async fn get_server_keys_route(State(services): State) -> Result { let server_name = services.globals.server_name(); - let verify_keys = services.server_keys.verify_keys_for(server_name).await; + let active_key_id = services.server_keys.active_key_id(); + let mut all_keys = services.server_keys.verify_keys_for(server_name).await; + + let verify_keys = all_keys + .remove_entry(active_key_id) + .expect("active verify_key is missing"); + + let old_verify_keys = all_keys + .into_iter() + .map(|(id, key)| (id, OldVerifyKey::new(expires_ts(), key.key))) + .collect(); + let server_key = ServerSigningKeys { - verify_keys, + verify_keys: [verify_keys].into(), + old_verify_keys, server_name: server_name.to_owned(), valid_until_ts: valid_until_ts(), - old_verify_keys: BTreeMap::new(), signatures: BTreeMap::new(), }; - let response = get_server_keys::v2::Response { - server_key: Raw::new(&server_key)?, - } - .try_into_http_response::>()?; + let server_key = Raw::new(&server_key)?; + let mut response = get_server_keys::v2::Response::new(server_key) + .try_into_http_response::>() + .map(|mut response| take(response.body_mut())) + .and_then(|body| serde_json::from_slice(&body).map_err(Into::into))?; - let mut response = serde_json::from_slice(response.body())?; services.server_keys.sign_json(&mut response)?; Ok(Json(response)) @@ -47,6 +62,11 @@ fn valid_until_ts() -> MilliSecondsSinceUnixEpoch { MilliSecondsSinceUnixEpoch::from_system_time(timepoint).expect("UInt should not overflow") } +fn expires_ts() -> MilliSecondsSinceUnixEpoch { + let timepoint = SystemTime::now(); + MilliSecondsSinceUnixEpoch::from_system_time(timepoint).expect("UInt should not overflow") +} + /// # `GET /_matrix/key/v2/server/{keyId}` /// /// Gets the public signing keys of this server. diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index dc09703c..dae45a51 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -44,7 +44,9 @@ pub type PubKeys = PublicKeySet; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let minimum_valid = Duration::from_secs(3600); + let (keypair, verify_keys) = keypair::init(args.db)?; + debug_assert!(verify_keys.len() == 1, "only one active verify_key supported"); Ok(Arc::new(Self { keypair, @@ -68,6 +70,21 @@ impl crate::Service for Service { #[inline] pub fn keypair(&self) -> &Ed25519KeyPair { &self.keypair } +#[implement(Service)] +#[inline] +pub fn active_key_id(&self) -> &ServerSigningKeyId { self.active_verify_key().0 } + +#[implement(Service)] +#[inline] +pub fn active_verify_key(&self) -> (&ServerSigningKeyId, &VerifyKey) { + debug_assert!(self.verify_keys.len() <= 1, "more than one active verify_key"); + self.verify_keys + .iter() + .next() + .map(|(id, key)| (id.as_ref(), key)) + .expect("missing active verify_key") +} + #[implement(Service)] async fn add_signing_keys(&self, new_keys: ServerSigningKeys) { let origin = &new_keys.server_name; From 89cc865868102697415802e45f4ce19bbaad33d2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 25 Oct 2024 14:45:22 -0400 Subject: [PATCH 0943/2091] bump conduwuit to 0.5.0 Signed-off-by: strawberry --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ac7cc35..31339b27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -592,7 +592,7 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "conduit" -version = "0.4.7" +version = "0.5.0" dependencies = [ "clap", "conduit_admin", @@ -621,7 +621,7 @@ dependencies = [ [[package]] name = "conduit_admin" -version = "0.4.7" +version = "0.5.0" dependencies = [ "clap", "conduit_api", @@ -642,7 +642,7 @@ dependencies = [ [[package]] name = "conduit_api" -version = "0.4.7" +version = "0.5.0" dependencies = [ "axum", "axum-client-ip", @@ -674,7 +674,7 @@ dependencies = [ [[package]] name = "conduit_core" -version = "0.4.7" +version = "0.5.0" dependencies = [ "argon2", "arrayvec", @@ -725,7 +725,7 @@ dependencies = [ [[package]] name = "conduit_database" -version = "0.4.7" +version = "0.5.0" dependencies = [ "arrayvec", "conduit_core", @@ -741,7 +741,7 @@ dependencies = [ [[package]] name = "conduit_macros" -version = "0.4.7" +version = "0.5.0" dependencies = [ "itertools 0.13.0", "proc-macro2", @@ -751,7 +751,7 @@ dependencies = [ [[package]] name = "conduit_router" -version = "0.4.7" +version = "0.5.0" dependencies = [ "axum", "axum-client-ip", @@ -784,7 +784,7 @@ dependencies = [ [[package]] name = "conduit_service" -version = "0.4.7" +version = "0.5.0" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 966c2818..64cd8ba3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" rust-version = "1.82.0" -version = "0.4.7" +version = "0.5.0" [workspace.metadata.crane] name = "conduit" From f29879288d00e24ec04d6a42bab6ef91e8bafda7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 25 Oct 2024 20:47:30 -0400 Subject: [PATCH 0944/2091] document conduwuit k8s helm chart Signed-off-by: strawberry --- docs/deploying/kubernetes.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 docs/deploying/kubernetes.md diff --git a/docs/deploying/kubernetes.md b/docs/deploying/kubernetes.md new file mode 100644 index 00000000..2a1bcb51 --- /dev/null +++ b/docs/deploying/kubernetes.md @@ -0,0 +1,4 @@ +# conduwuit for Kubernetes + +conduwuit doesn't support horizontal scalability or distributed loading natively, however a community maintained Helm Chart is available here to run conduwuit on Kubernetes: + From 652b04b9b6bc30f55b286645bb8cd706d429056c Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 25 Oct 2024 20:48:14 -0400 Subject: [PATCH 0945/2091] update conduwuit freebsd docs Signed-off-by: strawberry --- docs/deploying/freebsd.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/deploying/freebsd.md b/docs/deploying/freebsd.md index 4ac83515..65b40204 100644 --- a/docs/deploying/freebsd.md +++ b/docs/deploying/freebsd.md @@ -1,11 +1,5 @@ # conduwuit for FreeBSD -conduwuit at the moment does not provide FreeBSD builds. Building conduwuit on -FreeBSD requires a specific environment variable to use the system prebuilt -RocksDB library instead of rust-rocksdb / rust-librocksdb-sys which does *not* -work and will cause a build error or coredump. +conduwuit at the moment does not provide FreeBSD builds or have FreeBSD packaging, however conduwuit does build and work on FreeBSD using the system-provided RocksDB. -Use the following environment variable: `ROCKSDB_LIB_DIR=/usr/local/lib` - -Such example commandline with it can be: `ROCKSDB_LIB_DIR=/usr/local/lib cargo -build --release` +Contributions for getting conduwuit packaged are welcome. From 2ce91f33afbd08a722684f6d0e3928cc9a497696 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 25 Oct 2024 21:08:34 -0400 Subject: [PATCH 0946/2091] log method on tracing req spans, fix path sometimes being truncated Signed-off-by: strawberry --- src/api/router/auth.rs | 4 +--- src/router/layers.rs | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 6b90c5ff..28d6bc55 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -5,7 +5,6 @@ use axum_extra::{ TypedHeader, }; use conduit::{debug_error, err, warn, Err, Error, Result}; -use http::uri::PathAndQuery; use ruma::{ api::{client::error::ErrorKind, AuthScheme, Metadata}, server_util::authorization::XMatrix, @@ -190,12 +189,11 @@ async fn auth_server(services: &Services, request: &mut Request, body: Option<&C let destination = services.globals.server_name(); let origin = &x_matrix.origin; - #[allow(clippy::or_fun_call)] let signature_uri = request .parts .uri .path_and_query() - .unwrap_or(&PathAndQuery::from_static("/")) + .expect("all requests have a path") .to_string(); let signature: [Member; 1] = [(x_matrix.key.to_string(), Value::String(x_matrix.sig.to_string()))]; diff --git a/src/router/layers.rs b/src/router/layers.rs index a1a70bb8..908105d8 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -184,12 +184,20 @@ fn catch_panic(err: Box) -> http::Response(request: &http::Request) -> tracing::Span { - let path = request - .extensions() - .get::() - .map_or_else(|| request.uri().path(), truncated_matched_path); + let path = request.extensions().get::().map_or_else( + || { + request + .uri() + .path_and_query() + .expect("all requests have a path") + .as_str() + }, + truncated_matched_path, + ); - tracing::info_span!("router:", %path) + let method = request.method(); + + tracing::info_span!("router:", %method, %path) } fn truncated_matched_path(path: &MatchedPath) -> &str { From 0efe24a028f5954e9aa4969f533ed89a51115bbc Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 25 Oct 2024 21:13:14 -0400 Subject: [PATCH 0947/2091] remove spaces from CSP header to save a few bytes Signed-off-by: strawberry --- src/router/layers.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index 908105d8..fd68cc36 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -24,15 +24,15 @@ use tracing::Level; use crate::{request, router}; -const CONDUWUIT_CSP: &[&str] = &[ - "sandbox", +const CONDUWUIT_CSP: &[&str; 5] = &[ "default-src 'none'", "frame-ancestors 'none'", "form-action 'none'", "base-uri 'none'", + "sandbox", ]; -const CONDUWUIT_PERMISSIONS_POLICY: &[&str] = &["interest-cohort=()", "browsing-topics=()"]; +const CONDUWUIT_PERMISSIONS_POLICY: &[&str; 2] = &["interest-cohort=()", "browsing-topics=()"]; pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { let server = &services.server; @@ -78,7 +78,7 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { )) .layer(SetResponseHeaderLayer::if_not_present( header::CONTENT_SECURITY_POLICY, - HeaderValue::from_str(&CONDUWUIT_CSP.join("; "))?, + HeaderValue::from_str(&CONDUWUIT_CSP.join(";"))?, )) .layer(cors_layer(server)) .layer(body_limit_layer(server)) From d6991611f0d79d1ad4a1e3cdb5d1372a79b87ac7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 26 Oct 2024 12:32:47 -0400 Subject: [PATCH 0948/2091] add `require_auth_for_profile_requests` config option, check endpoint metadata instead of request string Signed-off-by: strawberry --- src/api/router/auth.rs | 36 ++++++++++++++++++++++++++++++------ src/core/config/mod.rs | 11 ++++++++++- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 28d6bc55..6b1bb1a9 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -6,7 +6,15 @@ use axum_extra::{ }; use conduit::{debug_error, err, warn, Err, Error, Result}; use ruma::{ - api::{client::error::ErrorKind, AuthScheme, Metadata}, + api::{ + client::{ + directory::get_public_rooms, + error::ErrorKind, + profile::{get_avatar_url, get_display_name, get_profile, get_profile_key, get_timezone_key}, + voip::get_turn_server_info, + }, + AuthScheme, IncomingRequest, Metadata, + }, server_util::authorization::XMatrix, CanonicalJsonObject, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; @@ -54,9 +62,8 @@ pub(super) async fn auth( }; if metadata.authentication == AuthScheme::None { - match request.parts.uri.path() { - // TODO: can we check this better? - "/_matrix/client/v3/publicRooms" | "/_matrix/client/r0/publicRooms" => { + match metadata { + &get_public_rooms::v3::Request::METADATA => { if !services .globals .config @@ -73,6 +80,23 @@ pub(super) async fn auth( } } }, + &get_profile::v3::Request::METADATA + | &get_profile_key::unstable::Request::METADATA + | &get_display_name::v3::Request::METADATA + | &get_avatar_url::v3::Request::METADATA + | &get_timezone_key::unstable::Request::METADATA => { + if services.globals.config.require_auth_for_profile_requests { + match token { + Token::Appservice(_) | Token::User(_) => { + // we should have validated the token above + // already + }, + Token::None | Token::Invalid => { + return Err(Error::BadRequest(ErrorKind::MissingToken, "Missing or invalid access token.")); + }, + } + } + }, _ => {}, }; } @@ -107,9 +131,9 @@ pub(super) async fn auth( appservice_info: Some(*info), }) }, - (AuthScheme::AccessToken, Token::None) => match request.parts.uri.path() { + (AuthScheme::AccessToken, Token::None) => match metadata { // TODO: can we check this better? - "/_matrix/client/v3/voip/turnServer" | "/_matrix/client/r0/voip/turnServer" => { + &get_turn_server_info::v3::Request::METADATA => { if services.globals.config.turn_allow_guests { Ok(Auth { origin: None, diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index ff214420..04e44fd7 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -58,7 +58,6 @@ pub struct Config { /// YOU NEED TO EDIT THIS pub server_name: OwnedServerName, - /// Database backend: Only rocksdb is supported. /// default address (IPv4 or IPv6) conduwuit will listen on. Generally you /// want this to be localhost (127.0.0.1 / ::1). If you are using Docker or /// a container NAT networking setup, you likely need this to be 0.0.0.0. @@ -94,6 +93,8 @@ pub struct Config { #[serde(default = "default_unix_socket_perms")] pub unix_socket_perms: u32, + /// Database backend: Only rocksdb is supported. + /// /// default: rocksdb #[serde(default = "default_database_backend")] pub database_backend: String, @@ -406,6 +407,14 @@ pub struct Config { #[serde(default)] pub federation_loopback: bool, + /// Set this to true to require authentication on the normally + /// unauthenticated profile retrieval endpoints (GET) + /// "/_matrix/client/v3/profile/{userId}". + /// + /// This can prevent profile scraping. + #[serde(default)] + pub require_auth_for_profile_requests: bool, + /// Set this to true to allow your server's public room directory to be /// federated. Set this to false to protect against /publicRooms spiders, /// but will forbid external users from viewing your server's public room From 60d84195c51c523b965c17d75ebca861290260e5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 26 Oct 2024 17:26:50 -0400 Subject: [PATCH 0949/2091] implement MSC4210, bump ruwuma Signed-off-by: strawberry --- Cargo.lock | 26 +++---- Cargo.toml | 3 +- src/admin/debug/commands.rs | 1 + src/api/client/push.rs | 100 +++++++++++++------------ src/service/rooms/event_handler/mod.rs | 1 + 5 files changed, 69 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 31339b27..c64d3cc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2976,7 +2976,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "assign", "js_int", @@ -2998,7 +2998,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "js_int", "ruma-common", @@ -3010,7 +3010,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "as_variant", "assign", @@ -3033,7 +3033,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "as_variant", "base64 0.22.1", @@ -3063,7 +3063,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3087,7 +3087,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "bytes", "http", @@ -3105,7 +3105,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "js_int", "thiserror", @@ -3114,7 +3114,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "js_int", "ruma-common", @@ -3124,7 +3124,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "cfg-if", "once_cell", @@ -3140,7 +3140,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "js_int", "ruma-common", @@ -3152,7 +3152,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "headers", "http", @@ -3165,7 +3165,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3181,7 +3181,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73#d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" dependencies = [ "futures-util", "itertools 0.13.0", diff --git a/Cargo.toml b/Cargo.toml index 64cd8ba3..73f16daf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -315,7 +315,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "d7baeb7e5c3ae28e79ad3fe81c5e8b207a26cc73" +rev = "39c1addd37a4eed612ac1135edc2cccd9d331d5e" features = [ "compat", "rand", @@ -346,6 +346,7 @@ features = [ "unstable-msc4121", "unstable-msc4125", "unstable-msc4186", + "unstable-msc4210", # remove legacy mentions "unstable-extensible-events", ] diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 7fe8addf..0fd3c91b 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -203,6 +203,7 @@ pub(super) async fn get_remote_pdu( &server, ruma::api::federation::event::get_event::v1::Request { event_id: event_id.clone().into(), + include_unredacted_content: None, }, ) .await diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 103c0c5e..de280b32 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -1,18 +1,18 @@ use axum::extract::State; -use conduit::err; +use conduit::{err, Err}; use ruma::{ api::client::{ error::ErrorKind, push::{ delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, get_pushrule_enabled, get_pushrules_all, - set_pusher, set_pushrule, set_pushrule_actions, set_pushrule_enabled, RuleScope, + set_pusher, set_pushrule, set_pushrule_actions, set_pushrule_enabled, }, }, events::{ push_rules::{PushRulesEvent, PushRulesEventContent}, GlobalAccountDataEventType, }, - push::{InsertPushRuleError, RemovePushRuleError, Ruleset}, + push::{InsertPushRuleError, PredefinedContentRuleId, PredefinedOverrideRuleId, RemovePushRuleError, Ruleset}, CanonicalJsonObject, CanonicalJsonValue, }; use service::Services; @@ -43,7 +43,24 @@ pub(crate) async fn get_pushrules_all_route( let account_data_content = serde_json::from_value::(content_value.into()) .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; - let global_ruleset: Ruleset = account_data_content.global; + let mut global_ruleset = account_data_content.global; + + // remove old deprecated mentions push rules as per MSC4210 + #[allow(deprecated)] + { + use ruma::push::RuleKind::*; + + global_ruleset + .remove(Override, PredefinedOverrideRuleId::ContainsDisplayName) + .ok(); + global_ruleset + .remove(Override, PredefinedOverrideRuleId::RoomNotif) + .ok(); + + global_ruleset + .remove(Content, PredefinedContentRuleId::ContainsUserName) + .ok(); + }; Ok(get_pushrules_all::v3::Response { global: global_ruleset, @@ -58,6 +75,15 @@ pub(crate) async fn get_pushrule_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + // remove old deprecated mentions push rules as per MSC4210 + #[allow(deprecated)] + if body.rule_id.as_str() == PredefinedContentRuleId::ContainsUserName.as_str() + || body.rule_id.as_str() == PredefinedOverrideRuleId::ContainsDisplayName.as_str() + || body.rule_id.as_str() == PredefinedOverrideRuleId::RoomNotif.as_str() + { + return Err!(Request(NotFound("Push rule not found."))); + } + let event: PushRulesEvent = services .account_data .get_global(sender_user, GlobalAccountDataEventType::PushRules) @@ -79,7 +105,7 @@ pub(crate) async fn get_pushrule_route( } } -/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}` +/// # `PUT /_matrix/client/r0/pushrules/global/{kind}/{ruleId}` /// /// Creates a single specified push rule for this user. pub(crate) async fn set_pushrule_route( @@ -88,13 +114,6 @@ pub(crate) async fn set_pushrule_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let body = body.body; - if body.scope != RuleScope::Global { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Scopes other than 'global' are not supported.", - )); - } - let mut account_data: PushRulesEvent = services .account_data .get_global(sender_user, GlobalAccountDataEventType::PushRules) @@ -145,7 +164,7 @@ pub(crate) async fn set_pushrule_route( Ok(set_pushrule::v3::Response {}) } -/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/actions` +/// # `GET /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/actions` /// /// Gets the actions of a single specified push rule for this user. pub(crate) async fn get_pushrule_actions_route( @@ -153,11 +172,13 @@ pub(crate) async fn get_pushrule_actions_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if body.scope != RuleScope::Global { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Scopes other than 'global' are not supported.", - )); + // remove old deprecated mentions push rules as per MSC4210 + #[allow(deprecated)] + if body.rule_id.as_str() == PredefinedContentRuleId::ContainsUserName.as_str() + || body.rule_id.as_str() == PredefinedOverrideRuleId::ContainsDisplayName.as_str() + || body.rule_id.as_str() == PredefinedOverrideRuleId::RoomNotif.as_str() + { + return Err!(Request(NotFound("Push rule not found."))); } let event: PushRulesEvent = services @@ -178,7 +199,7 @@ pub(crate) async fn get_pushrule_actions_route( }) } -/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/actions` +/// # `PUT /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/actions` /// /// Sets the actions of a single specified push rule for this user. pub(crate) async fn set_pushrule_actions_route( @@ -186,13 +207,6 @@ pub(crate) async fn set_pushrule_actions_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if body.scope != RuleScope::Global { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Scopes other than 'global' are not supported.", - )); - } - let mut account_data: PushRulesEvent = services .account_data .get_global(sender_user, GlobalAccountDataEventType::PushRules) @@ -221,7 +235,7 @@ pub(crate) async fn set_pushrule_actions_route( Ok(set_pushrule_actions::v3::Response {}) } -/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/enabled` +/// # `GET /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/enabled` /// /// Gets the enabled status of a single specified push rule for this user. pub(crate) async fn get_pushrule_enabled_route( @@ -229,11 +243,15 @@ pub(crate) async fn get_pushrule_enabled_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if body.scope != RuleScope::Global { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Scopes other than 'global' are not supported.", - )); + // remove old deprecated mentions push rules as per MSC4210 + #[allow(deprecated)] + if body.rule_id.as_str() == PredefinedContentRuleId::ContainsUserName.as_str() + || body.rule_id.as_str() == PredefinedOverrideRuleId::ContainsDisplayName.as_str() + || body.rule_id.as_str() == PredefinedOverrideRuleId::RoomNotif.as_str() + { + return Ok(get_pushrule_enabled::v3::Response { + enabled: false, + }); } let event: PushRulesEvent = services @@ -254,7 +272,7 @@ pub(crate) async fn get_pushrule_enabled_route( }) } -/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/enabled` +/// # `PUT /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/enabled` /// /// Sets the enabled status of a single specified push rule for this user. pub(crate) async fn set_pushrule_enabled_route( @@ -262,13 +280,6 @@ pub(crate) async fn set_pushrule_enabled_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if body.scope != RuleScope::Global { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Scopes other than 'global' are not supported.", - )); - } - let mut account_data: PushRulesEvent = services .account_data .get_global(sender_user, GlobalAccountDataEventType::PushRules) @@ -297,7 +308,7 @@ pub(crate) async fn set_pushrule_enabled_route( Ok(set_pushrule_enabled::v3::Response {}) } -/// # `DELETE /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}` +/// # `DELETE /_matrix/client/r0/pushrules/global/{kind}/{ruleId}` /// /// Deletes a single specified push rule for this user. pub(crate) async fn delete_pushrule_route( @@ -305,13 +316,6 @@ pub(crate) async fn delete_pushrule_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if body.scope != RuleScope::Global { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Scopes other than 'global' are not supported.", - )); - } - let mut account_data: PushRulesEvent = services .account_data .get_global(sender_user, GlobalAccountDataEventType::PushRules) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 0b2bbf73..026c5a4c 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1159,6 +1159,7 @@ impl Service { origin, get_event::v1::Request { event_id: (*next_id).to_owned(), + include_unredacted_content: None, }, ) .await From b921983a795f042dac0f348f1a832c73bd44de7f Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 26 Oct 2024 17:39:27 -0400 Subject: [PATCH 0950/2091] send room alias on pusher notification Signed-off-by: strawberry --- src/service/pusher/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index af15e332..2b90319e 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -332,6 +332,13 @@ impl Service { .await .ok(); + notifi.room_alias = self + .services + .state_accessor + .get_canonical_alias(&event.room_id) + .await + .ok(); + self.send_request(&http.url, send_event_notification::v1::Request::new(notifi)) .await?; } From 49343281d477cf414cfec737a00c150d5db34ba3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 24 Oct 2024 06:01:53 +0000 Subject: [PATCH 0951/2091] additional bool extensions Signed-off-by: Jason Volk --- src/core/utils/bool.rs | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/core/utils/bool.rs b/src/core/utils/bool.rs index d7ce78fe..d5fa85aa 100644 --- a/src/core/utils/bool.rs +++ b/src/core/utils/bool.rs @@ -2,12 +2,49 @@ /// Boolean extensions and chain.starters pub trait BoolExt { + fn map T>(self, f: F) -> T + where + Self: Sized; + + fn map_ok_or T>(self, err: E, f: F) -> Result; + + fn map_or T>(self, err: T, f: F) -> T; + + fn map_or_else T>(self, err: F, f: F) -> T; + + fn ok_or(self, err: E) -> Result<(), E>; + + fn ok_or_else E>(self, err: F) -> Result<(), E>; + fn or T>(self, f: F) -> Option; fn or_some(self, t: T) -> Option; } impl BoolExt for bool { + #[inline] + fn map T>(self, f: F) -> T + where + Self: Sized, + { + f(self) + } + + #[inline] + fn map_ok_or T>(self, err: E, f: F) -> Result { self.ok_or(err).map(|()| f()) } + + #[inline] + fn map_or T>(self, err: T, f: F) -> T { self.then(f).unwrap_or(err) } + + #[inline] + fn map_or_else T>(self, err: F, f: F) -> T { self.then(f).unwrap_or_else(err) } + + #[inline] + fn ok_or(self, err: E) -> Result<(), E> { self.then_some(()).ok_or(err) } + + #[inline] + fn ok_or_else E>(self, err: F) -> Result<(), E> { self.then_some(()).ok_or_else(err) } + #[inline] fn or T>(self, f: F) -> Option { (!self).then(f) } From efb28c1a9944840143af37bff65bd475f38df717 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 24 Oct 2024 06:03:45 +0000 Subject: [PATCH 0952/2091] add a Map::contains suite to db Signed-off-by: Jason Volk --- src/database/map.rs | 1 + src/database/map/contains.rs | 88 ++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 src/database/map/contains.rs diff --git a/src/database/map.rs b/src/database/map.rs index cac20d6a..d6b8bf38 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -1,3 +1,4 @@ +mod contains; mod count; mod get; mod insert; diff --git a/src/database/map/contains.rs b/src/database/map/contains.rs new file mode 100644 index 00000000..a98fe7c5 --- /dev/null +++ b/src/database/map/contains.rs @@ -0,0 +1,88 @@ +use std::{convert::AsRef, fmt::Debug, future::Future, io::Write}; + +use arrayvec::ArrayVec; +use conduit::{implement, utils::TryFutureExtExt, Err, Result}; +use futures::future::ready; +use serde::Serialize; + +use crate::{ser, util}; + +/// Returns true if the map contains the key. +/// - key is serialized into allocated buffer +/// - harder errors may not be reported +#[implement(super::Map)] +pub fn contains(&self, key: &K) -> impl Future + Send +where + K: Serialize + ?Sized + Debug, +{ + let mut buf = Vec::::with_capacity(64); + self.bcontains(key, &mut buf) +} + +/// Returns true if the map contains the key. +/// - key is serialized into stack-buffer +/// - harder errors will panic +#[implement(super::Map)] +pub fn acontains(&self, key: &K) -> impl Future + Send +where + K: Serialize + ?Sized + Debug, +{ + let mut buf = ArrayVec::::new(); + self.bcontains(key, &mut buf) +} + +/// Returns true if the map contains the key. +/// - key is serialized into provided buffer +/// - harder errors will panic +#[implement(super::Map)] +#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] +pub fn bcontains(&self, key: &K, buf: &mut B) -> impl Future + Send +where + K: Serialize + ?Sized + Debug, + B: Write + AsRef<[u8]>, +{ + let key = ser::serialize(buf, key).expect("failed to serialize query key"); + self.exists(key).is_ok() +} + +/// Returns Ok if the map contains the key. +/// - key is raw +#[implement(super::Map)] +pub fn exists(&self, key: &K) -> impl Future> + Send +where + K: AsRef<[u8]> + ?Sized + Debug, +{ + ready(self.exists_blocking(key)) +} + +/// Returns Ok if the map contains the key; NotFound otherwise. Harder errors +/// may not always be reported properly. +#[implement(super::Map)] +#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] +pub fn exists_blocking(&self, key: &K) -> Result<()> +where + K: AsRef<[u8]> + ?Sized + Debug, +{ + if self.maybe_exists_blocking(key) + && self + .db + .db + .get_pinned_cf_opt(&self.cf(), key, &self.read_options) + .map_err(util::map_err)? + .is_some() + { + Ok(()) + } else { + Err!(Request(NotFound("Not found in database"))) + } +} + +#[implement(super::Map)] +fn maybe_exists_blocking(&self, key: &K) -> bool +where + K: AsRef<[u8]> + ?Sized, +{ + self.db + .db + .key_may_exist_cf_opt(&self.cf(), key, &self.read_options) +} From 9438dc89e612ada5e5e44b48315877055498313b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 24 Oct 2024 10:58:08 +0000 Subject: [PATCH 0953/2091] merge and resplit/cleanup appservice service Signed-off-by: Jason Volk --- src/admin/query/appservice.rs | 6 +- src/service/appservice/data.rs | 50 ------ src/service/appservice/mod.rs | 173 ++++++-------------- src/service/appservice/namespace_regex.rs | 70 ++++++++ src/service/appservice/registration_info.rs | 39 +++++ 5 files changed, 157 insertions(+), 181 deletions(-) delete mode 100644 src/service/appservice/data.rs create mode 100644 src/service/appservice/namespace_regex.rs create mode 100644 src/service/appservice/registration_info.rs diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 4b97ef4e..02e89e7a 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -26,11 +26,7 @@ pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_> appservice_id, } => { let timer = tokio::time::Instant::now(); - let results = services - .appservice - .db - .get_registration(appservice_id.as_ref()) - .await; + let results = services.appservice.get_registration(&appservice_id).await; let query_time = timer.elapsed(); diff --git a/src/service/appservice/data.rs b/src/service/appservice/data.rs deleted file mode 100644 index 8fb7d958..00000000 --- a/src/service/appservice/data.rs +++ /dev/null @@ -1,50 +0,0 @@ -use std::sync::Arc; - -use conduit::{err, utils::stream::TryIgnore, Result}; -use database::{Database, Map}; -use futures::Stream; -use ruma::api::appservice::Registration; - -pub struct Data { - id_appserviceregistrations: Arc, -} - -impl Data { - pub(super) fn new(db: &Arc) -> Self { - Self { - id_appserviceregistrations: db["id_appserviceregistrations"].clone(), - } - } - - /// Registers an appservice and returns the ID to the caller - pub(super) fn register_appservice(&self, yaml: &Registration) -> Result { - let id = yaml.id.as_str(); - self.id_appserviceregistrations - .insert(id.as_bytes(), serde_yaml::to_string(&yaml).unwrap().as_bytes()); - - Ok(id.to_owned()) - } - - /// Remove an appservice registration - /// - /// # Arguments - /// - /// * `service_name` - the name you send to register the service previously - pub(super) fn unregister_appservice(&self, service_name: &str) -> Result<()> { - self.id_appserviceregistrations - .remove(service_name.as_bytes()); - Ok(()) - } - - pub async fn get_registration(&self, id: &str) -> Result { - self.id_appserviceregistrations - .get(id) - .await - .and_then(|ref bytes| serde_yaml::from_slice(bytes).map_err(Into::into)) - .map_err(|e| err!(Database("Invalid appservice {id:?} registration: {e:?}"))) - } - - pub(super) fn iter_ids(&self) -> impl Stream + Send + '_ { - self.id_appserviceregistrations.keys().ignore_err() - } -} diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 7e2dc738..1617e6e6 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -1,147 +1,49 @@ -mod data; +mod namespace_regex; +mod registration_info; use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; -use conduit::{err, Result}; -use data::Data; +use conduit::{err, utils::stream::TryIgnore, Result}; +use database::Map; use futures::{Future, StreamExt, TryStreamExt}; -use regex::RegexSet; -use ruma::{ - api::appservice::{Namespace, Registration}, - RoomAliasId, RoomId, UserId, -}; +use ruma::{api::appservice::Registration, RoomAliasId, RoomId, UserId}; use tokio::sync::RwLock; +pub use self::{namespace_regex::NamespaceRegex, registration_info::RegistrationInfo}; use crate::{sending, Dep}; -/// Compiled regular expressions for a namespace -#[derive(Clone, Debug)] -pub struct NamespaceRegex { - pub exclusive: Option, - pub non_exclusive: Option, -} - -impl NamespaceRegex { - /// Checks if this namespace has rights to a namespace - #[inline] - #[must_use] - pub fn is_match(&self, heystack: &str) -> bool { - if self.is_exclusive_match(heystack) { - return true; - } - - if let Some(non_exclusive) = &self.non_exclusive { - if non_exclusive.is_match(heystack) { - return true; - } - } - false - } - - /// Checks if this namespace has exlusive rights to a namespace - #[inline] - #[must_use] - pub fn is_exclusive_match(&self, heystack: &str) -> bool { - if let Some(exclusive) = &self.exclusive { - if exclusive.is_match(heystack) { - return true; - } - } - false - } -} - -impl RegistrationInfo { - #[must_use] - pub fn is_user_match(&self, user_id: &UserId) -> bool { - self.users.is_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() - } - - #[inline] - #[must_use] - pub fn is_exclusive_user_match(&self, user_id: &UserId) -> bool { - self.users.is_exclusive_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() - } -} - -impl TryFrom> for NamespaceRegex { - type Error = regex::Error; - - fn try_from(value: Vec) -> Result { - let mut exclusive = Vec::with_capacity(value.len()); - let mut non_exclusive = Vec::with_capacity(value.len()); - - for namespace in value { - if namespace.exclusive { - exclusive.push(namespace.regex); - } else { - non_exclusive.push(namespace.regex); - } - } - - Ok(Self { - exclusive: if exclusive.is_empty() { - None - } else { - Some(RegexSet::new(exclusive)?) - }, - non_exclusive: if non_exclusive.is_empty() { - None - } else { - Some(RegexSet::new(non_exclusive)?) - }, - }) - } -} - -/// Appservice registration combined with its compiled regular expressions. -#[derive(Clone, Debug)] -pub struct RegistrationInfo { - pub registration: Registration, - pub users: NamespaceRegex, - pub aliases: NamespaceRegex, - pub rooms: NamespaceRegex, -} - -impl TryFrom for RegistrationInfo { - type Error = regex::Error; - - fn try_from(value: Registration) -> Result { - Ok(Self { - users: value.namespaces.users.clone().try_into()?, - aliases: value.namespaces.aliases.clone().try_into()?, - rooms: value.namespaces.rooms.clone().try_into()?, - registration: value, - }) - } -} - pub struct Service { - pub db: Data, - services: Services, registration_info: RwLock>, + services: Services, + db: Data, } struct Services { sending: Dep, } +struct Data { + id_appserviceregistrations: Arc, +} + #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(args.db), + registration_info: RwLock::new(BTreeMap::new()), services: Services { sending: args.depend::("sending"), }, - registration_info: RwLock::new(BTreeMap::new()), + db: Data { + id_appserviceregistrations: args.db["id_appserviceregistrations"].clone(), + }, })) } async fn worker(self: Arc) -> Result<()> { // Inserting registrations into cache - for appservice in iter_ids(&self.db).await? { + for appservice in self.iter_db_ids().await? { self.registration_info.write().await.insert( appservice.0, appservice @@ -158,9 +60,6 @@ impl crate::Service for Service { } impl Service { - #[inline] - pub async fn all(&self) -> Result> { iter_ids(&self.db).await } - /// Registers an appservice and returns the ID to the caller pub async fn register_appservice(&self, yaml: Registration) -> Result { //TODO: Check for collisions between exclusive appservice namespaces @@ -169,7 +68,11 @@ impl Service { .await .insert(yaml.id.clone(), yaml.clone().try_into()?); - self.db.register_appservice(&yaml) + let id = yaml.id.as_str(); + let yaml = serde_yaml::to_string(&yaml)?; + self.db.id_appserviceregistrations.insert(id, yaml); + + Ok(id.to_owned()) } /// Remove an appservice registration @@ -186,7 +89,7 @@ impl Service { .ok_or(err!("Appservice not found"))?; // remove the appservice from the database - self.db.unregister_appservice(service_name)?; + self.db.id_appserviceregistrations.remove(service_name); // deletes all active requests for the appservice if there are any so we stop // sending to the URL @@ -254,11 +157,29 @@ impl Service { pub fn read(&self) -> impl Future>> { self.registration_info.read() } -} -async fn iter_ids(db: &Data) -> Result> { - db.iter_ids() - .then(|id| async move { Ok((id.clone(), db.get_registration(&id).await?)) }) - .try_collect() - .await + #[inline] + pub async fn all(&self) -> Result> { self.iter_db_ids().await } + + pub async fn get_db_registration(&self, id: &str) -> Result { + self.db + .id_appserviceregistrations + .get(id) + .await + .and_then(|ref bytes| serde_yaml::from_slice(bytes).map_err(Into::into)) + .map_err(|e| err!(Database("Invalid appservice {id:?} registration: {e:?}"))) + } + + async fn iter_db_ids(&self) -> Result> { + self.db + .id_appserviceregistrations + .keys() + .ignore_err() + .then(|id: String| async move { + let reg = self.get_db_registration(&id).await?; + Ok((id, reg)) + }) + .try_collect() + .await + } } diff --git a/src/service/appservice/namespace_regex.rs b/src/service/appservice/namespace_regex.rs new file mode 100644 index 00000000..3529fc0e --- /dev/null +++ b/src/service/appservice/namespace_regex.rs @@ -0,0 +1,70 @@ +use conduit::Result; +use regex::RegexSet; +use ruma::api::appservice::Namespace; + +/// Compiled regular expressions for a namespace +#[derive(Clone, Debug)] +pub struct NamespaceRegex { + pub exclusive: Option, + pub non_exclusive: Option, +} + +impl NamespaceRegex { + /// Checks if this namespace has rights to a namespace + #[inline] + #[must_use] + pub fn is_match(&self, heystack: &str) -> bool { + if self.is_exclusive_match(heystack) { + return true; + } + + if let Some(non_exclusive) = &self.non_exclusive { + if non_exclusive.is_match(heystack) { + return true; + } + } + false + } + + /// Checks if this namespace has exlusive rights to a namespace + #[inline] + #[must_use] + pub fn is_exclusive_match(&self, heystack: &str) -> bool { + if let Some(exclusive) = &self.exclusive { + if exclusive.is_match(heystack) { + return true; + } + } + false + } +} + +impl TryFrom> for NamespaceRegex { + type Error = regex::Error; + + fn try_from(value: Vec) -> Result { + let mut exclusive = Vec::with_capacity(value.len()); + let mut non_exclusive = Vec::with_capacity(value.len()); + + for namespace in value { + if namespace.exclusive { + exclusive.push(namespace.regex); + } else { + non_exclusive.push(namespace.regex); + } + } + + Ok(Self { + exclusive: if exclusive.is_empty() { + None + } else { + Some(RegexSet::new(exclusive)?) + }, + non_exclusive: if non_exclusive.is_empty() { + None + } else { + Some(RegexSet::new(non_exclusive)?) + }, + }) + } +} diff --git a/src/service/appservice/registration_info.rs b/src/service/appservice/registration_info.rs new file mode 100644 index 00000000..2c8595b1 --- /dev/null +++ b/src/service/appservice/registration_info.rs @@ -0,0 +1,39 @@ +use conduit::Result; +use ruma::{api::appservice::Registration, UserId}; + +use super::NamespaceRegex; + +/// Appservice registration combined with its compiled regular expressions. +#[derive(Clone, Debug)] +pub struct RegistrationInfo { + pub registration: Registration, + pub users: NamespaceRegex, + pub aliases: NamespaceRegex, + pub rooms: NamespaceRegex, +} + +impl RegistrationInfo { + #[must_use] + pub fn is_user_match(&self, user_id: &UserId) -> bool { + self.users.is_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() + } + + #[inline] + #[must_use] + pub fn is_exclusive_user_match(&self, user_id: &UserId) -> bool { + self.users.is_exclusive_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() + } +} + +impl TryFrom for RegistrationInfo { + type Error = regex::Error; + + fn try_from(value: Registration) -> Result { + Ok(Self { + users: value.namespaces.users.clone().try_into()?, + aliases: value.namespaces.aliases.clone().try_into()?, + rooms: value.namespaces.rooms.clone().try_into()?, + registration: value, + }) + } +} From 0e616f1d1267481ed97e9adc6d779d29fcf9ade2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 24 Oct 2024 11:16:44 +0000 Subject: [PATCH 0954/2091] add event macro log wrapper suite Signed-off-by: Jason Volk --- src/core/log/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 04d250a6..1cba236f 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -27,6 +27,11 @@ pub struct Log { // necessary but discouraged. Remember debug_ log macros are also exported to // the crate namespace like these. +#[macro_export] +macro_rules! event { + ( $level:expr, $($x:tt)+ ) => { ::tracing::event!( $level, $($x)+ ) } +} + #[macro_export] macro_rules! error { ( $($x:tt)+ ) => { ::tracing::error!( $($x)+ ) } From e175b7d28dffbea663c25e66babd3184e7fc5b1f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 24 Oct 2024 11:24:03 +0000 Subject: [PATCH 0955/2091] slightly cleanup prev_event eval loop Signed-off-by: Jason Volk --- src/service/rooms/event_handler/mod.rs | 41 +++++++++++++------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 026c5a4c..ec04e748 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -206,7 +206,7 @@ impl Service { debug!(events = ?sorted_prev_events, "Got previous events"); for prev_id in sorted_prev_events { self.services.server.check_running()?; - match self + if let Err(e) = self .handle_prev_pdu( origin, event_id, @@ -218,25 +218,26 @@ impl Service { ) .await { - Ok(()) => continue, - Err(e) => { - warn!("Prev event {prev_id} failed: {e}"); - match self - .services - .globals - .bad_event_ratelimiter - .write() - .expect("locked") - .entry((*prev_id).to_owned()) - { - hash_map::Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - hash_map::Entry::Occupied(mut e) => { - *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); - }, - }; - }, + use hash_map::Entry; + + let now = Instant::now(); + warn!("Prev event {prev_id} failed: {e}"); + + match self + .services + .globals + .bad_event_ratelimiter + .write() + .expect("locked") + .entry(prev_id.into()) + { + Entry::Vacant(e) => { + e.insert((now, 1)); + }, + Entry::Occupied(mut e) => { + *e.get_mut() = (now, e.get().1.saturating_add(1)); + }, + }; } } From 60cc07134f3d80f0ba25d4bc1b6736c30494f947 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 24 Oct 2024 11:24:37 +0000 Subject: [PATCH 0956/2091] log error for auth_chain corruption immediately Signed-off-by: Jason Volk --- src/service/rooms/auth_chain/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index f3861ca3..1387bc7d 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -167,10 +167,12 @@ impl Service { Err(e) => debug_error!(?event_id, ?e, "Could not find pdu mentioned in auth events"), Ok(pdu) => { if pdu.room_id != room_id { - return Err!(Request(Forbidden( - "auth event {event_id:?} for incorrect room {} which is not {room_id}", - pdu.room_id, - ))); + return Err!(Request(Forbidden(error!( + ?event_id, + ?room_id, + wrong_room_id = ?pdu.room_id, + "auth event for incorrect room" + )))); } for auth_event in &pdu.auth_events { From ee92a33a4de8db924ee5e203f5b3c64dade8dcc6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 24 Oct 2024 12:03:56 +0000 Subject: [PATCH 0957/2091] add some accessors to Ar for common patterns Signed-off-by: Jason Volk --- src/api/client/backup.rs | 107 ++++++++++++--------------- src/api/router/args.rs | 18 +++-- src/api/router/handler.rs | 2 +- src/api/server/backfill.rs | 7 +- src/api/server/event.rs | 6 +- src/api/server/event_auth.rs | 6 +- src/api/server/get_missing_events.rs | 8 +- src/api/server/hierarchy.rs | 4 +- src/api/server/invite.rs | 9 +-- src/api/server/make_join.rs | 13 ++-- src/api/server/make_leave.rs | 5 +- src/api/server/send.rs | 12 ++- src/api/server/send_join.rs | 19 ++--- src/api/server/send_leave.rs | 22 +----- src/api/server/state.rs | 6 +- src/api/server/state_ids.rs | 6 +- src/api/server/user.rs | 6 +- 17 files changed, 109 insertions(+), 147 deletions(-) diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index d52da80a..f435e086 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -18,10 +18,9 @@ use crate::{Result, Ruma}; pub(crate) async fn create_backup_version_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let version = services .key_backups - .create_backup(sender_user, &body.algorithm)?; + .create_backup(body.sender_user(), &body.algorithm)?; Ok(create_backup_version::v3::Response { version, @@ -35,10 +34,9 @@ pub(crate) async fn create_backup_version_route( pub(crate) async fn update_backup_version_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); services .key_backups - .update_backup(sender_user, &body.version, &body.algorithm) + .update_backup(body.sender_user(), &body.version, &body.algorithm) .await?; Ok(update_backup_version::v3::Response {}) @@ -50,19 +48,25 @@ pub(crate) async fn update_backup_version_route( pub(crate) async fn get_latest_backup_info_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let (version, algorithm) = services .key_backups - .get_latest_backup(sender_user) + .get_latest_backup(body.sender_user()) .await .map_err(|_| err!(Request(NotFound("Key backup does not exist."))))?; Ok(get_latest_backup_info::v3::Response { algorithm, - count: (UInt::try_from(services.key_backups.count_keys(sender_user, &version).await) - .expect("user backup keys count should not be that high")), - etag: services.key_backups.get_etag(sender_user, &version).await, + count: (UInt::try_from( + services + .key_backups + .count_keys(body.sender_user(), &version) + .await, + ) + .expect("user backup keys count should not be that high")), + etag: services + .key_backups + .get_etag(body.sender_user(), &version) + .await, version, }) } @@ -73,10 +77,9 @@ pub(crate) async fn get_latest_backup_info_route( pub(crate) async fn get_backup_info_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let algorithm = services .key_backups - .get_backup(sender_user, &body.version) + .get_backup(body.sender_user(), &body.version) .await .map_err(|_| err!(Request(NotFound("Key backup does not exist at version {:?}", body.version))))?; @@ -84,12 +87,12 @@ pub(crate) async fn get_backup_info_route( algorithm, count: services .key_backups - .count_keys(sender_user, &body.version) + .count_keys(body.sender_user(), &body.version) .await .try_into()?, etag: services .key_backups - .get_etag(sender_user, &body.version) + .get_etag(body.sender_user(), &body.version) .await, version: body.version.clone(), }) @@ -104,11 +107,9 @@ pub(crate) async fn get_backup_info_route( pub(crate) async fn delete_backup_version_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services .key_backups - .delete_backup(sender_user, &body.version) + .delete_backup(body.sender_user(), &body.version) .await; Ok(delete_backup_version::v3::Response {}) @@ -125,11 +126,9 @@ pub(crate) async fn delete_backup_version_route( pub(crate) async fn add_backup_keys_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if services .key_backups - .get_latest_backup_version(sender_user) + .get_latest_backup_version(body.sender_user()) .await .is_ok_and(|version| version != body.version) { @@ -142,7 +141,7 @@ pub(crate) async fn add_backup_keys_route( for (session_id, key_data) in &room.sessions { services .key_backups - .add_key(sender_user, &body.version, room_id, session_id, key_data) + .add_key(body.sender_user(), &body.version, room_id, session_id, key_data) .await?; } } @@ -150,12 +149,12 @@ pub(crate) async fn add_backup_keys_route( Ok(add_backup_keys::v3::Response { count: services .key_backups - .count_keys(sender_user, &body.version) + .count_keys(body.sender_user(), &body.version) .await .try_into()?, etag: services .key_backups - .get_etag(sender_user, &body.version) + .get_etag(body.sender_user(), &body.version) .await, }) } @@ -171,11 +170,9 @@ pub(crate) async fn add_backup_keys_route( pub(crate) async fn add_backup_keys_for_room_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if services .key_backups - .get_latest_backup_version(sender_user) + .get_latest_backup_version(body.sender_user()) .await .is_ok_and(|version| version != body.version) { @@ -187,19 +184,19 @@ pub(crate) async fn add_backup_keys_for_room_route( for (session_id, key_data) in &body.sessions { services .key_backups - .add_key(sender_user, &body.version, &body.room_id, session_id, key_data) + .add_key(body.sender_user(), &body.version, &body.room_id, session_id, key_data) .await?; } Ok(add_backup_keys_for_room::v3::Response { count: services .key_backups - .count_keys(sender_user, &body.version) + .count_keys(body.sender_user(), &body.version) .await .try_into()?, etag: services .key_backups - .get_etag(sender_user, &body.version) + .get_etag(body.sender_user(), &body.version) .await, }) } @@ -215,11 +212,9 @@ pub(crate) async fn add_backup_keys_for_room_route( pub(crate) async fn add_backup_keys_for_session_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if services .key_backups - .get_latest_backup_version(sender_user) + .get_latest_backup_version(body.sender_user()) .await .is_ok_and(|version| version != body.version) { @@ -230,18 +225,24 @@ pub(crate) async fn add_backup_keys_for_session_route( services .key_backups - .add_key(sender_user, &body.version, &body.room_id, &body.session_id, &body.session_data) + .add_key( + body.sender_user(), + &body.version, + &body.room_id, + &body.session_id, + &body.session_data, + ) .await?; Ok(add_backup_keys_for_session::v3::Response { count: services .key_backups - .count_keys(sender_user, &body.version) + .count_keys(body.sender_user(), &body.version) .await .try_into()?, etag: services .key_backups - .get_etag(sender_user, &body.version) + .get_etag(body.sender_user(), &body.version) .await, }) } @@ -252,11 +253,9 @@ pub(crate) async fn add_backup_keys_for_session_route( pub(crate) async fn get_backup_keys_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let rooms = services .key_backups - .get_all(sender_user, &body.version) + .get_all(body.sender_user(), &body.version) .await; Ok(get_backup_keys::v3::Response { @@ -270,11 +269,9 @@ pub(crate) async fn get_backup_keys_route( pub(crate) async fn get_backup_keys_for_room_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sessions = services .key_backups - .get_room(sender_user, &body.version, &body.room_id) + .get_room(body.sender_user(), &body.version, &body.room_id) .await; Ok(get_backup_keys_for_room::v3::Response { @@ -288,11 +285,9 @@ pub(crate) async fn get_backup_keys_for_room_route( pub(crate) async fn get_backup_keys_for_session_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let key_data = services .key_backups - .get_session(sender_user, &body.version, &body.room_id, &body.session_id) + .get_session(body.sender_user(), &body.version, &body.room_id, &body.session_id) .await .map_err(|_| err!(Request(NotFound(debug_error!("Backup key not found for this user's session.")))))?; @@ -307,22 +302,20 @@ pub(crate) async fn get_backup_keys_for_session_route( pub(crate) async fn delete_backup_keys_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services .key_backups - .delete_all_keys(sender_user, &body.version) + .delete_all_keys(body.sender_user(), &body.version) .await; Ok(delete_backup_keys::v3::Response { count: services .key_backups - .count_keys(sender_user, &body.version) + .count_keys(body.sender_user(), &body.version) .await .try_into()?, etag: services .key_backups - .get_etag(sender_user, &body.version) + .get_etag(body.sender_user(), &body.version) .await, }) } @@ -333,22 +326,20 @@ pub(crate) async fn delete_backup_keys_route( pub(crate) async fn delete_backup_keys_for_room_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services .key_backups - .delete_room_keys(sender_user, &body.version, &body.room_id) + .delete_room_keys(body.sender_user(), &body.version, &body.room_id) .await; Ok(delete_backup_keys_for_room::v3::Response { count: services .key_backups - .count_keys(sender_user, &body.version) + .count_keys(body.sender_user(), &body.version) .await .try_into()?, etag: services .key_backups - .get_etag(sender_user, &body.version) + .get_etag(body.sender_user(), &body.version) .await, }) } @@ -359,22 +350,20 @@ pub(crate) async fn delete_backup_keys_for_room_route( pub(crate) async fn delete_backup_keys_for_session_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services .key_backups - .delete_room_key(sender_user, &body.version, &body.room_id, &body.session_id) + .delete_room_key(body.sender_user(), &body.version, &body.room_id, &body.session_id) .await; Ok(delete_backup_keys_for_session::v3::Response { count: services .key_backups - .count_keys(sender_user, &body.version) + .count_keys(body.sender_user(), &body.version) .await .try_into()?, etag: services .key_backups - .get_etag(sender_user, &body.version) + .get_etag(body.sender_user(), &body.version) .await, }) } diff --git a/src/api/router/args.rs b/src/api/router/args.rs index 746e1cfc..cefacac1 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -3,17 +3,14 @@ use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, BytesMut}; use conduit::{debug, err, trace, utils::string::EMPTY, Error, Result}; -use ruma::{api::IncomingRequest, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId}; +use ruma::{api::IncomingRequest, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, ServerName, UserId}; use service::Services; use super::{auth, auth::Auth, request, request::Request}; use crate::{service::appservice::RegistrationInfo, State}; /// Extractor for Ruma request structs -pub(crate) struct Args -where - T: IncomingRequest + Send + Sync + 'static, -{ +pub(crate) struct Args { /// Request struct body pub(crate) body: T, @@ -38,6 +35,17 @@ where pub(crate) json_body: Option, } +impl Args +where + T: IncomingRequest + Send + Sync + 'static, +{ + #[inline] + pub(crate) fn sender_user(&self) -> &UserId { self.sender_user.as_deref().expect("user is authenticated") } + + #[inline] + pub(crate) fn origin(&self) -> &ServerName { self.origin.as_deref().expect("server is authenticated") } +} + #[async_trait] impl FromRequest for Args where diff --git a/src/api/router/handler.rs b/src/api/router/handler.rs index 3b7b1eeb..0022f06a 100644 --- a/src/api/router/handler.rs +++ b/src/api/router/handler.rs @@ -38,7 +38,7 @@ macro_rules! ruma_handler { where Fun: Fn($($tx,)* Ruma,) -> Fut + Send + Sync + 'static, Fut: Future> + Send, - Req: IncomingRequest + Send + Sync, + Req: IncomingRequest + Send + Sync + 'static, Err: IntoResponse + Send, ::OutgoingResponse: Send, $( $tx: FromRequestParts + Send + Sync + 'static, )* diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 2bbc95ca..088b891a 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -18,12 +18,10 @@ use crate::Ruma; pub(crate) async fn get_backfill_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - services .rooms .event_handler - .acl_check(origin, &body.room_id) + .acl_check(body.origin(), &body.room_id) .await?; if !services @@ -33,7 +31,7 @@ pub(crate) async fn get_backfill_route( .await && !services .rooms .state_cache - .server_in_room(origin, &body.room_id) + .server_in_room(body.origin(), &body.room_id) .await { return Err!(Request(Forbidden("Server is not in room."))); @@ -59,6 +57,7 @@ pub(crate) async fn get_backfill_route( .try_into() .expect("UInt could not be converted to usize"); + let origin = body.origin(); let pdus = services .rooms .timeline diff --git a/src/api/server/event.rs b/src/api/server/event.rs index e4eac794..64ce3e40 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -13,8 +13,6 @@ use crate::Ruma; pub(crate) async fn get_event_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - let event = services .rooms .timeline @@ -37,7 +35,7 @@ pub(crate) async fn get_event_route( .await && !services .rooms .state_cache - .server_in_room(origin, room_id) + .server_in_room(body.origin(), room_id) .await { return Err!(Request(Forbidden("Server is not in room."))); @@ -46,7 +44,7 @@ pub(crate) async fn get_event_route( if !services .rooms .state_accessor - .server_can_see_event(origin, room_id, &body.event_id) + .server_can_see_event(body.origin(), room_id, &body.event_id) .await? { return Err!(Request(Forbidden("Server is not allowed to see event."))); diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 8307a4ad..8fe96f81 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -18,12 +18,10 @@ use crate::Ruma; pub(crate) async fn get_event_authorization_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - services .rooms .event_handler - .acl_check(origin, &body.room_id) + .acl_check(body.origin(), &body.room_id) .await?; if !services @@ -33,7 +31,7 @@ pub(crate) async fn get_event_authorization_route( .await && !services .rooms .state_cache - .server_in_room(origin, &body.room_id) + .server_in_room(body.origin(), &body.room_id) .await { return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index e267898f..aee4fbe9 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -13,12 +13,10 @@ use crate::Ruma; pub(crate) async fn get_missing_events_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - services .rooms .event_handler - .acl_check(origin, &body.room_id) + .acl_check(body.origin(), &body.room_id) .await?; if !services @@ -28,7 +26,7 @@ pub(crate) async fn get_missing_events_route( .await && !services .rooms .state_cache - .server_in_room(origin, &body.room_id) + .server_in_room(body.origin(), &body.room_id) .await { return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room")); @@ -71,7 +69,7 @@ pub(crate) async fn get_missing_events_route( if !services .rooms .state_accessor - .server_can_see_event(origin, &body.room_id, &queued_events[i]) + .server_can_see_event(body.origin(), &body.room_id, &queued_events[i]) .await? { i = i.saturating_add(1); diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs index 002bd763..e3ce7108 100644 --- a/src/api/server/hierarchy.rs +++ b/src/api/server/hierarchy.rs @@ -10,13 +10,11 @@ use crate::{Error, Result, Ruma}; pub(crate) async fn get_hierarchy_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - if services.rooms.metadata.exists(&body.room_id).await { services .rooms .spaces - .get_federation_hierarchy(&body.room_id, origin, body.suggested_only) + .get_federation_hierarchy(&body.room_id, body.origin(), body.suggested_only) .await } else { Err(Error::BadRequest(ErrorKind::NotFound, "Room does not exist.")) diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index a9e404c5..b30a1b58 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -18,13 +18,11 @@ pub(crate) async fn create_invite_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - // ACL check origin services .rooms .event_handler - .acl_check(origin, &body.room_id) + .acl_check(body.origin(), &body.room_id) .await?; if !services @@ -55,10 +53,11 @@ pub(crate) async fn create_invite_route( .globals .config .forbidden_remote_server_names - .contains(origin) + .contains(body.origin()) { warn!( - "Received federated/remote invite from banned server {origin} for room ID {}. Rejecting.", + "Received federated/remote invite from banned server {} for room ID {}. Rejecting.", + body.origin(), body.room_id ); diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index 85668038..c3524f0e 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -30,8 +30,7 @@ pub(crate) async fn create_join_event_template_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } - let origin = body.origin.as_ref().expect("server is authenticated"); - if body.user_id.server_name() != origin { + if body.user_id.server_name() != body.origin() { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Not allowed to join on behalf of another server/user", @@ -42,19 +41,21 @@ pub(crate) async fn create_join_event_template_route( services .rooms .event_handler - .acl_check(origin, &body.room_id) + .acl_check(body.origin(), &body.room_id) .await?; if services .globals .config .forbidden_remote_server_names - .contains(origin) + .contains(body.origin()) { warn!( - "Server {origin} for remote user {} tried joining room ID {} which has a server name that is globally \ + "Server {} for remote user {} tried joining room ID {} which has a server name that is globally \ forbidden. Rejecting.", - &body.user_id, &body.room_id, + body.origin(), + &body.user_id, + &body.room_id, ); return Err(Error::BadRequest( ErrorKind::forbidden(), diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 81a32c86..33a94560 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -19,8 +19,7 @@ pub(crate) async fn create_leave_event_template_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); } - let origin = body.origin.as_ref().expect("server is authenticated"); - if body.user_id.server_name() != origin { + if body.user_id.server_name() != body.origin() { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Not allowed to leave on behalf of another server/user", @@ -31,7 +30,7 @@ pub(crate) async fn create_leave_event_template_route( services .rooms .event_handler - .acl_check(origin, &body.room_id) + .acl_check(body.origin(), &body.room_id) .await?; let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; diff --git a/src/api/server/send.rs b/src/api/server/send.rs index d5d3ffbb..2da99c93 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -41,9 +41,7 @@ pub(crate) async fn send_transaction_message_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - if *origin != body.body.origin { + if body.origin() != body.body.origin { return Err!(Request(Forbidden( "Not allowed to send transactions on behalf of other servers" ))); @@ -67,19 +65,19 @@ pub(crate) async fn send_transaction_message_route( edus = ?body.edus.len(), elapsed = ?txn_start_time.elapsed(), id = ?body.transaction_id, - origin =?body.origin, + origin =?body.origin(), "Starting txn", ); - let resolved_map = handle_pdus(&services, &client, &body.pdus, origin, &txn_start_time).await?; - handle_edus(&services, &client, &body.edus, origin).await; + let resolved_map = handle_pdus(&services, &client, &body.pdus, body.origin(), &txn_start_time).await?; + handle_edus(&services, &client, &body.edus, body.origin()).await; debug!( pdus = ?body.pdus.len(), edus = ?body.edus.len(), elapsed = ?txn_start_time.elapsed(), id = ?body.transaction_id, - origin =?body.origin, + origin =?body.origin(), "Finished txn", ); diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index d888d75e..c3273baf 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -217,16 +217,15 @@ async fn create_join_event( pub(crate) async fn create_join_event_v1_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - if services .globals .config .forbidden_remote_server_names - .contains(origin) + .contains(body.origin()) { warn!( - "Server {origin} tried joining room ID {} who has a server name that is globally forbidden. Rejecting.", + "Server {} tried joining room ID {} who has a server name that is globally forbidden. Rejecting.", + body.origin(), &body.room_id, ); return Err(Error::BadRequest( @@ -243,8 +242,8 @@ pub(crate) async fn create_join_event_v1_route( .contains(&server.to_owned()) { warn!( - "Server {origin} tried joining room ID {} which has a server name that is globally forbidden. \ - Rejecting.", + "Server {} tried joining room ID {} which has a server name that is globally forbidden. Rejecting.", + body.origin(), &body.room_id, ); return Err(Error::BadRequest( @@ -254,7 +253,7 @@ pub(crate) async fn create_join_event_v1_route( } } - let room_state = create_join_event(&services, origin, &body.room_id, &body.pdu).await?; + let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu).await?; Ok(create_join_event::v1::Response { room_state, @@ -267,13 +266,11 @@ pub(crate) async fn create_join_event_v1_route( pub(crate) async fn create_join_event_v2_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - if services .globals .config .forbidden_remote_server_names - .contains(origin) + .contains(body.origin()) { return Err(Error::BadRequest( ErrorKind::forbidden(), @@ -299,7 +296,7 @@ pub(crate) async fn create_join_event_v2_route( auth_chain, state, event, - } = create_join_event(&services, origin, &body.room_id, &body.pdu).await?; + } = create_join_event(&services, body.origin(), &body.room_id, &body.pdu).await?; let room_state = create_join_event::v2::RoomState { members_omitted: false, auth_chain, diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index 0530f9dd..7b4a8aee 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -8,7 +8,7 @@ use ruma::{ room::member::{MembershipState, RoomMemberEventContent}, StateEventType, }, - OwnedServerName, OwnedUserId, RoomId, ServerName, + OwnedUserId, RoomId, ServerName, }; use serde_json::value::RawValue as RawJsonValue; @@ -23,9 +23,7 @@ use crate::{ pub(crate) async fn create_leave_event_v1_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - create_leave_event(&services, origin, &body.room_id, &body.pdu).await?; + create_leave_event(&services, body.origin(), &body.room_id, &body.pdu).await?; Ok(create_leave_event::v1::Response::new()) } @@ -36,9 +34,7 @@ pub(crate) async fn create_leave_event_v1_route( pub(crate) async fn create_leave_event_v2_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - - create_leave_event(&services, origin, &body.room_id, &body.pdu).await?; + create_leave_event(&services, body.origin(), &body.room_id, &body.pdu).await?; Ok(create_leave_event::v2::Response::new()) } @@ -139,16 +135,6 @@ async fn create_leave_event( )); } - let origin: OwnedServerName = serde_json::from_value( - serde_json::to_value( - value - .get("origin") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?, - ) - .expect("CanonicalJson is valid json value"), - ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; - let mutex_lock = services .rooms .event_handler @@ -159,7 +145,7 @@ async fn create_leave_event( let pdu_id: Vec = services .rooms .event_handler - .handle_incoming_pdu(&origin, room_id, &event_id, value, true) + .handle_incoming_pdu(origin, room_id, &event_id, value, true) .await? .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; diff --git a/src/api/server/state.rs b/src/api/server/state.rs index 3a27cd0a..59bb6c7b 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -13,12 +13,10 @@ use crate::Ruma; pub(crate) async fn get_room_state_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - services .rooms .event_handler - .acl_check(origin, &body.room_id) + .acl_check(body.origin(), &body.room_id) .await?; if !services @@ -28,7 +26,7 @@ pub(crate) async fn get_room_state_route( .await && !services .rooms .state_cache - .server_in_room(origin, &body.room_id) + .server_in_room(body.origin(), &body.room_id) .await { return Err!(Request(Forbidden("Server is not in room."))); diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index b026abf1..957a2a86 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -14,12 +14,10 @@ use crate::{Result, Ruma}; pub(crate) async fn get_room_state_ids_route( State(services): State, body: Ruma, ) -> Result { - let origin = body.origin.as_ref().expect("server is authenticated"); - services .rooms .event_handler - .acl_check(origin, &body.room_id) + .acl_check(body.origin(), &body.room_id) .await?; if !services @@ -29,7 +27,7 @@ pub(crate) async fn get_room_state_ids_route( .await && !services .rooms .state_cache - .server_in_room(origin, &body.room_id) + .server_in_room(body.origin(), &body.room_id) .await { return Err!(Request(Forbidden("Server is not in room."))); diff --git a/src/api/server/user.rs b/src/api/server/user.rs index 0718da58..40f330a1 100644 --- a/src/api/server/user.rs +++ b/src/api/server/user.rs @@ -27,8 +27,6 @@ pub(crate) async fn get_devices_route( )); } - let origin = body.origin.as_ref().expect("server is authenticated"); - let user_id = &body.user_id; Ok(get_devices::v1::Response { user_id: user_id.clone(), @@ -66,12 +64,12 @@ pub(crate) async fn get_devices_route( .await, master_key: services .users - .get_master_key(None, &body.user_id, &|u| u.server_name() == origin) + .get_master_key(None, &body.user_id, &|u| u.server_name() == body.origin()) .await .ok(), self_signing_key: services .users - .get_self_signing_key(None, &body.user_id, &|u| u.server_name() == origin) + .get_self_signing_key(None, &body.user_id, &|u| u.server_name() == body.origin()) .await .ok(), }) From 8742266ff0422fb678c86306d2d7384ff7081fe4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 25 Oct 2024 01:16:01 +0000 Subject: [PATCH 0958/2091] split up core/pdu Signed-off-by: Jason Volk --- src/core/pdu/content.rs | 20 ++ src/core/pdu/id.rs | 27 +++ src/core/pdu/mod.rs | 456 +++----------------------------------- src/core/pdu/redact.rs | 93 ++++++++ src/core/pdu/state_res.rs | 30 +++ src/core/pdu/strip.rs | 208 +++++++++++++++++ src/core/pdu/unsigned.rs | 83 +++++++ 7 files changed, 492 insertions(+), 425 deletions(-) create mode 100644 src/core/pdu/content.rs create mode 100644 src/core/pdu/id.rs create mode 100644 src/core/pdu/redact.rs create mode 100644 src/core/pdu/state_res.rs create mode 100644 src/core/pdu/strip.rs create mode 100644 src/core/pdu/unsigned.rs diff --git a/src/core/pdu/content.rs b/src/core/pdu/content.rs new file mode 100644 index 00000000..a6d86554 --- /dev/null +++ b/src/core/pdu/content.rs @@ -0,0 +1,20 @@ +use serde::Deserialize; +use serde_json::value::Value as JsonValue; + +use crate::{err, implement, Result}; + +#[must_use] +#[implement(super::PduEvent)] +pub fn get_content_as_value(&self) -> JsonValue { + self.get_content() + .expect("pdu content must be a valid JSON value") +} + +#[implement(super::PduEvent)] +pub fn get_content(&self) -> Result +where + T: for<'de> Deserialize<'de>, +{ + serde_json::from_str(self.content.get()) + .map_err(|e| err!(Database("Failed to deserialize pdu content into type: {e}"))) +} diff --git a/src/core/pdu/id.rs b/src/core/pdu/id.rs new file mode 100644 index 00000000..ae5b85f9 --- /dev/null +++ b/src/core/pdu/id.rs @@ -0,0 +1,27 @@ +use ruma::{CanonicalJsonObject, OwnedEventId, RoomVersionId}; +use serde_json::value::RawValue as RawJsonValue; + +use crate::{err, Result}; + +/// Generates a correct eventId for the incoming pdu. +/// +/// Returns a tuple of the new `EventId` and the PDU as a `BTreeMap`. +pub fn gen_event_id_canonical_json( + pdu: &RawJsonValue, room_version_id: &RoomVersionId, +) -> Result<(OwnedEventId, CanonicalJsonObject)> { + let value: CanonicalJsonObject = serde_json::from_str(pdu.get()) + .map_err(|e| err!(BadServerResponse(warn!("Error parsing incoming event: {e:?}"))))?; + + let event_id = gen_event_id(&value, room_version_id)?; + + Ok((event_id, value)) +} + +/// Generates a correct eventId for the incoming pdu. +pub fn gen_event_id(value: &CanonicalJsonObject, room_version_id: &RoomVersionId) -> Result { + let reference_hash = ruma::signatures::reference_hash(value, room_version_id)?; + let event_id: OwnedEventId = format!("${reference_hash}").try_into()?; + + Ok(event_id) +} diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 274b96bd..9970c39e 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -1,44 +1,28 @@ mod builder; +mod content; mod count; +mod id; +mod redact; +mod state_res; +mod strip; +mod unsigned; -use std::{cmp::Ordering, collections::BTreeMap, sync::Arc}; +use std::{cmp::Ordering, sync::Arc}; use ruma::{ - canonical_json::redact_content_in_place, - events::{ - room::{member::RoomMemberEventContent, redaction::RoomRedactionEventContent}, - space::child::HierarchySpaceChildEvent, - AnyEphemeralRoomEvent, AnyMessageLikeEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncStateEvent, - AnySyncTimelineEvent, AnyTimelineEvent, StateEvent, TimelineEventType, - }, - serde::Raw, - state_res, CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, - OwnedUserId, RoomId, RoomVersionId, UInt, UserId, + events::TimelineEventType, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedRoomId, OwnedUserId, UInt, }; use serde::{Deserialize, Serialize}; -use serde_json::{ - json, - value::{to_raw_value, RawValue as RawJsonValue, Value as JsonValue}, -}; +use serde_json::value::RawValue as RawJsonValue; pub use self::{ builder::{Builder, Builder as PduBuilder}, count::PduCount, + id::*, }; -use crate::{err, is_true, warn, Error, Result}; - -#[derive(Deserialize)] -struct ExtractRedactedBecause { - redacted_because: Option, -} - -/// Content hashes of a PDU. -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct EventHash { - /// The SHA-256 hash. - pub sha256: String, -} +use crate::Result; +/// Persistent Data Unit (Event) #[derive(Clone, Deserialize, Serialize, Debug)] pub struct PduEvent { pub event_id: Arc, @@ -65,415 +49,37 @@ pub struct PduEvent { pub signatures: Option>, } +/// Content hashes of a PDU. +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct EventHash { + /// The SHA-256 hash. + pub sha256: String, +} + impl PduEvent { - #[tracing::instrument(skip(self), level = "debug")] - pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> Result<()> { - self.unsigned = None; - - let mut content = serde_json::from_str(self.content.get()) - .map_err(|_| Error::bad_database("PDU in db has invalid content."))?; - - redact_content_in_place(&mut content, &room_version_id, self.kind.to_string()) - .map_err(|e| Error::Redaction(self.sender.server_name().to_owned(), e))?; - - self.unsigned = Some( - to_raw_value(&json!({ - "redacted_because": serde_json::to_value(reason).expect("to_value(PduEvent) always works") - })) - .expect("to string always works"), - ); - - self.content = to_raw_value(&content).expect("to string always works"); - - Ok(()) - } - - #[must_use] - pub fn is_redacted(&self) -> bool { - let Some(unsigned) = &self.unsigned else { - return false; - }; - - let Ok(unsigned) = ExtractRedactedBecause::deserialize(&**unsigned) else { - return false; - }; - - unsigned.redacted_because.is_some() - } - - pub fn remove_transaction_id(&mut self) -> Result<()> { - let Some(unsigned) = &self.unsigned else { - return Ok(()); - }; - - let mut unsigned: BTreeMap> = - serde_json::from_str(unsigned.get()).map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; - - unsigned.remove("transaction_id"); - self.unsigned = to_raw_value(&unsigned) - .map(Some) - .expect("unsigned is valid"); - - Ok(()) - } - - pub fn add_age(&mut self) -> Result<()> { - let mut unsigned: BTreeMap> = self - .unsigned - .as_ref() - .map_or_else(|| Ok(BTreeMap::new()), |u| serde_json::from_str(u.get())) - .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; - - // deliberately allowing for the possibility of negative age - let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into(); - let then: i128 = self.origin_server_ts.into(); - let this_age = now.saturating_sub(then); - - unsigned.insert("age".to_owned(), to_raw_value(&this_age).expect("age is valid")); - self.unsigned = to_raw_value(&unsigned) - .map(Some) - .expect("unsigned is valid"); - - Ok(()) - } - - /// Copies the `redacts` property of the event to the `content` dict and - /// vice-versa. - /// - /// This follows the specification's - /// [recommendation](https://spec.matrix.org/v1.10/rooms/v11/#moving-the-redacts-property-of-mroomredaction-events-to-a-content-property): - /// - /// > For backwards-compatibility with older clients, servers should add a - /// > redacts - /// > property to the top level of m.room.redaction events in when serving - /// > such events - /// > over the Client-Server API. - /// - /// > For improved compatibility with newer clients, servers should add a - /// > redacts property - /// > to the content of m.room.redaction events in older room versions when - /// > serving - /// > such events over the Client-Server API. - #[must_use] - pub fn copy_redacts(&self) -> (Option>, Box) { - if self.kind == TimelineEventType::RoomRedaction { - if let Ok(mut content) = serde_json::from_str::(self.content.get()) { - if let Some(redacts) = content.redacts { - return (Some(redacts.into()), self.content.clone()); - } else if let Some(redacts) = self.redacts.clone() { - content.redacts = Some(redacts.into()); - return ( - self.redacts.clone(), - to_raw_value(&content).expect("Must be valid, we only added redacts field"), - ); - } - } - } - - (self.redacts.clone(), self.content.clone()) - } - - #[must_use] - pub fn get_content_as_value(&self) -> JsonValue { - self.get_content() - .expect("pdu content must be a valid JSON value") - } - - pub fn get_content(&self) -> Result - where - T: for<'de> Deserialize<'de>, - { - serde_json::from_str(self.content.get()) - .map_err(|e| err!(Database("Failed to deserialize pdu content into type: {e}"))) - } - - pub fn contains_unsigned_property(&self, property: &str, is_type: F) -> bool - where - F: FnOnce(&JsonValue) -> bool, - { - self.get_unsigned_as_value() - .get(property) - .map(is_type) - .is_some_and(is_true!()) - } - - pub fn get_unsigned_property(&self, property: &str) -> Result - where - T: for<'de> Deserialize<'de>, - { - self.get_unsigned_as_value() - .get_mut(property) - .map(JsonValue::take) - .map(serde_json::from_value) - .ok_or(err!(Request(NotFound("property not found in unsigned object"))))? - .map_err(|e| err!(Database("Failed to deserialize unsigned.{property} into type: {e}"))) - } - - #[must_use] - pub fn get_unsigned_as_value(&self) -> JsonValue { self.get_unsigned::().unwrap_or_default() } - - pub fn get_unsigned(&self) -> Result { - self.unsigned - .as_ref() - .map(|raw| raw.get()) - .map(serde_json::from_str) - .ok_or(err!(Request(NotFound("\"unsigned\" property not found in pdu"))))? - .map_err(|e| err!(Database("Failed to deserialize \"unsigned\" into value: {e}"))) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_sync_room_event(&self) -> Raw { - let (redacts, content) = self.copy_redacts(); - let mut json = json!({ - "content": content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &redacts { - json["redacts"] = json!(redacts); - } - - serde_json::from_value(json).expect("Raw::from_value always works") - } - - /// This only works for events that are also AnyRoomEvents. - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_any_event(&self) -> Raw { - let (redacts, content) = self.copy_redacts(); - let mut json = json!({ - "content": content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "room_id": self.room_id, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &redacts { - json["redacts"] = json!(redacts); - } - - serde_json::from_value(json).expect("Raw::from_value always works") - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_room_event(&self) -> Raw { - let (redacts, content) = self.copy_redacts(); - let mut json = json!({ - "content": content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "room_id": self.room_id, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &redacts { - json["redacts"] = json!(redacts); - } - - serde_json::from_value(json).expect("Raw::from_value always works") - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_message_like_event(&self) -> Raw { - let (redacts, content) = self.copy_redacts(); - let mut json = json!({ - "content": content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "room_id": self.room_id, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &redacts { - json["redacts"] = json!(redacts); - } - - serde_json::from_value(json).expect("Raw::from_value always works") - } - - #[must_use] - pub fn to_state_event_value(&self) -> JsonValue { - let mut json = json!({ - "content": self.content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "room_id": self.room_id, - "state_key": self.state_key, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - - json - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_state_event(&self) -> Raw { - serde_json::from_value(self.to_state_event_value()).expect("Raw::from_value always works") - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_sync_state_event(&self) -> Raw { - let mut json = json!({ - "content": self.content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "state_key": self.state_key, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - - serde_json::from_value(json).expect("Raw::from_value always works") - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_stripped_state_event(&self) -> Raw { - let json = json!({ - "content": self.content, - "type": self.kind, - "sender": self.sender, - "state_key": self.state_key, - }); - - serde_json::from_value(json).expect("Raw::from_value always works") - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_stripped_spacechild_state_event(&self) -> Raw { - let json = json!({ - "content": self.content, - "type": self.kind, - "sender": self.sender, - "state_key": self.state_key, - "origin_server_ts": self.origin_server_ts, - }); - - serde_json::from_value(json).expect("Raw::from_value always works") - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn to_member_event(&self) -> Raw> { - let mut json = json!({ - "content": self.content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "redacts": self.redacts, - "room_id": self.room_id, - "state_key": self.state_key, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - - serde_json::from_value(json).expect("Raw::from_value always works") - } - pub fn from_id_val(event_id: &EventId, mut json: CanonicalJsonObject) -> Result { - json.insert("event_id".into(), CanonicalJsonValue::String(event_id.into())); - - let value = serde_json::to_value(json)?; - let pdu = serde_json::from_value(value)?; - - Ok(pdu) + let event_id = CanonicalJsonValue::String(event_id.into()); + json.insert("event_id".into(), event_id); + serde_json::to_value(json) + .and_then(serde_json::from_value) + .map_err(Into::into) } } -impl state_res::Event for PduEvent { - type Id = Arc; - - fn event_id(&self) -> &Self::Id { &self.event_id } - - fn room_id(&self) -> &RoomId { &self.room_id } - - fn sender(&self) -> &UserId { &self.sender } - - fn event_type(&self) -> &TimelineEventType { &self.kind } - - fn content(&self) -> &RawJsonValue { &self.content } - - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { MilliSecondsSinceUnixEpoch(self.origin_server_ts) } - - fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } - - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { self.prev_events.iter() } - - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { self.auth_events.iter() } - - fn redacts(&self) -> Option<&Self::Id> { self.redacts.as_ref() } -} - -// These impl's allow us to dedup state snapshots when resolving state -// for incoming events (federation/send/{txn}). +/// Prevent derived equality which wouldn't limit itself to event_id impl Eq for PduEvent {} + +/// Equality determined by the Pdu's ID, not the memory representations. impl PartialEq for PduEvent { fn eq(&self, other: &Self) -> bool { self.event_id == other.event_id } } + +/// Ordering determined by the Pdu's ID, not the memory representations. impl PartialOrd for PduEvent { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } + +/// Ordering determined by the Pdu's ID, not the memory representations. impl Ord for PduEvent { fn cmp(&self, other: &Self) -> Ordering { self.event_id.cmp(&other.event_id) } } - -/// Generates a correct eventId for the incoming pdu. -/// -/// Returns a tuple of the new `EventId` and the PDU as a `BTreeMap`. -pub fn gen_event_id_canonical_json( - pdu: &RawJsonValue, room_version_id: &RoomVersionId, -) -> Result<(OwnedEventId, CanonicalJsonObject)> { - let value: CanonicalJsonObject = serde_json::from_str(pdu.get()) - .map_err(|e| err!(BadServerResponse(warn!("Error parsing incoming event: {e:?}"))))?; - - let event_id = gen_event_id(&value, room_version_id)?; - - Ok((event_id, value)) -} - -/// Generates a correct eventId for the incoming pdu. -pub fn gen_event_id(value: &CanonicalJsonObject, room_version_id: &RoomVersionId) -> Result { - let reference_hash = ruma::signatures::reference_hash(value, room_version_id)?; - let event_id: OwnedEventId = format!("${reference_hash}").try_into()?; - - Ok(event_id) -} diff --git a/src/core/pdu/redact.rs b/src/core/pdu/redact.rs new file mode 100644 index 00000000..647f54c0 --- /dev/null +++ b/src/core/pdu/redact.rs @@ -0,0 +1,93 @@ +use std::sync::Arc; + +use ruma::{ + canonical_json::redact_content_in_place, + events::{room::redaction::RoomRedactionEventContent, TimelineEventType}, + EventId, RoomVersionId, +}; +use serde::Deserialize; +use serde_json::{ + json, + value::{to_raw_value, RawValue as RawJsonValue}, +}; + +use crate::{implement, warn, Error, Result}; + +#[derive(Deserialize)] +struct ExtractRedactedBecause { + redacted_because: Option, +} + +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> Result<()> { + self.unsigned = None; + + let mut content = + serde_json::from_str(self.content.get()).map_err(|_| Error::bad_database("PDU in db has invalid content."))?; + + redact_content_in_place(&mut content, &room_version_id, self.kind.to_string()) + .map_err(|e| Error::Redaction(self.sender.server_name().to_owned(), e))?; + + self.unsigned = Some( + to_raw_value(&json!({ + "redacted_because": serde_json::to_value(reason).expect("to_value(PduEvent) always works") + })) + .expect("to string always works"), + ); + + self.content = to_raw_value(&content).expect("to string always works"); + + Ok(()) +} + +#[implement(super::PduEvent)] +#[must_use] +pub fn is_redacted(&self) -> bool { + let Some(unsigned) = &self.unsigned else { + return false; + }; + + let Ok(unsigned) = ExtractRedactedBecause::deserialize(&**unsigned) else { + return false; + }; + + unsigned.redacted_because.is_some() +} + +/// Copies the `redacts` property of the event to the `content` dict and +/// vice-versa. +/// +/// This follows the specification's +/// [recommendation](https://spec.matrix.org/v1.10/rooms/v11/#moving-the-redacts-property-of-mroomredaction-events-to-a-content-property): +/// +/// > For backwards-compatibility with older clients, servers should add a +/// > redacts +/// > property to the top level of m.room.redaction events in when serving +/// > such events +/// > over the Client-Server API. +/// +/// > For improved compatibility with newer clients, servers should add a +/// > redacts property +/// > to the content of m.room.redaction events in older room versions when +/// > serving +/// > such events over the Client-Server API. +#[implement(super::PduEvent)] +#[must_use] +pub fn copy_redacts(&self) -> (Option>, Box) { + if self.kind == TimelineEventType::RoomRedaction { + if let Ok(mut content) = serde_json::from_str::(self.content.get()) { + if let Some(redacts) = content.redacts { + return (Some(redacts.into()), self.content.clone()); + } else if let Some(redacts) = self.redacts.clone() { + content.redacts = Some(redacts.into()); + return ( + self.redacts.clone(), + to_raw_value(&content).expect("Must be valid, we only added redacts field"), + ); + } + } + } + + (self.redacts.clone(), self.content.clone()) +} diff --git a/src/core/pdu/state_res.rs b/src/core/pdu/state_res.rs new file mode 100644 index 00000000..a27c9822 --- /dev/null +++ b/src/core/pdu/state_res.rs @@ -0,0 +1,30 @@ +use std::sync::Arc; + +use ruma::{events::TimelineEventType, state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId}; +use serde_json::value::RawValue as RawJsonValue; + +use super::PduEvent; + +impl state_res::Event for PduEvent { + type Id = Arc; + + fn event_id(&self) -> &Self::Id { &self.event_id } + + fn room_id(&self) -> &RoomId { &self.room_id } + + fn sender(&self) -> &UserId { &self.sender } + + fn event_type(&self) -> &TimelineEventType { &self.kind } + + fn content(&self) -> &RawJsonValue { &self.content } + + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { MilliSecondsSinceUnixEpoch(self.origin_server_ts) } + + fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } + + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { self.prev_events.iter() } + + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { self.auth_events.iter() } + + fn redacts(&self) -> Option<&Self::Id> { self.redacts.as_ref() } +} diff --git a/src/core/pdu/strip.rs b/src/core/pdu/strip.rs new file mode 100644 index 00000000..8d20d982 --- /dev/null +++ b/src/core/pdu/strip.rs @@ -0,0 +1,208 @@ +use ruma::{ + events::{ + room::member::RoomMemberEventContent, space::child::HierarchySpaceChildEvent, AnyEphemeralRoomEvent, + AnyMessageLikeEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncStateEvent, AnySyncTimelineEvent, + AnyTimelineEvent, StateEvent, + }, + serde::Raw, +}; +use serde_json::{json, value::Value as JsonValue}; + +use crate::{implement, warn}; + +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn to_sync_room_event(&self) -> Raw { + let (redacts, content) = self.copy_redacts(); + let mut json = json!({ + "content": content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + }); + + if let Some(unsigned) = &self.unsigned { + json["unsigned"] = json!(unsigned); + } + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") +} + +/// This only works for events that are also AnyRoomEvents. +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn to_any_event(&self) -> Raw { + let (redacts, content) = self.copy_redacts(); + let mut json = json!({ + "content": content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "room_id": self.room_id, + }); + + if let Some(unsigned) = &self.unsigned { + json["unsigned"] = json!(unsigned); + } + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") +} + +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn to_room_event(&self) -> Raw { + let (redacts, content) = self.copy_redacts(); + let mut json = json!({ + "content": content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "room_id": self.room_id, + }); + + if let Some(unsigned) = &self.unsigned { + json["unsigned"] = json!(unsigned); + } + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") +} + +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn to_message_like_event(&self) -> Raw { + let (redacts, content) = self.copy_redacts(); + let mut json = json!({ + "content": content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "room_id": self.room_id, + }); + + if let Some(unsigned) = &self.unsigned { + json["unsigned"] = json!(unsigned); + } + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &redacts { + json["redacts"] = json!(redacts); + } + + serde_json::from_value(json).expect("Raw::from_value always works") +} + +#[implement(super::PduEvent)] +#[must_use] +pub fn to_state_event_value(&self) -> JsonValue { + let mut json = json!({ + "content": self.content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "room_id": self.room_id, + "state_key": self.state_key, + }); + + if let Some(unsigned) = &self.unsigned { + json["unsigned"] = json!(unsigned); + } + + json +} + +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn to_state_event(&self) -> Raw { + serde_json::from_value(self.to_state_event_value()).expect("Raw::from_value always works") +} + +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn to_sync_state_event(&self) -> Raw { + let mut json = json!({ + "content": self.content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "state_key": self.state_key, + }); + + if let Some(unsigned) = &self.unsigned { + json["unsigned"] = json!(unsigned); + } + + serde_json::from_value(json).expect("Raw::from_value always works") +} + +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn to_stripped_state_event(&self) -> Raw { + let json = json!({ + "content": self.content, + "type": self.kind, + "sender": self.sender, + "state_key": self.state_key, + }); + + serde_json::from_value(json).expect("Raw::from_value always works") +} + +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn to_stripped_spacechild_state_event(&self) -> Raw { + let json = json!({ + "content": self.content, + "type": self.kind, + "sender": self.sender, + "state_key": self.state_key, + "origin_server_ts": self.origin_server_ts, + }); + + serde_json::from_value(json).expect("Raw::from_value always works") +} + +#[implement(super::PduEvent)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn to_member_event(&self) -> Raw> { + let mut json = json!({ + "content": self.content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + "redacts": self.redacts, + "room_id": self.room_id, + "state_key": self.state_key, + }); + + if let Some(unsigned) = &self.unsigned { + json["unsigned"] = json!(unsigned); + } + + serde_json::from_value(json).expect("Raw::from_value always works") +} diff --git a/src/core/pdu/unsigned.rs b/src/core/pdu/unsigned.rs new file mode 100644 index 00000000..1c47e826 --- /dev/null +++ b/src/core/pdu/unsigned.rs @@ -0,0 +1,83 @@ +use std::collections::BTreeMap; + +use ruma::MilliSecondsSinceUnixEpoch; +use serde::Deserialize; +use serde_json::value::{to_raw_value, RawValue as RawJsonValue, Value as JsonValue}; + +use crate::{err, implement, is_true, Result}; + +#[implement(super::PduEvent)] +pub fn remove_transaction_id(&mut self) -> Result<()> { + let Some(unsigned) = &self.unsigned else { + return Ok(()); + }; + + let mut unsigned: BTreeMap> = + serde_json::from_str(unsigned.get()).map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; + + unsigned.remove("transaction_id"); + self.unsigned = to_raw_value(&unsigned) + .map(Some) + .expect("unsigned is valid"); + + Ok(()) +} + +#[implement(super::PduEvent)] +pub fn add_age(&mut self) -> Result<()> { + let mut unsigned: BTreeMap> = self + .unsigned + .as_ref() + .map_or_else(|| Ok(BTreeMap::new()), |u| serde_json::from_str(u.get())) + .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; + + // deliberately allowing for the possibility of negative age + let now: i128 = MilliSecondsSinceUnixEpoch::now().get().into(); + let then: i128 = self.origin_server_ts.into(); + let this_age = now.saturating_sub(then); + + unsigned.insert("age".to_owned(), to_raw_value(&this_age).expect("age is valid")); + self.unsigned = to_raw_value(&unsigned) + .map(Some) + .expect("unsigned is valid"); + + Ok(()) +} + +#[implement(super::PduEvent)] +pub fn contains_unsigned_property(&self, property: &str, is_type: F) -> bool +where + F: FnOnce(&JsonValue) -> bool, +{ + self.get_unsigned_as_value() + .get(property) + .map(is_type) + .is_some_and(is_true!()) +} + +#[implement(super::PduEvent)] +pub fn get_unsigned_property(&self, property: &str) -> Result +where + T: for<'de> Deserialize<'de>, +{ + self.get_unsigned_as_value() + .get_mut(property) + .map(JsonValue::take) + .map(serde_json::from_value) + .ok_or(err!(Request(NotFound("property not found in unsigned object"))))? + .map_err(|e| err!(Database("Failed to deserialize unsigned.{property} into type: {e}"))) +} + +#[implement(super::PduEvent)] +#[must_use] +pub fn get_unsigned_as_value(&self) -> JsonValue { self.get_unsigned::().unwrap_or_default() } + +#[implement(super::PduEvent)] +pub fn get_unsigned(&self) -> Result { + self.unsigned + .as_ref() + .map(|raw| raw.get()) + .map(serde_json::from_str) + .ok_or(err!(Request(NotFound("\"unsigned\" property not found in pdu"))))? + .map_err(|e| err!(Database("Failed to deserialize \"unsigned\" into value: {e}"))) +} From cf59f738b9f687aa0902bf2bd011219567e4fea5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 25 Oct 2024 02:01:29 +0000 Subject: [PATCH 0959/2091] move macros incorrectly moved out of utils to top level Signed-off-by: Jason Volk --- src/core/mod.rs | 56 ----------------------------------- src/core/utils/mod.rs | 68 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 56 deletions(-) diff --git a/src/core/mod.rs b/src/core/mod.rs index 79052554..d201709b 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -36,59 +36,3 @@ pub mod mods { () => {}; } } - -/// Functor for falsy -#[macro_export] -macro_rules! is_false { - () => { - |x| !x - }; -} - -/// Functor for truthy -#[macro_export] -macro_rules! is_true { - () => { - |x| !!x - }; -} - -/// Functor for equality to zero -#[macro_export] -macro_rules! is_zero { - () => { - $crate::is_matching!(0) - }; -} - -/// Functor for equality i.e. .is_some_and(is_equal!(2)) -#[macro_export] -macro_rules! is_equal_to { - ($val:expr) => { - |x| x == $val - }; -} - -/// Functor for less i.e. .is_some_and(is_less_than!(2)) -#[macro_export] -macro_rules! is_less_than { - ($val:expr) => { - |x| x < $val - }; -} - -/// Functor for matches! i.e. .is_some_and(is_matching!('A'..='Z')) -#[macro_export] -macro_rules! is_matching { - ($val:expr) => { - |x| matches!(x, $val) - }; -} - -/// Functor for !is_empty() -#[macro_export] -macro_rules! is_not_empty { - () => { - |x| !x.is_empty() - }; -} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 3943a8da..8e29c608 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -57,3 +57,71 @@ macro_rules! at { |t| t.$idx }; } + +/// Functor for equality i.e. .is_some_and(is_equal!(2)) +#[macro_export] +macro_rules! is_equal_to { + ($val:ident) => { + |x| x == $val + }; + + ($val:expr) => { + |x| x == $val + }; +} + +/// Functor for less i.e. .is_some_and(is_less_than!(2)) +#[macro_export] +macro_rules! is_less_than { + ($val:ident) => { + |x| x < $val + }; + + ($val:expr) => { + |x| x < $val + }; +} + +/// Functor for equality to zero +#[macro_export] +macro_rules! is_zero { + () => { + $crate::is_matching!(0) + }; +} + +/// Functor for matches! i.e. .is_some_and(is_matching!('A'..='Z')) +#[macro_export] +macro_rules! is_matching { + ($val:ident) => { + |x| matches!(x, $val) + }; + + ($val:expr) => { + |x| matches!(x, $val) + }; +} + +/// Functor for !is_empty() +#[macro_export] +macro_rules! is_not_empty { + () => { + |x| !x.is_empty() + }; +} + +/// Functor for truthy +#[macro_export] +macro_rules! is_true { + () => { + |x| !!x + }; +} + +/// Functor for falsy +#[macro_export] +macro_rules! is_false { + () => { + |x| !x + }; +} From b7369074d4d9e235c4bb9a7529e98c1aa5a662b1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 25 Oct 2024 02:56:24 +0000 Subject: [PATCH 0960/2091] add RoomEventFilter matcher for PduEvent Signed-off-by: Jason Volk --- src/core/pdu/filter.rs | 90 ++++++++++++++++++++++++++++++++++++++++++ src/core/pdu/mod.rs | 1 + 2 files changed, 91 insertions(+) create mode 100644 src/core/pdu/filter.rs diff --git a/src/core/pdu/filter.rs b/src/core/pdu/filter.rs new file mode 100644 index 00000000..bd232ebd --- /dev/null +++ b/src/core/pdu/filter.rs @@ -0,0 +1,90 @@ +use ruma::api::client::filter::{RoomEventFilter, UrlFilter}; +use serde_json::Value; + +use crate::{implement, is_equal_to}; + +#[implement(super::PduEvent)] +#[must_use] +pub fn matches(&self, filter: &RoomEventFilter) -> bool { + if !self.matches_sender(filter) { + return false; + } + + if !self.matches_room(filter) { + return false; + } + + if !self.matches_type(filter) { + return false; + } + + if !self.matches_url(filter) { + return false; + } + + true +} + +#[implement(super::PduEvent)] +fn matches_room(&self, filter: &RoomEventFilter) -> bool { + if filter.not_rooms.contains(&self.room_id) { + return false; + } + + if let Some(rooms) = filter.rooms.as_ref() { + if !rooms.contains(&self.room_id) { + return false; + } + } + + true +} + +#[implement(super::PduEvent)] +fn matches_sender(&self, filter: &RoomEventFilter) -> bool { + if filter.not_senders.contains(&self.sender) { + return false; + } + + if let Some(senders) = filter.senders.as_ref() { + if !senders.contains(&self.sender) { + return false; + } + } + + true +} + +#[implement(super::PduEvent)] +fn matches_type(&self, filter: &RoomEventFilter) -> bool { + let event_type = &self.kind.to_cow_str(); + if filter.not_types.iter().any(is_equal_to!(event_type)) { + return false; + } + + if let Some(types) = filter.types.as_ref() { + if !types.iter().any(is_equal_to!(event_type)) { + return false; + } + } + + true +} + +#[implement(super::PduEvent)] +fn matches_url(&self, filter: &RoomEventFilter) -> bool { + let Some(url_filter) = filter.url_filter.as_ref() else { + return true; + }; + + //TODO: might be better to use Ruma's Raw rather than serde here + let url = serde_json::from_str::(self.content.get()) + .expect("parsing content JSON failed") + .get("url") + .is_some_and(Value::is_string); + + match url_filter { + UrlFilter::EventsWithUrl => url, + UrlFilter::EventsWithoutUrl => !url, + } +} diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 9970c39e..ed11adbb 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -1,6 +1,7 @@ mod builder; mod content; mod count; +mod filter; mod id; mod redact; mod state_res; From 68086717516225af96fe6c7cff743836103188eb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 25 Oct 2024 05:22:50 +0000 Subject: [PATCH 0961/2091] merge search service w/ data Signed-off-by: Jason Volk --- src/service/rooms/search/data.rs | 113 ---------------------------- src/service/rooms/search/mod.rs | 123 ++++++++++++++++++++++++++----- 2 files changed, 103 insertions(+), 133 deletions(-) delete mode 100644 src/service/rooms/search/data.rs diff --git a/src/service/rooms/search/data.rs b/src/service/rooms/search/data.rs deleted file mode 100644 index de98beee..00000000 --- a/src/service/rooms/search/data.rs +++ /dev/null @@ -1,113 +0,0 @@ -use std::sync::Arc; - -use conduit::utils::{set, stream::TryIgnore, IterStream, ReadyExt}; -use database::Map; -use futures::StreamExt; -use ruma::RoomId; - -use crate::{rooms, Dep}; - -pub(super) struct Data { - tokenids: Arc, - services: Services, -} - -struct Services { - short: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - tokenids: db["tokenids"].clone(), - services: Services { - short: args.depend::("rooms::short"), - }, - } - } - - pub(super) fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { - let batch = tokenize(message_body) - .map(|word| { - let mut key = shortroomid.to_be_bytes().to_vec(); - key.extend_from_slice(word.as_bytes()); - key.push(0xFF); - key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here - (key, Vec::::new()) - }) - .collect::>(); - - self.tokenids.insert_batch(batch.iter()); - } - - pub(super) fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { - let batch = tokenize(message_body).map(|word| { - let mut key = shortroomid.to_be_bytes().to_vec(); - key.extend_from_slice(word.as_bytes()); - key.push(0xFF); - key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here - key - }); - - for token in batch { - self.tokenids.remove(&token); - } - } - - pub(super) async fn search_pdus( - &self, room_id: &RoomId, search_string: &str, - ) -> Option<(Vec>, Vec)> { - let prefix = self - .services - .short - .get_shortroomid(room_id) - .await - .ok()? - .to_be_bytes() - .to_vec(); - - let words: Vec<_> = tokenize(search_string).collect(); - - let bufs: Vec<_> = words - .clone() - .into_iter() - .stream() - .then(move |word| { - let mut prefix2 = prefix.clone(); - prefix2.extend_from_slice(word.as_bytes()); - prefix2.push(0xFF); - let prefix3 = prefix2.clone(); - - let mut last_possible_id = prefix2.clone(); - last_possible_id.extend_from_slice(&u64::MAX.to_be_bytes()); - - self.tokenids - .rev_raw_keys_from(&last_possible_id) // Newest pdus first - .ignore_err() - .ready_take_while(move |key| key.starts_with(&prefix2)) - .map(move |key| key[prefix3.len()..].to_vec()) - .collect::>() - }) - .collect() - .await; - - Some(( - set::intersection(bufs.iter().map(|buf| buf.iter())) - .cloned() - .collect(), - words, - )) - } -} - -/// Splits a string into tokens used as keys in the search inverted index -/// -/// This may be used to tokenize both message bodies (for indexing) or search -/// queries (for querying). -fn tokenize(body: &str) -> impl Iterator + Send + '_ { - body.split_terminator(|c: char| !c.is_alphanumeric()) - .filter(|s| !s.is_empty()) - .filter(|word| word.len() <= 50) - .map(str::to_lowercase) -} diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 80b58804..032ad55c 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,41 +1,124 @@ -mod data; - use std::sync::Arc; -use conduit::Result; -use data::Data; +use conduit::{ + implement, + utils::{set, stream::TryIgnore, IterStream, ReadyExt}, + Result, +}; +use database::Map; +use futures::StreamExt; use ruma::RoomId; +use crate::{rooms, Dep}; + pub struct Service { db: Data, + services: Services, +} + +struct Data { + tokenids: Arc, +} + +struct Services { + short: Dep, } impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data::new(&args), + db: Data { + tokenids: args.db["tokenids"].clone(), + }, + services: Services { + short: args.depend::("rooms::short"), + }, })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - #[inline] - #[tracing::instrument(skip(self), level = "debug")] - pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { - self.db.index_pdu(shortroomid, pdu_id, message_body); - } +#[implement(Service)] +pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { + let batch = tokenize(message_body) + .map(|word| { + let mut key = shortroomid.to_be_bytes().to_vec(); + key.extend_from_slice(word.as_bytes()); + key.push(0xFF); + key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here + (key, Vec::::new()) + }) + .collect::>(); - #[inline] - #[tracing::instrument(skip(self), level = "debug")] - pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { - self.db.deindex_pdu(shortroomid, pdu_id, message_body); - } + self.db.tokenids.insert_batch(batch.iter()); +} - #[inline] - #[tracing::instrument(skip(self), level = "debug")] - pub async fn search_pdus(&self, room_id: &RoomId, search_string: &str) -> Option<(Vec>, Vec)> { - self.db.search_pdus(room_id, search_string).await +#[implement(Service)] +pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { + let batch = tokenize(message_body).map(|word| { + let mut key = shortroomid.to_be_bytes().to_vec(); + key.extend_from_slice(word.as_bytes()); + key.push(0xFF); + key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here + key + }); + + for token in batch { + self.db.tokenids.remove(&token); } } + +#[implement(Service)] +pub async fn search_pdus(&self, room_id: &RoomId, search_string: &str) -> Option<(Vec>, Vec)> { + let prefix = self + .services + .short + .get_shortroomid(room_id) + .await + .ok()? + .to_be_bytes() + .to_vec(); + + let words: Vec<_> = tokenize(search_string).collect(); + + let bufs: Vec<_> = words + .clone() + .into_iter() + .stream() + .then(move |word| { + let mut prefix2 = prefix.clone(); + prefix2.extend_from_slice(word.as_bytes()); + prefix2.push(0xFF); + let prefix3 = prefix2.clone(); + + let mut last_possible_id = prefix2.clone(); + last_possible_id.extend_from_slice(&u64::MAX.to_be_bytes()); + + self.db.tokenids + .rev_raw_keys_from(&last_possible_id) // Newest pdus first + .ignore_err() + .ready_take_while(move |key| key.starts_with(&prefix2)) + .map(move |key| key[prefix3.len()..].to_vec()) + .collect::>() + }) + .collect() + .await; + + let bufs = bufs.iter().map(|buf| buf.iter()); + + let results = set::intersection(bufs).cloned().collect(); + + Some((results, words)) +} + +/// Splits a string into tokens used as keys in the search inverted index +/// +/// This may be used to tokenize both message bodies (for indexing) or search +/// queries (for querying). +fn tokenize(body: &str) -> impl Iterator + Send + '_ { + body.split_terminator(|c: char| !c.is_alphanumeric()) + .filter(|s| !s.is_empty()) + .filter(|word| word.len() <= 50) + .map(str::to_lowercase) +} From 0426f92ac032f03b8a4c86acec00b53c093a82d5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 25 Oct 2024 18:25:06 +0000 Subject: [PATCH 0962/2091] unify database record separator constants Signed-off-by: Jason Volk --- src/database/de.rs | 3 +-- src/database/mod.rs | 2 +- src/database/ser.rs | 5 ++++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/database/de.rs b/src/database/de.rs index e5fdf7cb..0e074fdb 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -41,8 +41,7 @@ pub struct Ignore; pub struct IgnoreAll; impl<'de> Deserializer<'de> { - /// Record separator; an intentionally invalid-utf8 byte. - const SEP: u8 = b'\xFF'; + const SEP: u8 = crate::ser::SEP; /// Determine if the input was fully consumed and error if bytes remaining. /// This is intended for debug assertions; not optimized for parsing logic. diff --git a/src/database/mod.rs b/src/database/mod.rs index 6d3b2079..dcd66a1e 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -29,7 +29,7 @@ pub use self::{ handle::Handle, keyval::{KeyVal, Slice}, map::Map, - ser::{serialize, serialize_to_array, serialize_to_vec, Interfix, Json, Separator}, + ser::{serialize, serialize_to_array, serialize_to_vec, Interfix, Json, Separator, SEP}, }; conduit::mod_ctor! {} diff --git a/src/database/ser.rs b/src/database/ser.rs index 742f1e34..0cc5c886 100644 --- a/src/database/ser.rs +++ b/src/database/ser.rs @@ -69,8 +69,11 @@ pub struct Interfix; #[derive(Debug, Serialize)] pub struct Separator; +/// Record separator; an intentionally invalid-utf8 byte. +pub const SEP: u8 = b'\xFF'; + impl Serializer<'_, W> { - const SEP: &'static [u8] = b"\xFF"; + const SEP: &'static [u8] = &[SEP]; fn tuple_start(&mut self) { debug_assert!(!self.sep, "Tuple start with separator set"); From 1e7207c23015f5bb8d9c22db30ccbb3669a9540a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 25 Oct 2024 19:53:08 +0000 Subject: [PATCH 0963/2091] start an ArrayVec extension trait Signed-off-by: Jason Volk --- src/core/utils/arrayvec.rs | 15 +++++++++++++++ src/core/utils/mod.rs | 2 ++ 2 files changed, 17 insertions(+) create mode 100644 src/core/utils/arrayvec.rs diff --git a/src/core/utils/arrayvec.rs b/src/core/utils/arrayvec.rs new file mode 100644 index 00000000..685aaf18 --- /dev/null +++ b/src/core/utils/arrayvec.rs @@ -0,0 +1,15 @@ +use ::arrayvec::ArrayVec; + +pub trait ArrayVecExt { + fn extend_from_slice(&mut self, other: &[T]) -> &mut Self; +} + +impl ArrayVecExt for ArrayVec { + #[inline] + fn extend_from_slice(&mut self, other: &[T]) -> &mut Self { + self.try_extend_from_slice(other) + .expect("Insufficient buffer capacity to extend from slice"); + + self + } +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 8e29c608..26b0484e 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -1,3 +1,4 @@ +pub mod arrayvec; pub mod bool; pub mod bytes; pub mod content_disposition; @@ -22,6 +23,7 @@ pub use ::conduit_macros::implement; pub use ::ctor::{ctor, dtor}; pub use self::{ + arrayvec::ArrayVecExt, bool::BoolExt, bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}, debug::slice_truncated as debug_slice_truncated, From f245389c0223ed96542969dafc90f0aeab1da9f5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 26 Oct 2024 22:20:16 +0000 Subject: [PATCH 0964/2091] add typedef for pdu_ids Signed-off-by: Jason Volk --- src/service/rooms/short/mod.rs | 4 ++++ src/service/rooms/timeline/mod.rs | 6 +++++- src/service/rooms/timeline/pduid.rs | 13 +++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 src/service/rooms/timeline/pduid.rs diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 609c0e07..02c449cc 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -24,6 +24,10 @@ struct Services { globals: Dep, } +pub type ShortEventId = ShortId; +pub type ShortRoomId = ShortId; +pub type ShortId = u64; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 902e50ff..e45bf7e5 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1,4 +1,5 @@ mod data; +mod pduid; use std::{ cmp, @@ -38,7 +39,10 @@ use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use self::data::Data; -pub use self::data::PdusIterItem; +pub use self::{ + data::PdusIterItem, + pduid::{PduId, RawPduId}, +}; use crate::{ account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms, rooms::state_compressor::CompressedStateEvent, sending, server_keys, users, Dep, diff --git a/src/service/rooms/timeline/pduid.rs b/src/service/rooms/timeline/pduid.rs new file mode 100644 index 00000000..b43c382c --- /dev/null +++ b/src/service/rooms/timeline/pduid.rs @@ -0,0 +1,13 @@ +use crate::rooms::short::{ShortEventId, ShortRoomId}; + +#[derive(Clone, Copy)] +pub struct PduId { + _room_id: ShortRoomId, + _event_id: ShortEventId, +} + +pub type RawPduId = [u8; PduId::LEN]; + +impl PduId { + pub const LEN: usize = size_of::() + size_of::(); +} From 21a67513f2480e6cb1cb0322e15016ba8d919dac Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 26 Oct 2024 22:21:23 +0000 Subject: [PATCH 0965/2091] refactor search system Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/api/client/search.rs | 339 +++++++++++++++++--------------- src/service/Cargo.toml | 1 + src/service/rooms/search/mod.rs | 176 +++++++++++++---- 4 files changed, 312 insertions(+), 205 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c64d3cc6..a8acce7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -786,6 +786,7 @@ dependencies = [ name = "conduit_service" version = "0.5.0" dependencies = [ + "arrayvec", "async-trait", "base64 0.22.1", "bytes", diff --git a/src/api/client/search.rs b/src/api/client/search.rs index b073640e..1e5384fe 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -2,25 +2,32 @@ use std::collections::BTreeMap; use axum::extract::State; use conduit::{ - debug, - utils::{IterStream, ReadyExt}, - Err, + at, is_true, + result::FlatOk, + utils::{stream::ReadyExt, IterStream}, + Err, PduEvent, Result, }; -use futures::{FutureExt, StreamExt}; +use futures::{future::OptionFuture, FutureExt, StreamExt, TryFutureExt}; use ruma::{ - api::client::{ - error::ErrorKind, - search::search_events::{ - self, - v3::{EventContextResult, ResultCategories, ResultRoomEvents, SearchResult}, - }, + api::client::search::search_events::{ + self, + v3::{Criteria, EventContextResult, ResultCategories, ResultRoomEvents, SearchResult}, }, events::AnyStateEvent, serde::Raw, - uint, OwnedRoomId, + OwnedRoomId, RoomId, UInt, UserId, }; +use search_events::v3::{Request, Response}; +use service::{rooms::search::RoomQuery, Services}; -use crate::{Error, Result, Ruma}; +use crate::Ruma; + +type RoomStates = BTreeMap; +type RoomState = Vec>; + +const LIMIT_DEFAULT: usize = 10; +const LIMIT_MAX: usize = 100; +const BATCH_MAX: usize = 20; /// # `POST /_matrix/client/r0/search` /// @@ -28,173 +35,177 @@ use crate::{Error, Result, Ruma}; /// /// - Only works if the user is currently joined to the room (TODO: Respect /// history visibility) -pub(crate) async fn search_events_route( - State(services): State, body: Ruma, -) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); +pub(crate) async fn search_events_route(State(services): State, body: Ruma) -> Result { + let sender_user = body.sender_user(); + let next_batch = body.next_batch.as_deref(); + let room_events_result: OptionFuture<_> = body + .search_categories + .room_events + .as_ref() + .map(|criteria| category_room_events(&services, sender_user, next_batch, criteria)) + .into(); - let search_criteria = body.search_categories.room_events.as_ref().unwrap(); - let filter = &search_criteria.filter; - let include_state = &search_criteria.include_state; + Ok(Response { + search_categories: ResultCategories { + room_events: room_events_result + .await + .unwrap_or_else(|| Ok(ResultRoomEvents::default()))?, + }, + }) +} - let room_ids = if let Some(room_ids) = &filter.rooms { - room_ids.clone() - } else { - services - .rooms - .state_cache - .rooms_joined(sender_user) - .map(ToOwned::to_owned) - .collect() - .await - }; +#[allow(clippy::map_unwrap_or)] +async fn category_room_events( + services: &Services, sender_user: &UserId, next_batch: Option<&str>, criteria: &Criteria, +) -> Result { + let filter = &criteria.filter; - // Use limit or else 10, with maximum 100 let limit: usize = filter .limit - .unwrap_or_else(|| uint!(10)) - .try_into() - .unwrap_or(10) - .min(100); + .map(TryInto::try_into) + .flat_ok() + .unwrap_or(LIMIT_DEFAULT) + .min(LIMIT_MAX); - let mut room_states: BTreeMap>> = BTreeMap::new(); + let next_batch: usize = next_batch + .map(str::parse) + .transpose()? + .unwrap_or(0) + .min(limit.saturating_mul(BATCH_MAX)); - if include_state.is_some_and(|include_state| include_state) { - for room_id in &room_ids { - if !services - .rooms - .state_cache - .is_joined(sender_user, room_id) - .await - { - return Err!(Request(Forbidden("You don't have permission to view this room."))); - } - - // check if sender_user can see state events - if services - .rooms - .state_accessor - .user_can_see_state_events(sender_user, room_id) - .await - { - let room_state: Vec<_> = services - .rooms - .state_accessor - .room_state_full(room_id) - .await? - .values() - .map(|pdu| pdu.to_state_event()) - .collect(); - - debug!("Room state: {:?}", room_state); - - room_states.insert(room_id.clone(), room_state); - } else { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this room.", - )); - } - } - } - - let mut search_vecs = Vec::new(); - - for room_id in &room_ids { - if !services - .rooms - .state_cache - .is_joined(sender_user, room_id) - .await - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this room.", - )); - } - - if let Some(search) = services - .rooms - .search - .search_pdus(room_id, &search_criteria.search_term) - .await - { - search_vecs.push(search.0); - } - } - - let mut searches: Vec<_> = search_vecs - .iter() - .map(|vec| vec.iter().peekable()) - .collect(); - - let skip: usize = match body.next_batch.as_ref().map(|s| s.parse()) { - Some(Ok(s)) => s, - Some(Err(_)) => return Err(Error::BadRequest(ErrorKind::InvalidParam, "Invalid next_batch token.")), - None => 0, // Default to the start - }; - - let mut results = Vec::new(); - let next_batch = skip.saturating_add(limit); - - for _ in 0..next_batch { - if let Some(s) = searches - .iter_mut() - .map(|s| (s.peek().copied(), s)) - .max_by_key(|(peek, _)| *peek) - .and_then(|(_, i)| i.next()) - { - results.push(s); - } - } - - let results: Vec<_> = results - .into_iter() - .skip(skip) - .stream() - .filter_map(|id| services.rooms.timeline.get_pdu_from_id(id).map(Result::ok)) - .ready_filter(|pdu| !pdu.is_redacted()) - .filter_map(|pdu| async move { + let rooms = filter + .rooms + .clone() + .map(IntoIterator::into_iter) + .map(IterStream::stream) + .map(StreamExt::boxed) + .unwrap_or_else(|| { services .rooms - .state_accessor - .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) + .state_cache + .rooms_joined(sender_user) + .map(ToOwned::to_owned) + .boxed() + }); + + let results: Vec<_> = rooms + .filter_map(|room_id| async move { + check_room_visible(services, sender_user, &room_id, criteria) .await - .then_some(pdu) + .is_ok() + .then_some(room_id) }) - .take(limit) - .map(|pdu| pdu.to_room_event()) - .map(|result| SearchResult { - context: EventContextResult { - end: None, - events_after: Vec::new(), - events_before: Vec::new(), - profile_info: BTreeMap::new(), - start: None, - }, - rank: None, - result: Some(result), + .filter_map(|room_id| async move { + let query = RoomQuery { + room_id: &room_id, + user_id: Some(sender_user), + criteria, + skip: next_batch, + limit, + }; + + let (count, results) = services.rooms.search.search_pdus(&query).await.ok()?; + + results + .collect::>() + .map(|results| (room_id.clone(), count, results)) + .map(Some) + .await }) .collect() - .boxed() .await; - let more_unloaded_results = searches.iter_mut().any(|s| s.peek().is_some()); + let total: UInt = results + .iter() + .fold(0, |a: usize, (_, count, _)| a.saturating_add(*count)) + .try_into()?; - let next_batch = more_unloaded_results.then(|| next_batch.to_string()); + let state: RoomStates = results + .iter() + .stream() + .ready_filter(|_| criteria.include_state.is_some_and(is_true!())) + .filter_map(|(room_id, ..)| async move { + procure_room_state(services, room_id) + .map_ok(|state| (room_id.clone(), state)) + .await + .ok() + }) + .collect() + .await; - Ok(search_events::v3::Response::new(ResultCategories { - room_events: ResultRoomEvents { - count: Some(results.len().try_into().unwrap_or_else(|_| uint!(0))), - groups: BTreeMap::new(), // TODO - next_batch, - results, - state: room_states, - highlights: search_criteria - .search_term - .split_terminator(|c: char| !c.is_alphanumeric()) - .map(str::to_lowercase) - .collect(), - }, - })) + let results: Vec = results + .into_iter() + .map(at!(2)) + .flatten() + .stream() + .map(|pdu| pdu.to_room_event()) + .map(|result| SearchResult { + rank: None, + result: Some(result), + context: EventContextResult { + profile_info: BTreeMap::new(), //TODO + events_after: Vec::new(), //TODO + events_before: Vec::new(), //TODO + start: None, //TODO + end: None, //TODO + }, + }) + .collect() + .await; + + let highlights = criteria + .search_term + .split_terminator(|c: char| !c.is_alphanumeric()) + .map(str::to_lowercase) + .collect(); + + let next_batch = (results.len() >= limit) + .then_some(next_batch.saturating_add(results.len())) + .as_ref() + .map(ToString::to_string); + + Ok(ResultRoomEvents { + count: Some(total), + next_batch, + results, + state, + highlights, + groups: BTreeMap::new(), // TODO + }) +} + +async fn procure_room_state(services: &Services, room_id: &RoomId) -> Result { + let state_map = services + .rooms + .state_accessor + .room_state_full(room_id) + .await?; + + let state_events = state_map + .values() + .map(AsRef::as_ref) + .map(PduEvent::to_state_event) + .collect(); + + Ok(state_events) +} + +async fn check_room_visible(services: &Services, user_id: &UserId, room_id: &RoomId, search: &Criteria) -> Result { + let check_visible = search.filter.rooms.is_some(); + let check_state = check_visible && search.include_state.is_some_and(is_true!()); + + let is_joined = !check_visible || services.rooms.state_cache.is_joined(user_id, room_id).await; + + let state_visible = !check_state + || services + .rooms + .state_accessor + .user_can_see_state_events(user_id, room_id) + .await; + + if !is_joined || !state_visible { + return Err!(Request(Forbidden("You don't have permission to view {room_id:?}"))); + } + + Ok(()) } diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 737a7039..7578ef64 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -40,6 +40,7 @@ release_max_log_level = [ ] [dependencies] +arrayvec.workspace = true async-trait.workspace = true base64.workspace = true bytes.workspace = true diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 032ad55c..8882ec99 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,15 +1,23 @@ -use std::sync::Arc; +use std::{iter, sync::Arc}; +use arrayvec::ArrayVec; use conduit::{ implement, - utils::{set, stream::TryIgnore, IterStream, ReadyExt}, - Result, + utils::{set, stream::TryIgnore, ArrayVecExt, IterStream, ReadyExt}, + PduEvent, Result, }; -use database::Map; -use futures::StreamExt; -use ruma::RoomId; +use database::{keyval::Val, Map}; +use futures::{Stream, StreamExt}; +use ruma::{api::client::search::search_events::v3::Criteria, RoomId, UserId}; -use crate::{rooms, Dep}; +use crate::{ + rooms, + rooms::{ + short::ShortRoomId, + timeline::{PduId, RawPduId}, + }, + Dep, +}; pub struct Service { db: Data, @@ -22,8 +30,24 @@ struct Data { struct Services { short: Dep, + state_accessor: Dep, + timeline: Dep, } +#[derive(Clone, Debug)] +pub struct RoomQuery<'a> { + pub room_id: &'a RoomId, + pub user_id: Option<&'a UserId>, + pub criteria: &'a Criteria, + pub limit: usize, + pub skip: usize, +} + +type TokenId = ArrayVec; + +const TOKEN_ID_MAX_LEN: usize = size_of::() + WORD_MAX_LEN + 1 + size_of::(); +const WORD_MAX_LEN: usize = 50; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -32,6 +56,8 @@ impl crate::Service for Service { }, services: Services { short: args.depend::("rooms::short"), + state_accessor: args.depend::("rooms::state_accessor"), + timeline: args.depend::("rooms::timeline"), }, })) } @@ -70,46 +96,92 @@ pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { } #[implement(Service)] -pub async fn search_pdus(&self, room_id: &RoomId, search_string: &str) -> Option<(Vec>, Vec)> { - let prefix = self - .services - .short - .get_shortroomid(room_id) - .await - .ok()? - .to_be_bytes() - .to_vec(); +pub async fn search_pdus<'a>( + &'a self, query: &'a RoomQuery<'a>, +) -> Result<(usize, impl Stream + Send + 'a)> { + let pdu_ids: Vec<_> = self.search_pdu_ids(query).await?.collect().await; - let words: Vec<_> = tokenize(search_string).collect(); - - let bufs: Vec<_> = words - .clone() + let count = pdu_ids.len(); + let pdus = pdu_ids .into_iter() .stream() - .then(move |word| { - let mut prefix2 = prefix.clone(); - prefix2.extend_from_slice(word.as_bytes()); - prefix2.push(0xFF); - let prefix3 = prefix2.clone(); - - let mut last_possible_id = prefix2.clone(); - last_possible_id.extend_from_slice(&u64::MAX.to_be_bytes()); - - self.db.tokenids - .rev_raw_keys_from(&last_possible_id) // Newest pdus first - .ignore_err() - .ready_take_while(move |key| key.starts_with(&prefix2)) - .map(move |key| key[prefix3.len()..].to_vec()) - .collect::>() + .filter_map(move |result_pdu_id: RawPduId| async move { + self.services + .timeline + .get_pdu_from_id(&result_pdu_id) + .await + .ok() }) - .collect() - .await; + .ready_filter(|pdu| !pdu.is_redacted()) + .filter_map(move |pdu| async move { + self.services + .state_accessor + .user_can_see_event(query.user_id?, &pdu.room_id, &pdu.event_id) + .await + .then_some(pdu) + }) + .skip(query.skip) + .take(query.limit); - let bufs = bufs.iter().map(|buf| buf.iter()); + Ok((count, pdus)) +} - let results = set::intersection(bufs).cloned().collect(); +// result is modeled as a stream such that callers don't have to be refactored +// though an additional async/wrap still exists for now +#[implement(Service)] +pub async fn search_pdu_ids(&self, query: &RoomQuery<'_>) -> Result + Send + '_> { + let shortroomid = self.services.short.get_shortroomid(query.room_id).await?; - Some((results, words)) + let pdu_ids = self.search_pdu_ids_query_room(query, shortroomid).await; + + let iters = pdu_ids.into_iter().map(IntoIterator::into_iter); + + Ok(set::intersection(iters).stream()) +} + +#[implement(Service)] +async fn search_pdu_ids_query_room(&self, query: &RoomQuery<'_>, shortroomid: ShortRoomId) -> Vec> { + tokenize(&query.criteria.search_term) + .stream() + .then(|word| async move { + self.search_pdu_ids_query_words(shortroomid, &word) + .collect::>() + .await + }) + .collect::>() + .await +} + +/// Iterate over PduId's containing a word +#[implement(Service)] +fn search_pdu_ids_query_words<'a>( + &'a self, shortroomid: ShortRoomId, word: &'a str, +) -> impl Stream + Send + '_ { + self.search_pdu_ids_query_word(shortroomid, word) + .ready_filter_map(move |key| { + key[prefix_len(word)..] + .chunks_exact(PduId::LEN) + .next() + .map(RawPduId::try_from) + .and_then(Result::ok) + }) +} + +/// Iterate over raw database results for a word +#[implement(Service)] +fn search_pdu_ids_query_word(&self, shortroomid: ShortRoomId, word: &str) -> impl Stream> + Send + '_ { + const PDUID_LEN: usize = PduId::LEN; + // rustc says const'ing this not yet stable + let end_id: ArrayVec = iter::repeat(u8::MAX).take(PduId::LEN).collect(); + + // Newest pdus first + let end = make_tokenid(shortroomid, word, end_id.as_slice()); + let prefix = make_prefix(shortroomid, word); + self.db + .tokenids + .rev_raw_keys_from(&end) + .ignore_err() + .ready_take_while(move |key| key.starts_with(&prefix)) } /// Splits a string into tokens used as keys in the search inverted index @@ -119,6 +191,28 @@ pub async fn search_pdus(&self, room_id: &RoomId, search_string: &str) -> Option fn tokenize(body: &str) -> impl Iterator + Send + '_ { body.split_terminator(|c: char| !c.is_alphanumeric()) .filter(|s| !s.is_empty()) - .filter(|word| word.len() <= 50) + .filter(|word| word.len() <= WORD_MAX_LEN) .map(str::to_lowercase) } + +fn make_tokenid(shortroomid: ShortRoomId, word: &str, pdu_id: &[u8]) -> TokenId { + debug_assert!(pdu_id.len() == PduId::LEN, "pdu_id size mismatch"); + + let mut key = make_prefix(shortroomid, word); + key.extend_from_slice(pdu_id); + key +} + +fn make_prefix(shortroomid: ShortRoomId, word: &str) -> TokenId { + let mut key = TokenId::new(); + key.extend_from_slice(&shortroomid.to_be_bytes()); + key.extend_from_slice(word.as_bytes()); + key.push(database::SEP); + key +} + +fn prefix_len(word: &str) -> usize { + size_of::() + .saturating_add(word.len()) + .saturating_add(1) +} From d281b8d3ae1818ea84be11ba38ac0325aaa84ffc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 26 Oct 2024 22:22:10 +0000 Subject: [PATCH 0966/2091] implement filters for search (#596) closes #596 Signed-off-by: Jason Volk --- src/service/rooms/search/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 8882ec99..70daded1 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -113,6 +113,7 @@ pub async fn search_pdus<'a>( .ok() }) .ready_filter(|pdu| !pdu.is_redacted()) + .ready_filter(|pdu| pdu.matches(&query.criteria.filter)) .filter_map(move |pdu| async move { self.services .state_accessor From 5e6dbaa27f5e08556422ee6b756efdc318654fd7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 27 Oct 2024 01:48:57 +0000 Subject: [PATCH 0967/2091] apply room event filter to messages endpoint (#596) Signed-off-by: Jason Volk --- src/api/client/message.rs | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 578b675b..094daa30 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -9,13 +9,13 @@ use conduit::{ use futures::{FutureExt, StreamExt}; use ruma::{ api::client::{ - filter::{RoomEventFilter, UrlFilter}, + filter::RoomEventFilter, message::{get_message_events, send_message_event}, }, events::{MessageLikeEventType, StateEventType, TimelineEventType::*}, UserId, }; -use serde_json::{from_str, Value}; +use serde_json::from_str; use service::rooms::timeline::PdusIterItem; use crate::{ @@ -151,7 +151,7 @@ pub(crate) async fn get_message_events_route( .timeline .pdus_after(sender_user, room_id, from) .await? - .ready_filter_map(|item| contains_url_filter(item, filter)) + .ready_filter_map(|item| event_filter(item, filter)) .filter_map(|item| visibility_filter(&services, item, sender_user)) .ready_take_while(|(count, _)| Some(*count) != to) // Stop at `to` .take(limit) @@ -225,7 +225,7 @@ pub(crate) async fn get_message_events_route( .timeline .pdus_until(sender_user, room_id, from) .await? - .ready_filter_map(|item| contains_url_filter(item, filter)) + .ready_filter_map(|item| event_filter(item, filter)) .filter_map(|(count, pdu)| async move { // list of safe and common non-state events to ignore if matches!( @@ -329,19 +329,7 @@ async fn visibility_filter(services: &Services, item: PdusIterItem, user_id: &Us .then_some(item) } -fn contains_url_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Option { +fn event_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Option { let (_, pdu) = &item; - - if filter.url_filter.is_none() { - return Some(item); - } - - let content: Value = from_str(pdu.content.get()).unwrap(); - let res = match filter.url_filter { - Some(UrlFilter::EventsWithoutUrl) => !content["url"].is_string(), - Some(UrlFilter::EventsWithUrl) => content["url"].is_string(), - None => true, - }; - - res.then_some(item) + pdu.matches(filter).then_some(item) } From 9787dfe77c7de1ef186c8bb934ba242f856ccc12 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 27 Oct 2024 00:30:30 +0000 Subject: [PATCH 0968/2091] fix clippy::ref_option fix needless borrow fix clippy::nonminimal_bool --- Cargo.toml | 1 + src/api/client/config.rs | 6 +++--- src/api/client/report.rs | 4 ++-- src/api/client/room.rs | 12 ++++++------ src/api/client/sync/v4.rs | 2 +- src/core/utils/future/try_ext_ext.rs | 1 + src/core/utils/stream/ready.rs | 1 + src/core/utils/stream/try_ready.rs | 1 + src/macros/config.rs | 18 ++++-------------- src/service/admin/mod.rs | 4 ++-- src/service/services.rs | 4 ++-- 11 files changed, 24 insertions(+), 30 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 73f16daf..2f9f196b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -772,6 +772,7 @@ unused-qualifications = "warn" #unused-results = "warn" # TODO ## some sadness +elided_named_lifetimes = "allow" # TODO! let_underscore_drop = "allow" missing_docs = "allow" # cfgs cannot be limited to expected cfgs or their de facto non-transitive/opt-in use-case e.g. diff --git a/src/api/client/config.rs b/src/api/client/config.rs index d06cc072..3cf71135 100644 --- a/src/api/client/config.rs +++ b/src/api/client/config.rs @@ -23,7 +23,7 @@ pub(crate) async fn set_global_account_data_route( set_account_data( &services, None, - &body.sender_user, + body.sender_user.as_ref(), &body.event_type.to_string(), body.data.json(), ) @@ -41,7 +41,7 @@ pub(crate) async fn set_room_account_data_route( set_account_data( &services, Some(&body.room_id), - &body.sender_user, + body.sender_user.as_ref(), &body.event_type.to_string(), body.data.json(), ) @@ -89,7 +89,7 @@ pub(crate) async fn get_room_account_data_route( } async fn set_account_data( - services: &Services, room_id: Option<&RoomId>, sender_user: &Option, event_type: &str, + services: &Services, room_id: Option<&RoomId>, sender_user: Option<&OwnedUserId>, event_type: &str, data: &RawJsonValue, ) -> Result<()> { let sender_user = sender_user.as_ref().expect("user is authenticated"); diff --git a/src/api/client/report.rs b/src/api/client/report.rs index cf789246..143c13e5 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -101,7 +101,7 @@ pub(crate) async fn report_event_route( &pdu.event_id, &body.room_id, sender_user, - &body.reason, + body.reason.as_ref(), body.score, &pdu, ) @@ -134,7 +134,7 @@ pub(crate) async fn report_event_route( /// check if report reasoning is less than or equal to 750 characters /// check if reporting user is in the reporting room async fn is_event_report_valid( - services: &Services, event_id: &EventId, room_id: &RoomId, sender_user: &UserId, reason: &Option, + services: &Services, event_id: &EventId, room_id: &RoomId, sender_user: &UserId, reason: Option<&String>, score: Option, pdu: &std::sync::Arc, ) -> Result<()> { debug_info!("Checking if report from user {sender_user} for event {event_id} in room {room_id} is valid"); diff --git a/src/api/client/room.rs b/src/api/client/room.rs index daadb724..4224d3fa 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -126,8 +126,8 @@ pub(crate) async fn create_room_route( .await; let state_lock = services.rooms.state.mutex.lock(&room_id).await; - let alias: Option = if let Some(alias) = &body.room_alias_name { - Some(room_alias_check(&services, alias, &body.appservice_info).await?) + let alias: Option = if let Some(alias) = body.room_alias_name.as_ref() { + Some(room_alias_check(&services, alias, body.appservice_info.as_ref()).await?) } else { None }; @@ -270,7 +270,7 @@ pub(crate) async fn create_room_route( } let power_levels_content = - default_power_levels_content(&body.power_level_content_override, &body.visibility, users)?; + default_power_levels_content(body.power_level_content_override.as_ref(), &body.visibility, users)?; services .rooms @@ -814,7 +814,7 @@ pub(crate) async fn upgrade_room_route( /// creates the power_levels_content for the PDU builder fn default_power_levels_content( - power_level_content_override: &Option>, visibility: &room::Visibility, + power_level_content_override: Option<&Raw>, visibility: &room::Visibility, users: BTreeMap, ) -> Result { let mut power_levels_content = serde_json::to_value(RoomPowerLevelsEventContent { @@ -864,7 +864,7 @@ fn default_power_levels_content( /// if a room is being created with a room alias, run our checks async fn room_alias_check( - services: &Services, room_alias_name: &str, appservice_info: &Option, + services: &Services, room_alias_name: &str, appservice_info: Option<&RegistrationInfo>, ) -> Result { // Basic checks on the room alias validity if room_alias_name.contains(':') { @@ -905,7 +905,7 @@ async fn room_alias_check( return Err(Error::BadRequest(ErrorKind::RoomInUse, "Room alias already exists.")); } - if let Some(ref info) = appservice_info { + if let Some(info) = appservice_info { if !info.aliases.is_match(full_room_alias.as_str()) { return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias is not in namespace.")); } diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 4f8323e6..f8ada81c 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -560,7 +560,7 @@ pub(crate) async fn sync_events_v4_route( for (_, pdu) in timeline_pdus { let ts = MilliSecondsSinceUnixEpoch(pdu.origin_server_ts); - if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) && !timestamp.is_some_and(|time| time > ts) { + if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) && timestamp.is_none_or(|time| time <= ts) { timestamp = Some(ts); } } diff --git a/src/core/utils/future/try_ext_ext.rs b/src/core/utils/future/try_ext_ext.rs index 7c0b36a2..f97ae885 100644 --- a/src/core/utils/future/try_ext_ext.rs +++ b/src/core/utils/future/try_ext_ext.rs @@ -1,4 +1,5 @@ //! Extended external extensions to futures::TryFutureExt +#![allow(clippy::type_complexity)] use futures::{ future::{MapOkOrElse, UnwrapOrElse}, diff --git a/src/core/utils/stream/ready.rs b/src/core/utils/stream/ready.rs index da5aec5a..c16d1246 100644 --- a/src/core/utils/stream/ready.rs +++ b/src/core/utils/stream/ready.rs @@ -1,4 +1,5 @@ //! Synchronous combinator extensions to futures::Stream +#![allow(clippy::type_complexity)] use futures::{ future::{ready, Ready}, diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs index df356456..feb38067 100644 --- a/src/core/utils/stream/try_ready.rs +++ b/src/core/utils/stream/try_ready.rs @@ -1,4 +1,5 @@ //! Synchronous combinator extensions to futures::TryStream +#![allow(clippy::type_complexity)] use futures::{ future::{ready, Ready}, diff --git a/src/macros/config.rs b/src/macros/config.rs index f8616352..6ccdb73c 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -164,11 +164,11 @@ fn get_default(field: &Field) -> Option { continue; }; - if !path + if path .segments .iter() .next() - .is_some_and(|s| s.ident == "serde") + .is_none_or(|s| s.ident == "serde") { continue; } @@ -218,12 +218,7 @@ fn get_doc_default(field: &Field) -> Option { continue; }; - if !path - .segments - .iter() - .next() - .is_some_and(|s| s.ident == "doc") - { + if path.segments.iter().next().is_none_or(|s| s.ident == "doc") { continue; } @@ -266,12 +261,7 @@ fn get_doc_comment(field: &Field) -> Option { continue; }; - if !path - .segments - .iter() - .next() - .is_some_and(|s| s.ident == "doc") - { + if path.segments.iter().next().is_none_or(|s| s.ident == "doc") { continue; } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 58cc012c..2860bd1b 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -370,9 +370,9 @@ impl Service { /// Sets the self-reference to crate::Services which will provide context to /// the admin commands. - pub(super) fn set_services(&self, services: &Option>) { + pub(super) fn set_services(&self, services: Option<&Arc>) { let receiver = &mut *self.services.services.write().expect("locked for writing"); - let weak = services.as_ref().map(Arc::downgrade); + let weak = services.map(Arc::downgrade); *receiver = weak; } } diff --git a/src/service/services.rs b/src/service/services.rs index ea81f434..c0af4249 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -113,7 +113,7 @@ impl Services { pub async fn start(self: &Arc) -> Result> { debug_info!("Starting services..."); - self.admin.set_services(&Some(Arc::clone(self))); + self.admin.set_services(Some(Arc::clone(self)).as_ref()); globals::migrations::migrations(self).await?; self.manager .lock() @@ -151,7 +151,7 @@ impl Services { manager.stop().await; } - self.admin.set_services(&None); + self.admin.set_services(None); debug_info!("Services shutdown complete."); } From e7e606300f33410bfb6bfdf7c9671b210e37f287 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 27 Oct 2024 19:17:41 +0000 Subject: [PATCH 0969/2091] slightly simplify reqwest/hickory hooks Signed-off-by: Jason Volk --- src/service/resolver/dns.rs | 59 ++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index b77bbb84..89129e03 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -1,15 +1,11 @@ -use std::{ - future, iter, - net::{IpAddr, SocketAddr}, - sync::Arc, - time::Duration, -}; +use std::{iter, net::SocketAddr, sync::Arc, time::Duration}; use conduit::{err, Result, Server}; +use futures::FutureExt; use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; -use super::cache::Cache; +use super::cache::{Cache, CachedOverride}; pub struct Resolver { pub(crate) resolver: Arc, @@ -21,6 +17,8 @@ pub(crate) struct Hooked { cache: Arc, } +type ResolvingResult = Result>; + impl Resolver { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] pub(super) fn build(server: &Arc, cache: Arc) -> Result> { @@ -82,12 +80,12 @@ impl Resolver { } impl Resolve for Resolver { - fn resolve(&self, name: Name) -> Resolving { resolve_to_reqwest(self.resolver.clone(), name) } + fn resolve(&self, name: Name) -> Resolving { resolve_to_reqwest(self.resolver.clone(), name).boxed() } } impl Resolve for Hooked { fn resolve(&self, name: Name) -> Resolving { - let cached = self + let cached: Option = self .cache .overrides .read() @@ -95,35 +93,30 @@ impl Resolve for Hooked { .get(name.as_str()) .cloned(); - if let Some(cached) = cached { - cached_to_reqwest(&cached.ips, cached.port) - } else { - resolve_to_reqwest(self.resolver.clone(), name) - } + cached.map_or_else( + || resolve_to_reqwest(self.resolver.clone(), name).boxed(), + |cached| cached_to_reqwest(cached).boxed(), + ) } } -fn cached_to_reqwest(override_name: &[IpAddr], port: u16) -> Resolving { - override_name +async fn cached_to_reqwest(cached: CachedOverride) -> ResolvingResult { + let first_ip = cached + .ips .first() - .map(|first_name| -> Resolving { - let saddr = SocketAddr::new(*first_name, port); - let result: Box + Send> = Box::new(iter::once(saddr)); - Box::pin(future::ready(Ok(result))) - }) - .expect("must provide at least one override name") + .expect("must provide at least one override"); + + let saddr = SocketAddr::new(*first_ip, cached.port); + + Ok(Box::new(iter::once(saddr))) } -fn resolve_to_reqwest(resolver: Arc, name: Name) -> Resolving { - Box::pin(async move { - let results = resolver - .lookup_ip(name.as_str()) - .await? - .into_iter() - .map(|ip| SocketAddr::new(ip, 0)); +async fn resolve_to_reqwest(resolver: Arc, name: Name) -> ResolvingResult { + let results = resolver + .lookup_ip(name.as_str()) + .await? + .into_iter() + .map(|ip| SocketAddr::new(ip, 0)); - let results: Addrs = Box::new(results); - - Ok(results) - }) + Ok(Box::new(results)) } From 6c9ecb031a62db0c589f383f7effe01ea30f38ce Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 27 Oct 2024 20:53:22 +0000 Subject: [PATCH 0970/2091] re-export ruma Event trait through core pdu Signed-off-by: Jason Volk --- src/core/mod.rs | 2 +- src/core/pdu/{state_res.rs => event.rs} | 5 +++-- src/core/pdu/mod.rs | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) rename src/core/pdu/{state_res.rs => event.rs} (85%) diff --git a/src/core/mod.rs b/src/core/mod.rs index d201709b..1b7b8fa1 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -17,7 +17,7 @@ pub use ::tracing; pub use config::Config; pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; -pub use pdu::{PduBuilder, PduCount, PduEvent}; +pub use pdu::{Event, PduBuilder, PduCount, PduEvent}; pub use server::Server; pub use utils::{ctor, dtor, implement, result, result::Result}; diff --git a/src/core/pdu/state_res.rs b/src/core/pdu/event.rs similarity index 85% rename from src/core/pdu/state_res.rs rename to src/core/pdu/event.rs index a27c9822..15117f92 100644 --- a/src/core/pdu/state_res.rs +++ b/src/core/pdu/event.rs @@ -1,11 +1,12 @@ use std::sync::Arc; -use ruma::{events::TimelineEventType, state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId}; +pub use ruma::state_res::Event; +use ruma::{events::TimelineEventType, EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId}; use serde_json::value::RawValue as RawJsonValue; use super::PduEvent; -impl state_res::Event for PduEvent { +impl Event for PduEvent { type Id = Arc; fn event_id(&self) -> &Self::Id { &self.event_id } diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index ed11adbb..9c3aaf9b 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -1,10 +1,10 @@ mod builder; mod content; mod count; +mod event; mod filter; mod id; mod redact; -mod state_res; mod strip; mod unsigned; @@ -19,6 +19,7 @@ use serde_json::value::RawValue as RawJsonValue; pub use self::{ builder::{Builder, Builder as PduBuilder}, count::PduCount, + event::Event, id::*, }; use crate::Result; From 7a09ac81e039a5ac1dc6d7e215824599b00aed36 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 27 Oct 2024 20:13:10 +0000 Subject: [PATCH 0971/2091] split send from messages; refactor client/messages; add filters to client/context Signed-off-by: Jason Volk --- src/api/client/context.rs | 264 +++++++++------------- src/api/client/message.rs | 457 +++++++++++++++----------------------- src/api/client/mod.rs | 2 + src/api/client/send.rs | 92 ++++++++ src/api/router/args.rs | 26 ++- 5 files changed, 403 insertions(+), 438 deletions(-) create mode 100644 src/api/client/send.rs diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 9a5c4e82..9bf0c467 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,14 +1,25 @@ -use std::collections::HashSet; +use std::iter::once; use axum::extract::State; -use conduit::{err, error, Err}; -use futures::StreamExt; +use conduit::{ + err, error, + utils::{future::TryExtExt, stream::ReadyExt, IterStream}, + Err, Result, +}; +use futures::{future::try_join, StreamExt, TryFutureExt}; use ruma::{ api::client::{context::get_context, filter::LazyLoadOptions}, - events::{StateEventType, TimelineEventType::*}, + events::StateEventType, + UserId, }; -use crate::{Result, Ruma}; +use crate::{ + client::message::{event_filter, ignored_filter, update_lazy, visibility_filter, LazySet}, + Ruma, +}; + +const LIMIT_MAX: usize = 100; +const LIMIT_DEFAULT: usize = 10; /// # `GET /_matrix/client/r0/rooms/{roomId}/context/{eventId}` /// @@ -19,33 +30,43 @@ use crate::{Result, Ruma}; pub(crate) async fn get_context_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + let filter = &body.filter; + let sender = body.sender(); + let (sender_user, _) = sender; + + // Use limit or else 10, with maximum 100 + let limit: usize = body + .limit + .try_into() + .unwrap_or(LIMIT_DEFAULT) + .min(LIMIT_MAX); // some clients, at least element, seem to require knowledge of redundant // members for "inline" profiles on the timeline to work properly - let (lazy_load_enabled, lazy_load_send_redundant) = match &body.filter.lazy_load_options { - LazyLoadOptions::Enabled { - include_redundant_members, - } => (true, *include_redundant_members), - LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")), - }; + let lazy_load_enabled = matches!(filter.lazy_load_options, LazyLoadOptions::Enabled { .. }); - let mut lazy_loaded = HashSet::with_capacity(100); + let lazy_load_redundant = if let LazyLoadOptions::Enabled { + include_redundant_members, + } = filter.lazy_load_options + { + include_redundant_members + } else { + false + }; let base_token = services .rooms .timeline .get_pdu_count(&body.event_id) - .await - .map_err(|_| err!(Request(NotFound("Base event id not found."))))?; + .map_err(|_| err!(Request(NotFound("Event not found.")))); let base_event = services .rooms .timeline .get_pdu(&body.event_id) - .await - .map_err(|_| err!(Request(NotFound("Base event not found."))))?; + .map_err(|_| err!(Request(NotFound("Base event not found.")))); + + let (base_token, base_event) = try_join(base_token, base_event).await?; let room_id = &base_event.room_id; @@ -58,136 +79,50 @@ pub(crate) async fn get_context_route( return Err!(Request(Forbidden("You don't have permission to view this event."))); } - if !services - .rooms - .lazy_loading - .lazy_load_was_sent_before(sender_user, sender_device, room_id, &base_event.sender) - .await || lazy_load_send_redundant - { - lazy_loaded.insert(base_event.sender.as_str().to_owned()); - } - - // Use limit or else 10, with maximum 100 - let limit = usize::try_from(body.limit).unwrap_or(10).min(100); - - let base_event = base_event.to_room_event(); - let events_before: Vec<_> = services .rooms .timeline .pdus_until(sender_user, room_id, base_token) .await? + .ready_filter_map(|item| event_filter(item, filter)) + .filter_map(|item| ignored_filter(&services, item, sender_user)) + .filter_map(|item| visibility_filter(&services, item, sender_user)) .take(limit / 2) - .filter_map(|(count, pdu)| async move { - // list of safe and common non-state events to ignore - if matches!( - &pdu.kind, - RoomMessage - | Sticker | CallInvite - | CallNotify | RoomEncrypted - | Image | File | Audio - | Voice | Video | UnstablePollStart - | PollStart | KeyVerificationStart - | Reaction | Emote - | Location - ) && services - .users - .user_is_ignored(&pdu.sender, sender_user) - .await - { - return None; - } - - services - .rooms - .state_accessor - .user_can_see_event(sender_user, room_id, &pdu.event_id) - .await - .then_some((count, pdu)) - }) .collect() .await; - for (_, event) in &events_before { - if !services - .rooms - .lazy_loading - .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) - .await || lazy_load_send_redundant - { - lazy_loaded.insert(event.sender.as_str().to_owned()); - } - } - - let start_token = events_before - .last() - .map_or_else(|| base_token.stringify(), |(count, _)| count.stringify()); - let events_after: Vec<_> = services .rooms .timeline .pdus_after(sender_user, room_id, base_token) .await? + .ready_filter_map(|item| event_filter(item, filter)) + .filter_map(|item| ignored_filter(&services, item, sender_user)) + .filter_map(|item| visibility_filter(&services, item, sender_user)) .take(limit / 2) - .filter_map(|(count, pdu)| async move { - // list of safe and common non-state events to ignore - if matches!( - &pdu.kind, - RoomMessage - | Sticker | CallInvite - | CallNotify | RoomEncrypted - | Image | File | Audio - | Voice | Video | UnstablePollStart - | PollStart | KeyVerificationStart - | Reaction | Emote - | Location - ) && services - .users - .user_is_ignored(&pdu.sender, sender_user) - .await - { - return None; - } - - services - .rooms - .state_accessor - .user_can_see_event(sender_user, room_id, &pdu.event_id) - .await - .then_some((count, pdu)) - }) .collect() .await; - for (_, event) in &events_after { - if !services - .rooms - .lazy_loading - .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) - .await || lazy_load_send_redundant - { - lazy_loaded.insert(event.sender.as_str().to_owned()); - } - } + let lazy = once(&(base_token, (*base_event).clone())) + .chain(events_before.iter()) + .chain(events_after.iter()) + .stream() + .fold(LazySet::new(), |lazy, item| { + update_lazy(&services, room_id, sender, lazy, item, lazy_load_redundant) + }) + .await; + + let state_id = events_after + .last() + .map_or(body.event_id.as_ref(), |(_, e)| e.event_id.as_ref()); let shortstatehash = services .rooms .state_accessor - .pdu_shortstatehash( - events_after - .last() - .map_or(&*body.event_id, |(_, e)| &*e.event_id), - ) + .pdu_shortstatehash(state_id) + .or_else(|_| services.rooms.state.get_room_shortstatehash(room_id)) .await - .map_or( - services - .rooms - .state - .get_room_shortstatehash(room_id) - .await - .expect("All rooms have state"), - |hash| hash, - ); + .map_err(|e| err!(Database("State hash not found: {e}")))?; let state_ids = services .rooms @@ -196,48 +131,61 @@ pub(crate) async fn get_context_route( .await .map_err(|e| err!(Database("State not found: {e}")))?; - let end_token = events_after - .last() - .map_or_else(|| base_token.stringify(), |(count, _)| count.stringify()); + let lazy = &lazy; + let state: Vec<_> = state_ids + .iter() + .stream() + .filter_map(|(shortstatekey, event_id)| { + services + .rooms + .short + .get_statekey_from_short(*shortstatekey) + .map_ok(move |(event_type, state_key)| (event_type, state_key, event_id)) + .ok() + }) + .filter_map(|(event_type, state_key, event_id)| async move { + if lazy_load_enabled && event_type == StateEventType::RoomMember { + let user_id: &UserId = state_key.as_str().try_into().ok()?; + if !lazy.contains(user_id) { + return None; + } + } - let mut state = Vec::with_capacity(state_ids.len()); - - for (shortstatekey, id) in state_ids { - let (event_type, state_key) = services - .rooms - .short - .get_statekey_from_short(shortstatekey) - .await?; - - if event_type != StateEventType::RoomMember { - let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { - error!("Pdu in state not found: {id}"); - continue; - }; - - state.push(pdu.to_state_event()); - } else if !lazy_load_enabled || lazy_loaded.contains(&state_key) { - let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { - error!("Pdu in state not found: {id}"); - continue; - }; - - state.push(pdu.to_state_event()); - } - } + services + .rooms + .timeline + .get_pdu(event_id) + .await + .inspect_err(|_| error!("Pdu in state not found: {event_id}")) + .map(|pdu| pdu.to_state_event()) + .ok() + }) + .collect() + .await; Ok(get_context::v3::Response { - start: Some(start_token), - end: Some(end_token), + event: Some(base_event.to_room_event()), + + start: events_before + .last() + .map_or_else(|| base_token.stringify(), |(count, _)| count.stringify()) + .into(), + + end: events_after + .last() + .map_or_else(|| base_token.stringify(), |(count, _)| count.stringify()) + .into(), + events_before: events_before - .iter() + .into_iter() .map(|(_, pdu)| pdu.to_room_event()) .collect(), - event: Some(base_event), + events_after: events_after - .iter() + .into_iter() .map(|(_, pdu)| pdu.to_room_event()) .collect(), + state, }) } diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 094daa30..4fc58d9f 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,111 +1,52 @@ -use std::collections::{BTreeMap, HashSet}; +use std::collections::HashSet; use axum::extract::State; use conduit::{ - err, - utils::{IterStream, ReadyExt}, - Err, PduCount, + at, is_equal_to, + utils::{ + result::{FlatOk, LogErr}, + IterStream, ReadyExt, + }, + Event, PduCount, Result, }; use futures::{FutureExt, StreamExt}; use ruma::{ - api::client::{ - filter::RoomEventFilter, - message::{get_message_events, send_message_event}, + api::{ + client::{filter::RoomEventFilter, message::get_message_events}, + Direction, }, - events::{MessageLikeEventType, StateEventType, TimelineEventType::*}, - UserId, + events::{AnyStateEvent, StateEventType, TimelineEventType, TimelineEventType::*}, + serde::Raw, + DeviceId, OwnedUserId, RoomId, UserId, }; -use serde_json::from_str; -use service::rooms::timeline::PdusIterItem; +use service::{rooms::timeline::PdusIterItem, Services}; -use crate::{ - service::{pdu::PduBuilder, Services}, - utils, Result, Ruma, -}; +use crate::Ruma; -/// # `PUT /_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}` -/// -/// Send a message event into the room. -/// -/// - Is a NOOP if the txn id was already used before and returns the same event -/// id again -/// - The only requirement for the content is that it has to be valid json -/// - Tries to send the event into the room, auth rules will determine if it is -/// allowed -pub(crate) async fn send_message_event_route( - State(services): State, body: Ruma, -) -> Result { - let sender_user = body.sender_user.as_deref().expect("user is authenticated"); - let sender_device = body.sender_device.as_deref(); - let appservice_info = body.appservice_info.as_ref(); +pub(crate) type LazySet = HashSet; - // Forbid m.room.encrypted if encryption is disabled - if MessageLikeEventType::RoomEncrypted == body.event_type && !services.globals.allow_encryption() { - return Err!(Request(Forbidden("Encryption has been disabled"))); - } +/// list of safe and common non-state events to ignore +const IGNORED_MESSAGE_TYPES: &[TimelineEventType] = &[ + RoomMessage, + Sticker, + CallInvite, + CallNotify, + RoomEncrypted, + Image, + File, + Audio, + Voice, + Video, + UnstablePollStart, + PollStart, + KeyVerificationStart, + Reaction, + Emote, + Location, +]; - let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - - if body.event_type == MessageLikeEventType::CallInvite - && services.rooms.directory.is_public_room(&body.room_id).await - { - return Err!(Request(Forbidden("Room call invites are not allowed in public rooms"))); - } - - // Check if this is a new transaction id - if let Ok(response) = services - .transaction_ids - .existing_txnid(sender_user, sender_device, &body.txn_id) - .await - { - // The client might have sent a txnid of the /sendToDevice endpoint - // This txnid has no response associated with it - if response.is_empty() { - return Err!(Request(InvalidParam( - "Tried to use txn id already used for an incompatible endpoint." - ))); - } - - return Ok(send_message_event::v3::Response { - event_id: utils::string_from_bytes(&response) - .map(TryInto::try_into) - .map_err(|e| err!(Database("Invalid event_id in txnid data: {e:?}")))??, - }); - } - - let mut unsigned = BTreeMap::new(); - unsigned.insert("transaction_id".to_owned(), body.txn_id.to_string().into()); - - let content = - from_str(body.body.body.json().get()).map_err(|e| err!(Request(BadJson("Invalid JSON body: {e}"))))?; - - let event_id = services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: body.event_type.clone().into(), - content, - unsigned: Some(unsigned), - timestamp: appservice_info.and(body.timestamp), - ..Default::default() - }, - sender_user, - &body.room_id, - &state_lock, - ) - .await?; - - services - .transaction_ids - .add_txnid(sender_user, sender_device, &body.txn_id, event_id.as_bytes()); - - drop(state_lock); - - Ok(send_message_event::v3::Response { - event_id: event_id.into(), - }) -} +const LIMIT_MAX: usize = 100; +const LIMIT_DEFAULT: usize = 10; /// # `GET /_matrix/client/r0/rooms/{roomId}/messages` /// @@ -116,209 +57,171 @@ pub(crate) async fn send_message_event_route( pub(crate) async fn get_message_events_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); - + let sender = body.sender(); + let (sender_user, sender_device) = sender; let room_id = &body.room_id; let filter = &body.filter; - let limit = usize::try_from(body.limit).unwrap_or(10).min(100); - let from = match body.from.as_ref() { - Some(from) => PduCount::try_from_string(from)?, - None => match body.dir { - ruma::api::Direction::Forward => PduCount::min(), - ruma::api::Direction::Backward => PduCount::max(), - }, + let from_default = match body.dir { + Direction::Forward => PduCount::min(), + Direction::Backward => PduCount::max(), }; - let to = body - .to - .as_ref() - .and_then(|t| PduCount::try_from_string(t).ok()); + let from = body + .from + .as_deref() + .map(PduCount::try_from_string) + .transpose()? + .unwrap_or(from_default); + + let to = body.to.as_deref().map(PduCount::try_from_string).flat_ok(); + + let limit: usize = body + .limit + .try_into() + .unwrap_or(LIMIT_DEFAULT) + .min(LIMIT_MAX); services .rooms .lazy_loading .lazy_load_confirm_delivery(sender_user, sender_device, room_id, from); - let mut resp = get_message_events::v3::Response::new(); - let mut lazy_loaded = HashSet::new(); - let next_token; - match body.dir { - ruma::api::Direction::Forward => { - let events_after: Vec = services - .rooms - .timeline - .pdus_after(sender_user, room_id, from) - .await? - .ready_filter_map(|item| event_filter(item, filter)) - .filter_map(|item| visibility_filter(&services, item, sender_user)) - .ready_take_while(|(count, _)| Some(*count) != to) // Stop at `to` - .take(limit) - .collect() - .boxed() - .await; - - for (_, event) in &events_after { - /* TODO: Remove the not "element_hacks" check when these are resolved: - * https://github.com/vector-im/element-android/issues/3417 - * https://github.com/vector-im/element-web/issues/21034 - */ - if !cfg!(feature = "element_hacks") - && !services - .rooms - .lazy_loading - .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) - .await - { - lazy_loaded.insert(event.sender.clone()); - } - - if cfg!(features = "element_hacks") { - lazy_loaded.insert(event.sender.clone()); - } - } - - next_token = events_after.last().map(|(count, _)| count).copied(); - - let events_after: Vec<_> = events_after - .into_iter() - .stream() - .filter_map(|(_, pdu)| async move { - // list of safe and common non-state events to ignore - if matches!( - &pdu.kind, - RoomMessage - | Sticker | CallInvite - | CallNotify | RoomEncrypted - | Image | File | Audio - | Voice | Video | UnstablePollStart - | PollStart | KeyVerificationStart - | Reaction | Emote | Location - ) && services - .users - .user_is_ignored(&pdu.sender, sender_user) - .await - { - return None; - } - - Some(pdu.to_room_event()) - }) - .collect() - .await; - - resp.start = from.stringify(); - resp.end = next_token.map(|count| count.stringify()); - resp.chunk = events_after; - }, - ruma::api::Direction::Backward => { - services - .rooms - .timeline - .backfill_if_required(room_id, from) - .boxed() - .await?; - - let events_before: Vec = services - .rooms - .timeline - .pdus_until(sender_user, room_id, from) - .await? - .ready_filter_map(|item| event_filter(item, filter)) - .filter_map(|(count, pdu)| async move { - // list of safe and common non-state events to ignore - if matches!( - &pdu.kind, - RoomMessage - | Sticker | CallInvite - | CallNotify | RoomEncrypted - | Image | File | Audio - | Voice | Video | UnstablePollStart - | PollStart | KeyVerificationStart - | Reaction | Emote | Location - ) && services - .users - .user_is_ignored(&pdu.sender, sender_user) - .await - { - return None; - } - - Some((count, pdu)) - }) - .filter_map(|item| visibility_filter(&services, item, sender_user)) - .ready_take_while(|(count, _)| Some(*count) != to) // Stop at `to` - .take(limit) - .collect() - .boxed() - .await; - - for (_, event) in &events_before { - /* TODO: Remove the not "element_hacks" check when these are resolved: - * https://github.com/vector-im/element-android/issues/3417 - * https://github.com/vector-im/element-web/issues/21034 - */ - if !cfg!(feature = "element_hacks") - && !services - .rooms - .lazy_loading - .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) - .await - { - lazy_loaded.insert(event.sender.clone()); - } - - if cfg!(features = "element_hacks") { - lazy_loaded.insert(event.sender.clone()); - } - } - - next_token = events_before.last().map(|(count, _)| count).copied(); - - let events_before: Vec<_> = events_before - .into_iter() - .map(|(_, pdu)| pdu.to_room_event()) - .collect(); - - resp.start = from.stringify(); - resp.end = next_token.map(|count| count.stringify()); - resp.chunk = events_before; - }, + if matches!(body.dir, Direction::Backward) { + services + .rooms + .timeline + .backfill_if_required(room_id, from) + .boxed() + .await + .log_err() + .ok(); } - resp.state = lazy_loaded - .iter() - .stream() - .filter_map(|ll_user_id| async move { - services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, ll_user_id.as_str()) - .await - .map(|member_event| member_event.to_state_event()) - .ok() - }) + let it = match body.dir { + Direction::Forward => services + .rooms + .timeline + .pdus_after(sender_user, room_id, from) + .await? + .boxed(), + + Direction::Backward => services + .rooms + .timeline + .pdus_until(sender_user, room_id, from) + .await? + .boxed(), + }; + + let events: Vec<_> = it + .ready_take_while(|(count, _)| Some(*count) != to) + .ready_filter_map(|item| event_filter(item, filter)) + .filter_map(|item| ignored_filter(&services, item, sender_user)) + .filter_map(|item| visibility_filter(&services, item, sender_user)) + .take(limit) .collect() .await; - // remove the feature check when we are sure clients like element can handle it + let lazy = events + .iter() + .stream() + .fold(LazySet::new(), |lazy, item| { + update_lazy(&services, room_id, sender, lazy, item, false) + }) + .await; + + let state = lazy + .iter() + .stream() + .filter_map(|user_id| get_member_event(&services, room_id, user_id)) + .collect() + .await; + + let next_token = events.last().map(|(count, _)| count).copied(); + if !cfg!(feature = "element_hacks") { if let Some(next_token) = next_token { - services.rooms.lazy_loading.lazy_load_mark_sent( - sender_user, - sender_device, - room_id, - lazy_loaded, - next_token, - ); + services + .rooms + .lazy_loading + .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy, next_token); } } - Ok(resp) + let chunk = events + .into_iter() + .map(at!(1)) + .map(|pdu| pdu.to_room_event()) + .collect(); + + Ok(get_message_events::v3::Response { + start: from.stringify(), + end: next_token.as_ref().map(PduCount::stringify), + chunk, + state, + }) } -async fn visibility_filter(services: &Services, item: PdusIterItem, user_id: &UserId) -> Option { +async fn get_member_event(services: &Services, room_id: &RoomId, user_id: &UserId) -> Option> { + services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str()) + .await + .map(|member_event| member_event.to_state_event()) + .ok() +} + +pub(crate) async fn update_lazy( + services: &Services, room_id: &RoomId, sender: (&UserId, &DeviceId), mut lazy: LazySet, item: &PdusIterItem, + force: bool, +) -> LazySet { + let (_, event) = &item; + let (sender_user, sender_device) = sender; + + /* TODO: Remove the not "element_hacks" check when these are resolved: + * https://github.com/vector-im/element-android/issues/3417 + * https://github.com/vector-im/element-web/issues/21034 + */ + if force || cfg!(features = "element_hacks") { + lazy.insert(event.sender().into()); + return lazy; + } + + if !services + .rooms + .lazy_loading + .lazy_load_was_sent_before(sender_user, sender_device, room_id, event.sender()) + .await + { + lazy.insert(event.sender().into()); + } + + lazy +} + +pub(crate) async fn ignored_filter(services: &Services, item: PdusIterItem, user_id: &UserId) -> Option { + let (_, pdu) = &item; + + if pdu.kind.to_cow_str() == "org.matrix.dummy_event" { + return None; + } + + if !IGNORED_MESSAGE_TYPES.iter().any(is_equal_to!(&pdu.kind)) { + return Some(item); + } + + if !services.users.user_is_ignored(&pdu.sender, user_id).await { + return Some(item); + } + + None +} + +pub(crate) async fn visibility_filter( + services: &Services, item: PdusIterItem, user_id: &UserId, +) -> Option { let (_, pdu) = &item; services @@ -329,7 +232,7 @@ async fn visibility_filter(services: &Services, item: PdusIterItem, user_id: &Us .then_some(item) } -fn event_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Option { +pub(crate) fn event_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Option { let (_, pdu) = &item; pdu.matches(filter).then_some(item) } diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index 2928be87..9ee88bec 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -23,6 +23,7 @@ pub(super) mod relations; pub(super) mod report; pub(super) mod room; pub(super) mod search; +pub(super) mod send; pub(super) mod session; pub(super) mod space; pub(super) mod state; @@ -65,6 +66,7 @@ pub(super) use relations::*; pub(super) use report::*; pub(super) use room::*; pub(super) use search::*; +pub(super) use send::*; pub(super) use session::*; pub(super) use space::*; pub(super) use state::*; diff --git a/src/api/client/send.rs b/src/api/client/send.rs new file mode 100644 index 00000000..ff011efa --- /dev/null +++ b/src/api/client/send.rs @@ -0,0 +1,92 @@ +use std::collections::BTreeMap; + +use axum::extract::State; +use conduit::{err, Err}; +use ruma::{api::client::message::send_message_event, events::MessageLikeEventType}; +use serde_json::from_str; + +use crate::{service::pdu::PduBuilder, utils, Result, Ruma}; + +/// # `PUT /_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}` +/// +/// Send a message event into the room. +/// +/// - Is a NOOP if the txn id was already used before and returns the same event +/// id again +/// - The only requirement for the content is that it has to be valid json +/// - Tries to send the event into the room, auth rules will determine if it is +/// allowed +pub(crate) async fn send_message_event_route( + State(services): State, body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + let sender_device = body.sender_device.as_deref(); + let appservice_info = body.appservice_info.as_ref(); + + // Forbid m.room.encrypted if encryption is disabled + if MessageLikeEventType::RoomEncrypted == body.event_type && !services.globals.allow_encryption() { + return Err!(Request(Forbidden("Encryption has been disabled"))); + } + + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + + if body.event_type == MessageLikeEventType::CallInvite + && services.rooms.directory.is_public_room(&body.room_id).await + { + return Err!(Request(Forbidden("Room call invites are not allowed in public rooms"))); + } + + // Check if this is a new transaction id + if let Ok(response) = services + .transaction_ids + .existing_txnid(sender_user, sender_device, &body.txn_id) + .await + { + // The client might have sent a txnid of the /sendToDevice endpoint + // This txnid has no response associated with it + if response.is_empty() { + return Err!(Request(InvalidParam( + "Tried to use txn id already used for an incompatible endpoint." + ))); + } + + return Ok(send_message_event::v3::Response { + event_id: utils::string_from_bytes(&response) + .map(TryInto::try_into) + .map_err(|e| err!(Database("Invalid event_id in txnid data: {e:?}")))??, + }); + } + + let mut unsigned = BTreeMap::new(); + unsigned.insert("transaction_id".to_owned(), body.txn_id.to_string().into()); + + let content = + from_str(body.body.body.json().get()).map_err(|e| err!(Request(BadJson("Invalid JSON body: {e}"))))?; + + let event_id = services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: body.event_type.clone().into(), + content, + unsigned: Some(unsigned), + timestamp: appservice_info.and(body.timestamp), + ..Default::default() + }, + sender_user, + &body.room_id, + &state_lock, + ) + .await?; + + services + .transaction_ids + .add_txnid(sender_user, sender_device, &body.txn_id, event_id.as_bytes()); + + drop(state_lock); + + Ok(send_message_event::v3::Response { + event_id: event_id.into(), + }) +} diff --git a/src/api/router/args.rs b/src/api/router/args.rs index cefacac1..38236db3 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -3,7 +3,9 @@ use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, BytesMut}; use conduit::{debug, err, trace, utils::string::EMPTY, Error, Result}; -use ruma::{api::IncomingRequest, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, ServerName, UserId}; +use ruma::{ + api::IncomingRequest, CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedServerName, OwnedUserId, ServerName, UserId, +}; use service::Services; use super::{auth, auth::Auth, request, request::Request}; @@ -40,10 +42,28 @@ where T: IncomingRequest + Send + Sync + 'static, { #[inline] - pub(crate) fn sender_user(&self) -> &UserId { self.sender_user.as_deref().expect("user is authenticated") } + pub(crate) fn sender(&self) -> (&UserId, &DeviceId) { (self.sender_user(), self.sender_device()) } #[inline] - pub(crate) fn origin(&self) -> &ServerName { self.origin.as_deref().expect("server is authenticated") } + pub(crate) fn sender_user(&self) -> &UserId { + self.sender_user + .as_deref() + .expect("user must be authenticated for this handler") + } + + #[inline] + pub(crate) fn sender_device(&self) -> &DeviceId { + self.sender_device + .as_deref() + .expect("user must be authenticated and device identified") + } + + #[inline] + pub(crate) fn origin(&self) -> &ServerName { + self.origin + .as_deref() + .expect("server must be authenticated for this handler") + } } #[async_trait] From 52e356d7805fd25c4e0b21757076f04d271d4241 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 28 Oct 2024 06:49:25 +0000 Subject: [PATCH 0972/2091] generate ActualDest https string on the fly Signed-off-by: Jason Volk --- src/service/resolver/actual.rs | 29 ++++++++++++++--------------- src/service/resolver/fed.rs | 19 ++++++++----------- src/service/sending/send.rs | 4 ++-- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index ea4b1100..660498f7 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -18,34 +18,33 @@ use crate::resolver::{ pub(crate) struct ActualDest { pub(crate) dest: FedDest, pub(crate) host: String, - pub(crate) string: String, pub(crate) cached: bool, } +impl ActualDest { + #[inline] + pub(crate) fn string(&self) -> String { self.dest.https_string() } +} + impl super::Service { #[tracing::instrument(skip_all, name = "resolve")] pub(crate) async fn get_actual_dest(&self, server_name: &ServerName) -> Result { - let cached; - let cached_result = self.get_cached_destination(server_name); + let (result, cached) = if let Some(result) = self.get_cached_destination(server_name) { + (result, true) + } else { + self.validate_dest(server_name)?; + (self.resolve_actual_dest(server_name, true).await?, false) + }; let CachedDest { dest, host, .. - } = if let Some(result) = cached_result { - cached = true; - result - } else { - cached = false; - self.validate_dest(server_name)?; - self.resolve_actual_dest(server_name, true).await? - }; + } = result; - let string = dest.clone().into_https_string(); Ok(ActualDest { dest, host, - string, cached, }) } @@ -89,7 +88,7 @@ impl super::Service { debug!("Actual destination: {actual_dest:?} hostname: {host:?}"); Ok(CachedDest { dest: actual_dest, - host: host.into_uri_string(), + host: host.uri_string(), expire: CachedDest::default_expire(), }) } @@ -109,7 +108,7 @@ impl super::Service { async fn actual_dest_3(&self, host: &mut String, cache: bool, delegated: String) -> Result { debug!("3: A .well-known file is available"); - *host = add_port_to_hostname(&delegated).into_uri_string(); + *host = add_port_to_hostname(&delegated).uri_string(); match get_ip_with_port(&delegated) { Some(host_and_port) => Self::actual_dest_3_1(host_and_port), None => { diff --git a/src/service/resolver/fed.rs b/src/service/resolver/fed.rs index 10cbbbdd..79f71f13 100644 --- a/src/service/resolver/fed.rs +++ b/src/service/resolver/fed.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Cow, fmt, net::{IpAddr, SocketAddr}, }; @@ -29,24 +30,25 @@ pub(crate) fn add_port_to_hostname(dest_str: &str) -> FedDest { } impl FedDest { - pub(crate) fn into_https_string(self) -> String { + pub(crate) fn https_string(&self) -> String { match self { Self::Literal(addr) => format!("https://{addr}"), Self::Named(host, port) => format!("https://{host}{port}"), } } - pub(crate) fn into_uri_string(self) -> String { + pub(crate) fn uri_string(&self) -> String { match self { Self::Literal(addr) => addr.to_string(), Self::Named(host, port) => format!("{host}{port}"), } } - pub(crate) fn hostname(&self) -> String { + #[inline] + pub(crate) fn hostname(&self) -> Cow<'_, str> { match &self { - Self::Literal(addr) => addr.ip().to_string(), - Self::Named(host, _) => host.clone(), + Self::Literal(addr) => addr.ip().to_string().into(), + Self::Named(host, _) => host.into(), } } @@ -61,10 +63,5 @@ impl FedDest { } impl fmt::Display for FedDest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Self::Named(host, port) => write!(f, "{host}{port}"), - Self::Literal(addr) => write!(f, "{addr}"), - } - } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.uri_string().as_str()) } } diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 62da59ef..2fbb3919 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -71,7 +71,7 @@ impl super::Service { trace!("Preparing request"); let mut http_request = req - .try_into_http_request::>(&actual.string, SATIR, &VERSIONS) + .try_into_http_request::>(actual.string().as_str(), SATIR, &VERSIONS) .map_err(|e| err!(BadServerResponse("Invalid destination: {e:?}")))?; self.sign_request::(dest, &mut http_request); @@ -107,7 +107,7 @@ where request_url = ?url, response_url = ?response.url(), "Received response from {}", - actual.string, + actual.string(), ); let mut http_response_builder = http::Response::builder() From d92f2c121f95f8d8beadc3a727b8a02376c46d3c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 27 Oct 2024 12:19:45 -0400 Subject: [PATCH 0973/2091] document nginx needing request_uri Signed-off-by: strawberry --- docs/deploying/generic.md | 12 ++++++++---- src/api/router/auth.rs | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 1e44ab54..31dc1845 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -119,12 +119,16 @@ is the recommended reverse proxy for new users and is very trivial to use (handles TLS, reverse proxy headers, etc transparently with proper defaults). Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization -header, making federation non-functional. If using Apache, you need to use -`nocanon` in your `ProxyPass` directive to prevent this (note that Apache -isn't very good as a general reverse proxy). +header, making federation non-functional. If a workaround is found, feel free to share to get it added to the documentation here. + +If using Apache, you need to use `nocanon` in your `ProxyPass` directive to prevent this (note that Apache isn't very good as a general reverse proxy and we discourage the usage of it if you can). + +If using Nginx, you need to give conduwuit the request URI using `$request_uri`, or like so: +- `proxy_pass http://127.0.0.1:6167$request_uri;` +- `proxy_pass http://127.0.0.1:6167;` Nginx users may need to set `proxy_buffering off;` if there are issues with -uploading media like images. +uploading media like images. This is due to Nginx storing the entire POST content in-memory (`/tmp`) and running out of memory if on low memory hardware. You will need to reverse proxy everything under following routes: - `/_matrix/` - core Matrix C-S and S-S APIs diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 6b1bb1a9..31e71f2f 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -247,8 +247,8 @@ async fn auth_server(services: &Services, request: &mut Request, body: Option<&C debug_error!("Failed to verify federation request from {origin}: {e}"); if request.parts.uri.to_string().contains('@') { warn!( - "Request uri contained '@' character. Make sure your reverse proxy gives Conduit the raw uri (apache: \ - use nocanon)" + "Request uri contained '@' character. Make sure your reverse proxy gives conduwuit the raw uri \ + (apache: use nocanon)" ); } From 065396f8f502e1b206c37b0d7dea92f79bfd8634 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 27 Oct 2024 12:37:44 -0400 Subject: [PATCH 0974/2091] better document allow_inbound_profile_lookup_federation_requests Signed-off-by: strawberry --- src/api/server/publicrooms.rs | 3 ++- src/api/server/query.rs | 6 +++++- src/core/config/mod.rs | 21 ++++++++++++++++----- src/service/globals/mod.rs | 4 ---- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index af8a5846..f6c41859 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -20,7 +20,8 @@ pub(crate) async fn get_public_rooms_filtered_route( ) -> Result { if !services .globals - .allow_public_room_directory_over_federation() + .config + .allow_public_room_directory_over_federation { return Err(Error::BadRequest(ErrorKind::forbidden(), "Room directory is not public")); } diff --git a/src/api/server/query.rs b/src/api/server/query.rs index 348b8c6e..bf515b3c 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -63,7 +63,11 @@ pub(crate) async fn get_room_information_route( pub(crate) async fn get_profile_information_route( State(services): State, body: Ruma, ) -> Result { - if !services.globals.allow_profile_lookup_federation_requests() { + if !services + .globals + .config + .allow_inbound_profile_lookup_federation_requests + { return Err(Error::BadRequest( ErrorKind::forbidden(), "Profile lookup over federation is not allowed on this homeserver.", diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 04e44fd7..7a5c6d08 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -458,11 +458,16 @@ pub struct Config { /// obtain the profiles of our local users from /// `/_matrix/federation/v1/query/profile` /// - /// This is inherently false if `allow_federation` is disabled + /// Increases privacy of your local user's such as display names, but some + /// remote users may get a false "this user does not exist" error when they + /// try to invite you to a DM or room. Also can protect against profile + /// spiders. /// - /// Defaults to true - #[serde(default = "true_fn")] - pub allow_profile_lookup_federation_requests: bool, + /// Defaults to true. + /// + /// This is inherently false if `allow_federation` is disabled + #[serde(default = "true_fn", alias = "allow_profile_lookup_federation_requests")] + pub allow_inbound_profile_lookup_federation_requests: bool, /// controls whether users are allowed to create rooms. /// appservices and admins are always allowed to create rooms @@ -1530,6 +1535,10 @@ impl fmt::Display for Config { line("Allow encryption", &self.allow_encryption.to_string()); line("Allow federation", &self.allow_federation.to_string()); line("Federation loopback", &self.federation_loopback.to_string()); + line( + "Require authentication for profile requests", + &self.require_auth_for_profile_requests.to_string(), + ); line( "Allow incoming federated presence requests (updates)", &self.allow_incoming_presence.to_string(), @@ -1577,7 +1586,9 @@ impl fmt::Display for Config { line("Allow device name federation", &self.allow_device_name_federation.to_string()); line( "Allow incoming profile lookup federation requests", - &self.allow_profile_lookup_federation_requests.to_string(), + &self + .allow_inbound_profile_lookup_federation_requests + .to_string(), ); line( "Auto deactivate banned room join attempts", diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 157c3944..0a7dda9f 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -212,10 +212,6 @@ impl Service { pub fn turn_username(&self) -> &String { &self.config.turn_username } - pub fn allow_profile_lookup_federation_requests(&self) -> bool { - self.config.allow_profile_lookup_federation_requests - } - pub fn notification_push_path(&self) -> &String { &self.config.notification_push_path } pub fn emergency_password(&self) -> &Option { &self.config.emergency_password } From 85890ed42502a4672d21218b07fc7366f7027ef3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 27 Oct 2024 13:21:16 -0400 Subject: [PATCH 0975/2091] remove some unnecessary HTML from admin commands Signed-off-by: strawberry --- src/admin/debug/commands.rs | 2 +- src/admin/federation/commands.rs | 28 +++++----------------------- src/admin/room/directory.rs | 29 +++++------------------------ src/admin/server/commands.rs | 5 ++++- src/core/config/mod.rs | 2 +- 5 files changed, 16 insertions(+), 50 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 0fd3c91b..2aa6078f 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -106,7 +106,7 @@ pub(super) async fn get_pdu(&self, event_id: Box) -> Result) -> Result { @@ -108,33 +108,15 @@ pub(super) async fn remote_user_in_rooms(&self, user_id: Box) -> Result< rooms.sort_by_key(|r| r.1); rooms.reverse(); - let output_plain = format!( - "Rooms {user_id} shares with us ({}):\n{}", + let output = format!( + "Rooms {user_id} shares with us ({}):\n```\n{}\n```", rooms.len(), rooms .iter() - .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) + .map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}")) .collect::>() .join("\n") ); - let output_html = format!( - "\n\t\t\n{}
    Rooms {user_id} shares with us \ - ({})
    idmembersname
    ", - rooms.len(), - rooms - .iter() - .fold(String::new(), |mut output, (id, members, name)| { - writeln!( - output, - "{}\t{}\t{}", - id, - members, - escape_html(name) - ) - .expect("should be able to write to string buffer"); - output - }) - ); - Ok(RoomMessageEventContent::text_html(output_plain, output_html)) + Ok(RoomMessageEventContent::text_markdown(output)) } diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index 1080356a..0bdaf56d 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -1,11 +1,9 @@ -use std::fmt::Write; - use clap::Subcommand; use conduit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId}; -use crate::{escape_html, get_room_info, Command, PAGE_SIZE}; +use crate::{get_room_info, Command, PAGE_SIZE}; #[derive(Debug, Subcommand)] pub(crate) enum RoomDirectoryCommand { @@ -68,32 +66,15 @@ pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_> return Ok(RoomMessageEventContent::text_plain("No more rooms.")); }; - let output_plain = format!( - "Rooms:\n{}", + let output = format!( + "Rooms (page {page}):\n```\n{}\n```", rooms .iter() - .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) + .map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}")) .collect::>() .join("\n") ); - let output_html = format!( - "\n\t\t\n{}
    Room directory - page \ - {page}
    idmembersname
    ", - rooms - .iter() - .fold(String::new(), |mut output, (id, members, name)| { - writeln!( - output, - "{}\t{}\t{}", - escape_html(id.as_ref()), - members, - escape_html(name.as_ref()) - ) - .expect("should be able to write to string buffer"); - output - }) - ); - Ok(RoomMessageEventContent::text_html(output_plain, output_html)) + Ok(RoomMessageEventContent::text_markdown(output)) }, } } diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index de6ad98a..f5879b03 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -21,7 +21,10 @@ pub(super) async fn uptime(&self) -> Result { #[admin_command] pub(super) async fn show_config(&self) -> Result { // Construct and send the response - Ok(RoomMessageEventContent::text_plain(format!("{}", self.services.globals.config))) + Ok(RoomMessageEventContent::text_markdown(format!( + "```\n{}\n```", + self.services.globals.config + ))) } #[admin_command] diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 7a5c6d08..512cb48b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1441,7 +1441,7 @@ impl Config { impl fmt::Display for Config { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - writeln!(f, "Active config values:\n\n").expect("wrote line to formatter stream"); + writeln!(f, "Active config values:\n").expect("wrote line to formatter stream"); let mut line = |key: &str, val: &str| { writeln!(f, "{key}: {val}").expect("wrote line to formatter stream"); }; From 0a281241efdc536d950f24edd4805364a8bbfd97 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 28 Oct 2024 16:53:53 -0400 Subject: [PATCH 0976/2091] bump few dependencies, bump ruwuma Signed-off-by: strawberry --- Cargo.lock | 68 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 10 ++++---- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8acce7d..44856753 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,7 +290,7 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-pemfile", "rustls-pki-types", "tokio", @@ -310,7 +310,7 @@ dependencies = [ "http", "http-body-util", "pin-project", - "rustls 0.23.15", + "rustls 0.23.16", "tokio", "tokio-rustls", "tokio-util", @@ -770,7 +770,7 @@ dependencies = [ "hyper-util", "log", "ruma", - "rustls 0.23.15", + "rustls 0.23.16", "sd-notify", "sentry", "sentry-tower", @@ -1202,9 +1202,9 @@ dependencies = [ [[package]] name = "fdeflate" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8090f921a24b04994d9929e204f50b498a33ea6ba559ffaa05e04f7ee7fb5ab" +checksum = "07c6f4c64c1d33a3111c4466f7365ebdcc37c5bd1ea0d62aae2e3d722aacbedb" dependencies = [ "simd-adler32", ] @@ -1708,7 +1708,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -2771,7 +2771,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.15", + "rustls 0.23.16", "socket2", "thiserror", "tokio", @@ -2788,7 +2788,7 @@ dependencies = [ "rand", "ring", "rustc-hash 2.0.0", - "rustls 0.23.15", + "rustls 0.23.16", "slab", "thiserror", "tinyvec", @@ -2902,9 +2902,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.8" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" +checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" dependencies = [ "async-compression", "base64 0.22.1", @@ -2928,7 +2928,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", @@ -2977,7 +2977,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "assign", "js_int", @@ -2999,7 +2999,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "js_int", "ruma-common", @@ -3011,7 +3011,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "as_variant", "assign", @@ -3034,7 +3034,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "as_variant", "base64 0.22.1", @@ -3064,7 +3064,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3088,7 +3088,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "bytes", "http", @@ -3106,7 +3106,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "js_int", "thiserror", @@ -3115,7 +3115,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "js_int", "ruma-common", @@ -3125,7 +3125,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "cfg-if", "once_cell", @@ -3141,7 +3141,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "js_int", "ruma-common", @@ -3153,7 +3153,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "headers", "http", @@ -3166,7 +3166,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3182,7 +3182,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=39c1addd37a4eed612ac1135edc2cccd9d331d5e#39c1addd37a4eed612ac1135edc2cccd9d331d5e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" dependencies = [ "futures-util", "itertools 0.13.0", @@ -3258,9 +3258,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.37" +version = "0.38.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" +checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" dependencies = [ "bitflags 2.6.0", "errno", @@ -3285,9 +3285,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.15" +version = "0.23.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fbb44d7acc4e873d613422379f69f237a1b141928c02f6bc6ccfddddc2d7993" +checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" dependencies = [ "aws-lc-rs", "log", @@ -3563,18 +3563,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.213" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ea7893ff5e2466df8d720bb615088341b295f849602c6956047f8f80f0e9bc1" +checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.213" +version = "1.0.214" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e85ad2009c50b58e87caa8cd6dac16bdf511bbfb7af6c33df902396aa480fa5" +checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", @@ -4106,7 +4106,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.15", + "rustls 0.23.16", "rustls-pki-types", "tokio", ] @@ -4472,7 +4472,7 @@ dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.23.15", + "rustls 0.23.16", "rustls-pki-types", "url", "webpki-roots", diff --git a/Cargo.toml b/Cargo.toml index 2f9f196b..e406c9e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -128,10 +128,10 @@ features = [ ] [workspace.dependencies.rustls] -version = "0.23.13" +version = "0.23.16" [workspace.dependencies.reqwest] -version = "0.12.8" +version = "0.12.9" default-features = false features = [ "rustls-tls-native-roots", @@ -141,7 +141,7 @@ features = [ ] [workspace.dependencies.serde] -version = "1.0.209" +version = "1.0.214" default-features = false features = ["rc"] @@ -257,7 +257,7 @@ features = [ ] [workspace.dependencies.hyper-util] -# 0.1.9 causes DNS issues +# 0.1.9 and above causes DNS issues version = "=0.1.8" default-features = false features = [ @@ -315,7 +315,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "39c1addd37a4eed612ac1135edc2cccd9d331d5e" +rev = "dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" features = [ "compat", "rand", From c71db93e225b44f15c652f4fbe0befaad508e48e Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 28 Oct 2024 18:28:56 -0400 Subject: [PATCH 0977/2091] implement admin command to force join list of local users Signed-off-by: strawberry --- src/admin/user/commands.rs | 139 ++++++++++++++++++++++++++++++++++++- src/admin/user/mod.rs | 15 ++++ 2 files changed, 153 insertions(+), 1 deletion(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index fb6d2bf1..d6946b4e 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -1,7 +1,11 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room}; -use conduit::{error, info, is_equal_to, utils, warn, PduBuilder, Result}; +use conduit::{ + debug_warn, error, info, is_equal_to, + utils::{self, ReadyExt}, + warn, PduBuilder, Result, +}; use conduit_api::client::{leave_all_rooms, update_avatar_url, update_displayname}; use futures::StreamExt; use ruma::{ @@ -376,6 +380,139 @@ pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result Result { + const REASON: &str = "Bulk force joining this room as initiated by the server admin."; + + if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + { + return Ok(RoomMessageEventContent::text_plain( + "Expected code block in command body. Add --help for details.", + )); + } + + if !yes_i_want_to_do_this { + return Ok(RoomMessageEventContent::notice_markdown( + "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force bulk join all \ + specified local users.", + )); + } + + let Ok(admin_room) = self.services.admin.get_admin_room().await else { + return Ok(RoomMessageEventContent::notice_markdown( + "There is not an admin room to check for server admins.", + )); + }; + + let (room_id, servers) = self + .services + .rooms + .alias + .resolve_with_servers(&room_id, None) + .await?; + + if !self + .services + .rooms + .state_cache + .server_in_room(self.services.globals.server_name(), &room_id) + .await + { + return Ok(RoomMessageEventContent::notice_markdown("We are not joined in this room.")); + } + + let server_admins: Vec<_> = self + .services + .rooms + .state_cache + .active_local_users_in_room(&admin_room) + .map(ToOwned::to_owned) + .collect() + .await; + + if !self + .services + .rooms + .state_cache + .room_members(&room_id) + .ready_any(|user_id| server_admins.contains(&user_id.to_owned())) + .await + { + return Ok(RoomMessageEventContent::notice_markdown( + "There is not a single server admin in the room.", + )); + } + + let usernames = self + .body + .to_vec() + .drain(1..self.body.len().saturating_sub(1)) + .collect::>(); + + let mut user_ids: Vec = Vec::with_capacity(usernames.len()); + + for username in usernames { + match parse_active_local_user_id(self.services, username).await { + Ok(user_id) => { + // don't make the server service account join + if user_id == self.services.globals.server_user { + self.services + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "{username} is the server service account, skipping over" + ))) + .await + .ok(); + continue; + } + + user_ids.push(user_id); + }, + Err(e) => { + self.services + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "{username} is not a valid username, skipping over: {e}" + ))) + .await + .ok(); + continue; + }, + } + } + + let mut failed_joins: usize = 0; + let mut successful_joins: usize = 0; + + for user_id in user_ids { + match join_room_by_id_helper( + self.services, + &user_id, + &room_id, + Some(String::from(REASON)), + &servers, + None, + &None, + ) + .await + { + Ok(_res) => { + successful_joins = successful_joins.saturating_add(1); + }, + Err(e) => { + debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}"); + failed_joins = failed_joins.saturating_add(1); + }, + }; + } + + Ok(RoomMessageEventContent::notice_markdown(format!( + "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins failed.", + ))) +} + #[admin_command] pub(super) async fn force_join_room( &self, user_id: String, room_id: OwnedRoomOrAliasId, diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index e7bb5c73..e1568269 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -124,4 +124,19 @@ pub(super) enum UserCommand { RedactEvent { event_id: Box, }, + + /// - Force joins a specified list of local users to join the specified + /// room. + /// + /// Specify a codeblock of usernames. + /// + /// At least 1 server admin must be in the room to prevent abuse. + /// + /// Requires the `--yes-i-want-to-do-this` flag. + ForceJoinListOfLocalUsers { + room_id: OwnedRoomOrAliasId, + + #[arg(long)] + yes_i_want_to_do_this: bool, + }, } From 567a4cb4417726d4400f81e1bedea12e46fac439 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 28 Oct 2024 19:06:53 -0400 Subject: [PATCH 0978/2091] implement admin command to force join all local users to room Signed-off-by: strawberry --- src/admin/user/commands.rs | 100 +++++++++++++++++++++++++++++++++++-- src/admin/user/mod.rs | 14 +++++- 2 files changed, 109 insertions(+), 5 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index d6946b4e..531ce490 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -18,7 +18,7 @@ use ruma::{ tag::{TagEvent, TagEventContent, TagInfo}, RoomAccountDataEventType, StateEventType, }, - EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, + EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, UserId, }; use crate::{ @@ -27,6 +27,7 @@ use crate::{ }; const AUTO_GEN_PASSWORD_LENGTH: usize = 25; +const BULK_JOIN_REASON: &str = "Bulk force joining this room as initiated by the server admin."; #[admin_command] pub(super) async fn list_users(&self) -> Result { @@ -384,8 +385,6 @@ pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result Result { - const REASON: &str = "Bulk force joining this room as initiated by the server admin."; - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( @@ -491,7 +490,100 @@ pub(super) async fn force_join_list_of_local_users( self.services, &user_id, &room_id, - Some(String::from(REASON)), + Some(String::from(BULK_JOIN_REASON)), + &servers, + None, + &None, + ) + .await + { + Ok(_res) => { + successful_joins = successful_joins.saturating_add(1); + }, + Err(e) => { + debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}"); + failed_joins = failed_joins.saturating_add(1); + }, + }; + } + + Ok(RoomMessageEventContent::notice_markdown(format!( + "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins failed.", + ))) +} + +#[admin_command] +pub(super) async fn force_join_all_local_users( + &self, room_id: OwnedRoomOrAliasId, yes_i_want_to_do_this: bool, +) -> Result { + if !yes_i_want_to_do_this { + return Ok(RoomMessageEventContent::notice_markdown( + "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force bulk join all local \ + users.", + )); + } + + let Ok(admin_room) = self.services.admin.get_admin_room().await else { + return Ok(RoomMessageEventContent::notice_markdown( + "There is not an admin room to check for server admins.", + )); + }; + + let (room_id, servers) = self + .services + .rooms + .alias + .resolve_with_servers(&room_id, None) + .await?; + + if !self + .services + .rooms + .state_cache + .server_in_room(self.services.globals.server_name(), &room_id) + .await + { + return Ok(RoomMessageEventContent::notice_markdown("We are not joined in this room.")); + } + + let server_admins: Vec<_> = self + .services + .rooms + .state_cache + .active_local_users_in_room(&admin_room) + .map(ToOwned::to_owned) + .collect() + .await; + + if !self + .services + .rooms + .state_cache + .room_members(&room_id) + .ready_any(|user_id| server_admins.contains(&user_id.to_owned())) + .await + { + return Ok(RoomMessageEventContent::notice_markdown( + "There is not a single server admin in the room.", + )); + } + + let mut failed_joins: usize = 0; + let mut successful_joins: usize = 0; + + for user_id in &self + .services + .users + .list_local_users() + .map(UserId::to_owned) + .collect::>() + .await + { + match join_room_by_id_helper( + self.services, + user_id, + &room_id, + Some(String::from(BULK_JOIN_REASON)), &servers, None, &None, diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index e1568269..649cdfb8 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -130,7 +130,7 @@ pub(super) enum UserCommand { /// /// Specify a codeblock of usernames. /// - /// At least 1 server admin must be in the room to prevent abuse. + /// At least 1 server admin must be in the room to reduce abuse. /// /// Requires the `--yes-i-want-to-do-this` flag. ForceJoinListOfLocalUsers { @@ -139,4 +139,16 @@ pub(super) enum UserCommand { #[arg(long)] yes_i_want_to_do_this: bool, }, + + /// - Force joins all local users to the specified room. + /// + /// At least 1 server admin must be in the room to reduce abuse. + /// + /// Requires the `--yes-i-want-to-do-this` flag. + ForceJoinAllLocalUsers { + room_id: OwnedRoomOrAliasId, + + #[arg(long)] + yes_i_want_to_do_this: bool, + }, } From 354dc9e703a16dfca96bfe59cba0ba83e65725cf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 29 Oct 2024 00:08:41 +0000 Subject: [PATCH 0979/2091] add map accessor to Database; move cork interface Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 2 +- src/database/cork.rs | 16 +++++++++++++++- src/database/database.rs | 26 ++++++++++---------------- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 2aa6078f..db102858 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -838,7 +838,7 @@ pub(super) async fn database_stats( let map_name = map.as_ref().map_or(EMPTY, String::as_str); let mut out = String::new(); - for (name, map) in self.services.db.iter_maps() { + for (name, map) in self.services.db.iter() { if !map_name.is_empty() && *map_name != *name { continue; } diff --git a/src/database/cork.rs b/src/database/cork.rs index 26c520a2..5fe5fd7a 100644 --- a/src/database/cork.rs +++ b/src/database/cork.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::Engine; +use crate::{Database, Engine}; pub struct Cork { db: Arc, @@ -8,6 +8,20 @@ pub struct Cork { sync: bool, } +impl Database { + #[inline] + #[must_use] + pub fn cork(&self) -> Cork { Cork::new(&self.db, false, false) } + + #[inline] + #[must_use] + pub fn cork_and_flush(&self) -> Cork { Cork::new(&self.db, true, false) } + + #[inline] + #[must_use] + pub fn cork_and_sync(&self) -> Cork { Cork::new(&self.db, true, true) } +} + impl Cork { #[inline] pub(super) fn new(db: &Arc, flush: bool, sync: bool) -> Self { diff --git a/src/database/database.rs b/src/database/database.rs index 4c29c840..099df87d 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -1,9 +1,8 @@ use std::{ops::Index, sync::Arc}; -use conduit::{Result, Server}; +use conduit::{err, Result, Server}; use crate::{ - cork::Cork, maps, maps::{Maps, MapsKey, MapsVal}, Engine, Map, @@ -11,7 +10,7 @@ use crate::{ pub struct Database { pub db: Arc, - map: Maps, + maps: Maps, } impl Database { @@ -20,24 +19,19 @@ impl Database { let db = Engine::open(server)?; Ok(Arc::new(Self { db: db.clone(), - map: maps::open(&db)?, + maps: maps::open(&db)?, })) } #[inline] - #[must_use] - pub fn cork(&self) -> Cork { Cork::new(&self.db, false, false) } + pub fn get(&self, name: &str) -> Result<&Arc> { + self.maps + .get(name) + .ok_or_else(|| err!(Request(NotFound("column not found")))) + } #[inline] - #[must_use] - pub fn cork_and_flush(&self) -> Cork { Cork::new(&self.db, true, false) } - - #[inline] - #[must_use] - pub fn cork_and_sync(&self) -> Cork { Cork::new(&self.db, true, true) } - - #[inline] - pub fn iter_maps(&self) -> impl Iterator + Send + '_ { self.map.iter() } + pub fn iter(&self) -> impl Iterator + Send + '_ { self.maps.iter() } #[inline] #[must_use] @@ -52,7 +46,7 @@ impl Index<&str> for Database { type Output = Arc; fn index(&self, name: &str) -> &Self::Output { - self.map + self.maps .get(name) .expect("column in database does not exist") } From 8ed9d49b73923c39524d442ed9c7878d99ff2189 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 31 Oct 2024 14:41:35 -0400 Subject: [PATCH 0980/2091] skip new flakey complement test Signed-off-by: strawberry --- bin/complement | 2 +- tests/test_results/complement/test_results.jsonl | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/complement b/bin/complement index 601edb5a..a1db4b32 100755 --- a/bin/complement +++ b/bin/complement @@ -18,7 +18,7 @@ RESULTS_FILE="$3" OCI_IMAGE="complement-conduwuit:main" # Complement tests that are skipped due to flakiness/reliability issues -SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*' +SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*|TestUnbanViaInvite.*' # $COMPLEMENT_SRC needs to be a directory to Complement source code if [ -f "$COMPLEMENT_SRC" ]; then diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index ff695bb7..575a22fe 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -225,7 +225,6 @@ {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/good_connectivity"} {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/interrupted_connectivity"} {"Action":"fail","Test":"TestToDeviceMessagesOverFederation/stopped_server"} -{"Action":"pass","Test":"TestUnbanViaInvite"} {"Action":"fail","Test":"TestUnknownEndpoints"} {"Action":"pass","Test":"TestUnknownEndpoints/Client-server_endpoints"} {"Action":"fail","Test":"TestUnknownEndpoints/Key_endpoints"} From 240c78e8101da122e35986c7c1414b4f2d655d31 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 1 Nov 2024 00:54:21 -0400 Subject: [PATCH 0981/2091] strong-type URL for URL previews to Url type Signed-off-by: strawberry --- src/api/client/media.rs | 25 ++++++++++++++++++------- src/api/client/media_legacy.rs | 25 ++++++++++++++++--------- src/service/media/preview.rs | 32 ++++++++++++-------------------- 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 12012711..71693618 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -11,6 +11,7 @@ use conduit_service::{ media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN, MXC_LENGTH}, Services, }; +use reqwest::Url; use ruma::{ api::client::{ authenticated_media::{ @@ -165,23 +166,33 @@ pub(crate) async fn get_media_preview_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let url = &body.url; - if !services.media.url_preview_allowed(url) { + let url = Url::parse(&body.url).map_err(|e| { + err!(Request(InvalidParam( + debug_warn!(%sender_user, %url, "Requested URL is not valid: {e}") + ))) + })?; + + if !services.media.url_preview_allowed(&url) { return Err!(Request(Forbidden( debug_warn!(%sender_user, %url, "URL is not allowed to be previewed") ))); } - let preview = services.media.get_url_preview(url).await.map_err(|error| { - err!(Request(Unknown( - debug_error!(%sender_user, %url, ?error, "Failed to fetch URL preview.") - ))) - })?; + let preview = services + .media + .get_url_preview(&url) + .await + .map_err(|error| { + err!(Request(Unknown( + debug_error!(%sender_user, %url, "Failed to fetch URL preview: {error}") + ))) + })?; serde_json::value::to_raw_value(&preview) .map(get_media_preview::v1::Response::from_raw_value) .map_err(|error| { err!(Request(Unknown( - debug_error!(%sender_user, %url, ?error, "Failed to parse URL preview.") + debug_error!(%sender_user, %url, "Failed to parse URL preview: {error}") ))) }) } diff --git a/src/api/client/media_legacy.rs b/src/api/client/media_legacy.rs index e87b9a2b..f6837462 100644 --- a/src/api/client/media_legacy.rs +++ b/src/api/client/media_legacy.rs @@ -8,6 +8,7 @@ use conduit::{ Err, Result, }; use conduit_service::media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN}; +use reqwest::Url; use ruma::{ api::client::media::{ create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, @@ -55,25 +56,31 @@ pub(crate) async fn get_media_preview_legacy_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let url = &body.url; - if !services.media.url_preview_allowed(url) { + let url = Url::parse(&body.url).map_err(|e| { + err!(Request(InvalidParam( + debug_warn!(%sender_user, %url, "Requested URL is not valid: {e}") + ))) + })?; + + if !services.media.url_preview_allowed(&url) { return Err!(Request(Forbidden( debug_warn!(%sender_user, %url, "URL is not allowed to be previewed") ))); } - let preview = services.media.get_url_preview(url).await.map_err(|e| { + let preview = services.media.get_url_preview(&url).await.map_err(|e| { err!(Request(Unknown( debug_error!(%sender_user, %url, "Failed to fetch a URL preview: {e}") ))) })?; - let res = serde_json::value::to_raw_value(&preview).map_err(|e| { - err!(Request(Unknown( - debug_error!(%sender_user, %url, "Failed to parse a URL preview: {e}") - ))) - })?; - - Ok(get_media_preview::v3::Response::from_raw_value(res)) + serde_json::value::to_raw_value(&preview) + .map(get_media_preview::v3::Response::from_raw_value) + .map_err(|error| { + err!(Request(Unknown( + debug_error!(%sender_user, %url, "Failed to parse URL preview: {error}") + ))) + }) } /// # `GET /_matrix/media/v1/preview_url` diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index 6b147383..acc9d8ed 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -1,6 +1,6 @@ use std::{io::Cursor, time::SystemTime}; -use conduit::{debug, utils, warn, Err, Result}; +use conduit::{debug, utils, Err, Result}; use conduit_core::implement; use image::ImageReader as ImgReader; use ipaddress::IPAddress; @@ -70,30 +70,30 @@ pub async fn download_image(&self, url: &str) -> Result { } #[implement(Service)] -pub async fn get_url_preview(&self, url: &str) -> Result { - if let Ok(preview) = self.db.get_url_preview(url).await { +pub async fn get_url_preview(&self, url: &Url) -> Result { + if let Ok(preview) = self.db.get_url_preview(url.as_str()).await { return Ok(preview); } // ensure that only one request is made per URL - let _request_lock = self.url_preview_mutex.lock(url).await; + let _request_lock = self.url_preview_mutex.lock(url.as_str()).await; - match self.db.get_url_preview(url).await { + match self.db.get_url_preview(url.as_str()).await { Ok(preview) => Ok(preview), Err(_) => self.request_url_preview(url).await, } } #[implement(Service)] -async fn request_url_preview(&self, url: &str) -> Result { - if let Ok(ip) = IPAddress::parse(url) { +async fn request_url_preview(&self, url: &Url) -> Result { + if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) { if !self.services.globals.valid_cidr_range(&ip) { return Err!(BadServerResponse("Requesting from this address is forbidden")); } } let client = &self.services.client.url_preview; - let response = client.head(url).send().await?; + let response = client.head(url.as_str()).send().await?; if let Some(remote_addr) = response.remote_addr() { if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { @@ -111,12 +111,12 @@ async fn request_url_preview(&self, url: &str) -> Result { return Err!(Request(Unknown("Unknown Content-Type"))); }; let data = match content_type { - html if html.starts_with("text/html") => self.download_html(url).await?, - img if img.starts_with("image/") => self.download_image(url).await?, + html if html.starts_with("text/html") => self.download_html(url.as_str()).await?, + img if img.starts_with("image/") => self.download_image(url.as_str()).await?, _ => return Err!(Request(Unknown("Unsupported Content-Type"))), }; - self.set_url_preview(url, &data).await?; + self.set_url_preview(url.as_str(), &data).await?; Ok(data) } @@ -159,15 +159,7 @@ async fn download_html(&self, url: &str) -> Result { } #[implement(Service)] -pub fn url_preview_allowed(&self, url_str: &str) -> bool { - let url: Url = match Url::parse(url_str) { - Ok(u) => u, - Err(e) => { - warn!("Failed to parse URL from a str: {}", e); - return false; - }, - }; - +pub fn url_preview_allowed(&self, url: &Url) -> bool { if ["http", "https"] .iter() .all(|&scheme| scheme != url.scheme().to_lowercase()) From 6cbaef2d12b24765dc16c0478b58c2e76dd972cd Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 2 Nov 2024 13:12:14 -0400 Subject: [PATCH 0982/2091] always set RUST_BACKTRACE=full in OCI images Signed-off-by: strawberry --- nix/pkgs/complement/default.nix | 1 + nix/pkgs/oci-image/default.nix | 3 +++ 2 files changed, 4 insertions(+) diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index 80e9ce27..399c4449 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -96,6 +96,7 @@ dockerTools.buildImage { Env = [ "SSL_CERT_FILE=/complement/ca/ca.crt" "CONDUWUIT_CONFIG=${./config.toml}" + "RUST_BACKTRACE=full" ]; ExposedPorts = { diff --git a/nix/pkgs/oci-image/default.nix b/nix/pkgs/oci-image/default.nix index 5078523b..9b641310 100644 --- a/nix/pkgs/oci-image/default.nix +++ b/nix/pkgs/oci-image/default.nix @@ -24,5 +24,8 @@ dockerTools.buildLayeredImage { Cmd = [ "${lib.getExe main}" ]; + Env = [ + "RUST_BACKTRACE=full" + ]; }; } From ee6af6c90e5f941584429b9e890bffa358b23720 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 2 Nov 2024 18:46:20 -0400 Subject: [PATCH 0983/2091] drop report delay response range to 2-5 secs Signed-off-by: strawberry --- src/api/client/report.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 143c13e5..32a254d8 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -180,7 +180,7 @@ async fn is_event_report_valid( /// random delay sending a response per spec suggestion regarding /// enumerating for potential events existing in our server. async fn delay_response() { - let time_to_wait = rand::thread_rng().gen_range(3..10); + let time_to_wait = rand::thread_rng().gen_range(2..5); debug_info!("Got successful /report request, waiting {time_to_wait} seconds before sending successful response."); sleep(Duration::from_secs(time_to_wait)).await; } From 9466aeb08876472f49da6ce4b2fb673ff3598c04 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 2 Nov 2024 18:52:25 -0400 Subject: [PATCH 0984/2091] remove some unnecessary debug prints on notices Signed-off-by: strawberry --- src/api/client/account.rs | 18 ++++++++++-------- src/api/client/report.rs | 25 ++++++++++++++----------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 97d36839..87e73c5a 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -100,8 +100,8 @@ pub(crate) async fn register_route( if !services.globals.allow_registration() && body.appservice_info.is_none() { info!( "Registration disabled and request not from known appservice, rejecting registration attempt for username \ - {:?}", - body.username + \"{}\"", + body.username.as_deref().unwrap_or("") ); return Err(Error::BadRequest(ErrorKind::forbidden(), "Registration has been disabled.")); } @@ -114,8 +114,8 @@ pub(crate) async fn register_route( { info!( "Guest registration disabled / registration enabled with token configured, rejecting guest registration \ - attempt, initial device name: {:?}", - body.initial_device_display_name + attempt, initial device name: \"{}\"", + body.initial_device_display_name.as_deref().unwrap_or("") ); return Err(Error::BadRequest( ErrorKind::GuestAccessForbidden, @@ -128,8 +128,8 @@ pub(crate) async fn register_route( if is_guest && services.users.count().await < 2 { warn!( "Guest account attempted to register before a real admin user has been registered, rejecting \ - registration. Guest's initial device name: {:?}", - body.initial_device_display_name + registration. Guest's initial device name: \"{}\"", + body.initial_device_display_name.as_deref().unwrap_or("") ); return Err(Error::BadRequest(ErrorKind::forbidden(), "Registration temporarily disabled.")); } @@ -312,12 +312,14 @@ pub(crate) async fn register_route( debug_info!(%user_id, %device_id, "User account was created"); - let device_display_name = body.initial_device_display_name.clone().unwrap_or_default(); + let device_display_name = body.initial_device_display_name.as_deref().unwrap_or(""); // log in conduit admin channel if a non-guest user registered if body.appservice_info.is_none() && !is_guest { if !device_display_name.is_empty() { - info!("New user \"{user_id}\" registered on this server with device display name: {device_display_name}"); + info!( + "New user \"{user_id}\" registered on this server with device display name: \"{device_display_name}\"" + ); if services.globals.config.admin_room_notices { services diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 32a254d8..e20fa8c2 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -33,10 +33,18 @@ pub(crate) async fn report_room_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); info!( - "Received room report by user {sender_user} for room {} with reason: {:?}", - body.room_id, body.reason + "Received room report by user {sender_user} for room {} with reason: \"{}\"", + body.room_id, + body.reason.as_deref().unwrap_or("") ); + if body.reason.as_ref().is_some_and(|s| s.len() > 750) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Reason too long, should be 750 characters or fewer", + )); + }; + delay_response().await; if !services @@ -50,13 +58,6 @@ pub(crate) async fn report_room_route( ))); } - if body.reason.as_ref().is_some_and(|s| s.len() > 750) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Reason too long, should be 750 characters or fewer", - )); - }; - // send admin room message that we received the report with an @room ping for // urgency services @@ -85,8 +86,10 @@ pub(crate) async fn report_event_route( let sender_user = body.sender_user.as_ref().expect("user is authenticated"); info!( - "Received event report by user {sender_user} for room {} and event ID {}, with reason: {:?}", - body.room_id, body.event_id, body.reason + "Received event report by user {sender_user} for room {} and event ID {}, with reason: \"{}\"", + body.room_id, + body.event_id, + body.reason.as_deref().unwrap_or("") ); delay_response().await; From 6f37a251fb5945c9f431ffeb3f32fcb5d3bbc470 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 2 Nov 2024 20:55:40 -0400 Subject: [PATCH 0985/2091] allow taking room aliases for `auto_join_rooms` config option Signed-off-by: strawberry --- src/admin/user/commands.rs | 20 ++++++++++++++++---- src/api/client/account.rs | 13 +++++++++---- src/core/config/mod.rs | 11 ++++++----- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 531ce490..444a7f37 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -108,24 +108,29 @@ pub(super) async fn create_user(&self, username: String, password: Option { + self.services + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "Failed to automatically join room {room} for user {user_id}: {e}" + ))) + .await + .ok(); // don't return this error so we don't fail registrations error!("Failed to automatically join room {room} for user {user_id}: {e}"); }, diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 87e73c5a..c340f529 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -398,23 +398,28 @@ pub(crate) async fn register_route( && (services.globals.allow_guests_auto_join_rooms() || !is_guest) { for room in &services.globals.config.auto_join_rooms { + let Ok(room_id) = services.rooms.alias.resolve(room).await else { + error!("Failed to resolve room alias to room ID when attempting to auto join {room}, skipping"); + continue; + }; + if !services .rooms .state_cache - .server_in_room(services.globals.server_name(), room) + .server_in_room(services.globals.server_name(), &room_id) .await { warn!("Skipping room {room} to automatically join as we have never joined before."); continue; } - if let Some(room_id_server_name) = room.server_name() { + if let Some(room_server_name) = room.server_name() { if let Err(e) = join_room_by_id_helper( &services, &user_id, - room, + &room_id, Some("Automatically joining this room upon registration".to_owned()), - &[room_id_server_name.to_owned(), services.globals.server_name().to_owned()], + &[services.globals.server_name().to_owned(), room_server_name.to_owned()], None, &body.appservice_info, ) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 512cb48b..a6216da2 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -18,7 +18,8 @@ pub use figment::{value::Value as FigmentValue, Figment}; use itertools::Itertools; use regex::RegexSet; use ruma::{ - api::client::discovery::discover_support::ContactRole, OwnedRoomId, OwnedServerName, OwnedUserId, RoomVersionId, + api::client::discovery::discover_support::ContactRole, OwnedRoomOrAliasId, OwnedServerName, OwnedUserId, + RoomVersionId, }; use serde::{de::IgnoredAny, Deserialize}; use url::Url; @@ -653,13 +654,13 @@ pub struct Config { #[serde(default = "default_turn_ttl")] pub turn_ttl: u64, - /// List/vector of room **IDs** that conduwuit will make newly registered - /// users join. The room IDs specified must be rooms that you have joined - /// at least once on the server, and must be public. + /// List/vector of room IDs or room aliases that conduwuit will make newly + /// registered users join. The rooms specified must be rooms that you + /// have joined at least once on the server, and must be public. /// /// No default. #[serde(default = "Vec::new")] - pub auto_join_rooms: Vec, + pub auto_join_rooms: Vec, /// Config option to automatically deactivate the account of any user who /// attempts to join a: From 038787106365cebb0c538af043cc90028e200cc3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 2 Nov 2024 21:20:36 -0400 Subject: [PATCH 0986/2091] add workaround for matrix-appservice-irc using historical localparts see https://github.com/matrix-org/matrix-appservice-irc/issues/1780 Signed-off-by: strawberry --- src/api/client/account.rs | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index c340f529..5ed4b312 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -48,10 +48,19 @@ pub(crate) async fn get_register_available_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { + // workaround for https://github.com/matrix-org/matrix-appservice-irc/issues/1780 due to inactivity of fixing the issue + let is_matrix_appservice_irc = body.appservice_info.as_ref().is_some_and(|appservice| { + appservice.registration.id == "irc" + || appservice.registration.id.contains("matrix-appservice-irc") + || appservice.registration.id.contains("matrix_appservice_irc") + }); + // Validate user id let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), services.globals.server_name()) .ok() - .filter(|user_id| !user_id.is_historical() && services.globals.user_is_local(user_id)) + .filter(|user_id| { + (!user_id.is_historical() || is_matrix_appservice_irc) && services.globals.user_is_local(user_id) + }) .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; // Check if username is creative enough @@ -134,12 +143,22 @@ pub(crate) async fn register_route( return Err(Error::BadRequest(ErrorKind::forbidden(), "Registration temporarily disabled.")); } + // workaround for https://github.com/matrix-org/matrix-appservice-irc/issues/1780 due to inactivity of fixing the issue + let is_matrix_appservice_irc = body.appservice_info.as_ref().is_some_and(|appservice| { + appservice.registration.id == "irc" + || appservice.registration.id.contains("matrix-appservice-irc") + || appservice.registration.id.contains("matrix_appservice_irc") + }); + let user_id = match (&body.username, is_guest) { (Some(username), false) => { let proposed_user_id = UserId::parse_with_server_name(username.to_lowercase(), services.globals.server_name()) .ok() - .filter(|user_id| !user_id.is_historical() && services.globals.user_is_local(user_id)) + .filter(|user_id| { + (!user_id.is_historical() || is_matrix_appservice_irc) + && services.globals.user_is_local(user_id) + }) .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; if services.users.exists(&proposed_user_id).await { From 1fbfc983e9606752770286f42d5812a34a820e63 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 29 Oct 2024 10:53:56 +0000 Subject: [PATCH 0987/2091] optimize FedDest::Named port Signed-off-by: Jason Volk --- src/service/resolver/actual.rs | 39 +++++++++++++++++++++++++--------- src/service/resolver/fed.rs | 26 +++++++++++++++++------ src/service/resolver/tests.rs | 4 ++-- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 660498f7..61eedca5 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -9,9 +9,9 @@ use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; use ruma::ServerName; -use crate::resolver::{ +use super::{ cache::{CachedDest, CachedOverride}, - fed::{add_port_to_hostname, get_ip_with_port, FedDest}, + fed::{add_port_to_hostname, get_ip_with_port, FedDest, PortString}, }; #[derive(Clone, Debug)] @@ -77,12 +77,12 @@ impl super::Service { let host = if let Ok(addr) = host.parse::() { FedDest::Literal(addr) } else if let Ok(addr) = host.parse::() { - FedDest::Named(addr.to_string(), ":8448".to_owned()) + FedDest::Named(addr.to_string(), FedDest::default_port()) } else if let Some(pos) = host.find(':') { let (host, port) = host.split_at(pos); - FedDest::Named(host.to_owned(), port.to_owned()) + FedDest::Named(host.to_owned(), port.try_into().unwrap_or_else(|_| FedDest::default_port())) } else { - FedDest::Named(host, ":8448".to_owned()) + FedDest::Named(host, FedDest::default_port()) }; debug!("Actual destination: {actual_dest:?} hostname: {host:?}"); @@ -103,7 +103,10 @@ impl super::Service { let (host, port) = dest.as_str().split_at(pos); self.conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache) .await?; - Ok(FedDest::Named(host.to_owned(), port.to_owned())) + Ok(FedDest::Named( + host.to_owned(), + port.try_into().unwrap_or_else(|_| FedDest::default_port()), + )) } async fn actual_dest_3(&self, host: &mut String, cache: bool, delegated: String) -> Result { @@ -136,7 +139,10 @@ impl super::Service { let (host, port) = delegated.split_at(pos); self.conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache) .await?; - Ok(FedDest::Named(host.to_owned(), port.to_owned())) + Ok(FedDest::Named( + host.to_owned(), + port.try_into().unwrap_or_else(|_| FedDest::default_port()), + )) } async fn actual_dest_3_3(&self, cache: bool, delegated: String, overrider: FedDest) -> Result { @@ -145,7 +151,13 @@ impl super::Service { self.conditional_query_and_cache_override(&delegated, &overrider.hostname(), force_port.unwrap_or(8448), cache) .await?; if let Some(port) = force_port { - Ok(FedDest::Named(delegated, format!(":{port}"))) + Ok(FedDest::Named( + delegated, + format!(":{port}") + .as_str() + .try_into() + .unwrap_or_else(|_| FedDest::default_port()), + )) } else { Ok(add_port_to_hostname(&delegated)) } @@ -164,7 +176,11 @@ impl super::Service { self.conditional_query_and_cache_override(host, &overrider.hostname(), force_port.unwrap_or(8448), cache) .await?; if let Some(port) = force_port { - Ok(FedDest::Named(host.to_owned(), format!(":{port}"))) + let port = format!(":{port}"); + Ok(FedDest::Named( + host.to_owned(), + PortString::from(port.as_str()).unwrap_or_else(|_| FedDest::default_port()), + )) } else { Ok(add_port_to_hostname(host)) } @@ -269,7 +285,10 @@ impl super::Service { srv.iter().next().map(|result| { FedDest::Named( result.target().to_string().trim_end_matches('.').to_owned(), - format!(":{}", result.port()), + format!(":{}", result.port()) + .as_str() + .try_into() + .unwrap_or_else(|_| FedDest::default_port()), ) }) } diff --git a/src/service/resolver/fed.rs b/src/service/resolver/fed.rs index 79f71f13..9c348b47 100644 --- a/src/service/resolver/fed.rs +++ b/src/service/resolver/fed.rs @@ -4,12 +4,19 @@ use std::{ net::{IpAddr, SocketAddr}, }; +use arrayvec::ArrayString; + #[derive(Clone, Debug, PartialEq, Eq)] pub enum FedDest { Literal(SocketAddr), - Named(String, String), + Named(String, PortString), } +/// numeric or service-name +pub type PortString = ArrayString<16>; + +const DEFAULT_PORT: &str = ":8448"; + pub(crate) fn get_ip_with_port(dest_str: &str) -> Option { if let Ok(dest) = dest_str.parse::() { Some(FedDest::Literal(dest)) @@ -20,13 +27,16 @@ pub(crate) fn get_ip_with_port(dest_str: &str) -> Option { } } -pub(crate) fn add_port_to_hostname(dest_str: &str) -> FedDest { - let (host, port) = match dest_str.find(':') { - None => (dest_str, ":8448"), - Some(pos) => dest_str.split_at(pos), +pub(crate) fn add_port_to_hostname(dest: &str) -> FedDest { + let (host, port) = match dest.find(':') { + None => (dest, DEFAULT_PORT), + Some(pos) => dest.split_at(pos), }; - FedDest::Named(host.to_owned(), port.to_owned()) + FedDest::Named( + host.to_owned(), + PortString::from(port).unwrap_or_else(|_| FedDest::default_port()), + ) } impl FedDest { @@ -60,6 +70,10 @@ impl FedDest { Self::Named(_, port) => port[1..].parse().ok(), } } + + #[inline] + #[must_use] + pub fn default_port() -> PortString { PortString::from(DEFAULT_PORT).expect("default port string") } } impl fmt::Display for FedDest { diff --git a/src/service/resolver/tests.rs b/src/service/resolver/tests.rs index 55cf0345..870f5eab 100644 --- a/src/service/resolver/tests.rs +++ b/src/service/resolver/tests.rs @@ -30,7 +30,7 @@ fn ips_keep_custom_ports() { fn hostnames_get_default_ports() { assert_eq!( add_port_to_hostname("example.com"), - FedDest::Named(String::from("example.com"), String::from(":8448")) + FedDest::Named(String::from("example.com"), ":8448".try_into().unwrap()) ); } @@ -38,6 +38,6 @@ fn hostnames_get_default_ports() { fn hostnames_keep_custom_ports() { assert_eq!( add_port_to_hostname("example.com:1337"), - FedDest::Named(String::from("example.com"), String::from(":1337")) + FedDest::Named(String::from("example.com"), ":1337".try_into().unwrap()) ); } From ad117641b88330aff3d1ba7ced57939df7862659 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 29 Oct 2024 00:08:41 +0000 Subject: [PATCH 0988/2091] add tuple-apply macro with length argument for now Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 26b0484e..b8640f3a 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -53,6 +53,25 @@ macro_rules! extract_variant { }; } +#[macro_export] +macro_rules! apply { + (1, $($idx:tt)+) => { + |t| (($($idx)+)(t.0),) + }; + + (2, $($idx:tt)+) => { + |t| (($($idx)+)(t.0), ($($idx)+)(t.1),) + }; + + (3, $($idx:tt)+) => { + |t| (($($idx)+)(t.0), ($($idx)+)(t.1), ($($idx)+)(t.2),) + }; + + (4, $($idx:tt)+) => { + |t| (($($idx)+)(t.0), ($($idx)+)(t.1), ($($idx)+)(t.2), ($($idx)+4)(t.3)) + }; +} + #[macro_export] macro_rules! at { ($idx:tt) => { @@ -112,6 +131,14 @@ macro_rules! is_not_empty { }; } +/// Functor for equality i.e. (a, b).map(is_equal!()) +#[macro_export] +macro_rules! is_equal { + () => { + |a, b| a == b + }; +} + /// Functor for truthy #[macro_export] macro_rules! is_true { From ed76797b55c8f32e11fcb0a3b8d0d29a4b93b6b8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 29 Oct 2024 03:10:18 +0000 Subject: [PATCH 0989/2091] add raw_ overloads for prefix/from counting Signed-off-by: Jason Volk --- src/database/database.rs | 3 +++ src/database/map/count.rs | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/database/database.rs b/src/database/database.rs index 099df87d..bf8c8855 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -33,6 +33,9 @@ impl Database { #[inline] pub fn iter(&self) -> impl Iterator + Send + '_ { self.maps.iter() } + #[inline] + pub fn keys(&self) -> impl Iterator + Send + '_ { self.maps.keys() } + #[inline] #[must_use] pub fn is_read_only(&self) -> bool { self.db.secondary || self.db.read_only } diff --git a/src/database/map/count.rs b/src/database/map/count.rs index dab45b7a..3e92279c 100644 --- a/src/database/map/count.rs +++ b/src/database/map/count.rs @@ -21,6 +21,18 @@ where self.keys_from_raw(from).count() } +/// Count the number of entries in the map starting from a lower-bound. +/// +/// - From is a raw +#[implement(super::Map)] +#[inline] +pub fn raw_count_from<'a, P>(&'a self, from: &'a P) -> impl Future + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, +{ + self.raw_keys_from(from).count() +} + /// Count the number of entries in the map matching a prefix. /// /// - Prefix is structured key @@ -32,3 +44,15 @@ where { self.keys_prefix_raw(prefix).count() } + +/// Count the number of entries in the map matching a prefix. +/// +/// - Prefix is raw +#[implement(super::Map)] +#[inline] +pub fn raw_count_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Future + Send + 'a +where + P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, +{ + self.raw_keys_prefix(prefix).count() +} From a7cb1c59518e8398e8a6aaedd784b9b7a222a2fd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 29 Oct 2024 23:31:53 +0000 Subject: [PATCH 0990/2091] slightly optimize request signing/verifying Signed-off-by: Jason Volk --- src/api/router/auth.rs | 40 +++++++----- src/service/sending/send.rs | 117 ++++++++++++++++++------------------ 2 files changed, 86 insertions(+), 71 deletions(-) diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 31e71f2f..2552dded 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -220,20 +220,32 @@ async fn auth_server(services: &Services, request: &mut Request, body: Option<&C .expect("all requests have a path") .to_string(); - let signature: [Member; 1] = [(x_matrix.key.to_string(), Value::String(x_matrix.sig.to_string()))]; - let signatures: [Member; 1] = [(origin.to_string(), Value::Object(signature.into()))]; - let authorization: [Member; 5] = [ - ("destination".into(), Value::String(destination.into())), - ("method".into(), Value::String(request.parts.method.to_string())), - ("origin".into(), Value::String(origin.to_string())), - ("signatures".into(), Value::Object(signatures.into())), - ("uri".into(), Value::String(signature_uri)), - ]; + let signature: [Member; 1] = [(x_matrix.key.as_str().into(), Value::String(x_matrix.sig.to_string()))]; - let mut authorization: Object = authorization.into(); - if let Some(body) = body { - authorization.insert("content".to_owned(), body.clone()); - } + let signatures: [Member; 1] = [(origin.as_str().into(), Value::Object(signature.into()))]; + + let authorization: Object = if let Some(body) = body.cloned() { + let authorization: [Member; 6] = [ + ("content".into(), body), + ("destination".into(), Value::String(destination.into())), + ("method".into(), Value::String(request.parts.method.as_str().into())), + ("origin".into(), Value::String(origin.as_str().into())), + ("signatures".into(), Value::Object(signatures.into())), + ("uri".into(), Value::String(signature_uri)), + ]; + + authorization.into() + } else { + let authorization: [Member; 5] = [ + ("destination".into(), Value::String(destination.into())), + ("method".into(), Value::String(request.parts.method.as_str().into())), + ("origin".into(), Value::String(origin.as_str().into())), + ("signatures".into(), Value::Object(signatures.into())), + ("uri".into(), Value::String(signature_uri)), + ]; + + authorization.into() + }; let key = services .server_keys @@ -242,7 +254,7 @@ async fn auth_server(services: &Services, request: &mut Request, body: Option<&C .map_err(|e| err!(Request(Forbidden(warn!("Failed to fetch signing keys: {e}")))))?; let keys: PubKeys = [(x_matrix.key.to_string(), key.key)].into(); - let keys: PubKeyMap = [(origin.to_string(), keys)].into(); + let keys: PubKeyMap = [(origin.as_str().into(), keys)].into(); if let Err(e) = ruma::signatures::verify_json(&keys, authorization) { debug_error!("Failed to verify federation request from {origin}: {e}"); if request.parts.uri.to_string().contains('@') { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 2fbb3919..939d6e73 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -14,7 +14,7 @@ use ruma::{ }, serde::Base64, server_util::authorization::XMatrix, - ServerName, + CanonicalJsonObject, CanonicalJsonValue, ServerName, ServerSigningKeyId, }; use crate::{ @@ -74,7 +74,7 @@ impl super::Service { .try_into_http_request::>(actual.string().as_str(), SATIR, &VERSIONS) .map_err(|e| err!(BadServerResponse("Invalid destination: {e:?}")))?; - self.sign_request::(dest, &mut http_request); + self.sign_request(&mut http_request, dest); let request = Request::try_from(http_request)?; self.validate_url(request.url())?; @@ -178,68 +178,71 @@ where } #[implement(super::Service)] -fn sign_request(&self, dest: &ServerName, http_request: &mut http::Request>) -where - T: OutgoingRequest + Debug + Send, -{ - let mut req_map = serde_json::Map::with_capacity(8); - if !http_request.body().is_empty() { - req_map.insert( - "content".to_owned(), - serde_json::from_slice(http_request.body()).expect("body is valid json, we just created it"), - ); +fn sign_request(&self, http_request: &mut http::Request>, dest: &ServerName) { + type Member = (String, Value); + type Value = CanonicalJsonValue; + type Object = CanonicalJsonObject; + + let origin = self.services.globals.server_name(); + let body = http_request.body(); + let uri = http_request + .uri() + .path_and_query() + .expect("http::Request missing path_and_query"); + + let mut req: Object = if !body.is_empty() { + let content: CanonicalJsonValue = serde_json::from_slice(body).expect("failed to serialize body"); + + let authorization: [Member; 5] = [ + ("content".into(), content), + ("destination".into(), dest.as_str().into()), + ("method".into(), http_request.method().as_str().into()), + ("origin".into(), origin.as_str().into()), + ("uri".into(), uri.to_string().into()), + ]; + + authorization.into() + } else { + let authorization: [Member; 4] = [ + ("destination".into(), dest.as_str().into()), + ("method".into(), http_request.method().as_str().into()), + ("origin".into(), origin.as_str().into()), + ("uri".into(), uri.to_string().into()), + ]; + + authorization.into() }; - req_map.insert("method".to_owned(), T::METADATA.method.to_string().into()); - req_map.insert( - "uri".to_owned(), - http_request - .uri() - .path_and_query() - .expect("all requests have a path") - .to_string() - .into(), - ); - req_map.insert("origin".to_owned(), self.services.globals.server_name().to_string().into()); - req_map.insert("destination".to_owned(), dest.as_str().into()); - - let mut req_json = serde_json::from_value(req_map.into()).expect("valid JSON is valid BTreeMap"); self.services .server_keys - .sign_json(&mut req_json) - .expect("our request json is what ruma expects"); + .sign_json(&mut req) + .expect("request signing failed"); - let req_json: serde_json::Map = - serde_json::from_slice(&serde_json::to_vec(&req_json).unwrap()).unwrap(); - - let signatures = req_json["signatures"] + let signatures = req["signatures"] .as_object() - .expect("signatures object") + .and_then(|object| object[origin.as_str()].as_object()) + .expect("origin signatures object"); + + let key: &ServerSigningKeyId = signatures + .keys() + .next() + .map(|k| k.as_str().try_into()) + .expect("at least one signature from this origin") + .expect("keyid is json string"); + + let sig: Base64 = signatures .values() - .map(|v| { - v.as_object() - .expect("server signatures object") - .iter() - .map(|(k, v)| (k, v.as_str().expect("server signature string"))) - }); + .next() + .map(|s| s.as_str().map(Base64::parse)) + .expect("at least one signature from this origin") + .expect("signature is json string") + .expect("signature is valid base64"); - for signature_server in signatures { - for s in signature_server { - let key = - s.0.as_str() - .try_into() - .expect("valid homeserver signing key ID"); - let sig = Base64::parse(s.1).expect("valid base64"); + let x_matrix = XMatrix::new(origin.into(), dest.into(), key.into(), sig); + let authorization = HeaderValue::from(&x_matrix); + let authorization = http_request + .headers_mut() + .insert(AUTHORIZATION, authorization); - http_request.headers_mut().insert( - AUTHORIZATION, - HeaderValue::from(&XMatrix::new( - self.services.globals.server_name().to_owned(), - dest.to_owned(), - key, - sig, - )), - ); - } - } + debug_assert!(authorization.is_none(), "Authorization header already present"); } From 9775694423943135bc6015ebb102a21288ec05a1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 30 Oct 2024 05:08:04 +0000 Subject: [PATCH 0991/2091] inline database stream interface functions lt 64B Signed-off-by: Jason Volk --- src/database/stream.rs | 3 +++ src/database/stream/items.rs | 2 ++ src/database/stream/items_rev.rs | 2 ++ src/database/stream/keys.rs | 3 +++ src/database/stream/keys_rev.rs | 3 +++ 5 files changed, 13 insertions(+) diff --git a/src/database/stream.rs b/src/database/stream.rs index d9b74215..a2a72e44 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -71,6 +71,7 @@ impl<'a> State<'a> { self } + #[inline] fn seek_fwd(&mut self) { if !exchange(&mut self.init, false) { self.inner.next(); @@ -79,6 +80,7 @@ impl<'a> State<'a> { } } + #[inline] fn seek_rev(&mut self) { if !exchange(&mut self.init, false) { self.inner.prev(); @@ -95,6 +97,7 @@ impl<'a> State<'a> { fn status(&self) -> Option { self.inner.status().map_err(map_err).err() } + #[inline] fn valid(&self) -> bool { self.inner.valid() } } diff --git a/src/database/stream/items.rs b/src/database/stream/items.rs index 31d5e9e8..54f8bc5c 100644 --- a/src/database/stream/items.rs +++ b/src/database/stream/items.rs @@ -28,6 +28,7 @@ impl<'a> Cursor<'a, KeyVal<'a>> for Items<'a> { fn fetch(&self) -> Option> { self.state.fetch().map(keyval_longevity) } + #[inline] fn seek(&mut self) { self.state.seek_fwd(); } } @@ -40,5 +41,6 @@ impl<'a> Stream for Items<'a> { } impl FusedStream for Items<'_> { + #[inline] fn is_terminated(&self) -> bool { !self.state.init && !self.state.valid() } } diff --git a/src/database/stream/items_rev.rs b/src/database/stream/items_rev.rs index ab57a250..26492db8 100644 --- a/src/database/stream/items_rev.rs +++ b/src/database/stream/items_rev.rs @@ -28,6 +28,7 @@ impl<'a> Cursor<'a, KeyVal<'a>> for ItemsRev<'a> { fn fetch(&self) -> Option> { self.state.fetch().map(keyval_longevity) } + #[inline] fn seek(&mut self) { self.state.seek_rev(); } } @@ -40,5 +41,6 @@ impl<'a> Stream for ItemsRev<'a> { } impl FusedStream for ItemsRev<'_> { + #[inline] fn is_terminated(&self) -> bool { !self.state.init && !self.state.valid() } } diff --git a/src/database/stream/keys.rs b/src/database/stream/keys.rs index 1c5d12e3..91884c8d 100644 --- a/src/database/stream/keys.rs +++ b/src/database/stream/keys.rs @@ -26,8 +26,10 @@ impl<'a> Keys<'a> { impl<'a> Cursor<'a, Key<'a>> for Keys<'a> { fn state(&self) -> &State<'a> { &self.state } + #[inline] fn fetch(&self) -> Option> { self.state.fetch_key().map(slice_longevity) } + #[inline] fn seek(&mut self) { self.state.seek_fwd(); } } @@ -40,5 +42,6 @@ impl<'a> Stream for Keys<'a> { } impl FusedStream for Keys<'_> { + #[inline] fn is_terminated(&self) -> bool { !self.state.init && !self.state.valid() } } diff --git a/src/database/stream/keys_rev.rs b/src/database/stream/keys_rev.rs index 26707483..59f66c2e 100644 --- a/src/database/stream/keys_rev.rs +++ b/src/database/stream/keys_rev.rs @@ -26,8 +26,10 @@ impl<'a> KeysRev<'a> { impl<'a> Cursor<'a, Key<'a>> for KeysRev<'a> { fn state(&self) -> &State<'a> { &self.state } + #[inline] fn fetch(&self) -> Option> { self.state.fetch_key().map(slice_longevity) } + #[inline] fn seek(&mut self) { self.state.seek_rev(); } } @@ -40,5 +42,6 @@ impl<'a> Stream for KeysRev<'a> { } impl FusedStream for KeysRev<'_> { + #[inline] fn is_terminated(&self) -> bool { !self.state.init && !self.state.valid() } } From 0eb67cfea00c19e7d0cf1981acc805986dfd05d6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 30 Oct 2024 06:41:03 +0000 Subject: [PATCH 0992/2091] additional bool extensions for Result/Option conversion Signed-off-by: Jason Volk --- src/core/utils/bool.rs | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/src/core/utils/bool.rs b/src/core/utils/bool.rs index d5fa85aa..e9f399d4 100644 --- a/src/core/utils/bool.rs +++ b/src/core/utils/bool.rs @@ -2,6 +2,23 @@ /// Boolean extensions and chain.starters pub trait BoolExt { + #[must_use] + fn clone_or(self, err: T, t: &T) -> T; + + #[must_use] + fn copy_or(self, err: T, t: T) -> T; + + #[must_use] + fn expect(self, msg: &str) -> Self; + + #[must_use] + fn expect_false(self, msg: &str) -> Self; + + fn into_option(self) -> Option<()>; + + #[allow(clippy::result_unit_err)] + fn into_result(self) -> Result<(), ()>; + fn map T>(self, f: F) -> T where Self: Sized; @@ -22,6 +39,24 @@ pub trait BoolExt { } impl BoolExt for bool { + #[inline] + fn clone_or(self, err: T, t: &T) -> T { self.map_or(err, || t.clone()) } + + #[inline] + fn copy_or(self, err: T, t: T) -> T { self.map_or(err, || t) } + + #[inline] + fn expect(self, msg: &str) -> Self { self.then_some(true).expect(msg) } + + #[inline] + fn expect_false(self, msg: &str) -> Self { (!self).then_some(false).expect(msg) } + + #[inline] + fn into_option(self) -> Option<()> { self.then_some(()) } + + #[inline] + fn into_result(self) -> Result<(), ()> { self.ok_or(()) } + #[inline] fn map T>(self, f: F) -> T where @@ -40,10 +75,10 @@ impl BoolExt for bool { fn map_or_else T>(self, err: F, f: F) -> T { self.then(f).unwrap_or_else(err) } #[inline] - fn ok_or(self, err: E) -> Result<(), E> { self.then_some(()).ok_or(err) } + fn ok_or(self, err: E) -> Result<(), E> { self.into_option().ok_or(err) } #[inline] - fn ok_or_else E>(self, err: F) -> Result<(), E> { self.then_some(()).ok_or_else(err) } + fn ok_or_else E>(self, err: F) -> Result<(), E> { self.into_option().ok_or_else(err) } #[inline] fn or T>(self, f: F) -> Option { (!self).then(f) } From 7fcc6d11a4993a174d4b0998e276bdc150594a15 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 30 Oct 2024 11:04:23 +0000 Subject: [PATCH 0993/2091] de-wrap state_accessor.server_can_see_event Signed-off-by: Jason Volk --- src/service/rooms/state_accessor/mod.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 561db18a..a2cc27e8 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -160,20 +160,18 @@ impl Service { /// Whether a server is allowed to see an event through federation, based on /// the room's history_visibility at that event's state. #[tracing::instrument(skip(self, origin, room_id, event_id))] - pub async fn server_can_see_event( - &self, origin: &ServerName, room_id: &RoomId, event_id: &EventId, - ) -> Result { + pub async fn server_can_see_event(&self, origin: &ServerName, room_id: &RoomId, event_id: &EventId) -> bool { let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { - return Ok(true); + return true; }; if let Some(visibility) = self .server_visibility_cache .lock() - .unwrap() + .expect("locked") .get_mut(&(origin.to_owned(), shortstatehash)) { - return Ok(*visibility); + return *visibility; } let history_visibility = self @@ -211,10 +209,10 @@ impl Service { self.server_visibility_cache .lock() - .unwrap() + .expect("locked") .insert((origin.to_owned(), shortstatehash), visibility); - Ok(visibility) + visibility } /// Whether a user is allowed to see an event, based on @@ -228,7 +226,7 @@ impl Service { if let Some(visibility) = self .user_visibility_cache .lock() - .unwrap() + .expect("locked") .get_mut(&(user_id.to_owned(), shortstatehash)) { return *visibility; @@ -262,7 +260,7 @@ impl Service { self.user_visibility_cache .lock() - .unwrap() + .expect("locked") .insert((user_id.to_owned(), shortstatehash), visibility); visibility From e49aee61c1f276fd613b8c61cde158b778c40763 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 30 Oct 2024 07:01:50 +0000 Subject: [PATCH 0994/2091] consolidate and parallelize api/server access check prologues Signed-off-by: Jason Volk --- Cargo.toml | 2 +- src/api/server/backfill.rs | 29 +++++--------- src/api/server/event.rs | 39 ++++++------------ src/api/server/event_auth.rs | 25 ++++-------- src/api/server/get_missing_events.rs | 32 +++++---------- src/api/server/mod.rs | 3 ++ src/api/server/state.rs | 27 +++++-------- src/api/server/state_ids.rs | 29 +++++--------- src/api/server/utils.rs | 60 ++++++++++++++++++++++++++++ 9 files changed, 123 insertions(+), 123 deletions(-) create mode 100644 src/api/server/utils.rs diff --git a/Cargo.toml b/Cargo.toml index e406c9e1..043790f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -213,7 +213,7 @@ features = [ [workspace.dependencies.futures] version = "0.3.30" default-features = false -features = ["std"] +features = ["std", "async-await"] [workspace.dependencies.tokio] version = "1.40.0" diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 088b891a..281bf2a2 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -2,13 +2,13 @@ use std::cmp; use axum::extract::State; use conduit::{ - is_equal_to, utils::{IterStream, ReadyExt}, - Err, PduCount, Result, + PduCount, Result, }; use futures::{FutureExt, StreamExt}; use ruma::{api::federation::backfill::get_backfill, uint, user_id, MilliSecondsSinceUnixEpoch}; +use super::AccessCheck; use crate::Ruma; /// # `GET /_matrix/federation/v1/backfill/` @@ -18,24 +18,14 @@ use crate::Ruma; pub(crate) async fn get_backfill_route( State(services): State, body: Ruma, ) -> Result { - services - .rooms - .event_handler - .acl_check(body.origin(), &body.room_id) - .await?; - - if !services - .rooms - .state_accessor - .is_world_readable(&body.room_id) - .await && !services - .rooms - .state_cache - .server_in_room(body.origin(), &body.room_id) - .await - { - return Err!(Request(Forbidden("Server is not in room."))); + AccessCheck { + services: &services, + origin: body.origin(), + room_id: &body.room_id, + event_id: None, } + .check() + .await?; let until = body .v @@ -70,7 +60,6 @@ pub(crate) async fn get_backfill_route( .state_accessor .server_can_see_event(origin, &pdu.room_id, &pdu.event_id) .await - .is_ok_and(is_equal_to!(true)) { return None; } diff --git a/src/api/server/event.rs b/src/api/server/event.rs index 64ce3e40..29d5d870 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -1,7 +1,8 @@ use axum::extract::State; -use conduit::{err, Err, Result}; +use conduit::{err, Result}; use ruma::{api::federation::event::get_event, MilliSecondsSinceUnixEpoch, RoomId}; +use super::AccessCheck; use crate::Ruma; /// # `GET /_matrix/federation/v1/event/{eventId}` @@ -20,35 +21,21 @@ pub(crate) async fn get_event_route( .await .map_err(|_| err!(Request(NotFound("Event not found."))))?; - let room_id_str = event + let room_id: &RoomId = event .get("room_id") .and_then(|val| val.as_str()) - .ok_or_else(|| err!(Database("Invalid event in database.")))?; + .ok_or_else(|| err!(Database("Invalid event in database.")))? + .try_into() + .map_err(|_| err!(Database("Invalid room_id in event in database.")))?; - let room_id = - <&RoomId>::try_from(room_id_str).map_err(|_| err!(Database("Invalid room_id in event in database.")))?; - - if !services - .rooms - .state_accessor - .is_world_readable(room_id) - .await && !services - .rooms - .state_cache - .server_in_room(body.origin(), room_id) - .await - { - return Err!(Request(Forbidden("Server is not in room."))); - } - - if !services - .rooms - .state_accessor - .server_can_see_event(body.origin(), room_id, &body.event_id) - .await? - { - return Err!(Request(Forbidden("Server is not allowed to see event."))); + AccessCheck { + services: &services, + origin: body.origin(), + room_id, + event_id: Some(&body.event_id), } + .check() + .await?; Ok(get_event::v1::Response { origin: services.globals.server_name().to_owned(), diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 8fe96f81..faeb2b99 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -8,6 +8,7 @@ use ruma::{ RoomId, }; +use super::AccessCheck; use crate::Ruma; /// # `GET /_matrix/federation/v1/event_auth/{roomId}/{eventId}` @@ -18,24 +19,14 @@ use crate::Ruma; pub(crate) async fn get_event_authorization_route( State(services): State, body: Ruma, ) -> Result { - services - .rooms - .event_handler - .acl_check(body.origin(), &body.room_id) - .await?; - - if !services - .rooms - .state_accessor - .is_world_readable(&body.room_id) - .await && !services - .rooms - .state_cache - .server_in_room(body.origin(), &body.room_id) - .await - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room.")); + AccessCheck { + services: &services, + origin: body.origin(), + room_id: &body.room_id, + event_id: None, } + .check() + .await?; let event = services .rooms diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index aee4fbe9..7dff44dc 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -5,6 +5,7 @@ use ruma::{ CanonicalJsonValue, EventId, RoomId, }; +use super::AccessCheck; use crate::Ruma; /// # `POST /_matrix/federation/v1/get_missing_events/{roomId}` @@ -13,29 +14,16 @@ use crate::Ruma; pub(crate) async fn get_missing_events_route( State(services): State, body: Ruma, ) -> Result { - services - .rooms - .event_handler - .acl_check(body.origin(), &body.room_id) - .await?; - - if !services - .rooms - .state_accessor - .is_world_readable(&body.room_id) - .await && !services - .rooms - .state_cache - .server_in_room(body.origin(), &body.room_id) - .await - { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Server is not in room")); + AccessCheck { + services: &services, + origin: body.origin(), + room_id: &body.room_id, + event_id: None, } + .check() + .await?; - let limit = body - .limit - .try_into() - .expect("UInt could not be converted to usize"); + let limit = body.limit.try_into()?; let mut queued_events = body.latest_events.clone(); // the vec will never have more entries the limit @@ -70,7 +58,7 @@ pub(crate) async fn get_missing_events_route( .rooms .state_accessor .server_can_see_event(body.origin(), &body.room_id, &queued_events[i]) - .await? + .await { i = i.saturating_add(1); continue; diff --git a/src/api/server/mod.rs b/src/api/server/mod.rs index 9a184f23..9b7d91cb 100644 --- a/src/api/server/mod.rs +++ b/src/api/server/mod.rs @@ -41,3 +41,6 @@ pub(super) use state_ids::*; pub(super) use user::*; pub(super) use version::*; pub(super) use well_known::*; + +mod utils; +use utils::AccessCheck; diff --git a/src/api/server/state.rs b/src/api/server/state.rs index 59bb6c7b..06a44a99 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -1,10 +1,11 @@ use std::borrow::Borrow; use axum::extract::State; -use conduit::{err, result::LogErr, utils::IterStream, Err, Result}; +use conduit::{err, result::LogErr, utils::IterStream, Result}; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::api::federation::event::get_room_state; +use super::AccessCheck; use crate::Ruma; /// # `GET /_matrix/federation/v1/state/{roomId}` @@ -13,24 +14,14 @@ use crate::Ruma; pub(crate) async fn get_room_state_route( State(services): State, body: Ruma, ) -> Result { - services - .rooms - .event_handler - .acl_check(body.origin(), &body.room_id) - .await?; - - if !services - .rooms - .state_accessor - .is_world_readable(&body.room_id) - .await && !services - .rooms - .state_cache - .server_in_room(body.origin(), &body.room_id) - .await - { - return Err!(Request(Forbidden("Server is not in room."))); + AccessCheck { + services: &services, + origin: body.origin(), + room_id: &body.room_id, + event_id: None, } + .check() + .await?; let shortstatehash = services .rooms diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index 957a2a86..52d8e7cc 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -1,11 +1,12 @@ use std::borrow::Borrow; use axum::extract::State; -use conduit::{err, Err}; +use conduit::{err, Result}; use futures::StreamExt; use ruma::api::federation::event::get_room_state_ids; -use crate::{Result, Ruma}; +use super::AccessCheck; +use crate::Ruma; /// # `GET /_matrix/federation/v1/state_ids/{roomId}` /// @@ -14,24 +15,14 @@ use crate::{Result, Ruma}; pub(crate) async fn get_room_state_ids_route( State(services): State, body: Ruma, ) -> Result { - services - .rooms - .event_handler - .acl_check(body.origin(), &body.room_id) - .await?; - - if !services - .rooms - .state_accessor - .is_world_readable(&body.room_id) - .await && !services - .rooms - .state_cache - .server_in_room(body.origin(), &body.room_id) - .await - { - return Err!(Request(Forbidden("Server is not in room."))); + AccessCheck { + services: &services, + origin: body.origin(), + room_id: &body.room_id, + event_id: None, } + .check() + .await?; let shortstatehash = services .rooms diff --git a/src/api/server/utils.rs b/src/api/server/utils.rs new file mode 100644 index 00000000..278465ca --- /dev/null +++ b/src/api/server/utils.rs @@ -0,0 +1,60 @@ +use conduit::{implement, is_false, Err, Result}; +use conduit_service::Services; +use futures::{future::OptionFuture, join, FutureExt}; +use ruma::{EventId, RoomId, ServerName}; + +pub(super) struct AccessCheck<'a> { + pub(super) services: &'a Services, + pub(super) origin: &'a ServerName, + pub(super) room_id: &'a RoomId, + pub(super) event_id: Option<&'a EventId>, +} + +#[implement(AccessCheck, params = "<'_>")] +pub(super) async fn check(&self) -> Result { + let acl_check = self + .services + .rooms + .event_handler + .acl_check(self.origin, self.room_id) + .map(|result| result.is_ok()); + + let world_readable = self + .services + .rooms + .state_accessor + .is_world_readable(self.room_id); + + let server_in_room = self + .services + .rooms + .state_cache + .server_in_room(self.origin, self.room_id); + + let server_can_see: OptionFuture<_> = self + .event_id + .map(|event_id| { + self.services + .rooms + .state_accessor + .server_can_see_event(self.origin, self.room_id, event_id) + }) + .into(); + + let (world_readable, server_in_room, server_can_see, acl_check) = + join!(world_readable, server_in_room, server_can_see, acl_check); + + if !acl_check { + return Err!(Request(Forbidden("Server access denied."))); + } + + if !world_readable && !server_in_room { + return Err!(Request(Forbidden("Server is not in room."))); + } + + if server_can_see.is_some_and(is_false!()) { + return Err!(Request(Forbidden("Server is not allowed to see event."))); + } + + Ok(()) +} From 6b0eb7608d06fbfce663e4775196cfd3c7bae643 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 31 Oct 2024 07:33:16 +0000 Subject: [PATCH 0995/2091] add Filter extension to Result Signed-off-by: Jason Volk --- src/core/utils/result.rs | 5 +++-- src/core/utils/result/filter.rs | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 src/core/utils/result/filter.rs diff --git a/src/core/utils/result.rs b/src/core/utils/result.rs index 9a60d19e..fb1b7b95 100644 --- a/src/core/utils/result.rs +++ b/src/core/utils/result.rs @@ -1,4 +1,5 @@ mod debug_inspect; +mod filter; mod flat_ok; mod into_is_ok; mod log_debug_err; @@ -8,8 +9,8 @@ mod not_found; mod unwrap_infallible; pub use self::{ - debug_inspect::DebugInspect, flat_ok::FlatOk, into_is_ok::IntoIsOk, log_debug_err::LogDebugErr, log_err::LogErr, - map_expect::MapExpect, not_found::NotFound, unwrap_infallible::UnwrapInfallible, + debug_inspect::DebugInspect, filter::Filter, flat_ok::FlatOk, into_is_ok::IntoIsOk, log_debug_err::LogDebugErr, + log_err::LogErr, map_expect::MapExpect, not_found::NotFound, unwrap_infallible::UnwrapInfallible, }; pub type Result = std::result::Result; diff --git a/src/core/utils/result/filter.rs b/src/core/utils/result/filter.rs new file mode 100644 index 00000000..f11d3632 --- /dev/null +++ b/src/core/utils/result/filter.rs @@ -0,0 +1,21 @@ +use super::Result; + +pub trait Filter { + /// Similar to Option::filter + #[must_use] + fn filter(self, predicate: P) -> Self + where + P: FnOnce(&T) -> Result<(), U>, + E: From; +} + +impl Filter for Result { + #[inline] + fn filter(self, predicate: P) -> Self + where + P: FnOnce(&T) -> Result<(), U>, + E: From, + { + self.and_then(move |t| predicate(&t).map(move |()| t).map_err(Into::into)) + } +} From 0bc6fdd5897c9a216ced0433532eb47a97f142ac Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 31 Oct 2024 08:19:37 +0000 Subject: [PATCH 0996/2091] Refactor ShortStateInfo et al to properly named structures Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 9 ++- src/api/client/membership.rs | 14 ++++- src/service/rooms/event_handler/mod.rs | 15 +++-- src/service/rooms/short/mod.rs | 1 + src/service/rooms/state/mod.rs | 8 +-- src/service/rooms/state_accessor/data.rs | 6 +- src/service/rooms/state_compressor/mod.rs | 77 +++++++++++++++-------- 7 files changed, 89 insertions(+), 41 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index db102858..754c9840 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -12,6 +12,7 @@ use ruma::{ events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, }; +use service::rooms::state_compressor::HashSetCompressStateEvent; use tracing_subscriber::EnvFilter; use crate::admin_command; @@ -632,7 +633,11 @@ pub(super) async fn force_set_room_state_from_server( .await?; info!("Forcing new room state"); - let (short_state_hash, new, removed) = self + let HashSetCompressStateEvent { + shortstatehash: short_state_hash, + added, + removed, + } = self .services .rooms .state_compressor @@ -643,7 +648,7 @@ pub(super) async fn force_set_room_state_from_server( self.services .rooms .state - .force_state(room_id.clone().as_ref(), short_state_hash, new, removed, &state_lock) + .force_state(room_id.clone().as_ref(), short_state_hash, added, removed, &state_lock) .await?; info!( diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 27de60c6..c41e93fa 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -39,7 +39,11 @@ use ruma::{ state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; -use service::{appservice::RegistrationInfo, rooms::state::RoomMutexGuard, Services}; +use service::{ + appservice::RegistrationInfo, + rooms::{state::RoomMutexGuard, state_compressor::HashSetCompressStateEvent}, + Services, +}; use crate::{client::full_user_deactivate, Ruma}; @@ -941,7 +945,11 @@ async fn join_room_by_id_helper_remote( .await; debug!("Saving compressed state"); - let (statehash_before_join, new, removed) = services + let HashSetCompressStateEvent { + shortstatehash: statehash_before_join, + added, + removed, + } = services .rooms .state_compressor .save_state(room_id, Arc::new(compressed)) @@ -951,7 +959,7 @@ async fn join_room_by_id_helper_remote( services .rooms .state - .force_state(room_id, statehash_before_join, new, removed, &state_lock) + .force_state(room_id, statehash_before_join, added, removed, &state_lock) .await?; info!("Updating joined counts for new room"); diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index ec04e748..adebd332 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -33,8 +33,11 @@ use ruma::{ RoomId, RoomVersionId, ServerName, UserId, }; -use super::state_compressor::CompressedStateEvent; -use crate::{globals, rooms, sending, server_keys, Dep}; +use crate::{ + globals, rooms, + rooms::state_compressor::{CompressedStateEvent, HashSetCompressStateEvent}, + sending, server_keys, Dep, +}; pub struct Service { services: Services, @@ -692,7 +695,11 @@ impl Service { // Set the new room state to the resolved state debug!("Forcing new room state"); - let (sstatehash, new, removed) = self + let HashSetCompressStateEvent { + shortstatehash, + added, + removed, + } = self .services .state_compressor .save_state(room_id, new_room_state) @@ -700,7 +707,7 @@ impl Service { self.services .state - .force_state(room_id, sstatehash, new, removed, &state_lock) + .force_state(room_id, shortstatehash, added, removed, &state_lock) .await?; } diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 02c449cc..62011605 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -24,6 +24,7 @@ struct Services { globals: Dep, } +pub type ShortStateKey = ShortId; pub type ShortEventId = ShortId; pub type ShortRoomId = ShortId; pub type ShortId = u64; diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 6abaa198..34fab079 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -182,12 +182,12 @@ impl Service { let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { let statediffnew: HashSet<_> = state_ids_compressed - .difference(&parent_stateinfo.1) + .difference(&parent_stateinfo.full_state) .copied() .collect(); let statediffremoved: HashSet<_> = parent_stateinfo - .1 + .full_state .difference(&state_ids_compressed) .copied() .collect(); @@ -259,7 +259,7 @@ impl Service { let replaces = states_parents .last() .map(|info| { - info.1 + info.full_state .iter() .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) }) @@ -421,7 +421,7 @@ impl Service { })? .pop() .expect("there is always one layer") - .1; + .full_state; let mut ret = HashMap::new(); for compressed in full_state.iter() { diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index adc26f00..f77a6d80 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -45,7 +45,7 @@ impl Data { .map_err(|e| err!(Database("Missing state IDs: {e}")))? .pop() .expect("there is always one layer") - .1; + .full_state; let mut result = HashMap::new(); let mut i: u8 = 0; @@ -78,7 +78,7 @@ impl Data { .await? .pop() .expect("there is always one layer") - .1; + .full_state; let mut result = HashMap::new(); let mut i: u8 = 0; @@ -123,7 +123,7 @@ impl Data { .map_err(|e| err!(Database(error!(?event_type, ?state_key, "Missing state: {e:?}"))))? .pop() .expect("there is always one layer") - .1; + .full_state; let compressed = full_state .iter() diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index be66c597..1f351f40 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -10,7 +10,7 @@ use database::Map; use lru_cache::LruCache; use ruma::{EventId, RoomId}; -use crate::{rooms, Dep}; +use crate::{rooms, rooms::short::ShortId, Dep}; pub struct Service { pub stateinfo_cache: Mutex, @@ -27,24 +27,33 @@ struct Data { shortstatehash_statediff: Arc, } +#[derive(Clone)] struct StateDiff { parent: Option, added: Arc>, removed: Arc>, } +#[derive(Clone, Default)] +pub struct ShortStateInfo { + pub shortstatehash: ShortStateHash, + pub full_state: Arc>, + pub added: Arc>, + pub removed: Arc>, +} + +#[derive(Clone, Default)] +pub struct HashSetCompressStateEvent { + pub shortstatehash: ShortStateHash, + pub added: Arc>, + pub removed: Arc>, +} + +pub type ShortStateHash = ShortId; +pub(crate) type CompressedStateEvent = [u8; 2 * size_of::()]; type StateInfoLruCache = LruCache; type ShortStateInfoVec = Vec; type ParentStatesVec = Vec; -type ShortStateInfo = ( - u64, // sstatehash - Arc>, // full state - Arc>, // added - Arc>, // removed -); - -type HashSetCompressStateEvent = (u64, Arc>, Arc>); -pub type CompressedStateEvent = [u8; 2 * size_of::()]; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -95,14 +104,19 @@ impl Service { if let Some(parent) = parent { let mut response = Box::pin(self.load_shortstatehash_info(parent)).await?; - let mut state = (*response.last().expect("at least one response").1).clone(); + let mut state = (*response.last().expect("at least one response").full_state).clone(); state.extend(added.iter().copied()); let removed = (*removed).clone(); for r in &removed { state.remove(r); } - response.push((shortstatehash, Arc::new(state), added, Arc::new(removed))); + response.push(ShortStateInfo { + shortstatehash, + full_state: Arc::new(state), + added, + removed: Arc::new(removed), + }); self.stateinfo_cache .lock() @@ -111,7 +125,13 @@ impl Service { Ok(response) } else { - let response = vec![(shortstatehash, added.clone(), added, removed)]; + let response = vec![ShortStateInfo { + shortstatehash, + full_state: added.clone(), + added, + removed, + }]; + self.stateinfo_cache .lock() .expect("locked") @@ -185,8 +205,8 @@ impl Service { // To many layers, we have to go deeper let parent = parent_states.pop().expect("parent must have a state"); - let mut parent_new = (*parent.2).clone(); - let mut parent_removed = (*parent.3).clone(); + let mut parent_new = (*parent.added).clone(); + let mut parent_removed = (*parent.removed).clone(); for removed in statediffremoved.iter() { if !parent_new.remove(removed) { @@ -236,14 +256,14 @@ impl Service { // 2. We replace a layer above let parent = parent_states.pop().expect("parent must have a state"); - let parent_2_len = parent.2.len(); - let parent_3_len = parent.3.len(); - let parent_diff = checked!(parent_2_len + parent_3_len)?; + let parent_added_len = parent.added.len(); + let parent_removed_len = parent.removed.len(); + let parent_diff = checked!(parent_added_len + parent_removed_len)?; if checked!(diffsum * diffsum)? >= checked!(2 * diff_to_sibling * parent_diff)? { // Diff too big, we replace above layer(s) - let mut parent_new = (*parent.2).clone(); - let mut parent_removed = (*parent.3).clone(); + let mut parent_new = (*parent.added).clone(); + let mut parent_removed = (*parent.removed).clone(); for removed in statediffremoved.iter() { if !parent_new.remove(removed) { @@ -275,7 +295,7 @@ impl Service { self.save_statediff( shortstatehash, &StateDiff { - parent: Some(parent.0), + parent: Some(parent.shortstatehash), added: statediffnew, removed: statediffremoved, }, @@ -311,7 +331,10 @@ impl Service { .await; if Some(new_shortstatehash) == previous_shortstatehash { - return Ok((new_shortstatehash, Arc::new(HashSet::new()), Arc::new(HashSet::new()))); + return Ok(HashSetCompressStateEvent { + shortstatehash: new_shortstatehash, + ..Default::default() + }); } let states_parents = if let Some(p) = previous_shortstatehash { @@ -322,12 +345,12 @@ impl Service { let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { let statediffnew: HashSet<_> = new_state_ids_compressed - .difference(&parent_stateinfo.1) + .difference(&parent_stateinfo.full_state) .copied() .collect(); let statediffremoved: HashSet<_> = parent_stateinfo - .1 + .full_state .difference(&new_state_ids_compressed) .copied() .collect(); @@ -347,7 +370,11 @@ impl Service { )?; }; - Ok((new_shortstatehash, statediffnew, statediffremoved)) + Ok(HashSetCompressStateEvent { + shortstatehash: new_shortstatehash, + added: statediffnew, + removed: statediffremoved, + }) } async fn get_statediff(&self, shortstatehash: u64) -> Result { From f746be82c158e0bae2a3311953d49c6c8be2c910 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 31 Oct 2024 08:41:43 +0000 Subject: [PATCH 0997/2091] typename some loose u64 ShortId's Signed-off-by: Jason Volk --- src/service/rooms/short/mod.rs | 33 ++++++++++++----------- src/service/rooms/state_accessor/data.rs | 12 ++++----- src/service/rooms/state_compressor/mod.rs | 25 +++++++++-------- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 62011605..a903ef22 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -24,6 +24,7 @@ struct Services { globals: Dep, } +pub type ShortStateHash = ShortId; pub type ShortStateKey = ShortId; pub type ShortEventId = ShortId; pub type ShortRoomId = ShortId; @@ -50,7 +51,7 @@ impl crate::Service for Service { } #[implement(Service)] -pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { +pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEventId { const BUFSIZE: usize = size_of::(); if let Ok(shorteventid) = self @@ -78,7 +79,7 @@ pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> u64 { } #[implement(Service)] -pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Vec { +pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Vec { self.db .eventid_shorteventid .get_batch_blocking(event_ids.iter()) @@ -106,7 +107,7 @@ pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> } #[implement(Service)] -pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { +pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { let key = (event_type, state_key); self.db .statekey_shortstatekey @@ -116,8 +117,8 @@ pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &s } #[implement(Service)] -pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> u64 { - const BUFSIZE: usize = size_of::(); +pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> ShortStateKey { + const BUFSIZE: usize = size_of::(); let key = (event_type, state_key); if let Ok(shortstatekey) = self @@ -145,8 +146,8 @@ pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, sta } #[implement(Service)] -pub async fn get_eventid_from_short(&self, shorteventid: u64) -> Result> { - const BUFSIZE: usize = size_of::(); +pub async fn get_eventid_from_short(&self, shorteventid: ShortEventId) -> Result> { + const BUFSIZE: usize = size_of::(); self.db .shorteventid_eventid @@ -157,8 +158,8 @@ pub async fn get_eventid_from_short(&self, shorteventid: u64) -> Result Vec>> { - const BUFSIZE: usize = size_of::(); +pub async fn multi_get_eventid_from_short(&self, shorteventid: &[ShortEventId]) -> Vec>> { + const BUFSIZE: usize = size_of::(); let keys: Vec<[u8; BUFSIZE]> = shorteventid .iter() @@ -174,8 +175,8 @@ pub async fn multi_get_eventid_from_short(&self, shorteventid: &[u64]) -> Vec Result<(StateEventType, String)> { - const BUFSIZE: usize = size_of::(); +pub async fn get_statekey_from_short(&self, shortstatekey: ShortStateKey) -> Result<(StateEventType, String)> { + const BUFSIZE: usize = size_of::(); self.db .shortstatekey_statekey @@ -191,8 +192,8 @@ pub async fn get_statekey_from_short(&self, shortstatekey: u64) -> Result<(State /// Returns (shortstatehash, already_existed) #[implement(Service)] -pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, bool) { - const BUFSIZE: usize = size_of::(); +pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (ShortStateHash, bool) { + const BUFSIZE: usize = size_of::(); if let Ok(shortstatehash) = self .db @@ -215,19 +216,19 @@ pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (u64, boo } #[implement(Service)] -pub async fn get_shortroomid(&self, room_id: &RoomId) -> Result { +pub async fn get_shortroomid(&self, room_id: &RoomId) -> Result { self.db.roomid_shortroomid.get(room_id).await.deserialized() } #[implement(Service)] -pub async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> u64 { +pub async fn get_or_create_shortroomid(&self, room_id: &RoomId) -> ShortRoomId { self.db .roomid_shortroomid .get(room_id) .await .deserialized() .unwrap_or_else(|_| { - const BUFSIZE: usize = size_of::(); + const BUFSIZE: usize = size_of::(); let short = self.services.globals.next_count().unwrap(); debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index f77a6d80..9c96785f 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -5,7 +5,7 @@ use database::{Deserialized, Map}; use futures::TryFutureExt; use ruma::{events::StateEventType, EventId, RoomId}; -use crate::{rooms, Dep}; +use crate::{rooms, rooms::short::ShortStateHash, Dep}; pub(super) struct Data { eventid_shorteventid: Arc, @@ -36,7 +36,7 @@ impl Data { } #[allow(unused_qualifications)] // async traits - pub(super) async fn state_full_ids(&self, shortstatehash: u64) -> Result>> { + pub(super) async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result>> { let full_state = self .services .state_compressor @@ -69,7 +69,7 @@ impl Data { #[allow(unused_qualifications)] // async traits pub(super) async fn state_full( - &self, shortstatehash: u64, + &self, shortstatehash: ShortStateHash, ) -> Result>> { let full_state = self .services @@ -107,7 +107,7 @@ impl Data { /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). #[allow(clippy::unused_self)] pub(super) async fn state_get_id( - &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, + &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result> { let shortstatekey = self .services @@ -147,7 +147,7 @@ impl Data { /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). pub(super) async fn state_get( - &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, + &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result> { self.state_get_id(shortstatehash, event_type, state_key) .and_then(|event_id| async move { self.services.timeline.get_pdu(&event_id).await }) @@ -155,7 +155,7 @@ impl Data { } /// Returns the state hash for this pdu. - pub(super) async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { + pub(super) async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { self.eventid_shorteventid .get(event_id) .and_then(|shorteventid| self.shorteventid_shortstatehash.get(&shorteventid)) diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 1f351f40..e213490b 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -10,7 +10,11 @@ use database::Map; use lru_cache::LruCache; use ruma::{EventId, RoomId}; -use crate::{rooms, rooms::short::ShortId, Dep}; +use crate::{ + rooms, + rooms::short::{ShortStateHash, ShortStateKey}, + Dep, +}; pub struct Service { pub stateinfo_cache: Mutex, @@ -49,9 +53,8 @@ pub struct HashSetCompressStateEvent { pub removed: Arc>, } -pub type ShortStateHash = ShortId; pub(crate) type CompressedStateEvent = [u8; 2 * size_of::()]; -type StateInfoLruCache = LruCache; +type StateInfoLruCache = LruCache; type ShortStateInfoVec = Vec; type ParentStatesVec = Vec; @@ -86,7 +89,7 @@ impl crate::Service for Service { impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. - pub async fn load_shortstatehash_info(&self, shortstatehash: u64) -> Result { + pub async fn load_shortstatehash_info(&self, shortstatehash: ShortStateHash) -> Result { if let Some(r) = self .stateinfo_cache .lock() @@ -141,7 +144,7 @@ impl Service { } } - pub async fn compress_state_event(&self, shortstatekey: u64, event_id: &EventId) -> CompressedStateEvent { + pub async fn compress_state_event(&self, shortstatekey: ShortStateKey, event_id: &EventId) -> CompressedStateEvent { let mut v = shortstatekey.to_be_bytes().to_vec(); v.extend_from_slice( &self @@ -159,7 +162,7 @@ impl Service { #[inline] pub async fn parse_compressed_state_event( &self, compressed_event: &CompressedStateEvent, - ) -> Result<(u64, Arc)> { + ) -> Result<(ShortStateKey, Arc)> { use utils::u64_from_u8; let shortstatekey = u64_from_u8(&compressed_event[0..size_of::()]); @@ -192,7 +195,7 @@ impl Service { /// added diff and removed diff for each parent layer #[tracing::instrument(skip_all, level = "debug")] pub fn save_state_from_diff( - &self, shortstatehash: u64, statediffnew: Arc>, + &self, shortstatehash: ShortStateHash, statediffnew: Arc>, statediffremoved: Arc>, diff_to_sibling: usize, mut parent_states: ParentStatesVec, ) -> Result { @@ -377,9 +380,9 @@ impl Service { }) } - async fn get_statediff(&self, shortstatehash: u64) -> Result { - const BUFSIZE: usize = size_of::(); - const STRIDE: usize = size_of::(); + async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result { + const BUFSIZE: usize = size_of::(); + const STRIDE: usize = size_of::(); let value = self .db @@ -418,7 +421,7 @@ impl Service { }) } - fn save_statediff(&self, shortstatehash: u64, diff: &StateDiff) { + fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) { let mut value = diff.parent.unwrap_or(0).to_be_bytes().to_vec(); for new in diff.added.iter() { value.extend_from_slice(&new[..]); From 1f1e2d547cceda68da71855186abc38a3a1ba713 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 31 Oct 2024 11:49:00 +0000 Subject: [PATCH 0998/2091] optimize override ips; utilize all ips from cache Signed-off-by: Jason Volk --- src/service/resolver/actual.rs | 6 +++--- src/service/resolver/cache.rs | 10 +++++++--- src/service/resolver/dns.rs | 12 +++++------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 61eedca5..5dc03d14 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -10,7 +10,7 @@ use ipaddress::IPAddress; use ruma::ServerName; use super::{ - cache::{CachedDest, CachedOverride}, + cache::{CachedDest, CachedOverride, MAX_IPS}, fed::{add_port_to_hostname, get_ip_with_port, FedDest, PortString}, }; @@ -266,9 +266,9 @@ impl super::Service { } self.set_cached_override( - overname.to_owned(), + overname, CachedOverride { - ips: override_ip.iter().collect(), + ips: override_ip.into_iter().take(MAX_IPS).collect(), port, expire: CachedOverride::default_expire(), }, diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index 465b5985..a13399dc 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -5,6 +5,7 @@ use std::{ time::SystemTime, }; +use arrayvec::ArrayVec; use conduit::{trace, utils::rand}; use ruma::{OwnedServerName, ServerName}; @@ -24,7 +25,7 @@ pub struct CachedDest { #[derive(Clone, Debug)] pub struct CachedOverride { - pub ips: Vec, + pub ips: IpAddrs, pub port: u16, pub expire: SystemTime, } @@ -32,6 +33,9 @@ pub struct CachedOverride { pub type WellKnownMap = HashMap; pub type TlsNameMap = HashMap; +pub type IpAddrs = ArrayVec; +pub(crate) const MAX_IPS: usize = 3; + impl Cache { pub(super) fn new() -> Arc { Arc::new(Self { @@ -61,13 +65,13 @@ impl super::Service { .cloned() } - pub fn set_cached_override(&self, name: String, over: CachedOverride) -> Option { + pub fn set_cached_override(&self, name: &str, over: CachedOverride) -> Option { trace!(?name, ?over, "set cached override"); self.cache .overrides .write() .expect("locked for writing") - .insert(name, over) + .insert(name.into(), over) } #[must_use] diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index 89129e03..d3e9f5c9 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -1,4 +1,4 @@ -use std::{iter, net::SocketAddr, sync::Arc, time::Duration}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; use conduit::{err, Result, Server}; use futures::FutureExt; @@ -101,14 +101,12 @@ impl Resolve for Hooked { } async fn cached_to_reqwest(cached: CachedOverride) -> ResolvingResult { - let first_ip = cached + let addrs = cached .ips - .first() - .expect("must provide at least one override"); + .into_iter() + .map(move |ip| SocketAddr::new(ip, cached.port)); - let saddr = SocketAddr::new(*first_ip, cached.port); - - Ok(Box::new(iter::once(saddr))) + Ok(Box::new(addrs)) } async fn resolve_to_reqwest(resolver: Arc, name: Name) -> ResolvingResult { From ba1c13468942dce17d78e394ffabe2796cfed577 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 1 Nov 2024 22:16:14 +0000 Subject: [PATCH 0999/2091] move migrations out of globals service Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 1 - src/service/media/migrations.rs | 4 ++-- src/service/{globals => }/migrations.rs | 0 src/service/mod.rs | 1 + src/service/services.rs | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) rename src/service/{globals => }/migrations.rs (100%) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 0a7dda9f..bd956964 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -1,5 +1,4 @@ mod data; -pub(super) mod migrations; use std::{ collections::HashMap, diff --git a/src/service/media/migrations.rs b/src/service/media/migrations.rs index 0e358d44..f1c6da7d 100644 --- a/src/service/media/migrations.rs +++ b/src/service/media/migrations.rs @@ -13,7 +13,7 @@ use conduit::{ warn, Config, Result, }; -use crate::{globals, Services}; +use crate::{migrations, Services}; /// Migrates a media directory from legacy base64 file names to sha2 file names. /// All errors are fatal. Upon success the database is keyed to not perform this @@ -50,7 +50,7 @@ pub(crate) async fn migrate_sha256_media(services: &Services) -> Result<()> { // Apply fix from when sha256_media was backward-incompat and bumped the schema // version from 13 to 14. For users satisfying these conditions we can go back. - if services.globals.db.database_version().await == 14 && globals::migrations::DATABASE_VERSION == 13 { + if services.globals.db.database_version().await == 14 && migrations::DATABASE_VERSION == 13 { services.globals.db.bump_database_version(13)?; } diff --git a/src/service/globals/migrations.rs b/src/service/migrations.rs similarity index 100% rename from src/service/globals/migrations.rs rename to src/service/migrations.rs diff --git a/src/service/mod.rs b/src/service/mod.rs index 604e3404..c7dcc0c6 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,6 +1,7 @@ #![allow(refining_impl_trait)] mod manager; +mod migrations; mod service; pub mod services; diff --git a/src/service/services.rs b/src/service/services.rs index c0af4249..b86e7a72 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -114,7 +114,7 @@ impl Services { debug_info!("Starting services..."); self.admin.set_services(Some(Arc::clone(self)).as_ref()); - globals::migrations::migrations(self).await?; + super::migrations::migrations(self).await?; self.manager .lock() .await From 87424370364fa391214d4a55feaf9d3606add9e0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 1 Nov 2024 22:43:26 +0000 Subject: [PATCH 1000/2091] wrap unimplemented ser/de branches with internal macro Signed-off-by: Jason Volk --- src/database/de.rs | 34 ++++++++++++++++--------------- src/database/mod.rs | 2 +- src/database/ser.rs | 48 ++++++++++++++++++++++---------------------- src/database/util.rs | 23 +++++++++++++++++++++ 4 files changed, 66 insertions(+), 41 deletions(-) diff --git a/src/database/de.rs b/src/database/de.rs index 0e074fdb..d7dc1102 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -5,6 +5,8 @@ use serde::{ Deserialize, }; +use crate::util::unhandled; + /// Deserialize into T from buffer. pub(crate) fn from_slice<'a, T>(buf: &'a [u8]) -> Result where @@ -192,7 +194,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { match name { "Ignore" => self.record_ignore(), "IgnoreAll" => self.record_ignore_all(), - _ => unimplemented!("Unrecognized deserialization Directive {name:?}"), + _ => unhandled!("Unrecognized deserialization Directive {name:?}"), }; visitor.visit_unit() @@ -214,27 +216,27 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { where V: Visitor<'de>, { - unimplemented!("deserialize Enum not implemented") + unhandled!("deserialize Enum not implemented") } fn deserialize_option>(self, _visitor: V) -> Result { - unimplemented!("deserialize Option not implemented") + unhandled!("deserialize Option not implemented") } fn deserialize_bool>(self, _visitor: V) -> Result { - unimplemented!("deserialize bool not implemented") + unhandled!("deserialize bool not implemented") } fn deserialize_i8>(self, _visitor: V) -> Result { - unimplemented!("deserialize i8 not implemented") + unhandled!("deserialize i8 not implemented") } fn deserialize_i16>(self, _visitor: V) -> Result { - unimplemented!("deserialize i16 not implemented") + unhandled!("deserialize i16 not implemented") } fn deserialize_i32>(self, _visitor: V) -> Result { - unimplemented!("deserialize i32 not implemented") + unhandled!("deserialize i32 not implemented") } fn deserialize_i64>(self, visitor: V) -> Result { @@ -244,15 +246,15 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } fn deserialize_u8>(self, _visitor: V) -> Result { - unimplemented!("deserialize u8 not implemented; try dereferencing the Handle for [u8] access instead") + unhandled!("deserialize u8 not implemented; try dereferencing the Handle for [u8] access instead") } fn deserialize_u16>(self, _visitor: V) -> Result { - unimplemented!("deserialize u16 not implemented") + unhandled!("deserialize u16 not implemented") } fn deserialize_u32>(self, _visitor: V) -> Result { - unimplemented!("deserialize u32 not implemented") + unhandled!("deserialize u32 not implemented") } fn deserialize_u64>(self, visitor: V) -> Result { @@ -262,15 +264,15 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } fn deserialize_f32>(self, _visitor: V) -> Result { - unimplemented!("deserialize f32 not implemented") + unhandled!("deserialize f32 not implemented") } fn deserialize_f64>(self, _visitor: V) -> Result { - unimplemented!("deserialize f64 not implemented") + unhandled!("deserialize f64 not implemented") } fn deserialize_char>(self, _visitor: V) -> Result { - unimplemented!("deserialize char not implemented") + unhandled!("deserialize char not implemented") } fn deserialize_str>(self, visitor: V) -> Result { @@ -291,11 +293,11 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } fn deserialize_byte_buf>(self, _visitor: V) -> Result { - unimplemented!("deserialize Byte Buf not implemented") + unhandled!("deserialize Byte Buf not implemented") } fn deserialize_unit>(self, _visitor: V) -> Result { - unimplemented!("deserialize Unit not implemented") + unhandled!("deserialize Unit not implemented") } // this only used for $serde_json::private::RawValue at this time; see MapAccess @@ -305,7 +307,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } fn deserialize_ignored_any>(self, _visitor: V) -> Result { - unimplemented!("deserialize Ignored Any not implemented") + unhandled!("deserialize Ignored Any not implemented") } fn deserialize_any>(self, visitor: V) -> Result { diff --git a/src/database/mod.rs b/src/database/mod.rs index dcd66a1e..f09c4a71 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -11,7 +11,7 @@ mod opts; mod ser; mod stream; mod tests; -mod util; +pub(crate) mod util; mod watchers; pub(crate) use self::{ diff --git a/src/database/ser.rs b/src/database/ser.rs index 0cc5c886..961d2700 100644 --- a/src/database/ser.rs +++ b/src/database/ser.rs @@ -4,6 +4,8 @@ use arrayvec::ArrayVec; use conduit::{debug::type_name, err, result::DebugInspect, utils::exchange, Error, Result}; use serde::{ser, Serialize}; +use crate::util::unhandled; + #[inline] pub fn serialize_to_array(val: T) -> Result> where @@ -146,17 +148,15 @@ impl ser::Serializer for &mut Serializer<'_, W> { fn serialize_tuple_variant( self, _name: &'static str, _idx: u32, _var: &'static str, _len: usize, ) -> Result { - unimplemented!("serialize Tuple Variant not implemented") + unhandled!("serialize Tuple Variant not implemented") } fn serialize_map(self, _len: Option) -> Result { - unimplemented!( - "serialize Map not implemented; did you mean to use database::Json() around your serde_json::Value?" - ) + unhandled!("serialize Map not implemented; did you mean to use database::Json() around your serde_json::Value?") } fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { - unimplemented!( + unhandled!( "serialize Struct not implemented at this time; did you mean to use database::Json() around your struct?" ) } @@ -164,7 +164,7 @@ impl ser::Serializer for &mut Serializer<'_, W> { fn serialize_struct_variant( self, _name: &'static str, _idx: u32, _var: &'static str, _len: usize, ) -> Result { - unimplemented!("serialize Struct Variant not implemented") + unhandled!("serialize Struct Variant not implemented") } #[allow(clippy::needless_borrows_for_generic_args)] // buggy @@ -179,14 +179,14 @@ impl ser::Serializer for &mut Serializer<'_, W> { match name { "Json" => serde_json::to_writer(&mut self.out, value).map_err(Into::into), - _ => unimplemented!("Unrecognized serialization Newtype {name:?}"), + _ => unhandled!("Unrecognized serialization Newtype {name:?}"), } } fn serialize_newtype_variant( self, _name: &'static str, _idx: u32, _var: &'static str, _value: &T, ) -> Result { - unimplemented!("serialize Newtype Variant not implemented") + unhandled!("serialize Newtype Variant not implemented") } fn serialize_unit_struct(self, name: &'static str) -> Result { @@ -197,14 +197,14 @@ impl ser::Serializer for &mut Serializer<'_, W> { "Separator" => { self.separator()?; }, - _ => unimplemented!("Unrecognized serialization directive: {name:?}"), + _ => unhandled!("Unrecognized serialization directive: {name:?}"), }; Ok(()) } fn serialize_unit_variant(self, _name: &'static str, _idx: u32, _var: &'static str) -> Result { - unimplemented!("serialize Unit Variant not implemented") + unhandled!("serialize Unit Variant not implemented") } fn serialize_some(self, val: &T) -> Result { val.serialize(self) } @@ -234,29 +234,29 @@ impl ser::Serializer for &mut Serializer<'_, W> { self.write(v) } - fn serialize_f64(self, _v: f64) -> Result { unimplemented!("serialize f64 not implemented") } + fn serialize_f64(self, _v: f64) -> Result { unhandled!("serialize f64 not implemented") } - fn serialize_f32(self, _v: f32) -> Result { unimplemented!("serialize f32 not implemented") } + fn serialize_f32(self, _v: f32) -> Result { unhandled!("serialize f32 not implemented") } fn serialize_i64(self, v: i64) -> Result { self.write(&v.to_be_bytes()) } fn serialize_i32(self, v: i32) -> Result { self.write(&v.to_be_bytes()) } - fn serialize_i16(self, _v: i16) -> Result { unimplemented!("serialize i16 not implemented") } + fn serialize_i16(self, _v: i16) -> Result { unhandled!("serialize i16 not implemented") } - fn serialize_i8(self, _v: i8) -> Result { unimplemented!("serialize i8 not implemented") } + fn serialize_i8(self, _v: i8) -> Result { unhandled!("serialize i8 not implemented") } fn serialize_u64(self, v: u64) -> Result { self.write(&v.to_be_bytes()) } fn serialize_u32(self, v: u32) -> Result { self.write(&v.to_be_bytes()) } - fn serialize_u16(self, _v: u16) -> Result { unimplemented!("serialize u16 not implemented") } + fn serialize_u16(self, _v: u16) -> Result { unhandled!("serialize u16 not implemented") } fn serialize_u8(self, v: u8) -> Result { self.write(&[v]) } - fn serialize_bool(self, _v: bool) -> Result { unimplemented!("serialize bool not implemented") } + fn serialize_bool(self, _v: bool) -> Result { unhandled!("serialize bool not implemented") } - fn serialize_unit(self) -> Result { unimplemented!("serialize unit not implemented") } + fn serialize_unit(self) -> Result { unhandled!("serialize unit not implemented") } } impl ser::SerializeSeq for &mut Serializer<'_, W> { @@ -309,14 +309,14 @@ impl ser::SerializeMap for &mut Serializer<'_, W> { type Ok = (); fn serialize_key(&mut self, _key: &T) -> Result { - unimplemented!("serialize Map Key not implemented") + unhandled!("serialize Map Key not implemented") } fn serialize_value(&mut self, _val: &T) -> Result { - unimplemented!("serialize Map Val not implemented") + unhandled!("serialize Map Val not implemented") } - fn end(self) -> Result { unimplemented!("serialize Map End not implemented") } + fn end(self) -> Result { unhandled!("serialize Map End not implemented") } } impl ser::SerializeStruct for &mut Serializer<'_, W> { @@ -324,10 +324,10 @@ impl ser::SerializeStruct for &mut Serializer<'_, W> { type Ok = (); fn serialize_field(&mut self, _key: &'static str, _val: &T) -> Result { - unimplemented!("serialize Struct Field not implemented") + unhandled!("serialize Struct Field not implemented") } - fn end(self) -> Result { unimplemented!("serialize Struct End not implemented") } + fn end(self) -> Result { unhandled!("serialize Struct End not implemented") } } impl ser::SerializeStructVariant for &mut Serializer<'_, W> { @@ -335,8 +335,8 @@ impl ser::SerializeStructVariant for &mut Serializer<'_, W> { type Ok = (); fn serialize_field(&mut self, _key: &'static str, _val: &T) -> Result { - unimplemented!("serialize Struct Variant Field not implemented") + unhandled!("serialize Struct Variant Field not implemented") } - fn end(self) -> Result { unimplemented!("serialize Struct Variant End not implemented") } + fn end(self) -> Result { unhandled!("serialize Struct Variant End not implemented") } } diff --git a/src/database/util.rs b/src/database/util.rs index d36e183f..ae076381 100644 --- a/src/database/util.rs +++ b/src/database/util.rs @@ -1,6 +1,29 @@ use conduit::{err, Result}; use rocksdb::{Direction, IteratorMode}; +//#[cfg(debug_assertions)] +macro_rules! unhandled { + ($msg:literal) => { + unimplemented!($msg) + }; +} + +// activate when stable; we're not ready for this yet +#[cfg(disable)] // #[cfg(not(debug_assertions))] +macro_rules! unhandled { + ($msg:literal) => { + // SAFETY: Eliminates branches for serializing and deserializing types never + // encountered in the codebase. This can promote optimization and reduce + // codegen. The developer must verify for every invoking callsite that the + // unhandled type is in no way involved and could not possibly be encountered. + unsafe { + std::hint::unreachable_unchecked(); + } + }; +} + +pub(crate) use unhandled; + #[inline] pub(crate) fn _into_direction(mode: &IteratorMode<'_>) -> Direction { use Direction::{Forward, Reverse}; From f191b4bad4cc6d15584f6c33cac57925e7b67abf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 2 Nov 2024 04:54:28 +0000 Subject: [PATCH 1001/2091] add map_expect for stream Signed-off-by: Jason Volk --- src/core/utils/result/map_expect.rs | 8 ++++---- src/core/utils/stream/expect.rs | 11 ++++++++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/core/utils/result/map_expect.rs b/src/core/utils/result/map_expect.rs index 8ce9195f..9cd498f7 100644 --- a/src/core/utils/result/map_expect.rs +++ b/src/core/utils/result/map_expect.rs @@ -2,14 +2,14 @@ use std::fmt::Debug; use super::Result; -pub trait MapExpect { +pub trait MapExpect<'a, T> { /// Calls expect(msg) on the mapped Result value. This is similar to /// map(Result::unwrap) but composes an expect call and message without /// requiring a closure. - fn map_expect(self, msg: &str) -> Option; + fn map_expect(self, msg: &'a str) -> T; } -impl MapExpect for Option> { +impl<'a, T, E: Debug> MapExpect<'a, Option> for Option> { #[inline] - fn map_expect(self, msg: &str) -> Option { self.map(|result| result.expect(msg)) } + fn map_expect(self, msg: &'a str) -> Option { self.map(|result| result.expect(msg)) } } diff --git a/src/core/utils/stream/expect.rs b/src/core/utils/stream/expect.rs index 3ab7181a..68ac24ce 100644 --- a/src/core/utils/stream/expect.rs +++ b/src/core/utils/stream/expect.rs @@ -4,14 +4,19 @@ use crate::Result; pub trait TryExpect<'a, Item> { fn expect_ok(self) -> impl Stream + Send + 'a; + + fn map_expect(self, msg: &'a str) -> impl Stream + Send + 'a; } impl<'a, T, Item> TryExpect<'a, Item> for T where T: Stream> + TryStream + Send + 'a, + Item: 'a, { #[inline] - fn expect_ok(self: T) -> impl Stream + Send + 'a { - self.map(|res| res.expect("stream expectation failure")) - } + fn expect_ok(self: T) -> impl Stream + Send + 'a { self.map_expect("stream expectation failure") } + + //TODO: move to impl MapExpect + #[inline] + fn map_expect(self, msg: &'a str) -> impl Stream + Send + 'a { self.map(|res| res.expect(msg)) } } From 52f09fdb51f895940a1d0895cd05775ed9aeacbd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 2 Nov 2024 01:59:06 +0000 Subject: [PATCH 1002/2091] add database migration for missing referencedevents separator Signed-off-by: Jason Volk --- src/service/migrations.rs | 59 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 2 deletions(-) diff --git a/src/service/migrations.rs b/src/service/migrations.rs index c953e7b1..45323fa2 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -1,7 +1,12 @@ +use std::cmp; + use conduit::{ - debug_info, debug_warn, error, info, + debug, debug_info, debug_warn, error, info, result::NotFound, - utils::{stream::TryIgnore, IterStream, ReadyExt}, + utils::{ + stream::{TryExpect, TryIgnore}, + IterStream, ReadyExt, + }, warn, Err, Result, }; use futures::{FutureExt, StreamExt}; @@ -120,6 +125,14 @@ async fn migrate(services: &Services) -> Result<()> { retroactively_fix_bad_data_from_roomuserid_joined(services).await?; } + if db["global"] + .get(b"fix_referencedevents_missing_sep") + .await + .is_not_found() + { + fix_referencedevents_missing_sep(services).await?; + } + let version_match = services.globals.db.database_version().await == DATABASE_VERSION || services.globals.db.database_version().await == CONDUIT_DATABASE_VERSION; @@ -444,3 +457,45 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) info!("Finished fixing"); Ok(()) } + +async fn fix_referencedevents_missing_sep(services: &Services) -> Result { + warn!("Fixing missing record separator between room_id and event_id in referencedevents"); + + let db = &services.db; + let cork = db.cork_and_sync(); + + let referencedevents = db["referencedevents"].clone(); + + let totals: (usize, usize) = (0, 0); + let (total, fixed) = referencedevents + .raw_stream() + .expect_ok() + .enumerate() + .ready_fold(totals, |mut a, (i, (key, val))| { + debug_assert!(val.is_empty(), "expected no value"); + + let has_sep = key.contains(&database::SEP); + + if !has_sep { + let key_str = std::str::from_utf8(key).expect("key not utf-8"); + let room_id_len = key_str.find('$').expect("missing '$' in key"); + let (room_id, event_id) = key_str.split_at(room_id_len); + debug!(?a, "fixing {room_id}, {event_id}"); + + let new_key = (room_id, event_id); + referencedevents.put_raw(new_key, val); + referencedevents.remove(key); + } + + a.0 = cmp::max(i, a.0); + a.1 = a.1.saturating_add((!has_sep).into()); + a + }) + .await; + + drop(cork); + info!(?total, ?fixed, "Fixed missing record separators in 'referencedevents'."); + + db["global"].insert(b"fix_referencedevents_missing_sep", []); + db.db.cleanup() +} From 8d251003a25aa697de052f01515e5cc23ce999e4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 3 Nov 2024 12:42:43 +0000 Subject: [PATCH 1003/2091] reduce Error-related codegen; add PoisonError Signed-off-by: Jason Volk --- src/core/config/mod.rs | 1 + src/core/error/err.rs | 1 + src/core/error/mod.rs | 10 +++++++++- src/core/error/panic.rs | 4 ++++ src/core/error/response.rs | 1 + 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index a6216da2..43cca4b8 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -2001,6 +2001,7 @@ fn default_rocksdb_stats_level() -> u8 { 1 } // I know, it's a great name #[must_use] +#[inline] pub fn default_default_room_version() -> RoomVersionId { RoomVersionId::V10 } fn default_ip_range_denylist() -> Vec { diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 82bb40b0..baeb992d 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -137,6 +137,7 @@ macro_rules! err_log { let visit = &mut |vs: ValueSet<'_>| { struct Visitor<'a>(&'a mut String); impl Visit for Visitor<'_> { + #[inline] fn record_debug(&mut self, field: &Field, val: &dyn fmt::Debug) { if field.name() == "message" { write!(self.0, "{:?}", val).expect("stream error"); diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 42250a0c..302d0f87 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -4,7 +4,7 @@ mod panic; mod response; mod serde; -use std::{any::Any, borrow::Cow, convert::Infallible, fmt}; +use std::{any::Any, borrow::Cow, convert::Infallible, fmt, sync::PoisonError}; pub use self::log::*; use crate::error; @@ -59,6 +59,8 @@ pub enum Error { JsTryFromInt(#[from] ruma::JsTryFromIntError), // js_int re-export #[error(transparent)] Path(#[from] axum::extract::rejection::PathRejection), + #[error("Mutex poisoned: {0}")] + Poison(Cow<'static, str>), #[error("Regex error: {0}")] Regex(#[from] regex::Error), #[error("Request error: {0}")] @@ -184,6 +186,12 @@ impl fmt::Debug for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.message()) } } +impl From> for Error { + #[cold] + #[inline(never)] + fn from(e: PoisonError) -> Self { Self::Poison(e.to_string().into()) } +} + #[allow(clippy::fallible_impl_from)] impl From for Error { #[cold] diff --git a/src/core/error/panic.rs b/src/core/error/panic.rs index c070f786..bec25132 100644 --- a/src/core/error/panic.rs +++ b/src/core/error/panic.rs @@ -10,11 +10,14 @@ impl UnwindSafe for Error {} impl RefUnwindSafe for Error {} impl Error { + #[inline] pub fn panic(self) -> ! { panic_any(self.into_panic()) } #[must_use] + #[inline] pub fn from_panic(e: Box) -> Self { Self::Panic(debug::panic_str(&e), e) } + #[inline] pub fn into_panic(self) -> Box { match self { Self::Panic(_, e) | Self::PanicAny(e) => e, @@ -24,6 +27,7 @@ impl Error { } /// Get the panic message string. + #[inline] pub fn panic_str(self) -> Option<&'static str> { self.is_panic() .then_some(debug::panic_str(&self.into_panic())) diff --git a/src/core/error/response.rs b/src/core/error/response.rs index 7568a1c0..21fbdcf2 100644 --- a/src/core/error/response.rs +++ b/src/core/error/response.rs @@ -26,6 +26,7 @@ impl axum::response::IntoResponse for Error { } impl From for UiaaResponse { + #[inline] fn from(error: Error) -> Self { if let Error::Uiaa(uiaainfo) = error { return Self::AuthResponse(uiaainfo); From 768e81741cbd2bb16e18edb0782b64d270a86dfd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 3 Nov 2024 11:22:38 +0000 Subject: [PATCH 1004/2091] use FnMut for ready_try_for_each extension Signed-off-by: Jason Volk --- src/core/utils/stream/try_ready.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs index feb38067..3fbcbc45 100644 --- a/src/core/utils/stream/try_ready.rs +++ b/src/core/utils/stream/try_ready.rs @@ -24,7 +24,7 @@ where self, f: F, ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> where - F: Fn(S::Ok) -> Result<(), E>; + F: FnMut(S::Ok) -> Result<(), E>; } impl TryReadyExt for S @@ -42,10 +42,10 @@ where #[inline] fn ready_try_for_each( - self, f: F, + self, mut f: F, ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> where - F: Fn(S::Ok) -> Result<(), E>, + F: FnMut(S::Ok) -> Result<(), E>, { self.try_for_each(move |t| ready(f(t))) } From 4a94a4c945740b3a5ee605af61397f37060d91bc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 4 Nov 2024 18:20:32 +0000 Subject: [PATCH 1005/2091] rename pdu/id to pdu/event_id Signed-off-by: Jason Volk --- src/core/pdu/{id.rs => event_id.rs} | 0 src/core/pdu/mod.rs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename src/core/pdu/{id.rs => event_id.rs} (100%) diff --git a/src/core/pdu/id.rs b/src/core/pdu/event_id.rs similarity index 100% rename from src/core/pdu/id.rs rename to src/core/pdu/event_id.rs diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 9c3aaf9b..53fcd0a9 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -2,8 +2,8 @@ mod builder; mod content; mod count; mod event; +mod event_id; mod filter; -mod id; mod redact; mod strip; mod unsigned; @@ -20,7 +20,7 @@ pub use self::{ builder::{Builder, Builder as PduBuilder}, count::PduCount, event::Event, - id::*, + event_id::*, }; use crate::Result; From 78aeb620bc4ecd1d2feadd72043a08a037615553 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 3 Nov 2024 18:23:35 +0000 Subject: [PATCH 1006/2091] add broad timeout on acquire_origins keys operation Signed-off-by: Jason Volk --- src/service/server_keys/acquire.rs | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/service/server_keys/acquire.rs b/src/service/server_keys/acquire.rs index 25b676b8..cdaf28b4 100644 --- a/src/service/server_keys/acquire.rs +++ b/src/service/server_keys/acquire.rs @@ -1,15 +1,17 @@ use std::{ borrow::Borrow, collections::{BTreeMap, BTreeSet}, + time::Duration, }; -use conduit::{debug, debug_warn, error, implement, result::FlatOk, warn}; +use conduit::{debug, debug_error, debug_warn, error, implement, result::FlatOk, trace, warn}; use futures::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::federation::discovery::ServerSigningKeys, serde::Raw, CanonicalJsonObject, OwnedServerName, OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, }; use serde_json::value::RawValue as RawJsonValue; +use tokio::time::{timeout_at, Instant}; use super::key_exists; @@ -136,8 +138,12 @@ async fn acquire_origins(&self, batch: I) -> Batch where I: Iterator)> + Send, { + let timeout = Instant::now() + .checked_add(Duration::from_secs(45)) + .expect("timeout overflows"); + let mut requests: FuturesUnordered<_> = batch - .map(|(origin, key_ids)| self.acquire_origin(origin, key_ids)) + .map(|(origin, key_ids)| self.acquire_origin(origin, key_ids, timeout)) .collect(); let mut missing = Batch::new(); @@ -152,11 +158,22 @@ where #[implement(super::Service)] async fn acquire_origin( - &self, origin: OwnedServerName, mut key_ids: Vec, + &self, origin: OwnedServerName, mut key_ids: Vec, timeout: Instant, ) -> (OwnedServerName, Vec) { - if let Ok(server_keys) = self.server_request(&origin).await { - self.add_signing_keys(server_keys.clone()).await; - key_ids.retain(|key_id| !key_exists(&server_keys, key_id)); + match timeout_at(timeout, self.server_request(&origin)).await { + Err(e) => debug_warn!(?origin, "timed out: {e}"), + Ok(Err(e)) => debug_error!(?origin, "{e}"), + Ok(Ok(server_keys)) => { + trace!( + %origin, + ?key_ids, + ?server_keys, + "received server_keys" + ); + + self.add_signing_keys(server_keys.clone()).await; + key_ids.retain(|key_id| !key_exists(&server_keys, key_id)); + }, } (origin, key_ids) From 2e4d9cb37cf7d47a9506ee3697775ddfadcb1d56 Mon Sep 17 00:00:00 2001 From: Kirill Hmelnitski Date: Thu, 31 Oct 2024 23:39:20 +0300 Subject: [PATCH 1007/2091] fix thread pagination refactor logic increase fetch limit for first relates apply other format Co-authored-by: Jason Volk Signed-off-by: Jason Volk --- src/api/client/relations.rs | 12 +- src/service/rooms/pdu_metadata/data.rs | 39 ++--- src/service/rooms/pdu_metadata/mod.rs | 197 +++++++++++++------------ 3 files changed, 125 insertions(+), 123 deletions(-) diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index d4384730..0456924c 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -20,8 +20,8 @@ pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( &body.event_id, body.event_type.clone().into(), body.rel_type.clone().into(), - body.from.as_ref(), - body.to.as_ref(), + body.from.as_deref(), + body.to.as_deref(), body.limit, body.recurse, body.dir, @@ -51,8 +51,8 @@ pub(crate) async fn get_relating_events_with_rel_type_route( &body.event_id, None, body.rel_type.clone().into(), - body.from.as_ref(), - body.to.as_ref(), + body.from.as_deref(), + body.to.as_deref(), body.limit, body.recurse, body.dir, @@ -82,8 +82,8 @@ pub(crate) async fn get_relating_events_route( &body.event_id, None, None, - body.from.as_ref(), - body.to.as_ref(), + body.from.as_deref(), + body.to.as_deref(), body.limit, body.recurse, body.dir, diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 4d570e6d..51a43714 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -8,7 +8,7 @@ use conduit::{ }; use database::Map; use futures::{Stream, StreamExt}; -use ruma::{EventId, RoomId, UserId}; +use ruma::{api::Direction, EventId, RoomId, UserId}; use crate::{rooms, Dep}; @@ -45,9 +45,9 @@ impl Data { self.tofrom_relation.aput_raw::(key, []); } - pub(super) fn relations_until<'a>( - &'a self, user_id: &'a UserId, shortroomid: u64, target: u64, until: PduCount, - ) -> impl Stream + Send + 'a + '_ { + pub(super) fn get_relations<'a>( + &'a self, user_id: &'a UserId, shortroomid: u64, target: u64, until: PduCount, dir: Direction, + ) -> impl Stream + Send + '_ { let prefix = target.to_be_bytes().to_vec(); let mut current = prefix.clone(); let count_raw = match until { @@ -59,22 +59,23 @@ impl Data { }; current.extend_from_slice(&count_raw.to_be_bytes()); - self.tofrom_relation - .rev_raw_keys_from(¤t) - .ignore_err() - .ready_take_while(move |key| key.starts_with(&prefix)) - .map(|to_from| utils::u64_from_u8(&to_from[(size_of::())..])) - .filter_map(move |from| async move { - let mut pduid = shortroomid.to_be_bytes().to_vec(); - pduid.extend_from_slice(&from.to_be_bytes()); - let mut pdu = self.services.timeline.get_pdu_from_id(&pduid).await.ok()?; + match dir { + Direction::Forward => self.tofrom_relation.raw_keys_from(¤t).boxed(), + Direction::Backward => self.tofrom_relation.rev_raw_keys_from(¤t).boxed(), + } + .ignore_err() + .ready_take_while(move |key| key.starts_with(&prefix)) + .map(|to_from| utils::u64_from_u8(&to_from[(size_of::())..])) + .filter_map(move |from| async move { + let mut pduid = shortroomid.to_be_bytes().to_vec(); + pduid.extend_from_slice(&from.to_be_bytes()); + let mut pdu = self.services.timeline.get_pdu_from_id(&pduid).await.ok()?; + if pdu.sender != user_id { + pdu.remove_transaction_id().log_err().ok(); + } - if pdu.sender != user_id { - pdu.remove_transaction_id().log_err().ok(); - } - - Some((PduCount::Normal(from), pdu)) - }) + Some((PduCount::Normal(from), pdu)) + }) } pub(super) fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) { diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index fb85d031..b1cf2049 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -1,12 +1,16 @@ mod data; use std::sync::Arc; -use conduit::{utils::stream::IterStream, PduCount, Result}; -use futures::StreamExt; +use conduit::{ + at, + utils::{result::FlatOk, stream::ReadyExt, IterStream}, + PduCount, Result, +}; +use futures::{FutureExt, StreamExt}; use ruma::{ api::{client::relations::get_relating_events, Direction}, events::{relation::RelationType, TimelineEventType}, - uint, EventId, RoomId, UInt, UserId, + EventId, RoomId, UInt, UserId, }; use serde::Deserialize; @@ -63,24 +67,24 @@ impl Service { #[allow(clippy::too_many_arguments)] pub async fn paginate_relations_with_filter( &self, sender_user: &UserId, room_id: &RoomId, target: &EventId, filter_event_type: Option, - filter_rel_type: Option, from: Option<&String>, to: Option<&String>, limit: Option, + filter_rel_type: Option, from: Option<&str>, to: Option<&str>, limit: Option, recurse: bool, dir: Direction, ) -> Result { - let from = match from { - Some(from) => PduCount::try_from_string(from)?, - None => match dir { + let from = from + .map(PduCount::try_from_string) + .transpose()? + .unwrap_or_else(|| match dir { Direction::Forward => PduCount::min(), Direction::Backward => PduCount::max(), - }, - }; + }); - let to = to.and_then(|t| PduCount::try_from_string(t).ok()); + let to = to.map(PduCount::try_from_string).flat_ok(); - // Use limit or else 10, with maximum 100 - let limit = limit - .unwrap_or_else(|| uint!(10)) - .try_into() - .unwrap_or(10) + // Use limit or else 30, with maximum 100 + let limit: usize = limit + .map(TryInto::try_into) + .flat_ok() + .unwrap_or(30) .min(100); // Spec (v1.10) recommends depth of at least 3 @@ -90,53 +94,96 @@ impl Service { 1 }; - let relations_until: Vec = self - .relations_until(sender_user, room_id, target, from, depth) - .await?; - - // TODO: should be relations_after - let events: Vec<_> = relations_until + let events: Vec = self + .get_relations(sender_user, room_id, target, from, limit, depth, dir) + .await .into_iter() - .filter(move |(_, pdu): &PdusIterItem| { - if !filter_event_type.as_ref().map_or(true, |t| pdu.kind == *t) { - return false; - } - - let Ok(content) = pdu.get_content::() else { - return false; - }; - - filter_rel_type + .filter(|(_, pdu)| { + filter_event_type .as_ref() - .map_or(true, |r| *r == content.relates_to.rel_type) + .is_none_or(|kind| *kind == pdu.kind) + }) + .filter(|(_, pdu)| { + filter_rel_type.as_ref().is_none_or(|rel_type| { + pdu.get_content() + .map(|c: ExtractRelatesToEventId| c.relates_to.rel_type) + .is_ok_and(|r| r == *rel_type) + }) }) - .take(limit) - .take_while(|(k, _)| Some(*k) != to) .stream() .filter_map(|item| self.visibility_filter(sender_user, item)) + .ready_take_while(|(count, _)| Some(*count) != to) + .take(limit) + .collect() + .boxed() + .await; + + let next_batch = match dir { + Direction::Backward => events.first(), + Direction::Forward => events.last(), + } + .map(at!(0)) + .map(|t| t.stringify()); + + Ok(get_relating_events::v1::Response { + next_batch, + prev_batch: Some(from.stringify()), + recursion_depth: recurse.then_some(depth.into()), + chunk: events + .into_iter() + .map(at!(1)) + .map(|pdu| pdu.to_message_like_event()) + .collect(), + }) + } + + #[allow(clippy::too_many_arguments)] + pub async fn get_relations( + &self, user_id: &UserId, room_id: &RoomId, target: &EventId, until: PduCount, limit: usize, max_depth: u8, + dir: Direction, + ) -> Vec { + let room_id = self.services.short.get_or_create_shortroomid(room_id).await; + + let target = match self.services.timeline.get_pdu_count(target).await { + Ok(PduCount::Normal(c)) => c, + // TODO: Support backfilled relations + _ => 0, // This will result in an empty iterator + }; + + let mut pdus: Vec<_> = self + .db + .get_relations(user_id, room_id, target, until, dir) .collect() .await; - let next_token = events.last().map(|(count, _)| count).copied(); + let mut stack: Vec<_> = pdus.iter().map(|pdu| (pdu.clone(), 1)).collect(); - let events_chunk: Vec<_> = match dir { - Direction::Forward => events - .into_iter() - .map(|(_, pdu)| pdu.to_message_like_event()) - .collect(), - Direction::Backward => events - .into_iter() - .rev() // relations are always most recent first - .map(|(_, pdu)| pdu.to_message_like_event()) - .collect(), - }; + 'limit: while let Some(stack_pdu) = stack.pop() { + let target = match stack_pdu.0 .0 { + PduCount::Normal(c) => c, + // TODO: Support backfilled relations + PduCount::Backfilled(_) => 0, // This will result in an empty iterator + }; - Ok(get_relating_events::v1::Response { - chunk: events_chunk, - next_batch: next_token.map(|t| t.stringify()), - prev_batch: Some(from.stringify()), - recursion_depth: recurse.then_some(depth.into()), - }) + let relations: Vec<_> = self + .db + .get_relations(user_id, room_id, target, until, dir) + .collect() + .await; + + for relation in relations { + if stack_pdu.1 < max_depth { + stack.push((relation.clone(), stack_pdu.1.saturating_add(1))); + } + + pdus.push(relation); + if pdus.len() >= limit { + break 'limit; + } + } + } + + pdus } async fn visibility_filter(&self, sender_user: &UserId, item: PdusIterItem) -> Option { @@ -149,52 +196,6 @@ impl Service { .then_some(item) } - pub async fn relations_until( - &self, user_id: &UserId, room_id: &RoomId, target: &EventId, until: PduCount, max_depth: u8, - ) -> Result> { - let room_id = self.services.short.get_or_create_shortroomid(room_id).await; - - let target = match self.services.timeline.get_pdu_count(target).await { - Ok(PduCount::Normal(c)) => c, - // TODO: Support backfilled relations - _ => 0, // This will result in an empty iterator - }; - - let mut pdus: Vec = self - .db - .relations_until(user_id, room_id, target, until) - .collect() - .await; - - let mut stack: Vec<_> = pdus.clone().into_iter().map(|pdu| (pdu, 1)).collect(); - - while let Some(stack_pdu) = stack.pop() { - let target = match stack_pdu.0 .0 { - PduCount::Normal(c) => c, - // TODO: Support backfilled relations - PduCount::Backfilled(_) => 0, // This will result in an empty iterator - }; - - let relations: Vec = self - .db - .relations_until(user_id, room_id, target, until) - .collect() - .await; - - for relation in relations { - if stack_pdu.1 < max_depth { - stack.push((relation.clone(), stack_pdu.1.saturating_add(1))); - } - - pdus.push(relation); - } - } - - pdus.sort_by(|a, b| a.0.cmp(&b.0)); - - Ok(pdus) - } - #[inline] #[tracing::instrument(skip_all, level = "debug")] pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) { From 9da523c004aba6e9d1d51c73de0524d5f6433bbd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 2 Nov 2024 06:12:54 +0000 Subject: [PATCH 1008/2091] refactor for stronger RawPduId type implement standard traits for PduCount enable serde for arrayvec typedef various shortid's pducount simplifications split parts of pdu_metadata service to core/pdu and api/relations remove some yields; improve var names/syntax tweak types for limit timeline limit arguments Signed-off-by: Jason Volk --- Cargo.lock | 3 + Cargo.toml | 1 + src/api/client/context.rs | 4 +- src/api/client/membership.rs | 7 +- src/api/client/message.rs | 20 +-- src/api/client/relations.rs | 199 +++++++++++++++------- src/api/client/sync/mod.rs | 11 +- src/api/client/sync/v3.rs | 65 +++---- src/api/client/sync/v4.rs | 24 ++- src/api/client/threads.rs | 40 ++--- src/api/server/send_join.rs | 4 +- src/api/server/send_leave.rs | 6 +- src/core/mod.rs | 2 +- src/core/pdu/count.rs | 142 ++++++++++++--- src/core/pdu/id.rs | 22 +++ src/core/pdu/mod.rs | 6 + src/core/pdu/raw_id.rs | 117 +++++++++++++ src/core/pdu/relation.rs | 22 +++ src/core/pdu/tests.rs | 19 +++ src/service/migrations.rs | 2 +- src/service/rooms/auth_chain/data.rs | 8 +- src/service/rooms/auth_chain/mod.rs | 18 +- src/service/rooms/event_handler/mod.rs | 13 +- src/service/rooms/pdu_metadata/data.rs | 49 +++--- src/service/rooms/pdu_metadata/mod.rs | 116 +------------ src/service/rooms/search/mod.rs | 36 ++-- src/service/rooms/short/mod.rs | 8 +- src/service/rooms/spaces/mod.rs | 6 +- src/service/rooms/state/mod.rs | 4 +- src/service/rooms/state_accessor/data.rs | 6 +- src/service/rooms/state_accessor/mod.rs | 30 ++-- src/service/rooms/state_compressor/mod.rs | 42 +++-- src/service/rooms/threads/data.rs | 52 +++--- src/service/rooms/threads/mod.rs | 6 +- src/service/rooms/timeline/data.rs | 124 ++++++-------- src/service/rooms/timeline/mod.rs | 75 ++++---- src/service/rooms/timeline/pduid.rs | 13 -- src/service/rooms/user/mod.rs | 6 +- src/service/sending/data.rs | 12 +- src/service/sending/mod.rs | 23 +-- src/service/sending/sender.rs | 6 +- 41 files changed, 796 insertions(+), 573 deletions(-) create mode 100644 src/core/pdu/id.rs create mode 100644 src/core/pdu/raw_id.rs create mode 100644 src/core/pdu/relation.rs create mode 100644 src/core/pdu/tests.rs delete mode 100644 src/service/rooms/timeline/pduid.rs diff --git a/Cargo.lock b/Cargo.lock index 44856753..f729d3d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,9 @@ name = "arrayvec" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +dependencies = [ + "serde", +] [[package]] name = "as_variant" diff --git a/Cargo.toml b/Cargo.toml index 043790f8..3ac1556c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,6 +27,7 @@ name = "conduit" [workspace.dependencies.arrayvec] version = "0.7.4" +features = ["std", "serde"] [workspace.dependencies.const-str] version = "0.5.7" diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 9bf0c467..5b492cb1 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -168,12 +168,12 @@ pub(crate) async fn get_context_route( start: events_before .last() - .map_or_else(|| base_token.stringify(), |(count, _)| count.stringify()) + .map_or_else(|| base_token.to_string(), |(count, _)| count.to_string()) .into(), end: events_after .last() - .map_or_else(|| base_token.stringify(), |(count, _)| count.stringify()) + .map_or_else(|| base_token.to_string(), |(count, _)| count.to_string()) .into(), events_before: events_before diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index c41e93fa..fa71c0c8 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1376,15 +1376,12 @@ pub(crate) async fn invite_helper( ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Origin field is invalid."))?; - let pdu_id: Vec = services + let pdu_id = services .rooms .event_handler .handle_incoming_pdu(&origin, room_id, &event_id, value, true) .await? - .ok_or(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not accept incoming PDU as timeline event.", - ))?; + .ok_or_else(|| err!(Request(InvalidParam("Could not accept incoming PDU as timeline event."))))?; services.sending.send_pdu_room(room_id, &pdu_id).await?; return Ok(()); diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 4fc58d9f..cb261a7f 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -62,19 +62,17 @@ pub(crate) async fn get_message_events_route( let room_id = &body.room_id; let filter = &body.filter; - let from_default = match body.dir { - Direction::Forward => PduCount::min(), - Direction::Backward => PduCount::max(), - }; - - let from = body + let from: PduCount = body .from .as_deref() - .map(PduCount::try_from_string) + .map(str::parse) .transpose()? - .unwrap_or(from_default); + .unwrap_or_else(|| match body.dir { + Direction::Forward => PduCount::min(), + Direction::Backward => PduCount::max(), + }); - let to = body.to.as_deref().map(PduCount::try_from_string).flat_ok(); + let to: Option = body.to.as_deref().map(str::parse).flat_ok(); let limit: usize = body .limit @@ -156,8 +154,8 @@ pub(crate) async fn get_message_events_route( .collect(); Ok(get_message_events::v3::Response { - start: from.stringify(), - end: next_token.as_ref().map(PduCount::stringify), + start: from.to_string(), + end: next_token.as_ref().map(PduCount::to_string), chunk, state, }) diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index 0456924c..ef7035e2 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,34 +1,43 @@ use axum::extract::State; -use ruma::api::client::relations::{ - get_relating_events, get_relating_events_with_rel_type, get_relating_events_with_rel_type_and_event_type, +use conduit::{ + at, + utils::{result::FlatOk, IterStream, ReadyExt}, + PduCount, Result, }; +use futures::{FutureExt, StreamExt}; +use ruma::{ + api::{ + client::relations::{ + get_relating_events, get_relating_events_with_rel_type, get_relating_events_with_rel_type_and_event_type, + }, + Direction, + }, + events::{relation::RelationType, TimelineEventType}, + EventId, RoomId, UInt, UserId, +}; +use service::{rooms::timeline::PdusIterItem, Services}; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}` pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_deref().expect("user is authenticated"); - - let res = services - .rooms - .pdu_metadata - .paginate_relations_with_filter( - sender_user, - &body.room_id, - &body.event_id, - body.event_type.clone().into(), - body.rel_type.clone().into(), - body.from.as_deref(), - body.to.as_deref(), - body.limit, - body.recurse, - body.dir, - ) - .await?; - - Ok(get_relating_events_with_rel_type_and_event_type::v1::Response { + paginate_relations_with_filter( + &services, + body.sender_user(), + &body.room_id, + &body.event_id, + body.event_type.clone().into(), + body.rel_type.clone().into(), + body.from.as_deref(), + body.to.as_deref(), + body.limit, + body.recurse, + body.dir, + ) + .await + .map(|res| get_relating_events_with_rel_type_and_event_type::v1::Response { chunk: res.chunk, next_batch: res.next_batch, prev_batch: res.prev_batch, @@ -40,26 +49,21 @@ pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( pub(crate) async fn get_relating_events_with_rel_type_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_deref().expect("user is authenticated"); - - let res = services - .rooms - .pdu_metadata - .paginate_relations_with_filter( - sender_user, - &body.room_id, - &body.event_id, - None, - body.rel_type.clone().into(), - body.from.as_deref(), - body.to.as_deref(), - body.limit, - body.recurse, - body.dir, - ) - .await?; - - Ok(get_relating_events_with_rel_type::v1::Response { + paginate_relations_with_filter( + &services, + body.sender_user(), + &body.room_id, + &body.event_id, + None, + body.rel_type.clone().into(), + body.from.as_deref(), + body.to.as_deref(), + body.limit, + body.recurse, + body.dir, + ) + .await + .map(|res| get_relating_events_with_rel_type::v1::Response { chunk: res.chunk, next_batch: res.next_batch, prev_batch: res.prev_batch, @@ -71,22 +75,103 @@ pub(crate) async fn get_relating_events_with_rel_type_route( pub(crate) async fn get_relating_events_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_deref().expect("user is authenticated"); + paginate_relations_with_filter( + &services, + body.sender_user(), + &body.room_id, + &body.event_id, + None, + None, + body.from.as_deref(), + body.to.as_deref(), + body.limit, + body.recurse, + body.dir, + ) + .await +} + +#[allow(clippy::too_many_arguments)] +async fn paginate_relations_with_filter( + services: &Services, sender_user: &UserId, room_id: &RoomId, target: &EventId, + filter_event_type: Option, filter_rel_type: Option, from: Option<&str>, + to: Option<&str>, limit: Option, recurse: bool, dir: Direction, +) -> Result { + let from: PduCount = from + .map(str::parse) + .transpose()? + .unwrap_or_else(|| match dir { + Direction::Forward => PduCount::min(), + Direction::Backward => PduCount::max(), + }); + + let to: Option = to.map(str::parse).flat_ok(); + + // Use limit or else 30, with maximum 100 + let limit: usize = limit + .map(TryInto::try_into) + .flat_ok() + .unwrap_or(30) + .min(100); + + // Spec (v1.10) recommends depth of at least 3 + let depth: u8 = if recurse { + 3 + } else { + 1 + }; + + let events: Vec = services + .rooms + .pdu_metadata + .get_relations(sender_user, room_id, target, from, limit, depth, dir) + .await + .into_iter() + .filter(|(_, pdu)| { + filter_event_type + .as_ref() + .is_none_or(|kind| *kind == pdu.kind) + }) + .filter(|(_, pdu)| { + filter_rel_type + .as_ref() + .is_none_or(|rel_type| pdu.relation_type_equal(rel_type)) + }) + .stream() + .filter_map(|item| visibility_filter(services, sender_user, item)) + .ready_take_while(|(count, _)| Some(*count) != to) + .take(limit) + .collect() + .boxed() + .await; + + let next_batch = match dir { + Direction::Backward => events.first(), + Direction::Forward => events.last(), + } + .map(at!(0)) + .as_ref() + .map(ToString::to_string); + + Ok(get_relating_events::v1::Response { + next_batch, + prev_batch: Some(from.to_string()), + recursion_depth: recurse.then_some(depth.into()), + chunk: events + .into_iter() + .map(at!(1)) + .map(|pdu| pdu.to_message_like_event()) + .collect(), + }) +} + +async fn visibility_filter(services: &Services, sender_user: &UserId, item: PdusIterItem) -> Option { + let (_, pdu) = &item; services .rooms - .pdu_metadata - .paginate_relations_with_filter( - sender_user, - &body.room_id, - &body.event_id, - None, - None, - body.from.as_deref(), - body.to.as_deref(), - body.limit, - body.recurse, - body.dir, - ) + .state_accessor + .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) .await + .then_some(item) } diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index ed22010c..7aec7186 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -1,10 +1,7 @@ mod v3; mod v4; -use conduit::{ - utils::{math::usize_from_u64_truncated, ReadyExt}, - PduCount, -}; +use conduit::{utils::ReadyExt, PduCount}; use futures::StreamExt; use ruma::{RoomId, UserId}; @@ -12,7 +9,7 @@ pub(crate) use self::{v3::sync_events_route, v4::sync_events_v4_route}; use crate::{service::Services, Error, PduEvent, Result}; async fn load_timeline( - services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: u64, + services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: usize, ) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> { let last_timeline_count = services .rooms @@ -29,12 +26,12 @@ async fn load_timeline( .timeline .pdus_until(sender_user, room_id, PduCount::max()) .await? - .ready_take_while(|(pducount, _)| pducount > &roomsincecount); + .ready_take_while(|(pducount, _)| *pducount > roomsincecount); // Take the last events for the timeline let timeline_pdus: Vec<_> = non_timeline_pdus .by_ref() - .take(usize_from_u64_truncated(limit)) + .take(limit) .collect::>() .await .into_iter() diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index ccca1f85..08048902 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -432,28 +432,26 @@ async fn handle_left_room( left_state_ids.insert(leave_shortstatekey, left_event_id); - let mut i: u8 = 0; - for (key, id) in left_state_ids { - if full_state || since_state_ids.get(&key) != Some(&id) { - let (event_type, state_key) = services.rooms.short.get_statekey_from_short(key).await?; + for (shortstatekey, event_id) in left_state_ids { + if full_state || since_state_ids.get(&shortstatekey) != Some(&event_id) { + let (event_type, state_key) = services + .rooms + .short + .get_statekey_from_short(shortstatekey) + .await?; + // TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565 if !lazy_load_enabled - || event_type != StateEventType::RoomMember - || full_state - // TODO: Delete the following line when this is resolved: https://github.com/vector-im/element-web/issues/22565 - || (cfg!(feature = "element_hacks") && *sender_user == state_key) + || event_type != StateEventType::RoomMember + || full_state + || (cfg!(feature = "element_hacks") && *sender_user == state_key) { - let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { - error!("Pdu in state not found: {}", id); + let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { + error!("Pdu in state not found: {event_id}"); continue; }; left_state_events.push(pdu.to_sync_state_event()); - - i = i.wrapping_add(1); - if i % 100 == 0 { - tokio::task::yield_now().await; - } } } } @@ -542,7 +540,7 @@ async fn load_joined_room( let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); - let (timeline_pdus, limited) = load_timeline(services, sender_user, room_id, sincecount, 10).await?; + let (timeline_pdus, limited) = load_timeline(services, sender_user, room_id, sincecount, 10_usize).await?; let send_notification_counts = !timeline_pdus.is_empty() || services @@ -678,8 +676,7 @@ async fn load_joined_room( let mut state_events = Vec::new(); let mut lazy_loaded = HashSet::new(); - let mut i: u8 = 0; - for (shortstatekey, id) in current_state_ids { + for (shortstatekey, event_id) in current_state_ids { let (event_type, state_key) = services .rooms .short @@ -687,24 +684,22 @@ async fn load_joined_room( .await?; if event_type != StateEventType::RoomMember { - let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { - error!("Pdu in state not found: {id}"); + let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { + error!("Pdu in state not found: {event_id}"); continue; }; - state_events.push(pdu); - i = i.wrapping_add(1); - if i % 100 == 0 { - tokio::task::yield_now().await; - } - } else if !lazy_load_enabled - || full_state - || timeline_users.contains(&state_key) - // TODO: Delete the following line when this is resolved: https://github.com/vector-im/element-web/issues/22565 - || (cfg!(feature = "element_hacks") && *sender_user == state_key) + state_events.push(pdu); + continue; + } + + // TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565 + if !lazy_load_enabled + || full_state || timeline_users.contains(&state_key) + || (cfg!(feature = "element_hacks") && *sender_user == state_key) { - let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { - error!("Pdu in state not found: {id}"); + let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { + error!("Pdu in state not found: {event_id}"); continue; }; @@ -712,12 +707,8 @@ async fn load_joined_room( if let Ok(uid) = UserId::parse(&state_key) { lazy_loaded.insert(uid); } - state_events.push(pdu); - i = i.wrapping_add(1); - if i % 100 == 0 { - tokio::task::yield_now().await; - } + state_events.push(pdu); } } diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index f8ada81c..11e3830c 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -8,7 +8,7 @@ use axum::extract::State; use conduit::{ debug, error, extract_variant, utils::{ - math::{ruma_from_usize, usize_from_ruma}, + math::{ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, BoolExt, IterStream, ReadyExt, TryFutureExtExt, }, warn, Error, PduCount, Result, @@ -350,14 +350,16 @@ pub(crate) async fn sync_events_v4_route( new_known_rooms.extend(room_ids.iter().cloned()); for room_id in &room_ids { - let todo_room = todo_rooms - .entry(room_id.clone()) - .or_insert((BTreeSet::new(), 0, u64::MAX)); + let todo_room = + todo_rooms + .entry(room_id.clone()) + .or_insert((BTreeSet::new(), 0_usize, u64::MAX)); - let limit = list + let limit: usize = list .room_details .timeline_limit - .map_or(10, u64::from) + .map(u64::from) + .map_or(10, usize_from_u64_truncated) .min(100); todo_room @@ -406,8 +408,14 @@ pub(crate) async fn sync_events_v4_route( } let todo_room = todo_rooms .entry(room_id.clone()) - .or_insert((BTreeSet::new(), 0, u64::MAX)); - let limit = room.timeline_limit.map_or(10, u64::from).min(100); + .or_insert((BTreeSet::new(), 0_usize, u64::MAX)); + + let limit: usize = room + .timeline_limit + .map(u64::from) + .map_or(10, usize_from_u64_truncated) + .min(100); + todo_room.0.extend(room.required_state.iter().cloned()); todo_room.1 = todo_room.1.max(limit); // 0 means unknown because it got out of date diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index 50f6cdfb..02cf7992 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -1,19 +1,14 @@ use axum::extract::State; -use conduit::PduEvent; +use conduit::{PduCount, PduEvent}; use futures::StreamExt; -use ruma::{ - api::client::{error::ErrorKind, threads::get_threads}, - uint, -}; +use ruma::{api::client::threads::get_threads, uint}; -use crate::{Error, Result, Ruma}; +use crate::{Result, Ruma}; /// # `GET /_matrix/client/r0/rooms/{roomId}/threads` pub(crate) async fn get_threads_route( - State(services): State, body: Ruma, + State(services): State, ref body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - // Use limit or else 10, with maximum 100 let limit = body .limit @@ -22,38 +17,39 @@ pub(crate) async fn get_threads_route( .unwrap_or(10) .min(100); - let from = if let Some(from) = &body.from { - from.parse() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, ""))? - } else { - u64::MAX - }; + let from: PduCount = body + .from + .as_deref() + .map(str::parse) + .transpose()? + .unwrap_or_else(PduCount::max); - let room_id = &body.room_id; - let threads: Vec<(u64, PduEvent)> = services + let threads: Vec<(PduCount, PduEvent)> = services .rooms .threads - .threads_until(sender_user, &body.room_id, from, &body.include) + .threads_until(body.sender_user(), &body.room_id, from, &body.include) .await? .take(limit) .filter_map(|(count, pdu)| async move { services .rooms .state_accessor - .user_can_see_event(sender_user, room_id, &pdu.event_id) + .user_can_see_event(body.sender_user(), &body.room_id, &pdu.event_id) .await .then_some((count, pdu)) }) .collect() .await; - let next_batch = threads.last().map(|(count, _)| count.to_string()); - Ok(get_threads::v1::Response { + next_batch: threads + .last() + .map(|(count, _)| count) + .map(ToString::to_string), + chunk: threads .into_iter() .map(|(_, pdu)| pdu.to_room_event()) .collect(), - next_batch, }) } diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index c3273baf..f2ede9d0 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -156,12 +156,12 @@ async fn create_join_event( .lock(room_id) .await; - let pdu_id: Vec = services + let pdu_id = services .rooms .event_handler .handle_incoming_pdu(&origin, room_id, &event_id, value.clone(), true) .await? - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; + .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; drop(mutex_lock); diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index 7b4a8aee..448e5de3 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -1,7 +1,7 @@ #![allow(deprecated)] use axum::extract::State; -use conduit::{utils::ReadyExt, Error, Result}; +use conduit::{err, utils::ReadyExt, Error, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_leave_event}, events::{ @@ -142,12 +142,12 @@ async fn create_leave_event( .lock(room_id) .await; - let pdu_id: Vec = services + let pdu_id = services .rooms .event_handler .handle_incoming_pdu(origin, room_id, &event_id, value, true) .await? - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Could not accept as timeline event."))?; + .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; drop(mutex_lock); diff --git a/src/core/mod.rs b/src/core/mod.rs index 1b7b8fa1..4ab84730 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -17,7 +17,7 @@ pub use ::tracing; pub use config::Config; pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; -pub use pdu::{Event, PduBuilder, PduCount, PduEvent}; +pub use pdu::{Event, PduBuilder, PduCount, PduEvent, PduId, RawPduId}; pub use server::Server; pub use utils::{ctor, dtor, implement, result, result::Result}; diff --git a/src/core/pdu/count.rs b/src/core/pdu/count.rs index 094988b6..90e552e8 100644 --- a/src/core/pdu/count.rs +++ b/src/core/pdu/count.rs @@ -1,38 +1,135 @@ -use std::cmp::Ordering; +#![allow(clippy::cast_possible_wrap, clippy::cast_sign_loss, clippy::as_conversions)] -use ruma::api::client::error::ErrorKind; +use std::{cmp::Ordering, fmt, fmt::Display, str::FromStr}; -use crate::{Error, Result}; +use crate::{err, Error, Result}; #[derive(Hash, PartialEq, Eq, Clone, Copy, Debug)] pub enum PduCount { - Backfilled(u64), Normal(u64), + Backfilled(i64), } impl PduCount { + #[inline] #[must_use] - pub fn min() -> Self { Self::Backfilled(u64::MAX) } + pub fn from_unsigned(unsigned: u64) -> Self { Self::from_signed(unsigned as i64) } + #[inline] #[must_use] - pub fn max() -> Self { Self::Normal(u64::MAX) } - - pub fn try_from_string(token: &str) -> Result { - if let Some(stripped_token) = token.strip_prefix('-') { - stripped_token.parse().map(PduCount::Backfilled) - } else { - token.parse().map(PduCount::Normal) + pub fn from_signed(signed: i64) -> Self { + match signed { + i64::MIN..=0 => Self::Backfilled(signed), + _ => Self::Normal(signed as u64), } - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid pagination token.")) } + #[inline] #[must_use] - pub fn stringify(&self) -> String { + pub fn into_unsigned(self) -> u64 { + self.debug_assert_valid(); match self { - Self::Backfilled(x) => format!("-{x}"), - Self::Normal(x) => x.to_string(), + Self::Normal(i) => i, + Self::Backfilled(i) => i as u64, } } + + #[inline] + #[must_use] + pub fn into_signed(self) -> i64 { + self.debug_assert_valid(); + match self { + Self::Normal(i) => i as i64, + Self::Backfilled(i) => i, + } + } + + #[inline] + #[must_use] + pub fn into_normal(self) -> Self { + self.debug_assert_valid(); + match self { + Self::Normal(i) => Self::Normal(i), + Self::Backfilled(_) => Self::Normal(0), + } + } + + #[inline] + pub fn checked_add(self, add: u64) -> Result { + Ok(match self { + Self::Normal(i) => Self::Normal( + i.checked_add(add) + .ok_or_else(|| err!(Arithmetic("PduCount::Normal overflow")))?, + ), + Self::Backfilled(i) => Self::Backfilled( + i.checked_add(add as i64) + .ok_or_else(|| err!(Arithmetic("PduCount::Backfilled overflow")))?, + ), + }) + } + + #[inline] + pub fn checked_sub(self, sub: u64) -> Result { + Ok(match self { + Self::Normal(i) => Self::Normal( + i.checked_sub(sub) + .ok_or_else(|| err!(Arithmetic("PduCount::Normal underflow")))?, + ), + Self::Backfilled(i) => Self::Backfilled( + i.checked_sub(sub as i64) + .ok_or_else(|| err!(Arithmetic("PduCount::Backfilled underflow")))?, + ), + }) + } + + #[inline] + #[must_use] + pub fn saturating_add(self, add: u64) -> Self { + match self { + Self::Normal(i) => Self::Normal(i.saturating_add(add)), + Self::Backfilled(i) => Self::Backfilled(i.saturating_add(add as i64)), + } + } + + #[inline] + #[must_use] + pub fn saturating_sub(self, sub: u64) -> Self { + match self { + Self::Normal(i) => Self::Normal(i.saturating_sub(sub)), + Self::Backfilled(i) => Self::Backfilled(i.saturating_sub(sub as i64)), + } + } + + #[inline] + #[must_use] + pub fn min() -> Self { Self::Backfilled(i64::MIN) } + + #[inline] + #[must_use] + pub fn max() -> Self { Self::Normal(i64::MAX as u64) } + + #[inline] + pub(crate) fn debug_assert_valid(&self) { + if let Self::Backfilled(i) = self { + debug_assert!(*i <= 0, "Backfilled sequence must be negative"); + } + } +} + +impl Display for PduCount { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + self.debug_assert_valid(); + match self { + Self::Normal(i) => write!(f, "{i}"), + Self::Backfilled(i) => write!(f, "{i}"), + } + } +} + +impl FromStr for PduCount { + type Err = Error; + + fn from_str(token: &str) -> Result { Ok(Self::from_signed(token.parse()?)) } } impl PartialOrd for PduCount { @@ -40,12 +137,9 @@ impl PartialOrd for PduCount { } impl Ord for PduCount { - fn cmp(&self, other: &Self) -> Ordering { - match (self, other) { - (Self::Normal(s), Self::Normal(o)) => s.cmp(o), - (Self::Backfilled(s), Self::Backfilled(o)) => o.cmp(s), - (Self::Normal(_), Self::Backfilled(_)) => Ordering::Greater, - (Self::Backfilled(_), Self::Normal(_)) => Ordering::Less, - } - } + fn cmp(&self, other: &Self) -> Ordering { self.into_signed().cmp(&other.into_signed()) } +} + +impl Default for PduCount { + fn default() -> Self { Self::Normal(0) } } diff --git a/src/core/pdu/id.rs b/src/core/pdu/id.rs new file mode 100644 index 00000000..05d11904 --- /dev/null +++ b/src/core/pdu/id.rs @@ -0,0 +1,22 @@ +use super::{PduCount, RawPduId}; +use crate::utils::u64_from_u8x8; + +pub type ShortRoomId = ShortId; +pub type ShortEventId = ShortId; +pub type ShortId = u64; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct PduId { + pub shortroomid: ShortRoomId, + pub shorteventid: PduCount, +} + +impl From for PduId { + #[inline] + fn from(raw: RawPduId) -> Self { + Self { + shortroomid: u64_from_u8x8(raw.shortroomid()), + shorteventid: PduCount::from_unsigned(u64_from_u8x8(raw.shorteventid())), + } + } +} diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 53fcd0a9..c785c99e 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -4,8 +4,12 @@ mod count; mod event; mod event_id; mod filter; +mod id; +mod raw_id; mod redact; +mod relation; mod strip; +mod tests; mod unsigned; use std::{cmp::Ordering, sync::Arc}; @@ -21,6 +25,8 @@ pub use self::{ count::PduCount, event::Event, event_id::*, + id::*, + raw_id::*, }; use crate::Result; diff --git a/src/core/pdu/raw_id.rs b/src/core/pdu/raw_id.rs new file mode 100644 index 00000000..faba1cbf --- /dev/null +++ b/src/core/pdu/raw_id.rs @@ -0,0 +1,117 @@ +use arrayvec::ArrayVec; + +use super::{PduCount, PduId, ShortEventId, ShortId, ShortRoomId}; + +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum RawPduId { + Normal(RawPduIdNormal), + Backfilled(RawPduIdBackfilled), +} + +type RawPduIdNormal = [u8; RawPduId::NORMAL_LEN]; +type RawPduIdBackfilled = [u8; RawPduId::BACKFILLED_LEN]; + +const INT_LEN: usize = size_of::(); + +impl RawPduId { + const BACKFILLED_LEN: usize = size_of::() + INT_LEN + size_of::(); + const MAX_LEN: usize = Self::BACKFILLED_LEN; + const NORMAL_LEN: usize = size_of::() + size_of::(); + + #[inline] + #[must_use] + pub fn pdu_count(&self) -> PduCount { + let id: PduId = (*self).into(); + id.shorteventid + } + + #[inline] + #[must_use] + pub fn shortroomid(self) -> [u8; INT_LEN] { + match self { + Self::Normal(raw) => raw[0..INT_LEN] + .try_into() + .expect("normal raw shortroomid array from slice"), + Self::Backfilled(raw) => raw[0..INT_LEN] + .try_into() + .expect("backfilled raw shortroomid array from slice"), + } + } + + #[inline] + #[must_use] + pub fn shorteventid(self) -> [u8; INT_LEN] { + match self { + Self::Normal(raw) => raw[INT_LEN..INT_LEN * 2] + .try_into() + .expect("normal raw shorteventid array from slice"), + Self::Backfilled(raw) => raw[INT_LEN * 2..INT_LEN * 3] + .try_into() + .expect("backfilled raw shorteventid array from slice"), + } + } + + #[inline] + #[must_use] + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::Normal(ref raw) => raw, + Self::Backfilled(ref raw) => raw, + } + } +} + +impl AsRef<[u8]> for RawPduId { + #[inline] + fn as_ref(&self) -> &[u8] { self.as_bytes() } +} + +impl From<&[u8]> for RawPduId { + #[inline] + fn from(id: &[u8]) -> Self { + match id.len() { + Self::NORMAL_LEN => Self::Normal( + id[0..Self::NORMAL_LEN] + .try_into() + .expect("normal RawPduId from [u8]"), + ), + Self::BACKFILLED_LEN => Self::Backfilled( + id[0..Self::BACKFILLED_LEN] + .try_into() + .expect("backfilled RawPduId from [u8]"), + ), + _ => unimplemented!("unrecognized RawPduId length"), + } + } +} + +impl From for RawPduId { + #[inline] + fn from(id: PduId) -> Self { + const MAX_LEN: usize = RawPduId::MAX_LEN; + type RawVec = ArrayVec; + + let mut vec = RawVec::new(); + vec.extend(id.shortroomid.to_be_bytes()); + id.shorteventid.debug_assert_valid(); + match id.shorteventid { + PduCount::Normal(shorteventid) => { + vec.extend(shorteventid.to_be_bytes()); + Self::Normal( + vec.as_ref() + .try_into() + .expect("RawVec into RawPduId::Normal"), + ) + }, + PduCount::Backfilled(shorteventid) => { + vec.extend(0_u64.to_be_bytes()); + vec.extend(shorteventid.to_be_bytes()); + Self::Backfilled( + vec.as_ref() + .try_into() + .expect("RawVec into RawPduId::Backfilled"), + ) + }, + } + } +} diff --git a/src/core/pdu/relation.rs b/src/core/pdu/relation.rs new file mode 100644 index 00000000..ae156a3d --- /dev/null +++ b/src/core/pdu/relation.rs @@ -0,0 +1,22 @@ +use ruma::events::relation::RelationType; +use serde::Deserialize; + +use crate::implement; + +#[derive(Clone, Debug, Deserialize)] +struct ExtractRelType { + rel_type: RelationType, +} +#[derive(Clone, Debug, Deserialize)] +struct ExtractRelatesToEventId { + #[serde(rename = "m.relates_to")] + relates_to: ExtractRelType, +} + +#[implement(super::PduEvent)] +#[must_use] +pub fn relation_type_equal(&self, rel_type: &RelationType) -> bool { + self.get_content() + .map(|c: ExtractRelatesToEventId| c.relates_to.rel_type) + .is_ok_and(|r| r == *rel_type) +} diff --git a/src/core/pdu/tests.rs b/src/core/pdu/tests.rs new file mode 100644 index 00000000..30ec23ba --- /dev/null +++ b/src/core/pdu/tests.rs @@ -0,0 +1,19 @@ +#![cfg(test)] + +use super::PduCount; + +#[test] +fn backfilled_parse() { + let count: PduCount = "-987654".parse().expect("parse() failed"); + let backfilled = matches!(count, PduCount::Backfilled(_)); + + assert!(backfilled, "not backfilled variant"); +} + +#[test] +fn normal_parse() { + let count: PduCount = "987654".parse().expect("parse() failed"); + let backfilled = matches!(count, PduCount::Backfilled(_)); + + assert!(!backfilled, "backfilled variant"); +} diff --git a/src/service/migrations.rs b/src/service/migrations.rs index 45323fa2..d6c342f8 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -71,7 +71,7 @@ async fn fresh(services: &Services) -> Result<()> { db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", []); // Create the admin room and server user on first run - crate::admin::create_admin_room(services).await?; + crate::admin::create_admin_room(services).boxed().await?; warn!( "Created new {} database with version {DATABASE_VERSION}", diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index 5c9dbda8..3c36928a 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -7,9 +7,11 @@ use conduit::{err, utils, utils::math::usize_from_f64, Err, Result}; use database::Map; use lru_cache::LruCache; +use crate::rooms::short::ShortEventId; + pub(super) struct Data { shorteventid_authchain: Arc, - pub(super) auth_chain_cache: Mutex, Arc<[u64]>>>, + pub(super) auth_chain_cache: Mutex, Arc<[ShortEventId]>>>, } impl Data { @@ -24,7 +26,7 @@ impl Data { } } - pub(super) async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { + pub(super) async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { debug_assert!(!key.is_empty(), "auth_chain key must not be empty"); // Check RAM cache @@ -63,7 +65,7 @@ impl Data { Ok(chain) } - pub(super) fn cache_auth_chain(&self, key: Vec, auth_chain: Arc<[u64]>) { + pub(super) fn cache_auth_chain(&self, key: Vec, auth_chain: Arc<[ShortEventId]>) { debug_assert!(!key.is_empty(), "auth_chain key must not be empty"); // Only persist single events in db diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 1387bc7d..c22732c2 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -10,7 +10,7 @@ use futures::Stream; use ruma::{EventId, RoomId}; use self::data::Data; -use crate::{rooms, Dep}; +use crate::{rooms, rooms::short::ShortEventId, Dep}; pub struct Service { services: Services, @@ -64,7 +64,7 @@ impl Service { } #[tracing::instrument(skip_all, name = "auth_chain")] - pub async fn get_auth_chain(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result> { + pub async fn get_auth_chain(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result> { const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); @@ -97,7 +97,7 @@ impl Service { continue; } - let chunk_key: Vec = chunk.iter().map(|(short, _)| short).copied().collect(); + let chunk_key: Vec = chunk.iter().map(|(short, _)| short).copied().collect(); if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { trace!("Found cache entry for whole chunk"); full_auth_chain.extend(cached.iter().copied()); @@ -156,7 +156,7 @@ impl Service { } #[tracing::instrument(skip(self, room_id))] - async fn get_auth_chain_inner(&self, room_id: &RoomId, event_id: &EventId) -> Result> { + async fn get_auth_chain_inner(&self, room_id: &RoomId, event_id: &EventId) -> Result> { let mut todo = vec![Arc::from(event_id)]; let mut found = HashSet::new(); @@ -195,19 +195,19 @@ impl Service { } #[inline] - pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { + pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { self.db.get_cached_eventid_authchain(key).await } #[tracing::instrument(skip(self), level = "debug")] - pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) { - let val = auth_chain.iter().copied().collect::>(); + pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) { + let val = auth_chain.iter().copied().collect::>(); self.db.cache_auth_chain(key, val); } #[tracing::instrument(skip(self), level = "debug")] - pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &Vec) { - let val = auth_chain.iter().copied().collect::>(); + pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &Vec) { + let val = auth_chain.iter().copied().collect::>(); self.db.cache_auth_chain(key, val); } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index adebd332..f76f817d 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -35,7 +35,10 @@ use ruma::{ use crate::{ globals, rooms, - rooms::state_compressor::{CompressedStateEvent, HashSetCompressStateEvent}, + rooms::{ + state_compressor::{CompressedStateEvent, HashSetCompressStateEvent}, + timeline::RawPduId, + }, sending, server_keys, Dep, }; @@ -136,10 +139,10 @@ impl Service { pub async fn handle_incoming_pdu<'a>( &self, origin: &'a ServerName, room_id: &'a RoomId, event_id: &'a EventId, value: BTreeMap, is_timeline_event: bool, - ) -> Result>> { + ) -> Result> { // 1. Skip the PDU if we already have it as a timeline event if let Ok(pdu_id) = self.services.timeline.get_pdu_id(event_id).await { - return Ok(Some(pdu_id.to_vec())); + return Ok(Some(pdu_id)); } // 1.1 Check the server is in the room @@ -488,7 +491,7 @@ impl Service { pub async fn upgrade_outlier_to_timeline_pdu( &self, incoming_pdu: Arc, val: BTreeMap, create_event: &PduEvent, origin: &ServerName, room_id: &RoomId, - ) -> Result>> { + ) -> Result> { // Skip the PDU if we already have it as a timeline event if let Ok(pduid) = self .services @@ -496,7 +499,7 @@ impl Service { .get_pdu_id(&incoming_pdu.event_id) .await { - return Ok(Some(pduid.to_vec())); + return Ok(Some(pduid)); } if self diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 51a43714..3fc06591 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -2,15 +2,21 @@ use std::{mem::size_of, sync::Arc}; use conduit::{ result::LogErr, - utils, - utils::{stream::TryIgnore, ReadyExt}, + utils::{stream::TryIgnore, u64_from_u8, ReadyExt}, PduCount, PduEvent, }; use database::Map; use futures::{Stream, StreamExt}; use ruma::{api::Direction, EventId, RoomId, UserId}; -use crate::{rooms, Dep}; +use crate::{ + rooms, + rooms::{ + short::{ShortEventId, ShortRoomId}, + timeline::{PduId, RawPduId}, + }, + Dep, +}; pub(super) struct Data { tofrom_relation: Arc, @@ -46,35 +52,36 @@ impl Data { } pub(super) fn get_relations<'a>( - &'a self, user_id: &'a UserId, shortroomid: u64, target: u64, until: PduCount, dir: Direction, + &'a self, user_id: &'a UserId, shortroomid: ShortRoomId, target: ShortEventId, from: PduCount, dir: Direction, ) -> impl Stream + Send + '_ { - let prefix = target.to_be_bytes().to_vec(); - let mut current = prefix.clone(); - let count_raw = match until { - PduCount::Normal(x) => x.saturating_sub(1), - PduCount::Backfilled(x) => { - current.extend_from_slice(&0_u64.to_be_bytes()); - u64::MAX.saturating_sub(x).saturating_sub(1) - }, - }; - current.extend_from_slice(&count_raw.to_be_bytes()); + let current: RawPduId = PduId { + shortroomid, + shorteventid: from, + } + .into(); match dir { Direction::Forward => self.tofrom_relation.raw_keys_from(¤t).boxed(), Direction::Backward => self.tofrom_relation.rev_raw_keys_from(¤t).boxed(), } .ignore_err() - .ready_take_while(move |key| key.starts_with(&prefix)) - .map(|to_from| utils::u64_from_u8(&to_from[(size_of::())..])) - .filter_map(move |from| async move { - let mut pduid = shortroomid.to_be_bytes().to_vec(); - pduid.extend_from_slice(&from.to_be_bytes()); - let mut pdu = self.services.timeline.get_pdu_from_id(&pduid).await.ok()?; + .ready_take_while(move |key| key.starts_with(&target.to_be_bytes())) + .map(|to_from| u64_from_u8(&to_from[8..16])) + .map(PduCount::from_unsigned) + .filter_map(move |shorteventid| async move { + let pdu_id: RawPduId = PduId { + shortroomid, + shorteventid, + } + .into(); + + let mut pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await.ok()?; + if pdu.sender != user_id { pdu.remove_transaction_id().log_err().ok(); } - Some((PduCount::Normal(from), pdu)) + Some((shorteventid, pdu)) }) } diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index b1cf2049..82d2ee35 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -1,18 +1,9 @@ mod data; use std::sync::Arc; -use conduit::{ - at, - utils::{result::FlatOk, stream::ReadyExt, IterStream}, - PduCount, Result, -}; -use futures::{FutureExt, StreamExt}; -use ruma::{ - api::{client::relations::get_relating_events, Direction}, - events::{relation::RelationType, TimelineEventType}, - EventId, RoomId, UInt, UserId, -}; -use serde::Deserialize; +use conduit::{PduCount, Result}; +use futures::StreamExt; +use ruma::{api::Direction, EventId, RoomId, UserId}; use self::data::{Data, PdusIterItem}; use crate::{rooms, Dep}; @@ -24,26 +15,14 @@ pub struct Service { struct Services { short: Dep, - state_accessor: Dep, timeline: Dep, } -#[derive(Clone, Debug, Deserialize)] -struct ExtractRelType { - rel_type: RelationType, -} -#[derive(Clone, Debug, Deserialize)] -struct ExtractRelatesToEventId { - #[serde(rename = "m.relates_to")] - relates_to: ExtractRelType, -} - impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { services: Services { short: args.depend::("rooms::short"), - state_accessor: args.depend::("rooms::state_accessor"), timeline: args.depend::("rooms::timeline"), }, db: Data::new(&args), @@ -64,82 +43,9 @@ impl Service { } } - #[allow(clippy::too_many_arguments)] - pub async fn paginate_relations_with_filter( - &self, sender_user: &UserId, room_id: &RoomId, target: &EventId, filter_event_type: Option, - filter_rel_type: Option, from: Option<&str>, to: Option<&str>, limit: Option, - recurse: bool, dir: Direction, - ) -> Result { - let from = from - .map(PduCount::try_from_string) - .transpose()? - .unwrap_or_else(|| match dir { - Direction::Forward => PduCount::min(), - Direction::Backward => PduCount::max(), - }); - - let to = to.map(PduCount::try_from_string).flat_ok(); - - // Use limit or else 30, with maximum 100 - let limit: usize = limit - .map(TryInto::try_into) - .flat_ok() - .unwrap_or(30) - .min(100); - - // Spec (v1.10) recommends depth of at least 3 - let depth: u8 = if recurse { - 3 - } else { - 1 - }; - - let events: Vec = self - .get_relations(sender_user, room_id, target, from, limit, depth, dir) - .await - .into_iter() - .filter(|(_, pdu)| { - filter_event_type - .as_ref() - .is_none_or(|kind| *kind == pdu.kind) - }) - .filter(|(_, pdu)| { - filter_rel_type.as_ref().is_none_or(|rel_type| { - pdu.get_content() - .map(|c: ExtractRelatesToEventId| c.relates_to.rel_type) - .is_ok_and(|r| r == *rel_type) - }) - }) - .stream() - .filter_map(|item| self.visibility_filter(sender_user, item)) - .ready_take_while(|(count, _)| Some(*count) != to) - .take(limit) - .collect() - .boxed() - .await; - - let next_batch = match dir { - Direction::Backward => events.first(), - Direction::Forward => events.last(), - } - .map(at!(0)) - .map(|t| t.stringify()); - - Ok(get_relating_events::v1::Response { - next_batch, - prev_batch: Some(from.stringify()), - recursion_depth: recurse.then_some(depth.into()), - chunk: events - .into_iter() - .map(at!(1)) - .map(|pdu| pdu.to_message_like_event()) - .collect(), - }) - } - #[allow(clippy::too_many_arguments)] pub async fn get_relations( - &self, user_id: &UserId, room_id: &RoomId, target: &EventId, until: PduCount, limit: usize, max_depth: u8, + &self, user_id: &UserId, room_id: &RoomId, target: &EventId, from: PduCount, limit: usize, max_depth: u8, dir: Direction, ) -> Vec { let room_id = self.services.short.get_or_create_shortroomid(room_id).await; @@ -152,7 +58,7 @@ impl Service { let mut pdus: Vec<_> = self .db - .get_relations(user_id, room_id, target, until, dir) + .get_relations(user_id, room_id, target, from, dir) .collect() .await; @@ -167,7 +73,7 @@ impl Service { let relations: Vec<_> = self .db - .get_relations(user_id, room_id, target, until, dir) + .get_relations(user_id, room_id, target, from, dir) .collect() .await; @@ -186,16 +92,6 @@ impl Service { pdus } - async fn visibility_filter(&self, sender_user: &UserId, item: PdusIterItem) -> Option { - let (_, pdu) = &item; - - self.services - .state_accessor - .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) - .await - .then_some(item) - } - #[inline] #[tracing::instrument(skip_all, level = "debug")] pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) { diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 70daded1..1af37d9e 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,10 +1,10 @@ -use std::{iter, sync::Arc}; +use std::sync::Arc; use arrayvec::ArrayVec; use conduit::{ implement, utils::{set, stream::TryIgnore, ArrayVecExt, IterStream, ReadyExt}, - PduEvent, Result, + PduCount, PduEvent, Result, }; use database::{keyval::Val, Map}; use futures::{Stream, StreamExt}; @@ -66,13 +66,13 @@ impl crate::Service for Service { } #[implement(Service)] -pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { +pub fn index_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) { let batch = tokenize(message_body) .map(|word| { let mut key = shortroomid.to_be_bytes().to_vec(); key.extend_from_slice(word.as_bytes()); key.push(0xFF); - key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here + key.extend_from_slice(pdu_id.as_ref()); // TODO: currently we save the room id a second time here (key, Vec::::new()) }) .collect::>(); @@ -81,12 +81,12 @@ pub fn index_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { } #[implement(Service)] -pub fn deindex_pdu(&self, shortroomid: u64, pdu_id: &[u8], message_body: &str) { +pub fn deindex_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_body: &str) { let batch = tokenize(message_body).map(|word| { let mut key = shortroomid.to_be_bytes().to_vec(); key.extend_from_slice(word.as_bytes()); key.push(0xFF); - key.extend_from_slice(pdu_id); // TODO: currently we save the room id a second time here + key.extend_from_slice(pdu_id.as_ref()); // TODO: currently we save the room id a second time here key }); @@ -159,24 +159,24 @@ fn search_pdu_ids_query_words<'a>( &'a self, shortroomid: ShortRoomId, word: &'a str, ) -> impl Stream + Send + '_ { self.search_pdu_ids_query_word(shortroomid, word) - .ready_filter_map(move |key| { - key[prefix_len(word)..] - .chunks_exact(PduId::LEN) - .next() - .map(RawPduId::try_from) - .and_then(Result::ok) + .map(move |key| -> RawPduId { + let key = &key[prefix_len(word)..]; + key.into() }) } /// Iterate over raw database results for a word #[implement(Service)] fn search_pdu_ids_query_word(&self, shortroomid: ShortRoomId, word: &str) -> impl Stream> + Send + '_ { - const PDUID_LEN: usize = PduId::LEN; // rustc says const'ing this not yet stable - let end_id: ArrayVec = iter::repeat(u8::MAX).take(PduId::LEN).collect(); + let end_id: RawPduId = PduId { + shortroomid, + shorteventid: PduCount::max(), + } + .into(); // Newest pdus first - let end = make_tokenid(shortroomid, word, end_id.as_slice()); + let end = make_tokenid(shortroomid, word, &end_id); let prefix = make_prefix(shortroomid, word); self.db .tokenids @@ -196,11 +196,9 @@ fn tokenize(body: &str) -> impl Iterator + Send + '_ { .map(str::to_lowercase) } -fn make_tokenid(shortroomid: ShortRoomId, word: &str, pdu_id: &[u8]) -> TokenId { - debug_assert!(pdu_id.len() == PduId::LEN, "pdu_id size mismatch"); - +fn make_tokenid(shortroomid: ShortRoomId, word: &str, pdu_id: &RawPduId) -> TokenId { let mut key = make_prefix(shortroomid, word); - key.extend_from_slice(pdu_id); + key.extend_from_slice(pdu_id.as_ref()); key } diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index a903ef22..9fddf099 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,5 +1,6 @@ use std::{mem::size_of_val, sync::Arc}; +pub use conduit::pdu::{ShortEventId, ShortId, ShortRoomId}; use conduit::{err, implement, utils, Result}; use database::{Deserialized, Map}; use ruma::{events::StateEventType, EventId, RoomId}; @@ -26,9 +27,6 @@ struct Services { pub type ShortStateHash = ShortId; pub type ShortStateKey = ShortId; -pub type ShortEventId = ShortId; -pub type ShortRoomId = ShortId; -pub type ShortId = u64; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -52,7 +50,7 @@ impl crate::Service for Service { #[implement(Service)] pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEventId { - const BUFSIZE: usize = size_of::(); + const BUFSIZE: usize = size_of::(); if let Ok(shorteventid) = self .db @@ -88,7 +86,7 @@ pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> .map(|(i, result)| match result { Ok(ref short) => utils::u64_from_u8(short), Err(_) => { - const BUFSIZE: usize = size_of::(); + const BUFSIZE: usize = size_of::(); let short = self.services.globals.next_count().unwrap(); debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 5aea5f6a..37272dca 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -33,7 +33,7 @@ use ruma::{ }; use tokio::sync::Mutex; -use crate::{rooms, sending, Dep}; +use crate::{rooms, rooms::short::ShortRoomId, sending, Dep}; pub struct CachedSpaceHierarchySummary { summary: SpaceHierarchyParentSummary, @@ -49,7 +49,7 @@ pub enum SummaryAccessibility { pub struct PaginationToken { /// Path down the hierarchy of the room to start the response at, /// excluding the root space. - pub short_room_ids: Vec, + pub short_room_ids: Vec, pub limit: UInt, pub max_depth: UInt, pub suggested_only: bool, @@ -448,7 +448,7 @@ impl Service { } pub async fn get_client_hierarchy( - &self, sender_user: &UserId, room_id: &RoomId, limit: usize, short_room_ids: Vec, max_depth: u64, + &self, sender_user: &UserId, room_id: &RoomId, limit: usize, short_room_ids: Vec, max_depth: u64, suggested_only: bool, ) -> Result { let mut parents = VecDeque::new(); diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 34fab079..71a3900c 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -95,7 +95,7 @@ impl Service { let event_ids = statediffnew.iter().stream().filter_map(|new| { self.services .state_compressor - .parse_compressed_state_event(new) + .parse_compressed_state_event(*new) .map_ok_or_else(|_| None, |(_, event_id)| Some(event_id)) }); @@ -428,7 +428,7 @@ impl Service { let Ok((shortstatekey, event_id)) = self .services .state_compressor - .parse_compressed_state_event(compressed) + .parse_compressed_state_event(*compressed) .await else { continue; diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 9c96785f..06cd648c 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -53,7 +53,7 @@ impl Data { let parsed = self .services .state_compressor - .parse_compressed_state_event(compressed) + .parse_compressed_state_event(*compressed) .await?; result.insert(parsed.0, parsed.1); @@ -86,7 +86,7 @@ impl Data { let (_, eventid) = self .services .state_compressor - .parse_compressed_state_event(compressed) + .parse_compressed_state_event(*compressed) .await?; if let Ok(pdu) = self.services.timeline.get_pdu(&eventid).await { @@ -132,7 +132,7 @@ impl Data { self.services .state_compressor - .parse_compressed_state_event(compressed) + .parse_compressed_state_event(*compressed) .map_ok(|(_, id)| id) .map_err(|e| { err!(Database(error!( diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index a2cc27e8..d51da8af 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -39,13 +39,17 @@ use ruma::{ use serde::Deserialize; use self::data::Data; -use crate::{rooms, rooms::state::RoomMutexGuard, Dep}; +use crate::{ + rooms, + rooms::{short::ShortStateHash, state::RoomMutexGuard}, + Dep, +}; pub struct Service { services: Services, db: Data, - pub server_visibility_cache: Mutex>, - pub user_visibility_cache: Mutex>, + pub server_visibility_cache: Mutex>, + pub user_visibility_cache: Mutex>, } struct Services { @@ -94,11 +98,13 @@ impl Service { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_full_ids(&self, shortstatehash: u64) -> Result>> { + pub async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result>> { self.db.state_full_ids(shortstatehash).await } - pub async fn state_full(&self, shortstatehash: u64) -> Result>> { + pub async fn state_full( + &self, shortstatehash: ShortStateHash, + ) -> Result>> { self.db.state_full(shortstatehash).await } @@ -106,7 +112,7 @@ impl Service { /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] pub async fn state_get_id( - &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, + &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result> { self.db .state_get_id(shortstatehash, event_type, state_key) @@ -117,7 +123,7 @@ impl Service { /// `state_key`). #[inline] pub async fn state_get( - &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, + &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result> { self.db .state_get(shortstatehash, event_type, state_key) @@ -126,7 +132,7 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). pub async fn state_get_content( - &self, shortstatehash: u64, event_type: &StateEventType, state_key: &str, + &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result where T: for<'de> Deserialize<'de> + Send, @@ -137,7 +143,7 @@ impl Service { } /// Get membership for given user in state - async fn user_membership(&self, shortstatehash: u64, user_id: &UserId) -> MembershipState { + async fn user_membership(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> MembershipState { self.state_get_content(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) .await .map_or(MembershipState::Leave, |c: RoomMemberEventContent| c.membership) @@ -145,14 +151,14 @@ impl Service { /// The user was a joined member at this state (potentially in the past) #[inline] - async fn user_was_joined(&self, shortstatehash: u64, user_id: &UserId) -> bool { + async fn user_was_joined(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { self.user_membership(shortstatehash, user_id).await == MembershipState::Join } /// The user was an invited or joined room member at this state (potentially /// in the past) #[inline] - async fn user_was_invited(&self, shortstatehash: u64, user_id: &UserId) -> bool { + async fn user_was_invited(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { let s = self.user_membership(shortstatehash, user_id).await; s == MembershipState::Join || s == MembershipState::Invite } @@ -285,7 +291,7 @@ impl Service { } /// Returns the state hash for this pdu. - pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { + pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { self.db.pdu_shortstatehash(event_id).await } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index e213490b..bf90d5c4 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -34,25 +34,26 @@ struct Data { #[derive(Clone)] struct StateDiff { parent: Option, - added: Arc>, - removed: Arc>, + added: Arc, + removed: Arc, } #[derive(Clone, Default)] pub struct ShortStateInfo { pub shortstatehash: ShortStateHash, - pub full_state: Arc>, - pub added: Arc>, - pub removed: Arc>, + pub full_state: Arc, + pub added: Arc, + pub removed: Arc, } #[derive(Clone, Default)] pub struct HashSetCompressStateEvent { pub shortstatehash: ShortStateHash, - pub added: Arc>, - pub removed: Arc>, + pub added: Arc, + pub removed: Arc, } +pub(crate) type CompressedState = HashSet; pub(crate) type CompressedStateEvent = [u8; 2 * size_of::()]; type StateInfoLruCache = LruCache; type ShortStateInfoVec = Vec; @@ -105,7 +106,7 @@ impl Service { removed, } = self.get_statediff(shortstatehash).await?; - if let Some(parent) = parent { + let response = if let Some(parent) = parent { let mut response = Box::pin(self.load_shortstatehash_info(parent)).await?; let mut state = (*response.last().expect("at least one response").full_state).clone(); state.extend(added.iter().copied()); @@ -121,27 +122,22 @@ impl Service { removed: Arc::new(removed), }); - self.stateinfo_cache - .lock() - .expect("locked") - .insert(shortstatehash, response.clone()); - - Ok(response) + response } else { - let response = vec![ShortStateInfo { + vec![ShortStateInfo { shortstatehash, full_state: added.clone(), added, removed, - }]; + }] + }; - self.stateinfo_cache - .lock() - .expect("locked") - .insert(shortstatehash, response.clone()); + self.stateinfo_cache + .lock() + .expect("locked") + .insert(shortstatehash, response.clone()); - Ok(response) - } + Ok(response) } pub async fn compress_state_event(&self, shortstatekey: ShortStateKey, event_id: &EventId) -> CompressedStateEvent { @@ -161,7 +157,7 @@ impl Service { /// Returns shortstatekey, event id #[inline] pub async fn parse_compressed_state_event( - &self, compressed_event: &CompressedStateEvent, + &self, compressed_event: CompressedStateEvent, ) -> Result<(ShortStateKey, Arc)> { use utils::u64_from_u8; diff --git a/src/service/rooms/threads/data.rs b/src/service/rooms/threads/data.rs index f50b812c..c26dabb4 100644 --- a/src/service/rooms/threads/data.rs +++ b/src/service/rooms/threads/data.rs @@ -1,17 +1,22 @@ -use std::{mem::size_of, sync::Arc}; +use std::sync::Arc; use conduit::{ - checked, result::LogErr, - utils, utils::{stream::TryIgnore, ReadyExt}, - PduEvent, Result, + PduCount, PduEvent, Result, }; use database::{Deserialized, Map}; use futures::{Stream, StreamExt}; use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId}; -use crate::{rooms, Dep}; +use crate::{ + rooms, + rooms::{ + short::ShortRoomId, + timeline::{PduId, RawPduId}, + }, + Dep, +}; pub(super) struct Data { threadid_userids: Arc, @@ -35,40 +40,39 @@ impl Data { } } + #[inline] pub(super) async fn threads_until<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, _include: &'a IncludeThreads, - ) -> Result + Send + 'a> { - let prefix = self - .services - .short - .get_shortroomid(room_id) - .await? - .to_be_bytes() - .to_vec(); + &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, _include: &'a IncludeThreads, + ) -> Result + Send + 'a> { + let shortroomid: ShortRoomId = self.services.short.get_shortroomid(room_id).await?; - let mut current = prefix.clone(); - current.extend_from_slice(&(checked!(until - 1)?).to_be_bytes()); + let current: RawPduId = PduId { + shortroomid, + shorteventid: until.saturating_sub(1), + } + .into(); let stream = self .threadid_userids .rev_raw_keys_from(¤t) .ignore_err() - .ready_take_while(move |key| key.starts_with(&prefix)) - .map(|pduid| (utils::u64_from_u8(&pduid[(size_of::())..]), pduid)) - .filter_map(move |(count, pduid)| async move { - let mut pdu = self.services.timeline.get_pdu_from_id(pduid).await.ok()?; + .map(RawPduId::from) + .ready_take_while(move |pdu_id| pdu_id.shortroomid() == shortroomid.to_be_bytes()) + .filter_map(move |pdu_id| async move { + let mut pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await.ok()?; + let pdu_id: PduId = pdu_id.into(); if pdu.sender != user_id { pdu.remove_transaction_id().log_err().ok(); } - Some((count, pdu)) + Some((pdu_id.shorteventid, pdu)) }); Ok(stream) } - pub(super) fn update_participants(&self, root_id: &[u8], participants: &[OwnedUserId]) -> Result<()> { + pub(super) fn update_participants(&self, root_id: &RawPduId, participants: &[OwnedUserId]) -> Result { let users = participants .iter() .map(|user| user.as_bytes()) @@ -80,7 +84,7 @@ impl Data { Ok(()) } - pub(super) async fn get_participants(&self, root_id: &[u8]) -> Result> { - self.threadid_userids.qry(root_id).await.deserialized() + pub(super) async fn get_participants(&self, root_id: &RawPduId) -> Result> { + self.threadid_userids.get(root_id).await.deserialized() } } diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index 2eafe5d5..02503030 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -2,7 +2,7 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{err, PduEvent, Result}; +use conduit::{err, PduCount, PduEvent, Result}; use data::Data; use futures::Stream; use ruma::{ @@ -37,8 +37,8 @@ impl crate::Service for Service { impl Service { pub async fn threads_until<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: u64, include: &'a IncludeThreads, - ) -> Result + Send + 'a> { + &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, include: &'a IncludeThreads, + ) -> Result + Send + 'a> { self.db .threads_until(user_id, room_id, until, include) .await diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 5428a3b9..19dc5325 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -1,14 +1,13 @@ use std::{ collections::{hash_map, HashMap}, - mem::size_of, sync::Arc, }; use conduit::{ - err, expected, + at, err, result::{LogErr, NotFound}, utils, - utils::{future::TryExtExt, stream::TryIgnore, u64_from_u8, ReadyExt}, + utils::{future::TryExtExt, stream::TryIgnore, ReadyExt}, Err, PduCount, PduEvent, Result, }; use database::{Database, Deserialized, Json, KeyVal, Map}; @@ -16,7 +15,8 @@ use futures::{Stream, StreamExt}; use ruma::{CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; use tokio::sync::Mutex; -use crate::{rooms, Dep}; +use super::{PduId, RawPduId}; +use crate::{rooms, rooms::short::ShortRoomId, Dep}; pub(super) struct Data { eventid_outlierpdu: Arc, @@ -58,30 +58,25 @@ impl Data { .lasttimelinecount_cache .lock() .await - .entry(room_id.to_owned()) + .entry(room_id.into()) { - hash_map::Entry::Vacant(v) => { - if let Some(last_count) = self - .pdus_until(sender_user, room_id, PduCount::max()) - .await? - .next() - .await - { - Ok(*v.insert(last_count.0)) - } else { - Ok(PduCount::Normal(0)) - } - }, hash_map::Entry::Occupied(o) => Ok(*o.get()), + hash_map::Entry::Vacant(v) => Ok(self + .pdus_until(sender_user, room_id, PduCount::max()) + .await? + .next() + .await + .map(at!(0)) + .filter(|&count| matches!(count, PduCount::Normal(_))) + .map_or_else(PduCount::max, |count| *v.insert(count))), } } /// Returns the `count` of this pdu's id. pub(super) async fn get_pdu_count(&self, event_id: &EventId) -> Result { - self.eventid_pduid - .get(event_id) + self.get_pdu_id(event_id) .await - .map(|pdu_id| pdu_count(&pdu_id)) + .map(|pdu_id| pdu_id.pdu_count()) } /// Returns the json of a pdu. @@ -102,8 +97,11 @@ impl Data { /// Returns the pdu's id. #[inline] - pub(super) async fn get_pdu_id(&self, event_id: &EventId) -> Result> { - self.eventid_pduid.get(event_id).await + pub(super) async fn get_pdu_id(&self, event_id: &EventId) -> Result { + self.eventid_pduid + .get(event_id) + .await + .map(|handle| RawPduId::from(&*handle)) } /// Returns the pdu directly from `eventid_pduid` only. @@ -154,34 +152,40 @@ impl Data { /// Returns the pdu. /// /// This does __NOT__ check the outliers `Tree`. - pub(super) async fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result { + pub(super) async fn get_pdu_from_id(&self, pdu_id: &RawPduId) -> Result { self.pduid_pdu.get(pdu_id).await.deserialized() } /// Returns the pdu as a `BTreeMap`. - pub(super) async fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result { + pub(super) async fn get_pdu_json_from_id(&self, pdu_id: &RawPduId) -> Result { self.pduid_pdu.get(pdu_id).await.deserialized() } - pub(super) async fn append_pdu(&self, pdu_id: &[u8], pdu: &PduEvent, json: &CanonicalJsonObject, count: u64) { + pub(super) async fn append_pdu( + &self, pdu_id: &RawPduId, pdu: &PduEvent, json: &CanonicalJsonObject, count: PduCount, + ) { + debug_assert!(matches!(count, PduCount::Normal(_)), "PduCount not Normal"); + self.pduid_pdu.raw_put(pdu_id, Json(json)); self.lasttimelinecount_cache .lock() .await - .insert(pdu.room_id.clone(), PduCount::Normal(count)); + .insert(pdu.room_id.clone(), count); self.eventid_pduid.insert(pdu.event_id.as_bytes(), pdu_id); self.eventid_outlierpdu.remove(pdu.event_id.as_bytes()); } - pub(super) fn prepend_backfill_pdu(&self, pdu_id: &[u8], event_id: &EventId, json: &CanonicalJsonObject) { + pub(super) fn prepend_backfill_pdu(&self, pdu_id: &RawPduId, event_id: &EventId, json: &CanonicalJsonObject) { self.pduid_pdu.raw_put(pdu_id, Json(json)); self.eventid_pduid.insert(event_id, pdu_id); self.eventid_outlierpdu.remove(event_id); } /// Removes a pdu and creates a new one with the same id. - pub(super) async fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, _pdu: &PduEvent) -> Result { + pub(super) async fn replace_pdu( + &self, pdu_id: &RawPduId, pdu_json: &CanonicalJsonObject, _pdu: &PduEvent, + ) -> Result { if self.pduid_pdu.get(pdu_id).await.is_not_found() { return Err!(Request(NotFound("PDU does not exist."))); } @@ -197,13 +201,14 @@ impl Data { pub(super) async fn pdus_until<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, ) -> Result + Send + 'a> { - let (prefix, current) = self.count_to_id(room_id, until, 1, true).await?; + let current = self.count_to_id(room_id, until, true).await?; + let prefix = current.shortroomid(); let stream = self .pduid_pdu .rev_raw_stream_from(¤t) .ignore_err() .ready_take_while(move |(key, _)| key.starts_with(&prefix)) - .map(move |item| Self::each_pdu(item, user_id)); + .map(|item| Self::each_pdu(item, user_id)); Ok(stream) } @@ -211,7 +216,8 @@ impl Data { pub(super) async fn pdus_after<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, from: PduCount, ) -> Result + Send + 'a> { - let (prefix, current) = self.count_to_id(room_id, from, 1, false).await?; + let current = self.count_to_id(room_id, from, false).await?; + let prefix = current.shortroomid(); let stream = self .pduid_pdu .raw_stream_from(¤t) @@ -223,6 +229,8 @@ impl Data { } fn each_pdu((pdu_id, pdu): KeyVal<'_>, user_id: &UserId) -> PdusIterItem { + let pdu_id: RawPduId = pdu_id.into(); + let mut pdu = serde_json::from_slice::(pdu).expect("PduEvent in pduid_pdu database column is invalid JSON"); @@ -231,9 +239,8 @@ impl Data { } pdu.add_age().log_err().ok(); - let count = pdu_count(pdu_id); - (count, pdu) + (pdu_id.pdu_count(), pdu) } pub(super) fn increment_notification_counts( @@ -256,56 +263,25 @@ impl Data { } } - pub(super) async fn count_to_id( - &self, room_id: &RoomId, count: PduCount, offset: u64, subtract: bool, - ) -> Result<(Vec, Vec)> { - let prefix = self + async fn count_to_id(&self, room_id: &RoomId, count: PduCount, subtract: bool) -> Result { + let shortroomid: ShortRoomId = self .services .short .get_shortroomid(room_id) .await - .map_err(|e| err!(Request(NotFound("Room {room_id:?} not found: {e:?}"))))? - .to_be_bytes() - .to_vec(); + .map_err(|e| err!(Request(NotFound("Room {room_id:?} not found: {e:?}"))))?; - let mut pdu_id = prefix.clone(); // +1 so we don't send the base event - let count_raw = match count { - PduCount::Normal(x) => { - if subtract { - x.saturating_sub(offset) - } else { - x.saturating_add(offset) - } - }, - PduCount::Backfilled(x) => { - pdu_id.extend_from_slice(&0_u64.to_be_bytes()); - let num = u64::MAX.saturating_sub(x); - if subtract { - num.saturating_sub(offset) - } else { - num.saturating_add(offset) - } + let pdu_id = PduId { + shortroomid, + shorteventid: if subtract { + count.checked_sub(1)? + } else { + count.checked_add(1)? }, }; - pdu_id.extend_from_slice(&count_raw.to_be_bytes()); - Ok((prefix, pdu_id)) - } -} - -/// Returns the `count` of this pdu's id. -pub(super) fn pdu_count(pdu_id: &[u8]) -> PduCount { - const STRIDE: usize = size_of::(); - - let pdu_id_len = pdu_id.len(); - let last_u64 = u64_from_u8(&pdu_id[expected!(pdu_id_len - STRIDE)..]); - let second_last_u64 = u64_from_u8(&pdu_id[expected!(pdu_id_len - 2 * STRIDE)..expected!(pdu_id_len - STRIDE)]); - - if second_last_u64 == 0 { - PduCount::Backfilled(u64::MAX.saturating_sub(last_u64)) - } else { - PduCount::Normal(last_u64) + Ok(pdu_id.into()) } } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index e45bf7e5..86a47919 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1,5 +1,4 @@ mod data; -mod pduid; use std::{ cmp, @@ -15,6 +14,7 @@ use conduit::{ utils::{stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, validated, warn, Err, Error, Result, Server, }; +pub use conduit::{PduId, RawPduId}; use futures::{future, future::ready, Future, FutureExt, Stream, StreamExt, TryStreamExt}; use ruma::{ api::federation, @@ -39,13 +39,13 @@ use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use self::data::Data; -pub use self::{ - data::PdusIterItem, - pduid::{PduId, RawPduId}, -}; +pub use self::data::PdusIterItem; use crate::{ - account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms, - rooms::state_compressor::CompressedStateEvent, sending, server_keys, users, Dep, + account_data, admin, appservice, + appservice::NamespaceRegex, + globals, pusher, rooms, + rooms::{short::ShortRoomId, state_compressor::CompressedStateEvent}, + sending, server_keys, users, Dep, }; // Update Relationships @@ -229,9 +229,7 @@ impl Service { /// Returns the pdu's id. #[inline] - pub async fn get_pdu_id(&self, event_id: &EventId) -> Result> { - self.db.get_pdu_id(event_id).await - } + pub async fn get_pdu_id(&self, event_id: &EventId) -> Result { self.db.get_pdu_id(event_id).await } /// Returns the pdu. /// @@ -256,16 +254,16 @@ impl Service { /// Returns the pdu. /// /// This does __NOT__ check the outliers `Tree`. - pub async fn get_pdu_from_id(&self, pdu_id: &[u8]) -> Result { self.db.get_pdu_from_id(pdu_id).await } + pub async fn get_pdu_from_id(&self, pdu_id: &RawPduId) -> Result { self.db.get_pdu_from_id(pdu_id).await } /// Returns the pdu as a `BTreeMap`. - pub async fn get_pdu_json_from_id(&self, pdu_id: &[u8]) -> Result { + pub async fn get_pdu_json_from_id(&self, pdu_id: &RawPduId) -> Result { self.db.get_pdu_json_from_id(pdu_id).await } /// Removes a pdu and creates a new one with the same id. #[tracing::instrument(skip(self), level = "debug")] - pub async fn replace_pdu(&self, pdu_id: &[u8], pdu_json: &CanonicalJsonObject, pdu: &PduEvent) -> Result<()> { + pub async fn replace_pdu(&self, pdu_id: &RawPduId, pdu_json: &CanonicalJsonObject, pdu: &PduEvent) -> Result<()> { self.db.replace_pdu(pdu_id, pdu_json, pdu).await } @@ -282,7 +280,7 @@ impl Service { mut pdu_json: CanonicalJsonObject, leaves: Vec, state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex - ) -> Result> { + ) -> Result { // Coalesce database writes for the remainder of this scope. let _cork = self.db.db.cork_and_flush(); @@ -359,9 +357,12 @@ impl Service { .user .reset_notification_counts(&pdu.sender, &pdu.room_id); - let count2 = self.services.globals.next_count().unwrap(); - let mut pdu_id = shortroomid.to_be_bytes().to_vec(); - pdu_id.extend_from_slice(&count2.to_be_bytes()); + let count2 = PduCount::Normal(self.services.globals.next_count().unwrap()); + let pdu_id: RawPduId = PduId { + shortroomid, + shorteventid: count2, + } + .into(); // Insert pdu self.db.append_pdu(&pdu_id, pdu, &pdu_json, count2).await; @@ -544,7 +545,7 @@ impl Service { if let Ok(related_pducount) = self.get_pdu_count(&content.relates_to.event_id).await { self.services .pdu_metadata - .add_relation(PduCount::Normal(count2), related_pducount); + .add_relation(count2, related_pducount); } } @@ -558,7 +559,7 @@ impl Service { if let Ok(related_pducount) = self.get_pdu_count(&in_reply_to.event_id).await { self.services .pdu_metadata - .add_relation(PduCount::Normal(count2), related_pducount); + .add_relation(count2, related_pducount); } }, Relation::Thread(thread) => { @@ -580,7 +581,7 @@ impl Service { { self.services .sending - .send_pdu_appservice(appservice.registration.id.clone(), pdu_id.clone())?; + .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; continue; } @@ -596,7 +597,7 @@ impl Service { if state_key_uid == appservice_uid { self.services .sending - .send_pdu_appservice(appservice.registration.id.clone(), pdu_id.clone())?; + .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; continue; } } @@ -623,7 +624,7 @@ impl Service { { self.services .sending - .send_pdu_appservice(appservice.registration.id.clone(), pdu_id.clone())?; + .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; } } @@ -935,7 +936,7 @@ impl Service { state_ids_compressed: Arc>, soft_fail: bool, state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex - ) -> Result>> { + ) -> Result> { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't // fail. @@ -993,7 +994,7 @@ impl Service { /// Replace a PDU with the redacted form. #[tracing::instrument(skip(self, reason))] - pub async fn redact_pdu(&self, event_id: &EventId, reason: &PduEvent, shortroomid: u64) -> Result<()> { + pub async fn redact_pdu(&self, event_id: &EventId, reason: &PduEvent, shortroomid: ShortRoomId) -> Result { // TODO: Don't reserialize, keep original json let Ok(pdu_id) = self.get_pdu_id(event_id).await else { // If event does not exist, just noop @@ -1133,7 +1134,6 @@ impl Service { // Skip the PDU if we already have it as a timeline event if let Ok(pdu_id) = self.get_pdu_id(&event_id).await { - let pdu_id = pdu_id.to_vec(); debug!("We already know {event_id} at {pdu_id:?}"); return Ok(()); } @@ -1158,11 +1158,13 @@ impl Service { let insert_lock = self.mutex_insert.lock(&room_id).await; - let max = u64::MAX; - let count = self.services.globals.next_count().unwrap(); - let mut pdu_id = shortroomid.to_be_bytes().to_vec(); - pdu_id.extend_from_slice(&0_u64.to_be_bytes()); - pdu_id.extend_from_slice(&(validated!(max - count)).to_be_bytes()); + let count: i64 = self.services.globals.next_count().unwrap().try_into()?; + + let pdu_id: RawPduId = PduId { + shortroomid, + shorteventid: PduCount::Backfilled(validated!(0 - count)), + } + .into(); // Insert pdu self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value); @@ -1246,16 +1248,3 @@ async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Res Ok(()) } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn comparisons() { - assert!(PduCount::Normal(1) < PduCount::Normal(2)); - assert!(PduCount::Backfilled(2) < PduCount::Backfilled(1)); - assert!(PduCount::Normal(1) > PduCount::Backfilled(1)); - assert!(PduCount::Backfilled(1) < PduCount::Normal(1)); - } -} diff --git a/src/service/rooms/timeline/pduid.rs b/src/service/rooms/timeline/pduid.rs deleted file mode 100644 index b43c382c..00000000 --- a/src/service/rooms/timeline/pduid.rs +++ /dev/null @@ -1,13 +0,0 @@ -use crate::rooms::short::{ShortEventId, ShortRoomId}; - -#[derive(Clone, Copy)] -pub struct PduId { - _room_id: ShortRoomId, - _event_id: ShortEventId, -} - -pub type RawPduId = [u8; PduId::LEN]; - -impl PduId { - pub const LEN: usize = size_of::() + size_of::(); -} diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index e484203d..99587134 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -5,7 +5,7 @@ use database::{Deserialized, Map}; use futures::{pin_mut, Stream, StreamExt}; use ruma::{RoomId, UserId}; -use crate::{globals, rooms, Dep}; +use crate::{globals, rooms, rooms::short::ShortStateHash, Dep}; pub struct Service { db: Data, @@ -93,7 +93,7 @@ pub async fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) - } #[implement(Service)] -pub async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: u64) { +pub async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: ShortStateHash) { let shortroomid = self .services .short @@ -108,7 +108,7 @@ pub async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, } #[implement(Service)] -pub async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result { +pub async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result { let shortroomid = self.services.short.get_shortroomid(room_id).await?; let key: &[u64] = &[shortroomid, token]; diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index f75a212c..cd25776a 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -115,10 +115,10 @@ impl Data { let mut keys = Vec::new(); for (event, destination) in requests { let mut key = destination.get_prefix(); - if let SendingEvent::Pdu(value) = &event { - key.extend_from_slice(value); + if let SendingEvent::Pdu(value) = event { + key.extend(value.as_ref()); } else { - key.extend_from_slice(&self.services.globals.next_count().unwrap().to_be_bytes()); + key.extend(&self.services.globals.next_count().unwrap().to_be_bytes()); } let value = if let SendingEvent::Edu(value) = &event { &**value @@ -175,7 +175,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se ( Destination::Appservice(server), if value.is_empty() { - SendingEvent::Pdu(event.to_vec()) + SendingEvent::Pdu(event.into()) } else { SendingEvent::Edu(value.to_vec()) }, @@ -202,7 +202,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se ( Destination::Push(user_id, pushkey_string), if value.is_empty() { - SendingEvent::Pdu(event.to_vec()) + SendingEvent::Pdu(event.into()) } else { // I'm pretty sure this should never be called SendingEvent::Edu(value.to_vec()) @@ -225,7 +225,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se .map_err(|_| Error::bad_database("Invalid server string in server_currenttransaction"))?, ), if value.is_empty() { - SendingEvent::Pdu(event.to_vec()) + SendingEvent::Pdu(event.into()) } else { SendingEvent::Edu(value.to_vec()) }, diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index ea266883..77997f69 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -24,7 +24,10 @@ pub use self::{ dest::Destination, sender::{EDU_LIMIT, PDU_LIMIT}, }; -use crate::{account_data, client, globals, presence, pusher, resolver, rooms, server_keys, users, Dep}; +use crate::{ + account_data, client, globals, presence, pusher, resolver, rooms, rooms::timeline::RawPduId, server_keys, users, + Dep, +}; pub struct Service { server: Arc, @@ -61,9 +64,9 @@ struct Msg { #[allow(clippy::module_name_repetitions)] #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum SendingEvent { - Pdu(Vec), // pduid - Edu(Vec), // pdu json - Flush, // none + Pdu(RawPduId), // pduid + Edu(Vec), // pdu json + Flush, // none } #[async_trait] @@ -110,9 +113,9 @@ impl crate::Service for Service { impl Service { #[tracing::instrument(skip(self, pdu_id, user, pushkey), level = "debug")] - pub fn send_pdu_push(&self, pdu_id: &[u8], user: &UserId, pushkey: String) -> Result<()> { + pub fn send_pdu_push(&self, pdu_id: &RawPduId, user: &UserId, pushkey: String) -> Result { let dest = Destination::Push(user.to_owned(), pushkey); - let event = SendingEvent::Pdu(pdu_id.to_owned()); + let event = SendingEvent::Pdu(*pdu_id); let _cork = self.db.db.cork(); let keys = self.db.queue_requests(&[(&event, &dest)]); self.dispatch(Msg { @@ -123,7 +126,7 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub fn send_pdu_appservice(&self, appservice_id: String, pdu_id: Vec) -> Result<()> { + pub fn send_pdu_appservice(&self, appservice_id: String, pdu_id: RawPduId) -> Result { let dest = Destination::Appservice(appservice_id); let event = SendingEvent::Pdu(pdu_id); let _cork = self.db.db.cork(); @@ -136,7 +139,7 @@ impl Service { } #[tracing::instrument(skip(self, room_id, pdu_id), level = "debug")] - pub async fn send_pdu_room(&self, room_id: &RoomId, pdu_id: &[u8]) -> Result<()> { + pub async fn send_pdu_room(&self, room_id: &RoomId, pdu_id: &RawPduId) -> Result { let servers = self .services .state_cache @@ -147,13 +150,13 @@ impl Service { } #[tracing::instrument(skip(self, servers, pdu_id), level = "debug")] - pub async fn send_pdu_servers<'a, S>(&self, servers: S, pdu_id: &[u8]) -> Result<()> + pub async fn send_pdu_servers<'a, S>(&self, servers: S, pdu_id: &RawPduId) -> Result where S: Stream + Send + 'a, { let _cork = self.db.db.cork(); let requests = servers - .map(|server| (Destination::Normal(server.into()), SendingEvent::Pdu(pdu_id.into()))) + .map(|server| (Destination::Normal(server.into()), SendingEvent::Pdu(pdu_id.to_owned()))) .collect::>() .await; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index d9087d44..464d186b 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -536,7 +536,8 @@ impl Service { &events .iter() .map(|e| match e { - SendingEvent::Edu(b) | SendingEvent::Pdu(b) => &**b, + SendingEvent::Edu(b) => &**b, + SendingEvent::Pdu(b) => b.as_ref(), SendingEvent::Flush => &[], }) .collect::>(), @@ -660,7 +661,8 @@ impl Service { &events .iter() .map(|e| match e { - SendingEvent::Edu(b) | SendingEvent::Pdu(b) => &**b, + SendingEvent::Edu(b) => &**b, + SendingEvent::Pdu(b) => b.as_ref(), SendingEvent::Flush => &[], }) .collect::>(), From 137e3008ea04d36f9562eeadc61b276032fd2ddf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 6 Nov 2024 21:02:23 +0000 Subject: [PATCH 1009/2091] merge rooms threads data and service Signed-off-by: Jason Volk --- src/api/client/relations.rs | 12 ++-- src/api/client/threads.rs | 10 ++- src/service/rooms/pdu_metadata/data.rs | 15 ++--- src/service/rooms/threads/data.rs | 90 -------------------------- src/service/rooms/threads/mod.rs | 88 +++++++++++++++++++------ 5 files changed, 91 insertions(+), 124 deletions(-) delete mode 100644 src/service/rooms/threads/data.rs diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index ef7035e2..b5d1485b 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -97,7 +97,7 @@ async fn paginate_relations_with_filter( filter_event_type: Option, filter_rel_type: Option, from: Option<&str>, to: Option<&str>, limit: Option, recurse: bool, dir: Direction, ) -> Result { - let from: PduCount = from + let start: PduCount = from .map(str::parse) .transpose()? .unwrap_or_else(|| match dir { @@ -124,7 +124,7 @@ async fn paginate_relations_with_filter( let events: Vec = services .rooms .pdu_metadata - .get_relations(sender_user, room_id, target, from, limit, depth, dir) + .get_relations(sender_user, room_id, target, start, limit, depth, dir) .await .into_iter() .filter(|(_, pdu)| { @@ -146,16 +146,20 @@ async fn paginate_relations_with_filter( .await; let next_batch = match dir { - Direction::Backward => events.first(), Direction::Forward => events.last(), + Direction::Backward => events.first(), } .map(at!(0)) + .map(|count| match dir { + Direction::Forward => count.saturating_add(1), + Direction::Backward => count.saturating_sub(1), + }) .as_ref() .map(ToString::to_string); Ok(get_relating_events::v1::Response { next_batch, - prev_batch: Some(from.to_string()), + prev_batch: from.map(Into::into), recursion_depth: recurse.then_some(depth.into()), chunk: events .into_iter() diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index 02cf7992..8d4e399b 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{PduCount, PduEvent}; +use conduit::{at, PduCount, PduEvent}; use futures::StreamExt; use ruma::{api::client::threads::get_threads, uint}; @@ -44,12 +44,16 @@ pub(crate) async fn get_threads_route( Ok(get_threads::v1::Response { next_batch: threads .last() - .map(|(count, _)| count) + .filter(|_| threads.len() >= limit) + .map(at!(0)) + .map(|count| count.saturating_sub(1)) + .as_ref() .map(ToString::to_string), chunk: threads .into_iter() - .map(|(_, pdu)| pdu.to_room_event()) + .map(at!(1)) + .map(|pdu| pdu.to_room_event()) .collect(), }) } diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 3fc06591..f3e1ced8 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,5 +1,6 @@ use std::{mem::size_of, sync::Arc}; +use arrayvec::ArrayVec; use conduit::{ result::LogErr, utils::{stream::TryIgnore, u64_from_u8, ReadyExt}, @@ -54,15 +55,13 @@ impl Data { pub(super) fn get_relations<'a>( &'a self, user_id: &'a UserId, shortroomid: ShortRoomId, target: ShortEventId, from: PduCount, dir: Direction, ) -> impl Stream + Send + '_ { - let current: RawPduId = PduId { - shortroomid, - shorteventid: from, - } - .into(); - + let mut current = ArrayVec::::new(); + current.extend(target.to_be_bytes()); + current.extend(from.into_unsigned().to_be_bytes()); + let current = current.as_slice(); match dir { - Direction::Forward => self.tofrom_relation.raw_keys_from(¤t).boxed(), - Direction::Backward => self.tofrom_relation.rev_raw_keys_from(¤t).boxed(), + Direction::Forward => self.tofrom_relation.raw_keys_from(current).boxed(), + Direction::Backward => self.tofrom_relation.rev_raw_keys_from(current).boxed(), } .ignore_err() .ready_take_while(move |key| key.starts_with(&target.to_be_bytes())) diff --git a/src/service/rooms/threads/data.rs b/src/service/rooms/threads/data.rs deleted file mode 100644 index c26dabb4..00000000 --- a/src/service/rooms/threads/data.rs +++ /dev/null @@ -1,90 +0,0 @@ -use std::sync::Arc; - -use conduit::{ - result::LogErr, - utils::{stream::TryIgnore, ReadyExt}, - PduCount, PduEvent, Result, -}; -use database::{Deserialized, Map}; -use futures::{Stream, StreamExt}; -use ruma::{api::client::threads::get_threads::v1::IncludeThreads, OwnedUserId, RoomId, UserId}; - -use crate::{ - rooms, - rooms::{ - short::ShortRoomId, - timeline::{PduId, RawPduId}, - }, - Dep, -}; - -pub(super) struct Data { - threadid_userids: Arc, - services: Services, -} - -struct Services { - short: Dep, - timeline: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - threadid_userids: db["threadid_userids"].clone(), - services: Services { - short: args.depend::("rooms::short"), - timeline: args.depend::("rooms::timeline"), - }, - } - } - - #[inline] - pub(super) async fn threads_until<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, _include: &'a IncludeThreads, - ) -> Result + Send + 'a> { - let shortroomid: ShortRoomId = self.services.short.get_shortroomid(room_id).await?; - - let current: RawPduId = PduId { - shortroomid, - shorteventid: until.saturating_sub(1), - } - .into(); - - let stream = self - .threadid_userids - .rev_raw_keys_from(¤t) - .ignore_err() - .map(RawPduId::from) - .ready_take_while(move |pdu_id| pdu_id.shortroomid() == shortroomid.to_be_bytes()) - .filter_map(move |pdu_id| async move { - let mut pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await.ok()?; - let pdu_id: PduId = pdu_id.into(); - - if pdu.sender != user_id { - pdu.remove_transaction_id().log_err().ok(); - } - - Some((pdu_id.shorteventid, pdu)) - }); - - Ok(stream) - } - - pub(super) fn update_participants(&self, root_id: &RawPduId, participants: &[OwnedUserId]) -> Result { - let users = participants - .iter() - .map(|user| user.as_bytes()) - .collect::>() - .join(&[0xFF][..]); - - self.threadid_userids.insert(root_id, &users); - - Ok(()) - } - - pub(super) async fn get_participants(&self, root_id: &RawPduId) -> Result> { - self.threadid_userids.get(root_id).await.deserialized() - } -} diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index 02503030..fcc629e1 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -1,34 +1,44 @@ -mod data; - use std::{collections::BTreeMap, sync::Arc}; -use conduit::{err, PduCount, PduEvent, Result}; -use data::Data; -use futures::Stream; +use conduit::{ + err, + utils::{stream::TryIgnore, ReadyExt}, + PduCount, PduEvent, PduId, RawPduId, Result, +}; +use database::{Deserialized, Map}; +use futures::{Stream, StreamExt}; use ruma::{ api::client::threads::get_threads::v1::IncludeThreads, events::relation::BundledThread, uint, CanonicalJsonValue, - EventId, RoomId, UserId, + EventId, OwnedUserId, RoomId, UserId, }; use serde_json::json; -use crate::{rooms, Dep}; +use crate::{rooms, rooms::short::ShortRoomId, Dep}; pub struct Service { - services: Services, db: Data, + services: Services, } struct Services { + short: Dep, timeline: Dep, } +pub(super) struct Data { + threadid_userids: Arc, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + db: Data { + threadid_userids: args.db["threadid_userids"].clone(), + }, services: Services { + short: args.depend::("rooms::short"), timeline: args.depend::("rooms::timeline"), }, - db: Data::new(&args), })) } @@ -36,14 +46,6 @@ impl crate::Service for Service { } impl Service { - pub async fn threads_until<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, include: &'a IncludeThreads, - ) -> Result + Send + 'a> { - self.db - .threads_until(user_id, room_id, until, include) - .await - } - pub async fn add_to_thread(&self, root_event_id: &EventId, pdu: &PduEvent) -> Result<()> { let root_id = self .services @@ -113,13 +115,61 @@ impl Service { } let mut users = Vec::new(); - if let Ok(userids) = self.db.get_participants(&root_id).await { + if let Ok(userids) = self.get_participants(&root_id).await { users.extend_from_slice(&userids); } else { users.push(root_pdu.sender); } users.push(pdu.sender.clone()); - self.db.update_participants(&root_id, &users) + self.update_participants(&root_id, &users) + } + + pub async fn threads_until<'a>( + &'a self, user_id: &'a UserId, room_id: &'a RoomId, shorteventid: PduCount, _inc: &'a IncludeThreads, + ) -> Result + Send + 'a> { + let shortroomid: ShortRoomId = self.services.short.get_shortroomid(room_id).await?; + + let current: RawPduId = PduId { + shortroomid, + shorteventid, + } + .into(); + + let stream = self + .db + .threadid_userids + .rev_raw_keys_from(¤t) + .ignore_err() + .map(RawPduId::from) + .ready_take_while(move |pdu_id| pdu_id.shortroomid() == shortroomid.to_be_bytes()) + .filter_map(move |pdu_id| async move { + let mut pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await.ok()?; + let pdu_id: PduId = pdu_id.into(); + + if pdu.sender != user_id { + pdu.remove_transaction_id().ok(); + } + + Some((pdu_id.shorteventid, pdu)) + }); + + Ok(stream) + } + + pub(super) fn update_participants(&self, root_id: &RawPduId, participants: &[OwnedUserId]) -> Result { + let users = participants + .iter() + .map(|user| user.as_bytes()) + .collect::>() + .join(&[0xFF][..]); + + self.db.threadid_userids.insert(root_id, &users); + + Ok(()) + } + + pub(super) async fn get_participants(&self, root_id: &RawPduId) -> Result> { + self.db.threadid_userids.get(root_id).await.deserialized() } } From 26c890d5ac18adf98109b4663c9eecdc289badef Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 4 Nov 2024 22:38:12 +0000 Subject: [PATCH 1010/2091] skip redundant receipts on syncs Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 08048902..2ac0bfea 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -28,7 +28,7 @@ use ruma::{ events::{ presence::PresenceEvent, room::member::{MembershipState, RoomMemberEventContent}, - AnyRawAccountDataEvent, StateEventType, + AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType::*, }, serde::Raw, @@ -983,20 +983,22 @@ async fn load_joined_room( .collect() .await; - let mut edus: Vec<_> = services + let edus: HashMap> = services .rooms .read_receipt .readreceipts_since(room_id, since) - .filter_map(|(read_user, _, v)| async move { - (!services + .filter_map(|(read_user, _, edu)| async move { + services .users .user_is_ignored(&read_user, sender_user) - .await) - .then_some(v) + .await + .or_some((read_user, edu)) }) .collect() .await; + let mut edus: Vec> = edus.into_values().collect(); + if services.rooms.typing.last_typing_update(room_id).await? > since { edus.push( serde_json::from_str( From 3ed2c17f980497a3ea7bdf2d438b5da7984572fd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 6 Nov 2024 01:24:44 +0000 Subject: [PATCH 1011/2091] move sync watcher from globals service to sync service Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 2 +- src/api/client/sync/v4.rs | 2 +- src/service/globals/data.rs | 140 +----------------------------------- src/service/globals/mod.rs | 6 +- src/service/sync/mod.rs | 51 ++++++++++++- src/service/sync/watch.rs | 117 ++++++++++++++++++++++++++++++ 6 files changed, 170 insertions(+), 148 deletions(-) create mode 100644 src/service/sync/watch.rs diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 2ac0bfea..00976c78 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -93,7 +93,7 @@ pub(crate) async fn sync_events_route( } // Setup watchers, so if there's no response, we can wait for them - let watcher = services.globals.watch(&sender_user, &sender_device); + let watcher = services.sync.watch(&sender_user, &sender_device); let next_batch = services.globals.current_count()?; let next_batchcount = PduCount::Normal(next_batch); diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 11e3830c..91abd24e 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -51,7 +51,7 @@ pub(crate) async fn sync_events_v4_route( let sender_device = body.sender_device.expect("user is authenticated"); let mut body = body.body; // Setup watchers, so if there's no response, we can wait for them - let watcher = services.globals.watch(sender_user, &sender_device); + let watcher = services.sync.watch(sender_user, &sender_device); let next_batch = services.globals.next_count()?; diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index eea7597a..bcfe101e 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -1,35 +1,12 @@ use std::sync::{Arc, RwLock}; -use conduit::{trace, utils, Result, Server}; +use conduit::{utils, Result}; use database::{Database, Deserialized, Map}; -use futures::{pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; -use ruma::{DeviceId, UserId}; - -use crate::{rooms, Dep}; pub struct Data { global: Arc, - todeviceid_events: Arc, - userroomid_joined: Arc, - userroomid_invitestate: Arc, - userroomid_leftstate: Arc, - userroomid_notificationcount: Arc, - userroomid_highlightcount: Arc, - pduid_pdu: Arc, - keychangeid_userid: Arc, - roomusertype_roomuserdataid: Arc, - readreceiptid_readreceipt: Arc, - userid_lastonetimekeyupdate: Arc, counter: RwLock, pub(super) db: Arc, - services: Services, -} - -struct Services { - server: Arc, - short: Dep, - state_cache: Dep, - typing: Dep, } const COUNTER: &[u8] = b"c"; @@ -39,25 +16,8 @@ impl Data { let db = &args.db; Self { global: db["global"].clone(), - todeviceid_events: db["todeviceid_events"].clone(), - userroomid_joined: db["userroomid_joined"].clone(), - userroomid_invitestate: db["userroomid_invitestate"].clone(), - userroomid_leftstate: db["userroomid_leftstate"].clone(), - userroomid_notificationcount: db["userroomid_notificationcount"].clone(), - userroomid_highlightcount: db["userroomid_highlightcount"].clone(), - pduid_pdu: db["pduid_pdu"].clone(), - keychangeid_userid: db["keychangeid_userid"].clone(), - roomusertype_roomuserdataid: db["roomusertype_roomuserdataid"].clone(), - readreceiptid_readreceipt: db["readreceiptid_readreceipt"].clone(), - userid_lastonetimekeyupdate: db["userid_lastonetimekeyupdate"].clone(), counter: RwLock::new(Self::stored_count(&db["global"]).expect("initialized global counter")), db: args.db.clone(), - services: Services { - server: args.server.clone(), - short: args.depend::("rooms::short"), - state_cache: args.depend::("rooms::state_cache"), - typing: args.depend::("rooms::typing"), - }, } } @@ -98,104 +58,6 @@ impl Data { .map_or(Ok(0_u64), utils::u64_from_bytes) } - #[tracing::instrument(skip(self), level = "debug")] - pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { - let userid_bytes = user_id.as_bytes().to_vec(); - let mut userid_prefix = userid_bytes.clone(); - userid_prefix.push(0xFF); - - let mut userdeviceid_prefix = userid_prefix.clone(); - userdeviceid_prefix.extend_from_slice(device_id.as_bytes()); - userdeviceid_prefix.push(0xFF); - - let mut futures = FuturesUnordered::new(); - - // Return when *any* user changed their key - // TODO: only send for user they share a room with - futures.push(self.todeviceid_events.watch_prefix(&userdeviceid_prefix)); - - futures.push(self.userroomid_joined.watch_prefix(&userid_prefix)); - futures.push(self.userroomid_invitestate.watch_prefix(&userid_prefix)); - futures.push(self.userroomid_leftstate.watch_prefix(&userid_prefix)); - futures.push( - self.userroomid_notificationcount - .watch_prefix(&userid_prefix), - ); - futures.push(self.userroomid_highlightcount.watch_prefix(&userid_prefix)); - - // Events for rooms we are in - let rooms_joined = self.services.state_cache.rooms_joined(user_id); - - pin_mut!(rooms_joined); - while let Some(room_id) = rooms_joined.next().await { - let Ok(short_roomid) = self.services.short.get_shortroomid(room_id).await else { - continue; - }; - - let roomid_bytes = room_id.as_bytes().to_vec(); - let mut roomid_prefix = roomid_bytes.clone(); - roomid_prefix.push(0xFF); - - // Key changes - futures.push(self.keychangeid_userid.watch_prefix(&roomid_prefix)); - - // Room account data - let mut roomuser_prefix = roomid_prefix.clone(); - roomuser_prefix.extend_from_slice(&userid_prefix); - - futures.push( - self.roomusertype_roomuserdataid - .watch_prefix(&roomuser_prefix), - ); - - // PDUs - let short_roomid = short_roomid.to_be_bytes().to_vec(); - futures.push(self.pduid_pdu.watch_prefix(&short_roomid)); - - // EDUs - let typing_room_id = room_id.to_owned(); - let typing_wait_for_update = async move { - self.services.typing.wait_for_update(&typing_room_id).await; - }; - - futures.push(typing_wait_for_update.boxed()); - futures.push(self.readreceiptid_readreceipt.watch_prefix(&roomid_prefix)); - } - - let mut globaluserdata_prefix = vec![0xFF]; - globaluserdata_prefix.extend_from_slice(&userid_prefix); - - futures.push( - self.roomusertype_roomuserdataid - .watch_prefix(&globaluserdata_prefix), - ); - - // More key changes (used when user is not joined to any rooms) - futures.push(self.keychangeid_userid.watch_prefix(&userid_prefix)); - - // One time keys - futures.push(self.userid_lastonetimekeyupdate.watch_prefix(&userid_bytes)); - - // Server shutdown - let server_shutdown = async move { - while self.services.server.running() { - self.services.server.signal.subscribe().recv().await.ok(); - } - }; - - futures.push(server_shutdown.boxed()); - if !self.services.server.running() { - return Ok(()); - } - - // Wait until one of them finds something - trace!(futures = futures.len(), "watch started"); - futures.next().await; - trace!(futures = futures.len(), "watch finished"); - - Ok(()) - } - pub async fn database_version(&self) -> u64 { self.global .get(b"version") diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index bd956964..55dd10aa 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -12,7 +12,7 @@ use data::Data; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ - api::client::discovery::discover_support::ContactRole, DeviceId, OwnedEventId, OwnedRoomAliasId, OwnedServerName, + api::client::discovery::discover_support::ContactRole, OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, }; use tokio::sync::Mutex; @@ -163,10 +163,6 @@ impl Service { #[inline] pub fn current_count(&self) -> Result { Ok(self.db.current_count()) } - pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> { - self.db.watch(user_id, device_id).await - } - #[inline] pub fn server_name(&self) -> &ServerName { self.config.server_name.as_ref() } diff --git a/src/service/sync/mod.rs b/src/service/sync/mod.rs index 1bf4610f..f1a6ae75 100644 --- a/src/service/sync/mod.rs +++ b/src/service/sync/mod.rs @@ -1,9 +1,12 @@ +mod watch; + use std::{ collections::{BTreeMap, BTreeSet}, sync::{Arc, Mutex, Mutex as StdMutex}, }; -use conduit::Result; +use conduit::{Result, Server}; +use database::Map; use ruma::{ api::client::sync::sync_events::{ self, @@ -12,10 +15,35 @@ use ruma::{ OwnedDeviceId, OwnedRoomId, OwnedUserId, }; +use crate::{rooms, Dep}; + pub struct Service { + db: Data, + services: Services, connections: DbConnections, } +pub struct Data { + todeviceid_events: Arc, + userroomid_joined: Arc, + userroomid_invitestate: Arc, + userroomid_leftstate: Arc, + userroomid_notificationcount: Arc, + userroomid_highlightcount: Arc, + pduid_pdu: Arc, + keychangeid_userid: Arc, + roomusertype_roomuserdataid: Arc, + readreceiptid_readreceipt: Arc, + userid_lastonetimekeyupdate: Arc, +} + +struct Services { + server: Arc, + short: Dep, + state_cache: Dep, + typing: Dep, +} + struct SlidingSyncCache { lists: BTreeMap, subscriptions: BTreeMap, @@ -28,8 +56,27 @@ type DbConnectionsKey = (OwnedUserId, OwnedDeviceId, String); type DbConnectionsVal = Arc>; impl crate::Service for Service { - fn build(_args: crate::Args<'_>) -> Result> { + fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + db: Data { + todeviceid_events: args.db["todeviceid_events"].clone(), + userroomid_joined: args.db["userroomid_joined"].clone(), + userroomid_invitestate: args.db["userroomid_invitestate"].clone(), + userroomid_leftstate: args.db["userroomid_leftstate"].clone(), + userroomid_notificationcount: args.db["userroomid_notificationcount"].clone(), + userroomid_highlightcount: args.db["userroomid_highlightcount"].clone(), + pduid_pdu: args.db["pduid_pdu"].clone(), + keychangeid_userid: args.db["keychangeid_userid"].clone(), + roomusertype_roomuserdataid: args.db["roomusertype_roomuserdataid"].clone(), + readreceiptid_readreceipt: args.db["readreceiptid_readreceipt"].clone(), + userid_lastonetimekeyupdate: args.db["userid_lastonetimekeyupdate"].clone(), + }, + services: Services { + server: args.server.clone(), + short: args.depend::("rooms::short"), + state_cache: args.depend::("rooms::state_cache"), + typing: args.depend::("rooms::typing"), + }, connections: StdMutex::new(BTreeMap::new()), })) } diff --git a/src/service/sync/watch.rs b/src/service/sync/watch.rs new file mode 100644 index 00000000..3eb663c1 --- /dev/null +++ b/src/service/sync/watch.rs @@ -0,0 +1,117 @@ +use conduit::{implement, trace, Result}; +use futures::{pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; +use ruma::{DeviceId, UserId}; + +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result { + let userid_bytes = user_id.as_bytes().to_vec(); + let mut userid_prefix = userid_bytes.clone(); + userid_prefix.push(0xFF); + + let mut userdeviceid_prefix = userid_prefix.clone(); + userdeviceid_prefix.extend_from_slice(device_id.as_bytes()); + userdeviceid_prefix.push(0xFF); + + let mut futures = FuturesUnordered::new(); + + // Return when *any* user changed their key + // TODO: only send for user they share a room with + futures.push(self.db.todeviceid_events.watch_prefix(&userdeviceid_prefix)); + + futures.push(self.db.userroomid_joined.watch_prefix(&userid_prefix)); + futures.push(self.db.userroomid_invitestate.watch_prefix(&userid_prefix)); + futures.push(self.db.userroomid_leftstate.watch_prefix(&userid_prefix)); + futures.push( + self.db + .userroomid_notificationcount + .watch_prefix(&userid_prefix), + ); + futures.push( + self.db + .userroomid_highlightcount + .watch_prefix(&userid_prefix), + ); + + // Events for rooms we are in + let rooms_joined = self.services.state_cache.rooms_joined(user_id); + + pin_mut!(rooms_joined); + while let Some(room_id) = rooms_joined.next().await { + let Ok(short_roomid) = self.services.short.get_shortroomid(room_id).await else { + continue; + }; + + let roomid_bytes = room_id.as_bytes().to_vec(); + let mut roomid_prefix = roomid_bytes.clone(); + roomid_prefix.push(0xFF); + + // Key changes + futures.push(self.db.keychangeid_userid.watch_prefix(&roomid_prefix)); + + // Room account data + let mut roomuser_prefix = roomid_prefix.clone(); + roomuser_prefix.extend_from_slice(&userid_prefix); + + futures.push( + self.db + .roomusertype_roomuserdataid + .watch_prefix(&roomuser_prefix), + ); + + // PDUs + let short_roomid = short_roomid.to_be_bytes().to_vec(); + futures.push(self.db.pduid_pdu.watch_prefix(&short_roomid)); + + // EDUs + let typing_room_id = room_id.to_owned(); + let typing_wait_for_update = async move { + self.services.typing.wait_for_update(&typing_room_id).await; + }; + + futures.push(typing_wait_for_update.boxed()); + futures.push( + self.db + .readreceiptid_readreceipt + .watch_prefix(&roomid_prefix), + ); + } + + let mut globaluserdata_prefix = vec![0xFF]; + globaluserdata_prefix.extend_from_slice(&userid_prefix); + + futures.push( + self.db + .roomusertype_roomuserdataid + .watch_prefix(&globaluserdata_prefix), + ); + + // More key changes (used when user is not joined to any rooms) + futures.push(self.db.keychangeid_userid.watch_prefix(&userid_prefix)); + + // One time keys + futures.push( + self.db + .userid_lastonetimekeyupdate + .watch_prefix(&userid_bytes), + ); + + // Server shutdown + let server_shutdown = async move { + while self.services.server.running() { + self.services.server.signal.subscribe().recv().await.ok(); + } + }; + + futures.push(server_shutdown.boxed()); + if !self.services.server.running() { + return Ok(()); + } + + // Wait until one of them finds something + trace!(futures = futures.len(), "watch started"); + futures.next().await; + trace!(futures = futures.len(), "watch finished"); + + Ok(()) +} From 7450c654ae37e8caa9af80f40dc674d9a65893b7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 6 Nov 2024 06:20:39 +0000 Subject: [PATCH 1012/2091] add get_pdu_owned sans Arc; improve client/room/event handler Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 3 +- src/api/client/membership.rs | 4 ++- src/api/client/room.rs | 48 +++++++++++++----------------- src/api/mod.rs | 2 ++ src/api/server/send_join.rs | 8 +++-- src/service/rooms/timeline/data.rs | 15 ++++++---- src/service/rooms/timeline/mod.rs | 8 ++++- 7 files changed, 50 insertions(+), 38 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 754c9840..f9d4a521 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,7 +6,7 @@ use std::{ }; use conduit::{debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result}; -use futures::StreamExt; +use futures::{FutureExt, StreamExt}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, events::room::message::RoomMessageEventContent, @@ -246,6 +246,7 @@ pub(super) async fn get_remote_pdu( .rooms .timeline .backfill_pdu(&server, response.pdu) + .boxed() .await?; let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json"); diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index fa71c0c8..bf8e5c33 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -374,7 +374,9 @@ pub(crate) async fn invite_user_route( return Ok(invite_user::v3::Response {}); } - invite_helper(&services, sender_user, user_id, &body.room_id, body.reason.clone(), false).await?; + invite_helper(&services, sender_user, user_id, &body.room_id, body.reason.clone(), false) + .boxed() + .await?; Ok(invite_user::v3::Response {}) } else { Err(Error::BadRequest(ErrorKind::NotFound, "User not found.")) diff --git a/src/api/client/room.rs b/src/api/client/room.rs index 4224d3fa..b6683ef4 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room.rs @@ -2,7 +2,7 @@ use std::{cmp::max, collections::BTreeMap}; use axum::extract::State; use conduit::{debug_info, debug_warn, err, Err}; -use futures::{FutureExt, StreamExt}; +use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{ api::client::{ error::ErrorKind, @@ -486,34 +486,28 @@ pub(crate) async fn create_room_route( /// - You have to currently be joined to the room (TODO: Respect history /// visibility) pub(crate) async fn get_room_event_route( - State(services): State, body: Ruma, + State(services): State, ref body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - - let event = services - .rooms - .timeline - .get_pdu(&body.event_id) - .await - .map_err(|_| err!(Request(NotFound("Event {} not found.", &body.event_id))))?; - - if !services - .rooms - .state_accessor - .user_can_see_event(sender_user, &event.room_id, &body.event_id) - .await - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this event.", - )); - } - - let mut event = (*event).clone(); - event.add_age()?; - Ok(get_room_event::v3::Response { - event: event.to_room_event(), + event: services + .rooms + .timeline + .get_pdu_owned(&body.event_id) + .map_err(|_| err!(Request(NotFound("Event {} not found.", &body.event_id)))) + .and_then(|event| async move { + services + .rooms + .state_accessor + .user_can_see_event(body.sender_user(), &event.room_id, &body.event_id) + .await + .then_some(event) + .ok_or_else(|| err!(Request(Forbidden("You don't have permission to view this event.")))) + }) + .map_ok(|mut event| { + event.add_age().ok(); + event.to_room_event() + }) + .await?, }) } diff --git a/src/api/mod.rs b/src/api/mod.rs index ed8aacf2..fc68af5b 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,3 +1,5 @@ +#![allow(clippy::toplevel_ref_arg)] + pub mod client; pub mod router; pub mod server; diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index f2ede9d0..60ec8c1f 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -253,7 +253,9 @@ pub(crate) async fn create_join_event_v1_route( } } - let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu).await?; + let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu) + .boxed() + .await?; Ok(create_join_event::v1::Response { room_state, @@ -296,7 +298,9 @@ pub(crate) async fn create_join_event_v2_route( auth_chain, state, event, - } = create_join_event(&services, body.origin(), &body.room_id, &body.pdu).await?; + } = create_join_event(&services, body.origin(), &body.room_id, &body.pdu) + .boxed() + .await?; let room_state = create_join_event::v2::RoomState { members_omitted: false, auth_chain, diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 19dc5325..f062e7e4 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -123,15 +123,18 @@ impl Data { /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. pub(super) async fn get_pdu(&self, event_id: &EventId) -> Result> { + self.get_pdu_owned(event_id).await.map(Arc::new) + } + + /// Returns the pdu. + /// + /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. + pub(super) async fn get_pdu_owned(&self, event_id: &EventId) -> Result { if let Ok(pdu) = self.get_non_outlier_pdu(event_id).await { - return Ok(Arc::new(pdu)); + return Ok(pdu); } - self.eventid_outlierpdu - .get(event_id) - .await - .deserialized() - .map(Arc::new) + self.eventid_outlierpdu.get(event_id).await.deserialized() } /// Like get_non_outlier_pdu(), but without the expense of fetching and diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 86a47919..8255be7d 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -244,6 +244,11 @@ impl Service { /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. pub async fn get_pdu(&self, event_id: &EventId) -> Result> { self.db.get_pdu(event_id).await } + /// Returns the pdu. + /// + /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. + pub async fn get_pdu_owned(&self, event_id: &EventId) -> Result { self.db.get_pdu_owned(event_id).await } + /// Checks if pdu exists /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. @@ -885,6 +890,7 @@ impl Service { vec![(*pdu.event_id).to_owned()], state_lock, ) + .boxed() .await?; // We set the room state after inserting the pdu, so that we never have a moment @@ -1104,7 +1110,7 @@ impl Service { match response { Ok(response) => { for pdu in response.pdus { - if let Err(e) = self.backfill_pdu(backfill_server, pdu).await { + if let Err(e) = self.backfill_pdu(backfill_server, pdu).boxed().await { warn!("Failed to add backfilled pdu in room {room_id}: {e}"); } } From f36757027eacc27f47f6415d998be6cf61cc4f0a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 6 Nov 2024 18:27:40 +0000 Subject: [PATCH 1013/2091] split api/client/room Signed-off-by: Jason Volk --- src/api/client/room/aliases.rs | 40 +++ src/api/client/{room.rs => room/create.rs} | 359 +-------------------- src/api/client/room/event.rs | 38 +++ src/api/client/room/mod.rs | 9 + src/api/client/room/upgrade.rs | 294 +++++++++++++++++ 5 files changed, 388 insertions(+), 352 deletions(-) create mode 100644 src/api/client/room/aliases.rs rename src/api/client/{room.rs => room/create.rs} (65%) create mode 100644 src/api/client/room/event.rs create mode 100644 src/api/client/room/mod.rs create mode 100644 src/api/client/room/upgrade.rs diff --git a/src/api/client/room/aliases.rs b/src/api/client/room/aliases.rs new file mode 100644 index 00000000..e530b260 --- /dev/null +++ b/src/api/client/room/aliases.rs @@ -0,0 +1,40 @@ +use axum::extract::State; +use conduit::{Error, Result}; +use futures::StreamExt; +use ruma::api::client::{error::ErrorKind, room::aliases}; + +use crate::Ruma; + +/// # `GET /_matrix/client/r0/rooms/{roomId}/aliases` +/// +/// Lists all aliases of the room. +/// +/// - Only users joined to the room are allowed to call this, or if +/// `history_visibility` is world readable in the room +pub(crate) async fn get_room_aliases_route( + State(services): State, body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if !services + .rooms + .state_accessor + .user_can_see_state_events(sender_user, &body.room_id) + .await + { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "You don't have permission to view this room.", + )); + } + + Ok(aliases::v3::Response { + aliases: services + .rooms + .alias + .local_aliases_for_room(&body.room_id) + .map(ToOwned::to_owned) + .collect() + .await, + }) +} diff --git a/src/api/client/room.rs b/src/api/client/room/create.rs similarity index 65% rename from src/api/client/room.rs rename to src/api/client/room/create.rs index b6683ef4..2ccb1c87 100644 --- a/src/api/client/room.rs +++ b/src/api/client/room/create.rs @@ -1,12 +1,12 @@ -use std::{cmp::max, collections::BTreeMap}; +use std::collections::BTreeMap; use axum::extract::State; -use conduit::{debug_info, debug_warn, err, Err}; -use futures::{FutureExt, StreamExt, TryFutureExt}; +use conduit::{debug_info, debug_warn, error, info, pdu::PduBuilder, warn, Err, Error, Result}; +use futures::FutureExt; use ruma::{ api::client::{ error::ErrorKind, - room::{self, aliases, create_room, get_room_event, upgrade_room}, + room::{self, create_room}, }, events::{ room::{ @@ -18,36 +18,18 @@ use ruma::{ member::{MembershipState, RoomMemberEventContent}, name::RoomNameEventContent, power_levels::RoomPowerLevelsEventContent, - tombstone::RoomTombstoneEventContent, topic::RoomTopicEventContent, }, - StateEventType, TimelineEventType, + TimelineEventType, }, int, serde::{JsonObject, Raw}, CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, RoomVersionId, }; use serde_json::{json, value::to_raw_value}; -use tracing::{error, info, warn}; +use service::{appservice::RegistrationInfo, Services}; -use super::invite_helper; -use crate::{ - service::{appservice::RegistrationInfo, pdu::PduBuilder, Services}, - Error, Result, Ruma, -}; - -/// Recommended transferable state events list from the spec -const TRANSFERABLE_STATE_EVENTS: &[StateEventType; 9] = &[ - StateEventType::RoomServerAcl, - StateEventType::RoomEncryption, - StateEventType::RoomName, - StateEventType::RoomAvatar, - StateEventType::RoomTopic, - StateEventType::RoomGuestAccess, - StateEventType::RoomHistoryVisibility, - StateEventType::RoomJoinRules, - StateEventType::RoomPowerLevels, -]; +use crate::{client::invite_helper, Ruma}; /// # `POST /_matrix/client/v3/createRoom` /// @@ -479,333 +461,6 @@ pub(crate) async fn create_room_route( Ok(create_room::v3::Response::new(room_id)) } -/// # `GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}` -/// -/// Gets a single event. -/// -/// - You have to currently be joined to the room (TODO: Respect history -/// visibility) -pub(crate) async fn get_room_event_route( - State(services): State, ref body: Ruma, -) -> Result { - Ok(get_room_event::v3::Response { - event: services - .rooms - .timeline - .get_pdu_owned(&body.event_id) - .map_err(|_| err!(Request(NotFound("Event {} not found.", &body.event_id)))) - .and_then(|event| async move { - services - .rooms - .state_accessor - .user_can_see_event(body.sender_user(), &event.room_id, &body.event_id) - .await - .then_some(event) - .ok_or_else(|| err!(Request(Forbidden("You don't have permission to view this event.")))) - }) - .map_ok(|mut event| { - event.add_age().ok(); - event.to_room_event() - }) - .await?, - }) -} - -/// # `GET /_matrix/client/r0/rooms/{roomId}/aliases` -/// -/// Lists all aliases of the room. -/// -/// - Only users joined to the room are allowed to call this, or if -/// `history_visibility` is world readable in the room -pub(crate) async fn get_room_aliases_route( - State(services): State, body: Ruma, -) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - - if !services - .rooms - .state_accessor - .user_can_see_state_events(sender_user, &body.room_id) - .await - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this room.", - )); - } - - Ok(aliases::v3::Response { - aliases: services - .rooms - .alias - .local_aliases_for_room(&body.room_id) - .map(ToOwned::to_owned) - .collect() - .await, - }) -} - -/// # `POST /_matrix/client/r0/rooms/{roomId}/upgrade` -/// -/// Upgrades the room. -/// -/// - Creates a replacement room -/// - Sends a tombstone event into the current room -/// - Sender user joins the room -/// - Transfers some state events -/// - Moves local aliases -/// - Modifies old room power levels to prevent users from speaking -pub(crate) async fn upgrade_room_route( - State(services): State, body: Ruma, -) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - - if !services - .globals - .supported_room_versions() - .contains(&body.new_version) - { - return Err(Error::BadRequest( - ErrorKind::UnsupportedRoomVersion, - "This server does not support that room version.", - )); - } - - // Create a replacement room - let replacement_room = RoomId::new(services.globals.server_name()); - - let _short_id = services - .rooms - .short - .get_or_create_shortroomid(&replacement_room) - .await; - - let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - - // Send a m.room.tombstone event to the old room to indicate that it is not - // intended to be used any further Fail if the sender does not have the required - // permissions - let tombstone_event_id = services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomTombstoneEventContent { - body: "This room has been replaced".to_owned(), - replacement_room: replacement_room.clone(), - }, - ), - sender_user, - &body.room_id, - &state_lock, - ) - .await?; - - // Change lock to replacement room - drop(state_lock); - let state_lock = services.rooms.state.mutex.lock(&replacement_room).await; - - // Get the old room creation event - let mut create_event_content: CanonicalJsonObject = services - .rooms - .state_accessor - .room_state_get_content(&body.room_id, &StateEventType::RoomCreate, "") - .await - .map_err(|_| err!(Database("Found room without m.room.create event.")))?; - - // Use the m.room.tombstone event as the predecessor - let predecessor = Some(ruma::events::room::create::PreviousRoom::new( - body.room_id.clone(), - (*tombstone_event_id).to_owned(), - )); - - // Send a m.room.create event containing a predecessor field and the applicable - // room_version - { - use RoomVersionId::*; - match body.new_version { - V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { - create_event_content.insert( - "creator".into(), - json!(&sender_user).try_into().map_err(|e| { - info!("Error forming creation event: {e}"); - Error::BadRequest(ErrorKind::BadJson, "Error forming creation event") - })?, - ); - }, - _ => { - // "creator" key no longer exists in V11+ rooms - create_event_content.remove("creator"); - }, - } - } - - create_event_content.insert( - "room_version".into(), - json!(&body.new_version) - .try_into() - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?, - ); - create_event_content.insert( - "predecessor".into(), - json!(predecessor) - .try_into() - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?, - ); - - // Validate creation event content - if serde_json::from_str::( - to_raw_value(&create_event_content) - .expect("Error forming creation event") - .get(), - ) - .is_err() - { - return Err(Error::BadRequest(ErrorKind::BadJson, "Error forming creation event")); - } - - services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomCreate, - content: to_raw_value(&create_event_content).expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(String::new()), - redacts: None, - timestamp: None, - }, - sender_user, - &replacement_room, - &state_lock, - ) - .await?; - - // Join the new room - services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: TimelineEventType::RoomMember, - content: to_raw_value(&RoomMemberEventContent { - membership: MembershipState::Join, - displayname: services.users.displayname(sender_user).await.ok(), - avatar_url: services.users.avatar_url(sender_user).await.ok(), - is_direct: None, - third_party_invite: None, - blurhash: services.users.blurhash(sender_user).await.ok(), - reason: None, - join_authorized_via_users_server: None, - }) - .expect("event is valid, we just created it"), - unsigned: None, - state_key: Some(sender_user.to_string()), - redacts: None, - timestamp: None, - }, - sender_user, - &replacement_room, - &state_lock, - ) - .await?; - - // Replicate transferable state events to the new room - for event_type in TRANSFERABLE_STATE_EVENTS { - let event_content = match services - .rooms - .state_accessor - .room_state_get(&body.room_id, event_type, "") - .await - { - Ok(v) => v.content.clone(), - Err(_) => continue, // Skipping missing events. - }; - - services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - event_type: event_type.to_string().into(), - content: event_content, - state_key: Some(String::new()), - ..Default::default() - }, - sender_user, - &replacement_room, - &state_lock, - ) - .await?; - } - - // Moves any local aliases to the new room - let mut local_aliases = services - .rooms - .alias - .local_aliases_for_room(&body.room_id) - .boxed(); - - while let Some(alias) = local_aliases.next().await { - services - .rooms - .alias - .remove_alias(alias, sender_user) - .await?; - - services - .rooms - .alias - .set_alias(alias, &replacement_room, sender_user)?; - } - - // Get the old room power levels - let power_levels_event_content: RoomPowerLevelsEventContent = services - .rooms - .state_accessor - .room_state_get_content(&body.room_id, &StateEventType::RoomPowerLevels, "") - .await - .map_err(|_| err!(Database("Found room without m.room.power_levels event.")))?; - - // Setting events_default and invite to the greater of 50 and users_default + 1 - let new_level = max( - int!(50), - power_levels_event_content - .users_default - .checked_add(int!(1)) - .ok_or_else(|| err!(Request(BadJson("users_default power levels event content is not valid"))))?, - ); - - // Modify the power levels in the old room to prevent sending of events and - // inviting new users - services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomPowerLevelsEventContent { - events_default: new_level, - invite: new_level, - ..power_levels_event_content - }, - ), - sender_user, - &body.room_id, - &state_lock, - ) - .await?; - - drop(state_lock); - - // Return the replacement room id - Ok(upgrade_room::v3::Response { - replacement_room, - }) -} - /// creates the power_levels_content for the PDU builder fn default_power_levels_content( power_level_content_override: Option<&Raw>, visibility: &room::Visibility, diff --git a/src/api/client/room/event.rs b/src/api/client/room/event.rs new file mode 100644 index 00000000..0f44f25d --- /dev/null +++ b/src/api/client/room/event.rs @@ -0,0 +1,38 @@ +use axum::extract::State; +use conduit::{err, Result}; +use futures::TryFutureExt; +use ruma::api::client::room::get_room_event; + +use crate::Ruma; + +/// # `GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}` +/// +/// Gets a single event. +/// +/// - You have to currently be joined to the room (TODO: Respect history +/// visibility) +pub(crate) async fn get_room_event_route( + State(services): State, ref body: Ruma, +) -> Result { + Ok(get_room_event::v3::Response { + event: services + .rooms + .timeline + .get_pdu_owned(&body.event_id) + .map_err(|_| err!(Request(NotFound("Event {} not found.", &body.event_id)))) + .and_then(|event| async move { + services + .rooms + .state_accessor + .user_can_see_event(body.sender_user(), &event.room_id, &body.event_id) + .await + .then_some(event) + .ok_or_else(|| err!(Request(Forbidden("You don't have permission to view this event.")))) + }) + .map_ok(|mut event| { + event.add_age().ok(); + event.to_room_event() + }) + .await?, + }) +} diff --git a/src/api/client/room/mod.rs b/src/api/client/room/mod.rs new file mode 100644 index 00000000..fa2d168f --- /dev/null +++ b/src/api/client/room/mod.rs @@ -0,0 +1,9 @@ +mod aliases; +mod create; +mod event; +mod upgrade; + +pub(crate) use self::{ + aliases::get_room_aliases_route, create::create_room_route, event::get_room_event_route, + upgrade::upgrade_room_route, +}; diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs new file mode 100644 index 00000000..ad5c356e --- /dev/null +++ b/src/api/client/room/upgrade.rs @@ -0,0 +1,294 @@ +use std::cmp::max; + +use axum::extract::State; +use conduit::{err, info, pdu::PduBuilder, Error, Result}; +use futures::StreamExt; +use ruma::{ + api::client::{error::ErrorKind, room::upgrade_room}, + events::{ + room::{ + member::{MembershipState, RoomMemberEventContent}, + power_levels::RoomPowerLevelsEventContent, + tombstone::RoomTombstoneEventContent, + }, + StateEventType, TimelineEventType, + }, + int, CanonicalJsonObject, RoomId, RoomVersionId, +}; +use serde_json::{json, value::to_raw_value}; + +use crate::Ruma; + +/// Recommended transferable state events list from the spec +const TRANSFERABLE_STATE_EVENTS: &[StateEventType; 9] = &[ + StateEventType::RoomServerAcl, + StateEventType::RoomEncryption, + StateEventType::RoomName, + StateEventType::RoomAvatar, + StateEventType::RoomTopic, + StateEventType::RoomGuestAccess, + StateEventType::RoomHistoryVisibility, + StateEventType::RoomJoinRules, + StateEventType::RoomPowerLevels, +]; + +/// # `POST /_matrix/client/r0/rooms/{roomId}/upgrade` +/// +/// Upgrades the room. +/// +/// - Creates a replacement room +/// - Sends a tombstone event into the current room +/// - Sender user joins the room +/// - Transfers some state events +/// - Moves local aliases +/// - Modifies old room power levels to prevent users from speaking +pub(crate) async fn upgrade_room_route( + State(services): State, body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if !services + .globals + .supported_room_versions() + .contains(&body.new_version) + { + return Err(Error::BadRequest( + ErrorKind::UnsupportedRoomVersion, + "This server does not support that room version.", + )); + } + + // Create a replacement room + let replacement_room = RoomId::new(services.globals.server_name()); + + let _short_id = services + .rooms + .short + .get_or_create_shortroomid(&replacement_room) + .await; + + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + + // Send a m.room.tombstone event to the old room to indicate that it is not + // intended to be used any further Fail if the sender does not have the required + // permissions + let tombstone_event_id = services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state( + String::new(), + &RoomTombstoneEventContent { + body: "This room has been replaced".to_owned(), + replacement_room: replacement_room.clone(), + }, + ), + sender_user, + &body.room_id, + &state_lock, + ) + .await?; + + // Change lock to replacement room + drop(state_lock); + let state_lock = services.rooms.state.mutex.lock(&replacement_room).await; + + // Get the old room creation event + let mut create_event_content: CanonicalJsonObject = services + .rooms + .state_accessor + .room_state_get_content(&body.room_id, &StateEventType::RoomCreate, "") + .await + .map_err(|_| err!(Database("Found room without m.room.create event.")))?; + + // Use the m.room.tombstone event as the predecessor + let predecessor = Some(ruma::events::room::create::PreviousRoom::new( + body.room_id.clone(), + (*tombstone_event_id).to_owned(), + )); + + // Send a m.room.create event containing a predecessor field and the applicable + // room_version + { + use RoomVersionId::*; + match body.new_version { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + create_event_content.insert( + "creator".into(), + json!(&sender_user).try_into().map_err(|e| { + info!("Error forming creation event: {e}"); + Error::BadRequest(ErrorKind::BadJson, "Error forming creation event") + })?, + ); + }, + _ => { + // "creator" key no longer exists in V11+ rooms + create_event_content.remove("creator"); + }, + } + } + + create_event_content.insert( + "room_version".into(), + json!(&body.new_version) + .try_into() + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?, + ); + create_event_content.insert( + "predecessor".into(), + json!(predecessor) + .try_into() + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Error forming creation event"))?, + ); + + // Validate creation event content + if serde_json::from_str::( + to_raw_value(&create_event_content) + .expect("Error forming creation event") + .get(), + ) + .is_err() + { + return Err(Error::BadRequest(ErrorKind::BadJson, "Error forming creation event")); + } + + services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomCreate, + content: to_raw_value(&create_event_content).expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(String::new()), + redacts: None, + timestamp: None, + }, + sender_user, + &replacement_room, + &state_lock, + ) + .await?; + + // Join the new room + services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: TimelineEventType::RoomMember, + content: to_raw_value(&RoomMemberEventContent { + membership: MembershipState::Join, + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + is_direct: None, + third_party_invite: None, + blurhash: services.users.blurhash(sender_user).await.ok(), + reason: None, + join_authorized_via_users_server: None, + }) + .expect("event is valid, we just created it"), + unsigned: None, + state_key: Some(sender_user.to_string()), + redacts: None, + timestamp: None, + }, + sender_user, + &replacement_room, + &state_lock, + ) + .await?; + + // Replicate transferable state events to the new room + for event_type in TRANSFERABLE_STATE_EVENTS { + let event_content = match services + .rooms + .state_accessor + .room_state_get(&body.room_id, event_type, "") + .await + { + Ok(v) => v.content.clone(), + Err(_) => continue, // Skipping missing events. + }; + + services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { + event_type: event_type.to_string().into(), + content: event_content, + state_key: Some(String::new()), + ..Default::default() + }, + sender_user, + &replacement_room, + &state_lock, + ) + .await?; + } + + // Moves any local aliases to the new room + let mut local_aliases = services + .rooms + .alias + .local_aliases_for_room(&body.room_id) + .boxed(); + + while let Some(alias) = local_aliases.next().await { + services + .rooms + .alias + .remove_alias(alias, sender_user) + .await?; + + services + .rooms + .alias + .set_alias(alias, &replacement_room, sender_user)?; + } + + // Get the old room power levels + let power_levels_event_content: RoomPowerLevelsEventContent = services + .rooms + .state_accessor + .room_state_get_content(&body.room_id, &StateEventType::RoomPowerLevels, "") + .await + .map_err(|_| err!(Database("Found room without m.room.power_levels event.")))?; + + // Setting events_default and invite to the greater of 50 and users_default + 1 + let new_level = max( + int!(50), + power_levels_event_content + .users_default + .checked_add(int!(1)) + .ok_or_else(|| err!(Request(BadJson("users_default power levels event content is not valid"))))?, + ); + + // Modify the power levels in the old room to prevent sending of events and + // inviting new users + services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state( + String::new(), + &RoomPowerLevelsEventContent { + events_default: new_level, + invite: new_level, + ..power_levels_event_content + }, + ), + sender_user, + &body.room_id, + &state_lock, + ) + .await?; + + drop(state_lock); + + // Return the replacement room id + Ok(upgrade_room::v3::Response { + replacement_room, + }) +} From e507c3130673099692143a59adc30a414ef6ca54 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 6 Nov 2024 22:21:51 +0000 Subject: [PATCH 1014/2091] make pdu batch tokens zeroith-indexed Signed-off-by: Jason Volk --- src/api/client/context.rs | 24 +++++++++++++++--------- src/api/client/message.rs | 15 ++++++++++----- src/api/client/relations.rs | 5 +---- src/api/client/sync/mod.rs | 2 +- src/api/client/sync/v3.rs | 15 +++++---------- src/api/server/backfill.rs | 2 +- src/core/pdu/count.rs | 19 +++++++++++++++++++ src/service/rooms/timeline/data.rs | 18 +++++++----------- src/service/rooms/timeline/mod.rs | 19 ++++++++----------- 9 files changed, 67 insertions(+), 52 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 5b492cb1..d07f6ac1 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -2,7 +2,7 @@ use std::iter::once; use axum::extract::State; use conduit::{ - err, error, + at, err, error, utils::{future::TryExtExt, stream::ReadyExt, IterStream}, Err, Result, }; @@ -82,7 +82,7 @@ pub(crate) async fn get_context_route( let events_before: Vec<_> = services .rooms .timeline - .pdus_until(sender_user, room_id, base_token) + .pdus_rev(sender_user, room_id, base_token.saturating_sub(1)) .await? .ready_filter_map(|item| event_filter(item, filter)) .filter_map(|item| ignored_filter(&services, item, sender_user)) @@ -94,7 +94,7 @@ pub(crate) async fn get_context_route( let events_after: Vec<_> = services .rooms .timeline - .pdus_after(sender_user, room_id, base_token) + .pdus(sender_user, room_id, base_token.saturating_add(1)) .await? .ready_filter_map(|item| event_filter(item, filter)) .filter_map(|item| ignored_filter(&services, item, sender_user)) @@ -168,22 +168,28 @@ pub(crate) async fn get_context_route( start: events_before .last() - .map_or_else(|| base_token.to_string(), |(count, _)| count.to_string()) - .into(), + .map(at!(0)) + .map(|count| count.saturating_sub(1)) + .as_ref() + .map(ToString::to_string), end: events_after .last() - .map_or_else(|| base_token.to_string(), |(count, _)| count.to_string()) - .into(), + .map(at!(0)) + .map(|count| count.saturating_add(1)) + .as_ref() + .map(ToString::to_string), events_before: events_before .into_iter() - .map(|(_, pdu)| pdu.to_room_event()) + .map(at!(1)) + .map(|pdu| pdu.to_room_event()) .collect(), events_after: events_after .into_iter() - .map(|(_, pdu)| pdu.to_room_event()) + .map(at!(1)) + .map(|pdu| pdu.to_room_event()) .collect(), state, diff --git a/src/api/client/message.rs b/src/api/client/message.rs index cb261a7f..e76325aa 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -100,14 +100,14 @@ pub(crate) async fn get_message_events_route( Direction::Forward => services .rooms .timeline - .pdus_after(sender_user, room_id, from) + .pdus(sender_user, room_id, from) .await? .boxed(), Direction::Backward => services .rooms .timeline - .pdus_until(sender_user, room_id, from) + .pdus_rev(sender_user, room_id, from) .await? .boxed(), }; @@ -136,7 +136,12 @@ pub(crate) async fn get_message_events_route( .collect() .await; - let next_token = events.last().map(|(count, _)| count).copied(); + let start_token = events.first().map(at!(0)).unwrap_or(from); + + let next_token = events + .last() + .map(at!(0)) + .map(|count| count.saturating_inc(body.dir)); if !cfg!(feature = "element_hacks") { if let Some(next_token) = next_token { @@ -154,8 +159,8 @@ pub(crate) async fn get_message_events_route( .collect(); Ok(get_message_events::v3::Response { - start: from.to_string(), - end: next_token.as_ref().map(PduCount::to_string), + start: start_token.to_string(), + end: next_token.as_ref().map(ToString::to_string), chunk, state, }) diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index b5d1485b..ee62dbfc 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -150,10 +150,7 @@ async fn paginate_relations_with_filter( Direction::Backward => events.first(), } .map(at!(0)) - .map(|count| match dir { - Direction::Forward => count.saturating_add(1), - Direction::Backward => count.saturating_sub(1), - }) + .map(|count| count.saturating_inc(dir)) .as_ref() .map(ToString::to_string); diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 7aec7186..f047d176 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -24,7 +24,7 @@ async fn load_timeline( let mut non_timeline_pdus = services .rooms .timeline - .pdus_until(sender_user, room_id, PduCount::max()) + .pdus_rev(sender_user, room_id, PduCount::max()) .await? .ready_take_while(|(pducount, _)| *pducount > roomsincecount); diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 00976c78..ea487d8e 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -6,7 +6,7 @@ use std::{ use axum::extract::State; use conduit::{ - err, error, extract_variant, is_equal_to, + at, err, error, extract_variant, is_equal_to, result::FlatOk, utils::{math::ruma_from_u64, BoolExt, IterStream, ReadyExt, TryFutureExtExt}, PduCount, @@ -945,15 +945,10 @@ async fn load_joined_room( let prev_batch = timeline_pdus .first() - .map_or(Ok::<_, Error>(None), |(pdu_count, _)| { - Ok(Some(match pdu_count { - PduCount::Backfilled(_) => { - error!("timeline in backfill state?!"); - "0".to_owned() - }, - PduCount::Normal(c) => c.to_string(), - })) - })?; + .map(at!(0)) + .map(|count| count.saturating_sub(1)) + .as_ref() + .map(ToString::to_string); let room_events: Vec<_> = timeline_pdus .iter() diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 281bf2a2..47f02841 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -51,7 +51,7 @@ pub(crate) async fn get_backfill_route( let pdus = services .rooms .timeline - .pdus_until(user_id!("@doesntmatter:conduit.rs"), &body.room_id, until) + .pdus_rev(user_id!("@doesntmatter:conduit.rs"), &body.room_id, until) .await? .take(limit) .filter_map(|(_, pdu)| async move { diff --git a/src/core/pdu/count.rs b/src/core/pdu/count.rs index 90e552e8..aceec1e8 100644 --- a/src/core/pdu/count.rs +++ b/src/core/pdu/count.rs @@ -2,6 +2,8 @@ use std::{cmp::Ordering, fmt, fmt::Display, str::FromStr}; +use ruma::api::Direction; + use crate::{err, Error, Result}; #[derive(Hash, PartialEq, Eq, Clone, Copy, Debug)] @@ -54,6 +56,14 @@ impl PduCount { } } + #[inline] + pub fn checked_inc(self, dir: Direction) -> Result { + match dir { + Direction::Forward => self.checked_add(1), + Direction::Backward => self.checked_sub(1), + } + } + #[inline] pub fn checked_add(self, add: u64) -> Result { Ok(match self { @@ -82,6 +92,15 @@ impl PduCount { }) } + #[inline] + #[must_use] + pub fn saturating_inc(self, dir: Direction) -> Self { + match dir { + Direction::Forward => self.saturating_add(1), + Direction::Backward => self.saturating_sub(1), + } + } + #[inline] #[must_use] pub fn saturating_add(self, add: u64) -> Self { diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index f062e7e4..f320e6a0 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -62,7 +62,7 @@ impl Data { { hash_map::Entry::Occupied(o) => Ok(*o.get()), hash_map::Entry::Vacant(v) => Ok(self - .pdus_until(sender_user, room_id, PduCount::max()) + .pdus_rev(sender_user, room_id, PduCount::max()) .await? .next() .await @@ -201,10 +201,10 @@ impl Data { /// Returns an iterator over all events and their tokens in a room that /// happened before the event with id `until` in reverse-chronological /// order. - pub(super) async fn pdus_until<'a>( + pub(super) async fn pdus_rev<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, ) -> Result + Send + 'a> { - let current = self.count_to_id(room_id, until, true).await?; + let current = self.count_to_id(room_id, until).await?; let prefix = current.shortroomid(); let stream = self .pduid_pdu @@ -216,10 +216,10 @@ impl Data { Ok(stream) } - pub(super) async fn pdus_after<'a>( + pub(super) async fn pdus<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, from: PduCount, ) -> Result + Send + 'a> { - let current = self.count_to_id(room_id, from, false).await?; + let current = self.count_to_id(room_id, from).await?; let prefix = current.shortroomid(); let stream = self .pduid_pdu @@ -266,7 +266,7 @@ impl Data { } } - async fn count_to_id(&self, room_id: &RoomId, count: PduCount, subtract: bool) -> Result { + async fn count_to_id(&self, room_id: &RoomId, shorteventid: PduCount) -> Result { let shortroomid: ShortRoomId = self .services .short @@ -277,11 +277,7 @@ impl Data { // +1 so we don't send the base event let pdu_id = PduId { shortroomid, - shorteventid: if subtract { - count.checked_sub(1)? - } else { - count.checked_add(1)? - }, + shorteventid, }; Ok(pdu_id.into()) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 8255be7d..81d372d7 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -177,7 +177,7 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub async fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result> { - self.pdus_until(user_id!("@placeholder:conduwuit.placeholder"), room_id, PduCount::max()) + self.pdus_rev(user_id!("@placeholder:conduwuit.placeholder"), room_id, PduCount::max()) .await? .next() .await @@ -976,26 +976,23 @@ impl Service { pub async fn all_pdus<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, ) -> Result + Send + 'a> { - self.pdus_after(user_id, room_id, PduCount::min()).await + self.pdus(user_id, room_id, PduCount::min()).await } - /// Returns an iterator over all events and their tokens in a room that - /// happened before the event with id `until` in reverse-chronological - /// order. + /// Reverse iteration starting at from. #[tracing::instrument(skip(self), level = "debug")] - pub async fn pdus_until<'a>( + pub async fn pdus_rev<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, ) -> Result + Send + 'a> { - self.db.pdus_until(user_id, room_id, until).await + self.db.pdus_rev(user_id, room_id, until).await } - /// Returns an iterator over all events and their token in a room that - /// happened after the event with id `from` in chronological order. + /// Forward iteration starting at from. #[tracing::instrument(skip(self), level = "debug")] - pub async fn pdus_after<'a>( + pub async fn pdus<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, from: PduCount, ) -> Result + Send + 'a> { - self.db.pdus_after(user_id, room_id, from).await + self.db.pdus(user_id, room_id, from).await } /// Replace a PDU with the redacted form. From 79c6b518605da12a65a7c0ae3a769931c6eed93b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 7 Nov 2024 03:30:47 +0000 Subject: [PATCH 1015/2091] renames for core pdu Signed-off-by: Jason Volk --- src/core/pdu/content.rs | 4 ++-- src/core/pdu/count.rs | 36 +++++++++++++++++++------------ src/core/pdu/event.rs | 4 ++-- src/core/pdu/filter.rs | 10 ++++----- src/core/pdu/id.rs | 12 +++++------ src/core/pdu/mod.rs | 15 +++++++------ src/core/pdu/raw_id.rs | 46 ++++++++++++++++++---------------------- src/core/pdu/redact.rs | 10 ++++----- src/core/pdu/relation.rs | 2 +- src/core/pdu/strip.rs | 20 ++++++++--------- src/core/pdu/tests.rs | 10 ++++----- src/core/pdu/unsigned.rs | 43 ++++++++++++++++++++++++++++++------- 12 files changed, 123 insertions(+), 89 deletions(-) diff --git a/src/core/pdu/content.rs b/src/core/pdu/content.rs index a6d86554..fa724cb2 100644 --- a/src/core/pdu/content.rs +++ b/src/core/pdu/content.rs @@ -4,13 +4,13 @@ use serde_json::value::Value as JsonValue; use crate::{err, implement, Result}; #[must_use] -#[implement(super::PduEvent)] +#[implement(super::Pdu)] pub fn get_content_as_value(&self) -> JsonValue { self.get_content() .expect("pdu content must be a valid JSON value") } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] pub fn get_content(&self) -> Result where T: for<'de> Deserialize<'de>, diff --git a/src/core/pdu/count.rs b/src/core/pdu/count.rs index aceec1e8..85222382 100644 --- a/src/core/pdu/count.rs +++ b/src/core/pdu/count.rs @@ -7,12 +7,12 @@ use ruma::api::Direction; use crate::{err, Error, Result}; #[derive(Hash, PartialEq, Eq, Clone, Copy, Debug)] -pub enum PduCount { +pub enum Count { Normal(u64), Backfilled(i64), } -impl PduCount { +impl Count { #[inline] #[must_use] pub fn from_unsigned(unsigned: u64) -> Self { Self::from_signed(unsigned as i64) } @@ -69,11 +69,11 @@ impl PduCount { Ok(match self { Self::Normal(i) => Self::Normal( i.checked_add(add) - .ok_or_else(|| err!(Arithmetic("PduCount::Normal overflow")))?, + .ok_or_else(|| err!(Arithmetic("Count::Normal overflow")))?, ), Self::Backfilled(i) => Self::Backfilled( i.checked_add(add as i64) - .ok_or_else(|| err!(Arithmetic("PduCount::Backfilled overflow")))?, + .ok_or_else(|| err!(Arithmetic("Count::Backfilled overflow")))?, ), }) } @@ -83,11 +83,11 @@ impl PduCount { Ok(match self { Self::Normal(i) => Self::Normal( i.checked_sub(sub) - .ok_or_else(|| err!(Arithmetic("PduCount::Normal underflow")))?, + .ok_or_else(|| err!(Arithmetic("Count::Normal underflow")))?, ), Self::Backfilled(i) => Self::Backfilled( i.checked_sub(sub as i64) - .ok_or_else(|| err!(Arithmetic("PduCount::Backfilled underflow")))?, + .ok_or_else(|| err!(Arithmetic("Count::Backfilled underflow")))?, ), }) } @@ -121,11 +121,11 @@ impl PduCount { #[inline] #[must_use] - pub fn min() -> Self { Self::Backfilled(i64::MIN) } + pub const fn min() -> Self { Self::Backfilled(i64::MIN) } #[inline] #[must_use] - pub fn max() -> Self { Self::Normal(i64::MAX as u64) } + pub const fn max() -> Self { Self::Normal(i64::MAX as u64) } #[inline] pub(crate) fn debug_assert_valid(&self) { @@ -135,7 +135,7 @@ impl PduCount { } } -impl Display for PduCount { +impl Display for Count { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { self.debug_assert_valid(); match self { @@ -145,20 +145,30 @@ impl Display for PduCount { } } -impl FromStr for PduCount { +impl From for Count { + #[inline] + fn from(signed: i64) -> Self { Self::from_signed(signed) } +} + +impl From for Count { + #[inline] + fn from(unsigned: u64) -> Self { Self::from_unsigned(unsigned) } +} + +impl FromStr for Count { type Err = Error; fn from_str(token: &str) -> Result { Ok(Self::from_signed(token.parse()?)) } } -impl PartialOrd for PduCount { +impl PartialOrd for Count { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl Ord for PduCount { +impl Ord for Count { fn cmp(&self, other: &Self) -> Ordering { self.into_signed().cmp(&other.into_signed()) } } -impl Default for PduCount { +impl Default for Count { fn default() -> Self { Self::Normal(0) } } diff --git a/src/core/pdu/event.rs b/src/core/pdu/event.rs index 15117f92..96a1e4ba 100644 --- a/src/core/pdu/event.rs +++ b/src/core/pdu/event.rs @@ -4,9 +4,9 @@ pub use ruma::state_res::Event; use ruma::{events::TimelineEventType, EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId}; use serde_json::value::RawValue as RawJsonValue; -use super::PduEvent; +use super::Pdu; -impl Event for PduEvent { +impl Event for Pdu { type Id = Arc; fn event_id(&self) -> &Self::Id { &self.event_id } diff --git a/src/core/pdu/filter.rs b/src/core/pdu/filter.rs index bd232ebd..c7c7316d 100644 --- a/src/core/pdu/filter.rs +++ b/src/core/pdu/filter.rs @@ -3,7 +3,7 @@ use serde_json::Value; use crate::{implement, is_equal_to}; -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[must_use] pub fn matches(&self, filter: &RoomEventFilter) -> bool { if !self.matches_sender(filter) { @@ -25,7 +25,7 @@ pub fn matches(&self, filter: &RoomEventFilter) -> bool { true } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] fn matches_room(&self, filter: &RoomEventFilter) -> bool { if filter.not_rooms.contains(&self.room_id) { return false; @@ -40,7 +40,7 @@ fn matches_room(&self, filter: &RoomEventFilter) -> bool { true } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] fn matches_sender(&self, filter: &RoomEventFilter) -> bool { if filter.not_senders.contains(&self.sender) { return false; @@ -55,7 +55,7 @@ fn matches_sender(&self, filter: &RoomEventFilter) -> bool { true } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] fn matches_type(&self, filter: &RoomEventFilter) -> bool { let event_type = &self.kind.to_cow_str(); if filter.not_types.iter().any(is_equal_to!(event_type)) { @@ -71,7 +71,7 @@ fn matches_type(&self, filter: &RoomEventFilter) -> bool { true } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] fn matches_url(&self, filter: &RoomEventFilter) -> bool { let Some(url_filter) = filter.url_filter.as_ref() else { return true; diff --git a/src/core/pdu/id.rs b/src/core/pdu/id.rs index 05d11904..0b23a29f 100644 --- a/src/core/pdu/id.rs +++ b/src/core/pdu/id.rs @@ -1,4 +1,4 @@ -use super::{PduCount, RawPduId}; +use super::{Count, RawId}; use crate::utils::u64_from_u8x8; pub type ShortRoomId = ShortId; @@ -6,17 +6,17 @@ pub type ShortEventId = ShortId; pub type ShortId = u64; #[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub struct PduId { +pub struct Id { pub shortroomid: ShortRoomId, - pub shorteventid: PduCount, + pub shorteventid: Count, } -impl From for PduId { +impl From for Id { #[inline] - fn from(raw: RawPduId) -> Self { + fn from(raw: RawId) -> Self { Self { shortroomid: u64_from_u8x8(raw.shortroomid()), - shorteventid: PduCount::from_unsigned(u64_from_u8x8(raw.shorteventid())), + shorteventid: Count::from_unsigned(u64_from_u8x8(raw.shorteventid())), } } } diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index c785c99e..2aa60ed1 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -22,17 +22,18 @@ use serde_json::value::RawValue as RawJsonValue; pub use self::{ builder::{Builder, Builder as PduBuilder}, - count::PduCount, + count::Count, event::Event, event_id::*, id::*, raw_id::*, + Count as PduCount, Id as PduId, Pdu as PduEvent, RawId as RawPduId, }; use crate::Result; /// Persistent Data Unit (Event) #[derive(Clone, Deserialize, Serialize, Debug)] -pub struct PduEvent { +pub struct Pdu { pub event_id: Arc, pub room_id: OwnedRoomId, pub sender: OwnedUserId, @@ -64,7 +65,7 @@ pub struct EventHash { pub sha256: String, } -impl PduEvent { +impl Pdu { pub fn from_id_val(event_id: &EventId, mut json: CanonicalJsonObject) -> Result { let event_id = CanonicalJsonValue::String(event_id.into()); json.insert("event_id".into(), event_id); @@ -75,19 +76,19 @@ impl PduEvent { } /// Prevent derived equality which wouldn't limit itself to event_id -impl Eq for PduEvent {} +impl Eq for Pdu {} /// Equality determined by the Pdu's ID, not the memory representations. -impl PartialEq for PduEvent { +impl PartialEq for Pdu { fn eq(&self, other: &Self) -> bool { self.event_id == other.event_id } } /// Ordering determined by the Pdu's ID, not the memory representations. -impl PartialOrd for PduEvent { +impl PartialOrd for Pdu { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } /// Ordering determined by the Pdu's ID, not the memory representations. -impl Ord for PduEvent { +impl Ord for Pdu { fn cmp(&self, other: &Self) -> Ordering { self.event_id.cmp(&other.event_id) } } diff --git a/src/core/pdu/raw_id.rs b/src/core/pdu/raw_id.rs index faba1cbf..ef8502f6 100644 --- a/src/core/pdu/raw_id.rs +++ b/src/core/pdu/raw_id.rs @@ -1,27 +1,27 @@ use arrayvec::ArrayVec; -use super::{PduCount, PduId, ShortEventId, ShortId, ShortRoomId}; +use super::{Count, Id, ShortEventId, ShortId, ShortRoomId}; #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -pub enum RawPduId { - Normal(RawPduIdNormal), - Backfilled(RawPduIdBackfilled), +pub enum RawId { + Normal(RawIdNormal), + Backfilled(RawIdBackfilled), } -type RawPduIdNormal = [u8; RawPduId::NORMAL_LEN]; -type RawPduIdBackfilled = [u8; RawPduId::BACKFILLED_LEN]; +type RawIdNormal = [u8; RawId::NORMAL_LEN]; +type RawIdBackfilled = [u8; RawId::BACKFILLED_LEN]; const INT_LEN: usize = size_of::(); -impl RawPduId { +impl RawId { const BACKFILLED_LEN: usize = size_of::() + INT_LEN + size_of::(); const MAX_LEN: usize = Self::BACKFILLED_LEN; const NORMAL_LEN: usize = size_of::() + size_of::(); #[inline] #[must_use] - pub fn pdu_count(&self) -> PduCount { - let id: PduId = (*self).into(); + pub fn pdu_count(&self) -> Count { + let id: Id = (*self).into(); id.shorteventid } @@ -61,55 +61,51 @@ impl RawPduId { } } -impl AsRef<[u8]> for RawPduId { +impl AsRef<[u8]> for RawId { #[inline] fn as_ref(&self) -> &[u8] { self.as_bytes() } } -impl From<&[u8]> for RawPduId { +impl From<&[u8]> for RawId { #[inline] fn from(id: &[u8]) -> Self { match id.len() { Self::NORMAL_LEN => Self::Normal( id[0..Self::NORMAL_LEN] .try_into() - .expect("normal RawPduId from [u8]"), + .expect("normal RawId from [u8]"), ), Self::BACKFILLED_LEN => Self::Backfilled( id[0..Self::BACKFILLED_LEN] .try_into() - .expect("backfilled RawPduId from [u8]"), + .expect("backfilled RawId from [u8]"), ), - _ => unimplemented!("unrecognized RawPduId length"), + _ => unimplemented!("unrecognized RawId length"), } } } -impl From for RawPduId { +impl From for RawId { #[inline] - fn from(id: PduId) -> Self { - const MAX_LEN: usize = RawPduId::MAX_LEN; + fn from(id: Id) -> Self { + const MAX_LEN: usize = RawId::MAX_LEN; type RawVec = ArrayVec; let mut vec = RawVec::new(); vec.extend(id.shortroomid.to_be_bytes()); id.shorteventid.debug_assert_valid(); match id.shorteventid { - PduCount::Normal(shorteventid) => { + Count::Normal(shorteventid) => { vec.extend(shorteventid.to_be_bytes()); - Self::Normal( - vec.as_ref() - .try_into() - .expect("RawVec into RawPduId::Normal"), - ) + Self::Normal(vec.as_ref().try_into().expect("RawVec into RawId::Normal")) }, - PduCount::Backfilled(shorteventid) => { + Count::Backfilled(shorteventid) => { vec.extend(0_u64.to_be_bytes()); vec.extend(shorteventid.to_be_bytes()); Self::Backfilled( vec.as_ref() .try_into() - .expect("RawVec into RawPduId::Backfilled"), + .expect("RawVec into RawId::Backfilled"), ) }, } diff --git a/src/core/pdu/redact.rs b/src/core/pdu/redact.rs index 647f54c0..e116e563 100644 --- a/src/core/pdu/redact.rs +++ b/src/core/pdu/redact.rs @@ -18,9 +18,9 @@ struct ExtractRedactedBecause { redacted_because: Option, } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[tracing::instrument(skip(self), level = "debug")] -pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> Result<()> { +pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> Result { self.unsigned = None; let mut content = @@ -31,7 +31,7 @@ pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> Resul self.unsigned = Some( to_raw_value(&json!({ - "redacted_because": serde_json::to_value(reason).expect("to_value(PduEvent) always works") + "redacted_because": serde_json::to_value(reason).expect("to_value(Pdu) always works") })) .expect("to string always works"), ); @@ -41,7 +41,7 @@ pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> Resul Ok(()) } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[must_use] pub fn is_redacted(&self) -> bool { let Some(unsigned) = &self.unsigned else { @@ -72,7 +72,7 @@ pub fn is_redacted(&self) -> bool { /// > to the content of m.room.redaction events in older room versions when /// > serving /// > such events over the Client-Server API. -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[must_use] pub fn copy_redacts(&self) -> (Option>, Box) { if self.kind == TimelineEventType::RoomRedaction { diff --git a/src/core/pdu/relation.rs b/src/core/pdu/relation.rs index ae156a3d..2968171e 100644 --- a/src/core/pdu/relation.rs +++ b/src/core/pdu/relation.rs @@ -13,7 +13,7 @@ struct ExtractRelatesToEventId { relates_to: ExtractRelType, } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[must_use] pub fn relation_type_equal(&self, rel_type: &RelationType) -> bool { self.get_content() diff --git a/src/core/pdu/strip.rs b/src/core/pdu/strip.rs index 8d20d982..30fee863 100644 --- a/src/core/pdu/strip.rs +++ b/src/core/pdu/strip.rs @@ -10,7 +10,7 @@ use serde_json::{json, value::Value as JsonValue}; use crate::{implement, warn}; -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[tracing::instrument(skip(self), level = "debug")] pub fn to_sync_room_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); @@ -36,7 +36,7 @@ pub fn to_sync_room_event(&self) -> Raw { } /// This only works for events that are also AnyRoomEvents. -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[tracing::instrument(skip(self), level = "debug")] pub fn to_any_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); @@ -62,7 +62,7 @@ pub fn to_any_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[tracing::instrument(skip(self), level = "debug")] pub fn to_room_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); @@ -88,7 +88,7 @@ pub fn to_room_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[tracing::instrument(skip(self), level = "debug")] pub fn to_message_like_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); @@ -114,7 +114,7 @@ pub fn to_message_like_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[must_use] pub fn to_state_event_value(&self) -> JsonValue { let mut json = json!({ @@ -134,13 +134,13 @@ pub fn to_state_event_value(&self) -> JsonValue { json } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[tracing::instrument(skip(self), level = "debug")] pub fn to_state_event(&self) -> Raw { serde_json::from_value(self.to_state_event_value()).expect("Raw::from_value always works") } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[tracing::instrument(skip(self), level = "debug")] pub fn to_sync_state_event(&self) -> Raw { let mut json = json!({ @@ -159,7 +159,7 @@ pub fn to_sync_state_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[tracing::instrument(skip(self), level = "debug")] pub fn to_stripped_state_event(&self) -> Raw { let json = json!({ @@ -172,7 +172,7 @@ pub fn to_stripped_state_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } -#[implement(super::PduEvent)] +#[implement(super::Pdu)] #[tracing::instrument(skip(self), level = "debug")] pub fn to_stripped_spacechild_state_event(&self) -> Raw { let json = json!({ @@ -186,7 +186,7 @@ pub fn to_stripped_spacechild_state_event(&self) -> Raw Raw> { let mut json = json!({ diff --git a/src/core/pdu/tests.rs b/src/core/pdu/tests.rs index 30ec23ba..ae3b1dd6 100644 --- a/src/core/pdu/tests.rs +++ b/src/core/pdu/tests.rs @@ -1,19 +1,19 @@ #![cfg(test)] -use super::PduCount; +use super::Count; #[test] fn backfilled_parse() { - let count: PduCount = "-987654".parse().expect("parse() failed"); - let backfilled = matches!(count, PduCount::Backfilled(_)); + let count: Count = "-987654".parse().expect("parse() failed"); + let backfilled = matches!(count, Count::Backfilled(_)); assert!(backfilled, "not backfilled variant"); } #[test] fn normal_parse() { - let count: PduCount = "987654".parse().expect("parse() failed"); - let backfilled = matches!(count, PduCount::Backfilled(_)); + let count: Count = "987654".parse().expect("parse() failed"); + let backfilled = matches!(count, Count::Backfilled(_)); assert!(!backfilled, "backfilled variant"); } diff --git a/src/core/pdu/unsigned.rs b/src/core/pdu/unsigned.rs index 1c47e826..6f3e4401 100644 --- a/src/core/pdu/unsigned.rs +++ b/src/core/pdu/unsigned.rs @@ -4,10 +4,11 @@ use ruma::MilliSecondsSinceUnixEpoch; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue, Value as JsonValue}; +use super::Pdu; use crate::{err, implement, is_true, Result}; -#[implement(super::PduEvent)] -pub fn remove_transaction_id(&mut self) -> Result<()> { +#[implement(Pdu)] +pub fn remove_transaction_id(&mut self) -> Result { let Some(unsigned) = &self.unsigned else { return Ok(()); }; @@ -23,8 +24,8 @@ pub fn remove_transaction_id(&mut self) -> Result<()> { Ok(()) } -#[implement(super::PduEvent)] -pub fn add_age(&mut self) -> Result<()> { +#[implement(Pdu)] +pub fn add_age(&mut self) -> Result { let mut unsigned: BTreeMap> = self .unsigned .as_ref() @@ -44,7 +45,33 @@ pub fn add_age(&mut self) -> Result<()> { Ok(()) } -#[implement(super::PduEvent)] +#[implement(Pdu)] +pub fn add_relation(&mut self, name: &str, pdu: &Pdu) -> Result { + let mut unsigned: BTreeMap = self + .unsigned + .as_ref() + .map_or_else(|| Ok(BTreeMap::new()), |u| serde_json::from_str(u.get())) + .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; + + let relations: &mut JsonValue = unsigned.entry("m.relations".into()).or_default(); + if relations.as_object_mut().is_none() { + let mut object = serde_json::Map::::new(); + _ = relations.as_object_mut().insert(&mut object); + } + + relations + .as_object_mut() + .expect("we just created it") + .insert(name.to_owned(), serde_json::to_value(pdu)?); + + self.unsigned = to_raw_value(&unsigned) + .map(Some) + .expect("unsigned is valid"); + + Ok(()) +} + +#[implement(Pdu)] pub fn contains_unsigned_property(&self, property: &str, is_type: F) -> bool where F: FnOnce(&JsonValue) -> bool, @@ -55,7 +82,7 @@ where .is_some_and(is_true!()) } -#[implement(super::PduEvent)] +#[implement(Pdu)] pub fn get_unsigned_property(&self, property: &str) -> Result where T: for<'de> Deserialize<'de>, @@ -68,11 +95,11 @@ where .map_err(|e| err!(Database("Failed to deserialize unsigned.{property} into type: {e}"))) } -#[implement(super::PduEvent)] +#[implement(Pdu)] #[must_use] pub fn get_unsigned_as_value(&self) -> JsonValue { self.get_unsigned::().unwrap_or_default() } -#[implement(super::PduEvent)] +#[implement(Pdu)] pub fn get_unsigned(&self) -> Result { self.unsigned .as_ref() From 27966221f106ef2c4c4e88cc9381d9f1e2d0468e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 5 Nov 2024 04:37:08 +0000 Subject: [PATCH 1016/2091] add ready_try_fold to utils Signed-off-by: Jason Volk --- src/core/utils/result/inspect_log.rs | 2 ++ src/core/utils/stream/ready.rs | 15 ++++++++++++ src/core/utils/stream/try_ready.rs | 36 +++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/core/utils/result/inspect_log.rs b/src/core/utils/result/inspect_log.rs index 577761c5..e9f32663 100644 --- a/src/core/utils/result/inspect_log.rs +++ b/src/core/utils/result/inspect_log.rs @@ -11,6 +11,7 @@ where { fn log_err(self, level: Level) -> Self; + #[inline] fn err_log(self) -> Self where Self: Sized, @@ -25,6 +26,7 @@ where { fn log_err_debug(self, level: Level) -> Self; + #[inline] fn err_debug_log(self) -> Self where Self: Sized, diff --git a/src/core/utils/stream/ready.rs b/src/core/utils/stream/ready.rs index c16d1246..f4eec7d1 100644 --- a/src/core/utils/stream/ready.rs +++ b/src/core/utils/stream/ready.rs @@ -32,6 +32,11 @@ where where F: Fn(T, Item) -> T; + fn ready_fold_default(self, f: F) -> Fold, T, impl FnMut(T, Item) -> Ready> + where + F: Fn(T, Item) -> T, + T: Default; + fn ready_for_each(self, f: F) -> ForEach, impl FnMut(Item) -> Ready<()>> where F: FnMut(Item); @@ -93,6 +98,15 @@ where self.fold(init, move |a, t| ready(f(a, t))) } + #[inline] + fn ready_fold_default(self, f: F) -> Fold, T, impl FnMut(T, Item) -> Ready> + where + F: Fn(T, Item) -> T, + T: Default, + { + self.ready_fold(T::default(), f) + } + #[inline] #[allow(clippy::unit_arg)] fn ready_for_each(self, mut f: F) -> ForEach, impl FnMut(Item) -> Ready<()>> @@ -120,6 +134,7 @@ where self.scan(init, move |s, t| ready(f(s, t))) } + #[inline] fn ready_scan_each( self, init: T, f: F, ) -> Scan>, impl FnMut(&mut T, Item) -> Ready>> diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs index 3fbcbc45..0daed26e 100644 --- a/src/core/utils/stream/try_ready.rs +++ b/src/core/utils/stream/try_ready.rs @@ -3,7 +3,7 @@ use futures::{ future::{ready, Ready}, - stream::{AndThen, TryForEach, TryStream, TryStreamExt}, + stream::{AndThen, TryFold, TryForEach, TryStream, TryStreamExt}, }; use crate::Result; @@ -25,6 +25,19 @@ where ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> where F: FnMut(S::Ok) -> Result<(), E>; + + fn ready_try_fold( + self, init: U, f: F, + ) -> TryFold>, U, impl FnMut(U, S::Ok) -> Ready>> + where + F: Fn(U, S::Ok) -> Result; + + fn ready_try_fold_default( + self, f: F, + ) -> TryFold>, U, impl FnMut(U, S::Ok) -> Ready>> + where + F: Fn(U, S::Ok) -> Result, + U: Default; } impl TryReadyExt for S @@ -49,4 +62,25 @@ where { self.try_for_each(move |t| ready(f(t))) } + + #[inline] + fn ready_try_fold( + self, init: U, f: F, + ) -> TryFold>, U, impl FnMut(U, S::Ok) -> Ready>> + where + F: Fn(U, S::Ok) -> Result, + { + self.try_fold(init, move |a, t| ready(f(a, t))) + } + + #[inline] + fn ready_try_fold_default( + self, f: F, + ) -> TryFold>, U, impl FnMut(U, S::Ok) -> Ready>> + where + F: Fn(U, S::Ok) -> Result, + U: Default, + { + self.ready_try_fold(U::default(), f) + } } From 13ef6dcbcf17e04f28ad8beaab64920e63c2aa31 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 7 Nov 2024 03:59:08 +0000 Subject: [PATCH 1017/2091] add standalone getters for shortid service Signed-off-by: Jason Volk --- src/service/rooms/short/mod.rs | 35 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 9fddf099..e8b00d9b 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -52,13 +52,7 @@ impl crate::Service for Service { pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEventId { const BUFSIZE: usize = size_of::(); - if let Ok(shorteventid) = self - .db - .eventid_shorteventid - .get(event_id) - .await - .deserialized() - { + if let Ok(shorteventid) = self.get_shorteventid(event_id).await { return shorteventid; } @@ -105,11 +99,10 @@ pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> } #[implement(Service)] -pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { - let key = (event_type, state_key); +pub async fn get_shorteventid(&self, event_id: &EventId) -> Result { self.db - .statekey_shortstatekey - .qry(&key) + .eventid_shorteventid + .get(event_id) .await .deserialized() } @@ -118,17 +111,11 @@ pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &s pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> ShortStateKey { const BUFSIZE: usize = size_of::(); - let key = (event_type, state_key); - if let Ok(shortstatekey) = self - .db - .statekey_shortstatekey - .qry(&key) - .await - .deserialized() - { + if let Ok(shortstatekey) = self.get_shortstatekey(event_type, state_key).await { return shortstatekey; } + let key = (event_type, state_key); let shortstatekey = self.services.globals.next_count().unwrap(); debug_assert!(size_of_val(&shortstatekey) == BUFSIZE, "buffer requirement changed"); @@ -143,6 +130,16 @@ pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, sta shortstatekey } +#[implement(Service)] +pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { + let key = (event_type, state_key); + self.db + .statekey_shortstatekey + .qry(&key) + .await + .deserialized() +} + #[implement(Service)] pub async fn get_eventid_from_short(&self, shorteventid: ShortEventId) -> Result> { const BUFSIZE: usize = size_of::(); From 1f2e939fd56319b85426457a9eb469228e287406 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 7 Nov 2024 04:49:01 +0000 Subject: [PATCH 1018/2091] optional arguments for timeline pdus iterations Signed-off-by: Jason Volk --- src/api/client/context.rs | 4 ++-- src/api/client/message.rs | 4 ++-- src/api/client/sync/mod.rs | 4 ++-- src/api/server/backfill.rs | 4 ++-- src/service/rooms/timeline/data.rs | 13 +++++++------ src/service/rooms/timeline/mod.rs | 18 +++++++++++------- 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index d07f6ac1..f5f981ba 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -82,7 +82,7 @@ pub(crate) async fn get_context_route( let events_before: Vec<_> = services .rooms .timeline - .pdus_rev(sender_user, room_id, base_token.saturating_sub(1)) + .pdus_rev(Some(sender_user), room_id, Some(base_token.saturating_sub(1))) .await? .ready_filter_map(|item| event_filter(item, filter)) .filter_map(|item| ignored_filter(&services, item, sender_user)) @@ -94,7 +94,7 @@ pub(crate) async fn get_context_route( let events_after: Vec<_> = services .rooms .timeline - .pdus(sender_user, room_id, base_token.saturating_add(1)) + .pdus(Some(sender_user), room_id, Some(base_token.saturating_add(1))) .await? .ready_filter_map(|item| event_filter(item, filter)) .filter_map(|item| ignored_filter(&services, item, sender_user)) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index e76325aa..e8306de9 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -100,14 +100,14 @@ pub(crate) async fn get_message_events_route( Direction::Forward => services .rooms .timeline - .pdus(sender_user, room_id, from) + .pdus(Some(sender_user), room_id, Some(from)) .await? .boxed(), Direction::Backward => services .rooms .timeline - .pdus_rev(sender_user, room_id, from) + .pdus_rev(Some(sender_user), room_id, Some(from)) .await? .boxed(), }; diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index f047d176..3201b827 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -14,7 +14,7 @@ async fn load_timeline( let last_timeline_count = services .rooms .timeline - .last_timeline_count(sender_user, room_id) + .last_timeline_count(Some(sender_user), room_id) .await?; if last_timeline_count <= roomsincecount { @@ -24,7 +24,7 @@ async fn load_timeline( let mut non_timeline_pdus = services .rooms .timeline - .pdus_rev(sender_user, room_id, PduCount::max()) + .pdus_rev(Some(sender_user), room_id, None) .await? .ready_take_while(|(pducount, _)| *pducount > roomsincecount); diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 47f02841..be770ee8 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -6,7 +6,7 @@ use conduit::{ PduCount, Result, }; use futures::{FutureExt, StreamExt}; -use ruma::{api::federation::backfill::get_backfill, uint, user_id, MilliSecondsSinceUnixEpoch}; +use ruma::{api::federation::backfill::get_backfill, uint, MilliSecondsSinceUnixEpoch}; use super::AccessCheck; use crate::Ruma; @@ -51,7 +51,7 @@ pub(crate) async fn get_backfill_route( let pdus = services .rooms .timeline - .pdus_rev(user_id!("@doesntmatter:conduit.rs"), &body.room_id, until) + .pdus_rev(None, &body.room_id, Some(until)) .await? .take(limit) .filter_map(|(_, pdu)| async move { diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index f320e6a0..7f1873ab 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Borrow, collections::{hash_map, HashMap}, sync::Arc, }; @@ -53,7 +54,7 @@ impl Data { } } - pub(super) async fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { + pub(super) async fn last_timeline_count(&self, sender_user: Option<&UserId>, room_id: &RoomId) -> Result { match self .lasttimelinecount_cache .lock() @@ -202,7 +203,7 @@ impl Data { /// happened before the event with id `until` in reverse-chronological /// order. pub(super) async fn pdus_rev<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, + &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, until: PduCount, ) -> Result + Send + 'a> { let current = self.count_to_id(room_id, until).await?; let prefix = current.shortroomid(); @@ -211,13 +212,13 @@ impl Data { .rev_raw_stream_from(¤t) .ignore_err() .ready_take_while(move |(key, _)| key.starts_with(&prefix)) - .map(|item| Self::each_pdu(item, user_id)); + .map(move |item| Self::each_pdu(item, user_id)); Ok(stream) } pub(super) async fn pdus<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, from: PduCount, + &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, from: PduCount, ) -> Result + Send + 'a> { let current = self.count_to_id(room_id, from).await?; let prefix = current.shortroomid(); @@ -231,13 +232,13 @@ impl Data { Ok(stream) } - fn each_pdu((pdu_id, pdu): KeyVal<'_>, user_id: &UserId) -> PdusIterItem { + fn each_pdu((pdu_id, pdu): KeyVal<'_>, user_id: Option<&UserId>) -> PdusIterItem { let pdu_id: RawPduId = pdu_id.into(); let mut pdu = serde_json::from_slice::(pdu).expect("PduEvent in pduid_pdu database column is invalid JSON"); - if pdu.sender != user_id { + if Some(pdu.sender.borrow()) != user_id { pdu.remove_transaction_id().log_err().ok(); } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 81d372d7..281879d2 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -177,7 +177,7 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub async fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result> { - self.pdus_rev(user_id!("@placeholder:conduwuit.placeholder"), room_id, PduCount::max()) + self.pdus_rev(None, room_id, None) .await? .next() .await @@ -186,7 +186,7 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub async fn last_timeline_count(&self, sender_user: &UserId, room_id: &RoomId) -> Result { + pub async fn last_timeline_count(&self, sender_user: Option<&UserId>, room_id: &RoomId) -> Result { self.db.last_timeline_count(sender_user, room_id).await } @@ -976,23 +976,27 @@ impl Service { pub async fn all_pdus<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, ) -> Result + Send + 'a> { - self.pdus(user_id, room_id, PduCount::min()).await + self.pdus(Some(user_id), room_id, None).await } /// Reverse iteration starting at from. #[tracing::instrument(skip(self), level = "debug")] pub async fn pdus_rev<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, until: PduCount, + &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, until: Option, ) -> Result + Send + 'a> { - self.db.pdus_rev(user_id, room_id, until).await + self.db + .pdus_rev(user_id, room_id, until.unwrap_or_else(PduCount::max)) + .await } /// Forward iteration starting at from. #[tracing::instrument(skip(self), level = "debug")] pub async fn pdus<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, from: PduCount, + &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, from: Option, ) -> Result + Send + 'a> { - self.db.pdus(user_id, room_id, from).await + self.db + .pdus(user_id, room_id, from.unwrap_or_else(PduCount::min)) + .await } /// Replace a PDU with the redacted form. From f59e8af73474aad18dd68300b245fd0ce2b8ab92 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 8 Nov 2024 05:49:28 +0000 Subject: [PATCH 1019/2091] slight cleanup/simplifications to backfil Signed-off-by: Jason Volk --- src/api/server/backfill.rs | 76 +++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index be770ee8..2858d9fd 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -16,7 +16,7 @@ use crate::Ruma; /// Retrieves events from before the sender joined the room, if the room's /// history visibility allows. pub(crate) async fn get_backfill_route( - State(services): State, body: Ruma, + State(services): State, ref body: Ruma, ) -> Result { AccessCheck { services: &services, @@ -27,7 +27,13 @@ pub(crate) async fn get_backfill_route( .check() .await?; - let until = body + let limit = body + .limit + .min(uint!(100)) + .try_into() + .expect("UInt could not be converted to usize"); + + let from = body .v .iter() .stream() @@ -38,46 +44,38 @@ pub(crate) async fn get_backfill_route( .get_pdu_count(event_id) .map(Result::ok) }) - .ready_fold(PduCount::Backfilled(0), cmp::max) - .await; - - let limit = body - .limit - .min(uint!(100)) - .try_into() - .expect("UInt could not be converted to usize"); - - let origin = body.origin(); - let pdus = services - .rooms - .timeline - .pdus_rev(None, &body.room_id, Some(until)) - .await? - .take(limit) - .filter_map(|(_, pdu)| async move { - if !services - .rooms - .state_accessor - .server_can_see_event(origin, &pdu.room_id, &pdu.event_id) - .await - { - return None; - } - - services - .rooms - .timeline - .get_pdu_json(&pdu.event_id) - .await - .ok() - }) - .then(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) - .collect() + .ready_fold(PduCount::min(), cmp::max) .await; Ok(get_backfill::v1::Response { - origin: services.globals.server_name().to_owned(), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - pdus, + + origin: services.globals.server_name().to_owned(), + + pdus: services + .rooms + .timeline + .pdus_rev(None, &body.room_id, Some(from)) + .await? + .take(limit) + .filter_map(|(_, pdu)| async move { + services + .rooms + .state_accessor + .server_can_see_event(body.origin(), &pdu.room_id, &pdu.event_id) + .await + .then_some(pdu) + }) + .filter_map(|pdu| async move { + services + .rooms + .timeline + .get_pdu_json(&pdu.event_id) + .await + .ok() + }) + .then(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) + .collect() + .await, }) } From 6eba36d7883439539b8ca0b65f04d2935e41ad05 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 8 Nov 2024 08:21:19 +0000 Subject: [PATCH 1020/2091] split make_body template Signed-off-by: Jason Volk --- src/api/router/args.rs | 47 ++++++++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/src/api/router/args.rs b/src/api/router/args.rs index 38236db3..4c0aff4c 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -1,8 +1,8 @@ use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; -use bytes::{BufMut, BytesMut}; -use conduit::{debug, err, trace, utils::string::EMPTY, Error, Result}; +use bytes::{BufMut, Bytes, BytesMut}; +use conduit::{debug, err, utils::string::EMPTY, Error, Result}; use ruma::{ api::IncomingRequest, CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedServerName, OwnedUserId, ServerName, UserId, }; @@ -103,7 +103,32 @@ fn make_body( where T: IncomingRequest + Send + Sync + 'static, { - let body = if let Some(CanonicalJsonValue::Object(json_body)) = json_body { + let body = take_body(services, request, json_body, auth); + let http_request = into_http_request(request, body); + T::try_from_http_request(http_request, &request.path).map_err(|e| err!(Request(BadJson(debug_warn!("{e}"))))) +} + +fn into_http_request(request: &Request, body: Bytes) -> hyper::Request { + let mut http_request = hyper::Request::builder() + .uri(request.parts.uri.clone()) + .method(request.parts.method.clone()); + + *http_request.headers_mut().expect("mutable http headers") = request.parts.headers.clone(); + + let http_request = http_request.body(body).expect("http request body"); + + let headers = http_request.headers(); + let method = http_request.method(); + let uri = http_request.uri(); + debug!("{method:?} {uri:?} {headers:?}"); + + http_request +} + +fn take_body( + services: &Services, request: &mut Request, json_body: &mut Option, auth: &Auth, +) -> Bytes { + if let Some(CanonicalJsonValue::Object(json_body)) = json_body { let user_id = auth.sender_user.clone().unwrap_or_else(|| { let server_name = services.globals.server_name(); UserId::parse_with_server_name(EMPTY, server_name).expect("valid user_id") @@ -131,19 +156,5 @@ where buf.into_inner().freeze() } else { mem::take(&mut request.body) - }; - - let mut http_request = hyper::Request::builder() - .uri(request.parts.uri.clone()) - .method(request.parts.method.clone()); - *http_request.headers_mut().expect("mutable http headers") = request.parts.headers.clone(); - let http_request = http_request.body(body).expect("http request body"); - - let headers = http_request.headers(); - let method = http_request.method(); - let uri = http_request.uri(); - debug!("{method:?} {uri:?} {headers:?}"); - trace!("{method:?} {uri:?} {json_body:?}"); - - T::try_from_http_request(http_request, &request.path).map_err(|e| err!(Request(BadJson(debug_warn!("{e}"))))) + } } From 1ce3db727fdd298ba94dd472d017c6fe7e8a92c2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 8 Nov 2024 07:30:52 +0000 Subject: [PATCH 1021/2091] split event_handler service Signed-off-by: Jason Volk --- src/service/rooms/event_handler/acl_check.rs | 35 + .../fetch_and_handle_outliers.rs | 181 +++ src/service/rooms/event_handler/fetch_prev.rs | 104 ++ .../rooms/event_handler/fetch_state.rs | 84 ++ .../event_handler/handle_incoming_pdu.rs | 172 +++ .../rooms/event_handler/handle_outlier_pdu.rs | 164 ++ .../rooms/event_handler/handle_prev_pdu.rs | 82 + src/service/rooms/event_handler/mod.rs | 1328 +---------------- .../rooms/event_handler/parse_incoming_pdu.rs | 41 +- .../rooms/event_handler/resolve_state.rs | 101 ++ .../rooms/event_handler/state_at_incoming.rs | 178 +++ .../event_handler/upgrade_outlier_pdu.rs | 298 ++++ 12 files changed, 1437 insertions(+), 1331 deletions(-) create mode 100644 src/service/rooms/event_handler/acl_check.rs create mode 100644 src/service/rooms/event_handler/fetch_and_handle_outliers.rs create mode 100644 src/service/rooms/event_handler/fetch_prev.rs create mode 100644 src/service/rooms/event_handler/fetch_state.rs create mode 100644 src/service/rooms/event_handler/handle_incoming_pdu.rs create mode 100644 src/service/rooms/event_handler/handle_outlier_pdu.rs create mode 100644 src/service/rooms/event_handler/handle_prev_pdu.rs create mode 100644 src/service/rooms/event_handler/resolve_state.rs create mode 100644 src/service/rooms/event_handler/state_at_incoming.rs create mode 100644 src/service/rooms/event_handler/upgrade_outlier_pdu.rs diff --git a/src/service/rooms/event_handler/acl_check.rs b/src/service/rooms/event_handler/acl_check.rs new file mode 100644 index 00000000..f2ff1b00 --- /dev/null +++ b/src/service/rooms/event_handler/acl_check.rs @@ -0,0 +1,35 @@ +use conduit::{debug, implement, trace, warn, Err, Result}; +use ruma::{ + events::{room::server_acl::RoomServerAclEventContent, StateEventType}, + RoomId, ServerName, +}; + +/// Returns Ok if the acl allows the server +#[implement(super::Service)] +#[tracing::instrument(skip_all)] +pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result { + let Ok(acl_event_content) = self + .services + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomServerAcl, "") + .await + .map(|c: RoomServerAclEventContent| c) + .inspect(|acl| trace!("ACL content found: {acl:?}")) + .inspect_err(|e| trace!("No ACL content found: {e:?}")) + else { + return Ok(()); + }; + + if acl_event_content.allow.is_empty() { + warn!("Ignoring broken ACL event (allow key is empty)"); + return Ok(()); + } + + if acl_event_content.is_allowed(server_name) { + trace!("server {server_name} is allowed by ACL"); + Ok(()) + } else { + debug!("Server {server_name} was denied by room ACL in {room_id}"); + Err!(Request(Forbidden("Server was denied by room ACL"))) + } +} diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs new file mode 100644 index 00000000..677b78f2 --- /dev/null +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -0,0 +1,181 @@ +use std::{ + collections::{hash_map, BTreeMap, HashSet}, + sync::Arc, + time::Instant, +}; + +use conduit::{ + debug, debug_error, implement, info, pdu, trace, utils::math::continue_exponential_backoff_secs, warn, PduEvent, +}; +use ruma::{api::federation::event::get_event, CanonicalJsonValue, EventId, RoomId, RoomVersionId, ServerName}; + +/// Find the event and auth it. Once the event is validated (steps 1 - 8) +/// it is appended to the outliers Tree. +/// +/// Returns pdu and if we fetched it over federation the raw json. +/// +/// a. Look in the main timeline (pduid_pdu tree) +/// b. Look at outlier pdu tree +/// c. Ask origin server over federation +/// d. TODO: Ask other servers over federation? +#[implement(super::Service)] +pub(super) async fn fetch_and_handle_outliers<'a>( + &self, origin: &'a ServerName, events: &'a [Arc], create_event: &'a PduEvent, room_id: &'a RoomId, + room_version_id: &'a RoomVersionId, +) -> Vec<(Arc, Option>)> { + let back_off = |id| match self + .services + .globals + .bad_event_ratelimiter + .write() + .expect("locked") + .entry(id) + { + hash_map::Entry::Vacant(e) => { + e.insert((Instant::now(), 1)); + }, + hash_map::Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)), + }; + + let mut events_with_auth_events = Vec::with_capacity(events.len()); + for id in events { + // a. Look in the main timeline (pduid_pdu tree) + // b. Look at outlier pdu tree + // (get_pdu_json checks both) + if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { + trace!("Found {id} in db"); + events_with_auth_events.push((id, Some(local_pdu), vec![])); + continue; + } + + // c. Ask origin server over federation + // We also handle its auth chain here so we don't get a stack overflow in + // handle_outlier_pdu. + let mut todo_auth_events = vec![Arc::clone(id)]; + let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); + let mut events_all = HashSet::with_capacity(todo_auth_events.len()); + while let Some(next_id) = todo_auth_events.pop() { + if let Some((time, tries)) = self + .services + .globals + .bad_event_ratelimiter + .read() + .expect("locked") + .get(&*next_id) + { + // Exponential backoff + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + info!("Backing off from {next_id}"); + continue; + } + } + + if events_all.contains(&next_id) { + continue; + } + + if self.services.timeline.pdu_exists(&next_id).await { + trace!("Found {next_id} in db"); + continue; + } + + debug!("Fetching {next_id} over federation."); + match self + .services + .sending + .send_federation_request( + origin, + get_event::v1::Request { + event_id: (*next_id).to_owned(), + include_unredacted_content: None, + }, + ) + .await + { + Ok(res) => { + debug!("Got {next_id} over federation"); + let Ok((calculated_event_id, value)) = pdu::gen_event_id_canonical_json(&res.pdu, room_version_id) + else { + back_off((*next_id).to_owned()); + continue; + }; + + if calculated_event_id != *next_id { + warn!( + "Server didn't return event id we requested: requested: {next_id}, we got \ + {calculated_event_id}. Event: {:?}", + &res.pdu + ); + } + + if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) { + for auth_event in auth_events { + if let Ok(auth_event) = serde_json::from_value(auth_event.clone().into()) { + let a: Arc = auth_event; + todo_auth_events.push(a); + } else { + warn!("Auth event id is not valid"); + } + } + } else { + warn!("Auth event list invalid"); + } + + events_in_reverse_order.push((next_id.clone(), value)); + events_all.insert(next_id); + }, + Err(e) => { + debug_error!("Failed to fetch event {next_id}: {e}"); + back_off((*next_id).to_owned()); + }, + } + } + events_with_auth_events.push((id, None, events_in_reverse_order)); + } + + let mut pdus = Vec::with_capacity(events_with_auth_events.len()); + for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { + // a. Look in the main timeline (pduid_pdu tree) + // b. Look at outlier pdu tree + // (get_pdu_json checks both) + if let Some(local_pdu) = local_pdu { + trace!("Found {id} in db"); + pdus.push((local_pdu.clone(), None)); + } + + for (next_id, value) in events_in_reverse_order.into_iter().rev() { + if let Some((time, tries)) = self + .services + .globals + .bad_event_ratelimiter + .read() + .expect("locked") + .get(&*next_id) + { + // Exponential backoff + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + debug!("Backing off from {next_id}"); + continue; + } + } + + match Box::pin(self.handle_outlier_pdu(origin, create_event, &next_id, room_id, value.clone(), true)).await + { + Ok((pdu, json)) => { + if next_id == *id { + pdus.push((pdu, Some(json))); + } + }, + Err(e) => { + warn!("Authentication of event {next_id} failed: {e:?}"); + back_off(next_id.into()); + }, + } + } + } + pdus +} diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs new file mode 100644 index 00000000..4acdba1d --- /dev/null +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -0,0 +1,104 @@ +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + sync::Arc, +}; + +use conduit::{debug_warn, err, implement, PduEvent, Result}; +use futures::{future, FutureExt}; +use ruma::{ + int, + state_res::{self}, + uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, ServerName, +}; + +use super::check_room_id; + +#[implement(super::Service)] +#[allow(clippy::type_complexity)] +#[tracing::instrument(skip_all)] +pub(super) async fn fetch_prev( + &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, + initial_set: Vec>, +) -> Result<( + Vec>, + HashMap, (Arc, BTreeMap)>, +)> { + let mut graph: HashMap, _> = HashMap::with_capacity(initial_set.len()); + let mut eventid_info = HashMap::new(); + let mut todo_outlier_stack: Vec> = initial_set; + + let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; + + let mut amount = 0; + + while let Some(prev_event_id) = todo_outlier_stack.pop() { + self.services.server.check_running()?; + + if let Some((pdu, mut json_opt)) = self + .fetch_and_handle_outliers(origin, &[prev_event_id.clone()], create_event, room_id, room_version_id) + .boxed() + .await + .pop() + { + check_room_id(room_id, &pdu)?; + + let limit = self.services.server.config.max_fetch_prev_events; + if amount > limit { + debug_warn!("Max prev event limit reached! Limit: {limit}"); + graph.insert(prev_event_id.clone(), HashSet::new()); + continue; + } + + if json_opt.is_none() { + json_opt = self + .services + .outlier + .get_outlier_pdu_json(&prev_event_id) + .await + .ok(); + } + + if let Some(json) = json_opt { + if pdu.origin_server_ts > first_pdu_in_room.origin_server_ts { + amount = amount.saturating_add(1); + for prev_prev in &pdu.prev_events { + if !graph.contains_key(prev_prev) { + todo_outlier_stack.push(prev_prev.clone()); + } + } + + graph.insert(prev_event_id.clone(), pdu.prev_events.iter().cloned().collect()); + } else { + // Time based check failed + graph.insert(prev_event_id.clone(), HashSet::new()); + } + + eventid_info.insert(prev_event_id.clone(), (pdu, json)); + } else { + // Get json failed, so this was not fetched over federation + graph.insert(prev_event_id.clone(), HashSet::new()); + } + } else { + // Fetch and handle failed + graph.insert(prev_event_id.clone(), HashSet::new()); + } + } + + let event_fetch = |event_id| { + let origin_server_ts = eventid_info + .get(&event_id) + .cloned() + .map_or_else(|| uint!(0), |info| info.0.origin_server_ts); + + // This return value is the key used for sorting events, + // events are then sorted by power level, time, + // and lexically by event_id. + future::ok((int!(0), MilliSecondsSinceUnixEpoch(origin_server_ts))) + }; + + let sorted = state_res::lexicographical_topological_sort(&graph, &event_fetch) + .await + .map_err(|e| err!(Database(error!("Error sorting prev events: {e}"))))?; + + Ok((sorted, eventid_info)) +} diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs new file mode 100644 index 00000000..74b0bb32 --- /dev/null +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -0,0 +1,84 @@ +use std::{ + collections::{hash_map, HashMap}, + sync::Arc, +}; + +use conduit::{debug, implement, warn, Err, Error, PduEvent, Result}; +use futures::FutureExt; +use ruma::{ + api::federation::event::get_room_state_ids, events::StateEventType, EventId, RoomId, RoomVersionId, ServerName, +}; + +/// Call /state_ids to find out what the state at this pdu is. We trust the +/// server's response to some extend (sic), but we still do a lot of checks +/// on the events +#[implement(super::Service)] +#[tracing::instrument(skip(self, create_event, room_version_id))] +pub(super) async fn fetch_state( + &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, + event_id: &EventId, +) -> Result>>> { + debug!("Fetching state ids"); + let res = self + .services + .sending + .send_synapse_request( + origin, + get_room_state_ids::v1::Request { + room_id: room_id.to_owned(), + event_id: (*event_id).to_owned(), + }, + ) + .await + .inspect_err(|e| warn!("Fetching state for event failed: {e}"))?; + + debug!("Fetching state events"); + let collect = res + .pdu_ids + .iter() + .map(|x| Arc::from(&**x)) + .collect::>(); + + let state_vec = self + .fetch_and_handle_outliers(origin, &collect, create_event, room_id, room_version_id) + .boxed() + .await; + + let mut state: HashMap<_, Arc> = HashMap::with_capacity(state_vec.len()); + for (pdu, _) in state_vec { + let state_key = pdu + .state_key + .clone() + .ok_or_else(|| Error::bad_database("Found non-state pdu in state events."))?; + + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&pdu.kind.to_string().into(), &state_key) + .await; + + match state.entry(shortstatekey) { + hash_map::Entry::Vacant(v) => { + v.insert(Arc::from(&*pdu.event_id)); + }, + hash_map::Entry::Occupied(_) => { + return Err(Error::bad_database( + "State event's type and state_key combination exists multiple times.", + )) + }, + } + } + + // The original create event must still be in the state + let create_shortstatekey = self + .services + .short + .get_shortstatekey(&StateEventType::RoomCreate, "") + .await?; + + if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(&create_event.event_id) { + return Err!(Database("Incoming event refers to wrong create event.")); + } + + Ok(Some(state)) +} diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs new file mode 100644 index 00000000..4d2d75d5 --- /dev/null +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -0,0 +1,172 @@ +use std::{ + collections::{hash_map, BTreeMap}, + time::Instant, +}; + +use conduit::{debug, err, implement, warn, Error, Result}; +use futures::FutureExt; +use ruma::{ + api::client::error::ErrorKind, events::StateEventType, CanonicalJsonValue, EventId, RoomId, ServerName, UserId, +}; + +use super::{check_room_id, get_room_version_id}; +use crate::rooms::timeline::RawPduId; + +/// When receiving an event one needs to: +/// 0. Check the server is in the room +/// 1. Skip the PDU if we already know about it +/// 1.1. Remove unsigned field +/// 2. Check signatures, otherwise drop +/// 3. Check content hash, redact if doesn't match +/// 4. Fetch any missing auth events doing all checks listed here starting at 1. +/// These are not timeline events +/// 5. Reject "due to auth events" if can't get all the auth events or some of +/// the auth events are also rejected "due to auth events" +/// 6. Reject "due to auth events" if the event doesn't pass auth based on the +/// auth events +/// 7. Persist this event as an outlier +/// 8. If not timeline event: stop +/// 9. Fetch any missing prev events doing all checks listed here starting at 1. +/// These are timeline events +/// 10. Fetch missing state and auth chain events by calling `/state_ids` at +/// backwards extremities doing all the checks in this list starting at +/// 1. These are not timeline events +/// 11. Check the auth of the event passes based on the state of the event +/// 12. Ensure that the state is derived from the previous current state (i.e. +/// we calculated by doing state res where one of the inputs was a +/// previously trusted set of state, don't just trust a set of state we got +/// from a remote) +/// 13. Use state resolution to find new room state +/// 14. Check if the event passes auth based on the "current state" of the room, +/// if not soft fail it +#[implement(super::Service)] +#[tracing::instrument(skip(self, origin, value, is_timeline_event), name = "pdu")] +pub async fn handle_incoming_pdu<'a>( + &self, origin: &'a ServerName, room_id: &'a RoomId, event_id: &'a EventId, + value: BTreeMap, is_timeline_event: bool, +) -> Result> { + // 1. Skip the PDU if we already have it as a timeline event + if let Ok(pdu_id) = self.services.timeline.get_pdu_id(event_id).await { + return Ok(Some(pdu_id)); + } + + // 1.1 Check the server is in the room + if !self.services.metadata.exists(room_id).await { + return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server")); + } + + // 1.2 Check if the room is disabled + if self.services.metadata.is_disabled(room_id).await { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Federation of this room is currently disabled on this server.", + )); + } + + // 1.3.1 Check room ACL on origin field/server + self.acl_check(origin, room_id).await?; + + // 1.3.2 Check room ACL on sender's server name + let sender: &UserId = value + .get("sender") + .try_into() + .map_err(|e| err!(Request(InvalidParam("PDU does not have a valid sender key: {e}"))))?; + + self.acl_check(sender.server_name(), room_id).await?; + + // Fetch create event + let create_event = self + .services + .state_accessor + .room_state_get(room_id, &StateEventType::RoomCreate, "") + .await?; + + // Procure the room version + let room_version_id = get_room_version_id(&create_event)?; + + let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; + + let (incoming_pdu, val) = self + .handle_outlier_pdu(origin, &create_event, event_id, room_id, value, false) + .boxed() + .await?; + + check_room_id(room_id, &incoming_pdu)?; + + // 8. if not timeline event: stop + if !is_timeline_event { + return Ok(None); + } + // Skip old events + if incoming_pdu.origin_server_ts < first_pdu_in_room.origin_server_ts { + return Ok(None); + } + + // 9. Fetch any missing prev events doing all checks listed here starting at 1. + // These are timeline events + let (sorted_prev_events, mut eventid_info) = self + .fetch_prev( + origin, + &create_event, + room_id, + &room_version_id, + incoming_pdu.prev_events.clone(), + ) + .await?; + + debug!(events = ?sorted_prev_events, "Got previous events"); + for prev_id in sorted_prev_events { + self.services.server.check_running()?; + if let Err(e) = self + .handle_prev_pdu( + origin, + event_id, + room_id, + &mut eventid_info, + &create_event, + &first_pdu_in_room, + &prev_id, + ) + .await + { + use hash_map::Entry; + + let now = Instant::now(); + warn!("Prev event {prev_id} failed: {e}"); + + match self + .services + .globals + .bad_event_ratelimiter + .write() + .expect("locked") + .entry(prev_id.into()) + { + Entry::Vacant(e) => { + e.insert((now, 1)); + }, + Entry::Occupied(mut e) => { + *e.get_mut() = (now, e.get().1.saturating_add(1)); + }, + }; + } + } + + // Done with prev events, now handling the incoming event + let start_time = Instant::now(); + self.federation_handletime + .write() + .expect("locked") + .insert(room_id.to_owned(), (event_id.to_owned(), start_time)); + + let r = self + .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, &create_event, origin, room_id) + .await; + + self.federation_handletime + .write() + .expect("locked") + .remove(&room_id.to_owned()); + + r +} diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs new file mode 100644 index 00000000..2d95ff63 --- /dev/null +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -0,0 +1,164 @@ +use std::{ + collections::{hash_map, BTreeMap, HashMap}, + sync::Arc, +}; + +use conduit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; +use futures::future::ready; +use ruma::{ + api::client::error::ErrorKind, + events::StateEventType, + state_res::{self, EventTypeExt}, + CanonicalJsonObject, CanonicalJsonValue, EventId, RoomId, ServerName, +}; + +use super::{check_room_id, get_room_version_id, to_room_version}; + +#[implement(super::Service)] +#[allow(clippy::too_many_arguments)] +pub(super) async fn handle_outlier_pdu<'a>( + &self, origin: &'a ServerName, create_event: &'a PduEvent, event_id: &'a EventId, room_id: &'a RoomId, + mut value: CanonicalJsonObject, auth_events_known: bool, +) -> Result<(Arc, BTreeMap)> { + // 1. Remove unsigned field + value.remove("unsigned"); + + // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we anywhere?: https://matrix.org/docs/spec/rooms/v6#canonical-json + + // 2. Check signatures, otherwise drop + // 3. check content hash, redact if doesn't match + let room_version_id = get_room_version_id(create_event)?; + let mut val = match self + .services + .server_keys + .verify_event(&value, Some(&room_version_id)) + .await + { + Ok(ruma::signatures::Verified::All) => value, + Ok(ruma::signatures::Verified::Signatures) => { + // Redact + debug_info!("Calculated hash does not match (redaction): {event_id}"); + let Ok(obj) = ruma::canonical_json::redact(value, &room_version_id, None) else { + return Err!(Request(InvalidParam("Redaction failed"))); + }; + + // Skip the PDU if it is redacted and we already have it as an outlier event + if self.services.timeline.pdu_exists(event_id).await { + return Err!(Request(InvalidParam("Event was redacted and we already knew about it"))); + } + + obj + }, + Err(e) => { + return Err!(Request(InvalidParam(debug_error!( + "Signature verification failed for {event_id}: {e}" + )))) + }, + }; + + // Now that we have checked the signature and hashes we can add the eventID and + // convert to our PduEvent type + val.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); + let incoming_pdu = + serde_json::from_value::(serde_json::to_value(&val).expect("CanonicalJsonObj is a valid JsonValue")) + .map_err(|_| Error::bad_database("Event is not a valid PDU."))?; + + check_room_id(room_id, &incoming_pdu)?; + + if !auth_events_known { + // 4. fetch any missing auth events doing all checks listed here starting at 1. + // These are not timeline events + // 5. Reject "due to auth events" if can't get all the auth events or some of + // the auth events are also rejected "due to auth events" + // NOTE: Step 5 is not applied anymore because it failed too often + debug!("Fetching auth events"); + Box::pin( + self.fetch_and_handle_outliers( + origin, + &incoming_pdu + .auth_events + .iter() + .map(|x| Arc::from(&**x)) + .collect::>>(), + create_event, + room_id, + &room_version_id, + ), + ) + .await; + } + + // 6. Reject "due to auth events" if the event doesn't pass auth based on the + // auth events + debug!("Checking based on auth events"); + // Build map of auth events + let mut auth_events = HashMap::with_capacity(incoming_pdu.auth_events.len()); + for id in &incoming_pdu.auth_events { + let Ok(auth_event) = self.services.timeline.get_pdu(id).await else { + warn!("Could not find auth event {id}"); + continue; + }; + + check_room_id(room_id, &auth_event)?; + + match auth_events.entry(( + auth_event.kind.to_string().into(), + auth_event + .state_key + .clone() + .expect("all auth events have state keys"), + )) { + hash_map::Entry::Vacant(v) => { + v.insert(auth_event); + }, + hash_map::Entry::Occupied(_) => { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Auth event's type and state_key combination exists multiple times.", + )); + }, + } + } + + // The original create event must be in the auth events + if !matches!( + auth_events + .get(&(StateEventType::RoomCreate, String::new())) + .map(AsRef::as_ref), + Some(_) | None + ) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Incoming event refers to wrong create event.", + )); + } + + let state_fetch = |ty: &'static StateEventType, sk: &str| { + let key = ty.with_state_key(sk); + ready(auth_events.get(&key)) + }; + + let auth_check = state_res::event_auth::auth_check( + &to_room_version(&room_version_id), + &incoming_pdu, + None, // TODO: third party invite + state_fetch, + ) + .await + .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; + + if !auth_check { + return Err!(Request(Forbidden("Auth check failed"))); + } + + trace!("Validation successful."); + + // 7. Persist the event as an outlier. + self.services + .outlier + .add_pdu_outlier(&incoming_pdu.event_id, &val); + + trace!("Added pdu as outlier."); + + Ok((Arc::new(incoming_pdu), val)) +} diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs new file mode 100644 index 00000000..90ff7f06 --- /dev/null +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -0,0 +1,82 @@ +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, + time::Instant, +}; + +use conduit::{debug, implement, utils::math::continue_exponential_backoff_secs, Error, PduEvent, Result}; +use ruma::{api::client::error::ErrorKind, CanonicalJsonValue, EventId, RoomId, ServerName}; + +#[implement(super::Service)] +#[allow(clippy::type_complexity)] +#[allow(clippy::too_many_arguments)] +#[tracing::instrument( + skip(self, origin, event_id, room_id, eventid_info, create_event, first_pdu_in_room), + name = "prev" +)] +pub(super) async fn handle_prev_pdu<'a>( + &self, origin: &'a ServerName, event_id: &'a EventId, room_id: &'a RoomId, + eventid_info: &mut HashMap, (Arc, BTreeMap)>, + create_event: &Arc, first_pdu_in_room: &Arc, prev_id: &EventId, +) -> Result { + // Check for disabled again because it might have changed + if self.services.metadata.is_disabled(room_id).await { + debug!( + "Federaton of room {room_id} is currently disabled on this server. Request by origin {origin} and event \ + ID {event_id}" + ); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Federation of this room is currently disabled on this server.", + )); + } + + if let Some((time, tries)) = self + .services + .globals + .bad_event_ratelimiter + .read() + .expect("locked") + .get(prev_id) + { + // Exponential backoff + const MIN_DURATION: u64 = 5 * 60; + const MAX_DURATION: u64 = 60 * 60 * 24; + if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + debug!( + ?tries, + duration = ?time.elapsed(), + "Backing off from prev_event" + ); + return Ok(()); + } + } + + if let Some((pdu, json)) = eventid_info.remove(prev_id) { + // Skip old events + if pdu.origin_server_ts < first_pdu_in_room.origin_server_ts { + return Ok(()); + } + + let start_time = Instant::now(); + self.federation_handletime + .write() + .expect("locked") + .insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time)); + + self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id) + .await?; + + self.federation_handletime + .write() + .expect("locked") + .remove(&room_id.to_owned()); + + debug!( + elapsed = ?start_time.elapsed(), + "Handled prev_event", + ); + } + + Ok(()) +} diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index f76f817d..f6440fe9 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -1,51 +1,34 @@ +mod acl_check; +mod fetch_and_handle_outliers; +mod fetch_prev; +mod fetch_state; +mod handle_incoming_pdu; +mod handle_outlier_pdu; +mod handle_prev_pdu; mod parse_incoming_pdu; +mod resolve_state; +mod state_at_incoming; +mod upgrade_outlier_pdu; use std::{ - borrow::Borrow, - collections::{hash_map, BTreeMap, HashMap, HashSet}, + collections::HashMap, fmt::Write, sync::{Arc, RwLock as StdRwLock}, time::Instant, }; -use conduit::{ - debug, debug_error, debug_info, debug_warn, err, info, pdu, - result::LogErr, - trace, - utils::{math::continue_exponential_backoff_secs, IterStream, MutexMap}, - warn, Err, Error, PduEvent, Result, Server, -}; -use futures::{future, future::ready, FutureExt, StreamExt, TryFutureExt}; +use conduit::{utils::MutexMap, Err, PduEvent, Result, Server}; use ruma::{ - api::{ - client::error::ErrorKind, - federation::event::{get_event, get_room_state_ids}, - }, - events::{ - room::{ - create::RoomCreateEventContent, redaction::RoomRedactionEventContent, server_acl::RoomServerAclEventContent, - }, - StateEventType, TimelineEventType, - }, - int, - state_res::{self, EventTypeExt, RoomVersion, StateMap}, - uint, CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, - RoomId, RoomVersionId, ServerName, UserId, + events::room::create::RoomCreateEventContent, state_res::RoomVersion, EventId, OwnedEventId, OwnedRoomId, RoomId, + RoomVersionId, }; -use crate::{ - globals, rooms, - rooms::{ - state_compressor::{CompressedStateEvent, HashSetCompressStateEvent}, - timeline::RawPduId, - }, - sending, server_keys, Dep, -}; +use crate::{globals, rooms, sending, server_keys, Dep}; pub struct Service { - services: Services, - pub federation_handletime: StdRwLock, pub mutex_federation: RoomMutexMap, + pub federation_handletime: StdRwLock, + services: Services, } struct Services { @@ -70,6 +53,8 @@ type HandleTimeMap = HashMap; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { + mutex_federation: RoomMutexMap::new(), + federation_handletime: HandleTimeMap::new().into(), services: Services { globals: args.depend::("globals"), sending: args.depend::("sending"), @@ -85,8 +70,6 @@ impl crate::Service for Service { timeline: args.depend::("rooms::timeline"), server: args.server.clone(), }, - federation_handletime: HandleTimeMap::new().into(), - mutex_federation: RoomMutexMap::new(), })) } @@ -108,1279 +91,6 @@ impl crate::Service for Service { } impl Service { - /// When receiving an event one needs to: - /// 0. Check the server is in the room - /// 1. Skip the PDU if we already know about it - /// 1.1. Remove unsigned field - /// 2. Check signatures, otherwise drop - /// 3. Check content hash, redact if doesn't match - /// 4. Fetch any missing auth events doing all checks listed here starting - /// at 1. These are not timeline events - /// 5. Reject "due to auth events" if can't get all the auth events or some - /// of the auth events are also rejected "due to auth events" - /// 6. Reject "due to auth events" if the event doesn't pass auth based on - /// the auth events - /// 7. Persist this event as an outlier - /// 8. If not timeline event: stop - /// 9. Fetch any missing prev events doing all checks listed here starting - /// at 1. These are timeline events - /// 10. Fetch missing state and auth chain events by calling `/state_ids` at - /// backwards extremities doing all the checks in this list starting at - /// 1. These are not timeline events - /// 11. Check the auth of the event passes based on the state of the event - /// 12. Ensure that the state is derived from the previous current state - /// (i.e. we calculated by doing state res where one of the inputs was a - /// previously trusted set of state, don't just trust a set of state we - /// got from a remote) - /// 13. Use state resolution to find new room state - /// 14. Check if the event passes auth based on the "current state" of the - /// room, if not soft fail it - #[tracing::instrument(skip(self, origin, value, is_timeline_event), name = "pdu")] - pub async fn handle_incoming_pdu<'a>( - &self, origin: &'a ServerName, room_id: &'a RoomId, event_id: &'a EventId, - value: BTreeMap, is_timeline_event: bool, - ) -> Result> { - // 1. Skip the PDU if we already have it as a timeline event - if let Ok(pdu_id) = self.services.timeline.get_pdu_id(event_id).await { - return Ok(Some(pdu_id)); - } - - // 1.1 Check the server is in the room - if !self.services.metadata.exists(room_id).await { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server")); - } - - // 1.2 Check if the room is disabled - if self.services.metadata.is_disabled(room_id).await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Federation of this room is currently disabled on this server.", - )); - } - - // 1.3.1 Check room ACL on origin field/server - self.acl_check(origin, room_id).await?; - - // 1.3.2 Check room ACL on sender's server name - let sender: &UserId = value - .get("sender") - .try_into() - .map_err(|e| err!(Request(InvalidParam("PDU does not have a valid sender key: {e}"))))?; - - self.acl_check(sender.server_name(), room_id).await?; - - // Fetch create event - let create_event = self - .services - .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "") - .await?; - - // Procure the room version - let room_version_id = get_room_version_id(&create_event)?; - - let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; - - let (incoming_pdu, val) = self - .handle_outlier_pdu(origin, &create_event, event_id, room_id, value, false) - .boxed() - .await?; - - check_room_id(room_id, &incoming_pdu)?; - - // 8. if not timeline event: stop - if !is_timeline_event { - return Ok(None); - } - // Skip old events - if incoming_pdu.origin_server_ts < first_pdu_in_room.origin_server_ts { - return Ok(None); - } - - // 9. Fetch any missing prev events doing all checks listed here starting at 1. - // These are timeline events - let (sorted_prev_events, mut eventid_info) = self - .fetch_prev( - origin, - &create_event, - room_id, - &room_version_id, - incoming_pdu.prev_events.clone(), - ) - .await?; - - debug!(events = ?sorted_prev_events, "Got previous events"); - for prev_id in sorted_prev_events { - self.services.server.check_running()?; - if let Err(e) = self - .handle_prev_pdu( - origin, - event_id, - room_id, - &mut eventid_info, - &create_event, - &first_pdu_in_room, - &prev_id, - ) - .await - { - use hash_map::Entry; - - let now = Instant::now(); - warn!("Prev event {prev_id} failed: {e}"); - - match self - .services - .globals - .bad_event_ratelimiter - .write() - .expect("locked") - .entry(prev_id.into()) - { - Entry::Vacant(e) => { - e.insert((now, 1)); - }, - Entry::Occupied(mut e) => { - *e.get_mut() = (now, e.get().1.saturating_add(1)); - }, - }; - } - } - - // Done with prev events, now handling the incoming event - let start_time = Instant::now(); - self.federation_handletime - .write() - .expect("locked") - .insert(room_id.to_owned(), (event_id.to_owned(), start_time)); - - let r = self - .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, &create_event, origin, room_id) - .await; - - self.federation_handletime - .write() - .expect("locked") - .remove(&room_id.to_owned()); - - r - } - - #[allow(clippy::type_complexity)] - #[allow(clippy::too_many_arguments)] - #[tracing::instrument( - skip(self, origin, event_id, room_id, eventid_info, create_event, first_pdu_in_room), - name = "prev" - )] - pub async fn handle_prev_pdu<'a>( - &self, origin: &'a ServerName, event_id: &'a EventId, room_id: &'a RoomId, - eventid_info: &mut HashMap, (Arc, BTreeMap)>, - create_event: &Arc, first_pdu_in_room: &Arc, prev_id: &EventId, - ) -> Result<()> { - // Check for disabled again because it might have changed - if self.services.metadata.is_disabled(room_id).await { - debug!( - "Federaton of room {room_id} is currently disabled on this server. Request by origin {origin} and \ - event ID {event_id}" - ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Federation of this room is currently disabled on this server.", - )); - } - - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .expect("locked") - .get(prev_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { - debug!( - ?tries, - duration = ?time.elapsed(), - "Backing off from prev_event" - ); - return Ok(()); - } - } - - if let Some((pdu, json)) = eventid_info.remove(prev_id) { - // Skip old events - if pdu.origin_server_ts < first_pdu_in_room.origin_server_ts { - return Ok(()); - } - - let start_time = Instant::now(); - self.federation_handletime - .write() - .expect("locked") - .insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time)); - - self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id) - .await?; - - self.federation_handletime - .write() - .expect("locked") - .remove(&room_id.to_owned()); - - debug!( - elapsed = ?start_time.elapsed(), - "Handled prev_event", - ); - } - - Ok(()) - } - - #[allow(clippy::too_many_arguments)] - async fn handle_outlier_pdu<'a>( - &self, origin: &'a ServerName, create_event: &'a PduEvent, event_id: &'a EventId, room_id: &'a RoomId, - mut value: CanonicalJsonObject, auth_events_known: bool, - ) -> Result<(Arc, BTreeMap)> { - // 1. Remove unsigned field - value.remove("unsigned"); - - // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we anywhere?: https://matrix.org/docs/spec/rooms/v6#canonical-json - - // 2. Check signatures, otherwise drop - // 3. check content hash, redact if doesn't match - let room_version_id = get_room_version_id(create_event)?; - let mut val = match self - .services - .server_keys - .verify_event(&value, Some(&room_version_id)) - .await - { - Ok(ruma::signatures::Verified::All) => value, - Ok(ruma::signatures::Verified::Signatures) => { - // Redact - debug_info!("Calculated hash does not match (redaction): {event_id}"); - let Ok(obj) = ruma::canonical_json::redact(value, &room_version_id, None) else { - return Err!(Request(InvalidParam("Redaction failed"))); - }; - - // Skip the PDU if it is redacted and we already have it as an outlier event - if self.services.timeline.pdu_exists(event_id).await { - return Err!(Request(InvalidParam("Event was redacted and we already knew about it"))); - } - - obj - }, - Err(e) => { - return Err!(Request(InvalidParam(debug_error!( - "Signature verification failed for {event_id}: {e}" - )))) - }, - }; - - // Now that we have checked the signature and hashes we can add the eventID and - // convert to our PduEvent type - val.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); - let incoming_pdu = serde_json::from_value::( - serde_json::to_value(&val).expect("CanonicalJsonObj is a valid JsonValue"), - ) - .map_err(|_| Error::bad_database("Event is not a valid PDU."))?; - - check_room_id(room_id, &incoming_pdu)?; - - if !auth_events_known { - // 4. fetch any missing auth events doing all checks listed here starting at 1. - // These are not timeline events - // 5. Reject "due to auth events" if can't get all the auth events or some of - // the auth events are also rejected "due to auth events" - // NOTE: Step 5 is not applied anymore because it failed too often - debug!("Fetching auth events"); - Box::pin( - self.fetch_and_handle_outliers( - origin, - &incoming_pdu - .auth_events - .iter() - .map(|x| Arc::from(&**x)) - .collect::>>(), - create_event, - room_id, - &room_version_id, - ), - ) - .await; - } - - // 6. Reject "due to auth events" if the event doesn't pass auth based on the - // auth events - debug!("Checking based on auth events"); - // Build map of auth events - let mut auth_events = HashMap::with_capacity(incoming_pdu.auth_events.len()); - for id in &incoming_pdu.auth_events { - let Ok(auth_event) = self.services.timeline.get_pdu(id).await else { - warn!("Could not find auth event {id}"); - continue; - }; - - check_room_id(room_id, &auth_event)?; - - match auth_events.entry(( - auth_event.kind.to_string().into(), - auth_event - .state_key - .clone() - .expect("all auth events have state keys"), - )) { - hash_map::Entry::Vacant(v) => { - v.insert(auth_event); - }, - hash_map::Entry::Occupied(_) => { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Auth event's type and state_key combination exists multiple times.", - )); - }, - } - } - - // The original create event must be in the auth events - if !matches!( - auth_events - .get(&(StateEventType::RoomCreate, String::new())) - .map(AsRef::as_ref), - Some(_) | None - ) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Incoming event refers to wrong create event.", - )); - } - - let state_fetch = |ty: &'static StateEventType, sk: &str| { - let key = ty.with_state_key(sk); - ready(auth_events.get(&key)) - }; - - let auth_check = state_res::event_auth::auth_check( - &to_room_version(&room_version_id), - &incoming_pdu, - None, // TODO: third party invite - state_fetch, - ) - .await - .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; - - if !auth_check { - return Err!(Request(Forbidden("Auth check failed"))); - } - - trace!("Validation successful."); - - // 7. Persist the event as an outlier. - self.services - .outlier - .add_pdu_outlier(&incoming_pdu.event_id, &val); - - trace!("Added pdu as outlier."); - - Ok((Arc::new(incoming_pdu), val)) - } - - pub async fn upgrade_outlier_to_timeline_pdu( - &self, incoming_pdu: Arc, val: BTreeMap, create_event: &PduEvent, - origin: &ServerName, room_id: &RoomId, - ) -> Result> { - // Skip the PDU if we already have it as a timeline event - if let Ok(pduid) = self - .services - .timeline - .get_pdu_id(&incoming_pdu.event_id) - .await - { - return Ok(Some(pduid)); - } - - if self - .services - .pdu_metadata - .is_event_soft_failed(&incoming_pdu.event_id) - .await - { - return Err!(Request(InvalidParam("Event has been soft failed"))); - } - - debug!("Upgrading to timeline pdu"); - let timer = Instant::now(); - let room_version_id = get_room_version_id(create_event)?; - - // 10. Fetch missing state and auth chain events by calling /state_ids at - // backwards extremities doing all the checks in this list starting at 1. - // These are not timeline events. - - debug!("Resolving state at event"); - let mut state_at_incoming_event = if incoming_pdu.prev_events.len() == 1 { - self.state_at_incoming_degree_one(&incoming_pdu).await? - } else { - self.state_at_incoming_resolved(&incoming_pdu, room_id, &room_version_id) - .await? - }; - - if state_at_incoming_event.is_none() { - state_at_incoming_event = self - .fetch_state(origin, create_event, room_id, &room_version_id, &incoming_pdu.event_id) - .await?; - } - - let state_at_incoming_event = state_at_incoming_event.expect("we always set this to some above"); - let room_version = to_room_version(&room_version_id); - - debug!("Performing auth check"); - // 11. Check the auth of the event passes based on the state of the event - let state_fetch_state = &state_at_incoming_event; - let state_fetch = |k: &'static StateEventType, s: String| async move { - let shortstatekey = self.services.short.get_shortstatekey(k, &s).await.ok()?; - - let event_id = state_fetch_state.get(&shortstatekey)?; - self.services.timeline.get_pdu(event_id).await.ok() - }; - - let auth_check = state_res::event_auth::auth_check( - &room_version, - &incoming_pdu, - None, // TODO: third party invite - |k, s| state_fetch(k, s.to_owned()), - ) - .await - .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; - - if !auth_check { - return Err!(Request(Forbidden("Event has failed auth check with state at the event."))); - } - - debug!("Gathering auth events"); - let auth_events = self - .services - .state - .get_auth_events( - room_id, - &incoming_pdu.kind, - &incoming_pdu.sender, - incoming_pdu.state_key.as_deref(), - &incoming_pdu.content, - ) - .await?; - - let state_fetch = |k: &'static StateEventType, s: &str| { - let key = k.with_state_key(s); - ready(auth_events.get(&key).cloned()) - }; - - let auth_check = state_res::event_auth::auth_check( - &room_version, - &incoming_pdu, - None, // third-party invite - state_fetch, - ) - .await - .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; - - // Soft fail check before doing state res - debug!("Performing soft-fail check"); - let soft_fail = { - use RoomVersionId::*; - - !auth_check - || incoming_pdu.kind == TimelineEventType::RoomRedaction - && match room_version_id { - V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { - if let Some(redact_id) = &incoming_pdu.redacts { - !self - .services - .state_accessor - .user_can_redact(redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) - .await? - } else { - false - } - }, - _ => { - let content: RoomRedactionEventContent = incoming_pdu.get_content()?; - if let Some(redact_id) = &content.redacts { - !self - .services - .state_accessor - .user_can_redact(redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) - .await? - } else { - false - } - }, - } - }; - - // 13. Use state resolution to find new room state - - // We start looking at current room state now, so lets lock the room - trace!("Locking the room"); - let state_lock = self.services.state.mutex.lock(room_id).await; - - // Now we calculate the set of extremities this room has after the incoming - // event has been applied. We start with the previous extremities (aka leaves) - trace!("Calculating extremities"); - let mut extremities: HashSet<_> = self - .services - .state - .get_forward_extremities(room_id) - .map(ToOwned::to_owned) - .collect() - .await; - - // Remove any forward extremities that are referenced by this incoming event's - // prev_events - trace!( - "Calculated {} extremities; checking against {} prev_events", - extremities.len(), - incoming_pdu.prev_events.len() - ); - for prev_event in &incoming_pdu.prev_events { - extremities.remove(&(**prev_event)); - } - - // Only keep those extremities were not referenced yet - let mut retained = HashSet::new(); - for id in &extremities { - if !self - .services - .pdu_metadata - .is_event_referenced(room_id, id) - .await - { - retained.insert(id.clone()); - } - } - - extremities.retain(|id| retained.contains(id)); - debug!("Retained {} extremities. Compressing state", extremities.len()); - - let mut state_ids_compressed = HashSet::new(); - for (shortstatekey, id) in &state_at_incoming_event { - state_ids_compressed.insert( - self.services - .state_compressor - .compress_state_event(*shortstatekey, id) - .await, - ); - } - - let state_ids_compressed = Arc::new(state_ids_compressed); - - if incoming_pdu.state_key.is_some() { - debug!("Event is a state-event. Deriving new room state"); - - // We also add state after incoming event to the fork states - let mut state_after = state_at_incoming_event.clone(); - if let Some(state_key) = &incoming_pdu.state_key { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&incoming_pdu.kind.to_string().into(), state_key) - .await; - - let event_id = &incoming_pdu.event_id; - state_after.insert(shortstatekey, event_id.clone()); - } - - let new_room_state = self - .resolve_state(room_id, &room_version_id, state_after) - .await?; - - // Set the new room state to the resolved state - debug!("Forcing new room state"); - let HashSetCompressStateEvent { - shortstatehash, - added, - removed, - } = self - .services - .state_compressor - .save_state(room_id, new_room_state) - .await?; - - self.services - .state - .force_state(room_id, shortstatehash, added, removed, &state_lock) - .await?; - } - - // 14. Check if the event passes auth based on the "current state" of the room, - // if not soft fail it - if soft_fail { - debug!("Soft failing event"); - self.services - .timeline - .append_incoming_pdu( - &incoming_pdu, - val, - extremities.iter().map(|e| (**e).to_owned()).collect(), - state_ids_compressed, - soft_fail, - &state_lock, - ) - .await?; - - // Soft fail, we keep the event as an outlier but don't add it to the timeline - warn!("Event was soft failed: {incoming_pdu:?}"); - self.services - .pdu_metadata - .mark_event_soft_failed(&incoming_pdu.event_id); - - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event has been soft failed")); - } - - trace!("Appending pdu to timeline"); - extremities.insert(incoming_pdu.event_id.clone().into()); - - // Now that the event has passed all auth it is added into the timeline. - // We use the `state_at_event` instead of `state_after` so we accurately - // represent the state for this event. - let pdu_id = self - .services - .timeline - .append_incoming_pdu( - &incoming_pdu, - val, - extremities.into_iter().collect(), - state_ids_compressed, - soft_fail, - &state_lock, - ) - .await?; - - // Event has passed all auth/stateres checks - drop(state_lock); - debug_info!( - elapsed = ?timer.elapsed(), - "Accepted", - ); - - Ok(pdu_id) - } - - #[tracing::instrument(skip_all, name = "resolve")] - pub async fn resolve_state( - &self, room_id: &RoomId, room_version_id: &RoomVersionId, incoming_state: HashMap>, - ) -> Result>> { - debug!("Loading current room state ids"); - let current_sstatehash = self - .services - .state - .get_room_shortstatehash(room_id) - .await - .map_err(|e| err!(Database(error!("No state for {room_id:?}: {e:?}"))))?; - - let current_state_ids = self - .services - .state_accessor - .state_full_ids(current_sstatehash) - .await?; - - let fork_states = [current_state_ids, incoming_state]; - let mut auth_chain_sets = Vec::with_capacity(fork_states.len()); - for state in &fork_states { - let starting_events: Vec<&EventId> = state.values().map(Borrow::borrow).collect(); - - let auth_chain: HashSet> = self - .services - .auth_chain - .get_event_ids(room_id, &starting_events) - .await? - .into_iter() - .collect(); - - auth_chain_sets.push(auth_chain); - } - - debug!("Loading fork states"); - let fork_states: Vec>> = fork_states - .into_iter() - .stream() - .then(|fork_state| { - fork_state - .into_iter() - .stream() - .filter_map(|(k, id)| { - self.services - .short - .get_statekey_from_short(k) - .map_ok_or_else(|_| None, move |(ty, st_key)| Some(((ty, st_key), id))) - }) - .collect() - }) - .collect() - .boxed() - .await; - - debug!("Resolving state"); - let lock = self.services.globals.stateres_mutex.lock(); - - let event_fetch = |event_id| self.event_fetch(event_id); - let event_exists = |event_id| self.event_exists(event_id); - let state = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) - .await - .map_err(|e| err!(Database(error!("State resolution failed: {e:?}"))))?; - - drop(lock); - - debug!("State resolution done. Compressing state"); - let mut new_room_state = HashSet::new(); - for ((event_type, state_key), event_id) in state { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&event_type.to_string().into(), &state_key) - .await; - - let compressed = self - .services - .state_compressor - .compress_state_event(shortstatekey, &event_id) - .await; - - new_room_state.insert(compressed); - } - - Ok(Arc::new(new_room_state)) - } - - // TODO: if we know the prev_events of the incoming event we can avoid the - // request and build the state from a known point and resolve if > 1 prev_event - #[tracing::instrument(skip_all, name = "state")] - pub async fn state_at_incoming_degree_one( - &self, incoming_pdu: &Arc, - ) -> Result>>> { - let prev_event = &*incoming_pdu.prev_events[0]; - let Ok(prev_event_sstatehash) = self - .services - .state_accessor - .pdu_shortstatehash(prev_event) - .await - else { - return Ok(None); - }; - - let Ok(mut state) = self - .services - .state_accessor - .state_full_ids(prev_event_sstatehash) - .await - .log_err() - else { - return Ok(None); - }; - - debug!("Using cached state"); - let prev_pdu = self - .services - .timeline - .get_pdu(prev_event) - .await - .map_err(|e| err!(Database("Could not find prev event, but we know the state: {e:?}")))?; - - if let Some(state_key) = &prev_pdu.state_key { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&prev_pdu.kind.to_string().into(), state_key) - .await; - - state.insert(shortstatekey, Arc::from(prev_event)); - // Now it's the state after the pdu - } - - debug_assert!(!state.is_empty(), "should be returning None for empty HashMap result"); - - Ok(Some(state)) - } - - #[tracing::instrument(skip_all, name = "state")] - pub async fn state_at_incoming_resolved( - &self, incoming_pdu: &Arc, room_id: &RoomId, room_version_id: &RoomVersionId, - ) -> Result>>> { - debug!("Calculating state at event using state res"); - let mut extremity_sstatehashes = HashMap::with_capacity(incoming_pdu.prev_events.len()); - - let mut okay = true; - for prev_eventid in &incoming_pdu.prev_events { - let Ok(prev_event) = self.services.timeline.get_pdu(prev_eventid).await else { - okay = false; - break; - }; - - let Ok(sstatehash) = self - .services - .state_accessor - .pdu_shortstatehash(prev_eventid) - .await - else { - okay = false; - break; - }; - - extremity_sstatehashes.insert(sstatehash, prev_event); - } - - if !okay { - return Ok(None); - } - - let mut fork_states = Vec::with_capacity(extremity_sstatehashes.len()); - let mut auth_chain_sets = Vec::with_capacity(extremity_sstatehashes.len()); - for (sstatehash, prev_event) in extremity_sstatehashes { - let Ok(mut leaf_state) = self - .services - .state_accessor - .state_full_ids(sstatehash) - .await - else { - continue; - }; - - if let Some(state_key) = &prev_event.state_key { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&prev_event.kind.to_string().into(), state_key) - .await; - - let event_id = &prev_event.event_id; - leaf_state.insert(shortstatekey, event_id.clone()); - // Now it's the state after the pdu - } - - let mut state = StateMap::with_capacity(leaf_state.len()); - let mut starting_events = Vec::with_capacity(leaf_state.len()); - for (k, id) in &leaf_state { - if let Ok((ty, st_key)) = self - .services - .short - .get_statekey_from_short(*k) - .await - .log_err() - { - // FIXME: Undo .to_string().into() when StateMap - // is updated to use StateEventType - state.insert((ty.to_string().into(), st_key), id.clone()); - } - - starting_events.push(id.borrow()); - } - - let auth_chain: HashSet> = self - .services - .auth_chain - .get_event_ids(room_id, &starting_events) - .await? - .into_iter() - .collect(); - - auth_chain_sets.push(auth_chain); - fork_states.push(state); - } - - let lock = self.services.globals.stateres_mutex.lock(); - - let event_fetch = |event_id| self.event_fetch(event_id); - let event_exists = |event_id| self.event_exists(event_id); - let result = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) - .await - .map_err(|e| err!(Database(warn!(?e, "State resolution on prev events failed.")))); - - drop(lock); - - let Ok(new_state) = result else { - return Ok(None); - }; - - new_state - .iter() - .stream() - .then(|((event_type, state_key), event_id)| { - self.services - .short - .get_or_create_shortstatekey(event_type, state_key) - .map(move |shortstatekey| (shortstatekey, event_id.clone())) - }) - .collect() - .map(Some) - .map(Ok) - .await - } - - /// Call /state_ids to find out what the state at this pdu is. We trust the - /// server's response to some extend (sic), but we still do a lot of checks - /// on the events - #[tracing::instrument(skip(self, create_event, room_version_id))] - async fn fetch_state( - &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, - event_id: &EventId, - ) -> Result>>> { - debug!("Fetching state ids"); - let res = self - .services - .sending - .send_synapse_request( - origin, - get_room_state_ids::v1::Request { - room_id: room_id.to_owned(), - event_id: (*event_id).to_owned(), - }, - ) - .await - .inspect_err(|e| warn!("Fetching state for event failed: {e}"))?; - - debug!("Fetching state events"); - let collect = res - .pdu_ids - .iter() - .map(|x| Arc::from(&**x)) - .collect::>(); - - let state_vec = self - .fetch_and_handle_outliers(origin, &collect, create_event, room_id, room_version_id) - .boxed() - .await; - - let mut state: HashMap<_, Arc> = HashMap::with_capacity(state_vec.len()); - for (pdu, _) in state_vec { - let state_key = pdu - .state_key - .clone() - .ok_or_else(|| Error::bad_database("Found non-state pdu in state events."))?; - - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&pdu.kind.to_string().into(), &state_key) - .await; - - match state.entry(shortstatekey) { - hash_map::Entry::Vacant(v) => { - v.insert(Arc::from(&*pdu.event_id)); - }, - hash_map::Entry::Occupied(_) => { - return Err(Error::bad_database( - "State event's type and state_key combination exists multiple times.", - )) - }, - } - } - - // The original create event must still be in the state - let create_shortstatekey = self - .services - .short - .get_shortstatekey(&StateEventType::RoomCreate, "") - .await?; - - if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(&create_event.event_id) { - return Err!(Database("Incoming event refers to wrong create event.")); - } - - Ok(Some(state)) - } - - /// Find the event and auth it. Once the event is validated (steps 1 - 8) - /// it is appended to the outliers Tree. - /// - /// Returns pdu and if we fetched it over federation the raw json. - /// - /// a. Look in the main timeline (pduid_pdu tree) - /// b. Look at outlier pdu tree - /// c. Ask origin server over federation - /// d. TODO: Ask other servers over federation? - pub async fn fetch_and_handle_outliers<'a>( - &self, origin: &'a ServerName, events: &'a [Arc], create_event: &'a PduEvent, room_id: &'a RoomId, - room_version_id: &'a RoomVersionId, - ) -> Vec<(Arc, Option>)> { - let back_off = |id| match self - .services - .globals - .bad_event_ratelimiter - .write() - .expect("locked") - .entry(id) - { - hash_map::Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - hash_map::Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)), - }; - - let mut events_with_auth_events = Vec::with_capacity(events.len()); - for id in events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { - trace!("Found {id} in db"); - events_with_auth_events.push((id, Some(local_pdu), vec![])); - continue; - } - - // c. Ask origin server over federation - // We also handle its auth chain here so we don't get a stack overflow in - // handle_outlier_pdu. - let mut todo_auth_events = vec![Arc::clone(id)]; - let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); - let mut events_all = HashSet::with_capacity(todo_auth_events.len()); - while let Some(next_id) = todo_auth_events.pop() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .expect("locked") - .get(&*next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { - info!("Backing off from {next_id}"); - continue; - } - } - - if events_all.contains(&next_id) { - continue; - } - - if self.services.timeline.pdu_exists(&next_id).await { - trace!("Found {next_id} in db"); - continue; - } - - debug!("Fetching {next_id} over federation."); - match self - .services - .sending - .send_federation_request( - origin, - get_event::v1::Request { - event_id: (*next_id).to_owned(), - include_unredacted_content: None, - }, - ) - .await - { - Ok(res) => { - debug!("Got {next_id} over federation"); - let Ok((calculated_event_id, value)) = - pdu::gen_event_id_canonical_json(&res.pdu, room_version_id) - else { - back_off((*next_id).to_owned()); - continue; - }; - - if calculated_event_id != *next_id { - warn!( - "Server didn't return event id we requested: requested: {next_id}, we got \ - {calculated_event_id}. Event: {:?}", - &res.pdu - ); - } - - if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) { - for auth_event in auth_events { - if let Ok(auth_event) = serde_json::from_value(auth_event.clone().into()) { - let a: Arc = auth_event; - todo_auth_events.push(a); - } else { - warn!("Auth event id is not valid"); - } - } - } else { - warn!("Auth event list invalid"); - } - - events_in_reverse_order.push((next_id.clone(), value)); - events_all.insert(next_id); - }, - Err(e) => { - debug_error!("Failed to fetch event {next_id}: {e}"); - back_off((*next_id).to_owned()); - }, - } - } - events_with_auth_events.push((id, None, events_in_reverse_order)); - } - - let mut pdus = Vec::with_capacity(events_with_auth_events.len()); - for (id, local_pdu, events_in_reverse_order) in events_with_auth_events { - // a. Look in the main timeline (pduid_pdu tree) - // b. Look at outlier pdu tree - // (get_pdu_json checks both) - if let Some(local_pdu) = local_pdu { - trace!("Found {id} in db"); - pdus.push((local_pdu.clone(), None)); - } - - for (next_id, value) in events_in_reverse_order.into_iter().rev() { - if let Some((time, tries)) = self - .services - .globals - .bad_event_ratelimiter - .read() - .expect("locked") - .get(&*next_id) - { - // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { - debug!("Backing off from {next_id}"); - continue; - } - } - - match Box::pin(self.handle_outlier_pdu(origin, create_event, &next_id, room_id, value.clone(), true)) - .await - { - Ok((pdu, json)) => { - if next_id == *id { - pdus.push((pdu, Some(json))); - } - }, - Err(e) => { - warn!("Authentication of event {next_id} failed: {e:?}"); - back_off(next_id.into()); - }, - } - } - } - pdus - } - - #[allow(clippy::type_complexity)] - #[tracing::instrument(skip_all)] - async fn fetch_prev( - &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, - initial_set: Vec>, - ) -> Result<( - Vec>, - HashMap, (Arc, BTreeMap)>, - )> { - let mut graph: HashMap, _> = HashMap::with_capacity(initial_set.len()); - let mut eventid_info = HashMap::new(); - let mut todo_outlier_stack: Vec> = initial_set; - - let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; - - let mut amount = 0; - - while let Some(prev_event_id) = todo_outlier_stack.pop() { - self.services.server.check_running()?; - - if let Some((pdu, mut json_opt)) = self - .fetch_and_handle_outliers(origin, &[prev_event_id.clone()], create_event, room_id, room_version_id) - .boxed() - .await - .pop() - { - check_room_id(room_id, &pdu)?; - - let limit = self.services.server.config.max_fetch_prev_events; - if amount > limit { - debug_warn!("Max prev event limit reached! Limit: {limit}"); - graph.insert(prev_event_id.clone(), HashSet::new()); - continue; - } - - if json_opt.is_none() { - json_opt = self - .services - .outlier - .get_outlier_pdu_json(&prev_event_id) - .await - .ok(); - } - - if let Some(json) = json_opt { - if pdu.origin_server_ts > first_pdu_in_room.origin_server_ts { - amount = amount.saturating_add(1); - for prev_prev in &pdu.prev_events { - if !graph.contains_key(prev_prev) { - todo_outlier_stack.push(prev_prev.clone()); - } - } - - graph.insert(prev_event_id.clone(), pdu.prev_events.iter().cloned().collect()); - } else { - // Time based check failed - graph.insert(prev_event_id.clone(), HashSet::new()); - } - - eventid_info.insert(prev_event_id.clone(), (pdu, json)); - } else { - // Get json failed, so this was not fetched over federation - graph.insert(prev_event_id.clone(), HashSet::new()); - } - } else { - // Fetch and handle failed - graph.insert(prev_event_id.clone(), HashSet::new()); - } - } - - let event_fetch = |event_id| { - let origin_server_ts = eventid_info - .get(&event_id) - .cloned() - .map_or_else(|| uint!(0), |info| info.0.origin_server_ts); - - // This return value is the key used for sorting events, - // events are then sorted by power level, time, - // and lexically by event_id. - future::ok((int!(0), MilliSecondsSinceUnixEpoch(origin_server_ts))) - }; - - let sorted = state_res::lexicographical_topological_sort(&graph, &event_fetch) - .await - .map_err(|e| err!(Database(error!("Error sorting prev events: {e}"))))?; - - Ok((sorted, eventid_info)) - } - - /// Returns Ok if the acl allows the server - #[tracing::instrument(skip_all)] - pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result<()> { - let Ok(acl_event_content) = self - .services - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomServerAcl, "") - .await - .map(|c: RoomServerAclEventContent| c) - .inspect(|acl| trace!("ACL content found: {acl:?}")) - .inspect_err(|e| trace!("No ACL content found: {e:?}")) - else { - return Ok(()); - }; - - if acl_event_content.allow.is_empty() { - warn!("Ignoring broken ACL event (allow key is empty)"); - return Ok(()); - } - - if acl_event_content.is_allowed(server_name) { - trace!("server {server_name} is allowed by ACL"); - Ok(()) - } else { - debug!("Server {server_name} was denied by room ACL in {room_id}"); - Err!(Request(Forbidden("Server was denied by room ACL"))) - } - } - async fn event_exists(&self, event_id: Arc) -> bool { self.services.timeline.pdu_exists(&event_id).await } async fn event_fetch(&self, event_id: Arc) -> Option> { diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 39920219..42f44dee 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,30 +1,27 @@ -use conduit::{err, pdu::gen_event_id_canonical_json, result::FlatOk, Result}; +use conduit::{err, implement, pdu::gen_event_id_canonical_json, result::FlatOk, Result}; use ruma::{CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, RoomId}; use serde_json::value::RawValue as RawJsonValue; -impl super::Service { - pub async fn parse_incoming_pdu( - &self, pdu: &RawJsonValue, - ) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { - let value = serde_json::from_str::(pdu.get()) - .map_err(|e| err!(BadServerResponse(debug_warn!("Error parsing incoming event {e:?}"))))?; +#[implement(super::Service)] +pub async fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { + let value = serde_json::from_str::(pdu.get()) + .map_err(|e| err!(BadServerResponse(debug_warn!("Error parsing incoming event {e:?}"))))?; - let room_id: OwnedRoomId = value - .get("room_id") - .and_then(CanonicalJsonValue::as_str) - .map(RoomId::parse) - .flat_ok_or(err!(Request(InvalidParam("Invalid room_id in pdu"))))?; + let room_id: OwnedRoomId = value + .get("room_id") + .and_then(CanonicalJsonValue::as_str) + .map(RoomId::parse) + .flat_ok_or(err!(Request(InvalidParam("Invalid room_id in pdu"))))?; - let room_version_id = self - .services - .state - .get_room_version(&room_id) - .await - .map_err(|_| err!("Server is not in room {room_id}"))?; + let room_version_id = self + .services + .state + .get_room_version(&room_id) + .await + .map_err(|_| err!("Server is not in room {room_id}"))?; - let (event_id, value) = gen_event_id_canonical_json(pdu, &room_version_id) - .map_err(|e| err!(Request(InvalidParam("Could not convert event to canonical json: {e}"))))?; + let (event_id, value) = gen_event_id_canonical_json(pdu, &room_version_id) + .map_err(|e| err!(Request(InvalidParam("Could not convert event to canonical json: {e}"))))?; - Ok((event_id, value, room_id)) - } + Ok((event_id, value, room_id)) } diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs new file mode 100644 index 00000000..0c9525dd --- /dev/null +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -0,0 +1,101 @@ +use std::{ + borrow::Borrow, + collections::{HashMap, HashSet}, + sync::Arc, +}; + +use conduit::{debug, err, implement, utils::IterStream, Result}; +use futures::{FutureExt, StreamExt, TryFutureExt}; +use ruma::{ + state_res::{self, StateMap}, + EventId, RoomId, RoomVersionId, +}; + +use crate::rooms::state_compressor::CompressedStateEvent; + +#[implement(super::Service)] +#[tracing::instrument(skip_all, name = "resolve")] +pub async fn resolve_state( + &self, room_id: &RoomId, room_version_id: &RoomVersionId, incoming_state: HashMap>, +) -> Result>> { + debug!("Loading current room state ids"); + let current_sstatehash = self + .services + .state + .get_room_shortstatehash(room_id) + .await + .map_err(|e| err!(Database(error!("No state for {room_id:?}: {e:?}"))))?; + + let current_state_ids = self + .services + .state_accessor + .state_full_ids(current_sstatehash) + .await?; + + let fork_states = [current_state_ids, incoming_state]; + let mut auth_chain_sets = Vec::with_capacity(fork_states.len()); + for state in &fork_states { + let starting_events: Vec<&EventId> = state.values().map(Borrow::borrow).collect(); + + let auth_chain: HashSet> = self + .services + .auth_chain + .get_event_ids(room_id, &starting_events) + .await? + .into_iter() + .collect(); + + auth_chain_sets.push(auth_chain); + } + + debug!("Loading fork states"); + let fork_states: Vec>> = fork_states + .into_iter() + .stream() + .then(|fork_state| { + fork_state + .into_iter() + .stream() + .filter_map(|(k, id)| { + self.services + .short + .get_statekey_from_short(k) + .map_ok_or_else(|_| None, move |(ty, st_key)| Some(((ty, st_key), id))) + }) + .collect() + }) + .collect() + .boxed() + .await; + + debug!("Resolving state"); + let lock = self.services.globals.stateres_mutex.lock(); + + let event_fetch = |event_id| self.event_fetch(event_id); + let event_exists = |event_id| self.event_exists(event_id); + let state = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) + .await + .map_err(|e| err!(Database(error!("State resolution failed: {e:?}"))))?; + + drop(lock); + + debug!("State resolution done. Compressing state"); + let mut new_room_state = HashSet::new(); + for ((event_type, state_key), event_id) in state { + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&event_type.to_string().into(), &state_key) + .await; + + let compressed = self + .services + .state_compressor + .compress_state_event(shortstatekey, &event_id) + .await; + + new_room_state.insert(compressed); + } + + Ok(Arc::new(new_room_state)) +} diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs new file mode 100644 index 00000000..a200ab56 --- /dev/null +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -0,0 +1,178 @@ +use std::{ + borrow::Borrow, + collections::{HashMap, HashSet}, + sync::Arc, +}; + +use conduit::{debug, err, implement, result::LogErr, utils::IterStream, PduEvent, Result}; +use futures::{FutureExt, StreamExt}; +use ruma::{ + state_res::{self, StateMap}, + EventId, RoomId, RoomVersionId, +}; + +// TODO: if we know the prev_events of the incoming event we can avoid the +#[implement(super::Service)] +// request and build the state from a known point and resolve if > 1 prev_event +#[tracing::instrument(skip_all, name = "state")] +pub(super) async fn state_at_incoming_degree_one( + &self, incoming_pdu: &Arc, +) -> Result>>> { + let prev_event = &*incoming_pdu.prev_events[0]; + let Ok(prev_event_sstatehash) = self + .services + .state_accessor + .pdu_shortstatehash(prev_event) + .await + else { + return Ok(None); + }; + + let Ok(mut state) = self + .services + .state_accessor + .state_full_ids(prev_event_sstatehash) + .await + .log_err() + else { + return Ok(None); + }; + + debug!("Using cached state"); + let prev_pdu = self + .services + .timeline + .get_pdu(prev_event) + .await + .map_err(|e| err!(Database("Could not find prev event, but we know the state: {e:?}")))?; + + if let Some(state_key) = &prev_pdu.state_key { + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&prev_pdu.kind.to_string().into(), state_key) + .await; + + state.insert(shortstatekey, Arc::from(prev_event)); + // Now it's the state after the pdu + } + + debug_assert!(!state.is_empty(), "should be returning None for empty HashMap result"); + + Ok(Some(state)) +} + +#[implement(super::Service)] +#[tracing::instrument(skip_all, name = "state")] +pub(super) async fn state_at_incoming_resolved( + &self, incoming_pdu: &Arc, room_id: &RoomId, room_version_id: &RoomVersionId, +) -> Result>>> { + debug!("Calculating state at event using state res"); + let mut extremity_sstatehashes = HashMap::with_capacity(incoming_pdu.prev_events.len()); + + let mut okay = true; + for prev_eventid in &incoming_pdu.prev_events { + let Ok(prev_event) = self.services.timeline.get_pdu(prev_eventid).await else { + okay = false; + break; + }; + + let Ok(sstatehash) = self + .services + .state_accessor + .pdu_shortstatehash(prev_eventid) + .await + else { + okay = false; + break; + }; + + extremity_sstatehashes.insert(sstatehash, prev_event); + } + + if !okay { + return Ok(None); + } + + let mut fork_states = Vec::with_capacity(extremity_sstatehashes.len()); + let mut auth_chain_sets = Vec::with_capacity(extremity_sstatehashes.len()); + for (sstatehash, prev_event) in extremity_sstatehashes { + let Ok(mut leaf_state) = self + .services + .state_accessor + .state_full_ids(sstatehash) + .await + else { + continue; + }; + + if let Some(state_key) = &prev_event.state_key { + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&prev_event.kind.to_string().into(), state_key) + .await; + + let event_id = &prev_event.event_id; + leaf_state.insert(shortstatekey, event_id.clone()); + // Now it's the state after the pdu + } + + let mut state = StateMap::with_capacity(leaf_state.len()); + let mut starting_events = Vec::with_capacity(leaf_state.len()); + for (k, id) in &leaf_state { + if let Ok((ty, st_key)) = self + .services + .short + .get_statekey_from_short(*k) + .await + .log_err() + { + // FIXME: Undo .to_string().into() when StateMap + // is updated to use StateEventType + state.insert((ty.to_string().into(), st_key), id.clone()); + } + + starting_events.push(id.borrow()); + } + + let auth_chain: HashSet> = self + .services + .auth_chain + .get_event_ids(room_id, &starting_events) + .await? + .into_iter() + .collect(); + + auth_chain_sets.push(auth_chain); + fork_states.push(state); + } + + let lock = self.services.globals.stateres_mutex.lock(); + + let event_fetch = |event_id| self.event_fetch(event_id); + let event_exists = |event_id| self.event_exists(event_id); + let result = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) + .await + .map_err(|e| err!(Database(warn!(?e, "State resolution on prev events failed.")))); + + drop(lock); + + let Ok(new_state) = result else { + return Ok(None); + }; + + new_state + .iter() + .stream() + .then(|((event_type, state_key), event_id)| { + self.services + .short + .get_or_create_shortstatekey(event_type, state_key) + .map(move |shortstatekey| (shortstatekey, event_id.clone())) + }) + .collect() + .map(Some) + .map(Ok) + .await +} diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs new file mode 100644 index 00000000..2a1e4662 --- /dev/null +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -0,0 +1,298 @@ +use std::{ + collections::{BTreeMap, HashSet}, + sync::Arc, + time::Instant, +}; + +use conduit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; +use futures::{future::ready, StreamExt}; +use ruma::{ + api::client::error::ErrorKind, + events::{room::redaction::RoomRedactionEventContent, StateEventType, TimelineEventType}, + state_res::{self, EventTypeExt}, + CanonicalJsonValue, RoomId, RoomVersionId, ServerName, +}; + +use super::{get_room_version_id, to_room_version}; +use crate::rooms::{state_compressor::HashSetCompressStateEvent, timeline::RawPduId}; + +#[implement(super::Service)] +pub(super) async fn upgrade_outlier_to_timeline_pdu( + &self, incoming_pdu: Arc, val: BTreeMap, create_event: &PduEvent, + origin: &ServerName, room_id: &RoomId, +) -> Result> { + // Skip the PDU if we already have it as a timeline event + if let Ok(pduid) = self + .services + .timeline + .get_pdu_id(&incoming_pdu.event_id) + .await + { + return Ok(Some(pduid)); + } + + if self + .services + .pdu_metadata + .is_event_soft_failed(&incoming_pdu.event_id) + .await + { + return Err!(Request(InvalidParam("Event has been soft failed"))); + } + + debug!("Upgrading to timeline pdu"); + let timer = Instant::now(); + let room_version_id = get_room_version_id(create_event)?; + + // 10. Fetch missing state and auth chain events by calling /state_ids at + // backwards extremities doing all the checks in this list starting at 1. + // These are not timeline events. + + debug!("Resolving state at event"); + let mut state_at_incoming_event = if incoming_pdu.prev_events.len() == 1 { + self.state_at_incoming_degree_one(&incoming_pdu).await? + } else { + self.state_at_incoming_resolved(&incoming_pdu, room_id, &room_version_id) + .await? + }; + + if state_at_incoming_event.is_none() { + state_at_incoming_event = self + .fetch_state(origin, create_event, room_id, &room_version_id, &incoming_pdu.event_id) + .await?; + } + + let state_at_incoming_event = state_at_incoming_event.expect("we always set this to some above"); + let room_version = to_room_version(&room_version_id); + + debug!("Performing auth check"); + // 11. Check the auth of the event passes based on the state of the event + let state_fetch_state = &state_at_incoming_event; + let state_fetch = |k: &'static StateEventType, s: String| async move { + let shortstatekey = self.services.short.get_shortstatekey(k, &s).await.ok()?; + + let event_id = state_fetch_state.get(&shortstatekey)?; + self.services.timeline.get_pdu(event_id).await.ok() + }; + + let auth_check = state_res::event_auth::auth_check( + &room_version, + &incoming_pdu, + None, // TODO: third party invite + |k, s| state_fetch(k, s.to_owned()), + ) + .await + .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; + + if !auth_check { + return Err!(Request(Forbidden("Event has failed auth check with state at the event."))); + } + + debug!("Gathering auth events"); + let auth_events = self + .services + .state + .get_auth_events( + room_id, + &incoming_pdu.kind, + &incoming_pdu.sender, + incoming_pdu.state_key.as_deref(), + &incoming_pdu.content, + ) + .await?; + + let state_fetch = |k: &'static StateEventType, s: &str| { + let key = k.with_state_key(s); + ready(auth_events.get(&key).cloned()) + }; + + let auth_check = state_res::event_auth::auth_check( + &room_version, + &incoming_pdu, + None, // third-party invite + state_fetch, + ) + .await + .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; + + // Soft fail check before doing state res + debug!("Performing soft-fail check"); + let soft_fail = { + use RoomVersionId::*; + + !auth_check + || incoming_pdu.kind == TimelineEventType::RoomRedaction + && match room_version_id { + V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + if let Some(redact_id) = &incoming_pdu.redacts { + !self + .services + .state_accessor + .user_can_redact(redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) + .await? + } else { + false + } + }, + _ => { + let content: RoomRedactionEventContent = incoming_pdu.get_content()?; + if let Some(redact_id) = &content.redacts { + !self + .services + .state_accessor + .user_can_redact(redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) + .await? + } else { + false + } + }, + } + }; + + // 13. Use state resolution to find new room state + + // We start looking at current room state now, so lets lock the room + trace!("Locking the room"); + let state_lock = self.services.state.mutex.lock(room_id).await; + + // Now we calculate the set of extremities this room has after the incoming + // event has been applied. We start with the previous extremities (aka leaves) + trace!("Calculating extremities"); + let mut extremities: HashSet<_> = self + .services + .state + .get_forward_extremities(room_id) + .map(ToOwned::to_owned) + .collect() + .await; + + // Remove any forward extremities that are referenced by this incoming event's + // prev_events + trace!( + "Calculated {} extremities; checking against {} prev_events", + extremities.len(), + incoming_pdu.prev_events.len() + ); + for prev_event in &incoming_pdu.prev_events { + extremities.remove(&(**prev_event)); + } + + // Only keep those extremities were not referenced yet + let mut retained = HashSet::new(); + for id in &extremities { + if !self + .services + .pdu_metadata + .is_event_referenced(room_id, id) + .await + { + retained.insert(id.clone()); + } + } + + extremities.retain(|id| retained.contains(id)); + debug!("Retained {} extremities. Compressing state", extremities.len()); + + let mut state_ids_compressed = HashSet::new(); + for (shortstatekey, id) in &state_at_incoming_event { + state_ids_compressed.insert( + self.services + .state_compressor + .compress_state_event(*shortstatekey, id) + .await, + ); + } + + let state_ids_compressed = Arc::new(state_ids_compressed); + + if incoming_pdu.state_key.is_some() { + debug!("Event is a state-event. Deriving new room state"); + + // We also add state after incoming event to the fork states + let mut state_after = state_at_incoming_event.clone(); + if let Some(state_key) = &incoming_pdu.state_key { + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&incoming_pdu.kind.to_string().into(), state_key) + .await; + + let event_id = &incoming_pdu.event_id; + state_after.insert(shortstatekey, event_id.clone()); + } + + let new_room_state = self + .resolve_state(room_id, &room_version_id, state_after) + .await?; + + // Set the new room state to the resolved state + debug!("Forcing new room state"); + let HashSetCompressStateEvent { + shortstatehash, + added, + removed, + } = self + .services + .state_compressor + .save_state(room_id, new_room_state) + .await?; + + self.services + .state + .force_state(room_id, shortstatehash, added, removed, &state_lock) + .await?; + } + + // 14. Check if the event passes auth based on the "current state" of the room, + // if not soft fail it + if soft_fail { + debug!("Soft failing event"); + self.services + .timeline + .append_incoming_pdu( + &incoming_pdu, + val, + extremities.iter().map(|e| (**e).to_owned()).collect(), + state_ids_compressed, + soft_fail, + &state_lock, + ) + .await?; + + // Soft fail, we keep the event as an outlier but don't add it to the timeline + warn!("Event was soft failed: {incoming_pdu:?}"); + self.services + .pdu_metadata + .mark_event_soft_failed(&incoming_pdu.event_id); + + return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event has been soft failed")); + } + + trace!("Appending pdu to timeline"); + extremities.insert(incoming_pdu.event_id.clone().into()); + + // Now that the event has passed all auth it is added into the timeline. + // We use the `state_at_event` instead of `state_after` so we accurately + // represent the state for this event. + let pdu_id = self + .services + .timeline + .append_incoming_pdu( + &incoming_pdu, + val, + extremities.into_iter().collect(), + state_ids_compressed, + soft_fail, + &state_lock, + ) + .await?; + + // Event has passed all auth/stateres checks + drop(state_lock); + debug_info!( + elapsed = ?timer.elapsed(), + "Accepted", + ); + + Ok(pdu_id) +} From 10be3016466076a76ab0e9270dabb80e2acf1afa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 9 Nov 2024 01:09:09 +0000 Subject: [PATCH 1022/2091] split large notary requests into batches Signed-off-by: Jason Volk --- src/core/config/mod.rs | 8 +++ src/service/server_keys/acquire.rs | 4 ++ src/service/server_keys/get.rs | 4 +- src/service/server_keys/mod.rs | 2 +- src/service/server_keys/request.rs | 89 +++++++++++++++++++----------- 5 files changed, 71 insertions(+), 36 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 43cca4b8..cd9c1b38 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -582,6 +582,12 @@ pub struct Config { #[serde(default)] pub only_query_trusted_key_servers: bool, + /// Maximum number of keys to request in each trusted server query. + /// + /// default: 1024 + #[serde(default = "default_trusted_server_batch_size")] + pub trusted_server_batch_size: usize, + /// max log level for conduwuit. allows debug, info, warn, or error /// see also: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives /// **Caveat**: @@ -2062,3 +2068,5 @@ fn parallelism_scaled_u32(val: u32) -> u32 { } fn parallelism_scaled(val: usize) -> usize { val.saturating_mul(sys::available_parallelism()) } + +fn default_trusted_server_batch_size() -> usize { 256 } diff --git a/src/service/server_keys/acquire.rs b/src/service/server_keys/acquire.rs index cdaf28b4..190b4239 100644 --- a/src/service/server_keys/acquire.rs +++ b/src/service/server_keys/acquire.rs @@ -110,6 +110,10 @@ where {requested_servers} total servers; some events may not be verifiable" ); } + + for (server, key_ids) in missing { + debug_warn!(?server, ?key_ids, "missing"); + } } #[implement(super::Service)] diff --git a/src/service/server_keys/get.rs b/src/service/server_keys/get.rs index 441e33d4..dc4627f7 100644 --- a/src/service/server_keys/get.rs +++ b/src/service/server_keys/get.rs @@ -89,8 +89,8 @@ pub async fn get_verify_key(&self, origin: &ServerName, key_id: &ServerSigningKe async fn get_verify_key_from_notaries(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> Result { for notary in self.services.globals.trusted_servers() { if let Ok(server_keys) = self.notary_request(notary, origin).await { - for server_key in &server_keys { - self.add_signing_keys(server_key.clone()).await; + for server_key in server_keys.clone() { + self.add_signing_keys(server_key).await; } for server_key in server_keys { diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index dae45a51..333970df 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -7,7 +7,7 @@ mod verify; use std::{collections::BTreeMap, sync::Arc, time::Duration}; -use conduit::{implement, utils::time::timepoint_from_now, Result, Server}; +use conduit::{implement, utils::timepoint_from_now, Result, Server}; use database::{Deserialized, Json, Map}; use ruma::{ api::federation::discovery::{ServerSigningKeys, VerifyKey}, diff --git a/src/service/server_keys/request.rs b/src/service/server_keys/request.rs index 84dd2871..7078f7cd 100644 --- a/src/service/server_keys/request.rs +++ b/src/service/server_keys/request.rs @@ -1,6 +1,6 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, fmt::Debug}; -use conduit::{implement, Err, Result}; +use conduit::{debug, implement, Err, Result}; use ruma::{ api::federation::discovery::{ get_remote_server_keys, @@ -25,34 +25,57 @@ where minimum_valid_until_ts: Some(self.minimum_valid_ts()), }; - let mut server_keys = RumaBatch::new(); - for (server, key_ids) in batch { - let entry = server_keys.entry(server.into()).or_default(); - for key_id in key_ids { - entry.insert(key_id.into(), criteria.clone()); - } - } + let mut server_keys = batch.fold(RumaBatch::new(), |mut batch, (server, key_ids)| { + batch + .entry(server.into()) + .or_default() + .extend(key_ids.map(|key_id| (key_id.into(), criteria.clone()))); + + batch + }); debug_assert!(!server_keys.is_empty(), "empty batch request to notary"); - let request = Request { - server_keys, - }; - self.services - .sending - .send_federation_request(notary, request) - .await - .map(|response| response.server_keys) - .map(|keys| { - keys.into_iter() - .map(|key| key.deserialize()) - .filter_map(Result::ok) - .collect() - }) + let mut results = Vec::new(); + while let Some(batch) = server_keys + .keys() + .rev() + .take(self.services.server.config.trusted_server_batch_size) + .last() + .cloned() + { + let request = Request { + server_keys: server_keys.split_off(&batch), + }; + + debug!( + ?notary, + ?batch, + remaining = %server_keys.len(), + requesting = ?request.server_keys.keys(), + "notary request" + ); + + let response = self + .services + .sending + .send_synapse_request(notary, request) + .await? + .server_keys + .into_iter() + .map(|key| key.deserialize()) + .filter_map(Result::ok); + + results.extend(response); + } + + Ok(results) } #[implement(super::Service)] -pub async fn notary_request(&self, notary: &ServerName, target: &ServerName) -> Result> { +pub async fn notary_request( + &self, notary: &ServerName, target: &ServerName, +) -> Result + Clone + Debug + Send> { use get_remote_server_keys::v2::Request; let request = Request { @@ -60,17 +83,17 @@ pub async fn notary_request(&self, notary: &ServerName, target: &ServerName) -> minimum_valid_until_ts: self.minimum_valid_ts(), }; - self.services + let response = self + .services .sending .send_federation_request(notary, request) - .await - .map(|response| response.server_keys) - .map(|keys| { - keys.into_iter() - .map(|key| key.deserialize()) - .filter_map(Result::ok) - .collect() - }) + .await? + .server_keys + .into_iter() + .map(|key| key.deserialize()) + .filter_map(Result::ok); + + Ok(response) } #[implement(super::Service)] From 14fce384034c348c6ba35fc946b6cbffaa970f3e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 9 Nov 2024 02:42:09 +0000 Subject: [PATCH 1023/2091] cork around send_join response processing Signed-off-by: Jason Volk --- src/api/client/membership.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index bf8e5c33..2906d35b 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -877,6 +877,7 @@ async fn join_room_by_id_helper_remote( .await; info!("Going through send_join response room_state"); + let cork = services.db.cork_and_flush(); let mut state = HashMap::new(); for result in send_join_response.room_state.state.iter().map(|pdu| { services @@ -902,8 +903,10 @@ async fn join_room_by_id_helper_remote( state.insert(shortstatekey, pdu.event_id.clone()); } } + drop(cork); info!("Going through send_join response auth_chain"); + let cork = services.db.cork_and_flush(); for result in send_join_response.room_state.auth_chain.iter().map(|pdu| { services .server_keys @@ -915,6 +918,7 @@ async fn join_room_by_id_helper_remote( services.rooms.outlier.add_pdu_outlier(&event_id, &value); } + drop(cork); debug!("Running send_join auth check"); let fetch_state = &state; From cc86feded32bb94b5171462bf9ce9c7b1adde04d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 10 Nov 2024 01:49:16 +0000 Subject: [PATCH 1024/2091] bump ruma fixes for key type changes Signed-off-by: Jason Volk --- Cargo.lock | 475 +++++++++++++++++++++++++++++++-------- Cargo.toml | 2 +- src/api/client/keys.rs | 19 +- src/api/server/key.rs | 5 +- src/service/users/mod.rs | 20 +- 5 files changed, 402 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f729d3d4..a1654ff9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -43,15 +43,15 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anyhow" -version = "1.0.91" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c042108f3ed77fd83760a5fd79b53be043192bb3b9dba91d8c574c0ada7850c8" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" [[package]] name = "arc-swap" @@ -127,7 +127,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -138,7 +138,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -373,7 +373,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.85", + "syn 2.0.87", "which", ] @@ -481,9 +481,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" dependencies = [ "jobserver", "libc", @@ -569,7 +569,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -713,7 +713,7 @@ dependencies = [ "serde_json", "serde_regex", "serde_yaml", - "thiserror", + "thiserror 1.0.68", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -749,7 +749,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1047,7 +1047,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" dependencies = [ "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1074,7 +1074,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1135,6 +1135,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "dunce" version = "1.0.5" @@ -1184,7 +1195,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1276,7 +1287,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" dependencies = [ "nonempty", - "thiserror", + "thiserror 1.0.68", ] [[package]] @@ -1350,7 +1361,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1461,9 +1472,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "hdrhistogram" @@ -1537,7 +1548,7 @@ dependencies = [ "ipnet", "once_cell", "rand", - "thiserror", + "thiserror 1.0.68", "tinyvec", "tokio", "tracing", @@ -1560,7 +1571,7 @@ dependencies = [ "rand", "resolv-conf", "smallvec", - "thiserror", + "thiserror 1.0.68", "tokio", "tracing", ] @@ -1616,7 +1627,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1753,6 +1764,124 @@ dependencies = [ "tracing", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "idna" version = "0.4.0" @@ -1765,19 +1894,30 @@ dependencies = [ [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] name = "image" -version = "0.25.4" +version = "0.25.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc144d44a31d753b02ce64093d532f55ff8dc4ebf2ffb8a63c0dda691385acae" +checksum = "cd6f44aed642f18953a158afeb30206f4d50da59fbc66ecb53c66488de73563b" dependencies = [ "bytemuck", "byteorder-lite", @@ -1817,7 +1957,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.1", "serde", ] @@ -1980,7 +2120,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -1997,9 +2137,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.161" +version = "0.2.162" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" [[package]] name = "libloading" @@ -2034,6 +2174,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -2356,7 +2502,7 @@ dependencies = [ "js-sys", "once_cell", "pin-project-lite", - "thiserror", + "thiserror 1.0.68", "urlencoding", ] @@ -2399,10 +2545,10 @@ dependencies = [ "glob", "once_cell", "opentelemetry", - "ordered-float 4.4.0", + "ordered-float 4.5.0", "percent-encoding", "rand", - "thiserror", + "thiserror 1.0.68", "tokio", "tokio-stream", ] @@ -2418,9 +2564,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.4.0" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e7ccb95e240b7c9506a3d544f10d935e142cc90b0a1d56954fb44d89ad6b97" +checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e" dependencies = [ "num-traits", ] @@ -2502,7 +2648,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2595,7 +2741,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2667,7 +2813,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2696,7 +2842,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "version_check", "yansi", ] @@ -2721,7 +2867,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -2776,7 +2922,7 @@ dependencies = [ "rustc-hash 2.0.0", "rustls 0.23.16", "socket2", - "thiserror", + "thiserror 1.0.68", "tokio", "tracing", ] @@ -2793,17 +2939,18 @@ dependencies = [ "rustc-hash 2.0.0", "rustls 0.23.16", "slab", - "thiserror", + "thiserror 1.0.68", "tinyvec", "tracing", ] [[package]] name = "quinn-udp" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" +checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" dependencies = [ + "cfg_aliases", "libc", "once_cell", "socket2", @@ -2980,7 +3127,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "assign", "js_int", @@ -3002,7 +3149,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "js_int", "ruma-common", @@ -3014,7 +3161,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "as_variant", "assign", @@ -3029,7 +3176,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror", + "thiserror 2.0.1", "url", "web-time 1.1.0", ] @@ -3037,7 +3184,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "as_variant", "base64 0.22.1", @@ -3055,7 +3202,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror", + "thiserror 2.0.1", "time", "tracing", "url", @@ -3067,7 +3214,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3081,7 +3228,7 @@ dependencies = [ "ruma-macros", "serde", "serde_json", - "thiserror", + "thiserror 2.0.1", "tracing", "url", "web-time 1.1.0", @@ -3091,7 +3238,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "bytes", "http", @@ -3109,16 +3256,16 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "js_int", - "thiserror", + "thiserror 2.0.1", ] [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "js_int", "ruma-common", @@ -3128,7 +3275,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "cfg-if", "once_cell", @@ -3137,14 +3284,14 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.85", + "syn 2.0.87", "toml", ] [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "js_int", "ruma-common", @@ -3156,20 +3303,20 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "headers", "http", "http-auth", "ruma-common", - "thiserror", + "thiserror 2.0.1", "tracing", ] [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3179,13 +3326,13 @@ dependencies = [ "serde_json", "sha2", "subslice", - "thiserror", + "thiserror 2.0.1", ] [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a#dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "futures-util", "itertools 0.13.0", @@ -3194,7 +3341,7 @@ dependencies = [ "ruma-events", "serde", "serde_json", - "thiserror", + "thiserror 2.0.1", "tracing", ] @@ -3261,9 +3408,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", @@ -3358,7 +3505,7 @@ dependencies = [ "futures-util", "pin-project", "thingbuf", - "thiserror", + "thiserror 1.0.68", "unicode-segmentation", "unicode-width", ] @@ -3415,9 +3562,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" +checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" dependencies = [ "core-foundation-sys", "libc", @@ -3558,7 +3705,7 @@ dependencies = [ "rand", "serde", "serde_json", - "thiserror", + "thiserror 1.0.68", "time", "url", "uuid", @@ -3581,7 +3728,7 @@ checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -3764,7 +3911,7 @@ checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" dependencies = [ "num-bigint", "num-traits", - "thiserror", + "thiserror 1.0.68", "time", ] @@ -3815,6 +3962,12 @@ dependencies = [ "der", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strict" version = "0.2.0" @@ -3875,9 +4028,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.85" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -3899,6 +4052,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "tendril" version = "0.4.3" @@ -3922,7 +4086,7 @@ dependencies = [ "lazy-regex", "minimad", "serde", - "thiserror", + "thiserror 1.0.68", "unicode-width", ] @@ -3938,22 +4102,42 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.68", +] + +[[package]] +name = "thiserror" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07c1e40dd48a282ae8edc36c732cbc219144b87fb6a4c7316d611c6b1f06ec0c" +dependencies = [ + "thiserror-impl 2.0.1", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874aa7e446f1da8d9c3a5c95b1c5eb41d800045252121dc7f8e0ba370cee55f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", ] [[package]] @@ -4047,6 +4231,16 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -4064,9 +4258,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.0" +version = "1.41.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145f3413504347a2be84393cc8a7d2fb4d863b375909ea59f2158261aa258bbb" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" dependencies = [ "backtrace", "bytes", @@ -4088,7 +4282,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4122,7 +4316,7 @@ checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" dependencies = [ "either", "futures-util", - "thiserror", + "thiserror 1.0.68", "tokio", ] @@ -4302,7 +4496,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b47 dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", ] [[package]] @@ -4483,12 +4677,12 @@ dependencies = [ [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.3", "percent-encoding", "serde", ] @@ -4505,6 +4699,18 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "1.11.0" @@ -4570,7 +4776,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-shared", ] @@ -4604,7 +4810,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4934,6 +5140,18 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xml5ever" version = "0.18.1" @@ -4951,6 +5169,30 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -4969,7 +5211,28 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.85", + "syn 2.0.87", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", + "synstructure", ] [[package]] @@ -4978,6 +5241,28 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/Cargo.toml b/Cargo.toml index 3ac1556c..5ea6b4e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -316,7 +316,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "dd8b13ed2fa2ec4d9fe5c6fbb18e701ac4d4d08a" +rev = "67ffedabbf43e1ff6934df0fbf770b21e101406f" features = [ "compat", "rand", diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 44d9164c..53ec12f9 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -16,7 +16,7 @@ use ruma::{ federation, }, serde::Raw, - DeviceKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, + OneTimeKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, }; use serde_json::json; @@ -36,13 +36,12 @@ use crate::{ pub(crate) async fn upload_keys_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + let (sender_user, sender_device) = body.sender(); - for (key_key, key_value) in &body.one_time_keys { + for (key_id, one_time_key) in &body.one_time_keys { services .users - .add_one_time_key(sender_user, sender_device, key_key, key_value) + .add_one_time_key(sender_user, sender_device, key_id, one_time_key) .await?; } @@ -400,16 +399,16 @@ where while let Some((server, response)) = futures.next().await { if let Ok(Ok(response)) = response { - for (user, masterkey) in response.master_keys { - let (master_key_id, mut master_key) = parse_master_key(&user, &masterkey)?; + for (user, master_key) in response.master_keys { + let (master_key_id, mut master_key) = parse_master_key(&user, &master_key)?; if let Ok(our_master_key) = services .users .get_key(&master_key_id, sender_user, &user, &allowed_signatures) .await { - let (_, our_master_key) = parse_master_key(&user, &our_master_key)?; - master_key.signatures.extend(our_master_key.signatures); + let (_, mut our_master_key) = parse_master_key(&user, &our_master_key)?; + master_key.signatures.append(&mut our_master_key.signatures); } let json = serde_json::to_value(master_key).expect("to_value always works"); let raw = serde_json::from_value(json).expect("Raw::from_value always works"); @@ -467,7 +466,7 @@ fn add_unsigned_device_display_name( } pub(crate) async fn claim_keys_helper( - services: &Services, one_time_keys_input: &BTreeMap>, + services: &Services, one_time_keys_input: &BTreeMap>, ) -> Result { let mut one_time_keys = BTreeMap::new(); diff --git a/src/api/server/key.rs b/src/api/server/key.rs index 5284593d..37fffa9f 100644 --- a/src/api/server/key.rs +++ b/src/api/server/key.rs @@ -1,5 +1,4 @@ use std::{ - collections::BTreeMap, mem::take, time::{Duration, SystemTime}, }; @@ -12,7 +11,7 @@ use ruma::{ OutgoingResponse, }, serde::Raw, - MilliSecondsSinceUnixEpoch, + MilliSecondsSinceUnixEpoch, Signatures, }; /// # `GET /_matrix/key/v2/server` @@ -42,7 +41,7 @@ pub(crate) async fn get_server_keys_route(State(services): State) old_verify_keys, server_name: server_name.to_owned(), valid_until_ts: valid_until_ts(), - signatures: BTreeMap::new(), + signatures: Signatures::new(), }; let server_key = Raw::new(&server_key)?; diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index b9183e12..1f8c56df 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -12,8 +12,8 @@ use ruma::{ encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, events::{ignored_user_list::IgnoredUserListEvent, AnyToDeviceEvent, GlobalAccountDataEventType}, serde::Raw, - DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, OwnedDeviceId, OwnedDeviceKeyId, - OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, + DeviceId, KeyId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, OneTimeKeyName, OwnedDeviceId, + OwnedKeyId, OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, }; use serde_json::json; @@ -341,9 +341,9 @@ impl Service { } pub async fn add_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, one_time_key_key: &DeviceKeyId, + &self, user_id: &UserId, device_id: &DeviceId, one_time_key_key: &KeyId, one_time_key_value: &Raw, - ) -> Result<()> { + ) -> Result { // All devices have metadata // Only existing devices should be able to call this, but we shouldn't assert // either... @@ -388,8 +388,8 @@ impl Service { } pub async fn take_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, key_algorithm: &DeviceKeyAlgorithm, - ) -> Result<(OwnedDeviceKeyId, Raw)> { + &self, user_id: &UserId, device_id: &DeviceId, key_algorithm: &OneTimeKeyAlgorithm, + ) -> Result<(OwnedKeyId, Raw)> { let count = self.services.globals.next_count()?.to_be_bytes(); self.db.userid_lastonetimekeyupdate.insert(user_id, count); @@ -433,23 +433,23 @@ impl Service { pub async fn count_one_time_keys( &self, user_id: &UserId, device_id: &DeviceId, - ) -> BTreeMap { + ) -> BTreeMap { type KeyVal<'a> = ((Ignore, Ignore, &'a Unquoted), Ignore); - let mut algorithm_counts = BTreeMap::::new(); + let mut algorithm_counts = BTreeMap::::new(); let query = (user_id, device_id); self.db .onetimekeyid_onetimekeys .stream_prefix(&query) .ignore_err() .ready_for_each(|((Ignore, Ignore, device_key_id), Ignore): KeyVal<'_>| { - let device_key_id: &DeviceKeyId = device_key_id + let one_time_key_id: &OneTimeKeyId = device_key_id .as_str() .try_into() .expect("Invalid DeviceKeyID in database"); let count: &mut UInt = algorithm_counts - .entry(device_key_id.algorithm()) + .entry(one_time_key_id.algorithm()) .or_default(); *count = count.saturating_add(1_u32.into()); From 5e74391c6c94e2843f6cf18aaf0b10e2a613690c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 10 Nov 2024 02:29:45 +0000 Subject: [PATCH 1025/2091] fix config generator macro matchers Signed-off-by: Jason Volk --- src/macros/config.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/macros/config.rs b/src/macros/config.rs index 6ccdb73c..d7f11535 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -168,7 +168,7 @@ fn get_default(field: &Field) -> Option { .segments .iter() .next() - .is_none_or(|s| s.ident == "serde") + .is_none_or(|s| s.ident != "serde") { continue; } @@ -218,7 +218,7 @@ fn get_doc_default(field: &Field) -> Option { continue; }; - if path.segments.iter().next().is_none_or(|s| s.ident == "doc") { + if path.segments.iter().next().is_none_or(|s| s.ident != "doc") { continue; } @@ -261,7 +261,7 @@ fn get_doc_comment(field: &Field) -> Option { continue; }; - if path.segments.iter().next().is_none_or(|s| s.ident == "doc") { + if path.segments.iter().next().is_none_or(|s| s.ident != "doc") { continue; } From 7e087bb93c316fb52ab1b0dad77530eaa6608dfa Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 10 Nov 2024 03:25:57 +0000 Subject: [PATCH 1026/2091] Fixes for CI Signed-off-by: Jason Volk --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d299514..f59c5048 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,7 @@ permissions: jobs: tests: name: Test - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main @@ -231,7 +231,7 @@ jobs: build: name: Build - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: tests strategy: matrix: @@ -245,7 +245,7 @@ jobs: - name: Sync repository uses: actions/checkout@v4 - - uses: nixbuild/nix-quick-install-action@v28 + - uses: nixbuild/nix-quick-install-action@master - name: Restore and cache Nix store uses: nix-community/cache-nix-action@v5.1.0 @@ -508,7 +508,7 @@ jobs: docker: name: Docker publish - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 needs: build if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' env: From f290d1a9c850008ce932680c91ac5a039d23c9f7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 10 Nov 2024 08:39:30 +0000 Subject: [PATCH 1027/2091] prevent retry for missing keys later in join process Signed-off-by: Jason Volk --- src/api/client/membership.rs | 79 ++++++++++++++++++------------- src/service/server_keys/mod.rs | 28 +++++++++-- src/service/server_keys/verify.rs | 20 ++++++++ 3 files changed, 91 insertions(+), 36 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 2906d35b..97aa1c69 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -878,46 +878,59 @@ async fn join_room_by_id_helper_remote( info!("Going through send_join response room_state"); let cork = services.db.cork_and_flush(); - let mut state = HashMap::new(); - for result in send_join_response.room_state.state.iter().map(|pdu| { - services - .server_keys - .validate_and_add_event_id(pdu, &room_version_id) - }) { - let Ok((event_id, value)) = result.await else { - continue; - }; + let state = send_join_response + .room_state + .state + .iter() + .stream() + .then(|pdu| { + services + .server_keys + .validate_and_add_event_id_no_fetch(pdu, &room_version_id) + }) + .ready_filter_map(Result::ok) + .fold(HashMap::new(), |mut state, (event_id, value)| async move { + let pdu = match PduEvent::from_id_val(&event_id, value.clone()) { + Ok(pdu) => pdu, + Err(e) => { + debug_warn!("Invalid PDU in send_join response: {e:?}: {value:#?}"); + return state; + }, + }; - let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| { - debug_warn!("Invalid PDU in send_join response: {value:#?}"); - err!(BadServerResponse("Invalid PDU in send_join response: {e:?}")) - })?; + services.rooms.outlier.add_pdu_outlier(&event_id, &value); + if let Some(state_key) = &pdu.state_key { + let shortstatekey = services + .rooms + .short + .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key) + .await; + + state.insert(shortstatekey, pdu.event_id.clone()); + } + + state + }) + .await; - services.rooms.outlier.add_pdu_outlier(&event_id, &value); - if let Some(state_key) = &pdu.state_key { - let shortstatekey = services - .rooms - .short - .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key) - .await; - state.insert(shortstatekey, pdu.event_id.clone()); - } - } drop(cork); info!("Going through send_join response auth_chain"); let cork = services.db.cork_and_flush(); - for result in send_join_response.room_state.auth_chain.iter().map(|pdu| { - services - .server_keys - .validate_and_add_event_id(pdu, &room_version_id) - }) { - let Ok((event_id, value)) = result.await else { - continue; - }; + send_join_response + .room_state + .auth_chain + .iter() + .stream() + .then(|pdu| { + services + .server_keys + .validate_and_add_event_id_no_fetch(pdu, &room_version_id) + }) + .ready_filter_map(Result::ok) + .ready_for_each(|(event_id, value)| services.rooms.outlier.add_pdu_outlier(&event_id, &value)) + .await; - services.rooms.outlier.add_pdu_outlier(&event_id, &value); - } drop(cork); debug!("Running send_join auth check"); diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index 333970df..08bcefb6 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -7,13 +7,19 @@ mod verify; use std::{collections::BTreeMap, sync::Arc, time::Duration}; -use conduit::{implement, utils::timepoint_from_now, Result, Server}; +use conduit::{ + implement, + utils::{timepoint_from_now, IterStream}, + Result, Server, +}; use database::{Deserialized, Json, Map}; +use futures::StreamExt; use ruma::{ api::federation::discovery::{ServerSigningKeys, VerifyKey}, serde::Raw, signatures::{Ed25519KeyPair, PublicKeyMap, PublicKeySet}, - MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, + CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, RoomVersionId, ServerName, + ServerSigningKeyId, }; use serde_json::value::RawValue as RawJsonValue; @@ -107,7 +113,23 @@ async fn add_signing_keys(&self, new_keys: ServerSigningKeys) { } #[implement(Service)] -async fn verify_key_exists(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> bool { +pub async fn required_keys_exist(&self, object: &CanonicalJsonObject, version: &RoomVersionId) -> bool { + use ruma::signatures::required_keys; + + let Ok(required_keys) = required_keys(object, version) else { + return false; + }; + + required_keys + .iter() + .flat_map(|(server, key_ids)| key_ids.iter().map(move |key_id| (server, key_id))) + .stream() + .all(|(server, key_id)| self.verify_key_exists(server, key_id)) + .await +} + +#[implement(Service)] +pub async fn verify_key_exists(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> bool { type KeysMap<'a> = BTreeMap<&'a ServerSigningKeyId, &'a RawJsonValue>; let Ok(keys) = self diff --git a/src/service/server_keys/verify.rs b/src/service/server_keys/verify.rs index ad20fec7..c836e324 100644 --- a/src/service/server_keys/verify.rs +++ b/src/service/server_keys/verify.rs @@ -16,6 +16,26 @@ pub async fn validate_and_add_event_id( Ok((event_id, value)) } +#[implement(super::Service)] +pub async fn validate_and_add_event_id_no_fetch( + &self, pdu: &RawJsonValue, room_version: &RoomVersionId, +) -> Result<(OwnedEventId, CanonicalJsonObject)> { + let (event_id, mut value) = gen_event_id_canonical_json(pdu, room_version)?; + if !self.required_keys_exist(&value, room_version).await { + return Err!(BadServerResponse(debug_warn!( + "Event {event_id} cannot be verified: missing keys." + ))); + } + + if let Err(e) = self.verify_event(&value, Some(room_version)).await { + return Err!(BadServerResponse(debug_error!("Event {event_id} failed verification: {e:?}"))); + } + + value.insert("event_id".into(), CanonicalJsonValue::String(event_id.as_str().into())); + + Ok((event_id, value)) +} + #[implement(super::Service)] pub async fn verify_event( &self, event: &CanonicalJsonObject, room_version: Option<&RoomVersionId>, From 1efc52c4401f3237124495c7120746a8f7aa4909 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 10 Nov 2024 11:09:48 +0000 Subject: [PATCH 1028/2091] increase logging during server keys acquire Signed-off-by: Jason Volk --- src/service/server_keys/acquire.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/service/server_keys/acquire.rs b/src/service/server_keys/acquire.rs index 190b4239..1080d79e 100644 --- a/src/service/server_keys/acquire.rs +++ b/src/service/server_keys/acquire.rs @@ -4,7 +4,7 @@ use std::{ time::Duration, }; -use conduit::{debug, debug_error, debug_warn, error, implement, result::FlatOk, trace, warn}; +use conduit::{debug, debug_error, debug_warn, error, implement, info, result::FlatOk, trace, warn}; use futures::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::federation::discovery::ServerSigningKeys, serde::Raw, CanonicalJsonObject, OwnedServerName, @@ -69,7 +69,7 @@ where return; } - debug!("missing {missing_keys} keys for {missing_servers} servers locally"); + info!("{missing_keys} keys for {missing_servers} servers will be acquired"); if notary_first_always || notary_first_on_join { missing = self.acquire_notary(missing.into_iter()).await; @@ -79,7 +79,7 @@ where return; } - debug_warn!("missing {missing_keys} keys for {missing_servers} servers from all notaries first"); + warn!("missing {missing_keys} keys for {missing_servers} servers from all notaries first"); } if !notary_only { @@ -107,7 +107,7 @@ where if missing_keys > 0 { warn!( "did not obtain {missing_keys} keys for {missing_servers} servers out of {requested_keys} total keys for \ - {requested_servers} total servers; some events may not be verifiable" + {requested_servers} total servers." ); } From 24a5ecb6b4dedf39184e9c38282ab94db1b12d5d Mon Sep 17 00:00:00 2001 From: OverPhoenix Date: Sun, 10 Nov 2024 21:45:37 +0000 Subject: [PATCH 1029/2091] fix incorrect user id for non-admin invites checking --- src/api/client/membership.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 97aa1c69..bde8dee8 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1306,7 +1306,7 @@ pub(crate) async fn invite_helper( services: &Services, sender_user: &UserId, user_id: &UserId, room_id: &RoomId, reason: Option, is_direct: bool, ) -> Result<()> { - if !services.users.is_admin(user_id).await && services.globals.block_non_admin_invites() { + if !services.users.is_admin(sender_user).await && services.globals.block_non_admin_invites() { info!("User {sender_user} is not an admin and attempted to send an invite to room {room_id}"); return Err(Error::BadRequest( ErrorKind::forbidden(), From 08a4e931a0d5353edc01716e371a489f2c14dba3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 11 Nov 2024 20:12:20 +0000 Subject: [PATCH 1030/2091] supplement a from_str for FmtSpan Signed-off-by: Jason Volk --- src/core/log/fmt_span.rs | 17 +++++++++++++++++ src/core/log/mod.rs | 1 + 2 files changed, 18 insertions(+) create mode 100644 src/core/log/fmt_span.rs diff --git a/src/core/log/fmt_span.rs b/src/core/log/fmt_span.rs new file mode 100644 index 00000000..5a340d0f --- /dev/null +++ b/src/core/log/fmt_span.rs @@ -0,0 +1,17 @@ +use tracing_subscriber::fmt::format::FmtSpan; + +use crate::Result; + +#[inline] +pub fn from_str(str: &str) -> Result { + match str.to_uppercase().as_str() { + "ENTER" => Ok(FmtSpan::ENTER), + "EXIT" => Ok(FmtSpan::EXIT), + "NEW" => Ok(FmtSpan::NEW), + "CLOSE" => Ok(FmtSpan::CLOSE), + "ACTIVE" => Ok(FmtSpan::ACTIVE), + "FULL" => Ok(FmtSpan::FULL), + "NONE" => Ok(FmtSpan::NONE), + _ => Err(FmtSpan::NONE), + } +} diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 1cba236f..1c415c6a 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -1,6 +1,7 @@ pub mod capture; pub mod color; pub mod fmt; +pub mod fmt_span; mod reload; mod suppress; From 9790a6edc992d24490e19161394c3041e137331d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 11 Nov 2024 20:33:56 +0000 Subject: [PATCH 1031/2091] add unwrap_or_err to result Signed-off-by: Jason Volk --- src/core/utils/result.rs | 2 ++ src/core/utils/result/unwrap_or_err.rs | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 src/core/utils/result/unwrap_or_err.rs diff --git a/src/core/utils/result.rs b/src/core/utils/result.rs index fb1b7b95..6b11ea66 100644 --- a/src/core/utils/result.rs +++ b/src/core/utils/result.rs @@ -7,10 +7,12 @@ mod log_err; mod map_expect; mod not_found; mod unwrap_infallible; +mod unwrap_or_err; pub use self::{ debug_inspect::DebugInspect, filter::Filter, flat_ok::FlatOk, into_is_ok::IntoIsOk, log_debug_err::LogDebugErr, log_err::LogErr, map_expect::MapExpect, not_found::NotFound, unwrap_infallible::UnwrapInfallible, + unwrap_or_err::UnwrapOrErr, }; pub type Result = std::result::Result; diff --git a/src/core/utils/result/unwrap_or_err.rs b/src/core/utils/result/unwrap_or_err.rs new file mode 100644 index 00000000..69901958 --- /dev/null +++ b/src/core/utils/result/unwrap_or_err.rs @@ -0,0 +1,15 @@ +use std::convert::identity; + +use super::Result; + +/// Returns the Ok value or the Err value. Available when the Ok and Err types +/// are the same. This is a way to default the result using the specific Err +/// value rather than unwrap_or_default() using Ok's default. +pub trait UnwrapOrErr { + fn unwrap_or_err(self) -> T; +} + +impl UnwrapOrErr for Result { + #[inline] + fn unwrap_or_err(self) -> T { self.unwrap_or_else(identity::) } +} From e2afaa9f039d26b85bcd518013aa6bb80ce11866 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 11 Nov 2024 20:49:25 +0000 Subject: [PATCH 1032/2091] add config item for with_span_events Signed-off-by: Jason Volk --- src/core/config/mod.rs | 9 +++++++++ src/main/tracing.rs | 8 ++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index cd9c1b38..eddab2fe 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -606,6 +606,12 @@ pub struct Config { #[serde(default = "true_fn", alias = "log_colours")] pub log_colors: bool, + /// configures the span events which will be outputted with the log + /// + /// default: "none" + #[serde(default = "default_log_span_events")] + pub log_span_events: String, + /// OpenID token expiration/TTL in seconds /// /// These are the OpenID tokens that are primarily used for Matrix account @@ -1958,6 +1964,9 @@ pub fn default_log() -> String { .to_owned() } +#[must_use] +pub fn default_log_span_events() -> String { "none".into() } + fn default_notification_push_path() -> String { "/_matrix/push/v1/notify".to_owned() } fn default_openid_token_ttl() -> u64 { 60 * 60 } diff --git a/src/main/tracing.rs b/src/main/tracing.rs index 9b4ad659..c28fef6b 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -3,7 +3,8 @@ use std::sync::Arc; use conduit::{ config::Config, debug_warn, err, - log::{capture, LogLevelReloadHandles}, + log::{capture, fmt_span, LogLevelReloadHandles}, + result::UnwrapOrErr, Result, }; use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; @@ -18,7 +19,10 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla let reload_handles = LogLevelReloadHandles::default(); let console_filter = EnvFilter::try_new(&config.log).map_err(|e| err!(Config("log", "{e}.")))?; - let console_layer = tracing_subscriber::fmt::Layer::new().with_ansi(config.log_colors); + let console_span_events = fmt_span::from_str(&config.log_span_events).unwrap_or_err(); + let console_layer = tracing_subscriber::fmt::Layer::new() + .with_ansi(config.log_colors) + .with_span_events(console_span_events); let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); reload_handles.add("console", Box::new(console_reload_handle)); From 61174dd0d3632f551735bea9c8ea22c0bf218427 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 11 Nov 2024 21:27:40 +0000 Subject: [PATCH 1033/2091] check if lazyset already contains user prior to querying Signed-off-by: Jason Volk --- src/api/client/message.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index e8306de9..cc636511 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -192,6 +192,10 @@ pub(crate) async fn update_lazy( return lazy; } + if lazy.contains(event.sender()) { + return lazy; + } + if !services .rooms .lazy_loading From 396233304328c75d1271465f28f55e4121e956b4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 11 Nov 2024 05:00:29 +0000 Subject: [PATCH 1034/2091] partially revert e507c3130673099692143a59adc30a414ef6ca54 Signed-off-by: Jason Volk --- src/api/client/context.rs | 6 ++---- src/api/client/message.rs | 5 +---- src/api/client/relations.rs | 1 - src/api/client/threads.rs | 1 - src/api/server/backfill.rs | 2 +- src/service/rooms/pdu_metadata/data.rs | 2 +- src/service/rooms/threads/mod.rs | 2 +- src/service/rooms/timeline/data.rs | 12 +++++++----- 8 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index f5f981ba..4359ae12 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -82,7 +82,7 @@ pub(crate) async fn get_context_route( let events_before: Vec<_> = services .rooms .timeline - .pdus_rev(Some(sender_user), room_id, Some(base_token.saturating_sub(1))) + .pdus_rev(Some(sender_user), room_id, Some(base_token)) .await? .ready_filter_map(|item| event_filter(item, filter)) .filter_map(|item| ignored_filter(&services, item, sender_user)) @@ -94,7 +94,7 @@ pub(crate) async fn get_context_route( let events_after: Vec<_> = services .rooms .timeline - .pdus(Some(sender_user), room_id, Some(base_token.saturating_add(1))) + .pdus(Some(sender_user), room_id, Some(base_token)) .await? .ready_filter_map(|item| event_filter(item, filter)) .filter_map(|item| ignored_filter(&services, item, sender_user)) @@ -169,14 +169,12 @@ pub(crate) async fn get_context_route( start: events_before .last() .map(at!(0)) - .map(|count| count.saturating_sub(1)) .as_ref() .map(ToString::to_string), end: events_after .last() .map(at!(0)) - .map(|count| count.saturating_add(1)) .as_ref() .map(ToString::to_string), diff --git a/src/api/client/message.rs b/src/api/client/message.rs index cc636511..88453de0 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -138,10 +138,7 @@ pub(crate) async fn get_message_events_route( let start_token = events.first().map(at!(0)).unwrap_or(from); - let next_token = events - .last() - .map(at!(0)) - .map(|count| count.saturating_inc(body.dir)); + let next_token = events.last().map(at!(0)); if !cfg!(feature = "element_hacks") { if let Some(next_token) = next_token { diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index ee62dbfc..902e6be6 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -150,7 +150,6 @@ async fn paginate_relations_with_filter( Direction::Backward => events.first(), } .map(at!(0)) - .map(|count| count.saturating_inc(dir)) .as_ref() .map(ToString::to_string); diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index 8d4e399b..906f779d 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -46,7 +46,6 @@ pub(crate) async fn get_threads_route( .last() .filter(|_| threads.len() >= limit) .map(at!(0)) - .map(|count| count.saturating_sub(1)) .as_ref() .map(ToString::to_string), diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 2858d9fd..b0bd48e8 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -55,7 +55,7 @@ pub(crate) async fn get_backfill_route( pdus: services .rooms .timeline - .pdus_rev(None, &body.room_id, Some(from)) + .pdus_rev(None, &body.room_id, Some(from.saturating_add(1))) .await? .take(limit) .filter_map(|(_, pdu)| async move { diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index f3e1ced8..b06e988e 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -57,7 +57,7 @@ impl Data { ) -> impl Stream + Send + '_ { let mut current = ArrayVec::::new(); current.extend(target.to_be_bytes()); - current.extend(from.into_unsigned().to_be_bytes()); + current.extend(from.saturating_inc(dir).into_unsigned().to_be_bytes()); let current = current.as_slice(); match dir { Direction::Forward => self.tofrom_relation.raw_keys_from(current).boxed(), diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index fcc629e1..5821f279 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -132,7 +132,7 @@ impl Service { let current: RawPduId = PduId { shortroomid, - shorteventid, + shorteventid: shorteventid.saturating_sub(1), } .into(); diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 7f1873ab..22a6c1d0 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -13,7 +13,7 @@ use conduit::{ }; use database::{Database, Deserialized, Json, KeyVal, Map}; use futures::{Stream, StreamExt}; -use ruma::{CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; +use ruma::{api::Direction, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; use tokio::sync::Mutex; use super::{PduId, RawPduId}; @@ -205,7 +205,9 @@ impl Data { pub(super) async fn pdus_rev<'a>( &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, until: PduCount, ) -> Result + Send + 'a> { - let current = self.count_to_id(room_id, until).await?; + let current = self + .count_to_id(room_id, until, Direction::Backward) + .await?; let prefix = current.shortroomid(); let stream = self .pduid_pdu @@ -220,7 +222,7 @@ impl Data { pub(super) async fn pdus<'a>( &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, from: PduCount, ) -> Result + Send + 'a> { - let current = self.count_to_id(room_id, from).await?; + let current = self.count_to_id(room_id, from, Direction::Forward).await?; let prefix = current.shortroomid(); let stream = self .pduid_pdu @@ -267,7 +269,7 @@ impl Data { } } - async fn count_to_id(&self, room_id: &RoomId, shorteventid: PduCount) -> Result { + async fn count_to_id(&self, room_id: &RoomId, shorteventid: PduCount, dir: Direction) -> Result { let shortroomid: ShortRoomId = self .services .short @@ -278,7 +280,7 @@ impl Data { // +1 so we don't send the base event let pdu_id = PduId { shortroomid, - shorteventid, + shorteventid: shorteventid.saturating_inc(dir), }; Ok(pdu_id.into()) From 999d731a65fe8f1313d6fe63d5139ee9f357a820 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 11 Nov 2024 22:18:14 +0000 Subject: [PATCH 1035/2091] move err macro visitor out-of-line; reduce codegen Signed-off-by: Jason Volk --- src/core/error/err.rs | 72 +++++++++++++++++++++++-------------------- src/core/error/mod.rs | 3 +- 2 files changed, 40 insertions(+), 35 deletions(-) diff --git a/src/core/error/err.rs b/src/core/error/err.rs index baeb992d..a24441e0 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -111,12 +111,8 @@ macro_rules! err { #[macro_export] macro_rules! err_log { ($out:ident, $level:ident, $($fields:tt)+) => {{ - use std::{fmt, fmt::Write}; - use $crate::tracing::{ - callsite, callsite2, level_enabled, metadata, valueset, Callsite, Event, __macro_support, - __tracing_log, - field::{Field, ValueSet, Visit}, + callsite, callsite2, metadata, valueset, Callsite, Level, }; @@ -134,34 +130,7 @@ macro_rules! err_log { fields: $($fields)+, }; - let visit = &mut |vs: ValueSet<'_>| { - struct Visitor<'a>(&'a mut String); - impl Visit for Visitor<'_> { - #[inline] - fn record_debug(&mut self, field: &Field, val: &dyn fmt::Debug) { - if field.name() == "message" { - write!(self.0, "{:?}", val).expect("stream error"); - } else { - write!(self.0, " {}={:?}", field.name(), val).expect("stream error"); - } - } - } - - let meta = __CALLSITE.metadata(); - let enabled = level_enabled!(LEVEL) && { - let interest = __CALLSITE.interest(); - !interest.is_never() && __macro_support::__is_enabled(meta, interest) - }; - - if enabled { - Event::dispatch(meta, &vs); - } - - __tracing_log!(LEVEL, __CALLSITE, &vs); - vs.record(&mut Visitor(&mut $out)); - }; - - (visit)(valueset!(__CALLSITE.metadata().fields(), $($fields)+)); + ($crate::error::visit)(&mut $out, LEVEL, &__CALLSITE, &mut valueset!(__CALLSITE.metadata().fields(), $($fields)+)); ($out).into() }} } @@ -192,3 +161,40 @@ macro_rules! err_lev { $crate::tracing::Level::ERROR }; } + +use std::{fmt, fmt::Write}; + +use tracing::{ + level_enabled, Callsite, Event, __macro_support, __tracing_log, + callsite::DefaultCallsite, + field::{Field, ValueSet, Visit}, + Level, +}; + +struct Visitor<'a>(&'a mut String); + +impl Visit for Visitor<'_> { + #[inline] + fn record_debug(&mut self, field: &Field, val: &dyn fmt::Debug) { + if field.name() == "message" { + write!(self.0, "{val:?}").expect("stream error"); + } else { + write!(self.0, " {}={val:?}", field.name()).expect("stream error"); + } + } +} + +pub fn visit(out: &mut String, level: Level, __callsite: &'static DefaultCallsite, vs: &mut ValueSet<'_>) { + let meta = __callsite.metadata(); + let enabled = level_enabled!(level) && { + let interest = __callsite.interest(); + !interest.is_never() && __macro_support::__is_enabled(meta, interest) + }; + + if enabled { + Event::dispatch(meta, vs); + } + + __tracing_log!(level, __callsite, vs); + vs.record(&mut Visitor(out)); +} diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 302d0f87..35bf9800 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -6,8 +6,7 @@ mod serde; use std::{any::Any, borrow::Cow, convert::Infallible, fmt, sync::PoisonError}; -pub use self::log::*; -use crate::error; +pub use self::{err::visit, log::*}; #[derive(thiserror::Error)] pub enum Error { From 86694f2d1d55605af2058b5347c71ebf977c5daf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 12 Nov 2024 08:01:23 +0000 Subject: [PATCH 1036/2091] move non-generic code out of generic; reduce codegen Signed-off-by: Jason Volk --- src/api/router/args.rs | 77 +++++++++++++++--------------- src/service/sending/send.rs | 93 ++++++++++++++++++++----------------- 2 files changed, 90 insertions(+), 80 deletions(-) diff --git a/src/api/router/args.rs b/src/api/router/args.rs index 4c0aff4c..0b693956 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -66,6 +66,15 @@ where } } +impl Deref for Args +where + T: IncomingRequest + Send + Sync + 'static, +{ + type Target = T; + + fn deref(&self) -> &Self::Target { &self.body } +} + #[async_trait] impl FromRequest for Args where @@ -78,7 +87,7 @@ where let mut json_body = serde_json::from_slice::(&request.body).ok(); let auth = auth::auth(services, &mut request, json_body.as_ref(), &T::METADATA).await?; Ok(Self { - body: make_body::(services, &mut request, &mut json_body, &auth)?, + body: make_body::(services, &mut request, json_body.as_mut(), &auth)?, origin: auth.origin, sender_user: auth.sender_user, sender_device: auth.sender_device, @@ -88,20 +97,11 @@ where } } -impl Deref for Args -where - T: IncomingRequest + Send + Sync + 'static, -{ - type Target = T; - - fn deref(&self) -> &Self::Target { &self.body } -} - fn make_body( - services: &Services, request: &mut Request, json_body: &mut Option, auth: &Auth, + services: &Services, request: &mut Request, json_body: Option<&mut CanonicalJsonValue>, auth: &Auth, ) -> Result where - T: IncomingRequest + Send + Sync + 'static, + T: IncomingRequest, { let body = take_body(services, request, json_body, auth); let http_request = into_http_request(request, body); @@ -125,36 +125,37 @@ fn into_http_request(request: &Request, body: Bytes) -> hyper::Request { http_request } +#[allow(clippy::needless_pass_by_value)] fn take_body( - services: &Services, request: &mut Request, json_body: &mut Option, auth: &Auth, + services: &Services, request: &mut Request, json_body: Option<&mut CanonicalJsonValue>, auth: &Auth, ) -> Bytes { - if let Some(CanonicalJsonValue::Object(json_body)) = json_body { - let user_id = auth.sender_user.clone().unwrap_or_else(|| { - let server_name = services.globals.server_name(); - UserId::parse_with_server_name(EMPTY, server_name).expect("valid user_id") + let Some(CanonicalJsonValue::Object(json_body)) = json_body else { + return mem::take(&mut request.body); + }; + + let user_id = auth.sender_user.clone().unwrap_or_else(|| { + let server_name = services.globals.server_name(); + UserId::parse_with_server_name(EMPTY, server_name).expect("valid user_id") + }); + + let uiaa_request = json_body + .get("auth") + .and_then(CanonicalJsonValue::as_object) + .and_then(|auth| auth.get("session")) + .and_then(CanonicalJsonValue::as_str) + .and_then(|session| { + services + .uiaa + .get_uiaa_request(&user_id, auth.sender_device.as_deref(), session) }); - let uiaa_request = json_body - .get("auth") - .and_then(CanonicalJsonValue::as_object) - .and_then(|auth| auth.get("session")) - .and_then(CanonicalJsonValue::as_str) - .and_then(|session| { - services - .uiaa - .get_uiaa_request(&user_id, auth.sender_device.as_deref(), session) - }); - - if let Some(CanonicalJsonValue::Object(initial_request)) = uiaa_request { - for (key, value) in initial_request { - json_body.entry(key).or_insert(value); - } + if let Some(CanonicalJsonValue::Object(initial_request)) = uiaa_request { + for (key, value) in initial_request { + json_body.entry(key).or_insert(value); } - - let mut buf = BytesMut::new().writer(); - serde_json::to_writer(&mut buf, &json_body).expect("value serialization can't fail"); - buf.into_inner().freeze() - } else { - mem::take(&mut request.body) } + + let mut buf = BytesMut::new().writer(); + serde_json::to_writer(&mut buf, &json_body).expect("value serialization can't fail"); + buf.into_inner().freeze() } diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 939d6e73..5bf48aaa 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,5 +1,6 @@ -use std::{fmt::Debug, mem}; +use std::mem; +use bytes::Bytes; use conduit::{ debug, debug_error, debug_warn, err, error::inspect_debug_log, implement, trace, utils::string::EMPTY, Err, Error, Result, @@ -23,10 +24,10 @@ use crate::{ }; impl super::Service { - #[tracing::instrument(skip(self, client, req), name = "send")] - pub async fn send(&self, client: &Client, dest: &ServerName, req: T) -> Result + #[tracing::instrument(skip(self, client, request), name = "send")] + pub async fn send(&self, client: &Client, dest: &ServerName, request: T) -> Result where - T: OutgoingRequest + Debug + Send, + T: OutgoingRequest + Send, { if !self.server.config.allow_federation { return Err!(Config("allow_federation", "Federation is disabled.")); @@ -42,7 +43,8 @@ impl super::Service { } let actual = self.services.resolver.get_actual_dest(dest).await?; - let request = self.prepare::(dest, &actual, req).await?; + let request = into_http_request::(&actual, request)?; + let request = self.prepare(dest, request)?; self.execute::(dest, &actual, request, client).await } @@ -50,7 +52,7 @@ impl super::Service { &self, dest: &ServerName, actual: &ActualDest, request: Request, client: &Client, ) -> Result where - T: OutgoingRequest + Debug + Send, + T: OutgoingRequest + Send, { let url = request.url().clone(); let method = request.method().clone(); @@ -58,25 +60,14 @@ impl super::Service { debug!(?method, ?url, "Sending request"); match client.execute(request).await { Ok(response) => handle_response::(&self.services.resolver, dest, actual, &method, &url, response).await, - Err(error) => handle_error::(dest, actual, &method, &url, error), + Err(error) => Err(handle_error(actual, &method, &url, error).expect_err("always returns error")), } } - async fn prepare(&self, dest: &ServerName, actual: &ActualDest, req: T) -> Result - where - T: OutgoingRequest + Debug + Send, - { - const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_11]; - const SATIR: SendAccessToken<'_> = SendAccessToken::IfRequired(EMPTY); + fn prepare(&self, dest: &ServerName, mut request: http::Request>) -> Result { + self.sign_request(&mut request, dest); - trace!("Preparing request"); - let mut http_request = req - .try_into_http_request::>(actual.string().as_str(), SATIR, &VERSIONS) - .map_err(|e| err!(BadServerResponse("Invalid destination: {e:?}")))?; - - self.sign_request(&mut http_request, dest); - - let request = Request::try_from(http_request)?; + let request = Request::try_from(request)?; self.validate_url(request.url())?; Ok(request) @@ -96,11 +87,31 @@ impl super::Service { async fn handle_response( resolver: &resolver::Service, dest: &ServerName, actual: &ActualDest, method: &Method, url: &Url, - mut response: Response, + response: Response, ) -> Result where - T: OutgoingRequest + Debug + Send, + T: OutgoingRequest + Send, { + let response = into_http_response(dest, actual, method, url, response).await?; + let result = T::IncomingResponse::try_from_http_response(response); + + if result.is_ok() && !actual.cached { + resolver.set_cached_destination( + dest.to_owned(), + CachedDest { + dest: actual.dest.clone(), + host: actual.host.clone(), + expire: CachedDest::default_expire(), + }, + ); + } + + result.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}"))) +} + +async fn into_http_response( + dest: &ServerName, actual: &ActualDest, method: &Method, url: &Url, mut response: Response, +) -> Result> { let status = response.status(); trace!( ?status, ?method, @@ -113,6 +124,7 @@ where let mut http_response_builder = http::Response::builder() .status(status) .version(response.version()); + mem::swap( response.headers_mut(), http_response_builder @@ -137,27 +149,10 @@ where return Err(Error::Federation(dest.to_owned(), RumaError::from_http_response(http_response))); } - let response = T::IncomingResponse::try_from_http_response(http_response); - if response.is_ok() && !actual.cached { - resolver.set_cached_destination( - dest.to_owned(), - CachedDest { - dest: actual.dest.clone(), - host: actual.host.clone(), - expire: CachedDest::default_expire(), - }, - ); - } - - response.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}"))) + Ok(http_response) } -fn handle_error( - _dest: &ServerName, actual: &ActualDest, method: &Method, url: &Url, mut e: reqwest::Error, -) -> Result -where - T: OutgoingRequest + Debug + Send, -{ +fn handle_error(actual: &ActualDest, method: &Method, url: &Url, mut e: reqwest::Error) -> Result { if e.is_timeout() || e.is_connect() { e = e.without_url(); debug_warn!("{e:?}"); @@ -246,3 +241,17 @@ fn sign_request(&self, http_request: &mut http::Request>, dest: &ServerN debug_assert!(authorization.is_none(), "Authorization header already present"); } + +fn into_http_request(actual: &ActualDest, request: T) -> Result>> +where + T: OutgoingRequest + Send, +{ + const VERSIONS: [MatrixVersion; 1] = [MatrixVersion::V1_11]; + const SATIR: SendAccessToken<'_> = SendAccessToken::IfRequired(EMPTY); + + let http_request = request + .try_into_http_request::>(actual.string().as_str(), SATIR, &VERSIONS) + .map_err(|e| err!(BadServerResponse("Invalid destination: {e:?}")))?; + + Ok(http_request) +} From c59f474aff0dbd96d2096d6d163629a7ecf460b5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 12 Nov 2024 05:01:11 +0000 Subject: [PATCH 1037/2091] fixes for gh workflow Signed-off-by: Jason Volk --- .github/workflows/ci.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f59c5048..2d253f69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,11 +76,10 @@ jobs: # large docker images sudo docker image prune --all --force || true # large packages - sudo apt-get purge -y '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true - sudo apt-get autoremove -y + sudo apt-get purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true sudo apt-get clean # large folders - sudo rm -rf /var/lib/apt/lists/* /usr/local/games /usr/local/sqlpackage /usr/local/.ghcup /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/local/share/vcpkg /usr/local/lib/python* /usr/local/lib/node_modules /usr/local/julia* /opt/mssql-tools /etc/skel /usr/share/vim /usr/share/postgresql /usr/share/man /usr/share/apache-maven-* /usr/share/R /usr/share/alsa /usr/share/miniconda /usr/share/grub /usr/share/gradle-* /usr/share/locale /usr/share/texinfo /usr/share/kotlinc /usr/share/swift /usr/share/doc /usr/share/az_9.3.0 /usr/share/sbt /usr/share/ri /usr/share/icons /usr/share/java /usr/share/fonts /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/R /usr/lib/postgresql /usr/lib/heroku /usr/lib/gcc + sudo rm -rf /var/lib/apt/lists/* /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/local/share/vcpkg /usr/local/julia* /opt/mssql-tools /usr/share/vim /usr/share/postgresql /usr/share/apache-maven-* /usr/share/R /usr/share/alsa /usr/share/miniconda /usr/share/grub /usr/share/gradle-* /usr/share/locale /usr/share/texinfo /usr/share/kotlinc /usr/share/swift /usr/share/sbt /usr/share/ri /usr/share/icons /usr/share/java /usr/share/fonts /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/R /usr/lib/postgresql /usr/lib/heroku set -o pipefail - name: Sync repository From feefa43e65e56f6d23fa96981128841fef609414 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 12 Nov 2024 22:01:32 +0000 Subject: [PATCH 1038/2091] add pretty/si-unit byte size parsing/printing utils Signed-off-by: Jason Volk --- Cargo.lock | 7 +++++++ Cargo.toml | 3 +++ src/core/Cargo.toml | 1 + src/core/utils/bytes.rs | 30 +++++++++++++++++++++++++++++- 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index a1654ff9..51571264 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -458,6 +458,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +[[package]] +name = "bytesize" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" + [[package]] name = "bzip2-sys" version = "0.1.11+1.0.8" @@ -683,6 +689,7 @@ dependencies = [ "arrayvec", "axum", "bytes", + "bytesize", "cargo_toml", "checked_ops", "chrono", diff --git a/Cargo.toml b/Cargo.toml index 5ea6b4e0..0173e7cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -466,6 +466,9 @@ version = "1.0.36" [workspace.dependencies.proc-macro2] version = "1.0.89" +[workspace.dependencies.bytesize] +version = "1.3.0" + # # Patches # diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 4fe413e9..b93f9a77 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -57,6 +57,7 @@ argon2.workspace = true arrayvec.workspace = true axum.workspace = true bytes.workspace = true +bytesize.workspace = true cargo_toml.workspace = true checked_ops.workspace = true chrono.workspace = true diff --git a/src/core/utils/bytes.rs b/src/core/utils/bytes.rs index e8975a49..441ba422 100644 --- a/src/core/utils/bytes.rs +++ b/src/core/utils/bytes.rs @@ -1,4 +1,32 @@ -use crate::Result; +use bytesize::ByteSize; + +use crate::{err, Result}; + +/// Parse a human-writable size string w/ si-unit suffix into integer +#[inline] +pub fn from_str(str: &str) -> Result { + let bytes: ByteSize = str + .parse() + .map_err(|e| err!(Arithmetic("Failed to parse byte size: {e}")))?; + + let bytes: usize = bytes + .as_u64() + .try_into() + .map_err(|e| err!(Arithmetic("Failed to convert u64 to usize: {e}")))?; + + Ok(bytes) +} + +/// Output a human-readable size string w/ si-unit suffix +#[inline] +#[must_use] +pub fn pretty(bytes: usize) -> String { + const SI_UNITS: bool = true; + + let bytes: u64 = bytes.try_into().expect("failed to convert usize to u64"); + + bytesize::to_string(bytes, SI_UNITS) +} #[inline] #[must_use] From 68582dd868032944a794f4eb7bfa2e71d29891f5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 13 Nov 2024 00:59:53 +0000 Subject: [PATCH 1039/2091] add parallel query for current membership state Signed-off-by: Jason Volk --- src/service/rooms/state_cache/mod.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 11684eab..6e330fdc 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -10,7 +10,7 @@ use conduit::{ warn, Result, }; use database::{serialize_to_vec, Deserialized, Ignore, Interfix, Json, Map}; -use futures::{stream::iter, Stream, StreamExt}; +use futures::{future::join4, stream::iter, Stream, StreamExt}; use itertools::Itertools; use ruma::{ events::{ @@ -566,6 +566,24 @@ impl Service { self.db.userroomid_leftstate.qry(&key).await.is_ok() } + pub async fn user_membership(&self, user_id: &UserId, room_id: &RoomId) -> Option { + let states = join4( + self.is_joined(user_id, room_id), + self.is_left(user_id, room_id), + self.is_invited(user_id, room_id), + self.once_joined(user_id, room_id), + ) + .await; + + match states { + (true, ..) => Some(MembershipState::Join), + (_, true, ..) => Some(MembershipState::Leave), + (_, _, true, ..) => Some(MembershipState::Invite), + (false, false, false, true) => Some(MembershipState::Ban), + _ => None, + } + } + #[tracing::instrument(skip(self), level = "debug")] pub fn servers_invite_via<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { type KeyVal<'a> = (Ignore, Vec<&'a ServerName>); From 77fab2c323b65d7d97e78dcbee946e7860cf3d1d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 13 Nov 2024 01:01:00 +0000 Subject: [PATCH 1040/2091] use ruma visibility enum in directory interface Signed-off-by: Jason Volk --- src/service/rooms/directory/mod.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index f366ffe2..63ed3519 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use conduit::{implement, utils::stream::TryIgnore, Result}; use database::Map; use futures::Stream; -use ruma::RoomId; +use ruma::{api::client::room::Visibility, RoomId}; pub struct Service { db: Data, @@ -32,7 +32,16 @@ pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_i pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id); } #[implement(Service)] -pub async fn is_public_room(&self, room_id: &RoomId) -> bool { self.db.publicroomids.get(room_id).await.is_ok() } +pub fn public_rooms(&self) -> impl Stream + Send { self.db.publicroomids.keys().ignore_err() } #[implement(Service)] -pub fn public_rooms(&self) -> impl Stream + Send { self.db.publicroomids.keys().ignore_err() } +pub async fn is_public_room(&self, room_id: &RoomId) -> bool { self.visibility(room_id).await == Visibility::Public } + +#[implement(Service)] +pub async fn visibility(&self, room_id: &RoomId) -> Visibility { + if self.db.publicroomids.get(room_id).await.is_ok() { + Visibility::Public + } else { + Visibility::Private + } +} From 004be3bf00f3d0aa22bb07e03bd6af146ad67c7b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 13 Nov 2024 05:28:15 +0000 Subject: [PATCH 1041/2091] prepare utf-8 check bypass for database deserializer Signed-off-by: Jason Volk --- src/database/de.rs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/database/de.rs b/src/database/de.rs index d7dc1102..f8a038ef 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -277,7 +277,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { fn deserialize_str>(self, visitor: V) -> Result { let input = self.record_next(); - let out = string::str_from_bytes(input)?; + let out = deserialize_str(input)?; visitor.visit_borrowed_str(out) } @@ -360,3 +360,18 @@ impl<'a, 'de: 'a> de::MapAccess<'de> for &'a mut Deserializer<'de> { seed.deserialize(&mut **self) } } + +// activate when stable; too soon now +//#[cfg(debug_assertions)] +#[inline] +fn deserialize_str(input: &[u8]) -> Result<&str> { string::str_from_bytes(input) } + +//#[cfg(not(debug_assertions))] +#[cfg(disable)] +#[inline] +fn deserialize_str(input: &[u8]) -> Result<&str> { + // SAFETY: Strings were written by the serializer to the database. Assuming no + // database corruption, the string will be valid. Database corruption is + // detected via rocksdb checksums. + unsafe { std::str::from_utf8_unchecked(input) } +} From 6ffdc1b2a654b2225b7ee6563e1defbf5019d32d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 13 Nov 2024 22:01:46 +0000 Subject: [PATCH 1042/2091] bump serde, image, loole, termimad etc Signed-off-by: Jason Volk --- Cargo.lock | 124 +++++++++++++++++++++++++++++------------------------ Cargo.toml | 8 ++-- 2 files changed, 71 insertions(+), 61 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 51571264..0e1845da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -487,9 +487,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.37" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40545c26d092346d8a8dab71ee48e7685a7a9cba76e634790c215b41a4a7b4cf" +checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8" dependencies = [ "jobserver", "libc", @@ -548,9 +548,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" +checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" dependencies = [ "clap_builder", "clap_derive", @@ -558,9 +558,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.20" +version = "4.5.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" +checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" dependencies = [ "anstyle", "clap_lex", @@ -580,9 +580,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" [[package]] name = "cmake" @@ -720,7 +720,7 @@ dependencies = [ "serde_json", "serde_regex", "serde_yaml", - "thiserror 1.0.68", + "thiserror 1.0.69", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -913,9 +913,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" +checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" dependencies = [ "libc", ] @@ -1294,7 +1294,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" dependencies = [ "nonempty", - "thiserror 1.0.68", + "thiserror 1.0.69", ] [[package]] @@ -1555,7 +1555,7 @@ dependencies = [ "ipnet", "once_cell", "rand", - "thiserror 1.0.68", + "thiserror 1.0.69", "tinyvec", "tokio", "tracing", @@ -1578,7 +1578,7 @@ dependencies = [ "rand", "resolv-conf", "smallvec", - "thiserror 1.0.68", + "thiserror 1.0.69", "tokio", "tracing", ] @@ -2205,9 +2205,13 @@ checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "loole" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad95468e4700cb37d8d1f198050db18cebe55e4b4c8aa9180a715deedb2f8965" +checksum = "a2998397c725c822c6b2ba605fd9eb4c6a7a0810f1629ba3cc232ef4f0308d96" +dependencies = [ + "futures-core", + "futures-sink", +] [[package]] name = "lru-cache" @@ -2509,7 +2513,7 @@ dependencies = [ "js-sys", "once_cell", "pin-project-lite", - "thiserror 1.0.68", + "thiserror 1.0.69", "urlencoding", ] @@ -2555,7 +2559,7 @@ dependencies = [ "ordered-float 4.5.0", "percent-encoding", "rand", - "thiserror 1.0.68", + "thiserror 1.0.69", "tokio", "tokio-stream", ] @@ -2918,9 +2922,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quinn" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" +checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" dependencies = [ "bytes", "pin-project-lite", @@ -2929,26 +2933,29 @@ dependencies = [ "rustc-hash 2.0.0", "rustls 0.23.16", "socket2", - "thiserror 1.0.68", + "thiserror 2.0.3", "tokio", "tracing", ] [[package]] name = "quinn-proto" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" +checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", + "getrandom", "rand", "ring", "rustc-hash 2.0.0", "rustls 0.23.16", + "rustls-pki-types", "slab", - "thiserror 1.0.68", + "thiserror 2.0.3", "tinyvec", "tracing", + "web-time 1.1.0", ] [[package]] @@ -3021,7 +3028,7 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.8", + "regex-automata 0.4.9", "regex-syntax 0.8.5", ] @@ -3036,9 +3043,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" dependencies = [ "aho-corasick", "memchr", @@ -3183,7 +3190,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror 2.0.1", + "thiserror 2.0.3", "url", "web-time 1.1.0", ] @@ -3209,7 +3216,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror 2.0.1", + "thiserror 2.0.3", "time", "tracing", "url", @@ -3235,7 +3242,7 @@ dependencies = [ "ruma-macros", "serde", "serde_json", - "thiserror 2.0.1", + "thiserror 2.0.3", "tracing", "url", "web-time 1.1.0", @@ -3266,7 +3273,7 @@ version = "0.9.5" source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" dependencies = [ "js_int", - "thiserror 2.0.1", + "thiserror 2.0.3", ] [[package]] @@ -3316,7 +3323,7 @@ dependencies = [ "http", "http-auth", "ruma-common", - "thiserror 2.0.1", + "thiserror 2.0.3", "tracing", ] @@ -3333,7 +3340,7 @@ dependencies = [ "serde_json", "sha2", "subslice", - "thiserror 2.0.1", + "thiserror 2.0.3", ] [[package]] @@ -3348,7 +3355,7 @@ dependencies = [ "ruma-events", "serde", "serde_json", - "thiserror 2.0.1", + "thiserror 2.0.3", "tracing", ] @@ -3415,9 +3422,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.39" +version = "0.38.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" +checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" dependencies = [ "bitflags 2.6.0", "errno", @@ -3483,6 +3490,9 @@ name = "rustls-pki-types" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +dependencies = [ + "web-time 1.1.0", +] [[package]] name = "rustls-webpki" @@ -3512,7 +3522,7 @@ dependencies = [ "futures-util", "pin-project", "thingbuf", - "thiserror 1.0.68", + "thiserror 1.0.69", "unicode-segmentation", "unicode-width", ] @@ -3712,7 +3722,7 @@ dependencies = [ "rand", "serde", "serde_json", - "thiserror 1.0.68", + "thiserror 1.0.69", "time", "url", "uuid", @@ -3720,18 +3730,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c3193aca71c12ad7890f1785d2b73e1b9f63a0bbc353c08ef26fe03fc56b5" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.214" +version = "1.0.215" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de523f781f095e28fa605cdce0f8307e451cc0fd14e2eb4cd2e98a355b147766" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", @@ -3918,7 +3928,7 @@ checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" dependencies = [ "num-bigint", "num-traits", - "thiserror 1.0.68", + "thiserror 1.0.69", "time", ] @@ -4083,9 +4093,9 @@ dependencies = [ [[package]] name = "termimad" -version = "0.30.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22117210909e9dfff30a558f554c7fb3edb198ef614e7691386785fb7679677c" +checksum = "9cda3a7471f9978706978454c45ef8dda67e9f8f3cdb9319eb2e9323deb6ae62" dependencies = [ "coolor", "crokey", @@ -4093,7 +4103,7 @@ dependencies = [ "lazy-regex", "minimad", "serde", - "thiserror 1.0.68", + "thiserror 1.0.69", "unicode-width", ] @@ -4109,27 +4119,27 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl 1.0.68", + "thiserror-impl 1.0.69", ] [[package]] name = "thiserror" -version = "2.0.1" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c1e40dd48a282ae8edc36c732cbc219144b87fb6a4c7316d611c6b1f06ec0c" +checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" dependencies = [ - "thiserror-impl 2.0.1", + "thiserror-impl 2.0.3", ] [[package]] name = "thiserror-impl" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", @@ -4138,9 +4148,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.1" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874aa7e446f1da8d9c3a5c95b1c5eb41d800045252121dc7f8e0ba370cee55f5" +checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ "proc-macro2", "quote", @@ -4323,7 +4333,7 @@ checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" dependencies = [ "either", "futures-util", - "thiserror 1.0.68", + "thiserror 1.0.69", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 0173e7cf..dde005a3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,7 +142,7 @@ features = [ ] [workspace.dependencies.serde] -version = "1.0.214" +version = "1.0.215" default-features = false features = ["rc"] @@ -171,7 +171,7 @@ default-features = false # Used to generate thumbnails for images [workspace.dependencies.image] -version = "0.25.1" +version = "0.25.5" default-features = false features = [ "jpeg", @@ -304,7 +304,7 @@ version = "2.1.1" # used to replace the channels of the tokio runtime [workspace.dependencies.loole] -version = "0.3.1" +version = "0.4.0" [workspace.dependencies.async-trait] version = "0.1.81" @@ -449,7 +449,7 @@ version = "0.4.3" default-features = false [workspace.dependencies.termimad] -version = "0.30.1" +version = "0.31.0" default-features = false [workspace.dependencies.checked_ops] From e228dec4f2d5abe02624f3d1a7cf572aab645e90 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 13 Nov 2024 01:01:33 +0000 Subject: [PATCH 1043/2091] add byte counting for compressed state caches Signed-off-by: Jason Volk --- src/service/rooms/state_compressor/mod.rs | 48 +++++++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index bf90d5c4..6b520ad3 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -1,18 +1,22 @@ use std::{ - collections::HashSet, + collections::{HashMap, HashSet}, fmt::Write, mem::size_of, sync::{Arc, Mutex}, }; -use conduit::{checked, err, expected, utils, utils::math::usize_from_f64, Result}; +use conduit::{ + at, checked, err, expected, utils, + utils::{bytes, math::usize_from_f64}, + Result, +}; use database::Map; use lru_cache::LruCache; use ruma::{EventId, RoomId}; use crate::{ rooms, - rooms::short::{ShortStateHash, ShortStateKey}, + rooms::short::{ShortId, ShortStateHash, ShortStateKey}, Dep, }; @@ -53,12 +57,13 @@ pub struct HashSetCompressStateEvent { pub removed: Arc, } -pub(crate) type CompressedState = HashSet; -pub(crate) type CompressedStateEvent = [u8; 2 * size_of::()]; type StateInfoLruCache = LruCache; type ShortStateInfoVec = Vec; type ParentStatesVec = Vec; +pub(crate) type CompressedState = HashSet; +pub(crate) type CompressedStateEvent = [u8; 2 * size_of::()]; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; @@ -75,9 +80,28 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { - let stateinfo_cache = self.stateinfo_cache.lock().expect("locked").len(); - writeln!(out, "stateinfo_cache: {stateinfo_cache}")?; + fn memory_usage(&self, out: &mut dyn Write) -> Result { + let (cache_len, ents) = { + let cache = self.stateinfo_cache.lock().expect("locked"); + let ents = cache + .iter() + .map(at!(1)) + .flat_map(|vec| vec.iter()) + .fold(HashMap::new(), |mut ents, ssi| { + ents.insert(Arc::as_ptr(&ssi.added), compressed_state_size(&ssi.added)); + ents.insert(Arc::as_ptr(&ssi.removed), compressed_state_size(&ssi.removed)); + ents.insert(Arc::as_ptr(&ssi.full_state), compressed_state_size(&ssi.full_state)); + ents + }); + + (cache.len(), ents) + }; + + let ents_len = ents.len(); + let bytes = ents.values().copied().fold(0_usize, usize::saturating_add); + + let bytes = bytes::pretty(bytes); + writeln!(out, "stateinfo_cache: {cache_len} {ents_len} ({bytes})")?; Ok(()) } @@ -435,3 +459,11 @@ impl Service { .insert(&shortstatehash.to_be_bytes(), &value); } } + +#[inline] +fn compressed_state_size(compressed_state: &CompressedState) -> usize { + compressed_state + .len() + .checked_mul(size_of::()) + .expect("CompressedState size overflow") +} From 4ec5d1e28e6cfff3d98c36c2b02aece196ee93c0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 14 Nov 2024 04:31:29 +0000 Subject: [PATCH 1044/2091] replace additional use tracing:: add log:: to disallowed-macros Signed-off-by: Jason Volk --- clippy.toml | 8 ++++++++ src/api/client/report.rs | 3 +-- src/api/server/make_join.rs | 6 ++++-- src/core/debug.rs | 2 ++ src/core/log/mod.rs | 2 ++ src/core/utils/sys.rs | 4 +--- src/router/serve/plain.rs | 3 +-- 7 files changed, 19 insertions(+), 9 deletions(-) diff --git a/clippy.toml b/clippy.toml index 08641fcc..b93b2377 100644 --- a/clippy.toml +++ b/clippy.toml @@ -5,3 +5,11 @@ future-size-threshold = 7745 # TODO reduce me ALARA stack-size-threshold = 196608 # reduce me ALARA too-many-lines-threshold = 700 # TODO reduce me to <= 100 type-complexity-threshold = 250 # reduce me to ~200 + +disallowed-macros = [ + { path = "log::error", reason = "use conduit_core::error" }, + { path = "log::warn", reason = "use conduit_core::warn" }, + { path = "log::info", reason = "use conduit_core::info" }, + { path = "log::debug", reason = "use conduit_core::debug" }, + { path = "log::trace", reason = "use conduit_core::trace" }, +] diff --git a/src/api/client/report.rs b/src/api/client/report.rs index e20fa8c2..a0133704 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -2,7 +2,7 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{utils::ReadyExt, Err}; +use conduit::{info, utils::ReadyExt, Err}; use rand::Rng; use ruma::{ api::client::{ @@ -13,7 +13,6 @@ use ruma::{ int, EventId, RoomId, UserId, }; use tokio::time::sleep; -use tracing::info; use crate::{ debug_info, diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index c3524f0e..af570064 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -1,5 +1,8 @@ use axum::extract::State; -use conduit::utils::{IterStream, ReadyExt}; +use conduit::{ + utils::{IterStream, ReadyExt}, + warn, +}; use futures::StreamExt; use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_join_event}, @@ -13,7 +16,6 @@ use ruma::{ CanonicalJsonObject, RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; -use tracing::warn; use crate::{ service::{pdu::PduBuilder, Services}, diff --git a/src/core/debug.rs b/src/core/debug.rs index 85574a2f..f7420784 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -1,3 +1,5 @@ +#![allow(clippy::disallowed_macros)] + use std::{any::Any, panic}; // Export debug proc_macros diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 1c415c6a..48b7f0f3 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -1,3 +1,5 @@ +#![allow(clippy::disallowed_macros)] + pub mod capture; pub mod color; pub mod fmt; diff --git a/src/core/utils/sys.rs b/src/core/utils/sys.rs index 6c396921..af8bd70b 100644 --- a/src/core/utils/sys.rs +++ b/src/core/utils/sys.rs @@ -1,6 +1,4 @@ -use tracing::debug; - -use crate::Result; +use crate::{debug, Result}; /// This is needed for opening lots of file descriptors, which tends to /// happen more often when using RocksDB and making lots of federation diff --git a/src/router/serve/plain.rs b/src/router/serve/plain.rs index 08263353..144bff85 100644 --- a/src/router/serve/plain.rs +++ b/src/router/serve/plain.rs @@ -5,9 +5,8 @@ use std::{ use axum::Router; use axum_server::{bind, Handle as ServerHandle}; -use conduit::{debug_info, Result, Server}; +use conduit::{debug_info, info, Result, Server}; use tokio::task::JoinSet; -use tracing::info; pub(super) async fn serve( server: &Arc, app: Router, handle: ServerHandle, addrs: Vec, From 08365bf5f440a4c9f086142c23044f1884c68033 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 10 Nov 2024 20:16:38 -0500 Subject: [PATCH 1045/2091] update config documentation, commit generated example config also removes the no-op/useless "database_backend" config option Signed-off-by: strawberry --- conduwuit-example.toml | 1923 +++++++++++++++++++++------------- src/api/server/make_knock.rs | 107 ++ src/api/server/send_knock.rs | 190 ++++ src/core/config/mod.rs | 573 ++++++---- src/service/migrations.rs | 11 +- 5 files changed, 1846 insertions(+), 958 deletions(-) create mode 100644 src/api/server/make_knock.rs create mode 100644 src/api/server/send_knock.rs diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 11735616..aa0d1e5d 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1,605 +1,867 @@ -# ============================================================================= -# This is the official example config for conduwuit. -# If you use it for your server, you will need to adjust it to your own needs. -# At the very least, change the server_name field! -# -# This documentation can also be found at https://conduwuit.puppyirl.gay/configuration.html -# ============================================================================= +### conduwuit Configuration +### +### THIS FILE IS GENERATED. CHANGES/CONTRIBUTIONS IN THE REPO WILL +### BE OVERWRITTEN! +### +### You should rename this file before configuring your server. Changes +### to documentation and defaults can be contributed in source code at +### src/core/config/mod.rs. This file is generated when building. +### +### Any values pre-populated are the default values for said config option. +### +### At the minimum, you MUST edit all the config options to your environment +### that say "YOU NEED TO EDIT THIS". +### See https://conduwuit.puppyirl.gay/configuration.html for ways to +### configure conduwuit [global] -# The server_name is the pretty name of this server. It is used as a suffix for user -# and room ids. Examples: matrix.org, conduit.rs - -# The Conduit server needs all /_matrix/ requests to be reachable at -# https://your.server.name/ on port 443 (client-server) and 8448 (federation). - -# If that's not possible for you, you can create /.well-known files to redirect -# requests (delegation). See -# https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixclient -# and -# https://spec.matrix.org/v1.9/server-server-api/#getwell-knownmatrixserver -# for more information - -# YOU NEED TO EDIT THIS -#server_name = "your.server.name" - -# Servers listed here will be used to gather public keys of other servers (notary trusted key servers). +# The server_name is the pretty name of this server. It is used as a +# suffix for user and room IDs/aliases. # -# The default behaviour for conduwuit is to attempt to query trusted key servers before querying the individual servers. -# This is done for performance reasons, but if you would like to query individual servers before the notary servers -# configured below, set to +# See the docs for reverse proxying and delegation: https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy +# Also see the `[global.well_known]` config section at the very bottom. # -# (Currently, conduwuit doesn't support batched key requests, so this list should only contain Synapse servers) -# Defaults to `matrix.org` -# trusted_servers = ["matrix.org"] - -# Sentry.io crash/panic reporting, performance monitoring/metrics, etc. This is NOT enabled by default. -# conduwuit's default Sentry reporting endpoint is o4506996327251968.ingest.us.sentry.io +# Examples of delegation: +# - https://puppygock.gay/.well-known/matrix/server +# - https://puppygock.gay/.well-known/matrix/client # -# Defaults to *false* -#sentry = false - -# Sentry reporting URL if a custom one is desired +# YOU NEED TO EDIT THIS. THIS CANNOT BE CHANGED AFTER WITHOUT A DATABASE +# WIPE. # -# Defaults to conduwuit's default Sentry endpoint: "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" -#sentry_endpoint = "" - -# Report your Conduwuit server_name in Sentry.io crash reports and metrics +# example: "conduwuit.woof" # -# Defaults to false -#sentry_send_server_name = false +#server_name = -# Performance monitoring/tracing sample rate for Sentry.io +# default address (IPv4 or IPv6) conduwuit will listen on. # -# Note that too high values may impact performance, and can be disabled by setting it to 0.0 (0%) -# This value is read as a percentage to Sentry, represented as a decimal +# If you are using Docker or a container NAT networking setup, this must +# be "0.0.0.0". # -# Defaults to 15% of traces (0.15) -#sentry_traces_sample_rate = 0.15 +# To listen on multiple addresses, specify a vector e.g. ["127.0.0.1", +# "::1"] +# +#address = ["127.0.0.1", "::1"] -# Whether to attach a stacktrace to Sentry reports. -#sentry_attach_stacktrace = false - -# Send panics to sentry. This is true by default, but sentry has to be enabled. -#sentry_send_panic = true - -# Send errors to sentry. This is true by default, but sentry has to be enabled. This option is -# only effective in release-mode; forced to false in debug-mode. -#sentry_send_error = true - -# Controls the tracing log level for Sentry to send things like breadcrumbs and transactions -# Defaults to "info" -#sentry_filter = "info" - - -### Database configuration - -# This is the only directory where conduwuit will save its data, including media. -# Note: this was previously "/var/lib/matrix-conduit" -database_path = "/var/lib/conduwuit" - -# Database backend: Only rocksdb is supported. -database_backend = "rocksdb" - - -### Network - -# The port(s) conduwuit will be running on. You need to set up a reverse proxy such as -# Caddy or Nginx so all requests to /_matrix on port 443 and 8448 will be -# forwarded to the conduwuit instance running on this port -# Docker users: Don't change this, you'll need to map an external port to this. +# The port(s) conduwuit will be running on. +# +# See https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy for reverse proxying. +# +# Docker users: Don't change this, you'll need to map an external port to +# this. +# # To listen on multiple ports, specify a vector e.g. [8080, 8448] # -# default if unspecified is 8008 -port = 6167 +#port = 8008 -# default address (IPv4 or IPv6) conduwuit will listen on. Generally you want this to be -# localhost (127.0.0.1 / ::1). If you are using Docker or a container NAT networking setup, you -# likely need this to be 0.0.0.0. -# To listen multiple addresses, specify a vector e.g. ["127.0.0.1", "::1"] +# Uncomment unix_socket_path to listen on a UNIX socket at the specified +# path. If listening on a UNIX socket, you MUST remove/comment the +# 'address' key if definedm AND add your reverse proxy to the 'conduwuit' +# group, unless world RW permissions are specified with unix_socket_perms +# (666 minimum). # -# default if unspecified is both IPv4 and IPv6 localhost: ["127.0.0.1", "::1"] -address = "127.0.0.1" +# example: "/run/conduwuit/conduwuit.sock" +# +#unix_socket_path = -# Max request size for file uploads -max_request_size = 20_000_000 # in bytes - -# Uncomment unix_socket_path to listen on a UNIX socket at the specified path. -# If listening on a UNIX socket, you must remove/comment the 'address' key if defined and add your -# reverse proxy to the 'conduwuit' group, unless world RW permissions are specified with unix_socket_perms (666 minimum). -#unix_socket_path = "/run/conduwuit/conduwuit.sock" +# The default permissions (in octal) to create the UNIX socket with. +# #unix_socket_perms = 660 -# Set this to true for conduwuit to compress HTTP response bodies using zstd. -# This option does nothing if conduwuit was not built with `zstd_compression` feature. -# Please be aware that enabling HTTP compression may weaken TLS. -# Most users should not need to enable this. -# See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before deciding to enable this. -zstd_compression = false - -# Set this to true for conduwuit to compress HTTP response bodies using gzip. -# This option does nothing if conduwuit was not built with `gzip_compression` feature. -# Please be aware that enabling HTTP compression may weaken TLS. -# Most users should not need to enable this. -# See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before deciding to enable this. -gzip_compression = false - -# Set this to true for conduwuit to compress HTTP response bodies using brotli. -# This option does nothing if conduwuit was not built with `brotli_compression` feature. -# Please be aware that enabling HTTP compression may weaken TLS. -# Most users should not need to enable this. -# See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before deciding to enable this. -brotli_compression = false - -# Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you do not want conduwuit to send outbound requests to. -# Defaults to RFC1918, unroutable, loopback, multicast, and testnet addresses for security. +# This is the only directory where conduwuit will save its data, including +# media. +# Note: this was previously "/var/lib/matrix-conduit" # -# To disable, set this to be an empty vector (`[]`). -# Please be aware that this is *not* a guarantee. You should be using a firewall with zones as doing this on the application layer may have bypasses. +# YOU NEED TO EDIT THIS. # -# Currently this does not account for proxies in use like Synapse does. -ip_range_denylist = [ - "127.0.0.0/8", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - "100.64.0.0/10", - "192.0.0.0/24", - "169.254.0.0/16", - "192.88.99.0/24", - "198.18.0.0/15", - "192.0.2.0/24", - "198.51.100.0/24", - "203.0.113.0/24", - "224.0.0.0/4", - "::1/128", - "fe80::/10", - "fc00::/7", - "2001:db8::/32", - "ff00::/8", - "fec0::/10", -] - - -### Moderation / Privacy / Security - -# Config option to control whether the legacy unauthenticated Matrix media repository endpoints will be enabled. -# These endpoints consist of: -# - /_matrix/media/*/config -# - /_matrix/media/*/upload -# - /_matrix/media/*/preview_url -# - /_matrix/media/*/download/* -# - /_matrix/media/*/thumbnail/* +# example: "/var/lib/conduwuit" # -# The authenticated equivalent endpoints are always enabled. +#database_path = + +# conduwuit supports online database backups using RocksDB's Backup engine +# API. To use this, set a database backup path that conduwuit can write +# to. # -# Defaults to true for now, but this is highly subject to change, likely in the next release. -#allow_legacy_media = true +# See https://conduwuit.puppyirl.gay/maintenance.html#backups for more information. +# +# example: "/opt/conduwuit-db-backups" +# +#database_backup_path = -# Set to true to allow user type "guest" registrations. Element attempts to register guest users automatically. -# Defaults to false -allow_guest_registration = false +# The amount of online RocksDB database backups to keep/retain, if using +# "database_backup_path", before deleting the oldest one. +# +#database_backups_to_keep = 1 -# Set to true to log guest registrations in the admin room. -# Defaults to false as it may be noisy or unnecessary. -log_guest_registrations = false +# Set this to any float value in megabytes for conduwuit to tell the +# database engine that this much memory is available for database-related +# caches. +# +# May be useful if you have significant memory to spare to increase +# performance. +# +# Similar to the individual LRU caches, this is scaled up with your CPU +# core count. +# +# This defaults to 128.0 + (64.0 * CPU core count) +# +#db_cache_capacity_mb = -# Set to true to allow guest registrations/users to auto join any rooms specified in `auto_join_rooms` -# Defaults to false -allow_guests_auto_join_rooms = false +# Option to control adding arbitrary text to the end of the user's +# displayname upon registration with a space before the text. This was the +# lightning bolt emoji option, just replaced with support for adding your +# own custom text or emojis. To disable, set this to "" (an empty string). +# +# The default is the trans pride flag. +# +# example: "🏳️⚧️" +# +#new_user_displayname_suffix = "🏳️⚧️" -# Vector list of servers that conduwuit will refuse to download remote media from. -# No default. -# prevent_media_downloads_from = ["example.com", "example.local"] +# If enabled, conduwuit will send a simple GET request periodically to +# `https://pupbrain.dev/check-for-updates/stable` for any new +# announcements made. Despite the name, this is not an update check +# endpoint, it is simply an announcement check endpoint. +# +# This is disabled by default as this is rarely used except for security +# updates or major updates. +# +#allow_check_for_updates = false + +# Set this to any float value to multiply conduwuit's in-memory LRU caches +# with such as "auth_chain_cache_capacity". +# +# May be useful if you have significant memory to spare to increase +# performance. This was previously called +# `conduit_cache_capacity_modifier`. +# +# If you have low memory, reducing this may be viable. +# +# By default, the individual caches such as "auth_chain_cache_capacity" +# are scaled by your CPU core count. +# +#cache_capacity_modifier = 1.0 + +# This item is undocumented. Please contribute documentation for it. +# +#pdu_cache_capacity = varies by system + +# This item is undocumented. Please contribute documentation for it. +# +#auth_chain_cache_capacity = varies by system + +# This item is undocumented. Please contribute documentation for it. +# +#shorteventid_cache_capacity = varies by system + +# This item is undocumented. Please contribute documentation for it. +# +#eventidshort_cache_capacity = varies by system + +# This item is undocumented. Please contribute documentation for it. +# +#shortstatekey_cache_capacity = varies by system + +# This item is undocumented. Please contribute documentation for it. +# +#statekeyshort_cache_capacity = varies by system + +# This item is undocumented. Please contribute documentation for it. +# +#server_visibility_cache_capacity = varies by system + +# This item is undocumented. Please contribute documentation for it. +# +#user_visibility_cache_capacity = varies by system + +# This item is undocumented. Please contribute documentation for it. +# +#stateinfo_cache_capacity = varies by system + +# This item is undocumented. Please contribute documentation for it. +# +#roomid_spacehierarchy_cache_capacity = varies by system + +# Maximum entries stored in DNS memory-cache. The size of an entry may +# vary so please take care if raising this value excessively. Only +# decrease this when using an external DNS cache. Please note +# that systemd-resolved does *not* count as an external cache, even when +# configured to do so. +# +#dns_cache_entries = 32768 + +# Minimum time-to-live in seconds for entries in the DNS cache. The +# default may appear high to most administrators; this is by design as the +# majority of NXDOMAINs are correct for a long time (e.g. the server is no +# longer running Matrix). Only decrease this if you are using an external +# DNS cache. +# +# default_dns_min_ttl: 259200 +# +#dns_min_ttl = + +# Minimum time-to-live in seconds for NXDOMAIN entries in the DNS cache. +# This value is critical for the server to federate efficiently. +# NXDOMAIN's are assumed to not be returning to the federation +# and aggressively cached rather than constantly rechecked. +# +# Defaults to 3 days as these are *very rarely* false negatives. +# +#dns_min_ttl_nxdomain = 259200 + +# Number of retries after a timeout. +# +#dns_attempts = 10 + +# The number of seconds to wait for a reply to a DNS query. Please note +# that recursive queries can take up to several seconds for some domains, +# so this value should not be too low, especially on slower hardware or +# resolvers. +# +#dns_timeout = 10 + +# Fallback to TCP on DNS errors. Set this to false if unsupported by +# nameserver. +# +#dns_tcp_fallback = true + +# Enable to query all nameservers until the domain is found. Referred to +# as "trust_negative_responses" in hickory_resolver. This can avoid +# useless DNS queries if the first nameserver responds with NXDOMAIN or +# an empty NOERROR response. +# +#query_all_nameservers = true + +# Enables using *only* TCP for querying your specified nameservers instead +# of UDP. +# +# If you are running conduwuit in a container environment, this config option may need to be enabled. See https://conduwuit.puppyirl.gay/troubleshooting.html#potential-dns-issues-when-using-docker for more details. +# +#query_over_tcp_only = false + +# DNS A/AAAA record lookup strategy +# +# Takes a number of one of the following options: +# 1 - Ipv4Only (Only query for A records, no AAAA/IPv6) +# +# 2 - Ipv6Only (Only query for AAAA records, no A/IPv4) +# +# 3 - Ipv4AndIpv6 (Query for A and AAAA records in parallel, uses whatever +# returns a successful response first) +# +# 4 - Ipv6thenIpv4 (Query for AAAA record, if that fails then query the A +# record) +# +# 5 - Ipv4thenIpv6 (Query for A record, if that fails then query the AAAA +# record) +# +# If you don't have IPv6 networking, then for better DNS performance it +# may be suitable to set this to Ipv4Only (1) as you will never ever use +# the AAAA record contents even if the AAAA record is successful instead +# of the A record. +# +#ip_lookup_strategy = 5 + +# Max request size for file uploads in bytes. Defaults to 20MB. +# +#max_request_size = 20971520 + +# This item is undocumented. Please contribute documentation for it. +# +#max_fetch_prev_events = 192 + +# Default/base connection timeout (seconds). This is used only by URL +# previews and update/news endpoint checks. +# +#request_conn_timeout = 10 + +# Default/base request timeout (seconds). The time waiting to receive more +# data from another server. This is used only by URL previews, +# update/news, and misc endpoint checks. +# +#request_timeout = 35 + +# Default/base request total timeout (seconds). The time limit for a whole +# request. This is set very high to not cancel healthy requests while +# serving as a backstop. This is used only by URL previews and +# update/news endpoint checks. +# +#request_total_timeout = 320 + +# Default/base idle connection pool timeout (seconds). This is used only +# by URL previews and update/news endpoint checks. +# +#request_idle_timeout = 5 + +# Default/base max idle connections per host. This is used only by URL +# previews and update/news endpoint checks. Defaults to 1 as generally the +# same open connection can be re-used. +# +#request_idle_per_host = 1 + +# Federation well-known resolution connection timeout (seconds) +# +#well_known_conn_timeout = 6 + +# Federation HTTP well-known resolution request timeout (seconds) +# +#well_known_timeout = 10 + +# Federation client request timeout (seconds). You most definitely want +# this to be high to account for extremely large room joins, slow +# homeservers, your own resources etc. +# +#federation_timeout = 300 + +# Federation client idle connection pool timeout (seconds) +# +#federation_idle_timeout = 25 + +# Federation client max idle connections per host. Defaults to 1 as +# generally the same open connection can be re-used +# +#federation_idle_per_host = 1 + +# Federation sender request timeout (seconds). The time it takes for the +# remote server to process sent transactions can take a while. +# +#sender_timeout = 180 + +# Federation sender idle connection pool timeout (seconds) +# +#sender_idle_timeout = 180 + +# Federation sender transaction retry backoff limit (seconds) +# +#sender_retry_backoff_limit = 86400 + +# Appservice URL request connection timeout. Defaults to 35 seconds as +# generally appservices are hosted within the same network. +# +#appservice_timeout = 35 + +# Appservice URL idle connection pool timeout (seconds) +# +#appservice_idle_timeout = 300 + +# Notification gateway pusher idle connection pool timeout +# +#pusher_idle_timeout = 15 # Enables registration. If set to false, no users can register on this # server. # -# If set to true without a token configured, users can register with no form of 2nd- -# step only if you set +# If set to true without a token configured, users can register with no +# form of 2nd-step only if you set # `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` to # true in your config. # # If you would like registration only via token reg, please configure # `registration_token` or `registration_token_file`. -allow_registration = false -# Please note that an open registration homeserver with no second-step verification -# is highly prone to abuse and potential defederation by homeservers, including -# matrix.org. +# +#allow_registration = false -# A static registration token that new users will have to provide when creating -# an account. If unset and `allow_registration` is true, registration is open -# without any condition. YOU NEED TO EDIT THIS. -registration_token = "change this token/string here or set registration_token_file" +# This item is undocumented. Please contribute documentation for it. +# +#yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = false -# Path to a file on the system that gets read for the registration token +# A static registration token that new users will have to provide when +# creating an account. If unset and `allow_registration` is true, +# registration is open without any condition. +# +# YOU NEED TO EDIT THIS OR USE registration_token_file. +# +# example: "o&^uCtes4HPf0Vu@F20jQeeWE7" +# +#registration_token = + +# Path to a file on the system that gets read for the registration token. +# this config option takes precedence/priority over "registration_token". # # conduwuit must be able to access the file, and it must not be empty # -# no default -#registration_token_file = "/etc/conduwuit/.reg_token" - -# controls whether federation is allowed or not -# defaults to true -# allow_federation = true - -# controls whether users are allowed to create rooms. -# appservices and admins are always allowed to create rooms -# defaults to true -# allow_room_creation = true - -# controls whether non-admin local users are forbidden from sending room invites (local and remote), -# and if non-admin users can receive remote room invites. admins are always allowed to send and receive all room invites. -# defaults to false -# block_non_admin_invites = false - -# List of forbidden username patterns/strings. Values in this list are matched as *contains*. -# This is checked upon username availability check, registration, and startup as warnings if any local users in your database -# have a forbidden username. -# No default. -# forbidden_usernames = [] - -# List of forbidden room aliases and room IDs as patterns/strings. Values in this list are matched as *contains*. -# This is checked upon room alias creation, custom room ID creation if used, and startup as warnings if any room aliases -# in your database have a forbidden room alias/ID. -# No default. -# forbidden_alias_names = [] - -# List of forbidden server names that we will block incoming AND outgoing federation with, and block client room joins / remote user invites. +# example: "/etc/conduwuit/.reg_token" # -# This check is applied on the room ID, room alias, sender server name, sender user's server name, inbound federation X-Matrix origin, and outbound federation handler. +#registration_token_file = + +# Controls whether encrypted rooms and events are allowed. # -# Basically "global" ACLs. No default. -# forbidden_remote_server_names = [] +#allow_encryption = true -# List of forbidden server names that we will block all outgoing federated room directory requests for. Useful for preventing our users from wandering into bad servers or spaces. -# No default. -# forbidden_remote_room_directory_server_names = [] - -# Set this to true to allow your server's public room directory to be federated. -# Set this to false to protect against /publicRooms spiders, but will forbid external users -# from viewing your server's public room directory. If federation is disabled entirely -# (`allow_federation`), this is inherently false. -allow_public_room_directory_over_federation = false - -# Set this to true to allow your server's public room directory to be queried without client -# authentication (access token) through the Client APIs. Set this to false to protect against /publicRooms spiders. -allow_public_room_directory_without_auth = false - -# Set this to true to lock down your server's public room directory and only allow admins to publish rooms to the room directory. -# Unpublishing is still allowed by all users with this enabled. +# Controls whether federation is allowed or not. It is not recommended to +# disable this after the fact due to potential federation breakage. # -# Defaults to false -lockdown_public_room_directory = false +#allow_federation = true -# Set this to true to allow federating device display names / allow external users to see your device display name. -# If federation is disabled entirely (`allow_federation`), this is inherently false. For privacy, this is best disabled. -allow_device_name_federation = false +# This item is undocumented. Please contribute documentation for it. +# +#federation_loopback = false -# Vector list of domains allowed to send requests to for URL previews. Defaults to none. -# Note: this is a *contains* match, not an explicit match. Putting "google.com" will match "https://google.com" and "http://mymaliciousdomainexamplegoogle.com" -# Setting this to "*" will allow all URL previews. Please note that this opens up significant attack surface to your server, you are expected to be aware of the risks by doing so. -url_preview_domain_contains_allowlist = [] +# Set this to true to require authentication on the normally +# unauthenticated profile retrieval endpoints (GET) +# "/_matrix/client/v3/profile/{userId}". +# +# This can prevent profile scraping. +# +#require_auth_for_profile_requests = false -# Vector list of explicit domains allowed to send requests to for URL previews. Defaults to none. -# Note: This is an *explicit* match, not a contains match. Putting "google.com" will match "https://google.com", "http://google.com", but not "https://mymaliciousdomainexamplegoogle.com" -# Setting this to "*" will allow all URL previews. Please note that this opens up significant attack surface to your server, you are expected to be aware of the risks by doing so. -url_preview_domain_explicit_allowlist = [] +# Set this to true to allow your server's public room directory to be +# federated. Set this to false to protect against /publicRooms spiders, +# but will forbid external users from viewing your server's public room +# directory. If federation is disabled entirely (`allow_federation`), +# this is inherently false. +# +#allow_public_room_directory_over_federation = false -# Vector list of URLs allowed to send requests to for URL previews. Defaults to none. -# Note that this is a *contains* match, not an explicit match. Putting "google.com" will match "https://google.com/", "https://google.com/url?q=https://mymaliciousdomainexample.com", and "https://mymaliciousdomainexample.com/hi/google.com" -# Setting this to "*" will allow all URL previews. Please note that this opens up significant attack surface to your server, you are expected to be aware of the risks by doing so. -url_preview_url_contains_allowlist = [] +# Set this to true to allow your server's public room directory to be +# queried without client authentication (access token) through the Client +# APIs. Set this to false to protect against /publicRooms spiders. +# +#allow_public_room_directory_without_auth = false -# Vector list of explicit domains not allowed to send requests to for URL previews. Defaults to none. -# Note: This is an *explicit* match, not a contains match. Putting "google.com" will match "https://google.com", "http://google.com", but not "https://mymaliciousdomainexamplegoogle.com" -# The denylist is checked first before allowlist. Setting this to "*" will not do anything. -url_preview_domain_explicit_denylist = [] +# allow guests/unauthenticated users to access TURN credentials +# +# this is the equivalent of Synapse's `turn_allow_guests` config option. +# this allows any unauthenticated user to call the endpoint +# `/_matrix/client/v3/voip/turnServer`. +# +# It is unlikely you need to enable this as all major clients support +# authentication for this endpoint and prevents misuse of your TURN server +# from potential bots. +# +#turn_allow_guests = false -# Maximum amount of bytes allowed in a URL preview body size when spidering. Defaults to 384KB (384_000 bytes) -url_preview_max_spider_size = 384_000 +# Set this to true to lock down your server's public room directory and +# only allow admins to publish rooms to the room directory. Unpublishing +# is still allowed by all users with this enabled. +# +#lockdown_public_room_directory = false -# Option to decide whether you would like to run the domain allowlist checks (contains and explicit) on the root domain or not. Does not apply to URL contains allowlist. Defaults to false. -# Example: If this is enabled and you have "wikipedia.org" allowed in the explicit and/or contains domain allowlist, it will allow all subdomains under "wikipedia.org" such as "en.m.wikipedia.org" as the root domain is checked and matched. -# Useful if the domain contains allowlist is still too broad for you but you still want to allow all the subdomains under a root domain. -url_preview_check_root_domain = false +# Set this to true to allow federating device display names / allow +# external users to see your device display name. If federation is +# disabled entirely (`allow_federation`), this is inherently false. For +# privacy reasons, this is best left disabled. +# +#allow_device_name_federation = false -# Config option to allow or disallow incoming federation requests that obtain the profiles -# of our local users from `/_matrix/federation/v1/query/profile` +# Config option to allow or disallow incoming federation requests that +# obtain the profiles of our local users from +# `/_matrix/federation/v1/query/profile` +# +# Increases privacy of your local user's such as display names, but some +# remote users may get a false "this user does not exist" error when they +# try to invite you to a DM or room. Also can protect against profile +# spiders. # # This is inherently false if `allow_federation` is disabled # -# Defaults to true -allow_profile_lookup_federation_requests = true +#allow_inbound_profile_lookup_federation_requests = true -# Config option to automatically deactivate the account of any user who attempts to join a: -# - banned room -# - forbidden room alias -# - room alias or ID with a forbidden server name +# controls whether standard users are allowed to create rooms. appservices +# and admins are always allowed to create rooms # -# This may be useful if all your banned lists consist of toxic rooms or servers that no good faith user would ever attempt to join, and -# to automatically remediate the problem without any admin user intervention. -# -# This will also make the user leave all rooms. Federation (e.g. remote room invites) are ignored here. -# -# Defaults to false as rooms can be banned for non-moderation-related reasons -#auto_deactivate_banned_room_attempts = false +#allow_room_creation = true - -### Admin Room and Console - -# Controls whether the conduwuit admin room console / CLI will immediately activate on startup. -# This option can also be enabled with `--console` conduwuit argument +# Set to false to disable users from joining or creating room versions +# that aren't 100% officially supported by conduwuit. # -# Defaults to false -#admin_console_automatic = false +# conduwuit officially supports room versions 6 - 11. +# +# conduwuit has slightly experimental (though works fine in practice) +# support for versions 3 - 5 +# +#allow_unstable_room_versions = true -# Controls what admin commands will be executed on startup. This is a vector list of strings of admin commands to run. +# default room version conduwuit will create rooms with. # -# An example of this can be: `admin_execute = ["debug ping puppygock.gay", "debug echo hi"]` +# per spec, room version 10 is the default. # -# This option can also be configured with the `--execute` conduwuit argument and can take standard shell commands and environment variables -# -# Such example could be: `./conduwuit --execute "server admin-notice conduwuit has started up at $(date)"` -# -# Defaults to nothing. -#admin_execute = [""] +#default_room_version = 10 -# Controls whether conduwuit should error and fail to start if an admin execute command (`--execute` / `admin_execute`) fails +# This item is undocumented. Please contribute documentation for it. # -# Defaults to false -#admin_execute_errors_ignore = false +#allow_jaeger = false -# Controls the max log level for admin command log captures (logs generated from running admin commands) +# This item is undocumented. Please contribute documentation for it. # -# Defaults to "info" on release builds, else "debug" on debug builds -#admin_log_capture = "info" +#jaeger_filter = "info" -# Allows admins to enter commands in rooms other than #admins by prefixing with \!admin. The reply -# will be publicly visible to the room, originating from the sender. -# defaults to true -#admin_escape_commands = true - -# Controls whether admin room notices like account registrations, password changes, account deactivations, -# room directory publications, etc will be sent to the admin room. +# If the 'perf_measurements' compile-time feature is enabled, enables +# collecting folded stack trace profile of tracing spans using +# tracing_flame. The resulting profile can be visualized with inferno[1], +# speedscope[2], or a number of other tools. # -# Update notices and normal admin command responses will still be sent. +# [1]: https://github.com/jonhoo/inferno +# [2]: www.speedscope.app # -# defaults to true -#admin_room_notices = true +#tracing_flame = false +# This item is undocumented. Please contribute documentation for it. +# +#tracing_flame_filter = "info" -### Misc +# This item is undocumented. Please contribute documentation for it. +# +#tracing_flame_output_path = "./tracing.folded" + +# Examples: +# - No proxy (default): +# proxy ="none" +# +# - For global proxy, create the section at the bottom of this file: +# [global.proxy] +# global = { url = "socks5h://localhost:9050" } +# +# - To proxy some domains: +# [global.proxy] +# [[global.proxy.by_domain]] +# url = "socks5h://localhost:9050" +# include = ["*.onion", "matrix.myspecial.onion"] +# exclude = ["*.myspecial.onion"] +# +# Include vs. Exclude: +# - If include is an empty list, it is assumed to be `["*"]`. +# - If a domain matches both the exclude and include list, the proxy will +# only be used if it was included because of a more specific rule than +# it was excluded. In the above example, the proxy would be used for +# `ordinary.onion`, `matrix.myspecial.onion`, but not +# `hello.myspecial.onion`. +# +#proxy = "none" + +# This item is undocumented. Please contribute documentation for it. +# +#jwt_secret = + +# Servers listed here will be used to gather public keys of other servers +# (notary trusted key servers). +# +# Currently, conduwuit doesn't support inbound batched key requests, so +# this list should only contain other Synapse servers +# +# example: ["matrix.org", "constellatory.net", "tchncs.de"] +# +#trusted_servers = ["matrix.org"] + +# Whether to query the servers listed in trusted_servers first or query +# the origin server first. For best security, querying the origin server +# first is advised to minimize the exposure to a compromised trusted +# server. For maximum federation/join performance this can be set to true, +# however other options exist to query trusted servers first under +# specific high-load circumstances and should be evaluated before setting +# this to true. +# +#query_trusted_key_servers_first = false + +# Whether to query the servers listed in trusted_servers first +# specifically on room joins. This option limits the exposure to a +# compromised trusted server to room joins only. The join operation +# requires gathering keys from many origin servers which can cause +# significant delays. Therefor this defaults to true to mitigate +# unexpected delays out-of-the-box. The security-paranoid or those +# willing to tolerate delays are advised to set this to false. Note that +# setting query_trusted_key_servers_first to true causes this option to +# be ignored. +# +#query_trusted_key_servers_first_on_join = true + +# Only query trusted servers for keys and never the origin server. This is +# intended for clusters or custom deployments using their trusted_servers +# as forwarding-agents to cache and deduplicate requests. Notary servers +# do not act as forwarding-agents by default, therefor do not enable this +# unless you know exactly what you are doing. +# +#only_query_trusted_key_servers = false + +# Maximum number of keys to request in each trusted server batch query. +# +#trusted_server_batch_size = 1024 # max log level for conduwuit. allows debug, info, warn, or error # see also: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives -# **Caveat**: -# For release builds, the tracing crate is configured to only implement levels higher than error to avoid unnecessary overhead in the compiled binary from trace macros. -# For debug builds, this restriction is not applied. # -# Defaults to "info" +# **Caveat**: +# For release builds, the tracing crate is configured to only implement +# levels higher than error to avoid unnecessary overhead in the compiled +# binary from trace macros. For debug builds, this restriction is not +# applied. +# #log = "info" # controls whether logs will be outputted with ANSI colours # -# defaults to true #log_colors = true -# controls whether encrypted rooms and events are allowed (default true) -#allow_encryption = false - -# if enabled, conduwuit will send a simple GET request periodically to `https://pupbrain.dev/check-for-updates/stable` -# for any new announcements made. Despite the name, this is not an update check -# endpoint, it is simply an announcement check endpoint. -# Defaults to false. -#allow_check_for_updates = false - -# Set to false to disable users from joining or creating room versions that aren't 100% officially supported by conduwuit. -# conduwuit officially supports room versions 6 - 10. conduwuit has experimental/unstable support for 3 - 5, and 11. -# Defaults to true. -#allow_unstable_room_versions = true - -# Option to control adding arbitrary text to the end of the user's displayname upon registration with a space before the text. -# This was the lightning bolt emoji option, just replaced with support for adding your own custom text or emojis. -# To disable, set this to "" (an empty string) -# Defaults to "🏳️‍⚧️" (trans pride flag) -#new_user_displayname_suffix = "🏳️‍⚧️" - -# Option to control whether conduwuit will query your list of trusted notary key servers (`trusted_servers`) for -# remote homeserver signing keys it doesn't know *first*, or query the individual servers first before falling back to the trusted -# key servers. -# -# The former/default behaviour makes federated/remote rooms joins generally faster because we're querying a single (or list of) server -# that we know works, is reasonably fast, and is reliable for just about all the homeserver signing keys in the room. Querying individual -# servers may take longer depending on the general infrastructure of everyone in there, how many dead servers there are, etc. -# -# However, this does create an increased reliance on one single or multiple large entities as `trusted_servers` should generally -# contain long-term and large servers who know a very large number of homeservers. -# -# If you don't know what any of this means, leave this and `trusted_servers` alone to their defaults. -# -# Defaults to true as this is the fastest option for federation. -#query_trusted_key_servers_first = true - -# List/vector of room **IDs** that conduwuit will make newly registered users join. -# The room IDs specified must be rooms that you have joined at least once on the server, and must be public. -# -# No default. -#auto_join_rooms = [] - -# Retry failed and incomplete messages to remote servers immediately upon startup. This is called bursting. -# If this is disabled, said messages may not be delivered until more messages are queued for that server. -# Do not change this option unless server resources are extremely limited or the scale of the server's -# deployment is huge. Do not disable this unless you know what you are doing. -#startup_netburst = true - -# Limit the startup netburst to the most recent (default: 50) messages queued for each remote server. All older -# messages are dropped and not reattempted. The `startup_netburst` option must be enabled for this value to have -# any effect. Do not change this value unless you know what you are doing. Set this value to -1 to reattempt -# every message without trimming the queues; this may consume significant disk. Set this value to 0 to drop all -# messages without any attempt at redelivery. -#startup_netburst_keep = 50 - -# If the 'perf_measurements' feature is enabled, enables collecting folded stack trace profile of tracing spans using -# tracing_flame. The resulting profile can be visualized with inferno[1], speedscope[2], or a number of other tools. -# [1]: https://github.com/jonhoo/inferno -# [2]: www.speedscope.app -# tracing_flame = false - -# If 'tracing_flame' is enabled, sets a filter for which events will be included in the profile. -# Supported syntax is documented at https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives -# tracing_flame_filter = "trace,h2=off" - -# If 'tracing_flame' is enabled, set the path to write the generated profile. -# tracing_flame_output_path = "./tracing.folded" - -# Enable the tokio-console. This option is only relevant to developers. -# See: docs/development.md#debugging-with-tokio-console for more information. -#tokio_console = false - -# Enable backward-compatibility with Conduit's media directory by creating symlinks of media. This -# option is only necessary if you plan on using Conduit again. Otherwise setting this to false -# reduces filesystem clutter and overhead for managing these symlinks in the directory. This is now -# disabled by default. You may still return to upstream Conduit but you have to run Conduwuit at -# least once with this set to true and allow the media_startup_check to take place before shutting -# down to return to Conduit. -# -# Disabled by default. -#media_compat_file_link = false - -# Prunes missing media from the database as part of the media startup checks. This means if you -# delete files from the media directory the corresponding entries will be removed from the -# database. This is disabled by default because if the media directory is accidentally moved or -# inaccessible the metadata entries in the database will be lost with sadness. -# -# Disabled by default. -#prune_missing_media = false - -# Checks consistency of the media directory at startup: -# 1. When `media_compat_file_link` is enbled, this check will upgrade media when switching back -# and forth between Conduit and Conduwuit. Both options must be enabled to handle this. -# 2. When media is deleted from the directory, this check will also delete its database entry. -# -# If none of these checks apply to your use cases, and your media directory is significantly large -# setting this to false may reduce startup time. -# -# Enabled by default. -#media_startup_check = true - # OpenID token expiration/TTL in seconds # -# These are the OpenID tokens that are primarily used for Matrix account integrations, *not* OIDC/OpenID Connect/etc +# These are the OpenID tokens that are primarily used for Matrix account +# integrations (e.g. Vector Integrations in Element), *not* OIDC/OpenID +# Connect/etc # -# Defaults to 3600 (1 hour) #openid_token_ttl = 3600 -# Emergency password feature. This password set here will let you login to the server service account (e.g. `@conduit`) -# and let you run admin commands, invite yourself to the admin room, etc. +# static TURN username to provide the client if not using a shared secret +# ("turn_secret"), It is recommended to use a shared secret over static +# credentials. # -# no default. -#emergency_password = "" +#turn_username = false - -### Generic database options - -# Set this to any float value to multiply conduwuit's in-memory LRU caches with. -# By default, the caches scale automatically with cpu-core-count. -# May be useful if you have significant memory to spare to increase performance. +# static TURN password to provide the client if not using a shared secret +# ("turn_secret"). It is recommended to use a shared secret over static +# credentials. # -# This was previously called `conduit_cache_capacity_modifier` +#turn_password = false + +# vector list of TURN URIs/servers to use # -# Defaults to 1.0. -#cache_capacity_modifier = 1.0 - -# Set this to any float value in megabytes for conduwuit to tell the database engine that this much memory is available for database-related caches. -# May be useful if you have significant memory to spare to increase performance. -# Defaults to 128.0 + (64.0 * CPU core count). -#db_cache_capacity_mb = 256.0 - - -### RocksDB options - -# Set this to true to use RocksDB config options that are tailored to HDDs (slower device storage) +# replace "example.turn.uri" with your TURN domain, such as the coturn +# "realm" config option. if using TURN over TLS, replace the URI prefix +# "turn:" with "turns:" # -# It is worth noting that by default, conduwuit will use RocksDB with Direct IO enabled. *Generally* speaking this improves performance as it bypasses buffered I/O (system page cache). -# However there is a potential chance that Direct IO may cause issues with database operations if your setup is uncommon. This has been observed with FUSE filesystems, and possibly ZFS filesystem. -# RocksDB generally deals/corrects these issues but it cannot account for all setups. -# If you experience any weird RocksDB issues, try enabling this option as it turns off Direct IO and feel free to report in the conduwuit Matrix room if this option fixes your DB issues. -# See https://github.com/facebook/rocksdb/wiki/Direct-IO for more information. +# example: ["turn:example.turn.uri?transport=udp", +# "turn:example.turn.uri?transport=tcp"] # -# Defaults to false -#rocksdb_optimize_for_spinning_disks = false +#turn_uris = [] -# Enables direct-io to increase database performance. This is enabled by default. Set this option to false if the -# database resides on a filesystem which does not support direct-io. -#rocksdb_direct_io = true +# TURN secret to use for generating the HMAC-SHA1 hash apart of username +# and password generation +# +# this is more secure, but if needed you can use traditional +# static username/password credentials. +# +#turn_secret = false -# RocksDB log level. This is not the same as conduwuit's log level. This is the log level for the RocksDB engine/library -# which show up in your database folder/path as `LOG` files. Defaults to error. conduwuit will typically log RocksDB errors as normal. +# TURN secret to use that's read from the file path specified +# +# this takes priority over "turn_secret" first, and falls back to +# "turn_secret" if invalid or failed to open. +# +# example: "/etc/conduwuit/.turn_secret" +# +#turn_secret_file = + +# TURN TTL in seconds +# +#turn_ttl = 86400 + +# List/vector of room IDs or room aliases that conduwuit will make newly +# registered users join. The rooms specified must be rooms that you +# have joined at least once on the server, and must be public. +# +# example: ["#conduwuit:puppygock.gay", +# "!eoIzvAvVwY23LPDay8:puppygock.gay"] +# +#auto_join_rooms = [] + +# Config option to automatically deactivate the account of any user who +# attempts to join a: +# - banned room +# - forbidden room alias +# - room alias or ID with a forbidden server name +# +# This may be useful if all your banned lists consist of toxic rooms or +# servers that no good faith user would ever attempt to join, and +# to automatically remediate the problem without any admin user +# intervention. +# +# This will also make the user leave all rooms. Federation (e.g. remote +# room invites) are ignored here. +# +# Defaults to false as rooms can be banned for non-moderation-related +# reasons +# +#auto_deactivate_banned_room_attempts = false + +# RocksDB log level. This is not the same as conduwuit's log level. This +# is the log level for the RocksDB engine/library which show up in your +# database folder/path as `LOG` files. conduwuit will log RocksDB errors +# as normal through tracing. +# #rocksdb_log_level = "error" -# Max RocksDB `LOG` file size before rotating in bytes. Defaults to 4MB. +# This item is undocumented. Please contribute documentation for it. +# +#rocksdb_log_stderr = false + +# Max RocksDB `LOG` file size before rotating in bytes. Defaults to 4MB in +# bytes. +# #rocksdb_max_log_file_size = 4194304 -# Time in seconds before RocksDB will forcibly rotate logs. Defaults to 0. +# Time in seconds before RocksDB will forcibly rotate logs. +# #rocksdb_log_time_to_roll = 0 -# Amount of threads that RocksDB will use for parallelism on database operatons such as cleanup, sync, flush, compaction, etc. Set to 0 to use all your logical threads. +# Set this to true to use RocksDB config options that are tailored to HDDs +# (slower device storage) +# +# It is worth noting that by default, conduwuit will use RocksDB with +# Direct IO enabled. *Generally* speaking this improves performance as it +# bypasses buffered I/O (system page cache). However there is a potential +# chance that Direct IO may cause issues with database operations if your +# setup is uncommon. This has been observed with FUSE filesystems, and +# possibly ZFS filesystem. RocksDB generally deals/corrects these issues +# but it cannot account for all setups. If you experience any weird +# RocksDB issues, try enabling this option as it turns off Direct IO and +# feel free to report in the conduwuit Matrix room if this option fixes +# your DB issues. +# +# See https://github.com/facebook/rocksdb/wiki/Direct-IO for more information. +# +#rocksdb_optimize_for_spinning_disks = false + +# Enables direct-io to increase database performance via unbuffered I/O. +# +# See https://github.com/facebook/rocksdb/wiki/Direct-IO for more details about Direct IO and RocksDB. +# +# Set this option to false if the database resides on a filesystem which +# does not support direct-io like FUSE, or any form of complex filesystem +# setup such as possibly ZFS. +# +#rocksdb_direct_io = true + +# Amount of threads that RocksDB will use for parallelism on database +# operatons such as cleanup, sync, flush, compaction, etc. Set to 0 to use +# all your logical threads. Defaults to your CPU logical thread count. # -# Defaults to your CPU logical thread count. #rocksdb_parallelism_threads = 0 -# Enables idle IO priority for compaction thread. This prevents any unexpected lag in the server's operation and -# is usually a good idea. Enabled by default. -#rocksdb_compaction_ioprio_idle = true - -# Enables idle CPU priority for compaction thread. This is not enabled by default to prevent compaction from -# falling too far behind on busy systems. -#rocksdb_compaction_prio_idle = false - -# Maximum number of LOG files RocksDB will keep. This must *not* be set to 0. It must be at least 1. -# Defaults to 3 as these are not very useful. +# Maximum number of LOG files RocksDB will keep. This must *not* be set to +# 0. It must be at least 1. Defaults to 3 as these are not very useful +# unless troubleshooting/debugging a RocksDB bug. +# #rocksdb_max_log_files = 3 # Type of RocksDB database compression to use. +# # Available options are "zstd", "zlib", "bz2", "lz4", or "none" -# It is best to use ZSTD as an overall good balance between speed/performance, storage, IO amplification, and CPU usage. -# For more performance but less compression (more storage used) and less CPU usage, use LZ4. -# See https://github.com/facebook/rocksdb/wiki/Compression for more details. +# +# It is best to use ZSTD as an overall good balance between +# speed/performance, storage, IO amplification, and CPU usage. +# For more performance but less compression (more storage used) and less +# CPU usage, use LZ4. See https://github.com/facebook/rocksdb/wiki/Compression for more details. # # "none" will disable compression. # -# Defaults to "zstd" #rocksdb_compression_algo = "zstd" -# Level of compression the specified compression algorithm for RocksDB to use. -# Default is 32767, which is internally read by RocksDB as the default magic number and -# translated to the library's default compression level as they all differ. +# Level of compression the specified compression algorithm for RocksDB to +# use. +# +# Default is 32767, which is internally read by RocksDB as the +# default magic number and translated to the library's default +# compression level as they all differ. # See their `kDefaultCompressionLevel`. # #rocksdb_compression_level = 32767 -# Level of compression the specified compression algorithm for the bottommost level/data for RocksDB to use. -# Default is 32767, which is internally read by RocksDB as the default magic number and -# translated to the library's default compression level as they all differ. +# Level of compression the specified compression algorithm for the +# bottommost level/data for RocksDB to use. Default is 32767, which is +# internally read by RocksDB as the default magic number and translated +# to the library's default compression level as they all differ. # See their `kDefaultCompressionLevel`. # -# Since this is the bottommost level (generally old and least used data), it may be desirable to have a very -# high compression level here as it's lesss likely for this data to be used. Research your chosen compression algorithm. +# Since this is the bottommost level (generally old and least used data), +# it may be desirable to have a very high compression level here as it's +# lesss likely for this data to be used. Research your chosen compression +# algorithm. # #rocksdb_bottommost_compression_level = 32767 -# Whether to enable RocksDB "bottommost_compression". -# At the expense of more CPU usage, this will further compress the database to reduce more storage. -# It is recommended to use ZSTD compression with this for best compression results. +# Whether to enable RocksDB's "bottommost_compression". +# +# At the expense of more CPU usage, this will further compress the +# database to reduce more storage. It is recommended to use ZSTD +# compression with this for best compression results. This may be useful +# if you're trying to reduce storage usage from the database. +# # See https://github.com/facebook/rocksdb/wiki/Compression for more details. # -# Defaults to false as this uses more CPU when compressing. #rocksdb_bottommost_compression = false -# Level of statistics collection. Some admin commands to display database statistics may require -# this option to be set. Database performance may be impacted by higher settings. +# Database recovery mode (for RocksDB WAL corruption) +# +# Use this option when the server reports corruption and refuses to start. +# Set mode 2 (PointInTime) to cleanly recover from this corruption. The +# server will continue from the last good state, several seconds or +# minutes prior to the crash. Clients may have to run "clear-cache & +# reload" to account for the rollback. Upon success, you may reset the +# mode back to default and restart again. Please note in some cases the +# corruption error may not be cleared for at least 30 minutes of +# operation in PointInTime mode. +# +# As a very last ditch effort, if PointInTime does not fix or resolve +# anything, you can try mode 3 (SkipAnyCorruptedRecord) but this will +# leave the server in a potentially inconsistent state. +# +# The default mode 1 (TolerateCorruptedTailRecords) will automatically +# drop the last entry in the database if corrupted during shutdown, but +# nothing more. It is extraordinarily unlikely this will desynchronize +# clients. To disable any form of silent rollback set mode 0 +# (AbsoluteConsistency). +# +# The options are: +# 0 = AbsoluteConsistency +# 1 = TolerateCorruptedTailRecords (default) +# 2 = PointInTime (use me if trying to recover) +# 3 = SkipAnyCorruptedRecord (you now voided your Conduwuit warranty) +# +# See https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes for more information on these modes. +# +# See https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption for more details on recovering a corrupt database. +# +#rocksdb_recovery_mode = 1 + +# Database repair mode (for RocksDB SST corruption) +# +# Use this option when the server reports corruption while running or +# panics. If the server refuses to start use the recovery mode options +# first. Corruption errors containing the acronym 'SST' which occur after +# startup will likely require this option. +# +# - Backing up your database directory is recommended prior to running the +# repair. +# - Disabling repair mode and restarting the server is recommended after +# running the repair. +# +# See https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption for more details on recovering a corrupt database. +# +#rocksdb_repair = false + +# This item is undocumented. Please contribute documentation for it. +# +#rocksdb_read_only = false + +# This item is undocumented. Please contribute documentation for it. +# +#rocksdb_secondary = false + +# Enables idle CPU priority for compaction thread. This is not enabled by +# default to prevent compaction from falling too far behind on busy +# systems. +# +#rocksdb_compaction_prio_idle = false + +# Enables idle IO priority for compaction thread. This prevents any +# unexpected lag in the server's operation and is usually a good idea. +# Enabled by default. +# +#rocksdb_compaction_ioprio_idle = true + +# Config option to disable RocksDB compaction. You should never ever have +# to disable this. If you for some reason find yourself needing to disable +# this as part of troubleshooting or a bug, please reach out to us in the +# conduwuit Matrix room with information and details. +# +# Disabling compaction will lead to a significantly bloated and +# explosively large database, gradually poor performance, unnecessarily +# excessive disk read/writes, and slower shutdowns and startups. +# +#rocksdb_compaction = true + +# Level of statistics collection. Some admin commands to display database +# statistics may require this option to be set. Database performance may +# be impacted by higher settings. # # Option is a number ranging from 0 to 6: # 0 = No statistics. @@ -608,338 +870,509 @@ allow_profile_lookup_federation_requests = true # 3 to 5 = Statistics with possible performance impact. # 6 = All statistics. # -# Defaults to 1 (No statistics, except in debug-mode) #rocksdb_stats_level = 1 -# Database repair mode (for RocksDB SST corruption) +# This is a password that can be configured that will let you login to the +# server bot account (currently `@conduit`) for emergency troubleshooting +# purposes such as recovering/recreating your admin room, or inviting +# yourself back. # -# Use this option when the server reports corruption while running or panics. If the server refuses -# to start use the recovery mode options first. Corruption errors containing the acronym 'SST' which -# occur after startup will likely require this option. +# See https://conduwuit.puppyirl.gay/troubleshooting.html#lost-access-to-admin-room for other ways to get back into your admin room. # -# - Backing up your database directory is recommended prior to running the repair. -# - Disabling repair mode and restarting the server is recommended after running the repair. +# Once this password is unset, all sessions will be logged out for +# security purposes. # -# Defaults to false -#rocksdb_repair = false - -# Database recovery mode (for RocksDB WAL corruption) +# example: "F670$2CP@Hw8mG7RY1$%!#Ic7YA" # -# Use this option when the server reports corruption and refuses to start. Set mode 2 (PointInTime) -# to cleanly recover from this corruption. The server will continue from the last good state, -# several seconds or minutes prior to the crash. Clients may have to run "clear-cache & reload" to -# account for the rollback. Upon success, you may reset the mode back to default and restart again. -# Please note in some cases the corruption error may not be cleared for at least 30 minutes of -# operation in PointInTime mode. +#emergency_password = + +# This item is undocumented. Please contribute documentation for it. # -# As a very last ditch effort, if PointInTime does not fix or resolve anything, you can try mode -# 3 (SkipAnyCorruptedRecord) but this will leave the server in a potentially inconsistent state. -# -# The default mode 1 (TolerateCorruptedTailRecords) will automatically drop the last entry in the -# database if corrupted during shutdown, but nothing more. It is extraordinarily unlikely this will -# desynchronize clients. To disable any form of silent rollback set mode 0 (AbsoluteConsistency). -# -# The options are: -# 0 = AbsoluteConsistency -# 1 = TolerateCorruptedTailRecords (default) -# 2 = PointInTime (use me if trying to recover) -# 3 = SkipAnyCorruptedRecord (you now voided your Conduwuit warranty) -# -# See https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes for more information -# -# Defaults to 1 (TolerateCorruptedTailRecords) -#rocksdb_recovery_mode = 1 +#notification_push_path = "/_matrix/push/v1/notify" - -### Domain Name Resolution and Caching - -# Maximum entries stored in DNS memory-cache. The size of an entry may vary so please take care if -# raising this value excessively. Only decrease this when using an external DNS cache. Please note -# that systemd does *not* count as an external cache, even when configured to do so. -#dns_cache_entries = 32768 - -# Minimum time-to-live in seconds for entries in the DNS cache. The default may appear high to most -# administrators; this is by design. Only decrease this if you are using an external DNS cache. -#dns_min_ttl = 10800 - -# Minimum time-to-live in seconds for NXDOMAIN entries in the DNS cache. This value is critical for -# the server to federate efficiently. NXDOMAIN's are assumed to not be returning to the federation -# and aggressively cached rather than constantly rechecked. -# -# Defaults to 3 days as these are *very rarely* false negatives. -#dns_min_ttl_nxdomain = 259200 - -# The number of seconds to wait for a reply to a DNS query. Please note that recursive queries can -# take up to several seconds for some domains, so this value should not be too low. -#dns_timeout = 10 - -# Number of retries after a timeout. -#dns_attempts = 10 - -# Fallback to TCP on DNS errors. Set this to false if unsupported by nameserver. -#dns_tcp_fallback = true - -# Enable to query all nameservers until the domain is found. Referred to as "trust_negative_responses" in hickory_resolver. -# This can avoid useless DNS queries if the first nameserver responds with NXDOMAIN or an empty NOERROR response. -# -# The default is to query one nameserver and stop (false). -#query_all_nameservers = true - -# Enables using *only* TCP for querying your specified nameservers instead of UDP. -# -# You very likely do *not* want this. hickory-resolver already falls back to TCP on UDP errors. -# Defaults to false -#query_over_tcp_only = false - -# DNS A/AAAA record lookup strategy -# -# Takes a number of one of the following options: -# 1 - Ipv4Only (Only query for A records, no AAAA/IPv6) -# 2 - Ipv6Only (Only query for AAAA records, no A/IPv4) -# 3 - Ipv4AndIpv6 (Query for A and AAAA records in parallel, uses whatever returns a successful response first) -# 4 - Ipv6thenIpv4 (Query for AAAA record, if that fails then query the A record) -# 5 - Ipv4thenIpv6 (Query for A record, if that fails then query the AAAA record) -# -# If you don't have IPv6 networking, then for better performance it may be suitable to set this to Ipv4Only (1) as -# you will never ever use the AAAA record contents even if the AAAA record is successful instead of the A record. -# -# Defaults to 5 - Ipv4ThenIpv6 as this is the most compatible and IPv4 networking is currently the most prevalent. -#ip_lookup_strategy = 5 - - -### Request Timeouts, Connection Timeouts, and Connection Pooling - -## Request Timeouts are HTTP response timeouts -## Connection Timeouts are TCP connection timeouts -## -## Connection Pooling Timeouts are timeouts for keeping an open idle connection alive. -## Connection pooling and keepalive is very useful for federation or other places where for performance reasons, -## we want to keep connections open that we will re-use frequently due to TCP and TLS 1.3 overhead/expensiveness. -## -## Generally these defaults are the best, but if you find a reason to need to change these they are here. - -# Default/base connection timeout. -# This is used only by URL previews and update/news endpoint checks -# -# Defaults to 10 seconds -#request_conn_timeout = 10 - -# Default/base request timeout. The time waiting to receive more data from another server. -# This is used only by URL previews, update/news, and misc endpoint checks -# -# Defaults to 35 seconds -#request_timeout = 35 - -# Default/base request total timeout. The time limit for a whole request. This is set very high to not -# cancel healthy requests while serving as a backstop. -# This is used only by URL previews and update/news endpoint checks -# -# Defaults to 320 seconds -#request_total_timeout = 320 - -# Default/base idle connection pool timeout -# This is used only by URL previews and update/news endpoint checks -# -# Defaults to 5 seconds -#request_idle_timeout = 5 - -# Default/base max idle connections per host -# This is used only by URL previews and update/news endpoint checks -# -# Defaults to 1 as generally the same open connection can be re-used -#request_idle_per_host = 1 - -# Federation well-known resolution connection timeout -# -# Defaults to 6 seconds -#well_known_conn_timeout = 6 - -# Federation HTTP well-known resolution request timeout -# -# Defaults to 10 seconds -#well_known_timeout = 10 - -# Federation client request timeout -# You most definitely want this to be high to account for extremely large room joins, slow homeservers, your own resources etc. -# -# Defaults to 300 seconds -#federation_timeout = 300 - -# Federation client idle connection pool timeout -# -# Defaults to 25 seconds -#federation_idle_timeout = 25 - -# Federation client max idle connections per host -# -# Defaults to 1 as generally the same open connection can be re-used -#federation_idle_per_host = 1 - -# Federation sender request timeout -# The time it takes for the remote server to process sent transactions can take a while. -# -# Defaults to 180 seconds -#sender_timeout = 180 - -# Federation sender idle connection pool timeout -# -# Defaults to 180 seconds -#sender_idle_timeout = 180 - -# Federation sender transaction retry backoff limit -# -# Defaults to 86400 seconds -#sender_retry_backoff_limit = 86400 - -# Appservice URL request connection timeout -# -# Defaults to 35 seconds as generally appservices are hosted within the same network -#appservice_timeout = 35 - -# Appservice URL idle connection pool timeout -# -# Defaults to 300 seconds -#appservice_idle_timeout = 300 - -# Notification gateway pusher idle connection pool timeout -# -# Defaults to 15 seconds -#pusher_idle_timeout = 15 - - -### Presence / Typing Indicators / Read Receipts - -# Config option to control local (your server only) presence updates/requests. Defaults to true. -# Note that presence on conduwuit is very fast unlike Synapse's. -# If using outgoing presence, this MUST be enabled. +# Config option to control local (your server only) presence +# updates/requests. Note that presence on conduwuit is +# very fast unlike Synapse's. If using outgoing presence, this MUST be +# enabled. # #allow_local_presence = true -# Config option to control incoming federated presence updates/requests. Defaults to true. -# This option receives presence updates from other servers, but does not send any unless `allow_outgoing_presence` is true. +# Config option to control incoming federated presence updates/requests. +# +# This option receives presence updates from other +# servers, but does not send any unless `allow_outgoing_presence` is true. # Note that presence on conduwuit is very fast unlike Synapse's. # #allow_incoming_presence = true -# Config option to control outgoing presence updates/requests. Defaults to true. -# This option sends presence updates to other servers, but does not receive any unless `allow_incoming_presence` is true. +# Config option to control outgoing presence updates/requests. +# +# This option sends presence updates to other servers, but does not +# receive any unless `allow_incoming_presence` is true. # Note that presence on conduwuit is very fast unlike Synapse's. -# If using outgoing presence, you MUST enable `allow_local_presence` as well. +# If using outgoing presence, you MUST enable `allow_local_presence` as +# well. # #allow_outgoing_presence = true -# Config option to enable the presence idle timer for remote users. Disabling is offered as an optimization for -# servers participating in many large rooms or when resources are limited. Disabling it may cause incorrect -# presence states (i.e. stuck online) to be seen for some remote users. Defaults to true. -#presence_timeout_remote_users = true - -# Config option to control how many seconds before presence updates that you are idle. Defaults to 5 minutes. +# Config option to control how many seconds before presence updates that +# you are idle. Defaults to 5 minutes. +# #presence_idle_timeout_s = 300 -# Config option to control how many seconds before presence updates that you are offline. Defaults to 30 minutes. +# Config option to control how many seconds before presence updates that +# you are offline. Defaults to 30 minutes. +# #presence_offline_timeout_s = 1800 -# Config option to control whether we should receive remote incoming read receipts. -# Defaults to true. +# Config option to enable the presence idle timer for remote users. +# Disabling is offered as an optimization for servers participating in +# many large rooms or when resources are limited. Disabling it may cause +# incorrect presence states (i.e. stuck online) to be seen for some +# remote users. +# +#presence_timeout_remote_users = true + +# Config option to control whether we should receive remote incoming read +# receipts. +# #allow_incoming_read_receipts = true -# Config option to control whether we should send read receipts to remote servers. -# Defaults to true. +# Config option to control whether we should send read receipts to remote +# servers. +# #allow_outgoing_read_receipts = true -# Config option to control outgoing typing updates to federation. Defaults to true. +# Config option to control outgoing typing updates to federation. +# #allow_outgoing_typing = true -# Config option to control incoming typing updates from federation. Defaults to true. +# Config option to control incoming typing updates from federation. +# #allow_incoming_typing = true -# Config option to control maximum time federation user can indicate typing. +# Config option to control maximum time federation user can indicate +# typing. +# #typing_federation_timeout_s = 30 -# Config option to control minimum time local client can indicate typing. This does not override -# a client's request to stop typing. It only enforces a minimum value in case of no stop request. +# Config option to control minimum time local client can indicate typing. +# This does not override a client's request to stop typing. It only +# enforces a minimum value in case of no stop request. +# #typing_client_timeout_min_s = 15 # Config option to control maximum time local client can indicate typing. +# #typing_client_timeout_max_s = 45 +# Set this to true for conduwuit to compress HTTP response bodies using +# zstd. This option does nothing if conduwuit was not built with +# `zstd_compression` feature. Please be aware that enabling HTTP +# compression may weaken TLS. Most users should not need to enable this. +# See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH +# before deciding to enable this. +# +#zstd_compression = false -### TURN / VoIP +# Set this to true for conduwuit to compress HTTP response bodies using +# gzip. This option does nothing if conduwuit was not built with +# `gzip_compression` feature. Please be aware that enabling HTTP +# compression may weaken TLS. Most users should not need to enable this. +# See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before +# deciding to enable this. +# +# If you are in a large amount of rooms, you may find that enabling this +# is necessary to reduce the significantly large response bodies. +# +#gzip_compression = false -# vector list of TURN URIs/servers to use +# Set this to true for conduwuit to compress HTTP response bodies using +# brotli. This option does nothing if conduwuit was not built with +# `brotli_compression` feature. Please be aware that enabling HTTP +# compression may weaken TLS. Most users should not need to enable this. +# See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before +# deciding to enable this. # -# replace "example.turn.uri" with your TURN domain, such as the coturn "realm". -# if using TURN over TLS, replace "turn:" with "turns:" -# -# No default -#turn_uris = ["turn:example.turn.uri?transport=udp", "turn:example.turn.uri?transport=tcp"] +#brotli_compression = false -# TURN secret to use that's read from the file path specified +# Set to true to allow user type "guest" registrations. Some clients like +# Element attempt to register guest users automatically. # -# this takes priority over "turn_secret" first, and falls back to "turn_secret" if invalid or -# failed to open. -# -# no default -#turn_secret_file = "/path/to/secret.txt" +#allow_guest_registration = false -# TURN secret to use for generating the HMAC-SHA1 hash apart of username and password generation +# Set to true to log guest registrations in the admin room. Note that +# these may be noisy or unnecessary if you're a public homeserver. # -# this is more secure, but if needed you can use traditional username/password below. -# -# no default -#turn_secret = "" +#log_guest_registrations = false -# TURN username to provide the client +# Set to true to allow guest registrations/users to auto join any rooms +# specified in `auto_join_rooms`. # -# no default -#turn_username = "" +#allow_guests_auto_join_rooms = false -# TURN password to provide the client +# Config option to control whether the legacy unauthenticated Matrix media +# repository endpoints will be enabled. These endpoints consist of: +# - /_matrix/media/*/config +# - /_matrix/media/*/upload +# - /_matrix/media/*/preview_url +# - /_matrix/media/*/download/* +# - /_matrix/media/*/thumbnail/* # -# no default -#turn_password = "" +# The authenticated equivalent endpoints are always enabled. +# +# Defaults to true for now, but this is highly subject to change, likely +# in the next release. +# +#allow_legacy_media = true -# TURN TTL +# This item is undocumented. Please contribute documentation for it. # -# Default is 86400 seconds -#turn_ttl = 86400 +#freeze_legacy_media = true -# allow guests/unauthenticated users to access TURN credentials +# Checks consistency of the media directory at startup: +# 1. When `media_compat_file_link` is enbled, this check will upgrade +# media when switching back and forth between Conduit and conduwuit. +# Both options must be enabled to handle this. +# 2. When media is deleted from the directory, this check will also delete +# its database entry. # -# this is the equivalent of Synapse's `turn_allow_guests` config option. this allows -# any unauthenticated user to call `/_matrix/client/v3/voip/turnServer`. +# If none of these checks apply to your use cases, and your media +# directory is significantly large setting this to false may reduce +# startup time. # -# defaults to false -#turn_allow_guests = false +#media_startup_check = true +# Enable backward-compatibility with Conduit's media directory by creating +# symlinks of media. This option is only necessary if you plan on using +# Conduit again. Otherwise setting this to false reduces filesystem +# clutter and overhead for managing these symlinks in the directory. This +# is now disabled by default. You may still return to upstream Conduit +# but you have to run conduwuit at least once with this set to true and +# allow the media_startup_check to take place before shutting +# down to return to Conduit. +# +#media_compat_file_link = false -# Other options not in [global]: +# Prunes missing media from the database as part of the media startup +# checks. This means if you delete files from the media directory the +# corresponding entries will be removed from the database. This is +# disabled by default because if the media directory is accidentally moved +# or inaccessible, the metadata entries in the database will be lost with +# sadness. +# +#prune_missing_media = false + +# Vector list of servers that conduwuit will refuse to download remote +# media from. +# +#prevent_media_downloads_from = [] + +# List of forbidden server names that we will block incoming AND outgoing +# federation with, and block client room joins / remote user invites. +# +# This check is applied on the room ID, room alias, sender server name, +# sender user's server name, inbound federation X-Matrix origin, and +# outbound federation handler. +# +# Basically "global" ACLs. +# +#forbidden_remote_server_names = [] + +# List of forbidden server names that we will block all outgoing federated +# room directory requests for. Useful for preventing our users from +# wandering into bad servers or spaces. +# +#forbidden_remote_room_directory_server_names = [] + +# Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you +# do not want conduwuit to send outbound requests to. Defaults to +# RFC1918, unroutable, loopback, multicast, and testnet addresses for +# security. +# +# Please be aware that this is *not* a guarantee. You should be using a +# firewall with zones as doing this on the application layer may have +# bypasses. +# +# Currently this does not account for proxies in use like Synapse does. +# +# To disable, set this to be an empty vector (`[]`). +# +# "192.168.0.0/16", "100.64.0.0/10", "192.0.0.0/24", "169.254.0.0/16", +# "192.88.99.0/24", "198.18.0.0/15", "192.0.2.0/24", "198.51.100.0/24", +# "203.0.113.0/24", "224.0.0.0/4", "::1/128", "fe80::/10", "fc00::/7", +# "2001:db8::/32", "ff00::/8", "fec0::/10"] +# +#ip_range_denylist = ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", + +# Vector list of domains allowed to send requests to for URL previews. +# Defaults to none. Note: this is a *contains* match, not an explicit +# match. Putting "google.com" will match "https://google.com" and +# "http://mymaliciousdomainexamplegoogle.com" Setting this to "*" will +# allow all URL previews. Please note that this opens up significant +# attack surface to your server, you are expected to be aware of the +# risks by doing so. +# +#url_preview_domain_contains_allowlist = [] + +# Vector list of explicit domains allowed to send requests to for URL +# previews. Defaults to none. Note: This is an *explicit* match, not a +# contains match. Putting "google.com" will match "https://google.com", +# "http://google.com", but not +# "https://mymaliciousdomainexamplegoogle.com". Setting this to "*" will +# allow all URL previews. Please note that this opens up significant +# attack surface to your server, you are expected to be aware of the +# risks by doing so. +# +#url_preview_domain_explicit_allowlist = [] + +# Vector list of explicit domains not allowed to send requests to for URL +# previews. Defaults to none. Note: This is an *explicit* match, not a +# contains match. Putting "google.com" will match "https://google.com", +# "http://google.com", but not +# "https://mymaliciousdomainexamplegoogle.com". The denylist is checked +# first before allowlist. Setting this to "*" will not do anything. +# +#url_preview_domain_explicit_denylist = [] + +# Vector list of URLs allowed to send requests to for URL previews. +# Defaults to none. Note that this is a *contains* match, not an +# explicit match. Putting "google.com" will match +# "https://google.com/", +# "https://google.com/url?q=https://mymaliciousdomainexample.com", and +# "https://mymaliciousdomainexample.com/hi/google.com" Setting this to +# "*" will allow all URL previews. Please note that this opens up +# significant attack surface to your server, you are expected to be +# aware of the risks by doing so. +# +#url_preview_url_contains_allowlist = [] + +# Maximum amount of bytes allowed in a URL preview body size when +# spidering. Defaults to 384KB in bytes. +# +#url_preview_max_spider_size = 384000 + +# Option to decide whether you would like to run the domain allowlist +# checks (contains and explicit) on the root domain or not. Does not apply +# to URL contains allowlist. Defaults to false. +# +# Example usecase: If this is +# enabled and you have "wikipedia.org" allowed in the explicit and/or +# contains domain allowlist, it will allow all subdomains under +# "wikipedia.org" such as "en.m.wikipedia.org" as the root domain is +# checked and matched. Useful if the domain contains allowlist is still +# too broad for you but you still want to allow all the subdomains under a +# root domain. +# +#url_preview_check_root_domain = false + +# List of forbidden room aliases and room IDs as strings of regex +# patterns. +# +# Regex can be used or explicit contains matches can be done by +# just specifying the words (see example). +# +# This is checked upon room alias creation, custom room ID creation if +# used, and startup as warnings if any room aliases in your database have +# a forbidden room alias/ID. +# +# example: ["19dollarfortnitecards", "b[4a]droom"] +# +#forbidden_alias_names = [] + +# List of forbidden username patterns/strings. +# +# Regex can be used or explicit contains matches can be done by just +# specifying the words (see example). +# +# This is checked upon username availability check, registration, and +# startup as warnings if any local users in your database have a forbidden +# username. +# +# example: ["administrator", "b[a4]dusernam[3e]"] +# +#forbidden_usernames = [] + +# Retry failed and incomplete messages to remote servers immediately upon +# startup. This is called bursting. If this is disabled, said messages +# may not be delivered until more messages are queued for that server. Do +# not change this option unless server resources are extremely limited or +# the scale of the server's deployment is huge. Do not disable this +# unless you know what you are doing. +# +#startup_netburst = true + +# messages are dropped and not reattempted. The `startup_netburst` option +# must be enabled for this value to have any effect. Do not change this +# value unless you know what you are doing. Set this value to -1 to +# reattempt every message without trimming the queues; this may consume +# significant disk. Set this value to 0 to drop all messages without any +# attempt at redelivery. +# +#startup_netburst_keep = 50 + +# controls whether non-admin local users are forbidden from sending room +# invites (local and remote), and if non-admin users can receive remote +# room invites. admins are always allowed to send and receive all room +# invites. +# +#block_non_admin_invites = false + +# Allows admins to enter commands in rooms other than "#admins" (admin +# room) by prefixing your message with "\!admin" or "\\!admin" followed +# up a normal conduwuit admin command. The reply will be publicly visible +# to the room, originating from the sender. +# +# example: \\!admin debug ping puppygock.gay +# +#admin_escape_commands = true + +# Controls whether the conduwuit admin room console / CLI will immediately +# activate on startup. This option can also be enabled with `--console` +# conduwuit argument. +# +#admin_console_automatic = false + +# Controls what admin commands will be executed on startup. This is a +# vector list of strings of admin commands to run. # # -# Enables running conduwuit with direct TLS support -# It is strongly recommended you use a reverse proxy instead. This is primarily relevant for test suites like complement that require a private CA setup. -# [global.tls] -# certs = "/path/to/my/certificate.crt" -# key = "/path/to/my/private_key.key" +# This option can also be configured with the `--execute` conduwuit +# argument and can take standard shell commands and environment variables # +# Such example could be: `./conduwuit --execute "server admin-notice +# conduwuit has started up at $(date)"` +# +# example: admin_execute = ["debug ping puppygock.gay", "debug echo hi"]` +# +#admin_execute = [] + +# Controls whether conduwuit should error and fail to start if an admin +# execute command (`--execute` / `admin_execute`) fails. +# +#admin_execute_errors_ignore = false + +# Controls the max log level for admin command log captures (logs +# generated from running admin commands). Defaults to "info" on release +# builds, else "debug" on debug builds. +# +#admin_log_capture = "info" + +# The default room tag to apply on the admin room. +# +# On some clients like Element, the room tag "m.server_notice" is a +# special pinned room at the very bottom of your room list. The conduwuit +# admin room can be pinned here so you always have an easy-to-access +# shortcut dedicated to your admin room. +# +#admin_room_tag = "m.server_notice" + +# Sentry.io crash/panic reporting, performance monitoring/metrics, etc. +# This is NOT enabled by default. conduwuit's default Sentry reporting +# endpoint is o4506996327251968.ingest.us.sentry.io +# +#sentry = false + +# Sentry reporting URL if a custom one is desired +# +#sentry_endpoint = "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" + +# Report your conduwuit server_name in Sentry.io crash reports and metrics +# +#sentry_send_server_name = false + +# Performance monitoring/tracing sample rate for Sentry.io +# +# Note that too high values may impact performance, and can be disabled by +# setting it to 0.0 (0%) This value is read as a percentage to Sentry, +# represented as a decimal. Defaults to 15% of traces (0.15) +# +#sentry_traces_sample_rate = 0.15 + +# Whether to attach a stacktrace to Sentry reports. +# +#sentry_attach_stacktrace = false + +# Send panics to sentry. This is true by default, but sentry has to be +# enabled. The global "sentry" config option must be enabled to send any +# data. +# +#sentry_send_panic = true + +# Send errors to sentry. This is true by default, but sentry has to be +# enabled. This option is only effective in release-mode; forced to false +# in debug-mode. +# +#sentry_send_error = true + +# Controls the tracing log level for Sentry to send things like +# breadcrumbs and transactions +# +#sentry_filter = "info" + +# Enable the tokio-console. This option is only relevant to developers. +# See https://conduwuit.puppyirl.gay/development.html#debugging-with-tokio-console for more information. +# +#tokio_console = false + +# This item is undocumented. Please contribute documentation for it. +# +#test = false + +# Controls whether admin room notices like account registrations, password +# changes, account deactivations, room directory publications, etc will +# be sent to the admin room. Update notices and normal admin command +# responses will still be sent. +# +#admin_room_notices = true + +[global.tls] + +# Path to a valid TLS certificate file. +# +# example: "/path/to/my/certificate.crt" +# +#certs = + +# Path to a valid TLS certificate private key. +# +# example: "/path/to/my/certificate.key" +# +#key = + # Whether to listen and allow for HTTP and HTTPS connections (insecure!) -# This config option is only available if conduwuit was built with `axum_dual_protocol` feature (not default feature) -# Defaults to false +# #dual_protocol = false +[global.well_known] -# If you are using delegation via well-known files and you cannot serve them from your reverse proxy, you can -# uncomment these to serve them directly from conduwuit. This requires proxying all requests to conduwuit, not just `/_matrix` to work. +# The server base domain of the URL with a specific port that the server +# well-known file will serve. This should contain a port at the end, and +# should not be a URL. # -#[global.well_known] -#server = "matrix.example.com:443" -#client = "https://matrix.example.com" +# example: "matrix.example.com:443" # -# A single contact and/or support page for /.well-known/matrix/support -# All options here are strings. Currently only supports 1 single contact. -# No default. +#server = + +# The server URL that the client well-known file will serve. This should +# not contain a port, and should just be a valid HTTPS URL. # -#support_page = "" -#support_role = "" -#support_email = "" -#support_mxid = "" +# example: "https://matrix.example.com" +# +#client = + +# This item is undocumented. Please contribute documentation for it. +# +#support_page = + +# This item is undocumented. Please contribute documentation for it. +# +#support_role = + +# This item is undocumented. Please contribute documentation for it. +# +#support_email = + +# This item is undocumented. Please contribute documentation for it. +# +#support_mxid = diff --git a/src/api/server/make_knock.rs b/src/api/server/make_knock.rs new file mode 100644 index 00000000..c1875a1f --- /dev/null +++ b/src/api/server/make_knock.rs @@ -0,0 +1,107 @@ +use axum::extract::State; +use conduit::Err; +use ruma::{ + api::{client::error::ErrorKind, federation::knock::create_knock_event_template}, + events::room::member::{MembershipState, RoomMemberEventContent}, + RoomVersionId, +}; +use serde_json::value::to_raw_value; +use tracing::warn; +use RoomVersionId::*; + +use crate::{service::pdu::PduBuilder, Error, Result, Ruma}; + +/// # `GET /_matrix/federation/v1/make_knock/{roomId}/{userId}` +/// +/// Creates a knock template. +pub(crate) async fn create_knock_event_template_route( + State(services): State, body: Ruma, +) -> Result { + if !services.rooms.metadata.exists(&body.room_id).await { + return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + } + + if body.user_id.server_name() != body.origin() { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to knock on behalf of another server/user", + )); + } + + // ACL check origin server + services + .rooms + .event_handler + .acl_check(body.origin(), &body.room_id) + .await?; + + if services + .globals + .config + .forbidden_remote_server_names + .contains(body.origin()) + { + warn!( + "Server {} for remote user {} tried knocking room ID {} which has a server name that is globally \ + forbidden. Rejecting.", + body.origin(), + &body.user_id, + &body.room_id, + ); + return Err!(Request(Forbidden("Server is banned on this homeserver."))); + } + + if let Some(server) = body.room_id.server_name() { + if services + .globals + .config + .forbidden_remote_server_names + .contains(&server.to_owned()) + { + return Err!(Request(Forbidden("Server is banned on this homeserver."))); + } + } + + let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; + + if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6) { + return Err(Error::BadRequest( + ErrorKind::IncompatibleRoomVersion { + room_version: room_version_id, + }, + "Room version does not support knocking.", + )); + } + + if !body.ver.contains(&room_version_id) { + return Err(Error::BadRequest( + ErrorKind::IncompatibleRoomVersion { + room_version: room_version_id, + }, + "Your homeserver does not support the features required to knock on this room.", + )); + } + + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + + let (_pdu, mut pdu_json) = services + .rooms + .timeline + .create_hash_and_sign_event( + PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Knock)), + &body.user_id, + &body.room_id, + &state_lock, + ) + .await?; + + drop(state_lock); + + // room v3 and above removed the "event_id" field from remote PDU format + super::maybe_strip_event_id(&mut pdu_json, &room_version_id)?; + + Ok(create_knock_event_template::v1::Response { + room_version: room_version_id, + event: to_raw_value(&pdu_json).expect("CanonicalJson can be serialized to JSON"), + }) +} diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs new file mode 100644 index 00000000..c57998ae --- /dev/null +++ b/src/api/server/send_knock.rs @@ -0,0 +1,190 @@ +use axum::extract::State; +use conduit::{err, pdu::gen_event_id_canonical_json, warn, Err, Error, PduEvent, Result}; +use ruma::{ + api::{client::error::ErrorKind, federation::knock::send_knock}, + events::{ + room::member::{MembershipState, RoomMemberEventContent}, + StateEventType, + }, + serde::JsonObject, + OwnedServerName, OwnedUserId, + RoomVersionId::*, +}; + +use crate::Ruma; + +/// # `PUT /_matrix/federation/v1/send_knock/{roomId}/{eventId}` +/// +/// Submits a signed knock event. +pub(crate) async fn create_knock_event_v1_route( + State(services): State, body: Ruma, +) -> Result { + if services + .globals + .config + .forbidden_remote_server_names + .contains(body.origin()) + { + warn!( + "Server {} tried knocking room ID {} who has a server name that is globally forbidden. Rejecting.", + body.origin(), + &body.room_id, + ); + return Err!(Request(Forbidden("Server is banned on this homeserver."))); + } + + if let Some(server) = body.room_id.server_name() { + if services + .globals + .config + .forbidden_remote_server_names + .contains(&server.to_owned()) + { + warn!( + "Server {} tried knocking room ID {} which has a server name that is globally forbidden. Rejecting.", + body.origin(), + &body.room_id, + ); + return Err!(Request(Forbidden("Server is banned on this homeserver."))); + } + } + + if !services.rooms.metadata.exists(&body.room_id).await { + return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + } + + // ACL check origin server + services + .rooms + .event_handler + .acl_check(body.origin(), &body.room_id) + .await?; + + let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; + + if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6) { + return Err!(Request(Forbidden("Room version does not support knocking."))); + } + + let Ok((event_id, value)) = gen_event_id_canonical_json(&body.pdu, &room_version_id) else { + // Event could not be converted to canonical json + return Err!(Request(InvalidParam("Could not convert event to canonical json."))); + }; + + let event_type: StateEventType = serde_json::from_value( + value + .get("type") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event has invalid event type."))?; + + if event_type != StateEventType::RoomMember { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to send non-membership state event to knock endpoint.", + )); + } + + let content: RoomMemberEventContent = serde_json::from_value( + value + .get("content") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property"))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid"))?; + + if content.membership != MembershipState::Knock { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to send a non-knock membership event to knock endpoint.", + )); + } + + // ACL check sender server name + let sender: OwnedUserId = serde_json::from_value( + value + .get("sender") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "sender is not a valid user ID."))?; + + services + .rooms + .event_handler + .acl_check(sender.server_name(), &body.room_id) + .await?; + + // check if origin server is trying to send for another server + if sender.server_name() != body.origin() { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Not allowed to knock on behalf of another server.", + )); + } + + let state_key: OwnedUserId = serde_json::from_value( + value + .get("state_key") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))? + .clone() + .into(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID."))?; + + if state_key != sender { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "State key does not match sender user", + )); + }; + + let origin: OwnedServerName = serde_json::from_value( + serde_json::to_value( + value + .get("origin") + .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?, + ) + .expect("CanonicalJson is valid json value"), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; + + let mut event: JsonObject = serde_json::from_str(body.pdu.get()) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid knock event PDU."))?; + + event.insert("event_id".to_owned(), "$placeholder".into()); + + let pdu: PduEvent = serde_json::from_value(event.into()) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid knock event PDU."))?; + + let mutex_lock = services + .rooms + .event_handler + .mutex_federation + .lock(&body.room_id) + .await; + + let pdu_id = services + .rooms + .event_handler + .handle_incoming_pdu(&origin, &body.room_id, &event_id, value.clone(), true) + .await? + .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; + + drop(mutex_lock); + + let knock_room_state = services.rooms.state.summary_stripped(&pdu).await; + + services + .sending + .send_pdu_room(&body.room_id, &pdu_id) + .await?; + + Ok(send_knock::v1::Response { + knock_room_state, + }) +} diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index eddab2fe..4bba1455 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -36,45 +36,51 @@ use crate::{err, error::Error, utils::sys, Result}; filename = "conduwuit-example.toml", section = "global", undocumented = "# This item is undocumented. Please contribute documentation for it.", - header = "### Conduwuit Configuration\n###\n### THIS FILE IS GENERATED. YOUR CHANGES WILL BE OVERWRITTEN!\n### \ - You should rename this file before configuring your server. Changes\n### to documentation and defaults \ - can be contributed in sourcecode at\n### src/core/config/mod.rs. This file is generated when \ - building.\n###\n", + header = "### conduwuit Configuration\n###\n### THIS FILE IS GENERATED. CHANGES/CONTRIBUTIONS IN THE REPO WILL\n### BE \ + OVERWRITTEN!\n###\n### You should rename this file before configuring your server. Changes\n### to \ + documentation and defaults can be contributed in source code at\n### src/core/config/mod.rs. This file \ + is generated when building.\n###\n### Any values pre-populated are the default values for said config \ + option.\n###\n### At the minimum, you MUST edit all the config options to your environment\n### that say \ + \"YOU NEED TO EDIT THIS\".\n### See https://conduwuit.puppyirl.gay/configuration.html for ways to\n### configure conduwuit\n", ignore = "catchall well_known tls" )] pub struct Config { /// The server_name is the pretty name of this server. It is used as a - /// suffix for user and room ids. Examples: matrix.org, conduit.rs + /// suffix for user and room IDs/aliases. /// - /// The Conduit server needs all /_matrix/ requests to be reachable at - /// https://your.server.name/ on port 443 (client-server) and 8448 (federation). + /// See the docs for reverse proxying and delegation: https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy + /// Also see the `[global.well_known]` config section at the very bottom. /// - /// If that's not possible for you, you can create /.well-known files to - /// redirect requests (delegation). See - /// https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixclient - /// and - /// https://spec.matrix.org/v1.9/server-server-api/#getwell-knownmatrixserver - /// for more information. + /// Examples of delegation: + /// - https://puppygock.gay/.well-known/matrix/server + /// - https://puppygock.gay/.well-known/matrix/client /// - /// YOU NEED TO EDIT THIS + /// YOU NEED TO EDIT THIS. THIS CANNOT BE CHANGED AFTER WITHOUT A DATABASE + /// WIPE. + /// + /// example: "conduwuit.woof" pub server_name: OwnedServerName, - /// default address (IPv4 or IPv6) conduwuit will listen on. Generally you - /// want this to be localhost (127.0.0.1 / ::1). If you are using Docker or - /// a container NAT networking setup, you likely need this to be 0.0.0.0. - /// To listen multiple addresses, specify a vector e.g. ["127.0.0.1", "::1"] - /// Default if unspecified is both IPv4 and IPv6 localhost. + /// default address (IPv4 or IPv6) conduwuit will listen on. + /// + /// If you are using Docker or a container NAT networking setup, this must + /// be "0.0.0.0". + /// + /// To listen on multiple addresses, specify a vector e.g. ["127.0.0.1", + /// "::1"] /// /// default: ["127.0.0.1", "::1"] #[serde(default = "default_address")] address: ListeningAddr, - /// The port(s) conduwuit will be running on. You need to set up a reverse - /// proxy such as Caddy or Nginx so all requests to /_matrix on port 443 - /// and 8448 will be forwarded to the conduwuit instance running on this - /// port Docker users: Don't change this, you'll need to map an external - /// port to this. To listen on multiple ports, specify a vector e.g. [8080, - /// 8448] + /// The port(s) conduwuit will be running on. + /// + /// See https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy for reverse proxying. + /// + /// Docker users: Don't change this, you'll need to map an external port to + /// this. + /// + /// To listen on multiple ports, specify a vector e.g. [8080, 8448] /// /// default: 8008 #[serde(default = "default_port")] @@ -84,108 +90,155 @@ pub struct Config { pub tls: Option, /// Uncomment unix_socket_path to listen on a UNIX socket at the specified - /// path. If listening on a UNIX socket, you must remove/comment the - /// 'address' key if defined and add your reverse proxy to the 'conduwuit' + /// path. If listening on a UNIX socket, you MUST remove/comment the + /// 'address' key if definedm AND add your reverse proxy to the 'conduwuit' /// group, unless world RW permissions are specified with unix_socket_perms /// (666 minimum). + /// + /// example: "/run/conduwuit/conduwuit.sock" pub unix_socket_path: Option, + /// The default permissions (in octal) to create the UNIX socket with. + /// /// default: 660 #[serde(default = "default_unix_socket_perms")] pub unix_socket_perms: u32, - /// Database backend: Only rocksdb is supported. - /// - /// default: rocksdb - #[serde(default = "default_database_backend")] - pub database_backend: String, - /// This is the only directory where conduwuit will save its data, including - /// media. Note: this was previously "/var/lib/matrix-conduit" + /// media. + /// Note: this was previously "/var/lib/matrix-conduit" + /// + /// YOU NEED TO EDIT THIS. + /// + /// example: "/var/lib/conduwuit" pub database_path: PathBuf, + /// conduwuit supports online database backups using RocksDB's Backup engine + /// API. To use this, set a database backup path that conduwuit can write + /// to. + /// + /// See https://conduwuit.puppyirl.gay/maintenance.html#backups for more information. + /// + /// example: "/opt/conduwuit-db-backups" pub database_backup_path: Option, + /// The amount of online RocksDB database backups to keep/retain, if using + /// "database_backup_path", before deleting the oldest one. + /// + /// default: 1 #[serde(default = "default_database_backups_to_keep")] pub database_backups_to_keep: i16, /// Set this to any float value in megabytes for conduwuit to tell the /// database engine that this much memory is available for database-related - /// caches. May be useful if you have significant memory to spare to - /// increase performance. + /// caches. /// - /// default: 256.0 + /// May be useful if you have significant memory to spare to increase + /// performance. + /// + /// Similar to the individual LRU caches, this is scaled up with your CPU + /// core count. + /// + /// This defaults to 128.0 + (64.0 * CPU core count) #[serde(default = "default_db_cache_capacity_mb")] pub db_cache_capacity_mb: f64, /// Option to control adding arbitrary text to the end of the user's /// displayname upon registration with a space before the text. This was the /// lightning bolt emoji option, just replaced with support for adding your - /// own custom text or emojis. To disable, set this to "" (an empty string) - /// Defaults to "🏳️⚧️" (trans pride flag) + /// own custom text or emojis. To disable, set this to "" (an empty string). /// - /// default: 🏳️⚧️ + /// The default is the trans pride flag. + /// + /// example: "🏳️⚧️" + /// + /// default: "🏳️⚧️" #[serde(default = "default_new_user_displayname_suffix")] pub new_user_displayname_suffix: String, /// If enabled, conduwuit will send a simple GET request periodically to /// `https://pupbrain.dev/check-for-updates/stable` for any new /// announcements made. Despite the name, this is not an update check - /// endpoint, it is simply an announcement check endpoint. Defaults to - /// false. + /// endpoint, it is simply an announcement check endpoint. + /// + /// This is disabled by default as this is rarely used except for security + /// updates or major updates. #[serde(default)] pub allow_check_for_updates: bool, - #[serde(default = "default_pdu_cache_capacity")] - pub pdu_cache_capacity: u32, - /// Set this to any float value to multiply conduwuit's in-memory LRU caches - /// with. May be useful if you have significant memory to spare to increase + /// with such as "auth_chain_cache_capacity". + /// + /// May be useful if you have significant memory to spare to increase /// performance. This was previously called /// `conduit_cache_capacity_modifier`. /// - /// default: 1.0. + /// If you have low memory, reducing this may be viable. + /// + /// By default, the individual caches such as "auth_chain_cache_capacity" + /// are scaled by your CPU core count. + /// + /// default: 1.0 #[serde(default = "default_cache_capacity_modifier", alias = "conduit_cache_capacity_modifier")] pub cache_capacity_modifier: f64, + /// default: varies by system + #[serde(default = "default_pdu_cache_capacity")] + pub pdu_cache_capacity: u32, + + /// default: varies by system #[serde(default = "default_auth_chain_cache_capacity")] pub auth_chain_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_shorteventid_cache_capacity")] pub shorteventid_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_eventidshort_cache_capacity")] pub eventidshort_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_shortstatekey_cache_capacity")] pub shortstatekey_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_statekeyshort_cache_capacity")] pub statekeyshort_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_server_visibility_cache_capacity")] pub server_visibility_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_user_visibility_cache_capacity")] pub user_visibility_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_stateinfo_cache_capacity")] pub stateinfo_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_roomid_spacehierarchy_cache_capacity")] pub roomid_spacehierarchy_cache_capacity: u32, /// Maximum entries stored in DNS memory-cache. The size of an entry may /// vary so please take care if raising this value excessively. Only /// decrease this when using an external DNS cache. Please note - /// that systemd does *not* count as an external cache, even when configured - /// to do so. + /// that systemd-resolved does *not* count as an external cache, even when + /// configured to do so. + /// + /// default: 32768 #[serde(default = "default_dns_cache_entries")] pub dns_cache_entries: u32, /// Minimum time-to-live in seconds for entries in the DNS cache. The - /// default may appear high to most administrators; this is by design. Only - /// decrease this if you are using an external DNS cache. + /// default may appear high to most administrators; this is by design as the + /// majority of NXDOMAINs are correct for a long time (e.g. the server is no + /// longer running Matrix). Only decrease this if you are using an external + /// DNS cache. + /// + /// default_dns_min_ttl: 259200 #[serde(default = "default_dns_min_ttl")] pub dns_min_ttl: u64, @@ -195,16 +248,23 @@ pub struct Config { /// and aggressively cached rather than constantly rechecked. /// /// Defaults to 3 days as these are *very rarely* false negatives. + /// + /// default: 259200 #[serde(default = "default_dns_min_ttl_nxdomain")] pub dns_min_ttl_nxdomain: u64, /// Number of retries after a timeout. + /// + /// default: 10 #[serde(default = "default_dns_attempts")] pub dns_attempts: u16, /// The number of seconds to wait for a reply to a DNS query. Please note /// that recursive queries can take up to several seconds for some domains, - /// so this value should not be too low. + /// so this value should not be too low, especially on slower hardware or + /// resolvers. + /// + /// default: 10 #[serde(default = "default_dns_timeout")] pub dns_timeout: u64, @@ -223,8 +283,7 @@ pub struct Config { /// Enables using *only* TCP for querying your specified nameservers instead /// of UDP. /// - /// You very likely do *not* want this. hickory-resolver already falls back - /// to TCP on UDP errors. Defaults to false + /// If you are running conduwuit in a container environment, this config option may need to be enabled. See https://conduwuit.puppyirl.gay/troubleshooting.html#potential-dns-issues-when-using-docker for more details. #[serde(default)] pub query_over_tcp_only: bool, @@ -232,30 +291,34 @@ pub struct Config { /// /// Takes a number of one of the following options: /// 1 - Ipv4Only (Only query for A records, no AAAA/IPv6) + /// /// 2 - Ipv6Only (Only query for AAAA records, no A/IPv4) + /// /// 3 - Ipv4AndIpv6 (Query for A and AAAA records in parallel, uses whatever - /// returns a successful response first) 4 - Ipv6thenIpv4 (Query for AAAA - /// record, if that fails then query the A record) 5 - Ipv4thenIpv6 (Query - /// for A record, if that fails then query the AAAA record) + /// returns a successful response first) /// - /// If you don't have IPv6 networking, then for better performance it may be - /// suitable to set this to Ipv4Only (1) as you will never ever use the - /// AAAA record contents even if the AAAA record is successful instead of - /// the A record. + /// 4 - Ipv6thenIpv4 (Query for AAAA record, if that fails then query the A + /// record) /// - /// Defaults to 5 - Ipv4ThenIpv6 as this is the most compatible and IPv4 - /// networking is currently the most prevalent. + /// 5 - Ipv4thenIpv6 (Query for A record, if that fails then query the AAAA + /// record) + /// + /// If you don't have IPv6 networking, then for better DNS performance it + /// may be suitable to set this to Ipv4Only (1) as you will never ever use + /// the AAAA record contents even if the AAAA record is successful instead + /// of the A record. /// /// default: 5 #[serde(default = "default_ip_lookup_strategy")] pub ip_lookup_strategy: u8, - /// Max request size for file uploads + /// Max request size for file uploads in bytes. Defaults to 20MB. /// /// default: 20971520 #[serde(default = "default_max_request_size")] pub max_request_size: usize, + /// default: 192 #[serde(default = "default_max_fetch_prev_events")] pub max_fetch_prev_events: u16, @@ -365,7 +428,7 @@ pub struct Config { /// Notification gateway pusher idle connection pool timeout /// - /// Defaults to 15 seconds + /// default: 15 #[serde(default = "default_pusher_idle_timeout")] pub pusher_idle_timeout: u64, @@ -373,7 +436,7 @@ pub struct Config { /// server. /// /// If set to true without a token configured, users can register with no - /// form of 2nd- step only if you set + /// form of 2nd-step only if you set /// `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` to /// true in your config. /// @@ -387,21 +450,27 @@ pub struct Config { /// A static registration token that new users will have to provide when /// creating an account. If unset and `allow_registration` is true, - /// registration is open without any condition. YOU NEED TO EDIT THIS. + /// registration is open without any condition. + /// + /// YOU NEED TO EDIT THIS OR USE registration_token_file. + /// + /// example: "o&^uCtes4HPf0Vu@F20jQeeWE7" pub registration_token: Option, - /// Path to a file on the system that gets read for the registration token + /// Path to a file on the system that gets read for the registration token. + /// this config option takes precedence/priority over "registration_token". /// /// conduwuit must be able to access the file, and it must not be empty /// - /// no default + /// example: "/etc/conduwuit/.reg_token" pub registration_token_file: Option, /// Controls whether encrypted rooms and events are allowed. #[serde(default = "true_fn")] pub allow_encryption: bool, - /// Controls whether federation is allowed or not. + /// Controls whether federation is allowed or not. It is not recommended to + /// disable this after the fact due to potential federation breakage. #[serde(default = "true_fn")] pub allow_federation: bool, @@ -433,25 +502,25 @@ pub struct Config { /// allow guests/unauthenticated users to access TURN credentials /// /// this is the equivalent of Synapse's `turn_allow_guests` config option. - /// this allows any unauthenticated user to call + /// this allows any unauthenticated user to call the endpoint /// `/_matrix/client/v3/voip/turnServer`. /// - /// defaults to false + /// It is unlikely you need to enable this as all major clients support + /// authentication for this endpoint and prevents misuse of your TURN server + /// from potential bots. #[serde(default)] pub turn_allow_guests: bool, /// Set this to true to lock down your server's public room directory and /// only allow admins to publish rooms to the room directory. Unpublishing /// is still allowed by all users with this enabled. - /// - /// Defaults to false #[serde(default)] pub lockdown_public_room_directory: bool, /// Set this to true to allow federating device display names / allow /// external users to see your device display name. If federation is /// disabled entirely (`allow_federation`), this is inherently false. For - /// privacy, this is best disabled. + /// privacy reasons, this is best left disabled. #[serde(default)] pub allow_device_name_federation: bool, @@ -464,25 +533,29 @@ pub struct Config { /// try to invite you to a DM or room. Also can protect against profile /// spiders. /// - /// Defaults to true. - /// /// This is inherently false if `allow_federation` is disabled #[serde(default = "true_fn", alias = "allow_profile_lookup_federation_requests")] pub allow_inbound_profile_lookup_federation_requests: bool, - /// controls whether users are allowed to create rooms. - /// appservices and admins are always allowed to create rooms - /// defaults to true + /// controls whether standard users are allowed to create rooms. appservices + /// and admins are always allowed to create rooms #[serde(default = "true_fn")] pub allow_room_creation: bool, /// Set to false to disable users from joining or creating room versions /// that aren't 100% officially supported by conduwuit. - /// conduwuit officially supports room versions 6 - 10. conduwuit has - /// experimental/unstable support for 3 - 5, and 11. Defaults to true. + /// + /// conduwuit officially supports room versions 6 - 11. + /// + /// conduwuit has slightly experimental (though works fine in practice) + /// support for versions 3 - 5 #[serde(default = "true_fn")] pub allow_unstable_room_versions: bool, + /// default room version conduwuit will create rooms with. + /// + /// per spec, room version 10 is the default. + /// /// default: 10 #[serde(default = "default_default_room_version")] pub default_room_version: RoomVersionId, @@ -498,10 +571,12 @@ pub struct Config { #[serde(default = "default_jaeger_filter")] pub jaeger_filter: String, - /// If the 'perf_measurements' feature is enabled, enables collecting folded - /// stack trace profile of tracing spans using tracing_flame. The resulting - /// profile can be visualized with inferno[1], speedscope[2], or a number of - /// other tools. [1]: https://github.com/jonhoo/inferno + /// If the 'perf_measurements' compile-time feature is enabled, enables + /// collecting folded stack trace profile of tracing spans using + /// tracing_flame. The resulting profile can be visualized with inferno[1], + /// speedscope[2], or a number of other tools. + /// + /// [1]: https://github.com/jonhoo/inferno /// [2]: www.speedscope.app #[serde(default)] pub tracing_flame: bool, @@ -546,8 +621,10 @@ pub struct Config { /// Servers listed here will be used to gather public keys of other servers /// (notary trusted key servers). /// - /// (Currently, conduwuit doesn't support batched key requests, so this list - /// should only contain other Synapse servers) Defaults to `matrix.org` + /// Currently, conduwuit doesn't support inbound batched key requests, so + /// this list should only contain other Synapse servers + /// + /// example: ["matrix.org", "constellatory.net", "tchncs.de"] /// /// default: ["matrix.org"] #[serde(default = "default_trusted_servers")] @@ -556,9 +633,10 @@ pub struct Config { /// Whether to query the servers listed in trusted_servers first or query /// the origin server first. For best security, querying the origin server /// first is advised to minimize the exposure to a compromised trusted - /// server. For maximum performance this can be set to true, however other - /// options exist to query trusted servers first under specific high-load - /// circumstances and should be evaluated before setting this to true. + /// server. For maximum federation/join performance this can be set to true, + /// however other options exist to query trusted servers first under + /// specific high-load circumstances and should be evaluated before setting + /// this to true. #[serde(default)] pub query_trusted_key_servers_first: bool, @@ -582,7 +660,7 @@ pub struct Config { #[serde(default)] pub only_query_trusted_key_servers: bool, - /// Maximum number of keys to request in each trusted server query. + /// Maximum number of keys to request in each trusted server batch query. /// /// default: 1024 #[serde(default = "default_trusted_server_batch_size")] @@ -590,6 +668,7 @@ pub struct Config { /// max log level for conduwuit. allows debug, info, warn, or error /// see also: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives + /// /// **Caveat**: /// For release builds, the tracing crate is configured to only implement /// levels higher than error to avoid unnecessary overhead in the compiled @@ -601,8 +680,6 @@ pub struct Config { pub log: String, /// controls whether logs will be outputted with ANSI colours - /// - /// default: true #[serde(default = "true_fn", alias = "log_colours")] pub log_colors: bool, @@ -615,40 +692,43 @@ pub struct Config { /// OpenID token expiration/TTL in seconds /// /// These are the OpenID tokens that are primarily used for Matrix account - /// integrations, *not* OIDC/OpenID Connect/etc + /// integrations (e.g. Vector Integrations in Element), *not* OIDC/OpenID + /// Connect/etc /// /// default: 3600 #[serde(default = "default_openid_token_ttl")] pub openid_token_ttl: u64, - /// TURN username to provide the client - /// - /// no default + /// static TURN username to provide the client if not using a shared secret + /// ("turn_secret"), It is recommended to use a shared secret over static + /// credentials. #[serde(default)] pub turn_username: String, - /// TURN password to provide the client - /// - /// no default + /// static TURN password to provide the client if not using a shared secret + /// ("turn_secret"). It is recommended to use a shared secret over static + /// credentials. #[serde(default)] pub turn_password: String, /// vector list of TURN URIs/servers to use /// /// replace "example.turn.uri" with your TURN domain, such as the coturn - /// "realm". if using TURN over TLS, replace "turn:" with "turns:" + /// "realm" config option. if using TURN over TLS, replace the URI prefix + /// "turn:" with "turns:" /// - /// No default - #[serde(default = "Vec::new")] + /// example: ["turn:example.turn.uri?transport=udp", + /// "turn:example.turn.uri?transport=tcp"] + /// + /// default: [] + #[serde(default)] pub turn_uris: Vec, /// TURN secret to use for generating the HMAC-SHA1 hash apart of username /// and password generation /// /// this is more secure, but if needed you can use traditional - /// username/password below. - /// - /// no default + /// static username/password credentials. #[serde(default)] pub turn_secret: String, @@ -657,7 +737,7 @@ pub struct Config { /// this takes priority over "turn_secret" first, and falls back to /// "turn_secret" if invalid or failed to open. /// - /// no default + /// example: "/etc/conduwuit/.turn_secret" pub turn_secret_file: Option, /// TURN TTL in seconds @@ -670,7 +750,10 @@ pub struct Config { /// registered users join. The rooms specified must be rooms that you /// have joined at least once on the server, and must be public. /// - /// No default. + /// example: ["#conduwuit:puppygock.gay", + /// "!eoIzvAvVwY23LPDay8:puppygock.gay"] + /// + /// default: [] #[serde(default = "Vec::new")] pub auto_join_rooms: Vec, @@ -695,15 +778,18 @@ pub struct Config { /// RocksDB log level. This is not the same as conduwuit's log level. This /// is the log level for the RocksDB engine/library which show up in your - /// database folder/path as `LOG` files. Defaults to error. conduwuit will - /// typically log RocksDB errors as normal. + /// database folder/path as `LOG` files. conduwuit will log RocksDB errors + /// as normal through tracing. + /// + /// default: "error" #[serde(default = "default_rocksdb_log_level")] pub rocksdb_log_level: String, #[serde(default)] pub rocksdb_log_stderr: bool, - /// Max RocksDB `LOG` file size before rotating in bytes. Defaults to 4MB. + /// Max RocksDB `LOG` file size before rotating in bytes. Defaults to 4MB in + /// bytes. /// /// default: 4194304 #[serde(default = "default_rocksdb_max_log_file_size")] @@ -727,13 +813,19 @@ pub struct Config { /// but it cannot account for all setups. If you experience any weird /// RocksDB issues, try enabling this option as it turns off Direct IO and /// feel free to report in the conduwuit Matrix room if this option fixes - /// your DB issues. See https://github.com/facebook/rocksdb/wiki/Direct-IO for more information. + /// your DB issues. + /// + /// See https://github.com/facebook/rocksdb/wiki/Direct-IO for more information. #[serde(default)] pub rocksdb_optimize_for_spinning_disks: bool, - /// Enables direct-io to increase database performance. This is enabled by - /// default. Set this option to false if the database resides on a - /// filesystem which does not support direct-io. + /// Enables direct-io to increase database performance via unbuffered I/O. + /// + /// See https://github.com/facebook/rocksdb/wiki/Direct-IO for more details about Direct IO and RocksDB. + /// + /// Set this option to false if the database resides on a filesystem which + /// does not support direct-io like FUSE, or any form of complex filesystem + /// setup such as possibly ZFS. #[serde(default = "true_fn")] pub rocksdb_direct_io: bool, @@ -746,14 +838,17 @@ pub struct Config { pub rocksdb_parallelism_threads: usize, /// Maximum number of LOG files RocksDB will keep. This must *not* be set to - /// 0. It must be at least 1. Defaults to 3 as these are not very useful. + /// 0. It must be at least 1. Defaults to 3 as these are not very useful + /// unless troubleshooting/debugging a RocksDB bug. /// /// default: 3 #[serde(default = "default_rocksdb_max_log_files")] pub rocksdb_max_log_files: usize, /// Type of RocksDB database compression to use. + /// /// Available options are "zstd", "zlib", "bz2", "lz4", or "none" + /// /// It is best to use ZSTD as an overall good balance between /// speed/performance, storage, IO amplification, and CPU usage. /// For more performance but less compression (more storage used) and less @@ -766,10 +861,14 @@ pub struct Config { pub rocksdb_compression_algo: String, /// Level of compression the specified compression algorithm for RocksDB to - /// use. Default is 32767, which is internally read by RocksDB as the + /// use. + /// + /// Default is 32767, which is internally read by RocksDB as the /// default magic number and translated to the library's default /// compression level as they all differ. /// See their `kDefaultCompressionLevel`. + /// + /// default: 32767 #[serde(default = "default_rocksdb_compression_level")] pub rocksdb_compression_level: i32, @@ -783,15 +882,19 @@ pub struct Config { /// it may be desirable to have a very high compression level here as it's /// lesss likely for this data to be used. Research your chosen compression /// algorithm. + /// + /// default: 32767 #[serde(default = "default_rocksdb_bottommost_compression_level")] pub rocksdb_bottommost_compression_level: i32, - /// Whether to enable RocksDB "bottommost_compression". + /// Whether to enable RocksDB's "bottommost_compression". + /// /// At the expense of more CPU usage, this will further compress the /// database to reduce more storage. It is recommended to use ZSTD - /// compression with this for best compression results. See https://github.com/facebook/rocksdb/wiki/Compression for more details. + /// compression with this for best compression results. This may be useful + /// if you're trying to reduce storage usage from the database. /// - /// Defaults to false as this uses more CPU when compressing. + /// See https://github.com/facebook/rocksdb/wiki/Compression for more details. #[serde(default)] pub rocksdb_bottommost_compression: bool, @@ -822,9 +925,9 @@ pub struct Config { /// 2 = PointInTime (use me if trying to recover) /// 3 = SkipAnyCorruptedRecord (you now voided your Conduwuit warranty) /// - /// See https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes for more information + /// See https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes for more information on these modes. /// - /// Defaults to 1 (TolerateCorruptedTailRecords) + /// See https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption for more details on recovering a corrupt database. /// /// default: 1 #[serde(default = "default_rocksdb_recovery_mode")] @@ -841,6 +944,8 @@ pub struct Config { /// repair. /// - Disabling repair mode and restarting the server is recommended after /// running the repair. + /// + /// See https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption for more details on recovering a corrupt database. #[serde(default)] pub rocksdb_repair: bool, @@ -862,6 +967,14 @@ pub struct Config { #[serde(default = "true_fn")] pub rocksdb_compaction_ioprio_idle: bool, + /// Config option to disable RocksDB compaction. You should never ever have + /// to disable this. If you for some reason find yourself needing to disable + /// this as part of troubleshooting or a bug, please reach out to us in the + /// conduwuit Matrix room with information and details. + /// + /// Disabling compaction will lead to a significantly bloated and + /// explosively large database, gradually poor performance, unnecessarily + /// excessive disk read/writes, and slower shutdowns and startups. #[serde(default = "true_fn")] pub rocksdb_compaction: bool, @@ -876,33 +989,45 @@ pub struct Config { /// 3 to 5 = Statistics with possible performance impact. /// 6 = All statistics. /// - /// Defaults to 1 (No statistics, except in debug-mode) - /// /// default: 1 #[serde(default = "default_rocksdb_stats_level")] pub rocksdb_stats_level: u8, + /// This is a password that can be configured that will let you login to the + /// server bot account (currently `@conduit`) for emergency troubleshooting + /// purposes such as recovering/recreating your admin room, or inviting + /// yourself back. + /// + /// See https://conduwuit.puppyirl.gay/troubleshooting.html#lost-access-to-admin-room for other ways to get back into your admin room. + /// + /// Once this password is unset, all sessions will be logged out for + /// security purposes. + /// + /// example: "F670$2CP@Hw8mG7RY1$%!#Ic7YA" pub emergency_password: Option, + /// default: "/_matrix/push/v1/notify" #[serde(default = "default_notification_push_path")] pub notification_push_path: String, /// Config option to control local (your server only) presence - /// updates/requests. Defaults to true. Note that presence on conduwuit is + /// updates/requests. Note that presence on conduwuit is /// very fast unlike Synapse's. If using outgoing presence, this MUST be /// enabled. #[serde(default = "true_fn")] pub allow_local_presence: bool, /// Config option to control incoming federated presence updates/requests. - /// Defaults to true. This option receives presence updates from other + /// + /// This option receives presence updates from other /// servers, but does not send any unless `allow_outgoing_presence` is true. /// Note that presence on conduwuit is very fast unlike Synapse's. #[serde(default = "true_fn")] pub allow_incoming_presence: bool, - /// Config option to control outgoing presence updates/requests. Defaults to - /// true. This option sends presence updates to other servers, but does not + /// Config option to control outgoing presence updates/requests. + /// + /// This option sends presence updates to other servers, but does not /// receive any unless `allow_incoming_presence` is true. /// Note that presence on conduwuit is very fast unlike Synapse's. /// If using outgoing presence, you MUST enable `allow_local_presence` as @@ -986,6 +1111,9 @@ pub struct Config { /// compression may weaken TLS. Most users should not need to enable this. /// See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before /// deciding to enable this. + /// + /// If you are in a large amount of rooms, you may find that enabling this + /// is necessary to reduce the significantly large response bodies. #[serde(default)] pub gzip_compression: bool, @@ -998,18 +1126,18 @@ pub struct Config { #[serde(default)] pub brotli_compression: bool, - /// Set to true to allow user type "guest" registrations. Element attempts - /// to register guest users automatically. Defaults to false. + /// Set to true to allow user type "guest" registrations. Some clients like + /// Element attempt to register guest users automatically. #[serde(default)] pub allow_guest_registration: bool, - /// Set to true to log guest registrations in the admin room. - /// Defaults to false as it may be noisy or unnecessary. + /// Set to true to log guest registrations in the admin room. Note that + /// these may be noisy or unnecessary if you're a public homeserver. #[serde(default)] pub log_guest_registrations: bool, /// Set to true to allow guest registrations/users to auto join any rooms - /// specified in `auto_join_rooms` Defaults to false. + /// specified in `auto_join_rooms`. #[serde(default)] pub allow_guests_auto_join_rooms: bool, @@ -1033,7 +1161,7 @@ pub struct Config { /// Checks consistency of the media directory at startup: /// 1. When `media_compat_file_link` is enbled, this check will upgrade - /// media when switching back and forth between Conduit and Conduwuit. + /// media when switching back and forth between Conduit and conduwuit. /// Both options must be enabled to handle this. /// 2. When media is deleted from the directory, this check will also delete /// its database entry. @@ -1041,8 +1169,6 @@ pub struct Config { /// If none of these checks apply to your use cases, and your media /// directory is significantly large setting this to false may reduce /// startup time. - /// - /// Enabled by default. #[serde(default = "true_fn")] pub media_startup_check: bool, @@ -1051,9 +1177,9 @@ pub struct Config { /// Conduit again. Otherwise setting this to false reduces filesystem /// clutter and overhead for managing these symlinks in the directory. This /// is now disabled by default. You may still return to upstream Conduit - /// but you have to run Conduwuit at least once with this set to true and + /// but you have to run conduwuit at least once with this set to true and /// allow the media_startup_check to take place before shutting - /// down to return to Conduit. Disabled by default. + /// down to return to Conduit. #[serde(default)] pub media_compat_file_link: bool, @@ -1061,14 +1187,16 @@ pub struct Config { /// checks. This means if you delete files from the media directory the /// corresponding entries will be removed from the database. This is /// disabled by default because if the media directory is accidentally moved - /// or inaccessible the metadata entries in the database will be lost with - /// sadness. Disabled by default. + /// or inaccessible, the metadata entries in the database will be lost with + /// sadness. #[serde(default)] pub prune_missing_media: bool, /// Vector list of servers that conduwuit will refuse to download remote - /// media from. No default. - #[serde(default = "HashSet::new")] + /// media from. + /// + /// default: [] + #[serde(default)] pub prevent_media_downloads_from: HashSet, /// List of forbidden server names that we will block incoming AND outgoing @@ -1078,13 +1206,17 @@ pub struct Config { /// sender user's server name, inbound federation X-Matrix origin, and /// outbound federation handler. /// - /// Basically "global" ACLs. No default. - #[serde(default = "HashSet::new")] + /// Basically "global" ACLs. + /// + /// default: [] + #[serde(default)] pub forbidden_remote_server_names: HashSet, /// List of forbidden server names that we will block all outgoing federated /// room directory requests for. Useful for preventing our users from - /// wandering into bad servers or spaces. No default. + /// wandering into bad servers or spaces. + /// + /// default: [] #[serde(default = "HashSet::new")] pub forbidden_remote_room_directory_server_names: HashSet, @@ -1100,28 +1232,12 @@ pub struct Config { /// Currently this does not account for proxies in use like Synapse does. /// /// To disable, set this to be an empty vector (`[]`). - /// The default is: - /// [ - /// "127.0.0.0/8", - /// "10.0.0.0/8", - /// "172.16.0.0/12", - /// "192.168.0.0/16", - /// "100.64.0.0/10", - /// "192.0.0.0/24", - /// "169.254.0.0/16", - /// "192.88.99.0/24", - /// "198.18.0.0/15", - /// "192.0.2.0/24", - /// "198.51.100.0/24", - /// "203.0.113.0/24", - /// "224.0.0.0/4", - /// "::1/128", - /// "fe80::/10", - /// "fc00::/7", - /// "2001:db8::/32", - /// "ff00::/8", - /// "fec0::/10", - /// ] + /// + /// default: ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", + /// "192.168.0.0/16", "100.64.0.0/10", "192.0.0.0/24", "169.254.0.0/16", + /// "192.88.99.0/24", "198.18.0.0/15", "192.0.2.0/24", "198.51.100.0/24", + /// "203.0.113.0/24", "224.0.0.0/4", "::1/128", "fe80::/10", "fc00::/7", + /// "2001:db8::/32", "ff00::/8", "fec0::/10"] #[serde(default = "default_ip_range_denylist")] pub ip_range_denylist: Vec, @@ -1132,7 +1248,9 @@ pub struct Config { /// allow all URL previews. Please note that this opens up significant /// attack surface to your server, you are expected to be aware of the /// risks by doing so. - #[serde(default = "Vec::new")] + /// + /// default: [] + #[serde(default)] pub url_preview_domain_contains_allowlist: Vec, /// Vector list of explicit domains allowed to send requests to for URL @@ -1143,7 +1261,9 @@ pub struct Config { /// allow all URL previews. Please note that this opens up significant /// attack surface to your server, you are expected to be aware of the /// risks by doing so. - #[serde(default = "Vec::new")] + /// + /// default: [] + #[serde(default)] pub url_preview_domain_explicit_allowlist: Vec, /// Vector list of explicit domains not allowed to send requests to for URL @@ -1152,7 +1272,9 @@ pub struct Config { /// "http://google.com", but not /// "https://mymaliciousdomainexamplegoogle.com". The denylist is checked /// first before allowlist. Setting this to "*" will not do anything. - #[serde(default = "Vec::new")] + /// + /// default: [] + #[serde(default)] pub url_preview_domain_explicit_denylist: Vec, /// Vector list of URLs allowed to send requests to for URL previews. @@ -1164,19 +1286,23 @@ pub struct Config { /// "*" will allow all URL previews. Please note that this opens up /// significant attack surface to your server, you are expected to be /// aware of the risks by doing so. - #[serde(default = "Vec::new")] + /// + /// default: [] + #[serde(default)] pub url_preview_url_contains_allowlist: Vec, /// Maximum amount of bytes allowed in a URL preview body size when - /// spidering. Defaults to 384KB. + /// spidering. Defaults to 384KB in bytes. /// - /// defaukt: 384000 + /// default: 384000 #[serde(default = "default_url_preview_max_spider_size")] pub url_preview_max_spider_size: usize, /// Option to decide whether you would like to run the domain allowlist /// checks (contains and explicit) on the root domain or not. Does not apply - /// to URL contains allowlist. Defaults to false. Example: If this is + /// to URL contains allowlist. Defaults to false. + /// + /// Example usecase: If this is /// enabled and you have "wikipedia.org" allowed in the explicit and/or /// contains domain allowlist, it will allow all subdomains under /// "wikipedia.org" such as "en.m.wikipedia.org" as the root domain is @@ -1186,21 +1312,36 @@ pub struct Config { #[serde(default)] pub url_preview_check_root_domain: bool, - /// List of forbidden room aliases and room IDs as patterns/strings. Values - /// in this list are matched as *contains*. This is checked upon room alias - /// creation, custom room ID creation if used, and startup as warnings if - /// any room aliases in your database have a forbidden room alias/ID. - /// No default. - #[serde(default = "RegexSet::empty")] + /// List of forbidden room aliases and room IDs as strings of regex + /// patterns. + /// + /// Regex can be used or explicit contains matches can be done by + /// just specifying the words (see example). + /// + /// This is checked upon room alias creation, custom room ID creation if + /// used, and startup as warnings if any room aliases in your database have + /// a forbidden room alias/ID. + /// + /// example: ["19dollarfortnitecards", "b[4a]droom"] + /// + /// default: [] + #[serde(default)] #[serde(with = "serde_regex")] pub forbidden_alias_names: RegexSet, - /// List of forbidden username patterns/strings. Values in this list are - /// matched as *contains*. This is checked upon username availability - /// check, registration, and startup as warnings if any local users in your - /// database have a forbidden username. - /// No default. - #[serde(default = "RegexSet::empty")] + /// List of forbidden username patterns/strings. + /// + /// Regex can be used or explicit contains matches can be done by just + /// specifying the words (see example). + /// + /// This is checked upon username availability check, registration, and + /// startup as warnings if any local users in your database have a forbidden + /// username. + /// + /// example: ["administrator", "b[a4]dusernam[3e]"] + /// + /// default: [] + #[serde(default)] #[serde(with = "serde_regex")] pub forbidden_usernames: RegexSet, @@ -1231,9 +1372,12 @@ pub struct Config { #[serde(default)] pub block_non_admin_invites: bool, - /// Allows admins to enter commands in rooms other than #admins by prefixing - /// with \!admin. The reply will be publicly visible to the room, - /// originating from the sender. + /// Allows admins to enter commands in rooms other than "#admins" (admin + /// room) by prefixing your message with "\!admin" or "\\!admin" followed + /// up a normal conduwuit admin command. The reply will be publicly visible + /// to the room, originating from the sender. + /// + /// example: \\!admin debug ping puppygock.gay #[serde(default = "true_fn")] pub admin_escape_commands: bool, @@ -1246,8 +1390,6 @@ pub struct Config { /// Controls what admin commands will be executed on startup. This is a /// vector list of strings of admin commands to run. /// - /// An example of this can be: `admin_execute = ["debug ping puppygock.gay", - /// "debug echo hi"]` /// /// This option can also be configured with the `--execute` conduwuit /// argument and can take standard shell commands and environment variables @@ -1255,6 +1397,8 @@ pub struct Config { /// Such example could be: `./conduwuit --execute "server admin-notice /// conduwuit has started up at $(date)"` /// + /// example: admin_execute = ["debug ping puppygock.gay", "debug echo hi"]` + /// /// default: [] #[serde(default)] pub admin_execute: Vec, @@ -1272,6 +1416,14 @@ pub struct Config { #[serde(default = "default_admin_log_capture")] pub admin_log_capture: String, + /// The default room tag to apply on the admin room. + /// + /// On some clients like Element, the room tag "m.server_notice" is a + /// special pinned room at the very bottom of your room list. The conduwuit + /// admin room can be pinned here so you always have an easy-to-access + /// shortcut dedicated to your admin room. + /// + /// default: "m.server_notice" #[serde(default = "default_admin_room_tag")] pub admin_room_tag: String, @@ -1283,12 +1435,11 @@ pub struct Config { /// Sentry reporting URL if a custom one is desired /// - /// Defaults to conduwuit's default Sentry endpoint: - /// "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" + /// default: "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" #[serde(default = "default_sentry_endpoint")] pub sentry_endpoint: Option, - /// Report your Conduwuit server_name in Sentry.io crash reports and metrics + /// Report your conduwuit server_name in Sentry.io crash reports and metrics #[serde(default)] pub sentry_send_server_name: bool, @@ -1307,7 +1458,8 @@ pub struct Config { pub sentry_attach_stacktrace: bool, /// Send panics to sentry. This is true by default, but sentry has to be - /// enabled. + /// enabled. The global "sentry" config option must be enabled to send any + /// data. #[serde(default = "true_fn")] pub sentry_send_panic: bool, @@ -1318,13 +1470,14 @@ pub struct Config { pub sentry_send_error: bool, /// Controls the tracing log level for Sentry to send things like - /// breadcrumbs and transactions Defaults to "info" + /// breadcrumbs and transactions + /// + /// default: "info" #[serde(default = "default_sentry_filter")] pub sentry_filter: String, /// Enable the tokio-console. This option is only relevant to developers. - /// See: docs/development.md#debugging-with-tokio-console for more - /// information. + /// See https://conduwuit.puppyirl.gay/development.html#debugging-with-tokio-console for more information. #[serde(default)] pub tokio_console: bool, @@ -1346,18 +1499,33 @@ pub struct Config { #[derive(Clone, Debug, Deserialize)] #[config_example_generator(filename = "conduwuit-example.toml", section = "global.tls")] pub struct TlsConfig { + /// Path to a valid TLS certificate file. + /// + /// example: "/path/to/my/certificate.crt" pub certs: String, + /// Path to a valid TLS certificate private key. + /// + /// example: "/path/to/my/certificate.key" pub key: String, - #[serde(default)] /// Whether to listen and allow for HTTP and HTTPS connections (insecure!) + #[serde(default)] pub dual_protocol: bool, } #[derive(Clone, Debug, Deserialize, Default)] #[config_example_generator(filename = "conduwuit-example.toml", section = "global.well_known")] pub struct WellKnownConfig { - pub client: Option, + /// The server base domain of the URL with a specific port that the server + /// well-known file will serve. This should contain a port at the end, and + /// should not be a URL. + /// + /// example: "matrix.example.com:443" pub server: Option, + /// The server URL that the client well-known file will serve. This should + /// not contain a port, and should just be a valid HTTPS URL. + /// + /// example: "https://matrix.example.com" + pub client: Option, pub support_page: Option, pub support_role: Option, pub support_email: Option, @@ -1460,7 +1628,6 @@ impl fmt::Display for Config { }; line("Server name", self.server_name.host()); - line("Database backend", &self.database_backend); line("Database path", &self.database_path.to_string_lossy()); line( "Database backup path", @@ -1861,8 +2028,6 @@ fn default_unix_socket_perms() -> u32 { 660 } fn default_database_backups_to_keep() -> i16 { 1 } -fn default_database_backend() -> String { "rocksdb".to_owned() } - fn default_db_cache_capacity_mb() -> f64 { 128.0 + parallelism_scaled_f64(64.0) } fn default_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } diff --git a/src/service/migrations.rs b/src/service/migrations.rs index d6c342f8..4c821fa3 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -59,7 +59,6 @@ pub(crate) async fn migrations(services: &Services) -> Result<()> { async fn fresh(services: &Services) -> Result<()> { let db = &services.db; - let config = &services.server.config; services .globals @@ -73,10 +72,7 @@ async fn fresh(services: &Services) -> Result<()> { // Create the admin room and server user on first run crate::admin::create_admin_room(services).boxed().await?; - warn!( - "Created new {} database with version {DATABASE_VERSION}", - config.database_backend, - ); + warn!("Created new RocksDB database with version {DATABASE_VERSION}"); Ok(()) } @@ -201,10 +197,7 @@ async fn migrate(services: &Services) -> Result<()> { } } - info!( - "Loaded {} database with schema version {DATABASE_VERSION}", - config.database_backend, - ); + info!("Loaded RocksDB database with schema version {DATABASE_VERSION}"); Ok(()) } From 4fe47903c24f3f9a0f33e871caecd96e7294dc49 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 10 Nov 2024 20:20:27 -0500 Subject: [PATCH 1046/2091] misc docs changes/improvements from example config Signed-off-by: strawberry --- docs/deploying/docker-compose.for-traefik.yml | 1 - docs/deploying/docker-compose.with-caddy.yml | 1 - docs/deploying/docker-compose.yml | 1 - docs/deploying/docker.md | 1 - docs/deploying/generic.md | 14 +++++++++++++- docs/troubleshooting.md | 9 +++++---- 6 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index ae93d52f..b4316426 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -14,7 +14,6 @@ services: environment: CONDUWUIT_SERVER_NAME: your.server.name.example # EDIT THIS CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit - CONDUWUIT_DATABASE_BACKEND: rocksdb CONDUWUIT_PORT: 6167 # should match the loadbalancer traefik label CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB CONDUWUIT_ALLOW_REGISTRATION: 'true' diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index 36924212..c080293f 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -30,7 +30,6 @@ services: environment: CONDUWUIT_SERVER_NAME: example.com # EDIT THIS CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit - CONDUWUIT_DATABASE_BACKEND: rocksdb CONDUWUIT_PORT: 6167 CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB CONDUWUIT_ALLOW_REGISTRATION: 'true' diff --git a/docs/deploying/docker-compose.yml b/docs/deploying/docker-compose.yml index 26145c5a..3b7d84ed 100644 --- a/docs/deploying/docker-compose.yml +++ b/docs/deploying/docker-compose.yml @@ -14,7 +14,6 @@ services: environment: CONDUWUIT_SERVER_NAME: your.server.name # EDIT THIS CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit - CONDUWUIT_DATABASE_BACKEND: rocksdb CONDUWUIT_PORT: 6167 CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB CONDUWUIT_ALLOW_REGISTRATION: 'true' diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index 7b8fd1a2..e9c49c71 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -40,7 +40,6 @@ When you have the image you can simply run it with docker run -d -p 8448:6167 \ -v db:/var/lib/conduwuit/ \ -e CONDUWUIT_SERVER_NAME="your.server.name" \ - -e CONDUWUIT_DATABASE_BACKEND="rocksdb" \ -e CONDUWUIT_ALLOW_REGISTRATION=false \ --name conduit $LINK ``` diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 31dc1845..6fe9709b 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -42,6 +42,9 @@ replace the binary / container image / etc. this will **NOT** work on conduwuit and you must configure delegation manually. This is not a mistake and no support for this feature will be added. +If you are using SQLite, you **MUST** migrate to RocksDB. You can use this +tool to migrate from SQLite to RocksDB: + See the `[global.well_known]` config section, or configure your web server appropriately to send the delegation responses. @@ -137,11 +140,20 @@ You will need to reverse proxy everything under following routes: You can optionally reverse proxy the following individual routes: - `/.well-known/matrix/client` and `/.well-known/matrix/server` if using -conduwuit to perform delegation +conduwuit to perform delegation (see the `[global.well_known]` config section) - `/.well-known/matrix/support` if using conduwuit to send the homeserver admin contact and support page (formerly known as MSC1929) - `/` if you would like to see `hewwo from conduwuit woof!` at the root +See the following spec pages for more details on these files: +- [`/.well-known/matrix/server`](https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixserver) +- [`/.well-known/matrix/client`](https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixclient) +- [`/.well-known/matrix/support`](https://spec.matrix.org/latest/client-server-api/#getwell-knownmatrixsupport) + +Examples of delegation: +- +- + ### Caddy Create `/etc/caddy/conf.d/conduwuit_caddyfile` and enter this (substitute for diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c1499f3a..74e19de7 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -47,10 +47,11 @@ and communicate with your host's DNS servers (host's `/etc/resolv.conf`) Some filesystems may not like RocksDB using [Direct IO](https://github.com/facebook/rocksdb/wiki/Direct-IO). Direct IO is for -non-buffered I/O which improves conduwuit performance, but at least FUSE is a -filesystem potentially known to not like this. See the [example -config](configuration/examples.md) for disabling it if needed. Issues from -Direct IO on unsupported filesystems are usually shown as startup errors. +non-buffered I/O which improves conduwuit performance and reduces system CPU +usage, but at least FUSE and possibly ZFS are filesystems potentially known +to not like this. See the [example config](configuration/examples.md) for +disabling it if needed. Issues from Direct IO on unsupported filesystems are +usually shown as startup errors. #### Database corruption From 4296d7174f97e380f4d8e28e0ebf7d89c26c9c4a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 2 Nov 2024 21:25:13 -0400 Subject: [PATCH 1047/2091] add receive_ephemeral check for appservice EDU sending (if it even works) Signed-off-by: strawberry --- src/service/sending/sender.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 464d186b..f4268293 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -524,8 +524,13 @@ impl Service { } }, SendingEvent::Edu(edu) => { - if let Ok(edu) = serde_json::from_slice(edu) { - edu_jsons.push(edu); + if appservice + .receive_ephemeral + .is_some_and(|receive_edus| receive_edus) + { + if let Ok(edu) = serde_json::from_slice(edu) { + edu_jsons.push(edu); + } } }, SendingEvent::Flush => {}, // flush only; no new content From fd2a0024809d5029fc679ff4008f97794fba4075 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 10 Nov 2024 20:30:34 -0500 Subject: [PATCH 1048/2091] dont build sentry or perf_measurements features for complement Signed-off-by: strawberry --- nix/pkgs/complement/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index 399c4449..36f12400 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -25,6 +25,7 @@ let "tokio_console" # sentry telemetry isn't useful for complement, disabled by default anyways "sentry_telemetry" + "perf_measurements" # the containers don't use or need systemd signal support "systemd" # this is non-functional on nix for some reason From 4f0bdb5194b8b496618fb8744a128f38b14a02b5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 10 Nov 2024 21:20:38 -0500 Subject: [PATCH 1049/2091] general misc bug fixes and slight improvements Signed-off-by: strawberry --- src/api/client/membership.rs | 21 +++++----- src/api/router/auth.rs | 48 +++++++++++++---------- src/api/server/invite.rs | 26 ++++++++++++- src/api/server/make_join.rs | 18 ++++----- src/api/server/send_leave.rs | 4 +- src/service/rooms/spaces/mod.rs | 51 +++++++++++-------------- src/service/rooms/state/mod.rs | 10 +++-- src/service/rooms/state_accessor/mod.rs | 41 +++++++++++--------- src/service/sending/send.rs | 2 +- src/service/sending/sender.rs | 6 ++- 10 files changed, 127 insertions(+), 100 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index bde8dee8..10e69f58 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1240,8 +1240,8 @@ async fn make_join_request( ) -> Result<(federation::membership::prepare_join_event::v1::Response, OwnedServerName)> { let mut make_join_response_and_server = Err!(BadServerResponse("No server available to assist in joining.")); - let mut make_join_counter: u16 = 0; - let mut incompatible_room_version_count: u8 = 0; + let mut make_join_counter: usize = 0; + let mut incompatible_room_version_count: usize = 0; for remote_server in servers { if services.globals.server_is_ours(remote_server) { @@ -1264,28 +1264,25 @@ async fn make_join_request( make_join_counter = make_join_counter.saturating_add(1); if let Err(ref e) = make_join_response { - trace!("make_join ErrorKind string: {:?}", e.kind().to_string()); - - // converting to a string is necessary (i think) because ruma is forcing us to - // fill in the struct for M_INCOMPATIBLE_ROOM_VERSION - if e.kind().to_string().contains("M_INCOMPATIBLE_ROOM_VERSION") - || e.kind().to_string().contains("M_UNSUPPORTED_ROOM_VERSION") - { + if matches!( + e.kind(), + ErrorKind::IncompatibleRoomVersion { .. } | ErrorKind::UnsupportedRoomVersion + ) { incompatible_room_version_count = incompatible_room_version_count.saturating_add(1); } if incompatible_room_version_count > 15 { info!( "15 servers have responded with M_INCOMPATIBLE_ROOM_VERSION or M_UNSUPPORTED_ROOM_VERSION, \ - assuming that Conduwuit does not support the room {room_id}: {e}" + assuming that conduwuit does not support the room {room_id}: {e}" ); make_join_response_and_server = Err!(BadServerResponse("Room version is not supported by Conduwuit")); return make_join_response_and_server; } - if make_join_counter > 50 { + if make_join_counter > 40 { warn!( - "50 servers failed to provide valid make_join response, assuming no server can assist in joining." + "40 servers failed to provide valid make_join response, assuming no server can assist in joining." ); make_join_response_and_server = Err!(BadServerResponse("No server available to assist in joining.")); return make_join_response_and_server; diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 2552dded..68abf5e2 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -13,6 +13,7 @@ use ruma::{ profile::{get_avatar_url, get_display_name, get_profile, get_profile_key, get_timezone_key}, voip::get_turn_server_info, }, + federation::openid::get_openid_userinfo, AuthScheme, IncomingRequest, Metadata, }, server_util::authorization::XMatrix, @@ -102,26 +103,6 @@ pub(super) async fn auth( } match (metadata.authentication, token) { - (_, Token::Invalid) => { - // OpenID endpoint uses a query param with the same name, drop this once query - // params for user auth are removed from the spec. This is required to make - // integration manager work. - if request.query.access_token.is_some() && request.parts.uri.path().contains("/openid/") { - Ok(Auth { - origin: None, - sender_user: None, - sender_device: None, - appservice_info: None, - }) - } else { - Err(Error::BadRequest( - ErrorKind::UnknownToken { - soft_logout: false, - }, - "Unknown access token.", - )) - } - }, (AuthScheme::AccessToken, Token::Appservice(info)) => Ok(auth_appservice(services, request, info).await?), (AuthScheme::None | AuthScheme::AccessTokenOptional | AuthScheme::AppserviceToken, Token::Appservice(info)) => { Ok(Auth { @@ -132,7 +113,6 @@ pub(super) async fn auth( }) }, (AuthScheme::AccessToken, Token::None) => match metadata { - // TODO: can we check this better? &get_turn_server_info::v3::Request::METADATA => { if services.globals.config.turn_allow_guests { Ok(Auth { @@ -171,6 +151,32 @@ pub(super) async fn auth( ErrorKind::Unauthorized, "Only appservice access tokens should be used on this endpoint.", )), + (AuthScheme::None, Token::Invalid) => { + // OpenID federation endpoint uses a query param with the same name, drop this + // once query params for user auth are removed from the spec. This is + // required to make integration manager work. + if request.query.access_token.is_some() && metadata == &get_openid_userinfo::v1::Request::METADATA { + Ok(Auth { + origin: None, + sender_user: None, + sender_device: None, + appservice_info: None, + }) + } else { + Err(Error::BadRequest( + ErrorKind::UnknownToken { + soft_logout: false, + }, + "Unknown access token.", + )) + } + }, + (_, Token::Invalid) => Err(Error::BadRequest( + ErrorKind::UnknownToken { + soft_logout: false, + }, + "Unknown access token.", + )), } } diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index b30a1b58..edf80cd6 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -1,5 +1,6 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; +use base64::{engine::general_purpose, Engine as _}; use conduit::{err, utils, warn, Err, Error, PduEvent, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_invite}, @@ -125,8 +126,10 @@ pub(crate) async fn create_invite_route( invite_state.push(pdu.to_stripped_state_event()); - // If we are active in the room, the remote server will notify us about the join - // via /send + // If we are active in the room, the remote server will notify us about the + // join/invite through /send. If we are not in the room, we need to manually + // record the invited state for client /sync through update_membership(), and + // send the invite PDU to the relevant appservices. if !services .rooms .state_cache @@ -148,6 +151,25 @@ pub(crate) async fn create_invite_route( .await?; } + for appservice in services.appservice.read().await.values() { + if appservice.is_user_match(&invited_user) { + services + .sending + .send_appservice_request( + appservice.registration.clone(), + ruma::api::appservice::event::push_events::v1::Request { + events: vec![pdu.to_room_event()], + txn_id: general_purpose::URL_SAFE_NO_PAD + .encode(utils::calculate_hash(&[pdu.event_id.as_bytes()])) + .into(), + ephemeral: Vec::new(), + to_device: Vec::new(), + }, + ) + .await?; + } + } + Ok(create_invite::v2::Response { event: services .sending diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index af570064..d5ea675e 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -80,6 +80,14 @@ pub(crate) async fn create_join_event_template_route( } let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; + if !body.ver.contains(&room_version_id) { + return Err(Error::BadRequest( + ErrorKind::IncompatibleRoomVersion { + room_version: room_version_id, + }, + "Room version not supported.", + )); + } let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; @@ -118,16 +126,6 @@ pub(crate) async fn create_join_event_template_route( None }; - let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; - if !body.ver.contains(&room_version_id) { - return Err(Error::BadRequest( - ErrorKind::IncompatibleRoomVersion { - room_version: room_version_id, - }, - "Room version not supported.", - )); - } - let (_pdu, mut pdu_json) = services .rooms .timeline diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index 448e5de3..e4f41833 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -157,7 +157,5 @@ async fn create_leave_event( .room_servers(room_id) .ready_filter(|server| !services.globals.server_is_ours(server)); - services.sending.send_pdu_servers(servers, &pdu_id).await?; - - Ok(()) + services.sending.send_pdu_servers(servers, &pdu_id).await } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 37272dca..0ef7ddf5 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -8,7 +8,7 @@ use std::{ }; use conduit::{ - checked, debug, debug_info, err, + checked, debug_info, err, utils::{math::usize_from_f64, IterStream}, Error, Result, }; @@ -234,27 +234,25 @@ impl Service { }); } - Ok( - if let Some(children_pdus) = self.get_stripped_space_child_events(current_room).await? { - let summary = self - .get_room_summary(current_room, children_pdus, &identifier) - .await; - if let Ok(summary) = summary { - self.roomid_spacehierarchy_cache.lock().await.insert( - current_room.clone(), - Some(CachedSpaceHierarchySummary { - summary: summary.clone(), - }), - ); + if let Some(children_pdus) = self.get_stripped_space_child_events(current_room).await? { + let summary = self + .get_room_summary(current_room, children_pdus, &identifier) + .await; + if let Ok(summary) = summary { + self.roomid_spacehierarchy_cache.lock().await.insert( + current_room.clone(), + Some(CachedSpaceHierarchySummary { + summary: summary.clone(), + }), + ); - Some(SummaryAccessibility::Accessible(Box::new(summary))) - } else { - None - } + Ok(Some(SummaryAccessibility::Accessible(Box::new(summary)))) } else { - None - }, - ) + Ok(None) + } + } else { + Ok(None) + } } /// Gets the summary of a space using solely federation @@ -393,7 +391,7 @@ impl Service { .is_accessible_child(current_room, &join_rule.clone().into(), identifier, &allowed_room_ids) .await { - debug!("User is not allowed to see room {room_id}"); + debug_info!("User is not allowed to see room {room_id}"); // This error will be caught later return Err(Error::BadRequest(ErrorKind::forbidden(), "User is not allowed to see the room")); } @@ -615,16 +613,13 @@ impl Service { &self, current_room: &OwnedRoomId, join_rule: &SpaceRoomJoinRule, identifier: &Identifier<'_>, allowed_room_ids: &Vec, ) -> bool { - // Note: unwrap_or_default for bool means false match identifier { Identifier::ServerName(server_name) => { - let room_id: &RoomId = current_room; - // Checks if ACLs allow for the server to participate if self .services .event_handler - .acl_check(server_name, room_id) + .acl_check(server_name, current_room) .await .is_err() { @@ -645,8 +640,9 @@ impl Service { return true; } }, - } // Takes care of join rules - match join_rule { + } + match &join_rule { + SpaceRoomJoinRule::Public | SpaceRoomJoinRule::Knock | SpaceRoomJoinRule::KnockRestricted => true, SpaceRoomJoinRule::Restricted => { for room in allowed_room_ids { match identifier { @@ -664,7 +660,6 @@ impl Service { } false }, - SpaceRoomJoinRule::Public | SpaceRoomJoinRule::Knock | SpaceRoomJoinRule::KnockRestricted => true, // Invite only, Private, or Custom join rule _ => false, } diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 71a3900c..7d8200f0 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -295,20 +295,22 @@ impl Service { } #[tracing::instrument(skip_all, level = "debug")] - pub async fn summary_stripped(&self, invite: &PduEvent) -> Vec> { + pub async fn summary_stripped(&self, event: &PduEvent) -> Vec> { let cells = [ (&StateEventType::RoomCreate, ""), (&StateEventType::RoomJoinRules, ""), (&StateEventType::RoomCanonicalAlias, ""), (&StateEventType::RoomName, ""), (&StateEventType::RoomAvatar, ""), - (&StateEventType::RoomMember, invite.sender.as_str()), // Add recommended events + (&StateEventType::RoomMember, event.sender.as_str()), // Add recommended events + (&StateEventType::RoomEncryption, ""), + (&StateEventType::RoomTopic, ""), ]; let fetches = cells.iter().map(|(event_type, state_key)| { self.services .state_accessor - .room_state_get(&invite.room_id, event_type, state_key) + .room_state_get(&event.room_id, event_type, state_key) }); join_all(fetches) @@ -316,7 +318,7 @@ impl Service { .into_iter() .filter_map(Result::ok) .map(|e| e.to_stripped_state_event()) - .chain(once(invite.to_stripped_state_event())) + .chain(once(event.to_stripped_state_event())) .collect() } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index d51da8af..4958c4ea 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -10,7 +10,7 @@ use conduit::{ err, error, pdu::PduBuilder, utils::{math::usize_from_f64, ReadyExt}, - Error, PduEvent, Result, + Err, Error, Event, PduEvent, Result, }; use futures::StreamExt; use lru_cache::LruCache; @@ -29,7 +29,7 @@ use ruma::{ power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, topic::RoomTopicEventContent, }, - StateEventType, + StateEventType, TimelineEventType, }, room::RoomType, space::SpaceRoomJoinRule, @@ -408,34 +408,41 @@ impl Service { pub async fn user_can_redact( &self, redacts: &EventId, sender: &UserId, room_id: &RoomId, federation: bool, ) -> Result { - if let Ok(event) = self + let redacting_event = self.services.timeline.get_pdu(redacts).await; + + if redacting_event + .as_ref() + .is_ok_and(|event| event.event_type() == &TimelineEventType::RoomCreate) + { + return Err!(Request(Forbidden("Redacting m.room.create is not safe, forbidding."))); + } + + if let Ok(pl_event_content) = self .room_state_get_content::(room_id, &StateEventType::RoomPowerLevels, "") .await { - let event: RoomPowerLevels = event.into(); - Ok(event.user_can_redact_event_of_other(sender) - || event.user_can_redact_own_event(sender) - && if let Ok(pdu) = self.services.timeline.get_pdu(redacts).await { + let pl_event: RoomPowerLevels = pl_event_content.into(); + Ok(pl_event.user_can_redact_event_of_other(sender) + || pl_event.user_can_redact_own_event(sender) + && if let Ok(redacting_event) = redacting_event { if federation { - pdu.sender.server_name() == sender.server_name() + redacting_event.sender.server_name() == sender.server_name() } else { - pdu.sender == sender + redacting_event.sender == sender } } else { false }) } else { // Falling back on m.room.create to judge power level - if let Ok(pdu) = self + if let Ok(room_create) = self .room_state_get(room_id, &StateEventType::RoomCreate, "") .await { - Ok(pdu.sender == sender - || if let Ok(pdu) = self.services.timeline.get_pdu(redacts).await { - pdu.sender == sender - } else { - false - }) + Ok(room_create.sender == sender + || redacting_event + .as_ref() + .is_ok_and(|redacting_event| redacting_event.sender == sender)) } else { Err(Error::bad_database( "No m.room.power_levels or m.room.create events in database for room", @@ -454,7 +461,7 @@ impl Service { /// Returns an empty vec if not a restricted room pub fn allowed_room_ids(&self, join_rule: JoinRule) -> Vec { - let mut room_ids = vec![]; + let mut room_ids = Vec::with_capacity(1); if let JoinRule::Restricted(r) | JoinRule::KnockRestricted(r) = join_rule { for rule in r.allow { if let AllowRule::RoomMembership(RoomMembership { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 5bf48aaa..6a8f1b1b 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -39,7 +39,7 @@ impl super::Service { .forbidden_remote_server_names .contains(dest) { - return Err!(Request(Forbidden(debug_warn!("Federation with this {dest} is not allowed.")))); + return Err!(Request(Forbidden(debug_warn!("Federation with {dest} is not allowed.")))); } let actual = self.services.resolver.get_actual_dest(dest).await?; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index f4268293..f5d87504 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -235,13 +235,15 @@ impl Service { fn select_events_current(&self, dest: Destination, statuses: &mut CurTransactionStatus) -> Result<(bool, bool)> { let (mut allow, mut retry) = (true, false); statuses - .entry(dest) + .entry(dest.clone()) // TODO: can we avoid cloning? .and_modify(|e| match e { TransactionStatus::Failed(tries, time) => { // Fail if a request has failed recently (exponential backoff) let min = self.server.config.sender_timeout; let max = self.server.config.sender_retry_backoff_limit; - if continue_exponential_backoff_secs(min, max, time.elapsed(), *tries) { + if continue_exponential_backoff_secs(min, max, time.elapsed(), *tries) + && !matches!(dest, Destination::Appservice(_)) + { allow = false; } else { retry = true; From 72fb8371f9828b5c039883467ea35e3b41b4b42c Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 13 Nov 2024 17:08:16 -0500 Subject: [PATCH 1050/2091] link to migrating from conduit on the README Signed-off-by: strawberry --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 962139d6..4e97f1f0 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/bad Visit the [conduwuit documentation](https://conduwuit.puppyirl.gay/) for more -information. +information and how to deploy/setup conduwuit. @@ -63,7 +63,9 @@ and we have no plans in stopping or slowing down any time soon! conduwuit is a complete drop-in replacement for Conduit. As long as you are using RocksDB, the only "migration" you need to do is replace the binary or container image. There -is no harm or additional steps required for using conduwuit. +is no harm or additional steps required for using conduwuit. See the +[Migrating from Conduit](https://conduwuit.puppyirl.gay/deploying/generic.html#migrating-from-conduit) section +on the generic deploying guide. From 011d44b749bc206732572c826b4ca9ae2a6111e3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 13 Nov 2024 20:06:25 -0500 Subject: [PATCH 1051/2091] add missing declared support for MSC3952 Signed-off-by: strawberry --- src/api/client/unversioned.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index d5bb14e5..3aee30c8 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -53,6 +53,7 @@ pub(crate) async fn get_supported_versions_route( ("org.matrix.msc2946".to_owned(), true), /* spaces/hierarchy summaries (https://github.com/matrix-org/matrix-spec-proposals/pull/2946) */ ("org.matrix.msc3026.busy_presence".to_owned(), true), /* busy presence status (https://github.com/matrix-org/matrix-spec-proposals/pull/3026) */ ("org.matrix.msc3827".to_owned(), true), /* filtering of /publicRooms by room type (https://github.com/matrix-org/matrix-spec-proposals/pull/3827) */ + ("org.matrix.msc3952_intentional_mentions".to_owned(), true), /* intentional mentions (https://github.com/matrix-org/matrix-spec-proposals/pull/3952) */ ("org.matrix.msc3575".to_owned(), true), /* sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/3575/files#r1588877046) */ ("org.matrix.msc3916.stable".to_owned(), true), /* authenticated media (https://github.com/matrix-org/matrix-spec-proposals/pull/3916) */ ("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */ From 44a7ac07036915263f93e43f572dca8446d7ef9f Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 09:40:04 -0500 Subject: [PATCH 1052/2091] add debug_assert is_sorted for inline content types Signed-off-by: strawberry --- src/core/utils/content_disposition.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index a2fe923c..3a264a74 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -45,9 +45,10 @@ pub fn content_disposition_type(content_type: Option<&str>) -> ContentDispositio return ContentDispositionType::Attachment; }; - // is_sorted is unstable - /* debug_assert!(ALLOWED_INLINE_CONTENT_TYPES.is_sorted(), - * "ALLOWED_INLINE_CONTENT_TYPES is not sorted"); */ + debug_assert!( + ALLOWED_INLINE_CONTENT_TYPES.is_sorted(), + "ALLOWED_INLINE_CONTENT_TYPES is not sorted" + ); let content_type: Cow<'_, str> = content_type .split(';') From dac1a01216e53c4aca4e54057db487dc7682f30a Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 09:43:58 -0500 Subject: [PATCH 1053/2091] update generated example config Signed-off-by: strawberry --- conduwuit-example.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index aa0d1e5d..2f3da71f 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -592,6 +592,10 @@ # #log_colors = true +# configures the span events which will be outputted with the log +# +#log_span_events = "none" + # OpenID token expiration/TTL in seconds # # These are the OpenID tokens that are primarily used for Matrix account From 3f69f2ee73960a8cc0f0a3f7a2e1202ad43928e6 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 09:44:29 -0500 Subject: [PATCH 1054/2091] replace deprecated sha-1 crate, try to reduce some unnecessary crates/features Signed-off-by: strawberry --- Cargo.lock | 49 +++++------------------------------- Cargo.toml | 62 +++++++++++++++++++++++++++------------------- src/api/Cargo.toml | 2 +- 3 files changed, 43 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e1845da..65eab0b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -676,7 +676,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "sha-1", + "sha1", "tokio", "tracing", ] @@ -720,7 +720,7 @@ dependencies = [ "serde_json", "serde_regex", "serde_yaml", - "thiserror 1.0.69", + "thiserror 2.0.3", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -1753,9 +1753,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -1766,7 +1766,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower 0.4.13", "tower-service", "tracing", ] @@ -2080,11 +2079,9 @@ checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" dependencies = [ "base64 0.21.7", "js-sys", - "pem", "ring", "serde", "serde_json", - "simple_asn1", ] [[package]] @@ -2662,16 +2659,6 @@ dependencies = [ "syn 2.0.87", ] -[[package]] -name = "pem" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" -dependencies = [ - "base64 0.22.1", - "serde", -] - [[package]] name = "percent-encoding" version = "2.3.1" @@ -3535,11 +3522,10 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "sanitize-filename" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ed72fbaf78e6f2d41744923916966c4fbe3d7c74e3037a8ee482f1115572603" +checksum = "bc984f4f9ceb736a7bb755c3e3bd17dc56370af2600c9780dcc48c66453da34d" dependencies = [ - "lazy_static", "regex", ] @@ -3827,17 +3813,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "sha-1" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5058ada175748e33390e40e872bd0fe59a19f265d0158daa551c5a88a76009c" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha1" version = "0.10.6" @@ -3920,18 +3895,6 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" -[[package]] -name = "simple_asn1" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" -dependencies = [ - "num-bigint", - "num-traits", - "thiserror 1.0.69", - "time", -] - [[package]] name = "siphasher" version = "0.3.11" diff --git a/Cargo.toml b/Cargo.toml index dde005a3..a84ff79f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,20 +46,20 @@ default-features = false features = ["parse"] [workspace.dependencies.sanitize-filename] -version = "0.5.0" +version = "0.6.0" [workspace.dependencies.jsonwebtoken] version = "9.3.0" +default-features = false [workspace.dependencies.base64] version = "0.22.1" +default-features = false # used for TURN server authentication [workspace.dependencies.hmac] version = "0.12.1" - -[workspace.dependencies.sha-1] -version = "0.10.1" +default-features = false # used for checking if an IP is in specific subnets / CIDR ranges easier [workspace.dependencies.ipaddress] @@ -70,16 +70,16 @@ version = "0.8.5" # Used for the http request / response body type for Ruma endpoints used with reqwest [workspace.dependencies.bytes] -version = "1.7.2" +version = "1.8.0" [workspace.dependencies.http-body-util] -version = "0.1.1" +version = "0.1.2" [workspace.dependencies.http] version = "1.1.0" [workspace.dependencies.regex] -version = "1.10.6" +version = "1.11.1" [workspace.dependencies.axum] version = "0.7.5" @@ -95,7 +95,7 @@ features = [ ] [workspace.dependencies.axum-extra] -version = "0.9.3" +version = "0.9.4" default-features = false features = ["typed-header", "tracing"] @@ -116,7 +116,7 @@ default-features = false features = ["util"] [workspace.dependencies.tower-http] -version = "0.6.0" +version = "0.6.1" default-features = false features = [ "add-extension", @@ -130,6 +130,8 @@ features = [ [workspace.dependencies.rustls] version = "0.23.16" +default-features = false +features = ["aws_lc_rs"] [workspace.dependencies.reqwest] version = "0.12.9" @@ -147,7 +149,7 @@ default-features = false features = ["rc"] [workspace.dependencies.serde_json] -version = "1.0.124" +version = "1.0.132" default-features = false features = ["raw_value"] @@ -189,9 +191,11 @@ version = "0.1.40" default-features = false [workspace.dependencies.tracing-subscriber] version = "0.3.18" -features = ["env-filter"] +default-features = false +features = ["env-filter", "std", "tracing", "tracing-log", "ansi", "fmt"] [workspace.dependencies.tracing-core] version = "0.1.32" +default-features = false # for URL previews [workspace.dependencies.webpage] @@ -200,12 +204,14 @@ default-features = false # used for conduit's CLI and admin room command parsing [workspace.dependencies.clap] -version = "4.5.20" +version = "4.5.21" default-features = false features = [ "std", "derive", "help", + #"color", Do we need these? + #"unicode", "usage", "error-context", "string", @@ -217,7 +223,7 @@ default-features = false features = ["std", "async-await"] [workspace.dependencies.tokio] -version = "1.40.0" +version = "1.41.1" default-features = false features = [ "fs", @@ -238,7 +244,7 @@ version = "0.8.5" # Validating urls in config, was already a transitive dependency [workspace.dependencies.url] -version = "2.5.0" +version = "2.5.3" default-features = false features = ["serde"] @@ -258,26 +264,23 @@ features = [ ] [workspace.dependencies.hyper-util] -# 0.1.9 and above causes DNS issues -version = "=0.1.8" +version = "0.1.10" default-features = false features = [ - "client", "server-auto", "server-graceful", - "service", "tokio", ] # to support multiple variations of setting a config option [workspace.dependencies.either] -version = "1.11.0" +version = "1.13.0" default-features = false features = ["serde"] # Used for reading the configuration from conduwuit.toml & environment variables [workspace.dependencies.figment] -version = "0.10.18" +version = "0.10.19" default-features = false features = ["env", "toml"] @@ -287,11 +290,13 @@ default-features = false # Used for conduit::Error type [workspace.dependencies.thiserror] -version = "1.0.63" +version = "2.0.3" +default-features = false # Used when hashing the state [workspace.dependencies.ring] version = "0.17.8" +default-features = false # Used to make working with iterators easier, was already a transitive depdendency [workspace.dependencies.itertools] @@ -307,7 +312,7 @@ version = "2.1.1" version = "0.4.0" [workspace.dependencies.async-trait] -version = "0.1.81" +version = "0.1.83" [workspace.dependencies.lru-cache] version = "0.1.2" @@ -363,9 +368,13 @@ features = [ "bzip2", ] -# optional SHA256 media keys feature [workspace.dependencies.sha2] version = "0.10.8" +default-features = false + +[workspace.dependencies.sha1] +version = "0.10.6" +default-features = false # optional opentelemetry, performance measurements, flamegraphs, etc for performance measurements and monitoring [workspace.dependencies.opentelemetry] @@ -433,7 +442,8 @@ default-features = false features = ["resource"] [workspace.dependencies.sd-notify] -version = "0.4.1" +version = "0.4.3" +default-features = false [workspace.dependencies.hardened_malloc-rs] version = "0.1.2" @@ -456,12 +466,12 @@ default-features = false version = "0.1" [workspace.dependencies.syn] -version = "2.0.76" +version = "2.0.87" default-features = false features = ["full", "extra-traits"] [workspace.dependencies.quote] -version = "1.0.36" +version = "1.0.37" [workspace.dependencies.proc-macro2] version = "1.0.89" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 6e37cb40..a0fc09de 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -59,7 +59,7 @@ ruma.workspace = true serde_html_form.workspace = true serde_json.workspace = true serde.workspace = true -sha-1.workspace = true +sha1.workspace = true tokio.workspace = true tracing.workspace = true From b4d809c68157a09bbfd2381bcea04407bf3b3dd2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 09:49:54 -0500 Subject: [PATCH 1055/2091] add more checks for gh pages deployment workflow Signed-off-by: strawberry --- .github/workflows/documentation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 506a87d9..ea720c43 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -39,7 +39,7 @@ concurrency: jobs: docs: name: Documentation and GitHub Pages - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 permissions: pages: write @@ -57,7 +57,7 @@ jobs: uses: actions/checkout@v4 - name: Setup GitHub Pages - if: github.event_name != 'pull_request' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') && (github.event.pull_request.user.login == 'girlbossceo') uses: actions/configure-pages@v5 - uses: nixbuild/nix-quick-install-action@master @@ -139,12 +139,12 @@ jobs: compression-level: 0 - name: Upload generated documentation (book) as GitHub Pages artifact - if: github.event_name != 'pull_request' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') && (github.event.pull_request.user.login == 'girlbossceo') uses: actions/upload-pages-artifact@v3 with: path: public - name: Deploy to GitHub Pages - if: github.event_name != 'pull_request' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') && (github.event.pull_request.user.login == 'girlbossceo') id: deployment uses: actions/deploy-pages@v4 From c1f553cf4f938da469b69bfd37060ca65a827762 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 12:25:47 -0500 Subject: [PATCH 1056/2091] bump rocksdb to v9.7.4, and ruwuma Signed-off-by: strawberry --- Cargo.lock | 55 ++++++++++++++++++------------------ Cargo.toml | 2 +- deps/rust-rocksdb/Cargo.toml | 2 +- flake.lock | 8 +++--- flake.nix | 2 +- 5 files changed, 34 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65eab0b5..b56005ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,9 +191,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504e3947307ac8326a5437504c517c4b56716c9d98fac0028c2acc7ca47d70ae" +checksum = "49c41b948da08fb481a94546cd874843adc1142278b0af4badf9b1b78599d68d" dependencies = [ "async-trait", "axum-core", @@ -257,9 +257,9 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c3220b188aea709cf1b6c5f9b01c3bd936bb08bd2b5184a12b35ac8131b1f9" +checksum = "37634d71e9f3c35cfb1c30c87c7cba500d55892f04c2dbe6a99383c664b820b0" dependencies = [ "axum", "axum-core", @@ -275,7 +275,6 @@ dependencies = [ "tower 0.5.1", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -487,9 +486,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aeb932158bd710538c73702db6945cb68a8fb08c519e6e12706b94263b36db8" +checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" dependencies = [ "jobserver", "libc", @@ -1264,9 +1263,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.34" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" dependencies = [ "crc32fast", "miniz_oxide", @@ -1740,9 +1739,9 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ "hyper", "hyper-util", @@ -3128,7 +3127,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "assign", "js_int", @@ -3150,7 +3149,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "js_int", "ruma-common", @@ -3162,7 +3161,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "as_variant", "assign", @@ -3185,7 +3184,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "as_variant", "base64 0.22.1", @@ -3215,7 +3214,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3239,7 +3238,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "bytes", "http", @@ -3257,7 +3256,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "js_int", "thiserror 2.0.3", @@ -3266,7 +3265,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "js_int", "ruma-common", @@ -3276,7 +3275,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "cfg-if", "once_cell", @@ -3292,7 +3291,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "js_int", "ruma-common", @@ -3304,7 +3303,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "headers", "http", @@ -3317,7 +3316,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3333,7 +3332,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=67ffedabbf43e1ff6934df0fbf770b21e101406f#67ffedabbf43e1ff6934df0fbf770b21e101406f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" dependencies = [ "futures-util", "itertools 0.13.0", @@ -3348,8 +3347,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.28.0+9.7.3" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=c1e5523eae095a893deaf9056128c7dbc2d5fd73#c1e5523eae095a893deaf9056128c7dbc2d5fd73" +version = "0.29.0+9.7.4" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2bc5495a9f8f75073390c326b47ee5928ab7c7f0#2bc5495a9f8f75073390c326b47ee5928ab7c7f0" dependencies = [ "bindgen", "bzip2-sys", @@ -3365,8 +3364,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.31.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=c1e5523eae095a893deaf9056128c7dbc2d5fd73#c1e5523eae095a893deaf9056128c7dbc2d5fd73" +version = "0.33.0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2bc5495a9f8f75073390c326b47ee5928ab7c7f0#2bc5495a9f8f75073390c326b47ee5928ab7c7f0" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index a84ff79f..814a435b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -321,7 +321,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "67ffedabbf43e1ff6934df0fbf770b21e101406f" +rev = "2ab432fba19eb8862c594d24af39d8f9f6b4eac6" features = [ "compat", "rand", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 8c168b24..908a2911 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "c1e5523eae095a893deaf9056128c7dbc2d5fd73" +rev = "2bc5495a9f8f75073390c326b47ee5928ab7c7f0" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index 271a2154..7740e925 100644 --- a/flake.lock +++ b/flake.lock @@ -922,16 +922,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1729712930, - "narHash": "sha256-jlp4kPkRTpoJaUdobEoHd8rCGAQNBy4ZHZ6y5zL/ibw=", + "lastModified": 1731690620, + "narHash": "sha256-Xd4TJYqPERMJLXaGa6r6Ny1Wlw8Uy5Cyf/8q7nS58QM=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "871eda6953c3f399aae39808dcfccdd014885beb", + "rev": "292446aa2bc41699204d817a1e4b091679a886eb", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.7.3", + "ref": "v9.7.4", "repo": "rocksdb", "type": "github" } diff --git a/flake.nix b/flake.nix index 85b7baa0..113757a7 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.7.3"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.7.4"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From a9c280bd4cd0616c57c9df28a11c3bc48ae8b5ba Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 12:58:02 -0500 Subject: [PATCH 1057/2091] document NAT hairpinning/loopback if needed Signed-off-by: strawberry --- docs/deploying/generic.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 6fe9709b..f0b85a25 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -68,13 +68,25 @@ sudo useradd -r --shell /usr/bin/nologin --no-create-home conduwuit ## Forwarding ports in the firewall or the router -conduwuit uses the ports 443 and 8448 both of which need to be open in the -firewall. +Matrix's default federation port is port 8448, and clients must be using port 443. +If you would like to use only port 443, or a different port, you will need to setup +delegation. conduwuit has config options for doing delegation, or you can configure +your reverse proxy to manually serve the necessary JSON files to do delegation +(see the `[global.well_known]` config section). If conduwuit runs behind a router or in a container and has a different public IP address than the host system these public ports need to be forwarded directly or indirectly to the port mentioned in the config. +Note for NAT users; if you have trouble connecting to your server from the inside +of your network, you need to research your router and see if it supports "NAT +hairpinning" or "NAT loopback". + +If your router does not support this feature, you need to research doing local +DNS overrides and force your Matrix DNS records to use your local IP internally. +This can be done at the host level using `/etc/hosts`. If you need this to be +on the network level, consider something like NextDNS or Pi-Hole. + ## Setting up a systemd service The systemd unit for conduwuit can be found From c23786d37f207f45632d8288affbcd51bfb5e5c8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 12:59:05 -0500 Subject: [PATCH 1058/2091] dont try to backfill empty, private rooms Signed-off-by: strawberry --- src/api/client/membership.rs | 2 +- src/service/rooms/timeline/mod.rs | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 10e69f58..c61185a7 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1481,7 +1481,7 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, .await { if let Err(e) = remote_leave_room(services, user_id, room_id).await { - warn!("Failed to leave room {user_id} remotely: {e}"); + warn!(%user_id, "Failed to leave room {room_id} remotely: {e}"); // Don't tell the client about this error } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 281879d2..2faa1c40 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1033,6 +1033,22 @@ impl Service { #[tracing::instrument(skip(self))] pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { + if self + .services + .state_cache + .room_joined_count(room_id) + .await + .is_ok_and(|count| count <= 1) + && !self + .services + .state_accessor + .is_world_readable(room_id) + .await + { + // Room is empty (1 user or none), there is no one that can backfill + return Ok(()); + } + let first_pdu = self .all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id) .await? @@ -1060,20 +1076,8 @@ impl Service { } }); - let room_alias_servers = self - .services - .alias - .local_aliases_for_room(room_id) - .ready_filter_map(|alias| { - self.services - .globals - .server_is_ours(alias.server_name()) - .then_some(alias.server_name()) - }); - let mut servers = room_mods .stream() - .chain(room_alias_servers) .map(ToOwned::to_owned) .chain( self.services From 9783bc78ba096f75db2c529f3e2a7f6cb76f51fe Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 13:58:44 -0500 Subject: [PATCH 1059/2091] remove sentry_telemetry from default features Signed-off-by: strawberry --- src/main/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index b3390bfb..b9122942 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -44,7 +44,6 @@ default = [ "jemalloc", "jemalloc_stats", "release_max_log_level", - "sentry_telemetry", "systemd", "zstd_compression", ] From 666989f74ce8a80b3d24132393c2e2e331fe719c Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 13:19:32 -0500 Subject: [PATCH 1060/2091] delete trivy as lately its been terribly unreliable Signed-off-by: strawberry --- .github/workflows/trivy.yml | 42 ------------------------------------- docs/differences.md | 3 +-- 2 files changed, 1 insertion(+), 44 deletions(-) delete mode 100644 .github/workflows/trivy.yml diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml deleted file mode 100644 index 1f0dd7df..00000000 --- a/.github/workflows/trivy.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Trivy code and vulnerability scanning - -on: - pull_request: - push: - branches: - - main - tags: - - '*' - schedule: - - cron: '00 12 * * *' - -permissions: - contents: read - -jobs: - trivy-scan: - name: Trivy Scan - runs-on: ubuntu-latest - permissions: - contents: read - security-events: write - actions: read - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Run Trivy code and vulnerability scanner on repo - uses: aquasecurity/trivy-action@0.28.0 - with: - scan-type: repo - format: sarif - output: trivy-results.sarif - severity: CRITICAL,HIGH,MEDIUM,LOW - - - name: Run Trivy code and vulnerability scanner on filesystem - uses: aquasecurity/trivy-action@0.28.0 - with: - scan-type: fs - format: sarif - output: trivy-results.sarif - severity: CRITICAL,HIGH,MEDIUM,LOW diff --git a/docs/differences.md b/docs/differences.md index 6815d248..18ea7a1f 100644 --- a/docs/differences.md +++ b/docs/differences.md @@ -241,8 +241,7 @@ both new users and power users - Fixed every single clippy (default lints) and rustc warnings, including some that were performance related or potential safety issues / unsoundness - Add a **lot** of other clippy and rustc lints and a rustfmt.toml file -- Repo uses [Renovate](https://docs.renovatebot.com/), -[Trivy](https://github.com/aquasecurity/trivy-action), and keeps ALL +- Repo uses [Renovate](https://docs.renovatebot.com/) and keeps ALL dependencies as up to date as possible - Purge unmaintained/irrelevant/broken database backends (heed, sled, persy) and other unnecessary code or overhead From f897b4daeea8147a96286e96e6d828d364136dd8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 13:20:47 -0500 Subject: [PATCH 1061/2091] ci: remove all free runner space steps due to flakiness Signed-off-by: strawberry --- .github/workflows/ci.yml | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d253f69..f4dcb88f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,21 +67,6 @@ jobs: name: Test runs-on: ubuntu-24.04 steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - - - name: Free up more runner space - run: | - set +o pipefail - # large docker images - sudo docker image prune --all --force || true - # large packages - sudo apt-get purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true - sudo apt-get clean - # large folders - sudo rm -rf /var/lib/apt/lists/* /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/local/share/vcpkg /usr/local/julia* /opt/mssql-tools /usr/share/vim /usr/share/postgresql /usr/share/apache-maven-* /usr/share/R /usr/share/alsa /usr/share/miniconda /usr/share/grub /usr/share/gradle-* /usr/share/locale /usr/share/texinfo /usr/share/kotlinc /usr/share/swift /usr/share/sbt /usr/share/ri /usr/share/icons /usr/share/java /usr/share/fonts /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/R /usr/lib/postgresql /usr/lib/heroku - set -o pipefail - - name: Sync repository uses: actions/checkout@v4 @@ -238,9 +223,6 @@ jobs: - target: aarch64-linux-musl - target: x86_64-linux-musl steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - - name: Sync repository uses: actions/checkout@v4 @@ -449,6 +431,7 @@ jobs: steps: - name: Sync repository uses: actions/checkout@v4 + - name: Tag comparison check if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} run: | @@ -459,14 +442,17 @@ jobs: echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY exit 1 fi + # use sccache for Rust - name: Run sccache-cache if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') uses: mozilla-actions/sccache-action@main + # use rust-cache - uses: Swatinem/rust-cache@v2 with: cache-all-crates: "true" + # Nix can't do portable macOS builds yet - name: Build macOS x86_64 binary if: ${{ matrix.os == 'macos-13' }} @@ -474,22 +460,26 @@ jobs: CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short HEAD)" cargo build --release cp -v -f target/release/conduit conduwuit-macos-x86_64 otool -L conduwuit-macos-x86_64 + # quick smoke test of the x86_64 macOS binary - name: Run x86_64 macOS release binary if: ${{ matrix.os == 'macos-13' }} run: | ./conduwuit-macos-x86_64 --version + - name: Build macOS arm64 binary if: ${{ matrix.os == 'macos-latest' }} run: | CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short HEAD)" cargo build --release cp -v -f target/release/conduit conduwuit-macos-arm64 otool -L conduwuit-macos-arm64 + # quick smoke test of the arm64 macOS binary - name: Run arm64 macOS release binary if: ${{ matrix.os == 'macos-latest' }} run: | ./conduwuit-macos-arm64 --version + - name: Upload macOS x86_64 binary if: ${{ matrix.os == 'macos-13' }} uses: actions/upload-artifact@v4 @@ -497,6 +487,7 @@ jobs: name: conduwuit-macos-x86_64 path: conduwuit-macos-x86_64 if-no-files-found: error + - name: Upload macOS arm64 binary if: ${{ matrix.os == 'macos-latest' }} uses: actions/upload-artifact@v4 From 6b1b464abcecacea98e06040aba679e9bdc3cec9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 14:48:10 -0500 Subject: [PATCH 1062/2091] add missing knock_restricted room type to /publicRooms Signed-off-by: strawberry --- src/api/client/directory.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 6cf7b13f..6120c7b3 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -407,7 +407,8 @@ async fn public_rooms_chunk(services: &Services, room_id: OwnedRoomId) -> Public .room_state_get_content(&room_id, &StateEventType::RoomJoinRules, "") .map_ok(|c: RoomJoinRulesEventContent| match c.join_rule { JoinRule::Public => PublicRoomJoinRule::Public, - JoinRule::Knock => PublicRoomJoinRule::Knock, + JoinRule::Knock => "knock".into(), + JoinRule::KnockRestricted(_) => "knock_restricted".into(), _ => "invite".into(), }) .await From 9c95a74d56bf80fb1f09984a9c05b16d25d320da Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 16:35:58 -0500 Subject: [PATCH 1063/2091] fix getting canonical alias server for backfill Signed-off-by: strawberry --- src/service/rooms/timeline/mod.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 2faa1c40..59fc8e93 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -4,6 +4,7 @@ use std::{ cmp, collections::{BTreeMap, HashSet}, fmt::Write, + iter::once, sync::Arc, }; @@ -1076,9 +1077,20 @@ impl Service { } }); + let canonical_room_alias_server = once( + self.services + .state_accessor + .get_canonical_alias(room_id) + .await, + ) + .filter_map(Result::ok) + .map(|alias| alias.server_name().to_owned()) + .stream(); + let mut servers = room_mods .stream() .map(ToOwned::to_owned) + .chain(canonical_room_alias_server) .chain( self.services .server From be5a04f47cf1da5629239804601978ddcb2a1db1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 17:09:36 -0500 Subject: [PATCH 1064/2091] ci: install liburing-dev Signed-off-by: strawberry --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f4dcb88f..96a1188b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -67,6 +67,10 @@ jobs: name: Test runs-on: ubuntu-24.04 steps: + - name: Install liburing + run: | + sudo apt install liburing-dev -y + - name: Sync repository uses: actions/checkout@v4 From 4b652f5236f3482316fc8dfc522f705c3b85b586 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 17:50:39 -0500 Subject: [PATCH 1065/2091] ok cargo doc Signed-off-by: strawberry --- src/core/config/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 4bba1455..cb9d087b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1524,7 +1524,7 @@ pub struct WellKnownConfig { /// The server URL that the client well-known file will serve. This should /// not contain a port, and should just be a valid HTTPS URL. /// - /// example: "https://matrix.example.com" + /// example: "" pub client: Option, pub support_page: Option, pub support_role: Option, From 59834a4b05784c6e5e9ba12c5c5cc06f5ba98825 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 14 Nov 2024 22:43:18 +0000 Subject: [PATCH 1066/2091] add is_read_only()/is_secondary() to Engine Signed-off-by: Jason Volk --- src/database/database.rs | 4 ++-- src/database/engine.rs | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/database/database.rs b/src/database/database.rs index bf8c8855..40aec312 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -38,11 +38,11 @@ impl Database { #[inline] #[must_use] - pub fn is_read_only(&self) -> bool { self.db.secondary || self.db.read_only } + pub fn is_read_only(&self) -> bool { self.db.is_read_only() } #[inline] #[must_use] - pub fn is_secondary(&self) -> bool { self.db.secondary } + pub fn is_secondary(&self) -> bool { self.db.is_secondary() } } impl Index<&str> for Database { diff --git a/src/database/engine.rs b/src/database/engine.rs index 99d971ed..b57fd75e 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -274,6 +274,14 @@ impl Engine { result(self.db.property_value_cf(cf, name)) .and_then(|val| val.map_or_else(|| Err!("Property {name:?} not found."), Ok)) } + + #[inline] + #[must_use] + pub fn is_read_only(&self) -> bool { self.secondary || self.read_only } + + #[inline] + #[must_use] + pub fn is_secondary(&self) -> bool { self.secondary } } pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> { From 20836cc3dbc2e22c6d7da99ec199930b6d4c7ad4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 14 Nov 2024 22:44:18 +0000 Subject: [PATCH 1067/2091] flush=false for database-backup in read-only/secondary modes; improve error Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 2 +- src/database/engine.rs | 14 ++++++++------ src/service/globals/data.rs | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index f5879b03..94f695ce 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -107,7 +107,7 @@ pub(super) async fn backup_database(&self) -> Result { .runtime() .spawn_blocking(move || match globals.db.backup() { Ok(()) => String::new(), - Err(e) => (*e).to_string(), + Err(e) => e.to_string(), }) .await?; diff --git a/src/database/engine.rs b/src/database/engine.rs index b57fd75e..1fa53b01 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -17,6 +17,7 @@ use rocksdb::{ use crate::{ opts::{cf_options, db_options}, or_else, result, + util::map_err, }; pub struct Engine { @@ -183,19 +184,20 @@ impl Engine { } #[tracing::instrument(skip(self))] - pub fn backup(&self) -> Result<(), Box> { + pub fn backup(&self) -> Result { let config = &self.server.config; let path = config.database_backup_path.as_ref(); if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { return Ok(()); } - let options = BackupEngineOptions::new(path.expect("valid database backup path"))?; - let mut engine = BackupEngine::open(&options, &self.env)?; + let options = BackupEngineOptions::new(path.expect("valid database backup path")).map_err(map_err)?; + let mut engine = BackupEngine::open(&options, &self.env).map_err(map_err)?; if config.database_backups_to_keep > 0 { - if let Err(e) = engine.create_new_backup_flush(&self.db, true) { - return Err(Box::new(e)); - } + let flush = !self.is_read_only(); + engine + .create_new_backup_flush(&self.db, flush) + .map_err(map_err)?; let engine_info = engine.get_backup_info(); let info = &engine_info.last().expect("backup engine info is not empty"); diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index bcfe101e..f715e944 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -73,7 +73,7 @@ impl Data { } #[inline] - pub fn backup(&self) -> Result<(), Box> { self.db.db.backup() } + pub fn backup(&self) -> Result { self.db.db.backup() } #[inline] pub fn backup_list(&self) -> Result { self.db.db.backup_list() } From 5f625216aa027aa000037c26fee461684e25689c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 14 Nov 2024 23:35:53 +0000 Subject: [PATCH 1068/2091] slight optimizations for statediff calculate with_capacity for set/get_statediff() etc Signed-off-by: Jason Volk --- src/service/rooms/state_compressor/mod.rs | 50 +++++++++++++++-------- 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 6b520ad3..06054f0d 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -5,6 +5,7 @@ use std::{ sync::{Arc, Mutex}, }; +use arrayvec::ArrayVec; use conduit::{ at, checked, err, expected, utils, utils::{bytes, math::usize_from_f64}, @@ -37,7 +38,7 @@ struct Data { #[derive(Clone)] struct StateDiff { - parent: Option, + parent: Option, added: Arc, removed: Arc, } @@ -165,17 +166,20 @@ impl Service { } pub async fn compress_state_event(&self, shortstatekey: ShortStateKey, event_id: &EventId) -> CompressedStateEvent { - let mut v = shortstatekey.to_be_bytes().to_vec(); - v.extend_from_slice( - &self - .services - .short - .get_or_create_shorteventid(event_id) - .await - .to_be_bytes(), - ); + const SIZE: usize = size_of::(); - v.try_into().expect("we checked the size above") + let shorteventid = self + .services + .short + .get_or_create_shorteventid(event_id) + .await; + + let mut v = ArrayVec::::new(); + v.extend(shortstatekey.to_be_bytes()); + v.extend(shorteventid.to_be_bytes()); + v.as_ref() + .try_into() + .expect("failed to create CompressedStateEvent") } /// Returns shortstatekey, event id @@ -185,11 +189,12 @@ impl Service { ) -> Result<(ShortStateKey, Arc)> { use utils::u64_from_u8; - let shortstatekey = u64_from_u8(&compressed_event[0..size_of::()]); + let shortstatekey = u64_from_u8(&compressed_event[0..size_of::()]); + let shorteventid = u64_from_u8(&compressed_event[size_of::()..]); let event_id = self .services .short - .get_eventid_from_short(u64_from_u8(&compressed_event[size_of::()..])) + .get_eventid_from_short(shorteventid) .await?; Ok((shortstatekey, event_id)) @@ -415,9 +420,12 @@ impl Service { .ok() .take_if(|parent| *parent != 0); + debug_assert!(value.len() % STRIDE == 0, "value not aligned to stride"); + let num_values = value.len() / STRIDE; + let mut add_mode = true; - let mut added = HashSet::new(); - let mut removed = HashSet::new(); + let mut added = HashSet::with_capacity(num_values); + let mut removed = HashSet::with_capacity(num_values); let mut i = STRIDE; while let Some(v) = value.get(i..expected!(i + 2 * STRIDE)) { @@ -434,6 +442,8 @@ impl Service { i = expected!(i + 2 * STRIDE); } + added.shrink_to_fit(); + removed.shrink_to_fit(); Ok(StateDiff { parent, added: Arc::new(added), @@ -442,7 +452,15 @@ impl Service { } fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) { - let mut value = diff.parent.unwrap_or(0).to_be_bytes().to_vec(); + let mut value = Vec::::with_capacity( + 2_usize + .saturating_add(diff.added.len()) + .saturating_add(diff.removed.len()), + ); + + let parent = diff.parent.unwrap_or(0_u64); + value.extend_from_slice(&parent.to_be_bytes()); + for new in diff.added.iter() { value.extend_from_slice(&new[..]); } From 9f7a4a012b38c4e3a59ea72bbb5d291ecdbf37f4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 15 Nov 2024 03:41:08 +0000 Subject: [PATCH 1069/2091] improve tracing/logging for state_compressor Signed-off-by: Jason Volk --- src/service/rooms/state_compressor/mod.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 06054f0d..f0c851de 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -7,7 +7,7 @@ use std::{ use arrayvec::ArrayVec; use conduit::{ - at, checked, err, expected, utils, + at, checked, debug, err, expected, utils, utils::{bytes, math::usize_from_f64}, Result, }; @@ -157,6 +157,13 @@ impl Service { }] }; + debug!( + ?parent, + ?shortstatehash, + vec_len = %response.len(), + "cache update" + ); + self.stateinfo_cache .lock() .expect("locked") @@ -218,7 +225,6 @@ impl Service { /// for this layer /// * `parent_states` - A stack with info on shortstatehash, full state, /// added diff and removed diff for each parent layer - #[tracing::instrument(skip_all, level = "debug")] pub fn save_state_from_diff( &self, shortstatehash: ShortStateHash, statediffnew: Arc>, statediffremoved: Arc>, diff_to_sibling: usize, @@ -335,6 +341,7 @@ impl Service { /// Returns the new shortstatehash, and the state diff from the previous /// room state + #[tracing::instrument(skip(self, new_state_ids_compressed), level = "debug")] pub async fn save_state( &self, room_id: &RoomId, new_state_ids_compressed: Arc>, ) -> Result { @@ -405,6 +412,7 @@ impl Service { }) } + #[tracing::instrument(skip(self), level = "debug", name = "get")] async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result { const BUFSIZE: usize = size_of::(); const STRIDE: usize = size_of::(); From 14e3b242dfafbb31b9a9c58b586989a828a022f3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 15 Nov 2024 03:44:04 +0000 Subject: [PATCH 1070/2091] add database get_batch stream wrapper Signed-off-by: Jason Volk --- src/database/map/get.rs | 22 +++++++++++++++------- src/service/rooms/auth_chain/mod.rs | 12 ++++++------ src/service/rooms/short/mod.rs | 13 +++++++------ 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 72382e36..2f7df031 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -1,8 +1,8 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write}; use arrayvec::ArrayVec; -use conduit::{err, implement, Result}; -use futures::future::ready; +use conduit::{err, implement, utils::IterStream, Result}; +use futures::{future::ready, Stream}; use rocksdb::DBPinnableSlice; use serde::Serialize; @@ -50,6 +50,7 @@ where /// Fetch a value from the database into cache, returning a reference-handle /// asynchronously. The key is referenced directly to perform the query. #[implement(super::Map)] +#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] pub fn get(&self, key: &K) -> impl Future>> + Send where K: AsRef<[u8]> + ?Sized + Debug, @@ -61,10 +62,9 @@ where /// The key is referenced directly to perform the query. This is a thread- /// blocking call. #[implement(super::Map)] -#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] pub fn get_blocking(&self, key: &K) -> Result> where - K: AsRef<[u8]> + ?Sized + Debug, + K: AsRef<[u8]> + ?Sized, { let res = self .db @@ -76,10 +76,19 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self, keys), fields(%self), level = "trace")] -pub fn get_batch_blocking<'a, I, K>(&self, keys: I) -> Vec>> +pub fn get_batch<'a, I, K>(&self, keys: I) -> impl Stream>> where I: Iterator + ExactSizeIterator + Send + Debug, - K: AsRef<[u8]> + Sized + Debug + 'a, + K: AsRef<[u8]> + Send + Sync + Sized + Debug + 'a, +{ + self.get_batch_blocking(keys).stream() +} + +#[implement(super::Map)] +pub fn get_batch_blocking<'a, I, K>(&self, keys: I) -> impl Iterator>> +where + I: Iterator + ExactSizeIterator + Send, + K: AsRef<[u8]> + Sized + 'a, { // Optimization can be `true` if key vector is pre-sorted **by the column // comparator**. @@ -91,7 +100,6 @@ where .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) .into_iter() .map(into_result_handle) - .collect() } fn into_result_handle(result: RocksdbResult<'_>) -> Result> { diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index c22732c2..cabb6f0c 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use conduit::{debug, debug_error, trace, utils::IterStream, validated, warn, Err, Result}; -use futures::Stream; +use futures::{Stream, StreamExt}; use ruma::{EventId, RoomId}; use self::data::Data; @@ -69,15 +69,15 @@ impl Service { const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); let started = std::time::Instant::now(); - let mut buckets = [BUCKET; NUM_BUCKETS]; - for (i, &short) in self + let mut starting_ids = self .services .short .multi_get_or_create_shorteventid(starting_events) - .await - .iter() .enumerate() - { + .boxed(); + + let mut buckets = [BUCKET; NUM_BUCKETS]; + while let Some((i, short)) = starting_ids.next().await { let bucket: usize = short.try_into()?; let bucket: usize = validated!(bucket % NUM_BUCKETS); buckets[bucket].insert((short, starting_events[i])); diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index e8b00d9b..703df796 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -3,6 +3,7 @@ use std::{mem::size_of_val, sync::Arc}; pub use conduit::pdu::{ShortEventId, ShortId, ShortRoomId}; use conduit::{err, implement, utils, Result}; use database::{Deserialized, Map}; +use futures::{Stream, StreamExt}; use ruma::{events::StateEventType, EventId, RoomId}; use crate::{globals, Dep}; @@ -71,11 +72,12 @@ pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEvent } #[implement(Service)] -pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> Vec { +pub fn multi_get_or_create_shorteventid<'a>( + &'a self, event_ids: &'a [&EventId], +) -> impl Stream + Send + 'a { self.db .eventid_shorteventid - .get_batch_blocking(event_ids.iter()) - .into_iter() + .get_batch(event_ids.iter()) .enumerate() .map(|(i, result)| match result { Ok(ref short) => utils::u64_from_u8(short), @@ -95,7 +97,6 @@ pub async fn multi_get_or_create_shorteventid(&self, event_ids: &[&EventId]) -> short }, }) - .collect() } #[implement(Service)] @@ -163,10 +164,10 @@ pub async fn multi_get_eventid_from_short(&self, shorteventid: &[ShortEventId]) self.db .shorteventid_eventid - .get_batch_blocking(keys.iter()) - .into_iter() + .get_batch(keys.iter()) .map(Deserialized::deserialized) .collect() + .await } #[implement(Service)] From 887ae84f1e3b3e0254e04afe011083e692af7e00 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 15 Nov 2024 22:20:28 +0000 Subject: [PATCH 1071/2091] optimize sha256 interface gather/vector inputs Signed-off-by: Jason Volk --- src/api/server/invite.rs | 4 +- src/core/utils/hash.rs | 9 +-- src/core/utils/hash/sha256.rs | 67 ++++++++++++++++++++--- src/core/utils/mod.rs | 2 +- src/service/rooms/state/mod.rs | 7 +-- src/service/rooms/state_compressor/mod.rs | 7 +-- src/service/sending/sender.rs | 39 ++++++------- 7 files changed, 85 insertions(+), 50 deletions(-) diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index edf80cd6..0ceb914f 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -1,7 +1,7 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use base64::{engine::general_purpose, Engine as _}; -use conduit::{err, utils, warn, Err, Error, PduEvent, Result}; +use conduit::{err, utils, utils::hash::sha256, warn, Err, Error, PduEvent, Result}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_invite}, events::room::member::{MembershipState, RoomMemberEventContent}, @@ -160,7 +160,7 @@ pub(crate) async fn create_invite_route( ruma::api::appservice::event::push_events::v1::Request { events: vec![pdu.to_room_event()], txn_id: general_purpose::URL_SAFE_NO_PAD - .encode(utils::calculate_hash(&[pdu.event_id.as_bytes()])) + .encode(sha256::hash(pdu.event_id.as_bytes())) .into(), ephemeral: Vec::new(), to_device: Vec::new(), diff --git a/src/core/utils/hash.rs b/src/core/utils/hash.rs index 5a3664cb..c12d4f66 100644 --- a/src/core/utils/hash.rs +++ b/src/core/utils/hash.rs @@ -1,13 +1,10 @@ mod argon; -mod sha256; +pub mod sha256; use crate::Result; -pub fn password(password: &str) -> Result { argon::password(password) } - -pub fn verify_password(password: &str, password_hash: &str) -> Result<()> { +pub fn verify_password(password: &str, password_hash: &str) -> Result { argon::verify_password(password, password_hash) } -#[must_use] -pub fn calculate_hash(keys: &[&[u8]]) -> Vec { sha256::hash(keys) } +pub fn password(password: &str) -> Result { argon::password(password) } diff --git a/src/core/utils/hash/sha256.rs b/src/core/utils/hash/sha256.rs index b2e5a94c..06e210a7 100644 --- a/src/core/utils/hash/sha256.rs +++ b/src/core/utils/hash/sha256.rs @@ -1,9 +1,62 @@ -use ring::{digest, digest::SHA256}; +use ring::{ + digest, + digest::{Context, SHA256, SHA256_OUTPUT_LEN}, +}; -#[tracing::instrument(skip_all, level = "debug")] -pub(super) fn hash(keys: &[&[u8]]) -> Vec { - // We only hash the pdu's event ids, not the whole pdu - let bytes = keys.join(&0xFF); - let hash = digest::digest(&SHA256, &bytes); - hash.as_ref().to_owned() +pub type Digest = [u8; SHA256_OUTPUT_LEN]; + +/// Sha256 hash (input gather joined by 0xFF bytes) +#[must_use] +#[tracing::instrument(skip(inputs), level = "trace")] +pub fn delimited<'a, T, I>(mut inputs: I) -> Digest +where + I: Iterator + 'a, + T: AsRef<[u8]> + 'a, +{ + let mut ctx = Context::new(&SHA256); + if let Some(input) = inputs.next() { + ctx.update(input.as_ref()); + for input in inputs { + ctx.update(b"\xFF"); + ctx.update(input.as_ref()); + } + } + + ctx.finish() + .as_ref() + .try_into() + .expect("failed to return Digest buffer") +} + +/// Sha256 hash (input gather) +#[must_use] +#[tracing::instrument(skip(inputs), level = "trace")] +pub fn concat<'a, T, I>(inputs: I) -> Digest +where + I: Iterator + 'a, + T: AsRef<[u8]> + 'a, +{ + inputs + .fold(Context::new(&SHA256), |mut ctx, input| { + ctx.update(input.as_ref()); + ctx + }) + .finish() + .as_ref() + .try_into() + .expect("failed to return Digest buffer") +} + +/// Sha256 hash +#[inline] +#[must_use] +#[tracing::instrument(skip(input), level = "trace")] +pub fn hash(input: T) -> Digest +where + T: AsRef<[u8]>, +{ + digest::digest(&SHA256, input.as_ref()) + .as_ref() + .try_into() + .expect("failed to return Digest buffer") } diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index b8640f3a..18c2dd6f 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -28,7 +28,7 @@ pub use self::{ bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}, debug::slice_truncated as debug_slice_truncated, future::TryExtExt as TryFutureExtExt, - hash::calculate_hash, + hash::sha256::delimited as calculate_hash, html::Escape as HtmlEscape, json::{deserialize_from_str, to_canonical_object}, math::clamp, diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 7d8200f0..29ffedfc 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -157,12 +157,7 @@ impl Service { let previous_shortstatehash = self.get_room_shortstatehash(room_id).await; - let state_hash = calculate_hash( - &state_ids_compressed - .iter() - .map(|s| &s[..]) - .collect::>(), - ); + let state_hash = calculate_hash(state_ids_compressed.iter().map(|s| &s[..])); let (shortstatehash, already_existed) = self .services diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index f0c851de..0466fb12 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -352,12 +352,7 @@ impl Service { .await .ok(); - let state_hash = utils::calculate_hash( - &new_state_ids_compressed - .iter() - .map(|bytes| &bytes[..]) - .collect::>(), - ); + let state_hash = utils::calculate_hash(new_state_ids_compressed.iter().map(|bytes| &bytes[..])); let (new_shortstatehash, already_existed) = self .services diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index f5d87504..ee818289 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -539,16 +539,13 @@ impl Service { } } - let txn_id = &*general_purpose::URL_SAFE_NO_PAD.encode(calculate_hash( - &events - .iter() - .map(|e| match e { - SendingEvent::Edu(b) => &**b, - SendingEvent::Pdu(b) => b.as_ref(), - SendingEvent::Flush => &[], - }) - .collect::>(), - )); + let txn_hash = calculate_hash(events.iter().filter_map(|e| match e { + SendingEvent::Edu(b) => Some(&**b), + SendingEvent::Pdu(b) => Some(b.as_ref()), + SendingEvent::Flush => None, + })); + + let txn_id = &*general_purpose::URL_SAFE_NO_PAD.encode(txn_hash); //debug_assert!(pdu_jsons.len() + edu_jsons.len() > 0, "sending empty // transaction"); @@ -664,23 +661,21 @@ impl Service { //debug_assert!(pdu_jsons.len() + edu_jsons.len() > 0, "sending empty // transaction"); - let transaction_id = &*general_purpose::URL_SAFE_NO_PAD.encode(calculate_hash( - &events - .iter() - .map(|e| match e { - SendingEvent::Edu(b) => &**b, - SendingEvent::Pdu(b) => b.as_ref(), - SendingEvent::Flush => &[], - }) - .collect::>(), - )); + + let txn_hash = calculate_hash(events.iter().filter_map(|e| match e { + SendingEvent::Edu(b) => Some(&**b), + SendingEvent::Pdu(b) => Some(b.as_ref()), + SendingEvent::Flush => None, + })); + + let txn_id = &*general_purpose::URL_SAFE_NO_PAD.encode(txn_hash); let request = send_transaction_message::v1::Request { origin: self.server.config.server_name.clone(), pdus: pdu_jsons, edus: edu_jsons, origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - transaction_id: transaction_id.into(), + transaction_id: txn_id.into(), }; let client = &self.services.client.sender; @@ -692,7 +687,7 @@ impl Service { .iter() .filter(|(_, res)| res.is_err()) .for_each( - |(pdu_id, res)| warn!(%transaction_id, %server, "error sending PDU {pdu_id} to remote server: {res:?}"), + |(pdu_id, res)| warn!(%txn_id, %server, "error sending PDU {pdu_id} to remote server: {res:?}"), ); }) .map(|_| dest.clone()) From cd2c473bfe627389f7e24f822ca4a19e696ca555 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 21:00:26 -0500 Subject: [PATCH 1072/2091] add missing fix_referencedevents_missing_sep key on fresh db creations Signed-off-by: strawberry --- src/service/migrations.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/service/migrations.rs b/src/service/migrations.rs index 4c821fa3..126d3c7e 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -68,6 +68,7 @@ async fn fresh(services: &Services) -> Result<()> { db["global"].insert(b"feat_sha256_media", []); db["global"].insert(b"fix_bad_double_separator_in_state_cache", []); db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", []); + db["global"].insert(b"fix_referencedevents_missing_sep", []); // Create the admin room and server user on first run crate::admin::create_admin_room(services).boxed().await?; From ead9d667970f77c6d4e1c9747e607d7d711a57e0 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 21:28:08 -0500 Subject: [PATCH 1073/2091] send the actual unsupported room version in join errors Signed-off-by: strawberry --- src/api/client/membership.rs | 69 ++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 31 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index c61185a7..9478e383 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -702,18 +702,20 @@ async fn join_room_by_id_helper_remote( info!("make_join finished"); - let room_version_id = match make_join_response.room_version { - Some(room_version) - if services - .globals - .supported_room_versions() - .contains(&room_version) => - { - room_version - }, - _ => return Err!(BadServerResponse("Room version is not supported")), + let Some(room_version_id) = make_join_response.room_version else { + return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); }; + if !services + .globals + .supported_room_versions() + .contains(&room_version_id) + { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } + let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) .map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?; @@ -1116,17 +1118,20 @@ async fn join_room_by_id_helper_local( warn!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); let (make_join_response, remote_server) = make_join_request(services, sender_user, room_id, servers).await?; - let room_version_id = match make_join_response.room_version { - Some(room_version_id) - if services - .globals - .supported_room_versions() - .contains(&room_version_id) => - { - room_version_id - }, - _ => return Err!(BadServerResponse("Room version is not supported")), + let Some(room_version_id) = make_join_response.room_version else { + return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); }; + + if !services + .globals + .supported_room_versions() + .contains(&room_version_id) + { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } + let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) .map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?; let join_authorized_via_users_server = join_event_stub @@ -1274,7 +1279,7 @@ async fn make_join_request( if incompatible_room_version_count > 15 { info!( "15 servers have responded with M_INCOMPATIBLE_ROOM_VERSION or M_UNSUPPORTED_ROOM_VERSION, \ - assuming that conduwuit does not support the room {room_id}: {e}" + assuming that conduwuit does not support the room version {room_id}: {e}" ); make_join_response_and_server = Err!(BadServerResponse("Room version is not supported by Conduwuit")); return make_join_response_and_server; @@ -1607,18 +1612,20 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room let (make_leave_response, remote_server) = make_leave_response_and_server?; - let room_version_id = match make_leave_response.room_version { - Some(version) - if services - .globals - .supported_room_versions() - .contains(&version) => - { - version - }, - _ => return Err!(BadServerResponse("Room version is not supported")), + let Some(room_version_id) = make_leave_response.room_version else { + return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); }; + if !services + .globals + .supported_room_versions() + .contains(&room_version_id) + { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } + let mut leave_event_stub = serde_json::from_str::(make_leave_response.event.get()) .map_err(|e| err!(BadServerResponse("Invalid make_leave event json received from server: {e:?}")))?; From 63193840729ab1dea026514d807aa39a98b3f1b1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 21:40:13 -0500 Subject: [PATCH 1074/2091] implement `GET /_matrix/client/v3/pushrules/global/` Signed-off-by: strawberry --- src/api/client/push.rs | 69 +++++++++++++++++++++++++++++++++++++++++- src/api/router.rs | 1 + 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/src/api/client/push.rs b/src/api/client/push.rs index de280b32..97243ab4 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -5,7 +5,7 @@ use ruma::{ error::ErrorKind, push::{ delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, get_pushrule_enabled, get_pushrules_all, - set_pusher, set_pushrule, set_pushrule_actions, set_pushrule_enabled, + get_pushrules_global_scope, set_pusher, set_pushrule, set_pushrule_actions, set_pushrule_enabled, }, }, events::{ @@ -67,6 +67,73 @@ pub(crate) async fn get_pushrules_all_route( }) } +/// # `GET /_matrix/client/r0/pushrules/global/` +/// +/// Retrieves the push rules event for this user. +/// +/// This appears to be the exact same as `GET /_matrix/client/r0/pushrules/`. +pub(crate) async fn get_pushrules_global_route( + State(services): State, body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + let Some(content_value) = services + .account_data + .get_global::(sender_user, GlobalAccountDataEventType::PushRules) + .await + .ok() + .and_then(|event| event.get("content").cloned()) + .filter(CanonicalJsonValue::is_object) + else { + // user somehow has non-existent push rule event. recreate it and return server + // default silently + services + .account_data + .update( + None, + sender_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(PushRulesEvent { + content: PushRulesEventContent { + global: Ruleset::server_default(sender_user), + }, + }) + .expect("to json always works"), + ) + .await?; + + return Ok(get_pushrules_global_scope::v3::Response { + global: Ruleset::server_default(sender_user), + }); + }; + + let account_data_content = serde_json::from_value::(content_value.into()) + .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; + + let mut global_ruleset = account_data_content.global; + + // remove old deprecated mentions push rules as per MSC4210 + #[allow(deprecated)] + { + use ruma::push::RuleKind::*; + + global_ruleset + .remove(Override, PredefinedOverrideRuleId::ContainsDisplayName) + .ok(); + global_ruleset + .remove(Override, PredefinedOverrideRuleId::RoomNotif) + .ok(); + + global_ruleset + .remove(Content, PredefinedContentRuleId::ContainsUserName) + .ok(); + }; + + Ok(get_pushrules_global_scope::v3::Response { + global: global_ruleset, + }) +} + /// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}` /// /// Retrieves a single specified push rule for this user. diff --git a/src/api/router.rs b/src/api/router.rs index ddd91d11..1df4342f 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -45,6 +45,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(&client::check_registration_token_validity) .ruma_route(&client::get_capabilities_route) .ruma_route(&client::get_pushrules_all_route) + .ruma_route(&client::get_pushrules_global_route) .ruma_route(&client::set_pushrule_route) .ruma_route(&client::get_pushrule_route) .ruma_route(&client::set_pushrule_enabled_route) From b92b4e043c03dfcc6f0163af92a219251b745351 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 22:16:11 -0500 Subject: [PATCH 1075/2091] drop hyper-util back down to 0.1.8 due to DNS issues Signed-off-by: strawberry --- Cargo.lock | 17 +++++++++-------- Cargo.toml | 3 ++- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b56005ff..3a95f83a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -164,9 +164,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdd82dba44d209fddb11c190e0a94b78651f95299598e472215667417a03ff1d" +checksum = "fe7c2840b66236045acd2607d5866e274380afd87ef99d6226e961e2cb47df45" dependencies = [ "aws-lc-sys", "mirai-annotations", @@ -176,9 +176,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df7a4168111d7eb622a31b214057b8509c0a7e1794f44c546d742330dc793972" +checksum = "ad3a619a9de81e1d7de1f1186dcba4506ed661a0e483d84410fdef0ee87b2f96" dependencies = [ "bindgen", "cc", @@ -1739,9 +1739,9 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.5.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" dependencies = [ "hyper", "hyper-util", @@ -1752,9 +1752,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -1765,6 +1765,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", + "tower 0.4.13", "tower-service", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index 814a435b..68c87c57 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -264,7 +264,8 @@ features = [ ] [workspace.dependencies.hyper-util] -version = "0.1.10" +# hyper-util >=0.1.9 seems to have DNS issues +version = "=0.1.8" default-features = false features = [ "server-auto", From 7f96b2f92ad11e9a357c5c0607f7be403f7c8a85 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 23:18:12 -0500 Subject: [PATCH 1076/2091] nix: remove libllvm, libgcc, and llvm from OCI images as well aarch64 OCI images love llvm?? Signed-off-by: strawberry --- nix/pkgs/main/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 1088b03c..d11c36cc 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -176,7 +176,7 @@ commonAttrs = { # # postInstall = with pkgsBuildHost; '' - find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${rustc.unwrapped} -t ${rustc} -t ${libidn2} -t ${libunistring} '{}' + + find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${libgcc} -t ${llvm} -t ${libllvm} -t ${rustc.unwrapped} -t ${rustc} -t ${libidn2} -t ${libunistring} '{}' + ''; }; in From 8f140485287adb8534d299ce553d15994ba1fab7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 15 Nov 2024 23:48:55 -0500 Subject: [PATCH 1077/2091] ci: free up a bit of runner space safely (again) Signed-off-by: strawberry --- .github/workflows/ci.yml | 9 +++++++++ .github/workflows/documentation.yml | 10 ++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96a1188b..9385c5e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,6 +71,15 @@ jobs: run: | sudo apt install liburing-dev -y + - name: Free up a bit of runner space + run: | + set +o pipefail + sudo docker image prune --all --force || true + sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true + sudo apt clean + sudo rm -v -rf /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/heroku + set -o pipefail + - name: Sync repository uses: actions/checkout@v4 diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ea720c43..17b1f9c1 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -50,8 +50,14 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main + - name: Free up a bit of runner space + run: | + set +o pipefail + sudo docker image prune --all --force || true + sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true + sudo apt clean + sudo rm -v -rf /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/heroku + set -o pipefail - name: Sync repository uses: actions/checkout@v4 From 2b2793fac6b2384217e70424dbd16752060dec14 Mon Sep 17 00:00:00 2001 From: Tamara Schmitz Date: Mon, 4 Nov 2024 14:27:43 +0100 Subject: [PATCH 1078/2091] docs: add note about the nixos service defaulting to sqlite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: June 🍓🦴 --- docs/deploying/nixos.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index 9147db7f..61fb3916 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -39,6 +39,15 @@ The `flake.nix` and `default.nix` do not currently provide a NixOS module (contr welcome!), so [`services.matrix-conduit`][module] from Nixpkgs can be used to configure conduwuit. +### Conduit NixOS Config Module and SQLite + +Beware! The [`services.matrix-conduit`][module] module defaults to SQLite as a database backend. +Conduwuit dropped SQLite support in favor of exclusively supporting the much faster RocksDB. +Make sure that you are using the RocksDB backend before migrating! + +There is a [tool to migrate a Conduit SQLite database to +RocksDB](https://github.com/ShadowJonathan/conduit_toolbox/). + If you want to run the latest code, you should get conduwuit from the `flake.nix` or `default.nix` and set [`services.matrix-conduit.package`][package] appropriately to use conduwuit instead of Conduit. From 85a6d8fc6bd25adaaebb395bc925dea043e3852a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 16 Nov 2024 01:57:31 -0500 Subject: [PATCH 1079/2091] ci: fix github pages publish check Signed-off-by: strawberry --- .github/workflows/documentation.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 17b1f9c1..20b1e4c6 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -63,7 +63,7 @@ jobs: uses: actions/checkout@v4 - name: Setup GitHub Pages - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') && (github.event.pull_request.user.login == 'girlbossceo') + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') uses: actions/configure-pages@v5 - uses: nixbuild/nix-quick-install-action@master @@ -145,12 +145,12 @@ jobs: compression-level: 0 - name: Upload generated documentation (book) as GitHub Pages artifact - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') && (github.event.pull_request.user.login == 'girlbossceo') + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') uses: actions/upload-pages-artifact@v3 with: path: public - name: Deploy to GitHub Pages - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') && (github.event.pull_request.user.login == 'girlbossceo') + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') id: deployment uses: actions/deploy-pages@v4 From 26bcc7e3127129e845e97e95913430df6f5b37fc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 17 Nov 2024 00:13:20 +0000 Subject: [PATCH 1080/2091] fix default stateinfo cache size Signed-off-by: Jason Volk --- src/core/config/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index cb9d087b..355881b3 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -2048,7 +2048,7 @@ fn default_server_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(50 fn default_user_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(1000) } -fn default_stateinfo_cache_capacity() -> u32 { parallelism_scaled_u32(1000) } +fn default_stateinfo_cache_capacity() -> u32 { parallelism_scaled_u32(100) } fn default_roomid_spacehierarchy_cache_capacity() -> u32 { parallelism_scaled_u32(1000) } From a05dc0310056c5efc5789158d643951964f8fce4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 17 Nov 2024 07:46:36 +0000 Subject: [PATCH 1081/2091] use debug_warn for backfill event evals Signed-off-by: Jason Volk --- src/service/rooms/timeline/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 59fc8e93..a3fc6a0b 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -9,7 +9,7 @@ use std::{ }; use conduit::{ - debug, err, error, implement, info, + debug, debug_warn, err, error, implement, info, pdu::{EventHash, PduBuilder, PduCount, PduEvent}, utils, utils::{stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, @@ -1128,7 +1128,7 @@ impl Service { Ok(response) => { for pdu in response.pdus { if let Err(e) = self.backfill_pdu(backfill_server, pdu).boxed().await { - warn!("Failed to add backfilled pdu in room {room_id}: {e}"); + debug_warn!("Failed to add backfilled pdu in room {room_id}: {e}"); } } return Ok(()); From 90106c4c33946cfda514e9a2cdf75f36605c08bd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 16 Nov 2024 23:13:27 +0000 Subject: [PATCH 1082/2091] streamline batch insertions Signed-off-by: Jason Volk --- src/database/map/insert.rs | 2 +- src/service/rooms/search/mod.rs | 6 ++- src/service/sending/data.rs | 84 ++++++++++++++++++++------------- src/service/sending/mod.rs | 16 +++---- src/service/sending/sender.rs | 4 +- 5 files changed, 63 insertions(+), 49 deletions(-) diff --git a/src/database/map/insert.rs b/src/database/map/insert.rs index 39a0c422..b8b08b34 100644 --- a/src/database/map/insert.rs +++ b/src/database/map/insert.rs @@ -203,7 +203,7 @@ where #[tracing::instrument(skip(self, iter), fields(%self), level = "trace")] pub fn insert_batch<'a, I, K, V>(&'a self, iter: I) where - I: Iterator + Send + Debug, + I: Iterator + Send + Debug, K: AsRef<[u8]> + Sized + Debug + 'a, V: AsRef<[u8]> + Sized + 'a, { diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 1af37d9e..d59d1d11 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -73,11 +73,13 @@ pub fn index_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_bod key.extend_from_slice(word.as_bytes()); key.push(0xFF); key.extend_from_slice(pdu_id.as_ref()); // TODO: currently we save the room id a second time here - (key, Vec::::new()) + key }) .collect::>(); - self.db.tokenids.insert_batch(batch.iter()); + self.db + .tokenids + .insert_batch(batch.iter().map(|k| (k.as_slice(), &[]))); } #[implement(Service)] diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index cd25776a..ca7ca19a 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -1,7 +1,7 @@ -use std::sync::Arc; +use std::{fmt::Debug, sync::Arc}; use conduit::{ - utils, + at, utils, utils::{stream::TryIgnore, ReadyExt}, Error, Result, }; @@ -69,20 +69,22 @@ impl Data { .await; } - pub(super) fn mark_as_active(&self, events: &[QueueItem]) { - for (key, e) in events { - if key.is_empty() { - continue; - } + pub(super) fn mark_as_active<'a, I>(&self, events: I) + where + I: Iterator, + { + events + .filter(|(key, _)| !key.is_empty()) + .for_each(|(key, val)| { + let val = if let SendingEvent::Edu(val) = &val { + &**val + } else { + &[] + }; - let value = if let SendingEvent::Edu(value) = &e { - &**value - } else { - &[] - }; - self.servercurrentevent_data.insert(key, value); - self.servernameevent_data.remove(key); - } + self.servercurrentevent_data.insert(key, val); + self.servernameevent_data.remove(key); + }); } #[inline] @@ -110,26 +112,40 @@ impl Data { }) } - pub(super) fn queue_requests(&self, requests: &[(&SendingEvent, &Destination)]) -> Vec> { - let mut batch = Vec::new(); - let mut keys = Vec::new(); - for (event, destination) in requests { - let mut key = destination.get_prefix(); - if let SendingEvent::Pdu(value) = event { - key.extend(value.as_ref()); - } else { - key.extend(&self.services.globals.next_count().unwrap().to_be_bytes()); - } - let value = if let SendingEvent::Edu(value) = &event { - &**value - } else { - &[] - }; - batch.push((key.clone(), value.to_owned())); - keys.push(key); - } + pub(super) fn queue_requests<'a, I>(&self, requests: I) -> Vec> + where + I: Iterator + Clone + Debug + Send, + { + let keys: Vec<_> = requests + .clone() + .map(|(event, dest)| { + let mut key = dest.get_prefix(); + if let SendingEvent::Pdu(value) = event { + key.extend(value.as_ref()); + } else { + let count = self.services.globals.next_count().unwrap(); + key.extend(&count.to_be_bytes()); + } + + key + }) + .collect(); + + self.servernameevent_data.insert_batch( + keys.iter() + .map(Vec::as_slice) + .zip(requests.map(at!(0))) + .map(|(key, event)| { + let value = if let SendingEvent::Edu(value) = &event { + &**value + } else { + &[] + }; + + (key, value) + }), + ); - self.servernameevent_data.insert_batch(batch.iter()); keys } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 77997f69..5a070306 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -4,7 +4,7 @@ mod dest; mod send; mod sender; -use std::{fmt::Debug, sync::Arc}; +use std::{fmt::Debug, iter::once, sync::Arc}; use async_trait::async_trait; use conduit::{ @@ -117,7 +117,7 @@ impl Service { let dest = Destination::Push(user.to_owned(), pushkey); let event = SendingEvent::Pdu(*pdu_id); let _cork = self.db.db.cork(); - let keys = self.db.queue_requests(&[(&event, &dest)]); + let keys = self.db.queue_requests(once((&event, &dest))); self.dispatch(Msg { dest, event, @@ -130,7 +130,7 @@ impl Service { let dest = Destination::Appservice(appservice_id); let event = SendingEvent::Pdu(pdu_id); let _cork = self.db.db.cork(); - let keys = self.db.queue_requests(&[(&event, &dest)]); + let keys = self.db.queue_requests(once((&event, &dest))); self.dispatch(Msg { dest, event, @@ -160,9 +160,7 @@ impl Service { .collect::>() .await; - let keys = self - .db - .queue_requests(&requests.iter().map(|(o, e)| (e, o)).collect::>()); + let keys = self.db.queue_requests(requests.iter().map(|(o, e)| (e, o))); for ((dest, event), queue_id) in requests.into_iter().zip(keys) { self.dispatch(Msg { @@ -180,7 +178,7 @@ impl Service { let dest = Destination::Normal(server.to_owned()); let event = SendingEvent::Edu(serialized); let _cork = self.db.db.cork(); - let keys = self.db.queue_requests(&[(&event, &dest)]); + let keys = self.db.queue_requests(once((&event, &dest))); self.dispatch(Msg { dest, event, @@ -210,9 +208,7 @@ impl Service { .collect::>() .await; - let keys = self - .db - .queue_requests(&requests.iter().map(|(o, e)| (e, o)).collect::>()); + let keys = self.db.queue_requests(requests.iter().map(|(o, e)| (e, o))); for ((dest, event), queue_id) in requests.into_iter().zip(keys) { self.dispatch(Msg { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index ee818289..0a0aae39 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -118,7 +118,7 @@ impl Service { // Insert any pdus we found if !new_events.is_empty() { - self.db.mark_as_active(&new_events); + self.db.mark_as_active(new_events.iter()); let new_events_vec = new_events.into_iter().map(|(_, event)| event).collect(); futures.push(self.send_events(dest.clone(), new_events_vec).boxed()); @@ -213,7 +213,7 @@ impl Service { // Compose the next transaction let _cork = self.db.db.cork(); if !new_events.is_empty() { - self.db.mark_as_active(&new_events); + self.db.mark_as_active(new_events.iter()); for (_, e) in new_events { events.push(e); } From 8fedc358e063cfc90a4f7598107d0e04685bb10a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 15 Nov 2024 22:23:42 +0000 Subject: [PATCH 1083/2091] typename additional shortids cleanup/split state_compressor load Signed-off-by: Jason Volk --- src/service/rooms/state/mod.rs | 19 ++++-- src/service/rooms/state_compressor/mod.rs | 83 ++++++++++++----------- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 29ffedfc..622b8325 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -23,8 +23,14 @@ use ruma::{ EventId, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, UserId, }; -use super::state_compressor::CompressedStateEvent; -use crate::{globals, rooms, Dep}; +use crate::{ + globals, rooms, + rooms::{ + short::{ShortEventId, ShortStateHash}, + state_compressor::CompressedStateEvent, + }, + Dep, +}; pub struct Service { pub mutex: RoomMutexMap, @@ -146,8 +152,9 @@ impl Service { #[tracing::instrument(skip(self, state_ids_compressed), level = "debug")] pub async fn set_event_state( &self, event_id: &EventId, room_id: &RoomId, state_ids_compressed: Arc>, - ) -> Result { - const BUFSIZE: usize = size_of::(); + ) -> Result { + const KEY_LEN: usize = size_of::(); + const VAL_LEN: usize = size_of::(); let shorteventid = self .services @@ -202,7 +209,7 @@ impl Service { self.db .shorteventid_shortstatehash - .aput::(shorteventid, shortstatehash); + .aput::(shorteventid, shortstatehash); Ok(shortstatehash) } @@ -343,7 +350,7 @@ impl Service { .map_err(|e| err!(Request(NotFound("No create event found: {e:?}")))) } - pub async fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result { + pub async fn get_room_shortstatehash(&self, room_id: &RoomId) -> Result { self.db .roomid_shortstatehash .get(room_id) diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 0466fb12..f9db6f67 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -89,9 +89,10 @@ impl crate::Service for Service { .map(at!(1)) .flat_map(|vec| vec.iter()) .fold(HashMap::new(), |mut ents, ssi| { - ents.insert(Arc::as_ptr(&ssi.added), compressed_state_size(&ssi.added)); - ents.insert(Arc::as_ptr(&ssi.removed), compressed_state_size(&ssi.removed)); - ents.insert(Arc::as_ptr(&ssi.full_state), compressed_state_size(&ssi.full_state)); + for cs in &[&ssi.added, &ssi.removed, &ssi.full_state] { + ents.insert(Arc::as_ptr(cs), compressed_state_size(cs)); + } + ents }); @@ -125,51 +126,57 @@ impl Service { return Ok(r.clone()); } - let StateDiff { - parent, - added, - removed, - } = self.get_statediff(shortstatehash).await?; - - let response = if let Some(parent) = parent { - let mut response = Box::pin(self.load_shortstatehash_info(parent)).await?; - let mut state = (*response.last().expect("at least one response").full_state).clone(); - state.extend(added.iter().copied()); - let removed = (*removed).clone(); - for r in &removed { - state.remove(r); - } - - response.push(ShortStateInfo { - shortstatehash, - full_state: Arc::new(state), - added, - removed: Arc::new(removed), - }); - - response - } else { - vec![ShortStateInfo { - shortstatehash, - full_state: added.clone(), - added, - removed, - }] - }; + let stack = self.new_shortstatehash_info(shortstatehash).await?; debug!( - ?parent, ?shortstatehash, - vec_len = %response.len(), + len = %stack.len(), "cache update" ); self.stateinfo_cache .lock() .expect("locked") - .insert(shortstatehash, response.clone()); + .insert(shortstatehash, stack.clone()); - Ok(response) + Ok(stack) + } + + async fn new_shortstatehash_info(&self, shortstatehash: ShortStateHash) -> Result { + let StateDiff { + parent, + added, + removed, + } = self.get_statediff(shortstatehash).await?; + + let Some(parent) = parent else { + return Ok(vec![ShortStateInfo { + shortstatehash, + full_state: added.clone(), + added, + removed, + }]); + }; + + let mut stack = Box::pin(self.load_shortstatehash_info(parent)).await?; + let top = stack.last().expect("at least one frame"); + + let mut full_state = (*top.full_state).clone(); + full_state.extend(added.iter().copied()); + + let removed = (*removed).clone(); + for r in &removed { + full_state.remove(r); + } + + stack.push(ShortStateInfo { + shortstatehash, + added, + removed: Arc::new(removed), + full_state: Arc::new(full_state), + }); + + Ok(stack) } pub async fn compress_state_event(&self, shortstatekey: ShortStateKey, event_id: &EventId) -> CompressedStateEvent { From 7680d1bd5e856630e670a739b912c4d183061911 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 19 Nov 2024 08:14:02 +0000 Subject: [PATCH 1084/2091] replace yields point with consume_budget Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 1 - src/database/map/get.rs | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index ea487d8e..77ba4c3f 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -757,7 +757,6 @@ async fn load_joined_room( }; delta_state_events.push(pdu); - tokio::task::yield_now().await; } } } diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 2f7df031..a3c6c492 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -2,9 +2,10 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write}; use arrayvec::ArrayVec; use conduit::{err, implement, utils::IterStream, Result}; -use futures::{future::ready, Stream}; +use futures::{FutureExt, Stream}; use rocksdb::DBPinnableSlice; use serde::Serialize; +use tokio::task; use crate::{ser, util, Handle}; @@ -55,7 +56,8 @@ pub fn get(&self, key: &K) -> impl Future>> + Send where K: AsRef<[u8]> + ?Sized + Debug, { - ready(self.get_blocking(key)) + let result = self.get_blocking(key); + task::consume_budget().map(move |()| result) } /// Fetch a value from the database into cache, returning a reference-handle. From 411c60009da6858f40e9f623c9c2420c09d346d8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 17 Nov 2024 03:57:21 +0000 Subject: [PATCH 1085/2091] enrich state iteration interface Signed-off-by: Jason Volk --- src/service/rooms/state/mod.rs | 32 ++--- src/service/rooms/state_accessor/data.rs | 155 ++++++++++++---------- src/service/rooms/state_accessor/mod.rs | 6 + src/service/rooms/state_compressor/mod.rs | 31 ++--- 4 files changed, 117 insertions(+), 107 deletions(-) diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 622b8325..9d702cd7 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -27,7 +27,7 @@ use crate::{ globals, rooms, rooms::{ short::{ShortEventId, ShortStateHash}, - state_compressor::CompressedStateEvent, + state_compressor::{parse_compressed_state_event, CompressedStateEvent}, }, Dep, }; @@ -98,12 +98,12 @@ impl Service { _statediffremoved: Arc>, state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex ) -> Result { - let event_ids = statediffnew.iter().stream().filter_map(|new| { - self.services - .state_compressor - .parse_compressed_state_event(*new) - .map_ok_or_else(|_| None, |(_, event_id)| Some(event_id)) - }); + let event_ids = statediffnew + .iter() + .stream() + .map(|&new| parse_compressed_state_event(new).1) + .then(|shorteventid| self.services.short.get_eventid_from_short(shorteventid)) + .ignore_err(); pin_mut!(event_ids); while let Some(event_id) = event_ids.next().await { @@ -428,17 +428,19 @@ impl Service { .full_state; let mut ret = HashMap::new(); - for compressed in full_state.iter() { - let Ok((shortstatekey, event_id)) = self - .services - .state_compressor - .parse_compressed_state_event(*compressed) - .await - else { + for &compressed in full_state.iter() { + let (shortstatekey, shorteventid) = parse_compressed_state_event(compressed); + + let Some((ty, state_key)) = sauthevents.remove(&shortstatekey) else { continue; }; - let Some((ty, state_key)) = sauthevents.remove(&shortstatekey) else { + let Ok(event_id) = self + .services + .short + .get_eventid_from_short(shorteventid) + .await + else { continue; }; diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 06cd648c..8df0d8b0 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,11 +1,22 @@ use std::{collections::HashMap, sync::Arc}; -use conduit::{err, PduEvent, Result}; +use conduit::{ + err, + utils::{future::TryExtExt, IterStream}, + PduEvent, Result, +}; use database::{Deserialized, Map}; -use futures::TryFutureExt; +use futures::{StreamExt, TryFutureExt}; use ruma::{events::StateEventType, EventId, RoomId}; -use crate::{rooms, rooms::short::ShortStateHash, Dep}; +use crate::{ + rooms, + rooms::{ + short::{ShortEventId, ShortStateHash, ShortStateKey}, + state_compressor::parse_compressed_state_event, + }, + Dep, +}; pub(super) struct Data { eventid_shorteventid: Arc, @@ -35,9 +46,55 @@ impl Data { } } - #[allow(unused_qualifications)] // async traits + pub(super) async fn state_full( + &self, shortstatehash: ShortStateHash, + ) -> Result>> { + Ok(self + .state_full_pdus(shortstatehash) + .await? + .into_iter() + .filter_map(|pdu| Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu))) + .collect()) + } + + pub(super) async fn state_full_pdus(&self, shortstatehash: ShortStateHash) -> Result>> { + Ok(self + .state_full_shortids(shortstatehash) + .await? + .iter() + .stream() + .filter_map(|(_, shorteventid)| { + self.services + .short + .get_eventid_from_short(*shorteventid) + .ok() + }) + .filter_map(|eventid| async move { self.services.timeline.get_pdu(&eventid).await.ok() }) + .collect() + .await) + } + pub(super) async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result>> { - let full_state = self + Ok(self + .state_full_shortids(shortstatehash) + .await? + .iter() + .stream() + .filter_map(|(shortstatekey, shorteventid)| { + self.services + .short + .get_eventid_from_short(*shorteventid) + .map_ok(move |eventid| (*shortstatekey, eventid)) + .ok() + }) + .collect() + .await) + } + + pub(super) async fn state_full_shortids( + &self, shortstatehash: ShortStateHash, + ) -> Result> { + Ok(self .services .state_compressor .load_shortstatehash_info(shortstatehash) @@ -45,63 +102,11 @@ impl Data { .map_err(|e| err!(Database("Missing state IDs: {e}")))? .pop() .expect("there is always one layer") - .full_state; - - let mut result = HashMap::new(); - let mut i: u8 = 0; - for compressed in full_state.iter() { - let parsed = self - .services - .state_compressor - .parse_compressed_state_event(*compressed) - .await?; - - result.insert(parsed.0, parsed.1); - - i = i.wrapping_add(1); - if i % 100 == 0 { - tokio::task::yield_now().await; - } - } - - Ok(result) - } - - #[allow(unused_qualifications)] // async traits - pub(super) async fn state_full( - &self, shortstatehash: ShortStateHash, - ) -> Result>> { - let full_state = self - .services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .await? - .pop() - .expect("there is always one layer") - .full_state; - - let mut result = HashMap::new(); - let mut i: u8 = 0; - for compressed in full_state.iter() { - let (_, eventid) = self - .services - .state_compressor - .parse_compressed_state_event(*compressed) - .await?; - - if let Ok(pdu) = self.services.timeline.get_pdu(&eventid).await { - if let Some(state_key) = pdu.state_key.as_ref() { - result.insert((pdu.kind.to_string().into(), state_key.clone()), pdu); - } - } - - i = i.wrapping_add(1); - if i % 100 == 0 { - tokio::task::yield_now().await; - } - } - - Ok(result) + .full_state + .iter() + .copied() + .map(parse_compressed_state_event) + .collect()) } /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). @@ -130,18 +135,11 @@ impl Data { .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) .ok_or(err!(Database("No shortstatekey in compressed state")))?; + let (_, shorteventid) = parse_compressed_state_event(*compressed); + self.services - .state_compressor - .parse_compressed_state_event(*compressed) - .map_ok(|(_, id)| id) - .map_err(|e| { - err!(Database(error!( - ?event_type, - ?state_key, - ?shortstatekey, - "Failed to parse compressed: {e:?}" - ))) - }) + .short + .get_eventid_from_short(shorteventid) .await } @@ -176,6 +174,17 @@ impl Data { .await } + /// Returns the full room state's pdus. + #[allow(unused_qualifications)] // async traits + pub(super) async fn room_state_full_pdus(&self, room_id: &RoomId) -> Result>> { + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_full_pdus(shortstatehash)) + .map_err(|e| err!(Database("Missing state pdus for {room_id:?}: {e:?}"))) + .await + } + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). pub(super) async fn room_state_get_id( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 4958c4ea..697f7236 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -301,6 +301,12 @@ impl Service { self.db.room_state_full(room_id).await } + /// Returns the full room state pdus + #[tracing::instrument(skip(self), level = "debug")] + pub async fn room_state_full_pdus(&self, room_id: &RoomId) -> Result>> { + self.db.room_state_full_pdus(room_id).await + } + /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index f9db6f67..52ad5437 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -17,7 +17,7 @@ use ruma::{EventId, RoomId}; use crate::{ rooms, - rooms::short::{ShortId, ShortStateHash, ShortStateKey}, + rooms::short::{ShortEventId, ShortId, ShortStateHash, ShortStateKey}, Dep, }; @@ -196,24 +196,6 @@ impl Service { .expect("failed to create CompressedStateEvent") } - /// Returns shortstatekey, event id - #[inline] - pub async fn parse_compressed_state_event( - &self, compressed_event: CompressedStateEvent, - ) -> Result<(ShortStateKey, Arc)> { - use utils::u64_from_u8; - - let shortstatekey = u64_from_u8(&compressed_event[0..size_of::()]); - let shorteventid = u64_from_u8(&compressed_event[size_of::()..]); - let event_id = self - .services - .short - .get_eventid_from_short(shorteventid) - .await?; - - Ok((shortstatekey, event_id)) - } - /// Creates a new shortstatehash that often is just a diff to an already /// existing shortstatehash and therefore very efficient. /// @@ -488,6 +470,17 @@ impl Service { } } +#[inline] +#[must_use] +pub fn parse_compressed_state_event(compressed_event: CompressedStateEvent) -> (ShortStateKey, ShortEventId) { + use utils::u64_from_u8; + + let shortstatekey = u64_from_u8(&compressed_event[0..size_of::()]); + let shorteventid = u64_from_u8(&compressed_event[size_of::()..]); + + (shortstatekey, shorteventid) +} + #[inline] fn compressed_state_size(compressed_state: &CompressedState) -> usize { compressed_state From e257512aa7c3a19ec2c4f695288c8de3d13663df Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 17 Nov 2024 04:45:45 +0000 Subject: [PATCH 1086/2091] relax state visibility for invited modes Signed-off-by: Jason Volk --- src/service/rooms/state_accessor/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 697f7236..89db88a6 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -287,7 +287,11 @@ impl Service { c.history_visibility }); - history_visibility == HistoryVisibility::WorldReadable + match history_visibility { + HistoryVisibility::Invited => self.services.state_cache.is_invited(user_id, room_id).await, + HistoryVisibility::WorldReadable => true, + _ => false, + } } /// Returns the state hash for this pdu. From 2f2cebe84d319608631273cef32c8236868e5baa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 12 Nov 2024 03:46:31 +0000 Subject: [PATCH 1087/2091] implement local room preview Signed-off-by: Jason Volk --- src/api/client/room/initial_sync.rs | 72 +++++++++++++++++++++++++++++ src/api/client/room/mod.rs | 3 +- src/api/router.rs | 7 +-- 3 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 src/api/client/room/initial_sync.rs diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs new file mode 100644 index 00000000..16b3a53b --- /dev/null +++ b/src/api/client/room/initial_sync.rs @@ -0,0 +1,72 @@ +use axum::extract::State; +use conduit::{at, utils::BoolExt, Err, Result}; +use futures::StreamExt; +use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response}; + +use crate::Ruma; + +const LIMIT_MAX: usize = 100; + +pub(crate) async fn room_initial_sync_route( + State(services): State, body: Ruma, +) -> Result { + let room_id = &body.room_id; + + if !services + .rooms + .state_accessor + .user_can_see_state_events(body.sender_user(), room_id) + .await + { + return Err!(Request(Forbidden("No room preview available."))); + } + + let limit = LIMIT_MAX; + let events: Vec<_> = services + .rooms + .timeline + .pdus_rev(None, room_id, None) + .await? + .take(limit) + .collect() + .await; + + let state: Vec<_> = services + .rooms + .state_accessor + .room_state_full_pdus(room_id) + .await? + .into_iter() + .map(|pdu| pdu.to_state_event()) + .collect(); + + let messages = PaginationChunk { + start: events.last().map(at!(0)).as_ref().map(ToString::to_string), + + end: events + .first() + .map(at!(0)) + .as_ref() + .map(ToString::to_string) + .unwrap_or_default(), + + chunk: events + .into_iter() + .map(at!(1)) + .map(|pdu| pdu.to_room_event()) + .collect(), + }; + + Ok(Response { + room_id: room_id.to_owned(), + account_data: None, + state: state.into(), + messages: messages.chunk.is_empty().or_some(messages), + visibility: services.rooms.directory.visibility(room_id).await.into(), + membership: services + .rooms + .state_cache + .user_membership(body.sender_user(), room_id) + .await, + }) +} diff --git a/src/api/client/room/mod.rs b/src/api/client/room/mod.rs index fa2d168f..16fcadab 100644 --- a/src/api/client/room/mod.rs +++ b/src/api/client/room/mod.rs @@ -1,9 +1,10 @@ mod aliases; mod create; mod event; +mod initial_sync; mod upgrade; pub(crate) use self::{ aliases::get_room_aliases_route, create::create_room_route, event::get_room_event_route, - upgrade::upgrade_room_route, + initial_sync::room_initial_sync_route, upgrade::upgrade_room_route, }; diff --git a/src/api/router.rs b/src/api/router.rs index 1df4342f..4bdd692d 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -183,8 +183,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(&client::well_known_support) .ruma_route(&client::well_known_client) .route("/_conduwuit/server_version", get(client::conduwuit_server_version)) - .route("/_matrix/client/r0/rooms/:room_id/initialSync", get(initial_sync)) - .route("/_matrix/client/v3/rooms/:room_id/initialSync", get(initial_sync)) + .ruma_route(&client::room_initial_sync_route) .route("/client/server.json", get(client::syncv3_client_server_json)); if config.allow_federation { @@ -285,10 +284,6 @@ async fn redirect_legacy_preview(uri: Uri) -> impl IntoResponse { Redirect::temporary(&uri) } -async fn initial_sync(_uri: Uri) -> impl IntoResponse { - err!(Request(GuestAccessForbidden("Guest access not implemented"))) -} - async fn legacy_media_disabled() -> impl IntoResponse { err!(Request(Forbidden("Unauthenticated media is disabled."))) } async fn federation_disabled() -> impl IntoResponse { err!(Request(Forbidden("Federation is disabled."))) } From 876c6e933cafd17d870ac51ff52c7a9ad052db16 Mon Sep 17 00:00:00 2001 From: nisbet-hubbard <87453615+nisbet-hubbard@users.noreply.github.com> Date: Sun, 17 Nov 2024 11:12:57 +0800 Subject: [PATCH 1088/2091] A minimal caveat --- docs/deploying/generic.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index f0b85a25..9eafbc46 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -54,13 +54,13 @@ While conduwuit can run as any user it is better to use dedicated users for different services. This also allows you to make sure that the file permissions are correctly set up. -In Debian or Fedora/RHEL, you can use this command to create a conduwuit user: +In Debian, you can use this command to create a conduwuit user: ```bash sudo adduser --system conduwuit --group --disabled-login --no-create-home ``` -For distros without `adduser`: +For distros without `adduser` (or where it's a symlink to `useradd`): ```bash sudo useradd -r --shell /usr/bin/nologin --no-create-home conduwuit @@ -142,8 +142,8 @@ If using Nginx, you need to give conduwuit the request URI using `$request_uri`, - `proxy_pass http://127.0.0.1:6167$request_uri;` - `proxy_pass http://127.0.0.1:6167;` -Nginx users may need to set `proxy_buffering off;` if there are issues with -uploading media like images. This is due to Nginx storing the entire POST content in-memory (`/tmp`) and running out of memory if on low memory hardware. +Nginx users need to increase `client_max_body_size` (default is 1M) to match +`max_request_size` defined in conduwuit.toml. You will need to reverse proxy everything under following routes: - `/_matrix/` - core Matrix C-S and S-S APIs From ee3c58f78f88464c1ba8b01812c8bd7de7f1b29a Mon Sep 17 00:00:00 2001 From: emily Date: Wed, 20 Nov 2024 16:29:42 +0100 Subject: [PATCH 1089/2091] docs: add workaround to use unix sockets with the nixos module --- docs/deploying/nixos.md | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index 61fb3916..0372228d 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -55,15 +55,31 @@ appropriately to use conduwuit instead of Conduit. ### UNIX sockets Due to the lack of a conduwuit NixOS module, when using the `services.matrix-conduit` module -it is not possible to use UNIX sockets. This is because the UNIX socket option does not exist -in Conduit, and their module forces listening on `[::1]:6167` by default if unspecified. +a workaround like the one below is necessary to use UNIX sockets. This is because the UNIX +socket option does not exist in Conduit, and the module forcibly sets the `address` and +`port` config options. + +```nix +options.services.matrix-conduit.settings = lib.mkOption { + apply = old: old // ( + if (old.global ? "unix_socket_path") + then { global = builtins.removeAttrs old.global [ "address" "port" ]; } + else { } + ); +}; + +``` Additionally, the [`matrix-conduit` systemd unit][systemd-unit] in the module does not allow the `AF_UNIX` socket address family in their systemd unit's `RestrictAddressFamilies=` which -disallows the namespace from accessing or creating UNIX sockets. +disallows the namespace from accessing or creating UNIX sockets and has to be enabled like so: -There is no known workaround these. A conduwuit NixOS configuration module must be developed and -published by the community. +```nix +systemd.services.conduit.serviceConfig.RestrictAddressFamilies = [ "AF_UNIX" ]; +``` + +Even though those workarounds are feasible a conduwuit NixOS configuration module, developed and +published by the community, would be appreciated. ### jemalloc and hardened profile From 336de49e6aeca51d886018aa39ca25f222c49695 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 20 Nov 2024 15:58:07 -0500 Subject: [PATCH 1090/2091] tiny optimisation in append_pdu push notif Signed-off-by: strawberry --- src/service/rooms/timeline/mod.rs | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index a3fc6a0b..5d5566cb 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -11,8 +11,7 @@ use std::{ use conduit::{ debug, debug_warn, err, error, implement, info, pdu::{EventHash, PduBuilder, PduCount, PduEvent}, - utils, - utils::{stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, + utils::{self, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, validated, warn, Err, Error, Result, Server, }; pub use conduit::{PduId, RawPduId}; @@ -386,17 +385,19 @@ impl Service { let sync_pdu = pdu.to_sync_room_event(); - let mut notifies = Vec::new(); - let mut highlights = Vec::new(); - let mut push_target: HashSet<_> = self .services .state_cache .active_local_users_in_room(&pdu.room_id) + // Don't notify the sender of their own events + .ready_filter(|user| user != &pdu.sender) .map(ToOwned::to_owned) .collect() .await; + let mut notifies = Vec::with_capacity(push_target.len().saturating_add(1)); + let mut highlights = Vec::with_capacity(push_target.len().saturating_add(1)); + if pdu.kind == TimelineEventType::RoomMember { if let Some(state_key) = &pdu.state_key { let target_user_id = UserId::parse(state_key.clone())?; @@ -408,11 +409,6 @@ impl Service { } for user in &push_target { - // Don't notify the user of their own events - if user == &pdu.sender { - continue; - } - let rules_for_user = self .services .account_data @@ -436,6 +432,11 @@ impl Service { }, _ => {}, }; + + // Break early if both conditions are true + if notify && highlight { + break; + } } if notify { From b6d53e97a6c560109e4bd270de80a73e60ca8605 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 20 Nov 2024 16:31:36 -0500 Subject: [PATCH 1091/2091] bump ruwuma and a few http deps Signed-off-by: strawberry --- Cargo.lock | 68 +++++++++++++++++++++++++++++------------------------- Cargo.toml | 19 +++++++-------- 2 files changed, 47 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a95f83a..d39375d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -191,9 +191,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49c41b948da08fb481a94546cd874843adc1142278b0af4badf9b1b78599d68d" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" dependencies = [ "async-trait", "axum-core", @@ -257,9 +257,9 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.9.5" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37634d71e9f3c35cfb1c30c87c7cba500d55892f04c2dbe6a99383c664b820b0" +checksum = "c794b30c904f0a1c2fb7740f7df7f7972dfaa14ef6f57cb6178dc63e5dca2f04" dependencies = [ "axum", "axum-core", @@ -1048,9 +1048,9 @@ dependencies = [ [[package]] name = "ctor" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edb49164822f3ee45b17acd4a208cfc1251410cf0cad9a833234c9890774dd9f" +checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", "syn 2.0.87", @@ -1699,9 +1699,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbff0a806a4728c99295b254c8838933b5b082d75e3cb70c8dab21fdfbcfa9a" +checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" dependencies = [ "bytes", "futures-channel", @@ -3128,7 +3128,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "assign", "js_int", @@ -3150,7 +3150,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "js_int", "ruma-common", @@ -3162,7 +3162,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "as_variant", "assign", @@ -3185,7 +3185,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "as_variant", "base64 0.22.1", @@ -3215,7 +3215,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3239,7 +3239,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "bytes", "http", @@ -3257,7 +3257,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "js_int", "thiserror 2.0.3", @@ -3266,7 +3266,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "js_int", "ruma-common", @@ -3276,7 +3276,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "cfg-if", "once_cell", @@ -3292,7 +3292,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "js_int", "ruma-common", @@ -3304,7 +3304,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "headers", "http", @@ -3317,7 +3317,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3333,7 +3333,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=2ab432fba19eb8862c594d24af39d8f9f6b4eac6#2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" dependencies = [ "futures-util", "itertools 0.13.0", @@ -3502,16 +3502,16 @@ checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" [[package]] name = "rustyline-async" version = "0.4.3" -source = "git+https://github.com/girlbossceo/rustyline-async?rev=9654cc84e19241f6e19021eb8e677892656f5071#9654cc84e19241f6e19021eb8e677892656f5071" +source = "git+https://github.com/girlbossceo/rustyline-async?rev=deaeb0694e2083f53d363b648da06e10fc13900c#deaeb0694e2083f53d363b648da06e10fc13900c" dependencies = [ "crossterm", "futures-channel", "futures-util", "pin-project", "thingbuf", - "thiserror 1.0.69", + "thiserror 2.0.3", "unicode-segmentation", - "unicode-width", + "unicode-width 0.2.0", ] [[package]] @@ -3749,9 +3749,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", @@ -4056,9 +4056,9 @@ dependencies = [ [[package]] name = "termimad" -version = "0.31.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cda3a7471f9978706978454c45ef8dda67e9f8f3cdb9319eb2e9323deb6ae62" +checksum = "ea6a5d4cf55d9f1cb04fcda48f725772d0733ae34e030dfc4dd36e738a5965f4" dependencies = [ "coolor", "crokey", @@ -4067,7 +4067,7 @@ dependencies = [ "minimad", "serde", "thiserror 1.0.69", - "unicode-width", + "unicode-width 0.1.14", ] [[package]] @@ -4425,9 +4425,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437150ab6bbc8c5f0f519e3d5ed4aa883a83dd4cdd3d1b21f9482936046cb97" +checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "async-compression", "bitflags 2.6.0", @@ -4628,6 +4628,12 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +[[package]] +name = "unicode-width" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" + [[package]] name = "unsafe-libyaml" version = "0.2.11" diff --git a/Cargo.toml b/Cargo.toml index 68c87c57..ed24435f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,7 +33,7 @@ features = ["std", "serde"] version = "0.5.7" [workspace.dependencies.ctor] -version = "0.2.8" +version = "0.2.9" [workspace.dependencies.cargo_toml] version = "0.20" @@ -82,7 +82,7 @@ version = "1.1.0" version = "1.11.1" [workspace.dependencies.axum] -version = "0.7.5" +version = "0.7.9" default-features = false features = [ "form", @@ -95,7 +95,7 @@ features = [ ] [workspace.dependencies.axum-extra] -version = "0.9.4" +version = "0.9.6" default-features = false features = ["typed-header", "tracing"] @@ -116,7 +116,7 @@ default-features = false features = ["util"] [workspace.dependencies.tower-http] -version = "0.6.1" +version = "0.6.2" default-features = false features = [ "add-extension", @@ -149,7 +149,7 @@ default-features = false features = ["rc"] [workspace.dependencies.serde_json] -version = "1.0.132" +version = "1.0.133" default-features = false features = ["raw_value"] @@ -255,7 +255,7 @@ features = ["alloc", "std"] default-features = false [workspace.dependencies.hyper] -version = "1.5.0" +version = "1.5.1" default-features = false features = [ "server", @@ -322,7 +322,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "2ab432fba19eb8862c594d24af39d8f9f6b4eac6" +rev = "97e2fb6df13f65532d33fc2f0f097ad5a449dd70" features = [ "compat", "rand", @@ -335,6 +335,7 @@ features = [ "server-util", "unstable-exhaustive-types", "ring-compat", + "compat-upload-signatures", "identifiers-validation", "unstable-unspecified", "unstable-msc2409", @@ -460,7 +461,7 @@ version = "0.4.3" default-features = false [workspace.dependencies.termimad] -version = "0.31.0" +version = "0.31.1" default-features = false [workspace.dependencies.checked_ops] @@ -504,7 +505,7 @@ rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" # adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b [patch.crates-io.rustyline-async] git = "https://github.com/girlbossceo/rustyline-async" -rev = "9654cc84e19241f6e19021eb8e677892656f5071" +rev = "deaeb0694e2083f53d363b648da06e10fc13900c" # # Our crates From 9100af9974c12d06a917ce0715b636b5d1b237f2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 21 Nov 2024 23:45:16 -0500 Subject: [PATCH 1092/2091] add eventid_pdu database cf cache Signed-off-by: strawberry --- src/core/config/mod.rs | 6 ++++++ src/database/opts.rs | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 355881b3..753a08fa 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -198,6 +198,10 @@ pub struct Config { #[serde(default = "default_eventidshort_cache_capacity")] pub eventidshort_cache_capacity: u32, + /// default: varies by system + #[serde(default = "default_eventid_pdu_cache_capacity")] + pub eventid_pdu_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_shortstatekey_cache_capacity")] pub shortstatekey_cache_capacity: u32, @@ -2040,6 +2044,8 @@ fn default_shorteventid_cache_capacity() -> u32 { parallelism_scaled_u32(50_000) fn default_eventidshort_cache_capacity() -> u32 { parallelism_scaled_u32(25_000).saturating_add(100_000) } +fn default_eventid_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(25_000).saturating_add(100_000) } + fn default_shortstatekey_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } fn default_statekeyshort_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } diff --git a/src/database/opts.rs b/src/database/opts.rs index 46fb4c54..732f571f 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -136,6 +136,14 @@ pub(crate) fn cf_options( cache_size(cfg, cfg.eventidshort_cache_capacity, 64)?, ), + "eventid_pduid" => set_table_with_new_cache( + &mut opts, + cfg, + cache, + name, + cache_size(cfg, cfg.eventid_pdu_cache_capacity, 64)?, + ), + "shorteventid_authchain" => { set_table_with_new_cache( &mut opts, From 5256cad396b97b520a1078e327b8c2adfb423e9b Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 21 Nov 2024 23:49:46 -0500 Subject: [PATCH 1093/2091] ignore bare_urls lint for well_known client config option Signed-off-by: strawberry --- src/core/config/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 753a08fa..66c78440 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1516,6 +1516,7 @@ pub struct TlsConfig { pub dual_protocol: bool, } +#[allow(rustdoc::broken_intra_doc_links, rustdoc::bare_urls)] #[derive(Clone, Debug, Deserialize, Default)] #[config_example_generator(filename = "conduwuit-example.toml", section = "global.well_known")] pub struct WellKnownConfig { @@ -1528,7 +1529,7 @@ pub struct WellKnownConfig { /// The server URL that the client well-known file will serve. This should /// not contain a port, and should just be a valid HTTPS URL. /// - /// example: "" + /// example: "https://matrix.example.com" pub client: Option, pub support_page: Option, pub support_role: Option, From bae06670661ada619f9a72c0ee4dcad2621c9dd4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 20 Nov 2024 19:42:34 +0000 Subject: [PATCH 1094/2091] limit sync response events to within the since/next_batch window fixes #606 Signed-off-by: Jason Volk --- src/api/client/sync/mod.rs | 6 ++++-- src/api/client/sync/v3.rs | 3 ++- src/api/client/sync/v4.rs | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 3201b827..a9715b84 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -9,7 +9,8 @@ pub(crate) use self::{v3::sync_events_route, v4::sync_events_v4_route}; use crate::{service::Services, Error, PduEvent, Result}; async fn load_timeline( - services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, limit: usize, + services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, + next_batch: Option, limit: usize, ) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> { let last_timeline_count = services .rooms @@ -26,7 +27,8 @@ async fn load_timeline( .timeline .pdus_rev(Some(sender_user), room_id, None) .await? - .ready_take_while(|(pducount, _)| *pducount > roomsincecount); + .ready_skip_while(|&(pducount, _)| pducount > next_batch.unwrap_or_else(PduCount::max)) + .ready_take_while(|&(pducount, _)| pducount > roomsincecount); // Take the last events for the timeline let timeline_pdus: Vec<_> = non_timeline_pdus diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 77ba4c3f..7a78ea74 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -540,7 +540,8 @@ async fn load_joined_room( let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); - let (timeline_pdus, limited) = load_timeline(services, sender_user, room_id, sincecount, 10_usize).await?; + let (timeline_pdus, limited) = + load_timeline(services, sender_user, room_id, sincecount, Some(next_batchcount), 10_usize).await?; let send_notification_counts = !timeline_pdus.is_empty() || services diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 91abd24e..57edc953 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -473,7 +473,7 @@ pub(crate) async fn sync_events_v4_route( (timeline_pdus, limited) = (Vec::new(), true); } else { (timeline_pdus, limited) = - match load_timeline(&services, sender_user, room_id, roomsincecount, *timeline_limit).await { + match load_timeline(&services, sender_user, room_id, roomsincecount, None, *timeline_limit).await { Ok(value) => value, Err(err) => { warn!("Encountered missing timeline in {}, error {}", room_id, err); From aea82183b2deb7318f6c0b1f8e4c2c86be9fbdcc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 21 Nov 2024 05:51:25 +0000 Subject: [PATCH 1095/2091] add set intersection util for two sorted streams Signed-off-by: Jason Volk --- src/core/utils/set.rs | 32 +++++++++++++++++++++++++++++++- src/core/utils/tests.rs | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/core/utils/set.rs b/src/core/utils/set.rs index 563f9df5..ddcf05ff 100644 --- a/src/core/utils/set.rs +++ b/src/core/utils/set.rs @@ -1,4 +1,10 @@ -use std::cmp::{Eq, Ord}; +use std::{ + cmp::{Eq, Ord}, + pin::Pin, + sync::Arc, +}; + +use futures::{Stream, StreamExt}; use crate::{is_equal_to, is_less_than}; @@ -45,3 +51,27 @@ where }) }) } + +/// Intersection of sets +/// +/// Outputs the set of elements common to both streams. Streams must be sorted. +pub fn intersection_sorted_stream2(a: S, b: S) -> impl Stream + Send +where + S: Stream + Send + Unpin, + Item: Eq + PartialOrd + Send + Sync, +{ + use tokio::sync::Mutex; + + let b = Arc::new(Mutex::new(b.peekable())); + a.map(move |ai| (ai, b.clone())) + .filter_map(|(ai, b)| async move { + let mut lock = b.lock().await; + while let Some(bi) = Pin::new(&mut *lock).next_if(|bi| *bi <= ai).await.as_ref() { + if ai == *bi { + return Some(ai); + } + } + + None + }) +} diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index 84d35936..f4f78b02 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -237,3 +237,42 @@ fn set_intersection_sorted_all() { let r = intersection_sorted(i.into_iter()); assert!(r.eq(["bar", "baz", "foo"].iter())); } + +#[tokio::test] +async fn set_intersection_sorted_stream2() { + use futures::StreamExt; + use utils::{set::intersection_sorted_stream2, IterStream}; + + let a = ["bar"]; + let b = ["bar", "foo"]; + let r = intersection_sorted_stream2(a.iter().stream(), b.iter().stream()) + .collect::>() + .await; + assert!(r.eq(&["bar"])); + + let r = intersection_sorted_stream2(b.iter().stream(), a.iter().stream()) + .collect::>() + .await; + assert!(r.eq(&["bar"])); + + let a = ["aaa", "ccc", "xxx", "yyy"]; + let b = ["hhh", "iii", "jjj", "zzz"]; + let r = intersection_sorted_stream2(a.iter().stream(), b.iter().stream()) + .collect::>() + .await; + assert!(r.is_empty()); + + let a = ["aaa", "ccc", "eee", "ggg"]; + let b = ["aaa", "bbb", "ccc", "ddd", "eee"]; + let r = intersection_sorted_stream2(a.iter().stream(), b.iter().stream()) + .collect::>() + .await; + assert!(r.eq(&["aaa", "ccc", "eee"])); + + let a = ["aaa", "ccc", "eee", "ggg", "hhh", "iii"]; + let b = ["bbb", "ccc", "ddd", "fff", "ggg", "iii"]; + let r = intersection_sorted_stream2(a.iter().stream(), b.iter().stream()) + .collect::>() + .await; + assert!(r.eq(&["ccc", "ggg", "iii"])); +} From 3968d038683592745799caca89e00e5547b6fde8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 20 Nov 2024 20:21:31 +0000 Subject: [PATCH 1096/2091] move and improve common-rooms related Signed-off-by: Jason Volk --- src/api/client/presence.rs | 4 ++-- src/api/client/sync/mod.rs | 2 +- src/api/client/unstable.rs | 2 +- src/api/client/user_directory.rs | 4 ++-- src/service/rooms/state_cache/mod.rs | 25 +++++++++++++++---------- src/service/rooms/user/mod.rs | 22 ---------------------- 6 files changed, 21 insertions(+), 38 deletions(-) diff --git a/src/api/client/presence.rs b/src/api/client/presence.rs index ba48808b..948d6caa 100644 --- a/src/api/client/presence.rs +++ b/src/api/client/presence.rs @@ -52,8 +52,8 @@ pub(crate) async fn get_presence_route( let has_shared_rooms = services .rooms - .user - .has_shared_rooms(sender_user, &body.user_id) + .state_cache + .user_sees_user(sender_user, &body.user_id) .await; if has_shared_rooms { diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index a9715b84..ba50d77c 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -52,7 +52,7 @@ async fn share_encrypted_room( ) -> bool { services .rooms - .user + .state_cache .get_shared_rooms(sender_user, user_id) .ready_filter(|&room_id| Some(room_id) != ignore_room) .any(|other_room_id| { diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index dc570295..5de41f44 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -55,7 +55,7 @@ pub(crate) async fn get_mutual_rooms_route( let mutual_rooms: Vec = services .rooms - .user + .state_cache .get_shared_rooms(sender_user, &body.user_id) .map(ToOwned::to_owned) .collect() diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index 868811a3..f3fee8d1 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -71,8 +71,8 @@ pub(crate) async fn search_users_route( } else { let user_is_in_shared_rooms = services .rooms - .user - .has_shared_rooms(sender_user, &user.user_id) + .state_cache + .user_sees_user(sender_user, &user.user_id) .await; if user_is_in_shared_rooms { diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 6e330fdc..156345fe 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -10,7 +10,7 @@ use conduit::{ warn, Result, }; use database::{serialize_to_vec, Deserialized, Ignore, Interfix, Json, Map}; -use futures::{future::join4, stream::iter, Stream, StreamExt}; +use futures::{future::join4, pin_mut, stream::iter, Stream, StreamExt}; use itertools::Itertools; use ruma::{ events::{ @@ -385,16 +385,21 @@ impl Service { /// Returns true if user_a and user_b share at least one room. #[tracing::instrument(skip(self), level = "debug")] pub async fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> bool { - // Minimize number of point-queries by iterating user with least nr rooms - let (a, b) = if self.rooms_joined(user_a).count().await < self.rooms_joined(user_b).count().await { - (user_a, user_b) - } else { - (user_b, user_a) - }; + let get_shared_rooms = self.get_shared_rooms(user_a, user_b); - self.rooms_joined(a) - .any(|room_id| self.is_joined(b, room_id)) - .await + pin_mut!(get_shared_rooms); + get_shared_rooms.next().await.is_some() + } + + /// List the rooms common between two users + pub fn get_shared_rooms<'a>( + &'a self, user_a: &'a UserId, user_b: &'a UserId, + ) -> impl Stream + Send + 'a { + use conduit::utils::set; + + let a = self.rooms_joined(user_a); + let b = self.rooms_joined(user_b); + set::intersection_sorted_stream2(a, b) } /// Returns an iterator of all joined members of a room. diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 99587134..948baa5e 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -2,7 +2,6 @@ use std::sync::Arc; use conduit::{implement, Result}; use database::{Deserialized, Map}; -use futures::{pin_mut, Stream, StreamExt}; use ruma::{RoomId, UserId}; use crate::{globals, rooms, rooms::short::ShortStateHash, Dep}; @@ -22,7 +21,6 @@ struct Data { struct Services { globals: Dep, short: Dep, - state_cache: Dep, } impl crate::Service for Service { @@ -38,7 +36,6 @@ impl crate::Service for Service { services: Services { globals: args.depend::("globals"), short: args.depend::("rooms::short"), - state_cache: args.depend::("rooms::state_cache"), }, })) } @@ -118,22 +115,3 @@ pub async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Re .await .deserialized() } - -#[implement(Service)] -pub async fn has_shared_rooms<'a>(&'a self, user_a: &'a UserId, user_b: &'a UserId) -> bool { - let get_shared_rooms = self.get_shared_rooms(user_a, user_b); - - pin_mut!(get_shared_rooms); - get_shared_rooms.next().await.is_some() -} - -//TODO: optimize; replace point-queries with dual iteration -#[implement(Service)] -pub fn get_shared_rooms<'a>( - &'a self, user_a: &'a UserId, user_b: &'a UserId, -) -> impl Stream + Send + 'a { - self.services - .state_cache - .rooms_joined(user_a) - .filter(|room_id| self.services.state_cache.is_joined(user_b, room_id)) -} From b94eeb9580af01d68d0c916a01a304b58e20e4ab Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 22 Nov 2024 03:57:29 +0000 Subject: [PATCH 1097/2091] fix deletions on readreceipt update Signed-off-by: Jason Volk --- src/service/rooms/read_receipt/data.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 80a35e88..1194598d 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -42,16 +42,14 @@ impl Data { } pub(super) async fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) { - type KeyVal<'a> = (&'a RoomId, u64, &'a UserId); - // Remove old entry let last_possible_key = (room_id, u64::MAX); self.readreceiptid_readreceipt - .rev_keys_from(&last_possible_key) + .rev_keys_from_raw(&last_possible_key) .ignore_err() - .ready_take_while(|(r, ..): &KeyVal<'_>| *r == room_id) - .ready_filter_map(|(r, c, u): KeyVal<'_>| (u == user_id).then_some((r, c, u))) - .ready_for_each(|old: KeyVal<'_>| self.readreceiptid_readreceipt.del(old)) + .ready_take_while(|key| key.starts_with(room_id.as_bytes())) + .ready_filter_map(|key| key.ends_with(user_id.as_bytes()).then_some(key)) + .ready_for_each(|key| self.readreceiptid_readreceipt.del(key)) .await; let count = self.services.globals.next_count().unwrap(); From f15370027e9712eb0fce07f0619a8d74cead149a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 23 Nov 2024 12:05:52 -0500 Subject: [PATCH 1098/2091] improve DNS error messages Signed-off-by: strawberry --- src/service/resolver/actual.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 5dc03d14..fec29133 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -4,7 +4,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, debug_error, debug_info, debug_warn, err, trace, Err, Result}; +use conduit::{debug, debug_error, debug_info, debug_warn, err, error, trace, Err, Result}; use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; use ipaddress::IPAddress; use ruma::ServerName; @@ -313,7 +313,6 @@ impl super::Service { Ok(None) } - #[allow(clippy::single_match_else)] fn handle_resolve_error(e: &ResolveError) -> Result<()> { use hickory_resolver::error::ResolveErrorKind; @@ -322,10 +321,21 @@ impl super::Service { .. } => { // Raise to debug_warn if we can find out the result wasn't from cache - debug!("{e}"); + debug!("No DNS records found: {e}"); Ok(()) }, - _ => Err!(error!("DNS {e}")), + ResolveErrorKind::Timeout => { + Err!(warn!("DNS {e}")) + }, + ResolveErrorKind::NoConnections => { + error!( + "Your DNS server is overloaded and has ran out of connections. It is strongly recommended you \ + remediate this issue to ensure proper federation connectivity." + ); + + Err!(error!("DNS error: {e}")) + }, + _ => Err!(error!("DNS error: {e}")), } } From 9d23a2b6f5e5a721d02c818f2c1bbd0caa716357 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 23 Nov 2024 12:53:26 -0500 Subject: [PATCH 1099/2091] add missing length checks on pushkey/appid, improve error msgs for pusher Signed-off-by: strawberry --- src/api/client/push.rs | 4 +-- src/service/pusher/mod.rs | 54 ++++++++++++++++++++++----------------- 2 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 97243ab4..f2376e7c 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -441,9 +441,9 @@ pub(crate) async fn set_pushers_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services.pusher.set_pusher(sender_user, &body.action); + services.pusher.set_pusher(sender_user, &body.action)?; - Ok(set_pusher::v3::Response::default()) + Ok(set_pusher::v3::Response::new()) } /// user somehow has bad push rules, these must always exist per spec. diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 2b90319e..fb43fdb8 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -2,9 +2,9 @@ use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; use conduit::{ - debug_error, err, trace, + debug_warn, err, trace, utils::{stream::TryIgnore, string_from_bytes}, - Err, PduEvent, Result, + warn, Err, PduEvent, Result, }; use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::{Stream, StreamExt}; @@ -65,17 +65,29 @@ impl crate::Service for Service { } impl Service { - pub fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) { + pub fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result { match pusher { set_pusher::v3::PusherAction::Post(data) => { - let key = (sender, &data.pusher.ids.pushkey); + let pushkey = data.pusher.ids.pushkey.as_str(); + + if pushkey.len() > 512 { + return Err!(Request(InvalidParam("Push key length cannot be greater than 512 bytes."))); + } + + if data.pusher.ids.app_id.as_str().len() > 64 { + return Err!(Request(InvalidParam("App ID length cannot be greater than 64 bytes."))); + } + + let key = (sender, data.pusher.ids.pushkey.as_str()); self.db.senderkey_pusher.put(key, Json(pusher)); }, set_pusher::v3::PusherAction::Delete(ids) => { - let key = (sender, &ids.pushkey); + let key = (sender, ids.pushkey.as_str()); self.db.senderkey_pusher.del(key); }, } + + Ok(()) } pub async fn get_pusher(&self, sender: &UserId, pushkey: &str) -> Result { @@ -166,8 +178,8 @@ impl Service { let body = response.bytes().await?; // TODO: handle timeout if !status.is_success() { - debug_error!("Push gateway response body: {:?}", string_from_bytes(&body)); - return Err!(BadServerResponse(error!( + debug_warn!("Push gateway response body: {:?}", string_from_bytes(&body)); + return Err!(BadServerResponse(warn!( "Push gateway {dest} returned unsuccessful HTTP response: {status}" ))); } @@ -178,10 +190,10 @@ impl Service { .expect("reqwest body is valid http body"), ); response - .map_err(|e| err!(BadServerResponse(error!("Push gateway {dest} returned invalid response: {e}")))) + .map_err(|e| err!(BadServerResponse(warn!("Push gateway {dest} returned invalid response: {e}")))) }, Err(e) => { - debug_error!("Could not send request to pusher {dest}: {e}"); + warn!("Could not send request to pusher {dest}: {e}"); Err(e.into()) }, } @@ -278,11 +290,7 @@ impl Service { // TODO: email match &pusher.kind { PusherKind::Http(http) => { - // TODO: - // Two problems with this - // 1. if "event_id_only" is the only format kind it seems we should never add - // more info - // 2. can pusher/devices have conflicting formats + // TODO (timo): can pusher/devices have conflicting formats let event_id_only = http.format == Some(PushFormat::EventIdOnly); let mut device = Device::new(pusher.ids.app_id.clone(), pusher.ids.pushkey.clone()); @@ -297,24 +305,24 @@ impl Service { let d = vec![device]; let mut notifi = Notification::new(d); - notifi.prio = NotificationPriority::Low; notifi.event_id = Some((*event.event_id).to_owned()); notifi.room_id = Some((*event.room_id).to_owned()); // TODO: missed calls notifi.counts = NotificationCounts::new(unread, uint!(0)); - if event.kind == TimelineEventType::RoomEncrypted - || tweaks - .iter() - .any(|t| matches!(t, Tweak::Highlight(true) | Tweak::Sound(_))) - { - notifi.prio = NotificationPriority::High; - } - if event_id_only { self.send_request(&http.url, send_event_notification::v1::Request::new(notifi)) .await?; } else { + if event.kind == TimelineEventType::RoomEncrypted + || tweaks + .iter() + .any(|t| matches!(t, Tweak::Highlight(true) | Tweak::Sound(_))) + { + notifi.prio = NotificationPriority::High; + } else { + notifi.prio = NotificationPriority::Low; + } notifi.sender = Some(event.sender.clone()); notifi.event_type = Some(event.kind.clone()); notifi.content = serde_json::value::to_raw_value(&event.content).ok(); From 3fe98f35f2571f80d39d251977208b1b074c1622 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 23 Nov 2024 13:45:27 -0500 Subject: [PATCH 1100/2091] remove queued push keys on pusher deletion, use more refs Signed-off-by: strawberry --- src/api/client/push.rs | 5 ++++- src/service/appservice/mod.rs | 14 ++++++-------- src/service/pusher/mod.rs | 12 ++++++++++-- src/service/sending/mod.rs | 35 ++++++++++++++++++++++++++++------- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/api/client/push.rs b/src/api/client/push.rs index f2376e7c..f27ead1f 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -441,7 +441,10 @@ pub(crate) async fn set_pushers_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services.pusher.set_pusher(sender_user, &body.action)?; + services + .pusher + .set_pusher(sender_user, &body.action) + .await?; Ok(set_pusher::v3::Response::new()) } diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 1617e6e6..4a20b130 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -79,26 +79,24 @@ impl Service { /// /// # Arguments /// - /// * `service_name` - the name you send to register the service previously - pub async fn unregister_appservice(&self, service_name: &str) -> Result<()> { + /// * `service_name` - the registration ID of the appservice + pub async fn unregister_appservice(&self, appservice_id: &str) -> Result<()> { // removes the appservice registration info self.registration_info .write() .await - .remove(service_name) + .remove(appservice_id) .ok_or(err!("Appservice not found"))?; // remove the appservice from the database - self.db.id_appserviceregistrations.remove(service_name); + self.db.id_appserviceregistrations.del(appservice_id); // deletes all active requests for the appservice if there are any so we stop // sending to the URL self.services .sending - .cleanup_events(service_name.to_owned()) - .await; - - Ok(()) + .cleanup_events(Some(appservice_id), None, None) + .await } pub async fn get_registration(&self, id: &str) -> Option { diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index fb43fdb8..6b02c7f8 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -26,7 +26,7 @@ use ruma::{ uint, RoomId, UInt, UserId, }; -use crate::{client, globals, rooms, users, Dep}; +use crate::{client, globals, rooms, sending, users, Dep}; pub struct Service { db: Data, @@ -39,6 +39,7 @@ struct Services { state_accessor: Dep, state_cache: Dep, users: Dep, + sending: Dep, } struct Data { @@ -57,6 +58,7 @@ impl crate::Service for Service { state_accessor: args.depend::("rooms::state_accessor"), state_cache: args.depend::("rooms::state_cache"), users: args.depend::("users"), + sending: args.depend::("sending"), }, })) } @@ -65,7 +67,7 @@ impl crate::Service for Service { } impl Service { - pub fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result { + pub async fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result { match pusher { set_pusher::v3::PusherAction::Post(data) => { let pushkey = data.pusher.ids.pushkey.as_str(); @@ -84,6 +86,12 @@ impl Service { set_pusher::v3::PusherAction::Delete(ids) => { let key = (sender, ids.pushkey.as_str()); self.db.senderkey_pusher.del(key); + + self.services + .sending + .cleanup_events(None, Some(sender), Some(ids.pushkey.as_str())) + .await + .ok(); }, } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 5a070306..611940be 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -8,7 +8,7 @@ use std::{fmt::Debug, iter::once, sync::Arc}; use async_trait::async_trait; use conduit::{ - err, + debug_warn, err, utils::{ReadyExt, TryReadyExt}, warn, Result, Server, }; @@ -285,13 +285,34 @@ impl Service { appservice::send_request(client, registration, request).await } - /// Cleanup event data - /// Used for instance after we remove an appservice registration + /// Clean up queued sending event data + /// + /// Used after we remove an appservice registration or a user deletes a push + /// key #[tracing::instrument(skip(self), level = "debug")] - pub async fn cleanup_events(&self, appservice_id: String) { - self.db - .delete_all_requests_for(&Destination::Appservice(appservice_id)) - .await; + pub async fn cleanup_events( + &self, appservice_id: Option<&str>, user_id: Option<&UserId>, push_key: Option<&str>, + ) -> Result { + match (appservice_id, user_id, push_key) { + (None, Some(user_id), Some(push_key)) => { + self.db + .delete_all_requests_for(&Destination::Push(user_id.to_owned(), push_key.to_owned())) + .await; + + Ok(()) + }, + (Some(appservice_id), None, None) => { + self.db + .delete_all_requests_for(&Destination::Appservice(appservice_id.to_owned())) + .await; + + Ok(()) + }, + _ => { + debug_warn!("cleanup_events called with too many or too few arguments"); + Ok(()) + }, + } } fn dispatch(&self, msg: Msg) -> Result<()> { From af772b0240c471d8b2e902d9cef0deab08d5c6fb Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 23 Nov 2024 22:35:54 -0500 Subject: [PATCH 1101/2091] various misc documentation improvements Signed-off-by: strawberry --- CONTRIBUTING.md | 3 ++- README.md | 3 +-- debian/README.md | 15 ++++++++----- docs/SUMMARY.md | 1 + docs/deploying/docker.md | 37 +++++++++++++++++++++---------- docs/deploying/kubernetes.md | 8 +++++-- docs/development.md | 24 ++++++++++++--------- docs/development/testing.md | 4 ++-- docs/troubleshooting.md | 42 +++++++++++++++++++++++++++--------- 9 files changed, 94 insertions(+), 43 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a11394e..fb540011 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -131,7 +131,8 @@ allowed to be licenced under the Apache-2.0 licence and all of your conduct is in line with the Contributor's Covenant, and conduwuit's Code of Conduct. Contribution by users who violate either of these code of conducts will not have -their contributions accepted. +their contributions accepted. This includes users who have been banned from +conduwuit Matrix rooms for Code of Conduct violations. [issues]: https://github.com/girlbossceo/conduwuit/issues [conduwuit-matrix]: https://matrix.to/#/#conduwuit:puppygock.gay diff --git a/README.md b/README.md index 4e97f1f0..4faf1ad7 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ # conduwuit -`main`: [![CI and -Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml) +[![conduwuit main room](https://img.shields.io/matrix/conduwuit%3Apuppygock.gay?server_fqdn=matrix.transfem.dev&style=flat&logo=matrix&logoColor=%23f5b3ff&label=%23conduwuit%3Apuppygock.gay&color=%23f652ff)](https://matrix.to/#/#conduwuit:puppygock.gay) [![conduwuit space](https://img.shields.io/matrix/conduwuit-space%3Apuppygock.gay?server_fqdn=matrix.transfem.dev&style=flat&logo=matrix&logoColor=%23f5b3ff&label=%23conduwuit-space%3Apuppygock.gay&color=%23f652ff)](https://matrix.to/#/#conduwuit-space:puppygock.gay) [![CI and Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml) diff --git a/debian/README.md b/debian/README.md index 62aa2112..89354469 100644 --- a/debian/README.md +++ b/debian/README.md @@ -1,17 +1,22 @@ # conduwuit for Debian -Information about downloading and deploying the Debian package. This may also be referenced for other `apt`-based distros such as Ubuntu. +Information about downloading and deploying the Debian package. This may also be +referenced for other `apt`-based distros such as Ubuntu. ### Installation -It is recommended to see the [generic deployment guide](../deploying/generic.md) for further information if needed as usage of the Debian package is generally related. +It is recommended to see the [generic deployment guide](../deploying/generic.md) +for further information if needed as usage of the Debian package is generally +related. ### Configuration -When installed, the example config is placed at `/etc/conduwuit/conduwuit.toml` as the default config. At the minimum, you will need to change your `server_name` here. +When installed, the example config is placed at `/etc/conduwuit/conduwuit.toml` +as the default config. The config mentions things required to be changed before +starting. -You can tweak more detailed settings by uncommenting and setting the config options -in `/etc/conduwuit/conduwuit.toml`. +You can tweak more detailed settings by uncommenting and setting the config +options in `/etc/conduwuit/conduwuit.toml`. ### Running diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 5e4155c4..8e07adc2 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -8,6 +8,7 @@ - [Generic](deploying/generic.md) - [NixOS](deploying/nixos.md) - [Docker](deploying/docker.md) + - [Kubernetes](deploying/kubernetes.md) - [Arch Linux](deploying/arch-linux.md) - [Debian](deploying/debian.md) - [FreeBSD](deploying/freebsd.md) diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index e9c49c71..fffa7770 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -11,9 +11,9 @@ OCI images for conduwuit are available in the registries listed below. | Registry | Image | Size | Notes | | --------------- | --------------------------------------------------------------- | ----------------------------- | ---------------------- | -| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:latest][gh] | ![Image Size][shield-latest] | Stable tagged image. | -| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:latest][gl] | ![Image Size][shield-latest] | Stable tagged image. | -| Docker Hub | [docker.io/girlbossceo/conduwuit:latest][dh] | ![Image Size][shield-latest] | Stable tagged image. | +| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:latest][gh] | ![Image Size][shield-latest] | Stable latest tagged image. | +| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:latest][gl] | ![Image Size][shield-latest] | Stable latest tagged image. | +| Docker Hub | [docker.io/girlbossceo/conduwuit:latest][dh] | ![Image Size][shield-latest] | Stable latest tagged image. | | GitHub Registry | [ghcr.io/girlbossceo/conduwuit:main][gh] | ![Image Size][shield-main] | Stable main branch. | | GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:main][gl] | ![Image Size][shield-main] | Stable main branch. | | Docker Hub | [docker.io/girlbossceo/conduwuit:main][dh] | ![Image Size][shield-main] | Stable main branch. | @@ -92,16 +92,28 @@ Additional info about deploying conduwuit can be found [here](generic.md). ### Build -To build the conduwuit image with docker-compose, you first need to open and -modify the `docker-compose.yml` file. There you need to comment the `image:` -option and uncomment the `build:` option. Then call docker compose with: +Official conduwuit images are built using Nix's +[`buildLayeredImage`][nix-buildlayeredimage]. This ensures all OCI images are +repeatable and reproducible by anyone, keeps the images lightweight, and can be +built offline. -```bash -docker compose up -``` +This also ensures portability of our images because `buildLayeredImage` builds +OCI images, not Docker images, and works with other container software. -This will also start the container right afterwards, so if want it to run in -detached mode, you also should use the `-d` flag. +The OCI images are OS-less with only a very minimal environment of the `tini` +init system, CA certificates, and the conduwuit binary. This does mean there is +not a shell, but in theory you can get a shell by adding the necessary layers +to the layered image. However it's very unlikely you will need a shell for any +real troubleshooting. + +The flake file for the OCI image definition is at [`nix/pkgs/oci-image/default.nix`][oci-image-def]. + +To build an OCI image using Nix, the following outputs can be built: +- `nix build -L .#oci-image` (default features, x86_64 glibc) +- `nix build -L .#oci-image-x86_64-linux-musl` (default features, x86_64 musl) +- `nix build -L .#oci-image-aarch64-linux-musl` (default features, aarch64 musl) +- `nix build -L .#oci-image-x86_64-linux-musl-all-features` (all features, x86_64 musl) +- `nix build -L .#oci-image-aarch64-linux-musl-all-features` (all features, aarch64 musl) ### Run @@ -136,3 +148,6 @@ those two files. ## Voice communication See the [TURN](../turn.md) page. + +[nix-buildlayeredimage]: https://ryantm.github.io/nixpkgs/builders/images/dockertools/#ssec-pkgs-dockerTools-buildLayeredImage +[oci-image-def]: https://github.com/girlbossceo/conduwuit/blob/main/nix/pkgs/oci-image/default.nix diff --git a/docs/deploying/kubernetes.md b/docs/deploying/kubernetes.md index 2a1bcb51..d7721722 100644 --- a/docs/deploying/kubernetes.md +++ b/docs/deploying/kubernetes.md @@ -1,4 +1,8 @@ # conduwuit for Kubernetes -conduwuit doesn't support horizontal scalability or distributed loading natively, however a community maintained Helm Chart is available here to run conduwuit on Kubernetes: - +conduwuit doesn't support horizontal scalability or distributed loading +natively, however a community maintained Helm Chart is available here to run +conduwuit on Kubernetes: + +Should changes need to be made, please reach out to the maintainer in our +Matrix room as this is not maintained/controlled by the conduwuit maintainers. diff --git a/docs/development.md b/docs/development.md index e1f36c0c..28b07667 100644 --- a/docs/development.md +++ b/docs/development.md @@ -75,21 +75,21 @@ development (unresponsive or slow upstream), conduwuit-specific usecases, or lack of time to upstream some things. - [ruma/ruma][1]: - various performance -improvements, more features, faster-paced development, client/server interop +improvements, more features, faster-paced development, better client/server interop hacks upstream won't accept, etc - [facebook/rocksdb][2]: - liburing -build fixes, GCC build fix, and logging callback C API for Rust tracing -integration +build fixes and GCC debug build fix - [tikv/jemallocator][3]: - musl -builds seem to be broken on upstream +builds seem to be broken on upstream, fixes some broken/suspicious code in +places, additional safety measures, and support redzones for Valgrind - [zyansheep/rustyline-async][4]: - tab completion callback and -`CTRL+\` signal quit event for CLI +`CTRL+\` signal quit event for conduwuit console CLI - [rust-rocksdb/rust-rocksdb][5]: - - [`@zaidoon1`'s][8] fork -has quicker updates, more up to date dependencies. Our changes fix musl build -issues, Rust part of the logging callback C API, removes unnecessary `gtest` -include, and uses our RocksDB and jemallocator + - [`@zaidoon1`][8]'s fork +has quicker updates, more up to date dependencies, etc. Our fork fixes musl build +issues, removes unnecessary `gtest` include, and uses our RocksDB and jemallocator +forks. - [tokio-rs/tracing][6]: - Implements `Clone` for `EnvFilter` to support dynamically changing tracing envfilter's alongside other logging/metrics things @@ -103,12 +103,16 @@ tokio_unstable` flag to enable experimental tokio APIs. A build might look like this: ```bash -RUSTFLAGS="--cfg tokio_unstable" cargo build \ +RUSTFLAGS="--cfg tokio_unstable" cargo +nightly build \ --release \ --no-default-features \ --features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console ``` +You will also need to enable the `tokio_console` config option in conduwuit when +starting it. This was due to tokio-console causing gradual memory leak/usage +if left enabled. + [1]: https://github.com/ruma/ruma/ [2]: https://github.com/facebook/rocksdb/ [3]: https://github.com/tikv/jemallocator/ diff --git a/docs/development/testing.md b/docs/development/testing.md index 06720dd8..2d421767 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -5,8 +5,8 @@ Have a look at [Complement's repository][complement] for an explanation of what it is. -To test against Complement, with Nix (or [Lix](https://lix.systems) and direnv installed -and set up, you can: +To test against Complement, with Nix (or [Lix](https://lix.systems) and direnv +installed and set up, you can: * Run `./bin/complement "$COMPLEMENT_SRC" ./path/to/logs.jsonl ./path/to/results.jsonl` to build a Complement image, run the tests, and output diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 74e19de7..c8655e06 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -106,24 +106,46 @@ Various debug commands can be found in `!admin debug`. #### Debug/Trace log level -conduwuit builds without debug or trace log levels by default for at least -performance reasons. This may change in the future and/or binaries providing -such configurations may be provided. If you need to access debug/trace log -levels, you will need to build without the `release_max_log_level` feature. +conduwuit builds without debug or trace log levels at compile time by default +for substantial performance gains in CPU usage and improved compile times. If +you need to access debug/trace log levels, you will need to build without the +`release_max_log_level` feature or use our provided static debug binaries. #### Changing log level dynamically conduwuit supports changing the tracing log environment filter on-the-fly using -the admin command `!admin debug change-log-level`. This accepts a string -**without quotes** the same format as the `log` config option. +the admin command `!admin debug change-log-level `. This accepts +a string **without quotes** the same format as the `log` config option. + +Example: `!admin debug change-log-level debug` + +This can also accept complex filters such as: +`!admin debug change-log-level info,conduit_service[{dest="example.com"}]=trace,ruma_state_res=trace` +`!admin debug change-log-level info,conduit_service[{dest="example.com"}]=trace,conduit_service[send{dest="example.org"}]=trace` + +And to reset the log level to the one that was set at startup / last config +load, simply pass the `--reset` flag. + +`!admin debug change-log-level --reset` #### Pinging servers -conduwuit can ping other servers using `!admin debug ping`. This takes a server -name and goes through the server discovery process and queries +conduwuit can ping other servers using `!admin debug ping `. This takes +a server name and goes through the server discovery process and queries `/_matrix/federation/v1/version`. Errors are outputted. +While it does measure the latency of the request, it is not indicative of +server performance on either side as that endpoint is completely unauthenticated +and simply fetches a string on a static JSON endpoint. It is very low cost both +bandwidth and computationally. + #### Allocator memory stats -When using jemalloc with jemallocator's `stats` feature, you can see conduwuit's -jemalloc memory stats by using `!admin debug memory-stats` +When using jemalloc with jemallocator's `stats` feature (`--enable-stats`), you +can see conduwuit's high-level allocator stats by using +`!admin server memory-usage` at the bottom. + +If you are a developer, you can also view the raw jemalloc statistics with +`!admin debug memory-stats`. Please note that this output is extremely large +which may only be visible in the conduwuit console CLI due to PDU size limits, +and is not easy for non-developers to understand. From 175e1c6453ac5ebe89f630acb7cbda0bec921d93 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 23 Nov 2024 22:36:22 -0500 Subject: [PATCH 1102/2091] correct admin cmd getting version and bin name Signed-off-by: strawberry --- src/admin/admin.rs | 2 +- src/main/clap.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/admin/admin.rs b/src/admin/admin.rs index fa497205..d1d8d394 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -9,7 +9,7 @@ use crate::{ }; #[derive(Debug, Parser)] -#[command(name = "admin", version = env!("CARGO_PKG_VERSION"))] +#[command(name = "conduwuit", version = conduit::version())] pub(super) enum AdminCommand { #[command(subcommand)] /// - Commands for managing appservices diff --git a/src/main/clap.rs b/src/main/clap.rs index 86b9fbd6..638398dd 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -10,7 +10,7 @@ use conduit::{ /// Commandline arguments #[derive(Parser, Debug)] -#[clap(version = conduit::version(), about, long_about = None)] +#[clap(version = conduit::version(), about, long_about = None, name = "conduwuit")] pub(crate) struct Args { #[arg(short, long)] /// Path to the config TOML file (optional) From 5f1cab6850cb78d4d4722315c914faf37fb809e3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 23 Nov 2024 03:45:11 +0000 Subject: [PATCH 1103/2091] passthru worker thread count from env Signed-off-by: Jason Volk --- Cargo.toml | 9 ++++----- src/main/clap.rs | 8 +++++++- src/main/main.rs | 5 ++--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ed24435f..058a15e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -207,14 +207,13 @@ default-features = false version = "4.5.21" default-features = false features = [ - "std", "derive", - "help", - #"color", Do we need these? - #"unicode", - "usage", + "env", "error-context", + "help", + "std", "string", + "usage", ] [workspace.dependencies.futures] diff --git a/src/main/clap.rs b/src/main/clap.rs index 638398dd..b10242be 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -5,7 +5,9 @@ use std::path::PathBuf; use clap::Parser; use conduit::{ config::{Figment, FigmentValue}, - err, toml, Err, Result, + err, toml, + utils::available_parallelism, + Err, Result, }; /// Commandline arguments @@ -32,6 +34,10 @@ pub(crate) struct Args { /// Set functional testing modes if available. Ex '--test=smoke' #[arg(long, hide(true))] pub(crate) test: Vec, + + /// Override the tokio worker_thread count. + #[arg(long, hide(true), env = "TOKIO_WORKER_THREADS", default_value = available_parallelism().to_string())] + pub(crate) worker_threads: usize, } /// Parse commandline arguments into structured data diff --git a/src/main/main.rs b/src/main/main.rs index 8e644a15..32d122f6 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -9,12 +9,11 @@ mod tracing; extern crate conduit_core as conduit; use std::{ - cmp, sync::{atomic::Ordering, Arc}, time::Duration, }; -use conduit::{debug_info, error, rustc_flags_capture, utils::available_parallelism, Error, Result}; +use conduit::{debug_info, error, rustc_flags_capture, Error, Result}; use server::Server; use tokio::runtime; @@ -30,7 +29,7 @@ fn main() -> Result<(), Error> { .enable_io() .enable_time() .thread_name(WORKER_NAME) - .worker_threads(cmp::max(WORKER_MIN, available_parallelism())) + .worker_threads(args.worker_threads.max(WORKER_MIN)) .thread_keep_alive(Duration::from_secs(WORKER_KEEPALIVE)) .build() .expect("built runtime"); From f30b08f015b00de2292d2b6938e352f0eb8d7daa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 24 Nov 2024 00:19:55 +0000 Subject: [PATCH 1104/2091] fix optional config section related split api/client well_known simplify well_known config access Signed-off-by: Jason Volk --- src/api/client/mod.rs | 2 + src/api/client/session.rs | 6 +- src/api/client/unversioned.rs | 104 +-------------------------------- src/api/client/well_known.rs | 105 ++++++++++++++++++++++++++++++++++ src/api/server/well_known.rs | 2 +- src/core/config/mod.rs | 26 ++++++--- src/router/serve/mod.rs | 2 +- src/router/serve/tls.rs | 19 +++--- src/service/globals/mod.rs | 16 +----- 9 files changed, 144 insertions(+), 138 deletions(-) create mode 100644 src/api/client/well_known.rs diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index 9ee88bec..3c9736ea 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -37,6 +37,7 @@ pub(super) mod unstable; pub(super) mod unversioned; pub(super) mod user_directory; pub(super) mod voip; +pub(super) mod well_known; pub use account::full_user_deactivate; pub(super) use account::*; @@ -80,6 +81,7 @@ pub(super) use unstable::*; pub(super) use unversioned::*; pub(super) use user_directory::*; pub(super) use voip::*; +pub(super) use well_known::*; /// generated device ID length const DEVICE_ID_LENGTH: usize = 10; diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 6347a2c9..573f3d97 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -198,8 +198,10 @@ pub(crate) async fn login_route( // send client well-known if specified so the client knows to reconfigure itself let client_discovery_info: Option = services - .globals - .well_known_client() + .server + .config + .well_known + .client .as_ref() .map(|server| DiscoveryInfo::new(HomeserverInfo::new(server.to_string()))); diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 3aee30c8..ed3ce37a 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -2,16 +2,9 @@ use std::collections::BTreeMap; use axum::{extract::State, response::IntoResponse, Json}; use futures::StreamExt; -use ruma::api::client::{ - discovery::{ - discover_homeserver::{self, HomeserverInfo, SlidingSyncProxyInfo}, - discover_support::{self, Contact}, - get_supported_versions, - }, - error::ErrorKind, -}; +use ruma::api::client::discovery::get_supported_versions; -use crate::{Error, Result, Ruma}; +use crate::{Result, Ruma}; /// # `GET /_matrix/client/versions` /// @@ -65,99 +58,6 @@ pub(crate) async fn get_supported_versions_route( Ok(resp) } -/// # `GET /.well-known/matrix/client` -/// -/// Returns the .well-known URL if it is configured, otherwise returns 404. -pub(crate) async fn well_known_client( - State(services): State, _body: Ruma, -) -> Result { - let client_url = match services.globals.well_known_client() { - Some(url) => url.to_string(), - None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), - }; - - Ok(discover_homeserver::Response { - homeserver: HomeserverInfo { - base_url: client_url.clone(), - }, - identity_server: None, - sliding_sync_proxy: Some(SlidingSyncProxyInfo { - url: client_url, - }), - tile_server: None, - }) -} - -/// # `GET /.well-known/matrix/support` -/// -/// Server support contact and support page of a homeserver's domain. -pub(crate) async fn well_known_support( - State(services): State, _body: Ruma, -) -> Result { - let support_page = services - .globals - .well_known_support_page() - .as_ref() - .map(ToString::to_string); - - let role = services.globals.well_known_support_role().clone(); - - // support page or role must be either defined for this to be valid - if support_page.is_none() && role.is_none() { - return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); - } - - let email_address = services.globals.well_known_support_email().clone(); - let matrix_id = services.globals.well_known_support_mxid().clone(); - - // if a role is specified, an email address or matrix id is required - if role.is_some() && (email_address.is_none() && matrix_id.is_none()) { - return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); - } - - // TOOD: support defining multiple contacts in the config - let mut contacts: Vec = vec![]; - - if let Some(role) = role { - let contact = Contact { - role, - email_address, - matrix_id, - }; - - contacts.push(contact); - } - - // support page or role+contacts must be either defined for this to be valid - if contacts.is_empty() && support_page.is_none() { - return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); - } - - Ok(discover_support::Response { - contacts, - support_page, - }) -} - -/// # `GET /client/server.json` -/// -/// Endpoint provided by sliding sync proxy used by some clients such as Element -/// Web as a non-standard health check. -pub(crate) async fn syncv3_client_server_json(State(services): State) -> Result { - let server_url = match services.globals.well_known_client() { - Some(url) => url.to_string(), - None => match services.globals.well_known_server() { - Some(url) => url.to_string(), - None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), - }, - }; - - Ok(Json(serde_json::json!({ - "server": server_url, - "version": conduit::version(), - }))) -} - /// # `GET /_conduwuit/server_version` /// /// Conduwuit-specific API to get the server version, results akin to diff --git a/src/api/client/well_known.rs b/src/api/client/well_known.rs new file mode 100644 index 00000000..674c9bb0 --- /dev/null +++ b/src/api/client/well_known.rs @@ -0,0 +1,105 @@ +use axum::{extract::State, response::IntoResponse, Json}; +use ruma::api::client::{ + discovery::{ + discover_homeserver::{self, HomeserverInfo, SlidingSyncProxyInfo}, + discover_support::{self, Contact}, + }, + error::ErrorKind, +}; + +use crate::{Error, Result, Ruma}; + +/// # `GET /.well-known/matrix/client` +/// +/// Returns the .well-known URL if it is configured, otherwise returns 404. +pub(crate) async fn well_known_client( + State(services): State, _body: Ruma, +) -> Result { + let client_url = match services.server.config.well_known.client.as_ref() { + Some(url) => url.to_string(), + None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), + }; + + Ok(discover_homeserver::Response { + homeserver: HomeserverInfo { + base_url: client_url.clone(), + }, + identity_server: None, + sliding_sync_proxy: Some(SlidingSyncProxyInfo { + url: client_url, + }), + tile_server: None, + }) +} + +/// # `GET /.well-known/matrix/support` +/// +/// Server support contact and support page of a homeserver's domain. +pub(crate) async fn well_known_support( + State(services): State, _body: Ruma, +) -> Result { + let support_page = services + .server + .config + .well_known + .support_page + .as_ref() + .map(ToString::to_string); + + let role = services.server.config.well_known.support_role.clone(); + + // support page or role must be either defined for this to be valid + if support_page.is_none() && role.is_none() { + return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); + } + + let email_address = services.server.config.well_known.support_email.clone(); + let matrix_id = services.server.config.well_known.support_mxid.clone(); + + // if a role is specified, an email address or matrix id is required + if role.is_some() && (email_address.is_none() && matrix_id.is_none()) { + return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); + } + + // TOOD: support defining multiple contacts in the config + let mut contacts: Vec = vec![]; + + if let Some(role) = role { + let contact = Contact { + role, + email_address, + matrix_id, + }; + + contacts.push(contact); + } + + // support page or role+contacts must be either defined for this to be valid + if contacts.is_empty() && support_page.is_none() { + return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); + } + + Ok(discover_support::Response { + contacts, + support_page, + }) +} + +/// # `GET /client/server.json` +/// +/// Endpoint provided by sliding sync proxy used by some clients such as Element +/// Web as a non-standard health check. +pub(crate) async fn syncv3_client_server_json(State(services): State) -> Result { + let server_url = match services.server.config.well_known.client.as_ref() { + Some(url) => url.to_string(), + None => match services.server.config.well_known.server.as_ref() { + Some(url) => url.to_string(), + None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), + }, + }; + + Ok(Json(serde_json::json!({ + "server": server_url, + "version": conduit::version(), + }))) +} diff --git a/src/api/server/well_known.rs b/src/api/server/well_known.rs index 2cc8f238..e6145aea 100644 --- a/src/api/server/well_known.rs +++ b/src/api/server/well_known.rs @@ -10,7 +10,7 @@ pub(crate) async fn well_known_server( State(services): State, _body: Ruma, ) -> Result { Ok(discover_homeserver::Response { - server: match services.globals.well_known_server() { + server: match services.server.config.well_known.server.as_ref() { Some(server_name) => server_name.to_owned(), None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), }, diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 66c78440..1754581d 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -87,7 +87,8 @@ pub struct Config { port: ListeningPort, // external structure; separate section - pub tls: Option, + #[serde(default)] + pub tls: TlsConfig, /// Uncomment unix_socket_path to listen on a UNIX socket at the specified /// path. If listening on a UNIX socket, you MUST remove/comment the @@ -1500,17 +1501,19 @@ pub struct Config { catchall: BTreeMap, } -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, Default)] #[config_example_generator(filename = "conduwuit-example.toml", section = "global.tls")] pub struct TlsConfig { /// Path to a valid TLS certificate file. /// /// example: "/path/to/my/certificate.crt" - pub certs: String, + pub certs: Option, + /// Path to a valid TLS certificate private key. /// /// example: "/path/to/my/certificate.key" - pub key: String, + pub key: Option, + /// Whether to listen and allow for HTTP and HTTPS connections (insecure!) #[serde(default)] pub dual_protocol: bool, @@ -1520,20 +1523,25 @@ pub struct TlsConfig { #[derive(Clone, Debug, Deserialize, Default)] #[config_example_generator(filename = "conduwuit-example.toml", section = "global.well_known")] pub struct WellKnownConfig { + /// The server URL that the client well-known file will serve. This should + /// not contain a port, and should just be a valid HTTPS URL. + /// + /// example: "https://matrix.example.com" + pub client: Option, + /// The server base domain of the URL with a specific port that the server /// well-known file will serve. This should contain a port at the end, and /// should not be a URL. /// /// example: "matrix.example.com:443" pub server: Option, - /// The server URL that the client well-known file will serve. This should - /// not contain a port, and should just be a valid HTTPS URL. - /// - /// example: "https://matrix.example.com" - pub client: Option, + pub support_page: Option, + pub support_role: Option, + pub support_email: Option, + pub support_mxid: Option, } diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index 858d3455..b0254772 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -23,7 +23,7 @@ pub(super) async fn serve( if cfg!(unix) && config.unix_socket_path.is_some() { unix::serve(server, app, shutdown).await - } else if config.tls.is_some() { + } else if config.tls.certs.is_some() { #[cfg(feature = "direct_tls")] return tls::serve(server, app, handle, addrs).await; diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index 08c5e7b6..f8d69048 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -6,17 +6,20 @@ use axum_server_dual_protocol::{ axum_server::{bind_rustls, tls_rustls::RustlsConfig}, ServerExt, }; -use conduit::{Result, Server}; +use conduit::{err, Result, Server}; use tokio::task::JoinSet; use tracing::{debug, info, warn}; -pub(super) async fn serve( - server: &Arc, app: Router, handle: ServerHandle, addrs: Vec, -) -> Result<()> { - let config = &server.config; - let tls = config.tls.as_ref().expect("TLS configuration"); - let certs = &tls.certs; - let key = &tls.key; +pub(super) async fn serve(server: &Arc, app: Router, handle: ServerHandle, addrs: Vec) -> Result { + let tls = &server.config.tls; + let certs = tls + .certs + .as_ref() + .ok_or(err!(Config("tls.certs", "Missing required value in tls config section")))?; + let key = tls + .key + .as_ref() + .ok_or(err!(Config("tls.key", "Missing required value in tls config section")))?; // we use ring for ruma and hashing state, but aws-lc-rs is the new default. // without this, TLS mode will panic. diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 55dd10aa..3eefe4b7 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -12,11 +12,9 @@ use data::Data; use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ - api::client::discovery::discover_support::ContactRole, OwnedEventId, OwnedRoomAliasId, OwnedServerName, - OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, + OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, }; use tokio::sync::Mutex; -use url::Url; use crate::service; @@ -243,14 +241,6 @@ impl Service { pub fn allow_outgoing_read_receipts(&self) -> bool { self.config.allow_outgoing_read_receipts } - pub fn well_known_support_page(&self) -> &Option { &self.config.well_known.support_page } - - pub fn well_known_support_role(&self) -> &Option { &self.config.well_known.support_role } - - pub fn well_known_support_email(&self) -> &Option { &self.config.well_known.support_email } - - pub fn well_known_support_mxid(&self) -> &Option { &self.config.well_known.support_mxid } - pub fn block_non_admin_invites(&self) -> bool { self.config.block_non_admin_invites } pub fn supported_room_versions(&self) -> Vec { @@ -265,10 +255,6 @@ impl Service { } } - pub fn well_known_client(&self) -> &Option { &self.config.well_known.client } - - pub fn well_known_server(&self) -> &Option { &self.config.well_known.server } - #[inline] pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool { for cidr in &self.cidr_range_denylist { From fd4c447a2d9f847c5b0e4de183e4ee95fcbda677 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 23 Nov 2024 23:25:14 +0000 Subject: [PATCH 1105/2091] move attribute argument extractor to utils Signed-off-by: Jason Volk --- src/macros/config.rs | 47 ++++++-------------------------------------- src/macros/utils.rs | 34 +++++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/src/macros/config.rs b/src/macros/config.rs index d7f11535..2934a0b2 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -1,9 +1,4 @@ -use std::{ - collections::{HashMap, HashSet}, - fmt::Write as _, - fs::OpenOptions, - io::Write as _, -}; +use std::{collections::HashSet, fmt::Write as _, fs::OpenOptions, io::Write as _}; use proc_macro::TokenStream; use proc_macro2::Span; @@ -13,7 +8,10 @@ use syn::{ ItemStruct, Lit, Meta, MetaList, MetaNameValue, Type, TypePath, }; -use crate::{utils::is_cargo_build, Result}; +use crate::{ + utils::{get_simple_settings, is_cargo_build}, + Result, +}; const UNDOCUMENTED: &str = "# This item is undocumented. Please contribute documentation for it."; @@ -29,7 +27,7 @@ pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result Result<()> { - let settings = get_settings(args); + let settings = get_simple_settings(args); let filename = settings .get("filename") @@ -120,39 +118,6 @@ fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { Ok(()) } -fn get_settings(args: &[Meta]) -> HashMap { - let mut map = HashMap::new(); - for arg in args { - let Meta::NameValue(MetaNameValue { - path, - value, - .. - }) = arg - else { - continue; - }; - - let Expr::Lit( - ExprLit { - lit: Lit::Str(str), - .. - }, - .., - ) = value - else { - continue; - }; - - let Some(key) = path.segments.iter().next().map(|s| s.ident.clone()) else { - continue; - }; - - map.insert(key.to_string(), str.value()); - } - - map -} - fn get_default(field: &Field) -> Option { for attr in &field.attrs { let Meta::List(MetaList { diff --git a/src/macros/utils.rs b/src/macros/utils.rs index e4ffc622..23c4c16f 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -1,7 +1,39 @@ -use syn::{parse_str, Expr, Generics, Lit, Meta}; +use std::collections::HashMap; + +use syn::{parse_str, Expr, ExprLit, Generics, Lit, Meta, MetaNameValue}; use crate::Result; +pub(crate) fn get_simple_settings(args: &[Meta]) -> HashMap { + args.iter().fold(HashMap::new(), |mut map, arg| { + let Meta::NameValue(MetaNameValue { + path, + value, + .. + }) = arg + else { + return map; + }; + + let Expr::Lit( + ExprLit { + lit: Lit::Str(str), + .. + }, + .., + ) = value + else { + return map; + }; + + if let Some(key) = path.segments.iter().next().map(|s| s.ident.clone()) { + map.insert(key.to_string(), str.value()); + } + + map + }) +} + pub(crate) fn is_cargo_build() -> bool { std::env::args() .find(|flag| flag.starts_with("--emit")) From 5da42fb859ba987c3488cfcbb7c1f65bca06ffa8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 19 Nov 2024 03:08:09 +0000 Subject: [PATCH 1106/2091] refactor account_data.changes_since to stream Signed-off-by: Jason Volk --- src/admin/query/account_data.rs | 6 ++-- src/api/client/sync/v3.rs | 14 ++++---- src/api/client/sync/v4.rs | 21 +++++------- src/service/account_data/mod.rs | 59 +++++++++++---------------------- 4 files changed, 39 insertions(+), 61 deletions(-) diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index ea45eb16..91217334 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -1,5 +1,6 @@ use clap::Subcommand; use conduit::Result; +use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId, UserId}; use crate::Command; @@ -39,10 +40,11 @@ pub(super) async fn process(subcommand: AccountDataCommand, context: &Command<'_ room_id, } => { let timer = tokio::time::Instant::now(); - let results = services + let results: Vec<_> = services .account_data .changes_since(room_id.as_deref(), &user_id, since) - .await?; + .collect() + .await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 7a78ea74..614970f0 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -275,10 +275,9 @@ pub(crate) async fn sync_events_route( events: services .account_data .changes_since(None, &sender_user, since) - .await? - .into_iter() - .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) - .collect(), + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) + .collect() + .await, }, device_lists: DeviceLists { changed: device_list_updates.into_iter().collect(), @@ -1023,10 +1022,9 @@ async fn load_joined_room( events: services .account_data .changes_since(Some(room_id), sender_user, since) - .await? - .into_iter() - .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) - .collect(), + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect() + .await, }, summary: RoomSummary { heroes, diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 57edc953..78b0b277 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -136,10 +136,9 @@ pub(crate) async fn sync_events_v4_route( account_data.global = services .account_data .changes_since(None, sender_user, globalsince) - .await? - .into_iter() - .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) - .collect(); + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) + .collect() + .await; if let Some(rooms) = body.extensions.account_data.rooms { for room in rooms { @@ -148,10 +147,9 @@ pub(crate) async fn sync_events_v4_route( services .account_data .changes_since(Some(&room), sender_user, globalsince) - .await? - .into_iter() - .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) - .collect(), + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect() + .await, ); } } @@ -487,10 +485,9 @@ pub(crate) async fn sync_events_v4_route( services .account_data .changes_since(Some(room_id), sender_user, *roomsince) - .await? - .into_iter() - .filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) - .collect(), + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect() + .await, ); let vector: Vec<_> = services diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index ac3f5f83..b752f9b8 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -1,12 +1,12 @@ -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; use conduit::{ - implement, - utils::{stream::TryIgnore, ReadyExt}, - Err, Error, Result, + err, implement, + utils::{result::LogErr, stream::TryIgnore, ReadyExt}, + Err, Result, }; -use database::{Deserialized, Handle, Json, Map}; -use futures::{StreamExt, TryFutureExt}; +use database::{Deserialized, Handle, Interfix, Json, Map}; +use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{ events::{ AnyGlobalAccountDataEvent, AnyRawAccountDataEvent, AnyRoomAccountDataEvent, GlobalAccountDataEventType, @@ -112,46 +112,27 @@ pub async fn get_raw(&self, room_id: Option<&RoomId>, user_id: &UserId, kind: &s /// Returns all changes to the account data that happened after `since`. #[implement(Service)] -pub async fn changes_since( - &self, room_id: Option<&RoomId>, user_id: &UserId, since: u64, -) -> Result> { - let mut userdata = HashMap::new(); - - let mut prefix = room_id - .map(ToString::to_string) - .unwrap_or_default() - .as_bytes() - .to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(user_id.as_bytes()); - prefix.push(0xFF); +pub fn changes_since<'a>( + &'a self, room_id: Option<&'a RoomId>, user_id: &'a UserId, since: u64, +) -> impl Stream + Send + 'a { + let prefix = (room_id, user_id, Interfix); + let prefix = database::serialize_to_vec(prefix).expect("failed to serialize prefix"); // Skip the data that's exactly at since, because we sent that last time - let mut first_possible = prefix.clone(); - first_possible.extend_from_slice(&(since.saturating_add(1)).to_be_bytes()); + let first_possible = (room_id, user_id, since.saturating_add(1)); self.db .roomuserdataid_accountdata - .raw_stream_from(&first_possible) + .stream_from_raw(&first_possible) .ignore_err() .ready_take_while(move |(k, _)| k.starts_with(&prefix)) - .map(|(k, v)| { - let v = match room_id { - None => serde_json::from_slice::>(v) - .map(AnyRawAccountDataEvent::Global) - .map_err(|_| Error::bad_database("Database contains invalid account data."))?, - Some(_) => serde_json::from_slice::>(v) - .map(AnyRawAccountDataEvent::Room) - .map_err(|_| Error::bad_database("Database contains invalid account data."))?, - }; - - Ok((k.to_owned(), v)) + .map(move |(_, v)| { + match room_id { + Some(_) => serde_json::from_slice::>(v).map(AnyRawAccountDataEvent::Room), + None => serde_json::from_slice::>(v).map(AnyRawAccountDataEvent::Global), + } + .map_err(|e| err!(Database("Database contains invalid account data: {e}"))) + .log_err() }) .ignore_err() - .ready_for_each(|(kind, data)| { - userdata.insert(kind, data); - }) - .await; - - Ok(userdata.into_values().collect()) } From 3789d60b6abf7d758bb75f898ccbaa7f1b4251aa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 22 Nov 2024 12:25:46 +0000 Subject: [PATCH 1107/2091] refactor to iterator inputs for auth_chain/short batch functions Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 3 +- src/api/server/event_auth.rs | 4 +- src/api/server/send_join.rs | 6 +- src/api/server/state.rs | 4 +- src/api/server/state_ids.rs | 4 +- src/database/map/get.rs | 8 +- src/service/rooms/auth_chain/mod.rs | 32 +++++--- .../rooms/event_handler/resolve_state.rs | 4 +- .../rooms/event_handler/state_at_incoming.rs | 2 +- src/service/rooms/short/mod.rs | 80 +++++++++---------- 10 files changed, 76 insertions(+), 71 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index f9d4a521..89e47d4e 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -1,6 +1,7 @@ use std::{ collections::HashMap, fmt::Write, + iter::once, sync::Arc, time::{Instant, SystemTime}, }; @@ -43,7 +44,7 @@ pub(super) async fn get_auth_chain(&self, event_id: Box) -> Result = state_ids.values().map(Borrow::borrow).collect(); + let starting_events = state_ids.values().map(Borrow::borrow); let auth_chain = services .rooms .auth_chain - .event_ids_iter(room_id, &starting_events) + .event_ids_iter(room_id, starting_events) .await? .map(Ok) .and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) diff --git a/src/api/server/state.rs b/src/api/server/state.rs index 06a44a99..b21fce68 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -1,4 +1,4 @@ -use std::borrow::Borrow; +use std::{borrow::Borrow, iter::once}; use axum::extract::State; use conduit::{err, result::LogErr, utils::IterStream, Result}; @@ -52,7 +52,7 @@ pub(crate) async fn get_room_state_route( let auth_chain = services .rooms .auth_chain - .event_ids_iter(&body.room_id, &[body.event_id.borrow()]) + .event_ids_iter(&body.room_id, once(body.event_id.borrow())) .await? .map(Ok) .and_then(|id| async move { services.rooms.timeline.get_pdu_json(&id).await }) diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index 52d8e7cc..0c023bf0 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -1,4 +1,4 @@ -use std::borrow::Borrow; +use std::{borrow::Borrow, iter::once}; use axum::extract::State; use conduit::{err, Result}; @@ -44,7 +44,7 @@ pub(crate) async fn get_room_state_ids_route( let auth_chain_ids = services .rooms .auth_chain - .event_ids_iter(&body.room_id, &[body.event_id.borrow()]) + .event_ids_iter(&body.room_id, once(body.event_id.borrow())) .await? .map(|id| (*id).to_owned()) .collect() diff --git a/src/database/map/get.rs b/src/database/map/get.rs index a3c6c492..3ee2a194 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -80,8 +80,8 @@ where #[tracing::instrument(skip(self, keys), fields(%self), level = "trace")] pub fn get_batch<'a, I, K>(&self, keys: I) -> impl Stream>> where - I: Iterator + ExactSizeIterator + Send + Debug, - K: AsRef<[u8]> + Send + Sync + Sized + Debug + 'a, + I: Iterator + ExactSizeIterator + Debug + Send, + K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, { self.get_batch_blocking(keys).stream() } @@ -89,8 +89,8 @@ where #[implement(super::Map)] pub fn get_batch_blocking<'a, I, K>(&self, keys: I) -> impl Iterator>> where - I: Iterator + ExactSizeIterator + Send, - K: AsRef<[u8]> + Sized + 'a, + I: Iterator + ExactSizeIterator + Debug + Send, + K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, { // Optimization can be `true` if key vector is pre-sorted **by the column // comparator**. diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index cabb6f0c..1d0490c2 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -2,6 +2,7 @@ mod data; use std::{ collections::{BTreeSet, HashSet}, + fmt::Debug, sync::Arc, }; @@ -37,9 +38,12 @@ impl crate::Service for Service { } impl Service { - pub async fn event_ids_iter( - &self, room_id: &RoomId, starting_events: &[&EventId], - ) -> Result> + Send + '_> { + pub async fn event_ids_iter<'a, I>( + &'a self, room_id: &RoomId, starting_events: I, + ) -> Result> + Send + '_> + where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, + { let stream = self .get_event_ids(room_id, starting_events) .await? @@ -49,12 +53,15 @@ impl Service { Ok(stream) } - pub async fn get_event_ids(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result>> { + pub async fn get_event_ids<'a, I>(&'a self, room_id: &RoomId, starting_events: I) -> Result>> + where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, + { let chain = self.get_auth_chain(room_id, starting_events).await?; let event_ids = self .services .short - .multi_get_eventid_from_short(&chain) + .multi_get_eventid_from_short(chain.into_iter()) .await .into_iter() .filter_map(Result::ok) @@ -64,7 +71,10 @@ impl Service { } #[tracing::instrument(skip_all, name = "auth_chain")] - pub async fn get_auth_chain(&self, room_id: &RoomId, starting_events: &[&EventId]) -> Result> { + pub async fn get_auth_chain<'a, I>(&'a self, room_id: &RoomId, starting_events: I) -> Result> + where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, + { const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); @@ -72,19 +82,19 @@ impl Service { let mut starting_ids = self .services .short - .multi_get_or_create_shorteventid(starting_events) - .enumerate() + .multi_get_or_create_shorteventid(starting_events.clone()) + .zip(starting_events.clone().stream()) .boxed(); let mut buckets = [BUCKET; NUM_BUCKETS]; - while let Some((i, short)) = starting_ids.next().await { + while let Some((short, starting_event)) = starting_ids.next().await { let bucket: usize = short.try_into()?; let bucket: usize = validated!(bucket % NUM_BUCKETS); - buckets[bucket].insert((short, starting_events[i])); + buckets[bucket].insert((short, starting_event)); } debug!( - starting_events = ?starting_events.len(), + starting_events = ?starting_events.count(), elapsed = ?started.elapsed(), "start", ); diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 0c9525dd..4863e340 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -35,12 +35,12 @@ pub async fn resolve_state( let fork_states = [current_state_ids, incoming_state]; let mut auth_chain_sets = Vec::with_capacity(fork_states.len()); for state in &fork_states { - let starting_events: Vec<&EventId> = state.values().map(Borrow::borrow).collect(); + let starting_events = state.values().map(Borrow::borrow); let auth_chain: HashSet> = self .services .auth_chain - .get_event_ids(room_id, &starting_events) + .get_event_ids(room_id, starting_events) .await? .into_iter() .collect(); diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index a200ab56..05a9d8ca 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -139,7 +139,7 @@ pub(super) async fn state_at_incoming_resolved( let auth_chain: HashSet> = self .services .auth_chain - .get_event_ids(room_id, &starting_events) + .get_event_ids(room_id, starting_events.into_iter()) .await? .into_iter() .collect(); diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 703df796..e4ff2975 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,7 +1,7 @@ -use std::{mem::size_of_val, sync::Arc}; +use std::{fmt::Debug, mem::size_of_val, sync::Arc}; pub use conduit::pdu::{ShortEventId, ShortId, ShortRoomId}; -use conduit::{err, implement, utils, Result}; +use conduit::{err, implement, utils, utils::stream::ReadyExt, Result}; use database::{Deserialized, Map}; use futures::{Stream, StreamExt}; use ruma::{events::StateEventType, EventId, RoomId}; @@ -51,52 +51,46 @@ impl crate::Service for Service { #[implement(Service)] pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEventId { - const BUFSIZE: usize = size_of::(); - if let Ok(shorteventid) = self.get_shorteventid(event_id).await { return shorteventid; } - let shorteventid = self.services.globals.next_count().unwrap(); - debug_assert!(size_of_val(&shorteventid) == BUFSIZE, "buffer requirement changed"); - - self.db - .eventid_shorteventid - .raw_aput::(event_id, shorteventid); - - self.db - .shorteventid_eventid - .aput_raw::(shorteventid, event_id); - - shorteventid + self.create_shorteventid(event_id) } #[implement(Service)] -pub fn multi_get_or_create_shorteventid<'a>( - &'a self, event_ids: &'a [&EventId], -) -> impl Stream + Send + 'a { +pub fn multi_get_or_create_shorteventid<'a, I>(&'a self, event_ids: I) -> impl Stream + Send + '_ +where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, + ::Item: AsRef<[u8]> + Send + Sync + 'a, +{ self.db .eventid_shorteventid - .get_batch(event_ids.iter()) - .enumerate() - .map(|(i, result)| match result { - Ok(ref short) => utils::u64_from_u8(short), - Err(_) => { - const BUFSIZE: usize = size_of::(); - - let short = self.services.globals.next_count().unwrap(); - debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); - - self.db - .eventid_shorteventid - .raw_aput::(event_ids[i], short); - self.db - .shorteventid_eventid - .aput_raw::(short, event_ids[i]); - - short - }, + .get_batch(event_ids.clone()) + .ready_scan(event_ids, |event_ids, result| { + event_ids.next().map(|event_id| (event_id, result)) }) + .map(|(event_id, result)| match result { + Ok(ref short) => utils::u64_from_u8(short), + Err(_) => self.create_shorteventid(event_id), + }) +} + +#[implement(Service)] +fn create_shorteventid(&self, event_id: &EventId) -> ShortEventId { + const BUFSIZE: usize = size_of::(); + + let short = self.services.globals.next_count().unwrap(); + debug_assert!(size_of_val(&short) == BUFSIZE, "buffer requirement changed"); + + self.db + .eventid_shorteventid + .raw_aput::(event_id, short); + self.db + .shorteventid_eventid + .aput_raw::(short, event_id); + + short } #[implement(Service)] @@ -154,13 +148,13 @@ pub async fn get_eventid_from_short(&self, shorteventid: ShortEventId) -> Result } #[implement(Service)] -pub async fn multi_get_eventid_from_short(&self, shorteventid: &[ShortEventId]) -> Vec>> { +pub async fn multi_get_eventid_from_short(&self, shorteventid: I) -> Vec>> +where + I: Iterator + Send, +{ const BUFSIZE: usize = size_of::(); - let keys: Vec<[u8; BUFSIZE]> = shorteventid - .iter() - .map(|short| short.to_be_bytes()) - .collect(); + let keys: Vec<[u8; BUFSIZE]> = shorteventid.map(u64::to_be_bytes).collect(); self.db .shorteventid_eventid From c519a40cb8d220e9d3d18702f4b4f593e0280505 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 22 Nov 2024 16:01:27 +0000 Subject: [PATCH 1108/2091] use multiget for shortid conversions Signed-off-by: Jason Volk --- src/service/rooms/state/mod.rs | 71 +++++++++++------------ src/service/rooms/state_accessor/data.rs | 73 ++++++++++++++---------- src/service/rooms/state_accessor/mod.rs | 12 +++- 3 files changed, 86 insertions(+), 70 deletions(-) diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 9d702cd7..4429e912 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use conduit::{ - err, + at, err, result::FlatOk, utils::{calculate_hash, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, warn, PduEvent, Result, @@ -398,59 +398,52 @@ impl Service { return Ok(HashMap::new()); }; - let auth_events = state_res::auth_types_for_event(kind, sender, state_key, content)?; - - let mut sauthevents: HashMap<_, _> = auth_events + let mut sauthevents: HashMap<_, _> = state_res::auth_types_for_event(kind, sender, state_key, content)? .iter() .stream() .filter_map(|(event_type, state_key)| { self.services .short .get_shortstatekey(event_type, state_key) - .map_ok(move |s| (s, (event_type, state_key))) + .map_ok(move |ssk| (ssk, (event_type, state_key))) .map(Result::ok) }) + .map(|(ssk, (event_type, state_key))| (ssk, (event_type.to_owned(), state_key.to_owned()))) .collect() .await; - let full_state = self + let auth_state: Vec<_> = self .services - .state_compressor - .load_shortstatehash_info(shortstatehash) + .state_accessor + .state_full_shortids(shortstatehash) .await - .map_err(|e| { - err!(Database( - "Missing shortstatehash info for {room_id:?} at {shortstatehash:?}: {e:?}" - )) - })? - .pop() - .expect("there is always one layer") - .full_state; + .map_err(|e| err!(Database(error!(?room_id, ?shortstatehash, "{e:?}"))))? + .into_iter() + .filter_map(|(shortstatekey, shorteventid)| { + sauthevents + .remove(&shortstatekey) + .map(|(event_type, state_key)| ((event_type, state_key), shorteventid)) + }) + .collect(); - let mut ret = HashMap::new(); - for &compressed in full_state.iter() { - let (shortstatekey, shorteventid) = parse_compressed_state_event(compressed); + let auth_pdus: Vec<_> = self + .services + .short + .multi_get_eventid_from_short(auth_state.iter().map(at!(1))) + .await + .into_iter() + .stream() + .and_then(|event_id| async move { self.services.timeline.get_pdu(&event_id).await }) + .collect() + .await; - let Some((ty, state_key)) = sauthevents.remove(&shortstatekey) else { - continue; - }; + let auth_pdus = auth_state + .into_iter() + .map(at!(0)) + .zip(auth_pdus.into_iter()) + .filter_map(|((event_type, state_key), pdu)| Some(((event_type, state_key), pdu.ok()?))) + .collect(); - let Ok(event_id) = self - .services - .short - .get_eventid_from_short(shorteventid) - .await - else { - continue; - }; - - let Ok(pdu) = self.services.timeline.get_pdu(&event_id).await else { - continue; - }; - - ret.insert((ty.to_owned(), state_key.to_owned()), pdu); - } - - Ok(ret) + Ok(auth_pdus) } } diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 8df0d8b0..80046d77 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, sync::Arc}; use conduit::{ - err, - utils::{future::TryExtExt, IterStream}, + at, err, + utils::stream::{IterStream, ReadyExt}, PduEvent, Result, }; use database::{Deserialized, Map}; @@ -49,52 +49,63 @@ impl Data { pub(super) async fn state_full( &self, shortstatehash: ShortStateHash, ) -> Result>> { - Ok(self + let state = self .state_full_pdus(shortstatehash) .await? .into_iter() .filter_map(|pdu| Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu))) - .collect()) + .collect(); + + Ok(state) } pub(super) async fn state_full_pdus(&self, shortstatehash: ShortStateHash) -> Result>> { - Ok(self + let short_ids = self .state_full_shortids(shortstatehash) .await? - .iter() + .into_iter() + .map(at!(1)); + + let event_ids = self + .services + .short + .multi_get_eventid_from_short(short_ids) + .await; + + let full_pdus = event_ids + .into_iter() .stream() - .filter_map(|(_, shorteventid)| { - self.services - .short - .get_eventid_from_short(*shorteventid) - .ok() - }) - .filter_map(|eventid| async move { self.services.timeline.get_pdu(&eventid).await.ok() }) + .ready_filter_map(Result::ok) + .filter_map(|event_id| async move { self.services.timeline.get_pdu(&event_id).await.ok() }) .collect() - .await) + .await; + + Ok(full_pdus) } pub(super) async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result>> { - Ok(self - .state_full_shortids(shortstatehash) - .await? - .iter() - .stream() - .filter_map(|(shortstatekey, shorteventid)| { - self.services - .short - .get_eventid_from_short(*shorteventid) - .map_ok(move |eventid| (*shortstatekey, eventid)) - .ok() - }) - .collect() - .await) + let short_ids = self.state_full_shortids(shortstatehash).await?; + + let event_ids = self + .services + .short + .multi_get_eventid_from_short(short_ids.iter().map(at!(1))) + .await; + + let full_ids = short_ids + .into_iter() + .map(at!(0)) + .zip(event_ids.into_iter()) + .filter_map(|(shortstatekey, event_id)| Some((shortstatekey, event_id.ok()?))) + .collect(); + + Ok(full_ids) } pub(super) async fn state_full_shortids( &self, shortstatehash: ShortStateHash, ) -> Result> { - Ok(self + let shortids = self .services .state_compressor .load_shortstatehash_info(shortstatehash) @@ -106,7 +117,9 @@ impl Data { .iter() .copied() .map(parse_compressed_state_event) - .collect()) + .collect(); + + Ok(shortids) } /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 89db88a6..e08fac66 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -41,7 +41,10 @@ use serde::Deserialize; use self::data::Data; use crate::{ rooms, - rooms::{short::ShortStateHash, state::RoomMutexGuard}, + rooms::{ + short::{ShortEventId, ShortStateHash, ShortStateKey}, + state::RoomMutexGuard, + }, Dep, }; @@ -102,6 +105,13 @@ impl Service { self.db.state_full_ids(shortstatehash).await } + #[inline] + pub async fn state_full_shortids( + &self, shortstatehash: ShortStateHash, + ) -> Result> { + self.db.state_full_shortids(shortstatehash).await + } + pub async fn state_full( &self, shortstatehash: ShortStateHash, ) -> Result>> { From 97ad9afc866d420354c44c9ffddf99d622c38f4b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 24 Nov 2024 06:04:54 +0000 Subject: [PATCH 1109/2091] default to main event for batch tokens fix prev_batch token for legacy sync timeline Signed-off-by: Jason Volk --- src/api/client/context.rs | 2 ++ src/api/client/message.rs | 4 +--- src/api/client/sync/v3.rs | 1 - 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 4359ae12..5b6b516e 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -169,12 +169,14 @@ pub(crate) async fn get_context_route( start: events_before .last() .map(at!(0)) + .or(Some(base_token)) .as_ref() .map(ToString::to_string), end: events_after .last() .map(at!(0)) + .or(Some(base_token)) .as_ref() .map(ToString::to_string), diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 88453de0..f1a10aa2 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -136,8 +136,6 @@ pub(crate) async fn get_message_events_route( .collect() .await; - let start_token = events.first().map(at!(0)).unwrap_or(from); - let next_token = events.last().map(at!(0)); if !cfg!(feature = "element_hacks") { @@ -156,7 +154,7 @@ pub(crate) async fn get_message_events_route( .collect(); Ok(get_message_events::v3::Response { - start: start_token.to_string(), + start: from.to_string(), end: next_token.as_ref().map(ToString::to_string), chunk, state, diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 614970f0..80aa8184 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -945,7 +945,6 @@ async fn load_joined_room( let prev_batch = timeline_pdus .first() .map(at!(0)) - .map(|count| count.saturating_sub(1)) .as_ref() .map(ToString::to_string); From 4e74a1811b3f61a18445740a4d3a84a5006e8499 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 24 Nov 2024 22:15:25 +0000 Subject: [PATCH 1110/2091] ci: set cancel-in-progress to true Signed-off-by: Jason Volk --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9385c5e3..b4b2c1f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ on: concurrency: group: ${{ github.head_ref || github.ref_name }} - cancel-in-progress: false + cancel-in-progress: true env: # sccache only on main repo From a582d0559a7e8cdf6f98a33082fe39984d959666 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 23 Nov 2024 22:31:44 +0000 Subject: [PATCH 1111/2091] bump url and cargo lock Signed-off-by: Jason Volk --- Cargo.lock | 202 +++++++++++++++++++++++++---------------------------- Cargo.toml | 2 +- 2 files changed, 98 insertions(+), 106 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d39375d1..e3af8ae0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,9 +94,9 @@ checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" [[package]] name = "async-compression" -version = "0.4.17" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cb8f1d480b0ea3783ab015936d2a55c87e219676f0c0b7dec61494043f21857" +checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ "brotli", "flate2", @@ -127,7 +127,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -138,7 +138,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -164,21 +164,20 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7c2840b66236045acd2607d5866e274380afd87ef99d6226e961e2cb47df45" +checksum = "f47bb8cc16b669d267eeccf585aea077d0882f4777b1c1f740217885d6e6e5a3" dependencies = [ "aws-lc-sys", - "mirai-annotations", "paste", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.23.0" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad3a619a9de81e1d7de1f1186dcba4506ed661a0e483d84410fdef0ee87b2f96" +checksum = "a2101df3813227bbaaaa0b04cd61c534c7954b22bd68d399b440be937dc63ff7" dependencies = [ "bindgen", "cc", @@ -215,7 +214,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tower 0.5.1", "tower-layer", @@ -249,7 +248,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", "tracing", @@ -292,7 +291,7 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls 0.23.16", + "rustls 0.23.18", "rustls-pemfile", "rustls-pki-types", "tokio", @@ -312,7 +311,7 @@ dependencies = [ "http", "http-body-util", "pin-project", - "rustls 0.23.16", + "rustls 0.23.18", "tokio", "tokio-rustls", "tokio-util", @@ -372,7 +371,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.87", + "syn 2.0.89", "which", ] @@ -435,9 +434,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.19.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" [[package]] name = "byteorder" @@ -574,7 +573,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -755,7 +754,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -779,7 +778,7 @@ dependencies = [ "hyper-util", "log", "ruma", - "rustls 0.23.16", + "rustls 0.23.18", "sd-notify", "sentry", "sentry-tower", @@ -896,9 +895,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" dependencies = [ "core-foundation-sys", "libc", @@ -912,9 +911,9 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "cpufeatures" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca741a962e1b0bff6d724a1a0958b686406e853bb14061f218562e1896f95e6" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" dependencies = [ "libc", ] @@ -1053,7 +1052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1080,7 +1079,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1149,7 +1148,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1201,7 +1200,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1367,7 +1366,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1447,9 +1446,9 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "h2" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" dependencies = [ "atomic-waker", "bytes", @@ -1633,7 +1632,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -1728,7 +1727,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.16", + "rustls 0.23.18", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -1885,7 +1884,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2031,9 +2030,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2" [[package]] name = "jobserver" @@ -2086,9 +2085,9 @@ dependencies = [ [[package]] name = "konst" -version = "0.3.9" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50a0ba6de5f7af397afff922f22c149ff605c766cd3269cf6c1cd5e466dbe3b9" +checksum = "b65f00fb3910881e52bf0850ae2a82aea411488a557e1c02820ceaa60963dce3" dependencies = [ "const_panic", "konst_kernel", @@ -2097,9 +2096,9 @@ dependencies = [ [[package]] name = "konst_kernel" -version = "0.3.9" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be0a455a1719220fd6adf756088e1c69a85bf14b6a9e24537a5cc04f503edb2b" +checksum = "599c1232f55c72c7fc378335a3efe1c878c92720838c8e6a4fd87784ef7764de" dependencies = [ "typewit", ] @@ -2124,7 +2123,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2141,9 +2140,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.162" +version = "0.2.164" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" +checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" [[package]] name = "libloading" @@ -2180,9 +2179,9 @@ checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" [[package]] name = "lock_api" @@ -2338,12 +2337,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "mirai-annotations" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9be0862c1b3f26a88803c4a49de6889c10e608b3ee9344e6ef5b45fb37ad3d1" - [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2656,7 +2649,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2739,7 +2732,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2811,7 +2804,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2825,9 +2818,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.89" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" dependencies = [ "unicode-ident", ] @@ -2840,7 +2833,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "version_check", "yansi", ] @@ -2865,7 +2858,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -2918,7 +2911,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.0.0", - "rustls 0.23.16", + "rustls 0.23.18", "socket2", "thiserror 2.0.3", "tokio", @@ -2936,7 +2929,7 @@ dependencies = [ "rand", "ring", "rustc-hash 2.0.0", - "rustls 0.23.16", + "rustls 0.23.18", "rustls-pki-types", "slab", "thiserror 2.0.3", @@ -3079,14 +3072,14 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.16", + "rustls 0.23.18", "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.1", + "sync_wrapper 1.0.2", "tokio", "tokio-rustls", "tokio-socks", @@ -3285,7 +3278,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.87", + "syn 2.0.89", "toml", ] @@ -3409,9 +3402,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.40" +version = "0.38.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99e4ea3e1cdc4b559b8e5650f9c8e5998e3e5c1343b4eaf034565f32318d63c0" +checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" dependencies = [ "bitflags 2.6.0", "errno", @@ -3436,9 +3429,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.16" +version = "0.23.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee87ff5d9b36712a58574e12e9f0ea80f915a5b0ac518d322b24a465617925e" +checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" dependencies = [ "aws-lc-rs", "log", @@ -3452,12 +3445,11 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a" +checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" dependencies = [ "openssl-probe", - "rustls-pemfile", "rustls-pki-types", "schannel", "security-framework", @@ -3531,9 +3523,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01227be5826fa0690321a2ba6c5cd57a19cf3f6a09e76973b58e61de6ab9d1c1" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" dependencies = [ "windows-sys 0.59.0", ] @@ -3552,9 +3544,9 @@ checksum = "1be20c5f7f393ee700f8b2f28ea35812e4e212f40774b550cd2a93ea91684451" [[package]] name = "security-framework" -version = "2.11.1" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -3731,7 +3723,7 @@ checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4008,9 +4000,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.87" +version = "2.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" dependencies = [ "proc-macro2", "quote", @@ -4025,9 +4017,9 @@ checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" [[package]] name = "sync_wrapper" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" dependencies = [ "futures-core", ] @@ -4040,7 +4032,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4106,7 +4098,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4117,7 +4109,7 @@ checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4262,7 +4254,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4283,7 +4275,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.16", + "rustls 0.23.18", "rustls-pki-types", "tokio", ] @@ -4476,7 +4468,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b47 dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] @@ -4603,9 +4595,9 @@ checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unicode-normalization" @@ -4655,7 +4647,7 @@ dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.23.16", + "rustls 0.23.18", "rustls-pki-types", "url", "webpki-roots", @@ -4663,9 +4655,9 @@ dependencies = [ [[package]] name = "url" -version = "2.5.3" +version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", "idna 1.0.3", @@ -4762,7 +4754,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "wasm-bindgen-shared", ] @@ -4796,7 +4788,7 @@ checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4851,9 +4843,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.6" +version = "0.26.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "841c67bff177718f1d4dfefde8d8f0e78f9b6589319ba88312f567fc5841a958" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" dependencies = [ "rustls-pki-types", ] @@ -5157,9 +5149,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" dependencies = [ "serde", "stable_deref_trait", @@ -5169,13 +5161,13 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "synstructure", ] @@ -5197,27 +5189,27 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] name = "zerofrom" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", "synstructure", ] @@ -5246,7 +5238,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.87", + "syn 2.0.89", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 058a15e0..02c3b5ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -243,7 +243,7 @@ version = "0.8.5" # Validating urls in config, was already a transitive dependency [workspace.dependencies.url] -version = "2.5.3" +version = "2.5.4" default-features = false features = ["serde"] From 1c751168c6e237a29dd0cf00b771e8ea3ca961f1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 24 Nov 2024 23:49:09 +0000 Subject: [PATCH 1112/2091] check-in missed example config changes Signed-off-by: Jason Volk --- conduwuit-example.toml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 2f3da71f..78136efb 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -158,6 +158,10 @@ # #eventidshort_cache_capacity = varies by system +# This item is undocumented. Please contribute documentation for it. +# +#eventid_pdu_cache_capacity = varies by system + # This item is undocumented. Please contribute documentation for it. # #shortstatekey_cache_capacity = varies by system @@ -1350,6 +1354,13 @@ [global.well_known] +# The server URL that the client well-known file will serve. This should +# not contain a port, and should just be a valid HTTPS URL. +# +# example: "https://matrix.example.com" +# +#client = + # The server base domain of the URL with a specific port that the server # well-known file will serve. This should contain a port at the end, and # should not be a URL. @@ -1358,13 +1369,6 @@ # #server = -# The server URL that the client well-known file will serve. This should -# not contain a port, and should just be a valid HTTPS URL. -# -# example: "https://matrix.example.com" -# -#client = - # This item is undocumented. Please contribute documentation for it. # #support_page = From b87362cbf1377592e7939d9667721183c25d14a9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 24 Nov 2024 19:17:07 -0500 Subject: [PATCH 1113/2091] ci: add test for validating generated example config is current Signed-off-by: strawberry --- engage.toml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/engage.toml b/engage.toml index 633cb95d..9a6ef8ca 100644 --- a/engage.toml +++ b/engage.toml @@ -188,6 +188,16 @@ cargo test \ --color=always """ +# Checks if the generated example config differs from the checked in repo's +# example config. +[[task]] +name = "example-config" +group = "tests" +depends = ["cargo/default"] +script = """ +git diff --exit-code conduwuit-example.toml +""" + # Ensure that the flake's default output can build and run without crashing # # This is a dynamically-linked jemalloc build, which is a case not covered by From 2675033aac4caf9c77a1dc5024468cae71cf92bc Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 24 Nov 2024 19:19:08 -0500 Subject: [PATCH 1114/2091] send plain txt admin room error responses fixes bracketed arguments not showing up on missing args Signed-off-by: strawberry --- src/admin/processor.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/admin/processor.rs b/src/admin/processor.rs index 3c1895ff..caaa7d2d 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -157,10 +157,7 @@ fn parse<'a>( let message = error .to_string() .replace("server.name", services.globals.server_name().as_str()); - Err(reply( - RoomMessageEventContent::notice_markdown(message), - input.reply_id.as_deref(), - )) + Err(reply(RoomMessageEventContent::notice_plain(message), input.reply_id.as_deref())) }, } } From 29c715a45fe1c6c177b1ea88452a94feb26d5174 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 24 Nov 2024 19:30:54 -0500 Subject: [PATCH 1115/2091] ci: remove some old/unnecessary paths-ignore Signed-off-by: strawberry --- .github/workflows/ci.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4b2c1f1..1441dd44 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,15 +3,10 @@ name: CI and Artifacts on: pull_request: push: - # documentation workflow deals with this or is not relevant for this workflow paths-ignore: - - '*.md' - - 'conduwuit-example.toml' - - 'book.toml' - '.gitlab-ci.yml' - '.gitignore' - 'renovate.json' - - 'docs/**' - 'debian/**' - 'docker/**' branches: From 6f1d50dda31b4c5d55e2297c701520553ba8327e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 25 Nov 2024 01:14:12 +0000 Subject: [PATCH 1116/2091] panic on otherwise ignored errors in debug mode Signed-off-by: Jason Volk --- src/core/utils/stream/ignore.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/core/utils/stream/ignore.rs b/src/core/utils/stream/ignore.rs index 997aa4ba..4e426557 100644 --- a/src/core/utils/stream/ignore.rs +++ b/src/core/utils/stream/ignore.rs @@ -13,6 +13,15 @@ where T: Stream> + TryStream + Send + 'a, Item: Send + 'a, { + #[cfg(debug_assertions)] + #[inline] + fn ignore_err(self: T) -> impl Stream + Send + 'a { + use super::TryExpect; + + self.expect_ok() + } + + #[cfg(not(debug_assertions))] #[inline] fn ignore_err(self: T) -> impl Stream + Send + 'a { self.filter_map(|res| ready(res.ok())) } From 343ec59a8b6b766b913538811c1d7b04327e8382 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 25 Nov 2024 01:01:39 +0000 Subject: [PATCH 1117/2091] use arrayvec for integer deserialization buffer Signed-off-by: Jason Volk --- src/core/error/mod.rs | 2 ++ src/database/de.rs | 29 ++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 35bf9800..b84f1b46 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -35,6 +35,8 @@ pub enum Error { // third-party #[error(transparent)] + CapacityError(#[from] arrayvec::CapacityError), + #[error(transparent)] CargoToml(#[from] cargo_toml::Error), #[error(transparent)] Clap(#[from] clap::error::Error), diff --git a/src/database/de.rs b/src/database/de.rs index f8a038ef..d303eab2 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -1,3 +1,4 @@ +use arrayvec::ArrayVec; use conduit::{checked, debug::DebugInspect, err, utils::string, Error, Result}; use serde::{ de, @@ -52,7 +53,7 @@ impl<'de> Deserializer<'de> { let len = self.buf.len(); let parsed = &self.buf[0..pos]; let unparsed = &self.buf[pos..]; - let remain = checked!(len - pos)?; + let remain = self.remaining()?; let trailing_sep = remain == 1 && unparsed[0] == Self::SEP; (remain == 0 || trailing_sep) .then_some(()) @@ -139,6 +140,14 @@ impl<'de> Deserializer<'de> { self.pos = self.pos.saturating_add(n); debug_assert!(self.pos <= self.buf.len(), "pos out of range"); } + + /// Unconsumed input bytes. + #[inline] + fn remaining(&self) -> Result { + let pos = self.pos; + let len = self.buf.len(); + checked!(len - pos) + } } impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { @@ -240,8 +249,13 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } fn deserialize_i64>(self, visitor: V) -> Result { - let bytes: [u8; size_of::()] = self.buf[self.pos..].try_into()?; - self.inc_pos(size_of::()); + const BYTES: usize = size_of::(); + + let end = self.pos.saturating_add(BYTES); + let bytes: ArrayVec = self.buf[self.pos..end].try_into()?; + let bytes = bytes.into_inner().expect("array size matches i64"); + + self.inc_pos(BYTES); visitor.visit_i64(i64::from_be_bytes(bytes)) } @@ -258,8 +272,13 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } fn deserialize_u64>(self, visitor: V) -> Result { - let bytes: [u8; size_of::()] = self.buf[self.pos..].try_into()?; - self.inc_pos(size_of::()); + const BYTES: usize = size_of::(); + + let end = self.pos.saturating_add(BYTES); + let bytes: ArrayVec = self.buf[self.pos..end].try_into()?; + let bytes = bytes.into_inner().expect("array size matches u64"); + + self.inc_pos(BYTES); visitor.visit_u64(u64::from_be_bytes(bytes)) } From c903a718075ddef637529294ccd45554d11d767d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 25 Nov 2024 01:55:57 +0000 Subject: [PATCH 1118/2091] refactor and optimize receipt service data Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 6 ++-- src/api/client/sync/v4.rs | 8 ++--- src/service/rooms/read_receipt/data.rs | 43 ++++++++------------------ src/service/rooms/read_receipt/mod.rs | 8 ++--- src/service/sending/sender.rs | 6 ++-- 5 files changed, 26 insertions(+), 45 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 80aa8184..b69cbc87 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -559,8 +559,6 @@ async fn load_joined_room( .lazy_loading .lazy_load_confirm_delivery(sender_user, sender_device, room_id, sincecount); - // Database queries: - let current_shortstatehash = services .rooms .state @@ -983,9 +981,9 @@ async fn load_joined_room( .filter_map(|(read_user, _, edu)| async move { services .users - .user_is_ignored(&read_user, sender_user) + .user_is_ignored(read_user, sender_user) .await - .or_some((read_user, edu)) + .or_some((read_user.to_owned(), edu)) }) .collect() .await; diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 78b0b277..0913336d 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -495,11 +495,11 @@ pub(crate) async fn sync_events_v4_route( .read_receipt .readreceipts_since(room_id, *roomsince) .filter_map(|(read_user, ts, v)| async move { - (!services + services .users - .user_is_ignored(&read_user, sender_user) - .await) - .then_some((read_user, ts, v)) + .user_is_ignored(read_user, sender_user) + .await + .or_some((read_user.to_owned(), ts, v)) }) .collect() .await; diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 1194598d..34639e27 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -1,16 +1,15 @@ -use std::{mem::size_of, sync::Arc}; +use std::sync::Arc; use conduit::{ - utils, utils::{stream::TryIgnore, ReadyExt}, - Error, Result, + Result, }; use database::{Deserialized, Json, Map}; use futures::{Stream, StreamExt}; use ruma::{ events::{receipt::ReceiptEvent, AnySyncEphemeralRoomEvent}, serde::Raw, - CanonicalJsonObject, OwnedUserId, RoomId, UserId, + CanonicalJsonObject, RoomId, UserId, }; use crate::{globals, Dep}; @@ -26,7 +25,7 @@ struct Services { globals: Dep, } -pub(super) type ReceiptItem = (OwnedUserId, u64, Raw); +pub(super) type ReceiptItem<'a> = (&'a UserId, u64, Raw); impl Data { pub(super) fn new(args: &crate::Args<'_>) -> Self { @@ -59,39 +58,23 @@ impl Data { pub(super) fn readreceipts_since<'a>( &'a self, room_id: &'a RoomId, since: u64, - ) -> impl Stream + Send + 'a { + ) -> impl Stream> + Send + 'a { + type Key<'a> = (&'a RoomId, u64, &'a UserId); + type KeyVal<'a> = (Key<'a>, CanonicalJsonObject); + let after_since = since.saturating_add(1); // +1 so we don't send the event at since let first_possible_edu = (room_id, after_since); - let mut prefix = room_id.as_bytes().to_vec(); - prefix.push(0xFF); - let prefix2 = prefix.clone(); - self.readreceiptid_readreceipt - .stream_from_raw(&first_possible_edu) + .stream_from(&first_possible_edu) .ignore_err() - .ready_take_while(move |(k, _)| k.starts_with(&prefix2)) - .map(move |(k, v)| { - let count_offset = prefix.len().saturating_add(size_of::()); - let user_id_offset = count_offset.saturating_add(1); - - let count = utils::u64_from_bytes(&k[prefix.len()..count_offset]) - .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?; - - let user_id_str = utils::string_from_bytes(&k[user_id_offset..]) - .map_err(|_| Error::bad_database("Invalid readreceiptid userid bytes in db."))?; - - let user_id = UserId::parse(user_id_str) - .map_err(|_| Error::bad_database("Invalid readreceiptid userid in db."))?; - - let mut json = serde_json::from_slice::(v) - .map_err(|_| Error::bad_database("Read receipt in roomlatestid_roomlatest is invalid json."))?; - + .ready_take_while(move |((r, ..), _): &KeyVal<'_>| *r == room_id) + .map(move |((_, count, user_id), mut json): KeyVal<'_>| { json.remove("room_id"); - let event = Raw::from_json(serde_json::value::to_raw_value(&json)?); + let event = serde_json::value::to_raw_value(&json)?; - Ok((user_id, count, event)) + Ok((user_id, count, Raw::from_json(event))) }) .ignore_err() } diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index ec34361e..e089d369 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -7,10 +7,10 @@ use futures::Stream; use ruma::{ events::{ receipt::{ReceiptEvent, ReceiptEventContent}, - SyncEphemeralRoomEvent, + AnySyncEphemeralRoomEvent, SyncEphemeralRoomEvent, }, serde::Raw, - RoomId, UserId, + OwnedUserId, RoomId, UserId, }; use self::data::{Data, ReceiptItem}; @@ -55,7 +55,7 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub fn readreceipts_since<'a>( &'a self, room_id: &'a RoomId, since: u64, - ) -> impl Stream + Send + 'a { + ) -> impl Stream> + Send + 'a { self.db.readreceipts_since(room_id, since) } @@ -83,7 +83,7 @@ impl Service { #[must_use] pub fn pack_receipts(receipts: I) -> Raw> where - I: Iterator, + I: Iterator)>, { let mut json = BTreeMap::new(); for (_, _, value) in receipts { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 0a0aae39..3c544725 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -376,7 +376,7 @@ impl Service { let mut read = BTreeMap::::new(); while let Some((user_id, count, read_receipt)) = receipts.next().await { *max_edu_count = cmp::max(count, *max_edu_count); - if !self.services.globals.user_is_local(&user_id) { + if !self.services.globals.user_is_local(user_id) { continue; } @@ -400,7 +400,7 @@ impl Service { let receipt = receipt .remove(&ReceiptType::Read) .expect("our read receipts always set this") - .remove(&user_id) + .remove(user_id) .expect("our read receipts always have the user here"); let receipt_data = ReceiptData { @@ -408,7 +408,7 @@ impl Service { event_ids: vec![event_id.clone()], }; - if read.insert(user_id, receipt_data).is_none() { + if read.insert(user_id.to_owned(), receipt_data).is_none() { *num = num.saturating_add(1); if *num >= SELECT_RECEIPT_LIMIT { break; From 2592f83b69de54e012a66652d43c09183074d77a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 25 Nov 2024 02:10:24 +0000 Subject: [PATCH 1119/2091] add migration fix for duplicate readreceipt entries Signed-off-by: Jason Volk --- src/service/migrations.rs | 61 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/src/service/migrations.rs b/src/service/migrations.rs index 126d3c7e..cded9bce 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -14,7 +14,7 @@ use itertools::Itertools; use ruma::{ events::{push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType}, push::Ruleset, - OwnedUserId, UserId, + OwnedUserId, RoomId, UserId, }; use crate::{media, Services}; @@ -69,6 +69,7 @@ async fn fresh(services: &Services) -> Result<()> { db["global"].insert(b"fix_bad_double_separator_in_state_cache", []); db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", []); db["global"].insert(b"fix_referencedevents_missing_sep", []); + db["global"].insert(b"fix_readreceiptid_readreceipt_duplicates", []); // Create the admin room and server user on first run crate::admin::create_admin_room(services).boxed().await?; @@ -130,6 +131,14 @@ async fn migrate(services: &Services) -> Result<()> { fix_referencedevents_missing_sep(services).await?; } + if db["global"] + .get(b"fix_readreceiptid_readreceipt_duplicates") + .await + .is_not_found() + { + fix_readreceiptid_readreceipt_duplicates(services).await?; + } + let version_match = services.globals.db.database_version().await == DATABASE_VERSION || services.globals.db.database_version().await == CONDUIT_DATABASE_VERSION; @@ -493,3 +502,53 @@ async fn fix_referencedevents_missing_sep(services: &Services) -> Result { db["global"].insert(b"fix_referencedevents_missing_sep", []); db.db.cleanup() } + +async fn fix_readreceiptid_readreceipt_duplicates(services: &Services) -> Result { + use ruma::identifiers_validation::MAX_BYTES; + type ArrayId = arrayvec::ArrayString; + type Key<'a> = (&'a RoomId, u64, &'a UserId); + + warn!("Fixing undeleted entries in readreceiptid_readreceipt..."); + + let db = &services.db; + let cork = db.cork_and_sync(); + let readreceiptid_readreceipt = db["readreceiptid_readreceipt"].clone(); + + let mut cur_room: Option = None; + let mut cur_user: Option = None; + let (mut total, mut fixed): (usize, usize) = (0, 0); + readreceiptid_readreceipt + .keys() + .expect_ok() + .ready_for_each(|key: Key<'_>| { + let (room_id, _, user_id) = key; + let last_room = cur_room.replace( + room_id + .as_str() + .try_into() + .expect("invalid room_id in database"), + ); + + let last_user = cur_user.replace( + user_id + .as_str() + .try_into() + .expect("invalid user_id in database"), + ); + + let is_dup = cur_room == last_room && cur_user == last_user; + if is_dup { + readreceiptid_readreceipt.del(key); + } + + fixed = fixed.saturating_add(is_dup.into()); + total = total.saturating_add(1); + }) + .await; + + drop(cork); + info!(?total, ?fixed, "Fixed undeleted entries in readreceiptid_readreceipt."); + + db["global"].insert(b"fix_readreceiptid_readreceipt_duplicates", []); + db.db.cleanup() +} From 8611cc0ee9afb4691f6f55fbe2fd583d4337cca2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 24 Nov 2024 22:30:14 -0500 Subject: [PATCH 1120/2091] fix ignored_filter check, exclude dummy events over sync Signed-off-by: strawberry --- src/api/client/message.rs | 18 +++++++++--------- src/api/client/sync/v3.rs | 25 +++---------------------- src/api/client/sync/v4.rs | 25 +++---------------------- 3 files changed, 15 insertions(+), 53 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index f1a10aa2..d8043855 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -25,8 +25,8 @@ use crate::Ruma; pub(crate) type LazySet = HashSet; -/// list of safe and common non-state events to ignore -const IGNORED_MESSAGE_TYPES: &[TimelineEventType] = &[ +/// list of safe and common non-state events to ignore if the user is ignored +const IGNORED_MESSAGE_TYPES: &[TimelineEventType; 16] = &[ RoomMessage, Sticker, CallInvite, @@ -206,19 +206,19 @@ pub(crate) async fn update_lazy( pub(crate) async fn ignored_filter(services: &Services, item: PdusIterItem, user_id: &UserId) -> Option { let (_, pdu) = &item; + // exclude Synapse's dummy events from bloating up response bodies. clients + // don't need to see this. if pdu.kind.to_cow_str() == "org.matrix.dummy_event" { return None; } - if !IGNORED_MESSAGE_TYPES.iter().any(is_equal_to!(&pdu.kind)) { - return Some(item); + if IGNORED_MESSAGE_TYPES.iter().any(is_equal_to!(&pdu.kind)) + && services.users.user_is_ignored(&pdu.sender, user_id).await + { + return None; } - if !services.users.user_is_ignored(&pdu.sender, user_id).await { - return Some(item); - } - - None + Some(item) } pub(crate) async fn visibility_filter( diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index b69cbc87..0cb22317 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -38,6 +38,7 @@ use tracing::{Instrument as _, Span}; use super::{load_timeline, share_encrypted_room}; use crate::{ + client::ignored_filter, service::{pdu::EventHash, Services}, utils, Error, PduEvent, Result, Ruma, RumaResponse, }; @@ -949,28 +950,8 @@ async fn load_joined_room( let room_events: Vec<_> = timeline_pdus .iter() .stream() - .filter_map(|(_, pdu)| async move { - // list of safe and common non-state events to ignore - if matches!( - &pdu.kind, - RoomMessage - | Sticker | CallInvite - | CallNotify | RoomEncrypted - | Image | File | Audio - | Voice | Video | UnstablePollStart - | PollStart | KeyVerificationStart - | Reaction | Emote - | Location - ) && services - .users - .user_is_ignored(&pdu.sender, sender_user) - .await - { - return None; - } - - Some(pdu.to_sync_room_event()) - }) + .filter_map(|item| ignored_filter(services, item.clone(), sender_user)) + .map(|(_, pdu)| pdu.to_sync_room_event()) .collect() .await; diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 0913336d..62c313e2 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -35,7 +35,7 @@ use ruma::{ use service::{rooms::read_receipt::pack_receipts, Services}; use super::{load_timeline, share_encrypted_room}; -use crate::Ruma; +use crate::{client::ignored_filter, Ruma}; const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = @@ -539,27 +539,8 @@ pub(crate) async fn sync_events_v4_route( let room_events: Vec<_> = timeline_pdus .iter() .stream() - .filter_map(|(_, pdu)| async move { - // list of safe and common non-state events to ignore - if matches!( - &pdu.kind, - RoomMessage - | Sticker | CallInvite - | CallNotify | RoomEncrypted - | Image | File | Audio - | Voice | Video | UnstablePollStart - | PollStart | KeyVerificationStart - | Reaction | Emote | Location - ) && services - .users - .user_is_ignored(&pdu.sender, sender_user) - .await - { - return None; - } - - Some(pdu.to_sync_room_event()) - }) + .filter_map(|item| ignored_filter(&services, item.clone(), sender_user)) + .map(|(_, pdu)| pdu.to_sync_room_event()) .collect() .await; From e9fee04eef52c781455468e0fe68cbb13f60dad2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 24 Nov 2024 23:08:40 -0500 Subject: [PATCH 1121/2091] fix needlessly strict appservice user existence check Signed-off-by: strawberry --- src/api/router/auth.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 68abf5e2..56c16550 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -197,10 +197,6 @@ async fn auth_appservice(services: &Services, request: &Request, info: Box Date: Sun, 24 Nov 2024 23:11:13 -0500 Subject: [PATCH 1122/2091] slightly refactor appservice registration command Signed-off-by: strawberry --- src/admin/appservice/commands.rs | 18 ++++++++++++------ src/service/appservice/mod.rs | 12 ++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/admin/appservice/commands.rs b/src/admin/appservice/commands.rs index 7d6378f3..6d661308 100644 --- a/src/admin/appservice/commands.rs +++ b/src/admin/appservice/commands.rs @@ -11,19 +11,25 @@ pub(super) async fn register(&self) -> Result { )); } - let appservice_config = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); - let parsed_config = serde_yaml::from_str::(&appservice_config); + let appservice_config_body = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); + let parsed_config = serde_yaml::from_str::(&appservice_config_body); match parsed_config { - Ok(yaml) => match self.services.appservice.register_appservice(yaml).await { - Ok(id) => Ok(RoomMessageEventContent::text_plain(format!( - "Appservice registered with ID: {id}." + Ok(registration) => match self + .services + .appservice + .register_appservice(®istration, &appservice_config_body) + .await + { + Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( + "Appservice registered with ID: {}", + registration.id ))), Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "Failed to register appservice: {e}" ))), }, Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Could not parse appservice config: {e}" + "Could not parse appservice config as YAML: {e}" ))), } } diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 4a20b130..a55d6534 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -61,18 +61,18 @@ impl crate::Service for Service { impl Service { /// Registers an appservice and returns the ID to the caller - pub async fn register_appservice(&self, yaml: Registration) -> Result { + pub async fn register_appservice(&self, registration: &Registration, appservice_config_body: &str) -> Result { //TODO: Check for collisions between exclusive appservice namespaces self.registration_info .write() .await - .insert(yaml.id.clone(), yaml.clone().try_into()?); + .insert(registration.id.clone(), registration.clone().try_into()?); - let id = yaml.id.as_str(); - let yaml = serde_yaml::to_string(&yaml)?; - self.db.id_appserviceregistrations.insert(id, yaml); + self.db + .id_appserviceregistrations + .insert(®istration.id, appservice_config_body); - Ok(id.to_owned()) + Ok(()) } /// Remove an appservice registration From 6c663919880f49738b502e051686601af7cb5079 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 25 Nov 2024 06:47:20 +0000 Subject: [PATCH 1123/2091] fix unnecessary serialization of sender query keys Signed-off-by: Jason Volk --- src/service/sending/data.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index ca7ca19a..118dab91 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -103,8 +103,9 @@ impl Data { pub fn active_requests_for(&self, destination: &Destination) -> impl Stream + Send + '_ { let prefix = destination.get_prefix(); self.servercurrentevent_data - .stream_prefix_raw(&prefix) + .raw_stream_from(&prefix) .ignore_err() + .ready_take_while(move |(key, _)| key.starts_with(&prefix)) .map(|(key, val)| { let (_, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); @@ -152,8 +153,9 @@ impl Data { pub fn queued_requests(&self, destination: &Destination) -> impl Stream + Send + '_ { let prefix = destination.get_prefix(); self.servernameevent_data - .stream_prefix_raw(&prefix) + .raw_stream_from(&prefix) .ignore_err() + .ready_take_while(move |(key, _)| key.starts_with(&prefix)) .map(|(key, val)| { let (_, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); From 62d560e2fbd866f6d1b9f33d5ea0d5753acecfbb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 25 Nov 2024 06:26:34 +0000 Subject: [PATCH 1124/2091] improve tracing instruments on database::map Signed-off-by: Jason Volk --- src/database/map/get.rs | 2 +- src/database/map/insert.rs | 1 + src/database/map/keys.rs | 1 - src/database/map/keys_from.rs | 6 ++---- src/database/map/keys_prefix.rs | 5 +---- src/database/map/remove.rs | 2 +- src/database/map/rev_keys.rs | 1 - src/database/map/rev_keys_from.rs | 6 ++---- src/database/map/rev_keys_prefix.rs | 5 +---- src/database/map/rev_stream.rs | 1 - src/database/map/rev_stream_from.rs | 6 ++---- src/database/map/rev_stream_prefix.rs | 5 +---- src/database/map/stream.rs | 1 - src/database/map/stream_from.rs | 6 ++---- src/database/map/stream_prefix.rs | 5 +---- 15 files changed, 15 insertions(+), 38 deletions(-) diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 3ee2a194..24649175 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -38,7 +38,7 @@ where /// Fetch a value from the database into cache, returning a reference-handle /// asynchronously. The key is serialized into a user-supplied Writer. #[implement(super::Map)] -#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] +#[tracing::instrument(skip(self, buf), level = "trace")] pub fn bqry(&self, key: &K, buf: &mut B) -> impl Future>> + Send where K: Serialize + ?Sized + Debug, diff --git a/src/database/map/insert.rs b/src/database/map/insert.rs index b8b08b34..9bebe7cf 100644 --- a/src/database/map/insert.rs +++ b/src/database/map/insert.rs @@ -149,6 +149,7 @@ where /// - Key is serialized to supplied buffer /// - Val is raw #[implement(super::Map)] +#[tracing::instrument(skip(self, val, buf), level = "trace")] pub fn bput_raw(&self, key: K, val: V, mut buf: Bk) where K: Serialize + Debug, diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs index 2396494c..9c4d66e4 100644 --- a/src/database/map/keys.rs +++ b/src/database/map/keys.rs @@ -5,7 +5,6 @@ use serde::Deserialize; use crate::{keyval, keyval::Key, stream}; #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn keys<'a, K>(&'a self) -> impl Stream>> + Send where K: Deserialize<'a> + Send, diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 4eb3b12e..630bf3fb 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -7,7 +7,6 @@ use serde::{Deserialize, Serialize}; use crate::{keyval, keyval::Key, ser, stream}; #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn keys_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -18,7 +17,7 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self), level = "trace")] pub fn keys_from_raw

    (&self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -28,7 +27,6 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn keys_raw_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, @@ -39,7 +37,7 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self, from), fields(%self), level = "trace")] pub fn raw_keys_from

    (&self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index 0ff755f3..df214af4 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize}; use crate::{keyval, keyval::Key, ser}; #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn keys_prefix<'a, K, P>(&'a self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -22,7 +21,7 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self), level = "trace")] pub fn keys_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -33,7 +32,6 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn keys_raw_prefix<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, @@ -44,7 +42,6 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn raw_keys_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, diff --git a/src/database/map/remove.rs b/src/database/map/remove.rs index 42eaa477..949817a0 100644 --- a/src/database/map/remove.rs +++ b/src/database/map/remove.rs @@ -25,7 +25,7 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] +#[tracing::instrument(skip(self, buf), level = "trace")] pub fn bdel(&self, key: K, buf: &mut B) where K: Serialize + Debug, diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index 449ccfff..e10a199c 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -5,7 +5,6 @@ use serde::Deserialize; use crate::{keyval, keyval::Key, stream}; #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_keys<'a, K>(&'a self) -> impl Stream>> + Send where K: Deserialize<'a> + Send, diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index b142718c..c1c6f3da 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -7,7 +7,6 @@ use serde::{Deserialize, Serialize}; use crate::{keyval, keyval::Key, ser, stream}; #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_keys_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -18,7 +17,7 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self), level = "trace")] pub fn rev_keys_from_raw

    (&self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -28,7 +27,6 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_keys_raw_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, @@ -39,7 +37,7 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self, from), fields(%self), level = "trace")] pub fn rev_raw_keys_from

    (&self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index 5297cecf..957b974e 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -11,7 +11,6 @@ use serde::{Deserialize, Serialize}; use crate::{keyval, keyval::Key, ser}; #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_keys_prefix<'a, K, P>(&'a self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -22,7 +21,7 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self), level = "trace")] pub fn rev_keys_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -33,7 +32,6 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_keys_raw_prefix<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, @@ -44,7 +42,6 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_raw_keys_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index de22fd5c..f4be69fd 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -8,7 +8,6 @@ use crate::{keyval, keyval::KeyVal, stream}; /// /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_stream<'a, K, V>(&'a self) -> impl Stream>> + Send where K: Deserialize<'a> + Send, diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 78318a7f..7ef25ee0 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -11,7 +11,6 @@ use crate::{keyval, keyval::KeyVal, ser, stream}; /// - Query is serialized /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_stream_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -27,7 +26,7 @@ where /// - Query is serialized /// - Result is raw #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self), level = "trace")] pub fn rev_stream_from_raw

    (&self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -41,7 +40,6 @@ where /// - Query is raw /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_stream_raw_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, @@ -57,7 +55,7 @@ where /// - Query is raw /// - Result is raw #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self, from), fields(%self), level = "trace")] pub fn rev_raw_stream_from

    (&self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index 601c3298..286cedca 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -15,7 +15,6 @@ use crate::{keyval, keyval::KeyVal, ser}; /// - Query is serialized /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_stream_prefix<'a, K, V, P>(&'a self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -31,7 +30,7 @@ where /// - Query is serialized /// - Result is raw #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self), level = "trace")] pub fn rev_stream_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -46,7 +45,6 @@ where /// - Query is raw /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_stream_raw_prefix<'a, K, V, P>( &'a self, prefix: &'a P, ) -> impl Stream>> + Send + 'a @@ -64,7 +62,6 @@ where /// - Query is raw /// - Result is raw #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_raw_stream_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index dfbea072..143b0d0c 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -8,7 +8,6 @@ use crate::{keyval, keyval::KeyVal, stream}; /// /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn stream<'a, K, V>(&'a self) -> impl Stream>> + Send where K: Deserialize<'a> + Send, diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 0d3bb1e1..fe89afe1 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -11,7 +11,6 @@ use crate::{keyval, keyval::KeyVal, ser, stream}; /// - Query is serialized /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn stream_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -27,7 +26,7 @@ where /// - Query is serialized /// - Result is raw #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self), level = "trace")] pub fn stream_from_raw

    (&self, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -41,7 +40,6 @@ where /// - Query is raw /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn stream_raw_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, @@ -57,7 +55,7 @@ where /// - Query is raw /// - Result is raw #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self, from), fields(%self), level = "trace")] pub fn raw_stream_from

    (&self, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index cab3dd09..ca4cfeaa 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -15,7 +15,6 @@ use crate::{keyval, keyval::KeyVal, ser}; /// - Query is serialized /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn stream_prefix<'a, K, V, P>(&'a self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -31,7 +30,7 @@ where /// - Query is serialized /// - Result is raw #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] +#[tracing::instrument(skip(self), level = "trace")] pub fn stream_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -46,7 +45,6 @@ where /// - Query is raw /// - Result is deserialized #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn stream_raw_prefix<'a, K, V, P>( &'a self, prefix: &'a P, ) -> impl Stream>> + Send + 'a @@ -64,7 +62,6 @@ where /// - Query is raw /// - Result is raw #[implement(super::Map)] -#[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn raw_stream_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, From b20bd65d383129a4cb59c6e9a2262e512ea7aeff Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 24 Nov 2024 23:46:34 -0500 Subject: [PATCH 1125/2091] fix matrix-appservice-irc workaround Signed-off-by: strawberry --- src/api/client/account.rs | 44 +++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 5ed4b312..47f6fec8 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -55,8 +55,15 @@ pub(crate) async fn get_register_available_route( || appservice.registration.id.contains("matrix_appservice_irc") }); + // don't force the username lowercase if it's from matrix-appservice-irc + let body_username = if is_matrix_appservice_irc { + body.username.clone() + } else { + body.username.to_lowercase() + }; + // Validate user id - let user_id = UserId::parse_with_server_name(body.username.to_lowercase(), services.globals.server_name()) + let user_id = UserId::parse_with_server_name(body_username, services.globals.server_name()) .ok() .filter(|user_id| { (!user_id.is_historical() || is_matrix_appservice_irc) && services.globals.user_is_local(user_id) @@ -143,23 +150,28 @@ pub(crate) async fn register_route( return Err(Error::BadRequest(ErrorKind::forbidden(), "Registration temporarily disabled.")); } - // workaround for https://github.com/matrix-org/matrix-appservice-irc/issues/1780 due to inactivity of fixing the issue - let is_matrix_appservice_irc = body.appservice_info.as_ref().is_some_and(|appservice| { - appservice.registration.id == "irc" - || appservice.registration.id.contains("matrix-appservice-irc") - || appservice.registration.id.contains("matrix_appservice_irc") - }); - let user_id = match (&body.username, is_guest) { (Some(username), false) => { - let proposed_user_id = - UserId::parse_with_server_name(username.to_lowercase(), services.globals.server_name()) - .ok() - .filter(|user_id| { - (!user_id.is_historical() || is_matrix_appservice_irc) - && services.globals.user_is_local(user_id) - }) - .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; + // workaround for https://github.com/matrix-org/matrix-appservice-irc/issues/1780 due to inactivity of fixing the issue + let is_matrix_appservice_irc = body.appservice_info.as_ref().is_some_and(|appservice| { + appservice.registration.id == "irc" + || appservice.registration.id.contains("matrix-appservice-irc") + || appservice.registration.id.contains("matrix_appservice_irc") + }); + + // don't force the username lowercase if it's from matrix-appservice-irc + let body_username = if is_matrix_appservice_irc { + username.clone() + } else { + username.to_lowercase() + }; + + let proposed_user_id = UserId::parse_with_server_name(body_username, services.globals.server_name()) + .ok() + .filter(|user_id| { + (!user_id.is_historical() || is_matrix_appservice_irc) && services.globals.user_is_local(user_id) + }) + .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; if services.users.exists(&proposed_user_id).await { return Err(Error::BadRequest(ErrorKind::UserInUse, "Desired user ID is already taken.")); From 63d1fcf213ada2841169c652869b72bc70f9f8d6 Mon Sep 17 00:00:00 2001 From: morguldir Date: Mon, 25 Nov 2024 16:08:30 -0500 Subject: [PATCH 1126/2091] add queued transactions rocksdb cf cache Signed-off-by: strawberry --- conduwuit-example.toml | 4 ++++ src/core/config/mod.rs | 6 ++++++ src/database/opts.rs | 9 +++++++++ 3 files changed, 19 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 78136efb..30da80e6 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -170,6 +170,10 @@ # #statekeyshort_cache_capacity = varies by system +# This item is undocumented. Please contribute documentation for it. +# +#servernameevent_data_cache_capacity = varies by system + # This item is undocumented. Please contribute documentation for it. # #server_visibility_cache_capacity = varies by system diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 1754581d..fafd3396 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -211,6 +211,10 @@ pub struct Config { #[serde(default = "default_statekeyshort_cache_capacity")] pub statekeyshort_cache_capacity: u32, + /// default: varies by system + #[serde(default = "default_servernameevent_data_cache_capacity")] + pub servernameevent_data_cache_capacity: u32, + /// default: varies by system #[serde(default = "default_server_visibility_cache_capacity")] pub server_visibility_cache_capacity: u32, @@ -2059,6 +2063,8 @@ fn default_shortstatekey_cache_capacity() -> u32 { parallelism_scaled_u32(10_000 fn default_statekeyshort_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } +fn default_servernameevent_data_cache_capacity() -> u32 { parallelism_scaled_u32(100_000).saturating_add(500_000) } + fn default_server_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(500) } fn default_user_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(1000) } diff --git a/src/database/opts.rs b/src/database/opts.rs index 732f571f..28a39cca 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -170,6 +170,15 @@ pub(crate) fn cf_options( cache_size(cfg, cfg.statekeyshort_cache_capacity, 1024)?, ), + "servernameevent_data" => set_table_with_new_cache( + &mut opts, + cfg, + cache, + name, + cache_size(cfg, cfg.servernameevent_data_cache_capacity, 128)?, /* Raw average value size = 102, key + * size = 34 */ + ), + "eventid_outlierpdu" => { set_table_with_new_cache(&mut opts, cfg, cache, name, cache_size(cfg, cfg.pdu_cache_capacity, 1536)?); }, From c5c74febb5bf0a300a61c905e0e614f2b7b806c2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 25 Nov 2024 16:32:09 -0500 Subject: [PATCH 1127/2091] bump rust-rocksdb to 4bce1bb97d8be6f0d47245c99d465ca9cef33aad Signed-off-by: strawberry --- Cargo.lock | 6 +++--- deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3af8ae0..28f0edad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2151,7 +2151,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -3342,7 +3342,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.29.0+9.7.4" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2bc5495a9f8f75073390c326b47ee5928ab7c7f0#2bc5495a9f8f75073390c326b47ee5928ab7c7f0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=4bce1bb97d8be6f0d47245c99d465ca9cef33aad#4bce1bb97d8be6f0d47245c99d465ca9cef33aad" dependencies = [ "bindgen", "bzip2-sys", @@ -3359,7 +3359,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.33.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2bc5495a9f8f75073390c326b47ee5928ab7c7f0#2bc5495a9f8f75073390c326b47ee5928ab7c7f0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=4bce1bb97d8be6f0d47245c99d465ca9cef33aad#4bce1bb97d8be6f0d47245c99d465ca9cef33aad" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 908a2911..64a8dcb6 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "2bc5495a9f8f75073390c326b47ee5928ab7c7f0" +rev = "4bce1bb97d8be6f0d47245c99d465ca9cef33aad" #branch = "master" default-features = false From 238523f17701e711a47b072646fa25043f62d4b6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 25 Nov 2024 21:01:15 +0000 Subject: [PATCH 1128/2091] cleanup: reuse api rather than querying db Signed-off-by: Jason Volk --- src/service/rooms/state_accessor/data.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 80046d77..62e1f550 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -19,7 +19,6 @@ use crate::{ }; pub(super) struct Data { - eventid_shorteventid: Arc, shorteventid_shortstatehash: Arc, services: Services, } @@ -35,7 +34,6 @@ impl Data { pub(super) fn new(args: &crate::Args<'_>) -> Self { let db = &args.db; Self { - eventid_shorteventid: db["eventid_shorteventid"].clone(), shorteventid_shortstatehash: db["shorteventid_shortstatehash"].clone(), services: Services { short: args.depend::("rooms::short"), @@ -167,9 +165,15 @@ impl Data { /// Returns the state hash for this pdu. pub(super) async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { - self.eventid_shorteventid - .get(event_id) - .and_then(|shorteventid| self.shorteventid_shortstatehash.get(&shorteventid)) + const BUFSIZE: usize = size_of::(); + + self.services + .short + .get_shorteventid(event_id) + .and_then(|shorteventid| { + self.shorteventid_shortstatehash + .aqry::(&shorteventid) + }) .await .deserialized() } From f69c596f5607e596ee5e79ff8347dcbba676144f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 26 Nov 2024 00:35:03 +0000 Subject: [PATCH 1129/2091] generalize return value wrapping to not require Arc Signed-off-by: Jason Volk --- src/service/rooms/short/mod.rs | 13 ++++++++++--- src/service/rooms/state/mod.rs | 8 ++++++-- src/service/rooms/state_accessor/data.rs | 4 ++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index e4ff2975..0f100348 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,10 +1,11 @@ -use std::{fmt::Debug, mem::size_of_val, sync::Arc}; +use std::{borrow::Borrow, fmt::Debug, mem::size_of_val, sync::Arc}; pub use conduit::pdu::{ShortEventId, ShortId, ShortRoomId}; use conduit::{err, implement, utils, utils::stream::ReadyExt, Result}; use database::{Deserialized, Map}; use futures::{Stream, StreamExt}; use ruma::{events::StateEventType, EventId, RoomId}; +use serde::Deserialize; use crate::{globals, Dep}; @@ -136,7 +137,11 @@ pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &s } #[implement(Service)] -pub async fn get_eventid_from_short(&self, shorteventid: ShortEventId) -> Result> { +pub async fn get_eventid_from_short(&self, shorteventid: ShortEventId) -> Result +where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + ::Owned: Borrow, +{ const BUFSIZE: usize = size_of::(); self.db @@ -148,8 +153,10 @@ pub async fn get_eventid_from_short(&self, shorteventid: ShortEventId) -> Result } #[implement(Service)] -pub async fn multi_get_eventid_from_short(&self, shorteventid: I) -> Vec>> +pub async fn multi_get_eventid_from_short(&self, shorteventid: I) -> Vec> where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + ::Owned: Borrow, I: Iterator + Send, { const BUFSIZE: usize = size_of::(); diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 4429e912..3227b935 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -102,7 +102,11 @@ impl Service { .iter() .stream() .map(|&new| parse_compressed_state_event(new).1) - .then(|shorteventid| self.services.short.get_eventid_from_short(shorteventid)) + .then(|shorteventid| { + self.services + .short + .get_eventid_from_short::>(shorteventid) + }) .ignore_err(); pin_mut!(event_ids); @@ -433,7 +437,7 @@ impl Service { .await .into_iter() .stream() - .and_then(|event_id| async move { self.services.timeline.get_pdu(&event_id).await }) + .and_then(|event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await }) .collect() .await; diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 62e1f550..1ef91221 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -7,7 +7,7 @@ use conduit::{ }; use database::{Deserialized, Map}; use futures::{StreamExt, TryFutureExt}; -use ruma::{events::StateEventType, EventId, RoomId}; +use ruma::{events::StateEventType, EventId, OwnedEventId, RoomId}; use crate::{ rooms, @@ -74,7 +74,7 @@ impl Data { .into_iter() .stream() .ready_filter_map(Result::ok) - .filter_map(|event_id| async move { self.services.timeline.get_pdu(&event_id).await.ok() }) + .filter_map(|event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await.ok() }) .collect() .await; From 4f97ff98d66cbd5b3dc9c5ca9a27232959b70ee7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 27 Nov 2024 02:55:38 +0000 Subject: [PATCH 1130/2091] enter the tokio runtime for the scope of main init Signed-off-by: Jason Volk --- src/main/server.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/server.rs b/src/main/server.rs index 4813d586..27fd0673 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -24,6 +24,8 @@ pub(crate) struct Server { impl Server { pub(crate) fn build(args: &Args, runtime: Option<&runtime::Handle>) -> Result, Error> { + let _runtime_guard = runtime.map(runtime::Handle::enter); + let raw_config = Config::load(args.config.as_deref())?; let raw_config = crate::clap::update(raw_config, args)?; let config = Config::new(&raw_config)?; From e83fa124512cfa79172fc8eefc4bbe0eee388902 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 26 Nov 2024 01:01:08 +0000 Subject: [PATCH 1131/2091] tweak dev profile Signed-off-by: Jason Volk --- Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 02c3b5ab..12163f6a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -633,7 +633,6 @@ opt-level = 0 panic = "unwind" debug-assertions = true incremental = true -codegen-units = 64 #rustflags = [ # '--cfg', 'conduit_mods', # '-Ztime-passes', @@ -675,7 +674,6 @@ incremental = false [profile.dev.package.conduit] inherits = "dev" -incremental = false #rustflags = [ # '--cfg', 'conduit_mods', # '-Ztime-passes', From 527494a34b5f55cb18f329d7f91c20df71def623 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 27 Nov 2024 04:48:40 +0000 Subject: [PATCH 1132/2091] fix oversized tracing span arguments; lints Signed-off-by: Jason Volk --- src/database/engine.rs | 2 +- src/service/rooms/auth_chain/mod.rs | 13 ++++++++----- src/service/rooms/state/mod.rs | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 1fa53b01..9da20803 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -296,7 +296,7 @@ pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> { Ok(()) } -#[tracing::instrument(skip_all, name = "rocksdb")] +#[tracing::instrument(skip_all, name = "rocksdb", level = "debug")] pub(crate) fn handle_log(level: LogLevel, msg: &str) { let msg = msg.trim(); if msg.starts_with("Options") { diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 1d0490c2..5face0b5 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -209,20 +209,23 @@ impl Service { self.db.get_cached_eventid_authchain(key).await } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip_all, level = "debug")] pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) { - let val = auth_chain.iter().copied().collect::>(); + let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); + self.db.cache_auth_chain(key, val); } - #[tracing::instrument(skip(self), level = "debug")] - pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &Vec) { - let val = auth_chain.iter().copied().collect::>(); + #[tracing::instrument(skip_all, level = "debug")] + pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &[ShortEventId]) { + let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); + self.db.cache_auth_chain(key, val); } pub fn get_cache_usage(&self) -> (usize, usize) { let cache = self.db.auth_chain_cache.lock().expect("locked"); + (cache.len(), cache.capacity()) } diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 3227b935..8bd5f7eb 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -393,7 +393,7 @@ impl Service { } /// This fetches auth events from the current state. - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self, content), level = "debug")] pub async fn get_auth_events( &self, room_id: &RoomId, kind: &TimelineEventType, sender: &UserId, state_key: Option<&str>, content: &serde_json::value::RawValue, From dd8c646b633ff05e9f8e49a274712d6bf435e83a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 25 Nov 2024 23:27:16 +0000 Subject: [PATCH 1133/2091] optimize state compressor I/O w/ batch operation Signed-off-by: Jason Volk --- .../rooms/event_handler/resolve_state.rs | 39 +++++++++------- .../event_handler/upgrade_outlier_pdu.rs | 20 +++++---- src/service/rooms/state_compressor/mod.rs | 45 ++++++++++++++----- 3 files changed, 69 insertions(+), 35 deletions(-) diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 4863e340..dc0edd13 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -79,23 +79,30 @@ pub async fn resolve_state( drop(lock); - debug!("State resolution done. Compressing state"); - let mut new_room_state = HashSet::new(); - for ((event_type, state_key), event_id) in state { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&event_type.to_string().into(), &state_key) - .await; + debug!("State resolution done."); + let state_events: Vec<_> = state + .iter() + .stream() + .then(|((event_type, state_key), event_id)| { + self.services + .short + .get_or_create_shortstatekey(event_type, state_key) + .map(move |shortstatekey| (shortstatekey, event_id)) + }) + .collect() + .await; - let compressed = self - .services - .state_compressor - .compress_state_event(shortstatekey, &event_id) - .await; - - new_room_state.insert(compressed); - } + debug!("Compressing state..."); + let new_room_state: HashSet<_> = self + .services + .state_compressor + .compress_state_events( + state_events + .iter() + .map(|(ref ssk, eid)| (ssk, (*eid).borrow())), + ) + .collect() + .await; Ok(Arc::new(new_room_state)) } diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 2a1e4662..13e2b281 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Borrow, collections::{BTreeMap, HashSet}, sync::Arc, time::Instant, @@ -193,15 +194,16 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( extremities.retain(|id| retained.contains(id)); debug!("Retained {} extremities. Compressing state", extremities.len()); - let mut state_ids_compressed = HashSet::new(); - for (shortstatekey, id) in &state_at_incoming_event { - state_ids_compressed.insert( - self.services - .state_compressor - .compress_state_event(*shortstatekey, id) - .await, - ); - } + let state_ids_compressed: HashSet<_> = self + .services + .state_compressor + .compress_state_events( + state_at_incoming_event + .iter() + .map(|(ssk, eid)| (ssk, eid.borrow())), + ) + .collect() + .await; let state_ids_compressed = Arc::new(state_ids_compressed); diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 52ad5437..8c6eccbe 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -1,6 +1,6 @@ use std::{ collections::{HashMap, HashSet}, - fmt::Write, + fmt::{Debug, Write}, mem::size_of, sync::{Arc, Mutex}, }; @@ -8,10 +8,11 @@ use std::{ use arrayvec::ArrayVec; use conduit::{ at, checked, debug, err, expected, utils, - utils::{bytes, math::usize_from_f64}, + utils::{bytes, math::usize_from_f64, stream::IterStream}, Result, }; use database::Map; +use futures::{Stream, StreamExt}; use lru_cache::LruCache; use ruma::{EventId, RoomId}; @@ -179,21 +180,32 @@ impl Service { Ok(stack) } - pub async fn compress_state_event(&self, shortstatekey: ShortStateKey, event_id: &EventId) -> CompressedStateEvent { - const SIZE: usize = size_of::(); + pub fn compress_state_events<'a, I>(&'a self, state: I) -> impl Stream + Send + 'a + where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, + { + let event_ids = state.clone().map(at!(1)); + let short_event_ids = self + .services + .short + .multi_get_or_create_shorteventid(event_ids); + + state + .stream() + .map(at!(0)) + .zip(short_event_ids) + .map(|(shortstatekey, shorteventid)| compress_state_event(*shortstatekey, shorteventid)) + } + + pub async fn compress_state_event(&self, shortstatekey: ShortStateKey, event_id: &EventId) -> CompressedStateEvent { let shorteventid = self .services .short .get_or_create_shorteventid(event_id) .await; - let mut v = ArrayVec::::new(); - v.extend(shortstatekey.to_be_bytes()); - v.extend(shorteventid.to_be_bytes()); - v.as_ref() - .try_into() - .expect("failed to create CompressedStateEvent") + compress_state_event(shortstatekey, shorteventid) } /// Creates a new shortstatehash that often is just a diff to an already @@ -470,6 +482,19 @@ impl Service { } } +#[inline] +#[must_use] +fn compress_state_event(shortstatekey: ShortStateKey, shorteventid: ShortEventId) -> CompressedStateEvent { + const SIZE: usize = size_of::(); + + let mut v = ArrayVec::::new(); + v.extend(shortstatekey.to_be_bytes()); + v.extend(shorteventid.to_be_bytes()); + v.as_ref() + .try_into() + .expect("failed to create CompressedStateEvent") +} + #[inline] #[must_use] pub fn parse_compressed_state_event(compressed_event: CompressedStateEvent) -> (ShortStateKey, ShortEventId) { From 2aeee4f5095993c40463193dc491d8a0279c48dc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 27 Nov 2024 05:57:20 +0000 Subject: [PATCH 1134/2091] parallel query for outlier/non-outlier pdu data Signed-off-by: Jason Volk --- src/service/rooms/timeline/data.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 22a6c1d0..c15d8e7f 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -12,7 +12,7 @@ use conduit::{ Err, PduCount, PduEvent, Result, }; use database::{Database, Deserialized, Json, KeyVal, Map}; -use futures::{Stream, StreamExt}; +use futures::{future::select_ok, FutureExt, Stream, StreamExt}; use ruma::{api::Direction, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; use tokio::sync::Mutex; @@ -82,11 +82,14 @@ impl Data { /// Returns the json of a pdu. pub(super) async fn get_pdu_json(&self, event_id: &EventId) -> Result { - if let Ok(pdu) = self.get_non_outlier_pdu_json(event_id).await { - return Ok(pdu); - } + let accepted = self.get_non_outlier_pdu_json(event_id).boxed(); + let outlier = self + .eventid_outlierpdu + .get(event_id) + .map(Deserialized::deserialized) + .boxed(); - self.eventid_outlierpdu.get(event_id).await.deserialized() + select_ok([accepted, outlier]).await.map(at!(0)) } /// Returns the json of a pdu. @@ -131,11 +134,14 @@ impl Data { /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. pub(super) async fn get_pdu_owned(&self, event_id: &EventId) -> Result { - if let Ok(pdu) = self.get_non_outlier_pdu(event_id).await { - return Ok(pdu); - } + let accepted = self.get_non_outlier_pdu(event_id).boxed(); + let outlier = self + .eventid_outlierpdu + .get(event_id) + .map(Deserialized::deserialized) + .boxed(); - self.eventid_outlierpdu.get(event_id).await.deserialized() + select_ok([accepted, outlier]).await.map(at!(0)) } /// Like get_non_outlier_pdu(), but without the expense of fetching and From 94d7b21cf01378ca8d8a0451d00c8eb4374c39c7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 27 Nov 2024 06:25:13 +0000 Subject: [PATCH 1135/2091] use stricter timeout for fetching state Signed-off-by: Jason Volk --- src/service/rooms/event_handler/fetch_state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 74b0bb32..228b7d0c 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -22,7 +22,7 @@ pub(super) async fn fetch_state( let res = self .services .sending - .send_synapse_request( + .send_federation_request( origin, get_room_state_ids::v1::Request { room_id: room_id.to_owned(), From c7ae9516767dff6767a725be3d144164c0d1ad56 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 27 Nov 2024 06:28:32 +0000 Subject: [PATCH 1136/2091] add frontend threadpool to database Signed-off-by: Jason Volk --- Cargo.lock | 51 +++++++++- Cargo.toml | 6 +- src/database/Cargo.toml | 1 + src/database/engine.rs | 9 +- src/database/map.rs | 11 ++- src/database/map/contains.rs | 49 +++++---- src/database/map/get.rs | 100 ++++++++++++++----- src/database/mod.rs | 1 + src/database/pool.rs | 186 +++++++++++++++++++++++++++++++++++ src/database/util.rs | 5 +- 10 files changed, 362 insertions(+), 57 deletions(-) create mode 100644 src/database/pool.rs diff --git a/Cargo.lock b/Cargo.lock index 28f0edad..5c79b4b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -92,6 +92,18 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" +[[package]] +name = "async-channel" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-compression" version = "0.4.18" @@ -597,6 +609,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "conduit" version = "0.5.0" @@ -736,6 +757,7 @@ name = "conduit_database" version = "0.5.0" dependencies = [ "arrayvec", + "async-channel", "conduit_core", "const-str", "futures", @@ -1219,6 +1241,27 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "event-listener" +version = "5.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fdeflate" version = "0.3.6" @@ -2151,7 +2194,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -2589,6 +2632,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.3" diff --git a/Cargo.toml b/Cargo.toml index 12163f6a..283c5a95 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -307,10 +307,14 @@ version = "0.13.0" [workspace.dependencies.cyborgtime] version = "2.1.1" -# used to replace the channels of the tokio runtime +# used for MPSC channels [workspace.dependencies.loole] version = "0.4.0" +# used for MPMC channels +[workspace.dependencies.async-channel] +version = "2.3.1" + [workspace.dependencies.async-trait] version = "0.1.83" diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 0e718aa7..c45931a5 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -36,6 +36,7 @@ zstd_compression = [ [dependencies] arrayvec.workspace = true +async-channel.workspace = true conduit-core.workspace = true const-str.workspace = true futures.workspace = true diff --git a/src/database/engine.rs b/src/database/engine.rs index 9da20803..837c7259 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -16,7 +16,9 @@ use rocksdb::{ use crate::{ opts::{cf_options, db_options}, - or_else, result, + or_else, pool, + pool::Pool, + result, util::map_err, }; @@ -31,6 +33,7 @@ pub struct Engine { corks: AtomicU32, pub(super) read_only: bool, pub(super) secondary: bool, + pub(crate) pool: Arc, } pub(crate) type Db = DBWithThreadMode; @@ -111,6 +114,7 @@ impl Engine { corks: AtomicU32::new(0), read_only: config.rocksdb_read_only, secondary: config.rocksdb_secondary, + pool: Pool::new(&pool::Opts::default())?, })) } @@ -316,6 +320,9 @@ impl Drop for Engine { fn drop(&mut self) { const BLOCKING: bool = true; + debug!("Joining request threads..."); + self.pool.close(); + debug!("Waiting for background tasks to finish..."); self.db.cancel_all_background_work(BLOCKING); diff --git a/src/database/map.rs b/src/database/map.rs index d6b8bf38..4b55fa54 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -27,7 +27,7 @@ use std::{ }; use conduit::Result; -use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, WriteOptions}; +use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, ReadTier, WriteOptions}; use crate::{watchers::Watchers, Engine}; @@ -38,6 +38,7 @@ pub struct Map { watchers: Watchers, write_options: WriteOptions, read_options: ReadOptions, + cache_read_options: ReadOptions, } impl Map { @@ -49,6 +50,7 @@ impl Map { watchers: Watchers::default(), write_options: write_options_default(), read_options: read_options_default(), + cache_read_options: cache_read_options_default(), })) } @@ -112,6 +114,13 @@ fn open(db: &Arc, name: &str) -> Result> { }) } +#[inline] +fn cache_read_options_default() -> ReadOptions { + let mut read_options = read_options_default(); + read_options.set_read_tier(ReadTier::BlockCache); + read_options +} + #[inline] fn read_options_default() -> ReadOptions { let mut read_options = ReadOptions::default(); diff --git a/src/database/map/contains.rs b/src/database/map/contains.rs index a98fe7c5..7acef2f6 100644 --- a/src/database/map/contains.rs +++ b/src/database/map/contains.rs @@ -1,17 +1,21 @@ -use std::{convert::AsRef, fmt::Debug, future::Future, io::Write}; +use std::{convert::AsRef, fmt::Debug, future::Future, io::Write, sync::Arc}; use arrayvec::ArrayVec; -use conduit::{implement, utils::TryFutureExtExt, Err, Result}; -use futures::future::ready; +use conduit::{ + err, implement, + utils::{future::TryExtExt, result::FlatOk}, + Result, +}; +use futures::FutureExt; use serde::Serialize; -use crate::{ser, util}; +use crate::ser; /// Returns true if the map contains the key. /// - key is serialized into allocated buffer /// - harder errors may not be reported #[implement(super::Map)] -pub fn contains(&self, key: &K) -> impl Future + Send +pub fn contains(self: &Arc, key: &K) -> impl Future + Send + '_ where K: Serialize + ?Sized + Debug, { @@ -23,7 +27,7 @@ where /// - key is serialized into stack-buffer /// - harder errors will panic #[implement(super::Map)] -pub fn acontains(&self, key: &K) -> impl Future + Send +pub fn acontains(self: &Arc, key: &K) -> impl Future + Send + '_ where K: Serialize + ?Sized + Debug, { @@ -36,7 +40,7 @@ where /// - harder errors will panic #[implement(super::Map)] #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] -pub fn bcontains(&self, key: &K, buf: &mut B) -> impl Future + Send +pub fn bcontains(self: &Arc, key: &K, buf: &mut B) -> impl Future + Send + '_ where K: Serialize + ?Sized + Debug, B: Write + AsRef<[u8]>, @@ -48,41 +52,36 @@ where /// Returns Ok if the map contains the key. /// - key is raw #[implement(super::Map)] -pub fn exists(&self, key: &K) -> impl Future> + Send +pub fn exists<'a, K>(self: &'a Arc, key: &K) -> impl Future + Send + 'a where - K: AsRef<[u8]> + ?Sized + Debug, + K: AsRef<[u8]> + ?Sized + Debug + 'a, { - ready(self.exists_blocking(key)) + self.get(key).map(|res| res.map(|_| ())) } /// Returns Ok if the map contains the key; NotFound otherwise. Harder errors /// may not always be reported properly. #[implement(super::Map)] #[tracing::instrument(skip(self, key), fields(%self), level = "trace")] -pub fn exists_blocking(&self, key: &K) -> Result<()> +pub fn exists_blocking(&self, key: &K) -> Result where K: AsRef<[u8]> + ?Sized + Debug, { - if self.maybe_exists_blocking(key) - && self - .db - .db - .get_pinned_cf_opt(&self.cf(), key, &self.read_options) - .map_err(util::map_err)? - .is_some() - { - Ok(()) - } else { - Err!(Request(NotFound("Not found in database"))) - } + self.maybe_exists(key) + .then(|| self.get_blocking(key)) + .flat_ok() + .map(|_| ()) + .ok_or_else(|| err!(Request(NotFound("Not found in database")))) } +/// Rocksdb limits this to kBlockCacheTier internally so this is not actually a +/// blocking call; in case that changes we set this as well in our read_options. #[implement(super::Map)] -fn maybe_exists_blocking(&self, key: &K) -> bool +pub(crate) fn maybe_exists(&self, key: &K) -> bool where K: AsRef<[u8]> + ?Sized, { self.db .db - .key_may_exist_cf_opt(&self.cf(), key, &self.read_options) + .key_may_exist_cf_opt(&self.cf(), key, &self.cache_read_options) } diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 24649175..a00eecfa 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -1,13 +1,16 @@ -use std::{convert::AsRef, fmt::Debug, future::Future, io::Write}; +use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; use arrayvec::ArrayVec; -use conduit::{err, implement, utils::IterStream, Result}; -use futures::{FutureExt, Stream}; +use conduit::{err, implement, utils::IterStream, Err, Result}; +use futures::{future, Future, FutureExt, Stream}; use rocksdb::DBPinnableSlice; use serde::Serialize; -use tokio::task; -use crate::{ser, util, Handle}; +use crate::{ + ser, + util::{is_incomplete, map_err, or_else}, + Handle, +}; type RocksdbResult<'a> = Result>, rocksdb::Error>; @@ -15,7 +18,7 @@ type RocksdbResult<'a> = Result>, rocksdb::Error>; /// asynchronously. The key is serialized into an allocated buffer to perform /// the query. #[implement(super::Map)] -pub fn qry(&self, key: &K) -> impl Future>> + Send +pub fn qry(self: &Arc, key: &K) -> impl Future>> + Send where K: Serialize + ?Sized + Debug, { @@ -27,7 +30,7 @@ where /// asynchronously. The key is serialized into a fixed-sized buffer to perform /// the query. The maximum size is supplied as const generic parameter. #[implement(super::Map)] -pub fn aqry(&self, key: &K) -> impl Future>> + Send +pub fn aqry(self: &Arc, key: &K) -> impl Future>> + Send where K: Serialize + ?Sized + Debug, { @@ -39,7 +42,7 @@ where /// asynchronously. The key is serialized into a user-supplied Writer. #[implement(super::Map)] #[tracing::instrument(skip(self, buf), level = "trace")] -pub fn bqry(&self, key: &K, buf: &mut B) -> impl Future>> + Send +pub fn bqry(self: &Arc, key: &K, buf: &mut B) -> impl Future>> + Send where K: Serialize + ?Sized + Debug, B: Write + AsRef<[u8]>, @@ -52,28 +55,28 @@ where /// asynchronously. The key is referenced directly to perform the query. #[implement(super::Map)] #[tracing::instrument(skip(self, key), fields(%self), level = "trace")] -pub fn get(&self, key: &K) -> impl Future>> + Send +pub fn get(self: &Arc, key: &K) -> impl Future>> + Send where - K: AsRef<[u8]> + ?Sized + Debug, + K: AsRef<[u8]> + Debug + ?Sized, { - let result = self.get_blocking(key); - task::consume_budget().map(move |()| result) -} + use crate::pool::{Cmd, Get}; -/// Fetch a value from the database into cache, returning a reference-handle. -/// The key is referenced directly to perform the query. This is a thread- -/// blocking call. -#[implement(super::Map)] -pub fn get_blocking(&self, key: &K) -> Result> -where - K: AsRef<[u8]> + ?Sized, -{ - let res = self - .db - .db - .get_pinned_cf_opt(&self.cf(), key, &self.read_options); + let cached = self.get_cached(key); + if matches!(cached, Err(_) | Ok(Some(_))) { + return future::ready(cached.map(|res| res.expect("Option is Some"))).boxed(); + } - into_result_handle(res) + debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete"); + let cmd = Cmd::Get(Get { + map: self.clone(), + res: None, + key: key + .as_ref() + .try_into() + .expect("failed to copy key into buffer"), + }); + + self.db.pool.execute(cmd).boxed() } #[implement(super::Map)] @@ -104,9 +107,52 @@ where .map(into_result_handle) } +/// Fetch a value from the database into cache, returning a reference-handle. +/// The key is referenced directly to perform the query. This is a thread- +/// blocking call. +#[implement(super::Map)] +pub fn get_blocking(&self, key: &K) -> Result> +where + K: AsRef<[u8]> + ?Sized, +{ + let res = self + .db + .db + .get_pinned_cf_opt(&self.cf(), key, &self.read_options); + + into_result_handle(res) +} + +/// Fetch a value from the cache without I/O. +#[implement(super::Map)] +#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] +pub(crate) fn get_cached(&self, key: &K) -> Result>> +where + K: AsRef<[u8]> + Debug + ?Sized, +{ + let res = self + .db + .db + .get_pinned_cf_opt(&self.cf(), key, &self.cache_read_options); + + match res { + // cache hit; not found + Ok(None) => Err!(Request(NotFound("Not found in database"))), + + // cache hit; value found + Ok(Some(res)) => Ok(Some(Handle::from(res))), + + // cache miss; unknown + Err(e) if is_incomplete(&e) => Ok(None), + + // some other error occurred + Err(e) => or_else(e), + } +} + fn into_result_handle(result: RocksdbResult<'_>) -> Result> { result - .map_err(util::map_err)? + .map_err(map_err)? .map(Handle::from) .ok_or(err!(Request(NotFound("Not found in database")))) } diff --git a/src/database/mod.rs b/src/database/mod.rs index f09c4a71..cd91fba2 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -8,6 +8,7 @@ pub mod keyval; mod map; pub mod maps; mod opts; +mod pool; mod ser; mod stream; mod tests; diff --git a/src/database/pool.rs b/src/database/pool.rs new file mode 100644 index 00000000..e2cb2605 --- /dev/null +++ b/src/database/pool.rs @@ -0,0 +1,186 @@ +use std::{ + convert::identity, + mem::take, + sync::{Arc, Mutex}, + thread::JoinHandle, +}; + +use arrayvec::ArrayVec; +use async_channel::{bounded, Receiver, Sender}; +use conduit::{debug, defer, err, implement, Result}; +use futures::channel::oneshot; + +use crate::{Handle, Map}; + +pub(crate) struct Pool { + workers: Mutex>>, + recv: Receiver, + send: Sender, +} + +#[derive(Default)] +pub(crate) struct Opts { + queue_size: Option, + worker_num: Option, +} + +const WORKER_THREAD_NAME: &str = "conduwuit:db"; +const DEFAULT_QUEUE_SIZE: usize = 1024; +const DEFAULT_WORKER_NUM: usize = 32; +const KEY_MAX_BYTES: usize = 384; + +#[derive(Debug)] +pub(crate) enum Cmd { + Get(Get), +} + +#[derive(Debug)] +pub(crate) struct Get { + pub(crate) map: Arc, + pub(crate) key: ArrayVec, + pub(crate) res: Option>>>, +} + +#[implement(Pool)] +pub(crate) fn new(opts: &Opts) -> Result> { + let queue_size = opts.queue_size.unwrap_or(DEFAULT_QUEUE_SIZE); + + let (send, recv) = bounded(queue_size); + let pool = Arc::new(Self { + workers: Vec::new().into(), + recv, + send, + }); + + let worker_num = opts.worker_num.unwrap_or(DEFAULT_WORKER_NUM); + pool.spawn_until(worker_num)?; + + Ok(pool) +} + +#[implement(Pool)] +fn spawn_until(self: &Arc, max: usize) -> Result { + let mut workers = self.workers.lock()?; + + while workers.len() < max { + self.clone().spawn_one(&mut workers)?; + } + + Ok(()) +} + +#[implement(Pool)] +fn spawn_one(self: Arc, workers: &mut Vec>) -> Result { + use std::thread::Builder; + + let id = workers.len(); + + debug!(?id, "spawning {WORKER_THREAD_NAME}..."); + let thread = Builder::new() + .name(WORKER_THREAD_NAME.into()) + .spawn(move || self.worker(id))?; + + workers.push(thread); + + Ok(id) +} + +#[implement(Pool)] +pub(crate) fn close(self: &Arc) { + debug!( + senders = %self.send.sender_count(), + receivers = %self.send.receiver_count(), + "Closing pool channel" + ); + + let closing = self.send.close(); + debug_assert!(closing, "channel is not closing"); + + debug!("Shutting down pool..."); + let mut workers = self.workers.lock().expect("locked"); + + debug!( + workers = %workers.len(), + "Waiting for workers to join..." + ); + take(&mut *workers) + .into_iter() + .map(JoinHandle::join) + .try_for_each(identity) + .expect("failed to join worker threads"); + + debug_assert!(self.send.is_empty(), "channel is not empty"); +} + +#[implement(Pool)] +#[tracing::instrument(skip(self, cmd), level = "trace")] +pub(crate) async fn execute(&self, mut cmd: Cmd) -> Result> { + let (send, recv) = oneshot::channel(); + match &mut cmd { + Cmd::Get(ref mut cmd) => { + _ = cmd.res.insert(send); + }, + }; + + self.send + .send(cmd) + .await + .map_err(|e| err!(error!("send failed {e:?}")))?; + + recv.await + .map(into_recv_result) + .map_err(|e| err!(error!("recv failed {e:?}")))? +} + +#[implement(Pool)] +#[tracing::instrument(skip(self))] +fn worker(self: Arc, id: usize) { + debug!(?id, "worker spawned"); + defer! {{ debug!(?id, "worker finished"); }} + self.worker_loop(id); +} + +#[implement(Pool)] +fn worker_loop(&self, id: usize) { + while let Ok(mut cmd) = self.recv.recv_blocking() { + self.handle(id, &mut cmd); + } +} + +#[implement(Pool)] +fn handle(&self, id: usize, cmd: &mut Cmd) { + match cmd { + Cmd::Get(get) => self.handle_get(id, get), + } +} + +#[implement(Pool)] +#[tracing::instrument(skip(self, cmd), fields(%cmd.map), level = "trace")] +fn handle_get(&self, id: usize, cmd: &mut Get) { + let chan = cmd.res.take().expect("missing result channel"); + + // If the future was dropped while the command was queued then we can bail + // without any query. This makes it more efficient to use select() variants and + // pessimistic parallel queries. + if chan.is_canceled() { + return; + } + + let result = cmd.map.get_blocking(&cmd.key); + let _sent = chan.send(into_send_result(result)).is_ok(); +} + +fn into_send_result(result: Result>) -> Result> { + // SAFETY: Necessary to send the Handle (rust_rocksdb::PinnableSlice) through + // the channel. The lifetime on the handle is a device by rust-rocksdb to + // associate a database lifetime with its assets, not a function of rocksdb or + // the asset. The Handle must be dropped before the database is dropped. The + // handle must pass through recv_handle() on the other end of the channel. + unsafe { std::mem::transmute(result) } +} + +fn into_recv_result(result: Result>) -> Result> { + // SAFETY: This is to receive the Handle from the channel. Previously it had + // passed through send_handle(). + unsafe { std::mem::transmute(result) } +} diff --git a/src/database/util.rs b/src/database/util.rs index ae076381..21764361 100644 --- a/src/database/util.rs +++ b/src/database/util.rs @@ -1,5 +1,5 @@ use conduit::{err, Result}; -use rocksdb::{Direction, IteratorMode}; +use rocksdb::{Direction, ErrorKind, IteratorMode}; //#[cfg(debug_assertions)] macro_rules! unhandled { @@ -45,6 +45,9 @@ pub(crate) fn and_then(t: T) -> Result { Ok(t) } pub(crate) fn or_else(e: rocksdb::Error) -> Result { Err(map_err(e)) } +#[inline] +pub(crate) fn is_incomplete(e: &rocksdb::Error) -> bool { e.kind() == ErrorKind::Incomplete } + pub(crate) fn map_err(e: rocksdb::Error) -> conduit::Error { let string = e.into_string(); err!(Database(error!("{string}"))) From 76c75cc05afd5a32bb8f07bdbb47d79a71c40a64 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 27 Nov 2024 20:58:04 -0500 Subject: [PATCH 1137/2091] bump tracing fork Signed-off-by: strawberry --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5c79b4b5..94a2b2bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4501,8 +4501,8 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" -source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" +version = "0.1.41" +source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" dependencies = [ "log", "pin-project-lite", @@ -4512,8 +4512,8 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.27" -source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" +version = "0.1.28" +source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" dependencies = [ "proc-macro2", "quote", @@ -4522,8 +4522,8 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" -source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" +version = "0.1.33" +source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" dependencies = [ "once_cell", "valuable", @@ -4543,7 +4543,7 @@ dependencies = [ [[package]] name = "tracing-log" version = "0.2.0" -source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" +source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" dependencies = [ "log", "once_cell", @@ -4571,7 +4571,7 @@ dependencies = [ [[package]] name = "tracing-subscriber" version = "0.3.18" -source = "git+https://github.com/girlbossceo/tracing?rev=4d78a14a5e03f539b8c6b475aefa08bb14e4de91#4d78a14a5e03f539b8c6b475aefa08bb14e4de91" +source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" dependencies = [ "matchers", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index 283c5a95..9557ae08 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -184,17 +184,17 @@ features = [ # logging [workspace.dependencies.log] -version = "0.4.21" +version = "0.4.22" default-features = false [workspace.dependencies.tracing] -version = "0.1.40" +version = "0.1.41" default-features = false [workspace.dependencies.tracing-subscriber] version = "0.3.18" default-features = false features = ["env-filter", "std", "tracing", "tracing-log", "ansi", "fmt"] [workspace.dependencies.tracing-core] -version = "0.1.32" +version = "0.1.33" default-features = false # for URL previews @@ -493,16 +493,16 @@ version = "1.3.0" # https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c [patch.crates-io.tracing-subscriber] git = "https://github.com/girlbossceo/tracing" -rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" +rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d" [patch.crates-io.tracing] git = "https://github.com/girlbossceo/tracing" -rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" +rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d" [patch.crates-io.tracing-core] git = "https://github.com/girlbossceo/tracing" -rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" +rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d" [patch.crates-io.tracing-log] git = "https://github.com/girlbossceo/tracing" -rev = "4d78a14a5e03f539b8c6b475aefa08bb14e4de91" +rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d" # adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50 # adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b From 3ad6aa59f9032b6f43a2e6fb2a7265d8bf3a93a1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 28 Nov 2024 05:54:34 +0000 Subject: [PATCH 1138/2091] use smallvec for db query buffering Signed-off-by: Jason Volk --- Cargo.lock | 4 +++ Cargo.toml | 11 +++++- src/database/Cargo.toml | 1 + src/database/handle.rs | 1 + src/database/keyval.rs | 35 +++++++++++++++++-- src/database/map/contains.rs | 7 ++-- src/database/map/get.rs | 10 +++--- src/database/map/insert.rs | 26 ++++++++++---- src/database/map/keys_from.rs | 13 +++---- src/database/map/keys_prefix.rs | 8 ++--- src/database/map/remove.rs | 6 ++-- src/database/map/rev_keys_from.rs | 11 +++--- src/database/map/rev_keys_prefix.rs | 8 ++--- src/database/map/rev_stream_from.rs | 11 +++--- src/database/map/rev_stream_prefix.rs | 8 ++--- src/database/map/stream_from.rs | 13 +++---- src/database/map/stream_prefix.rs | 8 ++--- src/database/mod.rs | 4 +-- src/database/pool.rs | 50 +++++++++++++++++---------- src/database/ser.rs | 16 +++------ src/service/account_data/mod.rs | 2 +- src/service/media/data.rs | 4 +-- src/service/rooms/state_cache/mod.rs | 14 ++++---- 23 files changed, 173 insertions(+), 98 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 94a2b2bc..49c4127e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -765,6 +765,7 @@ dependencies = [ "rust-rocksdb-uwu", "serde", "serde_json", + "smallvec", "tokio", "tracing", ] @@ -3956,6 +3957,9 @@ name = "smallvec" version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +dependencies = [ + "serde", +] [[package]] name = "socket2" diff --git a/Cargo.toml b/Cargo.toml index 9557ae08..0e8596f7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,7 +27,16 @@ name = "conduit" [workspace.dependencies.arrayvec] version = "0.7.4" -features = ["std", "serde"] +features = ["serde"] + +[workspace.dependencies.smallvec] +version = "1.13.2" +features = [ + "const_generics", + "const_new", + "serde", + "write", +] [workspace.dependencies.const-str] version = "0.5.7" diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index c45931a5..1deaf980 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -44,6 +44,7 @@ log.workspace = true rust-rocksdb.workspace = true serde.workspace = true serde_json.workspace = true +smallvec.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/src/database/handle.rs b/src/database/handle.rs index daee224d..356bd859 100644 --- a/src/database/handle.rs +++ b/src/database/handle.rs @@ -58,6 +58,7 @@ impl<'a> Deserialized for Result<&'a Handle<'a>> { } impl<'a> Deserialized for &'a Handle<'a> { + #[inline] fn map_de(self, f: F) -> Result where F: FnOnce(T) -> U, diff --git a/src/database/keyval.rs b/src/database/keyval.rs index a288f184..d4568600 100644 --- a/src/database/keyval.rs +++ b/src/database/keyval.rs @@ -1,13 +1,42 @@ use conduit::Result; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; +use smallvec::SmallVec; -use crate::de; +use crate::{de, ser}; pub type KeyVal<'a, K = &'a Slice, V = &'a Slice> = (Key<'a, K>, Val<'a, V>); pub type Key<'a, T = &'a Slice> = T; pub type Val<'a, T = &'a Slice> = T; -pub type Slice = [u8]; +pub type KeyBuf = KeyBuffer; +pub type ValBuf = ValBuffer; + +pub type KeyBuffer = Buffer; +pub type ValBuffer = Buffer; +pub type Buffer = SmallVec<[Byte; CAP]>; + +pub type Slice = [Byte]; +pub type Byte = u8; + +pub const KEY_STACK_CAP: usize = 128; +pub const VAL_STACK_CAP: usize = 512; +pub const DEF_STACK_CAP: usize = KEY_STACK_CAP; + +#[inline] +pub fn serialize_key(val: T) -> Result +where + T: Serialize, +{ + ser::serialize_to::(val) +} + +#[inline] +pub fn serialize_val(val: T) -> Result +where + T: Serialize, +{ + ser::serialize_to::(val) +} #[inline] pub(crate) fn _expect_deserialize<'a, K, V>(kv: Result>) -> KeyVal<'a, K, V> diff --git a/src/database/map/contains.rs b/src/database/map/contains.rs index 7acef2f6..7016b744 100644 --- a/src/database/map/contains.rs +++ b/src/database/map/contains.rs @@ -9,23 +9,25 @@ use conduit::{ use futures::FutureExt; use serde::Serialize; -use crate::ser; +use crate::{keyval::KeyBuf, ser}; /// Returns true if the map contains the key. /// - key is serialized into allocated buffer /// - harder errors may not be reported +#[inline] #[implement(super::Map)] pub fn contains(self: &Arc, key: &K) -> impl Future + Send + '_ where K: Serialize + ?Sized + Debug, { - let mut buf = Vec::::with_capacity(64); + let mut buf = KeyBuf::new(); self.bcontains(key, &mut buf) } /// Returns true if the map contains the key. /// - key is serialized into stack-buffer /// - harder errors will panic +#[inline] #[implement(super::Map)] pub fn acontains(self: &Arc, key: &K) -> impl Future + Send + '_ where @@ -51,6 +53,7 @@ where /// Returns Ok if the map contains the key. /// - key is raw +#[inline] #[implement(super::Map)] pub fn exists<'a, K>(self: &'a Arc, key: &K) -> impl Future + Send + 'a where diff --git a/src/database/map/get.rs b/src/database/map/get.rs index a00eecfa..befc0b24 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -7,6 +7,7 @@ use rocksdb::DBPinnableSlice; use serde::Serialize; use crate::{ + keyval::KeyBuf, ser, util::{is_incomplete, map_err, or_else}, Handle, @@ -18,11 +19,12 @@ type RocksdbResult<'a> = Result>, rocksdb::Error>; /// asynchronously. The key is serialized into an allocated buffer to perform /// the query. #[implement(super::Map)] +#[inline] pub fn qry(self: &Arc, key: &K) -> impl Future>> + Send where K: Serialize + ?Sized + Debug, { - let mut buf = Vec::::with_capacity(64); + let mut buf = KeyBuf::new(); self.bqry(key, &mut buf) } @@ -30,6 +32,7 @@ where /// asynchronously. The key is serialized into a fixed-sized buffer to perform /// the query. The maximum size is supplied as const generic parameter. #[implement(super::Map)] +#[inline] pub fn aqry(self: &Arc, key: &K) -> impl Future>> + Send where K: Serialize + ?Sized + Debug, @@ -69,11 +72,8 @@ where debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete"); let cmd = Cmd::Get(Get { map: self.clone(), + key: key.as_ref().into(), res: None, - key: key - .as_ref() - .try_into() - .expect("failed to copy key into buffer"), }); self.db.pool.execute(cmd).boxed() diff --git a/src/database/map/insert.rs b/src/database/map/insert.rs index 9bebe7cf..5b2e816c 100644 --- a/src/database/map/insert.rs +++ b/src/database/map/insert.rs @@ -10,20 +10,25 @@ use conduit::implement; use rocksdb::WriteBatchWithTransaction; use serde::Serialize; -use crate::{ser, util::or_else}; +use crate::{ + keyval::{KeyBuf, ValBuf}, + ser, + util::or_else, +}; /// Insert Key/Value /// /// - Key is serialized /// - Val is serialized #[implement(super::Map)] +#[inline] pub fn put(&self, key: K, val: V) where K: Serialize + Debug, V: Serialize, { - let mut key_buf = Vec::new(); - let mut val_buf = Vec::new(); + let mut key_buf = KeyBuf::new(); + let mut val_buf = ValBuf::new(); self.bput(key, val, (&mut key_buf, &mut val_buf)); } @@ -32,12 +37,13 @@ where /// - Key is serialized /// - Val is raw #[implement(super::Map)] +#[inline] pub fn put_raw(&self, key: K, val: V) where K: Serialize + Debug, V: AsRef<[u8]>, { - let mut key_buf = Vec::new(); + let mut key_buf = KeyBuf::new(); self.bput_raw(key, val, &mut key_buf); } @@ -46,12 +52,13 @@ where /// - Key is raw /// - Val is serialized #[implement(super::Map)] +#[inline] pub fn raw_put(&self, key: K, val: V) where K: AsRef<[u8]>, V: Serialize, { - let mut val_buf = Vec::new(); + let mut val_buf = ValBuf::new(); self.raw_bput(key, val, &mut val_buf); } @@ -60,12 +67,13 @@ where /// - Key is serialized /// - Val is serialized to stack-buffer #[implement(super::Map)] +#[inline] pub fn put_aput(&self, key: K, val: V) where K: Serialize + Debug, V: Serialize, { - let mut key_buf = Vec::new(); + let mut key_buf = KeyBuf::new(); let mut val_buf = ArrayVec::::new(); self.bput(key, val, (&mut key_buf, &mut val_buf)); } @@ -75,13 +83,14 @@ where /// - Key is serialized to stack-buffer /// - Val is serialized #[implement(super::Map)] +#[inline] pub fn aput_put(&self, key: K, val: V) where K: Serialize + Debug, V: Serialize, { let mut key_buf = ArrayVec::::new(); - let mut val_buf = Vec::new(); + let mut val_buf = ValBuf::new(); self.bput(key, val, (&mut key_buf, &mut val_buf)); } @@ -90,6 +99,7 @@ where /// - Key is serialized to stack-buffer /// - Val is serialized to stack-buffer #[implement(super::Map)] +#[inline] pub fn aput(&self, key: K, val: V) where K: Serialize + Debug, @@ -105,6 +115,7 @@ where /// - Key is serialized to stack-buffer /// - Val is raw #[implement(super::Map)] +#[inline] pub fn aput_raw(&self, key: K, val: V) where K: Serialize + Debug, @@ -119,6 +130,7 @@ where /// - Key is raw /// - Val is serialized to stack-buffer #[implement(super::Map)] +#[inline] pub fn raw_aput(&self, key: K, val: V) where K: AsRef<[u8]>, diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 630bf3fb..093f7fd6 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -4,7 +4,10 @@ use conduit::{implement, Result}; use futures::{Stream, StreamExt}; use serde::{Deserialize, Serialize}; -use crate::{keyval, keyval::Key, ser, stream}; +use crate::{ + keyval::{result_deserialize_key, serialize_key, Key}, + stream, +}; #[implement(super::Map)] pub fn keys_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send @@ -12,8 +15,7 @@ where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, { - self.keys_from_raw(from) - .map(keyval::result_deserialize_key::) + self.keys_from_raw(from).map(result_deserialize_key::) } #[implement(super::Map)] @@ -22,7 +24,7 @@ pub fn keys_from_raw

    (&self, from: &P) -> impl Stream>> where P: Serialize + ?Sized + Debug, { - let key = ser::serialize_to_vec(from).expect("failed to serialize query key"); + let key = serialize_key(from).expect("failed to serialize query key"); self.raw_keys_from(&key) } @@ -32,8 +34,7 @@ where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, { - self.raw_keys_from(from) - .map(keyval::result_deserialize_key::) + self.raw_keys_from(from).map(result_deserialize_key::) } #[implement(super::Map)] diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index df214af4..8963f002 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -8,7 +8,7 @@ use futures::{ }; use serde::{Deserialize, Serialize}; -use crate::{keyval, keyval::Key, ser}; +use crate::keyval::{result_deserialize_key, serialize_key, Key}; #[implement(super::Map)] pub fn keys_prefix<'a, K, P>(&'a self, prefix: &P) -> impl Stream>> + Send @@ -17,7 +17,7 @@ where K: Deserialize<'a> + Send, { self.keys_prefix_raw(prefix) - .map(keyval::result_deserialize_key::) + .map(result_deserialize_key::) } #[implement(super::Map)] @@ -26,7 +26,7 @@ pub fn keys_prefix_raw

    (&self, prefix: &P) -> impl Stream| future::ok(k.starts_with(&key))) } @@ -38,7 +38,7 @@ where K: Deserialize<'a> + Send + 'a, { self.raw_keys_prefix(prefix) - .map(keyval::result_deserialize_key::) + .map(result_deserialize_key::) } #[implement(super::Map)] diff --git a/src/database/map/remove.rs b/src/database/map/remove.rs index 949817a0..18080c64 100644 --- a/src/database/map/remove.rs +++ b/src/database/map/remove.rs @@ -4,18 +4,20 @@ use arrayvec::ArrayVec; use conduit::implement; use serde::Serialize; -use crate::{ser, util::or_else}; +use crate::{keyval::KeyBuf, ser, util::or_else}; #[implement(super::Map)] +#[inline] pub fn del(&self, key: K) where K: Serialize + Debug, { - let mut buf = Vec::::with_capacity(64); + let mut buf = KeyBuf::new(); self.bdel(key, &mut buf); } #[implement(super::Map)] +#[inline] pub fn adel(&self, key: K) where K: Serialize + Debug, diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index c1c6f3da..75d062b5 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -4,7 +4,10 @@ use conduit::{implement, Result}; use futures::{Stream, StreamExt}; use serde::{Deserialize, Serialize}; -use crate::{keyval, keyval::Key, ser, stream}; +use crate::{ + keyval::{result_deserialize_key, serialize_key, Key}, + stream, +}; #[implement(super::Map)] pub fn rev_keys_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send @@ -13,7 +16,7 @@ where K: Deserialize<'a> + Send, { self.rev_keys_from_raw(from) - .map(keyval::result_deserialize_key::) + .map(result_deserialize_key::) } #[implement(super::Map)] @@ -22,7 +25,7 @@ pub fn rev_keys_from_raw

    (&self, from: &P) -> impl Stream + Send, { self.rev_raw_keys_from(from) - .map(keyval::result_deserialize_key::) + .map(result_deserialize_key::) } #[implement(super::Map)] diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index 957b974e..c14909d4 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -8,7 +8,7 @@ use futures::{ }; use serde::{Deserialize, Serialize}; -use crate::{keyval, keyval::Key, ser}; +use crate::keyval::{result_deserialize_key, serialize_key, Key}; #[implement(super::Map)] pub fn rev_keys_prefix<'a, K, P>(&'a self, prefix: &P) -> impl Stream>> + Send @@ -17,7 +17,7 @@ where K: Deserialize<'a> + Send, { self.rev_keys_prefix_raw(prefix) - .map(keyval::result_deserialize_key::) + .map(result_deserialize_key::) } #[implement(super::Map)] @@ -26,7 +26,7 @@ pub fn rev_keys_prefix_raw

    (&self, prefix: &P) -> impl Stream| future::ok(k.starts_with(&key))) } @@ -38,7 +38,7 @@ where K: Deserialize<'a> + Send + 'a, { self.rev_raw_keys_prefix(prefix) - .map(keyval::result_deserialize_key::) + .map(result_deserialize_key::) } #[implement(super::Map)] diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 7ef25ee0..6ac1cd1a 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -4,7 +4,10 @@ use conduit::{implement, Result}; use futures::stream::{Stream, StreamExt}; use serde::{Deserialize, Serialize}; -use crate::{keyval, keyval::KeyVal, ser, stream}; +use crate::{ + keyval::{result_deserialize, serialize_key, KeyVal}, + stream, +}; /// Iterate key-value entries in the map starting from upper-bound. /// @@ -18,7 +21,7 @@ where V: Deserialize<'a> + Send, { self.rev_stream_from_raw(from) - .map(keyval::result_deserialize::) + .map(result_deserialize::) } /// Iterate key-value entries in the map starting from upper-bound. @@ -31,7 +34,7 @@ pub fn rev_stream_from_raw

    (&self, from: &P) -> impl Stream + Send, { self.rev_raw_stream_from(from) - .map(keyval::result_deserialize::) + .map(result_deserialize::) } /// Iterate key-value entries in the map starting from upper-bound. diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index 286cedca..fd0d93ff 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -8,7 +8,7 @@ use futures::{ }; use serde::{Deserialize, Serialize}; -use crate::{keyval, keyval::KeyVal, ser}; +use crate::keyval::{result_deserialize, serialize_key, KeyVal}; /// Iterate key-value entries in the map where the key matches a prefix. /// @@ -22,7 +22,7 @@ where V: Deserialize<'a> + Send, { self.rev_stream_prefix_raw(prefix) - .map(keyval::result_deserialize::) + .map(result_deserialize::) } /// Iterate key-value entries in the map where the key matches a prefix. @@ -35,7 +35,7 @@ pub fn rev_stream_prefix_raw

    (&self, prefix: &P) -> impl Stream| future::ok(k.starts_with(&key))) } @@ -54,7 +54,7 @@ where V: Deserialize<'a> + Send + 'a, { self.rev_raw_stream_prefix(prefix) - .map(keyval::result_deserialize::) + .map(result_deserialize::) } /// Iterate key-value entries in the map where the key matches a prefix. diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index fe89afe1..052a2e74 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -4,7 +4,10 @@ use conduit::{implement, Result}; use futures::stream::{Stream, StreamExt}; use serde::{Deserialize, Serialize}; -use crate::{keyval, keyval::KeyVal, ser, stream}; +use crate::{ + keyval::{result_deserialize, serialize_key, KeyVal}, + stream, +}; /// Iterate key-value entries in the map starting from lower-bound. /// @@ -17,8 +20,7 @@ where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, { - self.stream_from_raw(from) - .map(keyval::result_deserialize::) + self.stream_from_raw(from).map(result_deserialize::) } /// Iterate key-value entries in the map starting from lower-bound. @@ -31,7 +33,7 @@ pub fn stream_from_raw

    (&self, from: &P) -> impl Stream + Send, V: Deserialize<'a> + Send, { - self.raw_stream_from(from) - .map(keyval::result_deserialize::) + self.raw_stream_from(from).map(result_deserialize::) } /// Iterate key-value entries in the map starting from lower-bound. diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index ca4cfeaa..a08b1e2a 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -8,7 +8,7 @@ use futures::{ }; use serde::{Deserialize, Serialize}; -use crate::{keyval, keyval::KeyVal, ser}; +use crate::keyval::{result_deserialize, serialize_key, KeyVal}; /// Iterate key-value entries in the map where the key matches a prefix. /// @@ -22,7 +22,7 @@ where V: Deserialize<'a> + Send, { self.stream_prefix_raw(prefix) - .map(keyval::result_deserialize::) + .map(result_deserialize::) } /// Iterate key-value entries in the map where the key matches a prefix. @@ -35,7 +35,7 @@ pub fn stream_prefix_raw

    (&self, prefix: &P) -> impl Stream| future::ok(k.starts_with(&key))) } @@ -54,7 +54,7 @@ where V: Deserialize<'a> + Send + 'a, { self.raw_stream_prefix(prefix) - .map(keyval::result_deserialize::) + .map(result_deserialize::) } /// Iterate key-value entries in the map where the key matches a prefix. diff --git a/src/database/mod.rs b/src/database/mod.rs index cd91fba2..de060b3a 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -28,9 +28,9 @@ pub use self::{ de::{Ignore, IgnoreAll}, deserialized::Deserialized, handle::Handle, - keyval::{KeyVal, Slice}, + keyval::{serialize_key, serialize_val, KeyVal, Slice}, map::Map, - ser::{serialize, serialize_to_array, serialize_to_vec, Interfix, Json, Separator, SEP}, + ser::{serialize, serialize_to, serialize_to_vec, Interfix, Json, Separator, SEP}, }; conduit::mod_ctor! {} diff --git a/src/database/pool.rs b/src/database/pool.rs index e2cb2605..6e7a1e29 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -5,12 +5,11 @@ use std::{ thread::JoinHandle, }; -use arrayvec::ArrayVec; use async_channel::{bounded, Receiver, Sender}; use conduit::{debug, defer, err, implement, Result}; use futures::channel::oneshot; -use crate::{Handle, Map}; +use crate::{keyval::KeyBuf, Handle, Map}; pub(crate) struct Pool { workers: Mutex>>, @@ -27,7 +26,6 @@ pub(crate) struct Opts { const WORKER_THREAD_NAME: &str = "conduwuit:db"; const DEFAULT_QUEUE_SIZE: usize = 1024; const DEFAULT_WORKER_NUM: usize = 32; -const KEY_MAX_BYTES: usize = 384; #[derive(Debug)] pub(crate) enum Cmd { @@ -37,10 +35,12 @@ pub(crate) enum Cmd { #[derive(Debug)] pub(crate) struct Get { pub(crate) map: Arc, - pub(crate) key: ArrayVec, - pub(crate) res: Option>>>, + pub(crate) key: KeyBuf, + pub(crate) res: Option, } +type ResultSender = oneshot::Sender>>; + #[implement(Pool)] pub(crate) fn new(opts: &Opts) -> Result> { let queue_size = opts.queue_size.unwrap_or(DEFAULT_QUEUE_SIZE); @@ -92,7 +92,6 @@ pub(crate) fn close(self: &Arc) { receivers = %self.send.receiver_count(), "Closing pool channel" ); - let closing = self.send.close(); debug_assert!(closing, "channel is not closing"); @@ -116,11 +115,7 @@ pub(crate) fn close(self: &Arc) { #[tracing::instrument(skip(self, cmd), level = "trace")] pub(crate) async fn execute(&self, mut cmd: Cmd) -> Result> { let (send, recv) = oneshot::channel(); - match &mut cmd { - Cmd::Get(ref mut cmd) => { - _ = cmd.res.insert(send); - }, - }; + Self::prepare(&mut cmd, send); self.send .send(cmd) @@ -132,6 +127,15 @@ pub(crate) async fn execute(&self, mut cmd: Cmd) -> Result> { .map_err(|e| err!(error!("recv failed {e:?}")))? } +#[implement(Pool)] +fn prepare(cmd: &mut Cmd, send: ResultSender) { + match cmd { + Cmd::Get(ref mut cmd) => { + _ = cmd.res.insert(send); + }, + }; +} + #[implement(Pool)] #[tracing::instrument(skip(self))] fn worker(self: Arc, id: usize) { @@ -157,25 +161,35 @@ fn handle(&self, id: usize, cmd: &mut Cmd) { #[implement(Pool)] #[tracing::instrument(skip(self, cmd), fields(%cmd.map), level = "trace")] fn handle_get(&self, id: usize, cmd: &mut Get) { + debug_assert!(!cmd.key.is_empty(), "querying for empty key"); + + // Obtain the result channel. let chan = cmd.res.take().expect("missing result channel"); - // If the future was dropped while the command was queued then we can bail - // without any query. This makes it more efficient to use select() variants and - // pessimistic parallel queries. + // It is worth checking if the future was dropped while the command was queued + // so we can bail without paying for any query. if chan.is_canceled() { return; } + // Perform the actual database query. We reuse our database::Map interface but + // limited to the blocking calls, rather than creating another surface directly + // with rocksdb here. let result = cmd.map.get_blocking(&cmd.key); - let _sent = chan.send(into_send_result(result)).is_ok(); + + // Send the result back to the submitter. + let chan_result = chan.send(into_send_result(result)); + + // If the future was dropped during the query this will fail acceptably. + let _chan_sent = chan_result.is_ok(); } fn into_send_result(result: Result>) -> Result> { // SAFETY: Necessary to send the Handle (rust_rocksdb::PinnableSlice) through // the channel. The lifetime on the handle is a device by rust-rocksdb to - // associate a database lifetime with its assets, not a function of rocksdb or - // the asset. The Handle must be dropped before the database is dropped. The - // handle must pass through recv_handle() on the other end of the channel. + // associate a database lifetime with its assets. The Handle must be dropped + // before the database is dropped. The handle must pass through recv_handle() on + // the other end of the channel. unsafe { std::mem::transmute(result) } } diff --git a/src/database/ser.rs b/src/database/ser.rs index 961d2700..a60812aa 100644 --- a/src/database/ser.rs +++ b/src/database/ser.rs @@ -1,28 +1,20 @@ use std::io::Write; -use arrayvec::ArrayVec; use conduit::{debug::type_name, err, result::DebugInspect, utils::exchange, Error, Result}; use serde::{ser, Serialize}; use crate::util::unhandled; #[inline] -pub fn serialize_to_array(val: T) -> Result> -where - T: Serialize, -{ - let mut buf = ArrayVec::::new(); - serialize(&mut buf, val)?; - - Ok(buf) -} +pub fn serialize_to_vec(val: T) -> Result> { serialize_to::, T>(val) } #[inline] -pub fn serialize_to_vec(val: T) -> Result> +pub fn serialize_to(val: T) -> Result where + B: Default + Write + AsRef<[u8]>, T: Serialize, { - let mut buf = Vec::with_capacity(64); + let mut buf = B::default(); serialize(&mut buf, val)?; Ok(buf) diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index b752f9b8..5dc17640 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -116,7 +116,7 @@ pub fn changes_since<'a>( &'a self, room_id: Option<&'a RoomId>, user_id: &'a UserId, since: u64, ) -> impl Stream + Send + 'a { let prefix = (room_id, user_id, Interfix); - let prefix = database::serialize_to_vec(prefix).expect("failed to serialize prefix"); + let prefix = database::serialize_key(prefix).expect("failed to serialize prefix"); // Skip the data that's exactly at since, because we sent that last time let first_possible = (room_id, user_id, since.saturating_add(1)); diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 9afbd708..3922dec9 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -39,14 +39,14 @@ impl Data { ) -> Result> { let dim: &[u32] = &[dim.width, dim.height]; let key = (mxc, dim, content_disposition, content_type); - let key = database::serialize_to_vec(key)?; + let key = database::serialize_key(key)?; self.mediaid_file.insert(&key, []); if let Some(user) = user { let key = (mxc, user); self.mediaid_user.put_raw(key, user); } - Ok(key) + Ok(key.to_vec()) } pub(super) async fn delete_file_mxc(&self, mxc: &Mxc<'_>) { diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 156345fe..4a33224e 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -9,7 +9,7 @@ use conduit::{ utils::{stream::TryIgnore, ReadyExt, StreamTools}, warn, Result, }; -use database::{serialize_to_vec, Deserialized, Ignore, Interfix, Json, Map}; +use database::{serialize_key, Deserialized, Ignore, Interfix, Json, Map}; use futures::{future::join4, pin_mut, stream::iter, Stream, StreamExt}; use itertools::Itertools; use ruma::{ @@ -289,10 +289,10 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { let userroom_id = (user_id, room_id); - let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); self.db.userroomid_joined.insert(&userroom_id, []); self.db.roomuserid_joined.insert(&roomuser_id, []); @@ -312,10 +312,10 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) { let userroom_id = (user_id, room_id); - let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); // (timo) TODO let leftstate = Vec::>::new(); @@ -716,10 +716,10 @@ impl Service { invite_via: Option>, ) { let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_to_vec(roomuser_id).expect("failed to serialize roomuser_id"); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); let userroom_id = (user_id, room_id); - let userroom_id = serialize_to_vec(userroom_id).expect("failed to serialize userroom_id"); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); self.db .userroomid_invitestate From 2a9bb1ce11f931b1ec2e02865d4bf30ceec11175 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 28 Nov 2024 06:52:23 +0000 Subject: [PATCH 1139/2091] add configurables for frontend pool options Signed-off-by: Jason Volk --- conduwuit-example.toml | 12 ++++++++++++ src/core/config/mod.rs | 18 ++++++++++++++++++ src/database/engine.rs | 7 ++++++- src/database/pool.rs | 17 ++++++++--------- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 30da80e6..a0f05ebb 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1338,6 +1338,18 @@ # #admin_room_notices = true +# Sets the number of worker threads in the frontend-pool of the database. +# This number should reflect the I/O capabilities of the system, +# specifically the queue-depth or the number of simultaneous requests in +# flight. Defaults to 32 or number of CPU cores, whichever is greater. +# +#db_pool_workers = 32 + +# Size of the queue feeding the database's frontend-pool. Defaults to 256 +# or eight times the number of CPU cores, whichever is greater. +# +#db_pool_queue_size = 256 + [global.tls] # Path to a valid TLS certificate file. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index fafd3396..edbb7c26 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1500,6 +1500,20 @@ pub struct Config { #[serde(default = "true_fn")] pub admin_room_notices: bool, + /// Sets the number of worker threads in the frontend-pool of the database. + /// This number should reflect the I/O capabilities of the system, + /// specifically the queue-depth or the number of simultaneous requests in + /// flight. Defaults to 32 or number of CPU cores, whichever is greater. + /// default: 32 + #[serde(default = "default_db_pool_workers")] + pub db_pool_workers: usize, + + /// Size of the queue feeding the database's frontend-pool. Defaults to 256 + /// or eight times the number of CPU cores, whichever is greater. + /// default: 256 + #[serde(default = "default_db_pool_queue_size")] + pub db_pool_queue_size: usize, + #[serde(flatten)] #[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime catchall: BTreeMap, @@ -2265,3 +2279,7 @@ fn parallelism_scaled_u32(val: u32) -> u32 { fn parallelism_scaled(val: usize) -> usize { val.saturating_mul(sys::available_parallelism()) } fn default_trusted_server_batch_size() -> usize { 256 } + +fn default_db_pool_workers() -> usize { sys::available_parallelism().max(32) } + +fn default_db_pool_queue_size() -> usize { sys::available_parallelism().saturating_mul(8).max(256) } diff --git a/src/database/engine.rs b/src/database/engine.rs index 837c7259..e700be62 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -103,6 +103,11 @@ impl Engine { "Opened database." ); + let pool_opts = pool::Opts { + queue_size: config.db_pool_queue_size, + worker_num: config.db_pool_workers, + }; + Ok(Arc::new(Self { server: server.clone(), row_cache, @@ -114,7 +119,7 @@ impl Engine { corks: AtomicU32::new(0), read_only: config.rocksdb_read_only, secondary: config.rocksdb_secondary, - pool: Pool::new(&pool::Opts::default())?, + pool: Pool::new(&pool_opts)?, })) } diff --git a/src/database/pool.rs b/src/database/pool.rs index 6e7a1e29..ee3e67dd 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -17,15 +17,14 @@ pub(crate) struct Pool { send: Sender, } -#[derive(Default)] pub(crate) struct Opts { - queue_size: Option, - worker_num: Option, + pub(crate) queue_size: usize, + pub(crate) worker_num: usize, } +const QUEUE_LIMIT: (usize, usize) = (1, 8192); +const WORKER_LIMIT: (usize, usize) = (1, 512); const WORKER_THREAD_NAME: &str = "conduwuit:db"; -const DEFAULT_QUEUE_SIZE: usize = 1024; -const DEFAULT_WORKER_NUM: usize = 32; #[derive(Debug)] pub(crate) enum Cmd { @@ -43,7 +42,7 @@ type ResultSender = oneshot::Sender>>; #[implement(Pool)] pub(crate) fn new(opts: &Opts) -> Result> { - let queue_size = opts.queue_size.unwrap_or(DEFAULT_QUEUE_SIZE); + let queue_size = opts.queue_size.clamp(QUEUE_LIMIT.0, QUEUE_LIMIT.1); let (send, recv) = bounded(queue_size); let pool = Arc::new(Self { @@ -52,7 +51,7 @@ pub(crate) fn new(opts: &Opts) -> Result> { send, }); - let worker_num = opts.worker_num.unwrap_or(DEFAULT_WORKER_NUM); + let worker_num = opts.worker_num.clamp(WORKER_LIMIT.0, WORKER_LIMIT.1); pool.spawn_until(worker_num)?; Ok(pool) @@ -147,12 +146,12 @@ fn worker(self: Arc, id: usize) { #[implement(Pool)] fn worker_loop(&self, id: usize) { while let Ok(mut cmd) = self.recv.recv_blocking() { - self.handle(id, &mut cmd); + self.worker_handle(id, &mut cmd); } } #[implement(Pool)] -fn handle(&self, id: usize, cmd: &mut Cmd) { +fn worker_handle(&self, id: usize, cmd: &mut Cmd) { match cmd { Cmd::Get(get) => self.handle_get(id, get), } From 58be22e69557f7045ad3fd433a438b33150ecd83 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 28 Nov 2024 23:33:33 +0000 Subject: [PATCH 1140/2091] fix new lints; clippy::unnecessary-map-or Signed-off-by: Jason Volk --- src/core/log/reload.rs | 2 +- src/core/utils/sys.rs | 5 +---- src/service/media/migrations.rs | 2 +- src/service/migrations.rs | 4 ++-- src/service/rooms/spaces/mod.rs | 2 +- src/service/rooms/state_accessor/data.rs | 1 - src/service/rooms/timeline/mod.rs | 2 +- src/service/sending/dest.rs | 7 ++----- src/service/users/mod.rs | 2 +- 9 files changed, 10 insertions(+), 17 deletions(-) diff --git a/src/core/log/reload.rs b/src/core/log/reload.rs index 6d651065..12d14f48 100644 --- a/src/core/log/reload.rs +++ b/src/core/log/reload.rs @@ -54,7 +54,7 @@ impl LogLevelReloadHandles { .lock() .expect("locked") .iter() - .filter(|(name, _)| names.map_or(false, |names| names.contains(&name.as_str()))) + .filter(|(name, _)| names.is_some_and(|names| names.contains(&name.as_str()))) .for_each(|(_, handle)| { _ = handle.reload(new_value.clone()).or_else(error::else_log); }); diff --git a/src/core/utils/sys.rs b/src/core/utils/sys.rs index af8bd70b..f3ff08b6 100644 --- a/src/core/utils/sys.rs +++ b/src/core/utils/sys.rs @@ -58,8 +58,5 @@ pub unsafe fn current_exe() -> Result { /// accurate on all platforms; defaults to false. #[must_use] pub fn current_exe_deleted() -> bool { - std::env::current_exe().map_or(false, |exe| { - exe.to_str() - .map_or(false, |exe| exe.ends_with(" (deleted)")) - }) + std::env::current_exe().is_ok_and(|exe| exe.to_str().is_some_and(|exe| exe.ends_with(" (deleted)"))) } diff --git a/src/service/media/migrations.rs b/src/service/media/migrations.rs index f1c6da7d..3d9c395e 100644 --- a/src/service/media/migrations.rs +++ b/src/service/media/migrations.rs @@ -112,7 +112,7 @@ async fn handle_media_check( let old_is_symlink = || async { tokio::fs::symlink_metadata(old_path) .await - .map_or(false, |md| md.is_symlink()) + .is_ok_and(|md| md.is_symlink()) }; if config.prune_missing_media && !old_exists && !new_exists { diff --git a/src/service/migrations.rs b/src/service/migrations.rs index cded9bce..f9057036 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -413,7 +413,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) .rooms .state_accessor .get_member(room_id, user_id) - .map(|member| member.map_or(false, |member| member.membership == MembershipState::Join)) + .map(|member| member.is_ok_and(|member| member.membership == MembershipState::Join)) }) .collect::>() .await; @@ -426,7 +426,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) .rooms .state_accessor .get_member(room_id, user_id) - .map(|member| member.map_or(false, |member| member.membership == MembershipState::Join)) + .map(|member| member.is_ok_and(|member| member.membership == MembershipState::Join)) }) .collect::>() .await; diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 0ef7ddf5..2b80e3dc 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -757,7 +757,7 @@ fn get_parent_children_via( fn next_room_to_traverse( stack: &mut Vec)>>, parents: &mut VecDeque, ) -> Option<(OwnedRoomId, Vec)> { - while stack.last().map_or(false, Vec::is_empty) { + while stack.last().is_some_and(Vec::is_empty) { stack.pop(); parents.pop_back(); } diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 1ef91221..a6c2e429 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -179,7 +179,6 @@ impl Data { } /// Returns the full room state. - #[allow(unused_qualifications)] // async traits pub(super) async fn room_state_full( &self, room_id: &RoomId, ) -> Result>> { diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 5d5566cb..c0b48b9b 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -616,7 +616,7 @@ impl Service { && pdu .state_key .as_ref() - .map_or(false, |state_key| users.is_match(state_key)) + .is_some_and(|state_key| users.is_match(state_key)) }; let matching_aliases = |aliases: NamespaceRegex| { self.services diff --git a/src/service/sending/dest.rs b/src/service/sending/dest.rs index 234a0b90..d35350c5 100644 --- a/src/service/sending/dest.rs +++ b/src/service/sending/dest.rs @@ -24,10 +24,7 @@ pub(super) fn get_prefix(&self) -> Vec { }, Self::Appservice(server) => { let sigil = b"+"; - let len = sigil - .len() - .saturating_add(server.as_bytes().len()) - .saturating_add(1); + let len = sigil.len().saturating_add(server.len()).saturating_add(1); let mut p = Vec::with_capacity(len); p.extend_from_slice(sigil); @@ -41,7 +38,7 @@ pub(super) fn get_prefix(&self) -> Vec { .len() .saturating_add(user.as_bytes().len()) .saturating_add(1) - .saturating_add(pushkey.as_bytes().len()) + .saturating_add(pushkey.len()) .saturating_add(1); let mut p = Vec::with_capacity(len); diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 1f8c56df..2462dde3 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -101,7 +101,7 @@ impl Service { .account_data .get_global(recipient_user, GlobalAccountDataEventType::IgnoredUserList) .await - .map_or(false, |ignored: IgnoredUserListEvent| { + .is_ok_and(|ignored: IgnoredUserListEvent| { ignored .content .ignored_users From 6175e72f1c056bcaee5d318a75b014855b7956ee Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 29 Nov 2024 08:26:27 +0000 Subject: [PATCH 1141/2091] simplify get_pdu() interface; eliminate unconditional Arc Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 6 ++--- src/api/client/context.rs | 2 +- src/api/client/report.rs | 2 +- src/api/client/room/event.rs | 2 +- src/api/client/search.rs | 6 +---- src/api/client/state.rs | 4 ++-- src/api/client/sync/v3.rs | 2 +- .../fetch_and_handle_outliers.rs | 3 ++- .../event_handler/handle_incoming_pdu.rs | 4 +++- .../rooms/event_handler/handle_outlier_pdu.rs | 4 ++-- src/service/rooms/event_handler/mod.rs | 13 +++++++++-- src/service/rooms/state/mod.rs | 2 +- src/service/rooms/state_accessor/data.rs | 23 +++++++++++-------- src/service/rooms/state_accessor/mod.rs | 10 ++++---- src/service/rooms/timeline/data.rs | 9 +------- src/service/rooms/timeline/mod.rs | 11 +++------ 16 files changed, 51 insertions(+), 52 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 89e47d4e..d4c9a57b 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -266,15 +266,15 @@ pub(super) async fn get_remote_pdu( #[admin_command] pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result { let room_id = self.services.rooms.alias.resolve(&room).await?; - let room_state = self + let room_state: Vec<_> = self .services .rooms .state_accessor .room_state_full(&room_id) .await? .values() - .map(|pdu| pdu.to_state_event()) - .collect::>(); + .map(PduEvent::to_state_event) + .collect(); if room_state.is_empty() { return Ok(RoomMessageEventContent::text_plain( diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 5b6b516e..bf87f5e1 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -103,7 +103,7 @@ pub(crate) async fn get_context_route( .collect() .await; - let lazy = once(&(base_token, (*base_event).clone())) + let lazy = once(&(base_token, base_event.clone())) .chain(events_before.iter()) .chain(events_after.iter()) .stream() diff --git a/src/api/client/report.rs b/src/api/client/report.rs index a0133704..31667323 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -137,7 +137,7 @@ pub(crate) async fn report_event_route( /// check if reporting user is in the reporting room async fn is_event_report_valid( services: &Services, event_id: &EventId, room_id: &RoomId, sender_user: &UserId, reason: Option<&String>, - score: Option, pdu: &std::sync::Arc, + score: Option, pdu: &PduEvent, ) -> Result<()> { debug_info!("Checking if report from user {sender_user} for event {event_id} in room {room_id} is valid"); diff --git a/src/api/client/room/event.rs b/src/api/client/room/event.rs index 0f44f25d..090c70a7 100644 --- a/src/api/client/room/event.rs +++ b/src/api/client/room/event.rs @@ -18,7 +18,7 @@ pub(crate) async fn get_room_event_route( event: services .rooms .timeline - .get_pdu_owned(&body.event_id) + .get_pdu(&body.event_id) .map_err(|_| err!(Request(NotFound("Event {} not found.", &body.event_id)))) .and_then(|event| async move { services diff --git a/src/api/client/search.rs b/src/api/client/search.rs index 1e5384fe..38468abb 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -181,11 +181,7 @@ async fn procure_room_state(services: &Services, room_id: &RoomId) -> Result( // a. Look in the main timeline (pduid_pdu tree) // b. Look at outlier pdu tree // (get_pdu_json checks both) - if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { + if let Ok(local_pdu) = self.services.timeline.get_pdu(id).map_ok(Arc::new).await { trace!("Found {id} in db"); events_with_auth_events.push((id, Some(local_pdu), vec![])); continue; diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 4d2d75d5..19367582 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -1,10 +1,11 @@ use std::{ collections::{hash_map, BTreeMap}, + sync::Arc, time::Instant, }; use conduit::{debug, err, implement, warn, Error, Result}; -use futures::FutureExt; +use futures::{FutureExt, TryFutureExt}; use ruma::{ api::client::error::ErrorKind, events::StateEventType, CanonicalJsonValue, EventId, RoomId, ServerName, UserId, }; @@ -79,6 +80,7 @@ pub async fn handle_incoming_pdu<'a>( .services .state_accessor .room_state_get(room_id, &StateEventType::RoomCreate, "") + .map_ok(Arc::new) .await?; // Procure the room version diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 2d95ff63..21504b66 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -4,7 +4,7 @@ use std::{ }; use conduit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; -use futures::future::ready; +use futures::{future::ready, TryFutureExt}; use ruma::{ api::client::error::ErrorKind, events::StateEventType, @@ -94,7 +94,7 @@ pub(super) async fn handle_outlier_pdu<'a>( // Build map of auth events let mut auth_events = HashMap::with_capacity(incoming_pdu.auth_events.len()); for id in &incoming_pdu.auth_events { - let Ok(auth_event) = self.services.timeline.get_pdu(id).await else { + let Ok(auth_event) = self.services.timeline.get_pdu(id).map_ok(Arc::new).await else { warn!("Could not find auth event {id}"); continue; }; diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index f6440fe9..3fb7d5c4 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -17,7 +17,11 @@ use std::{ time::Instant, }; -use conduit::{utils::MutexMap, Err, PduEvent, Result, Server}; +use conduit::{ + utils::{MutexMap, TryFutureExtExt}, + Err, PduEvent, Result, Server, +}; +use futures::TryFutureExt; use ruma::{ events::room::create::RoomCreateEventContent, state_res::RoomVersion, EventId, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, @@ -94,7 +98,12 @@ impl Service { async fn event_exists(&self, event_id: Arc) -> bool { self.services.timeline.pdu_exists(&event_id).await } async fn event_fetch(&self, event_id: Arc) -> Option> { - self.services.timeline.get_pdu(&event_id).await.ok() + self.services + .timeline + .get_pdu(&event_id) + .map_ok(Arc::new) + .ok() + .await } } diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 8bd5f7eb..03e2d2e8 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -445,7 +445,7 @@ impl Service { .into_iter() .map(at!(0)) .zip(auth_pdus.into_iter()) - .filter_map(|((event_type, state_key), pdu)| Some(((event_type, state_key), pdu.ok()?))) + .filter_map(|((event_type, state_key), pdu)| Some(((event_type, state_key), pdu.ok()?.into()))) .collect(); Ok(auth_pdus) diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index a6c2e429..6c67b856 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -46,7 +46,7 @@ impl Data { pub(super) async fn state_full( &self, shortstatehash: ShortStateHash, - ) -> Result>> { + ) -> Result> { let state = self .state_full_pdus(shortstatehash) .await? @@ -57,24 +57,27 @@ impl Data { Ok(state) } - pub(super) async fn state_full_pdus(&self, shortstatehash: ShortStateHash) -> Result>> { + pub(super) async fn state_full_pdus(&self, shortstatehash: ShortStateHash) -> Result> { let short_ids = self .state_full_shortids(shortstatehash) .await? .into_iter() .map(at!(1)); - let event_ids = self + let event_ids: Vec = self .services .short .multi_get_eventid_from_short(short_ids) - .await; + .await + .into_iter() + .filter_map(Result::ok) + .collect(); let full_pdus = event_ids - .into_iter() + .iter() .stream() + .then(|event_id| self.services.timeline.get_pdu(event_id)) .ready_filter_map(Result::ok) - .filter_map(|event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await.ok() }) .collect() .await; @@ -157,7 +160,7 @@ impl Data { /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). pub(super) async fn state_get( &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, - ) -> Result> { + ) -> Result { self.state_get_id(shortstatehash, event_type, state_key) .and_then(|event_id| async move { self.services.timeline.get_pdu(&event_id).await }) .await @@ -181,7 +184,7 @@ impl Data { /// Returns the full room state. pub(super) async fn room_state_full( &self, room_id: &RoomId, - ) -> Result>> { + ) -> Result> { self.services .state .get_room_shortstatehash(room_id) @@ -192,7 +195,7 @@ impl Data { /// Returns the full room state's pdus. #[allow(unused_qualifications)] // async traits - pub(super) async fn room_state_full_pdus(&self, room_id: &RoomId) -> Result>> { + pub(super) async fn room_state_full_pdus(&self, room_id: &RoomId) -> Result> { self.services .state .get_room_shortstatehash(room_id) @@ -215,7 +218,7 @@ impl Data { /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). pub(super) async fn room_state_get( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result> { + ) -> Result { self.services .state .get_room_shortstatehash(room_id) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index e08fac66..18f999b4 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -114,7 +114,7 @@ impl Service { pub async fn state_full( &self, shortstatehash: ShortStateHash, - ) -> Result>> { + ) -> Result> { self.db.state_full(shortstatehash).await } @@ -134,7 +134,7 @@ impl Service { #[inline] pub async fn state_get( &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, - ) -> Result> { + ) -> Result { self.db .state_get(shortstatehash, event_type, state_key) .await @@ -311,13 +311,13 @@ impl Service { /// Returns the full room state. #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_full(&self, room_id: &RoomId) -> Result>> { + pub async fn room_state_full(&self, room_id: &RoomId) -> Result> { self.db.room_state_full(room_id).await } /// Returns the full room state pdus #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_full_pdus(&self, room_id: &RoomId) -> Result>> { + pub async fn room_state_full_pdus(&self, room_id: &RoomId) -> Result> { self.db.room_state_full_pdus(room_id).await } @@ -337,7 +337,7 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub async fn room_state_get( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result> { + ) -> Result { self.db.room_state_get(room_id, event_type, state_key).await } diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index c15d8e7f..c394dc3b 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -126,14 +126,7 @@ impl Data { /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - pub(super) async fn get_pdu(&self, event_id: &EventId) -> Result> { - self.get_pdu_owned(event_id).await.map(Arc::new) - } - - /// Returns the pdu. - /// - /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - pub(super) async fn get_pdu_owned(&self, event_id: &EventId) -> Result { + pub(super) async fn get_pdu(&self, event_id: &EventId) -> Result { let accepted = self.get_non_outlier_pdu(event_id).boxed(); let outlier = self .eventid_outlierpdu diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index c0b48b9b..07b406c4 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -242,12 +242,7 @@ impl Service { /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - pub async fn get_pdu(&self, event_id: &EventId) -> Result> { self.db.get_pdu(event_id).await } - - /// Returns the pdu. - /// - /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - pub async fn get_pdu_owned(&self, event_id: &EventId) -> Result { self.db.get_pdu_owned(event_id).await } + pub async fn get_pdu(&self, event_id: &EventId) -> Result { self.db.get_pdu(event_id).await } /// Checks if pdu exists /// @@ -327,11 +322,11 @@ impl Service { ); unsigned.insert( String::from("prev_sender"), - CanonicalJsonValue::String(prev_state.sender.clone().to_string()), + CanonicalJsonValue::String(prev_state.sender.to_string()), ); unsigned.insert( String::from("replaces_state"), - CanonicalJsonValue::String(prev_state.event_id.clone().to_string()), + CanonicalJsonValue::String(prev_state.event_id.to_string()), ); } } From b5266ad9f5b77f5fa04b305ebbb0ed29e862396f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 29 Nov 2024 03:25:29 +0000 Subject: [PATCH 1142/2091] parallelize sender edu selection Signed-off-by: Jason Volk --- src/service/sending/sender.rs | 109 +++++++++++++++++++++------------- 1 file changed, 68 insertions(+), 41 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 3c544725..b1e909c9 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -1,7 +1,7 @@ use std::{ - cmp, collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, + sync::atomic::{AtomicU64, AtomicUsize, Ordering}, time::{Duration, Instant}, }; @@ -13,7 +13,12 @@ use conduit::{ utils::{calculate_hash, math::continue_exponential_backoff_secs, ReadyExt}, warn, Error, Result, }; -use futures::{future::BoxFuture, pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; +use futures::{ + future::{BoxFuture, OptionFuture}, + join, pin_mut, + stream::FuturesUnordered, + FutureExt, StreamExt, +}; use ruma::{ api::{ appservice::event::push_events::v1::Edu as RumaEdu, @@ -261,33 +266,45 @@ impl Service { #[tracing::instrument(skip_all, level = "debug")] async fn select_edus(&self, server_name: &ServerName) -> Result<(Vec>, u64)> { - // u64: count of last edu + // selection window let since = self.db.get_latest_educount(server_name).await; - let mut max_edu_count = since; - let mut events = Vec::new(); + let since_upper = self.services.globals.current_count()?; + let batch = (since, since_upper); + debug_assert!(batch.0 <= batch.1, "since range must not be negative"); - self.select_edus_device_changes(server_name, since, &mut max_edu_count, &mut events) - .await; + let events_len = AtomicUsize::default(); + let max_edu_count = AtomicU64::new(since); - if self.server.config.allow_outgoing_read_receipts { - self.select_edus_receipts(server_name, since, &mut max_edu_count, &mut events) - .await; - } + let device_changes = self.select_edus_device_changes(server_name, batch, &max_edu_count, &events_len); - if self.server.config.allow_outgoing_presence { - self.select_edus_presence(server_name, since, &mut max_edu_count, &mut events) - .await; - } + let receipts: OptionFuture<_> = self + .server + .config + .allow_outgoing_read_receipts + .then(|| self.select_edus_receipts(server_name, batch, &max_edu_count)) + .into(); - Ok((events, max_edu_count)) + let presence: OptionFuture<_> = self + .server + .config + .allow_outgoing_presence + .then(|| self.select_edus_presence(server_name, batch, &max_edu_count)) + .into(); + + let (device_changes, receipts, presence) = join!(device_changes, receipts, presence); + + let mut events = device_changes; + events.extend(presence.into_iter().flatten()); + events.extend(receipts.into_iter().flatten()); + + Ok((events, max_edu_count.load(Ordering::Acquire))) } /// Look for presence async fn select_edus_device_changes( - &self, server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, - ) { - debug_assert!(events.len() < SELECT_EDU_LIMIT, "called when edu limit reached"); - + &self, server_name: &ServerName, since: (u64, u64), max_edu_count: &AtomicU64, events_len: &AtomicUsize, + ) -> Vec> { + let mut events = Vec::new(); let server_rooms = self.services.state_cache.server_rooms(server_name); pin_mut!(server_rooms); @@ -296,12 +313,16 @@ impl Service { let keys_changed = self .services .users - .room_keys_changed(room_id, since, None) + .room_keys_changed(room_id, since.0, None) .ready_filter(|(user_id, _)| self.services.globals.user_is_local(user_id)); pin_mut!(keys_changed); while let Some((user_id, count)) = keys_changed.next().await { - *max_edu_count = cmp::max(count, *max_edu_count); + if count > since.1 { + break; + } + + max_edu_count.fetch_max(count, Ordering::Relaxed); if !device_list_changes.insert(user_id.into()) { continue; } @@ -321,19 +342,19 @@ impl Service { let edu = serde_json::to_vec(&edu).expect("failed to serialize device list update to JSON"); events.push(edu); - if events.len() >= SELECT_EDU_LIMIT { - return; + if events_len.fetch_add(1, Ordering::Relaxed) >= SELECT_EDU_LIMIT - 1 { + return events; } } } + + events } /// Look for read receipts in this room async fn select_edus_receipts( - &self, server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, - ) { - debug_assert!(events.len() < EDU_LIMIT, "called when edu limit reached"); - + &self, server_name: &ServerName, since: (u64, u64), max_edu_count: &AtomicU64, + ) -> Option> { let server_rooms = self.services.state_cache.server_rooms(server_name); pin_mut!(server_rooms); @@ -350,7 +371,7 @@ impl Service { } if receipts.is_empty() { - return; + return None; } let receipt_content = Edu::Receipt(ReceiptContent { @@ -360,22 +381,26 @@ impl Service { let receipt_content = serde_json::to_vec(&receipt_content).expect("Failed to serialize Receipt EDU to JSON vec"); - events.push(receipt_content); + Some(receipt_content) } /// Look for read receipts in this room async fn select_edus_receipts_room( - &self, room_id: &RoomId, since: u64, max_edu_count: &mut u64, num: &mut usize, + &self, room_id: &RoomId, since: (u64, u64), max_edu_count: &AtomicU64, num: &mut usize, ) -> ReceiptMap { let receipts = self .services .read_receipt - .readreceipts_since(room_id, since); + .readreceipts_since(room_id, since.0); pin_mut!(receipts); let mut read = BTreeMap::::new(); while let Some((user_id, count, read_receipt)) = receipts.next().await { - *max_edu_count = cmp::max(count, *max_edu_count); + if count > since.1 { + break; + } + + max_edu_count.fetch_max(count, Ordering::Relaxed); if !self.services.globals.user_is_local(user_id) { continue; } @@ -423,16 +448,18 @@ impl Service { /// Look for presence async fn select_edus_presence( - &self, server_name: &ServerName, since: u64, max_edu_count: &mut u64, events: &mut Vec>, - ) { - debug_assert!(events.len() < EDU_LIMIT, "called when edu limit reached"); - - let presence_since = self.services.presence.presence_since(since); + &self, server_name: &ServerName, since: (u64, u64), max_edu_count: &AtomicU64, + ) -> Option> { + let presence_since = self.services.presence.presence_since(since.0); pin_mut!(presence_since); let mut presence_updates = HashMap::::new(); while let Some((user_id, count, presence_bytes)) = presence_since.next().await { - *max_edu_count = cmp::max(count, *max_edu_count); + if count > since.1 { + break; + } + + max_edu_count.fetch_max(count, Ordering::Relaxed); if !self.services.globals.user_is_local(user_id) { continue; } @@ -474,7 +501,7 @@ impl Service { } if presence_updates.is_empty() { - return; + return None; } let presence_content = Edu::Presence(PresenceContent { @@ -483,7 +510,7 @@ impl Service { let presence_content = serde_json::to_vec(&presence_content).expect("failed to serialize Presence EDU to JSON"); - events.push(presence_content); + Some(presence_content) } async fn send_events(&self, dest: Destination, events: Vec) -> SendingResult { From 4a3cc9fffa62636ba1c3e76494142da226f9aefd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 30 Nov 2024 08:09:51 +0000 Subject: [PATCH 1143/2091] de-arc state_full_ids Signed-off-by: Jason Volk --- src/api/client/context.rs | 6 ++--- src/api/client/sync/v3.rs | 8 +++--- src/api/client/sync/v4.rs | 6 ++--- src/api/server/send_join.rs | 6 ++--- src/api/server/state.rs | 10 ++++--- src/api/server/state_ids.rs | 5 ++-- src/service/rooms/spaces/mod.rs | 4 +-- src/service/rooms/state_accessor/data.rs | 34 +++++++++++++++++------- src/service/rooms/state_accessor/mod.rs | 29 ++++++++++++++------ 9 files changed, 69 insertions(+), 39 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index bf87f5e1..652e17f4 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,4 +1,4 @@ -use std::iter::once; +use std::{collections::HashMap, iter::once}; use axum::extract::State; use conduit::{ @@ -10,7 +10,7 @@ use futures::{future::try_join, StreamExt, TryFutureExt}; use ruma::{ api::client::{context::get_context, filter::LazyLoadOptions}, events::StateEventType, - UserId, + OwnedEventId, UserId, }; use crate::{ @@ -124,7 +124,7 @@ pub(crate) async fn get_context_route( .await .map_err(|e| err!(Database("State hash not found: {e}")))?; - let state_ids = services + let state_ids: HashMap<_, OwnedEventId> = services .rooms .state_accessor .state_full_ids(shortstatehash) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 9c1cefdb..5578077f 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -32,7 +32,7 @@ use ruma::{ TimelineEventType::*, }, serde::Raw, - uint, DeviceId, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId, + uint, DeviceId, EventId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId, }; use tracing::{Instrument as _, Span}; @@ -398,7 +398,7 @@ async fn handle_left_room( Err(_) => HashMap::new(), }; - let Ok(left_event_id) = services + let Ok(left_event_id): Result = services .rooms .state_accessor .room_state_get_id(room_id, &StateEventType::RoomMember, sender_user.as_str()) @@ -666,7 +666,7 @@ async fn load_joined_room( let (joined_member_count, invited_member_count, heroes) = calculate_counts().await?; - let current_state_ids = services + let current_state_ids: HashMap<_, OwnedEventId> = services .rooms .state_accessor .state_full_ids(current_shortstatehash) @@ -736,7 +736,7 @@ async fn load_joined_room( let mut delta_state_events = Vec::new(); if since_shortstatehash != current_shortstatehash { - let current_state_ids = services + let current_state_ids: HashMap<_, OwnedEventId> = services .rooms .state_accessor .state_full_ids(current_shortstatehash) diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 62c313e2..14d79c19 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -1,6 +1,6 @@ use std::{ cmp::{self, Ordering}, - collections::{BTreeMap, BTreeSet, HashSet}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, time::Duration, }; @@ -30,7 +30,7 @@ use ruma::{ TimelineEventType::{self, *}, }, state_res::Event, - uint, MilliSecondsSinceUnixEpoch, OwnedRoomId, UInt, UserId, + uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, UInt, UserId, }; use service::{rooms::read_receipt::pack_receipts, Services}; @@ -211,7 +211,7 @@ pub(crate) async fn sync_events_v4_route( let new_encrypted_room = encrypted_room && since_encryption.is_err(); if encrypted_room { - let current_state_ids = services + let current_state_ids: HashMap<_, OwnedEventId> = services .rooms .state_accessor .state_full_ids(current_shortstatehash) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 0ad07b1e..92ab3b50 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -1,6 +1,6 @@ #![allow(deprecated)] -use std::borrow::Borrow; +use std::{borrow::Borrow, collections::HashMap}; use axum::extract::State; use conduit::{err, pdu::gen_event_id_canonical_json, utils::IterStream, warn, Error, Result}; @@ -11,7 +11,7 @@ use ruma::{ room::member::{MembershipState, RoomMemberEventContent}, StateEventType, }, - CanonicalJsonValue, OwnedServerName, OwnedUserId, RoomId, ServerName, + CanonicalJsonValue, OwnedEventId, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::Services; @@ -165,7 +165,7 @@ async fn create_join_event( drop(mutex_lock); - let state_ids = services + let state_ids: HashMap<_, OwnedEventId> = services .rooms .state_accessor .state_full_ids(shortstatehash) diff --git a/src/api/server/state.rs b/src/api/server/state.rs index b21fce68..400b9237 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -3,7 +3,7 @@ use std::{borrow::Borrow, iter::once}; use axum::extract::State; use conduit::{err, result::LogErr, utils::IterStream, Result}; use futures::{FutureExt, StreamExt, TryStreamExt}; -use ruma::api::federation::event::get_room_state; +use ruma::{api::federation::event::get_room_state, OwnedEventId}; use super::AccessCheck; use crate::Ruma; @@ -30,14 +30,18 @@ pub(crate) async fn get_room_state_route( .await .map_err(|_| err!(Request(NotFound("PDU state not found."))))?; - let pdus = services + let state_ids: Vec = services .rooms .state_accessor .state_full_ids(shortstatehash) .await .log_err() .map_err(|_| err!(Request(NotFound("PDU state IDs not found."))))? - .values() + .into_values() + .collect(); + + let pdus = state_ids + .iter() .try_stream() .and_then(|id| services.rooms.timeline.get_pdu_json(id)) .and_then(|pdu| { diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index 0c023bf0..55662a40 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -3,7 +3,7 @@ use std::{borrow::Borrow, iter::once}; use axum::extract::State; use conduit::{err, Result}; use futures::StreamExt; -use ruma::api::federation::event::get_room_state_ids; +use ruma::{api::federation::event::get_room_state_ids, OwnedEventId}; use super::AccessCheck; use crate::Ruma; @@ -31,14 +31,13 @@ pub(crate) async fn get_room_state_ids_route( .await .map_err(|_| err!(Request(NotFound("Pdu state not found."))))?; - let pdu_ids = services + let pdu_ids: Vec = services .rooms .state_accessor .state_full_ids(shortstatehash) .await .map_err(|_| err!(Request(NotFound("State ids not found"))))? .into_values() - .map(|id| (*id).to_owned()) .collect(); let auth_chain_ids = services diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 2b80e3dc..3e972ca6 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -1,7 +1,7 @@ mod tests; use std::{ - collections::VecDeque, + collections::{HashMap, VecDeque}, fmt::{Display, Formatter}, str::FromStr, sync::Arc, @@ -572,7 +572,7 @@ impl Service { return Ok(None); }; - let state = self + let state: HashMap<_, Arc<_>> = self .services .state_accessor .state_full_ids(current_shortstatehash) diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 6c67b856..7760d5b6 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{borrow::Borrow, collections::HashMap, sync::Arc}; use conduit::{ at, err, @@ -8,6 +8,7 @@ use conduit::{ use database::{Deserialized, Map}; use futures::{StreamExt, TryFutureExt}; use ruma::{events::StateEventType, EventId, OwnedEventId, RoomId}; +use serde::Deserialize; use crate::{ rooms, @@ -84,7 +85,11 @@ impl Data { Ok(full_pdus) } - pub(super) async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result>> { + pub(super) async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result> + where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + ::Owned: Borrow, + { let short_ids = self.state_full_shortids(shortstatehash).await?; let event_ids = self @@ -123,11 +128,15 @@ impl Data { Ok(shortids) } - /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). - #[allow(clippy::unused_self)] - pub(super) async fn state_get_id( + /// Returns a single EventId from `room_id` with key + /// (`event_type`,`state_key`). + pub(super) async fn state_get_id( &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, - ) -> Result> { + ) -> Result + where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + ::Owned: Borrow, + { let shortstatekey = self .services .short @@ -162,7 +171,7 @@ impl Data { &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result { self.state_get_id(shortstatehash, event_type, state_key) - .and_then(|event_id| async move { self.services.timeline.get_pdu(&event_id).await }) + .and_then(|event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await }) .await } @@ -204,10 +213,15 @@ impl Data { .await } - /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). - pub(super) async fn room_state_get_id( + /// Returns a single EventId from `room_id` with key + /// (`event_type`,`state_key`). + pub(super) async fn room_state_get_id( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result> { + ) -> Result + where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + ::Owned: Borrow, + { self.services .state .get_room_shortstatehash(room_id) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 18f999b4..e42d3764 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -1,6 +1,7 @@ mod data; use std::{ + borrow::Borrow, collections::HashMap, fmt::Write, sync::{Arc, Mutex as StdMutex, Mutex}, @@ -101,8 +102,12 @@ impl Service { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result>> { - self.db.state_full_ids(shortstatehash).await + pub async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result> + where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + ::Owned: Borrow, + { + self.db.state_full_ids::(shortstatehash).await } #[inline] @@ -118,12 +123,16 @@ impl Service { self.db.state_full(shortstatehash).await } - /// Returns a single PDU from `room_id` with key (`event_type`, + /// Returns a single EventId from `room_id` with key (`event_type`, /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_get_id( + pub async fn state_get_id( &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, - ) -> Result> { + ) -> Result + where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + ::Owned: Borrow, + { self.db .state_get_id(shortstatehash, event_type, state_key) .await @@ -321,12 +330,16 @@ impl Service { self.db.room_state_full_pdus(room_id).await } - /// Returns a single PDU from `room_id` with key (`event_type`, + /// Returns a single EventId from `room_id` with key (`event_type`, /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_get_id( + pub async fn room_state_get_id( &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, - ) -> Result> { + ) -> Result + where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + ::Owned: Borrow, + { self.db .room_state_get_id(room_id, event_type, state_key) .await From 9263439af8e7409bc2e6cf0ada82e44f5989374e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 1 Dec 2024 10:49:46 +0000 Subject: [PATCH 1144/2091] fix is_matching macro argument designator Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 18c2dd6f..1ab7d717 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -118,8 +118,8 @@ macro_rules! is_matching { |x| matches!(x, $val) }; - ($val:expr) => { - |x| matches!(x, $val) + ($($val:tt)+) => { + |x| matches!(x, $($val)+) }; } From 3fbd74310f866c8f6292c17a89007f396aa748e1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 1 Dec 2024 09:18:35 +0000 Subject: [PATCH 1145/2091] impl transposed form of MapExpect Signed-off-by: Jason Volk --- src/core/utils/result/map_expect.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/utils/result/map_expect.rs b/src/core/utils/result/map_expect.rs index 9cd498f7..b315ad3d 100644 --- a/src/core/utils/result/map_expect.rs +++ b/src/core/utils/result/map_expect.rs @@ -13,3 +13,8 @@ impl<'a, T, E: Debug> MapExpect<'a, Option> for Option> { #[inline] fn map_expect(self, msg: &'a str) -> Option { self.map(|result| result.expect(msg)) } } + +impl<'a, T, E: Debug> MapExpect<'a, Result> for Result, E> { + #[inline] + fn map_expect(self, msg: &'a str) -> Result { self.map(|result| result.expect(msg)) } +} From 3b30bd3580af5082c8878ef490818411217648e3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 2 Dec 2024 10:49:08 +0000 Subject: [PATCH 1146/2091] add try_filter_map to TryReadyExt Signed-off-by: Jason Volk --- src/core/utils/stream/try_ready.rs | 35 +++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs index 0daed26e..b0a2b6c5 100644 --- a/src/core/utils/stream/try_ready.rs +++ b/src/core/utils/stream/try_ready.rs @@ -3,7 +3,7 @@ use futures::{ future::{ready, Ready}, - stream::{AndThen, TryFold, TryForEach, TryStream, TryStreamExt}, + stream::{AndThen, TryFilterMap, TryFold, TryForEach, TryStream, TryStreamExt}, }; use crate::Result; @@ -20,11 +20,11 @@ where where F: Fn(S::Ok) -> Result; - fn ready_try_for_each( + fn ready_try_filter_map( self, f: F, - ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> + ) -> TryFilterMap, E>>, impl FnMut(S::Ok) -> Ready, E>>> where - F: FnMut(S::Ok) -> Result<(), E>; + F: Fn(S::Ok) -> Result, E>; fn ready_try_fold( self, init: U, f: F, @@ -38,6 +38,12 @@ where where F: Fn(U, S::Ok) -> Result, U: Default; + + fn ready_try_for_each( + self, f: F, + ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> + where + F: FnMut(S::Ok) -> Result<(), E>; } impl TryReadyExt for S @@ -53,14 +59,13 @@ where self.and_then(move |t| ready(f(t))) } - #[inline] - fn ready_try_for_each( - self, mut f: F, - ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> + fn ready_try_filter_map( + self, f: F, + ) -> TryFilterMap, E>>, impl FnMut(S::Ok) -> Ready, E>>> where - F: FnMut(S::Ok) -> Result<(), E>, + F: Fn(S::Ok) -> Result, E>, { - self.try_for_each(move |t| ready(f(t))) + self.try_filter_map(move |t| ready(f(t))) } #[inline] @@ -83,4 +88,14 @@ where { self.ready_try_fold(U::default(), f) } + + #[inline] + fn ready_try_for_each( + self, mut f: F, + ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> + where + F: FnMut(S::Ok) -> Result<(), E>, + { + self.try_for_each(move |t| ready(f(t))) + } } From 61d9ac66fa367128fc539faf965d5422ef154f39 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 2 Dec 2024 11:54:03 +0000 Subject: [PATCH 1147/2091] add ref_at util macro Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 1ab7d717..f9310243 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -79,6 +79,13 @@ macro_rules! at { }; } +#[macro_export] +macro_rules! ref_at { + ($idx:tt) => { + |ref t| &t.$idx + }; +} + /// Functor for equality i.e. .is_some_and(is_equal!(2)) #[macro_export] macro_rules! is_equal_to { From 7d6710c03346f7157c74538b61e407c1b8536f64 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 30 Nov 2024 08:27:51 +0000 Subject: [PATCH 1148/2091] add broadband stream extensions Signed-off-by: Jason Volk --- src/core/utils/stream/broadband.rs | 84 ++++++++++++++++++++++++++++++ src/core/utils/stream/mod.rs | 4 ++ src/core/utils/stream/wideband.rs | 84 ++++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+) create mode 100644 src/core/utils/stream/broadband.rs create mode 100644 src/core/utils/stream/wideband.rs diff --git a/src/core/utils/stream/broadband.rs b/src/core/utils/stream/broadband.rs new file mode 100644 index 00000000..ce17830c --- /dev/null +++ b/src/core/utils/stream/broadband.rs @@ -0,0 +1,84 @@ +//! Broadband stream combinator extensions to futures::Stream +#![allow(clippy::type_complexity)] + +use std::convert::identity; + +use futures::{ + stream::{Stream, StreamExt}, + Future, +}; + +use super::ReadyExt; + +const WIDTH: usize = 32; + +/// Concurrency extensions to augment futures::StreamExt. broad_ combinators +/// produce out-of-order +pub trait BroadbandExt +where + Self: Stream + Send + Sized, +{ + /// Concurrent filter_map(); unordered results + fn broadn_filter_map(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future> + Send, + U: Send; + + fn broadn_then(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + U: Send; + + #[inline] + fn broad_filter_map(self, f: F) -> impl Stream + Send + where + F: Fn(Item) -> Fut + Send, + Fut: Future> + Send, + U: Send, + { + self.broadn_filter_map(None, f) + } + + #[inline] + fn broad_then(self, f: F) -> impl Stream + Send + where + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + U: Send, + { + self.broadn_then(None, f) + } +} + +impl BroadbandExt for S +where + S: Stream + Send + Sized, +{ + #[inline] + fn broadn_filter_map(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future> + Send, + U: Send, + { + self.map(f) + .buffer_unordered(n.into().unwrap_or(WIDTH)) + .ready_filter_map(identity) + } + + #[inline] + fn broadn_then(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + U: Send, + { + self.map(f).buffer_unordered(n.into().unwrap_or(WIDTH)) + } +} diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index 1111915b..45c2110d 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -1,3 +1,4 @@ +mod broadband; mod cloned; mod expect; mod ignore; @@ -5,7 +6,9 @@ mod iter_stream; mod ready; mod tools; mod try_ready; +mod wideband; +pub use broadband::BroadbandExt; pub use cloned::Cloned; pub use expect::TryExpect; pub use ignore::TryIgnore; @@ -13,3 +16,4 @@ pub use iter_stream::IterStream; pub use ready::ReadyExt; pub use tools::Tools; pub use try_ready::TryReadyExt; +pub use wideband::WidebandExt; diff --git a/src/core/utils/stream/wideband.rs b/src/core/utils/stream/wideband.rs new file mode 100644 index 00000000..100990b8 --- /dev/null +++ b/src/core/utils/stream/wideband.rs @@ -0,0 +1,84 @@ +//! Wideband stream combinator extensions to futures::Stream +#![allow(clippy::type_complexity)] + +use std::convert::identity; + +use futures::{ + stream::{Stream, StreamExt}, + Future, +}; + +use super::ReadyExt; + +const WIDTH: usize = 32; + +/// Concurrency extensions to augment futures::StreamExt. wideband_ combinators +/// produce in-order. +pub trait WidebandExt +where + Self: Stream + Send + Sized, +{ + /// Concurrent filter_map(); ordered results + fn widen_filter_map(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future> + Send, + U: Send; + + fn widen_then(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + U: Send; + + #[inline] + fn wide_filter_map(self, f: F) -> impl Stream + Send + where + F: Fn(Item) -> Fut + Send, + Fut: Future> + Send, + U: Send, + { + self.widen_filter_map(None, f) + } + + #[inline] + fn wide_then(self, f: F) -> impl Stream + Send + where + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + U: Send, + { + self.widen_then(None, f) + } +} + +impl WidebandExt for S +where + S: Stream + Send + Sized, +{ + #[inline] + fn widen_filter_map(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future> + Send, + U: Send, + { + self.map(f) + .buffered(n.into().unwrap_or(WIDTH)) + .ready_filter_map(identity) + } + + #[inline] + fn widen_then(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + U: Send, + { + self.map(f).buffered(n.into().unwrap_or(WIDTH)) + } +} From 89a158ab0b7ce47849b6b281e91fce584f3d4e93 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 30 Nov 2024 11:54:54 +0000 Subject: [PATCH 1149/2091] add delay before starting updates check Signed-off-by: Jason Volk --- src/service/updates/mod.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index adc85fe6..82decd26 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -1,7 +1,7 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{debug, info, warn, Result}; +use conduit::{debug, info, warn, Result, Server}; use database::{Deserialized, Map}; use ruma::events::room::message::RoomMessageEventContent; use serde::Deserialize; @@ -23,6 +23,7 @@ struct Services { admin: Dep, client: Dep, globals: Dep, + server: Arc, } #[derive(Debug, Deserialize)] @@ -52,11 +53,12 @@ impl crate::Service for Service { globals: args.depend::("globals"), admin: args.depend::("admin"), client: args.depend::("client"), + server: args.server.clone(), }, })) } - #[tracing::instrument(skip_all, name = "updates", level = "trace")] + #[tracing::instrument(skip_all, name = "updates", level = "debug")] async fn worker(self: Arc) -> Result<()> { if !self.services.globals.allow_check_for_updates() { debug!("Disabling update check"); @@ -65,6 +67,7 @@ impl crate::Service for Service { let mut i = interval(self.interval); i.set_missed_tick_behavior(MissedTickBehavior::Delay); + i.reset_after(self.interval); loop { tokio::select! { () = self.interrupt.notified() => break, @@ -85,8 +88,10 @@ impl crate::Service for Service { } impl Service { - #[tracing::instrument(skip_all, level = "trace")] + #[tracing::instrument(skip_all)] async fn check(&self) -> Result<()> { + debug_assert!(self.services.server.running(), "server must not be shutting down"); + let response = self .services .client @@ -108,6 +113,7 @@ impl Service { Ok(()) } + #[tracing::instrument(skip_all)] async fn handle(&self, update: &CheckForUpdatesResponseEntry) { info!("{} {:#}", update.date, update.message); self.services From 9a9c071e8204604d617b213c6a977f5649ba2fc0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 30 Nov 2024 03:16:57 +0000 Subject: [PATCH 1150/2091] use tokio for threadpool mgmt Signed-off-by: Jason Volk --- src/database/database.rs | 2 +- src/database/engine.rs | 10 +- src/database/map/get.rs | 30 +++--- src/database/pool.rs | 204 +++++++++++++++++++++++++-------------- 4 files changed, 155 insertions(+), 91 deletions(-) diff --git a/src/database/database.rs b/src/database/database.rs index 40aec312..3df95dce 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -16,7 +16,7 @@ pub struct Database { impl Database { /// Load an existing database or create a new one. pub async fn open(server: &Arc) -> Result> { - let db = Engine::open(server)?; + let db = Engine::open(server).await?; Ok(Arc::new(Self { db: db.clone(), maps: maps::open(&db)?, diff --git a/src/database/engine.rs b/src/database/engine.rs index e700be62..a6ed7d86 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -23,7 +23,7 @@ use crate::{ }; pub struct Engine { - server: Arc, + pub(crate) server: Arc, row_cache: Cache, col_cache: RwLock>, opts: Options, @@ -40,7 +40,7 @@ pub(crate) type Db = DBWithThreadMode; impl Engine { #[tracing::instrument(skip_all)] - pub(crate) fn open(server: &Arc) -> Result> { + pub(crate) async fn open(server: &Arc) -> Result> { let config = &server.config; let cache_capacity_bytes = config.db_cache_capacity_mb * 1024.0 * 1024.0; @@ -119,7 +119,7 @@ impl Engine { corks: AtomicU32::new(0), read_only: config.rocksdb_read_only, secondary: config.rocksdb_secondary, - pool: Pool::new(&pool_opts)?, + pool: Pool::new(server, &pool_opts).await?, })) } @@ -305,7 +305,7 @@ pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> { Ok(()) } -#[tracing::instrument(skip_all, name = "rocksdb", level = "debug")] +#[tracing::instrument(skip(msg), name = "rocksdb", level = "trace")] pub(crate) fn handle_log(level: LogLevel, msg: &str) { let msg = msg.trim(); if msg.starts_with("Options") { @@ -325,7 +325,7 @@ impl Drop for Engine { fn drop(&mut self) { const BLOCKING: bool = true; - debug!("Joining request threads..."); + debug!("Shutting down request pool..."); self.pool.close(); debug!("Waiting for background tasks to finish..."); diff --git a/src/database/map/get.rs b/src/database/map/get.rs index befc0b24..4699fec4 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -2,7 +2,7 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; use arrayvec::ArrayVec; use conduit::{err, implement, utils::IterStream, Err, Result}; -use futures::{future, Future, FutureExt, Stream}; +use futures::{future, Future, FutureExt, Stream, StreamExt}; use rocksdb::DBPinnableSlice; use serde::Serialize; @@ -54,6 +54,18 @@ where self.get(key) } +#[implement(super::Map)] +#[tracing::instrument(skip(self, keys), fields(%self), level = "trace")] +pub fn get_batch<'a, I, K>(self: &'a Arc, keys: I) -> impl Stream>> + Send + 'a +where + I: Iterator + ExactSizeIterator + Debug + Send + 'a, + K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, +{ + keys.stream() + .map(move |key| self.get(key)) + .buffered(self.db.server.config.db_pool_workers.saturating_mul(2)) +} + /// Fetch a value from the database into cache, returning a reference-handle /// asynchronously. The key is referenced directly to perform the query. #[implement(super::Map)] @@ -80,17 +92,8 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self, keys), fields(%self), level = "trace")] -pub fn get_batch<'a, I, K>(&self, keys: I) -> impl Stream>> -where - I: Iterator + ExactSizeIterator + Debug + Send, - K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, -{ - self.get_batch_blocking(keys).stream() -} - -#[implement(super::Map)] -pub fn get_batch_blocking<'a, I, K>(&self, keys: I) -> impl Iterator>> +#[tracing::instrument(skip(self, keys), name = "batch_blocking", level = "trace")] +pub(crate) fn get_batch_blocking<'a, I, K>(&self, keys: I) -> impl Iterator>> + Send where I: Iterator + ExactSizeIterator + Debug + Send, K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, @@ -111,6 +114,7 @@ where /// The key is referenced directly to perform the query. This is a thread- /// blocking call. #[implement(super::Map)] +#[tracing::instrument(skip(self, key), name = "blocking", level = "trace")] pub fn get_blocking(&self, key: &K) -> Result> where K: AsRef<[u8]> + ?Sized, @@ -125,7 +129,7 @@ where /// Fetch a value from the cache without I/O. #[implement(super::Map)] -#[tracing::instrument(skip(self, key), fields(%self), level = "trace")] +#[tracing::instrument(skip(self, key), name = "cache", level = "trace")] pub(crate) fn get_cached(&self, key: &K) -> Result>> where K: AsRef<[u8]> + Debug + ?Sized, diff --git a/src/database/pool.rs b/src/database/pool.rs index ee3e67dd..a9697625 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -1,20 +1,25 @@ use std::{ - convert::identity, mem::take, - sync::{Arc, Mutex}, - thread::JoinHandle, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, }; -use async_channel::{bounded, Receiver, Sender}; -use conduit::{debug, defer, err, implement, Result}; +use async_channel::{bounded, Receiver, RecvError, Sender}; +use conduit::{debug, debug_warn, defer, err, implement, result::DebugInspect, Result, Server}; use futures::channel::oneshot; +use tokio::{sync::Mutex, task::JoinSet}; use crate::{keyval::KeyBuf, Handle, Map}; pub(crate) struct Pool { - workers: Mutex>>, - recv: Receiver, - send: Sender, + server: Arc, + workers: Mutex>, + queue: Sender, + busy: AtomicUsize, + busy_max: AtomicUsize, + queued_max: AtomicUsize, } pub(crate) struct Opts { @@ -22,10 +27,6 @@ pub(crate) struct Opts { pub(crate) worker_num: usize, } -const QUEUE_LIMIT: (usize, usize) = (1, 8192); -const WORKER_LIMIT: (usize, usize) = (1, 512); -const WORKER_THREAD_NAME: &str = "conduwuit:db"; - #[derive(Debug)] pub(crate) enum Cmd { Get(Get), @@ -40,83 +41,111 @@ pub(crate) struct Get { type ResultSender = oneshot::Sender>>; -#[implement(Pool)] -pub(crate) fn new(opts: &Opts) -> Result> { - let queue_size = opts.queue_size.clamp(QUEUE_LIMIT.0, QUEUE_LIMIT.1); +const QUEUE_LIMIT: (usize, usize) = (1, 3072); +const WORKER_LIMIT: (usize, usize) = (1, 512); +impl Drop for Pool { + fn drop(&mut self) { + debug_assert!(self.queue.is_empty(), "channel must be empty on drop"); + debug_assert!(self.queue.is_closed(), "channel should be closed on drop"); + } +} + +#[implement(Pool)] +pub(crate) async fn new(server: &Arc, opts: &Opts) -> Result> { + let queue_size = opts.queue_size.clamp(QUEUE_LIMIT.0, QUEUE_LIMIT.1); let (send, recv) = bounded(queue_size); let pool = Arc::new(Self { - workers: Vec::new().into(), - recv, - send, + server: server.clone(), + workers: JoinSet::new().into(), + queue: send, + busy: AtomicUsize::default(), + busy_max: AtomicUsize::default(), + queued_max: AtomicUsize::default(), }); let worker_num = opts.worker_num.clamp(WORKER_LIMIT.0, WORKER_LIMIT.1); - pool.spawn_until(worker_num)?; + pool.spawn_until(recv, worker_num).await?; Ok(pool) } #[implement(Pool)] -fn spawn_until(self: &Arc, max: usize) -> Result { - let mut workers = self.workers.lock()?; +pub(crate) async fn _shutdown(self: &Arc) { + if !self.queue.is_closed() { + self.close(); + } + let workers = take(&mut *self.workers.lock().await); + debug!(workers = workers.len(), "Waiting for workers to join..."); + + workers.join_all().await; + debug_assert!(self.queue.is_empty(), "channel is not empty"); +} + +#[implement(Pool)] +pub(crate) fn close(&self) { + debug_assert!(!self.queue.is_closed(), "channel already closed"); + debug!( + senders = self.queue.sender_count(), + receivers = self.queue.receiver_count(), + "Closing pool channel" + ); + + let closing = self.queue.close(); + debug_assert!(closing, "channel is not closing"); +} + +#[implement(Pool)] +async fn spawn_until(self: &Arc, recv: Receiver, max: usize) -> Result { + let mut workers = self.workers.lock().await; while workers.len() < max { - self.clone().spawn_one(&mut workers)?; + self.spawn_one(&mut workers, recv.clone())?; } Ok(()) } #[implement(Pool)] -fn spawn_one(self: Arc, workers: &mut Vec>) -> Result { - use std::thread::Builder; - +fn spawn_one(self: &Arc, workers: &mut JoinSet<()>, recv: Receiver) -> Result { let id = workers.len(); - debug!(?id, "spawning {WORKER_THREAD_NAME}..."); - let thread = Builder::new() - .name(WORKER_THREAD_NAME.into()) - .spawn(move || self.worker(id))?; + debug!(?id, "spawning"); + let self_ = self.clone(); + let _abort = workers.spawn_blocking_on(move || self_.worker(id, recv), self.server.runtime()); - workers.push(thread); - - Ok(id) + Ok(()) } #[implement(Pool)] -pub(crate) fn close(self: &Arc) { - debug!( - senders = %self.send.sender_count(), - receivers = %self.send.receiver_count(), - "Closing pool channel" - ); - let closing = self.send.close(); - debug_assert!(closing, "channel is not closing"); - - debug!("Shutting down pool..."); - let mut workers = self.workers.lock().expect("locked"); - - debug!( - workers = %workers.len(), - "Waiting for workers to join..." - ); - take(&mut *workers) - .into_iter() - .map(JoinHandle::join) - .try_for_each(identity) - .expect("failed to join worker threads"); - - debug_assert!(self.send.is_empty(), "channel is not empty"); -} - -#[implement(Pool)] -#[tracing::instrument(skip(self, cmd), level = "trace")] +#[tracing::instrument( + level = "trace" + skip(self, cmd), + fields( + task = ?tokio::task::try_id(), + receivers = self.queue.receiver_count(), + senders = self.queue.sender_count(), + queued = self.queue.len(), + queued_max = self.queued_max.load(Ordering::Relaxed), + ), +)] pub(crate) async fn execute(&self, mut cmd: Cmd) -> Result> { let (send, recv) = oneshot::channel(); Self::prepare(&mut cmd, send); - self.send + if cfg!(debug_assertions) { + self.queued_max + .fetch_max(self.queue.len(), Ordering::Relaxed); + } + + if self.queue.is_full() { + debug_warn!( + capacity = ?self.queue.capacity(), + "pool queue is full" + ); + } + + self.queue .send(cmd) .await .map_err(|e| err!(error!("send failed {e:?}")))?; @@ -136,30 +165,61 @@ fn prepare(cmd: &mut Cmd, send: ResultSender) { } #[implement(Pool)] -#[tracing::instrument(skip(self))] -fn worker(self: Arc, id: usize) { - debug!(?id, "worker spawned"); - defer! {{ debug!(?id, "worker finished"); }} - self.worker_loop(id); +#[tracing::instrument(skip(self, recv))] +fn worker(self: Arc, id: usize, recv: Receiver) { + debug!("worker spawned"); + defer! {{ debug!("worker finished"); }} + + self.worker_loop(&recv); } #[implement(Pool)] -fn worker_loop(&self, id: usize) { - while let Ok(mut cmd) = self.recv.recv_blocking() { - self.worker_handle(id, &mut cmd); +fn worker_loop(&self, recv: &Receiver) { + // initial +1 needed prior to entering wait + self.busy.fetch_add(1, Ordering::Relaxed); + + while let Ok(mut cmd) = self.worker_wait(recv) { + self.worker_handle(&mut cmd); } } #[implement(Pool)] -fn worker_handle(&self, id: usize, cmd: &mut Cmd) { +#[tracing::instrument( + name = "wait", + level = "trace", + skip_all, + fields( + receivers = recv.receiver_count(), + senders = recv.sender_count(), + queued = recv.len(), + busy = self.busy.load(Ordering::Relaxed), + busy_max = self.busy_max.fetch_max( + self.busy.fetch_sub(1, Ordering::Relaxed), + Ordering::Relaxed + ), + ), +)] +fn worker_wait(&self, recv: &Receiver) -> Result { + recv.recv_blocking().debug_inspect(|_| { + self.busy.fetch_add(1, Ordering::Relaxed); + }) +} + +#[implement(Pool)] +fn worker_handle(&self, cmd: &mut Cmd) { match cmd { - Cmd::Get(get) => self.handle_get(id, get), + Cmd::Get(cmd) => self.handle_get(cmd), } } #[implement(Pool)] -#[tracing::instrument(skip(self, cmd), fields(%cmd.map), level = "trace")] -fn handle_get(&self, id: usize, cmd: &mut Get) { +#[tracing::instrument( + name = "get", + level = "trace", + skip_all, + fields(%cmd.map), +)] +fn handle_get(&self, cmd: &mut Get) { debug_assert!(!cmd.key.is_empty(), "querying for empty key"); // Obtain the result channel. From ed8c21ac9aa1be607195985249ec0f9915b41afd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 30 Nov 2024 15:13:35 +0000 Subject: [PATCH 1151/2091] modernize async srv lookup Signed-off-by: Jason Volk --- src/service/resolver/actual.rs | 41 +++++++++++++--------------------- src/service/resolver/mod.rs | 7 ------ 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index fec29133..6589a58b 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -1,11 +1,10 @@ use std::{ fmt::Debug, net::{IpAddr, SocketAddr}, - sync::Arc, }; use conduit::{debug, debug_error, debug_info, debug_warn, err, error, trace, Err, Result}; -use hickory_resolver::{error::ResolveError, lookup::SrvLookup}; +use hickory_resolver::error::ResolveError; use ipaddress::IPAddress; use ruma::ServerName; @@ -258,7 +257,7 @@ impl super::Service { #[tracing::instrument(skip_all, name = "ip")] async fn query_and_cache_override(&self, overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { - match self.raw().lookup_ip(hostname.to_owned()).await { + match self.resolver.resolver.lookup_ip(hostname.to_owned()).await { Err(e) => Self::handle_resolve_error(&e), Ok(override_ip) => { if hostname != overname { @@ -281,32 +280,24 @@ impl super::Service { #[tracing::instrument(skip_all, name = "srv")] async fn query_srv_record(&self, hostname: &'_ str) -> Result> { - fn handle_successful_srv(srv: &SrvLookup) -> Option { - srv.iter().next().map(|result| { - FedDest::Named( - result.target().to_string().trim_end_matches('.').to_owned(), - format!(":{}", result.port()) - .as_str() - .try_into() - .unwrap_or_else(|_| FedDest::default_port()), - ) - }) - } - - async fn lookup_srv( - resolver: Arc, hostname: &str, - ) -> Result { - debug!("querying SRV for {hostname:?}"); - let hostname = hostname.trim_end_matches('.'); - resolver.srv_lookup(hostname.to_owned()).await - } - let hostnames = [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")]; for hostname in hostnames { - match lookup_srv(self.raw(), &hostname).await { - Ok(result) => return Ok(handle_successful_srv(&result)), + debug!("querying SRV for {hostname:?}"); + let hostname = hostname.trim_end_matches('.'); + match self.resolver.resolver.srv_lookup(hostname).await { Err(e) => Self::handle_resolve_error(&e)?, + Ok(result) => { + return Ok(result.iter().next().map(|result| { + FedDest::Named( + result.target().to_string().trim_end_matches('.').to_owned(), + format!(":{}", result.port()) + .as_str() + .try_into() + .unwrap_or_else(|_| FedDest::default_port()), + ) + })) + }, } } diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 457ea9cc..28b7063d 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -7,7 +7,6 @@ mod tests; use std::{fmt::Write, sync::Arc}; use conduit::{Result, Server}; -use hickory_resolver::TokioAsyncResolver; use self::{cache::Cache, dns::Resolver}; use crate::{client, globals, Dep}; @@ -71,9 +70,3 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } - -impl Service { - #[inline] - #[must_use] - pub fn raw(&self) -> Arc { self.resolver.resolver.clone() } -} From 320b0680bdbbb2f08790a178ea2a75c5a0dee11f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 30 Nov 2024 08:31:28 +0000 Subject: [PATCH 1152/2091] pipeline various loops Signed-off-by: Jason Volk --- src/api/client/context.rs | 39 ++++++++++++------------ src/api/client/message.rs | 7 +++-- src/service/rooms/search/mod.rs | 12 +++++--- src/service/rooms/state/mod.rs | 29 +++++++++++------- src/service/rooms/state_accessor/data.rs | 14 ++++----- 5 files changed, 56 insertions(+), 45 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 652e17f4..af4e26f0 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,9 +1,13 @@ -use std::{collections::HashMap, iter::once}; +use std::iter::once; use axum::extract::State; use conduit::{ - at, err, error, - utils::{future::TryExtExt, stream::ReadyExt, IterStream}, + at, err, + utils::{ + future::TryExtExt, + stream::{BroadbandExt, ReadyExt, WidebandExt}, + IterStream, + }, Err, Result, }; use futures::{future::try_join, StreamExt, TryFutureExt}; @@ -85,8 +89,8 @@ pub(crate) async fn get_context_route( .pdus_rev(Some(sender_user), room_id, Some(base_token)) .await? .ready_filter_map(|item| event_filter(item, filter)) - .filter_map(|item| ignored_filter(&services, item, sender_user)) - .filter_map(|item| visibility_filter(&services, item, sender_user)) + .wide_filter_map(|item| ignored_filter(&services, item, sender_user)) + .wide_filter_map(|item| visibility_filter(&services, item, sender_user)) .take(limit / 2) .collect() .await; @@ -97,8 +101,8 @@ pub(crate) async fn get_context_route( .pdus(Some(sender_user), room_id, Some(base_token)) .await? .ready_filter_map(|item| event_filter(item, filter)) - .filter_map(|item| ignored_filter(&services, item, sender_user)) - .filter_map(|item| visibility_filter(&services, item, sender_user)) + .wide_filter_map(|item| ignored_filter(&services, item, sender_user)) + .wide_filter_map(|item| visibility_filter(&services, item, sender_user)) .take(limit / 2) .collect() .await; @@ -124,7 +128,7 @@ pub(crate) async fn get_context_route( .await .map_err(|e| err!(Database("State hash not found: {e}")))?; - let state_ids: HashMap<_, OwnedEventId> = services + let state_ids = services .rooms .state_accessor .state_full_ids(shortstatehash) @@ -133,17 +137,17 @@ pub(crate) async fn get_context_route( let lazy = &lazy; let state: Vec<_> = state_ids - .iter() + .into_iter() .stream() - .filter_map(|(shortstatekey, event_id)| { + .broad_filter_map(|(shortstatekey, event_id)| { services .rooms .short - .get_statekey_from_short(*shortstatekey) + .get_statekey_from_short(shortstatekey) .map_ok(move |(event_type, state_key)| (event_type, state_key, event_id)) .ok() }) - .filter_map(|(event_type, state_key, event_id)| async move { + .ready_filter_map(|(event_type, state_key, event_id)| { if lazy_load_enabled && event_type == StateEventType::RoomMember { let user_id: &UserId = state_key.as_str().try_into().ok()?; if !lazy.contains(user_id) { @@ -151,15 +155,10 @@ pub(crate) async fn get_context_route( } } - services - .rooms - .timeline - .get_pdu(event_id) - .await - .inspect_err(|_| error!("Pdu in state not found: {event_id}")) - .map(|pdu| pdu.to_state_event()) - .ok() + Some(event_id) }) + .broad_filter_map(|event_id: OwnedEventId| async move { services.rooms.timeline.get_pdu(&event_id).await.ok() }) + .map(|pdu| pdu.to_state_event()) .collect() .await; diff --git a/src/api/client/message.rs b/src/api/client/message.rs index d8043855..242c1681 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -5,6 +5,7 @@ use conduit::{ at, is_equal_to, utils::{ result::{FlatOk, LogErr}, + stream::{BroadbandExt, WidebandExt}, IterStream, ReadyExt, }, Event, PduCount, Result, @@ -115,8 +116,8 @@ pub(crate) async fn get_message_events_route( let events: Vec<_> = it .ready_take_while(|(count, _)| Some(*count) != to) .ready_filter_map(|item| event_filter(item, filter)) - .filter_map(|item| ignored_filter(&services, item, sender_user)) - .filter_map(|item| visibility_filter(&services, item, sender_user)) + .wide_filter_map(|item| ignored_filter(&services, item, sender_user)) + .wide_filter_map(|item| visibility_filter(&services, item, sender_user)) .take(limit) .collect() .await; @@ -132,7 +133,7 @@ pub(crate) async fn get_message_events_route( let state = lazy .iter() .stream() - .filter_map(|user_id| get_member_event(&services, room_id, user_id)) + .broad_filter_map(|user_id| get_member_event(&services, room_id, user_id)) .collect() .await; diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index d59d1d11..ae3567ce 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -3,7 +3,11 @@ use std::sync::Arc; use arrayvec::ArrayVec; use conduit::{ implement, - utils::{set, stream::TryIgnore, ArrayVecExt, IterStream, ReadyExt}, + utils::{ + set, + stream::{TryIgnore, WidebandExt}, + ArrayVecExt, IterStream, ReadyExt, + }, PduCount, PduEvent, Result, }; use database::{keyval::Val, Map}; @@ -107,7 +111,7 @@ pub async fn search_pdus<'a>( let pdus = pdu_ids .into_iter() .stream() - .filter_map(move |result_pdu_id: RawPduId| async move { + .wide_filter_map(move |result_pdu_id: RawPduId| async move { self.services .timeline .get_pdu_from_id(&result_pdu_id) @@ -116,7 +120,7 @@ pub async fn search_pdus<'a>( }) .ready_filter(|pdu| !pdu.is_redacted()) .ready_filter(|pdu| pdu.matches(&query.criteria.filter)) - .filter_map(move |pdu| async move { + .wide_filter_map(move |pdu| async move { self.services .state_accessor .user_can_see_event(query.user_id?, &pdu.room_id, &pdu.event_id) @@ -146,7 +150,7 @@ pub async fn search_pdu_ids(&self, query: &RoomQuery<'_>) -> Result, shortroomid: ShortRoomId) -> Vec> { tokenize(&query.criteria.search_term) .stream() - .then(|word| async move { + .wide_then(|word| async move { self.search_pdu_ids_query_words(shortroomid, &word) .collect::>() .await diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 03e2d2e8..838deacd 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -8,7 +8,11 @@ use std::{ use conduit::{ at, err, result::FlatOk, - utils::{calculate_hash, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, + utils::{ + calculate_hash, + stream::{BroadbandExt, TryIgnore}, + IterStream, MutexMap, MutexMapGuard, ReadyExt, + }, warn, PduEvent, Result, }; use database::{Deserialized, Ignore, Interfix, Map}; @@ -405,7 +409,7 @@ impl Service { let mut sauthevents: HashMap<_, _> = state_res::auth_types_for_event(kind, sender, state_key, content)? .iter() .stream() - .filter_map(|(event_type, state_key)| { + .broad_filter_map(|(event_type, state_key)| { self.services .short .get_shortstatekey(event_type, state_key) @@ -430,24 +434,27 @@ impl Service { }) .collect(); - let auth_pdus: Vec<_> = self + let auth_pdus = self .services .short .multi_get_eventid_from_short(auth_state.iter().map(at!(1))) .await .into_iter() .stream() - .and_then(|event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await }) + .zip(auth_state.into_iter().stream().map(at!(0))) + .ready_filter_map(|(event_id, tsk)| Some((tsk, event_id.ok()?))) + .broad_filter_map(|(tsk, event_id): (_, OwnedEventId)| async move { + self.services + .timeline + .get_pdu(&event_id) + .await + .map(Arc::new) + .map(move |pdu| (tsk, pdu)) + .ok() + }) .collect() .await; - let auth_pdus = auth_state - .into_iter() - .map(at!(0)) - .zip(auth_pdus.into_iter()) - .filter_map(|((event_type, state_key), pdu)| Some(((event_type, state_key), pdu.ok()?.into()))) - .collect(); - Ok(auth_pdus) } } diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 7760d5b6..2a670066 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -2,7 +2,7 @@ use std::{borrow::Borrow, collections::HashMap, sync::Arc}; use conduit::{ at, err, - utils::stream::{IterStream, ReadyExt}, + utils::stream::{BroadbandExt, IterStream}, PduEvent, Result, }; use database::{Deserialized, Map}; @@ -65,20 +65,20 @@ impl Data { .into_iter() .map(at!(1)); - let event_ids: Vec = self + let event_ids = self .services .short .multi_get_eventid_from_short(short_ids) .await .into_iter() - .filter_map(Result::ok) - .collect(); + .filter_map(Result::ok); let full_pdus = event_ids - .iter() + .into_iter() .stream() - .then(|event_id| self.services.timeline.get_pdu(event_id)) - .ready_filter_map(Result::ok) + .broad_filter_map( + |event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await.ok() }, + ) .collect() .await; From b5006a4c4128fc779add9cd6c35b93f0d8d6ba7e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 2 Dec 2024 08:51:59 +0000 Subject: [PATCH 1153/2091] offload initial iterator seeks to threadpool Signed-off-by: Jason Volk --- src/database/map/count.rs | 10 +- src/database/map/get.rs | 16 ++- src/database/map/keys.rs | 4 +- src/database/map/keys_from.rs | 32 ++++-- src/database/map/keys_prefix.rs | 12 +- src/database/map/rev_keys.rs | 4 +- src/database/map/rev_keys_from.rs | 32 ++++-- src/database/map/rev_keys_prefix.rs | 12 +- src/database/map/rev_stream.rs | 4 +- src/database/map/rev_stream_from.rs | 39 +++++-- src/database/map/rev_stream_prefix.rs | 14 ++- src/database/map/stream.rs | 4 +- src/database/map/stream_from.rs | 37 ++++-- src/database/map/stream_prefix.rs | 14 ++- src/database/pool.rs | 158 ++++++++++++++++++-------- src/database/stream.rs | 16 +-- src/database/stream/items.rs | 21 +++- src/database/stream/items_rev.rs | 21 +++- src/database/stream/keys.rs | 21 +++- src/database/stream/keys_rev.rs | 21 +++- 20 files changed, 361 insertions(+), 131 deletions(-) diff --git a/src/database/map/count.rs b/src/database/map/count.rs index 3e92279c..894fe12e 100644 --- a/src/database/map/count.rs +++ b/src/database/map/count.rs @@ -1,4 +1,4 @@ -use std::{fmt::Debug, future::Future}; +use std::{fmt::Debug, future::Future, sync::Arc}; use conduit::implement; use futures::stream::StreamExt; @@ -14,7 +14,7 @@ pub fn count(&self) -> impl Future + Send + '_ { self.raw_keys() /// - From is a structured key #[implement(super::Map)] #[inline] -pub fn count_from<'a, P>(&'a self, from: &P) -> impl Future + Send + 'a +pub fn count_from<'a, P>(self: &'a Arc, from: &P) -> impl Future + Send + 'a where P: Serialize + ?Sized + Debug + 'a, { @@ -26,7 +26,7 @@ where /// - From is a raw #[implement(super::Map)] #[inline] -pub fn raw_count_from<'a, P>(&'a self, from: &'a P) -> impl Future + Send + 'a +pub fn raw_count_from<'a, P>(self: &'a Arc, from: &'a P) -> impl Future + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { @@ -38,7 +38,7 @@ where /// - Prefix is structured key #[implement(super::Map)] #[inline] -pub fn count_prefix<'a, P>(&'a self, prefix: &P) -> impl Future + Send + 'a +pub fn count_prefix<'a, P>(self: &'a Arc, prefix: &P) -> impl Future + Send + 'a where P: Serialize + ?Sized + Debug + 'a, { @@ -50,7 +50,7 @@ where /// - Prefix is raw #[implement(super::Map)] #[inline] -pub fn raw_count_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Future + Send + 'a +pub fn raw_count_prefix<'a, P>(self: &'a Arc, prefix: &'a P) -> impl Future + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 4699fec4..ef552177 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -1,7 +1,11 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; use arrayvec::ArrayVec; -use conduit::{err, implement, utils::IterStream, Err, Result}; +use conduit::{ + err, implement, + utils::{result::MapExpect, IterStream}, + Err, Result, +}; use futures::{future, Future, FutureExt, Stream, StreamExt}; use rocksdb::DBPinnableSlice; use serde::Serialize; @@ -74,21 +78,21 @@ pub fn get(self: &Arc, key: &K) -> impl Future + Debug + ?Sized, { - use crate::pool::{Cmd, Get}; + use crate::pool::Get; let cached = self.get_cached(key); if matches!(cached, Err(_) | Ok(Some(_))) { - return future::ready(cached.map(|res| res.expect("Option is Some"))).boxed(); + return future::ready(cached.map_expect("data found in cache")).boxed(); } debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete"); - let cmd = Cmd::Get(Get { + let cmd = Get { map: self.clone(), key: key.as_ref().into(), res: None, - }); + }; - self.db.pool.execute(cmd).boxed() + self.db.pool.execute_get(cmd).boxed() } #[implement(super::Map)] diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs index 9c4d66e4..80cf1e15 100644 --- a/src/database/map/keys.rs +++ b/src/database/map/keys.rs @@ -2,7 +2,7 @@ use conduit::{implement, Result}; use futures::{Stream, StreamExt}; use serde::Deserialize; -use crate::{keyval, keyval::Key, stream}; +use crate::{keyval, keyval::Key, stream, stream::Cursor}; #[implement(super::Map)] pub fn keys<'a, K>(&'a self) -> impl Stream>> + Send @@ -16,5 +16,5 @@ where #[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn raw_keys(&self) -> impl Stream>> + Send { let opts = super::read_options_default(); - stream::Keys::new(&self.db, &self.cf, opts, None) + stream::Keys::new(&self.db, &self.cf, opts).init(None) } diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 093f7fd6..7be3dd1d 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -1,7 +1,8 @@ -use std::{convert::AsRef, fmt::Debug}; +use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduit::{implement, Result}; -use futures::{Stream, StreamExt}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use rocksdb::Direction; use serde::{Deserialize, Serialize}; use crate::{ @@ -10,7 +11,7 @@ use crate::{ }; #[implement(super::Map)] -pub fn keys_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn keys_from<'a, K, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -20,7 +21,7 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn keys_from_raw

    (&self, from: &P) -> impl Stream>> + Send +pub fn keys_from_raw

    (self: &Arc, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -29,7 +30,7 @@ where } #[implement(super::Map)] -pub fn keys_raw_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn keys_raw_from<'a, K, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, @@ -39,10 +40,27 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn raw_keys_from

    (&self, from: &P) -> impl Stream>> + Send +pub fn raw_keys_from

    (self: &Arc, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, { + use crate::pool::Seek; + let opts = super::read_options_default(); - stream::Keys::new(&self.db, &self.cf, opts, Some(from.as_ref())) + let state = stream::State::new(&self.db, &self.cf, opts); + let seek = Seek { + map: self.clone(), + dir: Direction::Forward, + key: Some(from.as_ref().into()), + state: crate::pool::into_send_seek(state), + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() } diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index 8963f002..9122d78e 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -1,4 +1,4 @@ -use std::{convert::AsRef, fmt::Debug}; +use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduit::{implement, Result}; use futures::{ @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use crate::keyval::{result_deserialize_key, serialize_key, Key}; #[implement(super::Map)] -pub fn keys_prefix<'a, K, P>(&'a self, prefix: &P) -> impl Stream>> + Send +pub fn keys_prefix<'a, K, P>(self: &'a Arc, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -22,7 +22,7 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn keys_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send +pub fn keys_prefix_raw

    (self: &Arc, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -32,7 +32,9 @@ where } #[implement(super::Map)] -pub fn keys_raw_prefix<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn keys_raw_prefix<'a, K, P>( + self: &'a Arc, prefix: &'a P, +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, K: Deserialize<'a> + Send + 'a, @@ -42,7 +44,7 @@ where } #[implement(super::Map)] -pub fn raw_keys_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn raw_keys_prefix<'a, P>(self: &'a Arc, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index e10a199c..45a0203f 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -2,7 +2,7 @@ use conduit::{implement, Result}; use futures::{Stream, StreamExt}; use serde::Deserialize; -use crate::{keyval, keyval::Key, stream}; +use crate::{keyval, keyval::Key, stream, stream::Cursor}; #[implement(super::Map)] pub fn rev_keys<'a, K>(&'a self) -> impl Stream>> + Send @@ -16,5 +16,5 @@ where #[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_raw_keys(&self) -> impl Stream>> + Send { let opts = super::read_options_default(); - stream::KeysRev::new(&self.db, &self.cf, opts, None) + stream::KeysRev::new(&self.db, &self.cf, opts).init(None) } diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index 75d062b5..2b59a5d7 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -1,7 +1,8 @@ -use std::{convert::AsRef, fmt::Debug}; +use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduit::{implement, Result}; -use futures::{Stream, StreamExt}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use rocksdb::Direction; use serde::{Deserialize, Serialize}; use crate::{ @@ -10,7 +11,7 @@ use crate::{ }; #[implement(super::Map)] -pub fn rev_keys_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn rev_keys_from<'a, K, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -21,7 +22,7 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn rev_keys_from_raw

    (&self, from: &P) -> impl Stream>> + Send +pub fn rev_keys_from_raw

    (self: &Arc, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -30,7 +31,7 @@ where } #[implement(super::Map)] -pub fn rev_keys_raw_from<'a, K, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn rev_keys_raw_from<'a, K, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, @@ -41,10 +42,27 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn rev_raw_keys_from

    (&self, from: &P) -> impl Stream>> + Send +pub fn rev_raw_keys_from

    (self: &Arc, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, { + use crate::pool::Seek; + let opts = super::read_options_default(); - stream::KeysRev::new(&self.db, &self.cf, opts, Some(from.as_ref())) + let state = stream::State::new(&self.db, &self.cf, opts); + let seek = Seek { + map: self.clone(), + dir: Direction::Reverse, + key: Some(from.as_ref().into()), + state: crate::pool::into_send_seek(state), + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() } diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index c14909d4..69dc54f2 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -1,4 +1,4 @@ -use std::{convert::AsRef, fmt::Debug}; +use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduit::{implement, Result}; use futures::{ @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use crate::keyval::{result_deserialize_key, serialize_key, Key}; #[implement(super::Map)] -pub fn rev_keys_prefix<'a, K, P>(&'a self, prefix: &P) -> impl Stream>> + Send +pub fn rev_keys_prefix<'a, K, P>(self: &'a Arc, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -22,7 +22,7 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn rev_keys_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send +pub fn rev_keys_prefix_raw

    (self: &Arc, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -32,7 +32,9 @@ where } #[implement(super::Map)] -pub fn rev_keys_raw_prefix<'a, K, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn rev_keys_raw_prefix<'a, K, P>( + self: &'a Arc, prefix: &'a P, +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, K: Deserialize<'a> + Send + 'a, @@ -42,7 +44,7 @@ where } #[implement(super::Map)] -pub fn rev_raw_keys_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn rev_raw_keys_prefix<'a, P>(self: &'a Arc, prefix: &'a P) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index f4be69fd..37b0d3b3 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -2,7 +2,7 @@ use conduit::{implement, Result}; use futures::stream::{Stream, StreamExt}; use serde::Deserialize; -use crate::{keyval, keyval::KeyVal, stream}; +use crate::{keyval, keyval::KeyVal, stream, stream::Cursor}; /// Iterate key-value entries in the map from the end. /// @@ -24,5 +24,5 @@ where #[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_raw_stream(&self) -> impl Stream>> + Send { let opts = super::read_options_default(); - stream::ItemsRev::new(&self.db, &self.cf, opts, None) + stream::ItemsRev::new(&self.db, &self.cf, opts).init(None) } diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 6ac1cd1a..9811d106 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -1,7 +1,11 @@ -use std::{convert::AsRef, fmt::Debug}; +use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduit::{implement, Result}; -use futures::stream::{Stream, StreamExt}; +use futures::{ + stream::{Stream, StreamExt}, + FutureExt, TryFutureExt, TryStreamExt, +}; +use rocksdb::Direction; use serde::{Deserialize, Serialize}; use crate::{ @@ -14,7 +18,9 @@ use crate::{ /// - Query is serialized /// - Result is deserialized #[implement(super::Map)] -pub fn rev_stream_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn rev_stream_from<'a, K, V, P>( + self: &'a Arc, from: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -30,7 +36,7 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn rev_stream_from_raw

    (&self, from: &P) -> impl Stream>> + Send +pub fn rev_stream_from_raw

    (self: &Arc, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -43,7 +49,9 @@ where /// - Query is raw /// - Result is deserialized #[implement(super::Map)] -pub fn rev_stream_raw_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn rev_stream_raw_from<'a, K, V, P>( + self: &'a Arc, from: &P, +) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, @@ -59,10 +67,27 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn rev_raw_stream_from

    (&self, from: &P) -> impl Stream>> + Send +pub fn rev_raw_stream_from

    (self: &Arc, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, { + use crate::pool::Seek; + let opts = super::read_options_default(); - stream::ItemsRev::new(&self.db, &self.cf, opts, Some(from.as_ref())) + let state = stream::State::new(&self.db, &self.cf, opts); + let seek = Seek { + map: self.clone(), + dir: Direction::Reverse, + key: Some(from.as_ref().into()), + state: crate::pool::into_send_seek(state), + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() } diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index fd0d93ff..e5c2fbea 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -1,4 +1,4 @@ -use std::{convert::AsRef, fmt::Debug}; +use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduit::{implement, Result}; use futures::{ @@ -15,7 +15,9 @@ use crate::keyval::{result_deserialize, serialize_key, KeyVal}; /// - Query is serialized /// - Result is deserialized #[implement(super::Map)] -pub fn rev_stream_prefix<'a, K, V, P>(&'a self, prefix: &P) -> impl Stream>> + Send +pub fn rev_stream_prefix<'a, K, V, P>( + self: &'a Arc, prefix: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -31,7 +33,7 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn rev_stream_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send +pub fn rev_stream_prefix_raw

    (self: &Arc, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -46,7 +48,7 @@ where /// - Result is deserialized #[implement(super::Map)] pub fn rev_stream_raw_prefix<'a, K, V, P>( - &'a self, prefix: &'a P, + self: &'a Arc, prefix: &'a P, ) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, @@ -62,7 +64,9 @@ where /// - Query is raw /// - Result is raw #[implement(super::Map)] -pub fn rev_raw_stream_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn rev_raw_stream_prefix<'a, P>( + self: &'a Arc, prefix: &'a P, +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index 143b0d0c..4f4fbd08 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -2,7 +2,7 @@ use conduit::{implement, Result}; use futures::stream::{Stream, StreamExt}; use serde::Deserialize; -use crate::{keyval, keyval::KeyVal, stream}; +use crate::{keyval, keyval::KeyVal, stream, stream::Cursor}; /// Iterate key-value entries in the map from the beginning. /// @@ -23,5 +23,5 @@ where #[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn raw_stream(&self) -> impl Stream>> + Send { let opts = super::read_options_default(); - stream::Items::new(&self.db, &self.cf, opts, None) + stream::Items::new(&self.db, &self.cf, opts).init(None) } diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 052a2e74..6468846f 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -1,7 +1,11 @@ -use std::{convert::AsRef, fmt::Debug}; +use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduit::{implement, Result}; -use futures::stream::{Stream, StreamExt}; +use futures::{ + stream::{Stream, StreamExt}, + FutureExt, TryFutureExt, TryStreamExt, +}; +use rocksdb::Direction; use serde::{Deserialize, Serialize}; use crate::{ @@ -14,7 +18,7 @@ use crate::{ /// - Query is serialized /// - Result is deserialized #[implement(super::Map)] -pub fn stream_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn stream_from<'a, K, V, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -29,7 +33,7 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn stream_from_raw

    (&self, from: &P) -> impl Stream>> + Send +pub fn stream_from_raw

    (self: &Arc, from: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -42,7 +46,9 @@ where /// - Query is raw /// - Result is deserialized #[implement(super::Map)] -pub fn stream_raw_from<'a, K, V, P>(&'a self, from: &P) -> impl Stream>> + Send +pub fn stream_raw_from<'a, K, V, P>( + self: &'a Arc, from: &P, +) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, @@ -57,10 +63,27 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn raw_stream_from

    (&self, from: &P) -> impl Stream>> + Send +pub fn raw_stream_from

    (self: &Arc, from: &P) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, { + use crate::pool::Seek; + let opts = super::read_options_default(); - stream::Items::new(&self.db, &self.cf, opts, Some(from.as_ref())) + let state = stream::State::new(&self.db, &self.cf, opts); + let seek = Seek { + map: self.clone(), + dir: Direction::Forward, + key: Some(from.as_ref().into()), + state: crate::pool::into_send_seek(state), + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() } diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index a08b1e2a..3c7bce2e 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -1,4 +1,4 @@ -use std::{convert::AsRef, fmt::Debug}; +use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduit::{implement, Result}; use futures::{ @@ -15,7 +15,9 @@ use crate::keyval::{result_deserialize, serialize_key, KeyVal}; /// - Query is serialized /// - Result is deserialized #[implement(super::Map)] -pub fn stream_prefix<'a, K, V, P>(&'a self, prefix: &P) -> impl Stream>> + Send +pub fn stream_prefix<'a, K, V, P>( + self: &'a Arc, prefix: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -31,7 +33,7 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn stream_prefix_raw

    (&self, prefix: &P) -> impl Stream>> + Send +pub fn stream_prefix_raw

    (self: &Arc, prefix: &P) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -46,7 +48,7 @@ where /// - Result is deserialized #[implement(super::Map)] pub fn stream_raw_prefix<'a, K, V, P>( - &'a self, prefix: &'a P, + self: &'a Arc, prefix: &'a P, ) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, @@ -62,7 +64,9 @@ where /// - Query is raw /// - Result is raw #[implement(super::Map)] -pub fn raw_stream_prefix<'a, P>(&'a self, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn raw_stream_prefix<'a, P>( + self: &'a Arc, prefix: &'a P, +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/pool.rs b/src/database/pool.rs index a9697625..136de67d 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -8,17 +8,18 @@ use std::{ use async_channel::{bounded, Receiver, RecvError, Sender}; use conduit::{debug, debug_warn, defer, err, implement, result::DebugInspect, Result, Server}; -use futures::channel::oneshot; +use futures::{channel::oneshot, TryFutureExt}; +use oneshot::Sender as ResultSender; +use rocksdb::Direction; use tokio::{sync::Mutex, task::JoinSet}; -use crate::{keyval::KeyBuf, Handle, Map}; +use crate::{keyval::KeyBuf, stream, Handle, Map}; pub(crate) struct Pool { server: Arc, workers: Mutex>, queue: Sender, busy: AtomicUsize, - busy_max: AtomicUsize, queued_max: AtomicUsize, } @@ -27,19 +28,24 @@ pub(crate) struct Opts { pub(crate) worker_num: usize, } -#[derive(Debug)] pub(crate) enum Cmd { Get(Get), + Iter(Seek), } -#[derive(Debug)] pub(crate) struct Get { pub(crate) map: Arc, pub(crate) key: KeyBuf, - pub(crate) res: Option, + pub(crate) res: Option>>>, } -type ResultSender = oneshot::Sender>>; +pub(crate) struct Seek { + pub(crate) map: Arc, + pub(crate) state: stream::State<'static>, + pub(crate) dir: Direction, + pub(crate) key: Option, + pub(crate) res: Option>>, +} const QUEUE_LIMIT: (usize, usize) = (1, 3072); const WORKER_LIMIT: (usize, usize) = (1, 512); @@ -60,7 +66,6 @@ pub(crate) async fn new(server: &Arc, opts: &Opts) -> Result> workers: JoinSet::new().into(), queue: send, busy: AtomicUsize::default(), - busy_max: AtomicUsize::default(), queued_max: AtomicUsize::default(), }); @@ -94,6 +99,8 @@ pub(crate) fn close(&self) { let closing = self.queue.close(); debug_assert!(closing, "channel is not closing"); + + std::thread::yield_now(); } #[implement(Pool)] @@ -117,22 +124,45 @@ fn spawn_one(self: &Arc, workers: &mut JoinSet<()>, recv: Receiver) - Ok(()) } +#[implement(Pool)] +#[tracing::instrument(level = "trace", name = "get", skip(self, cmd))] +pub(crate) async fn execute_get(&self, mut cmd: Get) -> Result> { + let (send, recv) = oneshot::channel(); + _ = cmd.res.insert(send); + self.execute(Cmd::Get(cmd)) + .and_then(|()| { + recv.map_ok(into_recv_get_result) + .map_err(|e| err!(error!("recv failed {e:?}"))) + }) + .await? +} + +#[implement(Pool)] +#[tracing::instrument(level = "trace", name = "iter", skip(self, cmd))] +pub(crate) async fn execute_iter(&self, mut cmd: Seek) -> Result> { + let (send, recv) = oneshot::channel(); + _ = cmd.res.insert(send); + self.execute(Cmd::Iter(cmd)) + .and_then(|()| { + recv.map_ok(into_recv_seek) + .map_err(|e| err!(error!("recv failed {e:?}"))) + }) + .await +} + #[implement(Pool)] #[tracing::instrument( - level = "trace" + level = "trace", + name = "execute", skip(self, cmd), fields( task = ?tokio::task::try_id(), receivers = self.queue.receiver_count(), - senders = self.queue.sender_count(), queued = self.queue.len(), queued_max = self.queued_max.load(Ordering::Relaxed), ), )] -pub(crate) async fn execute(&self, mut cmd: Cmd) -> Result> { - let (send, recv) = oneshot::channel(); - Self::prepare(&mut cmd, send); - +async fn execute(&self, cmd: Cmd) -> Result { if cfg!(debug_assertions) { self.queued_max .fetch_max(self.queue.len(), Ordering::Relaxed); @@ -148,20 +178,7 @@ pub(crate) async fn execute(&self, mut cmd: Cmd) -> Result> { self.queue .send(cmd) .await - .map_err(|e| err!(error!("send failed {e:?}")))?; - - recv.await - .map(into_recv_result) - .map_err(|e| err!(error!("recv failed {e:?}")))? -} - -#[implement(Pool)] -fn prepare(cmd: &mut Cmd, send: ResultSender) { - match cmd { - Cmd::Get(ref mut cmd) => { - _ = cmd.res.insert(send); - }, - }; + .map_err(|e| err!(error!("send failed {e:?}"))) } #[implement(Pool)] @@ -178,8 +195,8 @@ fn worker_loop(&self, recv: &Receiver) { // initial +1 needed prior to entering wait self.busy.fetch_add(1, Ordering::Relaxed); - while let Ok(mut cmd) = self.worker_wait(recv) { - self.worker_handle(&mut cmd); + while let Ok(cmd) = self.worker_wait(recv) { + self.worker_handle(cmd); } } @@ -190,13 +207,8 @@ fn worker_loop(&self, recv: &Receiver) { skip_all, fields( receivers = recv.receiver_count(), - senders = recv.sender_count(), queued = recv.len(), - busy = self.busy.load(Ordering::Relaxed), - busy_max = self.busy_max.fetch_max( - self.busy.fetch_sub(1, Ordering::Relaxed), - Ordering::Relaxed - ), + busy = self.busy.fetch_sub(1, Ordering::Relaxed) - 1, ), )] fn worker_wait(&self, recv: &Receiver) -> Result { @@ -206,12 +218,60 @@ fn worker_wait(&self, recv: &Receiver) -> Result { } #[implement(Pool)] -fn worker_handle(&self, cmd: &mut Cmd) { +fn worker_handle(&self, cmd: Cmd) { match cmd { Cmd::Get(cmd) => self.handle_get(cmd), + Cmd::Iter(cmd) => self.handle_iter(cmd), } } +#[implement(Pool)] +#[tracing::instrument( + name = "iter", + level = "trace", + skip_all, + fields(%cmd.map), +)] +fn handle_iter(&self, mut cmd: Seek) { + let chan = cmd.res.take().expect("missing result channel"); + + if chan.is_canceled() { + return; + } + + let from = cmd.key.as_deref().map(Into::into); + let result = match cmd.dir { + Direction::Forward => cmd.state.init_fwd(from), + Direction::Reverse => cmd.state.init_rev(from), + }; + + let chan_result = chan.send(into_send_seek(result)); + let _chan_sent = chan_result.is_ok(); +} + +#[implement(Pool)] +#[tracing::instrument( + name = "seek", + level = "trace", + skip_all, + fields(%cmd.map), +)] +fn _handle_seek(&self, mut cmd: Seek) { + let chan = cmd.res.take().expect("missing result channel"); + + if chan.is_canceled() { + return; + } + + match cmd.dir { + Direction::Forward => cmd.state.seek_fwd(), + Direction::Reverse => cmd.state.seek_rev(), + }; + + let chan_result = chan.send(into_send_seek(cmd.state)); + let _chan_sent = chan_result.is_ok(); +} + #[implement(Pool)] #[tracing::instrument( name = "get", @@ -219,7 +279,7 @@ fn worker_handle(&self, cmd: &mut Cmd) { skip_all, fields(%cmd.map), )] -fn handle_get(&self, cmd: &mut Get) { +fn handle_get(&self, mut cmd: Get) { debug_assert!(!cmd.key.is_empty(), "querying for empty key"); // Obtain the result channel. @@ -237,23 +297,31 @@ fn handle_get(&self, cmd: &mut Get) { let result = cmd.map.get_blocking(&cmd.key); // Send the result back to the submitter. - let chan_result = chan.send(into_send_result(result)); + let chan_result = chan.send(into_send_get_result(result)); // If the future was dropped during the query this will fail acceptably. let _chan_sent = chan_result.is_ok(); } -fn into_send_result(result: Result>) -> Result> { +fn into_send_get_result(result: Result>) -> Result> { // SAFETY: Necessary to send the Handle (rust_rocksdb::PinnableSlice) through // the channel. The lifetime on the handle is a device by rust-rocksdb to // associate a database lifetime with its assets. The Handle must be dropped - // before the database is dropped. The handle must pass through recv_handle() on - // the other end of the channel. + // before the database is dropped. unsafe { std::mem::transmute(result) } } -fn into_recv_result(result: Result>) -> Result> { - // SAFETY: This is to receive the Handle from the channel. Previously it had - // passed through send_handle(). +fn into_recv_get_result(result: Result>) -> Result> { + // SAFETY: This is to receive the Handle from the channel. + unsafe { std::mem::transmute(result) } +} + +pub(crate) fn into_send_seek(result: stream::State<'_>) -> stream::State<'static> { + // SAFETY: Necessary to send the State through the channel; see above. + unsafe { std::mem::transmute(result) } +} + +fn into_recv_seek(result: stream::State<'static>) -> stream::State<'_> { + // SAFETY: This is to receive the State from the channel; see above. unsafe { std::mem::transmute(result) } } diff --git a/src/database/stream.rs b/src/database/stream.rs index a2a72e44..38c46596 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -16,19 +16,21 @@ use crate::{ Engine, Slice, }; -struct State<'a> { +pub(crate) struct State<'a> { inner: Inner<'a>, seek: bool, init: bool, } -trait Cursor<'a, T> { +pub(crate) trait Cursor<'a, T> { fn state(&self) -> &State<'a>; fn fetch(&self) -> Option; fn seek(&mut self); + fn init(self, from: From<'a>) -> Self; + fn get(&self) -> Option> { self.fetch() .map(Ok) @@ -45,7 +47,7 @@ type Inner<'a> = DBRawIteratorWithThreadMode<'a, Db>; type From<'a> = Option>; impl<'a> State<'a> { - fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { + pub(super) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { Self { inner: db.db.raw_iterator_cf_opt(&**cf, opts), init: true, @@ -53,7 +55,7 @@ impl<'a> State<'a> { } } - fn init_fwd(mut self, from: From<'_>) -> Self { + pub(super) fn init_fwd(mut self, from: From<'_>) -> Self { if let Some(key) = from { self.inner.seek(key); self.seek = true; @@ -62,7 +64,7 @@ impl<'a> State<'a> { self } - fn init_rev(mut self, from: From<'_>) -> Self { + pub(super) fn init_rev(mut self, from: From<'_>) -> Self { if let Some(key) = from { self.inner.seek_for_prev(key); self.seek = true; @@ -72,7 +74,7 @@ impl<'a> State<'a> { } #[inline] - fn seek_fwd(&mut self) { + pub(super) fn seek_fwd(&mut self) { if !exchange(&mut self.init, false) { self.inner.next(); } else if !self.seek { @@ -81,7 +83,7 @@ impl<'a> State<'a> { } #[inline] - fn seek_rev(&mut self) { + pub(super) fn seek_rev(&mut self) { if !exchange(&mut self.init, false) { self.inner.prev(); } else if !self.seek { diff --git a/src/database/stream/items.rs b/src/database/stream/items.rs index 54f8bc5c..77b08a0b 100644 --- a/src/database/stream/items.rs +++ b/src/database/stream/items.rs @@ -1,4 +1,4 @@ -use std::{pin::Pin, sync::Arc}; +use std::{convert, pin::Pin, sync::Arc}; use conduit::Result; use futures::{ @@ -16,9 +16,17 @@ pub(crate) struct Items<'a> { } impl<'a> Items<'a> { - pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions, from: From<'_>) -> Self { + pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { Self { - state: State::new(db, cf, opts).init_fwd(from), + state: State::new(db, cf, opts), + } + } +} + +impl<'a> convert::From> for Items<'a> { + fn from(state: State<'a>) -> Self { + Self { + state, } } } @@ -30,6 +38,13 @@ impl<'a> Cursor<'a, KeyVal<'a>> for Items<'a> { #[inline] fn seek(&mut self) { self.state.seek_fwd(); } + + #[inline] + fn init(self, from: From<'a>) -> Self { + Self { + state: self.state.init_fwd(from), + } + } } impl<'a> Stream for Items<'a> { diff --git a/src/database/stream/items_rev.rs b/src/database/stream/items_rev.rs index 26492db8..dfd3a107 100644 --- a/src/database/stream/items_rev.rs +++ b/src/database/stream/items_rev.rs @@ -1,4 +1,4 @@ -use std::{pin::Pin, sync::Arc}; +use std::{convert, pin::Pin, sync::Arc}; use conduit::Result; use futures::{ @@ -16,9 +16,17 @@ pub(crate) struct ItemsRev<'a> { } impl<'a> ItemsRev<'a> { - pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions, from: From<'_>) -> Self { + pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { Self { - state: State::new(db, cf, opts).init_rev(from), + state: State::new(db, cf, opts), + } + } +} + +impl<'a> convert::From> for ItemsRev<'a> { + fn from(state: State<'a>) -> Self { + Self { + state, } } } @@ -30,6 +38,13 @@ impl<'a> Cursor<'a, KeyVal<'a>> for ItemsRev<'a> { #[inline] fn seek(&mut self) { self.state.seek_rev(); } + + #[inline] + fn init(self, from: From<'a>) -> Self { + Self { + state: self.state.init_rev(from), + } + } } impl<'a> Stream for ItemsRev<'a> { diff --git a/src/database/stream/keys.rs b/src/database/stream/keys.rs index 91884c8d..2ce88959 100644 --- a/src/database/stream/keys.rs +++ b/src/database/stream/keys.rs @@ -1,4 +1,4 @@ -use std::{pin::Pin, sync::Arc}; +use std::{convert, pin::Pin, sync::Arc}; use conduit::Result; use futures::{ @@ -16,9 +16,17 @@ pub(crate) struct Keys<'a> { } impl<'a> Keys<'a> { - pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions, from: From<'_>) -> Self { + pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { Self { - state: State::new(db, cf, opts).init_fwd(from), + state: State::new(db, cf, opts), + } + } +} + +impl<'a> convert::From> for Keys<'a> { + fn from(state: State<'a>) -> Self { + Self { + state, } } } @@ -31,6 +39,13 @@ impl<'a> Cursor<'a, Key<'a>> for Keys<'a> { #[inline] fn seek(&mut self) { self.state.seek_fwd(); } + + #[inline] + fn init(self, from: From<'a>) -> Self { + Self { + state: self.state.init_fwd(from), + } + } } impl<'a> Stream for Keys<'a> { diff --git a/src/database/stream/keys_rev.rs b/src/database/stream/keys_rev.rs index 59f66c2e..12dae759 100644 --- a/src/database/stream/keys_rev.rs +++ b/src/database/stream/keys_rev.rs @@ -1,4 +1,4 @@ -use std::{pin::Pin, sync::Arc}; +use std::{convert, pin::Pin, sync::Arc}; use conduit::Result; use futures::{ @@ -16,9 +16,17 @@ pub(crate) struct KeysRev<'a> { } impl<'a> KeysRev<'a> { - pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions, from: From<'_>) -> Self { + pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { Self { - state: State::new(db, cf, opts).init_rev(from), + state: State::new(db, cf, opts), + } + } +} + +impl<'a> convert::From> for KeysRev<'a> { + fn from(state: State<'a>) -> Self { + Self { + state, } } } @@ -31,6 +39,13 @@ impl<'a> Cursor<'a, Key<'a>> for KeysRev<'a> { #[inline] fn seek(&mut self) { self.state.seek_rev(); } + + #[inline] + fn init(self, from: From<'a>) -> Self { + Self { + state: self.state.init_rev(from), + } + } } impl<'a> Stream for KeysRev<'a> { From b7df0a14c6dd0b56348beab220637129c81942d2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 2 Dec 2024 11:56:02 +0000 Subject: [PATCH 1154/2091] parallelize events_before and events_after in api/client/context Signed-off-by: Jason Volk --- src/api/client/context.rs | 109 ++++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 52 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index af4e26f0..acd7d80b 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -2,7 +2,7 @@ use std::iter::once; use axum::extract::State; use conduit::{ - at, err, + at, err, ref_at, utils::{ future::TryExtExt, stream::{BroadbandExt, ReadyExt, WidebandExt}, @@ -10,7 +10,7 @@ use conduit::{ }, Err, Result, }; -use futures::{future::try_join, StreamExt, TryFutureExt}; +use futures::{join, try_join, FutureExt, StreamExt, TryFutureExt}; use ruma::{ api::client::{context::get_context, filter::LazyLoadOptions}, events::StateEventType, @@ -37,6 +37,7 @@ pub(crate) async fn get_context_route( let filter = &body.filter; let sender = body.sender(); let (sender_user, _) = sender; + let room_id = &body.room_id; // Use limit or else 10, with maximum 100 let limit: usize = body @@ -70,42 +71,63 @@ pub(crate) async fn get_context_route( .get_pdu(&body.event_id) .map_err(|_| err!(Request(NotFound("Base event not found.")))); - let (base_token, base_event) = try_join(base_token, base_event).await?; - - let room_id = &base_event.room_id; - - if !services + let visible = services .rooms .state_accessor - .user_can_see_event(sender_user, room_id, &body.event_id) - .await - { + .user_can_see_event(sender_user, &body.room_id, &body.event_id) + .map(Ok); + + let (base_token, base_event, visible) = try_join!(base_token, base_event, visible)?; + + if base_event.room_id != body.room_id { + return Err!(Request(NotFound("Base event not found."))); + } + + if !visible { return Err!(Request(Forbidden("You don't have permission to view this event."))); } - let events_before: Vec<_> = services + let events_before = services .rooms .timeline - .pdus_rev(Some(sender_user), room_id, Some(base_token)) - .await? - .ready_filter_map(|item| event_filter(item, filter)) - .wide_filter_map(|item| ignored_filter(&services, item, sender_user)) - .wide_filter_map(|item| visibility_filter(&services, item, sender_user)) - .take(limit / 2) - .collect() - .await; + .pdus_rev(Some(sender_user), room_id, Some(base_token)); - let events_after: Vec<_> = services + let events_after = services .rooms .timeline - .pdus(Some(sender_user), room_id, Some(base_token)) - .await? + .pdus(Some(sender_user), room_id, Some(base_token)); + + let (events_before, events_after) = try_join!(events_before, events_after)?; + + let events_before = events_before .ready_filter_map(|item| event_filter(item, filter)) .wide_filter_map(|item| ignored_filter(&services, item, sender_user)) .wide_filter_map(|item| visibility_filter(&services, item, sender_user)) .take(limit / 2) - .collect() - .await; + .collect(); + + let events_after = events_after + .ready_filter_map(|item| event_filter(item, filter)) + .wide_filter_map(|item| ignored_filter(&services, item, sender_user)) + .wide_filter_map(|item| visibility_filter(&services, item, sender_user)) + .take(limit / 2) + .collect(); + + let (events_before, events_after): (Vec<_>, Vec<_>) = join!(events_before, events_after); + + let state_at = events_after + .last() + .map(ref_at!(1)) + .map_or(body.event_id.as_ref(), |e| e.event_id.as_ref()); + + let state_ids = services + .rooms + .state_accessor + .pdu_shortstatehash(state_at) + .or_else(|_| services.rooms.state.get_room_shortstatehash(room_id)) + .and_then(|shortstatehash| services.rooms.state_accessor.state_full_ids(shortstatehash)) + .map_err(|e| err!(Database("State not found: {e}"))) + .await?; let lazy = once(&(base_token, base_event.clone())) .chain(events_before.iter()) @@ -116,48 +138,31 @@ pub(crate) async fn get_context_route( }) .await; - let state_id = events_after - .last() - .map_or(body.event_id.as_ref(), |(_, e)| e.event_id.as_ref()); - - let shortstatehash = services - .rooms - .state_accessor - .pdu_shortstatehash(state_id) - .or_else(|_| services.rooms.state.get_room_shortstatehash(room_id)) - .await - .map_err(|e| err!(Database("State hash not found: {e}")))?; - - let state_ids = services - .rooms - .state_accessor - .state_full_ids(shortstatehash) - .await - .map_err(|e| err!(Database("State not found: {e}")))?; - let lazy = &lazy; let state: Vec<_> = state_ids - .into_iter() + .iter() .stream() .broad_filter_map(|(shortstatekey, event_id)| { services .rooms .short - .get_statekey_from_short(shortstatekey) + .get_statekey_from_short(*shortstatekey) .map_ok(move |(event_type, state_key)| (event_type, state_key, event_id)) .ok() }) .ready_filter_map(|(event_type, state_key, event_id)| { - if lazy_load_enabled && event_type == StateEventType::RoomMember { - let user_id: &UserId = state_key.as_str().try_into().ok()?; - if !lazy.contains(user_id) { - return None; - } + if !lazy_load_enabled || event_type != StateEventType::RoomMember { + return Some(event_id); } - Some(event_id) + state_key + .as_str() + .try_into() + .ok() + .filter(|&user_id: &&UserId| lazy.contains(user_id)) + .map(|_| event_id) }) - .broad_filter_map(|event_id: OwnedEventId| async move { services.rooms.timeline.get_pdu(&event_id).await.ok() }) + .broad_filter_map(|event_id: &OwnedEventId| services.rooms.timeline.get_pdu(event_id).ok()) .map(|pdu| pdu.to_state_event()) .collect() .await; From ef9b1c63036868f336894b86f00b29cad0f48673 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 2 Dec 2024 13:50:09 +0000 Subject: [PATCH 1155/2091] simplify sender shutdown; prevent launching any retries Signed-off-by: Jason Volk --- src/service/sending/sender.rs | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index b1e909c9..5c00915b 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -7,7 +7,7 @@ use std::{ use base64::{engine::general_purpose, Engine as _}; use conduit::{ - debug, debug_warn, err, error, + debug, err, error, result::LogErr, trace, utils::{calculate_hash, math::continue_exponential_backoff_secs, ReadyExt}, @@ -35,7 +35,6 @@ use ruma::{ RoomVersionId, ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use tokio::time::sleep_until; use super::{appservice, data::QueueItem, Destination, Msg, SendingEvent, Service}; @@ -81,7 +80,7 @@ impl Service { }, } } - self.finish_responses(&mut futures, &mut statuses).await; + self.finish_responses(&mut futures).await; Ok(()) } @@ -146,22 +145,26 @@ impl Service { } } - async fn finish_responses<'a>(&'a self, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus) { + async fn finish_responses<'a>(&'a self, futures: &mut SendingFutures<'a>) { + use tokio::{ + select, + time::{sleep_until, Instant}, + }; + let now = Instant::now(); let timeout = Duration::from_millis(CLEANUP_TIMEOUT_MS); let deadline = now.checked_add(timeout).unwrap_or(now); loop { trace!("Waiting for {} requests to complete...", futures.len()); - tokio::select! { - () = sleep_until(deadline.into()) => break, + select! { + () = sleep_until(deadline) => return, response = futures.next() => match response { - Some(response) => self.handle_response(response, futures, statuses).await, + Some(Ok(dest)) => self.db.delete_all_active_requests_for(&dest).await, + Some(_) => continue, None => return, - } + }, } } - - debug_warn!("Leaving with {} unfinished requests...", futures.len()); } #[allow(clippy::needless_pass_by_ref_mut)] From 3109c0daba1115a4c58f46c9192d6ade8f078885 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 3 Dec 2024 06:34:56 +0000 Subject: [PATCH 1156/2091] perform async shutdown for database pool after services stop Signed-off-by: Jason Volk --- src/database/engine.rs | 5 +++-- src/database/pool.rs | 21 ++++++++++----------- src/router/run.rs | 12 +++++++++++- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index a6ed7d86..fe6602ae 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -148,6 +148,8 @@ impl Engine { .expect("column was created and exists") } + pub async fn shutdown_pool(&self) { self.pool.shutdown().await; } + pub fn flush(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, false)) } pub fn sync(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, true)) } @@ -325,8 +327,7 @@ impl Drop for Engine { fn drop(&mut self) { const BLOCKING: bool = true; - debug!("Shutting down request pool..."); - self.pool.close(); + debug_assert!(!self.pool.close(), "request pool was not closed"); debug!("Waiting for background tasks to finish..."); self.db.cancel_all_background_work(BLOCKING); diff --git a/src/database/pool.rs b/src/database/pool.rs index 136de67d..e7ffc807 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -76,10 +76,8 @@ pub(crate) async fn new(server: &Arc, opts: &Opts) -> Result> } #[implement(Pool)] -pub(crate) async fn _shutdown(self: &Arc) { - if !self.queue.is_closed() { - self.close(); - } +pub(crate) async fn shutdown(self: &Arc) { + self.close(); let workers = take(&mut *self.workers.lock().await); debug!(workers = workers.len(), "Waiting for workers to join..."); @@ -89,18 +87,19 @@ pub(crate) async fn _shutdown(self: &Arc) { } #[implement(Pool)] -pub(crate) fn close(&self) { - debug_assert!(!self.queue.is_closed(), "channel already closed"); +pub(crate) fn close(&self) -> bool { + if !self.queue.close() { + return false; + } + + std::thread::yield_now(); debug!( senders = self.queue.sender_count(), receivers = self.queue.receiver_count(), - "Closing pool channel" + "Closed pool channel" ); - let closing = self.queue.close(); - debug_assert!(closing, "channel is not closing"); - - std::thread::yield_now(); + true } #[implement(Pool)] diff --git a/src/router/run.rs b/src/router/run.rs index 395aa8c4..93b1339b 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -78,6 +78,11 @@ pub(crate) async fn stop(services: Arc) -> Result<()> { // unload and explode. services.stop().await; + // Check that Services and Database will drop as expected, The complex of Arc's + // used for various components can easily lead to references being held + // somewhere improperly; this can hang shutdowns. + debug!("Cleaning up..."); + let db = Arc::downgrade(&services.db); if let Err(services) = Arc::try_unwrap(services) { debug_error!( "{} dangling references to Services after shutdown", @@ -85,7 +90,12 @@ pub(crate) async fn stop(services: Arc) -> Result<()> { ); } - debug!("Cleaning up..."); + // The db threadpool requires async join if we use tokio/spawn_blocking to + // manage the threads. Without async-drop we have to wait here; for symmetry + // with Services construction it can't be done in services.stop(). + if let Some(db) = db.upgrade() { + db.db.shutdown_pool().await; + } #[cfg(feature = "systemd")] sd_notify::notify(true, &[sd_notify::NotifyState::Stopping]).expect("failed to notify systemd of stopping state"); From 9d9f403ad5f5656f8b1084a71fcba47a5ef03e5b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 3 Dec 2024 07:51:29 +0000 Subject: [PATCH 1157/2091] prevent adding presence timer for server's own user Signed-off-by: Jason Volk --- src/service/presence/mod.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index b2106f3f..8e1521c5 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -137,7 +137,9 @@ impl Service { .set_presence(user_id, presence_state, currently_active, last_active_ago, status_msg) .await?; - if self.timeout_remote_users || self.services.globals.user_is_local(user_id) { + if (self.timeout_remote_users || self.services.globals.user_is_local(user_id)) + && user_id != self.services.globals.server_user + { let timeout = match presence_state { PresenceState::Online => self.services.server.config.presence_idle_timeout_s, _ => self.services.server.config.presence_offline_timeout_s, From c01b049910c15e751b8782d33048fc8210983a8e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 3 Dec 2024 07:35:48 +0000 Subject: [PATCH 1158/2091] move cidr_range_denylist from globals to client service Signed-off-by: Jason Volk --- src/service/client/mod.rs | 22 +++++++++++++++++++++- src/service/globals/mod.rs | 24 +----------------------- src/service/media/preview.rs | 4 ++-- src/service/pusher/mod.rs | 4 ++-- src/service/resolver/actual.rs | 2 +- src/service/resolver/mod.rs | 4 +--- 6 files changed, 28 insertions(+), 32 deletions(-) diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index f9a89e99..2794efc1 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -1,6 +1,7 @@ use std::{sync::Arc, time::Duration}; -use conduit::{Config, Result}; +use conduit::{err, implement, trace, Config, Result}; +use ipaddress::IPAddress; use reqwest::redirect; use crate::{resolver, service}; @@ -15,6 +16,8 @@ pub struct Service { pub sender: reqwest::Client, pub appservice: reqwest::Client, pub pusher: reqwest::Client, + + pub cidr_range_denylist: Vec, } impl crate::Service for Service { @@ -86,6 +89,14 @@ impl crate::Service for Service { .pool_idle_timeout(Duration::from_secs(config.pusher_idle_timeout)) .redirect(redirect::Policy::limited(2)) .build()?, + + cidr_range_denylist: config + .ip_range_denylist + .iter() + .map(IPAddress::parse) + .inspect(|cidr| trace!("Denied CIDR range: {cidr:?}")) + .collect::>() + .map_err(|e| err!(Config("ip_range_denylist", e)))?, })) } @@ -152,3 +163,12 @@ fn base(config: &Config) -> Result { Ok(builder) } } + +#[inline] +#[must_use] +#[implement(Service)] +pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool { + self.cidr_range_denylist + .iter() + .all(|cidr| !cidr.includes(ip)) +} diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 3eefe4b7..4fb1ce2d 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,9 +7,8 @@ use std::{ time::Instant, }; -use conduit::{err, error, trace, Config, Result}; +use conduit::{error, Config, Result}; use data::Data; -use ipaddress::IPAddress; use regex::RegexSet; use ruma::{ OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, @@ -22,7 +21,6 @@ pub struct Service { pub db: Data, pub config: Config, - pub cidr_range_denylist: Vec, jwt_decoding_key: Option, pub stable_room_versions: Vec, pub unstable_room_versions: Vec, @@ -59,14 +57,6 @@ impl crate::Service for Service { // Experimental, partially supported room versions let unstable_room_versions = vec![RoomVersionId::V2, RoomVersionId::V3, RoomVersionId::V4, RoomVersionId::V5]; - let cidr_range_denylist: Vec<_> = config - .ip_range_denylist - .iter() - .map(IPAddress::parse) - .inspect(|cidr| trace!("Denied CIDR range: {cidr:?}")) - .collect::>() - .map_err(|e| err!(Config("ip_range_denylist", e)))?; - let turn_secret = config .turn_secret_file .as_ref() @@ -95,7 +85,6 @@ impl crate::Service for Service { let mut s = Self { db, config: config.clone(), - cidr_range_denylist, jwt_decoding_key, stable_room_versions, unstable_room_versions, @@ -255,17 +244,6 @@ impl Service { } } - #[inline] - pub fn valid_cidr_range(&self, ip: &IPAddress) -> bool { - for cidr in &self.cidr_range_denylist { - if cidr.includes(ip) { - return false; - } - } - - true - } - /// checks if `user_id` is local to us via server_name comparison #[inline] pub fn user_is_local(&self, user_id: &UserId) -> bool { self.server_is_ours(user_id.server_name()) } diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index acc9d8ed..eb9be560 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -87,7 +87,7 @@ pub async fn get_url_preview(&self, url: &Url) -> Result { #[implement(Service)] async fn request_url_preview(&self, url: &Url) -> Result { if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) { - if !self.services.globals.valid_cidr_range(&ip) { + if !self.services.client.valid_cidr_range(&ip) { return Err!(BadServerResponse("Requesting from this address is forbidden")); } } @@ -97,7 +97,7 @@ async fn request_url_preview(&self, url: &Url) -> Result { if let Some(remote_addr) = response.remote_addr() { if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { - if !self.services.globals.valid_cidr_range(&ip) { + if !self.services.client.valid_cidr_range(&ip) { return Err!(BadServerResponse("Requesting from this address is forbidden")); } } diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 6b02c7f8..df5852c6 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -151,7 +151,7 @@ impl Service { if let Some(url_host) = reqwest_request.url().host_str() { trace!("Checking request URL for IP"); if let Ok(ip) = IPAddress::parse(url_host) { - if !self.services.globals.valid_cidr_range(&ip) { + if !self.services.client.valid_cidr_range(&ip) { return Err!(BadServerResponse("Not allowed to send requests to this IP")); } } @@ -166,7 +166,7 @@ impl Service { trace!("Checking response destination's IP"); if let Some(remote_addr) = response.remote_addr() { if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { - if !self.services.globals.valid_cidr_range(&ip) { + if !self.services.client.valid_cidr_range(&ip) { return Err!(BadServerResponse("Not allowed to send requests to this IP")); } } diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 6589a58b..8553e8bb 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -358,7 +358,7 @@ impl super::Service { } pub(crate) fn validate_ip(&self, ip: &IPAddress) -> Result<()> { - if !self.services.globals.valid_cidr_range(ip) { + if !self.services.client.valid_cidr_range(ip) { return Err!(BadServerResponse("Not allowed to send requests to this IP")); } diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 28b7063d..111de292 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -9,7 +9,7 @@ use std::{fmt::Write, sync::Arc}; use conduit::{Result, Server}; use self::{cache::Cache, dns::Resolver}; -use crate::{client, globals, Dep}; +use crate::{client, Dep}; pub struct Service { pub cache: Arc, @@ -20,7 +20,6 @@ pub struct Service { struct Services { server: Arc, client: Dep, - globals: Dep, } impl crate::Service for Service { @@ -33,7 +32,6 @@ impl crate::Service for Service { services: Services { server: args.server.clone(), client: args.depend::("client"), - globals: args.depend::("globals"), }, })) } From 48703173bc2db590bfd9aeaf79b1248bc2a8327f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 3 Dec 2024 10:42:52 +0000 Subject: [PATCH 1159/2091] split get_batch from get.rs; add aqry_batch Signed-off-by: Jason Volk --- src/database/map.rs | 1 + src/database/map/get.rs | 59 +++++------------------------------ src/database/map/get_batch.rs | 57 +++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 52 deletions(-) create mode 100644 src/database/map/get_batch.rs diff --git a/src/database/map.rs b/src/database/map.rs index 4b55fa54..a15d5e9d 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -1,6 +1,7 @@ mod contains; mod count; mod get; +mod get_batch; mod insert; mod keys; mod keys_from; diff --git a/src/database/map/get.rs b/src/database/map/get.rs index ef552177..04f5d0ae 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -1,13 +1,8 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; use arrayvec::ArrayVec; -use conduit::{ - err, implement, - utils::{result::MapExpect, IterStream}, - Err, Result, -}; -use futures::{future, Future, FutureExt, Stream, StreamExt}; -use rocksdb::DBPinnableSlice; +use conduit::{err, implement, utils::result::MapExpect, Err, Result}; +use futures::{future, Future, FutureExt}; use serde::Serialize; use crate::{ @@ -17,8 +12,6 @@ use crate::{ Handle, }; -type RocksdbResult<'a> = Result>, rocksdb::Error>; - /// Fetch a value from the database into cache, returning a reference-handle /// asynchronously. The key is serialized into an allocated buffer to perform /// the query. @@ -58,18 +51,6 @@ where self.get(key) } -#[implement(super::Map)] -#[tracing::instrument(skip(self, keys), fields(%self), level = "trace")] -pub fn get_batch<'a, I, K>(self: &'a Arc, keys: I) -> impl Stream>> + Send + 'a -where - I: Iterator + ExactSizeIterator + Debug + Send + 'a, - K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, -{ - keys.stream() - .map(move |key| self.get(key)) - .buffered(self.db.server.config.db_pool_workers.saturating_mul(2)) -} - /// Fetch a value from the database into cache, returning a reference-handle /// asynchronously. The key is referenced directly to perform the query. #[implement(super::Map)] @@ -95,25 +76,6 @@ where self.db.pool.execute_get(cmd).boxed() } -#[implement(super::Map)] -#[tracing::instrument(skip(self, keys), name = "batch_blocking", level = "trace")] -pub(crate) fn get_batch_blocking<'a, I, K>(&self, keys: I) -> impl Iterator>> + Send -where - I: Iterator + ExactSizeIterator + Debug + Send, - K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, -{ - // Optimization can be `true` if key vector is pre-sorted **by the column - // comparator**. - const SORTED: bool = false; - - let read_options = &self.read_options; - self.db - .db - .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) - .into_iter() - .map(into_result_handle) -} - /// Fetch a value from the database into cache, returning a reference-handle. /// The key is referenced directly to perform the query. This is a thread- /// blocking call. @@ -123,12 +85,12 @@ pub fn get_blocking(&self, key: &K) -> Result> where K: AsRef<[u8]> + ?Sized, { - let res = self + self.db .db - .db - .get_pinned_cf_opt(&self.cf(), key, &self.read_options); - - into_result_handle(res) + .get_pinned_cf_opt(&self.cf(), key, &self.read_options) + .map_err(map_err)? + .map(Handle::from) + .ok_or(err!(Request(NotFound("Not found in database")))) } /// Fetch a value from the cache without I/O. @@ -157,10 +119,3 @@ where Err(e) => or_else(e), } } - -fn into_result_handle(result: RocksdbResult<'_>) -> Result> { - result - .map_err(map_err)? - .map(Handle::from) - .ok_or(err!(Request(NotFound("Not found in database")))) -} diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs new file mode 100644 index 00000000..0f1fdea7 --- /dev/null +++ b/src/database/map/get_batch.rs @@ -0,0 +1,57 @@ +use std::{convert::AsRef, fmt::Debug, sync::Arc}; + +use conduit::{err, implement, utils::IterStream, Result}; +use futures::{Stream, StreamExt}; +use serde::Serialize; + +use crate::{util::map_err, Handle}; + +#[implement(super::Map)] +#[tracing::instrument(skip(self, keys), level = "trace")] +pub fn aqry_batch<'b, 'a: 'b, const MAX: usize, I, K>( + self: &'a Arc, keys: I, +) -> impl Stream>> + Send + 'a +where + I: Iterator + Send + 'a, + K: Serialize + ?Sized + Debug + 'b, +{ + keys.stream() + .map(move |key| self.aqry::(&key)) + .buffered(self.db.server.config.db_pool_workers.saturating_mul(2)) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self, keys), level = "trace")] +pub fn get_batch<'a, I, K>(self: &'a Arc, keys: I) -> impl Stream>> + Send + 'a +where + I: Iterator + Debug + Send + 'a, + K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, +{ + keys.stream() + .map(move |key| self.get(key)) + .buffered(self.db.server.config.db_pool_workers.saturating_mul(2)) +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self, keys), name = "batch_blocking", level = "trace")] +pub(crate) fn get_batch_blocking<'a, I, K>(&self, keys: I) -> impl Iterator>> + Send +where + I: Iterator + ExactSizeIterator + Debug + Send, + K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, +{ + // Optimization can be `true` if key vector is pre-sorted **by the column + // comparator**. + const SORTED: bool = false; + + let read_options = &self.read_options; + self.db + .db + .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) + .into_iter() + .map(|result| { + result + .map_err(map_err)? + .map(Handle::from) + .ok_or(err!(Request(NotFound("Not found in database")))) + }) +} From 784ccd6bad25f845532bc3a6e82c5031c50c6444 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 3 Dec 2024 09:42:26 +0000 Subject: [PATCH 1160/2091] return stream from multi_get_eventid_from_short Signed-off-by: Jason Volk --- src/service/rooms/auth_chain/mod.rs | 15 +++++---- src/service/rooms/short/mod.rs | 14 +++----- src/service/rooms/state/mod.rs | 13 +++---- src/service/rooms/state_accessor/data.rs | 43 +++++++++--------------- src/service/rooms/state_accessor/mod.rs | 10 +++--- 5 files changed, 39 insertions(+), 56 deletions(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 5face0b5..e7e5edf4 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -6,7 +6,11 @@ use std::{ sync::Arc, }; -use conduit::{debug, debug_error, trace, utils::IterStream, validated, warn, Err, Result}; +use conduit::{ + debug, debug_error, trace, + utils::{stream::ReadyExt, IterStream}, + validated, warn, Err, Result, +}; use futures::{Stream, StreamExt}; use ruma::{EventId, RoomId}; @@ -61,11 +65,10 @@ impl Service { let event_ids = self .services .short - .multi_get_eventid_from_short(chain.into_iter()) - .await - .into_iter() - .filter_map(Result::ok) - .collect(); + .multi_get_eventid_from_short(chain.iter()) + .ready_filter_map(Result::ok) + .collect() + .await; Ok(event_ids) } diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 0f100348..a7c32856 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -139,7 +139,7 @@ pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &s #[implement(Service)] pub async fn get_eventid_from_short(&self, shorteventid: ShortEventId) -> Result where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + Id: for<'de> Deserialize<'de> + Sized + ToOwned, ::Owned: Borrow, { const BUFSIZE: usize = size_of::(); @@ -153,22 +153,18 @@ where } #[implement(Service)] -pub async fn multi_get_eventid_from_short(&self, shorteventid: I) -> Vec> +pub fn multi_get_eventid_from_short<'a, Id, I>(&'a self, shorteventid: I) -> impl Stream> + Send + 'a where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + I: Iterator + Send + 'a, + Id: for<'de> Deserialize<'de> + Sized + ToOwned + 'a, ::Owned: Borrow, - I: Iterator + Send, { const BUFSIZE: usize = size_of::(); - let keys: Vec<[u8; BUFSIZE]> = shorteventid.map(u64::to_be_bytes).collect(); - self.db .shorteventid_eventid - .get_batch(keys.iter()) + .aqry_batch::(shorteventid) .map(Deserialized::deserialized) - .collect() - .await } #[implement(Service)] diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 838deacd..d0d21fa8 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use conduit::{ - at, err, + err, result::FlatOk, utils::{ calculate_hash, @@ -420,7 +420,7 @@ impl Service { .collect() .await; - let auth_state: Vec<_> = self + let (state_keys, event_ids): (Vec<_>, Vec<_>) = self .services .state_accessor .state_full_shortids(shortstatehash) @@ -432,16 +432,13 @@ impl Service { .remove(&shortstatekey) .map(|(event_type, state_key)| ((event_type, state_key), shorteventid)) }) - .collect(); + .unzip(); let auth_pdus = self .services .short - .multi_get_eventid_from_short(auth_state.iter().map(at!(1))) - .await - .into_iter() - .stream() - .zip(auth_state.into_iter().stream().map(at!(0))) + .multi_get_eventid_from_short(event_ids.iter()) + .zip(state_keys.into_iter().stream()) .ready_filter_map(|(event_id, tsk)| Some((tsk, event_id.ok()?))) .broad_filter_map(|(tsk, event_id): (_, OwnedEventId)| async move { self.services diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 2a670066..bca54069 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,12 +1,12 @@ use std::{borrow::Borrow, collections::HashMap, sync::Arc}; use conduit::{ - at, err, - utils::stream::{BroadbandExt, IterStream}, + at, err, ref_at, + utils::stream::{BroadbandExt, IterStream, ReadyExt}, PduEvent, Result, }; use database::{Deserialized, Map}; -use futures::{StreamExt, TryFutureExt}; +use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{events::StateEventType, EventId, OwnedEventId, RoomId}; use serde::Deserialize; @@ -59,23 +59,13 @@ impl Data { } pub(super) async fn state_full_pdus(&self, shortstatehash: ShortStateHash) -> Result> { - let short_ids = self - .state_full_shortids(shortstatehash) - .await? - .into_iter() - .map(at!(1)); + let short_ids = self.state_full_shortids(shortstatehash).await?; - let event_ids = self + let full_pdus = self .services .short - .multi_get_eventid_from_short(short_ids) - .await - .into_iter() - .filter_map(Result::ok); - - let full_pdus = event_ids - .into_iter() - .stream() + .multi_get_eventid_from_short(short_ids.iter().map(ref_at!(1))) + .ready_filter_map(Result::ok) .broad_filter_map( |event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await.ok() }, ) @@ -92,19 +82,16 @@ impl Data { { let short_ids = self.state_full_shortids(shortstatehash).await?; - let event_ids = self + let full_ids = self .services .short - .multi_get_eventid_from_short(short_ids.iter().map(at!(1))) + .multi_get_eventid_from_short(short_ids.iter().map(ref_at!(1))) + .zip(short_ids.iter().stream().map(at!(0))) + .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) + .collect() + .boxed() .await; - let full_ids = short_ids - .into_iter() - .map(at!(0)) - .zip(event_ids.into_iter()) - .filter_map(|(shortstatekey, event_id)| Some((shortstatekey, event_id.ok()?))) - .collect(); - Ok(full_ids) } @@ -134,7 +121,7 @@ impl Data { &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + Id: for<'de> Deserialize<'de> + Sized + ToOwned, ::Owned: Borrow, { let shortstatekey = self @@ -219,7 +206,7 @@ impl Data { &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + Id: for<'de> Deserialize<'de> + Sized + ToOwned, ::Owned: Borrow, { self.services diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index e42d3764..ef1b63f5 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -102,7 +102,7 @@ impl Service { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result> + pub async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result> where Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, ::Owned: Borrow, @@ -130,7 +130,7 @@ impl Service { &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + Id: for<'de> Deserialize<'de> + Sized + ToOwned, ::Owned: Borrow, { self.db @@ -154,7 +154,7 @@ impl Service { &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result where - T: for<'de> Deserialize<'de> + Send, + T: for<'de> Deserialize<'de>, { self.state_get(shortstatehash, event_type, state_key) .await @@ -337,7 +337,7 @@ impl Service { &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + Id: for<'de> Deserialize<'de> + Sized + ToOwned, ::Owned: Borrow, { self.db @@ -359,7 +359,7 @@ impl Service { &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, ) -> Result where - T: for<'de> Deserialize<'de> + Send, + T: for<'de> Deserialize<'de>, { self.room_state_get(room_id, event_type, state_key) .await From e0494c1538526771695e63aa9dc410207e748704 Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 30 Nov 2024 14:18:46 -0500 Subject: [PATCH 1161/2091] add /bin/conduit to OCI image contents Signed-off-by: strawberry --- nix/pkgs/oci-image/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/pkgs/oci-image/default.nix b/nix/pkgs/oci-image/default.nix index 9b641310..152e00d1 100644 --- a/nix/pkgs/oci-image/default.nix +++ b/nix/pkgs/oci-image/default.nix @@ -14,6 +14,7 @@ dockerTools.buildLayeredImage { created = "@${toString inputs.self.lastModified}"; contents = [ dockerTools.caCertificates + main ]; config = { Entrypoint = if !stdenv.hostPlatform.isDarwin From 9db0325b42bb5afe4f32c5c9f47a2b7342c844a5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 30 Nov 2024 14:43:14 -0500 Subject: [PATCH 1162/2091] bump rust to 1.83.0 Signed-off-by: strawberry --- Cargo.toml | 2 +- flake.nix | 2 +- rust-toolchain.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0e8596f7..efb06a5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" -rust-version = "1.82.0" +rust-version = "1.83.0" version = "0.5.0" [workspace.metadata.crane] diff --git a/flake.nix b/flake.nix index 113757a7..281052a8 100644 --- a/flake.nix +++ b/flake.nix @@ -27,7 +27,7 @@ file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` - sha256 = "sha256-yMuSb5eQPO/bHv+Bcf/US8LVMbf/G/0MSfiPwBhiPpk="; + sha256 = "sha256-s1RPtyvDGJaX/BisLT+ifVfuhDT1nZkZ1NcK8sbwELM="; }; mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 086f7ba0..ddd952a2 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -9,7 +9,7 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.82.0" +channel = "1.83.0" profile = "minimal" components = [ # For rust-analyzer From 513236b3cec834e3f38939a1ce04574cf16bcbe1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 4 Dec 2024 00:00:00 +0000 Subject: [PATCH 1163/2091] bump ruma for async state-res optimizations Signed-off-by: Jason Volk --- Cargo.lock | 30 +++++++++---------- Cargo.toml | 2 +- .../rooms/event_handler/resolve_state.rs | 2 +- .../rooms/event_handler/state_at_incoming.rs | 1 + 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 49c4127e..d787739e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -3171,7 +3171,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "assign", "js_int", @@ -3193,7 +3193,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "js_int", "ruma-common", @@ -3205,7 +3205,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "as_variant", "assign", @@ -3228,7 +3228,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "as_variant", "base64 0.22.1", @@ -3258,7 +3258,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "as_variant", "indexmap 2.6.0", @@ -3282,7 +3282,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "bytes", "http", @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "js_int", "thiserror 2.0.3", @@ -3309,7 +3309,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "js_int", "ruma-common", @@ -3319,10 +3319,9 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "cfg-if", - "once_cell", "proc-macro-crate", "proc-macro2", "quote", @@ -3335,7 +3334,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "js_int", "ruma-common", @@ -3347,7 +3346,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "headers", "http", @@ -3360,7 +3359,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3376,10 +3375,9 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=97e2fb6df13f65532d33fc2f0f097ad5a449dd70#97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" dependencies = [ "futures-util", - "itertools 0.13.0", "js_int", "ruma-common", "ruma-events", diff --git a/Cargo.toml b/Cargo.toml index efb06a5b..673cab09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -334,7 +334,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "97e2fb6df13f65532d33fc2f0f097ad5a449dd70" +rev = "1a550585bf025cce48ef8b734339245092bc986e" features = [ "compat", "rand", diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index dc0edd13..81cf7733 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -65,7 +65,6 @@ pub async fn resolve_state( .collect() }) .collect() - .boxed() .await; debug!("Resolving state"); @@ -74,6 +73,7 @@ pub async fn resolve_state( let event_fetch = |event_id| self.event_fetch(event_id); let event_exists = |event_id| self.event_exists(event_id); let state = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) + .boxed() .await .map_err(|e| err!(Database(error!("State resolution failed: {e:?}"))))?; diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 05a9d8ca..96ee9907 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -153,6 +153,7 @@ pub(super) async fn state_at_incoming_resolved( let event_fetch = |event_id| self.event_fetch(event_id); let event_exists = |event_id| self.event_exists(event_id); let result = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) + .boxed() .await .map_err(|e| err!(Database(warn!(?e, "State resolution on prev events failed.")))); From c2d97aaa5ea4d304cb76d9d0ec31f3b9e1529043 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 4 Dec 2024 01:23:36 +0000 Subject: [PATCH 1164/2091] increase default db pool worker count for large systems Signed-off-by: Jason Volk --- conduwuit-example.toml | 3 ++- src/core/config/mod.rs | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index a0f05ebb..aee29f92 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1341,7 +1341,8 @@ # Sets the number of worker threads in the frontend-pool of the database. # This number should reflect the I/O capabilities of the system, # specifically the queue-depth or the number of simultaneous requests in -# flight. Defaults to 32 or number of CPU cores, whichever is greater. +# flight. Defaults to 32 or four times the number of CPU cores, whichever +# is greater. # #db_pool_workers = 32 diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index edbb7c26..cb8f3b87 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1503,7 +1503,8 @@ pub struct Config { /// Sets the number of worker threads in the frontend-pool of the database. /// This number should reflect the I/O capabilities of the system, /// specifically the queue-depth or the number of simultaneous requests in - /// flight. Defaults to 32 or number of CPU cores, whichever is greater. + /// flight. Defaults to 32 or four times the number of CPU cores, whichever + /// is greater. /// default: 32 #[serde(default = "default_db_pool_workers")] pub db_pool_workers: usize, @@ -2280,6 +2281,6 @@ fn parallelism_scaled(val: usize) -> usize { val.saturating_mul(sys::available_p fn default_trusted_server_batch_size() -> usize { 256 } -fn default_db_pool_workers() -> usize { sys::available_parallelism().max(32) } +fn default_db_pool_workers() -> usize { sys::available_parallelism().saturating_mul(4).max(32) } fn default_db_pool_queue_size() -> usize { sys::available_parallelism().saturating_mul(8).max(256) } From 59d5e3ebf1790e4dece9089bb3bce90cb976ffc3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 4 Dec 2024 02:11:43 +0000 Subject: [PATCH 1165/2091] additional stream extensions for any/all additional stream extension TryBroadbandExt Signed-off-by: Jason Volk --- src/core/utils/stream/broadband.rs | 55 +++++++++++++++++++++++++- src/core/utils/stream/iter_stream.rs | 11 +++++- src/core/utils/stream/mod.rs | 2 + src/core/utils/stream/ready.rs | 14 ++++++- src/core/utils/stream/try_broadband.rs | 43 ++++++++++++++++++++ src/core/utils/stream/wideband.rs | 1 - 6 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 src/core/utils/stream/try_broadband.rs diff --git a/src/core/utils/stream/broadband.rs b/src/core/utils/stream/broadband.rs index ce17830c..37416d63 100644 --- a/src/core/utils/stream/broadband.rs +++ b/src/core/utils/stream/broadband.rs @@ -1,5 +1,4 @@ //! Broadband stream combinator extensions to futures::Stream -#![allow(clippy::type_complexity)] use std::convert::identity; @@ -18,6 +17,18 @@ pub trait BroadbandExt where Self: Stream + Send + Sized, { + fn broadn_all(self, n: N, f: F) -> impl Future + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future + Send; + + fn broadn_any(self, n: N, f: F) -> impl Future + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future + Send; + /// Concurrent filter_map(); unordered results fn broadn_filter_map(self, n: N, f: F) -> impl Stream + Send where @@ -33,6 +44,24 @@ where Fut: Future + Send, U: Send; + #[inline] + fn broad_all(self, f: F) -> impl Future + Send + where + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + { + self.broadn_all(None, f) + } + + #[inline] + fn broad_any(self, f: F) -> impl Future + Send + where + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + { + self.broadn_any(None, f) + } + #[inline] fn broad_filter_map(self, f: F) -> impl Stream + Send where @@ -58,6 +87,30 @@ impl BroadbandExt for S where S: Stream + Send + Sized, { + #[inline] + fn broadn_all(self, n: N, f: F) -> impl Future + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + { + self.map(f) + .buffer_unordered(n.into().unwrap_or(WIDTH)) + .ready_all(identity) + } + + #[inline] + fn broadn_any(self, n: N, f: F) -> impl Future + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Future + Send, + { + self.map(f) + .buffer_unordered(n.into().unwrap_or(WIDTH)) + .ready_any(identity) + } + #[inline] fn broadn_filter_map(self, n: N, f: F) -> impl Stream + Send where diff --git a/src/core/utils/stream/iter_stream.rs b/src/core/utils/stream/iter_stream.rs index 69edf64f..2face4b0 100644 --- a/src/core/utils/stream/iter_stream.rs +++ b/src/core/utils/stream/iter_stream.rs @@ -4,12 +4,16 @@ use futures::{ StreamExt, }; +use crate::{Error, Result}; + pub trait IterStream { /// Convert an Iterator into a Stream fn stream(self) -> impl Stream::Item> + Send; /// Convert an Iterator into a TryStream - fn try_stream(self) -> impl TryStream::Item, Error = crate::Error> + Send; + fn try_stream( + self, + ) -> impl TryStream::Item, Error = Error, Item = Result<::Item, Error>> + Send; } impl IterStream for I @@ -21,7 +25,10 @@ where fn stream(self) -> impl Stream::Item> + Send { stream::iter(self) } #[inline] - fn try_stream(self) -> impl TryStream::Item, Error = crate::Error> + Send { + fn try_stream( + self, + ) -> impl TryStream::Item, Error = Error, Item = Result<::Item, Error>> + Send + { self.stream().map(Ok) } } diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index 45c2110d..c9138116 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -5,6 +5,7 @@ mod ignore; mod iter_stream; mod ready; mod tools; +mod try_broadband; mod try_ready; mod wideband; @@ -15,5 +16,6 @@ pub use ignore::TryIgnore; pub use iter_stream::IterStream; pub use ready::ReadyExt; pub use tools::Tools; +pub use try_broadband::TryBroadbandExt; pub use try_ready::TryReadyExt; pub use wideband::WidebandExt; diff --git a/src/core/utils/stream/ready.rs b/src/core/utils/stream/ready.rs index f4eec7d1..9bba589e 100644 --- a/src/core/utils/stream/ready.rs +++ b/src/core/utils/stream/ready.rs @@ -3,7 +3,7 @@ use futures::{ future::{ready, Ready}, - stream::{Any, Filter, FilterMap, Fold, ForEach, Scan, SkipWhile, Stream, StreamExt, TakeWhile}, + stream::{All, Any, Filter, FilterMap, Fold, ForEach, Scan, SkipWhile, Stream, StreamExt, TakeWhile}, }; /// Synchronous combinators to augment futures::StreamExt. Most Stream @@ -16,6 +16,10 @@ pub trait ReadyExt where Self: Stream + Send + Sized, { + fn ready_all(self, f: F) -> All, impl FnMut(Item) -> Ready> + where + F: Fn(Item) -> bool; + fn ready_any(self, f: F) -> Any, impl FnMut(Item) -> Ready> where F: Fn(Item) -> bool; @@ -66,6 +70,14 @@ impl ReadyExt for S where S: Stream + Send + Sized, { + #[inline] + fn ready_all(self, f: F) -> All, impl FnMut(Item) -> Ready> + where + F: Fn(Item) -> bool, + { + self.all(move |t| ready(f(t))) + } + #[inline] fn ready_any(self, f: F) -> Any, impl FnMut(Item) -> Ready> where diff --git a/src/core/utils/stream/try_broadband.rs b/src/core/utils/stream/try_broadband.rs new file mode 100644 index 00000000..59c488e0 --- /dev/null +++ b/src/core/utils/stream/try_broadband.rs @@ -0,0 +1,43 @@ +//! Synchronous combinator extensions to futures::TryStream + +use futures::{TryFuture, TryStream, TryStreamExt}; + +use crate::Result; + +const WIDTH: usize = 32; + +/// Concurrency extensions to augment futures::TryStreamExt. broad_ combinators +/// produce out-of-order +pub trait TryBroadbandExt +where + Self: TryStream> + Send + Sized, +{ + fn broadn_and_then(self, n: N, f: F) -> impl TryStream> + Send + where + N: Into>, + F: Fn(Self::Ok) -> Fut + Send + Sync, + Fut: TryFuture> + Send; + + fn broad_and_then(self, f: F) -> impl TryStream> + Send + where + F: Fn(Self::Ok) -> Fut + Send + Sync, + Fut: TryFuture> + Send, + { + self.broadn_and_then(None, f) + } +} + +impl TryBroadbandExt for S +where + S: TryStream> + Send + Sized, +{ + fn broadn_and_then(self, n: N, f: F) -> impl TryStream> + Send + where + N: Into>, + F: Fn(Self::Ok) -> Fut + Send + Sync, + Fut: TryFuture> + Send, + { + self.map_ok(f) + .try_buffer_unordered(n.into().unwrap_or(WIDTH)) + } +} diff --git a/src/core/utils/stream/wideband.rs b/src/core/utils/stream/wideband.rs index 100990b8..053a351f 100644 --- a/src/core/utils/stream/wideband.rs +++ b/src/core/utils/stream/wideband.rs @@ -1,5 +1,4 @@ //! Wideband stream combinator extensions to futures::Stream -#![allow(clippy::type_complexity)] use std::convert::identity; From 1d0285102825c903f90ceddfe3cb8763d8707dc4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 4 Dec 2024 00:00:40 +0000 Subject: [PATCH 1166/2091] implement several broadband loops Signed-off-by: Jason Volk --- src/api/client/relations.rs | 7 +++---- src/api/client/sync/mod.rs | 7 +++++-- src/api/server/send_join.rs | 20 +++++++++++++------ .../rooms/event_handler/resolve_state.rs | 12 +++++++---- .../rooms/event_handler/state_at_incoming.rs | 9 +++++++-- src/service/rooms/pdu_metadata/data.rs | 7 +++++-- src/service/rooms/threads/mod.rs | 7 +++++-- 7 files changed, 47 insertions(+), 22 deletions(-) diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index 902e6be6..de54c4e4 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,10 +1,10 @@ use axum::extract::State; use conduit::{ at, - utils::{result::FlatOk, IterStream, ReadyExt}, + utils::{result::FlatOk, stream::WidebandExt, IterStream, ReadyExt}, PduCount, Result, }; -use futures::{FutureExt, StreamExt}; +use futures::StreamExt; use ruma::{ api::{ client::relations::{ @@ -138,11 +138,10 @@ async fn paginate_relations_with_filter( .is_none_or(|rel_type| pdu.relation_type_equal(rel_type)) }) .stream() - .filter_map(|item| visibility_filter(services, sender_user, item)) .ready_take_while(|(count, _)| Some(*count) != to) + .wide_filter_map(|item| visibility_filter(services, sender_user, item)) .take(limit) .collect() - .boxed() .await; let next_batch = match dir { diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index ba50d77c..6f791860 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -1,7 +1,10 @@ mod v3; mod v4; -use conduit::{utils::ReadyExt, PduCount}; +use conduit::{ + utils::stream::{BroadbandExt, ReadyExt}, + PduCount, +}; use futures::StreamExt; use ruma::{RoomId, UserId}; @@ -55,7 +58,7 @@ async fn share_encrypted_room( .state_cache .get_shared_rooms(sender_user, user_id) .ready_filter(|&room_id| Some(room_id) != ignore_room) - .any(|other_room_id| { + .broad_any(|other_room_id| { services .rooms .state_accessor diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 92ab3b50..d1574e62 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -3,7 +3,12 @@ use std::{borrow::Borrow, collections::HashMap}; use axum::extract::State; -use conduit::{err, pdu::gen_event_id_canonical_json, utils::IterStream, warn, Error, Result}; +use conduit::{ + err, + pdu::gen_event_id_canonical_json, + utils::stream::{IterStream, TryBroadbandExt}, + warn, Error, Result, +}; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ api::{client::error::ErrorKind, federation::membership::create_join_event}, @@ -160,6 +165,7 @@ async fn create_join_event( .rooms .event_handler .handle_incoming_pdu(&origin, room_id, &event_id, value.clone(), true) + .boxed() .await? .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; @@ -172,16 +178,17 @@ async fn create_join_event( .await?; let state = state_ids - .iter() + .values() .try_stream() - .and_then(|(_, event_id)| services.rooms.timeline.get_pdu_json(event_id)) - .and_then(|pdu| { + .broad_and_then(|event_id| services.rooms.timeline.get_pdu_json(event_id)) + .broad_and_then(|pdu| { services .sending .convert_to_outgoing_federation_event(pdu) .map(Ok) }) .try_collect() + .boxed() .await?; let starting_events = state_ids.values().map(Borrow::borrow); @@ -191,14 +198,15 @@ async fn create_join_event( .event_ids_iter(room_id, starting_events) .await? .map(Ok) - .and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) - .and_then(|pdu| { + .broad_and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) + .broad_and_then(|pdu| { services .sending .convert_to_outgoing_federation_event(pdu) .map(Ok) }) .try_collect() + .boxed() .await?; services.sending.send_pdu_room(room_id, &pdu_id).await?; diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 81cf7733..3329a146 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -4,7 +4,11 @@ use std::{ sync::Arc, }; -use conduit::{debug, err, implement, utils::IterStream, Result}; +use conduit::{ + debug, err, implement, + utils::stream::{IterStream, WidebandExt}, + Result, +}; use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{ state_res::{self, StateMap}, @@ -52,11 +56,11 @@ pub async fn resolve_state( let fork_states: Vec>> = fork_states .into_iter() .stream() - .then(|fork_state| { + .wide_then(|fork_state| { fork_state .into_iter() .stream() - .filter_map(|(k, id)| { + .wide_filter_map(|(k, id)| { self.services .short .get_statekey_from_short(k) @@ -83,7 +87,7 @@ pub async fn resolve_state( let state_events: Vec<_> = state .iter() .stream() - .then(|((event_type, state_key), event_id)| { + .wide_then(|((event_type, state_key), event_id)| { self.services .short .get_or_create_shortstatekey(event_type, state_key) diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 96ee9907..9b30a830 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -4,7 +4,12 @@ use std::{ sync::Arc, }; -use conduit::{debug, err, implement, result::LogErr, utils::IterStream, PduEvent, Result}; +use conduit::{ + debug, err, implement, + result::LogErr, + utils::stream::{BroadbandExt, IterStream}, + PduEvent, Result, +}; use futures::{FutureExt, StreamExt}; use ruma::{ state_res::{self, StateMap}, @@ -166,7 +171,7 @@ pub(super) async fn state_at_incoming_resolved( new_state .iter() .stream() - .then(|((event_type, state_key), event_id)| { + .broad_then(|((event_type, state_key), event_id)| { self.services .short .get_or_create_shortstatekey(event_type, state_key) diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index b06e988e..3d05a1c8 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -3,7 +3,10 @@ use std::{mem::size_of, sync::Arc}; use arrayvec::ArrayVec; use conduit::{ result::LogErr, - utils::{stream::TryIgnore, u64_from_u8, ReadyExt}, + utils::{ + stream::{TryIgnore, WidebandExt}, + u64_from_u8, ReadyExt, + }, PduCount, PduEvent, }; use database::Map; @@ -67,7 +70,7 @@ impl Data { .ready_take_while(move |key| key.starts_with(&target.to_be_bytes())) .map(|to_from| u64_from_u8(&to_from[8..16])) .map(PduCount::from_unsigned) - .filter_map(move |shorteventid| async move { + .wide_filter_map(move |shorteventid| async move { let pdu_id: RawPduId = PduId { shortroomid, shorteventid, diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index 5821f279..a304e482 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -2,7 +2,10 @@ use std::{collections::BTreeMap, sync::Arc}; use conduit::{ err, - utils::{stream::TryIgnore, ReadyExt}, + utils::{ + stream::{TryIgnore, WidebandExt}, + ReadyExt, + }, PduCount, PduEvent, PduId, RawPduId, Result, }; use database::{Deserialized, Map}; @@ -143,7 +146,7 @@ impl Service { .ignore_err() .map(RawPduId::from) .ready_take_while(move |pdu_id| pdu_id.shortroomid() == shortroomid.to_be_bytes()) - .filter_map(move |pdu_id| async move { + .wide_filter_map(move |pdu_id| async move { let mut pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await.ok()?; let pdu_id: PduId = pdu_id.into(); From 68afdb22c7e4b0a7baeeba405c8257d9866263ff Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 4 Dec 2024 21:54:40 +0000 Subject: [PATCH 1167/2091] force Cargo.lock version to 3 Signed-off-by: Jason Volk --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index d787739e..52110f91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 4 +version = 3 [[package]] name = "addr2line" From ad0c5ceda409f9d5df9e49e335c82f5657beb1bf Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 4 Dec 2024 17:13:39 -0500 Subject: [PATCH 1168/2091] add origin to tracing instrument logs on /send Signed-off-by: strawberry --- src/api/server/send.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 2da99c93..3d861ad7 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -36,7 +36,7 @@ type ResolvedMap = BTreeMap>; /// # `PUT /_matrix/federation/v1/send/{txnId}` /// /// Push EDUs and PDUs to this server. -#[tracing::instrument(skip_all, fields(%client), name = "send")] +#[tracing::instrument(skip_all, fields(%client, origin = body.origin().as_str()), name = "send")] pub(crate) async fn send_transaction_message_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, From fe1ce521aa77f07bb6102c1305c440ca904938d5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 4 Dec 2024 18:33:12 -0500 Subject: [PATCH 1169/2091] add ignored user checks on /context and /event, misc cleanup Signed-off-by: strawberry --- src/api/client/context.rs | 8 +++-- src/api/client/room/event.rs | 65 ++++++++++++++++++++++-------------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index acd7d80b..45c04eb6 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -79,11 +79,15 @@ pub(crate) async fn get_context_route( let (base_token, base_event, visible) = try_join!(base_token, base_event, visible)?; - if base_event.room_id != body.room_id { + if base_event.room_id != body.room_id || base_event.event_id != body.event_id { return Err!(Request(NotFound("Base event not found."))); } - if !visible { + if !visible + || ignored_filter(&services, (base_token, base_event.clone()), sender_user) + .await + .is_none() + { return Err!(Request(Forbidden("You don't have permission to view this event."))); } diff --git a/src/api/client/room/event.rs b/src/api/client/room/event.rs index 090c70a7..6deb567f 100644 --- a/src/api/client/room/event.rs +++ b/src/api/client/room/event.rs @@ -1,38 +1,53 @@ use axum::extract::State; -use conduit::{err, Result}; -use futures::TryFutureExt; +use conduit::{err, Err, Event, Result}; +use futures::{try_join, FutureExt, TryFutureExt}; use ruma::api::client::room::get_room_event; -use crate::Ruma; +use crate::{client::ignored_filter, Ruma}; /// # `GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}` /// /// Gets a single event. -/// -/// - You have to currently be joined to the room (TODO: Respect history -/// visibility) pub(crate) async fn get_room_event_route( State(services): State, ref body: Ruma, ) -> Result { + let event = services + .rooms + .timeline + .get_pdu(&body.event_id) + .map_err(|_| err!(Request(NotFound("Event {} not found.", &body.event_id)))); + + let token = services + .rooms + .timeline + .get_pdu_count(&body.event_id) + .map_err(|_| err!(Request(NotFound("Event not found.")))); + + let visible = services + .rooms + .state_accessor + .user_can_see_event(body.sender_user(), &body.room_id, &body.event_id) + .map(Ok); + + let (token, mut event, visible) = try_join!(token, event, visible)?; + + if !visible + || ignored_filter(&services, (token, event.clone()), body.sender_user()) + .await + .is_none() + { + return Err!(Request(Forbidden("You don't have permission to view this event."))); + } + + if event.event_id() != &body.event_id || event.room_id() != body.room_id { + return Err!(Request(NotFound("Event not found"))); + } + + event.add_age().ok(); + + let event = event.to_room_event(); + Ok(get_room_event::v3::Response { - event: services - .rooms - .timeline - .get_pdu(&body.event_id) - .map_err(|_| err!(Request(NotFound("Event {} not found.", &body.event_id)))) - .and_then(|event| async move { - services - .rooms - .state_accessor - .user_can_see_event(body.sender_user(), &event.room_id, &body.event_id) - .await - .then_some(event) - .ok_or_else(|| err!(Request(Forbidden("You don't have permission to view this event.")))) - }) - .map_ok(|mut event| { - event.add_age().ok(); - event.to_room_event() - }) - .await?, + event, }) } From 52cee6574862fc68040a4831a91d0a32b2cb6a96 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 7 Dec 2024 23:27:56 +0000 Subject: [PATCH 1170/2091] add support for binding to a specific interface for url previews This is helpful to, for example, bind to an interface that can only access the public internet. The resulting setup is less maintenance-heavy / error-prone than manually maintaining a deny/ allowlist to protect internal resources. Signed-off-by: Jade Ellis --- conduwuit-example.toml | 10 ++++++++++ src/core/config/mod.rs | 19 +++++++++++++++++++ src/service/client/mod.rs | 16 ++++++++++++---- 3 files changed, 41 insertions(+), 4 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index aee29f92..b4bce140 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1117,6 +1117,16 @@ # #ip_range_denylist = ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", +# Optional interface to bind to with SO_BINDTODEVICE for URL previews. +# If not set, it will not bind to a specific interface. +# This uses [`reqwest::ClientBuilder::interface`] under the hood. +# +# To list the interfaces on your system, use the command `ip link show` +# +# Example: `"eth0"` +# +#url_preview_bound_interface = + # Vector list of domains allowed to send requests to for URL previews. # Defaults to none. Note: this is a *contains* match, not an explicit # match. Putting "google.com" will match "https://google.com" and diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index cb8f3b87..c613c819 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1250,6 +1250,16 @@ pub struct Config { #[serde(default = "default_ip_range_denylist")] pub ip_range_denylist: Vec, + /// Optional interface to bind to with SO_BINDTODEVICE for URL previews. + /// If not set, it will not bind to a specific interface. + /// This uses [`reqwest::ClientBuilder::interface`] under the hood. + /// + /// To list the interfaces on your system, use the command `ip link show` + /// + /// Example: `"eth0"` + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + pub url_preview_bound_interface: Option, + /// Vector list of domains allowed to send requests to for URL previews. /// Defaults to none. Note: this is a *contains* match, not an explicit /// match. Putting "google.com" will match "https://google.com" and @@ -1960,6 +1970,15 @@ impl fmt::Display for Config { line("Forbidden room aliases", { &self.forbidden_alias_names.patterns().iter().join(", ") }); + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + line( + "URL preview bound interface", + if let Some(interface) = &self.url_preview_bound_interface { + interface + } else { + "not set" + }, + ); line( "URL preview domain contains allowlist", &self.url_preview_domain_contains_allowlist.join(", "), diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index 2794efc1..bcd88158 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -25,15 +25,23 @@ impl crate::Service for Service { let config = &args.server.config; let resolver = args.require::("resolver"); + let url_preview_builder = base(config)? + .dns_resolver(resolver.resolver.clone()) + .redirect(redirect::Policy::limited(3)); + + #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] + let url_preview_builder = if let Some(interface) = &config.url_preview_bound_interface { + url_preview_builder.interface(interface) + } else { + url_preview_builder + }; + Ok(Arc::new(Self { default: base(config)? .dns_resolver(resolver.resolver.clone()) .build()?, - url_preview: base(config)? - .dns_resolver(resolver.resolver.clone()) - .redirect(redirect::Policy::limited(3)) - .build()?, + url_preview: url_preview_builder.build()?, extern_media: base(config)? .dns_resolver(resolver.resolver.clone()) From f0a1aaf7bcddbcec45cfa6c8040fada967a32cca Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 02:39:40 +0000 Subject: [PATCH 1171/2091] extend x-platform support for binding URL previews to interfaces via address Signed-off-by: Jason Volk --- Cargo.lock | 1 + conduwuit-example.toml | 12 ++++++----- src/core/config/check.rs | 12 +++++++++++ src/core/config/mod.rs | 31 ++++++++++++++++----------- src/service/Cargo.toml | 1 + src/service/client/mod.rs | 45 ++++++++++++++++++++++++++++++--------- 6 files changed, 74 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 52110f91..e6659e03 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -824,6 +824,7 @@ dependencies = [ "conduit_core", "conduit_database", "const-str", + "either", "futures", "hickory-resolver", "http", diff --git a/conduwuit-example.toml b/conduwuit-example.toml index b4bce140..bdccdb99 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1117,13 +1117,15 @@ # #ip_range_denylist = ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", -# Optional interface to bind to with SO_BINDTODEVICE for URL previews. -# If not set, it will not bind to a specific interface. -# This uses [`reqwest::ClientBuilder::interface`] under the hood. +# Optional IP address or network interface-name to bind as the source of +# URL preview requests. If not set, it will not bind to a specific +# address or interface. # -# To list the interfaces on your system, use the command `ip link show` +# Interface names only supported on Linux, Android, and Fuchsia platforms; +# all other platforms can specify the IP address. To list the interfaces +# on your system, use the command `ip link show`. # -# Example: `"eth0"` +# example: `"eth0"` or `"1.2.3.4"` # #url_preview_bound_interface = diff --git a/src/core/config/check.rs b/src/core/config/check.rs index c0d05533..c75fb31e 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -1,3 +1,6 @@ +use std::env::consts::OS; + +use either::Either; use figment::Figment; use super::DEPRECATED_KEYS; @@ -191,6 +194,15 @@ For security and safety reasons, conduwuit will shut down. If you are extra sure ); } + if let Some(Either::Right(_)) = config.url_preview_bound_interface.as_ref() { + if !matches!(OS, "android" | "fuchsia" | "linux") { + return Err!(Config( + "url_preview_bound_interface", + "Not a valid IP address. Interface names not supported on {OS}." + )); + } + } + Ok(()) } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index c613c819..117b4da5 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1250,15 +1250,19 @@ pub struct Config { #[serde(default = "default_ip_range_denylist")] pub ip_range_denylist: Vec, - /// Optional interface to bind to with SO_BINDTODEVICE for URL previews. - /// If not set, it will not bind to a specific interface. - /// This uses [`reqwest::ClientBuilder::interface`] under the hood. + /// Optional IP address or network interface-name to bind as the source of + /// URL preview requests. If not set, it will not bind to a specific + /// address or interface. /// - /// To list the interfaces on your system, use the command `ip link show` + /// Interface names only supported on Linux, Android, and Fuchsia platforms; + /// all other platforms can specify the IP address. To list the interfaces + /// on your system, use the command `ip link show`. /// - /// Example: `"eth0"` - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - pub url_preview_bound_interface: Option, + /// example: `"eth0"` or `"1.2.3.4"` + /// + /// default: + #[serde(default, with = "either::serde_untagged_optional")] + pub url_preview_bound_interface: Option>, /// Vector list of domains allowed to send requests to for URL previews. /// Defaults to none. Note: this is a *contains* match, not an explicit @@ -1970,14 +1974,15 @@ impl fmt::Display for Config { line("Forbidden room aliases", { &self.forbidden_alias_names.patterns().iter().join(", ") }); - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] line( "URL preview bound interface", - if let Some(interface) = &self.url_preview_bound_interface { - interface - } else { - "not set" - }, + self.url_preview_bound_interface + .as_ref() + .map(Either::as_ref) + .map(|either| either.map_left(ToString::to_string)) + .map(Either::either_into::) + .unwrap_or_default() + .as_str(), ); line( "URL preview domain contains allowlist", diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 7578ef64..19747847 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -47,6 +47,7 @@ bytes.workspace = true conduit-core.workspace = true conduit-database.workspace = true const-str.workspace = true +either.workspace = true futures.workspace = true hickory-resolver.workspace = true http.workspace = true diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index bcd88158..71545541 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -1,6 +1,7 @@ use std::{sync::Arc, time::Duration}; use conduit::{err, implement, trace, Config, Result}; +use either::Either; use ipaddress::IPAddress; use reqwest::redirect; @@ -25,23 +26,27 @@ impl crate::Service for Service { let config = &args.server.config; let resolver = args.require::("resolver"); - let url_preview_builder = base(config)? - .dns_resolver(resolver.resolver.clone()) - .redirect(redirect::Policy::limited(3)); + let url_preview_bind_addr = config + .url_preview_bound_interface + .clone() + .and_then(Either::left); - #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] - let url_preview_builder = if let Some(interface) = &config.url_preview_bound_interface { - url_preview_builder.interface(interface) - } else { - url_preview_builder - }; + let url_preview_bind_iface = config + .url_preview_bound_interface + .clone() + .and_then(Either::right); Ok(Arc::new(Self { default: base(config)? .dns_resolver(resolver.resolver.clone()) .build()?, - url_preview: url_preview_builder.build()?, + url_preview: base(config) + .and_then(|builder| builder_interface(builder, url_preview_bind_iface.as_deref()))? + .local_address(url_preview_bind_addr) + .dns_resolver(resolver.resolver.clone()) + .redirect(redirect::Policy::limited(3)) + .build()?, extern_media: base(config)? .dns_resolver(resolver.resolver.clone()) @@ -172,6 +177,26 @@ fn base(config: &Config) -> Result { } } +#[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] +fn builder_interface(builder: reqwest::ClientBuilder, config: Option<&str>) -> Result { + if let Some(iface) = config { + Ok(builder.interface(iface)) + } else { + Ok(builder) + } +} + +#[cfg(not(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] +fn builder_interface(builder: reqwest::ClientBuilder, config: Option<&str>) -> Result { + use conduit::Err; + + if let Some(iface) = config { + Err!("Binding to network-interface {iface:?} by name is not supported on this platform.") + } else { + Ok(builder) + } +} + #[inline] #[must_use] #[implement(Service)] From 8e8c6bfe07633eea75149ed067a04bc393433c15 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 5 Dec 2024 07:23:51 +0000 Subject: [PATCH 1172/2091] abstract supported room versions apis Signed-off-by: Jason Volk --- src/api/client/capabilities.rs | 16 ++-------- src/api/client/membership.rs | 20 +++--------- src/api/client/room/create.rs | 8 ++--- src/api/client/room/upgrade.rs | 6 +--- src/api/server/invite.rs | 6 +--- src/core/info/mod.rs | 1 + src/core/info/room_version.rs | 58 ++++++++++++++++++++++++++++++++++ src/service/admin/create.rs | 5 ++- src/service/globals/mod.rs | 43 +++---------------------- 9 files changed, 75 insertions(+), 88 deletions(-) create mode 100644 src/core/info/room_version.rs diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 89157e47..20f9cb58 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -19,23 +19,11 @@ use crate::{Result, Ruma}; pub(crate) async fn get_capabilities_route( State(services): State, _body: Ruma, ) -> Result { - let available: BTreeMap = services - .globals - .unstable_room_versions - .iter() - .map(|unstable_room_version| (unstable_room_version.clone(), RoomVersionStability::Unstable)) - .chain( - services - .globals - .stable_room_versions - .iter() - .map(|stable_room_version| (stable_room_version.clone(), RoomVersionStability::Stable)), - ) - .collect(); + let available: BTreeMap = services.server.available_room_versions().collect(); let mut capabilities = Capabilities::default(); capabilities.room_versions = RoomVersionsCapability { - default: services.globals.default_room_version(), + default: services.server.config.default_room_version.clone(), available, }; diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 9478e383..1cdf25a2 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -706,11 +706,7 @@ async fn join_room_by_id_helper_remote( return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); }; - if !services - .globals - .supported_room_versions() - .contains(&room_version_id) - { + if !services.server.supported_room_version(&room_version_id) { return Err!(BadServerResponse( "Remote room version {room_version_id} is not supported by conduwuit" )); @@ -1122,11 +1118,7 @@ async fn join_room_by_id_helper_local( return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); }; - if !services - .globals - .supported_room_versions() - .contains(&room_version_id) - { + if !services.server.supported_room_version(&room_version_id) { return Err!(BadServerResponse( "Remote room version {room_version_id} is not supported by conduwuit" )); @@ -1260,7 +1252,7 @@ async fn make_join_request( federation::membership::prepare_join_event::v1::Request { room_id: room_id.to_owned(), user_id: sender_user.to_owned(), - ver: services.globals.supported_room_versions(), + ver: services.server.supported_room_versions().collect(), }, ) .await; @@ -1616,11 +1608,7 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); }; - if !services - .globals - .supported_room_versions() - .contains(&room_version_id) - { + if !services.server.supported_room_version(&room_version_id) { return Err!(BadServerResponse( "Remote room version {room_version_id} is not supported by conduwuit" )); diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index 2ccb1c87..2e9852f8 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -116,11 +116,7 @@ pub(crate) async fn create_room_route( let room_version = match body.room_version.clone() { Some(room_version) => { - if services - .globals - .supported_room_versions() - .contains(&room_version) - { + if services.server.supported_room_version(&room_version) { room_version } else { return Err(Error::BadRequest( @@ -129,7 +125,7 @@ pub(crate) async fn create_room_route( )); } }, - None => services.globals.default_room_version(), + None => services.server.config.default_room_version.clone(), }; let create_content = match &body.creation_content { diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index ad5c356e..fafce2d1 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -47,11 +47,7 @@ pub(crate) async fn upgrade_room_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if !services - .globals - .supported_room_versions() - .contains(&body.new_version) - { + if !services.server.supported_room_version(&body.new_version) { return Err(Error::BadRequest( ErrorKind::UnsupportedRoomVersion, "This server does not support that room version.", diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 0ceb914f..f7919bb3 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -26,11 +26,7 @@ pub(crate) async fn create_invite_route( .acl_check(body.origin(), &body.room_id) .await?; - if !services - .globals - .supported_room_versions() - .contains(&body.room_version) - { + if !services.server.supported_room_version(&body.room_version) { return Err(Error::BadRequest( ErrorKind::IncompatibleRoomVersion { room_version: body.room_version.clone(), diff --git a/src/core/info/mod.rs b/src/core/info/mod.rs index e4a33b4e..22ea7d3d 100644 --- a/src/core/info/mod.rs +++ b/src/core/info/mod.rs @@ -2,6 +2,7 @@ //! etc information which can be queried by admins or used by developers. pub mod cargo; +pub mod room_version; pub mod rustc; pub mod version; diff --git a/src/core/info/room_version.rs b/src/core/info/room_version.rs new file mode 100644 index 00000000..59685b5f --- /dev/null +++ b/src/core/info/room_version.rs @@ -0,0 +1,58 @@ +//! Room version support + +use std::iter::once; + +use ruma::{api::client::discovery::get_capabilities::RoomVersionStability, RoomVersionId}; + +use crate::{at, is_equal_to}; + +/// Supported and stable room versions +pub const STABLE_ROOM_VERSIONS: &[RoomVersionId] = &[ + RoomVersionId::V6, + RoomVersionId::V7, + RoomVersionId::V8, + RoomVersionId::V9, + RoomVersionId::V10, + RoomVersionId::V11, +]; + +/// Experimental, partially supported room versions +pub const UNSTABLE_ROOM_VERSIONS: &[RoomVersionId] = + &[RoomVersionId::V2, RoomVersionId::V3, RoomVersionId::V4, RoomVersionId::V5]; + +impl crate::Server { + #[inline] + pub fn supported_room_version(&self, version: &RoomVersionId) -> bool { + self.supported_room_versions().any(is_equal_to!(*version)) + } + + #[inline] + pub fn supported_room_versions(&self) -> impl Iterator + '_ { + self.available_room_versions() + .filter(|(_, stability)| self.supported_stability(stability)) + .map(at!(0)) + } + + #[inline] + pub fn available_room_versions(&self) -> impl Iterator { + available_room_versions() + } + + #[inline] + fn supported_stability(&self, stability: &RoomVersionStability) -> bool { + self.config.allow_unstable_room_versions || *stability == RoomVersionStability::Stable + } +} + +pub fn available_room_versions() -> impl Iterator { + let unstable_room_versions = UNSTABLE_ROOM_VERSIONS + .iter() + .cloned() + .zip(once(RoomVersionStability::Unstable).cycle()); + + STABLE_ROOM_VERSIONS + .iter() + .cloned() + .zip(once(RoomVersionStability::Stable).cycle()) + .chain(unstable_room_versions) +} diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 1631f1cb..897d412d 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -25,6 +25,7 @@ use crate::Services; /// used to issue admin commands by talking to the server user inside it. pub async fn create_admin_room(services: &Services) -> Result<()> { let room_id = RoomId::new(services.globals.server_name()); + let room_version = &services.server.config.default_room_version; let _short_id = services .rooms @@ -38,8 +39,6 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { let server_user = &services.globals.server_user; services.users.create(server_user, None)?; - let room_version = services.globals.default_room_version(); - let create_content = { use RoomVersionId::*; match room_version { @@ -58,7 +57,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { &RoomCreateEventContent { federate: true, predecessor: None, - room_version, + room_version: room_version.clone(), ..create_content }, ), diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 4fb1ce2d..fefff3b5 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -10,9 +10,7 @@ use std::{ use conduit::{error, Config, Result}; use data::Data; use regex::RegexSet; -use ruma::{ - OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomAliasId, RoomVersionId, ServerName, UserId, -}; +use ruma::{OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomAliasId, ServerName, UserId}; use tokio::sync::Mutex; use crate::service; @@ -22,8 +20,6 @@ pub struct Service { pub config: Config, jwt_decoding_key: Option, - pub stable_room_versions: Vec, - pub unstable_room_versions: Vec, pub bad_event_ratelimiter: Arc>>, pub bad_query_ratelimiter: Arc>>, pub stateres_mutex: Arc>, @@ -45,18 +41,6 @@ impl crate::Service for Service { .as_ref() .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes())); - // Supported and stable room versions - let stable_room_versions = vec![ - RoomVersionId::V6, - RoomVersionId::V7, - RoomVersionId::V8, - RoomVersionId::V9, - RoomVersionId::V10, - RoomVersionId::V11, - ]; - // Experimental, partially supported room versions - let unstable_room_versions = vec![RoomVersionId::V2, RoomVersionId::V3, RoomVersionId::V4, RoomVersionId::V5]; - let turn_secret = config .turn_secret_file .as_ref() @@ -86,8 +70,6 @@ impl crate::Service for Service { db, config: config.clone(), jwt_decoding_key, - stable_room_versions, - unstable_room_versions, bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), stateres_mutex: Arc::new(Mutex::new(())), @@ -99,9 +81,9 @@ impl crate::Service for Service { registration_token, }; - if !s - .supported_room_versions() - .contains(&s.config.default_room_version) + if !args + .server + .supported_room_version(&config.default_room_version) { error!(config=?s.config.default_room_version, fallback=?conduit::config::default_default_room_version(), "Room version in config isn't supported, falling back to default version"); s.config.default_room_version = conduit::config::default_default_room_version(); @@ -173,11 +155,6 @@ impl Service { pub fn allow_room_creation(&self) -> bool { self.config.allow_room_creation } - pub fn allow_unstable_room_versions(&self) -> bool { self.config.allow_unstable_room_versions } - - #[inline] - pub fn default_room_version(&self) -> RoomVersionId { self.config.default_room_version.clone() } - pub fn new_user_displayname_suffix(&self) -> &String { &self.config.new_user_displayname_suffix } pub fn allow_check_for_updates(&self) -> bool { self.config.allow_check_for_updates } @@ -232,18 +209,6 @@ impl Service { pub fn block_non_admin_invites(&self) -> bool { self.config.block_non_admin_invites } - pub fn supported_room_versions(&self) -> Vec { - if self.config.allow_unstable_room_versions { - self.stable_room_versions - .clone() - .into_iter() - .chain(self.unstable_room_versions.clone()) - .collect() - } else { - self.stable_room_versions.clone() - } - } - /// checks if `user_id` is local to us via server_name comparison #[inline] pub fn user_is_local(&self, user_id: &UserId) -> bool { self.server_is_ours(user_id.server_name()) } From de3b137df801f4e362957235884b11e34a7c6626 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 6 Dec 2024 03:09:08 +0000 Subject: [PATCH 1173/2091] eliminate future wrapping stream for all_pdus() Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 1 - src/service/rooms/timeline/data.rs | 2 +- src/service/rooms/timeline/mod.rs | 17 ++++++++++------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 5578077f..3a78c9ad 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -609,7 +609,6 @@ async fn load_joined_room( .rooms .timeline .all_pdus(sender_user, room_id) - .await? .ready_filter(|(_, pdu)| pdu.kind == RoomMember) .filter_map(|(_, pdu)| async move { let content: RoomMemberEventContent = pdu.get_content().ok()?; diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index c394dc3b..94621385 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -220,7 +220,7 @@ impl Data { pub(super) async fn pdus<'a>( &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, from: PduCount, - ) -> Result + Send + 'a> { + ) -> Result + Send + Unpin + 'a> { let current = self.count_to_id(room_id, from, Direction::Forward).await?; let prefix = current.shortroomid(); let stream = self diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 07b406c4..0a96322b 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -15,7 +15,7 @@ use conduit::{ validated, warn, Err, Error, Result, Server, }; pub use conduit::{PduId, RawPduId}; -use futures::{future, future::ready, Future, FutureExt, Stream, StreamExt, TryStreamExt}; +use futures::{future, future::ready, Future, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{ api::federation, canonical_json::to_canonical_value, @@ -168,7 +168,6 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] pub async fn first_pdu_in_room(&self, room_id: &RoomId) -> Result> { self.all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id) - .await? .next() .await .map(|(_, p)| Arc::new(p)) @@ -968,12 +967,17 @@ impl Service { Ok(Some(pdu_id)) } - /// Returns an iterator over all PDUs in a room. + /// Returns an iterator over all PDUs in a room. Unknown rooms produce no + /// items. #[inline] - pub async fn all_pdus<'a>( + pub fn all_pdus<'a>( &'a self, user_id: &'a UserId, room_id: &'a RoomId, - ) -> Result + Send + 'a> { - self.pdus(Some(user_id), room_id, None).await + ) -> impl Stream + Send + Unpin + 'a { + self.pdus(Some(user_id), room_id, None) + .map_ok(|stream| stream.map(Ok)) + .try_flatten_stream() + .ignore_err() + .boxed() } /// Reverse iteration starting at from. @@ -1048,7 +1052,6 @@ impl Service { let first_pdu = self .all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id) - .await? .next() .await .expect("Room is not empty"); From 23cf2b2236405c64f990671e8610fa1a4471acd5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 6 Dec 2024 12:42:14 +0000 Subject: [PATCH 1174/2091] add is_err() to TryFuture extension add fold_default to Future tools extension Signed-off-by: Jason Volk --- src/core/utils/future/try_ext_ext.rs | 21 +++++++++++++++------ src/core/utils/stream/tools.rs | 16 ++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/core/utils/future/try_ext_ext.rs b/src/core/utils/future/try_ext_ext.rs index f97ae885..81c7aac0 100644 --- a/src/core/utils/future/try_ext_ext.rs +++ b/src/core/utils/future/try_ext_ext.rs @@ -1,5 +1,8 @@ //! Extended external extensions to futures::TryFutureExt #![allow(clippy::type_complexity)] +// is_ok() has to consume *self rather than borrow. This extension is for a +// caller only ever caring about result status while discarding all contents. +#![allow(clippy::wrong_self_convention)] use futures::{ future::{MapOkOrElse, UnwrapOrElse}, @@ -11,12 +14,10 @@ pub trait TryExtExt where Self: TryFuture + Send, { - /// Resolves to a bool for whether the TryFuture (Future of a Result) - /// resolved to Ok or Err. - /// - /// is_ok() has to consume *self rather than borrow. The intent of this - /// extension is therefor for a caller only ever caring about result status - /// while discarding all contents. + fn is_err(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> + where + Self: Sized; + #[allow(clippy::wrong_self_convention)] fn is_ok(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> where @@ -44,6 +45,14 @@ impl TryExtExt for Fut where Fut: TryFuture + Send, { + #[inline] + fn is_err(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> + where + Self: Sized, + { + self.map_ok_or(true, |_| false) + } + #[inline] fn is_ok(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> where diff --git a/src/core/utils/stream/tools.rs b/src/core/utils/stream/tools.rs index cc6b7ca9..b5b036cc 100644 --- a/src/core/utils/stream/tools.rs +++ b/src/core/utils/stream/tools.rs @@ -32,6 +32,12 @@ where fn counts_with_cap(self) -> impl Future> + Send where ::Item: Eq + Hash; + + fn fold_default(self, f: F) -> impl Future + Send + where + F: Fn(T, Item) -> Fut + Send, + Fut: Future + Send, + T: Default + Send; } impl Tools for S @@ -77,4 +83,14 @@ where counts }) } + + #[inline] + fn fold_default(self, f: F) -> impl Future + Send + where + F: Fn(T, Item) -> Fut + Send, + Fut: Future + Send, + T: Default + Send, + { + self.fold(T::default(), f) + } } From b77a1eb079b8d3e5144849a74805ec653cd722e6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 6 Dec 2024 12:44:10 +0000 Subject: [PATCH 1175/2091] move cork to remove_to_device_events fn add cork around write-heavy database routine called in loop; cleanup Signed-off-by: Jason Volk --- src/service/admin/console.rs | 2 +- src/service/rooms/user/mod.rs | 15 +++++++++------ src/service/users/mod.rs | 5 ++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 0f5016e1..37af7452 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -89,7 +89,7 @@ impl Console { } } - #[tracing::instrument(skip_all, name = "console")] + #[tracing::instrument(skip_all, name = "console", level = "trace")] async fn worker(self: Arc) { debug!("session starting"); while self.server.running() { diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 948baa5e..537fe69b 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use conduit::{implement, Result}; -use database::{Deserialized, Map}; +use database::{Database, Deserialized, Map}; use ruma::{RoomId, UserId}; use crate::{globals, rooms, rooms::short::ShortStateHash, Dep}; @@ -12,6 +12,7 @@ pub struct Service { } struct Data { + db: Arc, userroomid_notificationcount: Arc, userroomid_highlightcount: Arc, roomuserid_lastnotificationread: Arc, @@ -27,11 +28,12 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { db: Data { - userroomid_notificationcount: args.db["userroomid_notificationcount"].clone(), - userroomid_highlightcount: args.db["userroomid_highlightcount"].clone(), - roomuserid_lastnotificationread: args.db["userroomid_highlightcount"].clone(), //< NOTE: known bug from conduit - roomsynctoken_shortstatehash: args.db["roomsynctoken_shortstatehash"].clone(), - }, + db: args.db.clone(), + userroomid_notificationcount: args.db["userroomid_notificationcount"].clone(), + userroomid_highlightcount: args.db["userroomid_highlightcount"].clone(), + roomuserid_lastnotificationread: args.db["userroomid_highlightcount"].clone(), + roomsynctoken_shortstatehash: args.db["roomsynctoken_shortstatehash"].clone(), + }, services: Services { globals: args.depend::("globals"), @@ -98,6 +100,7 @@ pub async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, .await .expect("room exists"); + let _cork = self.db.db.cork(); let key: &[u64] = &[shortroomid, token]; self.db .roomsynctoken_shortstatehash diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 2462dde3..f17a6b9d 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -5,7 +5,7 @@ use conduit::{ utils::{stream::TryIgnore, string::Unquoted, ReadyExt}, Err, Error, Result, Server, }; -use database::{Deserialized, Ignore, Interfix, Json, Map}; +use database::{Database, Deserialized, Ignore, Interfix, Json, Map}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, @@ -26,6 +26,7 @@ pub struct Service { struct Services { server: Arc, + db: Arc, account_data: Dep, admin: Dep, globals: Dep, @@ -60,6 +61,7 @@ impl crate::Service for Service { Ok(Arc::new(Self { services: Services { server: args.server.clone(), + db: args.db.clone(), account_data: args.depend::("account_data"), admin: args.depend::("admin"), globals: args.depend::("globals"), @@ -721,6 +723,7 @@ impl Service { let mut last = prefix.clone(); last.extend_from_slice(&until.to_be_bytes()); + let _cork = self.services.db.cork_and_flush(); self.db .todeviceid_events .rev_raw_keys_from(&last) // this includes last From b7a41f283f0010d71f6475c8533c2384e20d629f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 6 Dec 2024 12:45:20 +0000 Subject: [PATCH 1176/2091] optimize sync v3 asynchronicity Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 1448 ++++++++++++++++++++----------------- 1 file changed, 788 insertions(+), 660 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 3a78c9ad..61a0ea5c 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -6,12 +6,25 @@ use std::{ use axum::extract::State; use conduit::{ - at, err, error, extract_variant, is_equal_to, - result::FlatOk, - utils::{math::ruma_from_u64, BoolExt, IterStream, ReadyExt, TryFutureExtExt}, - PduCount, + at, err, error, extract_variant, is_equal_to, is_false, + pdu::EventHash, + result::{FlatOk, LogDebugErr}, + utils, + utils::{ + math::ruma_from_u64, + stream::{BroadbandExt, Tools}, + BoolExt, IterStream, ReadyExt, TryFutureExtExt, + }, + Error, PduCount, PduEvent, Result, +}; +use conduit_service::{ + rooms::short::{ShortStateHash, ShortStateKey}, + Services, +}; +use futures::{ + future::{join, join3, join5, try_join, OptionFuture}, + FutureExt, StreamExt, TryFutureExt, }; -use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt}; use ruma::{ api::client::{ filter::{FilterDefinition, LazyLoadOptions}, @@ -34,14 +47,20 @@ use ruma::{ serde::Raw, uint, DeviceId, EventId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId, }; -use tracing::{Instrument as _, Span}; use super::{load_timeline, share_encrypted_room}; -use crate::{ - client::ignored_filter, - service::{pdu::EventHash, Services}, - utils, Error, PduEvent, Result, Ruma, RumaResponse, -}; +use crate::{client::ignored_filter, Ruma, RumaResponse}; + +#[derive(Default)] +struct StateChanges { + heroes: Option>, + joined_member_count: Option, + invited_member_count: Option, + joined_since_last_sync: bool, + state_events: Vec, +} + +type PresenceUpdates = HashMap; /// # `GET /_matrix/client/r0/sync` /// @@ -81,32 +100,30 @@ use crate::{ pub(crate) async fn sync_events_route( State(services): State, body: Ruma, ) -> Result> { - let sender_user = body.sender_user.expect("user is authenticated"); - let sender_device = body.sender_device.expect("user is authenticated"); - let body = body.body; + let (sender_user, sender_device) = body.sender(); // Presence update if services.globals.allow_local_presence() { services .presence - .ping_presence(&sender_user, &body.set_presence) + .ping_presence(sender_user, &body.body.set_presence) .await?; } // Setup watchers, so if there's no response, we can wait for them - let watcher = services.sync.watch(&sender_user, &sender_device); + let watcher = services.sync.watch(sender_user, sender_device); let next_batch = services.globals.current_count()?; let next_batchcount = PduCount::Normal(next_batch); let next_batch_string = next_batch.to_string(); // Load filter - let filter = match body.filter { + let filter = match body.body.filter.as_ref() { None => FilterDefinition::default(), - Some(Filter::FilterDefinition(filter)) => filter, - Some(Filter::FilterId(filter_id)) => services + Some(Filter::FilterDefinition(ref filter)) => filter.clone(), + Some(Filter::FilterId(ref filter_id)) => services .users - .get_filter(&sender_user, &filter_id) + .get_filter(sender_user, filter_id) .await .unwrap_or_default(), }; @@ -120,183 +137,190 @@ pub(crate) async fn sync_events_route( LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")), }; - let full_state = body.full_state; + let full_state = body.body.full_state; - let mut joined_rooms = BTreeMap::new(); let since = body + .body .since .as_ref() .and_then(|string| string.parse().ok()) .unwrap_or(0); let sincecount = PduCount::Normal(since); - let mut presence_updates = HashMap::new(); - let mut left_encrypted_users = HashSet::new(); // Users that have left any encrypted rooms the sender was in - let mut device_list_updates = HashSet::new(); - let mut device_list_left = HashSet::new(); - - // Look for device list updates of this account - device_list_updates.extend( - services - .users - .keys_changed(&sender_user, since, None) - .map(ToOwned::to_owned) - .collect::>() - .await, - ); - - if services.globals.allow_local_presence() { - process_presence_updates(&services, &mut presence_updates, since, &sender_user).await?; - } - - let all_joined_rooms: Vec<_> = services + let joined_related = services .rooms .state_cache - .rooms_joined(&sender_user) + .rooms_joined(sender_user) .map(ToOwned::to_owned) - .collect() - .await; + .broad_filter_map(|room_id| { + load_joined_room( + &services, + sender_user, + sender_device, + room_id.clone(), + since, + sincecount, + next_batch, + next_batchcount, + lazy_load_enabled, + lazy_load_send_redundant, + full_state, + ) + .map_ok(move |(joined_room, dlu, jeu)| (room_id, joined_room, dlu, jeu)) + .ok() + }) + .ready_fold( + (BTreeMap::new(), HashSet::new(), HashSet::new()), + |(mut joined_rooms, mut device_list_updates, mut left_encrypted_users), + (room_id, joined_room, dlu, leu)| { + device_list_updates.extend(dlu); + left_encrypted_users.extend(leu); + if !joined_room.is_empty() { + joined_rooms.insert(room_id, joined_room); + } - // Coalesce database writes for the remainder of this scope. - let _cork = services.db.cork_and_flush(); - - for room_id in all_joined_rooms { - if let Ok(joined_room) = load_joined_room( - &services, - &sender_user, - &sender_device, - &room_id, - since, - sincecount, - next_batch, - next_batchcount, - lazy_load_enabled, - lazy_load_send_redundant, - full_state, - &mut device_list_updates, - &mut left_encrypted_users, - ) - .await - { - if !joined_room.is_empty() { - joined_rooms.insert(room_id.clone(), joined_room); - } - } - } - - let mut left_rooms = BTreeMap::new(); - let all_left_rooms: Vec<_> = services - .rooms - .state_cache - .rooms_left(&sender_user) - .collect() - .await; - - for result in all_left_rooms { - handle_left_room( - &services, - since, - &result.0, - &sender_user, - &mut left_rooms, - &next_batch_string, - full_state, - lazy_load_enabled, - ) - .instrument(Span::current()) - .await?; - } - - let mut invited_rooms = BTreeMap::new(); - let all_invited_rooms: Vec<_> = services - .rooms - .state_cache - .rooms_invited(&sender_user) - .collect() - .await; - - for (room_id, invite_state_events) in all_invited_rooms { - // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; - drop(insert_lock); - - let invite_count = services - .rooms - .state_cache - .get_invite_count(&room_id, &sender_user) - .await - .ok(); - - // Invited before last sync - if Some(since) >= invite_count { - continue; - } - - invited_rooms.insert( - room_id.clone(), - InvitedRoom { - invite_state: InviteState { - events: invite_state_events, - }, + (joined_rooms, device_list_updates, left_encrypted_users) }, ); - } - for user_id in left_encrypted_users { - let dont_share_encrypted_room = !share_encrypted_room(&services, &sender_user, &user_id, None).await; + let left_rooms = services + .rooms + .state_cache + .rooms_left(sender_user) + .broad_filter_map(|(room_id, _)| { + handle_left_room( + &services, + since, + room_id.clone(), + sender_user, + &next_batch_string, + full_state, + lazy_load_enabled, + ) + .map_ok(move |left_room| (room_id, left_room)) + .ok() + }) + .ready_filter_map(|(room_id, left_room)| left_room.map(|left_room| (room_id, left_room))) + .collect(); - // If the user doesn't share an encrypted room with the target anymore, we need - // to tell them - if dont_share_encrypted_room { - device_list_left.insert(user_id); - } - } + let invited_rooms = services.rooms.state_cache.rooms_invited(sender_user).fold( + BTreeMap::new(), + |mut invited_rooms, (room_id, invite_state)| async move { + // Get and drop the lock to wait for remaining operations to finish + let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; + drop(insert_lock); + + let invite_count = services + .rooms + .state_cache + .get_invite_count(&room_id, sender_user) + .await + .ok(); + + // Invited before last sync + if Some(since) >= invite_count { + return invited_rooms; + } + + let invited_room = InvitedRoom { + invite_state: InviteState { + events: invite_state, + }, + }; + + invited_rooms.insert(room_id, invited_room); + invited_rooms + }, + ); + + let presence_updates: OptionFuture<_> = services + .globals + .allow_local_presence() + .then(|| process_presence_updates(&services, since, sender_user)) + .into(); + + let account_data = services + .account_data + .changes_since(None, sender_user, since) + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) + .collect(); + + // Look for device list updates of this account + let keys_changed = services + .users + .keys_changed(sender_user, since, None) + .map(ToOwned::to_owned) + .collect::>(); + + let to_device_events = services + .users + .get_to_device_events(sender_user, sender_device) + .collect::>(); + + let device_one_time_keys_count = services + .users + .count_one_time_keys(sender_user, sender_device); // Remove all to-device events the device received *last time* - services + let remove_to_device_events = services .users - .remove_to_device_events(&sender_user, &sender_device, since) + .remove_to_device_events(sender_user, sender_device, since); + + let rooms = join3(joined_related, left_rooms, invited_rooms); + let ephemeral = join3(remove_to_device_events, to_device_events, presence_updates); + let top = join5(account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms) + .boxed() + .await; + + let (account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms) = top; + let ((), to_device_events, presence_updates) = ephemeral; + let (joined_related, left_rooms, invited_rooms) = rooms; + let (joined_rooms, mut device_list_updates, left_encrypted_users) = joined_related; + device_list_updates.extend(keys_changed); + + // If the user doesn't share an encrypted room with the target anymore, we need + // to tell them + let device_list_left = left_encrypted_users + .into_iter() + .stream() + .broad_filter_map(|user_id| async move { + let no_shared_encrypted_room = !share_encrypted_room(&services, sender_user, &user_id, None).await; + no_shared_encrypted_room.then_some(user_id) + }) + .ready_fold(HashSet::new(), |mut device_list_left, user_id| { + device_list_left.insert(user_id); + device_list_left + }) .await; let response = sync_events::v3::Response { + account_data: GlobalAccountData { + events: account_data, + }, + device_lists: DeviceLists { + changed: device_list_updates.into_iter().collect(), + left: device_list_left.into_iter().collect(), + }, + device_one_time_keys_count, + // Fallback keys are not yet supported + device_unused_fallback_key_types: None, next_batch: next_batch_string, + presence: Presence { + events: presence_updates + .unwrap_or_default() + .into_values() + .map(|v| Raw::new(&v).expect("PresenceEvent always serializes successfully")) + .collect(), + }, rooms: Rooms { leave: left_rooms, join: joined_rooms, invite: invited_rooms, knock: BTreeMap::new(), // TODO }, - presence: Presence { - events: presence_updates - .into_values() - .map(|v| Raw::new(&v).expect("PresenceEvent always serializes successfully")) - .collect(), - }, - account_data: GlobalAccountData { - events: services - .account_data - .changes_since(None, &sender_user, since) - .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) - .collect() - .await, - }, - device_lists: DeviceLists { - changed: device_list_updates.into_iter().collect(), - left: device_list_left.into_iter().collect(), - }, - device_one_time_keys_count: services - .users - .count_one_time_keys(&sender_user, &sender_device) - .await, to_device: ToDevice { - events: services - .users - .get_to_device_events(&sender_user, &sender_device) - .collect() - .await, + events: to_device_events, }, - // Fallback keys are not yet supported - device_unused_fallback_key_types: None, }; // TODO: Retry the endpoint instead of returning @@ -310,37 +334,86 @@ pub(crate) async fn sync_events_route( // Hang a few seconds so requests are not spammed // Stop hanging if new info arrives let default = Duration::from_secs(30); - let duration = cmp::min(body.timeout.unwrap_or(default), default); + let duration = cmp::min(body.body.timeout.unwrap_or(default), default); _ = tokio::time::timeout(duration, watcher).await; } Ok(response) } +async fn process_presence_updates(services: &Services, since: u64, syncing_user: &UserId) -> PresenceUpdates { + services + .presence + .presence_since(since) + .filter(|(user_id, ..)| { + services + .rooms + .state_cache + .user_sees_user(syncing_user, user_id) + }) + .filter_map(|(user_id, _, presence_bytes)| { + services + .presence + .from_json_bytes_to_event(presence_bytes, user_id) + .map_ok(move |event| (user_id, event)) + .ok() + }) + .ready_fold(PresenceUpdates::new(), |mut updates, (user_id, event)| { + match updates.entry(user_id.into()) { + Entry::Vacant(slot) => { + slot.insert(event); + }, + Entry::Occupied(mut slot) => { + let curr_event = slot.get_mut(); + let curr_content = &mut curr_event.content; + let new_content = event.content; + + // Update existing presence event with more info + curr_content.presence = new_content.presence; + curr_content.status_msg = new_content + .status_msg + .or_else(|| curr_content.status_msg.take()); + curr_content.last_active_ago = new_content.last_active_ago.or(curr_content.last_active_ago); + curr_content.displayname = new_content + .displayname + .or_else(|| curr_content.displayname.take()); + curr_content.avatar_url = new_content + .avatar_url + .or_else(|| curr_content.avatar_url.take()); + curr_content.currently_active = new_content + .currently_active + .or(curr_content.currently_active); + }, + }; + + updates + }) + .await +} + #[allow(clippy::too_many_arguments)] #[tracing::instrument(skip_all, fields(user_id = %sender_user, room_id = %room_id), name = "left_room")] async fn handle_left_room( - services: &Services, since: u64, room_id: &RoomId, sender_user: &UserId, - left_rooms: &mut BTreeMap, next_batch_string: &str, full_state: bool, - lazy_load_enabled: bool, -) -> Result<()> { + services: &Services, since: u64, room_id: OwnedRoomId, sender_user: &UserId, next_batch_string: &str, + full_state: bool, lazy_load_enabled: bool, +) -> Result> { // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; + let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; drop(insert_lock); let left_count = services .rooms .state_cache - .get_left_count(room_id, sender_user) + .get_left_count(&room_id, sender_user) .await .ok(); // Left before last sync if Some(since) >= left_count { - return Ok(()); + return Ok(None); } - if !services.rooms.metadata.exists(room_id).await { + if !services.rooms.metadata.exists(&room_id).await { // This is just a rejected invite, not a room we know // Insert a leave event anyways let event = PduEvent { @@ -355,7 +428,7 @@ async fn handle_left_room( state_key: Some(sender_user.to_string()), unsigned: None, // The following keys are dropped on conversion - room_id: room_id.to_owned(), + room_id: room_id.clone(), prev_events: vec![], depth: uint!(1), auth_events: vec![], @@ -366,23 +439,19 @@ async fn handle_left_room( signatures: None, }; - left_rooms.insert( - room_id.to_owned(), - LeftRoom { - account_data: RoomAccountData { - events: Vec::new(), - }, - timeline: Timeline { - limited: false, - prev_batch: Some(next_batch_string.to_owned()), - events: Vec::new(), - }, - state: RoomState { - events: vec![event.to_sync_state_event()], - }, + return Ok(Some(LeftRoom { + account_data: RoomAccountData { + events: Vec::new(), }, - ); - return Ok(()); + timeline: Timeline { + limited: false, + prev_batch: Some(next_batch_string.to_owned()), + events: Vec::new(), + }, + state: RoomState { + events: vec![event.to_sync_state_event()], + }, + })); } let mut left_state_events = Vec::new(); @@ -390,7 +459,7 @@ async fn handle_left_room( let since_shortstatehash = services .rooms .user - .get_token_shortstatehash(room_id, since) + .get_token_shortstatehash(&room_id, since) .await; let since_state_ids = match since_shortstatehash { @@ -401,11 +470,11 @@ async fn handle_left_room( let Ok(left_event_id): Result = services .rooms .state_accessor - .room_state_get_id(room_id, &StateEventType::RoomMember, sender_user.as_str()) + .room_state_get_id(&room_id, &StateEventType::RoomMember, sender_user.as_str()) .await else { error!("Left room but no left state event"); - return Ok(()); + return Ok(None); }; let Ok(left_shortstatehash) = services @@ -415,7 +484,7 @@ async fn handle_left_room( .await else { error!(event_id = %left_event_id, "Leave event has no state"); - return Ok(()); + return Ok(None); }; let mut left_state_ids = services @@ -456,488 +525,102 @@ async fn handle_left_room( } } - left_rooms.insert( - room_id.to_owned(), - LeftRoom { - account_data: RoomAccountData { - events: Vec::new(), - }, - timeline: Timeline { - limited: false, - prev_batch: Some(next_batch_string.to_owned()), - events: Vec::new(), - }, - state: RoomState { - events: left_state_events, - }, + Ok(Some(LeftRoom { + account_data: RoomAccountData { + events: Vec::new(), }, - ); - Ok(()) -} - -async fn process_presence_updates( - services: &Services, presence_updates: &mut HashMap, since: u64, syncing_user: &UserId, -) -> Result<()> { - let presence_since = services.presence.presence_since(since); - - // Take presence updates - pin_mut!(presence_since); - while let Some((user_id, _, presence_bytes)) = presence_since.next().await { - if !services - .rooms - .state_cache - .user_sees_user(syncing_user, user_id) - .await - { - continue; - } - - let presence_event = services - .presence - .from_json_bytes_to_event(presence_bytes, user_id) - .await?; - - match presence_updates.entry(user_id.into()) { - Entry::Vacant(slot) => { - slot.insert(presence_event); - }, - Entry::Occupied(mut slot) => { - let curr_event = slot.get_mut(); - let curr_content = &mut curr_event.content; - let new_content = presence_event.content; - - // Update existing presence event with more info - curr_content.presence = new_content.presence; - curr_content.status_msg = new_content - .status_msg - .or_else(|| curr_content.status_msg.take()); - curr_content.last_active_ago = new_content.last_active_ago.or(curr_content.last_active_ago); - curr_content.displayname = new_content - .displayname - .or_else(|| curr_content.displayname.take()); - curr_content.avatar_url = new_content - .avatar_url - .or_else(|| curr_content.avatar_url.take()); - curr_content.currently_active = new_content - .currently_active - .or(curr_content.currently_active); - }, - }; - } - - Ok(()) + timeline: Timeline { + limited: false, + prev_batch: Some(next_batch_string.to_owned()), + events: Vec::new(), + }, + state: RoomState { + events: left_state_events, + }, + })) } #[allow(clippy::too_many_arguments)] async fn load_joined_room( - services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, since: u64, + services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: OwnedRoomId, since: u64, sincecount: PduCount, next_batch: u64, next_batchcount: PduCount, lazy_load_enabled: bool, - lazy_load_send_redundant: bool, full_state: bool, device_list_updates: &mut HashSet, - left_encrypted_users: &mut HashSet, -) -> Result { + lazy_load_send_redundant: bool, full_state: bool, +) -> Result<(JoinedRoom, HashSet, HashSet)> { + let mut device_list_updates = HashSet::::new(); + let mut left_encrypted_users = HashSet::::new(); + // Get and drop the lock to wait for remaining operations to finish // This will make sure the we have all events until next_batch - let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; + let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; drop(insert_lock); let (timeline_pdus, limited) = - load_timeline(services, sender_user, room_id, sincecount, Some(next_batchcount), 10_usize).await?; + load_timeline(services, sender_user, &room_id, sincecount, Some(next_batchcount), 10_usize).await?; let send_notification_counts = !timeline_pdus.is_empty() || services .rooms .user - .last_notification_read(sender_user, room_id) + .last_notification_read(sender_user, &room_id) .await > since; - let mut timeline_users = HashSet::new(); - for (_, event) in &timeline_pdus { - timeline_users.insert(event.sender.as_str().to_owned()); - } + let timeline_users = timeline_pdus + .iter() + .fold(HashSet::new(), |mut timeline_users, (_, event)| { + timeline_users.insert(event.sender.as_str().to_owned()); + timeline_users + }); services .rooms .lazy_loading - .lazy_load_confirm_delivery(sender_user, sender_device, room_id, sincecount); + .lazy_load_confirm_delivery(sender_user, sender_device, &room_id, sincecount); let current_shortstatehash = services .rooms .state - .get_room_shortstatehash(room_id) - .await - .map_err(|_| err!(Database(error!("Room {room_id} has no state"))))?; + .get_room_shortstatehash(&room_id) + .map_err(|_| err!(Database(error!("Room {room_id} has no state")))); let since_shortstatehash = services .rooms .user - .get_token_shortstatehash(room_id, since) - .await - .ok(); + .get_token_shortstatehash(&room_id, since) + .ok() + .map(Ok); - let (heroes, joined_member_count, invited_member_count, joined_since_last_sync, state_events) = if timeline_pdus - .is_empty() + let (current_shortstatehash, since_shortstatehash) = try_join(current_shortstatehash, since_shortstatehash).await?; + + let StateChanges { + heroes, + joined_member_count, + invited_member_count, + joined_since_last_sync, + state_events, + } = if timeline_pdus.is_empty() && (since_shortstatehash.is_none() || since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash))) { // No state changes - (Vec::new(), None, None, false, Vec::new()) + StateChanges::default() } else { - // Calculates joined_member_count, invited_member_count and heroes - let calculate_counts = || async { - let joined_member_count = services - .rooms - .state_cache - .room_joined_count(room_id) - .await - .unwrap_or(0); - - let invited_member_count = services - .rooms - .state_cache - .room_invited_count(room_id) - .await - .unwrap_or(0); - - if joined_member_count.saturating_add(invited_member_count) > 5 { - return Ok::<_, Error>((Some(joined_member_count), Some(invited_member_count), Vec::new())); - } - - // Go through all PDUs and for each member event, check if the user is still - // joined or invited until we have 5 or we reach the end - - // Recalculate heroes (first 5 members) - let heroes = services - .rooms - .timeline - .all_pdus(sender_user, room_id) - .ready_filter(|(_, pdu)| pdu.kind == RoomMember) - .filter_map(|(_, pdu)| async move { - let content: RoomMemberEventContent = pdu.get_content().ok()?; - let user_id: &UserId = pdu.state_key.as_deref().map(TryInto::try_into).flat_ok()?; - - if user_id == sender_user { - return None; - } - - // The membership was and still is invite or join - if !matches!(content.membership, MembershipState::Join | MembershipState::Invite) { - return None; - } - - let is_invited = services.rooms.state_cache.is_invited(user_id, room_id); - - let is_joined = services.rooms.state_cache.is_joined(user_id, room_id); - - if !is_joined.await && is_invited.await { - return None; - } - - Some(user_id.to_owned()) - }) - .collect::>() - .await; - - Ok::<_, Error>(( - Some(joined_member_count), - Some(invited_member_count), - heroes.into_iter().collect::>(), - )) - }; - - let get_sender_member_content = |short| { - services - .rooms - .state_accessor - .state_get_content(short, &StateEventType::RoomMember, sender_user.as_str()) - .ok() - }; - - let since_sender_member: OptionFuture<_> = since_shortstatehash.map(get_sender_member_content).into(); - - let joined_since_last_sync = since_sender_member - .await - .flatten() - .map_or(true, |content: RoomMemberEventContent| { - content.membership != MembershipState::Join - }); - - if since_shortstatehash.is_none() || joined_since_last_sync { - // Probably since = 0, we will do an initial sync - - let (joined_member_count, invited_member_count, heroes) = calculate_counts().await?; - - let current_state_ids: HashMap<_, OwnedEventId> = services - .rooms - .state_accessor - .state_full_ids(current_shortstatehash) - .await?; - - let mut state_events = Vec::new(); - let mut lazy_loaded = HashSet::new(); - - for (shortstatekey, event_id) in current_state_ids { - let (event_type, state_key) = services - .rooms - .short - .get_statekey_from_short(shortstatekey) - .await?; - - if event_type != StateEventType::RoomMember { - let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { - error!("Pdu in state not found: {event_id}"); - continue; - }; - - state_events.push(pdu); - continue; - } - - // TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565 - if !lazy_load_enabled - || full_state || timeline_users.contains(&state_key) - || (cfg!(feature = "element_hacks") && *sender_user == state_key) - { - let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { - error!("Pdu in state not found: {event_id}"); - continue; - }; - - // This check is in case a bad user ID made it into the database - if let Ok(uid) = UserId::parse(&state_key) { - lazy_loaded.insert(uid); - } - - state_events.push(pdu); - } - } - - // Reset lazy loading because this is an initial sync - services - .rooms - .lazy_loading - .lazy_load_reset(sender_user, sender_device, room_id) - .await; - - // The state_events above should contain all timeline_users, let's mark them as - // lazy loaded. - services.rooms.lazy_loading.lazy_load_mark_sent( - sender_user, - sender_device, - room_id, - lazy_loaded, - next_batchcount, - ); - - (heroes, joined_member_count, invited_member_count, true, state_events) - } else { - // Incremental /sync - let since_shortstatehash = since_shortstatehash.expect("missing since_shortstatehash on incremental sync"); - - let mut delta_state_events = Vec::new(); - - if since_shortstatehash != current_shortstatehash { - let current_state_ids: HashMap<_, OwnedEventId> = services - .rooms - .state_accessor - .state_full_ids(current_shortstatehash) - .await?; - - let since_state_ids = services - .rooms - .state_accessor - .state_full_ids(since_shortstatehash) - .await?; - - for (key, id) in current_state_ids { - if full_state || since_state_ids.get(&key) != Some(&id) { - let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { - error!("Pdu in state not found: {id}"); - continue; - }; - - delta_state_events.push(pdu); - } - } - } - - let encrypted_room = services - .rooms - .state_accessor - .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") - .await - .is_ok(); - - let since_encryption = services - .rooms - .state_accessor - .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") - .await; - - // Calculations: - let new_encrypted_room = encrypted_room && since_encryption.is_err(); - - let send_member_count = delta_state_events - .iter() - .any(|event| event.kind == RoomMember); - - if encrypted_room { - for state_event in &delta_state_events { - if state_event.kind != RoomMember { - continue; - } - - if let Some(state_key) = &state_event.state_key { - let user_id = UserId::parse(state_key.clone()) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; - - if user_id == sender_user { - continue; - } - - let content: RoomMemberEventContent = state_event.get_content()?; - - match content.membership { - MembershipState::Join => { - // A new user joined an encrypted room - if !share_encrypted_room(services, sender_user, &user_id, Some(room_id)).await { - device_list_updates.insert(user_id); - } - }, - MembershipState::Leave => { - // Write down users that have left encrypted rooms we are in - left_encrypted_users.insert(user_id); - }, - _ => {}, - } - } - } - } - - if joined_since_last_sync && encrypted_room || new_encrypted_room { - // If the user is in a new encrypted room, give them all joined users - device_list_updates.extend( - services - .rooms - .state_cache - .room_members(room_id) - // Don't send key updates from the sender to the sender - .ready_filter(|user_id| sender_user != *user_id) - // Only send keys if the sender doesn't share an encrypted room with the target - // already - .filter_map(|user_id| { - share_encrypted_room(services, sender_user, user_id, Some(room_id)) - .map(|res| res.or_some(user_id.to_owned())) - }) - .collect::>() - .await, - ); - } - - let (joined_member_count, invited_member_count, heroes) = if send_member_count { - calculate_counts().await? - } else { - (None, None, Vec::new()) - }; - - let mut state_events = delta_state_events; - let mut lazy_loaded = HashSet::new(); - - // Mark all member events we're returning as lazy-loaded - for pdu in &state_events { - if pdu.kind == RoomMember { - match UserId::parse( - pdu.state_key - .as_ref() - .expect("State event has state key") - .clone(), - ) { - Ok(state_key_userid) => { - lazy_loaded.insert(state_key_userid); - }, - Err(e) => error!("Invalid state key for member event: {}", e), - } - } - } - - // Fetch contextual member state events for events from the timeline, and - // mark them as lazy-loaded as well. - for (_, event) in &timeline_pdus { - if lazy_loaded.contains(&event.sender) { - continue; - } - - if !services - .rooms - .lazy_loading - .lazy_load_was_sent_before(sender_user, sender_device, room_id, &event.sender) - .await || lazy_load_send_redundant - { - if let Ok(member_event) = services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomMember, event.sender.as_str()) - .await - { - lazy_loaded.insert(event.sender.clone()); - state_events.push(member_event); - } - } - } - - services.rooms.lazy_loading.lazy_load_mark_sent( - sender_user, - sender_device, - room_id, - lazy_loaded, - next_batchcount, - ); - - ( - heroes, - joined_member_count, - invited_member_count, - joined_since_last_sync, - state_events, - ) - } - }; - - // Look for device list updates in this room - device_list_updates.extend( - services - .users - .room_keys_changed(room_id, since, None) - .map(|(user_id, _)| user_id) - .map(ToOwned::to_owned) - .collect::>() - .await, - ); - - let notification_count = if send_notification_counts { - Some( - services - .rooms - .user - .notification_count(sender_user, room_id) - .await - .try_into() - .expect("notification count can't go that high"), + calculate_state_changes( + services, + sender_user, + sender_device, + &room_id, + next_batchcount, + lazy_load_enabled, + lazy_load_send_redundant, + full_state, + &mut device_list_updates, + &mut left_encrypted_users, + since_shortstatehash, + current_shortstatehash, + &timeline_pdus, + &timeline_users, ) - } else { - None - }; - - let highlight_count = if send_notification_counts { - Some( - services - .rooms - .user - .highlight_count(sender_user, room_id) - .await - .try_into() - .expect("highlight count can't go that high"), - ) - } else { - None + .boxed() + .await? }; let prev_batch = timeline_pdus @@ -946,18 +629,45 @@ async fn load_joined_room( .as_ref() .map(ToString::to_string); - let room_events: Vec<_> = timeline_pdus + let notification_count: OptionFuture<_> = send_notification_counts + .then(|| { + services + .rooms + .user + .notification_count(sender_user, &room_id) + .map(TryInto::try_into) + .unwrap_or(uint!(0)) + }) + .into(); + + let highlight_count: OptionFuture<_> = send_notification_counts + .then(|| { + services + .rooms + .user + .highlight_count(sender_user, &room_id) + .map(TryInto::try_into) + .unwrap_or(uint!(0)) + }) + .into(); + + let room_events = timeline_pdus .iter() .stream() .filter_map(|item| ignored_filter(services, item.clone(), sender_user)) .map(|(_, pdu)| pdu.to_sync_room_event()) - .collect() - .await; + .collect(); - let edus: HashMap> = services + let account_data_events = services + .account_data + .changes_since(Some(&room_id), sender_user, since) + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect(); + + let receipt_events = services .rooms .read_receipt - .readreceipts_since(room_id, since) + .readreceipts_since(&room_id, since) .filter_map(|(read_user, _, edu)| async move { services .users @@ -965,19 +675,35 @@ async fn load_joined_room( .await .or_some((read_user.to_owned(), edu)) }) - .collect() + .collect::>>(); + + // Look for device list updates in this room + let device_updates = services + .users + .room_keys_changed(&room_id, since, None) + .map(|(user_id, _)| user_id) + .map(ToOwned::to_owned) + .collect::>(); + + let events = join3(room_events, account_data_events, receipt_events); + let unread_notifications = join(notification_count, highlight_count); + let (unread_notifications, events, device_updates) = join3(unread_notifications, events, device_updates) + .boxed() .await; - let mut edus: Vec> = edus.into_values().collect(); + let (room_events, account_data_events, receipt_events) = events; + let (notification_count, highlight_count) = unread_notifications; + device_list_updates.extend(device_updates); - if services.rooms.typing.last_typing_update(room_id).await? > since { + let mut edus: Vec> = receipt_events.into_values().collect(); + if services.rooms.typing.last_typing_update(&room_id).await? > since { edus.push( serde_json::from_str( &serde_json::to_string( &services .rooms .typing - .typings_all(room_id, sender_user) + .typings_all(&room_id, sender_user) .await?, ) .expect("event is valid, we just created it"), @@ -991,22 +717,22 @@ async fn load_joined_room( services .rooms .user - .associate_token_shortstatehash(room_id, next_batch, current_shortstatehash) + .associate_token_shortstatehash(&room_id, next_batch, current_shortstatehash) .await; - Ok(JoinedRoom { + let joined_room = JoinedRoom { account_data: RoomAccountData { - events: services - .account_data - .changes_since(Some(room_id), sender_user, since) - .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) - .collect() - .await, + events: account_data_events, }, summary: RoomSummary { - heroes, joined_member_count: joined_member_count.map(ruma_from_u64), invited_member_count: invited_member_count.map(ruma_from_u64), + heroes: heroes + .into_iter() + .flatten() + .map(TryInto::try_into) + .filter_map(Result::ok) + .collect(), }, unread_notifications: UnreadNotificationsCount { highlight_count, @@ -1027,5 +753,407 @@ async fn load_joined_room( events: edus, }, unread_thread_notifications: BTreeMap::new(), + }; + + Ok((joined_room, device_list_updates, left_encrypted_users)) +} + +#[allow(clippy::too_many_arguments)] +async fn calculate_state_changes( + services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, next_batchcount: PduCount, + lazy_load_enabled: bool, lazy_load_send_redundant: bool, full_state: bool, + device_list_updates: &mut HashSet, left_encrypted_users: &mut HashSet, + since_shortstatehash: Option, current_shortstatehash: ShortStateHash, + timeline_pdus: &Vec<(PduCount, PduEvent)>, timeline_users: &HashSet, +) -> Result { + let since_sender_member: OptionFuture<_> = since_shortstatehash + .map(|short| { + services + .rooms + .state_accessor + .state_get_content(short, &StateEventType::RoomMember, sender_user.as_str()) + .ok() + }) + .into(); + + let joined_since_last_sync = since_sender_member + .await + .flatten() + .map_or(true, |content: RoomMemberEventContent| { + content.membership != MembershipState::Join + }); + + if since_shortstatehash.is_none() || joined_since_last_sync { + calculate_state_initial( + services, + sender_user, + sender_device, + room_id, + next_batchcount, + lazy_load_enabled, + full_state, + current_shortstatehash, + timeline_users, + ) + .await + } else { + calculate_state_incremental( + services, + sender_user, + sender_device, + room_id, + next_batchcount, + lazy_load_send_redundant, + full_state, + device_list_updates, + left_encrypted_users, + since_shortstatehash, + current_shortstatehash, + timeline_pdus, + joined_since_last_sync, + ) + .await + } +} + +#[allow(clippy::too_many_arguments)] +async fn calculate_state_initial( + services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, next_batchcount: PduCount, + lazy_load_enabled: bool, full_state: bool, current_shortstatehash: ShortStateHash, + timeline_users: &HashSet, +) -> Result { + // Probably since = 0, we will do an initial sync + let state = services + .rooms + .state_accessor + .state_full_ids(current_shortstatehash) + .await? + .into_iter() + .stream() + .filter_map(|(shortstatekey, event_id): (ShortStateKey, OwnedEventId)| { + services + .rooms + .short + .get_statekey_from_short(shortstatekey) + .map_ok(move |(event_type, state_key)| ((event_type, state_key), event_id)) + .ok() + }) + .fold((Vec::new(), HashSet::new()), |a, item| async move { + let (mut state_events, mut lazy_loaded) = a; + let ((event_type, state_key), event_id) = item; + + if event_type != StateEventType::RoomMember { + let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { + error!("Pdu in state not found: {event_id}"); + return (state_events, lazy_loaded); + }; + + state_events.push(pdu); + return (state_events, lazy_loaded); + } + + // TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565 + if !lazy_load_enabled + || full_state + || timeline_users.contains(&state_key) + || (cfg!(feature = "element_hacks") && *sender_user == state_key) + { + let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { + error!("Pdu in state not found: {event_id}"); + return (state_events, lazy_loaded); + }; + + // This check is in case a bad user ID made it into the database + if let Ok(uid) = UserId::parse(&state_key) { + lazy_loaded.insert(uid); + } + + state_events.push(pdu); + } + + (state_events, lazy_loaded) + }) + .map(Ok); + + let counts = calculate_counts(services, room_id, sender_user); + let ((joined_member_count, invited_member_count, heroes), (state_events, lazy_loaded)) = + try_join(counts, state).boxed().await?; + + // Reset lazy loading because this is an initial sync + services + .rooms + .lazy_loading + .lazy_load_reset(sender_user, sender_device, room_id) + .await; + + // The state_events above should contain all timeline_users, let's mark them as + // lazy loaded. + services + .rooms + .lazy_loading + .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy_loaded, next_batchcount); + + Ok(StateChanges { + heroes, + joined_member_count, + invited_member_count, + joined_since_last_sync: true, + state_events, }) } + +#[allow(clippy::too_many_arguments)] +async fn calculate_state_incremental( + services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, next_batchcount: PduCount, + lazy_load_send_redundant: bool, full_state: bool, device_list_updates: &mut HashSet, + left_encrypted_users: &mut HashSet, since_shortstatehash: Option, + current_shortstatehash: ShortStateHash, timeline_pdus: &Vec<(PduCount, PduEvent)>, joined_since_last_sync: bool, +) -> Result { + // Incremental /sync + let since_shortstatehash = since_shortstatehash.expect("missing since_shortstatehash on incremental sync"); + + let mut delta_state_events = Vec::new(); + + if since_shortstatehash != current_shortstatehash { + let current_state_ids = services + .rooms + .state_accessor + .state_full_ids(current_shortstatehash); + + let since_state_ids = services + .rooms + .state_accessor + .state_full_ids(since_shortstatehash); + + let (current_state_ids, since_state_ids): (HashMap<_, OwnedEventId>, HashMap<_, OwnedEventId>) = + try_join(current_state_ids, since_state_ids).await?; + + current_state_ids + .iter() + .stream() + .ready_filter(|(key, id)| full_state || since_state_ids.get(key) != Some(id)) + .filter_map(|(_, id)| services.rooms.timeline.get_pdu(id).ok()) + .ready_for_each(|pdu| delta_state_events.push(pdu)) + .await; + } + + let encrypted_room = services + .rooms + .state_accessor + .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") + .is_ok(); + + let since_encryption = services + .rooms + .state_accessor + .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") + .is_ok(); + + let (encrypted_room, since_encryption) = join(encrypted_room, since_encryption).await; + + // Calculations: + let new_encrypted_room = encrypted_room && !since_encryption; + + let send_member_count = delta_state_events + .iter() + .any(|event| event.kind == RoomMember); + + if encrypted_room { + for state_event in &delta_state_events { + if state_event.kind != RoomMember { + continue; + } + + if let Some(state_key) = &state_event.state_key { + let user_id = UserId::parse(state_key.clone()) + .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; + + if user_id == sender_user { + continue; + } + + let content: RoomMemberEventContent = state_event.get_content()?; + + match content.membership { + MembershipState::Join => { + // A new user joined an encrypted room + if !share_encrypted_room(services, sender_user, &user_id, Some(room_id)).await { + device_list_updates.insert(user_id); + } + }, + MembershipState::Leave => { + // Write down users that have left encrypted rooms we are in + left_encrypted_users.insert(user_id); + }, + _ => {}, + } + } + } + } + + if joined_since_last_sync && encrypted_room || new_encrypted_room { + let updates: Vec = services + .rooms + .state_cache + .room_members(room_id) + .ready_filter(|user_id| sender_user != *user_id) + .filter_map(|user_id| { + share_encrypted_room(services, sender_user, user_id, Some(room_id)) + .map(|res| res.or_some(user_id.to_owned())) + }) + .collect() + .await; + + // If the user is in a new encrypted room, give them all joined users + device_list_updates.extend(updates); + } + + let (joined_member_count, invited_member_count, heroes) = if send_member_count { + calculate_counts(services, room_id, sender_user).await? + } else { + (None, None, None) + }; + + let mut state_events = delta_state_events; + + // Mark all member events we're returning as lazy-loaded + let mut lazy_loaded = state_events + .iter() + .filter(|pdu| pdu.kind == RoomMember) + .filter_map(|pdu| { + pdu.state_key + .clone() + .map(TryInto::try_into) + .map(LogDebugErr::log_debug_err) + .flat_ok() + }) + .fold(HashSet::new(), |mut lazy_loaded, user_id| { + lazy_loaded.insert(user_id); + lazy_loaded + }); + + // Fetch contextual member state events for events from the timeline, and + // mark them as lazy-loaded as well. + for (_, event) in timeline_pdus { + if lazy_loaded.contains(&event.sender) { + continue; + } + + let sent_before: OptionFuture<_> = (!lazy_load_send_redundant) + .then(|| { + services.rooms.lazy_loading.lazy_load_was_sent_before( + sender_user, + sender_device, + room_id, + &event.sender, + ) + }) + .into(); + + let member_event: OptionFuture<_> = sent_before + .await + .is_none_or(is_false!()) + .then(|| { + services.rooms.state_accessor.room_state_get( + room_id, + &StateEventType::RoomMember, + event.sender.as_str(), + ) + }) + .into(); + + let Some(Ok(member_event)) = member_event.await else { + continue; + }; + + lazy_loaded.insert(event.sender.clone()); + state_events.push(member_event); + } + + services + .rooms + .lazy_loading + .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy_loaded, next_batchcount); + + Ok(StateChanges { + heroes, + joined_member_count, + invited_member_count, + joined_since_last_sync, + state_events, + }) +} + +async fn calculate_counts( + services: &Services, room_id: &RoomId, sender_user: &UserId, +) -> Result<(Option, Option, Option>)> { + let joined_member_count = services + .rooms + .state_cache + .room_joined_count(room_id) + .unwrap_or(0); + + let invited_member_count = services + .rooms + .state_cache + .room_invited_count(room_id) + .unwrap_or(0); + + let (joined_member_count, invited_member_count) = join(joined_member_count, invited_member_count).await; + + let small_room = joined_member_count.saturating_add(invited_member_count) > 5; + + let heroes: OptionFuture<_> = small_room + .then(|| calculate_heroes(services, room_id, sender_user)) + .into(); + + Ok((Some(joined_member_count), Some(invited_member_count), heroes.await)) +} + +async fn calculate_heroes(services: &Services, room_id: &RoomId, sender_user: &UserId) -> Vec { + services + .rooms + .timeline + .all_pdus(sender_user, room_id) + .ready_filter(|(_, pdu)| pdu.kind == RoomMember) + .fold_default(|heroes: Vec<_>, (_, pdu)| fold_hero(heroes, services, room_id, sender_user, pdu)) + .await +} + +async fn fold_hero( + mut heroes: Vec, services: &Services, room_id: &RoomId, sender_user: &UserId, pdu: PduEvent, +) -> Vec { + let Some(user_id): Option<&UserId> = pdu.state_key.as_deref().map(TryInto::try_into).flat_ok() else { + return heroes; + }; + + if user_id == sender_user { + return heroes; + } + + let Ok(content): Result = pdu.get_content() else { + return heroes; + }; + + // The membership was and still is invite or join + if !matches!(content.membership, MembershipState::Join | MembershipState::Invite) { + return heroes; + } + + if heroes.iter().any(is_equal_to!(user_id)) { + return heroes; + } + + let (is_invited, is_joined) = join( + services.rooms.state_cache.is_invited(user_id, room_id), + services.rooms.state_cache.is_joined(user_id, room_id), + ) + .await; + + if !is_joined && is_invited { + return heroes; + } + + heroes.push(user_id.to_owned()); + heroes +} From da984d49cf1b3c4942662d325e04b40d8ef6d932 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 7 Dec 2024 04:42:35 +0000 Subject: [PATCH 1177/2091] remove unnecessary tracing of pdu conversions Signed-off-by: Jason Volk --- src/core/pdu/redact.rs | 7 +++---- src/core/pdu/strip.rs | 22 +++++++++++----------- src/service/rooms/timeline/mod.rs | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/core/pdu/redact.rs b/src/core/pdu/redact.rs index e116e563..01d9147c 100644 --- a/src/core/pdu/redact.rs +++ b/src/core/pdu/redact.rs @@ -11,7 +11,7 @@ use serde_json::{ value::{to_raw_value, RawValue as RawJsonValue}, }; -use crate::{implement, warn, Error, Result}; +use crate::{implement, Error, Result}; #[derive(Deserialize)] struct ExtractRedactedBecause { @@ -19,14 +19,13 @@ struct ExtractRedactedBecause { } #[implement(super::Pdu)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn redact(&mut self, room_version_id: RoomVersionId, reason: &Self) -> Result { +pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: &Self) -> Result { self.unsigned = None; let mut content = serde_json::from_str(self.content.get()).map_err(|_| Error::bad_database("PDU in db has invalid content."))?; - redact_content_in_place(&mut content, &room_version_id, self.kind.to_string()) + redact_content_in_place(&mut content, room_version_id, self.kind.to_string()) .map_err(|e| Error::Redaction(self.sender.server_name().to_owned(), e))?; self.unsigned = Some( diff --git a/src/core/pdu/strip.rs b/src/core/pdu/strip.rs index 30fee863..59457749 100644 --- a/src/core/pdu/strip.rs +++ b/src/core/pdu/strip.rs @@ -8,10 +8,10 @@ use ruma::{ }; use serde_json::{json, value::Value as JsonValue}; -use crate::{implement, warn}; +use crate::implement; +#[must_use] #[implement(super::Pdu)] -#[tracing::instrument(skip(self), level = "debug")] pub fn to_sync_room_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -36,8 +36,8 @@ pub fn to_sync_room_event(&self) -> Raw { } /// This only works for events that are also AnyRoomEvents. +#[must_use] #[implement(super::Pdu)] -#[tracing::instrument(skip(self), level = "debug")] pub fn to_any_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -62,8 +62,8 @@ pub fn to_any_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } +#[must_use] #[implement(super::Pdu)] -#[tracing::instrument(skip(self), level = "debug")] pub fn to_room_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -88,8 +88,8 @@ pub fn to_room_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } +#[must_use] #[implement(super::Pdu)] -#[tracing::instrument(skip(self), level = "debug")] pub fn to_message_like_event(&self) -> Raw { let (redacts, content) = self.copy_redacts(); let mut json = json!({ @@ -114,8 +114,8 @@ pub fn to_message_like_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } -#[implement(super::Pdu)] #[must_use] +#[implement(super::Pdu)] pub fn to_state_event_value(&self) -> JsonValue { let mut json = json!({ "content": self.content, @@ -134,14 +134,14 @@ pub fn to_state_event_value(&self) -> JsonValue { json } +#[must_use] #[implement(super::Pdu)] -#[tracing::instrument(skip(self), level = "debug")] pub fn to_state_event(&self) -> Raw { serde_json::from_value(self.to_state_event_value()).expect("Raw::from_value always works") } +#[must_use] #[implement(super::Pdu)] -#[tracing::instrument(skip(self), level = "debug")] pub fn to_sync_state_event(&self) -> Raw { let mut json = json!({ "content": self.content, @@ -159,8 +159,8 @@ pub fn to_sync_state_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } +#[must_use] #[implement(super::Pdu)] -#[tracing::instrument(skip(self), level = "debug")] pub fn to_stripped_state_event(&self) -> Raw { let json = json!({ "content": self.content, @@ -172,8 +172,8 @@ pub fn to_stripped_state_event(&self) -> Raw { serde_json::from_value(json).expect("Raw::from_value always works") } +#[must_use] #[implement(super::Pdu)] -#[tracing::instrument(skip(self), level = "debug")] pub fn to_stripped_spacechild_state_event(&self) -> Raw { let json = json!({ "content": self.content, @@ -186,8 +186,8 @@ pub fn to_stripped_spacechild_state_event(&self) -> Raw Raw> { let mut json = json!({ "content": self.content, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 0a96322b..b9fcdcd2 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1024,7 +1024,7 @@ impl Service { let room_version_id = self.services.state.get_room_version(&pdu.room_id).await?; - pdu.redact(room_version_id, reason)?; + pdu.redact(&room_version_id, reason)?; let obj = utils::to_canonical_object(&pdu) .map_err(|e| err!(Database(error!(?event_id, ?e, "Failed to convert PDU to canonical JSON"))))?; From d921b823764330959179e3dd0fe9a0e56437a4a4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 7 Dec 2024 06:02:33 +0000 Subject: [PATCH 1178/2091] add options for console tracing EnvFilter details Signed-off-by: Jason Volk --- conduwuit-example.toml | 9 +++++++++ src/core/config/mod.rs | 13 +++++++++++++ src/main/tracing.rs | 8 ++++++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index bdccdb99..446375b8 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -604,6 +604,15 @@ # #log_span_events = "none" +# configures whether CONDUWUIT_LOG EnvFilter matches values using regular +# expressions. See the tracing_subscriber documentation on Directives. +# +#log_filter_regex = true + +# toggles the display of ThreadId in tracing log output +# +#log_thread_ids = false + # OpenID token expiration/TTL in seconds # # These are the OpenID tokens that are primarily used for Matrix account diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 117b4da5..0e25b197 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -698,6 +698,19 @@ pub struct Config { #[serde(default = "default_log_span_events")] pub log_span_events: String, + /// configures whether CONDUWUIT_LOG EnvFilter matches values using regular + /// expressions. See the tracing_subscriber documentation on Directives. + /// + /// default: true + #[serde(default = "true_fn")] + pub log_filter_regex: bool, + + /// toggles the display of ThreadId in tracing log output + /// + /// default: false + #[serde(default)] + pub log_thread_ids: bool, + /// OpenID token expiration/TTL in seconds /// /// These are the OpenID tokens that are primarily used for Matrix account diff --git a/src/main/tracing.rs b/src/main/tracing.rs index c28fef6b..efa47bab 100644 --- a/src/main/tracing.rs +++ b/src/main/tracing.rs @@ -18,11 +18,15 @@ pub(crate) type TracingFlameGuard = (); pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFlameGuard, Arc)> { let reload_handles = LogLevelReloadHandles::default(); - let console_filter = EnvFilter::try_new(&config.log).map_err(|e| err!(Config("log", "{e}.")))?; let console_span_events = fmt_span::from_str(&config.log_span_events).unwrap_or_err(); + let console_filter = EnvFilter::builder() + .with_regex(config.log_filter_regex) + .parse(&config.log) + .map_err(|e| err!(Config("log", "{e}.")))?; let console_layer = tracing_subscriber::fmt::Layer::new() .with_ansi(config.log_colors) - .with_span_events(console_span_events); + .with_span_events(console_span_events) + .with_thread_ids(config.log_thread_ids); let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); reload_handles.add("console", Box::new(console_reload_handle)); From aa6d0fcaa7f5f5ca472b0c3eda09e3c7278b6fb6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 7 Dec 2024 06:02:59 +0000 Subject: [PATCH 1179/2091] add unwrap_or_default() to future TryExt extension start an OptionFuture extension Signed-off-by: Jason Volk --- src/core/utils/future/mod.rs | 2 ++ src/core/utils/future/option_ext.rs | 22 ++++++++++++++++++++++ src/core/utils/future/try_ext_ext.rs | 14 ++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 src/core/utils/future/option_ext.rs diff --git a/src/core/utils/future/mod.rs b/src/core/utils/future/mod.rs index 6d45b656..3d8ec8f4 100644 --- a/src/core/utils/future/mod.rs +++ b/src/core/utils/future/mod.rs @@ -1,3 +1,5 @@ +mod option_ext; mod try_ext_ext; +pub use option_ext::OptionExt; pub use try_ext_ext::TryExtExt; diff --git a/src/core/utils/future/option_ext.rs b/src/core/utils/future/option_ext.rs new file mode 100644 index 00000000..ed61de56 --- /dev/null +++ b/src/core/utils/future/option_ext.rs @@ -0,0 +1,22 @@ +#![allow(clippy::wrong_self_convention)] + +use futures::{future::OptionFuture, Future, FutureExt}; + +pub trait OptionExt { + fn is_none_or(self, f: impl FnOnce(&T) -> bool + Send) -> impl Future + Send; + + fn is_some_and(self, f: impl FnOnce(&T) -> bool + Send) -> impl Future + Send; +} + +impl OptionExt for OptionFuture +where + Fut: Future + Send, +{ + fn is_none_or(self, f: impl FnOnce(&T) -> bool + Send) -> impl Future + Send { + self.map(|o| o.as_ref().is_none_or(f)) + } + + fn is_some_and(self, f: impl FnOnce(&T) -> bool + Send) -> impl Future + Send { + self.map(|o| o.as_ref().is_some_and(f)) + } +} diff --git a/src/core/utils/future/try_ext_ext.rs b/src/core/utils/future/try_ext_ext.rs index 81c7aac0..977f74d2 100644 --- a/src/core/utils/future/try_ext_ext.rs +++ b/src/core/utils/future/try_ext_ext.rs @@ -39,6 +39,11 @@ where fn unwrap_or(self, default: Self::Ok) -> UnwrapOrElse Self::Ok> where Self: Sized; + + fn unwrap_or_default(self) -> UnwrapOrElse Self::Ok> + where + Self: Sized, + Self::Ok: Default; } impl TryExtExt for Fut @@ -89,4 +94,13 @@ where { self.unwrap_or_else(move |_| default) } + + #[inline] + fn unwrap_or_default(self) -> UnwrapOrElse Self::Ok> + where + Self: Sized, + Self::Ok: Default, + { + self.unwrap_or(Default::default()) + } } From 65a370836c5d824734a7d8b6229215fdd7b414ad Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 03:02:28 +0000 Subject: [PATCH 1180/2091] derive specific ReadOptions for iterators Signed-off-by: Jason Volk --- src/database/map.rs | 12 +++++++++--- src/database/map/keys.rs | 2 +- src/database/map/keys_from.rs | 2 +- src/database/map/rev_keys.rs | 2 +- src/database/map/rev_keys_from.rs | 2 +- src/database/map/rev_stream.rs | 2 +- src/database/map/rev_stream_from.rs | 2 +- src/database/map/stream.rs | 2 +- 8 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/database/map.rs b/src/database/map.rs index a15d5e9d..d9601599 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -116,18 +116,24 @@ fn open(db: &Arc, name: &str) -> Result> { } #[inline] -fn cache_read_options_default() -> ReadOptions { +pub(crate) fn iter_options_default() -> ReadOptions { + let mut read_options = read_options_default(); + read_options +} + +#[inline] +pub(crate) fn cache_read_options_default() -> ReadOptions { let mut read_options = read_options_default(); read_options.set_read_tier(ReadTier::BlockCache); read_options } #[inline] -fn read_options_default() -> ReadOptions { +pub(crate) fn read_options_default() -> ReadOptions { let mut read_options = ReadOptions::default(); read_options.set_total_order_seek(true); read_options } #[inline] -fn write_options_default() -> WriteOptions { WriteOptions::default() } +pub(crate) fn write_options_default() -> WriteOptions { WriteOptions::default() } diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs index 80cf1e15..3c4d491b 100644 --- a/src/database/map/keys.rs +++ b/src/database/map/keys.rs @@ -15,6 +15,6 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn raw_keys(&self) -> impl Stream>> + Send { - let opts = super::read_options_default(); + let opts = super::iter_options_default(); stream::Keys::new(&self.db, &self.cf, opts).init(None) } diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 7be3dd1d..40c608f2 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -46,7 +46,7 @@ where { use crate::pool::Seek; - let opts = super::read_options_default(); + let opts = super::iter_options_default(); let state = stream::State::new(&self.db, &self.cf, opts); let seek = Seek { map: self.clone(), diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index 45a0203f..1e000a47 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -15,6 +15,6 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_raw_keys(&self) -> impl Stream>> + Send { - let opts = super::read_options_default(); + let opts = super::iter_options_default(); stream::KeysRev::new(&self.db, &self.cf, opts).init(None) } diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index 2b59a5d7..b5565aa4 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -48,7 +48,7 @@ where { use crate::pool::Seek; - let opts = super::read_options_default(); + let opts = super::iter_options_default(); let state = stream::State::new(&self.db, &self.cf, opts); let seek = Seek { map: self.clone(), diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index 37b0d3b3..5f61cb08 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -23,6 +23,6 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn rev_raw_stream(&self) -> impl Stream>> + Send { - let opts = super::read_options_default(); + let opts = super::iter_options_default(); stream::ItemsRev::new(&self.db, &self.cf, opts).init(None) } diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 9811d106..542a5ba8 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -73,7 +73,7 @@ where { use crate::pool::Seek; - let opts = super::read_options_default(); + let opts = super::iter_options_default(); let state = stream::State::new(&self.db, &self.cf, opts); let seek = Seek { map: self.clone(), diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index 4f4fbd08..67cfff1b 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -22,6 +22,6 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] pub fn raw_stream(&self) -> impl Stream>> + Send { - let opts = super::read_options_default(); + let opts = super::iter_options_default(); stream::Items::new(&self.db, &self.cf, opts).init(None) } From 139a1ac5048aa6e18dae75ae8ec035f7e6572f11 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 03:07:08 +0000 Subject: [PATCH 1181/2091] configure the tokio runtime tunables Signed-off-by: Jason Volk --- src/main/main.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/main.rs b/src/main/main.rs index 32d122f6..67fb643f 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -20,6 +20,9 @@ use tokio::runtime; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; const WORKER_KEEPALIVE: u64 = 36; +const GLOBAL_QUEUE_INTERVAL: u32 = 192; +const SYSTEM_QUEUE_INTERVAL: u32 = 256; +const SYSTEM_EVENTS_PER_TICK: usize = 512; rustc_flags_capture! {} @@ -31,6 +34,9 @@ fn main() -> Result<(), Error> { .thread_name(WORKER_NAME) .worker_threads(args.worker_threads.max(WORKER_MIN)) .thread_keep_alive(Duration::from_secs(WORKER_KEEPALIVE)) + .global_queue_interval(GLOBAL_QUEUE_INTERVAL) + .event_interval(SYSTEM_QUEUE_INTERVAL) + .max_io_events_per_tick(SYSTEM_EVENTS_PER_TICK) .build() .expect("built runtime"); From 6aa35260e688a5b5cffa28c5c755db6bb9deab1c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 03:03:34 +0000 Subject: [PATCH 1182/2091] misc database options tweaks Signed-off-by: Jason Volk --- deps/rust-rocksdb/lib.rs | 1 + src/database/map.rs | 2 ++ src/database/opts.rs | 20 +++++++++++++++----- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/deps/rust-rocksdb/lib.rs b/deps/rust-rocksdb/lib.rs index 2d53d4b1..2e699efb 100644 --- a/deps/rust-rocksdb/lib.rs +++ b/deps/rust-rocksdb/lib.rs @@ -5,6 +5,7 @@ pub use rust_rocksdb::*; extern "C" { pub fn rocksdb_list_column_families(); pub fn rocksdb_logger_create_stderr_logger(); + pub fn rocksdb_logger_create_callback_logger(); pub fn rocksdb_options_set_info_log(); pub fn rocksdb_get_options_from_string(); pub fn rocksdb_writebatch_create(); diff --git a/src/database/map.rs b/src/database/map.rs index d9601599..0f4d740a 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -118,6 +118,8 @@ fn open(db: &Arc, name: &str) -> Result> { #[inline] pub(crate) fn iter_options_default() -> ReadOptions { let mut read_options = read_options_default(); + read_options.set_background_purge_on_iterator_cleanup(true); + //read_options.set_pin_data(true); read_options } diff --git a/src/database/opts.rs b/src/database/opts.rs index 28a39cca..f1b4d3a9 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -3,7 +3,7 @@ use std::{cmp, collections::HashMap, convert::TryFrom}; use conduit::{err, utils, Config, Result}; use rocksdb::{ statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, - LogLevel, Options, UniversalCompactOptions, UniversalCompactionStopStyle, + LogLevel, LruCacheOptions, Options, UniversalCompactOptions, UniversalCompactionStopStyle, }; /// Create database-wide options suitable for opening the database. This also @@ -24,9 +24,9 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ set_logging_defaults(&mut opts, config); // Processing - opts.set_enable_pipelined_write(true); opts.set_max_background_jobs(num_threads::(config)?); opts.set_max_subcompactions(num_threads::(config)?); + opts.set_avoid_unnecessary_blocking_io(true); opts.set_max_file_opening_threads(0); if config.rocksdb_compaction_prio_idle { env.lower_thread_pool_cpu_priority(); @@ -34,6 +34,7 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ // IO opts.set_manual_wal_flush(true); + opts.set_enable_pipelined_write(true); if config.rocksdb_direct_io { opts.set_use_direct_reads(true); opts.set_use_direct_io_for_flush_and_compaction(true); @@ -60,6 +61,7 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ opts.set_min_write_buffer_number(1); // Files + opts.set_table_cache_num_shard_bits(7); opts.set_max_total_wal_size(96 * 1024 * 1024); set_level_defaults(&mut opts, config); @@ -328,10 +330,16 @@ fn uc_options(_config: &Config) -> UniversalCompactOptions { } fn set_table_with_new_cache( - opts: &mut Options, config: &Config, cache: &mut HashMap, name: &str, size: usize, + opts: &mut Options, config: &Config, caches: &mut HashMap, name: &str, size: usize, ) { - cache.insert(name.to_owned(), Cache::new_lru_cache(size)); - set_table_with_shared_cache(opts, config, cache, name, name); + let mut cache_opts = LruCacheOptions::default(); + cache_opts.set_capacity(size); + cache_opts.set_num_shard_bits(7); + + let cache = Cache::new_lru_cache_opts(&cache_opts); + caches.insert(name.into(), cache); + + set_table_with_shared_cache(opts, config, caches, name, name); } fn set_table_with_shared_cache( @@ -343,6 +351,7 @@ fn set_table_with_shared_cache( .get(cache_name) .expect("existing cache to share with this column"), ); + opts.set_block_based_table_factory(&table); } @@ -361,6 +370,7 @@ fn table_options(_config: &Config) -> BlockBasedOptions { opts.set_block_size(4 * 1024); opts.set_metadata_block_size(4 * 1024); + opts.set_use_delta_encoding(false); opts.set_optimize_filters_for_memory(true); opts.set_cache_index_and_filter_blocks(true); opts.set_pin_top_level_index_and_filter(true); From ff56ee7413f973e4f70feaf71a027d926ee42341 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 05:45:15 +0000 Subject: [PATCH 1183/2091] add back the consume_budget on cache hits Signed-off-by: Jason Volk --- src/database/map/get.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 04f5d0ae..79556656 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -2,8 +2,9 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; use arrayvec::ArrayVec; use conduit::{err, implement, utils::result::MapExpect, Err, Result}; -use futures::{future, Future, FutureExt}; +use futures::{Future, FutureExt}; use serde::Serialize; +use tokio::task; use crate::{ keyval::KeyBuf, @@ -63,7 +64,9 @@ where let cached = self.get_cached(key); if matches!(cached, Err(_) | Ok(Some(_))) { - return future::ready(cached.map_expect("data found in cache")).boxed(); + return task::consume_budget() + .map(move |()| cached.map_expect("data found in cache")) + .boxed(); } debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete"); From 82133ee2ea91723c60f46cbd2ec538c3c11e92e7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 05:49:51 +0000 Subject: [PATCH 1184/2091] bump rust-rocksdb Signed-off-by: Jason Volk --- Cargo.lock | 4 ++-- deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e6659e03..fc8dc26d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3391,7 +3391,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.29.0+9.7.4" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=4bce1bb97d8be6f0d47245c99d465ca9cef33aad#4bce1bb97d8be6f0d47245c99d465ca9cef33aad" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=e2b8853ee8839dc886e88fd751fe50ed1c27b47e#e2b8853ee8839dc886e88fd751fe50ed1c27b47e" dependencies = [ "bindgen", "bzip2-sys", @@ -3408,7 +3408,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.33.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=4bce1bb97d8be6f0d47245c99d465ca9cef33aad#4bce1bb97d8be6f0d47245c99d465ca9cef33aad" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=e2b8853ee8839dc886e88fd751fe50ed1c27b47e#e2b8853ee8839dc886e88fd751fe50ed1c27b47e" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 64a8dcb6..c6c3fea5 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "4bce1bb97d8be6f0d47245c99d465ca9cef33aad" +rev = "e2b8853ee8839dc886e88fd751fe50ed1c27b47e" #branch = "master" default-features = false From f70fdca828d0c1e14601418fe337fb8444a76776 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 20:32:28 +0000 Subject: [PATCH 1185/2091] remove jemalloc_stats from main default features Signed-off-by: Jason Volk --- src/main/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index b9122942..ee434ceb 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -42,7 +42,6 @@ default = [ "gzip_compression", "io_uring", "jemalloc", - "jemalloc_stats", "release_max_log_level", "systemd", "zstd_compression", From a752fb99430b32e4cf7c3e2bd0a4cff9e8773980 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 13:47:02 +0000 Subject: [PATCH 1186/2091] rename mod with tracing inits due to macro issues Signed-off-by: Jason Volk --- src/main/{tracing.rs => logging.rs} | 0 src/main/main.rs | 2 +- src/main/server.rs | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/main/{tracing.rs => logging.rs} (100%) diff --git a/src/main/tracing.rs b/src/main/logging.rs similarity index 100% rename from src/main/tracing.rs rename to src/main/logging.rs diff --git a/src/main/main.rs b/src/main/main.rs index 67fb643f..c653444c 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -1,10 +1,10 @@ pub(crate) mod clap; +mod logging; mod mods; mod restart; mod sentry; mod server; mod signal; -mod tracing; extern crate conduit_core as conduit; diff --git a/src/main/server.rs b/src/main/server.rs index 27fd0673..179749b5 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use conduit::{config::Config, info, log::Log, utils::sys, Error, Result}; use tokio::{runtime, sync::Mutex}; -use crate::{clap::Args, tracing::TracingFlameGuard}; +use crate::{clap::Args, logging::TracingFlameGuard}; /// Server runtime state; complete pub(crate) struct Server { @@ -33,7 +33,7 @@ impl Server { #[cfg(feature = "sentry_telemetry")] let sentry_guard = crate::sentry::init(&config); - let (tracing_reload_handle, tracing_flame_guard, capture) = crate::tracing::init(&config)?; + let (tracing_reload_handle, tracing_flame_guard, capture) = crate::logging::init(&config)?; config.check()?; From 35e9d9b02e5bcb4562e84636ee5bd4d4e768d746 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Dec 2024 00:44:51 -0500 Subject: [PATCH 1187/2091] fix duplicate timezone keys being sent on profile lookup requests Signed-off-by: strawberry --- src/api/client/profile.rs | 16 +++++++++++----- src/api/server/query.rs | 4 ++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 32f7a723..690d0ad7 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -273,16 +273,22 @@ pub(crate) async fn get_profile_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); } + let mut custom_profile_fields: BTreeMap = services + .users + .all_profile_keys(&body.user_id) + .collect() + .await; + + // services.users.timezone will collect the MSC4175 timezone key if it exists + custom_profile_fields.remove("us.cloke.msc4175.tz"); + custom_profile_fields.remove("m.tz"); + Ok(get_profile::v3::Response { avatar_url: services.users.avatar_url(&body.user_id).await.ok(), blurhash: services.users.blurhash(&body.user_id).await.ok(), displayname: services.users.displayname(&body.user_id).await.ok(), tz: services.users.timezone(&body.user_id).await.ok(), - custom_profile_fields: services - .users - .all_profile_keys(&body.user_id) - .collect() - .await, + custom_profile_fields, }) } diff --git a/src/api/server/query.rs b/src/api/server/query.rs index bf515b3c..edbeee77 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -117,6 +117,10 @@ pub(crate) async fn get_profile_information_route( }, } + // services.users.timezone will collect the MSC4175 timezone key if it exists + custom_profile_fields.remove("us.cloke.msc4175.tz"); + custom_profile_fields.remove("m.tz"); + Ok(get_profile_information::v1::Response { displayname, avatar_url, From c070edc18954a4a8cf6cde6ebe6f1dd6d91e68fe Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Dec 2024 00:46:22 -0500 Subject: [PATCH 1188/2091] fix profile updates reusing old membership content + small parallelise + remove unnecessary Result Signed-off-by: strawberry --- src/admin/user/commands.rs | 12 ++-- src/api/client/account.rs | 10 +-- src/api/client/profile.rs | 143 +++++++++++++++++++++---------------- 3 files changed, 89 insertions(+), 76 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 444a7f37..61096625 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -213,8 +213,8 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> .await; full_user_deactivate(self.services, &user_id, &all_joined_rooms).await?; - update_displayname(self.services, &user_id, None, &all_joined_rooms).await?; - update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms).await?; + update_displayname(self.services, &user_id, None, &all_joined_rooms).await; + update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms).await; leave_all_rooms(self.services, &user_id).await; } @@ -327,12 +327,8 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> .await; full_user_deactivate(self.services, &user_id, &all_joined_rooms).await?; - update_displayname(self.services, &user_id, None, &all_joined_rooms) - .await - .ok(); - update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms) - .await - .ok(); + update_displayname(self.services, &user_id, None, &all_joined_rooms).await; + update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms).await; leave_all_rooms(self.services, &user_id).await; } }, diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 47f6fec8..3595f581 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -648,8 +648,8 @@ pub(crate) async fn deactivate_route( .collect() .await; - super::update_displayname(&services, sender_user, None, &all_joined_rooms).await?; - super::update_avatar_url(&services, sender_user, None, None, &all_joined_rooms).await?; + super::update_displayname(&services, sender_user, None, &all_joined_rooms).await; + super::update_avatar_url(&services, sender_user, None, None, &all_joined_rooms).await; full_user_deactivate(&services, sender_user, &all_joined_rooms).await?; @@ -744,9 +744,9 @@ pub(crate) async fn check_registration_token_validity( pub async fn full_user_deactivate( services: &Services, user_id: &UserId, all_joined_rooms: &[OwnedRoomId], ) -> Result<()> { - services.users.deactivate_account(user_id).await?; - super::update_displayname(services, user_id, None, all_joined_rooms).await?; - super::update_avatar_url(services, user_id, None, None, all_joined_rooms).await?; + services.users.deactivate_account(user_id).await.ok(); + super::update_displayname(services, user_id, None, all_joined_rooms).await; + super::update_avatar_url(services, user_id, None, None, all_joined_rooms).await; services .users diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 690d0ad7..97ea21ea 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -1,10 +1,12 @@ +use std::collections::BTreeMap; + use axum::extract::State; use conduit::{ pdu::PduBuilder, utils::{stream::TryIgnore, IterStream}, warn, Err, Error, Result, }; -use futures::{StreamExt, TryStreamExt}; +use futures::{future::join3, StreamExt, TryStreamExt}; use ruma::{ api::{ client::{ @@ -13,7 +15,7 @@ use ruma::{ }, federation, }, - events::{room::member::RoomMemberEventContent, StateEventType}, + events::room::member::{MembershipState, RoomMemberEventContent}, presence::PresenceState, OwnedMxcUri, OwnedRoomId, UserId, }; @@ -43,7 +45,7 @@ pub(crate) async fn set_displayname_route( .collect() .await; - update_displayname(&services, &body.user_id, body.displayname.clone(), &all_joined_rooms).await?; + update_displayname(&services, &body.user_id, body.displayname.clone(), &all_joined_rooms).await; if services.globals.allow_local_presence() { // Presence update @@ -138,7 +140,7 @@ pub(crate) async fn set_avatar_url_route( body.blurhash.clone(), &all_joined_rooms, ) - .await?; + .await; if services.globals.allow_local_presence() { // Presence update @@ -294,79 +296,43 @@ pub(crate) async fn get_profile_route( pub async fn update_displayname( services: &Services, user_id: &UserId, displayname: Option, all_joined_rooms: &[OwnedRoomId], -) -> Result<()> { - let current_display_name = services.users.displayname(user_id).await.ok(); +) { + let (current_avatar_url, current_blurhash, current_displayname) = join3( + services.users.avatar_url(user_id), + services.users.blurhash(user_id), + services.users.displayname(user_id), + ) + .await; - if displayname == current_display_name { - return Ok(()); + let current_avatar_url = current_avatar_url.ok(); + let current_blurhash = current_blurhash.ok(); + let current_displayname = current_displayname.ok(); + + if displayname == current_displayname { + return; } services.users.set_displayname(user_id, displayname.clone()); // Send a new join membership event into all joined rooms - let mut joined_rooms = Vec::new(); - for room_id in all_joined_rooms { - let Ok(content) = services - .rooms - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomMember, user_id.as_str()) - .await - else { - continue; - }; - - let pdu = PduBuilder::state( - user_id.to_string(), - &RoomMemberEventContent { - displayname: displayname.clone(), - join_authorized_via_users_server: None, - ..content - }, - ); - - joined_rooms.push((pdu, room_id)); - } - - update_all_rooms(services, joined_rooms, user_id).await; - - Ok(()) -} - -pub async fn update_avatar_url( - services: &Services, user_id: &UserId, avatar_url: Option, blurhash: Option, - all_joined_rooms: &[OwnedRoomId], -) -> Result<()> { - let current_avatar_url = services.users.avatar_url(user_id).await.ok(); - let current_blurhash = services.users.blurhash(user_id).await.ok(); - - if current_avatar_url == avatar_url && current_blurhash == blurhash { - return Ok(()); - } - - services.users.set_avatar_url(user_id, avatar_url.clone()); - - services.users.set_blurhash(user_id, blurhash.clone()); - - // Send a new join membership event into all joined rooms - let avatar_url = &avatar_url; - let blurhash = &blurhash; + let avatar_url = ¤t_avatar_url; + let blurhash = ¤t_blurhash; + let displayname = &displayname; let all_joined_rooms: Vec<_> = all_joined_rooms .iter() .try_stream() .and_then(|room_id: &OwnedRoomId| async move { - let content = services - .rooms - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomMember, user_id.as_str()) - .await?; - let pdu = PduBuilder::state( user_id.to_string(), &RoomMemberEventContent { + displayname: displayname.clone(), + membership: MembershipState::Join, avatar_url: avatar_url.clone(), blurhash: blurhash.clone(), join_authorized_via_users_server: None, - ..content + reason: None, + is_direct: None, + third_party_invite: None, }, ); @@ -377,8 +343,59 @@ pub async fn update_avatar_url( .await; update_all_rooms(services, all_joined_rooms, user_id).await; +} - Ok(()) +pub async fn update_avatar_url( + services: &Services, user_id: &UserId, avatar_url: Option, blurhash: Option, + all_joined_rooms: &[OwnedRoomId], +) { + let (current_avatar_url, current_blurhash, current_displayname) = join3( + services.users.avatar_url(user_id), + services.users.blurhash(user_id), + services.users.displayname(user_id), + ) + .await; + + let current_avatar_url = current_avatar_url.ok(); + let current_blurhash = current_blurhash.ok(); + let current_displayname = current_displayname.ok(); + + if current_avatar_url == avatar_url && current_blurhash == blurhash { + return; + } + + services.users.set_avatar_url(user_id, avatar_url.clone()); + services.users.set_blurhash(user_id, blurhash.clone()); + + // Send a new join membership event into all joined rooms + let avatar_url = &avatar_url; + let blurhash = &blurhash; + let displayname = ¤t_displayname; + let all_joined_rooms: Vec<_> = all_joined_rooms + .iter() + .try_stream() + .and_then(|room_id: &OwnedRoomId| async move { + let pdu = PduBuilder::state( + user_id.to_string(), + &RoomMemberEventContent { + avatar_url: avatar_url.clone(), + blurhash: blurhash.clone(), + membership: MembershipState::Join, + displayname: displayname.clone(), + join_authorized_via_users_server: None, + reason: None, + is_direct: None, + third_party_invite: None, + }, + ); + + Ok((pdu, room_id)) + }) + .ignore_err() + .collect() + .await; + + update_all_rooms(services, all_joined_rooms, user_id).await; } pub async fn update_all_rooms( @@ -392,7 +409,7 @@ pub async fn update_all_rooms( .build_and_append_pdu(pdu_builder, user_id, room_id, &state_lock) .await { - warn!(%user_id, %room_id, %e, "Failed to update/send new profile join membership update in room"); + warn!(%user_id, %room_id, "Failed to update/send new profile join membership update in room: {e}"); } } } From a1b03690334442733bfb8dd0ebb8ef77f3efd1d7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Dec 2024 00:53:15 -0500 Subject: [PATCH 1189/2091] reduce line width on banned_room_check Signed-off-by: strawberry --- src/api/client/membership.rs | 146 ++++++++++++++++++----------------- 1 file changed, 74 insertions(+), 72 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 1cdf25a2..fa671213 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -56,87 +56,89 @@ use crate::{client::full_user_deactivate, Ruma}; async fn banned_room_check( services: &Services, user_id: &UserId, room_id: Option<&RoomId>, server_name: Option<&ServerName>, client_ip: IpAddr, -) -> Result<()> { - if !services.users.is_admin(user_id).await { - if let Some(room_id) = room_id { - if services.rooms.metadata.is_banned(room_id).await - || services - .globals - .config - .forbidden_remote_server_names - .contains(&room_id.server_name().unwrap().to_owned()) - { - warn!( - "User {user_id} who is not an admin attempted to send an invite for or attempted to join a banned \ - room or banned room server name: {room_id}" - ); +) -> Result { + if services.users.is_admin(user_id).await { + return Ok(()); + } - if services.globals.config.auto_deactivate_banned_room_attempts { - warn!("Automatically deactivating user {user_id} due to attempted banned room join"); - - if services.globals.config.admin_room_notices { - services - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "Automatically deactivating user {user_id} due to attempted banned room join from IP \ - {client_ip}" - ))) - .await - .ok(); - } - - let all_joined_rooms: Vec = services - .rooms - .state_cache - .rooms_joined(user_id) - .map(Into::into) - .collect() - .await; - - full_user_deactivate(services, user_id, &all_joined_rooms).await?; - } - - return Err!(Request(Forbidden("This room is banned on this homeserver."))); - } - } else if let Some(server_name) = server_name { - if services + if let Some(room_id) = room_id { + if services.rooms.metadata.is_banned(room_id).await + || services .globals .config .forbidden_remote_server_names - .contains(&server_name.to_owned()) - { - warn!( - "User {user_id} who is not an admin tried joining a room which has the server name {server_name} \ - that is globally forbidden. Rejecting.", - ); + .contains(&room_id.server_name().unwrap().to_owned()) + { + warn!( + "User {user_id} who is not an admin attempted to send an invite for or attempted to join a banned \ + room or banned room server name: {room_id}" + ); - if services.globals.config.auto_deactivate_banned_room_attempts { - warn!("Automatically deactivating user {user_id} due to attempted banned room join"); + if services.globals.config.auto_deactivate_banned_room_attempts { + warn!("Automatically deactivating user {user_id} due to attempted banned room join"); - if services.globals.config.admin_room_notices { - services - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "Automatically deactivating user {user_id} due to attempted banned room join from IP \ - {client_ip}" - ))) - .await - .ok(); - } - - let all_joined_rooms: Vec = services - .rooms - .state_cache - .rooms_joined(user_id) - .map(Into::into) - .collect() - .await; - - full_user_deactivate(services, user_id, &all_joined_rooms).await?; + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "Automatically deactivating user {user_id} due to attempted banned room join from IP \ + {client_ip}" + ))) + .await + .ok(); } - return Err!(Request(Forbidden("This remote server is banned on this homeserver."))); + let all_joined_rooms: Vec = services + .rooms + .state_cache + .rooms_joined(user_id) + .map(Into::into) + .collect() + .await; + + full_user_deactivate(services, user_id, &all_joined_rooms).await?; } + + return Err!(Request(Forbidden("This room is banned on this homeserver."))); + } + } else if let Some(server_name) = server_name { + if services + .globals + .config + .forbidden_remote_server_names + .contains(&server_name.to_owned()) + { + warn!( + "User {user_id} who is not an admin tried joining a room which has the server name {server_name} that \ + is globally forbidden. Rejecting.", + ); + + if services.globals.config.auto_deactivate_banned_room_attempts { + warn!("Automatically deactivating user {user_id} due to attempted banned room join"); + + if services.globals.config.admin_room_notices { + services + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "Automatically deactivating user {user_id} due to attempted banned room join from IP \ + {client_ip}" + ))) + .await + .ok(); + } + + let all_joined_rooms: Vec = services + .rooms + .state_cache + .rooms_joined(user_id) + .map(Into::into) + .collect() + .await; + + full_user_deactivate(services, user_id, &all_joined_rooms).await?; + } + + return Err!(Request(Forbidden("This remote server is banned on this homeserver."))); } } From cf71aeef0b46015c53f39cf766abd2106d2c0bb8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Dec 2024 01:02:52 -0500 Subject: [PATCH 1190/2091] fix+improve membership endpoint UX, dont reuse old membership reasons, small parallelisation Signed-off-by: strawberry --- src/api/client/membership.rs | 114 ++++++++++++++++++++--------------- src/api/client/unstable.rs | 8 +-- 2 files changed, 69 insertions(+), 53 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index fa671213..7c3a597e 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -14,7 +14,7 @@ use conduit::{ utils::{shuffle, IterStream, ReadyExt}, warn, Err, Error, PduEvent, Result, }; -use futures::{FutureExt, StreamExt}; +use futures::{join, FutureExt, StreamExt}; use ruma::{ api::{ client::{ @@ -158,7 +158,7 @@ pub(crate) async fn join_room_by_id_route( State(services): State, InsecureClientIp(client_ip): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); banned_room_check( &services, @@ -334,9 +334,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( pub(crate) async fn leave_room_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - - leave_room(&services, sender_user, &body.room_id, body.reason.clone()).await?; + leave_room(&services, body.sender_user(), &body.room_id, body.reason.clone()).await?; Ok(leave_room::v3::Response::new()) } @@ -349,7 +347,7 @@ pub(crate) async fn invite_user_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if !services.users.is_admin(sender_user).await && services.globals.block_non_admin_invites() { info!( @@ -368,9 +366,17 @@ pub(crate) async fn invite_user_route( user_id, } = &body.recipient { - if services.users.user_is_ignored(sender_user, user_id).await { + let sender_ignored_recipient = services.users.user_is_ignored(sender_user, user_id); + let recipient_ignored_by_sender = services.users.user_is_ignored(user_id, sender_user); + + let (sender_ignored_recipient, recipient_ignored_by_sender) = + join!(sender_ignored_recipient, recipient_ignored_by_sender); + + if sender_ignored_recipient { return Err!(Request(Forbidden("You cannot invite users you have ignored to rooms."))); - } else if services.users.user_is_ignored(user_id, sender_user).await { + } + + if recipient_ignored_by_sender { // silently drop the invite to the recipient if they've been ignored by the // sender, pretend it worked return Ok(invite_user::v3::Response {}); @@ -379,6 +385,7 @@ pub(crate) async fn invite_user_route( invite_helper(&services, sender_user, user_id, &body.room_id, body.reason.clone(), false) .boxed() .await?; + Ok(invite_user::v3::Response {}) } else { Err(Error::BadRequest(ErrorKind::NotFound, "User not found.")) @@ -391,16 +398,18 @@ pub(crate) async fn invite_user_route( pub(crate) async fn kick_user_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let event: RoomMemberEventContent = services + let Ok(event) = services .rooms .state_accessor - .room_state_get_content(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) + .get_member(&body.room_id, &body.user_id) .await - .map_err(|_| err!(Request(BadState("Cannot kick member that's not in the room."))))?; + else { + // copy synapse's behaviour of returning 200 without any change to the state + // instead of erroring on left users + return Ok(kick_user::v3::Response::new()); + }; services .rooms @@ -411,10 +420,13 @@ pub(crate) async fn kick_user_route( &RoomMemberEventContent { membership: MembershipState::Leave, reason: body.reason.clone(), + is_direct: None, + join_authorized_via_users_server: None, + third_party_invite: None, ..event }, ), - sender_user, + body.sender_user(), &body.room_id, &state_lock, ) @@ -431,39 +443,38 @@ pub(crate) async fn kick_user_route( pub(crate) async fn ban_user_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); + + if sender_user == body.user_id { + return Err!(Request(Forbidden("You cannot ban yourself."))); + } let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let blurhash = services.users.blurhash(&body.user_id).await.ok(); - - let event = services + let current_member_content = services .rooms .state_accessor - .room_state_get_content(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) + .get_member(&body.room_id, &body.user_id) .await - .map_or_else( - |_| RoomMemberEventContent { - blurhash: blurhash.clone(), - reason: body.reason.clone(), - ..RoomMemberEventContent::new(MembershipState::Ban) - }, - |event| RoomMemberEventContent { - membership: MembershipState::Ban, - displayname: None, - avatar_url: None, - blurhash: blurhash.clone(), - reason: body.reason.clone(), - join_authorized_via_users_server: None, - ..event - }, - ); + .unwrap_or_else(|_| RoomMemberEventContent::new(MembershipState::Ban)); services .rooms .timeline .build_and_append_pdu( - PduBuilder::state(body.user_id.to_string(), &event), + PduBuilder::state( + body.user_id.to_string(), + &RoomMemberEventContent { + membership: MembershipState::Ban, + reason: body.reason.clone(), + displayname: None, // display name may be offensive + avatar_url: None, // avatar may be offensive + is_direct: None, + join_authorized_via_users_server: None, + third_party_invite: None, + ..current_member_content + }, + ), sender_user, &body.room_id, &state_lock, @@ -481,16 +492,21 @@ pub(crate) async fn ban_user_route( pub(crate) async fn unban_user_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let event: RoomMemberEventContent = services + let current_member_content = services .rooms .state_accessor - .room_state_get_content(&body.room_id, &StateEventType::RoomMember, body.user_id.as_ref()) + .get_member(&body.room_id, &body.user_id) .await - .map_err(|_| err!(Request(BadState("Cannot unban a user who is not banned."))))?; + .unwrap_or_else(|_| RoomMemberEventContent::new(MembershipState::Leave)); + + if current_member_content.membership != MembershipState::Ban { + return Err!(Request(Forbidden( + "Cannot ban a user who is not banned (current membership: {})", + current_member_content.membership + ))); + } services .rooms @@ -502,10 +518,12 @@ pub(crate) async fn unban_user_route( membership: MembershipState::Leave, reason: body.reason.clone(), join_authorized_via_users_server: None, - ..event + third_party_invite: None, + is_direct: None, + ..current_member_content }, ), - sender_user, + body.sender_user(), &body.room_id, &state_lock, ) @@ -528,7 +546,7 @@ pub(crate) async fn unban_user_route( pub(crate) async fn forget_room_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if services .rooms @@ -553,13 +571,11 @@ pub(crate) async fn forget_room_route( pub(crate) async fn joined_rooms_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - Ok(joined_rooms::v3::Response { joined_rooms: services .rooms .state_cache - .rooms_joined(sender_user) + .rooms_joined(body.sender_user()) .map(ToOwned::to_owned) .collect() .await, @@ -575,7 +591,7 @@ pub(crate) async fn joined_rooms_route( pub(crate) async fn get_member_events_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if !services .rooms @@ -608,7 +624,7 @@ pub(crate) async fn get_member_events_route( pub(crate) async fn joined_members_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if !services .rooms diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 5de41f44..3660d674 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -294,7 +294,7 @@ pub(crate) async fn set_profile_key_route( .collect() .await; - update_displayname(&services, &body.user_id, Some(profile_key_value.to_string()), &all_joined_rooms).await?; + update_displayname(&services, &body.user_id, Some(profile_key_value.to_string()), &all_joined_rooms).await; } else if body.key == "avatar_url" { let mxc = ruma::OwnedMxcUri::from(profile_key_value.to_string()); @@ -306,7 +306,7 @@ pub(crate) async fn set_profile_key_route( .collect() .await; - update_avatar_url(&services, &body.user_id, Some(mxc), None, &all_joined_rooms).await?; + update_avatar_url(&services, &body.user_id, Some(mxc), None, &all_joined_rooms).await; } else { services .users @@ -352,7 +352,7 @@ pub(crate) async fn delete_profile_key_route( .collect() .await; - update_displayname(&services, &body.user_id, None, &all_joined_rooms).await?; + update_displayname(&services, &body.user_id, None, &all_joined_rooms).await; } else if body.key == "avatar_url" { let all_joined_rooms: Vec = services .rooms @@ -362,7 +362,7 @@ pub(crate) async fn delete_profile_key_route( .collect() .await; - update_avatar_url(&services, &body.user_id, None, None, &all_joined_rooms).await?; + update_avatar_url(&services, &body.user_id, None, None, &all_joined_rooms).await; } else { services .users From 1606441d090952a7806140f060a0d1d7b34f94b9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 20 Nov 2024 20:23:13 -0500 Subject: [PATCH 1191/2091] disable admin_room_notices for complement, update welcome text Signed-off-by: strawberry --- nix/pkgs/complement/config.toml | 1 + src/service/admin/grant.rs | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index 7aaed2f4..631100fd 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -17,6 +17,7 @@ media_compat_file_link = false media_startup_check = false rocksdb_direct_io = false log_colors = false +admin_room_notices = false [global.tls] certs = "/certificate.crt" diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 405da982..090c0294 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -77,18 +77,20 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { } } - let welcome_message = String::from("## Thank you for trying out conduwuit!\n\nconduwuit is a fork of upstream Conduit which is in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Git and Documentation: https://github.com/girlbossceo/conduwuit\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `!admin --help`\n\nHere are some rooms you can join (by typing the command):\n\nconduwuit room (Ask questions and get notified on updates):\n`/join #conduwuit:puppygock.gay`"); + if self.services.server.config.admin_room_notices { + let welcome_message = String::from("## Thank you for trying out conduwuit!\n\nconduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status initially was inherited from Conduit, however overtime this Beta status is rapidly becoming less and less relevant as our codebase significantly diverges more and more. conduwuit is quite stable and very usable as a daily driver and for a low-medium sized homeserver. There is still a lot of more work to be done, but it is in a far better place than the project was in early 2024.\n\nHelpful links:\n> GitHub Repo: https://github.com/girlbossceo/conduwuit\n> Documentation: https://conduwuit.puppyirl.gay/\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `!admin --help`\n\nHere are some rooms you can join (by typing the command into your client) -\n\nconduwuit space: `/join #conduwuit-space:puppygock.gay`\nconduwuit main room (Ask questions and get notified on updates): `/join #conduwuit:puppygock.gay`\nconduwuit offtopic room: `/join #conduwuit-offtopic:puppygock.gay`"); - // Send welcome message - self.services - .timeline - .build_and_append_pdu( - PduBuilder::timeline(&RoomMessageEventContent::text_markdown(welcome_message)), - server_user, - &room_id, - &state_lock, - ) - .await?; + // Send welcome message + self.services + .timeline + .build_and_append_pdu( + PduBuilder::timeline(&RoomMessageEventContent::text_markdown(welcome_message)), + server_user, + &room_id, + &state_lock, + ) + .await?; + } Ok(()) } From 61670370edab2cc54445afd441891c7cd4d294fe Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Dec 2024 20:46:31 -0500 Subject: [PATCH 1192/2091] add rocksdb paranoid_file_checks config option, add some more config checks Signed-off-by: strawberry --- conduwuit-example.toml | 41 +++++++++++++++++------------- src/core/config/check.rs | 54 ++++++++++++++++++++++++---------------- src/core/config/mod.rs | 44 ++++++++++++++++++++------------ src/database/opts.rs | 2 ++ 4 files changed, 87 insertions(+), 54 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 446375b8..e179ce30 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -105,7 +105,7 @@ # # This defaults to 128.0 + (64.0 * CPU core count) # -#db_cache_capacity_mb = +#db_cache_capacity_mb = varies by system # Option to control adding arbitrary text to the end of the user's # displayname upon registration with a space before the text. This was the @@ -114,9 +114,9 @@ # # The default is the trans pride flag. # -# example: "🏳️⚧️" +# example: "🏳️‍⚧️" # -#new_user_displayname_suffix = "🏳️⚧️" +#new_user_displayname_suffix = "🏳️‍⚧️" # If enabled, conduwuit will send a simple GET request periodically to # `https://pupbrain.dev/check-for-updates/stable` for any new @@ -132,8 +132,7 @@ # with such as "auth_chain_cache_capacity". # # May be useful if you have significant memory to spare to increase -# performance. This was previously called -# `conduit_cache_capacity_modifier`. +# performance. # # If you have low memory, reducing this may be viable. # @@ -204,9 +203,7 @@ # longer running Matrix). Only decrease this if you are using an external # DNS cache. # -# default_dns_min_ttl: 259200 -# -#dns_min_ttl = +#dns_min_ttl = 10800 # Minimum time-to-live in seconds for NXDOMAIN entries in the DNS cache. # This value is critical for the server to federate efficiently. @@ -217,7 +214,7 @@ # #dns_min_ttl_nxdomain = 259200 -# Number of retries after a timeout. +# Number of DNS nameserver retries after a timeout or error. # #dns_attempts = 10 @@ -547,7 +544,7 @@ # Currently, conduwuit doesn't support inbound batched key requests, so # this list should only contain other Synapse servers # -# example: ["matrix.org", "constellatory.net", "tchncs.de"] +# example: ["matrix.org", "envs.net", "constellatory.net", "tchncs.de"] # #trusted_servers = ["matrix.org"] @@ -689,14 +686,14 @@ # room invites) are ignored here. # # Defaults to false as rooms can be banned for non-moderation-related -# reasons +# reasons and this performs a full user deactivation # #auto_deactivate_banned_room_attempts = false # RocksDB log level. This is not the same as conduwuit's log level. This # is the log level for the RocksDB engine/library which show up in your # database folder/path as `LOG` files. conduwuit will log RocksDB errors -# as normal through tracing. +# as normal through tracing or panics if severe for safety. # #rocksdb_log_level = "error" @@ -745,7 +742,7 @@ # operatons such as cleanup, sync, flush, compaction, etc. Set to 0 to use # all your logical threads. Defaults to your CPU logical thread count. # -#rocksdb_parallelism_threads = 0 +#rocksdb_parallelism_threads = varies by system # Maximum number of LOG files RocksDB will keep. This must *not* be set to # 0. It must be at least 1. Defaults to 3 as these are not very useful @@ -833,6 +830,14 @@ # #rocksdb_recovery_mode = 1 +# Enables or disables paranoid SST file checks. This can improve RocksDB +# database consistency at a potential performance impact due to further +# safety checks ran. +# +# See https://github.com/facebook/rocksdb/wiki/Online-Verification#columnfamilyoptionsparanoid_file_checks for more information. +# +#rocksdb_paranoid_file_checks = false + # Database repair mode (for RocksDB SST corruption) # # Use this option when the server reports corruption while running or @@ -1119,12 +1124,14 @@ # # To disable, set this to be an empty vector (`[]`). # +# Defaults to: +# ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", # "192.168.0.0/16", "100.64.0.0/10", "192.0.0.0/24", "169.254.0.0/16", # "192.88.99.0/24", "198.18.0.0/15", "192.0.2.0/24", "198.51.100.0/24", # "203.0.113.0/24", "224.0.0.0/4", "::1/128", "fe80::/10", "fc00::/7", # "2001:db8::/32", "ff00::/8", "fec0::/10"] # -#ip_range_denylist = ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", +#ip_range_denylist = # Optional IP address or network interface-name to bind as the source of # URL preview requests. If not set, it will not bind to a specific @@ -1181,9 +1188,9 @@ #url_preview_url_contains_allowlist = [] # Maximum amount of bytes allowed in a URL preview body size when -# spidering. Defaults to 384KB in bytes. +# spidering. Defaults to 256KB in bytes. # -#url_preview_max_spider_size = 384000 +#url_preview_max_spider_size = 256000 # Option to decide whether you would like to run the domain allowlist # checks (contains and explicit) on the root domain or not. Does not apply @@ -1302,7 +1309,7 @@ # Sentry.io crash/panic reporting, performance monitoring/metrics, etc. # This is NOT enabled by default. conduwuit's default Sentry reporting -# endpoint is o4506996327251968.ingest.us.sentry.io +# endpoint domain is o4506996327251968.ingest.us.sentry.io # #sentry = false diff --git a/src/core/config/check.rs b/src/core/config/check.rs index c75fb31e..b8415281 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -20,10 +20,7 @@ pub fn check(config: &Config) -> Result<()> { } if cfg!(all(feature = "hardened_malloc", feature = "jemalloc")) { - warn!( - "hardened_malloc and jemalloc are both enabled, this causes jemalloc to be used. If using --all-features, \ - this is harmless." - ); + info!("hardened_malloc and jemalloc compile-time features are both enabled, this causes jemalloc to be used."); } if cfg!(not(unix)) && config.unix_socket_path.is_some() { @@ -34,7 +31,15 @@ pub fn check(config: &Config) -> Result<()> { )); } - if cfg!(unix) && config.unix_socket_path.is_none() { + if config.unix_socket_path.is_none() && config.get_bind_hosts().is_empty() { + return Err!(Config("address", "No TCP addresses were specified to listen on")); + } + + if config.unix_socket_path.is_none() && config.get_bind_ports().is_empty() { + return Err!(Config("port", "No ports were specified to listen on")); + } + + if config.unix_socket_path.is_none() { config.get_bind_addrs().iter().for_each(|addr| { use std::path::Path; @@ -50,18 +55,14 @@ pub fn check(config: &Config) -> Result<()> { host and guest, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, \ you can ignore.", ); - } - - if Path::new("/.dockerenv").exists() { + } else if Path::new("/.dockerenv").exists() { error!( "You are detected using Docker with a loopback/localhost listening address of {addr}. If you \ are using a reverse proxy on the host and require communication to conduwuit in the Docker \ container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ If this is expected, you can ignore.", ); - } - - if Path::new("/run/.containerenv").exists() { + } else if Path::new("/run/.containerenv").exists() { error!( "You are detected using Podman with a loopback/localhost listening address of {addr}. If you \ are using a reverse proxy on the host and require communication to conduwuit in the Podman \ @@ -89,6 +90,13 @@ pub fn check(config: &Config) -> Result<()> { )); } + if config.emergency_password == Some(String::from("F670$2CP@Hw8mG7RY1$%!#Ic7YA")) { + return Err!(Config( + "emergency_password", + "The public example emergency password is being used, this is insecure. Please change this." + )); + } + // check if the user specified a registration token as `""` if config.registration_token == Some(String::new()) { return Err!(Config( @@ -113,17 +121,20 @@ pub fn check(config: &Config) -> Result<()> { )); } - if config.max_request_size < 5_120_000 { + if config.max_request_size < 10_000_000 { return Err!(Config( "max_request_size", - "Max request size is less than 5MB. Please increase it." + "Max request size is less than 10MB. Please increase it as this is too low for operable federation." )); } // check if user specified valid IP CIDR ranges on startup for cidr in &config.ip_range_denylist { if let Err(e) = ipaddress::IPAddress::parse(cidr) { - return Err!(Config("ip_range_denylist", "Parsing specified IP CIDR range from string: {e}.")); + return Err!(Config( + "ip_range_denylist", + "Parsing specified IP CIDR range from string failed: {e}." + )); } } @@ -135,10 +146,10 @@ pub fn check(config: &Config) -> Result<()> { return Err!(Config( "registration_token", "!! You have `allow_registration` enabled without a token configured in your config which means you are \ - allowing ANYONE to register on your conduwuit instance without any 2nd-step (e.g. registration token).\n -If this is not the intended behaviour, please set a registration token.\n -For security and safety reasons, conduwuit will shut down. If you are extra sure this is the desired behaviour you \ - want, please set the following config option to true: + allowing ANYONE to register on your conduwuit instance without any 2nd-step (e.g. registration token). \ + If this is not the intended behaviour, please set a registration token. For security and safety reasons, \ + conduwuit will shut down. If you are extra sure this is the desired behaviour you want, please set the \ + following config option to true: `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`" )); } @@ -151,15 +162,16 @@ For security and safety reasons, conduwuit will shut down. If you are extra sure warn!( "Open registration is enabled via setting \ `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` and `allow_registration` to \ - true without a registration token configured. You are expected to be aware of the risks now.\n - If this is not the desired behaviour, please set a registration token." + true without a registration token configured. You are expected to be aware of the risks now. If this is \ + not the desired behaviour, please set a registration token." ); } if config.allow_outgoing_presence && !config.allow_local_presence { return Err!(Config( "allow_local_presence", - "Outgoing presence requires allowing local presence. Please enable 'allow_local_presence'." + "Outgoing presence requires allowing local presence. Please enable 'allow_local_presence' or disable \ + outgoing presence." )); } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 0e25b197..413aa7f4 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -141,6 +141,8 @@ pub struct Config { /// core count. /// /// This defaults to 128.0 + (64.0 * CPU core count) + /// + /// default: varies by system #[serde(default = "default_db_cache_capacity_mb")] pub db_cache_capacity_mb: f64, @@ -151,9 +153,9 @@ pub struct Config { /// /// The default is the trans pride flag. /// - /// example: "🏳️⚧️" + /// example: "🏳️‍⚧️" /// - /// default: "🏳️⚧️" + /// default: "🏳️‍⚧️" #[serde(default = "default_new_user_displayname_suffix")] pub new_user_displayname_suffix: String, @@ -164,15 +166,14 @@ pub struct Config { /// /// This is disabled by default as this is rarely used except for security /// updates or major updates. - #[serde(default)] + #[serde(default, alias = "allow_announcements_check")] pub allow_check_for_updates: bool, /// Set this to any float value to multiply conduwuit's in-memory LRU caches /// with such as "auth_chain_cache_capacity". /// /// May be useful if you have significant memory to spare to increase - /// performance. This was previously called - /// `conduit_cache_capacity_modifier`. + /// performance. /// /// If you have low memory, reducing this may be viable. /// @@ -247,7 +248,7 @@ pub struct Config { /// longer running Matrix). Only decrease this if you are using an external /// DNS cache. /// - /// default_dns_min_ttl: 259200 + /// default: 10800 #[serde(default = "default_dns_min_ttl")] pub dns_min_ttl: u64, @@ -262,7 +263,7 @@ pub struct Config { #[serde(default = "default_dns_min_ttl_nxdomain")] pub dns_min_ttl_nxdomain: u64, - /// Number of retries after a timeout. + /// Number of DNS nameserver retries after a timeout or error. /// /// default: 10 #[serde(default = "default_dns_attempts")] @@ -633,7 +634,7 @@ pub struct Config { /// Currently, conduwuit doesn't support inbound batched key requests, so /// this list should only contain other Synapse servers /// - /// example: ["matrix.org", "constellatory.net", "tchncs.de"] + /// example: ["matrix.org", "envs.net", "constellatory.net", "tchncs.de"] /// /// default: ["matrix.org"] #[serde(default = "default_trusted_servers")] @@ -794,14 +795,14 @@ pub struct Config { /// room invites) are ignored here. /// /// Defaults to false as rooms can be banned for non-moderation-related - /// reasons + /// reasons and this performs a full user deactivation #[serde(default)] pub auto_deactivate_banned_room_attempts: bool, /// RocksDB log level. This is not the same as conduwuit's log level. This /// is the log level for the RocksDB engine/library which show up in your /// database folder/path as `LOG` files. conduwuit will log RocksDB errors - /// as normal through tracing. + /// as normal through tracing or panics if severe for safety. /// /// default: "error" #[serde(default = "default_rocksdb_log_level")] @@ -855,7 +856,7 @@ pub struct Config { /// operatons such as cleanup, sync, flush, compaction, etc. Set to 0 to use /// all your logical threads. Defaults to your CPU logical thread count. /// - /// default: 0 + /// default: varies by system #[serde(default = "default_rocksdb_parallelism_threads")] pub rocksdb_parallelism_threads: usize, @@ -955,6 +956,14 @@ pub struct Config { #[serde(default = "default_rocksdb_recovery_mode")] pub rocksdb_recovery_mode: u8, + /// Enables or disables paranoid SST file checks. This can improve RocksDB + /// database consistency at a potential performance impact due to further + /// safety checks ran. + /// + /// See https://github.com/facebook/rocksdb/wiki/Online-Verification#columnfamilyoptionsparanoid_file_checks for more information. + #[serde(default)] + pub rocksdb_paranoid_file_checks: bool, + /// Database repair mode (for RocksDB SST corruption) /// /// Use this option when the server reports corruption while running or @@ -1255,7 +1264,8 @@ pub struct Config { /// /// To disable, set this to be an empty vector (`[]`). /// - /// default: ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", + /// Defaults to: + /// ["127.0.0.0/8", "10.0.0.0/8", "172.16.0.0/12", /// "192.168.0.0/16", "100.64.0.0/10", "192.0.0.0/24", "169.254.0.0/16", /// "192.88.99.0/24", "198.18.0.0/15", "192.0.2.0/24", "198.51.100.0/24", /// "203.0.113.0/24", "224.0.0.0/4", "::1/128", "fe80::/10", "fc00::/7", @@ -1328,9 +1338,9 @@ pub struct Config { pub url_preview_url_contains_allowlist: Vec, /// Maximum amount of bytes allowed in a URL preview body size when - /// spidering. Defaults to 384KB in bytes. + /// spidering. Defaults to 256KB in bytes. /// - /// default: 384000 + /// default: 256000 #[serde(default = "default_url_preview_max_spider_size")] pub url_preview_max_spider_size: usize, @@ -1465,7 +1475,7 @@ pub struct Config { /// Sentry.io crash/panic reporting, performance monitoring/metrics, etc. /// This is NOT enabled by default. conduwuit's default Sentry reporting - /// endpoint is o4506996327251968.ingest.us.sentry.io + /// endpoint domain is o4506996327251968.ingest.us.sentry.io #[serde(default)] pub sentry: bool, @@ -1532,12 +1542,14 @@ pub struct Config { /// specifically the queue-depth or the number of simultaneous requests in /// flight. Defaults to 32 or four times the number of CPU cores, whichever /// is greater. + /// /// default: 32 #[serde(default = "default_db_pool_workers")] pub db_pool_workers: usize, /// Size of the queue feeding the database's frontend-pool. Defaults to 256 /// or eight times the number of CPU cores, whichever is greater. + /// /// default: 256 #[serde(default = "default_db_pool_queue_size")] pub db_pool_queue_size: usize, @@ -2282,7 +2294,7 @@ fn default_ip_range_denylist() -> Vec { } fn default_url_preview_max_spider_size() -> usize { - 384_000 // 384KB + 256_000 // 256KB } fn default_new_user_displayname_suffix() -> String { "🏳️‍⚧️".to_owned() } diff --git a/src/database/opts.rs b/src/database/opts.rs index f1b4d3a9..d0ae618c 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -95,6 +95,8 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ 4_u8..=u8::MAX => unimplemented!(), }); + opts.set_paranoid_checks(config.rocksdb_paranoid_file_checks); + opts.set_env(env); Ok(opts) } From 9d59f777d27bff30f776edec0f5a9788cfdba534 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 7 Dec 2024 01:07:01 -0500 Subject: [PATCH 1193/2091] refactor fed membership endpoints, add missing checks, some cleanup, reduce line width Signed-off-by: strawberry --- src/api/client/membership.rs | 500 +++++++++++++-------------- src/api/client/user_directory.rs | 2 +- src/api/server/invite.rs | 34 +- src/api/server/make_join.rs | 138 ++++---- src/api/server/make_knock.rs | 7 +- src/api/server/make_leave.rs | 11 +- src/api/server/send_join.rs | 159 +++++---- src/api/server/send_knock.rs | 5 +- src/api/server/send_leave.rs | 79 ++--- src/service/pusher/mod.rs | 6 +- src/service/rooms/state_cache/mod.rs | 17 +- src/service/rooms/timeline/mod.rs | 14 +- 12 files changed, 474 insertions(+), 498 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 7c3a597e..a61011b0 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -7,12 +7,12 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduit::{ - debug, debug_info, debug_warn, err, error, info, pdu, - pdu::{gen_event_id_canonical_json, PduBuilder}, + debug, debug_info, debug_warn, err, error, info, + pdu::{self, gen_event_id_canonical_json, PduBuilder}, result::FlatOk, - trace, utils, - utils::{shuffle, IterStream, ReadyExt}, - warn, Err, Error, PduEvent, Result, + trace, + utils::{self, shuffle, IterStream, ReadyExt}, + warn, Err, PduEvent, Result, }; use futures::{join, FutureExt, StreamExt}; use ruma::{ @@ -153,21 +153,14 @@ async fn banned_room_check( /// rules locally /// - If the server does not know about the room: asks other servers over /// federation -#[tracing::instrument(skip_all, fields(%client_ip), name = "join")] +#[tracing::instrument(skip_all, fields(%client), name = "join")] pub(crate) async fn join_room_by_id_route( - State(services): State, InsecureClientIp(client_ip): InsecureClientIp, + State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user(); - banned_room_check( - &services, - sender_user, - Some(&body.room_id), - body.room_id.server_name(), - client_ip, - ) - .await?; + banned_room_check(&services, sender_user, Some(&body.room_id), body.room_id.server_name(), client).await?; // There is no body.server_name for /roomId/join let mut servers: Vec<_> = services @@ -354,10 +347,7 @@ pub(crate) async fn invite_user_route( "User {sender_user} is not an admin and attempted to send an invite to room {}", &body.room_id ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Invites are not allowed on this server.", - )); + return Err!(Request(Forbidden("Invites are not allowed on this server."))); } banned_room_check(&services, sender_user, Some(&body.room_id), body.room_id.server_name(), client).await?; @@ -388,7 +378,7 @@ pub(crate) async fn invite_user_route( Ok(invite_user::v3::Response {}) } else { - Err(Error::BadRequest(ErrorKind::NotFound, "User not found.")) + Err!(Request(NotFound("User not found."))) } } @@ -686,6 +676,18 @@ pub async fn join_room_by_id_helper( }); } + if let Ok(membership) = services + .rooms + .state_accessor + .get_member(room_id, sender_user) + .await + { + if membership.membership == MembershipState::Ban { + debug_warn!("{sender_user} is banned from {room_id} but attempted to join"); + return Err!(Request(Forbidden("You are banned from the room."))); + } + } + let server_in_room = services .rooms .state_cache @@ -730,19 +732,29 @@ async fn join_room_by_id_helper_remote( )); } - let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) - .map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?; + let mut join_event_stub: CanonicalJsonObject = + serde_json::from_str(make_join_response.event.get()).map_err(|e| { + err!(BadServerResponse(warn!( + "Invalid make_join event json received from server: {e:?}" + ))) + })?; - let join_authorized_via_users_server = join_event_stub - .get("content") - .map(|s| { - s.as_object()? - .get("join_authorised_via_users_server")? - .as_str() - }) - .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()); + let join_authorized_via_users_server = { + use RoomVersionId::*; + if !matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { + join_event_stub + .get("content") + .map(|s| { + s.as_object()? + .get("join_authorised_via_users_server")? + .as_str() + }) + .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()) + } else { + None + } + }; - // TODO: Is origin needed? join_event_stub.insert( "origin".to_owned(), CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), @@ -811,65 +823,46 @@ async fn join_room_by_id_helper_remote( info!("send_join finished"); if join_authorized_via_users_server.is_some() { - use RoomVersionId::*; - match &room_version_id { - V1 | V2 | V3 | V4 | V5 | V6 | V7 => { - warn!( - "Found `join_authorised_via_users_server` but room {} is version {}. Ignoring.", - room_id, &room_version_id - ); - }, - // only room versions 8 and above using `join_authorized_via_users_server` (restricted joins) need to - // validate and send signatures - _ => { - if let Some(signed_raw) = &send_join_response.room_state.event { - debug_info!( - "There is a signed event. This room is probably using restricted joins. Adding signature to \ - our event" + if let Some(signed_raw) = &send_join_response.room_state.event { + debug_info!( + "There is a signed event with join_authorized_via_users_server. This room is probably using \ + restricted joins. Adding signature to our event" + ); + + let (signed_event_id, signed_value) = gen_event_id_canonical_json(signed_raw, &room_version_id) + .map_err(|e| err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))))?; + + if signed_event_id != event_id { + return Err!(Request(BadJson( + warn!(%signed_event_id, %event_id, "Server {remote_server} sent event with wrong event ID") + ))); + } + + match signed_value["signatures"] + .as_object() + .ok_or_else(|| err!(BadServerResponse(warn!("Server {remote_server} sent invalid signatures type")))) + .and_then(|e| { + e.get(remote_server.as_str()).ok_or_else(|| { + err!(BadServerResponse(warn!( + "Server {remote_server} did not send its signature for a restricted room" + ))) + }) + }) { + Ok(signature) => { + join_event + .get_mut("signatures") + .expect("we created a valid pdu") + .as_object_mut() + .expect("we created a valid pdu") + .insert(remote_server.to_string(), signature.clone()); + }, + Err(e) => { + warn!( + "Server {remote_server} sent invalid signature in send_join signatures for event \ + {signed_value:?}: {e:?}", ); - let Ok((signed_event_id, signed_value)) = gen_event_id_canonical_json(signed_raw, &room_version_id) - else { - // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); - }; - - if signed_event_id != event_id { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Server sent event with wrong event id", - )); - } - - match signed_value["signatures"] - .as_object() - .ok_or(Error::BadRequest( - ErrorKind::InvalidParam, - "Server sent invalid signatures type", - )) - .and_then(|e| { - e.get(remote_server.as_str()) - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Server did not send its signature")) - }) { - Ok(signature) => { - join_event - .get_mut("signatures") - .expect("we created a valid pdu") - .as_object_mut() - .expect("we created a valid pdu") - .insert(remote_server.to_string(), signature.clone()); - }, - Err(e) => { - warn!( - "Server {remote_server} sent invalid signature in sendjoin signatures for event \ - {signed_value:?}: {e:?}", - ); - }, - } - } - }, + }, + } } } @@ -1041,14 +1034,13 @@ async fn join_room_by_id_helper_local( services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, ) -> Result { - debug!("We can join locally"); + debug_info!("We can join locally"); let join_rules_event_content = services .rooms .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") - .await - .map(|content: RoomJoinRulesEventContent| content); + .room_state_get_content::(room_id, &StateEventType::RoomJoinRules, "") + .await; let restriction_rooms = match join_rules_event_content { Ok(RoomJoinRulesEventContent { @@ -1064,40 +1056,36 @@ async fn join_room_by_id_helper_local( _ => Vec::new(), }; - let local_members: Vec<_> = services - .rooms - .state_cache - .room_members(room_id) - .ready_filter(|user| services.globals.user_is_local(user)) - .map(ToOwned::to_owned) - .collect() - .await; - - let mut join_authorized_via_users_server: Option = None; - - if restriction_rooms - .iter() - .stream() - .any(|restriction_room_id| { + let join_authorized_via_users_server: Option = { + if restriction_rooms + .iter() + .stream() + .any(|restriction_room_id| { + services + .rooms + .state_cache + .is_joined(sender_user, restriction_room_id) + }) + .await + { services .rooms .state_cache - .is_joined(sender_user, restriction_room_id) - }) - .await - { - for user in local_members { - if services - .rooms - .state_accessor - .user_can_invite(room_id, &user, sender_user, &state_lock) + .local_users_in_room(room_id) + .filter(|user| { + services + .rooms + .state_accessor + .user_can_invite(room_id, user, sender_user, &state_lock) + }) + .boxed() + .next() .await - { - join_authorized_via_users_server = Some(user); - break; - } + .map(ToOwned::to_owned) + } else { + None } - } + }; let content = RoomMemberEventContent { displayname: services.users.displayname(sender_user).await.ok(), @@ -1109,7 +1097,7 @@ async fn join_room_by_id_helper_local( }; // Try normal join first - let error = match services + let Err(error) = services .rooms .timeline .build_and_append_pdu( @@ -1119,130 +1107,125 @@ async fn join_room_by_id_helper_local( &state_lock, ) .await - { - Ok(_) => return Ok(()), - Err(e) => e, + else { + return Ok(()); }; - if !restriction_rooms.is_empty() - && servers - .iter() - .any(|server_name| !services.globals.server_is_ours(server_name)) + if restriction_rooms.is_empty() + && (servers.is_empty() || servers.len() == 1 && services.globals.server_is_ours(&servers[0])) { - warn!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); - let (make_join_response, remote_server) = make_join_request(services, sender_user, room_id, servers).await?; + return Err(error); + } - let Some(room_version_id) = make_join_response.room_version else { - return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); - }; + warn!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); + let Ok((make_join_response, remote_server)) = make_join_request(services, sender_user, room_id, servers).await + else { + return Err(error); + }; - if !services.server.supported_room_version(&room_version_id) { - return Err!(BadServerResponse( - "Remote room version {room_version_id} is not supported by conduwuit" - )); - } + let Some(room_version_id) = make_join_response.room_version else { + return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); + }; - let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) - .map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?; - let join_authorized_via_users_server = join_event_stub - .get("content") - .map(|s| { - s.as_object()? - .get("join_authorised_via_users_server")? - .as_str() - }) - .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()); - // TODO: Is origin needed? - join_event_stub.insert( - "origin".to_owned(), - CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), - ); - join_event_stub.insert( - "origin_server_ts".to_owned(), - CanonicalJsonValue::Integer( - utils::millis_since_unix_epoch() - .try_into() - .expect("Timestamp is valid js_int value"), - ), - ); - join_event_stub.insert( - "content".to_owned(), - to_canonical_value(RoomMemberEventContent { - displayname: services.users.displayname(sender_user).await.ok(), - avatar_url: services.users.avatar_url(sender_user).await.ok(), - blurhash: services.users.blurhash(sender_user).await.ok(), - reason, - join_authorized_via_users_server, - ..RoomMemberEventContent::new(MembershipState::Join) - }) - .expect("event is valid, we just created it"), - ); + if !services.server.supported_room_version(&room_version_id) { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } - // We keep the "event_id" in the pdu only in v1 or - // v2 rooms - match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => { - join_event_stub.remove("event_id"); + let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) + .map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?; + + let join_authorized_via_users_server = join_event_stub + .get("content") + .map(|s| { + s.as_object()? + .get("join_authorised_via_users_server")? + .as_str() + }) + .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()); + + join_event_stub.insert( + "origin".to_owned(), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), + ); + join_event_stub.insert( + "origin_server_ts".to_owned(), + CanonicalJsonValue::Integer( + utils::millis_since_unix_epoch() + .try_into() + .expect("Timestamp is valid js_int value"), + ), + ); + join_event_stub.insert( + "content".to_owned(), + to_canonical_value(RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason, + join_authorized_via_users_server, + ..RoomMemberEventContent::new(MembershipState::Join) + }) + .expect("event is valid, we just created it"), + ); + + // We keep the "event_id" in the pdu only in v1 or + // v2 rooms + match room_version_id { + RoomVersionId::V1 | RoomVersionId::V2 => {}, + _ => { + join_event_stub.remove("event_id"); + }, + }; + + // In order to create a compatible ref hash (EventID) the `hashes` field needs + // to be present + services + .server_keys + .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; + + // Generate event id + let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?; + + // Add event_id back + join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + + // It has enough fields to be called a proper event now + let join_event = join_event_stub; + + let send_join_response = services + .sending + .send_synapse_request( + &remote_server, + federation::membership::create_join_event::v2::Request { + room_id: room_id.to_owned(), + event_id: event_id.clone(), + omit_members: false, + pdu: services + .sending + .convert_to_outgoing_federation_event(join_event.clone()) + .await, }, - }; + ) + .await?; - // In order to create a compatible ref hash (EventID) the `hashes` field needs - // to be present - services - .server_keys - .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; + if let Some(signed_raw) = send_join_response.room_state.event { + let (signed_event_id, signed_value) = gen_event_id_canonical_json(&signed_raw, &room_version_id) + .map_err(|e| err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))))?; - // Generate event id - let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?; - - // Add event_id back - join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); - - // It has enough fields to be called a proper event now - let join_event = join_event_stub; - - let send_join_response = services - .sending - .send_synapse_request( - &remote_server, - federation::membership::create_join_event::v2::Request { - room_id: room_id.to_owned(), - event_id: event_id.clone(), - omit_members: false, - pdu: services - .sending - .convert_to_outgoing_federation_event(join_event.clone()) - .await, - }, - ) - .await?; - - if let Some(signed_raw) = send_join_response.room_state.event { - let Ok((signed_event_id, signed_value)) = gen_event_id_canonical_json(&signed_raw, &room_version_id) else { - // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); - }; - - if signed_event_id != event_id { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Server sent event with wrong event id", - )); - } - - drop(state_lock); - services - .rooms - .event_handler - .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true) - .await?; - } else { - return Err(error); + if signed_event_id != event_id { + return Err!(Request(BadJson( + warn!(%signed_event_id, %event_id, "Server {remote_server} sent event with wrong event ID") + ))); } + + drop(state_lock); + services + .rooms + .event_handler + .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true) + .await?; } else { return Err(error); } @@ -1317,13 +1300,10 @@ async fn make_join_request( pub(crate) async fn invite_helper( services: &Services, sender_user: &UserId, user_id: &UserId, room_id: &RoomId, reason: Option, is_direct: bool, -) -> Result<()> { +) -> Result { if !services.users.is_admin(sender_user).await && services.globals.block_non_admin_invites() { info!("User {sender_user} is not an admin and attempted to send an invite to room {room_id}"); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Invites are not allowed on this server.", - )); + return Err!(Request(Forbidden("Invites are not allowed on this server."))); } if !services.globals.user_is_local(user_id) { @@ -1382,30 +1362,24 @@ pub(crate) async fn invite_helper( // We do not add the event_id field to the pdu here because of signature and // hashes checks - let Ok((event_id, value)) = gen_event_id_canonical_json(&response.event, &room_version_id) else { - // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); - }; + let (event_id, value) = gen_event_id_canonical_json(&response.event, &room_version_id) + .map_err(|e| err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))))?; - if *pdu.event_id != *event_id { - warn!( - "Server {} changed invite event, that's not allowed in the spec: ours: {pdu_json:?}, theirs: {value:?}", - user_id.server_name(), - ); + if pdu.event_id != event_id { + return Err!(Request(BadJson( + warn!(%pdu.event_id, %event_id, "Server {} sent event with wrong event ID", user_id.server_name()) + ))); } let origin: OwnedServerName = serde_json::from_value( serde_json::to_value( value .get("origin") - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Event needs an origin field."))?, + .ok_or_else(|| err!(Request(BadJson("Event missing origin field."))))?, ) .expect("CanonicalJson is valid json value"), ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Origin field is invalid."))?; + .map_err(|e| err!(Request(BadJson(warn!("Origin field in event is not a valid server name: {e}")))))?; let pdu_id = services .rooms @@ -1414,8 +1388,7 @@ pub(crate) async fn invite_helper( .await? .ok_or_else(|| err!(Request(InvalidParam("Could not accept incoming PDU as timeline event."))))?; - services.sending.send_pdu_room(room_id, &pdu_id).await?; - return Ok(()); + return services.sending.send_pdu_room(room_id, &pdu_id).await; } if !services @@ -1424,10 +1397,9 @@ pub(crate) async fn invite_helper( .is_joined(sender_user, room_id) .await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this room.", - )); + return Err!(Request(Forbidden( + "You must be joined in the room you are trying to invite from." + ))); } let state_lock = services.rooms.state.mutex.lock(room_id).await; @@ -1599,7 +1571,11 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room .map(|user| user.server_name().to_owned()), ); - debug!("servers in remote_leave_room: {servers:?}"); + if let Some(room_id_server_name) = room_id.server_name() { + servers.insert(room_id_server_name.to_owned()); + } + + debug_info!("servers in remote_leave_room: {servers:?}"); for remote_server in servers { let make_leave_response = services diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index f3fee8d1..feb48346 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -21,7 +21,7 @@ pub(crate) async fn search_users_route( State(services): State, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let limit = usize::try_from(body.limit).unwrap_or(10); // default limit is 10 + let limit = usize::try_from(body.limit).map_or(10, usize::from).min(100); // default limit is 10 let users = services.users.stream().filter_map(|user_id| async { // Filter out buggy users (they should not exist, but you never know...) diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index f7919bb3..49263d52 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -145,24 +145,24 @@ pub(crate) async fn create_invite_route( true, ) .await?; - } - for appservice in services.appservice.read().await.values() { - if appservice.is_user_match(&invited_user) { - services - .sending - .send_appservice_request( - appservice.registration.clone(), - ruma::api::appservice::event::push_events::v1::Request { - events: vec![pdu.to_room_event()], - txn_id: general_purpose::URL_SAFE_NO_PAD - .encode(sha256::hash(pdu.event_id.as_bytes())) - .into(), - ephemeral: Vec::new(), - to_device: Vec::new(), - }, - ) - .await?; + for appservice in services.appservice.read().await.values() { + if appservice.is_user_match(&invited_user) { + services + .sending + .send_appservice_request( + appservice.registration.clone(), + ruma::api::appservice::event::push_events::v1::Request { + events: vec![pdu.to_room_event()], + txn_id: general_purpose::URL_SAFE_NO_PAD + .encode(sha256::hash(pdu.event_id.as_bytes())) + .into(), + ephemeral: Vec::new(), + to_device: Vec::new(), + }, + ) + .await?; + } } } diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index d5ea675e..05a4125d 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -1,8 +1,5 @@ use axum::extract::State; -use conduit::{ - utils::{IterStream, ReadyExt}, - warn, -}; +use conduit::{debug_info, utils::IterStream, warn, Err}; use futures::StreamExt; use ruma::{ api::{client::error::ErrorKind, federation::membership::prepare_join_event}, @@ -13,7 +10,7 @@ use ruma::{ }, StateEventType, }, - CanonicalJsonObject, RoomId, RoomVersionId, UserId, + CanonicalJsonObject, OwnedUserId, RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; @@ -29,14 +26,11 @@ pub(crate) async fn create_join_event_template_route( State(services): State, body: Ruma, ) -> Result { if !services.rooms.metadata.exists(&body.room_id).await { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + return Err!(Request(NotFound("Room is unknown to this server."))); } if body.user_id.server_name() != body.origin() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to join on behalf of another server/user", - )); + return Err!(Request(BadJson("Not allowed to join on behalf of another server/user."))); } // ACL check origin server @@ -59,10 +53,7 @@ pub(crate) async fn create_join_event_template_route( &body.user_id, &body.room_id, ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + return Err!(Request(Forbidden("Server is banned on this homeserver."))); } if let Some(server) = body.room_id.server_name() { @@ -72,10 +63,9 @@ pub(crate) async fn create_join_event_template_route( .forbidden_remote_server_names .contains(&server.to_owned()) { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + return Err!(Request(Forbidden(warn!( + "Room ID server name {server} is banned on this homeserver." + )))); } } @@ -91,39 +81,35 @@ pub(crate) async fn create_join_event_template_route( let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - let join_authorized_via_users_server = if (services - .rooms - .state_cache - .is_left(&body.user_id, &body.room_id) - .await) - && user_can_perform_restricted_join(&services, &body.user_id, &body.room_id, &room_version_id).await? - { - let auth_user = services - .rooms - .state_cache - .room_members(&body.room_id) - .ready_filter(|user| user.server_name() == services.globals.server_name()) - .filter(|user| { - services - .rooms - .state_accessor - .user_can_invite(&body.room_id, user, &body.user_id, &state_lock) - }) - .boxed() - .next() - .await - .map(ToOwned::to_owned); - - if auth_user.is_some() { - auth_user + let join_authorized_via_users_server: Option = { + use RoomVersionId::*; + if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { + // room version does not support restricted join rules + None + } else if user_can_perform_restricted_join(&services, &body.user_id, &body.room_id, &room_version_id).await? { + let Some(auth_user) = services + .rooms + .state_cache + .local_users_in_room(&body.room_id) + .filter(|user| { + services + .rooms + .state_accessor + .user_can_invite(&body.room_id, user, &body.user_id, &state_lock) + }) + .boxed() + .next() + .await + .map(ToOwned::to_owned) + else { + return Err!(Request(UnableToGrantJoin( + "No user on this server is able to assist in joining." + ))); + }; + Some(auth_user) } else { - return Err(Error::BadRequest( - ErrorKind::UnableToGrantJoin, - "No user on this server is able to assist in joining.", - )); + None } - } else { - None }; let (_pdu, mut pdu_json) = services @@ -155,37 +141,39 @@ pub(crate) async fn create_join_event_template_route( } /// Checks whether the given user can join the given room via a restricted join. -/// This doesn't check the current user's membership. This should be done -/// externally, either by using the state cache or attempting to authorize the -/// event. pub(crate) async fn user_can_perform_restricted_join( services: &Services, user_id: &UserId, room_id: &RoomId, room_version_id: &RoomVersionId, ) -> Result { use RoomVersionId::*; - let join_rules_event = services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomJoinRules, "") - .await; - - let Ok(Ok(join_rules_event_content)) = join_rules_event.as_ref().map(|join_rules_event| { - serde_json::from_str::(join_rules_event.content.get()).map_err(|e| { - warn!("Invalid join rules event in database: {e}"); - Error::bad_database("Invalid join rules event in database") - }) - }) else { - return Ok(false); - }; - + // restricted rooms are not supported on <=v7 if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { return Ok(false); } + if services.rooms.state_cache.is_joined(user_id, room_id).await { + // joining user is already joined, there is nothing we need to do + return Ok(false); + } + + let Ok(join_rules_event_content) = services + .rooms + .state_accessor + .room_state_get_content::(room_id, &StateEventType::RoomJoinRules, "") + .await + else { + return Ok(false); + }; + let (JoinRule::Restricted(r) | JoinRule::KnockRestricted(r)) = join_rules_event_content.join_rule else { return Ok(false); }; + if r.allow.is_empty() { + debug_info!("{room_id} is restricted but the allow key is empty"); + return Ok(false); + } + if r.allow .iter() .filter_map(|rule| { @@ -201,22 +189,20 @@ pub(crate) async fn user_can_perform_restricted_join( { Ok(true) } else { - Err(Error::BadRequest( - ErrorKind::UnableToAuthorizeJoin, - "User is not known to be in any required room.", - )) + Err!(Request(UnableToAuthorizeJoin( + "Joining user is not known to be in any required room." + ))) } } -pub(crate) fn maybe_strip_event_id(pdu_json: &mut CanonicalJsonObject, room_version_id: &RoomVersionId) -> Result<()> { +pub(crate) fn maybe_strip_event_id(pdu_json: &mut CanonicalJsonObject, room_version_id: &RoomVersionId) -> Result { use RoomVersionId::*; match room_version_id { - V1 | V2 => {}, + V1 | V2 => Ok(()), _ => { pdu_json.remove("event_id"); + Ok(()) }, - }; - - Ok(()) + } } diff --git a/src/api/server/make_knock.rs b/src/api/server/make_knock.rs index c1875a1f..34883355 100644 --- a/src/api/server/make_knock.rs +++ b/src/api/server/make_knock.rs @@ -18,14 +18,11 @@ pub(crate) async fn create_knock_event_template_route( State(services): State, body: Ruma, ) -> Result { if !services.rooms.metadata.exists(&body.room_id).await { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + return Err!(Request(NotFound("Room is unknown to this server."))); } if body.user_id.server_name() != body.origin() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to knock on behalf of another server/user", - )); + return Err!(Request(BadJson("Not allowed to knock on behalf of another server/user."))); } // ACL check origin server diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 33a94560..1b81aea5 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -1,7 +1,7 @@ use axum::extract::State; -use conduit::{Error, Result}; +use conduit::{Err, Result}; use ruma::{ - api::{client::error::ErrorKind, federation::membership::prepare_leave_event}, + api::federation::membership::prepare_leave_event, events::room::member::{MembershipState, RoomMemberEventContent}, }; use serde_json::value::to_raw_value; @@ -16,14 +16,11 @@ pub(crate) async fn create_leave_event_template_route( State(services): State, body: Ruma, ) -> Result { if !services.rooms.metadata.exists(&body.room_id).await { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + return Err!(Request(NotFound("Room is unknown to this server."))); } if body.user_id.server_name() != body.origin() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to leave on behalf of another server/user", - )); + return Err!(Request(BadJson("Not allowed to leave on behalf of another server/user."))); } // ACL check origin diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index d1574e62..1e1e8fed 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -7,16 +7,16 @@ use conduit::{ err, pdu::gen_event_id_canonical_json, utils::stream::{IterStream, TryBroadbandExt}, - warn, Error, Result, + warn, Err, Result, }; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ - api::{client::error::ErrorKind, federation::membership::create_join_event}, + api::federation::membership::create_join_event, events::{ room::member::{MembershipState, RoomMemberEventContent}, StateEventType, }, - CanonicalJsonValue, OwnedEventId, OwnedServerName, OwnedUserId, RoomId, ServerName, + CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::Services; @@ -28,7 +28,7 @@ async fn create_join_event( services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, ) -> Result { if !services.rooms.metadata.exists(room_id).await { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + return Err!(Request(NotFound("Room is unknown to this server."))); } // ACL check origin server @@ -45,7 +45,7 @@ async fn create_join_event( .state .get_room_shortstatehash(room_id) .await - .map_err(|_| err!(Request(NotFound("Event state not found."))))?; + .map_err(|e| err!(Request(NotFound(error!("Room has no state: {e}")))))?; // We do not add the event_id field to the pdu here because of signature and // hashes checks @@ -53,53 +53,62 @@ async fn create_join_event( let Ok((event_id, mut value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); + return Err!(Request(BadJson("Could not convert event to canonical json."))); }; + let event_room_id: OwnedRoomId = serde_json::from_value( + serde_json::to_value( + value + .get("room_id") + .ok_or_else(|| err!(Request(BadJson("Event missing room_id property."))))?, + ) + .expect("CanonicalJson is valid json value"), + ) + .map_err(|e| err!(Request(BadJson(warn!("room_id field is not a valid room ID: {e}")))))?; + + if event_room_id != room_id { + return Err!(Request(BadJson("Event room_id does not match request path room ID."))); + } + let event_type: StateEventType = serde_json::from_value( value .get("type") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))? + .ok_or_else(|| err!(Request(BadJson("Event missing type property."))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event has invalid event type."))?; + .map_err(|e| err!(Request(BadJson(warn!("Event has invalid state event type: {e}")))))?; if event_type != StateEventType::RoomMember { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to send non-membership state event to join endpoint.", - )); + return Err!(Request(BadJson( + "Not allowed to send non-membership state event to join endpoint." + ))); } let content: RoomMemberEventContent = serde_json::from_value( value .get("content") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property"))? + .ok_or_else(|| err!(Request(BadJson("Event missing content property"))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid"))?; + .map_err(|e| err!(Request(BadJson(warn!("Event content is empty or invalid: {e}")))))?; if content.membership != MembershipState::Join { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to send a non-join membership event to join endpoint.", - )); + return Err!(Request(BadJson( + "Not allowed to send a non-join membership event to join endpoint." + ))); } - // ACL check sender server name + // ACL check sender user server name let sender: OwnedUserId = serde_json::from_value( value .get("sender") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))? + .ok_or_else(|| err!(Request(BadJson("Event missing sender property."))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "sender is not a valid user ID."))?; + .map_err(|e| err!(Request(BadJson(warn!("sender property is not a valid user ID: {e}")))))?; services .rooms @@ -109,50 +118,71 @@ async fn create_join_event( // check if origin server is trying to send for another server if sender.server_name() != origin { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to join on behalf of another server.", - )); + return Err!(Request(Forbidden("Not allowed to join on behalf of another server."))); } let state_key: OwnedUserId = serde_json::from_value( value .get("state_key") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))? + .ok_or_else(|| err!(Request(BadJson("Event missing state_key property."))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID."))?; + .map_err(|e| err!(Request(BadJson(warn!("State key is not a valid user ID: {e}")))))?; if state_key != sender { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "State key does not match sender user", - )); + return Err!(Request(BadJson("State key does not match sender user."))); }; - if content - .join_authorized_via_users_server - .is_some_and(|user| services.globals.user_is_local(&user)) - && super::user_can_perform_restricted_join(services, &sender, room_id, &room_version_id) + if let Some(authorising_user) = content.join_authorized_via_users_server { + use ruma::RoomVersionId::*; + + if !matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { + return Err!(Request(InvalidParam( + "Room version {room_version_id} does not support restricted rooms but \ + join_authorised_via_users_server ({authorising_user}) was found in the event." + ))); + } + + if !services.globals.user_is_local(&authorising_user) { + return Err!(Request(InvalidParam( + "Cannot authorise membership event through {authorising_user} as they do not belong to this homeserver" + ))); + } + + if !services + .rooms + .state_cache + .is_joined(&authorising_user, room_id) .await - .unwrap_or_default() - { - services - .server_keys - .hash_and_sign_event(&mut value, &room_version_id) - .map_err(|e| err!(Request(InvalidParam("Failed to sign event: {e}"))))?; + { + return Err!(Request(InvalidParam( + "Authorising user {authorising_user} is not in the room you are trying to join, they cannot authorise \ + your join." + ))); + } + + if !super::user_can_perform_restricted_join(services, &state_key, room_id, &room_version_id).await? { + return Err!(Request(UnableToAuthorizeJoin( + "Joining user did not pass restricted room's rules." + ))); + } } + services + .server_keys + .hash_and_sign_event(&mut value, &room_version_id) + .map_err(|e| err!(Request(InvalidParam(warn!("Failed to sign send_join event: {e}")))))?; + let origin: OwnedServerName = serde_json::from_value( serde_json::to_value( value .get("origin") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?, + .ok_or_else(|| err!(Request(BadJson("Event missing origin property."))))?, ) .expect("CanonicalJson is valid json value"), ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; + .map_err(|e| err!(Request(BadJson(warn!("origin field is not a valid server name: {e}")))))?; let mutex_lock = services .rooms @@ -214,7 +244,6 @@ async fn create_join_event( Ok(create_join_event::v1::RoomState { auth_chain, state, - // Event field is required if the room version supports restricted join rules. event: to_raw_value(&CanonicalJsonValue::Object(value)).ok(), }) } @@ -232,14 +261,12 @@ pub(crate) async fn create_join_event_v1_route( .contains(body.origin()) { warn!( - "Server {} tried joining room ID {} who has a server name that is globally forbidden. Rejecting.", + "Server {} tried joining room ID {} through us who has a server name that is globally forbidden. \ + Rejecting.", body.origin(), &body.room_id, ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + return Err!(Request(Forbidden("Server is banned on this homeserver."))); } if let Some(server) = body.room_id.server_name() { @@ -250,14 +277,14 @@ pub(crate) async fn create_join_event_v1_route( .contains(&server.to_owned()) { warn!( - "Server {} tried joining room ID {} which has a server name that is globally forbidden. Rejecting.", + "Server {} tried joining room ID {} through us which has a server name that is globally forbidden. \ + Rejecting.", body.origin(), &body.room_id, ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + return Err!(Request(Forbidden(warn!( + "Room ID server name {server} is banned on this homeserver." + )))); } } @@ -282,10 +309,7 @@ pub(crate) async fn create_join_event_v2_route( .forbidden_remote_server_names .contains(body.origin()) { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + return Err!(Request(Forbidden("Server is banned on this homeserver."))); } if let Some(server) = body.room_id.server_name() { @@ -295,10 +319,15 @@ pub(crate) async fn create_join_event_v2_route( .forbidden_remote_server_names .contains(&server.to_owned()) { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server is banned on this homeserver.", - )); + warn!( + "Server {} tried joining room ID {} through us which has a server name that is globally forbidden. \ + Rejecting.", + body.origin(), + &body.room_id, + ); + return Err!(Request(Forbidden(warn!( + "Room ID server name {server} is banned on this homeserver." + )))); } } diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs index c57998ae..9738c2eb 100644 --- a/src/api/server/send_knock.rs +++ b/src/api/server/send_knock.rs @@ -121,10 +121,7 @@ pub(crate) async fn create_knock_event_v1_route( // check if origin server is trying to send for another server if sender.server_name() != body.origin() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to knock on behalf of another server.", - )); + return Err!(Request(BadJson("Not allowed to knock on behalf of another server/user."))); } let state_key: OwnedUserId = serde_json::from_value( diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index e4f41833..0de485e2 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -1,14 +1,14 @@ #![allow(deprecated)] use axum::extract::State; -use conduit::{err, utils::ReadyExt, Error, Result}; +use conduit::{err, Err, Result}; use ruma::{ - api::{client::error::ErrorKind, federation::membership::create_leave_event}, + api::federation::membership::create_leave_event, events::{ room::member::{MembershipState, RoomMemberEventContent}, StateEventType, }, - OwnedUserId, RoomId, ServerName, + OwnedRoomId, OwnedUserId, RoomId, ServerName, }; use serde_json::value::RawValue as RawJsonValue; @@ -39,11 +39,9 @@ pub(crate) async fn create_leave_event_v2_route( Ok(create_leave_event::v2::Response::new()) } -async fn create_leave_event( - services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, -) -> Result<()> { +async fn create_leave_event(services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue) -> Result { if !services.rooms.metadata.exists(room_id).await { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + return Err!(Request(NotFound("Room is unknown to this server."))); } // ACL check origin @@ -58,53 +56,62 @@ async fn create_leave_event( let room_version_id = services.rooms.state.get_room_version(room_id).await?; let Ok((event_id, value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Could not convert event to canonical json.", - )); + return Err!(Request(BadJson("Could not convert event to canonical json."))); }; + let event_room_id: OwnedRoomId = serde_json::from_value( + serde_json::to_value( + value + .get("room_id") + .ok_or_else(|| err!(Request(BadJson("Event missing room_id property."))))?, + ) + .expect("CanonicalJson is valid json value"), + ) + .map_err(|e| err!(Request(BadJson(warn!("room_id field is not a valid room ID: {e}")))))?; + + if event_room_id != room_id { + return Err!(Request(BadJson("Event room_id does not match request path room ID."))); + } + let content: RoomMemberEventContent = serde_json::from_value( value .get("content") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property"))? + .ok_or_else(|| err!(Request(BadJson("Event missing content property."))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid"))?; + .map_err(|e| err!(Request(BadJson(warn!("Event content is empty or invalid: {e}")))))?; if content.membership != MembershipState::Leave { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to send a non-leave membership event to leave endpoint.", - )); + return Err!(Request(BadJson( + "Not allowed to send a non-leave membership event to leave endpoint." + ))); } let event_type: StateEventType = serde_json::from_value( value .get("type") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))? + .ok_or_else(|| err!(Request(BadJson("Event missing type property."))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Event does not have a valid state event type."))?; + .map_err(|e| err!(Request(BadJson(warn!("Event has invalid state event type: {e}")))))?; if event_type != StateEventType::RoomMember { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to send non-membership state event to leave endpoint.", - )); + return Err!(Request(BadJson( + "Not allowed to send non-membership state event to leave endpoint." + ))); } // ACL check sender server name let sender: OwnedUserId = serde_json::from_value( value .get("sender") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))? + .ok_or_else(|| err!(Request(BadJson("Event missing sender property."))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "User ID in sender is invalid."))?; + .map_err(|e| err!(Request(BadJson(warn!("sender property is not a valid user ID: {e}")))))?; services .rooms @@ -113,26 +120,20 @@ async fn create_leave_event( .await?; if sender.server_name() != origin { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to leave on behalf of another server.", - )); + return Err!(Request(BadJson("Not allowed to leave on behalf of another server/user."))); } let state_key: OwnedUserId = serde_json::from_value( value .get("state_key") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))? + .ok_or_else(|| err!(Request(BadJson("Event missing state_key property."))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID"))?; + .map_err(|e| err!(Request(BadJson(warn!("State key is not a valid user ID: {e}")))))?; if state_key != sender { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "state_key does not match sender user.", - )); + return Err!(Request(BadJson("State key does not match sender user."))); } let mutex_lock = services @@ -151,11 +152,5 @@ async fn create_leave_event( drop(mutex_lock); - let servers = services - .rooms - .state_cache - .room_servers(room_id) - .ready_filter(|server| !services.globals.server_is_ours(server)); - - services.sending.send_pdu_servers(servers, &pdu_id).await + services.sending.send_pdu_room(room_id, &pdu_id).await } diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index df5852c6..ff1837db 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -227,7 +227,7 @@ impl Service { for action in self .get_actions(user, &ruleset, &power_levels, &pdu.to_sync_room_event(), &pdu.room_id) - .await? + .await { let n = match action { Action::Notify => true, @@ -259,7 +259,7 @@ impl Service { pub async fn get_actions<'a>( &self, user: &UserId, ruleset: &'a Ruleset, power_levels: &RoomPowerLevelsEventContent, pdu: &Raw, room_id: &RoomId, - ) -> Result<&'a [Action]> { + ) -> &'a [Action] { let power_levels = PushConditionPowerLevelsCtx { users: power_levels.users.clone(), users_default: power_levels.users_default, @@ -290,7 +290,7 @@ impl Service { power_levels: Some(power_levels), }; - Ok(ruleset.get_actions(pdu, &ctx)) + ruleset.get_actions(pdu, &ctx) } #[tracing::instrument(skip(self, unread, pusher, tweaks, event))] diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 4a33224e..cbd72cdb 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -4,7 +4,7 @@ use std::{ }; use conduit::{ - err, is_not_empty, + is_not_empty, result::LogErr, utils::{stream::TryIgnore, ReadyExt, StreamTools}, warn, Result, @@ -600,11 +600,11 @@ impl Service { .map(|(_, servers): KeyVal<'_>| *servers.last().expect("at least one server")) } - /// Gets up to three servers that are likely to be in the room in the + /// Gets up to five servers that are likely to be in the room in the /// distant future. /// - /// See - #[tracing::instrument(skip(self))] + /// See + #[tracing::instrument(skip(self), level = "debug")] pub async fn servers_route_via(&self, room_id: &RoomId) -> Result> { let most_powerful_user_server = self .services @@ -618,8 +618,7 @@ impl Service { .max_by_key(|(_, power)| *power) .and_then(|x| (x.1 >= &int!(50)).then_some(x)) .map(|(user, _power)| user.server_name().to_owned()) - }) - .map_err(|e| err!(Database(error!(?e, "Invalid power levels event content in database."))))?; + }); let mut servers: Vec = self .room_members(room_id) @@ -629,12 +628,12 @@ impl Service { .sorted_by_key(|(_, users)| *users) .map(|(server, _)| server) .rev() - .take(3) + .take(5) .collect(); - if let Some(server) = most_powerful_user_server { + if let Ok(Some(server)) = most_powerful_user_server { servers.insert(0, server); - servers.truncate(3); + servers.truncate(5); } Ok(servers) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index b9fcdcd2..8603b7e7 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -417,7 +417,7 @@ impl Service { .services .pusher .get_actions(user, &rules_for_user, &power_levels, &sync_pdu, &pdu.room_id) - .await? + .await { match action { Action::Notify => notify = true, @@ -769,10 +769,8 @@ impl Service { } // Hash and sign - let mut pdu_json = utils::to_canonical_object(&pdu).map_err(|e| { - error!("Failed to convert PDU to canonical JSON: {e}"); - Error::bad_database("Failed to convert PDU to canonical JSON.") - })?; + let mut pdu_json = utils::to_canonical_object(&pdu) + .map_err(|e| err!(Request(BadJson(warn!("Failed to convert PDU to canonical JSON: {e}")))))?; // room v3 and above removed the "event_id" field from remote PDU format match room_version_id { @@ -794,8 +792,10 @@ impl Service { .hash_and_sign_event(&mut pdu_json, &room_version_id) { return match e { - Error::Signatures(ruma::signatures::Error::PduSize) => Err!(Request(TooLarge("Message is too long"))), - _ => Err!(Request(Unknown("Signing event failed"))), + Error::Signatures(ruma::signatures::Error::PduSize) => { + Err!(Request(TooLarge("Message/PDU is too long (exceeds 65535 bytes)"))) + }, + _ => Err!(Request(Unknown(warn!("Signing event failed: {e}")))), }; } From ab4a283870f6e0e986d818872d7c3c62efdefc17 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 8 Dec 2024 19:01:41 -0500 Subject: [PATCH 1194/2091] update complement config Signed-off-by: strawberry --- nix/pkgs/complement/config.toml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index 631100fd..f20abee2 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -9,15 +9,27 @@ database_path = "/database" log = "trace,h2=warn,hyper=warn" port = [8008, 8448] trusted_servers = [] +only_query_trusted_key_servers = false query_trusted_key_servers_first = false +query_trusted_key_servers_first_on_join = false yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true ip_range_denylist = [] url_preview_domain_contains_allowlist = ["*"] +url_preview_domain_explicit_denylist = ["*"] media_compat_file_link = false media_startup_check = false -rocksdb_direct_io = false +prune_missing_media = false log_colors = false admin_room_notices = false +allow_check_for_updates = false +allow_unstable_room_versions = true +rocksdb_log_level = "debug" +rocksdb_max_log_files = 1 +rocksdb_recovery_mode = 0 +rocksdb_paranoid_file_checks = true +log_guest_registrations = false +allow_legacy_media = true +startup_netburst = false [global.tls] certs = "/certificate.crt" From faf48405aa3ce491e142196b121268fa76b7a325 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 8 Dec 2024 20:40:03 -0500 Subject: [PATCH 1195/2091] bump rocksdb to v9.8.4, enable track_and_verify_wals_in_manifest by default Signed-off-by: strawberry --- Cargo.lock | 8 ++++---- deps/rust-rocksdb/Cargo.toml | 2 +- flake.lock | 8 ++++---- flake.nix | 2 +- src/database/opts.rs | 6 ++++++ 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fc8dc26d..e766fe46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3390,8 +3390,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.29.0+9.7.4" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=e2b8853ee8839dc886e88fd751fe50ed1c27b47e#e2b8853ee8839dc886e88fd751fe50ed1c27b47e" +version = "0.30.0+9.8.4" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=3a9640a6b9173d24819c22a49487e31d20a2e59e#3a9640a6b9173d24819c22a49487e31d20a2e59e" dependencies = [ "bindgen", "bzip2-sys", @@ -3407,8 +3407,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.33.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=e2b8853ee8839dc886e88fd751fe50ed1c27b47e#e2b8853ee8839dc886e88fd751fe50ed1c27b47e" +version = "0.34.0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=3a9640a6b9173d24819c22a49487e31d20a2e59e#3a9640a6b9173d24819c22a49487e31d20a2e59e" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index c6c3fea5..b6895ad3 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "e2b8853ee8839dc886e88fd751fe50ed1c27b47e" +rev = "3a9640a6b9173d24819c22a49487e31d20a2e59e" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index 7740e925..6049fded 100644 --- a/flake.lock +++ b/flake.lock @@ -922,16 +922,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1731690620, - "narHash": "sha256-Xd4TJYqPERMJLXaGa6r6Ny1Wlw8Uy5Cyf/8q7nS58QM=", + "lastModified": 1733704887, + "narHash": "sha256-4ijNmXACyTJWKRcTdlgObdbOVm2oN3Zefg55/4UPIL0=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "292446aa2bc41699204d817a1e4b091679a886eb", + "rev": "9c656e58c9f969aea28f25b22bc52ea03109677a", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.7.4", + "ref": "v9.8.4", "repo": "rocksdb", "type": "github" } diff --git a/flake.nix b/flake.nix index 281052a8..4eb72731 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.7.4"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.8.4"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; diff --git a/src/database/opts.rs b/src/database/opts.rs index d0ae618c..b0b33927 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -95,6 +95,12 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ 4_u8..=u8::MAX => unimplemented!(), }); + // + // "We recommend to set track_and_verify_wals_in_manifest to true for + // production, it has been enabled in production for the entire database cluster + // serving the social graph for all Meta apps." + opts.set_track_and_verify_wals_in_manifest(true); + opts.set_paranoid_checks(config.rocksdb_paranoid_file_checks); opts.set_env(env); From e1bd6a7c2d28853c4f19cd32e283b8c76434e15e Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 9 Dec 2024 12:31:08 -0500 Subject: [PATCH 1196/2091] nix: try bumping to latest complement Signed-off-by: strawberry --- flake.lock | 6 +++--- flake.nix | 7 +++---- nix/pkgs/main/default.nix | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/flake.lock b/flake.lock index 6049fded..56675fb0 100644 --- a/flake.lock +++ b/flake.lock @@ -120,11 +120,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1724347376, - "narHash": "sha256-y0e/ULDJ92IhNQZsS/06g0s+AYZ82aJfrIO9qEse94c=", + "lastModified": 1732612157, + "narHash": "sha256-uqQfVmwCUrsw7eHs9przqnhMoyx4r7WUU8AWfT2Buo4=", "owner": "matrix-org", "repo": "complement", - "rev": "39733c1b2f8314800776748cc7164f9a34650686", + "rev": "fc63446512261a496b794a7082a6598b6f98e925", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 4eb72731..817f4e6d 100644 --- a/flake.nix +++ b/flake.nix @@ -18,7 +18,6 @@ let pkgsHost = import inputs.nixpkgs{ inherit system; - config.permittedInsecurePackages = [ "olm-3.2.16" ]; }; pkgsHostStatic = pkgsHost.pkgsStatic; @@ -118,9 +117,9 @@ # code. COMPLEMENT_SRC = inputs.complement.outPath; - # Needed for Complement - CGO_CFLAGS = "-I${scope.pkgs.olm}/include"; - CGO_LDFLAGS = "-L${scope.pkgs.olm}/lib"; + # Needed for Complement: + CGO_CFLAGS = "-Wl,--no-gc-sections"; + CGO_LDFLAGS = "-Wl,--no-gc-sections"; }; # Development tools diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index d11c36cc..c40af176 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -176,7 +176,7 @@ commonAttrs = { # # postInstall = with pkgsBuildHost; '' - find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${libgcc} -t ${llvm} -t ${libllvm} -t ${rustc.unwrapped} -t ${rustc} -t ${libidn2} -t ${libunistring} '{}' + + find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${llvm} -t ${rustc.unwrapped} -t ${rustc} '{}' + ''; }; in From b6ac3649eea5e7b9d335ecb47e902654875b16b0 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 9 Dec 2024 13:04:13 -0500 Subject: [PATCH 1197/2091] use fork of complement Signed-off-by: strawberry --- flake.lock | 10 +++++----- flake.nix | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 56675fb0..ea067333 100644 --- a/flake.lock +++ b/flake.lock @@ -120,15 +120,15 @@ "complement": { "flake": false, "locked": { - "lastModified": 1732612157, - "narHash": "sha256-uqQfVmwCUrsw7eHs9przqnhMoyx4r7WUU8AWfT2Buo4=", - "owner": "matrix-org", + "lastModified": 1733767359, + "narHash": "sha256-3ZBFqvmTSE5Rtbb0Co89aut7q4ERBAeP7HntLN6sFik=", + "owner": "girlbossceo", "repo": "complement", - "rev": "fc63446512261a496b794a7082a6598b6f98e925", + "rev": "6ec9a3a28fb2baec9afb4fc8b9974a5f5d6525fc", "type": "github" }, "original": { - "owner": "matrix-org", + "owner": "girlbossceo", "ref": "main", "repo": "complement", "type": "github" diff --git a/flake.nix b/flake.nix index 817f4e6d..e4ba8e3d 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ inputs = { attic.url = "github:zhaofengli/attic?ref=main"; cachix.url = "github:cachix/cachix?ref=master"; - complement = { url = "github:matrix-org/complement?ref=main"; flake = false; }; + complement = { url = "github:girlbossceo/complement?ref=main"; flake = false; }; crane = { url = "github:ipetkov/crane?ref=master"; }; fenix = { url = "github:nix-community/fenix?ref=main"; inputs.nixpkgs.follows = "nixpkgs"; }; flake-compat = { url = "github:edolstra/flake-compat?ref=master"; flake = false; }; From 0ff37b829cc04ad84c7769f8676078cc13f7c6ca Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 9 Dec 2024 13:33:24 -0500 Subject: [PATCH 1198/2091] update default systemd units to use network-online.target fully Signed-off-by: strawberry --- arch/conduwuit.service | 3 ++- debian/conduwuit.service | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index b343c91a..7c05c259 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -1,6 +1,7 @@ [Unit] Description=conduwuit Matrix homeserver -After=network.target +Wants=network-online.target +After=network-online.target Documentation=https://conduwuit.puppyirl.gay/ RequiresMountsFor=/var/lib/private/conduwuit diff --git a/debian/conduwuit.service b/debian/conduwuit.service index d1b0d5d6..3c2ec49d 100644 --- a/debian/conduwuit.service +++ b/debian/conduwuit.service @@ -1,7 +1,8 @@ [Unit] Description=conduwuit Matrix homeserver -Documentation=https://conduwuit.puppyirl.gay/ +Wants=network-online.target After=network-online.target +Documentation=https://conduwuit.puppyirl.gay/ [Service] DynamicUser=yes From 9cb71e212ab44d27de1cf84650d7e0f0c5c9e90f Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 9 Dec 2024 17:03:54 -0500 Subject: [PATCH 1199/2091] nix: bump flake.lock finally after 3 months Signed-off-by: strawberry --- flake.lock | 598 +++++------------------- nix/pkgs/main/cross-compilation-env.nix | 35 +- nix/pkgs/main/default.nix | 2 - 3 files changed, 121 insertions(+), 514 deletions(-) diff --git a/flake.lock b/flake.lock index ea067333..25d6de81 100644 --- a/flake.lock +++ b/flake.lock @@ -5,15 +5,16 @@ "crane": "crane", "flake-compat": "flake-compat", "flake-parts": "flake-parts", + "nix-github-actions": "nix-github-actions", "nixpkgs": "nixpkgs", "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1729116596, - "narHash": "sha256-NnLMLIXGZtAscUF4dCShksuQ1nOGF6Y2dEeyj0rBbUg=", + "lastModified": 1731270564, + "narHash": "sha256-6KMC/NH/VWP5Eb+hA56hz0urel3jP6Y6cF2PX6xaTkk=", "owner": "zhaofengli", "repo": "attic", - "rev": "2b05b7d986cf6009b1c1ef7daa4961cd1a658782", + "rev": "47752427561f1c34debb16728a210d378f0ece36", "type": "github" }, "original": { @@ -26,16 +27,16 @@ "cachix": { "inputs": { "devenv": "devenv", - "flake-compat": "flake-compat_3", + "flake-compat": "flake-compat_2", "git-hooks": "git-hooks", "nixpkgs": "nixpkgs_4" }, "locked": { - "lastModified": 1728672398, - "narHash": "sha256-KxuGSoVUFnQLB2ZcYODW7AVPAh9JqRlD5BrfsC/Q4qs=", + "lastModified": 1733424942, + "narHash": "sha256-5t7Sl6EkOaoP4FvzLmH7HFDbdl9SizmLh53RjDQCbWQ=", "owner": "cachix", "repo": "cachix", - "rev": "aac51f698309fd0f381149214b7eee213c66ef0a", + "rev": "8b6b0e4694b9aa78b2ea4c93bff6e1a222dc7e4a", "type": "github" }, "original": { @@ -47,72 +48,31 @@ }, "cachix_2": { "inputs": { - "devenv": "devenv_2", + "devenv": [ + "cachix", + "devenv" + ], "flake-compat": [ "cachix", - "devenv", - "flake-compat" + "devenv" ], "git-hooks": [ "cachix", - "devenv", - "pre-commit-hooks" + "devenv" ], - "nixpkgs": [ - "cachix", - "devenv", - "nixpkgs" - ] + "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1726520618, - "narHash": "sha256-jOsaBmJ/EtX5t/vbylCdS7pWYcKGmWOKg4QKUzKr6dA=", + "lastModified": 1728672398, + "narHash": "sha256-KxuGSoVUFnQLB2ZcYODW7AVPAh9JqRlD5BrfsC/Q4qs=", "owner": "cachix", "repo": "cachix", - "rev": "695525f9086542dfb09fde0871dbf4174abbf634", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "cachix", - "type": "github" - } - }, - "cachix_3": { - "inputs": { - "devenv": "devenv_3", - "flake-compat": [ - "cachix", - "devenv", - "cachix", - "devenv", - "flake-compat" - ], - "nixpkgs": [ - "cachix", - "devenv", - "cachix", - "devenv", - "nixpkgs" - ], - "pre-commit-hooks": [ - "cachix", - "devenv", - "cachix", - "devenv", - "pre-commit-hooks" - ] - }, - "locked": { - "lastModified": 1712055811, - "narHash": "sha256-7FcfMm5A/f02yyzuavJe06zLa9hcMHsagE28ADcmQvk=", - "owner": "cachix", - "repo": "cachix", - "rev": "02e38da89851ec7fec3356a5c04bc8349cae0e30", + "rev": "aac51f698309fd0f381149214b7eee213c66ef0a", "type": "github" }, "original": { "owner": "cachix", + "ref": "latest", "repo": "cachix", "type": "github" } @@ -157,11 +117,11 @@ }, "crane_2": { "locked": { - "lastModified": 1729741221, - "narHash": "sha256-8AHZZXs1lFkERfBY0C8cZGElSo33D/et7NKEpLRmvzo=", + "lastModified": 1733688869, + "narHash": "sha256-KrhxxFj1CjESDrL5+u/zsVH0K+Ik9tvoac/oFPoxSB8=", "owner": "ipetkov", "repo": "crane", - "rev": "f235b656ee5b2bfd6d94c3bfd67896a575d4a6ed", + "rev": "604637106e420ad99907cae401e13ab6b452e7d9", "type": "github" }, "original": { @@ -178,100 +138,26 @@ "cachix", "flake-compat" ], - "nix": "nix_3", - "nixpkgs": [ - "cachix", - "nixpkgs" - ], - "pre-commit-hooks": [ + "git-hooks": [ "cachix", "git-hooks" - ] - }, - "locked": { - "lastModified": 1727963652, - "narHash": "sha256-os0EDjn7QVXL6RtHNb9TrZLXVm2Tc5/nZKk3KpbTzd8=", - "owner": "cachix", - "repo": "devenv", - "rev": "cb0052e25dbcc8267b3026160dc73cddaac7d5fd", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "devenv", - "type": "github" - } - }, - "devenv_2": { - "inputs": { - "cachix": "cachix_3", - "flake-compat": [ - "cachix", - "devenv", - "cachix", - "flake-compat" - ], - "nix": "nix_2", - "nixpkgs": [ - "cachix", - "devenv", - "cachix", - "nixpkgs" - ], - "pre-commit-hooks": [ - "cachix", - "devenv", - "cachix", - "git-hooks" - ] - }, - "locked": { - "lastModified": 1723156315, - "narHash": "sha256-0JrfahRMJ37Rf1i0iOOn+8Z4CLvbcGNwa2ChOAVrp/8=", - "owner": "cachix", - "repo": "devenv", - "rev": "ff5eb4f2accbcda963af67f1a1159e3f6c7f5f91", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "devenv", - "type": "github" - } - }, - "devenv_3": { - "inputs": { - "flake-compat": [ - "cachix", - "devenv", - "cachix", - "devenv", - "cachix", - "flake-compat" ], "nix": "nix", - "nixpkgs": "nixpkgs_2", - "poetry2nix": "poetry2nix", - "pre-commit-hooks": [ + "nixpkgs": [ "cachix", - "devenv", - "cachix", - "devenv", - "cachix", - "pre-commit-hooks" + "nixpkgs" ] }, "locked": { - "lastModified": 1708704632, - "narHash": "sha256-w+dOIW60FKMaHI1q5714CSibk99JfYxm0CzTinYWr+Q=", + "lastModified": 1733323168, + "narHash": "sha256-d5DwB4MZvlaQpN6OQ4SLYxb5jA4UH5EtV5t5WOtjLPU=", "owner": "cachix", "repo": "devenv", - "rev": "2ee4450b0f4b95a1b90f2eb5ffea98b90e48c196", + "rev": "efa9010b8b1cfd5dd3c7ed1e172a470c3b84a064", "type": "github" }, "original": { "owner": "cachix", - "ref": "python-rewrite", "repo": "devenv", "type": "github" } @@ -284,11 +170,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1729751566, - "narHash": "sha256-99u/hrgBdi8bxSXZc9ZbNkR5EL1htrkbd3lsbKzS60g=", + "lastModified": 1733726208, + "narHash": "sha256-Z6zL4AtpZWxkvTd3l0KkPZamu2wtTKl4nNiqNSlgsb4=", "owner": "nix-community", "repo": "fenix", - "rev": "f32a2d484091a6dc98220b1f4a2c2d60b7c97c64", + "rev": "d51a64e1d23e509f28a6955a6652cc62409dd4a8", "type": "github" }, "original": { @@ -317,11 +203,11 @@ "flake-compat_2": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", "type": "github" }, "original": { @@ -333,27 +219,11 @@ "flake-compat_3": { "flake": false, "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", "owner": "edolstra", "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "flake-compat_4": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", "type": "github" }, "original": { @@ -412,44 +282,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1689068808, - "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", - "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_2": { - "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "flake-utils_3": { - "inputs": { - "systems": "systems_2" - }, - "locked": { - "lastModified": 1710146030, - "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { @@ -473,11 +310,11 @@ "nixpkgs-stable": "nixpkgs-stable_2" }, "locked": { - "lastModified": 1727854478, - "narHash": "sha256-/odH2nUMAwkMgOS2nG2z0exLQNJS4S2LfMW0teqU7co=", + "lastModified": 1733318908, + "narHash": "sha256-SVQVsbafSM1dJ4fpgyBqLZ+Lft+jcQuMtEL3lQWx2Sk=", "owner": "cachix", "repo": "git-hooks.nix", - "rev": "5f58871c9657b5fc0a7f65670fe2ba99c26c1d79", + "rev": "6f4e2a2112050951a314d2733a994fbab94864c6", "type": "github" }, "original": { @@ -527,11 +364,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1725659644, - "narHash": "sha256-WjnpmopfvFoUbubIu9bki+Y6P4YXDfvnW4+72hniq3g=", + "lastModified": 1733603756, + "narHash": "sha256-eTKnZDZ1Ex++v+BI0DBcUBmCXAO/tE8hxK9MiyztZkU=", "owner": "axboe", "repo": "liburing", - "rev": "0fe5c09195c0918f89582dd6ff098a58a0bdf62a", + "rev": "c3d5d6270cd5ed48d817fc1e8e95f7c8b222f2ff", "type": "github" }, "original": { @@ -542,123 +379,26 @@ } }, "nix": { - "inputs": { - "flake-compat": "flake-compat_2", - "nixpkgs": [ - "cachix", - "devenv", - "cachix", - "devenv", - "cachix", - "devenv", - "nixpkgs" - ], - "nixpkgs-regression": "nixpkgs-regression" - }, - "locked": { - "lastModified": 1712911606, - "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", - "owner": "domenkozar", - "repo": "nix", - "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", - "type": "github" - }, - "original": { - "owner": "domenkozar", - "ref": "devenv-2.21", - "repo": "nix", - "type": "github" - } - }, - "nix-filter": { - "locked": { - "lastModified": 1710156097, - "narHash": "sha256-1Wvk8UP7PXdf8bCCaEoMnOT1qe5/Duqgj+rL8sRQsSM=", - "owner": "numtide", - "repo": "nix-filter", - "rev": "3342559a24e85fc164b295c3444e8a139924675b", - "type": "github" - }, - "original": { - "owner": "numtide", - "ref": "main", - "repo": "nix-filter", - "type": "github" - } - }, - "nix-github-actions": { - "inputs": { - "nixpkgs": [ - "cachix", - "devenv", - "cachix", - "devenv", - "cachix", - "devenv", - "poetry2nix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1688870561, - "narHash": "sha256-4UYkifnPEw1nAzqqPOTL2MvWtm3sNGw1UTYTalkTcGY=", - "owner": "nix-community", - "repo": "nix-github-actions", - "rev": "165b1650b753316aa7f1787f3005a8d2da0f5301", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "nix-github-actions", - "type": "github" - } - }, - "nix_2": { "inputs": { "flake-compat": [ "cachix", - "devenv", - "cachix", - "devenv", - "flake-compat" - ], - "nixpkgs": [ - "cachix", - "devenv", - "cachix", - "devenv", - "nixpkgs" - ], - "nixpkgs-regression": "nixpkgs-regression_2" - }, - "locked": { - "lastModified": 1712911606, - "narHash": "sha256-BGvBhepCufsjcUkXnEEXhEVjwdJAwPglCC2+bInc794=", - "owner": "domenkozar", - "repo": "nix", - "rev": "b24a9318ea3f3600c1e24b4a00691ee912d4de12", - "type": "github" - }, - "original": { - "owner": "domenkozar", - "ref": "devenv-2.21", - "repo": "nix", - "type": "github" - } - }, - "nix_3": { - "inputs": { - "flake-compat": [ - "cachix", - "devenv", - "flake-compat" + "devenv" ], "flake-parts": "flake-parts_2", "libgit2": "libgit2", "nixpkgs": "nixpkgs_3", - "nixpkgs-23-11": "nixpkgs-23-11", - "nixpkgs-regression": "nixpkgs-regression_3", - "pre-commit-hooks": "pre-commit-hooks" + "nixpkgs-23-11": [ + "cachix", + "devenv" + ], + "nixpkgs-regression": [ + "cachix", + "devenv" + ], + "pre-commit-hooks": [ + "cachix", + "devenv" + ] }, "locked": { "lastModified": 1727438425, @@ -675,6 +415,43 @@ "type": "github" } }, + "nix-filter": { + "locked": { + "lastModified": 1731533336, + "narHash": "sha256-oRam5PS1vcrr5UPgALW0eo1m/5/pls27Z/pabHNy2Ms=", + "owner": "numtide", + "repo": "nix-filter", + "rev": "f7653272fd234696ae94229839a99b73c9ab7de0", + "type": "github" + }, + "original": { + "owner": "numtide", + "ref": "main", + "repo": "nix-filter", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "attic", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1729742964, + "narHash": "sha256-B4mzTcQ0FZHdpeWcpDYPERtyjJd/NIuaQ9+BV1h+MpA=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "e04df33f62cdcf93d73e9a04142464753a16db67", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1726042813, @@ -691,70 +468,6 @@ "type": "github" } }, - "nixpkgs-23-11": { - "locked": { - "lastModified": 1717159533, - "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", - "type": "github" - } - }, - "nixpkgs-regression": { - "locked": { - "lastModified": 1643052045, - "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - } - }, - "nixpkgs-regression_2": { - "locked": { - "lastModified": 1643052045, - "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - } - }, - "nixpkgs-regression_3": { - "locked": { - "lastModified": 1643052045, - "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", - "type": "github" - } - }, "nixpkgs-stable": { "locked": { "lastModified": 1724316499, @@ -773,11 +486,11 @@ }, "nixpkgs-stable_2": { "locked": { - "lastModified": 1720386169, - "narHash": "sha256-NGKVY4PjzwAa4upkGtAMz1npHGoRzWotlSnVlqI40mo=", + "lastModified": 1730741070, + "narHash": "sha256-edm8WG19kWozJ/GqyYx2VjW99EdhjKwbY3ZwdlPAAlo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "194846768975b7ad2c4988bdb82572c00222c0d7", + "rev": "d063c1dd113c91ab27959ba540c0d9753409edf3", "type": "github" }, "original": { @@ -789,16 +502,16 @@ }, "nixpkgs_2": { "locked": { - "lastModified": 1692808169, - "narHash": "sha256-x9Opq06rIiwdwGeK2Ykj69dNc2IvUH1fY55Wm7atwrE=", + "lastModified": 1730531603, + "narHash": "sha256-Dqg6si5CqIzm87sp57j5nTaeBbWhHFaVyG7V6L8k3lY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9201b5ff357e781bf014d0330d18555695df7ba8", + "rev": "7ffd9ae656aec493492b44d0ddfb28e79a1ea25d", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "nixos-unstable", "repo": "nixpkgs", "type": "github" } @@ -821,11 +534,11 @@ }, "nixpkgs_4": { "locked": { - "lastModified": 1727802920, - "narHash": "sha256-HP89HZOT0ReIbI7IJZJQoJgxvB2Tn28V6XS3MNKnfLs=", + "lastModified": 1733212471, + "narHash": "sha256-M1+uCoV5igihRfcUKrr1riygbe73/dzNnzPsmaLCmpo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "27e30d177e57d912d614c88c622dcfdb2e6e6515", + "rev": "55d15ad12a74eb7d4646254e13638ad0c4128776", "type": "github" }, "original": { @@ -837,11 +550,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1725534445, - "narHash": "sha256-Yd0FK9SkWy+ZPuNqUgmVPXokxDgMJoGuNpMEtkfcf84=", + "lastModified": 1733656523, + "narHash": "sha256-w0FXPfpGhOihoJDiwMsyN1EzpsXi2F8VQ+NVZQSMtys=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "9bb1e7571aadf31ddb4af77fc64b2d59580f9a39", + "rev": "93dc9803a1ee435e590b02cde9589038d5cc3a4e", "type": "github" }, "original": { @@ -851,74 +564,6 @@ "type": "github" } }, - "poetry2nix": { - "inputs": { - "flake-utils": "flake-utils", - "nix-github-actions": "nix-github-actions", - "nixpkgs": [ - "cachix", - "devenv", - "cachix", - "devenv", - "cachix", - "devenv", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1692876271, - "narHash": "sha256-IXfZEkI0Mal5y1jr6IRWMqK8GW2/f28xJenZIPQqkY0=", - "owner": "nix-community", - "repo": "poetry2nix", - "rev": "d5006be9c2c2417dafb2e2e5034d83fabd207ee3", - "type": "github" - }, - "original": { - "owner": "nix-community", - "repo": "poetry2nix", - "type": "github" - } - }, - "pre-commit-hooks": { - "inputs": { - "flake-compat": [ - "cachix", - "devenv", - "nix" - ], - "flake-utils": "flake-utils_2", - "gitignore": [ - "cachix", - "devenv", - "nix" - ], - "nixpkgs": [ - "cachix", - "devenv", - "nix", - "nixpkgs" - ], - "nixpkgs-stable": [ - "cachix", - "devenv", - "nix", - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1712897695, - "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "type": "github" - } - }, "rocksdb": { "flake": false, "locked": { @@ -943,8 +588,8 @@ "complement": "complement", "crane": "crane_2", "fenix": "fenix", - "flake-compat": "flake-compat_4", - "flake-utils": "flake-utils_3", + "flake-compat": "flake-compat_3", + "flake-utils": "flake-utils", "liburing": "liburing", "nix-filter": "nix-filter", "nixpkgs": "nixpkgs_5", @@ -954,11 +599,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1729715509, - "narHash": "sha256-jUDN4e1kObbksb4sc+57NEeujBEDRdLCOu9wiE3RZdM=", + "lastModified": 1733642337, + "narHash": "sha256-I1uc97f/cNhOpCemIbBAUS+CV0R7jts0NW9lc8jrpxc=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "40492e15d49b89cf409e2c5536444131fac49429", + "rev": "4c755e62a617eeeef3066994731ce1cdd16504ac", "type": "github" }, "original": { @@ -982,21 +627,6 @@ "repo": "default", "type": "github" } - }, - "systems_2": { - "locked": { - "lastModified": 1681028828, - "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", - "owner": "nix-systems", - "repo": "default", - "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", - "type": "github" - }, - "original": { - "owner": "nix-systems", - "repo": "default", - "type": "github" - } } }, "root": "root", diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index c189d57c..3fb94d41 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -1,6 +1,5 @@ { lib , pkgsBuildHost -, pkgsBuildTarget , rust , stdenv }: @@ -23,25 +22,13 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { [ "-C" "relocation-model=static" ] ++ lib.optionals (stdenv.buildPlatform.config != stdenv.hostPlatform.config) - [ "-l" "c" ] - ++ lib.optionals - # This check has to match the one [here][0]. We only need to set - # these flags when using a different linker. Don't ask me why, - # though, because I don't know. All I know is it breaks otherwise. - # - # [0]: https://github.com/NixOS/nixpkgs/blob/5cdb38bb16c6d0a38779db14fcc766bc1b2394d6/pkgs/build-support/rust/lib/default.nix#L37-L40 - ( - # Nixpkgs doesn't check for x86_64 here but we do, because I - # observed a failure building statically for x86_64 without - # including it here. Linkers are weird. - (stdenv.hostPlatform.isAarch64 || stdenv.hostPlatform.isx86_64) - && stdenv.hostPlatform.isStatic - && !stdenv.hostPlatform.isDarwin - && !stdenv.cc.bintools.isLLVM - ) [ + "-l" + "c" + "-l" "stdc++" + "-L" "${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib" ] @@ -58,7 +45,7 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { ( let inherit (rust.lib) envVars; - shouldUseLLD = platform: platform.isAarch64 && platform.isStatic && !stdenv.hostPlatform.isDarwin; + #shouldUseLLD = platform: platform.isAarch64 && platform.isStatic && !stdenv.hostPlatform.isDarwin; in lib.optionalAttrs (stdenv.targetPlatform.rust.rustcTarget @@ -66,30 +53,22 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { ( let inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget; - linkerForTarget = if shouldUseLLD stdenv.targetPlatform - && !stdenv.cc.bintools.isLLVM # whether stdenv's linker is lld already - then "${pkgsBuildTarget.llvmPackages.bintools}/bin/${stdenv.cc.targetPrefix}ld.lld" - else envVars.ccForTarget; in { "CC_${cargoEnvVarTarget}" = envVars.ccForTarget; "CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = linkerForTarget; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForTarget; } ) // ( let inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget; - linkerForHost = if shouldUseLLD stdenv.targetPlatform - && !stdenv.cc.bintools.isLLVM - then "${pkgsBuildHost.llvmPackages.bintools}/bin/${stdenv.cc.targetPrefix}ld.lld" - else envVars.ccForHost; in { "CC_${cargoEnvVarTarget}" = envVars.ccForHost; "CXX_${cargoEnvVarTarget}" = envVars.cxxForHost; - "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = linkerForHost; + "CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForHost; CARGO_BUILD_TARGET = rustcTarget; } ) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index c40af176..abc85d82 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -6,7 +6,6 @@ , libiconv , liburing , pkgsBuildHost -, pkgsBuildTarget , rocksdb , removeReferencesTo , rust @@ -96,7 +95,6 @@ buildDepsOnlyEnv = inherit lib pkgsBuildHost - pkgsBuildTarget rust stdenv; }); From cd67ca6c57420f0c9550ce1861926dfd427b202a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 10 Dec 2024 04:15:24 +0000 Subject: [PATCH 1200/2091] fix .gitignore to allow /target to be a symlink Signed-off-by: Jason Volk --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index cf366522..b5fea66b 100644 --- a/.gitignore +++ b/.gitignore @@ -30,7 +30,7 @@ modules.xml .nfs* # Rust -/target/ +/target ### vscode ### .vscode/* From 5fc8e90e0231214a5b52ed0a4320d071f07b559f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 10 Dec 2024 05:33:04 +0000 Subject: [PATCH 1201/2091] fix runtime metrics command Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index d4c9a57b..a17d0800 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -773,7 +773,14 @@ pub(super) async fn memory_stats(&self) -> Result { pub(super) async fn runtime_metrics(&self) -> Result { let out = self.services.server.metrics.runtime_metrics().map_or_else( || "Runtime metrics are not available.".to_owned(), - |metrics| format!("```rs\n{metrics:#?}\n```"), + |metrics| { + format!( + "```rs\nnum_workers: {}\nnum_alive_tasks: {}\nglobal_queue_depth: {}\n```", + metrics.num_workers(), + metrics.num_alive_tasks(), + metrics.global_queue_depth() + ) + }, ); Ok(RoomMessageEventContent::text_markdown(out)) From aba88ccead6afef9197683890bc2bb279295db19 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 20:31:16 +0000 Subject: [PATCH 1202/2091] misc tracing span tweaks Signed-off-by: Jason Volk --- src/api/client/media.rs | 35 +++++++++++++++++++++---- src/api/client/sync/v3.rs | 19 +++++++++++++- src/database/engine.rs | 7 ++++- src/database/map/get_batch.rs | 2 +- src/database/maps.rs | 2 +- src/database/pool.rs | 17 +++++++++--- src/main/main.rs | 5 ++++ src/router/layers.rs | 4 +-- src/router/request.rs | 8 ++++-- src/router/serve/unix.rs | 2 +- src/service/manager.rs | 6 +++++ src/service/resolver/actual.rs | 2 +- src/service/rooms/state_accessor/mod.rs | 6 ++--- src/service/sending/data.rs | 1 - src/service/sending/send.rs | 2 +- src/service/sending/sender.rs | 2 +- 16 files changed, 96 insertions(+), 24 deletions(-) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 71693618..67a2bbdf 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -39,7 +39,12 @@ pub(crate) async fn get_media_config_route( /// /// - Some metadata will be saved in the database /// - Media will be saved in the media/ directory -#[tracing::instrument(skip_all, fields(%client), name = "media_upload")] +#[tracing::instrument( + name = "media_upload", + level = "debug", + skip_all, + fields(%client), +)] pub(crate) async fn create_content_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, @@ -67,7 +72,12 @@ pub(crate) async fn create_content_route( /// # `GET /_matrix/client/v1/media/thumbnail/{serverName}/{mediaId}` /// /// Load media thumbnail from our server or over federation. -#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] +#[tracing::instrument( + name = "media_thumbnail_get", + level = "debug", + skip_all, + fields(%client), +)] pub(crate) async fn get_content_thumbnail_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, @@ -98,7 +108,12 @@ pub(crate) async fn get_content_thumbnail_route( /// # `GET /_matrix/client/v1/media/download/{serverName}/{mediaId}` /// /// Load media from our server or over federation. -#[tracing::instrument(skip_all, fields(%client), name = "media_get")] +#[tracing::instrument( + name = "media_get", + level = "debug", + skip_all, + fields(%client), +)] pub(crate) async fn get_content_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, @@ -128,7 +143,12 @@ pub(crate) async fn get_content_route( /// # `GET /_matrix/client/v1/media/download/{serverName}/{mediaId}/{fileName}` /// /// Load media from our server or over federation as fileName. -#[tracing::instrument(skip_all, fields(%client), name = "media_get_af")] +#[tracing::instrument( + name = "media_get_af", + level = "debug", + skip_all, + fields(%client), +)] pub(crate) async fn get_content_as_filename_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, @@ -158,7 +178,12 @@ pub(crate) async fn get_content_as_filename_route( /// # `GET /_matrix/client/v1/media/preview_url` /// /// Returns URL preview. -#[tracing::instrument(skip_all, fields(%client), name = "url_preview")] +#[tracing::instrument( + name = "url_preview", + level = "debug", + skip_all, + fields(%client), +)] pub(crate) async fn get_media_preview_route( State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 61a0ea5c..31179d3c 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -97,6 +97,14 @@ type PresenceUpdates = HashMap; /// For left rooms: /// - If the user left after `since`: `prev_batch` token, empty state (TODO: /// subset of the state at the point of the leave) +#[tracing::instrument( + name = "sync", + level = "debug", + skip_all, + fields( + since = %body.body.since.as_deref().unwrap_or_default(), + ) +)] pub(crate) async fn sync_events_route( State(services): State, body: Ruma, ) -> Result> { @@ -391,8 +399,17 @@ async fn process_presence_updates(services: &Services, since: u64, syncing_user: .await } +#[tracing::instrument( + name = "left", + level = "debug", + skip_all, + fields( + room_id = %room_id, + full = %full_state, + ll = %lazy_load_enabled, + ), +)] #[allow(clippy::too_many_arguments)] -#[tracing::instrument(skip_all, fields(user_id = %sender_user, room_id = %room_id), name = "left_room")] async fn handle_left_room( services: &Services, since: u64, room_id: OwnedRoomId, sender_user: &UserId, next_batch_string: &str, full_state: bool, lazy_load_enabled: bool, diff --git a/src/database/engine.rs b/src/database/engine.rs index fe6602ae..d3bb727e 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -307,7 +307,12 @@ pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> { Ok(()) } -#[tracing::instrument(skip(msg), name = "rocksdb", level = "trace")] +#[tracing::instrument( + parent = None, + name = "rocksdb", + level = "trace" + skip(msg), +)] pub(crate) fn handle_log(level: LogLevel, msg: &str) { let msg = msg.trim(); if msg.starts_with("Options") { diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index 0f1fdea7..bb904943 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -33,7 +33,7 @@ where } #[implement(super::Map)] -#[tracing::instrument(skip(self, keys), name = "batch_blocking", level = "trace")] +#[tracing::instrument(name = "batch_blocking", level = "trace", skip_all)] pub(crate) fn get_batch_blocking<'a, I, K>(&self, keys: I) -> impl Iterator>> + Send where I: Iterator + ExactSizeIterator + Debug + Send, diff --git a/src/database/maps.rs b/src/database/maps.rs index 0e835abf..9b8d326a 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -10,7 +10,7 @@ pub(crate) type MapsKey = String; pub(crate) fn open(db: &Arc) -> Result { open_list(db, MAPS) } -#[tracing::instrument(skip_all, level = "debug")] +#[tracing::instrument(name = "maps", level = "debug", skip_all)] pub(crate) fn open_list(db: &Arc, maps: &[&str]) -> Result { Ok(maps .iter() diff --git a/src/database/pool.rs b/src/database/pool.rs index e7ffc807..79755ea6 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -113,10 +113,14 @@ async fn spawn_until(self: &Arc, recv: Receiver, max: usize) -> Resul } #[implement(Pool)] +#[tracing::instrument( + name = "spawn", + level = "trace", + skip_all, + fields(id = %workers.len()) +)] fn spawn_one(self: &Arc, workers: &mut JoinSet<()>, recv: Receiver) -> Result { let id = workers.len(); - - debug!(?id, "spawning"); let self_ = self.clone(); let _abort = workers.spawn_blocking_on(move || self_.worker(id, recv), self.server.runtime()); @@ -181,7 +185,14 @@ async fn execute(&self, cmd: Cmd) -> Result { } #[implement(Pool)] -#[tracing::instrument(skip(self, recv))] +#[tracing::instrument( + parent = None, + level = "debug", + skip(self, recv), + fields( + tid = ?std::thread::current().id(), + ), +)] fn worker(self: Arc, id: usize, recv: Receiver) { debug!("worker spawned"); defer! {{ debug!("worker finished"); }} diff --git a/src/main/main.rs b/src/main/main.rs index c653444c..41b21b58 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -59,6 +59,11 @@ fn main() -> Result<(), Error> { /// Operate the server normally in release-mode static builds. This will start, /// run and stop the server within the asynchronous runtime. #[cfg(not(conduit_mods))] +#[tracing::instrument( + name = "main", + parent = None, + skip_all +)] async fn async_main(server: &Arc) -> Result<(), Error> { extern crate conduit_router as router; diff --git a/src/router/layers.rs b/src/router/layers.rs index fd68cc36..bb3ffe87 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -150,8 +150,8 @@ fn cors_layer(_server: &Server) -> CorsLayer { fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max(server.config.max_request_size) } +#[tracing::instrument(name = "panic", level = "error", skip_all)] #[allow(clippy::needless_pass_by_value)] -#[tracing::instrument(skip_all, name = "panic")] fn catch_panic(err: Box) -> http::Response> { //TODO: XXX /* @@ -197,7 +197,7 @@ fn tracing_span(request: &http::Request) -> tracing::Span { let method = request.method(); - tracing::info_span!("router:", %method, %path) + tracing::debug_span!(parent: None, "router", %method, %path) } fn truncated_matched_path(path: &MatchedPath) -> &str { diff --git a/src/router/request.rs b/src/router/request.rs index 271d9573..13d36981 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -8,7 +8,11 @@ use conduit::{debug, debug_error, debug_warn, defer, err, error, trace, Result}; use conduit_service::Services; use http::{Method, StatusCode, Uri}; -#[tracing::instrument(skip_all, level = "debug")] +#[tracing::instrument( + parent = None, + level = "trace", + skip_all, +)] pub(crate) async fn spawn( State(services): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { @@ -34,7 +38,7 @@ pub(crate) async fn spawn( task.await.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) } -#[tracing::instrument(skip_all, name = "handle")] +#[tracing::instrument(level = "debug", skip_all)] pub(crate) async fn handle( State(services): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 5df41b61..dd2ecf6a 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -30,7 +30,7 @@ type MakeService = IntoMakeServiceWithConnectInfo; const NULL_ADDR: net::SocketAddr = net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); const FINI_POLL_INTERVAL: Duration = Duration::from_millis(750); -#[tracing::instrument(skip_all)] +#[tracing::instrument(skip_all, level = "debug")] pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broadcast::Receiver<()>) -> Result<()> { let mut tasks = JoinSet::<()>::new(); let executor = TokioExecutor::new(); diff --git a/src/service/manager.rs b/src/service/manager.rs index 21e0ed7c..7384e512 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -155,6 +155,12 @@ impl Manager { /// should never error with a panic, and if so it should propagate, but it may /// error with an Abort which the manager should handle along with results to /// determine if the worker should be restarted. +#[tracing::instrument( + parent = None, + level = "trace", + skip_all, + fields(service = %service.name()), +)] async fn worker(service: Arc) -> WorkerResult { let service_ = Arc::clone(&service); let result = AssertUnwindSafe(service_.worker()) diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 8553e8bb..8b720365 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -26,7 +26,7 @@ impl ActualDest { } impl super::Service { - #[tracing::instrument(skip_all, name = "resolve")] + #[tracing::instrument(skip_all, level = "debug", name = "resolve")] pub(crate) async fn get_actual_dest(&self, server_name: &ServerName) -> Result { let (result, cached) = if let Some(result) = self.get_cached_destination(server_name) { (result, true) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index ef1b63f5..bcc1263d 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -184,7 +184,7 @@ impl Service { /// Whether a server is allowed to see an event through federation, based on /// the room's history_visibility at that event's state. - #[tracing::instrument(skip(self, origin, room_id, event_id))] + #[tracing::instrument(skip_all, level = "trace")] pub async fn server_can_see_event(&self, origin: &ServerName, room_id: &RoomId, event_id: &EventId) -> bool { let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { return true; @@ -242,7 +242,7 @@ impl Service { /// Whether a user is allowed to see an event, based on /// the room's history_visibility at that event's state. - #[tracing::instrument(skip(self, user_id, room_id, event_id))] + #[tracing::instrument(skip_all, level = "trace")] pub async fn user_can_see_event(&self, user_id: &UserId, room_id: &RoomId, event_id: &EventId) -> bool { let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { return true; @@ -293,7 +293,7 @@ impl Service { /// Whether a user is allowed to see an event, based on /// the room's history_visibility at that event's state. - #[tracing::instrument(skip(self, user_id, room_id))] + #[tracing::instrument(skip_all, level = "trace")] pub async fn user_can_see_state_events(&self, user_id: &UserId, room_id: &RoomId) -> bool { if self.services.state_cache.is_joined(user_id, room_id).await { return true; diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 118dab91..bc70e875 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -176,7 +176,6 @@ impl Data { } } -#[tracing::instrument(skip(key), level = "debug")] fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, SendingEvent)> { // Appservices start with a plus Ok::<_, Error>(if key.starts_with(b"+") { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 6a8f1b1b..df39bcd1 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -24,7 +24,7 @@ use crate::{ }; impl super::Service { - #[tracing::instrument(skip(self, client, request), name = "send")] + #[tracing::instrument(skip_all, level = "debug")] pub async fn send(&self, client: &Client, dest: &ServerName, request: T) -> Result where T: OutgoingRequest + Send, diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 5c00915b..49ce1be4 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -62,7 +62,7 @@ pub const PDU_LIMIT: usize = 50; pub const EDU_LIMIT: usize = 100; impl Service { - #[tracing::instrument(skip_all, name = "sender")] + #[tracing::instrument(skip_all, level = "debug")] pub(super) async fn sender(&self) -> Result<()> { let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); let mut futures: SendingFutures<'_> = FuturesUnordered::new(); From 34f9e3260f44c8a9f642aeb3c9b94266431d5e25 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 8 Dec 2024 03:00:09 +0000 Subject: [PATCH 1203/2091] additional sync v3 refactoring/optimizations and tracing instruments Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 275 +++++++++++++++++++++----------------- 1 file changed, 154 insertions(+), 121 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 31179d3c..28ca1ea2 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -11,8 +11,9 @@ use conduit::{ result::{FlatOk, LogDebugErr}, utils, utils::{ + future::OptionExt, math::ruma_from_u64, - stream::{BroadbandExt, Tools}, + stream::{BroadbandExt, Tools, WidebandExt}, BoolExt, IterStream, ReadyExt, TryFutureExtExt, }, Error, PduCount, PduEvent, Result, @@ -22,7 +23,7 @@ use conduit_service::{ Services, }; use futures::{ - future::{join, join3, join5, try_join, OptionFuture}, + future::{join, join3, join4, join5, try_join, try_join3, OptionFuture}, FutureExt, StreamExt, TryFutureExt, }; use ruma::{ @@ -122,7 +123,6 @@ pub(crate) async fn sync_events_route( let watcher = services.sync.watch(sender_user, sender_device); let next_batch = services.globals.current_count()?; - let next_batchcount = PduCount::Normal(next_batch); let next_batch_string = next_batch.to_string(); // Load filter @@ -153,9 +153,8 @@ pub(crate) async fn sync_events_route( .as_ref() .and_then(|string| string.parse().ok()) .unwrap_or(0); - let sincecount = PduCount::Normal(since); - let joined_related = services + let joined_rooms = services .rooms .state_cache .rooms_joined(sender_user) @@ -167,9 +166,7 @@ pub(crate) async fn sync_events_route( sender_device, room_id.clone(), since, - sincecount, next_batch, - next_batchcount, lazy_load_enabled, lazy_load_send_redundant, full_state, @@ -211,9 +208,11 @@ pub(crate) async fn sync_events_route( .ready_filter_map(|(room_id, left_room)| left_room.map(|left_room| (room_id, left_room))) .collect(); - let invited_rooms = services.rooms.state_cache.rooms_invited(sender_user).fold( - BTreeMap::new(), - |mut invited_rooms, (room_id, invite_state)| async move { + let invited_rooms = services + .rooms + .state_cache + .rooms_invited(sender_user) + .fold_default(|mut invited_rooms: BTreeMap<_, _>, (room_id, invite_state)| async move { // Get and drop the lock to wait for remaining operations to finish let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; drop(insert_lock); @@ -238,8 +237,7 @@ pub(crate) async fn sync_events_route( invited_rooms.insert(room_id, invited_room); invited_rooms - }, - ); + }); let presence_updates: OptionFuture<_> = services .globals @@ -274,7 +272,7 @@ pub(crate) async fn sync_events_route( .users .remove_to_device_events(sender_user, sender_device, since); - let rooms = join3(joined_related, left_rooms, invited_rooms); + let rooms = join3(joined_rooms, left_rooms, invited_rooms); let ephemeral = join3(remove_to_device_events, to_device_events, presence_updates); let top = join5(account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms) .boxed() @@ -282,8 +280,8 @@ pub(crate) async fn sync_events_route( let (account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms) = top; let ((), to_device_events, presence_updates) = ephemeral; - let (joined_related, left_rooms, invited_rooms) = rooms; - let (joined_rooms, mut device_list_updates, left_encrypted_users) = joined_related; + let (joined_rooms, left_rooms, invited_rooms) = rooms; + let (joined_rooms, mut device_list_updates, left_encrypted_users) = joined_rooms; device_list_updates.extend(keys_changed); // If the user doesn't share an encrypted room with the target anymore, we need @@ -349,6 +347,7 @@ pub(crate) async fn sync_events_route( Ok(response) } +#[tracing::instrument(name = "presence", level = "debug", skip_all)] async fn process_presence_updates(services: &Services, since: u64, syncing_user: &UserId) -> PresenceUpdates { services .presence @@ -411,17 +410,17 @@ async fn process_presence_updates(services: &Services, since: u64, syncing_user: )] #[allow(clippy::too_many_arguments)] async fn handle_left_room( - services: &Services, since: u64, room_id: OwnedRoomId, sender_user: &UserId, next_batch_string: &str, + services: &Services, since: u64, ref room_id: OwnedRoomId, sender_user: &UserId, next_batch_string: &str, full_state: bool, lazy_load_enabled: bool, ) -> Result> { // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; + let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); let left_count = services .rooms .state_cache - .get_left_count(&room_id, sender_user) + .get_left_count(room_id, sender_user) .await .ok(); @@ -430,7 +429,7 @@ async fn handle_left_room( return Ok(None); } - if !services.rooms.metadata.exists(&room_id).await { + if !services.rooms.metadata.exists(room_id).await { // This is just a rejected invite, not a room we know // Insert a leave event anyways let event = PduEvent { @@ -476,7 +475,7 @@ async fn handle_left_room( let since_shortstatehash = services .rooms .user - .get_token_shortstatehash(&room_id, since) + .get_token_shortstatehash(room_id, since) .await; let since_state_ids = match since_shortstatehash { @@ -487,7 +486,7 @@ async fn handle_left_room( let Ok(left_event_id): Result = services .rooms .state_accessor - .room_state_get_id(&room_id, &StateEventType::RoomMember, sender_user.as_str()) + .room_state_get_id(room_id, &StateEventType::RoomMember, sender_user.as_str()) .await else { error!("Left room but no left state event"); @@ -557,30 +556,46 @@ async fn handle_left_room( })) } +#[tracing::instrument( + name = "joined", + level = "debug", + skip_all, + fields( + room_id = ?room_id, + ), +)] #[allow(clippy::too_many_arguments)] async fn load_joined_room( - services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: OwnedRoomId, since: u64, - sincecount: PduCount, next_batch: u64, next_batchcount: PduCount, lazy_load_enabled: bool, - lazy_load_send_redundant: bool, full_state: bool, + services: &Services, sender_user: &UserId, sender_device: &DeviceId, ref room_id: OwnedRoomId, since: u64, + next_batch: u64, lazy_load_enabled: bool, lazy_load_send_redundant: bool, full_state: bool, ) -> Result<(JoinedRoom, HashSet, HashSet)> { - let mut device_list_updates = HashSet::::new(); - let mut left_encrypted_users = HashSet::::new(); - // Get and drop the lock to wait for remaining operations to finish // This will make sure the we have all events until next_batch - let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; + let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; drop(insert_lock); - let (timeline_pdus, limited) = - load_timeline(services, sender_user, &room_id, sincecount, Some(next_batchcount), 10_usize).await?; + let sincecount = PduCount::Normal(since); + let next_batchcount = PduCount::Normal(next_batch); - let send_notification_counts = !timeline_pdus.is_empty() - || services - .rooms - .user - .last_notification_read(sender_user, &room_id) - .await > since; + let current_shortstatehash = services + .rooms + .state + .get_room_shortstatehash(room_id) + .map_err(|_| err!(Database(error!("Room {room_id} has no state")))); + let since_shortstatehash = services + .rooms + .user + .get_token_shortstatehash(room_id, since) + .ok() + .map(Ok); + + let timeline = load_timeline(services, sender_user, room_id, sincecount, Some(next_batchcount), 10_usize); + + let (current_shortstatehash, since_shortstatehash, timeline) = + try_join3(current_shortstatehash, since_shortstatehash, timeline).await?; + + let (timeline_pdus, limited) = timeline; let timeline_users = timeline_pdus .iter() .fold(HashSet::new(), |mut timeline_users, (_, event)| { @@ -588,43 +603,44 @@ async fn load_joined_room( timeline_users }); + let last_notification_read: OptionFuture<_> = timeline_pdus + .is_empty() + .then(|| { + services + .rooms + .user + .last_notification_read(sender_user, room_id) + }) + .into(); + + let send_notification_counts = last_notification_read + .is_none_or(|&count| count > since) + .await; + services .rooms .lazy_loading - .lazy_load_confirm_delivery(sender_user, sender_device, &room_id, sincecount); + .lazy_load_confirm_delivery(sender_user, sender_device, room_id, sincecount); - let current_shortstatehash = services - .rooms - .state - .get_room_shortstatehash(&room_id) - .map_err(|_| err!(Database(error!("Room {room_id} has no state")))); - - let since_shortstatehash = services - .rooms - .user - .get_token_shortstatehash(&room_id, since) - .ok() - .map(Ok); - - let (current_shortstatehash, since_shortstatehash) = try_join(current_shortstatehash, since_shortstatehash).await?; + let no_state_changes = timeline_pdus.is_empty() + && (since_shortstatehash.is_none() || since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash))); + let mut device_list_updates = HashSet::::new(); + let mut left_encrypted_users = HashSet::::new(); let StateChanges { heroes, joined_member_count, invited_member_count, joined_since_last_sync, state_events, - } = if timeline_pdus.is_empty() - && (since_shortstatehash.is_none() || since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash))) - { - // No state changes + } = if no_state_changes { StateChanges::default() } else { calculate_state_changes( services, sender_user, sender_device, - &room_id, + room_id, next_batchcount, lazy_load_enabled, lazy_load_send_redundant, @@ -636,22 +652,68 @@ async fn load_joined_room( &timeline_pdus, &timeline_users, ) - .boxed() .await? }; - let prev_batch = timeline_pdus - .first() - .map(at!(0)) - .as_ref() - .map(ToString::to_string); + let account_data_events = services + .account_data + .changes_since(Some(room_id), sender_user, since) + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect(); + + // Look for device list updates in this room + let device_updates = services + .users + .room_keys_changed(room_id, since, None) + .map(|(user_id, _)| user_id) + .map(ToOwned::to_owned) + .collect::>(); + + let room_events = timeline_pdus + .iter() + .stream() + .wide_filter_map(|item| ignored_filter(services, item.clone(), sender_user)) + .map(|(_, pdu)| pdu.to_sync_room_event()) + .collect(); + + let receipt_events = services + .rooms + .read_receipt + .readreceipts_since(room_id, since) + .filter_map(|(read_user, _, edu)| async move { + services + .users + .user_is_ignored(read_user, sender_user) + .await + .or_some((read_user.to_owned(), edu)) + }) + .collect::>>(); + + let typing_events = services + .rooms + .typing + .last_typing_update(room_id) + .and_then(|count| async move { + if count <= since { + return Ok(Vec::>::new()); + } + + let typings = services + .rooms + .typing + .typings_all(room_id, sender_user) + .await?; + + Ok(vec![serde_json::from_str(&serde_json::to_string(&typings)?)?]) + }) + .unwrap_or(Vec::new()); let notification_count: OptionFuture<_> = send_notification_counts .then(|| { services .rooms .user - .notification_count(sender_user, &room_id) + .notification_count(sender_user, room_id) .map(TryInto::try_into) .unwrap_or(uint!(0)) }) @@ -662,79 +724,33 @@ async fn load_joined_room( services .rooms .user - .highlight_count(sender_user, &room_id) + .highlight_count(sender_user, room_id) .map(TryInto::try_into) .unwrap_or(uint!(0)) }) .into(); - let room_events = timeline_pdus - .iter() - .stream() - .filter_map(|item| ignored_filter(services, item.clone(), sender_user)) - .map(|(_, pdu)| pdu.to_sync_room_event()) - .collect(); - - let account_data_events = services - .account_data - .changes_since(Some(&room_id), sender_user, since) - .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) - .collect(); - - let receipt_events = services - .rooms - .read_receipt - .readreceipts_since(&room_id, since) - .filter_map(|(read_user, _, edu)| async move { - services - .users - .user_is_ignored(read_user, sender_user) - .await - .or_some((read_user.to_owned(), edu)) - }) - .collect::>>(); - - // Look for device list updates in this room - let device_updates = services - .users - .room_keys_changed(&room_id, since, None) - .map(|(user_id, _)| user_id) - .map(ToOwned::to_owned) - .collect::>(); - - let events = join3(room_events, account_data_events, receipt_events); + let events = join4(room_events, account_data_events, receipt_events, typing_events); let unread_notifications = join(notification_count, highlight_count); let (unread_notifications, events, device_updates) = join3(unread_notifications, events, device_updates) .boxed() .await; - let (room_events, account_data_events, receipt_events) = events; + let (room_events, account_data_events, receipt_events, typing_events) = events; let (notification_count, highlight_count) = unread_notifications; - device_list_updates.extend(device_updates); - let mut edus: Vec> = receipt_events.into_values().collect(); - if services.rooms.typing.last_typing_update(&room_id).await? > since { - edus.push( - serde_json::from_str( - &serde_json::to_string( - &services - .rooms - .typing - .typings_all(&room_id, sender_user) - .await?, - ) - .expect("event is valid, we just created it"), - ) - .expect("event is valid, we just created it"), - ); - } + device_list_updates.extend(device_updates); + let edus: Vec> = receipt_events + .into_values() + .chain(typing_events.into_iter()) + .collect(); // Save the state after this sync so we can send the correct state diff next // sync services .rooms .user - .associate_token_shortstatehash(&room_id, next_batch, current_shortstatehash) + .associate_token_shortstatehash(room_id, next_batch, current_shortstatehash) .await; let joined_room = JoinedRoom { @@ -757,8 +773,12 @@ async fn load_joined_room( }, timeline: Timeline { limited: limited || joined_since_last_sync, - prev_batch, events: room_events, + prev_batch: timeline_pdus + .first() + .map(at!(0)) + .as_ref() + .map(ToString::to_string), }, state: RoomState { events: state_events @@ -775,6 +795,17 @@ async fn load_joined_room( Ok((joined_room, device_list_updates, left_encrypted_users)) } +#[tracing::instrument( + name = "state", + level = "trace", + skip_all, + fields( + full = %full_state, + ll = ?(lazy_load_enabled, lazy_load_send_redundant), + cs = %current_shortstatehash, + ss = ?since_shortstatehash, + ) +)] #[allow(clippy::too_many_arguments)] async fn calculate_state_changes( services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, next_batchcount: PduCount, @@ -833,6 +864,7 @@ async fn calculate_state_changes( } } +#[tracing::instrument(name = "initial", level = "trace", skip_all)] #[allow(clippy::too_many_arguments)] async fn calculate_state_initial( services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, next_batchcount: PduCount, @@ -847,7 +879,7 @@ async fn calculate_state_initial( .await? .into_iter() .stream() - .filter_map(|(shortstatekey, event_id): (ShortStateKey, OwnedEventId)| { + .broad_filter_map(|(shortstatekey, event_id): (ShortStateKey, OwnedEventId)| { services .rooms .short @@ -919,6 +951,7 @@ async fn calculate_state_initial( }) } +#[tracing::instrument(name = "incremental", level = "trace", skip_all)] #[allow(clippy::too_many_arguments)] async fn calculate_state_incremental( services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, next_batchcount: PduCount, @@ -949,7 +982,7 @@ async fn calculate_state_incremental( .iter() .stream() .ready_filter(|(key, id)| full_state || since_state_ids.get(key) != Some(id)) - .filter_map(|(_, id)| services.rooms.timeline.get_pdu(id).ok()) + .wide_filter_map(|(_, id)| services.rooms.timeline.get_pdu(id).ok()) .ready_for_each(|pdu| delta_state_events.push(pdu)) .await; } From badb83484fe913188f683b73e7d2afd86b0b29ba Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 10 Dec 2024 22:54:19 -0500 Subject: [PATCH 1204/2091] fix private read receipt support Signed-off-by: strawberry --- src/api/client/read_marker.rs | 129 ++++++++++++------------- src/api/client/sync/v3.rs | 21 +++- src/service/rooms/read_receipt/data.rs | 6 +- src/service/rooms/read_receipt/mod.rs | 64 ++++++++++-- 4 files changed, 140 insertions(+), 80 deletions(-) diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index f28b2aec..f6123614 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -1,9 +1,9 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::PduCount; +use conduit::{err, Err, PduCount}; use ruma::{ - api::client::{error::ErrorKind, read_marker::set_read_marker, receipt::create_receipt}, + api::client::{read_marker::set_read_marker, receipt::create_receipt}, events::{ receipt::{ReceiptThread, ReceiptType}, RoomAccountDataEventType, @@ -11,7 +11,7 @@ use ruma::{ MilliSecondsSinceUnixEpoch, }; -use crate::{Error, Result, Ruma}; +use crate::{Result, Ruma}; /// # `POST /_matrix/client/r0/rooms/{roomId}/read_markers` /// @@ -23,14 +23,15 @@ use crate::{Error, Result, Ruma}; pub(crate) async fn set_read_marker_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); - if let Some(fully_read) = &body.fully_read { + if let Some(event) = &body.fully_read { let fully_read_event = ruma::events::fully_read::FullyReadEvent { content: ruma::events::fully_read::FullyReadEventContent { - event_id: fully_read.clone(), + event_id: event.clone(), }, }; + services .account_data .update( @@ -49,44 +50,20 @@ pub(crate) async fn set_read_marker_route( .reset_notification_counts(sender_user, &body.room_id); } - if let Some(event) = &body.private_read_receipt { - let count = services - .rooms - .timeline - .get_pdu_count(event) - .await - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?; - - let count = match count { - PduCount::Backfilled(_) => { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Read receipt is in backfilled timeline", - )) - }, - PduCount::Normal(c) => c, - }; - services - .rooms - .read_receipt - .private_read_set(&body.room_id, sender_user, count); - } - if let Some(event) = &body.read_receipt { - let mut user_receipts = BTreeMap::new(); - user_receipts.insert( - sender_user.clone(), - ruma::events::receipt::Receipt { - ts: Some(MilliSecondsSinceUnixEpoch::now()), - thread: ReceiptThread::Unthreaded, - }, - ); - - let mut receipts = BTreeMap::new(); - receipts.insert(ReceiptType::Read, user_receipts); - - let mut receipt_content = BTreeMap::new(); - receipt_content.insert(event.to_owned(), receipts); + let receipt_content = BTreeMap::from_iter([( + event.to_owned(), + BTreeMap::from_iter([( + ReceiptType::Read, + BTreeMap::from_iter([( + sender_user.to_owned(), + ruma::events::receipt::Receipt { + ts: Some(MilliSecondsSinceUnixEpoch::now()), + thread: ReceiptThread::Unthreaded, + }, + )]), + )]), + )]); services .rooms @@ -102,6 +79,24 @@ pub(crate) async fn set_read_marker_route( .await; } + if let Some(event) = &body.private_read_receipt { + let count = services + .rooms + .timeline + .get_pdu_count(event) + .await + .map_err(|_| err!(Request(NotFound("Event not found."))))?; + + let PduCount::Normal(count) = count else { + return Err!(Request(InvalidParam("Event is a backfilled PDU and cannot be marked as read."))); + }; + + services + .rooms + .read_receipt + .private_read_set(&body.room_id, sender_user, count); + } + Ok(set_read_marker::v3::Response {}) } @@ -111,7 +106,7 @@ pub(crate) async fn set_read_marker_route( pub(crate) async fn create_receipt_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if matches!( &body.receipt_type, @@ -141,19 +136,19 @@ pub(crate) async fn create_receipt_route( .await?; }, create_receipt::v3::ReceiptType::Read => { - let mut user_receipts = BTreeMap::new(); - user_receipts.insert( - sender_user.clone(), - ruma::events::receipt::Receipt { - ts: Some(MilliSecondsSinceUnixEpoch::now()), - thread: ReceiptThread::Unthreaded, - }, - ); - let mut receipts = BTreeMap::new(); - receipts.insert(ReceiptType::Read, user_receipts); - - let mut receipt_content = BTreeMap::new(); - receipt_content.insert(body.event_id.clone(), receipts); + let receipt_content = BTreeMap::from_iter([( + body.event_id.clone(), + BTreeMap::from_iter([( + ReceiptType::Read, + BTreeMap::from_iter([( + sender_user.to_owned(), + ruma::events::receipt::Receipt { + ts: Some(MilliSecondsSinceUnixEpoch::now()), + thread: ReceiptThread::Unthreaded, + }, + )]), + )]), + )]); services .rooms @@ -174,23 +169,23 @@ pub(crate) async fn create_receipt_route( .timeline .get_pdu_count(&body.event_id) .await - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event does not exist."))?; + .map_err(|_| err!(Request(NotFound("Event not found."))))?; - let count = match count { - PduCount::Backfilled(_) => { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Read receipt is in backfilled timeline", - )) - }, - PduCount::Normal(c) => c, + let PduCount::Normal(count) = count else { + return Err!(Request(InvalidParam("Event is a backfilled PDU and cannot be marked as read."))); }; + services .rooms .read_receipt .private_read_set(&body.room_id, sender_user, count); }, - _ => return Err(Error::bad_database("Unsupported receipt type")), + _ => { + return Err!(Request(InvalidParam(warn!( + "Received unknown read receipt type: {}", + &body.receipt_type + )))) + }, } Ok(create_receipt::v3::Response {}) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 28ca1ea2..44572970 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -9,8 +9,8 @@ use conduit::{ at, err, error, extract_variant, is_equal_to, is_false, pdu::EventHash, result::{FlatOk, LogDebugErr}, - utils, utils::{ + self, future::OptionExt, math::ruma_from_u64, stream::{BroadbandExt, Tools, WidebandExt}, @@ -740,9 +740,28 @@ async fn load_joined_room( let (notification_count, highlight_count) = unread_notifications; device_list_updates.extend(device_updates); + + let last_privateread_update = services + .rooms + .read_receipt + .last_privateread_update(sender_user, room_id) + .await > since; + + let private_read_event = if last_privateread_update { + services + .rooms + .read_receipt + .private_read_get(room_id, sender_user) + .await + .ok() + } else { + None + }; + let edus: Vec> = receipt_events .into_values() .chain(typing_events.into_iter()) + .chain(private_read_event.into_iter()) .collect(); // Save the state after this sync so we can send the correct state diff next diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 34639e27..9a1dba45 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -79,15 +79,15 @@ impl Data { .ignore_err() } - pub(super) fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) { + pub(super) fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, pdu_count: u64) { let key = (room_id, user_id); let next_count = self.services.globals.next_count().unwrap(); - self.roomuserid_privateread.put(key, count); + self.roomuserid_privateread.put(key, pdu_count); self.roomuserid_lastprivatereadupdate.put(key, next_count); } - pub(super) async fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result { + pub(super) async fn private_read_get_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { let key = (room_id, user_id); self.roomuserid_privateread.qry(&key).await.deserialized() } diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index e089d369..a3cd7098 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -2,19 +2,19 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{debug, Result}; -use futures::Stream; +use conduit::{debug, err, warn, PduCount, PduId, RawPduId, Result}; +use futures::{try_join, Stream, TryFutureExt}; use ruma::{ events::{ receipt::{ReceiptEvent, ReceiptEventContent}, AnySyncEphemeralRoomEvent, SyncEphemeralRoomEvent, }, serde::Raw, - OwnedUserId, RoomId, UserId, + OwnedEventId, OwnedUserId, RoomId, UserId, }; use self::data::{Data, ReceiptItem}; -use crate::{sending, Dep}; +use crate::{rooms, sending, Dep}; pub struct Service { services: Services, @@ -23,6 +23,8 @@ pub struct Service { struct Services { sending: Dep, + short: Dep, + timeline: Dep, } impl crate::Service for Service { @@ -30,6 +32,8 @@ impl crate::Service for Service { Ok(Arc::new(Self { services: Services { sending: args.depend::("sending"), + short: args.depend::("rooms::short"), + timeline: args.depend::("rooms::timeline"), }, db: Data::new(&args), })) @@ -49,6 +53,48 @@ impl Service { .expect("room flush failed"); } + /// Gets the latest private read receipt from the user in the room + pub async fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { + let pdu_count = self + .private_read_get_count(room_id, user_id) + .map_err(|e| err!(Database(warn!("No private read receipt was set in {room_id}: {e}")))); + let shortroomid = self + .services + .short + .get_shortroomid(room_id) + .map_err(|e| err!(Database(warn!("Short room ID does not exist in database for {room_id}: {e}")))); + let (pdu_count, shortroomid) = try_join!(pdu_count, shortroomid)?; + + let shorteventid = PduCount::Normal(pdu_count); + let pdu_id: RawPduId = PduId { + shortroomid, + shorteventid, + } + .into(); + + let pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await?; + + let event_id: OwnedEventId = pdu.event_id.into(); + let receipt_content = BTreeMap::from_iter([( + event_id, + BTreeMap::from_iter([( + ruma::events::receipt::ReceiptType::ReadPrivate, + BTreeMap::from_iter([( + user_id, + ruma::events::receipt::Receipt { + ts: None, // TODO: start storing the timestamp so we can return one + thread: ruma::events::receipt::ReceiptThread::Unthreaded, + }, + )]), + )]), + )]); + //let receipt_json = Json + + let event = serde_json::value::to_raw_value(&receipt_content).expect("receipt_content created manually"); + + Ok(Raw::from_json(event)) + } + /// Returns an iterator over the most recent read_receipts in a room that /// happened after the event with id `since`. #[inline] @@ -59,21 +105,21 @@ impl Service { self.db.readreceipts_since(room_id, since) } - /// Sets a private read marker at `count`. + /// Sets a private read marker at PDU `count`. #[inline] #[tracing::instrument(skip(self), level = "debug")] pub fn private_read_set(&self, room_id: &RoomId, user_id: &UserId, count: u64) { self.db.private_read_set(room_id, user_id, count); } - /// Returns the private read marker. + /// Returns the private read marker PDU count. #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub async fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result { - self.db.private_read_get(room_id, user_id).await + pub async fn private_read_get_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + self.db.private_read_get_count(room_id, user_id).await } - /// Returns the count of the last typing update in this room. + /// Returns the PDU count of the last typing update in this room. #[inline] pub async fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> u64 { self.db.last_privateread_update(user_id, room_id).await From 1e3bb02ced7ba4994a37f4e917c866774100ee2f Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 14:26:15 -0500 Subject: [PATCH 1205/2091] bump cargo.lock, bump some deps Signed-off-by: strawberry --- Cargo.lock | 464 ++++++++++++++++++++++++----------------------------- Cargo.toml | 34 ++-- 2 files changed, 229 insertions(+), 269 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e766fe46..86b90c3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,9 +49,9 @@ checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" [[package]] name = "arc-swap" @@ -139,7 +139,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -150,7 +150,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -303,7 +303,7 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls 0.23.18", + "rustls", "rustls-pemfile", "rustls-pki-types", "tokio", @@ -323,7 +323,7 @@ dependencies = [ "http", "http-body-util", "pin-project", - "rustls 0.23.18", + "rustls", "tokio", "tokio-rustls", "tokio-util", @@ -383,7 +383,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.89", + "syn 2.0.90", "which", ] @@ -464,9 +464,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "bytesize" @@ -487,9 +487,9 @@ dependencies = [ [[package]] name = "cargo_toml" -version = "0.20.5" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88da5a13c620b4ca0078845707ea9c3faf11edbc3ffd8497d11d686211cd1ac0" +checksum = "5fbd1fe9db3ebf71b89060adaf7b0504c2d6a425cf061313099547e382c2e472" dependencies = [ "serde", "toml", @@ -497,9 +497,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd9de9f2205d5ef3fd67e685b0df337994ddd4495e2a28d185500d0e1edfea47" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "jobserver", "libc", @@ -538,9 +538,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "num-traits", ] @@ -558,9 +558,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.21" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb3b4b9e5a7c7514dfa52869339ee98b3156b0bfb4e8a77c4ff4babb64b1604f" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -568,9 +568,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.21" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17a95aa67cc7b5ebd32aa5370189aa0d79069ef1c64ce893bd30fb24bff20ec" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstyle", "clap_lex", @@ -585,20 +585,20 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] name = "clap_lex" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" -version = "0.1.51" +version = "0.1.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb1e43aa7fd152b1f968787f7dbcdeb306d1867ff373c69955211876c053f91a" +checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" dependencies = [ "cc", ] @@ -739,7 +739,7 @@ dependencies = [ "serde_json", "serde_regex", "serde_yaml", - "thiserror 2.0.3", + "thiserror 2.0.6", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -777,7 +777,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -801,7 +801,7 @@ dependencies = [ "hyper-util", "log", "ruma", - "rustls 0.23.18", + "rustls", "sd-notify", "sentry", "sentry-tower", @@ -1076,7 +1076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1103,7 +1103,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1172,7 +1172,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1224,7 +1224,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1235,12 +1235,12 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.9" +version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1256,9 +1256,9 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f214dc438f977e6d4e3500aaa277f5ad94ca83fbbd9b1a15713ce2344ccc5a1" +checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" dependencies = [ "event-listener", "pin-project-lite", @@ -1266,9 +1266,9 @@ dependencies = [ [[package]] name = "fdeflate" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c6f4c64c1d33a3111c4466f7365ebdcc37c5bd1ea0d62aae2e3d722aacbedb" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" dependencies = [ "simd-adler32", ] @@ -1411,7 +1411,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -1501,7 +1501,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.6.0", + "indexmap 2.7.0", "slab", "tokio", "tokio-util", @@ -1522,9 +1522,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.15.1" +version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" [[package]] name = "hdrhistogram" @@ -1583,9 +1583,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-proto" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" +checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" dependencies = [ "async-trait", "cfg-if", @@ -1594,7 +1594,7 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna 0.4.0", + "idna", "ipnet", "once_cell", "rand", @@ -1607,9 +1607,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.1" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" +checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" dependencies = [ "cfg-if", "futures-util", @@ -1677,14 +1677,14 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -1772,7 +1772,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.18", + "rustls", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -1929,17 +1929,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", -] - -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", + "syn 2.0.90", ] [[package]] @@ -2002,12 +1992,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", - "hashbrown 0.15.1", + "hashbrown 0.15.2", "serde", ] @@ -2075,9 +2065,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "jobserver" @@ -2090,10 +2080,11 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a88f1bda2bd75b0452a14784937d796722fdebfe50df998aeb3f0b7603019a9" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ + "once_cell", "wasm-bindgen", ] @@ -2130,9 +2121,9 @@ dependencies = [ [[package]] name = "konst" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b65f00fb3910881e52bf0850ae2a82aea411488a557e1c02820ceaa60963dce3" +checksum = "298ddf99f06a97c1ecd0e910932662b7842855046234b0d0376d35d93add087f" dependencies = [ "const_panic", "konst_kernel", @@ -2141,9 +2132,9 @@ dependencies = [ [[package]] name = "konst_kernel" -version = "0.3.12" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "599c1232f55c72c7fc378335a3efe1c878c92720838c8e6a4fd87784ef7764de" +checksum = "e4b1eb7788f3824c629b1116a7a9060d6e898c358ebff59070093d51103dcc3c" dependencies = [ "typewit", ] @@ -2168,7 +2159,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2185,15 +2176,15 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.164" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libloading" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", "windows-targets 0.52.6", @@ -2371,11 +2362,10 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ - "hermit-abi", "libc", "log", "wasi", @@ -2544,7 +2534,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.6.0", + "indexmap 2.7.0", "js-sys", "once_cell", "pin-project-lite", @@ -2619,9 +2609,9 @@ dependencies = [ [[package]] name = "os_info" -version = "3.8.2" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae99c7fa6dd38c7cafe1ec085e804f8f555a2f8659b0dbe03f1f9963a9b51092" +checksum = "e5ca711d8b83edbb00b44d504503cd247c9c0bd8b0fa2694f2a1a3d8165379ce" dependencies = [ "log", "serde", @@ -2700,7 +2690,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2783,7 +2773,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2816,9 +2806,9 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "png" -version = "0.17.14" +version = "0.17.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f9d46a34a05a6a57566bc2bfae066ef07585a6e3fa30fbbdff5936380623f0" +checksum = "b67582bd5b65bdff614270e2ea89a1cf15bef71245cc1e5f7ea126977144211d" dependencies = [ "bitflags 1.3.2", "crc32fast", @@ -2855,7 +2845,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" dependencies = [ "proc-macro2", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -2884,16 +2874,16 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "version_check", "yansi", ] [[package]] name = "prost" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" dependencies = [ "bytes", "prost-derive", @@ -2901,22 +2891,22 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] name = "prost-types" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" dependencies = [ "prost", ] @@ -2961,10 +2951,10 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.0.0", - "rustls 0.23.18", + "rustc-hash 2.1.0", + "rustls", "socket2", - "thiserror 2.0.3", + "thiserror 2.0.6", "tokio", "tracing", ] @@ -2979,11 +2969,11 @@ dependencies = [ "getrandom", "rand", "ring", - "rustc-hash 2.0.0", - "rustls 0.23.18", + "rustc-hash 2.1.0", + "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.3", + "thiserror 2.0.6", "tinyvec", "tracing", "web-time 1.1.0", @@ -2991,9 +2981,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" dependencies = [ "cfg_aliases", "libc", @@ -3123,7 +3113,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.18", + "rustls", "rustls-native-certs", "rustls-pemfile", "rustls-pki-types", @@ -3172,7 +3162,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "assign", "js_int", @@ -3194,7 +3184,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "js_int", "ruma-common", @@ -3206,7 +3196,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "as_variant", "assign", @@ -3221,7 +3211,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror 2.0.3", + "thiserror 2.0.6", "url", "web-time 1.1.0", ] @@ -3229,14 +3219,14 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "as_variant", "base64 0.22.1", "bytes", "form_urlencoded", "http", - "indexmap 2.6.0", + "indexmap 2.7.0", "js_int", "konst", "percent-encoding", @@ -3247,7 +3237,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror 2.0.3", + "thiserror 2.0.6", "time", "tracing", "url", @@ -3259,10 +3249,10 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "as_variant", - "indexmap 2.6.0", + "indexmap 2.7.0", "js_int", "js_option", "percent-encoding", @@ -3273,7 +3263,7 @@ dependencies = [ "ruma-macros", "serde", "serde_json", - "thiserror 2.0.3", + "thiserror 2.0.6", "tracing", "url", "web-time 1.1.0", @@ -3283,7 +3273,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "bytes", "http", @@ -3301,16 +3291,16 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "js_int", - "thiserror 2.0.3", + "thiserror 2.0.6", ] [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "js_int", "ruma-common", @@ -3320,7 +3310,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3328,14 +3318,14 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.89", + "syn 2.0.90", "toml", ] [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "js_int", "ruma-common", @@ -3347,20 +3337,20 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "headers", "http", "http-auth", "ruma-common", - "thiserror 2.0.3", + "thiserror 2.0.6", "tracing", ] [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3370,13 +3360,13 @@ dependencies = [ "serde_json", "sha2", "subslice", - "thiserror 2.0.3", + "thiserror 2.0.6", ] [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=1a550585bf025cce48ef8b734339245092bc986e#1a550585bf025cce48ef8b734339245092bc986e" +source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "futures-util", "js_int", @@ -3384,7 +3374,7 @@ dependencies = [ "ruma-events", "serde", "serde_json", - "thiserror 2.0.3", + "thiserror 2.0.6", "tracing", ] @@ -3436,9 +3426,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.0.0" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" +checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" [[package]] name = "rustc_version" @@ -3451,36 +3441,22 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.22.4" +version = "0.23.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" -dependencies = [ - "log", - "ring", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls" -version = "0.23.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9cc1d47e243d655ace55ed38201c19ae02c148ae56412ab8750e8f0166ab7f" +checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" dependencies = [ "aws-lc-rs", "log", @@ -3550,7 +3526,7 @@ dependencies = [ "futures-util", "pin-project", "thingbuf", - "thiserror 2.0.3", + "thiserror 2.0.6", "unicode-segmentation", "unicode-width 0.2.0", ] @@ -3622,13 +3598,13 @@ checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "sentry" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5484316556650182f03b43d4c746ce0e3e48074a21e2f51244b648b6542e1066" +checksum = "016958f51b96861dead7c1e02290f138411d05e94fad175c8636a835dee6e51e" dependencies = [ "httpdate", "reqwest", - "rustls 0.22.4", + "rustls", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -3644,9 +3620,9 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40aa225bb41e2ec9d7c90886834367f560efc1af028f1c5478a6cce6a59c463a" +checksum = "e57712c24e99252ef175b4b06c485294f10ad6bc5b5e1567ff3803ee7a0b7d3f" dependencies = [ "backtrace", "once_cell", @@ -3656,9 +3632,9 @@ dependencies = [ [[package]] name = "sentry-contexts" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a8dd746da3d16cb8c39751619cefd4fcdbd6df9610f3310fd646b55f6e39910" +checksum = "eba8754ec3b9279e00aa6d64916f211d44202370a1699afde1db2c16cbada089" dependencies = [ "hostname 0.4.0", "libc", @@ -3670,9 +3646,9 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161283cfe8e99c8f6f236a402b9ccf726b201f365988b5bb637ebca0abbd4a30" +checksum = "f9f8b6dcd4fbae1e3e22b447f32670360b27e31b62ab040f7fb04e0f80c04d92" dependencies = [ "once_cell", "rand", @@ -3683,9 +3659,9 @@ dependencies = [ [[package]] name = "sentry-debug-images" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc6b25e945fcaa5e97c43faee0267eebda9f18d4b09a251775d8fef1086238a" +checksum = "8982a69133d3f5e4efdbfa0776937fca43c3a2e275a8fe184f50b1b0aa92e07c" dependencies = [ "findshlibs", "once_cell", @@ -3694,9 +3670,9 @@ dependencies = [ [[package]] name = "sentry-log" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75bbcc61886955045a1dd4bdb173412a80bb2571be3c5bfcf7eb8f55a442bbf5" +checksum = "efcbfbb74628eaef033c1154d4bb082437c7592ce2282c7c5ccb455c4c97a06d" dependencies = [ "log", "sentry-core", @@ -3704,9 +3680,9 @@ dependencies = [ [[package]] name = "sentry-panic" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc74f229c7186dd971a9491ffcbe7883544aa064d1589bd30b83fb856cd22d63" +checksum = "de296dae6f01e931b65071ee5fe28d66a27909857f744018f107ed15fd1f6b25" dependencies = [ "sentry-backtrace", "sentry-core", @@ -3714,9 +3690,9 @@ dependencies = [ [[package]] name = "sentry-tower" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c90802b38c899a2c9e557dff25ad186362eddf755d5f5244001b172dd03bead" +checksum = "fcdaf9b1939589476bd57751d12a9653bbfe356610fc476d03d7683189183ab7" dependencies = [ "http", "pin-project", @@ -3728,9 +3704,9 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3c5faf2103cd01eeda779ea439b68c4ee15adcdb16600836e97feafab362ec" +checksum = "263f73c757ed7915d3e1e34625eae18cad498a95b4261603d4ce3f87b159a6f0" dependencies = [ "sentry-backtrace", "sentry-core", @@ -3740,9 +3716,9 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.34.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d68cdf6bc41b8ff3ae2a9c4671e97426dcdd154cc1d4b6b72813f285d6b163f" +checksum = "a71ed3a389948a6a6d92b98e997a2723ca22f09660c5a7b7388ecd509a70a527" dependencies = [ "debugid", "hex", @@ -3757,22 +3733,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -3782,7 +3758,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" dependencies = [ "form_urlencoded", - "indexmap 2.6.0", + "indexmap 2.7.0", "itoa", "ryu", "serde", @@ -3847,7 +3823,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "itoa", "ryu", "serde", @@ -3962,9 +3938,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" dependencies = [ "libc", "windows-sys 0.52.0", @@ -4052,9 +4028,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.89" +version = "2.0.90" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" +checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" dependencies = [ "proc-macro2", "quote", @@ -4084,7 +4060,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4135,11 +4111,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.6", ] [[package]] @@ -4150,18 +4126,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4226,9 +4202,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.36" +version = "0.3.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", "itoa", @@ -4247,9 +4223,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" dependencies = [ "num-conv", "time-core", @@ -4282,9 +4258,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -4306,14 +4282,14 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] name = "tokio-metrics" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eace09241d62c98b7eeb1107d4c5c64ca3bd7da92e8c218c153ab3a78f9be112" +checksum = "cb2bb07a8451c4c6fa8b3497ad198510d8b8dffa5df5cfb97a64102a58b113c8" dependencies = [ "futures-util", "pin-project-lite", @@ -4323,12 +4299,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.18", - "rustls-pki-types", + "rustls", "tokio", ] @@ -4346,9 +4321,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -4357,9 +4332,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -4395,7 +4370,7 @@ version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.6.0", + "indexmap 2.7.0", "serde", "serde_spanned", "toml_datetime", @@ -4520,7 +4495,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61 dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -4602,9 +4577,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "typewit" -version = "1.9.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fb9ae6a3cafaf0a5d14c2302ca525f9ae8e07a0f0e6949de88d882c37a6e24" +checksum = "cb77c29baba9e4d3a6182d51fa75e3215c7fd1dab8f4ea9d107c716878e55fc0" dependencies = [ "typewit_proc_macros", ] @@ -4639,27 +4614,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -4692,14 +4652,14 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.10.1" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74fc6b57825be3373f7054754755f03ac3a8f5d70015ccad699ba2029956f4a" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls 0.23.18", + "rustls", "rustls-pki-types", "url", "webpki-roots", @@ -4712,7 +4672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna 1.0.3", + "idna", "percent-encoding", "serde", ] @@ -4786,9 +4746,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "128d1e363af62632b8eb57219c8fd7877144af57558fb2ef0368d0087bddeb2e" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -4797,36 +4757,36 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb6dd4d3ca0ddffd1dd1c9c04f94b868c37ff5fac97c30b97cff2d74fce3a358" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.45" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7ec4f8827a71586374db3e87abdb5a2bb3a15afed140221307c3ec06b1f63b" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", + "once_cell", "wasm-bindgen", "web-sys", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e79384be7f8f5a9dd5d7167216f022090cf1f9ec128e6e6a482a2cb5c5422c56" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4834,28 +4794,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26c6ab57572f7a24a4985830b120de1594465e5d500f24afe89e16b4e833ef68" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.95" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65fc09f10666a9f147042251e0dda9c18f166ff7de300607007e96bdebc1068d" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "web-sys" -version = "0.3.72" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6488b90108c040df0fe62fa815cbdee25124641df01814dd7282749234c6112" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -5219,7 +5179,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "synstructure", ] @@ -5241,7 +5201,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] @@ -5261,7 +5221,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", "synstructure", ] @@ -5290,7 +5250,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.89", + "syn 2.0.90", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 673cab09..8da6728c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ version = "0.5.7" version = "0.2.9" [workspace.dependencies.cargo_toml] -version = "0.20" +version = "0.21" default-features = false features = ["features"] @@ -79,13 +79,13 @@ version = "0.8.5" # Used for the http request / response body type for Ruma endpoints used with reqwest [workspace.dependencies.bytes] -version = "1.8.0" +version = "1.9.0" [workspace.dependencies.http-body-util] version = "0.1.2" [workspace.dependencies.http] -version = "1.1.0" +version = "1.2.0" [workspace.dependencies.regex] version = "1.11.1" @@ -138,7 +138,7 @@ features = [ ] [workspace.dependencies.rustls] -version = "0.23.16" +version = "0.23.19" default-features = false features = ["aws_lc_rs"] @@ -153,7 +153,7 @@ features = [ ] [workspace.dependencies.serde] -version = "1.0.215" +version = "1.0.216" default-features = false features = ["rc"] @@ -199,7 +199,7 @@ default-features = false version = "0.1.41" default-features = false [workspace.dependencies.tracing-subscriber] -version = "0.3.18" +version = "=0.3.18" default-features = false features = ["env-filter", "std", "tracing", "tracing-log", "ansi", "fmt"] [workspace.dependencies.tracing-core] @@ -213,7 +213,7 @@ default-features = false # used for conduit's CLI and admin room command parsing [workspace.dependencies.clap] -version = "4.5.21" +version = "4.5.23" default-features = false features = [ "derive", @@ -231,7 +231,7 @@ default-features = false features = ["std", "async-await"] [workspace.dependencies.tokio] -version = "1.41.1" +version = "1.42.0" default-features = false features = [ "fs", @@ -245,10 +245,10 @@ features = [ ] [workspace.dependencies.tokio-metrics] -version = "0.3.1" +version = "0.4.0" [workspace.dependencies.libloading] -version = "0.8.5" +version = "0.8.6" # Validating urls in config, was already a transitive dependency [workspace.dependencies.url] @@ -294,12 +294,12 @@ default-features = false features = ["env", "toml"] [workspace.dependencies.hickory-resolver] -version = "0.24.1" +version = "0.24.2" default-features = false # Used for conduit::Error type [workspace.dependencies.thiserror] -version = "2.0.3" +version = "2.0.6" default-features = false # Used when hashing the state @@ -334,7 +334,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "1a550585bf025cce48ef8b734339245092bc986e" +rev = "08f58cd3236fdf175913b2bcaf8865359696d94d" features = [ "compat", "rand", @@ -410,7 +410,7 @@ features = ["rt-tokio"] # optional sentry metrics for crash/panic reporting [workspace.dependencies.sentry] -version = "0.34.0" +version = "0.35.0" default-features = false features = [ "backtrace", @@ -426,9 +426,9 @@ features = [ ] [workspace.dependencies.sentry-tracing] -version = "0.34.0" +version = "0.35.0" [workspace.dependencies.sentry-tower] -version = "0.34.0" +version = "0.35.0" # jemalloc usage [workspace.dependencies.tikv-jemalloc-sys] @@ -480,7 +480,7 @@ default-features = false version = "0.1" [workspace.dependencies.syn] -version = "2.0.87" +version = "2.0.90" default-features = false features = ["full", "extra-traits"] From ef8392cbbeec94906a4f85278765579935225dd3 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 14:27:08 -0500 Subject: [PATCH 1206/2091] allow empty POST request bodies (synapse behaviour) to improve compatibility Signed-off-by: strawberry --- src/api/router/args.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/api/router/args.rs b/src/api/router/args.rs index 0b693956..65dd8e9e 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -2,9 +2,10 @@ use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, Bytes, BytesMut}; -use conduit::{debug, err, utils::string::EMPTY, Error, Result}; +use conduit::{debug, debug_warn, err, trace, utils::string::EMPTY, Error, Result}; use ruma::{ - api::IncomingRequest, CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedServerName, OwnedUserId, ServerName, UserId, + api::IncomingRequest, CanonicalJsonObject, CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedServerName, + OwnedUserId, ServerName, UserId, }; use service::Services; @@ -85,6 +86,19 @@ where async fn from_request(request: hyper::Request, services: &State) -> Result { let mut request = request::from(services, request).await?; let mut json_body = serde_json::from_slice::(&request.body).ok(); + + // while very unusual and really shouldn't be recommended, Synapse accepts POST + // requests with a completely empty body. very old clients, libraries, and some + // appservices still call APIs like /join like this. so let's just default to + // empty object `{}` to copy synapse's behaviour + if json_body.is_none() + && request.parts.method == http::Method::POST + && !request.parts.uri.path().contains("/media/") + { + trace!("json_body from_request: {:?}", json_body.clone()); + debug_warn!("received a POST request with an empty body, defaulting/assuming to {{}} like Synapse does"); + json_body = Some(CanonicalJsonValue::Object(CanonicalJsonObject::new())); + } let auth = auth::auth(services, &mut request, json_body.as_ref(), &T::METADATA).await?; Ok(Self { body: make_body::(services, &mut request, json_body.as_mut(), &auth)?, From 80f9536d21fe98b18f429b6c26550a8115956972 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 14:28:54 -0500 Subject: [PATCH 1207/2091] rename config.rs to account_data.rs, refactor, add some missing checks Signed-off-by: strawberry --- src/api/client/{config.rs => account_data.rs} | 82 ++++++++++++------- src/api/client/mod.rs | 4 +- 2 files changed, 54 insertions(+), 32 deletions(-) rename src/api/client/{config.rs => account_data.rs} (54%) diff --git a/src/api/client/config.rs b/src/api/client/account_data.rs similarity index 54% rename from src/api/client/config.rs rename to src/api/client/account_data.rs index 3cf71135..86ef41ef 100644 --- a/src/api/client/config.rs +++ b/src/api/client/account_data.rs @@ -1,18 +1,20 @@ use axum::extract::State; -use conduit::err; +use conduit::{err, Err}; use ruma::{ - api::client::{ - config::{get_global_account_data, get_room_account_data, set_global_account_data, set_room_account_data}, - error::ErrorKind, + api::client::config::{ + get_global_account_data, get_room_account_data, set_global_account_data, set_room_account_data, + }, + events::{ + AnyGlobalAccountDataEventContent, AnyRoomAccountDataEventContent, GlobalAccountDataEventType, + RoomAccountDataEventType, }, - events::{AnyGlobalAccountDataEventContent, AnyRoomAccountDataEventContent}, serde::Raw, - OwnedUserId, RoomId, + RoomId, UserId, }; use serde::Deserialize; use serde_json::{json, value::RawValue as RawJsonValue}; -use crate::{service::Services, Error, Result, Ruma}; +use crate::{service::Services, Result, Ruma}; /// # `PUT /_matrix/client/r0/user/{userId}/account_data/{type}` /// @@ -20,14 +22,13 @@ use crate::{service::Services, Error, Result, Ruma}; pub(crate) async fn set_global_account_data_route( State(services): State, body: Ruma, ) -> Result { - set_account_data( - &services, - None, - body.sender_user.as_ref(), - &body.event_type.to_string(), - body.data.json(), - ) - .await?; + let sender_user = body.sender_user(); + + if sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot set account data for other users."))); + } + + set_account_data(&services, None, &body.user_id, &body.event_type.to_string(), body.data.json()).await?; Ok(set_global_account_data::v3::Response {}) } @@ -38,10 +39,16 @@ pub(crate) async fn set_global_account_data_route( pub(crate) async fn set_room_account_data_route( State(services): State, body: Ruma, ) -> Result { + let sender_user = body.sender_user(); + + if sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot set account data for other users."))); + } + set_account_data( &services, Some(&body.room_id), - body.sender_user.as_ref(), + &body.user_id, &body.event_type.to_string(), body.data.json(), ) @@ -56,11 +63,15 @@ pub(crate) async fn set_room_account_data_route( pub(crate) async fn get_global_account_data_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); + + if sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot get account data of other users."))); + } let account_data: ExtractGlobalEventContent = services .account_data - .get_global(sender_user, body.event_type.clone()) + .get_global(&body.user_id, body.event_type.clone()) .await .map_err(|_| err!(Request(NotFound("Data not found."))))?; @@ -75,11 +86,15 @@ pub(crate) async fn get_global_account_data_route( pub(crate) async fn get_room_account_data_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); + + if sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot get account data of other users."))); + } let account_data: ExtractRoomEventContent = services .account_data - .get_room(&body.room_id, sender_user, body.event_type.clone()) + .get_room(&body.room_id, &body.user_id, body.event_type.clone()) .await .map_err(|_| err!(Request(NotFound("Data not found."))))?; @@ -89,28 +104,35 @@ pub(crate) async fn get_room_account_data_route( } async fn set_account_data( - services: &Services, room_id: Option<&RoomId>, sender_user: Option<&OwnedUserId>, event_type: &str, - data: &RawJsonValue, -) -> Result<()> { - let sender_user = sender_user.as_ref().expect("user is authenticated"); + services: &Services, room_id: Option<&RoomId>, sender_user: &UserId, event_type_s: &str, data: &RawJsonValue, +) -> Result { + if event_type_s == RoomAccountDataEventType::FullyRead.to_cow_str() { + return Err!(Request(BadJson( + "This endpoint cannot be used for marking a room as fully read (setting m.fully_read)" + ))); + } + + if event_type_s == GlobalAccountDataEventType::PushRules.to_cow_str() { + return Err!(Request(BadJson( + "This endpoint cannot be used for setting/configuring push rules." + ))); + } let data: serde_json::Value = - serde_json::from_str(data.get()).map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?; + serde_json::from_str(data.get()).map_err(|e| err!(Request(BadJson(warn!("Invalid JSON provided: {e}")))))?; services .account_data .update( room_id, sender_user, - event_type.into(), + event_type_s.into(), &json!({ - "type": event_type, + "type": event_type_s, "content": data, }), ) - .await?; - - Ok(()) + .await } #[derive(Deserialize)] diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index 3c9736ea..be54e65f 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -1,9 +1,9 @@ pub(super) mod account; +pub(super) mod account_data; pub(super) mod alias; pub(super) mod appservice; pub(super) mod backup; pub(super) mod capabilities; -pub(super) mod config; pub(super) mod context; pub(super) mod device; pub(super) mod directory; @@ -41,11 +41,11 @@ pub(super) mod well_known; pub use account::full_user_deactivate; pub(super) use account::*; +pub(super) use account_data::*; pub(super) use alias::*; pub(super) use appservice::*; pub(super) use backup::*; pub(super) use capabilities::*; -pub(super) use config::*; pub(super) use context::*; pub(super) use device::*; pub(super) use directory::*; From ac0545f7733f2638703b5a4f4fc57ce392036b26 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 14:29:37 -0500 Subject: [PATCH 1208/2091] always set limited true on left timeline events in sync (for now) Signed-off-by: strawberry --- src/api/client/sync/v3.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 44572970..815ec87c 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -546,9 +546,9 @@ async fn handle_left_room( events: Vec::new(), }, timeline: Timeline { - limited: false, + limited: true, // TODO: support left timeline events so we dont need to set this to true prev_batch: Some(next_batch_string.to_owned()), - events: Vec::new(), + events: Vec::new(), // and so we dont need to set this to empty vec }, state: RoomState { events: left_state_events, From 39abd9a8278f2383b474211e7d4b737dd8f80927 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 16:05:06 -0500 Subject: [PATCH 1209/2091] (temp) drop sccache version down in CI to 0.8.2 Signed-off-by: strawberry --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1441dd44..5ab4581a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,6 +155,8 @@ jobs: - name: Run sccache-cache if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') uses: mozilla-actions/sccache-action@main + with: + version: "v0.8.2" # use rust-cache - uses: Swatinem/rust-cache@v2 @@ -292,6 +294,8 @@ jobs: - name: Run sccache-cache if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') uses: mozilla-actions/sccache-action@main + with: + version: "v0.8.2" # use rust-cache - uses: Swatinem/rust-cache@v2 From 9ca964a97e765af2060cbbc1fe9f244412e2d763 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 17:36:51 -0500 Subject: [PATCH 1210/2091] update stored default push rules when removing legacy mentions Signed-off-by: strawberry --- src/api/client/push.rs | 96 +++++++++++++++++++++++++++++++++--------- 1 file changed, 75 insertions(+), 21 deletions(-) diff --git a/src/api/client/push.rs b/src/api/client/push.rs index f27ead1f..7efa59e8 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -25,7 +25,7 @@ use crate::{Error, Result, Ruma}; pub(crate) async fn get_pushrules_all_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let Some(content_value) = services .account_data @@ -46,20 +46,47 @@ pub(crate) async fn get_pushrules_all_route( let mut global_ruleset = account_data_content.global; // remove old deprecated mentions push rules as per MSC4210 + // and update the stored server default push rules #[allow(deprecated)] { use ruma::push::RuleKind::*; + if global_ruleset + .get(Override, PredefinedOverrideRuleId::ContainsDisplayName.as_str()) + .is_some() + || global_ruleset + .get(Override, PredefinedOverrideRuleId::RoomNotif.as_str()) + .is_some() + || global_ruleset + .get(Content, PredefinedContentRuleId::ContainsUserName.as_str()) + .is_some() + { + global_ruleset + .remove(Override, PredefinedOverrideRuleId::ContainsDisplayName) + .ok(); + global_ruleset + .remove(Override, PredefinedOverrideRuleId::RoomNotif) + .ok(); + global_ruleset + .remove(Content, PredefinedContentRuleId::ContainsUserName) + .ok(); - global_ruleset - .remove(Override, PredefinedOverrideRuleId::ContainsDisplayName) - .ok(); - global_ruleset - .remove(Override, PredefinedOverrideRuleId::RoomNotif) - .ok(); + global_ruleset.update_with_server_default(Ruleset::server_default(sender_user)); - global_ruleset - .remove(Content, PredefinedContentRuleId::ContainsUserName) - .ok(); + services + .account_data + .update( + None, + sender_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(PushRulesEvent { + content: PushRulesEventContent { + global: global_ruleset.clone(), + }, + }) + .expect("to json always works"), + ) + .await?; + } }; Ok(get_pushrules_all::v3::Response { @@ -113,20 +140,47 @@ pub(crate) async fn get_pushrules_global_route( let mut global_ruleset = account_data_content.global; // remove old deprecated mentions push rules as per MSC4210 + // and update the stored server default push rules #[allow(deprecated)] { use ruma::push::RuleKind::*; + if global_ruleset + .get(Override, PredefinedOverrideRuleId::ContainsDisplayName.as_str()) + .is_some() + || global_ruleset + .get(Override, PredefinedOverrideRuleId::RoomNotif.as_str()) + .is_some() + || global_ruleset + .get(Content, PredefinedContentRuleId::ContainsUserName.as_str()) + .is_some() + { + global_ruleset + .remove(Override, PredefinedOverrideRuleId::ContainsDisplayName) + .ok(); + global_ruleset + .remove(Override, PredefinedOverrideRuleId::RoomNotif) + .ok(); + global_ruleset + .remove(Content, PredefinedContentRuleId::ContainsUserName) + .ok(); - global_ruleset - .remove(Override, PredefinedOverrideRuleId::ContainsDisplayName) - .ok(); - global_ruleset - .remove(Override, PredefinedOverrideRuleId::RoomNotif) - .ok(); + global_ruleset.update_with_server_default(Ruleset::server_default(sender_user)); - global_ruleset - .remove(Content, PredefinedContentRuleId::ContainsUserName) - .ok(); + services + .account_data + .update( + None, + sender_user, + GlobalAccountDataEventType::PushRules.to_string().into(), + &serde_json::to_value(PushRulesEvent { + content: PushRulesEventContent { + global: global_ruleset.clone(), + }, + }) + .expect("to json always works"), + ) + .await?; + } }; Ok(get_pushrules_global_scope::v3::Response { @@ -259,7 +313,7 @@ pub(crate) async fn get_pushrule_actions_route( .global .get(body.kind.clone(), &body.rule_id) .map(|rule| rule.actions().to_owned()) - .ok_or(err!(Request(NotFound("Push rule not found."))))?; + .ok_or_else(|| err!(Request(NotFound("Push rule not found."))))?; Ok(get_pushrule_actions::v3::Response { actions, @@ -332,7 +386,7 @@ pub(crate) async fn get_pushrule_enabled_route( .global .get(body.kind.clone(), &body.rule_id) .map(ruma::push::AnyPushRuleRef::enabled) - .ok_or(err!(Request(NotFound("Push rule not found."))))?; + .ok_or_else(|| err!(Request(NotFound("Push rule not found."))))?; Ok(get_pushrule_enabled::v3::Response { enabled, From af08d3f7bd7936f13e4e426cc004dc863edee8e0 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 18:12:27 -0500 Subject: [PATCH 1211/2091] document 'DNS No connections available' troubleshooting Signed-off-by: strawberry --- docs/troubleshooting.md | 51 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index c8655e06..283cdeee 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -41,6 +41,54 @@ workarounds for this are: - Don't use Docker's default DNS setup and instead allow the container to use and communicate with your host's DNS servers (host's `/etc/resolv.conf`) +#### DNS No connections available error message + +If you receive spurious amounts of error logs saying "DNS No connections +available", this is due to your DNS server (servers from `/etc/resolv.conf`) +being overloaded and unable to handle typical Matrix federation volume. Some +users have reported that the upstream servers are rate-limiting them as well +when they get this error (e.g. popular upstreams like Google DNS). + +Matrix federation is extremely heavy and sends wild amounts of DNS requests. +Unfortunately this is by design and has only gotten worse with more +server/destination resolution steps. Synapse also expects a very perfect DNS +setup. + +There are some ways you can reduce the amount of DNS queries, but ultimately +the best solution/fix is selfhosting a high quality caching DNS server like +[Unbound][unbound-arch] without any upstream resolvers, and without DNSSEC +validation enabled. + +DNSSEC validation is highly recommended to be **disabled** due to DNSSEC being +very computationally expensive, and is extremely susceptible to denial of +service, especially on Matrix. Many servers also strangely have broken DNSSEC +setups and will result in non-functional federation. + +conduwuit cannot provide a "works-for-everyone" Unbound DNS setup guide, but +the [official Unbound tuning guide][unbound-tuning] and the [Unbound Arch Linux wiki page][unbound-arch] +may be of interest. Disabling DNSSEC on Unbound is commenting out trust-anchors +config options and removing the `validator` module. + +**Avoid** using `systemd-resolved` as it does **not** perform very well under +high load, and we have identified its DNS caching to not be very effective. + +dnsmasq can possibly work, but it does **not** support TCP fallback which can be +problematic when receiving large DNS responses such as from large SRV records. +If you still want to use dnsmasq, make sure you **disable** `dns_tcp_fallback` +in conduwuit config. + +Raising `dns_cache_entries` in conduwuit config from the default can also assist +in DNS caching, but a full-fledged external caching resolver is better and more +reliable. + +If you don't have IPv6 connectivity, changing `ip_lookup_strategy` to match +your setup can help reduce unnecessary AAAA queries +(`1 - Ipv4Only (Only query for A records, no AAAA/IPv6)`). + +If your DNS server supports it, some users have reported enabling +`query_over_tcp_only` to force only TCP querying by default has improved DNS +reliability at a slight performance cost due to TCP overhead. + ## RocksDB / database issues #### Direct IO @@ -149,3 +197,6 @@ If you are a developer, you can also view the raw jemalloc statistics with `!admin debug memory-stats`. Please note that this output is extremely large which may only be visible in the conduwuit console CLI due to PDU size limits, and is not easy for non-developers to understand. + +[unbound-tuning]: https://unbound.docs.nlnetlabs.nl/en/latest/topics/core/performance.html +[unbound-arch]: https://wiki.archlinux.org/title/Unbound From 0740d7259ac7467610b1935d8d20eb2ed2f8cf4d Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 18:22:51 -0500 Subject: [PATCH 1212/2091] document modifying systemd unit if changing database path Signed-off-by: strawberry --- docs/deploying/generic.md | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 9eafbc46..70d4b561 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -89,11 +89,25 @@ on the network level, consider something like NextDNS or Pi-Hole. ## Setting up a systemd service -The systemd unit for conduwuit can be found -[here](../configuration/examples.md#example-systemd-unit-file). You may need to -change the `ExecStart=` path to where you placed the conduwuit binary. +Two example systemd units for conduwuit can be found +[on the configuration page](../configuration/examples.md#debian-systemd-unit-file). +You may need to change the `ExecStart=` path to where you placed the conduwuit +binary if it is not `/usr/bin/conduwuit`. -On systems where rsyslog is used alongside journald (i.e. Red Hat-based distros and OpenSUSE), put `$EscapeControlCharactersOnReceive off` inside `/etc/rsyslog.conf` to allow color in logs. +On systems where rsyslog is used alongside journald (i.e. Red Hat-based distros +and OpenSUSE), put `$EscapeControlCharactersOnReceive off` inside +`/etc/rsyslog.conf` to allow color in logs. + +If you are using a different `database_path` other than the systemd unit +configured default `/var/lib/conduwuit`, you need to add your path to the +systemd unit's `ReadWritePaths=`. This can be done by either directly editing +`conduwuit.service` and reloading systemd, or running `systemctl edit conduwuit.service` +and entering the following: + +``` +[Service] +ReadWritePaths=/path/to/custom/database/path +``` ## Creating the conduwuit configuration file @@ -101,7 +115,8 @@ Now we need to create the conduwuit's config file in `/etc/conduwuit/conduwuit.toml`. The example config can be found at [conduwuit-example.toml](../configuration/examples.md). -**Please take a moment to read the config. You need to change at least the server name.** +**Please take a moment to read the config. You need to change at least the +server name.** RocksDB is the only supported database backend. From 5d51adc29de4dea8e2995a2e88ec2923d6e4f650 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 19:17:23 -0500 Subject: [PATCH 1213/2091] update complement test results Signed-off-by: strawberry --- tests/test_results/complement/test_results.jsonl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 575a22fe..0db37515 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -141,6 +141,7 @@ {"Action":"fail","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} {"Action":"pass","Test":"TestLocalPngThumbnail"} {"Action":"pass","Test":"TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint"} +{"Action":"pass","Test":"TestLocalPngThumbnail/test_/_matrix/media/v3_endpoint"} {"Action":"pass","Test":"TestMediaFilenames"} {"Action":"pass","Test":"TestMediaFilenames/Parallel"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII"} @@ -185,22 +186,23 @@ {"Action":"pass","Test":"TestRemoteAliasRequestsUnderstandUnicode"} {"Action":"pass","Test":"TestRemotePngThumbnail"} {"Action":"pass","Test":"TestRemotePngThumbnail/test_/_matrix/client/v1/media_endpoint"} +{"Action":"pass","Test":"TestRemotePngThumbnail/test_/_matrix/media/v3_endpoint"} {"Action":"fail","Test":"TestRemotePresence"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members"} {"Action":"pass","Test":"TestRemoteTyping"} -{"Action":"fail","Test":"TestRestrictedRoomsLocalJoin"} +{"Action":"pass","Test":"TestRestrictedRoomsLocalJoin"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_fail_when_left_allowed_room"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_fail_with_mangled_join_rules"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_succeed_when_invited"} -{"Action":"fail","Test":"TestRestrictedRoomsLocalJoin/Join_should_succeed_when_joined_to_allowed_room"} -{"Action":"fail","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room"} +{"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_succeed_when_joined_to_allowed_room"} +{"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_when_left_allowed_room"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_with_mangled_join_rules"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_invited"} -{"Action":"fail","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room"} +{"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoin"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_fail_when_left_allowed_room"} From 3e64a5c907362774479c5e4d7a7bb8bcc3ed7f38 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 19:46:19 -0500 Subject: [PATCH 1214/2091] delete pointless bad_query_ratelimiter harming e2ee performance Signed-off-by: strawberry --- src/api/client/keys.rs | 45 +++++--------------------------------- src/service/globals/mod.rs | 14 ------------ 2 files changed, 5 insertions(+), 54 deletions(-) diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 53ec12f9..18f7d21c 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -1,10 +1,7 @@ -use std::{ - collections::{hash_map, BTreeMap, HashMap, HashSet}, - time::Instant, -}; +use std::collections::{BTreeMap, HashMap, HashSet}; use axum::extract::State; -use conduit::{err, utils, utils::math::continue_exponential_backoff_secs, Err, Error, Result}; +use conduit::{err, utils, Error, Result}; use futures::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::{ @@ -345,41 +342,9 @@ where let mut failures = BTreeMap::new(); - let back_off = |id| async { - match services - .globals - .bad_query_ratelimiter - .write() - .expect("locked") - .entry(id) - { - hash_map::Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - }, - hash_map::Entry::Occupied(mut e) => { - *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); - }, - } - }; - let mut futures: FuturesUnordered<_> = get_over_federation .into_iter() .map(|(server, vec)| async move { - if let Some((time, tries)) = services - .globals - .bad_query_ratelimiter - .read() - .expect("locked") - .get(server) - { - // Exponential backoff - const MIN: u64 = 5 * 60; - const MAX: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN, MAX, time.elapsed(), *tries) { - return (server, Err!(BadServerResponse("bad query from {server:?}, still backing off"))); - } - } - let mut device_keys_input_fed = BTreeMap::new(); for (user_id, keys) in vec { device_keys_input_fed.insert(user_id.to_owned(), keys.clone()); @@ -388,17 +353,18 @@ where let request = federation::keys::get_keys::v1::Request { device_keys: device_keys_input_fed, }; + let response = services .sending .send_federation_request(server, request) .await; - (server, Ok(response)) + (server, response) }) .collect(); while let Some((server, response)) = futures.next().await { - if let Ok(Ok(response)) = response { + if let Ok(response) = response { for (user, master_key) in response.master_keys { let (master_key_id, mut master_key) = parse_master_key(&user, &master_key)?; @@ -426,7 +392,6 @@ where self_signing_keys.extend(response.self_signing_keys); device_keys.extend(response.device_keys); } else { - back_off(server.to_owned()).await; failures.insert(server.to_string(), json!({})); } } diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index fefff3b5..081794e2 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -21,7 +21,6 @@ pub struct Service { pub config: Config, jwt_decoding_key: Option, pub bad_event_ratelimiter: Arc>>, - pub bad_query_ratelimiter: Arc>>, pub stateres_mutex: Arc>, pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, @@ -71,7 +70,6 @@ impl crate::Service for Service { config: config.clone(), jwt_decoding_key, bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), - bad_query_ratelimiter: Arc::new(RwLock::new(HashMap::new())), stateres_mutex: Arc::new(Mutex::new(())), admin_alias: RoomAliasId::parse(format!("#admins:{}", &config.server_name)) .expect("#admins:server_name is valid alias name"), @@ -100,13 +98,6 @@ impl crate::Service for Service { .len(); writeln!(out, "bad_event_ratelimiter: {bad_event_ratelimiter}")?; - let bad_query_ratelimiter = self - .bad_query_ratelimiter - .read() - .expect("locked for reading") - .len(); - writeln!(out, "bad_query_ratelimiter: {bad_query_ratelimiter}")?; - Ok(()) } @@ -115,11 +106,6 @@ impl crate::Service for Service { .write() .expect("locked for writing") .clear(); - - self.bad_query_ratelimiter - .write() - .expect("locked for writing") - .clear(); } fn name(&self) -> &str { service::make_name(std::module_path!()) } From 66a82447da0c0f3de6abf6ebd3346ae1483615da Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 21:24:53 -0500 Subject: [PATCH 1215/2091] log hostname in DNS error tracing spans Signed-off-by: strawberry --- src/service/resolver/actual.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 8b720365..a4475216 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -258,7 +258,7 @@ impl super::Service { #[tracing::instrument(skip_all, name = "ip")] async fn query_and_cache_override(&self, overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { match self.resolver.resolver.lookup_ip(hostname.to_owned()).await { - Err(e) => Self::handle_resolve_error(&e), + Err(e) => Self::handle_resolve_error(&e, hostname), Ok(override_ip) => { if hostname != overname { debug_info!("{overname:?} overriden by {hostname:?}"); @@ -286,7 +286,7 @@ impl super::Service { debug!("querying SRV for {hostname:?}"); let hostname = hostname.trim_end_matches('.'); match self.resolver.resolver.srv_lookup(hostname).await { - Err(e) => Self::handle_resolve_error(&e)?, + Err(e) => Self::handle_resolve_error(&e, hostname)?, Ok(result) => { return Ok(result.iter().next().map(|result| { FedDest::Named( @@ -304,7 +304,7 @@ impl super::Service { Ok(None) } - fn handle_resolve_error(e: &ResolveError) -> Result<()> { + fn handle_resolve_error(e: &ResolveError, host: &'_ str) -> Result<()> { use hickory_resolver::error::ResolveErrorKind; match *e.kind() { @@ -312,11 +312,11 @@ impl super::Service { .. } => { // Raise to debug_warn if we can find out the result wasn't from cache - debug!("No DNS records found: {e}"); + debug!(%host, "No DNS records found: {e}"); Ok(()) }, ResolveErrorKind::Timeout => { - Err!(warn!("DNS {e}")) + Err!(warn!(%host, "DNS {e}")) }, ResolveErrorKind::NoConnections => { error!( @@ -324,9 +324,9 @@ impl super::Service { remediate this issue to ensure proper federation connectivity." ); - Err!(error!("DNS error: {e}")) + Err!(error!(%host, "DNS error: {e}")) }, - _ => Err!(error!("DNS error: {e}")), + _ => Err!(error!(%host, "DNS error: {e}")), } } From 353dc17af4ad4e7266d67046090ddb93dca4bb01 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 21:25:07 -0500 Subject: [PATCH 1216/2091] construct private read receipt properly Signed-off-by: strawberry --- src/service/rooms/read_receipt/mod.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index a3cd7098..72f2e1ad 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -6,7 +6,7 @@ use conduit::{debug, err, warn, PduCount, PduId, RawPduId, Result}; use futures::{try_join, Stream, TryFutureExt}; use ruma::{ events::{ - receipt::{ReceiptEvent, ReceiptEventContent}, + receipt::{ReceiptEvent, ReceiptEventContent, Receipts}, AnySyncEphemeralRoomEvent, SyncEphemeralRoomEvent, }, serde::Raw, @@ -75,7 +75,8 @@ impl Service { let pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await?; let event_id: OwnedEventId = pdu.event_id.into(); - let receipt_content = BTreeMap::from_iter([( + let user_id: OwnedUserId = user_id.to_owned(); + let content: BTreeMap = BTreeMap::from_iter([( event_id, BTreeMap::from_iter([( ruma::events::receipt::ReceiptType::ReadPrivate, @@ -88,9 +89,12 @@ impl Service { )]), )]), )]); - //let receipt_json = Json + let receipt_event_content = ReceiptEventContent(content); + let receipt_sync_event = SyncEphemeralRoomEvent { + content: receipt_event_content, + }; - let event = serde_json::value::to_raw_value(&receipt_content).expect("receipt_content created manually"); + let event = serde_json::value::to_raw_value(&receipt_sync_event).expect("receipt created manually"); Ok(Raw::from_json(event)) } From c9bb943a343596bf604e9a982d1e1d9262ec6d01 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 21:25:40 -0500 Subject: [PATCH 1217/2091] mark device list updates when a device is deleted/removed Signed-off-by: strawberry --- src/api/client/session.rs | 6 ------ src/service/users/mod.rs | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 573f3d97..e889a867 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -243,9 +243,6 @@ pub(crate) async fn logout_route( .remove_device(sender_user, sender_device) .await; - // send device list update for user after logout - services.users.mark_device_key_update(sender_user).await; - Ok(logout::v3::Response::new()) } @@ -275,8 +272,5 @@ pub(crate) async fn logout_all_route( .for_each(|device_id| services.users.remove_device(sender_user, device_id)) .await; - // send device list update for user after logout - services.users.mark_device_key_update(sender_user).await; - Ok(logout_all::v3::Response::new()) } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index f17a6b9d..2aa80e32 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -300,6 +300,7 @@ impl Service { increment(&self.db.userid_devicelistversion, user_id.as_bytes()); self.db.userdeviceid_metadata.del(userdeviceid); + self.mark_device_key_update(user_id).await; } /// Returns an iterator over all device ids of this user. From a26b1c57220cfb1c0df3870a2f2b062f9e6dac19 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 22:02:48 -0500 Subject: [PATCH 1218/2091] send private read receipt on sliding stink (sync) Signed-off-by: strawberry --- src/api/client/sync/v4.rs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 14d79c19..3480489d 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -490,7 +490,25 @@ pub(crate) async fn sync_events_v4_route( .await, ); - let vector: Vec<_> = services + let last_privateread_update = services + .rooms + .read_receipt + .last_privateread_update(sender_user, room_id) + .await > *roomsince; + + let private_read_event = if last_privateread_update { + services + .rooms + .read_receipt + .private_read_get(room_id, sender_user) + .await + .ok() + .map(|read_event| (sender_user.to_owned(), 0_u64, read_event)) + } else { + None + }; + + let mut vector: Vec<_> = services .rooms .read_receipt .readreceipts_since(room_id, *roomsince) @@ -504,6 +522,10 @@ pub(crate) async fn sync_events_v4_route( .collect() .await; + if let Some(private_read_event) = private_read_event { + vector.push(private_read_event); + } + let receipt_size = vector.len(); receipts .rooms From e0446181c5960a7760e94747a68e769a0556bfc8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 11 Dec 2024 22:07:53 -0500 Subject: [PATCH 1219/2091] remove unnecessary args in pack_receipts Signed-off-by: strawberry --- src/api/client/sync/v4.rs | 10 +++++----- src/service/rooms/read_receipt/mod.rs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 3480489d..fddb81bf 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -26,9 +26,10 @@ use ruma::{ directory::RoomTypeFilter, events::{ room::member::{MembershipState, RoomMemberEventContent}, - AnyRawAccountDataEvent, StateEventType, + AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType::{self, *}, }, + serde::Raw, state_res::Event, uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, UInt, UserId, }; @@ -503,21 +504,20 @@ pub(crate) async fn sync_events_v4_route( .private_read_get(room_id, sender_user) .await .ok() - .map(|read_event| (sender_user.to_owned(), 0_u64, read_event)) } else { None }; - let mut vector: Vec<_> = services + let mut vector: Vec> = services .rooms .read_receipt .readreceipts_since(room_id, *roomsince) - .filter_map(|(read_user, ts, v)| async move { + .filter_map(|(read_user, _ts, v)| async move { services .users .user_is_ignored(read_user, sender_user) .await - .or_some((read_user.to_owned(), ts, v)) + .or_some(v) }) .collect() .await; diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 72f2e1ad..5cac535d 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -133,10 +133,10 @@ impl Service { #[must_use] pub fn pack_receipts(receipts: I) -> Raw> where - I: Iterator)>, + I: Iterator>, { let mut json = BTreeMap::new(); - for (_, _, value) in receipts { + for value in receipts { let receipt = serde_json::from_str::>(value.json().get()); if let Ok(value) = receipt { for (event, receipt) in value.content { From 76a5a67b6f6cedd94496e3abb695b2e1c4f88f05 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 12 Dec 2024 10:22:30 -0500 Subject: [PATCH 1220/2091] fix a couple rust 2024 lints introduced in rust 1.82.0 Signed-off-by: strawberry --- deps/rust-rocksdb/lib.rs | 114 +++++++++++++++++++-------------------- src/core/alloc/je.rs | 2 +- src/core/mods/macros.rs | 16 +++--- src/main/mods.rs | 2 +- src/router/mod.rs | 6 +-- 5 files changed, 70 insertions(+), 70 deletions(-) diff --git a/deps/rust-rocksdb/lib.rs b/deps/rust-rocksdb/lib.rs index 2e699efb..0551991a 100644 --- a/deps/rust-rocksdb/lib.rs +++ b/deps/rust-rocksdb/lib.rs @@ -2,61 +2,61 @@ pub use rust_rocksdb::*; #[cfg_attr(not(conduit_mods), link(name = "rocksdb"))] #[cfg_attr(conduit_mods, link(name = "rocksdb", kind = "static"))] -extern "C" { - pub fn rocksdb_list_column_families(); - pub fn rocksdb_logger_create_stderr_logger(); - pub fn rocksdb_logger_create_callback_logger(); - pub fn rocksdb_options_set_info_log(); - pub fn rocksdb_get_options_from_string(); - pub fn rocksdb_writebatch_create(); - pub fn rocksdb_writebatch_destroy(); - pub fn rocksdb_writebatch_put_cf(); - pub fn rocksdb_writebatch_delete_cf(); - pub fn rocksdb_iter_value(); - pub fn rocksdb_iter_seek_to_last(); - pub fn rocksdb_iter_seek_for_prev(); - pub fn rocksdb_iter_seek_to_first(); - pub fn rocksdb_iter_next(); - pub fn rocksdb_iter_prev(); - pub fn rocksdb_iter_seek(); - pub fn rocksdb_iter_valid(); - pub fn rocksdb_iter_get_error(); - pub fn rocksdb_iter_key(); - pub fn rocksdb_iter_destroy(); - pub fn rocksdb_livefiles(); - pub fn rocksdb_livefiles_count(); - pub fn rocksdb_livefiles_destroy(); - pub fn rocksdb_livefiles_column_family_name(); - pub fn rocksdb_livefiles_name(); - pub fn rocksdb_livefiles_size(); - pub fn rocksdb_livefiles_level(); - pub fn rocksdb_livefiles_smallestkey(); - pub fn rocksdb_livefiles_largestkey(); - pub fn rocksdb_livefiles_entries(); - pub fn rocksdb_livefiles_deletions(); - pub fn rocksdb_put_cf(); - pub fn rocksdb_delete_cf(); - pub fn rocksdb_get_pinned_cf(); - pub fn rocksdb_create_column_family(); - pub fn rocksdb_get_latest_sequence_number(); - pub fn rocksdb_batched_multi_get_cf(); - pub fn rocksdb_cancel_all_background_work(); - pub fn rocksdb_repair_db(); - pub fn rocksdb_list_column_families_destroy(); - pub fn rocksdb_flush(); - pub fn rocksdb_flush_wal(); - pub fn rocksdb_open_column_families(); - pub fn rocksdb_open_for_read_only_column_families(); - pub fn rocksdb_open_as_secondary_column_families(); - pub fn rocksdb_open_column_families_with_ttl(); - pub fn rocksdb_open(); - pub fn rocksdb_open_for_read_only(); - pub fn rocksdb_open_with_ttl(); - pub fn rocksdb_open_as_secondary(); - pub fn rocksdb_write(); - pub fn rocksdb_create_iterator_cf(); - pub fn rocksdb_backup_engine_create_new_backup_flush(); - pub fn rocksdb_backup_engine_options_create(); - pub fn rocksdb_write_buffer_manager_destroy(); - pub fn rocksdb_options_set_ttl(); +unsafe extern "C" { + pub unsafe fn rocksdb_list_column_families(); + pub unsafe fn rocksdb_logger_create_stderr_logger(); + pub unsafe fn rocksdb_logger_create_callback_logger(); + pub unsafe fn rocksdb_options_set_info_log(); + pub unsafe fn rocksdb_get_options_from_string(); + pub unsafe fn rocksdb_writebatch_create(); + pub unsafe fn rocksdb_writebatch_destroy(); + pub unsafe fn rocksdb_writebatch_put_cf(); + pub unsafe fn rocksdb_writebatch_delete_cf(); + pub unsafe fn rocksdb_iter_value(); + pub unsafe fn rocksdb_iter_seek_to_last(); + pub unsafe fn rocksdb_iter_seek_for_prev(); + pub unsafe fn rocksdb_iter_seek_to_first(); + pub unsafe fn rocksdb_iter_next(); + pub unsafe fn rocksdb_iter_prev(); + pub unsafe fn rocksdb_iter_seek(); + pub unsafe fn rocksdb_iter_valid(); + pub unsafe fn rocksdb_iter_get_error(); + pub unsafe fn rocksdb_iter_key(); + pub unsafe fn rocksdb_iter_destroy(); + pub unsafe fn rocksdb_livefiles(); + pub unsafe fn rocksdb_livefiles_count(); + pub unsafe fn rocksdb_livefiles_destroy(); + pub unsafe fn rocksdb_livefiles_column_family_name(); + pub unsafe fn rocksdb_livefiles_name(); + pub unsafe fn rocksdb_livefiles_size(); + pub unsafe fn rocksdb_livefiles_level(); + pub unsafe fn rocksdb_livefiles_smallestkey(); + pub unsafe fn rocksdb_livefiles_largestkey(); + pub unsafe fn rocksdb_livefiles_entries(); + pub unsafe fn rocksdb_livefiles_deletions(); + pub unsafe fn rocksdb_put_cf(); + pub unsafe fn rocksdb_delete_cf(); + pub unsafe fn rocksdb_get_pinned_cf(); + pub unsafe fn rocksdb_create_column_family(); + pub unsafe fn rocksdb_get_latest_sequence_number(); + pub unsafe fn rocksdb_batched_multi_get_cf(); + pub unsafe fn rocksdb_cancel_all_background_work(); + pub unsafe fn rocksdb_repair_db(); + pub unsafe fn rocksdb_list_column_families_destroy(); + pub unsafe fn rocksdb_flush(); + pub unsafe fn rocksdb_flush_wal(); + pub unsafe fn rocksdb_open_column_families(); + pub unsafe fn rocksdb_open_for_read_only_column_families(); + pub unsafe fn rocksdb_open_as_secondary_column_families(); + pub unsafe fn rocksdb_open_column_families_with_ttl(); + pub unsafe fn rocksdb_open(); + pub unsafe fn rocksdb_open_for_read_only(); + pub unsafe fn rocksdb_open_with_ttl(); + pub unsafe fn rocksdb_open_as_secondary(); + pub unsafe fn rocksdb_write(); + pub unsafe fn rocksdb_create_iterator_cf(); + pub unsafe fn rocksdb_backup_engine_create_new_backup_flush(); + pub unsafe fn rocksdb_backup_engine_options_create(); + pub unsafe fn rocksdb_write_buffer_manager_destroy(); + pub unsafe fn rocksdb_options_set_ttl(); } diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 7561eb95..a3a7acc0 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -59,7 +59,7 @@ pub fn memory_stats() -> Option { Some(format!("

    {str}
    ")) } -extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { +unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { // SAFETY: we have to trust the opaque points to our String let res: &mut String = unsafe { opaque.cast::().as_mut().unwrap() }; diff --git a/src/core/mods/macros.rs b/src/core/mods/macros.rs index aa0999c9..1f3b7f5f 100644 --- a/src/core/mods/macros.rs +++ b/src/core/mods/macros.rs @@ -23,11 +23,11 @@ macro_rules! mod_dtor { macro_rules! mod_init { ($body:block) => { #[used] - #[cfg_attr(target_family = "unix", link_section = ".init_array")] - static MOD_INIT: extern "C" fn() = { _mod_init }; + #[cfg_attr(target_family = "unix", unsafe(link_section = ".init_array"))] + static MOD_INIT: unsafe extern "C" fn() = { _mod_init }; - #[cfg_attr(target_family = "unix", link_section = ".text.startup")] - extern "C" fn _mod_init() -> () $body + #[cfg_attr(target_family = "unix", unsafe(link_section = ".text.startup"))] + unsafe extern "C" fn _mod_init() -> () $body }; } @@ -35,10 +35,10 @@ macro_rules! mod_init { macro_rules! mod_fini { ($body:block) => { #[used] - #[cfg_attr(target_family = "unix", link_section = ".fini_array")] - static MOD_FINI: extern "C" fn() = { _mod_fini }; + #[cfg_attr(target_family = "unix", unsafe(link_section = ".fini_array"))] + static MOD_FINI: unsafe extern "C" fn() = { _mod_fini }; - #[cfg_attr(target_family = "unix", link_section = ".text.startup")] - extern "C" fn _mod_fini() -> () $body + #[cfg_attr(target_family = "unix", unsafe(link_section = ".text.startup"))] + unsafe extern "C" fn _mod_fini() -> () $body }; } diff --git a/src/main/mods.rs b/src/main/mods.rs index e377e71c..16d975cc 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -1,6 +1,6 @@ #![cfg(conduit_mods)] -#[no_link] +#[unsafe(no_link)] extern crate conduit_service; use std::{ diff --git a/src/router/mod.rs b/src/router/mod.rs index 1580f605..215000cb 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -16,7 +16,7 @@ conduit::mod_ctor! {} conduit::mod_dtor! {} conduit::rustc_flags_capture! {} -#[no_mangle] +#[unsafe(no_mangle)] pub extern "Rust" fn start(server: &Arc) -> Pin>> + Send>> { AssertUnwindSafe(run::start(server.clone())) .catch_unwind() @@ -25,7 +25,7 @@ pub extern "Rust" fn start(server: &Arc) -> Pin) -> Pin> + Send>> { AssertUnwindSafe(run::stop(services)) .catch_unwind() @@ -34,7 +34,7 @@ pub extern "Rust" fn stop(services: Arc) -> Pin) -> Pin> + Send>> { AssertUnwindSafe(run::run(services.clone())) .catch_unwind() From b73d558cbac4f1f251a7a7f22db5c768549d81fa Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 13 Dec 2024 00:21:36 -0500 Subject: [PATCH 1221/2091] start producing haswell target optimised x86 binaries for rocksdb fast CRC32 support Signed-off-by: strawberry --- .github/workflows/ci.yml | 49 +++++++++++---- flake.nix | 79 +++++++++++++++++++++---- nix/pkgs/main/cross-compilation-env.nix | 1 - nix/pkgs/main/default.nix | 22 ++++++- 4 files changed, 126 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ab4581a..a1fa1895 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -302,7 +302,7 @@ jobs: with: cache-all-crates: "true" - - name: Build static ${{ matrix.target }} + - name: Build static ${{ matrix.target }}-all-features run: | if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] then @@ -325,15 +325,35 @@ jobs: mv -v target/release/conduwuit static-${{ matrix.target }} mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb + - name: Build static x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == "x86_64-linux-musl"}} + run: | + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduit target/release/conduwuit + cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + # -p conduit is the main crate name + direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb + mv -v target/release/conduwuit static-x86_64-linux-musl-x86_64-haswell-optimised + mv -v target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb x86_64-linux-musl-x86_64-haswell-optimised.deb + # quick smoke test of the x86_64 static release binary - - name: Run x86_64 static release binary + - name: Quick smoke test the x86_64 static release binary + if: ${{ matrix.target == "x86_64-linux-musl"}} run: | # GH actions default runners are x86_64 only if file result/bin/conduit | grep x86-64; then result/bin/conduit --version + result/bin/conduit --help + result/bin/conduit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown" fi - - name: Build static debug ${{ matrix.target }} + - name: Build static debug ${{ matrix.target }}-all-features run: | if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] then @@ -376,14 +396,14 @@ jobs: dpkg-deb --info ${{ matrix.target }}.deb dpkg-deb --info ${{ matrix.target }}-debug.deb - - name: Upload static-${{ matrix.target }} + - name: Upload static-${{ matrix.target }}-all-features uses: actions/upload-artifact@v4 with: name: static-${{ matrix.target }} path: static-${{ matrix.target }} if-no-files-found: error - - name: Upload deb ${{ matrix.target }} + - name: Upload deb ${{ matrix.target }}-all-features uses: actions/upload-artifact@v4 with: name: deb-${{ matrix.target }} @@ -391,14 +411,14 @@ jobs: if-no-files-found: error compression-level: 0 - - name: Upload static-${{ matrix.target }}-debug + - name: Upload static-${{ matrix.target }}-debug-all-features uses: actions/upload-artifact@v4 with: name: static-${{ matrix.target }}-debug path: static-${{ matrix.target }}-debug if-no-files-found: error - - name: Upload deb ${{ matrix.target }}-debug + - name: Upload deb ${{ matrix.target }}-debug-all-features uses: actions/upload-artifact@v4 with: name: deb-${{ matrix.target }}-debug @@ -406,19 +426,26 @@ jobs: if-no-files-found: error compression-level: 0 - - name: Build OCI image ${{ matrix.target }} + - name: Build OCI image ${{ matrix.target }}-all-features run: | bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features cp -v -f result oci-image-${{ matrix.target }}.tar.gz - - name: Build debug OCI image ${{ matrix.target }} + - name: Build OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == "x86_64-linux-musl"}} + run: | + bin/nix-build-and-cache just .#oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + cp -v -f result oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + + - name: Build debug OCI image ${{ matrix.target }}-all-features run: | bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz - - name: Upload OCI image ${{ matrix.target }} + - name: Upload OCI image ${{ matrix.target }}-all-features uses: actions/upload-artifact@v4 with: name: oci-image-${{ matrix.target }} @@ -426,7 +453,7 @@ jobs: if-no-files-found: error compression-level: 0 - - name: Upload OCI image ${{ matrix.target }}-debug + - name: Upload OCI image ${{ matrix.target }}-debug-all-features uses: actions/upload-artifact@v4 with: name: oci-image-${{ matrix.target }}-debug diff --git a/flake.nix b/flake.nix index e4ba8e3d..5041e468 100644 --- a/flake.nix +++ b/flake.nix @@ -72,9 +72,14 @@ "-DWITH_TESTS=1" # we use rust-rocksdb via C interface and dont need C++ RTTI "-DUSE_RTTI=1" + # this doesn't exist in RocksDB, and USE_SSE is deprecated for + # PORTABLE=$(march) + "-DFORCE_SSE42=1" ] old.cmakeFlags ++ [ + # no real reason to have snappy, no one uses this + "-DWITH_SNAPPY=0" # we dont need to use ldb or sst_dump (core_tools) "-DWITH_CORE_TOOLS=0" # we dont need trace tools @@ -272,6 +277,15 @@ value = scopeCrossStatic.main; } + # An output for a statically-linked binary with x86_64 haswell + # target optimisations + { + name = "${binaryName}-x86_64-haswell-optimised"; + value = scopeCrossStatic.main.override { + x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); + }; + } + # An output for a statically-linked unstripped debug ("dev") binary { name = "${binaryName}-debug"; @@ -306,6 +320,22 @@ }; } + # An output for a statically-linked binary with `--all-features` and with x86_64 haswell + # target optimisations + { + name = "${binaryName}-all-features-x86_64-haswell-optimised"; + value = scopeCrossStatic.main.override { + all_features = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; + x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); + }; + } + # An output for a statically-linked unstripped debug ("dev") binary with `--all-features` { name = "${binaryName}-all-features-debug"; @@ -337,6 +367,17 @@ value = scopeCrossStatic.oci-image; } + # An output for an OCI image based on that binary with x86_64 haswell + # target optimisations + { + name = "oci-image-${crossSystem}-x86_64-haswell-optimised"; + value = scopeCrossStatic.oci-image.override { + main = scopeCrossStatic.main.override { + x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); + }; + }; + } + # An output for an OCI image based on that unstripped debug ("dev") binary { name = "oci-image-${crossSystem}-debug"; @@ -365,21 +406,39 @@ }; } + # An output for an OCI image based on that binary with `--all-features` and with x86_64 haswell + # target optimisations + { + name = "oci-image-${crossSystem}-all-features-x86_64-haswell-optimised"; + value = scopeCrossStatic.oci-image.override { + main = scopeCrossStatic.main.override { + all_features = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; + x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); + }; + }; + } + # An output for an OCI image based on that unstripped debug ("dev") binary with `--all-features` { name = "oci-image-${crossSystem}-all-features-debug"; value = scopeCrossStatic.oci-image.override { main = scopeCrossStatic.main.override { - profile = "dev"; - all_features = true; - # debug build users expect full logs - disable_release_max_log_level = true; - disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" - # dont include experimental features - "experimental" - ]; + profile = "dev"; + all_features = true; + # debug build users expect full logs + disable_release_max_log_level = true; + disable_features = [ + # this is non-functional on nix for some reason + "hardened_malloc" + # dont include experimental features + "experimental" + ]; }; }; } diff --git a/nix/pkgs/main/cross-compilation-env.nix b/nix/pkgs/main/cross-compilation-env.nix index 3fb94d41..0f326c92 100644 --- a/nix/pkgs/main/cross-compilation-env.nix +++ b/nix/pkgs/main/cross-compilation-env.nix @@ -45,7 +45,6 @@ lib.optionalAttrs stdenv.hostPlatform.isStatic { ( let inherit (rust.lib) envVars; - #shouldUseLLD = platform: platform.isAarch64 && platform.isStatic && !stdenv.hostPlatform.isDarwin; in lib.optionalAttrs (stdenv.targetPlatform.rust.rustcTarget diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index abc85d82..32e11c06 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -13,12 +13,17 @@ , stdenv # Options (keep sorted) -, default_features ? true -, disable_release_max_log_level ? false , all_features ? false +, default_features ? true , disable_features ? [] +, disable_release_max_log_level ? false , features ? [] , profile ? "release" +# rocksdb compiled with -march=haswell and target-cpu=haswell rustflag +# haswell is pretty much any x86 cpu made in the last 12 years, and +# supports modern CPU extensions that rocksdb can make use of. +# disable if trying to make a portable x86_64 build for very old hardware +, x86_64_haswell_target_optimised ? false }: let @@ -79,6 +84,15 @@ buildDepsOnlyEnv = enableLiburing = enableLiburing; }).overrideAttrs (old: { enableLiburing = enableLiburing; + cmakeFlags = lib.optional x86_64_haswell_target_optimised (lib.subtractLists [ + # dont make a portable build if x86_64_haswell_target_optimised is enabled + "-DPORTABLE=1" + ] + old.cmakeFlags) + ++ lib.optionals x86_64_haswell_target_optimised [ + "-DPORTABLE=haswell" + ] + ++ old.cmakeFlags; }); in { @@ -105,7 +119,9 @@ buildPackageEnv = { # Only needed in static stdenv because these are transitive dependencies of rocksdb CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS + lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic) - " -L${lib.getLib liburing}/lib -luring"; + " -L${lib.getLib liburing}/lib -luring" + + lib.optionalString x86_64_haswell_target_optimised + " -Ctarget-cpu=haswell"; }; From c5aca8036d57e907d100f80554cfbecd9b90dbc7 Mon Sep 17 00:00:00 2001 From: Asen <78863300+AsenHu@users.noreply.github.com> Date: Fri, 13 Dec 2024 01:46:09 -0500 Subject: [PATCH 1222/2091] add auto upload assets, upload binary to webserver Co-authored-by: strawberry Signed-off-by: strawberry --- .github/workflows/release.yml | 106 ++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..902f0132 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,106 @@ +name: Upload Release Assets + +on: + release: + types: [published, prereleased] + workflow_dispatch: + inputs: + tag: + description: 'Tag to release' + required: true + type: string + action_id: + description: 'Action ID of the CI run' + required: true + type: string + +permissions: + contents: writes + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: get latest ci id + id: get_ci_id + env: + GH_TOKEN: ${{ github.token }} + run: | + if [ "${{ github.event_name }}" == "workflow_dispatch" ] + then + id="${{ github.event.inputs.action_id }}" + tag="${{ github.event.inputs.tag }}" + else + # get all runs of the ci workflow + json=$(gh api "repos/${{ github.repository }}/actions/workflows/ci.yml/runs") + + # find first run that is github sha and status is completed + id=$(echo "$json" | jq ".workflow_runs[] | select(.head_sha == \"${{ github.sha }}\" and .status == \"completed\") | .id" | head -n 1) + if [ ! "$id" ]; then + echo "No completed runs found" + echo "ci_id=0" >> "$GITHUB_OUTPUT" + exit 0 + fi + + tag="${{ github.event.release.tag_name }}" + fi + + echo "ci_id=$id" >> "$GITHUB_OUTPUT" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: get latest ci artifacts + if: steps.get_ci_id.outputs.ci_id != 0 + uses: actions/download-artifact@v4 + env: + GH_TOKEN: ${{ github.token }} + with: + merge-multiple: true + run-id: ${{ steps.get_ci_id.outputs.ci_id }} + github-token: ${{ github.token }} + + - run: | + ls + + - name: upload release assets + if: steps.get_ci_id.outputs.ci_id != 0 + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.get_ci_id.outputs.tag }} + run: | + for file in $(find . -type f); do + echo "Uploading $file..." + gh release upload $TAG "$file" --clobber --repo="${{github.repository}}" || echo "Something went wrong, skipping." + done + + - name: upload release assets to website + if: steps.get_ci_id.outputs.ci_id != 0 + env: + TAG: ${{ steps.get_ci_id.outputs.tag }} + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config < Date: Fri, 13 Dec 2024 01:47:53 -0500 Subject: [PATCH 1223/2091] upload CI binaries to webserver for easy direct download Signed-off-by: strawberry --- .github/workflows/ci.yml | 186 +++++++++++++++++++++++++++++++++++--- flake.nix | 2 + nix/pkgs/main/default.nix | 10 +- 3 files changed, 180 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1fa1895..5b44cbbb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,7 @@ env: accept-flake-config = true # complement uses libolm NIXPKGS_ALLOW_INSECURE: 1 + WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} permissions: packages: write @@ -62,6 +63,34 @@ jobs: name: Test runs-on: ubuntu-24.04 steps: + - name: Setup SSH web publish + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config < Date: Fri, 13 Dec 2024 11:10:23 -0500 Subject: [PATCH 1224/2091] use database::Json() to serialise MSC4133 profile value before insert Signed-off-by: strawberry --- src/service/users/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 2aa80e32..70868ec8 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -866,7 +866,7 @@ impl Service { let key = (user_id, profile_key); if let Some(value) = profile_key_value { - self.db.useridprofilekey_value.put(key, value); + self.db.useridprofilekey_value.put(key, Json(value)); } else { self.db.useridprofilekey_value.del(key); } From a9f6bb6169d60b5c2e9cd2cdbf11bba4464658cb Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 13 Dec 2024 13:35:36 -0500 Subject: [PATCH 1225/2091] remove jemalloc_prof and jemalloc_stats from all-features builds (exc. debug) Signed-off-by: strawberry --- flake.nix | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/flake.nix b/flake.nix index ba93333f..08120ffd 100644 --- a/flake.nix +++ b/flake.nix @@ -208,6 +208,10 @@ "hardened_malloc" # dont include experimental features "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" ]; }; all-features-debug = scopeHost.main.override { @@ -233,6 +237,10 @@ "hardened_malloc" # dont include experimental features "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" ]; }; }; @@ -318,6 +326,10 @@ "hardened_malloc" # dont include experimental features "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" ]; }; } @@ -333,6 +345,10 @@ "hardened_malloc" # dont include experimental features "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" ]; x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); }; @@ -403,6 +419,10 @@ "hardened_malloc" # dont include experimental features "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" ]; }; }; @@ -420,6 +440,10 @@ "hardened_malloc" # dont include experimental features "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" ]; x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); }; @@ -482,6 +506,10 @@ "hardened_malloc" # dont include experimental features "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" ]; }; })); From e4489a5d20ef0b1a7f36474f57e4740da1052530 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 13 Dec 2024 21:59:37 -0500 Subject: [PATCH 1226/2091] bump rust-rocksdb fork Signed-off-by: strawberry --- Cargo.lock | 4 ++-- deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86b90c3d..94533ae1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3381,7 +3381,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.30.0+9.8.4" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=3a9640a6b9173d24819c22a49487e31d20a2e59e#3a9640a6b9173d24819c22a49487e31d20a2e59e" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=fcb772d84572c63b1defc57baf9cdbdf5577c6a4#fcb772d84572c63b1defc57baf9cdbdf5577c6a4" dependencies = [ "bindgen", "bzip2-sys", @@ -3398,7 +3398,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.34.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=3a9640a6b9173d24819c22a49487e31d20a2e59e#3a9640a6b9173d24819c22a49487e31d20a2e59e" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=fcb772d84572c63b1defc57baf9cdbdf5577c6a4#fcb772d84572c63b1defc57baf9cdbdf5577c6a4" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index b6895ad3..e89c26d7 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "3a9640a6b9173d24819c22a49487e31d20a2e59e" +rev = "fcb772d84572c63b1defc57baf9cdbdf5577c6a4" #branch = "master" default-features = false From c6bf8f5ea15a6b963220a1de4bb50a639d0d0696 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 14 Dec 2024 21:00:33 -0500 Subject: [PATCH 1227/2091] improve gh actions security using zizmor https://github.com/woodruffw/zizmor Signed-off-by: strawberry --- .github/workflows/ci.yml | 166 +++++++++++++++------------- .github/workflows/documentation.yml | 8 +- .github/workflows/release.yml | 154 ++++++++++++++------------ 3 files changed, 176 insertions(+), 152 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b44cbbb..4dd47a30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,10 +53,10 @@ env: # complement uses libolm NIXPKGS_ALLOW_INSECURE: 1 WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} + GH_SHA: ${{ github.sha }} + GH_REF_NAME: ${{ github.ref_name }} -permissions: - packages: write - contents: read +permissions: {} jobs: tests: @@ -88,8 +88,8 @@ jobs: ssh -q website "echo test" echo "Creating commit rev directory on web server" - ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${{ github.sha }}/" - ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${{ github.sha }}/" + ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/$GITHUB_SHA/" + ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/$GITHUB_SHA/" - name: Install liburing run: | @@ -106,6 +106,8 @@ jobs: - name: Sync repository uses: actions/checkout@v4 + with: + persist-credentials: false - name: Tag comparison check if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} @@ -113,7 +115,7 @@ jobs: # Tag mismatch with latest repo tag check to prevent potential downgrades LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) - if [ $LATEST_TAG != ${{ github.ref_name }} ]; then + if [ ${LATEST_TAG} != ${GH_REF_NAME} ]; then echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY exit 1 @@ -162,8 +164,8 @@ jobs: if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }} run: | sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null < >(tee -a complement_diff_output.log) - name: Update Job Summary + env: + GH_JOB_STATUS: ${{ job.status }} if: success() || failure() run: | - if [ ${{ job.status }} == 'success' ]; then + if [ ${GH_JOB_STATUS} == 'success' ]; then echo '# ✅ completed suwuccessfully' >> $GITHUB_STEP_SUMMARY else echo '# CI failure' >> $GITHUB_STEP_SUMMARY @@ -264,6 +268,8 @@ jobs: steps: - name: Sync repository uses: actions/checkout@v4 + with: + persist-credentials: false - name: Setup SSH web publish if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' @@ -332,8 +338,8 @@ jobs: if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }} run: | sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <> $GITHUB_STEP_SUMMARY exit 1 @@ -671,14 +680,14 @@ jobs: if: ${{ matrix.os == 'macos-13' }} run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then - scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${{ github.sha }}/conduwuit-macos-x86_64 + scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-x86_64 fi - name: Upload macOS arm64 binary to webserver if: ${{ matrix.os == 'macos-latest' }} run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then - scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${{ github.sha }}/conduwuit-macos-arm64 + scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-arm64 fi - name: Upload macOS x86_64 binary @@ -701,6 +710,9 @@ jobs: name: Docker publish runs-on: ubuntu-24.04 needs: build + permissions: + packages: write + contents: read if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' env: DOCKER_ARM64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 @@ -756,92 +768,92 @@ jobs: if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | docker load -i oci-image-amd64.tar.gz - docker tag $(docker images -q conduit:main) ${{ env.DOCKER_AMD64 }} - docker tag $(docker images -q conduit:main) ${{ env.GHCR_AMD64 }} - docker tag $(docker images -q conduit:main) ${{ env.GLCR_AMD64 }} - docker push ${{ env.DOCKER_AMD64 }} - docker push ${{ env.GHCR_AMD64 }} - docker push ${{ env.GLCR_AMD64 }} + docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64} + docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64} + docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64} + docker push ${DOCKER_AMD64} + docker push ${GHCR_AMD64} + docker push ${GLCR_AMD64} - name: Load and push arm64 image if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | docker load -i oci-image-arm64v8.tar.gz - docker tag $(docker images -q conduit:main) ${{ env.DOCKER_ARM64 }} - docker tag $(docker images -q conduit:main) ${{ env.GHCR_ARM64 }} - docker tag $(docker images -q conduit:main) ${{ env.GLCR_ARM64 }} - docker push ${{ env.DOCKER_ARM64 }} - docker push ${{ env.GHCR_ARM64 }} - docker push ${{ env.GLCR_ARM64 }} + docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64} + docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64} + docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64} + docker push ${DOCKER_ARM64} + docker push ${GHCR_ARM64} + docker push ${GLCR_ARM64} - name: Load and push amd64 debug image if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | docker load -i oci-image-amd64-debug.tar.gz - docker tag $(docker images -q conduit:main) ${{ env.DOCKER_AMD64 }}-debug - docker tag $(docker images -q conduit:main) ${{ env.GHCR_AMD64 }}-debug - docker tag $(docker images -q conduit:main) ${{ env.GLCR_AMD64 }}-debug - docker push ${{ env.DOCKER_AMD64 }}-debug - docker push ${{ env.GHCR_AMD64 }}-debug - docker push ${{ env.GLCR_AMD64 }}-debug + docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64}-debug + docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64}-debug + docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64}-debug + docker push ${DOCKER_AMD64}-debug + docker push ${GHCR_AMD64}-debug + docker push ${GLCR_AMD64}-debug - name: Load and push arm64 debug image if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | docker load -i oci-image-arm64v8-debug.tar.gz - docker tag $(docker images -q conduit:main) ${{ env.DOCKER_ARM64 }}-debug - docker tag $(docker images -q conduit:main) ${{ env.GHCR_ARM64 }}-debug - docker tag $(docker images -q conduit:main) ${{ env.GLCR_ARM64 }}-debug - docker push ${{ env.DOCKER_ARM64 }}-debug - docker push ${{ env.GHCR_ARM64 }}-debug - docker push ${{ env.GLCR_ARM64 }}-debug + docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64}-debug + docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64}-debug + docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64}-debug + docker push ${DOCKER_ARM64}-debug + docker push ${GHCR_ARM64}-debug + docker push ${GLCR_ARM64}-debug - name: Create Docker combined manifests run: | # Dockerhub Container Registry - docker manifest create ${{ env.DOCKER_TAG }} --amend ${{ env.DOCKER_ARM64 }} --amend ${{ env.DOCKER_AMD64 }} - docker manifest create ${{ env.DOCKER_BRANCH }} --amend ${{ env.DOCKER_ARM64 }} --amend ${{ env.DOCKER_AMD64 }} + docker manifest create ${DOCKER_TAG} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64} + docker manifest create ${DOCKER_BRANCH} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64} # GitHub Container Registry - docker manifest create ${{ env.GHCR_TAG }} --amend ${{ env.GHCR_ARM64 }} --amend ${{ env.GHCR_AMD64 }} - docker manifest create ${{ env.GHCR_BRANCH }} --amend ${{ env.GHCR_ARM64 }} --amend ${{ env.GHCR_AMD64 }} + docker manifest create ${GHCR_TAG} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} + docker manifest create ${GHCR_BRANCH} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} # GitLab Container Registry - docker manifest create ${{ env.GLCR_TAG }} --amend ${{ env.GLCR_ARM64 }} --amend ${{ env.GLCR_AMD64 }} - docker manifest create ${{ env.GLCR_BRANCH }} --amend ${{ env.GLCR_ARM64 }} --amend ${{ env.GLCR_AMD64 }} + docker manifest create ${GLCR_TAG} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64} + docker manifest create ${GLCR_BRANCH} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64} - name: Create Docker combined debug manifests run: | # Dockerhub Container Registry - docker manifest create ${{ env.DOCKER_TAG }}-debug --amend ${{ env.DOCKER_ARM64 }}-debug --amend ${{ env.DOCKER_AMD64 }}-debug - docker manifest create ${{ env.DOCKER_BRANCH }}-debug --amend ${{ env.DOCKER_ARM64 }}-debug --amend ${{ env.DOCKER_AMD64 }}-debug + docker manifest create ${DOCKER_TAG}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug + docker manifest create ${DOCKER_BRANCH}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug # GitHub Container Registry - docker manifest create ${{ env.GHCR_TAG }}-debug --amend ${{ env.GHCR_ARM64 }}-debug --amend ${{ env.GHCR_AMD64 }}-debug - docker manifest create ${{ env.GHCR_BRANCH }}-debug --amend ${{ env.GHCR_ARM64 }}-debug --amend ${{ env.GHCR_AMD64 }}-debug + docker manifest create ${GHCR_TAG}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug + docker manifest create ${GHCR_BRANCH}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug # GitLab Container Registry - docker manifest create ${{ env.GLCR_TAG }}-debug --amend ${{ env.GLCR_ARM64 }}-debug --amend ${{ env.GLCR_AMD64 }}-debug - docker manifest create ${{ env.GLCR_BRANCH }}-debug --amend ${{ env.GLCR_ARM64 }}-debug --amend ${{ env.GLCR_AMD64 }}-debug + docker manifest create ${GLCR_TAG}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug + docker manifest create ${GLCR_BRANCH}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug - name: Push manifests to Docker registries if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | - docker manifest push ${{ env.DOCKER_TAG }} - docker manifest push ${{ env.DOCKER_BRANCH }} - docker manifest push ${{ env.GHCR_TAG }} - docker manifest push ${{ env.GHCR_BRANCH }} - docker manifest push ${{ env.GLCR_TAG }} - docker manifest push ${{ env.GLCR_BRANCH }} - docker manifest push ${{ env.DOCKER_TAG }}-debug - docker manifest push ${{ env.DOCKER_BRANCH }}-debug - docker manifest push ${{ env.GHCR_TAG }}-debug - docker manifest push ${{ env.GHCR_BRANCH }}-debug - docker manifest push ${{ env.GLCR_TAG }}-debug - docker manifest push ${{ env.GLCR_BRANCH }}-debug + docker manifest push ${DOCKER_TAG} + docker manifest push ${DOCKER_BRANCH} + docker manifest push ${GHCR_TAG} + docker manifest push ${GHCR_BRANCH} + docker manifest push ${GLCR_TAG} + docker manifest push ${GLCR_BRANCH} + docker manifest push ${DOCKER_TAG}-debug + docker manifest push ${DOCKER_BRANCH}-debug + docker manifest push ${GHCR_TAG}-debug + docker manifest push ${GHCR_BRANCH}-debug + docker manifest push ${GLCR_TAG}-debug + docker manifest push ${GLCR_BRANCH}-debug - name: Add Image Links to Job Summary if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | - echo "- \`docker pull ${{ env.DOCKER_TAG }}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${{ env.GHCR_TAG }}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${{ env.GLCR_TAG }}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${{ env.DOCKER_TAG }}-debug\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${{ env.GHCR_TAG }}-debug\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${{ env.GLCR_TAG }}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 20b1e4c6..4aa93f58 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -36,6 +36,8 @@ concurrency: group: "pages" cancel-in-progress: false +permissions: {} + jobs: docs: name: Documentation and GitHub Pages @@ -61,6 +63,8 @@ jobs: - name: Sync repository uses: actions/checkout@v4 + with: + persist-credentials: false - name: Setup GitHub Pages if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') @@ -109,8 +113,8 @@ jobs: if: ${{ (env.ATTIC_ENDPOINT != '') && (env.ATTIC_PUBLIC_KEY != '') }} run: | sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null <> "$GITHUB_OUTPUT" - exit 0 + # find first run that is github sha and status is completed + id=$(echo "$json" | jq ".workflow_runs[] | select(.head_sha == \"${GH_SHA}\" and .status == \"completed\") | .id" | head -n 1) + + if [ ! "$id" ]; then + echo "No completed runs found" + echo "ci_id=0" >> "$GITHUB_OUTPUT" + exit 0 + fi + + tag="${GH_TAG}}" fi - tag="${{ github.event.release.tag_name }}" - fi + echo "ci_id=$id" >> "$GITHUB_OUTPUT" + echo "tag=$tag" >> "$GITHUB_OUTPUT" - echo "ci_id=$id" >> "$GITHUB_OUTPUT" - echo "tag=$tag" >> "$GITHUB_OUTPUT" + - name: get latest ci artifacts + if: steps.get_ci_id.outputs.ci_id != 0 + uses: actions/download-artifact@v4 + env: + GH_TOKEN: ${{ github.token }} + with: + merge-multiple: true + run-id: ${{ steps.get_ci_id.outputs.ci_id }} + github-token: ${{ github.token }} - - name: get latest ci artifacts - if: steps.get_ci_id.outputs.ci_id != 0 - uses: actions/download-artifact@v4 - env: - GH_TOKEN: ${{ github.token }} - with: - merge-multiple: true - run-id: ${{ steps.get_ci_id.outputs.ci_id }} - github-token: ${{ github.token }} + - run: | + ls - - run: | - ls + - name: upload release assets + if: steps.get_ci_id.outputs.ci_id != 0 + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.get_ci_id.outputs.tag }} + run: | + for file in $(find . -type f); do + echo "Uploading $file..." + gh release upload $TAG "$file" --clobber --repo="${GH_REPOSITORY}" || echo "Something went wrong, skipping." + done - - name: upload release assets - if: steps.get_ci_id.outputs.ci_id != 0 - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ steps.get_ci_id.outputs.tag }} - run: | - for file in $(find . -type f); do - echo "Uploading $file..." - gh release upload $TAG "$file" --clobber --repo="${{github.repository}}" || echo "Something went wrong, skipping." - done + - name: upload release assets to website + if: steps.get_ci_id.outputs.ci_id != 0 + env: + TAG: ${{ steps.get_ci_id.outputs.tag }} + run: | + mkdir -p -v ~/.ssh - - name: upload release assets to website - if: steps.get_ci_id.outputs.ci_id != 0 - env: - TAG: ${{ steps.get_ci_id.outputs.tag }} - run: | - mkdir -p -v ~/.ssh + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 - echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts - echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 - chmod 600 ~/.ssh/id_ed25519 + cat >>~/.ssh/config <>~/.ssh/config < Date: Sat, 14 Dec 2024 21:58:01 -0500 Subject: [PATCH 1228/2091] rename conduit to conduwuit finally Signed-off-by: strawberry --- .github/workflows/ci.yml | 37 ++++---- Cargo.lock | 94 +++++++++--------- Cargo.toml | 52 +++++----- README.md | 6 ++ clippy.toml | 10 +- deps/rust-rocksdb/lib.rs | 4 +- docs/configuration.md | 2 +- docs/deploying/docker.md | 2 +- docs/deploying/generic.md | 8 +- docs/development.md | 2 +- src/admin/Cargo.toml | 12 +-- src/admin/admin.rs | 4 +- src/admin/appservice/mod.rs | 2 +- src/admin/check/commands.rs | 4 +- src/admin/check/mod.rs | 2 +- src/admin/command.rs | 2 +- src/admin/debug/commands.rs | 4 +- src/admin/debug/mod.rs | 2 +- src/admin/debug/tester.rs | 2 +- src/admin/federation/commands.rs | 2 +- src/admin/federation/mod.rs | 2 +- src/admin/media/commands.rs | 4 +- src/admin/media/mod.rs | 2 +- src/admin/mod.rs | 16 ++-- src/admin/processor.rs | 2 +- src/admin/query/account_data.rs | 2 +- src/admin/query/appservice.rs | 2 +- src/admin/query/globals.rs | 2 +- src/admin/query/mod.rs | 2 +- src/admin/query/presence.rs | 2 +- src/admin/query/pusher.rs | 2 +- src/admin/query/resolver.rs | 2 +- src/admin/query/room_alias.rs | 2 +- src/admin/query/room_state_cache.rs | 2 +- src/admin/query/sending.rs | 2 +- src/admin/query/users.rs | 2 +- src/admin/room/alias.rs | 2 +- src/admin/room/commands.rs | 2 +- src/admin/room/directory.rs | 2 +- src/admin/room/info.rs | 2 +- src/admin/room/mod.rs | 2 +- src/admin/room/moderation.rs | 2 +- src/admin/server/commands.rs | 6 +- src/admin/server/mod.rs | 2 +- src/admin/user/commands.rs | 4 +- src/admin/user/mod.rs | 2 +- src/admin/utils.rs | 2 +- src/api/Cargo.toml | 8 +- src/api/client/account.rs | 2 +- src/api/client/account_data.rs | 2 +- src/api/client/alias.rs | 2 +- src/api/client/appservice.rs | 2 +- src/api/client/backup.rs | 2 +- src/api/client/context.rs | 2 +- src/api/client/device.rs | 2 +- src/api/client/directory.rs | 2 +- src/api/client/filter.rs | 2 +- src/api/client/keys.rs | 2 +- src/api/client/media.rs | 4 +- src/api/client/media_legacy.rs | 4 +- src/api/client/membership.rs | 4 +- src/api/client/message.rs | 2 +- src/api/client/openid.rs | 2 +- src/api/client/profile.rs | 2 +- src/api/client/push.rs | 2 +- src/api/client/read_marker.rs | 2 +- src/api/client/relations.rs | 2 +- src/api/client/report.rs | 2 +- src/api/client/room/aliases.rs | 2 +- src/api/client/room/create.rs | 2 +- src/api/client/room/event.rs | 2 +- src/api/client/room/initial_sync.rs | 2 +- src/api/client/room/upgrade.rs | 2 +- src/api/client/search.rs | 2 +- src/api/client/send.rs | 2 +- src/api/client/session.rs | 2 +- src/api/client/state.rs | 2 +- src/api/client/sync/mod.rs | 2 +- src/api/client/sync/v3.rs | 4 +- src/api/client/sync/v4.rs | 2 +- src/api/client/threads.rs | 2 +- src/api/client/to_device.rs | 2 +- src/api/client/unstable.rs | 2 +- src/api/client/unversioned.rs | 4 +- src/api/client/user_directory.rs | 2 +- src/api/client/voip.rs | 2 +- src/api/client/well_known.rs | 2 +- src/api/mod.rs | 10 +- src/api/router.rs | 2 +- src/api/router/args.rs | 2 +- src/api/router/auth.rs | 2 +- src/api/router/handler.rs | 2 +- src/api/router/request.rs | 2 +- src/api/router/response.rs | 2 +- src/api/router/state.rs | 2 +- src/api/server/backfill.rs | 2 +- src/api/server/event.rs | 2 +- src/api/server/event_auth.rs | 2 +- src/api/server/get_missing_events.rs | 2 +- src/api/server/invite.rs | 2 +- src/api/server/key.rs | 2 +- src/api/server/make_join.rs | 2 +- src/api/server/make_knock.rs | 2 +- src/api/server/make_leave.rs | 2 +- src/api/server/media.rs | 4 +- src/api/server/query.rs | 2 +- src/api/server/send.rs | 2 +- src/api/server/send_join.rs | 2 +- src/api/server/send_knock.rs | 2 +- src/api/server/send_leave.rs | 2 +- src/api/server/state.rs | 2 +- src/api/server/state_ids.rs | 2 +- src/api/server/user.rs | 2 +- src/api/server/utils.rs | 4 +- src/api/server/version.rs | 4 +- src/core/Cargo.toml | 4 +- src/core/config/mod.rs | 2 +- src/core/debug.rs | 2 +- src/core/info/cargo.rs | 2 +- src/core/info/mod.rs | 2 +- src/core/mod.rs | 4 +- src/core/mods/mod.rs | 2 +- src/core/server.rs | 2 +- src/core/utils/debug.rs | 2 +- src/core/utils/mod.rs | 2 +- src/core/utils/string.rs | 2 +- src/database/Cargo.toml | 4 +- src/database/database.rs | 2 +- src/database/de.rs | 4 +- src/database/deserialized.rs | 2 +- src/database/engine.rs | 2 +- src/database/handle.rs | 2 +- src/database/keyval.rs | 2 +- src/database/map.rs | 2 +- src/database/map/contains.rs | 2 +- src/database/map/count.rs | 2 +- src/database/map/get.rs | 2 +- src/database/map/get_batch.rs | 2 +- src/database/map/insert.rs | 2 +- src/database/map/keys.rs | 2 +- src/database/map/keys_from.rs | 2 +- src/database/map/keys_prefix.rs | 2 +- src/database/map/remove.rs | 2 +- src/database/map/rev_keys.rs | 2 +- src/database/map/rev_keys_from.rs | 2 +- src/database/map/rev_keys_prefix.rs | 2 +- src/database/map/rev_stream.rs | 2 +- src/database/map/rev_stream_from.rs | 2 +- src/database/map/rev_stream_prefix.rs | 2 +- src/database/map/stream.rs | 2 +- src/database/map/stream_from.rs | 2 +- src/database/map/stream_prefix.rs | 2 +- src/database/maps.rs | 2 +- src/database/mod.rs | 8 +- src/database/opts.rs | 2 +- src/database/pool.rs | 2 +- src/database/ser.rs | 2 +- src/database/stream.rs | 2 +- src/database/stream/items.rs | 2 +- src/database/stream/items_rev.rs | 2 +- src/database/stream/keys.rs | 2 +- src/database/stream/keys_rev.rs | 2 +- src/database/tests.rs | 12 +-- src/database/util.rs | 10 +- src/macros/Cargo.toml | 4 +- src/macros/admin.rs | 2 +- src/macros/rustc.rs | 10 +- src/main/Cargo.toml | 95 +++++++++---------- src/main/clap.rs | 4 +- src/main/logging.rs | 2 +- src/main/main.rs | 10 +- src/main/mods.rs | 28 +++--- src/main/restart.rs | 2 +- src/main/sentry.rs | 4 +- src/main/server.rs | 16 ++-- src/main/signal.rs | 4 +- src/router/Cargo.toml | 10 +- src/router/layers.rs | 8 +- src/router/mod.rs | 12 +-- src/router/request.rs | 4 +- src/router/router.rs | 8 +- src/router/run.rs | 8 +- src/router/serve/mod.rs | 6 +- src/router/serve/plain.rs | 2 +- src/router/serve/tls.rs | 2 +- src/router/serve/unix.rs | 2 +- src/service/Cargo.toml | 6 +- src/service/account_data/mod.rs | 2 +- src/service/admin/console.rs | 2 +- src/service/admin/create.rs | 4 +- src/service/admin/grant.rs | 6 +- src/service/admin/mod.rs | 2 +- src/service/admin/startup.rs | 2 +- src/service/appservice/mod.rs | 2 +- src/service/appservice/namespace_regex.rs | 2 +- src/service/appservice/registration_info.rs | 2 +- src/service/client/mod.rs | 6 +- src/service/emergency/mod.rs | 18 ++-- src/service/globals/data.rs | 2 +- src/service/globals/mod.rs | 6 +- src/service/key_backups/mod.rs | 2 +- src/service/manager.rs | 2 +- src/service/media/data.rs | 2 +- src/service/media/migrations.rs | 2 +- src/service/media/mod.rs | 2 +- src/service/media/preview.rs | 4 +- src/service/media/remote.rs | 2 +- src/service/media/thumbnail.rs | 2 +- src/service/migrations.rs | 8 +- src/service/mod.rs | 12 +-- src/service/presence/data.rs | 2 +- src/service/presence/mod.rs | 2 +- src/service/presence/presence.rs | 2 +- src/service/pusher/mod.rs | 2 +- src/service/resolver/actual.rs | 2 +- src/service/resolver/cache.rs | 2 +- src/service/resolver/dns.rs | 2 +- src/service/resolver/mod.rs | 2 +- src/service/rooms/alias/mod.rs | 2 +- src/service/rooms/alias/remote.rs | 2 +- src/service/rooms/auth_chain/data.rs | 2 +- src/service/rooms/auth_chain/mod.rs | 2 +- src/service/rooms/directory/mod.rs | 2 +- src/service/rooms/event_handler/acl_check.rs | 2 +- .../fetch_and_handle_outliers.rs | 2 +- src/service/rooms/event_handler/fetch_prev.rs | 2 +- .../rooms/event_handler/fetch_state.rs | 2 +- .../event_handler/handle_incoming_pdu.rs | 2 +- .../rooms/event_handler/handle_outlier_pdu.rs | 2 +- .../rooms/event_handler/handle_prev_pdu.rs | 2 +- src/service/rooms/event_handler/mod.rs | 2 +- .../rooms/event_handler/parse_incoming_pdu.rs | 2 +- .../rooms/event_handler/resolve_state.rs | 2 +- .../rooms/event_handler/state_at_incoming.rs | 2 +- .../event_handler/upgrade_outlier_pdu.rs | 2 +- src/service/rooms/lazy_loading/mod.rs | 2 +- src/service/rooms/metadata/mod.rs | 2 +- src/service/rooms/outlier/mod.rs | 2 +- src/service/rooms/pdu_metadata/data.rs | 2 +- src/service/rooms/pdu_metadata/mod.rs | 2 +- src/service/rooms/read_receipt/data.rs | 2 +- src/service/rooms/read_receipt/mod.rs | 2 +- src/service/rooms/search/mod.rs | 2 +- src/service/rooms/short/mod.rs | 4 +- src/service/rooms/spaces/mod.rs | 2 +- src/service/rooms/state/mod.rs | 2 +- src/service/rooms/state_accessor/data.rs | 2 +- src/service/rooms/state_accessor/mod.rs | 2 +- src/service/rooms/state_cache/mod.rs | 4 +- src/service/rooms/state_compressor/mod.rs | 2 +- src/service/rooms/threads/mod.rs | 2 +- src/service/rooms/timeline/data.rs | 2 +- src/service/rooms/timeline/mod.rs | 4 +- src/service/rooms/typing/mod.rs | 2 +- src/service/rooms/user/mod.rs | 2 +- src/service/sending/appservice.rs | 2 +- src/service/sending/data.rs | 2 +- src/service/sending/dest.rs | 2 +- src/service/sending/mod.rs | 2 +- src/service/sending/send.rs | 2 +- src/service/sending/sender.rs | 2 +- src/service/server_keys/acquire.rs | 2 +- src/service/server_keys/get.rs | 2 +- src/service/server_keys/keypair.rs | 2 +- src/service/server_keys/mod.rs | 2 +- src/service/server_keys/request.rs | 2 +- src/service/server_keys/sign.rs | 2 +- src/service/server_keys/verify.rs | 2 +- src/service/service.rs | 2 +- src/service/services.rs | 2 +- src/service/sync/mod.rs | 2 +- src/service/sync/watch.rs | 2 +- src/service/transaction_ids/mod.rs | 2 +- src/service/uiaa/mod.rs | 2 +- src/service/updates/mod.rs | 2 +- src/service/users/mod.rs | 2 +- tests/cargo_smoke.sh | 4 +- 277 files changed, 559 insertions(+), 551 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4dd47a30..66c8f635 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -377,10 +377,9 @@ jobs: mkdir -v -p target/release/ mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ - cp -v -f result/bin/conduit target/release/conduwuit - cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit - # -p conduit is the main crate name - direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb mv -v target/release/conduwuit static-${{ matrix.target }} mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb @@ -394,10 +393,9 @@ jobs: mkdir -v -p target/release/ mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ - cp -v -f result/bin/conduit target/release/conduwuit - cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit - # -p conduit is the main crate name - direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb mv -v target/release/conduwuit static-x86_64-linux-musl-x86_64-haswell-optimised mv -v target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb x86_64-linux-musl-x86_64-haswell-optimised.deb @@ -406,10 +404,10 @@ jobs: if: ${{ matrix.target == 'x86_64-linux-musl' }} run: | # GH actions default runners are x86_64 only - if file result/bin/conduit | grep x86-64; then - result/bin/conduit --version - result/bin/conduit --help - result/bin/conduit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown" + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version + result/bin/conduwuit --help + result/bin/conduwuit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown" fi - name: Build static debug ${{ matrix.target }}-all-features @@ -430,10 +428,9 @@ jobs: # so we need to coerce cargo-deb into thinking this is a release binary mkdir -v -p target/release/ mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ - cp -v -f result/bin/conduit target/release/conduwuit - cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit - # -p conduit is the main crate name - direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}-debug.deb + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}-debug.deb mv -v target/release/conduwuit static-${{ matrix.target }}-debug mv -v target/release/${{ matrix.target }}-debug.deb ${{ matrix.target }}-debug.deb @@ -441,8 +438,8 @@ jobs: - name: Run x86_64 static debug binary run: | # GH actions default runners are x86_64 only - if file result/bin/conduit | grep x86-64; then - result/bin/conduit --version + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version fi # check validity of produced deb package, invalid debs will error on these commands @@ -654,7 +651,7 @@ jobs: if: ${{ matrix.os == 'macos-13' }} run: | CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release - cp -v -f target/release/conduit conduwuit-macos-x86_64 + cp -v -f target/release/conduwuit conduwuit-macos-x86_64 otool -L conduwuit-macos-x86_64 # quick smoke test of the x86_64 macOS binary @@ -667,7 +664,7 @@ jobs: if: ${{ matrix.os == 'macos-latest' }} run: | CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release - cp -v -f target/release/conduit conduwuit-macos-arm64 + cp -v -f target/release/conduwuit conduwuit-macos-arm64 otool -L conduwuit-macos-arm64 # quick smoke test of the arm64 macOS binary diff --git a/Cargo.lock b/Cargo.lock index 94533ae1..95ab8d04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -619,16 +619,16 @@ dependencies = [ ] [[package]] -name = "conduit" +name = "conduwuit" version = "0.5.0" dependencies = [ "clap", - "conduit_admin", - "conduit_api", - "conduit_core", - "conduit_database", - "conduit_router", - "conduit_service", + "conduwuit_admin", + "conduwuit_api", + "conduwuit_core", + "conduwuit_database", + "conduwuit_router", + "conduwuit_service", "console-subscriber", "const-str", "hardened_malloc-rs", @@ -648,15 +648,15 @@ dependencies = [ ] [[package]] -name = "conduit_admin" +name = "conduwuit_admin" version = "0.5.0" dependencies = [ "clap", - "conduit_api", - "conduit_core", - "conduit_database", - "conduit_macros", - "conduit_service", + "conduwuit_api", + "conduwuit_core", + "conduwuit_database", + "conduwuit_macros", + "conduwuit_service", "const-str", "futures", "log", @@ -669,7 +669,7 @@ dependencies = [ ] [[package]] -name = "conduit_api" +name = "conduwuit_api" version = "0.5.0" dependencies = [ "axum", @@ -677,9 +677,9 @@ dependencies = [ "axum-extra", "base64 0.22.1", "bytes", - "conduit_core", - "conduit_database", - "conduit_service", + "conduwuit_core", + "conduwuit_database", + "conduwuit_service", "const-str", "futures", "hmac", @@ -701,7 +701,7 @@ dependencies = [ ] [[package]] -name = "conduit_core" +name = "conduwuit_core" version = "0.5.0" dependencies = [ "argon2", @@ -713,7 +713,7 @@ dependencies = [ "checked_ops", "chrono", "clap", - "conduit_macros", + "conduwuit_macros", "const-str", "ctor", "cyborgtime", @@ -739,7 +739,7 @@ dependencies = [ "serde_json", "serde_regex", "serde_yaml", - "thiserror 2.0.6", + "thiserror 2.0.7", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -753,12 +753,12 @@ dependencies = [ ] [[package]] -name = "conduit_database" +name = "conduwuit_database" version = "0.5.0" dependencies = [ "arrayvec", "async-channel", - "conduit_core", + "conduwuit_core", "const-str", "futures", "log", @@ -771,7 +771,7 @@ dependencies = [ ] [[package]] -name = "conduit_macros" +name = "conduwuit_macros" version = "0.5.0" dependencies = [ "itertools 0.13.0", @@ -781,7 +781,7 @@ dependencies = [ ] [[package]] -name = "conduit_router" +name = "conduwuit_router" version = "0.5.0" dependencies = [ "axum", @@ -789,10 +789,10 @@ dependencies = [ "axum-server", "axum-server-dual-protocol", "bytes", - "conduit_admin", - "conduit_api", - "conduit_core", - "conduit_service", + "conduwuit_admin", + "conduwuit_api", + "conduwuit_core", + "conduwuit_service", "const-str", "futures", "http", @@ -814,15 +814,15 @@ dependencies = [ ] [[package]] -name = "conduit_service" +name = "conduwuit_service" version = "0.5.0" dependencies = [ "arrayvec", "async-trait", "base64 0.22.1", "bytes", - "conduit_core", - "conduit_database", + "conduwuit_core", + "conduwuit_database", "const-str", "either", "futures", @@ -2954,7 +2954,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls", "socket2", - "thiserror 2.0.6", + "thiserror 2.0.7", "tokio", "tracing", ] @@ -2973,7 +2973,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.6", + "thiserror 2.0.7", "tinyvec", "tracing", "web-time 1.1.0", @@ -3211,7 +3211,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.7", "url", "web-time 1.1.0", ] @@ -3237,7 +3237,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.7", "time", "tracing", "url", @@ -3263,7 +3263,7 @@ dependencies = [ "ruma-macros", "serde", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.7", "tracing", "url", "web-time 1.1.0", @@ -3294,7 +3294,7 @@ version = "0.9.5" source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" dependencies = [ "js_int", - "thiserror 2.0.6", + "thiserror 2.0.7", ] [[package]] @@ -3343,7 +3343,7 @@ dependencies = [ "http", "http-auth", "ruma-common", - "thiserror 2.0.6", + "thiserror 2.0.7", "tracing", ] @@ -3360,7 +3360,7 @@ dependencies = [ "serde_json", "sha2", "subslice", - "thiserror 2.0.6", + "thiserror 2.0.7", ] [[package]] @@ -3374,7 +3374,7 @@ dependencies = [ "ruma-events", "serde", "serde_json", - "thiserror 2.0.6", + "thiserror 2.0.7", "tracing", ] @@ -3526,7 +3526,7 @@ dependencies = [ "futures-util", "pin-project", "thingbuf", - "thiserror 2.0.6", + "thiserror 2.0.7", "unicode-segmentation", "unicode-width 0.2.0", ] @@ -4111,11 +4111,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.6" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" dependencies = [ - "thiserror-impl 2.0.6", + "thiserror-impl 2.0.7", ] [[package]] @@ -4131,9 +4131,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.6" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 8da6728c..2d99db02 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ rust-version = "1.83.0" version = "0.5.0" [workspace.metadata.crane] -name = "conduit" +name = "conduwuit" [workspace.dependencies.arrayvec] version = "0.7.4" @@ -211,7 +211,7 @@ default-features = false version = "2.0.1" default-features = false -# used for conduit's CLI and admin room command parsing +# used for conduwuit's CLI and admin room command parsing [workspace.dependencies.clap] version = "4.5.23" default-features = false @@ -297,9 +297,9 @@ features = ["env", "toml"] version = "0.24.2" default-features = false -# Used for conduit::Error type +# Used for conduwuit::Error type [workspace.dependencies.thiserror] -version = "2.0.6" +version = "2.0.7" default-features = false # Used when hashing the state @@ -523,38 +523,38 @@ rev = "deaeb0694e2083f53d363b648da06e10fc13900c" # Our crates # -[workspace.dependencies.conduit-router] -package = "conduit_router" +[workspace.dependencies.conduwuit-router] +package = "conduwuit_router" path = "src/router" default-features = false -[workspace.dependencies.conduit-admin] -package = "conduit_admin" +[workspace.dependencies.conduwuit-admin] +package = "conduwuit_admin" path = "src/admin" default-features = false -[workspace.dependencies.conduit-api] -package = "conduit_api" +[workspace.dependencies.conduwuit-api] +package = "conduwuit_api" path = "src/api" default-features = false -[workspace.dependencies.conduit-service] -package = "conduit_service" +[workspace.dependencies.conduwuit-service] +package = "conduwuit_service" path = "src/service" default-features = false -[workspace.dependencies.conduit-database] -package = "conduit_database" +[workspace.dependencies.conduwuit-database] +package = "conduwuit_database" path = "src/database" default-features = false -[workspace.dependencies.conduit-core] -package = "conduit_core" +[workspace.dependencies.conduwuit-core] +package = "conduwuit_core" path = "src/core" default-features = false -[workspace.dependencies.conduit-macros] -package = "conduit_macros" +[workspace.dependencies.conduwuit-macros] +package = "conduwuit_macros" path = "src/macros" default-features = false @@ -613,7 +613,7 @@ codegen-units = 32 # '-Clink-arg=-Wl,--no-gc-sections', #] -[profile.release-max-perf.package.conduit_macros] +[profile.release-max-perf.package.conduwuit_macros] inherits = "release-max-perf.build-override" #rustflags = [ # '-Crelocation-model=pic', @@ -647,7 +647,7 @@ panic = "unwind" debug-assertions = true incremental = true #rustflags = [ -# '--cfg', 'conduit_mods', +# '--cfg', 'conduwuit_mods', # '-Ztime-passes', # '-Zmir-opt-level=0', # '-Zvalidate-mir=false', @@ -664,11 +664,11 @@ incremental = true # '-Clink-arg=-Wl,-z,lazy', #] -[profile.dev.package.conduit_core] +[profile.dev.package.conduwuit_core] inherits = "dev" incremental = false #rustflags = [ -# '--cfg', 'conduit_mods', +# '--cfg', 'conduwuit_mods', # '-Ztime-passes', # '-Zmir-opt-level=0', # '-Ztls-model=initial-exec', @@ -685,10 +685,10 @@ incremental = false # '-Clink-arg=-Wl,-z,nodelete', #] -[profile.dev.package.conduit] +[profile.dev.package.conduwuit] inherits = "dev" #rustflags = [ -# '--cfg', 'conduit_mods', +# '--cfg', 'conduwuit_mods', # '-Ztime-passes', # '-Zmir-opt-level=0', # '-Zvalidate-mir=false', @@ -710,7 +710,7 @@ incremental = false codegen-units = 1 opt-level = 'z' #rustflags = [ -# '--cfg', 'conduit_mods', +# '--cfg', 'conduwuit_mods', # '-Ztls-model=initial-exec', # '-Cprefer-dynamic=true', # '-Zstaticlib-prefer-dynamic=true', @@ -731,7 +731,7 @@ incremental = false codegen-units = 1 opt-level = 'z' #rustflags = [ -# '--cfg', 'conduit_mods', +# '--cfg', 'conduwuit_mods', # '-Ztls-model=global-dynamic', # '-Cprefer-dynamic=true', # '-Zstaticlib-prefer-dynamic=true', diff --git a/README.md b/README.md index 4faf1ad7..e6ad1981 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,12 @@ is no harm or additional steps required for using conduwuit. See the [Migrating from Conduit](https://conduwuit.puppyirl.gay/deploying/generic.html#migrating-from-conduit) section on the generic deploying guide. +Note that as of conduwuit version 0.5.0, backwards compatibility with Conduit is +no longer supported. We only support migrating *from* Conduit, not back to +Conduit like before. If you are truly finding yourself wanting to migrate back +to Conduit, we would appreciate all your feedback and if we can assist with +any issues or concerns. + diff --git a/clippy.toml b/clippy.toml index b93b2377..3a0aa695 100644 --- a/clippy.toml +++ b/clippy.toml @@ -7,9 +7,9 @@ too-many-lines-threshold = 700 # TODO reduce me to <= 100 type-complexity-threshold = 250 # reduce me to ~200 disallowed-macros = [ - { path = "log::error", reason = "use conduit_core::error" }, - { path = "log::warn", reason = "use conduit_core::warn" }, - { path = "log::info", reason = "use conduit_core::info" }, - { path = "log::debug", reason = "use conduit_core::debug" }, - { path = "log::trace", reason = "use conduit_core::trace" }, + { path = "log::error", reason = "use conduwuit_core::error" }, + { path = "log::warn", reason = "use conduwuit_core::warn" }, + { path = "log::info", reason = "use conduwuit_core::info" }, + { path = "log::debug", reason = "use conduwuit_core::debug" }, + { path = "log::trace", reason = "use conduwuit_core::trace" }, ] diff --git a/deps/rust-rocksdb/lib.rs b/deps/rust-rocksdb/lib.rs index 0551991a..8dbbda98 100644 --- a/deps/rust-rocksdb/lib.rs +++ b/deps/rust-rocksdb/lib.rs @@ -1,7 +1,7 @@ pub use rust_rocksdb::*; -#[cfg_attr(not(conduit_mods), link(name = "rocksdb"))] -#[cfg_attr(conduit_mods, link(name = "rocksdb", kind = "static"))] +#[cfg_attr(not(conduwuit_mods), link(name = "rocksdb"))] +#[cfg_attr(conduwuit_mods, link(name = "rocksdb", kind = "static"))] unsafe extern "C" { pub unsafe fn rocksdb_list_column_families(); pub unsafe fn rocksdb_logger_create_stderr_logger(); diff --git a/docs/configuration.md b/docs/configuration.md index f4f7f4c7..0c670210 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -42,7 +42,7 @@ The syntax of this is a standard admin command without the prefix such as An example output of a success is: ``` -INFO conduit_service::admin::startup: Startup command #0 completed: +INFO conduwuit_service::admin::startup: Startup command #0 completed: Created user with user_id: @june:girlboss.ceo and password: `` ``` diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index fffa7770..d0aa13b3 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -41,7 +41,7 @@ docker run -d -p 8448:6167 \ -v db:/var/lib/conduwuit/ \ -e CONDUWUIT_SERVER_NAME="your.server.name" \ -e CONDUWUIT_ALLOW_REGISTRATION=false \ - --name conduit $LINK + --name conduwuit $LINK ``` or you can use [docker compose](#docker-compose). diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 70d4b561..398ba67f 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -26,7 +26,7 @@ allows easy cross-compilation. You can run the `nix build -L .#static-x86_64-linux-musl-all-features` or `nix build -L .#static-aarch64-linux-musl-all-features` commands based on architecture to cross-compile the necessary static binary located at -`result/bin/conduit`. This is reproducible with the static binaries produced in our CI. +`result/bin/conduwuit`. This is reproducible with the static binaries produced in our CI. Otherwise, follow standard Rust project build guides (installing git and cloning the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + @@ -38,6 +38,12 @@ As mentioned in the README, there is little to no steps needed to migrate from Conduit. As long as you are using the RocksDB database backend, just replace the binary / container image / etc. +**WARNING**: As of conduwuit 0.5.0, all database and backwards compatibility +with Conduit is no longer supported. We only support migrating *from* Conduit, +not back to Conduit like before. If you are truly finding yourself wanting to +migrate back to Conduit, we would appreciate all your feedback and if we can +assist with any issues or concerns. + **Note**: If you are relying on Conduit's "automatic delegation" feature, this will **NOT** work on conduwuit and you must configure delegation manually. This is not a mistake and no support for this feature will be added. diff --git a/docs/development.md b/docs/development.md index 28b07667..fa7519c0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -52,7 +52,7 @@ the said workspace crate(s) must define the feature there in its `Cargo.toml`. So, if this is adding a feature to the API such as `woof`, you define the feature in the `api` crate's `Cargo.toml` as `woof = []`. The feature definition in `main`'s -`Cargo.toml` will be `woof = ["conduit-api/woof"]`. +`Cargo.toml` will be `woof = ["conduwuit-api/woof"]`. The rationale for this is due to Rust / Cargo not supporting ["workspace level features"][9], we must make a choice of; either scattering diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index f5cab449..3f8fbf79 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "conduit_admin" +name = "conduwuit_admin" categories.workspace = true description.workspace = true edition.workspace = true @@ -27,11 +27,11 @@ release_max_log_level = [ [dependencies] clap.workspace = true -conduit-api.workspace = true -conduit-core.workspace = true -conduit-database.workspace = true -conduit-macros.workspace = true -conduit-service.workspace = true +conduwuit-api.workspace = true +conduwuit-core.workspace = true +conduwuit-database.workspace = true +conduwuit-macros.workspace = true +conduwuit-service.workspace = true const-str.workspace = true futures.workspace = true log.workspace = true diff --git a/src/admin/admin.rs b/src/admin/admin.rs index d1d8d394..7b707446 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -1,5 +1,5 @@ use clap::Parser; -use conduit::Result; +use conduwuit::Result; use ruma::events::room::message::RoomMessageEventContent; use crate::{ @@ -9,7 +9,7 @@ use crate::{ }; #[derive(Debug, Parser)] -#[command(name = "conduwuit", version = conduit::version())] +#[command(name = "conduwuit", version = conduwuit::version())] pub(super) enum AdminCommand { #[command(subcommand)] /// - Commands for managing appservices diff --git a/src/admin/appservice/mod.rs b/src/admin/appservice/mod.rs index ca5f46bb..2e0694aa 100644 --- a/src/admin/appservice/mod.rs +++ b/src/admin/appservice/mod.rs @@ -1,7 +1,7 @@ mod commands; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use crate::admin_command_dispatch; diff --git a/src/admin/check/commands.rs b/src/admin/check/commands.rs index 88fca462..ecb18c1a 100644 --- a/src/admin/check/commands.rs +++ b/src/admin/check/commands.rs @@ -1,5 +1,5 @@ -use conduit::Result; -use conduit_macros::implement; +use conduwuit::Result; +use conduwuit_macros::implement; use futures::StreamExt; use ruma::events::room::message::RoomMessageEventContent; diff --git a/src/admin/check/mod.rs b/src/admin/check/mod.rs index e543e5b5..44b1ddf2 100644 --- a/src/admin/check/mod.rs +++ b/src/admin/check/mod.rs @@ -1,7 +1,7 @@ mod commands; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use ruma::events::room::message::RoomMessageEventContent; use crate::Command; diff --git a/src/admin/command.rs b/src/admin/command.rs index c594736d..5277b976 100644 --- a/src/admin/command.rs +++ b/src/admin/command.rs @@ -1,6 +1,6 @@ use std::time::SystemTime; -use conduit_service::Services; +use conduwuit_service::Services; use ruma::EventId; pub(crate) struct Command<'a> { diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index a17d0800..9d77a1de 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,7 +6,7 @@ use std::{ time::{Instant, SystemTime}, }; -use conduit::{debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result}; +use conduwuit::{debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result}; use futures::{FutureExt, StreamExt}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, @@ -754,7 +754,7 @@ pub(super) async fn resolve_true_destination( #[admin_command] pub(super) async fn memory_stats(&self) -> Result { - let html_body = conduit::alloc::memory_stats(); + let html_body = conduwuit::alloc::memory_stats(); if html_body.is_none() { return Ok(RoomMessageEventContent::text_plain( diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index b74e9c36..c87dbb0a 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -2,7 +2,7 @@ mod commands; pub(crate) mod tester; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use ruma::{EventId, OwnedRoomOrAliasId, RoomId, ServerName}; use self::tester::TesterCommand; diff --git a/src/admin/debug/tester.rs b/src/admin/debug/tester.rs index c11f893e..5f922ece 100644 --- a/src/admin/debug/tester.rs +++ b/src/admin/debug/tester.rs @@ -1,4 +1,4 @@ -use conduit::Err; +use conduwuit::Err; use ruma::events::room::message::RoomMessageEventContent; use crate::{admin_command, admin_command_dispatch, Result}; diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 0c9df433..e0c1598d 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -1,6 +1,6 @@ use std::fmt::Write; -use conduit::Result; +use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId, ServerName, UserId}; diff --git a/src/admin/federation/mod.rs b/src/admin/federation/mod.rs index 8f5d3fae..3adfd459 100644 --- a/src/admin/federation/mod.rs +++ b/src/admin/federation/mod.rs @@ -1,7 +1,7 @@ mod commands; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use ruma::{RoomId, ServerName, UserId}; use crate::admin_command_dispatch; diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 82ac162e..977ecdae 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -1,7 +1,7 @@ use std::time::Duration; -use conduit::{debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, Result}; -use conduit_service::media::Dim; +use conduwuit::{debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, Result}; +use conduwuit_service::media::Dim; use ruma::{ events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, OwnedMxcUri, OwnedServerName, ServerName, }; diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index fbf6532b..046be556 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -1,7 +1,7 @@ mod commands; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use ruma::{EventId, MxcUri, OwnedMxcUri, OwnedServerName, ServerName}; use crate::admin_command_dispatch; diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 83db18fa..ac51104a 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -18,12 +18,12 @@ pub(crate) mod room; pub(crate) mod server; pub(crate) mod user; -extern crate conduit_api as api; -extern crate conduit_core as conduit; -extern crate conduit_service as service; +extern crate conduwuit_api as api; +extern crate conduwuit_core as conduwuit; +extern crate conduwuit_service as service; -pub(crate) use conduit::Result; -pub(crate) use conduit_macros::{admin_command, admin_command_dispatch}; +pub(crate) use conduwuit::Result; +pub(crate) use conduwuit_macros::{admin_command, admin_command_dispatch}; pub(crate) use crate::{ command::Command, @@ -32,9 +32,9 @@ pub(crate) use crate::{ pub(crate) const PAGE_SIZE: usize = 100; -conduit::mod_ctor! {} -conduit::mod_dtor! {} -conduit::rustc_flags_capture! {} +conduwuit::mod_ctor! {} +conduwuit::mod_dtor! {} +conduwuit::rustc_flags_capture! {} /// Install the admin command processor pub async fn init(admin_service: &service::admin::Service) { diff --git a/src/admin/processor.rs b/src/admin/processor.rs index caaa7d2d..e041a889 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -6,7 +6,7 @@ use std::{ }; use clap::{CommandFactory, Parser}; -use conduit::{ +use conduwuit::{ debug, error, log::{ capture, diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index 91217334..53ed64dd 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId, UserId}; diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 02e89e7a..6bf9b9ad 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use ruma::events::room::message::RoomMessageEventContent; use crate::Command; diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 837d34e6..68f68648 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use ruma::{events::room::message::RoomMessageEventContent, ServerName}; use crate::Command; diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index 1f0f5505..b1849091 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -10,7 +10,7 @@ mod sending; mod users; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use self::{ account_data::AccountDataCommand, appservice::AppserviceCommand, globals::GlobalsCommand, diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index 0963429e..73e33bf6 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, UserId}; diff --git a/src/admin/query/pusher.rs b/src/admin/query/pusher.rs index a1bd32f9..3004af8b 100644 --- a/src/admin/query/pusher.rs +++ b/src/admin/query/pusher.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use ruma::{events::room::message::RoomMessageEventContent, UserId}; use crate::Command; diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index e8340dad..43c63770 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -1,7 +1,7 @@ use std::fmt::Write; use clap::Subcommand; -use conduit::{utils::time, Result}; +use conduwuit::{utils::time, Result}; use ruma::{events::room::message::RoomMessageEventContent, OwnedServerName}; use crate::{admin_command, admin_command_dispatch}; diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index 382e4a78..19a727e9 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, RoomId}; diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index e32517fb..6e84507a 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId}; diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index eaab1f5e..efb4275f 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, ServerName, UserId}; use service::sending::Destination; diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index 0792e484..d58e2d2a 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use futures::stream::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, OwnedDeviceId, OwnedRoomId, OwnedUserId}; diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 1ccde47d..841bc51a 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -1,7 +1,7 @@ use std::fmt::Write; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId}; diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index 35e40c8b..d2fdbfca 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -1,4 +1,4 @@ -use conduit::Result; +use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId}; diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index 0bdaf56d..d3ed2575 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId}; diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index 13a74a9d..6a6ec695 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduit::{utils::ReadyExt, Result}; +use conduwuit::{utils::ReadyExt, Result}; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId}; diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index 8c6cbeaa..ad05d16d 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -5,7 +5,7 @@ mod info; mod moderation; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use ruma::OwnedRoomId; use self::{ diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index cfc048bd..427f3e42 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -1,6 +1,6 @@ use api::client::leave_room; use clap::Subcommand; -use conduit::{ +use conduwuit::{ debug, error, info, utils::{IterStream, ReadyExt}, warn, Result, diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 94f695ce..676f2d33 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,6 +1,6 @@ use std::{fmt::Write, sync::Arc}; -use conduit::{info, utils::time, warn, Err, Result}; +use conduwuit::{info, utils::time, warn, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; use crate::admin_command; @@ -73,7 +73,7 @@ pub(super) async fn list_features( pub(super) async fn memory_usage(&self) -> Result { let services_usage = self.services.memory_usage().await?; let database_usage = self.services.db.db.memory_usage()?; - let allocator_usage = conduit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}")); + let allocator_usage = conduwuit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}")); Ok(RoomMessageEventContent::text_plain(format!( "Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}", @@ -143,7 +143,7 @@ pub(super) async fn reload_mods(&self) -> Result { #[admin_command] #[cfg(unix)] pub(super) async fn restart(&self, force: bool) -> Result { - use conduit::utils::sys::current_exe_deleted; + use conduwuit::utils::sys::current_exe_deleted; if !force && current_exe_deleted() { return Err!( diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 222c537a..69ad7953 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -1,7 +1,7 @@ mod commands; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use crate::admin_command_dispatch; diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 61096625..017c6239 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -1,12 +1,12 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room}; -use conduit::{ +use conduwuit::{ debug_warn, error, info, is_equal_to, utils::{self, ReadyExt}, warn, PduBuilder, Result, }; -use conduit_api::client::{leave_all_rooms, update_avatar_url, update_displayname}; +use conduwuit_api::client::{leave_all_rooms, update_avatar_url, update_displayname}; use futures::StreamExt; use ruma::{ events::{ diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 649cdfb8..60db6de8 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -1,7 +1,7 @@ mod commands; use clap::Subcommand; -use conduit::Result; +use conduwuit::Result; use ruma::{EventId, OwnedRoomOrAliasId, RoomId}; use crate::admin_command_dispatch; diff --git a/src/admin/utils.rs b/src/admin/utils.rs index ba98bbea..8c67a0ca 100644 --- a/src/admin/utils.rs +++ b/src/admin/utils.rs @@ -1,4 +1,4 @@ -use conduit_core::{err, Err, Result}; +use conduwuit_core::{err, Err, Result}; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; use service::Services; diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index a0fc09de..f3a84664 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "conduit_api" +name = "conduwuit_api" categories.workspace = true description.workspace = true edition.workspace = true @@ -41,9 +41,9 @@ axum-extra.workspace = true axum.workspace = true base64.workspace = true bytes.workspace = true -conduit-core.workspace = true -conduit-database.workspace = true -conduit-service.workspace = true +conduwuit-core.workspace = true +conduwuit-database.workspace = true +conduwuit-service.workspace = true const-str.workspace = true futures.workspace = true hmac.workspace = true diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 3595f581..71e18fd3 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -2,7 +2,7 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{debug_info, error, info, is_equal_to, utils, utils::ReadyExt, warn, Error, PduBuilder, Result}; +use conduwuit::{debug_info, error, info, is_equal_to, utils, utils::ReadyExt, warn, Error, PduBuilder, Result}; use futures::{FutureExt, StreamExt}; use register::RegistrationKind; use ruma::{ diff --git a/src/api/client/account_data.rs b/src/api/client/account_data.rs index 86ef41ef..2fc78808 100644 --- a/src/api/client/account_data.rs +++ b/src/api/client/account_data.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{err, Err}; +use conduwuit::{err, Err}; use ruma::{ api::client::config::{ get_global_account_data, get_room_account_data, set_global_account_data, set_room_account_data, diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 83f3291d..c60a2f4c 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{debug, Err, Result}; +use conduwuit::{debug, Err, Result}; use futures::StreamExt; use rand::seq::SliceRandom; use ruma::{ diff --git a/src/api/client/appservice.rs b/src/api/client/appservice.rs index d299185e..9dbd141e 100644 --- a/src/api/client/appservice.rs +++ b/src/api/client/appservice.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{err, Err, Result}; +use conduwuit::{err, Err, Result}; use ruma::api::{appservice::ping, client::appservice::request_ping}; use crate::Ruma; diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index f435e086..9e4746ca 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{err, Err}; +use conduwuit::{err, Err}; use ruma::{ api::client::backup::{ add_backup_keys, add_backup_keys_for_room, add_backup_keys_for_session, create_backup_version, diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 45c04eb6..52f27692 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,7 +1,7 @@ use std::iter::once; use axum::extract::State; -use conduit::{ +use conduwuit::{ at, err, ref_at, utils::{ future::TryExtExt, diff --git a/src/api/client/device.rs b/src/api/client/device.rs index 7e56f27e..63f0c210 100644 --- a/src/api/client/device.rs +++ b/src/api/client/device.rs @@ -1,6 +1,6 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{err, Err}; +use conduwuit::{err, Err}; use futures::StreamExt; use ruma::{ api::client::{ diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 6120c7b3..9407a0bd 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,6 +1,6 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{info, warn, Err, Error, Result}; +use conduwuit::{info, warn, Err, Error, Result}; use futures::{StreamExt, TryFutureExt}; use ruma::{ api::{ diff --git a/src/api/client/filter.rs b/src/api/client/filter.rs index 2a8ebb9c..a1576e8e 100644 --- a/src/api/client/filter.rs +++ b/src/api/client/filter.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::err; +use conduwuit::err; use ruma::api::client::filter::{create_filter, get_filter}; use crate::{Result, Ruma}; diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 18f7d21c..13dc3467 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use axum::extract::State; -use conduit::{err, utils, Error, Result}; +use conduwuit::{err, utils, Error, Result}; use futures::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::{ diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 67a2bbdf..a9792c3b 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -2,12 +2,12 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{ +use conduwuit::{ err, utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize}, Err, Result, }; -use conduit_service::{ +use conduwuit_service::{ media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN, MXC_LENGTH}, Services, }; diff --git a/src/api/client/media_legacy.rs b/src/api/client/media_legacy.rs index f6837462..a8b366ed 100644 --- a/src/api/client/media_legacy.rs +++ b/src/api/client/media_legacy.rs @@ -2,12 +2,12 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{ +use conduwuit::{ err, utils::{content_disposition::make_content_disposition, math::ruma_from_usize}, Err, Result, }; -use conduit_service::media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN}; +use conduwuit_service::media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN}; use reqwest::Url; use ruma::{ api::client::media::{ diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index a61011b0..02166271 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -6,7 +6,7 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{ +use conduwuit::{ debug, debug_info, debug_warn, err, error, info, pdu::{self, gen_event_id_canonical_json, PduBuilder}, result::FlatOk, @@ -1457,7 +1457,7 @@ pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { } pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, reason: Option) -> Result<()> { - //use conduit::utils::stream::OptionStream; + //use conduwuit::utils::stream::OptionStream; use futures::TryFutureExt; // Ask a remote server if we don't have this room diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 242c1681..32e0b1e3 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,7 +1,7 @@ use std::collections::HashSet; use axum::extract::State; -use conduit::{ +use conduwuit::{ at, is_equal_to, utils::{ result::{FlatOk, LogErr}, diff --git a/src/api/client/openid.rs b/src/api/client/openid.rs index 3e4c6ca8..b8bbfb91 100644 --- a/src/api/client/openid.rs +++ b/src/api/client/openid.rs @@ -1,7 +1,7 @@ use std::time::Duration; use axum::extract::State; -use conduit::utils; +use conduwuit::utils; use ruma::{ api::client::{account, error::ErrorKind}, authentication::TokenType, diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 97ea21ea..ea74cdaf 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{ +use conduwuit::{ pdu::PduBuilder, utils::{stream::TryIgnore, IterStream}, warn, Err, Error, Result, diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 7efa59e8..e290c952 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{err, Err}; +use conduwuit::{err, Err}; use ruma::{ api::client::{ error::ErrorKind, diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index f6123614..d7f24101 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{err, Err, PduCount}; +use conduwuit::{err, Err, PduCount}; use ruma::{ api::client::{read_marker::set_read_marker, receipt::create_receipt}, events::{ diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index de54c4e4..c47bc60f 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{ +use conduwuit::{ at, utils::{result::FlatOk, stream::WidebandExt, IterStream, ReadyExt}, PduCount, Result, diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 31667323..429cd033 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -2,7 +2,7 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{info, utils::ReadyExt, Err}; +use conduwuit::{info, utils::ReadyExt, Err}; use rand::Rng; use ruma::{ api::client::{ diff --git a/src/api/client/room/aliases.rs b/src/api/client/room/aliases.rs index e530b260..f500e9c7 100644 --- a/src/api/client/room/aliases.rs +++ b/src/api/client/room/aliases.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{Error, Result}; +use conduwuit::{Error, Result}; use futures::StreamExt; use ruma::api::client::{error::ErrorKind, room::aliases}; diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index 2e9852f8..b98f1dab 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{debug_info, debug_warn, error, info, pdu::PduBuilder, warn, Err, Error, Result}; +use conduwuit::{debug_info, debug_warn, error, info, pdu::PduBuilder, warn, Err, Error, Result}; use futures::FutureExt; use ruma::{ api::client::{ diff --git a/src/api/client/room/event.rs b/src/api/client/room/event.rs index 6deb567f..ff9524e6 100644 --- a/src/api/client/room/event.rs +++ b/src/api/client/room/event.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{err, Err, Event, Result}; +use conduwuit::{err, Err, Event, Result}; use futures::{try_join, FutureExt, TryFutureExt}; use ruma::api::client::room::get_room_event; diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index 16b3a53b..8b2e45df 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{at, utils::BoolExt, Err, Result}; +use conduwuit::{at, utils::BoolExt, Err, Result}; use futures::StreamExt; use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response}; diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index fafce2d1..401bf800 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -1,7 +1,7 @@ use std::cmp::max; use axum::extract::State; -use conduit::{err, info, pdu::PduBuilder, Error, Result}; +use conduwuit::{err, info, pdu::PduBuilder, Error, Result}; use futures::StreamExt; use ruma::{ api::client::{error::ErrorKind, room::upgrade_room}, diff --git a/src/api/client/search.rs b/src/api/client/search.rs index 38468abb..28a8891c 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{ +use conduwuit::{ at, is_true, result::FlatOk, utils::{stream::ReadyExt, IterStream}, diff --git a/src/api/client/send.rs b/src/api/client/send.rs index ff011efa..e909ebbf 100644 --- a/src/api/client/send.rs +++ b/src/api/client/send.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{err, Err}; +use conduwuit::{err, Err}; use ruma::{api::client::message::send_message_event, events::MessageLikeEventType}; use serde_json::from_str; diff --git a/src/api/client/session.rs b/src/api/client/session.rs index e889a867..e30c94d7 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -1,6 +1,6 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::{debug, err, info, utils::ReadyExt, warn, Err}; +use conduwuit::{debug, err, info, utils::ReadyExt, warn, Err}; use futures::StreamExt; use ruma::{ api::client::{ diff --git a/src/api/client/state.rs b/src/api/client/state.rs index aeabd606..881eca98 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use axum::extract::State; -use conduit::{err, pdu::PduBuilder, utils::BoolExt, Err, Error, PduEvent, Result}; +use conduwuit::{err, pdu::PduBuilder, utils::BoolExt, Err, Error, PduEvent, Result}; use ruma::{ api::client::{ error::ErrorKind, diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 6f791860..2b8d478c 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -1,7 +1,7 @@ mod v3; mod v4; -use conduit::{ +use conduwuit::{ utils::stream::{BroadbandExt, ReadyExt}, PduCount, }; diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 815ec87c..c5ec3886 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -5,7 +5,7 @@ use std::{ }; use axum::extract::State; -use conduit::{ +use conduwuit::{ at, err, error, extract_variant, is_equal_to, is_false, pdu::EventHash, result::{FlatOk, LogDebugErr}, @@ -18,7 +18,7 @@ use conduit::{ }, Error, PduCount, PduEvent, Result, }; -use conduit_service::{ +use conduwuit_service::{ rooms::short::{ShortStateHash, ShortStateKey}, Services, }; diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index fddb81bf..140b647d 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -5,7 +5,7 @@ use std::{ }; use axum::extract::State; -use conduit::{ +use conduwuit::{ debug, error, extract_variant, utils::{ math::{ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index 906f779d..07badaf7 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::{at, PduCount, PduEvent}; +use conduwuit::{at, PduCount, PduEvent}; use futures::StreamExt; use ruma::{api::client::threads::get_threads, uint}; diff --git a/src/api/client/to_device.rs b/src/api/client/to_device.rs index 2b37a9ec..aab59394 100644 --- a/src/api/client/to_device.rs +++ b/src/api/client/to_device.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduit::{Error, Result}; +use conduwuit::{Error, Result}; use futures::StreamExt; use ruma::{ api::{ diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 3660d674..c546d6a7 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduit::Err; +use conduwuit::Err; use futures::StreamExt; use ruma::{ api::{ diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index ed3ce37a..91fe5a7c 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -64,8 +64,8 @@ pub(crate) async fn get_supported_versions_route( /// `/_matrix/federation/v1/version` pub(crate) async fn conduwuit_server_version() -> Result { Ok(Json(serde_json::json!({ - "name": conduit::version::name(), - "version": conduit::version::version(), + "name": conduwuit::version::name(), + "version": conduwuit::version::version(), }))) } diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index feb48346..bc7460f0 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduit::utils::TryFutureExtExt; +use conduwuit::utils::TryFutureExtExt; use futures::{pin_mut, StreamExt}; use ruma::{ api::client::user_directory::search_users, diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index 7ee4571f..b41cc8a1 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -2,7 +2,7 @@ use std::time::{Duration, SystemTime}; use axum::extract::State; use base64::{engine::general_purpose, Engine as _}; -use conduit::{utils, Err}; +use conduwuit::{utils, Err}; use hmac::{Hmac, Mac}; use ruma::{api::client::voip::get_turn_server_info, SecondsSinceUnixEpoch, UserId}; use sha1::Sha1; diff --git a/src/api/client/well_known.rs b/src/api/client/well_known.rs index 674c9bb0..b66217e8 100644 --- a/src/api/client/well_known.rs +++ b/src/api/client/well_known.rs @@ -100,6 +100,6 @@ pub(crate) async fn syncv3_client_server_json(State(services): State Result { Ok(get_server_version::v1::Response { server: Some(get_server_version::v1::Server { - name: Some(conduit::version::name().into()), - version: Some(conduit::version::version().into()), + name: Some(conduwuit::version::name().into()), + version: Some(conduwuit::version::version().into()), }), }) } diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index b93f9a77..27c6da52 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "conduit_core" +name = "conduwuit_core" categories.workspace = true description.workspace = true edition.workspace = true @@ -62,7 +62,7 @@ cargo_toml.workspace = true checked_ops.workspace = true chrono.workspace = true clap.workspace = true -conduit-macros.workspace = true +conduwuit-macros.workspace = true const-str.workspace = true ctor.workspace = true cyborgtime.workspace = true diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 413aa7f4..a34514de 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -8,7 +8,7 @@ use std::{ path::PathBuf, }; -use conduit_macros::config_example_generator; +use conduwuit_macros::config_example_generator; use either::{ Either, Either::{Left, Right}, diff --git a/src/core/debug.rs b/src/core/debug.rs index f7420784..e1980234 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -3,7 +3,7 @@ use std::{any::Any, panic}; // Export debug proc_macros -pub use conduit_macros::recursion_depth; +pub use conduwuit_macros::recursion_depth; // Export all of the ancillary tools from here as well. pub use crate::{result::DebugInspect, utils::debug::*}; diff --git a/src/core/info/cargo.rs b/src/core/info/cargo.rs index 012a08e0..735ccee6 100644 --- a/src/core/info/cargo.rs +++ b/src/core/info/cargo.rs @@ -6,7 +6,7 @@ use std::sync::OnceLock; use cargo_toml::{DepsSet, Manifest}; -use conduit_macros::cargo_manifest; +use conduwuit_macros::cargo_manifest; use crate::Result; diff --git a/src/core/info/mod.rs b/src/core/info/mod.rs index 22ea7d3d..ca39b348 100644 --- a/src/core/info/mod.rs +++ b/src/core/info/mod.rs @@ -6,7 +6,7 @@ pub mod room_version; pub mod rustc; pub mod version; -pub use conduit_macros::rustc_flags_capture; +pub use conduwuit_macros::rustc_flags_capture; pub const MODULE_ROOT: &str = const_str::split!(std::module_path!(), "::")[0]; pub const CRATE_PREFIX: &str = const_str::split!(MODULE_ROOT, '_')[0]; diff --git a/src/core/mod.rs b/src/core/mod.rs index 4ab84730..87cb58ae 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -21,11 +21,11 @@ pub use pdu::{Event, PduBuilder, PduCount, PduEvent, PduId, RawPduId}; pub use server::Server; pub use utils::{ctor, dtor, implement, result, result::Result}; -pub use crate as conduit_core; +pub use crate as conduwuit_core; rustc_flags_capture! {} -#[cfg(not(conduit_mods))] +#[cfg(not(conduwuit_mods))] pub mod mods { #[macro_export] macro_rules! mod_ctor { diff --git a/src/core/mods/mod.rs b/src/core/mods/mod.rs index 118bfc29..ac0c333b 100644 --- a/src/core/mods/mod.rs +++ b/src/core/mods/mod.rs @@ -1,4 +1,4 @@ -#![cfg(conduit_mods)] +#![cfg(conduwuit_mods)] pub(crate) use libloading::os::unix::{Library, Symbol}; diff --git a/src/core/server.rs b/src/core/server.rs index 627e125d..1e721517 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -56,7 +56,7 @@ impl Server { } pub fn reload(&self) -> Result<()> { - if cfg!(not(conduit_mods)) { + if cfg!(not(conduwuit_mods)) { return Err!("Reloading not enabled"); } diff --git a/src/core/utils/debug.rs b/src/core/utils/debug.rs index e4151f39..cb9d7bab 100644 --- a/src/core/utils/debug.rs +++ b/src/core/utils/debug.rs @@ -25,7 +25,7 @@ impl fmt::Debug for TruncatedSlice<'_, T> { /// See [`TruncatedSlice`]. Useful for `#[instrument]`: /// /// ``` -/// use conduit_core::utils::debug::slice_truncated; +/// use conduwuit_core::utils::debug::slice_truncated; /// /// #[tracing::instrument(fields(foos = slice_truncated(foos, 42)))] /// fn bar(foos: &[&str]); diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index f9310243..ad53bc42 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -19,7 +19,7 @@ pub mod sys; mod tests; pub mod time; -pub use ::conduit_macros::implement; +pub use ::conduwuit_macros::implement; pub use ::ctor::{ctor, dtor}; pub use self::{ diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index e65a3369..6baa9c35 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -84,7 +84,7 @@ where /// Find the common prefix from a collection of strings and return a slice /// ``` -/// use conduit_core::utils::string::common_prefix; +/// use conduwiit_core::utils::string::common_prefix; /// let input = ["conduwuit", "conduit", "construct"]; /// common_prefix(&input) == "con"; /// ``` diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 1deaf980..efd18a1a 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "conduit_database" +name = "conduwuit_database" categories.workspace = true description.workspace = true edition.workspace = true @@ -37,7 +37,7 @@ zstd_compression = [ [dependencies] arrayvec.workspace = true async-channel.workspace = true -conduit-core.workspace = true +conduwuit-core.workspace = true const-str.workspace = true futures.workspace = true log.workspace = true diff --git a/src/database/database.rs b/src/database/database.rs index 3df95dce..bd8dcb1a 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -1,6 +1,6 @@ use std::{ops::Index, sync::Arc}; -use conduit::{err, Result, Server}; +use conduwuit::{err, Result, Server}; use crate::{ maps, diff --git a/src/database/de.rs b/src/database/de.rs index d303eab2..ac8c021f 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -1,5 +1,5 @@ use arrayvec::ArrayVec; -use conduit::{checked, debug::DebugInspect, err, utils::string, Error, Result}; +use conduwuit::{checked, debug::DebugInspect, err, utils::string, Error, Result}; use serde::{ de, de::{DeserializeSeed, Visitor}, @@ -331,7 +331,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { fn deserialize_any>(self, visitor: V) -> Result { debug_assert_eq!( - conduit::debug::type_name::(), + conduwuit::debug::type_name::(), "serde_json::value::de::::deserialize::ValueVisitor", "deserialize_any: type not expected" diff --git a/src/database/deserialized.rs b/src/database/deserialized.rs index a59b2ce5..66541b2a 100644 --- a/src/database/deserialized.rs +++ b/src/database/deserialized.rs @@ -1,6 +1,6 @@ use std::convert::identity; -use conduit::Result; +use conduwuit::Result; use serde::Deserialize; pub trait Deserialized { diff --git a/src/database/engine.rs b/src/database/engine.rs index d3bb727e..72fe11e6 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -6,7 +6,7 @@ use std::{ sync::{atomic::AtomicU32, Arc, Mutex, RwLock}, }; -use conduit::{debug, error, info, utils::time::rfc2822_from_seconds, warn, Err, Result, Server}; +use conduwuit::{debug, error, info, utils::time::rfc2822_from_seconds, warn, Err, Result, Server}; use rocksdb::{ backup::{BackupEngine, BackupEngineOptions}, perf::get_memory_usage_stats, diff --git a/src/database/handle.rs b/src/database/handle.rs index 356bd859..b4d34b85 100644 --- a/src/database/handle.rs +++ b/src/database/handle.rs @@ -1,6 +1,6 @@ use std::{fmt, fmt::Debug, ops::Deref}; -use conduit::Result; +use conduwuit::Result; use rocksdb::DBPinnableSlice; use serde::{Deserialize, Serialize, Serializer}; diff --git a/src/database/keyval.rs b/src/database/keyval.rs index d4568600..056e53d1 100644 --- a/src/database/keyval.rs +++ b/src/database/keyval.rs @@ -1,4 +1,4 @@ -use conduit::Result; +use conduwuit::Result; use serde::{Deserialize, Serialize}; use smallvec::SmallVec; diff --git a/src/database/map.rs b/src/database/map.rs index 0f4d740a..09721b2a 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -27,7 +27,7 @@ use std::{ sync::Arc, }; -use conduit::Result; +use conduwuit::Result; use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, ReadTier, WriteOptions}; use crate::{watchers::Watchers, Engine}; diff --git a/src/database/map/contains.rs b/src/database/map/contains.rs index 7016b744..aa3ea676 100644 --- a/src/database/map/contains.rs +++ b/src/database/map/contains.rs @@ -1,7 +1,7 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write, sync::Arc}; use arrayvec::ArrayVec; -use conduit::{ +use conduwuit::{ err, implement, utils::{future::TryExtExt, result::FlatOk}, Result, diff --git a/src/database/map/count.rs b/src/database/map/count.rs index 894fe12e..7b632bb3 100644 --- a/src/database/map/count.rs +++ b/src/database/map/count.rs @@ -1,6 +1,6 @@ use std::{fmt::Debug, future::Future, sync::Arc}; -use conduit::implement; +use conduwuit::implement; use futures::stream::StreamExt; use serde::Serialize; diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 79556656..88023b1c 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -1,7 +1,7 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; use arrayvec::ArrayVec; -use conduit::{err, implement, utils::result::MapExpect, Err, Result}; +use conduwuit::{err, implement, utils::result::MapExpect, Err, Result}; use futures::{Future, FutureExt}; use serde::Serialize; use tokio::task; diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index bb904943..5f444cce 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduit::{err, implement, utils::IterStream, Result}; +use conduwuit::{err, implement, utils::IterStream, Result}; use futures::{Stream, StreamExt}; use serde::Serialize; diff --git a/src/database/map/insert.rs b/src/database/map/insert.rs index 5b2e816c..68c305af 100644 --- a/src/database/map/insert.rs +++ b/src/database/map/insert.rs @@ -6,7 +6,7 @@ use std::{convert::AsRef, fmt::Debug, io::Write}; use arrayvec::ArrayVec; -use conduit::implement; +use conduwuit::implement; use rocksdb::WriteBatchWithTransaction; use serde::Serialize; diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs index 3c4d491b..3ab5bacc 100644 --- a/src/database/map/keys.rs +++ b/src/database/map/keys.rs @@ -1,4 +1,4 @@ -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{Stream, StreamExt}; use serde::Deserialize; diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 40c608f2..2f6d2020 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index 9122d78e..24112ac1 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{ future, stream::{Stream, StreamExt}, diff --git a/src/database/map/remove.rs b/src/database/map/remove.rs index 18080c64..ec37bbfe 100644 --- a/src/database/map/remove.rs +++ b/src/database/map/remove.rs @@ -1,7 +1,7 @@ use std::{convert::AsRef, fmt::Debug, io::Write}; use arrayvec::ArrayVec; -use conduit::implement; +use conduwuit::implement; use serde::Serialize; use crate::{keyval::KeyBuf, ser, util::or_else}; diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index 1e000a47..7eb4ce63 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -1,4 +1,4 @@ -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{Stream, StreamExt}; use serde::Deserialize; diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index b5565aa4..c895105c 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index 69dc54f2..a5ed35ec 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{ future, stream::{Stream, StreamExt}, diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index 5f61cb08..81359800 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -1,4 +1,4 @@ -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::stream::{Stream, StreamExt}; use serde::Deserialize; diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 542a5ba8..2d6a76b7 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{ stream::{Stream, StreamExt}, FutureExt, TryFutureExt, TryStreamExt, diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index e5c2fbea..13aa40f2 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{ future, stream::{Stream, StreamExt}, diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index 67cfff1b..c2d9b6b8 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -1,4 +1,4 @@ -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::stream::{Stream, StreamExt}; use serde::Deserialize; diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 6468846f..c6fe439a 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{ stream::{Stream, StreamExt}, FutureExt, TryFutureExt, TryStreamExt, diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index 3c7bce2e..e7dad211 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -1,6 +1,6 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use futures::{ future, stream::{Stream, StreamExt}, diff --git a/src/database/maps.rs b/src/database/maps.rs index 9b8d326a..d69cc7fd 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduit::Result; +use conduwuit::Result; use crate::{Engine, Map}; diff --git a/src/database/mod.rs b/src/database/mod.rs index de060b3a..183cba8d 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -20,7 +20,7 @@ pub(crate) use self::{ util::{or_else, result}, }; -extern crate conduit_core as conduit; +extern crate conduwuit_core as conduwuit; extern crate rust_rocksdb as rocksdb; pub use self::{ @@ -33,6 +33,6 @@ pub use self::{ ser::{serialize, serialize_to, serialize_to_vec, Interfix, Json, Separator, SEP}, }; -conduit::mod_ctor! {} -conduit::mod_dtor! {} -conduit::rustc_flags_capture! {} +conduwuit::mod_ctor! {} +conduwuit::mod_dtor! {} +conduwuit::rustc_flags_capture! {} diff --git a/src/database/opts.rs b/src/database/opts.rs index b0b33927..52ed4818 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -1,6 +1,6 @@ use std::{cmp, collections::HashMap, convert::TryFrom}; -use conduit::{err, utils, Config, Result}; +use conduwuit::{err, utils, Config, Result}; use rocksdb::{ statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, LogLevel, LruCacheOptions, Options, UniversalCompactOptions, UniversalCompactionStopStyle, diff --git a/src/database/pool.rs b/src/database/pool.rs index 79755ea6..3301b821 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -7,7 +7,7 @@ use std::{ }; use async_channel::{bounded, Receiver, RecvError, Sender}; -use conduit::{debug, debug_warn, defer, err, implement, result::DebugInspect, Result, Server}; +use conduwuit::{debug, debug_warn, defer, err, implement, result::DebugInspect, Result, Server}; use futures::{channel::oneshot, TryFutureExt}; use oneshot::Sender as ResultSender; use rocksdb::Direction; diff --git a/src/database/ser.rs b/src/database/ser.rs index a60812aa..8abe5521 100644 --- a/src/database/ser.rs +++ b/src/database/ser.rs @@ -1,6 +1,6 @@ use std::io::Write; -use conduit::{debug::type_name, err, result::DebugInspect, utils::exchange, Error, Result}; +use conduwuit::{debug::type_name, err, result::DebugInspect, utils::exchange, Error, Result}; use serde::{ser, Serialize}; use crate::util::unhandled; diff --git a/src/database/stream.rs b/src/database/stream.rs index 38c46596..775fb930 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -5,7 +5,7 @@ mod keys_rev; use std::sync::Arc; -use conduit::{utils::exchange, Error, Result}; +use conduwuit::{utils::exchange, Error, Result}; use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode, ReadOptions}; pub(crate) use self::{items::Items, items_rev::ItemsRev, keys::Keys, keys_rev::KeysRev}; diff --git a/src/database/stream/items.rs b/src/database/stream/items.rs index 77b08a0b..06cb6de9 100644 --- a/src/database/stream/items.rs +++ b/src/database/stream/items.rs @@ -1,6 +1,6 @@ use std::{convert, pin::Pin, sync::Arc}; -use conduit::Result; +use conduwuit::Result; use futures::{ stream::FusedStream, task::{Context, Poll}, diff --git a/src/database/stream/items_rev.rs b/src/database/stream/items_rev.rs index dfd3a107..2d0c4639 100644 --- a/src/database/stream/items_rev.rs +++ b/src/database/stream/items_rev.rs @@ -1,6 +1,6 @@ use std::{convert, pin::Pin, sync::Arc}; -use conduit::Result; +use conduwuit::Result; use futures::{ stream::FusedStream, task::{Context, Poll}, diff --git a/src/database/stream/keys.rs b/src/database/stream/keys.rs index 2ce88959..a901b342 100644 --- a/src/database/stream/keys.rs +++ b/src/database/stream/keys.rs @@ -1,6 +1,6 @@ use std::{convert, pin::Pin, sync::Arc}; -use conduit::Result; +use conduwuit::Result; use futures::{ stream::FusedStream, task::{Context, Poll}, diff --git a/src/database/stream/keys_rev.rs b/src/database/stream/keys_rev.rs index 12dae759..73758a85 100644 --- a/src/database/stream/keys_rev.rs +++ b/src/database/stream/keys_rev.rs @@ -1,6 +1,6 @@ use std::{convert, pin::Pin, sync::Arc}; -use conduit::Result; +use conduwuit::Result; use futures::{ stream::FusedStream, task::{Context, Poll}, diff --git a/src/database/tests.rs b/src/database/tests.rs index bfab99ef..7f486966 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -4,7 +4,7 @@ use std::fmt::Debug; use arrayvec::ArrayVec; -use conduit::ruma::{serde::Raw, RoomId, UserId}; +use conduwuit::ruma::{serde::Raw, RoomId, UserId}; use serde::Serialize; use crate::{ @@ -53,7 +53,7 @@ fn ser_overflow() { #[test] fn ser_complex() { - use conduit::ruma::Mxc; + use conduwuit::ruma::Mxc; #[derive(Debug, Serialize)] struct Dim { @@ -90,7 +90,7 @@ fn ser_complex() { #[test] fn ser_json() { - use conduit::ruma::api::client::filter::FilterDefinition; + use conduwuit::ruma::api::client::filter::FilterDefinition; let filter = FilterDefinition { event_fields: Some(vec!["content.body".to_owned()]), @@ -105,7 +105,7 @@ fn ser_json() { #[test] fn ser_json_value() { - use conduit::ruma::api::client::filter::FilterDefinition; + use conduwuit::ruma::api::client::filter::FilterDefinition; let filter = FilterDefinition { event_fields: Some(vec!["content.body".to_owned()]), @@ -146,7 +146,7 @@ fn ser_json_macro() { #[test] #[should_panic(expected = "serializing string at the top-level")] fn ser_json_raw() { - use conduit::ruma::api::client::filter::FilterDefinition; + use conduwuit::ruma::api::client::filter::FilterDefinition; let filter = FilterDefinition { event_fields: Some(vec!["content.body".to_owned()]), @@ -162,7 +162,7 @@ fn ser_json_raw() { #[test] #[should_panic(expected = "you can skip serialization instead")] fn ser_json_raw_json() { - use conduit::ruma::api::client::filter::FilterDefinition; + use conduwuit::ruma::api::client::filter::FilterDefinition; let filter = FilterDefinition { event_fields: Some(vec!["content.body".to_owned()]), diff --git a/src/database/util.rs b/src/database/util.rs index 21764361..0fca3b68 100644 --- a/src/database/util.rs +++ b/src/database/util.rs @@ -1,4 +1,4 @@ -use conduit::{err, Result}; +use conduwuit::{err, Result}; use rocksdb::{Direction, ErrorKind, IteratorMode}; //#[cfg(debug_assertions)] @@ -36,19 +36,19 @@ pub(crate) fn _into_direction(mode: &IteratorMode<'_>) -> Direction { } #[inline] -pub(crate) fn result(r: std::result::Result) -> Result { +pub(crate) fn result(r: std::result::Result) -> Result { r.map_or_else(or_else, and_then) } #[inline(always)] -pub(crate) fn and_then(t: T) -> Result { Ok(t) } +pub(crate) fn and_then(t: T) -> Result { Ok(t) } -pub(crate) fn or_else(e: rocksdb::Error) -> Result { Err(map_err(e)) } +pub(crate) fn or_else(e: rocksdb::Error) -> Result { Err(map_err(e)) } #[inline] pub(crate) fn is_incomplete(e: &rocksdb::Error) -> bool { e.kind() == ErrorKind::Incomplete } -pub(crate) fn map_err(e: rocksdb::Error) -> conduit::Error { +pub(crate) fn map_err(e: rocksdb::Error) -> conduwuit::Error { let string = e.into_string(); err!(Database(error!("{string}"))) } diff --git a/src/macros/Cargo.toml b/src/macros/Cargo.toml index 9e866578..167de8c0 100644 --- a/src/macros/Cargo.toml +++ b/src/macros/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "conduit_macros" +name = "conduwuit_macros" categories.workspace = true description.workspace = true edition.workspace = true @@ -10,7 +10,7 @@ repository.workspace = true version.workspace = true [lib] -name = "conduit_macros" +name = "conduwuit_macros" path = "mod.rs" proc-macro = true diff --git a/src/macros/admin.rs b/src/macros/admin.rs index d4ce7ad5..b0dc1956 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -8,7 +8,7 @@ use crate::{utils::camel_to_snake_string, Result}; pub(super) fn command(mut item: ItemFn, _args: &[Meta]) -> Result { let attr: Attribute = parse_quote! { - #[conduit_macros::implement(crate::Command, params = "<'_>")] + #[conduwuit_macros::implement(crate::Command, params = "<'_>")] }; item.attrs.push(attr); diff --git a/src/macros/rustc.rs b/src/macros/rustc.rs index d286ed33..f484e5f5 100644 --- a/src/macros/rustc.rs +++ b/src/macros/rustc.rs @@ -5,7 +5,7 @@ pub(super) fn flags_capture(args: TokenStream) -> TokenStream { let cargo_crate_name = std::env::var("CARGO_CRATE_NAME"); let crate_name = match cargo_crate_name.as_ref() { Err(_) => return args, - Ok(crate_name) => crate_name.trim_start_matches("conduit_"), + Ok(crate_name) => crate_name.trim_start_matches("conduwuit_"), }; let flag = std::env::args().collect::>(); @@ -13,15 +13,15 @@ pub(super) fn flags_capture(args: TokenStream) -> TokenStream { let ret = quote! { pub static RUSTC_FLAGS: [&str; #flag_len] = [#( #flag ),*]; - #[conduit_core::ctor] + #[conduwuit_core::ctor] fn _set_rustc_flags() { - conduit_core::info::rustc::FLAGS.lock().expect("locked").insert(#crate_name, &RUSTC_FLAGS); + conduwuit_core::info::rustc::FLAGS.lock().expect("locked").insert(#crate_name, &RUSTC_FLAGS); } // static strings have to be yanked on module unload - #[conduit_core::dtor] + #[conduwuit_core::dtor] fn _unset_rustc_flags() { - conduit_core::info::rustc::FLAGS.lock().expect("locked").remove(#crate_name); + conduwuit_core::info::rustc::FLAGS.lock().expect("locked").remove(#crate_name); } }; diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index ee434ceb..99d41614 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -1,7 +1,6 @@ [package] -# TODO: when can we rename to conduwuit? -name = "conduit" -default-run = "conduit" +name = "conduwuit" +default-run = "conduwuit" authors.workspace = true categories.workspace = true description.workspace = true @@ -48,49 +47,49 @@ default = [ ] brotli_compression = [ - "conduit-api/brotli_compression", - "conduit-core/brotli_compression", - "conduit-router/brotli_compression", - "conduit-service/brotli_compression", + "conduwuit-api/brotli_compression", + "conduwuit-core/brotli_compression", + "conduwuit-router/brotli_compression", + "conduwuit-service/brotli_compression", ] console = [ - "conduit-service/console", + "conduwuit-service/console", ] #dev_release_log_level = [ -# "conduit-admin/dev_release_log_level", -# "conduit-api/dev_release_log_level", -# "conduit-core/dev_release_log_level", -# "conduit-database/dev_release_log_level", -# "conduit-router/dev_release_log_level", -# "conduit-service/dev_release_log_level", +# "conduwuit-admin/dev_release_log_level", +# "conduwuit-api/dev_release_log_level", +# "conduwuit-core/dev_release_log_level", +# "conduwuit-database/dev_release_log_level", +# "conduwuit-router/dev_release_log_level", +# "conduwuit-service/dev_release_log_level", #] direct_tls = [ - "conduit-router/direct_tls" + "conduwuit-router/direct_tls" ] element_hacks = [ - "conduit-api/element_hacks", - "conduit-service/element_hacks", + "conduwuit-api/element_hacks", + "conduwuit-service/element_hacks", ] gzip_compression = [ - "conduit-api/gzip_compression", - "conduit-router/gzip_compression", - "conduit-service/gzip_compression", + "conduwuit-api/gzip_compression", + "conduwuit-router/gzip_compression", + "conduwuit-service/gzip_compression", ] hardened_malloc = [ - "conduit-core/hardened_malloc", + "conduwuit-core/hardened_malloc", ] io_uring = [ - "conduit-database/io_uring", + "conduwuit-database/io_uring", ] jemalloc = [ - "conduit-core/jemalloc", - "conduit-database/jemalloc", + "conduwuit-core/jemalloc", + "conduwuit-database/jemalloc", ] jemalloc_prof = [ - "conduit-core/jemalloc_prof", + "conduwuit-core/jemalloc_prof", ] jemalloc_stats = [ - "conduit-core/jemalloc_stats", + "conduwuit-core/jemalloc_stats", ] perf_measurements = [ "dep:opentelemetry", @@ -98,8 +97,8 @@ perf_measurements = [ "dep:tracing-opentelemetry", "dep:opentelemetry_sdk", "dep:opentelemetry-jaeger", - "conduit-core/perf_measurements", - "conduit-core/sentry_telemetry", + "conduwuit-core/perf_measurements", + "conduwuit-core/sentry_telemetry", ] # increases performance, reduces build times, and reduces binary size by not compiling or # genreating code for log level filters that users will generally not use (debug and trace) @@ -108,22 +107,22 @@ release_max_log_level = [ "tracing/release_max_level_info", "log/max_level_trace", "log/release_max_level_info", - "conduit-admin/release_max_log_level", - "conduit-api/release_max_log_level", - "conduit-core/release_max_log_level", - "conduit-database/release_max_log_level", - "conduit-router/release_max_log_level", - "conduit-service/release_max_log_level", + "conduwuit-admin/release_max_log_level", + "conduwuit-api/release_max_log_level", + "conduwuit-core/release_max_log_level", + "conduwuit-database/release_max_log_level", + "conduwuit-router/release_max_log_level", + "conduwuit-service/release_max_log_level", ] sentry_telemetry = [ "dep:sentry", "dep:sentry-tracing", "dep:sentry-tower", - "conduit-core/sentry_telemetry", - "conduit-router/sentry_telemetry", + "conduwuit-core/sentry_telemetry", + "conduwuit-router/sentry_telemetry", ] systemd = [ - "conduit-router/systemd", + "conduwuit-router/systemd", ] # enable the tokio_console server ncompatible with release_max_log_level tokio_console = [ @@ -131,19 +130,19 @@ tokio_console = [ "tokio/tracing", ] zstd_compression = [ - "conduit-api/zstd_compression", - "conduit-core/zstd_compression", - "conduit-database/zstd_compression", - "conduit-router/zstd_compression", + "conduwuit-api/zstd_compression", + "conduwuit-core/zstd_compression", + "conduwuit-database/zstd_compression", + "conduwuit-router/zstd_compression", ] [dependencies] -conduit-admin.workspace = true -conduit-api.workspace = true -conduit-core.workspace = true -conduit-database.workspace = true -conduit-router.workspace = true -conduit-service.workspace = true +conduwuit-admin.workspace = true +conduwuit-api.workspace = true +conduwuit-core.workspace = true +conduwuit-database.workspace = true +conduwuit-router.workspace = true +conduwuit-service.workspace = true tokio.workspace = true log.workspace = true @@ -184,5 +183,5 @@ hardened_malloc-rs.optional = true workspace = true [[bin]] -name = "conduit" +name = "conduwuit" path = "main.rs" diff --git a/src/main/clap.rs b/src/main/clap.rs index b10242be..7e70bd80 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use clap::Parser; -use conduit::{ +use conduwuit::{ config::{Figment, FigmentValue}, err, toml, utils::available_parallelism, @@ -12,7 +12,7 @@ use conduit::{ /// Commandline arguments #[derive(Parser, Debug)] -#[clap(version = conduit::version(), about, long_about = None, name = "conduwuit")] +#[clap(version = conduwuit::version(), about, long_about = None, name = "conduwuit")] pub(crate) struct Args { #[arg(short, long)] /// Path to the config TOML file (optional) diff --git a/src/main/logging.rs b/src/main/logging.rs index efa47bab..999265e7 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{ +use conduwuit::{ config::Config, debug_warn, err, log::{capture, fmt_span, LogLevelReloadHandles}, diff --git a/src/main/main.rs b/src/main/main.rs index 41b21b58..5c066584 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -6,14 +6,14 @@ mod sentry; mod server; mod signal; -extern crate conduit_core as conduit; +extern crate conduwuit_core as conduwuit; use std::{ sync::{atomic::Ordering, Arc}, time::Duration, }; -use conduit::{debug_info, error, rustc_flags_capture, Error, Result}; +use conduwuit::{debug_info, error, rustc_flags_capture, Error, Result}; use server::Server; use tokio::runtime; @@ -58,14 +58,14 @@ fn main() -> Result<(), Error> { /// Operate the server normally in release-mode static builds. This will start, /// run and stop the server within the asynchronous runtime. -#[cfg(not(conduit_mods))] +#[cfg(not(conduwuit_mods))] #[tracing::instrument( name = "main", parent = None, skip_all )] async fn async_main(server: &Arc) -> Result<(), Error> { - extern crate conduit_router as router; + extern crate conduwuit_router as router; match router::start(&server.server).await { Ok(services) => server.services.lock().await.insert(services), @@ -110,7 +110,7 @@ async fn async_main(server: &Arc) -> Result<(), Error> { /// Operate the server in developer-mode dynamic builds. This will start, run, /// and hot-reload portions of the server as-needed before returning for an /// actual shutdown. This is not available in release-mode or static builds. -#[cfg(conduit_mods)] +#[cfg(conduwuit_mods)] async fn async_main(server: &Arc) -> Result<(), Error> { let mut starts = true; let mut reloads = true; diff --git a/src/main/mods.rs b/src/main/mods.rs index 16d975cc..ca984a64 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -1,7 +1,7 @@ -#![cfg(conduit_mods)] +#![cfg(conduwuit_mods)] #[unsafe(no_link)] -extern crate conduit_service; +extern crate conduwuit_service; use std::{ future::Future, @@ -9,13 +9,13 @@ use std::{ sync::{atomic::Ordering, Arc}, }; -use conduit::{debug, error, mods, Error, Result}; -use conduit_service::Services; +use conduwuit::{debug, error, mods, Error, Result}; +use conduwuit_service::Services; use crate::Server; type StartFuncResult = Pin>> + Send>>; -type StartFuncProto = fn(&Arc) -> StartFuncResult; +type StartFuncProto = fn(&Arc) -> StartFuncResult; type RunFuncResult = Pin> + Send>>; type RunFuncProto = fn(&Arc) -> RunFuncResult; @@ -23,19 +23,19 @@ type RunFuncProto = fn(&Arc) -> RunFuncResult; type StopFuncResult = Pin> + Send>>; type StopFuncProto = fn(Arc) -> StopFuncResult; -const RESTART_THRESH: &str = "conduit_service"; +const RESTART_THRESH: &str = "conduwuit_service"; const MODULE_NAMES: &[&str] = &[ - //"conduit_core", - "conduit_database", - "conduit_service", - "conduit_api", - "conduit_admin", - "conduit_router", + //"conduwuit_core", + "conduwuit_database", + "conduwuit_service", + "conduwuit_api", + "conduwuit_admin", + "conduwuit_router", ]; #[cfg(panic_trap)] -conduit::mod_init! {{ - conduit::debug::set_panic_trap(); +conduwuit::mod_init! {{ + conduwuit::debug::set_panic_trap(); }} pub(crate) async fn run(server: &Arc, starts: bool) -> Result<(bool, bool), Error> { diff --git a/src/main/restart.rs b/src/main/restart.rs index 009e5228..e6f45b82 100644 --- a/src/main/restart.rs +++ b/src/main/restart.rs @@ -2,7 +2,7 @@ use std::{env, os::unix::process::CommandExt, process::Command}; -use conduit::{debug, info, utils}; +use conduwuit::{debug, info, utils}; #[cold] pub(super) fn restart() -> ! { diff --git a/src/main/sentry.rs b/src/main/sentry.rs index 04ad8654..02835ec8 100644 --- a/src/main/sentry.rs +++ b/src/main/sentry.rs @@ -5,7 +5,7 @@ use std::{ sync::{Arc, OnceLock}, }; -use conduit::{config::Config, debug, trace}; +use conduwuit::{config::Config, debug, trace}; use sentry::{ types::{ protocol::v7::{Context, Event}, @@ -43,7 +43,7 @@ fn options(config: &Config) -> ClientOptions { traces_sample_rate: config.sentry_traces_sample_rate, debug: cfg!(debug_assertions), release: sentry::release_name!(), - user_agent: conduit::version::user_agent().into(), + user_agent: conduwuit::version::user_agent().into(), attach_stacktrace: config.sentry_attach_stacktrace, before_send: Some(Arc::new(before_send)), before_breadcrumb: Some(Arc::new(before_breadcrumb)), diff --git a/src/main/server.rs b/src/main/server.rs index 179749b5..7c3eec1f 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{config::Config, info, log::Log, utils::sys, Error, Result}; +use conduwuit::{config::Config, info, log::Log, utils::sys, Error, Result}; use tokio::{runtime, sync::Mutex}; use crate::{clap::Args, logging::TracingFlameGuard}; @@ -8,18 +8,18 @@ use crate::{clap::Args, logging::TracingFlameGuard}; /// Server runtime state; complete pub(crate) struct Server { /// Server runtime state; public portion - pub(crate) server: Arc, + pub(crate) server: Arc, - pub(crate) services: Mutex>>, + pub(crate) services: Mutex>>, _tracing_flame_guard: TracingFlameGuard, #[cfg(feature = "sentry_telemetry")] _sentry_guard: Option<::sentry::ClientInitGuard>, - #[cfg(conduit_mods)] + #[cfg(conduwuit_mods)] // Module instances; TODO: move to mods::loaded mgmt vector - pub(crate) mods: tokio::sync::RwLock>, + pub(crate) mods: tokio::sync::RwLock>, } impl Server { @@ -45,11 +45,11 @@ impl Server { database_path = ?config.database_path, log_levels = %config.log, "{}", - conduit::version(), + conduwuit::version(), ); Ok(Arc::new(Self { - server: Arc::new(conduit::Server::new( + server: Arc::new(conduwuit::Server::new( config, runtime.cloned(), Log { @@ -65,7 +65,7 @@ impl Server { #[cfg(feature = "sentry_telemetry")] _sentry_guard: sentry_guard, - #[cfg(conduit_mods)] + #[cfg(conduwuit_mods)] mods: tokio::sync::RwLock::new(Vec::new()), })) } diff --git a/src/main/signal.rs b/src/main/signal.rs index 139aab78..0f541099 100644 --- a/src/main/signal.rs +++ b/src/main/signal.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{debug_error, trace, warn}; +use conduwuit::{debug_error, trace, warn}; use tokio::signal; use super::server::Server; @@ -12,7 +12,7 @@ pub(super) async fn signal(server: Arc) { use unix::SignalKind; const CONSOLE: bool = cfg!(feature = "console"); - const RELOADING: bool = cfg!(all(conduit_mods, not(CONSOLE))); + const RELOADING: bool = cfg!(all(conduwuit_mods, not(CONSOLE))); let mut quit = unix::signal(SignalKind::quit()).expect("SIGQUIT handler"); let mut term = unix::signal(SignalKind::terminate()).expect("SIGTERM handler"); diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index e1535868..1b2c248e 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "conduit_router" +name = "conduwuit_router" categories.workspace = true description.workspace = true edition.workspace = true @@ -55,10 +55,10 @@ axum-server-dual-protocol.optional = true axum-server.workspace = true axum.workspace = true bytes.workspace = true -conduit-admin.workspace = true -conduit-api.workspace = true -conduit-core.workspace = true -conduit-service.workspace = true +conduwuit-admin.workspace = true +conduwuit-api.workspace = true +conduwuit-core.workspace = true +conduwuit-service.workspace = true const-str.workspace = true futures.workspace = true http.workspace = true diff --git a/src/router/layers.rs b/src/router/layers.rs index bb3ffe87..1c5beace 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -5,9 +5,9 @@ use axum::{ Router, }; use axum_client_ip::SecureClientIpSource; -use conduit::{error, Result, Server}; -use conduit_api::router::state::Guard; -use conduit_service::Services; +use conduwuit::{error, Result, Server}; +use conduwuit_api::router::state::Guard; +use conduwuit_service::Services; use http::{ header::{self, HeaderName}, HeaderValue, Method, StatusCode, @@ -155,7 +155,7 @@ fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max fn catch_panic(err: Box) -> http::Response> { //TODO: XXX /* - conduit_service::services() + conduwuit_service::services() .server .metrics .requests_panic diff --git a/src/router/mod.rs b/src/router/mod.rs index 215000cb..aab0b185 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -4,17 +4,17 @@ mod router; mod run; mod serve; -extern crate conduit_core as conduit; +extern crate conduwuit_core as conduwuit; use std::{panic::AssertUnwindSafe, pin::Pin, sync::Arc}; -use conduit::{Error, Result, Server}; -use conduit_service::Services; +use conduwuit::{Error, Result, Server}; +use conduwuit_service::Services; use futures::{Future, FutureExt, TryFutureExt}; -conduit::mod_ctor! {} -conduit::mod_dtor! {} -conduit::rustc_flags_capture! {} +conduwuit::mod_ctor! {} +conduwuit::mod_dtor! {} +conduwuit::rustc_flags_capture! {} #[unsafe(no_mangle)] pub extern "Rust" fn start(server: &Arc) -> Pin>> + Send>> { diff --git a/src/router/request.rs b/src/router/request.rs index 13d36981..b17e9c4f 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -4,8 +4,8 @@ use axum::{ extract::State, response::{IntoResponse, Response}, }; -use conduit::{debug, debug_error, debug_warn, defer, err, error, trace, Result}; -use conduit_service::Services; +use conduwuit::{debug, debug_error, debug_warn, defer, err, error, trace, Result}; +use conduwuit_service::Services; use http::{Method, StatusCode, Uri}; #[tracing::instrument( diff --git a/src/router/router.rs b/src/router/router.rs index 31b3f3e4..b3531418 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -1,16 +1,16 @@ use std::sync::Arc; use axum::{response::IntoResponse, routing::get, Router}; -use conduit::Error; -use conduit_api::router::{state, state::Guard}; -use conduit_service::Services; +use conduwuit::Error; +use conduwuit_api::router::{state, state::Guard}; +use conduwuit_service::Services; use http::{StatusCode, Uri}; use ruma::api::client::error::ErrorKind; pub(crate) fn build(services: &Arc) -> (Router, Guard) { let router = Router::::new(); let (state, guard) = state::create(services.clone()); - let router = conduit_api::router::build(router, &services.server) + let router = conduwuit_api::router::build(router, &services.server) .route("/", get(it_works)) .fallback(not_found) .with_state(state); diff --git a/src/router/run.rs b/src/router/run.rs index 93b1339b..6d5c8923 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -1,6 +1,6 @@ -extern crate conduit_admin as admin; -extern crate conduit_core as conduit; -extern crate conduit_service as service; +extern crate conduwuit_admin as admin; +extern crate conduwuit_core as conduwuit; +extern crate conduwuit_service as service; use std::{ sync::{atomic::Ordering, Arc}, @@ -8,7 +8,7 @@ use std::{ }; use axum_server::Handle as ServerHandle; -use conduit::{debug, debug_error, debug_info, error, info, Error, Result, Server}; +use conduwuit::{debug, debug_error, debug_info, error, info, Error, Result, Server}; use service::Services; use tokio::{ sync::broadcast::{self, Sender}, diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index b0254772..35792359 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -6,8 +6,8 @@ mod unix; use std::sync::Arc; use axum_server::Handle as ServerHandle; -use conduit::Result; -use conduit_service::Services; +use conduwuit::Result; +use conduwuit_service::Services; use tokio::sync::broadcast; use super::layers; @@ -28,7 +28,7 @@ pub(super) async fn serve( return tls::serve(server, app, handle, addrs).await; #[cfg(not(feature = "direct_tls"))] - return conduit::Err!(Config( + return conduwuit::Err!(Config( "tls", "conduwuit was not built with direct TLS support (\"direct_tls\")" )); diff --git a/src/router/serve/plain.rs b/src/router/serve/plain.rs index 144bff85..f6b6fba4 100644 --- a/src/router/serve/plain.rs +++ b/src/router/serve/plain.rs @@ -5,7 +5,7 @@ use std::{ use axum::Router; use axum_server::{bind, Handle as ServerHandle}; -use conduit::{debug_info, info, Result, Server}; +use conduwuit::{debug_info, info, Result, Server}; use tokio::task::JoinSet; pub(super) async fn serve( diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index f8d69048..7f54cfcc 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -6,7 +6,7 @@ use axum_server_dual_protocol::{ axum_server::{bind_rustls, tls_rustls::RustlsConfig}, ServerExt, }; -use conduit::{err, Result, Server}; +use conduwuit::{err, Result, Server}; use tokio::task::JoinSet; use tracing::{debug, info, warn}; diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index dd2ecf6a..fb37e125 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -10,7 +10,7 @@ use axum::{ extract::{connect_info::IntoMakeServiceWithConnectInfo, Request}, Router, }; -use conduit::{debug, debug_error, info, result::UnwrapInfallible, trace, warn, Err, Result, Server}; +use conduwuit::{debug, debug_error, info, result::UnwrapInfallible, trace, warn, Err, Result, Server}; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 19747847..85c4ead9 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "conduit_service" +name = "conduwuit_service" categories.workspace = true description.workspace = true edition.workspace = true @@ -44,8 +44,8 @@ arrayvec.workspace = true async-trait.workspace = true base64.workspace = true bytes.workspace = true -conduit-core.workspace = true -conduit-database.workspace = true +conduwuit-core.workspace = true +conduwuit-database.workspace = true const-str.workspace = true either.workspace = true futures.workspace = true diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 5dc17640..536a24e8 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{ +use conduwuit::{ err, implement, utils::{result::LogErr, stream::TryIgnore, ReadyExt}, Err, Result, diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 37af7452..c25eb6c6 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -4,7 +4,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{debug, defer, error, log, Server}; +use conduwuit::{debug, defer, error, log, Server}; use futures::future::{AbortHandle, Abortable}; use ruma::events::room::message::RoomMessageEventContent; use rustyline_async::{Readline, ReadlineError, ReadlineEvent}; diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 897d412d..971fdf67 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use conduit::{pdu::PduBuilder, Result}; +use conduwuit::{pdu::PduBuilder, Result}; use ruma::{ events::room::{ canonical_alias::RoomCanonicalAliasEventContent, @@ -67,7 +67,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { ) .await?; - // 2. Make conduit bot join + // 2. Make server user/bot join services .rooms .timeline diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 090c0294..2b05bfc7 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use conduit::{error, implement, Result}; +use conduwuit::{error, implement, Result}; use ruma::{ events::{ room::{ @@ -16,9 +16,9 @@ use ruma::{ use crate::pdu::PduBuilder; -/// Invite the user to the conduit admin room. +/// Invite the user to the conduwuit admin room. /// -/// In conduit, this is equivalent to granting admin privileges. +/// This is equivalent to granting server admin privileges. #[implement(super::Service)] pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { let Ok(room_id) = self.get_admin_room().await else { diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 2860bd1b..c4783565 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -10,7 +10,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{debug, err, error, error::default_log, pdu::PduBuilder, Error, PduEvent, Result, Server}; +use conduwuit::{debug, err, error, error::default_log, pdu::PduBuilder, Error, PduEvent, Result, Server}; pub use create::create_admin_room; use futures::{FutureExt, TryFutureExt}; use loole::{Receiver, Sender}; diff --git a/src/service/admin/startup.rs b/src/service/admin/startup.rs index 9c02c902..68ad4be1 100644 --- a/src/service/admin/startup.rs +++ b/src/service/admin/startup.rs @@ -1,4 +1,4 @@ -use conduit::{debug, debug_info, error, implement, info, Err, Result}; +use conduwuit::{debug, debug_info, error, implement, info, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; use tokio::time::{sleep, Duration}; diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index a55d6534..cf2921a7 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -4,7 +4,7 @@ mod registration_info; use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; -use conduit::{err, utils::stream::TryIgnore, Result}; +use conduwuit::{err, utils::stream::TryIgnore, Result}; use database::Map; use futures::{Future, StreamExt, TryStreamExt}; use ruma::{api::appservice::Registration, RoomAliasId, RoomId, UserId}; diff --git a/src/service/appservice/namespace_regex.rs b/src/service/appservice/namespace_regex.rs index 3529fc0e..fe0fd91f 100644 --- a/src/service/appservice/namespace_regex.rs +++ b/src/service/appservice/namespace_regex.rs @@ -1,4 +1,4 @@ -use conduit::Result; +use conduwuit::Result; use regex::RegexSet; use ruma::api::appservice::Namespace; diff --git a/src/service/appservice/registration_info.rs b/src/service/appservice/registration_info.rs index 2c8595b1..612933be 100644 --- a/src/service/appservice/registration_info.rs +++ b/src/service/appservice/registration_info.rs @@ -1,4 +1,4 @@ -use conduit::Result; +use conduwuit::Result; use ruma::{api::appservice::Registration, UserId}; use super::NamespaceRegex; diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index 71545541..f2bc8256 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -1,6 +1,6 @@ use std::{sync::Arc, time::Duration}; -use conduit::{err, implement, trace, Config, Result}; +use conduwuit::{err, implement, trace, Config, Result}; use either::Either; use ipaddress::IPAddress; use reqwest::redirect; @@ -124,7 +124,7 @@ fn base(config: &Config) -> Result { .timeout(Duration::from_secs(config.request_total_timeout)) .pool_idle_timeout(Duration::from_secs(config.request_idle_timeout)) .pool_max_idle_per_host(config.request_idle_per_host.into()) - .user_agent(conduit::version::user_agent()) + .user_agent(conduwuit::version::user_agent()) .redirect(redirect::Policy::limited(6)) .connection_verbose(true); @@ -188,7 +188,7 @@ fn builder_interface(builder: reqwest::ClientBuilder, config: Option<&str>) -> R #[cfg(not(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] fn builder_interface(builder: reqwest::ClientBuilder, config: Option<&str>) -> Result { - use conduit::Err; + use conduwuit::Err; if let Some(iface) = config { Err!("Binding to network-interface {iface:?} by name is not supported on this platform.") diff --git a/src/service/emergency/mod.rs b/src/service/emergency/mod.rs index c99a0891..5063fbd4 100644 --- a/src/service/emergency/mod.rs +++ b/src/service/emergency/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use async_trait::async_trait; -use conduit::{error, warn, Result}; +use conduwuit::{error, warn, Result}; use ruma::{ events::{push_rules::PushRulesEventContent, GlobalAccountDataEvent, GlobalAccountDataEventType}, push::Ruleset, @@ -38,7 +38,7 @@ impl crate::Service for Service { self.set_emergency_access() .await - .inspect_err(|e| error!("Could not set the configured emergency password for the conduit user: {e}"))?; + .inspect_err(|e| error!("Could not set the configured emergency password for the server user: {e}"))?; Ok(()) } @@ -47,17 +47,17 @@ impl crate::Service for Service { } impl Service { - /// Sets the emergency password and push rules for the @conduit account in - /// case emergency password is set + /// Sets the emergency password and push rules for the server user account + /// in case emergency password is set async fn set_emergency_access(&self) -> Result { - let conduit_user = &self.services.globals.server_user; + let server_user = &self.services.globals.server_user; self.services .users - .set_password(conduit_user, self.services.globals.emergency_password().as_deref())?; + .set_password(server_user, self.services.globals.emergency_password().as_deref())?; let (ruleset, pwd_set) = match self.services.globals.emergency_password() { - Some(_) => (Ruleset::server_default(conduit_user), true), + Some(_) => (Ruleset::server_default(server_user), true), None => (Ruleset::new(), false), }; @@ -65,7 +65,7 @@ impl Service { .account_data .update( None, - conduit_user, + server_user, GlobalAccountDataEventType::PushRules.to_string().into(), &serde_json::to_value(&GlobalAccountDataEvent { content: PushRulesEventContent { @@ -83,7 +83,7 @@ impl Service { ); } else { // logs out any users still in the server service account and removes sessions - self.services.users.deactivate_account(conduit_user).await?; + self.services.users.deactivate_account(server_user).await?; } Ok(pwd_set) diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index f715e944..5edd1f30 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -1,6 +1,6 @@ use std::sync::{Arc, RwLock}; -use conduit::{utils, Result}; +use conduwuit::{utils, Result}; use database::{Database, Deserialized, Map}; pub struct Data { diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 081794e2..2403b703 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,7 +7,7 @@ use std::{ time::Instant, }; -use conduit::{error, Config, Result}; +use conduwuit::{error, Config, Result}; use data::Data; use regex::RegexSet; use ruma::{OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomAliasId, ServerName, UserId}; @@ -83,8 +83,8 @@ impl crate::Service for Service { .server .supported_room_version(&config.default_room_version) { - error!(config=?s.config.default_room_version, fallback=?conduit::config::default_default_room_version(), "Room version in config isn't supported, falling back to default version"); - s.config.default_room_version = conduit::config::default_default_room_version(); + error!(config=?s.config.default_room_version, fallback=?conduwuit::config::default_default_room_version(), "Room version in config isn't supported, falling back to default version"); + s.config.default_room_version = conduwuit::config::default_default_room_version(); }; Ok(Arc::new(s)) diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index bae6f214..140fc701 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduit::{ +use conduwuit::{ err, implement, utils::stream::{ReadyExt, TryIgnore}, Err, Result, diff --git a/src/service/manager.rs b/src/service/manager.rs index 7384e512..3ce2e0a5 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -1,6 +1,6 @@ use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; -use conduit::{debug, debug_warn, error, trace, utils::time, warn, Err, Error, Result, Server}; +use conduwuit::{debug, debug_warn, error, trace, utils::time, warn, Err, Error, Result, Server}; use futures::FutureExt; use tokio::{ sync::{Mutex, MutexGuard}, diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 3922dec9..71fb1cdb 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -1,6 +1,6 @@ use std::{sync::Arc, time::Duration}; -use conduit::{ +use conduwuit::{ debug, debug_info, err, utils::{str_from_bytes, stream::TryIgnore, string_from_bytes, ReadyExt}, Err, Error, Result, diff --git a/src/service/media/migrations.rs b/src/service/media/migrations.rs index 3d9c395e..5932643b 100644 --- a/src/service/media/migrations.rs +++ b/src/service/media/migrations.rs @@ -7,7 +7,7 @@ use std::{ time::Instant, }; -use conduit::{ +use conduwuit::{ debug, debug_info, debug_warn, error, info, utils::{stream::TryIgnore, ReadyExt}, warn, Config, Result, diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index c0b15726..989e1c27 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -9,7 +9,7 @@ use std::{path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; use base64::{engine::general_purpose, Engine as _}; -use conduit::{ +use conduwuit::{ debug, debug_error, debug_info, debug_warn, err, error, trace, utils::{self, MutexMap}, warn, Err, Result, Server, diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index eb9be560..d571ac56 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -1,7 +1,7 @@ use std::{io::Cursor, time::SystemTime}; -use conduit::{debug, utils, Err, Result}; -use conduit_core::implement; +use conduwuit::{debug, utils, Err, Result}; +use conduwuit_core::implement; use image::ImageReader as ImgReader; use ipaddress::IPAddress; use ruma::Mxc; diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index 1c6c9ca0..8ec917b7 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -1,6 +1,6 @@ use std::{fmt::Debug, time::Duration}; -use conduit::{debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, Result}; +use conduwuit::{debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, Result}; use http::header::{HeaderValue, CONTENT_DISPOSITION, CONTENT_TYPE}; use ruma::{ api::{ diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 04ec0303..42fc40e7 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -1,6 +1,6 @@ use std::{cmp, io::Cursor, num::Saturating as Sat}; -use conduit::{checked, err, Result}; +use conduwuit::{checked, err, Result}; use image::{imageops::FilterType, DynamicImage}; use ruma::{http_headers::ContentDisposition, media::Method, Mxc, UInt, UserId}; use tokio::{ diff --git a/src/service/migrations.rs b/src/service/migrations.rs index f9057036..102ac7d4 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -1,6 +1,6 @@ use std::cmp; -use conduit::{ +use conduwuit::{ debug, debug_info, debug_warn, error, info, result::NotFound, utils::{ @@ -41,9 +41,9 @@ pub(crate) async fn migrations(services: &Services) -> Result<()> { // Matrix resource ownership is based on the server name; changing it // requires recreating the database from scratch. if users_count > 0 { - let conduit_user = &services.globals.server_user; - if !services.users.exists(conduit_user).await { - error!("The {conduit_user} server user does not exist, and the database is not new."); + let server_user = &services.globals.server_user; + if !services.users.exists(server_user).await { + error!("The {server_user} server user does not exist, and the database is not new."); return Err!(Database( "Cannot reuse an existing database after changing the server name, please delete the old one first.", )); diff --git a/src/service/mod.rs b/src/service/mod.rs index c7dcc0c6..789994d3 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -25,14 +25,14 @@ pub mod uiaa; pub mod updates; pub mod users; -extern crate conduit_core as conduit; -extern crate conduit_database as database; +extern crate conduwuit_core as conduwuit; +extern crate conduwuit_database as database; -pub use conduit::{pdu, PduBuilder, PduCount, PduEvent}; +pub use conduwuit::{pdu, PduBuilder, PduCount, PduEvent}; pub(crate) use service::{Args, Dep, Service}; pub use crate::services::Services; -conduit::mod_ctor! {} -conduit::mod_dtor! {} -conduit::rustc_flags_capture! {} +conduwuit::mod_ctor! {} +conduwuit::mod_dtor! {} +conduwuit::rustc_flags_capture! {} diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 68b2c3fe..dd0d8e72 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{ +use conduwuit::{ debug_warn, utils, utils::{stream::TryIgnore, ReadyExt}, Result, diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 8e1521c5..7e80e05e 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -4,7 +4,7 @@ mod presence; use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{checked, debug, error, result::LogErr, Error, Result, Server}; +use conduwuit::{checked, debug, error, result::LogErr, Error, Result, Server}; use futures::{stream::FuturesUnordered, Stream, StreamExt, TryFutureExt}; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; use tokio::{sync::Mutex, time::sleep}; diff --git a/src/service/presence/presence.rs b/src/service/presence/presence.rs index c4372003..aed4a3f8 100644 --- a/src/service/presence/presence.rs +++ b/src/service/presence/presence.rs @@ -1,4 +1,4 @@ -use conduit::{utils, Error, Result}; +use conduwuit::{utils, Error, Result}; use ruma::{ events::presence::{PresenceEvent, PresenceEventContent}, presence::PresenceState, diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index ff1837db..ffe822b7 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; -use conduit::{ +use conduwuit::{ debug_warn, err, trace, utils::{stream::TryIgnore, string_from_bytes}, warn, Err, PduEvent, Result, diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index a4475216..4d2da695 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -3,7 +3,7 @@ use std::{ net::{IpAddr, SocketAddr}, }; -use conduit::{debug, debug_error, debug_info, debug_warn, err, error, trace, Err, Result}; +use conduwuit::{debug, debug_error, debug_info, debug_warn, err, error, trace, Err, Result}; use hickory_resolver::error::ResolveError; use ipaddress::IPAddress; use ruma::ServerName; diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index a13399dc..ca65db35 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -6,7 +6,7 @@ use std::{ }; use arrayvec::ArrayVec; -use conduit::{trace, utils::rand}; +use conduwuit::{trace, utils::rand}; use ruma::{OwnedServerName, ServerName}; use super::fed::FedDest; diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index d3e9f5c9..54905647 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -1,6 +1,6 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; -use conduit::{err, Result, Server}; +use conduwuit::{err, Result, Server}; use futures::FutureExt; use hickory_resolver::TokioAsyncResolver; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 111de292..e18cf0bd 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -6,7 +6,7 @@ mod tests; use std::{fmt::Write, sync::Arc}; -use conduit::{Result, Server}; +use conduwuit::{Result, Server}; use self::{cache::Cache, dns::Resolver}; use crate::{client, Dep}; diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 0cdec8ee..57db0e15 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -2,7 +2,7 @@ mod remote; use std::sync::Arc; -use conduit::{ +use conduwuit::{ err, utils::{stream::TryIgnore, ReadyExt}, Err, Error, Result, diff --git a/src/service/rooms/alias/remote.rs b/src/service/rooms/alias/remote.rs index d9acccc9..00661da2 100644 --- a/src/service/rooms/alias/remote.rs +++ b/src/service/rooms/alias/remote.rs @@ -1,6 +1,6 @@ use std::iter::once; -use conduit::{debug, debug_error, err, implement, Result}; +use conduwuit::{debug, debug_error, err, implement, Result}; use federation::query::get_room_information::v1::Response; use ruma::{api::federation, OwnedRoomId, OwnedServerName, RoomAliasId, ServerName}; diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index 3c36928a..1548cd55 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{err, utils, utils::math::usize_from_f64, Err, Result}; +use conduwuit::{err, utils, utils::math::usize_from_f64, Err, Result}; use database::Map; use lru_cache::LruCache; diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index e7e5edf4..b875bf9c 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -6,7 +6,7 @@ use std::{ sync::Arc, }; -use conduit::{ +use conduwuit::{ debug, debug_error, trace, utils::{stream::ReadyExt, IterStream}, validated, warn, Err, Result, diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 63ed3519..3046a328 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{implement, utils::stream::TryIgnore, Result}; +use conduwuit::{implement, utils::stream::TryIgnore, Result}; use database::Map; use futures::Stream; use ruma::{api::client::room::Visibility, RoomId}; diff --git a/src/service/rooms/event_handler/acl_check.rs b/src/service/rooms/event_handler/acl_check.rs index f2ff1b00..ef12a25c 100644 --- a/src/service/rooms/event_handler/acl_check.rs +++ b/src/service/rooms/event_handler/acl_check.rs @@ -1,4 +1,4 @@ -use conduit::{debug, implement, trace, warn, Err, Result}; +use conduwuit::{debug, implement, trace, warn, Err, Result}; use ruma::{ events::{room::server_acl::RoomServerAclEventContent, StateEventType}, RoomId, ServerName, diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 8c418acb..316a1722 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -4,7 +4,7 @@ use std::{ time::Instant, }; -use conduit::{ +use conduwuit::{ debug, debug_error, implement, info, pdu, trace, utils::math::continue_exponential_backoff_secs, warn, PduEvent, }; use futures::TryFutureExt; diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 4acdba1d..577b3ff2 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -3,7 +3,7 @@ use std::{ sync::Arc, }; -use conduit::{debug_warn, err, implement, PduEvent, Result}; +use conduwuit::{debug_warn, err, implement, PduEvent, Result}; use futures::{future, FutureExt}; use ruma::{ int, diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 228b7d0c..9c7bc65e 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -3,7 +3,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, implement, warn, Err, Error, PduEvent, Result}; +use conduwuit::{debug, implement, warn, Err, Error, PduEvent, Result}; use futures::FutureExt; use ruma::{ api::federation::event::get_room_state_ids, events::StateEventType, EventId, RoomId, RoomVersionId, ServerName, diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 19367582..d63f96f9 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -4,7 +4,7 @@ use std::{ time::Instant, }; -use conduit::{debug, err, implement, warn, Error, Result}; +use conduwuit::{debug, err, implement, warn, Error, Result}; use futures::{FutureExt, TryFutureExt}; use ruma::{ api::client::error::ErrorKind, events::StateEventType, CanonicalJsonValue, EventId, RoomId, ServerName, UserId, diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 21504b66..9391ebf3 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -3,7 +3,7 @@ use std::{ sync::Arc, }; -use conduit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; +use conduwuit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; use futures::{future::ready, TryFutureExt}; use ruma::{ api::client::error::ErrorKind, diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index 90ff7f06..1b4e9fe2 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -4,7 +4,7 @@ use std::{ time::Instant, }; -use conduit::{debug, implement, utils::math::continue_exponential_backoff_secs, Error, PduEvent, Result}; +use conduwuit::{debug, implement, utils::math::continue_exponential_backoff_secs, Error, PduEvent, Result}; use ruma::{api::client::error::ErrorKind, CanonicalJsonValue, EventId, RoomId, ServerName}; #[implement(super::Service)] diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 3fb7d5c4..de3d2f49 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -17,7 +17,7 @@ use std::{ time::Instant, }; -use conduit::{ +use conduwuit::{ utils::{MutexMap, TryFutureExtExt}, Err, PduEvent, Result, Server, }; diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 42f44dee..6c19f43f 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,4 +1,4 @@ -use conduit::{err, implement, pdu::gen_event_id_canonical_json, result::FlatOk, Result}; +use conduwuit::{err, implement, pdu::gen_event_id_canonical_json, result::FlatOk, Result}; use ruma::{CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, RoomId}; use serde_json::value::RawValue as RawJsonValue; diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 3329a146..8ba4e4f4 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -4,7 +4,7 @@ use std::{ sync::Arc, }; -use conduit::{ +use conduwuit::{ debug, err, implement, utils::stream::{IterStream, WidebandExt}, Result, diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 9b30a830..6c76d9b5 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -4,7 +4,7 @@ use std::{ sync::Arc, }; -use conduit::{ +use conduwuit::{ debug, err, implement, result::LogErr, utils::stream::{BroadbandExt, IterStream}, diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 13e2b281..83267563 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -5,7 +5,7 @@ use std::{ time::Instant, }; -use conduit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; +use conduwuit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; use futures::{future::ready, StreamExt}; use ruma::{ api::client::error::ErrorKind, diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 7a4da2a6..a4bd4e8f 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -4,7 +4,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduit::{ +use conduwuit::{ implement, utils::{stream::TryIgnore, ReadyExt}, PduCount, Result, diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 4ee390a5..8f65eec3 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{implement, utils::stream::TryIgnore, Result}; +use conduwuit::{implement, utils::stream::TryIgnore, Result}; use database::Map; use futures::{Stream, StreamExt}; use ruma::RoomId; diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 03e77838..9cd3d805 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use database::{Deserialized, Json, Map}; use ruma::{CanonicalJsonObject, EventId}; diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 3d05a1c8..be9e4ba9 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,7 +1,7 @@ use std::{mem::size_of, sync::Arc}; use arrayvec::ArrayVec; -use conduit::{ +use conduwuit::{ result::LogErr, utils::{ stream::{TryIgnore, WidebandExt}, diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 82d2ee35..23c6d90b 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -1,7 +1,7 @@ mod data; use std::sync::Arc; -use conduit::{PduCount, Result}; +use conduwuit::{PduCount, Result}; use futures::StreamExt; use ruma::{api::Direction, EventId, RoomId, UserId}; diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 9a1dba45..03d04b65 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{ +use conduwuit::{ utils::{stream::TryIgnore, ReadyExt}, Result, }; diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 5cac535d..d92b9759 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -2,7 +2,7 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduit::{debug, err, warn, PduCount, PduId, RawPduId, Result}; +use conduwuit::{debug, err, warn, PduCount, PduId, RawPduId, Result}; use futures::{try_join, Stream, TryFutureExt}; use ruma::{ events::{ diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index ae3567ce..025fc42a 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use arrayvec::ArrayVec; -use conduit::{ +use conduwuit::{ implement, utils::{ set, diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index a7c32856..465ce1a9 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, fmt::Debug, mem::size_of_val, sync::Arc}; -pub use conduit::pdu::{ShortEventId, ShortId, ShortRoomId}; -use conduit::{err, implement, utils, utils::stream::ReadyExt, Result}; +pub use conduwuit::pdu::{ShortEventId, ShortId, ShortRoomId}; +use conduwuit::{err, implement, utils, utils::stream::ReadyExt, Result}; use database::{Deserialized, Map}; use futures::{Stream, StreamExt}; use ruma::{events::StateEventType, EventId, RoomId}; diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 3e972ca6..aa523871 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::Arc, }; -use conduit::{ +use conduwuit::{ checked, debug_info, err, utils::{math::usize_from_f64, IterStream}, Error, Result, diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index d0d21fa8..e083ed34 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -5,7 +5,7 @@ use std::{ sync::Arc, }; -use conduit::{ +use conduwuit::{ err, result::FlatOk, utils::{ diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index bca54069..9947b036 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,6 +1,6 @@ use std::{borrow::Borrow, collections::HashMap, sync::Arc}; -use conduit::{ +use conduwuit::{ at, err, ref_at, utils::stream::{BroadbandExt, IterStream, ReadyExt}, PduEvent, Result, diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index bcc1263d..798bcbed 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -7,7 +7,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; -use conduit::{ +use conduwuit::{ err, error, pdu::PduBuilder, utils::{math::usize_from_f64, ReadyExt}, diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index cbd72cdb..6b7d35d2 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, RwLock}, }; -use conduit::{ +use conduwuit::{ is_not_empty, result::LogErr, utils::{stream::TryIgnore, ReadyExt, StreamTools}, @@ -395,7 +395,7 @@ impl Service { pub fn get_shared_rooms<'a>( &'a self, user_a: &'a UserId, user_b: &'a UserId, ) -> impl Stream + Send + 'a { - use conduit::utils::set; + use conduwuit::utils::set; let a = self.rooms_joined(user_a); let b = self.rooms_joined(user_b); diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 8c6eccbe..36bc92c0 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use arrayvec::ArrayVec; -use conduit::{ +use conduwuit::{ at, checked, debug, err, expected, utils, utils::{bytes, math::usize_from_f64, stream::IterStream}, Result, diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index a304e482..f65be902 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduit::{ +use conduwuit::{ err, utils::{ stream::{TryIgnore, WidebandExt}, diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 94621385..a32ff54f 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -4,7 +4,7 @@ use std::{ sync::Arc, }; -use conduit::{ +use conduwuit::{ at, err, result::{LogErr, NotFound}, utils, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 8603b7e7..2bc5cf73 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -8,13 +8,13 @@ use std::{ sync::Arc, }; -use conduit::{ +use conduwuit::{ debug, debug_warn, err, error, implement, info, pdu::{EventHash, PduBuilder, PduCount, PduEvent}, utils::{self, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, validated, warn, Err, Error, Result, Server, }; -pub use conduit::{PduId, RawPduId}; +pub use conduwuit::{PduId, RawPduId}; use futures::{future, future::ready, Future, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{ api::federation, diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 8ee34f44..59862a85 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduit::{ +use conduwuit::{ debug_info, trace, utils::{self, IterStream}, Result, Server, diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 537fe69b..80d33de4 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use database::{Database, Deserialized, Map}; use ruma::{RoomId, UserId}; diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 55092523..9008a21f 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -1,7 +1,7 @@ use std::{fmt::Debug, mem}; use bytes::BytesMut; -use conduit::{debug_error, err, trace, utils, warn, Err, Result}; +use conduwuit::{debug_error, err, trace, utils, warn, Err, Result}; use reqwest::Client; use ruma::api::{appservice::Registration, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken}; diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index bc70e875..372d8e14 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -1,6 +1,6 @@ use std::{fmt::Debug, sync::Arc}; -use conduit::{ +use conduwuit::{ at, utils, utils::{stream::TryIgnore, ReadyExt}, Error, Result, diff --git a/src/service/sending/dest.rs b/src/service/sending/dest.rs index d35350c5..0a5893b7 100644 --- a/src/service/sending/dest.rs +++ b/src/service/sending/dest.rs @@ -1,6 +1,6 @@ use std::fmt::Debug; -use conduit::implement; +use conduwuit::implement; use ruma::{OwnedServerName, OwnedUserId}; #[derive(Clone, Debug, PartialEq, Eq, Hash)] diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 611940be..f9828178 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -7,7 +7,7 @@ mod sender; use std::{fmt::Debug, iter::once, sync::Arc}; use async_trait::async_trait; -use conduit::{ +use conduwuit::{ debug_warn, err, utils::{ReadyExt, TryReadyExt}, warn, Result, Server, diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index df39bcd1..862d2a42 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -1,7 +1,7 @@ use std::mem; use bytes::Bytes; -use conduit::{ +use conduwuit::{ debug, debug_error, debug_warn, err, error::inspect_debug_log, implement, trace, utils::string::EMPTY, Err, Error, Result, }; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 49ce1be4..77fd7d5c 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -6,7 +6,7 @@ use std::{ }; use base64::{engine::general_purpose, Engine as _}; -use conduit::{ +use conduwuit::{ debug, err, error, result::LogErr, trace, diff --git a/src/service/server_keys/acquire.rs b/src/service/server_keys/acquire.rs index 1080d79e..886c4750 100644 --- a/src/service/server_keys/acquire.rs +++ b/src/service/server_keys/acquire.rs @@ -4,7 +4,7 @@ use std::{ time::Duration, }; -use conduit::{debug, debug_error, debug_warn, error, implement, info, result::FlatOk, trace, warn}; +use conduwuit::{debug, debug_error, debug_warn, error, implement, info, result::FlatOk, trace, warn}; use futures::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::federation::discovery::ServerSigningKeys, serde::Raw, CanonicalJsonObject, OwnedServerName, diff --git a/src/service/server_keys/get.rs b/src/service/server_keys/get.rs index dc4627f7..94d2575a 100644 --- a/src/service/server_keys/get.rs +++ b/src/service/server_keys/get.rs @@ -1,6 +1,6 @@ use std::borrow::Borrow; -use conduit::{implement, Err, Result}; +use conduwuit::{implement, Err, Result}; use ruma::{api::federation::discovery::VerifyKey, CanonicalJsonObject, RoomVersionId, ServerName, ServerSigningKeyId}; use super::{extract_key, PubKeyMap, PubKeys}; diff --git a/src/service/server_keys/keypair.rs b/src/service/server_keys/keypair.rs index 31a24cdf..927171f3 100644 --- a/src/service/server_keys/keypair.rs +++ b/src/service/server_keys/keypair.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{debug, debug_info, err, error, utils, utils::string_from_bytes, Result}; +use conduwuit::{debug, debug_info, err, error, utils, utils::string_from_bytes, Result}; use database::Database; use ruma::{api::federation::discovery::VerifyKey, serde::Base64, signatures::Ed25519KeyPair}; diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index 08bcefb6..45c01c0b 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -7,7 +7,7 @@ mod verify; use std::{collections::BTreeMap, sync::Arc, time::Duration}; -use conduit::{ +use conduwuit::{ implement, utils::{timepoint_from_now, IterStream}, Result, Server, diff --git a/src/service/server_keys/request.rs b/src/service/server_keys/request.rs index 7078f7cd..da7fa08a 100644 --- a/src/service/server_keys/request.rs +++ b/src/service/server_keys/request.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, fmt::Debug}; -use conduit::{debug, implement, Err, Result}; +use conduwuit::{debug, implement, Err, Result}; use ruma::{ api::federation::discovery::{ get_remote_server_keys, diff --git a/src/service/server_keys/sign.rs b/src/service/server_keys/sign.rs index 28fd7e80..71565380 100644 --- a/src/service/server_keys/sign.rs +++ b/src/service/server_keys/sign.rs @@ -1,4 +1,4 @@ -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use ruma::{CanonicalJsonObject, RoomVersionId}; #[implement(super::Service)] diff --git a/src/service/server_keys/verify.rs b/src/service/server_keys/verify.rs index c836e324..d06b55ab 100644 --- a/src/service/server_keys/verify.rs +++ b/src/service/server_keys/verify.rs @@ -1,4 +1,4 @@ -use conduit::{implement, pdu::gen_event_id_canonical_json, Err, Result}; +use conduwuit::{implement, pdu::gen_event_id_canonical_json, Err, Result}; use ruma::{signatures::Verified, CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, RoomVersionId}; use serde_json::value::RawValue as RawJsonValue; diff --git a/src/service/service.rs b/src/service/service.rs index 7ec2ea0f..4b1774cc 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -7,7 +7,7 @@ use std::{ }; use async_trait::async_trait; -use conduit::{err, error::inspect_log, utils::string::SplitInfallible, Err, Result, Server}; +use conduwuit::{err, error::inspect_log, utils::string::SplitInfallible, Err, Result, Server}; use database::Database; /// Abstract interface for a Service diff --git a/src/service/services.rs b/src/service/services.rs index b86e7a72..9f9d10f5 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -5,7 +5,7 @@ use std::{ sync::{Arc, RwLock}, }; -use conduit::{debug, debug_info, info, trace, Result, Server}; +use conduwuit::{debug, debug_info, info, trace, Result, Server}; use database::Database; use tokio::sync::Mutex; diff --git a/src/service/sync/mod.rs b/src/service/sync/mod.rs index f1a6ae75..a4523246 100644 --- a/src/service/sync/mod.rs +++ b/src/service/sync/mod.rs @@ -5,7 +5,7 @@ use std::{ sync::{Arc, Mutex, Mutex as StdMutex}, }; -use conduit::{Result, Server}; +use conduwuit::{Result, Server}; use database::Map; use ruma::{ api::client::sync::sync_events::{ diff --git a/src/service/sync/watch.rs b/src/service/sync/watch.rs index 3eb663c1..50959ded 100644 --- a/src/service/sync/watch.rs +++ b/src/service/sync/watch.rs @@ -1,4 +1,4 @@ -use conduit::{implement, trace, Result}; +use conduwuit::{implement, trace, Result}; use futures::{pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; use ruma::{DeviceId, UserId}; diff --git a/src/service/transaction_ids/mod.rs b/src/service/transaction_ids/mod.rs index 72f60adb..2b979f99 100644 --- a/src/service/transaction_ids/mod.rs +++ b/src/service/transaction_ids/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduit::{implement, Result}; +use conduwuit::{implement, Result}; use database::{Handle, Map}; use ruma::{DeviceId, TransactionId, UserId}; diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index d2865d88..4d30a612 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, RwLock}, }; -use conduit::{ +use conduwuit::{ err, error, implement, utils, utils::{hash, string::EMPTY}, Error, Result, diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index 82decd26..1d13337d 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -1,7 +1,7 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduit::{debug, info, warn, Result, Server}; +use conduwuit::{debug, info, warn, Result, Server}; use database::{Deserialized, Map}; use ruma::events::room::message::RoomMessageEventContent; use serde::Deserialize; diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 70868ec8..8ceec2a0 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,6 +1,6 @@ use std::{collections::BTreeMap, mem, mem::size_of, sync::Arc}; -use conduit::{ +use conduwuit::{ debug_warn, err, utils, utils::{stream::TryIgnore, string::Unquoted, ReadyExt}, Err, Error, Result, Server, diff --git a/tests/cargo_smoke.sh b/tests/cargo_smoke.sh index 6ee20ee2..946790c3 100755 --- a/tests/cargo_smoke.sh +++ b/tests/cargo_smoke.sh @@ -38,8 +38,8 @@ element () { run cargo "$TOOLCHAIN" build $ELEMENT_OPTS --all-targets run cargo "$TOOLCHAIN" test $ELEMENT_OPTS --all-targets run cargo "$TOOLCHAIN" bench $ELEMENT_OPTS --all-targets - run cargo "$TOOLCHAIN" run $ELEMENT_OPTS --bin conduit -- -V - run conduwuit "$TOOLCHAIN" run $ELEMENT_OPTS --bin conduit + run cargo "$TOOLCHAIN" run $ELEMENT_OPTS --bin conduwuit -- -V + run conduwuit "$TOOLCHAIN" run $ELEMENT_OPTS --bin conduwuit fi } From 77e0b764080feb567f1d124ce04b037d7770f12f Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 00:05:47 -0500 Subject: [PATCH 1229/2091] apply new rustfmt.toml changes, fix some clippy lints Signed-off-by: strawberry --- .editorconfig | 1 + clippy.toml | 6 +- rustfmt.toml | 12 +- src/admin/admin.rs | 30 +- src/admin/appservice/commands.rs | 36 +- src/admin/check/commands.rs | 5 +- src/admin/check/mod.rs | 7 +- src/admin/debug/commands.rs | 229 +++++--- src/admin/federation/commands.rs | 33 +- src/admin/media/commands.rs | 121 +++-- src/admin/processor.rs | 41 +- src/admin/query/account_data.rs | 17 +- src/admin/query/appservice.rs | 11 +- src/admin/query/globals.rs | 15 +- src/admin/query/mod.rs | 5 +- src/admin/query/presence.rs | 13 +- src/admin/query/pusher.rs | 9 +- src/admin/query/resolver.rs | 23 +- src/admin/query/room_alias.rs | 15 +- src/admin/query/room_state_cache.rs | 71 +-- src/admin/query/sending.rs | 87 +-- src/admin/query/users.rs | 54 +- src/admin/room/alias.rs | 147 +++--- src/admin/room/commands.rs | 12 +- src/admin/room/directory.rs | 21 +- src/admin/room/info.rs | 6 +- src/admin/room/mod.rs | 3 +- src/admin/room/moderation.rs | 215 +++++--- src/admin/server/commands.rs | 34 +- src/admin/user/commands.rs | 177 ++++--- src/admin/utils.rs | 10 +- src/api/client/account.rs | 201 ++++--- src/api/client/account_data.rs | 49 +- src/api/client/alias.rs | 17 +- src/api/client/appservice.rs | 14 +- src/api/client/backup.rs | 75 +-- src/api/client/capabilities.rs | 22 +- src/api/client/context.rs | 21 +- src/api/client/device.rs | 31 +- src/api/client/directory.rs | 80 ++- src/api/client/filter.rs | 6 +- src/api/client/keys.rs | 87 +-- src/api/client/media.rs | 46 +- src/api/client/media_legacy.rs | 83 ++- src/api/client/membership.rs | 496 +++++++++++------- src/api/client/message.rs | 54 +- src/api/client/openid.rs | 3 +- src/api/client/presence.rs | 16 +- src/api/client/profile.rs | 94 ++-- src/api/client/push.rs | 135 ++--- src/api/client/read_marker.rs | 40 +- src/api/client/redact.rs | 11 +- src/api/client/relations.rs | 46 +- src/api/client/report.rs | 32 +- src/api/client/room/aliases.rs | 3 +- src/api/client/room/create.rs | 198 ++++--- src/api/client/room/event.rs | 7 +- src/api/client/room/initial_sync.rs | 3 +- src/api/client/room/upgrade.rs | 46 +- src/api/client/search.rs | 20 +- src/api/client/send.rs | 24 +- src/api/client/session.rs | 74 ++- src/api/client/space.rs | 3 +- src/api/client/state.rs | 74 ++- src/api/client/sync/mod.rs | 13 +- src/api/client/sync/v3.rs | 287 ++++++---- src/api/client/sync/v4.rs | 391 +++++++------- src/api/client/tag.rs | 25 +- src/api/client/thirdparty.rs | 4 +- src/api/client/threads.rs | 3 +- src/api/client/to_device.rs | 29 +- src/api/client/typing.rs | 3 +- src/api/client/unstable.rs | 64 ++- src/api/client/unversioned.rs | 4 +- src/api/client/user_directory.rs | 14 +- src/api/client/voip.rs | 6 +- src/api/client/well_known.rs | 41 +- src/api/router.rs | 13 +- src/api/router/args.rs | 31 +- src/api/router/auth.rs | 137 +++-- src/api/router/handler.rs | 18 +- src/api/router/request.rs | 16 +- src/api/router/state.rs | 4 +- src/api/server/backfill.rs | 3 +- src/api/server/event.rs | 3 +- src/api/server/event_auth.rs | 11 +- src/api/server/get_missing_events.rs | 7 +- src/api/server/hierarchy.rs | 3 +- src/api/server/invite.rs | 14 +- src/api/server/key.rs | 8 +- src/api/server/make_join.rs | 65 ++- src/api/server/make_leave.rs | 8 +- src/api/server/media.rs | 12 +- src/api/server/openid.rs | 3 +- src/api/server/publicrooms.rs | 14 +- src/api/server/query.rs | 19 +- src/api/server/send.rs | 131 +++-- src/api/server/send_join.rs | 65 ++- src/api/server/send_leave.rs | 13 +- src/api/server/state.rs | 8 +- src/api/server/state_ids.rs | 8 +- src/api/server/user.rs | 19 +- src/api/server/utils.rs | 9 +- src/api/server/well_known.rs | 7 +- src/core/alloc/je.rs | 5 +- src/core/alloc/mod.rs | 16 +- src/core/config/check.rs | 97 ++-- src/core/config/mod.rs | 87 +-- src/core/config/proxy.rs | 40 +- src/core/debug.rs | 8 +- src/core/error/err.rs | 7 +- src/core/error/log.rs | 24 +- src/core/error/mod.rs | 35 +- src/core/error/panic.rs | 12 +- src/core/error/response.rs | 34 +- src/core/info/cargo.rs | 13 +- src/core/info/room_version.rs | 4 +- src/core/log/capture/layer.rs | 16 +- src/core/log/capture/mod.rs | 4 +- src/core/log/capture/state.rs | 6 +- src/core/log/color.rs | 20 +- src/core/log/fmt.rs | 3 +- src/core/log/fmt_span.rs | 16 +- src/core/log/suppress.rs | 5 +- src/core/metrics/mod.rs | 4 +- src/core/pdu/builder.rs | 6 +- src/core/pdu/count.rs | 44 +- src/core/pdu/event.rs | 12 +- src/core/pdu/event_id.rs | 8 +- src/core/pdu/filter.rs | 4 +- src/core/pdu/mod.rs | 3 +- src/core/pdu/raw_id.rs | 22 +- src/core/pdu/redact.rs | 8 +- src/core/pdu/strip.rs | 6 +- src/core/pdu/unsigned.rs | 8 +- src/core/server.rs | 10 +- src/core/utils/bool.rs | 12 +- src/core/utils/bytes.rs | 4 +- src/core/utils/content_disposition.rs | 32 +- src/core/utils/debug.rs | 8 +- src/core/utils/defer.rs | 4 +- src/core/utils/future/try_ext_ext.rs | 46 +- src/core/utils/hash/argon.rs | 4 +- src/core/utils/html.rs | 12 +- src/core/utils/json.rs | 16 +- src/core/utils/math.rs | 14 +- src/core/utils/mod.rs | 4 +- src/core/utils/mutex_map.rs | 7 +- src/core/utils/result.rs | 6 +- src/core/utils/result/flat_ok.rs | 8 +- src/core/utils/result/log_err.rs | 4 +- src/core/utils/set.rs | 4 +- src/core/utils/stream/expect.rs | 8 +- src/core/utils/stream/ignore.rs | 8 +- src/core/utils/stream/iter_stream.rs | 13 +- src/core/utils/stream/ready.rs | 87 ++- src/core/utils/stream/tools.rs | 18 +- src/core/utils/stream/try_broadband.rs | 17 +- src/core/utils/stream/try_ready.rs | 48 +- src/core/utils/string/between.rs | 4 +- src/core/utils/string/split.rs | 8 +- src/core/utils/string/unquote.rs | 4 +- src/core/utils/sys.rs | 7 +- src/core/utils/tests.rs | 1 + src/core/utils/time.rs | 57 +- src/database/cork.rs | 6 +- src/database/database.rs | 9 +- src/database/de.rs | 44 +- src/database/engine.rs | 55 +- src/database/handle.rs | 6 +- src/database/map.rs | 13 +- src/database/map/contains.rs | 11 +- src/database/map/count.rs | 15 +- src/database/map/get.rs | 19 +- src/database/map/get_batch.rs | 13 +- src/database/map/keys_from.rs | 10 +- src/database/map/keys_prefix.rs | 18 +- src/database/map/rev_keys_from.rs | 20 +- src/database/map/rev_keys_prefix.rs | 18 +- src/database/map/rev_stream_from.rs | 16 +- src/database/map/rev_stream_prefix.rs | 14 +- src/database/map/stream_from.rs | 18 +- src/database/map/stream_prefix.rs | 14 +- src/database/opts.rs | 123 +++-- src/database/pool.rs | 12 +- src/database/ser.rs | 113 ++-- src/database/stream/items.rs | 16 +- src/database/stream/items_rev.rs | 16 +- src/database/stream/keys.rs | 16 +- src/database/stream/keys_rev.rs | 16 +- src/database/tests.rs | 24 +- src/database/util.rs | 8 +- src/database/watchers.rs | 9 +- src/macros/admin.rs | 6 +- src/macros/config.rs | 87 +-- src/macros/implement.rs | 11 +- src/macros/refutable.rs | 12 +- src/macros/rustc.rs | 4 +- src/macros/utils.rs | 16 +- src/main/logging.rs | 30 +- src/main/main.rs | 4 +- src/main/mods.rs | 4 +- src/main/server.rs | 23 +- src/router/layers.rs | 20 +- src/router/mod.rs | 12 +- src/router/request.rs | 8 +- src/router/run.rs | 24 +- src/router/serve/mod.rs | 4 +- src/router/serve/plain.rs | 8 +- src/router/serve/tls.rs | 14 +- src/router/serve/unix.rs | 21 +- src/service/account_data/mod.rs | 35 +- src/service/admin/console.rs | 23 +- src/service/admin/create.rs | 68 +-- src/service/admin/grant.rs | 30 +- src/service/admin/mod.rs | 49 +- src/service/admin/startup.rs | 6 +- src/service/appservice/mod.rs | 11 +- src/service/appservice/registration_info.rs | 6 +- src/service/client/mod.rs | 14 +- src/service/emergency/mod.rs | 34 +- src/service/globals/data.rs | 4 +- src/service/globals/mod.rs | 91 ++-- src/service/key_backups/mod.rs | 68 ++- src/service/manager.rs | 32 +- src/service/media/data.rs | 69 ++- src/service/media/migrations.rs | 11 +- src/service/media/mod.rs | 91 ++-- src/service/media/preview.rs | 54 +- src/service/media/remote.rs | 189 ++++--- src/service/media/tests.rs | 32 +- src/service/media/thumbnail.rs | 40 +- src/service/migrations.rs | 35 +- src/service/presence/data.rs | 35 +- src/service/presence/mod.rs | 53 +- src/service/presence/presence.rs | 14 +- src/service/pusher/mod.rs | 109 ++-- src/service/resolver/actual.rs | 161 +++--- src/service/resolver/cache.rs | 20 +- src/service/resolver/dns.rs | 19 +- src/service/resolver/fed.rs | 28 +- src/service/rooms/alias/mod.rs | 74 ++- src/service/rooms/alias/remote.rs | 30 +- src/service/rooms/auth_chain/data.rs | 8 +- src/service/rooms/auth_chain/mod.rs | 37 +- src/service/rooms/directory/mod.rs | 8 +- .../fetch_and_handle_outliers.rs | 80 ++- src/service/rooms/event_handler/fetch_prev.rs | 20 +- .../rooms/event_handler/fetch_state.rs | 27 +- .../event_handler/handle_incoming_pdu.rs | 15 +- .../rooms/event_handler/handle_outlier_pdu.rs | 33 +- .../rooms/event_handler/handle_prev_pdu.rs | 22 +- src/service/rooms/event_handler/mod.rs | 14 +- .../rooms/event_handler/parse_incoming_pdu.rs | 15 +- .../rooms/event_handler/resolve_state.rs | 19 +- .../rooms/event_handler/state_at_incoming.rs | 22 +- .../event_handler/upgrade_outlier_pdu.rs | 35 +- src/service/rooms/lazy_loading/mod.rs | 21 +- src/service/rooms/metadata/mod.rs | 16 +- src/service/rooms/pdu_metadata/data.rs | 21 +- src/service/rooms/pdu_metadata/mod.rs | 24 +- src/service/rooms/read_receipt/data.rs | 23 +- src/service/rooms/read_receipt/mod.rs | 71 +-- src/service/rooms/search/mod.rs | 34 +- src/service/rooms/short/mod.rs | 31 +- src/service/rooms/spaces/mod.rs | 200 ++++--- src/service/rooms/spaces/tests.rs | 21 +- src/service/rooms/state/mod.rs | 112 ++-- src/service/rooms/state_accessor/data.rs | 52 +- src/service/rooms/state_accessor/mod.rs | 155 ++++-- src/service/rooms/state_cache/mod.rs | 156 ++++-- src/service/rooms/state_compressor/mod.rs | 134 +++-- src/service/rooms/threads/mod.rs | 25 +- src/service/rooms/timeline/data.rs | 68 ++- src/service/rooms/timeline/mod.rs | 235 ++++++--- src/service/rooms/typing/mod.rs | 26 +- src/service/rooms/user/mod.rs | 13 +- src/service/sending/appservice.rs | 20 +- src/service/sending/data.rs | 50 +- src/service/sending/dest.rs | 6 +- src/service/sending/mod.rs | 56 +- src/service/sending/send.rs | 79 ++- src/service/sending/sender.rs | 183 ++++--- src/service/server_keys/acquire.rs | 47 +- src/service/server_keys/get.rs | 34 +- src/service/server_keys/keypair.rs | 7 +- src/service/server_keys/mod.rs | 13 +- src/service/server_keys/request.rs | 8 +- src/service/server_keys/sign.rs | 6 +- src/service/server_keys/verify.rs | 30 +- src/service/service.rs | 4 +- src/service/sync/mod.rs | 116 ++-- src/service/transaction_ids/mod.rs | 13 +- src/service/uiaa/mod.rs | 67 ++- src/service/updates/mod.rs | 7 +- src/service/users/mod.rs | 231 ++++++-- 296 files changed, 7147 insertions(+), 4300 deletions(-) diff --git a/.editorconfig b/.editorconfig index b2455005..2d7438a4 100644 --- a/.editorconfig +++ b/.editorconfig @@ -21,3 +21,4 @@ indent_size = 2 [*.rs] indent_style = tab +max_line_length = 98 diff --git a/clippy.toml b/clippy.toml index 3a0aa695..d9dd99ca 100644 --- a/clippy.toml +++ b/clippy.toml @@ -3,7 +3,7 @@ cognitive-complexity-threshold = 94 # TODO reduce me ALARA excessive-nesting-threshold = 11 # TODO reduce me to 4 or 5 future-size-threshold = 7745 # TODO reduce me ALARA stack-size-threshold = 196608 # reduce me ALARA -too-many-lines-threshold = 700 # TODO reduce me to <= 100 +too-many-lines-threshold = 780 # TODO reduce me to <= 100 type-complexity-threshold = 250 # reduce me to ~200 disallowed-macros = [ @@ -13,3 +13,7 @@ disallowed-macros = [ { path = "log::debug", reason = "use conduwuit_core::debug" }, { path = "log::trace", reason = "use conduwuit_core::trace" }, ] + +disallowed-methods = [ + { path = "tokio::spawn", reason = "use and pass conduuwit_core::server::Server::runtime() to spawn from"}, +] diff --git a/rustfmt.toml b/rustfmt.toml index fd912a19..635ec8f8 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -2,9 +2,8 @@ array_width = 80 chain_width = 60 comment_width = 80 condense_wildcard_suffixes = true -edition = "2021" +edition = "2024" fn_call_width = 80 -fn_params_layout = "Compressed" fn_single_line = true format_code_in_doc_comments = true format_macro_bodies = true @@ -14,13 +13,20 @@ group_imports = "StdExternalCrate" hard_tabs = true hex_literal_case = "Upper" imports_granularity = "Crate" +match_arm_blocks = false +match_arm_leading_pipes = "Always" match_block_trailing_comma = true -max_width = 120 +max_width = 98 newline_style = "Unix" normalize_comments = false +overflow_delimited_expr = true reorder_impl_items = true reorder_imports = true +single_line_if_else_max_width = 60 +single_line_let_else_max_width = 80 +struct_lit_width = 40 tab_spaces = 4 +unstable_features = true use_field_init_shorthand = true use_small_heuristics = "Off" use_try_shorthand = true diff --git a/src/admin/admin.rs b/src/admin/admin.rs index 7b707446..9097a613 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -3,9 +3,10 @@ use conduwuit::Result; use ruma::events::room::message::RoomMessageEventContent; use crate::{ - appservice, appservice::AppserviceCommand, check, check::CheckCommand, command::Command, debug, - debug::DebugCommand, federation, federation::FederationCommand, media, media::MediaCommand, query, - query::QueryCommand, room, room::RoomCommand, server, server::ServerCommand, user, user::UserCommand, + appservice, appservice::AppserviceCommand, check, check::CheckCommand, command::Command, + debug, debug::DebugCommand, federation, federation::FederationCommand, media, + media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, server, + server::ServerCommand, user, user::UserCommand, }; #[derive(Debug, Parser)] @@ -49,18 +50,21 @@ pub(super) enum AdminCommand { } #[tracing::instrument(skip_all, name = "command")] -pub(super) async fn process(command: AdminCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + command: AdminCommand, + context: &Command<'_>, +) -> Result { use AdminCommand::*; Ok(match command { - Appservices(command) => appservice::process(command, context).await?, - Media(command) => media::process(command, context).await?, - Users(command) => user::process(command, context).await?, - Rooms(command) => room::process(command, context).await?, - Federation(command) => federation::process(command, context).await?, - Server(command) => server::process(command, context).await?, - Debug(command) => debug::process(command, context).await?, - Query(command) => query::process(command, context).await?, - Check(command) => check::process(command, context).await?, + | Appservices(command) => appservice::process(command, context).await?, + | Media(command) => media::process(command, context).await?, + | Users(command) => user::process(command, context).await?, + | Rooms(command) => room::process(command, context).await?, + | Federation(command) => federation::process(command, context).await?, + | Server(command) => server::process(command, context).await?, + | Debug(command) => debug::process(command, context).await?, + | Query(command) => query::process(command, context).await?, + | Check(command) => check::process(command, context).await?, }) } diff --git a/src/admin/appservice/commands.rs b/src/admin/appservice/commands.rs index 6d661308..4f02531a 100644 --- a/src/admin/appservice/commands.rs +++ b/src/admin/appservice/commands.rs @@ -4,7 +4,9 @@ use crate::{admin_command, Result}; #[admin_command] pub(super) async fn register(&self) -> Result { - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -14,55 +16,63 @@ pub(super) async fn register(&self) -> Result { let appservice_config_body = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); let parsed_config = serde_yaml::from_str::(&appservice_config_body); match parsed_config { - Ok(registration) => match self + | Ok(registration) => match self .services .appservice .register_appservice(®istration, &appservice_config_body) .await { - Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( + | Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( "Appservice registered with ID: {}", registration.id ))), - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "Failed to register appservice: {e}" ))), }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "Could not parse appservice config as YAML: {e}" ))), } } #[admin_command] -pub(super) async fn unregister(&self, appservice_identifier: String) -> Result { +pub(super) async fn unregister( + &self, + appservice_identifier: String, +) -> Result { match self .services .appservice .unregister_appservice(&appservice_identifier) .await { - Ok(()) => Ok(RoomMessageEventContent::text_plain("Appservice unregistered.")), - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + | Ok(()) => Ok(RoomMessageEventContent::text_plain("Appservice unregistered.")), + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "Failed to unregister appservice: {e}" ))), } } #[admin_command] -pub(super) async fn show_appservice_config(&self, appservice_identifier: String) -> Result { +pub(super) async fn show_appservice_config( + &self, + appservice_identifier: String, +) -> Result { match self .services .appservice .get_registration(&appservice_identifier) .await { - Some(config) => { - let config_str = serde_yaml::to_string(&config).expect("config should've been validated on register"); - let output = format!("Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```",); + | Some(config) => { + let config_str = serde_yaml::to_string(&config) + .expect("config should've been validated on register"); + let output = + format!("Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```",); Ok(RoomMessageEventContent::notice_markdown(output)) }, - None => Ok(RoomMessageEventContent::text_plain("Appservice does not exist.")), + | None => Ok(RoomMessageEventContent::text_plain("Appservice does not exist.")), } } diff --git a/src/admin/check/commands.rs b/src/admin/check/commands.rs index ecb18c1a..7e27362f 100644 --- a/src/admin/check/commands.rs +++ b/src/admin/check/commands.rs @@ -19,8 +19,9 @@ pub(super) async fn check_all_users(&self) -> Result { let ok_count = users.iter().filter(|_user| true).count(); let message = format!( - "Database query completed in {query_time:?}:\n\n```\nTotal entries: {total:?}\nFailure/Invalid user count: \ - {err_count:?}\nSuccess/Valid user count: {ok_count:?}\n```" + "Database query completed in {query_time:?}:\n\n```\nTotal entries: \ + {total:?}\nFailure/Invalid user count: {err_count:?}\nSuccess/Valid user count: \ + {ok_count:?}\n```" ); Ok(RoomMessageEventContent::notice_markdown(message)) diff --git a/src/admin/check/mod.rs b/src/admin/check/mod.rs index 44b1ddf2..4790a6de 100644 --- a/src/admin/check/mod.rs +++ b/src/admin/check/mod.rs @@ -11,8 +11,11 @@ pub(super) enum CheckCommand { AllUsers, } -pub(super) async fn process(command: CheckCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + command: CheckCommand, + context: &Command<'_>, +) -> Result { Ok(match command { - CheckCommand::AllUsers => context.check_all_users().await?, + | CheckCommand::AllUsers => context.check_all_users().await?, }) } diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 9d77a1de..0c5a7cbd 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,7 +6,9 @@ use std::{ time::{Instant, SystemTime}, }; -use conduwuit::{debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result}; +use conduwuit::{ + debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result, +}; use futures::{FutureExt, StreamExt}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, @@ -26,7 +28,10 @@ pub(super) async fn echo(&self, message: Vec) -> Result) -> Result { +pub(super) async fn get_auth_chain( + &self, + event_id: Box, +) -> Result { let Ok(event) = self.services.rooms.timeline.get_pdu_json(&event_id).await else { return Ok(RoomMessageEventContent::notice_plain("Event not found.")); }; @@ -68,20 +73,26 @@ pub(super) async fn parse_pdu(&self) -> Result { let string = self.body[1..self.body.len().saturating_sub(1)].join("\n"); match serde_json::from_str(&string) { - Ok(value) => match ruma::signatures::reference_hash(&value, &RoomVersionId::V6) { - Ok(hash) => { + | Ok(value) => match ruma::signatures::reference_hash(&value, &RoomVersionId::V6) { + | Ok(hash) => { let event_id = EventId::parse(format!("${hash}")); - match serde_json::from_value::(serde_json::to_value(value).expect("value is json")) { - Ok(pdu) => Ok(RoomMessageEventContent::text_plain(format!("EventId: {event_id:?}\n{pdu:#?}"))), - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + match serde_json::from_value::( + serde_json::to_value(value).expect("value is json"), + ) { + | Ok(pdu) => Ok(RoomMessageEventContent::text_plain(format!( + "EventId: {event_id:?}\n{pdu:#?}" + ))), + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "EventId: {event_id:?}\nCould not parse event: {e}" ))), } }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Could not parse PDU JSON: {e:?}"))), + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + "Could not parse PDU JSON: {e:?}" + ))), }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "Invalid json in command body: {e}" ))), } @@ -103,8 +114,9 @@ pub(super) async fn get_pdu(&self, event_id: Box) -> Result { - let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json"); + | Ok(json) => { + let json_text = + serde_json::to_string_pretty(&json).expect("canonical json is valid json"); Ok(RoomMessageEventContent::notice_markdown(format!( "{}\n```json\n{}\n```", if outlier { @@ -115,13 +127,15 @@ pub(super) async fn get_pdu(&self, event_id: Box) -> Result Ok(RoomMessageEventContent::text_plain("PDU not found locally.")), + | Err(_) => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")), } } #[admin_command] pub(super) async fn get_remote_pdu_list( - &self, server: Box, force: bool, + &self, + server: Box, + force: bool, ) -> Result { if !self.services.globals.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( @@ -131,8 +145,8 @@ pub(super) async fn get_remote_pdu_list( if server == self.services.globals.server_name() { return Ok(RoomMessageEventContent::text_plain( - "Not allowed to send federation requests to ourselves. Please use `get-pdu` for fetching local PDUs from \ - the database.", + "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ + fetching local PDUs from the database.", )); } @@ -184,7 +198,9 @@ pub(super) async fn get_remote_pdu_list( #[admin_command] pub(super) async fn get_remote_pdu( - &self, event_id: Box, server: Box, + &self, + event_id: Box, + server: Box, ) -> Result { if !self.services.globals.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( @@ -194,30 +210,32 @@ pub(super) async fn get_remote_pdu( if server == self.services.globals.server_name() { return Ok(RoomMessageEventContent::text_plain( - "Not allowed to send federation requests to ourselves. Please use `get-pdu` for fetching local PDUs.", + "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ + fetching local PDUs.", )); } match self .services .sending - .send_federation_request( - &server, - ruma::api::federation::event::get_event::v1::Request { - event_id: event_id.clone().into(), - include_unredacted_content: None, - }, - ) + .send_federation_request(&server, ruma::api::federation::event::get_event::v1::Request { + event_id: event_id.clone().into(), + include_unredacted_content: None, + }) .await { - Ok(response) => { - let json: CanonicalJsonObject = serde_json::from_str(response.pdu.get()).map_err(|e| { - warn!( - "Requested event ID {event_id} from server but failed to convert from RawValue to \ - CanonicalJsonObject (malformed event/response?): {e}" - ); - Error::BadRequest(ErrorKind::Unknown, "Received response from server but failed to parse PDU") - })?; + | Ok(response) => { + let json: CanonicalJsonObject = + serde_json::from_str(response.pdu.get()).map_err(|e| { + warn!( + "Requested event ID {event_id} from server but failed to convert from \ + RawValue to CanonicalJsonObject (malformed event/response?): {e}" + ); + Error::BadRequest( + ErrorKind::Unknown, + "Received response from server but failed to parse PDU", + ) + })?; trace!("Attempting to parse PDU: {:?}", &response.pdu); let _parsed_pdu = { @@ -229,8 +247,8 @@ pub(super) async fn get_remote_pdu( .await; let (event_id, value, room_id) = match parsed_result { - Ok(t) => t, - Err(e) => { + | Ok(t) => t, + | Err(e) => { warn!("Failed to parse PDU: {e}"); info!("Full PDU: {:?}", &response.pdu); return Ok(RoomMessageEventContent::text_plain(format!( @@ -250,21 +268,27 @@ pub(super) async fn get_remote_pdu( .boxed() .await?; - let json_text = serde_json::to_string_pretty(&json).expect("canonical json is valid json"); + let json_text = + serde_json::to_string_pretty(&json).expect("canonical json is valid json"); Ok(RoomMessageEventContent::notice_markdown(format!( "{}\n```json\n{}\n```", - "Got PDU from specified server and handled as backfilled PDU successfully. Event body:", json_text + "Got PDU from specified server and handled as backfilled PDU successfully. \ + Event body:", + json_text ))) }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "Remote server did not have PDU or failed sending request to remote server: {e}" ))), } } #[admin_command] -pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result { +pub(super) async fn get_room_state( + &self, + room: OwnedRoomOrAliasId, +) -> Result { let room_id = self.services.rooms.alias.resolve(&room).await?; let room_state: Vec<_> = self .services @@ -285,7 +309,8 @@ pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result) -> Result { + | Ok(response) => { let ping_time = timer.elapsed(); let json_text_res = serde_json::to_string_pretty(&response.server); @@ -323,8 +351,11 @@ pub(super) async fn ping(&self, server: Box) -> Result { - warn!("Failed sending federation request to specified server from ping debug command: {e}"); + | Err(e) => { + warn!( + "Failed sending federation request to specified server from ping debug command: \ + {e}" + ); Ok(RoomMessageEventContent::text_plain(format!( "Failed sending federation request to specified server:\n\n{e}", ))) @@ -347,13 +378,17 @@ pub(super) async fn force_device_list_updates(&self) -> Result, reset: bool) -> Result { +pub(super) async fn change_log_level( + &self, + filter: Option, + reset: bool, +) -> Result { let handles = &["console"]; if reset { let old_filter_layer = match EnvFilter::try_new(&self.services.globals.config.log) { - Ok(s) => s, - Err(e) => { + | Ok(s) => s, + | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( "Log level from config appears to be invalid now: {e}" ))); @@ -367,13 +402,13 @@ pub(super) async fn change_log_level(&self, filter: Option, reset: bool) .reload .reload(&old_filter_layer, Some(handles)) { - Ok(()) => { + | Ok(()) => { return Ok(RoomMessageEventContent::text_plain(format!( "Successfully changed log level back to config value {}", self.services.globals.config.log ))); }, - Err(e) => { + | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( "Failed to modify and reload the global tracing log level: {e}" ))); @@ -383,8 +418,8 @@ pub(super) async fn change_log_level(&self, filter: Option, reset: bool) if let Some(filter) = filter { let new_filter_layer = match EnvFilter::try_new(filter) { - Ok(s) => s, - Err(e) => { + | Ok(s) => s, + | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( "Invalid log level filter specified: {e}" ))); @@ -398,10 +433,10 @@ pub(super) async fn change_log_level(&self, filter: Option, reset: bool) .reload .reload(&new_filter_layer, Some(handles)) { - Ok(()) => { + | Ok(()) => { return Ok(RoomMessageEventContent::text_plain("Successfully changed log level")); }, - Err(e) => { + | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( "Failed to modify and reload the global tracing log level: {e}" ))); @@ -414,7 +449,9 @@ pub(super) async fn change_log_level(&self, filter: Option, reset: bool) #[admin_command] pub(super) async fn sign_json(&self) -> Result { - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -423,21 +460,24 @@ pub(super) async fn sign_json(&self) -> Result { let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); match serde_json::from_str(&string) { - Ok(mut value) => { + | Ok(mut value) => { self.services .server_keys .sign_json(&mut value) .expect("our request json is what ruma expects"); - let json_text = serde_json::to_string_pretty(&value).expect("canonical json is valid json"); + let json_text = + serde_json::to_string_pretty(&value).expect("canonical json is valid json"); Ok(RoomMessageEventContent::text_plain(json_text)) }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), } } #[admin_command] pub(super) async fn verify_json(&self) -> Result { - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -446,13 +486,13 @@ pub(super) async fn verify_json(&self) -> Result { let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); match serde_json::from_str::(&string) { - Ok(value) => match self.services.server_keys.verify_json(&value, None).await { - Ok(()) => Ok(RoomMessageEventContent::text_plain("Signature correct")), - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + | Ok(value) => match self.services.server_keys.verify_json(&value, None).await { + | Ok(()) => Ok(RoomMessageEventContent::text_plain("Signature correct")), + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "Signature verification failed: {e}" ))), }, - Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), } } @@ -462,9 +502,10 @@ pub(super) async fn verify_pdu(&self, event_id: Box) -> Result "signatures OK, but content hash failed (redaction).", - Ok(ruma::signatures::Verified::All) => "signatures and hashes OK.", - Err(e) => return Err(e), + | Ok(ruma::signatures::Verified::Signatures) => + "signatures OK, but content hash failed (redaction).", + | Ok(ruma::signatures::Verified::All) => "signatures and hashes OK.", + | Err(e) => return Err(e), }; Ok(RoomMessageEventContent::notice_plain(msg)) @@ -472,7 +513,10 @@ pub(super) async fn verify_pdu(&self, event_id: Box) -> Result) -> Result { +pub(super) async fn first_pdu_in_room( + &self, + room_id: Box, +) -> Result { if !self .services .rooms @@ -498,7 +542,10 @@ pub(super) async fn first_pdu_in_room(&self, room_id: Box) -> Result) -> Result { +pub(super) async fn latest_pdu_in_room( + &self, + room_id: Box, +) -> Result { if !self .services .rooms @@ -525,7 +572,9 @@ pub(super) async fn latest_pdu_in_room(&self, room_id: Box) -> Result, server_name: Box, + &self, + room_id: Box, + server_name: Box, ) -> Result { if !self .services @@ -554,13 +603,10 @@ pub(super) async fn force_set_room_state_from_server( let remote_state_response = self .services .sending - .send_federation_request( - &server_name, - get_room_state::v1::Request { - room_id: room_id.clone().into(), - event_id: first_pdu.event_id.clone().into(), - }, - ) + .send_federation_request(&server_name, get_room_state::v1::Request { + room_id: room_id.clone().into(), + event_id: first_pdu.event_id.clone().into(), + }) .await?; for pdu in remote_state_response.pdus.clone() { @@ -571,8 +617,8 @@ pub(super) async fn force_set_room_state_from_server( .parse_incoming_pdu(&pdu) .await { - Ok(t) => t, - Err(e) => { + | Ok(t) => t, + | Err(e) => { warn!("Could not parse PDU, ignoring: {e}"); continue; }, @@ -654,8 +700,8 @@ pub(super) async fn force_set_room_state_from_server( .await?; info!( - "Updating joined counts for room just in case (e.g. we may have found a difference in the room's \ - m.room.member state" + "Updating joined counts for room just in case (e.g. we may have found a difference in \ + the room's m.room.member state" ); self.services .rooms @@ -672,9 +718,13 @@ pub(super) async fn force_set_room_state_from_server( #[admin_command] pub(super) async fn get_signing_keys( - &self, server_name: Option>, notary: Option>, query: bool, + &self, + server_name: Option>, + notary: Option>, + query: bool, ) -> Result { - let server_name = server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); + let server_name = + server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); if let Some(notary) = notary { let signing_keys = self @@ -706,8 +756,12 @@ pub(super) async fn get_signing_keys( } #[admin_command] -pub(super) async fn get_verify_keys(&self, server_name: Option>) -> Result { - let server_name = server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); +pub(super) async fn get_verify_keys( + &self, + server_name: Option>, +) -> Result { + let server_name = + server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); let keys = self .services @@ -727,7 +781,9 @@ pub(super) async fn get_verify_keys(&self, server_name: Option>) #[admin_command] pub(super) async fn resolve_true_destination( - &self, server_name: Box, no_cache: bool, + &self, + server_name: Box, + no_cache: bool, ) -> Result { if !self.services.globals.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( @@ -737,7 +793,8 @@ pub(super) async fn resolve_true_destination( if server_name == self.services.globals.config.server_name { return Ok(RoomMessageEventContent::text_plain( - "Not allowed to send federation requests to ourselves. Please use `get-pdu` for fetching local PDUs.", + "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ + fetching local PDUs.", )); } @@ -846,7 +903,9 @@ pub(super) async fn list_dependencies(&self, names: bool) -> Result, map: Option, + &self, + property: Option, + map: Option, ) -> Result { let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned()); let map_name = map.as_ref().map_or(EMPTY, String::as_str); diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index e0c1598d..75635b1b 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -2,7 +2,9 @@ use std::fmt::Write; use conduwuit::Result; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId, ServerName, UserId}; +use ruma::{ + events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId, ServerName, UserId, +}; use crate::{admin_command, get_room_info}; @@ -38,7 +40,10 @@ pub(super) async fn incoming_federation(&self) -> Result) -> Result { +pub(super) async fn fetch_support_well_known( + &self, + server_name: Box, +) -> Result { let response = self .services .client @@ -60,16 +65,20 @@ pub(super) async fn fetch_support_well_known(&self, server_name: Box } let json: serde_json::Value = match serde_json::from_str(&text) { - Ok(json) => json, - Err(_) => { - return Ok(RoomMessageEventContent::text_plain("Response text/body is not valid JSON.")); + | Ok(json) => json, + | Err(_) => { + return Ok(RoomMessageEventContent::text_plain( + "Response text/body is not valid JSON.", + )); }, }; let pretty_json: String = match serde_json::to_string_pretty(&json) { - Ok(json) => json, - Err(_) => { - return Ok(RoomMessageEventContent::text_plain("Response text/body is not valid JSON.")); + | Ok(json) => json, + | Err(_) => { + return Ok(RoomMessageEventContent::text_plain( + "Response text/body is not valid JSON.", + )); }, }; @@ -79,10 +88,14 @@ pub(super) async fn fetch_support_well_known(&self, server_name: Box } #[admin_command] -pub(super) async fn remote_user_in_rooms(&self, user_id: Box) -> Result { +pub(super) async fn remote_user_in_rooms( + &self, + user_id: Box, +) -> Result { if user_id.server_name() == self.services.globals.config.server_name { return Ok(RoomMessageEventContent::text_plain( - "User belongs to our server, please use `list-joined-rooms` user admin command instead.", + "User belongs to our server, please use `list-joined-rooms` user admin command \ + instead.", )); } diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 977ecdae..3d0a9473 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -1,16 +1,21 @@ use std::time::Duration; -use conduwuit::{debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, Result}; +use conduwuit::{ + debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, Result, +}; use conduwuit_service::media::Dim; use ruma::{ - events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, OwnedMxcUri, OwnedServerName, ServerName, + events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, OwnedMxcUri, + OwnedServerName, ServerName, }; use crate::{admin_command, utils::parse_local_user_id}; #[admin_command] pub(super) async fn delete( - &self, mxc: Option>, event_id: Option>, + &self, + mxc: Option>, + event_id: Option>, ) -> Result { if event_id.is_some() && mxc.is_some() { return Ok(RoomMessageEventContent::text_plain( @@ -52,7 +57,10 @@ pub(super) async fn delete( let final_url = url.to_string().replace('"', ""); mxc_urls.push(final_url); } else { - info!("Found a URL in the event ID {event_id} but did not start with mxc://, ignoring"); + info!( + "Found a URL in the event ID {event_id} but did not start with \ + mxc://, ignoring" + ); } } @@ -67,17 +75,24 @@ pub(super) async fn delete( debug!("Found a thumbnail_url in info key: {thumbnail_url}"); if thumbnail_url.to_string().starts_with("\"mxc://") { - debug!("Pushing thumbnail URL {thumbnail_url} to list of MXCs to delete"); - let final_thumbnail_url = thumbnail_url.to_string().replace('"', ""); + debug!( + "Pushing thumbnail URL {thumbnail_url} to list of MXCs \ + to delete" + ); + let final_thumbnail_url = + thumbnail_url.to_string().replace('"', ""); mxc_urls.push(final_thumbnail_url); } else { info!( - "Found a thumbnail URL in the event ID {event_id} but did not start with \ - mxc://, ignoring" + "Found a thumbnail URL in the event ID {event_id} but \ + did not start with mxc://, ignoring" ); } } else { - info!("No \"thumbnail_url\" key in \"info\" key, assuming no thumbnails."); + info!( + "No \"thumbnail_url\" key in \"info\" key, assuming no \ + thumbnails." + ); } } } @@ -98,8 +113,8 @@ pub(super) async fn delete( mxc_urls.push(final_url); } else { info!( - "Found a URL in the event ID {event_id} but did not start with mxc://, \ - ignoring" + "Found a URL in the event ID {event_id} but did not \ + start with mxc://, ignoring" ); } } else { @@ -109,13 +124,14 @@ pub(super) async fn delete( } } else { return Ok(RoomMessageEventContent::text_plain( - "Event ID does not have a \"content\" key or failed parsing the event ID JSON.", + "Event ID does not have a \"content\" key or failed parsing the event \ + ID JSON.", )); } } else { return Ok(RoomMessageEventContent::text_plain( - "Event ID does not have a \"content\" key, this is not a message or an event type that contains \ - media.", + "Event ID does not have a \"content\" key, this is not a message or an \ + event type that contains media.", )); } } else { @@ -126,7 +142,9 @@ pub(super) async fn delete( if mxc_urls.is_empty() { info!("Parsed event ID {event_id} but did not contain any MXC URLs."); - return Ok(RoomMessageEventContent::text_plain("Parsed event ID but found no MXC URLs.")); + return Ok(RoomMessageEventContent::text_plain( + "Parsed event ID but found no MXC URLs.", + )); } let mut mxc_deletion_count: usize = 0; @@ -138,11 +156,11 @@ pub(super) async fn delete( .delete(&mxc_url.as_str().try_into()?) .await { - Ok(()) => { + | Ok(()) => { debug_info!("Successfully deleted {mxc_url} from filesystem and database"); mxc_deletion_count = mxc_deletion_count.saturating_add(1); }, - Err(e) => { + | Err(e) => { debug_warn!("Failed to delete {mxc_url}, ignoring error and skipping: {e}"); continue; }, @@ -150,19 +168,22 @@ pub(super) async fn delete( } return Ok(RoomMessageEventContent::text_plain(format!( - "Deleted {mxc_deletion_count} total MXCs from our database and the filesystem from event ID {event_id}." + "Deleted {mxc_deletion_count} total MXCs from our database and the filesystem from \ + event ID {event_id}." ))); } Ok(RoomMessageEventContent::text_plain( - "Please specify either an MXC using --mxc or an event ID using --event-id of the message containing an image. \ - See --help for details.", + "Please specify either an MXC using --mxc or an event ID using --event-id of the \ + message containing an image. See --help for details.", )) } #[admin_command] pub(super) async fn delete_list(&self) -> Result { - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -192,11 +213,11 @@ pub(super) async fn delete_list(&self) -> Result { for mxc in &mxc_list { trace!(%failed_parsed_mxcs, %mxc_deletion_count, "Deleting MXC {mxc} in bulk"); match self.services.media.delete(mxc).await { - Ok(()) => { + | Ok(()) => { debug_info!("Successfully deleted {mxc} from filesystem and database"); mxc_deletion_count = mxc_deletion_count.saturating_add(1); }, - Err(e) => { + | Err(e) => { debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); continue; }, @@ -204,14 +225,18 @@ pub(super) async fn delete_list(&self) -> Result { } Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database and the filesystem. \ - {failed_parsed_mxcs} MXCs failed to be parsed from the database.", + "Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database \ + and the filesystem. {failed_parsed_mxcs} MXCs failed to be parsed from the database.", ))) } #[admin_command] pub(super) async fn delete_past_remote_media( - &self, duration: String, before: bool, after: bool, yes_i_want_to_delete_local_media: bool, + &self, + duration: String, + before: bool, + after: bool, + yes_i_want_to_delete_local_media: bool, ) -> Result { if before && after { return Ok(RoomMessageEventContent::text_plain( @@ -224,7 +249,12 @@ pub(super) async fn delete_past_remote_media( let deleted_count = self .services .media - .delete_all_remote_media_at_after_time(duration, before, after, yes_i_want_to_delete_local_media) + .delete_all_remote_media_at_after_time( + duration, + before, + after, + yes_i_want_to_delete_local_media, + ) .await?; Ok(RoomMessageEventContent::text_plain(format!( @@ -233,7 +263,10 @@ pub(super) async fn delete_past_remote_media( } #[admin_command] -pub(super) async fn delete_all_from_user(&self, username: String) -> Result { +pub(super) async fn delete_all_from_user( + &self, + username: String, +) -> Result { let user_id = parse_local_user_id(self.services, &username)?; let deleted_count = self.services.media.delete_from_user(&user_id).await?; @@ -245,7 +278,9 @@ pub(super) async fn delete_all_from_user(&self, username: String) -> Result, yes_i_want_to_delete_local_media: bool, + &self, + server_name: Box, + yes_i_want_to_delete_local_media: bool, ) -> Result { if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media { return Ok(RoomMessageEventContent::text_plain( @@ -260,20 +295,26 @@ pub(super) async fn delete_all_from_server( .await .inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}")) else { - return Ok(RoomMessageEventContent::text_plain("Failed to get MXC URIs from our database")); + return Ok(RoomMessageEventContent::text_plain( + "Failed to get MXC URIs from our database", + )); }; let mut deleted_count: usize = 0; for mxc in all_mxcs { let Ok(mxc_server_name) = mxc.server_name().inspect_err(|e| { - debug_warn!("Failed to parse MXC {mxc} server name from database, ignoring error and skipping: {e}"); + debug_warn!( + "Failed to parse MXC {mxc} server name from database, ignoring error and \ + skipping: {e}" + ); }) else { continue; }; if mxc_server_name != server_name - || (self.services.globals.server_is_ours(mxc_server_name) && !yes_i_want_to_delete_local_media) + || (self.services.globals.server_is_ours(mxc_server_name) + && !yes_i_want_to_delete_local_media) { trace!("skipping MXC URI {mxc}"); continue; @@ -282,10 +323,10 @@ pub(super) async fn delete_all_from_server( let mxc: Mxc<'_> = mxc.as_str().try_into()?; match self.services.media.delete(&mxc).await { - Ok(()) => { + | Ok(()) => { deleted_count = deleted_count.saturating_add(1); }, - Err(e) => { + | Err(e) => { debug_warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); continue; }, @@ -307,7 +348,10 @@ pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result, timeout: u32, + &self, + mxc: OwnedMxcUri, + server: Option, + timeout: u32, ) -> Result { let mxc: Mxc<'_> = mxc.as_str().try_into()?; let timeout = Duration::from_millis(timeout.into()); @@ -327,7 +371,12 @@ pub(super) async fn get_remote_file( #[admin_command] pub(super) async fn get_remote_thumbnail( - &self, mxc: OwnedMxcUri, server: Option, timeout: u32, width: u32, height: u32, + &self, + mxc: OwnedMxcUri, + server: Option, + timeout: u32, + width: u32, + height: u32, ) -> Result { let mxc: Mxc<'_> = mxc.as_str().try_into()?; let timeout = Duration::from_millis(timeout.into()); diff --git a/src/admin/processor.rs b/src/admin/processor.rs index e041a889..ed7d5ed1 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -53,8 +53,8 @@ async fn handle_command(services: Arc, command: CommandInput) -> Proce async fn process_command(services: Arc, input: &CommandInput) -> ProcessorResult { let (command, args, body) = match parse(&services, input) { - Err(error) => return Err(error), - Ok(parsed) => parsed, + | Err(error) => return Err(error), + | Ok(parsed) => parsed, }; let context = Command { @@ -68,7 +68,8 @@ async fn process_command(services: Arc, input: &CommandInput) -> Proce } fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult { - let link = "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; + let link = + "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}"); let content = RoomMessageEventContent::notice_markdown(msg); error!("Panic while processing command: {error:?}"); @@ -76,7 +77,11 @@ fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult { } // Parse and process a message from the admin room -async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) -> ProcessorResult { +async fn process( + context: &Command<'_>, + command: AdminCommand, + args: &[String], +) -> ProcessorResult { let (capture, logs) = capture_create(context); let capture_scope = capture.start(); @@ -100,11 +105,12 @@ async fn process(context: &Command<'_>, command: AdminCommand, args: &[String]) drop(logs); match result { - Ok(content) => { - write!(&mut output, "{0}", content.body()).expect("failed to format command result to output buffer"); + | Ok(content) => { + write!(&mut output, "{0}", content.body()) + .expect("failed to format command result to output buffer"); Ok(Some(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id))) }, - Err(error) => { + | Err(error) => { write!(&mut output, "Command failed with error:\n```\n{error:#?}\n```") .expect("failed to format command result to output"); Err(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id)) @@ -128,8 +134,9 @@ fn capture_create(context: &Command<'_>) -> (Arc, Arc>) { .and_then(LevelFilter::into_level) .unwrap_or(Level::DEBUG); - let filter = - move |data: capture::Data<'_>| data.level() <= log_level && data.our_modules() && data.scope.contains(&"admin"); + let filter = move |data: capture::Data<'_>| { + data.level() <= log_level && data.our_modules() && data.scope.contains(&"admin") + }; let logs = Arc::new(Mutex::new( collect_stream(|s| markdown_table_head(s)).expect("markdown table header"), @@ -146,14 +153,15 @@ fn capture_create(context: &Command<'_>) -> (Arc, Arc>) { // Parse chat messages from the admin room into an AdminCommand object fn parse<'a>( - services: &Arc, input: &'a CommandInput, + services: &Arc, + input: &'a CommandInput, ) -> Result<(AdminCommand, Vec, Vec<&'a str>), CommandOutput> { let lines = input.command.lines().filter(|line| !line.trim().is_empty()); let command_line = lines.clone().next().expect("command missing first line"); let body = lines.skip(1).collect(); match parse_command(command_line) { - Ok((command, args)) => Ok((command, args, body)), - Err(error) => { + | Ok((command, args)) => Ok((command, args, body)), + | Err(error) => { let message = error .to_string() .replace("server.name", services.globals.server_name().as_str()); @@ -255,11 +263,12 @@ fn parse_line(command_line: &str) -> Vec { argv } -fn reply(mut content: RoomMessageEventContent, reply_id: Option<&EventId>) -> RoomMessageEventContent { +fn reply( + mut content: RoomMessageEventContent, + reply_id: Option<&EventId>, +) -> RoomMessageEventContent { content.relates_to = reply_id.map(|event_id| Reply { - in_reply_to: InReplyTo { - event_id: event_id.to_owned(), - }, + in_reply_to: InReplyTo { event_id: event_id.to_owned() }, }); content diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index 53ed64dd..43762789 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -30,15 +30,14 @@ pub(crate) enum AccountDataCommand { } /// All the getters and iterators from src/database/key_value/account_data.rs -pub(super) async fn process(subcommand: AccountDataCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + subcommand: AccountDataCommand, + context: &Command<'_>, +) -> Result { let services = context.services; match subcommand { - AccountDataCommand::ChangesSince { - user_id, - since, - room_id, - } => { + | AccountDataCommand::ChangesSince { user_id, since, room_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .account_data @@ -51,11 +50,7 @@ pub(super) async fn process(subcommand: AccountDataCommand, context: &Command<'_ "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - AccountDataCommand::Get { - user_id, - kind, - room_id, - } => { + | AccountDataCommand::Get { user_id, kind, room_id } => { let timer = tokio::time::Instant::now(); let results = services .account_data diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 6bf9b9ad..fe4861bc 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -18,13 +18,14 @@ pub(crate) enum AppserviceCommand { } /// All the getters and iterators from src/database/key_value/appservice.rs -pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + subcommand: AppserviceCommand, + context: &Command<'_>, +) -> Result { let services = context.services; match subcommand { - AppserviceCommand::GetRegistration { - appservice_id, - } => { + | AppserviceCommand::GetRegistration { appservice_id } => { let timer = tokio::time::Instant::now(); let results = services.appservice.get_registration(&appservice_id).await; @@ -34,7 +35,7 @@ pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_> "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - AppserviceCommand::All => { + | AppserviceCommand::All => { let timer = tokio::time::Instant::now(); let results = services.appservice.all().await; let query_time = timer.elapsed(); diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 68f68648..e356453f 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -21,11 +21,14 @@ pub(crate) enum GlobalsCommand { } /// All the getters and iterators from src/database/key_value/globals.rs -pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + subcommand: GlobalsCommand, + context: &Command<'_>, +) -> Result { let services = context.services; match subcommand { - GlobalsCommand::DatabaseVersion => { + | GlobalsCommand::DatabaseVersion => { let timer = tokio::time::Instant::now(); let results = services.globals.db.database_version().await; let query_time = timer.elapsed(); @@ -34,7 +37,7 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - GlobalsCommand::CurrentCount => { + | GlobalsCommand::CurrentCount => { let timer = tokio::time::Instant::now(); let results = services.globals.db.current_count(); let query_time = timer.elapsed(); @@ -43,7 +46,7 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - GlobalsCommand::LastCheckForUpdatesId => { + | GlobalsCommand::LastCheckForUpdatesId => { let timer = tokio::time::Instant::now(); let results = services.updates.last_check_for_updates_id().await; let query_time = timer.elapsed(); @@ -52,9 +55,7 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - GlobalsCommand::SigningKeysFor { - origin, - } => { + | GlobalsCommand::SigningKeysFor { origin } => { let timer = tokio::time::Instant::now(); let results = services.server_keys.verify_keys_for(&origin).await; let query_time = timer.elapsed(); diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index b1849091..ab269a40 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -14,8 +14,9 @@ use conduwuit::Result; use self::{ account_data::AccountDataCommand, appservice::AppserviceCommand, globals::GlobalsCommand, - presence::PresenceCommand, pusher::PusherCommand, resolver::ResolverCommand, room_alias::RoomAliasCommand, - room_state_cache::RoomStateCacheCommand, sending::SendingCommand, users::UsersCommand, + presence::PresenceCommand, pusher::PusherCommand, resolver::ResolverCommand, + room_alias::RoomAliasCommand, room_state_cache::RoomStateCacheCommand, + sending::SendingCommand, users::UsersCommand, }; use crate::admin_command_dispatch; diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index 73e33bf6..45bb6bd9 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -23,13 +23,14 @@ pub(crate) enum PresenceCommand { } /// All the getters and iterators in key_value/presence.rs -pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + subcommand: PresenceCommand, + context: &Command<'_>, +) -> Result { let services = context.services; match subcommand { - PresenceCommand::GetPresence { - user_id, - } => { + | PresenceCommand::GetPresence { user_id } => { let timer = tokio::time::Instant::now(); let results = services.presence.db.get_presence(&user_id).await; let query_time = timer.elapsed(); @@ -38,9 +39,7 @@ pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - PresenceCommand::PresenceSince { - since, - } => { + | PresenceCommand::PresenceSince { since } => { let timer = tokio::time::Instant::now(); let results: Vec<(_, _, _)> = services .presence diff --git a/src/admin/query/pusher.rs b/src/admin/query/pusher.rs index 3004af8b..55532e54 100644 --- a/src/admin/query/pusher.rs +++ b/src/admin/query/pusher.rs @@ -13,13 +13,14 @@ pub(crate) enum PusherCommand { }, } -pub(super) async fn process(subcommand: PusherCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + subcommand: PusherCommand, + context: &Command<'_>, +) -> Result { let services = context.services; match subcommand { - PusherCommand::GetPushers { - user_id, - } => { + | PusherCommand::GetPushers { user_id } => { let timer = tokio::time::Instant::now(); let results = services.pusher.get_pushers(&user_id).await; let query_time = timer.elapsed(); diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index 43c63770..3b950d13 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -22,20 +22,16 @@ pub(crate) enum ResolverCommand { } #[admin_command] -async fn destinations_cache(&self, server_name: Option) -> Result { +async fn destinations_cache( + &self, + server_name: Option, +) -> Result { use service::resolver::cache::CachedDest; let mut out = String::new(); writeln!(out, "| Server Name | Destination | Hostname | Expires |")?; writeln!(out, "| ----------- | ----------- | -------- | ------- |")?; - let row = |( - name, - &CachedDest { - ref dest, - ref host, - expire, - }, - )| { + let row = |(name, &CachedDest { ref dest, ref host, expire })| { let expire = time::format(expire, "%+"); writeln!(out, "| {name} | {dest} | {host} | {expire} |").expect("wrote line"); }; @@ -64,14 +60,7 @@ async fn overrides_cache(&self, server_name: Option) -> Result) -> Result { +pub(super) async fn process( + subcommand: RoomAliasCommand, + context: &Command<'_>, +) -> Result { let services = context.services; match subcommand { - RoomAliasCommand::ResolveLocalAlias { - alias, - } => { + | RoomAliasCommand::ResolveLocalAlias { alias } => { let timer = tokio::time::Instant::now(); let results = services.rooms.alias.resolve_local_alias(&alias).await; let query_time = timer.elapsed(); @@ -39,9 +40,7 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomAliasCommand::LocalAliasesForRoom { - room_id, - } => { + | RoomAliasCommand::LocalAliasesForRoom { room_id } => { let timer = tokio::time::Instant::now(); let aliases: Vec<_> = services .rooms @@ -56,7 +55,7 @@ pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```" ))) }, - RoomAliasCommand::AllLocalAliases => { + | RoomAliasCommand::AllLocalAliases => { let timer = tokio::time::Instant::now(); let aliases = services .rooms diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index 6e84507a..cd7f5af7 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -77,15 +77,13 @@ pub(crate) enum RoomStateCacheCommand { } pub(super) async fn process( - subcommand: RoomStateCacheCommand, context: &Command<'_>, + subcommand: RoomStateCacheCommand, + context: &Command<'_>, ) -> Result { let services = context.services; match subcommand { - RoomStateCacheCommand::ServerInRoom { - server, - room_id, - } => { + | RoomStateCacheCommand::ServerInRoom { server, room_id } => { let timer = tokio::time::Instant::now(); let result = services .rooms @@ -98,9 +96,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" ))) }, - RoomStateCacheCommand::RoomServers { - room_id, - } => { + | RoomStateCacheCommand::RoomServers { room_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -115,9 +111,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::ServerRooms { - server, - } => { + | RoomStateCacheCommand::ServerRooms { server } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -132,9 +126,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::RoomMembers { - room_id, - } => { + | RoomStateCacheCommand::RoomMembers { room_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -149,9 +141,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::LocalUsersInRoom { - room_id, - } => { + | RoomStateCacheCommand::LocalUsersInRoom { room_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -166,9 +156,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::ActiveLocalUsersInRoom { - room_id, - } => { + | RoomStateCacheCommand::ActiveLocalUsersInRoom { room_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -183,9 +171,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::RoomJoinedCount { - room_id, - } => { + | RoomStateCacheCommand::RoomJoinedCount { room_id } => { let timer = tokio::time::Instant::now(); let results = services.rooms.state_cache.room_joined_count(&room_id).await; let query_time = timer.elapsed(); @@ -194,9 +180,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::RoomInvitedCount { - room_id, - } => { + | RoomStateCacheCommand::RoomInvitedCount { room_id } => { let timer = tokio::time::Instant::now(); let results = services .rooms @@ -209,9 +193,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::RoomUserOnceJoined { - room_id, - } => { + | RoomStateCacheCommand::RoomUserOnceJoined { room_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -226,9 +208,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::RoomMembersInvited { - room_id, - } => { + | RoomStateCacheCommand::RoomMembersInvited { room_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -243,10 +223,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::GetInviteCount { - room_id, - user_id, - } => { + | RoomStateCacheCommand::GetInviteCount { room_id, user_id } => { let timer = tokio::time::Instant::now(); let results = services .rooms @@ -259,10 +236,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::GetLeftCount { - room_id, - user_id, - } => { + | RoomStateCacheCommand::GetLeftCount { room_id, user_id } => { let timer = tokio::time::Instant::now(); let results = services .rooms @@ -275,9 +249,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::RoomsJoined { - user_id, - } => { + | RoomStateCacheCommand::RoomsJoined { user_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -292,9 +264,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::RoomsInvited { - user_id, - } => { + | RoomStateCacheCommand::RoomsInvited { user_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -308,9 +278,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::RoomsLeft { - user_id, - } => { + | RoomStateCacheCommand::RoomsLeft { user_id } => { let timer = tokio::time::Instant::now(); let results: Vec<_> = services .rooms @@ -324,10 +292,7 @@ pub(super) async fn process( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - RoomStateCacheCommand::InviteState { - user_id, - room_id, - } => { + | RoomStateCacheCommand::InviteState { user_id, room_id } => { let timer = tokio::time::Instant::now(); let results = services .rooms diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index efb4275f..696067b7 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -62,11 +62,14 @@ pub(crate) enum SendingCommand { } /// All the getters and iterators in key_value/sending.rs -pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + subcommand: SendingCommand, + context: &Command<'_>, +) -> Result { let services = context.services; match subcommand { - SendingCommand::ActiveRequests => { + | SendingCommand::ActiveRequests => { let timer = tokio::time::Instant::now(); let results = services.sending.db.active_requests(); let active_requests = results.collect::>().await; @@ -76,25 +79,29 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" ))) }, - SendingCommand::QueuedRequests { + | SendingCommand::QueuedRequests { appservice_id, server_name, user_id, push_key, } => { - if appservice_id.is_none() && server_name.is_none() && user_id.is_none() && push_key.is_none() { + if appservice_id.is_none() + && server_name.is_none() + && user_id.is_none() + && push_key.is_none() + { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via arguments. See \ - --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. See --help for more details.", )); } let timer = tokio::time::Instant::now(); let results = match (appservice_id, server_name, user_id, push_key) { - (Some(appservice_id), None, None, None) => { + | (Some(appservice_id), None, None, None) => { if appservice_id.is_empty() { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via \ - arguments. See --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. See --help for more details.", )); } @@ -103,15 +110,15 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - .db .queued_requests(&Destination::Appservice(appservice_id)) }, - (None, Some(server_name), None, None) => services + | (None, Some(server_name), None, None) => services .sending .db .queued_requests(&Destination::Normal(server_name.into())), - (None, None, Some(user_id), Some(push_key)) => { + | (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via \ - arguments. See --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. See --help for more details.", )); } @@ -120,16 +127,16 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - .db .queued_requests(&Destination::Push(user_id.into(), push_key)) }, - (Some(_), Some(_), Some(_), Some(_)) => { + | (Some(_), Some(_), Some(_), Some(_)) => { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via arguments. \ - Not all of them See --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. Not all of them See --help for more details.", )); }, - _ => { + | _ => { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via arguments. \ - See --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. See --help for more details.", )); }, }; @@ -141,26 +148,30 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - "Query completed in {query_time:?}:\n\n```rs\n{queued_requests:#?}\n```" ))) }, - SendingCommand::ActiveRequestsFor { + | SendingCommand::ActiveRequestsFor { appservice_id, server_name, user_id, push_key, } => { - if appservice_id.is_none() && server_name.is_none() && user_id.is_none() && push_key.is_none() { + if appservice_id.is_none() + && server_name.is_none() + && user_id.is_none() + && push_key.is_none() + { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via arguments. See \ - --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. See --help for more details.", )); } let timer = tokio::time::Instant::now(); let results = match (appservice_id, server_name, user_id, push_key) { - (Some(appservice_id), None, None, None) => { + | (Some(appservice_id), None, None, None) => { if appservice_id.is_empty() { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via \ - arguments. See --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. See --help for more details.", )); } @@ -169,15 +180,15 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - .db .active_requests_for(&Destination::Appservice(appservice_id)) }, - (None, Some(server_name), None, None) => services + | (None, Some(server_name), None, None) => services .sending .db .active_requests_for(&Destination::Normal(server_name.into())), - (None, None, Some(user_id), Some(push_key)) => { + | (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via \ - arguments. See --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. See --help for more details.", )); } @@ -186,16 +197,16 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - .db .active_requests_for(&Destination::Push(user_id.into(), push_key)) }, - (Some(_), Some(_), Some(_), Some(_)) => { + | (Some(_), Some(_), Some(_), Some(_)) => { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via arguments. \ - Not all of them See --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. Not all of them See --help for more details.", )); }, - _ => { + | _ => { return Ok(RoomMessageEventContent::text_plain( - "An appservice ID, server name, or a user ID with push key must be specified via arguments. \ - See --help for more details.", + "An appservice ID, server name, or a user ID with push key must be \ + specified via arguments. See --help for more details.", )); }, }; @@ -207,9 +218,7 @@ pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) - "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" ))) }, - SendingCommand::GetLatestEduCount { - server_name, - } => { + | SendingCommand::GetLatestEduCount { server_name } => { let timer = tokio::time::Instant::now(); let results = services.sending.db.get_latest_educount(&server_name).await; let query_time = timer.elapsed(); diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index d58e2d2a..2149a103 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -1,7 +1,9 @@ use clap::Subcommand; use conduwuit::Result; use futures::stream::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, OwnedDeviceId, OwnedRoomId, OwnedUserId}; +use ruma::{ + events::room::message::RoomMessageEventContent, OwnedDeviceId, OwnedRoomId, OwnedUserId, +}; use crate::{admin_command, admin_command_dispatch}; @@ -91,7 +93,11 @@ pub(crate) enum UsersCommand { #[admin_command] async fn get_backup_session( - &self, user_id: OwnedUserId, version: String, room_id: OwnedRoomId, session_id: String, + &self, + user_id: OwnedUserId, + version: String, + room_id: OwnedRoomId, + session_id: String, ) -> Result { let timer = tokio::time::Instant::now(); let result = self @@ -108,7 +114,10 @@ async fn get_backup_session( #[admin_command] async fn get_room_backups( - &self, user_id: OwnedUserId, version: String, room_id: OwnedRoomId, + &self, + user_id: OwnedUserId, + version: String, + room_id: OwnedRoomId, ) -> Result { let timer = tokio::time::Instant::now(); let result = self @@ -124,7 +133,11 @@ async fn get_room_backups( } #[admin_command] -async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result { +async fn get_all_backups( + &self, + user_id: OwnedUserId, + version: String, +) -> Result { let timer = tokio::time::Instant::now(); let result = self.services.key_backups.get_all(&user_id, &version).await; let query_time = timer.elapsed(); @@ -135,7 +148,11 @@ async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result } #[admin_command] -async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result { +async fn get_backup_algorithm( + &self, + user_id: OwnedUserId, + version: String, +) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -150,7 +167,10 @@ async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> R } #[admin_command] -async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result { +async fn get_latest_backup_version( + &self, + user_id: OwnedUserId, +) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -244,7 +264,11 @@ async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result Result { +async fn get_device_metadata( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, +) -> Result { let timer = tokio::time::Instant::now(); let device = self .services @@ -270,7 +294,11 @@ async fn get_devices_version(&self, user_id: OwnedUserId) -> Result Result { +async fn count_one_time_keys( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, +) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -285,7 +313,11 @@ async fn count_one_time_keys(&self, user_id: OwnedUserId, device_id: OwnedDevice } #[admin_command] -async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { +async fn get_device_keys( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, +) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -327,7 +359,9 @@ async fn get_master_key(&self, user_id: OwnedUserId) -> Result Result { let timer = tokio::time::Instant::now(); let result = self diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 841bc51a..835138b2 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -3,7 +3,10 @@ use std::fmt::Write; use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomAliasId, OwnedRoomId, RoomAliasId, RoomId}; +use ruma::{ + events::room::message::RoomMessageEventContent, OwnedRoomAliasId, OwnedRoomId, RoomAliasId, + RoomId, +}; use crate::{escape_html, Command}; @@ -42,82 +45,92 @@ pub(crate) enum RoomAliasCommand { }, } -pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + command: RoomAliasCommand, + context: &Command<'_>, +) -> Result { let services = context.services; let server_user = &services.globals.server_user; match command { - RoomAliasCommand::Set { - ref room_alias_localpart, - .. - } - | RoomAliasCommand::Remove { - ref room_alias_localpart, - } - | RoomAliasCommand::Which { - ref room_alias_localpart, - } => { - let room_alias_str = format!("#{}:{}", room_alias_localpart, services.globals.server_name()); + | RoomAliasCommand::Set { ref room_alias_localpart, .. } + | RoomAliasCommand::Remove { ref room_alias_localpart } + | RoomAliasCommand::Which { ref room_alias_localpart } => { + let room_alias_str = + format!("#{}:{}", room_alias_localpart, services.globals.server_name()); let room_alias = match RoomAliasId::parse_box(room_alias_str) { - Ok(alias) => alias, - Err(err) => return Ok(RoomMessageEventContent::text_plain(format!("Failed to parse alias: {err}"))), + | Ok(alias) => alias, + | Err(err) => + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to parse alias: {err}" + ))), }; match command { - RoomAliasCommand::Set { - force, - room_id, - .. - } => match (force, services.rooms.alias.resolve_local_alias(&room_alias).await) { - (true, Ok(id)) => match services - .rooms - .alias - .set_alias(&room_alias, &room_id, server_user) - { - Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( - "Successfully overwrote alias (formerly {id})" + | RoomAliasCommand::Set { force, room_id, .. } => + match (force, services.rooms.alias.resolve_local_alias(&room_alias).await) { + | (true, Ok(id)) => { + match services.rooms.alias.set_alias( + &room_alias, + &room_id, + server_user, + ) { + | Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( + "Successfully overwrote alias (formerly {id})" + ))), + | Err(err) => Ok(RoomMessageEventContent::text_plain(format!( + "Failed to remove alias: {err}" + ))), + } + }, + | (false, Ok(id)) => Ok(RoomMessageEventContent::text_plain(format!( + "Refusing to overwrite in use alias for {id}, use -f or --force to \ + overwrite" ))), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), + | (_, Err(_)) => { + match services.rooms.alias.set_alias( + &room_alias, + &room_id, + server_user, + ) { + | Ok(()) => Ok(RoomMessageEventContent::text_plain( + "Successfully set alias", + )), + | Err(err) => Ok(RoomMessageEventContent::text_plain(format!( + "Failed to remove alias: {err}" + ))), + } + }, }, - (false, Ok(id)) => Ok(RoomMessageEventContent::text_plain(format!( - "Refusing to overwrite in use alias for {id}, use -f or --force to overwrite" - ))), - (_, Err(_)) => match services - .rooms - .alias - .set_alias(&room_alias, &room_id, server_user) - { - Ok(()) => Ok(RoomMessageEventContent::text_plain("Successfully set alias")), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), + | RoomAliasCommand::Remove { .. } => + match services.rooms.alias.resolve_local_alias(&room_alias).await { + | Ok(id) => match services + .rooms + .alias + .remove_alias(&room_alias, server_user) + .await + { + | Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( + "Removed alias from {id}" + ))), + | Err(err) => Ok(RoomMessageEventContent::text_plain(format!( + "Failed to remove alias: {err}" + ))), + }, + | Err(_) => + Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), }, - }, - RoomAliasCommand::Remove { - .. - } => match services.rooms.alias.resolve_local_alias(&room_alias).await { - Ok(id) => match services - .rooms - .alias - .remove_alias(&room_alias, server_user) - .await - { - Ok(()) => Ok(RoomMessageEventContent::text_plain(format!("Removed alias from {id}"))), - Err(err) => Ok(RoomMessageEventContent::text_plain(format!("Failed to remove alias: {err}"))), + | RoomAliasCommand::Which { .. } => + match services.rooms.alias.resolve_local_alias(&room_alias).await { + | Ok(id) => Ok(RoomMessageEventContent::text_plain(format!( + "Alias resolves to {id}" + ))), + | Err(_) => + Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), }, - Err(_) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), - }, - RoomAliasCommand::Which { - .. - } => match services.rooms.alias.resolve_local_alias(&room_alias).await { - Ok(id) => Ok(RoomMessageEventContent::text_plain(format!("Alias resolves to {id}"))), - Err(_) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), - }, - RoomAliasCommand::List { - .. - } => unreachable!(), + | RoomAliasCommand::List { .. } => unreachable!(), } }, - RoomAliasCommand::List { - room_id, - } => { + | RoomAliasCommand::List { room_id } => if let Some(room_id) = room_id { let aliases: Vec = services .rooms @@ -128,7 +141,8 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> .await; let plain_list = aliases.iter().fold(String::new(), |mut output, alias| { - writeln!(output, "- {alias}").expect("should be able to write to string buffer"); + writeln!(output, "- {alias}") + .expect("should be able to write to string buffer"); output }); @@ -176,7 +190,6 @@ pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> let plain = format!("Aliases:\n{plain_list}"); let html = format!("Aliases:\n
      {html_list}
    "); Ok(RoomMessageEventContent::text_html(plain, html)) - } - }, + }, } } diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index d2fdbfca..b58d04c5 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -6,7 +6,11 @@ use crate::{admin_command, get_room_info, PAGE_SIZE}; #[admin_command] pub(super) async fn list_rooms( - &self, page: Option, exclude_disabled: bool, exclude_banned: bool, no_details: bool, + &self, + page: Option, + exclude_disabled: bool, + exclude_banned: bool, + no_details: bool, ) -> Result { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); @@ -16,10 +20,12 @@ pub(super) async fn list_rooms( .metadata .iter_ids() .filter_map(|room_id| async move { - (!exclude_disabled || !self.services.rooms.metadata.is_disabled(room_id).await).then_some(room_id) + (!exclude_disabled || !self.services.rooms.metadata.is_disabled(room_id).await) + .then_some(room_id) }) .filter_map(|room_id| async move { - (!exclude_banned || !self.services.rooms.metadata.is_banned(room_id).await).then_some(room_id) + (!exclude_banned || !self.services.rooms.metadata.is_banned(room_id).await) + .then_some(room_id) }) .then(|room_id| get_room_info(self.services, room_id)) .collect::>() diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index d3ed2575..81f25478 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -25,24 +25,21 @@ pub(crate) enum RoomDirectoryCommand { }, } -pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_>) -> Result { +pub(super) async fn process( + command: RoomDirectoryCommand, + context: &Command<'_>, +) -> Result { let services = context.services; match command { - RoomDirectoryCommand::Publish { - room_id, - } => { + | RoomDirectoryCommand::Publish { room_id } => { services.rooms.directory.set_public(&room_id); Ok(RoomMessageEventContent::notice_plain("Room published")) }, - RoomDirectoryCommand::Unpublish { - room_id, - } => { + | RoomDirectoryCommand::Unpublish { room_id } => { services.rooms.directory.set_not_public(&room_id); Ok(RoomMessageEventContent::notice_plain("Room unpublished")) }, - RoomDirectoryCommand::List { - page, - } => { + | RoomDirectoryCommand::List { page } => { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); let mut rooms: Vec<_> = services @@ -70,7 +67,9 @@ pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_> "Rooms (page {page}):\n```\n{}\n```", rooms .iter() - .map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}")) + .map(|(id, members, name)| format!( + "{id} | Members: {members} | Name: {name}" + )) .collect::>() .join("\n") ); diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index 6a6ec695..34abf8a9 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -27,7 +27,11 @@ pub(crate) enum RoomInfoCommand { } #[admin_command] -async fn list_joined_members(&self, room_id: Box, local_only: bool) -> Result { +async fn list_joined_members( + &self, + room_id: Box, + local_only: bool, +) -> Result { let room_name = self .services .rooms diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index ad05d16d..26d2c2d8 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -9,7 +9,8 @@ use conduwuit::Result; use ruma::OwnedRoomId; use self::{ - alias::RoomAliasCommand, directory::RoomDirectoryCommand, info::RoomInfoCommand, moderation::RoomModerationCommand, + alias::RoomAliasCommand, directory::RoomDirectoryCommand, info::RoomInfoCommand, + moderation::RoomModerationCommand, }; use crate::admin_command_dispatch; diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index 427f3e42..b16aff1f 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -6,7 +6,10 @@ use conduwuit::{ warn, Result, }; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId}; +use ruma::{ + events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, + RoomOrAliasId, +}; use crate::{admin_command, admin_command_dispatch, get_room_info}; @@ -75,7 +78,10 @@ pub(crate) enum RoomModerationCommand { #[admin_command] async fn ban_room( - &self, force: bool, disable_federation: bool, room: Box, + &self, + force: bool, + disable_federation: bool, + room: Box, ) -> Result { debug!("Got room alias or ID: {}", room); @@ -89,13 +95,13 @@ async fn ban_room( let room_id = if room.is_room_id() { let room_id = match RoomId::parse(&room) { - Ok(room_id) => room_id, - Err(e) => { + | Ok(room_id) => room_id, + | Err(e) => return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse room ID {room}. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" - ))) - }, + "Failed to parse room ID {room}. Please note that this requires a full room \ + ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`): {e}" + ))), }; debug!("Room specified is a room ID, banning room ID"); @@ -105,18 +111,18 @@ async fn ban_room( room_id } else if room.is_room_alias_id() { let room_alias = match RoomAliasId::parse(&room) { - Ok(room_alias) => room_alias, - Err(e) => { + | Ok(room_alias) => room_alias, + | Err(e) => return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse room ID {room}. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" - ))) - }, + "Failed to parse room ID {room}. Please note that this requires a full room \ + ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`): {e}" + ))), }; debug!( - "Room specified is not a room ID, attempting to resolve room alias to a room ID locally, if not using \ - get_alias_helper to fetch room ID remotely" + "Room specified is not a room ID, attempting to resolve room alias to a room ID \ + locally, if not using get_alias_helper to fetch room ID remotely" ); let room_id = if let Ok(room_id) = self @@ -128,7 +134,10 @@ async fn ban_room( { room_id } else { - debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); + debug!( + "We don't have this room alias to a room ID locally, attempting to fetch room \ + ID over federation" + ); match self .services @@ -137,11 +146,15 @@ async fn ban_room( .resolve_alias(&room_alias, None) .await { - Ok((room_id, servers)) => { - debug!(?room_id, ?servers, "Got federation response fetching room ID for {room}"); + | Ok((room_id, servers)) => { + debug!( + ?room_id, + ?servers, + "Got federation response fetching room ID for {room}" + ); room_id }, - Err(e) => { + | Err(e) => { return Ok(RoomMessageEventContent::notice_plain(format!( "Failed to resolve room alias {room} to a room ID: {e}" ))); @@ -154,8 +167,9 @@ async fn ban_room( room_id } else { return Ok(RoomMessageEventContent::text_plain( - "Room specified is not a room ID or room alias. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`)", + "Room specified is not a room ID or room alias. Please note that this requires a \ + full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`)", )); }; @@ -171,8 +185,8 @@ async fn ban_room( while let Some(local_user) = users.next().await { debug!( - "Attempting leave for user {local_user} in room {room_id} (forced, ignoring all errors, evicting \ - admins too)", + "Attempting leave for user {local_user} in room {room_id} (forced, ignoring all \ + errors, evicting admins too)", ); if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { @@ -196,12 +210,14 @@ async fn ban_room( debug!("Attempting leave for user {} in room {}", &local_user, &room_id); if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { error!( - "Error attempting to make local user {} leave room {} during room banning: {}", + "Error attempting to make local user {} leave room {} during room banning: \ + {}", &local_user, &room_id, e ); return Ok(RoomMessageEventContent::text_plain(format!( - "Error attempting to make local user {} leave room {} during room banning (room is still banned \ - but not removing any more users): {}\nIf you would like to ignore errors, use --force", + "Error attempting to make local user {} leave room {} during room banning \ + (room is still banned but not removing any more users): {}\nIf you would \ + like to ignore errors, use --force", &local_user, &room_id, e ))); } @@ -232,19 +248,26 @@ async fn ban_room( if disable_federation { self.services.rooms.metadata.disable_room(&room_id, true); return Ok(RoomMessageEventContent::text_plain( - "Room banned, removed all our local users, and disabled incoming federation with room.", + "Room banned, removed all our local users, and disabled incoming federation with \ + room.", )); } Ok(RoomMessageEventContent::text_plain( - "Room banned and removed all our local users, use `!admin federation disable-room` to stop receiving new \ - inbound federation events as well if needed.", + "Room banned and removed all our local users, use `!admin federation disable-room` to \ + stop receiving new inbound federation events as well if needed.", )) } #[admin_command] -async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Result { - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" +async fn ban_list_of_rooms( + &self, + force: bool, + disable_federation: bool, +) -> Result { + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -264,9 +287,10 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu for &room in &rooms_s { match <&RoomOrAliasId>::try_from(room) { - Ok(room_alias_or_id) => { + | Ok(room_alias_or_id) => { if let Ok(admin_room_id) = self.services.admin.get_admin_room().await { - if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias) { + if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias) + { info!("User specified admin room in bulk ban list, ignoring"); continue; } @@ -274,19 +298,20 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu if room_alias_or_id.is_room_id() { let room_id = match RoomId::parse(room_alias_or_id) { - Ok(room_id) => room_id, - Err(e) => { + | Ok(room_id) => room_id, + | Err(e) => { if force { // ignore rooms we failed to parse if we're force banning warn!( - "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ - logging here: {e}" + "Error parsing room \"{room}\" during bulk room banning, \ + ignoring error and logging here: {e}" ); continue; } return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try again: {e}" + "{room} is not a valid room ID or room alias, please fix the \ + list and try again: {e}" ))); }, }; @@ -296,7 +321,7 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu if room_alias_or_id.is_room_alias_id() { match RoomAliasId::parse(room_alias_or_id) { - Ok(room_alias) => { + | Ok(room_alias) => { let room_id = if let Ok(room_id) = self .services .rooms @@ -307,8 +332,8 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu room_id } else { debug!( - "We don't have this room alias to a room ID locally, attempting to fetch room ID \ - over federation" + "We don't have this room alias to a room ID locally, \ + attempting to fetch room ID over federation" ); match self @@ -318,7 +343,7 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu .resolve_alias(&room_alias, None) .await { - Ok((room_id, servers)) => { + | Ok((room_id, servers)) => { debug!( ?room_id, ?servers, @@ -326,15 +351,19 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu ); room_id }, - Err(e) => { + | Err(e) => { // don't fail if force blocking if force { - warn!("Failed to resolve room alias {room} to a room ID: {e}"); + warn!( + "Failed to resolve room alias {room} to a room \ + ID: {e}" + ); continue; } return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to resolve room alias {room} to a room ID: {e}" + "Failed to resolve room alias {room} to a room ID: \ + {e}" ))); }, } @@ -342,34 +371,37 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu room_ids.push(room_id); }, - Err(e) => { + | Err(e) => { if force { // ignore rooms we failed to parse if we're force deleting error!( - "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ - logging here: {e}" + "Error parsing room \"{room}\" during bulk room banning, \ + ignoring error and logging here: {e}" ); continue; } return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try again: {e}" + "{room} is not a valid room ID or room alias, please fix the \ + list and try again: {e}" ))); }, } } }, - Err(e) => { + | Err(e) => { if force { // ignore rooms we failed to parse if we're force deleting error!( - "Error parsing room \"{room}\" during bulk room banning, ignoring error and logging here: {e}" + "Error parsing room \"{room}\" during bulk room banning, ignoring error \ + and logging here: {e}" ); continue; } return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try again: {e}" + "{room} is not a valid room ID or room alias, please fix the list and try \ + again: {e}" ))); }, } @@ -393,8 +425,8 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu while let Some(local_user) = users.next().await { debug!( - "Attempting leave for user {local_user} in room {room_id} (forced, ignoring all errors, evicting \ - admins too)", + "Attempting leave for user {local_user} in room {room_id} (forced, ignoring \ + all errors, evicting admins too)", ); if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { @@ -418,14 +450,15 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu debug!("Attempting leave for user {local_user} in room {room_id}"); if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { error!( - "Error attempting to make local user {local_user} leave room {room_id} during bulk room \ - banning: {e}", + "Error attempting to make local user {local_user} leave room {room_id} \ + during bulk room banning: {e}", ); return Ok(RoomMessageEventContent::text_plain(format!( - "Error attempting to make local user {} leave room {} during room banning (room is still \ - banned but not removing any more users and not banning any more rooms): {}\nIf you would \ - like to ignore errors, use --force", + "Error attempting to make local user {} leave room {} during room \ + banning (room is still banned but not removing any more users and not \ + banning any more rooms): {}\nIf you would like to ignore errors, use \ + --force", &local_user, &room_id, e ))); } @@ -458,8 +491,8 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu if disable_federation { Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and disabled incoming \ - federation with the room." + "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, \ + and disabled incoming federation with the room." ))) } else { Ok(RoomMessageEventContent::text_plain(format!( @@ -469,16 +502,20 @@ async fn ban_list_of_rooms(&self, force: bool, disable_federation: bool) -> Resu } #[admin_command] -async fn unban_room(&self, enable_federation: bool, room: Box) -> Result { +async fn unban_room( + &self, + enable_federation: bool, + room: Box, +) -> Result { let room_id = if room.is_room_id() { let room_id = match RoomId::parse(&room) { - Ok(room_id) => room_id, - Err(e) => { + | Ok(room_id) => room_id, + | Err(e) => return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse room ID {room}. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" - ))) - }, + "Failed to parse room ID {room}. Please note that this requires a full room \ + ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`): {e}" + ))), }; debug!("Room specified is a room ID, unbanning room ID"); @@ -488,18 +525,18 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> room_id } else if room.is_room_alias_id() { let room_alias = match RoomAliasId::parse(&room) { - Ok(room_alias) => room_alias, - Err(e) => { + | Ok(room_alias) => room_alias, + | Err(e) => return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse room ID {room}. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`): {e}" - ))) - }, + "Failed to parse room ID {room}. Please note that this requires a full room \ + ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`): {e}" + ))), }; debug!( - "Room specified is not a room ID, attempting to resolve room alias to a room ID locally, if not using \ - get_alias_helper to fetch room ID remotely" + "Room specified is not a room ID, attempting to resolve room alias to a room ID \ + locally, if not using get_alias_helper to fetch room ID remotely" ); let room_id = if let Ok(room_id) = self @@ -511,7 +548,10 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> { room_id } else { - debug!("We don't have this room alias to a room ID locally, attempting to fetch room ID over federation"); + debug!( + "We don't have this room alias to a room ID locally, attempting to fetch room \ + ID over federation" + ); match self .services @@ -520,11 +560,15 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> .resolve_alias(&room_alias, None) .await { - Ok((room_id, servers)) => { - debug!(?room_id, ?servers, "Got federation response fetching room ID for room {room}"); + | Ok((room_id, servers)) => { + debug!( + ?room_id, + ?servers, + "Got federation response fetching room ID for room {room}" + ); room_id }, - Err(e) => { + | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( "Failed to resolve room alias {room} to a room ID: {e}" ))); @@ -537,8 +581,9 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> room_id } else { return Ok(RoomMessageEventContent::text_plain( - "Room specified is not a room ID or room alias. Please note that this requires a full room ID \ - (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias (`#roomalias:example.com`)", + "Room specified is not a room ID or room alias. Please note that this requires a \ + full room ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ + (`#roomalias:example.com`)", )); }; @@ -548,8 +593,8 @@ async fn unban_room(&self, enable_federation: bool, room: Box) -> } Ok(RoomMessageEventContent::text_plain( - "Room unbanned, you may need to re-enable federation with the room using enable-room if this is a remote room \ - to make it fully functional.", + "Room unbanned, you may need to re-enable federation with the room using enable-room if \ + this is a remote room to make it fully functional.", )) } diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 676f2d33..8d3358a8 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -29,13 +29,12 @@ pub(super) async fn show_config(&self) -> Result { #[admin_command] pub(super) async fn list_features( - &self, available: bool, enabled: bool, comma: bool, + &self, + available: bool, + enabled: bool, + comma: bool, ) -> Result { - let delim = if comma { - "," - } else { - " " - }; + let delim = if comma { "," } else { " " }; if enabled && !available { let features = info::rustc::features().join(delim); let out = format!("`\n{features}\n`"); @@ -53,16 +52,8 @@ pub(super) async fn list_features( let available = info::cargo::features(); for feature in available { let active = enabled.contains(&feature.as_str()); - let emoji = if active { - "✅" - } else { - "❌" - }; - let remark = if active { - "[enabled]" - } else { - "" - }; + let emoji = if active { "✅" } else { "❌" }; + let remark = if active { "[enabled]" } else { "" }; writeln!(features, "{emoji} {feature} {remark}")?; } @@ -73,7 +64,8 @@ pub(super) async fn list_features( pub(super) async fn memory_usage(&self) -> Result { let services_usage = self.services.memory_usage().await?; let database_usage = self.services.db.db.memory_usage()?; - let allocator_usage = conduwuit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}")); + let allocator_usage = + conduwuit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}")); Ok(RoomMessageEventContent::text_plain(format!( "Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}", @@ -106,8 +98,8 @@ pub(super) async fn backup_database(&self) -> Result { .server .runtime() .spawn_blocking(move || match globals.db.backup() { - Ok(()) => String::new(), - Err(e) => e.to_string(), + | Ok(()) => String::new(), + | Err(e) => e.to_string(), }) .await?; @@ -147,8 +139,8 @@ pub(super) async fn restart(&self, force: bool) -> Result Result { } #[admin_command] -pub(super) async fn create_user(&self, username: String, password: Option) -> Result { +pub(super) async fn create_user( + &self, + username: String, + password: Option, +) -> Result { // Validate user id let user_id = parse_local_user_id(self.services, &username)?; if self.services.users.exists(&user_id).await { - return Ok(RoomMessageEventContent::text_plain(format!("Userid {user_id} already exists"))); + return Ok(RoomMessageEventContent::text_plain(format!( + "Userid {user_id} already exists" + ))); } if user_id.is_historical() { @@ -120,7 +126,9 @@ pub(super) async fn create_user(&self, username: String, password: Option { + | Ok(_response) => { info!("Automatically joined room {room} for user {user_id}"); }, - Err(e) => { + | Err(e) => { self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( - "Failed to automatically join room {room} for user {user_id}: {e}" + "Failed to automatically join room {room} for user {user_id}: \ + {e}" ))) .await .ok(); // don't return this error so we don't fail registrations - error!("Failed to automatically join room {room} for user {user_id}: {e}"); + error!( + "Failed to automatically join room {room} for user {user_id}: {e}" + ); }, }; } @@ -181,7 +195,11 @@ pub(super) async fn create_user(&self, username: String, password: Option Result { +pub(super) async fn deactivate( + &self, + no_leave_rooms: bool, + user_id: String, +) -> Result { // Validate user id let user_id = parse_local_user_id(self.services, &user_id)?; @@ -229,7 +247,8 @@ pub(super) async fn reset_password(&self, username: String) -> Result Result Ok(RoomMessageEventContent::text_plain(format!( + | Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( "Successfully reset the password for user {user_id}: `{new_password}`" ))), - Err(e) => Ok(RoomMessageEventContent::text_plain(format!( + | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( "Couldn't reset the password for user {user_id}: {e}" ))), } } #[admin_command] -pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> Result { - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" +pub(super) async fn deactivate_all( + &self, + no_leave_rooms: bool, + force: bool, +) -> Result { + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -269,7 +294,7 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> for username in usernames { match parse_active_local_user_id(self.services, username).await { - Ok(user_id) => { + | Ok(user_id) => { if self.services.users.is_admin(&user_id).await && !force { self.services .admin @@ -296,7 +321,7 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> user_ids.push(user_id); }, - Err(e) => { + | Err(e) => { self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( @@ -313,7 +338,7 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> for user_id in user_ids { match self.services.users.deactivate_account(&user_id).await { - Ok(()) => { + | Ok(()) => { deactivation_count = deactivation_count.saturating_add(1); if !no_leave_rooms { info!("Forcing user {user_id} to leave all rooms apart of deactivate-all"); @@ -328,14 +353,17 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> full_user_deactivate(self.services, &user_id, &all_joined_rooms).await?; update_displayname(self.services, &user_id, None, &all_joined_rooms).await; - update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms).await; + update_avatar_url(self.services, &user_id, None, None, &all_joined_rooms) + .await; leave_all_rooms(self.services, &user_id).await; } }, - Err(e) => { + | Err(e) => { self.services .admin - .send_message(RoomMessageEventContent::text_plain(format!("Failed deactivating user: {e}"))) + .send_message(RoomMessageEventContent::text_plain(format!( + "Failed deactivating user: {e}" + ))) .await .ok(); }, @@ -348,8 +376,8 @@ pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> ))) } else { Ok(RoomMessageEventContent::text_plain(format!( - "Deactivated {deactivation_count} accounts.\nSkipped admin accounts: {}. Use --force to deactivate admin \ - accounts", + "Deactivated {deactivation_count} accounts.\nSkipped admin accounts: {}. Use \ + --force to deactivate admin accounts", admins.join(", ") ))) } @@ -391,9 +419,13 @@ pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result Result { - if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" + if self.body.len() < 2 + || !self.body[0].trim().starts_with("```") + || self.body.last().unwrap_or(&"").trim() != "```" { return Ok(RoomMessageEventContent::text_plain( "Expected code block in command body. Add --help for details.", @@ -402,8 +434,8 @@ pub(super) async fn force_join_list_of_local_users( if !yes_i_want_to_do_this { return Ok(RoomMessageEventContent::notice_markdown( - "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force bulk join all \ - specified local users.", + "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force \ + bulk join all specified local users.", )); } @@ -462,7 +494,7 @@ pub(super) async fn force_join_list_of_local_users( for username in usernames { match parse_active_local_user_id(self.services, username).await { - Ok(user_id) => { + | Ok(user_id) => { // don't make the server service account join if user_id == self.services.globals.server_user { self.services @@ -477,7 +509,7 @@ pub(super) async fn force_join_list_of_local_users( user_ids.push(user_id); }, - Err(e) => { + | Err(e) => { self.services .admin .send_message(RoomMessageEventContent::text_plain(format!( @@ -505,10 +537,10 @@ pub(super) async fn force_join_list_of_local_users( ) .await { - Ok(_res) => { + | Ok(_res) => { successful_joins = successful_joins.saturating_add(1); }, - Err(e) => { + | Err(e) => { debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}"); failed_joins = failed_joins.saturating_add(1); }, @@ -516,18 +548,21 @@ pub(super) async fn force_join_list_of_local_users( } Ok(RoomMessageEventContent::notice_markdown(format!( - "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins failed.", + "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins \ + failed.", ))) } #[admin_command] pub(super) async fn force_join_all_local_users( - &self, room_id: OwnedRoomOrAliasId, yes_i_want_to_do_this: bool, + &self, + room_id: OwnedRoomOrAliasId, + yes_i_want_to_do_this: bool, ) -> Result { if !yes_i_want_to_do_this { return Ok(RoomMessageEventContent::notice_markdown( - "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force bulk join all local \ - users.", + "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force \ + bulk join all local users.", )); } @@ -598,10 +633,10 @@ pub(super) async fn force_join_all_local_users( ) .await { - Ok(_res) => { + | Ok(_res) => { successful_joins = successful_joins.saturating_add(1); }, - Err(e) => { + | Err(e) => { debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}"); failed_joins = failed_joins.saturating_add(1); }, @@ -609,13 +644,16 @@ pub(super) async fn force_join_all_local_users( } Ok(RoomMessageEventContent::notice_markdown(format!( - "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins failed.", + "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins \ + failed.", ))) } #[admin_command] pub(super) async fn force_join_room( - &self, user_id: String, room_id: OwnedRoomOrAliasId, + &self, + user_id: String, + room_id: OwnedRoomOrAliasId, ) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; let (room_id, servers) = self @@ -629,7 +667,8 @@ pub(super) async fn force_join_room( self.services.globals.user_is_local(&user_id), "Parsed user_id must be a local user" ); - join_room_by_id_helper(self.services, &user_id, &room_id, None, &servers, None, &None).await?; + join_room_by_id_helper(self.services, &user_id, &room_id, None, &servers, None, &None) + .await?; Ok(RoomMessageEventContent::notice_markdown(format!( "{user_id} has been joined to {room_id}.", @@ -638,7 +677,9 @@ pub(super) async fn force_join_room( #[admin_command] pub(super) async fn force_leave_room( - &self, user_id: String, room_id: OwnedRoomOrAliasId, + &self, + user_id: String, + room_id: OwnedRoomOrAliasId, ) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; let room_id = self.services.rooms.alias.resolve(&room_id).await?; @@ -656,7 +697,9 @@ pub(super) async fn force_leave_room( #[admin_command] pub(super) async fn force_demote( - &self, user_id: String, room_id: OwnedRoomOrAliasId, + &self, + user_id: String, + room_id: OwnedRoomOrAliasId, ) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; let room_id = self.services.rooms.alias.resolve(&room_id).await?; @@ -672,14 +715,19 @@ pub(super) async fn force_demote( .services .rooms .state_accessor - .room_state_get_content::(&room_id, &StateEventType::RoomPowerLevels, "") + .room_state_get_content::( + &room_id, + &StateEventType::RoomPowerLevels, + "", + ) .await .ok(); let user_can_demote_self = room_power_levels .as_ref() .is_some_and(|power_levels_content| { - RoomPowerLevels::from(power_levels_content.clone()).user_can_change_user_power_level(&user_id, &user_id) + RoomPowerLevels::from(power_levels_content.clone()) + .user_can_change_user_power_level(&user_id, &user_id) }) || self .services .rooms @@ -710,7 +758,8 @@ pub(super) async fn force_demote( .await?; Ok(RoomMessageEventContent::notice_markdown(format!( - "User {user_id} demoted themselves to the room default power level in {room_id} - {event_id}" + "User {user_id} demoted themselves to the room default power level in {room_id} - \ + {event_id}" ))) } @@ -731,7 +780,10 @@ pub(super) async fn make_user_admin(&self, user_id: String) -> Result, tag: String, + &self, + user_id: String, + room_id: Box, + tag: String, ) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; @@ -741,9 +793,7 @@ pub(super) async fn put_room_tag( .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) .await .unwrap_or(TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, + content: TagEventContent { tags: BTreeMap::new() }, }); tags_event @@ -768,7 +818,10 @@ pub(super) async fn put_room_tag( #[admin_command] pub(super) async fn delete_room_tag( - &self, user_id: String, room_id: Box, tag: String, + &self, + user_id: String, + room_id: Box, + tag: String, ) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; @@ -778,9 +831,7 @@ pub(super) async fn delete_room_tag( .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) .await .unwrap_or(TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, + content: TagEventContent { tags: BTreeMap::new() }, }); tags_event.content.tags.remove(&tag.clone().into()); @@ -796,12 +847,17 @@ pub(super) async fn delete_room_tag( .await?; Ok(RoomMessageEventContent::text_plain(format!( - "Successfully updated room account data for {user_id} and room {room_id}, deleting room tag {tag}" + "Successfully updated room account data for {user_id} and room {room_id}, deleting room \ + tag {tag}" ))) } #[admin_command] -pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box) -> Result { +pub(super) async fn get_room_tags( + &self, + user_id: String, + room_id: Box, +) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; let tags_event = self @@ -810,9 +866,7 @@ pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box) .get_room(&room_id, &user_id, RoomAccountDataEventType::Tag) .await .unwrap_or(TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, + content: TagEventContent { tags: BTreeMap::new() }, }); Ok(RoomMessageEventContent::notice_markdown(format!( @@ -822,7 +876,10 @@ pub(super) async fn get_room_tags(&self, user_id: String, room_id: Box) } #[admin_command] -pub(super) async fn redact_event(&self, event_id: Box) -> Result { +pub(super) async fn redact_event( + &self, + event_id: Box, +) -> Result { let Ok(event) = self .services .rooms @@ -841,7 +898,9 @@ pub(super) async fn redact_event(&self, event_id: Box) -> Result String { .replace('>', ">") } -pub(crate) async fn get_room_info(services: &Services, room_id: &RoomId) -> (OwnedRoomId, u64, String) { +pub(crate) async fn get_room_info( + services: &Services, + room_id: &RoomId, +) -> (OwnedRoomId, u64, String) { ( room_id.into(), services @@ -44,7 +47,10 @@ pub(crate) fn parse_local_user_id(services: &Services, user_id: &str) -> Result< } /// Parses user ID that is an active (not guest or deactivated) local user -pub(crate) async fn parse_active_local_user_id(services: &Services, user_id: &str) -> Result { +pub(crate) async fn parse_active_local_user_id( + services: &Services, + user_id: &str, +) -> Result { let user_id = parse_local_user_id(services, user_id)?; if !services.users.exists(&user_id).await { diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 71e18fd3..e6748124 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -2,16 +2,19 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{debug_info, error, info, is_equal_to, utils, utils::ReadyExt, warn, Error, PduBuilder, Result}; +use conduwuit::{ + debug_info, error, info, is_equal_to, utils, utils::ReadyExt, warn, Error, PduBuilder, Result, +}; use futures::{FutureExt, StreamExt}; use register::RegistrationKind; use ruma::{ api::client::{ account::{ - change_password, check_registration_token_validity, deactivate, get_3pids, get_username_availability, + change_password, check_registration_token_validity, deactivate, get_3pids, + get_username_availability, register::{self, LoginType}, - request_3pid_management_token_via_email, request_3pid_management_token_via_msisdn, whoami, - ThirdPartyIdRemovalStatus, + request_3pid_management_token_via_email, request_3pid_management_token_via_msisdn, + whoami, ThirdPartyIdRemovalStatus, }, error::ErrorKind, uiaa::{AuthFlow, AuthType, UiaaInfo}, @@ -45,7 +48,8 @@ const RANDOM_USER_ID_LENGTH: usize = 10; /// invalid when trying to register #[tracing::instrument(skip_all, fields(%client), name = "register_available")] pub(crate) async fn get_register_available_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { // workaround for https://github.com/matrix-org/matrix-appservice-irc/issues/1780 due to inactivity of fixing the issue @@ -66,7 +70,8 @@ pub(crate) async fn get_register_available_route( let user_id = UserId::parse_with_server_name(body_username, services.globals.server_name()) .ok() .filter(|user_id| { - (!user_id.is_historical() || is_matrix_appservice_irc) && services.globals.user_is_local(user_id) + (!user_id.is_historical() || is_matrix_appservice_irc) + && services.globals.user_is_local(user_id) }) .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; @@ -86,9 +91,7 @@ pub(crate) async fn get_register_available_route( // TODO add check for appservice namespaces // If no if check is true we have an username that's available to be used. - Ok(get_username_availability::v3::Response { - available: true, - }) + Ok(get_username_availability::v3::Response { available: true }) } /// # `POST /_matrix/client/v3/register` @@ -111,12 +114,14 @@ pub(crate) async fn get_register_available_route( #[allow(clippy::doc_markdown)] #[tracing::instrument(skip_all, fields(%client), name = "register")] pub(crate) async fn register_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { if !services.globals.allow_registration() && body.appservice_info.is_none() { info!( - "Registration disabled and request not from known appservice, rejecting registration attempt for username \ - \"{}\"", + "Registration disabled and request not from known appservice, rejecting \ + registration attempt for username \"{}\"", body.username.as_deref().unwrap_or("") ); return Err(Error::BadRequest(ErrorKind::forbidden(), "Registration has been disabled.")); @@ -126,11 +131,12 @@ pub(crate) async fn register_route( if is_guest && (!services.globals.allow_guest_registration() - || (services.globals.allow_registration() && services.globals.registration_token.is_some())) + || (services.globals.allow_registration() + && services.globals.registration_token.is_some())) { info!( - "Guest registration disabled / registration enabled with token configured, rejecting guest registration \ - attempt, initial device name: \"{}\"", + "Guest registration disabled / registration enabled with token configured, \ + rejecting guest registration attempt, initial device name: \"{}\"", body.initial_device_display_name.as_deref().unwrap_or("") ); return Err(Error::BadRequest( @@ -143,21 +149,25 @@ pub(crate) async fn register_route( // generic user error. if is_guest && services.users.count().await < 2 { warn!( - "Guest account attempted to register before a real admin user has been registered, rejecting \ - registration. Guest's initial device name: \"{}\"", + "Guest account attempted to register before a real admin user has been registered, \ + rejecting registration. Guest's initial device name: \"{}\"", body.initial_device_display_name.as_deref().unwrap_or("") ); - return Err(Error::BadRequest(ErrorKind::forbidden(), "Registration temporarily disabled.")); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Registration temporarily disabled.", + )); } let user_id = match (&body.username, is_guest) { - (Some(username), false) => { + | (Some(username), false) => { // workaround for https://github.com/matrix-org/matrix-appservice-irc/issues/1780 due to inactivity of fixing the issue - let is_matrix_appservice_irc = body.appservice_info.as_ref().is_some_and(|appservice| { - appservice.registration.id == "irc" - || appservice.registration.id.contains("matrix-appservice-irc") - || appservice.registration.id.contains("matrix_appservice_irc") - }); + let is_matrix_appservice_irc = + body.appservice_info.as_ref().is_some_and(|appservice| { + appservice.registration.id == "irc" + || appservice.registration.id.contains("matrix-appservice-irc") + || appservice.registration.id.contains("matrix_appservice_irc") + }); // don't force the username lowercase if it's from matrix-appservice-irc let body_username = if is_matrix_appservice_irc { @@ -166,15 +176,23 @@ pub(crate) async fn register_route( username.to_lowercase() }; - let proposed_user_id = UserId::parse_with_server_name(body_username, services.globals.server_name()) - .ok() - .filter(|user_id| { - (!user_id.is_historical() || is_matrix_appservice_irc) && services.globals.user_is_local(user_id) - }) - .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; + let proposed_user_id = + UserId::parse_with_server_name(body_username, services.globals.server_name()) + .ok() + .filter(|user_id| { + (!user_id.is_historical() || is_matrix_appservice_irc) + && services.globals.user_is_local(user_id) + }) + .ok_or(Error::BadRequest( + ErrorKind::InvalidUsername, + "Username is invalid.", + ))?; if services.users.exists(&proposed_user_id).await { - return Err(Error::BadRequest(ErrorKind::UserInUse, "Desired user ID is already taken.")); + return Err(Error::BadRequest( + ErrorKind::UserInUse, + "Desired user ID is already taken.", + )); } if services @@ -187,7 +205,7 @@ pub(crate) async fn register_route( proposed_user_id }, - _ => loop { + | _ => loop { let proposed_user_id = UserId::parse_with_server_name( utils::random_string(RANDOM_USER_ID_LENGTH).to_lowercase(), services.globals.server_name(), @@ -228,9 +246,7 @@ pub(crate) async fn register_route( } else { // No registration token necessary, but clients must still go through the flow uiaainfo = UiaaInfo { - flows: vec![AuthFlow { - stages: vec![AuthType::Dummy], - }], + flows: vec![AuthFlow { stages: vec![AuthType::Dummy] }], completed: Vec::new(), params: Box::default(), session: None, @@ -244,7 +260,8 @@ pub(crate) async fn register_route( let (worked, uiaainfo) = services .uiaa .try_auth( - &UserId::parse_with_server_name("", services.globals.server_name()).expect("we know this is valid"), + &UserId::parse_with_server_name("", services.globals.server_name()) + .expect("we know this is valid"), "".into(), auth, &uiaainfo, @@ -257,7 +274,8 @@ pub(crate) async fn register_route( } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services.uiaa.create( - &UserId::parse_with_server_name("", services.globals.server_name()).expect("we know this is valid"), + &UserId::parse_with_server_name("", services.globals.server_name()) + .expect("we know this is valid"), "".into(), &uiaainfo, &json, @@ -268,11 +286,7 @@ pub(crate) async fn register_route( } } - let password = if is_guest { - None - } else { - body.password.as_deref() - }; + let password = if is_guest { None } else { body.password.as_deref() }; // Create user services.users.create(&user_id, password)?; @@ -282,7 +296,9 @@ pub(crate) async fn register_route( // If `new_user_displayname_suffix` is set, registration will push whatever // content is set to the user's display name with a space before it - if !services.globals.new_user_displayname_suffix().is_empty() && body.appservice_info.is_none() { + if !services.globals.new_user_displayname_suffix().is_empty() + && body.appservice_info.is_none() + { write!(displayname, " {}", services.globals.config.new_user_displayname_suffix) .expect("should be able to write to string buffer"); } @@ -319,12 +335,8 @@ pub(crate) async fn register_route( } // Generate new device id if the user didn't specify one - let device_id = if is_guest { - None - } else { - body.device_id.clone() - } - .unwrap_or_else(|| utils::random_string(DEVICE_ID_LENGTH).into()); + let device_id = if is_guest { None } else { body.device_id.clone() } + .unwrap_or_else(|| utils::random_string(DEVICE_ID_LENGTH).into()); // Generate new token for the device let token = utils::random_string(TOKEN_LENGTH); @@ -349,15 +361,16 @@ pub(crate) async fn register_route( if body.appservice_info.is_none() && !is_guest { if !device_display_name.is_empty() { info!( - "New user \"{user_id}\" registered on this server with device display name: \"{device_display_name}\"" + "New user \"{user_id}\" registered on this server with device display name: \ + \"{device_display_name}\"" ); if services.globals.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::notice_plain(format!( - "New user \"{user_id}\" registered on this server from IP {client} and device display name \ - \"{device_display_name}\"" + "New user \"{user_id}\" registered on this server from IP {client} and \ + device display name \"{device_display_name}\"" ))) .await .ok(); @@ -386,8 +399,8 @@ pub(crate) async fn register_route( services .admin .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with device display name \"{device_display_name}\" registered on \ - this server from IP {client}" + "Guest user \"{user_id}\" with device display name \ + \"{device_display_name}\" registered on this server from IP {client}" ))) .await .ok(); @@ -398,8 +411,8 @@ pub(crate) async fn register_route( services .admin .send_message(RoomMessageEventContent::notice_plain(format!( - "Guest user \"{user_id}\" with no device display name registered on this server from IP \ - {client}", + "Guest user \"{user_id}\" with no device display name registered on \ + this server from IP {client}", ))) .await .ok(); @@ -430,7 +443,10 @@ pub(crate) async fn register_route( { for room in &services.globals.config.auto_join_rooms { let Ok(room_id) = services.rooms.alias.resolve(room).await else { - error!("Failed to resolve room alias to room ID when attempting to auto join {room}, skipping"); + error!( + "Failed to resolve room alias to room ID when attempting to auto join \ + {room}, skipping" + ); continue; }; @@ -440,7 +456,9 @@ pub(crate) async fn register_route( .server_in_room(services.globals.server_name(), &room_id) .await { - warn!("Skipping room {room} to automatically join as we have never joined before."); + warn!( + "Skipping room {room} to automatically join as we have never joined before." + ); continue; } @@ -494,7 +512,8 @@ pub(crate) async fn register_route( /// - Triggers device list updates #[tracing::instrument(skip_all, fields(%client), name = "change_password")] pub(crate) async fn change_password_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { // Authentication for this endpoint was made optional, but we need @@ -506,9 +525,7 @@ pub(crate) async fn change_password_route( let sender_device = body.sender_device.as_ref().expect("user is authenticated"); let mut uiaainfo = UiaaInfo { - flows: vec![AuthFlow { - stages: vec![AuthType::Password], - }], + flows: vec![AuthFlow { stages: vec![AuthType::Password] }], completed: Vec::new(), params: Box::default(), session: None, @@ -572,7 +589,8 @@ pub(crate) async fn change_password_route( /// /// Note: Also works for Application Services pub(crate) async fn whoami_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let device_id = body.sender_device.clone(); @@ -580,7 +598,8 @@ pub(crate) async fn whoami_route( Ok(whoami::v3::Response { user_id: sender_user.clone(), device_id, - is_guest: services.users.is_deactivated(sender_user).await? && body.appservice_info.is_none(), + is_guest: services.users.is_deactivated(sender_user).await? + && body.appservice_info.is_none(), }) } @@ -597,7 +616,8 @@ pub(crate) async fn whoami_route( /// - Removes ability to log in again #[tracing::instrument(skip_all, fields(%client), name = "deactivate")] pub(crate) async fn deactivate_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { // Authentication for this endpoint was made optional, but we need @@ -609,9 +629,7 @@ pub(crate) async fn deactivate_route( let sender_device = body.sender_device.as_ref().expect("user is authenticated"); let mut uiaainfo = UiaaInfo { - flows: vec![AuthFlow { - stages: vec![AuthType::Password], - }], + flows: vec![AuthFlow { stages: vec![AuthType::Password] }], completed: Vec::new(), params: Box::default(), session: None, @@ -675,7 +693,9 @@ pub(crate) async fn deactivate_route( /// Get a list of third party identifiers associated with this account. /// /// - Currently always returns empty list -pub(crate) async fn third_party_route(body: Ruma) -> Result { +pub(crate) async fn third_party_route( + body: Ruma, +) -> Result { let _sender_user = body.sender_user.as_ref().expect("user is authenticated"); Ok(get_3pids::v3::Response::new(Vec::new())) @@ -720,7 +740,8 @@ pub(crate) async fn request_3pid_management_token_via_msisdn_route( /// Currently does not have any ratelimiting, and this isn't very practical as /// there is only one registration token allowed. pub(crate) async fn check_registration_token_validity( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let Some(reg_token) = services.globals.registration_token.clone() else { return Err(Error::BadRequest( @@ -729,9 +750,7 @@ pub(crate) async fn check_registration_token_validity( )); }; - Ok(check_registration_token_validity::v1::Response { - valid: reg_token == body.token, - }) + Ok(check_registration_token_validity::v1::Response { valid: reg_token == body.token }) } /// Runs through all the deactivation steps: @@ -742,7 +761,9 @@ pub(crate) async fn check_registration_token_validity( /// - Removing all profile data /// - Leaving all rooms (and forgets all of them) pub async fn full_user_deactivate( - services: &Services, user_id: &UserId, all_joined_rooms: &[OwnedRoomId], + services: &Services, + user_id: &UserId, + all_joined_rooms: &[OwnedRoomId], ) -> Result<()> { services.users.deactivate_account(user_id).await.ok(); super::update_displayname(services, user_id, None, all_joined_rooms).await; @@ -751,7 +772,9 @@ pub async fn full_user_deactivate( services .users .all_profile_keys(user_id) - .ready_for_each(|(profile_key, _)| services.users.set_profile_key(user_id, &profile_key, None)) + .ready_for_each(|(profile_key, _)| { + services.users.set_profile_key(user_id, &profile_key, None); + }) .await; for room_id in all_joined_rooms { @@ -760,20 +783,26 @@ pub async fn full_user_deactivate( let room_power_levels = services .rooms .state_accessor - .room_state_get_content::(room_id, &StateEventType::RoomPowerLevels, "") + .room_state_get_content::( + room_id, + &StateEventType::RoomPowerLevels, + "", + ) .await .ok(); - let user_can_demote_self = room_power_levels - .as_ref() - .is_some_and(|power_levels_content| { - RoomPowerLevels::from(power_levels_content.clone()).user_can_change_user_power_level(user_id, user_id) - }) || services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "") - .await - .is_ok_and(|event| event.sender == user_id); + let user_can_demote_self = + room_power_levels + .as_ref() + .is_some_and(|power_levels_content| { + RoomPowerLevels::from(power_levels_content.clone()) + .user_can_change_user_power_level(user_id, user_id) + }) || services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomCreate, "") + .await + .is_ok_and(|event| event.sender == user_id); if user_can_demote_self { let mut power_levels_content = room_power_levels.unwrap_or_default(); diff --git a/src/api/client/account_data.rs b/src/api/client/account_data.rs index 2fc78808..9f84f227 100644 --- a/src/api/client/account_data.rs +++ b/src/api/client/account_data.rs @@ -2,11 +2,12 @@ use axum::extract::State; use conduwuit::{err, Err}; use ruma::{ api::client::config::{ - get_global_account_data, get_room_account_data, set_global_account_data, set_room_account_data, + get_global_account_data, get_room_account_data, set_global_account_data, + set_room_account_data, }, events::{ - AnyGlobalAccountDataEventContent, AnyRoomAccountDataEventContent, GlobalAccountDataEventType, - RoomAccountDataEventType, + AnyGlobalAccountDataEventContent, AnyRoomAccountDataEventContent, + GlobalAccountDataEventType, RoomAccountDataEventType, }, serde::Raw, RoomId, UserId, @@ -20,7 +21,8 @@ use crate::{service::Services, Result, Ruma}; /// /// Sets some account data for the sender user. pub(crate) async fn set_global_account_data_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -28,7 +30,14 @@ pub(crate) async fn set_global_account_data_route( return Err!(Request(Forbidden("You cannot set account data for other users."))); } - set_account_data(&services, None, &body.user_id, &body.event_type.to_string(), body.data.json()).await?; + set_account_data( + &services, + None, + &body.user_id, + &body.event_type.to_string(), + body.data.json(), + ) + .await?; Ok(set_global_account_data::v3::Response {}) } @@ -37,7 +46,8 @@ pub(crate) async fn set_global_account_data_route( /// /// Sets some room account data for the sender user. pub(crate) async fn set_room_account_data_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -61,7 +71,8 @@ pub(crate) async fn set_room_account_data_route( /// /// Gets some account data for the sender user. pub(crate) async fn get_global_account_data_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -75,16 +86,15 @@ pub(crate) async fn get_global_account_data_route( .await .map_err(|_| err!(Request(NotFound("Data not found."))))?; - Ok(get_global_account_data::v3::Response { - account_data: account_data.content, - }) + Ok(get_global_account_data::v3::Response { account_data: account_data.content }) } /// # `GET /_matrix/client/r0/user/{userId}/rooms/{roomId}/account_data/{type}` /// /// Gets some room account data for the sender user. pub(crate) async fn get_room_account_data_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -98,17 +108,20 @@ pub(crate) async fn get_room_account_data_route( .await .map_err(|_| err!(Request(NotFound("Data not found."))))?; - Ok(get_room_account_data::v3::Response { - account_data: account_data.content, - }) + Ok(get_room_account_data::v3::Response { account_data: account_data.content }) } async fn set_account_data( - services: &Services, room_id: Option<&RoomId>, sender_user: &UserId, event_type_s: &str, data: &RawJsonValue, + services: &Services, + room_id: Option<&RoomId>, + sender_user: &UserId, + event_type_s: &str, + data: &RawJsonValue, ) -> Result { if event_type_s == RoomAccountDataEventType::FullyRead.to_cow_str() { return Err!(Request(BadJson( - "This endpoint cannot be used for marking a room as fully read (setting m.fully_read)" + "This endpoint cannot be used for marking a room as fully read (setting \ + m.fully_read)" ))); } @@ -118,8 +131,8 @@ async fn set_account_data( ))); } - let data: serde_json::Value = - serde_json::from_str(data.get()).map_err(|e| err!(Request(BadJson(warn!("Invalid JSON provided: {e}")))))?; + let data: serde_json::Value = serde_json::from_str(data.get()) + .map_err(|e| err!(Request(BadJson(warn!("Invalid JSON provided: {e}")))))?; services .account_data diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index c60a2f4c..e1af416e 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -14,7 +14,8 @@ use crate::Ruma; /// /// Creates a new room alias on this server. pub(crate) async fn create_alias_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -58,7 +59,8 @@ pub(crate) async fn create_alias_route( /// /// - TODO: Update canonical alias event pub(crate) async fn delete_alias_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -83,11 +85,13 @@ pub(crate) async fn delete_alias_route( /// /// Resolve an alias locally or over federation. pub(crate) async fn get_alias_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let room_alias = body.body.room_alias; - let Ok((room_id, servers)) = services.rooms.alias.resolve_alias(&room_alias, None).await else { + let Ok((room_id, servers)) = services.rooms.alias.resolve_alias(&room_alias, None).await + else { return Err!(Request(NotFound("Room with alias not found."))); }; @@ -98,7 +102,10 @@ pub(crate) async fn get_alias_route( } async fn room_available_servers( - services: &Services, room_id: &RoomId, room_alias: &RoomAliasId, pre_servers: Vec, + services: &Services, + room_id: &RoomId, + room_alias: &RoomAliasId, + pre_servers: Vec, ) -> Vec { // find active servers in room state cache to suggest let mut servers: Vec = services diff --git a/src/api/client/appservice.rs b/src/api/client/appservice.rs index 9dbd141e..e4071ab0 100644 --- a/src/api/client/appservice.rs +++ b/src/api/client/appservice.rs @@ -9,12 +9,12 @@ use crate::Ruma; /// Ask the homeserver to ping the application service to ensure the connection /// works. pub(crate) async fn appservice_ping( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { - let appservice_info = body - .appservice_info - .as_ref() - .ok_or_else(|| err!(Request(Forbidden("This endpoint can only be called by appservices."))))?; + let appservice_info = body.appservice_info.as_ref().ok_or_else(|| { + err!(Request(Forbidden("This endpoint can only be called by appservices."))) + })?; if body.appservice_id != appservice_info.registration.id { return Err!(Request(Forbidden( @@ -41,7 +41,5 @@ pub(crate) async fn appservice_ping( .await? .expect("We already validated if an appservice URL exists above"); - Ok(request_ping::v1::Response { - duration: timer.elapsed(), - }) + Ok(request_ping::v1::Response { duration: timer.elapsed() }) } diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index 9e4746ca..d330952d 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -2,10 +2,11 @@ use axum::extract::State; use conduwuit::{err, Err}; use ruma::{ api::client::backup::{ - add_backup_keys, add_backup_keys_for_room, add_backup_keys_for_session, create_backup_version, - delete_backup_keys, delete_backup_keys_for_room, delete_backup_keys_for_session, delete_backup_version, - get_backup_info, get_backup_keys, get_backup_keys_for_room, get_backup_keys_for_session, - get_latest_backup_info, update_backup_version, + add_backup_keys, add_backup_keys_for_room, add_backup_keys_for_session, + create_backup_version, delete_backup_keys, delete_backup_keys_for_room, + delete_backup_keys_for_session, delete_backup_version, get_backup_info, get_backup_keys, + get_backup_keys_for_room, get_backup_keys_for_session, get_latest_backup_info, + update_backup_version, }, UInt, }; @@ -16,15 +17,14 @@ use crate::{Result, Ruma}; /// /// Creates a new backup. pub(crate) async fn create_backup_version_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let version = services .key_backups .create_backup(body.sender_user(), &body.algorithm)?; - Ok(create_backup_version::v3::Response { - version, - }) + Ok(create_backup_version::v3::Response { version }) } /// # `PUT /_matrix/client/r0/room_keys/version/{version}` @@ -32,7 +32,8 @@ pub(crate) async fn create_backup_version_route( /// Update information about an existing backup. Only `auth_data` can be /// modified. pub(crate) async fn update_backup_version_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { services .key_backups @@ -46,7 +47,8 @@ pub(crate) async fn update_backup_version_route( /// /// Get information about the latest backup version. pub(crate) async fn get_latest_backup_info_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let (version, algorithm) = services .key_backups @@ -75,13 +77,16 @@ pub(crate) async fn get_latest_backup_info_route( /// /// Get information about an existing backup. pub(crate) async fn get_backup_info_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let algorithm = services .key_backups .get_backup(body.sender_user(), &body.version) .await - .map_err(|_| err!(Request(NotFound("Key backup does not exist at version {:?}", body.version))))?; + .map_err(|_| { + err!(Request(NotFound("Key backup does not exist at version {:?}", body.version))) + })?; Ok(get_backup_info::v3::Response { algorithm, @@ -105,7 +110,8 @@ pub(crate) async fn get_backup_info_route( /// - Deletes both information about the backup, as well as all key data related /// to the backup pub(crate) async fn delete_backup_version_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { services .key_backups @@ -124,7 +130,8 @@ pub(crate) async fn delete_backup_version_route( /// - Adds the keys to the backup /// - Returns the new number of keys in this backup and the etag pub(crate) async fn add_backup_keys_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if services .key_backups @@ -168,7 +175,8 @@ pub(crate) async fn add_backup_keys_route( /// - Adds the keys to the backup /// - Returns the new number of keys in this backup and the etag pub(crate) async fn add_backup_keys_for_room_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if services .key_backups @@ -210,7 +218,8 @@ pub(crate) async fn add_backup_keys_for_room_route( /// - Adds the keys to the backup /// - Returns the new number of keys in this backup and the etag pub(crate) async fn add_backup_keys_for_session_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if services .key_backups @@ -251,56 +260,56 @@ pub(crate) async fn add_backup_keys_for_session_route( /// /// Retrieves all keys from the backup. pub(crate) async fn get_backup_keys_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let rooms = services .key_backups .get_all(body.sender_user(), &body.version) .await; - Ok(get_backup_keys::v3::Response { - rooms, - }) + Ok(get_backup_keys::v3::Response { rooms }) } /// # `GET /_matrix/client/r0/room_keys/keys/{roomId}` /// /// Retrieves all keys from the backup for a given room. pub(crate) async fn get_backup_keys_for_room_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sessions = services .key_backups .get_room(body.sender_user(), &body.version, &body.room_id) .await; - Ok(get_backup_keys_for_room::v3::Response { - sessions, - }) + Ok(get_backup_keys_for_room::v3::Response { sessions }) } /// # `GET /_matrix/client/r0/room_keys/keys/{roomId}/{sessionId}` /// /// Retrieves a key from the backup. pub(crate) async fn get_backup_keys_for_session_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let key_data = services .key_backups .get_session(body.sender_user(), &body.version, &body.room_id, &body.session_id) .await - .map_err(|_| err!(Request(NotFound(debug_error!("Backup key not found for this user's session.")))))?; + .map_err(|_| { + err!(Request(NotFound(debug_error!("Backup key not found for this user's session.")))) + })?; - Ok(get_backup_keys_for_session::v3::Response { - key_data, - }) + Ok(get_backup_keys_for_session::v3::Response { key_data }) } /// # `DELETE /_matrix/client/r0/room_keys/keys` /// /// Delete the keys from the backup. pub(crate) async fn delete_backup_keys_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { services .key_backups @@ -324,7 +333,8 @@ pub(crate) async fn delete_backup_keys_route( /// /// Delete the keys from the backup for a given room. pub(crate) async fn delete_backup_keys_for_room_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { services .key_backups @@ -348,7 +358,8 @@ pub(crate) async fn delete_backup_keys_for_room_route( /// /// Delete a key from the backup. pub(crate) async fn delete_backup_keys_for_session_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { services .key_backups diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 20f9cb58..e122611f 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -3,8 +3,8 @@ use std::collections::BTreeMap; use axum::extract::State; use ruma::{ api::client::discovery::get_capabilities::{ - self, Capabilities, GetLoginTokenCapability, RoomVersionStability, RoomVersionsCapability, - ThirdPartyIdChangesCapability, + self, Capabilities, GetLoginTokenCapability, RoomVersionStability, + RoomVersionsCapability, ThirdPartyIdChangesCapability, }, RoomVersionId, }; @@ -17,9 +17,11 @@ use crate::{Result, Ruma}; /// Get information on the supported feature set and other relevent capabilities /// of this server. pub(crate) async fn get_capabilities_route( - State(services): State, _body: Ruma, + State(services): State, + _body: Ruma, ) -> Result { - let available: BTreeMap = services.server.available_room_versions().collect(); + let available: BTreeMap = + services.server.available_room_versions().collect(); let mut capabilities = Capabilities::default(); capabilities.room_versions = RoomVersionsCapability { @@ -28,21 +30,15 @@ pub(crate) async fn get_capabilities_route( }; // we do not implement 3PID stuff - capabilities.thirdparty_id_changes = ThirdPartyIdChangesCapability { - enabled: false, - }; + capabilities.thirdparty_id_changes = ThirdPartyIdChangesCapability { enabled: false }; // we dont support generating tokens yet - capabilities.get_login_token = GetLoginTokenCapability { - enabled: false, - }; + capabilities.get_login_token = GetLoginTokenCapability { enabled: false }; // MSC4133 capability capabilities .set("uk.tcpip.msc4133.profile_fields", json!({"enabled": true})) .expect("this is valid JSON we created"); - Ok(get_capabilities::v3::Response { - capabilities, - }) + Ok(get_capabilities::v3::Response { capabilities }) } diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 52f27692..30ba170d 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -32,7 +32,8 @@ const LIMIT_DEFAULT: usize = 10; /// - Only works if the user is joined (TODO: always allow, but only show events /// if the user was joined, depending on history_visibility) pub(crate) async fn get_context_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let filter = &body.filter; let sender = body.sender(); @@ -50,9 +51,8 @@ pub(crate) async fn get_context_route( // members for "inline" profiles on the timeline to work properly let lazy_load_enabled = matches!(filter.lazy_load_options, LazyLoadOptions::Enabled { .. }); - let lazy_load_redundant = if let LazyLoadOptions::Enabled { - include_redundant_members, - } = filter.lazy_load_options + let lazy_load_redundant = if let LazyLoadOptions::Enabled { include_redundant_members } = + filter.lazy_load_options { include_redundant_members } else { @@ -91,10 +91,11 @@ pub(crate) async fn get_context_route( return Err!(Request(Forbidden("You don't have permission to view this event."))); } - let events_before = services - .rooms - .timeline - .pdus_rev(Some(sender_user), room_id, Some(base_token)); + let events_before = + services + .rooms + .timeline + .pdus_rev(Some(sender_user), room_id, Some(base_token)); let events_after = services .rooms @@ -166,7 +167,9 @@ pub(crate) async fn get_context_route( .filter(|&user_id: &&UserId| lazy.contains(user_id)) .map(|_| event_id) }) - .broad_filter_map(|event_id: &OwnedEventId| services.rooms.timeline.get_pdu(event_id).ok()) + .broad_filter_map(|event_id: &OwnedEventId| { + services.rooms.timeline.get_pdu(event_id).ok() + }) .map(|pdu| pdu.to_state_event()) .collect() .await; diff --git a/src/api/client/device.rs b/src/api/client/device.rs index 63f0c210..bb0773dd 100644 --- a/src/api/client/device.rs +++ b/src/api/client/device.rs @@ -18,7 +18,8 @@ use crate::{utils, Error, Result, Ruma}; /// /// Get metadata on all devices of the sender user. pub(crate) async fn get_devices_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -28,16 +29,15 @@ pub(crate) async fn get_devices_route( .collect() .await; - Ok(get_devices::v3::Response { - devices, - }) + Ok(get_devices::v3::Response { devices }) } /// # `GET /_matrix/client/r0/devices/{deviceId}` /// /// Get metadata on a single device of the sender user. pub(crate) async fn get_device_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -47,9 +47,7 @@ pub(crate) async fn get_device_route( .await .map_err(|_| err!(Request(NotFound("Device not found."))))?; - Ok(get_device::v3::Response { - device, - }) + Ok(get_device::v3::Response { device }) } /// # `PUT /_matrix/client/r0/devices/{deviceId}` @@ -57,7 +55,8 @@ pub(crate) async fn get_device_route( /// Updates the metadata on a given device of the sender user. #[tracing::instrument(skip_all, fields(%client), name = "update_device")] pub(crate) async fn update_device_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -93,16 +92,15 @@ pub(crate) async fn update_device_route( /// - Forgets to-device events /// - Triggers device list updates pub(crate) async fn delete_device_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); // UIAA let mut uiaainfo = UiaaInfo { - flows: vec![AuthFlow { - stages: vec![AuthType::Password], - }], + flows: vec![AuthFlow { stages: vec![AuthType::Password] }], completed: Vec::new(), params: Box::default(), session: None, @@ -151,16 +149,15 @@ pub(crate) async fn delete_device_route( /// - Forgets to-device events /// - Triggers device list updates pub(crate) async fn delete_devices_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); // UIAA let mut uiaainfo = UiaaInfo { - flows: vec![AuthFlow { - stages: vec![AuthType::Password], - }], + flows: vec![AuthFlow { stages: vec![AuthType::Password] }], completed: Vec::new(), params: Box::default(), session: None, diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 9407a0bd..c8faaa46 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -5,7 +5,10 @@ use futures::{StreamExt, TryFutureExt}; use ruma::{ api::{ client::{ - directory::{get_public_rooms, get_public_rooms_filtered, get_room_visibility, set_room_visibility}, + directory::{ + get_public_rooms, get_public_rooms_filtered, get_room_visibility, + set_room_visibility, + }, error::ErrorKind, room, }, @@ -32,7 +35,8 @@ use crate::Ruma; /// - Rooms are ordered by the number of joined members #[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] pub(crate) async fn get_public_rooms_filtered_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if let Some(server) = &body.server { @@ -57,7 +61,10 @@ pub(crate) async fn get_public_rooms_filtered_route( .await .map_err(|e| { warn!(?body.server, "Failed to return /publicRooms: {e}"); - Error::BadRequest(ErrorKind::Unknown, "Failed to return the requested server's public room list.") + Error::BadRequest( + ErrorKind::Unknown, + "Failed to return the requested server's public room list.", + ) })?; Ok(response) @@ -70,7 +77,8 @@ pub(crate) async fn get_public_rooms_filtered_route( /// - Rooms are ordered by the number of joined members #[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] pub(crate) async fn get_public_rooms_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if let Some(server) = &body.server { @@ -95,7 +103,10 @@ pub(crate) async fn get_public_rooms_route( .await .map_err(|e| { warn!(?body.server, "Failed to return /publicRooms: {e}"); - Error::BadRequest(ErrorKind::Unknown, "Failed to return the requested server's public room list.") + Error::BadRequest( + ErrorKind::Unknown, + "Failed to return the requested server's public room list.", + ) })?; Ok(get_public_rooms::v3::Response { @@ -111,7 +122,8 @@ pub(crate) async fn get_public_rooms_route( /// Sets the visibility of a given room in the room directory. #[tracing::instrument(skip_all, fields(%client), name = "room_directory")] pub(crate) async fn set_room_visibility_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -139,14 +151,14 @@ pub(crate) async fn set_room_visibility_route( } match &body.visibility { - room::Visibility::Public => { + | room::Visibility::Public => { if services.globals.config.lockdown_public_room_directory && !services.users.is_admin(sender_user).await && body.appservice_info.is_none() { info!( - "Non-admin user {sender_user} tried to publish {0} to the room directory while \ - \"lockdown_public_room_directory\" is enabled", + "Non-admin user {sender_user} tried to publish {0} to the room directory \ + while \"lockdown_public_room_directory\" is enabled", body.room_id ); @@ -154,8 +166,8 @@ pub(crate) async fn set_room_visibility_route( services .admin .send_text(&format!( - "Non-admin user {sender_user} tried to publish {0} to the room directory while \ - \"lockdown_public_room_directory\" is enabled", + "Non-admin user {sender_user} tried to publish {0} to the room \ + directory while \"lockdown_public_room_directory\" is enabled", body.room_id )) .await; @@ -172,13 +184,16 @@ pub(crate) async fn set_room_visibility_route( if services.globals.config.admin_room_notices { services .admin - .send_text(&format!("{sender_user} made {} public to the room directory", body.room_id)) + .send_text(&format!( + "{sender_user} made {} public to the room directory", + body.room_id + )) .await; } info!("{sender_user} made {0} public to the room directory", body.room_id); }, - room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id), - _ => { + | room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id), + | _ => { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Room visibility type is not supported.", @@ -193,7 +208,8 @@ pub(crate) async fn set_room_visibility_route( /// /// Gets the visibility of a given room in the room directory. pub(crate) async fn get_room_visibility_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.rooms.metadata.exists(&body.room_id).await { // Return 404 if the room doesn't exist @@ -210,10 +226,16 @@ pub(crate) async fn get_room_visibility_route( } pub(crate) async fn get_public_rooms_filtered_helper( - services: &Services, server: Option<&ServerName>, limit: Option, since: Option<&str>, filter: &Filter, + services: &Services, + server: Option<&ServerName>, + limit: Option, + since: Option<&str>, + filter: &Filter, _network: &RoomNetwork, ) -> Result { - if let Some(other_server) = server.filter(|server_name| !services.globals.server_is_ours(server_name)) { + if let Some(other_server) = + server.filter(|server_name| !services.globals.server_is_ours(server_name)) + { let response = services .sending .send_federation_request( @@ -245,9 +267,10 @@ pub(crate) async fn get_public_rooms_filtered_helper( if let Some(s) = &since { let mut characters = s.chars(); let backwards = match characters.next() { - Some('n') => false, - Some('p') => true, - _ => return Err(Error::BadRequest(ErrorKind::InvalidParam, "Invalid `since` token")), + | Some('n') => false, + | Some('p') => true, + | _ => + return Err(Error::BadRequest(ErrorKind::InvalidParam, "Invalid `since` token")), }; num_since = characters @@ -337,7 +360,11 @@ pub(crate) async fn get_public_rooms_filtered_helper( /// Check whether the user can publish to the room directory via power levels of /// room history visibility event or room creator -async fn user_can_publish_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result { +async fn user_can_publish_room( + services: &Services, + user_id: &UserId, + room_id: &RoomId, +) -> Result { if let Ok(event) = services .rooms .state_accessor @@ -347,7 +374,8 @@ async fn user_can_publish_room(services: &Services, user_id: &UserId, room_id: & serde_json::from_str(event.content.get()) .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) .map(|content: RoomPowerLevelsEventContent| { - RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomHistoryVisibility) + RoomPowerLevels::from(content) + .user_can_send_state(user_id, StateEventType::RoomHistoryVisibility) }) } else if let Ok(event) = services .rooms @@ -406,10 +434,10 @@ async fn public_rooms_chunk(services: &Services, room_id: OwnedRoomId) -> Public .state_accessor .room_state_get_content(&room_id, &StateEventType::RoomJoinRules, "") .map_ok(|c: RoomJoinRulesEventContent| match c.join_rule { - JoinRule::Public => PublicRoomJoinRule::Public, - JoinRule::Knock => "knock".into(), - JoinRule::KnockRestricted(_) => "knock_restricted".into(), - _ => "invite".into(), + | JoinRule::Public => PublicRoomJoinRule::Public, + | JoinRule::Knock => "knock".into(), + | JoinRule::KnockRestricted(_) => "knock_restricted".into(), + | _ => "invite".into(), }) .await .unwrap_or_default(), diff --git a/src/api/client/filter.rs b/src/api/client/filter.rs index a1576e8e..84086452 100644 --- a/src/api/client/filter.rs +++ b/src/api/client/filter.rs @@ -10,7 +10,8 @@ use crate::{Result, Ruma}; /// /// - A user can only access their own filters pub(crate) async fn get_filter_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -26,7 +27,8 @@ pub(crate) async fn get_filter_route( /// /// Creates a new filter to be used by other endpoints. pub(crate) async fn create_filter_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 13dc3467..7bf0a5da 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -7,7 +7,10 @@ use ruma::{ api::{ client::{ error::ErrorKind, - keys::{claim_keys, get_key_changes, get_keys, upload_keys, upload_signatures, upload_signing_keys}, + keys::{ + claim_keys, get_key_changes, get_keys, upload_keys, upload_signatures, + upload_signing_keys, + }, uiaa::{AuthFlow, AuthType, UiaaInfo}, }, federation, @@ -31,7 +34,8 @@ use crate::{ /// - If there are no device keys yet: Adds device keys (TODO: merge with /// existing keys?) pub(crate) async fn upload_keys_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let (sender_user, sender_device) = body.sender(); @@ -75,7 +79,8 @@ pub(crate) async fn upload_keys_route( /// - The master and self-signing keys contain signatures that the user is /// allowed to see pub(crate) async fn get_keys_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -93,7 +98,8 @@ pub(crate) async fn get_keys_route( /// /// Claims one-time keys pub(crate) async fn claim_keys_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { claim_keys_helper(&services, &body.one_time_keys).await } @@ -104,16 +110,15 @@ pub(crate) async fn claim_keys_route( /// /// - Requires UIAA to verify password pub(crate) async fn upload_signing_keys_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); // UIAA let mut uiaainfo = UiaaInfo { - flows: vec![AuthFlow { - stages: vec![AuthType::Password], - }], + flows: vec![AuthFlow { stages: vec![AuthType::Password] }], completed: Vec::new(), params: Box::default(), session: None, @@ -161,7 +166,8 @@ pub(crate) async fn upload_signing_keys_route( /// /// Uploads end-to-end key signatures from the sender user. pub(crate) async fn upload_signatures_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -174,7 +180,10 @@ pub(crate) async fn upload_signatures_route( .get("signatures") .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Missing signatures field."))? .get(sender_user.to_string()) - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid user in signatures field."))? + .ok_or(Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid user in signatures field.", + ))? .as_object() .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid signature."))? .clone() @@ -185,7 +194,10 @@ pub(crate) async fn upload_signatures_route( signature .1 .as_str() - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid signature value."))? + .ok_or(Error::BadRequest( + ErrorKind::InvalidParam, + "Invalid signature value.", + ))? .to_owned(), ); @@ -209,7 +221,8 @@ pub(crate) async fn upload_signatures_route( /// /// - TODO: left users pub(crate) async fn get_key_changes_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -255,8 +268,11 @@ pub(crate) async fn get_key_changes_route( } pub(crate) async fn get_keys_helper( - services: &Services, sender_user: Option<&UserId>, device_keys_input: &BTreeMap>, - allowed_signatures: F, include_display_names: bool, + services: &Services, + sender_user: Option<&UserId>, + device_keys_input: &BTreeMap>, + allowed_signatures: F, + include_display_names: bool, ) -> Result where F: Fn(&UserId) -> bool + Send + Sync, @@ -289,7 +305,9 @@ where .users .get_device_metadata(user_id, device_id) .await - .map_err(|_| err!(Database("all_device_keys contained nonexistent device.")))?; + .map_err(|_| { + err!(Database("all_device_keys contained nonexistent device.")) + })?; add_unsigned_device_display_name(&mut keys, metadata, include_display_names) .map_err(|_| err!(Database("invalid device keys in database")))?; @@ -307,7 +325,11 @@ where .users .get_device_metadata(user_id, device_id) .await - .map_err(|_| err!(Request(InvalidParam("Tried to get keys for nonexistent device."))))?; + .map_err(|_| { + err!(Request(InvalidParam( + "Tried to get keys for nonexistent device." + ))) + })?; add_unsigned_device_display_name(&mut keys, metadata, include_display_names) .map_err(|_| err!(Database("invalid device keys in database")))?; @@ -350,9 +372,8 @@ where device_keys_input_fed.insert(user_id.to_owned(), keys.clone()); } - let request = federation::keys::get_keys::v1::Request { - device_keys: device_keys_input_fed, - }; + let request = + federation::keys::get_keys::v1::Request { device_keys: device_keys_input_fed }; let response = services .sending @@ -382,8 +403,8 @@ where .users .add_cross_signing_keys( &user, &raw, &None, &None, - false, /* Dont notify. A notification would trigger another key request resulting in an - * endless loop */ + false, /* Dont notify. A notification would trigger another key request + * resulting in an endless loop */ ) .await?; master_keys.insert(user.clone(), raw); @@ -406,7 +427,8 @@ where } fn add_unsigned_device_display_name( - keys: &mut Raw, metadata: ruma::api::client::device::Device, + keys: &mut Raw, + metadata: ruma::api::client::device::Device, include_display_names: bool, ) -> serde_json::Result<()> { if let Some(display_name) = metadata.display_name { @@ -431,7 +453,8 @@ fn add_unsigned_device_display_name( } pub(crate) async fn claim_keys_helper( - services: &Services, one_time_keys_input: &BTreeMap>, + services: &Services, + one_time_keys_input: &BTreeMap>, ) -> Result { let mut one_time_keys = BTreeMap::new(); @@ -473,12 +496,9 @@ pub(crate) async fn claim_keys_helper( server, services .sending - .send_federation_request( - server, - federation::keys::claim_keys::v1::Request { - one_time_keys: one_time_keys_input_fed, - }, - ) + .send_federation_request(server, federation::keys::claim_keys::v1::Request { + one_time_keys: one_time_keys_input_fed, + }) .await, ) }) @@ -486,17 +506,14 @@ pub(crate) async fn claim_keys_helper( while let Some((server, response)) = futures.next().await { match response { - Ok(keys) => { + | Ok(keys) => { one_time_keys.extend(keys.one_time_keys); }, - Err(_e) => { + | Err(_e) => { failures.insert(server.to_string(), json!({})); }, } } - Ok(claim_keys::v3::Response { - failures, - one_time_keys, - }) + Ok(claim_keys::v3::Response { failures, one_time_keys }) } diff --git a/src/api/client/media.rs b/src/api/client/media.rs index a9792c3b..e58ba626 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -15,7 +15,8 @@ use reqwest::Url; use ruma::{ api::client::{ authenticated_media::{ - get_content, get_content_as_filename, get_content_thumbnail, get_media_config, get_media_preview, + get_content, get_content_as_filename, get_content_thumbnail, get_media_config, + get_media_preview, }, media::create_content, }, @@ -26,7 +27,8 @@ use crate::Ruma; /// # `GET /_matrix/client/v1/media/config` pub(crate) async fn get_media_config_route( - State(services): State, _body: Ruma, + State(services): State, + _body: Ruma, ) -> Result { Ok(get_media_config::v1::Response { upload_size: ruma_from_usize(services.globals.config.max_request_size), @@ -46,7 +48,8 @@ pub(crate) async fn get_media_config_route( fields(%client), )] pub(crate) async fn create_content_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let user = body.sender_user.as_ref().expect("user is authenticated"); @@ -79,7 +82,8 @@ pub(crate) async fn create_content_route( fields(%client), )] pub(crate) async fn get_content_thumbnail_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let user = body.sender_user.as_ref().expect("user is authenticated"); @@ -115,7 +119,8 @@ pub(crate) async fn get_content_thumbnail_route( fields(%client), )] pub(crate) async fn get_content_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let user = body.sender_user.as_ref().expect("user is authenticated"); @@ -150,7 +155,8 @@ pub(crate) async fn get_content_route( fields(%client), )] pub(crate) async fn get_content_as_filename_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let user = body.sender_user.as_ref().expect("user is authenticated"); @@ -185,7 +191,8 @@ pub(crate) async fn get_content_as_filename_route( fields(%client), )] pub(crate) async fn get_media_preview_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -223,7 +230,11 @@ pub(crate) async fn get_media_preview_route( } async fn fetch_thumbnail( - services: &Services, mxc: &Mxc<'_>, user: &UserId, timeout_ms: Duration, dim: &Dim, + services: &Services, + mxc: &Mxc<'_>, + user: &UserId, + timeout_ms: Duration, + dim: &Dim, ) -> Result { let FileMeta { content, @@ -245,7 +256,11 @@ async fn fetch_thumbnail( } async fn fetch_file( - services: &Services, mxc: &Mxc<'_>, user: &UserId, timeout_ms: Duration, filename: Option<&str>, + services: &Services, + mxc: &Mxc<'_>, + user: &UserId, + timeout_ms: Duration, + filename: Option<&str>, ) -> Result { let FileMeta { content, @@ -267,7 +282,11 @@ async fn fetch_file( } async fn fetch_thumbnail_meta( - services: &Services, mxc: &Mxc<'_>, user: &UserId, timeout_ms: Duration, dim: &Dim, + services: &Services, + mxc: &Mxc<'_>, + user: &UserId, + timeout_ms: Duration, + dim: &Dim, ) -> Result { if let Some(filemeta) = services.media.get_thumbnail(mxc, dim).await? { return Ok(filemeta); @@ -283,7 +302,12 @@ async fn fetch_thumbnail_meta( .await } -async fn fetch_file_meta(services: &Services, mxc: &Mxc<'_>, user: &UserId, timeout_ms: Duration) -> Result { +async fn fetch_file_meta( + services: &Services, + mxc: &Mxc<'_>, + user: &UserId, + timeout_ms: Duration, +) -> Result { if let Some(filemeta) = services.media.get(mxc).await? { return Ok(filemeta); } diff --git a/src/api/client/media_legacy.rs b/src/api/client/media_legacy.rs index a8b366ed..6f54a683 100644 --- a/src/api/client/media_legacy.rs +++ b/src/api/client/media_legacy.rs @@ -11,8 +11,8 @@ use conduwuit_service::media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROS use reqwest::Url; use ruma::{ api::client::media::{ - create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, - get_media_preview, + create_content, get_content, get_content_as_filename, get_content_thumbnail, + get_media_config, get_media_preview, }, Mxc, }; @@ -23,7 +23,8 @@ use crate::{client::create_content_route, Ruma, RumaResponse}; /// /// Returns max upload size. pub(crate) async fn get_media_config_legacy_route( - State(services): State, _body: Ruma, + State(services): State, + _body: Ruma, ) -> Result { Ok(get_media_config::v3::Response { upload_size: ruma_from_usize(services.globals.config.max_request_size), @@ -38,7 +39,8 @@ pub(crate) async fn get_media_config_legacy_route( /// /// Returns max upload size. pub(crate) async fn get_media_config_legacy_legacy_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result> { get_media_config_legacy_route(State(services), body) .await @@ -50,7 +52,8 @@ pub(crate) async fn get_media_config_legacy_legacy_route( /// Returns URL preview. #[tracing::instrument(skip_all, fields(%client), name = "url_preview_legacy")] pub(crate) async fn get_media_preview_legacy_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -91,7 +94,8 @@ pub(crate) async fn get_media_preview_legacy_route( /// /// Returns URL preview. pub(crate) async fn get_media_preview_legacy_legacy_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { get_media_preview_legacy_route(State(services), InsecureClientIp(client), body) @@ -110,7 +114,8 @@ pub(crate) async fn get_media_preview_legacy_legacy_route( /// - Some metadata will be saved in the database /// - Media will be saved in the media/ directory pub(crate) async fn create_content_legacy_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { create_content_route(State(services), InsecureClientIp(client), body) @@ -128,7 +133,8 @@ pub(crate) async fn create_content_legacy_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")] pub(crate) async fn get_content_legacy_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let mxc = Mxc { @@ -142,7 +148,8 @@ pub(crate) async fn get_content_legacy_route( content_disposition, }) = services.media.get(&mxc).await? { - let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + let content_disposition = + make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); Ok(get_content::v3::Response { file: content.expect("entire file contents"), @@ -156,10 +163,15 @@ pub(crate) async fn get_content_legacy_route( .media .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) .await - .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; + .map_err(|e| { + err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))) + })?; - let content_disposition = - make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); + let content_disposition = make_content_disposition( + response.content_disposition.as_ref(), + response.content_type.as_deref(), + None, + ); Ok(get_content::v3::Response { file: response.file, @@ -187,7 +199,8 @@ pub(crate) async fn get_content_legacy_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")] pub(crate) async fn get_content_legacy_legacy_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { get_content_legacy_route(State(services), InsecureClientIp(client), body) @@ -205,7 +218,8 @@ pub(crate) async fn get_content_legacy_legacy_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")] pub(crate) async fn get_content_as_filename_legacy_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let mxc = Mxc { @@ -219,8 +233,11 @@ pub(crate) async fn get_content_as_filename_legacy_route( content_disposition, }) = services.media.get(&mxc).await? { - let content_disposition = - make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), Some(&body.filename)); + let content_disposition = make_content_disposition( + content_disposition.as_ref(), + content_type.as_deref(), + Some(&body.filename), + ); Ok(get_content_as_filename::v3::Response { file: content.expect("entire file contents"), @@ -234,10 +251,15 @@ pub(crate) async fn get_content_as_filename_legacy_route( .media .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) .await - .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; + .map_err(|e| { + err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))) + })?; - let content_disposition = - make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); + let content_disposition = make_content_disposition( + response.content_disposition.as_ref(), + response.content_type.as_deref(), + None, + ); Ok(get_content_as_filename::v3::Response { content_disposition: Some(content_disposition), @@ -264,7 +286,8 @@ pub(crate) async fn get_content_as_filename_legacy_route( /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds pub(crate) async fn get_content_as_filename_legacy_legacy_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { get_content_as_filename_legacy_route(State(services), InsecureClientIp(client), body) @@ -282,7 +305,8 @@ pub(crate) async fn get_content_as_filename_legacy_legacy_route( /// seconds #[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get_legacy")] pub(crate) async fn get_content_thumbnail_legacy_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let mxc = Mxc { @@ -297,7 +321,8 @@ pub(crate) async fn get_content_thumbnail_legacy_route( content_disposition, }) = services.media.get_thumbnail(&mxc, &dim).await? { - let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + let content_disposition = + make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); Ok(get_content_thumbnail::v3::Response { file: content.expect("entire file contents"), @@ -311,10 +336,15 @@ pub(crate) async fn get_content_thumbnail_legacy_route( .media .fetch_remote_thumbnail_legacy(&body) .await - .map_err(|e| err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))))?; + .map_err(|e| { + err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))) + })?; - let content_disposition = - make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); + let content_disposition = make_content_disposition( + response.content_disposition.as_ref(), + response.content_type.as_deref(), + None, + ); Ok(get_content_thumbnail::v3::Response { file: response.file, @@ -341,7 +371,8 @@ pub(crate) async fn get_content_thumbnail_legacy_route( /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds pub(crate) async fn get_content_thumbnail_legacy_legacy_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { get_content_thumbnail_legacy_route(State(services), InsecureClientIp(client), body) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 02166271..0ddcab32 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -20,7 +20,8 @@ use ruma::{ client::{ error::ErrorKind, membership::{ - ban_user, forget_room, get_member_events, invite_user, join_room_by_id, join_room_by_id_or_alias, + ban_user, forget_room, get_member_events, invite_user, join_room_by_id, + join_room_by_id_or_alias, joined_members::{self, v3::RoomMember}, joined_rooms, kick_user, leave_room, unban_user, ThirdPartySigned, }, @@ -36,8 +37,8 @@ use ruma::{ }, StateEventType, }, - state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, - RoomVersionId, ServerName, UserId, + state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedRoomId, OwnedServerName, + OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use service::{ appservice::RegistrationInfo, @@ -54,7 +55,10 @@ use crate::{client::full_user_deactivate, Ruma}; /// enabled #[tracing::instrument(skip(services))] async fn banned_room_check( - services: &Services, user_id: &UserId, room_id: Option<&RoomId>, server_name: Option<&ServerName>, + services: &Services, + user_id: &UserId, + room_id: Option<&RoomId>, + server_name: Option<&ServerName>, client_ip: IpAddr, ) -> Result { if services.users.is_admin(user_id).await { @@ -70,19 +74,21 @@ async fn banned_room_check( .contains(&room_id.server_name().unwrap().to_owned()) { warn!( - "User {user_id} who is not an admin attempted to send an invite for or attempted to join a banned \ - room or banned room server name: {room_id}" + "User {user_id} who is not an admin attempted to send an invite for or \ + attempted to join a banned room or banned room server name: {room_id}" ); if services.globals.config.auto_deactivate_banned_room_attempts { - warn!("Automatically deactivating user {user_id} due to attempted banned room join"); + warn!( + "Automatically deactivating user {user_id} due to attempted banned room join" + ); if services.globals.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::text_plain(format!( - "Automatically deactivating user {user_id} due to attempted banned room join from IP \ - {client_ip}" + "Automatically deactivating user {user_id} due to attempted banned \ + room join from IP {client_ip}" ))) .await .ok(); @@ -109,19 +115,21 @@ async fn banned_room_check( .contains(&server_name.to_owned()) { warn!( - "User {user_id} who is not an admin tried joining a room which has the server name {server_name} that \ - is globally forbidden. Rejecting.", + "User {user_id} who is not an admin tried joining a room which has the server \ + name {server_name} that is globally forbidden. Rejecting.", ); if services.globals.config.auto_deactivate_banned_room_attempts { - warn!("Automatically deactivating user {user_id} due to attempted banned room join"); + warn!( + "Automatically deactivating user {user_id} due to attempted banned room join" + ); if services.globals.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::text_plain(format!( - "Automatically deactivating user {user_id} due to attempted banned room join from IP \ - {client_ip}" + "Automatically deactivating user {user_id} due to attempted banned \ + room join from IP {client_ip}" ))) .await .ok(); @@ -155,12 +163,20 @@ async fn banned_room_check( /// federation #[tracing::instrument(skip_all, fields(%client), name = "join")] pub(crate) async fn join_room_by_id_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user(); - banned_room_check(&services, sender_user, Some(&body.room_id), body.room_id.server_name(), client).await?; + banned_room_check( + &services, + sender_user, + Some(&body.room_id), + body.room_id.server_name(), + client, + ) + .await?; // There is no body.server_name for /roomId/join let mut servers: Vec<_> = services @@ -216,7 +232,8 @@ pub(crate) async fn join_room_by_id_route( /// via room alias server name and room ID server name #[tracing::instrument(skip_all, fields(%client), name = "join")] pub(crate) async fn join_room_by_id_or_alias_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_deref().expect("user is authenticated"); @@ -224,8 +241,15 @@ pub(crate) async fn join_room_by_id_or_alias_route( let body = body.body; let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { - Ok(room_id) => { - banned_room_check(&services, sender_user, Some(&room_id), room_id.server_name(), client).await?; + | Ok(room_id) => { + banned_room_check( + &services, + sender_user, + Some(&room_id), + room_id.server_name(), + client, + ) + .await?; let mut servers = body.via.clone(); servers.extend( @@ -261,14 +285,21 @@ pub(crate) async fn join_room_by_id_or_alias_route( (servers, room_id) }, - Err(room_alias) => { + | Err(room_alias) => { let (room_id, mut servers) = services .rooms .alias .resolve_alias(&room_alias, Some(body.via.clone())) .await?; - banned_room_check(&services, sender_user, Some(&room_id), Some(room_alias.server_name()), client).await?; + banned_room_check( + &services, + sender_user, + Some(&room_id), + Some(room_alias.server_name()), + client, + ) + .await?; let addl_via_servers = services .rooms @@ -314,9 +345,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( .boxed() .await?; - Ok(join_room_by_id_or_alias::v3::Response { - room_id: join_room_response.room_id, - }) + Ok(join_room_by_id_or_alias::v3::Response { room_id: join_room_response.room_id }) } /// # `POST /_matrix/client/v3/rooms/{roomId}/leave` @@ -325,7 +354,8 @@ pub(crate) async fn join_room_by_id_or_alias_route( /// /// - This should always work if the user is currently joined. pub(crate) async fn leave_room_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { leave_room(&services, body.sender_user(), &body.room_id, body.reason.clone()).await?; @@ -337,7 +367,8 @@ pub(crate) async fn leave_room_route( /// Tries to send an invite event into the room. #[tracing::instrument(skip_all, fields(%client), name = "invite")] pub(crate) async fn invite_user_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -350,12 +381,16 @@ pub(crate) async fn invite_user_route( return Err!(Request(Forbidden("Invites are not allowed on this server."))); } - banned_room_check(&services, sender_user, Some(&body.room_id), body.room_id.server_name(), client).await?; + banned_room_check( + &services, + sender_user, + Some(&body.room_id), + body.room_id.server_name(), + client, + ) + .await?; - if let invite_user::v3::InvitationRecipient::UserId { - user_id, - } = &body.recipient - { + if let invite_user::v3::InvitationRecipient::UserId { user_id } = &body.recipient { let sender_ignored_recipient = services.users.user_is_ignored(sender_user, user_id); let recipient_ignored_by_sender = services.users.user_is_ignored(user_id, sender_user); @@ -363,7 +398,9 @@ pub(crate) async fn invite_user_route( join!(sender_ignored_recipient, recipient_ignored_by_sender); if sender_ignored_recipient { - return Err!(Request(Forbidden("You cannot invite users you have ignored to rooms."))); + return Err!(Request(Forbidden( + "You cannot invite users you have ignored to rooms." + ))); } if recipient_ignored_by_sender { @@ -386,7 +423,8 @@ pub(crate) async fn invite_user_route( /// /// Tries to send a kick event into the room. pub(crate) async fn kick_user_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; @@ -405,17 +443,14 @@ pub(crate) async fn kick_user_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - body.user_id.to_string(), - &RoomMemberEventContent { - membership: MembershipState::Leave, - reason: body.reason.clone(), - is_direct: None, - join_authorized_via_users_server: None, - third_party_invite: None, - ..event - }, - ), + PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { + membership: MembershipState::Leave, + reason: body.reason.clone(), + is_direct: None, + join_authorized_via_users_server: None, + third_party_invite: None, + ..event + }), body.sender_user(), &body.room_id, &state_lock, @@ -431,7 +466,8 @@ pub(crate) async fn kick_user_route( /// /// Tries to send a ban event into the room. pub(crate) async fn ban_user_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -452,19 +488,16 @@ pub(crate) async fn ban_user_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - body.user_id.to_string(), - &RoomMemberEventContent { - membership: MembershipState::Ban, - reason: body.reason.clone(), - displayname: None, // display name may be offensive - avatar_url: None, // avatar may be offensive - is_direct: None, - join_authorized_via_users_server: None, - third_party_invite: None, - ..current_member_content - }, - ), + PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { + membership: MembershipState::Ban, + reason: body.reason.clone(), + displayname: None, // display name may be offensive + avatar_url: None, // avatar may be offensive + is_direct: None, + join_authorized_via_users_server: None, + third_party_invite: None, + ..current_member_content + }), sender_user, &body.room_id, &state_lock, @@ -480,7 +513,8 @@ pub(crate) async fn ban_user_route( /// /// Tries to send an unban event into the room. pub(crate) async fn unban_user_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; @@ -502,17 +536,14 @@ pub(crate) async fn unban_user_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - body.user_id.to_string(), - &RoomMemberEventContent { - membership: MembershipState::Leave, - reason: body.reason.clone(), - join_authorized_via_users_server: None, - third_party_invite: None, - is_direct: None, - ..current_member_content - }, - ), + PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { + membership: MembershipState::Leave, + reason: body.reason.clone(), + join_authorized_via_users_server: None, + third_party_invite: None, + is_direct: None, + ..current_member_content + }), body.sender_user(), &body.room_id, &state_lock, @@ -534,7 +565,8 @@ pub(crate) async fn unban_user_route( /// Note: Other devices of the user have no way of knowing the room was /// forgotten, so this has to be called from every device pub(crate) async fn forget_room_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -559,7 +591,8 @@ pub(crate) async fn forget_room_route( /// /// Lists all rooms the user has joined. pub(crate) async fn joined_rooms_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { Ok(joined_rooms::v3::Response { joined_rooms: services @@ -579,7 +612,8 @@ pub(crate) async fn joined_rooms_route( /// /// - Only works if the user is currently joined pub(crate) async fn get_member_events_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -612,7 +646,8 @@ pub(crate) async fn get_member_events_route( /// - The sender user must be in the room /// - TODO: An appservice just needs a puppet joined pub(crate) async fn joined_members_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -631,25 +666,25 @@ pub(crate) async fn joined_members_route( .room_members(&body.room_id) .map(ToOwned::to_owned) .then(|user| async move { - ( - user.clone(), - RoomMember { - display_name: services.users.displayname(&user).await.ok(), - avatar_url: services.users.avatar_url(&user).await.ok(), - }, - ) + (user.clone(), RoomMember { + display_name: services.users.displayname(&user).await.ok(), + avatar_url: services.users.avatar_url(&user).await.ok(), + }) }) .collect() .await; - Ok(joined_members::v3::Response { - joined, - }) + Ok(joined_members::v3::Response { joined }) } pub async fn join_room_by_id_helper( - services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - third_party_signed: Option<&ThirdPartySigned>, appservice_info: &Option, + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + third_party_signed: Option<&ThirdPartySigned>, + appservice_info: &Option, ) -> Result { let state_lock = services.rooms.state.mutex.lock(room_id).await; @@ -671,9 +706,7 @@ pub async fn join_room_by_id_helper( .await { debug_warn!("{sender_user} is already joined in {room_id}"); - return Ok(join_room_by_id::v3::Response { - room_id: room_id.into(), - }); + return Ok(join_room_by_id::v3::Response { room_id: room_id.into() }); } if let Ok(membership) = services @@ -694,18 +727,35 @@ pub async fn join_room_by_id_helper( .server_in_room(services.globals.server_name(), room_id) .await; - let local_join = - server_in_room || servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); + let local_join = server_in_room + || servers.is_empty() + || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); if local_join { - join_room_by_id_helper_local(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) - .boxed() - .await?; + join_room_by_id_helper_local( + services, + sender_user, + room_id, + reason, + servers, + third_party_signed, + state_lock, + ) + .boxed() + .await?; } else { // Ask a remote server if we are not participating in this room - join_room_by_id_helper_remote(services, sender_user, room_id, reason, servers, third_party_signed, state_lock) - .boxed() - .await?; + join_room_by_id_helper_remote( + services, + sender_user, + room_id, + reason, + servers, + third_party_signed, + state_lock, + ) + .boxed() + .await?; } Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) @@ -713,12 +763,18 @@ pub async fn join_room_by_id_helper( #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote")] async fn join_room_by_id_helper_remote( - services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + _third_party_signed: Option<&ThirdPartySigned>, + state_lock: RoomMutexGuard, ) -> Result { info!("Joining {room_id} over federation."); - let (make_join_response, remote_server) = make_join_request(services, sender_user, room_id, servers).await?; + let (make_join_response, remote_server) = + make_join_request(services, sender_user, room_id, servers).await?; info!("make_join finished"); @@ -783,8 +839,8 @@ async fn join_room_by_id_helper_remote( // We keep the "event_id" in the pdu only in v1 or // v2 rooms match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => { + | RoomVersionId::V1 | RoomVersionId::V2 => {}, + | _ => { join_event_stub.remove("event_id"); }, }; @@ -799,7 +855,8 @@ async fn join_room_by_id_helper_remote( let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?; // Add event_id back - join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + join_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); // It has enough fields to be called a proper event now let mut join_event = join_event_stub; @@ -825,12 +882,16 @@ async fn join_room_by_id_helper_remote( if join_authorized_via_users_server.is_some() { if let Some(signed_raw) = &send_join_response.room_state.event { debug_info!( - "There is a signed event with join_authorized_via_users_server. This room is probably using \ - restricted joins. Adding signature to our event" + "There is a signed event with join_authorized_via_users_server. This room is \ + probably using restricted joins. Adding signature to our event" ); - let (signed_event_id, signed_value) = gen_event_id_canonical_json(signed_raw, &room_version_id) - .map_err(|e| err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))))?; + let (signed_event_id, signed_value) = + gen_event_id_canonical_json(signed_raw, &room_version_id).map_err(|e| { + err!(Request(BadJson(warn!( + "Could not convert event to canonical JSON: {e}" + )))) + })?; if signed_event_id != event_id { return Err!(Request(BadJson( @@ -840,15 +901,20 @@ async fn join_room_by_id_helper_remote( match signed_value["signatures"] .as_object() - .ok_or_else(|| err!(BadServerResponse(warn!("Server {remote_server} sent invalid signatures type")))) + .ok_or_else(|| { + err!(BadServerResponse(warn!( + "Server {remote_server} sent invalid signatures type" + ))) + }) .and_then(|e| { e.get(remote_server.as_str()).ok_or_else(|| { err!(BadServerResponse(warn!( - "Server {remote_server} did not send its signature for a restricted room" + "Server {remote_server} did not send its signature for a restricted \ + room" ))) }) }) { - Ok(signature) => { + | Ok(signature) => { join_event .get_mut("signatures") .expect("we created a valid pdu") @@ -856,10 +922,10 @@ async fn join_room_by_id_helper_remote( .expect("we created a valid pdu") .insert(remote_server.to_string(), signature.clone()); }, - Err(e) => { + | Err(e) => { warn!( - "Server {remote_server} sent invalid signature in send_join signatures for event \ - {signed_value:?}: {e:?}", + "Server {remote_server} sent invalid signature in send_join signatures \ + for event {signed_value:?}: {e:?}", ); }, } @@ -900,8 +966,8 @@ async fn join_room_by_id_helper_remote( .ready_filter_map(Result::ok) .fold(HashMap::new(), |mut state, (event_id, value)| async move { let pdu = match PduEvent::from_id_val(&event_id, value.clone()) { - Ok(pdu) => pdu, - Err(e) => { + | Ok(pdu) => pdu, + | Err(e) => { debug_warn!("Invalid PDU in send_join response: {e:?}: {value:#?}"); return state; }, @@ -937,7 +1003,9 @@ async fn join_room_by_id_helper_remote( .validate_and_add_event_id_no_fetch(pdu, &room_version_id) }) .ready_filter_map(Result::ok) - .ready_for_each(|(event_id, value)| services.rooms.outlier.add_pdu_outlier(&event_id, &value)) + .ready_for_each(|(event_id, value)| { + services.rooms.outlier.add_pdu_outlier(&event_id, &value); + }) .await; drop(cork); @@ -1031,29 +1099,38 @@ async fn join_room_by_id_helper_remote( #[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_local")] async fn join_room_by_id_helper_local( - services: &Services, sender_user: &UserId, room_id: &RoomId, reason: Option, servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, state_lock: RoomMutexGuard, + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + _third_party_signed: Option<&ThirdPartySigned>, + state_lock: RoomMutexGuard, ) -> Result { debug_info!("We can join locally"); let join_rules_event_content = services .rooms .state_accessor - .room_state_get_content::(room_id, &StateEventType::RoomJoinRules, "") + .room_state_get_content::( + room_id, + &StateEventType::RoomJoinRules, + "", + ) .await; let restriction_rooms = match join_rules_event_content { - Ok(RoomJoinRulesEventContent { + | Ok(RoomJoinRulesEventContent { join_rule: JoinRule::Restricted(restricted) | JoinRule::KnockRestricted(restricted), }) => restricted .allow .into_iter() .filter_map(|a| match a { - AllowRule::RoomMembership(r) => Some(r.room_id), - _ => None, + | AllowRule::RoomMembership(r) => Some(r.room_id), + | _ => None, }) .collect(), - _ => Vec::new(), + | _ => Vec::new(), }; let join_authorized_via_users_server: Option = { @@ -1073,10 +1150,12 @@ async fn join_room_by_id_helper_local( .state_cache .local_users_in_room(room_id) .filter(|user| { - services - .rooms - .state_accessor - .user_can_invite(room_id, user, sender_user, &state_lock) + services.rooms.state_accessor.user_can_invite( + room_id, + user, + sender_user, + &state_lock, + ) }) .boxed() .next() @@ -1112,13 +1191,18 @@ async fn join_room_by_id_helper_local( }; if restriction_rooms.is_empty() - && (servers.is_empty() || servers.len() == 1 && services.globals.server_is_ours(&servers[0])) + && (servers.is_empty() + || servers.len() == 1 && services.globals.server_is_ours(&servers[0])) { return Err(error); } - warn!("We couldn't do the join locally, maybe federation can help to satisfy the restricted join requirements"); - let Ok((make_join_response, remote_server)) = make_join_request(services, sender_user, room_id, servers).await + warn!( + "We couldn't do the join locally, maybe federation can help to satisfy the restricted \ + join requirements" + ); + let Ok((make_join_response, remote_server)) = + make_join_request(services, sender_user, room_id, servers).await else { return Err(error); }; @@ -1133,8 +1217,10 @@ async fn join_room_by_id_helper_local( )); } - let mut join_event_stub: CanonicalJsonObject = serde_json::from_str(make_join_response.event.get()) - .map_err(|e| err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")))?; + let mut join_event_stub: CanonicalJsonObject = + serde_json::from_str(make_join_response.event.get()).map_err(|e| { + err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")) + })?; let join_authorized_via_users_server = join_event_stub .get("content") @@ -1173,8 +1259,8 @@ async fn join_room_by_id_helper_local( // We keep the "event_id" in the pdu only in v1 or // v2 rooms match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => { + | RoomVersionId::V1 | RoomVersionId::V2 => {}, + | _ => { join_event_stub.remove("event_id"); }, }; @@ -1189,7 +1275,8 @@ async fn join_room_by_id_helper_local( let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?; // Add event_id back - join_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + join_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); // It has enough fields to be called a proper event now let join_event = join_event_stub; @@ -1211,8 +1298,10 @@ async fn join_room_by_id_helper_local( .await?; if let Some(signed_raw) = send_join_response.room_state.event { - let (signed_event_id, signed_value) = gen_event_id_canonical_json(&signed_raw, &room_version_id) - .map_err(|e| err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))))?; + let (signed_event_id, signed_value) = + gen_event_id_canonical_json(&signed_raw, &room_version_id).map_err(|e| { + err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))) + })?; if signed_event_id != event_id { return Err!(Request(BadJson( @@ -1234,9 +1323,13 @@ async fn join_room_by_id_helper_local( } async fn make_join_request( - services: &Services, sender_user: &UserId, room_id: &RoomId, servers: &[OwnedServerName], + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + servers: &[OwnedServerName], ) -> Result<(federation::membership::prepare_join_event::v1::Response, OwnedServerName)> { - let mut make_join_response_and_server = Err!(BadServerResponse("No server available to assist in joining.")); + let mut make_join_response_and_server = + Err!(BadServerResponse("No server available to assist in joining.")); let mut make_join_counter: usize = 0; let mut incompatible_room_version_count: usize = 0; @@ -1266,23 +1359,28 @@ async fn make_join_request( e.kind(), ErrorKind::IncompatibleRoomVersion { .. } | ErrorKind::UnsupportedRoomVersion ) { - incompatible_room_version_count = incompatible_room_version_count.saturating_add(1); + incompatible_room_version_count = + incompatible_room_version_count.saturating_add(1); } if incompatible_room_version_count > 15 { info!( - "15 servers have responded with M_INCOMPATIBLE_ROOM_VERSION or M_UNSUPPORTED_ROOM_VERSION, \ - assuming that conduwuit does not support the room version {room_id}: {e}" + "15 servers have responded with M_INCOMPATIBLE_ROOM_VERSION or \ + M_UNSUPPORTED_ROOM_VERSION, assuming that conduwuit does not support the \ + room version {room_id}: {e}" ); - make_join_response_and_server = Err!(BadServerResponse("Room version is not supported by Conduwuit")); + make_join_response_and_server = + Err!(BadServerResponse("Room version is not supported by Conduwuit")); return make_join_response_and_server; } if make_join_counter > 40 { warn!( - "40 servers failed to provide valid make_join response, assuming no server can assist in joining." + "40 servers failed to provide valid make_join response, assuming no server \ + can assist in joining." ); - make_join_response_and_server = Err!(BadServerResponse("No server available to assist in joining.")); + make_join_response_and_server = + Err!(BadServerResponse("No server available to assist in joining.")); return make_join_response_and_server; } } @@ -1298,11 +1396,18 @@ async fn make_join_request( } pub(crate) async fn invite_helper( - services: &Services, sender_user: &UserId, user_id: &UserId, room_id: &RoomId, reason: Option, + services: &Services, + sender_user: &UserId, + user_id: &UserId, + room_id: &RoomId, + reason: Option, is_direct: bool, ) -> Result { if !services.users.is_admin(sender_user).await && services.globals.block_non_admin_invites() { - info!("User {sender_user} is not an admin and attempted to send an invite to room {room_id}"); + info!( + "User {sender_user} is not an admin and attempted to send an invite to room \ + {room_id}" + ); return Err!(Request(Forbidden("Invites are not allowed on this server."))); } @@ -1339,31 +1444,30 @@ pub(crate) async fn invite_helper( let response = services .sending - .send_federation_request( - user_id.server_name(), - create_invite::v2::Request { - room_id: room_id.to_owned(), - event_id: (*pdu.event_id).to_owned(), - room_version: room_version_id.clone(), - event: services - .sending - .convert_to_outgoing_federation_event(pdu_json.clone()) - .await, - invite_room_state, - via: services - .rooms - .state_cache - .servers_route_via(room_id) - .await - .ok(), - }, - ) + .send_federation_request(user_id.server_name(), create_invite::v2::Request { + room_id: room_id.to_owned(), + event_id: (*pdu.event_id).to_owned(), + room_version: room_version_id.clone(), + event: services + .sending + .convert_to_outgoing_federation_event(pdu_json.clone()) + .await, + invite_room_state, + via: services + .rooms + .state_cache + .servers_route_via(room_id) + .await + .ok(), + }) .await?; // We do not add the event_id field to the pdu here because of signature and // hashes checks let (event_id, value) = gen_event_id_canonical_json(&response.event, &room_version_id) - .map_err(|e| err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))))?; + .map_err(|e| { + err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))) + })?; if pdu.event_id != event_id { return Err!(Request(BadJson( @@ -1379,14 +1483,18 @@ pub(crate) async fn invite_helper( ) .expect("CanonicalJson is valid json value"), ) - .map_err(|e| err!(Request(BadJson(warn!("Origin field in event is not a valid server name: {e}")))))?; + .map_err(|e| { + err!(Request(BadJson(warn!("Origin field in event is not a valid server name: {e}")))) + })?; let pdu_id = services .rooms .event_handler .handle_incoming_pdu(&origin, room_id, &event_id, value, true) .await? - .ok_or_else(|| err!(Request(InvalidParam("Could not accept incoming PDU as timeline event."))))?; + .ok_or_else(|| { + err!(Request(InvalidParam("Could not accept incoming PDU as timeline event."))) + })?; return services.sending.send_pdu_room(room_id, &pdu_id).await; } @@ -1456,7 +1564,12 @@ pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { } } -pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, reason: Option) -> Result<()> { +pub async fn leave_room( + services: &Services, + user_id: &UserId, + room_id: &RoomId, + reason: Option, +) -> Result<()> { //use conduwuit::utils::stream::OptionStream; use futures::TryFutureExt; @@ -1500,7 +1613,11 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, let Ok(event) = services .rooms .state_accessor - .room_state_get_content::(room_id, &StateEventType::RoomMember, user_id.as_str()) + .room_state_get_content::( + room_id, + &StateEventType::RoomMember, + user_id.as_str(), + ) .await else { // Fix for broken rooms @@ -1527,14 +1644,11 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - user_id.to_string(), - &RoomMemberEventContent { - membership: MembershipState::Leave, - reason, - ..event - }, - ), + PduBuilder::state(user_id.to_string(), &RoomMemberEventContent { + membership: MembershipState::Leave, + reason, + ..event + }), user_id, room_id, &state_lock, @@ -1545,8 +1659,13 @@ pub async fn leave_room(services: &Services, user_id: &UserId, room_id: &RoomId, Ok(()) } -async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &RoomId) -> Result<()> { - let mut make_leave_response_and_server = Err!(BadServerResponse("No server available to assist in leaving.")); +async fn remote_leave_room( + services: &Services, + user_id: &UserId, + room_id: &RoomId, +) -> Result<()> { + let mut make_leave_response_and_server = + Err!(BadServerResponse("No server available to assist in leaving.")); let invite_state = services .rooms @@ -1608,8 +1727,12 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room )); } - let mut leave_event_stub = serde_json::from_str::(make_leave_response.event.get()) - .map_err(|e| err!(BadServerResponse("Invalid make_leave event json received from server: {e:?}")))?; + let mut leave_event_stub = serde_json::from_str::( + make_leave_response.event.get(), + ) + .map_err(|e| { + err!(BadServerResponse("Invalid make_leave event json received from server: {e:?}")) + })?; // TODO: Is origin needed? leave_event_stub.insert( @@ -1627,8 +1750,8 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room // room v3 and above removed the "event_id" field from remote PDU format match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => { + | RoomVersionId::V1 | RoomVersionId::V2 => {}, + | _ => { leave_event_stub.remove("event_id"); }, }; @@ -1643,7 +1766,8 @@ async fn remote_leave_room(services: &Services, user_id: &UserId, room_id: &Room let event_id = pdu::gen_event_id(&leave_event_stub, &room_version_id)?; // Add event_id back - leave_event_stub.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + leave_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); // It has enough fields to be called a proper event now let leave_event = leave_event_stub; diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 32e0b1e3..93582de0 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -56,7 +56,8 @@ const LIMIT_DEFAULT: usize = 10; /// - Only works if the user is joined (TODO: always allow, but only show events /// where the user was joined, depending on `history_visibility`) pub(crate) async fn get_message_events_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender = body.sender(); let (sender_user, sender_device) = sender; @@ -69,8 +70,8 @@ pub(crate) async fn get_message_events_route( .map(str::parse) .transpose()? .unwrap_or_else(|| match body.dir { - Direction::Forward => PduCount::min(), - Direction::Backward => PduCount::max(), + | Direction::Forward => PduCount::min(), + | Direction::Backward => PduCount::max(), }); let to: Option = body.to.as_deref().map(str::parse).flat_ok(); @@ -81,10 +82,12 @@ pub(crate) async fn get_message_events_route( .unwrap_or(LIMIT_DEFAULT) .min(LIMIT_MAX); - services - .rooms - .lazy_loading - .lazy_load_confirm_delivery(sender_user, sender_device, room_id, from); + services.rooms.lazy_loading.lazy_load_confirm_delivery( + sender_user, + sender_device, + room_id, + from, + ); if matches!(body.dir, Direction::Backward) { services @@ -98,14 +101,14 @@ pub(crate) async fn get_message_events_route( } let it = match body.dir { - Direction::Forward => services + | Direction::Forward => services .rooms .timeline .pdus(Some(sender_user), room_id, Some(from)) .await? .boxed(), - Direction::Backward => services + | Direction::Backward => services .rooms .timeline .pdus_rev(Some(sender_user), room_id, Some(from)) @@ -141,10 +144,13 @@ pub(crate) async fn get_message_events_route( if !cfg!(feature = "element_hacks") { if let Some(next_token) = next_token { - services - .rooms - .lazy_loading - .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy, next_token); + services.rooms.lazy_loading.lazy_load_mark_sent( + sender_user, + sender_device, + room_id, + lazy, + next_token, + ); } } @@ -162,7 +168,11 @@ pub(crate) async fn get_message_events_route( }) } -async fn get_member_event(services: &Services, room_id: &RoomId, user_id: &UserId) -> Option> { +async fn get_member_event( + services: &Services, + room_id: &RoomId, + user_id: &UserId, +) -> Option> { services .rooms .state_accessor @@ -173,7 +183,11 @@ async fn get_member_event(services: &Services, room_id: &RoomId, user_id: &UserI } pub(crate) async fn update_lazy( - services: &Services, room_id: &RoomId, sender: (&UserId, &DeviceId), mut lazy: LazySet, item: &PdusIterItem, + services: &Services, + room_id: &RoomId, + sender: (&UserId, &DeviceId), + mut lazy: LazySet, + item: &PdusIterItem, force: bool, ) -> LazySet { let (_, event) = &item; @@ -204,7 +218,11 @@ pub(crate) async fn update_lazy( lazy } -pub(crate) async fn ignored_filter(services: &Services, item: PdusIterItem, user_id: &UserId) -> Option { +pub(crate) async fn ignored_filter( + services: &Services, + item: PdusIterItem, + user_id: &UserId, +) -> Option { let (_, pdu) = &item; // exclude Synapse's dummy events from bloating up response bodies. clients @@ -223,7 +241,9 @@ pub(crate) async fn ignored_filter(services: &Services, item: PdusIterItem, user } pub(crate) async fn visibility_filter( - services: &Services, item: PdusIterItem, user_id: &UserId, + services: &Services, + item: PdusIterItem, + user_id: &UserId, ) -> Option { let (_, pdu) = &item; diff --git a/src/api/client/openid.rs b/src/api/client/openid.rs index b8bbfb91..4b2ff727 100644 --- a/src/api/client/openid.rs +++ b/src/api/client/openid.rs @@ -16,7 +16,8 @@ use crate::{Error, Result, Ruma}; /// /// - The token generated is only valid for the OpenID API pub(crate) async fn create_openid_token_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/api/client/presence.rs b/src/api/client/presence.rs index 948d6caa..1a3ad26e 100644 --- a/src/api/client/presence.rs +++ b/src/api/client/presence.rs @@ -12,10 +12,14 @@ use crate::{Error, Result, Ruma}; /// /// Sets the presence state of the sender user. pub(crate) async fn set_presence_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.globals.allow_local_presence() { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Presence is disabled on this server")); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Presence is disabled on this server", + )); } let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -40,10 +44,14 @@ pub(crate) async fn set_presence_route( /// /// - Only works if you share a room with the user pub(crate) async fn get_presence_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.globals.allow_local_presence() { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Presence is disabled on this server")); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Presence is disabled on this server", + )); } let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index ea74cdaf..584adfc1 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -11,7 +11,9 @@ use ruma::{ api::{ client::{ error::ErrorKind, - profile::{get_avatar_url, get_display_name, get_profile, set_avatar_url, set_display_name}, + profile::{ + get_avatar_url, get_display_name, get_profile, set_avatar_url, set_display_name, + }, }, federation, }, @@ -29,7 +31,8 @@ use crate::Ruma; /// /// - Also makes sure other users receive the update using presence EDUs pub(crate) async fn set_displayname_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -45,7 +48,8 @@ pub(crate) async fn set_displayname_route( .collect() .await; - update_displayname(&services, &body.user_id, body.displayname.clone(), &all_joined_rooms).await; + update_displayname(&services, &body.user_id, body.displayname.clone(), &all_joined_rooms) + .await; if services.globals.allow_local_presence() { // Presence update @@ -65,7 +69,8 @@ pub(crate) async fn set_displayname_route( /// - If user is on another server and we do not have a local copy already fetch /// displayname over federation pub(crate) async fn get_displayname_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.globals.user_is_local(&body.user_id) { // Create and update our local copy of the user @@ -94,9 +99,7 @@ pub(crate) async fn get_displayname_route( .users .set_blurhash(&body.user_id, response.blurhash.clone()); - return Ok(get_display_name::v3::Response { - displayname: response.displayname, - }); + return Ok(get_display_name::v3::Response { displayname: response.displayname }); } } @@ -117,7 +120,8 @@ pub(crate) async fn get_displayname_route( /// /// - Also makes sure other users receive the update using presence EDUs pub(crate) async fn set_avatar_url_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -161,7 +165,8 @@ pub(crate) async fn set_avatar_url_route( /// - If user is on another server and we do not have a local copy already fetch /// `avatar_url` and blurhash over federation pub(crate) async fn get_avatar_url_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.globals.user_is_local(&body.user_id) { // Create and update our local copy of the user @@ -218,7 +223,8 @@ pub(crate) async fn get_avatar_url_route( /// - If user is on another server and we do not have a local copy already, /// fetch profile over federation. pub(crate) async fn get_profile_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.globals.user_is_local(&body.user_id) { // Create and update our local copy of the user @@ -254,9 +260,11 @@ pub(crate) async fn get_profile_route( .set_timezone(&body.user_id, response.tz.clone()); for (profile_key, profile_key_value) in &response.custom_profile_fields { - services - .users - .set_profile_key(&body.user_id, profile_key, Some(profile_key_value.clone())); + services.users.set_profile_key( + &body.user_id, + profile_key, + Some(profile_key_value.clone()), + ); } return Ok(get_profile::v3::Response { @@ -295,7 +303,10 @@ pub(crate) async fn get_profile_route( } pub async fn update_displayname( - services: &Services, user_id: &UserId, displayname: Option, all_joined_rooms: &[OwnedRoomId], + services: &Services, + user_id: &UserId, + displayname: Option, + all_joined_rooms: &[OwnedRoomId], ) { let (current_avatar_url, current_blurhash, current_displayname) = join3( services.users.avatar_url(user_id), @@ -322,19 +333,16 @@ pub async fn update_displayname( .iter() .try_stream() .and_then(|room_id: &OwnedRoomId| async move { - let pdu = PduBuilder::state( - user_id.to_string(), - &RoomMemberEventContent { - displayname: displayname.clone(), - membership: MembershipState::Join, - avatar_url: avatar_url.clone(), - blurhash: blurhash.clone(), - join_authorized_via_users_server: None, - reason: None, - is_direct: None, - third_party_invite: None, - }, - ); + let pdu = PduBuilder::state(user_id.to_string(), &RoomMemberEventContent { + displayname: displayname.clone(), + membership: MembershipState::Join, + avatar_url: avatar_url.clone(), + blurhash: blurhash.clone(), + join_authorized_via_users_server: None, + reason: None, + is_direct: None, + third_party_invite: None, + }); Ok((pdu, room_id)) }) @@ -346,7 +354,10 @@ pub async fn update_displayname( } pub async fn update_avatar_url( - services: &Services, user_id: &UserId, avatar_url: Option, blurhash: Option, + services: &Services, + user_id: &UserId, + avatar_url: Option, + blurhash: Option, all_joined_rooms: &[OwnedRoomId], ) { let (current_avatar_url, current_blurhash, current_displayname) = join3( @@ -375,19 +386,16 @@ pub async fn update_avatar_url( .iter() .try_stream() .and_then(|room_id: &OwnedRoomId| async move { - let pdu = PduBuilder::state( - user_id.to_string(), - &RoomMemberEventContent { - avatar_url: avatar_url.clone(), - blurhash: blurhash.clone(), - membership: MembershipState::Join, - displayname: displayname.clone(), - join_authorized_via_users_server: None, - reason: None, - is_direct: None, - third_party_invite: None, - }, - ); + let pdu = PduBuilder::state(user_id.to_string(), &RoomMemberEventContent { + avatar_url: avatar_url.clone(), + blurhash: blurhash.clone(), + membership: MembershipState::Join, + displayname: displayname.clone(), + join_authorized_via_users_server: None, + reason: None, + is_direct: None, + third_party_invite: None, + }); Ok((pdu, room_id)) }) @@ -399,7 +407,9 @@ pub async fn update_avatar_url( } pub async fn update_all_rooms( - services: &Services, all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, user_id: &UserId, + services: &Services, + all_joined_rooms: Vec<(PduBuilder, &OwnedRoomId)>, + user_id: &UserId, ) { for (pdu_builder, room_id) in all_joined_rooms { let state_lock = services.rooms.state.mutex.lock(room_id).await; diff --git a/src/api/client/push.rs b/src/api/client/push.rs index e290c952..ed7371e4 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -4,15 +4,19 @@ use ruma::{ api::client::{ error::ErrorKind, push::{ - delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, get_pushrule_enabled, get_pushrules_all, - get_pushrules_global_scope, set_pusher, set_pushrule, set_pushrule_actions, set_pushrule_enabled, + delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, + get_pushrule_enabled, get_pushrules_all, get_pushrules_global_scope, set_pusher, + set_pushrule, set_pushrule_actions, set_pushrule_enabled, }, }, events::{ push_rules::{PushRulesEvent, PushRulesEventContent}, GlobalAccountDataEventType, }, - push::{InsertPushRuleError, PredefinedContentRuleId, PredefinedOverrideRuleId, RemovePushRuleError, Ruleset}, + push::{ + InsertPushRuleError, PredefinedContentRuleId, PredefinedOverrideRuleId, + RemovePushRuleError, Ruleset, + }, CanonicalJsonObject, CanonicalJsonValue, }; use service::Services; @@ -23,7 +27,8 @@ use crate::{Error, Result, Ruma}; /// /// Retrieves the push rules event for this user. pub(crate) async fn get_pushrules_all_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -40,8 +45,10 @@ pub(crate) async fn get_pushrules_all_route( return recreate_push_rules_and_return(&services, sender_user).await; }; - let account_data_content = serde_json::from_value::(content_value.into()) - .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; + let account_data_content = + serde_json::from_value::(content_value.into()).map_err(|e| { + err!(Database(warn!("Invalid push rules account data event in database: {e}"))) + })?; let mut global_ruleset = account_data_content.global; @@ -79,9 +86,7 @@ pub(crate) async fn get_pushrules_all_route( sender_user, GlobalAccountDataEventType::PushRules.to_string().into(), &serde_json::to_value(PushRulesEvent { - content: PushRulesEventContent { - global: global_ruleset.clone(), - }, + content: PushRulesEventContent { global: global_ruleset.clone() }, }) .expect("to json always works"), ) @@ -89,9 +94,7 @@ pub(crate) async fn get_pushrules_all_route( } }; - Ok(get_pushrules_all::v3::Response { - global: global_ruleset, - }) + Ok(get_pushrules_all::v3::Response { global: global_ruleset }) } /// # `GET /_matrix/client/r0/pushrules/global/` @@ -100,7 +103,8 @@ pub(crate) async fn get_pushrules_all_route( /// /// This appears to be the exact same as `GET /_matrix/client/r0/pushrules/`. pub(crate) async fn get_pushrules_global_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -134,8 +138,10 @@ pub(crate) async fn get_pushrules_global_route( }); }; - let account_data_content = serde_json::from_value::(content_value.into()) - .map_err(|e| err!(Database(warn!("Invalid push rules account data event in database: {e}"))))?; + let account_data_content = + serde_json::from_value::(content_value.into()).map_err(|e| { + err!(Database(warn!("Invalid push rules account data event in database: {e}"))) + })?; let mut global_ruleset = account_data_content.global; @@ -173,9 +179,7 @@ pub(crate) async fn get_pushrules_global_route( sender_user, GlobalAccountDataEventType::PushRules.to_string().into(), &serde_json::to_value(PushRulesEvent { - content: PushRulesEventContent { - global: global_ruleset.clone(), - }, + content: PushRulesEventContent { global: global_ruleset.clone() }, }) .expect("to json always works"), ) @@ -183,16 +187,15 @@ pub(crate) async fn get_pushrules_global_route( } }; - Ok(get_pushrules_global_scope::v3::Response { - global: global_ruleset, - }) + Ok(get_pushrules_global_scope::v3::Response { global: global_ruleset }) } /// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}` /// /// Retrieves a single specified push rule for this user. pub(crate) async fn get_pushrule_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -218,9 +221,7 @@ pub(crate) async fn get_pushrule_route( .map(Into::into); if let Some(rule) = rule { - Ok(get_pushrule::v3::Response { - rule, - }) + Ok(get_pushrule::v3::Response { rule }) } else { Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")) } @@ -230,7 +231,8 @@ pub(crate) async fn get_pushrule_route( /// /// Creates a single specified push rule for this user. pub(crate) async fn set_pushrule_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let body = body.body; @@ -241,32 +243,33 @@ pub(crate) async fn set_pushrule_route( .await .map_err(|_| err!(Request(NotFound("PushRules event not found."))))?; - if let Err(error) = - account_data - .content - .global - .insert(body.rule.clone(), body.after.as_deref(), body.before.as_deref()) - { + if let Err(error) = account_data.content.global.insert( + body.rule.clone(), + body.after.as_deref(), + body.before.as_deref(), + ) { let err = match error { - InsertPushRuleError::ServerDefaultRuleId => Error::BadRequest( + | InsertPushRuleError::ServerDefaultRuleId => Error::BadRequest( ErrorKind::InvalidParam, "Rule IDs starting with a dot are reserved for server-default rules.", ), - InsertPushRuleError::InvalidRuleId => { - Error::BadRequest(ErrorKind::InvalidParam, "Rule ID containing invalid characters.") - }, - InsertPushRuleError::RelativeToServerDefaultRule => Error::BadRequest( + | InsertPushRuleError::InvalidRuleId => Error::BadRequest( + ErrorKind::InvalidParam, + "Rule ID containing invalid characters.", + ), + | InsertPushRuleError::RelativeToServerDefaultRule => Error::BadRequest( ErrorKind::InvalidParam, "Can't place a push rule relatively to a server-default rule.", ), - InsertPushRuleError::UnknownRuleId => { - Error::BadRequest(ErrorKind::NotFound, "The before or after rule could not be found.") - }, - InsertPushRuleError::BeforeHigherThanAfter => Error::BadRequest( + | InsertPushRuleError::UnknownRuleId => Error::BadRequest( + ErrorKind::NotFound, + "The before or after rule could not be found.", + ), + | InsertPushRuleError::BeforeHigherThanAfter => Error::BadRequest( ErrorKind::InvalidParam, "The before rule has a higher priority than the after rule.", ), - _ => Error::BadRequest(ErrorKind::InvalidParam, "Invalid data."), + | _ => Error::BadRequest(ErrorKind::InvalidParam, "Invalid data."), }; return Err(err); @@ -289,7 +292,8 @@ pub(crate) async fn set_pushrule_route( /// /// Gets the actions of a single specified push rule for this user. pub(crate) async fn get_pushrule_actions_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -315,16 +319,15 @@ pub(crate) async fn get_pushrule_actions_route( .map(|rule| rule.actions().to_owned()) .ok_or_else(|| err!(Request(NotFound("Push rule not found."))))?; - Ok(get_pushrule_actions::v3::Response { - actions, - }) + Ok(get_pushrule_actions::v3::Response { actions }) } /// # `PUT /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/actions` /// /// Sets the actions of a single specified push rule for this user. pub(crate) async fn set_pushrule_actions_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -360,7 +363,8 @@ pub(crate) async fn set_pushrule_actions_route( /// /// Gets the enabled status of a single specified push rule for this user. pub(crate) async fn get_pushrule_enabled_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -370,9 +374,7 @@ pub(crate) async fn get_pushrule_enabled_route( || body.rule_id.as_str() == PredefinedOverrideRuleId::ContainsDisplayName.as_str() || body.rule_id.as_str() == PredefinedOverrideRuleId::RoomNotif.as_str() { - return Ok(get_pushrule_enabled::v3::Response { - enabled: false, - }); + return Ok(get_pushrule_enabled::v3::Response { enabled: false }); } let event: PushRulesEvent = services @@ -388,16 +390,15 @@ pub(crate) async fn get_pushrule_enabled_route( .map(ruma::push::AnyPushRuleRef::enabled) .ok_or_else(|| err!(Request(NotFound("Push rule not found."))))?; - Ok(get_pushrule_enabled::v3::Response { - enabled, - }) + Ok(get_pushrule_enabled::v3::Response { enabled }) } /// # `PUT /_matrix/client/r0/pushrules/global/{kind}/{ruleId}/enabled` /// /// Sets the enabled status of a single specified push rule for this user. pub(crate) async fn set_pushrule_enabled_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -433,7 +434,8 @@ pub(crate) async fn set_pushrule_enabled_route( /// /// Deletes a single specified push rule for this user. pub(crate) async fn delete_pushrule_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -449,11 +451,13 @@ pub(crate) async fn delete_pushrule_route( .remove(body.kind.clone(), &body.rule_id) { let err = match error { - RemovePushRuleError::ServerDefault => { - Error::BadRequest(ErrorKind::InvalidParam, "Cannot delete a server-default pushrule.") - }, - RemovePushRuleError::NotFound => Error::BadRequest(ErrorKind::NotFound, "Push rule not found."), - _ => Error::BadRequest(ErrorKind::InvalidParam, "Invalid data."), + | RemovePushRuleError::ServerDefault => Error::BadRequest( + ErrorKind::InvalidParam, + "Cannot delete a server-default pushrule.", + ), + | RemovePushRuleError::NotFound => + Error::BadRequest(ErrorKind::NotFound, "Push rule not found."), + | _ => Error::BadRequest(ErrorKind::InvalidParam, "Invalid data."), }; return Err(err); @@ -476,7 +480,8 @@ pub(crate) async fn delete_pushrule_route( /// /// Gets all currently active pushers for the sender user. pub(crate) async fn get_pushers_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -491,7 +496,8 @@ pub(crate) async fn get_pushers_route( /// /// - TODO: Handle `append` pub(crate) async fn set_pushers_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -506,7 +512,8 @@ pub(crate) async fn set_pushers_route( /// user somehow has bad push rules, these must always exist per spec. /// so recreate it and return server default silently async fn recreate_push_rules_and_return( - services: &Services, sender_user: &ruma::UserId, + services: &Services, + sender_user: &ruma::UserId, ) -> Result { services .account_data diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index d7f24101..2e98afbc 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -21,15 +21,14 @@ use crate::{Result, Ruma}; /// - If `read_receipt` is set: Update private marker and public read receipt /// EDU pub(crate) async fn set_read_marker_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); if let Some(event) = &body.fully_read { let fully_read_event = ruma::events::fully_read::FullyReadEvent { - content: ruma::events::fully_read::FullyReadEventContent { - event_id: event.clone(), - }, + content: ruma::events::fully_read::FullyReadEventContent { event_id: event.clone() }, }; services @@ -55,13 +54,10 @@ pub(crate) async fn set_read_marker_route( event.to_owned(), BTreeMap::from_iter([( ReceiptType::Read, - BTreeMap::from_iter([( - sender_user.to_owned(), - ruma::events::receipt::Receipt { - ts: Some(MilliSecondsSinceUnixEpoch::now()), - thread: ReceiptThread::Unthreaded, - }, - )]), + BTreeMap::from_iter([(sender_user.to_owned(), ruma::events::receipt::Receipt { + ts: Some(MilliSecondsSinceUnixEpoch::now()), + thread: ReceiptThread::Unthreaded, + })]), )]), )]); @@ -88,7 +84,9 @@ pub(crate) async fn set_read_marker_route( .map_err(|_| err!(Request(NotFound("Event not found."))))?; let PduCount::Normal(count) = count else { - return Err!(Request(InvalidParam("Event is a backfilled PDU and cannot be marked as read."))); + return Err!(Request(InvalidParam( + "Event is a backfilled PDU and cannot be marked as read." + ))); }; services @@ -104,7 +102,8 @@ pub(crate) async fn set_read_marker_route( /// /// Sets private read marker and public read receipt EDU. pub(crate) async fn create_receipt_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); @@ -119,7 +118,7 @@ pub(crate) async fn create_receipt_route( } match body.receipt_type { - create_receipt::v3::ReceiptType::FullyRead => { + | create_receipt::v3::ReceiptType::FullyRead => { let fully_read_event = ruma::events::fully_read::FullyReadEvent { content: ruma::events::fully_read::FullyReadEventContent { event_id: body.event_id.clone(), @@ -135,7 +134,7 @@ pub(crate) async fn create_receipt_route( ) .await?; }, - create_receipt::v3::ReceiptType::Read => { + | create_receipt::v3::ReceiptType::Read => { let receipt_content = BTreeMap::from_iter([( body.event_id.clone(), BTreeMap::from_iter([( @@ -163,7 +162,7 @@ pub(crate) async fn create_receipt_route( ) .await; }, - create_receipt::v3::ReceiptType::ReadPrivate => { + | create_receipt::v3::ReceiptType::ReadPrivate => { let count = services .rooms .timeline @@ -172,7 +171,9 @@ pub(crate) async fn create_receipt_route( .map_err(|_| err!(Request(NotFound("Event not found."))))?; let PduCount::Normal(count) = count else { - return Err!(Request(InvalidParam("Event is a backfilled PDU and cannot be marked as read."))); + return Err!(Request(InvalidParam( + "Event is a backfilled PDU and cannot be marked as read." + ))); }; services @@ -180,12 +181,11 @@ pub(crate) async fn create_receipt_route( .read_receipt .private_read_set(&body.room_id, sender_user, count); }, - _ => { + | _ => return Err!(Request(InvalidParam(warn!( "Received unknown read receipt type: {}", &body.receipt_type - )))) - }, + )))), } Ok(create_receipt::v3::Response {}) diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index a986dc18..0b01238f 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -1,5 +1,7 @@ use axum::extract::State; -use ruma::{api::client::redact::redact_event, events::room::redaction::RoomRedactionEventContent}; +use ruma::{ + api::client::redact::redact_event, events::room::redaction::RoomRedactionEventContent, +}; use crate::{service::pdu::PduBuilder, Result, Ruma}; @@ -9,7 +11,8 @@ use crate::{service::pdu::PduBuilder, Result, Ruma}; /// /// - TODO: Handle txn id pub(crate) async fn redact_event_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let body = body.body; @@ -35,7 +38,5 @@ pub(crate) async fn redact_event_route( drop(state_lock); - Ok(redact_event::v3::Response { - event_id: event_id.into(), - }) + Ok(redact_event::v3::Response { event_id: event_id.into() }) } diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index c47bc60f..87fb1eac 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -8,7 +8,8 @@ use futures::StreamExt; use ruma::{ api::{ client::relations::{ - get_relating_events, get_relating_events_with_rel_type, get_relating_events_with_rel_type_and_event_type, + get_relating_events, get_relating_events_with_rel_type, + get_relating_events_with_rel_type_and_event_type, }, Direction, }, @@ -21,7 +22,8 @@ use crate::Ruma; /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}` pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { paginate_relations_with_filter( &services, @@ -47,7 +49,8 @@ pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}/{relType}` pub(crate) async fn get_relating_events_with_rel_type_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { paginate_relations_with_filter( &services, @@ -73,7 +76,8 @@ pub(crate) async fn get_relating_events_with_rel_type_route( /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}` pub(crate) async fn get_relating_events_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { paginate_relations_with_filter( &services, @@ -93,16 +97,24 @@ pub(crate) async fn get_relating_events_route( #[allow(clippy::too_many_arguments)] async fn paginate_relations_with_filter( - services: &Services, sender_user: &UserId, room_id: &RoomId, target: &EventId, - filter_event_type: Option, filter_rel_type: Option, from: Option<&str>, - to: Option<&str>, limit: Option, recurse: bool, dir: Direction, + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + target: &EventId, + filter_event_type: Option, + filter_rel_type: Option, + from: Option<&str>, + to: Option<&str>, + limit: Option, + recurse: bool, + dir: Direction, ) -> Result { let start: PduCount = from .map(str::parse) .transpose()? .unwrap_or_else(|| match dir { - Direction::Forward => PduCount::min(), - Direction::Backward => PduCount::max(), + | Direction::Forward => PduCount::min(), + | Direction::Backward => PduCount::max(), }); let to: Option = to.map(str::parse).flat_ok(); @@ -115,11 +127,7 @@ async fn paginate_relations_with_filter( .min(100); // Spec (v1.10) recommends depth of at least 3 - let depth: u8 = if recurse { - 3 - } else { - 1 - }; + let depth: u8 = if recurse { 3 } else { 1 }; let events: Vec = services .rooms @@ -145,8 +153,8 @@ async fn paginate_relations_with_filter( .await; let next_batch = match dir { - Direction::Forward => events.last(), - Direction::Backward => events.first(), + | Direction::Forward => events.last(), + | Direction::Backward => events.first(), } .map(at!(0)) .as_ref() @@ -164,7 +172,11 @@ async fn paginate_relations_with_filter( }) } -async fn visibility_filter(services: &Services, sender_user: &UserId, item: PdusIterItem) -> Option { +async fn visibility_filter( + services: &Services, + sender_user: &UserId, + item: PdusIterItem, +) -> Option { let (_, pdu) = &item; services diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 429cd033..fe23b7bd 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -25,7 +25,8 @@ use crate::{ /// Reports an abusive room to homeserver admins #[tracing::instrument(skip_all, fields(%client), name = "report_room")] pub(crate) async fn report_room_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { // user authentication @@ -78,14 +79,16 @@ pub(crate) async fn report_room_route( /// Reports an inappropriate event to homeserver admins #[tracing::instrument(skip_all, fields(%client), name = "report_event")] pub(crate) async fn report_event_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { // user authentication let sender_user = body.sender_user.as_ref().expect("user is authenticated"); info!( - "Received event report by user {sender_user} for room {} and event ID {}, with reason: \"{}\"", + "Received event report by user {sender_user} for room {} and event ID {}, with reason: \ + \"{}\"", body.room_id, body.event_id, body.reason.as_deref().unwrap_or("") @@ -114,8 +117,8 @@ pub(crate) async fn report_event_route( services .admin .send_message(message::RoomMessageEventContent::text_markdown(format!( - "@room Event report received from {} -\n\nEvent ID: {}\nRoom ID: {}\nSent By: {}\n\nReport Score: \ - {}\nReport Reason: {}", + "@room Event report received from {} -\n\nEvent ID: {}\nRoom ID: {}\nSent By: \ + {}\n\nReport Score: {}\nReport Reason: {}", sender_user.to_owned(), pdu.event_id, pdu.room_id, @@ -136,10 +139,18 @@ pub(crate) async fn report_event_route( /// check if report reasoning is less than or equal to 750 characters /// check if reporting user is in the reporting room async fn is_event_report_valid( - services: &Services, event_id: &EventId, room_id: &RoomId, sender_user: &UserId, reason: Option<&String>, - score: Option, pdu: &PduEvent, + services: &Services, + event_id: &EventId, + room_id: &RoomId, + sender_user: &UserId, + reason: Option<&String>, + score: Option, + pdu: &PduEvent, ) -> Result<()> { - debug_info!("Checking if report from user {sender_user} for event {event_id} in room {room_id} is valid"); + debug_info!( + "Checking if report from user {sender_user} for event {event_id} in room {room_id} is \ + valid" + ); if room_id != pdu.room_id { return Err(Error::BadRequest( @@ -183,6 +194,9 @@ async fn is_event_report_valid( /// enumerating for potential events existing in our server. async fn delay_response() { let time_to_wait = rand::thread_rng().gen_range(2..5); - debug_info!("Got successful /report request, waiting {time_to_wait} seconds before sending successful response."); + debug_info!( + "Got successful /report request, waiting {time_to_wait} seconds before sending \ + successful response." + ); sleep(Duration::from_secs(time_to_wait)).await; } diff --git a/src/api/client/room/aliases.rs b/src/api/client/room/aliases.rs index f500e9c7..3f0016af 100644 --- a/src/api/client/room/aliases.rs +++ b/src/api/client/room/aliases.rs @@ -12,7 +12,8 @@ use crate::Ruma; /// - Only users joined to the room are allowed to call this, or if /// `history_visibility` is world readable in the room pub(crate) async fn get_room_aliases_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index b98f1dab..89cddc0f 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -24,7 +24,8 @@ use ruma::{ }, int, serde::{JsonObject, Raw}, - CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, RoomVersionId, + CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, + RoomVersionId, }; use serde_json::{json, value::to_raw_value}; use service::{appservice::RegistrationInfo, Services}; @@ -49,7 +50,8 @@ use crate::{client::invite_helper, Ruma}; /// - Send invite events #[allow(clippy::large_stack_frames)] pub(crate) async fn create_room_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { use create_room::v3::RoomPreset; @@ -59,7 +61,10 @@ pub(crate) async fn create_room_route( && body.appservice_info.is_none() && !services.users.is_admin(sender_user).await { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Room creation has been disabled.")); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Room creation has been disabled.", + )); } let room_id: OwnedRoomId = if let Some(custom_room_id) = &body.room_id { @@ -91,8 +96,8 @@ pub(crate) async fn create_room_route( services .admin .send_text(&format!( - "Non-admin user {sender_user} tried to publish {0} to the room directory while \ - \"lockdown_public_room_directory\" is enabled", + "Non-admin user {sender_user} tried to publish {0} to the room directory \ + while \"lockdown_public_room_directory\" is enabled", &room_id )) .await; @@ -115,7 +120,7 @@ pub(crate) async fn create_room_route( }; let room_version = match body.room_version.clone() { - Some(room_version) => { + | Some(room_version) => if services.server.supported_room_version(&room_version) { room_version } else { @@ -123,13 +128,12 @@ pub(crate) async fn create_room_route( ErrorKind::UnsupportedRoomVersion, "This server does not support that room version.", )); - } - }, - None => services.server.config.default_room_version.clone(), + }, + | None => services.server.config.default_room_version.clone(), }; let create_content = match &body.creation_content { - Some(content) => { + | Some(content) => { use RoomVersionId::*; let mut content = content @@ -139,7 +143,7 @@ pub(crate) async fn create_room_route( Error::bad_database("Failed to deserialise content as canonical JSON.") })?; match room_version { - V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { content.insert( "creator".into(), json!(&sender_user).try_into().map_err(|e| { @@ -148,24 +152,25 @@ pub(crate) async fn create_room_route( })?, ); }, - _ => { + | _ => { // V11+ removed the "creator" key }, } content.insert( "room_version".into(), - json!(room_version.as_str()) - .try_into() - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid creation content"))?, + json!(room_version.as_str()).try_into().map_err(|_| { + Error::BadRequest(ErrorKind::BadJson, "Invalid creation content") + })?, ); content }, - None => { + | None => { use RoomVersionId::*; let content = match room_version { - V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => RoomCreateEventContent::new_v1(sender_user.clone()), - _ => RoomCreateEventContent::new_v11(), + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => + RoomCreateEventContent::new_v1(sender_user.clone()), + | _ => RoomCreateEventContent::new_v11(), }; let mut content = serde_json::from_str::( to_raw_value(&content) @@ -190,7 +195,8 @@ pub(crate) async fn create_room_route( .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomCreate, - content: to_raw_value(&create_content).expect("create event content serialization"), + content: to_raw_value(&create_content) + .expect("create event content serialization"), state_key: Some(String::new()), ..Default::default() }, @@ -206,16 +212,13 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - sender_user.to_string(), - &RoomMemberEventContent { - displayname: services.users.displayname(sender_user).await.ok(), - avatar_url: services.users.avatar_url(sender_user).await.ok(), - blurhash: services.users.blurhash(sender_user).await.ok(), - is_direct: Some(body.is_direct), - ..RoomMemberEventContent::new(MembershipState::Join) - }, - ), + PduBuilder::state(sender_user.to_string(), &RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + is_direct: Some(body.is_direct), + ..RoomMemberEventContent::new(MembershipState::Join) + }), sender_user, &room_id, &state_lock, @@ -227,8 +230,8 @@ pub(crate) async fn create_room_route( // Figure out preset. We need it for preset specific events let preset = body.preset.clone().unwrap_or(match &body.visibility { - room::Visibility::Public => RoomPreset::PublicChat, - _ => RoomPreset::PrivateChat, // Room visibility should not be custom + | room::Visibility::Public => RoomPreset::PublicChat, + | _ => RoomPreset::PrivateChat, // Room visibility should not be custom }); let mut users = BTreeMap::from_iter([(sender_user.clone(), int!(100))]); @@ -236,7 +239,9 @@ pub(crate) async fn create_room_route( if preset == RoomPreset::TrustedPrivateChat { for invite in &body.invite { if services.users.user_is_ignored(sender_user, invite).await { - return Err!(Request(Forbidden("You cannot invite users you have ignored to rooms."))); + return Err!(Request(Forbidden( + "You cannot invite users you have ignored to rooms." + ))); } else if services.users.user_is_ignored(invite, sender_user).await { // silently drop the invite to the recipient if they've been ignored by the // sender, pretend it worked @@ -247,8 +252,11 @@ pub(crate) async fn create_room_route( } } - let power_levels_content = - default_power_levels_content(body.power_level_content_override.as_ref(), &body.visibility, users)?; + let power_levels_content = default_power_levels_content( + body.power_level_content_override.as_ref(), + &body.visibility, + users, + )?; services .rooms @@ -256,7 +264,8 @@ pub(crate) async fn create_room_route( .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&power_levels_content).expect("serialized power_levels event content"), + content: to_raw_value(&power_levels_content) + .expect("serialized power_levels event content"), state_key: Some(String::new()), ..Default::default() }, @@ -273,13 +282,10 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomCanonicalAliasEventContent { - alias: Some(room_alias_id.to_owned()), - alt_aliases: vec![], - }, - ), + PduBuilder::state(String::new(), &RoomCanonicalAliasEventContent { + alias: Some(room_alias_id.to_owned()), + alt_aliases: vec![], + }), sender_user, &room_id, &state_lock, @@ -298,9 +304,9 @@ pub(crate) async fn create_room_route( PduBuilder::state( String::new(), &RoomJoinRulesEventContent::new(match preset { - RoomPreset::PublicChat => JoinRule::Public, + | RoomPreset::PublicChat => JoinRule::Public, // according to spec "invite" is the default - _ => JoinRule::Invite, + | _ => JoinRule::Invite, }), ), sender_user, @@ -334,8 +340,8 @@ pub(crate) async fn create_room_route( PduBuilder::state( String::new(), &RoomGuestAccessEventContent::new(match preset { - RoomPreset::PublicChat => GuestAccess::Forbidden, - _ => GuestAccess::CanJoin, + | RoomPreset::PublicChat => GuestAccess::Forbidden, + | _ => GuestAccess::CanJoin, }), ), sender_user, @@ -367,7 +373,9 @@ pub(crate) async fn create_room_route( pdu_builder.state_key.get_or_insert_with(String::new); // Silently skip encryption events if they are not allowed - if pdu_builder.event_type == TimelineEventType::RoomEncryption && !services.globals.allow_encryption() { + if pdu_builder.event_type == TimelineEventType::RoomEncryption + && !services.globals.allow_encryption() + { continue; } @@ -399,12 +407,7 @@ pub(crate) async fn create_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomTopicEventContent { - topic: topic.clone(), - }, - ), + PduBuilder::state(String::new(), &RoomTopicEventContent { topic: topic.clone() }), sender_user, &room_id, &state_lock, @@ -417,16 +420,19 @@ pub(crate) async fn create_room_route( drop(state_lock); for user_id in &body.invite { if services.users.user_is_ignored(sender_user, user_id).await { - return Err!(Request(Forbidden("You cannot invite users you have ignored to rooms."))); + return Err!(Request(Forbidden( + "You cannot invite users you have ignored to rooms." + ))); } else if services.users.user_is_ignored(user_id, sender_user).await { // silently drop the invite to the recipient if they've been ignored by the // sender, pretend it worked continue; } - if let Err(e) = invite_helper(&services, sender_user, user_id, &room_id, None, body.is_direct) - .boxed() - .await + if let Err(e) = + invite_helper(&services, sender_user, user_id, &room_id, None, body.is_direct) + .boxed() + .await { warn!(%e, "Failed to send invite"); } @@ -446,7 +452,10 @@ pub(crate) async fn create_room_route( if services.globals.config.admin_room_notices { services .admin - .send_text(&format!("{sender_user} made {} public to the room directory", &room_id)) + .send_text(&format!( + "{sender_user} made {} public to the room directory", + &room_id + )) .await; } info!("{sender_user} made {0} public to the room directory", &room_id); @@ -459,21 +468,24 @@ pub(crate) async fn create_room_route( /// creates the power_levels_content for the PDU builder fn default_power_levels_content( - power_level_content_override: Option<&Raw>, visibility: &room::Visibility, + power_level_content_override: Option<&Raw>, + visibility: &room::Visibility, users: BTreeMap, ) -> Result { - let mut power_levels_content = serde_json::to_value(RoomPowerLevelsEventContent { - users, - ..Default::default() - }) - .expect("event is valid, we just created it"); + let mut power_levels_content = + serde_json::to_value(RoomPowerLevelsEventContent { users, ..Default::default() }) + .expect("event is valid, we just created it"); // secure proper defaults of sensitive/dangerous permissions that moderators // (power level 50) should not have easy access to - power_levels_content["events"]["m.room.power_levels"] = serde_json::to_value(100).expect("100 is valid Value"); - power_levels_content["events"]["m.room.server_acl"] = serde_json::to_value(100).expect("100 is valid Value"); - power_levels_content["events"]["m.room.tombstone"] = serde_json::to_value(100).expect("100 is valid Value"); - power_levels_content["events"]["m.room.encryption"] = serde_json::to_value(100).expect("100 is valid Value"); + power_levels_content["events"]["m.room.power_levels"] = + serde_json::to_value(100).expect("100 is valid Value"); + power_levels_content["events"]["m.room.server_acl"] = + serde_json::to_value(100).expect("100 is valid Value"); + power_levels_content["events"]["m.room.tombstone"] = + serde_json::to_value(100).expect("100 is valid Value"); + power_levels_content["events"]["m.room.encryption"] = + serde_json::to_value(100).expect("100 is valid Value"); power_levels_content["events"]["m.room.history_visibility"] = serde_json::to_value(100).expect("100 is valid Value"); @@ -481,14 +493,18 @@ fn default_power_levels_content( // useful in read-only announcement rooms that post a public poll. power_levels_content["events"]["org.matrix.msc3381.poll.response"] = serde_json::to_value(0).expect("0 is valid Value"); - power_levels_content["events"]["m.poll.response"] = serde_json::to_value(0).expect("0 is valid Value"); + power_levels_content["events"]["m.poll.response"] = + serde_json::to_value(0).expect("0 is valid Value"); // synapse does this too. clients do not expose these permissions. it prevents // default users from calling public rooms, for obvious reasons. if *visibility == room::Visibility::Public { - power_levels_content["events"]["m.call.invite"] = serde_json::to_value(50).expect("50 is valid Value"); - power_levels_content["events"]["m.call"] = serde_json::to_value(50).expect("50 is valid Value"); - power_levels_content["events"]["m.call.member"] = serde_json::to_value(50).expect("50 is valid Value"); + power_levels_content["events"]["m.call.invite"] = + serde_json::to_value(50).expect("50 is valid Value"); + power_levels_content["events"]["m.call"] = + serde_json::to_value(50).expect("50 is valid Value"); + power_levels_content["events"]["m.call.member"] = + serde_json::to_value(50).expect("50 is valid Value"); power_levels_content["events"]["org.matrix.msc3401.call"] = serde_json::to_value(50).expect("50 is valid Value"); power_levels_content["events"]["org.matrix.msc3401.call.member"] = @@ -497,7 +513,9 @@ fn default_power_levels_content( if let Some(power_level_content_override) = power_level_content_override { let json: JsonObject = serde_json::from_str(power_level_content_override.json().get()) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override."))?; + .map_err(|_| { + Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.") + })?; for (key, value) in json { power_levels_content[key] = value; @@ -509,14 +527,16 @@ fn default_power_levels_content( /// if a room is being created with a room alias, run our checks async fn room_alias_check( - services: &Services, room_alias_name: &str, appservice_info: Option<&RegistrationInfo>, + services: &Services, + room_alias_name: &str, + appservice_info: Option<&RegistrationInfo>, ) -> Result { // Basic checks on the room alias validity if room_alias_name.contains(':') { return Err(Error::BadRequest( ErrorKind::InvalidParam, - "Room alias contained `:` which is not allowed. Please note that this expects a localpart, not the full \ - room alias.", + "Room alias contained `:` which is not allowed. Please note that this expects a \ + localpart, not the full room alias.", )); } else if room_alias_name.contains(char::is_whitespace) { return Err(Error::BadRequest( @@ -534,8 +554,11 @@ async fn room_alias_check( return Err(Error::BadRequest(ErrorKind::Unknown, "Room alias name is forbidden.")); } - let full_room_alias = RoomAliasId::parse(format!("#{}:{}", room_alias_name, services.globals.config.server_name)) - .map_err(|e| { + let full_room_alias = RoomAliasId::parse(format!( + "#{}:{}", + room_alias_name, services.globals.config.server_name + )) + .map_err(|e| { info!("Failed to parse room alias {room_alias_name}: {e}"); Error::BadRequest(ErrorKind::InvalidParam, "Invalid room alias specified.") })?; @@ -552,14 +575,20 @@ async fn room_alias_check( if let Some(info) = appservice_info { if !info.aliases.is_match(full_room_alias.as_str()) { - return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias is not in namespace.")); + return Err(Error::BadRequest( + ErrorKind::Exclusive, + "Room alias is not in namespace.", + )); } } else if services .appservice .is_exclusive_alias(&full_room_alias) .await { - return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias reserved by appservice.")); + return Err(Error::BadRequest( + ErrorKind::Exclusive, + "Room alias reserved by appservice.", + )); } debug_info!("Full room alias: {full_room_alias}"); @@ -581,8 +610,8 @@ fn custom_room_id_check(services: &Services, custom_room_id: &str) -> Result Result, ref body: Ruma, + State(services): State, + ref body: Ruma, ) -> Result { let event = services .rooms @@ -47,7 +48,5 @@ pub(crate) async fn get_room_event_route( let event = event.to_room_event(); - Ok(get_room_event::v3::Response { - event, - }) + Ok(get_room_event::v3::Response { event }) } diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index 8b2e45df..cc3c9420 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -8,7 +8,8 @@ use crate::Ruma; const LIMIT_MAX: usize = 100; pub(crate) async fn room_initial_sync_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let room_id = &body.room_id; diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index 401bf800..cc6cca5e 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -43,7 +43,8 @@ const TRANSFERABLE_STATE_EVENTS: &[StateEventType; 9] = &[ /// - Moves local aliases /// - Modifies old room power levels to prevent users from speaking pub(crate) async fn upgrade_room_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -72,13 +73,10 @@ pub(crate) async fn upgrade_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomTombstoneEventContent { - body: "This room has been replaced".to_owned(), - replacement_room: replacement_room.clone(), - }, - ), + PduBuilder::state(String::new(), &RoomTombstoneEventContent { + body: "This room has been replaced".to_owned(), + replacement_room: replacement_room.clone(), + }), sender_user, &body.room_id, &state_lock, @@ -108,7 +106,7 @@ pub(crate) async fn upgrade_room_route( { use RoomVersionId::*; match body.new_version { - V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { create_event_content.insert( "creator".into(), json!(&sender_user).try_into().map_err(|e| { @@ -117,7 +115,7 @@ pub(crate) async fn upgrade_room_route( })?, ); }, - _ => { + | _ => { // "creator" key no longer exists in V11+ rooms create_event_content.remove("creator"); }, @@ -154,7 +152,8 @@ pub(crate) async fn upgrade_room_route( .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomCreate, - content: to_raw_value(&create_event_content).expect("event is valid, we just created it"), + content: to_raw_value(&create_event_content) + .expect("event is valid, we just created it"), unsigned: None, state_key: Some(String::new()), redacts: None, @@ -203,8 +202,8 @@ pub(crate) async fn upgrade_room_route( .room_state_get(&body.room_id, event_type, "") .await { - Ok(v) => v.content.clone(), - Err(_) => continue, // Skipping missing events. + | Ok(v) => v.content.clone(), + | Err(_) => continue, // Skipping missing events. }; services @@ -258,7 +257,9 @@ pub(crate) async fn upgrade_room_route( power_levels_event_content .users_default .checked_add(int!(1)) - .ok_or_else(|| err!(Request(BadJson("users_default power levels event content is not valid"))))?, + .ok_or_else(|| { + err!(Request(BadJson("users_default power levels event content is not valid"))) + })?, ); // Modify the power levels in the old room to prevent sending of events and @@ -267,14 +268,11 @@ pub(crate) async fn upgrade_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomPowerLevelsEventContent { - events_default: new_level, - invite: new_level, - ..power_levels_event_content - }, - ), + PduBuilder::state(String::new(), &RoomPowerLevelsEventContent { + events_default: new_level, + invite: new_level, + ..power_levels_event_content + }), sender_user, &body.room_id, &state_lock, @@ -284,7 +282,5 @@ pub(crate) async fn upgrade_room_route( drop(state_lock); // Return the replacement room id - Ok(upgrade_room::v3::Response { - replacement_room, - }) + Ok(upgrade_room::v3::Response { replacement_room }) } diff --git a/src/api/client/search.rs b/src/api/client/search.rs index 28a8891c..e60bd26d 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -35,7 +35,10 @@ const BATCH_MAX: usize = 20; /// /// - Only works if the user is currently joined to the room (TODO: Respect /// history visibility) -pub(crate) async fn search_events_route(State(services): State, body: Ruma) -> Result { +pub(crate) async fn search_events_route( + State(services): State, + body: Ruma, +) -> Result { let sender_user = body.sender_user(); let next_batch = body.next_batch.as_deref(); let room_events_result: OptionFuture<_> = body @@ -56,7 +59,10 @@ pub(crate) async fn search_events_route(State(services): State, bo #[allow(clippy::map_unwrap_or)] async fn category_room_events( - services: &Services, sender_user: &UserId, next_batch: Option<&str>, criteria: &Criteria, + services: &Services, + sender_user: &UserId, + next_batch: Option<&str>, + criteria: &Criteria, ) -> Result { let filter = &criteria.filter; @@ -186,11 +192,17 @@ async fn procure_room_state(services: &Services, room_id: &RoomId) -> Result Result { +async fn check_room_visible( + services: &Services, + user_id: &UserId, + room_id: &RoomId, + search: &Criteria, +) -> Result { let check_visible = search.filter.rooms.is_some(); let check_state = check_visible && search.include_state.is_some_and(is_true!()); - let is_joined = !check_visible || services.rooms.state_cache.is_joined(user_id, room_id).await; + let is_joined = + !check_visible || services.rooms.state_cache.is_joined(user_id, room_id).await; let state_visible = !check_state || services diff --git a/src/api/client/send.rs b/src/api/client/send.rs index e909ebbf..9afa7e8c 100644 --- a/src/api/client/send.rs +++ b/src/api/client/send.rs @@ -17,14 +17,17 @@ use crate::{service::pdu::PduBuilder, utils, Result, Ruma}; /// - Tries to send the event into the room, auth rules will determine if it is /// allowed pub(crate) async fn send_message_event_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user(); let sender_device = body.sender_device.as_deref(); let appservice_info = body.appservice_info.as_ref(); // Forbid m.room.encrypted if encryption is disabled - if MessageLikeEventType::RoomEncrypted == body.event_type && !services.globals.allow_encryption() { + if MessageLikeEventType::RoomEncrypted == body.event_type + && !services.globals.allow_encryption() + { return Err!(Request(Forbidden("Encryption has been disabled"))); } @@ -60,8 +63,8 @@ pub(crate) async fn send_message_event_route( let mut unsigned = BTreeMap::new(); unsigned.insert("transaction_id".to_owned(), body.txn_id.to_string().into()); - let content = - from_str(body.body.body.json().get()).map_err(|e| err!(Request(BadJson("Invalid JSON body: {e}"))))?; + let content = from_str(body.body.body.json().get()) + .map_err(|e| err!(Request(BadJson("Invalid JSON body: {e}"))))?; let event_id = services .rooms @@ -80,13 +83,14 @@ pub(crate) async fn send_message_event_route( ) .await?; - services - .transaction_ids - .add_txnid(sender_user, sender_device, &body.txn_id, event_id.as_bytes()); + services.transaction_ids.add_txnid( + sender_user, + sender_device, + &body.txn_id, + event_id.as_bytes(), + ); drop(state_lock); - Ok(send_message_event::v3::Response { - event_id: event_id.into(), - }) + Ok(send_message_event::v3::Response { event_id: event_id.into() }) } diff --git a/src/api/client/session.rs b/src/api/client/session.rs index e30c94d7..90cd4cb4 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -37,7 +37,8 @@ struct Claims { /// the `type` field when logging in. #[tracing::instrument(skip_all, fields(%client), name = "login")] pub(crate) async fn get_login_types_route( - InsecureClientIp(client): InsecureClientIp, _body: Ruma, + InsecureClientIp(client): InsecureClientIp, + _body: Ruma, ) -> Result { Ok(get_login_types::v3::Response::new(vec![ get_login_types::v3::LoginType::Password(PasswordLoginType::default()), @@ -61,13 +62,15 @@ pub(crate) async fn get_login_types_route( /// supported login types. #[tracing::instrument(skip_all, fields(%client), name = "login")] pub(crate) async fn login_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { // Validate login method // TODO: Other login methods let user_id = match &body.login_info { #[allow(deprecated)] - login::v3::LoginInfo::Password(login::v3::Password { + | login::v3::LoginInfo::Password(login::v3::Password { identifier, password, user, @@ -75,7 +78,10 @@ pub(crate) async fn login_route( }) => { debug!("Got password login type"); let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { - UserId::parse_with_server_name(user_id.to_lowercase(), services.globals.server_name()) + UserId::parse_with_server_name( + user_id.to_lowercase(), + services.globals.server_name(), + ) } else if let Some(user) = user { UserId::parse(user) } else { @@ -100,22 +106,29 @@ pub(crate) async fn login_route( user_id }, - login::v3::LoginInfo::Token(login::v3::Token { - token, - }) => { + | login::v3::LoginInfo::Token(login::v3::Token { token }) => { debug!("Got token login type"); if let Some(jwt_decoding_key) = services.globals.jwt_decoding_key() { - let token = - jsonwebtoken::decode::(token, jwt_decoding_key, &jsonwebtoken::Validation::default()) - .map_err(|e| { - warn!("Failed to parse JWT token from user logging in: {e}"); - Error::BadRequest(ErrorKind::InvalidUsername, "Token is invalid.") - })?; + let token = jsonwebtoken::decode::( + token, + jwt_decoding_key, + &jsonwebtoken::Validation::default(), + ) + .map_err(|e| { + warn!("Failed to parse JWT token from user logging in: {e}"); + Error::BadRequest(ErrorKind::InvalidUsername, "Token is invalid.") + })?; let username = token.claims.sub.to_lowercase(); - UserId::parse_with_server_name(username, services.globals.server_name()) - .map_err(|e| err!(Request(InvalidUsername(debug_error!(?e, "Failed to parse login username")))))? + UserId::parse_with_server_name(username, services.globals.server_name()).map_err( + |e| { + err!(Request(InvalidUsername(debug_error!( + ?e, + "Failed to parse login username" + )))) + }, + )? } else { return Err!(Request(Unknown( "Token login is not supported (server has no jwt decoding key)." @@ -123,13 +136,16 @@ pub(crate) async fn login_route( } }, #[allow(deprecated)] - login::v3::LoginInfo::ApplicationService(login::v3::ApplicationService { + | login::v3::LoginInfo::ApplicationService(login::v3::ApplicationService { identifier, user, }) => { debug!("Got appservice login type"); let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { - UserId::parse_with_server_name(user_id.to_lowercase(), services.globals.server_name()) + UserId::parse_with_server_name( + user_id.to_lowercase(), + services.globals.server_name(), + ) } else if let Some(user) = user { UserId::parse(user) } else { @@ -143,18 +159,27 @@ pub(crate) async fn login_route( if let Some(ref info) = body.appservice_info { if !info.is_user_match(&user_id) { - return Err(Error::BadRequest(ErrorKind::Exclusive, "User is not in namespace.")); + return Err(Error::BadRequest( + ErrorKind::Exclusive, + "User is not in namespace.", + )); } } else { - return Err(Error::BadRequest(ErrorKind::MissingToken, "Missing appservice token.")); + return Err(Error::BadRequest( + ErrorKind::MissingToken, + "Missing appservice token.", + )); } user_id }, - _ => { + | _ => { warn!("Unsupported or unknown login type: {:?}", &body.login_info); debug!("JSON body: {:?}", &body.json_body); - return Err(Error::BadRequest(ErrorKind::Unknown, "Unsupported or unknown login type.")); + return Err(Error::BadRequest( + ErrorKind::Unknown, + "Unsupported or unknown login type.", + )); }, }; @@ -233,7 +258,9 @@ pub(crate) async fn login_route( /// - Triggers device list updates #[tracing::instrument(skip_all, fields(%client), name = "logout")] pub(crate) async fn logout_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, body: Ruma, + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_ref().expect("user is authenticated"); @@ -261,7 +288,8 @@ pub(crate) async fn logout_route( /// user. #[tracing::instrument(skip_all, fields(%client), name = "logout")] pub(crate) async fn logout_all_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/api/client/space.rs b/src/api/client/space.rs index a3031a3a..409c9083 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -13,7 +13,8 @@ use crate::{service::rooms::spaces::PaginationToken, Error, Result, Ruma}; /// Paginates over the space tree in a depth-first manner to locate child rooms /// of a given space. pub(crate) async fn get_hierarchy_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 881eca98..6a65f0f9 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -31,7 +31,8 @@ use crate::{Ruma, RumaResponse}; /// allowed /// - If event is new `canonical_alias`: Rejects if alias is incorrect pub(crate) async fn send_state_event_for_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -63,7 +64,8 @@ pub(crate) async fn send_state_event_for_key_route( /// allowed /// - If event is new `canonical_alias`: Rejects if alias is incorrect pub(crate) async fn send_state_event_for_empty_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result> { send_state_event_for_key_route(State(services), body) .await @@ -77,7 +79,8 @@ pub(crate) async fn send_state_event_for_empty_key_route( /// - If not joined: Only works if current room history visibility is world /// readable pub(crate) async fn get_state_events_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -111,7 +114,8 @@ pub(crate) async fn get_state_events_route( /// - If not joined: Only works if current room history visibility is world /// readable pub(crate) async fn get_state_events_for_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -157,7 +161,8 @@ pub(crate) async fn get_state_events_for_key_route( /// - If not joined: Only works if current room history visibility is world /// readable pub(crate) async fn get_state_events_for_empty_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result> { get_state_events_for_key_route(State(services), body) .await @@ -165,8 +170,13 @@ pub(crate) async fn get_state_events_for_empty_key_route( } async fn send_state_event_for_key_helper( - services: &Services, sender: &UserId, room_id: &RoomId, event_type: &StateEventType, - json: &Raw, state_key: String, timestamp: Option, + services: &Services, + sender: &UserId, + room_id: &RoomId, + event_type: &StateEventType, + json: &Raw, + state_key: String, + timestamp: Option, ) -> Result> { allowed_to_send_state_event(services, room_id, event_type, json).await?; let state_lock = services.rooms.state.mutex.lock(room_id).await; @@ -191,20 +201,27 @@ async fn send_state_event_for_key_helper( } async fn allowed_to_send_state_event( - services: &Services, room_id: &RoomId, event_type: &StateEventType, json: &Raw, + services: &Services, + room_id: &RoomId, + event_type: &StateEventType, + json: &Raw, ) -> Result { match event_type { // Forbid m.room.encryption if encryption is disabled - StateEventType::RoomEncryption => { + | StateEventType::RoomEncryption => if !services.globals.allow_encryption() { - return Err(Error::BadRequest(ErrorKind::forbidden(), "Encryption has been disabled")); - } - }, + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "Encryption has been disabled", + )); + }, // admin room is a sensitive room, it should not ever be made public - StateEventType::RoomJoinRules => { + | StateEventType::RoomJoinRules => { if let Ok(admin_room_id) = services.admin.get_admin_room().await { if admin_room_id == room_id { - if let Ok(join_rule) = serde_json::from_str::(json.json().get()) { + if let Ok(join_rule) = + serde_json::from_str::(json.json().get()) + { if join_rule.join_rule == JoinRule::Public { return Err(Error::BadRequest( ErrorKind::forbidden(), @@ -216,16 +233,20 @@ async fn allowed_to_send_state_event( } }, // admin room is a sensitive room, it should not ever be made world readable - StateEventType::RoomHistoryVisibility => { + | StateEventType::RoomHistoryVisibility => { if let Ok(admin_room_id) = services.admin.get_admin_room().await { if admin_room_id == room_id { - if let Ok(visibility_content) = - serde_json::from_str::(json.json().get()) + if let Ok(visibility_content) = serde_json::from_str::< + RoomHistoryVisibilityEventContent, + >(json.json().get()) { - if visibility_content.history_visibility == HistoryVisibility::WorldReadable { + if visibility_content.history_visibility + == HistoryVisibility::WorldReadable + { return Err(Error::BadRequest( ErrorKind::forbidden(), - "Admin room is not allowed to be made world readable (public room history).", + "Admin room is not allowed to be made world readable (public \ + room history).", )); } } @@ -233,8 +254,10 @@ async fn allowed_to_send_state_event( } }, // TODO: allow alias if it previously existed - StateEventType::RoomCanonicalAlias => { - if let Ok(canonical_alias) = serde_json::from_str::(json.json().get()) { + | StateEventType::RoomCanonicalAlias => { + if let Ok(canonical_alias) = + serde_json::from_str::(json.json().get()) + { let mut aliases = canonical_alias.alt_aliases.clone(); if let Some(alias) = canonical_alias.alias { @@ -243,7 +266,9 @@ async fn allowed_to_send_state_event( for alias in aliases { if !services.globals.server_is_ours(alias.server_name()) { - return Err!(Request(Forbidden("canonical_alias must be for this server"))); + return Err!(Request(Forbidden( + "canonical_alias must be for this server" + ))); } if !services @@ -255,13 +280,14 @@ async fn allowed_to_send_state_event( // Make sure it's the right room { return Err!(Request(Forbidden( - "You are only allowed to send canonical_alias events when its aliases already exist" + "You are only allowed to send canonical_alias events when its \ + aliases already exist" ))); } } } }, - _ => (), + | _ => (), } Ok(()) diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 2b8d478c..b772fbf1 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -12,8 +12,12 @@ pub(crate) use self::{v3::sync_events_route, v4::sync_events_v4_route}; use crate::{service::Services, Error, PduEvent, Result}; async fn load_timeline( - services: &Services, sender_user: &UserId, room_id: &RoomId, roomsincecount: PduCount, - next_batch: Option, limit: usize, + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + roomsincecount: PduCount, + next_batch: Option, + limit: usize, ) -> Result<(Vec<(PduCount, PduEvent)>, bool), Error> { let last_timeline_count = services .rooms @@ -51,7 +55,10 @@ async fn load_timeline( } async fn share_encrypted_room( - services: &Services, sender_user: &UserId, user_id: &UserId, ignore_room: Option<&RoomId>, + services: &Services, + sender_user: &UserId, + user_id: &UserId, + ignore_room: Option<&RoomId>, ) -> bool { services .rooms diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index c5ec3886..a05bcf98 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -32,8 +32,9 @@ use ruma::{ sync::sync_events::{ self, v3::{ - Ephemeral, Filter, GlobalAccountData, InviteState, InvitedRoom, JoinedRoom, LeftRoom, Presence, - RoomAccountData, RoomSummary, Rooms, State as RoomState, Timeline, ToDevice, + Ephemeral, Filter, GlobalAccountData, InviteState, InvitedRoom, JoinedRoom, + LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms, State as RoomState, + Timeline, ToDevice, }, DeviceLists, UnreadNotificationsCount, }, @@ -107,7 +108,8 @@ type PresenceUpdates = HashMap; ) )] pub(crate) async fn sync_events_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result> { let (sender_user, sender_device) = body.sender(); @@ -127,9 +129,9 @@ pub(crate) async fn sync_events_route( // Load filter let filter = match body.body.filter.as_ref() { - None => FilterDefinition::default(), - Some(Filter::FilterDefinition(ref filter)) => filter.clone(), - Some(Filter::FilterId(ref filter_id)) => services + | None => FilterDefinition::default(), + | Some(Filter::FilterDefinition(ref filter)) => filter.clone(), + | Some(Filter::FilterId(ref filter_id)) => services .users .get_filter(sender_user, filter_id) .await @@ -138,11 +140,11 @@ pub(crate) async fn sync_events_route( // some clients, at least element, seem to require knowledge of redundant // members for "inline" profiles on the timeline to work properly - let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options { - LazyLoadOptions::Enabled { - include_redundant_members, - } => (true, include_redundant_members), - LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")), + let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options + { + | LazyLoadOptions::Enabled { include_redundant_members } => + (true, include_redundant_members), + | LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")), }; let full_state = body.body.full_state; @@ -230,9 +232,7 @@ pub(crate) async fn sync_events_route( } let invited_room = InvitedRoom { - invite_state: InviteState { - events: invite_state, - }, + invite_state: InviteState { events: invite_state }, }; invited_rooms.insert(room_id, invited_room); @@ -268,9 +268,10 @@ pub(crate) async fn sync_events_route( .count_one_time_keys(sender_user, sender_device); // Remove all to-device events the device received *last time* - let remove_to_device_events = services - .users - .remove_to_device_events(sender_user, sender_device, since); + let remove_to_device_events = + services + .users + .remove_to_device_events(sender_user, sender_device, since); let rooms = join3(joined_rooms, left_rooms, invited_rooms); let ephemeral = join3(remove_to_device_events, to_device_events, presence_updates); @@ -290,7 +291,8 @@ pub(crate) async fn sync_events_route( .into_iter() .stream() .broad_filter_map(|user_id| async move { - let no_shared_encrypted_room = !share_encrypted_room(&services, sender_user, &user_id, None).await; + let no_shared_encrypted_room = + !share_encrypted_room(&services, sender_user, &user_id, None).await; no_shared_encrypted_room.then_some(user_id) }) .ready_fold(HashSet::new(), |mut device_list_left, user_id| { @@ -300,9 +302,7 @@ pub(crate) async fn sync_events_route( .await; let response = sync_events::v3::Response { - account_data: GlobalAccountData { - events: account_data, - }, + account_data: GlobalAccountData { events: account_data }, device_lists: DeviceLists { changed: device_list_updates.into_iter().collect(), left: device_list_left.into_iter().collect(), @@ -324,9 +324,7 @@ pub(crate) async fn sync_events_route( invite: invited_rooms, knock: BTreeMap::new(), // TODO }, - to_device: ToDevice { - events: to_device_events, - }, + to_device: ToDevice { events: to_device_events }, }; // TODO: Retry the endpoint instead of returning @@ -348,7 +346,11 @@ pub(crate) async fn sync_events_route( } #[tracing::instrument(name = "presence", level = "debug", skip_all)] -async fn process_presence_updates(services: &Services, since: u64, syncing_user: &UserId) -> PresenceUpdates { +async fn process_presence_updates( + services: &Services, + since: u64, + syncing_user: &UserId, +) -> PresenceUpdates { services .presence .presence_since(since) @@ -367,10 +369,10 @@ async fn process_presence_updates(services: &Services, since: u64, syncing_user: }) .ready_fold(PresenceUpdates::new(), |mut updates, (user_id, event)| { match updates.entry(user_id.into()) { - Entry::Vacant(slot) => { + | Entry::Vacant(slot) => { slot.insert(event); }, - Entry::Occupied(mut slot) => { + | Entry::Occupied(mut slot) => { let curr_event = slot.get_mut(); let curr_content = &mut curr_event.content; let new_content = event.content; @@ -380,7 +382,8 @@ async fn process_presence_updates(services: &Services, since: u64, syncing_user: curr_content.status_msg = new_content .status_msg .or_else(|| curr_content.status_msg.take()); - curr_content.last_active_ago = new_content.last_active_ago.or(curr_content.last_active_ago); + curr_content.last_active_ago = + new_content.last_active_ago.or(curr_content.last_active_ago); curr_content.displayname = new_content .displayname .or_else(|| curr_content.displayname.take()); @@ -410,8 +413,13 @@ async fn process_presence_updates(services: &Services, since: u64, syncing_user: )] #[allow(clippy::too_many_arguments)] async fn handle_left_room( - services: &Services, since: u64, ref room_id: OwnedRoomId, sender_user: &UserId, next_batch_string: &str, - full_state: bool, lazy_load_enabled: bool, + services: &Services, + since: u64, + ref room_id: OwnedRoomId, + sender_user: &UserId, + next_batch_string: &str, + full_state: bool, + lazy_load_enabled: bool, ) -> Result> { // Get and drop the lock to wait for remaining operations to finish let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; @@ -440,7 +448,8 @@ async fn handle_left_room( .try_into() .expect("Timestamp is valid js_int value"), kind: RoomMember, - content: serde_json::from_str(r#"{"membership":"leave"}"#).expect("this is valid JSON"), + content: serde_json::from_str(r#"{"membership":"leave"}"#) + .expect("this is valid JSON"), state_key: Some(sender_user.to_string()), unsigned: None, // The following keys are dropped on conversion @@ -449,16 +458,12 @@ async fn handle_left_room( depth: uint!(1), auth_events: vec![], redacts: None, - hashes: EventHash { - sha256: String::new(), - }, + hashes: EventHash { sha256: String::new() }, signatures: None, }; return Ok(Some(LeftRoom { - account_data: RoomAccountData { - events: Vec::new(), - }, + account_data: RoomAccountData { events: Vec::new() }, timeline: Timeline { limited: false, prev_batch: Some(next_batch_string.to_owned()), @@ -479,8 +484,8 @@ async fn handle_left_room( .await; let since_state_ids = match since_shortstatehash { - Ok(s) => services.rooms.state_accessor.state_full_ids(s).await?, - Err(_) => HashMap::new(), + | Ok(s) => services.rooms.state_accessor.state_full_ids(s).await?, + | Err(_) => HashMap::new(), }; let Ok(left_event_id): Result = services @@ -542,17 +547,14 @@ async fn handle_left_room( } Ok(Some(LeftRoom { - account_data: RoomAccountData { - events: Vec::new(), - }, + account_data: RoomAccountData { events: Vec::new() }, timeline: Timeline { - limited: true, // TODO: support left timeline events so we dont need to set this to true + limited: true, /* TODO: support left timeline events so we dont need to set this to + * true */ prev_batch: Some(next_batch_string.to_owned()), events: Vec::new(), // and so we dont need to set this to empty vec }, - state: RoomState { - events: left_state_events, - }, + state: RoomState { events: left_state_events }, })) } @@ -566,8 +568,15 @@ async fn handle_left_room( )] #[allow(clippy::too_many_arguments)] async fn load_joined_room( - services: &Services, sender_user: &UserId, sender_device: &DeviceId, ref room_id: OwnedRoomId, since: u64, - next_batch: u64, lazy_load_enabled: bool, lazy_load_send_redundant: bool, full_state: bool, + services: &Services, + sender_user: &UserId, + sender_device: &DeviceId, + ref room_id: OwnedRoomId, + since: u64, + next_batch: u64, + lazy_load_enabled: bool, + lazy_load_send_redundant: bool, + full_state: bool, ) -> Result<(JoinedRoom, HashSet, HashSet)> { // Get and drop the lock to wait for remaining operations to finish // This will make sure the we have all events until next_batch @@ -590,18 +599,26 @@ async fn load_joined_room( .ok() .map(Ok); - let timeline = load_timeline(services, sender_user, room_id, sincecount, Some(next_batchcount), 10_usize); + let timeline = load_timeline( + services, + sender_user, + room_id, + sincecount, + Some(next_batchcount), + 10_usize, + ); let (current_shortstatehash, since_shortstatehash, timeline) = try_join3(current_shortstatehash, since_shortstatehash, timeline).await?; let (timeline_pdus, limited) = timeline; - let timeline_users = timeline_pdus - .iter() - .fold(HashSet::new(), |mut timeline_users, (_, event)| { - timeline_users.insert(event.sender.as_str().to_owned()); - timeline_users - }); + let timeline_users = + timeline_pdus + .iter() + .fold(HashSet::new(), |mut timeline_users, (_, event)| { + timeline_users.insert(event.sender.as_str().to_owned()); + timeline_users + }); let last_notification_read: OptionFuture<_> = timeline_pdus .is_empty() @@ -617,13 +634,16 @@ async fn load_joined_room( .is_none_or(|&count| count > since) .await; - services - .rooms - .lazy_loading - .lazy_load_confirm_delivery(sender_user, sender_device, room_id, sincecount); + services.rooms.lazy_loading.lazy_load_confirm_delivery( + sender_user, + sender_device, + room_id, + sincecount, + ); let no_state_changes = timeline_pdus.is_empty() - && (since_shortstatehash.is_none() || since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash))); + && (since_shortstatehash.is_none() + || since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash))); let mut device_list_updates = HashSet::::new(); let mut left_encrypted_users = HashSet::::new(); @@ -732,9 +752,10 @@ async fn load_joined_room( let events = join4(room_events, account_data_events, receipt_events, typing_events); let unread_notifications = join(notification_count, highlight_count); - let (unread_notifications, events, device_updates) = join3(unread_notifications, events, device_updates) - .boxed() - .await; + let (unread_notifications, events, device_updates) = + join3(unread_notifications, events, device_updates) + .boxed() + .await; let (room_events, account_data_events, receipt_events, typing_events) = events; let (notification_count, highlight_count) = unread_notifications; @@ -773,9 +794,7 @@ async fn load_joined_room( .await; let joined_room = JoinedRoom { - account_data: RoomAccountData { - events: account_data_events, - }, + account_data: RoomAccountData { events: account_data_events }, summary: RoomSummary { joined_member_count: joined_member_count.map(ruma_from_u64), invited_member_count: invited_member_count.map(ruma_from_u64), @@ -786,10 +805,7 @@ async fn load_joined_room( .filter_map(Result::ok) .collect(), }, - unread_notifications: UnreadNotificationsCount { - highlight_count, - notification_count, - }, + unread_notifications: UnreadNotificationsCount { highlight_count, notification_count }, timeline: Timeline { limited: limited || joined_since_last_sync, events: room_events, @@ -805,9 +821,7 @@ async fn load_joined_room( .map(PduEvent::to_sync_state_event) .collect(), }, - ephemeral: Ephemeral { - events: edus, - }, + ephemeral: Ephemeral { events: edus }, unread_thread_notifications: BTreeMap::new(), }; @@ -827,11 +841,20 @@ async fn load_joined_room( )] #[allow(clippy::too_many_arguments)] async fn calculate_state_changes( - services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, next_batchcount: PduCount, - lazy_load_enabled: bool, lazy_load_send_redundant: bool, full_state: bool, - device_list_updates: &mut HashSet, left_encrypted_users: &mut HashSet, - since_shortstatehash: Option, current_shortstatehash: ShortStateHash, - timeline_pdus: &Vec<(PduCount, PduEvent)>, timeline_users: &HashSet, + services: &Services, + sender_user: &UserId, + sender_device: &DeviceId, + room_id: &RoomId, + next_batchcount: PduCount, + lazy_load_enabled: bool, + lazy_load_send_redundant: bool, + full_state: bool, + device_list_updates: &mut HashSet, + left_encrypted_users: &mut HashSet, + since_shortstatehash: Option, + current_shortstatehash: ShortStateHash, + timeline_pdus: &Vec<(PduCount, PduEvent)>, + timeline_users: &HashSet, ) -> Result { let since_sender_member: OptionFuture<_> = since_shortstatehash .map(|short| { @@ -843,12 +866,13 @@ async fn calculate_state_changes( }) .into(); - let joined_since_last_sync = since_sender_member - .await - .flatten() - .map_or(true, |content: RoomMemberEventContent| { - content.membership != MembershipState::Join - }); + let joined_since_last_sync = + since_sender_member + .await + .flatten() + .is_none_or(|content: RoomMemberEventContent| { + content.membership != MembershipState::Join + }); if since_shortstatehash.is_none() || joined_since_last_sync { calculate_state_initial( @@ -886,8 +910,14 @@ async fn calculate_state_changes( #[tracing::instrument(name = "initial", level = "trace", skip_all)] #[allow(clippy::too_many_arguments)] async fn calculate_state_initial( - services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, next_batchcount: PduCount, - lazy_load_enabled: bool, full_state: bool, current_shortstatehash: ShortStateHash, + services: &Services, + sender_user: &UserId, + sender_device: &DeviceId, + room_id: &RoomId, + next_batchcount: PduCount, + lazy_load_enabled: bool, + full_state: bool, + current_shortstatehash: ShortStateHash, timeline_users: &HashSet, ) -> Result { // Probably since = 0, we will do an initial sync @@ -956,10 +986,13 @@ async fn calculate_state_initial( // The state_events above should contain all timeline_users, let's mark them as // lazy loaded. - services - .rooms - .lazy_loading - .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy_loaded, next_batchcount); + services.rooms.lazy_loading.lazy_load_mark_sent( + sender_user, + sender_device, + room_id, + lazy_loaded, + next_batchcount, + ); Ok(StateChanges { heroes, @@ -973,13 +1006,23 @@ async fn calculate_state_initial( #[tracing::instrument(name = "incremental", level = "trace", skip_all)] #[allow(clippy::too_many_arguments)] async fn calculate_state_incremental( - services: &Services, sender_user: &UserId, sender_device: &DeviceId, room_id: &RoomId, next_batchcount: PduCount, - lazy_load_send_redundant: bool, full_state: bool, device_list_updates: &mut HashSet, - left_encrypted_users: &mut HashSet, since_shortstatehash: Option, - current_shortstatehash: ShortStateHash, timeline_pdus: &Vec<(PduCount, PduEvent)>, joined_since_last_sync: bool, + services: &Services, + sender_user: &UserId, + sender_device: &DeviceId, + room_id: &RoomId, + next_batchcount: PduCount, + lazy_load_send_redundant: bool, + full_state: bool, + device_list_updates: &mut HashSet, + left_encrypted_users: &mut HashSet, + since_shortstatehash: Option, + current_shortstatehash: ShortStateHash, + timeline_pdus: &Vec<(PduCount, PduEvent)>, + joined_since_last_sync: bool, ) -> Result { // Incremental /sync - let since_shortstatehash = since_shortstatehash.expect("missing since_shortstatehash on incremental sync"); + let since_shortstatehash = + since_shortstatehash.expect("missing since_shortstatehash on incremental sync"); let mut delta_state_events = Vec::new(); @@ -994,8 +1037,10 @@ async fn calculate_state_incremental( .state_accessor .state_full_ids(since_shortstatehash); - let (current_state_ids, since_state_ids): (HashMap<_, OwnedEventId>, HashMap<_, OwnedEventId>) = - try_join(current_state_ids, since_state_ids).await?; + let (current_state_ids, since_state_ids): ( + HashMap<_, OwnedEventId>, + HashMap<_, OwnedEventId>, + ) = try_join(current_state_ids, since_state_ids).await?; current_state_ids .iter() @@ -1044,17 +1089,19 @@ async fn calculate_state_incremental( let content: RoomMemberEventContent = state_event.get_content()?; match content.membership { - MembershipState::Join => { + | MembershipState::Join => { // A new user joined an encrypted room - if !share_encrypted_room(services, sender_user, &user_id, Some(room_id)).await { + if !share_encrypted_room(services, sender_user, &user_id, Some(room_id)) + .await + { device_list_updates.insert(user_id); } }, - MembershipState::Leave => { + | MembershipState::Leave => { // Write down users that have left encrypted rooms we are in left_encrypted_users.insert(user_id); }, - _ => {}, + | _ => {}, } } } @@ -1139,10 +1186,13 @@ async fn calculate_state_incremental( state_events.push(member_event); } - services - .rooms - .lazy_loading - .lazy_load_mark_sent(sender_user, sender_device, room_id, lazy_loaded, next_batchcount); + services.rooms.lazy_loading.lazy_load_mark_sent( + sender_user, + sender_device, + room_id, + lazy_loaded, + next_batchcount, + ); Ok(StateChanges { heroes, @@ -1154,7 +1204,9 @@ async fn calculate_state_incremental( } async fn calculate_counts( - services: &Services, room_id: &RoomId, sender_user: &UserId, + services: &Services, + room_id: &RoomId, + sender_user: &UserId, ) -> Result<(Option, Option, Option>)> { let joined_member_count = services .rooms @@ -1168,7 +1220,8 @@ async fn calculate_counts( .room_invited_count(room_id) .unwrap_or(0); - let (joined_member_count, invited_member_count) = join(joined_member_count, invited_member_count).await; + let (joined_member_count, invited_member_count) = + join(joined_member_count, invited_member_count).await; let small_room = joined_member_count.saturating_add(invited_member_count) > 5; @@ -1179,20 +1232,32 @@ async fn calculate_counts( Ok((Some(joined_member_count), Some(invited_member_count), heroes.await)) } -async fn calculate_heroes(services: &Services, room_id: &RoomId, sender_user: &UserId) -> Vec { +async fn calculate_heroes( + services: &Services, + room_id: &RoomId, + sender_user: &UserId, +) -> Vec { services .rooms .timeline .all_pdus(sender_user, room_id) .ready_filter(|(_, pdu)| pdu.kind == RoomMember) - .fold_default(|heroes: Vec<_>, (_, pdu)| fold_hero(heroes, services, room_id, sender_user, pdu)) + .fold_default(|heroes: Vec<_>, (_, pdu)| { + fold_hero(heroes, services, room_id, sender_user, pdu) + }) .await } async fn fold_hero( - mut heroes: Vec, services: &Services, room_id: &RoomId, sender_user: &UserId, pdu: PduEvent, + mut heroes: Vec, + services: &Services, + room_id: &RoomId, + sender_user: &UserId, + pdu: PduEvent, ) -> Vec { - let Some(user_id): Option<&UserId> = pdu.state_key.as_deref().map(TryInto::try_into).flat_ok() else { + let Some(user_id): Option<&UserId> = + pdu.state_key.as_deref().map(TryInto::try_into).flat_ok() + else { return heroes; }; diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 140b647d..7e24adff 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -46,7 +46,8 @@ const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = /// /// Sliding Sync endpoint (future endpoint: `/_matrix/client/v4/sync`) pub(crate) async fn sync_events_v4_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.expect("user is authenticated"); @@ -81,16 +82,19 @@ pub(crate) async fn sync_events_v4_route( } if globalsince == 0 { - services - .sync - .forget_sync_request_connection(sender_user.clone(), sender_device.clone(), conn_id.clone()); + services.sync.forget_sync_request_connection( + sender_user.clone(), + sender_device.clone(), + conn_id.clone(), + ); } // Get sticky parameters from cache - let known_rooms = - services - .sync - .update_sync_request_with_cache(sender_user.clone(), sender_device.clone(), &mut body); + let known_rooms = services.sync.update_sync_request_with_cache( + sender_user.clone(), + sender_device.clone(), + &mut body, + ); let all_joined_rooms: Vec<_> = services .rooms @@ -125,9 +129,7 @@ pub(crate) async fn sync_events_v4_route( let mut device_list_changes = HashSet::new(); let mut device_list_left = HashSet::new(); - let mut receipts = sync_events::v4::Receipts { - rooms: BTreeMap::new(), - }; + let mut receipts = sync_events::v4::Receipts { rooms: BTreeMap::new() }; let mut account_data = sync_events::v4::AccountData { global: Vec::new(), @@ -168,7 +170,9 @@ pub(crate) async fn sync_events_v4_route( ); for room_id in &all_joined_rooms { - let Ok(current_shortstatehash) = services.rooms.state.get_room_shortstatehash(room_id).await else { + let Ok(current_shortstatehash) = + services.rooms.state.get_room_shortstatehash(room_id).await + else { error!("Room {room_id} has no state"); continue; }; @@ -202,12 +206,17 @@ pub(crate) async fn sync_events_v4_route( let since_sender_member: Option = services .rooms .state_accessor - .state_get_content(since_shortstatehash, &StateEventType::RoomMember, sender_user.as_str()) + .state_get_content( + since_shortstatehash, + &StateEventType::RoomMember, + sender_user.as_str(), + ) .ok() .await; - let joined_since_last_sync = - since_sender_member.map_or(true, |member| member.membership != MembershipState::Join); + let joined_since_last_sync = since_sender_member + .as_ref() + .is_none_or(|member| member.membership != MembershipState::Join); let new_encrypted_room = encrypted_room && since_encryption.is_err(); @@ -232,8 +241,10 @@ pub(crate) async fn sync_events_v4_route( }; if pdu.kind == RoomMember { if let Some(state_key) = &pdu.state_key { - let user_id = UserId::parse(state_key.clone()) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; + let user_id = + UserId::parse(state_key.clone()).map_err(|_| { + Error::bad_database("Invalid UserId in member PDU.") + })?; if user_id == *sender_user { continue; @@ -241,19 +252,25 @@ pub(crate) async fn sync_events_v4_route( let content: RoomMemberEventContent = pdu.get_content()?; match content.membership { - MembershipState::Join => { + | MembershipState::Join => { // A new user joined an encrypted room - if !share_encrypted_room(&services, sender_user, &user_id, Some(room_id)) - .await + if !share_encrypted_room( + &services, + sender_user, + &user_id, + Some(room_id), + ) + .await { device_list_changes.insert(user_id); } }, - MembershipState::Leave => { - // Write down users that have left encrypted rooms we are in + | MembershipState::Leave => { + // Write down users that have left encrypted rooms we + // are in left_encrypted_users.insert(user_id); }, - _ => {}, + | _ => {}, } } } @@ -293,7 +310,8 @@ pub(crate) async fn sync_events_v4_route( } for user_id in left_encrypted_users { - let dont_share_encrypted_room = !share_encrypted_room(&services, sender_user, &user_id, None).await; + let dont_share_encrypted_room = + !share_encrypted_room(&services, sender_user, &user_id, None).await; // If the user doesn't share an encrypted room with the target anymore, we need // to tell them @@ -308,85 +326,85 @@ pub(crate) async fn sync_events_v4_route( for (list_id, list) in &body.lists { let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) { - Some(true) => &all_invited_rooms, - Some(false) => &all_joined_rooms, - None => &all_rooms, + | Some(true) => &all_invited_rooms, + | Some(false) => &all_joined_rooms, + | None => &all_rooms, }; let active_rooms = match list.filters.clone().map(|f| f.not_room_types) { - Some(filter) if filter.is_empty() => active_rooms.clone(), - Some(value) => filter_rooms(&services, active_rooms, &value, true).await, - None => active_rooms.clone(), + | Some(filter) if filter.is_empty() => active_rooms.clone(), + | Some(value) => filter_rooms(&services, active_rooms, &value, true).await, + | None => active_rooms.clone(), }; let active_rooms = match list.filters.clone().map(|f| f.room_types) { - Some(filter) if filter.is_empty() => active_rooms.clone(), - Some(value) => filter_rooms(&services, &active_rooms, &value, false).await, - None => active_rooms, + | Some(filter) if filter.is_empty() => active_rooms.clone(), + | Some(value) => filter_rooms(&services, &active_rooms, &value, false).await, + | None => active_rooms, }; let mut new_known_rooms = BTreeSet::new(); let ranges = list.ranges.clone(); - lists.insert( - list_id.clone(), - sync_events::v4::SyncList { - ops: ranges - .into_iter() - .map(|mut r| { - r.0 = r.0.clamp( - uint!(0), - UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), + lists.insert(list_id.clone(), sync_events::v4::SyncList { + ops: ranges + .into_iter() + .map(|mut r| { + r.0 = r.0.clamp( + uint!(0), + UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), + ); + r.1 = r.1.clamp( + r.0, + UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX), + ); + + let room_ids = if !active_rooms.is_empty() { + active_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec() + } else { + Vec::new() + }; + + new_known_rooms.extend(room_ids.iter().cloned()); + for room_id in &room_ids { + let todo_room = todo_rooms.entry(room_id.clone()).or_insert(( + BTreeSet::new(), + 0_usize, + u64::MAX, + )); + + let limit: usize = list + .room_details + .timeline_limit + .map(u64::from) + .map_or(10, usize_from_u64_truncated) + .min(100); + + todo_room + .0 + .extend(list.room_details.required_state.iter().cloned()); + + todo_room.1 = todo_room.1.max(limit); + // 0 means unknown because it got out of date + todo_room.2 = todo_room.2.min( + known_rooms + .get(list_id.as_str()) + .and_then(|k| k.get(room_id)) + .copied() + .unwrap_or(0), ); - r.1 = - r.1.clamp(r.0, UInt::try_from(active_rooms.len().saturating_sub(1)).unwrap_or(UInt::MAX)); - - let room_ids = if !active_rooms.is_empty() { - active_rooms[usize_from_ruma(r.0)..=usize_from_ruma(r.1)].to_vec() - } else { - Vec::new() - }; - - new_known_rooms.extend(room_ids.iter().cloned()); - for room_id in &room_ids { - let todo_room = - todo_rooms - .entry(room_id.clone()) - .or_insert((BTreeSet::new(), 0_usize, u64::MAX)); - - let limit: usize = list - .room_details - .timeline_limit - .map(u64::from) - .map_or(10, usize_from_u64_truncated) - .min(100); - - todo_room - .0 - .extend(list.room_details.required_state.iter().cloned()); - - todo_room.1 = todo_room.1.max(limit); - // 0 means unknown because it got out of date - todo_room.2 = todo_room.2.min( - known_rooms - .get(list_id.as_str()) - .and_then(|k| k.get(room_id)) - .copied() - .unwrap_or(0), - ); - } - sync_events::v4::SyncOp { - op: SlidingOp::Sync, - range: Some(r), - index: None, - room_ids, - room_id: None, - } - }) - .collect(), - count: ruma_from_usize(active_rooms.len()), - }, - ); + } + sync_events::v4::SyncOp { + op: SlidingOp::Sync, + range: Some(r), + index: None, + room_ids, + room_id: None, + } + }) + .collect(), + count: ruma_from_usize(active_rooms.len()), + }); if let Some(conn_id) = &body.conn_id { services.sync.update_sync_known_rooms( @@ -405,9 +423,10 @@ pub(crate) async fn sync_events_v4_route( if !services.rooms.metadata.exists(room_id).await { continue; } - let todo_room = todo_rooms - .entry(room_id.clone()) - .or_insert((BTreeSet::new(), 0_usize, u64::MAX)); + let todo_room = + todo_rooms + .entry(room_id.clone()) + .or_insert((BTreeSet::new(), 0_usize, u64::MAX)); let limit: usize = room .timeline_limit @@ -471,14 +490,22 @@ pub(crate) async fn sync_events_v4_route( (timeline_pdus, limited) = (Vec::new(), true); } else { - (timeline_pdus, limited) = - match load_timeline(&services, sender_user, room_id, roomsincecount, None, *timeline_limit).await { - Ok(value) => value, - Err(err) => { - warn!("Encountered missing timeline in {}, error {}", room_id, err); - continue; - }, - }; + (timeline_pdus, limited) = match load_timeline( + &services, + sender_user, + room_id, + roomsincecount, + None, + *timeline_limit, + ) + .await + { + | Ok(value) => value, + | Err(err) => { + warn!("Encountered missing timeline in {}, error {}", room_id, err); + continue; + }, + }; } account_data.rooms.insert( @@ -543,11 +570,11 @@ pub(crate) async fn sync_events_v4_route( .first() .map_or(Ok::<_, Error>(None), |(pdu_count, _)| { Ok(Some(match pdu_count { - PduCount::Backfilled(_) => { + | PduCount::Backfilled(_) => { error!("timeline in backfill state?!"); "0".to_owned() }, - PduCount::Normal(c) => c.to_string(), + | PduCount::Normal(c) => c.to_string(), })) })? .or_else(|| { @@ -568,7 +595,9 @@ pub(crate) async fn sync_events_v4_route( for (_, pdu) in timeline_pdus { let ts = MilliSecondsSinceUnixEpoch(pdu.origin_server_ts); - if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) && timestamp.is_none_or(|time| time <= ts) { + if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) + && timestamp.is_none_or(|time| time <= ts) + { timestamp = Some(ts); } } @@ -611,7 +640,7 @@ pub(crate) async fn sync_events_v4_route( .await; let name = match heroes.len().cmp(&(1_usize)) { - Ordering::Greater => { + | Ordering::Greater => { let firsts = heroes[1..] .iter() .map(|h| h.name.clone().unwrap_or_else(|| h.user_id.to_string())) @@ -625,13 +654,13 @@ pub(crate) async fn sync_events_v4_route( Some(format!("{firsts} and {last}")) }, - Ordering::Equal => Some( + | Ordering::Equal => Some( heroes[0] .name .clone() .unwrap_or_else(|| heroes[0].user_id.to_string()), ), - Ordering::Less => None, + | Ordering::Less => None, }; let heroes_avatar = if heroes.len() == 1 { @@ -640,77 +669,74 @@ pub(crate) async fn sync_events_v4_route( None }; - rooms.insert( - room_id.clone(), - sync_events::v4::SlidingSyncRoom { - name: services - .rooms - .state_accessor - .get_name(room_id) - .await - .ok() - .or(name), - avatar: if let Some(heroes_avatar) = heroes_avatar { - ruma::JsOption::Some(heroes_avatar) - } else { - match services.rooms.state_accessor.get_avatar(room_id).await { - ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url), - ruma::JsOption::Null => ruma::JsOption::Null, - ruma::JsOption::Undefined => ruma::JsOption::Undefined, - } - }, - initial: Some(roomsince == &0), - is_dm: None, - invite_state, - unread_notifications: UnreadNotificationsCount { - highlight_count: Some( - services - .rooms - .user - .highlight_count(sender_user, room_id) - .await - .try_into() - .expect("notification count can't go that high"), - ), - notification_count: Some( - services - .rooms - .user - .notification_count(sender_user, room_id) - .await - .try_into() - .expect("notification count can't go that high"), - ), - }, - timeline: room_events, - required_state, - prev_batch, - limited, - joined_count: Some( - services - .rooms - .state_cache - .room_joined_count(room_id) - .await - .unwrap_or(0) - .try_into() - .unwrap_or_else(|_| uint!(0)), - ), - invited_count: Some( - services - .rooms - .state_cache - .room_invited_count(room_id) - .await - .unwrap_or(0) - .try_into() - .unwrap_or_else(|_| uint!(0)), - ), - num_live: None, // Count events in timeline greater than global sync counter - timestamp, - heroes: Some(heroes), + rooms.insert(room_id.clone(), sync_events::v4::SlidingSyncRoom { + name: services + .rooms + .state_accessor + .get_name(room_id) + .await + .ok() + .or(name), + avatar: if let Some(heroes_avatar) = heroes_avatar { + ruma::JsOption::Some(heroes_avatar) + } else { + match services.rooms.state_accessor.get_avatar(room_id).await { + | ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url), + | ruma::JsOption::Null => ruma::JsOption::Null, + | ruma::JsOption::Undefined => ruma::JsOption::Undefined, + } }, - ); + initial: Some(roomsince == &0), + is_dm: None, + invite_state, + unread_notifications: UnreadNotificationsCount { + highlight_count: Some( + services + .rooms + .user + .highlight_count(sender_user, room_id) + .await + .try_into() + .expect("notification count can't go that high"), + ), + notification_count: Some( + services + .rooms + .user + .notification_count(sender_user, room_id) + .await + .try_into() + .expect("notification count can't go that high"), + ), + }, + timeline: room_events, + required_state, + prev_batch, + limited, + joined_count: Some( + services + .rooms + .state_cache + .room_joined_count(room_id) + .await + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), + ), + invited_count: Some( + services + .rooms + .state_cache + .room_invited_count(room_id) + .await + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), + ), + num_live: None, // Count events in timeline greater than global sync counter + timestamp, + heroes: Some(heroes), + }); } if rooms @@ -757,16 +783,17 @@ pub(crate) async fn sync_events_v4_route( }, account_data, receipts, - typing: sync_events::v4::Typing { - rooms: BTreeMap::new(), - }, + typing: sync_events::v4::Typing { rooms: BTreeMap::new() }, }, delta_token: None, }) } async fn filter_rooms( - services: &Services, rooms: &[OwnedRoomId], filter: &[RoomTypeFilter], negate: bool, + services: &Services, + rooms: &[OwnedRoomId], + filter: &[RoomTypeFilter], + negate: bool, ) -> Vec { rooms .iter() diff --git a/src/api/client/tag.rs b/src/api/client/tag.rs index b5fa19e3..820ee4a1 100644 --- a/src/api/client/tag.rs +++ b/src/api/client/tag.rs @@ -17,7 +17,8 @@ use crate::{Result, Ruma}; /// /// - Inserts the tag into the tag event of the room account data. pub(crate) async fn update_tag_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -26,9 +27,7 @@ pub(crate) async fn update_tag_route( .get_room(&body.room_id, sender_user, RoomAccountDataEventType::Tag) .await .unwrap_or(TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, + content: TagEventContent { tags: BTreeMap::new() }, }); tags_event @@ -55,7 +54,8 @@ pub(crate) async fn update_tag_route( /// /// - Removes the tag from the tag event of the room account data. pub(crate) async fn delete_tag_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -64,9 +64,7 @@ pub(crate) async fn delete_tag_route( .get_room(&body.room_id, sender_user, RoomAccountDataEventType::Tag) .await .unwrap_or(TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, + content: TagEventContent { tags: BTreeMap::new() }, }); tags_event.content.tags.remove(&body.tag.clone().into()); @@ -90,7 +88,8 @@ pub(crate) async fn delete_tag_route( /// /// - Gets the tag event of the room account data. pub(crate) async fn get_tags_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -99,12 +98,8 @@ pub(crate) async fn get_tags_route( .get_room(&body.room_id, sender_user, RoomAccountDataEventType::Tag) .await .unwrap_or(TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, + content: TagEventContent { tags: BTreeMap::new() }, }); - Ok(get_tags::v3::Response { - tags: tags_event.content.tags, - }) + Ok(get_tags::v3::Response { tags: tags_event.content.tags }) } diff --git a/src/api/client/thirdparty.rs b/src/api/client/thirdparty.rs index f6af8729..790b27d3 100644 --- a/src/api/client/thirdparty.rs +++ b/src/api/client/thirdparty.rs @@ -11,9 +11,7 @@ pub(crate) async fn get_protocols_route( _body: Ruma, ) -> Result { // TODO - Ok(get_protocols::v3::Response { - protocols: BTreeMap::new(), - }) + Ok(get_protocols::v3::Response { protocols: BTreeMap::new() }) } /// # `GET /_matrix/client/unstable/thirdparty/protocols` diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index 07badaf7..f0cbf467 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -7,7 +7,8 @@ use crate::{Result, Ruma}; /// # `GET /_matrix/client/r0/rooms/{roomId}/threads` pub(crate) async fn get_threads_route( - State(services): State, ref body: Ruma, + State(services): State, + ref body: Ruma, ) -> Result { // Use limit or else 10, with maximum 100 let limit = body diff --git a/src/api/client/to_device.rs b/src/api/client/to_device.rs index aab59394..2ded04e7 100644 --- a/src/api/client/to_device.rs +++ b/src/api/client/to_device.rs @@ -17,7 +17,8 @@ use crate::Ruma; /// /// Send a to-device event to a set of client devices. pub(crate) async fn send_event_to_device_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.as_deref(); @@ -43,12 +44,14 @@ pub(crate) async fn send_event_to_device_route( services.sending.send_edu_server( target_user_id.server_name(), - serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice(DirectDeviceContent { - sender: sender_user.clone(), - ev_type: body.event_type.clone(), - message_id: count.to_string().into(), - messages, - })) + serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice( + DirectDeviceContent { + sender: sender_user.clone(), + ev_type: body.event_type.clone(), + message_id: count.to_string().into(), + messages, + }, + )) .expect("DirectToDevice EDU can be serialized"), )?; @@ -62,14 +65,20 @@ pub(crate) async fn send_event_to_device_route( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid"))?; match target_device_id_maybe { - DeviceIdOrAllDevices::DeviceId(target_device_id) => { + | DeviceIdOrAllDevices::DeviceId(target_device_id) => { services .users - .add_to_device_event(sender_user, target_user_id, target_device_id, event_type, event) + .add_to_device_event( + sender_user, + target_user_id, + target_device_id, + event_type, + event, + ) .await; }, - DeviceIdOrAllDevices::AllDevices => { + | DeviceIdOrAllDevices::AllDevices => { let (event_type, event) = (&event_type, &event); services .users diff --git a/src/api/client/typing.rs b/src/api/client/typing.rs index 932d221e..4ae3ac5e 100644 --- a/src/api/client/typing.rs +++ b/src/api/client/typing.rs @@ -7,7 +7,8 @@ use crate::{utils, Error, Result, Ruma}; /// /// Sets the typing state of the sender user. pub(crate) async fn create_typing_event_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { use create_typing_event::v3::Typing; diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index c546d6a7..2c9add44 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -10,8 +10,8 @@ use ruma::{ error::ErrorKind, membership::mutual_rooms, profile::{ - delete_profile_key, delete_timezone_key, get_profile_key, get_timezone_key, set_profile_key, - set_timezone_key, + delete_profile_key, delete_timezone_key, get_profile_key, get_timezone_key, + set_profile_key, set_timezone_key, }, room::get_summary, }, @@ -34,7 +34,8 @@ use crate::{Error, Result, Ruma, RumaResponse}; /// An implementation of [MSC2666](https://github.com/matrix-org/matrix-spec-proposals/pull/2666) #[tracing::instrument(skip_all, fields(%client), name = "mutual_rooms")] pub(crate) async fn get_mutual_rooms_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -47,10 +48,7 @@ pub(crate) async fn get_mutual_rooms_route( } if !services.users.exists(&body.user_id).await { - return Ok(mutual_rooms::unstable::Response { - joined: vec![], - next_batch_token: None, - }); + return Ok(mutual_rooms::unstable::Response { joined: vec![], next_batch_token: None }); } let mutual_rooms: Vec = services @@ -77,7 +75,8 @@ pub(crate) async fn get_mutual_rooms_route( /// /// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) pub(crate) async fn get_room_summary_legacy( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result> { get_room_summary(State(services), InsecureClientIp(client), body) @@ -94,7 +93,8 @@ pub(crate) async fn get_room_summary_legacy( /// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) #[tracing::instrument(skip_all, fields(%client), name = "room_summary")] pub(crate) async fn get_room_summary( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref(); @@ -194,7 +194,8 @@ pub(crate) async fn get_room_summary( /// /// - Also makes sure other users receive the update using presence EDUs pub(crate) async fn delete_timezone_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -221,7 +222,8 @@ pub(crate) async fn delete_timezone_key_route( /// /// - Also makes sure other users receive the update using presence EDUs pub(crate) async fn set_timezone_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -248,7 +250,8 @@ pub(crate) async fn set_timezone_key_route( /// /// This also handles the avatar_url and displayname being updated. pub(crate) async fn set_profile_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -264,7 +267,9 @@ pub(crate) async fn set_profile_key_route( if body.kv_pair.len() > 1 { // TODO: support PATCH or "recursively" adding keys in some sort - return Err!(Request(BadJson("This endpoint can only take one key-value pair at a time"))); + return Err!(Request(BadJson( + "This endpoint can only take one key-value pair at a time" + ))); } let Some(profile_key_value) = body.kv_pair.get(&body.key) else { @@ -294,7 +299,13 @@ pub(crate) async fn set_profile_key_route( .collect() .await; - update_displayname(&services, &body.user_id, Some(profile_key_value.to_string()), &all_joined_rooms).await; + update_displayname( + &services, + &body.user_id, + Some(profile_key_value.to_string()), + &all_joined_rooms, + ) + .await; } else if body.key == "avatar_url" { let mxc = ruma::OwnedMxcUri::from(profile_key_value.to_string()); @@ -330,7 +341,8 @@ pub(crate) async fn set_profile_key_route( /// /// This also handles the avatar_url and displayname being updated. pub(crate) async fn delete_profile_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); @@ -340,7 +352,9 @@ pub(crate) async fn delete_profile_key_route( if body.kv_pair.len() > 1 { // TODO: support PATCH or "recursively" adding keys in some sort - return Err!(Request(BadJson("This endpoint can only take one key-value pair at a time"))); + return Err!(Request(BadJson( + "This endpoint can only take one key-value pair at a time" + ))); } if body.key == "displayname" { @@ -387,7 +401,8 @@ pub(crate) async fn delete_profile_key_route( /// - If user is on another server and we do not have a local copy already fetch /// `timezone` over federation pub(crate) async fn get_timezone_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.globals.user_is_local(&body.user_id) { // Create and update our local copy of the user @@ -422,9 +437,7 @@ pub(crate) async fn get_timezone_key_route( .users .set_timezone(&body.user_id, response.tz.clone()); - return Ok(get_timezone_key::unstable::Response { - tz: response.tz, - }); + return Ok(get_timezone_key::unstable::Response { tz: response.tz }); } } @@ -446,7 +459,8 @@ pub(crate) async fn get_timezone_key_route( /// - If user is on another server and we do not have a local copy already fetch /// `timezone` over federation pub(crate) async fn get_profile_key_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let mut profile_key_value: BTreeMap = BTreeMap::new(); @@ -492,9 +506,7 @@ pub(crate) async fn get_profile_key_route( return Err!(Request(NotFound("The requested profile key does not exist."))); } - return Ok(get_profile_key::unstable::Response { - value: profile_key_value, - }); + return Ok(get_profile_key::unstable::Response { value: profile_key_value }); } } @@ -510,7 +522,5 @@ pub(crate) async fn get_profile_key_route( return Err!(Request(NotFound("The requested profile key does not exist."))); } - Ok(get_profile_key::unstable::Response { - value: profile_key_value, - }) + Ok(get_profile_key::unstable::Response { value: profile_key_value }) } diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 91fe5a7c..b4856d72 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -74,7 +74,9 @@ pub(crate) async fn conduwuit_server_version() -> Result { /// conduwuit-specific API to return the amount of users registered on this /// homeserver. Endpoint is disabled if federation is disabled for privacy. This /// only includes active users (not deactivated, no guests, etc) -pub(crate) async fn conduwuit_local_user_count(State(services): State) -> Result { +pub(crate) async fn conduwuit_local_user_count( + State(services): State, +) -> Result { let user_count = services.users.list_local_users().count().await; Ok(Json(serde_json::json!({ diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index bc7460f0..182e30db 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -18,7 +18,8 @@ use crate::{Result, Ruma}; /// - Hides any local users that aren't in any public rooms (i.e. those that /// have the join rule set to public) and don't share a room with the sender pub(crate) async fn search_users_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let limit = usize::try_from(body.limit).map_or(10, usize::from).min(100); // default limit is 10 @@ -61,7 +62,11 @@ pub(crate) async fn search_users_route( services .rooms .state_accessor - .room_state_get_content::(room, &StateEventType::RoomJoinRules, "") + .room_state_get_content::( + room, + &StateEventType::RoomJoinRules, + "", + ) .map_ok_or(false, |content| content.join_rule == JoinRule::Public) }) .await; @@ -89,8 +94,5 @@ pub(crate) async fn search_users_route( let results = users.take(limit).collect().await; - Ok(search_users::v3::Response { - results, - limited, - }) + Ok(search_users::v3::Response { results, limited }) } diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index b41cc8a1..ec804570 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -17,7 +17,8 @@ type HmacSha1 = Hmac; /// /// TODO: Returns information about the recommended turn server. pub(crate) async fn turn_server_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { // MSC4166: return M_NOT_FOUND 404 if no TURN URIs are specified in any way if services.server.config.turn_uris.is_empty() { @@ -44,7 +45,8 @@ pub(crate) async fn turn_server_route( let username: String = format!("{}:{}", expiry.get(), user); - let mut mac = HmacSha1::new_from_slice(turn_secret.as_bytes()).expect("HMAC can take key of any size"); + let mut mac = HmacSha1::new_from_slice(turn_secret.as_bytes()) + .expect("HMAC can take key of any size"); mac.update(username.as_bytes()); let password: String = general_purpose::STANDARD.encode(mac.finalize().into_bytes()); diff --git a/src/api/client/well_known.rs b/src/api/client/well_known.rs index b66217e8..5c53d013 100644 --- a/src/api/client/well_known.rs +++ b/src/api/client/well_known.rs @@ -13,21 +13,18 @@ use crate::{Error, Result, Ruma}; /// /// Returns the .well-known URL if it is configured, otherwise returns 404. pub(crate) async fn well_known_client( - State(services): State, _body: Ruma, + State(services): State, + _body: Ruma, ) -> Result { let client_url = match services.server.config.well_known.client.as_ref() { - Some(url) => url.to_string(), - None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), + | Some(url) => url.to_string(), + | None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), }; Ok(discover_homeserver::Response { - homeserver: HomeserverInfo { - base_url: client_url.clone(), - }, + homeserver: HomeserverInfo { base_url: client_url.clone() }, identity_server: None, - sliding_sync_proxy: Some(SlidingSyncProxyInfo { - url: client_url, - }), + sliding_sync_proxy: Some(SlidingSyncProxyInfo { url: client_url }), tile_server: None, }) } @@ -36,7 +33,8 @@ pub(crate) async fn well_known_client( /// /// Server support contact and support page of a homeserver's domain. pub(crate) async fn well_known_support( - State(services): State, _body: Ruma, + State(services): State, + _body: Ruma, ) -> Result { let support_page = services .server @@ -65,11 +63,7 @@ pub(crate) async fn well_known_support( let mut contacts: Vec = vec![]; if let Some(role) = role { - let contact = Contact { - role, - email_address, - matrix_id, - }; + let contact = Contact { role, email_address, matrix_id }; contacts.push(contact); } @@ -79,22 +73,21 @@ pub(crate) async fn well_known_support( return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); } - Ok(discover_support::Response { - contacts, - support_page, - }) + Ok(discover_support::Response { contacts, support_page }) } /// # `GET /client/server.json` /// /// Endpoint provided by sliding sync proxy used by some clients such as Element /// Web as a non-standard health check. -pub(crate) async fn syncv3_client_server_json(State(services): State) -> Result { +pub(crate) async fn syncv3_client_server_json( + State(services): State, +) -> Result { let server_url = match services.server.config.well_known.client.as_ref() { - Some(url) => url.to_string(), - None => match services.server.config.well_known.server.as_ref() { - Some(url) => url.to_string(), - None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), + | Some(url) => url.to_string(), + | None => match services.server.config.well_known.server.as_ref() { + | Some(url) => url.to_string(), + | None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), }, }; diff --git a/src/api/router.rs b/src/api/router.rs index 99e066e4..1b38670d 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -190,7 +190,10 @@ pub fn build(router: Router, server: &Server) -> Router { router = router .ruma_route(&server::get_server_version_route) .route("/_matrix/key/v2/server", get(server::get_server_keys_route)) - .route("/_matrix/key/v2/server/:key_id", get(server::get_server_keys_deprecated_route)) + .route( + "/_matrix/key/v2/server/:key_id", + get(server::get_server_keys_deprecated_route), + ) .ruma_route(&server::get_public_rooms_route) .ruma_route(&server::get_public_rooms_filtered_route) .ruma_route(&server::send_transaction_message_route) @@ -284,6 +287,10 @@ async fn redirect_legacy_preview(uri: Uri) -> impl IntoResponse { Redirect::temporary(&uri) } -async fn legacy_media_disabled() -> impl IntoResponse { err!(Request(Forbidden("Unauthenticated media is disabled."))) } +async fn legacy_media_disabled() -> impl IntoResponse { + err!(Request(Forbidden("Unauthenticated media is disabled."))) +} -async fn federation_disabled() -> impl IntoResponse { err!(Request(Forbidden("Federation is disabled."))) } +async fn federation_disabled() -> impl IntoResponse { + err!(Request(Forbidden("Federation is disabled."))) +} diff --git a/src/api/router/args.rs b/src/api/router/args.rs index f5666ed6..582f0c56 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -4,8 +4,8 @@ use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, Bytes, BytesMut}; use conduwuit::{debug, debug_warn, err, trace, utils::string::EMPTY, Error, Result}; use ruma::{ - api::IncomingRequest, CanonicalJsonObject, CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedServerName, - OwnedUserId, ServerName, UserId, + api::IncomingRequest, CanonicalJsonObject, CanonicalJsonValue, DeviceId, OwnedDeviceId, + OwnedServerName, OwnedUserId, ServerName, UserId, }; use service::Services; @@ -43,7 +43,9 @@ where T: IncomingRequest + Send + Sync + 'static, { #[inline] - pub(crate) fn sender(&self) -> (&UserId, &DeviceId) { (self.sender_user(), self.sender_device()) } + pub(crate) fn sender(&self) -> (&UserId, &DeviceId) { + (self.sender_user(), self.sender_device()) + } #[inline] pub(crate) fn sender_user(&self) -> &UserId { @@ -83,7 +85,10 @@ where { type Rejection = Error; - async fn from_request(request: hyper::Request, services: &State) -> Result { + async fn from_request( + request: hyper::Request, + services: &State, + ) -> Result { let mut request = request::from(services, request).await?; let mut json_body = serde_json::from_slice::(&request.body).ok(); @@ -96,7 +101,10 @@ where && !request.parts.uri.path().contains("/media/") { trace!("json_body from_request: {:?}", json_body.clone()); - debug_warn!("received a POST request with an empty body, defaulting/assuming to {{}} like Synapse does"); + debug_warn!( + "received a POST request with an empty body, defaulting/assuming to {{}} like \ + Synapse does" + ); json_body = Some(CanonicalJsonValue::Object(CanonicalJsonObject::new())); } let auth = auth::auth(services, &mut request, json_body.as_ref(), &T::METADATA).await?; @@ -112,14 +120,18 @@ where } fn make_body( - services: &Services, request: &mut Request, json_body: Option<&mut CanonicalJsonValue>, auth: &Auth, + services: &Services, + request: &mut Request, + json_body: Option<&mut CanonicalJsonValue>, + auth: &Auth, ) -> Result where T: IncomingRequest, { let body = take_body(services, request, json_body, auth); let http_request = into_http_request(request, body); - T::try_from_http_request(http_request, &request.path).map_err(|e| err!(Request(BadJson(debug_warn!("{e}"))))) + T::try_from_http_request(http_request, &request.path) + .map_err(|e| err!(Request(BadJson(debug_warn!("{e}"))))) } fn into_http_request(request: &Request, body: Bytes) -> hyper::Request { @@ -141,7 +153,10 @@ fn into_http_request(request: &Request, body: Bytes) -> hyper::Request { #[allow(clippy::needless_pass_by_value)] fn take_body( - services: &Services, request: &mut Request, json_body: Option<&mut CanonicalJsonValue>, auth: &Auth, + services: &Services, + request: &mut Request, + json_body: Option<&mut CanonicalJsonValue>, + auth: &Auth, ) -> Bytes { let Some(CanonicalJsonValue::Object(json_body)) = json_body else { return mem::take(&mut request.body); diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 243fd46b..c5b040e0 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -10,7 +10,9 @@ use ruma::{ client::{ directory::get_public_rooms, error::ErrorKind, - profile::{get_avatar_url, get_display_name, get_profile, get_profile_key, get_timezone_key}, + profile::{ + get_avatar_url, get_display_name, get_profile, get_profile_key, get_timezone_key, + }, voip::get_turn_server_info, }, federation::openid::get_openid_userinfo, @@ -42,12 +44,15 @@ pub(super) struct Auth { } pub(super) async fn auth( - services: &Services, request: &mut Request, json_body: Option<&CanonicalJsonValue>, metadata: &Metadata, + services: &Services, + request: &mut Request, + json_body: Option<&CanonicalJsonValue>, + metadata: &Metadata, ) -> Result { let bearer: Option>> = request.parts.extract().await?; let token = match &bearer { - Some(TypedHeader(Authorization(bearer))) => Some(bearer.token()), - None => request.query.access_token.as_deref(), + | Some(TypedHeader(Authorization(bearer))) => Some(bearer.token()), + | None => request.query.access_token.as_deref(), }; let token = if let Some(token) = token { @@ -64,56 +69,64 @@ pub(super) async fn auth( if metadata.authentication == AuthScheme::None { match metadata { - &get_public_rooms::v3::Request::METADATA => { + | &get_public_rooms::v3::Request::METADATA => { if !services .globals .config .allow_public_room_directory_without_auth { match token { - Token::Appservice(_) | Token::User(_) => { + | Token::Appservice(_) | Token::User(_) => { // we should have validated the token above // already }, - Token::None | Token::Invalid => { - return Err(Error::BadRequest(ErrorKind::MissingToken, "Missing or invalid access token.")); + | Token::None | Token::Invalid => { + return Err(Error::BadRequest( + ErrorKind::MissingToken, + "Missing or invalid access token.", + )); }, } } }, - &get_profile::v3::Request::METADATA + | &get_profile::v3::Request::METADATA | &get_profile_key::unstable::Request::METADATA | &get_display_name::v3::Request::METADATA | &get_avatar_url::v3::Request::METADATA | &get_timezone_key::unstable::Request::METADATA => { if services.globals.config.require_auth_for_profile_requests { match token { - Token::Appservice(_) | Token::User(_) => { + | Token::Appservice(_) | Token::User(_) => { // we should have validated the token above // already }, - Token::None | Token::Invalid => { - return Err(Error::BadRequest(ErrorKind::MissingToken, "Missing or invalid access token.")); + | Token::None | Token::Invalid => { + return Err(Error::BadRequest( + ErrorKind::MissingToken, + "Missing or invalid access token.", + )); }, } } }, - _ => {}, + | _ => {}, }; } match (metadata.authentication, token) { - (AuthScheme::AccessToken, Token::Appservice(info)) => Ok(auth_appservice(services, request, info).await?), - (AuthScheme::None | AuthScheme::AccessTokenOptional | AuthScheme::AppserviceToken, Token::Appservice(info)) => { - Ok(Auth { - origin: None, - sender_user: None, - sender_device: None, - appservice_info: Some(*info), - }) - }, - (AuthScheme::AccessToken, Token::None) => match metadata { - &get_turn_server_info::v3::Request::METADATA => { + | (AuthScheme::AccessToken, Token::Appservice(info)) => + Ok(auth_appservice(services, request, info).await?), + | ( + AuthScheme::None | AuthScheme::AccessTokenOptional | AuthScheme::AppserviceToken, + Token::Appservice(info), + ) => Ok(Auth { + origin: None, + sender_user: None, + sender_device: None, + appservice_info: Some(*info), + }), + | (AuthScheme::AccessToken, Token::None) => match metadata { + | &get_turn_server_info::v3::Request::METADATA => { if services.globals.config.turn_allow_guests { Ok(Auth { origin: None, @@ -125,9 +138,9 @@ pub(super) async fn auth( Err(Error::BadRequest(ErrorKind::MissingToken, "Missing access token.")) } }, - _ => Err(Error::BadRequest(ErrorKind::MissingToken, "Missing access token.")), + | _ => Err(Error::BadRequest(ErrorKind::MissingToken, "Missing access token.")), }, - ( + | ( AuthScheme::AccessToken | AuthScheme::AccessTokenOptional | AuthScheme::None, Token::User((user_id, device_id)), ) => Ok(Auth { @@ -136,26 +149,33 @@ pub(super) async fn auth( sender_device: Some(device_id), appservice_info: None, }), - (AuthScheme::ServerSignatures, Token::None) => Ok(auth_server(services, request, json_body).await?), - (AuthScheme::None | AuthScheme::AppserviceToken | AuthScheme::AccessTokenOptional, Token::None) => Ok(Auth { + | (AuthScheme::ServerSignatures, Token::None) => + Ok(auth_server(services, request, json_body).await?), + | ( + AuthScheme::None | AuthScheme::AppserviceToken | AuthScheme::AccessTokenOptional, + Token::None, + ) => Ok(Auth { sender_user: None, sender_device: None, origin: None, appservice_info: None, }), - (AuthScheme::ServerSignatures, Token::Appservice(_) | Token::User(_)) => Err(Error::BadRequest( - ErrorKind::Unauthorized, - "Only server signatures should be used on this endpoint.", - )), - (AuthScheme::AppserviceToken, Token::User(_)) => Err(Error::BadRequest( + | (AuthScheme::ServerSignatures, Token::Appservice(_) | Token::User(_)) => + Err(Error::BadRequest( + ErrorKind::Unauthorized, + "Only server signatures should be used on this endpoint.", + )), + | (AuthScheme::AppserviceToken, Token::User(_)) => Err(Error::BadRequest( ErrorKind::Unauthorized, "Only appservice access tokens should be used on this endpoint.", )), - (AuthScheme::None, Token::Invalid) => { + | (AuthScheme::None, Token::Invalid) => { // OpenID federation endpoint uses a query param with the same name, drop this // once query params for user auth are removed from the spec. This is // required to make integration manager work. - if request.query.access_token.is_some() && metadata == &get_openid_userinfo::v1::Request::METADATA { + if request.query.access_token.is_some() + && metadata == &get_openid_userinfo::v1::Request::METADATA + { Ok(Auth { origin: None, sender_user: None, @@ -164,25 +184,29 @@ pub(super) async fn auth( }) } else { Err(Error::BadRequest( - ErrorKind::UnknownToken { - soft_logout: false, - }, + ErrorKind::UnknownToken { soft_logout: false }, "Unknown access token.", )) } }, - (_, Token::Invalid) => Err(Error::BadRequest( - ErrorKind::UnknownToken { - soft_logout: false, - }, + | (_, Token::Invalid) => Err(Error::BadRequest( + ErrorKind::UnknownToken { soft_logout: false }, "Unknown access token.", )), } } -async fn auth_appservice(services: &Services, request: &Request, info: Box) -> Result { - let user_id_default = - || UserId::parse_with_server_name(info.registration.sender_localpart.as_str(), services.globals.server_name()); +async fn auth_appservice( + services: &Services, + request: &Request, + info: Box, +) -> Result { + let user_id_default = || { + UserId::parse_with_server_name( + info.registration.sender_localpart.as_str(), + services.globals.server_name(), + ) + }; let Ok(user_id) = request .query @@ -205,7 +229,11 @@ async fn auth_appservice(services: &Services, request: &Request, info: Box) -> Result { +async fn auth_server( + services: &Services, + request: &mut Request, + body: Option<&CanonicalJsonValue>, +) -> Result { type Member = (String, CanonicalJsonValue); type Object = CanonicalJsonObject; type Value = CanonicalJsonValue; @@ -222,7 +250,8 @@ async fn auth_server(services: &Services, request: &mut Request, body: Option<&C .expect("all requests have a path") .to_string(); - let signature: [Member; 1] = [(x_matrix.key.as_str().into(), Value::String(x_matrix.sig.to_string()))]; + let signature: [Member; 1] = + [(x_matrix.key.as_str().into(), Value::String(x_matrix.sig.to_string()))]; let signatures: [Member; 1] = [(origin.as_str().into(), Value::Object(signature.into()))]; @@ -261,8 +290,8 @@ async fn auth_server(services: &Services, request: &mut Request, body: Option<&C debug_error!("Failed to verify federation request from {origin}: {e}"); if request.parts.uri.to_string().contains('@') { warn!( - "Request uri contained '@' character. Make sure your reverse proxy gives conduwuit the raw uri \ - (apache: use nocanon)" + "Request uri contained '@' character. Make sure your reverse proxy gives \ + conduwuit the raw uri (apache: use nocanon)" ); } @@ -294,7 +323,9 @@ fn auth_server_checks(services: &Services, x_matrix: &XMatrix) -> Result<()> { .forbidden_remote_server_names .contains(origin) { - return Err!(Request(Forbidden(debug_warn!("Federation requests from {origin} denied.")))); + return Err!(Request(Forbidden(debug_warn!( + "Federation requests from {origin} denied." + )))); } Ok(()) @@ -307,9 +338,9 @@ async fn parse_x_matrix(request: &mut Request) -> Result { .await .map_err(|e| { let msg = match e.reason() { - TypedHeaderRejectionReason::Missing => "Missing Authorization header.", - TypedHeaderRejectionReason::Error(_) => "Invalid X-Matrix signatures.", - _ => "Unknown header-related error", + | TypedHeaderRejectionReason::Missing => "Missing Authorization header.", + | TypedHeaderRejectionReason::Error(_) => "Invalid X-Matrix signatures.", + | _ => "Unknown header-related error", }; err!(Request(Forbidden(warn!("{msg}: {e}")))) diff --git a/src/api/router/handler.rs b/src/api/router/handler.rs index 246dc843..cfb8fb6e 100644 --- a/src/api/router/handler.rs +++ b/src/api/router/handler.rs @@ -66,14 +66,14 @@ ruma_handler!(T1, T2, T3, T4); const fn method_to_filter(method: &Method) -> MethodFilter { match *method { - Method::DELETE => MethodFilter::DELETE, - Method::GET => MethodFilter::GET, - Method::HEAD => MethodFilter::HEAD, - Method::OPTIONS => MethodFilter::OPTIONS, - Method::PATCH => MethodFilter::PATCH, - Method::POST => MethodFilter::POST, - Method::PUT => MethodFilter::PUT, - Method::TRACE => MethodFilter::TRACE, - _ => panic!("Unsupported HTTP method"), + | Method::DELETE => MethodFilter::DELETE, + | Method::GET => MethodFilter::GET, + | Method::HEAD => MethodFilter::HEAD, + | Method::OPTIONS => MethodFilter::OPTIONS, + | Method::PATCH => MethodFilter::PATCH, + | Method::POST => MethodFilter::POST, + | Method::PUT => MethodFilter::PUT, + | Method::TRACE => MethodFilter::TRACE, + | _ => panic!("Unsupported HTTP method"), } } diff --git a/src/api/router/request.rs b/src/api/router/request.rs index fd57d841..627abd30 100644 --- a/src/api/router/request.rs +++ b/src/api/router/request.rs @@ -20,14 +20,17 @@ pub(super) struct Request { pub(super) parts: Parts, } -pub(super) async fn from(services: &Services, request: hyper::Request) -> Result { +pub(super) async fn from( + services: &Services, + request: hyper::Request, +) -> Result { let limited = request.with_limited_body(); let (mut parts, body) = limited.into_parts(); let path: Path> = parts.extract().await?; let query = parts.uri.query().unwrap_or_default(); - let query = - serde_html_form::from_str(query).map_err(|e| err!(Request(Unknown("Failed to read query parameters: {e}"))))?; + let query = serde_html_form::from_str(query) + .map_err(|e| err!(Request(Unknown("Failed to read query parameters: {e}"))))?; let max_body_size = services.globals.config.max_request_size; @@ -35,10 +38,5 @@ pub(super) async fn from(services: &Services, request: hyper::Request) -> (State, Guard) { services: Arc::into_raw(services.clone()), }; - let guard = Guard { - services, - }; + let guard = Guard { services }; (state, guard) } diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 22fb6a8e..fac0e540 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -16,7 +16,8 @@ use crate::Ruma; /// Retrieves events from before the sender joined the room, if the room's /// history visibility allows. pub(crate) async fn get_backfill_route( - State(services): State, ref body: Ruma, + State(services): State, + ref body: Ruma, ) -> Result { AccessCheck { services: &services, diff --git a/src/api/server/event.rs b/src/api/server/event.rs index cbd9100b..629dd6a2 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -12,7 +12,8 @@ use crate::Ruma; /// - Only works if a user of this server is currently invited or joined the /// room pub(crate) async fn get_event_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let event = services .rooms diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 2d0893eb..93e867a0 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -17,7 +17,8 @@ use crate::Ruma; /// /// - This does not include the event itself pub(crate) async fn get_event_authorization_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { AccessCheck { services: &services, @@ -40,8 +41,8 @@ pub(crate) async fn get_event_authorization_route( .and_then(|val| val.as_str()) .ok_or_else(|| Error::bad_database("Invalid event in database."))?; - let room_id = - <&RoomId>::try_from(room_id_str).map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; + let room_id = <&RoomId>::try_from(room_id_str) + .map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; let auth_chain = services .rooms @@ -53,7 +54,5 @@ pub(crate) async fn get_event_authorization_route( .collect() .await; - Ok(get_event_authorization::v1::Response { - auth_chain, - }) + Ok(get_event_authorization::v1::Response { auth_chain }) } diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index a07a1a37..ea06015a 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -12,7 +12,8 @@ use crate::Ruma; /// /// Retrieves events that the sender is missing. pub(crate) async fn get_missing_events_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { AccessCheck { services: &services, @@ -87,7 +88,5 @@ pub(crate) async fn get_missing_events_route( i = i.saturating_add(1); } - Ok(get_missing_events::v1::Response { - events, - }) + Ok(get_missing_events::v1::Response { events }) } diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs index e3ce7108..a10df6ac 100644 --- a/src/api/server/hierarchy.rs +++ b/src/api/server/hierarchy.rs @@ -8,7 +8,8 @@ use crate::{Error, Result, Ruma}; /// Gets the space tree in a depth-first manner to locate child rooms of a given /// space. pub(crate) async fn get_hierarchy_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if services.rooms.metadata.exists(&body.room_id).await { services diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index f76573c7..a5b4a11c 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -16,7 +16,8 @@ use crate::Ruma; /// Invites a remote user to a room. #[tracing::instrument(skip_all, fields(%client), name = "invite")] pub(crate) async fn create_invite_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { // ACL check origin @@ -28,9 +29,7 @@ pub(crate) async fn create_invite_route( if !services.server.supported_room_version(&body.room_version) { return Err(Error::BadRequest( - ErrorKind::IncompatibleRoomVersion { - room_version: body.room_version.clone(), - }, + ErrorKind::IncompatibleRoomVersion { room_version: body.room_version.clone() }, "Server does not support this room version.", )); } @@ -102,11 +101,14 @@ pub(crate) async fn create_invite_route( .try_into() .map_err(|e| err!(Request(InvalidParam("Invalid sender property: {e}"))))?; - if services.rooms.metadata.is_banned(&body.room_id).await && !services.users.is_admin(&invited_user).await { + if services.rooms.metadata.is_banned(&body.room_id).await + && !services.users.is_admin(&invited_user).await + { return Err!(Request(Forbidden("This room is banned on this homeserver."))); } - if services.globals.block_non_admin_invites() && !services.users.is_admin(&invited_user).await { + if services.globals.block_non_admin_invites() && !services.users.is_admin(&invited_user).await + { return Err!(Request(Forbidden("This server does not allow room invites."))); } diff --git a/src/api/server/key.rs b/src/api/server/key.rs index fea79897..75801a7a 100644 --- a/src/api/server/key.rs +++ b/src/api/server/key.rs @@ -22,7 +22,9 @@ use ruma::{ /// this will be valid forever. // Response type for this endpoint is Json because we need to calculate a // signature for the response -pub(crate) async fn get_server_keys_route(State(services): State) -> Result { +pub(crate) async fn get_server_keys_route( + State(services): State, +) -> Result { let server_name = services.globals.server_name(); let active_key_id = services.server_keys.active_key_id(); let mut all_keys = services.server_keys.verify_keys_for(server_name).await; @@ -72,6 +74,8 @@ fn expires_ts() -> MilliSecondsSinceUnixEpoch { /// /// - Matrix does not support invalidating public keys, so the key returned by /// this will be valid forever. -pub(crate) async fn get_server_keys_deprecated_route(State(services): State) -> impl IntoResponse { +pub(crate) async fn get_server_keys_deprecated_route( + State(services): State, +) -> impl IntoResponse { get_server_keys_route(State(services)).await } diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index 3f62fbbe..3900c418 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -23,7 +23,8 @@ use crate::{ /// /// Creates a join template. pub(crate) async fn create_join_event_template_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.rooms.metadata.exists(&body.room_id).await { return Err!(Request(NotFound("Room is unknown to this server."))); @@ -47,8 +48,8 @@ pub(crate) async fn create_join_event_template_route( .contains(body.origin()) { warn!( - "Server {} for remote user {} tried joining room ID {} which has a server name that is globally \ - forbidden. Rejecting.", + "Server {} for remote user {} tried joining room ID {} which has a server name that \ + is globally forbidden. Rejecting.", body.origin(), &body.user_id, &body.room_id, @@ -72,9 +73,7 @@ pub(crate) async fn create_join_event_template_route( let room_version_id = services.rooms.state.get_room_version(&body.room_id).await?; if !body.ver.contains(&room_version_id) { return Err(Error::BadRequest( - ErrorKind::IncompatibleRoomVersion { - room_version: room_version_id, - }, + ErrorKind::IncompatibleRoomVersion { room_version: room_version_id }, "Room version not supported.", )); } @@ -86,16 +85,25 @@ pub(crate) async fn create_join_event_template_route( if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { // room version does not support restricted join rules None - } else if user_can_perform_restricted_join(&services, &body.user_id, &body.room_id, &room_version_id).await? { + } else if user_can_perform_restricted_join( + &services, + &body.user_id, + &body.room_id, + &room_version_id, + ) + .await? + { let Some(auth_user) = services .rooms .state_cache .local_users_in_room(&body.room_id) .filter(|user| { - services - .rooms - .state_accessor - .user_can_invite(&body.room_id, user, &body.user_id, &state_lock) + services.rooms.state_accessor.user_can_invite( + &body.room_id, + user, + &body.user_id, + &state_lock, + ) }) .boxed() .next() @@ -116,13 +124,10 @@ pub(crate) async fn create_join_event_template_route( .rooms .timeline .create_hash_and_sign_event( - PduBuilder::state( - body.user_id.to_string(), - &RoomMemberEventContent { - join_authorized_via_users_server, - ..RoomMemberEventContent::new(MembershipState::Join) - }, - ), + PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { + join_authorized_via_users_server, + ..RoomMemberEventContent::new(MembershipState::Join) + }), &body.user_id, &body.room_id, &state_lock, @@ -142,7 +147,10 @@ pub(crate) async fn create_join_event_template_route( /// Checks whether the given user can join the given room via a restricted join. pub(crate) async fn user_can_perform_restricted_join( - services: &Services, user_id: &UserId, room_id: &RoomId, room_version_id: &RoomVersionId, + services: &Services, + user_id: &UserId, + room_id: &RoomId, + room_version_id: &RoomVersionId, ) -> Result { use RoomVersionId::*; @@ -159,13 +167,19 @@ pub(crate) async fn user_can_perform_restricted_join( let Ok(join_rules_event_content) = services .rooms .state_accessor - .room_state_get_content::(room_id, &StateEventType::RoomJoinRules, "") + .room_state_get_content::( + room_id, + &StateEventType::RoomJoinRules, + "", + ) .await else { return Ok(false); }; - let (JoinRule::Restricted(r) | JoinRule::KnockRestricted(r)) = join_rules_event_content.join_rule else { + let (JoinRule::Restricted(r) | JoinRule::KnockRestricted(r)) = + join_rules_event_content.join_rule + else { return Ok(false); }; @@ -195,12 +209,15 @@ pub(crate) async fn user_can_perform_restricted_join( } } -pub(crate) fn maybe_strip_event_id(pdu_json: &mut CanonicalJsonObject, room_version_id: &RoomVersionId) -> Result { +pub(crate) fn maybe_strip_event_id( + pdu_json: &mut CanonicalJsonObject, + room_version_id: &RoomVersionId, +) -> Result { use RoomVersionId::*; match room_version_id { - V1 | V2 => Ok(()), - _ => { + | V1 | V2 => Ok(()), + | _ => { pdu_json.remove("event_id"); Ok(()) }, diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 2895d3bf..746a4858 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -13,7 +13,8 @@ use crate::{service::pdu::PduBuilder, Ruma}; /// /// Creates a leave template. pub(crate) async fn create_leave_event_template_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.rooms.metadata.exists(&body.room_id).await { return Err!(Request(NotFound("Room is unknown to this server."))); @@ -37,7 +38,10 @@ pub(crate) async fn create_leave_event_template_route( .rooms .timeline .create_hash_and_sign_event( - PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Leave)), + PduBuilder::state( + body.user_id.to_string(), + &RoomMemberEventContent::new(MembershipState::Leave), + ), &body.user_id, &body.room_id, &state_lock, diff --git a/src/api/server/media.rs b/src/api/server/media.rs index 8c3ff435..03ec7b51 100644 --- a/src/api/server/media.rs +++ b/src/api/server/media.rs @@ -16,7 +16,8 @@ use crate::Ruma; /// Load media from our server. #[tracing::instrument(skip_all, fields(%client), name = "media_get")] pub(crate) async fn get_content_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let mxc = Mxc { @@ -33,7 +34,8 @@ pub(crate) async fn get_content_route( return Err!(Request(NotFound("Media not found."))); }; - let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + let content_disposition = + make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); let content = Content { file: content.expect("entire file contents"), content_type: content_type.map(Into::into), @@ -51,7 +53,8 @@ pub(crate) async fn get_content_route( /// Load media thumbnail from our server. #[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] pub(crate) async fn get_content_thumbnail_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; @@ -69,7 +72,8 @@ pub(crate) async fn get_content_thumbnail_route( return Err!(Request(NotFound("Media not found."))); }; - let content_disposition = make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + let content_disposition = + make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); let content = Content { file: content.expect("entire file contents"), content_type: content_type.map(Into::into), diff --git a/src/api/server/openid.rs b/src/api/server/openid.rs index 9b54807a..4833fbe1 100644 --- a/src/api/server/openid.rs +++ b/src/api/server/openid.rs @@ -7,7 +7,8 @@ use crate::{Result, Ruma}; /// /// Get information about the user that generated the OpenID token. pub(crate) async fn get_openid_userinfo_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { Ok(get_openid_userinfo::v1::Response::new( services diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index f6c41859..77cde15f 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -15,7 +15,8 @@ use crate::{Error, Result, Ruma}; /// Lists the public rooms on this server. #[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] pub(crate) async fn get_public_rooms_filtered_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if !services @@ -35,7 +36,9 @@ pub(crate) async fn get_public_rooms_filtered_route( &body.room_network, ) .await - .map_err(|_| Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list."))?; + .map_err(|_| { + Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list.") + })?; Ok(get_public_rooms_filtered::v1::Response { chunk: response.chunk, @@ -50,7 +53,8 @@ pub(crate) async fn get_public_rooms_filtered_route( /// Lists the public rooms on this server. #[tracing::instrument(skip_all, fields(%client), "publicrooms")] pub(crate) async fn get_public_rooms_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if !services @@ -69,7 +73,9 @@ pub(crate) async fn get_public_rooms_route( &body.room_network, ) .await - .map_err(|_| Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list."))?; + .map_err(|_| { + Error::BadRequest(ErrorKind::Unknown, "Failed to return this server's public room list.") + })?; Ok(get_public_rooms::v1::Response { chunk: response.chunk, diff --git a/src/api/server/query.rs b/src/api/server/query.rs index 6536bb20..0e5f7e56 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -19,7 +19,8 @@ use crate::Ruma; /// /// Resolve a room alias to a room id. pub(crate) async fn get_room_information_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { let room_id = services .rooms @@ -50,10 +51,7 @@ pub(crate) async fn get_room_information_route( servers.insert(0, services.globals.server_name().to_owned()); } - Ok(get_room_information::v1::Response { - room_id, - servers, - }) + Ok(get_room_information::v1::Response { room_id, servers }) } /// # `GET /_matrix/federation/v1/query/profile` @@ -61,7 +59,8 @@ pub(crate) async fn get_room_information_route( /// /// Gets information on a profile. pub(crate) async fn get_profile_information_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services .globals @@ -88,14 +87,14 @@ pub(crate) async fn get_profile_information_route( let mut custom_profile_fields = BTreeMap::new(); match &body.field { - Some(ProfileField::DisplayName) => { + | Some(ProfileField::DisplayName) => { displayname = services.users.displayname(&body.user_id).await.ok(); }, - Some(ProfileField::AvatarUrl) => { + | Some(ProfileField::AvatarUrl) => { avatar_url = services.users.avatar_url(&body.user_id).await.ok(); blurhash = services.users.blurhash(&body.user_id).await.ok(); }, - Some(custom_field) => { + | Some(custom_field) => { if let Ok(value) = services .users .profile_key(&body.user_id, custom_field.as_str()) @@ -104,7 +103,7 @@ pub(crate) async fn get_profile_information_route( custom_profile_fields.insert(custom_field.to_string(), value); } }, - None => { + | None => { displayname = services.users.displayname(&body.user_id).await.ok(); avatar_url = services.users.avatar_url(&body.user_id).await.ok(); blurhash = services.users.blurhash(&body.user_id).await.ok(); diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 960c79b0..c5fc7118 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -2,15 +2,18 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{debug, debug_warn, err, error, result::LogErr, trace, utils::ReadyExt, warn, Err, Error, Result}; +use conduwuit::{ + debug, debug_warn, err, error, result::LogErr, trace, utils::ReadyExt, warn, Err, Error, + Result, +}; use futures::StreamExt; use ruma::{ api::{ client::error::ErrorKind, federation::transactions::{ edu::{ - DeviceListUpdateContent, DirectDeviceContent, Edu, PresenceContent, ReceiptContent, - SigningKeyUpdateContent, TypingContent, + DeviceListUpdateContent, DirectDeviceContent, Edu, PresenceContent, + ReceiptContent, SigningKeyUpdateContent, TypingContent, }, send_transaction_message, }, @@ -38,7 +41,8 @@ type ResolvedMap = BTreeMap>; /// Push EDUs and PDUs to this server. #[tracing::instrument(skip_all, fields(%client, origin = body.origin().as_str()), name = "send")] pub(crate) async fn send_transaction_message_route( - State(services): State, InsecureClientIp(client): InsecureClientIp, + State(services): State, + InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { if body.origin() != body.body.origin { @@ -69,7 +73,8 @@ pub(crate) async fn send_transaction_message_route( "Starting txn", ); - let resolved_map = handle_pdus(&services, &client, &body.pdus, body.origin(), &txn_start_time).await?; + let resolved_map = + handle_pdus(&services, &client, &body.pdus, body.origin(), &txn_start_time).await?; handle_edus(&services, &client, &body.edus, body.origin()).await; debug!( @@ -90,13 +95,17 @@ pub(crate) async fn send_transaction_message_route( } async fn handle_pdus( - services: &Services, _client: &IpAddr, pdus: &[Box], origin: &ServerName, txn_start_time: &Instant, + services: &Services, + _client: &IpAddr, + pdus: &[Box], + origin: &ServerName, + txn_start_time: &Instant, ) -> Result { let mut parsed_pdus = Vec::with_capacity(pdus.len()); for pdu in pdus { parsed_pdus.push(match services.rooms.event_handler.parse_incoming_pdu(pdu).await { - Ok(t) => t, - Err(e) => { + | Ok(t) => t, + | Err(e) => { debug_warn!("Could not parse PDU: {e}"); continue; }, @@ -145,26 +154,45 @@ async fn handle_pdus( Ok(resolved_map) } -async fn handle_edus(services: &Services, client: &IpAddr, edus: &[Raw], origin: &ServerName) { +async fn handle_edus( + services: &Services, + client: &IpAddr, + edus: &[Raw], + origin: &ServerName, +) { for edu in edus .iter() .filter_map(|edu| serde_json::from_str::(edu.json().get()).ok()) { match edu { - Edu::Presence(presence) => handle_edu_presence(services, client, origin, presence).await, - Edu::Receipt(receipt) => handle_edu_receipt(services, client, origin, receipt).await, - Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await, - Edu::DeviceListUpdate(content) => handle_edu_device_list_update(services, client, origin, content).await, - Edu::DirectToDevice(content) => handle_edu_direct_to_device(services, client, origin, content).await, - Edu::SigningKeyUpdate(content) => handle_edu_signing_key_update(services, client, origin, content).await, - Edu::_Custom(ref _custom) => { + | Edu::Presence(presence) => { + handle_edu_presence(services, client, origin, presence).await; + }, + | Edu::Receipt(receipt) => + handle_edu_receipt(services, client, origin, receipt).await, + | Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await, + | Edu::DeviceListUpdate(content) => { + handle_edu_device_list_update(services, client, origin, content).await; + }, + | Edu::DirectToDevice(content) => { + handle_edu_direct_to_device(services, client, origin, content).await; + }, + | Edu::SigningKeyUpdate(content) => { + handle_edu_signing_key_update(services, client, origin, content).await; + }, + | Edu::_Custom(ref _custom) => { debug_warn!(?edus, "received custom/unknown EDU"); }, } } } -async fn handle_edu_presence(services: &Services, _client: &IpAddr, origin: &ServerName, presence: PresenceContent) { +async fn handle_edu_presence( + services: &Services, + _client: &IpAddr, + origin: &ServerName, + presence: PresenceContent, +) { if !services.globals.allow_incoming_presence() { return; } @@ -193,7 +221,12 @@ async fn handle_edu_presence(services: &Services, _client: &IpAddr, origin: &Ser } } -async fn handle_edu_receipt(services: &Services, _client: &IpAddr, origin: &ServerName, receipt: ReceiptContent) { +async fn handle_edu_receipt( + services: &Services, + _client: &IpAddr, + origin: &ServerName, + receipt: ReceiptContent, +) { if !services.globals.allow_incoming_read_receipts() { return; } @@ -230,7 +263,8 @@ async fn handle_edu_receipt(services: &Services, _client: &IpAddr, origin: &Serv .await { for event_id in &user_updates.event_ids { - let user_receipts = BTreeMap::from([(user_id.clone(), user_updates.data.clone())]); + let user_receipts = + BTreeMap::from([(user_id.clone(), user_updates.data.clone())]); let receipts = BTreeMap::from([(ReceiptType::Read, user_receipts)]); let receipt_content = BTreeMap::from([(event_id.to_owned(), receipts)]); let event = ReceiptEvent { @@ -255,7 +289,12 @@ async fn handle_edu_receipt(services: &Services, _client: &IpAddr, origin: &Serv } } -async fn handle_edu_typing(services: &Services, _client: &IpAddr, origin: &ServerName, typing: TypingContent) { +async fn handle_edu_typing( + services: &Services, + _client: &IpAddr, + origin: &ServerName, + typing: TypingContent, +) { if !services.globals.config.allow_incoming_typing { return; } @@ -321,12 +360,12 @@ async fn handle_edu_typing(services: &Services, _client: &IpAddr, origin: &Serve } async fn handle_edu_device_list_update( - services: &Services, _client: &IpAddr, origin: &ServerName, content: DeviceListUpdateContent, + services: &Services, + _client: &IpAddr, + origin: &ServerName, + content: DeviceListUpdateContent, ) { - let DeviceListUpdateContent { - user_id, - .. - } = content; + let DeviceListUpdateContent { user_id, .. } = content; if user_id.server_name() != origin { debug_warn!( @@ -340,14 +379,12 @@ async fn handle_edu_device_list_update( } async fn handle_edu_direct_to_device( - services: &Services, _client: &IpAddr, origin: &ServerName, content: DirectDeviceContent, + services: &Services, + _client: &IpAddr, + origin: &ServerName, + content: DirectDeviceContent, ) { - let DirectDeviceContent { - sender, - ev_type, - message_id, - messages, - } = content; + let DirectDeviceContent { sender, ev_type, message_id, messages } = content; if sender.server_name() != origin { debug_warn!( @@ -369,23 +406,28 @@ async fn handle_edu_direct_to_device( for (target_user_id, map) in &messages { for (target_device_id_maybe, event) in map { - let Ok(event) = event - .deserialize_as() - .map_err(|e| err!(Request(InvalidParam(error!("To-Device event is invalid: {e}"))))) - else { + let Ok(event) = event.deserialize_as().map_err(|e| { + err!(Request(InvalidParam(error!("To-Device event is invalid: {e}")))) + }) else { continue; }; let ev_type = ev_type.to_string(); match target_device_id_maybe { - DeviceIdOrAllDevices::DeviceId(target_device_id) => { + | DeviceIdOrAllDevices::DeviceId(target_device_id) => { services .users - .add_to_device_event(&sender, target_user_id, target_device_id, &ev_type, event) + .add_to_device_event( + &sender, + target_user_id, + target_device_id, + &ev_type, + event, + ) .await; }, - DeviceIdOrAllDevices::AllDevices => { + | DeviceIdOrAllDevices::AllDevices => { let (sender, ev_type, event) = (&sender, &ev_type, &event); services .users @@ -412,13 +454,12 @@ async fn handle_edu_direct_to_device( } async fn handle_edu_signing_key_update( - services: &Services, _client: &IpAddr, origin: &ServerName, content: SigningKeyUpdateContent, + services: &Services, + _client: &IpAddr, + origin: &ServerName, + content: SigningKeyUpdateContent, ) { - let SigningKeyUpdateContent { - user_id, - master_key, - self_signing_key, - } = content; + let SigningKeyUpdateContent { user_id, master_key, self_signing_key } = content; if user_id.server_name() != origin { debug_warn!( diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 9c72de2c..6ac84907 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -16,7 +16,8 @@ use ruma::{ room::member::{MembershipState, RoomMemberEventContent}, StateEventType, }, - CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, + CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, + ServerName, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; use service::Services; @@ -25,7 +26,10 @@ use crate::Ruma; /// helper method for /send_join v1 and v2 async fn create_join_event( - services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, + services: &Services, + origin: &ServerName, + room_id: &RoomId, + pdu: &RawJsonValue, ) -> Result { if !services.rooms.metadata.exists(room_id).await { return Err!(Request(NotFound("Room is unknown to this server."))); @@ -146,7 +150,8 @@ async fn create_join_event( if !services.globals.user_is_local(&authorising_user) { return Err!(Request(InvalidParam( - "Cannot authorise membership event through {authorising_user} as they do not belong to this homeserver" + "Cannot authorise membership event through {authorising_user} as they do not \ + belong to this homeserver" ))); } @@ -157,12 +162,19 @@ async fn create_join_event( .await { return Err!(Request(InvalidParam( - "Authorising user {authorising_user} is not in the room you are trying to join, they cannot authorise \ - your join." + "Authorising user {authorising_user} is not in the room you are trying to join, \ + they cannot authorise your join." ))); } - if !super::user_can_perform_restricted_join(services, &state_key, room_id, &room_version_id).await? { + if !super::user_can_perform_restricted_join( + services, + &state_key, + room_id, + &room_version_id, + ) + .await? + { return Err!(Request(UnableToAuthorizeJoin( "Joining user did not pass restricted room's rules." ))); @@ -228,7 +240,9 @@ async fn create_join_event( .event_ids_iter(room_id, starting_events) .await? .map(Ok) - .broad_and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) + .broad_and_then(|event_id| async move { + services.rooms.timeline.get_pdu_json(&event_id).await + }) .broad_and_then(|pdu| { services .sending @@ -252,7 +266,8 @@ async fn create_join_event( /// /// Submits a signed join event. pub(crate) async fn create_join_event_v1_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if services .globals @@ -261,8 +276,8 @@ pub(crate) async fn create_join_event_v1_route( .contains(body.origin()) { warn!( - "Server {} tried joining room ID {} through us who has a server name that is globally forbidden. \ - Rejecting.", + "Server {} tried joining room ID {} through us who has a server name that is \ + globally forbidden. Rejecting.", body.origin(), &body.room_id, ); @@ -277,8 +292,8 @@ pub(crate) async fn create_join_event_v1_route( .contains(&server.to_owned()) { warn!( - "Server {} tried joining room ID {} through us which has a server name that is globally forbidden. \ - Rejecting.", + "Server {} tried joining room ID {} through us which has a server name that is \ + globally forbidden. Rejecting.", body.origin(), &body.room_id, ); @@ -292,16 +307,15 @@ pub(crate) async fn create_join_event_v1_route( .boxed() .await?; - Ok(create_join_event::v1::Response { - room_state, - }) + Ok(create_join_event::v1::Response { room_state }) } /// # `PUT /_matrix/federation/v2/send_join/{roomId}/{eventId}` /// /// Submits a signed join event. pub(crate) async fn create_join_event_v2_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if services .globals @@ -320,8 +334,8 @@ pub(crate) async fn create_join_event_v2_route( .contains(&server.to_owned()) { warn!( - "Server {} tried joining room ID {} through us which has a server name that is globally forbidden. \ - Rejecting.", + "Server {} tried joining room ID {} through us which has a server name that is \ + globally forbidden. Rejecting.", body.origin(), &body.room_id, ); @@ -331,13 +345,10 @@ pub(crate) async fn create_join_event_v2_route( } } - let create_join_event::v1::RoomState { - auth_chain, - state, - event, - } = create_join_event(&services, body.origin(), &body.room_id, &body.pdu) - .boxed() - .await?; + let create_join_event::v1::RoomState { auth_chain, state, event } = + create_join_event(&services, body.origin(), &body.room_id, &body.pdu) + .boxed() + .await?; let room_state = create_join_event::v2::RoomState { members_omitted: false, auth_chain, @@ -346,7 +357,5 @@ pub(crate) async fn create_join_event_v2_route( servers_in_room: None, }; - Ok(create_join_event::v2::Response { - room_state, - }) + Ok(create_join_event::v2::Response { room_state }) } diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index b58bae7e..016ed7fa 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -21,7 +21,8 @@ use crate::{ /// /// Submits a signed leave event. pub(crate) async fn create_leave_event_v1_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { create_leave_event(&services, body.origin(), &body.room_id, &body.pdu).await?; @@ -32,14 +33,20 @@ pub(crate) async fn create_leave_event_v1_route( /// /// Submits a signed leave event. pub(crate) async fn create_leave_event_v2_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { create_leave_event(&services, body.origin(), &body.room_id, &body.pdu).await?; Ok(create_leave_event::v2::Response::new()) } -async fn create_leave_event(services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue) -> Result { +async fn create_leave_event( + services: &Services, + origin: &ServerName, + room_id: &RoomId, + pdu: &RawJsonValue, +) -> Result { if !services.rooms.metadata.exists(room_id).await { return Err!(Request(NotFound("Room is unknown to this server."))); } diff --git a/src/api/server/state.rs b/src/api/server/state.rs index ba1c400f..42f7e538 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -12,7 +12,8 @@ use crate::Ruma; /// /// Retrieves a snapshot of a room's state at a given event. pub(crate) async fn get_room_state_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { AccessCheck { services: &services, @@ -69,8 +70,5 @@ pub(crate) async fn get_room_state_route( .try_collect() .await?; - Ok(get_room_state::v1::Response { - auth_chain, - pdus, - }) + Ok(get_room_state::v1::Response { auth_chain, pdus }) } diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index 503104df..186ef399 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -13,7 +13,8 @@ use crate::Ruma; /// Retrieves a snapshot of a room's state at a given event, in the form of /// event IDs. pub(crate) async fn get_room_state_ids_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { AccessCheck { services: &services, @@ -49,8 +50,5 @@ pub(crate) async fn get_room_state_ids_route( .collect() .await; - Ok(get_room_state_ids::v1::Response { - auth_chain_ids, - pdu_ids, - }) + Ok(get_room_state_ids::v1::Response { auth_chain_ids, pdu_ids }) } diff --git a/src/api/server/user.rs b/src/api/server/user.rs index 5072ed2f..321d0b66 100644 --- a/src/api/server/user.rs +++ b/src/api/server/user.rs @@ -18,7 +18,8 @@ use crate::{ /// /// Gets information on all devices of the user. pub(crate) async fn get_devices_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.globals.user_is_local(&body.user_id) { return Err(Error::BadRequest( @@ -52,11 +53,7 @@ pub(crate) async fn get_devices_route( services .users .get_device_keys(user_id, &device_id_clone) - .map_ok(|keys| UserDevice { - device_id, - keys, - device_display_name, - }) + .map_ok(|keys| UserDevice { device_id, keys, device_display_name }) .map(Result::ok) .await }) @@ -79,7 +76,8 @@ pub(crate) async fn get_devices_route( /// /// Gets devices and identity keys for the given users. pub(crate) async fn get_keys_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if body .device_keys @@ -112,7 +110,8 @@ pub(crate) async fn get_keys_route( /// /// Claims one-time keys. pub(crate) async fn claim_keys_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if body .one_time_keys @@ -127,7 +126,5 @@ pub(crate) async fn claim_keys_route( let result = claim_keys_helper(&services, &body.one_time_keys).await?; - Ok(claim_keys::v1::Response { - one_time_keys: result.one_time_keys, - }) + Ok(claim_keys::v1::Response { one_time_keys: result.one_time_keys }) } diff --git a/src/api/server/utils.rs b/src/api/server/utils.rs index 3e768513..112cf858 100644 --- a/src/api/server/utils.rs +++ b/src/api/server/utils.rs @@ -34,10 +34,11 @@ pub(super) async fn check(&self) -> Result { let server_can_see: OptionFuture<_> = self .event_id .map(|event_id| { - self.services - .rooms - .state_accessor - .server_can_see_event(self.origin, self.room_id, event_id) + self.services.rooms.state_accessor.server_can_see_event( + self.origin, + self.room_id, + event_id, + ) }) .into(); diff --git a/src/api/server/well_known.rs b/src/api/server/well_known.rs index e6145aea..48caa7d6 100644 --- a/src/api/server/well_known.rs +++ b/src/api/server/well_known.rs @@ -7,12 +7,13 @@ use crate::{Error, Result, Ruma}; /// /// Returns the .well-known URL if it is configured, otherwise returns 404. pub(crate) async fn well_known_server( - State(services): State, _body: Ruma, + State(services): State, + _body: Ruma, ) -> Result { Ok(discover_homeserver::Response { server: match services.server.config.well_known.server.as_ref() { - Some(server_name) => server_name.to_owned(), - None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), + | Some(server_name) => server_name.to_owned(), + | None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), }, }) } diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index a3a7acc0..423f5408 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -29,8 +29,9 @@ pub fn memory_usage() -> Option { let resident = mibs(stats::resident::read()); let retained = mibs(stats::retained::read()); Some(format!( - "allocated: {allocated:.2} MiB\nactive: {active:.2} MiB\nmapped: {mapped:.2} MiB\nmetadata: {metadata:.2} \ - MiB\nresident: {resident:.2} MiB\nretained: {retained:.2} MiB\n" + "allocated: {allocated:.2} MiB\nactive: {active:.2} MiB\nmapped: {mapped:.2} \ + MiB\nmetadata: {metadata:.2} MiB\nresident: {resident:.2} MiB\nretained: {retained:.2} \ + MiB\n" )) } diff --git a/src/core/alloc/mod.rs b/src/core/alloc/mod.rs index 7941a99c..31eb033c 100644 --- a/src/core/alloc/mod.rs +++ b/src/core/alloc/mod.rs @@ -8,10 +8,20 @@ pub use je::{memory_stats, memory_usage}; #[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", not(feature = "jemalloc")))] pub mod hardened; -#[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", not(feature = "jemalloc")))] +#[cfg(all( + not(target_env = "msvc"), + feature = "hardened_malloc", + not(feature = "jemalloc") +))] pub use hardened::{memory_stats, memory_usage}; -#[cfg(any(target_env = "msvc", all(not(feature = "hardened_malloc"), not(feature = "jemalloc"))))] +#[cfg(any( + target_env = "msvc", + all(not(feature = "hardened_malloc"), not(feature = "jemalloc")) +))] pub mod default; -#[cfg(any(target_env = "msvc", all(not(feature = "hardened_malloc"), not(feature = "jemalloc"))))] +#[cfg(any( + target_env = "msvc", + all(not(feature = "hardened_malloc"), not(feature = "jemalloc")) +))] pub use default::{memory_stats, memory_usage}; diff --git a/src/core/config/check.rs b/src/core/config/check.rs index b8415281..c242e2fd 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -16,18 +16,24 @@ pub fn check(config: &Config) -> Result<()> { warn_unknown_key(config); if config.sentry && config.sentry_endpoint.is_none() { - return Err!(Config("sentry_endpoint", "Sentry cannot be enabled without an endpoint set")); + return Err!(Config( + "sentry_endpoint", + "Sentry cannot be enabled without an endpoint set" + )); } if cfg!(all(feature = "hardened_malloc", feature = "jemalloc")) { - info!("hardened_malloc and jemalloc compile-time features are both enabled, this causes jemalloc to be used."); + info!( + "hardened_malloc and jemalloc compile-time features are both enabled, this causes \ + jemalloc to be used." + ); } if cfg!(not(unix)) && config.unix_socket_path.is_some() { return Err!(Config( "unix_socket_path", - "UNIX socket support is only available on *nix platforms. Please remove 'unix_socket_path' from your \ - config." + "UNIX socket support is only available on *nix platforms. Please remove \ + 'unix_socket_path' from your config." )); } @@ -44,30 +50,36 @@ pub fn check(config: &Config) -> Result<()> { use std::path::Path; if addr.ip().is_loopback() { - debug_info!("Found loopback listening address {addr}, running checks if we're in a container."); + debug_info!( + "Found loopback listening address {addr}, running checks if we're in a \ + container." + ); if Path::new("/proc/vz").exists() /* Guest */ && !Path::new("/proc/bz").exists() /* Host */ { error!( - "You are detected using OpenVZ with a loopback/localhost listening address of {addr}. If you \ - are using OpenVZ for containers and you use NAT-based networking to communicate with the \ - host and guest, this will NOT work. Please change this to \"0.0.0.0\". If this is expected, \ - you can ignore.", + "You are detected using OpenVZ with a loopback/localhost listening \ + address of {addr}. If you are using OpenVZ for containers and you use \ + NAT-based networking to communicate with the host and guest, this will \ + NOT work. Please change this to \"0.0.0.0\". If this is expected, you \ + can ignore.", ); } else if Path::new("/.dockerenv").exists() { error!( - "You are detected using Docker with a loopback/localhost listening address of {addr}. If you \ - are using a reverse proxy on the host and require communication to conduwuit in the Docker \ - container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ - If this is expected, you can ignore.", + "You are detected using Docker with a loopback/localhost listening \ + address of {addr}. If you are using a reverse proxy on the host and \ + require communication to conduwuit in the Docker container via \ + NAT-based networking, this will NOT work. Please change this to \ + \"0.0.0.0\". If this is expected, you can ignore.", ); } else if Path::new("/run/.containerenv").exists() { error!( - "You are detected using Podman with a loopback/localhost listening address of {addr}. If you \ - are using a reverse proxy on the host and require communication to conduwuit in the Podman \ - container via NAT-based networking, this will NOT work. Please change this to \"0.0.0.0\". \ - If this is expected, you can ignore.", + "You are detected using Podman with a loopback/localhost listening \ + address of {addr}. If you are using a reverse proxy on the host and \ + require communication to conduwuit in the Podman container via \ + NAT-based networking, this will NOT work. Please change this to \ + \"0.0.0.0\". If this is expected, you can ignore.", ); } } @@ -93,7 +105,8 @@ pub fn check(config: &Config) -> Result<()> { if config.emergency_password == Some(String::from("F670$2CP@Hw8mG7RY1$%!#Ic7YA")) { return Err!(Config( "emergency_password", - "The public example emergency password is being used, this is insecure. Please change this." + "The public example emergency password is being used, this is insecure. Please \ + change this." )); } @@ -124,7 +137,8 @@ pub fn check(config: &Config) -> Result<()> { if config.max_request_size < 10_000_000 { return Err!(Config( "max_request_size", - "Max request size is less than 10MB. Please increase it as this is too low for operable federation." + "Max request size is less than 10MB. Please increase it as this is too low for \ + operable federation." )); } @@ -145,11 +159,12 @@ pub fn check(config: &Config) -> Result<()> { { return Err!(Config( "registration_token", - "!! You have `allow_registration` enabled without a token configured in your config which means you are \ - allowing ANYONE to register on your conduwuit instance without any 2nd-step (e.g. registration token). \ - If this is not the intended behaviour, please set a registration token. For security and safety reasons, \ - conduwuit will shut down. If you are extra sure this is the desired behaviour you want, please set the \ - following config option to true: + "!! You have `allow_registration` enabled without a token configured in your config \ + which means you are allowing ANYONE to register on your conduwuit instance without \ + any 2nd-step (e.g. registration token). If this is not the intended behaviour, \ + please set a registration token. For security and safety reasons, conduwuit will \ + shut down. If you are extra sure this is the desired behaviour you want, please \ + set the following config option to true: `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse`" )); } @@ -161,17 +176,18 @@ pub fn check(config: &Config) -> Result<()> { { warn!( "Open registration is enabled via setting \ - `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` and `allow_registration` to \ - true without a registration token configured. You are expected to be aware of the risks now. If this is \ - not the desired behaviour, please set a registration token." + `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` and \ + `allow_registration` to true without a registration token configured. You are \ + expected to be aware of the risks now. If this is not the desired behaviour, \ + please set a registration token." ); } if config.allow_outgoing_presence && !config.allow_local_presence { return Err!(Config( "allow_local_presence", - "Outgoing presence requires allowing local presence. Please enable 'allow_local_presence' or disable \ - outgoing presence." + "Outgoing presence requires allowing local presence. Please enable \ + 'allow_local_presence' or disable outgoing presence." )); } @@ -180,9 +196,10 @@ pub fn check(config: &Config) -> Result<()> { .contains(&"*".to_owned()) { warn!( - "All URLs are allowed for URL previews via setting \"url_preview_domain_contains_allowlist\" to \"*\". \ - This opens up significant attack surface to your server. You are expected to be aware of the risks by \ - doing this." + "All URLs are allowed for URL previews via setting \ + \"url_preview_domain_contains_allowlist\" to \"*\". This opens up significant \ + attack surface to your server. You are expected to be aware of the risks by doing \ + this." ); } if config @@ -190,9 +207,10 @@ pub fn check(config: &Config) -> Result<()> { .contains(&"*".to_owned()) { warn!( - "All URLs are allowed for URL previews via setting \"url_preview_domain_explicit_allowlist\" to \"*\". \ - This opens up significant attack surface to your server. You are expected to be aware of the risks by \ - doing this." + "All URLs are allowed for URL previews via setting \ + \"url_preview_domain_explicit_allowlist\" to \"*\". This opens up significant \ + attack surface to your server. You are expected to be aware of the risks by doing \ + this." ); } if config @@ -200,9 +218,9 @@ pub fn check(config: &Config) -> Result<()> { .contains(&"*".to_owned()) { warn!( - "All URLs are allowed for URL previews via setting \"url_preview_url_contains_allowlist\" to \"*\". This \ - opens up significant attack surface to your server. You are expected to be aware of the risks by doing \ - this." + "All URLs are allowed for URL previews via setting \ + \"url_preview_url_contains_allowlist\" to \"*\". This opens up significant attack \ + surface to your server. You are expected to be aware of the risks by doing this." ); } @@ -260,7 +278,8 @@ pub(super) fn is_dual_listening(raw_config: &Figment) -> Result<()> { let contains_unix_socket = raw_config.contains("unix_socket_path"); if contains_address && contains_unix_socket { return Err!( - "TOML keys \"address\" and \"unix_socket_path\" were both defined. Please specify only one option." + "TOML keys \"address\" and \"unix_socket_path\" were both defined. Please specify \ + only one option." ); } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index a34514de..44b66f41 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -18,8 +18,8 @@ pub use figment::{value::Value as FigmentValue, Figment}; use itertools::Itertools; use regex::RegexSet; use ruma::{ - api::client::discovery::discover_support::ContactRole, OwnedRoomOrAliasId, OwnedServerName, OwnedUserId, - RoomVersionId, + api::client::discovery::discover_support::ContactRole, OwnedRoomOrAliasId, OwnedServerName, + OwnedUserId, RoomVersionId, }; use serde::{de::IgnoredAny, Deserialize}; use url::Url; @@ -181,7 +181,10 @@ pub struct Config { /// are scaled by your CPU core count. /// /// default: 1.0 - #[serde(default = "default_cache_capacity_modifier", alias = "conduit_cache_capacity_modifier")] + #[serde( + default = "default_cache_capacity_modifier", + alias = "conduit_cache_capacity_modifier" + )] pub cache_capacity_modifier: f64, /// default: varies by system @@ -1555,7 +1558,8 @@ pub struct Config { pub db_pool_queue_size: usize, #[serde(flatten)] - #[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime + #[allow(clippy::zero_sized_map_values)] + // this is a catchall, the map shouldn't be zero at runtime catchall: BTreeMap, } @@ -1676,15 +1680,15 @@ impl Config { fn get_bind_hosts(&self) -> Vec { match &self.address.addrs { - Left(addr) => vec![*addr], - Right(addrs) => addrs.clone(), + | Left(addr) => vec![*addr], + | Right(addrs) => addrs.clone(), } } fn get_bind_ports(&self) -> Vec { match &self.port.ports { - Left(port) => vec![*port], - Right(ports) => ports.clone(), + | Left(port) => vec![*port], + | Right(ports) => ports.clone(), } } @@ -1756,9 +1760,13 @@ impl fmt::Display for Config { line("Allow registration", &self.allow_registration.to_string()); line( "Registration token", - if self.registration_token.is_none() && self.registration_token_file.is_none() && self.allow_registration { + if self.registration_token.is_none() + && self.registration_token_file.is_none() + && self.allow_registration + { "not set (⚠️ open registration!)" - } else if self.registration_token.is_none() && self.registration_token_file.is_none() { + } else if self.registration_token.is_none() && self.registration_token_file.is_none() + { "not set" } else { "set" @@ -1811,7 +1819,8 @@ impl fmt::Display for Config { &self.allow_outgoing_read_receipts.to_string(), ); line( - "Block non-admin room invites (local and remote, admins can still send and receive invites)", + "Block non-admin room invites (local and remote, admins can still send and receive \ + invites)", &self.block_non_admin_invites.to_string(), ); line("Enable admin escape commands", &self.admin_escape_commands.to_string()); @@ -1859,13 +1868,10 @@ impl fmt::Display for Config { "Lockdown public room directory (only allow admins to publish)", &self.lockdown_public_room_directory.to_string(), ); - line( - "JWT secret", - match self.jwt_secret { - Some(_) => "set", - None => "not set", - }, - ); + line("JWT secret", match self.jwt_secret { + | Some(_) => "set", + | None => "not set", + }); line( "Trusted key servers", &self @@ -1979,7 +1985,8 @@ impl fmt::Display for Config { &lst.join(", ") }); line("Forbidden Remote Room Directory Server Names", { - let mut lst = Vec::with_capacity(self.forbidden_remote_room_directory_server_names.len()); + let mut lst = + Vec::with_capacity(self.forbidden_remote_room_directory_server_names.len()); for domain in &self.forbidden_remote_room_directory_server_names { lst.push(domain.host()); } @@ -2099,11 +2106,7 @@ fn default_address() -> ListeningAddr { } } -fn default_port() -> ListeningPort { - ListeningPort { - ports: Left(8008), - } -} +fn default_port() -> ListeningPort { ListeningPort { ports: Left(8008) } } fn default_unix_socket_perms() -> u32 { 660 } @@ -2115,19 +2118,33 @@ fn default_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturati fn default_cache_capacity_modifier() -> f64 { 1.0 } -fn default_auth_chain_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } +fn default_auth_chain_cache_capacity() -> u32 { + parallelism_scaled_u32(10_000).saturating_add(100_000) +} -fn default_shorteventid_cache_capacity() -> u32 { parallelism_scaled_u32(50_000).saturating_add(100_000) } +fn default_shorteventid_cache_capacity() -> u32 { + parallelism_scaled_u32(50_000).saturating_add(100_000) +} -fn default_eventidshort_cache_capacity() -> u32 { parallelism_scaled_u32(25_000).saturating_add(100_000) } +fn default_eventidshort_cache_capacity() -> u32 { + parallelism_scaled_u32(25_000).saturating_add(100_000) +} -fn default_eventid_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(25_000).saturating_add(100_000) } +fn default_eventid_pdu_cache_capacity() -> u32 { + parallelism_scaled_u32(25_000).saturating_add(100_000) +} -fn default_shortstatekey_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } +fn default_shortstatekey_cache_capacity() -> u32 { + parallelism_scaled_u32(10_000).saturating_add(100_000) +} -fn default_statekeyshort_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } +fn default_statekeyshort_cache_capacity() -> u32 { + parallelism_scaled_u32(10_000).saturating_add(100_000) +} -fn default_servernameevent_data_cache_capacity() -> u32 { parallelism_scaled_u32(100_000).saturating_add(500_000) } +fn default_servernameevent_data_cache_capacity() -> u32 { + parallelism_scaled_u32(100_000).saturating_add(500_000) +} fn default_server_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(500) } @@ -2203,7 +2220,9 @@ fn default_jaeger_filter() -> String { fn default_tracing_flame_output_path() -> String { "./tracing.folded".to_owned() } -fn default_trusted_servers() -> Vec { vec![OwnedServerName::try_from("matrix.org").unwrap()] } +fn default_trusted_servers() -> Vec { + vec![OwnedServerName::try_from("matrix.org").unwrap()] +} /// do debug logging by default for debug builds #[must_use] @@ -2332,4 +2351,6 @@ fn default_trusted_server_batch_size() -> usize { 256 } fn default_db_pool_workers() -> usize { sys::available_parallelism().saturating_mul(4).max(32) } -fn default_db_pool_queue_size() -> usize { sys::available_parallelism().saturating_mul(8).max(256) } +fn default_db_pool_queue_size() -> usize { + sys::available_parallelism().saturating_mul(8).max(256) +} diff --git a/src/core/config/proxy.rs b/src/core/config/proxy.rs index 48f883c6..ea388f24 100644 --- a/src/core/config/proxy.rs +++ b/src/core/config/proxy.rs @@ -42,11 +42,9 @@ pub enum ProxyConfig { impl ProxyConfig { pub fn to_proxy(&self) -> Result> { Ok(match self.clone() { - Self::None => None, - Self::Global { - url, - } => Some(Proxy::all(url)?), - Self::ByDomain(proxies) => Some(Proxy::custom(move |url| { + | Self::None => None, + | Self::Global { url } => Some(Proxy::all(url)?), + | Self::ByDomain(proxies) => Some(Proxy::custom(move |url| { // first matching proxy proxies.iter().find_map(|proxy| proxy.for_url(url)).cloned() })), @@ -76,24 +74,26 @@ impl PartialProxyConfig { for wc_domain in &self.include { if wc_domain.matches(domain) { match included_because { - Some(prev) if !wc_domain.more_specific_than(prev) => (), - _ => included_because = Some(wc_domain), + | Some(prev) if !wc_domain.more_specific_than(prev) => (), + | _ => included_because = Some(wc_domain), } } } for wc_domain in &self.exclude { if wc_domain.matches(domain) { match excluded_because { - Some(prev) if !wc_domain.more_specific_than(prev) => (), - _ => excluded_because = Some(wc_domain), + | Some(prev) if !wc_domain.more_specific_than(prev) => (), + | _ => excluded_because = Some(wc_domain), } } } match (included_because, excluded_because) { - (Some(a), Some(b)) if a.more_specific_than(b) => Some(&self.url), /* included for a more specific reason */ + | (Some(a), Some(b)) if a.more_specific_than(b) => Some(&self.url), /* included for + * a more specific + * reason */ // than excluded - (Some(_), None) => Some(&self.url), - _ => None, + | (Some(_), None) => Some(&self.url), + | _ => None, } } } @@ -108,19 +108,19 @@ enum WildCardedDomain { impl WildCardedDomain { fn matches(&self, domain: &str) -> bool { match self { - Self::WildCard => true, - Self::WildCarded(d) => domain.ends_with(d), - Self::Exact(d) => domain == d, + | Self::WildCard => true, + | Self::WildCarded(d) => domain.ends_with(d), + | Self::Exact(d) => domain == d, } } fn more_specific_than(&self, other: &Self) -> bool { match (self, other) { - (Self::WildCard, Self::WildCard) => false, - (_, Self::WildCard) => true, - (Self::Exact(a), Self::WildCarded(_)) => other.matches(a), - (Self::WildCarded(a), Self::WildCarded(b)) => a != b && a.ends_with(b), - _ => false, + | (Self::WildCard, Self::WildCard) => false, + | (_, Self::WildCard) => true, + | (Self::Exact(a), Self::WildCarded(_)) => other.matches(a), + | (Self::WildCarded(a), Self::WildCarded(b)) => a != b && a.ends_with(b), + | _ => false, } } } diff --git a/src/core/debug.rs b/src/core/debug.rs index e1980234..123cf820 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -84,7 +84,9 @@ pub fn trap() { } #[must_use] -pub fn panic_str(p: &Box) -> &'static str { p.downcast_ref::<&str>().copied().unwrap_or_default() } +pub fn panic_str(p: &Box) -> &'static str { + p.downcast_ref::<&str>().copied().unwrap_or_default() +} #[inline(always)] #[must_use] @@ -96,4 +98,6 @@ pub fn type_name() -> &'static str { std::any::type_name::() } #[must_use] #[inline] -pub const fn logging() -> bool { cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) } +pub const fn logging() -> bool { + cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) +} diff --git a/src/core/error/err.rs b/src/core/error/err.rs index a24441e0..60fa5bff 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -184,7 +184,12 @@ impl Visit for Visitor<'_> { } } -pub fn visit(out: &mut String, level: Level, __callsite: &'static DefaultCallsite, vs: &mut ValueSet<'_>) { +pub fn visit( + out: &mut String, + level: Level, + __callsite: &'static DefaultCallsite, + vs: &mut ValueSet<'_>, +) { let meta = __callsite.metadata(); let enabled = level_enabled!(level) && { let interest = __callsite.interest(); diff --git a/src/core/error/log.rs b/src/core/error/log.rs index 60bd7014..18c3a680 100644 --- a/src/core/error/log.rs +++ b/src/core/error/log.rs @@ -68,18 +68,20 @@ where pub fn inspect_log(error: &E) { inspect_log_level(error, Level::ERROR); } #[inline] -pub fn inspect_debug_log(error: &E) { inspect_debug_log_level(error, Level::ERROR); } +pub fn inspect_debug_log(error: &E) { + inspect_debug_log_level(error, Level::ERROR); +} #[inline] pub fn inspect_log_level(error: &E, level: Level) { use crate::{debug, error, info, trace, warn}; match level { - Level::ERROR => error!("{error}"), - Level::WARN => warn!("{error}"), - Level::INFO => info!("{error}"), - Level::DEBUG => debug!("{error}"), - Level::TRACE => trace!("{error}"), + | Level::ERROR => error!("{error}"), + | Level::WARN => warn!("{error}"), + | Level::INFO => info!("{error}"), + | Level::DEBUG => debug!("{error}"), + | Level::TRACE => trace!("{error}"), } } @@ -88,10 +90,10 @@ pub fn inspect_debug_log_level(error: &E, level: Level) { use crate::{debug, debug_error, debug_info, debug_warn, trace}; match level { - Level::ERROR => debug_error!("{error:?}"), - Level::WARN => debug_warn!("{error:?}"), - Level::INFO => debug_info!("{error:?}"), - Level::DEBUG => debug!("{error:?}"), - Level::TRACE => trace!("{error:?}"), + | Level::ERROR => debug_error!("{error:?}"), + | Level::WARN => debug_warn!("{error:?}"), + | Level::INFO => debug_info!("{error:?}"), + | Level::DEBUG => debug!("{error:?}"), + | Level::TRACE => trace!("{error:?}"), } } diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index b84f1b46..12ba0797 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -128,23 +128,25 @@ pub enum Error { impl Error { //#[deprecated] - pub fn bad_database(message: &'static str) -> Self { crate::err!(Database(error!("{message}"))) } + pub fn bad_database(message: &'static str) -> Self { + crate::err!(Database(error!("{message}"))) + } /// Sanitizes public-facing errors that can leak sensitive information. pub fn sanitized_message(&self) -> String { match self { - Self::Database(..) => String::from("Database error occurred."), - Self::Io(..) => String::from("I/O error occurred."), - _ => self.message(), + | Self::Database(..) => String::from("Database error occurred."), + | Self::Io(..) => String::from("I/O error occurred."), + | _ => self.message(), } } /// Generate the error message string. pub fn message(&self) -> String { match self { - Self::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), - Self::Ruma(ref error) => response::ruma_error_message(error), - _ => format!("{self}"), + | Self::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), + | Self::Ruma(ref error) => response::ruma_error_message(error), + | _ => format!("{self}"), } } @@ -154,9 +156,10 @@ impl Error { use ruma::api::client::error::ErrorKind::Unknown; match self { - Self::Federation(_, error) | Self::Ruma(error) => response::ruma_error_kind(error).clone(), - Self::BadRequest(kind, ..) | Self::Request(kind, ..) => kind.clone(), - _ => Unknown, + | Self::Federation(_, error) | Self::Ruma(error) => + response::ruma_error_kind(error).clone(), + | Self::BadRequest(kind, ..) | Self::Request(kind, ..) => kind.clone(), + | _ => Unknown, } } @@ -166,12 +169,12 @@ impl Error { use http::StatusCode; match self { - Self::Federation(_, error) | Self::Ruma(error) => error.status_code, - Self::Request(kind, _, code) => response::status_code(kind, *code), - Self::BadRequest(kind, ..) => response::bad_request_code(kind), - Self::Reqwest(error) => error.status().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), - Self::Conflict(_) => StatusCode::CONFLICT, - _ => StatusCode::INTERNAL_SERVER_ERROR, + | Self::Federation(_, error) | Self::Ruma(error) => error.status_code, + | Self::Request(kind, _, code) => response::status_code(kind, *code), + | Self::BadRequest(kind, ..) => response::bad_request_code(kind), + | Self::Reqwest(error) => error.status().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), + | Self::Conflict(_) => StatusCode::CONFLICT, + | _ => StatusCode::INTERNAL_SERVER_ERROR, } } diff --git a/src/core/error/panic.rs b/src/core/error/panic.rs index bec25132..c6a83ae0 100644 --- a/src/core/error/panic.rs +++ b/src/core/error/panic.rs @@ -20,9 +20,9 @@ impl Error { #[inline] pub fn into_panic(self) -> Box { match self { - Self::Panic(_, e) | Self::PanicAny(e) => e, - Self::JoinError(e) => e.into_panic(), - _ => Box::new(self), + | Self::Panic(_, e) | Self::PanicAny(e) => e, + | Self::JoinError(e) => e.into_panic(), + | _ => Box::new(self), } } @@ -37,9 +37,9 @@ impl Error { #[inline] pub fn is_panic(&self) -> bool { match &self { - Self::Panic(..) | Self::PanicAny(..) => true, - Self::JoinError(e) => e.is_panic(), - _ => false, + | Self::Panic(..) | Self::PanicAny(..) => true, + | Self::JoinError(e) => e.is_panic(), + | _ => false, } } } diff --git a/src/core/error/response.rs b/src/core/error/response.rs index 21fbdcf2..568238c3 100644 --- a/src/core/error/response.rs +++ b/src/core/error/response.rs @@ -57,49 +57,35 @@ pub(super) fn bad_request_code(kind: &ErrorKind) -> StatusCode { match kind { // 429 - LimitExceeded { - .. - } => StatusCode::TOO_MANY_REQUESTS, + | LimitExceeded { .. } => StatusCode::TOO_MANY_REQUESTS, // 413 - TooLarge => StatusCode::PAYLOAD_TOO_LARGE, + | TooLarge => StatusCode::PAYLOAD_TOO_LARGE, // 405 - Unrecognized => StatusCode::METHOD_NOT_ALLOWED, + | Unrecognized => StatusCode::METHOD_NOT_ALLOWED, // 404 - NotFound => StatusCode::NOT_FOUND, + | NotFound => StatusCode::NOT_FOUND, // 403 - GuestAccessForbidden + | GuestAccessForbidden | ThreepidAuthFailed | UserDeactivated | ThreepidDenied - | WrongRoomKeysVersion { - .. - } - | Forbidden { - .. - } => StatusCode::FORBIDDEN, + | WrongRoomKeysVersion { .. } + | Forbidden { .. } => StatusCode::FORBIDDEN, // 401 - UnknownToken { - .. - } - | MissingToken - | Unauthorized => StatusCode::UNAUTHORIZED, + | UnknownToken { .. } | MissingToken | Unauthorized => StatusCode::UNAUTHORIZED, // 400 - _ => StatusCode::BAD_REQUEST, + | _ => StatusCode::BAD_REQUEST, } } pub(super) fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { - if let ErrorBody::Standard { - message, - .. - } = &error.body - { + if let ErrorBody::Standard { message, .. } = &error.body { return message.to_string(); } diff --git a/src/core/info/cargo.rs b/src/core/info/cargo.rs index 735ccee6..c5a1d167 100644 --- a/src/core/info/cargo.rs +++ b/src/core/info/cargo.rs @@ -41,17 +41,22 @@ static FEATURES: OnceLock> = OnceLock::new(); static DEPENDENCIES: OnceLock = OnceLock::new(); #[must_use] -pub fn dependencies_names() -> Vec<&'static str> { dependencies().keys().map(String::as_str).collect() } +pub fn dependencies_names() -> Vec<&'static str> { + dependencies().keys().map(String::as_str).collect() +} pub fn dependencies() -> &'static DepsSet { - DEPENDENCIES - .get_or_init(|| init_dependencies().unwrap_or_else(|e| panic!("Failed to initialize dependencies: {e}"))) + DEPENDENCIES.get_or_init(|| { + init_dependencies().unwrap_or_else(|e| panic!("Failed to initialize dependencies: {e}")) + }) } /// List of all possible features for the project. For *enabled* features in /// this build see the companion function in info::rustc. pub fn features() -> &'static Vec { - FEATURES.get_or_init(|| init_features().unwrap_or_else(|e| panic!("Failed initialize features: {e}"))) + FEATURES.get_or_init(|| { + init_features().unwrap_or_else(|e| panic!("Failed initialize features: {e}")) + }) } fn init_features() -> Result> { diff --git a/src/core/info/room_version.rs b/src/core/info/room_version.rs index 59685b5f..40f0cf0a 100644 --- a/src/core/info/room_version.rs +++ b/src/core/info/room_version.rs @@ -34,7 +34,9 @@ impl crate::Server { } #[inline] - pub fn available_room_versions(&self) -> impl Iterator { + pub fn available_room_versions( + &self, + ) -> impl Iterator { available_room_versions() } diff --git a/src/core/log/capture/layer.rs b/src/core/log/capture/layer.rs index 674bc379..381a652f 100644 --- a/src/core/log/capture/layer.rs +++ b/src/core/log/capture/layer.rs @@ -22,11 +22,7 @@ type ScopeNames = ArrayVec<&'static str, 32>; impl Layer { #[inline] - pub fn new(state: &Arc) -> Self { - Self { - state: state.clone(), - } - } + pub fn new(state: &Arc) -> Self { Self { state: state.clone() } } } impl fmt::Debug for Layer { @@ -56,9 +52,7 @@ where S: Subscriber + for<'a> LookupSpan<'a>, { let names = ScopeNames::new(); - let mut visitor = Visitor { - values: Values::new(), - }; + let mut visitor = Visitor { values: Values::new() }; event.record(&mut visitor); let mut closure = capture.closure.lock().expect("exclusive lock"); @@ -83,7 +77,7 @@ where } } - capture.filter.as_ref().map_or(true, |filter| { + capture.filter.as_ref().is_none_or(|filter| { filter(Data { layer, event, @@ -95,7 +89,9 @@ where } impl Visit for Visitor { - fn record_debug(&mut self, f: &Field, v: &dyn fmt::Debug) { self.values.push((f.name(), format!("{v:?}"))); } + fn record_debug(&mut self, f: &Field, v: &dyn fmt::Debug) { + self.values.push((f.name(), format!("{v:?}"))); + } fn record_str(&mut self, f: &Field, v: &str) { self.values.push((f.name(), v.to_owned())); } } diff --git a/src/core/log/capture/mod.rs b/src/core/log/capture/mod.rs index c0fcd31e..20f70091 100644 --- a/src/core/log/capture/mod.rs +++ b/src/core/log/capture/mod.rs @@ -41,9 +41,7 @@ impl Capture { #[must_use] pub fn start(self: &Arc) -> Guard { self.state.add(self); - Guard { - capture: self.clone(), - } + Guard { capture: self.clone() } } pub fn stop(self: &Arc) { self.state.del(self); } diff --git a/src/core/log/capture/state.rs b/src/core/log/capture/state.rs index f2c401cd..dad6c8d8 100644 --- a/src/core/log/capture/state.rs +++ b/src/core/log/capture/state.rs @@ -13,11 +13,7 @@ impl Default for State { impl State { #[must_use] - pub fn new() -> Self { - Self { - active: RwLock::new(Vec::new()), - } - } + pub fn new() -> Self { Self { active: RwLock::new(Vec::new()) } } pub(super) fn add(&self, capture: &Arc) { self.active diff --git a/src/core/log/color.rs b/src/core/log/color.rs index 9baa99c2..23905d14 100644 --- a/src/core/log/color.rs +++ b/src/core/log/color.rs @@ -5,11 +5,11 @@ use super::Level; #[must_use] pub fn html(level: &Level) -> (&'static str, &'static str) { match *level { - Level::TRACE => ("#000000", "#A0A0A0"), - Level::DEBUG => ("#000000", "#FFFFFF"), - Level::ERROR => ("#000000", "#FF0000"), - Level::WARN => ("#000000", "#FFFF00"), - Level::INFO => ("#FFFFFF", "#008E00"), + | Level::TRACE => ("#000000", "#A0A0A0"), + | Level::DEBUG => ("#000000", "#FFFFFF"), + | Level::ERROR => ("#000000", "#FF0000"), + | Level::WARN => ("#000000", "#FFFF00"), + | Level::INFO => ("#FFFFFF", "#008E00"), } } @@ -18,10 +18,10 @@ pub fn html(level: &Level) -> (&'static str, &'static str) { #[must_use] pub fn code_tag(level: &Level) -> &'static str { match *level { - Level::TRACE => "#888888", - Level::DEBUG => "#C8C8C8", - Level::ERROR => "#FF0000", - Level::WARN => "#FFFF00", - Level::INFO => "#00FF00", + | Level::TRACE => "#888888", + | Level::DEBUG => "#C8C8C8", + | Level::ERROR => "#FF0000", + | Level::WARN => "#FFFF00", + | Level::INFO => "#00FF00", } } diff --git a/src/core/log/fmt.rs b/src/core/log/fmt.rs index 8766eb8d..353d4442 100644 --- a/src/core/log/fmt.rs +++ b/src/core/log/fmt.rs @@ -11,7 +11,8 @@ where let level = level.as_str().to_uppercase(); write!( out, - "{level:>5} {span:^12} {msg}
    " + "{level:>5} {span:^12} \ + {msg}
    " )?; Ok(()) diff --git a/src/core/log/fmt_span.rs b/src/core/log/fmt_span.rs index 5a340d0f..7ba86c4c 100644 --- a/src/core/log/fmt_span.rs +++ b/src/core/log/fmt_span.rs @@ -5,13 +5,13 @@ use crate::Result; #[inline] pub fn from_str(str: &str) -> Result { match str.to_uppercase().as_str() { - "ENTER" => Ok(FmtSpan::ENTER), - "EXIT" => Ok(FmtSpan::EXIT), - "NEW" => Ok(FmtSpan::NEW), - "CLOSE" => Ok(FmtSpan::CLOSE), - "ACTIVE" => Ok(FmtSpan::ACTIVE), - "FULL" => Ok(FmtSpan::FULL), - "NONE" => Ok(FmtSpan::NONE), - _ => Err(FmtSpan::NONE), + | "ENTER" => Ok(FmtSpan::ENTER), + | "EXIT" => Ok(FmtSpan::EXIT), + | "NEW" => Ok(FmtSpan::NEW), + | "CLOSE" => Ok(FmtSpan::CLOSE), + | "ACTIVE" => Ok(FmtSpan::ACTIVE), + | "FULL" => Ok(FmtSpan::FULL), + | "NONE" => Ok(FmtSpan::NONE), + | _ => Err(FmtSpan::NONE), } } diff --git a/src/core/log/suppress.rs b/src/core/log/suppress.rs index b13ee99e..55428d15 100644 --- a/src/core/log/suppress.rs +++ b/src/core/log/suppress.rs @@ -25,10 +25,7 @@ impl Suppress { .reload(&suppress, Some(&[handle])) .expect("log filter reloaded"); - Self { - server: server.clone(), - restore, - } + Self { server: server.clone(), restore } } } diff --git a/src/core/metrics/mod.rs b/src/core/metrics/mod.rs index 3ae139a8..d5f937d7 100644 --- a/src/core/metrics/mod.rs +++ b/src/core/metrics/mod.rs @@ -68,5 +68,7 @@ impl Metrics { pub fn task_root(&self) -> Option<&TaskMonitor> { self.task_monitor.as_ref() } - pub fn runtime_metrics(&self) -> Option<&runtime::RuntimeMetrics> { self.runtime_metrics.as_ref() } + pub fn runtime_metrics(&self) -> Option<&runtime::RuntimeMetrics> { + self.runtime_metrics.as_ref() + } } diff --git a/src/core/pdu/builder.rs b/src/core/pdu/builder.rs index 80ff0713..db3bb20a 100644 --- a/src/core/pdu/builder.rs +++ b/src/core/pdu/builder.rs @@ -35,7 +35,8 @@ impl Builder { { Self { event_type: content.event_type().into(), - content: to_raw_value(content).expect("Builder failed to serialize state event content to RawValue"), + content: to_raw_value(content) + .expect("Builder failed to serialize state event content to RawValue"), state_key: Some(state_key), ..Self::default() } @@ -47,7 +48,8 @@ impl Builder { { Self { event_type: content.event_type().into(), - content: to_raw_value(content).expect("Builder failed to serialize timeline event content to RawValue"), + content: to_raw_value(content) + .expect("Builder failed to serialize timeline event content to RawValue"), ..Self::default() } } diff --git a/src/core/pdu/count.rs b/src/core/pdu/count.rs index 85222382..0135cf28 100644 --- a/src/core/pdu/count.rs +++ b/src/core/pdu/count.rs @@ -21,8 +21,8 @@ impl Count { #[must_use] pub fn from_signed(signed: i64) -> Self { match signed { - i64::MIN..=0 => Self::Backfilled(signed), - _ => Self::Normal(signed as u64), + | i64::MIN..=0 => Self::Backfilled(signed), + | _ => Self::Normal(signed as u64), } } @@ -31,8 +31,8 @@ impl Count { pub fn into_unsigned(self) -> u64 { self.debug_assert_valid(); match self { - Self::Normal(i) => i, - Self::Backfilled(i) => i as u64, + | Self::Normal(i) => i, + | Self::Backfilled(i) => i as u64, } } @@ -41,8 +41,8 @@ impl Count { pub fn into_signed(self) -> i64 { self.debug_assert_valid(); match self { - Self::Normal(i) => i as i64, - Self::Backfilled(i) => i, + | Self::Normal(i) => i as i64, + | Self::Backfilled(i) => i, } } @@ -51,27 +51,27 @@ impl Count { pub fn into_normal(self) -> Self { self.debug_assert_valid(); match self { - Self::Normal(i) => Self::Normal(i), - Self::Backfilled(_) => Self::Normal(0), + | Self::Normal(i) => Self::Normal(i), + | Self::Backfilled(_) => Self::Normal(0), } } #[inline] pub fn checked_inc(self, dir: Direction) -> Result { match dir { - Direction::Forward => self.checked_add(1), - Direction::Backward => self.checked_sub(1), + | Direction::Forward => self.checked_add(1), + | Direction::Backward => self.checked_sub(1), } } #[inline] pub fn checked_add(self, add: u64) -> Result { Ok(match self { - Self::Normal(i) => Self::Normal( + | Self::Normal(i) => Self::Normal( i.checked_add(add) .ok_or_else(|| err!(Arithmetic("Count::Normal overflow")))?, ), - Self::Backfilled(i) => Self::Backfilled( + | Self::Backfilled(i) => Self::Backfilled( i.checked_add(add as i64) .ok_or_else(|| err!(Arithmetic("Count::Backfilled overflow")))?, ), @@ -81,11 +81,11 @@ impl Count { #[inline] pub fn checked_sub(self, sub: u64) -> Result { Ok(match self { - Self::Normal(i) => Self::Normal( + | Self::Normal(i) => Self::Normal( i.checked_sub(sub) .ok_or_else(|| err!(Arithmetic("Count::Normal underflow")))?, ), - Self::Backfilled(i) => Self::Backfilled( + | Self::Backfilled(i) => Self::Backfilled( i.checked_sub(sub as i64) .ok_or_else(|| err!(Arithmetic("Count::Backfilled underflow")))?, ), @@ -96,8 +96,8 @@ impl Count { #[must_use] pub fn saturating_inc(self, dir: Direction) -> Self { match dir { - Direction::Forward => self.saturating_add(1), - Direction::Backward => self.saturating_sub(1), + | Direction::Forward => self.saturating_add(1), + | Direction::Backward => self.saturating_sub(1), } } @@ -105,8 +105,8 @@ impl Count { #[must_use] pub fn saturating_add(self, add: u64) -> Self { match self { - Self::Normal(i) => Self::Normal(i.saturating_add(add)), - Self::Backfilled(i) => Self::Backfilled(i.saturating_add(add as i64)), + | Self::Normal(i) => Self::Normal(i.saturating_add(add)), + | Self::Backfilled(i) => Self::Backfilled(i.saturating_add(add as i64)), } } @@ -114,8 +114,8 @@ impl Count { #[must_use] pub fn saturating_sub(self, sub: u64) -> Self { match self { - Self::Normal(i) => Self::Normal(i.saturating_sub(sub)), - Self::Backfilled(i) => Self::Backfilled(i.saturating_sub(sub as i64)), + | Self::Normal(i) => Self::Normal(i.saturating_sub(sub)), + | Self::Backfilled(i) => Self::Backfilled(i.saturating_sub(sub as i64)), } } @@ -139,8 +139,8 @@ impl Display for Count { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { self.debug_assert_valid(); match self { - Self::Normal(i) => write!(f, "{i}"), - Self::Backfilled(i) => write!(f, "{i}"), + | Self::Normal(i) => write!(f, "{i}"), + | Self::Backfilled(i) => write!(f, "{i}"), } } } diff --git a/src/core/pdu/event.rs b/src/core/pdu/event.rs index 96a1e4ba..fb98d686 100644 --- a/src/core/pdu/event.rs +++ b/src/core/pdu/event.rs @@ -19,13 +19,19 @@ impl Event for Pdu { fn content(&self) -> &RawJsonValue { &self.content } - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { MilliSecondsSinceUnixEpoch(self.origin_server_ts) } + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { + MilliSecondsSinceUnixEpoch(self.origin_server_ts) + } fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { self.prev_events.iter() } + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + self.prev_events.iter() + } - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { self.auth_events.iter() } + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { + self.auth_events.iter() + } fn redacts(&self) -> Option<&Self::Id> { self.redacts.as_ref() } } diff --git a/src/core/pdu/event_id.rs b/src/core/pdu/event_id.rs index ae5b85f9..09b33edc 100644 --- a/src/core/pdu/event_id.rs +++ b/src/core/pdu/event_id.rs @@ -8,7 +8,8 @@ use crate::{err, Result}; /// Returns a tuple of the new `EventId` and the PDU as a `BTreeMap`. pub fn gen_event_id_canonical_json( - pdu: &RawJsonValue, room_version_id: &RoomVersionId, + pdu: &RawJsonValue, + room_version_id: &RoomVersionId, ) -> Result<(OwnedEventId, CanonicalJsonObject)> { let value: CanonicalJsonObject = serde_json::from_str(pdu.get()) .map_err(|e| err!(BadServerResponse(warn!("Error parsing incoming event: {e:?}"))))?; @@ -19,7 +20,10 @@ pub fn gen_event_id_canonical_json( } /// Generates a correct eventId for the incoming pdu. -pub fn gen_event_id(value: &CanonicalJsonObject, room_version_id: &RoomVersionId) -> Result { +pub fn gen_event_id( + value: &CanonicalJsonObject, + room_version_id: &RoomVersionId, +) -> Result { let reference_hash = ruma::signatures::reference_hash(value, room_version_id)?; let event_id: OwnedEventId = format!("${reference_hash}").try_into()?; diff --git a/src/core/pdu/filter.rs b/src/core/pdu/filter.rs index c7c7316d..aabf13db 100644 --- a/src/core/pdu/filter.rs +++ b/src/core/pdu/filter.rs @@ -84,7 +84,7 @@ fn matches_url(&self, filter: &RoomEventFilter) -> bool { .is_some_and(Value::is_string); match url_filter { - UrlFilter::EventsWithUrl => url, - UrlFilter::EventsWithoutUrl => !url, + | UrlFilter::EventsWithUrl => url, + | UrlFilter::EventsWithoutUrl => !url, } } diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 2aa60ed1..c90e174e 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -15,7 +15,8 @@ mod unsigned; use std::{cmp::Ordering, sync::Arc}; use ruma::{ - events::TimelineEventType, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedRoomId, OwnedUserId, UInt, + events::TimelineEventType, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedRoomId, + OwnedUserId, UInt, }; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue as RawJsonValue; diff --git a/src/core/pdu/raw_id.rs b/src/core/pdu/raw_id.rs index ef8502f6..e1fd2381 100644 --- a/src/core/pdu/raw_id.rs +++ b/src/core/pdu/raw_id.rs @@ -29,10 +29,10 @@ impl RawId { #[must_use] pub fn shortroomid(self) -> [u8; INT_LEN] { match self { - Self::Normal(raw) => raw[0..INT_LEN] + | Self::Normal(raw) => raw[0..INT_LEN] .try_into() .expect("normal raw shortroomid array from slice"), - Self::Backfilled(raw) => raw[0..INT_LEN] + | Self::Backfilled(raw) => raw[0..INT_LEN] .try_into() .expect("backfilled raw shortroomid array from slice"), } @@ -42,10 +42,10 @@ impl RawId { #[must_use] pub fn shorteventid(self) -> [u8; INT_LEN] { match self { - Self::Normal(raw) => raw[INT_LEN..INT_LEN * 2] + | Self::Normal(raw) => raw[INT_LEN..INT_LEN * 2] .try_into() .expect("normal raw shorteventid array from slice"), - Self::Backfilled(raw) => raw[INT_LEN * 2..INT_LEN * 3] + | Self::Backfilled(raw) => raw[INT_LEN * 2..INT_LEN * 3] .try_into() .expect("backfilled raw shorteventid array from slice"), } @@ -55,8 +55,8 @@ impl RawId { #[must_use] pub fn as_bytes(&self) -> &[u8] { match self { - Self::Normal(ref raw) => raw, - Self::Backfilled(ref raw) => raw, + | Self::Normal(ref raw) => raw, + | Self::Backfilled(ref raw) => raw, } } } @@ -70,17 +70,17 @@ impl From<&[u8]> for RawId { #[inline] fn from(id: &[u8]) -> Self { match id.len() { - Self::NORMAL_LEN => Self::Normal( + | Self::NORMAL_LEN => Self::Normal( id[0..Self::NORMAL_LEN] .try_into() .expect("normal RawId from [u8]"), ), - Self::BACKFILLED_LEN => Self::Backfilled( + | Self::BACKFILLED_LEN => Self::Backfilled( id[0..Self::BACKFILLED_LEN] .try_into() .expect("backfilled RawId from [u8]"), ), - _ => unimplemented!("unrecognized RawId length"), + | _ => unimplemented!("unrecognized RawId length"), } } } @@ -95,11 +95,11 @@ impl From for RawId { vec.extend(id.shortroomid.to_be_bytes()); id.shorteventid.debug_assert_valid(); match id.shorteventid { - Count::Normal(shorteventid) => { + | Count::Normal(shorteventid) => { vec.extend(shorteventid.to_be_bytes()); Self::Normal(vec.as_ref().try_into().expect("RawVec into RawId::Normal")) }, - Count::Backfilled(shorteventid) => { + | Count::Backfilled(shorteventid) => { vec.extend(0_u64.to_be_bytes()); vec.extend(shorteventid.to_be_bytes()); Self::Backfilled( diff --git a/src/core/pdu/redact.rs b/src/core/pdu/redact.rs index 01d9147c..dc97bfa8 100644 --- a/src/core/pdu/redact.rs +++ b/src/core/pdu/redact.rs @@ -22,8 +22,8 @@ struct ExtractRedactedBecause { pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: &Self) -> Result { self.unsigned = None; - let mut content = - serde_json::from_str(self.content.get()).map_err(|_| Error::bad_database("PDU in db has invalid content."))?; + let mut content = serde_json::from_str(self.content.get()) + .map_err(|_| Error::bad_database("PDU in db has invalid content."))?; redact_content_in_place(&mut content, room_version_id, self.kind.to_string()) .map_err(|e| Error::Redaction(self.sender.server_name().to_owned(), e))?; @@ -75,7 +75,9 @@ pub fn is_redacted(&self) -> bool { #[must_use] pub fn copy_redacts(&self) -> (Option>, Box) { if self.kind == TimelineEventType::RoomRedaction { - if let Ok(mut content) = serde_json::from_str::(self.content.get()) { + if let Ok(mut content) = + serde_json::from_str::(self.content.get()) + { if let Some(redacts) = content.redacts { return (Some(redacts.into()), self.content.clone()); } else if let Some(redacts) = self.redacts.clone() { diff --git a/src/core/pdu/strip.rs b/src/core/pdu/strip.rs index 59457749..8e1045db 100644 --- a/src/core/pdu/strip.rs +++ b/src/core/pdu/strip.rs @@ -1,8 +1,8 @@ use ruma::{ events::{ - room::member::RoomMemberEventContent, space::child::HierarchySpaceChildEvent, AnyEphemeralRoomEvent, - AnyMessageLikeEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncStateEvent, AnySyncTimelineEvent, - AnyTimelineEvent, StateEvent, + room::member::RoomMemberEventContent, space::child::HierarchySpaceChildEvent, + AnyEphemeralRoomEvent, AnyMessageLikeEvent, AnyStateEvent, AnyStrippedStateEvent, + AnySyncStateEvent, AnySyncTimelineEvent, AnyTimelineEvent, StateEvent, }, serde::Raw, }; diff --git a/src/core/pdu/unsigned.rs b/src/core/pdu/unsigned.rs index 6f3e4401..fa305d71 100644 --- a/src/core/pdu/unsigned.rs +++ b/src/core/pdu/unsigned.rs @@ -13,8 +13,8 @@ pub fn remove_transaction_id(&mut self) -> Result { return Ok(()); }; - let mut unsigned: BTreeMap> = - serde_json::from_str(unsigned.get()).map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; + let mut unsigned: BTreeMap> = serde_json::from_str(unsigned.get()) + .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; unsigned.remove("transaction_id"); self.unsigned = to_raw_value(&unsigned) @@ -97,7 +97,9 @@ where #[implement(Pdu)] #[must_use] -pub fn get_unsigned_as_value(&self) -> JsonValue { self.get_unsigned::().unwrap_or_default() } +pub fn get_unsigned_as_value(&self) -> JsonValue { + self.get_unsigned::().unwrap_or_default() +} #[implement(Pdu)] pub fn get_unsigned(&self) -> Result { diff --git a/src/core/server.rs b/src/core/server.rs index 1e721517..2a70ae4b 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -79,8 +79,9 @@ impl Server { return Err!("Restart already in progress"); } - self.shutdown() - .inspect_err(|_| self.restarting.store(false, Ordering::Release)) + self.shutdown().inspect_err(|_| { + self.restarting.store(false, Ordering::Release); + }) } pub fn shutdown(&self) -> Result<()> { @@ -88,8 +89,9 @@ impl Server { return Err!("Shutdown already in progress"); } - self.signal("SIGTERM") - .inspect_err(|_| self.stopping.store(false, Ordering::Release)) + self.signal("SIGTERM").inspect_err(|_| { + self.stopping.store(false, Ordering::Release); + }) } pub fn signal(&self, sig: &'static str) -> Result<()> { diff --git a/src/core/utils/bool.rs b/src/core/utils/bool.rs index e9f399d4..b4192daf 100644 --- a/src/core/utils/bool.rs +++ b/src/core/utils/bool.rs @@ -66,19 +66,25 @@ impl BoolExt for bool { } #[inline] - fn map_ok_or T>(self, err: E, f: F) -> Result { self.ok_or(err).map(|()| f()) } + fn map_ok_or T>(self, err: E, f: F) -> Result { + self.ok_or(err).map(|()| f()) + } #[inline] fn map_or T>(self, err: T, f: F) -> T { self.then(f).unwrap_or(err) } #[inline] - fn map_or_else T>(self, err: F, f: F) -> T { self.then(f).unwrap_or_else(err) } + fn map_or_else T>(self, err: F, f: F) -> T { + self.then(f).unwrap_or_else(err) + } #[inline] fn ok_or(self, err: E) -> Result<(), E> { self.into_option().ok_or(err) } #[inline] - fn ok_or_else E>(self, err: F) -> Result<(), E> { self.into_option().ok_or_else(err) } + fn ok_or_else E>(self, err: F) -> Result<(), E> { + self.into_option().ok_or_else(err) + } #[inline] fn or T>(self, f: F) -> Option { (!self).then(f) } diff --git a/src/core/utils/bytes.rs b/src/core/utils/bytes.rs index 441ba422..40316440 100644 --- a/src/core/utils/bytes.rs +++ b/src/core/utils/bytes.rs @@ -39,7 +39,9 @@ pub fn increment(old: Option<&[u8]>) -> [u8; 8] { /// Parses 8 big-endian bytes into an u64; panic on invalid argument #[inline] #[must_use] -pub fn u64_from_u8(bytes: &[u8]) -> u64 { u64_from_bytes(bytes).expect("must slice at least 8 bytes") } +pub fn u64_from_u8(bytes: &[u8]) -> u64 { + u64_from_bytes(bytes).expect("must slice at least 8 bytes") +} /// Parses the big-endian bytes into an u64. #[inline] diff --git a/src/core/utils/content_disposition.rs b/src/core/utils/content_disposition.rs index 3a264a74..82f11732 100644 --- a/src/core/utils/content_disposition.rs +++ b/src/core/utils/content_disposition.rs @@ -71,13 +71,10 @@ pub fn content_disposition_type(content_type: Option<&str>) -> ContentDispositio /// `sanitize_filename` crate #[tracing::instrument(level = "debug")] pub fn sanitise_filename(filename: &str) -> String { - sanitize_filename::sanitize_with_options( - filename, - sanitize_filename::Options { - truncate: false, - ..Default::default() - }, - ) + sanitize_filename::sanitize_with_options(filename, sanitize_filename::Options { + truncate: false, + ..Default::default() + }) } /// creates the final Content-Disposition based on whether the filename exists @@ -89,11 +86,16 @@ pub fn sanitise_filename(filename: &str) -> String { /// /// else: `Content-Disposition: attachment/inline` pub fn make_content_disposition( - content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, filename: Option<&str>, + content_disposition: Option<&ContentDisposition>, + content_type: Option<&str>, + filename: Option<&str>, ) -> ContentDisposition { ContentDisposition::new(content_disposition_type(content_type)).with_filename( filename - .or_else(|| content_disposition.and_then(|content_disposition| content_disposition.filename.as_deref())) + .or_else(|| { + content_disposition + .and_then(|content_disposition| content_disposition.filename.as_deref()) + }) .map(sanitise_filename), ) } @@ -102,8 +104,8 @@ pub fn make_content_disposition( mod tests { #[test] fn string_sanitisation() { - const SAMPLE: &str = - "🏳️‍⚧️this\\r\\n įs \r\\n ä \\r\nstrïng 🥴that\n\r ../../../../../../../may be\r\n malicious🏳️‍⚧️"; + const SAMPLE: &str = "🏳️‍⚧️this\\r\\n įs \r\\n ä \\r\nstrïng 🥴that\n\r \ + ../../../../../../../may be\r\n malicious🏳️‍⚧️"; const SANITISED: &str = "🏳️‍⚧️thisrn įs n ä rstrïng 🥴that ..............may be malicious🏳️‍⚧️"; let options = sanitize_filename::Options { @@ -125,14 +127,12 @@ mod tests { fn empty_sanitisation() { use crate::utils::string::EMPTY; - let result = sanitize_filename::sanitize_with_options( - EMPTY, - sanitize_filename::Options { + let result = + sanitize_filename::sanitize_with_options(EMPTY, sanitize_filename::Options { windows: true, truncate: true, replacement: "", - }, - ); + }); assert_eq!(EMPTY, result); } diff --git a/src/core/utils/debug.rs b/src/core/utils/debug.rs index cb9d7bab..b16ae754 100644 --- a/src/core/utils/debug.rs +++ b/src/core/utils/debug.rs @@ -31,10 +31,8 @@ impl fmt::Debug for TruncatedSlice<'_, T> { /// fn bar(foos: &[&str]); /// ``` pub fn slice_truncated( - slice: &[T], max_len: usize, + slice: &[T], + max_len: usize, ) -> tracing::field::DebugValue> { - tracing::field::debug(TruncatedSlice { - inner: slice, - max_len, - }) + tracing::field::debug(TruncatedSlice { inner: slice, max_len }) } diff --git a/src/core/utils/defer.rs b/src/core/utils/defer.rs index 29199700..60243e97 100644 --- a/src/core/utils/defer.rs +++ b/src/core/utils/defer.rs @@ -9,9 +9,7 @@ macro_rules! defer { fn drop(&mut self) { (self.closure)(); } } - let _defer_ = _Defer_ { - closure: || $body, - }; + let _defer_ = _Defer_ { closure: || $body }; }; ($body:expr) => { diff --git a/src/core/utils/future/try_ext_ext.rs b/src/core/utils/future/try_ext_ext.rs index 977f74d2..19761309 100644 --- a/src/core/utils/future/try_ext_ext.rs +++ b/src/core/utils/future/try_ext_ext.rs @@ -14,17 +14,23 @@ pub trait TryExtExt where Self: TryFuture + Send, { - fn is_err(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> + fn is_err( + self, + ) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> where Self: Sized; #[allow(clippy::wrong_self_convention)] - fn is_ok(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> + fn is_ok( + self, + ) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> where Self: Sized; fn map_ok_or( - self, default: U, f: F, + self, + default: U, + f: F, ) -> MapOkOrElse U, impl FnOnce(Self::Error) -> U> where F: FnOnce(Self::Ok) -> U, @@ -32,11 +38,18 @@ where fn ok( self, - ) -> MapOkOrElse Option, impl FnOnce(Self::Error) -> Option> + ) -> MapOkOrElse< + Self, + impl FnOnce(Self::Ok) -> Option, + impl FnOnce(Self::Error) -> Option, + > where Self: Sized; - fn unwrap_or(self, default: Self::Ok) -> UnwrapOrElse Self::Ok> + fn unwrap_or( + self, + default: Self::Ok, + ) -> UnwrapOrElse Self::Ok> where Self: Sized; @@ -51,7 +64,9 @@ where Fut: TryFuture + Send, { #[inline] - fn is_err(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> + fn is_err( + self, + ) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> where Self: Sized, { @@ -59,7 +74,9 @@ where } #[inline] - fn is_ok(self) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> + fn is_ok( + self, + ) -> MapOkOrElse bool, impl FnOnce(Self::Error) -> bool> where Self: Sized, { @@ -68,7 +85,9 @@ where #[inline] fn map_ok_or( - self, default: U, f: F, + self, + default: U, + f: F, ) -> MapOkOrElse U, impl FnOnce(Self::Error) -> U> where F: FnOnce(Self::Ok) -> U, @@ -80,7 +99,11 @@ where #[inline] fn ok( self, - ) -> MapOkOrElse Option, impl FnOnce(Self::Error) -> Option> + ) -> MapOkOrElse< + Self, + impl FnOnce(Self::Ok) -> Option, + impl FnOnce(Self::Error) -> Option, + > where Self: Sized, { @@ -88,7 +111,10 @@ where } #[inline] - fn unwrap_or(self, default: Self::Ok) -> UnwrapOrElse Self::Ok> + fn unwrap_or( + self, + default: Self::Ok, + ) -> UnwrapOrElse Self::Ok> where Self: Sized, { diff --git a/src/core/utils/hash/argon.rs b/src/core/utils/hash/argon.rs index 0a1e1e14..18146b47 100644 --- a/src/core/utils/hash/argon.rs +++ b/src/core/utils/hash/argon.rs @@ -1,8 +1,8 @@ use std::sync::OnceLock; use argon2::{ - password_hash, password_hash::SaltString, Algorithm, Argon2, Params, PasswordHash, PasswordHasher, - PasswordVerifier, Version, + password_hash, password_hash::SaltString, Algorithm, Argon2, Params, PasswordHash, + PasswordHasher, PasswordVerifier, Version, }; use crate::{err, Error, Result}; diff --git a/src/core/utils/html.rs b/src/core/utils/html.rs index fe07b2dd..f2b6d861 100644 --- a/src/core/utils/html.rs +++ b/src/core/utils/html.rs @@ -16,12 +16,12 @@ impl fmt::Display for Escape<'_> { let mut last = 0; for (i, ch) in s.char_indices() { let s = match ch { - '>' => ">", - '<' => "<", - '&' => "&", - '\'' => "'", - '"' => """, - _ => continue, + | '>' => ">", + | '<' => "<", + | '&' => "&", + | '\'' => "'", + | '"' => """, + | _ => continue, }; fmt.write_str(&pile_o_bits[last..i])?; fmt.write_str(s)?; diff --git a/src/core/utils/json.rs b/src/core/utils/json.rs index a9adad54..4a3fec8f 100644 --- a/src/core/utils/json.rs +++ b/src/core/utils/json.rs @@ -8,16 +8,24 @@ use crate::Result; /// `CanonicalJsonObject`. /// /// `value` must serialize to an `serde_json::Value::Object`. -pub fn to_canonical_object(value: T) -> Result { +pub fn to_canonical_object( + value: T, +) -> Result { use serde::ser::Error; match serde_json::to_value(value).map_err(CanonicalJsonError::SerDe)? { - serde_json::Value::Object(map) => try_from_json_map(map), - _ => Err(CanonicalJsonError::SerDe(serde_json::Error::custom("Value must be an object"))), + | serde_json::Value::Object(map) => try_from_json_map(map), + | _ => + Err(CanonicalJsonError::SerDe(serde_json::Error::custom("Value must be an object"))), } } -pub fn deserialize_from_str<'de, D: serde::de::Deserializer<'de>, T: FromStr, E: fmt::Display>( +pub fn deserialize_from_str< + 'de, + D: serde::de::Deserializer<'de>, + T: FromStr, + E: fmt::Display, +>( deserializer: D, ) -> Result { struct Visitor, E>(std::marker::PhantomData); diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index ccff6400..da2357d5 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -56,7 +56,12 @@ macro_rules! validated { /// Returns false if the exponential backoff has expired based on the inputs #[inline] #[must_use] -pub fn continue_exponential_backoff_secs(min: u64, max: u64, elapsed: Duration, tries: u32) -> bool { +pub fn continue_exponential_backoff_secs( + min: u64, + max: u64, + elapsed: Duration, + tries: u32, +) -> bool { let min = Duration::from_secs(min); let max = Duration::from_secs(max); continue_exponential_backoff(min, max, elapsed, tries) @@ -65,7 +70,12 @@ pub fn continue_exponential_backoff_secs(min: u64, max: u64, elapsed: Duration, /// Returns false if the exponential backoff has expired based on the inputs #[inline] #[must_use] -pub fn continue_exponential_backoff(min: Duration, max: Duration, elapsed: Duration, tries: u32) -> bool { +pub fn continue_exponential_backoff( + min: Duration, + max: Duration, + elapsed: Duration, + tries: u32, +) -> bool { let min = min.saturating_mul(tries).saturating_mul(tries); let min = cmp::min(min, max); elapsed < min diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index ad53bc42..a9b73fb6 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -47,8 +47,8 @@ pub fn exchange(state: &mut T, source: T) -> T { std::mem::replace(state, sou macro_rules! extract_variant { ($e:expr, $variant:path) => { match $e { - $variant(value) => Some(value), - _ => None, + | $variant(value) => Some(value), + | _ => None, } }; } diff --git a/src/core/utils/mutex_map.rs b/src/core/utils/mutex_map.rs index c3c51798..152a75d1 100644 --- a/src/core/utils/mutex_map.rs +++ b/src/core/utils/mutex_map.rs @@ -70,10 +70,9 @@ where impl Drop for Guard { fn drop(&mut self) { if Arc::strong_count(Omg::mutex(&self.val)) <= 2 { - self.map - .lock() - .expect("locked") - .retain(|_, val| !Arc::ptr_eq(val, Omg::mutex(&self.val)) || Arc::strong_count(val) > 2); + self.map.lock().expect("locked").retain(|_, val| { + !Arc::ptr_eq(val, Omg::mutex(&self.val)) || Arc::strong_count(val) > 2 + }); } } } diff --git a/src/core/utils/result.rs b/src/core/utils/result.rs index 6b11ea66..1ad5371b 100644 --- a/src/core/utils/result.rs +++ b/src/core/utils/result.rs @@ -10,9 +10,9 @@ mod unwrap_infallible; mod unwrap_or_err; pub use self::{ - debug_inspect::DebugInspect, filter::Filter, flat_ok::FlatOk, into_is_ok::IntoIsOk, log_debug_err::LogDebugErr, - log_err::LogErr, map_expect::MapExpect, not_found::NotFound, unwrap_infallible::UnwrapInfallible, - unwrap_or_err::UnwrapOrErr, + debug_inspect::DebugInspect, filter::Filter, flat_ok::FlatOk, into_is_ok::IntoIsOk, + log_debug_err::LogDebugErr, log_err::LogErr, map_expect::MapExpect, not_found::NotFound, + unwrap_infallible::UnwrapInfallible, unwrap_or_err::UnwrapOrErr, }; pub type Result = std::result::Result; diff --git a/src/core/utils/result/flat_ok.rs b/src/core/utils/result/flat_ok.rs index e378e5d0..8e7bb968 100644 --- a/src/core/utils/result/flat_ok.rs +++ b/src/core/utils/result/flat_ok.rs @@ -19,7 +19,9 @@ impl FlatOk for Option> { fn flat_ok_or(self, err: Ep) -> Result { self.flat_ok().ok_or(err) } #[inline] - fn flat_ok_or_else Ep>(self, err: F) -> Result { self.flat_ok().ok_or_else(err) } + fn flat_ok_or_else Ep>(self, err: F) -> Result { + self.flat_ok().ok_or_else(err) + } } impl FlatOk for Result, E> { @@ -30,5 +32,7 @@ impl FlatOk for Result, E> { fn flat_ok_or(self, err: Ep) -> Result { self.flat_ok().ok_or(err) } #[inline] - fn flat_ok_or_else Ep>(self, err: F) -> Result { self.flat_ok().ok_or_else(err) } + fn flat_ok_or_else Ep>(self, err: F) -> Result { + self.flat_ok().ok_or_else(err) + } } diff --git a/src/core/utils/result/log_err.rs b/src/core/utils/result/log_err.rs index 374a5e59..a1ce891f 100644 --- a/src/core/utils/result/log_err.rs +++ b/src/core/utils/result/log_err.rs @@ -20,5 +20,7 @@ pub trait LogErr { impl LogErr for Result { #[inline] - fn err_log(self, level: Level) -> Self { self.inspect_err(|error| error::inspect_log_level(&error, level)) } + fn err_log(self, level: Level) -> Self { + self.inspect_err(|error| error::inspect_log_level(&error, level)) + } } diff --git a/src/core/utils/set.rs b/src/core/utils/set.rs index ddcf05ff..8eac7157 100644 --- a/src/core/utils/set.rs +++ b/src/core/utils/set.rs @@ -32,7 +32,9 @@ where /// Intersection of sets /// /// Outputs the set of elements common to all input sets. Inputs must be sorted. -pub fn intersection_sorted(mut input: Iters) -> impl Iterator + Send +pub fn intersection_sorted( + mut input: Iters, +) -> impl Iterator + Send where Iters: Iterator + Clone + Send, Iter: Iterator + Send, diff --git a/src/core/utils/stream/expect.rs b/src/core/utils/stream/expect.rs index 68ac24ce..3509bb83 100644 --- a/src/core/utils/stream/expect.rs +++ b/src/core/utils/stream/expect.rs @@ -14,9 +14,13 @@ where Item: 'a, { #[inline] - fn expect_ok(self: T) -> impl Stream + Send + 'a { self.map_expect("stream expectation failure") } + fn expect_ok(self: T) -> impl Stream + Send + 'a { + self.map_expect("stream expectation failure") + } //TODO: move to impl MapExpect #[inline] - fn map_expect(self, msg: &'a str) -> impl Stream + Send + 'a { self.map(|res| res.expect(msg)) } + fn map_expect(self, msg: &'a str) -> impl Stream + Send + 'a { + self.map(|res| res.expect(msg)) + } } diff --git a/src/core/utils/stream/ignore.rs b/src/core/utils/stream/ignore.rs index 4e426557..9baa00f3 100644 --- a/src/core/utils/stream/ignore.rs +++ b/src/core/utils/stream/ignore.rs @@ -23,8 +23,12 @@ where #[cfg(not(debug_assertions))] #[inline] - fn ignore_err(self: T) -> impl Stream + Send + 'a { self.filter_map(|res| ready(res.ok())) } + fn ignore_err(self: T) -> impl Stream + Send + 'a { + self.filter_map(|res| ready(res.ok())) + } #[inline] - fn ignore_ok(self: T) -> impl Stream + Send + 'a { self.filter_map(|res| ready(res.err())) } + fn ignore_ok(self: T) -> impl Stream + Send + 'a { + self.filter_map(|res| ready(res.err())) + } } diff --git a/src/core/utils/stream/iter_stream.rs b/src/core/utils/stream/iter_stream.rs index 2face4b0..9077deac 100644 --- a/src/core/utils/stream/iter_stream.rs +++ b/src/core/utils/stream/iter_stream.rs @@ -13,7 +13,11 @@ pub trait IterStream { /// Convert an Iterator into a TryStream fn try_stream( self, - ) -> impl TryStream::Item, Error = Error, Item = Result<::Item, Error>> + Send; + ) -> impl TryStream< + Ok = ::Item, + Error = Error, + Item = Result<::Item, Error>, + > + Send; } impl IterStream for I @@ -27,8 +31,11 @@ where #[inline] fn try_stream( self, - ) -> impl TryStream::Item, Error = Error, Item = Result<::Item, Error>> + Send - { + ) -> impl TryStream< + Ok = ::Item, + Error = Error, + Item = Result<::Item, Error>, + > + Send { self.stream().map(Ok) } } diff --git a/src/core/utils/stream/ready.rs b/src/core/utils/stream/ready.rs index 9bba589e..d93187e9 100644 --- a/src/core/utils/stream/ready.rs +++ b/src/core/utils/stream/ready.rs @@ -3,7 +3,9 @@ use futures::{ future::{ready, Ready}, - stream::{All, Any, Filter, FilterMap, Fold, ForEach, Scan, SkipWhile, Stream, StreamExt, TakeWhile}, + stream::{ + All, Any, Filter, FilterMap, Fold, ForEach, Scan, SkipWhile, Stream, StreamExt, TakeWhile, + }, }; /// Synchronous combinators to augment futures::StreamExt. Most Stream @@ -24,19 +26,32 @@ where where F: Fn(Item) -> bool; - fn ready_filter<'a, F>(self, f: F) -> Filter, impl FnMut(&Item) -> Ready + 'a> + fn ready_filter<'a, F>( + self, + f: F, + ) -> Filter, impl FnMut(&Item) -> Ready + 'a> where F: Fn(&Item) -> bool + 'a; - fn ready_filter_map(self, f: F) -> FilterMap>, impl FnMut(Item) -> Ready>> + fn ready_filter_map( + self, + f: F, + ) -> FilterMap>, impl FnMut(Item) -> Ready>> where F: Fn(Item) -> Option; - fn ready_fold(self, init: T, f: F) -> Fold, T, impl FnMut(T, Item) -> Ready> + fn ready_fold( + self, + init: T, + f: F, + ) -> Fold, T, impl FnMut(T, Item) -> Ready> where F: Fn(T, Item) -> T; - fn ready_fold_default(self, f: F) -> Fold, T, impl FnMut(T, Item) -> Ready> + fn ready_fold_default( + self, + f: F, + ) -> Fold, T, impl FnMut(T, Item) -> Ready> where F: Fn(T, Item) -> T, T: Default; @@ -45,23 +60,33 @@ where where F: FnMut(Item); - fn ready_take_while<'a, F>(self, f: F) -> TakeWhile, impl FnMut(&Item) -> Ready + 'a> + fn ready_take_while<'a, F>( + self, + f: F, + ) -> TakeWhile, impl FnMut(&Item) -> Ready + 'a> where F: Fn(&Item) -> bool + 'a; fn ready_scan( - self, init: T, f: F, + self, + init: T, + f: F, ) -> Scan>, impl FnMut(&mut T, Item) -> Ready>> where F: Fn(&mut T, Item) -> Option; fn ready_scan_each( - self, init: T, f: F, + self, + init: T, + f: F, ) -> Scan>, impl FnMut(&mut T, Item) -> Ready>> where F: Fn(&mut T, &Item); - fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile, impl FnMut(&Item) -> Ready + 'a> + fn ready_skip_while<'a, F>( + self, + f: F, + ) -> SkipWhile, impl FnMut(&Item) -> Ready + 'a> where F: Fn(&Item) -> bool + 'a; } @@ -87,7 +112,10 @@ where } #[inline] - fn ready_filter<'a, F>(self, f: F) -> Filter, impl FnMut(&Item) -> Ready + 'a> + fn ready_filter<'a, F>( + self, + f: F, + ) -> Filter, impl FnMut(&Item) -> Ready + 'a> where F: Fn(&Item) -> bool + 'a, { @@ -95,7 +123,10 @@ where } #[inline] - fn ready_filter_map(self, f: F) -> FilterMap>, impl FnMut(Item) -> Ready>> + fn ready_filter_map( + self, + f: F, + ) -> FilterMap>, impl FnMut(Item) -> Ready>> where F: Fn(Item) -> Option, { @@ -103,7 +134,11 @@ where } #[inline] - fn ready_fold(self, init: T, f: F) -> Fold, T, impl FnMut(T, Item) -> Ready> + fn ready_fold( + self, + init: T, + f: F, + ) -> Fold, T, impl FnMut(T, Item) -> Ready> where F: Fn(T, Item) -> T, { @@ -111,7 +146,10 @@ where } #[inline] - fn ready_fold_default(self, f: F) -> Fold, T, impl FnMut(T, Item) -> Ready> + fn ready_fold_default( + self, + f: F, + ) -> Fold, T, impl FnMut(T, Item) -> Ready> where F: Fn(T, Item) -> T, T: Default, @@ -121,7 +159,10 @@ where #[inline] #[allow(clippy::unit_arg)] - fn ready_for_each(self, mut f: F) -> ForEach, impl FnMut(Item) -> Ready<()>> + fn ready_for_each( + self, + mut f: F, + ) -> ForEach, impl FnMut(Item) -> Ready<()>> where F: FnMut(Item), { @@ -129,7 +170,10 @@ where } #[inline] - fn ready_take_while<'a, F>(self, f: F) -> TakeWhile, impl FnMut(&Item) -> Ready + 'a> + fn ready_take_while<'a, F>( + self, + f: F, + ) -> TakeWhile, impl FnMut(&Item) -> Ready + 'a> where F: Fn(&Item) -> bool + 'a, { @@ -138,7 +182,9 @@ where #[inline] fn ready_scan( - self, init: T, f: F, + self, + init: T, + f: F, ) -> Scan>, impl FnMut(&mut T, Item) -> Ready>> where F: Fn(&mut T, Item) -> Option, @@ -148,7 +194,9 @@ where #[inline] fn ready_scan_each( - self, init: T, f: F, + self, + init: T, + f: F, ) -> Scan>, impl FnMut(&mut T, Item) -> Ready>> where F: Fn(&mut T, &Item), @@ -160,7 +208,10 @@ where } #[inline] - fn ready_skip_while<'a, F>(self, f: F) -> SkipWhile, impl FnMut(&Item) -> Ready + 'a> + fn ready_skip_while<'a, F>( + self, + f: F, + ) -> SkipWhile, impl FnMut(&Item) -> Ready + 'a> where F: Fn(&Item) -> bool + 'a, { diff --git a/src/core/utils/stream/tools.rs b/src/core/utils/stream/tools.rs index b5b036cc..7b24642e 100644 --- a/src/core/utils/stream/tools.rs +++ b/src/core/utils/stream/tools.rs @@ -24,12 +24,17 @@ where F: Fn(Item) -> K + Send, K: Eq + Hash + Send; - fn counts_by_with_cap(self, f: F) -> impl Future> + Send + fn counts_by_with_cap( + self, + f: F, + ) -> impl Future> + Send where F: Fn(Item) -> K + Send, K: Eq + Hash + Send; - fn counts_with_cap(self) -> impl Future> + Send + fn counts_with_cap( + self, + ) -> impl Future> + Send where ::Item: Eq + Hash; @@ -63,7 +68,10 @@ where } #[inline] - fn counts_by_with_cap(self, f: F) -> impl Future> + Send + fn counts_by_with_cap( + self, + f: F, + ) -> impl Future> + Send where F: Fn(Item) -> K + Send, K: Eq + Hash + Send, @@ -72,7 +80,9 @@ where } #[inline] - fn counts_with_cap(self) -> impl Future> + Send + fn counts_with_cap( + self, + ) -> impl Future> + Send where ::Item: Eq + Hash, { diff --git a/src/core/utils/stream/try_broadband.rs b/src/core/utils/stream/try_broadband.rs index 59c488e0..d1213174 100644 --- a/src/core/utils/stream/try_broadband.rs +++ b/src/core/utils/stream/try_broadband.rs @@ -12,13 +12,20 @@ pub trait TryBroadbandExt where Self: TryStream> + Send + Sized, { - fn broadn_and_then(self, n: N, f: F) -> impl TryStream> + Send + fn broadn_and_then( + self, + n: N, + f: F, + ) -> impl TryStream> + Send where N: Into>, F: Fn(Self::Ok) -> Fut + Send + Sync, Fut: TryFuture> + Send; - fn broad_and_then(self, f: F) -> impl TryStream> + Send + fn broad_and_then( + self, + f: F, + ) -> impl TryStream> + Send where F: Fn(Self::Ok) -> Fut + Send + Sync, Fut: TryFuture> + Send, @@ -31,7 +38,11 @@ impl TryBroadbandExt for S where S: TryStream> + Send + Sized, { - fn broadn_and_then(self, n: N, f: F) -> impl TryStream> + Send + fn broadn_and_then( + self, + n: N, + f: F, + ) -> impl TryStream> + Send where N: Into>, F: Fn(Self::Ok) -> Fut + Send + Sync, diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs index b0a2b6c5..d8da04ec 100644 --- a/src/core/utils/stream/try_ready.rs +++ b/src/core/utils/stream/try_ready.rs @@ -16,31 +16,43 @@ where S: TryStream> + Send + ?Sized, Self: TryStream + Send + Sized, { - fn ready_and_then(self, f: F) -> AndThen>, impl FnMut(S::Ok) -> Ready>> + fn ready_and_then( + self, + f: F, + ) -> AndThen>, impl FnMut(S::Ok) -> Ready>> where F: Fn(S::Ok) -> Result; fn ready_try_filter_map( - self, f: F, - ) -> TryFilterMap, E>>, impl FnMut(S::Ok) -> Ready, E>>> + self, + f: F, + ) -> TryFilterMap< + Self, + Ready, E>>, + impl FnMut(S::Ok) -> Ready, E>>, + > where F: Fn(S::Ok) -> Result, E>; fn ready_try_fold( - self, init: U, f: F, + self, + init: U, + f: F, ) -> TryFold>, U, impl FnMut(U, S::Ok) -> Ready>> where F: Fn(U, S::Ok) -> Result; fn ready_try_fold_default( - self, f: F, + self, + f: F, ) -> TryFold>, U, impl FnMut(U, S::Ok) -> Ready>> where F: Fn(U, S::Ok) -> Result, U: Default; fn ready_try_for_each( - self, f: F, + self, + f: F, ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> where F: FnMut(S::Ok) -> Result<(), E>; @@ -52,7 +64,10 @@ where Self: TryStream + Send + Sized, { #[inline] - fn ready_and_then(self, f: F) -> AndThen>, impl FnMut(S::Ok) -> Ready>> + fn ready_and_then( + self, + f: F, + ) -> AndThen>, impl FnMut(S::Ok) -> Ready>> where F: Fn(S::Ok) -> Result, { @@ -60,8 +75,13 @@ where } fn ready_try_filter_map( - self, f: F, - ) -> TryFilterMap, E>>, impl FnMut(S::Ok) -> Ready, E>>> + self, + f: F, + ) -> TryFilterMap< + Self, + Ready, E>>, + impl FnMut(S::Ok) -> Ready, E>>, + > where F: Fn(S::Ok) -> Result, E>, { @@ -70,7 +90,9 @@ where #[inline] fn ready_try_fold( - self, init: U, f: F, + self, + init: U, + f: F, ) -> TryFold>, U, impl FnMut(U, S::Ok) -> Ready>> where F: Fn(U, S::Ok) -> Result, @@ -80,7 +102,8 @@ where #[inline] fn ready_try_fold_default( - self, f: F, + self, + f: F, ) -> TryFold>, U, impl FnMut(U, S::Ok) -> Ready>> where F: Fn(U, S::Ok) -> Result, @@ -91,7 +114,8 @@ where #[inline] fn ready_try_for_each( - self, mut f: F, + self, + mut f: F, ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> where F: FnMut(S::Ok) -> Result<(), E>, diff --git a/src/core/utils/string/between.rs b/src/core/utils/string/between.rs index 209a9dab..05c137b4 100644 --- a/src/core/utils/string/between.rs +++ b/src/core/utils/string/between.rs @@ -15,7 +15,9 @@ pub trait Between<'a> { impl<'a> Between<'a> for &'a str { #[inline] - fn between_infallible(&self, delim: Delim<'_>) -> &'a str { self.between(delim).unwrap_or(self) } + fn between_infallible(&self, delim: Delim<'_>) -> &'a str { + self.between(delim).unwrap_or(self) + } #[inline] fn between(&self, delim: Delim<'_>) -> Option<&'a str> { diff --git a/src/core/utils/string/split.rs b/src/core/utils/string/split.rs index 96de28df..e643f51b 100644 --- a/src/core/utils/string/split.rs +++ b/src/core/utils/string/split.rs @@ -15,8 +15,12 @@ pub trait SplitInfallible<'a> { impl<'a> SplitInfallible<'a> for &'a str { #[inline] - fn rsplit_once_infallible(&self, delim: &str) -> Pair<'a> { self.rsplit_once(delim).unwrap_or((self, EMPTY)) } + fn rsplit_once_infallible(&self, delim: &str) -> Pair<'a> { + self.rsplit_once(delim).unwrap_or((self, EMPTY)) + } #[inline] - fn split_once_infallible(&self, delim: &str) -> Pair<'a> { self.split_once(delim).unwrap_or((self, EMPTY)) } + fn split_once_infallible(&self, delim: &str) -> Pair<'a> { + self.split_once(delim).unwrap_or((self, EMPTY)) + } } diff --git a/src/core/utils/string/unquote.rs b/src/core/utils/string/unquote.rs index eeded610..ea7ddbf9 100644 --- a/src/core/utils/string/unquote.rs +++ b/src/core/utils/string/unquote.rs @@ -26,7 +26,9 @@ impl<'a> Unquote<'a> for &'a str { } #[inline] - fn unquote(&self) -> Option<&'a str> { self.strip_prefix(QUOTE).and_then(|s| s.strip_suffix(QUOTE)) } + fn unquote(&self) -> Option<&'a str> { + self.strip_prefix(QUOTE).and_then(|s| s.strip_suffix(QUOTE)) + } #[inline] fn is_quoted(&self) -> bool { self.starts_with(QUOTE) && self.ends_with(QUOTE) } diff --git a/src/core/utils/sys.rs b/src/core/utils/sys.rs index f3ff08b6..05ef12ca 100644 --- a/src/core/utils/sys.rs +++ b/src/core/utils/sys.rs @@ -45,8 +45,8 @@ pub unsafe fn current_exe() -> Result { let exe = std::env::current_exe()?; match exe.to_str() { - None => Ok(exe), - Some(str) => Ok(str + | None => Ok(exe), + | Some(str) => Ok(str .strip_suffix(" (deleted)") .map(PathBuf::from) .unwrap_or(exe)), @@ -58,5 +58,6 @@ pub unsafe fn current_exe() -> Result { /// accurate on all platforms; defaults to false. #[must_use] pub fn current_exe_deleted() -> bool { - std::env::current_exe().is_ok_and(|exe| exe.to_str().is_some_and(|exe| exe.ends_with(" (deleted)"))) + std::env::current_exe() + .is_ok_and(|exe| exe.to_str().is_some_and(|exe| exe.ends_with(" (deleted)"))) } diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index f4f78b02..5c870730 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -1,4 +1,5 @@ #![cfg(test)] +#![allow(clippy::disallowed_methods)] use crate::utils; diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index f96a27d0..d65eb2d4 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -13,7 +13,9 @@ pub fn now_millis() -> u64 { } #[inline] -pub fn parse_timepoint_ago(ago: &str) -> Result { timepoint_ago(parse_duration(ago)?) } +pub fn parse_timepoint_ago(ago: &str) -> Result { + timepoint_ago(parse_duration(ago)?) +} #[inline] pub fn timepoint_ago(duration: Duration) -> Result { @@ -61,13 +63,13 @@ pub fn pretty(d: Duration) -> String { let gen64 = |w, f, u| fmt(w, (f * 100.0) as u32, u); let gen128 = |w, f, u| gen64(u64::try_from(w).expect("u128 to u64"), f, u); match whole_and_frac(d) { - (Days(whole), frac) => gen64(whole, frac, "days"), - (Hours(whole), frac) => gen64(whole, frac, "hours"), - (Mins(whole), frac) => gen64(whole, frac, "minutes"), - (Secs(whole), frac) => gen64(whole, frac, "seconds"), - (Millis(whole), frac) => gen128(whole, frac, "milliseconds"), - (Micros(whole), frac) => gen128(whole, frac, "microseconds"), - (Nanos(whole), frac) => gen128(whole, frac, "nanoseconds"), + | (Days(whole), frac) => gen64(whole, frac, "days"), + | (Hours(whole), frac) => gen64(whole, frac, "hours"), + | (Mins(whole), frac) => gen64(whole, frac, "minutes"), + | (Secs(whole), frac) => gen64(whole, frac, "seconds"), + | (Millis(whole), frac) => gen128(whole, frac, "milliseconds"), + | (Micros(whole), frac) => gen128(whole, frac, "microseconds"), + | (Nanos(whole), frac) => gen128(whole, frac, "nanoseconds"), } } @@ -80,18 +82,15 @@ pub fn whole_and_frac(d: Duration) -> (Unit, f64) { use Unit::*; let whole = whole_unit(d); - ( - whole, - match whole { - Days(_) => (d.as_secs() % 86_400) as f64 / 86_400.0, - Hours(_) => (d.as_secs() % 3_600) as f64 / 3_600.0, - Mins(_) => (d.as_secs() % 60) as f64 / 60.0, - Secs(_) => f64::from(d.subsec_millis()) / 1000.0, - Millis(_) => f64::from(d.subsec_micros()) / 1000.0, - Micros(_) => f64::from(d.subsec_nanos()) / 1000.0, - Nanos(_) => 0.0, - }, - ) + (whole, match whole { + | Days(_) => (d.as_secs() % 86_400) as f64 / 86_400.0, + | Hours(_) => (d.as_secs() % 3_600) as f64 / 3_600.0, + | Mins(_) => (d.as_secs() % 60) as f64 / 60.0, + | Secs(_) => f64::from(d.subsec_millis()) / 1000.0, + | Millis(_) => f64::from(d.subsec_micros()) / 1000.0, + | Micros(_) => f64::from(d.subsec_nanos()) / 1000.0, + | Nanos(_) => 0.0, + }) } /// Return the largest Unit which represents the duration. The value is @@ -101,18 +100,18 @@ pub fn whole_unit(d: Duration) -> Unit { use Unit::*; match d.as_secs() { - 86_400.. => Days(d.as_secs() / 86_400), - 3_600..=86_399 => Hours(d.as_secs() / 3_600), - 60..=3_599 => Mins(d.as_secs() / 60), + | 86_400.. => Days(d.as_secs() / 86_400), + | 3_600..=86_399 => Hours(d.as_secs() / 3_600), + | 60..=3_599 => Mins(d.as_secs() / 60), - _ => match d.as_micros() { - 1_000_000.. => Secs(d.as_secs()), - 1_000..=999_999 => Millis(d.subsec_millis().into()), + | _ => match d.as_micros() { + | 1_000_000.. => Secs(d.as_secs()), + | 1_000..=999_999 => Millis(d.subsec_millis().into()), - _ => match d.as_nanos() { - 1_000.. => Micros(d.subsec_micros().into()), + | _ => match d.as_nanos() { + | 1_000.. => Micros(d.subsec_micros().into()), - _ => Nanos(d.subsec_nanos().into()), + | _ => Nanos(d.subsec_nanos().into()), }, }, } diff --git a/src/database/cork.rs b/src/database/cork.rs index 5fe5fd7a..11b6efd7 100644 --- a/src/database/cork.rs +++ b/src/database/cork.rs @@ -26,11 +26,7 @@ impl Cork { #[inline] pub(super) fn new(db: &Arc, flush: bool, sync: bool) -> Self { db.cork(); - Self { - db: db.clone(), - flush, - sync, - } + Self { db: db.clone(), flush, sync } } } diff --git a/src/database/database.rs b/src/database/database.rs index bd8dcb1a..83d2c201 100644 --- a/src/database/database.rs +++ b/src/database/database.rs @@ -17,10 +17,7 @@ impl Database { /// Load an existing database or create a new one. pub async fn open(server: &Arc) -> Result> { let db = Engine::open(server).await?; - Ok(Arc::new(Self { - db: db.clone(), - maps: maps::open(&db)?, - })) + Ok(Arc::new(Self { db: db.clone(), maps: maps::open(&db)? })) } #[inline] @@ -31,7 +28,9 @@ impl Database { } #[inline] - pub fn iter(&self) -> impl Iterator + Send + '_ { self.maps.iter() } + pub fn iter(&self) -> impl Iterator + Send + '_ { + self.maps.iter() + } #[inline] pub fn keys(&self) -> impl Iterator + Send + '_ { self.maps.keys() } diff --git a/src/database/de.rs b/src/database/de.rs index ac8c021f..f648b89a 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -13,11 +13,7 @@ pub(crate) fn from_slice<'a, T>(buf: &'a [u8]) -> Result where T: Deserialize<'a>, { - let mut deserializer = Deserializer { - buf, - pos: 0, - seq: false, - }; + let mut deserializer = Deserializer { buf, pos: 0, seq: false }; T::deserialize(&mut deserializer).debug_inspect(|_| { deserializer @@ -169,7 +165,12 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { visitor.visit_seq(self) } - fn deserialize_tuple_struct(self, _name: &'static str, _len: usize, visitor: V) -> Result + fn deserialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + visitor: V, + ) -> Result where V: Visitor<'de>, { @@ -186,7 +187,12 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { d.deserialize_map(visitor).map_err(Into::into) } - fn deserialize_struct(self, name: &'static str, fields: &'static [&'static str], visitor: V) -> Result + fn deserialize_struct( + self, + name: &'static str, + fields: &'static [&'static str], + visitor: V, + ) -> Result where V: Visitor<'de>, { @@ -201,9 +207,9 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { V: Visitor<'de>, { match name { - "Ignore" => self.record_ignore(), - "IgnoreAll" => self.record_ignore_all(), - _ => unhandled!("Unrecognized deserialization Directive {name:?}"), + | "Ignore" => self.record_ignore(), + | "IgnoreAll" => self.record_ignore_all(), + | _ => unhandled!("Unrecognized deserialization Directive {name:?}"), }; visitor.visit_unit() @@ -214,13 +220,16 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { V: Visitor<'de>, { match name { - "$serde_json::private::RawValue" => visitor.visit_map(self), - _ => visitor.visit_newtype_struct(self), + | "$serde_json::private::RawValue" => visitor.visit_map(self), + | _ => visitor.visit_newtype_struct(self), } } fn deserialize_enum( - self, _name: &'static str, _variants: &'static [&'static str], _visitor: V, + self, + _name: &'static str, + _variants: &'static [&'static str], + _visitor: V, ) -> Result where V: Visitor<'de>, @@ -260,7 +269,10 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } fn deserialize_u8>(self, _visitor: V) -> Result { - unhandled!("deserialize u8 not implemented; try dereferencing the Handle for [u8] access instead") + unhandled!( + "deserialize u8 not implemented; try dereferencing the Handle for [u8] access \ + instead" + ) } fn deserialize_u16>(self, _visitor: V) -> Result { @@ -338,8 +350,8 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { ); match self.record_peek_byte() { - Some(b'{') => self.deserialize_map(visitor), - _ => self.deserialize_str(visitor), + | Some(b'{') => self.deserialize_map(visitor), + | _ => self.deserialize_str(visitor), } } } diff --git a/src/database/engine.rs b/src/database/engine.rs index 72fe11e6..63a6087d 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -6,12 +6,14 @@ use std::{ sync::{atomic::AtomicU32, Arc, Mutex, RwLock}, }; -use conduwuit::{debug, error, info, utils::time::rfc2822_from_seconds, warn, Err, Result, Server}; +use conduwuit::{ + debug, error, info, utils::time::rfc2822_from_seconds, warn, Err, Result, Server, +}; use rocksdb::{ backup::{BackupEngine, BackupEngineOptions}, perf::get_memory_usage_stats, - AsColumnFamilyRef, BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, DBWithThreadMode, Env, LogLevel, - MultiThreaded, Options, + AsColumnFamilyRef, BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, + DBWithThreadMode, Env, LogLevel, MultiThreaded, Options, }; use crate::{ @@ -169,11 +171,13 @@ impl Engine { pub fn memory_usage(&self) -> Result { let mut res = String::new(); - let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&self.row_cache])).or_else(or_else)?; + let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&self.row_cache])) + .or_else(or_else)?; let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0; writeln!( res, - "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow cache: {:.2} MiB", + "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow \ + cache: {:.2} MiB", mibs(stats.mem_table_total), mibs(stats.mem_table_unflushed), mibs(stats.mem_table_readers_total), @@ -202,7 +206,8 @@ impl Engine { return Ok(()); } - let options = BackupEngineOptions::new(path.expect("valid database backup path")).map_err(map_err)?; + let options = BackupEngineOptions::new(path.expect("valid database backup path")) + .map_err(map_err)?; let mut engine = BackupEngine::open(&options, &self.env).map_err(map_err)?; if config.database_backups_to_keep > 0 { let flush = !self.is_read_only(); @@ -232,13 +237,14 @@ impl Engine { let config = &self.server.config; let path = config.database_backup_path.as_ref(); if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { - return Ok( - "Configure database_backup_path to enable backups, or the path specified is not valid".to_owned(), - ); + return Ok("Configure database_backup_path to enable backups, or the path \ + specified is not valid" + .to_owned()); } let mut res = String::new(); - let options = BackupEngineOptions::new(path.expect("valid database backup path")).or_else(or_else)?; + let options = BackupEngineOptions::new(path.expect("valid database backup path")) + .or_else(or_else)?; let engine = BackupEngine::open(&options, &self.env).or_else(or_else)?; for info in engine.get_backup_info() { writeln!( @@ -256,8 +262,8 @@ impl Engine { pub fn file_list(&self) -> Result { match self.db.live_files() { - Err(e) => Ok(String::from(e)), - Ok(files) => { + | Err(e) => Ok(String::from(e)), + | Ok(files) => { let mut res = String::new(); writeln!(res, "| lev | sst | keys | dels | size | column |")?; writeln!(res, "| ---: | :--- | ---: | ---: | ---: | :--- |")?; @@ -265,7 +271,12 @@ impl Engine { writeln!( res, "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", - file.level, file.name, file.num_entries, file.num_deletions, file.size, file.column_family_name, + file.level, + file.name, + file.num_entries, + file.num_deletions, + file.size, + file.column_family_name, )?; } @@ -277,7 +288,11 @@ impl Engine { /// Query for database property by null-terminated name which is expected to /// have a result with an integer representation. This is intended for /// low-overhead programmatic use. - pub(crate) fn property_integer(&self, cf: &impl AsColumnFamilyRef, name: &CStr) -> Result { + pub(crate) fn property_integer( + &self, + cf: &impl AsColumnFamilyRef, + name: &CStr, + ) -> Result { result(self.db.property_int_value_cf(cf, name)) .and_then(|val| val.map_or_else(|| Err!("Property {name:?} not found."), Ok)) } @@ -300,8 +315,8 @@ impl Engine { pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> { warn!("Starting database repair. This may take a long time..."); match Db::repair(db_opts, path) { - Ok(()) => info!("Database repair successful."), - Err(e) => return Err!("Repair failed: {e:?}"), + | Ok(()) => info!("Database repair successful."), + | Err(e) => return Err!("Repair failed: {e:?}"), } Ok(()) @@ -320,10 +335,10 @@ pub(crate) fn handle_log(level: LogLevel, msg: &str) { } match level { - LogLevel::Header | LogLevel::Debug => debug!("{msg}"), - LogLevel::Error | LogLevel::Fatal => error!("{msg}"), - LogLevel::Info => debug!("{msg}"), - LogLevel::Warn => warn!("{msg}"), + | LogLevel::Header | LogLevel::Debug => debug!("{msg}"), + | LogLevel::Error | LogLevel::Fatal => error!("{msg}"), + | LogLevel::Info => debug!("{msg}"), + | LogLevel::Warn => warn!("{msg}"), }; } diff --git a/src/database/handle.rs b/src/database/handle.rs index b4d34b85..43b57839 100644 --- a/src/database/handle.rs +++ b/src/database/handle.rs @@ -11,11 +11,7 @@ pub struct Handle<'a> { } impl<'a> From> for Handle<'a> { - fn from(val: DBPinnableSlice<'a>) -> Self { - Self { - val, - } - } + fn from(val: DBPinnableSlice<'a>) -> Self { Self { val } } } impl Debug for Handle<'_> { diff --git a/src/database/map.rs b/src/database/map.rs index 09721b2a..af22a43c 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -56,7 +56,10 @@ impl Map { } #[inline] - pub fn watch_prefix<'a, K>(&'a self, prefix: &K) -> Pin + Send + 'a>> + pub fn watch_prefix<'a, K>( + &'a self, + prefix: &K, + ) -> Pin + Send + 'a>> where K: AsRef<[u8]> + ?Sized + Debug, { @@ -64,7 +67,9 @@ impl Map { } #[inline] - pub fn property_integer(&self, name: &CStr) -> Result { self.db.property_integer(&self.cf(), name) } + pub fn property_integer(&self, name: &CStr) -> Result { + self.db.property_integer(&self.cf(), name) + } #[inline] pub fn property(&self, name: &str) -> Result { self.db.property(&self.cf(), name) } @@ -76,7 +81,9 @@ impl Map { } impl Debug for Map { - fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { write!(out, "Map {{name: {0}}}", self.name) } + fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(out, "Map {{name: {0}}}", self.name) + } } impl Display for Map { diff --git a/src/database/map/contains.rs b/src/database/map/contains.rs index aa3ea676..424f8970 100644 --- a/src/database/map/contains.rs +++ b/src/database/map/contains.rs @@ -29,7 +29,10 @@ where /// - harder errors will panic #[inline] #[implement(super::Map)] -pub fn acontains(self: &Arc, key: &K) -> impl Future + Send + '_ +pub fn acontains( + self: &Arc, + key: &K, +) -> impl Future + Send + '_ where K: Serialize + ?Sized + Debug, { @@ -42,7 +45,11 @@ where /// - harder errors will panic #[implement(super::Map)] #[tracing::instrument(skip(self, buf), fields(%self), level = "trace")] -pub fn bcontains(self: &Arc, key: &K, buf: &mut B) -> impl Future + Send + '_ +pub fn bcontains( + self: &Arc, + key: &K, + buf: &mut B, +) -> impl Future + Send + '_ where K: Serialize + ?Sized + Debug, B: Write + AsRef<[u8]>, diff --git a/src/database/map/count.rs b/src/database/map/count.rs index 7b632bb3..b9b34613 100644 --- a/src/database/map/count.rs +++ b/src/database/map/count.rs @@ -26,7 +26,10 @@ where /// - From is a raw #[implement(super::Map)] #[inline] -pub fn raw_count_from<'a, P>(self: &'a Arc, from: &'a P) -> impl Future + Send + 'a +pub fn raw_count_from<'a, P>( + self: &'a Arc, + from: &'a P, +) -> impl Future + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { @@ -38,7 +41,10 @@ where /// - Prefix is structured key #[implement(super::Map)] #[inline] -pub fn count_prefix<'a, P>(self: &'a Arc, prefix: &P) -> impl Future + Send + 'a +pub fn count_prefix<'a, P>( + self: &'a Arc, + prefix: &P, +) -> impl Future + Send + 'a where P: Serialize + ?Sized + Debug + 'a, { @@ -50,7 +56,10 @@ where /// - Prefix is raw #[implement(super::Map)] #[inline] -pub fn raw_count_prefix<'a, P>(self: &'a Arc, prefix: &'a P) -> impl Future + Send + 'a +pub fn raw_count_prefix<'a, P>( + self: &'a Arc, + prefix: &'a P, +) -> impl Future + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 88023b1c..94a6b727 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -31,7 +31,10 @@ where /// the query. The maximum size is supplied as const generic parameter. #[implement(super::Map)] #[inline] -pub fn aqry(self: &Arc, key: &K) -> impl Future>> + Send +pub fn aqry( + self: &Arc, + key: &K, +) -> impl Future>> + Send where K: Serialize + ?Sized + Debug, { @@ -43,7 +46,11 @@ where /// asynchronously. The key is serialized into a user-supplied Writer. #[implement(super::Map)] #[tracing::instrument(skip(self, buf), level = "trace")] -pub fn bqry(self: &Arc, key: &K, buf: &mut B) -> impl Future>> + Send +pub fn bqry( + self: &Arc, + key: &K, + buf: &mut B, +) -> impl Future>> + Send where K: Serialize + ?Sized + Debug, B: Write + AsRef<[u8]>, @@ -110,15 +117,15 @@ where match res { // cache hit; not found - Ok(None) => Err!(Request(NotFound("Not found in database"))), + | Ok(None) => Err!(Request(NotFound("Not found in database"))), // cache hit; value found - Ok(Some(res)) => Ok(Some(Handle::from(res))), + | Ok(Some(res)) => Ok(Some(Handle::from(res))), // cache miss; unknown - Err(e) if is_incomplete(&e) => Ok(None), + | Err(e) if is_incomplete(&e) => Ok(None), // some other error occurred - Err(e) => or_else(e), + | Err(e) => or_else(e), } } diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index 5f444cce..49cd5920 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -9,7 +9,8 @@ use crate::{util::map_err, Handle}; #[implement(super::Map)] #[tracing::instrument(skip(self, keys), level = "trace")] pub fn aqry_batch<'b, 'a: 'b, const MAX: usize, I, K>( - self: &'a Arc, keys: I, + self: &'a Arc, + keys: I, ) -> impl Stream>> + Send + 'a where I: Iterator + Send + 'a, @@ -22,7 +23,10 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self, keys), level = "trace")] -pub fn get_batch<'a, I, K>(self: &'a Arc, keys: I) -> impl Stream>> + Send + 'a +pub fn get_batch<'a, I, K>( + self: &'a Arc, + keys: I, +) -> impl Stream>> + Send + 'a where I: Iterator + Debug + Send + 'a, K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, @@ -34,7 +38,10 @@ where #[implement(super::Map)] #[tracing::instrument(name = "batch_blocking", level = "trace", skip_all)] -pub(crate) fn get_batch_blocking<'a, I, K>(&self, keys: I) -> impl Iterator>> + Send +pub(crate) fn get_batch_blocking<'a, I, K>( + &self, + keys: I, +) -> impl Iterator>> + Send where I: Iterator + ExactSizeIterator + Debug + Send, K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 2f6d2020..2ffc68df 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -11,7 +11,10 @@ use crate::{ }; #[implement(super::Map)] -pub fn keys_from<'a, K, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send +pub fn keys_from<'a, K, P>( + self: &'a Arc, + from: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -30,7 +33,10 @@ where } #[implement(super::Map)] -pub fn keys_raw_from<'a, K, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send +pub fn keys_raw_from<'a, K, P>( + self: &'a Arc, + from: &P, +) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index 24112ac1..32a1f04c 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -11,7 +11,10 @@ use serde::{Deserialize, Serialize}; use crate::keyval::{result_deserialize_key, serialize_key, Key}; #[implement(super::Map)] -pub fn keys_prefix<'a, K, P>(self: &'a Arc, prefix: &P) -> impl Stream>> + Send +pub fn keys_prefix<'a, K, P>( + self: &'a Arc, + prefix: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -22,7 +25,10 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn keys_prefix_raw

    (self: &Arc, prefix: &P) -> impl Stream>> + Send +pub fn keys_prefix_raw

    ( + self: &Arc, + prefix: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -33,7 +39,8 @@ where #[implement(super::Map)] pub fn keys_raw_prefix<'a, K, P>( - self: &'a Arc, prefix: &'a P, + self: &'a Arc, + prefix: &'a P, ) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, @@ -44,7 +51,10 @@ where } #[implement(super::Map)] -pub fn raw_keys_prefix<'a, P>(self: &'a Arc, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn raw_keys_prefix<'a, P>( + self: &'a Arc, + prefix: &'a P, +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index c895105c..a398f315 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -11,7 +11,10 @@ use crate::{ }; #[implement(super::Map)] -pub fn rev_keys_from<'a, K, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send +pub fn rev_keys_from<'a, K, P>( + self: &'a Arc, + from: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -22,7 +25,10 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn rev_keys_from_raw

    (self: &Arc, from: &P) -> impl Stream>> + Send +pub fn rev_keys_from_raw

    ( + self: &Arc, + from: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -31,7 +37,10 @@ where } #[implement(super::Map)] -pub fn rev_keys_raw_from<'a, K, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send +pub fn rev_keys_raw_from<'a, K, P>( + self: &'a Arc, + from: &P, +) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, @@ -42,7 +51,10 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn rev_raw_keys_from

    (self: &Arc, from: &P) -> impl Stream>> + Send +pub fn rev_raw_keys_from

    ( + self: &Arc, + from: &P, +) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, { diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index a5ed35ec..9fda49a0 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -11,7 +11,10 @@ use serde::{Deserialize, Serialize}; use crate::keyval::{result_deserialize_key, serialize_key, Key}; #[implement(super::Map)] -pub fn rev_keys_prefix<'a, K, P>(self: &'a Arc, prefix: &P) -> impl Stream>> + Send +pub fn rev_keys_prefix<'a, K, P>( + self: &'a Arc, + prefix: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -22,7 +25,10 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn rev_keys_prefix_raw

    (self: &Arc, prefix: &P) -> impl Stream>> + Send +pub fn rev_keys_prefix_raw

    ( + self: &Arc, + prefix: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -33,7 +39,8 @@ where #[implement(super::Map)] pub fn rev_keys_raw_prefix<'a, K, P>( - self: &'a Arc, prefix: &'a P, + self: &'a Arc, + prefix: &'a P, ) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, @@ -44,7 +51,10 @@ where } #[implement(super::Map)] -pub fn rev_raw_keys_prefix<'a, P>(self: &'a Arc, prefix: &'a P) -> impl Stream>> + Send + 'a +pub fn rev_raw_keys_prefix<'a, P>( + self: &'a Arc, + prefix: &'a P, +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 2d6a76b7..6ddb9bc7 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -19,7 +19,8 @@ use crate::{ /// - Result is deserialized #[implement(super::Map)] pub fn rev_stream_from<'a, K, V, P>( - self: &'a Arc, from: &P, + self: &'a Arc, + from: &P, ) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -36,7 +37,10 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn rev_stream_from_raw

    (self: &Arc, from: &P) -> impl Stream>> + Send +pub fn rev_stream_from_raw

    ( + self: &Arc, + from: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -50,7 +54,8 @@ where /// - Result is deserialized #[implement(super::Map)] pub fn rev_stream_raw_from<'a, K, V, P>( - self: &'a Arc, from: &P, + self: &'a Arc, + from: &P, ) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, @@ -67,7 +72,10 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn rev_raw_stream_from

    (self: &Arc, from: &P) -> impl Stream>> + Send +pub fn rev_raw_stream_from

    ( + self: &Arc, + from: &P, +) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, { diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index 13aa40f2..857aa3a5 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -16,7 +16,8 @@ use crate::keyval::{result_deserialize, serialize_key, KeyVal}; /// - Result is deserialized #[implement(super::Map)] pub fn rev_stream_prefix<'a, K, V, P>( - self: &'a Arc, prefix: &P, + self: &'a Arc, + prefix: &P, ) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -33,7 +34,10 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn rev_stream_prefix_raw

    (self: &Arc, prefix: &P) -> impl Stream>> + Send +pub fn rev_stream_prefix_raw

    ( + self: &Arc, + prefix: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -48,7 +52,8 @@ where /// - Result is deserialized #[implement(super::Map)] pub fn rev_stream_raw_prefix<'a, K, V, P>( - self: &'a Arc, prefix: &'a P, + self: &'a Arc, + prefix: &'a P, ) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, @@ -65,7 +70,8 @@ where /// - Result is raw #[implement(super::Map)] pub fn rev_raw_stream_prefix<'a, P>( - self: &'a Arc, prefix: &'a P, + self: &'a Arc, + prefix: &'a P, ) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index c6fe439a..1dae9d78 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -18,7 +18,10 @@ use crate::{ /// - Query is serialized /// - Result is deserialized #[implement(super::Map)] -pub fn stream_from<'a, K, V, P>(self: &'a Arc, from: &P) -> impl Stream>> + Send +pub fn stream_from<'a, K, V, P>( + self: &'a Arc, + from: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -33,7 +36,10 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn stream_from_raw

    (self: &Arc, from: &P) -> impl Stream>> + Send +pub fn stream_from_raw

    ( + self: &Arc, + from: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -47,7 +53,8 @@ where /// - Result is deserialized #[implement(super::Map)] pub fn stream_raw_from<'a, K, V, P>( - self: &'a Arc, from: &P, + self: &'a Arc, + from: &P, ) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug + Sync, @@ -63,7 +70,10 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn raw_stream_from

    (self: &Arc, from: &P) -> impl Stream>> + Send +pub fn raw_stream_from

    ( + self: &Arc, + from: &P, +) -> impl Stream>> + Send where P: AsRef<[u8]> + ?Sized + Debug, { diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index e7dad211..a05e2fc5 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -16,7 +16,8 @@ use crate::keyval::{result_deserialize, serialize_key, KeyVal}; /// - Result is deserialized #[implement(super::Map)] pub fn stream_prefix<'a, K, V, P>( - self: &'a Arc, prefix: &P, + self: &'a Arc, + prefix: &P, ) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, @@ -33,7 +34,10 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn stream_prefix_raw

    (self: &Arc, prefix: &P) -> impl Stream>> + Send +pub fn stream_prefix_raw

    ( + self: &Arc, + prefix: &P, +) -> impl Stream>> + Send where P: Serialize + ?Sized + Debug, { @@ -48,7 +52,8 @@ where /// - Result is deserialized #[implement(super::Map)] pub fn stream_raw_prefix<'a, K, V, P>( - self: &'a Arc, prefix: &'a P, + self: &'a Arc, + prefix: &'a P, ) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, @@ -65,7 +70,8 @@ where /// - Result is raw #[implement(super::Map)] pub fn raw_stream_prefix<'a, P>( - self: &'a Arc, prefix: &'a P, + self: &'a Arc, + prefix: &'a P, ) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, diff --git a/src/database/opts.rs b/src/database/opts.rs index 52ed4818..933830bd 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -2,8 +2,9 @@ use std::{cmp, collections::HashMap, convert::TryFrom}; use conduwuit::{err, utils, Config, Result}; use rocksdb::{ - statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, DBRecoveryMode, Env, - LogLevel, LruCacheOptions, Options, UniversalCompactOptions, UniversalCompactionStopStyle, + statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, + DBRecoveryMode, Env, LogLevel, LruCacheOptions, Options, UniversalCompactOptions, + UniversalCompactionStopStyle, }; /// Create database-wide options suitable for opening the database. This also @@ -11,7 +12,12 @@ use rocksdb::{ /// resulting value. Note that we require special per-column options on some /// columns, therefor columns should only be opened after passing this result /// through cf_options(). -pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_cache: &Cache) -> Result { +pub(crate) fn db_options( + config: &Config, + env: &mut Env, + row_cache: &Cache, + col_cache: &Cache, +) -> Result { const DEFAULT_STATS_LEVEL: StatsLevel = if cfg!(debug_assertions) { StatsLevel::ExceptDetailedTimers } else { @@ -73,13 +79,13 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ opts.set_disable_auto_compactions(!config.rocksdb_compaction); opts.set_statistics_level(match config.rocksdb_stats_level { - 0 => StatsLevel::DisableAll, - 1 => DEFAULT_STATS_LEVEL, - 2 => StatsLevel::ExceptHistogramOrTimers, - 3 => StatsLevel::ExceptTimers, - 4 => StatsLevel::ExceptDetailedTimers, - 5 => StatsLevel::ExceptTimeForMutex, - 6_u8..=u8::MAX => StatsLevel::All, + | 0 => StatsLevel::DisableAll, + | 1 => DEFAULT_STATS_LEVEL, + | 2 => StatsLevel::ExceptHistogramOrTimers, + | 3 => StatsLevel::ExceptTimers, + | 4 => StatsLevel::ExceptDetailedTimers, + | 5 => StatsLevel::ExceptTimeForMutex, + | 6_u8..=u8::MAX => StatsLevel::All, }); // Default: https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes#ktoleratecorruptedtailrecords @@ -88,11 +94,11 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ // recovered in this manner as it's likely any lost information will be // restored via federation. opts.set_wal_recovery_mode(match config.rocksdb_recovery_mode { - 0 => DBRecoveryMode::AbsoluteConsistency, - 1 => DBRecoveryMode::TolerateCorruptedTailRecords, - 2 => DBRecoveryMode::PointInTime, - 3 => DBRecoveryMode::SkipAnyCorruptedRecord, - 4_u8..=u8::MAX => unimplemented!(), + | 0 => DBRecoveryMode::AbsoluteConsistency, + | 1 => DBRecoveryMode::TolerateCorruptedTailRecords, + | 2 => DBRecoveryMode::PointInTime, + | 3 => DBRecoveryMode::SkipAnyCorruptedRecord, + | 4_u8..=u8::MAX => unimplemented!(), }); // @@ -111,11 +117,14 @@ pub(crate) fn db_options(config: &Config, env: &mut Env, row_cache: &Cache, col_ /// db_options() as the argument to this function and use the return value in /// the arguments to open the specific column. pub(crate) fn cf_options( - cfg: &Config, name: &str, mut opts: Options, cache: &mut HashMap, + cfg: &Config, + name: &str, + mut opts: Options, + cache: &mut HashMap, ) -> Result { // Columns with non-default compaction options match name { - "backupid_algorithm" + | "backupid_algorithm" | "backupid_etag" | "backupkeyid_backup" | "roomid_shortroomid" @@ -125,12 +134,12 @@ pub(crate) fn cf_options( | "shortstatehash_statediff" | "userdevicetxnid_response" | "userfilterid_filter" => set_for_sequential_small_uc(&mut opts, cfg), - &_ => {}, + | &_ => {}, } // Columns with non-default table/cache configs match name { - "shorteventid_eventid" => set_table_with_new_cache( + | "shorteventid_eventid" => set_table_with_new_cache( &mut opts, cfg, cache, @@ -138,7 +147,7 @@ pub(crate) fn cf_options( cache_size(cfg, cfg.shorteventid_cache_capacity, 64)?, ), - "eventid_shorteventid" => set_table_with_new_cache( + | "eventid_shorteventid" => set_table_with_new_cache( &mut opts, cfg, cache, @@ -146,7 +155,7 @@ pub(crate) fn cf_options( cache_size(cfg, cfg.eventidshort_cache_capacity, 64)?, ), - "eventid_pduid" => set_table_with_new_cache( + | "eventid_pduid" => set_table_with_new_cache( &mut opts, cfg, cache, @@ -154,7 +163,7 @@ pub(crate) fn cf_options( cache_size(cfg, cfg.eventid_pdu_cache_capacity, 64)?, ), - "shorteventid_authchain" => { + | "shorteventid_authchain" => { set_table_with_new_cache( &mut opts, cfg, @@ -164,7 +173,7 @@ pub(crate) fn cf_options( ); }, - "shortstatekey_statekey" => set_table_with_new_cache( + | "shortstatekey_statekey" => set_table_with_new_cache( &mut opts, cfg, cache, @@ -172,7 +181,7 @@ pub(crate) fn cf_options( cache_size(cfg, cfg.shortstatekey_cache_capacity, 1024)?, ), - "statekey_shortstatekey" => set_table_with_new_cache( + | "statekey_shortstatekey" => set_table_with_new_cache( &mut opts, cfg, cache, @@ -180,22 +189,32 @@ pub(crate) fn cf_options( cache_size(cfg, cfg.statekeyshort_cache_capacity, 1024)?, ), - "servernameevent_data" => set_table_with_new_cache( + | "servernameevent_data" => set_table_with_new_cache( &mut opts, cfg, cache, name, - cache_size(cfg, cfg.servernameevent_data_cache_capacity, 128)?, /* Raw average value size = 102, key + cache_size(cfg, cfg.servernameevent_data_cache_capacity, 128)?, /* Raw average + * value size = + * 102, key * size = 34 */ ), - "eventid_outlierpdu" => { - set_table_with_new_cache(&mut opts, cfg, cache, name, cache_size(cfg, cfg.pdu_cache_capacity, 1536)?); + | "eventid_outlierpdu" => { + set_table_with_new_cache( + &mut opts, + cfg, + cache, + name, + cache_size(cfg, cfg.pdu_cache_capacity, 1536)?, + ); }, - "pduid_pdu" => set_table_with_shared_cache(&mut opts, cfg, cache, name, "eventid_outlierpdu"), + | "pduid_pdu" => { + set_table_with_shared_cache(&mut opts, cfg, cache, name, "eventid_outlierpdu"); + }, - &_ => {}, + | &_ => {}, } Ok(opts) @@ -203,11 +222,11 @@ pub(crate) fn cf_options( fn set_logging_defaults(opts: &mut Options, config: &Config) { let rocksdb_log_level = match config.rocksdb_log_level.as_ref() { - "debug" => LogLevel::Debug, - "info" => LogLevel::Info, - "warn" => LogLevel::Warn, - "fatal" => LogLevel::Fatal, - _ => LogLevel::Error, + | "debug" => LogLevel::Debug, + | "info" => LogLevel::Info, + | "warn" => LogLevel::Warn, + | "fatal" => LogLevel::Fatal, + | _ => LogLevel::Error, }; opts.set_log_level(rocksdb_log_level); @@ -225,13 +244,13 @@ fn set_logging_defaults(opts: &mut Options, config: &Config) { fn set_compression_defaults(opts: &mut Options, config: &Config) { let rocksdb_compression_algo = match config.rocksdb_compression_algo.as_ref() { - "snappy" => DBCompressionType::Snappy, - "zlib" => DBCompressionType::Zlib, - "bz2" => DBCompressionType::Bz2, - "lz4" => DBCompressionType::Lz4, - "lz4hc" => DBCompressionType::Lz4hc, - "none" => DBCompressionType::None, - _ => DBCompressionType::Zstd, + | "snappy" => DBCompressionType::Snappy, + | "zlib" => DBCompressionType::Zlib, + | "bz2" => DBCompressionType::Bz2, + | "lz4" => DBCompressionType::Lz4, + | "lz4hc" => DBCompressionType::Lz4hc, + | "none" => DBCompressionType::None, + | _ => DBCompressionType::Zstd, }; if config.rocksdb_bottommost_compression { @@ -239,7 +258,13 @@ fn set_compression_defaults(opts: &mut Options, config: &Config) { opts.set_bottommost_zstd_max_train_bytes(0, true); // -14 w_bits is only read by zlib. - opts.set_bottommost_compression_options(-14, config.rocksdb_bottommost_compression_level, 0, 0, true); + opts.set_bottommost_compression_options( + -14, + config.rocksdb_bottommost_compression_level, + 0, + 0, + true, + ); } // -14 w_bits is only read by zlib. @@ -338,7 +363,11 @@ fn uc_options(_config: &Config) -> UniversalCompactOptions { } fn set_table_with_new_cache( - opts: &mut Options, config: &Config, caches: &mut HashMap, name: &str, size: usize, + opts: &mut Options, + config: &Config, + caches: &mut HashMap, + name: &str, + size: usize, ) { let mut cache_opts = LruCacheOptions::default(); cache_opts.set_capacity(size); @@ -351,7 +380,11 @@ fn set_table_with_new_cache( } fn set_table_with_shared_cache( - opts: &mut Options, config: &Config, cache: &HashMap, _name: &str, cache_name: &str, + opts: &mut Options, + config: &Config, + cache: &HashMap, + _name: &str, + cache_name: &str, ) { let mut table = table_options(config); table.set_block_cache( diff --git a/src/database/pool.rs b/src/database/pool.rs index 3301b821..e4d78897 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -230,8 +230,8 @@ fn worker_wait(&self, recv: &Receiver) -> Result { #[implement(Pool)] fn worker_handle(&self, cmd: Cmd) { match cmd { - Cmd::Get(cmd) => self.handle_get(cmd), - Cmd::Iter(cmd) => self.handle_iter(cmd), + | Cmd::Get(cmd) => self.handle_get(cmd), + | Cmd::Iter(cmd) => self.handle_iter(cmd), } } @@ -251,8 +251,8 @@ fn handle_iter(&self, mut cmd: Seek) { let from = cmd.key.as_deref().map(Into::into); let result = match cmd.dir { - Direction::Forward => cmd.state.init_fwd(from), - Direction::Reverse => cmd.state.init_rev(from), + | Direction::Forward => cmd.state.init_fwd(from), + | Direction::Reverse => cmd.state.init_rev(from), }; let chan_result = chan.send(into_send_seek(result)); @@ -274,8 +274,8 @@ fn _handle_seek(&self, mut cmd: Seek) { } match cmd.dir { - Direction::Forward => cmd.state.seek_fwd(), - Direction::Reverse => cmd.state.seek_rev(), + | Direction::Forward => cmd.state.seek_fwd(), + | Direction::Reverse => cmd.state.seek_rev(), }; let chan_result = chan.send(into_send_seek(cmd.state)); diff --git a/src/database/ser.rs b/src/database/ser.rs index 8abe5521..878319fe 100644 --- a/src/database/ser.rs +++ b/src/database/ser.rs @@ -6,7 +6,9 @@ use serde::{ser, Serialize}; use crate::util::unhandled; #[inline] -pub fn serialize_to_vec(val: T) -> Result> { serialize_to::, T>(val) } +pub fn serialize_to_vec(val: T) -> Result> { + serialize_to::, T>(val) +} #[inline] pub fn serialize_to(val: T) -> Result @@ -26,17 +28,15 @@ where W: Write + AsRef<[u8]> + 'a, T: Serialize, { - let mut serializer = Serializer { - out, - depth: 0, - sep: false, - fin: false, - }; + let mut serializer = Serializer { out, depth: 0, sep: false, fin: false }; val.serialize(&mut serializer) .map_err(|error| err!(SerdeSer("{error}"))) .debug_inspect(|()| { - debug_assert_eq!(serializer.depth, 0, "Serialization completed at non-zero recursion level"); + debug_assert_eq!( + serializer.depth, 0, + "Serialization completed at non-zero recursion level" + ); })?; Ok((*out).as_ref()) @@ -132,29 +132,45 @@ impl ser::Serializer for &mut Serializer<'_, W> { Ok(self) } - fn serialize_tuple_struct(self, _name: &'static str, _len: usize) -> Result { + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { self.tuple_start(); Ok(self) } fn serialize_tuple_variant( - self, _name: &'static str, _idx: u32, _var: &'static str, _len: usize, + self, + _name: &'static str, + _idx: u32, + _var: &'static str, + _len: usize, ) -> Result { unhandled!("serialize Tuple Variant not implemented") } fn serialize_map(self, _len: Option) -> Result { - unhandled!("serialize Map not implemented; did you mean to use database::Json() around your serde_json::Value?") + unhandled!( + "serialize Map not implemented; did you mean to use database::Json() around your \ + serde_json::Value?" + ) } fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { unhandled!( - "serialize Struct not implemented at this time; did you mean to use database::Json() around your struct?" + "serialize Struct not implemented at this time; did you mean to use \ + database::Json() around your struct?" ) } fn serialize_struct_variant( - self, _name: &'static str, _idx: u32, _var: &'static str, _len: usize, + self, + _name: &'static str, + _idx: u32, + _var: &'static str, + _len: usize, ) -> Result { unhandled!("serialize Struct Variant not implemented") } @@ -170,36 +186,47 @@ impl ser::Serializer for &mut Serializer<'_, W> { ); match name { - "Json" => serde_json::to_writer(&mut self.out, value).map_err(Into::into), - _ => unhandled!("Unrecognized serialization Newtype {name:?}"), + | "Json" => serde_json::to_writer(&mut self.out, value).map_err(Into::into), + | _ => unhandled!("Unrecognized serialization Newtype {name:?}"), } } fn serialize_newtype_variant( - self, _name: &'static str, _idx: u32, _var: &'static str, _value: &T, + self, + _name: &'static str, + _idx: u32, + _var: &'static str, + _value: &T, ) -> Result { unhandled!("serialize Newtype Variant not implemented") } fn serialize_unit_struct(self, name: &'static str) -> Result { match name { - "Interfix" => { + | "Interfix" => { self.set_finalized(); }, - "Separator" => { + | "Separator" => { self.separator()?; }, - _ => unhandled!("Unrecognized serialization directive: {name:?}"), + | _ => unhandled!("Unrecognized serialization directive: {name:?}"), }; Ok(()) } - fn serialize_unit_variant(self, _name: &'static str, _idx: u32, _var: &'static str) -> Result { + fn serialize_unit_variant( + self, + _name: &'static str, + _idx: u32, + _var: &'static str, + ) -> Result { unhandled!("serialize Unit Variant not implemented") } - fn serialize_some(self, val: &T) -> Result { val.serialize(self) } + fn serialize_some(self, val: &T) -> Result { + val.serialize(self) + } fn serialize_none(self) -> Result { Ok(()) } @@ -226,27 +253,39 @@ impl ser::Serializer for &mut Serializer<'_, W> { self.write(v) } - fn serialize_f64(self, _v: f64) -> Result { unhandled!("serialize f64 not implemented") } + fn serialize_f64(self, _v: f64) -> Result { + unhandled!("serialize f64 not implemented") + } - fn serialize_f32(self, _v: f32) -> Result { unhandled!("serialize f32 not implemented") } + fn serialize_f32(self, _v: f32) -> Result { + unhandled!("serialize f32 not implemented") + } fn serialize_i64(self, v: i64) -> Result { self.write(&v.to_be_bytes()) } fn serialize_i32(self, v: i32) -> Result { self.write(&v.to_be_bytes()) } - fn serialize_i16(self, _v: i16) -> Result { unhandled!("serialize i16 not implemented") } + fn serialize_i16(self, _v: i16) -> Result { + unhandled!("serialize i16 not implemented") + } - fn serialize_i8(self, _v: i8) -> Result { unhandled!("serialize i8 not implemented") } + fn serialize_i8(self, _v: i8) -> Result { + unhandled!("serialize i8 not implemented") + } fn serialize_u64(self, v: u64) -> Result { self.write(&v.to_be_bytes()) } fn serialize_u32(self, v: u32) -> Result { self.write(&v.to_be_bytes()) } - fn serialize_u16(self, _v: u16) -> Result { unhandled!("serialize u16 not implemented") } + fn serialize_u16(self, _v: u16) -> Result { + unhandled!("serialize u16 not implemented") + } fn serialize_u8(self, v: u8) -> Result { self.write(&[v]) } - fn serialize_bool(self, _v: bool) -> Result { unhandled!("serialize bool not implemented") } + fn serialize_bool(self, _v: bool) -> Result { + unhandled!("serialize bool not implemented") + } fn serialize_unit(self) -> Result { unhandled!("serialize unit not implemented") } } @@ -255,7 +294,9 @@ impl ser::SerializeSeq for &mut Serializer<'_, W> { type Error = Error; type Ok = (); - fn serialize_element(&mut self, val: &T) -> Result { val.serialize(&mut **self) } + fn serialize_element(&mut self, val: &T) -> Result { + val.serialize(&mut **self) + } fn end(self) -> Result { self.sequence_end() } } @@ -315,7 +356,11 @@ impl ser::SerializeStruct for &mut Serializer<'_, W> { type Error = Error; type Ok = (); - fn serialize_field(&mut self, _key: &'static str, _val: &T) -> Result { + fn serialize_field( + &mut self, + _key: &'static str, + _val: &T, + ) -> Result { unhandled!("serialize Struct Field not implemented") } @@ -326,9 +371,15 @@ impl ser::SerializeStructVariant for &mut Serializer<'_, W> { type Error = Error; type Ok = (); - fn serialize_field(&mut self, _key: &'static str, _val: &T) -> Result { + fn serialize_field( + &mut self, + _key: &'static str, + _val: &T, + ) -> Result { unhandled!("serialize Struct Variant Field not implemented") } - fn end(self) -> Result { unhandled!("serialize Struct Variant End not implemented") } + fn end(self) -> Result { + unhandled!("serialize Struct Variant End not implemented") + } } diff --git a/src/database/stream/items.rs b/src/database/stream/items.rs index 06cb6de9..2a38d97e 100644 --- a/src/database/stream/items.rs +++ b/src/database/stream/items.rs @@ -17,18 +17,12 @@ pub(crate) struct Items<'a> { impl<'a> Items<'a> { pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { - Self { - state: State::new(db, cf, opts), - } + Self { state: State::new(db, cf, opts) } } } impl<'a> convert::From> for Items<'a> { - fn from(state: State<'a>) -> Self { - Self { - state, - } - } + fn from(state: State<'a>) -> Self { Self { state } } } impl<'a> Cursor<'a, KeyVal<'a>> for Items<'a> { @@ -40,11 +34,7 @@ impl<'a> Cursor<'a, KeyVal<'a>> for Items<'a> { fn seek(&mut self) { self.state.seek_fwd(); } #[inline] - fn init(self, from: From<'a>) -> Self { - Self { - state: self.state.init_fwd(from), - } - } + fn init(self, from: From<'a>) -> Self { Self { state: self.state.init_fwd(from) } } } impl<'a> Stream for Items<'a> { diff --git a/src/database/stream/items_rev.rs b/src/database/stream/items_rev.rs index 2d0c4639..c3a6cc7f 100644 --- a/src/database/stream/items_rev.rs +++ b/src/database/stream/items_rev.rs @@ -17,18 +17,12 @@ pub(crate) struct ItemsRev<'a> { impl<'a> ItemsRev<'a> { pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { - Self { - state: State::new(db, cf, opts), - } + Self { state: State::new(db, cf, opts) } } } impl<'a> convert::From> for ItemsRev<'a> { - fn from(state: State<'a>) -> Self { - Self { - state, - } - } + fn from(state: State<'a>) -> Self { Self { state } } } impl<'a> Cursor<'a, KeyVal<'a>> for ItemsRev<'a> { @@ -40,11 +34,7 @@ impl<'a> Cursor<'a, KeyVal<'a>> for ItemsRev<'a> { fn seek(&mut self) { self.state.seek_rev(); } #[inline] - fn init(self, from: From<'a>) -> Self { - Self { - state: self.state.init_rev(from), - } - } + fn init(self, from: From<'a>) -> Self { Self { state: self.state.init_rev(from) } } } impl<'a> Stream for ItemsRev<'a> { diff --git a/src/database/stream/keys.rs b/src/database/stream/keys.rs index a901b342..0696781d 100644 --- a/src/database/stream/keys.rs +++ b/src/database/stream/keys.rs @@ -17,18 +17,12 @@ pub(crate) struct Keys<'a> { impl<'a> Keys<'a> { pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { - Self { - state: State::new(db, cf, opts), - } + Self { state: State::new(db, cf, opts) } } } impl<'a> convert::From> for Keys<'a> { - fn from(state: State<'a>) -> Self { - Self { - state, - } - } + fn from(state: State<'a>) -> Self { Self { state } } } impl<'a> Cursor<'a, Key<'a>> for Keys<'a> { @@ -41,11 +35,7 @@ impl<'a> Cursor<'a, Key<'a>> for Keys<'a> { fn seek(&mut self) { self.state.seek_fwd(); } #[inline] - fn init(self, from: From<'a>) -> Self { - Self { - state: self.state.init_fwd(from), - } - } + fn init(self, from: From<'a>) -> Self { Self { state: self.state.init_fwd(from) } } } impl<'a> Stream for Keys<'a> { diff --git a/src/database/stream/keys_rev.rs b/src/database/stream/keys_rev.rs index 73758a85..42706d9f 100644 --- a/src/database/stream/keys_rev.rs +++ b/src/database/stream/keys_rev.rs @@ -17,18 +17,12 @@ pub(crate) struct KeysRev<'a> { impl<'a> KeysRev<'a> { pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { - Self { - state: State::new(db, cf, opts), - } + Self { state: State::new(db, cf, opts) } } } impl<'a> convert::From> for KeysRev<'a> { - fn from(state: State<'a>) -> Self { - Self { - state, - } - } + fn from(state: State<'a>) -> Self { Self { state } } } impl<'a> Cursor<'a, Key<'a>> for KeysRev<'a> { @@ -41,11 +35,7 @@ impl<'a> Cursor<'a, Key<'a>> for KeysRev<'a> { fn seek(&mut self) { self.state.seek_rev(); } #[inline] - fn init(self, from: From<'a>) -> Self { - Self { - state: self.state.init_rev(from), - } - } + fn init(self, from: From<'a>) -> Self { Self { state: self.state.init_rev(from) } } } impl<'a> Stream for KeysRev<'a> { diff --git a/src/database/tests.rs b/src/database/tests.rs index 7f486966..3d41a544 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -66,10 +66,7 @@ fn ser_complex() { media_id: "AbCdEfGhIjK", }; - let dim = Dim { - width: 123, - height: 456, - }; + let dim = Dim { width: 123, height: 456 }; let mut a = Vec::new(); a.extend_from_slice(b"mxc://"); @@ -128,9 +125,7 @@ fn ser_json_macro() { foo: String, } - let content = Foo { - foo: "bar".to_owned(), - }; + let content = Foo { foo: "bar".to_owned() }; let content = serde_json::to_value(content).expect("failed to serialize content"); let sender: &UserId = "@foo:example.com".try_into().unwrap(); let serialized = serialize_to_vec(Json(json!({ @@ -153,7 +148,8 @@ fn ser_json_raw() { ..Default::default() }; - let value = serde_json::value::to_raw_value(&filter).expect("failed to serialize to raw value"); + let value = + serde_json::value::to_raw_value(&filter).expect("failed to serialize to raw value"); let a = serialize_to_vec(value.get()).expect("failed to serialize raw value"); let s = String::from_utf8_lossy(&a); assert_eq!(&s, r#"{"event_fields":["content.body"]}"#); @@ -169,7 +165,8 @@ fn ser_json_raw_json() { ..Default::default() }; - let value = serde_json::value::to_raw_value(&filter).expect("failed to serialize to raw value"); + let value = + serde_json::value::to_raw_value(&filter).expect("failed to serialize to raw value"); let a = serialize_to_vec(Json(value)).expect("failed to serialize json value"); let s = String::from_utf8_lossy(&a); assert_eq!(&s, r#"{"event_fields":["content.body"]}"#); @@ -241,7 +238,8 @@ fn de_tuple_ignore() { let room_id: &RoomId = "!room:example.com".try_into().unwrap(); let raw: &[u8] = b"@user:example.com\xFF@user2:example.net\xFF!room:example.com"; - let (a, _, c): (&UserId, Ignore, &RoomId) = de::from_slice(raw).expect("failed to deserialize"); + let (a, _, c): (&UserId, Ignore, &RoomId) = + de::from_slice(raw).expect("failed to deserialize"); assert_eq!(a, user_id, "deserialized user_id does not match"); assert_eq!(c, room_id, "deserialized room_id does not match"); @@ -254,7 +252,8 @@ fn de_json_array() { let b: Raw>> = de::from_slice(&s).expect("failed to deserialize"); - let d: Vec = serde_json::from_str(b.json().get()).expect("failed to deserialize JSON"); + let d: Vec = + serde_json::from_str(b.json().get()).expect("failed to deserialize JSON"); for (i, a) in a.iter().enumerate() { assert_eq!(*a, d[i]); @@ -268,7 +267,8 @@ fn de_json_raw_array() { let b: Raw>> = de::from_slice(&s).expect("failed to deserialize"); - let c: Vec> = serde_json::from_str(b.json().get()).expect("failed to deserialize JSON"); + let c: Vec> = + serde_json::from_str(b.json().get()).expect("failed to deserialize JSON"); for (i, a) in a.iter().enumerate() { let c = serde_json::to_value(c[i].json()).expect("failed to deserialize JSON to string"); diff --git a/src/database/util.rs b/src/database/util.rs index 0fca3b68..c2a020e3 100644 --- a/src/database/util.rs +++ b/src/database/util.rs @@ -30,13 +30,15 @@ pub(crate) fn _into_direction(mode: &IteratorMode<'_>) -> Direction { use IteratorMode::{End, From, Start}; match mode { - Start | From(_, Forward) => Forward, - End | From(_, Reverse) => Reverse, + | Start | From(_, Forward) => Forward, + | End | From(_, Reverse) => Reverse, } } #[inline] -pub(crate) fn result(r: std::result::Result) -> Result { +pub(crate) fn result( + r: std::result::Result, +) -> Result { r.map_or_else(or_else, and_then) } diff --git a/src/database/watchers.rs b/src/database/watchers.rs index 93c82b44..9ce6f74c 100644 --- a/src/database/watchers.rs +++ b/src/database/watchers.rs @@ -15,10 +15,13 @@ pub(crate) struct Watchers { } impl Watchers { - pub(crate) fn watch<'a>(&'a self, prefix: &[u8]) -> Pin + Send + 'a>> { + pub(crate) fn watch<'a>( + &'a self, + prefix: &[u8], + ) -> Pin + Send + 'a>> { let mut rx = match self.watchers.write().unwrap().entry(prefix.to_vec()) { - hash_map::Entry::Occupied(o) => o.get().1.clone(), - hash_map::Entry::Vacant(v) => { + | hash_map::Entry::Occupied(o) => o.get().1.clone(), + | hash_map::Entry::Vacant(v) => { let (tx, rx) = watch::channel(()); v.insert((tx, rx.clone())); rx diff --git a/src/macros/admin.rs b/src/macros/admin.rs index b0dc1956..e98e914c 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -42,14 +42,14 @@ fn dispatch_arm(v: &Variant) -> Result { let target = camel_to_snake_string(&format!("{name}")); let handler = Ident::new(&target, Span::call_site().into()); let res = match &v.fields { - Fields::Named(fields) => { + | Fields::Named(fields) => { let field = fields.named.iter().filter_map(|f| f.ident.as_ref()); let arg = field.clone(); quote! { #name { #( #field ),* } => Box::pin(context.#handler(#( #arg ),*)).await?, } }, - Fields::Unnamed(fields) => { + | Fields::Unnamed(fields) => { let Some(ref field) = fields.unnamed.first() else { return Err(Error::new(Span::call_site().into(), "One unnamed field required")); }; @@ -57,7 +57,7 @@ fn dispatch_arm(v: &Variant) -> Result { #name ( #field ) => Box::pin(#handler::process(#field, context)).await?, } }, - Fields::Unit => { + | Fields::Unit => { quote! { #name => Box::pin(context.#handler()).await?, } diff --git a/src/macros/config.rs b/src/macros/config.rs index 2934a0b2..eb269e1e 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -4,8 +4,8 @@ use proc_macro::TokenStream; use proc_macro2::Span; use quote::ToTokens; use syn::{ - parse::Parser, punctuated::Punctuated, spanned::Spanned, Error, Expr, ExprLit, Field, Fields, FieldsNamed, - ItemStruct, Lit, Meta, MetaList, MetaNameValue, Type, TypePath, + parse::Parser, punctuated::Punctuated, spanned::Spanned, Error, Expr, ExprLit, Field, Fields, + FieldsNamed, ItemStruct, Lit, Meta, MetaList, MetaNameValue, Type, TypePath, }; use crate::{ @@ -29,9 +29,9 @@ pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result Result<()> { let settings = get_simple_settings(args); - let filename = settings - .get("filename") - .ok_or_else(|| Error::new(args[0].span(), "missing required 'filename' attribute argument"))?; + let filename = settings.get("filename").ok_or_else(|| { + Error::new(args[0].span(), "missing required 'filename' attribute argument") + })?; let undocumented = settings .get("undocumented") @@ -43,9 +43,9 @@ fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { .split(' ') .collect(); - let section = settings - .get("section") - .ok_or_else(|| Error::new(args[0].span(), "missing required 'section' attribute argument"))?; + let section = settings.get("section").ok_or_else(|| { + Error::new(args[0].span(), "missing required 'section' attribute argument") + })?; let mut file = OpenOptions::new() .write(true) @@ -53,7 +53,12 @@ fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { .truncate(section == "global") .append(section != "global") .open(filename) - .map_err(|e| Error::new(Span::call_site(), format!("Failed to open config file for generation: {e}")))?; + .map_err(|e| { + Error::new( + Span::call_site(), + format!("Failed to open config file for generation: {e}"), + ) + })?; if let Some(header) = settings.get("header") { file.write_all(header.as_bytes()) @@ -63,11 +68,7 @@ fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { file.write_fmt(format_args!("\n[{section}]\n")) .expect("written to config file"); - if let Fields::Named(FieldsNamed { - named, - .. - }) = &input.fields - { + if let Fields::Named(FieldsNamed { named, .. }) = &input.fields { for field in named { let Some(ident) = &field.ident else { continue; @@ -120,12 +121,7 @@ fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { fn get_default(field: &Field) -> Option { for attr in &field.attrs { - let Meta::List(MetaList { - path, - tokens, - .. - }) = &attr.meta - else { + let Meta::List(MetaList { path, tokens, .. }) = &attr.meta else { continue; }; @@ -149,23 +145,18 @@ fn get_default(field: &Field) -> Option { }; match arg { - Meta::NameValue(MetaNameValue { - value: Expr::Lit(ExprLit { - lit: Lit::Str(str), - .. - }), + | Meta::NameValue(MetaNameValue { + value: Expr::Lit(ExprLit { lit: Lit::Str(str), .. }), .. }) => { match str.value().as_str() { - "HashSet::new" | "Vec::new" | "RegexSet::empty" => Some("[]".to_owned()), - "true_fn" => return Some("true".to_owned()), - _ => return None, + | "HashSet::new" | "Vec::new" | "RegexSet::empty" => Some("[]".to_owned()), + | "true_fn" => return Some("true".to_owned()), + | _ => return None, }; }, - Meta::Path { - .. - } => return Some("false".to_owned()), - _ => return None, + | Meta::Path { .. } => return Some("false".to_owned()), + | _ => return None, }; } @@ -174,12 +165,7 @@ fn get_default(field: &Field) -> Option { fn get_doc_default(field: &Field) -> Option { for attr in &field.attrs { - let Meta::NameValue(MetaNameValue { - path, - value, - .. - }) = &attr.meta - else { + let Meta::NameValue(MetaNameValue { path, value, .. }) = &attr.meta else { continue; }; @@ -187,11 +173,7 @@ fn get_doc_default(field: &Field) -> Option { continue; } - let Expr::Lit(ExprLit { - lit, - .. - }) = &value - else { + let Expr::Lit(ExprLit { lit, .. }) = &value else { continue; }; @@ -217,12 +199,7 @@ fn get_doc_default(field: &Field) -> Option { fn get_doc_comment(field: &Field) -> Option { let mut out = String::new(); for attr in &field.attrs { - let Meta::NameValue(MetaNameValue { - path, - value, - .. - }) = &attr.meta - else { + let Meta::NameValue(MetaNameValue { path, value, .. }) = &attr.meta else { continue; }; @@ -230,11 +207,7 @@ fn get_doc_comment(field: &Field) -> Option { continue; } - let Expr::Lit(ExprLit { - lit, - .. - }) = &value - else { + let Expr::Lit(ExprLit { lit, .. }) = &value else { continue; }; @@ -254,11 +227,7 @@ fn get_doc_comment(field: &Field) -> Option { } fn get_type_name(field: &Field) -> Option { - let Type::Path(TypePath { - path, - .. - }) = &field.ty - else { + let Type::Path(TypePath { path, .. }) = &field.ty else { return None; }; diff --git a/src/macros/implement.rs b/src/macros/implement.rs index b798bae5..8d18f243 100644 --- a/src/macros/implement.rs +++ b/src/macros/implement.rs @@ -20,12 +20,15 @@ pub(super) fn implement(item: ItemFn, args: &[Meta]) -> Result { } fn get_receiver(args: &[Meta]) -> Result { - let receiver = &args - .first() - .ok_or_else(|| Error::new(Span::call_site().into(), "Missing required argument to receiver"))?; + let receiver = &args.first().ok_or_else(|| { + Error::new(Span::call_site().into(), "Missing required argument to receiver") + })?; let Meta::Path(receiver) = receiver else { - return Err(Error::new(Span::call_site().into(), "First argument is not path to receiver")); + return Err(Error::new( + Span::call_site().into(), + "First argument is not path to receiver", + )); }; Ok(receiver.clone()) diff --git a/src/macros/refutable.rs b/src/macros/refutable.rs index facb4729..66e0ebc3 100644 --- a/src/macros/refutable.rs +++ b/src/macros/refutable.rs @@ -9,11 +9,7 @@ pub(super) fn refutable(mut item: ItemFn, _args: &[Meta]) -> Result let stmt = &mut item.block.stmts; let sig = &mut item.sig; for (i, input) in inputs.iter().enumerate() { - let Typed(PatType { - pat, - .. - }) = input - else { + let Typed(PatType { pat, .. }) = input else { continue; }; @@ -24,11 +20,7 @@ pub(super) fn refutable(mut item: ItemFn, _args: &[Meta]) -> Result let variant = &pat.path; let fields = &pat.fields; - let Some(Typed(PatType { - ref mut pat, - .. - })) = sig.inputs.get_mut(i) - else { + let Some(Typed(PatType { ref mut pat, .. })) = sig.inputs.get_mut(i) else { continue; }; diff --git a/src/macros/rustc.rs b/src/macros/rustc.rs index f484e5f5..1220c8d4 100644 --- a/src/macros/rustc.rs +++ b/src/macros/rustc.rs @@ -4,8 +4,8 @@ use quote::quote; pub(super) fn flags_capture(args: TokenStream) -> TokenStream { let cargo_crate_name = std::env::var("CARGO_CRATE_NAME"); let crate_name = match cargo_crate_name.as_ref() { - Err(_) => return args, - Ok(crate_name) => crate_name.trim_start_matches("conduwuit_"), + | Err(_) => return args, + | Ok(crate_name) => crate_name.trim_start_matches("conduwuit_"), }; let flag = std::env::args().collect::>(); diff --git a/src/macros/utils.rs b/src/macros/utils.rs index 23c4c16f..9f799f7b 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -6,23 +6,11 @@ use crate::Result; pub(crate) fn get_simple_settings(args: &[Meta]) -> HashMap { args.iter().fold(HashMap::new(), |mut map, arg| { - let Meta::NameValue(MetaNameValue { - path, - value, - .. - }) = arg - else { + let Meta::NameValue(MetaNameValue { path, value, .. }) = arg else { return map; }; - let Expr::Lit( - ExprLit { - lit: Lit::Str(str), - .. - }, - .., - ) = value - else { + let Expr::Lit(ExprLit { lit: Lit::Str(str), .. }, ..) = value else { return map; }; diff --git a/src/main/logging.rs b/src/main/logging.rs index 999265e7..e8a18b10 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -10,12 +10,15 @@ use conduwuit::{ use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; #[cfg(feature = "perf_measurements")] -pub(crate) type TracingFlameGuard = Option>>; +pub(crate) type TracingFlameGuard = + Option>>; #[cfg(not(feature = "perf_measurements"))] pub(crate) type TracingFlameGuard = (); #[allow(clippy::redundant_clone)] -pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFlameGuard, Arc)> { +pub(crate) fn init( + config: &Config, +) -> Result<(LogLevelReloadHandles, TracingFlameGuard, Arc)> { let reload_handles = LogLevelReloadHandles::default(); let console_span_events = fmt_span::from_str(&config.log_span_events).unwrap_or_err(); @@ -27,7 +30,8 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla .with_ansi(config.log_colors) .with_span_events(console_span_events) .with_thread_ids(config.log_thread_ids); - let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); + let (console_reload_filter, console_reload_handle) = + reload::Layer::new(console_filter.clone()); reload_handles.add("console", Box::new(console_reload_handle)); let cap_state = Arc::new(capture::State::new()); @@ -39,8 +43,8 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla #[cfg(feature = "sentry_telemetry")] let subscriber = { - let sentry_filter = - EnvFilter::try_new(&config.sentry_filter).map_err(|e| err!(Config("sentry_filter", "{e}.")))?; + let sentry_filter = EnvFilter::try_new(&config.sentry_filter) + .map_err(|e| err!(Config("sentry_filter", "{e}.")))?; let sentry_layer = sentry_tracing::layer(); let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(sentry_filter); reload_handles.add("sentry", Box::new(sentry_reload_handle)); @@ -52,8 +56,9 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla let (flame_layer, flame_guard) = if config.tracing_flame { let flame_filter = EnvFilter::try_new(&config.tracing_flame_filter) .map_err(|e| err!(Config("tracing_flame_filter", "{e}.")))?; - let (flame_layer, flame_guard) = tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) - .map_err(|e| err!(Config("tracing_flame_output_path", "{e}.")))?; + let (flame_layer, flame_guard) = + tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) + .map_err(|e| err!(Config("tracing_flame_output_path", "{e}.")))?; let flame_layer = flame_layer .with_empty_samples(false) .with_filter(flame_filter); @@ -62,17 +67,20 @@ pub(crate) fn init(config: &Config) -> Result<(LogLevelReloadHandles, TracingFla (None, None) }; - let jaeger_filter = - EnvFilter::try_new(&config.jaeger_filter).map_err(|e| err!(Config("jaeger_filter", "{e}.")))?; + let jaeger_filter = EnvFilter::try_new(&config.jaeger_filter) + .map_err(|e| err!(Config("jaeger_filter", "{e}.")))?; let jaeger_layer = config.allow_jaeger.then(|| { - opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); + opentelemetry::global::set_text_map_propagator( + opentelemetry_jaeger::Propagator::new(), + ); let tracer = opentelemetry_jaeger::new_agent_pipeline() .with_auto_split_batch(true) .with_service_name("conduwuit") .install_batch(opentelemetry_sdk::runtime::Tokio) .expect("jaeger agent pipeline"); let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(jaeger_filter.clone()); + let (jaeger_reload_filter, jaeger_reload_handle) = + reload::Layer::new(jaeger_filter.clone()); reload_handles.add("jaeger", Box::new(jaeger_reload_handle)); Some(telemetry.with_filter(jaeger_reload_filter)) }); diff --git a/src/main/main.rs b/src/main/main.rs index 5c066584..0946e835 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -68,8 +68,8 @@ async fn async_main(server: &Arc) -> Result<(), Error> { extern crate conduwuit_router as router; match router::start(&server.server).await { - Ok(services) => server.services.lock().await.insert(services), - Err(error) => { + | Ok(services) => server.services.lock().await.insert(services), + | Err(error) => { error!("Critical error starting server: {error}"); return Err(error); }, diff --git a/src/main/mods.rs b/src/main/mods.rs index ca984a64..ca799b90 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -44,8 +44,8 @@ pub(crate) async fn run(server: &Arc, starts: bool) -> Result<(bool, boo if starts { let start = main_mod.get::("start")?; match start(&server.server).await { - Ok(services) => server.services.lock().await.insert(services), - Err(error) => { + | Ok(services) => server.services.lock().await.insert(services), + | Err(error) => { error!("Starting server: {error}"); return Err(error); }, diff --git a/src/main/server.rs b/src/main/server.rs index 7c3eec1f..00c7a6cc 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -23,7 +23,10 @@ pub(crate) struct Server { } impl Server { - pub(crate) fn build(args: &Args, runtime: Option<&runtime::Handle>) -> Result, Error> { + pub(crate) fn build( + args: &Args, + runtime: Option<&runtime::Handle>, + ) -> Result, Error> { let _runtime_guard = runtime.map(runtime::Handle::enter); let raw_config = Config::load(args.config.as_deref())?; @@ -33,12 +36,14 @@ impl Server { #[cfg(feature = "sentry_telemetry")] let sentry_guard = crate::sentry::init(&config); - let (tracing_reload_handle, tracing_flame_guard, capture) = crate::logging::init(&config)?; + let (tracing_reload_handle, tracing_flame_guard, capture) = + crate::logging::init(&config)?; config.check()?; #[cfg(unix)] - sys::maximize_fd_limit().expect("Unable to increase maximum soft and hard file descriptor limit"); + sys::maximize_fd_limit() + .expect("Unable to increase maximum soft and hard file descriptor limit"); info!( server_name = %config.server_name, @@ -49,14 +54,10 @@ impl Server { ); Ok(Arc::new(Self { - server: Arc::new(conduwuit::Server::new( - config, - runtime.cloned(), - Log { - reload: tracing_reload_handle, - capture, - }, - )), + server: Arc::new(conduwuit::Server::new(config, runtime.cloned(), Log { + reload: tracing_reload_handle, + capture, + })), services: None.into(), diff --git a/src/router/layers.rs b/src/router/layers.rs index 1c5beace..96bca4fd 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -41,7 +41,11 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { #[cfg(feature = "sentry_telemetry")] let layers = layers.layer(sentry_tower::NewSentryLayer::>::new_from_top()); - #[cfg(any(feature = "zstd_compression", feature = "gzip_compression", feature = "brotli_compression"))] + #[cfg(any( + feature = "zstd_compression", + feature = "gzip_compression", + feature = "brotli_compression" + ))] let layers = layers.layer(compression_layer(server)); let layers = layers @@ -88,7 +92,11 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { Ok((router.layer(layers), guard)) } -#[cfg(any(feature = "zstd_compression", feature = "gzip_compression", feature = "brotli_compression"))] +#[cfg(any( + feature = "zstd_compression", + feature = "gzip_compression", + feature = "brotli_compression" +))] fn compression_layer(server: &Server) -> tower_http::compression::CompressionLayer { let mut compression_layer = tower_http::compression::CompressionLayer::new(); @@ -148,11 +156,15 @@ fn cors_layer(_server: &Server) -> CorsLayer { .max_age(Duration::from_secs(86400)) } -fn body_limit_layer(server: &Server) -> DefaultBodyLimit { DefaultBodyLimit::max(server.config.max_request_size) } +fn body_limit_layer(server: &Server) -> DefaultBodyLimit { + DefaultBodyLimit::max(server.config.max_request_size) +} #[tracing::instrument(name = "panic", level = "error", skip_all)] #[allow(clippy::needless_pass_by_value)] -fn catch_panic(err: Box) -> http::Response> { +fn catch_panic( + err: Box, +) -> http::Response> { //TODO: XXX /* conduwuit_service::services() diff --git a/src/router/mod.rs b/src/router/mod.rs index aab0b185..f64dcb67 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -17,7 +17,9 @@ conduwuit::mod_dtor! {} conduwuit::rustc_flags_capture! {} #[unsafe(no_mangle)] -pub extern "Rust" fn start(server: &Arc) -> Pin>> + Send>> { +pub extern "Rust" fn start( + server: &Arc, +) -> Pin>> + Send>> { AssertUnwindSafe(run::start(server.clone())) .catch_unwind() .map_err(Error::from_panic) @@ -26,7 +28,9 @@ pub extern "Rust" fn start(server: &Arc) -> Pin) -> Pin> + Send>> { +pub extern "Rust" fn stop( + services: Arc, +) -> Pin> + Send>> { AssertUnwindSafe(run::stop(services)) .catch_unwind() .map_err(Error::from_panic) @@ -35,7 +39,9 @@ pub extern "Rust" fn stop(services: Arc) -> Pin) -> Pin> + Send>> { +pub extern "Rust" fn run( + services: &Arc, +) -> Pin> + Send>> { AssertUnwindSafe(run::run(services.clone())) .catch_unwind() .map_err(Error::from_panic) diff --git a/src/router/request.rs b/src/router/request.rs index b17e9c4f..559c7f88 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -14,7 +14,9 @@ use http::{Method, StatusCode, Uri}; skip_all, )] pub(crate) async fn spawn( - State(services): State>, req: http::Request, next: axum::middleware::Next, + State(services): State>, + req: http::Request, + next: axum::middleware::Next, ) -> Result { let server = &services.server; if !server.running() { @@ -40,7 +42,9 @@ pub(crate) async fn spawn( #[tracing::instrument(level = "debug", skip_all)] pub(crate) async fn handle( - State(services): State>, req: http::Request, next: axum::middleware::Next, + State(services): State>, + req: http::Request, + next: axum::middleware::Next, ) -> Result { let server = &services.server; if !server.running() { diff --git a/src/router/run.rs b/src/router/run.rs index 6d5c8923..248f7052 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -33,9 +33,10 @@ pub(crate) async fn run(services: Arc) -> Result<()> { .runtime() .spawn(signal(server.clone(), tx.clone(), handle.clone())); - let mut listener = server - .runtime() - .spawn(serve::serve(services.clone(), handle.clone(), tx.subscribe())); + let mut listener = + server + .runtime() + .spawn(serve::serve(services.clone(), handle.clone(), tx.subscribe())); // Focal point debug!("Running"); @@ -63,7 +64,8 @@ pub(crate) async fn start(server: Arc) -> Result> { let services = Services::build(server).await?.start().await?; #[cfg(feature = "systemd")] - sd_notify::notify(true, &[sd_notify::NotifyState::Ready]).expect("failed to notify systemd of ready state"); + sd_notify::notify(true, &[sd_notify::NotifyState::Ready]) + .expect("failed to notify systemd of ready state"); debug!("Started"); Ok(services) @@ -98,7 +100,8 @@ pub(crate) async fn stop(services: Arc) -> Result<()> { } #[cfg(feature = "systemd")] - sd_notify::notify(true, &[sd_notify::NotifyState::Stopping]).expect("failed to notify systemd of stopping state"); + sd_notify::notify(true, &[sd_notify::NotifyState::Stopping]) + .expect("failed to notify systemd of stopping state"); info!("Shutdown complete."); Ok(()) @@ -121,7 +124,12 @@ async fn signal(server: Arc, tx: Sender<()>, handle: axum_server::Handle } } -async fn handle_shutdown(server: &Arc, tx: &Sender<()>, handle: &axum_server::Handle, sig: &str) { +async fn handle_shutdown( + server: &Arc, + tx: &Sender<()>, + handle: &axum_server::Handle, + sig: &str, +) { debug!("Received signal {sig}"); if let Err(e) = tx.send(()) { error!("failed sending shutdown transaction to channel: {e}"); @@ -139,7 +147,9 @@ async fn handle_shutdown(server: &Arc, tx: &Sender<()>, handle: &axum_se } async fn handle_services_poll( - server: &Arc, result: Result<()>, listener: JoinHandle>, + server: &Arc, + result: Result<()>, + listener: JoinHandle>, ) -> Result<()> { debug!("Service manager finished: {result:?}"); diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index 35792359..f6262202 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -14,7 +14,9 @@ use super::layers; /// Serve clients pub(super) async fn serve( - services: Arc, handle: ServerHandle, shutdown: broadcast::Receiver<()>, + services: Arc, + handle: ServerHandle, + shutdown: broadcast::Receiver<()>, ) -> Result<()> { let server = &services.server; let config = &server.config; diff --git a/src/router/serve/plain.rs b/src/router/serve/plain.rs index f6b6fba4..0e971f3c 100644 --- a/src/router/serve/plain.rs +++ b/src/router/serve/plain.rs @@ -9,12 +9,16 @@ use conduwuit::{debug_info, info, Result, Server}; use tokio::task::JoinSet; pub(super) async fn serve( - server: &Arc, app: Router, handle: ServerHandle, addrs: Vec, + server: &Arc, + app: Router, + handle: ServerHandle, + addrs: Vec, ) -> Result<()> { let app = app.into_make_service_with_connect_info::(); let mut join_set = JoinSet::new(); for addr in &addrs { - join_set.spawn_on(bind(*addr).handle(handle.clone()).serve(app.clone()), server.runtime()); + join_set + .spawn_on(bind(*addr).handle(handle.clone()).serve(app.clone()), server.runtime()); } info!("Listening on {addrs:?}"); diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index 7f54cfcc..9d3fbd3b 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -10,7 +10,12 @@ use conduwuit::{err, Result, Server}; use tokio::task::JoinSet; use tracing::{debug, info, warn}; -pub(super) async fn serve(server: &Arc, app: Router, handle: ServerHandle, addrs: Vec) -> Result { +pub(super) async fn serve( + server: &Arc, + app: Router, + handle: ServerHandle, + addrs: Vec, +) -> Result { let tls = &server.config.tls; let certs = tls .certs @@ -29,7 +34,8 @@ pub(super) async fn serve(server: &Arc, app: Router, handle: ServerHandl debug!("Using direct TLS. Certificate path {certs} and certificate private key path {key}",); info!( - "Note: It is strongly recommended that you use a reverse proxy instead of running conduwuit directly with TLS." + "Note: It is strongly recommended that you use a reverse proxy instead of running \ + conduwuit directly with TLS." ); let conf = RustlsConfig::from_pem_file(certs, key).await?; @@ -58,8 +64,8 @@ pub(super) async fn serve(server: &Arc, app: Router, handle: ServerHandl if tls.dual_protocol { warn!( - "Listening on {addrs:?} with TLS certificate {certs} and supporting plain text (HTTP) connections too \ - (insecure!)", + "Listening on {addrs:?} with TLS certificate {certs} and supporting plain text \ + (HTTP) connections too (insecure!)", ); } else { info!("Listening on {addrs:?} with TLS certificate {certs}"); diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index fb37e125..bee3c8c7 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -10,7 +10,9 @@ use axum::{ extract::{connect_info::IntoMakeServiceWithConnectInfo, Request}, Router, }; -use conduwuit::{debug, debug_error, info, result::UnwrapInfallible, trace, warn, Err, Result, Server}; +use conduwuit::{ + debug, debug_error, info, result::UnwrapInfallible, trace, warn, Err, Result, Server, +}; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, @@ -31,7 +33,11 @@ const NULL_ADDR: net::SocketAddr = net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new const FINI_POLL_INTERVAL: Duration = Duration::from_millis(750); #[tracing::instrument(skip_all, level = "debug")] -pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broadcast::Receiver<()>) -> Result<()> { +pub(super) async fn serve( + server: &Arc, + app: Router, + mut shutdown: broadcast::Receiver<()>, +) -> Result<()> { let mut tasks = JoinSet::<()>::new(); let executor = TokioExecutor::new(); let app = app.into_make_service_with_connect_info::(); @@ -55,8 +61,12 @@ pub(super) async fn serve(server: &Arc, app: Router, mut shutdown: broad } async fn accept( - server: &Arc, listener: &UnixListener, tasks: &mut JoinSet<()>, mut app: MakeService, - builder: server::conn::auto::Builder, conn: (UnixStream, SocketAddr), + server: &Arc, + listener: &UnixListener, + tasks: &mut JoinSet<()>, + mut app: MakeService, + builder: server::conn::auto::Builder, + conn: (UnixStream, SocketAddr), ) { let (socket, remote) = conn; let socket = TokioIo::new(socket); @@ -103,7 +113,8 @@ async fn init(server: &Arc) -> Result { } let socket_perms = config.unix_socket_perms.to_string(); - let octal_perms = u32::from_str_radix(&socket_perms, 8).expect("failed to convert octal permissions"); + let octal_perms = + u32::from_str_radix(&socket_perms, 8).expect("failed to convert octal permissions"); let perms = std::fs::Permissions::from_mode(octal_perms); if let Err(e) = fs::set_permissions(&path, perms).await { return Err!("Failed to set socket {path:?} permissions: {e}"); diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 536a24e8..ddbc15a4 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -9,8 +9,8 @@ use database::{Deserialized, Handle, Interfix, Json, Map}; use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{ events::{ - AnyGlobalAccountDataEvent, AnyRawAccountDataEvent, AnyRoomAccountDataEvent, GlobalAccountDataEventType, - RoomAccountDataEventType, + AnyGlobalAccountDataEvent, AnyRawAccountDataEvent, AnyRoomAccountDataEvent, + GlobalAccountDataEventType, RoomAccountDataEventType, }, serde::Raw, RoomId, UserId, @@ -54,7 +54,11 @@ impl crate::Service for Service { #[allow(clippy::needless_pass_by_value)] #[implement(Service)] pub async fn update( - &self, room_id: Option<&RoomId>, user_id: &UserId, event_type: RoomAccountDataEventType, data: &serde_json::Value, + &self, + room_id: Option<&RoomId>, + user_id: &UserId, + event_type: RoomAccountDataEventType, + data: &serde_json::Value, ) -> Result<()> { if data.get("type").is_none() || data.get("content").is_none() { return Err!(Request(InvalidParam("Account data doesn't have all required fields."))); @@ -91,7 +95,12 @@ where /// Searches the global account data for a specific kind. #[implement(Service)] -pub async fn get_room(&self, room_id: &RoomId, user_id: &UserId, kind: RoomAccountDataEventType) -> Result +pub async fn get_room( + &self, + room_id: &RoomId, + user_id: &UserId, + kind: RoomAccountDataEventType, +) -> Result where T: for<'de> Deserialize<'de>, { @@ -101,7 +110,12 @@ where } #[implement(Service)] -pub async fn get_raw(&self, room_id: Option<&RoomId>, user_id: &UserId, kind: &str) -> Result> { +pub async fn get_raw( + &self, + room_id: Option<&RoomId>, + user_id: &UserId, + kind: &str, +) -> Result> { let key = (room_id, user_id, kind.to_owned()); self.db .roomusertype_roomuserdataid @@ -113,7 +127,10 @@ pub async fn get_raw(&self, room_id: Option<&RoomId>, user_id: &UserId, kind: &s /// Returns all changes to the account data that happened after `since`. #[implement(Service)] pub fn changes_since<'a>( - &'a self, room_id: Option<&'a RoomId>, user_id: &'a UserId, since: u64, + &'a self, + room_id: Option<&'a RoomId>, + user_id: &'a UserId, + since: u64, ) -> impl Stream + Send + 'a { let prefix = (room_id, user_id, Interfix); let prefix = database::serialize_key(prefix).expect("failed to serialize prefix"); @@ -128,8 +145,10 @@ pub fn changes_since<'a>( .ready_take_while(move |(k, _)| k.starts_with(&prefix)) .map(move |(_, v)| { match room_id { - Some(_) => serde_json::from_slice::>(v).map(AnyRawAccountDataEvent::Room), - None => serde_json::from_slice::>(v).map(AnyRawAccountDataEvent::Global), + | Some(_) => serde_json::from_slice::>(v) + .map(AnyRawAccountDataEvent::Room), + | None => serde_json::from_slice::>(v) + .map(AnyRawAccountDataEvent::Global), } .map_err(|e| err!(Database("Database contains invalid account data: {e}"))) .log_err() diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index c25eb6c6..0edcd2f3 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -94,15 +94,16 @@ impl Console { debug!("session starting"); while self.server.running() { match self.readline().await { - Ok(event) => match event { - ReadlineEvent::Line(string) => self.clone().handle(string).await, - ReadlineEvent::Interrupted => continue, - ReadlineEvent::Eof => break, - ReadlineEvent::Quit => self.server.shutdown().unwrap_or_else(error::default_log), + | Ok(event) => match event { + | ReadlineEvent::Line(string) => self.clone().handle(string).await, + | ReadlineEvent::Interrupted => continue, + | ReadlineEvent::Eof => break, + | ReadlineEvent::Quit => + self.server.shutdown().unwrap_or_else(error::default_log), }, - Err(error) => match error { - ReadlineError::Closed => break, - ReadlineError::IO(error) => { + | Err(error) => match error { + | ReadlineError::Closed => break, + | ReadlineError::IO(error) => { error!("console I/O: {error:?}"); break; }, @@ -158,9 +159,9 @@ impl Console { async fn process(self: Arc, line: String) { match self.admin.command_in_place(line, None).await { - Ok(Some(ref content)) => self.output(content), - Err(ref content) => self.output_err(content), - _ => unreachable!(), + | Ok(Some(ref content)) => self.output(content), + | Err(ref content) => self.output_err(content), + | _ => unreachable!(), } } diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 971fdf67..7b691fb1 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -42,8 +42,9 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { let create_content = { use RoomVersionId::*; match room_version { - V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => RoomCreateEventContent::new_v1(server_user.clone()), - _ => RoomCreateEventContent::new_v11(), + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => + RoomCreateEventContent::new_v1(server_user.clone()), + | _ => RoomCreateEventContent::new_v11(), } }; @@ -52,15 +53,12 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomCreateEventContent { - federate: true, - predecessor: None, - room_version: room_version.clone(), - ..create_content - }, - ), + PduBuilder::state(String::new(), &RoomCreateEventContent { + federate: true, + predecessor: None, + room_version: room_version.clone(), + ..create_content + }), server_user, &room_id, &state_lock, @@ -72,7 +70,10 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder::state(server_user.to_string(), &RoomMemberEventContent::new(MembershipState::Join)), + PduBuilder::state( + server_user.to_string(), + &RoomMemberEventContent::new(MembershipState::Join), + ), server_user, &room_id, &state_lock, @@ -86,13 +87,10 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomPowerLevelsEventContent { - users, - ..Default::default() - }, - ), + PduBuilder::state(String::new(), &RoomPowerLevelsEventContent { + users, + ..Default::default() + }), server_user, &room_id, &state_lock, @@ -131,7 +129,10 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder::state(String::new(), &RoomGuestAccessEventContent::new(GuestAccess::Forbidden)), + PduBuilder::state( + String::new(), + &RoomGuestAccessEventContent::new(GuestAccess::Forbidden), + ), server_user, &room_id, &state_lock, @@ -155,12 +156,9 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomTopicEventContent { - topic: format!("Manage {}", services.globals.server_name()), - }, - ), + PduBuilder::state(String::new(), &RoomTopicEventContent { + topic: format!("Manage {}", services.globals.server_name()), + }), server_user, &room_id, &state_lock, @@ -174,13 +172,10 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomCanonicalAliasEventContent { - alias: Some(alias.clone()), - alt_aliases: Vec::new(), - }, - ), + PduBuilder::state(String::new(), &RoomCanonicalAliasEventContent { + alias: Some(alias.clone()), + alt_aliases: Vec::new(), + }), server_user, &room_id, &state_lock, @@ -197,12 +192,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .rooms .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomPreviewUrlsEventContent { - disabled: true, - }, - ), + PduBuilder::state(String::new(), &RoomPreviewUrlsEventContent { disabled: true }), server_user, &room_id, &state_lock, diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 2b05bfc7..3ad9283f 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -34,7 +34,10 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { self.services .timeline .build_and_append_pdu( - PduBuilder::state(user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Invite)), + PduBuilder::state( + user_id.to_string(), + &RoomMemberEventContent::new(MembershipState::Invite), + ), server_user, &room_id, &state_lock, @@ -43,7 +46,10 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { self.services .timeline .build_and_append_pdu( - PduBuilder::state(user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Join)), + PduBuilder::state( + user_id.to_string(), + &RoomMemberEventContent::new(MembershipState::Join), + ), user_id, &room_id, &state_lock, @@ -51,18 +57,18 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { .await?; // Set power level - let users = BTreeMap::from_iter([(server_user.clone(), 100.into()), (user_id.to_owned(), 100.into())]); + let users = BTreeMap::from_iter([ + (server_user.clone(), 100.into()), + (user_id.to_owned(), 100.into()), + ]); self.services .timeline .build_and_append_pdu( - PduBuilder::state( - String::new(), - &RoomPowerLevelsEventContent { - users, - ..Default::default() - }, - ), + PduBuilder::state(String::new(), &RoomPowerLevelsEventContent { + users, + ..Default::default() + }), server_user, &room_id, &state_lock, @@ -103,9 +109,7 @@ async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> R .get_room(room_id, user_id, RoomAccountDataEventType::Tag) .await .unwrap_or_else(|_| TagEvent { - content: TagEventContent { - tags: BTreeMap::new(), - }, + content: TagEventContent { tags: BTreeMap::new() }, }); event diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index c4783565..59639e58 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -10,7 +10,9 @@ use std::{ }; use async_trait::async_trait; -use conduwuit::{debug, err, error, error::default_log, pdu::PduBuilder, Error, PduEvent, Result, Server}; +use conduwuit::{ + debug, err, error, error::default_log, pdu::PduBuilder, Error, PduEvent, Result, Server, +}; pub use create::create_admin_room; use futures::{FutureExt, TryFutureExt}; use loole::{Receiver, Sender}; @@ -158,21 +160,19 @@ impl Service { /// the queue is full. pub fn command(&self, command: String, reply_id: Option) -> Result<()> { self.sender - .send(CommandInput { - command, - reply_id, - }) + .send(CommandInput { command, reply_id }) .map_err(|e| err!("Failed to enqueue admin command: {e:?}")) } /// Dispatches a comamnd to the processor on the current task and waits for /// completion. - pub async fn command_in_place(&self, command: String, reply_id: Option) -> ProcessorResult { - self.process_command(CommandInput { - command, - reply_id, - }) - .await + pub async fn command_in_place( + &self, + command: String, + reply_id: Option, + ) -> ProcessorResult { + self.process_command(CommandInput { command, reply_id }) + .await } /// Invokes the tab-completer to complete the command. When unavailable, @@ -191,8 +191,8 @@ impl Service { async fn handle_command(&self, command: CommandInput) { match self.process_command(command).await { - Ok(None) => debug!("Command successful with no response"), - Ok(Some(output)) | Err(output) => self + | Ok(None) => debug!("Command successful with no response"), + | Ok(Some(output)) | Err(output) => self .handle_response(output) .await .unwrap_or_else(default_log), @@ -250,10 +250,7 @@ impl Service { } async fn handle_response(&self, content: RoomMessageEventContent) -> Result<()> { - let Some(Relation::Reply { - in_reply_to, - }) = content.relates_to.as_ref() - else { + let Some(Relation::Reply { in_reply_to }) = content.relates_to.as_ref() else { return Ok(()); }; @@ -277,7 +274,10 @@ impl Service { } async fn respond_to_room( - &self, content: RoomMessageEventContent, room_id: &RoomId, user_id: &UserId, + &self, + content: RoomMessageEventContent, + room_id: &RoomId, + user_id: &UserId, ) -> Result<()> { assert!(self.user_is_admin(user_id).await, "sender is not admin"); @@ -298,12 +298,16 @@ impl Service { } async fn handle_response_error( - &self, e: Error, room_id: &RoomId, user_id: &UserId, state_lock: &RoomMutexGuard, + &self, + e: Error, + room_id: &RoomId, + user_id: &UserId, + state_lock: &RoomMutexGuard, ) -> Result<()> { error!("Failed to build and append admin room response PDU: \"{e}\""); let content = RoomMessageEventContent::text_plain(format!( - "Failed to build and append admin room PDU: \"{e}\"\n\nThe original admin command may have finished \ - successfully, but we could not return the output." + "Failed to build and append admin room PDU: \"{e}\"\n\nThe original admin command \ + may have finished successfully, but we could not return the output." )); self.services @@ -321,7 +325,8 @@ impl Service { // Admin command with public echo (in admin room) let server_user = &self.services.globals.server_user; - let is_public_prefix = body.starts_with("!admin") || body.starts_with(server_user.as_str()); + let is_public_prefix = + body.starts_with("!admin") || body.starts_with(server_user.as_str()); // Expected backward branch if !is_public_escape && !is_public_prefix { diff --git a/src/service/admin/startup.rs b/src/service/admin/startup.rs index 68ad4be1..582e863d 100644 --- a/src/service/admin/startup.rs +++ b/src/service/admin/startup.rs @@ -65,9 +65,9 @@ async fn startup_execute_command(&self, i: usize, command: String) -> Result<()> debug!("Startup command #{i}: executing {command:?}"); match self.command_in_place(command, None).await { - Ok(Some(output)) => Self::startup_command_output(i, &output), - Err(output) => Self::startup_command_error(i, &output), - Ok(None) => { + | Ok(Some(output)) => Self::startup_command_output(i, &output), + | Err(output) => Self::startup_command_error(i, &output), + | Ok(None) => { info!("Startup command #{i} completed (no output)."); Ok(()) }, diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index cf2921a7..2a54ee09 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -61,7 +61,11 @@ impl crate::Service for Service { impl Service { /// Registers an appservice and returns the ID to the caller - pub async fn register_appservice(&self, registration: &Registration, appservice_config_body: &str) -> Result { + pub async fn register_appservice( + &self, + registration: &Registration, + appservice_config_body: &str, + ) -> Result { //TODO: Check for collisions between exclusive appservice namespaces self.registration_info .write() @@ -152,7 +156,10 @@ impl Service { .any(|info| info.rooms.is_exclusive_match(room_id.as_str())) } - pub fn read(&self) -> impl Future>> { + pub fn read( + &self, + ) -> impl Future>> + { self.registration_info.read() } diff --git a/src/service/appservice/registration_info.rs b/src/service/appservice/registration_info.rs index 612933be..9758e186 100644 --- a/src/service/appservice/registration_info.rs +++ b/src/service/appservice/registration_info.rs @@ -15,13 +15,15 @@ pub struct RegistrationInfo { impl RegistrationInfo { #[must_use] pub fn is_user_match(&self, user_id: &UserId) -> bool { - self.users.is_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() + self.users.is_match(user_id.as_str()) + || self.registration.sender_localpart == user_id.localpart() } #[inline] #[must_use] pub fn is_exclusive_user_match(&self, user_id: &UserId) -> bool { - self.users.is_exclusive_match(user_id.as_str()) || self.registration.sender_localpart == user_id.localpart() + self.users.is_exclusive_match(user_id.as_str()) + || self.registration.sender_localpart == user_id.localpart() } } diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index f2bc8256..f63d78b8 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -42,7 +42,9 @@ impl crate::Service for Service { .build()?, url_preview: base(config) - .and_then(|builder| builder_interface(builder, url_preview_bind_iface.as_deref()))? + .and_then(|builder| { + builder_interface(builder, url_preview_bind_iface.as_deref()) + })? .local_address(url_preview_bind_addr) .dns_resolver(resolver.resolver.clone()) .redirect(redirect::Policy::limited(3)) @@ -178,7 +180,10 @@ fn base(config: &Config) -> Result { } #[cfg(any(target_os = "android", target_os = "fuchsia", target_os = "linux"))] -fn builder_interface(builder: reqwest::ClientBuilder, config: Option<&str>) -> Result { +fn builder_interface( + builder: reqwest::ClientBuilder, + config: Option<&str>, +) -> Result { if let Some(iface) = config { Ok(builder.interface(iface)) } else { @@ -187,7 +192,10 @@ fn builder_interface(builder: reqwest::ClientBuilder, config: Option<&str>) -> R } #[cfg(not(any(target_os = "android", target_os = "fuchsia", target_os = "linux")))] -fn builder_interface(builder: reqwest::ClientBuilder, config: Option<&str>) -> Result { +fn builder_interface( + builder: reqwest::ClientBuilder, + config: Option<&str>, +) -> Result { use conduwuit::Err; if let Some(iface) = config { diff --git a/src/service/emergency/mod.rs b/src/service/emergency/mod.rs index 5063fbd4..9b2e4025 100644 --- a/src/service/emergency/mod.rs +++ b/src/service/emergency/mod.rs @@ -3,7 +3,9 @@ use std::sync::Arc; use async_trait::async_trait; use conduwuit::{error, warn, Result}; use ruma::{ - events::{push_rules::PushRulesEventContent, GlobalAccountDataEvent, GlobalAccountDataEventType}, + events::{ + push_rules::PushRulesEventContent, GlobalAccountDataEvent, GlobalAccountDataEventType, + }, push::Ruleset, }; @@ -31,16 +33,14 @@ impl crate::Service for Service { })) } - async fn worker(self: Arc) -> Result<()> { + async fn worker(self: Arc) -> Result { if self.services.globals.is_read_only() { return Ok(()); } - self.set_emergency_access() - .await - .inspect_err(|e| error!("Could not set the configured emergency password for the server user: {e}"))?; - - Ok(()) + self.set_emergency_access().await.inspect_err(|e| { + error!("Could not set the configured emergency password for the server user: {e}"); + }) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } @@ -49,7 +49,7 @@ impl crate::Service for Service { impl Service { /// Sets the emergency password and push rules for the server user account /// in case emergency password is set - async fn set_emergency_access(&self) -> Result { + async fn set_emergency_access(&self) -> Result { let server_user = &self.services.globals.server_user; self.services @@ -57,8 +57,8 @@ impl Service { .set_password(server_user, self.services.globals.emergency_password().as_deref())?; let (ruleset, pwd_set) = match self.services.globals.emergency_password() { - Some(_) => (Ruleset::server_default(server_user), true), - None => (Ruleset::new(), false), + | Some(_) => (Ruleset::server_default(server_user), true), + | None => (Ruleset::new(), false), }; self.services @@ -68,9 +68,7 @@ impl Service { server_user, GlobalAccountDataEventType::PushRules.to_string().into(), &serde_json::to_value(&GlobalAccountDataEvent { - content: PushRulesEventContent { - global: ruleset, - }, + content: PushRulesEventContent { global: ruleset }, }) .expect("to json value always works"), ) @@ -78,14 +76,14 @@ impl Service { if pwd_set { warn!( - "The server account emergency password is set! Please unset it as soon as you finish admin account \ - recovery! You will be logged out of the server service account when you finish." + "The server account emergency password is set! Please unset it as soon as you \ + finish admin account recovery! You will be logged out of the server service \ + account when you finish." ); + Ok(()) } else { // logs out any users still in the server service account and removes sessions - self.services.users.deactivate_account(server_user).await?; + self.services.users.deactivate_account(server_user).await } - - Ok(pwd_set) } } diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 5edd1f30..07b4ac2c 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -16,7 +16,9 @@ impl Data { let db = &args.db; Self { global: db["global"].clone(), - counter: RwLock::new(Self::stored_count(&db["global"]).expect("initialized global counter")), + counter: RwLock::new( + Self::stored_count(&db["global"]).expect("initialized global counter"), + ), db: args.db.clone(), } } diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 2403b703..88199f5f 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -10,7 +10,9 @@ use std::{ use conduwuit::{error, Config, Result}; use data::Data; use regex::RegexSet; -use ruma::{OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomAliasId, ServerName, UserId}; +use ruma::{ + OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomAliasId, ServerName, UserId, +}; use tokio::sync::Mutex; use crate::service; @@ -40,31 +42,31 @@ impl crate::Service for Service { .as_ref() .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes())); - let turn_secret = config - .turn_secret_file - .as_ref() - .map_or(config.turn_secret.clone(), |path| { - std::fs::read_to_string(path).unwrap_or_else(|e| { - error!("Failed to read the TURN secret file: {e}"); - - config.turn_secret.clone() - }) - }); - - let registration_token = + let turn_secret = config - .registration_token_file + .turn_secret_file .as_ref() - .map_or(config.registration_token.clone(), |path| { - let Ok(token) = std::fs::read_to_string(path).inspect_err(|e| { - error!("Failed to read the registration token file: {e}"); - }) else { - return config.registration_token.clone(); - }; + .map_or(config.turn_secret.clone(), |path| { + std::fs::read_to_string(path).unwrap_or_else(|e| { + error!("Failed to read the TURN secret file: {e}"); - Some(token) + config.turn_secret.clone() + }) }); + let registration_token = config.registration_token_file.as_ref().map_or( + config.registration_token.clone(), + |path| { + let Ok(token) = std::fs::read_to_string(path).inspect_err(|e| { + error!("Failed to read the registration token file: {e}"); + }) else { + return config.registration_token.clone(); + }; + + Some(token) + }, + ); + let mut s = Self { db, config: config.clone(), @@ -73,8 +75,11 @@ impl crate::Service for Service { stateres_mutex: Arc::new(Mutex::new(())), admin_alias: RoomAliasId::parse(format!("#admins:{}", &config.server_name)) .expect("#admins:server_name is valid alias name"), - server_user: UserId::parse_with_server_name(String::from("conduit"), &config.server_name) - .expect("@conduit:server_name is valid"), + server_user: UserId::parse_with_server_name( + String::from("conduit"), + &config.server_name, + ) + .expect("@conduit:server_name is valid"), turn_secret, registration_token, }; @@ -125,7 +130,9 @@ impl Service { pub fn allow_guest_registration(&self) -> bool { self.config.allow_guest_registration } - pub fn allow_guests_auto_join_rooms(&self) -> bool { self.config.allow_guests_auto_join_rooms } + pub fn allow_guests_auto_join_rooms(&self) -> bool { + self.config.allow_guests_auto_join_rooms + } pub fn log_guest_registrations(&self) -> bool { self.config.log_guest_registrations } @@ -137,17 +144,23 @@ impl Service { self.config.allow_public_room_directory_over_federation } - pub fn allow_device_name_federation(&self) -> bool { self.config.allow_device_name_federation } + pub fn allow_device_name_federation(&self) -> bool { + self.config.allow_device_name_federation + } pub fn allow_room_creation(&self) -> bool { self.config.allow_room_creation } - pub fn new_user_displayname_suffix(&self) -> &String { &self.config.new_user_displayname_suffix } + pub fn new_user_displayname_suffix(&self) -> &String { + &self.config.new_user_displayname_suffix + } pub fn allow_check_for_updates(&self) -> bool { self.config.allow_check_for_updates } pub fn trusted_servers(&self) -> &[OwnedServerName] { &self.config.trusted_servers } - pub fn jwt_decoding_key(&self) -> Option<&jsonwebtoken::DecodingKey> { self.jwt_decoding_key.as_ref() } + pub fn jwt_decoding_key(&self) -> Option<&jsonwebtoken::DecodingKey> { + self.jwt_decoding_key.as_ref() + } pub fn turn_password(&self) -> &String { &self.config.turn_password } @@ -173,11 +186,15 @@ impl Service { &self.config.url_preview_domain_explicit_denylist } - pub fn url_preview_url_contains_allowlist(&self) -> &Vec { &self.config.url_preview_url_contains_allowlist } + pub fn url_preview_url_contains_allowlist(&self) -> &Vec { + &self.config.url_preview_url_contains_allowlist + } pub fn url_preview_max_spider_size(&self) -> usize { self.config.url_preview_max_spider_size } - pub fn url_preview_check_root_domain(&self) -> bool { self.config.url_preview_check_root_domain } + pub fn url_preview_check_root_domain(&self) -> bool { + self.config.url_preview_check_root_domain + } pub fn forbidden_alias_names(&self) -> &RegexSet { &self.config.forbidden_alias_names } @@ -189,18 +206,26 @@ impl Service { pub fn allow_outgoing_presence(&self) -> bool { self.config.allow_outgoing_presence } - pub fn allow_incoming_read_receipts(&self) -> bool { self.config.allow_incoming_read_receipts } + pub fn allow_incoming_read_receipts(&self) -> bool { + self.config.allow_incoming_read_receipts + } - pub fn allow_outgoing_read_receipts(&self) -> bool { self.config.allow_outgoing_read_receipts } + pub fn allow_outgoing_read_receipts(&self) -> bool { + self.config.allow_outgoing_read_receipts + } pub fn block_non_admin_invites(&self) -> bool { self.config.block_non_admin_invites } /// checks if `user_id` is local to us via server_name comparison #[inline] - pub fn user_is_local(&self, user_id: &UserId) -> bool { self.server_is_ours(user_id.server_name()) } + pub fn user_is_local(&self, user_id: &UserId) -> bool { + self.server_is_ours(user_id.server_name()) + } #[inline] - pub fn server_is_ours(&self, server_name: &ServerName) -> bool { server_name == self.config.server_name } + pub fn server_is_ours(&self, server_name: &ServerName) -> bool { + server_name == self.config.server_name + } #[inline] pub fn is_read_only(&self) -> bool { self.db.db.is_read_only() } diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 140fc701..1165c3ed 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -48,7 +48,11 @@ impl crate::Service for Service { } #[implement(Service)] -pub fn create_backup(&self, user_id: &UserId, backup_metadata: &Raw) -> Result { +pub fn create_backup( + &self, + user_id: &UserId, + backup_metadata: &Raw, +) -> Result { let version = self.services.globals.next_count()?.to_string(); let count = self.services.globals.next_count()?; @@ -71,13 +75,18 @@ pub async fn delete_backup(&self, user_id: &UserId, version: &str) { .backupkeyid_backup .keys_prefix_raw(&key) .ignore_err() - .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) + .ready_for_each(|outdated_key| { + self.db.backupkeyid_backup.remove(outdated_key); + }) .await; } #[implement(Service)] pub async fn update_backup<'a>( - &self, user_id: &UserId, version: &'a str, backup_metadata: &Raw, + &self, + user_id: &UserId, + version: &'a str, + backup_metadata: &Raw, ) -> Result<&'a str> { let key = (user_id, version); if self.db.backupid_algorithm.qry(&key).await.is_err() { @@ -110,7 +119,10 @@ pub async fn get_latest_backup_version(&self, user_id: &UserId) -> Result Result<(String, Raw)> { +pub async fn get_latest_backup( + &self, + user_id: &UserId, +) -> Result<(String, Raw)> { type Key<'a> = (&'a UserId, &'a str); type KeyVal<'a> = (Key<'a>, Raw); @@ -134,7 +146,12 @@ pub async fn get_backup(&self, user_id: &UserId, version: &str) -> Result, + &self, + user_id: &UserId, + version: &str, + room_id: &RoomId, + session_id: &str, + key_data: &Raw, ) -> Result<()> { let key = (user_id, version); if self.db.backupid_algorithm.qry(&key).await.is_err() { @@ -176,14 +193,16 @@ pub async fn get_etag(&self, user_id: &UserId, version: &str) -> String { } #[implement(Service)] -pub async fn get_all(&self, user_id: &UserId, version: &str) -> BTreeMap { +pub async fn get_all( + &self, + user_id: &UserId, + version: &str, +) -> BTreeMap { type Key<'a> = (Ignore, Ignore, &'a RoomId, &'a str); type KeyVal<'a> = (Key<'a>, Raw); let mut rooms = BTreeMap::::new(); - let default = || RoomKeyBackup { - sessions: BTreeMap::new(), - }; + let default = || RoomKeyBackup { sessions: BTreeMap::new() }; let prefix = (user_id, version, Interfix); self.db @@ -204,7 +223,10 @@ pub async fn get_all(&self, user_id: &UserId, version: &str) -> BTreeMap BTreeMap> { type KeyVal<'a> = ((Ignore, Ignore, Ignore, &'a str), Raw); @@ -213,14 +235,20 @@ pub async fn get_room( .backupkeyid_backup .stream_prefix(&prefix) .ignore_err() - .map(|((.., session_id), key_backup_data): KeyVal<'_>| (session_id.to_owned(), key_backup_data)) + .map(|((.., session_id), key_backup_data): KeyVal<'_>| { + (session_id.to_owned(), key_backup_data) + }) .collect() .await } #[implement(Service)] pub async fn get_session( - &self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str, + &self, + user_id: &UserId, + version: &str, + room_id: &RoomId, + session_id: &str, ) -> Result> { let key = (user_id, version, room_id, session_id); @@ -245,17 +273,27 @@ pub async fn delete_room_keys(&self, user_id: &UserId, version: &str, room_id: & .backupkeyid_backup .keys_prefix_raw(&key) .ignore_err() - .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) + .ready_for_each(|outdated_key| { + self.db.backupkeyid_backup.remove(outdated_key); + }) .await; } #[implement(Service)] -pub async fn delete_room_key(&self, user_id: &UserId, version: &str, room_id: &RoomId, session_id: &str) { +pub async fn delete_room_key( + &self, + user_id: &UserId, + version: &str, + room_id: &RoomId, + session_id: &str, +) { let key = (user_id, version, room_id, session_id); self.db .backupkeyid_backup .keys_prefix_raw(&key) .ignore_err() - .ready_for_each(|outdated_key| self.db.backupkeyid_backup.remove(outdated_key)) + .ready_for_each(|outdated_key| { + self.db.backupkeyid_backup.remove(outdated_key); + }) .await; } diff --git a/src/service/manager.rs b/src/service/manager.rs index 3ce2e0a5..ea33d285 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -102,21 +102,32 @@ impl Manager { unimplemented!("unexpected worker task abort {error:?}"); } - async fn handle_result(&self, workers: &mut WorkersLocked<'_>, result: WorkerResult) -> Result<()> { + async fn handle_result( + &self, + workers: &mut WorkersLocked<'_>, + result: WorkerResult, + ) -> Result<()> { let (service, result) = result; match result { - Ok(()) => self.handle_finished(workers, &service).await, - Err(error) => self.handle_error(workers, &service, error).await, + | Ok(()) => self.handle_finished(workers, &service).await, + | Err(error) => self.handle_error(workers, &service, error).await, } } - async fn handle_finished(&self, _workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + async fn handle_finished( + &self, + _workers: &mut WorkersLocked<'_>, + service: &Arc, + ) -> Result<()> { debug!("service {:?} worker finished", service.name()); Ok(()) } async fn handle_error( - &self, workers: &mut WorkersLocked<'_>, service: &Arc, error: Error, + &self, + workers: &mut WorkersLocked<'_>, + service: &Arc, + error: Error, ) -> Result<()> { let name = service.name(); error!("service {name:?} aborted: {error}"); @@ -138,9 +149,16 @@ impl Manager { } /// Start the worker in a task for the service. - async fn start_worker(&self, workers: &mut WorkersLocked<'_>, service: &Arc) -> Result<()> { + async fn start_worker( + &self, + workers: &mut WorkersLocked<'_>, + service: &Arc, + ) -> Result<()> { if !self.server.running() { - return Err!("Service {:?} worker not starting during server shutdown.", service.name()); + return Err!( + "Service {:?} worker not starting during server shutdown.", + service.name() + ); } debug!("Service {:?} worker starting...", service.name()); diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 71fb1cdb..43310515 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -34,7 +34,11 @@ impl Data { } pub(super) fn create_file_metadata( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, dim: &Dim, content_disposition: Option<&ContentDisposition>, + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + dim: &Dim, + content_disposition: Option<&ContentDisposition>, content_type: Option<&str>, ) -> Result> { let dim: &[u32] = &[dim.width, dim.height]; @@ -63,7 +67,10 @@ impl Data { .stream_prefix_raw(&prefix) .ignore_err() .ready_for_each(|(key, val)| { - debug_assert!(key.starts_with(mxc.to_string().as_bytes()), "key should start with the mxc"); + debug_assert!( + key.starts_with(mxc.to_string().as_bytes()), + "key should start with the mxc" + ); let user = str_from_bytes(val).unwrap_or_default(); debug_info!("Deleting key {key:?} which was uploaded by user {user}"); @@ -95,7 +102,11 @@ impl Data { Ok(keys) } - pub(super) async fn search_file_metadata(&self, mxc: &Mxc<'_>, dim: &Dim) -> Result { + pub(super) async fn search_file_metadata( + &self, + mxc: &Mxc<'_>, + dim: &Dim, + ) -> Result { let dim: &[u32] = &[dim.width, dim.height]; let prefix = (mxc, dim, Interfix); @@ -113,8 +124,9 @@ impl Data { let content_type = parts .next() .map(|bytes| { - string_from_bytes(bytes) - .map_err(|_| Error::bad_database("Content type in mediaid_file is invalid unicode.")) + string_from_bytes(bytes).map_err(|_| { + Error::bad_database("Content type in mediaid_file is invalid unicode.") + }) }) .transpose()?; @@ -127,16 +139,16 @@ impl Data { } else { Some( string_from_bytes(content_disposition_bytes) - .map_err(|_| Error::bad_database("Content Disposition in mediaid_file is invalid unicode."))? + .map_err(|_| { + Error::bad_database( + "Content Disposition in mediaid_file is invalid unicode.", + ) + })? .parse()?, ) }; - Ok(Metadata { - content_disposition, - content_type, - key, - }) + Ok(Metadata { content_disposition, content_type, key }) } /// Gets all the MXCs associated with a user @@ -144,7 +156,9 @@ impl Data { self.mediaid_user .stream() .ignore_err() - .ready_filter_map(|(key, user): (&str, &UserId)| (user == user_id).then(|| key.into())) + .ready_filter_map(|(key, user): (&str, &UserId)| { + (user == user_id).then(|| key.into()) + }) .collect() .await } @@ -166,7 +180,12 @@ impl Data { Ok(()) } - pub(super) fn set_url_preview(&self, url: &str, data: &UrlPreviewData, timestamp: Duration) -> Result<()> { + pub(super) fn set_url_preview( + &self, + url: &str, + data: &UrlPreviewData, + timestamp: Duration, + ) -> Result<()> { let mut value = Vec::::new(); value.extend_from_slice(×tamp.as_secs().to_be_bytes()); value.push(0xFF); @@ -218,43 +237,43 @@ impl Data { .next() .and_then(|b| String::from_utf8(b.to_vec()).ok()) { - Some(s) if s.is_empty() => None, - x => x, + | Some(s) if s.is_empty() => None, + | x => x, }; let description = match values .next() .and_then(|b| String::from_utf8(b.to_vec()).ok()) { - Some(s) if s.is_empty() => None, - x => x, + | Some(s) if s.is_empty() => None, + | x => x, }; let image = match values .next() .and_then(|b| String::from_utf8(b.to_vec()).ok()) { - Some(s) if s.is_empty() => None, - x => x, + | Some(s) if s.is_empty() => None, + | x => x, }; let image_size = match values .next() .map(|b| usize::from_be_bytes(b.try_into().unwrap_or_default())) { - Some(0) => None, - x => x, + | Some(0) => None, + | x => x, }; let image_width = match values .next() .map(|b| u32::from_be_bytes(b.try_into().unwrap_or_default())) { - Some(0) => None, - x => x, + | Some(0) => None, + | x => x, }; let image_height = match values .next() .map(|b| u32::from_be_bytes(b.try_into().unwrap_or_default())) { - Some(0) => None, - x => x, + | Some(0) => None, + | x => x, }; Ok(UrlPreviewData { diff --git a/src/service/media/migrations.rs b/src/service/media/migrations.rs index 5932643b..9555edd7 100644 --- a/src/service/media/migrations.rs +++ b/src/service/media/migrations.rs @@ -83,7 +83,8 @@ pub(crate) async fn checkup_sha256_media(services: &Services) -> Result<()> { for key in media.db.get_all_media_keys().await { let new_path = media.get_media_file_sha256(&key).into_os_string(); let old_path = media.get_media_file_b64(&key).into_os_string(); - if let Err(e) = handle_media_check(&dbs, config, &files, &key, &new_path, &old_path).await { + if let Err(e) = handle_media_check(&dbs, config, &files, &key, &new_path, &old_path).await + { error!( media_id = ?encode_key(&key), ?new_path, ?old_path, "Failed to resolve media check failure: {e}" @@ -100,8 +101,12 @@ pub(crate) async fn checkup_sha256_media(services: &Services) -> Result<()> { } async fn handle_media_check( - dbs: &(&Arc, &Arc), config: &Config, files: &HashSet, key: &[u8], - new_path: &OsStr, old_path: &OsStr, + dbs: &(&Arc, &Arc), + config: &Config, + files: &HashSet, + key: &[u8], + new_path: &OsStr, + old_path: &OsStr, ) -> Result<()> { use crate::media::encode_key; diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 989e1c27..7e77090c 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -80,13 +80,21 @@ impl crate::Service for Service { impl Service { /// Uploads a file. pub async fn create( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, content_disposition: Option<&ContentDisposition>, - content_type: Option<&str>, file: &[u8], + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + content_disposition: Option<&ContentDisposition>, + content_type: Option<&str>, + file: &[u8], ) -> Result<()> { // Width, Height = 0 if it's not a thumbnail - let key = self - .db - .create_file_metadata(mxc, user, &Dim::default(), content_disposition, content_type)?; + let key = self.db.create_file_metadata( + mxc, + user, + &Dim::default(), + content_disposition, + content_type, + )?; //TODO: Dangling metadata in database if creation fails let mut f = self.create_media_file(&key).await?; @@ -132,10 +140,10 @@ impl Service { debug_info!(%deletion_count, "Deleting MXC {mxc} by user {user} from database and filesystem"); match self.delete(&mxc).await { - Ok(()) => { + | Ok(()) => { deletion_count = deletion_count.saturating_add(1); }, - Err(e) => { + | Err(e) => { debug_error!(%deletion_count, "Failed to delete {mxc} from user {user}, ignoring error: {e}"); }, } @@ -146,11 +154,8 @@ impl Service { /// Downloads a file. pub async fn get(&self, mxc: &Mxc<'_>) -> Result> { - if let Ok(Metadata { - content_disposition, - content_type, - key, - }) = self.db.search_file_metadata(mxc, &Dim::default()).await + if let Ok(Metadata { content_disposition, content_type, key }) = + self.db.search_file_metadata(mxc, &Dim::default()).await { let mut content = Vec::new(); let path = self.get_media_file(&key); @@ -181,13 +186,19 @@ impl Service { let mxc = parts .next() .map(|bytes| { - utils::string_from_bytes(bytes) - .map_err(|e| err!(Database(error!("Failed to parse MXC unicode bytes from our database: {e}")))) + utils::string_from_bytes(bytes).map_err(|e| { + err!(Database(error!( + "Failed to parse MXC unicode bytes from our database: {e}" + ))) + }) }) .transpose()?; let Some(mxc_s) = mxc else { - debug_warn!(?mxc, "Parsed MXC URL unicode bytes from database but is still invalid"); + debug_warn!( + ?mxc, + "Parsed MXC URL unicode bytes from database but is still invalid" + ); continue; }; @@ -207,7 +218,11 @@ impl Service { /// Deletes all remote only media files in the given at or after /// time/duration. Returns a usize with the amount of media files deleted. pub async fn delete_all_remote_media_at_after_time( - &self, time: SystemTime, before: bool, after: bool, yes_i_want_to_delete_local_media: bool, + &self, + time: SystemTime, + before: bool, + after: bool, + yes_i_want_to_delete_local_media: bool, ) -> Result { let all_keys = self.db.get_all_media_keys().await; let mut remote_mxcs = Vec::with_capacity(all_keys.len()); @@ -218,19 +233,26 @@ impl Service { let mxc = parts .next() .map(|bytes| { - utils::string_from_bytes(bytes) - .map_err(|e| err!(Database(error!("Failed to parse MXC unicode bytes from our database: {e}")))) + utils::string_from_bytes(bytes).map_err(|e| { + err!(Database(error!( + "Failed to parse MXC unicode bytes from our database: {e}" + ))) + }) }) .transpose()?; let Some(mxc_s) = mxc else { - debug_warn!(?mxc, "Parsed MXC URL unicode bytes from database but is still invalid"); + debug_warn!( + ?mxc, + "Parsed MXC URL unicode bytes from database but is still invalid" + ); continue; }; trace!("Parsed MXC key to URL: {mxc_s}"); let mxc = OwnedMxcUri::from(mxc_s); - if (mxc.server_name() == Ok(self.services.globals.server_name()) && !yes_i_want_to_delete_local_media) + if (mxc.server_name() == Ok(self.services.globals.server_name()) + && !yes_i_want_to_delete_local_media) || !mxc.is_valid() { debug!("Ignoring local or broken media MXC: {mxc}"); @@ -240,9 +262,12 @@ impl Service { let path = self.get_media_file(&key); let file_metadata = match fs::metadata(path.clone()).await { - Ok(file_metadata) => file_metadata, - Err(e) => { - error!("Failed to obtain file metadata for MXC {mxc} at file path \"{path:?}\", skipping: {e}"); + | Ok(file_metadata) => file_metadata, + | Err(e) => { + error!( + "Failed to obtain file metadata for MXC {mxc} at file path \ + \"{path:?}\", skipping: {e}" + ); continue; }, }; @@ -250,12 +275,12 @@ impl Service { trace!(%mxc, ?path, "File metadata: {file_metadata:?}"); let file_created_at = match file_metadata.created() { - Ok(value) => value, - Err(err) if err.kind() == std::io::ErrorKind::Unsupported => { + | Ok(value) => value, + | Err(err) if err.kind() == std::io::ErrorKind::Unsupported => { debug!("btime is unsupported, using mtime instead"); file_metadata.modified()? }, - Err(err) => { + | Err(err) => { error!("Could not delete MXC {mxc} at path {path:?}: {err:?}. Skipping..."); continue; }, @@ -264,10 +289,16 @@ impl Service { debug!("File created at: {file_created_at:?}"); if file_created_at >= time && before { - debug!("File is within (before) user duration, pushing to list of file paths and keys to delete."); + debug!( + "File is within (before) user duration, pushing to list of file paths and \ + keys to delete." + ); remote_mxcs.push(mxc.to_string()); } else if file_created_at <= time && after { - debug!("File is not within (after) user duration, pushing to list of file paths and keys to delete."); + debug!( + "File is not within (after) user duration, pushing to list of file paths \ + and keys to delete." + ); remote_mxcs.push(mxc.to_string()); } } @@ -289,10 +320,10 @@ impl Service { debug_info!("Deleting MXC {mxc} from database and filesystem"); match self.delete(&mxc).await { - Ok(()) => { + | Ok(()) => { deletion_count = deletion_count.saturating_add(1); }, - Err(e) => { + | Err(e) => { warn!("Failed to delete {mxc}, ignoring error and skipping: {e}"); continue; }, diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index d571ac56..b1c53305 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -53,10 +53,10 @@ pub async fn download_image(&self, url: &str) -> Result { self.create(&mxc, None, None, None, &image).await?; let (width, height) = match ImgReader::new(Cursor::new(&image)).with_guessed_format() { - Err(_) => (None, None), - Ok(reader) => match reader.into_dimensions() { - Err(_) => (None, None), - Ok((width, height)) => (Some(width), Some(height)), + | Err(_) => (None, None), + | Ok(reader) => match reader.into_dimensions() { + | Err(_) => (None, None), + | Ok((width, height)) => (Some(width), Some(height)), }, }; @@ -79,8 +79,8 @@ pub async fn get_url_preview(&self, url: &Url) -> Result { let _request_lock = self.url_preview_mutex.lock(url.as_str()).await; match self.db.get_url_preview(url.as_str()).await { - Ok(preview) => Ok(preview), - Err(_) => self.request_url_preview(url).await, + | Ok(preview) => Ok(preview), + | Err(_) => self.request_url_preview(url).await, } } @@ -111,9 +111,9 @@ async fn request_url_preview(&self, url: &Url) -> Result { return Err!(Request(Unknown("Unknown Content-Type"))); }; let data = match content_type { - html if html.starts_with("text/html") => self.download_html(url.as_str()).await?, - img if img.starts_with("image/") => self.download_image(url.as_str()).await?, - _ => return Err!(Request(Unknown("Unsupported Content-Type"))), + | html if html.starts_with("text/html") => self.download_html(url.as_str()).await?, + | img if img.starts_with("image/") => self.download_image(url.as_str()).await?, + | _ => return Err!(Request(Unknown("Unsupported Content-Type"))), }; self.set_url_preview(url.as_str(), &data).await?; @@ -131,8 +131,9 @@ async fn download_html(&self, url: &str) -> Result { bytes.extend_from_slice(&chunk); if bytes.len() > self.services.globals.url_preview_max_spider_size() { debug!( - "Response body from URL {} exceeds url_preview_max_spider_size ({}), not processing the rest of the \ - response body and assuming our necessary data is in this range.", + "Response body from URL {} exceeds url_preview_max_spider_size ({}), not \ + processing the rest of the response body and assuming our necessary data is in \ + this range.", url, self.services.globals.url_preview_max_spider_size() ); @@ -145,8 +146,8 @@ async fn download_html(&self, url: &str) -> Result { }; let mut data = match html.opengraph.images.first() { - None => UrlPreviewData::default(), - Some(obj) => self.download_image(&obj.url).await?, + | None => UrlPreviewData::default(), + | Some(obj) => self.download_image(&obj.url).await?, }; let props = html.opengraph.properties; @@ -169,11 +170,11 @@ pub fn url_preview_allowed(&self, url: &Url) -> bool { } let host = match url.host_str() { - None => { + | None => { debug!("Ignoring URL preview for a URL that does not have a host (?): {}", url); return false; }, - Some(h) => h.to_owned(), + | Some(h) => h.to_owned(), }; let allowlist_domain_contains = self @@ -205,7 +206,10 @@ pub fn url_preview_allowed(&self, url: &Url) -> bool { } if allowlist_domain_explicit.contains(&host) { - debug!("Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)", &host); + debug!( + "Host {} is allowed by url_preview_domain_explicit_allowlist (check 2/4)", + &host + ); return true; } @@ -213,7 +217,10 @@ pub fn url_preview_allowed(&self, url: &Url) -> bool { .iter() .any(|domain_s| domain_s.contains(&host.clone())) { - debug!("Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)", &host); + debug!( + "Host {} is allowed by url_preview_domain_contains_allowlist (check 3/4)", + &host + ); return true; } @@ -229,11 +236,12 @@ pub fn url_preview_allowed(&self, url: &Url) -> bool { if self.services.globals.url_preview_check_root_domain() { debug!("Checking root domain"); match host.split_once('.') { - None => return false, - Some((_, root_domain)) => { + | None => return false, + | Some((_, root_domain)) => { if denylist_domain_explicit.contains(&root_domain.to_owned()) { debug!( - "Root domain {} is not allowed by url_preview_domain_explicit_denylist (check 1/3)", + "Root domain {} is not allowed by \ + url_preview_domain_explicit_denylist (check 1/3)", &root_domain ); return true; @@ -241,7 +249,8 @@ pub fn url_preview_allowed(&self, url: &Url) -> bool { if allowlist_domain_explicit.contains(&root_domain.to_owned()) { debug!( - "Root domain {} is allowed by url_preview_domain_explicit_allowlist (check 2/3)", + "Root domain {} is allowed by url_preview_domain_explicit_allowlist \ + (check 2/3)", &root_domain ); return true; @@ -252,7 +261,8 @@ pub fn url_preview_allowed(&self, url: &Url) -> bool { .any(|domain_s| domain_s.contains(&root_domain.to_owned())) { debug!( - "Root domain {} is allowed by url_preview_domain_contains_allowlist (check 3/3)", + "Root domain {} is allowed by url_preview_domain_contains_allowlist \ + (check 3/3)", &root_domain ); return true; diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index 8ec917b7..d5ad5391 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -1,6 +1,9 @@ use std::{fmt::Debug, time::Duration}; -use conduwuit::{debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, Result}; +use conduwuit::{ + debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, + Result, +}; use http::header::{HeaderValue, CONTENT_DISPOSITION, CONTENT_TYPE}; use ruma::{ api::{ @@ -19,7 +22,12 @@ use super::{Dim, FileMeta}; #[implement(super::Service)] pub async fn fetch_remote_thumbnail( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, dim: &Dim, + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, + dim: &Dim, ) -> Result { self.check_fetch_authorized(mxc)?; @@ -38,7 +46,11 @@ pub async fn fetch_remote_thumbnail( #[implement(super::Service)] pub async fn fetch_remote_content( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, ) -> Result { self.check_fetch_authorized(mxc)?; @@ -57,7 +69,12 @@ pub async fn fetch_remote_content( #[implement(super::Service)] async fn fetch_thumbnail_authenticated( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, dim: &Dim, + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, + dim: &Dim, ) -> Result { use federation::authenticated_media::get_content_thumbnail::v1::{Request, Response}; @@ -70,20 +87,22 @@ async fn fetch_thumbnail_authenticated( timeout_ms, }; - let Response { - content, - .. - } = self.federation_request(mxc, user, server, request).await?; + let Response { content, .. } = self.federation_request(mxc, user, server, request).await?; match content { - FileOrLocation::File(content) => self.handle_thumbnail_file(mxc, user, dim, content).await, - FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await, + | FileOrLocation::File(content) => + self.handle_thumbnail_file(mxc, user, dim, content).await, + | FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await, } } #[implement(super::Service)] async fn fetch_content_authenticated( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, ) -> Result { use federation::authenticated_media::get_content::v1::{Request, Response}; @@ -92,21 +111,23 @@ async fn fetch_content_authenticated( timeout_ms, }; - let Response { - content, - .. - } = self.federation_request(mxc, user, server, request).await?; + let Response { content, .. } = self.federation_request(mxc, user, server, request).await?; match content { - FileOrLocation::File(content) => self.handle_content_file(mxc, user, content).await, - FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await, + | FileOrLocation::File(content) => self.handle_content_file(mxc, user, content).await, + | FileOrLocation::Location(location) => self.handle_location(mxc, user, &location).await, } } #[allow(deprecated)] #[implement(super::Service)] async fn fetch_thumbnail_unauthenticated( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, dim: &Dim, + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, + dim: &Dim, ) -> Result { use media::get_content_thumbnail::v3::{Request, Response}; @@ -123,17 +144,10 @@ async fn fetch_thumbnail_unauthenticated( }; let Response { - file, - content_type, - content_disposition, - .. + file, content_type, content_disposition, .. } = self.federation_request(mxc, user, server, request).await?; - let content = Content { - file, - content_type, - content_disposition, - }; + let content = Content { file, content_type, content_disposition }; self.handle_thumbnail_file(mxc, user, dim, content).await } @@ -141,7 +155,11 @@ async fn fetch_thumbnail_unauthenticated( #[allow(deprecated)] #[implement(super::Service)] async fn fetch_content_unauthenticated( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, timeout_ms: Duration, + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + timeout_ms: Duration, ) -> Result { use media::get_content::v3::{Request, Response}; @@ -154,27 +172,27 @@ async fn fetch_content_unauthenticated( }; let Response { - file, - content_type, - content_disposition, - .. + file, content_type, content_disposition, .. } = self.federation_request(mxc, user, server, request).await?; - let content = Content { - file, - content_type, - content_disposition, - }; + let content = Content { file, content_type, content_disposition }; self.handle_content_file(mxc, user, content).await } #[implement(super::Service)] async fn handle_thumbnail_file( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, dim: &Dim, content: Content, + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + dim: &Dim, + content: Content, ) -> Result { - let content_disposition = - make_content_disposition(content.content_disposition.as_ref(), content.content_type.as_deref(), None); + let content_disposition = make_content_disposition( + content.content_disposition.as_ref(), + content.content_type.as_deref(), + None, + ); self.upload_thumbnail( mxc, @@ -193,9 +211,17 @@ async fn handle_thumbnail_file( } #[implement(super::Service)] -async fn handle_content_file(&self, mxc: &Mxc<'_>, user: Option<&UserId>, content: Content) -> Result { - let content_disposition = - make_content_disposition(content.content_disposition.as_ref(), content.content_type.as_deref(), None); +async fn handle_content_file( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + content: Content, +) -> Result { + let content_disposition = make_content_disposition( + content.content_disposition.as_ref(), + content.content_type.as_deref(), + None, + ); self.create( mxc, @@ -213,7 +239,12 @@ async fn handle_content_file(&self, mxc: &Mxc<'_>, user: Option<&UserId>, conten } #[implement(super::Service)] -async fn handle_location(&self, mxc: &Mxc<'_>, user: Option<&UserId>, location: &str) -> Result { +async fn handle_location( + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + location: &str, +) -> Result { self.location_request(location).await.map_err(|error| { err!(Request(NotFound( debug_warn!(%mxc, ?user, ?location, ?error, "Fetching media from location failed") @@ -263,7 +294,11 @@ async fn location_request(&self, location: &str) -> Result { #[implement(super::Service)] async fn federation_request( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, request: Request, + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + request: Request, ) -> Result where Request: OutgoingRequest + Send + Debug, @@ -277,7 +312,12 @@ where // Handles and adjusts the error for the caller to determine if they should // request the fallback endpoint or give up. -fn handle_federation_error(mxc: &Mxc<'_>, user: Option<&UserId>, server: Option<&ServerName>, error: Error) -> Error { +fn handle_federation_error( + mxc: &Mxc<'_>, + user: Option<&UserId>, + server: Option<&ServerName>, + error: Error, +) -> Error { let fallback = || { err!(Request(NotFound( debug_error!(%mxc, ?user, ?server, ?error, "Remote media not found") @@ -303,7 +343,8 @@ fn handle_federation_error(mxc: &Mxc<'_>, user: Option<&UserId>, server: Option< #[implement(super::Service)] #[allow(deprecated)] pub async fn fetch_remote_thumbnail_legacy( - &self, body: &media::get_content_thumbnail::v3::Request, + &self, + body: &media::get_content_thumbnail::v3::Request, ) -> Result { let mxc = Mxc { server_name: &body.server_name, @@ -315,20 +356,17 @@ pub async fn fetch_remote_thumbnail_legacy( let reponse = self .services .sending - .send_federation_request( - mxc.server_name, - media::get_content_thumbnail::v3::Request { - allow_remote: body.allow_remote, - height: body.height, - width: body.width, - method: body.method.clone(), - server_name: body.server_name.clone(), - media_id: body.media_id.clone(), - timeout_ms: body.timeout_ms, - allow_redirect: body.allow_redirect, - animated: body.animated, - }, - ) + .send_federation_request(mxc.server_name, media::get_content_thumbnail::v3::Request { + allow_remote: body.allow_remote, + height: body.height, + width: body.width, + method: body.method.clone(), + server_name: body.server_name.clone(), + media_id: body.media_id.clone(), + timeout_ms: body.timeout_ms, + allow_redirect: body.allow_redirect, + animated: body.animated, + }) .await?; let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; @@ -341,27 +379,30 @@ pub async fn fetch_remote_thumbnail_legacy( #[implement(super::Service)] #[allow(deprecated)] pub async fn fetch_remote_content_legacy( - &self, mxc: &Mxc<'_>, allow_redirect: bool, timeout_ms: Duration, + &self, + mxc: &Mxc<'_>, + allow_redirect: bool, + timeout_ms: Duration, ) -> Result { self.check_legacy_freeze()?; self.check_fetch_authorized(mxc)?; let response = self .services .sending - .send_federation_request( - mxc.server_name, - media::get_content::v3::Request { - allow_remote: true, - server_name: mxc.server_name.into(), - media_id: mxc.media_id.into(), - timeout_ms, - allow_redirect, - }, - ) + .send_federation_request(mxc.server_name, media::get_content::v3::Request { + allow_remote: true, + server_name: mxc.server_name.into(), + media_id: mxc.media_id.into(), + timeout_ms, + allow_redirect, + }) .await?; - let content_disposition = - make_content_disposition(response.content_disposition.as_ref(), response.content_type.as_deref(), None); + let content_disposition = make_content_disposition( + response.content_disposition.as_ref(), + response.content_type.as_deref(), + None, + ); self.create( mxc, diff --git a/src/service/media/tests.rs b/src/service/media/tests.rs index b2f31e6f..1d6dce30 100644 --- a/src/service/media/tests.rs +++ b/src/service/media/tests.rs @@ -13,7 +13,12 @@ async fn long_file_names_works() { impl Data for MockedKVDatabase { fn create_file_metadata( - &self, _sender_user: Option<&str>, mxc: String, width: u32, height: u32, content_disposition: Option<&str>, + &self, + _sender_user: Option<&str>, + mxc: String, + width: u32, + height: u32, + content_disposition: Option<&str>, content_type: Option<&str>, ) -> Result> { // copied from src/database/key_value/media.rs @@ -46,14 +51,22 @@ async fn long_file_names_works() { fn get_all_media_keys(&self) -> Vec> { todo!() } fn search_file_metadata( - &self, _mxc: String, _width: u32, _height: u32, + &self, + _mxc: String, + _width: u32, + _height: u32, ) -> Result<(Option, Option, Vec)> { todo!() } fn remove_url_preview(&self, _url: &str) -> Result<()> { todo!() } - fn set_url_preview(&self, _url: &str, _data: &UrlPreviewData, _timestamp: std::time::Duration) -> Result<()> { + fn set_url_preview( + &self, + _url: &str, + _data: &UrlPreviewData, + _timestamp: std::time::Duration, + ) -> Result<()> { todo!() } @@ -64,11 +77,18 @@ async fn long_file_names_works() { let mxc = "mxc://example.com/ascERGshawAWawugaAcauga".to_owned(); let width = 100; let height = 100; - let content_disposition = "attachment; filename=\"this is a very long file name with spaces and special \ - characters like äöüß and even emoji like 🦀.png\""; + let content_disposition = "attachment; filename=\"this is a very long file name with spaces \ + and special characters like äöüß and even emoji like 🦀.png\""; let content_type = "image/png"; let key = db - .create_file_metadata(None, mxc, width, height, Some(content_disposition), Some(content_type)) + .create_file_metadata( + None, + mxc, + width, + height, + Some(content_disposition), + Some(content_type), + ) .unwrap(); let mut r = PathBuf::from("/tmp/media"); // r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD)); diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 42fc40e7..5c8063cb 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -22,12 +22,17 @@ impl super::Service { /// Uploads or replaces a file thumbnail. #[allow(clippy::too_many_arguments)] pub async fn upload_thumbnail( - &self, mxc: &Mxc<'_>, user: Option<&UserId>, content_disposition: Option<&ContentDisposition>, - content_type: Option<&str>, dim: &Dim, file: &[u8], + &self, + mxc: &Mxc<'_>, + user: Option<&UserId>, + content_disposition: Option<&ContentDisposition>, + content_type: Option<&str>, + dim: &Dim, + file: &[u8], ) -> Result<()> { - let key = self - .db - .create_file_metadata(mxc, user, dim, content_disposition, content_type)?; + let key = + self.db + .create_file_metadata(mxc, user, dim, content_disposition, content_type)?; //TODO: Dangling metadata in database if creation fails let mut f = self.create_media_file(&key).await?; @@ -78,7 +83,12 @@ impl super::Service { /// Generate a thumbnail #[tracing::instrument(skip(self), name = "generate", level = "debug")] - async fn get_thumbnail_generate(&self, mxc: &Mxc<'_>, dim: &Dim, data: Metadata) -> Result> { + async fn get_thumbnail_generate( + &self, + mxc: &Mxc<'_>, + dim: &Dim, + data: Metadata, + ) -> Result> { let mut content = Vec::new(); let path = self.get_media_file(&data.key); fs::File::open(path) @@ -117,11 +127,7 @@ impl super::Service { fn thumbnail_generate(image: &DynamicImage, requested: &Dim) -> Result { let thumbnail = if !requested.crop() { - let Dim { - width, - height, - .. - } = requested.scaled(&Dim { + let Dim { width, height, .. } = requested.scaled(&Dim { width: image.width(), height: image.height(), ..Dim::default() @@ -202,12 +208,12 @@ impl Dim { #[must_use] pub fn normalized(&self) -> Self { match (self.width, self.height) { - (0..=32, 0..=32) => Self::new(32, 32, Some(Method::Crop)), - (0..=96, 0..=96) => Self::new(96, 96, Some(Method::Crop)), - (0..=320, 0..=240) => Self::new(320, 240, Some(Method::Scale)), - (0..=640, 0..=480) => Self::new(640, 480, Some(Method::Scale)), - (0..=800, 0..=600) => Self::new(800, 600, Some(Method::Scale)), - _ => Self::default(), + | (0..=32, 0..=32) => Self::new(32, 32, Some(Method::Crop)), + | (0..=96, 0..=96) => Self::new(96, 96, Some(Method::Crop)), + | (0..=320, 0..=240) => Self::new(320, 240, Some(Method::Scale)), + | (0..=640, 0..=480) => Self::new(640, 480, Some(Method::Scale)), + | (0..=800, 0..=600) => Self::new(800, 600, Some(Method::Scale)), + | _ => Self::default(), } } diff --git a/src/service/migrations.rs b/src/service/migrations.rs index 102ac7d4..adf75c0b 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -12,7 +12,9 @@ use conduwuit::{ use futures::{FutureExt, StreamExt}; use itertools::Itertools; use ruma::{ - events::{push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType}, + events::{ + push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType, + }, push::Ruleset, OwnedUserId, RoomId, UserId, }; @@ -45,7 +47,8 @@ pub(crate) async fn migrations(services: &Services) -> Result<()> { if !services.users.exists(server_user).await { error!("The {server_user} server user does not exist, and the database is not new."); return Err!(Database( - "Cannot reuse an existing database after changing the server name, please delete the old one first.", + "Cannot reuse an existing database after changing the server name, please \ + delete the old one first.", )); } } @@ -144,7 +147,8 @@ async fn migrate(services: &Services) -> Result<()> { assert!( version_match, - "Failed asserting local database version {} is equal to known latest conduwuit database version {}", + "Failed asserting local database version {} is equal to known latest conduwuit database \ + version {}", services.globals.db.database_version().await, DATABASE_VERSION, ); @@ -192,7 +196,8 @@ async fn migrate(services: &Services) -> Result<()> { let matches = patterns.matches(room_alias.alias()); if matches.matched_any() { warn!( - "Room with alias {} ({}) matches the following forbidden room name patterns: {}", + "Room with alias {} ({}) matches the following forbidden room \ + name patterns: {}", room_alias, &room_id, matches @@ -223,8 +228,8 @@ async fn db_lt_12(services: &Services) -> Result<()> { .await { let user = match UserId::parse_with_server_name(username.as_str(), &config.server_name) { - Ok(u) => u, - Err(e) => { + | Ok(u) => u, + | Err(e) => { warn!("Invalid username {username}: {e}"); continue; }, @@ -240,7 +245,8 @@ async fn db_lt_12(services: &Services) -> Result<()> { //content rule { - let content_rule_transformation = [".m.rules.contains_user_name", ".m.rule.contains_user_name"]; + let content_rule_transformation = + [".m.rules.contains_user_name", ".m.rule.contains_user_name"]; let rule = rules_list.content.get(content_rule_transformation[0]); if rule.is_some() { @@ -301,8 +307,8 @@ async fn db_lt_13(services: &Services) -> Result<()> { .await { let user = match UserId::parse_with_server_name(username.as_str(), &config.server_name) { - Ok(u) => u, - Err(e) => { + | Ok(u) => u, + | Err(e) => { warn!("Invalid username {username}: {e}"); continue; }, @@ -413,7 +419,9 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) .rooms .state_accessor .get_member(room_id, user_id) - .map(|member| member.is_ok_and(|member| member.membership == MembershipState::Join)) + .map(|member| { + member.is_ok_and(|member| member.membership == MembershipState::Join) + }) }) .collect::>() .await; @@ -426,7 +434,9 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) .rooms .state_accessor .get_member(room_id, user_id) - .map(|member| member.is_ok_and(|member| member.membership == MembershipState::Join)) + .map(|member| { + member.is_ok_and(|member| member.membership == MembershipState::Join) + }) }) .collect::>() .await; @@ -444,7 +454,8 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) for room_id in &room_ids { debug_info!( - "Updating joined count for room {room_id} to fix servers in room after correcting membership states" + "Updating joined count for room {room_id} to fix servers in room after correcting \ + membership states" ); services diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index dd0d8e72..3d614333 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -53,18 +53,22 @@ impl Data { } pub(super) async fn set_presence( - &self, user_id: &UserId, presence_state: &PresenceState, currently_active: Option, - last_active_ago: Option, status_msg: Option, + &self, + user_id: &UserId, + presence_state: &PresenceState, + currently_active: Option, + last_active_ago: Option, + status_msg: Option, ) -> Result<()> { let last_presence = self.get_presence(user_id).await; let state_changed = match last_presence { - Err(_) => true, - Ok(ref presence) => presence.1.content.presence != *presence_state, + | Err(_) => true, + | Ok(ref presence) => presence.1.content.presence != *presence_state, }; let status_msg_changed = match last_presence { - Err(_) => true, - Ok(ref last_presence) => { + | Err(_) => true, + | Ok(ref last_presence) => { let old_msg = last_presence .1 .content @@ -80,18 +84,22 @@ impl Data { let now = utils::millis_since_unix_epoch(); let last_last_active_ts = match last_presence { - Err(_) => 0, - Ok((_, ref presence)) => now.saturating_sub(presence.content.last_active_ago.unwrap_or_default().into()), + | Err(_) => 0, + | Ok((_, ref presence)) => + now.saturating_sub(presence.content.last_active_ago.unwrap_or_default().into()), }; let last_active_ts = match last_active_ago { - None => now, - Some(last_active_ago) => now.saturating_sub(last_active_ago.into()), + | None => now, + | Some(last_active_ago) => now.saturating_sub(last_active_ago.into()), }; // TODO: tighten for state flicker? if !status_msg_changed && !state_changed && last_active_ts < last_last_active_ts { - debug_warn!("presence spam {user_id:?} last_active_ts:{last_active_ts:?} < {last_last_active_ts:?}",); + debug_warn!( + "presence spam {user_id:?} last_active_ts:{last_active_ts:?} < \ + {last_last_active_ts:?}", + ); return Ok(()); } @@ -138,7 +146,10 @@ impl Data { } #[inline] - pub(super) fn presence_since(&self, since: u64) -> impl Stream + Send + '_ { + pub(super) fn presence_since( + &self, + since: u64, + ) -> impl Stream + Send + '_ { self.presenceid_presence .raw_stream() .ignore_err() diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 7e80e05e..1f9f63d9 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -99,13 +99,14 @@ impl Service { let last_presence = self.db.get_presence(user_id).await; let state_changed = match last_presence { - Err(_) => true, - Ok((_, ref presence)) => presence.content.presence != *new_state, + | Err(_) => true, + | Ok((_, ref presence)) => presence.content.presence != *new_state, }; let last_last_active_ago = match last_presence { - Err(_) => 0_u64, - Ok((_, ref presence)) => presence.content.last_active_ago.unwrap_or_default().into(), + | Err(_) => 0_u64, + | Ok((_, ref presence)) => + presence.content.last_active_ago.unwrap_or_default().into(), }; if !state_changed && last_last_active_ago < REFRESH_TIMEOUT { @@ -113,8 +114,8 @@ impl Service { } let status_msg = match last_presence { - Ok((_, ref presence)) => presence.content.status_msg.clone(), - Err(_) => Some(String::new()), + | Ok((_, ref presence)) => presence.content.status_msg.clone(), + | Err(_) => Some(String::new()), }; let last_active_ago = UInt::new(0); @@ -125,12 +126,16 @@ impl Service { /// Adds a presence event which will be saved until a new event replaces it. pub async fn set_presence( - &self, user_id: &UserId, state: &PresenceState, currently_active: Option, last_active_ago: Option, + &self, + user_id: &UserId, + state: &PresenceState, + currently_active: Option, + last_active_ago: Option, status_msg: Option, ) -> Result<()> { let presence_state = match state.as_str() { - "" => &PresenceState::Offline, // default an empty string to 'offline' - &_ => state, + | "" => &PresenceState::Offline, // default an empty string to 'offline' + | &_ => state, }; self.db @@ -141,8 +146,8 @@ impl Service { && user_id != self.services.globals.server_user { let timeout = match presence_state { - PresenceState::Online => self.services.server.config.presence_idle_timeout_s, - _ => self.services.server.config.presence_offline_timeout_s, + | PresenceState::Online => self.services.server.config.presence_idle_timeout_s, + | _ => self.services.server.config.presence_offline_timeout_s, }; self.timer_sender @@ -160,16 +165,25 @@ impl Service { /// /// TODO: Why is this not used? #[allow(dead_code)] - pub async fn remove_presence(&self, user_id: &UserId) { self.db.remove_presence(user_id).await } + pub async fn remove_presence(&self, user_id: &UserId) { + self.db.remove_presence(user_id).await; + } /// Returns the most recent presence updates that happened after the event /// with id `since`. - pub fn presence_since(&self, since: u64) -> impl Stream + Send + '_ { + pub fn presence_since( + &self, + since: u64, + ) -> impl Stream + Send + '_ { self.db.presence_since(since) } #[inline] - pub async fn from_json_bytes_to_event(&self, bytes: &[u8], user_id: &UserId) -> Result { + pub async fn from_json_bytes_to_event( + &self, + bytes: &[u8], + user_id: &UserId, + ) -> Result { let presence = Presence::from_json_bytes(bytes)?; let event = presence .to_presence_event(user_id, &self.services.users) @@ -192,13 +206,16 @@ impl Service { } let new_state = match (&presence_state, last_active_ago.map(u64::from)) { - (PresenceState::Online, Some(ago)) if ago >= self.idle_timeout => Some(PresenceState::Unavailable), - (PresenceState::Unavailable, Some(ago)) if ago >= self.offline_timeout => Some(PresenceState::Offline), - _ => None, + | (PresenceState::Online, Some(ago)) if ago >= self.idle_timeout => + Some(PresenceState::Unavailable), + | (PresenceState::Unavailable, Some(ago)) if ago >= self.offline_timeout => + Some(PresenceState::Offline), + | _ => None, }; debug!( - "Processed presence timer for user '{user_id}': Old state = {presence_state}, New state = {new_state:?}" + "Processed presence timer for user '{user_id}': Old state = {presence_state}, New \ + state = {new_state:?}" ); if let Some(new_state) = new_state { diff --git a/src/service/presence/presence.rs b/src/service/presence/presence.rs index aed4a3f8..b88a004b 100644 --- a/src/service/presence/presence.rs +++ b/src/service/presence/presence.rs @@ -21,7 +21,10 @@ pub(super) struct Presence { impl Presence { #[must_use] pub(super) fn new( - state: PresenceState, currently_active: bool, last_active_ts: u64, status_msg: Option, + state: PresenceState, + currently_active: bool, + last_active_ts: u64, + status_msg: Option, ) -> Self { Self { state, @@ -32,11 +35,16 @@ impl Presence { } pub(super) fn from_json_bytes(bytes: &[u8]) -> Result { - serde_json::from_slice(bytes).map_err(|_| Error::bad_database("Invalid presence data in database")) + serde_json::from_slice(bytes) + .map_err(|_| Error::bad_database("Invalid presence data in database")) } /// Creates a PresenceEvent from available data. - pub(super) async fn to_presence_event(&self, user_id: &UserId, users: &users::Service) -> PresenceEvent { + pub(super) async fn to_presence_event( + &self, + user_id: &UserId, + users: &users::Service, + ) -> PresenceEvent { let now = utils::millis_since_unix_epoch(); let last_active_ago = if self.currently_active { None diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index ffe822b7..cea3ba35 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -19,9 +19,12 @@ use ruma::{ IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, }, events::{ - room::power_levels::RoomPowerLevelsEventContent, AnySyncTimelineEvent, StateEventType, TimelineEventType, + room::power_levels::RoomPowerLevelsEventContent, AnySyncTimelineEvent, StateEventType, + TimelineEventType, + }, + push::{ + Action, PushConditionPowerLevelsCtx, PushConditionRoomCtx, PushFormat, Ruleset, Tweak, }, - push::{Action, PushConditionPowerLevelsCtx, PushConditionRoomCtx, PushFormat, Ruleset, Tweak}, serde::Raw, uint, RoomId, UInt, UserId, }; @@ -55,7 +58,8 @@ impl crate::Service for Service { services: Services { globals: args.depend::("globals"), client: args.depend::("client"), - state_accessor: args.depend::("rooms::state_accessor"), + state_accessor: args + .depend::("rooms::state_accessor"), state_cache: args.depend::("rooms::state_cache"), users: args.depend::("users"), sending: args.depend::("sending"), @@ -67,23 +71,31 @@ impl crate::Service for Service { } impl Service { - pub async fn set_pusher(&self, sender: &UserId, pusher: &set_pusher::v3::PusherAction) -> Result { + pub async fn set_pusher( + &self, + sender: &UserId, + pusher: &set_pusher::v3::PusherAction, + ) -> Result { match pusher { - set_pusher::v3::PusherAction::Post(data) => { + | set_pusher::v3::PusherAction::Post(data) => { let pushkey = data.pusher.ids.pushkey.as_str(); if pushkey.len() > 512 { - return Err!(Request(InvalidParam("Push key length cannot be greater than 512 bytes."))); + return Err!(Request(InvalidParam( + "Push key length cannot be greater than 512 bytes." + ))); } if data.pusher.ids.app_id.as_str().len() > 64 { - return Err!(Request(InvalidParam("App ID length cannot be greater than 64 bytes."))); + return Err!(Request(InvalidParam( + "App ID length cannot be greater than 64 bytes." + ))); } let key = (sender, data.pusher.ids.pushkey.as_str()); self.db.senderkey_pusher.put(key, Json(pusher)); }, - set_pusher::v3::PusherAction::Delete(ids) => { + | set_pusher::v3::PusherAction::Delete(ids) => { let key = (sender, ids.pushkey.as_str()); self.db.senderkey_pusher.del(key); @@ -118,7 +130,10 @@ impl Service { .await } - pub fn get_pushkeys<'a>(&'a self, sender: &'a UserId) -> impl Stream + Send + 'a { + pub fn get_pushkeys<'a>( + &'a self, + sender: &'a UserId, + ) -> impl Stream + Send + 'a { let prefix = (sender, Interfix); self.db .senderkey_pusher @@ -160,14 +175,16 @@ impl Service { let response = self.services.client.pusher.execute(reqwest_request).await; match response { - Ok(mut response) => { + | Ok(mut response) => { // reqwest::Response -> http::Response conversion trace!("Checking response destination's IP"); if let Some(remote_addr) = response.remote_addr() { if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { if !self.services.client.valid_cidr_range(&ip) { - return Err!(BadServerResponse("Not allowed to send requests to this IP")); + return Err!(BadServerResponse( + "Not allowed to send requests to this IP" + )); } } } @@ -197,10 +214,13 @@ impl Service { .body(body) .expect("reqwest body is valid http body"), ); - response - .map_err(|e| err!(BadServerResponse(warn!("Push gateway {dest} returned invalid response: {e}")))) + response.map_err(|e| { + err!(BadServerResponse(warn!( + "Push gateway {dest} returned invalid response: {e}" + ))) + }) }, - Err(e) => { + | Err(e) => { warn!("Could not send request to pusher {dest}: {e}"); Err(e.into()) }, @@ -209,7 +229,12 @@ impl Service { #[tracing::instrument(skip(self, user, unread, pusher, ruleset, pdu))] pub async fn send_push_notice( - &self, user: &UserId, unread: UInt, pusher: &Pusher, ruleset: Ruleset, pdu: &PduEvent, + &self, + user: &UserId, + unread: UInt, + pusher: &Pusher, + ruleset: Ruleset, + pdu: &PduEvent, ) -> Result<()> { let mut notify = None; let mut tweaks = Vec::new(); @@ -220,8 +245,9 @@ impl Service { .room_state_get(&pdu.room_id, &StateEventType::RoomPowerLevels, "") .await .and_then(|ev| { - serde_json::from_str(ev.content.get()) - .map_err(|e| err!(Database(error!("invalid m.room.power_levels event: {e:?}")))) + serde_json::from_str(ev.content.get()).map_err(|e| { + err!(Database(error!("invalid m.room.power_levels event: {e:?}"))) + }) }) .unwrap_or_default(); @@ -230,12 +256,12 @@ impl Service { .await { let n = match action { - Action::Notify => true, - Action::SetTweak(tweak) => { + | Action::Notify => true, + | Action::SetTweak(tweak) => { tweaks.push(tweak.clone()); continue; }, - _ => false, + | _ => false, }; if notify.is_some() { @@ -257,8 +283,12 @@ impl Service { #[tracing::instrument(skip(self, user, ruleset, pdu), level = "debug")] pub async fn get_actions<'a>( - &self, user: &UserId, ruleset: &'a Ruleset, power_levels: &RoomPowerLevelsEventContent, - pdu: &Raw, room_id: &RoomId, + &self, + user: &UserId, + ruleset: &'a Ruleset, + power_levels: &RoomPowerLevelsEventContent, + pdu: &Raw, + room_id: &RoomId, ) -> &'a [Action] { let power_levels = PushConditionPowerLevelsCtx { users: power_levels.users.clone(), @@ -294,14 +324,21 @@ impl Service { } #[tracing::instrument(skip(self, unread, pusher, tweaks, event))] - async fn send_notice(&self, unread: UInt, pusher: &Pusher, tweaks: Vec, event: &PduEvent) -> Result<()> { + async fn send_notice( + &self, + unread: UInt, + pusher: &Pusher, + tweaks: Vec, + event: &PduEvent, + ) -> Result<()> { // TODO: email match &pusher.kind { - PusherKind::Http(http) => { + | PusherKind::Http(http) => { // TODO (timo): can pusher/devices have conflicting formats let event_id_only = http.format == Some(PushFormat::EventIdOnly); - let mut device = Device::new(pusher.ids.app_id.clone(), pusher.ids.pushkey.clone()); + let mut device = + Device::new(pusher.ids.app_id.clone(), pusher.ids.pushkey.clone()); device.data.default_payload = http.default_payload.clone(); device.data.format.clone_from(&http.format); @@ -319,8 +356,11 @@ impl Service { notifi.counts = NotificationCounts::new(unread, uint!(0)); if event_id_only { - self.send_request(&http.url, send_event_notification::v1::Request::new(notifi)) - .await?; + self.send_request( + &http.url, + send_event_notification::v1::Request::new(notifi), + ) + .await?; } else { if event.kind == TimelineEventType::RoomEncrypted || tweaks @@ -336,10 +376,12 @@ impl Service { notifi.content = serde_json::value::to_raw_value(&event.content).ok(); if event.kind == TimelineEventType::RoomMember { - notifi.user_is_target = event.state_key.as_deref() == Some(event.sender.as_str()); + notifi.user_is_target = + event.state_key.as_deref() == Some(event.sender.as_str()); } - notifi.sender_display_name = self.services.users.displayname(&event.sender).await.ok(); + notifi.sender_display_name = + self.services.users.displayname(&event.sender).await.ok(); notifi.room_name = self .services @@ -355,15 +397,18 @@ impl Service { .await .ok(); - self.send_request(&http.url, send_event_notification::v1::Request::new(notifi)) - .await?; + self.send_request( + &http.url, + send_event_notification::v1::Request::new(notifi), + ) + .await?; } Ok(()) }, // TODO: Handle email //PusherKind::Email(_) => Ok(()), - _ => Ok(()), + | _ => Ok(()), } } } diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 4d2da695..63de6539 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -35,17 +35,9 @@ impl super::Service { (self.resolve_actual_dest(server_name, true).await?, false) }; - let CachedDest { - dest, - host, - .. - } = result; + let CachedDest { dest, host, .. } = result; - Ok(ActualDest { - dest, - host, - cached, - }) + Ok(ActualDest { dest, host, cached }) } /// Returns: `actual_destination`, host header @@ -53,12 +45,16 @@ impl super::Service { /// Numbers in comments below refer to bullet points in linked section of /// specification #[tracing::instrument(skip_all, name = "actual")] - pub async fn resolve_actual_dest(&self, dest: &ServerName, cache: bool) -> Result { + pub async fn resolve_actual_dest( + &self, + dest: &ServerName, + cache: bool, + ) -> Result { trace!("Finding actual destination for {dest}"); let mut host = dest.as_str().to_owned(); let actual_dest = match get_ip_with_port(dest.as_str()) { - Some(host_port) => Self::actual_dest_1(host_port)?, - None => { + | Some(host_port) => Self::actual_dest_1(host_port)?, + | None => if let Some(pos) = dest.as_str().find(':') { self.actual_dest_2(dest, cache, pos).await? } else if let Some(delegated) = self.request_well_known(dest.as_str()).await? { @@ -67,8 +63,7 @@ impl super::Service { self.actual_dest_4(&host, cache, overrider).await? } else { self.actual_dest_5(dest, cache).await? - } - }, + }, }; // Can't use get_ip_with_port here because we don't want to add a port @@ -79,7 +74,10 @@ impl super::Service { FedDest::Named(addr.to_string(), FedDest::default_port()) } else if let Some(pos) = host.find(':') { let (host, port) = host.split_at(pos); - FedDest::Named(host.to_owned(), port.try_into().unwrap_or_else(|_| FedDest::default_port())) + FedDest::Named( + host.to_owned(), + port.try_into().unwrap_or_else(|_| FedDest::default_port()), + ) } else { FedDest::Named(host, FedDest::default_port()) }; @@ -100,20 +98,30 @@ impl super::Service { async fn actual_dest_2(&self, dest: &ServerName, cache: bool, pos: usize) -> Result { debug!("2: Hostname with included port"); let (host, port) = dest.as_str().split_at(pos); - self.conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache) - .await?; + self.conditional_query_and_cache_override( + host, + host, + port.parse::().unwrap_or(8448), + cache, + ) + .await?; Ok(FedDest::Named( host.to_owned(), port.try_into().unwrap_or_else(|_| FedDest::default_port()), )) } - async fn actual_dest_3(&self, host: &mut String, cache: bool, delegated: String) -> Result { + async fn actual_dest_3( + &self, + host: &mut String, + cache: bool, + delegated: String, + ) -> Result { debug!("3: A .well-known file is available"); *host = add_port_to_hostname(&delegated).uri_string(); match get_ip_with_port(&delegated) { - Some(host_and_port) => Self::actual_dest_3_1(host_and_port), - None => { + | Some(host_and_port) => Self::actual_dest_3_1(host_and_port), + | None => if let Some(pos) = delegated.find(':') { self.actual_dest_3_2(cache, delegated, pos).await } else { @@ -123,8 +131,7 @@ impl super::Service { } else { self.actual_dest_3_4(cache, delegated).await } - } - }, + }, } } @@ -133,22 +140,42 @@ impl super::Service { Ok(host_and_port) } - async fn actual_dest_3_2(&self, cache: bool, delegated: String, pos: usize) -> Result { + async fn actual_dest_3_2( + &self, + cache: bool, + delegated: String, + pos: usize, + ) -> Result { debug!("3.2: Hostname with port in .well-known file"); let (host, port) = delegated.split_at(pos); - self.conditional_query_and_cache_override(host, host, port.parse::().unwrap_or(8448), cache) - .await?; + self.conditional_query_and_cache_override( + host, + host, + port.parse::().unwrap_or(8448), + cache, + ) + .await?; Ok(FedDest::Named( host.to_owned(), port.try_into().unwrap_or_else(|_| FedDest::default_port()), )) } - async fn actual_dest_3_3(&self, cache: bool, delegated: String, overrider: FedDest) -> Result { + async fn actual_dest_3_3( + &self, + cache: bool, + delegated: String, + overrider: FedDest, + ) -> Result { debug!("3.3: SRV lookup successful"); let force_port = overrider.port(); - self.conditional_query_and_cache_override(&delegated, &overrider.hostname(), force_port.unwrap_or(8448), cache) - .await?; + self.conditional_query_and_cache_override( + &delegated, + &overrider.hostname(), + force_port.unwrap_or(8448), + cache, + ) + .await?; if let Some(port) = force_port { Ok(FedDest::Named( delegated, @@ -169,11 +196,21 @@ impl super::Service { Ok(add_port_to_hostname(&delegated)) } - async fn actual_dest_4(&self, host: &str, cache: bool, overrider: FedDest) -> Result { + async fn actual_dest_4( + &self, + host: &str, + cache: bool, + overrider: FedDest, + ) -> Result { debug!("4: No .well-known; SRV record found"); let force_port = overrider.port(); - self.conditional_query_and_cache_override(host, &overrider.hostname(), force_port.unwrap_or(8448), cache) - .await?; + self.conditional_query_and_cache_override( + host, + &overrider.hostname(), + force_port.unwrap_or(8448), + cache, + ) + .await?; if let Some(port) = force_port { let port = format!(":{port}"); Ok(FedDest::Named( @@ -245,7 +282,11 @@ impl super::Service { #[inline] async fn conditional_query_and_cache_override( - &self, overname: &str, hostname: &str, port: u16, cache: bool, + &self, + overname: &str, + hostname: &str, + port: u16, + cache: bool, ) -> Result<()> { if cache { self.query_and_cache_override(overname, hostname, port) @@ -256,22 +297,24 @@ impl super::Service { } #[tracing::instrument(skip_all, name = "ip")] - async fn query_and_cache_override(&self, overname: &'_ str, hostname: &'_ str, port: u16) -> Result<()> { + async fn query_and_cache_override( + &self, + overname: &'_ str, + hostname: &'_ str, + port: u16, + ) -> Result<()> { match self.resolver.resolver.lookup_ip(hostname.to_owned()).await { - Err(e) => Self::handle_resolve_error(&e, hostname), - Ok(override_ip) => { + | Err(e) => Self::handle_resolve_error(&e, hostname), + | Ok(override_ip) => { if hostname != overname { debug_info!("{overname:?} overriden by {hostname:?}"); } - self.set_cached_override( - overname, - CachedOverride { - ips: override_ip.into_iter().take(MAX_IPS).collect(), - port, - expire: CachedOverride::default_expire(), - }, - ); + self.set_cached_override(overname, CachedOverride { + ips: override_ip.into_iter().take(MAX_IPS).collect(), + port, + expire: CachedOverride::default_expire(), + }); Ok(()) }, @@ -280,14 +323,15 @@ impl super::Service { #[tracing::instrument(skip_all, name = "srv")] async fn query_srv_record(&self, hostname: &'_ str) -> Result> { - let hostnames = [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")]; + let hostnames = + [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")]; for hostname in hostnames { debug!("querying SRV for {hostname:?}"); let hostname = hostname.trim_end_matches('.'); match self.resolver.resolver.srv_lookup(hostname).await { - Err(e) => Self::handle_resolve_error(&e, hostname)?, - Ok(result) => { + | Err(e) => Self::handle_resolve_error(&e, hostname)?, + | Ok(result) => return Ok(result.iter().next().map(|result| { FedDest::Named( result.target().to_string().trim_end_matches('.').to_owned(), @@ -296,8 +340,7 @@ impl super::Service { .try_into() .unwrap_or_else(|_| FedDest::default_port()), ) - })) - }, + })), } } @@ -308,25 +351,24 @@ impl super::Service { use hickory_resolver::error::ResolveErrorKind; match *e.kind() { - ResolveErrorKind::NoRecordsFound { - .. - } => { + | ResolveErrorKind::NoRecordsFound { .. } => { // Raise to debug_warn if we can find out the result wasn't from cache debug!(%host, "No DNS records found: {e}"); Ok(()) }, - ResolveErrorKind::Timeout => { + | ResolveErrorKind::Timeout => { Err!(warn!(%host, "DNS {e}")) }, - ResolveErrorKind::NoConnections => { + | ResolveErrorKind::NoConnections => { error!( - "Your DNS server is overloaded and has ran out of connections. It is strongly recommended you \ - remediate this issue to ensure proper federation connectivity." + "Your DNS server is overloaded and has ran out of connections. It is \ + strongly recommended you remediate this issue to ensure proper federation \ + connectivity." ); Err!(error!(%host, "DNS error: {e}")) }, - _ => Err!(error!(%host, "DNS error: {e}")), + | _ => Err!(error!(%host, "DNS error: {e}")), } } @@ -349,8 +391,9 @@ impl super::Service { dest.is_ip_literal() || !IPAddress::is_valid(dest.host()), "Destination is not an IP literal." ); - let ip = IPAddress::parse(dest.host()) - .map_err(|e| err!(BadServerResponse(debug_error!("Failed to parse IP literal from string: {e}"))))?; + let ip = IPAddress::parse(dest.host()).map_err(|e| { + err!(BadServerResponse(debug_error!("Failed to parse IP literal from string: {e}"))) + })?; self.validate_ip(&ip)?; diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index ca65db35..3e961f4c 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -46,7 +46,11 @@ impl Cache { } impl super::Service { - pub fn set_cached_destination(&self, name: OwnedServerName, dest: CachedDest) -> Option { + pub fn set_cached_destination( + &self, + name: OwnedServerName, + dest: CachedDest, + ) -> Option { trace!(?name, ?dest, "set cached destination"); self.cache .destinations @@ -65,7 +69,11 @@ impl super::Service { .cloned() } - pub fn set_cached_override(&self, name: &str, over: CachedOverride) -> Option { + pub fn set_cached_override( + &self, + name: &str, + over: CachedOverride, + ) -> Option { trace!(?name, ?over, "set cached override"); self.cache .overrides @@ -102,7 +110,9 @@ impl CachedDest { //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } #[must_use] - pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } + pub(crate) fn default_expire() -> SystemTime { + rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) + } } impl CachedOverride { @@ -113,5 +123,7 @@ impl CachedOverride { //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } #[must_use] - pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } + pub(crate) fn default_expire() -> SystemTime { + rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) + } } diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index 54905647..633b397a 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -60,27 +60,26 @@ impl Resolver { opts.shuffle_dns_servers = true; opts.rotate = true; opts.ip_strategy = match config.ip_lookup_strategy { - 1 => hickory_resolver::config::LookupIpStrategy::Ipv4Only, - 2 => hickory_resolver::config::LookupIpStrategy::Ipv6Only, - 3 => hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6, - 4 => hickory_resolver::config::LookupIpStrategy::Ipv6thenIpv4, - _ => hickory_resolver::config::LookupIpStrategy::Ipv4thenIpv6, + | 1 => hickory_resolver::config::LookupIpStrategy::Ipv4Only, + | 2 => hickory_resolver::config::LookupIpStrategy::Ipv6Only, + | 3 => hickory_resolver::config::LookupIpStrategy::Ipv4AndIpv6, + | 4 => hickory_resolver::config::LookupIpStrategy::Ipv6thenIpv4, + | _ => hickory_resolver::config::LookupIpStrategy::Ipv4thenIpv6, }; opts.authentic_data = false; let resolver = Arc::new(TokioAsyncResolver::tokio(conf, opts)); Ok(Arc::new(Self { resolver: resolver.clone(), - hooked: Arc::new(Hooked { - resolver, - cache, - }), + hooked: Arc::new(Hooked { resolver, cache }), })) } } impl Resolve for Resolver { - fn resolve(&self, name: Name) -> Resolving { resolve_to_reqwest(self.resolver.clone(), name).boxed() } + fn resolve(&self, name: Name) -> Resolving { + resolve_to_reqwest(self.resolver.clone(), name).boxed() + } } impl Resolve for Hooked { diff --git a/src/service/resolver/fed.rs b/src/service/resolver/fed.rs index 9c348b47..3986db8e 100644 --- a/src/service/resolver/fed.rs +++ b/src/service/resolver/fed.rs @@ -29,8 +29,8 @@ pub(crate) fn get_ip_with_port(dest_str: &str) -> Option { pub(crate) fn add_port_to_hostname(dest: &str) -> FedDest { let (host, port) = match dest.find(':') { - None => (dest, DEFAULT_PORT), - Some(pos) => dest.split_at(pos), + | None => (dest, DEFAULT_PORT), + | Some(pos) => dest.split_at(pos), }; FedDest::Named( @@ -42,23 +42,23 @@ pub(crate) fn add_port_to_hostname(dest: &str) -> FedDest { impl FedDest { pub(crate) fn https_string(&self) -> String { match self { - Self::Literal(addr) => format!("https://{addr}"), - Self::Named(host, port) => format!("https://{host}{port}"), + | Self::Literal(addr) => format!("https://{addr}"), + | Self::Named(host, port) => format!("https://{host}{port}"), } } pub(crate) fn uri_string(&self) -> String { match self { - Self::Literal(addr) => addr.to_string(), - Self::Named(host, port) => format!("{host}{port}"), + | Self::Literal(addr) => addr.to_string(), + | Self::Named(host, port) => format!("{host}{port}"), } } #[inline] pub(crate) fn hostname(&self) -> Cow<'_, str> { match &self { - Self::Literal(addr) => addr.ip().to_string().into(), - Self::Named(host, _) => host.into(), + | Self::Literal(addr) => addr.ip().to_string().into(), + | Self::Named(host, _) => host.into(), } } @@ -66,16 +66,20 @@ impl FedDest { #[allow(clippy::string_slice)] pub(crate) fn port(&self) -> Option { match &self { - Self::Literal(addr) => Some(addr.port()), - Self::Named(_, port) => port[1..].parse().ok(), + | Self::Literal(addr) => Some(addr.port()), + | Self::Named(_, port) => port[1..].parse().ok(), } } #[inline] #[must_use] - pub fn default_port() -> PortString { PortString::from(DEFAULT_PORT).expect("default port string") } + pub fn default_port() -> PortString { + PortString::from(DEFAULT_PORT).expect("default port string") + } } impl fmt::Display for FedDest { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.uri_string().as_str()) } + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.uri_string().as_str()) + } } diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 57db0e15..0790c376 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -52,7 +52,8 @@ impl crate::Service for Service { appservice: args.depend::("appservice"), globals: args.depend::("globals"), sending: args.depend::("sending"), - state_accessor: args.depend::("rooms::state_accessor"), + state_accessor: args + .depend::("rooms::state_accessor"), }, })) } @@ -62,8 +63,15 @@ impl crate::Service for Service { impl Service { #[tracing::instrument(skip(self))] - pub fn set_alias(&self, alias: &RoomAliasId, room_id: &RoomId, user_id: &UserId) -> Result<()> { - if alias == self.services.globals.admin_alias && user_id != self.services.globals.server_user { + pub fn set_alias( + &self, + alias: &RoomAliasId, + room_id: &RoomId, + user_id: &UserId, + ) -> Result<()> { + if alias == self.services.globals.admin_alias + && user_id != self.services.globals.server_user + { return Err(Error::BadRequest( ErrorKind::forbidden(), "Only the server user can set this alias", @@ -120,7 +128,9 @@ impl Service { } pub async fn resolve_with_servers( - &self, room: &RoomOrAliasId, servers: Option>, + &self, + room: &RoomOrAliasId, + servers: Option>, ) -> Result<(OwnedRoomId, Vec)> { if room.is_room_id() { let room_id = RoomId::parse(room).expect("valid RoomId"); @@ -133,14 +143,16 @@ impl Service { #[tracing::instrument(skip(self), name = "resolve")] pub async fn resolve_alias( - &self, room_alias: &RoomAliasId, servers: Option>, + &self, + room_alias: &RoomAliasId, + servers: Option>, ) -> Result<(OwnedRoomId, Vec)> { let server_name = room_alias.server_name(); let server_is_ours = self.services.globals.server_is_ours(server_name); let servers_contains_ours = || { - servers - .as_ref() - .is_some_and(|servers| servers.contains(&self.services.globals.config.server_name)) + servers.as_ref().is_some_and(|servers| { + servers.contains(&self.services.globals.config.server_name) + }) }; if !server_is_ours && !servers_contains_ours() { @@ -150,8 +162,8 @@ impl Service { } let room_id = match self.resolve_local_alias(room_alias).await { - Ok(r) => Some(r), - Err(_) => self.resolve_appservice_alias(room_alias).await?, + | Ok(r) => Some(r), + | Err(_) => self.resolve_appservice_alias(room_alias).await?, }; room_id.map_or_else( @@ -166,7 +178,10 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub fn local_aliases_for_room<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + pub fn local_aliases_for_room<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .aliasid_alias @@ -208,10 +223,15 @@ impl Service { if let Ok(content) = self .services .state_accessor - .room_state_get_content::(&room_id, &StateEventType::RoomPowerLevels, "") + .room_state_get_content::( + &room_id, + &StateEventType::RoomPowerLevels, + "", + ) .await { - return Ok(RoomPowerLevels::from(content).user_can_send_state(user_id, StateEventType::RoomCanonicalAlias)); + return Ok(RoomPowerLevels::from(content) + .user_can_send_state(user_id, StateEventType::RoomCanonicalAlias)); } // If there is no power levels event, only the room creator can change @@ -232,7 +252,10 @@ impl Service { self.db.alias_userid.get(alias.alias()).await.deserialized() } - async fn resolve_appservice_alias(&self, room_alias: &RoomAliasId) -> Result> { + async fn resolve_appservice_alias( + &self, + room_alias: &RoomAliasId, + ) -> Result> { use ruma::api::appservice::query::query_room_alias; for appservice in self.services.appservice.read().await.values() { @@ -242,9 +265,7 @@ impl Service { .sending .send_appservice_request( appservice.registration.clone(), - query_room_alias::v1::Request { - room_alias: room_alias.to_owned(), - }, + query_room_alias::v1::Request { room_alias: room_alias.to_owned() }, ) .await, Ok(Some(_opt_result)) @@ -261,19 +282,27 @@ impl Service { } pub async fn appservice_checks( - &self, room_alias: &RoomAliasId, appservice_info: &Option, + &self, + room_alias: &RoomAliasId, + appservice_info: &Option, ) -> Result<()> { if !self .services .globals .server_is_ours(room_alias.server_name()) { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Alias is from another server.")); + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Alias is from another server.", + )); } if let Some(ref info) = appservice_info { if !info.aliases.is_match(room_alias.as_str()) { - return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias is not in namespace.")); + return Err(Error::BadRequest( + ErrorKind::Exclusive, + "Room alias is not in namespace.", + )); } } else if self .services @@ -281,7 +310,10 @@ impl Service { .is_exclusive_alias(room_alias) .await { - return Err(Error::BadRequest(ErrorKind::Exclusive, "Room alias reserved by appservice.")); + return Err(Error::BadRequest( + ErrorKind::Exclusive, + "Room alias reserved by appservice.", + )); } Ok(()) diff --git a/src/service/rooms/alias/remote.rs b/src/service/rooms/alias/remote.rs index 00661da2..7744bee2 100644 --- a/src/service/rooms/alias/remote.rs +++ b/src/service/rooms/alias/remote.rs @@ -6,7 +6,9 @@ use ruma::{api::federation, OwnedRoomId, OwnedServerName, RoomAliasId, ServerNam #[implement(super::Service)] pub(super) async fn remote_resolve( - &self, room_alias: &RoomAliasId, servers: Vec, + &self, + room_alias: &RoomAliasId, + servers: Vec, ) -> Result<(OwnedRoomId, Vec)> { debug!(?room_alias, servers = ?servers, "resolve"); let servers = once(room_alias.server_name()) @@ -17,12 +19,12 @@ pub(super) async fn remote_resolve( let mut resolved_room_id: Option = None; for server in servers { match self.remote_request(room_alias, &server).await { - Err(e) => debug_error!("Failed to query for {room_alias:?} from {server}: {e}"), - Ok(Response { - room_id, - servers, - }) => { - debug!("Server {server} answered with {room_id:?} for {room_alias:?} servers: {servers:?}"); + | Err(e) => debug_error!("Failed to query for {room_alias:?} from {server}: {e}"), + | Ok(Response { room_id, servers }) => { + debug!( + "Server {server} answered with {room_id:?} for {room_alias:?} servers: \ + {servers:?}" + ); resolved_room_id.get_or_insert(room_id); add_server(&mut resolved_servers, server); @@ -37,16 +39,20 @@ pub(super) async fn remote_resolve( resolved_room_id .map(|room_id| (room_id, resolved_servers)) - .ok_or_else(|| err!(Request(NotFound("No servers could assist in resolving the room alias")))) + .ok_or_else(|| { + err!(Request(NotFound("No servers could assist in resolving the room alias"))) + }) } #[implement(super::Service)] -async fn remote_request(&self, room_alias: &RoomAliasId, server: &ServerName) -> Result { +async fn remote_request( + &self, + room_alias: &RoomAliasId, + server: &ServerName, +) -> Result { use federation::query::get_room_information::v1::Request; - let request = Request { - room_alias: room_alias.to_owned(), - }; + let request = Request { room_alias: room_alias.to_owned() }; self.services .sending diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index 1548cd55..af8ae364 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -19,14 +19,18 @@ impl Data { let db = &args.db; let config = &args.server.config; let cache_size = f64::from(config.auth_chain_cache_capacity); - let cache_size = usize_from_f64(cache_size * config.cache_capacity_modifier).expect("valid cache size"); + let cache_size = usize_from_f64(cache_size * config.cache_capacity_modifier) + .expect("valid cache size"); Self { shorteventid_authchain: db["shorteventid_authchain"].clone(), auth_chain_cache: Mutex::new(LruCache::new(cache_size)), } } - pub(super) async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { + pub(super) async fn get_cached_eventid_authchain( + &self, + key: &[u64], + ) -> Result> { debug_assert!(!key.is_empty(), "auth_chain key must not be empty"); // Check RAM cache diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index b875bf9c..87992f2d 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -43,7 +43,9 @@ impl crate::Service for Service { impl Service { pub async fn event_ids_iter<'a, I>( - &'a self, room_id: &RoomId, starting_events: I, + &'a self, + room_id: &RoomId, + starting_events: I, ) -> Result> + Send + '_> where I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, @@ -57,7 +59,11 @@ impl Service { Ok(stream) } - pub async fn get_event_ids<'a, I>(&'a self, room_id: &RoomId, starting_events: I) -> Result>> + pub async fn get_event_ids<'a, I>( + &'a self, + room_id: &RoomId, + starting_events: I, + ) -> Result>> where I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, { @@ -74,7 +80,11 @@ impl Service { } #[tracing::instrument(skip_all, name = "auth_chain")] - pub async fn get_auth_chain<'a, I>(&'a self, room_id: &RoomId, starting_events: I) -> Result> + pub async fn get_auth_chain<'a, I>( + &'a self, + room_id: &RoomId, + starting_events: I, + ) -> Result> where I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, { @@ -110,7 +120,8 @@ impl Service { continue; } - let chunk_key: Vec = chunk.iter().map(|(short, _)| short).copied().collect(); + let chunk_key: Vec = + chunk.iter().map(|(short, _)| short).copied().collect(); if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { trace!("Found cache entry for whole chunk"); full_auth_chain.extend(cached.iter().copied()); @@ -169,7 +180,11 @@ impl Service { } #[tracing::instrument(skip(self, room_id))] - async fn get_auth_chain_inner(&self, room_id: &RoomId, event_id: &EventId) -> Result> { + async fn get_auth_chain_inner( + &self, + room_id: &RoomId, + event_id: &EventId, + ) -> Result> { let mut todo = vec![Arc::from(event_id)]; let mut found = HashSet::new(); @@ -177,8 +192,10 @@ impl Service { trace!(?event_id, "processing auth event"); match self.services.timeline.get_pdu(&event_id).await { - Err(e) => debug_error!(?event_id, ?e, "Could not find pdu mentioned in auth events"), - Ok(pdu) => { + | Err(e) => { + debug_error!(?event_id, ?e, "Could not find pdu mentioned in auth events"); + }, + | Ok(pdu) => { if pdu.room_id != room_id { return Err!(Request(Forbidden(error!( ?event_id, @@ -196,7 +213,11 @@ impl Service { .await; if found.insert(sauthevent) { - trace!(?event_id, ?auth_event, "adding auth event to processing queue"); + trace!( + ?event_id, + ?auth_event, + "adding auth event to processing queue" + ); todo.push(auth_event.clone()); } } diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 3046a328..039efca7 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -32,10 +32,14 @@ pub fn set_public(&self, room_id: &RoomId) { self.db.publicroomids.insert(room_i pub fn set_not_public(&self, room_id: &RoomId) { self.db.publicroomids.remove(room_id); } #[implement(Service)] -pub fn public_rooms(&self) -> impl Stream + Send { self.db.publicroomids.keys().ignore_err() } +pub fn public_rooms(&self) -> impl Stream + Send { + self.db.publicroomids.keys().ignore_err() +} #[implement(Service)] -pub async fn is_public_room(&self, room_id: &RoomId) -> bool { self.visibility(room_id).await == Visibility::Public } +pub async fn is_public_room(&self, room_id: &RoomId) -> bool { + self.visibility(room_id).await == Visibility::Public +} #[implement(Service)] pub async fn visibility(&self, room_id: &RoomId) -> Visibility { diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 316a1722..af0ef67d 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -5,10 +5,14 @@ use std::{ }; use conduwuit::{ - debug, debug_error, implement, info, pdu, trace, utils::math::continue_exponential_backoff_secs, warn, PduEvent, + debug, debug_error, implement, info, pdu, trace, + utils::math::continue_exponential_backoff_secs, warn, PduEvent, }; use futures::TryFutureExt; -use ruma::{api::federation::event::get_event, CanonicalJsonValue, EventId, RoomId, RoomVersionId, ServerName}; +use ruma::{ + api::federation::event::get_event, CanonicalJsonValue, EventId, RoomId, RoomVersionId, + ServerName, +}; /// Find the event and auth it. Once the event is validated (steps 1 - 8) /// it is appended to the outliers Tree. @@ -21,7 +25,11 @@ use ruma::{api::federation::event::get_event, CanonicalJsonValue, EventId, RoomI /// d. TODO: Ask other servers over federation? #[implement(super::Service)] pub(super) async fn fetch_and_handle_outliers<'a>( - &self, origin: &'a ServerName, events: &'a [Arc], create_event: &'a PduEvent, room_id: &'a RoomId, + &self, + origin: &'a ServerName, + events: &'a [Arc], + create_event: &'a PduEvent, + room_id: &'a RoomId, room_version_id: &'a RoomVersionId, ) -> Vec<(Arc, Option>)> { let back_off = |id| match self @@ -32,10 +40,12 @@ pub(super) async fn fetch_and_handle_outliers<'a>( .expect("locked") .entry(id) { - hash_map::Entry::Vacant(e) => { + | hash_map::Entry::Vacant(e) => { e.insert((Instant::now(), 1)); }, - hash_map::Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)), + | hash_map::Entry::Occupied(mut e) => { + *e.get_mut() = (Instant::now(), e.get().1.saturating_add(1)); + }, }; let mut events_with_auth_events = Vec::with_capacity(events.len()); @@ -67,7 +77,12 @@ pub(super) async fn fetch_and_handle_outliers<'a>( // Exponential backoff const MIN_DURATION: u64 = 5 * 60; const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + if continue_exponential_backoff_secs( + MIN_DURATION, + MAX_DURATION, + time.elapsed(), + *tries, + ) { info!("Backing off from {next_id}"); continue; } @@ -86,18 +101,16 @@ pub(super) async fn fetch_and_handle_outliers<'a>( match self .services .sending - .send_federation_request( - origin, - get_event::v1::Request { - event_id: (*next_id).to_owned(), - include_unredacted_content: None, - }, - ) + .send_federation_request(origin, get_event::v1::Request { + event_id: (*next_id).to_owned(), + include_unredacted_content: None, + }) .await { - Ok(res) => { + | Ok(res) => { debug!("Got {next_id} over federation"); - let Ok((calculated_event_id, value)) = pdu::gen_event_id_canonical_json(&res.pdu, room_version_id) + let Ok((calculated_event_id, value)) = + pdu::gen_event_id_canonical_json(&res.pdu, room_version_id) else { back_off((*next_id).to_owned()); continue; @@ -105,15 +118,18 @@ pub(super) async fn fetch_and_handle_outliers<'a>( if calculated_event_id != *next_id { warn!( - "Server didn't return event id we requested: requested: {next_id}, we got \ - {calculated_event_id}. Event: {:?}", + "Server didn't return event id we requested: requested: {next_id}, \ + we got {calculated_event_id}. Event: {:?}", &res.pdu ); } - if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) { + if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) + { for auth_event in auth_events { - if let Ok(auth_event) = serde_json::from_value(auth_event.clone().into()) { + if let Ok(auth_event) = + serde_json::from_value(auth_event.clone().into()) + { let a: Arc = auth_event; todo_auth_events.push(a); } else { @@ -127,7 +143,7 @@ pub(super) async fn fetch_and_handle_outliers<'a>( events_in_reverse_order.push((next_id.clone(), value)); events_all.insert(next_id); }, - Err(e) => { + | Err(e) => { debug_error!("Failed to fetch event {next_id}: {e}"); back_off((*next_id).to_owned()); }, @@ -158,20 +174,32 @@ pub(super) async fn fetch_and_handle_outliers<'a>( // Exponential backoff const MIN_DURATION: u64 = 5 * 60; const MAX_DURATION: u64 = 60 * 60 * 24; - if continue_exponential_backoff_secs(MIN_DURATION, MAX_DURATION, time.elapsed(), *tries) { + if continue_exponential_backoff_secs( + MIN_DURATION, + MAX_DURATION, + time.elapsed(), + *tries, + ) { debug!("Backing off from {next_id}"); continue; } } - match Box::pin(self.handle_outlier_pdu(origin, create_event, &next_id, room_id, value.clone(), true)).await + match Box::pin(self.handle_outlier_pdu( + origin, + create_event, + &next_id, + room_id, + value.clone(), + true, + )) + .await { - Ok((pdu, json)) => { + | Ok((pdu, json)) => if next_id == *id { pdus.push((pdu, Some(json))); - } - }, - Err(e) => { + }, + | Err(e) => { warn!("Authentication of event {next_id} failed: {e:?}"); back_off(next_id.into()); }, diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 577b3ff2..b271958f 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -8,7 +8,8 @@ use futures::{future, FutureExt}; use ruma::{ int, state_res::{self}, - uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, ServerName, + uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, + ServerName, }; use super::check_room_id; @@ -17,7 +18,11 @@ use super::check_room_id; #[allow(clippy::type_complexity)] #[tracing::instrument(skip_all)] pub(super) async fn fetch_prev( - &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, + &self, + origin: &ServerName, + create_event: &PduEvent, + room_id: &RoomId, + room_version_id: &RoomVersionId, initial_set: Vec>, ) -> Result<( Vec>, @@ -35,7 +40,13 @@ pub(super) async fn fetch_prev( self.services.server.check_running()?; if let Some((pdu, mut json_opt)) = self - .fetch_and_handle_outliers(origin, &[prev_event_id.clone()], create_event, room_id, room_version_id) + .fetch_and_handle_outliers( + origin, + &[prev_event_id.clone()], + create_event, + room_id, + room_version_id, + ) .boxed() .await .pop() @@ -67,7 +78,8 @@ pub(super) async fn fetch_prev( } } - graph.insert(prev_event_id.clone(), pdu.prev_events.iter().cloned().collect()); + graph + .insert(prev_event_id.clone(), pdu.prev_events.iter().cloned().collect()); } else { // Time based check failed graph.insert(prev_event_id.clone(), HashSet::new()); diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 9c7bc65e..9ea3e081 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -6,7 +6,8 @@ use std::{ use conduwuit::{debug, implement, warn, Err, Error, PduEvent, Result}; use futures::FutureExt; use ruma::{ - api::federation::event::get_room_state_ids, events::StateEventType, EventId, RoomId, RoomVersionId, ServerName, + api::federation::event::get_room_state_ids, events::StateEventType, EventId, RoomId, + RoomVersionId, ServerName, }; /// Call /state_ids to find out what the state at this pdu is. We trust the @@ -15,20 +16,21 @@ use ruma::{ #[implement(super::Service)] #[tracing::instrument(skip(self, create_event, room_version_id))] pub(super) async fn fetch_state( - &self, origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, + &self, + origin: &ServerName, + create_event: &PduEvent, + room_id: &RoomId, + room_version_id: &RoomVersionId, event_id: &EventId, ) -> Result>>> { debug!("Fetching state ids"); let res = self .services .sending - .send_federation_request( - origin, - get_room_state_ids::v1::Request { - room_id: room_id.to_owned(), - event_id: (*event_id).to_owned(), - }, - ) + .send_federation_request(origin, get_room_state_ids::v1::Request { + room_id: room_id.to_owned(), + event_id: (*event_id).to_owned(), + }) .await .inspect_err(|e| warn!("Fetching state for event failed: {e}"))?; @@ -58,14 +60,13 @@ pub(super) async fn fetch_state( .await; match state.entry(shortstatekey) { - hash_map::Entry::Vacant(v) => { + | hash_map::Entry::Vacant(v) => { v.insert(Arc::from(&*pdu.event_id)); }, - hash_map::Entry::Occupied(_) => { + | hash_map::Entry::Occupied(_) => return Err(Error::bad_database( "State event's type and state_key combination exists multiple times.", - )) - }, + )), } } diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index d63f96f9..ca56228d 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -7,7 +7,8 @@ use std::{ use conduwuit::{debug, err, implement, warn, Error, Result}; use futures::{FutureExt, TryFutureExt}; use ruma::{ - api::client::error::ErrorKind, events::StateEventType, CanonicalJsonValue, EventId, RoomId, ServerName, UserId, + api::client::error::ErrorKind, events::StateEventType, CanonicalJsonValue, EventId, RoomId, + ServerName, UserId, }; use super::{check_room_id, get_room_version_id}; @@ -43,8 +44,12 @@ use crate::rooms::timeline::RawPduId; #[implement(super::Service)] #[tracing::instrument(skip(self, origin, value, is_timeline_event), name = "pdu")] pub async fn handle_incoming_pdu<'a>( - &self, origin: &'a ServerName, room_id: &'a RoomId, event_id: &'a EventId, - value: BTreeMap, is_timeline_event: bool, + &self, + origin: &'a ServerName, + room_id: &'a RoomId, + event_id: &'a EventId, + value: BTreeMap, + is_timeline_event: bool, ) -> Result> { // 1. Skip the PDU if we already have it as a timeline event if let Ok(pdu_id) = self.services.timeline.get_pdu_id(event_id).await { @@ -144,10 +149,10 @@ pub async fn handle_incoming_pdu<'a>( .expect("locked") .entry(prev_id.into()) { - Entry::Vacant(e) => { + | Entry::Vacant(e) => { e.insert((now, 1)); }, - Entry::Occupied(mut e) => { + | Entry::Occupied(mut e) => { *e.get_mut() = (now, e.get().1.saturating_add(1)); }, }; diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 9391ebf3..59ef27ba 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -17,8 +17,13 @@ use super::{check_room_id, get_room_version_id, to_room_version}; #[implement(super::Service)] #[allow(clippy::too_many_arguments)] pub(super) async fn handle_outlier_pdu<'a>( - &self, origin: &'a ServerName, create_event: &'a PduEvent, event_id: &'a EventId, room_id: &'a RoomId, - mut value: CanonicalJsonObject, auth_events_known: bool, + &self, + origin: &'a ServerName, + create_event: &'a PduEvent, + event_id: &'a EventId, + room_id: &'a RoomId, + mut value: CanonicalJsonObject, + auth_events_known: bool, ) -> Result<(Arc, BTreeMap)> { // 1. Remove unsigned field value.remove("unsigned"); @@ -34,8 +39,8 @@ pub(super) async fn handle_outlier_pdu<'a>( .verify_event(&value, Some(&room_version_id)) .await { - Ok(ruma::signatures::Verified::All) => value, - Ok(ruma::signatures::Verified::Signatures) => { + | Ok(ruma::signatures::Verified::All) => value, + | Ok(ruma::signatures::Verified::Signatures) => { // Redact debug_info!("Calculated hash does not match (redaction): {event_id}"); let Ok(obj) = ruma::canonical_json::redact(value, &room_version_id, None) else { @@ -44,24 +49,26 @@ pub(super) async fn handle_outlier_pdu<'a>( // Skip the PDU if it is redacted and we already have it as an outlier event if self.services.timeline.pdu_exists(event_id).await { - return Err!(Request(InvalidParam("Event was redacted and we already knew about it"))); + return Err!(Request(InvalidParam( + "Event was redacted and we already knew about it" + ))); } obj }, - Err(e) => { + | Err(e) => return Err!(Request(InvalidParam(debug_error!( "Signature verification failed for {event_id}: {e}" - )))) - }, + )))), }; // Now that we have checked the signature and hashes we can add the eventID and // convert to our PduEvent type val.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); - let incoming_pdu = - serde_json::from_value::(serde_json::to_value(&val).expect("CanonicalJsonObj is a valid JsonValue")) - .map_err(|_| Error::bad_database("Event is not a valid PDU."))?; + let incoming_pdu = serde_json::from_value::( + serde_json::to_value(&val).expect("CanonicalJsonObj is a valid JsonValue"), + ) + .map_err(|_| Error::bad_database("Event is not a valid PDU."))?; check_room_id(room_id, &incoming_pdu)?; @@ -108,10 +115,10 @@ pub(super) async fn handle_outlier_pdu<'a>( .clone() .expect("all auth events have state keys"), )) { - hash_map::Entry::Vacant(v) => { + | hash_map::Entry::Vacant(v) => { v.insert(auth_event); }, - hash_map::Entry::Occupied(_) => { + | hash_map::Entry::Occupied(_) => { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Auth event's type and state_key combination exists multiple times.", diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index 1b4e9fe2..becaeb17 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -4,7 +4,9 @@ use std::{ time::Instant, }; -use conduwuit::{debug, implement, utils::math::continue_exponential_backoff_secs, Error, PduEvent, Result}; +use conduwuit::{ + debug, implement, utils::math::continue_exponential_backoff_secs, Error, PduEvent, Result, +}; use ruma::{api::client::error::ErrorKind, CanonicalJsonValue, EventId, RoomId, ServerName}; #[implement(super::Service)] @@ -15,15 +17,23 @@ use ruma::{api::client::error::ErrorKind, CanonicalJsonValue, EventId, RoomId, S name = "prev" )] pub(super) async fn handle_prev_pdu<'a>( - &self, origin: &'a ServerName, event_id: &'a EventId, room_id: &'a RoomId, - eventid_info: &mut HashMap, (Arc, BTreeMap)>, - create_event: &Arc, first_pdu_in_room: &Arc, prev_id: &EventId, + &self, + origin: &'a ServerName, + event_id: &'a EventId, + room_id: &'a RoomId, + eventid_info: &mut HashMap< + Arc, + (Arc, BTreeMap), + >, + create_event: &Arc, + first_pdu_in_room: &Arc, + prev_id: &EventId, ) -> Result { // Check for disabled again because it might have changed if self.services.metadata.is_disabled(room_id).await { debug!( - "Federaton of room {room_id} is currently disabled on this server. Request by origin {origin} and event \ - ID {event_id}" + "Federaton of room {room_id} is currently disabled on this server. Request by \ + origin {origin} and event ID {event_id}" ); return Err(Error::BadRequest( ErrorKind::forbidden(), diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index de3d2f49..bfc5a014 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -23,8 +23,8 @@ use conduwuit::{ }; use futures::TryFutureExt; use ruma::{ - events::room::create::RoomCreateEventContent, state_res::RoomVersion, EventId, OwnedEventId, OwnedRoomId, RoomId, - RoomVersionId, + events::room::create::RoomCreateEventContent, state_res::RoomVersion, EventId, OwnedEventId, + OwnedRoomId, RoomId, RoomVersionId, }; use crate::{globals, rooms, sending, server_keys, Dep}; @@ -69,8 +69,10 @@ impl crate::Service for Service { pdu_metadata: args.depend::("rooms::pdu_metadata"), short: args.depend::("rooms::short"), state: args.depend::("rooms::state"), - state_accessor: args.depend::("rooms::state_accessor"), - state_compressor: args.depend::("rooms::state_compressor"), + state_accessor: args + .depend::("rooms::state_accessor"), + state_compressor: args + .depend::("rooms::state_compressor"), timeline: args.depend::("rooms::timeline"), server: args.server.clone(), }, @@ -95,7 +97,9 @@ impl crate::Service for Service { } impl Service { - async fn event_exists(&self, event_id: Arc) -> bool { self.services.timeline.pdu_exists(&event_id).await } + async fn event_exists(&self, event_id: Arc) -> bool { + self.services.timeline.pdu_exists(&event_id).await + } async fn event_fetch(&self, event_id: Arc) -> Option> { self.services diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 6c19f43f..f3c75f36 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -3,9 +3,13 @@ use ruma::{CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, R use serde_json::value::RawValue as RawJsonValue; #[implement(super::Service)] -pub async fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { - let value = serde_json::from_str::(pdu.get()) - .map_err(|e| err!(BadServerResponse(debug_warn!("Error parsing incoming event {e:?}"))))?; +pub async fn parse_incoming_pdu( + &self, + pdu: &RawJsonValue, +) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { + let value = serde_json::from_str::(pdu.get()).map_err(|e| { + err!(BadServerResponse(debug_warn!("Error parsing incoming event {e:?}"))) + })?; let room_id: OwnedRoomId = value .get("room_id") @@ -20,8 +24,9 @@ pub async fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result<(OwnedEvent .await .map_err(|_| err!("Server is not in room {room_id}"))?; - let (event_id, value) = gen_event_id_canonical_json(pdu, &room_version_id) - .map_err(|e| err!(Request(InvalidParam("Could not convert event to canonical json: {e}"))))?; + let (event_id, value) = gen_event_id_canonical_json(pdu, &room_version_id).map_err(|e| { + err!(Request(InvalidParam("Could not convert event to canonical json: {e}"))) + })?; Ok((event_id, value, room_id)) } diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 8ba4e4f4..d507c9c3 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -20,7 +20,10 @@ use crate::rooms::state_compressor::CompressedStateEvent; #[implement(super::Service)] #[tracing::instrument(skip_all, name = "resolve")] pub async fn resolve_state( - &self, room_id: &RoomId, room_version_id: &RoomVersionId, incoming_state: HashMap>, + &self, + room_id: &RoomId, + room_version_id: &RoomVersionId, + incoming_state: HashMap>, ) -> Result>> { debug!("Loading current room state ids"); let current_sstatehash = self @@ -76,10 +79,16 @@ pub async fn resolve_state( let event_fetch = |event_id| self.event_fetch(event_id); let event_exists = |event_id| self.event_exists(event_id); - let state = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) - .boxed() - .await - .map_err(|e| err!(Database(error!("State resolution failed: {e:?}"))))?; + let state = state_res::resolve( + room_version_id, + &fork_states, + &auth_chain_sets, + &event_fetch, + &event_exists, + ) + .boxed() + .await + .map_err(|e| err!(Database(error!("State resolution failed: {e:?}"))))?; drop(lock); diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 6c76d9b5..51879d0d 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -21,7 +21,8 @@ use ruma::{ // request and build the state from a known point and resolve if > 1 prev_event #[tracing::instrument(skip_all, name = "state")] pub(super) async fn state_at_incoming_degree_one( - &self, incoming_pdu: &Arc, + &self, + incoming_pdu: &Arc, ) -> Result>>> { let prev_event = &*incoming_pdu.prev_events[0]; let Ok(prev_event_sstatehash) = self @@ -70,7 +71,10 @@ pub(super) async fn state_at_incoming_degree_one( #[implement(super::Service)] #[tracing::instrument(skip_all, name = "state")] pub(super) async fn state_at_incoming_resolved( - &self, incoming_pdu: &Arc, room_id: &RoomId, room_version_id: &RoomVersionId, + &self, + incoming_pdu: &Arc, + room_id: &RoomId, + room_version_id: &RoomVersionId, ) -> Result>>> { debug!("Calculating state at event using state res"); let mut extremity_sstatehashes = HashMap::with_capacity(incoming_pdu.prev_events.len()); @@ -157,10 +161,16 @@ pub(super) async fn state_at_incoming_resolved( let event_fetch = |event_id| self.event_fetch(event_id); let event_exists = |event_id| self.event_exists(event_id); - let result = state_res::resolve(room_version_id, &fork_states, &auth_chain_sets, &event_fetch, &event_exists) - .boxed() - .await - .map_err(|e| err!(Database(warn!(?e, "State resolution on prev events failed.")))); + let result = state_res::resolve( + room_version_id, + &fork_states, + &auth_chain_sets, + &event_fetch, + &event_exists, + ) + .boxed() + .await + .map_err(|e| err!(Database(warn!(?e, "State resolution on prev events failed.")))); drop(lock); diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 83267563..aa484cd6 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -19,8 +19,12 @@ use crate::rooms::{state_compressor::HashSetCompressStateEvent, timeline::RawPdu #[implement(super::Service)] pub(super) async fn upgrade_outlier_to_timeline_pdu( - &self, incoming_pdu: Arc, val: BTreeMap, create_event: &PduEvent, - origin: &ServerName, room_id: &RoomId, + &self, + incoming_pdu: Arc, + val: BTreeMap, + create_event: &PduEvent, + origin: &ServerName, + room_id: &RoomId, ) -> Result> { // Skip the PDU if we already have it as a timeline event if let Ok(pduid) = self @@ -63,7 +67,8 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( .await?; } - let state_at_incoming_event = state_at_incoming_event.expect("we always set this to some above"); + let state_at_incoming_event = + state_at_incoming_event.expect("we always set this to some above"); let room_version = to_room_version(&room_version_id); debug!("Performing auth check"); @@ -124,24 +129,34 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( !auth_check || incoming_pdu.kind == TimelineEventType::RoomRedaction && match room_version_id { - V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &incoming_pdu.redacts { !self .services .state_accessor - .user_can_redact(redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) + .user_can_redact( + redact_id, + &incoming_pdu.sender, + &incoming_pdu.room_id, + true, + ) .await? } else { false } }, - _ => { + | _ => { let content: RoomRedactionEventContent = incoming_pdu.get_content()?; if let Some(redact_id) = &content.redacts { !self .services .state_accessor - .user_can_redact(redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) + .user_can_redact( + redact_id, + &incoming_pdu.sender, + &incoming_pdu.room_id, + true, + ) .await? } else { false @@ -229,11 +244,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( // Set the new room state to the resolved state debug!("Forcing new room state"); - let HashSetCompressStateEvent { - shortstatehash, - added, - removed, - } = self + let HashSetCompressStateEvent { shortstatehash, added, removed } = self .services .state_compressor .save_state(room_id, new_room_state) diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index a4bd4e8f..c3c27b9e 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -51,7 +51,11 @@ impl crate::Service for Service { #[tracing::instrument(skip(self), level = "debug")] #[inline] pub async fn lazy_load_was_sent_before( - &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, ll_user: &UserId, + &self, + user_id: &UserId, + device_id: &DeviceId, + room_id: &RoomId, + ll_user: &UserId, ) -> bool { let key = (user_id, device_id, room_id, ll_user); self.db.lazyloadedids.qry(&key).await.is_ok() @@ -60,7 +64,12 @@ pub async fn lazy_load_was_sent_before( #[implement(Service)] #[tracing::instrument(skip(self), level = "debug")] pub fn lazy_load_mark_sent( - &self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, lazy_load: HashSet, count: PduCount, + &self, + user_id: &UserId, + device_id: &DeviceId, + room_id: &RoomId, + lazy_load: HashSet, + count: PduCount, ) { let key = (user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), count); @@ -72,7 +81,13 @@ pub fn lazy_load_mark_sent( #[implement(Service)] #[tracing::instrument(skip(self), level = "debug")] -pub fn lazy_load_confirm_delivery(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId, since: PduCount) { +pub fn lazy_load_confirm_delivery( + &self, + user_id: &UserId, + device_id: &DeviceId, + room_id: &RoomId, + since: PduCount, +) { let key = (user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), since); let Some(user_ids) = self.lazy_load_waiting.lock().expect("locked").remove(&key) else { diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 8f65eec3..6d5a85a0 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -58,7 +58,9 @@ pub async fn exists(&self, room_id: &RoomId) -> bool { } #[implement(Service)] -pub fn iter_ids(&self) -> impl Stream + Send + '_ { self.db.roomid_shortroomid.keys().ignore_err() } +pub fn iter_ids(&self) -> impl Stream + Send + '_ { + self.db.roomid_shortroomid.keys().ignore_err() +} #[implement(Service)] #[inline] @@ -81,12 +83,18 @@ pub fn ban_room(&self, room_id: &RoomId, banned: bool) { } #[implement(Service)] -pub fn list_banned_rooms(&self) -> impl Stream + Send + '_ { self.db.bannedroomids.keys().ignore_err() } +pub fn list_banned_rooms(&self) -> impl Stream + Send + '_ { + self.db.bannedroomids.keys().ignore_err() +} #[implement(Service)] #[inline] -pub async fn is_disabled(&self, room_id: &RoomId) -> bool { self.db.disabledroomids.get(room_id).await.is_ok() } +pub async fn is_disabled(&self, room_id: &RoomId) -> bool { + self.db.disabledroomids.get(room_id).await.is_ok() +} #[implement(Service)] #[inline] -pub async fn is_banned(&self, room_id: &RoomId) -> bool { self.db.bannedroomids.get(room_id).await.is_ok() } +pub async fn is_banned(&self, room_id: &RoomId) -> bool { + self.db.bannedroomids.get(room_id).await.is_ok() +} diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index be9e4ba9..01950975 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -56,26 +56,27 @@ impl Data { } pub(super) fn get_relations<'a>( - &'a self, user_id: &'a UserId, shortroomid: ShortRoomId, target: ShortEventId, from: PduCount, dir: Direction, + &'a self, + user_id: &'a UserId, + shortroomid: ShortRoomId, + target: ShortEventId, + from: PduCount, + dir: Direction, ) -> impl Stream + Send + '_ { let mut current = ArrayVec::::new(); current.extend(target.to_be_bytes()); current.extend(from.saturating_inc(dir).into_unsigned().to_be_bytes()); let current = current.as_slice(); match dir { - Direction::Forward => self.tofrom_relation.raw_keys_from(current).boxed(), - Direction::Backward => self.tofrom_relation.rev_raw_keys_from(current).boxed(), + | Direction::Forward => self.tofrom_relation.raw_keys_from(current).boxed(), + | Direction::Backward => self.tofrom_relation.rev_raw_keys_from(current).boxed(), } .ignore_err() .ready_take_while(move |key| key.starts_with(&target.to_be_bytes())) .map(|to_from| u64_from_u8(&to_from[8..16])) .map(PduCount::from_unsigned) .wide_filter_map(move |shorteventid| async move { - let pdu_id: RawPduId = PduId { - shortroomid, - shorteventid, - } - .into(); + let pdu_id: RawPduId = PduId { shortroomid, shorteventid }.into(); let mut pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await.ok()?; @@ -99,7 +100,9 @@ impl Data { self.referencedevents.qry(&key).await.is_ok() } - pub(super) fn mark_event_soft_failed(&self, event_id: &EventId) { self.softfailedeventids.insert(event_id, []); } + pub(super) fn mark_event_soft_failed(&self, event_id: &EventId) { + self.softfailedeventids.insert(event_id, []); + } pub(super) async fn is_event_soft_failed(&self, event_id: &EventId) -> bool { self.softfailedeventids.get(event_id).await.is_ok() diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 23c6d90b..ac70dbc3 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -36,8 +36,8 @@ impl Service { #[tracing::instrument(skip(self, from, to), level = "debug")] pub fn add_relation(&self, from: PduCount, to: PduCount) { match (from, to) { - (PduCount::Normal(f), PduCount::Normal(t)) => self.db.add_relation(f, t), - _ => { + | (PduCount::Normal(f), PduCount::Normal(t)) => self.db.add_relation(f, t), + | _ => { // TODO: Relations with backfilled pdus }, } @@ -45,15 +45,21 @@ impl Service { #[allow(clippy::too_many_arguments)] pub async fn get_relations( - &self, user_id: &UserId, room_id: &RoomId, target: &EventId, from: PduCount, limit: usize, max_depth: u8, + &self, + user_id: &UserId, + room_id: &RoomId, + target: &EventId, + from: PduCount, + limit: usize, + max_depth: u8, dir: Direction, ) -> Vec { let room_id = self.services.short.get_or_create_shortroomid(room_id).await; let target = match self.services.timeline.get_pdu_count(target).await { - Ok(PduCount::Normal(c)) => c, + | Ok(PduCount::Normal(c)) => c, // TODO: Support backfilled relations - _ => 0, // This will result in an empty iterator + | _ => 0, // This will result in an empty iterator }; let mut pdus: Vec<_> = self @@ -66,9 +72,9 @@ impl Service { 'limit: while let Some(stack_pdu) = stack.pop() { let target = match stack_pdu.0 .0 { - PduCount::Normal(c) => c, + | PduCount::Normal(c) => c, // TODO: Support backfilled relations - PduCount::Backfilled(_) => 0, // This will result in an empty iterator + | PduCount::Backfilled(_) => 0, // This will result in an empty iterator }; let relations: Vec<_> = self @@ -106,7 +112,9 @@ impl Service { #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub fn mark_event_soft_failed(&self, event_id: &EventId) { self.db.mark_event_soft_failed(event_id) } + pub fn mark_event_soft_failed(&self, event_id: &EventId) { + self.db.mark_event_soft_failed(event_id); + } #[inline] #[tracing::instrument(skip(self), level = "debug")] diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 03d04b65..c21ad36c 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -40,7 +40,12 @@ impl Data { } } - pub(super) async fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) { + pub(super) async fn readreceipt_update( + &self, + user_id: &UserId, + room_id: &RoomId, + event: &ReceiptEvent, + ) { // Remove old entry let last_possible_key = (room_id, u64::MAX); self.readreceiptid_readreceipt @@ -57,7 +62,9 @@ impl Data { } pub(super) fn readreceipts_since<'a>( - &'a self, room_id: &'a RoomId, since: u64, + &'a self, + room_id: &'a RoomId, + since: u64, ) -> impl Stream> + Send + 'a { type Key<'a> = (&'a RoomId, u64, &'a UserId); type KeyVal<'a> = (Key<'a>, CanonicalJsonObject); @@ -87,12 +94,20 @@ impl Data { self.roomuserid_lastprivatereadupdate.put(key, next_count); } - pub(super) async fn private_read_get_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + pub(super) async fn private_read_get_count( + &self, + room_id: &RoomId, + user_id: &UserId, + ) -> Result { let key = (room_id, user_id); self.roomuserid_privateread.qry(&key).await.deserialized() } - pub(super) async fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> u64 { + pub(super) async fn last_privateread_update( + &self, + user_id: &UserId, + room_id: &RoomId, + ) -> u64 { let key = (room_id, user_id); self.roomuserid_lastprivatereadupdate .qry(&key) diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index d92b9759..53e64957 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -44,7 +44,12 @@ impl crate::Service for Service { impl Service { /// Replaces the previous read receipt. - pub async fn readreceipt_update(&self, user_id: &UserId, room_id: &RoomId, event: &ReceiptEvent) { + pub async fn readreceipt_update( + &self, + user_id: &UserId, + room_id: &RoomId, + event: &ReceiptEvent, + ) { self.db.readreceipt_update(user_id, room_id, event).await; self.services .sending @@ -54,23 +59,21 @@ impl Service { } /// Gets the latest private read receipt from the user in the room - pub async fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result> { - let pdu_count = self - .private_read_get_count(room_id, user_id) - .map_err(|e| err!(Database(warn!("No private read receipt was set in {room_id}: {e}")))); - let shortroomid = self - .services - .short - .get_shortroomid(room_id) - .map_err(|e| err!(Database(warn!("Short room ID does not exist in database for {room_id}: {e}")))); + pub async fn private_read_get( + &self, + room_id: &RoomId, + user_id: &UserId, + ) -> Result> { + let pdu_count = self.private_read_get_count(room_id, user_id).map_err(|e| { + err!(Database(warn!("No private read receipt was set in {room_id}: {e}"))) + }); + let shortroomid = self.services.short.get_shortroomid(room_id).map_err(|e| { + err!(Database(warn!("Short room ID does not exist in database for {room_id}: {e}"))) + }); let (pdu_count, shortroomid) = try_join!(pdu_count, shortroomid)?; let shorteventid = PduCount::Normal(pdu_count); - let pdu_id: RawPduId = PduId { - shortroomid, - shorteventid, - } - .into(); + let pdu_id: RawPduId = PduId { shortroomid, shorteventid }.into(); let pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await?; @@ -80,21 +83,17 @@ impl Service { event_id, BTreeMap::from_iter([( ruma::events::receipt::ReceiptType::ReadPrivate, - BTreeMap::from_iter([( - user_id, - ruma::events::receipt::Receipt { - ts: None, // TODO: start storing the timestamp so we can return one - thread: ruma::events::receipt::ReceiptThread::Unthreaded, - }, - )]), + BTreeMap::from_iter([(user_id, ruma::events::receipt::Receipt { + ts: None, // TODO: start storing the timestamp so we can return one + thread: ruma::events::receipt::ReceiptThread::Unthreaded, + })]), )]), )]); let receipt_event_content = ReceiptEventContent(content); - let receipt_sync_event = SyncEphemeralRoomEvent { - content: receipt_event_content, - }; + let receipt_sync_event = SyncEphemeralRoomEvent { content: receipt_event_content }; - let event = serde_json::value::to_raw_value(&receipt_sync_event).expect("receipt created manually"); + let event = serde_json::value::to_raw_value(&receipt_sync_event) + .expect("receipt created manually"); Ok(Raw::from_json(event)) } @@ -104,7 +103,9 @@ impl Service { #[inline] #[tracing::instrument(skip(self), level = "debug")] pub fn readreceipts_since<'a>( - &'a self, room_id: &'a RoomId, since: u64, + &'a self, + room_id: &'a RoomId, + since: u64, ) -> impl Stream> + Send + 'a { self.db.readreceipts_since(room_id, since) } @@ -119,7 +120,11 @@ impl Service { /// Returns the private read marker PDU count. #[inline] #[tracing::instrument(skip(self), level = "debug")] - pub async fn private_read_get_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + pub async fn private_read_get_count( + &self, + room_id: &RoomId, + user_id: &UserId, + ) -> Result { self.db.private_read_get_count(room_id, user_id).await } @@ -137,7 +142,9 @@ where { let mut json = BTreeMap::new(); for value in receipts { - let receipt = serde_json::from_str::>(value.json().get()); + let receipt = serde_json::from_str::>( + value.json().get(), + ); if let Ok(value) = receipt { for (event, receipt) in value.content { json.insert(event, receipt); @@ -149,9 +156,7 @@ where let content = ReceiptEventContent::from_iter(json); Raw::from_json( - serde_json::value::to_raw_value(&SyncEphemeralRoomEvent { - content, - }) - .expect("received valid json"), + serde_json::value::to_raw_value(&SyncEphemeralRoomEvent { content }) + .expect("received valid json"), ) } diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 025fc42a..35cfd444 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -49,18 +49,18 @@ pub struct RoomQuery<'a> { type TokenId = ArrayVec; -const TOKEN_ID_MAX_LEN: usize = size_of::() + WORD_MAX_LEN + 1 + size_of::(); +const TOKEN_ID_MAX_LEN: usize = + size_of::() + WORD_MAX_LEN + 1 + size_of::(); const WORD_MAX_LEN: usize = 50; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - db: Data { - tokenids: args.db["tokenids"].clone(), - }, + db: Data { tokenids: args.db["tokenids"].clone() }, services: Services { short: args.depend::("rooms::short"), - state_accessor: args.depend::("rooms::state_accessor"), + state_accessor: args + .depend::("rooms::state_accessor"), timeline: args.depend::("rooms::timeline"), }, })) @@ -103,7 +103,8 @@ pub fn deindex_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_b #[implement(Service)] pub async fn search_pdus<'a>( - &'a self, query: &'a RoomQuery<'a>, + &'a self, + query: &'a RoomQuery<'a>, ) -> Result<(usize, impl Stream + Send + 'a)> { let pdu_ids: Vec<_> = self.search_pdu_ids(query).await?.collect().await; @@ -136,7 +137,10 @@ pub async fn search_pdus<'a>( // result is modeled as a stream such that callers don't have to be refactored // though an additional async/wrap still exists for now #[implement(Service)] -pub async fn search_pdu_ids(&self, query: &RoomQuery<'_>) -> Result + Send + '_> { +pub async fn search_pdu_ids( + &self, + query: &RoomQuery<'_>, +) -> Result + Send + '_> { let shortroomid = self.services.short.get_shortroomid(query.room_id).await?; let pdu_ids = self.search_pdu_ids_query_room(query, shortroomid).await; @@ -147,7 +151,11 @@ pub async fn search_pdu_ids(&self, query: &RoomQuery<'_>) -> Result, shortroomid: ShortRoomId) -> Vec> { +async fn search_pdu_ids_query_room( + &self, + query: &RoomQuery<'_>, + shortroomid: ShortRoomId, +) -> Vec> { tokenize(&query.criteria.search_term) .stream() .wide_then(|word| async move { @@ -162,7 +170,9 @@ async fn search_pdu_ids_query_room(&self, query: &RoomQuery<'_>, shortroomid: Sh /// Iterate over PduId's containing a word #[implement(Service)] fn search_pdu_ids_query_words<'a>( - &'a self, shortroomid: ShortRoomId, word: &'a str, + &'a self, + shortroomid: ShortRoomId, + word: &'a str, ) -> impl Stream + Send + '_ { self.search_pdu_ids_query_word(shortroomid, word) .map(move |key| -> RawPduId { @@ -173,7 +183,11 @@ fn search_pdu_ids_query_words<'a>( /// Iterate over raw database results for a word #[implement(Service)] -fn search_pdu_ids_query_word(&self, shortroomid: ShortRoomId, word: &str) -> impl Stream> + Send + '_ { +fn search_pdu_ids_query_word( + &self, + shortroomid: ShortRoomId, + word: &str, +) -> impl Stream> + Send + '_ { // rustc says const'ing this not yet stable let end_id: RawPduId = PduId { shortroomid, diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 465ce1a9..f814411b 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -60,7 +60,10 @@ pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEvent } #[implement(Service)] -pub fn multi_get_or_create_shorteventid<'a, I>(&'a self, event_ids: I) -> impl Stream + Send + '_ +pub fn multi_get_or_create_shorteventid<'a, I>( + &'a self, + event_ids: I, +) -> impl Stream + Send + '_ where I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, ::Item: AsRef<[u8]> + Send + Sync + 'a, @@ -72,8 +75,8 @@ where event_ids.next().map(|event_id| (event_id, result)) }) .map(|(event_id, result)| match result { - Ok(ref short) => utils::u64_from_u8(short), - Err(_) => self.create_shorteventid(event_id), + | Ok(ref short) => utils::u64_from_u8(short), + | Err(_) => self.create_shorteventid(event_id), }) } @@ -104,7 +107,11 @@ pub async fn get_shorteventid(&self, event_id: &EventId) -> Result } #[implement(Service)] -pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> ShortStateKey { +pub async fn get_or_create_shortstatekey( + &self, + event_type: &StateEventType, + state_key: &str, +) -> ShortStateKey { const BUFSIZE: usize = size_of::(); if let Ok(shortstatekey) = self.get_shortstatekey(event_type, state_key).await { @@ -127,7 +134,11 @@ pub async fn get_or_create_shortstatekey(&self, event_type: &StateEventType, sta } #[implement(Service)] -pub async fn get_shortstatekey(&self, event_type: &StateEventType, state_key: &str) -> Result { +pub async fn get_shortstatekey( + &self, + event_type: &StateEventType, + state_key: &str, +) -> Result { let key = (event_type, state_key); self.db .statekey_shortstatekey @@ -153,7 +164,10 @@ where } #[implement(Service)] -pub fn multi_get_eventid_from_short<'a, Id, I>(&'a self, shorteventid: I) -> impl Stream> + Send + 'a +pub fn multi_get_eventid_from_short<'a, Id, I>( + &'a self, + shorteventid: I, +) -> impl Stream> + Send + 'a where I: Iterator + Send + 'a, Id: for<'de> Deserialize<'de> + Sized + ToOwned + 'a, @@ -168,7 +182,10 @@ where } #[implement(Service)] -pub async fn get_statekey_from_short(&self, shortstatekey: ShortStateKey) -> Result<(StateEventType, String)> { +pub async fn get_statekey_from_short( + &self, + shortstatekey: ShortStateKey, +) -> Result<(StateEventType, String)> { const BUFSIZE: usize = size_of::(); self.db diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index aa523871..d60c4c9e 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -125,7 +125,8 @@ enum Identifier<'a> { pub struct Service { services: Services, - pub roomid_spacehierarchy_cache: Mutex>>, + pub roomid_spacehierarchy_cache: + Mutex>>, } struct Services { @@ -145,11 +146,13 @@ impl crate::Service for Service { let cache_size = cache_size * config.cache_capacity_modifier; Ok(Arc::new(Self { services: Services { - state_accessor: args.depend::("rooms::state_accessor"), + state_accessor: args + .depend::("rooms::state_accessor"), state_cache: args.depend::("rooms::state_cache"), state: args.depend::("rooms::state"), short: args.depend::("rooms::short"), - event_handler: args.depend::("rooms::event_handler"), + event_handler: args + .depend::("rooms::event_handler"), timeline: args.depend::("rooms::timeline"), sending: args.depend::("sending"), }, @@ -166,28 +169,37 @@ impl Service { /// Errors if the room does not exist, so a check if the room exists should /// be done pub async fn get_federation_hierarchy( - &self, room_id: &RoomId, server_name: &ServerName, suggested_only: bool, + &self, + room_id: &RoomId, + server_name: &ServerName, + suggested_only: bool, ) -> Result { match self - .get_summary_and_children_local(&room_id.to_owned(), Identifier::ServerName(server_name)) + .get_summary_and_children_local( + &room_id.to_owned(), + Identifier::ServerName(server_name), + ) .await? { - Some(SummaryAccessibility::Accessible(room)) => { + | Some(SummaryAccessibility::Accessible(room)) => { let mut children = Vec::new(); let mut inaccessible_children = Vec::new(); for (child, _via) in get_parent_children_via(&room, suggested_only) { match self - .get_summary_and_children_local(&child, Identifier::ServerName(server_name)) + .get_summary_and_children_local( + &child, + Identifier::ServerName(server_name), + ) .await? { - Some(SummaryAccessibility::Accessible(summary)) => { + | Some(SummaryAccessibility::Accessible(summary)) => { children.push((*summary).into()); }, - Some(SummaryAccessibility::Inaccessible) => { + | Some(SummaryAccessibility::Inaccessible) => { inaccessible_children.push(child); }, - None => (), + | None => (), } } @@ -197,16 +209,18 @@ impl Service { inaccessible_children, }) }, - Some(SummaryAccessibility::Inaccessible) => { - Err(Error::BadRequest(ErrorKind::NotFound, "The requested room is inaccessible")) - }, - None => Err(Error::BadRequest(ErrorKind::NotFound, "The requested room was not found")), + | Some(SummaryAccessibility::Inaccessible) => + Err(Error::BadRequest(ErrorKind::NotFound, "The requested room is inaccessible")), + | None => + Err(Error::BadRequest(ErrorKind::NotFound, "The requested room was not found")), } } /// Gets the summary of a space using solely local information async fn get_summary_and_children_local( - &self, current_room: &OwnedRoomId, identifier: Identifier<'_>, + &self, + current_room: &OwnedRoomId, + identifier: Identifier<'_>, ) -> Result> { if let Some(cached) = self .roomid_spacehierarchy_cache @@ -241,9 +255,7 @@ impl Service { if let Ok(summary) = summary { self.roomid_spacehierarchy_cache.lock().await.insert( current_room.clone(), - Some(CachedSpaceHierarchySummary { - summary: summary.clone(), - }), + Some(CachedSpaceHierarchySummary { summary: summary.clone() }), ); Ok(Some(SummaryAccessibility::Accessible(Box::new(summary)))) @@ -258,20 +270,21 @@ impl Service { /// Gets the summary of a space using solely federation #[tracing::instrument(skip(self))] async fn get_summary_and_children_federation( - &self, current_room: &OwnedRoomId, suggested_only: bool, user_id: &UserId, via: &[OwnedServerName], + &self, + current_room: &OwnedRoomId, + suggested_only: bool, + user_id: &UserId, + via: &[OwnedServerName], ) -> Result> { for server in via { debug_info!("Asking {server} for /hierarchy"); let Ok(response) = self .services .sending - .send_federation_request( - server, - federation::space::get_hierarchy::v1::Request { - room_id: current_room.to_owned(), - suggested_only, - }, - ) + .send_federation_request(server, federation::space::get_hierarchy::v1::Request { + room_id: current_room.to_owned(), + suggested_only, + }) .await else { continue; @@ -282,9 +295,7 @@ impl Service { self.roomid_spacehierarchy_cache.lock().await.insert( current_room.clone(), - Some(CachedSpaceHierarchySummary { - summary: summary.clone(), - }), + Some(CachedSpaceHierarchySummary { summary: summary.clone() }), ); for child in response.children { @@ -356,7 +367,11 @@ impl Service { /// Gets the summary of a space using either local or remote (federation) /// sources async fn get_summary_and_children_client( - &self, current_room: &OwnedRoomId, suggested_only: bool, user_id: &UserId, via: &[OwnedServerName], + &self, + current_room: &OwnedRoomId, + suggested_only: bool, + user_id: &UserId, + via: &[OwnedServerName], ) -> Result> { if let Ok(Some(response)) = self .get_summary_and_children_local(current_room, Identifier::UserId(user_id)) @@ -370,7 +385,9 @@ impl Service { } async fn get_room_summary( - &self, current_room: &OwnedRoomId, children_state: Vec>, + &self, + current_room: &OwnedRoomId, + children_state: Vec>, identifier: &Identifier<'_>, ) -> Result { let room_id: &RoomId = current_room; @@ -388,12 +405,20 @@ impl Service { .allowed_room_ids(join_rule.clone()); if !self - .is_accessible_child(current_room, &join_rule.clone().into(), identifier, &allowed_room_ids) + .is_accessible_child( + current_room, + &join_rule.clone().into(), + identifier, + &allowed_room_ids, + ) .await { debug_info!("User is not allowed to see room {room_id}"); // This error will be caught later - return Err(Error::BadRequest(ErrorKind::forbidden(), "User is not allowed to see the room")); + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "User is not allowed to see the room", + )); } Ok(SpaceHierarchyParentSummary { @@ -446,7 +471,12 @@ impl Service { } pub async fn get_client_hierarchy( - &self, sender_user: &UserId, room_id: &RoomId, limit: usize, short_room_ids: Vec, max_depth: u64, + &self, + sender_user: &UserId, + room_id: &RoomId, + limit: usize, + short_room_ids: Vec, + max_depth: u64, suggested_only: bool, ) -> Result { let mut parents = VecDeque::new(); @@ -454,27 +484,30 @@ impl Service { // Don't start populating the results if we have to start at a specific room. let mut populate_results = short_room_ids.is_empty(); - let mut stack = vec![vec![( - room_id.to_owned(), - match room_id.server_name() { - Some(server_name) => vec![server_name.into()], - None => vec![], - }, - )]]; + let mut stack = vec![vec![(room_id.to_owned(), match room_id.server_name() { + | Some(server_name) => vec![server_name.into()], + | None => vec![], + })]]; let mut results = Vec::with_capacity(limit); - while let Some((current_room, via)) = { next_room_to_traverse(&mut stack, &mut parents) } { + while let Some((current_room, via)) = { next_room_to_traverse(&mut stack, &mut parents) } + { if results.len() >= limit { break; } match ( - self.get_summary_and_children_client(¤t_room, suggested_only, sender_user, &via) - .await?, + self.get_summary_and_children_client( + ¤t_room, + suggested_only, + sender_user, + &via, + ) + .await?, current_room == room_id, ) { - (Some(SummaryAccessibility::Accessible(summary)), _) => { + | (Some(SummaryAccessibility::Accessible(summary)), _) => { let mut children: Vec<(OwnedRoomId, Vec)> = get_parent_children_via(&summary, suggested_only) .into_iter() @@ -493,7 +526,9 @@ impl Service { self.services .short .get_shortroomid(room) - .map_ok(|short| Some(&short) != short_room_ids.get(parents.len())) + .map_ok(|short| { + Some(&short) != short_room_ids.get(parents.len()) + }) .unwrap_or_else(|_| false) }) .map(Clone::clone) @@ -525,14 +560,20 @@ impl Service { // Root room in the space hierarchy, we return an error // if this one fails. }, - (Some(SummaryAccessibility::Inaccessible), true) => { - return Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room is inaccessible")); + | (Some(SummaryAccessibility::Inaccessible), true) => { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "The requested room is inaccessible", + )); }, - (None, true) => { - return Err(Error::BadRequest(ErrorKind::forbidden(), "The requested room was not found")); + | (None, true) => { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "The requested room was not found", + )); }, // Just ignore other unavailable rooms - (None | Some(SummaryAccessibility::Inaccessible), false) => (), + | (None | Some(SummaryAccessibility::Inaccessible), false) => (), } } @@ -544,15 +585,19 @@ impl Service { let short_room_ids: Vec<_> = parents .iter() .stream() - .filter_map(|room_id| async move { self.services.short.get_shortroomid(room_id).await.ok() }) + .filter_map(|room_id| async move { + self.services.short.get_shortroomid(room_id).await.ok() + }) .collect() .await; Some( PaginationToken { short_room_ids, - limit: UInt::new(max_depth).expect("When sent in request it must have been valid UInt"), - max_depth: UInt::new(max_depth).expect("When sent in request it must have been valid UInt"), + limit: UInt::new(max_depth) + .expect("When sent in request it must have been valid UInt"), + max_depth: UInt::new(max_depth) + .expect("When sent in request it must have been valid UInt"), suggested_only, } .to_string(), @@ -566,9 +611,12 @@ impl Service { /// Simply returns the stripped m.space.child events of a room async fn get_stripped_space_child_events( - &self, room_id: &RoomId, + &self, + room_id: &RoomId, ) -> Result>>, Error> { - let Ok(current_shortstatehash) = self.services.state.get_room_shortstatehash(room_id).await else { + let Ok(current_shortstatehash) = + self.services.state.get_room_shortstatehash(room_id).await + else { return Ok(None); }; @@ -581,18 +629,17 @@ impl Service { let mut children_pdus = Vec::with_capacity(state.len()); for (key, id) in state { - let (event_type, state_key) = self.services.short.get_statekey_from_short(key).await?; + let (event_type, state_key) = + self.services.short.get_statekey_from_short(key).await?; if event_type != StateEventType::SpaceChild { continue; } - let pdu = self - .services - .timeline - .get_pdu(&id) - .await - .map_err(|e| err!(Database("Event {id:?} in space state not found: {e:?}")))?; + let pdu = + self.services.timeline.get_pdu(&id).await.map_err(|e| { + err!(Database("Event {id:?} in space state not found: {e:?}")) + })?; if let Ok(content) = pdu.get_content::() { if content.via.is_empty() { @@ -610,11 +657,14 @@ impl Service { /// With the given identifier, checks if a room is accessable async fn is_accessible_child( - &self, current_room: &OwnedRoomId, join_rule: &SpaceRoomJoinRule, identifier: &Identifier<'_>, + &self, + current_room: &OwnedRoomId, + join_rule: &SpaceRoomJoinRule, + identifier: &Identifier<'_>, allowed_room_ids: &Vec, ) -> bool { match identifier { - Identifier::ServerName(server_name) => { + | Identifier::ServerName(server_name) => { // Checks if ACLs allow for the server to participate if self .services @@ -626,7 +676,7 @@ impl Service { return false; } }, - Identifier::UserId(user_id) => { + | Identifier::UserId(user_id) => { if self .services .state_cache @@ -642,16 +692,18 @@ impl Service { }, } match &join_rule { - SpaceRoomJoinRule::Public | SpaceRoomJoinRule::Knock | SpaceRoomJoinRule::KnockRestricted => true, - SpaceRoomJoinRule::Restricted => { + | SpaceRoomJoinRule::Public + | SpaceRoomJoinRule::Knock + | SpaceRoomJoinRule::KnockRestricted => true, + | SpaceRoomJoinRule::Restricted => { for room in allowed_room_ids { match identifier { - Identifier::UserId(user) => { + | Identifier::UserId(user) => { if self.services.state_cache.is_joined(user, room).await { return true; } }, - Identifier::ServerName(server) => { + | Identifier::ServerName(server) => { if self.services.state_cache.server_in_room(server, room).await { return true; } @@ -661,7 +713,7 @@ impl Service { false }, // Invite only, Private, or Custom join rule - _ => false, + | _ => false, } } } @@ -737,7 +789,8 @@ fn summary_to_chunk(summary: SpaceHierarchyParentSummary) -> SpaceHierarchyRooms /// Returns the children of a SpaceHierarchyParentSummary, making use of the /// children_state field fn get_parent_children_via( - parent: &SpaceHierarchyParentSummary, suggested_only: bool, + parent: &SpaceHierarchyParentSummary, + suggested_only: bool, ) -> Vec<(OwnedRoomId, Vec)> { parent .children_state @@ -755,7 +808,8 @@ fn get_parent_children_via( } fn next_room_to_traverse( - stack: &mut Vec)>>, parents: &mut VecDeque, + stack: &mut Vec)>>, + parents: &mut VecDeque, ) -> Option<(OwnedRoomId, Vec)> { while stack.last().is_some_and(Vec::is_empty) { stack.pop(); diff --git a/src/service/rooms/spaces/tests.rs b/src/service/rooms/spaces/tests.rs index 71640035..b4c387d7 100644 --- a/src/service/rooms/spaces/tests.rs +++ b/src/service/rooms/spaces/tests.rs @@ -69,18 +69,15 @@ fn get_summary_children() { } .into(); - assert_eq!( - get_parent_children_via(&summary, false), - vec![ - (owned_room_id!("!foo:example.org"), vec![owned_server_name!("example.org")]), - (owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")]), - (owned_room_id!("!baz:example.org"), vec![owned_server_name!("example.org")]) - ] - ); - assert_eq!( - get_parent_children_via(&summary, true), - vec![(owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")])] - ); + assert_eq!(get_parent_children_via(&summary, false), vec![ + (owned_room_id!("!foo:example.org"), vec![owned_server_name!("example.org")]), + (owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")]), + (owned_room_id!("!baz:example.org"), vec![owned_server_name!("example.org")]) + ]); + assert_eq!(get_parent_children_via(&summary, true), vec![( + owned_room_id!("!bar:example.org"), + vec![owned_server_name!("example.org")] + )]); } #[test] diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index e083ed34..a7f79e94 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -16,7 +16,9 @@ use conduwuit::{ warn, PduEvent, Result, }; use database::{Deserialized, Ignore, Interfix, Map}; -use futures::{future::join_all, pin_mut, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{ + future::join_all, pin_mut, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, +}; use ruma::{ events::{ room::{create::RoomCreateEventContent, member::RoomMemberEventContent}, @@ -70,8 +72,10 @@ impl crate::Service for Service { short: args.depend::("rooms::short"), spaces: args.depend::("rooms::spaces"), state_cache: args.depend::("rooms::state_cache"), - state_accessor: args.depend::("rooms::state_accessor"), - state_compressor: args.depend::("rooms::state_compressor"), + state_accessor: args + .depend::("rooms::state_accessor"), + state_compressor: args + .depend::("rooms::state_compressor"), timeline: args.depend::("rooms::timeline"), }, db: Data { @@ -100,7 +104,8 @@ impl Service { shortstatehash: u64, statediffnew: Arc>, _statediffremoved: Arc>, - state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room state + * mutex */ ) -> Result { let event_ids = statediffnew .iter() @@ -120,8 +125,9 @@ impl Service { }; match pdu.kind { - TimelineEventType::RoomMember => { - let Some(user_id) = pdu.state_key.as_ref().map(UserId::parse).flat_ok() else { + | TimelineEventType::RoomMember => { + let Some(user_id) = pdu.state_key.as_ref().map(UserId::parse).flat_ok() + else { continue; }; @@ -131,10 +137,18 @@ impl Service { self.services .state_cache - .update_membership(room_id, &user_id, membership_event, &pdu.sender, None, None, false) + .update_membership( + room_id, + &user_id, + membership_event, + &pdu.sender, + None, + None, + false, + ) .await?; }, - TimelineEventType::SpaceChild => { + | TimelineEventType::SpaceChild => { self.services .spaces .roomid_spacehierarchy_cache @@ -142,7 +156,7 @@ impl Service { .await .remove(&pdu.room_id); }, - _ => continue, + | _ => continue, } } @@ -159,7 +173,10 @@ impl Service { /// to `stateid_pduid` and adds the incoming event to `eventid_statehash`. #[tracing::instrument(skip(self, state_ids_compressed), level = "debug")] pub async fn set_event_state( - &self, event_id: &EventId, room_id: &RoomId, state_ids_compressed: Arc>, + &self, + event_id: &EventId, + room_id: &RoomId, + state_ids_compressed: Arc>, ) -> Result { const KEY_LEN: usize = size_of::(); const VAL_LEN: usize = size_of::(); @@ -190,22 +207,23 @@ impl Service { Vec::new() }; - let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew: HashSet<_> = state_ids_compressed - .difference(&parent_stateinfo.full_state) - .copied() - .collect(); + let (statediffnew, statediffremoved) = + if let Some(parent_stateinfo) = states_parents.last() { + let statediffnew: HashSet<_> = state_ids_compressed + .difference(&parent_stateinfo.full_state) + .copied() + .collect(); - let statediffremoved: HashSet<_> = parent_stateinfo - .full_state - .difference(&state_ids_compressed) - .copied() - .collect(); + let statediffremoved: HashSet<_> = parent_stateinfo + .full_state + .difference(&state_ids_compressed) + .copied() + .collect(); - (Arc::new(statediffnew), Arc::new(statediffremoved)) - } else { - (state_ids_compressed, Arc::new(HashSet::new())) - }; + (Arc::new(statediffnew), Arc::new(statediffremoved)) + } else { + (state_ids_compressed, Arc::new(HashSet::new())) + }; self.services.state_compressor.save_state_from_diff( shortstatehash, statediffnew, @@ -338,7 +356,8 @@ impl Service { &self, room_id: &RoomId, shortstatehash: u64, - _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room + * state mutex */ ) { const BUFSIZE: usize = size_of::(); @@ -366,7 +385,10 @@ impl Service { .deserialized() } - pub fn get_forward_extremities<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + '_ { + pub fn get_forward_extremities<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + '_ { let prefix = (room_id, Interfix); self.db @@ -380,7 +402,8 @@ impl Service { &self, room_id: &RoomId, event_ids: Vec, - _state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex + _state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room + * state mutex */ ) { let prefix = (room_id, Interfix); self.db @@ -399,26 +422,33 @@ impl Service { /// This fetches auth events from the current state. #[tracing::instrument(skip(self, content), level = "debug")] pub async fn get_auth_events( - &self, room_id: &RoomId, kind: &TimelineEventType, sender: &UserId, state_key: Option<&str>, + &self, + room_id: &RoomId, + kind: &TimelineEventType, + sender: &UserId, + state_key: Option<&str>, content: &serde_json::value::RawValue, ) -> Result>> { let Ok(shortstatehash) = self.get_room_shortstatehash(room_id).await else { return Ok(HashMap::new()); }; - let mut sauthevents: HashMap<_, _> = state_res::auth_types_for_event(kind, sender, state_key, content)? - .iter() - .stream() - .broad_filter_map(|(event_type, state_key)| { - self.services - .short - .get_shortstatekey(event_type, state_key) - .map_ok(move |ssk| (ssk, (event_type, state_key))) - .map(Result::ok) - }) - .map(|(ssk, (event_type, state_key))| (ssk, (event_type.to_owned(), state_key.to_owned()))) - .collect() - .await; + let mut sauthevents: HashMap<_, _> = + state_res::auth_types_for_event(kind, sender, state_key, content)? + .iter() + .stream() + .broad_filter_map(|(event_type, state_key)| { + self.services + .short + .get_shortstatekey(event_type, state_key) + .map_ok(move |ssk| (ssk, (event_type, state_key))) + .map(Result::ok) + }) + .map(|(ssk, (event_type, state_key))| { + (ssk, (event_type.to_owned(), state_key.to_owned())) + }) + .collect() + .await; let (state_keys, event_ids): (Vec<_>, Vec<_>) = self .services diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index 9947b036..d60e505e 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -39,14 +39,16 @@ impl Data { services: Services { short: args.depend::("rooms::short"), state: args.depend::("rooms::state"), - state_compressor: args.depend::("rooms::state_compressor"), + state_compressor: args + .depend::("rooms::state_compressor"), timeline: args.depend::("rooms::timeline"), }, } } pub(super) async fn state_full( - &self, shortstatehash: ShortStateHash, + &self, + shortstatehash: ShortStateHash, ) -> Result> { let state = self .state_full_pdus(shortstatehash) @@ -58,7 +60,10 @@ impl Data { Ok(state) } - pub(super) async fn state_full_pdus(&self, shortstatehash: ShortStateHash) -> Result> { + pub(super) async fn state_full_pdus( + &self, + shortstatehash: ShortStateHash, + ) -> Result> { let short_ids = self.state_full_shortids(shortstatehash).await?; let full_pdus = self @@ -66,16 +71,19 @@ impl Data { .short .multi_get_eventid_from_short(short_ids.iter().map(ref_at!(1))) .ready_filter_map(Result::ok) - .broad_filter_map( - |event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await.ok() }, - ) + .broad_filter_map(|event_id: OwnedEventId| async move { + self.services.timeline.get_pdu(&event_id).await.ok() + }) .collect() .await; Ok(full_pdus) } - pub(super) async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result> + pub(super) async fn state_full_ids( + &self, + shortstatehash: ShortStateHash, + ) -> Result> where Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, ::Owned: Borrow, @@ -96,7 +104,8 @@ impl Data { } pub(super) async fn state_full_shortids( - &self, shortstatehash: ShortStateHash, + &self, + shortstatehash: ShortStateHash, ) -> Result> { let shortids = self .services @@ -118,7 +127,10 @@ impl Data { /// Returns a single EventId from `room_id` with key /// (`event_type`,`state_key`). pub(super) async fn state_get_id( - &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, ) -> Result where Id: for<'de> Deserialize<'de> + Sized + ToOwned, @@ -155,10 +167,15 @@ impl Data { /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). pub(super) async fn state_get( - &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, ) -> Result { self.state_get_id(shortstatehash, event_type, state_key) - .and_then(|event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await }) + .and_then(|event_id: OwnedEventId| async move { + self.services.timeline.get_pdu(&event_id).await + }) .await } @@ -179,7 +196,8 @@ impl Data { /// Returns the full room state. pub(super) async fn room_state_full( - &self, room_id: &RoomId, + &self, + room_id: &RoomId, ) -> Result> { self.services .state @@ -203,7 +221,10 @@ impl Data { /// Returns a single EventId from `room_id` with key /// (`event_type`,`state_key`). pub(super) async fn room_state_get_id( - &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, ) -> Result where Id: for<'de> Deserialize<'de> + Sized + ToOwned, @@ -218,7 +239,10 @@ impl Data { /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). pub(super) async fn room_state_get( - &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, ) -> Result { self.services .state diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 798bcbed..d8093dd7 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -34,8 +34,8 @@ use ruma::{ }, room::RoomType, space::SpaceRoomJoinRule, - EventEncryptionAlgorithm, EventId, JsOption, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, - ServerName, UserId, + EventEncryptionAlgorithm, EventId, JsOption, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, + OwnedUserId, RoomId, ServerName, UserId, }; use serde::Deserialize; @@ -75,8 +75,12 @@ impl crate::Service for Service { timeline: args.depend::("rooms::timeline"), }, db: Data::new(&args), - server_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64(server_visibility_cache_capacity)?)), - user_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64(user_visibility_cache_capacity)?)), + server_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64( + server_visibility_cache_capacity, + )?)), + user_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64( + user_visibility_cache_capacity, + )?)), })) } @@ -102,7 +106,10 @@ impl Service { /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_full_ids(&self, shortstatehash: ShortStateHash) -> Result> + pub async fn state_full_ids( + &self, + shortstatehash: ShortStateHash, + ) -> Result> where Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, ::Owned: Borrow, @@ -112,13 +119,15 @@ impl Service { #[inline] pub async fn state_full_shortids( - &self, shortstatehash: ShortStateHash, + &self, + shortstatehash: ShortStateHash, ) -> Result> { self.db.state_full_shortids(shortstatehash).await } pub async fn state_full( - &self, shortstatehash: ShortStateHash, + &self, + shortstatehash: ShortStateHash, ) -> Result> { self.db.state_full(shortstatehash).await } @@ -127,7 +136,10 @@ impl Service { /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] pub async fn state_get_id( - &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, ) -> Result where Id: for<'de> Deserialize<'de> + Sized + ToOwned, @@ -142,7 +154,10 @@ impl Service { /// `state_key`). #[inline] pub async fn state_get( - &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, ) -> Result { self.db .state_get(shortstatehash, event_type, state_key) @@ -151,7 +166,10 @@ impl Service { /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). pub async fn state_get_content( - &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, ) -> Result where T: for<'de> Deserialize<'de>, @@ -162,7 +180,11 @@ impl Service { } /// Get membership for given user in state - async fn user_membership(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> MembershipState { + async fn user_membership( + &self, + shortstatehash: ShortStateHash, + user_id: &UserId, + ) -> MembershipState { self.state_get_content(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) .await .map_or(MembershipState::Leave, |c: RoomMemberEventContent| c.membership) @@ -185,7 +207,12 @@ impl Service { /// Whether a server is allowed to see an event through federation, based on /// the room's history_visibility at that event's state. #[tracing::instrument(skip_all, level = "trace")] - pub async fn server_can_see_event(&self, origin: &ServerName, room_id: &RoomId, event_id: &EventId) -> bool { + pub async fn server_can_see_event( + &self, + origin: &ServerName, + room_id: &RoomId, + event_id: &EventId, + ) -> bool { let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { return true; }; @@ -213,20 +240,20 @@ impl Service { .ready_filter(|member| member.server_name() == origin); let visibility = match history_visibility { - HistoryVisibility::WorldReadable | HistoryVisibility::Shared => true, - HistoryVisibility::Invited => { + | HistoryVisibility::WorldReadable | HistoryVisibility::Shared => true, + | HistoryVisibility::Invited => { // Allow if any member on requesting server was AT LEAST invited, else deny current_server_members .any(|member| self.user_was_invited(shortstatehash, member)) .await }, - HistoryVisibility::Joined => { + | HistoryVisibility::Joined => { // Allow if any member on requested server was joined, else deny current_server_members .any(|member| self.user_was_joined(shortstatehash, member)) .await }, - _ => { + | _ => { error!("Unknown history visibility {history_visibility}"); false }, @@ -243,7 +270,12 @@ impl Service { /// Whether a user is allowed to see an event, based on /// the room's history_visibility at that event's state. #[tracing::instrument(skip_all, level = "trace")] - pub async fn user_can_see_event(&self, user_id: &UserId, room_id: &RoomId, event_id: &EventId) -> bool { + pub async fn user_can_see_event( + &self, + user_id: &UserId, + room_id: &RoomId, + event_id: &EventId, + ) -> bool { let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { return true; }; @@ -267,17 +299,17 @@ impl Service { }); let visibility = match history_visibility { - HistoryVisibility::WorldReadable => true, - HistoryVisibility::Shared => currently_member, - HistoryVisibility::Invited => { + | HistoryVisibility::WorldReadable => true, + | HistoryVisibility::Shared => currently_member, + | HistoryVisibility::Invited => { // Allow if any member on requesting server was AT LEAST invited, else deny self.user_was_invited(shortstatehash, user_id).await }, - HistoryVisibility::Joined => { + | HistoryVisibility::Joined => { // Allow if any member on requested server was joined, else deny self.user_was_joined(shortstatehash, user_id).await }, - _ => { + | _ => { error!("Unknown history visibility {history_visibility}"); false }, @@ -307,9 +339,10 @@ impl Service { }); match history_visibility { - HistoryVisibility::Invited => self.services.state_cache.is_invited(user_id, room_id).await, - HistoryVisibility::WorldReadable => true, - _ => false, + | HistoryVisibility::Invited => + self.services.state_cache.is_invited(user_id, room_id).await, + | HistoryVisibility::WorldReadable => true, + | _ => false, } } @@ -320,7 +353,10 @@ impl Service { /// Returns the full room state. #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_full(&self, room_id: &RoomId) -> Result> { + pub async fn room_state_full( + &self, + room_id: &RoomId, + ) -> Result> { self.db.room_state_full(room_id).await } @@ -334,7 +370,10 @@ impl Service { /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] pub async fn room_state_get_id( - &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, ) -> Result where Id: for<'de> Deserialize<'de> + Sized + ToOwned, @@ -349,14 +388,20 @@ impl Service { /// `state_key`). #[tracing::instrument(skip(self), level = "debug")] pub async fn room_state_get( - &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, ) -> Result { self.db.room_state_get(room_id, event_type, state_key).await } /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). pub async fn room_state_get_content( - &self, room_id: &RoomId, event_type: &StateEventType, state_key: &str, + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, ) -> Result where T: for<'de> Deserialize<'de>, @@ -381,18 +426,29 @@ impl Service { JsOption::from_option(content) } - pub async fn get_member(&self, room_id: &RoomId, user_id: &UserId) -> Result { + pub async fn get_member( + &self, + room_id: &RoomId, + user_id: &UserId, + ) -> Result { self.room_state_get_content(room_id, &StateEventType::RoomMember, user_id.as_str()) .await } pub async fn user_can_invite( - &self, room_id: &RoomId, sender: &UserId, target_user: &UserId, state_lock: &RoomMutexGuard, + &self, + room_id: &RoomId, + sender: &UserId, + target_user: &UserId, + state_lock: &RoomMutexGuard, ) -> bool { self.services .timeline .create_hash_and_sign_event( - PduBuilder::state(target_user.into(), &RoomMemberEventContent::new(MembershipState::Invite)), + PduBuilder::state( + target_user.into(), + &RoomMemberEventContent::new(MembershipState::Invite), + ), sender, room_id, state_lock, @@ -405,7 +461,9 @@ impl Service { pub async fn is_world_readable(&self, room_id: &RoomId) -> bool { self.room_state_get_content(room_id, &StateEventType::RoomHistoryVisibility, "") .await - .map(|c: RoomHistoryVisibilityEventContent| c.history_visibility == HistoryVisibility::WorldReadable) + .map(|c: RoomHistoryVisibilityEventContent| { + c.history_visibility == HistoryVisibility::WorldReadable + }) .unwrap_or(false) } @@ -439,7 +497,11 @@ impl Service { /// If federation is true, it allows redaction events from any user of the /// same server as the original event sender pub async fn user_can_redact( - &self, redacts: &EventId, sender: &UserId, room_id: &RoomId, federation: bool, + &self, + redacts: &EventId, + sender: &UserId, + room_id: &RoomId, + federation: bool, ) -> Result { let redacting_event = self.services.timeline.get_pdu(redacts).await; @@ -451,7 +513,11 @@ impl Service { } if let Ok(pl_event_content) = self - .room_state_get_content::(room_id, &StateEventType::RoomPowerLevels, "") + .room_state_get_content::( + room_id, + &StateEventType::RoomPowerLevels, + "", + ) .await { let pl_event: RoomPowerLevels = pl_event_content.into(); @@ -485,10 +551,15 @@ impl Service { } /// Returns the join rule (`SpaceRoomJoinRule`) for a given room - pub async fn get_join_rule(&self, room_id: &RoomId) -> Result<(SpaceRoomJoinRule, Vec)> { + pub async fn get_join_rule( + &self, + room_id: &RoomId, + ) -> Result<(SpaceRoomJoinRule, Vec)> { self.room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") .await - .map(|c: RoomJoinRulesEventContent| (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule))) + .map(|c: RoomJoinRulesEventContent| { + (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule)) + }) .or_else(|_| Ok((SpaceRoomJoinRule::Invite, vec![]))) } @@ -497,10 +568,7 @@ impl Service { let mut room_ids = Vec::with_capacity(1); if let JoinRule::Restricted(r) | JoinRule::KnockRestricted(r) = join_rule { for rule in r.allow { - if let AllowRule::RoomMembership(RoomMembership { - room_id: membership, - }) = rule - { + if let AllowRule::RoomMembership(RoomMembership { room_id: membership }) = rule { room_ids.push(membership.clone()); } } @@ -520,7 +588,10 @@ impl Service { /// Gets the room's encryption algorithm if `m.room.encryption` state event /// is found - pub async fn get_room_encryption(&self, room_id: &RoomId) -> Result { + pub async fn get_room_encryption( + &self, + room_id: &RoomId, + ) -> Result { self.room_state_get_content(room_id, &StateEventType::RoomEncryption, "") .await .map(|content: RoomEncryptionEventContent| content.algorithm) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 6b7d35d2..c2de8b62 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -20,7 +20,8 @@ use ruma::{ member::{MembershipState, RoomMemberEventContent}, power_levels::RoomPowerLevelsEventContent, }, - AnyStrippedStateEvent, AnySyncStateEvent, GlobalAccountDataEventType, RoomAccountDataEventType, StateEventType, + AnyStrippedStateEvent, AnySyncStateEvent, GlobalAccountDataEventType, + RoomAccountDataEventType, StateEventType, }, int, serde::Raw, @@ -68,7 +69,8 @@ impl crate::Service for Service { services: Services { account_data: args.depend::("account_data"), globals: args.depend::("globals"), - state_accessor: args.depend::("rooms::state_accessor"), + state_accessor: args + .depend::("rooms::state_accessor"), users: args.depend::("users"), }, db: Data { @@ -96,8 +98,13 @@ impl Service { #[tracing::instrument(skip(self, last_state))] #[allow(clippy::too_many_arguments)] pub async fn update_membership( - &self, room_id: &RoomId, user_id: &UserId, membership_event: RoomMemberEventContent, sender: &UserId, - last_state: Option>>, invite_via: Option>, + &self, + room_id: &RoomId, + user_id: &UserId, + membership_event: RoomMemberEventContent, + sender: &UserId, + last_state: Option>>, + invite_via: Option>, update_joined_count: bool, ) -> Result<()> { let membership = membership_event.membership; @@ -138,7 +145,7 @@ impl Service { } match &membership { - MembershipState::Join => { + | MembershipState::Join => { // Check if the user never joined this room if !self.once_joined(user_id, room_id).await { // Add the user ID to the join list then @@ -181,12 +188,21 @@ impl Service { if let Ok(tag_event) = self .services .account_data - .get_room(&predecessor.room_id, user_id, RoomAccountDataEventType::Tag) + .get_room( + &predecessor.room_id, + user_id, + RoomAccountDataEventType::Tag, + ) .await { self.services .account_data - .update(Some(room_id), user_id, RoomAccountDataEventType::Tag, &tag_event) + .update( + Some(room_id), + user_id, + RoomAccountDataEventType::Tag, + &tag_event, + ) .await .ok(); }; @@ -195,7 +211,10 @@ impl Service { if let Ok(mut direct_event) = self .services .account_data - .get_global::(user_id, GlobalAccountDataEventType::Direct) + .get_global::( + user_id, + GlobalAccountDataEventType::Direct, + ) .await { let mut room_ids_updated = false; @@ -213,7 +232,8 @@ impl Service { None, user_id, GlobalAccountDataEventType::Direct.to_string().into(), - &serde_json::to_value(&direct_event).expect("to json always works"), + &serde_json::to_value(&direct_event) + .expect("to json always works"), ) .await?; } @@ -223,7 +243,7 @@ impl Service { self.mark_as_joined(user_id, room_id); }, - MembershipState::Invite => { + | MembershipState::Invite => { // We want to know if the sender is ignored by the receiver if self.services.users.user_is_ignored(sender, user_id).await { return Ok(()); @@ -232,10 +252,10 @@ impl Service { self.mark_as_invited(user_id, room_id, last_state, invite_via) .await; }, - MembershipState::Leave | MembershipState::Ban => { + | MembershipState::Leave | MembershipState::Ban => { self.mark_as_left(user_id, room_id); }, - _ => {}, + | _ => {}, } if update_joined_count { @@ -246,7 +266,11 @@ impl Service { } #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] - pub async fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> bool { + pub async fn appservice_in_room( + &self, + room_id: &RoomId, + appservice: &RegistrationInfo, + ) -> bool { if let Some(cached) = self .appservice_in_room_cache .read() @@ -347,7 +371,10 @@ impl Service { /// Returns an iterator of all servers participating in this room. #[tracing::instrument(skip(self), level = "debug")] - pub fn room_servers<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + pub fn room_servers<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .roomserverids @@ -357,7 +384,11 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub async fn server_in_room<'a>(&'a self, server: &'a ServerName, room_id: &'a RoomId) -> bool { + pub async fn server_in_room<'a>( + &'a self, + server: &'a ServerName, + room_id: &'a RoomId, + ) -> bool { let key = (server, room_id); self.db.serverroomids.qry(&key).await.is_ok() } @@ -365,7 +396,10 @@ impl Service { /// Returns an iterator of all rooms a server participates in (as far as we /// know). #[tracing::instrument(skip(self), level = "debug")] - pub fn server_rooms<'a>(&'a self, server: &'a ServerName) -> impl Stream + Send + 'a { + pub fn server_rooms<'a>( + &'a self, + server: &'a ServerName, + ) -> impl Stream + Send + 'a { let prefix = (server, Interfix); self.db .serverroomids @@ -393,7 +427,9 @@ impl Service { /// List the rooms common between two users pub fn get_shared_rooms<'a>( - &'a self, user_a: &'a UserId, user_b: &'a UserId, + &'a self, + user_a: &'a UserId, + user_b: &'a UserId, ) -> impl Stream + Send + 'a { use conduwuit::utils::set; @@ -404,7 +440,10 @@ impl Service { /// Returns an iterator of all joined members of a room. #[tracing::instrument(skip(self), level = "debug")] - pub fn room_members<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + pub fn room_members<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .roomuserid_joined @@ -422,7 +461,10 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] /// Returns an iterator of all our local users in the room, even if they're /// deactivated/guests - pub fn local_users_in_room<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + pub fn local_users_in_room<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { self.room_members(room_id) .ready_filter(|user| self.services.globals.user_is_local(user)) } @@ -430,7 +472,10 @@ impl Service { #[tracing::instrument(skip(self), level = "debug")] /// Returns an iterator of all our local joined users in a room who are /// active (not deactivated, not guest) - pub fn active_local_users_in_room<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + pub fn active_local_users_in_room<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { self.local_users_in_room(room_id) .filter(|user| self.services.users.is_active(user)) } @@ -447,7 +492,10 @@ impl Service { /// Returns an iterator over all User IDs who ever joined a room. #[tracing::instrument(skip(self), level = "debug")] - pub fn room_useroncejoined<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + pub fn room_useroncejoined<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .roomuseroncejoinedids @@ -458,7 +506,10 @@ impl Service { /// Returns an iterator over all invited members of a room. #[tracing::instrument(skip(self), level = "debug")] - pub fn room_members_invited<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + pub fn room_members_invited<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .roomuserid_invitecount @@ -485,7 +536,10 @@ impl Service { /// Returns an iterator over all rooms this user joined. #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_joined<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + pub fn rooms_joined<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { self.db .userroomid_joined .keys_raw_prefix(user_id) @@ -495,7 +549,10 @@ impl Service { /// Returns an iterator over all rooms a user was invited to. #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_invited<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + pub fn rooms_invited<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { type KeyVal<'a> = (Key<'a>, Raw>); type Key<'a> = (&'a UserId, &'a RoomId); @@ -510,30 +567,45 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub async fn invite_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>> { + pub async fn invite_state( + &self, + user_id: &UserId, + room_id: &RoomId, + ) -> Result>> { let key = (user_id, room_id); self.db .userroomid_invitestate .qry(&key) .await .deserialized() - .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) + .and_then(|val: Raw>| { + val.deserialize_as().map_err(Into::into) + }) } #[tracing::instrument(skip(self), level = "debug")] - pub async fn left_state(&self, user_id: &UserId, room_id: &RoomId) -> Result>> { + pub async fn left_state( + &self, + user_id: &UserId, + room_id: &RoomId, + ) -> Result>> { let key = (user_id, room_id); self.db .userroomid_leftstate .qry(&key) .await .deserialized() - .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) + .and_then(|val: Raw>| { + val.deserialize_as().map_err(Into::into) + }) } /// Returns an iterator over all rooms a user left. #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_left<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + pub fn rooms_left<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { type KeyVal<'a> = (Key<'a>, Raw>>); type Key<'a> = (&'a UserId, &'a RoomId); @@ -571,7 +643,11 @@ impl Service { self.db.userroomid_leftstate.qry(&key).await.is_ok() } - pub async fn user_membership(&self, user_id: &UserId, room_id: &RoomId) -> Option { + pub async fn user_membership( + &self, + user_id: &UserId, + room_id: &RoomId, + ) -> Option { let states = join4( self.is_joined(user_id, room_id), self.is_left(user_id, room_id), @@ -581,16 +657,19 @@ impl Service { .await; match states { - (true, ..) => Some(MembershipState::Join), - (_, true, ..) => Some(MembershipState::Leave), - (_, _, true, ..) => Some(MembershipState::Invite), - (false, false, false, true) => Some(MembershipState::Ban), - _ => None, + | (true, ..) => Some(MembershipState::Join), + | (_, true, ..) => Some(MembershipState::Leave), + | (_, _, true, ..) => Some(MembershipState::Invite), + | (false, false, false, true) => Some(MembershipState::Ban), + | _ => None, } } #[tracing::instrument(skip(self), level = "debug")] - pub fn servers_invite_via<'a>(&'a self, room_id: &'a RoomId) -> impl Stream + Send + 'a { + pub fn servers_invite_via<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { type KeyVal<'a> = (Ignore, Vec<&'a ServerName>); self.db @@ -711,7 +790,10 @@ impl Service { } pub async fn mark_as_invited( - &self, user_id: &UserId, room_id: &RoomId, last_state: Option>>, + &self, + user_id: &UserId, + room_id: &RoomId, + last_state: Option>>, invite_via: Option>, ) { let roomuser_id = (room_id, user_id); diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 36bc92c0..dbe0a386 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -69,7 +69,8 @@ pub(crate) type CompressedStateEvent = [u8; 2 * size_of::()]; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; - let cache_capacity = f64::from(config.stateinfo_cache_capacity) * config.cache_capacity_modifier; + let cache_capacity = + f64::from(config.stateinfo_cache_capacity) * config.cache_capacity_modifier; Ok(Arc::new(Self { stateinfo_cache: LruCache::new(usize_from_f64(cache_capacity)?).into(), db: Data { @@ -85,17 +86,16 @@ impl crate::Service for Service { fn memory_usage(&self, out: &mut dyn Write) -> Result { let (cache_len, ents) = { let cache = self.stateinfo_cache.lock().expect("locked"); - let ents = cache - .iter() - .map(at!(1)) - .flat_map(|vec| vec.iter()) - .fold(HashMap::new(), |mut ents, ssi| { + let ents = cache.iter().map(at!(1)).flat_map(|vec| vec.iter()).fold( + HashMap::new(), + |mut ents, ssi| { for cs in &[&ssi.added, &ssi.removed, &ssi.full_state] { ents.insert(Arc::as_ptr(cs), compressed_state_size(cs)); } ents - }); + }, + ); (cache.len(), ents) }; @@ -117,7 +117,10 @@ impl crate::Service for Service { impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. - pub async fn load_shortstatehash_info(&self, shortstatehash: ShortStateHash) -> Result { + pub async fn load_shortstatehash_info( + &self, + shortstatehash: ShortStateHash, + ) -> Result { if let Some(r) = self .stateinfo_cache .lock() @@ -143,12 +146,11 @@ impl Service { Ok(stack) } - async fn new_shortstatehash_info(&self, shortstatehash: ShortStateHash) -> Result { - let StateDiff { - parent, - added, - removed, - } = self.get_statediff(shortstatehash).await?; + async fn new_shortstatehash_info( + &self, + shortstatehash: ShortStateHash, + ) -> Result { + let StateDiff { parent, added, removed } = self.get_statediff(shortstatehash).await?; let Some(parent) = parent else { return Ok(vec![ShortStateInfo { @@ -180,9 +182,17 @@ impl Service { Ok(stack) } - pub fn compress_state_events<'a, I>(&'a self, state: I) -> impl Stream + Send + 'a + pub fn compress_state_events<'a, I>( + &'a self, + state: I, + ) -> impl Stream + Send + 'a where - I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, + I: Iterator + + Clone + + Debug + + ExactSizeIterator + + Send + + 'a, { let event_ids = state.clone().map(at!(1)); @@ -195,10 +205,16 @@ impl Service { .stream() .map(at!(0)) .zip(short_event_ids) - .map(|(shortstatekey, shorteventid)| compress_state_event(*shortstatekey, shorteventid)) + .map(|(shortstatekey, shorteventid)| { + compress_state_event(*shortstatekey, shorteventid) + }) } - pub async fn compress_state_event(&self, shortstatekey: ShortStateKey, event_id: &EventId) -> CompressedStateEvent { + pub async fn compress_state_event( + &self, + shortstatekey: ShortStateKey, + event_id: &EventId, + ) -> CompressedStateEvent { let shorteventid = self .services .short @@ -227,8 +243,11 @@ impl Service { /// * `parent_states` - A stack with info on shortstatehash, full state, /// added diff and removed diff for each parent layer pub fn save_state_from_diff( - &self, shortstatehash: ShortStateHash, statediffnew: Arc>, - statediffremoved: Arc>, diff_to_sibling: usize, + &self, + shortstatehash: ShortStateHash, + statediffnew: Arc>, + statediffremoved: Arc>, + diff_to_sibling: usize, mut parent_states: ParentStatesVec, ) -> Result { let statediffnew_len = statediffnew.len(); @@ -274,14 +293,11 @@ impl Service { if parent_states.is_empty() { // There is no parent layer, create a new state - self.save_statediff( - shortstatehash, - &StateDiff { - parent: None, - added: statediffnew, - removed: statediffremoved, - }, - ); + self.save_statediff(shortstatehash, &StateDiff { + parent: None, + added: statediffnew, + removed: statediffremoved, + }); return Ok(()); }; @@ -327,14 +343,11 @@ impl Service { )?; } else { // Diff small enough, we add diff as layer on top of parent - self.save_statediff( - shortstatehash, - &StateDiff { - parent: Some(parent.shortstatehash), - added: statediffnew, - removed: statediffremoved, - }, - ); + self.save_statediff(shortstatehash, &StateDiff { + parent: Some(parent.shortstatehash), + added: statediffnew, + removed: statediffremoved, + }); } Ok(()) @@ -344,7 +357,9 @@ impl Service { /// room state #[tracing::instrument(skip(self, new_state_ids_compressed), level = "debug")] pub async fn save_state( - &self, room_id: &RoomId, new_state_ids_compressed: Arc>, + &self, + room_id: &RoomId, + new_state_ids_compressed: Arc>, ) -> Result { let previous_shortstatehash = self .services @@ -353,7 +368,8 @@ impl Service { .await .ok(); - let state_hash = utils::calculate_hash(new_state_ids_compressed.iter().map(|bytes| &bytes[..])); + let state_hash = + utils::calculate_hash(new_state_ids_compressed.iter().map(|bytes| &bytes[..])); let (new_shortstatehash, already_existed) = self .services @@ -374,22 +390,23 @@ impl Service { ShortStateInfoVec::new() }; - let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew: HashSet<_> = new_state_ids_compressed - .difference(&parent_stateinfo.full_state) - .copied() - .collect(); + let (statediffnew, statediffremoved) = + if let Some(parent_stateinfo) = states_parents.last() { + let statediffnew: HashSet<_> = new_state_ids_compressed + .difference(&parent_stateinfo.full_state) + .copied() + .collect(); - let statediffremoved: HashSet<_> = parent_stateinfo - .full_state - .difference(&new_state_ids_compressed) - .copied() - .collect(); + let statediffremoved: HashSet<_> = parent_stateinfo + .full_state + .difference(&new_state_ids_compressed) + .copied() + .collect(); - (Arc::new(statediffnew), Arc::new(statediffremoved)) - } else { - (new_state_ids_compressed, Arc::new(HashSet::new())) - }; + (Arc::new(statediffnew), Arc::new(statediffremoved)) + } else { + (new_state_ids_compressed, Arc::new(HashSet::new())) + }; if !already_existed { self.save_state_from_diff( @@ -418,7 +435,9 @@ impl Service { .shortstatehash_statediff .aqry::(&shortstatehash) .await - .map_err(|e| err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")))?; + .map_err(|e| { + err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")) + })?; let parent = utils::u64_from_bytes(&value[0..size_of::()]) .ok() @@ -484,7 +503,10 @@ impl Service { #[inline] #[must_use] -fn compress_state_event(shortstatekey: ShortStateKey, shorteventid: ShortEventId) -> CompressedStateEvent { +fn compress_state_event( + shortstatekey: ShortStateKey, + shorteventid: ShortEventId, +) -> CompressedStateEvent { const SIZE: usize = size_of::(); let mut v = ArrayVec::::new(); @@ -497,7 +519,9 @@ fn compress_state_event(shortstatekey: ShortStateKey, shorteventid: ShortEventId #[inline] #[must_use] -pub fn parse_compressed_state_event(compressed_event: CompressedStateEvent) -> (ShortStateKey, ShortEventId) { +pub fn parse_compressed_state_event( + compressed_event: CompressedStateEvent, +) -> (ShortStateKey, ShortEventId) { use utils::u64_from_u8; let shortstatekey = u64_from_u8(&compressed_event[0..size_of::()]); diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index f65be902..bc995e27 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -11,8 +11,8 @@ use conduwuit::{ use database::{Deserialized, Map}; use futures::{Stream, StreamExt}; use ruma::{ - api::client::threads::get_threads::v1::IncludeThreads, events::relation::BundledThread, uint, CanonicalJsonValue, - EventId, OwnedUserId, RoomId, UserId, + api::client::threads::get_threads::v1::IncludeThreads, events::relation::BundledThread, uint, + CanonicalJsonValue, EventId, OwnedUserId, RoomId, UserId, }; use serde_json::json; @@ -55,7 +55,9 @@ impl Service { .timeline .get_pdu_id(root_event_id) .await - .map_err(|e| err!(Request(InvalidParam("Invalid event_id in thread message: {e:?}"))))?; + .map_err(|e| { + err!(Request(InvalidParam("Invalid event_id in thread message: {e:?}"))) + })?; let root_pdu = self .services @@ -79,8 +81,9 @@ impl Service { .get("m.relations") .and_then(|r| r.as_object()) .and_then(|r| r.get("m.thread")) - .and_then(|relations| serde_json::from_value::(relations.clone().into()).ok()) - { + .and_then(|relations| { + serde_json::from_value::(relations.clone().into()).ok() + }) { // Thread already existed relations.count = relations.count.saturating_add(uint!(1)); relations.latest_event = pdu.to_message_like_event(); @@ -129,7 +132,11 @@ impl Service { } pub async fn threads_until<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, shorteventid: PduCount, _inc: &'a IncludeThreads, + &'a self, + user_id: &'a UserId, + room_id: &'a RoomId, + shorteventid: PduCount, + _inc: &'a IncludeThreads, ) -> Result + Send + 'a> { let shortroomid: ShortRoomId = self.services.short.get_shortroomid(room_id).await?; @@ -160,7 +167,11 @@ impl Service { Ok(stream) } - pub(super) fn update_participants(&self, root_id: &RawPduId, participants: &[OwnedUserId]) -> Result { + pub(super) fn update_participants( + &self, + root_id: &RawPduId, + participants: &[OwnedUserId], + ) -> Result { let users = participants .iter() .map(|user| user.as_bytes()) diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index a32ff54f..0be8aa52 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -13,7 +13,9 @@ use conduwuit::{ }; use database::{Database, Deserialized, Json, KeyVal, Map}; use futures::{future::select_ok, FutureExt, Stream, StreamExt}; -use ruma::{api::Direction, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId}; +use ruma::{ + api::Direction, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId, +}; use tokio::sync::Mutex; use super::{PduId, RawPduId}; @@ -54,15 +56,19 @@ impl Data { } } - pub(super) async fn last_timeline_count(&self, sender_user: Option<&UserId>, room_id: &RoomId) -> Result { + pub(super) async fn last_timeline_count( + &self, + sender_user: Option<&UserId>, + room_id: &RoomId, + ) -> Result { match self .lasttimelinecount_cache .lock() .await .entry(room_id.into()) { - hash_map::Entry::Occupied(o) => Ok(*o.get()), - hash_map::Entry::Vacant(v) => Ok(self + | hash_map::Entry::Occupied(o) => Ok(*o.get()), + | hash_map::Entry::Vacant(v) => Ok(self .pdus_rev(sender_user, room_id, PduCount::max()) .await? .next() @@ -93,7 +99,10 @@ impl Data { } /// Returns the json of a pdu. - pub(super) async fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result { + pub(super) async fn get_non_outlier_pdu_json( + &self, + event_id: &EventId, + ) -> Result { let pduid = self.get_pdu_id(event_id).await?; self.pduid_pdu.get(&pduid).await.deserialized() @@ -160,12 +169,19 @@ impl Data { } /// Returns the pdu as a `BTreeMap`. - pub(super) async fn get_pdu_json_from_id(&self, pdu_id: &RawPduId) -> Result { + pub(super) async fn get_pdu_json_from_id( + &self, + pdu_id: &RawPduId, + ) -> Result { self.pduid_pdu.get(pdu_id).await.deserialized() } pub(super) async fn append_pdu( - &self, pdu_id: &RawPduId, pdu: &PduEvent, json: &CanonicalJsonObject, count: PduCount, + &self, + pdu_id: &RawPduId, + pdu: &PduEvent, + json: &CanonicalJsonObject, + count: PduCount, ) { debug_assert!(matches!(count, PduCount::Normal(_)), "PduCount not Normal"); @@ -179,7 +195,12 @@ impl Data { self.eventid_outlierpdu.remove(pdu.event_id.as_bytes()); } - pub(super) fn prepend_backfill_pdu(&self, pdu_id: &RawPduId, event_id: &EventId, json: &CanonicalJsonObject) { + pub(super) fn prepend_backfill_pdu( + &self, + pdu_id: &RawPduId, + event_id: &EventId, + json: &CanonicalJsonObject, + ) { self.pduid_pdu.raw_put(pdu_id, Json(json)); self.eventid_pduid.insert(event_id, pdu_id); self.eventid_outlierpdu.remove(event_id); @@ -187,7 +208,10 @@ impl Data { /// Removes a pdu and creates a new one with the same id. pub(super) async fn replace_pdu( - &self, pdu_id: &RawPduId, pdu_json: &CanonicalJsonObject, _pdu: &PduEvent, + &self, + pdu_id: &RawPduId, + pdu_json: &CanonicalJsonObject, + _pdu: &PduEvent, ) -> Result { if self.pduid_pdu.get(pdu_id).await.is_not_found() { return Err!(Request(NotFound("PDU does not exist."))); @@ -202,7 +226,10 @@ impl Data { /// happened before the event with id `until` in reverse-chronological /// order. pub(super) async fn pdus_rev<'a>( - &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, until: PduCount, + &'a self, + user_id: Option<&'a UserId>, + room_id: &'a RoomId, + until: PduCount, ) -> Result + Send + 'a> { let current = self .count_to_id(room_id, until, Direction::Backward) @@ -219,7 +246,10 @@ impl Data { } pub(super) async fn pdus<'a>( - &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, from: PduCount, + &'a self, + user_id: Option<&'a UserId>, + room_id: &'a RoomId, + from: PduCount, ) -> Result + Send + Unpin + 'a> { let current = self.count_to_id(room_id, from, Direction::Forward).await?; let prefix = current.shortroomid(); @@ -236,8 +266,8 @@ impl Data { fn each_pdu((pdu_id, pdu): KeyVal<'_>, user_id: Option<&UserId>) -> PdusIterItem { let pdu_id: RawPduId = pdu_id.into(); - let mut pdu = - serde_json::from_slice::(pdu).expect("PduEvent in pduid_pdu database column is invalid JSON"); + let mut pdu = serde_json::from_slice::(pdu) + .expect("PduEvent in pduid_pdu database column is invalid JSON"); if Some(pdu.sender.borrow()) != user_id { pdu.remove_transaction_id().log_err().ok(); @@ -249,7 +279,10 @@ impl Data { } pub(super) fn increment_notification_counts( - &self, room_id: &RoomId, notifies: Vec, highlights: Vec, + &self, + room_id: &RoomId, + notifies: Vec, + highlights: Vec, ) { let _cork = self.db.cork(); @@ -268,7 +301,12 @@ impl Data { } } - async fn count_to_id(&self, room_id: &RoomId, shorteventid: PduCount, dir: Direction) -> Result { + async fn count_to_id( + &self, + room_id: &RoomId, + shorteventid: PduCount, + dir: Direction, + ) -> Result { let shortroomid: ShortRoomId = self .services .short diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 2bc5cf73..2ae66546 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -15,7 +15,9 @@ use conduwuit::{ validated, warn, Err, Error, Result, Server, }; pub use conduwuit::{PduId, RawPduId}; -use futures::{future, future::ready, Future, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{ + future, future::ready, Future, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, +}; use ruma::{ api::federation, canonical_json::to_canonical_value, @@ -32,8 +34,8 @@ use ruma::{ }, push::{Action, Ruleset, Tweak}, state_res::{self, Event, RoomVersion}, - uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, - RoomId, RoomVersionId, ServerName, UserId, + uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, + OwnedServerName, RoomId, RoomVersionId, ServerName, UserId, }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; @@ -116,7 +118,8 @@ impl crate::Service for Service { short: args.depend::("rooms::short"), state: args.depend::("rooms::state"), state_cache: args.depend::("rooms::state_cache"), - state_accessor: args.depend::("rooms::state_accessor"), + state_accessor: args + .depend::("rooms::state_accessor"), pdu_metadata: args.depend::("rooms::pdu_metadata"), read_receipt: args.depend::("rooms::read_receipt"), sending: args.depend::("sending"), @@ -127,7 +130,8 @@ impl crate::Service for Service { threads: args.depend::("rooms::threads"), search: args.depend::("rooms::search"), spaces: args.depend::("rooms::spaces"), - event_handler: args.depend::("rooms::event_handler"), + event_handler: args + .depend::("rooms::event_handler"), }, db: Data::new(&args), mutex_insert: RoomMutexMap::new(), @@ -185,12 +189,18 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub async fn last_timeline_count(&self, sender_user: Option<&UserId>, room_id: &RoomId) -> Result { + pub async fn last_timeline_count( + &self, + sender_user: Option<&UserId>, + room_id: &RoomId, + ) -> Result { self.db.last_timeline_count(sender_user, room_id).await } /// Returns the `count` of this pdu's id. - pub async fn get_pdu_count(&self, event_id: &EventId) -> Result { self.db.get_pdu_count(event_id).await } + pub async fn get_pdu_count(&self, event_id: &EventId) -> Result { + self.db.get_pdu_count(event_id).await + } // TODO Is this the same as the function above? /* @@ -222,13 +232,18 @@ impl Service { /// Returns the json of a pdu. #[inline] - pub async fn get_non_outlier_pdu_json(&self, event_id: &EventId) -> Result { + pub async fn get_non_outlier_pdu_json( + &self, + event_id: &EventId, + ) -> Result { self.db.get_non_outlier_pdu_json(event_id).await } /// Returns the pdu's id. #[inline] - pub async fn get_pdu_id(&self, event_id: &EventId) -> Result { self.db.get_pdu_id(event_id).await } + pub async fn get_pdu_id(&self, event_id: &EventId) -> Result { + self.db.get_pdu_id(event_id).await + } /// Returns the pdu. /// @@ -241,19 +256,26 @@ impl Service { /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - pub async fn get_pdu(&self, event_id: &EventId) -> Result { self.db.get_pdu(event_id).await } + pub async fn get_pdu(&self, event_id: &EventId) -> Result { + self.db.get_pdu(event_id).await + } /// Checks if pdu exists /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - pub fn pdu_exists<'a>(&'a self, event_id: &'a EventId) -> impl Future + Send + 'a { + pub fn pdu_exists<'a>( + &'a self, + event_id: &'a EventId, + ) -> impl Future + Send + 'a { self.db.pdu_exists(event_id) } /// Returns the pdu. /// /// This does __NOT__ check the outliers `Tree`. - pub async fn get_pdu_from_id(&self, pdu_id: &RawPduId) -> Result { self.db.get_pdu_from_id(pdu_id).await } + pub async fn get_pdu_from_id(&self, pdu_id: &RawPduId) -> Result { + self.db.get_pdu_from_id(pdu_id).await + } /// Returns the pdu as a `BTreeMap`. pub async fn get_pdu_json_from_id(&self, pdu_id: &RawPduId) -> Result { @@ -262,7 +284,12 @@ impl Service { /// Removes a pdu and creates a new one with the same id. #[tracing::instrument(skip(self), level = "debug")] - pub async fn replace_pdu(&self, pdu_id: &RawPduId, pdu_json: &CanonicalJsonObject, pdu: &PduEvent) -> Result<()> { + pub async fn replace_pdu( + &self, + pdu_id: &RawPduId, + pdu_json: &CanonicalJsonObject, + pdu: &PduEvent, + ) -> Result<()> { self.db.replace_pdu(pdu_id, pdu_json, pdu).await } @@ -278,7 +305,8 @@ impl Service { pdu: &PduEvent, mut pdu_json: CanonicalJsonObject, leaves: Vec, - state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room state + * mutex */ ) -> Result { // Coalesce database writes for the remainder of this scope. let _cork = self.db.db.cork_and_flush(); @@ -313,10 +341,16 @@ impl Service { unsigned.insert( "prev_content".to_owned(), CanonicalJsonValue::Object( - utils::to_canonical_object(prev_state.content.clone()).map_err(|e| { - error!("Failed to convert prev_state to canonical JSON: {e}"); - Error::bad_database("Failed to convert prev_state to canonical JSON.") - })?, + utils::to_canonical_object(prev_state.content.clone()).map_err( + |e| { + error!( + "Failed to convert prev_state to canonical JSON: {e}" + ); + Error::bad_database( + "Failed to convert prev_state to canonical JSON.", + ) + }, + )?, ), ); unsigned.insert( @@ -357,11 +391,7 @@ impl Service { .reset_notification_counts(&pdu.sender, &pdu.room_id); let count2 = PduCount::Normal(self.services.globals.next_count().unwrap()); - let pdu_id: RawPduId = PduId { - shortroomid, - shorteventid: count2, - } - .into(); + let pdu_id: RawPduId = PduId { shortroomid, shorteventid: count2 }.into(); // Insert pdu self.db.append_pdu(&pdu_id, pdu, &pdu_json, count2).await; @@ -408,7 +438,10 @@ impl Service { .account_data .get_global(user, GlobalAccountDataEventType::PushRules) .await - .map_or_else(|_| Ruleset::server_default(user), |ev: PushRulesEvent| ev.content.global); + .map_or_else( + |_| Ruleset::server_default(user), + |ev: PushRulesEvent| ev.content.global, + ); let mut highlight = false; let mut notify = false; @@ -420,11 +453,11 @@ impl Service { .await { match action { - Action::Notify => notify = true, - Action::SetTweak(Tweak::Highlight(true)) => { + | Action::Notify => notify = true, + | Action::SetTweak(Tweak::Highlight(true)) => { highlight = true; }, - _ => {}, + | _ => {}, }; // Break early if both conditions are true @@ -457,12 +490,12 @@ impl Service { .increment_notification_counts(&pdu.room_id, notifies, highlights); match pdu.kind { - TimelineEventType::RoomRedaction => { + | TimelineEventType::RoomRedaction => { use RoomVersionId::*; let room_version_id = self.services.state.get_room_version(&pdu.room_id).await?; match room_version_id { - V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { if self .services @@ -474,7 +507,7 @@ impl Service { } } }, - _ => { + | _ => { let content: RoomRedactionEventContent = pdu.get_content()?; if let Some(redact_id) = &content.redacts { if self @@ -489,7 +522,7 @@ impl Service { }, }; }, - TimelineEventType::SpaceChild => { + | TimelineEventType::SpaceChild => if let Some(_state_key) = &pdu.state_key { self.services .spaces @@ -497,18 +530,18 @@ impl Service { .lock() .await .remove(&pdu.room_id); - } - }, - TimelineEventType::RoomMember => { + }, + | TimelineEventType::RoomMember => { if let Some(state_key) = &pdu.state_key { // if the state_key fails - let target_user_id = - UserId::parse(state_key.clone()).expect("This state_key was previously validated"); + let target_user_id = UserId::parse(state_key.clone()) + .expect("This state_key was previously validated"); let content: RoomMemberEventContent = pdu.get_content()?; let invite_state = match content.membership { - MembershipState::Invite => self.services.state.summary_stripped(pdu).await.into(), - _ => None, + | MembershipState::Invite => + self.services.state.summary_stripped(pdu).await.into(), + | _ => None, }; // Update our membership info, we do this here incase a user is invited @@ -527,7 +560,7 @@ impl Service { .await?; } }, - TimelineEventType::RoomMessage => { + | TimelineEventType::RoomMessage => { let content: ExtractBody = pdu.get_content()?; if let Some(body) = content.body { self.services.search.index_pdu(shortroomid, &pdu_id, &body); @@ -539,7 +572,7 @@ impl Service { } } }, - _ => {}, + | _ => {}, } if let Ok(content) = pdu.get_content::() { @@ -552,24 +585,23 @@ impl Service { if let Ok(content) = pdu.get_content::() { match content.relates_to { - Relation::Reply { - in_reply_to, - } => { + | Relation::Reply { in_reply_to } => { // We need to do it again here, because replies don't have // event_id as a top level field - if let Ok(related_pducount) = self.get_pdu_count(&in_reply_to.event_id).await { + if let Ok(related_pducount) = self.get_pdu_count(&in_reply_to.event_id).await + { self.services .pdu_metadata .add_relation(count2, related_pducount); } }, - Relation::Thread(thread) => { + | Relation::Thread(thread) => { self.services .threads .add_to_thread(&thread.event_id, pdu) .await?; }, - _ => {}, // TODO: Aggregate other types + | _ => {}, // TODO: Aggregate other types } } @@ -637,7 +669,8 @@ impl Service { pdu_builder: PduBuilder, sender: &UserId, room_id: &RoomId, - _mutex_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room + * state mutex */ ) -> Result<(PduEvent, CanonicalJsonObject)> { let PduBuilder { event_type, @@ -707,7 +740,8 @@ impl Service { unsigned.insert("prev_content".to_owned(), prev_pdu.get_content_as_value()); unsigned.insert( "prev_sender".to_owned(), - serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"), + serde_json::to_value(&prev_pdu.sender) + .expect("UserId::to_value always works"), ); unsigned.insert( "replaces_state".to_owned(), @@ -744,9 +778,7 @@ impl Service { } else { Some(to_raw_value(&unsigned).expect("to_raw_value always works")) }, - hashes: EventHash { - sha256: "aaa".to_owned(), - }, + hashes: EventHash { sha256: "aaa".to_owned() }, signatures: None, }; @@ -769,13 +801,14 @@ impl Service { } // Hash and sign - let mut pdu_json = utils::to_canonical_object(&pdu) - .map_err(|e| err!(Request(BadJson(warn!("Failed to convert PDU to canonical JSON: {e}")))))?; + let mut pdu_json = utils::to_canonical_object(&pdu).map_err(|e| { + err!(Request(BadJson(warn!("Failed to convert PDU to canonical JSON: {e}")))) + })?; // room v3 and above removed the "event_id" field from remote PDU format match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => { + | RoomVersionId::V1 | RoomVersionId::V2 => {}, + | _ => { pdu_json.remove("event_id"); }, }; @@ -783,7 +816,8 @@ impl Service { // Add origin because synapse likes that (and it's required in the spec) pdu_json.insert( "origin".to_owned(), - to_canonical_value(self.services.globals.server_name()).expect("server name is a valid CanonicalJsonValue"), + to_canonical_value(self.services.globals.server_name()) + .expect("server name is a valid CanonicalJsonValue"), ); if let Err(e) = self @@ -792,17 +826,18 @@ impl Service { .hash_and_sign_event(&mut pdu_json, &room_version_id) { return match e { - Error::Signatures(ruma::signatures::Error::PduSize) => { + | Error::Signatures(ruma::signatures::Error::PduSize) => { Err!(Request(TooLarge("Message/PDU is too long (exceeds 65535 bytes)"))) }, - _ => Err!(Request(Unknown(warn!("Signing event failed: {e}")))), + | _ => Err!(Request(Unknown(warn!("Signing event failed: {e}")))), }; } // Generate event id pdu.event_id = EventId::parse_arc(format!( "${}", - ruma::signatures::reference_hash(&pdu_json, &room_version_id).expect("ruma can calculate reference hashes") + ruma::signatures::reference_hash(&pdu_json, &room_version_id) + .expect("ruma can calculate reference hashes") )) .expect("ruma's reference hashes are valid event ids"); @@ -830,7 +865,8 @@ impl Service { pdu_builder: PduBuilder, sender: &UserId, room_id: &RoomId, - state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room state + * mutex */ ) -> Result> { let (pdu, pdu_json) = self .create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock) @@ -844,7 +880,7 @@ impl Service { if pdu.kind == TimelineEventType::RoomRedaction { use RoomVersionId::*; match self.services.state.get_room_version(&pdu.room_id).await? { - V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { if let Some(redact_id) = &pdu.redacts { if !self .services @@ -856,7 +892,7 @@ impl Service { } }; }, - _ => { + | _ => { let content: RoomRedactionEventContent = pdu.get_content()?; if let Some(redact_id) = &content.redacts { if !self @@ -937,7 +973,8 @@ impl Service { new_room_leaves: Vec, state_ids_compressed: Arc>, soft_fail: bool, - state_lock: &RoomMutexGuard, // Take mutex guard to make sure users get the room state mutex + state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room state + * mutex */ ) -> Result> { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't @@ -971,7 +1008,9 @@ impl Service { /// items. #[inline] pub fn all_pdus<'a>( - &'a self, user_id: &'a UserId, room_id: &'a RoomId, + &'a self, + user_id: &'a UserId, + room_id: &'a RoomId, ) -> impl Stream + Send + Unpin + 'a { self.pdus(Some(user_id), room_id, None) .map_ok(|stream| stream.map(Ok)) @@ -983,7 +1022,10 @@ impl Service { /// Reverse iteration starting at from. #[tracing::instrument(skip(self), level = "debug")] pub async fn pdus_rev<'a>( - &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, until: Option, + &'a self, + user_id: Option<&'a UserId>, + room_id: &'a RoomId, + until: Option, ) -> Result + Send + 'a> { self.db .pdus_rev(user_id, room_id, until.unwrap_or_else(PduCount::max)) @@ -993,7 +1035,10 @@ impl Service { /// Forward iteration starting at from. #[tracing::instrument(skip(self), level = "debug")] pub async fn pdus<'a>( - &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, from: Option, + &'a self, + user_id: Option<&'a UserId>, + room_id: &'a RoomId, + from: Option, ) -> Result + Send + 'a> { self.db .pdus(user_id, room_id, from.unwrap_or_else(PduCount::min)) @@ -1002,17 +1047,21 @@ impl Service { /// Replace a PDU with the redacted form. #[tracing::instrument(skip(self, reason))] - pub async fn redact_pdu(&self, event_id: &EventId, reason: &PduEvent, shortroomid: ShortRoomId) -> Result { + pub async fn redact_pdu( + &self, + event_id: &EventId, + reason: &PduEvent, + shortroomid: ShortRoomId, + ) -> Result { // TODO: Don't reserialize, keep original json let Ok(pdu_id) = self.get_pdu_id(event_id).await else { // If event does not exist, just noop return Ok(()); }; - let mut pdu = self - .get_pdu_from_id(&pdu_id) - .await - .map_err(|e| err!(Database(error!(?pdu_id, ?event_id, ?e, "PDU ID points to invalid PDU."))))?; + let mut pdu = self.get_pdu_from_id(&pdu_id).await.map_err(|e| { + err!(Database(error!(?pdu_id, ?event_id, ?e, "PDU ID points to invalid PDU."))) + })?; if let Ok(content) = pdu.get_content::() { if let Some(body) = content.body { @@ -1026,8 +1075,9 @@ impl Service { pdu.redact(&room_version_id, reason)?; - let obj = utils::to_canonical_object(&pdu) - .map_err(|e| err!(Database(error!(?event_id, ?e, "Failed to convert PDU to canonical JSON"))))?; + let obj = utils::to_canonical_object(&pdu).map_err(|e| { + err!(Database(error!(?event_id, ?e, "Failed to convert PDU to canonical JSON"))) + })?; self.replace_pdu(&pdu_id, &obj, &pdu).await } @@ -1069,7 +1119,9 @@ impl Service { .unwrap_or_default(); let room_mods = power_levels.users.iter().filter_map(|(user_id, level)| { - if level > &power_levels.users_default && !self.services.globals.user_is_local(user_id) { + if level > &power_levels.users_default + && !self.services.globals.user_is_local(user_id) + { Some(user_id.server_name()) } else { None @@ -1124,7 +1176,7 @@ impl Service { ) .await; match response { - Ok(response) => { + | Ok(response) => { for pdu in response.pdus { if let Err(e) = self.backfill_pdu(backfill_server, pdu).boxed().await { debug_warn!("Failed to add backfilled pdu in room {room_id}: {e}"); @@ -1132,7 +1184,7 @@ impl Service { } return Ok(()); }, - Err(e) => { + | Err(e) => { warn!("{backfill_server} failed to provide backfill for room {room_id}: {e}"); }, } @@ -1144,7 +1196,8 @@ impl Service { #[tracing::instrument(skip(self, pdu))] pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> Result<()> { - let (event_id, value, room_id) = self.services.event_handler.parse_incoming_pdu(&pdu).await?; + let (event_id, value, room_id) = + self.services.event_handler.parse_incoming_pdu(&pdu).await?; // Lock so we cannot backfill the same pdu twice at the same time let mutex_lock = self @@ -1210,10 +1263,10 @@ impl Service { #[tracing::instrument(skip_all, level = "debug")] async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Result<()> { match pdu.event_type() { - TimelineEventType::RoomEncryption => { + | TimelineEventType::RoomEncryption => { return Err!(Request(Forbidden(error!("Encryption not supported in admins room.")))); }, - TimelineEventType::RoomMember => { + | TimelineEventType::RoomMember => { let target = pdu .state_key() .filter(|v| v.starts_with('@')) @@ -1223,9 +1276,11 @@ async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Res let content: RoomMemberEventContent = pdu.get_content()?; match content.membership { - MembershipState::Leave => { + | MembershipState::Leave => { if target == server_user { - return Err!(Request(Forbidden(error!("Server user cannot leave the admins room.")))); + return Err!(Request(Forbidden(error!( + "Server user cannot leave the admins room." + )))); } let count = self @@ -1239,13 +1294,17 @@ async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Res .await; if count < 2 { - return Err!(Request(Forbidden(error!("Last admin cannot leave the admins room.")))); + return Err!(Request(Forbidden(error!( + "Last admin cannot leave the admins room." + )))); } }, - MembershipState::Ban if pdu.state_key().is_some() => { + | MembershipState::Ban if pdu.state_key().is_some() => { if target == server_user { - return Err!(Request(Forbidden(error!("Server cannot be banned from admins room.")))); + return Err!(Request(Forbidden(error!( + "Server cannot be banned from admins room." + )))); } let count = self @@ -1259,13 +1318,15 @@ async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Res .await; if count < 2 { - return Err!(Request(Forbidden(error!("Last admin cannot be banned from admins room.")))); + return Err!(Request(Forbidden(error!( + "Last admin cannot be banned from admins room." + )))); } }, - _ => {}, + | _ => {}, }; }, - _ => {}, + | _ => {}, }; Ok(()) diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 59862a85..a6123322 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -52,7 +52,12 @@ impl crate::Service for Service { impl Service { /// Sets a user as typing until the timeout timestamp is reached or /// roomtyping_remove is called. - pub async fn typing_add(&self, user_id: &UserId, room_id: &RoomId, timeout: u64) -> Result<()> { + pub async fn typing_add( + &self, + user_id: &UserId, + room_id: &RoomId, + timeout: u64, + ) -> Result<()> { debug_info!("typing started {user_id:?} in {room_id:?} timeout:{timeout:?}"); // update clients self.typing @@ -177,15 +182,15 @@ impl Service { /// Returns a new typing EDU. pub async fn typings_all( - &self, room_id: &RoomId, sender_user: &UserId, + &self, + room_id: &RoomId, + sender_user: &UserId, ) -> Result> { let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); let Some(typing_indicators) = room_typing_indicators else { return Ok(SyncEphemeralRoomEvent { - content: ruma::events::typing::TypingEventContent { - user_ids: Vec::new(), - }, + content: ruma::events::typing::TypingEventContent { user_ids: Vec::new() }, }); }; @@ -204,13 +209,16 @@ impl Service { .await; Ok(SyncEphemeralRoomEvent { - content: ruma::events::typing::TypingEventContent { - user_ids, - }, + content: ruma::events::typing::TypingEventContent { user_ids }, }) } - async fn federation_send(&self, room_id: &RoomId, user_id: &UserId, typing: bool) -> Result<()> { + async fn federation_send( + &self, + room_id: &RoomId, + user_id: &UserId, + typing: bool, + ) -> Result<()> { debug_assert!( self.services.globals.user_is_local(user_id), "tried to broadcast typing status of remote user", diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 80d33de4..6a0c6aa1 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -92,7 +92,12 @@ pub async fn last_notification_read(&self, user_id: &UserId, room_id: &RoomId) - } #[implement(Service)] -pub async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, shortstatehash: ShortStateHash) { +pub async fn associate_token_shortstatehash( + &self, + room_id: &RoomId, + token: u64, + shortstatehash: ShortStateHash, +) { let shortroomid = self .services .short @@ -108,7 +113,11 @@ pub async fn associate_token_shortstatehash(&self, room_id: &RoomId, token: u64, } #[implement(Service)] -pub async fn get_token_shortstatehash(&self, room_id: &RoomId, token: u64) -> Result { +pub async fn get_token_shortstatehash( + &self, + room_id: &RoomId, + token: u64, +) -> Result { let shortroomid = self.services.short.get_shortroomid(room_id).await?; let key: &[u64] = &[shortroomid, token]; diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 9008a21f..6b58d964 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -3,14 +3,18 @@ use std::{fmt::Debug, mem}; use bytes::BytesMut; use conduwuit::{debug_error, err, trace, utils, warn, Err, Result}; use reqwest::Client; -use ruma::api::{appservice::Registration, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken}; +use ruma::api::{ + appservice::Registration, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, +}; /// Sends a request to an appservice /// /// Only returns Ok(None) if there is no url specified in the appservice /// registration file pub(crate) async fn send_request( - client: &Client, registration: Registration, request: T, + client: &Client, + registration: Registration, + request: T, ) -> Result> where T: OutgoingRequest + Debug + Send, @@ -25,17 +29,17 @@ where let hs_token = registration.hs_token.as_str(); let mut http_request = request - .try_into_http_request::(&dest, SendAccessToken::IfRequired(hs_token), &VERSIONS) + .try_into_http_request::( + &dest, + SendAccessToken::IfRequired(hs_token), + &VERSIONS, + ) .map_err(|e| err!(BadServerResponse(warn!("Failed to find destination {dest}: {e}"))))? .map(BytesMut::freeze); let mut parts = http_request.uri().clone().into_parts(); let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned(); - let symbol = if old_path_and_query.contains('?') { - "&" - } else { - "?" - }; + let symbol = if old_path_and_query.contains('?') { "&" } else { "?" }; parts.path_and_query = Some( (old_path_and_query + symbol + "access_token=" + hs_token) diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 372d8e14..ac06424f 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -43,7 +43,9 @@ impl Data { } } - pub(super) fn delete_active_request(&self, key: &[u8]) { self.servercurrentevent_data.remove(key); } + pub(super) fn delete_active_request(&self, key: &[u8]) { + self.servercurrentevent_data.remove(key); + } pub(super) async fn delete_all_active_requests_for(&self, destination: &Destination) { let prefix = destination.get_prefix(); @@ -76,11 +78,7 @@ impl Data { events .filter(|(key, _)| !key.is_empty()) .for_each(|(key, val)| { - let val = if let SendingEvent::Edu(val) = &val { - &**val - } else { - &[] - }; + let val = if let SendingEvent::Edu(val) = &val { &**val } else { &[] }; self.servercurrentevent_data.insert(key, val); self.servernameevent_data.remove(key); @@ -93,21 +91,26 @@ impl Data { .raw_stream() .ignore_err() .map(|(key, val)| { - let (dest, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); + let (dest, event) = + parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); (key.to_vec(), event, dest) }) } #[inline] - pub fn active_requests_for(&self, destination: &Destination) -> impl Stream + Send + '_ { + pub fn active_requests_for( + &self, + destination: &Destination, + ) -> impl Stream + Send + '_ { let prefix = destination.get_prefix(); self.servercurrentevent_data .raw_stream_from(&prefix) .ignore_err() .ready_take_while(move |(key, _)| key.starts_with(&prefix)) .map(|(key, val)| { - let (_, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); + let (_, event) = + parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); (key.to_vec(), event) }) @@ -150,14 +153,18 @@ impl Data { keys } - pub fn queued_requests(&self, destination: &Destination) -> impl Stream + Send + '_ { + pub fn queued_requests( + &self, + destination: &Destination, + ) -> impl Stream + Send + '_ { let prefix = destination.get_prefix(); self.servernameevent_data .raw_stream_from(&prefix) .ignore_err() .ready_take_while(move |(key, _)| key.starts_with(&prefix)) .map(|(key, val)| { - let (_, event) = parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); + let (_, event) = + parse_servercurrentevent(key, val).expect("invalid servercurrentevent"); (key.to_vec(), event) }) @@ -186,8 +193,9 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se .next() .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?; - let server = utils::string_from_bytes(server) - .map_err(|_| Error::bad_database("Invalid server bytes in server_currenttransaction"))?; + let server = utils::string_from_bytes(server).map_err(|_| { + Error::bad_database("Invalid server bytes in server_currenttransaction") + })?; ( Destination::Appservice(server), @@ -203,8 +211,8 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se let user = parts.next().expect("splitn always returns one element"); let user_string = utils::string_from_bytes(user) .map_err(|_| Error::bad_database("Invalid user string in servercurrentevent"))?; - let user_id = - UserId::parse(user_string).map_err(|_| Error::bad_database("Invalid user id in servercurrentevent"))?; + let user_id = UserId::parse(user_string) + .map_err(|_| Error::bad_database("Invalid user id in servercurrentevent"))?; let pushkey = parts .next() @@ -233,14 +241,14 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se .next() .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?; - let server = utils::string_from_bytes(server) - .map_err(|_| Error::bad_database("Invalid server bytes in server_currenttransaction"))?; + let server = utils::string_from_bytes(server).map_err(|_| { + Error::bad_database("Invalid server bytes in server_currenttransaction") + })?; ( - Destination::Normal( - ServerName::parse(server) - .map_err(|_| Error::bad_database("Invalid server string in server_currenttransaction"))?, - ), + Destination::Normal(ServerName::parse(server).map_err(|_| { + Error::bad_database("Invalid server string in server_currenttransaction") + })?), if value.is_empty() { SendingEvent::Pdu(event.into()) } else { diff --git a/src/service/sending/dest.rs b/src/service/sending/dest.rs index 0a5893b7..2c6063cc 100644 --- a/src/service/sending/dest.rs +++ b/src/service/sending/dest.rs @@ -14,7 +14,7 @@ pub enum Destination { #[must_use] pub(super) fn get_prefix(&self) -> Vec { match self { - Self::Normal(server) => { + | Self::Normal(server) => { let len = server.as_bytes().len().saturating_add(1); let mut p = Vec::with_capacity(len); @@ -22,7 +22,7 @@ pub(super) fn get_prefix(&self) -> Vec { p.push(0xFF); p }, - Self::Appservice(server) => { + | Self::Appservice(server) => { let sigil = b"+"; let len = sigil.len().saturating_add(server.len()).saturating_add(1); @@ -32,7 +32,7 @@ pub(super) fn get_prefix(&self) -> Vec { p.push(0xFF); p }, - Self::Push(user, pushkey) => { + | Self::Push(user, pushkey) => { let sigil = b"$"; let len = sigil .len() diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index f9828178..2038f4eb 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -25,8 +25,8 @@ pub use self::{ sender::{EDU_LIMIT, PDU_LIMIT}, }; use crate::{ - account_data, client, globals, presence, pusher, resolver, rooms, rooms::timeline::RawPduId, server_keys, users, - Dep, + account_data, client, globals, presence, pusher, resolver, rooms, rooms::timeline::RawPduId, + server_keys, users, Dep, }; pub struct Service { @@ -156,18 +156,16 @@ impl Service { { let _cork = self.db.db.cork(); let requests = servers - .map(|server| (Destination::Normal(server.into()), SendingEvent::Pdu(pdu_id.to_owned()))) + .map(|server| { + (Destination::Normal(server.into()), SendingEvent::Pdu(pdu_id.to_owned())) + }) .collect::>() .await; let keys = self.db.queue_requests(requests.iter().map(|(o, e)| (e, o))); for ((dest, event), queue_id) in requests.into_iter().zip(keys) { - self.dispatch(Msg { - dest, - event, - queue_id, - })?; + self.dispatch(Msg { dest, event, queue_id })?; } Ok(()) @@ -204,18 +202,16 @@ impl Service { { let _cork = self.db.db.cork(); let requests = servers - .map(|server| (Destination::Normal(server.to_owned()), SendingEvent::Edu(serialized.clone()))) + .map(|server| { + (Destination::Normal(server.to_owned()), SendingEvent::Edu(serialized.clone())) + }) .collect::>() .await; let keys = self.db.queue_requests(requests.iter().map(|(o, e)| (e, o))); for ((dest, event), queue_id) in requests.into_iter().zip(keys) { - self.dispatch(Msg { - dest, - event, - queue_id, - })?; + self.dispatch(Msg { dest, event, queue_id })?; } Ok(()) @@ -253,7 +249,11 @@ impl Service { /// Sends a request to a federation server #[tracing::instrument(skip_all, name = "request")] - pub async fn send_federation_request(&self, dest: &ServerName, request: T) -> Result + pub async fn send_federation_request( + &self, + dest: &ServerName, + request: T, + ) -> Result where T: OutgoingRequest + Debug + Send, { @@ -263,7 +263,11 @@ impl Service { /// Like send_federation_request() but with a very large timeout #[tracing::instrument(skip_all, name = "synapse")] - pub async fn send_synapse_request(&self, dest: &ServerName, request: T) -> Result + pub async fn send_synapse_request( + &self, + dest: &ServerName, + request: T, + ) -> Result where T: OutgoingRequest + Debug + Send, { @@ -276,7 +280,9 @@ impl Service { /// Only returns None if there is no url specified in the appservice /// registration file pub async fn send_appservice_request( - &self, registration: Registration, request: T, + &self, + registration: Registration, + request: T, ) -> Result> where T: OutgoingRequest + Debug + Send, @@ -291,24 +297,30 @@ impl Service { /// key #[tracing::instrument(skip(self), level = "debug")] pub async fn cleanup_events( - &self, appservice_id: Option<&str>, user_id: Option<&UserId>, push_key: Option<&str>, + &self, + appservice_id: Option<&str>, + user_id: Option<&UserId>, + push_key: Option<&str>, ) -> Result { match (appservice_id, user_id, push_key) { - (None, Some(user_id), Some(push_key)) => { + | (None, Some(user_id), Some(push_key)) => { self.db - .delete_all_requests_for(&Destination::Push(user_id.to_owned(), push_key.to_owned())) + .delete_all_requests_for(&Destination::Push( + user_id.to_owned(), + push_key.to_owned(), + )) .await; Ok(()) }, - (Some(appservice_id), None, None) => { + | (Some(appservice_id), None, None) => { self.db .delete_all_requests_for(&Destination::Appservice(appservice_id.to_owned())) .await; Ok(()) }, - _ => { + | _ => { debug_warn!("cleanup_events called with too many or too few arguments"); Ok(()) }, diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 862d2a42..81467c16 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -2,16 +2,16 @@ use std::mem; use bytes::Bytes; use conduwuit::{ - debug, debug_error, debug_warn, err, error::inspect_debug_log, implement, trace, utils::string::EMPTY, Err, Error, - Result, + debug, debug_error, debug_warn, err, error::inspect_debug_log, implement, trace, + utils::string::EMPTY, Err, Error, Result, }; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; use reqwest::{Client, Method, Request, Response, Url}; use ruma::{ api::{ - client::error::Error as RumaError, EndpointError, IncomingResponse, MatrixVersion, OutgoingRequest, - SendAccessToken, + client::error::Error as RumaError, EndpointError, IncomingResponse, MatrixVersion, + OutgoingRequest, SendAccessToken, }, serde::Base64, server_util::authorization::XMatrix, @@ -25,7 +25,12 @@ use crate::{ impl super::Service { #[tracing::instrument(skip_all, level = "debug")] - pub async fn send(&self, client: &Client, dest: &ServerName, request: T) -> Result + pub async fn send( + &self, + client: &Client, + dest: &ServerName, + request: T, + ) -> Result where T: OutgoingRequest + Send, { @@ -39,7 +44,9 @@ impl super::Service { .forbidden_remote_server_names .contains(dest) { - return Err!(Request(Forbidden(debug_warn!("Federation with {dest} is not allowed.")))); + return Err!(Request(Forbidden(debug_warn!( + "Federation with {dest} is not allowed." + )))); } let actual = self.services.resolver.get_actual_dest(dest).await?; @@ -49,7 +56,11 @@ impl super::Service { } async fn execute( - &self, dest: &ServerName, actual: &ActualDest, request: Request, client: &Client, + &self, + dest: &ServerName, + actual: &ActualDest, + request: Request, + client: &Client, ) -> Result where T: OutgoingRequest + Send, @@ -59,8 +70,18 @@ impl super::Service { debug!(?method, ?url, "Sending request"); match client.execute(request).await { - Ok(response) => handle_response::(&self.services.resolver, dest, actual, &method, &url, response).await, - Err(error) => Err(handle_error(actual, &method, &url, error).expect_err("always returns error")), + | Ok(response) => + handle_response::( + &self.services.resolver, + dest, + actual, + &method, + &url, + response, + ) + .await, + | Err(error) => + Err(handle_error(actual, &method, &url, error).expect_err("always returns error")), } } @@ -86,7 +107,11 @@ impl super::Service { } async fn handle_response( - resolver: &resolver::Service, dest: &ServerName, actual: &ActualDest, method: &Method, url: &Url, + resolver: &resolver::Service, + dest: &ServerName, + actual: &ActualDest, + method: &Method, + url: &Url, response: Response, ) -> Result where @@ -96,21 +121,22 @@ where let result = T::IncomingResponse::try_from_http_response(response); if result.is_ok() && !actual.cached { - resolver.set_cached_destination( - dest.to_owned(), - CachedDest { - dest: actual.dest.clone(), - host: actual.host.clone(), - expire: CachedDest::default_expire(), - }, - ); + resolver.set_cached_destination(dest.to_owned(), CachedDest { + dest: actual.dest.clone(), + host: actual.host.clone(), + expire: CachedDest::default_expire(), + }); } result.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}"))) } async fn into_http_response( - dest: &ServerName, actual: &ActualDest, method: &Method, url: &Url, mut response: Response, + dest: &ServerName, + actual: &ActualDest, + method: &Method, + url: &Url, + mut response: Response, ) -> Result> { let status = response.status(); trace!( @@ -146,13 +172,21 @@ async fn into_http_response( debug!("Got {status:?} for {method} {url}"); if !status.is_success() { - return Err(Error::Federation(dest.to_owned(), RumaError::from_http_response(http_response))); + return Err(Error::Federation( + dest.to_owned(), + RumaError::from_http_response(http_response), + )); } Ok(http_response) } -fn handle_error(actual: &ActualDest, method: &Method, url: &Url, mut e: reqwest::Error) -> Result { +fn handle_error( + actual: &ActualDest, + method: &Method, + url: &Url, + mut e: reqwest::Error, +) -> Result { if e.is_timeout() || e.is_connect() { e = e.without_url(); debug_warn!("{e:?}"); @@ -186,7 +220,8 @@ fn sign_request(&self, http_request: &mut http::Request>, dest: &ServerN .expect("http::Request missing path_and_query"); let mut req: Object = if !body.is_empty() { - let content: CanonicalJsonValue = serde_json::from_slice(body).expect("failed to serialize body"); + let content: CanonicalJsonValue = + serde_json::from_slice(body).expect("failed to serialize body"); let authorization: [Member; 5] = [ ("content".into(), content), diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 77fd7d5c..1f462f39 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -24,15 +24,19 @@ use ruma::{ appservice::event::push_events::v1::Edu as RumaEdu, federation::transactions::{ edu::{ - DeviceListUpdateContent, Edu, PresenceContent, PresenceUpdate, ReceiptContent, ReceiptData, ReceiptMap, + DeviceListUpdateContent, Edu, PresenceContent, PresenceUpdate, ReceiptContent, + ReceiptData, ReceiptMap, }, send_transaction_message, }, }, device_id, - events::{push_rules::PushRulesEvent, receipt::ReceiptType, AnySyncEphemeralRoomEvent, GlobalAccountDataEventType}, - push, uint, CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, - RoomVersionId, ServerName, UInt, + events::{ + push_rules::PushRulesEvent, receipt::ReceiptType, AnySyncEphemeralRoomEvent, + GlobalAccountDataEventType, + }, + push, uint, CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedServerName, + OwnedUserId, RoomId, RoomVersionId, ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; @@ -86,11 +90,14 @@ impl Service { } async fn handle_response<'a>( - &'a self, response: SendingResult, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, + &'a self, + response: SendingResult, + futures: &mut SendingFutures<'a>, + statuses: &mut CurTransactionStatus, ) { match response { - Ok(dest) => self.handle_response_ok(&dest, futures, statuses).await, - Err((dest, e)) => Self::handle_response_err(dest, statuses, &e), + | Ok(dest) => self.handle_response_ok(&dest, futures, statuses).await, + | Err((dest, e)) => Self::handle_response_err(dest, statuses, &e), }; } @@ -98,16 +105,22 @@ impl Service { debug!(dest = ?dest, "{e:?}"); statuses.entry(dest).and_modify(|e| { *e = match e { - TransactionStatus::Running => TransactionStatus::Failed(1, Instant::now()), - TransactionStatus::Retrying(ref n) => TransactionStatus::Failed(n.saturating_add(1), Instant::now()), - TransactionStatus::Failed(..) => panic!("Request that was not even running failed?!"), + | TransactionStatus::Running => TransactionStatus::Failed(1, Instant::now()), + | TransactionStatus::Retrying(ref n) => + TransactionStatus::Failed(n.saturating_add(1), Instant::now()), + | TransactionStatus::Failed(..) => { + panic!("Request that was not even running failed?!") + }, } }); } #[allow(clippy::needless_pass_by_ref_mut)] async fn handle_response_ok<'a>( - &'a self, dest: &Destination, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, + &'a self, + dest: &Destination, + futures: &mut SendingFutures<'a>, + statuses: &mut CurTransactionStatus, ) { let _cork = self.db.db.cork(); self.db.delete_all_active_requests_for(dest).await; @@ -133,7 +146,10 @@ impl Service { #[allow(clippy::needless_pass_by_ref_mut)] async fn handle_request<'a>( - &'a self, msg: Msg, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, + &'a self, + msg: Msg, + futures: &mut SendingFutures<'a>, + statuses: &mut CurTransactionStatus, ) { let iv = vec![(msg.queue_id, msg.event)]; if let Ok(Some(events)) = self.select_events(&msg.dest, iv, statuses).await { @@ -168,8 +184,13 @@ impl Service { } #[allow(clippy::needless_pass_by_ref_mut)] - async fn initial_requests<'a>(&'a self, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus) { - let keep = usize::try_from(self.server.config.startup_netburst_keep).unwrap_or(usize::MAX); + async fn initial_requests<'a>( + &'a self, + futures: &mut SendingFutures<'a>, + statuses: &mut CurTransactionStatus, + ) { + let keep = + usize::try_from(self.server.config.startup_netburst_keep).unwrap_or(usize::MAX); let mut txns = HashMap::>::new(); let mut active = self.db.active_requests().boxed(); @@ -240,7 +261,11 @@ impl Service { } #[tracing::instrument(skip_all, level = "debug")] - fn select_events_current(&self, dest: Destination, statuses: &mut CurTransactionStatus) -> Result<(bool, bool)> { + fn select_events_current( + &self, + dest: Destination, + statuses: &mut CurTransactionStatus, + ) -> Result<(bool, bool)> { let (mut allow, mut retry) = (true, false); statuses .entry(dest.clone()) // TODO: can we avoid cloning? @@ -278,7 +303,8 @@ impl Service { let events_len = AtomicUsize::default(); let max_edu_count = AtomicU64::new(since); - let device_changes = self.select_edus_device_changes(server_name, batch, &max_edu_count, &events_len); + let device_changes = + self.select_edus_device_changes(server_name, batch, &max_edu_count, &events_len); let receipts: OptionFuture<_> = self .server @@ -305,7 +331,11 @@ impl Service { /// Look for presence async fn select_edus_device_changes( - &self, server_name: &ServerName, since: (u64, u64), max_edu_count: &AtomicU64, events_len: &AtomicUsize, + &self, + server_name: &ServerName, + since: (u64, u64), + max_edu_count: &AtomicU64, + events_len: &AtomicUsize, ) -> Vec> { let mut events = Vec::new(); let server_rooms = self.services.state_cache.server_rooms(server_name); @@ -342,7 +372,8 @@ impl Service { keys: None, }); - let edu = serde_json::to_vec(&edu).expect("failed to serialize device list update to JSON"); + let edu = serde_json::to_vec(&edu) + .expect("failed to serialize device list update to JSON"); events.push(edu); if events_len.fetch_add(1, Ordering::Relaxed) >= SELECT_EDU_LIMIT - 1 { @@ -356,7 +387,10 @@ impl Service { /// Look for read receipts in this room async fn select_edus_receipts( - &self, server_name: &ServerName, since: (u64, u64), max_edu_count: &AtomicU64, + &self, + server_name: &ServerName, + since: (u64, u64), + max_edu_count: &AtomicU64, ) -> Option> { let server_rooms = self.services.state_cache.server_rooms(server_name); @@ -377,19 +411,21 @@ impl Service { return None; } - let receipt_content = Edu::Receipt(ReceiptContent { - receipts, - }); + let receipt_content = Edu::Receipt(ReceiptContent { receipts }); - let receipt_content = - serde_json::to_vec(&receipt_content).expect("Failed to serialize Receipt EDU to JSON vec"); + let receipt_content = serde_json::to_vec(&receipt_content) + .expect("Failed to serialize Receipt EDU to JSON vec"); Some(receipt_content) } /// Look for read receipts in this room async fn select_edus_receipts_room( - &self, room_id: &RoomId, since: (u64, u64), max_edu_count: &AtomicU64, num: &mut usize, + &self, + room_id: &RoomId, + since: (u64, u64), + max_edu_count: &AtomicU64, + num: &mut usize, ) -> ReceiptMap { let receipts = self .services @@ -444,14 +480,15 @@ impl Service { } } - ReceiptMap { - read, - } + ReceiptMap { read } } /// Look for presence async fn select_edus_presence( - &self, server_name: &ServerName, since: (u64, u64), max_edu_count: &AtomicU64, + &self, + server_name: &ServerName, + since: (u64, u64), + max_edu_count: &AtomicU64, ) -> Option> { let presence_since = self.services.presence.presence_since(since.0); @@ -511,7 +548,8 @@ impl Service { push: presence_updates.into_values().collect(), }); - let presence_content = serde_json::to_vec(&presence_content).expect("failed to serialize Presence EDU to JSON"); + let presence_content = serde_json::to_vec(&presence_content) + .expect("failed to serialize Presence EDU to JSON"); Some(presence_content) } @@ -519,21 +557,28 @@ impl Service { async fn send_events(&self, dest: Destination, events: Vec) -> SendingResult { //debug_assert!(!events.is_empty(), "sending empty transaction"); match dest { - Destination::Normal(ref server) => self.send_events_dest_normal(&dest, server, events).await, - Destination::Appservice(ref id) => self.send_events_dest_appservice(&dest, id, events).await, - Destination::Push(ref userid, ref pushkey) => { + | Destination::Normal(ref server) => + self.send_events_dest_normal(&dest, server, events).await, + | Destination::Appservice(ref id) => + self.send_events_dest_appservice(&dest, id, events).await, + | Destination::Push(ref userid, ref pushkey) => self.send_events_dest_push(&dest, userid, pushkey, events) - .await - }, + .await, } } #[tracing::instrument(skip(self, dest, events), name = "appservice")] async fn send_events_dest_appservice( - &self, dest: &Destination, id: &str, events: Vec, + &self, + dest: &Destination, + id: &str, + events: Vec, ) -> SendingResult { let Some(appservice) = self.services.appservice.get_registration(id).await else { - return Err((dest.clone(), err!(Database(warn!(?id, "Missing appservice registration"))))); + return Err(( + dest.clone(), + err!(Database(warn!(?id, "Missing appservice registration"))), + )); }; let mut pdu_jsons = Vec::with_capacity( @@ -550,12 +595,12 @@ impl Service { ); for event in &events { match event { - SendingEvent::Pdu(pdu_id) => { + | SendingEvent::Pdu(pdu_id) => { if let Ok(pdu) = self.services.timeline.get_pdu_from_id(pdu_id).await { pdu_jsons.push(pdu.to_room_event()); } }, - SendingEvent::Edu(edu) => { + | SendingEvent::Edu(edu) => { if appservice .receive_ephemeral .is_some_and(|receive_edus| receive_edus) @@ -565,14 +610,14 @@ impl Service { } } }, - SendingEvent::Flush => {}, // flush only; no new content + | SendingEvent::Flush => {}, // flush only; no new content } } let txn_hash = calculate_hash(events.iter().filter_map(|e| match e { - SendingEvent::Edu(b) => Some(&**b), - SendingEvent::Pdu(b) => Some(b.as_ref()), - SendingEvent::Flush => None, + | SendingEvent::Edu(b) => Some(&**b), + | SendingEvent::Pdu(b) => Some(b.as_ref()), + | SendingEvent::Flush => None, })); let txn_id = &*general_purpose::URL_SAFE_NO_PAD.encode(txn_hash); @@ -592,28 +637,35 @@ impl Service { ) .await { - Ok(_) => Ok(dest.clone()), - Err(e) => Err((dest.clone(), e)), + | Ok(_) => Ok(dest.clone()), + | Err(e) => Err((dest.clone(), e)), } } #[tracing::instrument(skip(self, dest, events), name = "push")] async fn send_events_dest_push( - &self, dest: &Destination, userid: &OwnedUserId, pushkey: &str, events: Vec, + &self, + dest: &Destination, + userid: &OwnedUserId, + pushkey: &str, + events: Vec, ) -> SendingResult { let Ok(pusher) = self.services.pusher.get_pusher(userid, pushkey).await else { - return Err((dest.clone(), err!(Database(error!(?userid, ?pushkey, "Missing pusher"))))); + return Err(( + dest.clone(), + err!(Database(error!(?userid, ?pushkey, "Missing pusher"))), + )); }; let mut pdus = Vec::new(); for event in &events { match event { - SendingEvent::Pdu(pdu_id) => { + | SendingEvent::Pdu(pdu_id) => { if let Ok(pdu) = self.services.timeline.get_pdu_from_id(pdu_id).await { pdus.push(pdu); } }, - SendingEvent::Edu(_) | SendingEvent::Flush => { + | SendingEvent::Edu(_) | SendingEvent::Flush => { // Push gateways don't need EDUs (?) and flush only; // no new content }, @@ -657,7 +709,10 @@ impl Service { #[tracing::instrument(skip(self, dest, events), name = "", level = "debug")] async fn send_events_dest_normal( - &self, dest: &Destination, server: &OwnedServerName, events: Vec, + &self, + dest: &Destination, + server: &OwnedServerName, + events: Vec, ) -> SendingResult { let mut pdu_jsons = Vec::with_capacity( events @@ -675,17 +730,16 @@ impl Service { for event in &events { match event { // TODO: check room version and remove event_id if needed - SendingEvent::Pdu(pdu_id) => { + | SendingEvent::Pdu(pdu_id) => { if let Ok(pdu) = self.services.timeline.get_pdu_json_from_id(pdu_id).await { pdu_jsons.push(self.convert_to_outgoing_federation_event(pdu).await); } }, - SendingEvent::Edu(edu) => { + | SendingEvent::Edu(edu) => if let Ok(raw) = serde_json::from_slice(edu) { edu_jsons.push(raw); - } - }, - SendingEvent::Flush => {}, // flush only; no new content + }, + | SendingEvent::Flush => {}, // flush only; no new content } } @@ -693,9 +747,9 @@ impl Service { // transaction"); let txn_hash = calculate_hash(events.iter().filter_map(|e| match e { - SendingEvent::Edu(b) => Some(&**b), - SendingEvent::Pdu(b) => Some(b.as_ref()), - SendingEvent::Flush => None, + | SendingEvent::Edu(b) => Some(&**b), + | SendingEvent::Pdu(b) => Some(b.as_ref()), + | SendingEvent::Flush => None, })); let txn_id = &*general_purpose::URL_SAFE_NO_PAD.encode(txn_hash); @@ -725,7 +779,10 @@ impl Service { } /// This does not return a full `Pdu` it is only to satisfy ruma's types. - pub async fn convert_to_outgoing_federation_event(&self, mut pdu_json: CanonicalJsonObject) -> Box { + pub async fn convert_to_outgoing_federation_event( + &self, + mut pdu_json: CanonicalJsonObject, + ) -> Box { if let Some(unsigned) = pdu_json .get_mut("unsigned") .and_then(|val| val.as_object_mut()) @@ -739,11 +796,11 @@ impl Service { .and_then(|val| RoomId::parse(val.as_str()?).ok()) { match self.services.state.get_room_version(&room_id).await { - Ok(room_version_id) => match room_version_id { - RoomVersionId::V1 | RoomVersionId::V2 => {}, - _ => _ = pdu_json.remove("event_id"), + | Ok(room_version_id) => match room_version_id { + | RoomVersionId::V1 | RoomVersionId::V2 => {}, + | _ => _ = pdu_json.remove("event_id"), }, - Err(_) => _ = pdu_json.remove("event_id"), + | Err(_) => _ = pdu_json.remove("event_id"), } } else { pdu_json.remove("event_id"); diff --git a/src/service/server_keys/acquire.rs b/src/service/server_keys/acquire.rs index 886c4750..305cbfef 100644 --- a/src/service/server_keys/acquire.rs +++ b/src/service/server_keys/acquire.rs @@ -4,11 +4,13 @@ use std::{ time::Duration, }; -use conduwuit::{debug, debug_error, debug_warn, error, implement, info, result::FlatOk, trace, warn}; +use conduwuit::{ + debug, debug_error, debug_warn, error, implement, info, result::FlatOk, trace, warn, +}; use futures::{stream::FuturesUnordered, StreamExt}; use ruma::{ - api::federation::discovery::ServerSigningKeys, serde::Raw, CanonicalJsonObject, OwnedServerName, - OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, + api::federation::discovery::ServerSigningKeys, serde::Raw, CanonicalJsonObject, + OwnedServerName, OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, }; use serde_json::value::RawValue as RawJsonValue; use tokio::time::{timeout_at, Instant}; @@ -79,7 +81,9 @@ where return; } - warn!("missing {missing_keys} keys for {missing_servers} servers from all notaries first"); + warn!( + "missing {missing_keys} keys for {missing_servers} servers from all notaries first" + ); } if !notary_only { @@ -101,13 +105,15 @@ where return; } - debug_warn!("still missing {missing_keys} keys for {missing_servers} servers from all notaries."); + debug_warn!( + "still missing {missing_keys} keys for {missing_servers} servers from all notaries." + ); } if missing_keys > 0 { warn!( - "did not obtain {missing_keys} keys for {missing_servers} servers out of {requested_keys} total keys for \ - {requested_servers} total servers." + "did not obtain {missing_keys} keys for {missing_servers} servers out of \ + {requested_keys} total keys for {requested_servers} total servers." ); } @@ -162,12 +168,15 @@ where #[implement(super::Service)] async fn acquire_origin( - &self, origin: OwnedServerName, mut key_ids: Vec, timeout: Instant, + &self, + origin: OwnedServerName, + mut key_ids: Vec, + timeout: Instant, ) -> (OwnedServerName, Vec) { match timeout_at(timeout, self.server_request(&origin)).await { - Err(e) => debug_warn!(?origin, "timed out: {e}"), - Ok(Err(e)) => debug_error!(?origin, "{e}"), - Ok(Ok(server_keys)) => { + | Err(e) => debug_warn!(?origin, "timed out: {e}"), + | Ok(Err(e)) => debug_error!(?origin, "{e}"), + | Ok(Ok(server_keys)) => { trace!( %origin, ?key_ids, @@ -192,19 +201,21 @@ where for notary in self.services.globals.trusted_servers() { let missing_keys = keys_count(&missing); let missing_servers = missing.len(); - debug!("Asking notary {notary} for {missing_keys} missing keys from {missing_servers} servers"); + debug!( + "Asking notary {notary} for {missing_keys} missing keys from {missing_servers} \ + servers" + ); let batch = missing .iter() .map(|(server, keys)| (server.borrow(), keys.iter().map(Borrow::borrow))); match self.batch_notary_request(notary, batch).await { - Err(e) => error!("Failed to contact notary {notary:?}: {e}"), - Ok(results) => { + | Err(e) => error!("Failed to contact notary {notary:?}: {e}"), + | Ok(results) => for server_keys in results { self.acquire_notary_result(&mut missing, server_keys).await; - } - }, + }, } } @@ -224,4 +235,6 @@ async fn acquire_notary_result(&self, missing: &mut Batch, server_keys: ServerSi } } -fn keys_count(batch: &Batch) -> usize { batch.iter().flat_map(|(_, key_ids)| key_ids.iter()).count() } +fn keys_count(batch: &Batch) -> usize { + batch.iter().flat_map(|(_, key_ids)| key_ids.iter()).count() +} diff --git a/src/service/server_keys/get.rs b/src/service/server_keys/get.rs index 94d2575a..5a027d64 100644 --- a/src/service/server_keys/get.rs +++ b/src/service/server_keys/get.rs @@ -1,17 +1,25 @@ use std::borrow::Borrow; use conduwuit::{implement, Err, Result}; -use ruma::{api::federation::discovery::VerifyKey, CanonicalJsonObject, RoomVersionId, ServerName, ServerSigningKeyId}; +use ruma::{ + api::federation::discovery::VerifyKey, CanonicalJsonObject, RoomVersionId, ServerName, + ServerSigningKeyId, +}; use super::{extract_key, PubKeyMap, PubKeys}; #[implement(super::Service)] -pub async fn get_event_keys(&self, object: &CanonicalJsonObject, version: &RoomVersionId) -> Result { +pub async fn get_event_keys( + &self, + object: &CanonicalJsonObject, + version: &RoomVersionId, +) -> Result { use ruma::signatures::required_keys; let required = match required_keys(object, version) { - Ok(required) => required, - Err(e) => return Err!(BadServerResponse("Failed to determine keys required to verify: {e}")), + | Ok(required) => required, + | Err(e) => + return Err!(BadServerResponse("Failed to determine keys required to verify: {e}")), }; let batch = required @@ -52,7 +60,11 @@ where } #[implement(super::Service)] -pub async fn get_verify_key(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> Result { +pub async fn get_verify_key( + &self, + origin: &ServerName, + key_id: &ServerSigningKeyId, +) -> Result { let notary_first = self.services.server.config.query_trusted_key_servers_first; let notary_only = self.services.server.config.only_query_trusted_key_servers; @@ -86,7 +98,11 @@ pub async fn get_verify_key(&self, origin: &ServerName, key_id: &ServerSigningKe } #[implement(super::Service)] -async fn get_verify_key_from_notaries(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> Result { +async fn get_verify_key_from_notaries( + &self, + origin: &ServerName, + key_id: &ServerSigningKeyId, +) -> Result { for notary in self.services.globals.trusted_servers() { if let Ok(server_keys) = self.notary_request(notary, origin).await { for server_key in server_keys.clone() { @@ -105,7 +121,11 @@ async fn get_verify_key_from_notaries(&self, origin: &ServerName, key_id: &Serve } #[implement(super::Service)] -async fn get_verify_key_from_origin(&self, origin: &ServerName, key_id: &ServerSigningKeyId) -> Result { +async fn get_verify_key_from_origin( + &self, + origin: &ServerName, + key_id: &ServerSigningKeyId, +) -> Result { if let Ok(server_key) = self.server_request(origin).await { self.add_signing_keys(server_key.clone()).await; if let Some(result) = extract_key(server_key, key_id) { diff --git a/src/service/server_keys/keypair.rs b/src/service/server_keys/keypair.rs index 927171f3..6f983c26 100644 --- a/src/service/server_keys/keypair.rs +++ b/src/service/server_keys/keypair.rs @@ -39,14 +39,15 @@ fn load(db: &Arc) -> Result> { create(db) })?; - let key = - Ed25519KeyPair::from_der(&key, version).map_err(|e| err!("Failed to load ed25519 keypair from der: {e:?}"))?; + let key = Ed25519KeyPair::from_der(&key, version) + .map_err(|e| err!("Failed to load ed25519 keypair from der: {e:?}"))?; Ok(Box::new(key)) } fn create(db: &Arc) -> Result<(String, Vec)> { - let keypair = Ed25519KeyPair::generate().map_err(|e| err!("Failed to generate new ed25519 keypair: {e:?}"))?; + let keypair = Ed25519KeyPair::generate() + .map_err(|e| err!("Failed to generate new ed25519 keypair: {e:?}"))?; let id = utils::rand::string(8); debug_info!("Generated new Ed25519 keypair: {id:?}"); diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index 45c01c0b..3f6a3039 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -18,8 +18,8 @@ use ruma::{ api::federation::discovery::{ServerSigningKeys, VerifyKey}, serde::Raw, signatures::{Ed25519KeyPair, PublicKeyMap, PublicKeySet}, - CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, RoomVersionId, ServerName, - ServerSigningKeyId, + CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, RoomVersionId, + ServerName, ServerSigningKeyId, }; use serde_json::value::RawValue as RawJsonValue; @@ -113,7 +113,11 @@ async fn add_signing_keys(&self, new_keys: ServerSigningKeys) { } #[implement(Service)] -pub async fn required_keys_exist(&self, object: &CanonicalJsonObject, version: &RoomVersionId) -> bool { +pub async fn required_keys_exist( + &self, + object: &CanonicalJsonObject, + version: &RoomVersionId, +) -> bool { use ruma::signatures::required_keys; let Ok(required_keys) = required_keys(object, version) else { @@ -179,7 +183,8 @@ pub async fn signing_keys_for(&self, origin: &ServerName) -> Result MilliSecondsSinceUnixEpoch { - let timepoint = timepoint_from_now(self.minimum_valid).expect("SystemTime should not overflow"); + let timepoint = + timepoint_from_now(self.minimum_valid).expect("SystemTime should not overflow"); MilliSecondsSinceUnixEpoch::from_system_time(timepoint).expect("UInt should not overflow") } diff --git a/src/service/server_keys/request.rs b/src/service/server_keys/request.rs index da7fa08a..afe8958b 100644 --- a/src/service/server_keys/request.rs +++ b/src/service/server_keys/request.rs @@ -12,7 +12,9 @@ use ruma::{ #[implement(super::Service)] pub(super) async fn batch_notary_request<'a, S, K>( - &self, notary: &ServerName, batch: S, + &self, + notary: &ServerName, + batch: S, ) -> Result> where S: Iterator + Send, @@ -74,7 +76,9 @@ where #[implement(super::Service)] pub async fn notary_request( - &self, notary: &ServerName, target: &ServerName, + &self, + notary: &ServerName, + target: &ServerName, ) -> Result + Clone + Debug + Send> { use get_remote_server_keys::v2::Request; diff --git a/src/service/server_keys/sign.rs b/src/service/server_keys/sign.rs index 71565380..8d6f108c 100644 --- a/src/service/server_keys/sign.rs +++ b/src/service/server_keys/sign.rs @@ -10,7 +10,11 @@ pub fn sign_json(&self, object: &mut CanonicalJsonObject) -> Result { } #[implement(super::Service)] -pub fn hash_and_sign_event(&self, object: &mut CanonicalJsonObject, room_version: &RoomVersionId) -> Result { +pub fn hash_and_sign_event( + &self, + object: &mut CanonicalJsonObject, + room_version: &RoomVersionId, +) -> Result { use ruma::signatures::hash_and_sign_event; let server_name = self.services.globals.server_name().as_str(); diff --git a/src/service/server_keys/verify.rs b/src/service/server_keys/verify.rs index d06b55ab..0f03e59e 100644 --- a/src/service/server_keys/verify.rs +++ b/src/service/server_keys/verify.rs @@ -1,14 +1,20 @@ use conduwuit::{implement, pdu::gen_event_id_canonical_json, Err, Result}; -use ruma::{signatures::Verified, CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, RoomVersionId}; +use ruma::{ + signatures::Verified, CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, RoomVersionId, +}; use serde_json::value::RawValue as RawJsonValue; #[implement(super::Service)] pub async fn validate_and_add_event_id( - &self, pdu: &RawJsonValue, room_version: &RoomVersionId, + &self, + pdu: &RawJsonValue, + room_version: &RoomVersionId, ) -> Result<(OwnedEventId, CanonicalJsonObject)> { let (event_id, mut value) = gen_event_id_canonical_json(pdu, room_version)?; if let Err(e) = self.verify_event(&value, Some(room_version)).await { - return Err!(BadServerResponse(debug_error!("Event {event_id} failed verification: {e:?}"))); + return Err!(BadServerResponse(debug_error!( + "Event {event_id} failed verification: {e:?}" + ))); } value.insert("event_id".into(), CanonicalJsonValue::String(event_id.as_str().into())); @@ -18,7 +24,9 @@ pub async fn validate_and_add_event_id( #[implement(super::Service)] pub async fn validate_and_add_event_id_no_fetch( - &self, pdu: &RawJsonValue, room_version: &RoomVersionId, + &self, + pdu: &RawJsonValue, + room_version: &RoomVersionId, ) -> Result<(OwnedEventId, CanonicalJsonObject)> { let (event_id, mut value) = gen_event_id_canonical_json(pdu, room_version)?; if !self.required_keys_exist(&value, room_version).await { @@ -28,7 +36,9 @@ pub async fn validate_and_add_event_id_no_fetch( } if let Err(e) = self.verify_event(&value, Some(room_version)).await { - return Err!(BadServerResponse(debug_error!("Event {event_id} failed verification: {e:?}"))); + return Err!(BadServerResponse(debug_error!( + "Event {event_id} failed verification: {e:?}" + ))); } value.insert("event_id".into(), CanonicalJsonValue::String(event_id.as_str().into())); @@ -38,7 +48,9 @@ pub async fn validate_and_add_event_id_no_fetch( #[implement(super::Service)] pub async fn verify_event( - &self, event: &CanonicalJsonObject, room_version: Option<&RoomVersionId>, + &self, + event: &CanonicalJsonObject, + room_version: Option<&RoomVersionId>, ) -> Result { let room_version = room_version.unwrap_or(&RoomVersionId::V11); let keys = self.get_event_keys(event, room_version).await?; @@ -46,7 +58,11 @@ pub async fn verify_event( } #[implement(super::Service)] -pub async fn verify_json(&self, event: &CanonicalJsonObject, room_version: Option<&RoomVersionId>) -> Result { +pub async fn verify_json( + &self, + event: &CanonicalJsonObject, + room_version: Option<&RoomVersionId>, +) -> Result { let room_version = room_version.unwrap_or(&RoomVersionId::V11); let keys = self.get_event_keys(event, room_version).await?; ruma::signatures::verify_json(&keys, event.clone()).map_err(Into::into) diff --git a/src/service/service.rs b/src/service/service.rs index 4b1774cc..7adb189e 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -114,7 +114,9 @@ impl<'a> Args<'a> { /// Create a reference immediately to a service when constructing another /// Service. The other service must be constructed. #[inline] - pub(crate) fn require(&'a self, name: &str) -> Arc { require::(self.service, name) } + pub(crate) fn require(&'a self, name: &str) -> Arc { + require::(self.service, name) + } } /// Reference a Service by name. Panics if the Service does not exist or was diff --git a/src/service/sync/mod.rs b/src/service/sync/mod.rs index a4523246..97f4ce9c 100644 --- a/src/service/sync/mod.rs +++ b/src/service/sync/mod.rs @@ -47,7 +47,8 @@ struct Services { struct SlidingSyncCache { lists: BTreeMap, subscriptions: BTreeMap, - known_rooms: BTreeMap>, // For every room, the roomsince number + known_rooms: BTreeMap>, /* For every room, the + * roomsince number */ extensions: ExtensionsConfig, } @@ -85,14 +86,24 @@ impl crate::Service for Service { } impl Service { - pub fn remembered(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) -> bool { + pub fn remembered( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + conn_id: String, + ) -> bool { self.connections .lock() .unwrap() .contains_key(&(user_id, device_id, conn_id)) } - pub fn forget_sync_request_connection(&self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String) { + pub fn forget_sync_request_connection( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + conn_id: String, + ) { self.connections .lock() .expect("locked") @@ -100,25 +111,26 @@ impl Service { } pub fn update_sync_request_with_cache( - &self, user_id: OwnedUserId, device_id: OwnedDeviceId, request: &mut sync_events::v4::Request, + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + request: &mut sync_events::v4::Request, ) -> BTreeMap> { let Some(conn_id) = request.conn_id.clone() else { return BTreeMap::new(); }; let mut cache = self.connections.lock().expect("locked"); - let cached = Arc::clone( - cache - .entry((user_id, device_id, conn_id)) - .or_insert_with(|| { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }), - ); + let cached = Arc::clone(cache.entry((user_id, device_id, conn_id)).or_insert_with( + || { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + }, + )); let cached = &mut cached.lock().expect("locked"); drop(cache); @@ -141,13 +153,15 @@ impl Service { .clone() .or_else(|| cached_list.include_old_rooms.clone()); match (&mut list.filters, cached_list.filters.clone()) { - (Some(list_filters), Some(cached_filters)) => { + | (Some(list_filters), Some(cached_filters)) => { list_filters.is_dm = list_filters.is_dm.or(cached_filters.is_dm); if list_filters.spaces.is_empty() { list_filters.spaces = cached_filters.spaces; } - list_filters.is_encrypted = list_filters.is_encrypted.or(cached_filters.is_encrypted); - list_filters.is_invite = list_filters.is_invite.or(cached_filters.is_invite); + list_filters.is_encrypted = + list_filters.is_encrypted.or(cached_filters.is_encrypted); + list_filters.is_invite = + list_filters.is_invite.or(cached_filters.is_invite); if list_filters.room_types.is_empty() { list_filters.room_types = cached_filters.room_types; } @@ -165,9 +179,9 @@ impl Service { list_filters.not_tags = cached_filters.not_tags; } }, - (_, Some(cached_filters)) => list.filters = Some(cached_filters), - (Some(list_filters), _) => list.filters = Some(list_filters.clone()), - (..) => {}, + | (_, Some(cached_filters)) => list.filters = Some(cached_filters), + | (Some(list_filters), _) => list.filters = Some(list_filters.clone()), + | (..) => {}, } if list.bump_event_types.is_empty() { list.bump_event_types @@ -220,22 +234,23 @@ impl Service { } pub fn update_sync_subscriptions( - &self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String, + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + conn_id: String, subscriptions: BTreeMap, ) { let mut cache = self.connections.lock().expect("locked"); - let cached = Arc::clone( - cache - .entry((user_id, device_id, conn_id)) - .or_insert_with(|| { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }), - ); + let cached = Arc::clone(cache.entry((user_id, device_id, conn_id)).or_insert_with( + || { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + }, + )); let cached = &mut cached.lock().expect("locked"); drop(cache); @@ -243,22 +258,25 @@ impl Service { } pub fn update_sync_known_rooms( - &self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String, list_id: String, - new_cached_rooms: BTreeSet, globalsince: u64, + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + conn_id: String, + list_id: String, + new_cached_rooms: BTreeSet, + globalsince: u64, ) { let mut cache = self.connections.lock().expect("locked"); - let cached = Arc::clone( - cache - .entry((user_id, device_id, conn_id)) - .or_insert_with(|| { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }), - ); + let cached = Arc::clone(cache.entry((user_id, device_id, conn_id)).or_insert_with( + || { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + }, + )); let cached = &mut cached.lock().expect("locked"); drop(cache); diff --git a/src/service/transaction_ids/mod.rs b/src/service/transaction_ids/mod.rs index 2b979f99..912c0b49 100644 --- a/src/service/transaction_ids/mod.rs +++ b/src/service/transaction_ids/mod.rs @@ -25,7 +25,13 @@ impl crate::Service for Service { } #[implement(Service)] -pub fn add_txnid(&self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, data: &[u8]) { +pub fn add_txnid( + &self, + user_id: &UserId, + device_id: Option<&DeviceId>, + txn_id: &TransactionId, + data: &[u8], +) { let mut key = user_id.as_bytes().to_vec(); key.push(0xFF); key.extend_from_slice(device_id.map(DeviceId::as_bytes).unwrap_or_default()); @@ -38,7 +44,10 @@ pub fn add_txnid(&self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: // If there's no entry, this is a new transaction #[implement(Service)] pub async fn existing_txnid( - &self, user_id: &UserId, device_id: Option<&DeviceId>, txn_id: &TransactionId, + &self, + user_id: &UserId, + device_id: Option<&DeviceId>, + txn_id: &TransactionId, ) -> Result> { let key = (user_id, device_id, txn_id); self.db.userdevicetxnid_response.qry(&key).await diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 4d30a612..f7e55251 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -58,7 +58,13 @@ impl crate::Service for Service { /// Creates a new Uiaa session. Make sure the session token is unique. #[implement(Service)] -pub fn create(&self, user_id: &UserId, device_id: &DeviceId, uiaainfo: &UiaaInfo, json_body: &CanonicalJsonValue) { +pub fn create( + &self, + user_id: &UserId, + device_id: &DeviceId, + uiaainfo: &UiaaInfo, + json_body: &CanonicalJsonValue, +) { // TODO: better session error handling (why is uiaainfo.session optional in // ruma?) self.set_uiaa_request( @@ -78,7 +84,11 @@ pub fn create(&self, user_id: &UserId, device_id: &DeviceId, uiaainfo: &UiaaInfo #[implement(Service)] pub async fn try_auth( - &self, user_id: &UserId, device_id: &DeviceId, auth: &AuthData, uiaainfo: &UiaaInfo, + &self, + user_id: &UserId, + device_id: &DeviceId, + auth: &AuthData, + uiaainfo: &UiaaInfo, ) -> Result<(bool, UiaaInfo)> { let mut uiaainfo = if let Some(session) = auth.session() { self.get_uiaa_session(user_id, device_id, session).await? @@ -92,7 +102,7 @@ pub async fn try_auth( match auth { // Find out what the user completed - AuthData::Password(Password { + | AuthData::Password(Password { identifier, password, #[cfg(feature = "element_hacks")] @@ -105,17 +115,26 @@ pub async fn try_auth( } else if let Some(username) = user { username } else { - return Err(Error::BadRequest(ErrorKind::Unrecognized, "Identifier type not recognized.")); + return Err(Error::BadRequest( + ErrorKind::Unrecognized, + "Identifier type not recognized.", + )); }; #[cfg(not(feature = "element_hacks"))] let Some(UserIdentifier::UserIdOrLocalpart(username)) = identifier else { - return Err(Error::BadRequest(ErrorKind::Unrecognized, "Identifier type not recognized.")); + return Err(Error::BadRequest( + ErrorKind::Unrecognized, + "Identifier type not recognized.", + )); }; - let user_id = UserId::parse_with_server_name(username.clone(), self.services.globals.server_name()) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid."))?; + let user_id = UserId::parse_with_server_name( + username.clone(), + self.services.globals.server_name(), + ) + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid."))?; // Check if password is correct if let Ok(hash) = self.services.users.password_hash(&user_id).await { @@ -132,7 +151,7 @@ pub async fn try_auth( // Password was correct! Let's add it to `completed` uiaainfo.completed.push(AuthType::Password); }, - AuthData::RegistrationToken(t) => { + | AuthData::RegistrationToken(t) => { if self .services .globals @@ -149,10 +168,10 @@ pub async fn try_auth( return Ok((false, uiaainfo)); } }, - AuthData::Dummy(_) => { + | AuthData::Dummy(_) => { uiaainfo.completed.push(AuthType::Dummy); }, - k => error!("type not supported: {:?}", k), + | k => error!("type not supported: {:?}", k), } // Check if a flow now succeeds @@ -190,7 +209,13 @@ pub async fn try_auth( } #[implement(Service)] -fn set_uiaa_request(&self, user_id: &UserId, device_id: &DeviceId, session: &str, request: &CanonicalJsonValue) { +fn set_uiaa_request( + &self, + user_id: &UserId, + device_id: &DeviceId, + session: &str, + request: &CanonicalJsonValue, +) { let key = (user_id.to_owned(), device_id.to_owned(), session.to_owned()); self.userdevicesessionid_uiaarequest .write() @@ -200,7 +225,10 @@ fn set_uiaa_request(&self, user_id: &UserId, device_id: &DeviceId, session: &str #[implement(Service)] pub fn get_uiaa_request( - &self, user_id: &UserId, device_id: Option<&DeviceId>, session: &str, + &self, + user_id: &UserId, + device_id: Option<&DeviceId>, + session: &str, ) -> Option { let key = ( user_id.to_owned(), @@ -216,7 +244,13 @@ pub fn get_uiaa_request( } #[implement(Service)] -fn update_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: &str, uiaainfo: Option<&UiaaInfo>) { +fn update_uiaa_session( + &self, + user_id: &UserId, + device_id: &DeviceId, + session: &str, + uiaainfo: Option<&UiaaInfo>, +) { let key = (user_id, device_id, session); if let Some(uiaainfo) = uiaainfo { @@ -229,7 +263,12 @@ fn update_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: & } #[implement(Service)] -async fn get_uiaa_session(&self, user_id: &UserId, device_id: &DeviceId, session: &str) -> Result { +async fn get_uiaa_session( + &self, + user_id: &UserId, + device_id: &DeviceId, + session: &str, +) -> Result { let key = (user_id, device_id, session); self.db .userdevicesessionid_uiaainfo diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index 1d13337d..1f499692 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -119,7 +119,8 @@ impl Service { self.services .admin .send_message(RoomMessageEventContent::text_markdown(format!( - "### the following is a message from the conduwuit puppy\n\nit was sent on `{}`:\n\n@room: {}", + "### the following is a message from the conduwuit puppy\n\nit was sent on \ + `{}`:\n\n@room: {}", update.date, update.message ))) .await @@ -127,7 +128,9 @@ impl Service { } #[inline] - pub fn update_check_for_updates_id(&self, id: u64) { self.db.raw_put(LAST_CHECK_FOR_UPDATES_COUNT, id); } + pub fn update_check_for_updates_id(&self, id: u64) { + self.db.raw_put(LAST_CHECK_FOR_UPDATES_COUNT, id); + } pub async fn last_check_for_updates_id(&self) -> u64 { self.db diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 8ceec2a0..1d7f4248 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -10,10 +10,12 @@ use futures::{FutureExt, Stream, StreamExt, TryFutureExt}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, - events::{ignored_user_list::IgnoredUserListEvent, AnyToDeviceEvent, GlobalAccountDataEventType}, + events::{ + ignored_user_list::IgnoredUserListEvent, AnyToDeviceEvent, GlobalAccountDataEventType, + }, serde::Raw, - DeviceId, KeyId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, OneTimeKeyName, OwnedDeviceId, - OwnedKeyId, OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, + DeviceId, KeyId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, + OneTimeKeyName, OwnedDeviceId, OwnedKeyId, OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, }; use serde_json::json; @@ -65,7 +67,8 @@ impl crate::Service for Service { account_data: args.depend::("account_data"), admin: args.depend::("admin"), globals: args.depend::("globals"), - state_accessor: args.depend::("rooms::state_accessor"), + state_accessor: args + .depend::("rooms::state_accessor"), state_cache: args.depend::("rooms::state_cache"), }, db: Data { @@ -114,7 +117,9 @@ impl Service { /// Check if a user is an admin #[inline] - pub async fn is_admin(&self, user_id: &UserId) -> bool { self.services.admin.user_is_admin(user_id).await } + pub async fn is_admin(&self, user_id: &UserId) -> bool { + self.services.admin.user_is_admin(user_id).await + } /// Create a new user account on this homeserver. #[inline] @@ -141,7 +146,9 @@ impl Service { /// Check if a user has an account on this homeserver. #[inline] - pub async fn exists(&self, user_id: &UserId) -> bool { self.db.userid_password.get(user_id).await.is_ok() } + pub async fn exists(&self, user_id: &UserId) -> bool { + self.db.userid_password.get(user_id).await.is_ok() + } /// Check if account is deactivated pub async fn is_deactivated(&self, user_id: &UserId) -> Result { @@ -154,7 +161,9 @@ impl Service { } /// Check if account is active, infallible - pub async fn is_active(&self, user_id: &UserId) -> bool { !self.is_deactivated(user_id).await.unwrap_or(true) } + pub async fn is_active(&self, user_id: &UserId) -> bool { + !self.is_deactivated(user_id).await.unwrap_or(true) + } /// Check if account is active, infallible pub async fn is_active_local(&self, user_id: &UserId) -> bool { @@ -173,10 +182,14 @@ impl Service { /// Returns an iterator over all users on this homeserver (offered for /// compatibility) #[allow(clippy::iter_without_into_iter, clippy::iter_not_returning_iterator)] - pub fn iter(&self) -> impl Stream + Send + '_ { self.stream().map(ToOwned::to_owned) } + pub fn iter(&self) -> impl Stream + Send + '_ { + self.stream().map(ToOwned::to_owned) + } /// Returns an iterator over all users on this homeserver. - pub fn stream(&self) -> impl Stream + Send { self.db.userid_password.keys().ignore_err() } + pub fn stream(&self) -> impl Stream + Send { + self.db.userid_password.keys().ignore_err() + } /// Returns a list of local users as list of usernames. /// @@ -200,7 +213,9 @@ impl Service { password .map(utils::hash::password) .transpose() - .map_err(|e| err!(Request(InvalidParam("Password does not meet the requirements: {e}"))))? + .map_err(|e| { + err!(Request(InvalidParam("Password does not meet the requirements: {e}"))) + })? .map_or_else( || self.db.userid_password.insert(user_id, b""), |hash| self.db.userid_password.insert(user_id, hash), @@ -254,13 +269,19 @@ impl Service { /// Adds a new device to a user. pub async fn create_device( - &self, user_id: &UserId, device_id: &DeviceId, token: &str, initial_device_display_name: Option, + &self, + user_id: &UserId, + device_id: &DeviceId, + token: &str, + initial_device_display_name: Option, client_ip: Option, ) -> Result<()> { // This method should never be called for nonexistent users. We shouldn't assert // though... if !self.exists(user_id).await { - return Err!(Request(InvalidParam(error!("Called create_device for non-existent {user_id}")))); + return Err!(Request(InvalidParam(error!( + "Called create_device for non-existent {user_id}" + )))); } let key = (user_id, device_id); @@ -304,7 +325,10 @@ impl Service { } /// Returns an iterator over all device ids of this user. - pub fn all_device_ids<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + pub fn all_device_ids<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { let prefix = (user_id, Interfix); self.db .userdeviceid_metadata @@ -319,7 +343,12 @@ impl Service { } /// Replaces the access token of one device. - pub async fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> { + pub async fn set_token( + &self, + user_id: &UserId, + device_id: &DeviceId, + token: &str, + ) -> Result<()> { let key = (user_id, device_id); // should not be None, but we shouldn't assert either lol... if self.db.userdeviceid_metadata.qry(&key).await.is_err() { @@ -344,7 +373,10 @@ impl Service { } pub async fn add_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, one_time_key_key: &KeyId, + &self, + user_id: &UserId, + device_id: &DeviceId, + one_time_key_key: &KeyId, one_time_key_value: &Raw, ) -> Result { // All devices have metadata @@ -391,7 +423,10 @@ impl Service { } pub async fn take_one_time_key( - &self, user_id: &UserId, device_id: &DeviceId, key_algorithm: &OneTimeKeyAlgorithm, + &self, + user_id: &UserId, + device_id: &DeviceId, + key_algorithm: &OneTimeKeyAlgorithm, ) -> Result<(OwnedKeyId, Raw)> { let count = self.services.globals.next_count()?.to_be_bytes(); self.db.userid_lastonetimekeyupdate.insert(user_id, count); @@ -435,7 +470,9 @@ impl Service { } pub async fn count_one_time_keys( - &self, user_id: &UserId, device_id: &DeviceId, + &self, + user_id: &UserId, + device_id: &DeviceId, ) -> BTreeMap { type KeyVal<'a> = ((Ignore, Ignore, &'a Unquoted), Ignore); @@ -462,7 +499,12 @@ impl Service { algorithm_counts } - pub async fn add_device_keys(&self, user_id: &UserId, device_id: &DeviceId, device_keys: &Raw) { + pub async fn add_device_keys( + &self, + user_id: &UserId, + device_id: &DeviceId, + device_keys: &Raw, + ) { let key = (user_id, device_id); self.db.keyid_key.put(key, Json(device_keys)); @@ -470,8 +512,12 @@ impl Service { } pub async fn add_cross_signing_keys( - &self, user_id: &UserId, master_key: &Raw, self_signing_key: &Option>, - user_signing_key: &Option>, notify: bool, + &self, + user_id: &UserId, + master_key: &Raw, + self_signing_key: &Option>, + user_signing_key: &Option>, + notify: bool, ) -> Result<()> { // TODO: Check signatures let mut prefix = user_id.as_bytes().to_vec(); @@ -495,9 +541,10 @@ impl Service { .keys .into_values(); - let self_signing_key_id = self_signing_key_ids - .next() - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Self signing key contained no key."))?; + let self_signing_key_id = self_signing_key_ids.next().ok_or(Error::BadRequest( + ErrorKind::InvalidParam, + "Self signing key contained no key.", + ))?; if self_signing_key_ids.next().is_some() { return Err(Error::BadRequest( @@ -531,7 +578,9 @@ impl Service { .ok_or(err!(Request(InvalidParam("User signing key contained no key."))))?; if user_signing_key_ids.next().is_some() { - return Err!(Request(InvalidParam("User signing key contained more than one key."))); + return Err!(Request(InvalidParam( + "User signing key contained more than one key." + ))); } let mut user_signing_key_key = prefix; @@ -554,7 +603,11 @@ impl Service { } pub async fn sign_key( - &self, target_id: &UserId, key_id: &str, signature: (String, String), sender_id: &UserId, + &self, + target_id: &UserId, + key_id: &str, + signature: (String, String), + sender_id: &UserId, ) -> Result<()> { let key = (target_id, key_id); @@ -590,7 +643,10 @@ impl Service { #[inline] pub fn keys_changed<'a>( - &'a self, user_id: &'a UserId, from: u64, to: Option, + &'a self, + user_id: &'a UserId, + from: u64, + to: Option, ) -> impl Stream + Send + 'a { self.keys_changed_user_or_room(user_id.as_str(), from, to) .map(|(user_id, ..)| user_id) @@ -598,13 +654,19 @@ impl Service { #[inline] pub fn room_keys_changed<'a>( - &'a self, room_id: &'a RoomId, from: u64, to: Option, + &'a self, + room_id: &'a RoomId, + from: u64, + to: Option, ) -> impl Stream + Send + 'a { self.keys_changed_user_or_room(room_id.as_str(), from, to) } fn keys_changed_user_or_room<'a>( - &'a self, user_or_room_id: &'a str, from: u64, to: Option, + &'a self, + user_or_room_id: &'a str, + from: u64, + to: Option, ) -> impl Stream + Send + 'a { type KeyVal<'a> = ((&'a str, u64), &'a UserId); @@ -614,7 +676,9 @@ impl Service { .keychangeid_userid .stream_from(&start) .ignore_err() - .ready_take_while(move |((prefix, count), _): &KeyVal<'_>| *prefix == user_or_room_id && *count <= to) + .ready_take_while(move |((prefix, count), _): &KeyVal<'_>| { + *prefix == user_or_room_id && *count <= to + }) .map(|((_, count), user_id): KeyVal<'_>| (user_id, count)) } @@ -636,13 +700,21 @@ impl Service { self.db.keychangeid_userid.put_raw(key, user_id); } - pub async fn get_device_keys<'a>(&'a self, user_id: &'a UserId, device_id: &DeviceId) -> Result> { + pub async fn get_device_keys<'a>( + &'a self, + user_id: &'a UserId, + device_id: &DeviceId, + ) -> Result> { let key_id = (user_id, device_id); self.db.keyid_key.qry(&key_id).await.deserialized() } pub async fn get_key( - &self, key_id: &[u8], sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &F, + &self, + key_id: &[u8], + sender_user: Option<&UserId>, + user_id: &UserId, + allowed_signatures: &F, ) -> Result> where F: Fn(&UserId) -> bool + Send + Sync, @@ -655,7 +727,10 @@ impl Service { } pub async fn get_master_key( - &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &F, + &self, + sender_user: Option<&UserId>, + user_id: &UserId, + allowed_signatures: &F, ) -> Result> where F: Fn(&UserId) -> bool + Send + Sync, @@ -667,7 +742,10 @@ impl Service { } pub async fn get_self_signing_key( - &self, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &F, + &self, + sender_user: Option<&UserId>, + user_id: &UserId, + allowed_signatures: &F, ) -> Result> where F: Fn(&UserId) -> bool + Send + Sync, @@ -688,7 +766,11 @@ impl Service { } pub async fn add_to_device_event( - &self, sender: &UserId, target_user_id: &UserId, target_device_id: &DeviceId, event_type: &str, + &self, + sender: &UserId, + target_user_id: &UserId, + target_device_id: &DeviceId, + event_type: &str, content: serde_json::Value, ) { let count = self.services.globals.next_count().unwrap(); @@ -705,7 +787,9 @@ impl Service { } pub fn get_to_device_events<'a>( - &'a self, user_id: &'a UserId, device_id: &'a DeviceId, + &'a self, + user_id: &'a UserId, + device_id: &'a DeviceId, ) -> impl Stream> + Send + 'a { let prefix = (user_id, device_id, Interfix); self.db @@ -715,7 +799,12 @@ impl Service { .map(|(_, val): (Ignore, Raw)| val) } - pub async fn remove_to_device_events(&self, user_id: &UserId, device_id: &DeviceId, until: u64) { + pub async fn remove_to_device_events( + &self, + user_id: &UserId, + device_id: &DeviceId, + until: u64, + ) { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); prefix.extend_from_slice(device_id.as_bytes()); @@ -742,7 +831,12 @@ impl Service { .await; } - pub async fn update_device_metadata(&self, user_id: &UserId, device_id: &DeviceId, device: &Device) -> Result<()> { + pub async fn update_device_metadata( + &self, + user_id: &UserId, + device_id: &DeviceId, + device: &Device, + ) -> Result<()> { increment(&self.db.userid_devicelistversion, user_id.as_bytes()); let key = (user_id, device_id); @@ -752,7 +846,11 @@ impl Service { } /// Get device metadata. - pub async fn get_device_metadata(&self, user_id: &UserId, device_id: &DeviceId) -> Result { + pub async fn get_device_metadata( + &self, + user_id: &UserId, + device_id: &DeviceId, + ) -> Result { self.db .userdeviceid_metadata .qry(&(user_id, device_id)) @@ -768,7 +866,10 @@ impl Service { .deserialized() } - pub fn all_devices_metadata<'a>(&'a self, user_id: &'a UserId) -> impl Stream + Send + 'a { + pub fn all_devices_metadata<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { let key = (user_id, Interfix); self.db .userdeviceid_metadata @@ -787,7 +888,11 @@ impl Service { filter_id } - pub async fn get_filter(&self, user_id: &UserId, filter_id: &str) -> Result { + pub async fn get_filter( + &self, + user_id: &UserId, + filter_id: &str, + ) -> Result { let key = (user_id, filter_id); self.db.userfilterid_filter.qry(&key).await.deserialized() } @@ -817,11 +922,10 @@ impl Service { }; let (expires_at_bytes, user_bytes) = value.split_at(0_u64.to_be_bytes().len()); - let expires_at = u64::from_be_bytes( - expires_at_bytes - .try_into() - .map_err(|e| err!(Database("expires_at in openid_userid is invalid u64. {e}")))?, - ); + let expires_at = + u64::from_be_bytes(expires_at_bytes.try_into().map_err(|e| { + err!(Database("expires_at in openid_userid is invalid u64. {e}")) + })?); if expires_at < utils::millis_since_unix_epoch() { debug_warn!("OpenID token is expired, removing"); @@ -833,11 +937,16 @@ impl Service { let user_string = utils::string_from_bytes(user_bytes) .map_err(|e| err!(Database("User ID in openid_userid is invalid unicode. {e}")))?; - UserId::parse(user_string).map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) + UserId::parse(user_string) + .map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) } /// Gets a specific user profile key - pub async fn profile_key(&self, user_id: &UserId, profile_key: &str) -> Result { + pub async fn profile_key( + &self, + user_id: &UserId, + profile_key: &str, + ) -> Result { let key = (user_id, profile_key); self.db .useridprofilekey_value @@ -848,7 +957,8 @@ impl Service { /// Gets all the user's profile keys and values in an iterator pub fn all_profile_keys<'a>( - &'a self, user_id: &'a UserId, + &'a self, + user_id: &'a UserId, ) -> impl Stream + 'a + Send { type KeyVal = ((Ignore, String), serde_json::Value); @@ -861,7 +971,12 @@ impl Service { } /// Sets a new profile key value, removes the key if value is None - pub fn set_profile_key(&self, user_id: &UserId, profile_key: &str, profile_key_value: Option) { + pub fn set_profile_key( + &self, + user_id: &UserId, + profile_key: &str, + profile_key_value: Option, + ) { // TODO: insert to the stable MSC4175 key when it's stable let key = (user_id, profile_key); @@ -901,7 +1016,10 @@ impl Service { } } -pub fn parse_master_key(user_id: &UserId, master_key: &Raw) -> Result<(Vec, CrossSigningKey)> { +pub fn parse_master_key( + user_id: &UserId, + master_key: &Raw, +) -> Result<(Vec, CrossSigningKey)> { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); @@ -925,7 +1043,10 @@ pub fn parse_master_key(user_id: &UserId, master_key: &Raw) -> /// Ensure that a user only sees signatures from themselves and the target user fn clean_signatures( - mut cross_signing_key: serde_json::Value, sender_user: Option<&UserId>, user_id: &UserId, allowed_signatures: &F, + mut cross_signing_key: serde_json::Value, + sender_user: Option<&UserId>, + user_id: &UserId, + allowed_signatures: &F, ) -> Result where F: Fn(&UserId) -> bool + Send + Sync, @@ -937,9 +1058,11 @@ where // Don't allocate for the full size of the current signatures, but require // at most one resize if nothing is dropped let new_capacity = signatures.len() / 2; - for (user, signature) in mem::replace(signatures, serde_json::Map::with_capacity(new_capacity)) { - let sid = - <&UserId>::try_from(user.as_str()).map_err(|_| Error::bad_database("Invalid user ID in database."))?; + for (user, signature) in + mem::replace(signatures, serde_json::Map::with_capacity(new_capacity)) + { + let sid = <&UserId>::try_from(user.as_str()) + .map_err(|_| Error::bad_database("Invalid user ID in database."))?; if sender_user == Some(user_id) || sid == user_id || allowed_signatures(sid) { signatures.insert(user, signature); } From 52693db4779d8c19c446b4bd967815a8d6cd1d34 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 01:02:52 -0500 Subject: [PATCH 1230/2091] hide messages from users servers on the forbidden_remote_server_names list Signed-off-by: strawberry --- src/api/client/message.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 93582de0..800dce9d 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -237,6 +237,17 @@ pub(crate) async fn ignored_filter( return None; } + if IGNORED_MESSAGE_TYPES.iter().any(is_equal_to!(&pdu.kind)) + && services + .server + .config + .forbidden_remote_server_names + .iter() + .any(is_equal_to!(pdu.sender().server_name())) + { + return None; + } + Some(item) } From 6c96acc482baaacb784b8dc1b1c1e4c2a34f3c3b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 01:41:50 -0500 Subject: [PATCH 1231/2091] use fast binary_search for some const slices Signed-off-by: strawberry --- src/api/client/message.rs | 53 ++++++++++++------------- src/api/client/room/upgrade.rs | 12 ++++-- src/api/client/sync/v4.rs | 7 ++-- src/service/rooms/state_accessor/mod.rs | 4 +- src/service/rooms/timeline/mod.rs | 2 +- src/service/updates/mod.rs | 2 +- 6 files changed, 41 insertions(+), 39 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 800dce9d..58f4f916 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -27,23 +27,24 @@ use crate::Ruma; pub(crate) type LazySet = HashSet; /// list of safe and common non-state events to ignore if the user is ignored -const IGNORED_MESSAGE_TYPES: &[TimelineEventType; 16] = &[ +const IGNORED_MESSAGE_TYPES: &[TimelineEventType; 17] = &[ + Audio, + CallInvite, + Emote, + File, + Image, + KeyVerificationStart, + Location, + PollStart, + UnstablePollStart, + Beacon, + Reaction, + RoomEncrypted, RoomMessage, Sticker, - CallInvite, - CallNotify, - RoomEncrypted, - Image, - File, - Audio, - Voice, Video, - UnstablePollStart, - PollStart, - KeyVerificationStart, - Reaction, - Emote, - Location, + Voice, + CallNotify, ]; const LIMIT_MAX: usize = 100; @@ -59,6 +60,7 @@ pub(crate) async fn get_message_events_route( State(services): State, body: Ruma, ) -> Result { + debug_assert!(IGNORED_MESSAGE_TYPES.is_sorted(), "IGNORED_MESSAGE_TYPES is not sorted"); let sender = body.sender(); let (sender_user, sender_device) = sender; let room_id = &body.room_id; @@ -193,7 +195,7 @@ pub(crate) async fn update_lazy( let (_, event) = &item; let (sender_user, sender_device) = sender; - /* TODO: Remove the not "element_hacks" check when these are resolved: + /* TODO: Remove the "element_hacks" check when these are resolved: * https://github.com/vector-im/element-android/issues/3417 * https://github.com/vector-im/element-web/issues/21034 */ @@ -231,19 +233,14 @@ pub(crate) async fn ignored_filter( return None; } - if IGNORED_MESSAGE_TYPES.iter().any(is_equal_to!(&pdu.kind)) - && services.users.user_is_ignored(&pdu.sender, user_id).await - { - return None; - } - - if IGNORED_MESSAGE_TYPES.iter().any(is_equal_to!(&pdu.kind)) - && services - .server - .config - .forbidden_remote_server_names - .iter() - .any(is_equal_to!(pdu.sender().server_name())) + if IGNORED_MESSAGE_TYPES.binary_search(&pdu.kind).is_ok() + && (services.users.user_is_ignored(&pdu.sender, user_id).await + || services + .server + .config + .forbidden_remote_server_names + .iter() + .any(is_equal_to!(pdu.sender().server_name()))) { return None; } diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index cc6cca5e..2f9706f4 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -21,15 +21,15 @@ use crate::Ruma; /// Recommended transferable state events list from the spec const TRANSFERABLE_STATE_EVENTS: &[StateEventType; 9] = &[ - StateEventType::RoomServerAcl, - StateEventType::RoomEncryption, - StateEventType::RoomName, StateEventType::RoomAvatar, - StateEventType::RoomTopic, + StateEventType::RoomEncryption, StateEventType::RoomGuestAccess, StateEventType::RoomHistoryVisibility, StateEventType::RoomJoinRules, + StateEventType::RoomName, StateEventType::RoomPowerLevels, + StateEventType::RoomServerAcl, + StateEventType::RoomTopic, ]; /// # `POST /_matrix/client/r0/rooms/{roomId}/upgrade` @@ -46,6 +46,10 @@ pub(crate) async fn upgrade_room_route( State(services): State, body: Ruma, ) -> Result { + debug_assert!( + TRANSFERABLE_STATE_EVENTS.is_sorted(), + "TRANSFERABLE_STATE_EVENTS is not sorted" + ); let sender_user = body.sender_user.as_ref().expect("user is authenticated"); if !services.server.supported_room_version(&body.new_version) { diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 7e24adff..0c6ea650 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -30,7 +30,6 @@ use ruma::{ TimelineEventType::{self, *}, }, serde::Raw, - state_res::Event, uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, UInt, UserId, }; use service::{rooms::read_receipt::pack_receipts, Services}; @@ -39,8 +38,9 @@ use super::{load_timeline, share_encrypted_room}; use crate::{client::ignored_filter, Ruma}; const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; + const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = - &[RoomMessage, RoomEncrypted, Sticker, CallInvite, PollStart, Beacon]; + &[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker]; /// POST `/_matrix/client/unstable/org.matrix.msc3575/sync` /// @@ -49,6 +49,7 @@ pub(crate) async fn sync_events_v4_route( State(services): State, body: Ruma, ) -> Result { + debug_assert!(DEFAULT_BUMP_TYPES.is_sorted(), "DEFAULT_BUMP_TYPES is not sorted"); let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let sender_device = body.sender_device.expect("user is authenticated"); let mut body = body.body; @@ -595,7 +596,7 @@ pub(crate) async fn sync_events_v4_route( for (_, pdu) in timeline_pdus { let ts = MilliSecondsSinceUnixEpoch(pdu.origin_server_ts); - if DEFAULT_BUMP_TYPES.contains(pdu.event_type()) + if DEFAULT_BUMP_TYPES.binary_search(&pdu.kind).is_ok() && timestamp.is_none_or(|time| time <= ts) { timestamp = Some(ts); diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index d8093dd7..6ddf198d 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -11,7 +11,7 @@ use conduwuit::{ err, error, pdu::PduBuilder, utils::{math::usize_from_f64, ReadyExt}, - Err, Error, Event, PduEvent, Result, + Err, Error, PduEvent, Result, }; use futures::StreamExt; use lru_cache::LruCache; @@ -507,7 +507,7 @@ impl Service { if redacting_event .as_ref() - .is_ok_and(|event| event.event_type() == &TimelineEventType::RoomCreate) + .is_ok_and(|pdu| pdu.kind == TimelineEventType::RoomCreate) { return Err!(Request(Forbidden("Redacting m.room.create is not safe, forbidding."))); } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 2ae66546..028b270f 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1262,7 +1262,7 @@ impl Service { #[implement(Service)] #[tracing::instrument(skip_all, level = "debug")] async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Result<()> { - match pdu.event_type() { + match &pdu.kind { | TimelineEventType::RoomEncryption => { return Err!(Request(Forbidden(error!("Encryption not supported in admins room.")))); }, diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index 1f499692..7fd93b6c 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -40,7 +40,7 @@ struct CheckForUpdatesResponseEntry { const CHECK_FOR_UPDATES_URL: &str = "https://pupbrain.dev/check-for-updates/stable"; const CHECK_FOR_UPDATES_INTERVAL: u64 = 7200; // 2 hours -const LAST_CHECK_FOR_UPDATES_COUNT: &[u8] = b"u"; +const LAST_CHECK_FOR_UPDATES_COUNT: &[u8; 1] = b"u"; #[async_trait] impl crate::Service for Service { From c8349988aac40f24368085a3d7404536c9a26410 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 02:53:32 -0500 Subject: [PATCH 1232/2091] ping online presence on typing updates like synapse Signed-off-by: strawberry --- src/api/client/typing.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/api/client/typing.rs b/src/api/client/typing.rs index 4ae3ac5e..6eabe96a 100644 --- a/src/api/client/typing.rs +++ b/src/api/client/typing.rs @@ -1,7 +1,8 @@ use axum::extract::State; -use ruma::api::client::{error::ErrorKind, typing::create_typing_event}; +use conduwuit::Err; +use ruma::api::client::typing::create_typing_event; -use crate::{utils, Error, Result, Ruma}; +use crate::{utils, Result, Ruma}; /// # `PUT /_matrix/client/r0/rooms/{roomId}/typing/{userId}` /// @@ -11,8 +12,11 @@ pub(crate) async fn create_typing_event_route( body: Ruma, ) -> Result { use create_typing_event::v3::Typing; + let sender_user = body.sender_user(); - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + if sender_user != body.user_id && body.appservice_info.is_none() { + return Err!(Request(Forbidden("You cannot update typing status of other users."))); + } if !services .rooms @@ -20,7 +24,7 @@ pub(crate) async fn create_typing_event_route( .is_joined(sender_user, &body.room_id) .await { - return Err(Error::BadRequest(ErrorKind::forbidden(), "You are not in this room.")); + return Err!(Request(Forbidden("You are not in this room."))); } if let Typing::Yes(duration) = body.state { @@ -58,5 +62,13 @@ pub(crate) async fn create_typing_event_route( .await?; } + // ping presence + if services.globals.allow_local_presence() { + services + .presence + .ping_presence(&body.user_id, &ruma::presence::PresenceState::Online) + .await?; + } + Ok(create_typing_event::v3::Response {}) } From 9c6908b8c166fe451f3ca5bfb86ea2a132cc6c7a Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 14:05:28 -0500 Subject: [PATCH 1233/2091] bump ruwuma, implement MSC4076, add more pusher HTTP URL validation Signed-off-by: strawberry --- Cargo.lock | 26 +++++++------- Cargo.toml | 2 +- src/service/pusher/mod.rs | 73 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 83 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95ab8d04..f7c7079f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3162,7 +3162,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "assign", "js_int", @@ -3184,7 +3184,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "js_int", "ruma-common", @@ -3196,7 +3196,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "as_variant", "assign", @@ -3219,7 +3219,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "as_variant", "base64 0.22.1", @@ -3249,7 +3249,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3273,7 +3273,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "bytes", "http", @@ -3291,7 +3291,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "js_int", "ruma-common", @@ -3310,7 +3310,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3325,7 +3325,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "js_int", "ruma-common", @@ -3337,7 +3337,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "headers", "http", @@ -3350,7 +3350,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3366,7 +3366,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=08f58cd3236fdf175913b2bcaf8865359696d94d#08f58cd3236fdf175913b2bcaf8865359696d94d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 2d99db02..ea9cfa3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -334,7 +334,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "08f58cd3236fdf175913b2bcaf8865359696d94d" +rev = "5a826d31a32b6473671a5b9f813ad2e4b47676b4" features = [ "compat", "rand", diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index cea3ba35..43d60c08 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -92,6 +92,36 @@ impl Service { ))); } + // add some validation to the pusher URL + let pusher_kind = &data.pusher.kind; + if let PusherKind::Http(http) = pusher_kind { + let url = &http.url; + let url = url::Url::parse(&http.url).map_err(|e| { + err!(Request(InvalidParam( + warn!(%url, "HTTP pusher URL is not a valid URL: {e}") + ))) + })?; + + if ["http", "https"] + .iter() + .all(|&scheme| scheme != url.scheme().to_lowercase()) + { + return Err!(Request(InvalidParam( + warn!(%url, "HTTP pusher URL is not a valid HTTP/HTTPS URL") + ))); + } + + if let Ok(ip) = + IPAddress::parse(url.host_str().expect("URL previously validated")) + { + if !self.services.client.valid_cidr_range(&ip) { + return Err!(Request(InvalidParam( + warn!(%url, "HTTP pusher URL is a forbidden remote address") + ))); + } + } + } + let key = (sender, data.pusher.ids.pushkey.as_str()); self.db.senderkey_pusher.put(key, Json(pusher)); }, @@ -330,16 +360,42 @@ impl Service { pusher: &Pusher, tweaks: Vec, event: &PduEvent, - ) -> Result<()> { + ) -> Result { // TODO: email match &pusher.kind { | PusherKind::Http(http) => { + let url = &http.url; + let url = url::Url::parse(&http.url).map_err(|e| { + err!(Request(InvalidParam( + warn!(%url, "HTTP pusher URL is not a valid URL: {e}") + ))) + })?; + + if ["http", "https"] + .iter() + .all(|&scheme| scheme != url.scheme().to_lowercase()) + { + return Err!(Request(InvalidParam( + warn!(%url, "HTTP pusher URL is not a valid HTTP/HTTPS URL") + ))); + } + + if let Ok(ip) = + IPAddress::parse(url.host_str().expect("URL previously validated")) + { + if !self.services.client.valid_cidr_range(&ip) { + return Err!(Request(InvalidParam( + warn!(%url, "HTTP pusher URL is a forbidden remote address") + ))); + } + } + // TODO (timo): can pusher/devices have conflicting formats let event_id_only = http.format == Some(PushFormat::EventIdOnly); let mut device = Device::new(pusher.ids.app_id.clone(), pusher.ids.pushkey.clone()); - device.data.default_payload = http.default_payload.clone(); + device.data.data.clone_from(&http.data); device.data.format.clone_from(&http.format); // Tweaks are only added if the format is NOT event_id_only @@ -352,8 +408,17 @@ impl Service { notifi.event_id = Some((*event.event_id).to_owned()); notifi.room_id = Some((*event.room_id).to_owned()); - // TODO: missed calls - notifi.counts = NotificationCounts::new(unread, uint!(0)); + if http + .data + .get("org.matrix.msc4076.disable_badge_count") + .is_none() && http.data.get("disable_badge_count").is_none() + { + notifi.counts = NotificationCounts::new(unread, uint!(0)); + } else { + // counts will not be serialised if it's the default (0, 0) + // skip_serializing_if = "NotificationCounts::is_default" + notifi.counts = NotificationCounts::default(); + } if event_id_only { self.send_request( From 468071336b1d8e834d7db4c55d1a3c5e0c84b211 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 14:12:00 -0500 Subject: [PATCH 1234/2091] ping online presence on read updates like synapse Signed-off-by: strawberry --- src/api/client/read_marker.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index 2e98afbc..89fe003a 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -49,6 +49,14 @@ pub(crate) async fn set_read_marker_route( .reset_notification_counts(sender_user, &body.room_id); } + // ping presence + if services.globals.allow_local_presence() { + services + .presence + .ping_presence(sender_user, &ruma::presence::PresenceState::Online) + .await?; + } + if let Some(event) = &body.read_receipt { let receipt_content = BTreeMap::from_iter([( event.to_owned(), @@ -117,6 +125,14 @@ pub(crate) async fn create_receipt_route( .reset_notification_counts(sender_user, &body.room_id); } + // ping presence + if services.globals.allow_local_presence() { + services + .presence + .ping_presence(sender_user, &ruma::presence::PresenceState::Online) + .await?; + } + match body.receipt_type { | create_receipt::v3::ReceiptType::FullyRead => { let fully_read_event = ruma::events::fully_read::FullyReadEvent { From 2ba0400758b21eec043f7c75cb8642017c01de20 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 14:23:13 -0500 Subject: [PATCH 1235/2091] return proper error for attempting to update m.room.create Signed-off-by: strawberry --- src/api/client/state.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 6a65f0f9..f56444c7 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -207,6 +207,11 @@ async fn allowed_to_send_state_event( json: &Raw, ) -> Result { match event_type { + | StateEventType::RoomCreate => { + return Err!(Request(BadJson( + "You cannot update m.room.create after a room has been created." + ))); + }, // Forbid m.room.encryption if encryption is disabled | StateEventType::RoomEncryption => if !services.globals.allow_encryption() { From 74c5bfd3118822035ff242274d8bcefd70afa453 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 14:31:11 -0500 Subject: [PATCH 1236/2091] dont allow m.room.server_acl to be redacted Signed-off-by: strawberry --- src/service/rooms/state_accessor/mod.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 6ddf198d..fd132798 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -512,6 +512,16 @@ impl Service { return Err!(Request(Forbidden("Redacting m.room.create is not safe, forbidding."))); } + if redacting_event + .as_ref() + .is_ok_and(|pdu| pdu.kind == TimelineEventType::RoomServerAcl) + { + return Err!(Request(Forbidden( + "Redacting m.room.server_acl will result in the room being inaccessible for \ + everyone (empty allow key), forbidding." + ))); + } + if let Ok(pl_event_content) = self .room_state_get_content::( room_id, From aeae67a7ee975c1987b33a91aca4e9d5ae29c2f2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 15:20:28 -0500 Subject: [PATCH 1237/2091] bump ruwuma to stop sending empty content rulesets Signed-off-by: strawberry --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7c7079f..c86904d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3162,7 +3162,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "assign", "js_int", @@ -3184,7 +3184,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "js_int", "ruma-common", @@ -3196,7 +3196,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "as_variant", "assign", @@ -3219,7 +3219,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "as_variant", "base64 0.22.1", @@ -3249,7 +3249,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3273,7 +3273,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "bytes", "http", @@ -3291,7 +3291,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "js_int", "ruma-common", @@ -3310,7 +3310,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3325,7 +3325,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "js_int", "ruma-common", @@ -3337,7 +3337,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "headers", "http", @@ -3350,7 +3350,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3366,7 +3366,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5a826d31a32b6473671a5b9f813ad2e4b47676b4#5a826d31a32b6473671a5b9f813ad2e4b47676b4" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index ea9cfa3c..cb2ab916 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -334,7 +334,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "5a826d31a32b6473671a5b9f813ad2e4b47676b4" +rev = "a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" features = [ "compat", "rand", From 004671b437020e4463ed4defe26ad2882138aa13 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 15:41:19 -0500 Subject: [PATCH 1238/2091] dont copy join_authorized_via_users_server on leaving rooms Signed-off-by: strawberry --- src/api/client/membership.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 0ddcab32..3eb52138 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -7,7 +7,7 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - debug, debug_info, debug_warn, err, error, info, + debug, debug_info, debug_warn, err, info, pdu::{self, gen_event_id_canonical_json, PduBuilder}, result::FlatOk, trace, @@ -1621,7 +1621,9 @@ pub async fn leave_room( .await else { // Fix for broken rooms - error!("Trying to leave a room you are not a member of."); + warn!( + "Trying to leave a room you are not a member of, marking room as left locally." + ); services .rooms @@ -1647,6 +1649,8 @@ pub async fn leave_room( PduBuilder::state(user_id.to_string(), &RoomMemberEventContent { membership: MembershipState::Leave, reason, + join_authorized_via_users_server: None, + is_direct: None, ..event }), user_id, From c8c9d73b20f698a8c5cc5e2aec9f04b35e0de680 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 17:24:45 -0500 Subject: [PATCH 1239/2091] enable edns0 on hickory resolver config Signed-off-by: strawberry --- src/service/resolver/dns.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index 633b397a..c331dfba 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -57,6 +57,7 @@ impl Resolver { opts.attempts = config.dns_attempts as usize; opts.try_tcp_on_error = config.dns_tcp_fallback; opts.num_concurrent_reqs = 1; + opts.edns0 = true; opts.shuffle_dns_servers = true; opts.rotate = true; opts.ip_strategy = match config.ip_lookup_strategy { From 7ad710d96ae2860b2a7f15cd0574d58ea4cfd3f9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 18:00:49 -0500 Subject: [PATCH 1240/2091] bump complement Signed-off-by: strawberry --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index 25d6de81..28437890 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1733767359, - "narHash": "sha256-3ZBFqvmTSE5Rtbb0Co89aut7q4ERBAeP7HntLN6sFik=", + "lastModified": 1734303596, + "narHash": "sha256-HjDRyLR4MBqQ3IjfMM6eE+8ayztXlbz3gXdyDmFla68=", "owner": "girlbossceo", "repo": "complement", - "rev": "6ec9a3a28fb2baec9afb4fc8b9974a5f5d6525fc", + "rev": "14cc5be797b774f1a2b9f826f38181066d4952b8", "type": "github" }, "original": { From 8dcdb4b09a3b6f5347ebbb1a26b7e591e2154cb9 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 22:46:56 -0500 Subject: [PATCH 1241/2091] update some more docs Signed-off-by: strawberry --- debian/README.md | 2 ++ docs/deploying/docker.md | 3 ++ docs/deploying/generic.md | 43 ++++++++++++++++++++------- docs/deploying/nixos.md | 6 ++++ docs/maintenance.md | 62 +++++++++++++++++++++++++++++++-------- docs/troubleshooting.md | 10 ------- 6 files changed, 93 insertions(+), 33 deletions(-) diff --git a/debian/README.md b/debian/README.md index 89354469..800a2e09 100644 --- a/debian/README.md +++ b/debian/README.md @@ -9,6 +9,8 @@ It is recommended to see the [generic deployment guide](../deploying/generic.md) for further information if needed as usage of the Debian package is generally related. +No `apt` repository is currently offered yet, it is in the works/development. + ### Configuration When installed, the example config is placed at `/etc/conduwuit/conduwuit.toml` diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index d0aa13b3..bdbfb59c 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -24,6 +24,9 @@ OCI images for conduwuit are available in the registries listed below. [shield-latest]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/latest [shield-main]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/main +OCI image `.tar.gz` files are also hosted directly at when uploaded by CI with a +commit hash/revision or a tagged release: + Use ```bash diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 398ba67f..cc50544e 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -1,6 +1,6 @@ # Generic deployment documentation -> ## Getting help +> ### Getting help > > If you run into any problems while setting up conduwuit, ask us in > `#conduwuit:puppygock.gay` or [open an issue on @@ -8,29 +8,50 @@ ## Installing conduwuit -You may simply download the binary that fits your machine. Run `uname -m` to see -what you need. +### Static prebuilt binary + +You may simply download the binary that fits your machine architecture (x86_64 +or aarch64). Run `uname -m` to see what you need. Prebuilt fully static musl binaries can be downloaded from the latest tagged release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or -`main` CI branch workflow artifact output. These also include Debian/Ubuntu packages. +`main` CI branch workflow artifact output. These also include Debian/Ubuntu +packages. + +Binaries are also available on my website directly at: + +These can be curl'd directly from. `ci-bins` are CI workflow binaries by commit +hash/revision, and `releases` are tagged releases. Sort by descending last +modified for the latest. These binaries have jemalloc and io_uring statically linked and included with them, so no additional dynamic dependencies need to be installed. +For the **best** performance; if using an `x86_64` CPU made in the last ~15 years, +we recommend using the `-haswell-` optimised binaries. This sets +`-march=haswell` which is the most compatible and highest performance with +optimised binaries. The database backend, RocksDB, most benefits from this as it +will then use hardware accelerated CRC32 hashing/checksumming which is critical +for performance. + +### Compiling + Alternatively, you may compile the binary yourself. We recommend using -Nix (or [Lix](https://lix.systems)) to build conduwuit as this has the most guaranteed -reproducibiltiy and easiest to get a build environment and output going. This also -allows easy cross-compilation. +Nix (or [Lix](https://lix.systems)) to build conduwuit as this has the most +guaranteed reproducibiltiy and easiest to get a build environment and output +going. This also allows easy cross-compilation. You can run the `nix build -L .#static-x86_64-linux-musl-all-features` or `nix build -L .#static-aarch64-linux-musl-all-features` commands based on architecture to cross-compile the necessary static binary located at -`result/bin/conduwuit`. This is reproducible with the static binaries produced in our CI. +`result/bin/conduwuit`. This is reproducible with the static binaries produced +in our CI. -Otherwise, follow standard Rust project build guides (installing git and cloning -the repo, getting the Rust toolchain via rustup, installing LLVM toolchain + -libclang for RocksDB, installing liburing for io_uring and RocksDB, etc). +If wanting to build using standard Rust toolchains, make sure you install: +- `liburing-dev` on the compiling machine, and `liburing` on the target host +- LLVM and libclang for RocksDB + +You can build conduwuit using `cargo build --release --all-features` ## Migrating from Conduit diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index 0372228d..3c5b0e69 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -29,6 +29,12 @@ conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= ``` +If needed, we have a binary cache on Cachix but it is only limited to 5GB: + +``` +https://conduwuit.cachix.org +conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= +``` If specifying a Git remote URL in your flake, you can use any remotes that are specified on the README (the mirrors), such as the GitHub: `github:girlbossceo/conduwuit` diff --git a/docs/maintenance.md b/docs/maintenance.md index c8df95af..5c8c853a 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -22,23 +22,59 @@ conduwuit has moderation admin commands for: Any commands with `-list` in them will require a codeblock in the message with each object being newline delimited. An example of doing this is: -```` !admin rooms moderation ban-list-of-rooms ``` !roomid1:server.name -!roomid2:server.name !roomid3:server.name ``` ```` +```` +!admin rooms moderation ban-list-of-rooms +``` +!roomid1:server.name +#badroomalias1:server.name +!roomid2:server.name +!roomid3:server.name +#badroomalias2:server.name +``` +```` -## Database +## Database (RocksDB) -If using RocksDB, there's very little you need to do. Compaction is ran -automatically based on various defined thresholds tuned for conduwuit to be high -performance with the least I/O amplifcation or overhead. Manually running -compaction is not recommended, or compaction via a timer. RocksDB is built with -io_uring support via liburing for async read I/O. +Generally there is very little you need to do. [Compaction][rocksdb-compaction] +is ran automatically based on various defined thresholds tuned for conduwuit to +be high performance with the least I/O amplifcation or overhead. Manually +running compaction is not recommended, or compaction via a timer, due to +creating unnecessary I/O amplification. RocksDB is built with io_uring support +via liburing for improved read performance. + +RocksDB troubleshooting can be found [in the RocksDB section of troubleshooting](troubleshooting.md). + +### Compression Some RocksDB settings can be adjusted such as the compression method chosen. See -the RocksDB section in the [example config](configuration/examples.md). btrfs -users may benefit from disabling compression on RocksDB if CoW is in use. +the RocksDB section in the [example config](configuration/examples.md). -RocksDB troubleshooting can be found [in the RocksDB section of -troubleshooting](troubleshooting.md). +btrfs users have reported that database compression does not need to be disabled +on conduwuit as the filesystem already does not attempt to compress. This can be +validated by using `filefrag -v` on a `.SST` file in your database, and ensure +the `physical_offset` matches (no filesystem compression). It is very important +to ensure no additional filesystem compression takes place as this can render +unbuffered Direct IO inoperable, significantly slowing down read and write +performance. See + +> Compression is done using the COW mechanism so it’s incompatible with +> nodatacow. Direct IO read works on compressed files but will fall back to +> buffered writes and leads to no compression even if force compression is set. +> Currently nodatasum and compression don’t work together. + +### Files in database + +Do not touch any of the files in the database directory. This must be said due +to users being mislead by the `.log` files in the RocksDB directory, thinking +they're server logs or database logs, however they are critical RocksDB files +related to WAL tracking. + +The only safe files that can be deleted are the `LOG` files (all caps). These +are the real RocksDB telemetry/log files, however conduwuit has already +configured to only store up to 3 RocksDB `LOG` files due to generall being +useless for average users unless troubleshooting something low-level. If you +would like to store nearly none at all, see the `rocksdb_max_log_files` +config option. ## Backups @@ -95,3 +131,5 @@ Built-in S3 support is also planned, but for now using a "S3 filesystem" on `media/` works. conduwuit also sends a `Cache-Control` header of 1 year and immutable for all media requests (download and thumbnail) to reduce unnecessary media requests from browsers, reduce bandwidth usage, and reduce load. + +[rocksdb-compaction]: https://github.com/facebook/rocksdb/wiki/Compaction diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 283cdeee..d25c9762 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -91,16 +91,6 @@ reliability at a slight performance cost due to TCP overhead. ## RocksDB / database issues -#### Direct IO - -Some filesystems may not like RocksDB using [Direct -IO](https://github.com/facebook/rocksdb/wiki/Direct-IO). Direct IO is for -non-buffered I/O which improves conduwuit performance and reduces system CPU -usage, but at least FUSE and possibly ZFS are filesystems potentially known -to not like this. See the [example config](configuration/examples.md) for -disabling it if needed. Issues from Direct IO on unsupported filesystems are -usually shown as startup errors. - #### Database corruption If your database is corrupted *and* is failing to start (e.g. checksum From 5813419f4b3a2cce7d55182a2181206ec9b0e906 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 15 Dec 2024 22:49:34 -0500 Subject: [PATCH 1242/2091] free up ci runner space on both jobs Signed-off-by: strawberry --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66c8f635..d0d78238 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -266,6 +266,15 @@ jobs: - target: aarch64-linux-musl - target: x86_64-linux-musl steps: + - name: Free up a bit of runner space + run: | + set +o pipefail + sudo docker image prune --all --force || true + sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true + sudo apt clean + sudo rm -v -rf /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/heroku + set -o pipefail + - name: Sync repository uses: actions/checkout@v4 with: From 63c4975483446d14b08c431c6378d2024cd4401a Mon Sep 17 00:00:00 2001 From: AsenHu <78863300+AsenHu@users.noreply.github.com> Date: Tue, 17 Dec 2024 02:08:52 +0000 Subject: [PATCH 1243/2091] fix bug --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b0b1ec8d..511d6a0e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: Upload Release Assets on: release: - types: [published, prereleased] + types: [published] workflow_dispatch: inputs: tag: @@ -51,7 +51,7 @@ jobs: exit 0 fi - tag="${GH_TAG}}" + tag="${GH_TAG}" fi echo "ci_id=$id" >> "$GITHUB_OUTPUT" From 9ec35cf68483b14857684c69076e95d905090f90 Mon Sep 17 00:00:00 2001 From: AsenHu <78863300+AsenHu@users.noreply.github.com> Date: Tue, 17 Dec 2024 03:01:40 +0000 Subject: [PATCH 1244/2091] skip uploading JSON files in release workflow --- .github/workflows/release.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 511d6a0e..cfe72d2a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -77,8 +77,10 @@ jobs: TAG: ${{ steps.get_ci_id.outputs.tag }} run: | for file in $(find . -type f); do - echo "Uploading $file..." - gh release upload $TAG "$file" --clobber --repo="${GH_REPOSITORY}" || echo "Something went wrong, skipping." + case "$file" in + *json*) echo "Skipping $file...";; + *) echo "Uploading $file..."; gh release upload $TAG "$file" --clobber --repo="${GH_REPOSITORY}" || echo "Something went wrong, skipping.";; + esac done - name: upload release assets to website @@ -109,6 +111,8 @@ jobs: ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/releases/$TAG/" for file in $(find . -type f); do - echo "Uploading $file to website" - scp $file website:/var/www/girlboss.ceo/~strawberry/conduwuit/releases/$TAG/$file + case "$file" in + *json*) echo "Skipping $file...";; + *) echo "Uploading $file to website"; scp $file website:/var/www/girlboss.ceo/~strawberry/conduwuit/releases/$TAG/$file;; + esac done From 5b6ff3869c118c4d5cde7b43c48f0b4be0d406ae Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 17 Dec 2024 19:29:46 +0000 Subject: [PATCH 1245/2091] Fix typo in test from rename --- src/core/utils/string.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 6baa9c35..cc692c14 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -84,7 +84,7 @@ where /// Find the common prefix from a collection of strings and return a slice /// ``` -/// use conduwiit_core::utils::string::common_prefix; +/// use conduwuit_core::utils::string::common_prefix; /// let input = ["conduwuit", "conduit", "construct"]; /// common_prefix(&input) == "con"; /// ``` From 3675c941f8fe4b92b5426f24a667e0b03cabd9da Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 18 Dec 2024 03:04:39 +0000 Subject: [PATCH 1246/2091] Send read reciept and typing indicator EDUs to appservices with receive_ephemeral --- Cargo.lock | 26 ++++++------- Cargo.toml | 4 +- src/api/client/read_marker.rs | 14 +++---- src/api/server/send.rs | 2 +- src/service/rooms/read_receipt/mod.rs | 18 +++++++-- src/service/rooms/typing/mod.rs | 54 +++++++++++++++++++++++---- src/service/sending/mod.rs | 48 +++++++++++++++++++++++- src/service/sending/sender.rs | 15 +++----- 8 files changed, 135 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c86904d7..2c0ae75c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3162,7 +3162,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "assign", "js_int", @@ -3184,7 +3184,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "js_int", "ruma-common", @@ -3196,7 +3196,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "as_variant", "assign", @@ -3219,7 +3219,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "as_variant", "base64 0.22.1", @@ -3249,7 +3249,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3273,7 +3273,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "bytes", "http", @@ -3291,7 +3291,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "js_int", "ruma-common", @@ -3310,7 +3310,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3325,7 +3325,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "js_int", "ruma-common", @@ -3337,7 +3337,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "headers", "http", @@ -3350,7 +3350,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3366,7 +3366,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index cb2ab916..38d6d729 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -334,7 +334,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +rev = "112ccc24cb14de26757715d611285d0806d5d91f" features = [ "compat", "rand", @@ -350,7 +350,6 @@ features = [ "compat-upload-signatures", "identifiers-validation", "unstable-unspecified", - "unstable-msc2409", "unstable-msc2448", "unstable-msc2666", "unstable-msc2867", @@ -366,6 +365,7 @@ features = [ "unstable-msc4121", "unstable-msc4125", "unstable-msc4186", + "unstable-msc4203", # sending to-device events to appservices "unstable-msc4210", # remove legacy mentions "unstable-extensible-events", ] diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index 89fe003a..ab7cc6ad 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -72,14 +72,10 @@ pub(crate) async fn set_read_marker_route( services .rooms .read_receipt - .readreceipt_update( - sender_user, - &body.room_id, - &ruma::events::receipt::ReceiptEvent { - content: ruma::events::receipt::ReceiptEventContent(receipt_content), - room_id: body.room_id.clone(), - }, - ) + .readreceipt_update(sender_user, &body.room_id, ruma::events::receipt::ReceiptEvent { + content: ruma::events::receipt::ReceiptEventContent(receipt_content), + room_id: body.room_id.clone(), + }) .await; } @@ -171,7 +167,7 @@ pub(crate) async fn create_receipt_route( .readreceipt_update( sender_user, &body.room_id, - &ruma::events::receipt::ReceiptEvent { + ruma::events::receipt::ReceiptEvent { content: ruma::events::receipt::ReceiptEventContent(receipt_content), room_id: body.room_id.clone(), }, diff --git a/src/api/server/send.rs b/src/api/server/send.rs index c5fc7118..db6fd748 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -275,7 +275,7 @@ async fn handle_edu_receipt( services .rooms .read_receipt - .readreceipt_update(&user_id, &room_id, &event) + .readreceipt_update(&user_id, &room_id, event) .await; } } else { diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 53e64957..4075c447 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -2,9 +2,10 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduwuit::{debug, err, warn, PduCount, PduId, RawPduId, Result}; +use conduwuit::{debug, err, result::LogErr, warn, PduCount, PduId, RawPduId, Result}; use futures::{try_join, Stream, TryFutureExt}; use ruma::{ + api::appservice::event::push_events::v1::EphemeralData, events::{ receipt::{ReceiptEvent, ReceiptEventContent, Receipts}, AnySyncEphemeralRoomEvent, SyncEphemeralRoomEvent, @@ -48,14 +49,25 @@ impl Service { &self, user_id: &UserId, room_id: &RoomId, - event: &ReceiptEvent, + event: ReceiptEvent, ) { - self.db.readreceipt_update(user_id, room_id, event).await; + self.db.readreceipt_update(user_id, room_id, &event).await; self.services .sending .flush_room(room_id) .await .expect("room flush failed"); + // update appservices + let edu = EphemeralData::Receipt(event); + let _ = self + .services + .sending + .send_edu_appservice_room( + room_id, + serde_json::to_vec(&edu).expect("Serialized EphemeralData::Receipt"), + ) + .await + .log_err(); } /// Gets the latest private read receipt from the user in the room diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index a6123322..31ea40ae 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -7,8 +7,11 @@ use conduwuit::{ }; use futures::StreamExt; use ruma::{ - api::federation::transactions::edu::{Edu, TypingContent}, - events::SyncEphemeralRoomEvent, + api::{ + appservice::event::push_events::v1::EphemeralData, + federation::transactions::edu::{Edu, TypingContent}, + }, + events::{typing::TypingEventContent, EphemeralRoomEvent, SyncEphemeralRoomEvent}, OwnedRoomId, OwnedUserId, RoomId, UserId, }; use tokio::sync::{broadcast, RwLock}; @@ -76,6 +79,9 @@ impl Service { trace!("receiver found what it was looking for and is no longer interested"); } + // update appservices + self.appservice_send(room_id).await?; + // update federation if self.services.globals.user_is_local(user_id) { self.federation_send(room_id, user_id, true).await?; @@ -103,7 +109,8 @@ impl Service { if self.typing_update_sender.send(room_id.to_owned()).is_err() { trace!("receiver found what it was looking for and is no longer interested"); } - + // update appservices + self.appservice_send(room_id).await?; // update federation if self.services.globals.user_is_local(user_id) { self.federation_send(room_id, user_id, false).await?; @@ -157,6 +164,9 @@ impl Service { trace!("receiver found what it was looking for and is no longer interested"); } + // update appservices + self.appservice_send(room_id).await?; + // update federation for user in &removable { if self.services.globals.user_is_local(user) { @@ -180,17 +190,30 @@ impl Service { .unwrap_or(0)) } + /// Returns a new typing EDU. + pub async fn typings_content(&self, room_id: &RoomId) -> Result { + let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); + + let Some(typing_indicators) = room_typing_indicators else { + return Ok(TypingEventContent { user_ids: Vec::new() }); + }; + + let user_ids: Vec<_> = typing_indicators.into_keys().collect(); + + Ok(TypingEventContent { user_ids }) + } + /// Returns a new typing EDU. pub async fn typings_all( &self, room_id: &RoomId, sender_user: &UserId, - ) -> Result> { + ) -> Result> { let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); let Some(typing_indicators) = room_typing_indicators else { return Ok(SyncEphemeralRoomEvent { - content: ruma::events::typing::TypingEventContent { user_ids: Vec::new() }, + content: TypingEventContent { user_ids: Vec::new() }, }); }; @@ -208,9 +231,7 @@ impl Service { .collect() .await; - Ok(SyncEphemeralRoomEvent { - content: ruma::events::typing::TypingEventContent { user_ids }, - }) + Ok(SyncEphemeralRoomEvent { content: TypingEventContent { user_ids } }) } async fn federation_send( @@ -237,4 +258,21 @@ impl Service { Ok(()) } + + async fn appservice_send(&self, room_id: &RoomId) -> Result<()> { + let edu = EphemeralData::Typing(EphemeralRoomEvent { + content: self.typings_content(room_id).await?, + room_id: room_id.into(), + }); + + self.services + .sending + .send_edu_appservice_room( + room_id, + serde_json::to_vec(&edu).expect("Serialized EphemeralData::Typing"), + ) + .await?; + + Ok(()) + } } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 2038f4eb..2b571034 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -25,7 +25,10 @@ pub use self::{ sender::{EDU_LIMIT, PDU_LIMIT}, }; use crate::{ - account_data, client, globals, presence, pusher, resolver, rooms, rooms::timeline::RawPduId, + account_data, + appservice::NamespaceRegex, + client, globals, presence, pusher, resolver, + rooms::{self, timeline::RawPduId}, server_keys, users, Dep, }; @@ -38,6 +41,7 @@ pub struct Service { } struct Services { + alias: Dep, client: Dep, globals: Dep, resolver: Dep, @@ -76,6 +80,7 @@ impl crate::Service for Service { Ok(Arc::new(Self { server: args.server.clone(), services: Services { + alias: args.depend::("rooms::alias"), client: args.depend::("client"), globals: args.depend::("globals"), resolver: args.depend::("resolver"), @@ -184,6 +189,47 @@ impl Service { }) } + #[tracing::instrument(skip(self, serialized), level = "debug")] + pub fn send_edu_appservice(&self, appservice_id: String, serialized: Vec) -> Result { + let dest = Destination::Appservice(appservice_id); + let event = SendingEvent::Edu(serialized); + let _cork = self.db.db.cork(); + let keys = self.db.queue_requests(once((&event, &dest))); + self.dispatch(Msg { + dest, + event, + queue_id: keys.into_iter().next().expect("request queue key"), + }) + } + + #[tracing::instrument(skip(self, room_id, serialized), level = "debug")] + pub async fn send_edu_appservice_room( + &self, + room_id: &RoomId, + serialized: Vec, + ) -> Result<()> { + for appservice in self.services.appservice.read().await.values() { + let matching_aliases = |aliases: NamespaceRegex| { + self.services + .alias + .local_aliases_for_room(room_id) + .ready_any(move |room_alias| aliases.is_match(room_alias.as_str())) + }; + + if appservice.rooms.is_match(room_id.as_str()) + || matching_aliases(appservice.aliases.clone()).await + || self + .services + .state_cache + .appservice_in_room(room_id, appservice) + .await + { + self.send_edu_appservice(appservice.registration.id.clone(), serialized.clone())?; + } + } + Ok(()) + } + #[tracing::instrument(skip(self, room_id, serialized), level = "debug")] pub async fn send_edu_room(&self, room_id: &RoomId, serialized: Vec) -> Result<()> { let servers = self diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 1f462f39..1589101b 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -1,3 +1,4 @@ +use core::str; use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, @@ -21,7 +22,7 @@ use futures::{ }; use ruma::{ api::{ - appservice::event::push_events::v1::Edu as RumaEdu, + appservice::event::push_events::v1::EphemeralData, federation::transactions::{ edu::{ DeviceListUpdateContent, Edu, PresenceContent, PresenceUpdate, ReceiptContent, @@ -587,7 +588,7 @@ impl Service { .filter(|event| matches!(event, SendingEvent::Pdu(_))) .count(), ); - let mut edu_jsons: Vec = Vec::with_capacity( + let mut edu_jsons: Vec = Vec::with_capacity( events .iter() .filter(|event| matches!(event, SendingEvent::Edu(_))) @@ -600,16 +601,12 @@ impl Service { pdu_jsons.push(pdu.to_room_event()); } }, - | SendingEvent::Edu(edu) => { - if appservice - .receive_ephemeral - .is_some_and(|receive_edus| receive_edus) - { + | SendingEvent::Edu(edu) => + if appservice.receive_ephemeral { if let Ok(edu) = serde_json::from_slice(edu) { edu_jsons.push(edu); } - } - }, + }, | SendingEvent::Flush => {}, // flush only; no new content } } From 4dd809fdc4c84725d87f23f55bad510b63c27f1b Mon Sep 17 00:00:00 2001 From: strawberry Date: Tue, 17 Dec 2024 23:06:04 -0500 Subject: [PATCH 1247/2091] misc typing cleanup Signed-off-by: strawberry --- src/api/client/sync/v3.rs | 2 +- src/service/rooms/read_receipt/mod.rs | 2 +- src/service/rooms/typing/mod.rs | 72 +++++++++++++-------------- src/service/sending/mod.rs | 6 +-- 4 files changed, 40 insertions(+), 42 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index a05bcf98..fe3877aa 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -722,7 +722,7 @@ async fn load_joined_room( .rooms .typing .typings_all(room_id, sender_user) - .await?; + .await; Ok(vec![serde_json::from_str(&serde_json::to_string(&typings)?)?]) }) diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 4075c447..04c1426e 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -59,7 +59,7 @@ impl Service { .expect("room flush failed"); // update appservices let edu = EphemeralData::Receipt(event); - let _ = self + _ = self .services .sending .send_edu_appservice_room( diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 31ea40ae..1de91e5e 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -146,32 +146,34 @@ impl Service { } }; - if !removable.is_empty() { - let typing = &mut self.typing.write().await; - let room = typing.entry(room_id.to_owned()).or_default(); - for user in &removable { - debug_info!("typing timeout {user:?} in {room_id:?}"); - room.remove(user); - } + if removable.is_empty() { + return Ok(()); + } - // update clients - self.last_typing_update - .write() - .await - .insert(room_id.to_owned(), self.services.globals.next_count()?); + let typing = &mut self.typing.write().await; + let room = typing.entry(room_id.to_owned()).or_default(); + for user in &removable { + debug_info!("typing timeout {user:?} in {room_id:?}"); + room.remove(user); + } - if self.typing_update_sender.send(room_id.to_owned()).is_err() { - trace!("receiver found what it was looking for and is no longer interested"); - } + // update clients + self.last_typing_update + .write() + .await + .insert(room_id.to_owned(), self.services.globals.next_count()?); - // update appservices - self.appservice_send(room_id).await?; + if self.typing_update_sender.send(room_id.to_owned()).is_err() { + trace!("receiver found what it was looking for and is no longer interested"); + } - // update federation - for user in &removable { - if self.services.globals.user_is_local(user) { - self.federation_send(room_id, user, false).await?; - } + // update appservices + self.appservice_send(room_id).await?; + + // update federation + for user in &removable { + if self.services.globals.user_is_local(user) { + self.federation_send(room_id, user, false).await?; } } @@ -190,17 +192,17 @@ impl Service { .unwrap_or(0)) } - /// Returns a new typing EDU. - pub async fn typings_content(&self, room_id: &RoomId) -> Result { + /// Returns a new typing EDU's content. + pub async fn typings_content(&self, room_id: &RoomId) -> TypingEventContent { let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); let Some(typing_indicators) = room_typing_indicators else { - return Ok(TypingEventContent { user_ids: Vec::new() }); + return TypingEventContent { user_ids: Vec::new() }; }; let user_ids: Vec<_> = typing_indicators.into_keys().collect(); - Ok(TypingEventContent { user_ids }) + TypingEventContent { user_ids } } /// Returns a new typing EDU. @@ -208,13 +210,13 @@ impl Service { &self, room_id: &RoomId, sender_user: &UserId, - ) -> Result> { + ) -> SyncEphemeralRoomEvent { let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); let Some(typing_indicators) = room_typing_indicators else { - return Ok(SyncEphemeralRoomEvent { + return SyncEphemeralRoomEvent { content: TypingEventContent { user_ids: Vec::new() }, - }); + }; }; let user_ids: Vec<_> = typing_indicators @@ -231,7 +233,7 @@ impl Service { .collect() .await; - Ok(SyncEphemeralRoomEvent { content: TypingEventContent { user_ids } }) + SyncEphemeralRoomEvent { content: TypingEventContent { user_ids } } } async fn federation_send( @@ -254,14 +256,12 @@ impl Service { self.services .sending .send_edu_room(room_id, serde_json::to_vec(&edu).expect("Serialized Edu::Typing")) - .await?; - - Ok(()) + .await } async fn appservice_send(&self, room_id: &RoomId) -> Result<()> { let edu = EphemeralData::Typing(EphemeralRoomEvent { - content: self.typings_content(room_id).await?, + content: self.typings_content(room_id).await, room_id: room_id.into(), }); @@ -271,8 +271,6 @@ impl Service { room_id, serde_json::to_vec(&edu).expect("Serialized EphemeralData::Typing"), ) - .await?; - - Ok(()) + .await } } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 2b571034..a100367c 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -190,8 +190,8 @@ impl Service { } #[tracing::instrument(skip(self, serialized), level = "debug")] - pub fn send_edu_appservice(&self, appservice_id: String, serialized: Vec) -> Result { - let dest = Destination::Appservice(appservice_id); + pub fn send_edu_appservice(&self, appservice_id: &str, serialized: Vec) -> Result { + let dest = Destination::Appservice(appservice_id.to_owned()); let event = SendingEvent::Edu(serialized); let _cork = self.db.db.cork(); let keys = self.db.queue_requests(once((&event, &dest))); @@ -224,7 +224,7 @@ impl Service { .appservice_in_room(room_id, appservice) .await { - self.send_edu_appservice(appservice.registration.id.clone(), serialized.clone())?; + self.send_edu_appservice(&appservice.registration.id, serialized.clone())?; } } Ok(()) From 9040ad054e172661bb9869b955cae02a51f6abae Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 18 Dec 2024 11:26:04 -0500 Subject: [PATCH 1248/2091] Revert "misc typing cleanup" This reverts commit 4dd809fdc4c84725d87f23f55bad510b63c27f1b. --- src/api/client/sync/v3.rs | 2 +- src/service/rooms/read_receipt/mod.rs | 2 +- src/service/rooms/typing/mod.rs | 72 ++++++++++++++------------- src/service/sending/mod.rs | 6 +-- 4 files changed, 42 insertions(+), 40 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index fe3877aa..a05bcf98 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -722,7 +722,7 @@ async fn load_joined_room( .rooms .typing .typings_all(room_id, sender_user) - .await; + .await?; Ok(vec![serde_json::from_str(&serde_json::to_string(&typings)?)?]) }) diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 04c1426e..4075c447 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -59,7 +59,7 @@ impl Service { .expect("room flush failed"); // update appservices let edu = EphemeralData::Receipt(event); - _ = self + let _ = self .services .sending .send_edu_appservice_room( diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 1de91e5e..31ea40ae 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -146,34 +146,32 @@ impl Service { } }; - if removable.is_empty() { - return Ok(()); - } + if !removable.is_empty() { + let typing = &mut self.typing.write().await; + let room = typing.entry(room_id.to_owned()).or_default(); + for user in &removable { + debug_info!("typing timeout {user:?} in {room_id:?}"); + room.remove(user); + } - let typing = &mut self.typing.write().await; - let room = typing.entry(room_id.to_owned()).or_default(); - for user in &removable { - debug_info!("typing timeout {user:?} in {room_id:?}"); - room.remove(user); - } + // update clients + self.last_typing_update + .write() + .await + .insert(room_id.to_owned(), self.services.globals.next_count()?); - // update clients - self.last_typing_update - .write() - .await - .insert(room_id.to_owned(), self.services.globals.next_count()?); + if self.typing_update_sender.send(room_id.to_owned()).is_err() { + trace!("receiver found what it was looking for and is no longer interested"); + } - if self.typing_update_sender.send(room_id.to_owned()).is_err() { - trace!("receiver found what it was looking for and is no longer interested"); - } + // update appservices + self.appservice_send(room_id).await?; - // update appservices - self.appservice_send(room_id).await?; - - // update federation - for user in &removable { - if self.services.globals.user_is_local(user) { - self.federation_send(room_id, user, false).await?; + // update federation + for user in &removable { + if self.services.globals.user_is_local(user) { + self.federation_send(room_id, user, false).await?; + } } } @@ -192,17 +190,17 @@ impl Service { .unwrap_or(0)) } - /// Returns a new typing EDU's content. - pub async fn typings_content(&self, room_id: &RoomId) -> TypingEventContent { + /// Returns a new typing EDU. + pub async fn typings_content(&self, room_id: &RoomId) -> Result { let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); let Some(typing_indicators) = room_typing_indicators else { - return TypingEventContent { user_ids: Vec::new() }; + return Ok(TypingEventContent { user_ids: Vec::new() }); }; let user_ids: Vec<_> = typing_indicators.into_keys().collect(); - TypingEventContent { user_ids } + Ok(TypingEventContent { user_ids }) } /// Returns a new typing EDU. @@ -210,13 +208,13 @@ impl Service { &self, room_id: &RoomId, sender_user: &UserId, - ) -> SyncEphemeralRoomEvent { + ) -> Result> { let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); let Some(typing_indicators) = room_typing_indicators else { - return SyncEphemeralRoomEvent { + return Ok(SyncEphemeralRoomEvent { content: TypingEventContent { user_ids: Vec::new() }, - }; + }); }; let user_ids: Vec<_> = typing_indicators @@ -233,7 +231,7 @@ impl Service { .collect() .await; - SyncEphemeralRoomEvent { content: TypingEventContent { user_ids } } + Ok(SyncEphemeralRoomEvent { content: TypingEventContent { user_ids } }) } async fn federation_send( @@ -256,12 +254,14 @@ impl Service { self.services .sending .send_edu_room(room_id, serde_json::to_vec(&edu).expect("Serialized Edu::Typing")) - .await + .await?; + + Ok(()) } async fn appservice_send(&self, room_id: &RoomId) -> Result<()> { let edu = EphemeralData::Typing(EphemeralRoomEvent { - content: self.typings_content(room_id).await, + content: self.typings_content(room_id).await?, room_id: room_id.into(), }); @@ -271,6 +271,8 @@ impl Service { room_id, serde_json::to_vec(&edu).expect("Serialized EphemeralData::Typing"), ) - .await + .await?; + + Ok(()) } } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index a100367c..2b571034 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -190,8 +190,8 @@ impl Service { } #[tracing::instrument(skip(self, serialized), level = "debug")] - pub fn send_edu_appservice(&self, appservice_id: &str, serialized: Vec) -> Result { - let dest = Destination::Appservice(appservice_id.to_owned()); + pub fn send_edu_appservice(&self, appservice_id: String, serialized: Vec) -> Result { + let dest = Destination::Appservice(appservice_id); let event = SendingEvent::Edu(serialized); let _cork = self.db.db.cork(); let keys = self.db.queue_requests(once((&event, &dest))); @@ -224,7 +224,7 @@ impl Service { .appservice_in_room(room_id, appservice) .await { - self.send_edu_appservice(&appservice.registration.id, serialized.clone())?; + self.send_edu_appservice(appservice.registration.id.clone(), serialized.clone())?; } } Ok(()) From f54a62dda0e989eda9a28f09121a142f692638c2 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 18 Dec 2024 11:26:18 -0500 Subject: [PATCH 1249/2091] Revert "Send read reciept and typing indicator EDUs to appservices with receive_ephemeral" This reverts commit 3675c941f8fe4b92b5426f24a667e0b03cabd9da. --- Cargo.lock | 26 ++++++------- Cargo.toml | 4 +- src/api/client/read_marker.rs | 14 ++++--- src/api/server/send.rs | 2 +- src/service/rooms/read_receipt/mod.rs | 18 ++------- src/service/rooms/typing/mod.rs | 54 ++++----------------------- src/service/sending/mod.rs | 48 +----------------------- src/service/sending/sender.rs | 15 +++++--- 8 files changed, 46 insertions(+), 135 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c0ae75c..c86904d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3162,7 +3162,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "assign", "js_int", @@ -3184,7 +3184,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "js_int", "ruma-common", @@ -3196,7 +3196,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "as_variant", "assign", @@ -3219,7 +3219,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "as_variant", "base64 0.22.1", @@ -3249,7 +3249,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3273,7 +3273,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "bytes", "http", @@ -3291,7 +3291,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "js_int", "ruma-common", @@ -3310,7 +3310,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3325,7 +3325,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "js_int", "ruma-common", @@ -3337,7 +3337,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "headers", "http", @@ -3350,7 +3350,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3366,7 +3366,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 38d6d729..cb2ab916 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -334,7 +334,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "112ccc24cb14de26757715d611285d0806d5d91f" +rev = "a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" features = [ "compat", "rand", @@ -350,6 +350,7 @@ features = [ "compat-upload-signatures", "identifiers-validation", "unstable-unspecified", + "unstable-msc2409", "unstable-msc2448", "unstable-msc2666", "unstable-msc2867", @@ -365,7 +366,6 @@ features = [ "unstable-msc4121", "unstable-msc4125", "unstable-msc4186", - "unstable-msc4203", # sending to-device events to appservices "unstable-msc4210", # remove legacy mentions "unstable-extensible-events", ] diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index ab7cc6ad..89fe003a 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -72,10 +72,14 @@ pub(crate) async fn set_read_marker_route( services .rooms .read_receipt - .readreceipt_update(sender_user, &body.room_id, ruma::events::receipt::ReceiptEvent { - content: ruma::events::receipt::ReceiptEventContent(receipt_content), - room_id: body.room_id.clone(), - }) + .readreceipt_update( + sender_user, + &body.room_id, + &ruma::events::receipt::ReceiptEvent { + content: ruma::events::receipt::ReceiptEventContent(receipt_content), + room_id: body.room_id.clone(), + }, + ) .await; } @@ -167,7 +171,7 @@ pub(crate) async fn create_receipt_route( .readreceipt_update( sender_user, &body.room_id, - ruma::events::receipt::ReceiptEvent { + &ruma::events::receipt::ReceiptEvent { content: ruma::events::receipt::ReceiptEventContent(receipt_content), room_id: body.room_id.clone(), }, diff --git a/src/api/server/send.rs b/src/api/server/send.rs index db6fd748..c5fc7118 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -275,7 +275,7 @@ async fn handle_edu_receipt( services .rooms .read_receipt - .readreceipt_update(&user_id, &room_id, event) + .readreceipt_update(&user_id, &room_id, &event) .await; } } else { diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 4075c447..53e64957 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -2,10 +2,9 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduwuit::{debug, err, result::LogErr, warn, PduCount, PduId, RawPduId, Result}; +use conduwuit::{debug, err, warn, PduCount, PduId, RawPduId, Result}; use futures::{try_join, Stream, TryFutureExt}; use ruma::{ - api::appservice::event::push_events::v1::EphemeralData, events::{ receipt::{ReceiptEvent, ReceiptEventContent, Receipts}, AnySyncEphemeralRoomEvent, SyncEphemeralRoomEvent, @@ -49,25 +48,14 @@ impl Service { &self, user_id: &UserId, room_id: &RoomId, - event: ReceiptEvent, + event: &ReceiptEvent, ) { - self.db.readreceipt_update(user_id, room_id, &event).await; + self.db.readreceipt_update(user_id, room_id, event).await; self.services .sending .flush_room(room_id) .await .expect("room flush failed"); - // update appservices - let edu = EphemeralData::Receipt(event); - let _ = self - .services - .sending - .send_edu_appservice_room( - room_id, - serde_json::to_vec(&edu).expect("Serialized EphemeralData::Receipt"), - ) - .await - .log_err(); } /// Gets the latest private read receipt from the user in the room diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index 31ea40ae..a6123322 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -7,11 +7,8 @@ use conduwuit::{ }; use futures::StreamExt; use ruma::{ - api::{ - appservice::event::push_events::v1::EphemeralData, - federation::transactions::edu::{Edu, TypingContent}, - }, - events::{typing::TypingEventContent, EphemeralRoomEvent, SyncEphemeralRoomEvent}, + api::federation::transactions::edu::{Edu, TypingContent}, + events::SyncEphemeralRoomEvent, OwnedRoomId, OwnedUserId, RoomId, UserId, }; use tokio::sync::{broadcast, RwLock}; @@ -79,9 +76,6 @@ impl Service { trace!("receiver found what it was looking for and is no longer interested"); } - // update appservices - self.appservice_send(room_id).await?; - // update federation if self.services.globals.user_is_local(user_id) { self.federation_send(room_id, user_id, true).await?; @@ -109,8 +103,7 @@ impl Service { if self.typing_update_sender.send(room_id.to_owned()).is_err() { trace!("receiver found what it was looking for and is no longer interested"); } - // update appservices - self.appservice_send(room_id).await?; + // update federation if self.services.globals.user_is_local(user_id) { self.federation_send(room_id, user_id, false).await?; @@ -164,9 +157,6 @@ impl Service { trace!("receiver found what it was looking for and is no longer interested"); } - // update appservices - self.appservice_send(room_id).await?; - // update federation for user in &removable { if self.services.globals.user_is_local(user) { @@ -190,30 +180,17 @@ impl Service { .unwrap_or(0)) } - /// Returns a new typing EDU. - pub async fn typings_content(&self, room_id: &RoomId) -> Result { - let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); - - let Some(typing_indicators) = room_typing_indicators else { - return Ok(TypingEventContent { user_ids: Vec::new() }); - }; - - let user_ids: Vec<_> = typing_indicators.into_keys().collect(); - - Ok(TypingEventContent { user_ids }) - } - /// Returns a new typing EDU. pub async fn typings_all( &self, room_id: &RoomId, sender_user: &UserId, - ) -> Result> { + ) -> Result> { let room_typing_indicators = self.typing.read().await.get(room_id).cloned(); let Some(typing_indicators) = room_typing_indicators else { return Ok(SyncEphemeralRoomEvent { - content: TypingEventContent { user_ids: Vec::new() }, + content: ruma::events::typing::TypingEventContent { user_ids: Vec::new() }, }); }; @@ -231,7 +208,9 @@ impl Service { .collect() .await; - Ok(SyncEphemeralRoomEvent { content: TypingEventContent { user_ids } }) + Ok(SyncEphemeralRoomEvent { + content: ruma::events::typing::TypingEventContent { user_ids }, + }) } async fn federation_send( @@ -258,21 +237,4 @@ impl Service { Ok(()) } - - async fn appservice_send(&self, room_id: &RoomId) -> Result<()> { - let edu = EphemeralData::Typing(EphemeralRoomEvent { - content: self.typings_content(room_id).await?, - room_id: room_id.into(), - }); - - self.services - .sending - .send_edu_appservice_room( - room_id, - serde_json::to_vec(&edu).expect("Serialized EphemeralData::Typing"), - ) - .await?; - - Ok(()) - } } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 2b571034..2038f4eb 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -25,10 +25,7 @@ pub use self::{ sender::{EDU_LIMIT, PDU_LIMIT}, }; use crate::{ - account_data, - appservice::NamespaceRegex, - client, globals, presence, pusher, resolver, - rooms::{self, timeline::RawPduId}, + account_data, client, globals, presence, pusher, resolver, rooms, rooms::timeline::RawPduId, server_keys, users, Dep, }; @@ -41,7 +38,6 @@ pub struct Service { } struct Services { - alias: Dep, client: Dep, globals: Dep, resolver: Dep, @@ -80,7 +76,6 @@ impl crate::Service for Service { Ok(Arc::new(Self { server: args.server.clone(), services: Services { - alias: args.depend::("rooms::alias"), client: args.depend::("client"), globals: args.depend::("globals"), resolver: args.depend::("resolver"), @@ -189,47 +184,6 @@ impl Service { }) } - #[tracing::instrument(skip(self, serialized), level = "debug")] - pub fn send_edu_appservice(&self, appservice_id: String, serialized: Vec) -> Result { - let dest = Destination::Appservice(appservice_id); - let event = SendingEvent::Edu(serialized); - let _cork = self.db.db.cork(); - let keys = self.db.queue_requests(once((&event, &dest))); - self.dispatch(Msg { - dest, - event, - queue_id: keys.into_iter().next().expect("request queue key"), - }) - } - - #[tracing::instrument(skip(self, room_id, serialized), level = "debug")] - pub async fn send_edu_appservice_room( - &self, - room_id: &RoomId, - serialized: Vec, - ) -> Result<()> { - for appservice in self.services.appservice.read().await.values() { - let matching_aliases = |aliases: NamespaceRegex| { - self.services - .alias - .local_aliases_for_room(room_id) - .ready_any(move |room_alias| aliases.is_match(room_alias.as_str())) - }; - - if appservice.rooms.is_match(room_id.as_str()) - || matching_aliases(appservice.aliases.clone()).await - || self - .services - .state_cache - .appservice_in_room(room_id, appservice) - .await - { - self.send_edu_appservice(appservice.registration.id.clone(), serialized.clone())?; - } - } - Ok(()) - } - #[tracing::instrument(skip(self, room_id, serialized), level = "debug")] pub async fn send_edu_room(&self, room_id: &RoomId, serialized: Vec) -> Result<()> { let servers = self diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 1589101b..1f462f39 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -1,4 +1,3 @@ -use core::str; use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, @@ -22,7 +21,7 @@ use futures::{ }; use ruma::{ api::{ - appservice::event::push_events::v1::EphemeralData, + appservice::event::push_events::v1::Edu as RumaEdu, federation::transactions::{ edu::{ DeviceListUpdateContent, Edu, PresenceContent, PresenceUpdate, ReceiptContent, @@ -588,7 +587,7 @@ impl Service { .filter(|event| matches!(event, SendingEvent::Pdu(_))) .count(), ); - let mut edu_jsons: Vec = Vec::with_capacity( + let mut edu_jsons: Vec = Vec::with_capacity( events .iter() .filter(|event| matches!(event, SendingEvent::Edu(_))) @@ -601,12 +600,16 @@ impl Service { pdu_jsons.push(pdu.to_room_event()); } }, - | SendingEvent::Edu(edu) => - if appservice.receive_ephemeral { + | SendingEvent::Edu(edu) => { + if appservice + .receive_ephemeral + .is_some_and(|receive_edus| receive_edus) + { if let Ok(edu) = serde_json::from_slice(edu) { edu_jsons.push(edu); } - }, + } + }, | SendingEvent::Flush => {}, // flush only; no new content } } From ad8cbcaac1ab77b0b074132c477c7d1943c93919 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 17 Dec 2024 10:43:14 +0000 Subject: [PATCH 1250/2091] check cache prior to offloading iterator seek Signed-off-by: Jason Volk --- src/database/map/keys_from.rs | 5 +++++ src/database/map/rev_keys_from.rs | 5 +++++ src/database/map/rev_stream.rs | 17 +++++++++++++++++ src/database/map/rev_stream_from.rs | 23 +++++++++++++++++++++++ src/database/map/stream.rs | 17 +++++++++++++++++ src/database/map/stream_from.rs | 20 ++++++++++++++++++++ src/database/stream.rs | 14 +++++++++----- 7 files changed, 96 insertions(+), 5 deletions(-) diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 2ffc68df..95c6611b 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -5,6 +5,7 @@ use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; +use super::stream_from::is_cached; use crate::{ keyval::{result_deserialize_key, serialize_key, Key}, stream, @@ -54,6 +55,10 @@ where let opts = super::iter_options_default(); let state = stream::State::new(&self.db, &self.cf, opts); + if is_cached(self, from) { + return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed(); + } + let seek = Seek { map: self.clone(), dir: Direction::Forward, diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index a398f315..e208c505 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -5,6 +5,7 @@ use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; +use super::rev_stream_from::is_cached; use crate::{ keyval::{result_deserialize_key, serialize_key, Key}, stream, @@ -62,6 +63,10 @@ where let opts = super::iter_options_default(); let state = stream::State::new(&self.db, &self.cf, opts); + if is_cached(self, from) { + return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed(); + } + let seek = Seek { map: self.clone(), dir: Direction::Reverse, diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index 81359800..7f58582f 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -26,3 +26,20 @@ pub fn rev_raw_stream(&self) -> impl Stream>> + Send { let opts = super::iter_options_default(); stream::ItemsRev::new(&self.db, &self.cf, opts).init(None) } + +#[tracing::instrument( + name = "cached", + level = "trace", + skip_all, + fields(%map), +)] +pub(super) fn _is_cached

    (map: &super::Map) -> bool +where + P: AsRef<[u8]> + ?Sized, +{ + let opts = super::cache_read_options_default(); + let mut state = stream::State::new(&map.db, &map.cf, opts); + + state.seek_rev(); + !state.is_incomplete() +} diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 6ddb9bc7..d166aa0f 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize}; use crate::{ keyval::{result_deserialize, serialize_key, KeyVal}, stream, + util::is_incomplete, }; /// Iterate key-value entries in the map starting from upper-bound. @@ -83,6 +84,10 @@ where let opts = super::iter_options_default(); let state = stream::State::new(&self.db, &self.cf, opts); + if is_cached(self, from) { + return stream::ItemsRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed(); + }; + let seek = Seek { map: self.clone(), dir: Direction::Reverse, @@ -99,3 +104,21 @@ where .try_flatten() .boxed() } + +#[tracing::instrument( + name = "cached", + level = "trace", + skip(map, from), + fields(%map), +)] +pub(super) fn is_cached

    (map: &Arc, from: &P) -> bool +where + P: AsRef<[u8]> + ?Sized, +{ + let cache_opts = super::cache_read_options_default(); + let cache_status = stream::State::new(&map.db, &map.cf, cache_opts) + .init_rev(from.as_ref().into()) + .status(); + + !matches!(cache_status, Some(e) if is_incomplete(&e)) +} diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index c2d9b6b8..1a90b8fb 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -25,3 +25,20 @@ pub fn raw_stream(&self) -> impl Stream>> + Send { let opts = super::iter_options_default(); stream::Items::new(&self.db, &self.cf, opts).init(None) } + +#[tracing::instrument( + name = "cached", + level = "trace", + skip_all, + fields(%map), +)] +pub(super) fn _is_cached

    (map: &super::Map) -> bool +where + P: AsRef<[u8]> + ?Sized, +{ + let opts = super::cache_read_options_default(); + let mut state = stream::State::new(&map.db, &map.cf, opts); + + state.seek_fwd(); + !state.is_incomplete() +} diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 1dae9d78..107ce4b1 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -81,6 +81,10 @@ where let opts = super::read_options_default(); let state = stream::State::new(&self.db, &self.cf, opts); + if is_cached(self, from) { + return stream::Items::<'_>::from(state.init_fwd(from.as_ref().into())).boxed(); + }; + let seek = Seek { map: self.clone(), dir: Direction::Forward, @@ -97,3 +101,19 @@ where .try_flatten() .boxed() } + +#[tracing::instrument( + name = "cached", + level = "trace", + skip(map, from), + fields(%map), +)] +pub(super) fn is_cached

    (map: &Arc, from: &P) -> bool +where + P: AsRef<[u8]> + ?Sized, +{ + let opts = super::cache_read_options_default(); + let state = stream::State::new(&map.db, &map.cf, opts).init_fwd(from.as_ref().into()); + + !state.is_incomplete() +} diff --git a/src/database/stream.rs b/src/database/stream.rs index 775fb930..f849d08f 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -5,14 +5,14 @@ mod keys_rev; use std::sync::Arc; -use conduwuit::{utils::exchange, Error, Result}; +use conduwuit::{utils::exchange, Result}; use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode, ReadOptions}; pub(crate) use self::{items::Items, items_rev::ItemsRev, keys::Keys, keys_rev::KeysRev}; use crate::{ engine::Db, keyval::{Key, KeyVal, Val}, - util::map_err, + util::{is_incomplete, map_err}, Engine, Slice, }; @@ -34,7 +34,7 @@ pub(crate) trait Cursor<'a, T> { fn get(&self) -> Option> { self.fetch() .map(Ok) - .or_else(|| self.state().status().map(Err)) + .or_else(|| self.state().status().map(map_err).map(Err)) } fn seek_and_get(&mut self) -> Option> { @@ -91,16 +91,20 @@ impl<'a> State<'a> { } } + pub(super) fn is_incomplete(&self) -> bool { + matches!(self.status(), Some(e) if is_incomplete(&e)) + } + fn fetch_key(&self) -> Option> { self.inner.key().map(Key::from) } fn _fetch_val(&self) -> Option> { self.inner.value().map(Val::from) } fn fetch(&self) -> Option> { self.inner.item().map(KeyVal::from) } - fn status(&self) -> Option { self.inner.status().map_err(map_err).err() } + pub(super) fn status(&self) -> Option { self.inner.status().err() } #[inline] - fn valid(&self) -> bool { self.inner.valid() } + pub(super) fn valid(&self) -> bool { self.inner.valid() } } fn keyval_longevity<'a, 'b: 'a>(item: KeyVal<'a>) -> KeyVal<'b> { From f78104a95914acb3f8a8761cd1d5853c6cdfee82 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 17 Dec 2024 14:52:12 +0000 Subject: [PATCH 1251/2091] relax Send on input iterator Signed-off-by: Jason Volk --- src/core/utils/set.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/utils/set.rs b/src/core/utils/set.rs index 8eac7157..032a9835 100644 --- a/src/core/utils/set.rs +++ b/src/core/utils/set.rs @@ -17,7 +17,7 @@ pub fn intersection(mut input: Iters) -> impl Iterator + Clone + Send, Iter: Iterator + Send, - Item: Eq + Send, + Item: Eq, { input.next().into_iter().flat_map(move |first| { let input = input.clone(); @@ -38,7 +38,7 @@ pub fn intersection_sorted( where Iters: Iterator + Clone + Send, Iter: Iterator + Send, - Item: Eq + Ord + Send, + Item: Eq + Ord, { input.next().into_iter().flat_map(move |first| { let mut input = input.clone().collect::>(); From 5f7636f177ddde0c630f24f347a8f638a7304278 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 17 Dec 2024 14:51:21 +0000 Subject: [PATCH 1252/2091] set dbpool thread name (gated by tokio_unstable) Signed-off-by: Jason Volk Signed-off-by: strawberry --- Cargo.toml | 1 + src/database/pool.rs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index cb2ab916..cffbebfa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -242,6 +242,7 @@ features = [ "time", "rt-multi-thread", "io-util", + "tracing", ] [workspace.dependencies.tokio-metrics] diff --git a/src/database/pool.rs b/src/database/pool.rs index e4d78897..65012527 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -122,8 +122,16 @@ async fn spawn_until(self: &Arc, recv: Receiver, max: usize) -> Resul fn spawn_one(self: &Arc, workers: &mut JoinSet<()>, recv: Receiver) -> Result { let id = workers.len(); let self_ = self.clone(); + + #[cfg(not(tokio_unstable))] let _abort = workers.spawn_blocking_on(move || self_.worker(id, recv), self.server.runtime()); + #[cfg(tokio_unstable)] + let _abort = workers + .build_task() + .name("conduwuit:dbpool") + .spawn_blocking_on(move || self_.worker(id, recv), self.server.runtime()); + Ok(()) } From 71673b2a886cd7ec1693f09bff77c3f12f7a7aa2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 17 Dec 2024 14:32:54 +0000 Subject: [PATCH 1253/2091] add worker_affinity feature split runtime init from main.rs Signed-off-by: Jason Volk --- Cargo.lock | 12 ++++ Cargo.toml | 3 + src/main/Cargo.toml | 57 ++++++++-------- src/main/main.rs | 29 ++------ src/main/runtime.rs | 163 ++++++++++++++++++++++++++++++++++++++++++++ src/main/server.rs | 2 +- 6 files changed, 212 insertions(+), 54 deletions(-) create mode 100644 src/main/runtime.rs diff --git a/Cargo.lock b/Cargo.lock index c86904d7..f05da4eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -631,6 +631,7 @@ dependencies = [ "conduwuit_service", "console-subscriber", "const-str", + "core_affinity", "hardened_malloc-rs", "log", "opentelemetry", @@ -933,6 +934,17 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_affinity" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622892f5635ce1fc38c8f16dfc938553ed64af482edb5e150bf4caedbfcb2304" +dependencies = [ + "libc", + "num_cpus", + "winapi", +] + [[package]] name = "cpufeatures" version = "0.2.16" diff --git a/Cargo.toml b/Cargo.toml index cffbebfa..05fc3bc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -494,6 +494,9 @@ version = "1.0.89" [workspace.dependencies.bytesize] version = "1.3.0" +[workspace.dependencies.core_affinity] +version = "0.8.1" + # # Patches # diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 99d41614..fe24d4c1 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -129,6 +129,9 @@ tokio_console = [ "dep:console-subscriber", "tokio/tracing", ] +worker_affinity = [ + "dep:core_affinity", +] zstd_compression = [ "conduwuit-api/zstd_compression", "conduwuit-core/zstd_compression", @@ -144,36 +147,34 @@ conduwuit-database.workspace = true conduwuit-router.workspace = true conduwuit-service.workspace = true -tokio.workspace = true -log.workspace = true -tracing.workspace = true -tracing-subscriber.workspace = true clap.workspace = true -const-str.workspace = true - -opentelemetry.workspace = true -opentelemetry.optional = true -tracing-flame.workspace = true -tracing-flame.optional = true -tracing-opentelemetry.workspace = true -tracing-opentelemetry.optional = true -opentelemetry_sdk.workspace = true -opentelemetry_sdk.optional = true -opentelemetry-jaeger.workspace = true -opentelemetry-jaeger.optional = true - -sentry.workspace = true -sentry.optional = true -sentry-tracing.workspace = true -sentry-tracing.optional = true -sentry-tower.workspace = true -sentry-tower.optional = true - -tokio-metrics.workspace = true -tokio-metrics.optional = true - -console-subscriber.workspace = true console-subscriber.optional = true +console-subscriber.workspace = true +const-str.workspace = true +core_affinity.optional = true +core_affinity.workspace = true +log.workspace = true +opentelemetry-jaeger.optional = true +opentelemetry-jaeger.workspace = true +opentelemetry.optional = true +opentelemetry.workspace = true +opentelemetry_sdk.optional = true +opentelemetry_sdk.workspace = true +sentry-tower.optional = true +sentry-tower.workspace = true +sentry-tracing.optional = true +sentry-tracing.workspace = true +sentry.optional = true +sentry.workspace = true +tokio-metrics.optional = true +tokio-metrics.workspace = true +tokio.workspace = true +tracing-flame.optional = true +tracing-flame.workspace = true +tracing-opentelemetry.optional = true +tracing-opentelemetry.workspace = true +tracing-subscriber.workspace = true +tracing.workspace = true [target.'cfg(all(not(target_env = "msvc"), target_os = "linux"))'.dependencies] hardened_malloc-rs.workspace = true diff --git a/src/main/main.rs b/src/main/main.rs index 0946e835..e7aaf3fc 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -2,45 +2,24 @@ pub(crate) mod clap; mod logging; mod mods; mod restart; +mod runtime; mod sentry; mod server; mod signal; extern crate conduwuit_core as conduwuit; -use std::{ - sync::{atomic::Ordering, Arc}, - time::Duration, -}; +use std::sync::{atomic::Ordering, Arc}; use conduwuit::{debug_info, error, rustc_flags_capture, Error, Result}; use server::Server; -use tokio::runtime; - -const WORKER_NAME: &str = "conduwuit:worker"; -const WORKER_MIN: usize = 2; -const WORKER_KEEPALIVE: u64 = 36; -const GLOBAL_QUEUE_INTERVAL: u32 = 192; -const SYSTEM_QUEUE_INTERVAL: u32 = 256; -const SYSTEM_EVENTS_PER_TICK: usize = 512; rustc_flags_capture! {} fn main() -> Result<(), Error> { let args = clap::parse(); - let runtime = runtime::Builder::new_multi_thread() - .enable_io() - .enable_time() - .thread_name(WORKER_NAME) - .worker_threads(args.worker_threads.max(WORKER_MIN)) - .thread_keep_alive(Duration::from_secs(WORKER_KEEPALIVE)) - .global_queue_interval(GLOBAL_QUEUE_INTERVAL) - .event_interval(SYSTEM_QUEUE_INTERVAL) - .max_io_events_per_tick(SYSTEM_EVENTS_PER_TICK) - .build() - .expect("built runtime"); - - let server: Arc = Server::build(&args, Some(runtime.handle()))?; + let runtime = runtime::new(&args)?; + let server = Server::new(&args, Some(runtime.handle()))?; runtime.spawn(signal::signal(server.clone())); runtime.block_on(async_main(&server))?; diff --git a/src/main/runtime.rs b/src/main/runtime.rs new file mode 100644 index 00000000..ad0c3cde --- /dev/null +++ b/src/main/runtime.rs @@ -0,0 +1,163 @@ +use std::{thread, time::Duration}; + +use conduwuit::Result; +use tokio::runtime::Builder; + +use crate::clap::Args; + +const WORKER_NAME: &str = "conduwuit:worker"; +const WORKER_MIN: usize = 2; +const WORKER_KEEPALIVE: u64 = 36; +const GLOBAL_QUEUE_INTERVAL: u32 = 192; +const KERNEL_QUEUE_INTERVAL: u32 = 256; +const KERNEL_EVENTS_PER_TICK: usize = 512; + +pub(super) fn new(args: &Args) -> Result { + let mut builder = Builder::new_multi_thread(); + + builder + .enable_io() + .enable_time() + .thread_name(WORKER_NAME) + .worker_threads(args.worker_threads.max(WORKER_MIN)) + .thread_keep_alive(Duration::from_secs(WORKER_KEEPALIVE)) + .max_io_events_per_tick(KERNEL_EVENTS_PER_TICK) + .event_interval(KERNEL_QUEUE_INTERVAL) + .global_queue_interval(GLOBAL_QUEUE_INTERVAL) + .on_thread_start(thread_start) + .on_thread_stop(thread_stop) + .on_thread_unpark(thread_unpark) + .on_thread_park(thread_park); + + #[cfg(tokio_unstable)] + builder + .on_task_spawn(task_spawn) + .on_task_terminate(task_terminate); + + #[cfg(tokio_unstable)] + enable_histogram(&mut builder); + + builder.build().map_err(Into::into) +} + +#[cfg(tokio_unstable)] +fn enable_histogram(builder: &mut Builder) { + use tokio::runtime::{HistogramConfiguration, LogHistogram}; + + let config = LogHistogram::builder() + .min_value(Duration::from_micros(10)) + .max_value(Duration::from_millis(1)) + .max_error(0.5) + .max_buckets(32) + .expect("erroneous histogram configuration"); + + builder + .enable_metrics_poll_time_histogram() + .metrics_poll_time_histogram_configuration(HistogramConfiguration::log(config)); +} + +#[tracing::instrument( + name = "fork", + level = "debug", + skip_all, + fields( + id = ?thread::current().id(), + name = %thread::current().name().unwrap_or("None"), + ), +)] +fn thread_start() { + #[cfg(feature = "worker_affinity")] + set_worker_affinity(); +} + +#[cfg(feature = "worker_affinity")] +fn set_worker_affinity() { + use std::sync::{ + atomic::{AtomicUsize, Ordering}, + LazyLock, + }; + + static CORES_OCCUPIED: AtomicUsize = AtomicUsize::new(0); + static CORES_AVAILABLE: LazyLock>> = LazyLock::new(|| { + core_affinity::get_core_ids().map(|mut cores| { + cores.sort_unstable(); + cores + }) + }); + + let Some(cores) = CORES_AVAILABLE.as_ref() else { + return; + }; + + if thread::current().name() != Some(WORKER_NAME) { + return; + } + + let handle = tokio::runtime::Handle::current(); + let num_workers = handle.metrics().num_workers(); + let i = CORES_OCCUPIED.fetch_add(1, Ordering::Relaxed); + if i >= num_workers { + return; + } + + let Some(id) = cores.get(i) else { + return; + }; + + let _set = core_affinity::set_for_current(*id); +} + +#[tracing::instrument( + name = "join", + level = "debug", + skip_all, + fields( + id = ?thread::current().id(), + name = %thread::current().name().unwrap_or("None"), + ), +)] +fn thread_stop() {} + +#[tracing::instrument( + name = "work", + level = "trace", + skip_all, + fields( + id = ?thread::current().id(), + name = %thread::current().name().unwrap_or("None"), + ), +)] +fn thread_unpark() {} + +#[tracing::instrument( + name = "park", + level = "trace", + skip_all, + fields( + id = ?thread::current().id(), + name = %thread::current().name().unwrap_or("None"), + ), +)] +fn thread_park() {} + +#[cfg(tokio_unstable)] +#[tracing::instrument( + name = "spawn", + level = "trace", + skip_all, + fields( + id = %meta.id(), + ), +)] +fn task_spawn(meta: &tokio::runtime::TaskMeta<'_>) {} + +#[cfg(tokio_unstable)] +#[tracing::instrument( + name = "finish", + level = "trace", + skip_all, + fields( + id = %meta.id() + ), +)] +fn task_terminate(meta: &tokio::runtime::TaskMeta<'_>) {} diff --git a/src/main/server.rs b/src/main/server.rs index 00c7a6cc..bc2cff85 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -23,7 +23,7 @@ pub(crate) struct Server { } impl Server { - pub(crate) fn build( + pub(crate) fn new( args: &Args, runtime: Option<&runtime::Handle>, ) -> Result, Error> { From 6c42ac2726fe8fa0fbb09ac9ffd9753965459881 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 17 Dec 2024 22:34:09 +0000 Subject: [PATCH 1254/2091] add num_workers() convenience to core metrics Signed-off-by: Jason Volk --- src/core/metrics/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/core/metrics/mod.rs b/src/core/metrics/mod.rs index d5f937d7..f2022166 100644 --- a/src/core/metrics/mod.rs +++ b/src/core/metrics/mod.rs @@ -66,8 +66,16 @@ impl Metrics { .expect("next interval") } + #[inline] pub fn task_root(&self) -> Option<&TaskMonitor> { self.task_monitor.as_ref() } + #[inline] + pub fn num_workers(&self) -> usize { + self.runtime_metrics() + .map_or(0, runtime::RuntimeMetrics::num_workers) + } + + #[inline] pub fn runtime_metrics(&self) -> Option<&runtime::RuntimeMetrics> { self.runtime_metrics.as_ref() } From 7b8320e0eb369e74149a30e1b55fbe871a075d0b Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 18 Dec 2024 13:41:30 -0500 Subject: [PATCH 1255/2091] bump rocksdb to v9.9.3 Signed-off-by: strawberry --- Cargo.lock | 8 ++++---- deps/rust-rocksdb/Cargo.toml | 2 +- flake.lock | 8 ++++---- flake.nix | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f05da4eb..ad22c8c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3392,8 +3392,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.30.0+9.8.4" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=fcb772d84572c63b1defc57baf9cdbdf5577c6a4#fcb772d84572c63b1defc57baf9cdbdf5577c6a4" +version = "0.31.0+9.9.3" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=123d6302fed23fc706344becb2f19623265a83f8#123d6302fed23fc706344becb2f19623265a83f8" dependencies = [ "bindgen", "bzip2-sys", @@ -3409,8 +3409,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.34.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=fcb772d84572c63b1defc57baf9cdbdf5577c6a4#fcb772d84572c63b1defc57baf9cdbdf5577c6a4" +version = "0.35.0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=123d6302fed23fc706344becb2f19623265a83f8#123d6302fed23fc706344becb2f19623265a83f8" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index e89c26d7..f06c44e8 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "fcb772d84572c63b1defc57baf9cdbdf5577c6a4" +rev = "123d6302fed23fc706344becb2f19623265a83f8" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index 28437890..94cef4dd 100644 --- a/flake.lock +++ b/flake.lock @@ -567,16 +567,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1733704887, - "narHash": "sha256-4ijNmXACyTJWKRcTdlgObdbOVm2oN3Zefg55/4UPIL0=", + "lastModified": 1734469478, + "narHash": "sha256-IcQ4N8xADYal79K+ONmNq4RLlIwdgUqgrVzgNgiIaG8=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "9c656e58c9f969aea28f25b22bc52ea03109677a", + "rev": "8b4808e7de2fbb5d119d8d72cdca76d8ab84bc47", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.8.4", + "ref": "v9.9.3", "repo": "rocksdb", "type": "github" } diff --git a/flake.nix b/flake.nix index 08120ffd..d8ad47a8 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.8.4"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.9.3"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From 4d46df2af5f678bdf1566b9a9509f8b628c58921 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 18 Dec 2024 03:29:42 +0000 Subject: [PATCH 1256/2091] abort tasks for non-async pool shudown Signed-off-by: Jason Volk --- src/database/engine.rs | 3 ++- src/database/pool.rs | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 63a6087d..73ea559d 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -347,7 +347,8 @@ impl Drop for Engine { fn drop(&mut self) { const BLOCKING: bool = true; - debug_assert!(!self.pool.close(), "request pool was not closed"); + debug!("Closing frontend pool"); + self.pool.close(); debug!("Waiting for background tasks to finish..."); self.db.cancel_all_background_work(BLOCKING); diff --git a/src/database/pool.rs b/src/database/pool.rs index 65012527..28eb38bd 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -2,7 +2,7 @@ use std::{ mem::take, sync::{ atomic::{AtomicUsize, Ordering}, - Arc, + Arc, Mutex, }, }; @@ -11,7 +11,7 @@ use conduwuit::{debug, debug_warn, defer, err, implement, result::DebugInspect, use futures::{channel::oneshot, TryFutureExt}; use oneshot::Sender as ResultSender; use rocksdb::Direction; -use tokio::{sync::Mutex, task::JoinSet}; +use tokio::task::JoinSet; use crate::{keyval::KeyBuf, stream, Handle, Map}; @@ -79,7 +79,7 @@ pub(crate) async fn new(server: &Arc, opts: &Opts) -> Result> pub(crate) async fn shutdown(self: &Arc) { self.close(); - let workers = take(&mut *self.workers.lock().await); + let workers = take(&mut *self.workers.lock().expect("locked")); debug!(workers = workers.len(), "Waiting for workers to join..."); workers.join_all().await; @@ -92,7 +92,13 @@ pub(crate) fn close(&self) -> bool { return false; } + let mut workers = take(&mut *self.workers.lock().expect("locked")); + debug!(workers = workers.len(), "Waiting for workers to join..."); + workers.abort_all(); + drop(workers); + std::thread::yield_now(); + debug_assert!(self.queue.is_empty(), "channel is not empty"); debug!( senders = self.queue.sender_count(), receivers = self.queue.receiver_count(), @@ -104,7 +110,7 @@ pub(crate) fn close(&self) -> bool { #[implement(Pool)] async fn spawn_until(self: &Arc, recv: Receiver, max: usize) -> Result { - let mut workers = self.workers.lock().await; + let mut workers = self.workers.lock().expect("locked"); while workers.len() < max { self.spawn_one(&mut workers, recv.clone())?; } From 60a952508e0591c380e293d71c442a02584ae08d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 18 Dec 2024 03:32:58 +0000 Subject: [PATCH 1257/2091] use macro for error constructions Signed-off-by: Jason Volk --- src/service/rooms/alias/mod.rs | 35 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 0790c376..9dcf9d8e 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -5,12 +5,11 @@ use std::sync::Arc; use conduwuit::{ err, utils::{stream::TryIgnore, ReadyExt}, - Err, Error, Result, + Err, Result, }; use database::{Deserialized, Ignore, Interfix, Map}; -use futures::{Stream, StreamExt}; +use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{ - api::client::error::ErrorKind, events::{ room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, StateEventType, @@ -72,10 +71,7 @@ impl Service { if alias == self.services.globals.admin_alias && user_id != self.services.globals.server_user { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Only the server user can set this alias", - )); + return Err!(Request(Forbidden("Only the server user can set this alias"))); } // Comes first as we don't want a stuck alias @@ -220,7 +216,7 @@ impl Service { } // Checking whether the user is able to change canonical aliases of the room - if let Ok(content) = self + if let Ok(power_levels) = self .services .state_accessor .room_state_get_content::( @@ -228,10 +224,12 @@ impl Service { &StateEventType::RoomPowerLevels, "", ) + .map_ok(RoomPowerLevels::from) .await { - return Ok(RoomPowerLevels::from(content) - .user_can_send_state(user_id, StateEventType::RoomCanonicalAlias)); + return Ok( + power_levels.user_can_send_state(user_id, StateEventType::RoomCanonicalAlias) + ); } // If there is no power levels event, only the room creator can change @@ -291,18 +289,12 @@ impl Service { .globals .server_is_ours(room_alias.server_name()) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Alias is from another server.", - )); + return Err!(Request(InvalidParam("Alias is from another server."))); } - if let Some(ref info) = appservice_info { + if let Some(info) = appservice_info { if !info.aliases.is_match(room_alias.as_str()) { - return Err(Error::BadRequest( - ErrorKind::Exclusive, - "Room alias is not in namespace.", - )); + return Err!(Request(Exclusive("Room alias is not in namespace."))); } } else if self .services @@ -310,10 +302,7 @@ impl Service { .is_exclusive_alias(room_alias) .await { - return Err(Error::BadRequest( - ErrorKind::Exclusive, - "Room alias reserved by appservice.", - )); + return Err!(Request(Exclusive("Room alias reserved by appservice."))); } Ok(()) From 802395bdb7f83b04a54c0d4b7a356ae2c4919254 Mon Sep 17 00:00:00 2001 From: "admin@dimensionproject.net" Date: Wed, 18 Dec 2024 20:46:01 -0500 Subject: [PATCH 1258/2091] add password argument to reset-password admin cmd Signed-off-by: strawberry --- src/admin/user/commands.rs | 8 ++++++-- src/admin/user/mod.rs | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 803fbcf2..1cbbf856 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -242,7 +242,11 @@ pub(super) async fn deactivate( } #[admin_command] -pub(super) async fn reset_password(&self, username: String) -> Result { +pub(super) async fn reset_password( + &self, + username: String, + password: Option, +) -> Result { let user_id = parse_local_user_id(self.services, &username)?; if user_id == self.services.globals.server_user { @@ -252,7 +256,7 @@ pub(super) async fn reset_password(&self, username: String) -> Result, }, /// - Deactivate a user From 8f73caae0b673ac3a69a0ef5f47b4c8f60ea9481 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 19 Dec 2024 18:56:58 +0000 Subject: [PATCH 1259/2091] fix github actions runs on forks This commit contains two separate fixes: 1) use lowercased github username/repository for the docker repository name. This is because image registries only accept lowercase in image repository names, but github stores the repository and username in a case-sensitive manner. This broke image uploads for me, as my username has uppercase chars. 2) change run conditions for some steps. It will no longer attempt to set up SSH web publishing if the SSH private key is not set. It will also run the image registry upload steps if registry usernames are missing, instead skipping individual uploads where the token for that registry is missing. Finally, it simplifies the sccache run conditions to use the `SCCACHE_GHA_ENABLED` env variable, rather than duplicating that logic. Signed-off-by: Jade Ellis --- .github/workflows/ci.yml | 179 +++++++++++++++++++++++++-------------- 1 file changed, 114 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0d78238..6c2c2822 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,7 +64,9 @@ jobs: runs-on: ubuntu-24.04 steps: - name: Setup SSH web publish - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | mkdir -p -v ~/.ssh @@ -184,7 +186,7 @@ jobs: # use sccache for Rust - name: Run sccache-cache - if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') + if: (env.SCCACHE_GHA_ENABLED == 'true') uses: mozilla-actions/sccache-action@main with: version: "v0.8.2" @@ -281,7 +283,9 @@ jobs: persist-credentials: false - name: Setup SSH web publish - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | mkdir -p -v ~/.ssh @@ -360,7 +364,7 @@ jobs: # use sccache for Rust - name: Run sccache-cache - if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') + if: (env.SCCACHE_GHA_ENABLED == 'true') uses: mozilla-actions/sccache-action@main with: version: "v0.8.2" @@ -492,7 +496,7 @@ jobs: fi - name: Upload static-${{ matrix.target }}-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }} @@ -506,7 +510,7 @@ jobs: fi - name: Upload static deb ${{ matrix.target }}-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}.deb @@ -528,14 +532,14 @@ jobs: compression-level: 0 - name: Upload static-${{ matrix.target }}-debug-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }}-debug fi - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}-debug.deb @@ -584,14 +588,14 @@ jobs: fi - name: Upload OCI image ${{ matrix.target }}-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}.tar.gz fi - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}-debug.tar.gz @@ -610,7 +614,9 @@ jobs: persist-credentials: false - name: Setup SSH web publish - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | mkdir -p -v ~/.ssh @@ -647,7 +653,7 @@ jobs: # use sccache for Rust - name: Run sccache-cache - if: (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') + if: (env.SCCACHE_GHA_ENABLED == 'true') uses: mozilla-actions/sccache-action@main # use rust-cache @@ -711,24 +717,35 @@ jobs: name: conduwuit-macos-arm64 path: conduwuit-macos-arm64 if-no-files-found: error - + variables: + outputs: + github_repository: ${{ steps.var.outputs.github_repository }} + runs-on: "ubuntu-latest" + steps: + - name: Setting global variables + uses: actions/github-script@v7 + id: var + with: + script: | + console.log('${{ github.repository }}'.toLowerCase()) + core.setOutput('github_repository', '${{ github.repository }}'.toLowerCase()) docker: name: Docker publish runs-on: ubuntu-24.04 - needs: build + needs: [build, variables] permissions: packages: write contents: read - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && github.event.pull_request.user.login != 'renovate[bot]' + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' env: - DOCKER_ARM64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 - DOCKER_AMD64: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 - DOCKER_TAG: docker.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - DOCKER_BRANCH: docker.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} - GHCR_ARM64: ghcr.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 - GHCR_AMD64: ghcr.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 - GHCR_TAG: ghcr.io/${{ github.repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - GHCR_BRANCH: ghcr.io/${{ github.repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + DOCKER_ARM64: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 + DOCKER_AMD64: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 + DOCKER_TAG: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + DOCKER_BRANCH: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + GHCR_ARM64: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 + GHCR_AMD64: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 + GHCR_TAG: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + GHCR_BRANCH: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} GLCR_ARM64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 GLCR_AMD64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 GLCR_TAG: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} @@ -737,6 +754,12 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} steps: + - name: log variables + uses: actions/github-script@v7 + id: var + with: + script: | + console.log(`${{ toJson(needs) }}`) - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: @@ -771,95 +794,121 @@ jobs: mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz - name: Load and push amd64 image - if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | docker load -i oci-image-amd64.tar.gz - docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64} + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64} + docker push ${DOCKER_AMD64} + fi docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64} - docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64} - docker push ${DOCKER_AMD64} docker push ${GHCR_AMD64} - docker push ${GLCR_AMD64} + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64} + docker push ${GLCR_AMD64} + fi - name: Load and push arm64 image - if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | docker load -i oci-image-arm64v8.tar.gz - docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64} + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64} + docker push ${DOCKER_ARM64} + fi docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64} - docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64} - docker push ${DOCKER_ARM64} docker push ${GHCR_ARM64} - docker push ${GLCR_ARM64} + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64} + docker push ${GLCR_ARM64} + fi - name: Load and push amd64 debug image - if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | docker load -i oci-image-amd64-debug.tar.gz - docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64}-debug + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64}-debug + docker push ${DOCKER_AMD64}-debug + fi docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64}-debug - docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64}-debug - docker push ${DOCKER_AMD64}-debug docker push ${GHCR_AMD64}-debug - docker push ${GLCR_AMD64}-debug + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64}-debug + docker push ${GLCR_AMD64}-debug + fi - name: Load and push arm64 debug image - if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | docker load -i oci-image-arm64v8-debug.tar.gz - docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64}-debug + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64}-debug + docker push ${DOCKER_ARM64}-debug + fi docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64}-debug - docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64}-debug - docker push ${DOCKER_ARM64}-debug docker push ${GHCR_ARM64}-debug - docker push ${GLCR_ARM64}-debug + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64}-debug + docker push ${GLCR_ARM64}-debug + fi - name: Create Docker combined manifests run: | # Dockerhub Container Registry - docker manifest create ${DOCKER_TAG} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64} - docker manifest create ${DOCKER_BRANCH} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64} + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_TAG} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64} + docker manifest create ${DOCKER_BRANCH} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64} + fi # GitHub Container Registry docker manifest create ${GHCR_TAG} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} docker manifest create ${GHCR_BRANCH} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} # GitLab Container Registry - docker manifest create ${GLCR_TAG} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64} - docker manifest create ${GLCR_BRANCH} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64} + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_TAG} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64} + docker manifest create ${GLCR_BRANCH} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64} + fi - name: Create Docker combined debug manifests run: | # Dockerhub Container Registry - docker manifest create ${DOCKER_TAG}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug - docker manifest create ${DOCKER_BRANCH}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_TAG}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug + docker manifest create ${DOCKER_BRANCH}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug + fi # GitHub Container Registry docker manifest create ${GHCR_TAG}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug docker manifest create ${GHCR_BRANCH}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug # GitLab Container Registry - docker manifest create ${GLCR_TAG}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug - docker manifest create ${GLCR_BRANCH}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_TAG}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug + docker manifest create ${GLCR_BRANCH}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug + fi - name: Push manifests to Docker registries - if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | - docker manifest push ${DOCKER_TAG} - docker manifest push ${DOCKER_BRANCH} + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest push ${DOCKER_TAG} + docker manifest push ${DOCKER_BRANCH} + docker manifest push ${DOCKER_TAG}-debug + docker manifest push ${DOCKER_BRANCH}-debug + fi docker manifest push ${GHCR_TAG} docker manifest push ${GHCR_BRANCH} - docker manifest push ${GLCR_TAG} - docker manifest push ${GLCR_BRANCH} - docker manifest push ${DOCKER_TAG}-debug - docker manifest push ${DOCKER_BRANCH}-debug docker manifest push ${GHCR_TAG}-debug docker manifest push ${GHCR_BRANCH}-debug - docker manifest push ${GLCR_TAG}-debug - docker manifest push ${GLCR_BRANCH}-debug + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest push ${GLCR_TAG} + docker manifest push ${GLCR_BRANCH} + docker manifest push ${GLCR_TAG}-debug + docker manifest push ${GLCR_BRANCH}-debug + fi - name: Add Image Links to Job Summary - if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} run: | - echo "- \`docker pull ${DOCKER_TAG}\`" >> $GITHUB_STEP_SUMMARY + if [ ! -z $DOCKERHUB_TOKEN ]; then + echo "- \`docker pull ${DOCKER_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + fi echo "- \`docker pull ${GHCR_TAG}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${GLCR_TAG}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${DOCKER_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY echo "- \`docker pull ${GHCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${GLCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + if [ ! -z $GITLAB_TOKEN ]; then + echo "- \`docker pull ${GLCR_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + fi \ No newline at end of file From 0602e38ada1ef310671f4806b920732b76f0aa0a Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 19 Dec 2024 19:07:50 +0000 Subject: [PATCH 1260/2091] remove logging --- .github/workflows/ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c2c2822..ec06323a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -727,7 +727,6 @@ jobs: id: var with: script: | - console.log('${{ github.repository }}'.toLowerCase()) core.setOutput('github_repository', '${{ github.repository }}'.toLowerCase()) docker: name: Docker publish @@ -754,12 +753,6 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} steps: - - name: log variables - uses: actions/github-script@v7 - id: var - with: - script: | - console.log(`${{ toJson(needs) }}`) - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: From 025950139e733907a7b5ccd80a056378d6fb0c65 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 19 Dec 2024 22:11:26 +0000 Subject: [PATCH 1261/2091] do not try to push to GHCR on pull requests from forks --- .github/workflows/ci.yml | 53 ++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec06323a..5b20a43e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -752,6 +752,7 @@ jobs: DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GHCR_ENABLED: "${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}" steps: - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -793,8 +794,10 @@ jobs: docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64} docker push ${DOCKER_AMD64} fi - docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64} - docker push ${GHCR_AMD64} + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64} + docker push ${GHCR_AMD64} + fi if [ ! -z $GITLAB_TOKEN ]; then docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64} docker push ${GLCR_AMD64} @@ -807,8 +810,10 @@ jobs: docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64} docker push ${DOCKER_ARM64} fi - docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64} - docker push ${GHCR_ARM64} + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64} + docker push ${GHCR_ARM64} + fi if [ ! -z $GITLAB_TOKEN ]; then docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64} docker push ${GLCR_ARM64} @@ -821,8 +826,10 @@ jobs: docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64}-debug docker push ${DOCKER_AMD64}-debug fi - docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64}-debug - docker push ${GHCR_AMD64}-debug + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64}-debug + docker push ${GHCR_AMD64}-debug + fi if [ ! -z $GITLAB_TOKEN ]; then docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64}-debug docker push ${GLCR_AMD64}-debug @@ -835,8 +842,10 @@ jobs: docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64}-debug docker push ${DOCKER_ARM64}-debug fi - docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64}-debug - docker push ${GHCR_ARM64}-debug + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64}-debug + docker push ${GHCR_ARM64}-debug + fi if [ ! -z $GITLAB_TOKEN ]; then docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64}-debug docker push ${GLCR_ARM64}-debug @@ -850,8 +859,10 @@ jobs: docker manifest create ${DOCKER_BRANCH} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64} fi # GitHub Container Registry - docker manifest create ${GHCR_TAG} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} - docker manifest create ${GHCR_BRANCH} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_TAG} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} + docker manifest create ${GHCR_BRANCH} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} + fi # GitLab Container Registry if [ ! -z $GITLAB_TOKEN ]; then docker manifest create ${GLCR_TAG} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64} @@ -866,8 +877,10 @@ jobs: docker manifest create ${DOCKER_BRANCH}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug fi # GitHub Container Registry - docker manifest create ${GHCR_TAG}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug - docker manifest create ${GHCR_BRANCH}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_TAG}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug + docker manifest create ${GHCR_BRANCH}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug + fi # GitLab Container Registry if [ ! -z $GITLAB_TOKEN ]; then docker manifest create ${GLCR_TAG}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug @@ -882,10 +895,12 @@ jobs: docker manifest push ${DOCKER_TAG}-debug docker manifest push ${DOCKER_BRANCH}-debug fi - docker manifest push ${GHCR_TAG} - docker manifest push ${GHCR_BRANCH} - docker manifest push ${GHCR_TAG}-debug - docker manifest push ${GHCR_BRANCH}-debug + if [ $GHCR_ENABLED = "true" ]; then + docker manifest push ${GHCR_TAG} + docker manifest push ${GHCR_BRANCH} + docker manifest push ${GHCR_TAG}-debug + docker manifest push ${GHCR_BRANCH}-debug + fi if [ ! -z $GITLAB_TOKEN ]; then docker manifest push ${GLCR_TAG} docker manifest push ${GLCR_BRANCH} @@ -899,8 +914,10 @@ jobs: echo "- \`docker pull ${DOCKER_TAG}\`" >> $GITHUB_STEP_SUMMARY echo "- \`docker pull ${DOCKER_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY fi - echo "- \`docker pull ${GHCR_TAG}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${GHCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + if [ $GHCR_ENABLED = "true" ]; then + echo "- \`docker pull ${GHCR_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + fi if [ ! -z $GITLAB_TOKEN ]; then echo "- \`docker pull ${GLCR_TAG}\`" >> $GITHUB_STEP_SUMMARY echo "- \`docker pull ${GLCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY From 5b5735f653e1169ebf5eeaa7add51070fd7cd1cc Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 19 Dec 2024 20:19:05 +0000 Subject: [PATCH 1262/2091] update ruwuma --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 4 ++-- src/service/sending/sender.rs | 14 +++++--------- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad22c8c2..d65ae18f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3174,7 +3174,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "assign", "js_int", @@ -3196,7 +3196,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "js_int", "ruma-common", @@ -3208,7 +3208,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "as_variant", "assign", @@ -3231,7 +3231,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "as_variant", "base64 0.22.1", @@ -3261,7 +3261,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3285,7 +3285,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "bytes", "http", @@ -3303,7 +3303,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "js_int", "ruma-common", @@ -3322,7 +3322,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3337,7 +3337,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "js_int", "ruma-common", @@ -3349,7 +3349,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "headers", "http", @@ -3362,7 +3362,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3378,7 +3378,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2#a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 05fc3bc5..15f054bf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -335,7 +335,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "a204cb56dbc20f72a1cbd0e9d6c827bbfd4082f2" +rev = "112ccc24cb14de26757715d611285d0806d5d91f" features = [ "compat", "rand", @@ -351,7 +351,6 @@ features = [ "compat-upload-signatures", "identifiers-validation", "unstable-unspecified", - "unstable-msc2409", "unstable-msc2448", "unstable-msc2666", "unstable-msc2867", @@ -367,6 +366,7 @@ features = [ "unstable-msc4121", "unstable-msc4125", "unstable-msc4186", + "unstable-msc4203", # sending to-device events to appservices "unstable-msc4210", # remove legacy mentions "unstable-extensible-events", ] diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 1f462f39..f6b83e83 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -21,7 +21,7 @@ use futures::{ }; use ruma::{ api::{ - appservice::event::push_events::v1::Edu as RumaEdu, + appservice::event::push_events::v1::EphemeralData, federation::transactions::{ edu::{ DeviceListUpdateContent, Edu, PresenceContent, PresenceUpdate, ReceiptContent, @@ -587,7 +587,7 @@ impl Service { .filter(|event| matches!(event, SendingEvent::Pdu(_))) .count(), ); - let mut edu_jsons: Vec = Vec::with_capacity( + let mut edu_jsons: Vec = Vec::with_capacity( events .iter() .filter(|event| matches!(event, SendingEvent::Edu(_))) @@ -600,16 +600,12 @@ impl Service { pdu_jsons.push(pdu.to_room_event()); } }, - | SendingEvent::Edu(edu) => { - if appservice - .receive_ephemeral - .is_some_and(|receive_edus| receive_edus) - { + | SendingEvent::Edu(edu) => + if appservice.receive_ephemeral { if let Ok(edu) = serde_json::from_slice(edu) { edu_jsons.push(edu); } - } - }, + }, | SendingEvent::Flush => {}, // flush only; no new content } } From 7f645ff0e9111cc6e05e3abc1abad7d0b1f6a5a9 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Thu, 19 Dec 2024 23:06:01 +0000 Subject: [PATCH 1263/2091] Make some doc changes to `admin media delete-past-remote-media` Adjust before/after doc to clarify internationally Courtesy of @Aranjedeath Run `cargo fmt`, properly Fix stupid doc command issue Signed-off-by: strawberry --- src/admin/media/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 046be556..d212aab4 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -26,23 +26,23 @@ pub(super) enum MediaCommand { /// filesystem. This will always ignore errors. DeleteList, - /// - Deletes all remote media in the last/after "X" time using filesystem - /// metadata first created at date, or fallback to last modified date. - /// This will always ignore errors by default. - /// - /// Synapse + /// - Deletes all remote (and optionally local) media created before or + /// after \[duration] time using filesystem metadata first created at + /// date, or fallback to last modified date. This will always ignore + /// errors by default. DeletePastRemoteMedia { - /// - The duration (at or after/before), e.g. "5m" to delete all media - /// in the past or up to 5 minutes + /// - The relative time (e.g. 30s, 5m, 7d) within which to search duration: String, + /// - Only delete media created more recently than \[duration] ago #[arg(long, short)] before: bool, + /// - Only delete media created after \[duration] ago #[arg(long, short)] after: bool, - /// Long argument to delete local media + /// - Long argument to additionally delete local media #[arg(long)] yes_i_want_to_delete_local_media: bool, }, From c53e9e07f23756d92ecd124da10a9a970756977c Mon Sep 17 00:00:00 2001 From: Neek <33990922+neektwothousand@users.noreply.github.com> Date: Fri, 27 Dec 2024 19:41:16 +0100 Subject: [PATCH 1264/2091] update cohost link added an archive.org link to the cohost post as it will fully shut down at the end of the year --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e6ad1981..796f96f8 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ heavily appreciated! 💜🥺 Original repo and Matrix room picture was from bran (<3). Current banner image and logo is directly from [this cohost -post](https://cohost.org/RatBaby/post/1028290-finally-a-flag-for). +post](https://web.archive.org/web/20241126004041/https://cohost.org/RatBaby/post/1028290-finally-a-flag-for). #### Is it conduwuit or Conduwuit? From 5bce0a3a4630c234fc7c64eba10a49127bd77d65 Mon Sep 17 00:00:00 2001 From: Neil Svedberg Date: Tue, 31 Dec 2024 17:19:28 -0500 Subject: [PATCH 1265/2091] Improve documentation for configuration Although the configuration file was mostly wrapped to a line-width of 80, some lines were wrapped slightly shorter. I fixed this. In general, all sentences were changed to start with a capital letter and end with a period or other punctuation mark. Many of the documentation commets read as, "config option to do XYZ". I shortened these to simply "do XYZ". --- clippy.toml | 2 +- conduwuit-example.toml | 487 +++++++++++++++++++++------------------- src/core/config/mod.rs | 499 ++++++++++++++++++++++------------------- 3 files changed, 530 insertions(+), 458 deletions(-) diff --git a/clippy.toml b/clippy.toml index d9dd99ca..42427101 100644 --- a/clippy.toml +++ b/clippy.toml @@ -15,5 +15,5 @@ disallowed-macros = [ ] disallowed-methods = [ - { path = "tokio::spawn", reason = "use and pass conduuwit_core::server::Server::runtime() to spawn from"}, + { path = "tokio::spawn", reason = "use and pass conduuwit_core::server::Server::runtime() to spawn from" }, ] diff --git a/conduwuit-example.toml b/conduwuit-example.toml index e179ce30..a82d8f69 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1,25 +1,28 @@ ### conduwuit Configuration ### -### THIS FILE IS GENERATED. CHANGES/CONTRIBUTIONS IN THE REPO WILL -### BE OVERWRITTEN! +### THIS FILE IS GENERATED. CHANGES/CONTRIBUTIONS IN THE REPO WILL BE +### OVERWRITTEN! ### -### You should rename this file before configuring your server. Changes -### to documentation and defaults can be contributed in source code at +### You should rename this file before configuring your server. Changes to +### documentation and defaults can be contributed in source code at ### src/core/config/mod.rs. This file is generated when building. ### ### Any values pre-populated are the default values for said config option. ### ### At the minimum, you MUST edit all the config options to your environment ### that say "YOU NEED TO EDIT THIS". -### See https://conduwuit.puppyirl.gay/configuration.html for ways to -### configure conduwuit +### +### For more information, see: +### https://conduwuit.puppyirl.gay/configuration.html [global] # The server_name is the pretty name of this server. It is used as a # suffix for user and room IDs/aliases. # -# See the docs for reverse proxying and delegation: https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy +# See the docs for reverse proxying and delegation: +# https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy +# # Also see the `[global.well_known]` config section at the very bottom. # # Examples of delegation: @@ -33,7 +36,7 @@ # #server_name = -# default address (IPv4 or IPv6) conduwuit will listen on. +# The default address (IPv4 or IPv6) conduwuit will listen on. # # If you are using Docker or a container NAT networking setup, this must # be "0.0.0.0". @@ -43,22 +46,26 @@ # #address = ["127.0.0.1", "::1"] -# The port(s) conduwuit will be running on. +# The port(s) conduwuit will listen on. # -# See https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy for reverse proxying. +# For reverse proxying, see: +# https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy # -# Docker users: Don't change this, you'll need to map an external port to -# this. +# If you are using Docker, don't change this, you'll need to map an +# external port to this. # # To listen on multiple ports, specify a vector e.g. [8080, 8448] # #port = 8008 -# Uncomment unix_socket_path to listen on a UNIX socket at the specified -# path. If listening on a UNIX socket, you MUST remove/comment the -# 'address' key if definedm AND add your reverse proxy to the 'conduwuit' -# group, unless world RW permissions are specified with unix_socket_perms -# (666 minimum). +# The UNIX socket conduwuit will listen on. +# +# conduwuit cannot listen on both an IP address and a UNIX socket. If +# listening on a UNIX socket, you MUST remove/comment the `address` key. +# +# Remember to make sure that your reverse proxy has access to this socket +# file, either by adding your reverse proxy to the 'conduwuit' group or +# granting world R/W permissions with `unix_socket_perms` (666 minimum). # # example: "/run/conduwuit/conduwuit.sock" # @@ -69,8 +76,7 @@ #unix_socket_perms = 660 # This is the only directory where conduwuit will save its data, including -# media. -# Note: this was previously "/var/lib/matrix-conduit" +# media. Note: this was previously "/var/lib/matrix-conduit". # # YOU NEED TO EDIT THIS. # @@ -82,7 +88,8 @@ # API. To use this, set a database backup path that conduwuit can write # to. # -# See https://conduwuit.puppyirl.gay/maintenance.html#backups for more information. +# For more information, see: +# https://conduwuit.puppyirl.gay/maintenance.html#backups # # example: "/opt/conduwuit-db-backups" # @@ -103,14 +110,15 @@ # Similar to the individual LRU caches, this is scaled up with your CPU # core count. # -# This defaults to 128.0 + (64.0 * CPU core count) +# This defaults to 128.0 + (64.0 * CPU core count). # #db_cache_capacity_mb = varies by system -# Option to control adding arbitrary text to the end of the user's -# displayname upon registration with a space before the text. This was the -# lightning bolt emoji option, just replaced with support for adding your -# own custom text or emojis. To disable, set this to "" (an empty string). +# Text which will be added to the end of the user's displayname upon +# registration with a space before the text. In Conduit, this was the +# lightning bolt emoji. +# +# To disable, set this to "" (an empty string). # # The default is the trans pride flag. # @@ -191,8 +199,8 @@ # Maximum entries stored in DNS memory-cache. The size of an entry may # vary so please take care if raising this value excessively. Only -# decrease this when using an external DNS cache. Please note -# that systemd-resolved does *not* count as an external cache, even when +# decrease this when using an external DNS cache. Please note that +# systemd-resolved does *not* count as an external cache, even when # configured to do so. # #dns_cache_entries = 32768 @@ -207,8 +215,8 @@ # Minimum time-to-live in seconds for NXDOMAIN entries in the DNS cache. # This value is critical for the server to federate efficiently. -# NXDOMAIN's are assumed to not be returning to the federation -# and aggressively cached rather than constantly rechecked. +# NXDOMAIN's are assumed to not be returning to the federation and +# aggressively cached rather than constantly rechecked. # # Defaults to 3 days as these are *very rarely* false negatives. # @@ -237,10 +245,12 @@ # #query_all_nameservers = true -# Enables using *only* TCP for querying your specified nameservers instead +# Enable using *only* TCP for querying your specified nameservers instead # of UDP. # -# If you are running conduwuit in a container environment, this config option may need to be enabled. See https://conduwuit.puppyirl.gay/troubleshooting.html#potential-dns-issues-when-using-docker for more details. +# If you are running conduwuit in a container environment, this config +# option may need to be enabled. For more details, see: +# https://conduwuit.puppyirl.gay/troubleshooting.html#potential-dns-issues-when-using-docker # #query_over_tcp_only = false @@ -288,8 +298,8 @@ # Default/base request total timeout (seconds). The time limit for a whole # request. This is set very high to not cancel healthy requests while -# serving as a backstop. This is used only by URL previews and -# update/news endpoint checks. +# serving as a backstop. This is used only by URL previews and update/news +# endpoint checks. # #request_total_timeout = 320 @@ -304,11 +314,11 @@ # #request_idle_per_host = 1 -# Federation well-known resolution connection timeout (seconds) +# Federation well-known resolution connection timeout (seconds). # #well_known_conn_timeout = 6 -# Federation HTTP well-known resolution request timeout (seconds) +# Federation HTTP well-known resolution request timeout (seconds). # #well_known_timeout = 10 @@ -318,12 +328,12 @@ # #federation_timeout = 300 -# Federation client idle connection pool timeout (seconds) +# Federation client idle connection pool timeout (seconds). # #federation_idle_timeout = 25 # Federation client max idle connections per host. Defaults to 1 as -# generally the same open connection can be re-used +# generally the same open connection can be re-used. # #federation_idle_per_host = 1 @@ -332,11 +342,11 @@ # #sender_timeout = 180 -# Federation sender idle connection pool timeout (seconds) +# Federation sender idle connection pool timeout (seconds). # #sender_idle_timeout = 180 -# Federation sender transaction retry backoff limit (seconds) +# Federation sender transaction retry backoff limit (seconds). # #sender_retry_backoff_limit = 86400 @@ -345,11 +355,11 @@ # #appservice_timeout = 35 -# Appservice URL idle connection pool timeout (seconds) +# Appservice URL idle connection pool timeout (seconds). # #appservice_idle_timeout = 300 -# Notification gateway pusher idle connection pool timeout +# Notification gateway pusher idle connection pool timeout. # #pusher_idle_timeout = 15 @@ -357,9 +367,8 @@ # server. # # If set to true without a token configured, users can register with no -# form of 2nd-step only if you set -# `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` to -# true in your config. +# form of 2nd-step only if you set the following option to true: +# `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` # # If you would like registration only via token reg, please configure # `registration_token` or `registration_token_file`. @@ -413,8 +422,8 @@ # Set this to true to allow your server's public room directory to be # federated. Set this to false to protect against /publicRooms spiders, # but will forbid external users from viewing your server's public room -# directory. If federation is disabled entirely (`allow_federation`), -# this is inherently false. +# directory. If federation is disabled entirely (`allow_federation`), this +# is inherently false. # #allow_public_room_directory_over_federation = false @@ -424,10 +433,10 @@ # #allow_public_room_directory_without_auth = false -# allow guests/unauthenticated users to access TURN credentials +# Allow guests/unauthenticated users to access TURN credentials. # -# this is the equivalent of Synapse's `turn_allow_guests` config option. -# this allows any unauthenticated user to call the endpoint +# This is the equivalent of Synapse's `turn_allow_guests` config option. +# This allows any unauthenticated user to call the endpoint # `/_matrix/client/v3/voip/turnServer`. # # It is unlikely you need to enable this as all major clients support @@ -462,24 +471,24 @@ # #allow_inbound_profile_lookup_federation_requests = true -# controls whether standard users are allowed to create rooms. appservices -# and admins are always allowed to create rooms +# Allow standard users to create rooms. Appservices and admins are always +# allowed to create rooms # #allow_room_creation = true # Set to false to disable users from joining or creating room versions -# that aren't 100% officially supported by conduwuit. +# that aren't officially supported by conduwuit. # # conduwuit officially supports room versions 6 - 11. # # conduwuit has slightly experimental (though works fine in practice) -# support for versions 3 - 5 +# support for versions 3 - 5. # #allow_unstable_room_versions = true -# default room version conduwuit will create rooms with. +# Default room version conduwuit will create rooms with. # -# per spec, room version 10 is the default. +# Per spec, room version 10 is the default. # #default_room_version = 10 @@ -510,22 +519,28 @@ #tracing_flame_output_path = "./tracing.folded" # Examples: +# # - No proxy (default): -# proxy ="none" +# +# proxy = "none" # # - For global proxy, create the section at the bottom of this file: -# [global.proxy] -# global = { url = "socks5h://localhost:9050" } +# +# [global.proxy] +# global = { url = "socks5h://localhost:9050" } # # - To proxy some domains: -# [global.proxy] -# [[global.proxy.by_domain]] -# url = "socks5h://localhost:9050" -# include = ["*.onion", "matrix.myspecial.onion"] -# exclude = ["*.myspecial.onion"] +# +# [global.proxy] +# [[global.proxy.by_domain]] +# url = "socks5h://localhost:9050" +# include = ["*.onion", "matrix.myspecial.onion"] +# exclude = ["*.myspecial.onion"] # # Include vs. Exclude: +# # - If include is an empty list, it is assumed to be `["*"]`. +# # - If a domain matches both the exclude and include list, the proxy will # only be used if it was included because of a more specific rule than # it was excluded. In the above example, the proxy would be used for @@ -542,7 +557,7 @@ # (notary trusted key servers). # # Currently, conduwuit doesn't support inbound batched key requests, so -# this list should only contain other Synapse servers +# this list should only contain other Synapse servers. # # example: ["matrix.org", "envs.net", "constellatory.net", "tchncs.de"] # @@ -563,10 +578,10 @@ # compromised trusted server to room joins only. The join operation # requires gathering keys from many origin servers which can cause # significant delays. Therefor this defaults to true to mitigate -# unexpected delays out-of-the-box. The security-paranoid or those -# willing to tolerate delays are advised to set this to false. Note that -# setting query_trusted_key_servers_first to true causes this option to -# be ignored. +# unexpected delays out-of-the-box. The security-paranoid or those willing +# to tolerate delays are advised to set this to false. Note that setting +# query_trusted_key_servers_first to true causes this option to be +# ignored. # #query_trusted_key_servers_first_on_join = true @@ -582,8 +597,10 @@ # #trusted_server_batch_size = 1024 -# max log level for conduwuit. allows debug, info, warn, or error -# see also: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives +# Max log level for conduwuit. Allows debug, info, warn, or error. +# +# See also: +# https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives # # **Caveat**: # For release builds, the tracing crate is configured to only implement @@ -593,48 +610,48 @@ # #log = "info" -# controls whether logs will be outputted with ANSI colours +# Output logs with ANSI colours. # #log_colors = true -# configures the span events which will be outputted with the log +# Configures the span events which will be outputted with the log. # #log_span_events = "none" -# configures whether CONDUWUIT_LOG EnvFilter matches values using regular +# Configures whether CONDUWUIT_LOG EnvFilter matches values using regular # expressions. See the tracing_subscriber documentation on Directives. # #log_filter_regex = true -# toggles the display of ThreadId in tracing log output +# Toggles the display of ThreadId in tracing log output. # #log_thread_ids = false -# OpenID token expiration/TTL in seconds +# OpenID token expiration/TTL in seconds. # # These are the OpenID tokens that are primarily used for Matrix account # integrations (e.g. Vector Integrations in Element), *not* OIDC/OpenID -# Connect/etc +# Connect/etc. # #openid_token_ttl = 3600 -# static TURN username to provide the client if not using a shared secret +# Static TURN username to provide the client if not using a shared secret # ("turn_secret"), It is recommended to use a shared secret over static # credentials. # #turn_username = false -# static TURN password to provide the client if not using a shared secret +# Static TURN password to provide the client if not using a shared secret # ("turn_secret"). It is recommended to use a shared secret over static # credentials. # #turn_password = false -# vector list of TURN URIs/servers to use +# Vector list of TURN URIs/servers to use. # -# replace "example.turn.uri" with your TURN domain, such as the coturn -# "realm" config option. if using TURN over TLS, replace the URI prefix -# "turn:" with "turns:" +# Replace "example.turn.uri" with your TURN domain, such as the coturn +# "realm" config option. If using TURN over TLS, replace the URI prefix +# "turn:" with "turns:". # # example: ["turn:example.turn.uri?transport=udp", # "turn:example.turn.uri?transport=tcp"] @@ -642,29 +659,29 @@ #turn_uris = [] # TURN secret to use for generating the HMAC-SHA1 hash apart of username -# and password generation +# and password generation. # -# this is more secure, but if needed you can use traditional -# static username/password credentials. +# This is more secure, but if needed you can use traditional static +# username/password credentials. # #turn_secret = false -# TURN secret to use that's read from the file path specified +# TURN secret to use that's read from the file path specified. # -# this takes priority over "turn_secret" first, and falls back to +# This takes priority over "turn_secret" first, and falls back to # "turn_secret" if invalid or failed to open. # # example: "/etc/conduwuit/.turn_secret" # #turn_secret_file = -# TURN TTL in seconds +# TURN TTL, in seconds. # #turn_ttl = 86400 # List/vector of room IDs or room aliases that conduwuit will make newly -# registered users join. The rooms specified must be rooms that you -# have joined at least once on the server, and must be public. +# registered users join. The rooms specified must be rooms that you have +# joined at least once on the server, and must be public. # # example: ["#conduwuit:puppygock.gay", # "!eoIzvAvVwY23LPDay8:puppygock.gay"] @@ -686,7 +703,7 @@ # room invites) are ignored here. # # Defaults to false as rooms can be banned for non-moderation-related -# reasons and this performs a full user deactivation +# reasons and this performs a full user deactivation. # #auto_deactivate_banned_room_attempts = false @@ -711,7 +728,7 @@ #rocksdb_log_time_to_roll = 0 # Set this to true to use RocksDB config options that are tailored to HDDs -# (slower device storage) +# (slower device storage). # # It is worth noting that by default, conduwuit will use RocksDB with # Direct IO enabled. *Generally* speaking this improves performance as it @@ -724,13 +741,15 @@ # feel free to report in the conduwuit Matrix room if this option fixes # your DB issues. # -# See https://github.com/facebook/rocksdb/wiki/Direct-IO for more information. +# For more information, see: +# https://github.com/facebook/rocksdb/wiki/Direct-IO # #rocksdb_optimize_for_spinning_disks = false # Enables direct-io to increase database performance via unbuffered I/O. # -# See https://github.com/facebook/rocksdb/wiki/Direct-IO for more details about Direct IO and RocksDB. +# For more details about direct I/O and RockDB, see: +# https://github.com/facebook/rocksdb/wiki/Direct-IO # # Set this option to false if the database resides on a filesystem which # does not support direct-io like FUSE, or any form of complex filesystem @@ -739,25 +758,28 @@ #rocksdb_direct_io = true # Amount of threads that RocksDB will use for parallelism on database -# operatons such as cleanup, sync, flush, compaction, etc. Set to 0 to use -# all your logical threads. Defaults to your CPU logical thread count. +# operations such as cleanup, sync, flush, compaction, etc. Set to 0 to +# use all your logical threads. Defaults to your CPU logical thread count. # #rocksdb_parallelism_threads = varies by system # Maximum number of LOG files RocksDB will keep. This must *not* be set to # 0. It must be at least 1. Defaults to 3 as these are not very useful -# unless troubleshooting/debugging a RocksDB bug. +# unless troubleshooting/debugging a RocksDB bug. # #rocksdb_max_log_files = 3 # Type of RocksDB database compression to use. # -# Available options are "zstd", "zlib", "bz2", "lz4", or "none" +# Available options are "zstd", "zlib", "bz2", "lz4", or "none". # # It is best to use ZSTD as an overall good balance between -# speed/performance, storage, IO amplification, and CPU usage. -# For more performance but less compression (more storage used) and less -# CPU usage, use LZ4. See https://github.com/facebook/rocksdb/wiki/Compression for more details. +# speed/performance, storage, IO amplification, and CPU usage. For more +# performance but less compression (more storage used) and less CPU usage, +# use LZ4. +# +# For more details, see: +# https://github.com/facebook/rocksdb/wiki/Compression # # "none" will disable compression. # @@ -766,22 +788,21 @@ # Level of compression the specified compression algorithm for RocksDB to # use. # -# Default is 32767, which is internally read by RocksDB as the -# default magic number and translated to the library's default -# compression level as they all differ. -# See their `kDefaultCompressionLevel`. +# Default is 32767, which is internally read by RocksDB as the default +# magic number and translated to the library's default compression level +# as they all differ. See their `kDefaultCompressionLevel`. # #rocksdb_compression_level = 32767 # Level of compression the specified compression algorithm for the # bottommost level/data for RocksDB to use. Default is 32767, which is -# internally read by RocksDB as the default magic number and translated -# to the library's default compression level as they all differ. -# See their `kDefaultCompressionLevel`. +# internally read by RocksDB as the default magic number and translated to +# the library's default compression level as they all differ. See their +# `kDefaultCompressionLevel`. # # Since this is the bottommost level (generally old and least used data), # it may be desirable to have a very high compression level here as it's -# lesss likely for this data to be used. Research your chosen compression +# less likely for this data to be used. Research your chosen compression # algorithm. # #rocksdb_bottommost_compression_level = 32767 @@ -797,7 +818,7 @@ # #rocksdb_bottommost_compression = false -# Database recovery mode (for RocksDB WAL corruption) +# Database recovery mode (for RocksDB WAL corruption). # # Use this option when the server reports corruption and refuses to start. # Set mode 2 (PointInTime) to cleanly recover from this corruption. The @@ -805,8 +826,8 @@ # minutes prior to the crash. Clients may have to run "clear-cache & # reload" to account for the rollback. Upon success, you may reset the # mode back to default and restart again. Please note in some cases the -# corruption error may not be cleared for at least 30 minutes of -# operation in PointInTime mode. +# corruption error may not be cleared for at least 30 minutes of operation +# in PointInTime mode. # # As a very last ditch effort, if PointInTime does not fix or resolve # anything, you can try mode 3 (SkipAnyCorruptedRecord) but this will @@ -824,9 +845,11 @@ # 2 = PointInTime (use me if trying to recover) # 3 = SkipAnyCorruptedRecord (you now voided your Conduwuit warranty) # -# See https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes for more information on these modes. +# For more information on these modes, see: +# https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes # -# See https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption for more details on recovering a corrupt database. +# For more details on recovering a corrupt database, see: +# https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption # #rocksdb_recovery_mode = 1 @@ -834,11 +857,12 @@ # database consistency at a potential performance impact due to further # safety checks ran. # -# See https://github.com/facebook/rocksdb/wiki/Online-Verification#columnfamilyoptionsparanoid_file_checks for more information. +# For more information, see: +# https://github.com/facebook/rocksdb/wiki/Online-Verification#columnfamilyoptionsparanoid_file_checks # #rocksdb_paranoid_file_checks = false -# Database repair mode (for RocksDB SST corruption) +# Database repair mode (for RocksDB SST corruption). # # Use this option when the server reports corruption while running or # panics. If the server refuses to start use the recovery mode options @@ -847,6 +871,7 @@ # # - Backing up your database directory is recommended prior to running the # repair. +# # - Disabling repair mode and restarting the server is recommended after # running the repair. # @@ -874,10 +899,10 @@ # #rocksdb_compaction_ioprio_idle = true -# Config option to disable RocksDB compaction. You should never ever have -# to disable this. If you for some reason find yourself needing to disable -# this as part of troubleshooting or a bug, please reach out to us in the -# conduwuit Matrix room with information and details. +# Disables RocksDB compaction. You should never ever have to set this +# option to true. If you for some reason find yourself needing to use this +# option as part of troubleshooting or a bug, please reach out to us in +# the conduwuit Matrix room with information and details. # # Disabling compaction will lead to a significantly bloated and # explosively large database, gradually poor performance, unnecessarily @@ -916,79 +941,76 @@ # #notification_push_path = "/_matrix/push/v1/notify" -# Config option to control local (your server only) presence -# updates/requests. Note that presence on conduwuit is -# very fast unlike Synapse's. If using outgoing presence, this MUST be -# enabled. +# Allow local (your server only) presence updates/requests. +# +# Note that presence on conduwuit is very fast unlike Synapse's. If using +# outgoing presence, this MUST be enabled. # #allow_local_presence = true -# Config option to control incoming federated presence updates/requests. +# Allow incoming federated presence updates/requests. # -# This option receives presence updates from other -# servers, but does not send any unless `allow_outgoing_presence` is true. -# Note that presence on conduwuit is very fast unlike Synapse's. +# This option receives presence updates from other servers, but does not +# send any unless `allow_outgoing_presence` is true. Note that presence on +# conduwuit is very fast unlike Synapse's. # #allow_incoming_presence = true -# Config option to control outgoing presence updates/requests. +# Allow outgoing presence updates/requests. # # This option sends presence updates to other servers, but does not -# receive any unless `allow_incoming_presence` is true. -# Note that presence on conduwuit is very fast unlike Synapse's. -# If using outgoing presence, you MUST enable `allow_local_presence` as -# well. +# receive any unless `allow_incoming_presence` is true. Note that presence +# on conduwuit is very fast unlike Synapse's. If using outgoing presence, +# you MUST enable `allow_local_presence` as well. # #allow_outgoing_presence = true -# Config option to control how many seconds before presence updates that -# you are idle. Defaults to 5 minutes. +# How many seconds without presence updates before you become idle. +# Defaults to 5 minutes. # #presence_idle_timeout_s = 300 -# Config option to control how many seconds before presence updates that -# you are offline. Defaults to 30 minutes. +# How many seconds without presence updates before you become offline. +# Defaults to 30 minutes. # #presence_offline_timeout_s = 1800 -# Config option to enable the presence idle timer for remote users. +# Enable the presence idle timer for remote users. +# # Disabling is offered as an optimization for servers participating in # many large rooms or when resources are limited. Disabling it may cause -# incorrect presence states (i.e. stuck online) to be seen for some -# remote users. +# incorrect presence states (i.e. stuck online) to be seen for some remote +# users. # #presence_timeout_remote_users = true -# Config option to control whether we should receive remote incoming read -# receipts. +# Allow receiving incoming read receipts from remote servers. # #allow_incoming_read_receipts = true -# Config option to control whether we should send read receipts to remote -# servers. +# Allow sending read receipts to remote servers. # #allow_outgoing_read_receipts = true -# Config option to control outgoing typing updates to federation. +# Allow outgoing typing updates to federation. # #allow_outgoing_typing = true -# Config option to control incoming typing updates from federation. +# Allow incoming typing updates from federation. # #allow_incoming_typing = true -# Config option to control maximum time federation user can indicate -# typing. +# Maximum time federation user can indicate typing. # #typing_federation_timeout_s = 30 -# Config option to control minimum time local client can indicate typing. -# This does not override a client's request to stop typing. It only -# enforces a minimum value in case of no stop request. +# Minimum time local client can indicate typing. This does not override a +# client's request to stop typing. It only enforces a minimum value in +# case of no stop request. # #typing_client_timeout_min_s = 15 -# Config option to control maximum time local client can indicate typing. +# Maximum time local client can indicate typing. # #typing_client_timeout_max_s = 45 @@ -1017,8 +1039,8 @@ # brotli. This option does nothing if conduwuit was not built with # `brotli_compression` feature. Please be aware that enabling HTTP # compression may weaken TLS. Most users should not need to enable this. -# See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before -# deciding to enable this. +# See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH +# before deciding to enable this. # #brotli_compression = false @@ -1037,8 +1059,8 @@ # #allow_guests_auto_join_rooms = false -# Config option to control whether the legacy unauthenticated Matrix media -# repository endpoints will be enabled. These endpoints consist of: +# Enable the legacy unauthenticated Matrix media repository endpoints. +# These endpoints consist of: # - /_matrix/media/*/config # - /_matrix/media/*/upload # - /_matrix/media/*/preview_url @@ -1056,8 +1078,8 @@ # #freeze_legacy_media = true -# Checks consistency of the media directory at startup: -# 1. When `media_compat_file_link` is enbled, this check will upgrade +# Check consistency of the media directory at startup: +# 1. When `media_compat_file_link` is enabled, this check will upgrade # media when switching back and forth between Conduit and conduwuit. # Both options must be enabled to handle this. # 2. When media is deleted from the directory, this check will also delete @@ -1070,18 +1092,22 @@ #media_startup_check = true # Enable backward-compatibility with Conduit's media directory by creating -# symlinks of media. This option is only necessary if you plan on using -# Conduit again. Otherwise setting this to false reduces filesystem -# clutter and overhead for managing these symlinks in the directory. This -# is now disabled by default. You may still return to upstream Conduit -# but you have to run conduwuit at least once with this set to true and -# allow the media_startup_check to take place before shutting -# down to return to Conduit. +# symlinks of media. +# +# This option is only necessary if you plan on using Conduit again. +# Otherwise setting this to false reduces filesystem clutter and overhead +# for managing these symlinks in the directory. This is now disabled by +# default. You may still return to upstream Conduit but you have to run +# conduwuit at least once with this set to true and allow the +# media_startup_check to take place before shutting down to return to +# Conduit. # #media_compat_file_link = false -# Prunes missing media from the database as part of the media startup -# checks. This means if you delete files from the media directory the +# Prune missing media from the database as part of the media startup +# checks. +# +# This means if you delete files from the media directory the # corresponding entries will be removed from the database. This is # disabled by default because if the media directory is accidentally moved # or inaccessible, the metadata entries in the database will be lost with @@ -1146,44 +1172,47 @@ #url_preview_bound_interface = # Vector list of domains allowed to send requests to for URL previews. -# Defaults to none. Note: this is a *contains* match, not an explicit -# match. Putting "google.com" will match "https://google.com" and +# +# This is a *contains* match, not an explicit match. Putting "google.com" +# will match "https://google.com" and # "http://mymaliciousdomainexamplegoogle.com" Setting this to "*" will # allow all URL previews. Please note that this opens up significant -# attack surface to your server, you are expected to be aware of the -# risks by doing so. +# attack surface to your server, you are expected to be aware of the risks +# by doing so. # #url_preview_domain_contains_allowlist = [] # Vector list of explicit domains allowed to send requests to for URL -# previews. Defaults to none. Note: This is an *explicit* match, not a -# contains match. Putting "google.com" will match "https://google.com", -# "http://google.com", but not +# previews. +# +# This is an *explicit* match, not a contains match. Putting "google.com" +# will match "https://google.com", "http://google.com", but not # "https://mymaliciousdomainexamplegoogle.com". Setting this to "*" will # allow all URL previews. Please note that this opens up significant -# attack surface to your server, you are expected to be aware of the -# risks by doing so. +# attack surface to your server, you are expected to be aware of the risks +# by doing so. # #url_preview_domain_explicit_allowlist = [] # Vector list of explicit domains not allowed to send requests to for URL -# previews. Defaults to none. Note: This is an *explicit* match, not a -# contains match. Putting "google.com" will match "https://google.com", -# "http://google.com", but not +# previews. +# +# This is an *explicit* match, not a contains match. Putting "google.com" +# will match "https://google.com", "http://google.com", but not # "https://mymaliciousdomainexamplegoogle.com". The denylist is checked # first before allowlist. Setting this to "*" will not do anything. # #url_preview_domain_explicit_denylist = [] # Vector list of URLs allowed to send requests to for URL previews. -# Defaults to none. Note that this is a *contains* match, not an -# explicit match. Putting "google.com" will match -# "https://google.com/", +# +# Note that this is a *contains* match, not an explicit match. Putting +# "google.com" will match "https://google.com/", # "https://google.com/url?q=https://mymaliciousdomainexample.com", and -# "https://mymaliciousdomainexample.com/hi/google.com" Setting this to -# "*" will allow all URL previews. Please note that this opens up -# significant attack surface to your server, you are expected to be -# aware of the risks by doing so. +# "https://mymaliciousdomainexample.com/hi/google.com" Setting this to "*" +# will allow all URL previews. Please note that this opens up significant +# attack surface to your server, you are expected to be aware of the risks +# by doing so. # #url_preview_url_contains_allowlist = [] @@ -1196,21 +1225,20 @@ # checks (contains and explicit) on the root domain or not. Does not apply # to URL contains allowlist. Defaults to false. # -# Example usecase: If this is -# enabled and you have "wikipedia.org" allowed in the explicit and/or -# contains domain allowlist, it will allow all subdomains under -# "wikipedia.org" such as "en.m.wikipedia.org" as the root domain is -# checked and matched. Useful if the domain contains allowlist is still -# too broad for you but you still want to allow all the subdomains under a -# root domain. +# Example usecase: If this is enabled and you have "wikipedia.org" allowed +# in the explicit and/or contains domain allowlist, it will allow all +# subdomains under "wikipedia.org" such as "en.m.wikipedia.org" as the +# root domain is checked and matched. Useful if the domain contains +# allowlist is still too broad for you but you still want to allow all the +# subdomains under a root domain. # #url_preview_check_root_domain = false # List of forbidden room aliases and room IDs as strings of regex # patterns. # -# Regex can be used or explicit contains matches can be done by -# just specifying the words (see example). +# Regex can be used or explicit contains matches can be done by just +# specifying the words (see example). # # This is checked upon room alias creation, custom room ID creation if # used, and startup as warnings if any room aliases in your database have @@ -1234,15 +1262,15 @@ #forbidden_usernames = [] # Retry failed and incomplete messages to remote servers immediately upon -# startup. This is called bursting. If this is disabled, said messages -# may not be delivered until more messages are queued for that server. Do -# not change this option unless server resources are extremely limited or -# the scale of the server's deployment is huge. Do not disable this -# unless you know what you are doing. +# startup. This is called bursting. If this is disabled, said messages may +# not be delivered until more messages are queued for that server. Do not +# change this option unless server resources are extremely limited or the +# scale of the server's deployment is huge. Do not disable this unless you +# know what you are doing. # #startup_netburst = true -# messages are dropped and not reattempted. The `startup_netburst` option +# Messages are dropped and not reattempted. The `startup_netburst` option # must be enabled for this value to have any effect. Do not change this # value unless you know what you are doing. Set this value to -1 to # reattempt every message without trimming the queues; this may consume @@ -1251,44 +1279,44 @@ # #startup_netburst_keep = 50 -# controls whether non-admin local users are forbidden from sending room -# invites (local and remote), and if non-admin users can receive remote -# room invites. admins are always allowed to send and receive all room -# invites. +# Block non-admin local users from sending room invites (local and +# remote), and block non-admin users from receiving remote room invites. +# +# Admins are always allowed to send and receive all room invites. # #block_non_admin_invites = false -# Allows admins to enter commands in rooms other than "#admins" (admin -# room) by prefixing your message with "\!admin" or "\\!admin" followed -# up a normal conduwuit admin command. The reply will be publicly visible -# to the room, originating from the sender. +# Allow admins to enter commands in rooms other than "#admins" (admin +# room) by prefixing your message with "\!admin" or "\\!admin" followed up +# a normal conduwuit admin command. The reply will be publicly visible to +# the room, originating from the sender. # # example: \\!admin debug ping puppygock.gay # #admin_escape_commands = true -# Controls whether the conduwuit admin room console / CLI will immediately -# activate on startup. This option can also be enabled with `--console` -# conduwuit argument. +# Automatically activate the conduwuit admin room console / CLI on +# startup. This option can also be enabled with `--console` conduwuit +# argument. # #admin_console_automatic = false -# Controls what admin commands will be executed on startup. This is a -# vector list of strings of admin commands to run. -# +# List of admin commands to execute on startup. # # This option can also be configured with the `--execute` conduwuit # argument and can take standard shell commands and environment variables # -# Such example could be: `./conduwuit --execute "server admin-notice -# conduwuit has started up at $(date)"` +# For example: `./conduwuit --execute "server admin-notice conduwuit has +# started up at $(date)"` # # example: admin_execute = ["debug ping puppygock.gay", "debug echo hi"]` # #admin_execute = [] -# Controls whether conduwuit should error and fail to start if an admin -# execute command (`--execute` / `admin_execute`) fails. +# Ignore errors in startup commands. +# +# If false, conduwuit will error and fail to start if an admin execute +# command (`--execute` / `admin_execute`) fails. # #admin_execute_errors_ignore = false @@ -1309,19 +1337,20 @@ # Sentry.io crash/panic reporting, performance monitoring/metrics, etc. # This is NOT enabled by default. conduwuit's default Sentry reporting -# endpoint domain is o4506996327251968.ingest.us.sentry.io +# endpoint domain is `o4506996327251968.ingest.us.sentry.io`. # #sentry = false -# Sentry reporting URL if a custom one is desired +# Sentry reporting URL, if a custom one is desired. # #sentry_endpoint = "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" -# Report your conduwuit server_name in Sentry.io crash reports and metrics +# Report your conduwuit server_name in Sentry.io crash reports and +# metrics. # #sentry_send_server_name = false -# Performance monitoring/tracing sample rate for Sentry.io +# Performance monitoring/tracing sample rate for Sentry.io. # # Note that too high values may impact performance, and can be disabled by # setting it to 0.0 (0%) This value is read as a percentage to Sentry, @@ -1333,8 +1362,8 @@ # #sentry_attach_stacktrace = false -# Send panics to sentry. This is true by default, but sentry has to be -# enabled. The global "sentry" config option must be enabled to send any +# Send panics to Sentry. This is true by default, but Sentry has to be +# enabled. The global `sentry` config option must be enabled to send any # data. # #sentry_send_panic = true @@ -1351,7 +1380,9 @@ #sentry_filter = "info" # Enable the tokio-console. This option is only relevant to developers. -# See https://conduwuit.puppyirl.gay/development.html#debugging-with-tokio-console for more information. +# +# For more information, see: +# https://conduwuit.puppyirl.gay/development.html#debugging-with-tokio-console # #tokio_console = false @@ -1360,8 +1391,8 @@ #test = false # Controls whether admin room notices like account registrations, password -# changes, account deactivations, room directory publications, etc will -# be sent to the admin room. Update notices and normal admin command +# changes, account deactivations, room directory publications, etc will be +# sent to the admin room. Update notices and normal admin command # responses will still be sent. # #admin_room_notices = true diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 44b66f41..8fd5621f 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -28,7 +28,7 @@ pub use self::check::check; use self::proxy::ProxyConfig; use crate::{err, error::Error, utils::sys, Result}; -/// all the config options for conduwuit +/// All the config options for conduwuit. #[allow(clippy::struct_excessive_bools)] #[allow(rustdoc::broken_intra_doc_links, rustdoc::bare_urls)] #[derive(Clone, Debug, Deserialize)] @@ -36,19 +36,32 @@ use crate::{err, error::Error, utils::sys, Result}; filename = "conduwuit-example.toml", section = "global", undocumented = "# This item is undocumented. Please contribute documentation for it.", - header = "### conduwuit Configuration\n###\n### THIS FILE IS GENERATED. CHANGES/CONTRIBUTIONS IN THE REPO WILL\n### BE \ - OVERWRITTEN!\n###\n### You should rename this file before configuring your server. Changes\n### to \ - documentation and defaults can be contributed in source code at\n### src/core/config/mod.rs. This file \ - is generated when building.\n###\n### Any values pre-populated are the default values for said config \ - option.\n###\n### At the minimum, you MUST edit all the config options to your environment\n### that say \ - \"YOU NEED TO EDIT THIS\".\n### See https://conduwuit.puppyirl.gay/configuration.html for ways to\n### configure conduwuit\n", + header = r#"### conduwuit Configuration +### +### THIS FILE IS GENERATED. CHANGES/CONTRIBUTIONS IN THE REPO WILL BE +### OVERWRITTEN! +### +### You should rename this file before configuring your server. Changes to +### documentation and defaults can be contributed in source code at +### src/core/config/mod.rs. This file is generated when building. +### +### Any values pre-populated are the default values for said config option. +### +### At the minimum, you MUST edit all the config options to your environment +### that say "YOU NEED TO EDIT THIS". +### +### For more information, see: +### https://conduwuit.puppyirl.gay/configuration.html +"#, ignore = "catchall well_known tls" )] pub struct Config { /// The server_name is the pretty name of this server. It is used as a /// suffix for user and room IDs/aliases. /// - /// See the docs for reverse proxying and delegation: https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy + /// See the docs for reverse proxying and delegation: + /// https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy + /// /// Also see the `[global.well_known]` config section at the very bottom. /// /// Examples of delegation: @@ -61,7 +74,7 @@ pub struct Config { /// example: "conduwuit.woof" pub server_name: OwnedServerName, - /// default address (IPv4 or IPv6) conduwuit will listen on. + /// The default address (IPv4 or IPv6) conduwuit will listen on. /// /// If you are using Docker or a container NAT networking setup, this must /// be "0.0.0.0". @@ -73,12 +86,13 @@ pub struct Config { #[serde(default = "default_address")] address: ListeningAddr, - /// The port(s) conduwuit will be running on. + /// The port(s) conduwuit will listen on. /// - /// See https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy for reverse proxying. + /// For reverse proxying, see: + /// https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy /// - /// Docker users: Don't change this, you'll need to map an external port to - /// this. + /// If you are using Docker, don't change this, you'll need to map an + /// external port to this. /// /// To listen on multiple ports, specify a vector e.g. [8080, 8448] /// @@ -90,11 +104,14 @@ pub struct Config { #[serde(default)] pub tls: TlsConfig, - /// Uncomment unix_socket_path to listen on a UNIX socket at the specified - /// path. If listening on a UNIX socket, you MUST remove/comment the - /// 'address' key if definedm AND add your reverse proxy to the 'conduwuit' - /// group, unless world RW permissions are specified with unix_socket_perms - /// (666 minimum). + /// The UNIX socket conduwuit will listen on. + /// + /// conduwuit cannot listen on both an IP address and a UNIX socket. If + /// listening on a UNIX socket, you MUST remove/comment the `address` key. + /// + /// Remember to make sure that your reverse proxy has access to this socket + /// file, either by adding your reverse proxy to the 'conduwuit' group or + /// granting world R/W permissions with `unix_socket_perms` (666 minimum). /// /// example: "/run/conduwuit/conduwuit.sock" pub unix_socket_path: Option, @@ -106,8 +123,7 @@ pub struct Config { pub unix_socket_perms: u32, /// This is the only directory where conduwuit will save its data, including - /// media. - /// Note: this was previously "/var/lib/matrix-conduit" + /// media. Note: this was previously "/var/lib/matrix-conduit". /// /// YOU NEED TO EDIT THIS. /// @@ -118,7 +134,8 @@ pub struct Config { /// API. To use this, set a database backup path that conduwuit can write /// to. /// - /// See https://conduwuit.puppyirl.gay/maintenance.html#backups for more information. + /// For more information, see: + /// https://conduwuit.puppyirl.gay/maintenance.html#backups /// /// example: "/opt/conduwuit-db-backups" pub database_backup_path: Option, @@ -140,16 +157,17 @@ pub struct Config { /// Similar to the individual LRU caches, this is scaled up with your CPU /// core count. /// - /// This defaults to 128.0 + (64.0 * CPU core count) + /// This defaults to 128.0 + (64.0 * CPU core count). /// /// default: varies by system #[serde(default = "default_db_cache_capacity_mb")] pub db_cache_capacity_mb: f64, - /// Option to control adding arbitrary text to the end of the user's - /// displayname upon registration with a space before the text. This was the - /// lightning bolt emoji option, just replaced with support for adding your - /// own custom text or emojis. To disable, set this to "" (an empty string). + /// Text which will be added to the end of the user's displayname upon + /// registration with a space before the text. In Conduit, this was the + /// lightning bolt emoji. + /// + /// To disable, set this to "" (an empty string). /// /// The default is the trans pride flag. /// @@ -237,8 +255,8 @@ pub struct Config { /// Maximum entries stored in DNS memory-cache. The size of an entry may /// vary so please take care if raising this value excessively. Only - /// decrease this when using an external DNS cache. Please note - /// that systemd-resolved does *not* count as an external cache, even when + /// decrease this when using an external DNS cache. Please note that + /// systemd-resolved does *not* count as an external cache, even when /// configured to do so. /// /// default: 32768 @@ -257,8 +275,8 @@ pub struct Config { /// Minimum time-to-live in seconds for NXDOMAIN entries in the DNS cache. /// This value is critical for the server to federate efficiently. - /// NXDOMAIN's are assumed to not be returning to the federation - /// and aggressively cached rather than constantly rechecked. + /// NXDOMAIN's are assumed to not be returning to the federation and + /// aggressively cached rather than constantly rechecked. /// /// Defaults to 3 days as these are *very rarely* false negatives. /// @@ -293,10 +311,12 @@ pub struct Config { #[serde(default = "true_fn")] pub query_all_nameservers: bool, - /// Enables using *only* TCP for querying your specified nameservers instead + /// Enable using *only* TCP for querying your specified nameservers instead /// of UDP. /// - /// If you are running conduwuit in a container environment, this config option may need to be enabled. See https://conduwuit.puppyirl.gay/troubleshooting.html#potential-dns-issues-when-using-docker for more details. + /// If you are running conduwuit in a container environment, this config + /// option may need to be enabled. For more details, see: + /// https://conduwuit.puppyirl.gay/troubleshooting.html#potential-dns-issues-when-using-docker #[serde(default)] pub query_over_tcp_only: bool, @@ -352,8 +372,8 @@ pub struct Config { /// Default/base request total timeout (seconds). The time limit for a whole /// request. This is set very high to not cancel healthy requests while - /// serving as a backstop. This is used only by URL previews and - /// update/news endpoint checks. + /// serving as a backstop. This is used only by URL previews and update/news + /// endpoint checks. /// /// default: 320 #[serde(default = "default_request_total_timeout")] @@ -374,13 +394,13 @@ pub struct Config { #[serde(default = "default_request_idle_per_host")] pub request_idle_per_host: u16, - /// Federation well-known resolution connection timeout (seconds) + /// Federation well-known resolution connection timeout (seconds). /// /// default: 6 #[serde(default = "default_well_known_conn_timeout")] pub well_known_conn_timeout: u64, - /// Federation HTTP well-known resolution request timeout (seconds) + /// Federation HTTP well-known resolution request timeout (seconds). /// /// default: 10 #[serde(default = "default_well_known_timeout")] @@ -394,14 +414,14 @@ pub struct Config { #[serde(default = "default_federation_timeout")] pub federation_timeout: u64, - /// Federation client idle connection pool timeout (seconds) + /// Federation client idle connection pool timeout (seconds). /// /// default: 25 #[serde(default = "default_federation_idle_timeout")] pub federation_idle_timeout: u64, /// Federation client max idle connections per host. Defaults to 1 as - /// generally the same open connection can be re-used + /// generally the same open connection can be re-used. /// /// default: 1 #[serde(default = "default_federation_idle_per_host")] @@ -414,13 +434,13 @@ pub struct Config { #[serde(default = "default_sender_timeout")] pub sender_timeout: u64, - /// Federation sender idle connection pool timeout (seconds) + /// Federation sender idle connection pool timeout (seconds). /// /// default: 180 #[serde(default = "default_sender_idle_timeout")] pub sender_idle_timeout: u64, - /// Federation sender transaction retry backoff limit (seconds) + /// Federation sender transaction retry backoff limit (seconds). /// /// default: 86400 #[serde(default = "default_sender_retry_backoff_limit")] @@ -433,13 +453,13 @@ pub struct Config { #[serde(default = "default_appservice_timeout")] pub appservice_timeout: u64, - /// Appservice URL idle connection pool timeout (seconds) + /// Appservice URL idle connection pool timeout (seconds). /// /// default: 300 #[serde(default = "default_appservice_idle_timeout")] pub appservice_idle_timeout: u64, - /// Notification gateway pusher idle connection pool timeout + /// Notification gateway pusher idle connection pool timeout. /// /// default: 15 #[serde(default = "default_pusher_idle_timeout")] @@ -449,9 +469,8 @@ pub struct Config { /// server. /// /// If set to true without a token configured, users can register with no - /// form of 2nd-step only if you set - /// `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` to - /// true in your config. + /// form of 2nd-step only if you set the following option to true: + /// `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` /// /// If you would like registration only via token reg, please configure /// `registration_token` or `registration_token_file`. @@ -501,8 +520,8 @@ pub struct Config { /// Set this to true to allow your server's public room directory to be /// federated. Set this to false to protect against /publicRooms spiders, /// but will forbid external users from viewing your server's public room - /// directory. If federation is disabled entirely (`allow_federation`), - /// this is inherently false. + /// directory. If federation is disabled entirely (`allow_federation`), this + /// is inherently false. #[serde(default)] pub allow_public_room_directory_over_federation: bool, @@ -512,10 +531,10 @@ pub struct Config { #[serde(default)] pub allow_public_room_directory_without_auth: bool, - /// allow guests/unauthenticated users to access TURN credentials + /// Allow guests/unauthenticated users to access TURN credentials. /// - /// this is the equivalent of Synapse's `turn_allow_guests` config option. - /// this allows any unauthenticated user to call the endpoint + /// This is the equivalent of Synapse's `turn_allow_guests` config option. + /// This allows any unauthenticated user to call the endpoint /// `/_matrix/client/v3/voip/turnServer`. /// /// It is unlikely you need to enable this as all major clients support @@ -550,24 +569,24 @@ pub struct Config { #[serde(default = "true_fn", alias = "allow_profile_lookup_federation_requests")] pub allow_inbound_profile_lookup_federation_requests: bool, - /// controls whether standard users are allowed to create rooms. appservices - /// and admins are always allowed to create rooms + /// Allow standard users to create rooms. Appservices and admins are always + /// allowed to create rooms #[serde(default = "true_fn")] pub allow_room_creation: bool, /// Set to false to disable users from joining or creating room versions - /// that aren't 100% officially supported by conduwuit. + /// that aren't officially supported by conduwuit. /// /// conduwuit officially supports room versions 6 - 11. /// /// conduwuit has slightly experimental (though works fine in practice) - /// support for versions 3 - 5 + /// support for versions 3 - 5. #[serde(default = "true_fn")] pub allow_unstable_room_versions: bool, - /// default room version conduwuit will create rooms with. + /// Default room version conduwuit will create rooms with. /// - /// per spec, room version 10 is the default. + /// Per spec, room version 10 is the default. /// /// default: 10 #[serde(default = "default_default_room_version")] @@ -603,22 +622,28 @@ pub struct Config { pub tracing_flame_output_path: String, /// Examples: + /// /// - No proxy (default): - /// proxy ="none" + /// + /// proxy = "none" /// /// - For global proxy, create the section at the bottom of this file: - /// [global.proxy] - /// global = { url = "socks5h://localhost:9050" } + /// + /// [global.proxy] + /// global = { url = "socks5h://localhost:9050" } /// /// - To proxy some domains: - /// [global.proxy] - /// [[global.proxy.by_domain]] - /// url = "socks5h://localhost:9050" - /// include = ["*.onion", "matrix.myspecial.onion"] - /// exclude = ["*.myspecial.onion"] + /// + /// [global.proxy] + /// [[global.proxy.by_domain]] + /// url = "socks5h://localhost:9050" + /// include = ["*.onion", "matrix.myspecial.onion"] + /// exclude = ["*.myspecial.onion"] /// /// Include vs. Exclude: + /// /// - If include is an empty list, it is assumed to be `["*"]`. + /// /// - If a domain matches both the exclude and include list, the proxy will /// only be used if it was included because of a more specific rule than /// it was excluded. In the above example, the proxy would be used for @@ -635,7 +660,7 @@ pub struct Config { /// (notary trusted key servers). /// /// Currently, conduwuit doesn't support inbound batched key requests, so - /// this list should only contain other Synapse servers + /// this list should only contain other Synapse servers. /// /// example: ["matrix.org", "envs.net", "constellatory.net", "tchncs.de"] /// @@ -658,10 +683,10 @@ pub struct Config { /// compromised trusted server to room joins only. The join operation /// requires gathering keys from many origin servers which can cause /// significant delays. Therefor this defaults to true to mitigate - /// unexpected delays out-of-the-box. The security-paranoid or those - /// willing to tolerate delays are advised to set this to false. Note that - /// setting query_trusted_key_servers_first to true causes this option to - /// be ignored. + /// unexpected delays out-of-the-box. The security-paranoid or those willing + /// to tolerate delays are advised to set this to false. Note that setting + /// query_trusted_key_servers_first to true causes this option to be + /// ignored. #[serde(default = "true_fn")] pub query_trusted_key_servers_first_on_join: bool, @@ -679,8 +704,10 @@ pub struct Config { #[serde(default = "default_trusted_server_batch_size")] pub trusted_server_batch_size: usize, - /// max log level for conduwuit. allows debug, info, warn, or error - /// see also: https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives + /// Max log level for conduwuit. Allows debug, info, warn, or error. + /// + /// See also: + /// https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives /// /// **Caveat**: /// For release builds, the tracing crate is configured to only implement @@ -692,56 +719,56 @@ pub struct Config { #[serde(default = "default_log")] pub log: String, - /// controls whether logs will be outputted with ANSI colours + /// Output logs with ANSI colours. #[serde(default = "true_fn", alias = "log_colours")] pub log_colors: bool, - /// configures the span events which will be outputted with the log + /// Configures the span events which will be outputted with the log. /// /// default: "none" #[serde(default = "default_log_span_events")] pub log_span_events: String, - /// configures whether CONDUWUIT_LOG EnvFilter matches values using regular + /// Configures whether CONDUWUIT_LOG EnvFilter matches values using regular /// expressions. See the tracing_subscriber documentation on Directives. /// /// default: true #[serde(default = "true_fn")] pub log_filter_regex: bool, - /// toggles the display of ThreadId in tracing log output + /// Toggles the display of ThreadId in tracing log output. /// /// default: false #[serde(default)] pub log_thread_ids: bool, - /// OpenID token expiration/TTL in seconds + /// OpenID token expiration/TTL in seconds. /// /// These are the OpenID tokens that are primarily used for Matrix account /// integrations (e.g. Vector Integrations in Element), *not* OIDC/OpenID - /// Connect/etc + /// Connect/etc. /// /// default: 3600 #[serde(default = "default_openid_token_ttl")] pub openid_token_ttl: u64, - /// static TURN username to provide the client if not using a shared secret + /// Static TURN username to provide the client if not using a shared secret /// ("turn_secret"), It is recommended to use a shared secret over static /// credentials. #[serde(default)] pub turn_username: String, - /// static TURN password to provide the client if not using a shared secret + /// Static TURN password to provide the client if not using a shared secret /// ("turn_secret"). It is recommended to use a shared secret over static /// credentials. #[serde(default)] pub turn_password: String, - /// vector list of TURN URIs/servers to use + /// Vector list of TURN URIs/servers to use. /// - /// replace "example.turn.uri" with your TURN domain, such as the coturn - /// "realm" config option. if using TURN over TLS, replace the URI prefix - /// "turn:" with "turns:" + /// Replace "example.turn.uri" with your TURN domain, such as the coturn + /// "realm" config option. If using TURN over TLS, replace the URI prefix + /// "turn:" with "turns:". /// /// example: ["turn:example.turn.uri?transport=udp", /// "turn:example.turn.uri?transport=tcp"] @@ -751,30 +778,30 @@ pub struct Config { pub turn_uris: Vec, /// TURN secret to use for generating the HMAC-SHA1 hash apart of username - /// and password generation + /// and password generation. /// - /// this is more secure, but if needed you can use traditional - /// static username/password credentials. + /// This is more secure, but if needed you can use traditional static + /// username/password credentials. #[serde(default)] pub turn_secret: String, - /// TURN secret to use that's read from the file path specified + /// TURN secret to use that's read from the file path specified. /// - /// this takes priority over "turn_secret" first, and falls back to + /// This takes priority over "turn_secret" first, and falls back to /// "turn_secret" if invalid or failed to open. /// /// example: "/etc/conduwuit/.turn_secret" pub turn_secret_file: Option, - /// TURN TTL in seconds + /// TURN TTL, in seconds. /// /// default: 86400 #[serde(default = "default_turn_ttl")] pub turn_ttl: u64, /// List/vector of room IDs or room aliases that conduwuit will make newly - /// registered users join. The rooms specified must be rooms that you - /// have joined at least once on the server, and must be public. + /// registered users join. The rooms specified must be rooms that you have + /// joined at least once on the server, and must be public. /// /// example: ["#conduwuit:puppygock.gay", /// "!eoIzvAvVwY23LPDay8:puppygock.gay"] @@ -798,7 +825,7 @@ pub struct Config { /// room invites) are ignored here. /// /// Defaults to false as rooms can be banned for non-moderation-related - /// reasons and this performs a full user deactivation + /// reasons and this performs a full user deactivation. #[serde(default)] pub auto_deactivate_banned_room_attempts: bool, @@ -828,7 +855,7 @@ pub struct Config { pub rocksdb_log_time_to_roll: usize, /// Set this to true to use RocksDB config options that are tailored to HDDs - /// (slower device storage) + /// (slower device storage). /// /// It is worth noting that by default, conduwuit will use RocksDB with /// Direct IO enabled. *Generally* speaking this improves performance as it @@ -841,13 +868,15 @@ pub struct Config { /// feel free to report in the conduwuit Matrix room if this option fixes /// your DB issues. /// - /// See https://github.com/facebook/rocksdb/wiki/Direct-IO for more information. + /// For more information, see: + /// https://github.com/facebook/rocksdb/wiki/Direct-IO #[serde(default)] pub rocksdb_optimize_for_spinning_disks: bool, /// Enables direct-io to increase database performance via unbuffered I/O. /// - /// See https://github.com/facebook/rocksdb/wiki/Direct-IO for more details about Direct IO and RocksDB. + /// For more details about direct I/O and RockDB, see: + /// https://github.com/facebook/rocksdb/wiki/Direct-IO /// /// Set this option to false if the database resides on a filesystem which /// does not support direct-io like FUSE, or any form of complex filesystem @@ -856,8 +885,8 @@ pub struct Config { pub rocksdb_direct_io: bool, /// Amount of threads that RocksDB will use for parallelism on database - /// operatons such as cleanup, sync, flush, compaction, etc. Set to 0 to use - /// all your logical threads. Defaults to your CPU logical thread count. + /// operations such as cleanup, sync, flush, compaction, etc. Set to 0 to + /// use all your logical threads. Defaults to your CPU logical thread count. /// /// default: varies by system #[serde(default = "default_rocksdb_parallelism_threads")] @@ -865,7 +894,7 @@ pub struct Config { /// Maximum number of LOG files RocksDB will keep. This must *not* be set to /// 0. It must be at least 1. Defaults to 3 as these are not very useful - /// unless troubleshooting/debugging a RocksDB bug. + /// unless troubleshooting/debugging a RocksDB bug. /// /// default: 3 #[serde(default = "default_rocksdb_max_log_files")] @@ -873,12 +902,15 @@ pub struct Config { /// Type of RocksDB database compression to use. /// - /// Available options are "zstd", "zlib", "bz2", "lz4", or "none" + /// Available options are "zstd", "zlib", "bz2", "lz4", or "none". /// /// It is best to use ZSTD as an overall good balance between - /// speed/performance, storage, IO amplification, and CPU usage. - /// For more performance but less compression (more storage used) and less - /// CPU usage, use LZ4. See https://github.com/facebook/rocksdb/wiki/Compression for more details. + /// speed/performance, storage, IO amplification, and CPU usage. For more + /// performance but less compression (more storage used) and less CPU usage, + /// use LZ4. + /// + /// For more details, see: + /// https://github.com/facebook/rocksdb/wiki/Compression /// /// "none" will disable compression. /// @@ -889,10 +921,9 @@ pub struct Config { /// Level of compression the specified compression algorithm for RocksDB to /// use. /// - /// Default is 32767, which is internally read by RocksDB as the - /// default magic number and translated to the library's default - /// compression level as they all differ. - /// See their `kDefaultCompressionLevel`. + /// Default is 32767, which is internally read by RocksDB as the default + /// magic number and translated to the library's default compression level + /// as they all differ. See their `kDefaultCompressionLevel`. /// /// default: 32767 #[serde(default = "default_rocksdb_compression_level")] @@ -900,13 +931,13 @@ pub struct Config { /// Level of compression the specified compression algorithm for the /// bottommost level/data for RocksDB to use. Default is 32767, which is - /// internally read by RocksDB as the default magic number and translated - /// to the library's default compression level as they all differ. - /// See their `kDefaultCompressionLevel`. + /// internally read by RocksDB as the default magic number and translated to + /// the library's default compression level as they all differ. See their + /// `kDefaultCompressionLevel`. /// /// Since this is the bottommost level (generally old and least used data), /// it may be desirable to have a very high compression level here as it's - /// lesss likely for this data to be used. Research your chosen compression + /// less likely for this data to be used. Research your chosen compression /// algorithm. /// /// default: 32767 @@ -924,7 +955,7 @@ pub struct Config { #[serde(default)] pub rocksdb_bottommost_compression: bool, - /// Database recovery mode (for RocksDB WAL corruption) + /// Database recovery mode (for RocksDB WAL corruption). /// /// Use this option when the server reports corruption and refuses to start. /// Set mode 2 (PointInTime) to cleanly recover from this corruption. The @@ -932,8 +963,8 @@ pub struct Config { /// minutes prior to the crash. Clients may have to run "clear-cache & /// reload" to account for the rollback. Upon success, you may reset the /// mode back to default and restart again. Please note in some cases the - /// corruption error may not be cleared for at least 30 minutes of - /// operation in PointInTime mode. + /// corruption error may not be cleared for at least 30 minutes of operation + /// in PointInTime mode. /// /// As a very last ditch effort, if PointInTime does not fix or resolve /// anything, you can try mode 3 (SkipAnyCorruptedRecord) but this will @@ -951,9 +982,11 @@ pub struct Config { /// 2 = PointInTime (use me if trying to recover) /// 3 = SkipAnyCorruptedRecord (you now voided your Conduwuit warranty) /// - /// See https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes for more information on these modes. + /// For more information on these modes, see: + /// https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes /// - /// See https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption for more details on recovering a corrupt database. + /// For more details on recovering a corrupt database, see: + /// https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption /// /// default: 1 #[serde(default = "default_rocksdb_recovery_mode")] @@ -963,11 +996,12 @@ pub struct Config { /// database consistency at a potential performance impact due to further /// safety checks ran. /// - /// See https://github.com/facebook/rocksdb/wiki/Online-Verification#columnfamilyoptionsparanoid_file_checks for more information. + /// For more information, see: + /// https://github.com/facebook/rocksdb/wiki/Online-Verification#columnfamilyoptionsparanoid_file_checks #[serde(default)] pub rocksdb_paranoid_file_checks: bool, - /// Database repair mode (for RocksDB SST corruption) + /// Database repair mode (for RocksDB SST corruption). /// /// Use this option when the server reports corruption while running or /// panics. If the server refuses to start use the recovery mode options @@ -976,6 +1010,7 @@ pub struct Config { /// /// - Backing up your database directory is recommended prior to running the /// repair. + /// /// - Disabling repair mode and restarting the server is recommended after /// running the repair. /// @@ -1001,10 +1036,10 @@ pub struct Config { #[serde(default = "true_fn")] pub rocksdb_compaction_ioprio_idle: bool, - /// Config option to disable RocksDB compaction. You should never ever have - /// to disable this. If you for some reason find yourself needing to disable - /// this as part of troubleshooting or a bug, please reach out to us in the - /// conduwuit Matrix room with information and details. + /// Disables RocksDB compaction. You should never ever have to set this + /// option to true. If you for some reason find yourself needing to use this + /// option as part of troubleshooting or a bug, please reach out to us in + /// the conduwuit Matrix room with information and details. /// /// Disabling compaction will lead to a significantly bloated and /// explosively large database, gradually poor performance, unnecessarily @@ -1044,87 +1079,84 @@ pub struct Config { #[serde(default = "default_notification_push_path")] pub notification_push_path: String, - /// Config option to control local (your server only) presence - /// updates/requests. Note that presence on conduwuit is - /// very fast unlike Synapse's. If using outgoing presence, this MUST be - /// enabled. + /// Allow local (your server only) presence updates/requests. + /// + /// Note that presence on conduwuit is very fast unlike Synapse's. If using + /// outgoing presence, this MUST be enabled. #[serde(default = "true_fn")] pub allow_local_presence: bool, - /// Config option to control incoming federated presence updates/requests. + /// Allow incoming federated presence updates/requests. /// - /// This option receives presence updates from other - /// servers, but does not send any unless `allow_outgoing_presence` is true. - /// Note that presence on conduwuit is very fast unlike Synapse's. + /// This option receives presence updates from other servers, but does not + /// send any unless `allow_outgoing_presence` is true. Note that presence on + /// conduwuit is very fast unlike Synapse's. #[serde(default = "true_fn")] pub allow_incoming_presence: bool, - /// Config option to control outgoing presence updates/requests. + /// Allow outgoing presence updates/requests. /// /// This option sends presence updates to other servers, but does not - /// receive any unless `allow_incoming_presence` is true. - /// Note that presence on conduwuit is very fast unlike Synapse's. - /// If using outgoing presence, you MUST enable `allow_local_presence` as - /// well. + /// receive any unless `allow_incoming_presence` is true. Note that presence + /// on conduwuit is very fast unlike Synapse's. If using outgoing presence, + /// you MUST enable `allow_local_presence` as well. #[serde(default = "true_fn")] pub allow_outgoing_presence: bool, - /// Config option to control how many seconds before presence updates that - /// you are idle. Defaults to 5 minutes. + /// How many seconds without presence updates before you become idle. + /// Defaults to 5 minutes. /// /// default: 300 #[serde(default = "default_presence_idle_timeout_s")] pub presence_idle_timeout_s: u64, - /// Config option to control how many seconds before presence updates that - /// you are offline. Defaults to 30 minutes. + /// How many seconds without presence updates before you become offline. + /// Defaults to 30 minutes. /// /// default: 1800 #[serde(default = "default_presence_offline_timeout_s")] pub presence_offline_timeout_s: u64, - /// Config option to enable the presence idle timer for remote users. + /// Enable the presence idle timer for remote users. + /// /// Disabling is offered as an optimization for servers participating in /// many large rooms or when resources are limited. Disabling it may cause - /// incorrect presence states (i.e. stuck online) to be seen for some - /// remote users. + /// incorrect presence states (i.e. stuck online) to be seen for some remote + /// users. #[serde(default = "true_fn")] pub presence_timeout_remote_users: bool, - /// Config option to control whether we should receive remote incoming read - /// receipts. + /// Allow receiving incoming read receipts from remote servers. #[serde(default = "true_fn")] pub allow_incoming_read_receipts: bool, - /// Config option to control whether we should send read receipts to remote - /// servers. + /// Allow sending read receipts to remote servers. #[serde(default = "true_fn")] pub allow_outgoing_read_receipts: bool, - /// Config option to control outgoing typing updates to federation. + /// Allow outgoing typing updates to federation. #[serde(default = "true_fn")] pub allow_outgoing_typing: bool, - /// Config option to control incoming typing updates from federation. + /// Allow incoming typing updates from federation. #[serde(default = "true_fn")] pub allow_incoming_typing: bool, - /// Config option to control maximum time federation user can indicate - /// typing. + /// Maximum time federation user can indicate typing. /// /// default: 30 #[serde(default = "default_typing_federation_timeout_s")] pub typing_federation_timeout_s: u64, - /// Config option to control minimum time local client can indicate typing. - /// This does not override a client's request to stop typing. It only - /// enforces a minimum value in case of no stop request. + /// Minimum time local client can indicate typing. This does not override a + /// client's request to stop typing. It only enforces a minimum value in + /// case of no stop request. /// /// default: 15 #[serde(default = "default_typing_client_timeout_min_s")] pub typing_client_timeout_min_s: u64, - /// Config option to control maximum time local client can indicate typing. + /// Maximum time local client can indicate typing. /// /// default: 45 #[serde(default = "default_typing_client_timeout_max_s")] @@ -1155,8 +1187,8 @@ pub struct Config { /// brotli. This option does nothing if conduwuit was not built with /// `brotli_compression` feature. Please be aware that enabling HTTP /// compression may weaken TLS. Most users should not need to enable this. - /// See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before - /// deciding to enable this. + /// See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH + /// before deciding to enable this. #[serde(default)] pub brotli_compression: bool, @@ -1175,8 +1207,8 @@ pub struct Config { #[serde(default)] pub allow_guests_auto_join_rooms: bool, - /// Config option to control whether the legacy unauthenticated Matrix media - /// repository endpoints will be enabled. These endpoints consist of: + /// Enable the legacy unauthenticated Matrix media repository endpoints. + /// These endpoints consist of: /// - /_matrix/media/*/config /// - /_matrix/media/*/upload /// - /_matrix/media/*/preview_url @@ -1193,8 +1225,8 @@ pub struct Config { #[serde(default = "true_fn")] pub freeze_legacy_media: bool, - /// Checks consistency of the media directory at startup: - /// 1. When `media_compat_file_link` is enbled, this check will upgrade + /// Check consistency of the media directory at startup: + /// 1. When `media_compat_file_link` is enabled, this check will upgrade /// media when switching back and forth between Conduit and conduwuit. /// Both options must be enabled to handle this. /// 2. When media is deleted from the directory, this check will also delete @@ -1207,18 +1239,22 @@ pub struct Config { pub media_startup_check: bool, /// Enable backward-compatibility with Conduit's media directory by creating - /// symlinks of media. This option is only necessary if you plan on using - /// Conduit again. Otherwise setting this to false reduces filesystem - /// clutter and overhead for managing these symlinks in the directory. This - /// is now disabled by default. You may still return to upstream Conduit - /// but you have to run conduwuit at least once with this set to true and - /// allow the media_startup_check to take place before shutting - /// down to return to Conduit. + /// symlinks of media. + /// + /// This option is only necessary if you plan on using Conduit again. + /// Otherwise setting this to false reduces filesystem clutter and overhead + /// for managing these symlinks in the directory. This is now disabled by + /// default. You may still return to upstream Conduit but you have to run + /// conduwuit at least once with this set to true and allow the + /// media_startup_check to take place before shutting down to return to + /// Conduit. #[serde(default)] pub media_compat_file_link: bool, - /// Prunes missing media from the database as part of the media startup - /// checks. This means if you delete files from the media directory the + /// Prune missing media from the database as part of the media startup + /// checks. + /// + /// This means if you delete files from the media directory the /// corresponding entries will be removed from the database. This is /// disabled by default because if the media directory is accidentally moved /// or inaccessible, the metadata entries in the database will be lost with @@ -1291,34 +1327,37 @@ pub struct Config { pub url_preview_bound_interface: Option>, /// Vector list of domains allowed to send requests to for URL previews. - /// Defaults to none. Note: this is a *contains* match, not an explicit - /// match. Putting "google.com" will match "https://google.com" and + /// + /// This is a *contains* match, not an explicit match. Putting "google.com" + /// will match "https://google.com" and /// "http://mymaliciousdomainexamplegoogle.com" Setting this to "*" will /// allow all URL previews. Please note that this opens up significant - /// attack surface to your server, you are expected to be aware of the - /// risks by doing so. + /// attack surface to your server, you are expected to be aware of the risks + /// by doing so. /// /// default: [] #[serde(default)] pub url_preview_domain_contains_allowlist: Vec, /// Vector list of explicit domains allowed to send requests to for URL - /// previews. Defaults to none. Note: This is an *explicit* match, not a - /// contains match. Putting "google.com" will match "https://google.com", - /// "http://google.com", but not + /// previews. + /// + /// This is an *explicit* match, not a contains match. Putting "google.com" + /// will match "https://google.com", "http://google.com", but not /// "https://mymaliciousdomainexamplegoogle.com". Setting this to "*" will /// allow all URL previews. Please note that this opens up significant - /// attack surface to your server, you are expected to be aware of the - /// risks by doing so. + /// attack surface to your server, you are expected to be aware of the risks + /// by doing so. /// /// default: [] #[serde(default)] pub url_preview_domain_explicit_allowlist: Vec, /// Vector list of explicit domains not allowed to send requests to for URL - /// previews. Defaults to none. Note: This is an *explicit* match, not a - /// contains match. Putting "google.com" will match "https://google.com", - /// "http://google.com", but not + /// previews. + /// + /// This is an *explicit* match, not a contains match. Putting "google.com" + /// will match "https://google.com", "http://google.com", but not /// "https://mymaliciousdomainexamplegoogle.com". The denylist is checked /// first before allowlist. Setting this to "*" will not do anything. /// @@ -1327,14 +1366,14 @@ pub struct Config { pub url_preview_domain_explicit_denylist: Vec, /// Vector list of URLs allowed to send requests to for URL previews. - /// Defaults to none. Note that this is a *contains* match, not an - /// explicit match. Putting "google.com" will match - /// "https://google.com/", + /// + /// Note that this is a *contains* match, not an explicit match. Putting + /// "google.com" will match "https://google.com/", /// "https://google.com/url?q=https://mymaliciousdomainexample.com", and - /// "https://mymaliciousdomainexample.com/hi/google.com" Setting this to - /// "*" will allow all URL previews. Please note that this opens up - /// significant attack surface to your server, you are expected to be - /// aware of the risks by doing so. + /// "https://mymaliciousdomainexample.com/hi/google.com" Setting this to "*" + /// will allow all URL previews. Please note that this opens up significant + /// attack surface to your server, you are expected to be aware of the risks + /// by doing so. /// /// default: [] #[serde(default)] @@ -1351,21 +1390,20 @@ pub struct Config { /// checks (contains and explicit) on the root domain or not. Does not apply /// to URL contains allowlist. Defaults to false. /// - /// Example usecase: If this is - /// enabled and you have "wikipedia.org" allowed in the explicit and/or - /// contains domain allowlist, it will allow all subdomains under - /// "wikipedia.org" such as "en.m.wikipedia.org" as the root domain is - /// checked and matched. Useful if the domain contains allowlist is still - /// too broad for you but you still want to allow all the subdomains under a - /// root domain. + /// Example usecase: If this is enabled and you have "wikipedia.org" allowed + /// in the explicit and/or contains domain allowlist, it will allow all + /// subdomains under "wikipedia.org" such as "en.m.wikipedia.org" as the + /// root domain is checked and matched. Useful if the domain contains + /// allowlist is still too broad for you but you still want to allow all the + /// subdomains under a root domain. #[serde(default)] pub url_preview_check_root_domain: bool, /// List of forbidden room aliases and room IDs as strings of regex /// patterns. /// - /// Regex can be used or explicit contains matches can be done by - /// just specifying the words (see example). + /// Regex can be used or explicit contains matches can be done by just + /// specifying the words (see example). /// /// This is checked upon room alias creation, custom room ID creation if /// used, and startup as warnings if any room aliases in your database have @@ -1395,15 +1433,15 @@ pub struct Config { pub forbidden_usernames: RegexSet, /// Retry failed and incomplete messages to remote servers immediately upon - /// startup. This is called bursting. If this is disabled, said messages - /// may not be delivered until more messages are queued for that server. Do - /// not change this option unless server resources are extremely limited or - /// the scale of the server's deployment is huge. Do not disable this - /// unless you know what you are doing. + /// startup. This is called bursting. If this is disabled, said messages may + /// not be delivered until more messages are queued for that server. Do not + /// change this option unless server resources are extremely limited or the + /// scale of the server's deployment is huge. Do not disable this unless you + /// know what you are doing. #[serde(default = "true_fn")] pub startup_netburst: bool, - /// messages are dropped and not reattempted. The `startup_netburst` option + /// Messages are dropped and not reattempted. The `startup_netburst` option /// must be enabled for this value to have any effect. Do not change this /// value unless you know what you are doing. Set this value to -1 to /// reattempt every message without trimming the queues; this may consume @@ -1414,37 +1452,35 @@ pub struct Config { #[serde(default = "default_startup_netburst_keep")] pub startup_netburst_keep: i64, - /// controls whether non-admin local users are forbidden from sending room - /// invites (local and remote), and if non-admin users can receive remote - /// room invites. admins are always allowed to send and receive all room - /// invites. + /// Block non-admin local users from sending room invites (local and + /// remote), and block non-admin users from receiving remote room invites. + /// + /// Admins are always allowed to send and receive all room invites. #[serde(default)] pub block_non_admin_invites: bool, - /// Allows admins to enter commands in rooms other than "#admins" (admin - /// room) by prefixing your message with "\!admin" or "\\!admin" followed - /// up a normal conduwuit admin command. The reply will be publicly visible - /// to the room, originating from the sender. + /// Allow admins to enter commands in rooms other than "#admins" (admin + /// room) by prefixing your message with "\!admin" or "\\!admin" followed up + /// a normal conduwuit admin command. The reply will be publicly visible to + /// the room, originating from the sender. /// /// example: \\!admin debug ping puppygock.gay #[serde(default = "true_fn")] pub admin_escape_commands: bool, - /// Controls whether the conduwuit admin room console / CLI will immediately - /// activate on startup. This option can also be enabled with `--console` - /// conduwuit argument. + /// Automatically activate the conduwuit admin room console / CLI on + /// startup. This option can also be enabled with `--console` conduwuit + /// argument. #[serde(default)] pub admin_console_automatic: bool, - /// Controls what admin commands will be executed on startup. This is a - /// vector list of strings of admin commands to run. - /// + /// List of admin commands to execute on startup. /// /// This option can also be configured with the `--execute` conduwuit /// argument and can take standard shell commands and environment variables /// - /// Such example could be: `./conduwuit --execute "server admin-notice - /// conduwuit has started up at $(date)"` + /// For example: `./conduwuit --execute "server admin-notice conduwuit has + /// started up at $(date)"` /// /// example: admin_execute = ["debug ping puppygock.gay", "debug echo hi"]` /// @@ -1452,8 +1488,10 @@ pub struct Config { #[serde(default)] pub admin_execute: Vec, - /// Controls whether conduwuit should error and fail to start if an admin - /// execute command (`--execute` / `admin_execute`) fails. + /// Ignore errors in startup commands. + /// + /// If false, conduwuit will error and fail to start if an admin execute + /// command (`--execute` / `admin_execute`) fails. #[serde(default)] pub admin_execute_errors_ignore: bool, @@ -1478,21 +1516,22 @@ pub struct Config { /// Sentry.io crash/panic reporting, performance monitoring/metrics, etc. /// This is NOT enabled by default. conduwuit's default Sentry reporting - /// endpoint domain is o4506996327251968.ingest.us.sentry.io + /// endpoint domain is `o4506996327251968.ingest.us.sentry.io`. #[serde(default)] pub sentry: bool, - /// Sentry reporting URL if a custom one is desired + /// Sentry reporting URL, if a custom one is desired. /// /// default: "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" #[serde(default = "default_sentry_endpoint")] pub sentry_endpoint: Option, - /// Report your conduwuit server_name in Sentry.io crash reports and metrics + /// Report your conduwuit server_name in Sentry.io crash reports and + /// metrics. #[serde(default)] pub sentry_send_server_name: bool, - /// Performance monitoring/tracing sample rate for Sentry.io + /// Performance monitoring/tracing sample rate for Sentry.io. /// /// Note that too high values may impact performance, and can be disabled by /// setting it to 0.0 (0%) This value is read as a percentage to Sentry, @@ -1506,8 +1545,8 @@ pub struct Config { #[serde(default)] pub sentry_attach_stacktrace: bool, - /// Send panics to sentry. This is true by default, but sentry has to be - /// enabled. The global "sentry" config option must be enabled to send any + /// Send panics to Sentry. This is true by default, but Sentry has to be + /// enabled. The global `sentry` config option must be enabled to send any /// data. #[serde(default = "true_fn")] pub sentry_send_panic: bool, @@ -1526,7 +1565,9 @@ pub struct Config { pub sentry_filter: String, /// Enable the tokio-console. This option is only relevant to developers. - /// See https://conduwuit.puppyirl.gay/development.html#debugging-with-tokio-console for more information. + /// + /// For more information, see: + /// https://conduwuit.puppyirl.gay/development.html#debugging-with-tokio-console #[serde(default)] pub tokio_console: bool, @@ -1534,8 +1575,8 @@ pub struct Config { pub test: BTreeSet, /// Controls whether admin room notices like account registrations, password - /// changes, account deactivations, room directory publications, etc will - /// be sent to the admin room. Update notices and normal admin command + /// changes, account deactivations, room directory publications, etc will be + /// sent to the admin room. Update notices and normal admin command /// responses will still be sent. #[serde(default = "true_fn")] pub admin_room_notices: bool, From 14341bb906c0428269f59aae6c10071da55d7608 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 18 Dec 2024 22:54:16 +0000 Subject: [PATCH 1266/2091] simplify and improve db iter State init interface Signed-off-by: Jason Volk --- src/database/pool.rs | 23 ----------------------- src/database/stream.rs | 14 ++++++++++++-- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/src/database/pool.rs b/src/database/pool.rs index 28eb38bd..4f018a38 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -273,29 +273,6 @@ fn handle_iter(&self, mut cmd: Seek) { let _chan_sent = chan_result.is_ok(); } -#[implement(Pool)] -#[tracing::instrument( - name = "seek", - level = "trace", - skip_all, - fields(%cmd.map), -)] -fn _handle_seek(&self, mut cmd: Seek) { - let chan = cmd.res.take().expect("missing result channel"); - - if chan.is_canceled() { - return; - } - - match cmd.dir { - | Direction::Forward => cmd.state.seek_fwd(), - | Direction::Reverse => cmd.state.seek_rev(), - }; - - let chan_result = chan.send(into_send_seek(cmd.state)); - let _chan_sent = chan_result.is_ok(); -} - #[implement(Pool)] #[tracing::instrument( name = "get", diff --git a/src/database/stream.rs b/src/database/stream.rs index f849d08f..f8e6733d 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -56,20 +56,30 @@ impl<'a> State<'a> { } pub(super) fn init_fwd(mut self, from: From<'_>) -> Self { + debug_assert!(self.init, "init must be set to make this call"); + debug_assert!(!self.seek, "seek must not be set to make this call"); + if let Some(key) = from { self.inner.seek(key); - self.seek = true; + } else { + self.inner.seek_to_first(); } + self.seek = true; self } pub(super) fn init_rev(mut self, from: From<'_>) -> Self { + debug_assert!(self.init, "init must be set to make this call"); + debug_assert!(!self.seek, "seek must not be set to make this call"); + if let Some(key) = from { self.inner.seek_for_prev(key); - self.seek = true; + } else { + self.inner.seek_to_last(); } + self.seek = true; self } From 98e6c81e4962791a71ec479cf4f43c63bd119e13 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 18 Dec 2024 22:56:53 +0000 Subject: [PATCH 1267/2091] offload remaining db iterator initial seeks on cache miss consume task budget on cache hit Signed-off-by: Jason Volk --- src/database/map/count.rs | 4 +- src/database/map/keys.rs | 45 +++++++++++++++++---- src/database/map/keys_prefix.rs | 6 +-- src/database/map/rev_keys.rs | 45 +++++++++++++++++---- src/database/map/rev_keys_prefix.rs | 6 +-- src/database/map/rev_stream.rs | 56 ++++++++++++++++++++------- src/database/map/rev_stream_from.rs | 13 ++++--- src/database/map/rev_stream_prefix.rs | 6 +-- src/database/map/stream.rs | 56 ++++++++++++++++++++------- src/database/map/stream_from.rs | 13 ++++--- src/database/map/stream_prefix.rs | 6 +-- src/database/stream.rs | 2 - src/database/stream/items.rs | 18 ++------- src/database/stream/items_rev.rs | 18 ++------- src/database/stream/keys.rs | 18 ++------- src/database/stream/keys_rev.rs | 18 ++------- 16 files changed, 199 insertions(+), 131 deletions(-) diff --git a/src/database/map/count.rs b/src/database/map/count.rs index b9b34613..22b298b9 100644 --- a/src/database/map/count.rs +++ b/src/database/map/count.rs @@ -7,7 +7,9 @@ use serde::Serialize; /// Count the total number of entries in the map. #[implement(super::Map)] #[inline] -pub fn count(&self) -> impl Future + Send + '_ { self.raw_keys().count() } +pub fn count(self: &Arc) -> impl Future + Send + '_ { + self.raw_keys().count() +} /// Count the number of entries in the map starting from a lower-bound. /// diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs index 3ab5bacc..7d09f3da 100644 --- a/src/database/map/keys.rs +++ b/src/database/map/keys.rs @@ -1,11 +1,16 @@ -use conduwuit::{implement, Result}; -use futures::{Stream, StreamExt}; -use serde::Deserialize; +use std::sync::Arc; -use crate::{keyval, keyval::Key, stream, stream::Cursor}; +use conduwuit::{implement, Result}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use rocksdb::Direction; +use serde::Deserialize; +use tokio::task; + +use super::stream::is_cached; +use crate::{keyval, keyval::Key, stream}; #[implement(super::Map)] -pub fn keys<'a, K>(&'a self) -> impl Stream>> + Send +pub fn keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send where K: Deserialize<'a> + Send, { @@ -14,7 +19,33 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn raw_keys(&self) -> impl Stream>> + Send { +pub fn raw_keys(self: &Arc) -> impl Stream>> + Send { + use crate::pool::Seek; + let opts = super::iter_options_default(); - stream::Keys::new(&self.db, &self.cf, opts).init(None) + let state = stream::State::new(&self.db, &self.cf, opts); + if is_cached(self) { + let state = state.init_fwd(None); + return task::consume_budget() + .map(move |()| stream::Keys::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Forward, + state: crate::pool::into_send_seek(state), + key: None, + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() } diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index 32a1f04c..28bc7ccd 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -1,11 +1,7 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduwuit::{implement, Result}; -use futures::{ - future, - stream::{Stream, StreamExt}, - TryStreamExt, -}; +use futures::{future, Stream, StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use crate::keyval::{result_deserialize_key, serialize_key, Key}; diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index 7eb4ce63..0ca6ad0f 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -1,11 +1,16 @@ -use conduwuit::{implement, Result}; -use futures::{Stream, StreamExt}; -use serde::Deserialize; +use std::sync::Arc; -use crate::{keyval, keyval::Key, stream, stream::Cursor}; +use conduwuit::{implement, Result}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use rocksdb::Direction; +use serde::Deserialize; +use tokio::task; + +use super::rev_stream::is_cached; +use crate::{keyval, keyval::Key, stream}; #[implement(super::Map)] -pub fn rev_keys<'a, K>(&'a self) -> impl Stream>> + Send +pub fn rev_keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send where K: Deserialize<'a> + Send, { @@ -14,7 +19,33 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_raw_keys(&self) -> impl Stream>> + Send { +pub fn rev_raw_keys(self: &Arc) -> impl Stream>> + Send { + use crate::pool::Seek; + let opts = super::iter_options_default(); - stream::KeysRev::new(&self.db, &self.cf, opts).init(None) + let state = stream::State::new(&self.db, &self.cf, opts); + if is_cached(self) { + let state = state.init_rev(None); + return task::consume_budget() + .map(move |()| stream::KeysRev::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + } + + let seek = Seek { + map: self.clone(), + dir: Direction::Reverse, + state: crate::pool::into_send_seek(state), + key: None, + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() } diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index 9fda49a0..fb29acaf 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -1,11 +1,7 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduwuit::{implement, Result}; -use futures::{ - future, - stream::{Stream, StreamExt}, - TryStreamExt, -}; +use futures::{future, Stream, StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use crate::keyval::{result_deserialize_key, serialize_key, Key}; diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index 7f58582f..d882dd91 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -1,14 +1,20 @@ -use conduwuit::{implement, Result}; -use futures::stream::{Stream, StreamExt}; -use serde::Deserialize; +use std::sync::Arc; -use crate::{keyval, keyval::KeyVal, stream, stream::Cursor}; +use conduwuit::{implement, Result}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use rocksdb::Direction; +use serde::Deserialize; +use tokio::task; + +use crate::{keyval, keyval::KeyVal, stream}; /// Iterate key-value entries in the map from the end. /// /// - Result is deserialized #[implement(super::Map)] -pub fn rev_stream<'a, K, V>(&'a self) -> impl Stream>> + Send +pub fn rev_stream<'a, K, V>( + self: &'a Arc, +) -> impl Stream>> + Send where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, @@ -22,9 +28,35 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn rev_raw_stream(&self) -> impl Stream>> + Send { - let opts = super::iter_options_default(); - stream::ItemsRev::new(&self.db, &self.cf, opts).init(None) +pub fn rev_raw_stream(self: &Arc) -> impl Stream>> + Send { + use crate::pool::Seek; + + let opts = super::read_options_default(); + let state = stream::State::new(&self.db, &self.cf, opts); + if is_cached(self) { + let state = state.init_rev(None); + return task::consume_budget() + .map(move |()| stream::ItemsRev::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + }; + + let seek = Seek { + map: self.clone(), + dir: Direction::Reverse, + state: crate::pool::into_send_seek(state), + key: None, + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() } #[tracing::instrument( @@ -33,13 +65,9 @@ pub fn rev_raw_stream(&self) -> impl Stream>> + Send { skip_all, fields(%map), )] -pub(super) fn _is_cached

    (map: &super::Map) -> bool -where - P: AsRef<[u8]> + ?Sized, -{ +pub(super) fn is_cached(map: &super::Map) -> bool { let opts = super::cache_read_options_default(); - let mut state = stream::State::new(&map.db, &map.cf, opts); + let state = stream::State::new(&map.db, &map.cf, opts).init_rev(None); - state.seek_rev(); !state.is_incomplete() } diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index d166aa0f..72fc739c 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -1,12 +1,10 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduwuit::{implement, Result}; -use futures::{ - stream::{Stream, StreamExt}, - FutureExt, TryFutureExt, TryStreamExt, -}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; +use tokio::task; use crate::{ keyval::{result_deserialize, serialize_key, KeyVal}, @@ -85,7 +83,12 @@ where let opts = super::iter_options_default(); let state = stream::State::new(&self.db, &self.cf, opts); if is_cached(self, from) { - return stream::ItemsRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed(); + let state = state.init_rev(from.as_ref().into()); + return task::consume_budget() + .map(move |()| stream::ItemsRev::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); }; let seek = Seek { diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index 857aa3a5..22a2ce53 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -1,11 +1,7 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduwuit::{implement, Result}; -use futures::{ - future, - stream::{Stream, StreamExt}, - TryStreamExt, -}; +use futures::{future, Stream, StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use crate::keyval::{result_deserialize, serialize_key, KeyVal}; diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index 1a90b8fb..11b0676c 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -1,14 +1,20 @@ -use conduwuit::{implement, Result}; -use futures::stream::{Stream, StreamExt}; -use serde::Deserialize; +use std::sync::Arc; -use crate::{keyval, keyval::KeyVal, stream, stream::Cursor}; +use conduwuit::{implement, Result}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use rocksdb::Direction; +use serde::Deserialize; +use tokio::task; + +use crate::{keyval, keyval::KeyVal, stream}; /// Iterate key-value entries in the map from the beginning. /// /// - Result is deserialized #[implement(super::Map)] -pub fn stream<'a, K, V>(&'a self) -> impl Stream>> + Send +pub fn stream<'a, K, V>( + self: &'a Arc, +) -> impl Stream>> + Send where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, @@ -21,9 +27,35 @@ where /// - Result is raw #[implement(super::Map)] #[tracing::instrument(skip(self), fields(%self), level = "trace")] -pub fn raw_stream(&self) -> impl Stream>> + Send { - let opts = super::iter_options_default(); - stream::Items::new(&self.db, &self.cf, opts).init(None) +pub fn raw_stream(self: &Arc) -> impl Stream>> + Send { + use crate::pool::Seek; + + let opts = super::read_options_default(); + let state = stream::State::new(&self.db, &self.cf, opts); + if is_cached(self) { + let state = state.init_fwd(None); + return task::consume_budget() + .map(move |()| stream::Items::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); + }; + + let seek = Seek { + map: self.clone(), + dir: Direction::Forward, + state: crate::pool::into_send_seek(state), + key: None, + res: None, + }; + + self.db + .pool + .execute_iter(seek) + .ok_into::>() + .into_stream() + .try_flatten() + .boxed() } #[tracing::instrument( @@ -32,13 +64,9 @@ pub fn raw_stream(&self) -> impl Stream>> + Send { skip_all, fields(%map), )] -pub(super) fn _is_cached

    (map: &super::Map) -> bool -where - P: AsRef<[u8]> + ?Sized, -{ +pub(super) fn is_cached(map: &super::Map) -> bool { let opts = super::cache_read_options_default(); - let mut state = stream::State::new(&map.db, &map.cf, opts); + let state = stream::State::new(&map.db, &map.cf, opts).init_fwd(None); - state.seek_fwd(); !state.is_incomplete() } diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 107ce4b1..79ea8f51 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -1,12 +1,10 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduwuit::{implement, Result}; -use futures::{ - stream::{Stream, StreamExt}, - FutureExt, TryFutureExt, TryStreamExt, -}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; +use tokio::task; use crate::{ keyval::{result_deserialize, serialize_key, KeyVal}, @@ -82,7 +80,12 @@ where let opts = super::read_options_default(); let state = stream::State::new(&self.db, &self.cf, opts); if is_cached(self, from) { - return stream::Items::<'_>::from(state.init_fwd(from.as_ref().into())).boxed(); + let state = state.init_fwd(from.as_ref().into()); + return task::consume_budget() + .map(move |()| stream::Items::<'_>::from(state)) + .into_stream() + .flatten() + .boxed(); }; let seek = Seek { diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index a05e2fc5..adacfc81 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -1,11 +1,7 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduwuit::{implement, Result}; -use futures::{ - future, - stream::{Stream, StreamExt}, - TryStreamExt, -}; +use futures::{future, Stream, StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use crate::keyval::{result_deserialize, serialize_key, KeyVal}; diff --git a/src/database/stream.rs b/src/database/stream.rs index f8e6733d..d7cb16c6 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -29,8 +29,6 @@ pub(crate) trait Cursor<'a, T> { fn seek(&mut self); - fn init(self, from: From<'a>) -> Self; - fn get(&self) -> Option> { self.fetch() .map(Ok) diff --git a/src/database/stream/items.rs b/src/database/stream/items.rs index 2a38d97e..cd81b4a0 100644 --- a/src/database/stream/items.rs +++ b/src/database/stream/items.rs @@ -1,4 +1,4 @@ -use std::{convert, pin::Pin, sync::Arc}; +use std::pin::Pin; use conduwuit::Result; use futures::{ @@ -6,22 +6,15 @@ use futures::{ task::{Context, Poll}, Stream, }; -use rocksdb::{ColumnFamily, ReadOptions}; -use super::{keyval_longevity, Cursor, From, State}; -use crate::{keyval::KeyVal, Engine}; +use super::{keyval_longevity, Cursor, State}; +use crate::keyval::KeyVal; pub(crate) struct Items<'a> { state: State<'a>, } -impl<'a> Items<'a> { - pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { - Self { state: State::new(db, cf, opts) } - } -} - -impl<'a> convert::From> for Items<'a> { +impl<'a> From> for Items<'a> { fn from(state: State<'a>) -> Self { Self { state } } } @@ -32,9 +25,6 @@ impl<'a> Cursor<'a, KeyVal<'a>> for Items<'a> { #[inline] fn seek(&mut self) { self.state.seek_fwd(); } - - #[inline] - fn init(self, from: From<'a>) -> Self { Self { state: self.state.init_fwd(from) } } } impl<'a> Stream for Items<'a> { diff --git a/src/database/stream/items_rev.rs b/src/database/stream/items_rev.rs index c3a6cc7f..c6cf9b53 100644 --- a/src/database/stream/items_rev.rs +++ b/src/database/stream/items_rev.rs @@ -1,4 +1,4 @@ -use std::{convert, pin::Pin, sync::Arc}; +use std::pin::Pin; use conduwuit::Result; use futures::{ @@ -6,22 +6,15 @@ use futures::{ task::{Context, Poll}, Stream, }; -use rocksdb::{ColumnFamily, ReadOptions}; -use super::{keyval_longevity, Cursor, From, State}; -use crate::{keyval::KeyVal, Engine}; +use super::{keyval_longevity, Cursor, State}; +use crate::keyval::KeyVal; pub(crate) struct ItemsRev<'a> { state: State<'a>, } -impl<'a> ItemsRev<'a> { - pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { - Self { state: State::new(db, cf, opts) } - } -} - -impl<'a> convert::From> for ItemsRev<'a> { +impl<'a> From> for ItemsRev<'a> { fn from(state: State<'a>) -> Self { Self { state } } } @@ -32,9 +25,6 @@ impl<'a> Cursor<'a, KeyVal<'a>> for ItemsRev<'a> { #[inline] fn seek(&mut self) { self.state.seek_rev(); } - - #[inline] - fn init(self, from: From<'a>) -> Self { Self { state: self.state.init_rev(from) } } } impl<'a> Stream for ItemsRev<'a> { diff --git a/src/database/stream/keys.rs b/src/database/stream/keys.rs index 0696781d..9bf27507 100644 --- a/src/database/stream/keys.rs +++ b/src/database/stream/keys.rs @@ -1,4 +1,4 @@ -use std::{convert, pin::Pin, sync::Arc}; +use std::pin::Pin; use conduwuit::Result; use futures::{ @@ -6,22 +6,15 @@ use futures::{ task::{Context, Poll}, Stream, }; -use rocksdb::{ColumnFamily, ReadOptions}; -use super::{slice_longevity, Cursor, From, State}; -use crate::{keyval::Key, Engine}; +use super::{slice_longevity, Cursor, State}; +use crate::keyval::Key; pub(crate) struct Keys<'a> { state: State<'a>, } -impl<'a> Keys<'a> { - pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { - Self { state: State::new(db, cf, opts) } - } -} - -impl<'a> convert::From> for Keys<'a> { +impl<'a> From> for Keys<'a> { fn from(state: State<'a>) -> Self { Self { state } } } @@ -33,9 +26,6 @@ impl<'a> Cursor<'a, Key<'a>> for Keys<'a> { #[inline] fn seek(&mut self) { self.state.seek_fwd(); } - - #[inline] - fn init(self, from: From<'a>) -> Self { Self { state: self.state.init_fwd(from) } } } impl<'a> Stream for Keys<'a> { diff --git a/src/database/stream/keys_rev.rs b/src/database/stream/keys_rev.rs index 42706d9f..8657df0f 100644 --- a/src/database/stream/keys_rev.rs +++ b/src/database/stream/keys_rev.rs @@ -1,4 +1,4 @@ -use std::{convert, pin::Pin, sync::Arc}; +use std::pin::Pin; use conduwuit::Result; use futures::{ @@ -6,22 +6,15 @@ use futures::{ task::{Context, Poll}, Stream, }; -use rocksdb::{ColumnFamily, ReadOptions}; -use super::{slice_longevity, Cursor, From, State}; -use crate::{keyval::Key, Engine}; +use super::{slice_longevity, Cursor, State}; +use crate::keyval::Key; pub(crate) struct KeysRev<'a> { state: State<'a>, } -impl<'a> KeysRev<'a> { - pub(crate) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { - Self { state: State::new(db, cf, opts) } - } -} - -impl<'a> convert::From> for KeysRev<'a> { +impl<'a> From> for KeysRev<'a> { fn from(state: State<'a>) -> Self { Self { state } } } @@ -33,9 +26,6 @@ impl<'a> Cursor<'a, Key<'a>> for KeysRev<'a> { #[inline] fn seek(&mut self) { self.state.seek_rev(); } - - #[inline] - fn init(self, from: From<'a>) -> Self { Self { state: self.state.init_rev(from) } } } impl<'a> Stream for KeysRev<'a> { From af3d6a2e372b7054126d8f31e34eb42d51aa8829 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 18 Dec 2024 03:34:56 +0000 Subject: [PATCH 1268/2091] shard sender into multiple task workers by destination hash rename Destination::Normal variant tracing instruments Signed-off-by: Jason Volk --- conduwuit-example.toml | 7 ++ src/admin/query/sending.rs | 4 +- src/core/config/mod.rs | 9 ++ src/service/resolver/actual.rs | 3 +- src/service/sending/data.rs | 2 +- src/service/sending/dest.rs | 4 +- src/service/sending/mod.rs | 119 ++++++++++++++---- src/service/sending/send.rs | 5 +- src/service/sending/sender.rs | 217 +++++++++++++++++++++++---------- 9 files changed, 275 insertions(+), 95 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index a82d8f69..3669961a 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1410,6 +1410,13 @@ # #db_pool_queue_size = 256 +# Number of sender task workers; determines sender parallelism. Default is +# '0' which means the value is determined internally, likely matching the +# number of tokio worker-threads or number of cores, etc. Override by +# setting a non-zero value. +# +#sender_workers = 0 + [global.tls] # Path to a valid TLS certificate file. diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 696067b7..3edbbe87 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -113,7 +113,7 @@ pub(super) async fn process( | (None, Some(server_name), None, None) => services .sending .db - .queued_requests(&Destination::Normal(server_name.into())), + .queued_requests(&Destination::Federation(server_name.into())), | (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { return Ok(RoomMessageEventContent::text_plain( @@ -183,7 +183,7 @@ pub(super) async fn process( | (None, Some(server_name), None, None) => services .sending .db - .active_requests_for(&Destination::Normal(server_name.into())), + .active_requests_for(&Destination::Federation(server_name.into())), | (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { return Ok(RoomMessageEventContent::text_plain( diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 8fd5621f..23feb0ca 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1598,6 +1598,15 @@ pub struct Config { #[serde(default = "default_db_pool_queue_size")] pub db_pool_queue_size: usize, + /// Number of sender task workers; determines sender parallelism. Default is + /// '0' which means the value is determined internally, likely matching the + /// number of tokio worker-threads or number of cores, etc. Override by + /// setting a non-zero value. + /// + /// default: 0 + #[serde(default)] + pub sender_workers: usize, + #[serde(flatten)] #[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 63de6539..3f609b95 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -4,6 +4,7 @@ use std::{ }; use conduwuit::{debug, debug_error, debug_info, debug_warn, err, error, trace, Err, Result}; +use futures::FutureExt; use hickory_resolver::error::ResolveError; use ipaddress::IPAddress; use ruma::ServerName; @@ -32,7 +33,7 @@ impl super::Service { (result, true) } else { self.validate_dest(server_name)?; - (self.resolve_actual_dest(server_name, true).await?, false) + (self.resolve_actual_dest(server_name, true).boxed().await?, false) }; let CachedDest { dest, host, .. } = result; diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index ac06424f..a699b8ee 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -246,7 +246,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se })?; ( - Destination::Normal(ServerName::parse(server).map_err(|_| { + Destination::Federation(ServerName::parse(server).map_err(|_| { Error::bad_database("Invalid server string in server_currenttransaction") })?), if value.is_empty() { diff --git a/src/service/sending/dest.rs b/src/service/sending/dest.rs index 2c6063cc..4099d372 100644 --- a/src/service/sending/dest.rs +++ b/src/service/sending/dest.rs @@ -7,14 +7,14 @@ use ruma::{OwnedServerName, OwnedUserId}; pub enum Destination { Appservice(String), Push(OwnedUserId, String), // user and pushkey - Normal(OwnedServerName), + Federation(OwnedServerName), } #[implement(Destination)] #[must_use] pub(super) fn get_prefix(&self) -> Vec { match self { - | Self::Normal(server) => { + | Self::Federation(server) => { let len = server.as_bytes().len().saturating_add(1); let mut p = Vec::with_capacity(len); diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 2038f4eb..5ccba249 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -4,20 +4,25 @@ mod dest; mod send; mod sender; -use std::{fmt::Debug, iter::once, sync::Arc}; +use std::{ + fmt::Debug, + hash::{DefaultHasher, Hash, Hasher}, + iter::once, + sync::Arc, +}; use async_trait::async_trait; use conduwuit::{ - debug_warn, err, - utils::{ReadyExt, TryReadyExt}, + debug, debug_warn, err, error, + utils::{available_parallelism, math::usize_from_u64_truncated, ReadyExt, TryReadyExt}, warn, Result, Server, }; -use futures::{Stream, StreamExt}; +use futures::{FutureExt, Stream, StreamExt}; use ruma::{ api::{appservice::Registration, OutgoingRequest}, RoomId, ServerName, UserId, }; -use tokio::sync::Mutex; +use tokio::task::JoinSet; use self::data::Data; pub use self::{ @@ -30,11 +35,10 @@ use crate::{ }; pub struct Service { + pub db: Data, server: Arc, services: Services, - pub db: Data, - sender: loole::Sender, - receiver: Mutex>, + channels: Vec<(loole::Sender, loole::Receiver)>, } struct Services { @@ -72,8 +76,9 @@ pub enum SendingEvent { #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { - let (sender, receiver) = loole::unbounded(); + let num_senders = num_senders(&args); Ok(Arc::new(Self { + db: Data::new(&args), server: args.server.clone(), services: Services { client: args.depend::("client"), @@ -91,20 +96,41 @@ impl crate::Service for Service { pusher: args.depend::("pusher"), server_keys: args.depend::("server_keys"), }, - db: Data::new(&args), - sender, - receiver: Mutex::new(receiver), + channels: (0..num_senders).map(|_| loole::unbounded()).collect(), })) } - async fn worker(self: Arc) -> Result<()> { - // trait impl can't be split between files so this just glues to mod sender - self.sender().await + async fn worker(self: Arc) -> Result { + let mut senders = + self.channels + .iter() + .enumerate() + .fold(JoinSet::new(), |mut joinset, (id, _)| { + let self_ = self.clone(); + let runtime = self.server.runtime(); + let _abort = joinset.spawn_on(self_.sender(id).boxed(), runtime); + joinset + }); + + while let Some(ret) = senders.join_next_with_id().await { + match ret { + | Ok((id, _)) => { + debug!(?id, "sender worker finished"); + }, + | Err(error) => { + error!(id = ?error.id(), ?error, "sender worker finished"); + }, + }; + } + + Ok(()) } fn interrupt(&self) { - if !self.sender.is_closed() { - self.sender.close(); + for (sender, _) in &self.channels { + if !sender.is_closed() { + sender.close(); + } } } @@ -157,7 +183,7 @@ impl Service { let _cork = self.db.db.cork(); let requests = servers .map(|server| { - (Destination::Normal(server.into()), SendingEvent::Pdu(pdu_id.to_owned())) + (Destination::Federation(server.into()), SendingEvent::Pdu(pdu_id.to_owned())) }) .collect::>() .await; @@ -173,7 +199,7 @@ impl Service { #[tracing::instrument(skip(self, server, serialized), level = "debug")] pub fn send_edu_server(&self, server: &ServerName, serialized: Vec) -> Result<()> { - let dest = Destination::Normal(server.to_owned()); + let dest = Destination::Federation(server.to_owned()); let event = SendingEvent::Edu(serialized); let _cork = self.db.db.cork(); let keys = self.db.queue_requests(once((&event, &dest))); @@ -203,7 +229,10 @@ impl Service { let _cork = self.db.db.cork(); let requests = servers .map(|server| { - (Destination::Normal(server.to_owned()), SendingEvent::Edu(serialized.clone())) + ( + Destination::Federation(server.to_owned()), + SendingEvent::Edu(serialized.clone()), + ) }) .collect::>() .await; @@ -235,7 +264,7 @@ impl Service { { servers .map(ToOwned::to_owned) - .map(Destination::Normal) + .map(Destination::Federation) .map(Ok) .ready_try_for_each(|dest| { self.dispatch(Msg { @@ -327,9 +356,49 @@ impl Service { } } - fn dispatch(&self, msg: Msg) -> Result<()> { - debug_assert!(!self.sender.is_full(), "channel full"); - debug_assert!(!self.sender.is_closed(), "channel closed"); - self.sender.send(msg).map_err(|e| err!("{e}")) + fn dispatch(&self, msg: Msg) -> Result { + let shard = self.shard_id(&msg.dest); + let sender = &self + .channels + .get(shard) + .expect("missing sender worker channels") + .0; + + debug_assert!(!sender.is_full(), "channel full"); + debug_assert!(!sender.is_closed(), "channel closed"); + sender.send(msg).map_err(|e| err!("{e}")) + } + + pub(super) fn shard_id(&self, dest: &Destination) -> usize { + if self.channels.len() <= 1 { + return 0; + } + + let mut hash = DefaultHasher::default(); + dest.hash(&mut hash); + + let hash: u64 = hash.finish(); + let hash = usize_from_u64_truncated(hash); + + let chans = self.channels.len().max(1); + hash.overflowing_rem(chans).0 } } + +fn num_senders(args: &crate::Args<'_>) -> usize { + const MIN_SENDERS: usize = 1; + // Limit the number of senders to the number of workers threads or number of + // cores, conservatively. + let max_senders = args + .server + .metrics + .num_workers() + .min(available_parallelism()); + + // If the user doesn't override the default 0, this is intended to then default + // to 1 for now as multiple senders is experimental. + args.server + .config + .sender_workers + .clamp(MIN_SENDERS, max_senders) +} diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 81467c16..3d13a3b0 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -24,7 +24,10 @@ use crate::{ }; impl super::Service { - #[tracing::instrument(skip_all, level = "debug")] + #[tracing::instrument( + level = "debug" + skip(self, client, request), + )] pub async fn send( &self, client: &Client, diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index f6b83e83..4e806ce8 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -1,7 +1,10 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, - sync::atomic::{AtomicU64, AtomicUsize, Ordering}, + sync::{ + atomic::{AtomicU64, AtomicUsize, Ordering}, + Arc, + }, time::{Duration, Instant}, }; @@ -66,29 +69,56 @@ pub const PDU_LIMIT: usize = 50; pub const EDU_LIMIT: usize = 100; impl Service { - #[tracing::instrument(skip_all, level = "debug")] - pub(super) async fn sender(&self) -> Result<()> { + #[tracing::instrument(skip(self), level = "debug")] + pub(super) async fn sender(self: Arc, id: usize) -> Result { let mut statuses: CurTransactionStatus = CurTransactionStatus::new(); let mut futures: SendingFutures<'_> = FuturesUnordered::new(); - let receiver = self.receiver.lock().await; - self.initial_requests(&mut futures, &mut statuses).await; - while !receiver.is_closed() { - tokio::select! { - request = receiver.recv_async() => match request { - Ok(request) => self.handle_request(request, &mut futures, &mut statuses).await, - Err(_) => break, - }, - Some(response) = futures.next() => { - self.handle_response(response, &mut futures, &mut statuses).await; - }, - } - } - self.finish_responses(&mut futures).await; + self.startup_netburst(id, &mut futures, &mut statuses) + .boxed() + .await; + + self.work_loop(id, &mut futures, &mut statuses).await; + + self.finish_responses(&mut futures).boxed().await; Ok(()) } + #[tracing::instrument( + name = "work", + level = "trace" + skip_all, + fields( + futures = %futures.len(), + statuses = %statuses.len(), + ), + )] + async fn work_loop<'a>( + &'a self, + id: usize, + futures: &mut SendingFutures<'a>, + statuses: &mut CurTransactionStatus, + ) { + let receiver = self + .channels + .get(id) + .map(|(_, receiver)| receiver.clone()) + .expect("Missing channel for sender worker"); + loop { + tokio::select! { + Some(response) = futures.next() => { + self.handle_response(response, futures, statuses).await; + }, + request = receiver.recv_async() => match request { + Ok(request) => self.handle_request(request, futures, statuses).await, + Err(_) => return, + }, + } + } + } + + #[tracing::instrument(name = "response", level = "debug", skip_all)] async fn handle_response<'a>( &'a self, response: SendingResult, @@ -138,13 +168,14 @@ impl Service { self.db.mark_as_active(new_events.iter()); let new_events_vec = new_events.into_iter().map(|(_, event)| event).collect(); - futures.push(self.send_events(dest.clone(), new_events_vec).boxed()); + futures.push(self.send_events(dest.clone(), new_events_vec)); } else { statuses.remove(dest); } } #[allow(clippy::needless_pass_by_ref_mut)] + #[tracing::instrument(name = "request", level = "debug", skip_all)] async fn handle_request<'a>( &'a self, msg: Msg, @@ -154,13 +185,19 @@ impl Service { let iv = vec![(msg.queue_id, msg.event)]; if let Ok(Some(events)) = self.select_events(&msg.dest, iv, statuses).await { if !events.is_empty() { - futures.push(self.send_events(msg.dest, events).boxed()); + futures.push(self.send_events(msg.dest, events)); } else { statuses.remove(&msg.dest); } } } + #[tracing::instrument( + name = "finish", + level = "info", + skip_all, + fields(futures = %futures.len()), + )] async fn finish_responses<'a>(&'a self, futures: &mut SendingFutures<'a>) { use tokio::{ select, @@ -183,9 +220,16 @@ impl Service { } } + #[tracing::instrument( + name = "netburst", + level = "debug", + skip_all, + fields(futures = %futures.len()), + )] #[allow(clippy::needless_pass_by_ref_mut)] - async fn initial_requests<'a>( + async fn startup_netburst<'a>( &'a self, + id: usize, futures: &mut SendingFutures<'a>, statuses: &mut CurTransactionStatus, ) { @@ -195,6 +239,10 @@ impl Service { let mut active = self.db.active_requests().boxed(); while let Some((key, event, dest)) = active.next().await { + if self.shard_id(&dest) != id { + continue; + } + let entry = txns.entry(dest.clone()).or_default(); if self.server.config.startup_netburst_keep >= 0 && entry.len() >= keep { warn!("Dropping unsent event {dest:?} {:?}", String::from_utf8_lossy(&key)); @@ -207,19 +255,27 @@ impl Service { for (dest, events) in txns { if self.server.config.startup_netburst && !events.is_empty() { statuses.insert(dest.clone(), TransactionStatus::Running); - futures.push(self.send_events(dest.clone(), events).boxed()); + futures.push(self.send_events(dest.clone(), events)); } } } - #[tracing::instrument(skip_all, level = "debug")] + #[tracing::instrument( + name = "select",, + level = "debug", + skip_all, + fields( + ?dest, + new_events = %new_events.len(), + ) + )] async fn select_events( &self, dest: &Destination, new_events: Vec, // Events we want to send: event and full key statuses: &mut CurTransactionStatus, ) -> Result>> { - let (allow, retry) = self.select_events_current(dest.clone(), statuses)?; + let (allow, retry) = self.select_events_current(dest, statuses)?; // Nothing can be done for this remote, bail out. if !allow { @@ -249,7 +305,7 @@ impl Service { } // Add EDU's into the transaction - if let Destination::Normal(server_name) = dest { + if let Destination::Federation(server_name) = dest { if let Ok((select_edus, last_count)) = self.select_edus(server_name).await { debug_assert!(select_edus.len() <= EDU_LIMIT, "exceeded edus limit"); events.extend(select_edus.into_iter().map(SendingEvent::Edu)); @@ -260,10 +316,9 @@ impl Service { Ok(Some(events)) } - #[tracing::instrument(skip_all, level = "debug")] fn select_events_current( &self, - dest: Destination, + dest: &Destination, statuses: &mut CurTransactionStatus, ) -> Result<(bool, bool)> { let (mut allow, mut retry) = (true, false); @@ -292,7 +347,11 @@ impl Service { Ok((allow, retry)) } - #[tracing::instrument(skip_all, level = "debug")] + #[tracing::instrument( + name = "edus",, + level = "debug", + skip_all, + )] async fn select_edus(&self, server_name: &ServerName) -> Result<(Vec>, u64)> { // selection window let since = self.db.get_latest_educount(server_name).await; @@ -329,7 +388,12 @@ impl Service { Ok((events, max_edu_count.load(Ordering::Acquire))) } - /// Look for presence + /// Look for device changes + #[tracing::instrument( + name = "device_changes", + level = "trace", + skip(self, server_name, max_edu_count) + )] async fn select_edus_device_changes( &self, server_name: &ServerName, @@ -386,6 +450,11 @@ impl Service { } /// Look for read receipts in this room + #[tracing::instrument( + name = "receipts", + level = "trace", + skip(self, server_name, max_edu_count) + )] async fn select_edus_receipts( &self, server_name: &ServerName, @@ -420,6 +489,7 @@ impl Service { } /// Look for read receipts in this room + #[tracing::instrument(name = "receipts", level = "trace", skip(self, since, max_edu_count))] async fn select_edus_receipts_room( &self, room_id: &RoomId, @@ -484,6 +554,11 @@ impl Service { } /// Look for presence + #[tracing::instrument( + name = "presence", + level = "trace", + skip(self, server_name, max_edu_count) + )] async fn select_edus_presence( &self, server_name: &ServerName, @@ -554,29 +629,33 @@ impl Service { Some(presence_content) } - async fn send_events(&self, dest: Destination, events: Vec) -> SendingResult { + fn send_events(&self, dest: Destination, events: Vec) -> SendingFuture<'_> { //debug_assert!(!events.is_empty(), "sending empty transaction"); match dest { - | Destination::Normal(ref server) => - self.send_events_dest_normal(&dest, server, events).await, - | Destination::Appservice(ref id) => - self.send_events_dest_appservice(&dest, id, events).await, - | Destination::Push(ref userid, ref pushkey) => - self.send_events_dest_push(&dest, userid, pushkey, events) - .await, + | Destination::Federation(server) => + self.send_events_dest_federation(server, events).boxed(), + | Destination::Appservice(id) => self.send_events_dest_appservice(id, events).boxed(), + | Destination::Push(user_id, pushkey) => + self.send_events_dest_push(user_id, pushkey, events).boxed(), } } - #[tracing::instrument(skip(self, dest, events), name = "appservice")] + #[tracing::instrument( + name = "appservice", + level = "debug", + skip(self, events), + fields( + events = %events.len(), + ), + )] async fn send_events_dest_appservice( &self, - dest: &Destination, - id: &str, + id: String, events: Vec, ) -> SendingResult { - let Some(appservice) = self.services.appservice.get_registration(id).await else { + let Some(appservice) = self.services.appservice.get_registration(&id).await else { return Err(( - dest.clone(), + Destination::Appservice(id.clone()), err!(Database(warn!(?id, "Missing appservice registration"))), )); }; @@ -633,23 +712,29 @@ impl Service { ) .await { - | Ok(_) => Ok(dest.clone()), - | Err(e) => Err((dest.clone(), e)), + | Ok(_) => Ok(Destination::Appservice(id)), + | Err(e) => Err((Destination::Appservice(id), e)), } } - #[tracing::instrument(skip(self, dest, events), name = "push")] + #[tracing::instrument( + name = "push", + level = "info", + skip(self, events), + fields( + events = %events.len(), + ), + )] async fn send_events_dest_push( &self, - dest: &Destination, - userid: &OwnedUserId, - pushkey: &str, + user_id: OwnedUserId, + pushkey: String, events: Vec, ) -> SendingResult { - let Ok(pusher) = self.services.pusher.get_pusher(userid, pushkey).await else { + let Ok(pusher) = self.services.pusher.get_pusher(&user_id, &pushkey).await else { return Err(( - dest.clone(), - err!(Database(error!(?userid, ?pushkey, "Missing pusher"))), + Destination::Push(user_id.clone(), pushkey.clone()), + err!(Database(error!(?user_id, ?pushkey, "Missing pusher"))), )); }; @@ -677,17 +762,17 @@ impl Service { let rules_for_user = self .services .account_data - .get_global(userid, GlobalAccountDataEventType::PushRules) + .get_global(&user_id, GlobalAccountDataEventType::PushRules) .await .map_or_else( - |_| push::Ruleset::server_default(userid), + |_| push::Ruleset::server_default(&user_id), |ev: PushRulesEvent| ev.content.global, ); let unread: UInt = self .services .user - .notification_count(userid, &pdu.room_id) + .notification_count(&user_id, &pdu.room_id) .await .try_into() .expect("notification count can't go that high"); @@ -695,19 +780,25 @@ impl Service { let _response = self .services .pusher - .send_push_notice(userid, unread, &pusher, rules_for_user, &pdu) + .send_push_notice(&user_id, unread, &pusher, rules_for_user, &pdu) .await - .map_err(|e| (dest.clone(), e)); + .map_err(|e| (Destination::Push(user_id.clone(), pushkey.clone()), e)); } - Ok(dest.clone()) + Ok(Destination::Push(user_id, pushkey)) } - #[tracing::instrument(skip(self, dest, events), name = "", level = "debug")] - async fn send_events_dest_normal( + #[tracing::instrument( + name = "fed", + level = "debug", + skip(self, events), + fields( + events = %events.len(), + ), + )] + async fn send_events_dest_federation( &self, - dest: &Destination, - server: &OwnedServerName, + server: OwnedServerName, events: Vec, ) -> SendingResult { let mut pdu_jsons = Vec::with_capacity( @@ -759,7 +850,7 @@ impl Service { }; let client = &self.services.client.sender; - self.send(client, server, request) + self.send(client, &server, request) .await .inspect(|response| { response @@ -770,8 +861,8 @@ impl Service { |(pdu_id, res)| warn!(%txn_id, %server, "error sending PDU {pdu_id} to remote server: {res:?}"), ); }) - .map(|_| dest.clone()) - .map_err(|e| (dest.clone(), e)) + .map_err(|e| (Destination::Federation(server.clone()), e)) + .map(|_| Destination::Federation(server)) } /// This does not return a full `Pdu` it is only to satisfy ruma's types. From 674acc86571a78ea722a03200babdabb78a21100 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 19 Dec 2024 07:01:15 +0000 Subject: [PATCH 1269/2091] add program options for tokio histogram settings Signed-off-by: Jason Volk --- src/main/clap.rs | 12 ++++++++++++ src/main/runtime.rs | 18 +++++++----------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/main/clap.rs b/src/main/clap.rs index 7e70bd80..28577356 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -38,6 +38,18 @@ pub(crate) struct Args { /// Override the tokio worker_thread count. #[arg(long, hide(true), env = "TOKIO_WORKER_THREADS", default_value = available_parallelism().to_string())] pub(crate) worker_threads: usize, + + /// Set the histogram bucket size, in microseconds (tokio_unstable). Default + /// is 25 microseconds. If the values of the histogram don't approach zero + /// with the exception of the last bucket, try increasing this value to e.g. + /// 50 or 100. Inversely, decrease to 10 etc if the histogram lacks + /// resolution. + #[arg(long, hide(true), env = "CONDUWUIT_RUNTIME_HISTOGRAM_INTERVAL", default_value = "25")] + pub(crate) worker_histogram_interval: u64, + + /// Set the histogram bucket count (tokio_unstable). Default is 20. + #[arg(long, hide(true), env = "CONDUWUIT_RUNTIME_HISTOGRAM_BUCKETS", default_value = "20")] + pub(crate) worker_histogram_buckets: usize, } /// Parse commandline arguments into structured data diff --git a/src/main/runtime.rs b/src/main/runtime.rs index ad0c3cde..6fc405a7 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -35,25 +35,21 @@ pub(super) fn new(args: &Args) -> Result { .on_task_terminate(task_terminate); #[cfg(tokio_unstable)] - enable_histogram(&mut builder); + enable_histogram(&mut builder, args); builder.build().map_err(Into::into) } #[cfg(tokio_unstable)] -fn enable_histogram(builder: &mut Builder) { - use tokio::runtime::{HistogramConfiguration, LogHistogram}; - - let config = LogHistogram::builder() - .min_value(Duration::from_micros(10)) - .max_value(Duration::from_millis(1)) - .max_error(0.5) - .max_buckets(32) - .expect("erroneous histogram configuration"); +fn enable_histogram(builder: &mut Builder, args: &Args) { + use tokio::runtime::HistogramConfiguration; + let buckets = args.worker_histogram_buckets; + let interval = Duration::from_micros(args.worker_histogram_interval); + let linear = HistogramConfiguration::linear(interval, buckets); builder .enable_metrics_poll_time_histogram() - .metrics_poll_time_histogram_configuration(HistogramConfiguration::log(config)); + .metrics_poll_time_histogram_configuration(linear); } #[tracing::instrument( From 503210c3bffbc90a5fbcabf4c12a6e5fa9036d8a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 19 Dec 2024 07:33:29 +0000 Subject: [PATCH 1270/2091] toggle worker_affinity feature from program argument Signed-off-by: Jason Volk --- src/main/clap.rs | 15 ++++++++++++++- src/main/runtime.rs | 19 +++++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/clap.rs b/src/main/clap.rs index 28577356..c447deae 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -2,7 +2,7 @@ use std::path::PathBuf; -use clap::Parser; +use clap::{ArgAction, Parser}; use conduwuit::{ config::{Figment, FigmentValue}, err, toml, @@ -50,6 +50,19 @@ pub(crate) struct Args { /// Set the histogram bucket count (tokio_unstable). Default is 20. #[arg(long, hide(true), env = "CONDUWUIT_RUNTIME_HISTOGRAM_BUCKETS", default_value = "20")] pub(crate) worker_histogram_buckets: usize, + + /// Toggles worker affinity feature. + #[arg( + long, + hide(true), + env = "CONDUWUIT_RUNTIME_WORKER_AFFINITY", + action = ArgAction::Set, + num_args = 0..=1, + require_equals(false), + default_value = "true", + default_missing_value = "true", + )] + pub(crate) worker_affinity: bool, } /// Parse commandline arguments into structured data diff --git a/src/main/runtime.rs b/src/main/runtime.rs index 6fc405a7..ff8ab322 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -1,4 +1,4 @@ -use std::{thread, time::Duration}; +use std::{sync::OnceLock, thread, time::Duration}; use conduwuit::Result; use tokio::runtime::Builder; @@ -12,9 +12,14 @@ const GLOBAL_QUEUE_INTERVAL: u32 = 192; const KERNEL_QUEUE_INTERVAL: u32 = 256; const KERNEL_EVENTS_PER_TICK: usize = 512; -pub(super) fn new(args: &Args) -> Result { - let mut builder = Builder::new_multi_thread(); +static WORKER_AFFINITY: OnceLock = OnceLock::new(); +pub(super) fn new(args: &Args) -> Result { + WORKER_AFFINITY + .set(args.worker_affinity) + .expect("set WORKER_AFFINITY from program argument"); + + let mut builder = Builder::new_multi_thread(); builder .enable_io() .enable_time() @@ -63,7 +68,13 @@ fn enable_histogram(builder: &mut Builder, args: &Args) { )] fn thread_start() { #[cfg(feature = "worker_affinity")] - set_worker_affinity(); + if WORKER_AFFINITY + .get() + .copied() + .expect("WORKER_AFFINITY initialized by runtime::new()") + { + set_worker_affinity(); + } } #[cfg(feature = "worker_affinity")] From aefc4b4e69e17bac99b8608a5c02fbcfd66c8f78 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 19 Dec 2024 09:42:28 +0000 Subject: [PATCH 1271/2091] add more program options for tokio Signed-off-by: Jason Volk --- src/main/clap.rs | 19 ++++++++++++++++++- src/main/runtime.rs | 11 +++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/main/clap.rs b/src/main/clap.rs index c447deae..ad5c815a 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -36,9 +36,26 @@ pub(crate) struct Args { pub(crate) test: Vec, /// Override the tokio worker_thread count. - #[arg(long, hide(true), env = "TOKIO_WORKER_THREADS", default_value = available_parallelism().to_string())] + #[arg( + long, + hide(true), + env = "TOKIO_WORKER_THREADS", + default_value = available_parallelism().to_string(), + )] pub(crate) worker_threads: usize, + /// Override the tokio global_queue_interval. + #[arg(long, hide(true), env = "TOKIO_GLOBAL_QUEUE_INTERVAL", default_value = "192")] + pub(crate) global_event_interval: u32, + + /// Override the tokio event_interval. + #[arg(long, hide(true), env = "TOKIO_EVENT_INTERVAL", default_value = "512")] + pub(crate) kernel_event_interval: u32, + + /// Override the tokio max_io_events_per_tick. + #[arg(long, hide(true), env = "TOKIO_MAX_IO_EVENTS_PER_TICK", default_value = "512")] + pub(crate) kernel_events_per_tick: usize, + /// Set the histogram bucket size, in microseconds (tokio_unstable). Default /// is 25 microseconds. If the values of the histogram don't approach zero /// with the exception of the last bucket, try increasing this value to e.g. diff --git a/src/main/runtime.rs b/src/main/runtime.rs index ff8ab322..e98e73d6 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -8,9 +8,7 @@ use crate::clap::Args; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; const WORKER_KEEPALIVE: u64 = 36; -const GLOBAL_QUEUE_INTERVAL: u32 = 192; -const KERNEL_QUEUE_INTERVAL: u32 = 256; -const KERNEL_EVENTS_PER_TICK: usize = 512; +const MAX_BLOCKING_THREADS: usize = 1024; static WORKER_AFFINITY: OnceLock = OnceLock::new(); @@ -25,10 +23,11 @@ pub(super) fn new(args: &Args) -> Result { .enable_time() .thread_name(WORKER_NAME) .worker_threads(args.worker_threads.max(WORKER_MIN)) + .max_blocking_threads(MAX_BLOCKING_THREADS) .thread_keep_alive(Duration::from_secs(WORKER_KEEPALIVE)) - .max_io_events_per_tick(KERNEL_EVENTS_PER_TICK) - .event_interval(KERNEL_QUEUE_INTERVAL) - .global_queue_interval(GLOBAL_QUEUE_INTERVAL) + .global_queue_interval(args.global_event_interval) + .event_interval(args.kernel_event_interval) + .max_io_events_per_tick(args.kernel_events_per_tick) .on_thread_start(thread_start) .on_thread_stop(thread_stop) .on_thread_unpark(thread_unpark) From a3d87be22ff394fb3060502ac1952883b8b58783 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 21 Dec 2024 21:32:10 +0000 Subject: [PATCH 1272/2091] enable atomic_flush for database columns Signed-off-by: Jason Volk --- src/database/opts.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/database/opts.rs b/src/database/opts.rs index 933830bd..24128f14 100644 --- a/src/database/opts.rs +++ b/src/database/opts.rs @@ -39,8 +39,9 @@ pub(crate) fn db_options( } // IO + opts.set_atomic_flush(true); opts.set_manual_wal_flush(true); - opts.set_enable_pipelined_write(true); + opts.set_enable_pipelined_write(false); if config.rocksdb_direct_io { opts.set_use_direct_reads(true); opts.set_use_direct_io_for_flush_and_compaction(true); From 749f29aaaba1ca117c3cd8a5bb4ebb3a031bd84f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 22 Dec 2024 07:03:32 +0000 Subject: [PATCH 1273/2091] add and/and_then to BoolExt Signed-off-by: Jason Volk --- src/core/utils/bool.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/core/utils/bool.rs b/src/core/utils/bool.rs index b4192daf..b16f63e9 100644 --- a/src/core/utils/bool.rs +++ b/src/core/utils/bool.rs @@ -2,6 +2,10 @@ /// Boolean extensions and chain.starters pub trait BoolExt { + fn and(self, t: Option) -> Option; + + fn and_then Option>(self, f: F) -> Option; + #[must_use] fn clone_or(self, err: T, t: &T) -> T; @@ -39,6 +43,12 @@ pub trait BoolExt { } impl BoolExt for bool { + #[inline] + fn and(self, t: Option) -> Option { self.then_some(t).flatten() } + + #[inline] + fn and_then Option>(self, f: F) -> Option { self.then(f).flatten() } + #[inline] fn clone_or(self, err: T, t: &T) -> T { self.map_or(err, || t.clone()) } From 5fdb8895b1834fa77f59bd0e44a9705bab3288f3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 21 Dec 2024 22:12:42 +0000 Subject: [PATCH 1274/2091] add utils to detect device io characteristics Signed-off-by: Jason Volk --- Cargo.lock | 76 +++++++++++++++++++++-- Cargo.toml | 5 ++ src/core/Cargo.toml | 2 + src/core/utils/mod.rs | 2 +- src/core/utils/sys.rs | 23 +++---- src/core/utils/sys/compute.rs | 74 ++++++++++++++++++++++ src/core/utils/sys/storage.rs | 112 ++++++++++++++++++++++++++++++++++ src/main/Cargo.toml | 5 -- src/main/runtime.rs | 36 +++++------ 9 files changed, 288 insertions(+), 47 deletions(-) create mode 100644 src/core/utils/sys/compute.rs create mode 100644 src/core/utils/sys/storage.rs diff --git a/Cargo.lock b/Cargo.lock index d65ae18f..6522aa55 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -631,7 +631,6 @@ dependencies = [ "conduwuit_service", "console-subscriber", "const-str", - "core_affinity", "hardened_malloc-rs", "log", "opentelemetry", @@ -716,6 +715,7 @@ dependencies = [ "clap", "conduwuit_macros", "const-str", + "core_affinity", "ctor", "cyborgtime", "either", @@ -740,6 +740,7 @@ dependencies = [ "serde_json", "serde_regex", "serde_yaml", + "sysinfo", "thiserror 2.0.7", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -1675,7 +1676,7 @@ checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" dependencies = [ "cfg-if", "libc", - "windows", + "windows 0.52.0", ] [[package]] @@ -4075,6 +4076,18 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "sysinfo" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "948512566b1895f93b1592c7574baeb2de842f224f2aab158799ecadb8ebbb46" +dependencies = [ + "core-foundation-sys", + "libc", + "serde", + "windows 0.57.0", +] + [[package]] name = "tendril" version = "0.4.3" @@ -4932,7 +4945,17 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core", + "windows-core 0.52.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" +dependencies = [ + "windows-core 0.57.0", "windows-targets 0.52.6", ] @@ -4945,17 +4968,60 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result 0.1.2", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "windows-interface" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "windows-registry" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-strings", "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-result" version = "0.2.0" @@ -4971,7 +5037,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-targets 0.52.6", ] diff --git a/Cargo.toml b/Cargo.toml index 15f054bf..689f2b5c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -497,6 +497,11 @@ version = "1.3.0" [workspace.dependencies.core_affinity] version = "0.8.1" +[workspace.dependencies.sysinfo] +version = "0.33.0" +default-features = false +features = ["disk", "serde"] + # # Patches # diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 27c6da52..d249f647 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -64,6 +64,7 @@ chrono.workspace = true clap.workspace = true conduwuit-macros.workspace = true const-str.workspace = true +core_affinity.workspace = true ctor.workspace = true cyborgtime.workspace = true either.workspace = true @@ -86,6 +87,7 @@ serde_json.workspace = true serde_regex.workspace = true serde_yaml.workspace = true serde.workspace = true +sysinfo.workspace = true thiserror.workspace = true tikv-jemallocator.optional = true tikv-jemallocator.workspace = true diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index a9b73fb6..38232820 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -36,7 +36,7 @@ pub use self::{ rand::{shuffle, string as random_string}, stream::{IterStream, ReadyExt, Tools as StreamTools, TryReadyExt}, string::{str_from_bytes, string_from_bytes}, - sys::available_parallelism, + sys::compute::parallelism as available_parallelism, time::{now_millis as millis_since_unix_epoch, timepoint_ago, timepoint_from_now}, }; diff --git a/src/core/utils/sys.rs b/src/core/utils/sys.rs index 05ef12ca..5c5564c4 100644 --- a/src/core/utils/sys.rs +++ b/src/core/utils/sys.rs @@ -1,3 +1,10 @@ +pub mod compute; +pub mod storage; + +use std::path::PathBuf; + +pub use compute::parallelism as available_parallelism; + use crate::{debug, Result}; /// This is needed for opening lots of file descriptors, which tends to @@ -21,18 +28,6 @@ pub fn maximize_fd_limit() -> Result<(), nix::errno::Errno> { Ok(()) } -/// Get the number of threads which could execute in parallel based on the -/// hardware and administrative constraints of this system. This value should be -/// used to hint the size of thread-pools and divide-and-conquer algorithms. -/// -/// * -#[must_use] -pub fn available_parallelism() -> usize { - std::thread::available_parallelism() - .expect("Unable to query for available parallelism.") - .get() -} - /// Return a possibly corrected std::env::current_exe() even if the path is /// marked deleted. /// @@ -40,9 +35,7 @@ pub fn available_parallelism() -> usize { /// This function is declared unsafe because the original result was altered for /// security purposes, and altering it back ignores those urposes and should be /// understood by the user. -pub unsafe fn current_exe() -> Result { - use std::path::PathBuf; - +pub unsafe fn current_exe() -> Result { let exe = std::env::current_exe()?; match exe.to_str() { | None => Ok(exe), diff --git a/src/core/utils/sys/compute.rs b/src/core/utils/sys/compute.rs new file mode 100644 index 00000000..4e9ef743 --- /dev/null +++ b/src/core/utils/sys/compute.rs @@ -0,0 +1,74 @@ +//! System utilities related to compute/processing + +use std::{cell::Cell, sync::LazyLock}; + +use crate::is_equal_to; + +/// The list of cores available to the process (at startup) +static CORES_AVAILABLE: LazyLock = LazyLock::new(|| { + core_affinity::get_core_ids() + .unwrap_or_default() + .into_iter() + .map(|core_id| core_id.id) + .inspect(|&id| assert!(id < 128, "Core ID must be < 128 at least for now")) + .fold(0_u128, |mask, id| mask | (1 << id)) +}); + +thread_local! { + /// Tracks the affinity for this thread. This is updated when affinities + /// are set via our set_affinity() interface. + static CORE_AFFINITY: Cell = Cell::default(); +} + +/// Set the core affinity for this thread. The ID should be listed in +/// CORES_AVAILABLE. Empty input is a no-op; prior affinity unchanged. +pub fn set_affinity(ids: I) +where + I: Iterator, +{ + use core_affinity::{set_for_current, CoreId}; + + let mask: u128 = ids.fold(0, |mask, id| { + debug_assert!(is_core_available(id), "setting affinity to unavailable core"); + set_for_current(CoreId { id }); + mask | (1 << id) + }); + + if mask.count_ones() > 0 { + CORE_AFFINITY.replace(mask); + } +} + +/// Get the core affinity for this thread. +pub fn get_affinity() -> impl Iterator { + (0..128).filter(|&i| ((CORE_AFFINITY.get() & (1 << i)) != 0)) +} + +/// Gets the ID of the nth core available. This bijects our sequence of cores to +/// actual ID's which may have gaps for cores which are not available. +#[inline] +#[must_use] +pub fn get_core_available(i: usize) -> Option { cores_available().nth(i) } + +/// Determine if core (by id) is available to the process. +#[inline] +#[must_use] +pub fn is_core_available(id: usize) -> bool { cores_available().any(is_equal_to!(id)) } + +/// Get the list of cores available. The values were recorded at program start. +#[inline] +pub fn cores_available() -> impl Iterator { + (0..128).filter(|&i| ((*CORES_AVAILABLE & (1 << i)) != 0)) +} + +/// Get the number of threads which could execute in parallel based on the +/// hardware and administrative constraints of this system. This value should be +/// used to hint the size of thread-pools and divide-and-conquer algorithms. +/// +/// * +#[must_use] +pub fn parallelism() -> usize { + std::thread::available_parallelism() + .expect("Unable to query for available parallelism.") + .get() +} diff --git a/src/core/utils/sys/storage.rs b/src/core/utils/sys/storage.rs new file mode 100644 index 00000000..8dc75236 --- /dev/null +++ b/src/core/utils/sys/storage.rs @@ -0,0 +1,112 @@ +//! System utilities related to devices/peripherals + +use std::{ + ffi::{OsStr, OsString}, + fs, + fs::{read_to_string, FileType}, + iter::IntoIterator, + path::Path, +}; + +use crate::{result::FlatOk, Result}; + +/// Device characteristics useful for random access throughput +#[derive(Clone, Debug, Default)] +pub struct Parallelism { + /// Number of requests for the device. + pub nr_requests: Option, + + /// Individual queue characteristics. + pub mq: Vec, +} + +/// Device queue characteristics +#[derive(Clone, Debug, Default)] +pub struct Queue { + /// Queue's indice. + pub id: usize, + + /// Number of requests for the queue. + pub nr_tags: Option, + + /// CPU affinities for the queue. + pub cpu_list: Vec, +} + +/// Get device characteristics useful for random access throughput by name. +#[must_use] +pub fn parallelism(name: &OsStr) -> Parallelism { + let name = name + .to_str() + .expect("device name expected to be utf-8 representable"); + + let block_path = Path::new("/").join("sys/").join("block/"); + + let mq_path = Path::new(&block_path).join(format!("{name}/mq/")); + + let nr_requests_path = Path::new(&block_path).join(format!("{name}/queue/nr_requests")); + + Parallelism { + nr_requests: read_to_string(&nr_requests_path) + .ok() + .as_deref() + .map(str::trim) + .map(str::parse) + .flat_ok(), + + mq: fs::read_dir(&mq_path) + .into_iter() + .flat_map(IntoIterator::into_iter) + .filter_map(Result::ok) + .filter(|entry| entry.file_type().as_ref().is_ok_and(FileType::is_dir)) + .map(|dir| queue_parallelism(&dir.path())) + .collect(), + } +} + +/// Get device queue characteristics by mq path on sysfs(5) +fn queue_parallelism(dir: &Path) -> Queue { + let queue_id = dir.file_name(); + + let nr_tags_path = dir.join("nr_tags"); + + let cpu_list_path = dir.join("cpu_list"); + + Queue { + id: queue_id + .and_then(OsStr::to_str) + .map(str::parse) + .flat_ok() + .expect("queue has some numerical identifier"), + + nr_tags: read_to_string(&nr_tags_path) + .ok() + .as_deref() + .map(str::trim) + .map(str::parse) + .flat_ok(), + + cpu_list: read_to_string(&cpu_list_path) + .iter() + .flat_map(|list| list.trim().split(',')) + .map(str::trim) + .map(str::parse) + .filter_map(Result::ok) + .collect(), + } +} + +/// Get the name of the device on which Path is mounted. +#[must_use] +pub fn name_from_path(path: &Path) -> Option { + sysinfo::Disks::new_with_refreshed_list() + .into_iter() + .filter(|disk| path.starts_with(disk.mount_point())) + .max_by(|a, b| { + let a = a.mount_point().ancestors().count(); + let b = b.mount_point().ancestors().count(); + a.cmp(&b) + }) + .map(|disk| Path::new(disk.name())) + .and_then(|path| path.file_name().map(ToOwned::to_owned)) +} diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index fe24d4c1..eeb6f2bc 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -129,9 +129,6 @@ tokio_console = [ "dep:console-subscriber", "tokio/tracing", ] -worker_affinity = [ - "dep:core_affinity", -] zstd_compression = [ "conduwuit-api/zstd_compression", "conduwuit-core/zstd_compression", @@ -151,8 +148,6 @@ clap.workspace = true console-subscriber.optional = true console-subscriber.workspace = true const-str.workspace = true -core_affinity.optional = true -core_affinity.workspace = true log.workspace = true opentelemetry-jaeger.optional = true opentelemetry-jaeger.workspace = true diff --git a/src/main/runtime.rs b/src/main/runtime.rs index e98e73d6..cee093ea 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -1,6 +1,17 @@ -use std::{sync::OnceLock, thread, time::Duration}; +use std::{ + iter::once, + sync::{ + atomic::{AtomicUsize, Ordering}, + OnceLock, + }, + thread, + time::Duration, +}; -use conduwuit::Result; +use conduwuit::{ + utils::sys::compute::{get_core_available, set_affinity}, + Result, +}; use tokio::runtime::Builder; use crate::clap::Args; @@ -66,7 +77,6 @@ fn enable_histogram(builder: &mut Builder, args: &Args) { ), )] fn thread_start() { - #[cfg(feature = "worker_affinity")] if WORKER_AFFINITY .get() .copied() @@ -76,24 +86,8 @@ fn thread_start() { } } -#[cfg(feature = "worker_affinity")] fn set_worker_affinity() { - use std::sync::{ - atomic::{AtomicUsize, Ordering}, - LazyLock, - }; - static CORES_OCCUPIED: AtomicUsize = AtomicUsize::new(0); - static CORES_AVAILABLE: LazyLock>> = LazyLock::new(|| { - core_affinity::get_core_ids().map(|mut cores| { - cores.sort_unstable(); - cores - }) - }); - - let Some(cores) = CORES_AVAILABLE.as_ref() else { - return; - }; if thread::current().name() != Some(WORKER_NAME) { return; @@ -106,11 +100,11 @@ fn set_worker_affinity() { return; } - let Some(id) = cores.get(i) else { + let Some(id) = get_core_available(i) else { return; }; - let _set = core_affinity::set_for_current(*id); + set_affinity(once(id)); } #[tracing::instrument( From 03f2ac9cafd9b070c36ca6fdf0fccf3d5fb8e953 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 22 Dec 2024 22:58:37 +0000 Subject: [PATCH 1275/2091] simplify usage of mpmc channels which don't require receiver lock Signed-off-by: Jason Volk --- src/service/admin/mod.rs | 19 +++++++++---------- src/service/presence/mod.rs | 21 +++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 59639e58..399055aa 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -20,14 +20,13 @@ use ruma::{ events::room::message::{Relation, RoomMessageEventContent}, OwnedEventId, OwnedRoomId, RoomId, UserId, }; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::RwLock; use crate::{account_data, globals, rooms, rooms::state::RoomMutexGuard, Dep}; pub struct Service { services: Services, - sender: Sender, - receiver: Mutex>, + channel: (Sender, Receiver), pub handle: RwLock>, pub complete: StdRwLock>, #[cfg(feature = "console")] @@ -78,7 +77,6 @@ const COMMAND_QUEUE_LIMIT: usize = 512; #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { - let (sender, receiver) = loole::bounded(COMMAND_QUEUE_LIMIT); Ok(Arc::new(Self { services: Services { server: args.server.clone(), @@ -90,8 +88,7 @@ impl crate::Service for Service { account_data: args.depend::("account_data"), services: None.into(), }, - sender, - receiver: Mutex::new(receiver), + channel: loole::bounded(COMMAND_QUEUE_LIMIT), handle: RwLock::new(None), complete: StdRwLock::new(None), #[cfg(feature = "console")] @@ -100,8 +97,8 @@ impl crate::Service for Service { } async fn worker(self: Arc) -> Result<()> { - let receiver = self.receiver.lock().await; let mut signals = self.services.server.signal.subscribe(); + let receiver = self.channel.1.clone(); self.startup_execute().await?; self.console_auto_start().await; @@ -128,8 +125,9 @@ impl crate::Service for Service { #[cfg(feature = "console")] self.console.interrupt(); - if !self.sender.is_closed() { - self.sender.close(); + let (sender, _) = &self.channel; + if !sender.is_closed() { + sender.close(); } } @@ -159,7 +157,8 @@ impl Service { /// will take place on the service worker's task asynchronously. Errors if /// the queue is full. pub fn command(&self, command: String, reply_id: Option) -> Result<()> { - self.sender + self.channel + .0 .send(CommandInput { command, reply_id }) .map_err(|e| err!("Failed to enqueue admin command: {e:?}")) } diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 1f9f63d9..bf5258e1 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -6,15 +6,15 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; use conduwuit::{checked, debug, error, result::LogErr, Error, Result, Server}; use futures::{stream::FuturesUnordered, Stream, StreamExt, TryFutureExt}; +use loole::{Receiver, Sender}; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; -use tokio::{sync::Mutex, time::sleep}; +use tokio::time::sleep; use self::{data::Data, presence::Presence}; use crate::{globals, users, Dep}; pub struct Service { - timer_sender: loole::Sender, - timer_receiver: Mutex>, + timer_channel: (Sender, Receiver), timeout_remote_users: bool, idle_timeout: u64, offline_timeout: u64, @@ -36,10 +36,8 @@ impl crate::Service for Service { let config = &args.server.config; let idle_timeout_s = config.presence_idle_timeout_s; let offline_timeout_s = config.presence_offline_timeout_s; - let (timer_sender, timer_receiver) = loole::unbounded(); Ok(Arc::new(Self { - timer_sender, - timer_receiver: Mutex::new(timer_receiver), + timer_channel: loole::unbounded(), timeout_remote_users: config.presence_timeout_remote_users, idle_timeout: checked!(idle_timeout_s * 1_000)?, offline_timeout: checked!(offline_timeout_s * 1_000)?, @@ -53,8 +51,9 @@ impl crate::Service for Service { } async fn worker(self: Arc) -> Result<()> { + let receiver = self.timer_channel.1.clone(); + let mut presence_timers = FuturesUnordered::new(); - let receiver = self.timer_receiver.lock().await; while !receiver.is_closed() { tokio::select! { Some(user_id) = presence_timers.next() => { @@ -74,8 +73,9 @@ impl crate::Service for Service { } fn interrupt(&self) { - if !self.timer_sender.is_closed() { - self.timer_sender.close(); + let (timer_sender, _) = &self.timer_channel; + if !timer_sender.is_closed() { + timer_sender.close(); } } @@ -150,7 +150,8 @@ impl Service { | _ => self.services.server.config.presence_offline_timeout_s, }; - self.timer_sender + self.timer_channel + .0 .send((user_id.to_owned(), Duration::from_secs(timeout))) .map_err(|e| { error!("Failed to add presence timer: {}", e); From e5a1309583c3f280308ddb368d4b1a72cfcb30ad Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 22 Dec 2024 22:59:43 +0000 Subject: [PATCH 1276/2091] additional interruption points to hasten shutdown Signed-off-by: Jason Volk --- src/router/serve/unix.rs | 2 +- src/service/resolver/actual.rs | 8 +++++++- src/service/sending/send.rs | 1 + src/service/sending/sender.rs | 3 ++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index bee3c8c7..f5018455 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -43,7 +43,7 @@ pub(super) async fn serve( let app = app.into_make_service_with_connect_info::(); let builder = server::conn::auto::Builder::new(executor); let listener = init(server).await?; - loop { + while server.running() { let app = app.clone(); let builder = builder.clone(); tokio::select! { diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 3f609b95..0506be2c 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -232,11 +232,12 @@ impl super::Service { #[tracing::instrument(skip_all, name = "well-known")] async fn request_well_known(&self, dest: &str) -> Result> { - trace!("Requesting well known for {dest}"); if !self.has_cached_override(dest) { self.query_and_cache_override(dest, dest, 8448).await?; } + self.services.server.check_running()?; + trace!("Requesting well known for {dest}"); let response = self .services .client @@ -304,6 +305,9 @@ impl super::Service { hostname: &'_ str, port: u16, ) -> Result<()> { + self.services.server.check_running()?; + + debug!("querying IP for {overname:?} ({hostname:?}:{port})"); match self.resolver.resolver.lookup_ip(hostname.to_owned()).await { | Err(e) => Self::handle_resolve_error(&e, hostname), | Ok(override_ip) => { @@ -328,6 +332,8 @@ impl super::Service { [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")]; for hostname in hostnames { + self.services.server.check_running()?; + debug!("querying SRV for {hostname:?}"); let hostname = hostname.trim_end_matches('.'); match self.resolver.resolver.srv_lookup(hostname).await { diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 3d13a3b0..e2981068 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -93,6 +93,7 @@ impl super::Service { let request = Request::try_from(request)?; self.validate_url(request.url())?; + self.server.check_running()?; Ok(request) } diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 4e806ce8..339236a5 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -105,7 +105,8 @@ impl Service { .get(id) .map(|(_, receiver)| receiver.clone()) .expect("Missing channel for sender worker"); - loop { + + while !receiver.is_closed() { tokio::select! { Some(response) = futures.next() => { self.handle_response(response, futures, statuses).await; From b1951070535674b4bcae429c8dee740e72fe104f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 22 Dec 2024 15:09:30 +0000 Subject: [PATCH 1277/2091] optimize for multi-queue storage topologies with affinity Signed-off-by: Jason Volk --- conduwuit-example.toml | 32 ++++++- src/core/config/mod.rs | 52 ++++++++-- src/database/engine.rs | 12 +-- src/database/pool.rs | 170 ++++++++++++++++++++++----------- src/database/pool/configure.rs | 106 ++++++++++++++++++++ src/main/runtime.rs | 2 +- 6 files changed, 297 insertions(+), 77 deletions(-) create mode 100644 src/database/pool/configure.rs diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 3669961a..111acb05 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1397,18 +1397,42 @@ # #admin_room_notices = true +# Enable database pool affinity support. On supporting systems, block +# device queue topologies are detected and the request pool is optimized +# for the hardware; db_pool_workers is determined automatically. +# +#db_pool_affinity = true + # Sets the number of worker threads in the frontend-pool of the database. # This number should reflect the I/O capabilities of the system, -# specifically the queue-depth or the number of simultaneous requests in +# such as the queue-depth or the number of simultaneous requests in # flight. Defaults to 32 or four times the number of CPU cores, whichever # is greater. # +# Note: This value is only used if db_pool_affinity is disabled or not +# detected on the system, otherwise it is determined automatically. +# #db_pool_workers = 32 -# Size of the queue feeding the database's frontend-pool. Defaults to 256 -# or eight times the number of CPU cores, whichever is greater. +# When db_pool_affinity is enabled and detected, the size of any worker +# group will not exceed the determined value. This is necessary when +# thread-pooling approach does not scale to the full capabilities of +# high-end hardware; using detected values without limitation could +# degrade performance. # -#db_pool_queue_size = 256 +# The value is multiplied by the number of cores which share a device +# queue, since group workers can be scheduled on any of those cores. +# +#db_pool_workers_limit = 64 + +# Determines the size of the queues feeding the database's frontend-pool. +# The size of the queue is determined by multiplying this value with the +# number of pool workers. When this queue is full, tokio tasks conducting +# requests will yield until space is available; this is good for +# flow-control by avoiding buffer-bloat, but can inhibit throughput if +# too low. +# +#db_pool_queue_mult = 4 # Number of sender task workers; determines sender parallelism. Default is # '0' which means the value is determined internally, likely matching the diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 23feb0ca..3772aa16 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1581,22 +1581,50 @@ pub struct Config { #[serde(default = "true_fn")] pub admin_room_notices: bool, + /// Enable database pool affinity support. On supporting systems, block + /// device queue topologies are detected and the request pool is optimized + /// for the hardware; db_pool_workers is determined automatically. + /// + /// default: true + #[serde(default = "true_fn")] + pub db_pool_affinity: bool, + /// Sets the number of worker threads in the frontend-pool of the database. /// This number should reflect the I/O capabilities of the system, - /// specifically the queue-depth or the number of simultaneous requests in + /// such as the queue-depth or the number of simultaneous requests in /// flight. Defaults to 32 or four times the number of CPU cores, whichever /// is greater. /// + /// Note: This value is only used if db_pool_affinity is disabled or not + /// detected on the system, otherwise it is determined automatically. + /// /// default: 32 #[serde(default = "default_db_pool_workers")] pub db_pool_workers: usize, - /// Size of the queue feeding the database's frontend-pool. Defaults to 256 - /// or eight times the number of CPU cores, whichever is greater. + /// When db_pool_affinity is enabled and detected, the size of any worker + /// group will not exceed the determined value. This is necessary when + /// thread-pooling approach does not scale to the full capabilities of + /// high-end hardware; using detected values without limitation could + /// degrade performance. /// - /// default: 256 - #[serde(default = "default_db_pool_queue_size")] - pub db_pool_queue_size: usize, + /// The value is multiplied by the number of cores which share a device + /// queue, since group workers can be scheduled on any of those cores. + /// + /// default: 64 + #[serde(default = "default_db_pool_workers_limit")] + pub db_pool_workers_limit: usize, + + /// Determines the size of the queues feeding the database's frontend-pool. + /// The size of the queue is determined by multiplying this value with the + /// number of pool workers. When this queue is full, tokio tasks conducting + /// requests will yield until space is available; this is good for + /// flow-control by avoiding buffer-bloat, but can inhibit throughput if + /// too low. + /// + /// default: 4 + #[serde(default = "default_db_pool_queue_mult")] + pub db_pool_queue_mult: usize, /// Number of sender task workers; determines sender parallelism. Default is /// '0' which means the value is determined internally, likely matching the @@ -2399,8 +2427,12 @@ fn parallelism_scaled(val: usize) -> usize { val.saturating_mul(sys::available_p fn default_trusted_server_batch_size() -> usize { 256 } -fn default_db_pool_workers() -> usize { sys::available_parallelism().saturating_mul(4).max(32) } - -fn default_db_pool_queue_size() -> usize { - sys::available_parallelism().saturating_mul(8).max(256) +fn default_db_pool_workers() -> usize { + sys::available_parallelism() + .saturating_mul(4) + .clamp(32, 1024) } + +fn default_db_pool_workers_limit() -> usize { 64 } + +fn default_db_pool_queue_mult() -> usize { 4 } diff --git a/src/database/engine.rs b/src/database/engine.rs index 73ea559d..faf5b131 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -18,7 +18,7 @@ use rocksdb::{ use crate::{ opts::{cf_options, db_options}, - or_else, pool, + or_else, pool::Pool, result, util::map_err, @@ -87,8 +87,9 @@ impl Engine { .map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts)) .collect::>(); - debug!("Opening database..."); let path = &config.database_path; + + debug!("Opening database..."); let res = if config.rocksdb_read_only { Db::open_cf_descriptors_read_only(&db_opts, path, cfds, false) } else if config.rocksdb_secondary { @@ -105,11 +106,6 @@ impl Engine { "Opened database." ); - let pool_opts = pool::Opts { - queue_size: config.db_pool_queue_size, - worker_num: config.db_pool_workers, - }; - Ok(Arc::new(Self { server: server.clone(), row_cache, @@ -121,7 +117,7 @@ impl Engine { corks: AtomicU32::new(0), read_only: config.rocksdb_read_only, secondary: config.rocksdb_secondary, - pool: Pool::new(server, &pool_opts).await?, + pool: Pool::new(server).await?, })) } diff --git a/src/database/pool.rs b/src/database/pool.rs index 4f018a38..51e705ce 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -1,3 +1,5 @@ +mod configure; + use std::{ mem::take, sync::{ @@ -6,39 +8,50 @@ use std::{ }, }; -use async_channel::{bounded, Receiver, RecvError, Sender}; -use conduwuit::{debug, debug_warn, defer, err, implement, result::DebugInspect, Result, Server}; +use async_channel::{Receiver, RecvError, Sender}; +use conduwuit::{ + debug, debug_warn, defer, err, implement, + result::DebugInspect, + trace, + utils::sys::compute::{get_affinity, get_core_available, set_affinity}, + Result, Server, +}; use futures::{channel::oneshot, TryFutureExt}; use oneshot::Sender as ResultSender; use rocksdb::Direction; use tokio::task::JoinSet; +use self::configure::configure; use crate::{keyval::KeyBuf, stream, Handle, Map}; +/// Frontend thread-pool. Operating system threads are used to make database +/// requests which are not cached. These thread-blocking requests are offloaded +/// from the tokio async workers and executed on this threadpool. pub(crate) struct Pool { server: Arc, + queues: Vec>, workers: Mutex>, - queue: Sender, + topology: Vec, busy: AtomicUsize, queued_max: AtomicUsize, } -pub(crate) struct Opts { - pub(crate) queue_size: usize, - pub(crate) worker_num: usize, -} - +/// Operations which can be submitted to the pool. pub(crate) enum Cmd { Get(Get), Iter(Seek), } +/// Point-query pub(crate) struct Get { pub(crate) map: Arc, pub(crate) key: KeyBuf, pub(crate) res: Option>>>, } +/// Iterator-seek. +/// Note: only initial seek is supported at this time on the assumption rocksdb +/// prefetching prevents mid-iteration polls from blocking on I/O. pub(crate) struct Seek { pub(crate) map: Arc, pub(crate) state: stream::State<'static>, @@ -47,34 +60,44 @@ pub(crate) struct Seek { pub(crate) res: Option>>, } -const QUEUE_LIMIT: (usize, usize) = (1, 3072); -const WORKER_LIMIT: (usize, usize) = (1, 512); - -impl Drop for Pool { - fn drop(&mut self) { - debug_assert!(self.queue.is_empty(), "channel must be empty on drop"); - debug_assert!(self.queue.is_closed(), "channel should be closed on drop"); - } -} +const WORKER_LIMIT: (usize, usize) = (1, 1024); +const QUEUE_LIMIT: (usize, usize) = (1, 2048); #[implement(Pool)] -pub(crate) async fn new(server: &Arc, opts: &Opts) -> Result> { - let queue_size = opts.queue_size.clamp(QUEUE_LIMIT.0, QUEUE_LIMIT.1); - let (send, recv) = bounded(queue_size); +pub(crate) async fn new(server: &Arc) -> Result> { + let (total_workers, queue_sizes, topology) = configure(server); + + let (senders, receivers) = queue_sizes.into_iter().map(async_channel::bounded).unzip(); + let pool = Arc::new(Self { server: server.clone(), + + queues: senders, + workers: JoinSet::new().into(), - queue: send, + + topology, + busy: AtomicUsize::default(), + queued_max: AtomicUsize::default(), }); - let worker_num = opts.worker_num.clamp(WORKER_LIMIT.0, WORKER_LIMIT.1); - pool.spawn_until(recv, worker_num).await?; + pool.spawn_until(receivers, total_workers).await?; Ok(pool) } +impl Drop for Pool { + fn drop(&mut self) { + debug_assert!(self.queues.iter().all(Sender::is_empty), "channel must be empty on drop"); + debug_assert!( + self.queues.iter().all(Sender::is_closed), + "channel should be closed on drop" + ); + } +} + #[implement(Pool)] pub(crate) async fn shutdown(self: &Arc) { self.close(); @@ -83,36 +106,39 @@ pub(crate) async fn shutdown(self: &Arc) { debug!(workers = workers.len(), "Waiting for workers to join..."); workers.join_all().await; - debug_assert!(self.queue.is_empty(), "channel is not empty"); } #[implement(Pool)] -pub(crate) fn close(&self) -> bool { - if !self.queue.close() { - return false; - } +pub(crate) fn close(&self) { + let senders = self.queues.iter().map(Sender::sender_count).sum::(); - let mut workers = take(&mut *self.workers.lock().expect("locked")); - debug!(workers = workers.len(), "Waiting for workers to join..."); - workers.abort_all(); - drop(workers); + let receivers = self + .queues + .iter() + .map(Sender::receiver_count) + .sum::(); - std::thread::yield_now(); - debug_assert!(self.queue.is_empty(), "channel is not empty"); debug!( - senders = self.queue.sender_count(), - receivers = self.queue.receiver_count(), - "Closed pool channel" + queues = self.queues.len(), + workers = self.workers.lock().expect("locked").len(), + ?senders, + ?receivers, + "Closing pool..." ); - true + for queue in &self.queues { + queue.close(); + } + + self.workers.lock().expect("locked").abort_all(); + std::thread::yield_now(); } #[implement(Pool)] -async fn spawn_until(self: &Arc, recv: Receiver, max: usize) -> Result { +async fn spawn_until(self: &Arc, recv: Vec>, count: usize) -> Result { let mut workers = self.workers.lock().expect("locked"); - while workers.len() < max { - self.spawn_one(&mut workers, recv.clone())?; + while workers.len() < count { + self.spawn_one(&mut workers, &recv)?; } Ok(()) @@ -125,8 +151,13 @@ async fn spawn_until(self: &Arc, recv: Receiver, max: usize) -> Resul skip_all, fields(id = %workers.len()) )] -fn spawn_one(self: &Arc, workers: &mut JoinSet<()>, recv: Receiver) -> Result { +fn spawn_one(self: &Arc, workers: &mut JoinSet<()>, recv: &[Receiver]) -> Result { + debug_assert!(!self.queues.is_empty(), "Must have at least one queue"); + debug_assert!(!recv.is_empty(), "Must have at least one receiver"); + let id = workers.len(); + let group = id.overflowing_rem(self.queues.len()).0; + let recv = recv[group].clone(); let self_ = self.clone(); #[cfg(not(tokio_unstable))] @@ -146,7 +177,9 @@ fn spawn_one(self: &Arc, workers: &mut JoinSet<()>, recv: Receiver) - pub(crate) async fn execute_get(&self, mut cmd: Get) -> Result> { let (send, recv) = oneshot::channel(); _ = cmd.res.insert(send); - self.execute(Cmd::Get(cmd)) + + let queue = self.select_queue(); + self.execute(queue, Cmd::Get(cmd)) .and_then(|()| { recv.map_ok(into_recv_get_result) .map_err(|e| err!(error!("recv failed {e:?}"))) @@ -159,7 +192,9 @@ pub(crate) async fn execute_get(&self, mut cmd: Get) -> Result> { pub(crate) async fn execute_iter(&self, mut cmd: Seek) -> Result> { let (send, recv) = oneshot::channel(); _ = cmd.res.insert(send); - self.execute(Cmd::Iter(cmd)) + + let queue = self.select_queue(); + self.execute(queue, Cmd::Iter(cmd)) .and_then(|()| { recv.map_ok(into_recv_seek) .map_err(|e| err!(error!("recv failed {e:?}"))) @@ -167,6 +202,13 @@ pub(crate) async fn execute_iter(&self, mut cmd: Seek) -> Result &Sender { + let core_id = get_affinity().next().unwrap_or(0); + let chan_id = self.topology[core_id]; + self.queues.get(chan_id).unwrap_or_else(|| &self.queues[0]) +} + #[implement(Pool)] #[tracing::instrument( level = "trace", @@ -174,25 +216,24 @@ pub(crate) async fn execute_iter(&self, mut cmd: Seek) -> Result Result { +async fn execute(&self, queue: &Sender, cmd: Cmd) -> Result { if cfg!(debug_assertions) { - self.queued_max - .fetch_max(self.queue.len(), Ordering::Relaxed); + self.queued_max.fetch_max(queue.len(), Ordering::Relaxed); } - if self.queue.is_full() { + if queue.is_full() { debug_warn!( - capacity = ?self.queue.capacity(), + capacity = ?queue.capacity(), "pool queue is full" ); } - self.queue + queue .send(cmd) .await .map_err(|e| err!(error!("send failed {e:?}"))) @@ -208,12 +249,33 @@ async fn execute(&self, cmd: Cmd) -> Result { ), )] fn worker(self: Arc, id: usize, recv: Receiver) { - debug!("worker spawned"); - defer! {{ debug!("worker finished"); }} + defer! {{ trace!("worker finished"); }} + trace!("worker spawned"); + self.worker_init(id); self.worker_loop(&recv); } +#[implement(Pool)] +fn worker_init(&self, id: usize) { + let group = id.overflowing_rem(self.queues.len()).0; + let affinity = self + .topology + .iter() + .enumerate() + .filter(|_| self.queues.len() > 1) + .filter_map(|(core_id, &queue_id)| (group == queue_id).then_some(core_id)) + .filter_map(get_core_available); + + // affinity is empty (no-op) if there's only one queue + set_affinity(affinity.clone()); + debug!( + ?group, + affinity = ?affinity.collect::>(), + "worker ready" + ); +} + #[implement(Pool)] fn worker_loop(&self, recv: &Receiver) { // initial +1 needed prior to entering wait diff --git a/src/database/pool/configure.rs b/src/database/pool/configure.rs new file mode 100644 index 00000000..8353a265 --- /dev/null +++ b/src/database/pool/configure.rs @@ -0,0 +1,106 @@ +use std::{ffi::OsStr, sync::Arc}; + +use conduwuit::{ + debug_info, + utils::{ + sys::{compute::is_core_available, storage}, + BoolExt, + }, + Server, +}; + +use super::{QUEUE_LIMIT, WORKER_LIMIT}; + +pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) { + let config = &server.config; + + // This finds the block device and gathers all the properties we need. + let (device_name, device_prop) = config + .db_pool_affinity + .and_then(|| storage::name_from_path(&config.database_path)) + .map(|device_name| (device_name.clone(), storage::parallelism(&device_name))) + .unzip(); + + // The default worker count is masked-on if we didn't find better information. + let default_worker_count = device_prop + .as_ref() + .is_none_or(|prop| prop.mq.is_empty()) + .then_some(config.db_pool_workers); + + // Determine the worker groupings. Each indice represents a hardware queue and + // contains the number of workers which will service it. + let worker_counts: Vec<_> = device_prop + .iter() + .map(|dev| &dev.mq) + .flat_map(|mq| mq.iter()) + .filter(|mq| mq.cpu_list.iter().copied().any(is_core_available)) + .map(|mq| { + mq.nr_tags.unwrap_or_default().min( + config.db_pool_workers_limit.saturating_mul( + mq.cpu_list + .iter() + .filter(|&&id| is_core_available(id)) + .count() + .max(1), + ), + ) + }) + .chain(default_worker_count) + .collect(); + + // Determine our software queue size for each hardware queue. This is the mpmc + // between the tokio worker and the pool worker. + let queue_sizes: Vec<_> = worker_counts + .iter() + .map(|worker_count| { + worker_count + .saturating_mul(config.db_pool_queue_mult) + .clamp(QUEUE_LIMIT.0, QUEUE_LIMIT.1) + }) + .collect(); + + // Determine the CPU affinities of each hardware queue. Each indice is a cpu and + // each value is the associated hardware queue. There is a little shiftiness + // going on because cpu's which are not available to the process are filtered + // out, similar to the worker_counts. + let topology = device_prop + .iter() + .map(|dev| &dev.mq) + .flat_map(|mq| mq.iter()) + .fold(vec![0; 128], |mut topology, mq| { + mq.cpu_list + .iter() + .filter(|&&id| is_core_available(id)) + .for_each(|&id| { + topology[id] = mq.id; + }); + + topology + }); + + // Regardless of the capacity of all queues we establish some limit on the total + // number of workers; this is hopefully hinted by nr_requests. + let max_workers = device_prop + .as_ref() + .and_then(|prop| prop.nr_requests) + .unwrap_or(WORKER_LIMIT.1); + + // Determine the final worker count which we'll be spawning. + let total_workers = worker_counts + .iter() + .sum::() + .clamp(WORKER_LIMIT.0, max_workers); + + debug_info!( + device_name = ?device_name + .as_deref() + .and_then(OsStr::to_str) + .unwrap_or("None"), + ?worker_counts, + ?queue_sizes, + ?total_workers, + "Frontend topology", + ); + + (total_workers, queue_sizes, topology) +} diff --git a/src/main/runtime.rs b/src/main/runtime.rs index cee093ea..bfd2ef74 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -19,7 +19,7 @@ use crate::clap::Args; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; const WORKER_KEEPALIVE: u64 = 36; -const MAX_BLOCKING_THREADS: usize = 1024; +const MAX_BLOCKING_THREADS: usize = 2048; static WORKER_AFFINITY: OnceLock = OnceLock::new(); From 7a6d6575585cd001f0c889aa41c1a2663b2435d0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 23 Dec 2024 04:32:28 +0000 Subject: [PATCH 1278/2091] configurable dynamic stream concurrency scalar Signed-off-by: Jason Volk --- conduwuit-example.toml | 23 +++++++++++++++++ src/core/config/mod.rs | 31 +++++++++++++++++++++++ src/core/utils/stream/broadband.rs | 13 +++++----- src/core/utils/stream/mod.rs | 29 ++++++++++++++++++++++ src/core/utils/stream/try_broadband.rs | 5 ++-- src/core/utils/stream/wideband.rs | 9 +++---- src/database/map/get_batch.rs | 10 +++++--- src/database/pool/configure.rs | 34 +++++++++++++++++++++++++- src/main/server.rs | 10 +++++++- 9 files changed, 144 insertions(+), 20 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 111acb05..c64b18e8 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1434,6 +1434,29 @@ # #db_pool_queue_mult = 4 +# Sets the initial value for the concurrency of streams. This value simply +# allows overriding the default in the code. The default is 32, which is +# the same as the default in the code. Note this value is itself +# overridden by the computed stream_width_scale, unless that is disabled; +# this value can serve as a fixed-width instead. +# +#stream_width_default = 32 + +# Scales the stream width starting from a base value detected for the +# specific system. The base value is the database pool worker count +# determined from the hardware queue size (e.g. 32 for SSD or 64 or 128+ +# for NVMe). This float allows scaling the width up or down by multiplying +# it (e.g. 1.5, 2.0, etc). The maximum result can be the size of the pool +# queue (see: db_pool_queue_mult) as any larger value will stall the tokio +# task. The value can also be scaled down (e.g. 0.5) to improve +# responsiveness for many users at the cost of throughput for each. +# +# Setting this value to 0.0 causes the stream width to be fixed at the +# value of stream_width_default. The default is 1.0 to match the +# capabilities detected for the system. +# +#stream_width_scale = 1.0 + # Number of sender task workers; determines sender parallelism. Default is # '0' which means the value is determined internally, likely matching the # number of tokio worker-threads or number of cores, etc. Override by diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 3772aa16..e1f578c8 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1626,6 +1626,33 @@ pub struct Config { #[serde(default = "default_db_pool_queue_mult")] pub db_pool_queue_mult: usize, + /// Sets the initial value for the concurrency of streams. This value simply + /// allows overriding the default in the code. The default is 32, which is + /// the same as the default in the code. Note this value is itself + /// overridden by the computed stream_width_scale, unless that is disabled; + /// this value can serve as a fixed-width instead. + /// + /// default: 32 + #[serde(default = "default_stream_width_default")] + pub stream_width_default: usize, + + /// Scales the stream width starting from a base value detected for the + /// specific system. The base value is the database pool worker count + /// determined from the hardware queue size (e.g. 32 for SSD or 64 or 128+ + /// for NVMe). This float allows scaling the width up or down by multiplying + /// it (e.g. 1.5, 2.0, etc). The maximum result can be the size of the pool + /// queue (see: db_pool_queue_mult) as any larger value will stall the tokio + /// task. The value can also be scaled down (e.g. 0.5) to improve + /// responsiveness for many users at the cost of throughput for each. + /// + /// Setting this value to 0.0 causes the stream width to be fixed at the + /// value of stream_width_default. The default is 1.0 to match the + /// capabilities detected for the system. + /// + /// default: 1.0 + #[serde(default = "default_stream_width_scale")] + pub stream_width_scale: f32, + /// Number of sender task workers; determines sender parallelism. Default is /// '0' which means the value is determined internally, likely matching the /// number of tokio worker-threads or number of cores, etc. Override by @@ -2436,3 +2463,7 @@ fn default_db_pool_workers() -> usize { fn default_db_pool_workers_limit() -> usize { 64 } fn default_db_pool_queue_mult() -> usize { 4 } + +fn default_stream_width_default() -> usize { 32 } + +fn default_stream_width_scale() -> f32 { 1.0 } diff --git a/src/core/utils/stream/broadband.rs b/src/core/utils/stream/broadband.rs index 37416d63..6d1ff6fe 100644 --- a/src/core/utils/stream/broadband.rs +++ b/src/core/utils/stream/broadband.rs @@ -7,9 +7,7 @@ use futures::{ Future, }; -use super::ReadyExt; - -const WIDTH: usize = 32; +use super::{automatic_width, ReadyExt}; /// Concurrency extensions to augment futures::StreamExt. broad_ combinators /// produce out-of-order @@ -95,7 +93,7 @@ where Fut: Future + Send, { self.map(f) - .buffer_unordered(n.into().unwrap_or(WIDTH)) + .buffer_unordered(n.into().unwrap_or_else(automatic_width)) .ready_all(identity) } @@ -107,7 +105,7 @@ where Fut: Future + Send, { self.map(f) - .buffer_unordered(n.into().unwrap_or(WIDTH)) + .buffer_unordered(n.into().unwrap_or_else(automatic_width)) .ready_any(identity) } @@ -120,7 +118,7 @@ where U: Send, { self.map(f) - .buffer_unordered(n.into().unwrap_or(WIDTH)) + .buffer_unordered(n.into().unwrap_or_else(automatic_width)) .ready_filter_map(identity) } @@ -132,6 +130,7 @@ where Fut: Future + Send, U: Send, { - self.map(f).buffer_unordered(n.into().unwrap_or(WIDTH)) + self.map(f) + .buffer_unordered(n.into().unwrap_or_else(automatic_width)) } } diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index c9138116..4456784f 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -19,3 +19,32 @@ pub use tools::Tools; pub use try_broadband::TryBroadbandExt; pub use try_ready::TryReadyExt; pub use wideband::WidebandExt; + +/// Stream concurrency factor; this is a live value. +static WIDTH: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(32); + +/// Practicable limits on the stream width +pub const WIDTH_LIMIT: (usize, usize) = (1, 1024); + +/// Sets the live concurrency factor. The first return value is the previous +/// width which was replaced. The second return value is the value which was set +/// after any applied limits. +pub fn set_width(width: usize) -> (usize, usize) { + use std::sync::atomic::Ordering; + + let width = width.clamp(WIDTH_LIMIT.0, WIDTH_LIMIT.1); + (WIDTH.swap(width, Ordering::Relaxed), width) +} + +/// Used by stream operations where the concurrency factor hasn't been manually +/// supplied by the caller (most uses). Instead we provide a default value which +/// is adjusted at startup for the specific system and also dynamically. +#[inline] +pub fn automatic_width() -> usize { + use std::sync::atomic::Ordering; + + let width = WIDTH.load(Ordering::Relaxed); + debug_assert!(width >= WIDTH_LIMIT.0, "WIDTH should not be zero"); + debug_assert!(width <= WIDTH_LIMIT.1, "WIDTH is probably too large"); + width +} diff --git a/src/core/utils/stream/try_broadband.rs b/src/core/utils/stream/try_broadband.rs index d1213174..c72fcc2c 100644 --- a/src/core/utils/stream/try_broadband.rs +++ b/src/core/utils/stream/try_broadband.rs @@ -2,10 +2,9 @@ use futures::{TryFuture, TryStream, TryStreamExt}; +use super::automatic_width; use crate::Result; -const WIDTH: usize = 32; - /// Concurrency extensions to augment futures::TryStreamExt. broad_ combinators /// produce out-of-order pub trait TryBroadbandExt @@ -49,6 +48,6 @@ where Fut: TryFuture> + Send, { self.map_ok(f) - .try_buffer_unordered(n.into().unwrap_or(WIDTH)) + .try_buffer_unordered(n.into().unwrap_or_else(automatic_width)) } } diff --git a/src/core/utils/stream/wideband.rs b/src/core/utils/stream/wideband.rs index 053a351f..a8560bb4 100644 --- a/src/core/utils/stream/wideband.rs +++ b/src/core/utils/stream/wideband.rs @@ -7,9 +7,7 @@ use futures::{ Future, }; -use super::ReadyExt; - -const WIDTH: usize = 32; +use super::{automatic_width, ReadyExt}; /// Concurrency extensions to augment futures::StreamExt. wideband_ combinators /// produce in-order. @@ -66,7 +64,7 @@ where U: Send, { self.map(f) - .buffered(n.into().unwrap_or(WIDTH)) + .buffered(n.into().unwrap_or_else(automatic_width)) .ready_filter_map(identity) } @@ -78,6 +76,7 @@ where Fut: Future + Send, U: Send, { - self.map(f).buffered(n.into().unwrap_or(WIDTH)) + self.map(f) + .buffered(n.into().unwrap_or_else(automatic_width)) } } diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index 49cd5920..631692fe 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -1,6 +1,10 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{err, implement, utils::IterStream, Result}; +use conduwuit::{ + err, implement, + utils::{stream::automatic_width, IterStream}, + Result, +}; use futures::{Stream, StreamExt}; use serde::Serialize; @@ -18,7 +22,7 @@ where { keys.stream() .map(move |key| self.aqry::(&key)) - .buffered(self.db.server.config.db_pool_workers.saturating_mul(2)) + .buffered(automatic_width()) } #[implement(super::Map)] @@ -33,7 +37,7 @@ where { keys.stream() .map(move |key| self.get(key)) - .buffered(self.db.server.config.db_pool_workers.saturating_mul(2)) + .buffered(automatic_width()) } #[implement(super::Map)] diff --git a/src/database/pool/configure.rs b/src/database/pool/configure.rs index 8353a265..9361a534 100644 --- a/src/database/pool/configure.rs +++ b/src/database/pool/configure.rs @@ -1,8 +1,11 @@ use std::{ffi::OsStr, sync::Arc}; use conduwuit::{ - debug_info, + debug, debug_info, expected, utils::{ + math::usize_from_f64, + stream, + stream::WIDTH_LIMIT, sys::{compute::is_core_available, storage}, BoolExt, }, @@ -91,6 +94,13 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) .sum::() .clamp(WORKER_LIMIT.0, max_workers); + // After computing all of the above we can update the global automatic stream + // width, hopefully with a better value tailored to this system. + if config.stream_width_scale > 0.0 { + let num_queues = queue_sizes.len(); + update_stream_width(server, num_queues, total_workers); + } + debug_info!( device_name = ?device_name .as_deref() @@ -99,8 +109,30 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) ?worker_counts, ?queue_sizes, ?total_workers, + stream_width = ?stream::automatic_width(), "Frontend topology", ); (total_workers, queue_sizes, topology) } + +#[allow(clippy::as_conversions, clippy::cast_precision_loss)] +fn update_stream_width(server: &Arc, num_queues: usize, total_workers: usize) { + let config = &server.config; + let scale: f64 = config.stream_width_scale.min(100.0).into(); + let req_width = expected!(total_workers / num_queues).next_multiple_of(2); + let req_width = req_width as f64; + let req_width = usize_from_f64(req_width * scale) + .expect("failed to convert f64 to usize") + .clamp(WIDTH_LIMIT.0, WIDTH_LIMIT.1); + + let (old_width, new_width) = stream::set_width(req_width); + debug!( + scale = ?config.stream_width_scale, + ?num_queues, + ?req_width, + ?old_width, + ?new_width, + "Updated global stream width" + ); +} diff --git a/src/main/server.rs b/src/main/server.rs index bc2cff85..e1389f6d 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,6 +1,12 @@ use std::sync::Arc; -use conduwuit::{config::Config, info, log::Log, utils::sys, Error, Result}; +use conduwuit::{ + config::Config, + info, + log::Log, + utils::{stream, sys}, + Error, Result, +}; use tokio::{runtime, sync::Mutex}; use crate::{clap::Args, logging::TracingFlameGuard}; @@ -45,6 +51,8 @@ impl Server { sys::maximize_fd_limit() .expect("Unable to increase maximum soft and hard file descriptor limit"); + let (_old_width, _new_width) = stream::set_width(config.stream_width_default); + info!( server_name = %config.server_name, database_path = ?config.database_path, From a407909d09f09c6fe0f714a8ab6ae872c0b8ef6a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 24 Dec 2024 01:07:53 +0000 Subject: [PATCH 1279/2091] reduce unnecessary info-level spans to debug Signed-off-by: Jason Volk --- src/core/utils/sys/compute.rs | 2 +- src/service/resolver/actual.rs | 4 ++-- src/service/rooms/timeline/mod.rs | 6 +++--- src/service/sending/mod.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/core/utils/sys/compute.rs b/src/core/utils/sys/compute.rs index 4e9ef743..e947b579 100644 --- a/src/core/utils/sys/compute.rs +++ b/src/core/utils/sys/compute.rs @@ -10,7 +10,7 @@ static CORES_AVAILABLE: LazyLock = LazyLock::new(|| { .unwrap_or_default() .into_iter() .map(|core_id| core_id.id) - .inspect(|&id| assert!(id < 128, "Core ID must be < 128 at least for now")) + .inspect(|&id| debug_assert!(id < 128, "Core ID must be < 128 at least for now")) .fold(0_u128, |mask, id| mask | (1 << id)) }); diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 0506be2c..f8bc0f6a 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -45,7 +45,7 @@ impl super::Service { /// Implemented according to the specification at /// Numbers in comments below refer to bullet points in linked section of /// specification - #[tracing::instrument(skip_all, name = "actual")] + #[tracing::instrument(skip(self, cache), name = "actual")] pub async fn resolve_actual_dest( &self, dest: &ServerName, @@ -298,7 +298,7 @@ impl super::Service { } } - #[tracing::instrument(skip_all, name = "ip")] + #[tracing::instrument(skip(self, overname, port), name = "ip")] async fn query_and_cache_override( &self, overname: &'_ str, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 028b270f..3b81e2e5 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -859,7 +859,7 @@ impl Service { /// Creates a new persisted data unit and adds it to a room. This function /// takes a roomid_mutex_state, meaning that only this function is able to /// mutate the room state. - #[tracing::instrument(skip(self, state_lock))] + #[tracing::instrument(skip(self, state_lock), level = "debug")] pub async fn build_and_append_pdu( &self, pdu_builder: PduBuilder, @@ -1082,7 +1082,7 @@ impl Service { self.replace_pdu(&pdu_id, &obj, &pdu).await } - #[tracing::instrument(skip(self))] + #[tracing::instrument(skip(self), level = "debug")] pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { if self .services @@ -1194,7 +1194,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, pdu))] + #[tracing::instrument(skip(self, pdu), level = "debug")] pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> Result<()> { let (event_id, value, room_id) = self.services.event_handler.parse_incoming_pdu(&pdu).await?; diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 5ccba249..e52bfb25 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -277,7 +277,7 @@ impl Service { } /// Sends a request to a federation server - #[tracing::instrument(skip_all, name = "request")] + #[tracing::instrument(skip_all, name = "request", level = "debug")] pub async fn send_federation_request( &self, dest: &ServerName, @@ -291,7 +291,7 @@ impl Service { } /// Like send_federation_request() but with a very large timeout - #[tracing::instrument(skip_all, name = "synapse")] + #[tracing::instrument(skip_all, name = "synapse", level = "debug")] pub async fn send_synapse_request( &self, dest: &ServerName, From 3b95af9a1887a07070f7b7918ce3cc91fb911918 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 24 Dec 2024 14:37:26 +0000 Subject: [PATCH 1280/2091] move stream width related into unit Signed-off-by: Jason Volk --- src/core/utils/stream/band.rs | 26 ++++++++++++++++++++++++++ src/core/utils/stream/mod.rs | 31 ++----------------------------- 2 files changed, 28 insertions(+), 29 deletions(-) create mode 100644 src/core/utils/stream/band.rs diff --git a/src/core/utils/stream/band.rs b/src/core/utils/stream/band.rs new file mode 100644 index 00000000..76f2a85a --- /dev/null +++ b/src/core/utils/stream/band.rs @@ -0,0 +1,26 @@ +use std::sync::atomic::{AtomicUsize, Ordering}; + +/// Stream concurrency factor; this is a live value. +static WIDTH: AtomicUsize = AtomicUsize::new(32); + +/// Practicable limits on the stream width +pub const WIDTH_LIMIT: (usize, usize) = (1, 1024); + +/// Sets the live concurrency factor. The first return value is the previous +/// width which was replaced. The second return value is the value which was set +/// after any applied limits. +pub fn set_width(width: usize) -> (usize, usize) { + let width = width.clamp(WIDTH_LIMIT.0, WIDTH_LIMIT.1); + (WIDTH.swap(width, Ordering::Relaxed), width) +} + +/// Used by stream operations where the concurrency factor hasn't been manually +/// supplied by the caller (most uses). Instead we provide a default value which +/// is adjusted at startup for the specific system and also dynamically. +#[inline] +pub fn automatic_width() -> usize { + let width = WIDTH.load(Ordering::Relaxed); + debug_assert!(width >= WIDTH_LIMIT.0, "WIDTH should not be zero"); + debug_assert!(width <= WIDTH_LIMIT.1, "WIDTH is probably too large"); + width +} diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index 4456784f..a5ef17c5 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -1,3 +1,4 @@ +mod band; mod broadband; mod cloned; mod expect; @@ -9,6 +10,7 @@ mod try_broadband; mod try_ready; mod wideband; +pub use band::{automatic_width, set_width, WIDTH_LIMIT}; pub use broadband::BroadbandExt; pub use cloned::Cloned; pub use expect::TryExpect; @@ -19,32 +21,3 @@ pub use tools::Tools; pub use try_broadband::TryBroadbandExt; pub use try_ready::TryReadyExt; pub use wideband::WidebandExt; - -/// Stream concurrency factor; this is a live value. -static WIDTH: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(32); - -/// Practicable limits on the stream width -pub const WIDTH_LIMIT: (usize, usize) = (1, 1024); - -/// Sets the live concurrency factor. The first return value is the previous -/// width which was replaced. The second return value is the value which was set -/// after any applied limits. -pub fn set_width(width: usize) -> (usize, usize) { - use std::sync::atomic::Ordering; - - let width = width.clamp(WIDTH_LIMIT.0, WIDTH_LIMIT.1); - (WIDTH.swap(width, Ordering::Relaxed), width) -} - -/// Used by stream operations where the concurrency factor hasn't been manually -/// supplied by the caller (most uses). Instead we provide a default value which -/// is adjusted at startup for the specific system and also dynamically. -#[inline] -pub fn automatic_width() -> usize { - use std::sync::atomic::Ordering; - - let width = WIDTH.load(Ordering::Relaxed); - debug_assert!(width >= WIDTH_LIMIT.0, "WIDTH should not be zero"); - debug_assert!(width <= WIDTH_LIMIT.1, "WIDTH is probably too large"); - width -} From 5a335933b84c4598106d4b7b448b0ea4e8e48504 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 24 Dec 2024 14:43:28 +0000 Subject: [PATCH 1281/2091] pass stream width to ruma state res Signed-off-by: Jason Volk --- Cargo.lock | 26 +++++------ Cargo.toml | 2 +- .../rooms/event_handler/resolve_state.rs | 45 ++++++++++++------- .../rooms/event_handler/state_at_incoming.rs | 28 +++--------- 4 files changed, 48 insertions(+), 53 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6522aa55..9134b290 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3175,7 +3175,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "assign", "js_int", @@ -3197,7 +3197,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "js_int", "ruma-common", @@ -3209,7 +3209,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "as_variant", "assign", @@ -3232,7 +3232,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "as_variant", "base64 0.22.1", @@ -3262,7 +3262,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3286,7 +3286,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "bytes", "http", @@ -3304,7 +3304,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3313,7 +3313,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "js_int", "ruma-common", @@ -3323,7 +3323,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3338,7 +3338,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "js_int", "ruma-common", @@ -3350,7 +3350,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "headers", "http", @@ -3363,7 +3363,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3379,7 +3379,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=112ccc24cb14de26757715d611285d0806d5d91f#112ccc24cb14de26757715d611285d0806d5d91f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 689f2b5c..1546df3a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -335,7 +335,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "112ccc24cb14de26757715d611285d0806d5d91f" +rev = "d3ed3194ebe96b921d06d1d3e607f0bf7873f792" features = [ "compat", "rand", diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index d507c9c3..108be18d 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -6,7 +6,7 @@ use std::{ use conduwuit::{ debug, err, implement, - utils::stream::{IterStream, WidebandExt}, + utils::stream::{automatic_width, IterStream, WidebandExt}, Result, }; use futures::{FutureExt, StreamExt, TryFutureExt}; @@ -75,22 +75,10 @@ pub async fn resolve_state( .await; debug!("Resolving state"); - let lock = self.services.globals.stateres_mutex.lock(); - - let event_fetch = |event_id| self.event_fetch(event_id); - let event_exists = |event_id| self.event_exists(event_id); - let state = state_res::resolve( - room_version_id, - &fork_states, - &auth_chain_sets, - &event_fetch, - &event_exists, - ) - .boxed() - .await - .map_err(|e| err!(Database(error!("State resolution failed: {e:?}"))))?; - - drop(lock); + let state = self + .state_resolution(room_version_id, &fork_states, &auth_chain_sets) + .boxed() + .await?; debug!("State resolution done."); let state_events: Vec<_> = state @@ -119,3 +107,26 @@ pub async fn resolve_state( Ok(Arc::new(new_room_state)) } + +#[implement(super::Service)] +#[tracing::instrument(name = "ruma", level = "debug", skip_all)] +pub async fn state_resolution( + &self, + room_version: &RoomVersionId, + state_sets: &[StateMap>], + auth_chain_sets: &Vec>>, +) -> Result>> { + //TODO: ??? + let _lock = self.services.globals.stateres_mutex.lock(); + + state_res::resolve( + room_version, + state_sets.iter(), + auth_chain_sets, + &|event_id| self.event_fetch(event_id), + &|event_id| self.event_exists(event_id), + automatic_width(), + ) + .await + .map_err(|e| err!(error!("State resolution failed: {e:?}"))) +} diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 51879d0d..658fb904 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -11,10 +11,7 @@ use conduwuit::{ PduEvent, Result, }; use futures::{FutureExt, StreamExt}; -use ruma::{ - state_res::{self, StateMap}, - EventId, RoomId, RoomVersionId, -}; +use ruma::{state_res::StateMap, EventId, RoomId, RoomVersionId}; // TODO: if we know the prev_events of the incoming event we can avoid the #[implement(super::Service)] @@ -157,24 +154,11 @@ pub(super) async fn state_at_incoming_resolved( fork_states.push(state); } - let lock = self.services.globals.stateres_mutex.lock(); - - let event_fetch = |event_id| self.event_fetch(event_id); - let event_exists = |event_id| self.event_exists(event_id); - let result = state_res::resolve( - room_version_id, - &fork_states, - &auth_chain_sets, - &event_fetch, - &event_exists, - ) - .boxed() - .await - .map_err(|e| err!(Database(warn!(?e, "State resolution on prev events failed.")))); - - drop(lock); - - let Ok(new_state) = result else { + let Ok(new_state) = self + .state_resolution(room_version_id, &fork_states, &auth_chain_sets) + .boxed() + .await + else { return Ok(None); }; From 6458f4b1957a54691dc660a1c70e8c89beea402c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 28 Dec 2024 00:57:02 +0000 Subject: [PATCH 1282/2091] refactor various Arc to OwnedEventId Signed-off-by: Jason Volk --- Cargo.lock | 28 +++++++++-------- Cargo.toml | 3 +- src/admin/debug/commands.rs | 10 +++--- src/admin/user/commands.rs | 2 +- src/api/client/redact.rs | 4 +-- src/api/client/send.rs | 2 +- src/api/client/state.rs | 9 ++---- src/api/client/sync/v3.rs | 2 +- src/api/server/send.rs | 11 +++++-- src/api/server/send_leave.rs | 8 ++++- src/core/pdu/builder.rs | 6 ++-- src/core/pdu/event.rs | 6 ++-- src/core/pdu/mod.rs | 14 ++++----- src/core/pdu/redact.rs | 10 +++--- src/service/rooms/auth_chain/mod.rs | 8 ++--- .../fetch_and_handle_outliers.rs | 17 +++++----- src/service/rooms/event_handler/fetch_prev.rs | 12 +++---- .../rooms/event_handler/fetch_state.rs | 31 +++++++------------ .../event_handler/handle_incoming_pdu.rs | 2 +- .../rooms/event_handler/handle_outlier_pdu.rs | 20 +++++------- .../rooms/event_handler/handle_prev_pdu.rs | 14 +++------ src/service/rooms/event_handler/mod.rs | 6 ++-- .../rooms/event_handler/resolve_state.rs | 14 ++++----- .../rooms/event_handler/state_at_incoming.rs | 12 +++---- .../event_handler/upgrade_outlier_pdu.rs | 2 +- src/service/rooms/pdu_metadata/data.rs | 6 +++- src/service/rooms/pdu_metadata/mod.rs | 6 ++-- src/service/rooms/read_receipt/mod.rs | 2 +- src/service/rooms/timeline/mod.rs | 27 ++++++---------- 29 files changed, 142 insertions(+), 152 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9134b290..8b41bf35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3175,7 +3175,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "assign", "js_int", @@ -3197,7 +3197,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "js_int", "ruma-common", @@ -3209,7 +3209,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "as_variant", "assign", @@ -3232,7 +3232,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "as_variant", "base64 0.22.1", @@ -3250,6 +3250,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", + "smallvec", "thiserror 2.0.7", "time", "tracing", @@ -3262,7 +3263,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3276,6 +3277,7 @@ dependencies = [ "ruma-macros", "serde", "serde_json", + "smallvec", "thiserror 2.0.7", "tracing", "url", @@ -3286,7 +3288,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "bytes", "http", @@ -3304,7 +3306,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3313,7 +3315,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "js_int", "ruma-common", @@ -3323,7 +3325,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3338,7 +3340,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "js_int", "ruma-common", @@ -3350,7 +3352,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "headers", "http", @@ -3363,7 +3365,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3379,7 +3381,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d3ed3194ebe96b921d06d1d3e607f0bf7873f792#d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 1546df3a..ea153fda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ features = [ "const_generics", "const_new", "serde", + "union", "write", ] @@ -335,7 +336,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "d3ed3194ebe96b921d06d1d3e607f0bf7873f792" +rev = "54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" features = [ "compat", "rand", diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 0c5a7cbd..85a014b9 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -2,7 +2,6 @@ use std::{ collections::HashMap, fmt::Write, iter::once, - sync::Arc, time::{Instant, SystemTime}, }; @@ -13,7 +12,8 @@ use futures::{FutureExt, StreamExt}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, events::room::message::RoomMessageEventContent, - CanonicalJsonObject, EventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, + CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, + ServerName, }; use service::rooms::state_compressor::HashSetCompressStateEvent; use tracing_subscriber::EnvFilter; @@ -598,14 +598,14 @@ pub(super) async fn force_set_room_state_from_server( let room_version = self.services.rooms.state.get_room_version(&room_id).await?; - let mut state: HashMap> = HashMap::new(); + let mut state: HashMap = HashMap::new(); let remote_state_response = self .services .sending .send_federation_request(&server_name, get_room_state::v1::Request { room_id: room_id.clone().into(), - event_id: first_pdu.event_id.clone().into(), + event_id: first_pdu.event_id.clone(), }) .await?; @@ -677,7 +677,7 @@ pub(super) async fn force_set_room_state_from_server( .services .rooms .event_handler - .resolve_state(room_id.clone().as_ref(), &room_version, state) + .resolve_state(&room_id, &room_version, state) .await?; info!("Forcing new room state"); diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 1cbbf856..5758d937 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -922,7 +922,7 @@ pub(super) async fn redact_event( PduBuilder { redacts: Some(event.event_id.clone()), ..PduBuilder::timeline(&RoomRedactionEventContent { - redacts: Some(event.event_id.clone().into()), + redacts: Some(event.event_id.clone()), reason: Some(reason), }) }, diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index 0b01238f..ba59a010 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -24,7 +24,7 @@ pub(crate) async fn redact_event_route( .timeline .build_and_append_pdu( PduBuilder { - redacts: Some(body.event_id.clone().into()), + redacts: Some(body.event_id.clone()), ..PduBuilder::timeline(&RoomRedactionEventContent { redacts: Some(body.event_id.clone()), reason: body.reason.clone(), @@ -38,5 +38,5 @@ pub(crate) async fn redact_event_route( drop(state_lock); - Ok(redact_event::v3::Response { event_id: event_id.into() }) + Ok(redact_event::v3::Response { event_id }) } diff --git a/src/api/client/send.rs b/src/api/client/send.rs index 9afa7e8c..39340070 100644 --- a/src/api/client/send.rs +++ b/src/api/client/send.rs @@ -92,5 +92,5 @@ pub(crate) async fn send_message_event_route( drop(state_lock); - Ok(send_message_event::v3::Response { event_id: event_id.into() }) + Ok(send_message_event::v3::Response { event_id }) } diff --git a/src/api/client/state.rs b/src/api/client/state.rs index f56444c7..e5a7dd28 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,5 +1,3 @@ -use std::sync::Arc; - use axum::extract::State; use conduwuit::{err, pdu::PduBuilder, utils::BoolExt, Err, Error, PduEvent, Result}; use ruma::{ @@ -16,7 +14,7 @@ use ruma::{ AnyStateEventContent, StateEventType, }, serde::Raw, - EventId, RoomId, UserId, + OwnedEventId, RoomId, UserId, }; use service::Services; @@ -50,8 +48,7 @@ pub(crate) async fn send_state_event_for_key_route( None }, ) - .await? - .into(), + .await?, }) } @@ -177,7 +174,7 @@ async fn send_state_event_for_key_helper( json: &Raw, state_key: String, timestamp: Option, -) -> Result> { +) -> Result { allowed_to_send_state_event(services, room_id, event_type, json).await?; let state_lock = services.rooms.state.mutex.lock(room_id).await; let event_id = services diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index a05bcf98..fbf42f92 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -441,7 +441,7 @@ async fn handle_left_room( // This is just a rejected invite, not a room we know // Insert a leave event anyways let event = PduEvent { - event_id: EventId::new(services.globals.server_name()).into(), + event_id: EventId::new(services.globals.server_name()), sender: sender_user.to_owned(), origin: None, origin_server_ts: utils::millis_since_unix_epoch() diff --git a/src/api/server/send.rs b/src/api/server/send.rs index c5fc7118..dbe0108f 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -6,7 +6,7 @@ use conduwuit::{ debug, debug_warn, err, error, result::LogErr, trace, utils::ReadyExt, warn, Err, Error, Result, }; -use futures::StreamExt; +use futures::{FutureExt, StreamExt}; use ruma::{ api::{ client::error::ErrorKind, @@ -74,8 +74,13 @@ pub(crate) async fn send_transaction_message_route( ); let resolved_map = - handle_pdus(&services, &client, &body.pdus, body.origin(), &txn_start_time).await?; - handle_edus(&services, &client, &body.edus, body.origin()).await; + handle_pdus(&services, &client, &body.pdus, body.origin(), &txn_start_time) + .boxed() + .await?; + + handle_edus(&services, &client, &body.edus, body.origin()) + .boxed() + .await; debug!( pdus = ?body.pdus.len(), diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index 016ed7fa..e955a267 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -2,6 +2,7 @@ use axum::extract::State; use conduwuit::{err, Err, Result}; +use futures::FutureExt; use ruma::{ api::federation::membership::create_leave_event, events::{ @@ -154,10 +155,15 @@ async fn create_leave_event( .rooms .event_handler .handle_incoming_pdu(origin, room_id, &event_id, value, true) + .boxed() .await? .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; drop(mutex_lock); - services.sending.send_pdu_room(room_id, &pdu_id).await + services + .sending + .send_pdu_room(room_id, &pdu_id) + .boxed() + .await } diff --git a/src/core/pdu/builder.rs b/src/core/pdu/builder.rs index db3bb20a..b25d4e9e 100644 --- a/src/core/pdu/builder.rs +++ b/src/core/pdu/builder.rs @@ -1,8 +1,8 @@ -use std::{collections::BTreeMap, sync::Arc}; +use std::collections::BTreeMap; use ruma::{ events::{EventContent, MessageLikeEventType, StateEventType, TimelineEventType}, - EventId, MilliSecondsSinceUnixEpoch, + MilliSecondsSinceUnixEpoch, OwnedEventId, }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; @@ -19,7 +19,7 @@ pub struct Builder { pub state_key: Option, - pub redacts: Option>, + pub redacts: Option, /// For timestamped messaging, should only be used for appservices. /// Will be set to current time if None diff --git a/src/core/pdu/event.rs b/src/core/pdu/event.rs index fb98d686..6a92afe8 100644 --- a/src/core/pdu/event.rs +++ b/src/core/pdu/event.rs @@ -1,13 +1,11 @@ -use std::sync::Arc; - pub use ruma::state_res::Event; -use ruma::{events::TimelineEventType, EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId}; +use ruma::{events::TimelineEventType, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId}; use serde_json::value::RawValue as RawJsonValue; use super::Pdu; impl Event for Pdu { - type Id = Arc; + type Id = OwnedEventId; fn event_id(&self) -> &Self::Id { &self.event_id } diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index c90e174e..ba1d2ce1 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -12,11 +12,11 @@ mod strip; mod tests; mod unsigned; -use std::{cmp::Ordering, sync::Arc}; +use std::cmp::Ordering; use ruma::{ - events::TimelineEventType, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedRoomId, - OwnedUserId, UInt, + events::TimelineEventType, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, + OwnedRoomId, OwnedUserId, UInt, }; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue as RawJsonValue; @@ -35,7 +35,7 @@ use crate::Result; /// Persistent Data Unit (Event) #[derive(Clone, Deserialize, Serialize, Debug)] pub struct Pdu { - pub event_id: Arc, + pub event_id: OwnedEventId, pub room_id: OwnedRoomId, pub sender: OwnedUserId, #[serde(skip_serializing_if = "Option::is_none")] @@ -46,11 +46,11 @@ pub struct Pdu { pub content: Box, #[serde(skip_serializing_if = "Option::is_none")] pub state_key: Option, - pub prev_events: Vec>, + pub prev_events: Vec, pub depth: UInt, - pub auth_events: Vec>, + pub auth_events: Vec, #[serde(skip_serializing_if = "Option::is_none")] - pub redacts: Option>, + pub redacts: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub unsigned: Option>, pub hashes: EventHash, diff --git a/src/core/pdu/redact.rs b/src/core/pdu/redact.rs index dc97bfa8..5d33eeca 100644 --- a/src/core/pdu/redact.rs +++ b/src/core/pdu/redact.rs @@ -1,9 +1,7 @@ -use std::sync::Arc; - use ruma::{ canonical_json::redact_content_in_place, events::{room::redaction::RoomRedactionEventContent, TimelineEventType}, - EventId, RoomVersionId, + OwnedEventId, RoomVersionId, }; use serde::Deserialize; use serde_json::{ @@ -73,15 +71,15 @@ pub fn is_redacted(&self) -> bool { /// > such events over the Client-Server API. #[implement(super::Pdu)] #[must_use] -pub fn copy_redacts(&self) -> (Option>, Box) { +pub fn copy_redacts(&self) -> (Option, Box) { if self.kind == TimelineEventType::RoomRedaction { if let Ok(mut content) = serde_json::from_str::(self.content.get()) { if let Some(redacts) = content.redacts { - return (Some(redacts.into()), self.content.clone()); + return (Some(redacts), self.content.clone()); } else if let Some(redacts) = self.redacts.clone() { - content.redacts = Some(redacts.into()); + content.redacts = Some(redacts); return ( self.redacts.clone(), to_raw_value(&content).expect("Must be valid, we only added redacts field"), diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 87992f2d..796eb5d2 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -12,7 +12,7 @@ use conduwuit::{ validated, warn, Err, Result, }; use futures::{Stream, StreamExt}; -use ruma::{EventId, RoomId}; +use ruma::{EventId, OwnedEventId, RoomId}; use self::data::Data; use crate::{rooms, rooms::short::ShortEventId, Dep}; @@ -46,7 +46,7 @@ impl Service { &'a self, room_id: &RoomId, starting_events: I, - ) -> Result> + Send + '_> + ) -> Result + Send + '_> where I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, { @@ -63,7 +63,7 @@ impl Service { &'a self, room_id: &RoomId, starting_events: I, - ) -> Result>> + ) -> Result> where I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, { @@ -185,7 +185,7 @@ impl Service { room_id: &RoomId, event_id: &EventId, ) -> Result> { - let mut todo = vec![Arc::from(event_id)]; + let mut todo = vec![event_id.to_owned()]; let mut found = HashSet::new(); while let Some(event_id) = todo.pop() { diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index af0ef67d..f980ca24 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -10,7 +10,7 @@ use conduwuit::{ }; use futures::TryFutureExt; use ruma::{ - api::federation::event::get_event, CanonicalJsonValue, EventId, RoomId, RoomVersionId, + api::federation::event::get_event, CanonicalJsonValue, OwnedEventId, RoomId, RoomVersionId, ServerName, }; @@ -27,7 +27,7 @@ use ruma::{ pub(super) async fn fetch_and_handle_outliers<'a>( &self, origin: &'a ServerName, - events: &'a [Arc], + events: &'a [OwnedEventId], create_event: &'a PduEvent, room_id: &'a RoomId, room_version_id: &'a RoomVersionId, @@ -62,7 +62,7 @@ pub(super) async fn fetch_and_handle_outliers<'a>( // c. Ask origin server over federation // We also handle its auth chain here so we don't get a stack overflow in // handle_outlier_pdu. - let mut todo_auth_events = vec![Arc::clone(id)]; + let mut todo_auth_events = vec![id.clone()]; let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); let mut events_all = HashSet::with_capacity(todo_auth_events.len()); while let Some(next_id) = todo_auth_events.pop() { @@ -124,14 +124,15 @@ pub(super) async fn fetch_and_handle_outliers<'a>( ); } - if let Some(auth_events) = value.get("auth_events").and_then(|c| c.as_array()) + if let Some(auth_events) = value + .get("auth_events") + .and_then(CanonicalJsonValue::as_array) { for auth_event in auth_events { if let Ok(auth_event) = - serde_json::from_value(auth_event.clone().into()) + serde_json::from_value::(auth_event.clone().into()) { - let a: Arc = auth_event; - todo_auth_events.push(a); + todo_auth_events.push(auth_event); } else { warn!("Auth event id is not valid"); } @@ -201,7 +202,7 @@ pub(super) async fn fetch_and_handle_outliers<'a>( }, | Err(e) => { warn!("Authentication of event {next_id} failed: {e:?}"); - back_off(next_id.into()); + back_off(next_id); }, } } diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index b271958f..ca93e11d 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -8,7 +8,7 @@ use futures::{future, FutureExt}; use ruma::{ int, state_res::{self}, - uint, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, + uint, CanonicalJsonValue, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, ServerName, }; @@ -23,14 +23,14 @@ pub(super) async fn fetch_prev( create_event: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, - initial_set: Vec>, + initial_set: Vec, ) -> Result<( - Vec>, - HashMap, (Arc, BTreeMap)>, + Vec, + HashMap, BTreeMap)>, )> { - let mut graph: HashMap, _> = HashMap::with_capacity(initial_set.len()); + let mut graph: HashMap = HashMap::with_capacity(initial_set.len()); let mut eventid_info = HashMap::new(); - let mut todo_outlier_stack: Vec> = initial_set; + let mut todo_outlier_stack: Vec = initial_set; let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 9ea3e081..edc47194 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,15 +1,14 @@ -use std::{ - collections::{hash_map, HashMap}, - sync::Arc, -}; +use std::collections::{hash_map, HashMap}; use conduwuit::{debug, implement, warn, Err, Error, PduEvent, Result}; use futures::FutureExt; use ruma::{ - api::federation::event::get_room_state_ids, events::StateEventType, EventId, RoomId, - RoomVersionId, ServerName, + api::federation::event::get_room_state_ids, events::StateEventType, EventId, OwnedEventId, + RoomId, RoomVersionId, ServerName, }; +use crate::rooms::short::ShortStateKey; + /// Call /state_ids to find out what the state at this pdu is. We trust the /// server's response to some extend (sic), but we still do a lot of checks /// on the events @@ -22,31 +21,25 @@ pub(super) async fn fetch_state( room_id: &RoomId, room_version_id: &RoomVersionId, event_id: &EventId, -) -> Result>>> { +) -> Result>> { debug!("Fetching state ids"); let res = self .services .sending .send_federation_request(origin, get_room_state_ids::v1::Request { room_id: room_id.to_owned(), - event_id: (*event_id).to_owned(), + event_id: event_id.to_owned(), }) .await .inspect_err(|e| warn!("Fetching state for event failed: {e}"))?; debug!("Fetching state events"); - let collect = res - .pdu_ids - .iter() - .map(|x| Arc::from(&**x)) - .collect::>(); - let state_vec = self - .fetch_and_handle_outliers(origin, &collect, create_event, room_id, room_version_id) + .fetch_and_handle_outliers(origin, &res.pdu_ids, create_event, room_id, room_version_id) .boxed() .await; - let mut state: HashMap<_, Arc> = HashMap::with_capacity(state_vec.len()); + let mut state: HashMap = HashMap::with_capacity(state_vec.len()); for (pdu, _) in state_vec { let state_key = pdu .state_key @@ -61,10 +54,10 @@ pub(super) async fn fetch_state( match state.entry(shortstatekey) { | hash_map::Entry::Vacant(v) => { - v.insert(Arc::from(&*pdu.event_id)); + v.insert(pdu.event_id.clone()); }, | hash_map::Entry::Occupied(_) => - return Err(Error::bad_database( + return Err!(Database( "State event's type and state_key combination exists multiple times.", )), } @@ -77,7 +70,7 @@ pub(super) async fn fetch_state( .get_shortstatekey(&StateEventType::RoomCreate, "") .await?; - if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(&create_event.event_id) { + if state.get(&create_shortstatekey) != Some(&create_event.event_id) { return Err!(Database("Incoming event refers to wrong create event.")); } diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index ca56228d..0e0409b4 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -147,7 +147,7 @@ pub async fn handle_incoming_pdu<'a>( .bad_event_ratelimiter .write() .expect("locked") - .entry(prev_id.into()) + .entry(prev_id) { | Entry::Vacant(e) => { e.insert((now, 1)); diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 59ef27ba..c3278329 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -79,19 +79,13 @@ pub(super) async fn handle_outlier_pdu<'a>( // the auth events are also rejected "due to auth events" // NOTE: Step 5 is not applied anymore because it failed too often debug!("Fetching auth events"); - Box::pin( - self.fetch_and_handle_outliers( - origin, - &incoming_pdu - .auth_events - .iter() - .map(|x| Arc::from(&**x)) - .collect::>>(), - create_event, - room_id, - &room_version_id, - ), - ) + Box::pin(self.fetch_and_handle_outliers( + origin, + &incoming_pdu.auth_events, + create_event, + room_id, + &room_version_id, + )) .await; } diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index becaeb17..a8893160 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -5,9 +5,9 @@ use std::{ }; use conduwuit::{ - debug, implement, utils::math::continue_exponential_backoff_secs, Error, PduEvent, Result, + debug, implement, utils::math::continue_exponential_backoff_secs, Err, PduEvent, Result, }; -use ruma::{api::client::error::ErrorKind, CanonicalJsonValue, EventId, RoomId, ServerName}; +use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName}; #[implement(super::Service)] #[allow(clippy::type_complexity)] @@ -22,7 +22,7 @@ pub(super) async fn handle_prev_pdu<'a>( event_id: &'a EventId, room_id: &'a RoomId, eventid_info: &mut HashMap< - Arc, + OwnedEventId, (Arc, BTreeMap), >, create_event: &Arc, @@ -31,14 +31,10 @@ pub(super) async fn handle_prev_pdu<'a>( ) -> Result { // Check for disabled again because it might have changed if self.services.metadata.is_disabled(room_id).await { - debug!( + return Err!(Request(Forbidden(debug_warn!( "Federaton of room {room_id} is currently disabled on this server. Request by \ origin {origin} and event ID {event_id}" - ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Federation of this room is currently disabled on this server.", - )); + )))); } if let Some((time, tries)) = self diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index bfc5a014..8bcbc48b 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -23,7 +23,7 @@ use conduwuit::{ }; use futures::TryFutureExt; use ruma::{ - events::room::create::RoomCreateEventContent, state_res::RoomVersion, EventId, OwnedEventId, + events::room::create::RoomCreateEventContent, state_res::RoomVersion, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, }; @@ -97,11 +97,11 @@ impl crate::Service for Service { } impl Service { - async fn event_exists(&self, event_id: Arc) -> bool { + async fn event_exists(&self, event_id: OwnedEventId) -> bool { self.services.timeline.pdu_exists(&event_id).await } - async fn event_fetch(&self, event_id: Arc) -> Option> { + async fn event_fetch(&self, event_id: OwnedEventId) -> Option> { self.services .timeline .get_pdu(&event_id) diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 108be18d..b47111d9 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -12,7 +12,7 @@ use conduwuit::{ use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{ state_res::{self, StateMap}, - EventId, RoomId, RoomVersionId, + OwnedEventId, RoomId, RoomVersionId, }; use crate::rooms::state_compressor::CompressedStateEvent; @@ -23,7 +23,7 @@ pub async fn resolve_state( &self, room_id: &RoomId, room_version_id: &RoomVersionId, - incoming_state: HashMap>, + incoming_state: HashMap, ) -> Result>> { debug!("Loading current room state ids"); let current_sstatehash = self @@ -44,7 +44,7 @@ pub async fn resolve_state( for state in &fork_states { let starting_events = state.values().map(Borrow::borrow); - let auth_chain: HashSet> = self + let auth_chain: HashSet = self .services .auth_chain .get_event_ids(room_id, starting_events) @@ -56,7 +56,7 @@ pub async fn resolve_state( } debug!("Loading fork states"); - let fork_states: Vec>> = fork_states + let fork_states: Vec> = fork_states .into_iter() .stream() .wide_then(|fork_state| { @@ -113,9 +113,9 @@ pub async fn resolve_state( pub async fn state_resolution( &self, room_version: &RoomVersionId, - state_sets: &[StateMap>], - auth_chain_sets: &Vec>>, -) -> Result>> { + state_sets: &[StateMap], + auth_chain_sets: &Vec>, +) -> Result> { //TODO: ??? let _lock = self.services.globals.stateres_mutex.lock(); diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 658fb904..fa2ce1cd 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -11,7 +11,7 @@ use conduwuit::{ PduEvent, Result, }; use futures::{FutureExt, StreamExt}; -use ruma::{state_res::StateMap, EventId, RoomId, RoomVersionId}; +use ruma::{state_res::StateMap, OwnedEventId, RoomId, RoomVersionId}; // TODO: if we know the prev_events of the incoming event we can avoid the #[implement(super::Service)] @@ -20,8 +20,8 @@ use ruma::{state_res::StateMap, EventId, RoomId, RoomVersionId}; pub(super) async fn state_at_incoming_degree_one( &self, incoming_pdu: &Arc, -) -> Result>>> { - let prev_event = &*incoming_pdu.prev_events[0]; +) -> Result>> { + let prev_event = &incoming_pdu.prev_events[0]; let Ok(prev_event_sstatehash) = self .services .state_accessor @@ -56,7 +56,7 @@ pub(super) async fn state_at_incoming_degree_one( .get_or_create_shortstatekey(&prev_pdu.kind.to_string().into(), state_key) .await; - state.insert(shortstatekey, Arc::from(prev_event)); + state.insert(shortstatekey, prev_event.clone()); // Now it's the state after the pdu } @@ -72,7 +72,7 @@ pub(super) async fn state_at_incoming_resolved( incoming_pdu: &Arc, room_id: &RoomId, room_version_id: &RoomVersionId, -) -> Result>>> { +) -> Result>> { debug!("Calculating state at event using state res"); let mut extremity_sstatehashes = HashMap::with_capacity(incoming_pdu.prev_events.len()); @@ -142,7 +142,7 @@ pub(super) async fn state_at_incoming_resolved( starting_events.push(id.borrow()); } - let auth_chain: HashSet> = self + let auth_chain: HashSet = self .services .auth_chain .get_event_ids(room_id, starting_events.into_iter()) diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index aa484cd6..8adf4246 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -282,7 +282,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( } trace!("Appending pdu to timeline"); - extremities.insert(incoming_pdu.event_id.clone().into()); + extremities.insert(incoming_pdu.event_id.clone()); // Now that the event has passed all auth it is added into the timeline. // We use the `state_at_event` instead of `state_after` so we accurately diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 01950975..2e6ecbb5 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -88,7 +88,11 @@ impl Data { }) } - pub(super) fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) { + #[inline] + pub(super) fn mark_as_referenced<'a, I>(&self, room_id: &RoomId, event_ids: I) + where + I: Iterator, + { for prev in event_ids { let key = (room_id, prev); self.referencedevents.put_raw(key, []); diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index ac70dbc3..4cb14ebc 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -98,9 +98,11 @@ impl Service { pdus } - #[inline] #[tracing::instrument(skip_all, level = "debug")] - pub fn mark_as_referenced(&self, room_id: &RoomId, event_ids: &[Arc]) { + pub fn mark_as_referenced<'a, I>(&self, room_id: &RoomId, event_ids: I) + where + I: Iterator, + { self.db.mark_as_referenced(room_id, event_ids); } diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 53e64957..9777faeb 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -77,7 +77,7 @@ impl Service { let pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await?; - let event_id: OwnedEventId = pdu.event_id.into(); + let event_id: OwnedEventId = pdu.event_id; let user_id: OwnedUserId = user_id.to_owned(); let content: BTreeMap = BTreeMap::from_iter([( event_id, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 3b81e2e5..a2f1ab1b 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -10,7 +10,7 @@ use std::{ use conduwuit::{ debug, debug_warn, err, error, implement, info, - pdu::{EventHash, PduBuilder, PduCount, PduEvent}, + pdu::{gen_event_id, EventHash, PduBuilder, PduCount, PduEvent}, utils::{self, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, validated, warn, Err, Error, Result, Server, }; @@ -371,7 +371,7 @@ impl Service { // We must keep track of all events that have been referenced. self.services .pdu_metadata - .mark_as_referenced(&pdu.room_id, &pdu.prev_events); + .mark_as_referenced(&pdu.room_id, pdu.prev_events.iter().map(AsRef::as_ref)); self.services .state @@ -681,12 +681,12 @@ impl Service { timestamp, } = pdu_builder; - let prev_events: Vec<_> = self + let prev_events: Vec = self .services .state .get_forward_extremities(room_id) .take(20) - .map(Arc::from) + .map(Into::into) .collect() .await; @@ -834,17 +834,10 @@ impl Service { } // Generate event id - pdu.event_id = EventId::parse_arc(format!( - "${}", - ruma::signatures::reference_hash(&pdu_json, &room_version_id) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); + pdu.event_id = gen_event_id(&pdu_json, &room_version_id)?; - pdu_json.insert( - "event_id".to_owned(), - CanonicalJsonValue::String(pdu.event_id.as_str().to_owned()), - ); + pdu_json + .insert("event_id".into(), CanonicalJsonValue::String(pdu.event_id.clone().into())); // Generate short event id let _shorteventid = self @@ -867,7 +860,7 @@ impl Service { room_id: &RoomId, state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room state * mutex */ - ) -> Result> { + ) -> Result { let (pdu, pdu_json) = self .create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock) .await?; @@ -987,7 +980,7 @@ impl Service { if soft_fail { self.services .pdu_metadata - .mark_as_referenced(&pdu.room_id, &pdu.prev_events); + .mark_as_referenced(&pdu.room_id, pdu.prev_events.iter().map(AsRef::as_ref)); self.services .state @@ -1170,7 +1163,7 @@ impl Service { backfill_server, federation::backfill::get_backfill::v1::Request { room_id: room_id.to_owned(), - v: vec![first_pdu.1.event_id.as_ref().to_owned()], + v: vec![first_pdu.1.event_id.clone()], limit: uint!(100), }, ) From 74eb30c106dcd88e97ba1367a29bba9669e5d9d0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 25 Dec 2024 06:17:18 +0000 Subject: [PATCH 1283/2091] add LIFO scheduling for database frontend pool workers Signed-off-by: Jason Volk --- Cargo.lock | 6 ++---- Cargo.toml | 8 ++++++++ src/database/pool.rs | 9 +++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b41bf35..0404f778 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,8 +95,7 @@ checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" [[package]] name = "async-channel" version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b47800b0be77592da0afd425cc03468052844aff33b84e33cc696f64e77b6a" +source = "git+https://github.com/jevolk/async-channel?rev=fefa543ca5eddf21237d75776fce98b7e09e924a#fefa543ca5eddf21237d75776fce98b7e09e924a" dependencies = [ "concurrent-queue", "event-listener-strategy", @@ -1259,8 +1258,7 @@ dependencies = [ [[package]] name = "event-listener" version = "5.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6032be9bd27023a771701cc49f9f053c751055f71efb2e0ae5c15809093675ba" +source = "git+https://github.com/jevolk/event-listener?rev=96d7e0fc026d8f708b19bc9267a382676a50354c#96d7e0fc026d8f708b19bc9267a382676a50354c" dependencies = [ "concurrent-queue", "parking", diff --git a/Cargo.toml b/Cargo.toml index ea153fda..36f6c1ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -529,6 +529,14 @@ rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d" git = "https://github.com/girlbossceo/rustyline-async" rev = "deaeb0694e2083f53d363b648da06e10fc13900c" +# adds LIFO queue scheduling; this should be updated with PR progress. +[patch.crates-io.event-listener] +git = "https://github.com/jevolk/event-listener" +rev = "96d7e0fc026d8f708b19bc9267a382676a50354c" +[patch.crates-io.async-channel] +git = "https://github.com/jevolk/async-channel" +rev = "fefa543ca5eddf21237d75776fce98b7e09e924a" + # # Our crates # diff --git a/src/database/pool.rs b/src/database/pool.rs index 51e705ce..1c55c456 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -8,7 +8,7 @@ use std::{ }, }; -use async_channel::{Receiver, RecvError, Sender}; +use async_channel::{Receiver, RecvError, Sched, Sender}; use conduwuit::{ debug, debug_warn, defer, err, implement, result::DebugInspect, @@ -65,9 +65,14 @@ const QUEUE_LIMIT: (usize, usize) = (1, 2048); #[implement(Pool)] pub(crate) async fn new(server: &Arc) -> Result> { + const CHAN_SCHED: (Sched, Sched) = (Sched::Fifo, Sched::Lifo); + let (total_workers, queue_sizes, topology) = configure(server); - let (senders, receivers) = queue_sizes.into_iter().map(async_channel::bounded).unzip(); + let (senders, receivers) = queue_sizes + .into_iter() + .map(|cap| async_channel::bounded_with_sched(cap, CHAN_SCHED)) + .unzip(); let pool = Arc::new(Self { server: server.clone(), From e21403a4d4abba20d48c5801a703efa5cde82452 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 28 Dec 2024 02:51:30 +0000 Subject: [PATCH 1284/2091] use VecDeque for todo queues Signed-off-by: Jason Volk --- src/service/rooms/auth_chain/mod.rs | 9 +++++---- .../rooms/event_handler/fetch_and_handle_outliers.rs | 8 ++++---- src/service/rooms/event_handler/fetch_prev.rs | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 796eb5d2..67883d01 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -1,7 +1,7 @@ mod data; use std::{ - collections::{BTreeSet, HashSet}, + collections::{BTreeSet, HashSet, VecDeque}, fmt::Debug, sync::Arc, }; @@ -185,10 +185,10 @@ impl Service { room_id: &RoomId, event_id: &EventId, ) -> Result> { - let mut todo = vec![event_id.to_owned()]; + let mut todo: VecDeque<_> = [event_id.to_owned()].into(); let mut found = HashSet::new(); - while let Some(event_id) = todo.pop() { + while let Some(event_id) = todo.pop_front() { trace!(?event_id, "processing auth event"); match self.services.timeline.get_pdu(&event_id).await { @@ -218,7 +218,8 @@ impl Service { ?auth_event, "adding auth event to processing queue" ); - todo.push(auth_event.clone()); + + todo.push_back(auth_event.clone()); } } }, diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index f980ca24..1bad4659 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -1,5 +1,5 @@ use std::{ - collections::{hash_map, BTreeMap, HashSet}, + collections::{hash_map, BTreeMap, HashSet, VecDeque}, sync::Arc, time::Instant, }; @@ -62,10 +62,10 @@ pub(super) async fn fetch_and_handle_outliers<'a>( // c. Ask origin server over federation // We also handle its auth chain here so we don't get a stack overflow in // handle_outlier_pdu. - let mut todo_auth_events = vec![id.clone()]; + let mut todo_auth_events: VecDeque<_> = [id.clone()].into(); let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); let mut events_all = HashSet::with_capacity(todo_auth_events.len()); - while let Some(next_id) = todo_auth_events.pop() { + while let Some(next_id) = todo_auth_events.pop_front() { if let Some((time, tries)) = self .services .globals @@ -132,7 +132,7 @@ pub(super) async fn fetch_and_handle_outliers<'a>( if let Ok(auth_event) = serde_json::from_value::(auth_event.clone().into()) { - todo_auth_events.push(auth_event); + todo_auth_events.push_back(auth_event); } else { warn!("Auth event id is not valid"); } diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index ca93e11d..3f121f69 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,5 +1,5 @@ use std::{ - collections::{BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet, VecDeque}, sync::Arc, }; @@ -30,13 +30,13 @@ pub(super) async fn fetch_prev( )> { let mut graph: HashMap = HashMap::with_capacity(initial_set.len()); let mut eventid_info = HashMap::new(); - let mut todo_outlier_stack: Vec = initial_set; + let mut todo_outlier_stack: VecDeque = initial_set.into(); let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; let mut amount = 0; - while let Some(prev_event_id) = todo_outlier_stack.pop() { + while let Some(prev_event_id) = todo_outlier_stack.pop_front() { self.services.server.check_running()?; if let Some((pdu, mut json_opt)) = self @@ -74,7 +74,7 @@ pub(super) async fn fetch_prev( amount = amount.saturating_add(1); for prev_prev in &pdu.prev_events { if !graph.contains_key(prev_prev) { - todo_outlier_stack.push(prev_prev.clone()); + todo_outlier_stack.push_back(prev_prev.clone()); } } From 3a2c5be4f4d6b74527ae40ae05f9a5b0e786314e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 28 Dec 2024 04:51:19 +0000 Subject: [PATCH 1285/2091] reduce exponential backoff for prev fetch Signed-off-by: Jason Volk --- src/service/rooms/event_handler/fetch_and_handle_outliers.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 1bad4659..5c561789 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -75,8 +75,8 @@ pub(super) async fn fetch_and_handle_outliers<'a>( .get(&*next_id) { // Exponential backoff - const MIN_DURATION: u64 = 5 * 60; - const MAX_DURATION: u64 = 60 * 60 * 24; + const MIN_DURATION: u64 = 60 * 2; + const MAX_DURATION: u64 = 60 * 60 * 8; if continue_exponential_backoff_secs( MIN_DURATION, MAX_DURATION, From d91570d0e61d7c06b0a4659b3803ced3e52ccf35 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 18 Dec 2024 20:48:40 +0000 Subject: [PATCH 1286/2091] add Error variant for FeatureDisabled Signed-off-by: Jason Volk --- src/core/error/mod.rs | 6 +++++- src/core/error/response.rs | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 12ba0797..f38178e6 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -98,6 +98,8 @@ pub enum Error { ContentDisposition(#[from] ruma::http_headers::ContentDispositionParseError), #[error("{0}")] Database(Cow<'static, str>), + #[error("Feature '{0}' is not available on this server.")] + FeatureDisabled(Cow<'static, str>), #[error("Remote server {0} responded with: {1}")] Federation(ruma::OwnedServerName, ruma::api::client::error::Error), #[error("{0} in {1}")] @@ -153,12 +155,13 @@ impl Error { /// Returns the Matrix error code / error kind #[inline] pub fn kind(&self) -> ruma::api::client::error::ErrorKind { - use ruma::api::client::error::ErrorKind::Unknown; + use ruma::api::client::error::ErrorKind::{FeatureDisabled, Unknown}; match self { | Self::Federation(_, error) | Self::Ruma(error) => response::ruma_error_kind(error).clone(), | Self::BadRequest(kind, ..) | Self::Request(kind, ..) => kind.clone(), + | Self::FeatureDisabled(..) => FeatureDisabled, | _ => Unknown, } } @@ -172,6 +175,7 @@ impl Error { | Self::Federation(_, error) | Self::Ruma(error) => error.status_code, | Self::Request(kind, _, code) => response::status_code(kind, *code), | Self::BadRequest(kind, ..) => response::bad_request_code(kind), + | Self::FeatureDisabled(..) => response::bad_request_code(&self.kind()), | Self::Reqwest(error) => error.status().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), | Self::Conflict(_) => StatusCode::CONFLICT, | _ => StatusCode::INTERNAL_SERVER_ERROR, diff --git a/src/core/error/response.rs b/src/core/error/response.rs index 568238c3..335fddab 100644 --- a/src/core/error/response.rs +++ b/src/core/error/response.rs @@ -66,7 +66,7 @@ pub(super) fn bad_request_code(kind: &ErrorKind) -> StatusCode { | Unrecognized => StatusCode::METHOD_NOT_ALLOWED, // 404 - | NotFound => StatusCode::NOT_FOUND, + | NotFound | NotImplemented | FeatureDisabled => StatusCode::NOT_FOUND, // 403 | GuestAccessForbidden From 0a9b6c136fcc29d4f72996dc7eaf7ed4948e820a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 28 Dec 2024 23:31:24 +0000 Subject: [PATCH 1287/2091] refactor for ruma identifiers optimizations Signed-off-by: Jason Volk --- Cargo.lock | 26 ++++++------ Cargo.toml | 2 +- src/admin/debug/commands.rs | 2 +- src/admin/room/alias.rs | 5 +-- src/admin/room/moderation.rs | 28 ++++++------- src/api/client/room/create.rs | 40 +++++++++---------- src/api/client/session.rs | 6 +-- src/api/client/sync/v3.rs | 10 ++--- src/api/client/sync/v4.rs | 4 +- src/api/router/auth.rs | 2 +- src/api/server/invite.rs | 4 +- src/service/globals/mod.rs | 6 +-- src/service/rooms/alias/mod.rs | 6 +-- .../rooms/event_handler/parse_incoming_pdu.rs | 4 +- src/service/rooms/state/mod.rs | 2 +- src/service/rooms/timeline/mod.rs | 10 ++--- src/service/sending/data.rs | 8 ++-- src/service/sending/sender.rs | 2 +- src/service/users/mod.rs | 2 +- 19 files changed, 81 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0404f778..5b00a313 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3173,7 +3173,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "assign", "js_int", @@ -3195,7 +3195,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "js_int", "ruma-common", @@ -3207,7 +3207,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "as_variant", "assign", @@ -3230,7 +3230,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "as_variant", "base64 0.22.1", @@ -3261,7 +3261,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3286,7 +3286,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "bytes", "http", @@ -3304,7 +3304,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3313,7 +3313,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "js_int", "ruma-common", @@ -3323,7 +3323,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3338,7 +3338,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "js_int", "ruma-common", @@ -3350,7 +3350,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "headers", "http", @@ -3363,7 +3363,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3379,7 +3379,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a#54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 36f6c1ad..24e6eb79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -336,7 +336,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "54da128bbe09a0c6d14c9a6bf7b6b54a2d7c835a" +rev = "30a08ff7be779df6858036c3f01f3ba64c1ee785" features = [ "compat", "rand", diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 85a014b9..5bc65d9b 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -75,7 +75,7 @@ pub(super) async fn parse_pdu(&self) -> Result { match serde_json::from_str(&string) { | Ok(value) => match ruma::signatures::reference_hash(&value, &RoomVersionId::V6) { | Ok(hash) => { - let event_id = EventId::parse(format!("${hash}")); + let event_id = OwnedEventId::parse(format!("${hash}")); match serde_json::from_value::( serde_json::to_value(value).expect("value is json"), diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 835138b2..4490600d 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -4,8 +4,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; use ruma::{ - events::room::message::RoomMessageEventContent, OwnedRoomAliasId, OwnedRoomId, RoomAliasId, - RoomId, + events::room::message::RoomMessageEventContent, OwnedRoomAliasId, OwnedRoomId, RoomId, }; use crate::{escape_html, Command}; @@ -58,7 +57,7 @@ pub(super) async fn process( | RoomAliasCommand::Which { ref room_alias_localpart } => { let room_alias_str = format!("#{}:{}", room_alias_localpart, services.globals.server_name()); - let room_alias = match RoomAliasId::parse_box(room_alias_str) { + let room_alias = match OwnedRoomAliasId::parse(room_alias_str) { | Ok(alias) => alias, | Err(err) => return Ok(RoomMessageEventContent::text_plain(format!( diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index b16aff1f..bf54505e 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -105,10 +105,9 @@ async fn ban_room( }; debug!("Room specified is a room ID, banning room ID"); + self.services.rooms.metadata.ban_room(room_id, true); - self.services.rooms.metadata.ban_room(&room_id, true); - - room_id + room_id.to_owned() } else if room.is_room_alias_id() { let room_alias = match RoomAliasId::parse(&room) { | Ok(room_alias) => room_alias, @@ -129,7 +128,7 @@ async fn ban_room( .services .rooms .alias - .resolve_local_alias(&room_alias) + .resolve_local_alias(room_alias) .await { room_id @@ -143,20 +142,20 @@ async fn ban_room( .services .rooms .alias - .resolve_alias(&room_alias, None) + .resolve_alias(room_alias, None) .await { | Ok((room_id, servers)) => { debug!( ?room_id, ?servers, - "Got federation response fetching room ID for {room}" + "Got federation response fetching room ID for {room_id}" ); room_id }, | Err(e) => { return Ok(RoomMessageEventContent::notice_plain(format!( - "Failed to resolve room alias {room} to a room ID: {e}" + "Failed to resolve room alias {room_alias} to a room ID: {e}" ))); }, } @@ -316,7 +315,7 @@ async fn ban_list_of_rooms( }, }; - room_ids.push(room_id); + room_ids.push(room_id.to_owned()); } if room_alias_or_id.is_room_alias_id() { @@ -326,7 +325,7 @@ async fn ban_list_of_rooms( .services .rooms .alias - .resolve_local_alias(&room_alias) + .resolve_local_alias(room_alias) .await { room_id @@ -340,7 +339,7 @@ async fn ban_list_of_rooms( .services .rooms .alias - .resolve_alias(&room_alias, None) + .resolve_alias(room_alias, None) .await { | Ok((room_id, servers)) => { @@ -519,10 +518,9 @@ async fn unban_room( }; debug!("Room specified is a room ID, unbanning room ID"); + self.services.rooms.metadata.ban_room(room_id, false); - self.services.rooms.metadata.ban_room(&room_id, false); - - room_id + room_id.to_owned() } else if room.is_room_alias_id() { let room_alias = match RoomAliasId::parse(&room) { | Ok(room_alias) => room_alias, @@ -543,7 +541,7 @@ async fn unban_room( .services .rooms .alias - .resolve_local_alias(&room_alias) + .resolve_local_alias(room_alias) .await { room_id @@ -557,7 +555,7 @@ async fn unban_room( .services .rooms .alias - .resolve_alias(&room_alias, None) + .resolve_alias(room_alias, None) .await { | Ok((room_id, servers)) => { diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index 89cddc0f..a78242ca 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -1,7 +1,9 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduwuit::{debug_info, debug_warn, error, info, pdu::PduBuilder, warn, Err, Error, Result}; +use conduwuit::{ + debug_info, debug_warn, err, error, info, pdu::PduBuilder, warn, Err, Error, Result, +}; use futures::FutureExt; use ruma::{ api::client::{ @@ -24,8 +26,7 @@ use ruma::{ }, int, serde::{JsonObject, Raw}, - CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomAliasId, RoomId, - RoomVersionId, + CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId, }; use serde_json::{json, value::to_raw_value}; use service::{appservice::RegistrationInfo, Services}; @@ -554,14 +555,15 @@ async fn room_alias_check( return Err(Error::BadRequest(ErrorKind::Unknown, "Room alias name is forbidden.")); } - let full_room_alias = RoomAliasId::parse(format!( - "#{}:{}", - room_alias_name, services.globals.config.server_name - )) - .map_err(|e| { - info!("Failed to parse room alias {room_alias_name}: {e}"); - Error::BadRequest(ErrorKind::InvalidParam, "Invalid room alias specified.") - })?; + let server_name = services.globals.server_name(); + let full_room_alias = OwnedRoomAliasId::parse(format!("#{room_alias_name}:{server_name}")) + .map_err(|e| { + err!(Request(InvalidParam(debug_error!( + ?e, + ?room_alias_name, + "Failed to parse room alias.", + )))) + })?; if services .rooms @@ -620,15 +622,11 @@ fn custom_room_id_check(services: &Services, custom_room_id: &str) -> Result { // A new user joined an encrypted room - if !share_encrypted_room(services, sender_user, &user_id, Some(room_id)) + if !share_encrypted_room(services, sender_user, user_id, Some(room_id)) .await { - device_list_updates.insert(user_id); + device_list_updates.insert(user_id.into()); } }, | MembershipState::Leave => { // Write down users that have left encrypted rooms we are in - left_encrypted_users.insert(user_id); + left_encrypted_users.insert(user_id.into()); }, | _ => {}, } diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 0c6ea650..9915752e 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -30,7 +30,7 @@ use ruma::{ TimelineEventType::{self, *}, }, serde::Raw, - uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, UInt, UserId, + uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, UInt, }; use service::{rooms::read_receipt::pack_receipts, Services}; @@ -243,7 +243,7 @@ pub(crate) async fn sync_events_v4_route( if pdu.kind == RoomMember { if let Some(state_key) = &pdu.state_key { let user_id = - UserId::parse(state_key.clone()).map_err(|_| { + OwnedUserId::parse(state_key.clone()).map_err(|_| { Error::bad_database("Invalid UserId in member PDU.") })?; diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index c5b040e0..dd25e091 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -212,7 +212,7 @@ async fn auth_appservice( .query .user_id .clone() - .map_or_else(user_id_default, UserId::parse) + .map_or_else(user_id_default, OwnedUserId::parse) else { return Err!(Request(InvalidUsername("Username is invalid."))); }; diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index a5b4a11c..6d3be04c 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -6,7 +6,7 @@ use ruma::{ api::{client::error::ErrorKind, federation::membership::create_invite}, events::room::member::{MembershipState, RoomMemberEventContent}, serde::JsonObject, - CanonicalJsonValue, EventId, OwnedUserId, UserId, + CanonicalJsonValue, OwnedEventId, OwnedUserId, UserId, }; use crate::Ruma; @@ -86,7 +86,7 @@ pub(crate) async fn create_invite_route( .map_err(|e| err!(Request(InvalidParam("Failed to sign event: {e}"))))?; // Generate event id - let event_id = EventId::parse(format!( + let event_id = OwnedEventId::parse(format!( "${}", ruma::signatures::reference_hash(&signed_event, &body.room_version) .expect("ruma can calculate reference hashes") diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 88199f5f..3f98babe 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -10,9 +10,7 @@ use std::{ use conduwuit::{error, Config, Result}; use data::Data; use regex::RegexSet; -use ruma::{ - OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, RoomAliasId, ServerName, UserId, -}; +use ruma::{OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, ServerName, UserId}; use tokio::sync::Mutex; use crate::service; @@ -73,7 +71,7 @@ impl crate::Service for Service { jwt_decoding_key, bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), stateres_mutex: Arc::new(Mutex::new(())), - admin_alias: RoomAliasId::parse(format!("#admins:{}", &config.server_name)) + admin_alias: OwnedRoomAliasId::try_from(format!("#admins:{}", &config.server_name)) .expect("#admins:server_name is valid alias name"), server_user: UserId::parse_with_server_name( String::from("conduit"), diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 9dcf9d8e..0acbb116 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -129,10 +129,10 @@ impl Service { servers: Option>, ) -> Result<(OwnedRoomId, Vec)> { if room.is_room_id() { - let room_id = RoomId::parse(room).expect("valid RoomId"); - Ok((room_id, servers.unwrap_or_default())) + let room_id: &RoomId = room.try_into().expect("valid RoomId"); + Ok((room_id.to_owned(), servers.unwrap_or_default())) } else { - let alias = &RoomAliasId::parse(room).expect("valid RoomAliasId"); + let alias: &RoomAliasId = room.try_into().expect("valid RoomAliasId"); self.resolve_alias(alias, servers).await } } diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index f3c75f36..0c11314d 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,5 +1,5 @@ use conduwuit::{err, implement, pdu::gen_event_id_canonical_json, result::FlatOk, Result}; -use ruma::{CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, RoomId}; +use ruma::{CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId}; use serde_json::value::RawValue as RawJsonValue; #[implement(super::Service)] @@ -14,7 +14,7 @@ pub async fn parse_incoming_pdu( let room_id: OwnedRoomId = value .get("room_id") .and_then(CanonicalJsonValue::as_str) - .map(RoomId::parse) + .map(OwnedRoomId::parse) .flat_ok_or(err!(Request(InvalidParam("Invalid room_id in pdu"))))?; let room_version_id = self diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index a7f79e94..2769beb8 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -139,7 +139,7 @@ impl Service { .state_cache .update_membership( room_id, - &user_id, + user_id, membership_event, &pdu.sender, None, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index a2f1ab1b..81df7b35 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -35,7 +35,7 @@ use ruma::{ push::{Action, Ruleset, Tweak}, state_res::{self, Event, RoomVersion}, uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, - OwnedServerName, RoomId, RoomVersionId, ServerName, UserId, + OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; @@ -424,7 +424,7 @@ impl Service { if pdu.kind == TimelineEventType::RoomMember { if let Some(state_key) = &pdu.state_key { - let target_user_id = UserId::parse(state_key.clone())?; + let target_user_id = OwnedUserId::parse(state_key)?; if self.services.users.is_active_local(&target_user_id).await { push_target.insert(target_user_id); @@ -534,7 +534,7 @@ impl Service { | TimelineEventType::RoomMember => { if let Some(state_key) = &pdu.state_key { // if the state_key fails - let target_user_id = UserId::parse(state_key.clone()) + let target_user_id = UserId::parse(state_key) .expect("This state_key was previously validated"); let content: RoomMemberEventContent = pdu.get_content()?; @@ -550,7 +550,7 @@ impl Service { .state_cache .update_membership( &pdu.room_id, - &target_user_id, + target_user_id, content, &pdu.sender, invite_state, @@ -627,7 +627,7 @@ impl Service { .and_then(|state_key| UserId::parse(state_key.as_str()).ok()) { let appservice_uid = appservice.registration.sender_localpart.as_str(); - if state_key_uid == appservice_uid { + if state_key_uid == &appservice_uid { self.services .sending .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index a699b8ee..436f633e 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -7,7 +7,7 @@ use conduwuit::{ }; use database::{Database, Deserialized, Map}; use futures::{Stream, StreamExt}; -use ruma::{ServerName, UserId}; +use ruma::{OwnedServerName, ServerName, UserId}; use super::{Destination, SendingEvent}; use crate::{globals, Dep}; @@ -209,7 +209,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se let mut parts = key[1..].splitn(3, |&b| b == 0xFF); let user = parts.next().expect("splitn always returns one element"); - let user_string = utils::string_from_bytes(user) + let user_string = utils::str_from_bytes(user) .map_err(|_| Error::bad_database("Invalid user string in servercurrentevent"))?; let user_id = UserId::parse(user_string) .map_err(|_| Error::bad_database("Invalid user id in servercurrentevent"))?; @@ -225,7 +225,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?; ( - Destination::Push(user_id, pushkey_string), + Destination::Push(user_id.to_owned(), pushkey_string), if value.is_empty() { SendingEvent::Pdu(event.into()) } else { @@ -246,7 +246,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se })?; ( - Destination::Federation(ServerName::parse(server).map_err(|_| { + Destination::Federation(OwnedServerName::parse(&server).map_err(|_| { Error::bad_database("Invalid server string in server_currenttransaction") })?), if value.is_empty() { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 339236a5..bf4ebafb 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -883,7 +883,7 @@ impl Service { .get("room_id") .and_then(|val| RoomId::parse(val.as_str()?).ok()) { - match self.services.state.get_room_version(&room_id).await { + match self.services.state.get_room_version(room_id).await { | Ok(room_version_id) => match room_version_id { | RoomVersionId::V1 | RoomVersionId::V2 => {}, | _ => _ = pdu_json.remove("event_id"), diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 1d7f4248..fe064d9c 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -937,7 +937,7 @@ impl Service { let user_string = utils::string_from_bytes(user_bytes) .map_err(|e| err!(Database("User ID in openid_userid is invalid unicode. {e}")))?; - UserId::parse(user_string) + OwnedUserId::try_from(user_string) .map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) } From 9eb99f8070db4711845592287fd0a793fdd7cd7f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 29 Dec 2024 02:21:22 +0000 Subject: [PATCH 1288/2091] abstract async shutdown waiting loop Signed-off-by: Jason Volk --- src/core/server.rs | 12 +++++++++++- src/service/sync/watch.rs | 9 ++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/core/server.rs b/src/core/server.rs index 2a70ae4b..8a4d9f66 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -1,5 +1,8 @@ use std::{ - sync::atomic::{AtomicBool, Ordering}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, time::SystemTime, }; @@ -102,6 +105,13 @@ impl Server { Ok(()) } + #[inline] + pub async fn until_shutdown(self: Arc) { + while self.running() { + self.signal.subscribe().recv().await.ok(); + } + } + #[inline] pub fn runtime(&self) -> &runtime::Handle { self.runtime diff --git a/src/service/sync/watch.rs b/src/service/sync/watch.rs index 50959ded..2b351c3a 100644 --- a/src/service/sync/watch.rs +++ b/src/service/sync/watch.rs @@ -97,13 +97,8 @@ pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result { ); // Server shutdown - let server_shutdown = async move { - while self.services.server.running() { - self.services.server.signal.subscribe().recv().await.ok(); - } - }; - - futures.push(server_shutdown.boxed()); + let server_shutdown = self.services.server.clone().until_shutdown().boxed(); + futures.push(server_shutdown); if !self.services.server.running() { return Ok(()); } From 7c8eeaf4ea117645de6af9c8e9c1cd5a13703835 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 29 Dec 2024 04:23:12 +0000 Subject: [PATCH 1289/2091] simplify multi_get_or_create/related stream implementations Signed-off-by: Jason Volk --- src/service/rooms/short/mod.rs | 12 +++++------- src/service/rooms/state_compressor/mod.rs | 7 +------ 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index f814411b..00c1d16c 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, fmt::Debug, mem::size_of_val, sync::Arc}; pub use conduwuit::pdu::{ShortEventId, ShortId, ShortRoomId}; -use conduwuit::{err, implement, utils, utils::stream::ReadyExt, Result}; +use conduwuit::{err, implement, utils, utils::IterStream, Result}; use database::{Deserialized, Map}; use futures::{Stream, StreamExt}; use ruma::{events::StateEventType, EventId, RoomId}; @@ -65,16 +65,13 @@ pub fn multi_get_or_create_shorteventid<'a, I>( event_ids: I, ) -> impl Stream + Send + '_ where - I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, - ::Item: AsRef<[u8]> + Send + Sync + 'a, + I: Iterator + Clone + Debug + Send + 'a, { self.db .eventid_shorteventid .get_batch(event_ids.clone()) - .ready_scan(event_ids, |event_ids, result| { - event_ids.next().map(|event_id| (event_id, result)) - }) - .map(|(event_id, result)| match result { + .zip(event_ids.into_iter().stream()) + .map(|(result, event_id)| match result { | Ok(ref short) => utils::u64_from_u8(short), | Err(_) => self.create_shorteventid(event_id), }) @@ -90,6 +87,7 @@ fn create_shorteventid(&self, event_id: &EventId) -> ShortEventId { self.db .eventid_shorteventid .raw_aput::(event_id, short); + self.db .shorteventid_eventid .aput_raw::(short, event_id); diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index dbe0a386..a61a66a1 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -187,12 +187,7 @@ impl Service { state: I, ) -> impl Stream + Send + 'a where - I: Iterator - + Clone - + Debug - + ExactSizeIterator - + Send - + 'a, + I: Iterator + Clone + Debug + Send + 'a, { let event_ids = state.clone().map(at!(1)); From b56e480b3cd039b89af6295f68654b1d16efe9eb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 30 Dec 2024 08:57:39 +0000 Subject: [PATCH 1290/2091] integrate some std io error kinds with our status code abstraction associate rocksdb error kinds with std io error kinds Signed-off-by: Jason Volk --- src/core/error/mod.rs | 1 + src/core/error/response.rs | 14 ++++++++++++++ src/database/util.rs | 26 ++++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index f38178e6..f1e3b924 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -178,6 +178,7 @@ impl Error { | Self::FeatureDisabled(..) => response::bad_request_code(&self.kind()), | Self::Reqwest(error) => error.status().unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), | Self::Conflict(_) => StatusCode::CONFLICT, + | Self::Io(error) => response::io_error_code(error.kind()), | _ => StatusCode::INTERNAL_SERVER_ERROR, } } diff --git a/src/core/error/response.rs b/src/core/error/response.rs index 335fddab..ede1a05d 100644 --- a/src/core/error/response.rs +++ b/src/core/error/response.rs @@ -95,3 +95,17 @@ pub(super) fn ruma_error_message(error: &ruma::api::client::error::Error) -> Str pub(super) fn ruma_error_kind(e: &ruma::api::client::error::Error) -> &ErrorKind { e.error_kind().unwrap_or(&ErrorKind::Unknown) } + +pub(super) fn io_error_code(kind: std::io::ErrorKind) -> StatusCode { + use std::io::ErrorKind; + + match kind { + | ErrorKind::InvalidInput => StatusCode::BAD_REQUEST, + | ErrorKind::PermissionDenied => StatusCode::FORBIDDEN, + | ErrorKind::NotFound => StatusCode::NOT_FOUND, + | ErrorKind::TimedOut => StatusCode::GATEWAY_TIMEOUT, + | ErrorKind::FileTooLarge => StatusCode::PAYLOAD_TOO_LARGE, + | ErrorKind::StorageFull => StatusCode::INSUFFICIENT_STORAGE, + | _ => StatusCode::INTERNAL_SERVER_ERROR, + } +} diff --git a/src/database/util.rs b/src/database/util.rs index c2a020e3..caef94f1 100644 --- a/src/database/util.rs +++ b/src/database/util.rs @@ -1,4 +1,4 @@ -use conduwuit::{err, Result}; +use conduwuit::Result; use rocksdb::{Direction, ErrorKind, IteratorMode}; //#[cfg(debug_assertions)] @@ -51,6 +51,28 @@ pub(crate) fn or_else(e: rocksdb::Error) -> Result { Err pub(crate) fn is_incomplete(e: &rocksdb::Error) -> bool { e.kind() == ErrorKind::Incomplete } pub(crate) fn map_err(e: rocksdb::Error) -> conduwuit::Error { + let kind = io_error_kind(&e.kind()); let string = e.into_string(); - err!(Database(error!("{string}"))) + + std::io::Error::new(kind, string).into() +} + +fn io_error_kind(e: &ErrorKind) -> std::io::ErrorKind { + use std::io; + + match e { + | ErrorKind::NotFound => io::ErrorKind::NotFound, + | ErrorKind::Corruption => io::ErrorKind::InvalidData, + | ErrorKind::InvalidArgument => io::ErrorKind::InvalidInput, + | ErrorKind::Aborted => io::ErrorKind::Interrupted, + | ErrorKind::NotSupported => io::ErrorKind::Unsupported, + | ErrorKind::CompactionTooLarge => io::ErrorKind::FileTooLarge, + | ErrorKind::MergeInProgress | ErrorKind::Busy => io::ErrorKind::ResourceBusy, + | ErrorKind::Expired | ErrorKind::TimedOut => io::ErrorKind::TimedOut, + | ErrorKind::Incomplete | ErrorKind::TryAgain => io::ErrorKind::WouldBlock, + | ErrorKind::ColumnFamilyDropped + | ErrorKind::ShutdownInProgress + | ErrorKind::IOError + | ErrorKind::Unknown => io::ErrorKind::Other, + } } From a1fc4d49ac3c0c830d7d3b6edf0db37924e8088b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 30 Dec 2024 12:35:31 +0000 Subject: [PATCH 1291/2091] reduce non-debug startup warnings Signed-off-by: Jason Volk --- src/core/config/check.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index c242e2fd..06ae5ebb 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -4,12 +4,12 @@ use either::Either; use figment::Figment; use super::DEPRECATED_KEYS; -use crate::{debug, debug_info, error, info, warn, Config, Err, Result}; +use crate::{debug, debug_info, debug_warn, error, warn, Config, Err, Result}; #[allow(clippy::cognitive_complexity)] pub fn check(config: &Config) -> Result<()> { if cfg!(debug_assertions) { - info!("Note: conduwuit was built without optimisations (i.e. debug build)"); + warn!("Note: conduwuit was built without optimisations (i.e. debug build)"); } warn_deprecated(config); @@ -23,7 +23,7 @@ pub fn check(config: &Config) -> Result<()> { } if cfg!(all(feature = "hardened_malloc", feature = "jemalloc")) { - info!( + debug_warn!( "hardened_malloc and jemalloc compile-time features are both enabled, this causes \ jemalloc to be used." ); From 9c6b5b44070c16a72a5149b4a0f104d2ef6ba9dd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 30 Dec 2024 13:56:21 +0000 Subject: [PATCH 1292/2091] add faster interruption to resolver (#649) Signed-off-by: Jason Volk --- src/service/resolver/dns.rs | 38 +++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index c331dfba..5c9018ab 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -2,7 +2,7 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; use conduwuit::{err, Result, Server}; use futures::FutureExt; -use hickory_resolver::TokioAsyncResolver; +use hickory_resolver::{lookup_ip::LookupIp, TokioAsyncResolver}; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use super::cache::{Cache, CachedOverride}; @@ -10,11 +10,13 @@ use super::cache::{Cache, CachedOverride}; pub struct Resolver { pub(crate) resolver: Arc, pub(crate) hooked: Arc, + server: Arc, } pub(crate) struct Hooked { resolver: Arc, cache: Arc, + server: Arc, } type ResolvingResult = Result>; @@ -72,14 +74,15 @@ impl Resolver { let resolver = Arc::new(TokioAsyncResolver::tokio(conf, opts)); Ok(Arc::new(Self { resolver: resolver.clone(), - hooked: Arc::new(Hooked { resolver, cache }), + hooked: Arc::new(Hooked { resolver, cache, server: server.clone() }), + server: server.clone(), })) } } impl Resolve for Resolver { fn resolve(&self, name: Name) -> Resolving { - resolve_to_reqwest(self.resolver.clone(), name).boxed() + resolve_to_reqwest(self.server.clone(), self.resolver.clone(), name).boxed() } } @@ -94,12 +97,29 @@ impl Resolve for Hooked { .cloned(); cached.map_or_else( - || resolve_to_reqwest(self.resolver.clone(), name).boxed(), + || resolve_to_reqwest(self.server.clone(), self.resolver.clone(), name).boxed(), |cached| cached_to_reqwest(cached).boxed(), ) } } +async fn resolve_to_reqwest( + server: Arc, + resolver: Arc, + name: Name, +) -> ResolvingResult { + use std::{io, io::ErrorKind::Interrupted}; + + let handle_shutdown = || Box::new(io::Error::new(Interrupted, "Server shutting down")); + let handle_results = + |results: LookupIp| Box::new(results.into_iter().map(|ip| SocketAddr::new(ip, 0))); + + tokio::select! { + results = resolver.lookup_ip(name.as_str()) => Ok(handle_results(results?)), + () = server.until_shutdown() => Err(handle_shutdown()), + } +} + async fn cached_to_reqwest(cached: CachedOverride) -> ResolvingResult { let addrs = cached .ips @@ -108,13 +128,3 @@ async fn cached_to_reqwest(cached: CachedOverride) -> ResolvingResult { Ok(Box::new(addrs)) } - -async fn resolve_to_reqwest(resolver: Arc, name: Name) -> ResolvingResult { - let results = resolver - .lookup_ip(name.as_str()) - .await? - .into_iter() - .map(|ip| SocketAddr::new(ip, 0)); - - Ok(Box::new(results)) -} From 203cf57fdf060c10d955fd6a57ade5ba9fcbb834 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 30 Dec 2024 12:28:18 +0000 Subject: [PATCH 1293/2091] add console log event format hook Signed-off-by: Jason Volk --- src/core/debug.rs | 2 +- src/core/log/console.rs | 94 +++++++++++++++++++++++++++++++++++++++++ src/core/log/mod.rs | 2 + src/main/logging.rs | 12 +++--- 4 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 src/core/log/console.rs diff --git a/src/core/debug.rs b/src/core/debug.rs index 123cf820..c904f23c 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -17,7 +17,7 @@ pub use crate::{result::DebugInspect, utils::debug::*}; macro_rules! debug_event { ( $level:expr, $($x:tt)+ ) => { if $crate::debug::logging() { - ::tracing::event!( $level, $($x)+ ) + ::tracing::event!( $level, _debug = true, $($x)+ ) } else { ::tracing::debug!( $($x)+ ) } diff --git a/src/core/log/console.rs b/src/core/log/console.rs new file mode 100644 index 00000000..0bc44fa7 --- /dev/null +++ b/src/core/log/console.rs @@ -0,0 +1,94 @@ +use tracing::{ + field::{Field, Visit}, + Event, Level, Subscriber, +}; +use tracing_subscriber::{ + field::RecordFields, + fmt, + fmt::{ + format::{Compact, DefaultVisitor, Format, Full, Pretty, Writer}, + FmtContext, FormatEvent, FormatFields, + }, + registry::LookupSpan, +}; + +use crate::{Config, Result}; + +pub struct ConsoleFormat { + _compact: Format, + full: Format, + pretty: Format, +} + +struct ConsoleVisitor<'a> { + visitor: DefaultVisitor<'a>, +} + +impl ConsoleFormat { + #[must_use] + pub fn new(config: &Config) -> Self { + Self { + _compact: fmt::format().compact(), + + full: Format::::default() + .with_thread_ids(config.log_thread_ids) + .with_ansi(config.log_colors), + + pretty: fmt::format() + .pretty() + .with_ansi(config.log_colors) + .with_thread_names(true) + .with_thread_ids(true) + .with_target(true) + .with_file(true) + .with_line_number(true) + .with_source_location(true), + } + } +} + +impl FormatEvent for ConsoleFormat +where + S: Subscriber + for<'a> LookupSpan<'a>, + N: for<'a> FormatFields<'a> + 'static, +{ + fn format_event( + &self, + ctx: &FmtContext<'_, S, N>, + writer: Writer<'_>, + event: &Event<'_>, + ) -> Result<(), std::fmt::Error> { + let is_debug = + cfg!(debug_assertions) && event.fields().any(|field| field.name() == "_debug"); + + match *event.metadata().level() { + | Level::ERROR if !is_debug => self.pretty.format_event(ctx, writer, event), + | _ => self.full.format_event(ctx, writer, event), + } + } +} + +impl<'writer> FormatFields<'writer> for ConsoleFormat { + fn format_fields(&self, writer: Writer<'writer>, fields: R) -> Result<(), std::fmt::Error> + where + R: RecordFields, + { + let mut visitor = ConsoleVisitor { + visitor: DefaultVisitor::<'_>::new(writer, true), + }; + + fields.record(&mut visitor); + + Ok(()) + } +} + +impl Visit for ConsoleVisitor<'_> { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + if field.name().starts_with('_') { + return; + } + + self.visitor.record_debug(field, value); + } +} diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 48b7f0f3..0c51a383 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -2,12 +2,14 @@ pub mod capture; pub mod color; +mod console; pub mod fmt; pub mod fmt_span; mod reload; mod suppress; pub use capture::Capture; +pub use console::ConsoleFormat; pub use reload::{LogLevelReloadHandles, ReloadHandle}; pub use suppress::Suppress; pub use tracing::Level; diff --git a/src/main/logging.rs b/src/main/logging.rs index e8a18b10..85945e8a 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -3,11 +3,11 @@ use std::sync::Arc; use conduwuit::{ config::Config, debug_warn, err, - log::{capture, fmt_span, LogLevelReloadHandles}, + log::{capture, fmt_span, ConsoleFormat, LogLevelReloadHandles}, result::UnwrapOrErr, Result, }; -use tracing_subscriber::{layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; +use tracing_subscriber::{fmt, layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; #[cfg(feature = "perf_measurements")] pub(crate) type TracingFlameGuard = @@ -26,10 +26,12 @@ pub(crate) fn init( .with_regex(config.log_filter_regex) .parse(&config.log) .map_err(|e| err!(Config("log", "{e}.")))?; - let console_layer = tracing_subscriber::fmt::Layer::new() - .with_ansi(config.log_colors) + let console_layer = fmt::Layer::new() .with_span_events(console_span_events) - .with_thread_ids(config.log_thread_ids); + .event_format(ConsoleFormat::new(config)) + .fmt_fields(ConsoleFormat::new(config)) + .map_writer(|w| w); + let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); reload_handles.add("console", Box::new(console_reload_handle)); From 2bbb28bb888a12644e9f19843b740294f55f87b8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 30 Dec 2024 09:06:47 +0000 Subject: [PATCH 1294/2091] get device info with libc using major/minor Signed-off-by: Jason Volk --- Cargo.lock | 75 +++------------------------------- Cargo.toml | 6 +-- src/core/Cargo.toml | 2 +- src/core/utils/sys/storage.rs | 72 +++++++++++++++++++++----------- src/database/pool/configure.rs | 12 ++++-- 5 files changed, 64 insertions(+), 103 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5b00a313..0c1890c4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -726,6 +726,7 @@ dependencies = [ "image", "ipaddress", "itertools 0.13.0", + "libc", "libloading", "log", "nix", @@ -739,7 +740,6 @@ dependencies = [ "serde_json", "serde_regex", "serde_yaml", - "sysinfo", "thiserror 2.0.7", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -1674,7 +1674,7 @@ checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" dependencies = [ "cfg-if", "libc", - "windows 0.52.0", + "windows", ] [[package]] @@ -4076,18 +4076,6 @@ dependencies = [ "syn 2.0.90", ] -[[package]] -name = "sysinfo" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "948512566b1895f93b1592c7574baeb2de842f224f2aab158799ecadb8ebbb46" -dependencies = [ - "core-foundation-sys", - "libc", - "serde", - "windows 0.57.0", -] - [[package]] name = "tendril" version = "0.4.3" @@ -4945,17 +4933,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core 0.52.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143" -dependencies = [ - "windows-core 0.57.0", + "windows-core", "windows-targets 0.52.6", ] @@ -4968,60 +4946,17 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-core" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-result 0.1.2", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-implement" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.90", -] - -[[package]] -name = "windows-interface" -version = "0.57.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.90", -] - [[package]] name = "windows-registry" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" dependencies = [ - "windows-result 0.2.0", + "windows-result", "windows-strings", "windows-targets 0.52.6", ] -[[package]] -name = "windows-result" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-result" version = "0.2.0" @@ -5037,7 +4972,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-result 0.2.0", + "windows-result", "windows-targets 0.52.6", ] diff --git a/Cargo.toml b/Cargo.toml index 24e6eb79..022baaa3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -498,10 +498,8 @@ version = "1.3.0" [workspace.dependencies.core_affinity] version = "0.8.1" -[workspace.dependencies.sysinfo] -version = "0.33.0" -default-features = false -features = ["disk", "serde"] +[workspace.dependencies.libc] +version = "0.2" # # Patches diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index d249f647..2873a05d 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -75,6 +75,7 @@ http.workspace = true image.workspace = true ipaddress.workspace = true itertools.workspace = true +libc.workspace = true libloading.workspace = true log.workspace = true rand.workspace = true @@ -87,7 +88,6 @@ serde_json.workspace = true serde_regex.workspace = true serde_yaml.workspace = true serde.workspace = true -sysinfo.workspace = true thiserror.workspace = true tikv-jemallocator.optional = true tikv-jemallocator.workspace = true diff --git a/src/core/utils/sys/storage.rs b/src/core/utils/sys/storage.rs index 8dc75236..25b17904 100644 --- a/src/core/utils/sys/storage.rs +++ b/src/core/utils/sys/storage.rs @@ -1,14 +1,20 @@ //! System utilities related to devices/peripherals use std::{ - ffi::{OsStr, OsString}, + ffi::OsStr, fs, fs::{read_to_string, FileType}, iter::IntoIterator, - path::Path, + path::{Path, PathBuf}, }; -use crate::{result::FlatOk, Result}; +use libc::dev_t; + +use crate::{ + result::FlatOk, + utils::{result::LogDebugErr, string::SplitInfallible}, + Result, +}; /// Device characteristics useful for random access throughput #[derive(Clone, Debug, Default)] @@ -35,16 +41,12 @@ pub struct Queue { /// Get device characteristics useful for random access throughput by name. #[must_use] -pub fn parallelism(name: &OsStr) -> Parallelism { - let name = name - .to_str() - .expect("device name expected to be utf-8 representable"); +pub fn parallelism(path: &Path) -> Parallelism { + let dev_id = dev_from_path(path).log_debug_err().unwrap_or_default(); - let block_path = Path::new("/").join("sys/").join("block/"); + let mq_path = block_path(dev_id).join("mq/"); - let mq_path = Path::new(&block_path).join(format!("{name}/mq/")); - - let nr_requests_path = Path::new(&block_path).join(format!("{name}/queue/nr_requests")); + let nr_requests_path = block_path(dev_id).join("queue/nr_requests"); Parallelism { nr_requests: read_to_string(&nr_requests_path) @@ -96,17 +98,39 @@ fn queue_parallelism(dir: &Path) -> Queue { } } -/// Get the name of the device on which Path is mounted. -#[must_use] -pub fn name_from_path(path: &Path) -> Option { - sysinfo::Disks::new_with_refreshed_list() - .into_iter() - .filter(|disk| path.starts_with(disk.mount_point())) - .max_by(|a, b| { - let a = a.mount_point().ancestors().count(); - let b = b.mount_point().ancestors().count(); - a.cmp(&b) - }) - .map(|disk| Path::new(disk.name())) - .and_then(|path| path.file_name().map(ToOwned::to_owned)) +/// Get the name of the block device on which Path is mounted. +pub fn name_from_path(path: &Path) -> Result { + use std::io::{Error, ErrorKind::NotFound}; + + let (major, minor) = dev_from_path(path)?; + let path = block_path((major, minor)).join("uevent"); + read_to_string(path) + .iter() + .map(String::as_str) + .flat_map(str::lines) + .map(|line| line.split_once_infallible("=")) + .find_map(|(key, val)| (key == "DEVNAME").then_some(val)) + .ok_or_else(|| Error::new(NotFound, "DEVNAME not found.")) + .map_err(Into::into) + .map(Into::into) +} + +/// Get the (major, minor) of the block device on which Path is mounted. +#[allow(clippy::useless_conversion, clippy::unnecessary_fallible_conversions)] +pub fn dev_from_path(path: &Path) -> Result<(dev_t, dev_t)> { + #[cfg(target_family = "unix")] + use std::os::unix::fs::MetadataExt; + + let stat = fs::metadata(path)?; + let dev_id = stat.dev().try_into()?; + + // SAFETY: These functions may not need to be marked as unsafe. + // see: https://github.com/rust-lang/libc/issues/3759 + let (major, minor) = unsafe { (libc::major(dev_id), libc::minor(dev_id)) }; + + Ok((major.try_into()?, minor.try_into()?)) +} + +fn block_path((major, minor): (dev_t, dev_t)) -> PathBuf { + format!("/sys/dev/block/{major}:{minor}/").into() } diff --git a/src/database/pool/configure.rs b/src/database/pool/configure.rs index 9361a534..2a192a9c 100644 --- a/src/database/pool/configure.rs +++ b/src/database/pool/configure.rs @@ -1,9 +1,10 @@ -use std::{ffi::OsStr, sync::Arc}; +use std::{path::PathBuf, sync::Arc}; use conduwuit::{ debug, debug_info, expected, utils::{ math::usize_from_f64, + result::LogDebugErr, stream, stream::WIDTH_LIMIT, sys::{compute::is_core_available, storage}, @@ -20,8 +21,12 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) // This finds the block device and gathers all the properties we need. let (device_name, device_prop) = config .db_pool_affinity - .and_then(|| storage::name_from_path(&config.database_path)) - .map(|device_name| (device_name.clone(), storage::parallelism(&device_name))) + .and_then(|| { + let path: PathBuf = config.database_path.clone(); + let name = storage::name_from_path(&path).log_debug_err().ok(); + let prop = storage::parallelism(&path); + name.map(|name| (name, prop)) + }) .unzip(); // The default worker count is masked-on if we didn't find better information. @@ -104,7 +109,6 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) debug_info!( device_name = ?device_name .as_deref() - .and_then(OsStr::to_str) .unwrap_or("None"), ?worker_counts, ?queue_sizes, From c5b94be5b8d1816c3c03f4a7c15a702b7b8e5215 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 30 Dec 2024 17:01:27 +0000 Subject: [PATCH 1295/2091] remove dev_release_log_level cfg Signed-off-by: Jason Volk --- src/admin/Cargo.toml | 1 - src/api/Cargo.toml | 1 - src/core/Cargo.toml | 1 - src/core/debug.rs | 7 +------ src/database/Cargo.toml | 1 - src/main/Cargo.toml | 8 -------- src/router/Cargo.toml | 1 - src/service/Cargo.toml | 1 - 8 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 3f8fbf79..ca865969 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -17,7 +17,6 @@ crate-type = [ ] [features] -#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index f3a84664..1bc73624 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -18,7 +18,6 @@ crate-type = [ [features] element_hacks = [] -#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 2873a05d..49874c9c 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -17,7 +17,6 @@ crate-type = [ ] [features] -#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/core/debug.rs b/src/core/debug.rs index c904f23c..aebfc833 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -10,9 +10,6 @@ pub use crate::{result::DebugInspect, utils::debug::*}; /// Log event at given level in debug-mode (when debug-assertions are enabled). /// In release-mode it becomes DEBUG level, and possibly subject to elision. -/// -/// Release-mode can be simulated in debug-mode builds by enabling the feature -/// 'dev_release_log_level'. #[macro_export] macro_rules! debug_event { ( $level:expr, $($x:tt)+ ) => { @@ -98,6 +95,4 @@ pub fn type_name() -> &'static str { std::any::type_name::() } #[must_use] #[inline] -pub const fn logging() -> bool { - cfg!(debug_assertions) && cfg!(not(feature = "dev_release_log_level")) -} +pub const fn logging() -> bool { cfg!(debug_assertions) } diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index efd18a1a..09eedaf4 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -17,7 +17,6 @@ crate-type = [ ] [features] -#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index eeb6f2bc..a6421b34 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -55,14 +55,6 @@ brotli_compression = [ console = [ "conduwuit-service/console", ] -#dev_release_log_level = [ -# "conduwuit-admin/dev_release_log_level", -# "conduwuit-api/dev_release_log_level", -# "conduwuit-core/dev_release_log_level", -# "conduwuit-database/dev_release_log_level", -# "conduwuit-router/dev_release_log_level", -# "conduwuit-service/dev_release_log_level", -#] direct_tls = [ "conduwuit-router/direct_tls" ] diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 1b2c248e..1623590b 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -17,7 +17,6 @@ crate-type = [ ] [features] -#dev_release_log_level = [] release_max_log_level = [ "tracing/max_level_trace", "tracing/release_max_level_info", diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 85c4ead9..26f737ee 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -24,7 +24,6 @@ console = [ "dep:rustyline-async", "dep:termimad", ] -#dev_release_log_level = [] element_hacks = [] gzip_compression = [ "reqwest/gzip", From 3ffe2cd56e27aaa34c4df94387be1334d5c379c1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 30 Dec 2024 21:26:38 +0000 Subject: [PATCH 1296/2091] reduce backoff warning from infolog Signed-off-by: Jason Volk --- .../rooms/event_handler/fetch_and_handle_outliers.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 5c561789..2f6940ed 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -5,7 +5,7 @@ use std::{ }; use conduwuit::{ - debug, debug_error, implement, info, pdu, trace, + debug, debug_error, debug_warn, implement, pdu, trace, utils::math::continue_exponential_backoff_secs, warn, PduEvent, }; use futures::TryFutureExt; @@ -83,7 +83,11 @@ pub(super) async fn fetch_and_handle_outliers<'a>( time.elapsed(), *tries, ) { - info!("Backing off from {next_id}"); + debug_warn!( + tried = ?*tries, + elapsed = ?time.elapsed(), + "Backing off from {next_id}", + ); continue; } } From 0e110bb48b0d49e24528395ae480f3b74bb62da3 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 30 Dec 2024 18:33:55 -0500 Subject: [PATCH 1297/2091] fix flipped room ver check on send_join Signed-off-by: June Clementine Strawberry --- src/api/server/send_join.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 6ac84907..fe0277d1 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -141,7 +141,7 @@ async fn create_join_event( if let Some(authorising_user) = content.join_authorized_via_users_server { use ruma::RoomVersionId::*; - if !matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { + if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { return Err!(Request(InvalidParam( "Room version {room_version_id} does not support restricted rooms but \ join_authorised_via_users_server ({authorising_user}) was found in the event." From 0b2fca5ad9caa4e0325d9653722ee14218cb5f3c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 1 Jan 2025 10:18:38 +0000 Subject: [PATCH 1298/2091] add complement diff from e2f6753a3280 --- tests/test_results/complement/test_results.jsonl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 0db37515..26ae6931 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -207,16 +207,16 @@ {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_fail_when_left_allowed_room"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_fail_with_mangled_join_rules"} -{"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_succeed_when_invited"} -{"Action":"fail","Test":"TestRestrictedRoomsRemoteJoin/Join_should_succeed_when_joined_to_allowed_room"} +{"Action":"fail","Test":"TestRestrictedRoomsRemoteJoin/Join_should_succeed_when_invited"} +{"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_succeed_when_joined_to_allowed_room"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinFailOver"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinFailOverInMSC3787Room"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_when_left_allowed_room"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_with_mangled_join_rules"} -{"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_invited"} -{"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room"} +{"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_invited"} +{"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinLocalUser"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinLocalUserInMSC3787Room"} {"Action":"pass","Test":"TestRestrictedRoomsSpacesSummaryFederation"} From 1792711d0967f34a7a07c9e7d832d4447f433987 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 30 Dec 2024 09:35:44 +0000 Subject: [PATCH 1299/2091] support core affinity mask Signed-off-by: Jason Volk --- Cargo.lock | 3 +-- Cargo.toml | 4 ++++ src/core/utils/sys/compute.rs | 43 ++++++++++++++++++++++++----------- src/database/pool.rs | 4 ++-- src/main/runtime.rs | 4 ++-- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0c1890c4..7f9ef547 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -937,8 +937,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core_affinity" version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622892f5635ce1fc38c8f16dfc938553ed64af482edb5e150bf4caedbfcb2304" +source = "git+https://github.com/jevolk/core_affinity_rs?rev=9c8e51510c35077df888ee72a36b4b05637147da#9c8e51510c35077df888ee72a36b4b05637147da" dependencies = [ "libc", "num_cpus", diff --git a/Cargo.toml b/Cargo.toml index 022baaa3..c66dfcff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -535,6 +535,10 @@ rev = "96d7e0fc026d8f708b19bc9267a382676a50354c" git = "https://github.com/jevolk/async-channel" rev = "fefa543ca5eddf21237d75776fce98b7e09e924a" +[patch.crates-io.core_affinity] +git = "https://github.com/jevolk/core_affinity_rs" +rev = "9c8e51510c35077df888ee72a36b4b05637147da" + # # Our crates # diff --git a/src/core/utils/sys/compute.rs b/src/core/utils/sys/compute.rs index e947b579..9e90fc90 100644 --- a/src/core/utils/sys/compute.rs +++ b/src/core/utils/sys/compute.rs @@ -1,6 +1,6 @@ //! System utilities related to compute/processing -use std::{cell::Cell, sync::LazyLock}; +use std::{cell::Cell, fmt::Debug, sync::LazyLock}; use crate::is_equal_to; @@ -22,33 +22,48 @@ thread_local! { /// Set the core affinity for this thread. The ID should be listed in /// CORES_AVAILABLE. Empty input is a no-op; prior affinity unchanged. -pub fn set_affinity(ids: I) +#[tracing::instrument( + level = "debug", + skip_all, + fields( + id = ?std::thread::current().id(), + name = %std::thread::current().name().unwrap_or("None"), + set = ?ids.by_ref().collect::>(), + CURRENT = %format!("[b{:b}]", CORE_AFFINITY.get()), + AVAILABLE = %format!("[b{:b}]", *CORES_AVAILABLE), + ), +)] +pub fn set_affinity(mut ids: I) where - I: Iterator, + I: Iterator + Clone + Debug, { - use core_affinity::{set_for_current, CoreId}; + use core_affinity::{set_each_for_current, set_for_current, CoreId}; - let mask: u128 = ids.fold(0, |mask, id| { + let n = ids.clone().count(); + let mask: u128 = ids.clone().fold(0, |mask, id| { debug_assert!(is_core_available(id), "setting affinity to unavailable core"); - set_for_current(CoreId { id }); mask | (1 << id) }); + if n > 1 { + set_each_for_current(ids.map(|id| CoreId { id })); + } else if n > 0 { + set_for_current(CoreId { id: ids.next().expect("n > 0") }); + } + if mask.count_ones() > 0 { CORE_AFFINITY.replace(mask); } } /// Get the core affinity for this thread. -pub fn get_affinity() -> impl Iterator { - (0..128).filter(|&i| ((CORE_AFFINITY.get() & (1 << i)) != 0)) -} +pub fn get_affinity() -> impl Iterator { iter_bits(CORE_AFFINITY.get()) } /// Gets the ID of the nth core available. This bijects our sequence of cores to /// actual ID's which may have gaps for cores which are not available. #[inline] #[must_use] -pub fn get_core_available(i: usize) -> Option { cores_available().nth(i) } +pub fn nth_core_available(i: usize) -> Option { cores_available().nth(i) } /// Determine if core (by id) is available to the process. #[inline] @@ -57,9 +72,7 @@ pub fn is_core_available(id: usize) -> bool { cores_available().any(is_equal_to! /// Get the list of cores available. The values were recorded at program start. #[inline] -pub fn cores_available() -> impl Iterator { - (0..128).filter(|&i| ((*CORES_AVAILABLE & (1 << i)) != 0)) -} +pub fn cores_available() -> impl Iterator { iter_bits(*CORES_AVAILABLE) } /// Get the number of threads which could execute in parallel based on the /// hardware and administrative constraints of this system. This value should be @@ -72,3 +85,7 @@ pub fn parallelism() -> usize { .expect("Unable to query for available parallelism.") .get() } + +fn iter_bits(v: u128) -> impl Iterator { + (0..128).filter(move |&i| (v & (1 << i)) != 0) +} diff --git a/src/database/pool.rs b/src/database/pool.rs index 1c55c456..8182f217 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -13,7 +13,7 @@ use conduwuit::{ debug, debug_warn, defer, err, implement, result::DebugInspect, trace, - utils::sys::compute::{get_affinity, get_core_available, set_affinity}, + utils::sys::compute::{get_affinity, nth_core_available, set_affinity}, Result, Server, }; use futures::{channel::oneshot, TryFutureExt}; @@ -270,7 +270,7 @@ fn worker_init(&self, id: usize) { .enumerate() .filter(|_| self.queues.len() > 1) .filter_map(|(core_id, &queue_id)| (group == queue_id).then_some(core_id)) - .filter_map(get_core_available); + .filter_map(nth_core_available); // affinity is empty (no-op) if there's only one queue set_affinity(affinity.clone()); diff --git a/src/main/runtime.rs b/src/main/runtime.rs index bfd2ef74..b9dfc866 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -9,7 +9,7 @@ use std::{ }; use conduwuit::{ - utils::sys::compute::{get_core_available, set_affinity}, + utils::sys::compute::{nth_core_available, set_affinity}, Result, }; use tokio::runtime::Builder; @@ -100,7 +100,7 @@ fn set_worker_affinity() { return; } - let Some(id) = get_core_available(i) else { + let Some(id) = nth_core_available(i) else { return; }; From 2259e2c82f890415b84e5999242c3ee102973d23 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 1 Jan 2025 06:08:20 +0000 Subject: [PATCH 1300/2091] batch queries to maximize throughput query-side streams for first level of callsites Signed-off-by: Jason Volk --- conduwuit-example.toml | 13 ++++ src/core/config/mod.rs | 17 ++++++ src/core/utils/stream/band.rs | 26 +++++++- src/core/utils/stream/mod.rs | 5 +- src/database/map/get.rs | 10 +++- src/database/map/get_batch.rs | 66 +++++++++++++------- src/database/pool.rs | 76 ++++++++++++++++++------ src/database/pool/configure.rs | 11 +++- src/main/server.rs | 1 + src/service/rooms/auth_chain/mod.rs | 2 +- src/service/rooms/short/mod.rs | 12 ++-- src/service/rooms/state/mod.rs | 2 +- src/service/rooms/state_accessor/data.rs | 6 +- 13 files changed, 191 insertions(+), 56 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index c64b18e8..526e9fe2 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1457,6 +1457,19 @@ # #stream_width_scale = 1.0 +# Sets the initial amplification factor. This controls batch sizes of +# requests made by each pool worker, multiplying the throughput of each +# stream. This value is somewhat abstract from specific hardware +# characteristics and can be significantly larger than any thread count or +# queue size. This is because each database query may require several +# index lookups, thus many database queries in a batch may make progress +# independently while also sharing index and data blocks which may or may +# not be cached. It is worthwhile to submit huge batches to reduce +# complexity. The maximum value is 32768, though sufficient hardware is +# still advised for that. +# +#stream_amplification = 1024 + # Number of sender task workers; determines sender parallelism. Default is # '0' which means the value is determined internally, likely matching the # number of tokio worker-threads or number of cores, etc. Override by diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index e1f578c8..b1ede844 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1653,6 +1653,21 @@ pub struct Config { #[serde(default = "default_stream_width_scale")] pub stream_width_scale: f32, + /// Sets the initial amplification factor. This controls batch sizes of + /// requests made by each pool worker, multiplying the throughput of each + /// stream. This value is somewhat abstract from specific hardware + /// characteristics and can be significantly larger than any thread count or + /// queue size. This is because each database query may require several + /// index lookups, thus many database queries in a batch may make progress + /// independently while also sharing index and data blocks which may or may + /// not be cached. It is worthwhile to submit huge batches to reduce + /// complexity. The maximum value is 32768, though sufficient hardware is + /// still advised for that. + /// + /// default: 1024 + #[serde(default = "default_stream_amplification")] + pub stream_amplification: usize, + /// Number of sender task workers; determines sender parallelism. Default is /// '0' which means the value is determined internally, likely matching the /// number of tokio worker-threads or number of cores, etc. Override by @@ -2467,3 +2482,5 @@ fn default_db_pool_queue_mult() -> usize { 4 } fn default_stream_width_default() -> usize { 32 } fn default_stream_width_scale() -> f32 { 1.0 } + +fn default_stream_amplification() -> usize { 1024 } diff --git a/src/core/utils/stream/band.rs b/src/core/utils/stream/band.rs index 76f2a85a..45ad7d94 100644 --- a/src/core/utils/stream/band.rs +++ b/src/core/utils/stream/band.rs @@ -3,9 +3,15 @@ use std::sync::atomic::{AtomicUsize, Ordering}; /// Stream concurrency factor; this is a live value. static WIDTH: AtomicUsize = AtomicUsize::new(32); -/// Practicable limits on the stream width +/// Stream throughput amplifier; this is a live value. +static AMPLIFICATION: AtomicUsize = AtomicUsize::new(1024); + +/// Practicable limits on the stream width. pub const WIDTH_LIMIT: (usize, usize) = (1, 1024); +/// Practicable limits on the stream amplifier. +pub const AMPLIFICATION_LIMIT: (usize, usize) = (32, 32768); + /// Sets the live concurrency factor. The first return value is the previous /// width which was replaced. The second return value is the value which was set /// after any applied limits. @@ -14,6 +20,14 @@ pub fn set_width(width: usize) -> (usize, usize) { (WIDTH.swap(width, Ordering::Relaxed), width) } +/// Sets the live concurrency amplification. The first return value is the +/// previous width which was replaced. The second return value is the value +/// which was set after any applied limits. +pub fn set_amplification(width: usize) -> (usize, usize) { + let width = width.clamp(AMPLIFICATION_LIMIT.0, AMPLIFICATION_LIMIT.1); + (AMPLIFICATION.swap(width, Ordering::Relaxed), width) +} + /// Used by stream operations where the concurrency factor hasn't been manually /// supplied by the caller (most uses). Instead we provide a default value which /// is adjusted at startup for the specific system and also dynamically. @@ -24,3 +38,13 @@ pub fn automatic_width() -> usize { debug_assert!(width <= WIDTH_LIMIT.1, "WIDTH is probably too large"); width } + +/// Used by stream operations where the amplification hasn't been manually +/// supplied by the caller. Instead we provide a computed value. +#[inline] +pub fn automatic_amplification() -> usize { + let amplification = AMPLIFICATION.load(Ordering::Relaxed); + debug_assert!(amplification >= AMPLIFICATION_LIMIT.0, "amplification is too low"); + debug_assert!(amplification <= AMPLIFICATION_LIMIT.1, "amplification is too high"); + amplification +} diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index a5ef17c5..61ae993d 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -10,7 +10,10 @@ mod try_broadband; mod try_ready; mod wideband; -pub use band::{automatic_width, set_width, WIDTH_LIMIT}; +pub use band::{ + automatic_amplification, automatic_width, set_amplification, set_width, AMPLIFICATION_LIMIT, + WIDTH_LIMIT, +}; pub use broadband::BroadbandExt; pub use cloned::Cloned; pub use expect::TryExpect; diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 94a6b727..e64ef2ec 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -2,7 +2,7 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; use arrayvec::ArrayVec; use conduwuit::{err, implement, utils::result::MapExpect, Err, Result}; -use futures::{Future, FutureExt}; +use futures::{future::ready, Future, FutureExt, TryFutureExt}; use serde::Serialize; use tokio::task; @@ -79,11 +79,15 @@ where debug_assert!(matches!(cached, Ok(None)), "expected status Incomplete"); let cmd = Get { map: self.clone(), - key: key.as_ref().into(), + key: [key.as_ref().into()].into(), res: None, }; - self.db.pool.execute_get(cmd).boxed() + self.db + .pool + .execute_get(cmd) + .and_then(|mut res| ready(res.remove(0))) + .boxed() } /// Fetch a value from the database into cache, returning a reference-handle. diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index 631692fe..452697f1 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -2,42 +2,68 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduwuit::{ err, implement, - utils::{stream::automatic_width, IterStream}, + utils::{ + stream::{automatic_amplification, automatic_width, WidebandExt}, + IterStream, + }, Result, }; -use futures::{Stream, StreamExt}; +use futures::{Stream, StreamExt, TryStreamExt}; use serde::Serialize; -use crate::{util::map_err, Handle}; +use crate::{keyval::KeyBuf, ser, util::map_err, Handle}; #[implement(super::Map)] #[tracing::instrument(skip(self, keys), level = "trace")] -pub fn aqry_batch<'b, 'a: 'b, const MAX: usize, I, K>( +pub fn qry_batch<'a, S, K>( self: &'a Arc, - keys: I, -) -> impl Stream>> + Send + 'a + keys: S, +) -> impl Stream>> + Send + 'a where - I: Iterator + Send + 'a, - K: Serialize + ?Sized + Debug + 'b, + S: Stream + Send + 'a, + K: Serialize + Debug + 'a, { - keys.stream() - .map(move |key| self.aqry::(&key)) - .buffered(automatic_width()) + use crate::pool::Get; + + keys.ready_chunks(automatic_amplification()) + .widen_then(automatic_width(), |chunk| { + let keys = chunk + .iter() + .map(ser::serialize_to::) + .map(|result| result.expect("failed to serialize query key")) + .map(Into::into) + .collect(); + + self.db + .pool + .execute_get(Get { map: self.clone(), key: keys, res: None }) + }) + .map_ok(|results| results.into_iter().stream()) + .try_flatten() } #[implement(super::Map)] #[tracing::instrument(skip(self, keys), level = "trace")] -pub fn get_batch<'a, I, K>( +pub fn get_batch<'a, S, K>( self: &'a Arc, - keys: I, + keys: S, ) -> impl Stream>> + Send + 'a where - I: Iterator + Debug + Send + 'a, - K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, + S: Stream + Send + 'a, + K: AsRef<[u8]> + Send + Sync + 'a, { - keys.stream() - .map(move |key| self.get(key)) - .buffered(automatic_width()) + use crate::pool::Get; + + keys.ready_chunks(automatic_amplification()) + .widen_then(automatic_width(), |chunk| { + self.db.pool.execute_get(Get { + map: self.clone(), + key: chunk.iter().map(AsRef::as_ref).map(Into::into).collect(), + res: None, + }) + }) + .map_ok(|results| results.into_iter().stream()) + .try_flatten() } #[implement(super::Map)] @@ -47,8 +73,8 @@ pub(crate) fn get_batch_blocking<'a, I, K>( keys: I, ) -> impl Iterator>> + Send where - I: Iterator + ExactSizeIterator + Debug + Send, - K: AsRef<[u8]> + Debug + Send + ?Sized + Sync + 'a, + I: Iterator + ExactSizeIterator + Send, + K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, { // Optimization can be `true` if key vector is pre-sorted **by the column // comparator**. diff --git a/src/database/pool.rs b/src/database/pool.rs index 8182f217..bcf20de8 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -19,6 +19,7 @@ use conduwuit::{ use futures::{channel::oneshot, TryFutureExt}; use oneshot::Sender as ResultSender; use rocksdb::Direction; +use smallvec::SmallVec; use tokio::task::JoinSet; use self::configure::configure; @@ -42,11 +43,11 @@ pub(crate) enum Cmd { Iter(Seek), } -/// Point-query +/// Multi-point-query pub(crate) struct Get { pub(crate) map: Arc, - pub(crate) key: KeyBuf, - pub(crate) res: Option>>>, + pub(crate) key: BatchQuery<'static>, + pub(crate) res: Option>>, } /// Iterator-seek. @@ -60,8 +61,13 @@ pub(crate) struct Seek { pub(crate) res: Option>>, } +pub(crate) type BatchQuery<'a> = SmallVec<[KeyBuf; BATCH_INLINE]>; +pub(crate) type BatchResult<'a> = SmallVec<[ResultHandle<'a>; BATCH_INLINE]>; +pub(crate) type ResultHandle<'a> = Result>; + const WORKER_LIMIT: (usize, usize) = (1, 1024); const QUEUE_LIMIT: (usize, usize) = (1, 2048); +const BATCH_INLINE: usize = 1; #[implement(Pool)] pub(crate) async fn new(server: &Arc) -> Result> { @@ -179,22 +185,24 @@ fn spawn_one(self: &Arc, workers: &mut JoinSet<()>, recv: &[Receiver] #[implement(Pool)] #[tracing::instrument(level = "trace", name = "get", skip(self, cmd))] -pub(crate) async fn execute_get(&self, mut cmd: Get) -> Result> { +pub(crate) async fn execute_get(self: &Arc, mut cmd: Get) -> Result> { let (send, recv) = oneshot::channel(); _ = cmd.res.insert(send); let queue = self.select_queue(); self.execute(queue, Cmd::Get(cmd)) - .and_then(|()| { - recv.map_ok(into_recv_get_result) + .and_then(move |()| { + recv.map_ok(into_recv_get) .map_err(|e| err!(error!("recv failed {e:?}"))) }) - .await? + .await + .map(Into::into) + .map_err(Into::into) } #[implement(Pool)] #[tracing::instrument(level = "trace", name = "iter", skip(self, cmd))] -pub(crate) async fn execute_iter(&self, mut cmd: Seek) -> Result> { +pub(crate) async fn execute_iter(self: &Arc, mut cmd: Seek) -> Result> { let (send, recv) = oneshot::channel(); _ = cmd.res.insert(send); @@ -282,7 +290,7 @@ fn worker_init(&self, id: usize) { } #[implement(Pool)] -fn worker_loop(&self, recv: &Receiver) { +fn worker_loop(self: &Arc, recv: &Receiver) { // initial +1 needed prior to entering wait self.busy.fetch_add(1, Ordering::Relaxed); @@ -302,18 +310,19 @@ fn worker_loop(&self, recv: &Receiver) { busy = self.busy.fetch_sub(1, Ordering::Relaxed) - 1, ), )] -fn worker_wait(&self, recv: &Receiver) -> Result { +fn worker_wait(self: &Arc, recv: &Receiver) -> Result { recv.recv_blocking().debug_inspect(|_| { self.busy.fetch_add(1, Ordering::Relaxed); }) } #[implement(Pool)] -fn worker_handle(&self, cmd: Cmd) { +fn worker_handle(self: &Arc, cmd: Cmd) { match cmd { - | Cmd::Get(cmd) => self.handle_get(cmd), + | Cmd::Get(cmd) if cmd.key.len() == 1 => self.handle_get(cmd), + | Cmd::Get(cmd) => self.handle_batch(cmd), | Cmd::Iter(cmd) => self.handle_iter(cmd), - } + }; } #[implement(Pool)] @@ -331,12 +340,43 @@ fn handle_iter(&self, mut cmd: Seek) { } let from = cmd.key.as_deref().map(Into::into); + let result = match cmd.dir { | Direction::Forward => cmd.state.init_fwd(from), | Direction::Reverse => cmd.state.init_rev(from), }; let chan_result = chan.send(into_send_seek(result)); + + let _chan_sent = chan_result.is_ok(); +} + +#[implement(Pool)] +#[tracing::instrument( + name = "batch", + level = "trace", + skip_all, + fields( + %cmd.map, + keys = %cmd.key.len(), + ), +)] +fn handle_batch(self: &Arc, mut cmd: Get) { + debug_assert!(cmd.key.len() > 1, "should have more than one key"); + debug_assert!(!cmd.key.iter().any(SmallVec::is_empty), "querying for empty key"); + + let chan = cmd.res.take().expect("missing result channel"); + + if chan.is_canceled() { + return; + } + + let keys = cmd.key.iter().map(Into::into); + + let result: SmallVec<_> = cmd.map.get_batch_blocking(keys).collect(); + + let chan_result = chan.send(into_send_get(result)); + let _chan_sent = chan_result.is_ok(); } @@ -348,7 +388,7 @@ fn handle_iter(&self, mut cmd: Seek) { fields(%cmd.map), )] fn handle_get(&self, mut cmd: Get) { - debug_assert!(!cmd.key.is_empty(), "querying for empty key"); + debug_assert!(!cmd.key[0].is_empty(), "querying for empty key"); // Obtain the result channel. let chan = cmd.res.take().expect("missing result channel"); @@ -362,16 +402,16 @@ fn handle_get(&self, mut cmd: Get) { // Perform the actual database query. We reuse our database::Map interface but // limited to the blocking calls, rather than creating another surface directly // with rocksdb here. - let result = cmd.map.get_blocking(&cmd.key); + let result = cmd.map.get_blocking(&cmd.key[0]); // Send the result back to the submitter. - let chan_result = chan.send(into_send_get_result(result)); + let chan_result = chan.send(into_send_get([result].into())); // If the future was dropped during the query this will fail acceptably. let _chan_sent = chan_result.is_ok(); } -fn into_send_get_result(result: Result>) -> Result> { +fn into_send_get(result: BatchResult<'_>) -> BatchResult<'static> { // SAFETY: Necessary to send the Handle (rust_rocksdb::PinnableSlice) through // the channel. The lifetime on the handle is a device by rust-rocksdb to // associate a database lifetime with its assets. The Handle must be dropped @@ -379,7 +419,7 @@ fn into_send_get_result(result: Result>) -> Result> { unsafe { std::mem::transmute(result) } } -fn into_recv_get_result(result: Result>) -> Result> { +fn into_recv_get<'a>(result: BatchResult<'static>) -> BatchResult<'a> { // SAFETY: This is to receive the Handle from the channel. unsafe { std::mem::transmute(result) } } diff --git a/src/database/pool/configure.rs b/src/database/pool/configure.rs index 2a192a9c..6cac58e7 100644 --- a/src/database/pool/configure.rs +++ b/src/database/pool/configure.rs @@ -6,7 +6,7 @@ use conduwuit::{ math::usize_from_f64, result::LogDebugErr, stream, - stream::WIDTH_LIMIT, + stream::{AMPLIFICATION_LIMIT, WIDTH_LIMIT}, sys::{compute::is_core_available, storage}, BoolExt, }, @@ -124,19 +124,28 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) fn update_stream_width(server: &Arc, num_queues: usize, total_workers: usize) { let config = &server.config; let scale: f64 = config.stream_width_scale.min(100.0).into(); + let req_width = expected!(total_workers / num_queues).next_multiple_of(2); let req_width = req_width as f64; let req_width = usize_from_f64(req_width * scale) .expect("failed to convert f64 to usize") .clamp(WIDTH_LIMIT.0, WIDTH_LIMIT.1); + let req_amp = config.stream_amplification as f64; + let req_amp = usize_from_f64(req_amp * scale) + .expect("failed to convert f64 to usize") + .clamp(AMPLIFICATION_LIMIT.0, AMPLIFICATION_LIMIT.1); + let (old_width, new_width) = stream::set_width(req_width); + let (old_amp, new_amp) = stream::set_amplification(req_amp); debug!( scale = ?config.stream_width_scale, ?num_queues, ?req_width, ?old_width, ?new_width, + ?old_amp, + ?new_amp, "Updated global stream width" ); } diff --git a/src/main/server.rs b/src/main/server.rs index e1389f6d..a81b708d 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -52,6 +52,7 @@ impl Server { .expect("Unable to increase maximum soft and hard file descriptor limit"); let (_old_width, _new_width) = stream::set_width(config.stream_width_default); + let (_old_amp, _new_amp) = stream::set_amplification(config.stream_amplification); info!( server_name = %config.server_name, diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 67883d01..f6534825 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -71,7 +71,7 @@ impl Service { let event_ids = self .services .short - .multi_get_eventid_from_short(chain.iter()) + .multi_get_eventid_from_short(chain.into_iter().stream()) .ready_filter_map(Result::ok) .collect() .await; diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 00c1d16c..b645f9f1 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -69,7 +69,7 @@ where { self.db .eventid_shorteventid - .get_batch(event_ids.clone()) + .get_batch(event_ids.clone().stream()) .zip(event_ids.into_iter().stream()) .map(|(result, event_id)| match result { | Ok(ref short) => utils::u64_from_u8(short), @@ -162,20 +162,18 @@ where } #[implement(Service)] -pub fn multi_get_eventid_from_short<'a, Id, I>( +pub fn multi_get_eventid_from_short<'a, Id, S>( &'a self, - shorteventid: I, + shorteventid: S, ) -> impl Stream> + Send + 'a where - I: Iterator + Send + 'a, + S: Stream + Send + 'a, Id: for<'de> Deserialize<'de> + Sized + ToOwned + 'a, ::Owned: Borrow, { - const BUFSIZE: usize = size_of::(); - self.db .shorteventid_eventid - .aqry_batch::(shorteventid) + .qry_batch(shorteventid) .map(Deserialized::deserialized) } diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 2769beb8..fd303667 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -467,7 +467,7 @@ impl Service { let auth_pdus = self .services .short - .multi_get_eventid_from_short(event_ids.iter()) + .multi_get_eventid_from_short(event_ids.into_iter().stream()) .zip(state_keys.into_iter().stream()) .ready_filter_map(|(event_id, tsk)| Some((tsk, event_id.ok()?))) .broad_filter_map(|(tsk, event_id): (_, OwnedEventId)| async move { diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs index d60e505e..29b27a05 100644 --- a/src/service/rooms/state_accessor/data.rs +++ b/src/service/rooms/state_accessor/data.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, collections::HashMap, sync::Arc}; use conduwuit::{ - at, err, ref_at, + at, err, utils::stream::{BroadbandExt, IterStream, ReadyExt}, PduEvent, Result, }; @@ -69,7 +69,7 @@ impl Data { let full_pdus = self .services .short - .multi_get_eventid_from_short(short_ids.iter().map(ref_at!(1))) + .multi_get_eventid_from_short(short_ids.into_iter().map(at!(1)).stream()) .ready_filter_map(Result::ok) .broad_filter_map(|event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await.ok() @@ -93,7 +93,7 @@ impl Data { let full_ids = self .services .short - .multi_get_eventid_from_short(short_ids.iter().map(ref_at!(1))) + .multi_get_eventid_from_short(short_ids.iter().map(at!(1)).stream()) .zip(short_ids.iter().stream().map(at!(0))) .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) .collect() From 7e4453620eb3b9eb618ff195b72fc0d5df5d898b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 1 Jan 2025 20:09:41 +0000 Subject: [PATCH 1301/2091] move exponential backoff util to different submod Signed-off-by: Jason Volk --- src/core/utils/math.rs | 30 +------------------ src/core/utils/mod.rs | 5 +++- src/core/utils/time.rs | 2 ++ src/core/utils/time/exponential_backoff.rs | 29 ++++++++++++++++++ .../fetch_and_handle_outliers.rs | 2 +- .../rooms/event_handler/handle_prev_pdu.rs | 2 +- src/service/sending/sender.rs | 2 +- 7 files changed, 39 insertions(+), 33 deletions(-) create mode 100644 src/core/utils/time/exponential_backoff.rs diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index da2357d5..c5a785e2 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -1,4 +1,4 @@ -use std::{cmp, convert::TryFrom, time::Duration}; +use std::{cmp, convert::TryFrom}; pub use checked_ops::checked_ops; @@ -53,34 +53,6 @@ macro_rules! validated { ($($input:tt)+) => { $crate::expected!($($input)+) } } -/// Returns false if the exponential backoff has expired based on the inputs -#[inline] -#[must_use] -pub fn continue_exponential_backoff_secs( - min: u64, - max: u64, - elapsed: Duration, - tries: u32, -) -> bool { - let min = Duration::from_secs(min); - let max = Duration::from_secs(max); - continue_exponential_backoff(min, max, elapsed, tries) -} - -/// Returns false if the exponential backoff has expired based on the inputs -#[inline] -#[must_use] -pub fn continue_exponential_backoff( - min: Duration, - max: Duration, - elapsed: Duration, - tries: u32, -) -> bool { - let min = min.saturating_mul(tries).saturating_mul(tries); - let min = cmp::min(min, max); - elapsed < min -} - #[inline] #[allow(clippy::as_conversions)] pub fn usize_from_f64(val: f64) -> Result { diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 38232820..16072765 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -37,7 +37,10 @@ pub use self::{ stream::{IterStream, ReadyExt, Tools as StreamTools, TryReadyExt}, string::{str_from_bytes, string_from_bytes}, sys::compute::parallelism as available_parallelism, - time::{now_millis as millis_since_unix_epoch, timepoint_ago, timepoint_from_now}, + time::{ + exponential_backoff::{continue_exponential_backoff, continue_exponential_backoff_secs}, + now_millis as millis_since_unix_epoch, timepoint_ago, timepoint_from_now, + }, }; #[inline] diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index d65eb2d4..81fdda2a 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -1,3 +1,5 @@ +pub mod exponential_backoff; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; use crate::{err, Result}; diff --git a/src/core/utils/time/exponential_backoff.rs b/src/core/utils/time/exponential_backoff.rs new file mode 100644 index 00000000..682c2592 --- /dev/null +++ b/src/core/utils/time/exponential_backoff.rs @@ -0,0 +1,29 @@ +use std::{cmp, time::Duration}; + +/// Returns false if the exponential backoff has expired based on the inputs +#[inline] +#[must_use] +pub fn continue_exponential_backoff_secs( + min: u64, + max: u64, + elapsed: Duration, + tries: u32, +) -> bool { + let min = Duration::from_secs(min); + let max = Duration::from_secs(max); + continue_exponential_backoff(min, max, elapsed, tries) +} + +/// Returns false if the exponential backoff has expired based on the inputs +#[inline] +#[must_use] +pub fn continue_exponential_backoff( + min: Duration, + max: Duration, + elapsed: Duration, + tries: u32, +) -> bool { + let min = min.saturating_mul(tries).saturating_mul(tries); + let min = cmp::min(min, max); + elapsed < min +} diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 2f6940ed..84d0edd0 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -6,7 +6,7 @@ use std::{ use conduwuit::{ debug, debug_error, debug_warn, implement, pdu, trace, - utils::math::continue_exponential_backoff_secs, warn, PduEvent, + utils::continue_exponential_backoff_secs, warn, PduEvent, }; use futures::TryFutureExt; use ruma::{ diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index a8893160..0a5295dc 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -5,7 +5,7 @@ use std::{ }; use conduwuit::{ - debug, implement, utils::math::continue_exponential_backoff_secs, Err, PduEvent, Result, + debug, implement, utils::continue_exponential_backoff_secs, Err, PduEvent, Result, }; use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName}; diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index bf4ebafb..482c31cf 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -13,7 +13,7 @@ use conduwuit::{ debug, err, error, result::LogErr, trace, - utils::{calculate_hash, math::continue_exponential_backoff_secs, ReadyExt}, + utils::{calculate_hash, continue_exponential_backoff_secs, ReadyExt}, warn, Error, Result, }; use futures::{ From 1a71798859786645ef8520c19ede5576b4d15bae Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 1 Jan 2025 20:20:41 +0000 Subject: [PATCH 1302/2091] add Expected trait to utils; use (already transitive) num-traits. Signed-off-by: Jason Volk --- Cargo.lock | 1 + Cargo.toml | 3 ++ src/core/Cargo.toml | 1 + src/core/utils/math.rs | 3 ++ src/core/utils/math/expected.rs | 52 +++++++++++++++++++++++++++++++++ 5 files changed, 60 insertions(+) create mode 100644 src/core/utils/math/expected.rs diff --git a/Cargo.lock b/Cargo.lock index 7f9ef547..a8a6ae0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -730,6 +730,7 @@ dependencies = [ "libloading", "log", "nix", + "num-traits", "rand", "regex", "reqwest", diff --git a/Cargo.toml b/Cargo.toml index c66dfcff..805c7d7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -501,6 +501,9 @@ version = "0.8.1" [workspace.dependencies.libc] version = "0.2" +[workspace.dependencies.num-traits] +version = "0.2" + # # Patches # diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 49874c9c..dd8f634a 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -77,6 +77,7 @@ itertools.workspace = true libc.workspace = true libloading.workspace = true log.workspace = true +num-traits.workspace = true rand.workspace = true regex.workspace = true reqwest.workspace = true diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index c5a785e2..a08cb206 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -1,7 +1,10 @@ +mod expected; + use std::{cmp, convert::TryFrom}; pub use checked_ops::checked_ops; +pub use self::expected::Expected; use crate::{debug::type_name, err, Err, Error, Result}; /// Checked arithmetic expression. Returns a Result diff --git a/src/core/utils/math/expected.rs b/src/core/utils/math/expected.rs new file mode 100644 index 00000000..f0f71854 --- /dev/null +++ b/src/core/utils/math/expected.rs @@ -0,0 +1,52 @@ +use num_traits::ops::checked::{CheckedAdd, CheckedDiv, CheckedMul, CheckedRem, CheckedSub}; + +use crate::expected; + +pub trait Expected { + #[inline] + #[must_use] + fn expected_add(self, rhs: Self) -> Self + where + Self: CheckedAdd + Sized, + { + expected!(self + rhs) + } + + #[inline] + #[must_use] + fn expected_sub(self, rhs: Self) -> Self + where + Self: CheckedSub + Sized, + { + expected!(self - rhs) + } + + #[inline] + #[must_use] + fn expected_mul(self, rhs: Self) -> Self + where + Self: CheckedMul + Sized, + { + expected!(self * rhs) + } + + #[inline] + #[must_use] + fn expected_div(self, rhs: Self) -> Self + where + Self: CheckedDiv + Sized, + { + expected!(self / rhs) + } + + #[inline] + #[must_use] + fn expected_rem(self, rhs: Self) -> Self + where + Self: CheckedRem + Sized, + { + expected!(self % rhs) + } +} + +impl Expected for T {} From 33635e11d1ad1f52744d368defc12e6ec25ec59d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 1 Jan 2025 23:47:42 +0000 Subject: [PATCH 1303/2091] bump ruma tweak tracing instrument Signed-off-by: Jason Volk --- Cargo.lock | 26 +++++++++---------- Cargo.toml | 2 +- src/service/rooms/event_handler/acl_check.rs | 2 +- .../rooms/event_handler/resolve_state.rs | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8a6ae0a..cf08509f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3173,7 +3173,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "assign", "js_int", @@ -3195,7 +3195,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "js_int", "ruma-common", @@ -3207,7 +3207,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "as_variant", "assign", @@ -3230,7 +3230,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "as_variant", "base64 0.22.1", @@ -3261,7 +3261,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3286,7 +3286,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "bytes", "http", @@ -3304,7 +3304,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3313,7 +3313,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "js_int", "ruma-common", @@ -3323,7 +3323,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3338,7 +3338,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "js_int", "ruma-common", @@ -3350,7 +3350,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "headers", "http", @@ -3363,7 +3363,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3379,7 +3379,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=30a08ff7be779df6858036c3f01f3ba64c1ee785#30a08ff7be779df6858036c3f01f3ba64c1ee785" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 805c7d7c..a59dcfcd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -336,7 +336,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "30a08ff7be779df6858036c3f01f3ba64c1ee785" +rev = "ac37db212aed236d4c415cfecb96aa42cfebc6d9" features = [ "compat", "rand", diff --git a/src/service/rooms/event_handler/acl_check.rs b/src/service/rooms/event_handler/acl_check.rs index ef12a25c..714b6fc1 100644 --- a/src/service/rooms/event_handler/acl_check.rs +++ b/src/service/rooms/event_handler/acl_check.rs @@ -6,7 +6,7 @@ use ruma::{ /// Returns Ok if the acl allows the server #[implement(super::Service)] -#[tracing::instrument(skip_all)] +#[tracing::instrument(skip_all, level = "debug")] pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Result { let Ok(acl_event_content) = self .services diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index b47111d9..0e4b919d 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -114,7 +114,7 @@ pub async fn state_resolution( &self, room_version: &RoomVersionId, state_sets: &[StateMap], - auth_chain_sets: &Vec>, + auth_chain_sets: &[HashSet], ) -> Result> { //TODO: ??? let _lock = self.services.globals.stateres_mutex.lock(); From b8f8f686349445e69b19d203a89cb97f71ed2330 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 1 Jan 2025 23:49:08 +0000 Subject: [PATCH 1304/2091] improve memory-usage output w/ more byte-sizes Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 21 ++++++++----- src/service/resolver/cache.rs | 18 +++++++++++- src/service/resolver/fed.rs | 10 +++++++ src/service/resolver/mod.rs | 35 ++++++++++++---------- src/service/rooms/state_accessor/mod.rs | 39 +++++++++++++++++++++---- 5 files changed, 92 insertions(+), 31 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 3f98babe..3a514d5c 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,7 +7,7 @@ use std::{ time::Instant, }; -use conduwuit::{error, Config, Result}; +use conduwuit::{error, utils::bytes::pretty, Config, Result}; use data::Data; use regex::RegexSet; use ruma::{OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, ServerName, UserId}; @@ -93,13 +93,18 @@ impl crate::Service for Service { Ok(Arc::new(s)) } - fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { - let bad_event_ratelimiter = self - .bad_event_ratelimiter - .read() - .expect("locked for reading") - .len(); - writeln!(out, "bad_event_ratelimiter: {bad_event_ratelimiter}")?; + fn memory_usage(&self, out: &mut dyn Write) -> Result { + let (ber_count, ber_bytes) = self.bad_event_ratelimiter.read()?.iter().fold( + (0_usize, 0_usize), + |(mut count, mut bytes), (event_id, _)| { + bytes = bytes.saturating_add(event_id.capacity()); + bytes = bytes.saturating_add(size_of::()); + count = count.saturating_add(1); + (count, bytes) + }, + ); + + writeln!(out, "bad_event_ratelimiter: {ber_count} ({})", pretty(ber_bytes))?; Ok(()) } diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index 3e961f4c..e309a129 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -6,7 +6,10 @@ use std::{ }; use arrayvec::ArrayVec; -use conduwuit::{trace, utils::rand}; +use conduwuit::{ + trace, + utils::{math::Expected, rand}, +}; use ruma::{OwnedServerName, ServerName}; use super::fed::FedDest; @@ -113,6 +116,15 @@ impl CachedDest { pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) } + + #[inline] + #[must_use] + pub fn size(&self) -> usize { + self.dest + .size() + .expected_add(self.host.len()) + .expected_add(size_of_val(&self.expire)) + } } impl CachedOverride { @@ -126,4 +138,8 @@ impl CachedOverride { pub(crate) fn default_expire() -> SystemTime { rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) } + + #[inline] + #[must_use] + pub fn size(&self) -> usize { size_of_val(self) } } diff --git a/src/service/resolver/fed.rs b/src/service/resolver/fed.rs index 3986db8e..76fc6894 100644 --- a/src/service/resolver/fed.rs +++ b/src/service/resolver/fed.rs @@ -5,6 +5,7 @@ use std::{ }; use arrayvec::ArrayString; +use conduwuit::utils::math::Expected; #[derive(Clone, Debug, PartialEq, Eq)] pub enum FedDest { @@ -76,6 +77,15 @@ impl FedDest { pub fn default_port() -> PortString { PortString::from(DEFAULT_PORT).expect("default port string") } + + #[inline] + #[must_use] + pub fn size(&self) -> usize { + match self { + | Self::Literal(saddr) => size_of_val(saddr), + | Self::Named(host, port) => host.len().expected_add(port.capacity()), + } + } } impl fmt::Display for FedDest { diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index e18cf0bd..6a6289b6 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -6,7 +6,7 @@ mod tests; use std::{fmt::Write, sync::Arc}; -use conduwuit::{Result, Server}; +use conduwuit::{utils, utils::math::Expected, Result, Server}; use self::{cache::Cache, dns::Resolver}; use crate::{client, Dep}; @@ -36,22 +36,25 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { - let resolver_overrides_cache = self - .cache - .overrides - .read() - .expect("locked for reading") - .len(); - writeln!(out, "resolver_overrides_cache: {resolver_overrides_cache}")?; + fn memory_usage(&self, out: &mut dyn Write) -> Result { + use utils::bytes::pretty; - let resolver_destinations_cache = self - .cache - .destinations - .read() - .expect("locked for reading") - .len(); - writeln!(out, "resolver_destinations_cache: {resolver_destinations_cache}")?; + let (oc_count, oc_bytes) = self.cache.overrides.read()?.iter().fold( + (0_usize, 0_usize), + |(count, bytes), (key, val)| { + (count.expected_add(1), bytes.expected_add(key.len()).expected_add(val.size())) + }, + ); + + let (dc_count, dc_bytes) = self.cache.destinations.read()?.iter().fold( + (0_usize, 0_usize), + |(count, bytes), (key, val)| { + (count.expected_add(1), bytes.expected_add(key.len()).expected_add(val.size())) + }, + ); + + writeln!(out, "resolver_overrides_cache: {oc_count} ({})", pretty(oc_bytes))?; + writeln!(out, "resolver_destinations_cache: {dc_count} ({})", pretty(dc_bytes))?; Ok(()) } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index fd132798..d89c8835 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -10,7 +10,11 @@ use std::{ use conduwuit::{ err, error, pdu::PduBuilder, - utils::{math::usize_from_f64, ReadyExt}, + utils, + utils::{ + math::{usize_from_f64, Expected}, + ReadyExt, + }, Err, Error, PduEvent, Result, }; use futures::StreamExt; @@ -84,12 +88,35 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { - let server_visibility_cache = self.server_visibility_cache.lock().expect("locked").len(); - writeln!(out, "server_visibility_cache: {server_visibility_cache}")?; + fn memory_usage(&self, out: &mut dyn Write) -> Result { + use utils::bytes::pretty; - let user_visibility_cache = self.user_visibility_cache.lock().expect("locked").len(); - writeln!(out, "user_visibility_cache: {user_visibility_cache}")?; + let (svc_count, svc_bytes) = self.server_visibility_cache.lock()?.iter().fold( + (0_usize, 0_usize), + |(count, bytes), (key, _)| { + ( + count.expected_add(1), + bytes + .expected_add(key.0.capacity()) + .expected_add(size_of_val(&key.1)), + ) + }, + ); + + let (uvc_count, uvc_bytes) = self.user_visibility_cache.lock()?.iter().fold( + (0_usize, 0_usize), + |(count, bytes), (key, _)| { + ( + count.expected_add(1), + bytes + .expected_add(key.0.capacity()) + .expected_add(size_of_val(&key.1)), + ) + }, + ); + + writeln!(out, "server_visibility_cache: {svc_count} ({})", pretty(svc_bytes))?; + writeln!(out, "user_visibility_cache: {uvc_count} ({})", pretty(uvc_bytes))?; Ok(()) } From b4ef646485de48eb660563f277cfc939d938ff17 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 2 Jan 2025 00:11:12 +0000 Subject: [PATCH 1305/2091] translate 5xx to 404 on remote media request Signed-off-by: Jason Volk --- src/service/media/remote.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index d5ad5391..ca73c3ef 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -331,7 +331,10 @@ fn handle_federation_error( // If we get these from any middleware we'll try the other endpoint rather than // giving up too early. - if error.status_code().is_client_error() || error.status_code().is_redirection() { + if error.status_code().is_redirection() + || error.status_code().is_client_error() + || error.status_code().is_server_error() + { return fallback(); } From 72797532b69b37d0ce532521f187f8a1396fc53c Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 1 Jan 2025 23:19:51 -0500 Subject: [PATCH 1306/2091] fix MSC4133 fields not being returned as original types Signed-off-by: strawberry --- src/api/client/unstable.rs | 8 ++++++++ src/service/users/mod.rs | 13 ++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 2c9add44..66cb31d5 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -506,6 +506,10 @@ pub(crate) async fn get_profile_key_route( return Err!(Request(NotFound("The requested profile key does not exist."))); } + if profile_key_value.is_empty() { + return Err!(Request(NotFound("The requested profile key does not exist."))); + } + return Ok(get_profile_key::unstable::Response { value: profile_key_value }); } } @@ -522,5 +526,9 @@ pub(crate) async fn get_profile_key_route( return Err!(Request(NotFound("The requested profile key does not exist."))); } + if profile_key_value.is_empty() { + return Err!(Request(NotFound("The requested profile key does not exist."))); + } + Ok(get_profile_key::unstable::Response { value: profile_key_value }) } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index fe064d9c..3c43968a 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -947,12 +947,13 @@ impl Service { user_id: &UserId, profile_key: &str, ) -> Result { - let key = (user_id, profile_key); self.db .useridprofilekey_value - .qry(&key) + .qry(&(user_id, profile_key)) .await - .deserialized() + .deserialized::>() + .map(serde_json::to_value)? + .map_err(Into::into) } /// Gets all the user's profile keys and values in an iterator @@ -960,14 +961,16 @@ impl Service { &'a self, user_id: &'a UserId, ) -> impl Stream + 'a + Send { - type KeyVal = ((Ignore, String), serde_json::Value); + type KeyVal = ((Ignore, String), Raw); let prefix = (user_id, Interfix); self.db .useridprofilekey_value .stream_prefix(&prefix) .ignore_err() - .map(|((_, key), val): KeyVal| (key, val)) + .ready_filter_map(|((_, key), val): KeyVal| { + Some((key, serde_json::to_value(val).ok()?)) + }) } /// Sets a new profile key value, removes the key if value is None From 77d7e8a3ad743f4043ff652939346ae6c7be0312 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 2 Jan 2025 06:35:33 +0000 Subject: [PATCH 1307/2091] add json array to serde_json::Value db deserialization without Raw --- src/database/de.rs | 4 ++++ src/service/users/mod.rs | 13 +++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/database/de.rs b/src/database/de.rs index f648b89a..4f5be6fc 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -351,6 +351,10 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { match self.record_peek_byte() { | Some(b'{') => self.deserialize_map(visitor), + | Some(b'[') => serde_json::Deserializer::from_slice(self.record_next()) + .deserialize_seq(visitor) + .map_err(Into::into), + | _ => self.deserialize_str(visitor), } } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 3c43968a..fe064d9c 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -947,13 +947,12 @@ impl Service { user_id: &UserId, profile_key: &str, ) -> Result { + let key = (user_id, profile_key); self.db .useridprofilekey_value - .qry(&(user_id, profile_key)) + .qry(&key) .await - .deserialized::>() - .map(serde_json::to_value)? - .map_err(Into::into) + .deserialized() } /// Gets all the user's profile keys and values in an iterator @@ -961,16 +960,14 @@ impl Service { &'a self, user_id: &'a UserId, ) -> impl Stream + 'a + Send { - type KeyVal = ((Ignore, String), Raw); + type KeyVal = ((Ignore, String), serde_json::Value); let prefix = (user_id, Interfix); self.db .useridprofilekey_value .stream_prefix(&prefix) .ignore_err() - .ready_filter_map(|((_, key), val): KeyVal| { - Some((key, serde_json::to_value(val).ok()?)) - }) + .map(|((_, key), val): KeyVal| (key, val)) } /// Sets a new profile key value, removes the key if value is None From b1182fe8a4065eb6edcfc992118b630f7baf5afc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 2 Jan 2025 07:16:46 +0000 Subject: [PATCH 1308/2091] improve some router tracing spans Signed-off-by: Jason Volk --- src/router/request.rs | 69 ++++++++++++++++++++++++++-------------- src/router/serve/unix.rs | 50 +++++++++++++++++++++-------- 2 files changed, 83 insertions(+), 36 deletions(-) diff --git a/src/router/request.rs b/src/router/request.rs index 559c7f88..ab98fe4f 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -12,6 +12,18 @@ use http::{Method, StatusCode, Uri}; parent = None, level = "trace", skip_all, + fields( + handled = %services + .server + .metrics + .requests_spawn_finished + .fetch_add(1, Ordering::Relaxed), + active = %services + .server + .metrics + .requests_spawn_active + .fetch_add(1, Ordering::Relaxed), + ) )] pub(crate) async fn spawn( State(services): State>, @@ -19,34 +31,56 @@ pub(crate) async fn spawn( next: axum::middleware::Next, ) -> Result { let server = &services.server; + + #[cfg(debug_assertions)] + defer! {{ + _ = server + .metrics + .requests_spawn_active + .fetch_sub(1, Ordering::Relaxed); + }}; + if !server.running() { debug_warn!("unavailable pending shutdown"); return Err(StatusCode::SERVICE_UNAVAILABLE); } - let active = server - .metrics - .requests_spawn_active - .fetch_add(1, Ordering::Relaxed); - trace!(active, "enter"); - defer! {{ - let active = server.metrics.requests_spawn_active.fetch_sub(1, Ordering::Relaxed); - let finished = server.metrics.requests_spawn_finished.fetch_add(1, Ordering::Relaxed); - trace!(active, finished, "leave"); - }}; - let fut = next.run(req); let task = server.runtime().spawn(fut); task.await.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) } -#[tracing::instrument(level = "debug", skip_all)] +#[tracing::instrument( + level = "debug", + skip_all, + fields( + handled = %services + .server + .metrics + .requests_handle_finished + .fetch_add(1, Ordering::Relaxed), + active = %services + .server + .metrics + .requests_handle_active + .fetch_add(1, Ordering::Relaxed), + ) +)] pub(crate) async fn handle( State(services): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { let server = &services.server; + + #[cfg(debug_assertions)] + defer! {{ + _ = server + .metrics + .requests_handle_active + .fetch_sub(1, Ordering::Relaxed); + }}; + if !server.running() { debug_warn!( method = %req.method(), @@ -57,17 +91,6 @@ pub(crate) async fn handle( return Err(StatusCode::SERVICE_UNAVAILABLE); } - let active = server - .metrics - .requests_handle_active - .fetch_add(1, Ordering::Relaxed); - trace!(active, "enter"); - defer! {{ - let active = server.metrics.requests_handle_active.fetch_sub(1, Ordering::Relaxed); - let finished = server.metrics.requests_handle_finished.fetch_add(1, Ordering::Relaxed); - trace!(active, finished, "leave"); - }}; - let uri = req.uri().clone(); let method = req.method().clone(); let result = next.run(req).await; diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index f5018455..fcd361f0 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -2,6 +2,7 @@ use std::{ net::{self, IpAddr, Ipv4Addr}, + os::fd::AsRawFd, path::Path, sync::{atomic::Ordering, Arc}, }; @@ -60,31 +61,54 @@ pub(super) async fn serve( Ok(()) } +#[tracing::instrument( + level = "trace", + skip_all, + fields( + ?listener, + socket = ?conn.0, + ), +)] async fn accept( server: &Arc, listener: &UnixListener, tasks: &mut JoinSet<()>, - mut app: MakeService, + app: MakeService, builder: server::conn::auto::Builder, conn: (UnixStream, SocketAddr), ) { - let (socket, remote) = conn; - let socket = TokioIo::new(socket); - trace!(?listener, ?socket, ?remote, "accepted"); - - let called = app.call(NULL_ADDR).await.unwrap_infallible(); - - let service = move |req: Request| called.clone().oneshot(req); - let handler = service_fn(service); - let task = async move { - // bug on darwin causes all results to be errors. do not unwrap this - _ = builder.serve_connection(socket, handler).await; - }; + let (socket, _) = conn; + let server_ = server.clone(); + let task = async move { accepted(server_, builder, socket, app).await }; _ = tasks.spawn_on(task, server.runtime()); while tasks.try_join_next().is_some() {} } +#[tracing::instrument( + level = "trace", + skip_all, + fields( + fd = %socket.as_raw_fd(), + path = ?socket.local_addr(), + ), +)] +async fn accepted( + server: Arc, + builder: server::conn::auto::Builder, + socket: UnixStream, + mut app: MakeService, +) { + let socket = TokioIo::new(socket); + let called = app.call(NULL_ADDR).await.unwrap_infallible(); + let service = move |req: Request| called.clone().oneshot(req); + let handler = service_fn(service); + trace!(?socket, ?handler, "serving connection"); + + // bug on darwin causes all results to be errors. do not unwrap this + _ = builder.serve_connection(socket, handler).await; +} + async fn init(server: &Arc) -> Result { use std::os::unix::fs::PermissionsExt; From 50cadbee9624f1a24f2395ce28439d7140a433fa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 2 Jan 2025 07:31:16 +0000 Subject: [PATCH 1309/2091] add faster shutdown point in unix socket listener Signed-off-by: Jason Volk --- src/router/serve/unix.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index fcd361f0..6855b34c 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -106,7 +106,10 @@ async fn accepted( trace!(?socket, ?handler, "serving connection"); // bug on darwin causes all results to be errors. do not unwrap this - _ = builder.serve_connection(socket, handler).await; + tokio::select! { + () = server.until_shutdown() => (), + _ = builder.serve_connection(socket, handler) => (), + }; } async fn init(server: &Arc) -> Result { From a5d70f73566044369231a43ae2f2be3bc5e9d715 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 2 Jan 2025 18:16:00 -0500 Subject: [PATCH 1310/2091] add some m.room.member checks on putting direct state events Signed-off-by: strawberry --- src/api/client/state.rs | 118 +++++++++++------- src/api/server/send_join.rs | 11 +- .../event_handler/handle_incoming_pdu.rs | 16 +-- .../rooms/event_handler/handle_outlier_pdu.rs | 2 +- src/service/rooms/timeline/mod.rs | 23 ++++ src/service/sending/sender.rs | 7 +- 6 files changed, 117 insertions(+), 60 deletions(-) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index e5a7dd28..d00ee5e5 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,15 +1,13 @@ use axum::extract::State; -use conduwuit::{err, pdu::PduBuilder, utils::BoolExt, Err, Error, PduEvent, Result}; +use conduwuit::{err, pdu::PduBuilder, utils::BoolExt, Err, PduEvent, Result}; use ruma::{ - api::client::{ - error::ErrorKind, - state::{get_state_events, get_state_events_for_key, send_state_event}, - }, + api::client::state::{get_state_events, get_state_events_for_key, send_state_event}, events::{ room::{ canonical_alias::RoomCanonicalAliasEventContent, history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, }, AnyStateEventContent, StateEventType, }, @@ -23,11 +21,6 @@ use crate::{Ruma, RumaResponse}; /// # `PUT /_matrix/client/*/rooms/{roomId}/state/{eventType}/{stateKey}` /// /// Sends a state event into the room. -/// -/// - The only requirement for the content is that it has to be valid json -/// - Tries to send the event into the room, auth rules will determine if it is -/// allowed -/// - If event is new `canonical_alias`: Rejects if alias is incorrect pub(crate) async fn send_state_event_for_key_route( State(services): State, body: Ruma, @@ -41,7 +34,7 @@ pub(crate) async fn send_state_event_for_key_route( &body.room_id, &body.event_type, &body.body.body, - body.state_key.clone(), + &body.state_key, if body.appservice_info.is_some() { body.timestamp } else { @@ -55,11 +48,6 @@ pub(crate) async fn send_state_event_for_key_route( /// # `PUT /_matrix/client/*/rooms/{roomId}/state/{eventType}` /// /// Sends a state event into the room. -/// -/// - The only requirement for the content is that it has to be valid json -/// - Tries to send the event into the room, auth rules will determine if it is -/// allowed -/// - If event is new `canonical_alias`: Rejects if alias is incorrect pub(crate) async fn send_state_event_for_empty_key_route( State(services): State, body: Ruma, @@ -172,10 +160,10 @@ async fn send_state_event_for_key_helper( room_id: &RoomId, event_type: &StateEventType, json: &Raw, - state_key: String, + state_key: &str, timestamp: Option, ) -> Result { - allowed_to_send_state_event(services, room_id, event_type, json).await?; + allowed_to_send_state_event(services, room_id, event_type, state_key, json).await?; let state_lock = services.rooms.state.mutex.lock(room_id).await; let event_id = services .rooms @@ -184,7 +172,7 @@ async fn send_state_event_for_key_helper( PduBuilder { event_type: event_type.to_string().into(), content: serde_json::from_str(json.json().get())?, - state_key: Some(state_key), + state_key: Some(String::from(state_key)), timestamp, ..Default::default() }, @@ -201,6 +189,7 @@ async fn allowed_to_send_state_event( services: &Services, room_id: &RoomId, event_type: &StateEventType, + state_key: &str, json: &Raw, ) -> Result { match event_type { @@ -212,10 +201,7 @@ async fn allowed_to_send_state_event( // Forbid m.room.encryption if encryption is disabled | StateEventType::RoomEncryption => if !services.globals.allow_encryption() { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Encryption has been disabled", - )); + return Err!(Request(Forbidden("Encryption is disabled on this homeserver."))); }, // admin room is a sensitive room, it should not ever be made public | StateEventType::RoomJoinRules => { @@ -225,10 +211,9 @@ async fn allowed_to_send_state_event( serde_json::from_str::(json.json().get()) { if join_rule.join_rule == JoinRule::Public { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Admin room is not allowed to be public.", - )); + return Err!(Request(Forbidden( + "Admin room is a sensitive room, it cannot be made public" + ))); } } } @@ -236,26 +221,22 @@ async fn allowed_to_send_state_event( }, // admin room is a sensitive room, it should not ever be made world readable | StateEventType::RoomHistoryVisibility => { - if let Ok(admin_room_id) = services.admin.get_admin_room().await { - if admin_room_id == room_id { - if let Ok(visibility_content) = serde_json::from_str::< - RoomHistoryVisibilityEventContent, - >(json.json().get()) - { - if visibility_content.history_visibility + if let Ok(visibility_content) = + serde_json::from_str::(json.json().get()) + { + if let Ok(admin_room_id) = services.admin.get_admin_room().await { + if admin_room_id == room_id + && visibility_content.history_visibility == HistoryVisibility::WorldReadable - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Admin room is not allowed to be made world readable (public \ - room history).", - )); - } + { + return Err!(Request(Forbidden( + "Admin room is a sensitive room, it cannot be made world readable \ + (public room history)." + ))); } } } }, - // TODO: allow alias if it previously existed | StateEventType::RoomCanonicalAlias => { if let Ok(canonical_alias) = serde_json::from_str::(json.json().get()) @@ -289,6 +270,59 @@ async fn allowed_to_send_state_event( } } }, + | StateEventType::RoomMember => { + let Ok(membership_content) = + serde_json::from_str::(json.json().get()) + else { + return Err!(Request(BadJson( + "Membership content must have a valid JSON body with at least a valid \ + membership state." + ))); + }; + + let Ok(state_key) = UserId::parse(state_key) else { + return Err!(Request(BadJson( + "Membership event has invalid or non-existent state key" + ))); + }; + + if let Some(authorising_user) = membership_content.join_authorized_via_users_server { + if membership_content.membership != MembershipState::Join { + return Err!(Request(BadJson( + "join_authorised_via_users_server is only for member joins" + ))); + } + + if services + .rooms + .state_cache + .is_joined(state_key, room_id) + .await + { + return Err!(Request(InvalidParam( + "{state_key} is already joined, an authorising user is not required." + ))); + } + + if !services.globals.user_is_local(&authorising_user) { + return Err!(Request(InvalidParam( + "Authorising user {authorising_user} does not belong to this homeserver" + ))); + } + + if !services + .rooms + .state_cache + .is_joined(&authorising_user, room_id) + .await + { + return Err!(Request(InvalidParam( + "Authorising user {authorising_user} is not in the room, they cannot \ + authorise the join." + ))); + } + } + }, | _ => (), } diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index fe0277d1..6cbe5143 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -61,12 +61,11 @@ async fn create_join_event( }; let event_room_id: OwnedRoomId = serde_json::from_value( - serde_json::to_value( - value - .get("room_id") - .ok_or_else(|| err!(Request(BadJson("Event missing room_id property."))))?, - ) - .expect("CanonicalJson is valid json value"), + value + .get("room_id") + .ok_or_else(|| err!(Request(BadJson("Event missing room_id property."))))? + .clone() + .into(), ) .map_err(|e| err!(Request(BadJson(warn!("room_id field is not a valid room ID: {e}")))))?; diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 0e0409b4..4c2fb2f7 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -4,12 +4,9 @@ use std::{ time::Instant, }; -use conduwuit::{debug, err, implement, warn, Error, Result}; +use conduwuit::{debug, err, implement, warn, Err, Result}; use futures::{FutureExt, TryFutureExt}; -use ruma::{ - api::client::error::ErrorKind, events::StateEventType, CanonicalJsonValue, EventId, RoomId, - ServerName, UserId, -}; +use ruma::{events::StateEventType, CanonicalJsonValue, EventId, RoomId, ServerName, UserId}; use super::{check_room_id, get_room_version_id}; use crate::rooms::timeline::RawPduId; @@ -58,15 +55,14 @@ pub async fn handle_incoming_pdu<'a>( // 1.1 Check the server is in the room if !self.services.metadata.exists(room_id).await { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server")); + return Err!(Request(NotFound("Room is unknown to this server"))); } // 1.2 Check if the room is disabled if self.services.metadata.is_disabled(room_id).await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Federation of this room is currently disabled on this server.", - )); + return Err!(Request(Forbidden( + "Federation of this room is currently disabled on this server." + ))); } // 1.3.1 Check room ACL on origin field/server diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index c3278329..3ad73295 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -68,7 +68,7 @@ pub(super) async fn handle_outlier_pdu<'a>( let incoming_pdu = serde_json::from_value::( serde_json::to_value(&val).expect("CanonicalJsonObj is a valid JsonValue"), ) - .map_err(|_| Error::bad_database("Event is not a valid PDU."))?; + .map_err(|e| err!(Request(BadJson(debug_warn!("Event is not a valid PDU: {e}")))))?; check_room_id(room_id, &incoming_pdu)?; diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 81df7b35..2a272c38 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -901,6 +901,29 @@ impl Service { } }; + if pdu.kind == TimelineEventType::RoomMember { + let content: RoomMemberEventContent = pdu.get_content()?; + + if content.join_authorized_via_users_server.is_some() + && content.membership != MembershipState::Join + { + return Err!(Request(BadJson( + "join_authorised_via_users_server is only for member joins" + ))); + } + + if content + .join_authorized_via_users_server + .as_ref() + .is_some_and(|authorising_user| { + !self.services.globals.user_is_local(authorising_user) + }) { + return Err!(Request(InvalidParam( + "Authorising user does not belong to this homeserver" + ))); + } + } + // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't // fail. diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 482c31cf..a9abada4 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -739,7 +739,12 @@ impl Service { )); }; - let mut pdus = Vec::new(); + let mut pdus = Vec::with_capacity( + events + .iter() + .filter(|event| matches!(event, SendingEvent::Pdu(_))) + .count(), + ); for event in &events { match event { | SendingEvent::Pdu(pdu_id) => { From 6c8a1b5e9b3854c036e5fc74733357f223dc437f Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 2 Jan 2025 18:48:04 -0500 Subject: [PATCH 1311/2091] bump ruwuma Signed-off-by: strawberry --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cf08509f..2005202d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3173,7 +3173,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "assign", "js_int", @@ -3195,7 +3195,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "js_int", "ruma-common", @@ -3207,7 +3207,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "as_variant", "assign", @@ -3230,7 +3230,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "as_variant", "base64 0.22.1", @@ -3261,7 +3261,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3286,7 +3286,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "bytes", "http", @@ -3304,7 +3304,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "js_int", "thiserror 2.0.7", @@ -3313,7 +3313,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "js_int", "ruma-common", @@ -3323,7 +3323,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3338,7 +3338,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "js_int", "ruma-common", @@ -3350,7 +3350,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "headers", "http", @@ -3363,7 +3363,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3379,7 +3379,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ac37db212aed236d4c415cfecb96aa42cfebc6d9#ac37db212aed236d4c415cfecb96aa42cfebc6d9" +source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index a59dcfcd..3d65e4cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -336,7 +336,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "ac37db212aed236d4c415cfecb96aa42cfebc6d9" +rev = "c4f55b39900b33b2d443dd12a6a2dab50961fdfb" features = [ "compat", "rand", From 7a349fdc58bb1322991a7b403178b8dccedf2e59 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 2 Jan 2025 18:55:52 -0500 Subject: [PATCH 1312/2091] use latest sccache, bump nix flake lock, cache everything in rust-cache Signed-off-by: strawberry --- .github/workflows/ci.yml | 12 +++++++----- Cargo.lock | 6 +++--- Cargo.toml | 10 +++++----- flake.lock | 24 ++++++++++++------------ src/database/pool.rs | 6 +++--- 5 files changed, 30 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b20a43e..453d9df8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -188,13 +188,13 @@ jobs: - name: Run sccache-cache if: (env.SCCACHE_GHA_ENABLED == 'true') uses: mozilla-actions/sccache-action@main - with: - version: "v0.8.2" # use rust-cache - uses: Swatinem/rust-cache@v2 with: cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" - name: Run CI tests env: @@ -366,13 +366,13 @@ jobs: - name: Run sccache-cache if: (env.SCCACHE_GHA_ENABLED == 'true') uses: mozilla-actions/sccache-action@main - with: - version: "v0.8.2" # use rust-cache - uses: Swatinem/rust-cache@v2 with: cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" - name: Build static ${{ matrix.target }}-all-features run: | @@ -660,6 +660,8 @@ jobs: - uses: Swatinem/rust-cache@v2 with: cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" # Nix can't do portable macOS builds yet - name: Build macOS x86_64 binary @@ -921,4 +923,4 @@ jobs: if [ ! -z $GITLAB_TOKEN ]; then echo "- \`docker pull ${GLCR_TAG}\`" >> $GITHUB_STEP_SUMMARY echo "- \`docker pull ${GLCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY - fi \ No newline at end of file + fi diff --git a/Cargo.lock b/Cargo.lock index 2005202d..3f900a11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,7 +95,7 @@ checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" [[package]] name = "async-channel" version = "2.3.1" -source = "git+https://github.com/jevolk/async-channel?rev=fefa543ca5eddf21237d75776fce98b7e09e924a#fefa543ca5eddf21237d75776fce98b7e09e924a" +source = "git+https://github.com/girlbossceo/async-channel?rev=92e5e74063bf2a3b10414bcc8a0d68b235644280#92e5e74063bf2a3b10414bcc8a0d68b235644280" dependencies = [ "concurrent-queue", "event-listener-strategy", @@ -938,7 +938,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core_affinity" version = "0.8.1" -source = "git+https://github.com/jevolk/core_affinity_rs?rev=9c8e51510c35077df888ee72a36b4b05637147da#9c8e51510c35077df888ee72a36b4b05637147da" +source = "git+https://github.com/girlbossceo/core_affinity_rs?rev=9c8e51510c35077df888ee72a36b4b05637147da#9c8e51510c35077df888ee72a36b4b05637147da" dependencies = [ "libc", "num_cpus", @@ -1258,7 +1258,7 @@ dependencies = [ [[package]] name = "event-listener" version = "5.3.1" -source = "git+https://github.com/jevolk/event-listener?rev=96d7e0fc026d8f708b19bc9267a382676a50354c#96d7e0fc026d8f708b19bc9267a382676a50354c" +source = "git+https://github.com/girlbossceo/event-listener?rev=fe4aebeeaae435af60087ddd56b573a2e0be671d#fe4aebeeaae435af60087ddd56b573a2e0be671d" dependencies = [ "concurrent-queue", "parking", diff --git a/Cargo.toml b/Cargo.toml index 3d65e4cd..76acda80 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -532,14 +532,14 @@ rev = "deaeb0694e2083f53d363b648da06e10fc13900c" # adds LIFO queue scheduling; this should be updated with PR progress. [patch.crates-io.event-listener] -git = "https://github.com/jevolk/event-listener" -rev = "96d7e0fc026d8f708b19bc9267a382676a50354c" +git = "https://github.com/girlbossceo/event-listener" +rev = "fe4aebeeaae435af60087ddd56b573a2e0be671d" [patch.crates-io.async-channel] -git = "https://github.com/jevolk/async-channel" -rev = "fefa543ca5eddf21237d75776fce98b7e09e924a" +git = "https://github.com/girlbossceo/async-channel" +rev = "92e5e74063bf2a3b10414bcc8a0d68b235644280" [patch.crates-io.core_affinity] -git = "https://github.com/jevolk/core_affinity_rs" +git = "https://github.com/girlbossceo/core_affinity_rs" rev = "9c8e51510c35077df888ee72a36b4b05637147da" # diff --git a/flake.lock b/flake.lock index 94cef4dd..35029076 100644 --- a/flake.lock +++ b/flake.lock @@ -117,11 +117,11 @@ }, "crane_2": { "locked": { - "lastModified": 1733688869, - "narHash": "sha256-KrhxxFj1CjESDrL5+u/zsVH0K+Ik9tvoac/oFPoxSB8=", + "lastModified": 1734808813, + "narHash": "sha256-3aH/0Y6ajIlfy7j52FGZ+s4icVX0oHhqBzRdlOeztqg=", "owner": "ipetkov", "repo": "crane", - "rev": "604637106e420ad99907cae401e13ab6b452e7d9", + "rev": "72e2d02dbac80c8c86bf6bf3e785536acf8ee926", "type": "github" }, "original": { @@ -170,11 +170,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1733726208, - "narHash": "sha256-Z6zL4AtpZWxkvTd3l0KkPZamu2wtTKl4nNiqNSlgsb4=", + "lastModified": 1735799625, + "narHash": "sha256-lFadwWDvVIub11bwfZhsh2WUByf9LOi6yjsSUMmE0xk=", "owner": "nix-community", "repo": "fenix", - "rev": "d51a64e1d23e509f28a6955a6652cc62409dd4a8", + "rev": "a9d84a1545814910cb4ab0515ed6921e8b07ee95", "type": "github" }, "original": { @@ -550,11 +550,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1733656523, - "narHash": "sha256-w0FXPfpGhOihoJDiwMsyN1EzpsXi2F8VQ+NVZQSMtys=", + "lastModified": 1735685343, + "narHash": "sha256-h1CpBzdJDNtSUb5QMyfFHKHocTTky+4McgQEBQBM+xA=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "93dc9803a1ee435e590b02cde9589038d5cc3a4e", + "rev": "81934660d6e9ea54d2f0cdee821e8533b10c221a", "type": "github" }, "original": { @@ -599,11 +599,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1733642337, - "narHash": "sha256-I1uc97f/cNhOpCemIbBAUS+CV0R7jts0NW9lc8jrpxc=", + "lastModified": 1735742096, + "narHash": "sha256-q3a80h8Jf8wfmPURUgRR46nQCB3I5fhZ+/swulTF5HY=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "4c755e62a617eeeef3066994731ce1cdd16504ac", + "rev": "7e639ee3dda6ed9cecc79d41f6d38235121e483d", "type": "github" }, "original": { diff --git a/src/database/pool.rs b/src/database/pool.rs index bcf20de8..0295e0e9 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -8,7 +8,7 @@ use std::{ }, }; -use async_channel::{Receiver, RecvError, Sched, Sender}; +use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ debug, debug_warn, defer, err, implement, result::DebugInspect, @@ -71,13 +71,13 @@ const BATCH_INLINE: usize = 1; #[implement(Pool)] pub(crate) async fn new(server: &Arc) -> Result> { - const CHAN_SCHED: (Sched, Sched) = (Sched::Fifo, Sched::Lifo); + const CHAN_SCHED: (QueueStrategy, QueueStrategy) = (QueueStrategy::Fifo, QueueStrategy::Lifo); let (total_workers, queue_sizes, topology) = configure(server); let (senders, receivers) = queue_sizes .into_iter() - .map(|cap| async_channel::bounded_with_sched(cap, CHAN_SCHED)) + .map(|cap| async_channel::bounded_with_queue_strategy(cap, CHAN_SCHED)) .unzip(); let pool = Arc::new(Self { From 3b8a90ad1381e740c5420846ccad7a418581521c Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 2 Jan 2025 20:56:27 -0500 Subject: [PATCH 1313/2091] delete more stuff to free up runner space Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 453d9df8..4a4df488 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -103,7 +103,7 @@ jobs: sudo docker image prune --all --force || true sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true sudo apt clean - sudo rm -v -rf /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/heroku + sudo rm -rf /usr/local/lib/android /usr/local/julia* /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/local/lib/heroku /usr/lib/heroku /usr/local/share/boost /usr/share/dotnet /usr/local/bin/cmake* /usr/local/bin/stack /usr/local/bin/terraform /opt/microsoft/powershell /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go /opt/hostedtoolcache/PyPy /usr/local/bin/sam || true set -o pipefail - name: Sync repository @@ -274,7 +274,7 @@ jobs: sudo docker image prune --all --force || true sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true sudo apt clean - sudo rm -v -rf /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/heroku + sudo rm -rf /usr/local/lib/android /usr/local/julia* /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/local/lib/heroku /usr/lib/heroku /usr/local/share/boost /usr/share/dotnet /usr/local/bin/cmake* /usr/local/bin/stack /usr/local/bin/terraform /opt/microsoft/powershell /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go /opt/hostedtoolcache/PyPy /usr/local/bin/sam || true set -o pipefail - name: Sync repository From f1af7ec08c9af0efbfebcd814111fb546e258552 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 3 Jan 2025 02:34:41 +0000 Subject: [PATCH 1314/2091] fix unused import in release mode Signed-off-by: Jason Volk --- src/router/request.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/router/request.rs b/src/router/request.rs index ab98fe4f..ca063338 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -4,7 +4,7 @@ use axum::{ extract::State, response::{IntoResponse, Response}, }; -use conduwuit::{debug, debug_error, debug_warn, defer, err, error, trace, Result}; +use conduwuit::{debug, debug_error, debug_warn, err, error, trace, Result}; use conduwuit_service::Services; use http::{Method, StatusCode, Uri}; @@ -33,7 +33,7 @@ pub(crate) async fn spawn( let server = &services.server; #[cfg(debug_assertions)] - defer! {{ + conduwuit::defer! {{ _ = server .metrics .requests_spawn_active @@ -74,7 +74,7 @@ pub(crate) async fn handle( let server = &services.server; #[cfg(debug_assertions)] - defer! {{ + conduwuit::defer! {{ _ = server .metrics .requests_handle_active From e5049cae4a3890dc5f61ead53281f23b36bf4c97 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 2 Jan 2025 22:02:11 -0500 Subject: [PATCH 1315/2091] update complement test results Signed-off-by: June Clementine Strawberry --- tests/test_results/complement/test_results.jsonl | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 26ae6931..9b4d2838 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -191,32 +191,32 @@ {"Action":"fail","Test":"TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members"} {"Action":"pass","Test":"TestRemoteTyping"} -{"Action":"pass","Test":"TestRestrictedRoomsLocalJoin"} +{"Action":"fail","Test":"TestRestrictedRoomsLocalJoin"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_fail_when_left_allowed_room"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_fail_with_mangled_join_rules"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_succeed_when_invited"} -{"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_succeed_when_joined_to_allowed_room"} -{"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room"} +{"Action":"fail","Test":"TestRestrictedRoomsLocalJoin/Join_should_succeed_when_joined_to_allowed_room"} +{"Action":"fail","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_when_left_allowed_room"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_fail_with_mangled_join_rules"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_invited"} -{"Action":"pass","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room"} +{"Action":"fail","Test":"TestRestrictedRoomsLocalJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoin"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_fail_when_left_allowed_room"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_fail_with_mangled_join_rules"} -{"Action":"fail","Test":"TestRestrictedRoomsRemoteJoin/Join_should_succeed_when_invited"} -{"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_succeed_when_joined_to_allowed_room"} +{"Action":"pass","Test":"TestRestrictedRoomsRemoteJoin/Join_should_succeed_when_invited"} +{"Action":"fail","Test":"TestRestrictedRoomsRemoteJoin/Join_should_succeed_when_joined_to_allowed_room"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinFailOver"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinFailOverInMSC3787Room"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_when_left_allowed_room"} {"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_fail_with_mangled_join_rules"} -{"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_invited"} -{"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room"} +{"Action":"pass","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_invited"} +{"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinInMSC3787Room/Join_should_succeed_when_joined_to_allowed_room"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinLocalUser"} {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinLocalUserInMSC3787Room"} {"Action":"pass","Test":"TestRestrictedRoomsSpacesSummaryFederation"} From 8c74e35e7640a041c1f3496d82585e5240294352 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 4 Jan 2025 03:15:48 +0000 Subject: [PATCH 1316/2091] automatically retry returning data in syncv3 (#652) * automatically retry returning data in syncv3 * reference service * clippy fixes --- src/api/client/sync/v3.rs | 50 ++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 406b497d..9f9ccfab 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -124,6 +124,33 @@ pub(crate) async fn sync_events_route( // Setup watchers, so if there's no response, we can wait for them let watcher = services.sync.watch(sender_user, sender_device); + let response = build_sync_events(&services, &body).await?; + if body.body.full_state + || !(response.rooms.is_empty() + && response.presence.is_empty() + && response.account_data.is_empty() + && response.device_lists.is_empty() + && response.to_device.is_empty()) + { + return Ok(response); + } + + // Hang a few seconds so requests are not spammed + // Stop hanging if new info arrives + let default = Duration::from_secs(30); + let duration = cmp::min(body.body.timeout.unwrap_or(default), default); + _ = tokio::time::timeout(duration, watcher).await; + + // Retry returning data + build_sync_events(&services, &body).await +} + +pub(crate) async fn build_sync_events( + services: &Services, + body: &Ruma, +) -> Result> { + let (sender_user, sender_device) = body.sender(); + let next_batch = services.globals.current_count()?; let next_batch_string = next_batch.to_string(); @@ -163,7 +190,7 @@ pub(crate) async fn sync_events_route( .map(ToOwned::to_owned) .broad_filter_map(|room_id| { load_joined_room( - &services, + services, sender_user, sender_device, room_id.clone(), @@ -196,7 +223,7 @@ pub(crate) async fn sync_events_route( .rooms_left(sender_user) .broad_filter_map(|(room_id, _)| { handle_left_room( - &services, + services, since, room_id.clone(), sender_user, @@ -242,7 +269,7 @@ pub(crate) async fn sync_events_route( let presence_updates: OptionFuture<_> = services .globals .allow_local_presence() - .then(|| process_presence_updates(&services, since, sender_user)) + .then(|| process_presence_updates(services, since, sender_user)) .into(); let account_data = services @@ -292,7 +319,7 @@ pub(crate) async fn sync_events_route( .stream() .broad_filter_map(|user_id| async move { let no_shared_encrypted_room = - !share_encrypted_room(&services, sender_user, &user_id, None).await; + !share_encrypted_room(services, sender_user, &user_id, None).await; no_shared_encrypted_room.then_some(user_id) }) .ready_fold(HashSet::new(), |mut device_list_left, user_id| { @@ -327,21 +354,6 @@ pub(crate) async fn sync_events_route( to_device: ToDevice { events: to_device_events }, }; - // TODO: Retry the endpoint instead of returning - if !full_state - && response.rooms.is_empty() - && response.presence.is_empty() - && response.account_data.is_empty() - && response.device_lists.is_empty() - && response.to_device.is_empty() - { - // Hang a few seconds so requests are not spammed - // Stop hanging if new info arrives - let default = Duration::from_secs(30); - let duration = cmp::min(body.body.timeout.unwrap_or(default), default); - _ = tokio::time::timeout(duration, watcher).await; - } - Ok(response) } From 7526ba9d6fc4221a338056802f8456733da314d6 Mon Sep 17 00:00:00 2001 From: Neil Svedberg Date: Mon, 6 Jan 2025 14:20:52 -0500 Subject: [PATCH 1317/2091] Add header to console When the console is launched, it now prints this message: conduwuit VERSION admin console "help" for help, ^D to exit the console, ^\ to stop the server --- src/service/admin/console.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 0edcd2f3..de201f4b 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -58,6 +58,7 @@ impl Console { pub async fn close(self: &Arc) { self.interrupt(); + let Some(worker_join) = self.worker_join.lock().expect("locked").take() else { return; }; @@ -92,6 +93,12 @@ impl Console { #[tracing::instrument(skip_all, name = "console", level = "trace")] async fn worker(self: Arc) { debug!("session starting"); + + self.output + .print_inline(&format!("**conduwuit {}** admin console\n", conduwuit::version())); + self.output + .print_text("\"help\" for help, ^D to exit the console, ^\\ to stop the server\n"); + while self.server.running() { match self.readline().await { | Ok(event) => match event { @@ -147,6 +154,7 @@ impl Console { self.add_history(line.clone()); let future = self.clone().process(line); + let (abort, abort_reg) = AbortHandle::new_pair(); let future = Abortable::new(future, abort_reg); _ = self.command_abort.lock().expect("locked").insert(abort); From 82168b972a175720f79e68ec6a5ffce8dced9f9e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 8 Jan 2025 01:20:42 +0000 Subject: [PATCH 1318/2091] fix heroes calculation regression Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 9f9ccfab..b7ecd6b9 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -1235,7 +1235,7 @@ async fn calculate_counts( let (joined_member_count, invited_member_count) = join(joined_member_count, invited_member_count).await; - let small_room = joined_member_count.saturating_add(invited_member_count) > 5; + let small_room = joined_member_count.saturating_add(invited_member_count) <= 5; let heroes: OptionFuture<_> = small_room .then(|| calculate_heroes(services, room_id, sender_user)) From a3f9432da879ff0c34fd2a94dcc6f25d6e5fb9e3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 5 Jan 2025 03:18:30 +0000 Subject: [PATCH 1319/2091] eliminate the state-res mutex hazard Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 3 --- src/service/rooms/event_handler/resolve_state.rs | 3 --- 2 files changed, 6 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 3a514d5c..38d7f786 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -11,7 +11,6 @@ use conduwuit::{error, utils::bytes::pretty, Config, Result}; use data::Data; use regex::RegexSet; use ruma::{OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, ServerName, UserId}; -use tokio::sync::Mutex; use crate::service; @@ -21,7 +20,6 @@ pub struct Service { pub config: Config, jwt_decoding_key: Option, pub bad_event_ratelimiter: Arc>>, - pub stateres_mutex: Arc>, pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, pub turn_secret: String, @@ -70,7 +68,6 @@ impl crate::Service for Service { config: config.clone(), jwt_decoding_key, bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), - stateres_mutex: Arc::new(Mutex::new(())), admin_alias: OwnedRoomAliasId::try_from(format!("#admins:{}", &config.server_name)) .expect("#admins:server_name is valid alias name"), server_user: UserId::parse_with_server_name( diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 0e4b919d..f21f7b66 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -116,9 +116,6 @@ pub async fn state_resolution( state_sets: &[StateMap], auth_chain_sets: &[HashSet], ) -> Result> { - //TODO: ??? - let _lock = self.services.globals.stateres_mutex.lock(); - state_res::resolve( room_version, state_sets.iter(), From 27328cbc01e4d10bdade4c92a1823873c170d582 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 31 Dec 2024 01:11:58 +0000 Subject: [PATCH 1320/2091] additional futures extension utils Signed-off-by: Jason Volk --- src/core/utils/future/ext_ext.rs | 34 +++++++++++++++++++++ src/core/utils/future/mod.rs | 2 ++ src/core/utils/future/try_ext_ext.rs | 23 ++++++++++++++- src/core/utils/stream/mod.rs | 2 ++ src/core/utils/stream/try_ready.rs | 20 ++++++++++++- src/core/utils/stream/try_tools.rs | 44 ++++++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 src/core/utils/future/ext_ext.rs create mode 100644 src/core/utils/stream/try_tools.rs diff --git a/src/core/utils/future/ext_ext.rs b/src/core/utils/future/ext_ext.rs new file mode 100644 index 00000000..38decaae --- /dev/null +++ b/src/core/utils/future/ext_ext.rs @@ -0,0 +1,34 @@ +//! Extended external extensions to futures::FutureExt + +use std::marker::Unpin; + +use futures::{future, future::Select, Future}; + +/// This interface is not necessarily complete; feel free to add as-needed. +pub trait ExtExt +where + Self: Future + Send, +{ + fn until(self, f: F) -> Select + where + Self: Sized, + F: FnOnce() -> B, + A: Future + From + Send + Unpin, + B: Future + Send + Unpin; +} + +impl ExtExt for Fut +where + Fut: Future + Send, +{ + #[inline] + fn until(self, f: F) -> Select + where + Self: Sized, + F: FnOnce() -> B, + A: Future + From + Send + Unpin, + B: Future + Send + Unpin, + { + future::select(self.into(), f()) + } +} diff --git a/src/core/utils/future/mod.rs b/src/core/utils/future/mod.rs index 3d8ec8f4..153dcfe1 100644 --- a/src/core/utils/future/mod.rs +++ b/src/core/utils/future/mod.rs @@ -1,5 +1,7 @@ +mod ext_ext; mod option_ext; mod try_ext_ext; +pub use ext_ext::ExtExt; pub use option_ext::OptionExt; pub use try_ext_ext::TryExtExt; diff --git a/src/core/utils/future/try_ext_ext.rs b/src/core/utils/future/try_ext_ext.rs index 19761309..aa3d72e4 100644 --- a/src/core/utils/future/try_ext_ext.rs +++ b/src/core/utils/future/try_ext_ext.rs @@ -4,8 +4,11 @@ // caller only ever caring about result status while discarding all contents. #![allow(clippy::wrong_self_convention)] +use std::marker::Unpin; + use futures::{ - future::{MapOkOrElse, UnwrapOrElse}, + future, + future::{MapOkOrElse, TrySelect, UnwrapOrElse}, TryFuture, TryFutureExt, }; @@ -46,6 +49,13 @@ where where Self: Sized; + fn try_until(self, f: F) -> TrySelect + where + Self: Sized, + F: FnOnce() -> B, + A: TryFuture + From + Send + Unpin, + B: TryFuture + Send + Unpin; + fn unwrap_or( self, default: Self::Ok, @@ -110,6 +120,17 @@ where self.map_ok_or(None, Some) } + #[inline] + fn try_until(self, f: F) -> TrySelect + where + Self: Sized, + F: FnOnce() -> B, + A: TryFuture + From + Send + Unpin, + B: TryFuture + Send + Unpin, + { + future::try_select(self.into(), f()) + } + #[inline] fn unwrap_or( self, diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index 61ae993d..0fee0a3a 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -8,6 +8,7 @@ mod ready; mod tools; mod try_broadband; mod try_ready; +mod try_tools; mod wideband; pub use band::{ @@ -23,4 +24,5 @@ pub use ready::ReadyExt; pub use tools::Tools; pub use try_broadband::TryBroadbandExt; pub use try_ready::TryReadyExt; +pub use try_tools::TryTools; pub use wideband::WidebandExt; diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs index d8da04ec..3261acb6 100644 --- a/src/core/utils/stream/try_ready.rs +++ b/src/core/utils/stream/try_ready.rs @@ -3,7 +3,7 @@ use futures::{ future::{ready, Ready}, - stream::{AndThen, TryFilterMap, TryFold, TryForEach, TryStream, TryStreamExt}, + stream::{AndThen, TryFilterMap, TryFold, TryForEach, TryStream, TryStreamExt, TryTakeWhile}, }; use crate::Result; @@ -56,6 +56,13 @@ where ) -> TryForEach>, impl FnMut(S::Ok) -> Ready>> where F: FnMut(S::Ok) -> Result<(), E>; + + fn ready_try_take_while( + self, + f: F, + ) -> TryTakeWhile>, impl FnMut(&S::Ok) -> Ready>> + where + F: Fn(&S::Ok) -> Result; } impl TryReadyExt for S @@ -122,4 +129,15 @@ where { self.try_for_each(move |t| ready(f(t))) } + + #[inline] + fn ready_try_take_while( + self, + f: F, + ) -> TryTakeWhile>, impl FnMut(&S::Ok) -> Ready>> + where + F: Fn(&S::Ok) -> Result, + { + self.try_take_while(move |t| ready(f(t))) + } } diff --git a/src/core/utils/stream/try_tools.rs b/src/core/utils/stream/try_tools.rs new file mode 100644 index 00000000..3ddce6ad --- /dev/null +++ b/src/core/utils/stream/try_tools.rs @@ -0,0 +1,44 @@ +//! TryStreamTools for futures::TryStream +#![allow(clippy::type_complexity)] + +use futures::{future, future::Ready, stream::TryTakeWhile, TryStream, TryStreamExt}; + +use crate::Result; + +/// TryStreamTools +pub trait TryTools +where + S: TryStream> + Send + ?Sized, + Self: TryStream + Send + Sized, +{ + fn try_take( + self, + n: usize, + ) -> TryTakeWhile< + Self, + Ready>, + impl FnMut(&S::Ok) -> Ready>, + >; +} + +impl TryTools for S +where + S: TryStream> + Send + ?Sized, + Self: TryStream + Send + Sized, +{ + #[inline] + fn try_take( + self, + mut n: usize, + ) -> TryTakeWhile< + Self, + Ready>, + impl FnMut(&S::Ok) -> Ready>, + > { + self.try_take_while(move |_| { + let res = future::ok(n > 0); + n = n.saturating_sub(1); + res + }) + } +} From 925061b92de1a60c6d7af9d28bdd158b7cc9901b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 4 Jan 2025 04:12:50 +0000 Subject: [PATCH 1321/2091] flatten timeline pdus iterations; increase concurrency Signed-off-by: Jason Volk --- src/admin/query/room_timeline.rs | 61 ++++++++ src/api/client/context.rs | 67 ++++----- src/api/client/membership.rs | 2 + src/api/client/message.rs | 6 +- src/api/client/room/initial_sync.rs | 15 +- src/api/client/sync/mod.rs | 20 ++- src/api/server/backfill.rs | 30 ++-- src/api/server/send.rs | 1 + .../rooms/event_handler/handle_prev_pdu.rs | 2 +- src/service/rooms/timeline/data.rs | 136 +++++++++--------- src/service/rooms/timeline/mod.rs | 135 ++++++----------- 11 files changed, 238 insertions(+), 237 deletions(-) create mode 100644 src/admin/query/room_timeline.rs diff --git a/src/admin/query/room_timeline.rs b/src/admin/query/room_timeline.rs new file mode 100644 index 00000000..3fe653e3 --- /dev/null +++ b/src/admin/query/room_timeline.rs @@ -0,0 +1,61 @@ +use clap::Subcommand; +use conduwuit::{utils::stream::TryTools, PduCount, Result}; +use futures::TryStreamExt; +use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomOrAliasId}; + +use crate::{admin_command, admin_command_dispatch}; + +#[admin_command_dispatch] +#[derive(Debug, Subcommand)] +/// Query tables from database +pub(crate) enum RoomTimelineCommand { + Pdus { + room_id: OwnedRoomOrAliasId, + + from: Option, + + #[arg(short, long)] + limit: Option, + }, + + Last { + room_id: OwnedRoomOrAliasId, + }, +} + +#[admin_command] +pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result { + let room_id = self.services.rooms.alias.resolve(&room_id).await?; + + let result = self + .services + .rooms + .timeline + .last_timeline_count(None, &room_id) + .await?; + + Ok(RoomMessageEventContent::notice_markdown(format!("{result:#?}"))) +} + +#[admin_command] +pub(super) async fn pdus( + &self, + room_id: OwnedRoomOrAliasId, + from: Option, + limit: Option, +) -> Result { + let room_id = self.services.rooms.alias.resolve(&room_id).await?; + + let from: Option = from.as_deref().map(str::parse).transpose()?; + + let result: Vec<_> = self + .services + .rooms + .timeline + .pdus_rev(None, &room_id, from) + .try_take(limit.unwrap_or(3)) + .try_collect() + .await?; + + Ok(RoomMessageEventContent::notice_markdown(format!("{result:#?}"))) +} diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 30ba170d..b957561c 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,14 +1,12 @@ -use std::iter::once; - use axum::extract::State; use conduwuit::{ at, err, ref_at, utils::{ future::TryExtExt, - stream::{BroadbandExt, ReadyExt, WidebandExt}, + stream::{BroadbandExt, ReadyExt, TryIgnore, WidebandExt}, IterStream, }, - Err, Result, + Err, PduEvent, Result, }; use futures::{join, try_join, FutureExt, StreamExt, TryFutureExt}; use ruma::{ @@ -59,13 +57,13 @@ pub(crate) async fn get_context_route( false }; - let base_token = services + let base_id = services .rooms .timeline - .get_pdu_count(&body.event_id) + .get_pdu_id(&body.event_id) .map_err(|_| err!(Request(NotFound("Event not found.")))); - let base_event = services + let base_pdu = services .rooms .timeline .get_pdu(&body.event_id) @@ -77,48 +75,44 @@ pub(crate) async fn get_context_route( .user_can_see_event(sender_user, &body.room_id, &body.event_id) .map(Ok); - let (base_token, base_event, visible) = try_join!(base_token, base_event, visible)?; + let (base_id, base_pdu, visible) = try_join!(base_id, base_pdu, visible)?; - if base_event.room_id != body.room_id || base_event.event_id != body.event_id { + if base_pdu.room_id != body.room_id || base_pdu.event_id != body.event_id { return Err!(Request(NotFound("Base event not found."))); } - if !visible - || ignored_filter(&services, (base_token, base_event.clone()), sender_user) - .await - .is_none() - { + if !visible { return Err!(Request(Forbidden("You don't have permission to view this event."))); } - let events_before = - services - .rooms - .timeline - .pdus_rev(Some(sender_user), room_id, Some(base_token)); + let base_count = base_id.pdu_count(); + + let base_event = ignored_filter(&services, (base_count, base_pdu), sender_user); + + let events_before = services + .rooms + .timeline + .pdus_rev(Some(sender_user), room_id, Some(base_count)) + .ignore_err() + .ready_filter_map(|item| event_filter(item, filter)) + .wide_filter_map(|item| ignored_filter(&services, item, sender_user)) + .wide_filter_map(|item| visibility_filter(&services, item, sender_user)) + .take(limit / 2) + .collect(); let events_after = services .rooms .timeline - .pdus(Some(sender_user), room_id, Some(base_token)); - - let (events_before, events_after) = try_join!(events_before, events_after)?; - - let events_before = events_before + .pdus(Some(sender_user), room_id, Some(base_count)) + .ignore_err() .ready_filter_map(|item| event_filter(item, filter)) .wide_filter_map(|item| ignored_filter(&services, item, sender_user)) .wide_filter_map(|item| visibility_filter(&services, item, sender_user)) .take(limit / 2) .collect(); - let events_after = events_after - .ready_filter_map(|item| event_filter(item, filter)) - .wide_filter_map(|item| ignored_filter(&services, item, sender_user)) - .wide_filter_map(|item| visibility_filter(&services, item, sender_user)) - .take(limit / 2) - .collect(); - - let (events_before, events_after): (Vec<_>, Vec<_>) = join!(events_before, events_after); + let (base_event, events_before, events_after): (_, Vec<_>, Vec<_>) = + join!(base_event, events_before, events_after); let state_at = events_after .last() @@ -134,7 +128,8 @@ pub(crate) async fn get_context_route( .map_err(|e| err!(Database("State not found: {e}"))) .await?; - let lazy = once(&(base_token, base_event.clone())) + let lazy = base_event + .iter() .chain(events_before.iter()) .chain(events_after.iter()) .stream() @@ -175,19 +170,19 @@ pub(crate) async fn get_context_route( .await; Ok(get_context::v3::Response { - event: Some(base_event.to_room_event()), + event: base_event.map(at!(1)).as_ref().map(PduEvent::to_room_event), start: events_before .last() .map(at!(0)) - .or(Some(base_token)) + .or(Some(base_count)) .as_ref() .map(ToString::to_string), end: events_after .last() .map(at!(0)) - .or(Some(base_token)) + .or(Some(base_count)) .as_ref() .map(ToString::to_string), diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 3eb52138..4046b493 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1314,6 +1314,7 @@ async fn join_room_by_id_helper_local( .rooms .event_handler .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true) + .boxed() .await?; } else { return Err(error); @@ -1491,6 +1492,7 @@ pub(crate) async fn invite_helper( .rooms .event_handler .handle_incoming_pdu(&origin, room_id, &event_id, value, true) + .boxed() .await? .ok_or_else(|| { err!(Request(InvalidParam("Could not accept incoming PDU as timeline event."))) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 58f4f916..ec9a14d5 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -5,7 +5,7 @@ use conduwuit::{ at, is_equal_to, utils::{ result::{FlatOk, LogErr}, - stream::{BroadbandExt, WidebandExt}, + stream::{BroadbandExt, TryIgnore, WidebandExt}, IterStream, ReadyExt, }, Event, PduCount, Result, @@ -107,14 +107,14 @@ pub(crate) async fn get_message_events_route( .rooms .timeline .pdus(Some(sender_user), room_id, Some(from)) - .await? + .ignore_err() .boxed(), | Direction::Backward => services .rooms .timeline .pdus_rev(Some(sender_user), room_id, Some(from)) - .await? + .ignore_err() .boxed(), }; diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index cc3c9420..301b6e8d 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -1,6 +1,10 @@ use axum::extract::State; -use conduwuit::{at, utils::BoolExt, Err, Result}; -use futures::StreamExt; +use conduwuit::{ + at, + utils::{stream::TryTools, BoolExt}, + Err, Result, +}; +use futures::TryStreamExt; use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response}; use crate::Ruma; @@ -27,10 +31,9 @@ pub(crate) async fn room_initial_sync_route( .rooms .timeline .pdus_rev(None, room_id, None) - .await? - .take(limit) - .collect() - .await; + .try_take(limit) + .try_collect() + .await?; let state: Vec<_> = services .rooms diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index b772fbf1..79e4b1ca 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -2,10 +2,10 @@ mod v3; mod v4; use conduwuit::{ - utils::stream::{BroadbandExt, ReadyExt}, + utils::stream::{BroadbandExt, ReadyExt, TryIgnore}, PduCount, }; -use futures::StreamExt; +use futures::{pin_mut, StreamExt}; use ruma::{RoomId, UserId}; pub(crate) use self::{v3::sync_events_route, v4::sync_events_v4_route}; @@ -29,23 +29,19 @@ async fn load_timeline( return Ok((Vec::new(), false)); } - let mut non_timeline_pdus = services + let non_timeline_pdus = services .rooms .timeline .pdus_rev(Some(sender_user), room_id, None) - .await? + .ignore_err() .ready_skip_while(|&(pducount, _)| pducount > next_batch.unwrap_or_else(PduCount::max)) .ready_take_while(|&(pducount, _)| pducount > roomsincecount); // Take the last events for the timeline - let timeline_pdus: Vec<_> = non_timeline_pdus - .by_ref() - .take(limit) - .collect::>() - .await - .into_iter() - .rev() - .collect(); + pin_mut!(non_timeline_pdus); + let timeline_pdus: Vec<_> = non_timeline_pdus.by_ref().take(limit).collect().await; + + let timeline_pdus: Vec<_> = timeline_pdus.into_iter().rev().collect(); // They /sync response doesn't always return all messages, so we say the output // is limited unless there are events in non_timeline_pdus diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index fac0e540..b44db67c 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -2,10 +2,10 @@ use std::cmp; use axum::extract::State; use conduwuit::{ - utils::{IterStream, ReadyExt}, + utils::{stream::TryTools, IterStream, ReadyExt}, PduCount, Result, }; -use futures::{FutureExt, StreamExt}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{api::federation::backfill::get_backfill, uint, MilliSecondsSinceUnixEpoch}; use super::AccessCheck; @@ -57,26 +57,30 @@ pub(crate) async fn get_backfill_route( .rooms .timeline .pdus_rev(None, &body.room_id, Some(from.saturating_add(1))) - .await? - .take(limit) - .filter_map(|(_, pdu)| async move { - services + .try_take(limit) + .try_filter_map(|(_, pdu)| async move { + Ok(services .rooms .state_accessor .server_can_see_event(body.origin(), &pdu.room_id, &pdu.event_id) .await - .then_some(pdu) + .then_some(pdu)) }) - .filter_map(|pdu| async move { - services + .try_filter_map(|pdu| async move { + Ok(services .rooms .timeline .get_pdu_json(&pdu.event_id) .await - .ok() + .ok()) }) - .then(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) - .collect() - .await, + .and_then(|pdu| { + services + .sending + .convert_to_outgoing_federation_event(pdu) + .map(Ok) + }) + .try_collect() + .await?, }) } diff --git a/src/api/server/send.rs b/src/api/server/send.rs index dbe0108f..c0c8a0c9 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -135,6 +135,7 @@ async fn handle_pdus( .rooms .event_handler .handle_incoming_pdu(origin, &room_id, &event_id, value, true) + .boxed() .await .map(|_| ()); diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index 0a5295dc..9bd4450e 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -26,7 +26,7 @@ pub(super) async fn handle_prev_pdu<'a>( (Arc, BTreeMap), >, create_event: &Arc, - first_pdu_in_room: &Arc, + first_pdu_in_room: &PduEvent, prev_id: &EventId, ) -> Result { // Check for disabled again because it might have changed diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 0be8aa52..457c1e8d 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -1,22 +1,15 @@ -use std::{ - borrow::Borrow, - collections::{hash_map, HashMap}, - sync::Arc, -}; +use std::{borrow::Borrow, sync::Arc}; use conduwuit::{ at, err, result::{LogErr, NotFound}, utils, - utils::{future::TryExtExt, stream::TryIgnore, ReadyExt}, + utils::stream::TryReadyExt, Err, PduCount, PduEvent, Result, }; use database::{Database, Deserialized, Json, KeyVal, Map}; -use futures::{future::select_ok, FutureExt, Stream, StreamExt}; -use ruma::{ - api::Direction, CanonicalJsonObject, EventId, OwnedRoomId, OwnedUserId, RoomId, UserId, -}; -use tokio::sync::Mutex; +use futures::{future::select_ok, pin_mut, FutureExt, Stream, TryFutureExt, TryStreamExt}; +use ruma::{api::Direction, CanonicalJsonObject, EventId, OwnedUserId, RoomId, UserId}; use super::{PduId, RawPduId}; use crate::{rooms, rooms::short::ShortRoomId, Dep}; @@ -27,7 +20,6 @@ pub(super) struct Data { pduid_pdu: Arc, userroomid_highlightcount: Arc, userroomid_notificationcount: Arc, - pub(super) lasttimelinecount_cache: LastTimelineCountCache, pub(super) db: Arc, services: Services, } @@ -37,7 +29,6 @@ struct Services { } pub type PdusIterItem = (PduCount, PduEvent); -type LastTimelineCountCache = Mutex>; impl Data { pub(super) fn new(args: &crate::Args<'_>) -> Self { @@ -48,7 +39,6 @@ impl Data { pduid_pdu: db["pduid_pdu"].clone(), userroomid_highlightcount: db["userroomid_highlightcount"].clone(), userroomid_notificationcount: db["userroomid_notificationcount"].clone(), - lasttimelinecount_cache: Mutex::new(HashMap::new()), db: args.db.clone(), services: Services { short: args.depend::("rooms::short"), @@ -56,27 +46,39 @@ impl Data { } } + #[inline] pub(super) async fn last_timeline_count( &self, sender_user: Option<&UserId>, room_id: &RoomId, ) -> Result { - match self - .lasttimelinecount_cache - .lock() - .await - .entry(room_id.into()) - { - | hash_map::Entry::Occupied(o) => Ok(*o.get()), - | hash_map::Entry::Vacant(v) => Ok(self - .pdus_rev(sender_user, room_id, PduCount::max()) - .await? - .next() - .await - .map(at!(0)) - .filter(|&count| matches!(count, PduCount::Normal(_))) - .map_or_else(PduCount::max, |count| *v.insert(count))), - } + let pdus_rev = self.pdus_rev(sender_user, room_id, PduCount::max()); + + pin_mut!(pdus_rev); + let last_count = pdus_rev + .try_next() + .await? + .map(at!(0)) + .filter(|&count| matches!(count, PduCount::Normal(_))) + .unwrap_or_else(PduCount::max); + + Ok(last_count) + } + + #[inline] + pub(super) async fn latest_pdu_in_room( + &self, + sender_user: Option<&UserId>, + room_id: &RoomId, + ) -> Result { + let pdus_rev = self.pdus_rev(sender_user, room_id, PduCount::max()); + + pin_mut!(pdus_rev); + pdus_rev + .try_next() + .await? + .map(at!(1)) + .ok_or_else(|| err!(Request(NotFound("no PDU's found in room")))) } /// Returns the `count` of this pdu's id. @@ -129,7 +131,7 @@ impl Data { pub(super) async fn non_outlier_pdu_exists(&self, event_id: &EventId) -> Result { let pduid = self.get_pdu_id(event_id).await?; - self.pduid_pdu.get(&pduid).await.map(|_| ()) + self.pduid_pdu.exists(&pduid).await } /// Returns the pdu. @@ -148,17 +150,17 @@ impl Data { /// Like get_non_outlier_pdu(), but without the expense of fetching and /// parsing the PduEvent + #[inline] pub(super) async fn outlier_pdu_exists(&self, event_id: &EventId) -> Result { - self.eventid_outlierpdu.get(event_id).await.map(|_| ()) + self.eventid_outlierpdu.exists(event_id).await } /// Like get_pdu(), but without the expense of fetching and parsing the data - pub(super) async fn pdu_exists(&self, event_id: &EventId) -> bool { - let non_outlier = self.non_outlier_pdu_exists(event_id).is_ok(); - let outlier = self.outlier_pdu_exists(event_id).is_ok(); + pub(super) async fn pdu_exists(&self, event_id: &EventId) -> Result { + let non_outlier = self.non_outlier_pdu_exists(event_id).boxed(); + let outlier = self.outlier_pdu_exists(event_id).boxed(); - //TODO: parallelize - non_outlier.await || outlier.await + select_ok([non_outlier, outlier]).await.map(at!(0)) } /// Returns the pdu. @@ -186,11 +188,6 @@ impl Data { debug_assert!(matches!(count, PduCount::Normal(_)), "PduCount not Normal"); self.pduid_pdu.raw_put(pdu_id, Json(json)); - self.lasttimelinecount_cache - .lock() - .await - .insert(pdu.room_id.clone(), count); - self.eventid_pduid.insert(pdu.event_id.as_bytes(), pdu_id); self.eventid_outlierpdu.remove(pdu.event_id.as_bytes()); } @@ -225,49 +222,44 @@ impl Data { /// Returns an iterator over all events and their tokens in a room that /// happened before the event with id `until` in reverse-chronological /// order. - pub(super) async fn pdus_rev<'a>( + pub(super) fn pdus_rev<'a>( &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, until: PduCount, - ) -> Result + Send + 'a> { - let current = self - .count_to_id(room_id, until, Direction::Backward) - .await?; - let prefix = current.shortroomid(); - let stream = self - .pduid_pdu - .rev_raw_stream_from(¤t) - .ignore_err() - .ready_take_while(move |(key, _)| key.starts_with(&prefix)) - .map(move |item| Self::each_pdu(item, user_id)); - - Ok(stream) + ) -> impl Stream> + Send + 'a { + self.count_to_id(room_id, until, Direction::Backward) + .map_ok(move |current| { + let prefix = current.shortroomid(); + self.pduid_pdu + .rev_raw_stream_from(¤t) + .ready_try_take_while(move |(key, _)| Ok(key.starts_with(&prefix))) + .ready_and_then(move |item| Self::each_pdu(item, user_id)) + }) + .try_flatten_stream() } - pub(super) async fn pdus<'a>( + pub(super) fn pdus<'a>( &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, from: PduCount, - ) -> Result + Send + Unpin + 'a> { - let current = self.count_to_id(room_id, from, Direction::Forward).await?; - let prefix = current.shortroomid(); - let stream = self - .pduid_pdu - .raw_stream_from(¤t) - .ignore_err() - .ready_take_while(move |(key, _)| key.starts_with(&prefix)) - .map(move |item| Self::each_pdu(item, user_id)); - - Ok(stream) + ) -> impl Stream> + Send + 'a { + self.count_to_id(room_id, from, Direction::Forward) + .map_ok(move |current| { + let prefix = current.shortroomid(); + self.pduid_pdu + .raw_stream_from(¤t) + .ready_try_take_while(move |(key, _)| Ok(key.starts_with(&prefix))) + .ready_and_then(move |item| Self::each_pdu(item, user_id)) + }) + .try_flatten_stream() } - fn each_pdu((pdu_id, pdu): KeyVal<'_>, user_id: Option<&UserId>) -> PdusIterItem { + fn each_pdu((pdu_id, pdu): KeyVal<'_>, user_id: Option<&UserId>) -> Result { let pdu_id: RawPduId = pdu_id.into(); - let mut pdu = serde_json::from_slice::(pdu) - .expect("PduEvent in pduid_pdu database column is invalid JSON"); + let mut pdu = serde_json::from_slice::(pdu)?; if Some(pdu.sender.borrow()) != user_id { pdu.remove_transaction_id().log_err().ok(); @@ -275,7 +267,7 @@ impl Data { pdu.add_age().log_err().ok(); - (pdu_id.pdu_count(), pdu) + Ok((pdu_id.pdu_count(), pdu)) } pub(super) fn increment_notification_counts( diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 2a272c38..fe7f885a 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -9,14 +9,16 @@ use std::{ }; use conduwuit::{ - debug, debug_warn, err, error, implement, info, + at, debug, debug_warn, err, error, implement, info, pdu::{gen_event_id, EventHash, PduBuilder, PduCount, PduEvent}, - utils::{self, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt}, + utils::{ + self, future::TryExtExt, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt, + }, validated, warn, Err, Error, Result, Server, }; pub use conduwuit::{PduId, RawPduId}; use futures::{ - future, future::ready, Future, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, + future, future::ready, pin_mut, Future, FutureExt, Stream, StreamExt, TryStreamExt, }; use ruma::{ api::federation, @@ -34,7 +36,7 @@ use ruma::{ }, push::{Action, Ruleset, Tweak}, state_res::{self, Event, RoomVersion}, - uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, + uint, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use serde::Deserialize; @@ -139,53 +141,34 @@ impl crate::Service for Service { } fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { - /* - let lasttimelinecount_cache = self - .db - .lasttimelinecount_cache - .lock() - .expect("locked") - .len(); - writeln!(out, "lasttimelinecount_cache: {lasttimelinecount_cache}")?; - */ - let mutex_insert = self.mutex_insert.len(); writeln!(out, "insert_mutex: {mutex_insert}")?; Ok(()) } - fn clear_cache(&self) { - /* - self.db - .lasttimelinecount_cache - .lock() - .expect("locked") - .clear(); - */ - } - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } impl Service { #[tracing::instrument(skip(self), level = "debug")] - pub async fn first_pdu_in_room(&self, room_id: &RoomId) -> Result> { - self.all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id) - .next() - .await - .map(|(_, p)| Arc::new(p)) + pub async fn first_pdu_in_room(&self, room_id: &RoomId) -> Result { + self.first_item_in_room(room_id).await.map(at!(1)) + } + + #[tracing::instrument(skip(self), level = "debug")] + pub async fn first_item_in_room(&self, room_id: &RoomId) -> Result<(PduCount, PduEvent)> { + let pdus = self.pdus(None, room_id, None); + + pin_mut!(pdus); + pdus.try_next() + .await? .ok_or_else(|| err!(Request(NotFound("No PDU found in room")))) } #[tracing::instrument(skip(self), level = "debug")] - pub async fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result> { - self.pdus_rev(None, room_id, None) - .await? - .next() - .await - .map(|(_, p)| Arc::new(p)) - .ok_or_else(|| err!(Request(NotFound("No PDU found in room")))) + pub async fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result { + self.db.latest_pdu_in_room(None, room_id).await } #[tracing::instrument(skip(self), level = "debug")] @@ -202,29 +185,6 @@ impl Service { self.db.get_pdu_count(event_id).await } - // TODO Is this the same as the function above? - /* - #[tracing::instrument(skip(self))] - pub fn latest_pdu_count(&self, room_id: &RoomId) -> Result { - let prefix = self - .get_shortroomid(room_id)? - .expect("room exists") - .to_be_bytes() - .to_vec(); - - let mut last_possible_key = prefix.clone(); - last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes()); - - self.pduid_pdu - .iter_from(&last_possible_key, true) - .take_while(move |(k, _)| k.starts_with(&prefix)) - .next() - .map(|b| self.pdu_count(&b.0)) - .transpose() - .map(|op| op.unwrap_or_default()) - } - */ - /// Returns the json of a pdu. pub async fn get_pdu_json(&self, event_id: &EventId) -> Result { self.db.get_pdu_json(event_id).await @@ -260,16 +220,6 @@ impl Service { self.db.get_pdu(event_id).await } - /// Checks if pdu exists - /// - /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. - pub fn pdu_exists<'a>( - &'a self, - event_id: &'a EventId, - ) -> impl Future + Send + 'a { - self.db.pdu_exists(event_id) - } - /// Returns the pdu. /// /// This does __NOT__ check the outliers `Tree`. @@ -282,6 +232,16 @@ impl Service { self.db.get_pdu_json_from_id(pdu_id).await } + /// Checks if pdu exists + /// + /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. + pub fn pdu_exists<'a>( + &'a self, + event_id: &'a EventId, + ) -> impl Future + Send + 'a { + self.db.pdu_exists(event_id).is_ok() + } + /// Removes a pdu and creates a new one with the same id. #[tracing::instrument(skip(self), level = "debug")] pub async fn replace_pdu( @@ -1027,38 +987,32 @@ impl Service { &'a self, user_id: &'a UserId, room_id: &'a RoomId, - ) -> impl Stream + Send + Unpin + 'a { - self.pdus(Some(user_id), room_id, None) - .map_ok(|stream| stream.map(Ok)) - .try_flatten_stream() - .ignore_err() - .boxed() + ) -> impl Stream + Send + 'a { + self.pdus(Some(user_id), room_id, None).ignore_err() } /// Reverse iteration starting at from. #[tracing::instrument(skip(self), level = "debug")] - pub async fn pdus_rev<'a>( + pub fn pdus_rev<'a>( &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, until: Option, - ) -> Result + Send + 'a> { + ) -> impl Stream> + Send + 'a { self.db .pdus_rev(user_id, room_id, until.unwrap_or_else(PduCount::max)) - .await } /// Forward iteration starting at from. #[tracing::instrument(skip(self), level = "debug")] - pub async fn pdus<'a>( + pub fn pdus<'a>( &'a self, user_id: Option<&'a UserId>, room_id: &'a RoomId, from: Option, - ) -> Result + Send + 'a> { + ) -> impl Stream> + Send + 'a { self.db .pdus(user_id, room_id, from.unwrap_or_else(PduCount::min)) - .await } /// Replace a PDU with the redacted form. @@ -1117,8 +1071,7 @@ impl Service { } let first_pdu = self - .all_pdus(user_id!("@doesntmatter:conduit.rs"), room_id) - .next() + .first_item_in_room(room_id) .await .expect("Room is not empty"); @@ -1232,20 +1185,14 @@ impl Service { self.services .event_handler .handle_incoming_pdu(origin, &room_id, &event_id, value, false) + .boxed() .await?; - let value = self - .get_pdu_json(&event_id) - .await - .expect("We just created it"); - let pdu = self.get_pdu(&event_id).await.expect("We just created it"); + let value = self.get_pdu_json(&event_id).await?; - let shortroomid = self - .services - .short - .get_shortroomid(&room_id) - .await - .expect("room exists"); + let pdu = self.get_pdu(&event_id).await?; + + let shortroomid = self.services.short.get_shortroomid(&room_id).await?; let insert_lock = self.mutex_insert.lock(&room_id).await; From d36167ab64560221ac08e848e479b285d8c4dc48 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 5 Jan 2025 23:33:27 +0000 Subject: [PATCH 1322/2091] partially revert 9a9c071e82; use std threads for db pool. Signed-off-by: Jason Volk --- src/database/engine.rs | 26 +++++----- src/database/pool.rs | 107 +++++++++++++++++++++++------------------ src/main/runtime.rs | 2 +- src/router/run.rs | 12 ++--- 4 files changed, 80 insertions(+), 67 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index faf5b131..670817b5 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -31,11 +31,11 @@ pub struct Engine { opts: Options, env: Env, cfs: Mutex>, + pub(crate) pool: Arc, pub(crate) db: Db, corks: AtomicU32, pub(super) read_only: bool, pub(super) secondary: bool, - pub(crate) pool: Arc, } pub(crate) type Db = DBWithThreadMode; @@ -44,6 +44,8 @@ impl Engine { #[tracing::instrument(skip_all)] pub(crate) async fn open(server: &Arc) -> Result> { let config = &server.config; + let path = &config.database_path; + let cache_capacity_bytes = config.db_cache_capacity_mb * 1024.0 * 1024.0; #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] @@ -64,30 +66,32 @@ impl Engine { col_cache.get("primary").expect("primary cache exists"), )?; - let load_time = std::time::Instant::now(); - if config.rocksdb_repair { - repair(&db_opts, &config.database_path)?; - } - debug!("Listing column families in database"); let cfs = Db::list_cf(&db_opts, &config.database_path) .unwrap_or_default() .into_iter() .collect::>(); - debug!("Opening {} column family descriptors in database", cfs.len()); + debug!("Configuring {} column families found in database", cfs.len()); let cfopts = cfs .iter() .map(|name| cf_options(config, name, db_opts.clone(), &mut col_cache)) .collect::>>()?; + debug!("Opening {} column family descriptors in database", cfs.len()); let cfds = cfs .iter() .zip(cfopts.into_iter()) .map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts)) .collect::>(); - let path = &config.database_path; + debug!("Starting frontend request pool"); + let pool = Pool::new(server)?; + + let load_time = std::time::Instant::now(); + if config.rocksdb_repair { + repair(&db_opts, &config.database_path)?; + } debug!("Opening database..."); let res = if config.rocksdb_read_only { @@ -113,11 +117,11 @@ impl Engine { opts: db_opts, env: db_env, cfs: Mutex::new(cfs), - db, corks: AtomicU32::new(0), read_only: config.rocksdb_read_only, secondary: config.rocksdb_secondary, - pool: Pool::new(server).await?, + pool, + db, })) } @@ -146,8 +150,6 @@ impl Engine { .expect("column was created and exists") } - pub async fn shutdown_pool(&self) { self.pool.shutdown().await; } - pub fn flush(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, false)) } pub fn sync(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, true)) } diff --git a/src/database/pool.rs b/src/database/pool.rs index 0295e0e9..1560c8b0 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -6,21 +6,22 @@ use std::{ atomic::{AtomicUsize, Ordering}, Arc, Mutex, }, + thread, + thread::JoinHandle, }; use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ - debug, debug_warn, defer, err, implement, + debug, debug_warn, defer, err, error, implement, result::DebugInspect, trace, utils::sys::compute::{get_affinity, nth_core_available, set_affinity}, - Result, Server, + Error, Result, Server, }; use futures::{channel::oneshot, TryFutureExt}; use oneshot::Sender as ResultSender; use rocksdb::Direction; use smallvec::SmallVec; -use tokio::task::JoinSet; use self::configure::configure; use crate::{keyval::KeyBuf, stream, Handle, Map}; @@ -29,9 +30,9 @@ use crate::{keyval::KeyBuf, stream, Handle, Map}; /// requests which are not cached. These thread-blocking requests are offloaded /// from the tokio async workers and executed on this threadpool. pub(crate) struct Pool { - server: Arc, + _server: Arc, queues: Vec>, - workers: Mutex>, + workers: Mutex>>, topology: Vec, busy: AtomicUsize, queued_max: AtomicUsize, @@ -69,39 +70,42 @@ const WORKER_LIMIT: (usize, usize) = (1, 1024); const QUEUE_LIMIT: (usize, usize) = (1, 2048); const BATCH_INLINE: usize = 1; +const WORKER_STACK_SIZE: usize = 1_048_576; +const WORKER_NAME: &str = "conduwuit:db"; + #[implement(Pool)] -pub(crate) async fn new(server: &Arc) -> Result> { +pub(crate) fn new(server: &Arc) -> Result> { const CHAN_SCHED: (QueueStrategy, QueueStrategy) = (QueueStrategy::Fifo, QueueStrategy::Lifo); let (total_workers, queue_sizes, topology) = configure(server); - let (senders, receivers) = queue_sizes + let (senders, receivers): (Vec<_>, Vec<_>) = queue_sizes .into_iter() .map(|cap| async_channel::bounded_with_queue_strategy(cap, CHAN_SCHED)) .unzip(); let pool = Arc::new(Self { - server: server.clone(), - + _server: server.clone(), queues: senders, - - workers: JoinSet::new().into(), - + workers: Vec::new().into(), topology, - busy: AtomicUsize::default(), - queued_max: AtomicUsize::default(), }); - pool.spawn_until(receivers, total_workers).await?; + pool.spawn_until(&receivers, total_workers)?; Ok(pool) } impl Drop for Pool { fn drop(&mut self) { - debug_assert!(self.queues.iter().all(Sender::is_empty), "channel must be empty on drop"); + self.close(); + + debug_assert!( + self.queues.iter().all(Sender::is_empty), + "channel must should not have requests queued on drop" + ); debug_assert!( self.queues.iter().all(Sender::is_closed), "channel should be closed on drop" @@ -110,17 +114,10 @@ impl Drop for Pool { } #[implement(Pool)] -pub(crate) async fn shutdown(self: &Arc) { - self.close(); - - let workers = take(&mut *self.workers.lock().expect("locked")); - debug!(workers = workers.len(), "Waiting for workers to join..."); - - workers.join_all().await; -} - -#[implement(Pool)] +#[tracing::instrument(skip_all)] pub(crate) fn close(&self) { + let workers = take(&mut *self.workers.lock().expect("locked")); + let senders = self.queues.iter().map(Sender::sender_count).sum::(); let receivers = self @@ -129,27 +126,40 @@ pub(crate) fn close(&self) { .map(Sender::receiver_count) .sum::(); - debug!( - queues = self.queues.len(), - workers = self.workers.lock().expect("locked").len(), - ?senders, - ?receivers, - "Closing pool..." - ); - for queue in &self.queues { queue.close(); } - self.workers.lock().expect("locked").abort_all(); - std::thread::yield_now(); + if workers.is_empty() { + return; + } + + debug!( + senders, + receivers, + queues = self.queues.len(), + workers = workers.len(), + "Closing pool. Waiting for workers to join..." + ); + + workers + .into_iter() + .map(JoinHandle::join) + .map(|result| result.map_err(Error::from_panic)) + .enumerate() + .for_each(|(id, result)| { + match result { + | Ok(()) => trace!(?id, "worker joined"), + | Err(error) => error!(?id, "worker joined with error: {error}"), + }; + }); } #[implement(Pool)] -async fn spawn_until(self: &Arc, recv: Vec>, count: usize) -> Result { +fn spawn_until(self: &Arc, recv: &[Receiver], count: usize) -> Result { let mut workers = self.workers.lock().expect("locked"); while workers.len() < count { - self.spawn_one(&mut workers, &recv)?; + self.clone().spawn_one(&mut workers, recv)?; } Ok(()) @@ -162,23 +172,24 @@ async fn spawn_until(self: &Arc, recv: Vec>, count: usize) - skip_all, fields(id = %workers.len()) )] -fn spawn_one(self: &Arc, workers: &mut JoinSet<()>, recv: &[Receiver]) -> Result { +fn spawn_one( + self: Arc, + workers: &mut Vec>, + recv: &[Receiver], +) -> Result { debug_assert!(!self.queues.is_empty(), "Must have at least one queue"); debug_assert!(!recv.is_empty(), "Must have at least one receiver"); let id = workers.len(); let group = id.overflowing_rem(self.queues.len()).0; let recv = recv[group].clone(); - let self_ = self.clone(); - #[cfg(not(tokio_unstable))] - let _abort = workers.spawn_blocking_on(move || self_.worker(id, recv), self.server.runtime()); + let handle = thread::Builder::new() + .name(WORKER_NAME.into()) + .stack_size(WORKER_STACK_SIZE) + .spawn(move || self.worker(id, recv))?; - #[cfg(tokio_unstable)] - let _abort = workers - .build_task() - .name("conduwuit:dbpool") - .spawn_blocking_on(move || self_.worker(id, recv), self.server.runtime()); + workers.push(handle); Ok(()) } @@ -258,7 +269,7 @@ async fn execute(&self, queue: &Sender, cmd: Cmd) -> Result { level = "debug", skip(self, recv), fields( - tid = ?std::thread::current().id(), + tid = ?thread::current().id(), ), )] fn worker(self: Arc, id: usize, recv: Receiver) { diff --git a/src/main/runtime.rs b/src/main/runtime.rs index b9dfc866..3039ef1b 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -19,7 +19,7 @@ use crate::clap::Args; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; const WORKER_KEEPALIVE: u64 = 36; -const MAX_BLOCKING_THREADS: usize = 2048; +const MAX_BLOCKING_THREADS: usize = 1024; static WORKER_AFFINITY: OnceLock = OnceLock::new(); diff --git a/src/router/run.rs b/src/router/run.rs index 248f7052..1b4d7437 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -3,7 +3,7 @@ extern crate conduwuit_core as conduwuit; extern crate conduwuit_service as service; use std::{ - sync::{atomic::Ordering, Arc}, + sync::{atomic::Ordering, Arc, Weak}, time::Duration, }; @@ -92,11 +92,11 @@ pub(crate) async fn stop(services: Arc) -> Result<()> { ); } - // The db threadpool requires async join if we use tokio/spawn_blocking to - // manage the threads. Without async-drop we have to wait here; for symmetry - // with Services construction it can't be done in services.stop(). - if let Some(db) = db.upgrade() { - db.db.shutdown_pool().await; + if Weak::strong_count(&db) > 0 { + debug_error!( + "{} dangling references to Database after shutdown", + Weak::strong_count(&db) + ); } #[cfg(feature = "systemd")] From 94c8683836764e7adfa8fe978692ac7154b0d8ec Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 6 Jan 2025 01:30:22 +0000 Subject: [PATCH 1323/2091] improve db pool topology configuration Signed-off-by: Jason Volk --- conduwuit-example.toml | 2 +- src/core/config/mod.rs | 2 +- src/database/pool.rs | 7 ++-- src/database/pool/configure.rs | 65 +++++++++++++++++----------------- 4 files changed, 39 insertions(+), 37 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 526e9fe2..e2ed5daa 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1452,7 +1452,7 @@ # responsiveness for many users at the cost of throughput for each. # # Setting this value to 0.0 causes the stream width to be fixed at the -# value of stream_width_default. The default is 1.0 to match the +# value of stream_width_default. The default scale is 1.0 to match the # capabilities detected for the system. # #stream_width_scale = 1.0 diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index b1ede844..bf6a4ba6 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1646,7 +1646,7 @@ pub struct Config { /// responsiveness for many users at the cost of throughput for each. /// /// Setting this value to 0.0 causes the stream width to be fixed at the - /// value of stream_width_default. The default is 1.0 to match the + /// value of stream_width_default. The default scale is 1.0 to match the /// capabilities detected for the system. /// /// default: 1.0 diff --git a/src/database/pool.rs b/src/database/pool.rs index 1560c8b0..11871ff6 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -30,7 +30,7 @@ use crate::{keyval::KeyBuf, stream, Handle, Map}; /// requests which are not cached. These thread-blocking requests are offloaded /// from the tokio async workers and executed on this threadpool. pub(crate) struct Pool { - _server: Arc, + server: Arc, queues: Vec>, workers: Mutex>>, topology: Vec, @@ -67,7 +67,7 @@ pub(crate) type BatchResult<'a> = SmallVec<[ResultHandle<'a>; BATCH_INLINE]>; pub(crate) type ResultHandle<'a> = Result>; const WORKER_LIMIT: (usize, usize) = (1, 1024); -const QUEUE_LIMIT: (usize, usize) = (1, 2048); +const QUEUE_LIMIT: (usize, usize) = (1, 4096); const BATCH_INLINE: usize = 1; const WORKER_STACK_SIZE: usize = 1_048_576; @@ -85,7 +85,7 @@ pub(crate) fn new(server: &Arc) -> Result> { .unzip(); let pool = Arc::new(Self { - _server: server.clone(), + server: server.clone(), queues: senders, workers: Vec::new().into(), topology, @@ -288,6 +288,7 @@ fn worker_init(&self, id: usize) { .iter() .enumerate() .filter(|_| self.queues.len() > 1) + .filter(|_| self.server.config.db_pool_affinity) .filter_map(|(core_id, &queue_id)| (group == queue_id).then_some(core_id)) .filter_map(nth_core_available); diff --git a/src/database/pool/configure.rs b/src/database/pool/configure.rs index 6cac58e7..ff42ef51 100644 --- a/src/database/pool/configure.rs +++ b/src/database/pool/configure.rs @@ -1,14 +1,13 @@ use std::{path::PathBuf, sync::Arc}; use conduwuit::{ - debug, debug_info, expected, + debug, debug_info, expected, is_equal_to, utils::{ math::usize_from_f64, result::LogDebugErr, stream, stream::{AMPLIFICATION_LIMIT, WIDTH_LIMIT}, sys::{compute::is_core_available, storage}, - BoolExt, }, Server, }; @@ -19,39 +18,32 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) let config = &server.config; // This finds the block device and gathers all the properties we need. - let (device_name, device_prop) = config - .db_pool_affinity - .and_then(|| { - let path: PathBuf = config.database_path.clone(); - let name = storage::name_from_path(&path).log_debug_err().ok(); - let prop = storage::parallelism(&path); - name.map(|name| (name, prop)) - }) - .unzip(); + let path: PathBuf = config.database_path.clone(); + let device_name = storage::name_from_path(&path).log_debug_err().ok(); + let device_prop = storage::parallelism(&path); // The default worker count is masked-on if we didn't find better information. - let default_worker_count = device_prop - .as_ref() - .is_none_or(|prop| prop.mq.is_empty()) - .then_some(config.db_pool_workers); + let default_worker_count = device_prop.mq.is_empty().then_some(config.db_pool_workers); // Determine the worker groupings. Each indice represents a hardware queue and // contains the number of workers which will service it. let worker_counts: Vec<_> = device_prop + .mq .iter() - .map(|dev| &dev.mq) - .flat_map(|mq| mq.iter()) .filter(|mq| mq.cpu_list.iter().copied().any(is_core_available)) .map(|mq| { - mq.nr_tags.unwrap_or_default().min( - config.db_pool_workers_limit.saturating_mul( - mq.cpu_list - .iter() - .filter(|&&id| is_core_available(id)) - .count() - .max(1), - ), - ) + let shares = mq + .cpu_list + .iter() + .filter(|&&id| is_core_available(id)) + .count() + .max(1); + + let limit = config.db_pool_workers_limit.saturating_mul(shares); + + let limit = device_prop.nr_requests.map_or(limit, |nr| nr.min(limit)); + + mq.nr_tags.unwrap_or(WORKER_LIMIT.0).min(limit) }) .chain(default_worker_count) .collect(); @@ -72,9 +64,8 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) // going on because cpu's which are not available to the process are filtered // out, similar to the worker_counts. let topology = device_prop + .mq .iter() - .map(|dev| &dev.mq) - .flat_map(|mq| mq.iter()) .fold(vec![0; 128], |mut topology, mq| { mq.cpu_list .iter() @@ -89,9 +80,12 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) // Regardless of the capacity of all queues we establish some limit on the total // number of workers; this is hopefully hinted by nr_requests. let max_workers = device_prop - .as_ref() - .and_then(|prop| prop.nr_requests) - .unwrap_or(WORKER_LIMIT.1); + .mq + .iter() + .filter_map(|mq| mq.nr_tags) + .chain(default_worker_count) + .fold(0_usize, usize::saturating_add) + .clamp(WORKER_LIMIT.0, WORKER_LIMIT.1); // Determine the final worker count which we'll be spawning. let total_workers = worker_counts @@ -102,7 +96,7 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) // After computing all of the above we can update the global automatic stream // width, hopefully with a better value tailored to this system. if config.stream_width_scale > 0.0 { - let num_queues = queue_sizes.len(); + let num_queues = queue_sizes.len().max(1); update_stream_width(server, num_queues, total_workers); } @@ -117,6 +111,13 @@ pub(super) fn configure(server: &Arc) -> (usize, Vec, Vec) "Frontend topology", ); + assert!(total_workers > 0, "some workers expected"); + assert!(!queue_sizes.is_empty(), "some queues expected"); + assert!( + !queue_sizes.iter().copied().any(is_equal_to!(0)), + "positive queue sizes expected" + ); + (total_workers, queue_sizes, topology) } From 44e6b1af3c24f4fdf664a138db3a9ecde9c4026c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 8 Jan 2025 20:43:02 +0000 Subject: [PATCH 1324/2091] fixes for tests to be run in release-mode Signed-off-by: Jason Volk --- src/core/pdu/mod.rs | 1 + src/core/pdu/tests.rs | 2 -- src/core/utils/mod.rs | 1 + src/core/utils/tests.rs | 1 - src/database/mod.rs | 1 + src/database/tests.rs | 7 +++---- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index ba1d2ce1..1a8f6a70 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -9,6 +9,7 @@ mod raw_id; mod redact; mod relation; mod strip; +#[cfg(test)] mod tests; mod unsigned; diff --git a/src/core/pdu/tests.rs b/src/core/pdu/tests.rs index ae3b1dd6..ed9b7caa 100644 --- a/src/core/pdu/tests.rs +++ b/src/core/pdu/tests.rs @@ -1,5 +1,3 @@ -#![cfg(test)] - use super::Count; #[test] diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 16072765..631b2820 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -16,6 +16,7 @@ pub mod set; pub mod stream; pub mod string; pub mod sys; +#[cfg(test)] mod tests; pub mod time; diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index 5c870730..1bcb92b8 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -1,4 +1,3 @@ -#![cfg(test)] #![allow(clippy::disallowed_methods)] use crate::utils; diff --git a/src/database/mod.rs b/src/database/mod.rs index 183cba8d..bdb7d3ea 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -11,6 +11,7 @@ mod opts; mod pool; mod ser; mod stream; +#[cfg(test)] mod tests; pub(crate) mod util; mod watchers; diff --git a/src/database/tests.rs b/src/database/tests.rs index 3d41a544..0c9fb41a 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -1,4 +1,3 @@ -#![cfg(test)] #![allow(clippy::needless_borrows_for_generic_args)] use std::fmt::Debug; @@ -14,7 +13,7 @@ use crate::{ }; #[test] -#[should_panic(expected = "serializing string at the top-level")] +#[cfg_attr(debug_assertions, should_panic(expected = "serializing string at the top-level"))] fn ser_str() { let user_id: &UserId = "@user:example.com".try_into().unwrap(); let s = serialize_to_vec(&user_id).expect("failed to serialize user_id"); @@ -139,7 +138,7 @@ fn ser_json_macro() { } #[test] -#[should_panic(expected = "serializing string at the top-level")] +#[cfg_attr(debug_assertions, should_panic(expected = "serializing string at the top-level"))] fn ser_json_raw() { use conduwuit::ruma::api::client::filter::FilterDefinition; @@ -156,7 +155,7 @@ fn ser_json_raw() { } #[test] -#[should_panic(expected = "you can skip serialization instead")] +#[cfg_attr(debug_assertions, should_panic(expected = "you can skip serialization instead"))] fn ser_json_raw_json() { use conduwuit::ruma::api::client::filter::FilterDefinition; From 5dae086197cd751e2a386cc4e8701623ef12bc8a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 8 Jan 2025 23:59:19 +0000 Subject: [PATCH 1325/2091] exclude config item from doctest Signed-off-by: Jason Volk --- src/core/config/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index bf6a4ba6..56580fda 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -621,6 +621,7 @@ pub struct Config { #[serde(default = "default_tracing_flame_output_path")] pub tracing_flame_output_path: String, + #[cfg(not(doctest))] /// Examples: /// /// - No proxy (default): From 0238f27605e01847219a5cf1061aa4b597636c61 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 10 Jan 2025 06:59:12 +0000 Subject: [PATCH 1326/2091] prevent example-config generating in test builds Signed-off-by: Jason Volk --- src/macros/config.rs | 4 ++-- src/macros/utils.rs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/macros/config.rs b/src/macros/config.rs index eb269e1e..0fb79728 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -9,7 +9,7 @@ use syn::{ }; use crate::{ - utils::{get_simple_settings, is_cargo_build}, + utils::{get_simple_settings, is_cargo_build, is_cargo_test}, Result, }; @@ -17,7 +17,7 @@ const UNDOCUMENTED: &str = "# This item is undocumented. Please contribute docum #[allow(clippy::needless_pass_by_value)] pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result { - if is_cargo_build() { + if is_cargo_build() && !is_cargo_test() { generate_example(&input, args)?; } diff --git a/src/macros/utils.rs b/src/macros/utils.rs index 9f799f7b..e33ee8b4 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -32,6 +32,8 @@ pub(crate) fn is_cargo_build() -> bool { .is_some() } +pub(crate) fn is_cargo_test() -> bool { std::env::args().any(|flag| flag == "--test") } + pub(crate) fn get_named_generics(args: &[Meta], name: &str) -> Result { const DEFAULT: &str = "<>"; From cc1889d135763edc44b9eb28c991bbeee8badbc0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 18 Dec 2024 21:29:30 +0000 Subject: [PATCH 1327/2091] Add default-enabled feature-gates for url_preview and media_thumbnail Signed-off-by: Jason Volk --- Cargo.lock | 1 - src/core/Cargo.toml | 1 - src/core/error/mod.rs | 2 - src/main/Cargo.toml | 8 ++ src/service/Cargo.toml | 13 ++- src/service/media/data.rs | 37 +++----- src/service/media/preview.rs | 93 ++++++++++++-------- src/service/media/thumbnail.rs | 150 ++++++++++++++++++++------------- 8 files changed, 182 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f900a11..d25197e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -723,7 +723,6 @@ dependencies = [ "hardened_malloc-rs", "http", "http-body-util", - "image", "ipaddress", "itertools 0.13.0", "libc", diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index dd8f634a..4a9cc462 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -71,7 +71,6 @@ figment.workspace = true futures.workspace = true http-body-util.workspace = true http.workspace = true -image.workspace = true ipaddress.workspace = true itertools.workspace = true libc.workspace = true diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index f1e3b924..ffa829d9 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -48,8 +48,6 @@ pub enum Error { Http(#[from] http::Error), #[error(transparent)] HttpHeader(#[from] http::header::InvalidHeaderValue), - #[error("Image error: {0}")] - Image(#[from] image::error::ImageError), #[error("Join error: {0}")] JoinError(#[from] tokio::task::JoinError), #[error(transparent)] diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index a6421b34..38eb7188 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -41,8 +41,10 @@ default = [ "gzip_compression", "io_uring", "jemalloc", + "media_thumbnail", "release_max_log_level", "systemd", + "url_preview", "zstd_compression", ] @@ -83,6 +85,9 @@ jemalloc_prof = [ jemalloc_stats = [ "conduwuit-core/jemalloc_stats", ] +media_thumbnail = [ + "conduwuit-service/media_thumbnail", +] perf_measurements = [ "dep:opentelemetry", "dep:tracing-flame", @@ -121,6 +126,9 @@ tokio_console = [ "dep:console-subscriber", "tokio/tracing", ] +url_preview = [ + "conduwuit-service/url_preview", +] zstd_compression = [ "conduwuit-api/zstd_compression", "conduwuit-core/zstd_compression", diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 26f737ee..4708ff4e 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -28,8 +28,8 @@ element_hacks = [] gzip_compression = [ "reqwest/gzip", ] -zstd_compression = [ - "reqwest/zstd", +media_thumbnail = [ + "dep:image", ] release_max_log_level = [ "tracing/max_level_trace", @@ -37,6 +37,13 @@ release_max_log_level = [ "log/max_level_trace", "log/release_max_level_info", ] +url_preview = [ + "dep:image", + "dep:webpage", +] +zstd_compression = [ + "reqwest/zstd", +] [dependencies] arrayvec.workspace = true @@ -51,6 +58,7 @@ futures.workspace = true hickory-resolver.workspace = true http.workspace = true image.workspace = true +image.optional = true ipaddress.workspace = true itertools.workspace = true jsonwebtoken.workspace = true @@ -73,6 +81,7 @@ tokio.workspace = true tracing.workspace = true url.workspace = true webpage.workspace = true +webpage.optional = true [lints] workspace = true diff --git a/src/service/media/data.rs b/src/service/media/data.rs index 43310515..f48482ea 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -3,7 +3,7 @@ use std::{sync::Arc, time::Duration}; use conduwuit::{ debug, debug_info, err, utils::{str_from_bytes, stream::TryIgnore, string_from_bytes, ReadyExt}, - Err, Error, Result, + Err, Result, }; use database::{Database, Interfix, Map}; use futures::StreamExt; @@ -123,30 +123,21 @@ impl Data { let content_type = parts .next() - .map(|bytes| { - string_from_bytes(bytes).map_err(|_| { - Error::bad_database("Content type in mediaid_file is invalid unicode.") - }) - }) - .transpose()?; + .map(string_from_bytes) + .transpose() + .map_err(|e| err!(Database(error!(?mxc, "Content-type is invalid: {e}"))))?; - let content_disposition_bytes = parts + let content_disposition = parts .next() - .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?; - - let content_disposition = if content_disposition_bytes.is_empty() { - None - } else { - Some( - string_from_bytes(content_disposition_bytes) - .map_err(|_| { - Error::bad_database( - "Content Disposition in mediaid_file is invalid unicode.", - ) - })? - .parse()?, - ) - }; + .map(Some) + .ok_or_else(|| err!(Database(error!(?mxc, "Media ID in db is invalid."))))? + .filter(|bytes| !bytes.is_empty()) + .map(string_from_bytes) + .transpose() + .map_err(|e| err!(Database(error!(?mxc, "Content-type is invalid: {e}"))))? + .as_deref() + .map(str::parse) + .transpose()?; Ok(Metadata { content_disposition, content_type, key }) } diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index b1c53305..e7f76bab 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -1,15 +1,19 @@ -use std::{io::Cursor, time::SystemTime}; +//! URL Previews +//! +//! This functionality is gated by 'url_preview', but not at the unit level for +//! historical and simplicity reasons. Instead the feature gates the inclusion +//! of dependencies and nulls out results through the existing interface when +//! not featured. -use conduwuit::{debug, utils, Err, Result}; +use std::time::SystemTime; + +use conduwuit::{debug, Err, Result}; use conduwuit_core::implement; -use image::ImageReader as ImgReader; use ipaddress::IPAddress; -use ruma::Mxc; use serde::Serialize; use url::Url; -use webpage::HTML; -use super::{Service, MXC_LENGTH}; +use super::Service; #[derive(Serialize, Default)] pub struct UrlPreviewData { @@ -41,34 +45,6 @@ pub async fn set_url_preview(&self, url: &str, data: &UrlPreviewData) -> Result< self.db.set_url_preview(url, data, now) } -#[implement(Service)] -pub async fn download_image(&self, url: &str) -> Result { - let client = &self.services.client.url_preview; - let image = client.get(url).send().await?.bytes().await?; - let mxc = Mxc { - server_name: self.services.globals.server_name(), - media_id: &utils::random_string(MXC_LENGTH), - }; - - self.create(&mxc, None, None, None, &image).await?; - - let (width, height) = match ImgReader::new(Cursor::new(&image)).with_guessed_format() { - | Err(_) => (None, None), - | Ok(reader) => match reader.into_dimensions() { - | Err(_) => (None, None), - | Ok((width, height)) => (Some(width), Some(height)), - }, - }; - - Ok(UrlPreviewData { - image: Some(mxc.to_string()), - image_size: Some(image.len()), - image_width: width, - image_height: height, - ..Default::default() - }) -} - #[implement(Service)] pub async fn get_url_preview(&self, url: &Url) -> Result { if let Ok(preview) = self.db.get_url_preview(url.as_str()).await { @@ -121,8 +97,51 @@ async fn request_url_preview(&self, url: &Url) -> Result { Ok(data) } +#[cfg(feature = "url_preview")] +#[implement(Service)] +pub async fn download_image(&self, url: &str) -> Result { + use conduwuit::utils::random_string; + use image::ImageReader; + use ruma::Mxc; + + let image = self.services.client.url_preview.get(url).send().await?; + let image = image.bytes().await?; + let mxc = Mxc { + server_name: self.services.globals.server_name(), + media_id: &random_string(super::MXC_LENGTH), + }; + + self.create(&mxc, None, None, None, &image).await?; + + let cursor = std::io::Cursor::new(&image); + let (width, height) = match ImageReader::new(cursor).with_guessed_format() { + | Err(_) => (None, None), + | Ok(reader) => match reader.into_dimensions() { + | Err(_) => (None, None), + | Ok((width, height)) => (Some(width), Some(height)), + }, + }; + + Ok(UrlPreviewData { + image: Some(mxc.to_string()), + image_size: Some(image.len()), + image_width: width, + image_height: height, + ..Default::default() + }) +} + +#[cfg(not(feature = "url_preview"))] +#[implement(Service)] +pub async fn download_image(&self, _url: &str) -> Result { + Err!(FeatureDisabled("url_preview")) +} + +#[cfg(feature = "url_preview")] #[implement(Service)] async fn download_html(&self, url: &str) -> Result { + use webpage::HTML; + let client = &self.services.client.url_preview; let mut response = client.get(url).send().await?; @@ -159,6 +178,12 @@ async fn download_html(&self, url: &str) -> Result { Ok(data) } +#[cfg(not(feature = "url_preview"))] +#[implement(Service)] +async fn download_html(&self, _url: &str) -> Result { + Err!(FeatureDisabled("url_preview")) +} + #[implement(Service)] pub fn url_preview_allowed(&self, url: &Url) -> bool { if ["http", "https"] diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 5c8063cb..7350b3a1 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -1,7 +1,13 @@ -use std::{cmp, io::Cursor, num::Saturating as Sat}; +//! Media Thumbnails +//! +//! This functionality is gated by 'media_thumbnail', but not at the unit level +//! for historical and simplicity reasons. Instead the feature gates the +//! inclusion of dependencies and nulls out results using the existing interface +//! when not featured. -use conduwuit::{checked, err, Result}; -use image::{imageops::FilterType, DynamicImage}; +use std::{cmp, num::Saturating as Sat}; + +use conduwuit::{checked, err, implement, Result}; use ruma::{http_headers::ContentDisposition, media::Method, Mxc, UInt, UserId}; use tokio::{ fs, @@ -67,65 +73,89 @@ impl super::Service { Ok(None) } } - - /// Using saved thumbnail - #[tracing::instrument(skip(self), name = "saved", level = "debug")] - async fn get_thumbnail_saved(&self, data: Metadata) -> Result> { - let mut content = Vec::new(); - let path = self.get_media_file(&data.key); - fs::File::open(path) - .await? - .read_to_end(&mut content) - .await?; - - Ok(Some(into_filemeta(data, content))) - } - - /// Generate a thumbnail - #[tracing::instrument(skip(self), name = "generate", level = "debug")] - async fn get_thumbnail_generate( - &self, - mxc: &Mxc<'_>, - dim: &Dim, - data: Metadata, - ) -> Result> { - let mut content = Vec::new(); - let path = self.get_media_file(&data.key); - fs::File::open(path) - .await? - .read_to_end(&mut content) - .await?; - - let Ok(image) = image::load_from_memory(&content) else { - // Couldn't parse file to generate thumbnail, send original - return Ok(Some(into_filemeta(data, content))); - }; - - if dim.width > image.width() || dim.height > image.height() { - return Ok(Some(into_filemeta(data, content))); - } - - let mut thumbnail_bytes = Vec::new(); - let thumbnail = thumbnail_generate(&image, dim)?; - thumbnail.write_to(&mut Cursor::new(&mut thumbnail_bytes), image::ImageFormat::Png)?; - - // Save thumbnail in database so we don't have to generate it again next time - let thumbnail_key = self.db.create_file_metadata( - mxc, - None, - dim, - data.content_disposition.as_ref(), - data.content_type.as_deref(), - )?; - - let mut f = self.create_media_file(&thumbnail_key).await?; - f.write_all(&thumbnail_bytes).await?; - - Ok(Some(into_filemeta(data, thumbnail_bytes))) - } } -fn thumbnail_generate(image: &DynamicImage, requested: &Dim) -> Result { +/// Using saved thumbnail +#[implement(super::Service)] +#[tracing::instrument(name = "saved", level = "debug", skip(self, data))] +async fn get_thumbnail_saved(&self, data: Metadata) -> Result> { + let mut content = Vec::new(); + let path = self.get_media_file(&data.key); + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; + + Ok(Some(into_filemeta(data, content))) +} + +/// Generate a thumbnail +#[cfg(feature = "media_thumbnail")] +#[implement(super::Service)] +#[tracing::instrument(name = "generate", level = "debug", skip(self, data))] +async fn get_thumbnail_generate( + &self, + mxc: &Mxc<'_>, + dim: &Dim, + data: Metadata, +) -> Result> { + let mut content = Vec::new(); + let path = self.get_media_file(&data.key); + fs::File::open(path) + .await? + .read_to_end(&mut content) + .await?; + + let Ok(image) = image::load_from_memory(&content) else { + // Couldn't parse file to generate thumbnail, send original + return Ok(Some(into_filemeta(data, content))); + }; + + if dim.width > image.width() || dim.height > image.height() { + return Ok(Some(into_filemeta(data, content))); + } + + let mut thumbnail_bytes = Vec::new(); + let thumbnail = thumbnail_generate(&image, dim)?; + let mut cursor = std::io::Cursor::new(&mut thumbnail_bytes); + thumbnail + .write_to(&mut cursor, image::ImageFormat::Png) + .map_err(|error| err!(error!(?error, "Error writing PNG thumbnail.")))?; + + // Save thumbnail in database so we don't have to generate it again next time + let thumbnail_key = self.db.create_file_metadata( + mxc, + None, + dim, + data.content_disposition.as_ref(), + data.content_type.as_deref(), + )?; + + let mut f = self.create_media_file(&thumbnail_key).await?; + f.write_all(&thumbnail_bytes).await?; + + Ok(Some(into_filemeta(data, thumbnail_bytes))) +} + +#[cfg(not(feature = "media_thumbnail"))] +#[implement(super::Service)] +#[tracing::instrument(name = "fallback", level = "debug", skip_all)] +async fn get_thumbnail_generate( + &self, + _mxc: &Mxc<'_>, + _dim: &Dim, + data: Metadata, +) -> Result> { + self.get_thumbnail_saved(data).await +} + +#[cfg(feature = "media_thumbnail")] +fn thumbnail_generate( + image: &image::DynamicImage, + requested: &Dim, +) -> Result { + use image::imageops::FilterType; + let thumbnail = if !requested.crop() { let Dim { width, height, .. } = requested.scaled(&Dim { width: image.width(), From 685b127f99d3168f28f82e4a6c7171ace4e8cd6a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 6 Jan 2025 05:12:01 +0000 Subject: [PATCH 1328/2091] simplify iterator state constructor arguments Signed-off-by: Jason Volk --- src/database/map.rs | 6 +++++- src/database/map/keys.rs | 2 +- src/database/map/keys_from.rs | 2 +- src/database/map/rev_keys.rs | 2 +- src/database/map/rev_keys_from.rs | 2 +- src/database/map/rev_stream.rs | 6 +++--- src/database/map/rev_stream_from.rs | 4 ++-- src/database/map/stream.rs | 6 +++--- src/database/map/stream_from.rs | 4 ++-- src/database/stream.rs | 8 ++++---- 10 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/database/map.rs b/src/database/map.rs index af22a43c..de37b8f9 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -77,7 +77,11 @@ impl Map { #[inline] pub fn name(&self) -> &str { &self.name } - fn cf(&self) -> impl AsColumnFamilyRef + '_ { &*self.cf } + #[inline] + pub(crate) fn db(&self) -> &Arc { &self.db } + + #[inline] + pub(crate) fn cf(&self) -> impl AsColumnFamilyRef + '_ { &*self.cf } } impl Debug for Map { diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs index 7d09f3da..60742334 100644 --- a/src/database/map/keys.rs +++ b/src/database/map/keys.rs @@ -23,7 +23,7 @@ pub fn raw_keys(self: &Arc) -> impl Stream>> + Send use crate::pool::Seek; let opts = super::iter_options_default(); - let state = stream::State::new(&self.db, &self.cf, opts); + let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_fwd(None); return task::consume_budget() diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 95c6611b..b83775ac 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -54,7 +54,7 @@ where use crate::pool::Seek; let opts = super::iter_options_default(); - let state = stream::State::new(&self.db, &self.cf, opts); + let state = stream::State::new(self, opts); if is_cached(self, from) { return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed(); } diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index 0ca6ad0f..a559d04b 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -23,7 +23,7 @@ pub fn rev_raw_keys(self: &Arc) -> impl Stream>> + S use crate::pool::Seek; let opts = super::iter_options_default(); - let state = stream::State::new(&self.db, &self.cf, opts); + let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_rev(None); return task::consume_budget() diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index e208c505..5b159195 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -62,7 +62,7 @@ where use crate::pool::Seek; let opts = super::iter_options_default(); - let state = stream::State::new(&self.db, &self.cf, opts); + let state = stream::State::new(self, opts); if is_cached(self, from) { return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed(); } diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index d882dd91..1d5d3d10 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -32,7 +32,7 @@ pub fn rev_raw_stream(self: &Arc) -> impl Stream> use crate::pool::Seek; let opts = super::read_options_default(); - let state = stream::State::new(&self.db, &self.cf, opts); + let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_rev(None); return task::consume_budget() @@ -65,9 +65,9 @@ pub fn rev_raw_stream(self: &Arc) -> impl Stream> skip_all, fields(%map), )] -pub(super) fn is_cached(map: &super::Map) -> bool { +pub(super) fn is_cached(map: &Arc) -> bool { let opts = super::cache_read_options_default(); - let state = stream::State::new(&map.db, &map.cf, opts).init_rev(None); + let state = stream::State::new(map, opts).init_rev(None); !state.is_incomplete() } diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 72fc739c..1b66e8cc 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -81,7 +81,7 @@ where use crate::pool::Seek; let opts = super::iter_options_default(); - let state = stream::State::new(&self.db, &self.cf, opts); + let state = stream::State::new(self, opts); if is_cached(self, from) { let state = state.init_rev(from.as_ref().into()); return task::consume_budget() @@ -119,7 +119,7 @@ where P: AsRef<[u8]> + ?Sized, { let cache_opts = super::cache_read_options_default(); - let cache_status = stream::State::new(&map.db, &map.cf, cache_opts) + let cache_status = stream::State::new(map, cache_opts) .init_rev(from.as_ref().into()) .status(); diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index 11b0676c..fa3b0ad7 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -31,7 +31,7 @@ pub fn raw_stream(self: &Arc) -> impl Stream>> + use crate::pool::Seek; let opts = super::read_options_default(); - let state = stream::State::new(&self.db, &self.cf, opts); + let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_fwd(None); return task::consume_budget() @@ -64,9 +64,9 @@ pub fn raw_stream(self: &Arc) -> impl Stream>> + skip_all, fields(%map), )] -pub(super) fn is_cached(map: &super::Map) -> bool { +pub(super) fn is_cached(map: &Arc) -> bool { let opts = super::cache_read_options_default(); - let state = stream::State::new(&map.db, &map.cf, opts).init_fwd(None); + let state = stream::State::new(map, opts).init_fwd(None); !state.is_incomplete() } diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 79ea8f51..4296b6f6 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -78,7 +78,7 @@ where use crate::pool::Seek; let opts = super::read_options_default(); - let state = stream::State::new(&self.db, &self.cf, opts); + let state = stream::State::new(self, opts); if is_cached(self, from) { let state = state.init_fwd(from.as_ref().into()); return task::consume_budget() @@ -116,7 +116,7 @@ where P: AsRef<[u8]> + ?Sized, { let opts = super::cache_read_options_default(); - let state = stream::State::new(&map.db, &map.cf, opts).init_fwd(from.as_ref().into()); + let state = stream::State::new(map, opts).init_fwd(from.as_ref().into()); !state.is_incomplete() } diff --git a/src/database/stream.rs b/src/database/stream.rs index d7cb16c6..5f0fc0a1 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -6,14 +6,14 @@ mod keys_rev; use std::sync::Arc; use conduwuit::{utils::exchange, Result}; -use rocksdb::{ColumnFamily, DBRawIteratorWithThreadMode, ReadOptions}; +use rocksdb::{DBRawIteratorWithThreadMode, ReadOptions}; pub(crate) use self::{items::Items, items_rev::ItemsRev, keys::Keys, keys_rev::KeysRev}; use crate::{ engine::Db, keyval::{Key, KeyVal, Val}, util::{is_incomplete, map_err}, - Engine, Slice, + Map, Slice, }; pub(crate) struct State<'a> { @@ -45,9 +45,9 @@ type Inner<'a> = DBRawIteratorWithThreadMode<'a, Db>; type From<'a> = Option>; impl<'a> State<'a> { - pub(super) fn new(db: &'a Arc, cf: &'a Arc, opts: ReadOptions) -> Self { + pub(super) fn new(map: &'a Arc, opts: ReadOptions) -> Self { Self { - inner: db.db.raw_iterator_cf_opt(&**cf, opts), + inner: map.db().db.raw_iterator_cf_opt(&map.cf(), opts), init: true, seek: false, } From 02f19cf951d5bfe90313bf7acb674f020a2e125e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 2 Jan 2025 05:30:51 +0000 Subject: [PATCH 1329/2091] tweak tracing spans; inlines db deserializer tracing instrument cover Signed-off-by: Jason Volk --- src/core/utils/mutex_map.rs | 13 +++-- src/database/de.rs | 55 +++++++++++++++++++ src/database/engine.rs | 2 + src/database/pool.rs | 2 - src/database/ser.rs | 2 + src/database/stream.rs | 13 +++++ src/database/stream/items.rs | 3 + src/database/stream/items_rev.rs | 3 + src/database/stream/keys.rs | 2 + src/database/stream/keys_rev.rs | 2 + src/service/rooms/event_handler/fetch_prev.rs | 6 +- .../rooms/event_handler/fetch_state.rs | 11 ++-- .../event_handler/handle_incoming_pdu.rs | 7 ++- .../rooms/event_handler/handle_prev_pdu.rs | 6 +- src/service/rooms/state_cache/mod.rs | 49 +++++++++++------ src/service/sending/sender.rs | 4 +- 16 files changed, 146 insertions(+), 34 deletions(-) diff --git a/src/core/utils/mutex_map.rs b/src/core/utils/mutex_map.rs index 152a75d1..9b9821fe 100644 --- a/src/core/utils/mutex_map.rs +++ b/src/core/utils/mutex_map.rs @@ -19,8 +19,8 @@ type Value = Arc>; impl MutexMap where - Key: Send + Hash + Eq + Clone, - Val: Send + Default, + Key: Clone + Eq + Hash + Send, + Val: Default + Send, { #[must_use] pub fn new() -> Self { @@ -29,10 +29,10 @@ where } } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(level = "trace", skip(self))] pub async fn lock(&self, k: &K) -> Guard where - K: ?Sized + Send + Sync + Debug, + K: Debug + Send + ?Sized + Sync, Key: for<'a> From<&'a K>, { let val = self @@ -61,13 +61,14 @@ where impl Default for MutexMap where - Key: Send + Hash + Eq + Clone, - Val: Send + Default, + Key: Clone + Eq + Hash + Send, + Val: Default + Send, { fn default() -> Self { Self::new() } } impl Drop for Guard { + #[tracing::instrument(name = "unlock", level = "trace", skip_all)] fn drop(&mut self) { if Arc::strong_count(Omg::mutex(&self.val)) <= 2 { self.map.lock().expect("locked").retain(|_, val| { diff --git a/src/database/de.rs b/src/database/de.rs index 4f5be6fc..48bc9f64 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -9,6 +9,15 @@ use serde::{ use crate::util::unhandled; /// Deserialize into T from buffer. +#[cfg_attr( + unabridged, + tracing::instrument( + name = "deserialize", + level = "trace", + skip_all, + fields(len = %buf.len()), + ) +)] pub(crate) fn from_slice<'a, T>(buf: &'a [u8]) -> Result where T: Deserialize<'a>, @@ -132,6 +141,17 @@ impl<'de> Deserializer<'de> { /// Increment the position pointer. #[inline] + #[cfg_attr( + unabridged, + tracing::instrument( + level = "trace", + skip(self), + fields( + len = self.buf.len(), + rem = self.remaining().unwrap_or_default().saturating_sub(n), + ), + ) + )] fn inc_pos(&mut self, n: usize) { self.pos = self.pos.saturating_add(n); debug_assert!(self.pos <= self.buf.len(), "pos out of range"); @@ -149,6 +169,7 @@ impl<'de> Deserializer<'de> { impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { type Error = Error; + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_seq(self, visitor: V) -> Result where V: Visitor<'de>, @@ -157,6 +178,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { visitor.visit_seq(self) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip(self, visitor)))] fn deserialize_tuple(self, _len: usize, visitor: V) -> Result where V: Visitor<'de>, @@ -165,6 +187,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { visitor.visit_seq(self) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip(self, visitor)))] fn deserialize_tuple_struct( self, _name: &'static str, @@ -178,6 +201,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { visitor.visit_seq(self) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_map(self, visitor: V) -> Result where V: Visitor<'de>, @@ -187,6 +211,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { d.deserialize_map(visitor).map_err(Into::into) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip(self, visitor)))] fn deserialize_struct( self, name: &'static str, @@ -202,6 +227,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { .map_err(Into::into) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip(self, visitor)))] fn deserialize_unit_struct(self, name: &'static str, visitor: V) -> Result where V: Visitor<'de>, @@ -215,6 +241,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { visitor.visit_unit() } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip(self, visitor)))] fn deserialize_newtype_struct(self, name: &'static str, visitor: V) -> Result where V: Visitor<'de>, @@ -225,6 +252,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip(self, _visitor)))] fn deserialize_enum( self, _name: &'static str, @@ -237,26 +265,32 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { unhandled!("deserialize Enum not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_option>(self, _visitor: V) -> Result { unhandled!("deserialize Option not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_bool>(self, _visitor: V) -> Result { unhandled!("deserialize bool not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_i8>(self, _visitor: V) -> Result { unhandled!("deserialize i8 not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_i16>(self, _visitor: V) -> Result { unhandled!("deserialize i16 not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_i32>(self, _visitor: V) -> Result { unhandled!("deserialize i32 not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_i64>(self, visitor: V) -> Result { const BYTES: usize = size_of::(); @@ -268,6 +302,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { visitor.visit_i64(i64::from_be_bytes(bytes)) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_u8>(self, _visitor: V) -> Result { unhandled!( "deserialize u8 not implemented; try dereferencing the Handle for [u8] access \ @@ -275,14 +310,17 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { ) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_u16>(self, _visitor: V) -> Result { unhandled!("deserialize u16 not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_u32>(self, _visitor: V) -> Result { unhandled!("deserialize u32 not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_u64>(self, visitor: V) -> Result { const BYTES: usize = size_of::(); @@ -294,53 +332,67 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { visitor.visit_u64(u64::from_be_bytes(bytes)) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_f32>(self, _visitor: V) -> Result { unhandled!("deserialize f32 not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_f64>(self, _visitor: V) -> Result { unhandled!("deserialize f64 not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_char>(self, _visitor: V) -> Result { unhandled!("deserialize char not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_str>(self, visitor: V) -> Result { let input = self.record_next(); let out = deserialize_str(input)?; visitor.visit_borrowed_str(out) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_string>(self, visitor: V) -> Result { let input = self.record_next(); let out = string::string_from_bytes(input)?; visitor.visit_string(out) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_bytes>(self, visitor: V) -> Result { let input = self.record_trail(); visitor.visit_borrowed_bytes(input) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_byte_buf>(self, _visitor: V) -> Result { unhandled!("deserialize Byte Buf not implemented") } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_unit>(self, _visitor: V) -> Result { unhandled!("deserialize Unit not implemented") } // this only used for $serde_json::private::RawValue at this time; see MapAccess + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_identifier>(self, visitor: V) -> Result { let input = "$serde_json::private::RawValue"; visitor.visit_borrowed_str(input) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] fn deserialize_ignored_any>(self, _visitor: V) -> Result { unhandled!("deserialize Ignored Any not implemented") } + #[cfg_attr( + unabridged, + tracing::instrument(level = "trace", skip_all, fields(?self.buf)) + )] fn deserialize_any>(self, visitor: V) -> Result { debug_assert_eq!( conduwuit::debug::type_name::(), @@ -363,6 +415,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { impl<'a, 'de: 'a> de::SeqAccess<'de> for &'a mut Deserializer<'de> { type Error = Error; + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip(self, seed)))] fn next_element_seed(&mut self, seed: T) -> Result> where T: DeserializeSeed<'de>, @@ -381,6 +434,7 @@ impl<'a, 'de: 'a> de::SeqAccess<'de> for &'a mut Deserializer<'de> { impl<'a, 'de: 'a> de::MapAccess<'de> for &'a mut Deserializer<'de> { type Error = Error; + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip(self, seed)))] fn next_key_seed(&mut self, seed: K) -> Result> where K: DeserializeSeed<'de>, @@ -388,6 +442,7 @@ impl<'a, 'de: 'a> de::MapAccess<'de> for &'a mut Deserializer<'de> { seed.deserialize(&mut **self).map(Some) } + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip(self, seed)))] fn next_value_seed(&mut self, seed: V) -> Result where V: DeserializeSeed<'de>, diff --git a/src/database/engine.rs b/src/database/engine.rs index 670817b5..3d554eac 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -157,11 +157,13 @@ impl Engine { #[inline] pub fn corked(&self) -> bool { self.corks.load(std::sync::atomic::Ordering::Relaxed) > 0 } + #[inline] pub(crate) fn cork(&self) { self.corks .fetch_add(1, std::sync::atomic::Ordering::Relaxed); } + #[inline] pub(crate) fn uncork(&self) { self.corks .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); diff --git a/src/database/pool.rs b/src/database/pool.rs index 11871ff6..b972e763 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -207,8 +207,6 @@ pub(crate) async fn execute_get(self: &Arc, mut cmd: Get) -> Result(out: &'a mut W, val: T) -> Result<&'a [u8]> where W: Write + AsRef<[u8]> + 'a, diff --git a/src/database/stream.rs b/src/database/stream.rs index 5f0fc0a1..f3063bb3 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -29,12 +29,14 @@ pub(crate) trait Cursor<'a, T> { fn seek(&mut self); + #[inline] fn get(&self) -> Option> { self.fetch() .map(Ok) .or_else(|| self.state().status().map(map_err).map(Err)) } + #[inline] fn seek_and_get(&mut self) -> Option> { self.seek(); self.get() @@ -45,6 +47,7 @@ type Inner<'a> = DBRawIteratorWithThreadMode<'a, Db>; type From<'a> = Option>; impl<'a> State<'a> { + #[inline] pub(super) fn new(map: &'a Arc, opts: ReadOptions) -> Self { Self { inner: map.db().db.raw_iterator_cf_opt(&map.cf(), opts), @@ -53,6 +56,8 @@ impl<'a> State<'a> { } } + #[inline] + #[tracing::instrument(level = "trace", skip_all)] pub(super) fn init_fwd(mut self, from: From<'_>) -> Self { debug_assert!(self.init, "init must be set to make this call"); debug_assert!(!self.seek, "seek must not be set to make this call"); @@ -67,6 +72,8 @@ impl<'a> State<'a> { self } + #[inline] + #[tracing::instrument(level = "trace", skip_all)] pub(super) fn init_rev(mut self, from: From<'_>) -> Self { debug_assert!(self.init, "init must be set to make this call"); debug_assert!(!self.seek, "seek must not be set to make this call"); @@ -82,6 +89,7 @@ impl<'a> State<'a> { } #[inline] + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] pub(super) fn seek_fwd(&mut self) { if !exchange(&mut self.init, false) { self.inner.next(); @@ -91,6 +99,7 @@ impl<'a> State<'a> { } #[inline] + #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] pub(super) fn seek_rev(&mut self) { if !exchange(&mut self.init, false) { self.inner.prev(); @@ -103,12 +112,16 @@ impl<'a> State<'a> { matches!(self.status(), Some(e) if is_incomplete(&e)) } + #[inline] fn fetch_key(&self) -> Option> { self.inner.key().map(Key::from) } + #[inline] fn _fetch_val(&self) -> Option> { self.inner.value().map(Val::from) } + #[inline] fn fetch(&self) -> Option> { self.inner.item().map(KeyVal::from) } + #[inline] pub(super) fn status(&self) -> Option { self.inner.status().err() } #[inline] diff --git a/src/database/stream/items.rs b/src/database/stream/items.rs index cd81b4a0..8814419e 100644 --- a/src/database/stream/items.rs +++ b/src/database/stream/items.rs @@ -15,12 +15,15 @@ pub(crate) struct Items<'a> { } impl<'a> From> for Items<'a> { + #[inline] fn from(state: State<'a>) -> Self { Self { state } } } impl<'a> Cursor<'a, KeyVal<'a>> for Items<'a> { + #[inline] fn state(&self) -> &State<'a> { &self.state } + #[inline] fn fetch(&self) -> Option> { self.state.fetch().map(keyval_longevity) } #[inline] diff --git a/src/database/stream/items_rev.rs b/src/database/stream/items_rev.rs index c6cf9b53..f6fcb0e5 100644 --- a/src/database/stream/items_rev.rs +++ b/src/database/stream/items_rev.rs @@ -15,12 +15,15 @@ pub(crate) struct ItemsRev<'a> { } impl<'a> From> for ItemsRev<'a> { + #[inline] fn from(state: State<'a>) -> Self { Self { state } } } impl<'a> Cursor<'a, KeyVal<'a>> for ItemsRev<'a> { + #[inline] fn state(&self) -> &State<'a> { &self.state } + #[inline] fn fetch(&self) -> Option> { self.state.fetch().map(keyval_longevity) } #[inline] diff --git a/src/database/stream/keys.rs b/src/database/stream/keys.rs index 9bf27507..b953f51c 100644 --- a/src/database/stream/keys.rs +++ b/src/database/stream/keys.rs @@ -15,10 +15,12 @@ pub(crate) struct Keys<'a> { } impl<'a> From> for Keys<'a> { + #[inline] fn from(state: State<'a>) -> Self { Self { state } } } impl<'a> Cursor<'a, Key<'a>> for Keys<'a> { + #[inline] fn state(&self) -> &State<'a> { &self.state } #[inline] diff --git a/src/database/stream/keys_rev.rs b/src/database/stream/keys_rev.rs index 8657df0f..acf78d88 100644 --- a/src/database/stream/keys_rev.rs +++ b/src/database/stream/keys_rev.rs @@ -15,10 +15,12 @@ pub(crate) struct KeysRev<'a> { } impl<'a> From> for KeysRev<'a> { + #[inline] fn from(state: State<'a>) -> Self { Self { state } } } impl<'a> Cursor<'a, Key<'a>> for KeysRev<'a> { + #[inline] fn state(&self) -> &State<'a> { &self.state } #[inline] diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 3f121f69..0d64e98e 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -15,8 +15,12 @@ use ruma::{ use super::check_room_id; #[implement(super::Service)] +#[tracing::instrument( + level = "warn", + skip_all, + fields(%origin), +)] #[allow(clippy::type_complexity)] -#[tracing::instrument(skip_all)] pub(super) async fn fetch_prev( &self, origin: &ServerName, diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index edc47194..cc4a3e46 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,6 +1,6 @@ use std::collections::{hash_map, HashMap}; -use conduwuit::{debug, implement, warn, Err, Error, PduEvent, Result}; +use conduwuit::{debug, debug_warn, implement, Err, Error, PduEvent, Result}; use futures::FutureExt; use ruma::{ api::federation::event::get_room_state_ids, events::StateEventType, EventId, OwnedEventId, @@ -13,7 +13,11 @@ use crate::rooms::short::ShortStateKey; /// server's response to some extend (sic), but we still do a lot of checks /// on the events #[implement(super::Service)] -#[tracing::instrument(skip(self, create_event, room_version_id))] +#[tracing::instrument( + level = "warn", + skip_all, + fields(%origin), +)] pub(super) async fn fetch_state( &self, origin: &ServerName, @@ -22,7 +26,6 @@ pub(super) async fn fetch_state( room_version_id: &RoomVersionId, event_id: &EventId, ) -> Result>> { - debug!("Fetching state ids"); let res = self .services .sending @@ -31,7 +34,7 @@ pub(super) async fn fetch_state( event_id: event_id.to_owned(), }) .await - .inspect_err(|e| warn!("Fetching state for event failed: {e}"))?; + .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; debug!("Fetching state events"); let state_vec = self diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 4c2fb2f7..c2e6ccc9 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -39,7 +39,12 @@ use crate::rooms::timeline::RawPduId; /// 14. Check if the event passes auth based on the "current state" of the room, /// if not soft fail it #[implement(super::Service)] -#[tracing::instrument(skip(self, origin, value, is_timeline_event), name = "pdu")] +#[tracing::instrument( + name = "pdu", + level = "warn", + skip_all, + fields(%room_id, %event_id), +)] pub async fn handle_incoming_pdu<'a>( &self, origin: &'a ServerName, diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index 9bd4450e..ad71c173 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -13,8 +13,10 @@ use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName}; #[allow(clippy::type_complexity)] #[allow(clippy::too_many_arguments)] #[tracing::instrument( - skip(self, origin, event_id, room_id, eventid_info, create_event, first_pdu_in_room), - name = "prev" + name = "prev", + level = "warn", + skip_all, + fields(%prev_id), )] pub(super) async fn handle_prev_pdu<'a>( &self, diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index c2de8b62..89421dfd 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -95,7 +95,16 @@ impl crate::Service for Service { impl Service { /// Update current membership data. - #[tracing::instrument(skip(self, last_state))] + #[tracing::instrument( + level = "debug", + skip_all, + fields( + %room_id, + %user_id, + %sender, + ?membership_event, + ), + )] #[allow(clippy::too_many_arguments)] pub async fn update_membership( &self, @@ -265,7 +274,7 @@ impl Service { Ok(()) } - #[tracing::instrument(skip(self, room_id, appservice), level = "debug")] + #[tracing::instrument(level = "trace", skip_all)] pub async fn appservice_in_room( &self, room_id: &RoomId, @@ -383,7 +392,7 @@ impl Service { .map(|(_, server): (Ignore, &ServerName)| server) } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn server_in_room<'a>( &'a self, server: &'a ServerName, @@ -409,7 +418,7 @@ impl Service { } /// Returns true if server can see user by sharing at least one room. - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn server_sees_user(&self, server: &ServerName, user_id: &UserId) -> bool { self.server_rooms(server) .any(|room_id| self.is_joined(user_id, room_id)) @@ -417,7 +426,7 @@ impl Service { } /// Returns true if user_a and user_b share at least one room. - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> bool { let get_shared_rooms = self.get_shared_rooms(user_a, user_b); @@ -426,6 +435,7 @@ impl Service { } /// List the rooms common between two users + #[tracing::instrument(skip(self), level = "debug")] pub fn get_shared_rooms<'a>( &'a self, user_a: &'a UserId, @@ -453,7 +463,7 @@ impl Service { } /// Returns the number of users which are currently in a room - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn room_joined_count(&self, room_id: &RoomId) -> Result { self.db.roomid_joinedcount.get(room_id).await.deserialized() } @@ -469,9 +479,9 @@ impl Service { .ready_filter(|user| self.services.globals.user_is_local(user)) } - #[tracing::instrument(skip(self), level = "debug")] /// Returns an iterator of all our local joined users in a room who are /// active (not deactivated, not guest) + #[tracing::instrument(skip(self), level = "trace")] pub fn active_local_users_in_room<'a>( &'a self, room_id: &'a RoomId, @@ -481,7 +491,7 @@ impl Service { } /// Returns the number of users which are currently invited to a room - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn room_invited_count(&self, room_id: &RoomId) -> Result { self.db .roomid_invitedcount @@ -518,7 +528,7 @@ impl Service { .map(|(_, user_id): (Ignore, &UserId)| user_id) } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { let key = (room_id, user_id); self.db @@ -528,7 +538,7 @@ impl Service { .deserialized() } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { let key = (room_id, user_id); self.db.roomuserid_leftcount.qry(&key).await.deserialized() @@ -566,7 +576,7 @@ impl Service { .ignore_err() } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn invite_state( &self, user_id: &UserId, @@ -583,7 +593,7 @@ impl Service { }) } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn left_state( &self, user_id: &UserId, @@ -625,24 +635,25 @@ impl Service { self.db.roomuseroncejoinedids.qry(&key).await.is_ok() } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn is_joined<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { let key = (user_id, room_id); self.db.userroomid_joined.qry(&key).await.is_ok() } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> bool { let key = (user_id, room_id); self.db.userroomid_invitestate.qry(&key).await.is_ok() } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> bool { let key = (user_id, room_id); self.db.userroomid_leftstate.qry(&key).await.is_ok() } + #[tracing::instrument(skip(self), level = "trace")] pub async fn user_membership( &self, user_id: &UserId, @@ -683,7 +694,7 @@ impl Service { /// distant future. /// /// See - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "trace")] pub async fn servers_route_via(&self, room_id: &RoomId) -> Result> { let most_powerful_user_server = self .services @@ -724,6 +735,7 @@ impl Service { (cache.len(), cache.capacity()) } + #[tracing::instrument(level = "debug", skip_all)] pub fn clear_appservice_in_room_cache(&self) { self.appservice_in_room_cache .write() @@ -731,6 +743,7 @@ impl Service { .clear(); } + #[tracing::instrument(level = "debug", skip(self))] pub async fn update_joined_count(&self, room_id: &RoomId) { let mut joinedcount = 0_u64; let mut invitedcount = 0_u64; @@ -784,11 +797,13 @@ impl Service { .remove(room_id); } + #[tracing::instrument(level = "debug", skip(self))] fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) { let key = (user_id, room_id); self.db.roomuseroncejoinedids.put_raw(key, []); } + #[tracing::instrument(level = "debug", skip(self, last_state, invite_via))] pub async fn mark_as_invited( &self, user_id: &UserId, @@ -821,7 +836,7 @@ impl Service { } } - #[tracing::instrument(skip(self, servers), level = "debug")] + #[tracing::instrument(level = "debug", skip(self, servers))] pub async fn add_servers_invite_via(&self, room_id: &RoomId, servers: Vec) { let mut servers: Vec<_> = self .servers_invite_via(room_id) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index a9abada4..5fd4cf91 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -80,7 +80,9 @@ impl Service { self.work_loop(id, &mut futures, &mut statuses).await; - self.finish_responses(&mut futures).boxed().await; + if !futures.is_empty() { + self.finish_responses(&mut futures).boxed().await; + } Ok(()) } From 6a0f9add0c1f882d4cbae1b595fcdaa1577183be Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 6 Jan 2025 14:07:26 +0000 Subject: [PATCH 1330/2091] refactor database engine/options; add column descriptors Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 4 +- src/database/database.rs | 55 ---- src/database/engine.rs | 336 +++----------------- src/database/engine/backup.rs | 73 +++++ src/database/engine/cf_opts.rs | 214 +++++++++++++ src/database/engine/context.rs | 73 +++++ src/database/engine/db_opts.rs | 133 ++++++++ src/database/engine/descriptor.rs | 89 ++++++ src/database/engine/files.rs | 32 ++ src/database/engine/logger.rs | 22 ++ src/database/engine/memory_usage.rs | 30 ++ src/database/engine/open.rs | 121 ++++++++ src/database/engine/repair.rs | 16 + src/database/map.rs | 74 +---- src/database/map/open.rs | 37 +++ src/database/map/options.rs | 26 ++ src/database/maps.rs | 466 ++++++++++++++++++++++------ src/database/mod.rs | 77 ++++- src/database/opts.rs | 433 -------------------------- src/service/migrations.rs | 8 +- src/service/services.rs | 4 +- 21 files changed, 1358 insertions(+), 965 deletions(-) delete mode 100644 src/database/database.rs create mode 100644 src/database/engine/backup.rs create mode 100644 src/database/engine/cf_opts.rs create mode 100644 src/database/engine/context.rs create mode 100644 src/database/engine/db_opts.rs create mode 100644 src/database/engine/descriptor.rs create mode 100644 src/database/engine/files.rs create mode 100644 src/database/engine/logger.rs create mode 100644 src/database/engine/memory_usage.rs create mode 100644 src/database/engine/open.rs create mode 100644 src/database/engine/repair.rs create mode 100644 src/database/map/open.rs create mode 100644 src/database/map/options.rs delete mode 100644 src/database/opts.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 5bc65d9b..07daaf0a 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -911,8 +911,8 @@ pub(super) async fn database_stats( let map_name = map.as_ref().map_or(EMPTY, String::as_str); let mut out = String::new(); - for (name, map) in self.services.db.iter() { - if !map_name.is_empty() && *map_name != *name { + for (&name, map) in self.services.db.iter() { + if !map_name.is_empty() && map_name != name { continue; } diff --git a/src/database/database.rs b/src/database/database.rs deleted file mode 100644 index 83d2c201..00000000 --- a/src/database/database.rs +++ /dev/null @@ -1,55 +0,0 @@ -use std::{ops::Index, sync::Arc}; - -use conduwuit::{err, Result, Server}; - -use crate::{ - maps, - maps::{Maps, MapsKey, MapsVal}, - Engine, Map, -}; - -pub struct Database { - pub db: Arc, - maps: Maps, -} - -impl Database { - /// Load an existing database or create a new one. - pub async fn open(server: &Arc) -> Result> { - let db = Engine::open(server).await?; - Ok(Arc::new(Self { db: db.clone(), maps: maps::open(&db)? })) - } - - #[inline] - pub fn get(&self, name: &str) -> Result<&Arc> { - self.maps - .get(name) - .ok_or_else(|| err!(Request(NotFound("column not found")))) - } - - #[inline] - pub fn iter(&self) -> impl Iterator + Send + '_ { - self.maps.iter() - } - - #[inline] - pub fn keys(&self) -> impl Iterator + Send + '_ { self.maps.keys() } - - #[inline] - #[must_use] - pub fn is_read_only(&self) -> bool { self.db.is_read_only() } - - #[inline] - #[must_use] - pub fn is_secondary(&self) -> bool { self.db.is_secondary() } -} - -impl Index<&str> for Database { - type Output = Arc; - - fn index(&self, name: &str) -> &Self::Output { - self.maps - .get(name) - .expect("column in database does not exist") - } -} diff --git a/src/database/engine.rs b/src/database/engine.rs index 3d554eac..2958f73f 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -1,288 +1,64 @@ +mod backup; +mod cf_opts; +pub(crate) mod context; +mod db_opts; +pub(crate) mod descriptor; +mod files; +mod logger; +mod memory_usage; +mod open; +mod repair; + use std::{ - collections::{BTreeSet, HashMap}, ffi::CStr, - fmt::Write, - path::PathBuf, - sync::{atomic::AtomicU32, Arc, Mutex, RwLock}, + sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }, }; -use conduwuit::{ - debug, error, info, utils::time::rfc2822_from_seconds, warn, Err, Result, Server, -}; -use rocksdb::{ - backup::{BackupEngine, BackupEngineOptions}, - perf::get_memory_usage_stats, - AsColumnFamilyRef, BoundColumnFamily, Cache, ColumnFamilyDescriptor, DBCommon, - DBWithThreadMode, Env, LogLevel, MultiThreaded, Options, -}; +use conduwuit::{debug, info, warn, Err, Result}; +use rocksdb::{AsColumnFamilyRef, BoundColumnFamily, DBCommon, DBWithThreadMode, MultiThreaded}; -use crate::{ - opts::{cf_options, db_options}, - or_else, - pool::Pool, - result, - util::map_err, -}; +use crate::{pool::Pool, result, Context}; pub struct Engine { - pub(crate) server: Arc, - row_cache: Cache, - col_cache: RwLock>, - opts: Options, - env: Env, - cfs: Mutex>, - pub(crate) pool: Arc, - pub(crate) db: Db, - corks: AtomicU32, pub(super) read_only: bool, pub(super) secondary: bool, + corks: AtomicU32, + pub(crate) db: Db, + pub(crate) pool: Arc, + pub(crate) ctx: Arc, } pub(crate) type Db = DBWithThreadMode; impl Engine { - #[tracing::instrument(skip_all)] - pub(crate) async fn open(server: &Arc) -> Result> { - let config = &server.config; - let path = &config.database_path; - - let cache_capacity_bytes = config.db_cache_capacity_mb * 1024.0 * 1024.0; - - #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] - let row_cache_capacity_bytes = (cache_capacity_bytes * 0.50) as usize; - - #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] - let col_cache_capacity_bytes = (cache_capacity_bytes * 0.50) as usize; - - let mut col_cache = HashMap::new(); - col_cache.insert("primary".to_owned(), Cache::new_lru_cache(col_cache_capacity_bytes)); - - let mut db_env = Env::new().or_else(or_else)?; - let row_cache = Cache::new_lru_cache(row_cache_capacity_bytes); - let db_opts = db_options( - config, - &mut db_env, - &row_cache, - col_cache.get("primary").expect("primary cache exists"), - )?; - - debug!("Listing column families in database"); - let cfs = Db::list_cf(&db_opts, &config.database_path) - .unwrap_or_default() - .into_iter() - .collect::>(); - - debug!("Configuring {} column families found in database", cfs.len()); - let cfopts = cfs - .iter() - .map(|name| cf_options(config, name, db_opts.clone(), &mut col_cache)) - .collect::>>()?; - - debug!("Opening {} column family descriptors in database", cfs.len()); - let cfds = cfs - .iter() - .zip(cfopts.into_iter()) - .map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts)) - .collect::>(); - - debug!("Starting frontend request pool"); - let pool = Pool::new(server)?; - - let load_time = std::time::Instant::now(); - if config.rocksdb_repair { - repair(&db_opts, &config.database_path)?; - } - - debug!("Opening database..."); - let res = if config.rocksdb_read_only { - Db::open_cf_descriptors_read_only(&db_opts, path, cfds, false) - } else if config.rocksdb_secondary { - Db::open_cf_descriptors_as_secondary(&db_opts, path, path, cfds) - } else { - Db::open_cf_descriptors(&db_opts, path, cfds) - }; - - let db = res.or_else(or_else)?; - info!( - columns = cfs.len(), - sequence = %db.latest_sequence_number(), - time = ?load_time.elapsed(), - "Opened database." - ); - - Ok(Arc::new(Self { - server: server.clone(), - row_cache, - col_cache: RwLock::new(col_cache), - opts: db_opts, - env: db_env, - cfs: Mutex::new(cfs), - corks: AtomicU32::new(0), - read_only: config.rocksdb_read_only, - secondary: config.rocksdb_secondary, - pool, - db, - })) - } - - #[tracing::instrument(skip(self), level = "trace")] - pub(crate) fn open_cf(&self, name: &str) -> Result>> { - let mut cfs = self.cfs.lock().expect("locked"); - if !cfs.contains(name) { - debug!("Creating new column family in database: {name}"); - - let mut col_cache = self.col_cache.write().expect("locked"); - let opts = cf_options(&self.server.config, name, self.opts.clone(), &mut col_cache)?; - if let Err(e) = self.db.create_cf(name, &opts) { - error!(?name, "Failed to create new column family: {e}"); - return or_else(e); - } - - cfs.insert(name.to_owned()); - } - - Ok(self.cf(name)) - } - pub(crate) fn cf(&self, name: &str) -> Arc> { self.db .cf_handle(name) - .expect("column was created and exists") - } - - pub fn flush(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, false)) } - - pub fn sync(&self) -> Result<()> { result(DBCommon::flush_wal(&self.db, true)) } - - #[inline] - pub fn corked(&self) -> bool { self.corks.load(std::sync::atomic::Ordering::Relaxed) > 0 } - - #[inline] - pub(crate) fn cork(&self) { - self.corks - .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + .expect("column must be described prior to database open") } #[inline] - pub(crate) fn uncork(&self) { - self.corks - .fetch_sub(1, std::sync::atomic::Ordering::Relaxed); - } + pub(crate) fn cork(&self) { self.corks.fetch_add(1, Ordering::Relaxed); } - pub fn memory_usage(&self) -> Result { - let mut res = String::new(); - let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&self.row_cache])) - .or_else(or_else)?; - let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0; - writeln!( - res, - "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow \ - cache: {:.2} MiB", - mibs(stats.mem_table_total), - mibs(stats.mem_table_unflushed), - mibs(stats.mem_table_readers_total), - mibs(u64::try_from(self.row_cache.get_usage())?), - )?; + #[inline] + pub(crate) fn uncork(&self) { self.corks.fetch_sub(1, Ordering::Relaxed); } - for (name, cache) in &*self.col_cache.read().expect("locked") { - writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?; - } - - Ok(res) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn cleanup(&self) -> Result<()> { - debug!("Running flush_opt"); - let flushoptions = rocksdb::FlushOptions::default(); - result(DBCommon::flush_opt(&self.db, &flushoptions)) - } + #[inline] + pub fn corked(&self) -> bool { self.corks.load(Ordering::Relaxed) > 0 } #[tracing::instrument(skip(self))] - pub fn backup(&self) -> Result { - let config = &self.server.config; - let path = config.database_backup_path.as_ref(); - if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { - return Ok(()); - } + pub fn sync(&self) -> Result { result(DBCommon::flush_wal(&self.db, true)) } - let options = BackupEngineOptions::new(path.expect("valid database backup path")) - .map_err(map_err)?; - let mut engine = BackupEngine::open(&options, &self.env).map_err(map_err)?; - if config.database_backups_to_keep > 0 { - let flush = !self.is_read_only(); - engine - .create_new_backup_flush(&self.db, flush) - .map_err(map_err)?; + #[tracing::instrument(skip(self), level = "debug")] + pub fn flush(&self) -> Result { result(DBCommon::flush_wal(&self.db, false)) } - let engine_info = engine.get_backup_info(); - let info = &engine_info.last().expect("backup engine info is not empty"); - info!( - "Created database backup #{} using {} bytes in {} files", - info.backup_id, info.size, info.num_files, - ); - } - - if config.database_backups_to_keep >= 0 { - let keep = u32::try_from(config.database_backups_to_keep)?; - if let Err(e) = engine.purge_old_backups(keep.try_into()?) { - error!("Failed to purge old backup: {e:?}"); - } - } - - Ok(()) - } - - pub fn backup_list(&self) -> Result { - let config = &self.server.config; - let path = config.database_backup_path.as_ref(); - if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { - return Ok("Configure database_backup_path to enable backups, or the path \ - specified is not valid" - .to_owned()); - } - - let mut res = String::new(); - let options = BackupEngineOptions::new(path.expect("valid database backup path")) - .or_else(or_else)?; - let engine = BackupEngine::open(&options, &self.env).or_else(or_else)?; - for info in engine.get_backup_info() { - writeln!( - res, - "#{} {}: {} bytes, {} files", - info.backup_id, - rfc2822_from_seconds(info.timestamp), - info.size, - info.num_files, - )?; - } - - Ok(res) - } - - pub fn file_list(&self) -> Result { - match self.db.live_files() { - | Err(e) => Ok(String::from(e)), - | Ok(files) => { - let mut res = String::new(); - writeln!(res, "| lev | sst | keys | dels | size | column |")?; - writeln!(res, "| ---: | :--- | ---: | ---: | ---: | :--- |")?; - for file in files { - writeln!( - res, - "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", - file.level, - file.name, - file.num_entries, - file.num_deletions, - file.size, - file.column_family_name, - )?; - } - - Ok(res) - }, - } + #[tracing::instrument(skip(self), level = "debug")] + pub fn sort(&self) -> Result { + let flushoptions = rocksdb::FlushOptions::default(); + result(DBCommon::flush_opt(&self.db, &flushoptions)) } /// Query for database property by null-terminated name which is expected to @@ -312,56 +88,14 @@ impl Engine { pub fn is_secondary(&self) -> bool { self.secondary } } -pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result<()> { - warn!("Starting database repair. This may take a long time..."); - match Db::repair(db_opts, path) { - | Ok(()) => info!("Database repair successful."), - | Err(e) => return Err!("Repair failed: {e:?}"), - } - - Ok(()) -} - -#[tracing::instrument( - parent = None, - name = "rocksdb", - level = "trace" - skip(msg), -)] -pub(crate) fn handle_log(level: LogLevel, msg: &str) { - let msg = msg.trim(); - if msg.starts_with("Options") { - return; - } - - match level { - | LogLevel::Header | LogLevel::Debug => debug!("{msg}"), - | LogLevel::Error | LogLevel::Fatal => error!("{msg}"), - | LogLevel::Info => debug!("{msg}"), - | LogLevel::Warn => warn!("{msg}"), - }; -} - impl Drop for Engine { #[cold] fn drop(&mut self) { const BLOCKING: bool = true; - debug!("Closing frontend pool"); - self.pool.close(); - debug!("Waiting for background tasks to finish..."); self.db.cancel_all_background_work(BLOCKING); - debug!("Shutting down background threads"); - self.env.set_high_priority_background_threads(0); - self.env.set_low_priority_background_threads(0); - self.env.set_bottom_priority_background_threads(0); - self.env.set_background_threads(0); - - debug!("Joining background threads..."); - self.env.join_all_threads(); - info!( sequence = %self.db.latest_sequence_number(), "Closing database..." diff --git a/src/database/engine/backup.rs b/src/database/engine/backup.rs new file mode 100644 index 00000000..db718c2c --- /dev/null +++ b/src/database/engine/backup.rs @@ -0,0 +1,73 @@ +use std::fmt::Write; + +use conduwuit::{error, implement, info, utils::time::rfc2822_from_seconds, warn, Result}; +use rocksdb::backup::{BackupEngine, BackupEngineOptions}; + +use super::Engine; +use crate::{or_else, util::map_err}; + +#[implement(Engine)] +#[tracing::instrument(skip(self))] +pub fn backup(&self) -> Result { + let server = &self.ctx.server; + let config = &server.config; + let path = config.database_backup_path.as_ref(); + if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { + return Ok(()); + } + + let options = + BackupEngineOptions::new(path.expect("valid database backup path")).map_err(map_err)?; + let mut engine = BackupEngine::open(&options, &*self.ctx.env.lock()?).map_err(map_err)?; + if config.database_backups_to_keep > 0 { + let flush = !self.is_read_only(); + engine + .create_new_backup_flush(&self.db, flush) + .map_err(map_err)?; + + let engine_info = engine.get_backup_info(); + let info = &engine_info.last().expect("backup engine info is not empty"); + info!( + "Created database backup #{} using {} bytes in {} files", + info.backup_id, info.size, info.num_files, + ); + } + + if config.database_backups_to_keep >= 0 { + let keep = u32::try_from(config.database_backups_to_keep)?; + if let Err(e) = engine.purge_old_backups(keep.try_into()?) { + error!("Failed to purge old backup: {e:?}"); + } + } + + Ok(()) +} + +#[implement(Engine)] +pub fn backup_list(&self) -> Result { + let server = &self.ctx.server; + let config = &server.config; + let path = config.database_backup_path.as_ref(); + if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { + return Ok("Configure database_backup_path to enable backups, or the path specified is \ + not valid" + .to_owned()); + } + + let mut res = String::new(); + let options = + BackupEngineOptions::new(path.expect("valid database backup path")).or_else(or_else)?; + let engine = BackupEngine::open(&options, &*self.ctx.env.lock()?).or_else(or_else)?; + for info in engine.get_backup_info() { + writeln!( + res, + "#{} {}: {} bytes, {} files", + info.backup_id, + rfc2822_from_seconds(info.timestamp), + info.size, + info.num_files, + )?; + } + + Ok(res) +} diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs new file mode 100644 index 00000000..98d74044 --- /dev/null +++ b/src/database/engine/cf_opts.rs @@ -0,0 +1,214 @@ +use conduwuit::{ + err, + utils::{math::Expected, BoolExt}, + Config, Result, +}; +use rocksdb::{ + BlockBasedIndexType, BlockBasedOptions, BlockBasedPinningTier, Cache, + DBCompressionType as CompressionType, DataBlockIndexType, LruCacheOptions, Options, + UniversalCompactOptions, UniversalCompactionStopStyle, +}; + +use super::descriptor::{CacheDisp, Descriptor}; +use crate::Context; + +/// Adjust options for the specific column by name. Provide the result of +/// db_options() as the argument to this function and use the return value in +/// the arguments to open the specific column. +pub(crate) fn cf_options(ctx: &Context, opts: Options, desc: &Descriptor) -> Result { + let cache = get_cache(ctx, desc); + let config = &ctx.server.config; + descriptor_cf_options(opts, desc.clone(), config, cache.as_ref()) +} + +fn descriptor_cf_options( + mut opts: Options, + mut desc: Descriptor, + config: &Config, + cache: Option<&Cache>, +) -> Result { + set_compression(&mut desc, config); + set_table_options(&mut opts, &desc, cache); + + opts.set_min_write_buffer_number(1); + opts.set_max_write_buffer_number(2); + if let Some(write_size) = desc.write_size { + opts.set_write_buffer_size(write_size); + } + + opts.set_target_file_size_base(desc.file_size); + opts.set_target_file_size_multiplier(desc.file_shape[0]); + + opts.set_level_zero_file_num_compaction_trigger(desc.level0_width); + opts.set_level_compaction_dynamic_level_bytes(false); + opts.set_ttl(desc.ttl); + + opts.set_max_bytes_for_level_base(desc.level_size); + opts.set_max_bytes_for_level_multiplier(1.0); + opts.set_max_bytes_for_level_multiplier_additional(&desc.level_shape); + + opts.set_compaction_style(desc.compaction); + opts.set_compaction_pri(desc.compaction_pri); + opts.set_universal_compaction_options(&uc_options(&desc)); + + opts.set_compression_type(desc.compression); + opts.set_compression_options(-14, desc.compression_level, 0, 0); // -14 w_bits used by zlib. + if let Some(&bottommost_level) = desc.bottommost_level.as_ref() { + opts.set_bottommost_compression_type(desc.compression); + opts.set_bottommost_zstd_max_train_bytes(0, true); + opts.set_bottommost_compression_options( + -14, // -14 w_bits is only read by zlib. + bottommost_level, + 0, + 0, + true, + ); + } + + Ok(opts) +} + +fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache>) { + let mut table = table_options(desc); + if let Some(cache) = cache { + table.set_block_cache(cache); + } else { + table.disable_cache(); + } + + opts.set_block_based_table_factory(&table); +} + +fn set_compression(desc: &mut Descriptor, config: &Config) { + desc.compression = match config.rocksdb_compression_algo.as_ref() { + | "snappy" => CompressionType::Snappy, + | "zlib" => CompressionType::Zlib, + | "bz2" => CompressionType::Bz2, + | "lz4" => CompressionType::Lz4, + | "lz4hc" => CompressionType::Lz4hc, + | "none" => CompressionType::None, + | _ => CompressionType::Zstd, + }; + + desc.compression_level = config.rocksdb_compression_level; + desc.bottommost_level = config + .rocksdb_bottommost_compression + .then_some(config.rocksdb_bottommost_compression_level); +} + +fn uc_options(desc: &Descriptor) -> UniversalCompactOptions { + let mut opts = UniversalCompactOptions::default(); + opts.set_stop_style(UniversalCompactionStopStyle::Total); + opts.set_min_merge_width(desc.merge_width.0); + opts.set_max_merge_width(desc.merge_width.1); + opts.set_max_size_amplification_percent(10000); + opts.set_compression_size_percent(-1); + opts.set_size_ratio(1); + + opts +} + +fn table_options(desc: &Descriptor) -> BlockBasedOptions { + let mut opts = BlockBasedOptions::default(); + + opts.set_block_size(desc.block_size); + opts.set_metadata_block_size(desc.index_size); + + opts.set_cache_index_and_filter_blocks(true); + opts.set_pin_top_level_index_and_filter(false); + opts.set_pin_l0_filter_and_index_blocks_in_cache(false); + opts.set_partition_pinning_tier(BlockBasedPinningTier::None); + opts.set_unpartitioned_pinning_tier(BlockBasedPinningTier::None); + opts.set_top_level_index_pinning_tier(BlockBasedPinningTier::None); + + opts.set_use_delta_encoding(false); + opts.set_index_type(BlockBasedIndexType::TwoLevelIndexSearch); + opts.set_data_block_index_type( + desc.block_index_hashing + .map_or(DataBlockIndexType::BinarySearch, || DataBlockIndexType::BinaryAndHash), + ); + + opts +} + +fn get_cache(ctx: &Context, desc: &Descriptor) -> Option { + let config = &ctx.server.config; + + // Some cache capacities are overriden by server config in a strange but + // legacy-compat way + let cap = match desc.name { + | "eventid_pduid" => Some(config.eventid_pdu_cache_capacity), + | "eventid_shorteventid" => Some(config.eventidshort_cache_capacity), + | "shorteventid_eventid" => Some(config.shorteventid_cache_capacity), + | "shorteventid_authchain" => Some(config.auth_chain_cache_capacity), + | "shortstatekey_statekey" => Some(config.shortstatekey_cache_capacity), + | "statekey_shortstatekey" => Some(config.statekeyshort_cache_capacity), + | "servernameevent_data" => Some(config.servernameevent_data_cache_capacity), + | "pduid_pdu" | "eventid_outlierpdu" => Some(config.pdu_cache_capacity), + | _ => None, + } + .map(TryInto::try_into) + .transpose() + .expect("u32 to usize"); + + let ent_size: usize = desc + .key_size_hint + .unwrap_or_default() + .expected_add(desc.val_size_hint.unwrap_or_default()); + + let size = match cap { + | Some(cap) => cache_size(config, cap, ent_size), + | _ => desc.cache_size, + }; + + let shard_bits: i32 = desc + .cache_shards + .ilog2() + .try_into() + .expect("u32 to i32 conversion"); + + debug_assert!(shard_bits <= 6, "cache shards limited to 64"); + let mut cache_opts = LruCacheOptions::default(); + cache_opts.set_num_shard_bits(shard_bits); + cache_opts.set_capacity(size); + + let mut caches = ctx.col_cache.lock().expect("locked"); + match desc.cache_disp { + | CacheDisp::Unique if desc.cache_size == 0 => None, + | CacheDisp::Unique => { + let cache = Cache::new_lru_cache_opts(&cache_opts); + caches.insert(desc.name.into(), cache.clone()); + Some(cache) + }, + + | CacheDisp::SharedWith(other) if !caches.contains_key(other) => { + let cache = Cache::new_lru_cache_opts(&cache_opts); + caches.insert(desc.name.into(), cache.clone()); + Some(cache) + }, + + | CacheDisp::SharedWith(other) => Some( + caches + .get(other) + .cloned() + .expect("caches.contains_key(other) must be true"), + ), + + | CacheDisp::Shared => Some( + caches + .get("Shared") + .cloned() + .expect("shared cache must already exist"), + ), + } +} + +#[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] +pub(crate) fn cache_size(config: &Config, base_size: u32, entity_size: usize) -> usize { + let ents = f64::from(base_size) * config.cache_capacity_modifier; + + (ents as usize) + .checked_mul(entity_size) + .ok_or_else(|| err!(Config("cache_capacity_modifier", "Cache size is too large."))) + .expect("invalid cache size") +} diff --git a/src/database/engine/context.rs b/src/database/engine/context.rs new file mode 100644 index 00000000..76238f7d --- /dev/null +++ b/src/database/engine/context.rs @@ -0,0 +1,73 @@ +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, +}; + +use conduwuit::{debug, utils::math::usize_from_f64, Result, Server}; +use rocksdb::{Cache, Env}; + +use crate::{or_else, pool::Pool}; + +/// Some components are constructed prior to opening the database and must +/// outlive the database. These can also be shared between database instances +/// though at the time of this comment we only open one database per process. +/// These assets are housed in the shared Context. +pub(crate) struct Context { + pub(crate) pool: Arc, + pub(crate) col_cache: Mutex>, + pub(crate) row_cache: Mutex, + pub(crate) env: Mutex, + pub(crate) server: Arc, +} + +impl Context { + pub(crate) fn new(server: &Arc) -> Result> { + let config = &server.config; + let cache_capacity_bytes = config.db_cache_capacity_mb * 1024.0 * 1024.0; + + let row_cache_capacity_bytes = usize_from_f64(cache_capacity_bytes * 0.50)?; + let row_cache = Cache::new_lru_cache(row_cache_capacity_bytes); + + let col_cache_capacity_bytes = usize_from_f64(cache_capacity_bytes * 0.50)?; + let col_cache = Cache::new_lru_cache(col_cache_capacity_bytes); + + let col_cache: BTreeMap<_, _> = [("Shared".to_owned(), col_cache)].into(); + + let mut env = Env::new().or_else(or_else)?; + + if config.rocksdb_compaction_prio_idle { + env.lower_thread_pool_cpu_priority(); + } + + if config.rocksdb_compaction_ioprio_idle { + env.lower_thread_pool_io_priority(); + } + + Ok(Arc::new(Self { + pool: Pool::new(server)?, + col_cache: col_cache.into(), + row_cache: row_cache.into(), + env: env.into(), + server: server.clone(), + })) + } +} + +impl Drop for Context { + #[cold] + fn drop(&mut self) { + debug!("Closing frontend pool"); + self.pool.close(); + + let mut env = self.env.lock().expect("locked"); + + debug!("Shutting down background threads"); + env.set_high_priority_background_threads(0); + env.set_low_priority_background_threads(0); + env.set_bottom_priority_background_threads(0); + env.set_background_threads(0); + + debug!("Joining background threads..."); + env.join_all_threads(); + } +} diff --git a/src/database/engine/db_opts.rs b/src/database/engine/db_opts.rs new file mode 100644 index 00000000..211265de --- /dev/null +++ b/src/database/engine/db_opts.rs @@ -0,0 +1,133 @@ +use std::{cmp, convert::TryFrom}; + +use conduwuit::{utils, Config, Result}; +use rocksdb::{statistics::StatsLevel, Cache, DBRecoveryMode, Env, LogLevel, Options}; + +use super::{cf_opts::cache_size, logger::handle as handle_log}; + +/// Create database-wide options suitable for opening the database. This also +/// sets our default column options in case of opening a column with the same +/// resulting value. Note that we require special per-column options on some +/// columns, therefor columns should only be opened after passing this result +/// through cf_options(). +pub(crate) fn db_options(config: &Config, env: &Env, row_cache: &Cache) -> Result { + const DEFAULT_STATS_LEVEL: StatsLevel = if cfg!(debug_assertions) { + StatsLevel::ExceptDetailedTimers + } else { + StatsLevel::DisableAll + }; + + let mut opts = Options::default(); + + // Logging + set_logging_defaults(&mut opts, config); + + // Processing + opts.set_max_background_jobs(num_threads::(config)?); + opts.set_max_subcompactions(num_threads::(config)?); + opts.set_avoid_unnecessary_blocking_io(true); + opts.set_max_file_opening_threads(0); + + // IO + opts.set_atomic_flush(true); + opts.set_manual_wal_flush(true); + opts.set_enable_pipelined_write(false); + if config.rocksdb_direct_io { + opts.set_use_direct_reads(true); + opts.set_use_direct_io_for_flush_and_compaction(true); + } + if config.rocksdb_optimize_for_spinning_disks { + // speeds up opening DB on hard drives + opts.set_skip_checking_sst_file_sizes_on_db_open(true); + opts.set_skip_stats_update_on_db_open(true); + //opts.set_max_file_opening_threads(threads.try_into().unwrap()); + } + + // Blocks + opts.set_row_cache(row_cache); + + // Files + opts.set_table_cache_num_shard_bits(7); + opts.set_wal_size_limit_mb(1024 * 1024 * 1024); + opts.set_max_total_wal_size(1024 * 1024 * 512); + opts.set_db_write_buffer_size(cache_size(config, 1024 * 1024 * 32, 1)); + + // Misc + opts.set_disable_auto_compactions(!config.rocksdb_compaction); + opts.create_missing_column_families(true); + opts.create_if_missing(true); + + opts.set_statistics_level(match config.rocksdb_stats_level { + | 0 => StatsLevel::DisableAll, + | 1 => DEFAULT_STATS_LEVEL, + | 2 => StatsLevel::ExceptHistogramOrTimers, + | 3 => StatsLevel::ExceptTimers, + | 4 => StatsLevel::ExceptDetailedTimers, + | 5 => StatsLevel::ExceptTimeForMutex, + | 6_u8..=u8::MAX => StatsLevel::All, + }); + + opts.set_report_bg_io_stats(match config.rocksdb_stats_level { + | 0..=1 => false, + | 2_u8..=u8::MAX => true, + }); + + // Default: https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes#ktoleratecorruptedtailrecords + // + // Unclean shutdowns of a Matrix homeserver are likely to be fine when + // recovered in this manner as it's likely any lost information will be + // restored via federation. + opts.set_wal_recovery_mode(match config.rocksdb_recovery_mode { + | 0 => DBRecoveryMode::AbsoluteConsistency, + | 1 => DBRecoveryMode::TolerateCorruptedTailRecords, + | 2 => DBRecoveryMode::PointInTime, + | 3 => DBRecoveryMode::SkipAnyCorruptedRecord, + | 4_u8..=u8::MAX => unimplemented!(), + }); + + // + // "We recommend to set track_and_verify_wals_in_manifest to true for + // production, it has been enabled in production for the entire database cluster + // serving the social graph for all Meta apps." + opts.set_track_and_verify_wals_in_manifest(true); + + opts.set_paranoid_checks(config.rocksdb_paranoid_file_checks); + + opts.set_env(env); + + Ok(opts) +} + +fn set_logging_defaults(opts: &mut Options, config: &Config) { + let rocksdb_log_level = match config.rocksdb_log_level.as_ref() { + | "debug" => LogLevel::Debug, + | "info" => LogLevel::Info, + | "warn" => LogLevel::Warn, + | "fatal" => LogLevel::Fatal, + | _ => LogLevel::Error, + }; + + opts.set_log_level(rocksdb_log_level); + opts.set_max_log_file_size(config.rocksdb_max_log_file_size); + opts.set_log_file_time_to_roll(config.rocksdb_log_time_to_roll); + opts.set_keep_log_file_num(config.rocksdb_max_log_files); + opts.set_stats_dump_period_sec(0); + + if config.rocksdb_log_stderr { + opts.set_stderr_logger(rocksdb_log_level, "rocksdb"); + } else { + opts.set_callback_logger(rocksdb_log_level, &handle_log); + } +} + +fn num_threads>(config: &Config) -> Result { + const MIN_PARALLELISM: usize = 2; + + let requested = if config.rocksdb_parallelism_threads != 0 { + config.rocksdb_parallelism_threads + } else { + utils::available_parallelism() + }; + + utils::math::try_into::(cmp::max(MIN_PARALLELISM, requested)) +} diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs new file mode 100644 index 00000000..f0fd83f1 --- /dev/null +++ b/src/database/engine/descriptor.rs @@ -0,0 +1,89 @@ +use conduwuit::utils::string::EMPTY; +use rocksdb::{ + DBCompactionPri as CompactionPri, DBCompactionStyle as CompactionStyle, + DBCompressionType as CompressionType, +}; + +#[derive(Debug, Clone, Copy)] +pub(crate) enum CacheDisp { + Unique, + Shared, + SharedWith(&'static str), +} + +#[derive(Debug, Clone)] +pub(crate) struct Descriptor { + pub(crate) name: &'static str, + pub(crate) cache_disp: CacheDisp, + pub(crate) key_size_hint: Option, + pub(crate) val_size_hint: Option, + pub(crate) block_size: usize, + pub(crate) index_size: usize, + pub(crate) write_size: Option, + pub(crate) cache_size: usize, + pub(crate) level_size: u64, + pub(crate) level_shape: [i32; 7], + pub(crate) file_size: u64, + pub(crate) file_shape: [i32; 1], + pub(crate) level0_width: i32, + pub(crate) merge_width: (i32, i32), + pub(crate) ttl: u64, + pub(crate) compaction: CompactionStyle, + pub(crate) compaction_pri: CompactionPri, + pub(crate) compression: CompressionType, + pub(crate) compression_level: i32, + pub(crate) bottommost_level: Option, + pub(crate) block_index_hashing: bool, + pub(crate) cache_shards: u32, +} + +pub(crate) static BASE: Descriptor = Descriptor { + name: EMPTY, + cache_disp: CacheDisp::Shared, + key_size_hint: None, + val_size_hint: None, + block_size: 1024 * 4, + index_size: 1024 * 4, + write_size: None, + cache_size: 1024 * 1024 * 4, + level_size: 1024 * 1024 * 8, + level_shape: [1, 1, 1, 3, 7, 15, 31], + file_size: 1024 * 1024, + file_shape: [2], + level0_width: 2, + merge_width: (2, 16), + ttl: 60 * 60 * 24 * 21, + compaction: CompactionStyle::Level, + compaction_pri: CompactionPri::MinOverlappingRatio, + compression: CompressionType::Zstd, + compression_level: 32767, + bottommost_level: Some(32767), + block_index_hashing: false, + cache_shards: 64, +}; + +pub(crate) static RANDOM: Descriptor = Descriptor { + compaction_pri: CompactionPri::OldestSmallestSeqFirst, + ..BASE +}; + +pub(crate) static SEQUENTIAL: Descriptor = Descriptor { + compaction_pri: CompactionPri::OldestLargestSeqFirst, + level_size: 1024 * 1024 * 32, + file_size: 1024 * 1024 * 2, + ..BASE +}; + +pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { + compaction: CompactionStyle::Universal, + level_size: 1024 * 512, + file_size: 1024 * 128, + ..RANDOM +}; + +pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { + compaction: CompactionStyle::Universal, + level_size: 1024 * 1024, + file_size: 1024 * 512, + ..SEQUENTIAL +}; diff --git a/src/database/engine/files.rs b/src/database/engine/files.rs new file mode 100644 index 00000000..f603c57b --- /dev/null +++ b/src/database/engine/files.rs @@ -0,0 +1,32 @@ +use std::fmt::Write; + +use conduwuit::{implement, Result}; + +use super::Engine; + +#[implement(Engine)] +pub fn file_list(&self) -> Result { + match self.db.live_files() { + | Err(e) => Ok(String::from(e)), + | Ok(mut files) => { + files.sort_by_key(|f| f.name.clone()); + let mut res = String::new(); + writeln!(res, "| lev | sst | keys | dels | size | column |")?; + writeln!(res, "| ---: | :--- | ---: | ---: | ---: | :--- |")?; + for file in files { + writeln!( + res, + "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", + file.level, + file.name, + file.num_entries, + file.num_deletions, + file.size, + file.column_family_name, + )?; + } + + Ok(res) + }, + } +} diff --git a/src/database/engine/logger.rs b/src/database/engine/logger.rs new file mode 100644 index 00000000..a1898e30 --- /dev/null +++ b/src/database/engine/logger.rs @@ -0,0 +1,22 @@ +use conduwuit::{debug, error, warn}; +use rocksdb::LogLevel; + +#[tracing::instrument( + parent = None, + name = "rocksdb", + level = "trace" + skip(msg), +)] +pub(crate) fn handle(level: LogLevel, msg: &str) { + let msg = msg.trim(); + if msg.starts_with("Options") { + return; + } + + match level { + | LogLevel::Header | LogLevel::Debug => debug!("{msg}"), + | LogLevel::Error | LogLevel::Fatal => error!("{msg}"), + | LogLevel::Info => debug!("{msg}"), + | LogLevel::Warn => warn!("{msg}"), + }; +} diff --git a/src/database/engine/memory_usage.rs b/src/database/engine/memory_usage.rs new file mode 100644 index 00000000..01859815 --- /dev/null +++ b/src/database/engine/memory_usage.rs @@ -0,0 +1,30 @@ +use std::fmt::Write; + +use conduwuit::{implement, Result}; +use rocksdb::perf::get_memory_usage_stats; + +use super::Engine; +use crate::or_else; + +#[implement(Engine)] +pub fn memory_usage(&self) -> Result { + let mut res = String::new(); + let stats = get_memory_usage_stats(Some(&[&self.db]), Some(&[&*self.ctx.row_cache.lock()?])) + .or_else(or_else)?; + let mibs = |input| f64::from(u32::try_from(input / 1024).unwrap_or(0)) / 1024.0; + writeln!( + res, + "Memory buffers: {:.2} MiB\nPending write: {:.2} MiB\nTable readers: {:.2} MiB\nRow \ + cache: {:.2} MiB", + mibs(stats.mem_table_total), + mibs(stats.mem_table_unflushed), + mibs(stats.mem_table_readers_total), + mibs(u64::try_from(self.ctx.row_cache.lock()?.get_usage())?), + )?; + + for (name, cache) in &*self.ctx.col_cache.lock()? { + writeln!(res, "{name} cache: {:.2} MiB", mibs(u64::try_from(cache.get_usage())?))?; + } + + Ok(res) +} diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs new file mode 100644 index 00000000..9999296b --- /dev/null +++ b/src/database/engine/open.rs @@ -0,0 +1,121 @@ +use std::{ + collections::BTreeSet, + path::Path, + sync::{atomic::AtomicU32, Arc}, +}; + +use conduwuit::{debug, debug_warn, implement, info, warn, Result}; +use rocksdb::{ColumnFamilyDescriptor, Options}; + +use super::{ + cf_opts::cf_options, db_opts::db_options, descriptor::Descriptor, repair::repair, Db, Engine, +}; +use crate::{or_else, Context}; + +#[implement(Engine)] +#[tracing::instrument(skip_all)] +pub(crate) async fn open(ctx: Arc, desc: &[Descriptor]) -> Result> { + let server = &ctx.server; + let config = &server.config; + let path = &config.database_path; + + let db_opts = db_options( + config, + &ctx.env.lock().expect("environment locked"), + &ctx.row_cache.lock().expect("row cache locked"), + )?; + + let cfds = Self::configure_cfds(&ctx, &db_opts, desc)?; + let num_cfds = cfds.len(); + debug!("Configured {num_cfds} column descriptors..."); + + let load_time = std::time::Instant::now(); + if config.rocksdb_repair { + repair(&db_opts, &config.database_path)?; + } + + debug!("Opening database..."); + let db = if config.rocksdb_read_only { + Db::open_cf_descriptors_read_only(&db_opts, path, cfds, false) + } else if config.rocksdb_secondary { + Db::open_cf_descriptors_as_secondary(&db_opts, path, path, cfds) + } else { + Db::open_cf_descriptors(&db_opts, path, cfds) + } + .or_else(or_else)?; + + info!( + columns = num_cfds, + sequence = %db.latest_sequence_number(), + time = ?load_time.elapsed(), + "Opened database." + ); + + Ok(Arc::new(Self { + read_only: config.rocksdb_read_only, + secondary: config.rocksdb_secondary, + corks: AtomicU32::new(0), + pool: ctx.pool.clone(), + db, + ctx, + })) +} + +#[implement(Engine)] +#[tracing::instrument(name = "configure", skip_all)] +fn configure_cfds( + ctx: &Arc, + db_opts: &Options, + desc: &[Descriptor], +) -> Result> { + let server = &ctx.server; + let config = &server.config; + let path = &config.database_path; + let existing = Self::discover_cfs(path, db_opts); + debug!( + "Found {} existing columns; have {} described columns", + existing.len(), + desc.len() + ); + + existing + .iter() + .filter(|&name| name != "default") + .filter(|&name| !desc.iter().any(|desc| desc.name == name)) + .for_each(|name| { + debug_warn!("Found unknown column {name:?} in database which will not be opened."); + }); + + desc.iter() + .filter(|desc| !existing.contains(desc.name)) + .for_each(|desc| { + debug!( + "Creating new column {:?} which was not found in the existing database.", + desc.name, + ); + }); + + let cfopts: Vec<_> = desc + .iter() + .map(|desc| cf_options(ctx, db_opts.clone(), desc)) + .collect::>()?; + + let cfds: Vec<_> = desc + .iter() + .map(|desc| desc.name) + .map(ToOwned::to_owned) + .zip(cfopts.into_iter()) + .map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts)) + .collect(); + + Ok(cfds) +} + +#[implement(Engine)] +#[tracing::instrument(name = "discover", skip_all)] +fn discover_cfs(path: &Path, opts: &Options) -> BTreeSet { + Db::list_cf(opts, path) + .unwrap_or_default() + .into_iter() + .collect::>() +} diff --git a/src/database/engine/repair.rs b/src/database/engine/repair.rs new file mode 100644 index 00000000..61283904 --- /dev/null +++ b/src/database/engine/repair.rs @@ -0,0 +1,16 @@ +use std::path::PathBuf; + +use conduwuit::{info, warn, Err, Result}; +use rocksdb::Options; + +use super::Db; + +pub(crate) fn repair(db_opts: &Options, path: &PathBuf) -> Result { + warn!("Starting database repair. This may take a long time..."); + match Db::repair(db_opts, path) { + | Ok(()) => info!("Database repair successful."), + | Err(e) => return Err!("Repair failed: {e:?}"), + } + + Ok(()) +} diff --git a/src/database/map.rs b/src/database/map.rs index de37b8f9..60d66585 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -6,6 +6,8 @@ mod insert; mod keys; mod keys_from; mod keys_prefix; +mod open; +mod options; mod remove; mod rev_keys; mod rev_keys_from; @@ -28,12 +30,15 @@ use std::{ }; use conduwuit::Result; -use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, ReadTier, WriteOptions}; +use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, WriteOptions}; +pub(crate) use self::options::{ + cache_read_options_default, iter_options_default, read_options_default, write_options_default, +}; use crate::{watchers::Watchers, Engine}; pub struct Map { - name: String, + name: &'static str, db: Arc, cf: Arc, watchers: Watchers, @@ -43,11 +48,11 @@ pub struct Map { } impl Map { - pub(crate) fn open(db: &Arc, name: &str) -> Result> { + pub(crate) fn open(db: &Arc, name: &'static str) -> Result> { Ok(Arc::new(Self { - name: name.to_owned(), + name, db: db.clone(), - cf: open(db, name)?, + cf: open::open(db, name), watchers: Watchers::default(), write_options: write_options_default(), read_options: read_options_default(), @@ -75,7 +80,7 @@ impl Map { pub fn property(&self, name: &str) -> Result { self.db.property(&self.cf(), name) } #[inline] - pub fn name(&self) -> &str { &self.name } + pub fn name(&self) -> &str { self.name } #[inline] pub(crate) fn db(&self) -> &Arc { &self.db } @@ -93,60 +98,3 @@ impl Debug for Map { impl Display for Map { fn fmt(&self, out: &mut fmt::Formatter<'_>) -> fmt::Result { write!(out, "{0}", self.name) } } - -fn open(db: &Arc, name: &str) -> Result> { - let bounded_arc = db.open_cf(name)?; - let bounded_ptr = Arc::into_raw(bounded_arc); - let cf_ptr = bounded_ptr.cast::(); - - // SAFETY: Column family handles out of RocksDB are basic pointers and can - // be invalidated: 1. when the database closes. 2. when the column is dropped or - // closed. rust_rocksdb wraps this for us by storing handles in their own - // `RwLock` map and returning an Arc>` to - // provide expected safety. Similarly in "single-threaded mode" we would - // receive `&'_ ColumnFamily`. - // - // PROBLEM: We need to hold these handles in a field, otherwise we have to take - // a lock and get them by name from this map for every query, which is what - // conduit was doing, but we're not going to make a query for every query so we - // need to be holding it right. The lifetime parameter on these references makes - // that complicated. If this can be done without polluting the userspace - // with lifetimes on every instance of `Map` then this `unsafe` might not be - // necessary. - // - // SOLUTION: After investigating the underlying types it appears valid to - // Arc-swap `BoundColumnFamily<'_>` for `ColumnFamily`. They have the - // same inner data, the same Drop behavior, Deref, etc. We're just losing the - // lifetime parameter. We should not hold this handle, even in its Arc, after - // closing the database (dropping `Engine`). Since `Arc` is a sibling - // member along with this handle in `Map`, that is prevented. - Ok(unsafe { - Arc::increment_strong_count(cf_ptr); - Arc::from_raw(cf_ptr) - }) -} - -#[inline] -pub(crate) fn iter_options_default() -> ReadOptions { - let mut read_options = read_options_default(); - read_options.set_background_purge_on_iterator_cleanup(true); - //read_options.set_pin_data(true); - read_options -} - -#[inline] -pub(crate) fn cache_read_options_default() -> ReadOptions { - let mut read_options = read_options_default(); - read_options.set_read_tier(ReadTier::BlockCache); - read_options -} - -#[inline] -pub(crate) fn read_options_default() -> ReadOptions { - let mut read_options = ReadOptions::default(); - read_options.set_total_order_seek(true); - read_options -} - -#[inline] -pub(crate) fn write_options_default() -> WriteOptions { WriteOptions::default() } diff --git a/src/database/map/open.rs b/src/database/map/open.rs new file mode 100644 index 00000000..6ecec044 --- /dev/null +++ b/src/database/map/open.rs @@ -0,0 +1,37 @@ +use std::sync::Arc; + +use rocksdb::ColumnFamily; + +use crate::Engine; + +pub(super) fn open(db: &Arc, name: &str) -> Arc { + let bounded_arc = db.cf(name); + let bounded_ptr = Arc::into_raw(bounded_arc); + let cf_ptr = bounded_ptr.cast::(); + + // SAFETY: Column family handles out of RocksDB are basic pointers and can + // be invalidated: 1. when the database closes. 2. when the column is dropped or + // closed. rust_rocksdb wraps this for us by storing handles in their own + // `RwLock` map and returning an Arc>` to + // provide expected safety. Similarly in "single-threaded mode" we would + // receive `&'_ ColumnFamily`. + // + // PROBLEM: We need to hold these handles in a field, otherwise we have to take + // a lock and get them by name from this map for every query, which is what + // conduit was doing, but we're not going to make a query for every query so we + // need to be holding it right. The lifetime parameter on these references makes + // that complicated. If this can be done without polluting the userspace + // with lifetimes on every instance of `Map` then this `unsafe` might not be + // necessary. + // + // SOLUTION: After investigating the underlying types it appears valid to + // Arc-swap `BoundColumnFamily<'_>` for `ColumnFamily`. They have the + // same inner data, the same Drop behavior, Deref, etc. We're just losing the + // lifetime parameter. We should not hold this handle, even in its Arc, after + // closing the database (dropping `Engine`). Since `Arc` is a sibling + // member along with this handle in `Map`, that is prevented. + unsafe { + Arc::increment_strong_count(cf_ptr); + Arc::from_raw(cf_ptr) + } +} diff --git a/src/database/map/options.rs b/src/database/map/options.rs new file mode 100644 index 00000000..90dc0261 --- /dev/null +++ b/src/database/map/options.rs @@ -0,0 +1,26 @@ +use rocksdb::{ReadOptions, ReadTier, WriteOptions}; + +#[inline] +pub(crate) fn iter_options_default() -> ReadOptions { + let mut read_options = read_options_default(); + read_options.set_background_purge_on_iterator_cleanup(true); + //read_options.set_pin_data(true); + read_options +} + +#[inline] +pub(crate) fn cache_read_options_default() -> ReadOptions { + let mut read_options = read_options_default(); + read_options.set_read_tier(ReadTier::BlockCache); + read_options +} + +#[inline] +pub(crate) fn read_options_default() -> ReadOptions { + let mut read_options = ReadOptions::default(); + read_options.set_total_order_seek(true); + read_options +} + +#[inline] +pub(crate) fn write_options_default() -> WriteOptions { WriteOptions::default() } diff --git a/src/database/maps.rs b/src/database/maps.rs index d69cc7fd..e9b26818 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -2,103 +2,383 @@ use std::{collections::BTreeMap, sync::Arc}; use conduwuit::Result; -use crate::{Engine, Map}; +use crate::{ + engine::descriptor::{self, CacheDisp, Descriptor}, + Engine, Map, +}; -pub type Maps = BTreeMap; -pub(crate) type MapsVal = Arc; -pub(crate) type MapsKey = String; +pub(super) type Maps = BTreeMap; +pub(super) type MapsKey = &'static str; +pub(super) type MapsVal = Arc; -pub(crate) fn open(db: &Arc) -> Result { open_list(db, MAPS) } +pub(super) fn open(db: &Arc) -> Result { open_list(db, MAPS) } #[tracing::instrument(name = "maps", level = "debug", skip_all)] -pub(crate) fn open_list(db: &Arc, maps: &[&str]) -> Result { - Ok(maps - .iter() - .map(|&name| (name.to_owned(), Map::open(db, name).expect("valid column opened"))) - .collect::()) +pub(super) fn open_list(db: &Arc, maps: &[Descriptor]) -> Result { + maps.iter() + .map(|desc| Ok((desc.name, Map::open(db, desc.name)?))) + .collect() } -pub const MAPS: &[&str] = &[ - "alias_roomid", - "alias_userid", - "aliasid_alias", - "backupid_algorithm", - "backupid_etag", - "backupkeyid_backup", - "bannedroomids", - "disabledroomids", - "eventid_outlierpdu", - "eventid_pduid", - "eventid_shorteventid", - "global", - "id_appserviceregistrations", - "keychangeid_userid", - "keyid_key", - "lazyloadedids", - "mediaid_file", - "mediaid_user", - "onetimekeyid_onetimekeys", - "pduid_pdu", - "presenceid_presence", - "publicroomids", - "readreceiptid_readreceipt", - "referencedevents", - "roomid_invitedcount", - "roomid_inviteviaservers", - "roomid_joinedcount", - "roomid_pduleaves", - "roomid_shortroomid", - "roomid_shortstatehash", - "roomserverids", - "roomsynctoken_shortstatehash", - "roomuserdataid_accountdata", - "roomuserid_invitecount", - "roomuserid_joined", - "roomuserid_lastprivatereadupdate", - "roomuserid_leftcount", - "roomuserid_privateread", - "roomuseroncejoinedids", - "roomusertype_roomuserdataid", - "senderkey_pusher", - "server_signingkeys", - "servercurrentevent_data", - "servername_educount", - "servernameevent_data", - "serverroomids", - "shorteventid_authchain", - "shorteventid_eventid", - "shorteventid_shortstatehash", - "shortstatehash_statediff", - "shortstatekey_statekey", - "softfailedeventids", - "statehash_shortstatehash", - "statekey_shortstatekey", - "threadid_userids", - "todeviceid_events", - "tofrom_relation", - "token_userdeviceid", - "tokenids", - "url_previews", - "userdeviceid_metadata", - "userdeviceid_token", - "userdevicesessionid_uiaainfo", - "userdevicetxnid_response", - "userfilterid_filter", - "userid_avatarurl", - "userid_blurhash", - "userid_devicelistversion", - "userid_displayname", - "userid_lastonetimekeyupdate", - "userid_masterkeyid", - "userid_password", - "userid_presenceid", - "userid_selfsigningkeyid", - "userid_usersigningkeyid", - "useridprofilekey_value", - "openidtoken_expiresatuserid", - "userroomid_highlightcount", - "userroomid_invitestate", - "userroomid_joined", - "userroomid_leftstate", - "userroomid_notificationcount", +pub(super) static MAPS: &[Descriptor] = &[ + Descriptor { + name: "alias_roomid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "alias_userid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "aliasid_alias", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "backupid_algorithm", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "backupid_etag", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "backupkeyid_backup", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "bannedroomids", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "disabledroomids", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "eventid_outlierpdu", + cache_disp: CacheDisp::SharedWith("pduid_pdu"), + key_size_hint: Some(48), + val_size_hint: Some(1488), + ..descriptor::RANDOM + }, + Descriptor { + name: "eventid_pduid", + cache_disp: CacheDisp::Unique, + key_size_hint: Some(48), + val_size_hint: Some(16), + ..descriptor::RANDOM + }, + Descriptor { + name: "eventid_shorteventid", + cache_disp: CacheDisp::Unique, + key_size_hint: Some(48), + val_size_hint: Some(8), + ..descriptor::RANDOM + }, + Descriptor { + name: "global", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "id_appserviceregistrations", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "keychangeid_userid", + ..descriptor::RANDOM + }, + Descriptor { + name: "keyid_key", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "lazyloadedids", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "mediaid_file", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "mediaid_user", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "onetimekeyid_onetimekeys", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "pduid_pdu", + cache_disp: CacheDisp::SharedWith("eventid_outlierpdu"), + key_size_hint: Some(16), + val_size_hint: Some(1520), + ..descriptor::SEQUENTIAL + }, + Descriptor { + name: "presenceid_presence", + ..descriptor::SEQUENTIAL_SMALL + }, + Descriptor { + name: "publicroomids", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "readreceiptid_readreceipt", + ..descriptor::RANDOM + }, + Descriptor { + name: "referencedevents", + ..descriptor::RANDOM + }, + Descriptor { + name: "roomid_invitedcount", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomid_inviteviaservers", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomid_joinedcount", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomid_pduleaves", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomid_shortroomid", + val_size_hint: Some(8), + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomid_shortstatehash", + val_size_hint: Some(8), + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomserverids", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomsynctoken_shortstatehash", + val_size_hint: Some(8), + ..descriptor::SEQUENTIAL + }, + Descriptor { + name: "roomuserdataid_accountdata", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomuserid_invitecount", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomuserid_joined", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomuserid_lastprivatereadupdate", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomuserid_leftcount", + ..descriptor::RANDOM + }, + Descriptor { + name: "roomuserid_privateread", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "roomuseroncejoinedids", + ..descriptor::RANDOM + }, + Descriptor { + name: "roomusertype_roomuserdataid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "senderkey_pusher", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "server_signingkeys", + ..descriptor::RANDOM + }, + Descriptor { + name: "servercurrentevent_data", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "servername_educount", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "servernameevent_data", + cache_disp: CacheDisp::Unique, + val_size_hint: Some(128), + ..descriptor::RANDOM + }, + Descriptor { + name: "serverroomids", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "shorteventid_authchain", + cache_disp: CacheDisp::Unique, + key_size_hint: Some(8), + ..descriptor::SEQUENTIAL + }, + Descriptor { + name: "shorteventid_eventid", + cache_disp: CacheDisp::Unique, + key_size_hint: Some(8), + val_size_hint: Some(48), + ..descriptor::SEQUENTIAL_SMALL + }, + Descriptor { + name: "shorteventid_shortstatehash", + key_size_hint: Some(8), + val_size_hint: Some(8), + ..descriptor::SEQUENTIAL + }, + Descriptor { + name: "shortstatehash_statediff", + key_size_hint: Some(8), + ..descriptor::SEQUENTIAL_SMALL + }, + Descriptor { + name: "shortstatekey_statekey", + cache_disp: CacheDisp::Unique, + key_size_hint: Some(8), + val_size_hint: Some(1016), + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "softfailedeventids", + key_size_hint: Some(48), + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "statehash_shortstatehash", + val_size_hint: Some(8), + ..descriptor::RANDOM + }, + Descriptor { + name: "statekey_shortstatekey", + cache_disp: CacheDisp::Unique, + key_size_hint: Some(1016), + val_size_hint: Some(8), + ..descriptor::RANDOM + }, + Descriptor { + name: "threadid_userids", + ..descriptor::SEQUENTIAL_SMALL + }, + Descriptor { + name: "todeviceid_events", + ..descriptor::RANDOM + }, + Descriptor { + name: "tofrom_relation", + key_size_hint: Some(8), + val_size_hint: Some(8), + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "token_userdeviceid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { name: "tokenids", ..descriptor::RANDOM }, + Descriptor { + name: "url_previews", + ..descriptor::RANDOM + }, + Descriptor { + name: "userdeviceid_metadata", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userdeviceid_token", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userdevicesessionid_uiaainfo", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userdevicetxnid_response", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userfilterid_filter", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userid_avatarurl", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userid_blurhash", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userid_devicelistversion", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userid_displayname", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userid_lastonetimekeyupdate", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userid_masterkeyid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userid_password", + ..descriptor::RANDOM + }, + Descriptor { + name: "userid_presenceid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userid_selfsigningkeyid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userid_usersigningkeyid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "useridprofilekey_value", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "openidtoken_expiresatuserid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userroomid_highlightcount", + ..descriptor::RANDOM + }, + Descriptor { + name: "userroomid_invitestate", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "userroomid_joined", + ..descriptor::RANDOM + }, + Descriptor { + name: "userroomid_leftstate", + ..descriptor::RANDOM + }, + Descriptor { + name: "userroomid_notificationcount", + ..descriptor::RANDOM + }, ]; diff --git a/src/database/mod.rs b/src/database/mod.rs index bdb7d3ea..6e3f8c96 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,5 +1,11 @@ +extern crate conduwuit_core as conduwuit; +extern crate rust_rocksdb as rocksdb; + +conduwuit::mod_ctor! {} +conduwuit::mod_dtor! {} +conduwuit::rustc_flags_capture! {} + mod cork; -mod database; mod de; mod deserialized; mod engine; @@ -7,7 +13,6 @@ mod handle; pub mod keyval; mod map; pub mod maps; -mod opts; mod pool; mod ser; mod stream; @@ -16,16 +21,11 @@ mod tests; pub(crate) mod util; mod watchers; -pub(crate) use self::{ - engine::Engine, - util::{or_else, result}, -}; +use std::{ops::Index, sync::Arc}; -extern crate conduwuit_core as conduwuit; -extern crate rust_rocksdb as rocksdb; +use conduwuit::{err, Result, Server}; pub use self::{ - database::Database, de::{Ignore, IgnoreAll}, deserialized::Deserialized, handle::Handle, @@ -33,7 +33,60 @@ pub use self::{ map::Map, ser::{serialize, serialize_to, serialize_to_vec, Interfix, Json, Separator, SEP}, }; +pub(crate) use self::{ + engine::{context::Context, Engine}, + util::{or_else, result}, +}; +use crate::maps::{Maps, MapsKey, MapsVal}; -conduwuit::mod_ctor! {} -conduwuit::mod_dtor! {} -conduwuit::rustc_flags_capture! {} +pub struct Database { + maps: Maps, + pub db: Arc, + pub(crate) _ctx: Arc, +} + +impl Database { + /// Load an existing database or create a new one. + pub async fn open(server: &Arc) -> Result> { + let ctx = Context::new(server)?; + let db = Engine::open(ctx.clone(), maps::MAPS).await?; + Ok(Arc::new(Self { + maps: maps::open(&db)?, + db: db.clone(), + _ctx: ctx, + })) + } + + #[inline] + pub fn get(&self, name: &str) -> Result<&Arc> { + self.maps + .get(name) + .ok_or_else(|| err!(Request(NotFound("column not found")))) + } + + #[inline] + pub fn iter(&self) -> impl Iterator + Send + '_ { + self.maps.iter() + } + + #[inline] + pub fn keys(&self) -> impl Iterator + Send + '_ { self.maps.keys() } + + #[inline] + #[must_use] + pub fn is_read_only(&self) -> bool { self.db.is_read_only() } + + #[inline] + #[must_use] + pub fn is_secondary(&self) -> bool { self.db.is_secondary() } +} + +impl Index<&str> for Database { + type Output = Arc; + + fn index(&self, name: &str) -> &Self::Output { + self.maps + .get(name) + .expect("column in database does not exist") + } +} diff --git a/src/database/opts.rs b/src/database/opts.rs deleted file mode 100644 index 24128f14..00000000 --- a/src/database/opts.rs +++ /dev/null @@ -1,433 +0,0 @@ -use std::{cmp, collections::HashMap, convert::TryFrom}; - -use conduwuit::{err, utils, Config, Result}; -use rocksdb::{ - statistics::StatsLevel, BlockBasedOptions, Cache, DBCompactionStyle, DBCompressionType, - DBRecoveryMode, Env, LogLevel, LruCacheOptions, Options, UniversalCompactOptions, - UniversalCompactionStopStyle, -}; - -/// Create database-wide options suitable for opening the database. This also -/// sets our default column options in case of opening a column with the same -/// resulting value. Note that we require special per-column options on some -/// columns, therefor columns should only be opened after passing this result -/// through cf_options(). -pub(crate) fn db_options( - config: &Config, - env: &mut Env, - row_cache: &Cache, - col_cache: &Cache, -) -> Result { - const DEFAULT_STATS_LEVEL: StatsLevel = if cfg!(debug_assertions) { - StatsLevel::ExceptDetailedTimers - } else { - StatsLevel::DisableAll - }; - - let mut opts = Options::default(); - - // Logging - set_logging_defaults(&mut opts, config); - - // Processing - opts.set_max_background_jobs(num_threads::(config)?); - opts.set_max_subcompactions(num_threads::(config)?); - opts.set_avoid_unnecessary_blocking_io(true); - opts.set_max_file_opening_threads(0); - if config.rocksdb_compaction_prio_idle { - env.lower_thread_pool_cpu_priority(); - } - - // IO - opts.set_atomic_flush(true); - opts.set_manual_wal_flush(true); - opts.set_enable_pipelined_write(false); - if config.rocksdb_direct_io { - opts.set_use_direct_reads(true); - opts.set_use_direct_io_for_flush_and_compaction(true); - } - if config.rocksdb_optimize_for_spinning_disks { - // speeds up opening DB on hard drives - opts.set_skip_checking_sst_file_sizes_on_db_open(true); - opts.set_skip_stats_update_on_db_open(true); - //opts.set_max_file_opening_threads(threads.try_into().unwrap()); - } - if config.rocksdb_compaction_ioprio_idle { - env.lower_thread_pool_io_priority(); - } - - // Blocks - let mut table_opts = table_options(config); - table_opts.set_block_cache(col_cache); - opts.set_block_based_table_factory(&table_opts); - opts.set_row_cache(row_cache); - - // Buffers - opts.set_write_buffer_size(2 * 1024 * 1024); - opts.set_max_write_buffer_number(2); - opts.set_min_write_buffer_number(1); - - // Files - opts.set_table_cache_num_shard_bits(7); - opts.set_max_total_wal_size(96 * 1024 * 1024); - set_level_defaults(&mut opts, config); - - // Compression - set_compression_defaults(&mut opts, config); - - // Misc - opts.create_if_missing(true); - opts.set_disable_auto_compactions(!config.rocksdb_compaction); - - opts.set_statistics_level(match config.rocksdb_stats_level { - | 0 => StatsLevel::DisableAll, - | 1 => DEFAULT_STATS_LEVEL, - | 2 => StatsLevel::ExceptHistogramOrTimers, - | 3 => StatsLevel::ExceptTimers, - | 4 => StatsLevel::ExceptDetailedTimers, - | 5 => StatsLevel::ExceptTimeForMutex, - | 6_u8..=u8::MAX => StatsLevel::All, - }); - - // Default: https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes#ktoleratecorruptedtailrecords - // - // Unclean shutdowns of a Matrix homeserver are likely to be fine when - // recovered in this manner as it's likely any lost information will be - // restored via federation. - opts.set_wal_recovery_mode(match config.rocksdb_recovery_mode { - | 0 => DBRecoveryMode::AbsoluteConsistency, - | 1 => DBRecoveryMode::TolerateCorruptedTailRecords, - | 2 => DBRecoveryMode::PointInTime, - | 3 => DBRecoveryMode::SkipAnyCorruptedRecord, - | 4_u8..=u8::MAX => unimplemented!(), - }); - - // - // "We recommend to set track_and_verify_wals_in_manifest to true for - // production, it has been enabled in production for the entire database cluster - // serving the social graph for all Meta apps." - opts.set_track_and_verify_wals_in_manifest(true); - - opts.set_paranoid_checks(config.rocksdb_paranoid_file_checks); - - opts.set_env(env); - Ok(opts) -} - -/// Adjust options for the specific column by name. Provide the result of -/// db_options() as the argument to this function and use the return value in -/// the arguments to open the specific column. -pub(crate) fn cf_options( - cfg: &Config, - name: &str, - mut opts: Options, - cache: &mut HashMap, -) -> Result { - // Columns with non-default compaction options - match name { - | "backupid_algorithm" - | "backupid_etag" - | "backupkeyid_backup" - | "roomid_shortroomid" - | "shorteventid_shortstatehash" - | "shorteventid_eventid" - | "shortstatekey_statekey" - | "shortstatehash_statediff" - | "userdevicetxnid_response" - | "userfilterid_filter" => set_for_sequential_small_uc(&mut opts, cfg), - | &_ => {}, - } - - // Columns with non-default table/cache configs - match name { - | "shorteventid_eventid" => set_table_with_new_cache( - &mut opts, - cfg, - cache, - name, - cache_size(cfg, cfg.shorteventid_cache_capacity, 64)?, - ), - - | "eventid_shorteventid" => set_table_with_new_cache( - &mut opts, - cfg, - cache, - name, - cache_size(cfg, cfg.eventidshort_cache_capacity, 64)?, - ), - - | "eventid_pduid" => set_table_with_new_cache( - &mut opts, - cfg, - cache, - name, - cache_size(cfg, cfg.eventid_pdu_cache_capacity, 64)?, - ), - - | "shorteventid_authchain" => { - set_table_with_new_cache( - &mut opts, - cfg, - cache, - name, - cache_size(cfg, cfg.auth_chain_cache_capacity, 192)?, - ); - }, - - | "shortstatekey_statekey" => set_table_with_new_cache( - &mut opts, - cfg, - cache, - name, - cache_size(cfg, cfg.shortstatekey_cache_capacity, 1024)?, - ), - - | "statekey_shortstatekey" => set_table_with_new_cache( - &mut opts, - cfg, - cache, - name, - cache_size(cfg, cfg.statekeyshort_cache_capacity, 1024)?, - ), - - | "servernameevent_data" => set_table_with_new_cache( - &mut opts, - cfg, - cache, - name, - cache_size(cfg, cfg.servernameevent_data_cache_capacity, 128)?, /* Raw average - * value size = - * 102, key - * size = 34 */ - ), - - | "eventid_outlierpdu" => { - set_table_with_new_cache( - &mut opts, - cfg, - cache, - name, - cache_size(cfg, cfg.pdu_cache_capacity, 1536)?, - ); - }, - - | "pduid_pdu" => { - set_table_with_shared_cache(&mut opts, cfg, cache, name, "eventid_outlierpdu"); - }, - - | &_ => {}, - } - - Ok(opts) -} - -fn set_logging_defaults(opts: &mut Options, config: &Config) { - let rocksdb_log_level = match config.rocksdb_log_level.as_ref() { - | "debug" => LogLevel::Debug, - | "info" => LogLevel::Info, - | "warn" => LogLevel::Warn, - | "fatal" => LogLevel::Fatal, - | _ => LogLevel::Error, - }; - - opts.set_log_level(rocksdb_log_level); - opts.set_max_log_file_size(config.rocksdb_max_log_file_size); - opts.set_log_file_time_to_roll(config.rocksdb_log_time_to_roll); - opts.set_keep_log_file_num(config.rocksdb_max_log_files); - opts.set_stats_dump_period_sec(0); - - if config.rocksdb_log_stderr { - opts.set_stderr_logger(rocksdb_log_level, "rocksdb"); - } else { - opts.set_callback_logger(rocksdb_log_level, &super::engine::handle_log); - } -} - -fn set_compression_defaults(opts: &mut Options, config: &Config) { - let rocksdb_compression_algo = match config.rocksdb_compression_algo.as_ref() { - | "snappy" => DBCompressionType::Snappy, - | "zlib" => DBCompressionType::Zlib, - | "bz2" => DBCompressionType::Bz2, - | "lz4" => DBCompressionType::Lz4, - | "lz4hc" => DBCompressionType::Lz4hc, - | "none" => DBCompressionType::None, - | _ => DBCompressionType::Zstd, - }; - - if config.rocksdb_bottommost_compression { - opts.set_bottommost_compression_type(rocksdb_compression_algo); - opts.set_bottommost_zstd_max_train_bytes(0, true); - - // -14 w_bits is only read by zlib. - opts.set_bottommost_compression_options( - -14, - config.rocksdb_bottommost_compression_level, - 0, - 0, - true, - ); - } - - // -14 w_bits is only read by zlib. - opts.set_compression_options(-14, config.rocksdb_compression_level, 0, 0); - opts.set_compression_type(rocksdb_compression_algo); -} - -#[allow(dead_code)] -fn set_for_random_small_uc(opts: &mut Options, config: &Config) { - let uco = uc_options(config); - set_for_random_small(opts, config); - opts.set_universal_compaction_options(&uco); - opts.set_compaction_style(DBCompactionStyle::Universal); -} - -fn set_for_sequential_small_uc(opts: &mut Options, config: &Config) { - let uco = uc_options(config); - set_for_sequential_small(opts, config); - opts.set_universal_compaction_options(&uco); - opts.set_compaction_style(DBCompactionStyle::Universal); -} - -#[allow(dead_code)] -fn set_for_random_small(opts: &mut Options, config: &Config) { - set_for_random(opts, config); - - opts.set_write_buffer_size(1024 * 128); - opts.set_target_file_size_base(1024 * 128); - opts.set_target_file_size_multiplier(2); - opts.set_max_bytes_for_level_base(1024 * 512); - opts.set_max_bytes_for_level_multiplier(2.0); -} - -fn set_for_sequential_small(opts: &mut Options, config: &Config) { - set_for_sequential(opts, config); - - opts.set_write_buffer_size(1024 * 512); - opts.set_target_file_size_base(1024 * 512); - opts.set_target_file_size_multiplier(2); - opts.set_max_bytes_for_level_base(1024 * 1024); - opts.set_max_bytes_for_level_multiplier(2.0); -} - -fn set_for_random(opts: &mut Options, config: &Config) { - set_level_defaults(opts, config); - - let pri = "compaction_pri=kOldestSmallestSeqFirst"; - opts.set_options_from_string(pri) - .expect("set compaction priority string"); - - opts.set_max_bytes_for_level_base(8 * 1024 * 1024); - opts.set_max_bytes_for_level_multiplier(1.0); - opts.set_max_bytes_for_level_multiplier_additional(&[0, 1, 1, 3, 7, 15, 31]); -} - -fn set_for_sequential(opts: &mut Options, config: &Config) { - set_level_defaults(opts, config); - - let pri = "compaction_pri=kOldestLargestSeqFirst"; - opts.set_options_from_string(pri) - .expect("set compaction priority string"); - - opts.set_target_file_size_base(2 * 1024 * 1024); - opts.set_target_file_size_multiplier(2); - - opts.set_max_bytes_for_level_base(32 * 1024 * 1024); - opts.set_max_bytes_for_level_multiplier(1.0); - opts.set_max_bytes_for_level_multiplier_additional(&[0, 1, 1, 3, 7, 15, 31]); -} - -fn set_level_defaults(opts: &mut Options, _config: &Config) { - opts.set_level_zero_file_num_compaction_trigger(2); - - opts.set_target_file_size_base(1024 * 1024); - opts.set_target_file_size_multiplier(2); - - opts.set_level_compaction_dynamic_level_bytes(false); - opts.set_max_bytes_for_level_base(16 * 1024 * 1024); - opts.set_max_bytes_for_level_multiplier(2.0); - - opts.set_ttl(21 * 24 * 60 * 60); -} - -fn uc_options(_config: &Config) -> UniversalCompactOptions { - let mut opts = UniversalCompactOptions::default(); - - opts.set_stop_style(UniversalCompactionStopStyle::Total); - opts.set_max_size_amplification_percent(10000); - opts.set_compression_size_percent(-1); - opts.set_size_ratio(1); - - opts.set_min_merge_width(2); - opts.set_max_merge_width(16); - - opts -} - -fn set_table_with_new_cache( - opts: &mut Options, - config: &Config, - caches: &mut HashMap, - name: &str, - size: usize, -) { - let mut cache_opts = LruCacheOptions::default(); - cache_opts.set_capacity(size); - cache_opts.set_num_shard_bits(7); - - let cache = Cache::new_lru_cache_opts(&cache_opts); - caches.insert(name.into(), cache); - - set_table_with_shared_cache(opts, config, caches, name, name); -} - -fn set_table_with_shared_cache( - opts: &mut Options, - config: &Config, - cache: &HashMap, - _name: &str, - cache_name: &str, -) { - let mut table = table_options(config); - table.set_block_cache( - cache - .get(cache_name) - .expect("existing cache to share with this column"), - ); - - opts.set_block_based_table_factory(&table); -} - -fn cache_size(config: &Config, base_size: u32, entity_size: usize) -> Result { - let ents = f64::from(base_size) * config.cache_capacity_modifier; - - #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] - (ents as usize) - .checked_mul(entity_size) - .ok_or_else(|| err!(Config("cache_capacity_modifier", "Cache size is too large."))) -} - -fn table_options(_config: &Config) -> BlockBasedOptions { - let mut opts = BlockBasedOptions::default(); - - opts.set_block_size(4 * 1024); - opts.set_metadata_block_size(4 * 1024); - - opts.set_use_delta_encoding(false); - opts.set_optimize_filters_for_memory(true); - opts.set_cache_index_and_filter_blocks(true); - opts.set_pin_top_level_index_and_filter(true); - - opts -} - -fn num_threads>(config: &Config) -> Result { - const MIN_PARALLELISM: usize = 2; - - let requested = if config.rocksdb_parallelism_threads != 0 { - config.rocksdb_parallelism_threads - } else { - utils::available_parallelism() - }; - - utils::math::try_into::(cmp::max(MIN_PARALLELISM, requested)) -} diff --git a/src/service/migrations.rs b/src/service/migrations.rs index adf75c0b..c42c0324 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -379,7 +379,7 @@ async fn fix_bad_double_separator_in_state_cache(services: &Services) -> Result< }) .await; - db.db.cleanup()?; + db.db.sort()?; db["global"].insert(b"fix_bad_double_separator_in_state_cache", []); info!("Finished fixing"); @@ -465,7 +465,7 @@ async fn retroactively_fix_bad_data_from_roomuserid_joined(services: &Services) .await; } - db.db.cleanup()?; + db.db.sort()?; db["global"].insert(b"retroactively_fix_bad_data_from_roomuserid_joined", []); info!("Finished fixing"); @@ -511,7 +511,7 @@ async fn fix_referencedevents_missing_sep(services: &Services) -> Result { info!(?total, ?fixed, "Fixed missing record separators in 'referencedevents'."); db["global"].insert(b"fix_referencedevents_missing_sep", []); - db.db.cleanup() + db.db.sort() } async fn fix_readreceiptid_readreceipt_duplicates(services: &Services) -> Result { @@ -561,5 +561,5 @@ async fn fix_readreceiptid_readreceipt_duplicates(services: &Services) -> Result info!(?total, ?fixed, "Fixed undeleted entries in readreceiptid_readreceipt."); db["global"].insert(b"fix_readreceiptid_readreceipt_duplicates", []); - db.db.cleanup() + db.db.sort() } diff --git a/src/service/services.rs b/src/service/services.rs index 9f9d10f5..c955834e 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -124,7 +124,7 @@ impl Services { .await?; // set the server user as online - if self.server.config.allow_local_presence { + if self.server.config.allow_local_presence && !self.db.is_read_only() { _ = self .presence .ping_presence(&self.globals.server_user, &ruma::presence::PresenceState::Online) @@ -139,7 +139,7 @@ impl Services { info!("Shutting down services..."); // set the server user as offline - if self.server.config.allow_local_presence { + if self.server.config.allow_local_presence && !self.db.is_read_only() { _ = self .presence .ping_presence(&self.globals.server_user, &ruma::presence::PresenceState::Offline) From 16fa2eca87ef2b3b006a8467dd620ef3cce240a0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 9 Jan 2025 19:55:25 +0000 Subject: [PATCH 1331/2091] add conf item for write buffer size Signed-off-by: Jason Volk --- conduwuit-example.toml | 42 ++++++++++++++++++---------- src/core/config/mod.rs | 50 +++++++++++++++++++++++----------- src/database/engine/cf_opts.rs | 26 ++++++++++++++---- src/database/engine/db_opts.rs | 11 ++++++-- 4 files changed, 92 insertions(+), 37 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index e2ed5daa..9eefedbb 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -100,20 +100,6 @@ # #database_backups_to_keep = 1 -# Set this to any float value in megabytes for conduwuit to tell the -# database engine that this much memory is available for database-related -# caches. -# -# May be useful if you have significant memory to spare to increase -# performance. -# -# Similar to the individual LRU caches, this is scaled up with your CPU -# core count. -# -# This defaults to 128.0 + (64.0 * CPU core count). -# -#db_cache_capacity_mb = varies by system - # Text which will be added to the end of the user's displayname upon # registration with a space before the text. In Conduit, this was the # lightning bolt emoji. @@ -149,6 +135,34 @@ # #cache_capacity_modifier = 1.0 +# Set this to any float value in megabytes for conduwuit to tell the +# database engine that this much memory is available for database read +# caches. +# +# May be useful if you have significant memory to spare to increase +# performance. +# +# Similar to the individual LRU caches, this is scaled up with your CPU +# core count. +# +# This defaults to 128.0 + (64.0 * CPU core count). +# +#db_cache_capacity_mb = varies by system + +# Set this to any float value in megabytes for conduwuit to tell the +# database engine that this much memory is available for database write +# caches. +# +# May be useful if you have significant memory to spare to increase +# performance. +# +# Similar to the individual LRU caches, this is scaled up with your CPU +# core count. +# +# This defaults to 48.0 + (4.0 * CPU core count). +# +#db_write_buffer_capacity_mb = varies by system + # This item is undocumented. Please contribute documentation for it. # #pdu_cache_capacity = varies by system diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 56580fda..97ecbeaf 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -147,22 +147,6 @@ pub struct Config { #[serde(default = "default_database_backups_to_keep")] pub database_backups_to_keep: i16, - /// Set this to any float value in megabytes for conduwuit to tell the - /// database engine that this much memory is available for database-related - /// caches. - /// - /// May be useful if you have significant memory to spare to increase - /// performance. - /// - /// Similar to the individual LRU caches, this is scaled up with your CPU - /// core count. - /// - /// This defaults to 128.0 + (64.0 * CPU core count). - /// - /// default: varies by system - #[serde(default = "default_db_cache_capacity_mb")] - pub db_cache_capacity_mb: f64, - /// Text which will be added to the end of the user's displayname upon /// registration with a space before the text. In Conduit, this was the /// lightning bolt emoji. @@ -205,6 +189,38 @@ pub struct Config { )] pub cache_capacity_modifier: f64, + /// Set this to any float value in megabytes for conduwuit to tell the + /// database engine that this much memory is available for database read + /// caches. + /// + /// May be useful if you have significant memory to spare to increase + /// performance. + /// + /// Similar to the individual LRU caches, this is scaled up with your CPU + /// core count. + /// + /// This defaults to 128.0 + (64.0 * CPU core count). + /// + /// default: varies by system + #[serde(default = "default_db_cache_capacity_mb")] + pub db_cache_capacity_mb: f64, + + /// Set this to any float value in megabytes for conduwuit to tell the + /// database engine that this much memory is available for database write + /// caches. + /// + /// May be useful if you have significant memory to spare to increase + /// performance. + /// + /// Similar to the individual LRU caches, this is scaled up with your CPU + /// core count. + /// + /// This defaults to 48.0 + (4.0 * CPU core count). + /// + /// default: varies by system + #[serde(default = "default_db_write_buffer_capacity_mb")] + pub db_write_buffer_capacity_mb: f64, + /// default: varies by system #[serde(default = "default_pdu_cache_capacity")] pub pdu_cache_capacity: u32, @@ -2233,6 +2249,8 @@ fn default_unix_socket_perms() -> u32 { 660 } fn default_database_backups_to_keep() -> i16 { 1 } +fn default_db_write_buffer_capacity_mb() -> f64 { 48.0 + parallelism_scaled_f64(4.0) } + fn default_db_cache_capacity_mb() -> f64 { 128.0 + parallelism_scaled_f64(64.0) } fn default_pdu_cache_capacity() -> u32 { parallelism_scaled_u32(10_000).saturating_add(100_000) } diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 98d74044..006d36fe 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -10,7 +10,7 @@ use rocksdb::{ }; use super::descriptor::{CacheDisp, Descriptor}; -use crate::Context; +use crate::{util::map_err, Context}; /// Adjust options for the specific column by name. Provide the result of /// db_options() as the argument to this function and use the return value in @@ -28,7 +28,7 @@ fn descriptor_cf_options( cache: Option<&Cache>, ) -> Result { set_compression(&mut desc, config); - set_table_options(&mut opts, &desc, cache); + set_table_options(&mut opts, &desc, cache)?; opts.set_min_write_buffer_number(1); opts.set_max_write_buffer_number(2); @@ -65,10 +65,13 @@ fn descriptor_cf_options( ); } + opts.set_options_from_string("{{arena_block_size=2097152;}}") + .map_err(map_err)?; + Ok(opts) } -fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache>) { +fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache>) -> Result { let mut table = table_options(desc); if let Some(cache) = cache { table.set_block_cache(cache); @@ -76,7 +79,15 @@ fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache table.disable_cache(); } + opts.set_options_from_string( + "{{block_based_table_factory={num_file_reads_for_auto_readahead=0;\ + max_auto_readahead_size=524288;initial_auto_readahead_size=16384}}}", + ) + .map_err(map_err)?; + opts.set_block_based_table_factory(&table); + + Ok(()) } fn set_compression(desc: &mut Descriptor, config: &Config) { @@ -121,6 +132,7 @@ fn table_options(desc: &Descriptor) -> BlockBasedOptions { opts.set_unpartitioned_pinning_tier(BlockBasedPinningTier::None); opts.set_top_level_index_pinning_tier(BlockBasedPinningTier::None); + opts.set_partition_filters(true); opts.set_use_delta_encoding(false); opts.set_index_type(BlockBasedIndexType::TwoLevelIndexSearch); opts.set_data_block_index_type( @@ -203,9 +215,13 @@ fn get_cache(ctx: &Context, desc: &Descriptor) -> Option { } } -#[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] pub(crate) fn cache_size(config: &Config, base_size: u32, entity_size: usize) -> usize { - let ents = f64::from(base_size) * config.cache_capacity_modifier; + cache_size_f64(config, f64::from(base_size), entity_size) +} + +#[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] +pub(crate) fn cache_size_f64(config: &Config, base_size: f64, entity_size: usize) -> usize { + let ents = base_size * config.cache_capacity_modifier; (ents as usize) .checked_mul(entity_size) diff --git a/src/database/engine/db_opts.rs b/src/database/engine/db_opts.rs index 211265de..26f53825 100644 --- a/src/database/engine/db_opts.rs +++ b/src/database/engine/db_opts.rs @@ -3,7 +3,7 @@ use std::{cmp, convert::TryFrom}; use conduwuit::{utils, Config, Result}; use rocksdb::{statistics::StatsLevel, Cache, DBRecoveryMode, Env, LogLevel, Options}; -use super::{cf_opts::cache_size, logger::handle as handle_log}; +use super::{cf_opts::cache_size_f64, logger::handle as handle_log}; /// Create database-wide options suitable for opening the database. This also /// sets our default column options in case of opening a column with the same @@ -41,16 +41,23 @@ pub(crate) fn db_options(config: &Config, env: &Env, row_cache: &Cache) -> Resul opts.set_skip_checking_sst_file_sizes_on_db_open(true); opts.set_skip_stats_update_on_db_open(true); //opts.set_max_file_opening_threads(threads.try_into().unwrap()); + } else { + opts.set_compaction_readahead_size(1024 * 512); } // Blocks opts.set_row_cache(row_cache); + opts.set_db_write_buffer_size(cache_size_f64( + config, + config.db_write_buffer_capacity_mb, + 1_048_576, + )); // Files opts.set_table_cache_num_shard_bits(7); opts.set_wal_size_limit_mb(1024 * 1024 * 1024); opts.set_max_total_wal_size(1024 * 1024 * 512); - opts.set_db_write_buffer_size(cache_size(config, 1024 * 1024 * 32, 1)); + opts.set_writable_file_max_buffer_size(1024 * 1024 * 2); // Misc opts.set_disable_auto_compactions(!config.rocksdb_compaction); From 66231676f15eb8400bac310f837cc23df0ef735a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 10 Jan 2025 22:23:02 +0000 Subject: [PATCH 1332/2091] gracefully ignore unknown columns; add dropped flag in descriptor Signed-off-by: Jason Volk --- src/database/engine/cf_opts.rs | 11 ++++--- src/database/engine/descriptor.rs | 2 ++ src/database/engine/open.rs | 54 +++++++++++++++++++------------ 3 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 006d36fe..a68eb8b6 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -72,7 +72,7 @@ fn descriptor_cf_options( } fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache>) -> Result { - let mut table = table_options(desc); + let mut table = table_options(desc, cache.is_some()); if let Some(cache) = cache { table.set_block_cache(cache); } else { @@ -119,13 +119,13 @@ fn uc_options(desc: &Descriptor) -> UniversalCompactOptions { opts } -fn table_options(desc: &Descriptor) -> BlockBasedOptions { +fn table_options(desc: &Descriptor, has_cache: bool) -> BlockBasedOptions { let mut opts = BlockBasedOptions::default(); opts.set_block_size(desc.block_size); opts.set_metadata_block_size(desc.index_size); - opts.set_cache_index_and_filter_blocks(true); + opts.set_cache_index_and_filter_blocks(has_cache); opts.set_pin_top_level_index_and_filter(false); opts.set_pin_l0_filter_and_index_blocks_in_cache(false); opts.set_partition_pinning_tier(BlockBasedPinningTier::None); @@ -144,10 +144,13 @@ fn table_options(desc: &Descriptor) -> BlockBasedOptions { } fn get_cache(ctx: &Context, desc: &Descriptor) -> Option { - let config = &ctx.server.config; + if desc.dropped { + return None; + } // Some cache capacities are overriden by server config in a strange but // legacy-compat way + let config = &ctx.server.config; let cap = match desc.name { | "eventid_pduid" => Some(config.eventid_pdu_cache_capacity), | "eventid_shorteventid" => Some(config.eventidshort_cache_capacity), diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index f0fd83f1..9cf57d8f 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -14,6 +14,7 @@ pub(crate) enum CacheDisp { #[derive(Debug, Clone)] pub(crate) struct Descriptor { pub(crate) name: &'static str, + pub(crate) dropped: bool, pub(crate) cache_disp: CacheDisp, pub(crate) key_size_hint: Option, pub(crate) val_size_hint: Option, @@ -39,6 +40,7 @@ pub(crate) struct Descriptor { pub(crate) static BASE: Descriptor = Descriptor { name: EMPTY, + dropped: false, cache_disp: CacheDisp::Shared, key_size_hint: None, val_size_hint: None, diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs index 9999296b..6a801878 100644 --- a/src/database/engine/open.rs +++ b/src/database/engine/open.rs @@ -4,11 +4,15 @@ use std::{ sync::{atomic::AtomicU32, Arc}, }; -use conduwuit::{debug, debug_warn, implement, info, warn, Result}; +use conduwuit::{debug, implement, info, warn, Result}; use rocksdb::{ColumnFamilyDescriptor, Options}; use super::{ - cf_opts::cf_options, db_opts::db_options, descriptor::Descriptor, repair::repair, Db, Engine, + cf_opts::cf_options, + db_opts::db_options, + descriptor::{self, Descriptor}, + repair::repair, + Db, Engine, }; use crate::{or_else, Context}; @@ -72,38 +76,46 @@ fn configure_cfds( let config = &server.config; let path = &config.database_path; let existing = Self::discover_cfs(path, db_opts); - debug!( - "Found {} existing columns; have {} described columns", - existing.len(), - desc.len() - ); - existing + let creating = desc.iter().filter(|desc| !existing.contains(desc.name)); + + let missing = existing .iter() .filter(|&name| name != "default") - .filter(|&name| !desc.iter().any(|desc| desc.name == name)) - .for_each(|name| { - debug_warn!("Found unknown column {name:?} in database which will not be opened."); - }); + .filter(|&name| !desc.iter().any(|desc| desc.name == name)); - desc.iter() - .filter(|desc| !existing.contains(desc.name)) - .for_each(|desc| { - debug!( - "Creating new column {:?} which was not found in the existing database.", - desc.name, - ); - }); + debug!( + existing = existing.len(), + described = desc.len(), + missing = missing.clone().count(), + creating = creating.clone().count(), + "Discovered database columns" + ); + + missing.clone().for_each(|name| { + debug!("Found unrecognized column {name:?} in existing database."); + }); + + creating.map(|desc| desc.name).for_each(|name| { + debug!("Creating new column {name:?} not previously found in existing database."); + }); + + let missing_descriptors = missing + .clone() + .map(|_| Descriptor { dropped: true, ..descriptor::BASE }); let cfopts: Vec<_> = desc .iter() - .map(|desc| cf_options(ctx, db_opts.clone(), desc)) + .cloned() + .chain(missing_descriptors) + .map(|ref desc| cf_options(ctx, db_opts.clone(), desc)) .collect::>()?; let cfds: Vec<_> = desc .iter() .map(|desc| desc.name) .map(ToOwned::to_owned) + .chain(missing.cloned()) .zip(cfopts.into_iter()) .map(|(name, opts)| ColumnFamilyDescriptor::new(name, opts)) .collect(); From 53d03bbb1f5001695d69f83047ead96d2079a36a Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 10 Jan 2025 10:25:07 -0500 Subject: [PATCH 1333/2091] gate sd_notify to linux target_os only Signed-off-by: strawberry --- src/router/Cargo.toml | 2 +- src/router/run.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 1623590b..51e15aed 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -80,7 +80,7 @@ tower.workspace = true tower-http.workspace = true tracing.workspace = true -[target.'cfg(unix)'.dependencies] +[target.'cfg(all(unix, target_os = "linux"))'.dependencies] sd-notify.workspace = true sd-notify.optional = true diff --git a/src/router/run.rs b/src/router/run.rs index 1b4d7437..95d12559 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -63,7 +63,7 @@ pub(crate) async fn start(server: Arc) -> Result> { let services = Services::build(server).await?.start().await?; - #[cfg(feature = "systemd")] + #[cfg(all(feature = "systemd", target_os = "linux"))] sd_notify::notify(true, &[sd_notify::NotifyState::Ready]) .expect("failed to notify systemd of ready state"); @@ -99,7 +99,7 @@ pub(crate) async fn stop(services: Arc) -> Result<()> { ); } - #[cfg(feature = "systemd")] + #[cfg(all(feature = "systemd", target_os = "linux"))] sd_notify::notify(true, &[sd_notify::NotifyState::Stopping]) .expect("failed to notify systemd of stopping state"); From 4c2999ccd15506a4acbc948d62c7ec0c03d46167 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 10 Jan 2025 10:46:32 -0500 Subject: [PATCH 1334/2091] gate libloading to conduwuit_mods feature and cfg only Signed-off-by: strawberry --- flake.nix | 22 ++++++++++++++++++++++ src/core/Cargo.toml | 4 ++++ src/core/mod.rs | 2 +- src/core/mods/mod.rs | 2 +- src/core/server.rs | 2 +- src/main/Cargo.toml | 3 +++ src/main/main.rs | 4 ++-- src/main/mods.rs | 2 +- src/main/server.rs | 4 ++-- src/main/signal.rs | 2 +- 10 files changed, 38 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index d8ad47a8..e3497d85 100644 --- a/flake.nix +++ b/flake.nix @@ -212,6 +212,8 @@ # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; all-features-debug = scopeHost.main.override { @@ -224,6 +226,8 @@ "hardened_malloc" # dont include experimental features "experimental" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; hmalloc = scopeHost.main.override { features = ["hardened_malloc"]; }; @@ -241,6 +245,8 @@ # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; }; @@ -255,6 +261,8 @@ "hardened_malloc" # dont include experimental features "experimental" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; }; @@ -330,6 +338,8 @@ # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; } @@ -349,6 +359,8 @@ # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); }; @@ -367,6 +379,8 @@ "hardened_malloc" # dont include experimental features "experimental" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; } @@ -423,6 +437,8 @@ # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; }; @@ -444,6 +460,8 @@ # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); }; @@ -464,6 +482,8 @@ "hardened_malloc" # dont include experimental features "experimental" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; }; @@ -510,6 +530,8 @@ # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; })); diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 4a9cc462..c716e9c2 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -50,6 +50,9 @@ zstd_compression = [ ] perf_measurements = [] sentry_telemetry = [] +conduwuit_mods = [ + "dep:libloading" +] [dependencies] argon2.workspace = true @@ -75,6 +78,7 @@ ipaddress.workspace = true itertools.workspace = true libc.workspace = true libloading.workspace = true +libloading.optional = true log.workspace = true num-traits.workspace = true rand.workspace = true diff --git a/src/core/mod.rs b/src/core/mod.rs index 87cb58ae..1416ed9e 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -25,7 +25,7 @@ pub use crate as conduwuit_core; rustc_flags_capture! {} -#[cfg(not(conduwuit_mods))] +#[cfg(any(not(conduwuit_mods), not(feature = "conduwuit_mods")))] pub mod mods { #[macro_export] macro_rules! mod_ctor { diff --git a/src/core/mods/mod.rs b/src/core/mods/mod.rs index ac0c333b..b8f06f29 100644 --- a/src/core/mods/mod.rs +++ b/src/core/mods/mod.rs @@ -1,4 +1,4 @@ -#![cfg(conduwuit_mods)] +#![cfg(all(conduwuit_mods, feature = "conduwuit_mods"))] pub(crate) use libloading::os::unix::{Library, Symbol}; diff --git a/src/core/server.rs b/src/core/server.rs index 8a4d9f66..948eea36 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -59,7 +59,7 @@ impl Server { } pub fn reload(&self) -> Result<()> { - if cfg!(not(conduwuit_mods)) { + if cfg!(any(not(conduwuit_mods), not(feature = "conduwuit_mods"))) { return Err!("Reloading not enabled"); } diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 38eb7188..baf5336f 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -135,6 +135,9 @@ zstd_compression = [ "conduwuit-database/zstd_compression", "conduwuit-router/zstd_compression", ] +conduwuit_mods = [ + "conduwuit-core/conduwuit_mods", +] [dependencies] conduwuit-admin.workspace = true diff --git a/src/main/main.rs b/src/main/main.rs index e7aaf3fc..dacc2a2e 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -37,7 +37,7 @@ fn main() -> Result<(), Error> { /// Operate the server normally in release-mode static builds. This will start, /// run and stop the server within the asynchronous runtime. -#[cfg(not(conduwuit_mods))] +#[cfg(any(not(conduwuit_mods), not(feature = "conduwuit_mods")))] #[tracing::instrument( name = "main", parent = None, @@ -89,7 +89,7 @@ async fn async_main(server: &Arc) -> Result<(), Error> { /// Operate the server in developer-mode dynamic builds. This will start, run, /// and hot-reload portions of the server as-needed before returning for an /// actual shutdown. This is not available in release-mode or static builds. -#[cfg(conduwuit_mods)] +#[cfg(all(conduwuit_mods, feature = "conduwuit_mods"))] async fn async_main(server: &Arc) -> Result<(), Error> { let mut starts = true; let mut reloads = true; diff --git a/src/main/mods.rs b/src/main/mods.rs index ca799b90..9ab36e6c 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -1,4 +1,4 @@ -#![cfg(conduwuit_mods)] +#![cfg(all(conduwuit_mods, feature = "conduwuit_mods"))] #[unsafe(no_link)] extern crate conduwuit_service; diff --git a/src/main/server.rs b/src/main/server.rs index a81b708d..359a029c 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -23,7 +23,7 @@ pub(crate) struct Server { #[cfg(feature = "sentry_telemetry")] _sentry_guard: Option<::sentry::ClientInitGuard>, - #[cfg(conduwuit_mods)] + #[cfg(all(conduwuit_mods, feature = "conduwuit_mods"))] // Module instances; TODO: move to mods::loaded mgmt vector pub(crate) mods: tokio::sync::RwLock>, } @@ -75,7 +75,7 @@ impl Server { #[cfg(feature = "sentry_telemetry")] _sentry_guard: sentry_guard, - #[cfg(conduwuit_mods)] + #[cfg(all(conduwuit_mods, feature = "conduwuit_mods"))] mods: tokio::sync::RwLock::new(Vec::new()), })) } diff --git a/src/main/signal.rs b/src/main/signal.rs index 0f541099..cecb718b 100644 --- a/src/main/signal.rs +++ b/src/main/signal.rs @@ -12,7 +12,7 @@ pub(super) async fn signal(server: Arc) { use unix::SignalKind; const CONSOLE: bool = cfg!(feature = "console"); - const RELOADING: bool = cfg!(all(conduwuit_mods, not(CONSOLE))); + const RELOADING: bool = cfg!(all(conduwuit_mods, feature = "conduwuit_mods", not(CONSOLE))); let mut quit = unix::signal(SignalKind::quit()).expect("SIGQUIT handler"); let mut term = unix::signal(SignalKind::terminate()).expect("SIGTERM handler"); From 5b6279b1c514ea65e00399c26bb7cdd007061e6d Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 10 Jan 2025 10:51:39 -0500 Subject: [PATCH 1335/2091] ci: require docker publishing to pass tests Signed-off-by: strawberry --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a4df488..3ccbf5d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -733,7 +733,7 @@ jobs: docker: name: Docker publish runs-on: ubuntu-24.04 - needs: [build, variables] + needs: [build, variables, tests] permissions: packages: write contents: read From 1852eeebf22d2bedfeff63b9c205640f5832ca49 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 10 Jan 2025 22:45:48 -0500 Subject: [PATCH 1336/2091] disable more unnecessary features in various build outputs Signed-off-by: strawberry --- flake.nix | 146 ++++++++++++++++++++------------ nix/pkgs/complement/default.nix | 10 +++ nix/pkgs/main/default.nix | 14 ++- 3 files changed, 115 insertions(+), 55 deletions(-) diff --git a/flake.nix b/flake.nix index e3497d85..fb40cae7 100644 --- a/flake.nix +++ b/flake.nix @@ -191,27 +191,57 @@ in { packages = { - default = scopeHost.main; - default-debug = scopeHost.main.override { - profile = "dev"; - # debug build users expect full logs - disable_release_max_log_level = true; - }; - default-test = scopeHost.main.override { - profile = "test"; - disable_release_max_log_level = true; - }; - all-features = scopeHost.main.override { - all_features = true; + default = scopeHost.main.override { disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" # dont include experimental features "experimental" # jemalloc profiling/stats features are expensive and shouldn't # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" + ]; + }; + default-debug = scopeHost.main.override { + profile = "dev"; + # debug build users expect full logs + disable_release_max_log_level = true; + disable_features = [ + # dont include experimental features + "experimental" + # this is non-functional on nix for some reason + "hardened_malloc" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" + ]; + }; + # just a test profile used for things like CI and complement + default-test = scopeHost.main.override { + profile = "test"; + disable_release_max_log_level = true; + disable_features = [ + # dont include experimental features + "experimental" + # this is non-functional on nix for some reason + "hardened_malloc" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" + ]; + }; + all-features = scopeHost.main.override { + all_features = true; + disable_features = [ + # dont include experimental features + "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" # conduwuit_mods is a development-only hot reload feature "conduwuit_mods" ]; @@ -222,10 +252,10 @@ # debug build users expect full logs disable_release_max_log_level = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" # dont include experimental features "experimental" + # this is non-functional on nix for some reason + "hardened_malloc" # conduwuit_mods is a development-only hot reload feature "conduwuit_mods" ]; @@ -237,14 +267,14 @@ main = scopeHost.main.override { all_features = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" # dont include experimental features "experimental" # jemalloc profiling/stats features are expensive and shouldn't # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" # conduwuit_mods is a development-only hot reload feature "conduwuit_mods" ]; @@ -257,10 +287,10 @@ # debug build users expect full logs disable_release_max_log_level = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" # dont include experimental features "experimental" + # this is non-functional on nix for some reason + "hardened_malloc" # conduwuit_mods is a development-only hot reload feature "conduwuit_mods" ]; @@ -321,6 +351,14 @@ value = scopeCrossStatic.main.override { profile = "test"; disable_release_max_log_level = true; + disable_features = [ + # dont include experimental features + "experimental" + # this is non-functional on nix for some reason + "hardened_malloc" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" + ]; }; } @@ -330,14 +368,14 @@ value = scopeCrossStatic.main.override { all_features = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" # dont include experimental features "experimental" # jemalloc profiling/stats features are expensive and shouldn't # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" # conduwuit_mods is a development-only hot reload feature "conduwuit_mods" ]; @@ -351,14 +389,14 @@ value = scopeCrossStatic.main.override { all_features = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" # dont include experimental features "experimental" # jemalloc profiling/stats features are expensive and shouldn't # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" # conduwuit_mods is a development-only hot reload feature "conduwuit_mods" ]; @@ -375,10 +413,10 @@ # debug build users expect full logs disable_release_max_log_level = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" # dont include experimental features "experimental" + # this is non-functional on nix for some reason + "hardened_malloc" # conduwuit_mods is a development-only hot reload feature "conduwuit_mods" ]; @@ -429,16 +467,16 @@ main = scopeCrossStatic.main.override { all_features = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" - # dont include experimental features - "experimental" - # jemalloc profiling/stats features are expensive and shouldn't - # be expected on non-debug builds. - "jemalloc_prof" - "jemalloc_stats" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" + # dont include experimental features + "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; }; @@ -452,16 +490,16 @@ main = scopeCrossStatic.main.override { all_features = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" - # dont include experimental features - "experimental" - # jemalloc profiling/stats features are expensive and shouldn't - # be expected on non-debug builds. - "jemalloc_prof" - "jemalloc_stats" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" + # dont include experimental features + "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; x86_64_haswell_target_optimised = (if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false); }; @@ -478,12 +516,12 @@ # debug build users expect full logs disable_release_max_log_level = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" - # dont include experimental features - "experimental" - # conduwuit_mods is a development-only hot reload feature - "conduwuit_mods" + # dont include experimental features + "experimental" + # this is non-functional on nix for some reason + "hardened_malloc" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" ]; }; }; @@ -522,14 +560,14 @@ main = prev.main.override { all_features = true; disable_features = [ - # this is non-functional on nix for some reason - "hardened_malloc" # dont include experimental features "experimental" # jemalloc profiling/stats features are expensive and shouldn't # be expected on non-debug builds. "jemalloc_prof" "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" # conduwuit_mods is a development-only hot reload feature "conduwuit_mods" ]; diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index 36f12400..e35cbf04 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -20,6 +20,8 @@ let disable_features = [ # no reason to use jemalloc for complement, just has compatibility/build issues "jemalloc" + "jemalloc_stats" + "jemalloc_prof" # console/CLI stuff isn't used or relevant for complement "console" "tokio_console" @@ -32,6 +34,14 @@ let "hardened_malloc" # dont include experimental features "experimental" + # compression isn't needed for complement + "brotli_compression" + "gzip_compression" + "zstd_compression" + # complement doesn't need hot reloading + "conduwuit_mods" + # complement doesn't have URL preview media tests + "url_preview" ]; }; diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index a785e7f2..d7424d11 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -15,7 +15,19 @@ # Options (keep sorted) , all_features ? false , default_features ? true -, disable_features ? [] +# default list of disabled features +, disable_features ? [ + # dont include experimental features + "experimental" + # jemalloc profiling/stats features are expensive and shouldn't + # be expected on non-debug builds. + "jemalloc_prof" + "jemalloc_stats" + # this is non-functional on nix for some reason + "hardened_malloc" + # conduwuit_mods is a development-only hot reload feature + "conduwuit_mods" +] , disable_release_max_log_level ? false , features ? [] , profile ? "release" From 0074f903d8a0574b63b588438efba996ef467c26 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 11 Jan 2025 21:43:46 -0500 Subject: [PATCH 1337/2091] delete lix binary cache because it has terrible reliability Signed-off-by: strawberry --- .gitea/workflows/ci.yml | 8 ++++---- .github/workflows/ci.yml | 8 ++++---- .github/workflows/documentation.yml | 4 ++-- .gitlab-ci.yml | 8 ++------ 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 9ce7c993..ef436734 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -87,8 +87,8 @@ jobs: - name: Apply Nix binary cache configuration run: | sudo tee -a /etc/nix/nix.conf > /dev/null < /dev/null < /dev/null < /dev/null < /dev/null < /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi - # Add Lix binary cache - - if command -v nix > /dev/null; then echo "extra-substituters = https://cache.lix.systems" >> /etc/nix/nix.conf; fi - - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o=" >> /etc/nix/nix.conf; fi - # Add crane binary cache - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi From 6f15c9b3f47143715280e485db2a5bcab35d77fd Mon Sep 17 00:00:00 2001 From: Holger Huo <50446405+HolgerHuo@users.noreply.github.com> Date: Wed, 8 Jan 2025 17:57:12 +0800 Subject: [PATCH 1338/2091] fix: presence timer not working --- src/api/client/presence.rs | 13 +++++++++---- src/api/client/sync/v3.rs | 17 ++++++++++++++--- src/service/presence/presence.rs | 6 +----- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/api/client/presence.rs b/src/api/client/presence.rs index 1a3ad26e..d19e6ae1 100644 --- a/src/api/client/presence.rs +++ b/src/api/client/presence.rs @@ -82,14 +82,19 @@ pub(crate) async fn get_presence_route( presence.content.status_msg }; + let last_active_ago = match presence.content.currently_active { + | Some(true) => None, + | _ => presence + .content + .last_active_ago + .map(|millis| Duration::from_millis(millis.into())), + }; + Ok(get_presence::v3::Response { // TODO: Should ruma just use the presenceeventcontent type here? status_msg, currently_active: presence.content.currently_active, - last_active_ago: presence - .content - .last_active_ago - .map(|millis| Duration::from_millis(millis.into())), + last_active_ago, presence: presence.content.presence, }) } else { diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index b7ecd6b9..910a15d4 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -382,7 +382,16 @@ async fn process_presence_updates( .ready_fold(PresenceUpdates::new(), |mut updates, (user_id, event)| { match updates.entry(user_id.into()) { | Entry::Vacant(slot) => { - slot.insert(event); + let mut new_event = event; + new_event.content.last_active_ago = match new_event.content.currently_active { + | Some(true) => None, + | _ => new_event + .content + .last_active_ago + .or(new_event.content.last_active_ago), + }; + + slot.insert(new_event); }, | Entry::Occupied(mut slot) => { let curr_event = slot.get_mut(); @@ -394,8 +403,6 @@ async fn process_presence_updates( curr_content.status_msg = new_content .status_msg .or_else(|| curr_content.status_msg.take()); - curr_content.last_active_ago = - new_content.last_active_ago.or(curr_content.last_active_ago); curr_content.displayname = new_content .displayname .or_else(|| curr_content.displayname.take()); @@ -405,6 +412,10 @@ async fn process_presence_updates( curr_content.currently_active = new_content .currently_active .or(curr_content.currently_active); + curr_content.last_active_ago = match curr_content.currently_active { + | Some(true) => None, + | _ => new_content.last_active_ago.or(curr_content.last_active_ago), + }; }, }; diff --git a/src/service/presence/presence.rs b/src/service/presence/presence.rs index b88a004b..b322dfb4 100644 --- a/src/service/presence/presence.rs +++ b/src/service/presence/presence.rs @@ -46,11 +46,7 @@ impl Presence { users: &users::Service, ) -> PresenceEvent { let now = utils::millis_since_unix_epoch(); - let last_active_ago = if self.currently_active { - None - } else { - Some(UInt::new_saturating(now.saturating_sub(self.last_active_ts))) - }; + let last_active_ago = Some(UInt::new_saturating(now.saturating_sub(self.last_active_ts))); PresenceEvent { sender: user_id.to_owned(), From 8451ea3bc32748fc8ff64d817685cc7f344e2e64 Mon Sep 17 00:00:00 2001 From: Holger Huo <50446405+HolgerHuo@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:24:27 +0800 Subject: [PATCH 1339/2091] update: refresh timeout greater than idle timeout --- src/service/presence/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index bf5258e1..3b0bdd15 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -95,7 +95,7 @@ impl Service { /// Pings the presence of the given user in the given room, setting the /// specified state. pub async fn ping_presence(&self, user_id: &UserId, new_state: &PresenceState) -> Result<()> { - const REFRESH_TIMEOUT: u64 = 60 * 25 * 1000; + const REFRESH_TIMEOUT: u64 = 60 * 1000; let last_presence = self.db.get_presence(user_id).await; let state_changed = match last_presence { From b71201cf19cddd72c689ada532dbe1400f6a997d Mon Sep 17 00:00:00 2001 From: Holger Huo <50446405+HolgerHuo@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:42:46 +0800 Subject: [PATCH 1340/2091] add: clear online status on server boot --- src/service/presence/mod.rs | 38 +++++++++++++++++++++++++++++++++++++ src/service/services.rs | 5 +++++ 2 files changed, 43 insertions(+) diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 3b0bdd15..a6eb1bcd 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -170,6 +170,44 @@ impl Service { self.db.remove_presence(user_id).await; } + // Unset online/unavailable presence to offline on startup + pub async fn unset_all_presence(&self) -> Result<()> { + for user_id in &self + .services + .users + .list_local_users() + .map(UserId::to_owned) + .collect::>() + .await + { + let presence = self.db.get_presence(user_id).await; + + let presence = match presence { + | Ok((_, ref presence)) => &presence.content, + | _ => return Ok(()), + }; + + let need_reset = match presence.presence { + | PresenceState::Unavailable | PresenceState::Online => true, + | _ => false, + }; + + if !need_reset { + return Ok(()); + } + + self.set_presence( + user_id, + &PresenceState::Offline, + Some(false), + presence.last_active_ago, + presence.status_msg.clone(), + ) + .await?; + } + Ok(()) + } + /// Returns the most recent presence updates that happened after the event /// with id `since`. pub fn presence_since( diff --git a/src/service/services.rs b/src/service/services.rs index c955834e..9e099759 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -123,6 +123,11 @@ impl Services { .start() .await?; + // clear online statuses + if self.server.config.allow_local_presence { + _ = self.presence.unset_all_presence().await; + } + // set the server user as online if self.server.config.allow_local_presence && !self.db.is_read_only() { _ = self From fde1b94e26f22f21c3c6e012331f23164c27d776 Mon Sep 17 00:00:00 2001 From: Holger Huo <50446405+HolgerHuo@users.noreply.github.com> Date: Fri, 10 Jan 2025 23:51:51 +0800 Subject: [PATCH 1341/2091] fix: logic mistake --- src/api/client/sync/v3.rs | 5 +---- src/service/presence/mod.rs | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 910a15d4..95c8c2d4 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -385,10 +385,7 @@ async fn process_presence_updates( let mut new_event = event; new_event.content.last_active_ago = match new_event.content.currently_active { | Some(true) => None, - | _ => new_event - .content - .last_active_ago - .or(new_event.content.last_active_ago), + | _ => new_event.content.last_active_ago, }; slot.insert(new_event); diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index a6eb1bcd..ab7c76a1 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -193,7 +193,7 @@ impl Service { }; if !need_reset { - return Ok(()); + continue; } self.set_presence( From 8c18481d1dab411bd9270ed56e2b6b5c1f465d3f Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 10 Jan 2025 23:51:08 -0500 Subject: [PATCH 1342/2091] optimise resetting all user presences Signed-off-by: strawberry --- src/admin/query/presence.rs | 2 +- src/service/presence/data.rs | 4 +-- src/service/presence/mod.rs | 53 +++++++++++++++++++++++------------- src/service/services.rs | 9 ++---- 4 files changed, 40 insertions(+), 28 deletions(-) diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index 45bb6bd9..0de6b696 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -32,7 +32,7 @@ pub(super) async fn process( match subcommand { | PresenceCommand::GetPresence { user_id } => { let timer = tokio::time::Instant::now(); - let results = services.presence.db.get_presence(&user_id).await; + let results = services.presence.get_presence(&user_id).await; let query_time = timer.elapsed(); Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 3d614333..4ec0a7ee 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -12,7 +12,7 @@ use ruma::{events::presence::PresenceEvent, presence::PresenceState, UInt, UserI use super::Presence; use crate::{globals, users, Dep}; -pub struct Data { +pub(crate) struct Data { presenceid_presence: Arc, userid_presenceid: Arc, services: Services, @@ -36,7 +36,7 @@ impl Data { } } - pub async fn get_presence(&self, user_id: &UserId) -> Result<(u64, PresenceEvent)> { + pub(super) async fn get_presence(&self, user_id: &UserId) -> Result<(u64, PresenceEvent)> { let count = self .userid_presenceid .get(user_id) diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index ab7c76a1..eb4105e5 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -4,7 +4,10 @@ mod presence; use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduwuit::{checked, debug, error, result::LogErr, Error, Result, Server}; +use conduwuit::{ + checked, debug, debug_warn, error, result::LogErr, trace, Error, Result, Server, +}; +use database::Database; use futures::{stream::FuturesUnordered, Stream, StreamExt, TryFutureExt}; use loole::{Receiver, Sender}; use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; @@ -18,12 +21,13 @@ pub struct Service { timeout_remote_users: bool, idle_timeout: u64, offline_timeout: u64, - pub db: Data, + db: Data, services: Services, } struct Services { server: Arc, + db: Arc, globals: Dep, users: Dep, } @@ -44,6 +48,7 @@ impl crate::Service for Service { db: Data::new(&args), services: Services { server: args.server.clone(), + db: args.db.clone(), globals: args.depend::("globals"), users: args.depend::("users"), }, @@ -171,7 +176,9 @@ impl Service { } // Unset online/unavailable presence to offline on startup - pub async fn unset_all_presence(&self) -> Result<()> { + pub async fn unset_all_presence(&self) { + let _cork = self.services.db.cork(); + for user_id in &self .services .users @@ -184,28 +191,36 @@ impl Service { let presence = match presence { | Ok((_, ref presence)) => &presence.content, - | _ => return Ok(()), + | _ => continue, }; - let need_reset = match presence.presence { - | PresenceState::Unavailable | PresenceState::Online => true, - | _ => false, - }; - - if !need_reset { + if !matches!( + presence.presence, + PresenceState::Unavailable | PresenceState::Online | PresenceState::Busy + ) { + trace!(?user_id, ?presence, "Skipping user"); continue; } - self.set_presence( - user_id, - &PresenceState::Offline, - Some(false), - presence.last_active_ago, - presence.status_msg.clone(), - ) - .await?; + trace!(?user_id, ?presence, "Resetting presence to offline"); + + _ = self + .set_presence( + user_id, + &PresenceState::Offline, + Some(false), + presence.last_active_ago, + presence.status_msg.clone(), + ) + .await + .inspect_err(|e| { + debug_warn!( + ?presence, + "{user_id} has invalid presence in database and failed to reset it to \ + offline: {e}" + ); + }); } - Ok(()) } /// Returns the most recent presence updates that happened after the event diff --git a/src/service/services.rs b/src/service/services.rs index 9e099759..1aa87f58 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -123,13 +123,10 @@ impl Services { .start() .await?; - // clear online statuses - if self.server.config.allow_local_presence { - _ = self.presence.unset_all_presence().await; - } - - // set the server user as online + // reset dormant online/away statuses to offline, and set the server user as + // online if self.server.config.allow_local_presence && !self.db.is_read_only() { + self.presence.unset_all_presence().await; _ = self .presence .ping_presence(&self.globals.server_user, &ruma::presence::PresenceState::Online) From 9bda5a43e5ac742fb6b30ae14c1d1e89c0a68c36 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 11 Jan 2025 00:24:51 -0500 Subject: [PATCH 1343/2091] fix /kick endpoint unbanning banned users Signed-off-by: strawberry --- src/api/client/membership.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 4046b493..0c493a37 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -439,6 +439,16 @@ pub(crate) async fn kick_user_route( return Ok(kick_user::v3::Response::new()); }; + if !matches!( + event.membership, + MembershipState::Invite | MembershipState::Knock | MembershipState::Join, + ) { + return Err!(Request(Forbidden( + "Cannot kick a user who is not apart of the room (current membership: {})", + event.membership + ))); + } + services .rooms .timeline @@ -527,7 +537,7 @@ pub(crate) async fn unban_user_route( if current_member_content.membership != MembershipState::Ban { return Err!(Request(Forbidden( - "Cannot ban a user who is not banned (current membership: {})", + "Cannot unban a user who is not banned (current membership: {})", current_member_content.membership ))); } From 5e21b43f2505fd8369f4ce09ef36950983d12182 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 11 Jan 2025 00:25:10 -0500 Subject: [PATCH 1344/2091] run direnv exec in engage default steps Signed-off-by: strawberry --- .github/workflows/ci.yml | 2 -- engage.toml | 18 +++++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d73af24f..d06de5e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,8 +50,6 @@ env: experimental-features = nix-command flakes extra-experimental-features = nix-command flakes accept-flake-config = true - # complement uses libolm - NIXPKGS_ALLOW_INSECURE: 1 WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} GH_SHA: ${{ github.sha }} GH_REF_NAME: ${{ github.ref_name }} diff --git a/engage.toml b/engage.toml index 9a6ef8ca..1d6a5475 100644 --- a/engage.toml +++ b/engage.toml @@ -97,6 +97,7 @@ env DIRENV_DEVSHELL=all-features \ name = "clippy/default" group = "lints" script = """ +direnv exec . \ cargo clippy \ --workspace \ --profile test \ @@ -126,6 +127,7 @@ env DIRENV_DEVSHELL=all-features \ name = "clippy/jemalloc" group = "lints" script = """ +direnv exec . \ cargo clippy \ --workspace \ --profile test \ @@ -179,13 +181,15 @@ env DIRENV_DEVSHELL=all-features \ name = "cargo/default" group = "tests" script = """ -cargo test \ - --workspace \ - --profile test \ - --all-targets \ - --color=always \ - -- \ - --color=always +env DIRENV_DEVSHELL=default \ + direnv exec . \ + cargo test \ + --workspace \ + --profile test \ + --all-targets \ + --color=always \ + -- \ + --color=always """ # Checks if the generated example config differs from the checked in repo's From fabd3cf567c9c676d8d546ced79e81f69bb70ae4 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 12 Jan 2025 19:18:07 -0500 Subject: [PATCH 1345/2091] ci: set binary as executable before uploading to webserver Signed-off-by: strawberry --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d06de5e3..007adace 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -490,6 +490,7 @@ jobs: if: ${{ matrix.target == 'x86_64-linux-musl' }} run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-x86_64-linux-musl-x86_64-haswell-optimised fi @@ -497,6 +498,7 @@ jobs: if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + chmod +x static-${{ matrix.target }} scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }} fi @@ -692,6 +694,7 @@ jobs: if: ${{ matrix.os == 'macos-13' }} run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + chmod +x conduwuit-macos-x86_64 scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-x86_64 fi @@ -699,6 +702,7 @@ jobs: if: ${{ matrix.os == 'macos-latest' }} run: | if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + chmod +x conduwuit-macos-arm64 scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-arm64 fi From 5a1c41e66b4fec8ab76fd268fc9c9e282fd19428 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 11 Jan 2025 18:43:54 -0500 Subject: [PATCH 1346/2091] knocking implementation Signed-off-by: strawberry add sync bit of knocking Signed-off-by: strawberry --- src/api/client/membership.rs | 716 +++++++++++++++++++++++++-- src/api/client/sync/v3.rs | 39 +- src/api/client/sync/v4.rs | 9 + src/api/router.rs | 3 + src/api/server/invite.rs | 14 +- src/api/server/make_knock.rs | 38 +- src/api/server/make_leave.rs | 6 +- src/api/server/mod.rs | 4 + src/api/server/send_join.rs | 13 +- src/api/server/send_knock.rs | 75 ++- src/api/server/utils.rs | 17 +- src/database/maps.rs | 8 + src/service/rooms/state_cache/mod.rs | 142 +++++- src/service/rooms/timeline/mod.rs | 11 +- 14 files changed, 978 insertions(+), 117 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 0c493a37..d94fc3c7 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1,4 +1,5 @@ use std::{ + borrow::Borrow, collections::{BTreeMap, HashMap, HashSet}, net::IpAddr, sync::Arc, @@ -8,7 +9,7 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ debug, debug_info, debug_warn, err, info, - pdu::{self, gen_event_id_canonical_json, PduBuilder}, + pdu::{gen_event_id_canonical_json, PduBuilder}, result::FlatOk, trace, utils::{self, shuffle, IterStream, ReadyExt}, @@ -19,6 +20,7 @@ use ruma::{ api::{ client::{ error::ErrorKind, + knock::knock_room, membership::{ ban_user, forget_room, get_member_events, invite_user, join_room_by_id, join_room_by_id_or_alias, @@ -37,11 +39,12 @@ use ruma::{ }, StateEventType, }, - state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedRoomId, OwnedServerName, - OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, + state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, + OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use service::{ appservice::RegistrationInfo, + pdu::gen_event_id, rooms::{state::RoomMutexGuard, state_compressor::HashSetCompressStateEvent}, Services, }; @@ -348,6 +351,116 @@ pub(crate) async fn join_room_by_id_or_alias_route( Ok(join_room_by_id_or_alias::v3::Response { room_id: join_room_response.room_id }) } +/// # `POST /_matrix/client/*/knock/{roomIdOrAlias}` +/// +/// Tries to knock the room to ask permission to join for the sender user. +#[tracing::instrument(skip_all, fields(%client), name = "knock")] +pub(crate) async fn knock_room_route( + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let body = body.body; + + let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { + | Ok(room_id) => { + banned_room_check( + &services, + sender_user, + Some(&room_id), + room_id.server_name(), + client, + ) + .await?; + + let mut servers = body.via.clone(); + servers.extend( + services + .rooms + .state_cache + .servers_invite_via(&room_id) + .map(ToOwned::to_owned) + .collect::>() + .await, + ); + + servers.extend( + services + .rooms + .state_cache + .invite_state(sender_user, &room_id) + .await + .unwrap_or_default() + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), + ); + + if let Some(server) = room_id.server_name() { + servers.push(server.to_owned()); + } + + servers.sort_unstable(); + servers.dedup(); + shuffle(&mut servers); + + (servers, room_id) + }, + | Err(room_alias) => { + let (room_id, mut servers) = services + .rooms + .alias + .resolve_alias(&room_alias, Some(body.via.clone())) + .await?; + + banned_room_check( + &services, + sender_user, + Some(&room_id), + Some(room_alias.server_name()), + client, + ) + .await?; + + let addl_via_servers = services + .rooms + .state_cache + .servers_invite_via(&room_id) + .map(ToOwned::to_owned); + + let addl_state_servers = services + .rooms + .state_cache + .invite_state(sender_user, &room_id) + .await + .unwrap_or_default(); + + let mut addl_servers: Vec<_> = addl_state_servers + .iter() + .map(|event| event.get_field("sender")) + .filter_map(FlatOk::flat_ok) + .map(|user: &UserId| user.server_name().to_owned()) + .stream() + .chain(addl_via_servers) + .collect() + .await; + + addl_servers.sort_unstable(); + addl_servers.dedup(); + shuffle(&mut addl_servers); + servers.append(&mut addl_servers); + + (servers, room_id) + }, + }; + + knock_room_by_id_helper(&services, sender_user, &room_id, body.reason.clone(), &servers) + .boxed() + .await +} + /// # `POST /_matrix/client/v3/rooms/{roomId}/leave` /// /// Tries to leave the sender user from a room. @@ -403,6 +516,17 @@ pub(crate) async fn invite_user_route( ))); } + if let Ok(target_user_membership) = services + .rooms + .state_accessor + .get_member(&body.room_id, user_id) + .await + { + if target_user_membership.membership == MembershipState::Ban { + return Err!(Request(Forbidden("User is banned from this room."))); + } + } + if recipient_ignored_by_sender { // silently drop the invite to the recipient if they've been ignored by the // sender, pretend it worked @@ -862,7 +986,7 @@ async fn join_room_by_id_helper_remote( .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; // Generate event id - let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?; + let event_id = gen_event_id(&join_event_stub, &room_version_id)?; // Add event_id back join_event_stub @@ -1030,7 +1154,7 @@ async fn join_room_by_id_helper_remote( }; let auth_check = state_res::event_auth::auth_check( - &state_res::RoomVersion::new(&room_version_id).expect("room version is supported"), + &state_res::RoomVersion::new(&room_version_id)?, &parsed_join_pdu, None, // TODO: third party invite |k, s| state_fetch(k, s.to_owned()), @@ -1043,10 +1167,10 @@ async fn join_room_by_id_helper_remote( } info!("Compressing state from send_join"); - let compressed = state - .iter() - .stream() - .then(|(&k, id)| services.rooms.state_compressor.compress_state_event(k, id)) + let compressed: HashSet<_> = services + .rooms + .state_compressor + .compress_state_events(state.iter().map(|(ssk, eid)| (ssk, eid.borrow()))) .collect() .await; @@ -1282,7 +1406,7 @@ async fn join_room_by_id_helper_local( .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; // Generate event id - let event_id = pdu::gen_event_id(&join_event_stub, &room_version_id)?; + let event_id = gen_event_id(&join_event_stub, &room_version_id)?; // Add event_id back join_event_stub @@ -1392,6 +1516,7 @@ async fn make_join_request( ); make_join_response_and_server = Err!(BadServerResponse("No server available to assist in joining.")); + return make_join_response_and_server; } } @@ -1569,7 +1694,7 @@ pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { for room_id in all_rooms { // ignore errors if let Err(e) = leave_room(services, user_id, &room_id, None).await { - warn!(%room_id, %user_id, %e, "Failed to leave room"); + warn!(%user_id, "Failed to leave {room_id} remotely: {e}"); } services.rooms.state_cache.forget(&room_id, user_id); @@ -1585,11 +1710,15 @@ pub async fn leave_room( //use conduwuit::utils::stream::OptionStream; use futures::TryFutureExt; - // Ask a remote server if we don't have this room + // Ask a remote server if we don't have this room and are not knocking on it if !services .rooms .state_cache .server_in_room(services.globals.server_name(), room_id) + .await && !services + .rooms + .state_cache + .is_knocked(user_id, room_id) .await { if let Err(e) = remote_leave_room(services, user_id, room_id).await { @@ -1601,7 +1730,8 @@ pub async fn leave_room( .rooms .state_cache .invite_state(user_id, room_id) - .map_err(|_| services.rooms.state_cache.left_state(user_id, room_id)) + .or_else(|_| services.rooms.state_cache.knock_state(user_id, room_id)) + .or_else(|_| services.rooms.state_cache.left_state(user_id, room_id)) .await .ok(); @@ -1683,13 +1813,6 @@ async fn remote_leave_room( let mut make_leave_response_and_server = Err!(BadServerResponse("No server available to assist in leaving.")); - let invite_state = services - .rooms - .state_cache - .invite_state(user_id, room_id) - .await - .map_err(|_| err!(Request(BadState("User is not invited."))))?; - let mut servers: HashSet = services .rooms .state_cache @@ -1698,13 +1821,39 @@ async fn remote_leave_room( .collect() .await; - servers.extend( - invite_state - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()), - ); + if let Ok(invite_state) = services + .rooms + .state_cache + .invite_state(user_id, room_id) + .await + { + servers.extend( + invite_state + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), + ); + } else if let Ok(knock_state) = services + .rooms + .state_cache + .knock_state(user_id, room_id) + .await + { + servers.extend( + knock_state + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .filter_map(|sender| { + if !services.globals.user_is_local(sender) { + Some(sender.server_name().to_owned()) + } else { + None + } + }), + ); + } if let Some(room_id_server_name) = room_id.server_name() { servers.insert(room_id_server_name.to_owned()); @@ -1779,7 +1928,7 @@ async fn remote_leave_room( .hash_and_sign_event(&mut leave_event_stub, &room_version_id)?; // Generate event id - let event_id = pdu::gen_event_id(&leave_event_stub, &room_version_id)?; + let event_id = gen_event_id(&leave_event_stub, &room_version_id)?; // Add event_id back leave_event_stub @@ -1805,3 +1954,514 @@ async fn remote_leave_room( Ok(()) } + +async fn knock_room_by_id_helper( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], +) -> Result { + let state_lock = services.rooms.state.mutex.lock(room_id).await; + + if services + .rooms + .state_cache + .is_invited(sender_user, room_id) + .await + { + debug_warn!("{sender_user} is already invited in {room_id} but attempted to knock"); + return Err!(Request(Forbidden( + "You cannot knock on a room you are already invited/accepted to." + ))); + } + + if services + .rooms + .state_cache + .is_joined(sender_user, room_id) + .await + { + debug_warn!("{sender_user} is already joined in {room_id} but attempted to knock"); + return Err!(Request(Forbidden("You cannot knock on a room you are already joined in."))); + } + + if services + .rooms + .state_cache + .is_knocked(sender_user, room_id) + .await + { + debug_warn!("{sender_user} is already knocked in {room_id}"); + return Ok(knock_room::v3::Response { room_id: room_id.into() }); + } + + if let Ok(membership) = services + .rooms + .state_accessor + .get_member(room_id, sender_user) + .await + { + if membership.membership == MembershipState::Ban { + debug_warn!("{sender_user} is banned from {room_id} but attempted to knock"); + return Err!(Request(Forbidden("You cannot knock on a room you are banned from."))); + } + } + + let server_in_room = services + .rooms + .state_cache + .server_in_room(services.globals.server_name(), room_id) + .await; + + let local_knock = server_in_room + || servers.is_empty() + || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); + + if local_knock { + knock_room_helper_local(services, sender_user, room_id, reason, servers, state_lock) + .boxed() + .await?; + } else { + knock_room_helper_remote(services, sender_user, room_id, reason, servers, state_lock) + .boxed() + .await?; + } + + Ok(knock_room::v3::Response::new(room_id.to_owned())) +} + +async fn knock_room_helper_local( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + state_lock: RoomMutexGuard, +) -> Result { + debug_info!("We can knock locally"); + + let room_version_id = services.rooms.state.get_room_version(room_id).await?; + + if matches!( + room_version_id, + RoomVersionId::V1 + | RoomVersionId::V2 + | RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + ) { + return Err!(Request(Forbidden("This room does not support knocking."))); + } + + let content = RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason: reason.clone(), + ..RoomMemberEventContent::new(MembershipState::Knock) + }; + + // Try normal knock first + let Err(error) = services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state(sender_user.to_string(), &content), + sender_user, + room_id, + &state_lock, + ) + .await + else { + return Ok(()); + }; + + if servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])) + { + return Err(error); + } + + warn!("We couldn't do the knock locally, maybe federation can help to satisfy the knock"); + + let (make_knock_response, remote_server) = + make_knock_request(services, sender_user, room_id, servers).await?; + + info!("make_knock finished"); + + let room_version_id = make_knock_response.room_version; + + if !services.server.supported_room_version(&room_version_id) { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } + + let mut knock_event_stub = serde_json::from_str::( + make_knock_response.event.get(), + ) + .map_err(|e| { + err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}")) + })?; + + knock_event_stub.insert( + "origin".to_owned(), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), + ); + knock_event_stub.insert( + "origin_server_ts".to_owned(), + CanonicalJsonValue::Integer( + utils::millis_since_unix_epoch() + .try_into() + .expect("Timestamp is valid js_int value"), + ), + ); + knock_event_stub.insert( + "content".to_owned(), + to_canonical_value(RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason, + ..RoomMemberEventContent::new(MembershipState::Knock) + }) + .expect("event is valid, we just created it"), + ); + + // In order to create a compatible ref hash (EventID) the `hashes` field needs + // to be present + services + .server_keys + .hash_and_sign_event(&mut knock_event_stub, &room_version_id)?; + + // Generate event id + let event_id = gen_event_id(&knock_event_stub, &room_version_id)?; + + // Add event_id + knock_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + + // It has enough fields to be called a proper event now + let knock_event = knock_event_stub; + + info!("Asking {remote_server} for send_knock in room {room_id}"); + let send_knock_request = federation::knock::send_knock::v1::Request { + room_id: room_id.to_owned(), + event_id: event_id.clone(), + pdu: services + .sending + .convert_to_outgoing_federation_event(knock_event.clone()) + .await, + }; + + let send_knock_response = services + .sending + .send_federation_request(&remote_server, send_knock_request) + .await?; + + info!("send_knock finished"); + + services + .rooms + .short + .get_or_create_shortroomid(room_id) + .await; + + info!("Parsing knock event"); + + let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone()) + .map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?; + + info!("Updating membership locally to knock state with provided stripped state events"); + services + .rooms + .state_cache + .update_membership( + room_id, + sender_user, + parsed_knock_pdu + .get_content::() + .expect("we just created this"), + sender_user, + Some(send_knock_response.knock_room_state), + None, + false, + ) + .await?; + + info!("Appending room knock event locally"); + services + .rooms + .timeline + .append_pdu( + &parsed_knock_pdu, + knock_event, + vec![(*parsed_knock_pdu.event_id).to_owned()], + &state_lock, + ) + .await?; + + Ok(()) +} + +async fn knock_room_helper_remote( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + state_lock: RoomMutexGuard, +) -> Result { + info!("Knocking {room_id} over federation."); + + let (make_knock_response, remote_server) = + make_knock_request(services, sender_user, room_id, servers).await?; + + info!("make_knock finished"); + + let room_version_id = make_knock_response.room_version; + + if !services.server.supported_room_version(&room_version_id) { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } + + let mut knock_event_stub: CanonicalJsonObject = + serde_json::from_str(make_knock_response.event.get()).map_err(|e| { + err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}")) + })?; + + knock_event_stub.insert( + "origin".to_owned(), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), + ); + knock_event_stub.insert( + "origin_server_ts".to_owned(), + CanonicalJsonValue::Integer( + utils::millis_since_unix_epoch() + .try_into() + .expect("Timestamp is valid js_int value"), + ), + ); + knock_event_stub.insert( + "content".to_owned(), + to_canonical_value(RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason, + ..RoomMemberEventContent::new(MembershipState::Knock) + }) + .expect("event is valid, we just created it"), + ); + + // In order to create a compatible ref hash (EventID) the `hashes` field needs + // to be present + services + .server_keys + .hash_and_sign_event(&mut knock_event_stub, &room_version_id)?; + + // Generate event id + let event_id = gen_event_id(&knock_event_stub, &room_version_id)?; + + // Add event_id + knock_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + + // It has enough fields to be called a proper event now + let knock_event = knock_event_stub; + + info!("Asking {remote_server} for send_knock in room {room_id}"); + let send_knock_request = federation::knock::send_knock::v1::Request { + room_id: room_id.to_owned(), + event_id: event_id.clone(), + pdu: services + .sending + .convert_to_outgoing_federation_event(knock_event.clone()) + .await, + }; + + let send_knock_response = services + .sending + .send_federation_request(&remote_server, send_knock_request) + .await?; + + info!("send_knock finished"); + + services + .rooms + .short + .get_or_create_shortroomid(room_id) + .await; + + info!("Parsing knock event"); + let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone()) + .map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?; + + info!("Going through send_knock response knock state events"); + let state = send_knock_response + .knock_room_state + .iter() + .map(|event| serde_json::from_str::(event.clone().into_json().get())) + .filter_map(Result::ok); + + let mut state_map: HashMap = HashMap::new(); + + for event in state { + let Some(state_key) = event.get("state_key") else { + debug_warn!("send_knock stripped state event missing state_key: {event:?}"); + continue; + }; + let Some(event_type) = event.get("type") else { + debug_warn!("send_knock stripped state event missing event type: {event:?}"); + continue; + }; + + let Ok(state_key) = serde_json::from_value::(state_key.clone().into()) else { + debug_warn!("send_knock stripped state event has invalid state_key: {event:?}"); + continue; + }; + let Ok(event_type) = serde_json::from_value::(event_type.clone().into()) + else { + debug_warn!("send_knock stripped state event has invalid event type: {event:?}"); + continue; + }; + + let event_id = gen_event_id(&event, &room_version_id)?; + let shortstatekey = services + .rooms + .short + .get_or_create_shortstatekey(&event_type, &state_key) + .await; + + services.rooms.outlier.add_pdu_outlier(&event_id, &event); + state_map.insert(shortstatekey, event_id.clone()); + } + + info!("Compressing state from send_knock"); + let compressed: HashSet<_> = services + .rooms + .state_compressor + .compress_state_events(state_map.iter().map(|(ssk, eid)| (ssk, eid.borrow()))) + .collect() + .await; + + debug!("Saving compressed state"); + let HashSetCompressStateEvent { + shortstatehash: statehash_before_knock, + added, + removed, + } = services + .rooms + .state_compressor + .save_state(room_id, Arc::new(compressed)) + .await?; + + debug!("Forcing state for new room"); + services + .rooms + .state + .force_state(room_id, statehash_before_knock, added, removed, &state_lock) + .await?; + + let statehash_after_knock = services + .rooms + .state + .append_to_state(&parsed_knock_pdu) + .await?; + + info!("Updating membership locally to knock state with provided stripped state events"); + services + .rooms + .state_cache + .update_membership( + room_id, + sender_user, + parsed_knock_pdu + .get_content::() + .expect("we just created this"), + sender_user, + Some(send_knock_response.knock_room_state), + None, + false, + ) + .await?; + + info!("Appending room knock event locally"); + services + .rooms + .timeline + .append_pdu( + &parsed_knock_pdu, + knock_event, + vec![(*parsed_knock_pdu.event_id).to_owned()], + &state_lock, + ) + .await?; + + info!("Setting final room state for new room"); + // We set the room state after inserting the pdu, so that we never have a moment + // in time where events in the current room state do not exist + services + .rooms + .state + .set_room_state(room_id, statehash_after_knock, &state_lock); + + Ok(()) +} + +async fn make_knock_request( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + servers: &[OwnedServerName], +) -> Result<(federation::knock::create_knock_event_template::v1::Response, OwnedServerName)> { + let mut make_knock_response_and_server = + Err!(BadServerResponse("No server available to assist in knocking.")); + + let mut make_knock_counter: usize = 0; + + for remote_server in servers { + if services.globals.server_is_ours(remote_server) { + continue; + } + + info!("Asking {remote_server} for make_knock ({make_knock_counter})"); + + let make_knock_response = services + .sending + .send_federation_request( + remote_server, + federation::knock::create_knock_event_template::v1::Request { + room_id: room_id.to_owned(), + user_id: sender_user.to_owned(), + ver: services.server.supported_room_versions().collect(), + }, + ) + .await; + + trace!("make_knock response: {make_knock_response:?}"); + make_knock_counter = make_knock_counter.saturating_add(1); + + make_knock_response_and_server = make_knock_response.map(|r| (r, remote_server.clone())); + + if make_knock_response_and_server.is_ok() { + break; + } + + if make_knock_counter > 40 { + warn!( + "50 servers failed to provide valid make_knock response, assuming no server can \ + assist in knocking." + ); + make_knock_response_and_server = + Err!(BadServerResponse("No server available to assist in knocking.")); + + return make_knock_response_and_server; + } + } + + make_knock_response_and_server +} diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 95c8c2d4..a4dc0205 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -33,8 +33,8 @@ use ruma::{ self, v3::{ Ephemeral, Filter, GlobalAccountData, InviteState, InvitedRoom, JoinedRoom, - LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms, State as RoomState, - Timeline, ToDevice, + KnockState, KnockedRoom, LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms, + State as RoomState, Timeline, ToDevice, }, DeviceLists, UnreadNotificationsCount, }, @@ -266,6 +266,35 @@ pub(crate) async fn build_sync_events( invited_rooms }); + let knocked_rooms = services + .rooms + .state_cache + .rooms_knocked(sender_user) + .fold_default(|mut knocked_rooms: BTreeMap<_, _>, (room_id, knock_state)| async move { + // Get and drop the lock to wait for remaining operations to finish + let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; + drop(insert_lock); + + let knock_count = services + .rooms + .state_cache + .get_knock_count(&room_id, sender_user) + .await + .ok(); + + // Knocked before last sync + if Some(since) >= knock_count { + return knocked_rooms; + } + + let knocked_room = KnockedRoom { + knock_state: KnockState { events: knock_state }, + }; + + knocked_rooms.insert(room_id, knocked_room); + knocked_rooms + }); + let presence_updates: OptionFuture<_> = services .globals .allow_local_presence() @@ -300,7 +329,7 @@ pub(crate) async fn build_sync_events( .users .remove_to_device_events(sender_user, sender_device, since); - let rooms = join3(joined_rooms, left_rooms, invited_rooms); + let rooms = join4(joined_rooms, left_rooms, invited_rooms, knocked_rooms); let ephemeral = join3(remove_to_device_events, to_device_events, presence_updates); let top = join5(account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms) .boxed() @@ -308,7 +337,7 @@ pub(crate) async fn build_sync_events( let (account_data, ephemeral, device_one_time_keys_count, keys_changed, rooms) = top; let ((), to_device_events, presence_updates) = ephemeral; - let (joined_rooms, left_rooms, invited_rooms) = rooms; + let (joined_rooms, left_rooms, invited_rooms, knocked_rooms) = rooms; let (joined_rooms, mut device_list_updates, left_encrypted_users) = joined_rooms; device_list_updates.extend(keys_changed); @@ -349,7 +378,7 @@ pub(crate) async fn build_sync_events( leave: left_rooms, join: joined_rooms, invite: invited_rooms, - knock: BTreeMap::new(), // TODO + knock: knocked_rooms, }, to_device: ToDevice { events: to_device_events }, }; diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 9915752e..24c7e286 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -113,9 +113,18 @@ pub(crate) async fn sync_events_v4_route( .collect() .await; + let all_knocked_rooms: Vec<_> = services + .rooms + .state_cache + .rooms_knocked(sender_user) + .map(|r| r.0) + .collect() + .await; + let all_rooms = all_joined_rooms .iter() .chain(all_invited_rooms.iter()) + .chain(all_knocked_rooms.iter()) .map(Clone::clone) .collect(); diff --git a/src/api/router.rs b/src/api/router.rs index 1b38670d..1d42fc5e 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -99,6 +99,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(&client::join_room_by_id_route) .ruma_route(&client::join_room_by_id_or_alias_route) .ruma_route(&client::joined_members_route) + .ruma_route(&client::knock_room_route) .ruma_route(&client::leave_room_route) .ruma_route(&client::forget_room_route) .ruma_route(&client::joined_rooms_route) @@ -204,8 +205,10 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(&server::get_room_state_route) .ruma_route(&server::get_room_state_ids_route) .ruma_route(&server::create_leave_event_template_route) + .ruma_route(&server::create_knock_event_template_route) .ruma_route(&server::create_leave_event_v1_route) .ruma_route(&server::create_leave_event_v2_route) + .ruma_route(&server::create_knock_event_v1_route) .ruma_route(&server::create_join_event_template_route) .ruma_route(&server::create_join_event_v1_route) .ruma_route(&server::create_join_event_v2_route) diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 6d3be04c..1fea268b 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -6,8 +6,9 @@ use ruma::{ api::{client::error::ErrorKind, federation::membership::create_invite}, events::room::member::{MembershipState, RoomMemberEventContent}, serde::JsonObject, - CanonicalJsonValue, OwnedEventId, OwnedUserId, UserId, + CanonicalJsonValue, OwnedUserId, UserId, }; +use service::pdu::gen_event_id; use crate::Ruma; @@ -86,12 +87,7 @@ pub(crate) async fn create_invite_route( .map_err(|e| err!(Request(InvalidParam("Failed to sign event: {e}"))))?; // Generate event id - let event_id = OwnedEventId::parse(format!( - "${}", - ruma::signatures::reference_hash(&signed_event, &body.room_version) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); + let event_id = gen_event_id(&signed_event, &body.room_version)?; // Add event_id back signed_event.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.to_string())); @@ -115,12 +111,12 @@ pub(crate) async fn create_invite_route( let mut invite_state = body.invite_room_state.clone(); let mut event: JsonObject = serde_json::from_str(body.event.get()) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event bytes."))?; + .map_err(|e| err!(Request(BadJson("Invalid invite event PDU: {e}"))))?; event.insert("event_id".to_owned(), "$placeholder".into()); let pdu: PduEvent = serde_json::from_value(event.into()) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid invite event."))?; + .map_err(|e| err!(Request(BadJson("Invalid invite event PDU: {e}"))))?; invite_state.push(pdu.to_stripped_state_event()); diff --git a/src/api/server/make_knock.rs b/src/api/server/make_knock.rs index 6d9d6d55..90b9b629 100644 --- a/src/api/server/make_knock.rs +++ b/src/api/server/make_knock.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduwuit::Err; +use conduwuit::{debug_warn, Err}; use ruma::{ api::{client::error::ErrorKind, federation::knock::create_knock_event_template}, events::room::member::{MembershipState, RoomMemberEventContent}, @@ -15,7 +15,8 @@ use crate::{service::pdu::PduBuilder, Error, Result, Ruma}; /// /// Creates a knock template. pub(crate) async fn create_knock_event_template_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if !services.rooms.metadata.exists(&body.room_id).await { return Err!(Request(NotFound("Room is unknown to this server."))); @@ -39,8 +40,8 @@ pub(crate) async fn create_knock_event_template_route( .contains(body.origin()) { warn!( - "Server {} for remote user {} tried knocking room ID {} which has a server name that is globally \ - forbidden. Rejecting.", + "Server {} for remote user {} tried knocking room ID {} which has a server name \ + that is globally forbidden. Rejecting.", body.origin(), &body.user_id, &body.room_id, @@ -63,29 +64,44 @@ pub(crate) async fn create_knock_event_template_route( if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6) { return Err(Error::BadRequest( - ErrorKind::IncompatibleRoomVersion { - room_version: room_version_id, - }, + ErrorKind::IncompatibleRoomVersion { room_version: room_version_id }, "Room version does not support knocking.", )); } if !body.ver.contains(&room_version_id) { return Err(Error::BadRequest( - ErrorKind::IncompatibleRoomVersion { - room_version: room_version_id, - }, + ErrorKind::IncompatibleRoomVersion { room_version: room_version_id }, "Your homeserver does not support the features required to knock on this room.", )); } let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + if let Ok(membership) = services + .rooms + .state_accessor + .get_member(&body.room_id, &body.user_id) + .await + { + if membership.membership == MembershipState::Ban { + debug_warn!( + "Remote user {} is banned from {} but attempted to knock", + &body.user_id, + &body.room_id + ); + return Err!(Request(Forbidden("You cannot knock on a room you are banned from."))); + } + } + let (_pdu, mut pdu_json) = services .rooms .timeline .create_hash_and_sign_event( - PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent::new(MembershipState::Knock)), + PduBuilder::state( + body.user_id.to_string(), + &RoomMemberEventContent::new(MembershipState::Knock), + ), &body.user_id, &body.room_id, &state_lock, diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 746a4858..936e0fbb 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -9,7 +9,7 @@ use serde_json::value::to_raw_value; use super::make_join::maybe_strip_event_id; use crate::{service::pdu::PduBuilder, Ruma}; -/// # `PUT /_matrix/federation/v1/make_leave/{roomId}/{eventId}` +/// # `GET /_matrix/federation/v1/make_leave/{roomId}/{eventId}` /// /// Creates a leave template. pub(crate) async fn create_leave_event_template_route( @@ -21,7 +21,9 @@ pub(crate) async fn create_leave_event_template_route( } if body.user_id.server_name() != body.origin() { - return Err!(Request(BadJson("Not allowed to leave on behalf of another server/user."))); + return Err!(Request(Forbidden( + "Not allowed to leave on behalf of another server/user." + ))); } // ACL check origin diff --git a/src/api/server/mod.rs b/src/api/server/mod.rs index 9b7d91cb..5c1ff3f7 100644 --- a/src/api/server/mod.rs +++ b/src/api/server/mod.rs @@ -6,6 +6,7 @@ pub(super) mod hierarchy; pub(super) mod invite; pub(super) mod key; pub(super) mod make_join; +pub(super) mod make_knock; pub(super) mod make_leave; pub(super) mod media; pub(super) mod openid; @@ -13,6 +14,7 @@ pub(super) mod publicrooms; pub(super) mod query; pub(super) mod send; pub(super) mod send_join; +pub(super) mod send_knock; pub(super) mod send_leave; pub(super) mod state; pub(super) mod state_ids; @@ -28,6 +30,7 @@ pub(super) use hierarchy::*; pub(super) use invite::*; pub(super) use key::*; pub(super) use make_join::*; +pub(super) use make_knock::*; pub(super) use make_leave::*; pub(super) use media::*; pub(super) use openid::*; @@ -35,6 +38,7 @@ pub(super) use publicrooms::*; pub(super) use query::*; pub(super) use send::*; pub(super) use send_join::*; +pub(super) use send_knock::*; pub(super) use send_leave::*; pub(super) use state::*; pub(super) use state_ids::*; diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 6cbe5143..97a65bf8 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -186,14 +186,13 @@ async fn create_join_event( .map_err(|e| err!(Request(InvalidParam(warn!("Failed to sign send_join event: {e}")))))?; let origin: OwnedServerName = serde_json::from_value( - serde_json::to_value( - value - .get("origin") - .ok_or_else(|| err!(Request(BadJson("Event missing origin property."))))?, - ) - .expect("CanonicalJson is valid json value"), + value + .get("origin") + .ok_or_else(|| err!(Request(BadJson("Event does not have an origin server name."))))? + .clone() + .into(), ) - .map_err(|e| err!(Request(BadJson(warn!("origin field is not a valid server name: {e}")))))?; + .map_err(|e| err!(Request(BadJson("Event has an invalid origin server name: {e}"))))?; let mutex_lock = services .rooms diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs index 49ec4bf8..95478081 100644 --- a/src/api/server/send_knock.rs +++ b/src/api/server/send_knock.rs @@ -1,7 +1,8 @@ use axum::extract::State; -use conduwuit::{err, pdu::gen_event_id_canonical_json, warn, Err, Error, PduEvent, Result}; +use conduwuit::{err, pdu::gen_event_id_canonical_json, warn, Err, PduEvent, Result}; +use futures::FutureExt; use ruma::{ - api::{client::error::ErrorKind, federation::knock::send_knock}, + api::federation::knock::send_knock, events::{ room::member::{MembershipState, RoomMemberEventContent}, StateEventType, @@ -17,7 +18,8 @@ use crate::Ruma; /// /// Submits a signed knock event. pub(crate) async fn create_knock_event_v1_route( - State(services): State, body: Ruma, + State(services): State, + body: Ruma, ) -> Result { if services .globals @@ -26,7 +28,8 @@ pub(crate) async fn create_knock_event_v1_route( .contains(body.origin()) { warn!( - "Server {} tried knocking room ID {} who has a server name that is globally forbidden. Rejecting.", + "Server {} tried knocking room ID {} who has a server name that is globally \ + forbidden. Rejecting.", body.origin(), &body.room_id, ); @@ -41,7 +44,8 @@ pub(crate) async fn create_knock_event_v1_route( .contains(&server.to_owned()) { warn!( - "Server {} tried knocking room ID {} which has a server name that is globally forbidden. Rejecting.", + "Server {} tried knocking room ID {} which has a server name that is globally \ + forbidden. Rejecting.", body.origin(), &body.room_id, ); @@ -50,7 +54,7 @@ pub(crate) async fn create_knock_event_v1_route( } if !services.rooms.metadata.exists(&body.room_id).await { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server.")); + return Err!(Request(NotFound("Room is unknown to this server."))); } // ACL check origin server @@ -74,44 +78,42 @@ pub(crate) async fn create_knock_event_v1_route( let event_type: StateEventType = serde_json::from_value( value .get("type") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing type property."))? + .ok_or_else(|| err!(Request(InvalidParam("Event has no event type."))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event has invalid event type."))?; + .map_err(|e| err!(Request(InvalidParam("Event has invalid event type: {e}"))))?; if event_type != StateEventType::RoomMember { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, + return Err!(Request(InvalidParam( "Not allowed to send non-membership state event to knock endpoint.", - )); + ))); } let content: RoomMemberEventContent = serde_json::from_value( value .get("content") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing content property"))? + .ok_or_else(|| err!(Request(InvalidParam("Membership event has no content"))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Event content is empty or invalid"))?; + .map_err(|e| err!(Request(InvalidParam("Event has invalid membership content: {e}"))))?; if content.membership != MembershipState::Knock { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to send a non-knock membership event to knock endpoint.", - )); + return Err!(Request(InvalidParam( + "Not allowed to send a non-knock membership event to knock endpoint." + ))); } // ACL check sender server name let sender: OwnedUserId = serde_json::from_value( value .get("sender") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing sender property."))? + .ok_or_else(|| err!(Request(InvalidParam("Event has no sender user ID."))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "sender is not a valid user ID."))?; + .map_err(|e| err!(Request(BadJson("Event sender is not a valid user ID: {e}"))))?; services .rooms @@ -127,36 +129,32 @@ pub(crate) async fn create_knock_event_v1_route( let state_key: OwnedUserId = serde_json::from_value( value .get("state_key") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing state_key property."))? + .ok_or_else(|| err!(Request(InvalidParam("Event does not have a state_key"))))? .clone() .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "state_key is invalid or not a user ID."))?; + .map_err(|e| err!(Request(BadJson("Event does not have a valid state_key: {e}"))))?; if state_key != sender { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "State key does not match sender user", - )); + return Err!(Request(InvalidParam("state_key does not match sender user of event."))); }; let origin: OwnedServerName = serde_json::from_value( - serde_json::to_value( - value - .get("origin") - .ok_or_else(|| Error::BadRequest(ErrorKind::InvalidParam, "Event missing origin property."))?, - ) - .expect("CanonicalJson is valid json value"), + value + .get("origin") + .ok_or_else(|| err!(Request(BadJson("Event does not have an origin server name."))))? + .clone() + .into(), ) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "origin is not a server name."))?; + .map_err(|e| err!(Request(BadJson("Event has an invalid origin server name: {e}"))))?; let mut event: JsonObject = serde_json::from_str(body.pdu.get()) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid knock event PDU."))?; + .map_err(|e| err!(Request(InvalidParam("Invalid knock event PDU: {e}"))))?; event.insert("event_id".to_owned(), "$placeholder".into()); let pdu: PduEvent = serde_json::from_value(event.into()) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid knock event PDU."))?; + .map_err(|e| err!(Request(InvalidParam("Invalid knock event PDU: {e}"))))?; let mutex_lock = services .rooms @@ -169,19 +167,18 @@ pub(crate) async fn create_knock_event_v1_route( .rooms .event_handler .handle_incoming_pdu(&origin, &body.room_id, &event_id, value.clone(), true) + .boxed() .await? .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; drop(mutex_lock); - let knock_room_state = services.rooms.state.summary_stripped(&pdu).await; - services .sending .send_pdu_room(&body.room_id, &pdu_id) .await?; - Ok(send_knock::v1::Response { - knock_room_state, - }) + let knock_room_state = services.rooms.state.summary_stripped(&pdu).await; + + Ok(send_knock::v1::Response { knock_room_state }) } diff --git a/src/api/server/utils.rs b/src/api/server/utils.rs index 112cf858..4f3fa245 100644 --- a/src/api/server/utils.rs +++ b/src/api/server/utils.rs @@ -1,6 +1,6 @@ use conduwuit::{implement, is_false, Err, Result}; use conduwuit_service::Services; -use futures::{future::OptionFuture, join, FutureExt}; +use futures::{future::OptionFuture, join, FutureExt, StreamExt}; use ruma::{EventId, RoomId, ServerName}; pub(super) struct AccessCheck<'a> { @@ -31,6 +31,15 @@ pub(super) async fn check(&self) -> Result { .state_cache .server_in_room(self.origin, self.room_id); + // if any user on our homeserver is trying to knock this room, we'll need to + // acknowledge bans or leaves + let user_is_knocking = self + .services + .rooms + .state_cache + .room_members_knocked(self.room_id) + .count(); + let server_can_see: OptionFuture<_> = self .event_id .map(|event_id| { @@ -42,14 +51,14 @@ pub(super) async fn check(&self) -> Result { }) .into(); - let (world_readable, server_in_room, server_can_see, acl_check) = - join!(world_readable, server_in_room, server_can_see, acl_check); + let (world_readable, server_in_room, server_can_see, acl_check, user_is_knocking) = + join!(world_readable, server_in_room, server_can_see, acl_check, user_is_knocking); if !acl_check { return Err!(Request(Forbidden("Server access denied."))); } - if !world_readable && !server_in_room { + if !world_readable && !server_in_room && user_is_knocking == 0 { return Err!(Request(Forbidden("Server is not in room."))); } diff --git a/src/database/maps.rs b/src/database/maps.rs index e9b26818..bc409919 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -184,6 +184,10 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "roomuserid_leftcount", ..descriptor::RANDOM }, + Descriptor { + name: "roomuserid_knockedcount", + ..descriptor::RANDOM_SMALL + }, Descriptor { name: "roomuserid_privateread", ..descriptor::RANDOM_SMALL @@ -377,6 +381,10 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "userroomid_leftstate", ..descriptor::RANDOM }, + Descriptor { + name: "userroomid_knockedstate", + ..descriptor::RANDOM_SMALL + }, Descriptor { name: "userroomid_notificationcount", ..descriptor::RANDOM diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 89421dfd..0d25142d 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -10,7 +10,7 @@ use conduwuit::{ warn, Result, }; use database::{serialize_key, Deserialized, Ignore, Interfix, Json, Map}; -use futures::{future::join4, pin_mut, stream::iter, Stream, StreamExt}; +use futures::{future::join5, pin_mut, stream::iter, Stream, StreamExt}; use itertools::Itertools; use ruma::{ events::{ @@ -51,11 +51,13 @@ struct Data { roomuserid_invitecount: Arc, roomuserid_joined: Arc, roomuserid_leftcount: Arc, + roomuserid_knockedcount: Arc, roomuseroncejoinedids: Arc, serverroomids: Arc, userroomid_invitestate: Arc, userroomid_joined: Arc, userroomid_leftstate: Arc, + userroomid_knockedstate: Arc, } type AppServiceInRoomCache = RwLock>>; @@ -81,11 +83,13 @@ impl crate::Service for Service { roomuserid_invitecount: args.db["roomuserid_invitecount"].clone(), roomuserid_joined: args.db["roomuserid_joined"].clone(), roomuserid_leftcount: args.db["roomuserid_leftcount"].clone(), + roomuserid_knockedcount: args.db["roomuserid_knockedcount"].clone(), roomuseroncejoinedids: args.db["roomuseroncejoinedids"].clone(), serverroomids: args.db["serverroomids"].clone(), userroomid_invitestate: args.db["userroomid_invitestate"].clone(), userroomid_joined: args.db["userroomid_joined"].clone(), userroomid_leftstate: args.db["userroomid_leftstate"].clone(), + userroomid_knockedstate: args.db["userroomid_knockedstate"].clone(), }, })) } @@ -336,6 +340,9 @@ impl Service { self.db.userroomid_leftstate.remove(&userroom_id); self.db.roomuserid_leftcount.remove(&roomuser_id); + self.db.userroomid_knockedstate.remove(&userroom_id); + self.db.roomuserid_knockedcount.remove(&roomuser_id); + self.db.roomid_inviteviaservers.remove(room_id); } @@ -352,12 +359,13 @@ impl Service { // (timo) TODO let leftstate = Vec::>::new(); - let count = self.services.globals.next_count().unwrap(); self.db .userroomid_leftstate .raw_put(&userroom_id, Json(leftstate)); - self.db.roomuserid_leftcount.raw_put(&roomuser_id, count); + self.db + .roomuserid_leftcount + .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); self.db.userroomid_joined.remove(&userroom_id); self.db.roomuserid_joined.remove(&roomuser_id); @@ -365,6 +373,44 @@ impl Service { self.db.userroomid_invitestate.remove(&userroom_id); self.db.roomuserid_invitecount.remove(&roomuser_id); + self.db.userroomid_knockedstate.remove(&userroom_id); + self.db.roomuserid_knockedcount.remove(&roomuser_id); + + self.db.roomid_inviteviaservers.remove(room_id); + } + + /// Direct DB function to directly mark a user as knocked. It is not + /// recommended to use this directly. You most likely should use + /// `update_membership` instead + #[tracing::instrument(skip(self), level = "debug")] + pub fn mark_as_knocked( + &self, + user_id: &UserId, + room_id: &RoomId, + knocked_state: Option>>, + ) { + let userroom_id = (user_id, room_id); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); + + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + + self.db + .userroomid_knockedstate + .raw_put(&userroom_id, Json(knocked_state.unwrap_or_default())); + self.db + .roomuserid_knockedcount + .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); + + self.db.userroomid_joined.remove(&userroom_id); + self.db.roomuserid_joined.remove(&roomuser_id); + + self.db.userroomid_invitestate.remove(&userroom_id); + self.db.roomuserid_invitecount.remove(&roomuser_id); + + self.db.userroomid_leftstate.remove(&userroom_id); + self.db.roomuserid_leftcount.remove(&roomuser_id); + self.db.roomid_inviteviaservers.remove(room_id); } @@ -528,6 +574,20 @@ impl Service { .map(|(_, user_id): (Ignore, &UserId)| user_id) } + /// Returns an iterator over all knocked members of a room. + #[tracing::instrument(skip(self), level = "debug")] + pub fn room_members_knocked<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuserid_knockedcount + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, &UserId)| user_id) + } + #[tracing::instrument(skip(self), level = "trace")] pub async fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { let key = (room_id, user_id); @@ -538,6 +598,16 @@ impl Service { .deserialized() } + #[tracing::instrument(skip(self), level = "trace")] + pub async fn get_knock_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.db + .roomuserid_knockedcount + .qry(&key) + .await + .deserialized() + } + #[tracing::instrument(skip(self), level = "trace")] pub async fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { let key = (room_id, user_id); @@ -576,6 +646,25 @@ impl Service { .ignore_err() } + /// Returns an iterator over all rooms a user is currently knocking. + #[tracing::instrument(skip(self), level = "trace")] + pub fn rooms_knocked<'a>( + &'a self, + user_id: &'a UserId, + ) -> impl Stream + Send + 'a { + type KeyVal<'a> = (Key<'a>, Raw>); + type Key<'a> = (&'a UserId, &'a RoomId); + + let prefix = (user_id, Interfix); + self.db + .userroomid_knockedstate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) + .ignore_err() + } + #[tracing::instrument(skip(self), level = "trace")] pub async fn invite_state( &self, @@ -593,6 +682,23 @@ impl Service { }) } + #[tracing::instrument(skip(self), level = "trace")] + pub async fn knock_state( + &self, + user_id: &UserId, + room_id: &RoomId, + ) -> Result>> { + let key = (user_id, room_id); + self.db + .userroomid_knockedstate + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| { + val.deserialize_as().map_err(Into::into) + }) + } + #[tracing::instrument(skip(self), level = "trace")] pub async fn left_state( &self, @@ -641,6 +747,12 @@ impl Service { self.db.userroomid_joined.qry(&key).await.is_ok() } + #[tracing::instrument(skip(self), level = "trace")] + pub async fn is_knocked<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_knockedstate.qry(&key).await.is_ok() + } + #[tracing::instrument(skip(self), level = "trace")] pub async fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> bool { let key = (user_id, room_id); @@ -659,9 +771,10 @@ impl Service { user_id: &UserId, room_id: &RoomId, ) -> Option { - let states = join4( + let states = join5( self.is_joined(user_id, room_id), self.is_left(user_id, room_id), + self.is_knocked(user_id, room_id), self.is_invited(user_id, room_id), self.once_joined(user_id, room_id), ) @@ -670,8 +783,9 @@ impl Service { match states { | (true, ..) => Some(MembershipState::Join), | (_, true, ..) => Some(MembershipState::Leave), - | (_, _, true, ..) => Some(MembershipState::Invite), - | (false, false, false, true) => Some(MembershipState::Ban), + | (_, _, true, ..) => Some(MembershipState::Knock), + | (_, _, _, true, ..) => Some(MembershipState::Invite), + | (false, false, false, false, true) => Some(MembershipState::Ban), | _ => None, } } @@ -747,6 +861,7 @@ impl Service { pub async fn update_joined_count(&self, room_id: &RoomId) { let mut joinedcount = 0_u64; let mut invitedcount = 0_u64; + let mut knockedcount = 0_u64; let mut joined_servers = HashSet::new(); self.room_members(room_id) @@ -764,8 +879,19 @@ impl Service { .unwrap_or(0), ); + knockedcount = knockedcount.saturating_add( + self.room_members_knocked(room_id) + .count() + .await + .try_into() + .unwrap_or(0), + ); + self.db.roomid_joinedcount.raw_put(room_id, joinedcount); self.db.roomid_invitedcount.raw_put(room_id, invitedcount); + self.db + .roomuserid_knockedcount + .raw_put(room_id, knockedcount); self.room_servers(room_id) .ready_for_each(|old_joined_server| { @@ -820,7 +946,6 @@ impl Service { self.db .userroomid_invitestate .raw_put(&userroom_id, Json(last_state.unwrap_or_default())); - self.db .roomuserid_invitecount .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); @@ -831,6 +956,9 @@ impl Service { self.db.userroomid_leftstate.remove(&userroom_id); self.db.roomuserid_leftcount.remove(&roomuser_id); + self.db.userroomid_knockedstate.remove(&userroom_id); + self.db.roomuserid_knockedcount.remove(&roomuser_id); + if let Some(servers) = invite_via.filter(is_not_empty!()) { self.add_servers_invite_via(room_id, servers).await; } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index fe7f885a..3ebc432f 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -498,14 +498,15 @@ impl Service { .expect("This state_key was previously validated"); let content: RoomMemberEventContent = pdu.get_content()?; - let invite_state = match content.membership { - | MembershipState::Invite => + let stripped_state = match content.membership { + | MembershipState::Invite | MembershipState::Knock => self.services.state.summary_stripped(pdu).await.into(), | _ => None, }; - // Update our membership info, we do this here incase a user is invited - // and immediately leaves we need the DB to record the invite event for auth + // Update our membership info, we do this here incase a user is invited or + // knocked and immediately leaves we need the DB to record the invite or + // knock event for auth self.services .state_cache .update_membership( @@ -513,7 +514,7 @@ impl Service { target_user_id, content, &pdu.sender, - invite_state, + stripped_state, None, true, ) From 9dd058de60cc2a07a61a0c783b8967a779d7881c Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 12 Jan 2025 21:02:03 -0500 Subject: [PATCH 1347/2091] update complement results Signed-off-by: strawberry --- .../complement/test_results.jsonl | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 9b4d2838..11339049 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -1,12 +1,12 @@ {"Action":"pass","Test":"TestACLs"} {"Action":"pass","Test":"TestBannedUserCannotSendJoin"} -{"Action":"fail","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room"} -{"Action":"fail","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/event_with_mismatched_state_key"} -{"Action":"fail","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/invite_event"} -{"Action":"fail","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/join_event"} -{"Action":"fail","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/leave_event"} -{"Action":"fail","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/non-state_membership_event"} -{"Action":"fail","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/regular_event"} +{"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room"} +{"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/event_with_mismatched_state_key"} +{"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/invite_event"} +{"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/join_event"} +{"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/leave_event"} +{"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/non-state_membership_event"} +{"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/regular_event"} {"Action":"pass","Test":"TestCannotSendNonJoinViaSendJoinV1"} {"Action":"pass","Test":"TestCannotSendNonJoinViaSendJoinV1/event_with_mismatched_state_key"} {"Action":"pass","Test":"TestCannotSendNonJoinViaSendJoinV1/invite_event"} @@ -21,13 +21,13 @@ {"Action":"pass","Test":"TestCannotSendNonJoinViaSendJoinV2/leave_event"} {"Action":"pass","Test":"TestCannotSendNonJoinViaSendJoinV2/non-state_membership_event"} {"Action":"pass","Test":"TestCannotSendNonJoinViaSendJoinV2/regular_event"} -{"Action":"fail","Test":"TestCannotSendNonKnockViaSendKnock"} -{"Action":"fail","Test":"TestCannotSendNonKnockViaSendKnock/event_with_mismatched_state_key"} -{"Action":"fail","Test":"TestCannotSendNonKnockViaSendKnock/invite_event"} -{"Action":"fail","Test":"TestCannotSendNonKnockViaSendKnock/join_event"} -{"Action":"fail","Test":"TestCannotSendNonKnockViaSendKnock/leave_event"} -{"Action":"fail","Test":"TestCannotSendNonKnockViaSendKnock/non-state_membership_event"} -{"Action":"fail","Test":"TestCannotSendNonKnockViaSendKnock/regular_event"} +{"Action":"pass","Test":"TestCannotSendNonKnockViaSendKnock"} +{"Action":"pass","Test":"TestCannotSendNonKnockViaSendKnock/event_with_mismatched_state_key"} +{"Action":"pass","Test":"TestCannotSendNonKnockViaSendKnock/invite_event"} +{"Action":"pass","Test":"TestCannotSendNonKnockViaSendKnock/join_event"} +{"Action":"pass","Test":"TestCannotSendNonKnockViaSendKnock/leave_event"} +{"Action":"pass","Test":"TestCannotSendNonKnockViaSendKnock/non-state_membership_event"} +{"Action":"pass","Test":"TestCannotSendNonKnockViaSendKnock/regular_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV1"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV1/event_with_mismatched_state_key"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV1/invite_event"} @@ -90,10 +90,10 @@ {"Action":"fail","Test":"TestKnocking"} {"Action":"fail","Test":"TestKnocking/A_user_can_knock_on_a_room_without_a_reason"} {"Action":"fail","Test":"TestKnocking/A_user_can_knock_on_a_room_without_a_reason#01"} -{"Action":"fail","Test":"TestKnocking/A_user_cannot_knock_on_a_room_they_are_already_in"} -{"Action":"fail","Test":"TestKnocking/A_user_cannot_knock_on_a_room_they_are_already_in#01"} -{"Action":"fail","Test":"TestKnocking/A_user_cannot_knock_on_a_room_they_are_already_invited_to"} -{"Action":"fail","Test":"TestKnocking/A_user_cannot_knock_on_a_room_they_are_already_invited_to#01"} +{"Action":"pass","Test":"TestKnocking/A_user_cannot_knock_on_a_room_they_are_already_in"} +{"Action":"pass","Test":"TestKnocking/A_user_cannot_knock_on_a_room_they_are_already_in#01"} +{"Action":"pass","Test":"TestKnocking/A_user_cannot_knock_on_a_room_they_are_already_invited_to"} +{"Action":"pass","Test":"TestKnocking/A_user_cannot_knock_on_a_room_they_are_already_invited_to#01"} {"Action":"pass","Test":"TestKnocking/A_user_in_the_room_can_accept_a_knock"} {"Action":"pass","Test":"TestKnocking/A_user_in_the_room_can_accept_a_knock#01"} {"Action":"fail","Test":"TestKnocking/A_user_in_the_room_can_reject_a_knock"} @@ -101,25 +101,25 @@ {"Action":"fail","Test":"TestKnocking/A_user_that_has_already_knocked_is_allowed_to_knock_again_on_the_same_room"} {"Action":"fail","Test":"TestKnocking/A_user_that_has_already_knocked_is_allowed_to_knock_again_on_the_same_room#01"} {"Action":"fail","Test":"TestKnocking/A_user_that_has_knocked_on_a_local_room_can_rescind_their_knock_and_then_knock_again"} -{"Action":"fail","Test":"TestKnocking/A_user_that_is_banned_from_a_room_cannot_knock_on_it"} -{"Action":"fail","Test":"TestKnocking/A_user_that_is_banned_from_a_room_cannot_knock_on_it#01"} +{"Action":"pass","Test":"TestKnocking/A_user_that_is_banned_from_a_room_cannot_knock_on_it"} +{"Action":"pass","Test":"TestKnocking/A_user_that_is_banned_from_a_room_cannot_knock_on_it#01"} {"Action":"pass","Test":"TestKnocking/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail"} {"Action":"pass","Test":"TestKnocking/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail#01"} {"Action":"pass","Test":"TestKnocking/Change_the_join_rule_of_a_room_from_'invite'_to_'knock'"} {"Action":"pass","Test":"TestKnocking/Change_the_join_rule_of_a_room_from_'invite'_to_'knock'#01"} -{"Action":"fail","Test":"TestKnocking/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail"} -{"Action":"fail","Test":"TestKnocking/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail#01"} +{"Action":"pass","Test":"TestKnocking/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail"} +{"Action":"pass","Test":"TestKnocking/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail#01"} {"Action":"fail","Test":"TestKnocking/Knocking_on_a_room_with_join_rule_'knock'_should_succeed"} {"Action":"fail","Test":"TestKnocking/Knocking_on_a_room_with_join_rule_'knock'_should_succeed#01"} -{"Action":"fail","Test":"TestKnocking/Users_in_the_room_see_a_user's_membership_update_when_they_knock"} -{"Action":"fail","Test":"TestKnocking/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} +{"Action":"pass","Test":"TestKnocking/Users_in_the_room_see_a_user's_membership_update_when_they_knock"} +{"Action":"pass","Test":"TestKnocking/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_can_knock_on_a_room_without_a_reason"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_can_knock_on_a_room_without_a_reason#01"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_in"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_in#01"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to#01"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_in"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_in#01"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_cannot_knock_on_a_room_they_are_already_invited_to#01"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_in_the_room_can_accept_a_knock#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_in_the_room_can_reject_a_knock"} @@ -127,18 +127,18 @@ {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_that_has_already_knocked_is_allowed_to_knock_again_on_the_same_room"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_that_has_already_knocked_is_allowed_to_knock_again_on_the_same_room#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_that_has_knocked_on_a_local_room_can_rescind_their_knock_and_then_knock_again"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it#01"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/A_user_that_is_banned_from_a_room_cannot_knock_on_it#01"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/Attempting_to_join_a_room_with_join_rule_'knock'_without_an_invite_should_fail#01"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/Change_the_join_rule_of_a_room_from_'invite'_to_'knock'"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/Change_the_join_rule_of_a_room_from_'invite'_to_'knock'#01"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail#01"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_a_join_rule_other_than_'knock'_should_fail#01"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed"} {"Action":"fail","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed#01"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock"} -{"Action":"fail","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock"} +{"Action":"pass","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} {"Action":"pass","Test":"TestLocalPngThumbnail"} {"Action":"pass","Test":"TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint"} {"Action":"pass","Test":"TestLocalPngThumbnail/test_/_matrix/media/v3_endpoint"} From be16f84410c09db478eaa7998f451952bbe0fabe Mon Sep 17 00:00:00 2001 From: morguldir Date: Wed, 6 Nov 2024 03:17:50 +0100 Subject: [PATCH 1348/2091] syncv3: use a function for repeated pattern of fetching sticky params --- src/service/sync/mod.rs | 124 +++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 72 deletions(-) diff --git a/src/service/sync/mod.rs b/src/service/sync/mod.rs index 97f4ce9c..02658a70 100644 --- a/src/service/sync/mod.rs +++ b/src/service/sync/mod.rs @@ -85,6 +85,17 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } +fn list_or_sticky(target: &mut Vec, cached: &Vec) { + if target.is_empty() { + target.clone_from(cached); + } +} +fn some_or_sticky(target: &mut Option, cached: Option) { + if target.is_none() { + *target = cached; + } +} + impl Service { pub fn remembered( &self, @@ -136,57 +147,27 @@ impl Service { for (list_id, list) in &mut request.lists { if let Some(cached_list) = cached.lists.get(list_id) { - if list.sort.is_empty() { - list.sort.clone_from(&cached_list.sort); - }; - if list.room_details.required_state.is_empty() { - list.room_details - .required_state - .clone_from(&cached_list.room_details.required_state); - }; - list.room_details.timeline_limit = list - .room_details - .timeline_limit - .or(cached_list.room_details.timeline_limit); - list.include_old_rooms = list - .include_old_rooms - .clone() - .or_else(|| cached_list.include_old_rooms.clone()); + list_or_sticky(&mut list.sort, &cached_list.sort); + list_or_sticky(&mut list.room_details.required_state, &cached_list.room_details.required_state); + some_or_sticky(&mut list.room_details.timeline_limit, cached_list.room_details.timeline_limit); + some_or_sticky(&mut list.include_old_rooms, cached_list.include_old_rooms.clone()); match (&mut list.filters, cached_list.filters.clone()) { - | (Some(list_filters), Some(cached_filters)) => { - list_filters.is_dm = list_filters.is_dm.or(cached_filters.is_dm); - if list_filters.spaces.is_empty() { - list_filters.spaces = cached_filters.spaces; - } - list_filters.is_encrypted = - list_filters.is_encrypted.or(cached_filters.is_encrypted); - list_filters.is_invite = - list_filters.is_invite.or(cached_filters.is_invite); - if list_filters.room_types.is_empty() { - list_filters.room_types = cached_filters.room_types; - } - if list_filters.not_room_types.is_empty() { - list_filters.not_room_types = cached_filters.not_room_types; - } - list_filters.room_name_like = list_filters - .room_name_like - .clone() - .or(cached_filters.room_name_like); - if list_filters.tags.is_empty() { - list_filters.tags = cached_filters.tags; - } - if list_filters.not_tags.is_empty() { - list_filters.not_tags = cached_filters.not_tags; - } + (Some(list_filters), Some(cached_filters)) => { + some_or_sticky(&mut list_filters.is_dm, cached_filters.is_dm); + list_or_sticky(&mut list_filters.spaces, &cached_filters.spaces); + some_or_sticky(&mut list_filters.is_encrypted, cached_filters.is_encrypted); + some_or_sticky(&mut list_filters.is_invite, cached_filters.is_invite); + list_or_sticky(&mut list_filters.room_types, &cached_filters.room_types); + list_or_sticky(&mut list_filters.not_room_types, &cached_filters.not_room_types); + some_or_sticky(&mut list_filters.room_name_like, cached_filters.room_name_like); + list_or_sticky(&mut list_filters.tags, &cached_filters.tags); + list_or_sticky(&mut list_filters.not_tags, &cached_filters.not_tags); }, | (_, Some(cached_filters)) => list.filters = Some(cached_filters), | (Some(list_filters), _) => list.filters = Some(list_filters.clone()), | (..) => {}, } - if list.bump_event_types.is_empty() { - list.bump_event_types - .clone_from(&cached_list.bump_event_types); - }; + list_or_sticky(&mut list.bump_event_types, &cached_list.bump_event_types); } cached.lists.insert(list_id.clone(), list.clone()); } @@ -241,16 +222,18 @@ impl Service { subscriptions: BTreeMap, ) { let mut cache = self.connections.lock().expect("locked"); - let cached = Arc::clone(cache.entry((user_id, device_id, conn_id)).or_insert_with( - || { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }, - )); + let cached = Arc::clone( + cache + .entry((user_id, device_id, conn_id)) + .or_insert_with(|| { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + }), + ); let cached = &mut cached.lock().expect("locked"); drop(cache); @@ -258,25 +241,22 @@ impl Service { } pub fn update_sync_known_rooms( - &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, - conn_id: String, - list_id: String, - new_cached_rooms: BTreeSet, - globalsince: u64, + &self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String, list_id: String, + new_cached_rooms: BTreeSet, globalsince: u64, ) { let mut cache = self.connections.lock().expect("locked"); - let cached = Arc::clone(cache.entry((user_id, device_id, conn_id)).or_insert_with( - || { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }, - )); + let cached = Arc::clone( + cache + .entry((user_id, device_id, conn_id)) + .or_insert_with(|| { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + }), + ); let cached = &mut cached.lock().expect("locked"); drop(cache); From 6cb3275be0828d7d64da30bacb759b8c796c3c99 Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 3 Jan 2025 08:32:54 +0100 Subject: [PATCH 1349/2091] Add initial MSC4186 (Simplified Sliding Sync) implementation Signed-off-by: morguldir Signed-off-by: strawberry --- src/api/client/sync/mod.rs | 51 +- src/api/client/sync/v4.rs | 94 +-- src/api/client/sync/v5.rs | 886 ++++++++++++++++++++++++++ src/api/client/unversioned.rs | 1 + src/api/router.rs | 1 + src/service/rooms/read_receipt/mod.rs | 1 + src/service/sync/mod.rs | 273 +++++++- 7 files changed, 1214 insertions(+), 93 deletions(-) create mode 100644 src/api/client/sync/v5.rs diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 79e4b1ca..1967f4a2 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -1,16 +1,31 @@ mod v3; mod v4; +mod v5; use conduwuit::{ - utils::stream::{BroadbandExt, ReadyExt, TryIgnore}, + utils::{ + stream::{BroadbandExt, ReadyExt, TryIgnore}, + IterStream, + }, PduCount, }; use futures::{pin_mut, StreamExt}; -use ruma::{RoomId, UserId}; +use ruma::{ + directory::RoomTypeFilter, + events::TimelineEventType::{ + self, Beacon, CallInvite, PollStart, RoomEncrypted, RoomMessage, Sticker, + }, + RoomId, UserId, +}; -pub(crate) use self::{v3::sync_events_route, v4::sync_events_v4_route}; +pub(crate) use self::{ + v3::sync_events_route, v4::sync_events_v4_route, v5::sync_events_v5_route, +}; use crate::{service::Services, Error, PduEvent, Result}; +pub(crate) const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = + &[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker]; + async fn load_timeline( services: &Services, sender_user: &UserId, @@ -69,3 +84,33 @@ async fn share_encrypted_room( }) .await } + +pub(crate) async fn filter_rooms<'a>( + services: &Services, + rooms: &[&'a RoomId], + filter: &[RoomTypeFilter], + negate: bool, +) -> Vec<&'a RoomId> { + rooms + .iter() + .stream() + .filter_map(|r| async move { + let room_type = services.rooms.state_accessor.get_room_type(r).await; + + if room_type.as_ref().is_err_and(|e| !e.is_not_found()) { + return None; + } + + let room_type_filter = RoomTypeFilter::from(room_type.ok()); + + let include = if negate { + !filter.contains(&room_type_filter) + } else { + filter.is_empty() || filter.contains(&room_type_filter) + }; + + include.then_some(r) + }) + .collect() + .await +} diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 24c7e286..a82e9309 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -23,24 +23,23 @@ use ruma::{ DeviceLists, UnreadNotificationsCount, }, }, - directory::RoomTypeFilter, events::{ room::member::{MembershipState, RoomMemberEventContent}, AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, - TimelineEventType::{self, *}, + TimelineEventType::*, }, serde::Raw, - uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, UInt, + uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UInt, }; -use service::{rooms::read_receipt::pack_receipts, Services}; +use service::rooms::read_receipt::pack_receipts; use super::{load_timeline, share_encrypted_room}; -use crate::{client::ignored_filter, Ruma}; +use crate::{ + client::{filter_rooms, ignored_filter, sync::v5::TodoRooms, DEFAULT_BUMP_TYPES}, + Ruma, +}; -const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; - -const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = - &[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker]; +pub(crate) const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; /// POST `/_matrix/client/unstable/org.matrix.msc3575/sync` /// @@ -121,13 +120,19 @@ pub(crate) async fn sync_events_v4_route( .collect() .await; - let all_rooms = all_joined_rooms + let all_invited_rooms: Vec<&RoomId> = all_invited_rooms.iter().map(AsRef::as_ref).collect(); + let all_knocked_rooms: Vec<&RoomId> = all_knocked_rooms.iter().map(AsRef::as_ref).collect(); + + let all_rooms: Vec<&RoomId> = all_joined_rooms .iter() - .chain(all_invited_rooms.iter()) - .chain(all_knocked_rooms.iter()) - .map(Clone::clone) + .map(AsRef::as_ref) + .chain(all_invited_rooms.iter().map(AsRef::as_ref)) + .chain(all_knocked_rooms.iter().map(AsRef::as_ref)) .collect(); + let all_joined_rooms = all_joined_rooms.iter().map(AsRef::as_ref).collect(); + let all_invited_rooms = all_invited_rooms.iter().map(AsRef::as_ref).collect(); + if body.extensions.to_device.enabled.unwrap_or(false) { services .users @@ -180,6 +185,7 @@ pub(crate) async fn sync_events_v4_route( ); for room_id in &all_joined_rooms { + let room_id: &&RoomId = room_id; let Ok(current_shortstatehash) = services.rooms.state.get_room_shortstatehash(room_id).await else { @@ -332,7 +338,7 @@ pub(crate) async fn sync_events_v4_route( } let mut lists = BTreeMap::new(); - let mut todo_rooms = BTreeMap::new(); // and required state + let mut todo_rooms: TodoRooms = BTreeMap::new(); // and required state for (list_id, list) in &body.lists { let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) { @@ -353,7 +359,7 @@ pub(crate) async fn sync_events_v4_route( | None => active_rooms, }; - let mut new_known_rooms = BTreeSet::new(); + let mut new_known_rooms: BTreeSet = BTreeSet::new(); let ranges = list.ranges.clone(); lists.insert(list_id.clone(), sync_events::v4::SyncList { @@ -375,9 +381,9 @@ pub(crate) async fn sync_events_v4_route( Vec::new() }; - new_known_rooms.extend(room_ids.iter().cloned()); + new_known_rooms.extend(room_ids.clone().into_iter().map(ToOwned::to_owned)); for room_id in &room_ids { - let todo_room = todo_rooms.entry(room_id.clone()).or_insert(( + let todo_room = todo_rooms.entry((*room_id).to_owned()).or_insert(( BTreeSet::new(), 0_usize, u64::MAX, @@ -399,7 +405,7 @@ pub(crate) async fn sync_events_v4_route( todo_room.2 = todo_room.2.min( known_rooms .get(list_id.as_str()) - .and_then(|k| k.get(room_id)) + .and_then(|k| k.get(*room_id)) .copied() .unwrap_or(0), ); @@ -408,7 +414,7 @@ pub(crate) async fn sync_events_v4_route( op: SlidingOp::Sync, range: Some(r), index: None, - room_ids, + room_ids: room_ids.into_iter().map(ToOwned::to_owned).collect(), room_id: None, } }) @@ -418,8 +424,8 @@ pub(crate) async fn sync_events_v4_route( if let Some(conn_id) = &body.conn_id { services.sync.update_sync_known_rooms( - sender_user.clone(), - sender_device.clone(), + sender_user, + &sender_device, conn_id.clone(), list_id.clone(), new_known_rooms, @@ -464,8 +470,8 @@ pub(crate) async fn sync_events_v4_route( if let Some(conn_id) = &body.conn_id { services.sync.update_sync_known_rooms( - sender_user.clone(), - sender_device.clone(), + sender_user, + &sender_device, conn_id.clone(), "subscriptions".to_owned(), known_subscription_rooms, @@ -489,7 +495,8 @@ pub(crate) async fn sync_events_v4_route( let mut timestamp: Option<_> = None; let mut invite_state = None; let (timeline_pdus, limited); - if all_invited_rooms.contains(room_id) { + let new_room_id: &RoomId = (*room_id).as_ref(); + if all_invited_rooms.contains(&new_room_id) { // TODO: figure out a timestamp we can use for remote invites invite_state = services .rooms @@ -519,7 +526,7 @@ pub(crate) async fn sync_events_v4_route( } account_data.rooms.insert( - room_id.clone(), + room_id.to_owned(), services .account_data .changes_since(Some(room_id), sender_user, *roomsince) @@ -749,10 +756,9 @@ pub(crate) async fn sync_events_v4_route( }); } - if rooms - .iter() - .all(|(_, r)| r.timeline.is_empty() && r.required_state.is_empty()) - { + if rooms.iter().all(|(id, r)| { + r.timeline.is_empty() && r.required_state.is_empty() && !receipts.rooms.contains_key(id) + }) { // Hang a few seconds so requests are not spammed // Stop hanging if new info arrives let default = Duration::from_secs(30); @@ -798,33 +804,3 @@ pub(crate) async fn sync_events_v4_route( delta_token: None, }) } - -async fn filter_rooms( - services: &Services, - rooms: &[OwnedRoomId], - filter: &[RoomTypeFilter], - negate: bool, -) -> Vec { - rooms - .iter() - .stream() - .filter_map(|r| async move { - let room_type = services.rooms.state_accessor.get_room_type(r).await; - - if room_type.as_ref().is_err_and(|e| !e.is_not_found()) { - return None; - } - - let room_type_filter = RoomTypeFilter::from(room_type.ok()); - - let include = if negate { - !filter.contains(&room_type_filter) - } else { - filter.is_empty() || filter.contains(&room_type_filter) - }; - - include.then_some(r.to_owned()) - }) - .collect() - .await -} diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs new file mode 100644 index 00000000..1c4f3504 --- /dev/null +++ b/src/api/client/sync/v5.rs @@ -0,0 +1,886 @@ +use std::{ + cmp::{self, Ordering}, + collections::{BTreeMap, BTreeSet, HashMap, HashSet}, + time::Duration, +}; + +use axum::extract::State; +use conduwuit::{ + debug, error, extract_variant, trace, + utils::{ + math::{ruma_from_usize, usize_from_ruma}, + BoolExt, IterStream, ReadyExt, TryFutureExtExt, + }, + warn, Error, Result, +}; +use futures::{FutureExt, StreamExt, TryFutureExt}; +use ruma::{ + api::client::{ + error::ErrorKind, + sync::sync_events::{self, DeviceLists, UnreadNotificationsCount}, + }, + events::{ + room::member::{MembershipState, RoomMemberEventContent}, + AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType, + }, + serde::Raw, + state_res::TypeStateKey, + uint, DeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId, +}; +use service::{rooms::read_receipt::pack_receipts, PduCount}; + +use super::{filter_rooms, share_encrypted_room}; +use crate::{ + client::{ignored_filter, sync::load_timeline, DEFAULT_BUMP_TYPES}, + Ruma, +}; + +type SyncInfo<'a> = (&'a UserId, &'a DeviceId, u64, &'a sync_events::v5::Request); + +/// `POST /_matrix/client/unstable/org.matrix.simplified_msc3575/sync` +/// ([MSC4186]) +/// +/// A simplified version of sliding sync ([MSC3575]). +/// +/// Get all new events in a sliding window of rooms since the last sync or a +/// given point in time. +/// +/// [MSC3575]: https://github.com/matrix-org/matrix-spec-proposals/pull/3575 +/// [MSC4186]: https://github.com/matrix-org/matrix-spec-proposals/pull/4186 +pub(crate) async fn sync_events_v5_route( + State(services): State, + body: Ruma, +) -> Result { + debug_assert!(DEFAULT_BUMP_TYPES.is_sorted(), "DEFAULT_BUMP_TYPES is not sorted"); + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + let mut body = body.body; + + // Setup watchers, so if there's no response, we can wait for them + let watcher = services.sync.watch(sender_user, sender_device); + + let next_batch = services.globals.next_count()?; + + let conn_id = body.conn_id.clone(); + + let globalsince = body + .pos + .as_ref() + .and_then(|string| string.parse().ok()) + .unwrap_or(0); + + if globalsince != 0 + && !services.sync.snake_connection_cached( + sender_user.clone(), + sender_device.clone(), + conn_id.clone(), + ) { + debug!("Restarting sync stream because it was gone from the database"); + return Err(Error::Request( + ErrorKind::UnknownPos, + "Connection data lost since last time".into(), + http::StatusCode::BAD_REQUEST, + )); + } + + // Client / User requested an initial sync + if globalsince == 0 { + services.sync.forget_snake_sync_connection( + sender_user.clone(), + sender_device.clone(), + conn_id.clone(), + ); + } + + // Get sticky parameters from cache + let known_rooms = services.sync.update_snake_sync_request_with_cache( + sender_user.clone(), + sender_device.clone(), + &mut body, + ); + + let all_joined_rooms: Vec<_> = services + .rooms + .state_cache + .rooms_joined(sender_user) + .map(ToOwned::to_owned) + .collect() + .await; + + let all_invited_rooms: Vec<_> = services + .rooms + .state_cache + .rooms_invited(sender_user) + .map(|r| r.0) + .collect() + .await; + + let all_knocked_rooms: Vec<_> = services + .rooms + .state_cache + .rooms_knocked(sender_user) + .map(|r| r.0) + .collect() + .await; + + let all_rooms: Vec<&RoomId> = all_joined_rooms + .iter() + .map(AsRef::as_ref) + .chain(all_invited_rooms.iter().map(AsRef::as_ref)) + .chain(all_knocked_rooms.iter().map(AsRef::as_ref)) + .collect(); + + let all_joined_rooms = all_joined_rooms.iter().map(AsRef::as_ref).collect(); + let all_invited_rooms = all_invited_rooms.iter().map(AsRef::as_ref).collect(); + + let pos = next_batch.clone().to_string(); + + let mut todo_rooms: TodoRooms = BTreeMap::new(); + + let sync_info: SyncInfo<'_> = (sender_user, sender_device, globalsince, &body); + let mut response = sync_events::v5::Response { + txn_id: body.txn_id.clone(), + pos, + lists: BTreeMap::new(), + rooms: BTreeMap::new(), + extensions: sync_events::v5::response::Extensions { + account_data: collect_account_data(services, sync_info).await, + e2ee: collect_e2ee(services, sync_info, &all_joined_rooms).await?, + to_device: collect_to_device(services, sync_info, next_batch).await, + receipts: collect_receipts(services).await, + typing: sync_events::v5::response::Typing::default(), + }, + }; + + handle_lists( + services, + sync_info, + &all_invited_rooms, + &all_joined_rooms, + &all_rooms, + &mut todo_rooms, + &known_rooms, + &mut response, + ) + .await; + + fetch_subscriptions(services, sync_info, &known_rooms, &mut todo_rooms).await; + + response.rooms = process_rooms( + services, + sender_user, + next_batch, + &all_invited_rooms, + &todo_rooms, + &mut response, + &body, + ) + .await?; + + if response.rooms.iter().all(|(id, r)| { + r.timeline.is_empty() + && r.required_state.is_empty() + && !response.extensions.receipts.rooms.contains_key(id) + }) && response + .extensions + .to_device + .clone() + .is_none_or(|to| to.events.is_empty()) + { + // Hang a few seconds so requests are not spammed + // Stop hanging if new info arrives + let default = Duration::from_secs(30); + let duration = cmp::min(body.timeout.unwrap_or(default), default); + _ = tokio::time::timeout(duration, watcher).await; + } + + trace!( + rooms=?response.rooms.len(), + account_data=?response.extensions.account_data.rooms.len(), + receipts=?response.extensions.receipts.rooms.len(), + "responding to request with" + ); + Ok(response) +} + +type KnownRooms = BTreeMap>; +pub(crate) type TodoRooms = BTreeMap, usize, u64)>; + +async fn fetch_subscriptions( + services: crate::State, + (sender_user, sender_device, globalsince, body): SyncInfo<'_>, + known_rooms: &KnownRooms, + todo_rooms: &mut TodoRooms, +) { + let mut known_subscription_rooms = BTreeSet::new(); + for (room_id, room) in &body.room_subscriptions { + if !services.rooms.metadata.exists(room_id).await { + continue; + } + let todo_room = + todo_rooms + .entry(room_id.clone()) + .or_insert((BTreeSet::new(), 0_usize, u64::MAX)); + + let limit: UInt = room.timeline_limit; + + todo_room.0.extend(room.required_state.iter().cloned()); + todo_room.1 = todo_room.1.max(usize_from_ruma(limit)); + // 0 means unknown because it got out of date + todo_room.2 = todo_room.2.min( + known_rooms + .get("subscriptions") + .and_then(|k| k.get(room_id)) + .copied() + .unwrap_or(0), + ); + known_subscription_rooms.insert(room_id.clone()); + } + // where this went (protomsc says it was removed) + //for r in body.unsubscribe_rooms { + // known_subscription_rooms.remove(&r); + // body.room_subscriptions.remove(&r); + //} + + if let Some(conn_id) = &body.conn_id { + services.sync.update_snake_sync_known_rooms( + sender_user, + sender_device, + conn_id.clone(), + "subscriptions".to_owned(), + known_subscription_rooms, + globalsince, + ); + } +} + +#[allow(clippy::too_many_arguments)] +async fn handle_lists<'a>( + services: crate::State, + (sender_user, sender_device, globalsince, body): SyncInfo<'_>, + all_invited_rooms: &Vec<&'a RoomId>, + all_joined_rooms: &Vec<&'a RoomId>, + all_rooms: &Vec<&'a RoomId>, + todo_rooms: &'a mut TodoRooms, + known_rooms: &'a KnownRooms, + response: &'_ mut sync_events::v5::Response, +) -> KnownRooms { + for (list_id, list) in &body.lists { + let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) { + | Some(true) => all_invited_rooms, + | Some(false) => all_joined_rooms, + | None => all_rooms, + }; + + let active_rooms = match list.filters.clone().map(|f| f.not_room_types) { + | Some(filter) if filter.is_empty() => active_rooms, + | Some(value) => &filter_rooms(&services, active_rooms, &value, true).await, + | None => active_rooms, + }; + + let mut new_known_rooms: BTreeSet = BTreeSet::new(); + + let ranges = list.ranges.clone(); + + for mut range in ranges { + range.0 = uint!(0); + range.1 = range + .1 + .clamp(range.0, UInt::try_from(active_rooms.len()).unwrap_or(UInt::MAX)); + + let room_ids = + active_rooms[usize_from_ruma(range.0)..usize_from_ruma(range.1)].to_vec(); + + let new_rooms: BTreeSet = + room_ids.clone().into_iter().map(From::from).collect(); + new_known_rooms.extend(new_rooms); + //new_known_rooms.extend(room_ids..cloned()); + for room_id in room_ids { + let todo_room = todo_rooms.entry(room_id.to_owned()).or_insert(( + BTreeSet::new(), + 0_usize, + u64::MAX, + )); + + let limit: usize = usize_from_ruma(list.room_details.timeline_limit).min(100); + + todo_room + .0 + .extend(list.room_details.required_state.iter().cloned()); + + todo_room.1 = todo_room.1.max(limit); + // 0 means unknown because it got out of date + todo_room.2 = todo_room.2.min( + known_rooms + .get(list_id.as_str()) + .and_then(|k| k.get(room_id)) + .copied() + .unwrap_or(0), + ); + } + } + response + .lists + .insert(list_id.clone(), sync_events::v5::response::List { + count: ruma_from_usize(active_rooms.len()), + }); + + if let Some(conn_id) = &body.conn_id { + services.sync.update_snake_sync_known_rooms( + sender_user, + sender_device, + conn_id.clone(), + list_id.clone(), + new_known_rooms, + globalsince, + ); + } + } + BTreeMap::default() +} + +async fn process_rooms( + services: crate::State, + sender_user: &UserId, + next_batch: u64, + all_invited_rooms: &[&RoomId], + todo_rooms: &TodoRooms, + response: &mut sync_events::v5::Response, + body: &sync_events::v5::Request, +) -> Result> { + let mut rooms = BTreeMap::new(); + for (room_id, (required_state_request, timeline_limit, roomsince)) in todo_rooms { + let roomsincecount = PduCount::Normal(*roomsince); + + let mut timestamp: Option<_> = None; + let mut invite_state = None; + let (timeline_pdus, limited); + let new_room_id: &RoomId = (*room_id).as_ref(); + if all_invited_rooms.contains(&new_room_id) { + // TODO: figure out a timestamp we can use for remote invites + invite_state = services + .rooms + .state_cache + .invite_state(sender_user, room_id) + .await + .ok(); + + (timeline_pdus, limited) = (Vec::new(), true); + } else { + (timeline_pdus, limited) = match load_timeline( + &services, + sender_user, + room_id, + roomsincecount, + Some(PduCount::from(next_batch)), + *timeline_limit, + ) + .await + { + | Ok(value) => value, + | Err(err) => { + warn!("Encountered missing timeline in {}, error {}", room_id, err); + continue; + }, + }; + } + + if body.extensions.account_data.enabled == Some(true) { + response.extensions.account_data.rooms.insert( + room_id.to_owned(), + services + .account_data + .changes_since(Some(room_id), sender_user, *roomsince) + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect() + .await, + ); + } + + let last_privateread_update = services + .rooms + .read_receipt + .last_privateread_update(sender_user, room_id) + .await > *roomsince; + + let private_read_event = if last_privateread_update { + services + .rooms + .read_receipt + .private_read_get(room_id, sender_user) + .await + .ok() + } else { + None + }; + + let mut receipts: Vec> = services + .rooms + .read_receipt + .readreceipts_since(room_id, *roomsince) + .filter_map(|(read_user, _ts, v)| async move { + services + .users + .user_is_ignored(read_user, sender_user) + .await + .or_some(v) + }) + .collect() + .await; + + if let Some(private_read_event) = private_read_event { + receipts.push(private_read_event); + } + + let receipt_size = receipts.len(); + + if receipt_size > 0 { + response + .extensions + .receipts + .rooms + .insert(room_id.clone(), pack_receipts(Box::new(receipts.into_iter()))); + } + + if roomsince != &0 + && timeline_pdus.is_empty() + && response + .extensions + .account_data + .rooms + .get(room_id) + .is_none_or(Vec::is_empty) + && receipt_size == 0 + { + continue; + } + + let prev_batch = timeline_pdus + .first() + .map_or(Ok::<_, Error>(None), |(pdu_count, _)| { + Ok(Some(match pdu_count { + | PduCount::Backfilled(_) => { + error!("timeline in backfill state?!"); + "0".to_owned() + }, + | PduCount::Normal(c) => c.to_string(), + })) + })? + .or_else(|| { + if roomsince != &0 { + Some(roomsince.to_string()) + } else { + None + } + }); + + let room_events: Vec<_> = timeline_pdus + .iter() + .stream() + .filter_map(|item| ignored_filter(&services, item.clone(), sender_user)) + .map(|(_, pdu)| pdu.to_sync_room_event()) + .collect() + .await; + + for (_, pdu) in timeline_pdus { + let ts = pdu.origin_server_ts; + if DEFAULT_BUMP_TYPES.binary_search(&pdu.kind).is_ok() + && timestamp.is_none_or(|time| time <= ts) + { + timestamp = Some(ts); + } + } + + let required_state = required_state_request + .iter() + .stream() + .filter_map(|state| async move { + services + .rooms + .state_accessor + .room_state_get(room_id, &state.0, &state.1) + .await + .map(|s| s.to_sync_state_event()) + .ok() + }) + .collect() + .await; + + // Heroes + let heroes: Vec<_> = services + .rooms + .state_cache + .room_members(room_id) + .ready_filter(|member| *member != sender_user) + .filter_map(|user_id| { + services + .rooms + .state_accessor + .get_member(room_id, user_id) + .map_ok(|memberevent| sync_events::v5::response::Hero { + user_id: user_id.into(), + name: memberevent.displayname, + avatar: memberevent.avatar_url, + }) + .ok() + }) + .take(5) + .collect() + .await; + + let name = match heroes.len().cmp(&(1_usize)) { + | Ordering::Greater => { + let firsts = heroes[1..] + .iter() + .map(|h| h.name.clone().unwrap_or_else(|| h.user_id.to_string())) + .collect::>() + .join(", "); + + let last = heroes[0] + .name + .clone() + .unwrap_or_else(|| heroes[0].user_id.to_string()); + + Some(format!("{firsts} and {last}")) + }, + | Ordering::Equal => Some( + heroes[0] + .name + .clone() + .unwrap_or_else(|| heroes[0].user_id.to_string()), + ), + | Ordering::Less => None, + }; + + let heroes_avatar = if heroes.len() == 1 { + heroes[0].avatar.clone() + } else { + None + }; + + rooms.insert(room_id.clone(), sync_events::v5::response::Room { + name: services + .rooms + .state_accessor + .get_name(room_id) + .await + .ok() + .or(name), + avatar: if let Some(heroes_avatar) = heroes_avatar { + ruma::JsOption::Some(heroes_avatar) + } else { + match services.rooms.state_accessor.get_avatar(room_id).await { + | ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url), + | ruma::JsOption::Null => ruma::JsOption::Null, + | ruma::JsOption::Undefined => ruma::JsOption::Undefined, + } + }, + initial: Some(roomsince == &0), + is_dm: None, + invite_state, + unread_notifications: UnreadNotificationsCount { + highlight_count: Some( + services + .rooms + .user + .highlight_count(sender_user, room_id) + .await + .try_into() + .expect("notification count can't go that high"), + ), + notification_count: Some( + services + .rooms + .user + .notification_count(sender_user, room_id) + .await + .try_into() + .expect("notification count can't go that high"), + ), + }, + timeline: room_events, + required_state, + prev_batch, + limited, + joined_count: Some( + services + .rooms + .state_cache + .room_joined_count(room_id) + .await + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), + ), + invited_count: Some( + services + .rooms + .state_cache + .room_invited_count(room_id) + .await + .unwrap_or(0) + .try_into() + .unwrap_or_else(|_| uint!(0)), + ), + num_live: None, // Count events in timeline greater than global sync counter + bump_stamp: timestamp, + heroes: Some(heroes), + }); + } + Ok(rooms) +} +async fn collect_account_data( + services: crate::State, + (sender_user, _, globalsince, body): (&UserId, &DeviceId, u64, &sync_events::v5::Request), +) -> sync_events::v5::response::AccountData { + let mut account_data = sync_events::v5::response::AccountData { + global: Vec::new(), + rooms: BTreeMap::new(), + }; + + if !body.extensions.account_data.enabled.unwrap_or(false) { + return sync_events::v5::response::AccountData::default(); + } + + account_data.global = services + .account_data + .changes_since(None, sender_user, globalsince) + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) + .collect() + .await; + + if let Some(rooms) = &body.extensions.account_data.rooms { + for room in rooms { + account_data.rooms.insert( + room.clone(), + services + .account_data + .changes_since(Some(room), sender_user, globalsince) + .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) + .collect() + .await, + ); + } + } + + account_data +} + +async fn collect_e2ee<'a>( + services: crate::State, + (sender_user, sender_device, globalsince, body): ( + &UserId, + &DeviceId, + u64, + &sync_events::v5::Request, + ), + all_joined_rooms: &'a Vec<&'a RoomId>, +) -> Result { + if !body.extensions.e2ee.enabled.unwrap_or(false) { + return Ok(sync_events::v5::response::E2EE::default()); + } + let mut left_encrypted_users = HashSet::new(); // Users that have left any encrypted rooms the sender was in + let mut device_list_changes = HashSet::new(); + let mut device_list_left = HashSet::new(); + // Look for device list updates of this account + device_list_changes.extend( + services + .users + .keys_changed(sender_user, globalsince, None) + .map(ToOwned::to_owned) + .collect::>() + .await, + ); + + for room_id in all_joined_rooms { + let Ok(current_shortstatehash) = + services.rooms.state.get_room_shortstatehash(room_id).await + else { + error!("Room {room_id} has no state"); + continue; + }; + + let since_shortstatehash = services + .rooms + .user + .get_token_shortstatehash(room_id, globalsince) + .await + .ok(); + + let encrypted_room = services + .rooms + .state_accessor + .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") + .await + .is_ok(); + + if let Some(since_shortstatehash) = since_shortstatehash { + // Skip if there are only timeline changes + if since_shortstatehash == current_shortstatehash { + continue; + } + + let since_encryption = services + .rooms + .state_accessor + .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") + .await; + + let since_sender_member: Option = services + .rooms + .state_accessor + .state_get_content( + since_shortstatehash, + &StateEventType::RoomMember, + sender_user.as_str(), + ) + .ok() + .await; + + let joined_since_last_sync = since_sender_member + .as_ref() + .is_none_or(|member| member.membership != MembershipState::Join); + + let new_encrypted_room = encrypted_room && since_encryption.is_err(); + + if encrypted_room { + let current_state_ids: HashMap<_, OwnedEventId> = services + .rooms + .state_accessor + .state_full_ids(current_shortstatehash) + .await?; + + let since_state_ids = services + .rooms + .state_accessor + .state_full_ids(since_shortstatehash) + .await?; + + for (key, id) in current_state_ids { + if since_state_ids.get(&key) != Some(&id) { + let Ok(pdu) = services.rooms.timeline.get_pdu(&id).await else { + error!("Pdu in state not found: {id}"); + continue; + }; + if pdu.kind == TimelineEventType::RoomMember { + if let Some(state_key) = &pdu.state_key { + let user_id = + OwnedUserId::parse(state_key.clone()).map_err(|_| { + Error::bad_database("Invalid UserId in member PDU.") + })?; + + if user_id == *sender_user { + continue; + } + + let content: RoomMemberEventContent = pdu.get_content()?; + match content.membership { + | MembershipState::Join => { + // A new user joined an encrypted room + if !share_encrypted_room( + &services, + sender_user, + &user_id, + Some(room_id), + ) + .await + { + device_list_changes.insert(user_id); + } + }, + | MembershipState::Leave => { + // Write down users that have left encrypted rooms we + // are in + left_encrypted_users.insert(user_id); + }, + | _ => {}, + } + } + } + } + } + if joined_since_last_sync || new_encrypted_room { + // If the user is in a new encrypted room, give them all joined users + device_list_changes.extend( + services + .rooms + .state_cache + .room_members(room_id) + // Don't send key updates from the sender to the sender + .ready_filter(|user_id| sender_user != *user_id) + // Only send keys if the sender doesn't share an encrypted room with the target + // already + .filter_map(|user_id| { + share_encrypted_room(&services, sender_user, user_id, Some(room_id)) + .map(|res| res.or_some(user_id.to_owned())) + }) + .collect::>() + .await, + ); + } + } + } + // Look for device list updates in this room + device_list_changes.extend( + services + .users + .room_keys_changed(room_id, globalsince, None) + .map(|(user_id, _)| user_id) + .map(ToOwned::to_owned) + .collect::>() + .await, + ); + } + + for user_id in left_encrypted_users { + let dont_share_encrypted_room = + !share_encrypted_room(&services, sender_user, &user_id, None).await; + + // If the user doesn't share an encrypted room with the target anymore, we need + // to tell them + if dont_share_encrypted_room { + device_list_left.insert(user_id); + } + } + + Ok(sync_events::v5::response::E2EE { + device_lists: DeviceLists { + changed: device_list_changes.into_iter().collect(), + left: device_list_left.into_iter().collect(), + }, + device_one_time_keys_count: services + .users + .count_one_time_keys(sender_user, sender_device) + .await, + device_unused_fallback_key_types: None, + }) +} + +async fn collect_to_device( + services: crate::State, + (sender_user, sender_device, globalsince, body): SyncInfo<'_>, + next_batch: u64, +) -> Option { + if !body.extensions.to_device.enabled.unwrap_or(false) { + return None; + } + + services + .users + .remove_to_device_events(sender_user, sender_device, globalsince) + .await; + + Some(sync_events::v5::response::ToDevice { + next_batch: next_batch.to_string(), + events: services + .users + .get_to_device_events(sender_user, sender_device) + .collect() + .await, + }) +} + +async fn collect_receipts(_services: crate::State) -> sync_events::v5::response::Receipts { + sync_events::v5::response::Receipts { rooms: BTreeMap::new() } + // TODO: get explicitly requested read receipts +} diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index b4856d72..904f1d2f 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -52,6 +52,7 @@ pub(crate) async fn get_supported_versions_route( ("org.matrix.msc4180".to_owned(), true), /* stable flag for 3916 (https://github.com/matrix-org/matrix-spec-proposals/pull/4180) */ ("uk.tcpip.msc4133".to_owned(), true), /* Extending User Profile API with Key:Value Pairs (https://github.com/matrix-org/matrix-spec-proposals/pull/4133) */ ("us.cloke.msc4175".to_owned(), true), /* Profile field for user time zone (https://github.com/matrix-org/matrix-spec-proposals/pull/4175) */ + ("org.matrix.simplified_msc3575".to_owned(), true), /* Simplified Sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/4186) */ ]), }; diff --git a/src/api/router.rs b/src/api/router.rs index 1d42fc5e..e7cd368d 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -145,6 +145,7 @@ pub fn build(router: Router, server: &Server) -> Router { ) .ruma_route(&client::sync_events_route) .ruma_route(&client::sync_events_v4_route) + .ruma_route(&client::sync_events_v5_route) .ruma_route(&client::get_context_route) .ruma_route(&client::get_message_events_route) .ruma_route(&client::search_events_route) diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 9777faeb..2bc21355 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -155,6 +155,7 @@ where } let content = ReceiptEventContent::from_iter(json); + conduwuit::trace!(?content); Raw::from_json( serde_json::value::to_raw_value(&SyncEphemeralRoomEvent { content }) .expect("received valid json"), diff --git a/src/service/sync/mod.rs b/src/service/sync/mod.rs index 02658a70..0b86377a 100644 --- a/src/service/sync/mod.rs +++ b/src/service/sync/mod.rs @@ -11,8 +11,9 @@ use ruma::{ api::client::sync::sync_events::{ self, v4::{ExtensionsConfig, SyncRequestList}, + v5, }, - OwnedDeviceId, OwnedRoomId, OwnedUserId, + DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, UserId, }; use crate::{rooms, Dep}; @@ -20,7 +21,8 @@ use crate::{rooms, Dep}; pub struct Service { db: Data, services: Services, - connections: DbConnections, + connections: DbConnections, + snake_connections: DbConnections, } pub struct Data { @@ -52,9 +54,19 @@ struct SlidingSyncCache { extensions: ExtensionsConfig, } -type DbConnections = Mutex>; +#[derive(Default)] +struct SnakeSyncCache { + lists: BTreeMap, + subscriptions: BTreeMap, + known_rooms: BTreeMap>, + extensions: v5::request::Extensions, +} + +type DbConnections = Mutex>; type DbConnectionsKey = (OwnedUserId, OwnedDeviceId, String); type DbConnectionsVal = Arc>; +type SnakeConnectionsKey = (OwnedUserId, OwnedDeviceId, Option); +type SnakeConnectionsVal = Arc>; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -79,12 +91,15 @@ impl crate::Service for Service { typing: args.depend::("rooms::typing"), }, connections: StdMutex::new(BTreeMap::new()), + snake_connections: StdMutex::new(BTreeMap::new()), })) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } +/// load params from cache if body doesn't contain it, as long as it's allowed +/// in some cases we may need to allow an empty list as an actual value fn list_or_sticky(target: &mut Vec, cached: &Vec) { if target.is_empty() { target.clone_from(cached); @@ -97,6 +112,30 @@ fn some_or_sticky(target: &mut Option, cached: Option) { } impl Service { + pub fn snake_connection_cached( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + conn_id: Option, + ) -> bool { + self.snake_connections + .lock() + .unwrap() + .contains_key(&(user_id, device_id, conn_id)) + } + + pub fn forget_snake_sync_connection( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + conn_id: Option, + ) { + self.snake_connections + .lock() + .expect("locked") + .remove(&(user_id, device_id, conn_id)); + } + pub fn remembered( &self, user_id: OwnedUserId, @@ -121,6 +160,112 @@ impl Service { .remove(&(user_id, device_id, conn_id)); } + pub fn update_snake_sync_request_with_cache( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + request: &mut v5::Request, + ) -> BTreeMap> { + let conn_id = request.conn_id.clone(); + let mut cache = self.snake_connections.lock().expect("locked"); + let cached = Arc::clone( + cache + .entry((user_id, device_id, conn_id)) + .or_insert_with(|| Arc::new(Mutex::new(SnakeSyncCache::default()))), + ); + let cached = &mut cached.lock().expect("locked"); + drop(cache); + + //v5::Request::try_from_http_request(req, path_args); + for (list_id, list) in &mut request.lists { + if let Some(cached_list) = cached.lists.get(list_id) { + list_or_sticky( + &mut list.room_details.required_state, + &cached_list.room_details.required_state, + ); + some_or_sticky(&mut list.include_heroes, cached_list.include_heroes); + + match (&mut list.filters, cached_list.filters.clone()) { + | (Some(filters), Some(cached_filters)) => { + some_or_sticky(&mut filters.is_invite, cached_filters.is_invite); + // TODO (morguldir): Find out how a client can unset this, probably need + // to change into an option inside ruma + list_or_sticky( + &mut filters.not_room_types, + &cached_filters.not_room_types, + ); + }, + | (_, Some(cached_filters)) => list.filters = Some(cached_filters), + | (Some(list_filters), _) => list.filters = Some(list_filters.clone()), + | (..) => {}, + } + } + cached.lists.insert(list_id.clone(), list.clone()); + } + + cached + .subscriptions + .extend(request.room_subscriptions.clone()); + request + .room_subscriptions + .extend(cached.subscriptions.clone()); + + request.extensions.e2ee.enabled = request + .extensions + .e2ee + .enabled + .or(cached.extensions.e2ee.enabled); + + request.extensions.to_device.enabled = request + .extensions + .to_device + .enabled + .or(cached.extensions.to_device.enabled); + + request.extensions.account_data.enabled = request + .extensions + .account_data + .enabled + .or(cached.extensions.account_data.enabled); + request.extensions.account_data.lists = request + .extensions + .account_data + .lists + .clone() + .or_else(|| cached.extensions.account_data.lists.clone()); + request.extensions.account_data.rooms = request + .extensions + .account_data + .rooms + .clone() + .or_else(|| cached.extensions.account_data.rooms.clone()); + + some_or_sticky(&mut request.extensions.typing.enabled, cached.extensions.typing.enabled); + some_or_sticky( + &mut request.extensions.typing.rooms, + cached.extensions.typing.rooms.clone(), + ); + some_or_sticky( + &mut request.extensions.typing.lists, + cached.extensions.typing.lists.clone(), + ); + some_or_sticky( + &mut request.extensions.receipts.enabled, + cached.extensions.receipts.enabled, + ); + some_or_sticky( + &mut request.extensions.receipts.rooms, + cached.extensions.receipts.rooms.clone(), + ); + some_or_sticky( + &mut request.extensions.receipts.lists, + cached.extensions.receipts.lists.clone(), + ); + + cached.extensions = request.extensions.clone(); + cached.known_rooms.clone() + } + pub fn update_sync_request_with_cache( &self, user_id: OwnedUserId, @@ -148,20 +293,30 @@ impl Service { for (list_id, list) in &mut request.lists { if let Some(cached_list) = cached.lists.get(list_id) { list_or_sticky(&mut list.sort, &cached_list.sort); - list_or_sticky(&mut list.room_details.required_state, &cached_list.room_details.required_state); - some_or_sticky(&mut list.room_details.timeline_limit, cached_list.room_details.timeline_limit); - some_or_sticky(&mut list.include_old_rooms, cached_list.include_old_rooms.clone()); + list_or_sticky( + &mut list.room_details.required_state, + &cached_list.room_details.required_state, + ); + some_or_sticky( + &mut list.room_details.timeline_limit, + cached_list.room_details.timeline_limit, + ); + some_or_sticky( + &mut list.include_old_rooms, + cached_list.include_old_rooms.clone(), + ); match (&mut list.filters, cached_list.filters.clone()) { - (Some(list_filters), Some(cached_filters)) => { - some_or_sticky(&mut list_filters.is_dm, cached_filters.is_dm); - list_or_sticky(&mut list_filters.spaces, &cached_filters.spaces); - some_or_sticky(&mut list_filters.is_encrypted, cached_filters.is_encrypted); - some_or_sticky(&mut list_filters.is_invite, cached_filters.is_invite); - list_or_sticky(&mut list_filters.room_types, &cached_filters.room_types); - list_or_sticky(&mut list_filters.not_room_types, &cached_filters.not_room_types); - some_or_sticky(&mut list_filters.room_name_like, cached_filters.room_name_like); - list_or_sticky(&mut list_filters.tags, &cached_filters.tags); - list_or_sticky(&mut list_filters.not_tags, &cached_filters.not_tags); + | (Some(filter), Some(cached_filter)) => { + some_or_sticky(&mut filter.is_dm, cached_filter.is_dm); + list_or_sticky(&mut filter.spaces, &cached_filter.spaces); + some_or_sticky(&mut filter.is_encrypted, cached_filter.is_encrypted); + some_or_sticky(&mut filter.is_invite, cached_filter.is_invite); + list_or_sticky(&mut filter.room_types, &cached_filter.room_types); + // Should be made possible to change + list_or_sticky(&mut filter.not_room_types, &cached_filter.not_room_types); + some_or_sticky(&mut filter.room_name_like, cached_filter.room_name_like); + list_or_sticky(&mut filter.tags, &cached_filter.tags); + list_or_sticky(&mut filter.not_tags, &cached_filter.not_tags); }, | (_, Some(cached_filters)) => list.filters = Some(cached_filters), | (Some(list_filters), _) => list.filters = Some(list_filters.clone()), @@ -222,18 +377,16 @@ impl Service { subscriptions: BTreeMap, ) { let mut cache = self.connections.lock().expect("locked"); - let cached = Arc::clone( - cache - .entry((user_id, device_id, conn_id)) - .or_insert_with(|| { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }), - ); + let cached = Arc::clone(cache.entry((user_id, device_id, conn_id)).or_insert_with( + || { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + }, + )); let cached = &mut cached.lock().expect("locked"); drop(cache); @@ -241,13 +394,18 @@ impl Service { } pub fn update_sync_known_rooms( - &self, user_id: OwnedUserId, device_id: OwnedDeviceId, conn_id: String, list_id: String, - new_cached_rooms: BTreeSet, globalsince: u64, + &self, + user_id: &UserId, + device_id: &DeviceId, + conn_id: String, + list_id: String, + new_cached_rooms: BTreeSet, + globalsince: u64, ) { let mut cache = self.connections.lock().expect("locked"); let cached = Arc::clone( cache - .entry((user_id, device_id, conn_id)) + .entry((user_id.to_owned(), device_id.to_owned(), conn_id)) .or_insert_with(|| { Arc::new(Mutex::new(SlidingSyncCache { lists: BTreeMap::new(), @@ -275,4 +433,57 @@ impl Service { list.insert(roomid, globalsince); } } + + pub fn update_snake_sync_known_rooms( + &self, + user_id: &UserId, + device_id: &DeviceId, + conn_id: String, + list_id: String, + new_cached_rooms: BTreeSet, + globalsince: u64, + ) { + let mut cache = self.snake_connections.lock().expect("locked"); + let cached = Arc::clone( + cache + .entry((user_id.to_owned(), device_id.to_owned(), Some(conn_id))) + .or_insert_with(|| Arc::new(Mutex::new(SnakeSyncCache::default()))), + ); + let cached = &mut cached.lock().expect("locked"); + drop(cache); + + for (roomid, lastsince) in cached + .known_rooms + .entry(list_id.clone()) + .or_default() + .iter_mut() + { + if !new_cached_rooms.contains(roomid) { + *lastsince = 0; + } + } + let list = cached.known_rooms.entry(list_id).or_default(); + for roomid in new_cached_rooms { + list.insert(roomid, globalsince); + } + } + + pub fn update_snake_sync_subscriptions( + &self, + user_id: OwnedUserId, + device_id: OwnedDeviceId, + conn_id: Option, + subscriptions: BTreeMap, + ) { + let mut cache = self.snake_connections.lock().expect("locked"); + let cached = Arc::clone( + cache + .entry((user_id, device_id, conn_id)) + .or_insert_with(|| Arc::new(Mutex::new(SnakeSyncCache::default()))), + ); + let cached = &mut cached.lock().expect("locked"); + drop(cache); + + cached.subscriptions = subscriptions; + } } From f59e3d8850bfa93244f855e757fa69d610d16e2b Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 14 Jan 2025 13:05:25 -0500 Subject: [PATCH 1350/2091] bump nix lockfile, cargo.lock, rust to 1.84, and fix tracing fork Signed-off-by: June Clementine Strawberry --- Cargo.lock | 457 ++++++++++++++++++++++---------------------- Cargo.toml | 10 +- flake.lock | 30 +-- flake.nix | 2 +- rust-toolchain.toml | 2 +- 5 files changed, 252 insertions(+), 249 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d25197e0..f777a50c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,9 +49,9 @@ checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "arc-swap" @@ -138,18 +138,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -175,9 +175,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f47bb8cc16b669d267eeccf585aea077d0882f4777b1c1f740217885d6e6e5a3" +checksum = "f409eb70b561706bf8abba8ca9c112729c481595893fd06a2dd9af8ed8441148" dependencies = [ "aws-lc-sys", "paste", @@ -186,16 +186,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.23.1" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2101df3813227bbaaaa0b04cd61c534c7954b22bd68d399b440be937dc63ff7" +checksum = "923ded50f602b3007e5e63e3f094c479d9c8a9b42d7f4034e4afe456aa48bfd2" dependencies = [ "bindgen", "cc", "cmake", "dunce", "fs_extra", - "libc", "paste", ] @@ -225,9 +224,9 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -259,7 +258,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 1.0.2", + "sync_wrapper", "tower-layer", "tower-service", "tracing", @@ -282,7 +281,7 @@ dependencies = [ "mime", "pin-project-lite", "serde", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", ] @@ -369,7 +368,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -382,7 +381,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.90", + "syn 2.0.96", "which", ] @@ -394,9 +393,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" [[package]] name = "blake2" @@ -445,9 +444,9 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.20.0" +version = "1.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b37c88a63ffd85d15b406896cc343916d7cf57838a847b3a6f2ca5d39a5695a" +checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" [[package]] name = "byteorder" @@ -496,9 +495,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.3" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" dependencies = [ "jobserver", "libc", @@ -557,9 +556,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" dependencies = [ "clap_builder", "clap_derive", @@ -567,9 +566,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ "anstyle", "clap_lex", @@ -577,14 +576,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -740,7 +739,7 @@ dependencies = [ "serde_json", "serde_regex", "serde_yaml", - "thiserror 2.0.7", + "thiserror 2.0.11", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -778,7 +777,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -809,7 +808,7 @@ dependencies = [ "sentry-tracing", "serde_json", "tokio", - "tower 0.5.1", + "tower 0.5.2", "tower-http", "tracing", ] @@ -905,9 +904,9 @@ checksum = "3618cccc083bb987a415d85c02ca6c9994ea5b44731ec28b9ecf09658655fba9" [[package]] name = "const_panic" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "013b6c2c3a14d678f38cd23994b02da3a1a1b6a5d1eedddfe63a5a5f11b13a81" +checksum = "2459fc9262a1aa204eb4b5764ad4f189caec88aea9634389c0a25f8be7f6265e" [[package]] name = "coolor" @@ -1003,18 +1002,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -1031,18 +1030,18 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.11" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crossterm" @@ -1050,7 +1049,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "crossterm_winapi", "futures-core", "mio", @@ -1087,7 +1086,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1114,7 +1113,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1183,7 +1182,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1235,7 +1234,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1421,7 +1420,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1495,9 +1494,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" @@ -1647,11 +1646,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1687,7 +1686,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1753,9 +1752,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -1774,9 +1773,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http", @@ -1939,7 +1938,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -1982,9 +1981,9 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e031e8e3d94711a9ccb5d6ea357439ef3dcbed361798bd4071dc4d9793fbe22f" +checksum = "b77d01e822461baa8409e156015a1d91735549f0f2c17691bd2d996bef238f7f" dependencies = [ "byteorder-lite", "quick-error 2.0.1", @@ -2090,9 +2089,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -2131,9 +2130,9 @@ dependencies = [ [[package]] name = "konst" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298ddf99f06a97c1ecd0e910932662b7842855046234b0d0376d35d93add087f" +checksum = "4381b9b00c55f251f2ebe9473aef7c117e96828def1a7cb3bd3f0f903c6894e9" dependencies = [ "const_panic", "konst_kernel", @@ -2151,9 +2150,9 @@ dependencies = [ [[package]] name = "lazy-regex" -version = "3.3.0" +version = "3.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d8e41c97e6bc7ecb552016274b99fbb5d035e8de288c582d9b933af6677bfda" +checksum = "60c7310b93682b36b98fa7ea4de998d3463ccbebd94d935d6b48ba5b6ffa7126" dependencies = [ "lazy-regex-proc_macros", "once_cell", @@ -2162,14 +2161,14 @@ dependencies = [ [[package]] name = "lazy-regex-proc_macros" -version = "3.3.0" +version = "3.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76e1d8b05d672c53cb9c7b920bbba8783845ae4f0b076e02a3db1d02c81b4163" +checksum = "4ba01db5ef81e17eb10a5e0f2109d1b3a3e29bac3070fdbd7d156bf7dbd206a1" dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2186,9 +2185,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.168" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libloading" @@ -2202,9 +2201,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.20" +version = "1.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" +checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" dependencies = [ "cc", "pkg-config", @@ -2219,9 +2218,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" @@ -2241,9 +2240,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "loole" @@ -2362,9 +2361,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", "simd-adler32", @@ -2394,7 +2393,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "cfg-if", "cfg_aliases", "libc", @@ -2517,9 +2516,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -2591,7 +2590,7 @@ dependencies = [ "glob", "once_cell", "opentelemetry", - "ordered-float 4.5.0", + "ordered-float 4.6.0", "percent-encoding", "rand", "thiserror 1.0.69", @@ -2610,18 +2609,18 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.5.0" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" dependencies = [ "num-traits", ] [[package]] name = "os_info" -version = "3.9.0" +version = "3.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ca711d8b83edbb00b44d504503cd247c9c0bd8b0fa2694f2a1a3d8165379ce" +checksum = "6e6520c8cc998c5741ee68ec1dc369fc47e5f0ea5320018ecf2a1ccd6328f48b" dependencies = [ "log", "serde", @@ -2700,7 +2699,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2711,21 +2710,21 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_shared 0.11.2", + "phf_shared 0.11.3", ] [[package]] name = "phf_codegen" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator 0.11.2", - "phf_shared 0.11.2", + "phf_generator 0.11.3", + "phf_shared 0.11.3", ] [[package]] @@ -2740,11 +2739,11 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared 0.11.2", + "phf_shared 0.11.3", "rand", ] @@ -2754,43 +2753,43 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" dependencies = [ - "siphasher", + "siphasher 0.3.11", ] [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ - "siphasher", + "siphasher 1.0.1", ] [[package]] name = "pin-project" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be57f64e946e500c8ee36ef6331845d40a93055567ec57e8fae13efd33759b95" +checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" +checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2816,9 +2815,9 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "png" -version = "0.17.15" +version = "0.17.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67582bd5b65bdff614270e2ea89a1cf15bef71245cc1e5f7ea126977144211d" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" dependencies = [ "bitflags 1.3.2", "crc32fast", @@ -2850,12 +2849,12 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2869,9 +2868,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -2884,7 +2883,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "version_check", "yansi", ] @@ -2909,7 +2908,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -2927,7 +2926,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "memchr", "pulldown-cmark-escape", "unicase", @@ -2964,7 +2963,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls", "socket2", - "thiserror 2.0.7", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -2983,7 +2982,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.7", + "thiserror 2.0.11", "tinyvec", "tracing", "web-time 1.1.0", @@ -2991,9 +2990,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -3005,9 +3004,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -3044,11 +3043,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", ] [[package]] @@ -3130,7 +3129,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper 1.0.2", + "sync_wrapper", "tokio", "tokio-rustls", "tokio-socks", @@ -3221,7 +3220,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror 2.0.7", + "thiserror 2.0.11", "url", "web-time 1.1.0", ] @@ -3248,7 +3247,7 @@ dependencies = [ "serde_html_form", "serde_json", "smallvec", - "thiserror 2.0.7", + "thiserror 2.0.11", "time", "tracing", "url", @@ -3275,7 +3274,7 @@ dependencies = [ "serde", "serde_json", "smallvec", - "thiserror 2.0.7", + "thiserror 2.0.11", "tracing", "url", "web-time 1.1.0", @@ -3306,7 +3305,7 @@ version = "0.9.5" source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" dependencies = [ "js_int", - "thiserror 2.0.7", + "thiserror 2.0.11", ] [[package]] @@ -3330,7 +3329,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.90", + "syn 2.0.96", "toml", ] @@ -3355,7 +3354,7 @@ dependencies = [ "http", "http-auth", "ruma-common", - "thiserror 2.0.7", + "thiserror 2.0.11", "tracing", ] @@ -3372,7 +3371,7 @@ dependencies = [ "serde_json", "sha2", "subslice", - "thiserror 2.0.7", + "thiserror 2.0.11", ] [[package]] @@ -3386,7 +3385,7 @@ dependencies = [ "ruma-events", "serde", "serde_json", - "thiserror 2.0.7", + "thiserror 2.0.11", "tracing", ] @@ -3453,11 +3452,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.42" +version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "errno", "libc", "linux-raw-sys", @@ -3466,9 +3465,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "aws-lc-rs", "log", @@ -3503,9 +3502,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time 1.1.0", ] @@ -3524,9 +3523,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "rustyline-async" @@ -3538,7 +3537,7 @@ dependencies = [ "futures-util", "pin-project", "thingbuf", - "thiserror 2.0.7", + "thiserror 2.0.11", "unicode-segmentation", "unicode-width 0.2.0", ] @@ -3581,11 +3580,11 @@ checksum = "1be20c5f7f393ee700f8b2f28ea35812e4e212f40774b550cd2a93ea91684451" [[package]] name = "security-framework" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.7.0", "core-foundation", "core-foundation-sys", "libc", @@ -3594,9 +3593,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -3604,9 +3603,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "sentry" @@ -3745,29 +3744,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "serde_html_form" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de514ef58196f1fc96dcaef80fe6170a1ce6215df9687a93fe8300e773fefc5" +checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", "indexmap 2.7.0", @@ -3778,9 +3777,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", @@ -3930,6 +3929,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.9" @@ -4040,21 +4045,15 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4072,7 +4071,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -4123,11 +4122,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.7" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.7", + "thiserror-impl 2.0.11", ] [[package]] @@ -4138,18 +4137,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "thiserror-impl" -version = "2.0.7" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -4255,9 +4254,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -4270,9 +4269,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -4288,13 +4287,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -4441,14 +4440,14 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper 0.1.2", + "sync_wrapper", "tokio", "tower-layer", "tower-service", @@ -4461,7 +4460,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "async-compression", - "bitflags 2.6.0", + "bitflags 2.7.0", "bytes", "futures-core", "futures-util", @@ -4471,7 +4470,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower 0.5.1", + "tower 0.5.2", "tower-layer", "tower-service", "tracing", @@ -4492,7 +4491,7 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" version = "0.1.41" -source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" +source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" dependencies = [ "log", "pin-project-lite", @@ -4503,17 +4502,17 @@ dependencies = [ [[package]] name = "tracing-attributes" version = "0.1.28" -source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" +source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] name = "tracing-core" version = "0.1.33" -source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" +source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" dependencies = [ "once_cell", "valuable", @@ -4533,7 +4532,7 @@ dependencies = [ [[package]] name = "tracing-log" version = "0.2.0" -source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" +source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" dependencies = [ "log", "once_cell", @@ -4561,7 +4560,7 @@ dependencies = [ [[package]] name = "tracing-subscriber" version = "0.3.18" -source = "git+https://github.com/girlbossceo/tracing?rev=ccc4fbd8238c2d5ba354e61ec17ac610af11401d#ccc4fbd8238c2d5ba354e61ec17ac610af11401d" +source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" dependencies = [ "matchers", "nu-ansi-term", @@ -4622,9 +4621,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.8.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e51b68083f157f853b6379db119d1c1be0e6e4dec98101079dec41f6f5cf6df" +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" @@ -4715,9 +4714,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" dependencies = [ "getrandom", "serde", @@ -4758,34 +4757,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", @@ -4796,9 +4796,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4806,28 +4806,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -5125,9 +5128,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.6.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" dependencies = [ "memchr", ] @@ -5191,7 +5194,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "synstructure", ] @@ -5213,7 +5216,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -5233,7 +5236,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", "synstructure", ] @@ -5262,7 +5265,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.96", ] [[package]] @@ -5301,9 +5304,9 @@ checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" [[package]] name = "zune-jpeg" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16099418600b4d8f028622f73ff6e3deaabdff330fb9a2a131dea781ee8b0768" +checksum = "99a5bab8d7dedf81405c4bb1f2b83ea057643d9cb28778cea9eecddeedd2e028" dependencies = [ "zune-core", ] diff --git a/Cargo.toml b/Cargo.toml index 76acda80..855b8dda 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" -rust-version = "1.83.0" +rust-version = "1.84.0" version = "0.5.0" [workspace.metadata.crane] @@ -513,16 +513,16 @@ version = "0.2" # https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c [patch.crates-io.tracing-subscriber] git = "https://github.com/girlbossceo/tracing" -rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d" +rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c" [patch.crates-io.tracing] git = "https://github.com/girlbossceo/tracing" -rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d" +rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c" [patch.crates-io.tracing-core] git = "https://github.com/girlbossceo/tracing" -rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d" +rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c" [patch.crates-io.tracing-log] git = "https://github.com/girlbossceo/tracing" -rev = "ccc4fbd8238c2d5ba354e61ec17ac610af11401d" +rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c" # adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50 # adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b diff --git a/flake.lock b/flake.lock index 35029076..210e8e08 100644 --- a/flake.lock +++ b/flake.lock @@ -117,11 +117,11 @@ }, "crane_2": { "locked": { - "lastModified": 1734808813, - "narHash": "sha256-3aH/0Y6ajIlfy7j52FGZ+s4icVX0oHhqBzRdlOeztqg=", + "lastModified": 1736566337, + "narHash": "sha256-SC0eDcZPqISVt6R0UfGPyQLrI0+BppjjtQ3wcSlk0oI=", "owner": "ipetkov", "repo": "crane", - "rev": "72e2d02dbac80c8c86bf6bf3e785536acf8ee926", + "rev": "9172acc1ee6c7e1cbafc3044ff850c568c75a5a3", "type": "github" }, "original": { @@ -170,11 +170,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1735799625, - "narHash": "sha256-lFadwWDvVIub11bwfZhsh2WUByf9LOi6yjsSUMmE0xk=", + "lastModified": 1736836313, + "narHash": "sha256-zdZ7/T6yG0/hzoVOiNpDiR/sW3zR6oSMrfIFJK2BrrE=", "owner": "nix-community", "repo": "fenix", - "rev": "a9d84a1545814910cb4ab0515ed6921e8b07ee95", + "rev": "056c9393c821a4df356df6ce7f14c722dc8717ec", "type": "github" }, "original": { @@ -364,11 +364,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1733603756, - "narHash": "sha256-eTKnZDZ1Ex++v+BI0DBcUBmCXAO/tE8hxK9MiyztZkU=", + "lastModified": 1736719310, + "narHash": "sha256-Turvx60THwzTiUHb49WV3upUgsPuktr7tVy2Lwu2xJg=", "owner": "axboe", "repo": "liburing", - "rev": "c3d5d6270cd5ed48d817fc1e8e95f7c8b222f2ff", + "rev": "3124a4619e4daf26b06d48ccf0186a947070c415", "type": "github" }, "original": { @@ -550,11 +550,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1735685343, - "narHash": "sha256-h1CpBzdJDNtSUb5QMyfFHKHocTTky+4McgQEBQBM+xA=", + "lastModified": 1736817698, + "narHash": "sha256-1m+JP9RUsbeLVv/tF1DX3Ew9Vl/fatXnlh/g5k3jcSk=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "81934660d6e9ea54d2f0cdee821e8533b10c221a", + "rev": "2b1fca3296ddd1602d2c4f104a4050e006f4b0cb", "type": "github" }, "original": { @@ -599,11 +599,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1735742096, - "narHash": "sha256-q3a80h8Jf8wfmPURUgRR46nQCB3I5fhZ+/swulTF5HY=", + "lastModified": 1736690231, + "narHash": "sha256-g9gyxX+F6CrkT5gRIMKPnCPom0o9ZDzYnzzeNF86D6Q=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "7e639ee3dda6ed9cecc79d41f6d38235121e483d", + "rev": "8364ef299790cb6ec22b9e09e873c97dbe9f2cb5", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index fb40cae7..920d3d14 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,7 @@ file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` - sha256 = "sha256-s1RPtyvDGJaX/BisLT+ifVfuhDT1nZkZ1NcK8sbwELM="; + sha256 = "sha256-lMLAupxng4Fd9F1oDw8gx+qA0RuF7ou7xhNU8wgs0PU="; }; mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index ddd952a2..97e33c91 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -9,7 +9,7 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.83.0" +channel = "1.84.0" profile = "minimal" components = [ # For rust-analyzer From 9ebb39ca4f35789e54b73cd33805943b362819ae Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 15 Jan 2025 14:34:21 -0500 Subject: [PATCH 1351/2091] add missing cfg_attr on deserialisation db test Signed-off-by: strawberry --- src/database/tests.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/database/tests.rs b/src/database/tests.rs index 0c9fb41a..0a42ad60 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -219,7 +219,10 @@ fn de_tuple_incomplete_with_sep() { } #[test] -#[should_panic(expected = "deserialization failed to consume trailing bytes")] +#[cfg_attr( + debug_assertions, + should_panic(expected = "deserialization failed to consume trailing bytes") +)] fn de_tuple_unfinished() { let user_id: &UserId = "@user:example.com".try_into().unwrap(); let room_id: &RoomId = "!room:example.com".try_into().unwrap(); From afe9e5536bc8afded76c30304dc782deeda3c9c4 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 11 Jan 2025 16:04:19 +0000 Subject: [PATCH 1352/2091] remove undocumented jwt token login --- Cargo.lock | 15 --------------- Cargo.toml | 4 ---- conduwuit-example.toml | 4 ---- src/api/Cargo.toml | 1 - src/api/client/session.rs | 38 ++++---------------------------------- src/core/config/mod.rs | 6 ------ src/service/Cargo.toml | 1 - src/service/globals/mod.rs | 11 ----------- 8 files changed, 4 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f777a50c..18bd7aab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -685,7 +685,6 @@ dependencies = [ "http-body-util", "hyper", "ipaddress", - "jsonwebtoken", "log", "rand", "reqwest", @@ -831,7 +830,6 @@ dependencies = [ "image", "ipaddress", "itertools 0.13.0", - "jsonwebtoken", "log", "loole", "lru-cache", @@ -2115,19 +2113,6 @@ dependencies = [ "serde", ] -[[package]] -name = "jsonwebtoken" -version = "9.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ae10193d25051e74945f1ea2d0b42e03cc3b890f7e4cc5faa44997d808193f" -dependencies = [ - "base64 0.21.7", - "js-sys", - "ring", - "serde", - "serde_json", -] - [[package]] name = "konst" version = "0.3.16" diff --git a/Cargo.toml b/Cargo.toml index 855b8dda..c0b31a69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,10 +58,6 @@ features = ["parse"] [workspace.dependencies.sanitize-filename] version = "0.6.0" -[workspace.dependencies.jsonwebtoken] -version = "9.3.0" -default-features = false - [workspace.dependencies.base64] version = "0.22.1" default-features = false diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 9eefedbb..28e7012b 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -563,10 +563,6 @@ # #proxy = "none" -# This item is undocumented. Please contribute documentation for it. -# -#jwt_secret = - # Servers listed here will be used to gather public keys of other servers # (notary trusted key servers). # diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 1bc73624..1b463fbc 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -50,7 +50,6 @@ http.workspace = true http-body-util.workspace = true hyper.workspace = true ipaddress.workspace = true -jsonwebtoken.workspace = true log.workspace = true rand.workspace = true reqwest.workspace = true diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 26377c55..21b8786c 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -20,17 +20,10 @@ use ruma::{ }, OwnedUserId, UserId, }; -use serde::Deserialize; use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; use crate::{utils, utils::hash, Error, Result, Ruma}; -#[derive(Debug, Deserialize)] -struct Claims { - sub: String, - //exp: usize, -} - /// # `GET /_matrix/client/v3/login` /// /// Get the supported login types of this server. One of these should be used as @@ -106,34 +99,11 @@ pub(crate) async fn login_route( user_id }, - | login::v3::LoginInfo::Token(login::v3::Token { token }) => { + | login::v3::LoginInfo::Token(login::v3::Token { token: _ }) => { debug!("Got token login type"); - if let Some(jwt_decoding_key) = services.globals.jwt_decoding_key() { - let token = jsonwebtoken::decode::( - token, - jwt_decoding_key, - &jsonwebtoken::Validation::default(), - ) - .map_err(|e| { - warn!("Failed to parse JWT token from user logging in: {e}"); - Error::BadRequest(ErrorKind::InvalidUsername, "Token is invalid.") - })?; - - let username = token.claims.sub.to_lowercase(); - - UserId::parse_with_server_name(username, services.globals.server_name()).map_err( - |e| { - err!(Request(InvalidUsername(debug_error!( - ?e, - "Failed to parse login username" - )))) - }, - )? - } else { - return Err!(Request(Unknown( - "Token login is not supported (server has no jwt decoding key)." - ))); - } + return Err!(Request(Unknown( + "Token login is not supported." + ))); }, #[allow(deprecated)] | login::v3::LoginInfo::ApplicationService(login::v3::ApplicationService { diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 97ecbeaf..d65d3812 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -671,8 +671,6 @@ pub struct Config { #[serde(default)] pub proxy: ProxyConfig, - pub jwt_secret: Option, - /// Servers listed here will be used to gather public keys of other servers /// (notary trusted key servers). /// @@ -2005,10 +2003,6 @@ impl fmt::Display for Config { "Lockdown public room directory (only allow admins to publish)", &self.lockdown_public_room_directory.to_string(), ); - line("JWT secret", match self.jwt_secret { - | Some(_) => "set", - | None => "not set", - }); line( "Trusted key servers", &self diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 4708ff4e..21fbb417 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -61,7 +61,6 @@ image.workspace = true image.optional = true ipaddress.workspace = true itertools.workspace = true -jsonwebtoken.workspace = true log.workspace = true loole.workspace = true lru-cache.workspace = true diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 38d7f786..f6ff2b09 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -18,7 +18,6 @@ pub struct Service { pub db: Data, pub config: Config, - jwt_decoding_key: Option, pub bad_event_ratelimiter: Arc>>, pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, @@ -33,11 +32,6 @@ impl crate::Service for Service { let db = Data::new(&args); let config = &args.server.config; - let jwt_decoding_key = config - .jwt_secret - .as_ref() - .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes())); - let turn_secret = config .turn_secret_file @@ -66,7 +60,6 @@ impl crate::Service for Service { let mut s = Self { db, config: config.clone(), - jwt_decoding_key, bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), admin_alias: OwnedRoomAliasId::try_from(format!("#admins:{}", &config.server_name)) .expect("#admins:server_name is valid alias name"), @@ -158,10 +151,6 @@ impl Service { pub fn trusted_servers(&self) -> &[OwnedServerName] { &self.config.trusted_servers } - pub fn jwt_decoding_key(&self) -> Option<&jsonwebtoken::DecodingKey> { - self.jwt_decoding_key.as_ref() - } - pub fn turn_password(&self) -> &String { &self.config.turn_password } pub fn turn_ttl(&self) -> u64 { self.config.turn_ttl } From 2cc6ad8df32610531eb56a5e3bf06320afafde97 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 11 Jan 2025 18:49:21 +0000 Subject: [PATCH 1353/2091] implement `/login/get_token` (MSC3882) --- conduwuit-example.toml | 16 +++++ src/api/client/capabilities.rs | 5 +- src/api/client/session.rs | 124 +++++++++++++++++++++++++++------ src/api/router.rs | 1 + src/core/config/mod.rs | 20 ++++++ src/database/maps.rs | 4 ++ src/service/users/mod.rs | 50 +++++++++++++ 7 files changed, 196 insertions(+), 24 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 28e7012b..96578da3 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -645,6 +645,22 @@ # #openid_token_ttl = 3600 +# Allow an existing session to mint a login token for another client. +# This requires interactive authentication, but has security ramifications +# as a malicious client could use the mechanism to spawn more than one +# session. +# Enabled by default. +# +#login_via_existing_session = true + +# Login token expiration/TTL in milliseconds. +# +# These are short-lived tokens for the m.login.token endpoint. +# This is used to allow existing sessions to create new sessions. +# see login_via_existing_session. +# +#login_token_ttl = 120000 + # Static TURN username to provide the client if not using a shared secret # ("turn_secret"), It is recommended to use a shared secret over static # credentials. diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index e122611f..87cdb43d 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -32,8 +32,9 @@ pub(crate) async fn get_capabilities_route( // we do not implement 3PID stuff capabilities.thirdparty_id_changes = ThirdPartyIdChangesCapability { enabled: false }; - // we dont support generating tokens yet - capabilities.get_login_token = GetLoginTokenCapability { enabled: false }; + capabilities.get_login_token = GetLoginTokenCapability { + enabled: services.server.config.login_via_existing_session, + }; // MSC4133 capability capabilities diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 21b8786c..4881ade7 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{debug, err, info, utils::ReadyExt, warn, Err}; @@ -6,9 +8,10 @@ use ruma::{ api::client::{ error::ErrorKind, session::{ + get_login_token, get_login_types::{ self, - v3::{ApplicationServiceLoginType, PasswordLoginType}, + v3::{ApplicationServiceLoginType, PasswordLoginType, TokenLoginType}, }, login::{ self, @@ -16,10 +19,11 @@ use ruma::{ }, logout, logout_all, }, - uiaa::UserIdentifier, + uiaa, }, OwnedUserId, UserId, }; +use service::uiaa::SESSION_ID_LENGTH; use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; use crate::{utils, utils::hash, Error, Result, Ruma}; @@ -30,12 +34,16 @@ use crate::{utils, utils::hash, Error, Result, Ruma}; /// the `type` field when logging in. #[tracing::instrument(skip_all, fields(%client), name = "login")] pub(crate) async fn get_login_types_route( + State(services): State, InsecureClientIp(client): InsecureClientIp, _body: Ruma, ) -> Result { Ok(get_login_types::v3::Response::new(vec![ get_login_types::v3::LoginType::Password(PasswordLoginType::default()), get_login_types::v3::LoginType::ApplicationService(ApplicationServiceLoginType::default()), + get_login_types::v3::LoginType::Token(TokenLoginType { + get_login_token: services.server.config.login_via_existing_session, + }), ])) } @@ -70,7 +78,9 @@ pub(crate) async fn login_route( .. }) => { debug!("Got password login type"); - let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { + let user_id = if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = + identifier + { UserId::parse_with_server_name( user_id.to_lowercase(), services.globals.server_name(), @@ -99,11 +109,12 @@ pub(crate) async fn login_route( user_id }, - | login::v3::LoginInfo::Token(login::v3::Token { token: _ }) => { + | login::v3::LoginInfo::Token(login::v3::Token { token }) => { debug!("Got token login type"); - return Err!(Request(Unknown( - "Token login is not supported." - ))); + if !services.server.config.login_via_existing_session { + return Err!(Request(Unknown("Token login is not enabled."))); + } + services.users.find_from_login_token(token).await? }, #[allow(deprecated)] | login::v3::LoginInfo::ApplicationService(login::v3::ApplicationService { @@ -111,21 +122,22 @@ pub(crate) async fn login_route( user, }) => { debug!("Got appservice login type"); - let user_id = if let Some(UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { - UserId::parse_with_server_name( - user_id.to_lowercase(), - services.globals.server_name(), - ) - } else if let Some(user) = user { - OwnedUserId::parse(user) - } else { - warn!("Bad login type: {:?}", &body.login_info); - return Err(Error::BadRequest(ErrorKind::forbidden(), "Bad login type.")); - } - .map_err(|e| { - warn!("Failed to parse username from appservice logging in: {e}"); - Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.") - })?; + let user_id = + if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { + UserId::parse_with_server_name( + user_id.to_lowercase(), + services.globals.server_name(), + ) + } else if let Some(user) = user { + OwnedUserId::parse(user) + } else { + warn!("Bad login type: {:?}", &body.login_info); + return Err(Error::BadRequest(ErrorKind::forbidden(), "Bad login type.")); + } + .map_err(|e| { + warn!("Failed to parse username from appservice logging in: {e}"); + Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.") + })?; if let Some(ref info) = body.appservice_info { if !info.is_user_match(&user_id) { @@ -217,6 +229,74 @@ pub(crate) async fn login_route( }) } +/// # `POST /_matrix/client/v1/login/get_token` +/// +/// Allows a logged-in user to get a short-lived token which can be used +/// to log in with the m.login.token flow. +/// +/// +#[tracing::instrument(skip_all, fields(%client), name = "login_token")] +pub(crate) async fn login_token_route( + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + if !services.server.config.login_via_existing_session { + return Err!(Request(Unknown("Login via an existing session is not enabled"))); + } + // Authentication for this endpoint was made optional, but we need + // authentication. + let sender_user = body + .sender_user + .as_ref() + .ok_or_else(|| Error::BadRequest(ErrorKind::MissingToken, "Missing access token."))?; + let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + + // This route SHOULD have UIA + // TODO: How do we make only UIA sessions that have not been used before valid? + + let mut uiaainfo = uiaa::UiaaInfo { + flows: vec![uiaa::AuthFlow { stages: vec![uiaa::AuthType::Password] }], + completed: Vec::new(), + params: Box::default(), + session: None, + auth_error: None, + }; + + if let Some(auth) = &body.auth { + let (worked, uiaainfo) = services + .uiaa + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; + + if !worked { + return Err(Error::Uiaa(uiaainfo)); + } + + // Success! + } else if let Some(json) = body.json_body { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); + services + .uiaa + .create(sender_user, sender_device, &uiaainfo, &json); + + return Err(Error::Uiaa(uiaainfo)); + } else { + return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + } + + let login_token = utils::random_string(TOKEN_LENGTH); + + let expires_in = services + .users + .create_login_token(sender_user, &login_token)?; + + Ok(get_login_token::v1::Response { + expires_in: Duration::from_millis(expires_in), + login_token, + }) +} + /// # `POST /_matrix/client/v3/logout` /// /// Log out the current device. diff --git a/src/api/router.rs b/src/api/router.rs index e7cd368d..7855ddfa 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -34,6 +34,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(&client::register_route) .ruma_route(&client::get_login_types_route) .ruma_route(&client::login_route) + .ruma_route(&client::login_token_route) .ruma_route(&client::whoami_route) .ruma_route(&client::logout_route) .ruma_route(&client::logout_all_route) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index d65d3812..84b88c7c 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -767,6 +767,24 @@ pub struct Config { #[serde(default = "default_openid_token_ttl")] pub openid_token_ttl: u64, + /// Allow an existing session to mint a login token for another client. + /// This requires interactive authentication, but has security ramifications + /// as a malicious client could use the mechanism to spawn more than one + /// session. + /// Enabled by default. + #[serde(default = "true_fn")] + pub login_via_existing_session: bool, + + /// Login token expiration/TTL in milliseconds. + /// + /// These are short-lived tokens for the m.login.token endpoint. + /// This is used to allow existing sessions to create new sessions. + /// see login_via_existing_session. + /// + /// default: 120000 + #[serde(default = "default_login_token_ttl")] + pub login_token_ttl: u64, + /// Static TURN username to provide the client if not using a shared secret /// ("turn_secret"), It is recommended to use a shared secret over static /// credentials. @@ -2373,6 +2391,8 @@ fn default_notification_push_path() -> String { "/_matrix/push/v1/notify".to_own fn default_openid_token_ttl() -> u64 { 60 * 60 } +fn default_login_token_ttl() -> u64 { 2 * 60 * 1000 } + fn default_turn_ttl() -> u64 { 60 * 60 * 24 } fn default_presence_idle_timeout_s() -> u64 { 5 * 60 } diff --git a/src/database/maps.rs b/src/database/maps.rs index bc409919..19e19955 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -365,6 +365,10 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "openidtoken_expiresatuserid", ..descriptor::RANDOM_SMALL }, + Descriptor { + name: "logintoken_expiresatuserid", + ..descriptor::RANDOM_SMALL + }, Descriptor { name: "userroomid_highlightcount", ..descriptor::RANDOM diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index fe064d9c..971cea7c 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -41,6 +41,7 @@ struct Data { keyid_key: Arc, onetimekeyid_onetimekeys: Arc, openidtoken_expiresatuserid: Arc, + logintoken_expiresatuserid: Arc, todeviceid_events: Arc, token_userdeviceid: Arc, userdeviceid_metadata: Arc, @@ -76,6 +77,7 @@ impl crate::Service for Service { keyid_key: args.db["keyid_key"].clone(), onetimekeyid_onetimekeys: args.db["onetimekeyid_onetimekeys"].clone(), openidtoken_expiresatuserid: args.db["openidtoken_expiresatuserid"].clone(), + logintoken_expiresatuserid: args.db["logintoken_expiresatuserid"].clone(), todeviceid_events: args.db["todeviceid_events"].clone(), token_userdeviceid: args.db["token_userdeviceid"].clone(), userdeviceid_metadata: args.db["userdeviceid_metadata"].clone(), @@ -941,6 +943,54 @@ impl Service { .map_err(|e| err!(Database("User ID in openid_userid is invalid. {e}"))) } + /// Creates a short-lived login token, which can be used to log in using the + /// `m.login.token` mechanism. + pub fn create_login_token(&self, user_id: &UserId, token: &str) -> Result { + use std::num::Saturating as Sat; + + let expires_in = self.services.server.config.login_token_ttl; + let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in); + + let mut value = expires_at.0.to_be_bytes().to_vec(); + value.extend_from_slice(user_id.as_bytes()); + + self.db + .logintoken_expiresatuserid + .insert(token.as_bytes(), value.as_slice()); + + Ok(expires_in) + } + + /// Find out which user a login token belongs to. + /// Removes the token to prevent double-use attacks. + pub async fn find_from_login_token(&self, token: &str) -> Result { + let Ok(value) = self.db.logintoken_expiresatuserid.get(token).await else { + return Err!(Request(Unauthorized("Login token is unrecognised"))); + }; + + let (expires_at_bytes, user_bytes) = value.split_at(0_u64.to_be_bytes().len()); + let expires_at = u64::from_be_bytes( + expires_at_bytes + .try_into() + .map_err(|e| err!(Database("expires_at in login_userid is invalid u64. {e}")))?, + ); + + if expires_at < utils::millis_since_unix_epoch() { + debug_warn!("Login token is expired, removing"); + self.db.openidtoken_expiresatuserid.remove(token.as_bytes()); + + return Err!(Request(Unauthorized("Login token is expired"))); + } + + self.db.openidtoken_expiresatuserid.remove(token.as_bytes()); + + let user_string = utils::string_from_bytes(user_bytes) + .map_err(|e| err!(Database("User ID in login_userid is invalid unicode. {e}")))?; + + OwnedUserId::try_from(user_string) + .map_err(|e| err!(Database("User ID in login_userid is invalid. {e}"))) + } + /// Gets a specific user profile key pub async fn profile_key( &self, From 5b8464252c2c03edf65e43153be026dbb768a12a Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 17 Jan 2025 00:01:47 -0500 Subject: [PATCH 1354/2091] cleanup+fix login get_token code, use db ser/deser instead Signed-off-by: strawberry --- src/api/client/session.rs | 23 ++++++++-------------- src/service/users/mod.rs | 41 +++++++++++++-------------------------- 2 files changed, 22 insertions(+), 42 deletions(-) diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 4881ade7..7155351c 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -242,15 +242,11 @@ pub(crate) async fn login_token_route( body: Ruma, ) -> Result { if !services.server.config.login_via_existing_session { - return Err!(Request(Unknown("Login via an existing session is not enabled"))); + return Err!(Request(Forbidden("Login via an existing session is not enabled"))); } - // Authentication for this endpoint was made optional, but we need - // authentication. - let sender_user = body - .sender_user - .as_ref() - .ok_or_else(|| Error::BadRequest(ErrorKind::MissingToken, "Missing access token."))?; - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + + let sender_user = body.sender_user(); + let sender_device = body.sender_device(); // This route SHOULD have UIA // TODO: How do we make only UIA sessions that have not been used before valid? @@ -274,22 +270,19 @@ pub(crate) async fn login_token_route( } // Success! - } else if let Some(json) = body.json_body { + } else if let Some(json) = body.json_body.as_ref() { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + .create(sender_user, sender_device, &uiaainfo, json); return Err(Error::Uiaa(uiaainfo)); } else { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + return Err!(Request(NotJson("No JSON body was sent when required."))); } let login_token = utils::random_string(TOKEN_LENGTH); - - let expires_in = services - .users - .create_login_token(sender_user, &login_token)?; + let expires_in = services.users.create_login_token(sender_user, &login_token); Ok(get_login_token::v1::Response { expires_in: Duration::from_millis(expires_in), diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 971cea7c..b2d3a94a 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,8 +1,8 @@ use std::{collections::BTreeMap, mem, mem::size_of, sync::Arc}; use conduwuit::{ - debug_warn, err, utils, - utils::{stream::TryIgnore, string::Unquoted, ReadyExt}, + debug_warn, err, trace, + utils::{self, stream::TryIgnore, string::Unquoted, ReadyExt}, Err, Error, Result, Server, }; use database::{Database, Deserialized, Ignore, Interfix, Json, Map}; @@ -945,50 +945,37 @@ impl Service { /// Creates a short-lived login token, which can be used to log in using the /// `m.login.token` mechanism. - pub fn create_login_token(&self, user_id: &UserId, token: &str) -> Result { + pub fn create_login_token(&self, user_id: &UserId, token: &str) -> u64 { use std::num::Saturating as Sat; let expires_in = self.services.server.config.login_token_ttl; let expires_at = Sat(utils::millis_since_unix_epoch()) + Sat(expires_in); - let mut value = expires_at.0.to_be_bytes().to_vec(); - value.extend_from_slice(user_id.as_bytes()); + let value = (expires_at.0, user_id); + self.db.logintoken_expiresatuserid.raw_put(token, value); - self.db - .logintoken_expiresatuserid - .insert(token.as_bytes(), value.as_slice()); - - Ok(expires_in) + expires_in } /// Find out which user a login token belongs to. /// Removes the token to prevent double-use attacks. pub async fn find_from_login_token(&self, token: &str) -> Result { let Ok(value) = self.db.logintoken_expiresatuserid.get(token).await else { - return Err!(Request(Unauthorized("Login token is unrecognised"))); + return Err!(Request(Forbidden("Login token is unrecognised"))); }; - - let (expires_at_bytes, user_bytes) = value.split_at(0_u64.to_be_bytes().len()); - let expires_at = u64::from_be_bytes( - expires_at_bytes - .try_into() - .map_err(|e| err!(Database("expires_at in login_userid is invalid u64. {e}")))?, - ); + let (expires_at, user_id): (u64, OwnedUserId) = value.deserialized()?; if expires_at < utils::millis_since_unix_epoch() { - debug_warn!("Login token is expired, removing"); - self.db.openidtoken_expiresatuserid.remove(token.as_bytes()); + trace!(?user_id, ?token, "Removing expired login token"); - return Err!(Request(Unauthorized("Login token is expired"))); + self.db.logintoken_expiresatuserid.remove(token); + + return Err!(Request(Forbidden("Login token is expired"))); } - self.db.openidtoken_expiresatuserid.remove(token.as_bytes()); + self.db.logintoken_expiresatuserid.remove(token); - let user_string = utils::string_from_bytes(user_bytes) - .map_err(|e| err!(Database("User ID in login_userid is invalid unicode. {e}")))?; - - OwnedUserId::try_from(user_string) - .map_err(|e| err!(Database("User ID in login_userid is invalid. {e}"))) + Ok(user_id) } /// Gets a specific user profile key From afcd0bfeef8e68232dd92aeddfd20397493a409e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 14 Jan 2025 05:55:49 +0000 Subject: [PATCH 1355/2091] add deref_at macro util Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 631b2820..2bbadb50 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -90,6 +90,13 @@ macro_rules! ref_at { }; } +#[macro_export] +macro_rules! deref_at { + ($idx:tt) => { + |t| *t.$idx + }; +} + /// Functor for equality i.e. .is_some_and(is_equal!(2)) #[macro_export] macro_rules! is_equal_to { From e56d3c6cb3939bfe9b10c5a18a62104f85a02fef Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 14 Jan 2025 05:56:06 +0000 Subject: [PATCH 1356/2091] add multi_get_statekey_from_short Signed-off-by: Jason Volk --- src/service/rooms/short/mod.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index b645f9f1..4a591592 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -196,6 +196,20 @@ pub async fn get_statekey_from_short( }) } +#[implement(Service)] +pub fn multi_get_statekey_from_short<'a, S>( + &'a self, + shortstatekey: S, +) -> impl Stream> + Send + 'a +where + S: Stream + Send + 'a, +{ + self.db + .shortstatekey_statekey + .qry_batch(shortstatekey) + .map(Deserialized::deserialized) +} + /// Returns (shortstatehash, already_existed) #[implement(Service)] pub async fn get_or_create_shortstatehash(&self, state_hash: &[u8]) -> (ShortStateHash, bool) { From 5167e1f06dce7bbf2cb521dbfb5ca28c15b2a547 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 9 Jan 2025 21:01:58 +0000 Subject: [PATCH 1357/2091] add option to disable listeners Signed-off-by: Jason Volk --- conduwuit-example.toml | 5 +++++ src/core/config/mod.rs | 5 +++++ src/router/serve/mod.rs | 14 ++++++++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 96578da3..54143ced 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1503,6 +1503,11 @@ # #sender_workers = 0 +# Enables listener sockets; can be set to false to disable listening. This +# option is intended for developer/diagnostic purposes only. +# +#listening = true + [global.tls] # Path to a valid TLS certificate file. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 84b88c7c..cb42940b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1710,6 +1710,11 @@ pub struct Config { #[serde(default)] pub sender_workers: usize, + /// Enables listener sockets; can be set to false to disable listening. This + /// option is intended for developer/diagnostic purposes only. + #[serde(default = "true_fn")] + pub listening: bool, + #[serde(flatten)] #[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index f6262202..5c822f2b 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -6,7 +6,7 @@ mod unix; use std::sync::Arc; use axum_server::Handle as ServerHandle; -use conduwuit::Result; +use conduwuit::{err, Result}; use conduwuit_service::Services; use tokio::sync::broadcast; @@ -16,13 +16,19 @@ use super::layers; pub(super) async fn serve( services: Arc, handle: ServerHandle, - shutdown: broadcast::Receiver<()>, -) -> Result<()> { + mut shutdown: broadcast::Receiver<()>, +) -> Result { let server = &services.server; let config = &server.config; + if !config.listening { + return shutdown + .recv() + .await + .map_err(|e| err!(error!("channel error: {e}"))); + } + let addrs = config.get_bind_addrs(); let (app, _guard) = layers::build(&services)?; - if cfg!(unix) && config.unix_socket_path.is_some() { unix::serve(server, app, shutdown).await } else if config.tls.certs.is_some() { From 98d8e5c63cc7019d6ecbe235f380a3c954b9e6b5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 16 Jan 2025 08:00:01 +0000 Subject: [PATCH 1358/2091] add standard error trait and thread access error conversions Signed-off-by: Jason Volk --- src/core/error/mod.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index ffa829d9..2468811e 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -4,7 +4,7 @@ mod panic; mod response; mod serde; -use std::{any::Any, borrow::Cow, convert::Infallible, fmt, sync::PoisonError}; +use std::{any::Any, borrow::Cow, convert::Infallible, sync::PoisonError}; pub use self::{err::visit, log::*}; @@ -17,7 +17,7 @@ pub enum Error { // std #[error(transparent)] - Fmt(#[from] fmt::Error), + Fmt(#[from] std::fmt::Error), #[error(transparent)] FromUtf8(#[from] std::string::FromUtf8Error), #[error("I/O error: {0}")] @@ -27,6 +27,10 @@ pub enum Error { #[error(transparent)] ParseInt(#[from] std::num::ParseIntError), #[error(transparent)] + Std(#[from] Box), + #[error(transparent)] + ThreadAccessError(#[from] std::thread::AccessError), + #[error(transparent)] TryFromInt(#[from] std::num::TryFromIntError), #[error(transparent)] TryFromSlice(#[from] std::array::TryFromSliceError), @@ -189,8 +193,10 @@ impl Error { pub fn is_not_found(&self) -> bool { self.status_code() == http::StatusCode::NOT_FOUND } } -impl fmt::Debug for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.message()) } +impl std::fmt::Debug for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.message()) + } } impl From> for Error { From 80832cb0bb2fc725164370bb808e192ee3480172 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 16 Jan 2025 12:04:31 +0000 Subject: [PATCH 1359/2091] add checked math wrapper Signed-off-by: Jason Volk --- src/core/utils/math.rs | 3 ++- src/core/utils/math/tried.rs | 47 ++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 src/core/utils/math/tried.rs diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index a08cb206..ed157daf 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -1,10 +1,11 @@ mod expected; +mod tried; use std::{cmp, convert::TryFrom}; pub use checked_ops::checked_ops; -pub use self::expected::Expected; +pub use self::{expected::Expected, tried::Tried}; use crate::{debug::type_name, err, Err, Error, Result}; /// Checked arithmetic expression. Returns a Result diff --git a/src/core/utils/math/tried.rs b/src/core/utils/math/tried.rs new file mode 100644 index 00000000..2006d2d5 --- /dev/null +++ b/src/core/utils/math/tried.rs @@ -0,0 +1,47 @@ +use num_traits::ops::checked::{CheckedAdd, CheckedDiv, CheckedMul, CheckedRem, CheckedSub}; + +use crate::{checked, Result}; + +pub trait Tried { + #[inline] + fn try_add(self, rhs: Self) -> Result + where + Self: CheckedAdd + Sized, + { + checked!(self + rhs) + } + + #[inline] + fn try_sub(self, rhs: Self) -> Result + where + Self: CheckedSub + Sized, + { + checked!(self - rhs) + } + + #[inline] + fn try_mul(self, rhs: Self) -> Result + where + Self: CheckedMul + Sized, + { + checked!(self * rhs) + } + + #[inline] + fn try_div(self, rhs: Self) -> Result + where + Self: CheckedDiv + Sized, + { + checked!(self / rhs) + } + + #[inline] + fn try_rem(self, rhs: Self) -> Result + where + Self: CheckedRem + Sized, + { + checked!(self % rhs) + } +} + +impl Tried for T {} From 7a8ca8842af65435239ae9358587fdabb18ee6c0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 16 Jan 2025 08:58:40 +0000 Subject: [PATCH 1360/2091] add jemallctl base; add trim to interface w/ console cmd Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 9 +++ src/admin/debug/mod.rs | 3 + src/core/alloc/default.rs | 3 + src/core/alloc/hardened.rs | 2 + src/core/alloc/je.rs | 127 +++++++++++++++++++++++++++++++++++- src/core/alloc/mod.rs | 6 +- 6 files changed, 144 insertions(+), 6 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 07daaf0a..d027fa73 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -923,3 +923,12 @@ pub(super) async fn database_stats( Ok(RoomMessageEventContent::notice_markdown(out)) } + +#[admin_command] +pub(super) async fn trim_memory(&self) -> Result { + conduwuit::alloc::trim()?; + + writeln!(self, "done").await?; + + Ok(RoomMessageEventContent::notice_plain("")) +} diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index c87dbb0a..cc2a8ddd 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -207,6 +207,9 @@ pub(super) enum DebugCommand { map: Option, }, + /// - Trim memory usage + TrimMemory, + /// - Developer test stubs #[command(subcommand)] #[allow(non_snake_case)] diff --git a/src/core/alloc/default.rs b/src/core/alloc/default.rs index 83bfca7d..5db02884 100644 --- a/src/core/alloc/default.rs +++ b/src/core/alloc/default.rs @@ -1,5 +1,8 @@ //! Default allocator with no special features +/// Always returns Ok +pub fn trim() -> crate::Result { Ok(()) } + /// Always returns None #[must_use] pub fn memory_stats() -> Option { None } diff --git a/src/core/alloc/hardened.rs b/src/core/alloc/hardened.rs index 335a3307..e2d9b28e 100644 --- a/src/core/alloc/hardened.rs +++ b/src/core/alloc/hardened.rs @@ -3,6 +3,8 @@ #[global_allocator] static HMALLOC: hardened_malloc_rs::HardenedMalloc = hardened_malloc_rs::HardenedMalloc; +pub fn trim() -> crate::Result { Ok(()) } + #[must_use] //TODO: get usage pub fn memory_usage() -> Option { None } diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 423f5408..b2c1fe85 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -1,18 +1,45 @@ //! jemalloc allocator -use std::ffi::{c_char, c_void}; +use std::{ + cell::OnceCell, + ffi::{c_char, c_void}, + fmt::{Debug, Write}, +}; +use arrayvec::ArrayVec; +use tikv_jemalloc_ctl as mallctl; use tikv_jemalloc_sys as ffi; use tikv_jemallocator as jemalloc; +use crate::{err, is_equal_to, utils::math::Tried, Result}; + +#[cfg(feature = "jemalloc_conf")] +#[no_mangle] +pub static malloc_conf: &[u8] = b"\ +metadata_thp:always\ +,percpu_arena:percpu\ +,background_thread:true\ +,max_background_threads:-1\ +,lg_extent_max_active_fit:4\ +,oversize_threshold:33554432\ +,tcache_max:2097152\ +,dirty_decay_ms:16000\ +,muzzy_decay_ms:144000\ +\0"; + #[global_allocator] static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc; +type Key = ArrayVec; +type Name = ArrayVec; + +const KEY_SEGS: usize = 8; +const NAME_MAX: usize = 128; + #[must_use] #[cfg(feature = "jemalloc_stats")] pub fn memory_usage() -> Option { use mallctl::stats; - use tikv_jemalloc_ctl as mallctl; let mibs = |input: Result| { let input = input.unwrap_or_default(); @@ -62,7 +89,12 @@ pub fn memory_stats() -> Option { unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { // SAFETY: we have to trust the opaque points to our String - let res: &mut String = unsafe { opaque.cast::().as_mut().unwrap() }; + let res: &mut String = unsafe { + opaque + .cast::() + .as_mut() + .expect("failed to cast void* to &mut String") + }; // SAFETY: we have to trust the string is null terminated. let msg = unsafe { std::ffi::CStr::from_ptr(msg) }; @@ -70,3 +102,92 @@ unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { let msg = String::from_utf8_lossy(msg.to_bytes()); res.push_str(msg.as_ref()); } + +macro_rules! mallctl { + ($name:literal) => {{ + thread_local! { + static KEY: OnceCell = OnceCell::default(); + }; + + KEY.with(|once| { + once.get_or_init(move || key($name).expect("failed to translate name into mib key")) + .clone() + }) + }}; +} + +pub fn trim() -> Result { set(&mallctl!("arena.4096.purge"), ()) } + +pub fn decay() -> Result { set(&mallctl!("arena.4096.purge"), ()) } + +pub fn set_by_name(name: &str, val: T) -> Result { set(&key(name)?, val) } + +pub fn get_by_name(name: &str) -> Result { get(&key(name)?) } + +pub mod this_thread { + use super::{get, key, set, Key, OnceCell, Result}; + + pub fn trim() -> Result { + let mut key = mallctl!("arena.0.purge"); + key[1] = arena_id()?.try_into()?; + set(&key, ()) + } + + pub fn decay() -> Result { + let mut key = mallctl!("arena.0.decay"); + key[1] = arena_id()?.try_into()?; + set(&key, ()) + } + + pub fn cache(enable: bool) -> Result { + set(&mallctl!("thread.tcache.enabled"), u8::from(enable)) + } + + pub fn flush() -> Result { set(&mallctl!("thread.tcache.flush"), ()) } + + pub fn allocated() -> Result { get::(&mallctl!("thread.allocated")) } + + pub fn deallocated() -> Result { get::(&mallctl!("thread.deallocated")) } + + pub fn arena_id() -> Result { get::(&mallctl!("thread.arena")) } +} + +fn set(key: &Key, val: T) -> Result +where + T: Copy + Debug, +{ + // SAFETY: T must be the exact expected type. + unsafe { mallctl::raw::write_mib(key.as_slice(), val) }.map_err(map_err) +} + +fn get(key: &Key) -> Result +where + T: Copy + Debug, +{ + // SAFETY: T must be perfectly valid to receive value. + unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err) +} + +fn key(name: &str) -> Result { + // tikv asserts the output buffer length is tight to the number of required mibs + // so we slice that down here. + let segs = name.chars().filter(is_equal_to!(&'.')).count().try_add(1)?; + + let name = self::name(name)?; + let mut buf = [0_usize; KEY_SEGS]; + mallctl::raw::name_to_mib(name.as_slice(), &mut buf[0..segs]) + .map_err(map_err) + .map(move |()| buf.into_iter().take(segs).collect()) +} + +fn name(name: &str) -> Result { + let mut buf = Name::new(); + buf.try_extend_from_slice(name.as_bytes())?; + buf.try_extend_from_slice(b"\0")?; + + Ok(buf) +} + +fn map_err(error: tikv_jemalloc_ctl::Error) -> crate::Error { + err!("mallctl: {}", error.to_string()) +} diff --git a/src/core/alloc/mod.rs b/src/core/alloc/mod.rs index 31eb033c..0ed1b1a6 100644 --- a/src/core/alloc/mod.rs +++ b/src/core/alloc/mod.rs @@ -4,7 +4,7 @@ #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] pub mod je; #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] -pub use je::{memory_stats, memory_usage}; +pub use je::{memory_stats, memory_usage, trim}; #[cfg(all(not(target_env = "msvc"), feature = "hardened_malloc", not(feature = "jemalloc")))] pub mod hardened; @@ -13,7 +13,7 @@ pub mod hardened; feature = "hardened_malloc", not(feature = "jemalloc") ))] -pub use hardened::{memory_stats, memory_usage}; +pub use hardened::{memory_stats, memory_usage, trim}; #[cfg(any( target_env = "msvc", @@ -24,4 +24,4 @@ pub mod default; target_env = "msvc", all(not(feature = "hardened_malloc"), not(feature = "jemalloc")) ))] -pub use default::{memory_stats, memory_usage}; +pub use default::{memory_stats, memory_usage, trim}; From 77d8e26efe3b46e73325386837a0f95107213842 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 16 Jan 2025 19:08:54 +0000 Subject: [PATCH 1361/2091] integrate trim-on-park memory reclamation to runtime Signed-off-by: Jason Volk --- src/main/clap.rs | 12 ++++++++++++ src/main/runtime.rs | 22 +++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/main/clap.rs b/src/main/clap.rs index ad5c815a..d3d40491 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -80,6 +80,18 @@ pub(crate) struct Args { default_missing_value = "true", )] pub(crate) worker_affinity: bool, + + /// Toggles feature to promote memory reclamation by the operating system + /// when tokio worker runs out of work. + #[arg( + long, + hide(true), + env = "CONDUWUIT_RUNTIME_GC_ON_PARK", + action = ArgAction::Set, + num_args = 0..=1, + require_equals(false), + )] + pub(crate) gc_on_park: Option, } /// Parse commandline arguments into structured data diff --git a/src/main/runtime.rs b/src/main/runtime.rs index 3039ef1b..315336b0 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -9,6 +9,7 @@ use std::{ }; use conduwuit::{ + result::LogErr, utils::sys::compute::{nth_core_available, set_affinity}, Result, }; @@ -22,12 +23,17 @@ const WORKER_KEEPALIVE: u64 = 36; const MAX_BLOCKING_THREADS: usize = 1024; static WORKER_AFFINITY: OnceLock = OnceLock::new(); +static GC_ON_PARK: OnceLock> = OnceLock::new(); pub(super) fn new(args: &Args) -> Result { WORKER_AFFINITY .set(args.worker_affinity) .expect("set WORKER_AFFINITY from program argument"); + GC_ON_PARK + .set(args.gc_on_park) + .expect("set GC_ON_PARK from program argument"); + let mut builder = Builder::new_multi_thread(); builder .enable_io() @@ -138,7 +144,21 @@ fn thread_unpark() {} name = %thread::current().name().unwrap_or("None"), ), )] -fn thread_park() {} +fn thread_park() { + match GC_ON_PARK + .get() + .as_ref() + .expect("GC_ON_PARK initialized by runtime::new()") + { + | Some(true) | None if cfg!(feature = "jemalloc_conf") => gc_on_park(), + | _ => (), + } +} + +fn gc_on_park() { + #[cfg(feature = "jemalloc")] + conduwuit::alloc::je::this_thread::decay().log_err().ok(); +} #[cfg(tokio_unstable)] #[tracing::instrument( From 3759d1be6ca01a687b29df9e68c24c7723a55427 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 11 Jan 2025 00:08:35 +0000 Subject: [PATCH 1362/2091] tweak per-column write_buffer down from default Signed-off-by: Jason Volk --- src/database/engine/cf_opts.rs | 4 +--- src/database/engine/descriptor.rs | 8 ++++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index a68eb8b6..8cb659ac 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -32,9 +32,7 @@ fn descriptor_cf_options( opts.set_min_write_buffer_number(1); opts.set_max_write_buffer_number(2); - if let Some(write_size) = desc.write_size { - opts.set_write_buffer_size(write_size); - } + opts.set_write_buffer_size(desc.write_size); opts.set_target_file_size_base(desc.file_size); opts.set_target_file_size_multiplier(desc.file_shape[0]); diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index 9cf57d8f..ef08945e 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -20,7 +20,7 @@ pub(crate) struct Descriptor { pub(crate) val_size_hint: Option, pub(crate) block_size: usize, pub(crate) index_size: usize, - pub(crate) write_size: Option, + pub(crate) write_size: usize, pub(crate) cache_size: usize, pub(crate) level_size: u64, pub(crate) level_shape: [i32; 7], @@ -46,7 +46,7 @@ pub(crate) static BASE: Descriptor = Descriptor { val_size_hint: None, block_size: 1024 * 4, index_size: 1024 * 4, - write_size: None, + write_size: 1024 * 1024 * 2, cache_size: 1024 * 1024 * 4, level_size: 1024 * 1024 * 8, level_shape: [1, 1, 1, 3, 7, 15, 31], @@ -66,11 +66,13 @@ pub(crate) static BASE: Descriptor = Descriptor { pub(crate) static RANDOM: Descriptor = Descriptor { compaction_pri: CompactionPri::OldestSmallestSeqFirst, + write_size: 1024 * 1024 * 32, ..BASE }; pub(crate) static SEQUENTIAL: Descriptor = Descriptor { compaction_pri: CompactionPri::OldestLargestSeqFirst, + write_size: 1024 * 1024 * 64, level_size: 1024 * 1024 * 32, file_size: 1024 * 1024 * 2, ..BASE @@ -78,6 +80,7 @@ pub(crate) static SEQUENTIAL: Descriptor = Descriptor { pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { compaction: CompactionStyle::Universal, + write_size: 1024 * 1024 * 16, level_size: 1024 * 512, file_size: 1024 * 128, ..RANDOM @@ -85,6 +88,7 @@ pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { compaction: CompactionStyle::Universal, + write_size: 1024 * 1024 * 16, level_size: 1024 * 1024, file_size: 1024 * 512, ..SEQUENTIAL From aad42bdaa0734d76ae390b7c865673a1bf68c4ca Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 16 Jan 2025 20:47:00 +0000 Subject: [PATCH 1363/2091] reduce block size on small tables Signed-off-by: Jason Volk --- src/database/engine/descriptor.rs | 5 +++++ src/database/maps.rs | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index ef08945e..06e1a29b 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -83,6 +83,9 @@ pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { write_size: 1024 * 1024 * 16, level_size: 1024 * 512, file_size: 1024 * 128, + index_size: 512, + block_size: 512, + cache_shards: 64, ..RANDOM }; @@ -91,5 +94,7 @@ pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { write_size: 1024 * 1024 * 16, level_size: 1024 * 1024, file_size: 1024 * 512, + block_size: 512, + cache_shards: 64, ..SEQUENTIAL }; diff --git a/src/database/maps.rs b/src/database/maps.rs index 19e19955..82c5a4a8 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -58,6 +58,8 @@ pub(super) static MAPS: &[Descriptor] = &[ cache_disp: CacheDisp::SharedWith("pduid_pdu"), key_size_hint: Some(48), val_size_hint: Some(1488), + block_size: 1024, + index_size: 512, ..descriptor::RANDOM }, Descriptor { @@ -65,6 +67,8 @@ pub(super) static MAPS: &[Descriptor] = &[ cache_disp: CacheDisp::Unique, key_size_hint: Some(48), val_size_hint: Some(16), + block_size: 512, + index_size: 512, ..descriptor::RANDOM }, Descriptor { @@ -72,6 +76,8 @@ pub(super) static MAPS: &[Descriptor] = &[ cache_disp: CacheDisp::Unique, key_size_hint: Some(48), val_size_hint: Some(8), + block_size: 512, + index_size: 512, ..descriptor::RANDOM }, Descriptor { @@ -111,6 +117,8 @@ pub(super) static MAPS: &[Descriptor] = &[ cache_disp: CacheDisp::SharedWith("eventid_outlierpdu"), key_size_hint: Some(16), val_size_hint: Some(1520), + block_size: 2048, + index_size: 512, ..descriptor::SEQUENTIAL }, Descriptor { @@ -162,6 +170,7 @@ pub(super) static MAPS: &[Descriptor] = &[ Descriptor { name: "roomsynctoken_shortstatehash", val_size_hint: Some(8), + block_size: 512, ..descriptor::SEQUENTIAL }, Descriptor { @@ -243,6 +252,8 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "shorteventid_shortstatehash", key_size_hint: Some(8), val_size_hint: Some(8), + block_size: 512, + index_size: 512, ..descriptor::SEQUENTIAL }, Descriptor { @@ -292,7 +303,11 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "token_userdeviceid", ..descriptor::RANDOM_SMALL }, - Descriptor { name: "tokenids", ..descriptor::RANDOM }, + Descriptor { + name: "tokenids", + block_size: 512, + ..descriptor::RANDOM + }, Descriptor { name: "url_previews", ..descriptor::RANDOM From bab40a374707f9a50e2b440ab830d1a4456dd678 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 17 Jan 2025 20:34:56 +0000 Subject: [PATCH 1364/2091] enable hashing on large-block indexes Signed-off-by: Jason Volk --- src/database/engine/cf_opts.rs | 16 +++++++--------- src/database/engine/descriptor.rs | 5 +++-- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 8cb659ac..158fb3c8 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -1,8 +1,4 @@ -use conduwuit::{ - err, - utils::{math::Expected, BoolExt}, - Config, Result, -}; +use conduwuit::{err, utils::math::Expected, Config, Result}; use rocksdb::{ BlockBasedIndexType, BlockBasedOptions, BlockBasedPinningTier, Cache, DBCompressionType as CompressionType, DataBlockIndexType, LruCacheOptions, Options, @@ -133,10 +129,12 @@ fn table_options(desc: &Descriptor, has_cache: bool) -> BlockBasedOptions { opts.set_partition_filters(true); opts.set_use_delta_encoding(false); opts.set_index_type(BlockBasedIndexType::TwoLevelIndexSearch); - opts.set_data_block_index_type( - desc.block_index_hashing - .map_or(DataBlockIndexType::BinarySearch, || DataBlockIndexType::BinaryAndHash), - ); + + opts.set_data_block_index_type(match desc.block_index_hashing { + | None if desc.index_size > 512 => DataBlockIndexType::BinaryAndHash, + | Some(enable) if enable => DataBlockIndexType::BinaryAndHash, + | Some(_) | None => DataBlockIndexType::BinarySearch, + }); opts } diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index 06e1a29b..d668862b 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -34,7 +34,7 @@ pub(crate) struct Descriptor { pub(crate) compression: CompressionType, pub(crate) compression_level: i32, pub(crate) bottommost_level: Option, - pub(crate) block_index_hashing: bool, + pub(crate) block_index_hashing: Option, pub(crate) cache_shards: u32, } @@ -60,7 +60,7 @@ pub(crate) static BASE: Descriptor = Descriptor { compression: CompressionType::Zstd, compression_level: 32767, bottommost_level: Some(32767), - block_index_hashing: false, + block_index_hashing: None, cache_shards: 64, }; @@ -96,5 +96,6 @@ pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { file_size: 1024 * 512, block_size: 512, cache_shards: 64, + block_index_hashing: Some(false), ..SEQUENTIAL }; From 819e35f81fdfb6d7045b24f7fdf7d647e75ffb22 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 14 Jan 2025 08:01:12 +0000 Subject: [PATCH 1365/2091] remove mutex lock/unlock during sync iteration. Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index a4dc0205..d6b9f15c 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -242,10 +242,6 @@ pub(crate) async fn build_sync_events( .state_cache .rooms_invited(sender_user) .fold_default(|mut invited_rooms: BTreeMap<_, _>, (room_id, invite_state)| async move { - // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; - drop(insert_lock); - let invite_count = services .rooms .state_cache @@ -271,10 +267,6 @@ pub(crate) async fn build_sync_events( .state_cache .rooms_knocked(sender_user) .fold_default(|mut knocked_rooms: BTreeMap<_, _>, (room_id, knock_state)| async move { - // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services.rooms.timeline.mutex_insert.lock(&room_id).await; - drop(insert_lock); - let knock_count = services .rooms .state_cache @@ -470,10 +462,6 @@ async fn handle_left_room( full_state: bool, lazy_load_enabled: bool, ) -> Result> { - // Get and drop the lock to wait for remaining operations to finish - let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; - drop(insert_lock); - let left_count = services .rooms .state_cache @@ -627,11 +615,6 @@ async fn load_joined_room( lazy_load_send_redundant: bool, full_state: bool, ) -> Result<(JoinedRoom, HashSet, HashSet)> { - // Get and drop the lock to wait for remaining operations to finish - // This will make sure the we have all events until next_batch - let insert_lock = services.rooms.timeline.mutex_insert.lock(room_id).await; - drop(insert_lock); - let sincecount = PduCount::Normal(since); let next_batchcount = PduCount::Normal(next_batch); From fc1170e12a17f47cb1c0b60fc873dcfae127ccea Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 14 Jan 2025 05:20:42 +0000 Subject: [PATCH 1366/2091] additional tracing span tweaks Signed-off-by: Jason Volk --- src/api/server/send.rs | 10 ++++- src/database/pool.rs | 5 +-- src/service/rooms/auth_chain/mod.rs | 4 +- src/service/rooms/event_handler/fetch_prev.rs | 2 +- .../rooms/event_handler/fetch_state.rs | 2 +- .../event_handler/handle_incoming_pdu.rs | 2 +- .../rooms/event_handler/handle_prev_pdu.rs | 2 +- .../rooms/event_handler/resolve_state.rs | 2 +- .../rooms/event_handler/state_at_incoming.rs | 4 +- src/service/rooms/state_compressor/mod.rs | 43 +++++++++++-------- src/service/rooms/timeline/mod.rs | 4 +- 11 files changed, 47 insertions(+), 33 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index c0c8a0c9..56a17c22 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -39,7 +39,15 @@ type ResolvedMap = BTreeMap>; /// # `PUT /_matrix/federation/v1/send/{txnId}` /// /// Push EDUs and PDUs to this server. -#[tracing::instrument(skip_all, fields(%client, origin = body.origin().as_str()), name = "send")] +#[tracing::instrument( + name = "send", + level = "debug", + skip_all, + fields( + %client, + origin = body.origin().as_str() + ), +)] pub(crate) async fn send_transaction_message_route( State(services): State, InsecureClientIp(client): InsecureClientIp, diff --git a/src/database/pool.rs b/src/database/pool.rs index b972e763..f5600c36 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -12,7 +12,7 @@ use std::{ use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ - debug, debug_warn, defer, err, error, implement, + debug, debug_warn, err, error, implement, result::DebugInspect, trace, utils::sys::compute::{get_affinity, nth_core_available, set_affinity}, @@ -271,9 +271,6 @@ async fn execute(&self, queue: &Sender, cmd: Cmd) -> Result { ), )] fn worker(self: Arc, id: usize, recv: Receiver) { - defer! {{ trace!("worker finished"); }} - trace!("worker spawned"); - self.worker_init(id); self.worker_loop(&recv); } diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index f6534825..74064701 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -79,7 +79,7 @@ impl Service { Ok(event_ids) } - #[tracing::instrument(skip_all, name = "auth_chain")] + #[tracing::instrument(name = "auth_chain", level = "debug", skip_all)] pub async fn get_auth_chain<'a, I>( &'a self, room_id: &RoomId, @@ -179,7 +179,7 @@ impl Service { Ok(full_auth_chain) } - #[tracing::instrument(skip(self, room_id))] + #[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))] async fn get_auth_chain_inner( &self, room_id: &RoomId, diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 0d64e98e..5966aeba 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -16,7 +16,7 @@ use super::check_room_id; #[implement(super::Service)] #[tracing::instrument( - level = "warn", + level = "debug", skip_all, fields(%origin), )] diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index cc4a3e46..0892655e 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -14,7 +14,7 @@ use crate::rooms::short::ShortStateKey; /// on the events #[implement(super::Service)] #[tracing::instrument( - level = "warn", + level = "debug", skip_all, fields(%origin), )] diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index c2e6ccc9..4e6f0b0c 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -41,7 +41,7 @@ use crate::rooms::timeline::RawPduId; #[implement(super::Service)] #[tracing::instrument( name = "pdu", - level = "warn", + level = "debug", skip_all, fields(%room_id, %event_id), )] diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index ad71c173..2bec4eba 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -14,7 +14,7 @@ use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName}; #[allow(clippy::too_many_arguments)] #[tracing::instrument( name = "prev", - level = "warn", + level = "debug", skip_all, fields(%prev_id), )] diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index f21f7b66..8640c582 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -18,7 +18,7 @@ use ruma::{ use crate::rooms::state_compressor::CompressedStateEvent; #[implement(super::Service)] -#[tracing::instrument(skip_all, name = "resolve")] +#[tracing::instrument(name = "resolve", level = "debug", skip_all)] pub async fn resolve_state( &self, room_id: &RoomId, diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index fa2ce1cd..9e7f8d2a 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -16,7 +16,7 @@ use ruma::{state_res::StateMap, OwnedEventId, RoomId, RoomVersionId}; // TODO: if we know the prev_events of the incoming event we can avoid the #[implement(super::Service)] // request and build the state from a known point and resolve if > 1 prev_event -#[tracing::instrument(skip_all, name = "state")] +#[tracing::instrument(name = "state", level = "debug", skip_all)] pub(super) async fn state_at_incoming_degree_one( &self, incoming_pdu: &Arc, @@ -66,7 +66,7 @@ pub(super) async fn state_at_incoming_degree_one( } #[implement(super::Service)] -#[tracing::instrument(skip_all, name = "state")] +#[tracing::instrument(name = "state", level = "debug", skip_all)] pub(super) async fn state_at_incoming_resolved( &self, incoming_pdu: &Arc, diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index a61a66a1..532df360 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -7,7 +7,7 @@ use std::{ use arrayvec::ArrayVec; use conduwuit::{ - at, checked, debug, err, expected, utils, + at, checked, err, expected, utils, utils::{bytes, math::usize_from_f64, stream::IterStream}, Result, }; @@ -117,35 +117,44 @@ impl crate::Service for Service { impl Service { /// Returns a stack with info on shortstatehash, full state, added diff and /// removed diff for the selected shortstatehash and each parent layer. + #[tracing::instrument(name = "load", level = "debug", skip(self))] pub async fn load_shortstatehash_info( &self, shortstatehash: ShortStateHash, ) -> Result { - if let Some(r) = self - .stateinfo_cache - .lock() - .expect("locked") - .get_mut(&shortstatehash) - { + if let Some(r) = self.stateinfo_cache.lock()?.get_mut(&shortstatehash) { return Ok(r.clone()); } let stack = self.new_shortstatehash_info(shortstatehash).await?; - debug!( - ?shortstatehash, - len = %stack.len(), - "cache update" - ); - - self.stateinfo_cache - .lock() - .expect("locked") - .insert(shortstatehash, stack.clone()); + self.cache_shortstatehash_info(shortstatehash, stack.clone()) + .await?; Ok(stack) } + /// Returns a stack with info on shortstatehash, full state, added diff and + /// removed diff for the selected shortstatehash and each parent layer. + #[tracing::instrument( + name = "cache", + level = "debug", + skip_all, + fields( + ?shortstatehash, + stack = stack.len(), + ), + )] + async fn cache_shortstatehash_info( + &self, + shortstatehash: ShortStateHash, + stack: ShortStateInfoVec, + ) -> Result { + self.stateinfo_cache.lock()?.insert(shortstatehash, stack); + + Ok(()) + } + async fn new_shortstatehash_info( &self, shortstatehash: ShortStateHash, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 3ebc432f..bd60e40e 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -259,7 +259,7 @@ impl Service { /// happens in `append_pdu`. /// /// Returns pdu id - #[tracing::instrument(skip_all)] + #[tracing::instrument(level = "debug", skip_all)] pub async fn append_pdu( &self, pdu: &PduEvent, @@ -942,7 +942,7 @@ impl Service { /// Append the incoming event setting the state snapshot to the state from /// the server that sent the event. - #[tracing::instrument(skip_all)] + #[tracing::instrument(level = "debug", skip_all)] pub async fn append_incoming_pdu( &self, pdu: &PduEvent, From 96e85adc32f68e2080da9cf0088d9da84858747e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 01:34:14 +0000 Subject: [PATCH 1367/2091] use cache builder for row and table cache options add cache check using multi-get path Signed-off-by: Jason Volk --- src/database/engine/cf_opts.rs | 2 +- src/database/engine/context.rs | 19 +++++++--- src/database/engine/descriptor.rs | 2 ++ src/database/map/get.rs | 60 +++++++++++++++++++++---------- src/database/map/get_batch.rs | 41 ++++++++++++++++----- 5 files changed, 90 insertions(+), 34 deletions(-) diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 158fb3c8..7b3a1d49 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -178,7 +178,7 @@ fn get_cache(ctx: &Context, desc: &Descriptor) -> Option { .try_into() .expect("u32 to i32 conversion"); - debug_assert!(shard_bits <= 6, "cache shards limited to 64"); + debug_assert!(shard_bits <= 10, "cache shards probably too large"); let mut cache_opts = LruCacheOptions::default(); cache_opts.set_num_shard_bits(shard_bits); cache_opts.set_capacity(size); diff --git a/src/database/engine/context.rs b/src/database/engine/context.rs index 76238f7d..04e08854 100644 --- a/src/database/engine/context.rs +++ b/src/database/engine/context.rs @@ -4,7 +4,7 @@ use std::{ }; use conduwuit::{debug, utils::math::usize_from_f64, Result, Server}; -use rocksdb::{Cache, Env}; +use rocksdb::{Cache, Env, LruCacheOptions}; use crate::{or_else, pool::Pool}; @@ -25,12 +25,21 @@ impl Context { let config = &server.config; let cache_capacity_bytes = config.db_cache_capacity_mb * 1024.0 * 1024.0; - let row_cache_capacity_bytes = usize_from_f64(cache_capacity_bytes * 0.50)?; - let row_cache = Cache::new_lru_cache(row_cache_capacity_bytes); - + let col_shard_bits = 7; let col_cache_capacity_bytes = usize_from_f64(cache_capacity_bytes * 0.50)?; - let col_cache = Cache::new_lru_cache(col_cache_capacity_bytes); + let row_shard_bits = 7; + let row_cache_capacity_bytes = usize_from_f64(cache_capacity_bytes * 0.50)?; + + let mut row_cache_opts = LruCacheOptions::default(); + row_cache_opts.set_num_shard_bits(row_shard_bits); + row_cache_opts.set_capacity(row_cache_capacity_bytes); + let row_cache = Cache::new_lru_cache_opts(&row_cache_opts); + + let mut col_cache_opts = LruCacheOptions::default(); + col_cache_opts.set_num_shard_bits(col_shard_bits); + col_cache_opts.set_capacity(col_cache_capacity_bytes); + let col_cache = Cache::new_lru_cache_opts(&col_cache_opts); let col_cache: BTreeMap<_, _> = [("Shared".to_owned(), col_cache)].into(); let mut env = Env::new().or_else(or_else)?; diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index d668862b..234ca2bf 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -67,6 +67,7 @@ pub(crate) static BASE: Descriptor = Descriptor { pub(crate) static RANDOM: Descriptor = Descriptor { compaction_pri: CompactionPri::OldestSmallestSeqFirst, write_size: 1024 * 1024 * 32, + cache_shards: 128, ..BASE }; @@ -75,6 +76,7 @@ pub(crate) static SEQUENTIAL: Descriptor = Descriptor { write_size: 1024 * 1024 * 64, level_size: 1024 * 1024 * 32, file_size: 1024 * 1024 * 2, + cache_shards: 128, ..BASE }; diff --git a/src/database/map/get.rs b/src/database/map/get.rs index e64ef2ec..73182042 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -3,6 +3,7 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; use arrayvec::ArrayVec; use conduwuit::{err, implement, utils::result::MapExpect, Err, Result}; use futures::{future::ready, Future, FutureExt, TryFutureExt}; +use rocksdb::{DBPinnableSlice, ReadOptions}; use serde::Serialize; use tokio::task; @@ -90,6 +91,17 @@ where .boxed() } +/// Fetch a value from the cache without I/O. +#[implement(super::Map)] +#[tracing::instrument(skip(self, key), name = "cache", level = "trace")] +pub(crate) fn get_cached(&self, key: &K) -> Result>> +where + K: AsRef<[u8]> + Debug + ?Sized, +{ + let res = self.get_blocking_opts(key, &self.cache_read_options); + cached_handle_from(res) +} + /// Fetch a value from the database into cache, returning a reference-handle. /// The key is referenced directly to perform the query. This is a thread- /// blocking call. @@ -99,37 +111,47 @@ pub fn get_blocking(&self, key: &K) -> Result> where K: AsRef<[u8]> + ?Sized, { - self.db - .db - .get_pinned_cf_opt(&self.cf(), key, &self.read_options) + let res = self.get_blocking_opts(key, &self.read_options); + handle_from(res) +} + +#[implement(super::Map)] +fn get_blocking_opts( + &self, + key: &K, + read_options: &ReadOptions, +) -> Result>, rocksdb::Error> +where + K: AsRef<[u8]> + ?Sized, +{ + self.db.db.get_pinned_cf_opt(&self.cf(), key, read_options) +} + +#[inline] +pub(super) fn handle_from( + result: Result>, rocksdb::Error>, +) -> Result> { + result .map_err(map_err)? .map(Handle::from) .ok_or(err!(Request(NotFound("Not found in database")))) } -/// Fetch a value from the cache without I/O. -#[implement(super::Map)] -#[tracing::instrument(skip(self, key), name = "cache", level = "trace")] -pub(crate) fn get_cached(&self, key: &K) -> Result>> -where - K: AsRef<[u8]> + Debug + ?Sized, -{ - let res = self - .db - .db - .get_pinned_cf_opt(&self.cf(), key, &self.cache_read_options); - - match res { +#[inline] +pub(super) fn cached_handle_from( + result: Result>, rocksdb::Error>, +) -> Result>> { + match result { // cache hit; not found | Ok(None) => Err!(Request(NotFound("Not found in database"))), // cache hit; value found - | Ok(Some(res)) => Ok(Some(Handle::from(res))), + | Ok(Some(result)) => Ok(Some(Handle::from(result))), // cache miss; unknown - | Err(e) if is_incomplete(&e) => Ok(None), + | Err(error) if is_incomplete(&error) => Ok(None), // some other error occurred - | Err(e) => or_else(e), + | Err(error) => or_else(error), } } diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index 452697f1..ee9269e3 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -1,7 +1,7 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; use conduwuit::{ - err, implement, + implement, utils::{ stream::{automatic_amplification, automatic_width, WidebandExt}, IterStream, @@ -9,9 +9,11 @@ use conduwuit::{ Result, }; use futures::{Stream, StreamExt, TryStreamExt}; +use rocksdb::{DBPinnableSlice, ReadOptions}; use serde::Serialize; -use crate::{keyval::KeyBuf, ser, util::map_err, Handle}; +use super::get::{cached_handle_from, handle_from}; +use crate::{keyval::KeyBuf, ser, Handle}; #[implement(super::Map)] #[tracing::instrument(skip(self, keys), level = "trace")] @@ -66,12 +68,40 @@ where .try_flatten() } +#[implement(super::Map)] +#[tracing::instrument(name = "batch_cached", level = "trace", skip_all)] +pub(crate) fn get_batch_cached<'a, I, K>( + &self, + keys: I, +) -> impl Iterator>>> + Send +where + I: Iterator + ExactSizeIterator + Send, + K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, +{ + self.get_batch_blocking_opts(keys, &self.cache_read_options) + .map(cached_handle_from) +} + #[implement(super::Map)] #[tracing::instrument(name = "batch_blocking", level = "trace", skip_all)] pub(crate) fn get_batch_blocking<'a, I, K>( &self, keys: I, ) -> impl Iterator>> + Send +where + I: Iterator + ExactSizeIterator + Send, + K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, +{ + self.get_batch_blocking_opts(keys, &self.read_options) + .map(handle_from) +} + +#[implement(super::Map)] +fn get_batch_blocking_opts<'a, I, K>( + &self, + keys: I, + read_options: &ReadOptions, +) -> impl Iterator>, rocksdb::Error>> + Send where I: Iterator + ExactSizeIterator + Send, K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, @@ -80,15 +110,8 @@ where // comparator**. const SORTED: bool = false; - let read_options = &self.read_options; self.db .db .batched_multi_get_cf_opt(&self.cf(), keys, SORTED, read_options) .into_iter() - .map(|result| { - result - .map_err(map_err)? - .map(Handle::from) - .ok_or(err!(Request(NotFound("Not found in database")))) - }) } From abf33013e37e89e1581c8ca9ab9b1411d51d7513 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 01:38:09 +0000 Subject: [PATCH 1368/2091] check-in additional database test related Signed-off-by: Jason Volk --- src/database/tests.rs | 97 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/src/database/tests.rs b/src/database/tests.rs index 0a42ad60..2f143698 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -35,6 +35,29 @@ fn ser_tuple() { assert_eq!(a, b); } +#[test] +fn ser_tuple_option() { + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + + let mut a = Vec::::new(); + a.push(0xFF); + a.extend_from_slice(user_id.as_bytes()); + + let mut aa = Vec::::new(); + aa.extend_from_slice(room_id.as_bytes()); + aa.push(0xFF); + aa.extend_from_slice(user_id.as_bytes()); + + let b: (Option<&RoomId>, &UserId) = (None, user_id); + let b = serialize_to_vec(&b).expect("failed to serialize tuple"); + assert_eq!(a, b); + + let bb: (Option<&RoomId>, &UserId) = (Some(room_id), user_id); + let bb = serialize_to_vec(&bb).expect("failed to serialize tuple"); + assert_eq!(aa, bb); +} + #[test] #[should_panic(expected = "I/O error: failed to write whole buffer")] fn ser_overflow() { @@ -284,6 +307,8 @@ fn ser_array() { let b: u64 = 987_654; let arr: &[u64] = &[a, b]; + let vec: Vec = vec![a, b]; + let arv: ArrayVec = [a, b].into(); let mut v = Vec::new(); v.extend_from_slice(&a.to_be_bytes()); @@ -291,4 +316,76 @@ fn ser_array() { let s = serialize_to_vec(arr).expect("failed to serialize"); assert_eq!(&s, &v, "serialization does not match"); + + let s = serialize_to_vec(arv.as_slice()).expect("failed to serialize arrayvec"); + assert_eq!(&s, &v, "arrayvec serialization does not match"); + + let s = serialize_to_vec(&vec).expect("failed to serialize vec"); + assert_eq!(&s, &v, "vec serialization does not match"); +} + +#[cfg(todo)] +#[test] +fn de_array() { + let a: u64 = 123_456; + let b: u64 = 987_654; + + let mut v: Vec = Vec::new(); + v.extend_from_slice(&a.to_be_bytes()); + v.extend_from_slice(&b.to_be_bytes()); + + let arv: ArrayVec = de::from_slice::>(v.as_slice()) + .map(TryInto::try_into) + .expect("failed to deserialize to arrayvec") + .expect("failed to deserialize into"); + + assert_eq!(arv[0], a, "deserialized arv [0] does not match"); + assert_eq!(arv[1], b, "deserialized arv [1] does not match"); + + let arr: [u64; 2] = de::from_slice::<[u64; 2]>(v.as_slice()) + .map(TryInto::try_into) + .expect("failed to deserialize to array") + .expect("failed to deserialize into"); + + assert_eq!(arr[0], a, "deserialized arr [0] does not match"); + assert_eq!(arr[1], b, "deserialized arr [1] does not match"); + + let vec: Vec = de::from_slice(v.as_slice()).expect("failed to deserialize to vec"); + + assert_eq!(vec[0], a, "deserialized vec [0] does not match"); + assert_eq!(vec[1], b, "deserialized vec [1] does not match"); +} + +#[cfg(todo)] +#[test] +fn de_complex() { + type Key<'a> = (&'a UserId, ArrayVec, &'a RoomId); + + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + let a: u64 = 123_456; + let b: u64 = 987_654; + + let mut v = Vec::new(); + v.extend_from_slice(user_id.as_bytes()); + v.extend_from_slice(b"\xFF"); + v.extend_from_slice(&a.to_be_bytes()); + v.extend_from_slice(&b.to_be_bytes()); + v.extend_from_slice(b"\xFF"); + v.extend_from_slice(room_id.as_bytes()); + + let arr: &[u64] = &[a, b]; + let key = (user_id, arr, room_id); + let s = serialize_to_vec(&key).expect("failed to serialize"); + + assert_eq!(&s, &v, "serialization does not match"); + + let key = (user_id, [a, b].into(), room_id); + let arr: Key<'_> = de::from_slice(&v).expect("failed to deserialize"); + + assert_eq!(arr, key, "deserialization does not match"); + + let arr: Key<'_> = de::from_slice(&s).expect("failed to deserialize"); + + assert_eq!(arr, key, "deserialization of serialization does not match"); } From 8141ca34448452fd8fd910c85626a2568a3ebe55 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 4 Jan 2025 16:57:07 +0000 Subject: [PATCH 1369/2091] refactor admin outputs to asyncwrite Signed-off-by: Jason Volk --- src/admin/admin.rs | 12 +- src/admin/check/mod.rs | 15 +- src/admin/command.rs | 30 +- src/admin/debug/commands.rs | 39 ++- src/admin/debug/mod.rs | 12 +- src/admin/debug/tester.rs | 2 +- src/admin/processor.rs | 43 ++- src/admin/query/account_data.rs | 74 ++--- src/admin/query/appservice.rs | 15 +- src/admin/query/globals.rs | 24 +- src/admin/query/mod.rs | 20 +- src/admin/query/presence.rs | 16 +- src/admin/query/pusher.rs | 12 +- src/admin/query/raw.rs | 457 ++++++++++++++++++++++++++++ src/admin/query/resolver.rs | 82 ++--- src/admin/query/room_alias.rs | 20 +- src/admin/query/room_state_cache.rs | 47 +-- src/admin/query/sending.rs | 9 +- src/admin/query/short.rs | 45 +++ src/admin/query/users.rs | 46 +++ src/admin/room/alias.rs | 9 +- src/admin/room/directory.rs | 8 +- src/admin/user/commands.rs | 51 ++-- src/macros/admin.rs | 16 +- 24 files changed, 877 insertions(+), 227 deletions(-) create mode 100644 src/admin/query/raw.rs create mode 100644 src/admin/query/short.rs diff --git a/src/admin/admin.rs b/src/admin/admin.rs index 9097a613..b6de1ec6 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -1,6 +1,5 @@ use clap::Parser; use conduwuit::Result; -use ruma::events::room::message::RoomMessageEventContent; use crate::{ appservice, appservice::AppserviceCommand, check, check::CheckCommand, command::Command, @@ -50,13 +49,10 @@ pub(super) enum AdminCommand { } #[tracing::instrument(skip_all, name = "command")] -pub(super) async fn process( - command: AdminCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(command: AdminCommand, context: &Command<'_>) -> Result { use AdminCommand::*; - Ok(match command { + match command { | Appservices(command) => appservice::process(command, context).await?, | Media(command) => media::process(command, context).await?, | Users(command) => user::process(command, context).await?, @@ -66,5 +62,7 @@ pub(super) async fn process( | Debug(command) => debug::process(command, context).await?, | Query(command) => query::process(command, context).await?, | Check(command) => check::process(command, context).await?, - }) + }; + + Ok(()) } diff --git a/src/admin/check/mod.rs b/src/admin/check/mod.rs index 4790a6de..30b335c4 100644 --- a/src/admin/check/mod.rs +++ b/src/admin/check/mod.rs @@ -2,20 +2,11 @@ mod commands; use clap::Subcommand; use conduwuit::Result; -use ruma::events::room::message::RoomMessageEventContent; -use crate::Command; +use crate::admin_command_dispatch; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] pub(super) enum CheckCommand { - AllUsers, -} - -pub(super) async fn process( - command: CheckCommand, - context: &Command<'_>, -) -> Result { - Ok(match command { - | CheckCommand::AllUsers => context.check_all_users().await?, - }) + CheckAllUsers, } diff --git a/src/admin/command.rs b/src/admin/command.rs index 5277b976..5ad9e581 100644 --- a/src/admin/command.rs +++ b/src/admin/command.rs @@ -1,6 +1,12 @@ -use std::time::SystemTime; +use std::{fmt, time::SystemTime}; +use conduwuit::Result; use conduwuit_service::Services; +use futures::{ + io::{AsyncWriteExt, BufWriter}, + lock::Mutex, + Future, FutureExt, +}; use ruma::EventId; pub(crate) struct Command<'a> { @@ -8,4 +14,26 @@ pub(crate) struct Command<'a> { pub(crate) body: &'a [&'a str], pub(crate) timer: SystemTime, pub(crate) reply_id: Option<&'a EventId>, + pub(crate) output: Mutex>>, +} + +impl Command<'_> { + pub(crate) fn write_fmt( + &self, + arguments: fmt::Arguments<'_>, + ) -> impl Future + Send + '_ { + let buf = format!("{arguments}"); + self.output.lock().then(|mut output| async move { + output.write_all(buf.as_bytes()).await.map_err(Into::into) + }) + } + + pub(crate) fn write_str<'a>( + &'a self, + s: &'a str, + ) -> impl Future + Send + 'a { + self.output.lock().then(move |mut output| async move { + output.write_all(s.as_bytes()).await.map_err(Into::into) + }) + } } diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index d027fa73..b6189f6a 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,7 +6,8 @@ use std::{ }; use conduwuit::{ - debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, Result, + debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, PduId, + RawPduId, Result, }; use futures::{FutureExt, StreamExt}; use ruma::{ @@ -15,7 +16,10 @@ use ruma::{ CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, }; -use service::rooms::state_compressor::HashSetCompressStateEvent; +use service::rooms::{ + short::{ShortEventId, ShortRoomId}, + state_compressor::HashSetCompressStateEvent, +}; use tracing_subscriber::EnvFilter; use crate::admin_command; @@ -131,6 +135,35 @@ pub(super) async fn get_pdu(&self, event_id: Box) -> Result Result { + let pdu_id: RawPduId = PduId { + shortroomid, + shorteventid: shorteventid.into(), + } + .into(); + + let pdu_json = self + .services + .rooms + .timeline + .get_pdu_json_from_id(&pdu_id) + .await; + + match pdu_json { + | Ok(json) => { + let json_text = + serde_json::to_string_pretty(&json).expect("canonical json is valid json"); + Ok(RoomMessageEventContent::notice_markdown(format!("```json\n{json_text}\n```",))) + }, + | Err(_) => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")), + } +} + #[admin_command] pub(super) async fn get_remote_pdu_list( &self, @@ -895,7 +928,7 @@ pub(super) async fn list_dependencies(&self, names: bool) -> Result, }, + /// - Retrieve and print a PDU by PduId from the conduwuit database + GetShortPdu { + /// Shortroomid integer + shortroomid: ShortRoomId, + + /// Shorteventid integer + shorteventid: ShortEventId, + }, + /// - Attempts to retrieve a PDU from a remote server. Inserts it into our /// database/timeline if found and we do not have this PDU already /// (following normal event auth rules, handles it as an incoming PDU). diff --git a/src/admin/debug/tester.rs b/src/admin/debug/tester.rs index 5f922ece..5200fa0d 100644 --- a/src/admin/debug/tester.rs +++ b/src/admin/debug/tester.rs @@ -31,7 +31,7 @@ async fn failure(&self) -> Result { #[admin_command] async fn tester(&self) -> Result { - Ok(RoomMessageEventContent::notice_plain("completed")) + Ok(RoomMessageEventContent::notice_plain("legacy")) } #[inline(never)] diff --git a/src/admin/processor.rs b/src/admin/processor.rs index ed7d5ed1..eefcdcd6 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -1,5 +1,6 @@ use std::{ fmt::Write, + mem::take, panic::AssertUnwindSafe, sync::{Arc, Mutex}, time::SystemTime, @@ -17,7 +18,7 @@ use conduwuit::{ utils::string::{collect_stream, common_prefix}, warn, Error, Result, }; -use futures::future::FutureExt; +use futures::{future::FutureExt, io::BufWriter, AsyncWriteExt}; use ruma::{ events::{ relation::InReplyTo, @@ -62,9 +63,32 @@ async fn process_command(services: Arc, input: &CommandInput) -> Proce body: &body, timer: SystemTime::now(), reply_id: input.reply_id.as_deref(), + output: BufWriter::new(Vec::new()).into(), }; - process(&context, command, &args).await + let (result, mut logs) = process(&context, command, &args).await; + + let output = &mut context.output.lock().await; + output.flush().await.expect("final flush of output stream"); + + let output = + String::from_utf8(take(output.get_mut())).expect("invalid utf8 in command output stream"); + + match result { + | Ok(()) if logs.is_empty() => + Ok(Some(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id))), + + | Ok(()) => { + logs.write_str(output.as_str()).expect("output buffer"); + Ok(Some(reply(RoomMessageEventContent::notice_markdown(logs), context.reply_id))) + }, + | Err(error) => { + write!(&mut logs, "Command failed with error:\n```\n{error:#?}\n```") + .expect("output buffer"); + + Err(reply(RoomMessageEventContent::notice_markdown(logs), context.reply_id)) + }, + } } fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult { @@ -81,7 +105,7 @@ async fn process( context: &Command<'_>, command: AdminCommand, args: &[String], -) -> ProcessorResult { +) -> (Result, String) { let (capture, logs) = capture_create(context); let capture_scope = capture.start(); @@ -104,18 +128,7 @@ async fn process( } drop(logs); - match result { - | Ok(content) => { - write!(&mut output, "{0}", content.body()) - .expect("failed to format command result to output buffer"); - Ok(Some(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id))) - }, - | Err(error) => { - write!(&mut output, "Command failed with error:\n```\n{error:#?}\n```") - .expect("failed to format command result to output"); - Err(reply(RoomMessageEventContent::notice_markdown(output), context.reply_id)) - }, - } + (result, output) } fn capture_create(context: &Command<'_>) -> (Arc, Arc>) { diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index 43762789..b75d8234 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -3,8 +3,9 @@ use conduwuit::Result; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId, UserId}; -use crate::Command; +use crate::{admin_command, admin_command_dispatch}; +#[admin_command_dispatch] #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/account_data.rs pub(crate) enum AccountDataCommand { @@ -19,7 +20,7 @@ pub(crate) enum AccountDataCommand { }, /// - Searches the account data for a specific kind. - Get { + AccountDataGet { /// Full user ID user_id: Box, /// Account data event type @@ -29,38 +30,43 @@ pub(crate) enum AccountDataCommand { }, } -/// All the getters and iterators from src/database/key_value/account_data.rs -pub(super) async fn process( - subcommand: AccountDataCommand, - context: &Command<'_>, +#[admin_command] +async fn changes_since( + &self, + user_id: Box, + since: u64, + room_id: Option>, ) -> Result { - let services = context.services; + let timer = tokio::time::Instant::now(); + let results: Vec<_> = self + .services + .account_data + .changes_since(room_id.as_deref(), &user_id, since) + .collect() + .await; + let query_time = timer.elapsed(); - match subcommand { - | AccountDataCommand::ChangesSince { user_id, since, room_id } => { - let timer = tokio::time::Instant::now(); - let results: Vec<_> = services - .account_data - .changes_since(room_id.as_deref(), &user_id, since) - .collect() - .await; - let query_time = timer.elapsed(); - - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) - }, - | AccountDataCommand::Get { user_id, kind, room_id } => { - let timer = tokio::time::Instant::now(); - let results = services - .account_data - .get_raw(room_id.as_deref(), &user_id, &kind) - .await; - let query_time = timer.elapsed(); - - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) - }, - } + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) +} + +#[admin_command] +async fn account_data_get( + &self, + user_id: Box, + kind: String, + room_id: Option>, +) -> Result { + let timer = tokio::time::Instant::now(); + let results = self + .services + .account_data + .get_raw(room_id.as_deref(), &user_id, &kind) + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + ))) } diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index fe4861bc..f9e1fd2c 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -1,6 +1,5 @@ use clap::Subcommand; use conduwuit::Result; -use ruma::events::room::message::RoomMessageEventContent; use crate::Command; @@ -18,10 +17,7 @@ pub(crate) enum AppserviceCommand { } /// All the getters and iterators from src/database/key_value/appservice.rs -pub(super) async fn process( - subcommand: AppserviceCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_>) -> Result { let services = context.services; match subcommand { @@ -31,18 +27,15 @@ pub(super) async fn process( let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, | AppserviceCommand::All => { let timer = tokio::time::Instant::now(); let results = services.appservice.all().await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, } + .await } diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index e356453f..1642f7cd 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -1,6 +1,6 @@ use clap::Subcommand; use conduwuit::Result; -use ruma::{events::room::message::RoomMessageEventContent, ServerName}; +use ruma::ServerName; use crate::Command; @@ -21,10 +21,7 @@ pub(crate) enum GlobalsCommand { } /// All the getters and iterators from src/database/key_value/globals.rs -pub(super) async fn process( - subcommand: GlobalsCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) -> Result { let services = context.services; match subcommand { @@ -33,36 +30,29 @@ pub(super) async fn process( let results = services.globals.db.database_version().await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, | GlobalsCommand::CurrentCount => { let timer = tokio::time::Instant::now(); let results = services.globals.db.current_count(); let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, | GlobalsCommand::LastCheckForUpdatesId => { let timer = tokio::time::Instant::now(); let results = services.updates.last_check_for_updates_id().await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, | GlobalsCommand::SigningKeysFor { origin } => { let timer = tokio::time::Instant::now(); let results = services.server_keys.verify_keys_for(&origin).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, } + .await } diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index ab269a40..da27eb1d 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -3,10 +3,13 @@ mod appservice; mod globals; mod presence; mod pusher; +mod raw; mod resolver; mod room_alias; mod room_state_cache; +mod room_timeline; mod sending; +mod short; mod users; use clap::Subcommand; @@ -14,9 +17,10 @@ use conduwuit::Result; use self::{ account_data::AccountDataCommand, appservice::AppserviceCommand, globals::GlobalsCommand, - presence::PresenceCommand, pusher::PusherCommand, resolver::ResolverCommand, + presence::PresenceCommand, pusher::PusherCommand, raw::RawCommand, resolver::ResolverCommand, room_alias::RoomAliasCommand, room_state_cache::RoomStateCacheCommand, - sending::SendingCommand, users::UsersCommand, + room_timeline::RoomTimelineCommand, sending::SendingCommand, short::ShortCommand, + users::UsersCommand, }; use crate::admin_command_dispatch; @@ -44,6 +48,10 @@ pub(super) enum QueryCommand { #[command(subcommand)] RoomStateCache(RoomStateCacheCommand), + /// - rooms/timeline iterators and getters + #[command(subcommand)] + RoomTimeline(RoomTimelineCommand), + /// - globals.rs iterators and getters #[command(subcommand)] Globals(GlobalsCommand), @@ -63,4 +71,12 @@ pub(super) enum QueryCommand { /// - pusher service #[command(subcommand)] Pusher(PusherCommand), + + /// - short service + #[command(subcommand)] + Short(ShortCommand), + + /// - raw service + #[command(subcommand)] + Raw(RawCommand), } diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index 0de6b696..38272749 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, UserId}; +use ruma::UserId; use crate::Command; @@ -23,10 +23,7 @@ pub(crate) enum PresenceCommand { } /// All the getters and iterators in key_value/presence.rs -pub(super) async fn process( - subcommand: PresenceCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) -> Result { let services = context.services; match subcommand { @@ -35,9 +32,7 @@ pub(super) async fn process( let results = services.presence.get_presence(&user_id).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, | PresenceCommand::PresenceSince { since } => { let timer = tokio::time::Instant::now(); @@ -49,9 +44,8 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, } + .await } diff --git a/src/admin/query/pusher.rs b/src/admin/query/pusher.rs index 55532e54..34edf4db 100644 --- a/src/admin/query/pusher.rs +++ b/src/admin/query/pusher.rs @@ -1,6 +1,6 @@ use clap::Subcommand; use conduwuit::Result; -use ruma::{events::room::message::RoomMessageEventContent, UserId}; +use ruma::UserId; use crate::Command; @@ -13,10 +13,7 @@ pub(crate) enum PusherCommand { }, } -pub(super) async fn process( - subcommand: PusherCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(subcommand: PusherCommand, context: &Command<'_>) -> Result { let services = context.services; match subcommand { @@ -25,9 +22,8 @@ pub(super) async fn process( let results = services.pusher.get_pushers(&user_id).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, } + .await } diff --git a/src/admin/query/raw.rs b/src/admin/query/raw.rs new file mode 100644 index 00000000..678d21c9 --- /dev/null +++ b/src/admin/query/raw.rs @@ -0,0 +1,457 @@ +use std::{borrow::Cow, collections::BTreeMap, ops::Deref}; + +use clap::Subcommand; +use conduwuit::{ + apply, at, + utils::{ + stream::{ReadyExt, TryIgnore}, + string::EMPTY, + IterStream, + }, + Result, +}; +use futures::{FutureExt, StreamExt, TryStreamExt}; +use ruma::events::room::message::RoomMessageEventContent; +use tokio::time::Instant; + +use crate::{admin_command, admin_command_dispatch}; + +#[admin_command_dispatch] +#[derive(Debug, Subcommand)] +#[allow(clippy::enum_variant_names)] +/// Query tables from database +pub(crate) enum RawCommand { + /// - List database maps + RawMaps, + + /// - Raw database query + RawGet { + /// Map name + map: String, + + /// Key + key: String, + }, + + /// - Raw database keys iteration + RawKeys { + /// Map name + map: String, + + /// Key prefix + prefix: Option, + }, + + /// - Raw database key size breakdown + RawKeysSizes { + /// Map name + map: Option, + + /// Key prefix + prefix: Option, + }, + + /// - Raw database keys total bytes + RawKeysTotal { + /// Map name + map: Option, + + /// Key prefix + prefix: Option, + }, + + /// - Raw database values size breakdown + RawValsSizes { + /// Map name + map: Option, + + /// Key prefix + prefix: Option, + }, + + /// - Raw database values total bytes + RawValsTotal { + /// Map name + map: Option, + + /// Key prefix + prefix: Option, + }, + + /// - Raw database items iteration + RawIter { + /// Map name + map: String, + + /// Key prefix + prefix: Option, + }, + + /// - Raw database keys iteration + RawKeysFrom { + /// Map name + map: String, + + /// Lower-bound + start: String, + + /// Limit + #[arg(short, long)] + limit: Option, + }, + + /// - Raw database items iteration + RawIterFrom { + /// Map name + map: String, + + /// Lower-bound + start: String, + + /// Limit + #[arg(short, long)] + limit: Option, + }, + + /// - Raw database record count + RawCount { + /// Map name + map: Option, + + /// Key prefix + prefix: Option, + }, +} + +#[admin_command] +pub(super) async fn raw_count( + &self, + map: Option, + prefix: Option, +) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); + + let default_all_maps = map + .is_none() + .then(|| self.services.db.keys().map(Deref::deref)) + .into_iter() + .flatten(); + + let maps: Vec<_> = map + .iter() + .map(String::as_str) + .chain(default_all_maps) + .map(|map| self.services.db.get(map)) + .filter_map(Result::ok) + .cloned() + .collect(); + + let timer = Instant::now(); + let count = maps + .iter() + .stream() + .then(|map| map.raw_count_prefix(&prefix)) + .ready_fold(0_usize, usize::saturating_add) + .await; + + let query_time = timer.elapsed(); + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```")) + .await?; + + Ok(RoomMessageEventContent::text_plain("")) +} + +#[admin_command] +pub(super) async fn raw_keys( + &self, + map: String, + prefix: Option, +) -> Result { + writeln!(self, "```").boxed().await?; + + let map = self.services.db.get(map.as_str())?; + let timer = Instant::now(); + prefix + .as_deref() + .map_or_else(|| map.raw_keys().boxed(), |prefix| map.raw_keys_prefix(prefix).boxed()) + .map_ok(String::from_utf8_lossy) + .try_for_each(|str| writeln!(self, "{str:?}")) + .boxed() + .await?; + + let query_time = timer.elapsed(); + let out = format!("\n```\n\nQuery completed in {query_time:?}"); + self.write_str(out.as_str()).await?; + + Ok(RoomMessageEventContent::text_plain("")) +} + +#[admin_command] +pub(super) async fn raw_keys_sizes( + &self, + map: Option, + prefix: Option, +) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); + + let default_all_maps = map + .is_none() + .then(|| self.services.db.keys().map(Deref::deref)) + .into_iter() + .flatten(); + + let maps: Vec<_> = map + .iter() + .map(String::as_str) + .chain(default_all_maps) + .map(|map| self.services.db.get(map)) + .filter_map(Result::ok) + .cloned() + .collect(); + + let timer = Instant::now(); + let result = maps + .iter() + .stream() + .map(|map| map.raw_keys_prefix(&prefix)) + .flatten() + .ignore_err() + .map(<[u8]>::len) + .ready_fold_default(|mut map: BTreeMap<_, usize>, len| { + let entry = map.entry(len).or_default(); + *entry = entry.saturating_add(1); + map + }) + .await; + + let query_time = timer.elapsed(); + let result = format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"); + self.write_str(result.as_str()).await?; + + Ok(RoomMessageEventContent::text_plain("")) +} + +#[admin_command] +pub(super) async fn raw_keys_total( + &self, + map: Option, + prefix: Option, +) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); + + let default_all_maps = map + .is_none() + .then(|| self.services.db.keys().map(Deref::deref)) + .into_iter() + .flatten(); + + let maps: Vec<_> = map + .iter() + .map(String::as_str) + .chain(default_all_maps) + .map(|map| self.services.db.get(map)) + .filter_map(Result::ok) + .cloned() + .collect(); + + let timer = Instant::now(); + let result = maps + .iter() + .stream() + .map(|map| map.raw_keys_prefix(&prefix)) + .flatten() + .ignore_err() + .map(<[u8]>::len) + .ready_fold_default(|acc: usize, len| acc.saturating_add(len)) + .await; + + let query_time = timer.elapsed(); + + self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}")) + .await?; + + Ok(RoomMessageEventContent::text_plain("")) +} + +#[admin_command] +pub(super) async fn raw_vals_sizes( + &self, + map: Option, + prefix: Option, +) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); + + let default_all_maps = map + .is_none() + .then(|| self.services.db.keys().map(Deref::deref)) + .into_iter() + .flatten(); + + let maps: Vec<_> = map + .iter() + .map(String::as_str) + .chain(default_all_maps) + .map(|map| self.services.db.get(map)) + .filter_map(Result::ok) + .cloned() + .collect(); + + let timer = Instant::now(); + let result = maps + .iter() + .stream() + .map(|map| map.raw_stream_prefix(&prefix)) + .flatten() + .ignore_err() + .map(at!(1)) + .map(<[u8]>::len) + .ready_fold_default(|mut map: BTreeMap<_, usize>, len| { + let entry = map.entry(len).or_default(); + *entry = entry.saturating_add(1); + map + }) + .await; + + let query_time = timer.elapsed(); + let result = format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"); + self.write_str(result.as_str()).await?; + + Ok(RoomMessageEventContent::text_plain("")) +} + +#[admin_command] +pub(super) async fn raw_vals_total( + &self, + map: Option, + prefix: Option, +) -> Result { + let prefix = prefix.as_deref().unwrap_or(EMPTY); + + let default_all_maps = map + .is_none() + .then(|| self.services.db.keys().map(Deref::deref)) + .into_iter() + .flatten(); + + let maps: Vec<_> = map + .iter() + .map(String::as_str) + .chain(default_all_maps) + .map(|map| self.services.db.get(map)) + .filter_map(Result::ok) + .cloned() + .collect(); + + let timer = Instant::now(); + let result = maps + .iter() + .stream() + .map(|map| map.raw_stream_prefix(&prefix)) + .flatten() + .ignore_err() + .map(at!(1)) + .map(<[u8]>::len) + .ready_fold_default(|acc: usize, len| acc.saturating_add(len)) + .await; + + let query_time = timer.elapsed(); + + self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}")) + .await?; + + Ok(RoomMessageEventContent::text_plain("")) +} + +#[admin_command] +pub(super) async fn raw_iter( + &self, + map: String, + prefix: Option, +) -> Result { + writeln!(self, "```").await?; + + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + prefix + .as_deref() + .map_or_else(|| map.raw_stream().boxed(), |prefix| map.raw_stream_prefix(prefix).boxed()) + .map_ok(apply!(2, String::from_utf8_lossy)) + .map_ok(apply!(2, Cow::into_owned)) + .try_for_each(|keyval| writeln!(self, "{keyval:?}")) + .boxed() + .await?; + + let query_time = timer.elapsed(); + self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) + .await?; + + Ok(RoomMessageEventContent::text_plain("")) +} + +#[admin_command] +pub(super) async fn raw_keys_from( + &self, + map: String, + start: String, + limit: Option, +) -> Result { + writeln!(self, "```").await?; + + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + map.raw_keys_from(&start) + .map_ok(String::from_utf8_lossy) + .take(limit.unwrap_or(usize::MAX)) + .try_for_each(|str| writeln!(self, "{str:?}")) + .boxed() + .await?; + + let query_time = timer.elapsed(); + self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) + .await?; + + Ok(RoomMessageEventContent::text_plain("")) +} + +#[admin_command] +pub(super) async fn raw_iter_from( + &self, + map: String, + start: String, + limit: Option, +) -> Result { + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + let result = map + .raw_stream_from(&start) + .map_ok(apply!(2, String::from_utf8_lossy)) + .map_ok(apply!(2, Cow::into_owned)) + .take(limit.unwrap_or(usize::MAX)) + .try_collect::>() + .await?; + + let query_time = timer.elapsed(); + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) +} + +#[admin_command] +pub(super) async fn raw_get(&self, map: String, key: String) -> Result { + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + let handle = map.get(&key).await?; + let query_time = timer.elapsed(); + let result = String::from_utf8_lossy(&handle); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```" + ))) +} + +#[admin_command] +pub(super) async fn raw_maps(&self) -> Result { + let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect(); + + Ok(RoomMessageEventContent::notice_markdown(format!("{list:#?}"))) +} diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index 3b950d13..b53661fc 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -28,56 +28,66 @@ async fn destinations_cache( ) -> Result { use service::resolver::cache::CachedDest; + writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?; + writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?; + let mut out = String::new(); - writeln!(out, "| Server Name | Destination | Hostname | Expires |")?; - writeln!(out, "| ----------- | ----------- | -------- | ------- |")?; - let row = |(name, &CachedDest { ref dest, ref host, expire })| { - let expire = time::format(expire, "%+"); - writeln!(out, "| {name} | {dest} | {host} | {expire} |").expect("wrote line"); - }; + { + let map = self + .services + .resolver + .cache + .destinations + .read() + .expect("locked"); - let map = self - .services - .resolver - .cache - .destinations - .read() - .expect("locked"); + for (name, &CachedDest { ref dest, ref host, expire }) in map.iter() { + if let Some(server_name) = server_name.as_ref() { + if name != server_name { + continue; + } + } - if let Some(server_name) = server_name.as_ref() { - map.get_key_value(server_name).map(row); - } else { - map.iter().for_each(row); + let expire = time::format(expire, "%+"); + writeln!(out, "| {name} | {dest} | {host} | {expire} |")?; + } } - Ok(RoomMessageEventContent::notice_markdown(out)) + self.write_str(out.as_str()).await?; + + Ok(RoomMessageEventContent::notice_plain("")) } #[admin_command] async fn overrides_cache(&self, server_name: Option) -> Result { use service::resolver::cache::CachedOverride; + writeln!(self, "| Server Name | IP | Port | Expires |").await?; + writeln!(self, "| ----------- | --- | ----:| ------- |").await?; + let mut out = String::new(); - writeln!(out, "| Server Name | IP | Port | Expires |")?; - writeln!(out, "| ----------- | --- | ----:| ------- |")?; - let row = |(name, &CachedOverride { ref ips, port, expire })| { - let expire = time::format(expire, "%+"); - writeln!(out, "| {name} | {ips:?} | {port} | {expire} |").expect("wrote line"); - }; + { + let map = self + .services + .resolver + .cache + .overrides + .read() + .expect("locked"); - let map = self - .services - .resolver - .cache - .overrides - .read() - .expect("locked"); + for (name, &CachedOverride { ref ips, port, expire }) in map.iter() { + if let Some(server_name) = server_name.as_ref() { + if name != server_name { + continue; + } + } - if let Some(server_name) = server_name.as_ref() { - map.get_key_value(server_name).map(row); - } else { - map.iter().for_each(row); + let expire = time::format(expire, "%+"); + writeln!(out, "| {name} | {ips:?} | {port} | {expire} |")?; + } } - Ok(RoomMessageEventContent::notice_markdown(out)) + self.write_str(out.as_str()).await?; + + Ok(RoomMessageEventContent::notice_plain("")) } diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index e1bf1622..2d4d8104 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, RoomAliasId, RoomId}; +use ruma::{RoomAliasId, RoomId}; use crate::Command; @@ -24,10 +24,7 @@ pub(crate) enum RoomAliasCommand { } /// All the getters and iterators in src/database/key_value/rooms/alias.rs -pub(super) async fn process( - subcommand: RoomAliasCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(subcommand: RoomAliasCommand, context: &Command<'_>) -> Result { let services = context.services; match subcommand { @@ -36,9 +33,7 @@ pub(super) async fn process( let results = services.rooms.alias.resolve_local_alias(&alias).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, | RoomAliasCommand::LocalAliasesForRoom { room_id } => { let timer = tokio::time::Instant::now(); @@ -51,9 +46,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```") }, | RoomAliasCommand::AllLocalAliases => { let timer = tokio::time::Instant::now(); @@ -66,9 +59,8 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```" - ))) + write!(context, "Query completed in {query_time:?}:\n\n```rs\n{aliases:#?}\n```") }, } + .await } diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index cd7f5af7..71dadc99 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -1,5 +1,5 @@ use clap::Subcommand; -use conduwuit::Result; +use conduwuit::{Error, Result}; use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId}; @@ -76,13 +76,10 @@ pub(crate) enum RoomStateCacheCommand { }, } -pub(super) async fn process( - subcommand: RoomStateCacheCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command<'_>) -> Result { let services = context.services; - match subcommand { + let c = match subcommand { | RoomStateCacheCommand::ServerInRoom { server, room_id } => { let timer = tokio::time::Instant::now(); let result = services @@ -92,7 +89,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" ))) }, @@ -107,7 +104,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -122,7 +119,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -137,7 +134,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -152,7 +149,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -167,7 +164,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -176,7 +173,7 @@ pub(super) async fn process( let results = services.rooms.state_cache.room_joined_count(&room_id).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -189,7 +186,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -204,7 +201,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -219,7 +216,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -232,7 +229,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -245,7 +242,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -260,7 +257,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -274,7 +271,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -288,7 +285,7 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, @@ -301,9 +298,13 @@ pub(super) async fn process( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( + Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" ))) }, - } + }?; + + context.write_str(c.body()).await?; + + Ok(()) } diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 3edbbe87..8c6fb25f 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -62,7 +62,14 @@ pub(crate) enum SendingCommand { } /// All the getters and iterators in key_value/sending.rs -pub(super) async fn process( +pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -> Result { + let c = reprocess(subcommand, context).await?; + context.write_str(c.body()).await?; + Ok(()) +} + +/// All the getters and iterators in key_value/sending.rs +pub(super) async fn reprocess( subcommand: SendingCommand, context: &Command<'_>, ) -> Result { diff --git a/src/admin/query/short.rs b/src/admin/query/short.rs new file mode 100644 index 00000000..7f0f3449 --- /dev/null +++ b/src/admin/query/short.rs @@ -0,0 +1,45 @@ +use clap::Subcommand; +use conduwuit::Result; +use ruma::{events::room::message::RoomMessageEventContent, OwnedEventId, OwnedRoomOrAliasId}; + +use crate::{admin_command, admin_command_dispatch}; + +#[admin_command_dispatch] +#[derive(Debug, Subcommand)] +/// Query tables from database +pub(crate) enum ShortCommand { + ShortEventId { + event_id: OwnedEventId, + }, + + ShortRoomId { + room_id: OwnedRoomOrAliasId, + }, +} + +#[admin_command] +pub(super) async fn short_event_id( + &self, + event_id: OwnedEventId, +) -> Result { + let shortid = self + .services + .rooms + .short + .get_shorteventid(&event_id) + .await?; + + Ok(RoomMessageEventContent::notice_markdown(format!("{shortid:#?}"))) +} + +#[admin_command] +pub(super) async fn short_room_id( + &self, + room_id: OwnedRoomOrAliasId, +) -> Result { + let room_id = self.services.rooms.alias.resolve(&room_id).await?; + + let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?; + + Ok(RoomMessageEventContent::notice_markdown(format!("{shortid:#?}"))) +} diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index 2149a103..3715ac25 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -15,6 +15,8 @@ pub(crate) enum UsersCommand { IterUsers, + IterUsers2, + PasswordHash { user_id: OwnedUserId, }, @@ -89,6 +91,33 @@ pub(crate) enum UsersCommand { room_id: OwnedRoomId, session_id: String, }, + + GetSharedRooms { + user_a: OwnedUserId, + user_b: OwnedUserId, + }, +} + +#[admin_command] +async fn get_shared_rooms( + &self, + user_a: OwnedUserId, + user_b: OwnedUserId, +) -> Result { + let timer = tokio::time::Instant::now(); + let result: Vec<_> = self + .services + .rooms + .state_cache + .get_shared_rooms(&user_a, &user_b) + .map(ToOwned::to_owned) + .collect() + .await; + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + ))) } #[admin_command] @@ -207,6 +236,23 @@ async fn iter_users(&self) -> Result { ))) } +#[admin_command] +async fn iter_users2(&self) -> Result { + let timer = tokio::time::Instant::now(); + let result: Vec<_> = self.services.users.stream().collect().await; + let result: Vec<_> = result + .into_iter() + .map(ruma::UserId::as_bytes) + .map(String::from_utf8_lossy) + .collect(); + + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```" + ))) +} + #[admin_command] async fn count_users(&self) -> Result { let timer = tokio::time::Instant::now(); diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 4490600d..9710cfc8 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -44,7 +44,14 @@ pub(crate) enum RoomAliasCommand { }, } -pub(super) async fn process( +pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> Result { + let c = reprocess(command, context).await?; + context.write_str(c.body()).await?; + + Ok(()) +} + +pub(super) async fn reprocess( command: RoomAliasCommand, context: &Command<'_>, ) -> Result { diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index 81f25478..791b9204 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -25,7 +25,13 @@ pub(crate) enum RoomDirectoryCommand { }, } -pub(super) async fn process( +pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_>) -> Result { + let c = reprocess(command, context).await?; + context.write_str(c.body()).await?; + Ok(()) +} + +pub(super) async fn reprocess( command: RoomDirectoryCommand, context: &Command<'_>, ) -> Result { diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 5758d937..57aedd9c 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -31,19 +31,21 @@ const BULK_JOIN_REASON: &str = "Bulk force joining this room as initiated by the #[admin_command] pub(super) async fn list_users(&self) -> Result { - let users = self + let users: Vec<_> = self .services .users .list_local_users() .map(ToString::to_string) - .collect::>() + .collect() .await; let mut plain_msg = format!("Found {} local user account(s):\n```\n", users.len()); plain_msg += users.join("\n").as_str(); plain_msg += "\n```"; - Ok(RoomMessageEventContent::notice_markdown(plain_msg)) + self.write_str(plain_msg.as_str()).await?; + + Ok(RoomMessageEventContent::text_plain("")) } #[admin_command] @@ -912,29 +914,30 @@ pub(super) async fn redact_event( self.services.globals.server_name() ); - let state_lock = self.services.rooms.state.mutex.lock(&room_id).await; + let redaction_event_id = { + let state_lock = self.services.rooms.state.mutex.lock(&room_id).await; - let redaction_event_id = self - .services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder { - redacts: Some(event.event_id.clone()), - ..PduBuilder::timeline(&RoomRedactionEventContent { + self.services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder { redacts: Some(event.event_id.clone()), - reason: Some(reason), - }) - }, - &sender_user, - &room_id, - &state_lock, - ) - .await?; + ..PduBuilder::timeline(&RoomRedactionEventContent { + redacts: Some(event.event_id.clone()), + reason: Some(reason), + }) + }, + &sender_user, + &room_id, + &state_lock, + ) + .await? + }; - drop(state_lock); + let out = format!("Successfully redacted event. Redaction event ID: {redaction_event_id}"); - Ok(RoomMessageEventContent::text_plain(format!( - "Successfully redacted event. Redaction event ID: {redaction_event_id}" - ))) + self.write_str(out.as_str()).await?; + + Ok(RoomMessageEventContent::text_plain("")) } diff --git a/src/macros/admin.rs b/src/macros/admin.rs index e98e914c..e35bd586 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -22,7 +22,7 @@ pub(super) fn command_dispatch(item: ItemEnum, _args: &[Meta]) -> Result - ) -> Result { + ) -> Result { use #name::*; #[allow(non_snake_case)] Ok(match command { @@ -46,7 +46,10 @@ fn dispatch_arm(v: &Variant) -> Result { let field = fields.named.iter().filter_map(|f| f.ident.as_ref()); let arg = field.clone(); quote! { - #name { #( #field ),* } => Box::pin(context.#handler(#( #arg ),*)).await?, + #name { #( #field ),* } => { + let c = Box::pin(context.#handler(#( #arg ),*)).await?; + Box::pin(context.write_str(c.body())).await?; + }, } }, | Fields::Unnamed(fields) => { @@ -54,12 +57,17 @@ fn dispatch_arm(v: &Variant) -> Result { return Err(Error::new(Span::call_site().into(), "One unnamed field required")); }; quote! { - #name ( #field ) => Box::pin(#handler::process(#field, context)).await?, + #name ( #field ) => { + Box::pin(#handler::process(#field, context)).await?; + } } }, | Fields::Unit => { quote! { - #name => Box::pin(context.#handler()).await?, + #name => { + let c = Box::pin(context.#handler()).await?; + Box::pin(context.write_str(c.body())).await?; + }, } }, }; From f9e76d6239632bd3e74cd0b1c76dd72dbc24dc7a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 01:32:37 +0000 Subject: [PATCH 1370/2091] improve debug memory-stats options Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 30 +++++++++++++++++++----------- src/admin/debug/mod.rs | 8 +++++++- src/core/alloc/default.rs | 2 +- src/core/alloc/hardened.rs | 4 ++-- src/core/alloc/je.rs | 14 ++++++-------- 5 files changed, 35 insertions(+), 23 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index b6189f6a..ad61440c 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -843,19 +843,27 @@ pub(super) async fn resolve_true_destination( } #[admin_command] -pub(super) async fn memory_stats(&self) -> Result { - let html_body = conduwuit::alloc::memory_stats(); +pub(super) async fn memory_stats(&self, opts: Option) -> Result { + const OPTS: &str = "abcdefghijklmnopqrstuvwxyz"; - if html_body.is_none() { - return Ok(RoomMessageEventContent::text_plain( - "malloc stats are not supported on your compiled malloc.", - )); - } + let opts: String = OPTS + .chars() + .filter(|&c| { + let allow_any = opts.as_ref().is_some_and(|opts| opts == "*"); - Ok(RoomMessageEventContent::text_html( - "This command's output can only be viewed by clients that render HTML.".to_owned(), - html_body.expect("string result"), - )) + let allow = allow_any || opts.as_ref().is_some_and(|opts| opts.contains(c)); + + !allow + }) + .collect(); + + let stats = conduwuit::alloc::memory_stats(&opts).unwrap_or_default(); + + self.write_str("```\n").await?; + self.write_str(&stats).await?; + self.write_str("\n```").await?; + + Ok(RoomMessageEventContent::text_plain("")) } #[cfg(tokio_unstable)] diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 2367f80d..07f7296b 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -191,7 +191,13 @@ pub(super) enum DebugCommand { }, /// - Print extended memory usage - MemoryStats, + /// + /// Optional argument is a character mask (a sequence of characters in any + /// order) which enable additional extended statistics. Known characters are + /// "abdeglmx". For convenience, a '*' will enable everything. + MemoryStats { + opts: Option, + }, /// - Print general tokio runtime metric totals. RuntimeMetrics, diff --git a/src/core/alloc/default.rs b/src/core/alloc/default.rs index 5db02884..56e8c407 100644 --- a/src/core/alloc/default.rs +++ b/src/core/alloc/default.rs @@ -5,7 +5,7 @@ pub fn trim() -> crate::Result { Ok(()) } /// Always returns None #[must_use] -pub fn memory_stats() -> Option { None } +pub fn memory_stats(_opts: &str) -> Option { None } /// Always returns None #[must_use] diff --git a/src/core/alloc/hardened.rs b/src/core/alloc/hardened.rs index e2d9b28e..ff10cf2b 100644 --- a/src/core/alloc/hardened.rs +++ b/src/core/alloc/hardened.rs @@ -7,9 +7,9 @@ pub fn trim() -> crate::Result { Ok(()) } #[must_use] //TODO: get usage -pub fn memory_usage() -> Option { None } +pub fn memory_usage() -> Option { None } #[must_use] -pub fn memory_stats() -> Option { +pub fn memory_stats(_opts: &str) -> Option { Some("Extended statistics are not available from hardened_malloc.".to_owned()) } diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index b2c1fe85..ccb213c9 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -3,7 +3,7 @@ use std::{ cell::OnceCell, ffi::{c_char, c_void}, - fmt::{Debug, Write}, + fmt::Debug, }; use arrayvec::ArrayVec; @@ -66,15 +66,12 @@ pub fn memory_usage() -> Option { #[cfg(not(feature = "jemalloc_stats"))] pub fn memory_usage() -> Option { None } -#[must_use] -pub fn memory_stats() -> Option { - const MAX_LENGTH: usize = 65536 - 4096; +pub fn memory_stats(opts: &str) -> Option { + const MAX_LENGTH: usize = 1_048_576; - let opts_s = "d"; let mut str = String::new(); - let opaque = std::ptr::from_mut(&mut str).cast::(); - let opts_p: *const c_char = std::ffi::CString::new(opts_s) + let opts_p: *const c_char = std::ffi::CString::new(opts) .expect("cstring") .into_raw() .cast_const(); @@ -84,7 +81,8 @@ pub fn memory_stats() -> Option { unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts_p) }; str.truncate(MAX_LENGTH); - Some(format!("

    {str}
    ")) + + Some(str) } unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { From afdf5a07b52bd7102d5b98c4d3b6aa1b1fc905ce Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 05:32:17 +0000 Subject: [PATCH 1371/2091] abstract hidden line related in config generator macro --- src/macros/config.rs | 70 +++++++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/src/macros/config.rs b/src/macros/config.rs index 0fb79728..452abd20 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -15,6 +15,8 @@ use crate::{ const UNDOCUMENTED: &str = "# This item is undocumented. Please contribute documentation for it."; +const HIDDEN: &[&str] = &["default"]; + #[allow(clippy::needless_pass_by_value)] pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result { if is_cargo_build() && !is_cargo_test() { @@ -93,7 +95,7 @@ fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { format!("{doc}\n#\n") }; - let default = get_doc_default(field) + let default = get_doc_comment_line(field, "default") .or_else(|| get_default(field)) .unwrap_or_default(); @@ -163,40 +165,40 @@ fn get_default(field: &Field) -> Option { None } -fn get_doc_default(field: &Field) -> Option { - for attr in &field.attrs { - let Meta::NameValue(MetaNameValue { path, value, .. }) = &attr.meta else { - continue; - }; +fn get_doc_comment(field: &Field) -> Option { + let comment = get_doc_comment_full(field)?; - if path.segments.iter().next().is_none_or(|s| s.ident != "doc") { - continue; - } + let out = comment + .lines() + .filter(|line| { + !HIDDEN.iter().any(|key| { + line.trim().starts_with(key) && line.trim().chars().nth(key.len()) == Some(':') + }) + }) + .fold(String::new(), |full, line| full + "#" + line + "\n"); - let Expr::Lit(ExprLit { lit, .. }) = &value else { - continue; - }; - - let Lit::Str(token) = &lit else { - continue; - }; - - let value = token.value(); - if !value.trim().starts_with("default:") { - continue; - } - - return value - .split_once(':') - .map(|(_, v)| v) - .map(str::trim) - .map(ToOwned::to_owned); - } - - None + (!out.is_empty()).then_some(out) } -fn get_doc_comment(field: &Field) -> Option { +fn get_doc_comment_line(field: &Field, label: &str) -> Option { + let comment = get_doc_comment_full(field)?; + + comment + .lines() + .map(str::trim) + .filter(|line| line.starts_with(label)) + .filter(|line| line.chars().nth(label.len()) == Some(':')) + .map(|line| { + line.split_once(':') + .map(|(_, v)| v) + .map(str::trim) + .map(ToOwned::to_owned) + }) + .next() + .flatten() +} + +fn get_doc_comment_full(field: &Field) -> Option { let mut out = String::new(); for attr in &field.attrs { let Meta::NameValue(MetaNameValue { path, value, .. }) = &attr.meta else { @@ -216,11 +218,7 @@ fn get_doc_comment(field: &Field) -> Option { }; let value = token.value(); - if value.trim().starts_with("default:") { - continue; - } - - writeln!(&mut out, "#{value}").expect("wrote to output string buffer"); + writeln!(&mut out, "{value}").expect("wrote to output string buffer"); } (!out.is_empty()).then_some(out) From c6ae6adc80e562a44f96e10f03eb4d14dc312984 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 09:47:17 +0000 Subject: [PATCH 1372/2091] pre-allocate some amount of media read buffer Signed-off-by: Jason Volk --- src/service/media/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 7e77090c..0d98853d 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -157,7 +157,7 @@ impl Service { if let Ok(Metadata { content_disposition, content_type, key }) = self.db.search_file_metadata(mxc, &Dim::default()).await { - let mut content = Vec::new(); + let mut content = Vec::with_capacity(8192); let path = self.get_media_file(&key); BufReader::new(fs::File::open(path).await?) .read_to_end(&mut content) From 7045481fae69150eea84983a55b762ecfaa04e2f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 21:58:49 +0000 Subject: [PATCH 1373/2091] add from_errno construction to Error Signed-off-by: Jason Volk --- src/core/error/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 2468811e..88ac6d09 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -131,6 +131,10 @@ pub enum Error { } impl Error { + #[inline] + #[must_use] + pub fn from_errno() -> Self { Self::Io(std::io::Error::last_os_error()) } + //#[deprecated] pub fn bad_database(message: &'static str) -> Self { crate::err!(Database(error!("{message}"))) From df3eb95d4f18f61839b296f48401ca75f61ad750 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 4 Jan 2025 01:32:45 +0000 Subject: [PATCH 1374/2091] additional affinity utils Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 2 +- src/core/utils/sys.rs | 2 +- src/core/utils/sys/compute.rs | 145 ++++++++++++++++++++++++++-------- 3 files changed, 116 insertions(+), 33 deletions(-) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 2bbadb50..4b5330ed 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -37,7 +37,7 @@ pub use self::{ rand::{shuffle, string as random_string}, stream::{IterStream, ReadyExt, Tools as StreamTools, TryReadyExt}, string::{str_from_bytes, string_from_bytes}, - sys::compute::parallelism as available_parallelism, + sys::compute::available_parallelism, time::{ exponential_backoff::{continue_exponential_backoff, continue_exponential_backoff_secs}, now_millis as millis_since_unix_epoch, timepoint_ago, timepoint_from_now, diff --git a/src/core/utils/sys.rs b/src/core/utils/sys.rs index 5c5564c4..a0d5be52 100644 --- a/src/core/utils/sys.rs +++ b/src/core/utils/sys.rs @@ -3,7 +3,7 @@ pub mod storage; use std::path::PathBuf; -pub use compute::parallelism as available_parallelism; +pub use compute::available_parallelism; use crate::{debug, Result}; diff --git a/src/core/utils/sys/compute.rs b/src/core/utils/sys/compute.rs index 9e90fc90..ce2aa504 100644 --- a/src/core/utils/sys/compute.rs +++ b/src/core/utils/sys/compute.rs @@ -1,23 +1,31 @@ //! System utilities related to compute/processing -use std::{cell::Cell, fmt::Debug, sync::LazyLock}; +use std::{cell::Cell, fmt::Debug, path::PathBuf, sync::LazyLock}; -use crate::is_equal_to; +use crate::{is_equal_to, Result}; -/// The list of cores available to the process (at startup) -static CORES_AVAILABLE: LazyLock = LazyLock::new(|| { - core_affinity::get_core_ids() - .unwrap_or_default() - .into_iter() - .map(|core_id| core_id.id) - .inspect(|&id| debug_assert!(id < 128, "Core ID must be < 128 at least for now")) - .fold(0_u128, |mask, id| mask | (1 << id)) -}); +type Id = usize; + +type Mask = u128; +type Masks = [Mask; MASK_BITS]; + +const MASK_BITS: usize = 128; + +/// The mask of logical cores available to the process (at startup). +static CORES_AVAILABLE: LazyLock = LazyLock::new(|| into_mask(query_cores_available())); + +/// Stores the mask of logical-cores with thread/HT/SMT association. Each group +/// here makes up a physical-core. +static SMT_TOPOLOGY: LazyLock = LazyLock::new(init_smt_topology); + +/// Stores the mask of logical-core associations on a node/socket. Bits are set +/// for all logical cores within all physical cores of the node. +static NODE_TOPOLOGY: LazyLock = LazyLock::new(init_node_topology); thread_local! { /// Tracks the affinity for this thread. This is updated when affinities /// are set via our set_affinity() interface. - static CORE_AFFINITY: Cell = Cell::default(); + static CORE_AFFINITY: Cell = Cell::default(); } /// Set the core affinity for this thread. The ID should be listed in @@ -28,19 +36,19 @@ thread_local! { fields( id = ?std::thread::current().id(), name = %std::thread::current().name().unwrap_or("None"), - set = ?ids.by_ref().collect::>(), + set = ?ids.clone().collect::>(), CURRENT = %format!("[b{:b}]", CORE_AFFINITY.get()), AVAILABLE = %format!("[b{:b}]", *CORES_AVAILABLE), ), )] pub fn set_affinity(mut ids: I) where - I: Iterator + Clone + Debug, + I: Iterator + Clone + Debug, { use core_affinity::{set_each_for_current, set_for_current, CoreId}; let n = ids.clone().count(); - let mask: u128 = ids.clone().fold(0, |mask, id| { + let mask: Mask = ids.clone().fold(0, |mask, id| { debug_assert!(is_core_available(id), "setting affinity to unavailable core"); mask | (1 << id) }); @@ -57,35 +65,110 @@ where } /// Get the core affinity for this thread. -pub fn get_affinity() -> impl Iterator { iter_bits(CORE_AFFINITY.get()) } +pub fn get_affinity() -> impl Iterator { from_mask(CORE_AFFINITY.get()) } + +/// List the cores sharing SMT-tier resources +pub fn smt_siblings() -> impl Iterator { + from_mask(get_affinity().fold(0_u128, |mask, id| { + mask | SMT_TOPOLOGY.get(id).expect("ID must not exceed max cpus") + })) +} + +/// List the cores sharing Node-tier resources relative to this threads current +/// affinity. +pub fn node_siblings() -> impl Iterator { + from_mask(get_affinity().fold(0_u128, |mask, id| { + mask | NODE_TOPOLOGY.get(id).expect("Id must not exceed max cpus") + })) +} + +/// Get the cores sharing SMT resources relative to id. +#[inline] +pub fn smt_affinity(id: Id) -> impl Iterator { + from_mask(*SMT_TOPOLOGY.get(id).expect("ID must not exceed max cpus")) +} + +/// Get the cores sharing Node resources relative to id. +#[inline] +pub fn node_affinity(id: Id) -> impl Iterator { + from_mask(*NODE_TOPOLOGY.get(id).expect("ID must not exceed max cpus")) +} + +/// Get the number of threads which could execute in parallel based on hardware +/// constraints of this system. +#[inline] +#[must_use] +pub fn available_parallelism() -> usize { cores_available().count() } /// Gets the ID of the nth core available. This bijects our sequence of cores to /// actual ID's which may have gaps for cores which are not available. #[inline] #[must_use] -pub fn nth_core_available(i: usize) -> Option { cores_available().nth(i) } +pub fn nth_core_available(i: usize) -> Option { cores_available().nth(i) } /// Determine if core (by id) is available to the process. #[inline] #[must_use] -pub fn is_core_available(id: usize) -> bool { cores_available().any(is_equal_to!(id)) } +pub fn is_core_available(id: Id) -> bool { cores_available().any(is_equal_to!(id)) } /// Get the list of cores available. The values were recorded at program start. #[inline] -pub fn cores_available() -> impl Iterator { iter_bits(*CORES_AVAILABLE) } +pub fn cores_available() -> impl Iterator { from_mask(*CORES_AVAILABLE) } -/// Get the number of threads which could execute in parallel based on the -/// hardware and administrative constraints of this system. This value should be -/// used to hint the size of thread-pools and divide-and-conquer algorithms. -/// -/// * -#[must_use] -pub fn parallelism() -> usize { - std::thread::available_parallelism() - .expect("Unable to query for available parallelism.") - .get() +#[cfg(target_os = "linux")] +#[inline] +pub fn getcpu() -> Result { + use crate::{utils::math, Error}; + + // SAFETY: This is part of an interface with many low-level calls taking many + // raw params, but it's unclear why this specific call is unsafe. Nevertheless + // the value obtained here is semantically unsafe because it can change on the + // instruction boundary trailing its own acquisition and also any other time. + let ret: i32 = unsafe { libc::sched_getcpu() }; + + #[cfg(target_os = "linux")] + // SAFETY: On modern linux systems with a vdso if we can optimize away the branch checking + // for error (see getcpu(2)) then this system call becomes a memory access. + unsafe { + std::hint::assert_unchecked(ret >= 0); + }; + + if ret == -1 { + return Err(Error::from_errno()); + } + + math::try_into(ret) } -fn iter_bits(v: u128) -> impl Iterator { - (0..128).filter(move |&i| (v & (1 << i)) != 0) +#[cfg(not(target_os = "linux"))] +#[inline] +pub fn getcpu() -> Result { Err(crate::Error::Io(std::io::ErrorKind::Unsupported.into())) } + +fn query_cores_available() -> impl Iterator { + core_affinity::get_core_ids() + .unwrap_or_default() + .into_iter() + .map(|core_id| core_id.id) +} + +fn init_smt_topology() -> [Mask; MASK_BITS] { [Mask::default(); MASK_BITS] } + +fn init_node_topology() -> [Mask; MASK_BITS] { [Mask::default(); MASK_BITS] } + +fn into_mask(ids: I) -> Mask +where + I: Iterator, +{ + ids.inspect(|&id| { + debug_assert!(id < MASK_BITS, "Core ID must be < Mask::BITS at least for now"); + }) + .fold(Mask::default(), |mask, id| mask | (1 << id)) +} + +fn from_mask(v: Mask) -> impl Iterator { + (0..MASK_BITS).filter(move |&i| (v & (1 << i)) != 0) +} + +fn _sys_path(id: usize, suffix: &str) -> PathBuf { + format!("/sys/devices/system/cpu/cpu{id}/{suffix}").into() } From 4fbbfe5d3056669982becae1ae7ec0f3edd80439 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 19:56:16 +0000 Subject: [PATCH 1375/2091] add alt argument format for detecting cargo build phase Signed-off-by: Jason Volk --- src/macros/utils.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/macros/utils.rs b/src/macros/utils.rs index e33ee8b4..af2519a7 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -23,6 +23,16 @@ pub(crate) fn get_simple_settings(args: &[Meta]) -> HashMap { } pub(crate) fn is_cargo_build() -> bool { + legacy_is_cargo_build() + || std::env::args() + .skip_while(|flag| !flag.starts_with("--emit")) + .nth(1) + .iter() + .flat_map(|flag| flag.split(',')) + .any(|elem| elem == "link") +} + +pub(crate) fn legacy_is_cargo_build() -> bool { std::env::args() .find(|flag| flag.starts_with("--emit")) .as_ref() From 3eed408b2975564ed2c0b103a665f1a022e150b8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 20 Jan 2025 04:42:01 +0000 Subject: [PATCH 1376/2091] additional util macros and reorg Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 85 ++++++++++++++++++++++++++----------------- 1 file changed, 52 insertions(+), 33 deletions(-) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 4b5330ed..1a4b52da 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -57,6 +57,14 @@ macro_rules! extract_variant { }; } +/// Functor for !is_empty() +#[macro_export] +macro_rules! is_not_empty { + () => { + |x| !x.is_empty() + }; +} + #[macro_export] macro_rules! apply { (1, $($idx:tt)+) => { @@ -76,24 +84,35 @@ macro_rules! apply { }; } +/// Functor for truthy #[macro_export] -macro_rules! at { - ($idx:tt) => { - |t| t.$idx +macro_rules! is_true { + () => { + |x| !!x }; } +/// Functor for falsy #[macro_export] -macro_rules! ref_at { - ($idx:tt) => { - |ref t| &t.$idx +macro_rules! is_false { + () => { + |x| !x }; } +/// Functor for equality to non-zero #[macro_export] -macro_rules! deref_at { - ($idx:tt) => { - |t| *t.$idx +macro_rules! is_nonzero { + () => { + |x| x != 0 + }; +} + +/// Functor for equality to zero +#[macro_export] +macro_rules! is_zero { + () => { + $crate::is_matching!(0) }; } @@ -121,14 +140,6 @@ macro_rules! is_less_than { }; } -/// Functor for equality to zero -#[macro_export] -macro_rules! is_zero { - () => { - $crate::is_matching!(0) - }; -} - /// Functor for matches! i.e. .is_some_and(is_matching!('A'..='Z')) #[macro_export] macro_rules! is_matching { @@ -141,14 +152,6 @@ macro_rules! is_matching { }; } -/// Functor for !is_empty() -#[macro_export] -macro_rules! is_not_empty { - () => { - |x| !x.is_empty() - }; -} - /// Functor for equality i.e. (a, b).map(is_equal!()) #[macro_export] macro_rules! is_equal { @@ -157,18 +160,34 @@ macro_rules! is_equal { }; } -/// Functor for truthy +/// Functor for |x| *x.$i #[macro_export] -macro_rules! is_true { - () => { - |x| !!x +macro_rules! deref_at { + ($idx:tt) => { + |t| *t.$idx }; } -/// Functor for falsy +/// Functor for |ref x| x.$i #[macro_export] -macro_rules! is_false { - () => { - |x| !x +macro_rules! ref_at { + ($idx:tt) => { + |ref t| &t.$idx + }; +} + +/// Functor for |&x| x.$i +#[macro_export] +macro_rules! val_at { + ($idx:tt) => { + |&t| t.$idx + }; +} + +/// Functor for |x| x.$i +#[macro_export] +macro_rules! at { + ($idx:tt) => { + |t| t.$idx }; } From 3dae02b886a3428c58d0a11e1cc19271722b4b47 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 01:30:41 +0000 Subject: [PATCH 1377/2091] add preferred jemalloc config add muzzy/dirty configuration mallctl interface add program argument for --gc-muzzy=false Signed-off-by: Jason Volk --- Cargo.lock | 57 +++++----- Cargo.toml | 16 ++- deps/rust-rocksdb/Cargo.toml | 2 +- src/admin/debug/commands.rs | 2 +- src/core/Cargo.toml | 1 + src/core/alloc/default.rs | 2 +- src/core/alloc/hardened.rs | 2 +- src/core/alloc/je.rs | 194 +++++++++++++++++++++++++++++------ src/database/pool.rs | 16 ++- src/main/Cargo.toml | 4 + src/main/clap.rs | 16 +++ src/main/runtime.rs | 59 ++++++++--- 12 files changed, 289 insertions(+), 82 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18bd7aab..8de3abf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -175,9 +175,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-lc-rs" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f409eb70b561706bf8abba8ca9c112729c481595893fd06a2dd9af8ed8441148" +checksum = "1ea835662a0af02443aa1396d39be523bbf8f11ee6fad20329607c480bea48c3" dependencies = [ "aws-lc-sys", "paste", @@ -186,9 +186,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.24.1" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "923ded50f602b3007e5e63e3f094c479d9c8a9b42d7f4034e4afe456aa48bfd2" +checksum = "71b2ddd3ada61a305e1d8bb6c005d1eaa7d14d903681edfc400406d523a9b491" dependencies = [ "bindgen", "cc", @@ -368,7 +368,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -393,9 +393,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "blake2" @@ -495,9 +495,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.9" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" +checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" dependencies = [ "jobserver", "libc", @@ -1047,7 +1047,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "crossterm_winapi", "futures-core", "mio", @@ -1122,9 +1122,9 @@ checksum = "817fa642fb0ee7fe42e95783e00e0969927b96091bdd4b9b1af082acd943913b" [[package]] name = "data-encoding" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" [[package]] name = "date_header" @@ -2378,7 +2378,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "cfg-if", "cfg_aliases", "libc", @@ -2911,7 +2911,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "memchr", "pulldown-cmark-escape", "unicase", @@ -3032,7 +3032,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", ] [[package]] @@ -3377,7 +3377,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.31.0+9.9.3" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=123d6302fed23fc706344becb2f19623265a83f8#123d6302fed23fc706344becb2f19623265a83f8" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2d31cf323df7c6d95396ef0213e28936c2218bd6#2d31cf323df7c6d95396ef0213e28936c2218bd6" dependencies = [ "bindgen", "bzip2-sys", @@ -3394,7 +3394,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.35.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=123d6302fed23fc706344becb2f19623265a83f8#123d6302fed23fc706344becb2f19623265a83f8" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2d31cf323df7c6d95396ef0213e28936c2218bd6#2d31cf323df7c6d95396ef0213e28936c2218bd6" dependencies = [ "libc", "rust-librocksdb-sys", @@ -3441,7 +3441,7 @@ version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", @@ -3559,9 +3559,12 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sd-notify" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1be20c5f7f393ee700f8b2f28ea35812e4e212f40774b550cd2a93ea91684451" +checksum = "561e6b346a5e59e0b8a07894004897d7160567e3352d2ebd6c3741d4e086b6f5" +dependencies = [ + "libc", +] [[package]] name = "security-framework" @@ -3569,7 +3572,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.7.0", + "bitflags 2.8.0", "core-foundation", "core-foundation-sys", "libc", @@ -4171,7 +4174,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-ctl" version = "0.6.0" -source = "git+https://github.com/girlbossceo/jemallocator?rev=d87938bfddc26377dd7fdf14bbcd345f3ab19442#d87938bfddc26377dd7fdf14bbcd345f3ab19442" +source = "git+https://github.com/girlbossceo/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" dependencies = [ "libc", "paste", @@ -4181,7 +4184,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" -source = "git+https://github.com/girlbossceo/jemallocator?rev=d87938bfddc26377dd7fdf14bbcd345f3ab19442#d87938bfddc26377dd7fdf14bbcd345f3ab19442" +source = "git+https://github.com/girlbossceo/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" dependencies = [ "cc", "libc", @@ -4190,7 +4193,7 @@ dependencies = [ [[package]] name = "tikv-jemallocator" version = "0.6.0" -source = "git+https://github.com/girlbossceo/jemallocator?rev=d87938bfddc26377dd7fdf14bbcd345f3ab19442#d87938bfddc26377dd7fdf14bbcd345f3ab19442" +source = "git+https://github.com/girlbossceo/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" dependencies = [ "libc", "tikv-jemalloc-sys", @@ -4445,7 +4448,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "async-compression", - "bitflags 2.7.0", + "bitflags 2.8.0", "bytes", "futures-core", "futures-util", @@ -4709,9 +4712,9 @@ dependencies = [ [[package]] name = "valuable" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" [[package]] name = "vcpkg" diff --git a/Cargo.toml b/Cargo.toml index c0b31a69..4d738a11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -431,17 +431,23 @@ version = "0.35.0" # jemalloc usage [workspace.dependencies.tikv-jemalloc-sys] git = "https://github.com/girlbossceo/jemallocator" -rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442" +rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" default-features = false -features = ["unprefixed_malloc_on_supported_platforms"] +features = [ + "background_threads_runtime_support", + "unprefixed_malloc_on_supported_platforms", +] [workspace.dependencies.tikv-jemallocator] git = "https://github.com/girlbossceo/jemallocator" -rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442" +rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" default-features = false -features = ["unprefixed_malloc_on_supported_platforms"] +features = [ + "background_threads_runtime_support", + "unprefixed_malloc_on_supported_platforms", +] [workspace.dependencies.tikv-jemalloc-ctl] git = "https://github.com/girlbossceo/jemallocator" -rev = "d87938bfddc26377dd7fdf14bbcd345f3ab19442" +rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" default-features = false features = ["use_std"] diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index f06c44e8..96554aed 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "123d6302fed23fc706344becb2f19623265a83f8" +rev = "2d31cf323df7c6d95396ef0213e28936c2218bd6" #branch = "master" default-features = false diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index ad61440c..a77587b0 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -967,7 +967,7 @@ pub(super) async fn database_stats( #[admin_command] pub(super) async fn trim_memory(&self) -> Result { - conduwuit::alloc::trim()?; + conduwuit::alloc::trim(None)?; writeln!(self, "done").await?; diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index c716e9c2..ef2df4ff 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -36,6 +36,7 @@ jemalloc_stats = [ "tikv-jemalloc-ctl/stats", "tikv-jemallocator/stats", ] +jemalloc_conf = [] hardened_malloc = [ "dep:hardened_malloc-rs" ] diff --git a/src/core/alloc/default.rs b/src/core/alloc/default.rs index 56e8c407..65354b7d 100644 --- a/src/core/alloc/default.rs +++ b/src/core/alloc/default.rs @@ -1,7 +1,7 @@ //! Default allocator with no special features /// Always returns Ok -pub fn trim() -> crate::Result { Ok(()) } +pub fn trim>>(_: I) -> crate::Result { Ok(()) } /// Always returns None #[must_use] diff --git a/src/core/alloc/hardened.rs b/src/core/alloc/hardened.rs index ff10cf2b..5f850673 100644 --- a/src/core/alloc/hardened.rs +++ b/src/core/alloc/hardened.rs @@ -3,7 +3,7 @@ #[global_allocator] static HMALLOC: hardened_malloc_rs::HardenedMalloc = hardened_malloc_rs::HardenedMalloc; -pub fn trim() -> crate::Result { Ok(()) } +pub fn trim>>(_: I) -> crate::Result { Ok(()) } #[must_use] //TODO: get usage diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index ccb213c9..119ff45e 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -2,8 +2,9 @@ use std::{ cell::OnceCell, - ffi::{c_char, c_void}, + ffi::{c_char, c_void, CStr}, fmt::Debug, + sync::RwLock, }; use arrayvec::ArrayVec; @@ -11,10 +12,14 @@ use tikv_jemalloc_ctl as mallctl; use tikv_jemalloc_sys as ffi; use tikv_jemallocator as jemalloc; -use crate::{err, is_equal_to, utils::math::Tried, Result}; +use crate::{ + err, is_equal_to, is_nonzero, + utils::{math, math::Tried}, + Result, +}; #[cfg(feature = "jemalloc_conf")] -#[no_mangle] +#[unsafe(no_mangle)] pub static malloc_conf: &[u8] = b"\ metadata_thp:always\ ,percpu_arena:percpu\ @@ -22,19 +27,26 @@ metadata_thp:always\ ,max_background_threads:-1\ ,lg_extent_max_active_fit:4\ ,oversize_threshold:33554432\ -,tcache_max:2097152\ +,tcache_max:1048576\ ,dirty_decay_ms:16000\ ,muzzy_decay_ms:144000\ \0"; #[global_allocator] static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc; +static CONTROL: RwLock<()> = RwLock::new(()); -type Key = ArrayVec; type Name = ArrayVec; +type Key = ArrayVec; -const KEY_SEGS: usize = 8; const NAME_MAX: usize = 128; +const KEY_SEGS: usize = 8; + +#[crate::ctor] +fn _static_initialization() { + acq_epoch().expect("pre-initialization of jemalloc failed"); + acq_epoch().expect("pre-initialization of jemalloc failed"); +} #[must_use] #[cfg(feature = "jemalloc_stats")] @@ -49,6 +61,9 @@ pub fn memory_usage() -> Option { kibs / 1024.0 }; + // Acquire the epoch; ensure latest stats are pulled in + acq_epoch().ok()?; + let allocated = mibs(stats::allocated::read()); let active = mibs(stats::active::read()); let mapped = mibs(stats::mapped::read()); @@ -76,6 +91,9 @@ pub fn memory_stats(opts: &str) -> Option { .into_raw() .cast_const(); + // Acquire the epoch; ensure latest stats are pulled in + acq_epoch().ok()?; + // SAFETY: calls malloc_stats_print() with our string instance which must remain // in this frame. https://docs.rs/tikv-jemalloc-sys/latest/tikv_jemalloc_sys/fn.malloc_stats_print.html unsafe { ffi::malloc_stats_print(Some(malloc_stats_cb), opaque, opts_p) }; @@ -95,7 +113,7 @@ unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { }; // SAFETY: we have to trust the string is null terminated. - let msg = unsafe { std::ffi::CStr::from_ptr(msg) }; + let msg = unsafe { CStr::from_ptr(msg) }; let msg = String::from_utf8_lossy(msg.to_bytes()); res.push_str(msg.as_ref()); @@ -114,58 +132,168 @@ macro_rules! mallctl { }}; } -pub fn trim() -> Result { set(&mallctl!("arena.4096.purge"), ()) } - -pub fn decay() -> Result { set(&mallctl!("arena.4096.purge"), ()) } - -pub fn set_by_name(name: &str, val: T) -> Result { set(&key(name)?, val) } - -pub fn get_by_name(name: &str) -> Result { get(&key(name)?) } - pub mod this_thread { - use super::{get, key, set, Key, OnceCell, Result}; + use super::{is_nonzero, key, math, Debug, Key, OnceCell, Result}; - pub fn trim() -> Result { - let mut key = mallctl!("arena.0.purge"); - key[1] = arena_id()?.try_into()?; - set(&key, ()) + pub fn trim() -> Result { notify(mallctl!("arena.0.purge")) } + + pub fn decay() -> Result { notify(mallctl!("arena.0.decay")) } + + pub fn flush() -> Result { super::notify(&mallctl!("thread.tcache.flush")) } + + pub fn set_muzzy_decay(decay_ms: isize) -> Result { + set(mallctl!("arena.0.muzzy_decay_ms"), decay_ms) } - pub fn decay() -> Result { - let mut key = mallctl!("arena.0.decay"); - key[1] = arena_id()?.try_into()?; - set(&key, ()) + pub fn get_muzzy_decay() -> Result { get(mallctl!("arena.0.muzzy_decay_ms")) } + + pub fn set_dirty_decay(decay_ms: isize) -> Result { + set(mallctl!("arena.0.dirty_decay_ms"), decay_ms) } - pub fn cache(enable: bool) -> Result { - set(&mallctl!("thread.tcache.enabled"), u8::from(enable)) + pub fn get_dirty_decay() -> Result { get(mallctl!("arena.0.dirty_decay_ms")) } + + pub fn enable_cache(enable: bool) -> Result { + super::set::(&mallctl!("thread.tcache.enabled"), enable.into()).map(is_nonzero!()) } - pub fn flush() -> Result { set(&mallctl!("thread.tcache.flush"), ()) } + pub fn is_cache_enabled() -> Result { + super::get::(&mallctl!("thread.tcache.enabled")).map(is_nonzero!()) + } - pub fn allocated() -> Result { get::(&mallctl!("thread.allocated")) } + pub fn set_arena(id: usize) -> Result { + super::set::(&mallctl!("thread.arena"), id.try_into()?).and_then(math::try_into) + } - pub fn deallocated() -> Result { get::(&mallctl!("thread.deallocated")) } + pub fn arena_id() -> Result { + super::get::(&mallctl!("thread.arena")).and_then(math::try_into) + } - pub fn arena_id() -> Result { get::(&mallctl!("thread.arena")) } + pub fn allocated() -> Result { super::get(&mallctl!("thread.allocated")) } + + pub fn deallocated() -> Result { super::get(&mallctl!("thread.deallocated")) } + + fn notify(key: Key) -> Result { super::notify_by_arena(Some(arena_id()?), key) } + + fn set(key: Key, val: T) -> Result + where + T: Copy + Debug, + { + super::set_by_arena(Some(arena_id()?), key, val) + } + + fn get(key: Key) -> Result + where + T: Copy + Debug, + { + super::get_by_arena(Some(arena_id()?), key) + } } -fn set(key: &Key, val: T) -> Result +pub fn trim>>(arena: I) -> Result { + notify_by_arena(arena.into(), mallctl!("arena.4096.purge")) +} + +pub fn decay>>(arena: I) -> Result { + notify_by_arena(arena.into(), mallctl!("arena.4096.decay")) +} + +pub fn set_muzzy_decay>>(arena: I, decay_ms: isize) -> Result { + if let Some(arena) = arena.into() { + set_by_arena(Some(arena), mallctl!("arena.4096.muzzy_decay_ms"), decay_ms) + } else { + set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms) + } +} + +pub fn set_dirty_decay>>(arena: I, decay_ms: isize) -> Result { + if let Some(arena) = arena.into() { + set_by_arena(Some(arena), mallctl!("arena.4096.dirty_decay_ms"), decay_ms) + } else { + set(&mallctl!("arenas.dirty_decay_ms"), decay_ms) + } +} + +#[inline] +#[must_use] +pub fn is_affine_arena() -> bool { is_percpu_arena() || is_phycpu_arena() } + +#[inline] +#[must_use] +pub fn is_percpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("percpu")) } + +#[inline] +#[must_use] +pub fn is_phycpu_arena() -> bool { percpu_arenas().is_ok_and(is_equal_to!("phycpu")) } + +pub fn percpu_arenas() -> Result<&'static str> { + let ptr = get::<*const c_char>(&mallctl!("opt.percpu_arena"))?; + //SAFETY: ptr points to a null-terminated string returned for opt.percpu_arena. + let cstr = unsafe { CStr::from_ptr(ptr) }; + cstr.to_str().map_err(Into::into) +} + +pub fn arenas() -> Result { + get::(&mallctl!("arenas.narenas")).and_then(math::try_into) +} + +pub fn inc_epoch() -> Result { xchg(&mallctl!("epoch"), 1_u64) } + +pub fn acq_epoch() -> Result { xchg(&mallctl!("epoch"), 0_u64) } + +fn notify_by_arena(id: Option, mut key: Key) -> Result { + key[1] = id.unwrap_or(4096); + notify(&key) +} + +fn set_by_arena(id: Option, mut key: Key, val: T) -> Result where T: Copy + Debug, { - // SAFETY: T must be the exact expected type. - unsafe { mallctl::raw::write_mib(key.as_slice(), val) }.map_err(map_err) + key[1] = id.unwrap_or(4096); + set(&key, val) +} + +fn get_by_arena(id: Option, mut key: Key) -> Result +where + T: Copy + Debug, +{ + key[1] = id.unwrap_or(4096); + get(&key) +} + +fn notify(key: &Key) -> Result { xchg(key, ()) } + +fn set(key: &Key, val: T) -> Result +where + T: Copy + Debug, +{ + let _lock = CONTROL.write()?; + let res = xchg(key, val)?; + inc_epoch()?; + + Ok(res) } fn get(key: &Key) -> Result where T: Copy + Debug, { + acq_epoch()?; + acq_epoch()?; + // SAFETY: T must be perfectly valid to receive value. unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err) } +fn xchg(key: &Key, val: T) -> Result +where + T: Copy + Debug, +{ + // SAFETY: T must be the exact expected type. + unsafe { mallctl::raw::update_mib(key.as_slice(), val) }.map_err(map_err) +} + fn key(name: &str) -> Result { // tikv asserts the output buffer length is tight to the number of required mibs // so we slice that down here. diff --git a/src/database/pool.rs b/src/database/pool.rs index f5600c36..86516c31 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -13,7 +13,7 @@ use std::{ use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ debug, debug_warn, err, error, implement, - result::DebugInspect, + result::{DebugInspect, LogDebugErr}, trace, utils::sys::compute::{get_affinity, nth_core_available, set_affinity}, Error, Result, Server, @@ -289,6 +289,20 @@ fn worker_init(&self, id: usize) { // affinity is empty (no-op) if there's only one queue set_affinity(affinity.clone()); + + #[cfg(feature = "jemalloc")] + if affinity.clone().count() == 1 && conduwuit::alloc::je::is_affine_arena() { + use conduwuit::alloc::je::this_thread::{arena_id, set_arena}; + + let id = affinity.clone().next().expect("at least one id"); + + if let Ok(arena) = arena_id() { + if arena != id { + set_arena(id).log_debug_err().ok(); + } + } + } + debug!( ?group, affinity = ?affinity.collect::>(), diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index baf5336f..f774c37a 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -41,6 +41,7 @@ default = [ "gzip_compression", "io_uring", "jemalloc", + "jemalloc_conf", "media_thumbnail", "release_max_log_level", "systemd", @@ -85,6 +86,9 @@ jemalloc_prof = [ jemalloc_stats = [ "conduwuit-core/jemalloc_stats", ] +jemalloc_conf = [ + "conduwuit-core/jemalloc_conf", +] media_thumbnail = [ "conduwuit-service/media_thumbnail", ] diff --git a/src/main/clap.rs b/src/main/clap.rs index d3d40491..2bb6f3f2 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -92,6 +92,22 @@ pub(crate) struct Args { require_equals(false), )] pub(crate) gc_on_park: Option, + + /// Toggles muzzy decay for jemalloc arenas associated with a tokio + /// worker (when worker-affinity is enabled). Setting to false releases + /// memory to the operating system using MADV_FREE without MADV_DONTNEED. + /// Setting to false increases performance by reducing pagefaults, but + /// resident memory usage appears high until there is memory pressure. The + /// default is true unless the system has four or more cores. + #[arg( + long, + hide(true), + env = "CONDUWUIT_RUNTIME_GC_MUZZY", + action = ArgAction::Set, + num_args = 0..=1, + require_equals(false), + )] + pub(crate) gc_muzzy: Option, } /// Parse commandline arguments into structured data diff --git a/src/main/runtime.rs b/src/main/runtime.rs index 315336b0..9f4f60f8 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -9,8 +9,12 @@ use std::{ }; use conduwuit::{ - result::LogErr, - utils::sys::compute::{nth_core_available, set_affinity}, + is_true, + result::LogDebugErr, + utils::{ + available_parallelism, + sys::compute::{nth_core_available, set_affinity}, + }, Result, }; use tokio::runtime::Builder; @@ -21,9 +25,11 @@ const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; const WORKER_KEEPALIVE: u64 = 36; const MAX_BLOCKING_THREADS: usize = 1024; +const DISABLE_MUZZY_THRESHOLD: usize = 4; static WORKER_AFFINITY: OnceLock = OnceLock::new(); static GC_ON_PARK: OnceLock> = OnceLock::new(); +static GC_MUZZY: OnceLock> = OnceLock::new(); pub(super) fn new(args: &Args) -> Result { WORKER_AFFINITY @@ -34,6 +40,10 @@ pub(super) fn new(args: &Args) -> Result { .set(args.gc_on_park) .expect("set GC_ON_PARK from program argument"); + GC_MUZZY + .set(args.gc_muzzy) + .expect("set GC_MUZZY from program argument"); + let mut builder = Builder::new_multi_thread(); builder .enable_io() @@ -83,11 +93,13 @@ fn enable_histogram(builder: &mut Builder, args: &Args) { ), )] fn thread_start() { - if WORKER_AFFINITY - .get() - .copied() - .expect("WORKER_AFFINITY initialized by runtime::new()") - { + debug_assert_eq!( + Some(WORKER_NAME), + thread::current().name(), + "tokio worker name mismatch at thread start" + ); + + if WORKER_AFFINITY.get().is_some_and(is_true!()) { set_worker_affinity(); } } @@ -95,10 +107,6 @@ fn thread_start() { fn set_worker_affinity() { static CORES_OCCUPIED: AtomicUsize = AtomicUsize::new(0); - if thread::current().name() != Some(WORKER_NAME) { - return; - } - let handle = tokio::runtime::Handle::current(); let num_workers = handle.metrics().num_workers(); let i = CORES_OCCUPIED.fetch_add(1, Ordering::Relaxed); @@ -111,8 +119,33 @@ fn set_worker_affinity() { }; set_affinity(once(id)); + set_worker_mallctl(id); } +#[cfg(feature = "jemalloc")] +fn set_worker_mallctl(id: usize) { + use conduwuit::alloc::je::{ + is_affine_arena, + this_thread::{set_arena, set_muzzy_decay}, + }; + + if is_affine_arena() { + set_arena(id).log_debug_err().ok(); + } + + let muzzy_option = GC_MUZZY + .get() + .expect("GC_MUZZY initialized by runtime::new()"); + + let muzzy_auto_disable = available_parallelism() >= DISABLE_MUZZY_THRESHOLD; + if matches!(muzzy_option, Some(false) | None if muzzy_auto_disable) { + set_muzzy_decay(-1).log_debug_err().ok(); + } +} + +#[cfg(not(feature = "jemalloc"))] +fn set_worker_mallctl(_: usize) {} + #[tracing::instrument( name = "join", level = "debug", @@ -157,7 +190,9 @@ fn thread_park() { fn gc_on_park() { #[cfg(feature = "jemalloc")] - conduwuit::alloc::je::this_thread::decay().log_err().ok(); + conduwuit::alloc::je::this_thread::decay() + .log_debug_err() + .ok(); } #[cfg(tokio_unstable)] From ac944496c15bc476bc9964e034b7fcea737cc733 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 14 Jan 2025 19:17:45 +0000 Subject: [PATCH 1378/2091] optimize statekey-from-short loopsite Signed-off-by: Jason Volk --- .../rooms/event_handler/resolve_state.rs | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 8640c582..edce880d 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -6,10 +6,10 @@ use std::{ use conduwuit::{ debug, err, implement, - utils::stream::{automatic_width, IterStream, WidebandExt}, + utils::stream::{automatic_width, IterStream, ReadyExt, WidebandExt}, Result, }; -use futures::{FutureExt, StreamExt, TryFutureExt}; +use futures::{FutureExt, StreamExt}; use ruma::{ state_res::{self, StateMap}, OwnedEventId, RoomId, RoomVersionId, @@ -59,17 +59,18 @@ pub async fn resolve_state( let fork_states: Vec> = fork_states .into_iter() .stream() - .wide_then(|fork_state| { - fork_state - .into_iter() - .stream() - .wide_filter_map(|(k, id)| { - self.services - .short - .get_statekey_from_short(k) - .map_ok_or_else(|_| None, move |(ty, st_key)| Some(((ty, st_key), id))) - }) + .wide_then(|fork_state| async move { + let shortstatekeys = fork_state.keys().copied().stream(); + + let event_ids = fork_state.values().cloned().stream().boxed(); + + self.services + .short + .multi_get_statekey_from_short(shortstatekeys) + .zip(event_ids) + .ready_filter_map(|(ty_sk, id)| Some((ty_sk.ok()?, id))) .collect() + .await }) .collect() .await; From 388730d6dd7dc69e1243218e556946bb35fd7461 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 20 Jan 2025 07:38:19 +0000 Subject: [PATCH 1379/2091] add TryWideband trait to similar to TryBroadband Signed-off-by: Jason Volk --- src/core/utils/stream/mod.rs | 2 + src/core/utils/stream/try_wideband.rs | 57 +++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 src/core/utils/stream/try_wideband.rs diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index 0fee0a3a..c7bfa021 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -9,6 +9,7 @@ mod tools; mod try_broadband; mod try_ready; mod try_tools; +mod try_wideband; mod wideband; pub use band::{ @@ -25,4 +26,5 @@ pub use tools::Tools; pub use try_broadband::TryBroadbandExt; pub use try_ready::TryReadyExt; pub use try_tools::TryTools; +pub use try_wideband::TryWidebandExt; pub use wideband::WidebandExt; diff --git a/src/core/utils/stream/try_wideband.rs b/src/core/utils/stream/try_wideband.rs new file mode 100644 index 00000000..0af3c8ec --- /dev/null +++ b/src/core/utils/stream/try_wideband.rs @@ -0,0 +1,57 @@ +//! Synchronous combinator extensions to futures::TryStream + +use futures::{TryFuture, TryStream, TryStreamExt}; + +use super::automatic_width; +use crate::Result; + +/// Concurrency extensions to augment futures::TryStreamExt. wide_ combinators +/// produce in-order results +pub trait TryWidebandExt +where + Self: TryStream> + Send + Sized, +{ + fn widen_and_then( + self, + n: N, + f: F, + ) -> impl TryStream> + Send + where + N: Into>, + F: Fn(Self::Ok) -> Fut + Send, + Fut: TryFuture> + Send, + U: Send; + + fn wide_and_then( + self, + f: F, + ) -> impl TryStream> + Send + where + F: Fn(Self::Ok) -> Fut + Send, + Fut: TryFuture> + Send, + U: Send, + { + self.widen_and_then(None, f) + } +} + +impl TryWidebandExt for S +where + S: TryStream> + Send + Sized, + E: Send, +{ + fn widen_and_then( + self, + n: N, + f: F, + ) -> impl TryStream> + Send + where + N: Into>, + F: Fn(Self::Ok) -> Fut + Send, + Fut: TryFuture> + Send, + U: Send, + { + self.map_ok(f) + .try_buffered(n.into().unwrap_or_else(automatic_width)) + } +} From ea25dc04b26fb4c3dffb456fd1c8e4b7c066b95f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 20 Jan 2025 07:38:32 +0000 Subject: [PATCH 1380/2091] parallelize current and incoming fork-state fetch Signed-off-by: Jason Volk --- .../rooms/event_handler/resolve_state.rs | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index edce880d..0526d31c 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -6,10 +6,10 @@ use std::{ use conduwuit::{ debug, err, implement, - utils::stream::{automatic_width, IterStream, ReadyExt, WidebandExt}, + utils::stream::{automatic_width, IterStream, ReadyExt, TryWidebandExt, WidebandExt}, Result, }; -use futures::{FutureExt, StreamExt}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ state_res::{self, StateMap}, OwnedEventId, RoomId, RoomVersionId, @@ -40,20 +40,24 @@ pub async fn resolve_state( .await?; let fork_states = [current_state_ids, incoming_state]; - let mut auth_chain_sets = Vec::with_capacity(fork_states.len()); - for state in &fork_states { - let starting_events = state.values().map(Borrow::borrow); + let auth_chain_sets: Vec> = fork_states + .iter() + .try_stream() + .wide_and_then(|state| async move { + let starting_events = state.values().map(Borrow::borrow); - let auth_chain: HashSet = self - .services - .auth_chain - .get_event_ids(room_id, starting_events) - .await? - .into_iter() - .collect(); + let auth_chain = self + .services + .auth_chain + .get_event_ids(room_id, starting_events) + .await? + .into_iter() + .collect(); - auth_chain_sets.push(auth_chain); - } + Ok(auth_chain) + }) + .try_collect() + .await?; debug!("Loading fork states"); let fork_states: Vec> = fork_states From 4c0ae8c2f708cc2d950f6a8269844ae42069d55a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 20 Jan 2025 09:02:50 +0000 Subject: [PATCH 1381/2091] parallelize get_auth_chain outer Signed-off-by: Jason Volk --- src/service/rooms/auth_chain/mod.rs | 112 ++++++++++++++-------------- 1 file changed, 55 insertions(+), 57 deletions(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 74064701..fb7b6163 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -7,11 +7,14 @@ use std::{ }; use conduwuit::{ - debug, debug_error, trace, - utils::{stream::ReadyExt, IterStream}, + at, debug, debug_error, trace, + utils::{ + stream::{ReadyExt, TryBroadbandExt}, + IterStream, + }, validated, warn, Err, Result, }; -use futures::{Stream, StreamExt}; +use futures::{Stream, StreamExt, TryStreamExt}; use ruma::{EventId, OwnedEventId, RoomId}; use self::data::Data; @@ -112,66 +115,61 @@ impl Service { "start", ); - let mut hits: usize = 0; - let mut misses: usize = 0; - let mut full_auth_chain = Vec::with_capacity(buckets.len()); - for chunk in buckets { - if chunk.is_empty() { - continue; - } + let full_auth_chain: Vec<_> = buckets + .into_iter() + .try_stream() + .broad_and_then(|chunk| async move { + let chunk_key: Vec = chunk.iter().map(at!(0)).collect(); - let chunk_key: Vec = - chunk.iter().map(|(short, _)| short).copied().collect(); - if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { - trace!("Found cache entry for whole chunk"); - full_auth_chain.extend(cached.iter().copied()); - hits = hits.saturating_add(1); - continue; - } + if chunk_key.is_empty() { + return Ok(Vec::new()); + } - let mut hits2: usize = 0; - let mut misses2: usize = 0; - let mut chunk_cache = Vec::with_capacity(chunk.len()); - for (sevent_id, event_id) in chunk { - if let Ok(cached) = self.get_cached_eventid_authchain(&[sevent_id]).await { - trace!(?event_id, "Found cache entry for event"); - chunk_cache.extend(cached.iter().copied()); - hits2 = hits2.saturating_add(1); - } else { - let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; - self.cache_auth_chain(vec![sevent_id], &auth_chain); - chunk_cache.extend(auth_chain.iter()); - misses2 = misses2.saturating_add(1); - debug!( - event_id = ?event_id, - chain_length = ?auth_chain.len(), - chunk_cache_length = ?chunk_cache.len(), - elapsed = ?started.elapsed(), - "Cache missed event" - ); - }; - } + if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { + return Ok(cached.to_vec()); + } - chunk_cache.sort_unstable(); - chunk_cache.dedup(); - self.cache_auth_chain_vec(chunk_key, &chunk_cache); - full_auth_chain.extend(chunk_cache.iter()); - misses = misses.saturating_add(1); - debug!( - chunk_cache_length = ?chunk_cache.len(), - hits = ?hits2, - misses = ?misses2, - elapsed = ?started.elapsed(), - "Chunk missed", - ); - } + let chunk_cache: Vec<_> = chunk + .into_iter() + .try_stream() + .broad_and_then(|(shortid, event_id)| async move { + if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await { + return Ok(cached.to_vec()); + } + let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; + self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice()); + debug!( + ?event_id, + elapsed = ?started.elapsed(), + "Cache missed event" + ); + + Ok(auth_chain) + }) + .try_collect() + .await?; + + let mut chunk_cache: Vec<_> = chunk_cache.into_iter().flatten().collect(); + chunk_cache.sort_unstable(); + chunk_cache.dedup(); + self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice()); + debug!( + chunk_cache_length = ?chunk_cache.len(), + elapsed = ?started.elapsed(), + "Cache missed chunk", + ); + + Ok(chunk_cache) + }) + .try_collect() + .await?; + + let mut full_auth_chain: Vec<_> = full_auth_chain.into_iter().flatten().collect(); full_auth_chain.sort_unstable(); full_auth_chain.dedup(); debug!( chain_length = ?full_auth_chain.len(), - hits = ?hits, - misses = ?misses, elapsed = ?started.elapsed(), "done", ); @@ -184,7 +182,7 @@ impl Service { &self, room_id: &RoomId, event_id: &EventId, - ) -> Result> { + ) -> Result> { let mut todo: VecDeque<_> = [event_id.to_owned()].into(); let mut found = HashSet::new(); @@ -226,7 +224,7 @@ impl Service { } } - Ok(found) + Ok(found.into_iter().collect()) } #[inline] From 610129d16265f702b7bfbf0ada019fc77766e10f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 20 Jan 2025 09:05:49 +0000 Subject: [PATCH 1382/2091] outdent auth_chain Service impl Signed-off-by: Jason Volk --- src/service/rooms/auth_chain/mod.rs | 429 ++++++++++++++-------------- 1 file changed, 219 insertions(+), 210 deletions(-) diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index fb7b6163..df2663b2 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -7,14 +7,14 @@ use std::{ }; use conduwuit::{ - at, debug, debug_error, trace, + at, debug, debug_error, implement, trace, utils::{ stream::{ReadyExt, TryBroadbandExt}, IterStream, }, validated, warn, Err, Result, }; -use futures::{Stream, StreamExt, TryStreamExt}; +use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{EventId, OwnedEventId, RoomId}; use self::data::Data; @@ -44,213 +44,222 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - pub async fn event_ids_iter<'a, I>( - &'a self, - room_id: &RoomId, - starting_events: I, - ) -> Result + Send + '_> - where - I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, - { - let stream = self - .get_event_ids(room_id, starting_events) - .await? - .into_iter() - .stream(); +#[implement(Service)] +pub async fn event_ids_iter<'a, I>( + &'a self, + room_id: &RoomId, + starting_events: I, +) -> Result + Send + '_> +where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, +{ + let stream = self + .get_event_ids(room_id, starting_events) + .await? + .into_iter() + .stream(); - Ok(stream) - } - - pub async fn get_event_ids<'a, I>( - &'a self, - room_id: &RoomId, - starting_events: I, - ) -> Result> - where - I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, - { - let chain = self.get_auth_chain(room_id, starting_events).await?; - let event_ids = self - .services - .short - .multi_get_eventid_from_short(chain.into_iter().stream()) - .ready_filter_map(Result::ok) - .collect() - .await; - - Ok(event_ids) - } - - #[tracing::instrument(name = "auth_chain", level = "debug", skip_all)] - pub async fn get_auth_chain<'a, I>( - &'a self, - room_id: &RoomId, - starting_events: I, - ) -> Result> - where - I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, - { - const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? - const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); - - let started = std::time::Instant::now(); - let mut starting_ids = self - .services - .short - .multi_get_or_create_shorteventid(starting_events.clone()) - .zip(starting_events.clone().stream()) - .boxed(); - - let mut buckets = [BUCKET; NUM_BUCKETS]; - while let Some((short, starting_event)) = starting_ids.next().await { - let bucket: usize = short.try_into()?; - let bucket: usize = validated!(bucket % NUM_BUCKETS); - buckets[bucket].insert((short, starting_event)); - } - - debug!( - starting_events = ?starting_events.count(), - elapsed = ?started.elapsed(), - "start", - ); - - let full_auth_chain: Vec<_> = buckets - .into_iter() - .try_stream() - .broad_and_then(|chunk| async move { - let chunk_key: Vec = chunk.iter().map(at!(0)).collect(); - - if chunk_key.is_empty() { - return Ok(Vec::new()); - } - - if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { - return Ok(cached.to_vec()); - } - - let chunk_cache: Vec<_> = chunk - .into_iter() - .try_stream() - .broad_and_then(|(shortid, event_id)| async move { - if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await { - return Ok(cached.to_vec()); - } - - let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; - self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice()); - debug!( - ?event_id, - elapsed = ?started.elapsed(), - "Cache missed event" - ); - - Ok(auth_chain) - }) - .try_collect() - .await?; - - let mut chunk_cache: Vec<_> = chunk_cache.into_iter().flatten().collect(); - chunk_cache.sort_unstable(); - chunk_cache.dedup(); - self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice()); - debug!( - chunk_cache_length = ?chunk_cache.len(), - elapsed = ?started.elapsed(), - "Cache missed chunk", - ); - - Ok(chunk_cache) - }) - .try_collect() - .await?; - - let mut full_auth_chain: Vec<_> = full_auth_chain.into_iter().flatten().collect(); - full_auth_chain.sort_unstable(); - full_auth_chain.dedup(); - debug!( - chain_length = ?full_auth_chain.len(), - elapsed = ?started.elapsed(), - "done", - ); - - Ok(full_auth_chain) - } - - #[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))] - async fn get_auth_chain_inner( - &self, - room_id: &RoomId, - event_id: &EventId, - ) -> Result> { - let mut todo: VecDeque<_> = [event_id.to_owned()].into(); - let mut found = HashSet::new(); - - while let Some(event_id) = todo.pop_front() { - trace!(?event_id, "processing auth event"); - - match self.services.timeline.get_pdu(&event_id).await { - | Err(e) => { - debug_error!(?event_id, ?e, "Could not find pdu mentioned in auth events"); - }, - | Ok(pdu) => { - if pdu.room_id != room_id { - return Err!(Request(Forbidden(error!( - ?event_id, - ?room_id, - wrong_room_id = ?pdu.room_id, - "auth event for incorrect room" - )))); - } - - for auth_event in &pdu.auth_events { - let sauthevent = self - .services - .short - .get_or_create_shorteventid(auth_event) - .await; - - if found.insert(sauthevent) { - trace!( - ?event_id, - ?auth_event, - "adding auth event to processing queue" - ); - - todo.push_back(auth_event.clone()); - } - } - }, - } - } - - Ok(found.into_iter().collect()) - } - - #[inline] - pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { - self.db.get_cached_eventid_authchain(key).await - } - - #[tracing::instrument(skip_all, level = "debug")] - pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) { - let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); - - self.db.cache_auth_chain(key, val); - } - - #[tracing::instrument(skip_all, level = "debug")] - pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &[ShortEventId]) { - let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); - - self.db.cache_auth_chain(key, val); - } - - pub fn get_cache_usage(&self) -> (usize, usize) { - let cache = self.db.auth_chain_cache.lock().expect("locked"); - - (cache.len(), cache.capacity()) - } - - pub fn clear_cache(&self) { self.db.auth_chain_cache.lock().expect("locked").clear(); } + Ok(stream) } + +#[implement(Service)] +pub async fn get_event_ids<'a, I>( + &'a self, + room_id: &RoomId, + starting_events: I, +) -> Result> +where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, +{ + let chain = self.get_auth_chain(room_id, starting_events).await?; + let event_ids = self + .services + .short + .multi_get_eventid_from_short(chain.into_iter().stream()) + .ready_filter_map(Result::ok) + .collect() + .await; + + Ok(event_ids) +} + +#[implement(Service)] +#[tracing::instrument(name = "auth_chain", level = "debug", skip_all)] +pub async fn get_auth_chain<'a, I>( + &'a self, + room_id: &RoomId, + starting_events: I, +) -> Result> +where + I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, +{ + const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? + const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); + + let started = std::time::Instant::now(); + let mut starting_ids = self + .services + .short + .multi_get_or_create_shorteventid(starting_events.clone()) + .zip(starting_events.clone().stream()) + .boxed(); + + let mut buckets = [BUCKET; NUM_BUCKETS]; + while let Some((short, starting_event)) = starting_ids.next().await { + let bucket: usize = short.try_into()?; + let bucket: usize = validated!(bucket % NUM_BUCKETS); + buckets[bucket].insert((short, starting_event)); + } + + debug!( + starting_events = ?starting_events.count(), + elapsed = ?started.elapsed(), + "start", + ); + + let full_auth_chain: Vec = buckets + .into_iter() + .try_stream() + .broad_and_then(|chunk| async move { + let chunk_key: Vec = chunk.iter().map(at!(0)).collect(); + + if chunk_key.is_empty() { + return Ok(Vec::new()); + } + + if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { + return Ok(cached.to_vec()); + } + + let chunk_cache: Vec<_> = chunk + .into_iter() + .try_stream() + .broad_and_then(|(shortid, event_id)| async move { + if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await { + return Ok(cached.to_vec()); + } + + let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; + self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice()); + debug!( + ?event_id, + elapsed = ?started.elapsed(), + "Cache missed event" + ); + + Ok(auth_chain) + }) + .try_collect() + .map_ok(|chunk_cache: Vec<_>| chunk_cache.into_iter().flatten().collect()) + .map_ok(|mut chunk_cache: Vec<_>| { + chunk_cache.sort_unstable(); + chunk_cache.dedup(); + chunk_cache + }) + .await?; + + self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice()); + debug!( + chunk_cache_length = ?chunk_cache.len(), + elapsed = ?started.elapsed(), + "Cache missed chunk", + ); + + Ok(chunk_cache) + }) + .try_collect() + .map_ok(|auth_chain: Vec<_>| auth_chain.into_iter().flatten().collect()) + .map_ok(|mut full_auth_chain: Vec<_>| { + full_auth_chain.sort_unstable(); + full_auth_chain.dedup(); + full_auth_chain + }) + .await?; + + debug!( + chain_length = ?full_auth_chain.len(), + elapsed = ?started.elapsed(), + "done", + ); + + Ok(full_auth_chain) +} + +#[implement(Service)] +#[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))] +async fn get_auth_chain_inner( + &self, + room_id: &RoomId, + event_id: &EventId, +) -> Result> { + let mut todo: VecDeque<_> = [event_id.to_owned()].into(); + let mut found = HashSet::new(); + + while let Some(event_id) = todo.pop_front() { + trace!(?event_id, "processing auth event"); + + match self.services.timeline.get_pdu(&event_id).await { + | Err(e) => { + debug_error!(?event_id, ?e, "Could not find pdu mentioned in auth events"); + }, + | Ok(pdu) => { + if pdu.room_id != room_id { + return Err!(Request(Forbidden(error!( + ?event_id, + ?room_id, + wrong_room_id = ?pdu.room_id, + "auth event for incorrect room" + )))); + } + + for auth_event in &pdu.auth_events { + let sauthevent = self + .services + .short + .get_or_create_shorteventid(auth_event) + .await; + + if found.insert(sauthevent) { + trace!(?event_id, ?auth_event, "adding auth event to processing queue"); + + todo.push_back(auth_event.clone()); + } + } + }, + } + } + + Ok(found.into_iter().collect()) +} + +#[implement(Service)] +#[inline] +pub async fn get_cached_eventid_authchain(&self, key: &[u64]) -> Result> { + self.db.get_cached_eventid_authchain(key).await +} + +#[implement(Service)] +#[tracing::instrument(skip_all, level = "debug")] +pub fn cache_auth_chain(&self, key: Vec, auth_chain: &HashSet) { + let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); + + self.db.cache_auth_chain(key, val); +} + +#[implement(Service)] +#[tracing::instrument(skip_all, level = "debug")] +pub fn cache_auth_chain_vec(&self, key: Vec, auth_chain: &[ShortEventId]) { + let val: Arc<[ShortEventId]> = auth_chain.iter().copied().collect(); + + self.db.cache_auth_chain(key, val); +} + +#[implement(Service)] +pub fn get_cache_usage(&self) -> (usize, usize) { + let cache = self.db.auth_chain_cache.lock().expect("locked"); + + (cache.len(), cache.capacity()) +} + +#[implement(Service)] +pub fn clear_cache(&self) { self.db.auth_chain_cache.lock().expect("locked").clear(); } From 277b4951e8e7f28e6319a17e91229e77c9db090d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 20 Jan 2025 11:50:17 +0000 Subject: [PATCH 1383/2091] add compression-shaping; tweak default compression levels Signed-off-by: Jason Volk --- conduwuit-example.toml | 8 +++++++- src/core/config/mod.rs | 8 +++++++- src/database/engine/cf_opts.rs | 32 +++++++++++++++++++++++++++---- src/database/engine/descriptor.rs | 19 ++++++++++++++++-- 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 54143ced..79efbd14 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -818,6 +818,9 @@ # magic number and translated to the library's default compression level # as they all differ. See their `kDefaultCompressionLevel`. # +# Note when using the default value we may override it with a setting +# tailored specifically conduwuit. +# #rocksdb_compression_level = 32767 # Level of compression the specified compression algorithm for the @@ -831,6 +834,9 @@ # less likely for this data to be used. Research your chosen compression # algorithm. # +# Note when using the default value we may override it with a setting +# tailored specifically conduwuit. +# #rocksdb_bottommost_compression_level = 32767 # Whether to enable RocksDB's "bottommost_compression". @@ -842,7 +848,7 @@ # # See https://github.com/facebook/rocksdb/wiki/Compression for more details. # -#rocksdb_bottommost_compression = false +#rocksdb_bottommost_compression = true # Database recovery mode (for RocksDB WAL corruption). # diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index cb42940b..5cfed0b9 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -958,6 +958,9 @@ pub struct Config { /// magic number and translated to the library's default compression level /// as they all differ. See their `kDefaultCompressionLevel`. /// + /// Note when using the default value we may override it with a setting + /// tailored specifically conduwuit. + /// /// default: 32767 #[serde(default = "default_rocksdb_compression_level")] pub rocksdb_compression_level: i32, @@ -973,6 +976,9 @@ pub struct Config { /// less likely for this data to be used. Research your chosen compression /// algorithm. /// + /// Note when using the default value we may override it with a setting + /// tailored specifically conduwuit. + /// /// default: 32767 #[serde(default = "default_rocksdb_bottommost_compression_level")] pub rocksdb_bottommost_compression_level: i32, @@ -985,7 +991,7 @@ pub struct Config { /// if you're trying to reduce storage usage from the database. /// /// See https://github.com/facebook/rocksdb/wiki/Compression for more details. - #[serde(default)] + #[serde(default = "true_fn")] pub rocksdb_bottommost_compression: bool, /// Database recovery mode (for RocksDB WAL corruption). diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 7b3a1d49..da636718 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -8,6 +8,8 @@ use rocksdb::{ use super::descriptor::{CacheDisp, Descriptor}; use crate::{util::map_err, Context}; +pub(super) const SENTINEL_COMPRESSION_LEVEL: i32 = 32767; + /// Adjust options for the specific column by name. Provide the result of /// db_options() as the argument to this function and use the return value in /// the arguments to open the specific column. @@ -45,7 +47,15 @@ fn descriptor_cf_options( opts.set_compaction_pri(desc.compaction_pri); opts.set_universal_compaction_options(&uc_options(&desc)); + let compression_shape: Vec<_> = desc + .compression_shape + .into_iter() + .map(|val| (val > 0).then_some(desc.compression)) + .map(|val| val.unwrap_or(CompressionType::None)) + .collect(); + opts.set_compression_type(desc.compression); + opts.set_compression_per_level(compression_shape.as_slice()); opts.set_compression_options(-14, desc.compression_level, 0, 0); // -14 w_bits used by zlib. if let Some(&bottommost_level) = desc.bottommost_level.as_ref() { opts.set_bottommost_compression_type(desc.compression); @@ -95,10 +105,24 @@ fn set_compression(desc: &mut Descriptor, config: &Config) { | _ => CompressionType::Zstd, }; - desc.compression_level = config.rocksdb_compression_level; - desc.bottommost_level = config - .rocksdb_bottommost_compression - .then_some(config.rocksdb_bottommost_compression_level); + let can_override_level = config.rocksdb_compression_level == SENTINEL_COMPRESSION_LEVEL + && desc.compression == CompressionType::Zstd; + + if !can_override_level { + desc.compression_level = config.rocksdb_compression_level; + } + + let can_override_bottom = config.rocksdb_bottommost_compression_level + == SENTINEL_COMPRESSION_LEVEL + && desc.compression == CompressionType::Zstd; + + if !can_override_bottom { + desc.bottommost_level = Some(config.rocksdb_bottommost_compression_level); + } + + if !config.rocksdb_bottommost_compression { + desc.bottommost_level = None; + } } fn uc_options(desc: &Descriptor) -> UniversalCompactOptions { diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index 234ca2bf..2c84ac53 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -4,6 +4,8 @@ use rocksdb::{ DBCompressionType as CompressionType, }; +use super::cf_opts::SENTINEL_COMPRESSION_LEVEL; + #[derive(Debug, Clone, Copy)] pub(crate) enum CacheDisp { Unique, @@ -32,6 +34,7 @@ pub(crate) struct Descriptor { pub(crate) compaction: CompactionStyle, pub(crate) compaction_pri: CompactionPri, pub(crate) compression: CompressionType, + pub(crate) compression_shape: [i32; 7], pub(crate) compression_level: i32, pub(crate) bottommost_level: Option, pub(crate) block_index_hashing: Option, @@ -58,8 +61,9 @@ pub(crate) static BASE: Descriptor = Descriptor { compaction: CompactionStyle::Level, compaction_pri: CompactionPri::MinOverlappingRatio, compression: CompressionType::Zstd, - compression_level: 32767, - bottommost_level: Some(32767), + compression_shape: [0, 0, 0, 1, 1, 1, 1], + compression_level: SENTINEL_COMPRESSION_LEVEL, + bottommost_level: Some(SENTINEL_COMPRESSION_LEVEL), block_index_hashing: None, cache_shards: 64, }; @@ -68,6 +72,8 @@ pub(crate) static RANDOM: Descriptor = Descriptor { compaction_pri: CompactionPri::OldestSmallestSeqFirst, write_size: 1024 * 1024 * 32, cache_shards: 128, + compression_level: -3, + bottommost_level: Some(4), ..BASE }; @@ -77,6 +83,9 @@ pub(crate) static SEQUENTIAL: Descriptor = Descriptor { level_size: 1024 * 1024 * 32, file_size: 1024 * 1024 * 2, cache_shards: 128, + compression_level: -1, + bottommost_level: Some(6), + compression_shape: [0, 0, 1, 1, 1, 1, 1], ..BASE }; @@ -88,6 +97,9 @@ pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { index_size: 512, block_size: 512, cache_shards: 64, + compression_level: -4, + bottommost_level: Some(1), + compression_shape: [0, 0, 0, 0, 0, 1, 1], ..RANDOM }; @@ -99,5 +111,8 @@ pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { block_size: 512, cache_shards: 64, block_index_hashing: Some(false), + compression_level: -2, + bottommost_level: Some(4), + compression_shape: [0, 0, 0, 0, 1, 1, 1], ..SEQUENTIAL }; From 19f6d9d0e1dd2e40bb710bdc0e876e4f2fc02917 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 20 Jan 2025 12:12:44 +0000 Subject: [PATCH 1384/2091] add index-compression and auto-readahead to descriptor Signed-off-by: Jason Volk --- src/database/engine/cf_opts.rs | 16 +++++++++++----- src/database/engine/descriptor.rs | 12 ++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index da636718..1230081c 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -83,11 +83,17 @@ fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache table.disable_cache(); } - opts.set_options_from_string( - "{{block_based_table_factory={num_file_reads_for_auto_readahead=0;\ - max_auto_readahead_size=524288;initial_auto_readahead_size=16384}}}", - ) - .map_err(map_err)?; + let string = format!( + "{{block_based_table_factory={{num_file_reads_for_auto_readahead={0};\ + max_auto_readahead_size={1};initial_auto_readahead_size={2};\ + enable_index_compression={3}}}}}", + desc.auto_readahead_thresh, + desc.auto_readahead_max, + desc.auto_readahead_init, + desc.compressed_index, + ); + + opts.set_options_from_string(&string).map_err(map_err)?; opts.set_block_based_table_factory(&table); diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index 2c84ac53..6ce8b5ad 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -34,11 +34,15 @@ pub(crate) struct Descriptor { pub(crate) compaction: CompactionStyle, pub(crate) compaction_pri: CompactionPri, pub(crate) compression: CompressionType, + pub(crate) compressed_index: bool, pub(crate) compression_shape: [i32; 7], pub(crate) compression_level: i32, pub(crate) bottommost_level: Option, pub(crate) block_index_hashing: Option, pub(crate) cache_shards: u32, + pub(crate) auto_readahead_thresh: u32, + pub(crate) auto_readahead_init: usize, + pub(crate) auto_readahead_max: usize, } pub(crate) static BASE: Descriptor = Descriptor { @@ -61,11 +65,15 @@ pub(crate) static BASE: Descriptor = Descriptor { compaction: CompactionStyle::Level, compaction_pri: CompactionPri::MinOverlappingRatio, compression: CompressionType::Zstd, + compressed_index: true, compression_shape: [0, 0, 0, 1, 1, 1, 1], compression_level: SENTINEL_COMPRESSION_LEVEL, bottommost_level: Some(SENTINEL_COMPRESSION_LEVEL), block_index_hashing: None, cache_shards: 64, + auto_readahead_thresh: 0, + auto_readahead_init: 1024 * 16, + auto_readahead_max: 1024 * 1024 * 2, }; pub(crate) static RANDOM: Descriptor = Descriptor { @@ -74,6 +82,7 @@ pub(crate) static RANDOM: Descriptor = Descriptor { cache_shards: 128, compression_level: -3, bottommost_level: Some(4), + compressed_index: true, ..BASE }; @@ -86,6 +95,7 @@ pub(crate) static SEQUENTIAL: Descriptor = Descriptor { compression_level: -1, bottommost_level: Some(6), compression_shape: [0, 0, 1, 1, 1, 1, 1], + compressed_index: false, ..BASE }; @@ -100,6 +110,7 @@ pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { compression_level: -4, bottommost_level: Some(1), compression_shape: [0, 0, 0, 0, 0, 1, 1], + compressed_index: false, ..RANDOM }; @@ -114,5 +125,6 @@ pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { compression_level: -2, bottommost_level: Some(4), compression_shape: [0, 0, 0, 0, 1, 1, 1], + compressed_index: false, ..SEQUENTIAL }; From 8ab825b12c08324977898b26e9513b197750b9bb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 21 Jan 2025 20:43:38 +0000 Subject: [PATCH 1385/2091] add stream parallelism extension combinators Signed-off-by: Jason Volk --- src/core/utils/stream/mod.rs | 2 + src/core/utils/stream/try_broadband.rs | 6 +-- src/core/utils/stream/try_parallel.rs | 71 ++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 src/core/utils/stream/try_parallel.rs diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index c7bfa021..23455322 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -7,6 +7,7 @@ mod iter_stream; mod ready; mod tools; mod try_broadband; +mod try_parallel; mod try_ready; mod try_tools; mod try_wideband; @@ -24,6 +25,7 @@ pub use iter_stream::IterStream; pub use ready::ReadyExt; pub use tools::Tools; pub use try_broadband::TryBroadbandExt; +pub use try_parallel::TryParallelExt; pub use try_ready::TryReadyExt; pub use try_tools::TryTools; pub use try_wideband::TryWidebandExt; diff --git a/src/core/utils/stream/try_broadband.rs b/src/core/utils/stream/try_broadband.rs index c72fcc2c..361b4a92 100644 --- a/src/core/utils/stream/try_broadband.rs +++ b/src/core/utils/stream/try_broadband.rs @@ -18,7 +18,7 @@ where ) -> impl TryStream> + Send where N: Into>, - F: Fn(Self::Ok) -> Fut + Send + Sync, + F: Fn(Self::Ok) -> Fut + Send, Fut: TryFuture> + Send; fn broad_and_then( @@ -26,7 +26,7 @@ where f: F, ) -> impl TryStream> + Send where - F: Fn(Self::Ok) -> Fut + Send + Sync, + F: Fn(Self::Ok) -> Fut + Send, Fut: TryFuture> + Send, { self.broadn_and_then(None, f) @@ -44,7 +44,7 @@ where ) -> impl TryStream> + Send where N: Into>, - F: Fn(Self::Ok) -> Fut + Send + Sync, + F: Fn(Self::Ok) -> Fut + Send, Fut: TryFuture> + Send, { self.map_ok(f) diff --git a/src/core/utils/stream/try_parallel.rs b/src/core/utils/stream/try_parallel.rs new file mode 100644 index 00000000..7f8a63b1 --- /dev/null +++ b/src/core/utils/stream/try_parallel.rs @@ -0,0 +1,71 @@ +//! Parallelism stream combinator extensions to futures::Stream + +use futures::{stream::TryStream, TryFutureExt}; +use tokio::{runtime, task::JoinError}; + +use super::TryBroadbandExt; +use crate::{utils::sys::available_parallelism, Error, Result}; + +/// Parallelism extensions to augment futures::StreamExt. These combinators are +/// for computation-oriented workloads, unlike -band combinators for I/O +/// workloads; these default to the available compute parallelism for the +/// system. Threads are currently drawn from the tokio-spawn pool. Results are +/// unordered. +pub trait TryParallelExt +where + Self: TryStream> + Send + Sized, + E: From + From + Send + 'static, + T: Send + 'static, +{ + fn paralleln_and_then( + self, + h: H, + n: N, + f: F, + ) -> impl TryStream> + Send + where + N: Into>, + H: Into>, + F: Fn(Self::Ok) -> Result + Clone + Send + 'static, + U: Send + 'static; + + fn parallel_and_then( + self, + h: H, + f: F, + ) -> impl TryStream> + Send + where + H: Into>, + F: Fn(Self::Ok) -> Result + Clone + Send + 'static, + U: Send + 'static, + { + self.paralleln_and_then(h, None, f) + } +} + +impl TryParallelExt for S +where + S: TryStream> + Send + Sized, + E: From + From + Send + 'static, + T: Send + 'static, +{ + fn paralleln_and_then( + self, + h: H, + n: N, + f: F, + ) -> impl TryStream> + Send + where + N: Into>, + H: Into>, + F: Fn(Self::Ok) -> Result + Clone + Send + 'static, + U: Send + 'static, + { + let n = n.into().unwrap_or_else(available_parallelism); + let h = h.into().unwrap_or_else(runtime::Handle::current); + self.broadn_and_then(n, move |val| { + let (h, f) = (h.clone(), f.clone()); + async move { h.spawn_blocking(move || f(val)).map_err(E::from).await? } + }) + } +} From dda27ffcb1a6d9f1ff6dafebb6203cb9cb8c2f22 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 12:05:07 +0000 Subject: [PATCH 1386/2091] add some compaction related interfaces Signed-off-by: Jason Volk --- Cargo.toml | 1 + src/admin/mod.rs | 1 + src/admin/query/raw.rs | 104 ++++++++++++++++++++++++++++++++++-- src/database/engine.rs | 23 ++++++-- src/database/map.rs | 1 + src/database/map/compact.rs | 62 +++++++++++++++++++++ src/database/mod.rs | 4 +- 7 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 src/database/map/compact.rs diff --git a/Cargo.toml b/Cargo.toml index 4d738a11..f9e3b6db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -875,6 +875,7 @@ enum_glob_use = { level = "allow", priority = 1 } if_not_else = { level = "allow", priority = 1 } if_then_some_else_none = { level = "allow", priority = 1 } inline_always = { level = "allow", priority = 1 } +match_bool = { level = "allow", priority = 1 } missing_docs_in_private_items = { level = "allow", priority = 1 } missing_errors_doc = { level = "allow", priority = 1 } missing_panics_doc = { level = "allow", priority = 1 } diff --git a/src/admin/mod.rs b/src/admin/mod.rs index ac51104a..695155e8 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -1,6 +1,7 @@ #![recursion_limit = "192"] #![allow(clippy::wildcard_imports)] #![allow(clippy::enum_glob_use)] +#![allow(clippy::too_many_arguments)] pub(crate) mod admin; pub(crate) mod command; diff --git a/src/admin/query/raw.rs b/src/admin/query/raw.rs index 678d21c9..ac5e8976 100644 --- a/src/admin/query/raw.rs +++ b/src/admin/query/raw.rs @@ -2,13 +2,13 @@ use std::{borrow::Cow, collections::BTreeMap, ops::Deref}; use clap::Subcommand; use conduwuit::{ - apply, at, + apply, at, is_zero, utils::{ - stream::{ReadyExt, TryIgnore}, + stream::{ReadyExt, TryIgnore, TryParallelExt}, string::EMPTY, IterStream, }, - Result, + Err, Result, }; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::events::room::message::RoomMessageEventContent; @@ -121,6 +121,104 @@ pub(crate) enum RawCommand { /// Key prefix prefix: Option, }, + + /// - Compact database + Compact { + #[arg(short, long, alias("column"))] + map: Option>, + + #[arg(long)] + start: Option, + + #[arg(long)] + stop: Option, + + #[arg(long)] + from: Option, + + #[arg(long)] + into: Option, + + /// There is one compaction job per column; then this controls how many + /// columns are compacted in parallel. If zero, one compaction job is + /// still run at a time here, but in exclusive-mode blocking any other + /// automatic compaction jobs until complete. + #[arg(long)] + parallelism: Option, + + #[arg(long, default_value("false"))] + exhaustive: bool, + }, +} + +#[admin_command] +pub(super) async fn compact( + &self, + map: Option>, + start: Option, + stop: Option, + from: Option, + into: Option, + parallelism: Option, + exhaustive: bool, +) -> Result { + use conduwuit_database::compact::Options; + + let default_all_maps = map + .is_none() + .then(|| { + self.services + .db + .keys() + .map(Deref::deref) + .map(ToOwned::to_owned) + }) + .into_iter() + .flatten(); + + let maps: Vec<_> = map + .unwrap_or_default() + .into_iter() + .chain(default_all_maps) + .map(|map| self.services.db.get(&map)) + .filter_map(Result::ok) + .cloned() + .collect(); + + if maps.is_empty() { + return Err!("--map argument invalid. not found in database"); + } + + let range = ( + start.as_ref().map(String::as_bytes).map(Into::into), + stop.as_ref().map(String::as_bytes).map(Into::into), + ); + + let options = Options { + range, + level: (from, into), + exclusive: parallelism.is_some_and(is_zero!()), + exhaustive, + }; + + let runtime = self.services.server.runtime().clone(); + let parallelism = parallelism.unwrap_or(1); + let results = maps + .into_iter() + .try_stream() + .paralleln_and_then(runtime, parallelism, move |map| { + map.compact_blocking(options.clone())?; + Ok(map.name().to_owned()) + }) + .collect::>(); + + let timer = Instant::now(); + let results = results.await; + let query_time = timer.elapsed(); + self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")) + .await?; + + Ok(RoomMessageEventContent::text_plain("")) } #[admin_command] diff --git a/src/database/engine.rs b/src/database/engine.rs index 2958f73f..8be9eecc 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -18,9 +18,16 @@ use std::{ }; use conduwuit::{debug, info, warn, Err, Result}; -use rocksdb::{AsColumnFamilyRef, BoundColumnFamily, DBCommon, DBWithThreadMode, MultiThreaded}; +use rocksdb::{ + AsColumnFamilyRef, BoundColumnFamily, DBCommon, DBWithThreadMode, MultiThreaded, + WaitForCompactOptions, +}; -use crate::{pool::Pool, result, Context}; +use crate::{ + pool::Pool, + util::{map_err, result}, + Context, +}; pub struct Engine { pub(super) read_only: bool, @@ -55,12 +62,22 @@ impl Engine { #[tracing::instrument(skip(self), level = "debug")] pub fn flush(&self) -> Result { result(DBCommon::flush_wal(&self.db, false)) } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(skip(self), level = "info")] pub fn sort(&self) -> Result { let flushoptions = rocksdb::FlushOptions::default(); result(DBCommon::flush_opt(&self.db, &flushoptions)) } + #[tracing::instrument(skip(self), level = "info")] + pub fn wait_compactions(&self) -> Result { + let mut opts = WaitForCompactOptions::default(); + opts.set_abort_on_pause(true); + opts.set_flush(false); + opts.set_timeout(0); + + self.db.wait_for_compact(&opts).map_err(map_err) + } + /// Query for database property by null-terminated name which is expected to /// have a result with an integer representation. This is intended for /// low-overhead programmatic use. diff --git a/src/database/map.rs b/src/database/map.rs index 60d66585..33cae594 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -1,3 +1,4 @@ +pub mod compact; mod contains; mod count; mod get; diff --git a/src/database/map/compact.rs b/src/database/map/compact.rs new file mode 100644 index 00000000..c0381eb4 --- /dev/null +++ b/src/database/map/compact.rs @@ -0,0 +1,62 @@ +use conduwuit::{implement, Err, Result}; +use rocksdb::{BottommostLevelCompaction, CompactOptions}; + +use crate::keyval::KeyBuf; + +#[derive(Clone, Debug, Default)] +pub struct Options { + /// Key range to start and stop compaction. + pub range: (Option, Option), + + /// (None, None) - all levels to all necessary levels + /// (None, Some(1)) - compact all levels into level 1 + /// (Some(1), None) - compact level 1 into level 1 + /// (Some(_), Some(_) - currently unsupported + pub level: (Option, Option), + + /// run compaction until complete. if false only one pass is made, and the + /// results of that pass are not further recompacted. + pub exhaustive: bool, + + /// waits for other compactions to complete, then runs this compaction + /// exclusively before allowing automatic compactions to resume. + pub exclusive: bool, +} + +#[implement(super::Map)] +#[tracing::instrument( + name = "compact", + level = "info" + skip(self), + fields(%self), +)] +pub fn compact_blocking(&self, opts: Options) -> Result { + let mut co = CompactOptions::default(); + co.set_exclusive_manual_compaction(opts.exclusive); + co.set_bottommost_level_compaction(match opts.exhaustive { + | true => BottommostLevelCompaction::Force, + | false => BottommostLevelCompaction::ForceOptimized, + }); + + match opts.level { + | (None, None) => { + co.set_change_level(true); + co.set_target_level(-1); + }, + | (None, Some(level)) => { + co.set_change_level(true); + co.set_target_level(level.try_into()?); + }, + | (Some(level), None) => { + co.set_change_level(false); + co.set_target_level(level.try_into()?); + }, + | (Some(_), Some(_)) => return Err!("compacting between specific levels not supported"), + }; + + self.db + .db + .compact_range_cf_opt(&self.cf(), opts.range.0, opts.range.1, &co); + + Ok(()) +} diff --git a/src/database/mod.rs b/src/database/mod.rs index 6e3f8c96..8ae8dcf5 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -30,12 +30,12 @@ pub use self::{ deserialized::Deserialized, handle::Handle, keyval::{serialize_key, serialize_val, KeyVal, Slice}, - map::Map, + map::{compact, Map}, ser::{serialize, serialize_to, serialize_to_vec, Interfix, Json, Separator, SEP}, }; pub(crate) use self::{ engine::{context::Context, Engine}, - util::{or_else, result}, + util::or_else, }; use crate::maps::{Maps, MapsKey, MapsVal}; From 9ab381e4ebf8b2953c3cd697185c79e14c0ae309 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 18 Jan 2025 07:12:20 +0000 Subject: [PATCH 1387/2091] generate fmt::Display for Config Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 4 +- src/core/config/mod.rs | 401 ----------------------------------- src/macros/config.rs | 93 +++++--- 3 files changed, 63 insertions(+), 435 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 8d3358a8..6469a0e9 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -22,8 +22,8 @@ pub(super) async fn uptime(&self) -> Result { pub(super) async fn show_config(&self) -> Result { // Construct and send the response Ok(RoomMessageEventContent::text_markdown(format!( - "```\n{}\n```", - self.services.globals.config + "{}", + self.services.server.config ))) } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 5cfed0b9..d6983540 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -3,7 +3,6 @@ pub mod proxy; use std::{ collections::{BTreeMap, BTreeSet, HashSet}, - fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, path::PathBuf, }; @@ -15,7 +14,6 @@ use either::{ }; use figment::providers::{Env, Format, Toml}; pub use figment::{value::Value as FigmentValue, Figment}; -use itertools::Itertools; use regex::RegexSet; use ruma::{ api::client::discovery::discover_support::ContactRole, OwnedRoomOrAliasId, OwnedServerName, @@ -1859,405 +1857,6 @@ impl Config { pub fn check(&self) -> Result<(), Error> { check(self) } } -impl fmt::Display for Config { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - writeln!(f, "Active config values:\n").expect("wrote line to formatter stream"); - let mut line = |key: &str, val: &str| { - writeln!(f, "{key}: {val}").expect("wrote line to formatter stream"); - }; - - line("Server name", self.server_name.host()); - line("Database path", &self.database_path.to_string_lossy()); - line( - "Database backup path", - self.database_backup_path - .as_ref() - .map_or("", |path| path.to_str().unwrap_or("")), - ); - line("Database backups to keep", &self.database_backups_to_keep.to_string()); - line("Database cache capacity (MB)", &self.db_cache_capacity_mb.to_string()); - line("Cache capacity modifier", &self.cache_capacity_modifier.to_string()); - line("PDU cache capacity", &self.pdu_cache_capacity.to_string()); - line("Auth chain cache capacity", &self.auth_chain_cache_capacity.to_string()); - line("Short eventid cache capacity", &self.shorteventid_cache_capacity.to_string()); - line("Eventid short cache capacity", &self.eventidshort_cache_capacity.to_string()); - line("Short statekey cache capacity", &self.shortstatekey_cache_capacity.to_string()); - line("Statekey short cache capacity", &self.statekeyshort_cache_capacity.to_string()); - line( - "Server visibility cache capacity", - &self.server_visibility_cache_capacity.to_string(), - ); - line( - "User visibility cache capacity", - &self.user_visibility_cache_capacity.to_string(), - ); - line("Stateinfo cache capacity", &self.stateinfo_cache_capacity.to_string()); - line( - "Roomid space hierarchy cache capacity", - &self.roomid_spacehierarchy_cache_capacity.to_string(), - ); - line("DNS cache entry limit", &self.dns_cache_entries.to_string()); - line("DNS minimum TTL", &self.dns_min_ttl.to_string()); - line("DNS minimum NXDOMAIN TTL", &self.dns_min_ttl_nxdomain.to_string()); - line("DNS attempts", &self.dns_attempts.to_string()); - line("DNS timeout", &self.dns_timeout.to_string()); - line("DNS fallback to TCP", &self.dns_tcp_fallback.to_string()); - line("DNS query over TCP only", &self.query_over_tcp_only.to_string()); - line("Query all nameservers", &self.query_all_nameservers.to_string()); - line("Maximum request size (bytes)", &self.max_request_size.to_string()); - line("Sender retry backoff limit", &self.sender_retry_backoff_limit.to_string()); - line("Request connect timeout", &self.request_conn_timeout.to_string()); - line("Request timeout", &self.request_timeout.to_string()); - line("Request total timeout", &self.request_total_timeout.to_string()); - line("Idle connections per host", &self.request_idle_per_host.to_string()); - line("Request pool idle timeout", &self.request_idle_timeout.to_string()); - line("Well_known connect timeout", &self.well_known_conn_timeout.to_string()); - line("Well_known timeout", &self.well_known_timeout.to_string()); - line("Federation timeout", &self.federation_timeout.to_string()); - line("Federation pool idle per host", &self.federation_idle_per_host.to_string()); - line("Federation pool idle timeout", &self.federation_idle_timeout.to_string()); - line("Sender timeout", &self.sender_timeout.to_string()); - line("Sender pool idle timeout", &self.sender_idle_timeout.to_string()); - line("Appservice timeout", &self.appservice_timeout.to_string()); - line("Appservice pool idle timeout", &self.appservice_idle_timeout.to_string()); - line("Pusher pool idle timeout", &self.pusher_idle_timeout.to_string()); - line("Allow registration", &self.allow_registration.to_string()); - line( - "Registration token", - if self.registration_token.is_none() - && self.registration_token_file.is_none() - && self.allow_registration - { - "not set (⚠️ open registration!)" - } else if self.registration_token.is_none() && self.registration_token_file.is_none() - { - "not set" - } else { - "set" - }, - ); - line( - "Registration token file path", - self.registration_token_file - .as_ref() - .map_or("", |path| path.to_str().unwrap_or_default()), - ); - line( - "Allow guest registration (inherently false if allow registration is false)", - &self.allow_guest_registration.to_string(), - ); - line( - "Log guest registrations in admin room", - &self.log_guest_registrations.to_string(), - ); - line( - "Allow guests to auto join rooms", - &self.allow_guests_auto_join_rooms.to_string(), - ); - line("New user display name suffix", &self.new_user_displayname_suffix); - line("Allow encryption", &self.allow_encryption.to_string()); - line("Allow federation", &self.allow_federation.to_string()); - line("Federation loopback", &self.federation_loopback.to_string()); - line( - "Require authentication for profile requests", - &self.require_auth_for_profile_requests.to_string(), - ); - line( - "Allow incoming federated presence requests (updates)", - &self.allow_incoming_presence.to_string(), - ); - line( - "Allow outgoing federated presence requests (updates)", - &self.allow_outgoing_presence.to_string(), - ); - line( - "Allow local presence requests (updates)", - &self.allow_local_presence.to_string(), - ); - line( - "Allow incoming remote read receipts", - &self.allow_incoming_read_receipts.to_string(), - ); - line( - "Allow outgoing remote read receipts", - &self.allow_outgoing_read_receipts.to_string(), - ); - line( - "Block non-admin room invites (local and remote, admins can still send and receive \ - invites)", - &self.block_non_admin_invites.to_string(), - ); - line("Enable admin escape commands", &self.admin_escape_commands.to_string()); - line( - "Activate admin console after startup", - &self.admin_console_automatic.to_string(), - ); - line("Execute admin commands after startup", &self.admin_execute.join(", ")); - line( - "Continue startup even if some commands fail", - &self.admin_execute_errors_ignore.to_string(), - ); - line("Filter for admin command log capture", &self.admin_log_capture); - line("Admin room tag", &self.admin_room_tag); - line("Allow outgoing federated typing", &self.allow_outgoing_typing.to_string()); - line("Allow incoming federated typing", &self.allow_incoming_typing.to_string()); - line( - "Incoming federated typing timeout", - &self.typing_federation_timeout_s.to_string(), - ); - line("Client typing timeout minimum", &self.typing_client_timeout_min_s.to_string()); - line("Client typing timeout maxmimum", &self.typing_client_timeout_max_s.to_string()); - line("Allow device name federation", &self.allow_device_name_federation.to_string()); - line( - "Allow incoming profile lookup federation requests", - &self - .allow_inbound_profile_lookup_federation_requests - .to_string(), - ); - line( - "Auto deactivate banned room join attempts", - &self.auto_deactivate_banned_room_attempts.to_string(), - ); - line("Notification push path", &self.notification_push_path); - line("Allow room creation", &self.allow_room_creation.to_string()); - line( - "Allow public room directory over federation", - &self.allow_public_room_directory_over_federation.to_string(), - ); - line( - "Allow public room directory without authentication", - &self.allow_public_room_directory_without_auth.to_string(), - ); - line( - "Lockdown public room directory (only allow admins to publish)", - &self.lockdown_public_room_directory.to_string(), - ); - line( - "Trusted key servers", - &self - .trusted_servers - .iter() - .map(|server| server.host()) - .join(", "), - ); - line("OpenID Token TTL", &self.openid_token_ttl.to_string()); - line( - "TURN username", - if self.turn_username.is_empty() { - "not set" - } else { - &self.turn_username - }, - ); - line("TURN password", { - if self.turn_password.is_empty() { - "not set" - } else { - "set" - } - }); - line("TURN secret", { - if self.turn_secret.is_empty() && self.turn_secret_file.is_none() { - "not set" - } else { - "set" - } - }); - line("TURN secret file path", { - self.turn_secret_file - .as_ref() - .map_or("", |path| path.to_str().unwrap_or_default()) - }); - line("Turn TTL", &self.turn_ttl.to_string()); - line("Turn URIs", { - let mut lst = Vec::with_capacity(self.turn_uris.len()); - for item in self.turn_uris.iter().cloned().enumerate() { - let (_, uri): (usize, String) = item; - lst.push(uri); - } - &lst.join(", ") - }); - line("Auto Join Rooms", { - let mut lst = Vec::with_capacity(self.auto_join_rooms.len()); - for room in &self.auto_join_rooms { - lst.push(room); - } - &lst.into_iter().join(", ") - }); - line("Zstd HTTP Compression", &self.zstd_compression.to_string()); - line("Gzip HTTP Compression", &self.gzip_compression.to_string()); - line("Brotli HTTP Compression", &self.brotli_compression.to_string()); - line("RocksDB database LOG level", &self.rocksdb_log_level); - line("RocksDB database LOG to stderr", &self.rocksdb_log_stderr.to_string()); - line("RocksDB database LOG time-to-roll", &self.rocksdb_log_time_to_roll.to_string()); - line("RocksDB Max LOG Files", &self.rocksdb_max_log_files.to_string()); - line( - "RocksDB database max LOG file size", - &self.rocksdb_max_log_file_size.to_string(), - ); - line( - "RocksDB database optimize for spinning disks", - &self.rocksdb_optimize_for_spinning_disks.to_string(), - ); - line("RocksDB Direct-IO", &self.rocksdb_direct_io.to_string()); - line("RocksDB Parallelism Threads", &self.rocksdb_parallelism_threads.to_string()); - line("RocksDB Compression Algorithm", &self.rocksdb_compression_algo); - line("RocksDB Compression Level", &self.rocksdb_compression_level.to_string()); - line( - "RocksDB Bottommost Compression Level", - &self.rocksdb_bottommost_compression_level.to_string(), - ); - line( - "RocksDB Bottommost Level Compression", - &self.rocksdb_bottommost_compression.to_string(), - ); - line("RocksDB Recovery Mode", &self.rocksdb_recovery_mode.to_string()); - line("RocksDB Repair Mode", &self.rocksdb_repair.to_string()); - line("RocksDB Read-only Mode", &self.rocksdb_read_only.to_string()); - line("RocksDB Secondary Mode", &self.rocksdb_secondary.to_string()); - line( - "RocksDB Compaction Idle Priority", - &self.rocksdb_compaction_prio_idle.to_string(), - ); - line( - "RocksDB Compaction Idle IOPriority", - &self.rocksdb_compaction_ioprio_idle.to_string(), - ); - line("RocksDB Compaction enabled", &self.rocksdb_compaction.to_string()); - line("RocksDB Statistics level", &self.rocksdb_stats_level.to_string()); - line("Media integrity checks on startup", &self.media_startup_check.to_string()); - line("Media compatibility filesystem links", &self.media_compat_file_link.to_string()); - line("Prune missing media from database", &self.prune_missing_media.to_string()); - line("Allow legacy (unauthenticated) media", &self.allow_legacy_media.to_string()); - line("Freeze legacy (unauthenticated) media", &self.freeze_legacy_media.to_string()); - line("Prevent Media Downloads From", { - let mut lst = Vec::with_capacity(self.prevent_media_downloads_from.len()); - for domain in &self.prevent_media_downloads_from { - lst.push(domain.host()); - } - &lst.join(", ") - }); - line("Forbidden Remote Server Names (\"Global\" ACLs)", { - let mut lst = Vec::with_capacity(self.forbidden_remote_server_names.len()); - for domain in &self.forbidden_remote_server_names { - lst.push(domain.host()); - } - &lst.join(", ") - }); - line("Forbidden Remote Room Directory Server Names", { - let mut lst = - Vec::with_capacity(self.forbidden_remote_room_directory_server_names.len()); - for domain in &self.forbidden_remote_room_directory_server_names { - lst.push(domain.host()); - } - &lst.join(", ") - }); - line("Outbound Request IP Range (CIDR) Denylist", { - let mut lst = Vec::with_capacity(self.ip_range_denylist.len()); - for item in self.ip_range_denylist.iter().cloned().enumerate() { - let (_, ip): (usize, String) = item; - lst.push(ip); - } - &lst.join(", ") - }); - line("Forbidden usernames", { - &self.forbidden_usernames.patterns().iter().join(", ") - }); - line("Forbidden room aliases", { - &self.forbidden_alias_names.patterns().iter().join(", ") - }); - line( - "URL preview bound interface", - self.url_preview_bound_interface - .as_ref() - .map(Either::as_ref) - .map(|either| either.map_left(ToString::to_string)) - .map(Either::either_into::) - .unwrap_or_default() - .as_str(), - ); - line( - "URL preview domain contains allowlist", - &self.url_preview_domain_contains_allowlist.join(", "), - ); - line( - "URL preview domain explicit allowlist", - &self.url_preview_domain_explicit_allowlist.join(", "), - ); - line( - "URL preview domain explicit denylist", - &self.url_preview_domain_explicit_denylist.join(", "), - ); - line( - "URL preview URL contains allowlist", - &self.url_preview_url_contains_allowlist.join(", "), - ); - line("URL preview maximum spider size", &self.url_preview_max_spider_size.to_string()); - line("URL preview check root domain", &self.url_preview_check_root_domain.to_string()); - line( - "Allow check for updates / announcements check", - &self.allow_check_for_updates.to_string(), - ); - line("Enable netburst on startup", &self.startup_netburst.to_string()); - #[cfg(feature = "sentry_telemetry")] - line("Sentry.io reporting and tracing", &self.sentry.to_string()); - #[cfg(feature = "sentry_telemetry")] - line("Sentry.io send server_name in logs", &self.sentry_send_server_name.to_string()); - #[cfg(feature = "sentry_telemetry")] - line("Sentry.io tracing sample rate", &self.sentry_traces_sample_rate.to_string()); - line("Sentry.io attach stacktrace", &self.sentry_attach_stacktrace.to_string()); - line("Sentry.io send panics", &self.sentry_send_panic.to_string()); - line("Sentry.io send errors", &self.sentry_send_error.to_string()); - line("Sentry.io tracing filter", &self.sentry_filter); - line( - "Well-known server name", - self.well_known - .server - .as_ref() - .map_or("", |server| server.as_str()), - ); - line( - "Well-known client URL", - self.well_known - .client - .as_ref() - .map_or("", |url| url.as_str()), - ); - line( - "Well-known support email", - self.well_known - .support_email - .as_ref() - .map_or("", |str| str.as_ref()), - ); - line( - "Well-known support Matrix ID", - self.well_known - .support_mxid - .as_ref() - .map_or("", |mxid| mxid.as_str()), - ); - line( - "Well-known support role", - self.well_known - .support_role - .as_ref() - .map_or("", |role| role.as_str()), - ); - line( - "Well-known support page/URL", - self.well_known - .support_page - .as_ref() - .map_or("", |url| url.as_str()), - ); - line("Enable the tokio-console", &self.tokio_console.to_string()); - line("Admin room notices", &self.admin_room_notices.to_string()); - - Ok(()) - } -} - fn true_fn() -> bool { true } fn default_address() -> ListeningAddr { diff --git a/src/macros/config.rs b/src/macros/config.rs index 452abd20..90d6ef15 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -1,8 +1,8 @@ use std::{collections::HashSet, fmt::Write as _, fs::OpenOptions, io::Write as _}; use proc_macro::TokenStream; -use proc_macro2::Span; -use quote::ToTokens; +use proc_macro2::{Span, TokenStream as TokenStream2}; +use quote::{quote, ToTokens}; use syn::{ parse::Parser, punctuated::Punctuated, spanned::Spanned, Error, Expr, ExprLit, Field, Fields, FieldsNamed, ItemStruct, Lit, Meta, MetaList, MetaNameValue, Type, TypePath, @@ -19,18 +19,24 @@ const HIDDEN: &[&str] = &["default"]; #[allow(clippy::needless_pass_by_value)] pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result { - if is_cargo_build() && !is_cargo_test() { - generate_example(&input, args)?; - } + let write = is_cargo_build() && !is_cargo_test(); + let additional = generate_example(&input, args, write)?; - Ok(input.to_token_stream().into()) + Ok([input.to_token_stream(), additional] + .into_iter() + .collect::() + .into()) } #[allow(clippy::needless_pass_by_value)] #[allow(unused_variables)] -fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { +fn generate_example(input: &ItemStruct, args: &[Meta], write: bool) -> Result { let settings = get_simple_settings(args); + let section = settings.get("section").ok_or_else(|| { + Error::new(args[0].span(), "missing required 'section' attribute argument") + })?; + let filename = settings.get("filename").ok_or_else(|| { Error::new(args[0].span(), "missing required 'filename' attribute argument") })?; @@ -45,31 +51,33 @@ fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { .split(' ') .collect(); - let section = settings.get("section").ok_or_else(|| { - Error::new(args[0].span(), "missing required 'section' attribute argument") - })?; - - let mut file = OpenOptions::new() + let fopts = OpenOptions::new() .write(true) .create(section == "global") .truncate(section == "global") .append(section != "global") - .open(filename) - .map_err(|e| { - Error::new( - Span::call_site(), - format!("Failed to open config file for generation: {e}"), - ) - })?; + .clone(); - if let Some(header) = settings.get("header") { - file.write_all(header.as_bytes()) + let mut file = write + .then(|| { + fopts.open(filename).map_err(|e| { + let msg = format!("Failed to open file for config generation: {e}"); + Error::new(Span::call_site(), msg) + }) + }) + .transpose()?; + + if let Some(file) = file.as_mut() { + if let Some(header) = settings.get("header") { + file.write_all(header.as_bytes()) + .expect("written to config file"); + } + + file.write_fmt(format_args!("\n[{section}]\n")) .expect("written to config file"); } - file.write_fmt(format_args!("\n[{section}]\n")) - .expect("written to config file"); - + let mut summary: Vec = Vec::new(); if let Fields::Named(FieldsNamed { named, .. }) = &input.fields { for field in named { let Some(ident) = &field.ident else { @@ -105,20 +113,41 @@ fn generate_example(input: &ItemStruct, args: &[Meta]) -> Result<()> { default }; - file.write_fmt(format_args!("\n{doc}")) - .expect("written to config file"); + if let Some(file) = file.as_mut() { + file.write_fmt(format_args!("\n{doc}")) + .expect("written to config file"); - file.write_fmt(format_args!("#{ident} ={default}\n")) + file.write_fmt(format_args!("#{ident} ={default}\n")) + .expect("written to config file"); + } + + let name = ident.to_string(); + summary.push(quote! { + writeln!(out, "| {} | {:?} |", #name, self.#ident)?; + }); + } + } + + if let Some(file) = file.as_mut() { + if let Some(footer) = settings.get("footer") { + file.write_all(footer.as_bytes()) .expect("written to config file"); } } - if let Some(footer) = settings.get("footer") { - file.write_all(footer.as_bytes()) - .expect("written to config file"); - } + let struct_name = &input.ident; + let display = quote! { + impl std::fmt::Display for #struct_name { + fn fmt(&self, out: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(out, "| name | value |")?; + writeln!(out, "| :--- | :--- |")?; + #( #summary )* + Ok(()) + } + } + }; - Ok(()) + Ok(display) } fn get_default(field: &Field) -> Option { From 1f31e74024bcdc23efcbafc32c0d9572df83fb82 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 22 Jan 2025 03:50:51 +0000 Subject: [PATCH 1388/2091] add del to raw suite Signed-off-by: Jason Volk --- src/admin/query/raw.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/admin/query/raw.rs b/src/admin/query/raw.rs index ac5e8976..5a6006ec 100644 --- a/src/admin/query/raw.rs +++ b/src/admin/query/raw.rs @@ -33,6 +33,15 @@ pub(crate) enum RawCommand { key: String, }, + /// - Raw database delete (for string keys) + RawDel { + /// Map name + map: String, + + /// Key + key: String, + }, + /// - Raw database keys iteration RawKeys { /// Map name @@ -534,6 +543,18 @@ pub(super) async fn raw_iter_from( ))) } +#[admin_command] +pub(super) async fn raw_del(&self, map: String, key: String) -> Result { + let map = self.services.db.get(&map)?; + let timer = Instant::now(); + map.remove(&key); + let query_time = timer.elapsed(); + + Ok(RoomMessageEventContent::notice_markdown(format!( + "Operation completed in {query_time:?}" + ))) +} + #[admin_command] pub(super) async fn raw_get(&self, map: String, key: String) -> Result { let map = self.services.db.get(&map)?; From 0c96891008713b1e121f1896fdba59f94570cc29 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 22 Jan 2025 00:52:48 +0000 Subject: [PATCH 1389/2091] add CBOR support to database schema Signed-off-by: Jason Volk --- Cargo.lock | 32 ++++++++++++++++++++++++++++++++ Cargo.toml | 8 ++++++++ src/database/Cargo.toml | 2 ++ src/database/de.rs | 4 ++++ src/database/mod.rs | 2 +- src/database/ser.rs | 14 +++++++++++++- 6 files changed, 60 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8de3abf4..d9758e6f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -761,6 +761,8 @@ dependencies = [ "const-str", "futures", "log", + "minicbor", + "minicbor-serde", "rust-rocksdb-uwu", "serde", "serde_json", @@ -2329,6 +2331,36 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minicbor" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0452a60c1863c1f50b5f77cd295e8d2786849f35883f0b9e18e7e6e1b5691b0" +dependencies = [ + "minicbor-derive", +] + +[[package]] +name = "minicbor-derive" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + +[[package]] +name = "minicbor-serde" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "becf18ac384ecf6f53b2db3b1549eebff664c67ecf259ae99be5912193291686" +dependencies = [ + "minicbor", + "serde", +] + [[package]] name = "minimad" version = "0.13.1" diff --git a/Cargo.toml b/Cargo.toml index f9e3b6db..042587fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -506,6 +506,14 @@ version = "0.2" [workspace.dependencies.num-traits] version = "0.2" +[workspace.dependencies.minicbor] +version = "0.25.1" +features = ["std"] + +[workspace.dependencies.minicbor-serde] +version = "0.3.2" +features = ["std"] + # # Patches # diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 09eedaf4..557c9a3e 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -40,6 +40,8 @@ conduwuit-core.workspace = true const-str.workspace = true futures.workspace = true log.workspace = true +minicbor.workspace = true +minicbor-serde.workspace = true rust-rocksdb.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/src/database/de.rs b/src/database/de.rs index 48bc9f64..4fdc2251 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -248,6 +248,10 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { { match name { | "$serde_json::private::RawValue" => visitor.visit_map(self), + | "Cbor" => visitor + .visit_newtype_struct(&mut minicbor_serde::Deserializer::new(self.record_trail())) + .map_err(|e| Self::Error::SerdeDe(e.to_string().into())), + | _ => visitor.visit_newtype_struct(self), } } diff --git a/src/database/mod.rs b/src/database/mod.rs index 8ae8dcf5..42b7f5e3 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -31,7 +31,7 @@ pub use self::{ handle::Handle, keyval::{serialize_key, serialize_val, KeyVal, Slice}, map::{compact, Map}, - ser::{serialize, serialize_to, serialize_to_vec, Interfix, Json, Separator, SEP}, + ser::{serialize, serialize_to, serialize_to_vec, Cbor, Interfix, Json, Separator, SEP}, }; pub(crate) use self::{ engine::{context::Context, Engine}, diff --git a/src/database/ser.rs b/src/database/ser.rs index e6de5f7f..372b7522 100644 --- a/src/database/ser.rs +++ b/src/database/ser.rs @@ -1,7 +1,7 @@ use std::io::Write; use conduwuit::{debug::type_name, err, result::DebugInspect, utils::exchange, Error, Result}; -use serde::{ser, Serialize}; +use serde::{ser, Deserialize, Serialize}; use crate::util::unhandled; @@ -55,6 +55,10 @@ pub(crate) struct Serializer<'a, W: Write> { #[derive(Debug, Serialize)] pub struct Json(pub T); +/// Newtype for CBOR serialization. +#[derive(Debug, Deserialize, Serialize)] +pub struct Cbor(pub T); + /// Directive to force separator serialization specifically for prefix keying /// use. This is a quirk of the database schema and prefix iterations. #[derive(Debug, Serialize)] @@ -189,6 +193,14 @@ impl ser::Serializer for &mut Serializer<'_, W> { match name { | "Json" => serde_json::to_writer(&mut self.out, value).map_err(Into::into), + | "Cbor" => { + use minicbor::encode::write::Writer; + use minicbor_serde::Serializer; + + value + .serialize(&mut Serializer::new(&mut Writer::new(&mut self.out))) + .map_err(|e| Self::Error::SerdeSer(e.to_string().into())) + }, | _ => unhandled!("Unrecognized serialization Newtype {name:?}"), } } From 49023aa295da8b4d975389a1494611696d4cc63d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 22 Jan 2025 02:16:51 +0000 Subject: [PATCH 1390/2091] use database for resolver caches Signed-off-by: Jason Volk --- src/admin/query/resolver.rs | 59 +++++--------- src/database/maps.rs | 8 ++ src/service/resolver/actual.rs | 6 +- src/service/resolver/cache.rs | 135 ++++++++++++++++----------------- src/service/resolver/dns.rs | 24 +++--- src/service/resolver/fed.rs | 3 +- src/service/resolver/mod.rs | 39 +--------- src/service/sending/send.rs | 2 +- 8 files changed, 114 insertions(+), 162 deletions(-) diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index b53661fc..0b6da6fd 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -1,7 +1,6 @@ -use std::fmt::Write; - use clap::Subcommand; use conduwuit::{utils::time, Result}; +use futures::StreamExt; use ruma::{events::room::message::RoomMessageEventContent, OwnedServerName}; use crate::{admin_command, admin_command_dispatch}; @@ -31,29 +30,19 @@ async fn destinations_cache( writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?; writeln!(self, "| ----------- | ----------- | -------- | ------- |").await?; - let mut out = String::new(); - { - let map = self - .services - .resolver - .cache - .destinations - .read() - .expect("locked"); + let mut destinations = self.services.resolver.cache.destinations().boxed(); - for (name, &CachedDest { ref dest, ref host, expire }) in map.iter() { - if let Some(server_name) = server_name.as_ref() { - if name != server_name { - continue; - } + while let Some((name, CachedDest { dest, host, expire })) = destinations.next().await { + if let Some(server_name) = server_name.as_ref() { + if name != server_name { + continue; } - - let expire = time::format(expire, "%+"); - writeln!(out, "| {name} | {dest} | {host} | {expire} |")?; } - } - self.write_str(out.as_str()).await?; + let expire = time::format(expire, "%+"); + self.write_str(&format!("| {name} | {dest} | {host} | {expire} |\n")) + .await?; + } Ok(RoomMessageEventContent::notice_plain("")) } @@ -65,29 +54,19 @@ async fn overrides_cache(&self, server_name: Option) -> Result Result { - let (result, cached) = if let Some(result) = self.get_cached_destination(server_name) { + let (result, cached) = if let Ok(result) = self.cache.get_destination(server_name).await { (result, true) } else { self.validate_dest(server_name)?; @@ -232,7 +232,7 @@ impl super::Service { #[tracing::instrument(skip_all, name = "well-known")] async fn request_well_known(&self, dest: &str) -> Result> { - if !self.has_cached_override(dest) { + if !self.cache.has_override(dest).await { self.query_and_cache_override(dest, dest, 8448).await?; } @@ -315,7 +315,7 @@ impl super::Service { debug_info!("{overname:?} overriden by {hostname:?}"); } - self.set_cached_override(overname, CachedOverride { + self.cache.set_override(overname, CachedOverride { ips: override_ip.into_iter().take(MAX_IPS).collect(), port, expire: CachedOverride::default_expire(), diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index e309a129..11e6c9bd 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -1,108 +1,103 @@ -use std::{ - collections::HashMap, - net::IpAddr, - sync::{Arc, RwLock}, - time::SystemTime, -}; +use std::{net::IpAddr, sync::Arc, time::SystemTime}; use arrayvec::ArrayVec; use conduwuit::{ - trace, - utils::{math::Expected, rand}, + at, implement, + utils::{math::Expected, rand, stream::TryIgnore}, + Result, }; -use ruma::{OwnedServerName, ServerName}; +use database::{Cbor, Deserialized, Map}; +use futures::{Stream, StreamExt}; +use ruma::ServerName; +use serde::{Deserialize, Serialize}; use super::fed::FedDest; pub struct Cache { - pub destinations: RwLock, // actual_destination, host - pub overrides: RwLock, + destinations: Arc, + overrides: Arc, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct CachedDest { pub dest: FedDest, pub host: String, pub expire: SystemTime, } -#[derive(Clone, Debug)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct CachedOverride { pub ips: IpAddrs, pub port: u16, pub expire: SystemTime, } -pub type WellKnownMap = HashMap; -pub type TlsNameMap = HashMap; - pub type IpAddrs = ArrayVec; pub(crate) const MAX_IPS: usize = 3; impl Cache { - pub(super) fn new() -> Arc { + pub(super) fn new(args: &crate::Args<'_>) -> Arc { Arc::new(Self { - destinations: RwLock::new(WellKnownMap::new()), - overrides: RwLock::new(TlsNameMap::new()), + destinations: args.db["servername_destination"].clone(), + overrides: args.db["servername_override"].clone(), }) } } -impl super::Service { - pub fn set_cached_destination( - &self, - name: OwnedServerName, - dest: CachedDest, - ) -> Option { - trace!(?name, ?dest, "set cached destination"); - self.cache - .destinations - .write() - .expect("locked for writing") - .insert(name, dest) - } +#[implement(Cache)] +pub fn set_destination(&self, name: &ServerName, dest: CachedDest) { + self.destinations.raw_put(name, Cbor(dest)); +} - #[must_use] - pub fn get_cached_destination(&self, name: &ServerName) -> Option { - self.cache - .destinations - .read() - .expect("locked for reading") - .get(name) - .cloned() - } +#[implement(Cache)] +pub fn set_override(&self, name: &str, over: CachedOverride) { + self.overrides.raw_put(name, Cbor(over)); +} - pub fn set_cached_override( - &self, - name: &str, - over: CachedOverride, - ) -> Option { - trace!(?name, ?over, "set cached override"); - self.cache - .overrides - .write() - .expect("locked for writing") - .insert(name.into(), over) - } +#[implement(Cache)] +pub async fn get_destination(&self, name: &ServerName) -> Result { + self.destinations + .get(name) + .await + .deserialized::>() + .map(at!(0)) +} - #[must_use] - pub fn get_cached_override(&self, name: &str) -> Option { - self.cache - .overrides - .read() - .expect("locked for reading") - .get(name) - .cloned() - } +#[implement(Cache)] +pub async fn get_override(&self, name: &str) -> Result { + self.overrides + .get(name) + .await + .deserialized::>() + .map(at!(0)) +} - #[must_use] - pub fn has_cached_override(&self, name: &str) -> bool { - self.cache - .overrides - .read() - .expect("locked for reading") - .contains_key(name) - } +#[implement(Cache)] +#[must_use] +pub async fn has_destination(&self, destination: &str) -> bool { + self.destinations.exists(destination).await.is_ok() +} + +#[implement(Cache)] +#[must_use] +pub async fn has_override(&self, destination: &str) -> bool { + self.overrides.exists(destination).await.is_ok() +} + +#[implement(Cache)] +pub fn destinations(&self) -> impl Stream + Send + '_ { + self.destinations + .stream() + .ignore_err() + .map(|item: (&ServerName, Cbor<_>)| (item.0, item.1 .0)) +} + +#[implement(Cache)] +pub fn overrides(&self) -> impl Stream + Send + '_ { + self.overrides + .stream() + .ignore_err() + .map(|item: (&ServerName, Cbor<_>)| (item.0, item.1 .0)) } impl CachedDest { diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index 5c9018ab..ad7768bc 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -88,18 +88,20 @@ impl Resolve for Resolver { impl Resolve for Hooked { fn resolve(&self, name: Name) -> Resolving { - let cached: Option = self - .cache - .overrides - .read() - .expect("locked for reading") - .get(name.as_str()) - .cloned(); + hooked_resolve(self.cache.clone(), self.server.clone(), self.resolver.clone(), name) + .boxed() + } +} - cached.map_or_else( - || resolve_to_reqwest(self.server.clone(), self.resolver.clone(), name).boxed(), - |cached| cached_to_reqwest(cached).boxed(), - ) +async fn hooked_resolve( + cache: Arc, + server: Arc, + resolver: Arc, + name: Name, +) -> Result> { + match cache.get_override(name.as_str()).await { + | Ok(cached) => cached_to_reqwest(cached).await, + | Err(_) => resolve_to_reqwest(server, resolver, name).boxed().await, } } diff --git a/src/service/resolver/fed.rs b/src/service/resolver/fed.rs index 76fc6894..bfe100e7 100644 --- a/src/service/resolver/fed.rs +++ b/src/service/resolver/fed.rs @@ -6,8 +6,9 @@ use std::{ use arrayvec::ArrayString; use conduwuit::utils::math::Expected; +use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, PartialEq, Eq)] +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] pub enum FedDest { Literal(SocketAddr), Named(String, PortString), diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 6a6289b6..3163b0d0 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -4,9 +4,9 @@ mod dns; pub mod fed; mod tests; -use std::{fmt::Write, sync::Arc}; +use std::sync::Arc; -use conduwuit::{utils, utils::math::Expected, Result, Server}; +use conduwuit::{Result, Server}; use self::{cache::Cache, dns::Resolver}; use crate::{client, Dep}; @@ -25,7 +25,7 @@ struct Services { impl crate::Service for Service { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] fn build(args: crate::Args<'_>) -> Result> { - let cache = Cache::new(); + let cache = Cache::new(&args); Ok(Arc::new(Self { cache: cache.clone(), resolver: Resolver::build(args.server, cache)?, @@ -36,38 +36,5 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result { - use utils::bytes::pretty; - - let (oc_count, oc_bytes) = self.cache.overrides.read()?.iter().fold( - (0_usize, 0_usize), - |(count, bytes), (key, val)| { - (count.expected_add(1), bytes.expected_add(key.len()).expected_add(val.size())) - }, - ); - - let (dc_count, dc_bytes) = self.cache.destinations.read()?.iter().fold( - (0_usize, 0_usize), - |(count, bytes), (key, val)| { - (count.expected_add(1), bytes.expected_add(key.len()).expected_add(val.size())) - }, - ); - - writeln!(out, "resolver_overrides_cache: {oc_count} ({})", pretty(oc_bytes))?; - writeln!(out, "resolver_destinations_cache: {dc_count} ({})", pretty(dc_bytes))?; - - Ok(()) - } - - fn clear_cache(&self) { - self.cache.overrides.write().expect("write locked").clear(); - self.cache - .destinations - .write() - .expect("write locked") - .clear(); - self.resolver.resolver.clear_cache(); - } - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index e2981068..831a1dd8 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -125,7 +125,7 @@ where let result = T::IncomingResponse::try_from_http_response(response); if result.is_ok() && !actual.cached { - resolver.set_cached_destination(dest.to_owned(), CachedDest { + resolver.cache.set_destination(dest, CachedDest { dest: actual.dest.clone(), host: actual.host.clone(), expire: CachedDest::default_expire(), From 7c0c029a4a90ded986d76910c162377f73360b64 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 22 Jan 2025 06:40:07 +0000 Subject: [PATCH 1391/2091] add try_lock to MutexMap; allow TryFrom constructions Signed-off-by: Jason Volk --- Cargo.lock | 26 ++++++++-------- Cargo.toml | 2 +- src/admin/debug/commands.rs | 2 +- src/core/utils/mutex_map.rs | 60 ++++++++++++++++++++++++++++++++++--- 4 files changed, 71 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9758e6f..7985a411 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3188,7 +3188,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "assign", "js_int", @@ -3210,7 +3210,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "js_int", "ruma-common", @@ -3222,7 +3222,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "as_variant", "assign", @@ -3245,7 +3245,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "as_variant", "base64 0.22.1", @@ -3276,7 +3276,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3301,7 +3301,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "bytes", "http", @@ -3319,7 +3319,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3328,7 +3328,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "js_int", "ruma-common", @@ -3338,7 +3338,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3353,7 +3353,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "js_int", "ruma-common", @@ -3365,7 +3365,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "headers", "http", @@ -3378,7 +3378,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3394,7 +3394,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=c4f55b39900b33b2d443dd12a6a2dab50961fdfb#c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 042587fc..b8c145ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -332,7 +332,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "c4f55b39900b33b2d443dd12a6a2dab50961fdfb" +rev = "427877d5bc14988ed877e500bbb27f8bc08b84e8" features = [ "compat", "rand", diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index a77587b0..49078dde 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -725,7 +725,7 @@ pub(super) async fn force_set_room_state_from_server( .save_state(room_id.clone().as_ref(), new_room_state) .await?; - let state_lock = self.services.rooms.state.mutex.lock(&room_id).await; + let state_lock = self.services.rooms.state.mutex.lock(&*room_id).await; self.services .rooms .state diff --git a/src/core/utils/mutex_map.rs b/src/core/utils/mutex_map.rs index 9b9821fe..03a4adf1 100644 --- a/src/core/utils/mutex_map.rs +++ b/src/core/utils/mutex_map.rs @@ -1,7 +1,13 @@ -use std::{fmt::Debug, hash::Hash, sync::Arc}; +use std::{ + fmt::Debug, + hash::Hash, + sync::{Arc, TryLockError::WouldBlock}, +}; use tokio::sync::OwnedMutexGuard as Omg; +use crate::{err, Result}; + /// Map of Mutexes pub struct MutexMap { map: Map, @@ -30,16 +36,17 @@ where } #[tracing::instrument(level = "trace", skip(self))] - pub async fn lock(&self, k: &K) -> Guard + pub async fn lock<'a, K>(&'a self, k: &'a K) -> Guard where K: Debug + Send + ?Sized + Sync, - Key: for<'a> From<&'a K>, + Key: TryFrom<&'a K>, + >::Error: Debug, { let val = self .map .lock() .expect("locked") - .entry(k.into()) + .entry(k.try_into().expect("failed to construct key")) .or_default() .clone(); @@ -49,6 +56,51 @@ where } } + #[tracing::instrument(level = "trace", skip(self))] + pub fn try_lock<'a, K>(&self, k: &'a K) -> Result> + where + K: Debug + Send + ?Sized + Sync, + Key: TryFrom<&'a K>, + >::Error: Debug, + { + let val = self + .map + .lock() + .expect("locked") + .entry(k.try_into().expect("failed to construct key")) + .or_default() + .clone(); + + Ok(Guard:: { + map: Arc::clone(&self.map), + val: val.try_lock_owned().map_err(|_| err!("would yield"))?, + }) + } + + #[tracing::instrument(level = "trace", skip(self))] + pub fn try_try_lock<'a, K>(&self, k: &'a K) -> Result> + where + K: Debug + Send + ?Sized + Sync, + Key: TryFrom<&'a K>, + >::Error: Debug, + { + let val = self + .map + .try_lock() + .map_err(|e| match e { + | WouldBlock => err!("would block"), + | _ => panic!("{e:?}"), + })? + .entry(k.try_into().expect("failed to construct key")) + .or_default() + .clone(); + + Ok(Guard:: { + map: Arc::clone(&self.map), + val: val.try_lock_owned().map_err(|_| err!("would yield"))?, + }) + } + #[must_use] pub fn contains(&self, k: &Key) -> bool { self.map.lock().expect("locked").contains_key(k) } From f75d9fa79e1ef1bfbd7454cfa470acf1910d6a99 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 22 Jan 2025 06:57:18 +0000 Subject: [PATCH 1392/2091] deduplicate name resolutions Signed-off-by: Jason Volk --- src/service/resolver/actual.rs | 55 +++++++++++++++++++++++----------- src/service/resolver/mod.rs | 8 ++++- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 5676d7b1..afe5a1e5 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -4,7 +4,7 @@ use std::{ }; use conduwuit::{debug, debug_error, debug_info, debug_warn, err, error, trace, Err, Result}; -use futures::FutureExt; +use futures::{FutureExt, TryFutureExt}; use hickory_resolver::error::ResolveError; use ipaddress::IPAddress; use ruma::ServerName; @@ -29,18 +29,31 @@ impl ActualDest { impl super::Service { #[tracing::instrument(skip_all, level = "debug", name = "resolve")] pub(crate) async fn get_actual_dest(&self, server_name: &ServerName) -> Result { - let (result, cached) = if let Ok(result) = self.cache.get_destination(server_name).await { - (result, true) - } else { - self.validate_dest(server_name)?; - (self.resolve_actual_dest(server_name, true).boxed().await?, false) - }; - - let CachedDest { dest, host, .. } = result; + let (CachedDest { dest, host, .. }, cached) = + self.lookup_actual_dest(server_name).await?; Ok(ActualDest { dest, host, cached }) } + pub(crate) async fn lookup_actual_dest( + &self, + server_name: &ServerName, + ) -> Result<(CachedDest, bool)> { + if let Ok(result) = self.cache.get_destination(server_name).await { + return Ok((result, true)); + } + + let _dedup = self.resolving.lock(server_name.as_str()); + if let Ok(result) = self.cache.get_destination(server_name).await { + return Ok((result, true)); + } + + self.resolve_actual_dest(server_name, true) + .map_ok(|result| (result, false)) + .boxed() + .await + } + /// Returns: `actual_destination`, host header /// Implemented according to the specification at /// Numbers in comments below refer to bullet points in linked section of @@ -51,7 +64,7 @@ impl super::Service { dest: &ServerName, cache: bool, ) -> Result { - trace!("Finding actual destination for {dest}"); + self.validate_dest(dest)?; let mut host = dest.as_str().to_owned(); let actual_dest = match get_ip_with_port(dest.as_str()) { | Some(host_port) => Self::actual_dest_1(host_port)?, @@ -106,6 +119,7 @@ impl super::Service { cache, ) .await?; + Ok(FedDest::Named( host.to_owned(), port.try_into().unwrap_or_else(|_| FedDest::default_port()), @@ -156,6 +170,7 @@ impl super::Service { cache, ) .await?; + Ok(FedDest::Named( host.to_owned(), port.try_into().unwrap_or_else(|_| FedDest::default_port()), @@ -177,17 +192,18 @@ impl super::Service { cache, ) .await?; + if let Some(port) = force_port { - Ok(FedDest::Named( + return Ok(FedDest::Named( delegated, format!(":{port}") .as_str() .try_into() .unwrap_or_else(|_| FedDest::default_port()), - )) - } else { - Ok(add_port_to_hostname(&delegated)) + )); } + + Ok(add_port_to_hostname(&delegated)) } async fn actual_dest_3_4(&self, cache: bool, delegated: String) -> Result { @@ -212,21 +228,24 @@ impl super::Service { cache, ) .await?; + if let Some(port) = force_port { let port = format!(":{port}"); - Ok(FedDest::Named( + + return Ok(FedDest::Named( host.to_owned(), PortString::from(port.as_str()).unwrap_or_else(|_| FedDest::default_port()), - )) - } else { - Ok(add_port_to_hostname(host)) + )); } + + Ok(add_port_to_hostname(host)) } async fn actual_dest_5(&self, dest: &ServerName, cache: bool) -> Result { debug!("5: No SRV record found"); self.conditional_query_and_cache_override(dest.as_str(), dest.as_str(), 8448, cache) .await?; + Ok(add_port_to_hostname(dest.as_str())) } diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 3163b0d0..090e562d 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -6,7 +6,8 @@ mod tests; use std::sync::Arc; -use conduwuit::{Result, Server}; +use arrayvec::ArrayString; +use conduwuit::{utils::MutexMap, Result, Server}; use self::{cache::Cache, dns::Resolver}; use crate::{client, Dep}; @@ -14,6 +15,7 @@ use crate::{client, Dep}; pub struct Service { pub cache: Arc, pub resolver: Arc, + resolving: Resolving, services: Services, } @@ -22,6 +24,9 @@ struct Services { client: Dep, } +type Resolving = MutexMap; +type NameBuf = ArrayString<256>; + impl crate::Service for Service { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] fn build(args: crate::Args<'_>) -> Result> { @@ -29,6 +34,7 @@ impl crate::Service for Service { Ok(Arc::new(Self { cache: cache.clone(), resolver: Resolver::build(args.server, cache)?, + resolving: MutexMap::new(), services: Services { server: args.server.clone(), client: args.depend::("client"), From 607e338ac2bdb03b9e08cfe207bc7253aa8a8a2e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 22 Jan 2025 07:56:24 +0000 Subject: [PATCH 1393/2091] cache result of resolution at completion of resolution Signed-off-by: Jason Volk --- src/service/resolver/actual.rs | 8 ++++---- src/service/resolver/cache.rs | 4 ++-- src/service/sending/send.rs | 29 ++++------------------------- 3 files changed, 10 insertions(+), 31 deletions(-) diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index afe5a1e5..1a36936d 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -18,7 +18,6 @@ use super::{ pub(crate) struct ActualDest { pub(crate) dest: FedDest, pub(crate) host: String, - pub(crate) cached: bool, } impl ActualDest { @@ -29,10 +28,10 @@ impl ActualDest { impl super::Service { #[tracing::instrument(skip_all, level = "debug", name = "resolve")] pub(crate) async fn get_actual_dest(&self, server_name: &ServerName) -> Result { - let (CachedDest { dest, host, .. }, cached) = + let (CachedDest { dest, host, .. }, _cached) = self.lookup_actual_dest(server_name).await?; - Ok(ActualDest { dest, host, cached }) + Ok(ActualDest { dest, host }) } pub(crate) async fn lookup_actual_dest( @@ -49,6 +48,7 @@ impl super::Service { } self.resolve_actual_dest(server_name, true) + .inspect_ok(|result| self.cache.set_destination(server_name, result)) .map_ok(|result| (result, false)) .boxed() .await @@ -334,7 +334,7 @@ impl super::Service { debug_info!("{overname:?} overriden by {hostname:?}"); } - self.cache.set_override(overname, CachedOverride { + self.cache.set_override(overname, &CachedOverride { ips: override_ip.into_iter().take(MAX_IPS).collect(), port, expire: CachedOverride::default_expire(), diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index 11e6c9bd..657718b3 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -45,12 +45,12 @@ impl Cache { } #[implement(Cache)] -pub fn set_destination(&self, name: &ServerName, dest: CachedDest) { +pub fn set_destination(&self, name: &ServerName, dest: &CachedDest) { self.destinations.raw_put(name, Cbor(dest)); } #[implement(Cache)] -pub fn set_override(&self, name: &str, over: CachedOverride) { +pub fn set_override(&self, name: &str, over: &CachedOverride) { self.overrides.raw_put(name, Cbor(over)); } diff --git a/src/service/sending/send.rs b/src/service/sending/send.rs index 831a1dd8..c8a64f3c 100644 --- a/src/service/sending/send.rs +++ b/src/service/sending/send.rs @@ -18,10 +18,7 @@ use ruma::{ CanonicalJsonObject, CanonicalJsonValue, ServerName, ServerSigningKeyId, }; -use crate::{ - resolver, - resolver::{actual::ActualDest, cache::CachedDest}, -}; +use crate::resolver::actual::ActualDest; impl super::Service { #[tracing::instrument( @@ -73,16 +70,7 @@ impl super::Service { debug!(?method, ?url, "Sending request"); match client.execute(request).await { - | Ok(response) => - handle_response::( - &self.services.resolver, - dest, - actual, - &method, - &url, - response, - ) - .await, + | Ok(response) => handle_response::(dest, actual, &method, &url, response).await, | Err(error) => Err(handle_error(actual, &method, &url, error).expect_err("always returns error")), } @@ -111,7 +99,6 @@ impl super::Service { } async fn handle_response( - resolver: &resolver::Service, dest: &ServerName, actual: &ActualDest, method: &Method, @@ -122,17 +109,9 @@ where T: OutgoingRequest + Send, { let response = into_http_response(dest, actual, method, url, response).await?; - let result = T::IncomingResponse::try_from_http_response(response); - if result.is_ok() && !actual.cached { - resolver.cache.set_destination(dest, CachedDest { - dest: actual.dest.clone(), - host: actual.host.clone(), - expire: CachedDest::default_expire(), - }); - } - - result.map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}"))) + T::IncomingResponse::try_from_http_response(response) + .map_err(|e| err!(BadServerResponse("Server returned bad 200 response: {e:?}"))) } async fn into_http_response( From da9f1ae5d7daf68d8e8568f07d38cbf8c065634a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 22 Jan 2025 08:51:15 +0000 Subject: [PATCH 1394/2091] expire resolver cache entries Signed-off-by: Jason Volk --- src/core/utils/rand.rs | 2 +- src/service/resolver/cache.rs | 44 ++++++++++++++++++----------------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/core/utils/rand.rs b/src/core/utils/rand.rs index 9e6fc7a8..1d289c6e 100644 --- a/src/core/utils/rand.rs +++ b/src/core/utils/rand.rs @@ -33,7 +33,7 @@ pub fn string_array() -> ArrayString { #[inline] #[must_use] -pub fn timepoint_secs(range: Range) -> SystemTime { +pub fn time_from_now_secs(range: Range) -> SystemTime { SystemTime::now() .checked_add(secs(range)) .expect("range does not overflow SystemTime") diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index 657718b3..e64878d4 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -2,7 +2,7 @@ use std::{net::IpAddr, sync::Arc, time::SystemTime}; use arrayvec::ArrayVec; use conduwuit::{ - at, implement, + at, err, implement, utils::{math::Expected, rand, stream::TryIgnore}, Result, }; @@ -54,6 +54,18 @@ pub fn set_override(&self, name: &str, over: &CachedOverride) { self.overrides.raw_put(name, Cbor(over)); } +#[implement(Cache)] +#[must_use] +pub async fn has_destination(&self, destination: &ServerName) -> bool { + self.get_destination(destination).await.is_ok() +} + +#[implement(Cache)] +#[must_use] +pub async fn has_override(&self, destination: &str) -> bool { + self.get_override(destination).await.is_ok() +} + #[implement(Cache)] pub async fn get_destination(&self, name: &ServerName) -> Result { self.destinations @@ -61,6 +73,9 @@ pub async fn get_destination(&self, name: &ServerName) -> Result { .await .deserialized::>() .map(at!(0)) + .into_iter() + .find(CachedDest::valid) + .ok_or(err!(Request(NotFound("Expired from cache")))) } #[implement(Cache)] @@ -70,18 +85,9 @@ pub async fn get_override(&self, name: &str) -> Result { .await .deserialized::>() .map(at!(0)) -} - -#[implement(Cache)] -#[must_use] -pub async fn has_destination(&self, destination: &str) -> bool { - self.destinations.exists(destination).await.is_ok() -} - -#[implement(Cache)] -#[must_use] -pub async fn has_override(&self, destination: &str) -> bool { - self.overrides.exists(destination).await.is_ok() + .into_iter() + .find(CachedOverride::valid) + .ok_or(err!(Request(NotFound("Expired from cache")))) } #[implement(Cache)] @@ -103,13 +109,11 @@ pub fn overrides(&self) -> impl Stream + S impl CachedDest { #[inline] #[must_use] - pub fn valid(&self) -> bool { true } - - //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + pub fn valid(&self) -> bool { self.expire > SystemTime::now() } #[must_use] pub(crate) fn default_expire() -> SystemTime { - rand::timepoint_secs(60 * 60 * 18..60 * 60 * 36) + rand::time_from_now_secs(60 * 60 * 18..60 * 60 * 36) } #[inline] @@ -125,13 +129,11 @@ impl CachedDest { impl CachedOverride { #[inline] #[must_use] - pub fn valid(&self) -> bool { true } - - //pub fn valid(&self) -> bool { self.expire > SystemTime::now() } + pub fn valid(&self) -> bool { self.expire > SystemTime::now() } #[must_use] pub(crate) fn default_expire() -> SystemTime { - rand::timepoint_secs(60 * 60 * 6..60 * 60 * 12) + rand::time_from_now_secs(60 * 60 * 6..60 * 60 * 12) } #[inline] From 265802d54608eb10295560f54f26bd106e4930e9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 22 Jan 2025 21:38:15 +0000 Subject: [PATCH 1395/2091] additional prof/stats jemalloc related interface tweaks Signed-off-by: Jason Volk --- src/core/alloc/je.rs | 61 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 6 deletions(-) diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 119ff45e..81fbd3ea 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -26,10 +26,11 @@ metadata_thp:always\ ,background_thread:true\ ,max_background_threads:-1\ ,lg_extent_max_active_fit:4\ -,oversize_threshold:33554432\ -,tcache_max:1048576\ +,oversize_threshold:16777216\ +,tcache_max:2097152\ ,dirty_decay_ms:16000\ ,muzzy_decay_ms:144000\ +,prof_active:false\ \0"; #[global_allocator] @@ -120,7 +121,7 @@ unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { } macro_rules! mallctl { - ($name:literal) => {{ + ($name:expr) => {{ thread_local! { static KEY: OnceCell = OnceCell::default(); }; @@ -135,6 +136,13 @@ macro_rules! mallctl { pub mod this_thread { use super::{is_nonzero, key, math, Debug, Key, OnceCell, Result}; + thread_local! { + static ALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() }; + static DEALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() }; + } + + pub fn idle() -> Result { super::notify(&mallctl!("thread.idle")) } + pub fn trim() -> Result { notify(mallctl!("arena.0.purge")) } pub fn decay() -> Result { notify(mallctl!("arena.0.decay")) } @@ -153,7 +161,7 @@ pub mod this_thread { pub fn get_dirty_decay() -> Result { get(mallctl!("arena.0.dirty_decay_ms")) } - pub fn enable_cache(enable: bool) -> Result { + pub fn cache_enable(enable: bool) -> Result { super::set::(&mallctl!("thread.tcache.enabled"), enable.into()).map(is_nonzero!()) } @@ -169,9 +177,29 @@ pub mod this_thread { super::get::(&mallctl!("thread.arena")).and_then(math::try_into) } - pub fn allocated() -> Result { super::get(&mallctl!("thread.allocated")) } + pub fn prof_enable(enable: bool) -> Result { + super::set::(&mallctl!("thread.prof.active"), enable.into()).map(is_nonzero!()) + } - pub fn deallocated() -> Result { super::get(&mallctl!("thread.deallocated")) } + pub fn is_prof_enabled() -> Result { + super::get::(&mallctl!("thread.prof.active")).map(is_nonzero!()) + } + + pub fn reset_peak() -> Result { super::notify(&mallctl!("thread.peak.reset")) } + + pub fn peak() -> Result { super::get(&mallctl!("thread.peak.read")) } + + #[inline] + #[must_use] + pub fn allocated() -> u64 { + *ALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.allocatedp")) + } + + #[inline] + #[must_use] + pub fn deallocated() -> u64 { + *DEALLOCATED_BYTES.with(|once| init_tls_cell(once, "thread.deallocatedp")) + } fn notify(key: Key) -> Result { super::notify_by_arena(Some(arena_id()?), key) } @@ -188,6 +216,27 @@ pub mod this_thread { { super::get_by_arena(Some(arena_id()?), key) } + + fn init_tls_cell(cell: &OnceCell<&'static u64>, name: &str) -> &'static u64 { + cell.get_or_init(|| { + let ptr: *const u64 = super::get(&mallctl!(name)).expect("failed to obtain pointer"); + + // SAFETY: ptr points directly to the internal state of jemalloc for this thread + unsafe { ptr.as_ref() }.expect("pointer must not be null") + }) + } +} + +pub fn stats_reset() -> Result { notify(&mallctl!("stats.mutexes.reset")) } + +pub fn prof_reset() -> Result { notify(&mallctl!("prof.reset")) } + +pub fn prof_enable(enable: bool) -> Result { + set::(&mallctl!("prof.active"), enable.into()).map(is_nonzero!()) +} + +pub fn is_prof_enabled() -> Result { + get::(&mallctl!("prof.active")).map(is_nonzero!()) } pub fn trim>>(arena: I) -> Result { From a5520e8b1bc1c4ddb9090dc9b93ef76899e58d9a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 22 Jan 2025 23:07:13 +0000 Subject: [PATCH 1396/2091] fix SRV override loss on cache expiration Signed-off-by: Jason Volk --- src/admin/query/resolver.rs | 10 +++--- src/service/resolver/actual.rs | 62 ++++++++++++++++++---------------- src/service/resolver/cache.rs | 9 ++--- src/service/resolver/dns.rs | 22 ++++++++++-- 4 files changed, 63 insertions(+), 40 deletions(-) diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index 0b6da6fd..08b5d171 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -51,12 +51,14 @@ async fn destinations_cache( async fn overrides_cache(&self, server_name: Option) -> Result { use service::resolver::cache::CachedOverride; - writeln!(self, "| Server Name | IP | Port | Expires |").await?; - writeln!(self, "| ----------- | --- | ----:| ------- |").await?; + writeln!(self, "| Server Name | IP | Port | Expires | Overriding |").await?; + writeln!(self, "| ----------- | --- | ----:| ------- | ---------- |").await?; let mut overrides = self.services.resolver.cache.overrides().boxed(); - while let Some((name, CachedOverride { ips, port, expire })) = overrides.next().await { + while let Some((name, CachedOverride { ips, port, expire, overriding })) = + overrides.next().await + { if let Some(server_name) = server_name.as_ref() { if name != server_name { continue; @@ -64,7 +66,7 @@ async fn overrides_cache(&self, server_name: Option) -> Result Result { debug!("2: Hostname with included port"); let (host, port) = dest.as_str().split_at(pos); - self.conditional_query_and_cache_override( - host, - host, - port.parse::().unwrap_or(8448), - cache, - ) - .await?; + self.conditional_query_and_cache(host, port.parse::().unwrap_or(8448), cache) + .await?; Ok(FedDest::Named( host.to_owned(), @@ -163,13 +158,8 @@ impl super::Service { ) -> Result { debug!("3.2: Hostname with port in .well-known file"); let (host, port) = delegated.split_at(pos); - self.conditional_query_and_cache_override( - host, - host, - port.parse::().unwrap_or(8448), - cache, - ) - .await?; + self.conditional_query_and_cache(host, port.parse::().unwrap_or(8448), cache) + .await?; Ok(FedDest::Named( host.to_owned(), @@ -208,7 +198,7 @@ impl super::Service { async fn actual_dest_3_4(&self, cache: bool, delegated: String) -> Result { debug!("3.4: No SRV records, just use the hostname from .well-known"); - self.conditional_query_and_cache_override(&delegated, &delegated, 8448, cache) + self.conditional_query_and_cache(&delegated, 8448, cache) .await?; Ok(add_port_to_hostname(&delegated)) } @@ -243,7 +233,7 @@ impl super::Service { async fn actual_dest_5(&self, dest: &ServerName, cache: bool) -> Result { debug!("5: No SRV record found"); - self.conditional_query_and_cache_override(dest.as_str(), dest.as_str(), 8448, cache) + self.conditional_query_and_cache(dest.as_str(), 8448, cache) .await?; Ok(add_port_to_hostname(dest.as_str())) @@ -251,9 +241,7 @@ impl super::Service { #[tracing::instrument(skip_all, name = "well-known")] async fn request_well_known(&self, dest: &str) -> Result> { - if !self.cache.has_override(dest).await { - self.query_and_cache_override(dest, dest, 8448).await?; - } + self.conditional_query_and_cache(dest, 8448, true).await?; self.services.server.check_running()?; trace!("Requesting well known for {dest}"); @@ -301,6 +289,17 @@ impl super::Service { Ok(Some(m_server.to_owned())) } + #[inline] + async fn conditional_query_and_cache( + &self, + hostname: &str, + port: u16, + cache: bool, + ) -> Result { + self.conditional_query_and_cache_override(hostname, hostname, port, cache) + .await + } + #[inline] async fn conditional_query_and_cache_override( &self, @@ -308,13 +307,17 @@ impl super::Service { hostname: &str, port: u16, cache: bool, - ) -> Result<()> { - if cache { - self.query_and_cache_override(overname, hostname, port) - .await - } else { - Ok(()) + ) -> Result { + if !cache { + return Ok(()); } + + if self.cache.has_override(overname).await { + return Ok(()); + } + + self.query_and_cache_override(overname, hostname, port) + .await } #[tracing::instrument(skip(self, overname, port), name = "ip")] @@ -323,21 +326,20 @@ impl super::Service { overname: &'_ str, hostname: &'_ str, port: u16, - ) -> Result<()> { + ) -> Result { self.services.server.check_running()?; debug!("querying IP for {overname:?} ({hostname:?}:{port})"); match self.resolver.resolver.lookup_ip(hostname.to_owned()).await { | Err(e) => Self::handle_resolve_error(&e, hostname), | Ok(override_ip) => { - if hostname != overname { - debug_info!("{overname:?} overriden by {hostname:?}"); - } - self.cache.set_override(overname, &CachedOverride { ips: override_ip.into_iter().take(MAX_IPS).collect(), port, expire: CachedOverride::default_expire(), + overriding: (hostname != overname) + .then_some(hostname.into()) + .inspect(|_| debug_info!("{overname:?} overriden by {hostname:?}")), }); Ok(()) diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index e64878d4..22a92865 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -30,6 +30,7 @@ pub struct CachedOverride { pub ips: IpAddrs, pub port: u16, pub expire: SystemTime, + pub overriding: Option, } pub type IpAddrs = ArrayVec; @@ -63,7 +64,10 @@ pub async fn has_destination(&self, destination: &ServerName) -> bool { #[implement(Cache)] #[must_use] pub async fn has_override(&self, destination: &str) -> bool { - self.get_override(destination).await.is_ok() + self.get_override(destination) + .await + .iter() + .any(CachedOverride::valid) } #[implement(Cache)] @@ -85,9 +89,6 @@ pub async fn get_override(&self, name: &str) -> Result { .await .deserialized::>() .map(at!(0)) - .into_iter() - .find(CachedOverride::valid) - .ok_or(err!(Request(NotFound("Expired from cache")))) } #[implement(Cache)] diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index ad7768bc..ca6106e2 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -93,6 +93,11 @@ impl Resolve for Hooked { } } +#[tracing::instrument( + level = "debug", + skip_all, + fields(name = ?name.as_str()) +)] async fn hooked_resolve( cache: Arc, server: Arc, @@ -100,8 +105,21 @@ async fn hooked_resolve( name: Name, ) -> Result> { match cache.get_override(name.as_str()).await { - | Ok(cached) => cached_to_reqwest(cached).await, - | Err(_) => resolve_to_reqwest(server, resolver, name).boxed().await, + | Ok(cached) if cached.valid() => cached_to_reqwest(cached).await, + | Ok(CachedOverride { overriding, .. }) if overriding.is_some() => + resolve_to_reqwest( + server, + resolver, + overriding + .as_deref() + .map(str::parse) + .expect("overriding is set for this record") + .expect("overriding is a valid internet name"), + ) + .boxed() + .await, + + | _ => resolve_to_reqwest(server, resolver, name).boxed().await, } } From 52adae7553e896bb07aacce4224a7fe8ff1bc992 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 23 Jan 2025 20:05:20 +0000 Subject: [PATCH 1397/2091] add sequence method to db engine; improve engine interface/tracing Signed-off-by: Jason Volk --- src/database/engine.rs | 87 +++++++++++++++++++++++++++++------------- 1 file changed, 60 insertions(+), 27 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index 8be9eecc..76b2889b 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -41,12 +41,49 @@ pub struct Engine { pub(crate) type Db = DBWithThreadMode; impl Engine { - pub(crate) fn cf(&self, name: &str) -> Arc> { - self.db - .cf_handle(name) - .expect("column must be described prior to database open") + #[tracing::instrument( + level = "info", + skip_all, + fields( + sequence = ?self.current_sequence(), + ), + )] + pub fn wait_compactions_blocking(&self) -> Result { + let mut opts = WaitForCompactOptions::default(); + opts.set_abort_on_pause(true); + opts.set_flush(false); + opts.set_timeout(0); + + self.db.wait_for_compact(&opts).map_err(map_err) } + #[tracing::instrument( + level = "info", + skip_all, + fields( + sequence = ?self.current_sequence(), + ), + )] + pub fn sort(&self) -> Result { + let flushoptions = rocksdb::FlushOptions::default(); + result(DBCommon::flush_opt(&self.db, &flushoptions)) + } + + #[tracing::instrument( + level = "debug", + skip_all, + fields( + sequence = ?self.current_sequence(), + ), + )] + pub fn update(&self) -> Result { self.db.try_catch_up_with_primary().map_err(map_err) } + + #[tracing::instrument(level = "info", skip_all)] + pub fn sync(&self) -> Result { result(DBCommon::flush_wal(&self.db, true)) } + + #[tracing::instrument(level = "debug", skip_all)] + pub fn flush(&self) -> Result { result(DBCommon::flush_wal(&self.db, false)) } + #[inline] pub(crate) fn cork(&self) { self.corks.fetch_add(1, Ordering::Relaxed); } @@ -56,28 +93,6 @@ impl Engine { #[inline] pub fn corked(&self) -> bool { self.corks.load(Ordering::Relaxed) > 0 } - #[tracing::instrument(skip(self))] - pub fn sync(&self) -> Result { result(DBCommon::flush_wal(&self.db, true)) } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn flush(&self) -> Result { result(DBCommon::flush_wal(&self.db, false)) } - - #[tracing::instrument(skip(self), level = "info")] - pub fn sort(&self) -> Result { - let flushoptions = rocksdb::FlushOptions::default(); - result(DBCommon::flush_opt(&self.db, &flushoptions)) - } - - #[tracing::instrument(skip(self), level = "info")] - pub fn wait_compactions(&self) -> Result { - let mut opts = WaitForCompactOptions::default(); - opts.set_abort_on_pause(true); - opts.set_flush(false); - opts.set_timeout(0); - - self.db.wait_for_compact(&opts).map_err(map_err) - } - /// Query for database property by null-terminated name which is expected to /// have a result with an integer representation. This is intended for /// low-overhead programmatic use. @@ -96,6 +111,24 @@ impl Engine { .and_then(|val| val.map_or_else(|| Err!("Property {name:?} not found."), Ok)) } + pub(crate) fn cf(&self, name: &str) -> Arc> { + self.db + .cf_handle(name) + .expect("column must be described prior to database open") + } + + #[inline] + #[must_use] + #[tracing::instrument(name = "sequence", level = "debug", skip_all, fields(sequence))] + pub fn current_sequence(&self) -> u64 { + let sequence = self.db.latest_sequence_number(); + + #[cfg(debug_assertions)] + tracing::Span::current().record("sequence", sequence); + + sequence + } + #[inline] #[must_use] pub fn is_read_only(&self) -> bool { self.secondary || self.read_only } @@ -114,7 +147,7 @@ impl Drop for Engine { self.db.cancel_all_background_work(BLOCKING); info!( - sequence = %self.db.latest_sequence_number(), + sequence = %self.current_sequence(), "Closing database..." ); } From 6e7c73336c49bd43cdb143212f36fe82f749209a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 24 Jan 2025 06:12:52 +0000 Subject: [PATCH 1398/2091] move room version config check out of services.globals make available_room_versions() non-member associated Signed-off-by: Jason Volk --- src/api/client/capabilities.rs | 5 +++-- src/core/config/check.rs | 12 +++++++++++- src/core/info/room_version.rs | 10 +++++----- src/service/globals/mod.rs | 8 -------- 4 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 87cdb43d..7188aa23 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; +use conduwuit::{Result, Server}; use ruma::{ api::client::discovery::get_capabilities::{ self, Capabilities, GetLoginTokenCapability, RoomVersionStability, @@ -10,7 +11,7 @@ use ruma::{ }; use serde_json::json; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/client/v3/capabilities` /// @@ -21,7 +22,7 @@ pub(crate) async fn get_capabilities_route( _body: Ruma, ) -> Result { let available: BTreeMap = - services.server.available_room_versions().collect(); + Server::available_room_versions().collect(); let mut capabilities = Capabilities::default(); capabilities.room_versions = RoomVersionsCapability { diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 06ae5ebb..d7be54b1 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -4,7 +4,7 @@ use either::Either; use figment::Figment; use super::DEPRECATED_KEYS; -use crate::{debug, debug_info, debug_warn, error, warn, Config, Err, Result}; +use crate::{debug, debug_info, debug_warn, error, warn, Config, Err, Result, Server}; #[allow(clippy::cognitive_complexity)] pub fn check(config: &Config) -> Result<()> { @@ -233,6 +233,16 @@ pub fn check(config: &Config) -> Result<()> { } } + if !Server::available_room_versions() + .any(|(version, _)| version == config.default_room_version) + { + return Err!(Config( + "default_room_version", + "Room version {:?} is not available", + config.default_room_version + )); + } + Ok(()) } diff --git a/src/core/info/room_version.rs b/src/core/info/room_version.rs index 40f0cf0a..b33a8562 100644 --- a/src/core/info/room_version.rs +++ b/src/core/info/room_version.rs @@ -20,6 +20,8 @@ pub const STABLE_ROOM_VERSIONS: &[RoomVersionId] = &[ pub const UNSTABLE_ROOM_VERSIONS: &[RoomVersionId] = &[RoomVersionId::V2, RoomVersionId::V3, RoomVersionId::V4, RoomVersionId::V5]; +type RoomVersion = (RoomVersionId, RoomVersionStability); + impl crate::Server { #[inline] pub fn supported_room_version(&self, version: &RoomVersionId) -> bool { @@ -28,15 +30,13 @@ impl crate::Server { #[inline] pub fn supported_room_versions(&self) -> impl Iterator + '_ { - self.available_room_versions() + Self::available_room_versions() .filter(|(_, stability)| self.supported_stability(stability)) .map(at!(0)) } #[inline] - pub fn available_room_versions( - &self, - ) -> impl Iterator { + pub fn available_room_versions() -> impl Iterator { available_room_versions() } @@ -46,7 +46,7 @@ impl crate::Server { } } -pub fn available_room_versions() -> impl Iterator { +pub fn available_room_versions() -> impl Iterator { let unstable_room_versions = UNSTABLE_ROOM_VERSIONS .iter() .cloned() diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index f6ff2b09..fe84578a 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -72,14 +72,6 @@ impl crate::Service for Service { registration_token, }; - if !args - .server - .supported_room_version(&config.default_room_version) - { - error!(config=?s.config.default_room_version, fallback=?conduwuit::config::default_default_room_version(), "Room version in config isn't supported, falling back to default version"); - s.config.default_room_version = conduwuit::config::default_default_room_version(); - }; - Ok(Arc::new(s)) } From 1351d07735719525da6af3485afcc6039de67b8c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 24 Jan 2025 06:58:26 +0000 Subject: [PATCH 1399/2091] improve path argument to Config::load and constructions Signed-off-by: Jason Volk --- src/core/config/mod.rs | 17 ++++++++++------- src/main/server.rs | 15 +++++++++++---- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index d6983540..beaabe5d 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -4,7 +4,7 @@ pub mod proxy; use std::{ collections::{BTreeMap, BTreeSet, HashSet}, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, - path::PathBuf, + path::{Path, PathBuf}, }; use conduwuit_macros::config_example_generator; @@ -1797,14 +1797,17 @@ const DEPRECATED_KEYS: &[&str; 9] = &[ impl Config { /// Pre-initialize config - pub fn load(paths: Option<&[PathBuf]>) -> Result { - let paths_files = paths.into_iter().flatten().map(Toml::file); - + pub fn load<'a, I>(paths: I) -> Result + where + I: Iterator, + { let envs = [Env::var("CONDUIT_CONFIG"), Env::var("CONDUWUIT_CONFIG")]; - let envs_files = envs.into_iter().flatten().map(Toml::file); - let config = envs_files - .chain(paths_files) + let config = envs + .into_iter() + .flatten() + .map(Toml::file) + .chain(paths.map(Toml::file)) .fold(Figment::new(), |config, file| config.merge(file.nested())) .merge(Env::prefixed("CONDUIT_").global().split("__")) .merge(Env::prefixed("CONDUWUIT_").global().split("__")); diff --git a/src/main/server.rs b/src/main/server.rs index 359a029c..74859f2b 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{path::PathBuf, sync::Arc}; use conduwuit::{ config::Config, @@ -35,9 +35,16 @@ impl Server { ) -> Result, Error> { let _runtime_guard = runtime.map(runtime::Handle::enter); - let raw_config = Config::load(args.config.as_deref())?; - let raw_config = crate::clap::update(raw_config, args)?; - let config = Config::new(&raw_config)?; + let config_paths = args + .config + .as_deref() + .into_iter() + .flat_map(<[_]>::iter) + .map(PathBuf::as_path); + + let config = Config::load(config_paths) + .and_then(|raw| crate::clap::update(raw, args)) + .and_then(|raw| Config::new(&raw))?; #[cfg(feature = "sentry_telemetry")] let sentry_guard = crate::sentry::init(&config); From 7c6b8b132aea086fc95c5a6def4af14d1c35d0f8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 24 Jan 2025 06:15:10 +0000 Subject: [PATCH 1400/2091] add config reloading indirector Signed-off-by: Jason Volk --- src/core/config/manager.rs | 128 +++++++++++++++++++++++++++++++++++++ src/core/config/mod.rs | 3 +- 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 src/core/config/manager.rs diff --git a/src/core/config/manager.rs b/src/core/config/manager.rs new file mode 100644 index 00000000..0c95ca15 --- /dev/null +++ b/src/core/config/manager.rs @@ -0,0 +1,128 @@ +use std::{ + cell::{Cell, RefCell}, + ops::Deref, + ptr, + ptr::null_mut, + sync::{ + atomic::{AtomicPtr, Ordering}, + Arc, + }, +}; + +use super::Config; +use crate::{implement, Result}; + +/// The configuration manager is an indirection to reload the configuration for +/// the server while it is running. In order to not burden or clutter the many +/// callsites which query for configuration items, this object implements Deref +/// for the actively loaded configuration. +pub struct Manager { + active: AtomicPtr, +} + +thread_local! { + static INDEX: Cell = 0.into(); + static HANDLE: RefCell = const { + RefCell::new([const { None }; HISTORY]) + }; +} + +type Handle = Option>; +type Handles = [Handle; HISTORY]; + +const HISTORY: usize = 8; + +impl Manager { + pub(crate) fn new(config: Config) -> Self { + let config = Arc::new(config); + Self { + active: AtomicPtr::new(Arc::into_raw(config).cast_mut()), + } + } +} + +impl Drop for Manager { + fn drop(&mut self) { + let config = self.active.swap(null_mut(), Ordering::AcqRel); + + // SAFETY: The active pointer was set using an Arc::into_raw(). We're obliged to + // reconstitute that into Arc otherwise it will leak. + unsafe { Arc::from_raw(config) }; + } +} + +impl Deref for Manager { + type Target = Arc; + + fn deref(&self) -> &Self::Target { HANDLE.with_borrow_mut(|handle| self.load(handle)) } +} + +/// Update the active configuration, returning prior configuration. +#[implement(Manager)] +#[tracing::instrument(skip_all)] +pub fn update(&self, config: Config) -> Result> { + let config = Arc::new(config); + let new = Arc::into_raw(config); + let old = self.active.swap(new.cast_mut(), Ordering::AcqRel); + + // SAFETY: The old active pointer was set using an Arc::into_raw(). We're + // obliged to reconstitute that into Arc otherwise it will leak. + Ok(unsafe { Arc::from_raw(old) }) +} + +#[implement(Manager)] +fn load(&self, handle: &mut [Option>]) -> &'static Arc { + let config = self.active.load(Ordering::Acquire); + + // Branch taken after config reload or first access by this thread. + if handle[INDEX.get()] + .as_ref() + .is_none_or(|handle| !ptr::eq(config, Arc::as_ptr(handle))) + { + INDEX.set(INDEX.get().wrapping_add(1).wrapping_rem(HISTORY)); + return load_miss(handle, INDEX.get(), config); + } + + let config: &Arc = handle[INDEX.get()] + .as_ref() + .expect("handle was already cached for this thread"); + + // SAFETY: The caller should not hold multiple references at a time directly + // into Config, as a subsequent reference might invalidate the thread's cache + // causing another reference to dangle. + // + // This is a highly unusual pattern as most config values are copied by value or + // used immediately without running overlap with another value. Even if it does + // actually occur somewhere, the window of danger is limited to the config being + // reloaded while the reference is held and another access is made by the same + // thread into a different config value. This is mitigated by creating a buffer + // of old configs rather than discarding at the earliest opportunity; the odds + // of this scenario are thus astronomical. + unsafe { std::mem::transmute(config) } +} + +#[tracing::instrument( + name = "miss", + level = "trace", + skip_all, + fields(%index, ?config) +)] +#[allow(clippy::transmute_ptr_to_ptr)] +fn load_miss( + handle: &mut [Option>], + index: usize, + config: *const Config, +) -> &'static Arc { + // SAFETY: The active pointer was set prior and always remains valid. We're + // reconstituting the Arc here but as a new reference, so the count is + // incremented. This instance will be cached in the thread-local. + let config = unsafe { + Arc::increment_strong_count(config); + Arc::from_raw(config) + }; + + // SAFETY: See the note on the transmute above. The caller should not hold more + // than one reference at a time directly into Config, as the second access + // might invalidate the thread's cache, dangling the reference to the first. + unsafe { std::mem::transmute(handle[index].insert(config)) } +} diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index beaabe5d..e459f50b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1,4 +1,5 @@ pub mod check; +pub mod manager; pub mod proxy; use std::{ @@ -22,8 +23,8 @@ use ruma::{ use serde::{de::IgnoredAny, Deserialize}; use url::Url; -pub use self::check::check; use self::proxy::ProxyConfig; +pub use self::{check::check, manager::Manager}; use crate::{err, error::Error, utils::sys, Result}; /// All the config options for conduwuit. From 5be07ebc0f4bbedb3f0d93d35d290720d042fd0d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 24 Jan 2025 07:02:56 +0000 Subject: [PATCH 1401/2091] eliminate references to services.globals.config Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 18 +++---- src/admin/federation/commands.rs | 2 +- src/admin/user/commands.rs | 8 +-- src/api/client/account.rs | 18 +++---- src/api/client/directory.rs | 6 +-- src/api/client/media.rs | 2 +- src/api/client/media_legacy.rs | 2 +- src/api/client/membership.rs | 12 ++--- src/api/client/openid.rs | 2 +- src/api/client/report.rs | 2 +- src/api/client/room/create.rs | 8 +-- src/api/client/typing.rs | 12 ++--- src/api/client/voip.rs | 2 +- src/api/router/auth.rs | 6 +-- src/api/router/request.rs | 2 +- src/api/server/invite.rs | 4 +- src/api/server/make_join.rs | 4 +- src/api/server/make_knock.rs | 4 +- src/api/server/publicrooms.rs | 2 +- src/api/server/query.rs | 2 +- src/api/server/send.rs | 4 +- src/api/server/send_join.rs | 8 +-- src/api/server/send_knock.rs | 4 +- src/service/admin/mod.rs | 2 +- src/service/globals/mod.rs | 84 ++++++++++++++++---------------- src/service/rooms/alias/mod.rs | 10 ++-- 26 files changed, 116 insertions(+), 114 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 49078dde..af7bd79f 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -170,7 +170,7 @@ pub(super) async fn get_remote_pdu_list( server: Box, force: bool, ) -> Result { - if !self.services.globals.config.allow_federation { + if !self.services.server.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( "Federation is disabled on this homeserver.", )); @@ -235,7 +235,7 @@ pub(super) async fn get_remote_pdu( event_id: Box, server: Box, ) -> Result { - if !self.services.globals.config.allow_federation { + if !self.services.server.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( "Federation is disabled on this homeserver.", )); @@ -419,7 +419,7 @@ pub(super) async fn change_log_level( let handles = &["console"]; if reset { - let old_filter_layer = match EnvFilter::try_new(&self.services.globals.config.log) { + let old_filter_layer = match EnvFilter::try_new(&self.services.server.config.log) { | Ok(s) => s, | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( @@ -438,7 +438,7 @@ pub(super) async fn change_log_level( | Ok(()) => { return Ok(RoomMessageEventContent::text_plain(format!( "Successfully changed log level back to config value {}", - self.services.globals.config.log + self.services.server.config.log ))); }, | Err(e) => { @@ -554,7 +554,7 @@ pub(super) async fn first_pdu_in_room( .services .rooms .state_cache - .server_in_room(&self.services.globals.config.server_name, &room_id) + .server_in_room(&self.services.server.config.server_name, &room_id) .await { return Ok(RoomMessageEventContent::text_plain( @@ -583,7 +583,7 @@ pub(super) async fn latest_pdu_in_room( .services .rooms .state_cache - .server_in_room(&self.services.globals.config.server_name, &room_id) + .server_in_room(&self.services.server.config.server_name, &room_id) .await { return Ok(RoomMessageEventContent::text_plain( @@ -613,7 +613,7 @@ pub(super) async fn force_set_room_state_from_server( .services .rooms .state_cache - .server_in_room(&self.services.globals.config.server_name, &room_id) + .server_in_room(&self.services.server.config.server_name, &room_id) .await { return Ok(RoomMessageEventContent::text_plain( @@ -818,13 +818,13 @@ pub(super) async fn resolve_true_destination( server_name: Box, no_cache: bool, ) -> Result { - if !self.services.globals.config.allow_federation { + if !self.services.server.config.allow_federation { return Ok(RoomMessageEventContent::text_plain( "Federation is disabled on this homeserver.", )); } - if server_name == self.services.globals.config.server_name { + if server_name == self.services.server.config.server_name { return Ok(RoomMessageEventContent::text_plain( "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ fetching local PDUs.", diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 75635b1b..be91ef0a 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -92,7 +92,7 @@ pub(super) async fn remote_user_in_rooms( &self, user_id: Box, ) -> Result { - if user_id.server_name() == self.services.globals.config.server_name { + if user_id.server_name() == self.services.server.config.server_name { return Ok(RoomMessageEventContent::text_plain( "User belongs to our server, please use `list-joined-rooms` user admin command \ instead.", diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 57aedd9c..64767a36 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -83,12 +83,12 @@ pub(super) async fn create_user( // content is set to the user's display name with a space before it if !self .services - .globals + .server .config .new_user_displayname_suffix .is_empty() { - write!(displayname, " {}", self.services.globals.config.new_user_displayname_suffix) + write!(displayname, " {}", self.services.server.config.new_user_displayname_suffix) .expect("should be able to write to string buffer"); } @@ -114,8 +114,8 @@ pub(super) async fn create_user( ) .await?; - if !self.services.globals.config.auto_join_rooms.is_empty() { - for room in &self.services.globals.config.auto_join_rooms { + if !self.services.server.config.auto_join_rooms.is_empty() { + for room in &self.services.server.config.auto_join_rooms { let Ok(room_id) = self.services.rooms.alias.resolve(room).await else { error!(%user_id, "Failed to resolve room alias to room ID when attempting to auto join {room}, skipping"); continue; diff --git a/src/api/client/account.rs b/src/api/client/account.rs index e6748124..cb25b276 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -299,7 +299,7 @@ pub(crate) async fn register_route( if !services.globals.new_user_displayname_suffix().is_empty() && body.appservice_info.is_none() { - write!(displayname, " {}", services.globals.config.new_user_displayname_suffix) + write!(displayname, " {}", services.server.config.new_user_displayname_suffix) .expect("should be able to write to string buffer"); } @@ -365,7 +365,7 @@ pub(crate) async fn register_route( \"{device_display_name}\"" ); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::notice_plain(format!( @@ -378,7 +378,7 @@ pub(crate) async fn register_route( } else { info!("New user \"{user_id}\" registered on this server."); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::notice_plain(format!( @@ -395,7 +395,7 @@ pub(crate) async fn register_route( info!("New guest user \"{user_id}\" registered on this server."); if !device_display_name.is_empty() { - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::notice_plain(format!( @@ -407,7 +407,7 @@ pub(crate) async fn register_route( } } else { #[allow(clippy::collapsible_else_if)] - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::notice_plain(format!( @@ -438,10 +438,10 @@ pub(crate) async fn register_route( } if body.appservice_info.is_none() - && !services.globals.config.auto_join_rooms.is_empty() + && !services.server.config.auto_join_rooms.is_empty() && (services.globals.allow_guests_auto_join_rooms() || !is_guest) { - for room in &services.globals.config.auto_join_rooms { + for room in &services.server.config.auto_join_rooms { let Ok(room_id) = services.rooms.alias.resolve(room).await else { error!( "Failed to resolve room alias to room ID when attempting to auto join \ @@ -570,7 +570,7 @@ pub(crate) async fn change_password_route( info!("User {sender_user} changed their password."); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::notice_plain(format!( @@ -673,7 +673,7 @@ pub(crate) async fn deactivate_route( info!("User {sender_user} deactivated their account."); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::notice_plain(format!( diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index c8faaa46..9166eed9 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -152,7 +152,7 @@ pub(crate) async fn set_room_visibility_route( match &body.visibility { | room::Visibility::Public => { - if services.globals.config.lockdown_public_room_directory + if services.server.config.lockdown_public_room_directory && !services.users.is_admin(sender_user).await && body.appservice_info.is_none() { @@ -162,7 +162,7 @@ pub(crate) async fn set_room_visibility_route( body.room_id ); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_text(&format!( @@ -181,7 +181,7 @@ pub(crate) async fn set_room_visibility_route( services.rooms.directory.set_public(&body.room_id); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_text(&format!( diff --git a/src/api/client/media.rs b/src/api/client/media.rs index e58ba626..afbc218a 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -31,7 +31,7 @@ pub(crate) async fn get_media_config_route( _body: Ruma, ) -> Result { Ok(get_media_config::v1::Response { - upload_size: ruma_from_usize(services.globals.config.max_request_size), + upload_size: ruma_from_usize(services.server.config.max_request_size), }) } diff --git a/src/api/client/media_legacy.rs b/src/api/client/media_legacy.rs index 6f54a683..29cf3069 100644 --- a/src/api/client/media_legacy.rs +++ b/src/api/client/media_legacy.rs @@ -27,7 +27,7 @@ pub(crate) async fn get_media_config_legacy_route( _body: Ruma, ) -> Result { Ok(get_media_config::v3::Response { - upload_size: ruma_from_usize(services.globals.config.max_request_size), + upload_size: ruma_from_usize(services.server.config.max_request_size), }) } diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index d94fc3c7..2e23dab9 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -71,7 +71,7 @@ async fn banned_room_check( if let Some(room_id) = room_id { if services.rooms.metadata.is_banned(room_id).await || services - .globals + .server .config .forbidden_remote_server_names .contains(&room_id.server_name().unwrap().to_owned()) @@ -81,12 +81,12 @@ async fn banned_room_check( attempted to join a banned room or banned room server name: {room_id}" ); - if services.globals.config.auto_deactivate_banned_room_attempts { + if services.server.config.auto_deactivate_banned_room_attempts { warn!( "Automatically deactivating user {user_id} due to attempted banned room join" ); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::text_plain(format!( @@ -112,7 +112,7 @@ async fn banned_room_check( } } else if let Some(server_name) = server_name { if services - .globals + .server .config .forbidden_remote_server_names .contains(&server_name.to_owned()) @@ -122,12 +122,12 @@ async fn banned_room_check( name {server_name} that is globally forbidden. Rejecting.", ); - if services.globals.config.auto_deactivate_banned_room_attempts { + if services.server.config.auto_deactivate_banned_room_attempts { warn!( "Automatically deactivating user {user_id} due to attempted banned room join" ); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_message(RoomMessageEventContent::text_plain(format!( diff --git a/src/api/client/openid.rs b/src/api/client/openid.rs index 4b2ff727..3547d284 100644 --- a/src/api/client/openid.rs +++ b/src/api/client/openid.rs @@ -37,7 +37,7 @@ pub(crate) async fn create_openid_token_route( Ok(account::request_openid_token::v3::Response { access_token, token_type: TokenType::Bearer, - matrix_server_name: services.globals.config.server_name.clone(), + matrix_server_name: services.server.config.server_name.clone(), expires_in: Duration::from_secs(expires_in), }) } diff --git a/src/api/client/report.rs b/src/api/client/report.rs index fe23b7bd..2b25b518 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -50,7 +50,7 @@ pub(crate) async fn report_room_route( if !services .rooms .state_cache - .server_in_room(&services.globals.config.server_name, &body.room_id) + .server_in_room(&services.server.config.server_name, &body.room_id) .await { return Err!(Request(NotFound( diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index a78242ca..1b6e8667 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -71,7 +71,7 @@ pub(crate) async fn create_room_route( let room_id: OwnedRoomId = if let Some(custom_room_id) = &body.room_id { custom_room_id_check(&services, custom_room_id)? } else { - RoomId::new(&services.globals.config.server_name) + RoomId::new(&services.server.config.server_name) }; // check if room ID doesn't already exist instead of erroring on auth check @@ -83,7 +83,7 @@ pub(crate) async fn create_room_route( } if body.visibility == room::Visibility::Public - && services.globals.config.lockdown_public_room_directory + && services.server.config.lockdown_public_room_directory && !services.users.is_admin(sender_user).await && body.appservice_info.is_none() { @@ -93,7 +93,7 @@ pub(crate) async fn create_room_route( &room_id ); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_text(&format!( @@ -450,7 +450,7 @@ pub(crate) async fn create_room_route( if body.visibility == room::Visibility::Public { services.rooms.directory.set_public(&room_id); - if services.globals.config.admin_room_notices { + if services.server.config.admin_room_notices { services .admin .send_text(&format!( diff --git a/src/api/client/typing.rs b/src/api/client/typing.rs index 6eabe96a..b311295b 100644 --- a/src/api/client/typing.rs +++ b/src/api/client/typing.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduwuit::Err; +use conduwuit::{utils::math::Tried, Err}; use ruma::api::client::typing::create_typing_event; use crate::{utils, Result, Ruma}; @@ -31,17 +31,15 @@ pub(crate) async fn create_typing_event_route( let duration = utils::clamp( duration.as_millis().try_into().unwrap_or(u64::MAX), services - .globals + .server .config .typing_client_timeout_min_s - .checked_mul(1000) - .unwrap(), + .try_mul(1000)?, services - .globals + .server .config .typing_client_timeout_max_s - .checked_mul(1000) - .unwrap(), + .try_mul(1000)?, ); services .rooms diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index ec804570..c08b1fdf 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -38,7 +38,7 @@ pub(crate) async fn turn_server_route( let user = body.sender_user.unwrap_or_else(|| { UserId::parse_with_server_name( utils::random_string(RANDOM_USER_ID_LENGTH).to_lowercase(), - &services.globals.config.server_name, + &services.server.config.server_name, ) .unwrap() }); diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index dd25e091..ecea305b 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -71,7 +71,7 @@ pub(super) async fn auth( match metadata { | &get_public_rooms::v3::Request::METADATA => { if !services - .globals + .server .config .allow_public_room_directory_without_auth { @@ -94,7 +94,7 @@ pub(super) async fn auth( | &get_display_name::v3::Request::METADATA | &get_avatar_url::v3::Request::METADATA | &get_timezone_key::unstable::Request::METADATA => { - if services.globals.config.require_auth_for_profile_requests { + if services.server.config.require_auth_for_profile_requests { match token { | Token::Appservice(_) | Token::User(_) => { // we should have validated the token above @@ -127,7 +127,7 @@ pub(super) async fn auth( }), | (AuthScheme::AccessToken, Token::None) => match metadata { | &get_turn_server_info::v3::Request::METADATA => { - if services.globals.config.turn_allow_guests { + if services.server.config.turn_allow_guests { Ok(Auth { origin: None, sender_user: None, diff --git a/src/api/router/request.rs b/src/api/router/request.rs index 627abd30..615a8bff 100644 --- a/src/api/router/request.rs +++ b/src/api/router/request.rs @@ -32,7 +32,7 @@ pub(super) async fn from( let query = serde_html_form::from_str(query) .map_err(|e| err!(Request(Unknown("Failed to read query parameters: {e}"))))?; - let max_body_size = services.globals.config.max_request_size; + let max_body_size = services.server.config.max_request_size; let body = axum::body::to_bytes(body, max_body_size) .await diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 1fea268b..27a4485c 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -37,7 +37,7 @@ pub(crate) async fn create_invite_route( if let Some(server) = body.room_id.server_name() { if services - .globals + .server .config .forbidden_remote_server_names .contains(&server.to_owned()) @@ -47,7 +47,7 @@ pub(crate) async fn create_invite_route( } if services - .globals + .server .config .forbidden_remote_server_names .contains(body.origin()) diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index 3900c418..b753346c 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -42,7 +42,7 @@ pub(crate) async fn create_join_event_template_route( .await?; if services - .globals + .server .config .forbidden_remote_server_names .contains(body.origin()) @@ -59,7 +59,7 @@ pub(crate) async fn create_join_event_template_route( if let Some(server) = body.room_id.server_name() { if services - .globals + .server .config .forbidden_remote_server_names .contains(&server.to_owned()) diff --git a/src/api/server/make_knock.rs b/src/api/server/make_knock.rs index 90b9b629..423e202d 100644 --- a/src/api/server/make_knock.rs +++ b/src/api/server/make_knock.rs @@ -34,7 +34,7 @@ pub(crate) async fn create_knock_event_template_route( .await?; if services - .globals + .server .config .forbidden_remote_server_names .contains(body.origin()) @@ -51,7 +51,7 @@ pub(crate) async fn create_knock_event_template_route( if let Some(server) = body.room_id.server_name() { if services - .globals + .server .config .forbidden_remote_server_names .contains(&server.to_owned()) diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index 77cde15f..2c09385b 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -20,7 +20,7 @@ pub(crate) async fn get_public_rooms_filtered_route( body: Ruma, ) -> Result { if !services - .globals + .server .config .allow_public_room_directory_over_federation { diff --git a/src/api/server/query.rs b/src/api/server/query.rs index 0e5f7e56..69f62e94 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -63,7 +63,7 @@ pub(crate) async fn get_profile_information_route( body: Ruma, ) -> Result { if !services - .globals + .server .config .allow_inbound_profile_lookup_federation_requests { diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 56a17c22..eec9bd11 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -309,7 +309,7 @@ async fn handle_edu_typing( origin: &ServerName, typing: TypingContent, ) { - if !services.globals.config.allow_incoming_typing { + if !services.server.config.allow_incoming_typing { return; } @@ -344,7 +344,7 @@ async fn handle_edu_typing( if typing.typing { let timeout = utils::millis_since_unix_epoch().saturating_add( services - .globals + .server .config .typing_federation_timeout_s .saturating_mul(1000), diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 97a65bf8..e62089b4 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -268,7 +268,7 @@ pub(crate) async fn create_join_event_v1_route( body: Ruma, ) -> Result { if services - .globals + .server .config .forbidden_remote_server_names .contains(body.origin()) @@ -284,7 +284,7 @@ pub(crate) async fn create_join_event_v1_route( if let Some(server) = body.room_id.server_name() { if services - .globals + .server .config .forbidden_remote_server_names .contains(&server.to_owned()) @@ -316,7 +316,7 @@ pub(crate) async fn create_join_event_v2_route( body: Ruma, ) -> Result { if services - .globals + .server .config .forbidden_remote_server_names .contains(body.origin()) @@ -326,7 +326,7 @@ pub(crate) async fn create_join_event_v2_route( if let Some(server) = body.room_id.server_name() { if services - .globals + .server .config .forbidden_remote_server_names .contains(&server.to_owned()) diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs index 95478081..b07620af 100644 --- a/src/api/server/send_knock.rs +++ b/src/api/server/send_knock.rs @@ -22,7 +22,7 @@ pub(crate) async fn create_knock_event_v1_route( body: Ruma, ) -> Result { if services - .globals + .server .config .forbidden_remote_server_names .contains(body.origin()) @@ -38,7 +38,7 @@ pub(crate) async fn create_knock_event_v1_route( if let Some(server) = body.room_id.server_name() { if services - .globals + .server .config .forbidden_remote_server_names .contains(&server.to_owned()) diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 399055aa..bc410631 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -338,7 +338,7 @@ impl Service { } // Check if server-side command-escape is disabled by configuration - if is_public_escape && !self.services.globals.config.admin_escape_commands { + if is_public_escape && !self.services.server.config.admin_escape_commands { return false; } diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index fe84578a..ef34054f 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,7 +7,7 @@ use std::{ time::Instant, }; -use conduwuit::{error, utils::bytes::pretty, Config, Result}; +use conduwuit::{error, utils::bytes::pretty, Result, Server}; use data::Data; use regex::RegexSet; use ruma::{OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, ServerName, UserId}; @@ -16,8 +16,8 @@ use crate::service; pub struct Service { pub db: Data, + server: Arc, - pub config: Config, pub bad_event_ratelimiter: Arc>>, pub server_user: OwnedUserId, pub admin_alias: OwnedRoomAliasId, @@ -57,9 +57,9 @@ impl crate::Service for Service { }, ); - let mut s = Self { + Ok(Arc::new(Self { db, - config: config.clone(), + server: args.server.clone(), bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), admin_alias: OwnedRoomAliasId::try_from(format!("#admins:{}", &config.server_name)) .expect("#admins:server_name is valid alias name"), @@ -70,9 +70,7 @@ impl crate::Service for Service { .expect("@conduit:server_name is valid"), turn_secret, registration_token, - }; - - Ok(Arc::new(s)) + })) } fn memory_usage(&self, out: &mut dyn Write) -> Result { @@ -109,93 +107,97 @@ impl Service { pub fn current_count(&self) -> Result { Ok(self.db.current_count()) } #[inline] - pub fn server_name(&self) -> &ServerName { self.config.server_name.as_ref() } + pub fn server_name(&self) -> &ServerName { self.server.config.server_name.as_ref() } - pub fn allow_registration(&self) -> bool { self.config.allow_registration } + pub fn allow_registration(&self) -> bool { self.server.config.allow_registration } - pub fn allow_guest_registration(&self) -> bool { self.config.allow_guest_registration } + pub fn allow_guest_registration(&self) -> bool { self.server.config.allow_guest_registration } pub fn allow_guests_auto_join_rooms(&self) -> bool { - self.config.allow_guests_auto_join_rooms + self.server.config.allow_guests_auto_join_rooms } - pub fn log_guest_registrations(&self) -> bool { self.config.log_guest_registrations } + pub fn log_guest_registrations(&self) -> bool { self.server.config.log_guest_registrations } - pub fn allow_encryption(&self) -> bool { self.config.allow_encryption } + pub fn allow_encryption(&self) -> bool { self.server.config.allow_encryption } - pub fn allow_federation(&self) -> bool { self.config.allow_federation } + pub fn allow_federation(&self) -> bool { self.server.config.allow_federation } pub fn allow_public_room_directory_over_federation(&self) -> bool { - self.config.allow_public_room_directory_over_federation + self.server + .config + .allow_public_room_directory_over_federation } pub fn allow_device_name_federation(&self) -> bool { - self.config.allow_device_name_federation + self.server.config.allow_device_name_federation } - pub fn allow_room_creation(&self) -> bool { self.config.allow_room_creation } + pub fn allow_room_creation(&self) -> bool { self.server.config.allow_room_creation } pub fn new_user_displayname_suffix(&self) -> &String { - &self.config.new_user_displayname_suffix + &self.server.config.new_user_displayname_suffix } - pub fn allow_check_for_updates(&self) -> bool { self.config.allow_check_for_updates } + pub fn allow_check_for_updates(&self) -> bool { self.server.config.allow_check_for_updates } - pub fn trusted_servers(&self) -> &[OwnedServerName] { &self.config.trusted_servers } + pub fn trusted_servers(&self) -> &[OwnedServerName] { &self.server.config.trusted_servers } - pub fn turn_password(&self) -> &String { &self.config.turn_password } + pub fn turn_password(&self) -> &String { &self.server.config.turn_password } - pub fn turn_ttl(&self) -> u64 { self.config.turn_ttl } + pub fn turn_ttl(&self) -> u64 { self.server.config.turn_ttl } - pub fn turn_uris(&self) -> &[String] { &self.config.turn_uris } + pub fn turn_uris(&self) -> &[String] { &self.server.config.turn_uris } - pub fn turn_username(&self) -> &String { &self.config.turn_username } + pub fn turn_username(&self) -> &String { &self.server.config.turn_username } - pub fn notification_push_path(&self) -> &String { &self.config.notification_push_path } + pub fn notification_push_path(&self) -> &String { &self.server.config.notification_push_path } - pub fn emergency_password(&self) -> &Option { &self.config.emergency_password } + pub fn emergency_password(&self) -> &Option { &self.server.config.emergency_password } pub fn url_preview_domain_contains_allowlist(&self) -> &Vec { - &self.config.url_preview_domain_contains_allowlist + &self.server.config.url_preview_domain_contains_allowlist } pub fn url_preview_domain_explicit_allowlist(&self) -> &Vec { - &self.config.url_preview_domain_explicit_allowlist + &self.server.config.url_preview_domain_explicit_allowlist } pub fn url_preview_domain_explicit_denylist(&self) -> &Vec { - &self.config.url_preview_domain_explicit_denylist + &self.server.config.url_preview_domain_explicit_denylist } pub fn url_preview_url_contains_allowlist(&self) -> &Vec { - &self.config.url_preview_url_contains_allowlist + &self.server.config.url_preview_url_contains_allowlist } - pub fn url_preview_max_spider_size(&self) -> usize { self.config.url_preview_max_spider_size } + pub fn url_preview_max_spider_size(&self) -> usize { + self.server.config.url_preview_max_spider_size + } pub fn url_preview_check_root_domain(&self) -> bool { - self.config.url_preview_check_root_domain + self.server.config.url_preview_check_root_domain } - pub fn forbidden_alias_names(&self) -> &RegexSet { &self.config.forbidden_alias_names } + pub fn forbidden_alias_names(&self) -> &RegexSet { &self.server.config.forbidden_alias_names } - pub fn forbidden_usernames(&self) -> &RegexSet { &self.config.forbidden_usernames } + pub fn forbidden_usernames(&self) -> &RegexSet { &self.server.config.forbidden_usernames } - pub fn allow_local_presence(&self) -> bool { self.config.allow_local_presence } + pub fn allow_local_presence(&self) -> bool { self.server.config.allow_local_presence } - pub fn allow_incoming_presence(&self) -> bool { self.config.allow_incoming_presence } + pub fn allow_incoming_presence(&self) -> bool { self.server.config.allow_incoming_presence } - pub fn allow_outgoing_presence(&self) -> bool { self.config.allow_outgoing_presence } + pub fn allow_outgoing_presence(&self) -> bool { self.server.config.allow_outgoing_presence } pub fn allow_incoming_read_receipts(&self) -> bool { - self.config.allow_incoming_read_receipts + self.server.config.allow_incoming_read_receipts } pub fn allow_outgoing_read_receipts(&self) -> bool { - self.config.allow_outgoing_read_receipts + self.server.config.allow_outgoing_read_receipts } - pub fn block_non_admin_invites(&self) -> bool { self.config.block_non_admin_invites } + pub fn block_non_admin_invites(&self) -> bool { self.server.config.block_non_admin_invites } /// checks if `user_id` is local to us via server_name comparison #[inline] @@ -205,7 +207,7 @@ impl Service { #[inline] pub fn server_is_ours(&self, server_name: &ServerName) -> bool { - server_name == self.config.server_name + server_name == self.server.config.server_name } #[inline] diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 0acbb116..91797d01 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use conduwuit::{ err, utils::{stream::TryIgnore, ReadyExt}, - Err, Result, + Err, Result, Server, }; use database::{Deserialized, Ignore, Interfix, Map}; use futures::{Stream, StreamExt, TryFutureExt}; @@ -31,6 +31,7 @@ struct Data { } struct Services { + server: Arc, admin: Dep, appservice: Dep, globals: Dep, @@ -47,6 +48,7 @@ impl crate::Service for Service { aliasid_alias: args.db["aliasid_alias"].clone(), }, services: Services { + server: args.server.clone(), admin: args.depend::("admin"), appservice: args.depend::("appservice"), globals: args.depend::("globals"), @@ -146,9 +148,9 @@ impl Service { let server_name = room_alias.server_name(); let server_is_ours = self.services.globals.server_is_ours(server_name); let servers_contains_ours = || { - servers.as_ref().is_some_and(|servers| { - servers.contains(&self.services.globals.config.server_name) - }) + servers + .as_ref() + .is_some_and(|servers| servers.contains(&self.services.server.config.server_name)) }; if !server_is_ours && !servers_contains_ours() { From b5c167de121e17696c2542b34a6b7904dade8c21 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 24 Jan 2025 11:29:36 +0000 Subject: [PATCH 1402/2091] call decay prior to purge for trim-memory Signed-off-by: Jason Volk --- src/core/alloc/je.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 81fbd3ea..6bdf8b33 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -141,12 +141,14 @@ pub mod this_thread { static DEALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() }; } - pub fn idle() -> Result { super::notify(&mallctl!("thread.idle")) } + pub fn trim() -> Result { decay().and_then(|()| purge()) } - pub fn trim() -> Result { notify(mallctl!("arena.0.purge")) } + pub fn purge() -> Result { notify(mallctl!("arena.0.purge")) } pub fn decay() -> Result { notify(mallctl!("arena.0.decay")) } + pub fn idle() -> Result { super::notify(&mallctl!("thread.idle")) } + pub fn flush() -> Result { super::notify(&mallctl!("thread.tcache.flush")) } pub fn set_muzzy_decay(decay_ms: isize) -> Result { @@ -239,7 +241,11 @@ pub fn is_prof_enabled() -> Result { get::(&mallctl!("prof.active")).map(is_nonzero!()) } -pub fn trim>>(arena: I) -> Result { +pub fn trim> + Copy>(arena: I) -> Result { + decay(arena).and_then(|()| purge(arena)) +} + +pub fn purge>>(arena: I) -> Result { notify_by_arena(arena.into(), mallctl!("arena.4096.purge")) } From 184a3b0f0cccfbd0a6f4d95f65504b4d1d9bb21f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 23 Jan 2025 19:07:13 +0000 Subject: [PATCH 1403/2091] reduce some tracing span levels; bump ruma Signed-off-by: Jason Volk --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- src/api/client/media_legacy.rs | 10 +++++----- src/api/server/media.rs | 14 ++++++++++++-- src/service/resolver/actual.rs | 24 ++++++++++++------------ src/service/rooms/timeline/mod.rs | 4 ++-- 6 files changed, 45 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7985a411..cd914dfd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3188,7 +3188,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "assign", "js_int", @@ -3210,7 +3210,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "js_int", "ruma-common", @@ -3222,7 +3222,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "as_variant", "assign", @@ -3245,7 +3245,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "as_variant", "base64 0.22.1", @@ -3276,7 +3276,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3301,7 +3301,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "bytes", "http", @@ -3319,7 +3319,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3328,7 +3328,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "js_int", "ruma-common", @@ -3338,7 +3338,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3353,7 +3353,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "js_int", "ruma-common", @@ -3365,7 +3365,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "headers", "http", @@ -3378,7 +3378,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3394,7 +3394,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=427877d5bc14988ed877e500bbb27f8bc08b84e8#427877d5bc14988ed877e500bbb27f8bc08b84e8" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index b8c145ca..d52ce974 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -332,7 +332,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "427877d5bc14988ed877e500bbb27f8bc08b84e8" +rev = "b560338b2a50dbf61ecfe80808b9b095ad4cec00" features = [ "compat", "rand", diff --git a/src/api/client/media_legacy.rs b/src/api/client/media_legacy.rs index 29cf3069..4fa0b52e 100644 --- a/src/api/client/media_legacy.rs +++ b/src/api/client/media_legacy.rs @@ -50,7 +50,7 @@ pub(crate) async fn get_media_config_legacy_legacy_route( /// # `GET /_matrix/media/v3/preview_url` /// /// Returns URL preview. -#[tracing::instrument(skip_all, fields(%client), name = "url_preview_legacy")] +#[tracing::instrument(skip_all, fields(%client), name = "url_preview_legacy", level = "debug")] pub(crate) async fn get_media_preview_legacy_route( State(services): State, InsecureClientIp(client): InsecureClientIp, @@ -131,7 +131,7 @@ pub(crate) async fn create_content_legacy_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")] +#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy", level = "debug")] pub(crate) async fn get_content_legacy_route( State(services): State, InsecureClientIp(client): InsecureClientIp, @@ -197,7 +197,7 @@ pub(crate) async fn get_content_legacy_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")] +#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy", level = "debug")] pub(crate) async fn get_content_legacy_legacy_route( State(services): State, InsecureClientIp(client): InsecureClientIp, @@ -216,7 +216,7 @@ pub(crate) async fn get_content_legacy_legacy_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy")] +#[tracing::instrument(skip_all, fields(%client), name = "media_get_legacy", level = "debug")] pub(crate) async fn get_content_as_filename_legacy_route( State(services): State, InsecureClientIp(client): InsecureClientIp, @@ -303,7 +303,7 @@ pub(crate) async fn get_content_as_filename_legacy_legacy_route( /// - Only redirects if `allow_redirect` is true /// - Uses client-provided `timeout_ms` if available, else defaults to 20 /// seconds -#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get_legacy")] +#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get_legacy", level = "debug")] pub(crate) async fn get_content_thumbnail_legacy_route( State(services): State, InsecureClientIp(client): InsecureClientIp, diff --git a/src/api/server/media.rs b/src/api/server/media.rs index 03ec7b51..e56f5b9d 100644 --- a/src/api/server/media.rs +++ b/src/api/server/media.rs @@ -14,7 +14,12 @@ use crate::Ruma; /// # `GET /_matrix/federation/v1/media/download/{mediaId}` /// /// Load media from our server. -#[tracing::instrument(skip_all, fields(%client), name = "media_get")] +#[tracing::instrument( + name = "media_get", + level = "debug", + skip_all, + fields(%client) +)] pub(crate) async fn get_content_route( State(services): State, InsecureClientIp(client): InsecureClientIp, @@ -51,7 +56,12 @@ pub(crate) async fn get_content_route( /// # `GET /_matrix/federation/v1/media/thumbnail/{mediaId}` /// /// Load media thumbnail from our server. -#[tracing::instrument(skip_all, fields(%client), name = "media_thumbnail_get")] +#[tracing::instrument( + name = "media_thumbnail_get", + level = "debug", + skip_all, + fields(%client) +)] pub(crate) async fn get_content_thumbnail_route( State(services): State, InsecureClientIp(client): InsecureClientIp, diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index c5451c58..33374240 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -58,7 +58,7 @@ impl super::Service { /// Implemented according to the specification at /// Numbers in comments below refer to bullet points in linked section of /// specification - #[tracing::instrument(skip(self, cache), name = "actual")] + #[tracing::instrument(name = "actual", level = "debug", skip(self, cache))] pub async fn resolve_actual_dest( &self, dest: &ServerName, @@ -239,7 +239,7 @@ impl super::Service { Ok(add_port_to_hostname(dest.as_str())) } - #[tracing::instrument(skip_all, name = "well-known")] + #[tracing::instrument(name = "well-known", level = "debug", skip(self, dest))] async fn request_well_known(&self, dest: &str) -> Result> { self.conditional_query_and_cache(dest, 8448, true).await?; @@ -303,7 +303,7 @@ impl super::Service { #[inline] async fn conditional_query_and_cache_override( &self, - overname: &str, + untername: &str, hostname: &str, port: u16, cache: bool, @@ -312,34 +312,34 @@ impl super::Service { return Ok(()); } - if self.cache.has_override(overname).await { + if self.cache.has_override(untername).await { return Ok(()); } - self.query_and_cache_override(overname, hostname, port) + self.query_and_cache_override(untername, hostname, port) .await } - #[tracing::instrument(skip(self, overname, port), name = "ip")] + #[tracing::instrument(name = "ip", level = "debug", skip(self))] async fn query_and_cache_override( &self, - overname: &'_ str, + untername: &'_ str, hostname: &'_ str, port: u16, ) -> Result { self.services.server.check_running()?; - debug!("querying IP for {overname:?} ({hostname:?}:{port})"); + debug!("querying IP for {untername:?} ({hostname:?}:{port})"); match self.resolver.resolver.lookup_ip(hostname.to_owned()).await { | Err(e) => Self::handle_resolve_error(&e, hostname), | Ok(override_ip) => { - self.cache.set_override(overname, &CachedOverride { + self.cache.set_override(untername, &CachedOverride { ips: override_ip.into_iter().take(MAX_IPS).collect(), port, expire: CachedOverride::default_expire(), - overriding: (hostname != overname) + overriding: (hostname != untername) .then_some(hostname.into()) - .inspect(|_| debug_info!("{overname:?} overriden by {hostname:?}")), + .inspect(|_| debug_info!("{untername:?} overriden by {hostname:?}")), }); Ok(()) @@ -347,7 +347,7 @@ impl super::Service { } } - #[tracing::instrument(skip_all, name = "srv")] + #[tracing::instrument(name = "srv", level = "debug", skip(self))] async fn query_srv_record(&self, hostname: &'_ str) -> Result> { let hostnames = [format!("_matrix-fed._tcp.{hostname}."), format!("_matrix._tcp.{hostname}.")]; diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index bd60e40e..362bfab5 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1017,7 +1017,7 @@ impl Service { } /// Replace a PDU with the redacted form. - #[tracing::instrument(skip(self, reason))] + #[tracing::instrument(name = "redact", level = "debug", skip(self))] pub async fn redact_pdu( &self, event_id: &EventId, @@ -1053,7 +1053,7 @@ impl Service { self.replace_pdu(&pdu_id, &obj, &pdu).await } - #[tracing::instrument(skip(self), level = "debug")] + #[tracing::instrument(name = "backfill", level = "debug", skip(self))] pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { if self .services From b1b6dc0479538a207b3cf62ac90b58abb38ae103 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 24 Jan 2025 07:04:29 +0000 Subject: [PATCH 1404/2091] reloadable configuration Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 22 +++++++++++++++++++--- src/admin/server/mod.rs | 7 +++++++ src/core/server.rs | 6 +++--- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 6469a0e9..3ea27883 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,6 +1,6 @@ -use std::{fmt::Write, sync::Arc}; +use std::{fmt::Write, path::PathBuf, sync::Arc}; -use conduwuit::{info, utils::time, warn, Err, Result}; +use conduwuit::{info, utils::time, warn, Config, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; use crate::admin_command; @@ -23,10 +23,26 @@ pub(super) async fn show_config(&self) -> Result { // Construct and send the response Ok(RoomMessageEventContent::text_markdown(format!( "{}", - self.services.server.config + *self.services.server.config ))) } +#[admin_command] +pub(super) async fn reload_config( + &self, + path: Option, +) -> Result { + let path = path.as_deref().into_iter(); + let config = Config::load(path).and_then(|raw| Config::new(&raw))?; + if config.server_name != self.services.server.config.server_name { + return Err!("You can't change the server name."); + } + + let _old = self.services.server.config.update(config)?; + + Ok(RoomMessageEventContent::text_plain("Successfully reconfigured.")) +} + #[admin_command] pub(super) async fn list_features( &self, diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 69ad7953..3f3d6c5e 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -1,5 +1,7 @@ mod commands; +use std::path::PathBuf; + use clap::Subcommand; use conduwuit::Result; @@ -14,6 +16,11 @@ pub(super) enum ServerCommand { /// - Show configuration values ShowConfig, + /// - Reload configuration values + ReloadConfig { + path: Option, + }, + /// - List the features built into the server ListFeatures { #[arg(short, long)] diff --git a/src/core/server.rs b/src/core/server.rs index 948eea36..6838c9c9 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -8,12 +8,12 @@ use std::{ use tokio::{runtime, sync::broadcast}; -use crate::{config::Config, err, log::Log, metrics::Metrics, Err, Result}; +use crate::{config, config::Config, err, log::Log, metrics::Metrics, Err, Result}; /// Server runtime state; public portion pub struct Server { /// Server-wide configuration instance - pub config: Config, + pub config: config::Manager, /// Timestamp server was started; used for uptime. pub started: SystemTime, @@ -46,7 +46,7 @@ impl Server { #[must_use] pub fn new(config: Config, runtime: Option, log: Log) -> Self { Self { - config, + config: config::Manager::new(config), started: SystemTime::now(), stopping: AtomicBool::new(false), reloading: AtomicBool::new(false), From d59f68a51aa35e7e5491da79d667ad4dd497be5e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 24 Jan 2025 22:49:10 +0000 Subject: [PATCH 1405/2091] add sensitive-field directives to config display Signed-off-by: Jason Volk --- src/core/config/mod.rs | 9 +++++++++ src/macros/config.rs | 27 ++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index e459f50b..c541c7e4 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -502,6 +502,8 @@ pub struct Config { /// YOU NEED TO EDIT THIS OR USE registration_token_file. /// /// example: "o&^uCtes4HPf0Vu@F20jQeeWE7" + /// + /// display: sensitive pub registration_token: Option, /// Path to a file on the system that gets read for the registration token. @@ -793,6 +795,8 @@ pub struct Config { /// Static TURN password to provide the client if not using a shared secret /// ("turn_secret"). It is recommended to use a shared secret over static /// credentials. + /// + /// display: sensitive #[serde(default)] pub turn_password: String, @@ -814,6 +818,8 @@ pub struct Config { /// /// This is more secure, but if needed you can use traditional static /// username/password credentials. + /// + /// display: sensitive #[serde(default)] pub turn_secret: String, @@ -1111,6 +1117,8 @@ pub struct Config { /// security purposes. /// /// example: "F670$2CP@Hw8mG7RY1$%!#Ic7YA" + /// + /// display: sensitive pub emergency_password: Option, /// default: "/_matrix/push/v1/notify" @@ -1560,6 +1568,7 @@ pub struct Config { /// Sentry reporting URL, if a custom one is desired. /// + /// display: sensitive /// default: "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" #[serde(default = "default_sentry_endpoint")] pub sentry_endpoint: Option, diff --git a/src/macros/config.rs b/src/macros/config.rs index 90d6ef15..50feefa8 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -15,7 +15,7 @@ use crate::{ const UNDOCUMENTED: &str = "# This item is undocumented. Please contribute documentation for it."; -const HIDDEN: &[&str] = &["default"]; +const HIDDEN: &[&str] = &["default", "display"]; #[allow(clippy::needless_pass_by_value)] pub(super) fn example_generator(input: ItemStruct, args: &[Meta]) -> Result { @@ -121,10 +121,27 @@ fn generate_example(input: &ItemStruct, args: &[Meta], write: bool) -> Result Date: Fri, 24 Jan 2025 23:45:35 +0000 Subject: [PATCH 1406/2091] fix missing iteration-optimized read options on several stream types Signed-off-by: Jason Volk --- src/database/map.rs | 3 ++- src/database/map/options.rs | 29 +++++++++++++++++++---------- src/database/map/rev_stream.rs | 4 ++-- src/database/map/rev_stream_from.rs | 2 +- src/database/map/stream.rs | 4 ++-- src/database/map/stream_from.rs | 4 ++-- 6 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/database/map.rs b/src/database/map.rs index 33cae594..97e90659 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -34,7 +34,8 @@ use conduwuit::Result; use rocksdb::{AsColumnFamilyRef, ColumnFamily, ReadOptions, WriteOptions}; pub(crate) use self::options::{ - cache_read_options_default, iter_options_default, read_options_default, write_options_default, + cache_iter_options_default, cache_read_options_default, iter_options_default, + read_options_default, write_options_default, }; use crate::{watchers::Watchers, Engine}; diff --git a/src/database/map/options.rs b/src/database/map/options.rs index 90dc0261..f726036d 100644 --- a/src/database/map/options.rs +++ b/src/database/map/options.rs @@ -2,24 +2,33 @@ use rocksdb::{ReadOptions, ReadTier, WriteOptions}; #[inline] pub(crate) fn iter_options_default() -> ReadOptions { - let mut read_options = read_options_default(); - read_options.set_background_purge_on_iterator_cleanup(true); - //read_options.set_pin_data(true); - read_options + let mut options = read_options_default(); + options.set_background_purge_on_iterator_cleanup(true); + //options.set_pin_data(true); + options +} + +#[inline] +pub(crate) fn cache_iter_options_default() -> ReadOptions { + let mut options = cache_read_options_default(); + options.set_background_purge_on_iterator_cleanup(true); + //options.set_pin_data(true); + options } #[inline] pub(crate) fn cache_read_options_default() -> ReadOptions { - let mut read_options = read_options_default(); - read_options.set_read_tier(ReadTier::BlockCache); - read_options + let mut options = read_options_default(); + options.set_read_tier(ReadTier::BlockCache); + options.fill_cache(false); + options } #[inline] pub(crate) fn read_options_default() -> ReadOptions { - let mut read_options = ReadOptions::default(); - read_options.set_total_order_seek(true); - read_options + let mut options = ReadOptions::default(); + options.set_total_order_seek(true); + options } #[inline] diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index 1d5d3d10..56b20b9b 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -31,7 +31,7 @@ where pub fn rev_raw_stream(self: &Arc) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::read_options_default(); + let opts = super::iter_options_default(); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_rev(None); @@ -66,7 +66,7 @@ pub fn rev_raw_stream(self: &Arc) -> impl Stream> fields(%map), )] pub(super) fn is_cached(map: &Arc) -> bool { - let opts = super::cache_read_options_default(); + let opts = super::cache_iter_options_default(); let state = stream::State::new(map, opts).init_rev(None); !state.is_incomplete() diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 1b66e8cc..83832bdd 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -118,7 +118,7 @@ pub(super) fn is_cached

    (map: &Arc, from: &P) -> bool where P: AsRef<[u8]> + ?Sized, { - let cache_opts = super::cache_read_options_default(); + let cache_opts = super::cache_iter_options_default(); let cache_status = stream::State::new(map, cache_opts) .init_rev(from.as_ref().into()) .status(); diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index fa3b0ad7..f1b5fdc3 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -30,7 +30,7 @@ where pub fn raw_stream(self: &Arc) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::read_options_default(); + let opts = super::iter_options_default(); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_fwd(None); @@ -65,7 +65,7 @@ pub fn raw_stream(self: &Arc) -> impl Stream>> + fields(%map), )] pub(super) fn is_cached(map: &Arc) -> bool { - let opts = super::cache_read_options_default(); + let opts = super::cache_iter_options_default(); let state = stream::State::new(map, opts).init_fwd(None); !state.is_incomplete() diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 4296b6f6..562ab6b1 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -77,7 +77,7 @@ where { use crate::pool::Seek; - let opts = super::read_options_default(); + let opts = super::iter_options_default(); let state = stream::State::new(self, opts); if is_cached(self, from) { let state = state.init_fwd(from.as_ref().into()); @@ -115,7 +115,7 @@ pub(super) fn is_cached

    (map: &Arc, from: &P) -> bool where P: AsRef<[u8]> + ?Sized, { - let opts = super::cache_read_options_default(); + let opts = super::cache_iter_options_default(); let state = stream::State::new(map, opts).init_fwd(from.as_ref().into()); !state.is_incomplete() From 72daf7ea6816ebf46c43b02e137d58bed5bee883 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 25 Jan 2025 02:01:20 -0500 Subject: [PATCH 1407/2091] bump rocksdb to v9.10.0, reverts upstream rocksdb regression causing deadlocks on corrupt WAL files Signed-off-by: June Clementine Strawberry --- Cargo.lock | 8 +++---- deps/rust-rocksdb/Cargo.toml | 2 +- flake.lock | 44 ++++++++++++++++++------------------ flake.nix | 2 +- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd914dfd..cb1458db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3408,8 +3408,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.31.0+9.9.3" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2d31cf323df7c6d95396ef0213e28936c2218bd6#2d31cf323df7c6d95396ef0213e28936c2218bd6" +version = "0.32.0+9.10.0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e#24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e" dependencies = [ "bindgen", "bzip2-sys", @@ -3425,8 +3425,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.35.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2d31cf323df7c6d95396ef0213e28936c2218bd6#2d31cf323df7c6d95396ef0213e28936c2218bd6" +version = "0.36.0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e#24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 96554aed..40c0541e 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "2d31cf323df7c6d95396ef0213e28936c2218bd6" +rev = "24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index 210e8e08..d245ccdd 100644 --- a/flake.lock +++ b/flake.lock @@ -32,11 +32,11 @@ "nixpkgs": "nixpkgs_4" }, "locked": { - "lastModified": 1733424942, - "narHash": "sha256-5t7Sl6EkOaoP4FvzLmH7HFDbdl9SizmLh53RjDQCbWQ=", + "lastModified": 1737621947, + "narHash": "sha256-8HFvG7fvIFbgtaYAY2628Tb89fA55nPm2jSiNs0/Cws=", "owner": "cachix", "repo": "cachix", - "rev": "8b6b0e4694b9aa78b2ea4c93bff6e1a222dc7e4a", + "rev": "f65a3cd5e339c223471e64c051434616e18cc4f5", "type": "github" }, "original": { @@ -117,11 +117,11 @@ }, "crane_2": { "locked": { - "lastModified": 1736566337, - "narHash": "sha256-SC0eDcZPqISVt6R0UfGPyQLrI0+BppjjtQ3wcSlk0oI=", + "lastModified": 1737689766, + "narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=", "owner": "ipetkov", "repo": "crane", - "rev": "9172acc1ee6c7e1cbafc3044ff850c568c75a5a3", + "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527", "type": "github" }, "original": { @@ -170,11 +170,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1736836313, - "narHash": "sha256-zdZ7/T6yG0/hzoVOiNpDiR/sW3zR6oSMrfIFJK2BrrE=", + "lastModified": 1737700483, + "narHash": "sha256-1778bR4GDDc51/iZQvcshGLZ4JU87zCzqei8Hn7vU1A=", "owner": "nix-community", "repo": "fenix", - "rev": "056c9393c821a4df356df6ce7f14c722dc8717ec", + "rev": "bab2a2840bc2d5ae7c6a133602185edbe4ca7daa", "type": "github" }, "original": { @@ -364,11 +364,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1736719310, - "narHash": "sha256-Turvx60THwzTiUHb49WV3upUgsPuktr7tVy2Lwu2xJg=", + "lastModified": 1737600516, + "narHash": "sha256-EKyLQ3pbcjoU5jH5atge59F4fzuhTsb6yalUj6Ve2t8=", "owner": "axboe", "repo": "liburing", - "rev": "3124a4619e4daf26b06d48ccf0186a947070c415", + "rev": "6c509e2b0c881a13b83b259a221bf15fc9b3f681", "type": "github" }, "original": { @@ -550,11 +550,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1736817698, - "narHash": "sha256-1m+JP9RUsbeLVv/tF1DX3Ew9Vl/fatXnlh/g5k3jcSk=", + "lastModified": 1737717945, + "narHash": "sha256-ET91TMkab3PmOZnqiJQYOtSGvSTvGeHoegAv4zcTefM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "2b1fca3296ddd1602d2c4f104a4050e006f4b0cb", + "rev": "ecd26a469ac56357fd333946a99086e992452b6a", "type": "github" }, "original": { @@ -567,16 +567,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1734469478, - "narHash": "sha256-IcQ4N8xADYal79K+ONmNq4RLlIwdgUqgrVzgNgiIaG8=", + "lastModified": 1737761947, + "narHash": "sha256-FqpAOeFGuA+luV36jaf5aVz3UB183n6wUrTbFxCwjjQ=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "8b4808e7de2fbb5d119d8d72cdca76d8ab84bc47", + "rev": "d078ca31e802696b26d972bda7bed86ee1382156", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.9.3", + "ref": "v9.10.0", "repo": "rocksdb", "type": "github" } @@ -599,11 +599,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1736690231, - "narHash": "sha256-g9gyxX+F6CrkT5gRIMKPnCPom0o9ZDzYnzzeNF86D6Q=", + "lastModified": 1737634189, + "narHash": "sha256-AG5G9KDsl0Ngby9EfWvlemma7WWG0KCADTIccPJuzUE=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "8364ef299790cb6ec22b9e09e873c97dbe9f2cb5", + "rev": "84d44d0a574630aa8500ed62b6c01ccd3fae2473", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 920d3d14..1d38f80f 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.9.3"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.10.0"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From 6a7fe3ab7c0ddd489250f1c2922c7808d67bff43 Mon Sep 17 00:00:00 2001 From: morguldir Date: Tue, 21 Jan 2025 12:11:49 +0100 Subject: [PATCH 1408/2091] limit wal archive size to 1gb Signed-off-by: morguldir --- src/database/engine/db_opts.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/engine/db_opts.rs b/src/database/engine/db_opts.rs index 26f53825..01847257 100644 --- a/src/database/engine/db_opts.rs +++ b/src/database/engine/db_opts.rs @@ -55,7 +55,7 @@ pub(crate) fn db_options(config: &Config, env: &Env, row_cache: &Cache) -> Resul // Files opts.set_table_cache_num_shard_bits(7); - opts.set_wal_size_limit_mb(1024 * 1024 * 1024); + opts.set_wal_size_limit_mb(1024); opts.set_max_total_wal_size(1024 * 1024 * 512); opts.set_writable_file_max_buffer_size(1024 * 1024 * 2); From eed3291625d6b6f454ff24e41888d17ed492e1b7 Mon Sep 17 00:00:00 2001 From: morguldir Date: Tue, 21 Jan 2025 13:32:10 +0100 Subject: [PATCH 1409/2091] ci: set variable after ssh has been configured, mainly for draft pull requests --- .github/workflows/ci.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 007adace..ce662101 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -306,6 +306,8 @@ jobs: echo "Checking connection" ssh -q website "echo test" + echo "SSH_WEBSITE=1" >> "$GITHUB_ENV" + - uses: nixbuild/nix-quick-install-action@master - name: Restore and cache Nix store @@ -489,7 +491,7 @@ jobs: - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver if: ${{ matrix.target == 'x86_64-linux-musl' }} run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-x86_64-linux-musl-x86_64-haswell-optimised fi @@ -497,7 +499,7 @@ jobs: - name: Upload static-${{ matrix.target }}-all-features to webserver if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then chmod +x static-${{ matrix.target }} scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }} fi @@ -505,14 +507,14 @@ jobs: - name: Upload static deb x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver if: ${{ matrix.target == 'x86_64-linux-musl' }} run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/x86_64-linux-musl-x86_64-haswell-optimised.deb fi - name: Upload static deb ${{ matrix.target }}-all-features to webserver if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}.deb fi @@ -534,14 +536,14 @@ jobs: - name: Upload static-${{ matrix.target }}-debug-all-features to webserver if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }}-debug fi - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE]; then scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}-debug.deb fi @@ -583,21 +585,21 @@ jobs: - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz to webserver if: ${{ matrix.target == 'x86_64-linux-musl' }} run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz fi - name: Upload OCI image ${{ matrix.target }}-all-features to webserver if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}.tar.gz fi - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}-debug.tar.gz fi @@ -693,7 +695,7 @@ jobs: - name: Upload macOS x86_64 binary to webserver if: ${{ matrix.os == 'macos-13' }} run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then chmod +x conduwuit-macos-x86_64 scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-x86_64 fi @@ -701,7 +703,7 @@ jobs: - name: Upload macOS arm64 binary to webserver if: ${{ matrix.os == 'macos-latest' }} run: | - if [ ! -z $WEB_UPLOAD_SSH_USERNAME ]; then + if [ ! -z $SSH_WEBSITE ]; then chmod +x conduwuit-macos-arm64 scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-arm64 fi From cd5d4f48bec719a938f50cb17b667668105a1141 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 25 Jan 2025 03:08:35 -0500 Subject: [PATCH 1410/2091] add mau.dev mirror of conduwuit Signed-off-by: June Clementine Strawberry --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 796f96f8..74b6bddf 100644 --- a/README.md +++ b/README.md @@ -109,7 +109,8 @@ Both, but I prefer conduwuit. - GitLab: - git.girlcock.ceo: - git.gay: -- Codeberg: +- mau.dev: +- Codeberg: - sourcehut: From 2abf15b9e9587e6b625c0f40bf29bef75368630e Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 25 Jan 2025 13:14:53 -0500 Subject: [PATCH 1411/2091] revert back to v9.9.3 due to upstream build issue with macos Signed-off-by: June Clementine Strawberry --- Cargo.lock | 12 ++++++------ deps/rust-rocksdb/Cargo.toml | 2 +- flake.lock | 20 ++++++++++---------- flake.nix | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb1458db..5848cc46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1250,7 +1250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -2183,7 +2183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -3016,7 +3016,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3409,7 +3409,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.32.0+9.10.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e#24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=1f032427d3a0e7b0f13c04b4e34712bd8610291b#1f032427d3a0e7b0f13c04b4e34712bd8610291b" dependencies = [ "bindgen", "bzip2-sys", @@ -3426,7 +3426,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.36.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e#24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=1f032427d3a0e7b0f13c04b4e34712bd8610291b#1f032427d3a0e7b0f13c04b4e34712bd8610291b" dependencies = [ "libc", "rust-librocksdb-sys", @@ -3477,7 +3477,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 40c0541e..ba8259a3 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "24826ed7e9cf9ed9ba7ba2f1ec63091006025c9e" +rev = "1f032427d3a0e7b0f13c04b4e34712bd8610291b" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index d245ccdd..5af6ec43 100644 --- a/flake.lock +++ b/flake.lock @@ -170,11 +170,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1737700483, - "narHash": "sha256-1778bR4GDDc51/iZQvcshGLZ4JU87zCzqei8Hn7vU1A=", + "lastModified": 1737786656, + "narHash": "sha256-ubCW9Jy7ZUOF354bWxTgLDpVnTvIpNr6qR4H/j7I0oo=", "owner": "nix-community", "repo": "fenix", - "rev": "bab2a2840bc2d5ae7c6a133602185edbe4ca7daa", + "rev": "2f721f527886f801403f389a9cabafda8f1e3b7f", "type": "github" }, "original": { @@ -567,16 +567,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1737761947, - "narHash": "sha256-FqpAOeFGuA+luV36jaf5aVz3UB183n6wUrTbFxCwjjQ=", + "lastModified": 1737828695, + "narHash": "sha256-8Ev6zzhNPU798JNvU27a7gj5X+6SDG3jBweUkQ59DbA=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "d078ca31e802696b26d972bda7bed86ee1382156", + "rev": "a4d9230dcc9d03be428b9a728133f8f646c0065c", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.10.0", + "ref": "v9.9.3", "repo": "rocksdb", "type": "github" } @@ -599,11 +599,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1737634189, - "narHash": "sha256-AG5G9KDsl0Ngby9EfWvlemma7WWG0KCADTIccPJuzUE=", + "lastModified": 1737728869, + "narHash": "sha256-U4pl3Hi0lT6GP4ecN3q9wdD2sdaKMbmD/5NJ1NdJ9AM=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "84d44d0a574630aa8500ed62b6c01ccd3fae2473", + "rev": "6e4c29f7ce18cea7d3d31237a4661ab932eab636", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 1d38f80f..920d3d14 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.10.0"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.9.3"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From 9514064c1c709dc7c437b1478b224bb0d711ec05 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 25 Jan 2025 13:40:31 -0500 Subject: [PATCH 1412/2091] use --locked for macOS builds Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce662101..de6dbc77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -669,7 +669,7 @@ jobs: - name: Build macOS x86_64 binary if: ${{ matrix.os == 'macos-13' }} run: | - CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release + CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release --locked --features=perf_measurements,sentry_telemetry,direct_tls cp -v -f target/release/conduwuit conduwuit-macos-x86_64 otool -L conduwuit-macos-x86_64 @@ -677,12 +677,13 @@ jobs: - name: Run x86_64 macOS release binary if: ${{ matrix.os == 'macos-13' }} run: | + ./conduwuit-macos-x86_64 --help ./conduwuit-macos-x86_64 --version - name: Build macOS arm64 binary if: ${{ matrix.os == 'macos-latest' }} run: | - CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release + CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release --locked --features=perf_measurements,sentry_telemetry,direct_tls cp -v -f target/release/conduwuit conduwuit-macos-arm64 otool -L conduwuit-macos-arm64 @@ -690,6 +691,7 @@ jobs: - name: Run arm64 macOS release binary if: ${{ matrix.os == 'macos-latest' }} run: | + ./conduwuit-macos-arm64 --help ./conduwuit-macos-arm64 --version - name: Upload macOS x86_64 binary to webserver From 1d26eec82d8d75f3d67cd973482bc7aa604e6381 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 11 Jan 2025 14:12:08 +0000 Subject: [PATCH 1413/2091] publish README to docker hub --- .github/workflows/docker-hub-description.yml | 36 ++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/docker-hub-description.yml diff --git a/.github/workflows/docker-hub-description.yml b/.github/workflows/docker-hub-description.yml new file mode 100644 index 00000000..5ff5f666 --- /dev/null +++ b/.github/workflows/docker-hub-description.yml @@ -0,0 +1,36 @@ +name: Update Docker Hub Description + +on: + push: + branches: + - main + paths: + - README.md + - .github/workflows/docker-hub-description.yml + +jobs: + dockerHubDescription: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Setting variables + uses: actions/github-script@v7 + id: var + with: + script: | + const githubRepo = '${{ github.repository }}'.toLowerCase() + const repoId = githubRepo.split('/')[1] + + core.setOutput('github_repository', githubRepo) + const dockerRepo = '${{ vars.DOCKER_USERNAME }}'.toLowerCase() + '/' + repoId + core.setOutput('docker_repo', dockerRepo) + - name: Docker Hub Description + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: ${{ steps.var.outputs.docker_repo }} + short-description: ${{ github.event.repository.description }} + enable-url-completion: true From d86061084cf2d544b99e16890914001b116ab2ca Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 26 Jan 2025 01:05:41 +0000 Subject: [PATCH 1414/2091] Publish haswell images to image registries (#674) * push haswell images to docker repos * Add OCI labels to image * fixup! Add OCI labels to image * fixup! push haswell images to docker repos * fixup! Add OCI labels to image * fixup! Add OCI labels to image * fixup! Add OCI labels to image --- .github/workflows/ci.yml | 174 ++++++++++++++++++++------------- nix/pkgs/oci-image/default.nix | 6 ++ 2 files changed, 114 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de6dbc77..345713aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -566,6 +566,14 @@ jobs: cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz + - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub + if: ${{ matrix.target == 'x86_64-linux-musl' }} + uses: actions/upload-artifact@v4 + with: + name: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + path: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + if-no-files-found: error + compression-level: 0 - name: Upload OCI image ${{ matrix.target }}-all-features to GitHub uses: actions/upload-artifact@v4 with: @@ -745,18 +753,11 @@ jobs: contents: read if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' env: - DOCKER_ARM64: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 - DOCKER_AMD64: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 - DOCKER_TAG: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - DOCKER_BRANCH: docker.io/${{ needs.variables.outputs.github_repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} - GHCR_ARM64: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 - GHCR_AMD64: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 - GHCR_TAG: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - GHCR_BRANCH: ghcr.io/${{ needs.variables.outputs.github_repository }}:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} - GLCR_ARM64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-arm64v8 - GLCR_AMD64: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }}-amd64 - GLCR_TAG: registry.gitlab.com/conduwuit/conduwuit:${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - GLCR_BRANCH: registry.gitlab.com/conduwuit/conduwuit:${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + DOCKER_HUB_REPO: docker.io/${{ needs.variables.outputs.github_repository }} + GHCR_REPO: ghcr.io/${{ needs.variables.outputs.github_repository }} + GLCR_REPO: registry.gitlab.com/conduwuit/conduwuit + UNIQUE_TAG: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + BRANCH_TAG: ${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} @@ -790,143 +791,184 @@ jobs: - name: Move OCI images into position run: | + mv -v oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised/*.tar.gz oci-image-amd64-haswell-optimised.tar.gz mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz + - name: Load and push amd64 haswell image + run: | + docker load -i oci-image-amd64.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + fi + - name: Load and push amd64 image run: | docker load -i oci-image-amd64.tar.gz if [ ! -z $DOCKERHUB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64} - docker push ${DOCKER_AMD64} + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 fi if [ $GHCR_ENABLED = "true" ]; then - docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64} - docker push ${GHCR_AMD64} + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64 fi if [ ! -z $GITLAB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64} - docker push ${GLCR_AMD64} + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64 fi - name: Load and push arm64 image run: | docker load -i oci-image-arm64v8.tar.gz if [ ! -z $DOCKERHUB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64} - docker push ${DOCKER_ARM64} + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 fi if [ $GHCR_ENABLED = "true" ]; then - docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64} - docker push ${GHCR_ARM64} + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 fi if [ ! -z $GITLAB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64} - docker push ${GLCR_ARM64} + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 fi - name: Load and push amd64 debug image run: | docker load -i oci-image-amd64-debug.tar.gz if [ ! -z $DOCKERHUB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${DOCKER_AMD64}-debug - docker push ${DOCKER_AMD64}-debug + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug fi if [ $GHCR_ENABLED = "true" ]; then - docker tag $(docker images -q conduwuit:main) ${GHCR_AMD64}-debug - docker push ${GHCR_AMD64}-debug + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug fi if [ ! -z $GITLAB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${GLCR_AMD64}-debug - docker push ${GLCR_AMD64}-debug + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug fi - name: Load and push arm64 debug image run: | docker load -i oci-image-arm64v8-debug.tar.gz if [ ! -z $DOCKERHUB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${DOCKER_ARM64}-debug - docker push ${DOCKER_ARM64}-debug + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug fi if [ $GHCR_ENABLED = "true" ]; then - docker tag $(docker images -q conduwuit:main) ${GHCR_ARM64}-debug - docker push ${GHCR_ARM64}-debug + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug fi if [ ! -z $GITLAB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${GLCR_ARM64}-debug - docker push ${GLCR_ARM64}-debug + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + + - name: Create Docker haswell manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell fi - name: Create Docker combined manifests run: | # Dockerhub Container Registry if [ ! -z $DOCKERHUB_TOKEN ]; then - docker manifest create ${DOCKER_TAG} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64} - docker manifest create ${DOCKER_BRANCH} --amend ${DOCKER_ARM64} --amend ${DOCKER_AMD64} + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 fi # GitHub Container Registry if [ $GHCR_ENABLED = "true" ]; then - docker manifest create ${GHCR_TAG} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} - docker manifest create ${GHCR_BRANCH} --amend ${GHCR_ARM64} --amend ${GHCR_AMD64} + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GHCR_REPO}:${BRANCH_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 fi # GitLab Container Registry if [ ! -z $GITLAB_TOKEN ]; then - docker manifest create ${GLCR_TAG} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64} - docker manifest create ${GLCR_BRANCH} --amend ${GLCR_ARM64} --amend ${GLCR_AMD64} + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GLCR_REPO}:${BRANCH_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 fi - name: Create Docker combined debug manifests run: | # Dockerhub Container Registry if [ ! -z $DOCKERHUB_TOKEN ]; then - docker manifest create ${DOCKER_TAG}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug - docker manifest create ${DOCKER_BRANCH}-debug --amend ${DOCKER_ARM64}-debug --amend ${DOCKER_AMD64}-debug + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug fi # GitHub Container Registry if [ $GHCR_ENABLED = "true" ]; then - docker manifest create ${GHCR_TAG}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug - docker manifest create ${GHCR_BRANCH}-debug --amend ${GHCR_ARM64}-debug --amend ${GHCR_AMD64}-debug + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug fi # GitLab Container Registry if [ ! -z $GITLAB_TOKEN ]; then - docker manifest create ${GLCR_TAG}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug - docker manifest create ${GLCR_BRANCH}-debug --amend ${GLCR_ARM64}-debug --amend ${GLCR_AMD64}-debug + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug fi - name: Push manifests to Docker registries run: | if [ ! -z $DOCKERHUB_TOKEN ]; then - docker manifest push ${DOCKER_TAG} - docker manifest push ${DOCKER_BRANCH} - docker manifest push ${DOCKER_TAG}-debug - docker manifest push ${DOCKER_BRANCH}-debug + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell fi if [ $GHCR_ENABLED = "true" ]; then - docker manifest push ${GHCR_TAG} - docker manifest push ${GHCR_BRANCH} - docker manifest push ${GHCR_TAG}-debug - docker manifest push ${GHCR_BRANCH}-debug + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GHCR_REPO}:${BRANCH_TAG} + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-haswell fi if [ ! -z $GITLAB_TOKEN ]; then - docker manifest push ${GLCR_TAG} - docker manifest push ${GLCR_BRANCH} - docker manifest push ${GLCR_TAG}-debug - docker manifest push ${GLCR_BRANCH}-debug + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GLCR_REPO}:${BRANCH_TAG} + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-haswell fi - name: Add Image Links to Job Summary run: | if [ ! -z $DOCKERHUB_TOKEN ]; then - echo "- \`docker pull ${DOCKER_TAG}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${DOCKER_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY fi if [ $GHCR_ENABLED = "true" ]; then - echo "- \`docker pull ${GHCR_TAG}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${GHCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY fi if [ ! -z $GITLAB_TOKEN ]; then - echo "- \`docker pull ${GLCR_TAG}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${GLCR_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY fi diff --git a/nix/pkgs/oci-image/default.nix b/nix/pkgs/oci-image/default.nix index 152e00d1..d378d017 100644 --- a/nix/pkgs/oci-image/default.nix +++ b/nix/pkgs/oci-image/default.nix @@ -28,5 +28,11 @@ dockerTools.buildLayeredImage { Env = [ "RUST_BACKTRACE=full" ]; + Labels = { + "org.opencontainers.image.title" = main.pname; + "org.opencontainers.image.version" = main.version; + "org.opencontainers.image.revision" = inputs.self.rev or inputs.self.dirtyRev or ""; + # "org.opencontainers.image.created" = builtins.formatTime "%Y-%m-%dT%H:%M:%SZ" inputs.self.lastModified; + }; }; } From 9dcf289c7a1f57bfb512f406ddcfb22895e30846 Mon Sep 17 00:00:00 2001 From: bumpsoo Date: Sun, 5 Jan 2025 11:37:40 +0900 Subject: [PATCH 1415/2091] (doc): Update docker-compose.yml and conduwuit-example.toml The server cannot start without a registration token when registration is configured Signed-off-by: bumpsoo --- conduwuit-example.toml | 7 +++++-- docs/deploying/docker-compose.for-traefik.yml | 2 ++ docs/deploying/docker-compose.with-caddy.yml | 2 ++ docs/deploying/docker-compose.yml | 2 ++ src/core/config/mod.rs | 6 +++++- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 79efbd14..3ecc1628 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -389,13 +389,16 @@ # #allow_registration = false -# This item is undocumented. Please contribute documentation for it. +# Enabling this setting opens registration to anyone without restrictions. +# This makes your server vulnerable to abuse # #yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = false # A static registration token that new users will have to provide when # creating an account. If unset and `allow_registration` is true, -# registration is open without any condition. +# you must set +# `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` +# to true to allow open registration without any conditions. # # YOU NEED TO EDIT THIS OR USE registration_token_file. # diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index b4316426..366f6999 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -17,6 +17,8 @@ services: CONDUWUIT_PORT: 6167 # should match the loadbalancer traefik label CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB CONDUWUIT_ALLOW_REGISTRATION: 'true' + CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed. + #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true' CONDUWUIT_ALLOW_FEDERATION: 'true' CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index c080293f..431cf2d4 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -33,6 +33,8 @@ services: CONDUWUIT_PORT: 6167 CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB CONDUWUIT_ALLOW_REGISTRATION: 'true' + CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed. + #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true' CONDUWUIT_ALLOW_FEDERATION: 'true' CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' diff --git a/docs/deploying/docker-compose.yml b/docs/deploying/docker-compose.yml index 3b7d84ed..ca33b5f5 100644 --- a/docs/deploying/docker-compose.yml +++ b/docs/deploying/docker-compose.yml @@ -17,6 +17,8 @@ services: CONDUWUIT_PORT: 6167 CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB CONDUWUIT_ALLOW_REGISTRATION: 'true' + CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed. + #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true' CONDUWUIT_ALLOW_FEDERATION: 'true' CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index c541c7e4..133f0887 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -492,12 +492,16 @@ pub struct Config { #[serde(default)] pub allow_registration: bool, + /// Enabling this setting opens registration to anyone without restrictions. + /// This makes your server vulnerable to abuse #[serde(default)] pub yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse: bool, /// A static registration token that new users will have to provide when /// creating an account. If unset and `allow_registration` is true, - /// registration is open without any condition. + /// you must set + /// `yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse` + /// to true to allow open registration without any conditions. /// /// YOU NEED TO EDIT THIS OR USE registration_token_file. /// From 5b5ccba64e3d36a9235f4e0d449f40d859046dad Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 25 Jan 2025 21:14:38 -0500 Subject: [PATCH 1416/2091] make conduwuit lowercase in the user-agent again Signed-off-by: June Clementine Strawberry --- src/core/info/version.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/info/version.rs b/src/core/info/version.rs index fb71d4e1..37580210 100644 --- a/src/core/info/version.rs +++ b/src/core/info/version.rs @@ -7,7 +7,7 @@ use std::sync::OnceLock; -static BRANDING: &str = "Conduwuit"; +static BRANDING: &str = "conduwuit"; static SEMANTIC: &str = env!("CARGO_PKG_VERSION"); static VERSION: OnceLock = OnceLock::new(); From c323894497e263514e92bfe0dee8397085305bc0 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 25 Jan 2025 21:20:29 -0500 Subject: [PATCH 1417/2091] use test in postrm deb script before deleting Signed-off-by: June Clementine Strawberry --- debian/postrm | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/debian/postrm b/debian/postrm index f516f2a2..3c0b1c09 100644 --- a/debian/postrm +++ b/debian/postrm @@ -10,21 +10,33 @@ CONDUWUIT_DATABASE_PATH_SYMLINK=/var/lib/matrix-conduit case $1 in purge) # Remove debconf changes from the db - db_purge + #db_purge # Per https://www.debian.org/doc/debian-policy/ch-files.html#behavior # "configuration files must be preserved when the package is removed, and # only deleted when the package is purged." + + # + if [ -d "$CONDUWUIT_CONFIG_PATH" ]; then - rm -v -r "$CONDUWUIT_CONFIG_PATH" + if test -L "$CONDUWUIT_CONFIG_PATH"; then + echo "Deleting conduwuit configuration files" + rm -v -r "$CONDUWUIT_CONFIG_PATH" + fi fi if [ -d "$CONDUWUIT_DATABASE_PATH" ]; then - rm -v -r "$CONDUWUIT_DATABASE_PATH" + if test -L "$CONDUWUIT_DATABASE_PATH"; then + echo "Deleting conduwuit database directory" + rm -r "$CONDUWUIT_DATABASE_PATH" + fi fi if [ -d "$CONDUWUIT_DATABASE_PATH_SYMLINK" ]; then - rm -v -r "$CONDUWUIT_DATABASE_PATH_SYMLINK" + if test -L "$CONDUWUIT_DATABASE_SYMLINK"; then + echo "Removing matrix-conduit symlink" + rm -r "$CONDUWUIT_DATABASE_PATH_SYMLINK" + fi fi ;; esac From 4b331fe50e568241f6703d92b005149da9dc4a52 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 25 Jan 2025 22:42:16 -0500 Subject: [PATCH 1418/2091] update README.md, crate metadata, and OCI image metadata Signed-off-by: June Clementine Strawberry --- Cargo.toml | 9 +++-- README.md | 72 ++++++++++++++++++++++++---------- nix/pkgs/oci-image/default.nix | 13 ++++-- 3 files changed, 66 insertions(+), 28 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d52ce974..c4af4a7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,14 +7,15 @@ default-members = ["src/*"] [workspace.package] authors = [ - "strawberry ", - "timokoesters ", + "June Clementine Strawberry ", + "strawberry ", # woof + "Jason Volk ", ] categories = ["network-programming"] -description = "a very cool fork of Conduit, a Matrix homeserver written in Rust" +description = "a very cool Matrix chat homeserver written in Rust" edition = "2021" homepage = "https://conduwuit.puppyirl.gay/" -keywords = ["chat", "matrix", "server", "uwu"] +keywords = ["chat", "matrix", "networking", "server", "uwu"] license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" diff --git a/README.md b/README.md index 74b6bddf..13a1c67f 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ -### a very cool, featureful fork of [Conduit](https://conduit.rs/) +### a very cool [Matrix](https://matrix.org/) chat homeserver written in Rust @@ -15,16 +15,15 @@ information and how to deploy/setup conduwuit. #### What is Matrix? -[Matrix](https://matrix.org) is an open network for secure and decentralized -communication. Users from every Matrix homeserver can chat with users from all -other Matrix servers. You can even use bridges (also called Matrix Appservices) -to communicate with users outside of Matrix, like a community on Discord. +[Matrix](https://matrix.org) is an open, federated, and extensible network for +decentralised communication. Users from any Matrix homeserver can chat with users from all +other homeservers over federation. Matrix is designed to be extensible and built on top of. +You can even use bridges such as Matrix Appservices to communicate with users outside of Matrix, like a community on Discord. #### What is the goal? -A high-performance and efficient Matrix homeserver that's easy to set up and -just works. You can install it on a mini-computer like the Raspberry Pi to -host Matrix for your family, friends or company. +A high-performance, efficient, low-cost, and featureful Matrix homeserver that's +easy to set up and just works with minimal configuration needed. #### Can I try it out? @@ -37,17 +36,22 @@ homeserver". This means there are rules, so please read the rules: [https://transfem.dev/homeserver_rules.txt](https://transfem.dev/homeserver_rules.txt) transfem.dev is also listed at -[servers.joinmatrix.org](https://servers.joinmatrix.org/) +[servers.joinmatrix.org](https://servers.joinmatrix.org/), which is a list of +popular public Matrix homeservers, including some others that run conduwuit. #### What is the current status? -conduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status -initially was inherited from Conduit, however overtime this Beta status is rapidly -becoming less and less relevant as our codebase significantly diverges more and more. +conduwuit is technically a hard fork of [Conduit](https://conduit.rs/), which is in beta. +The beta status initially was inherited from Conduit, however the huge amount of +codebase divergance, changes, fixes, and improvements have effectively made this +beta status not entirely applicable to us anymore. -conduwuit is quite stable and very usable as a daily driver and for a low-medium -sized homeserver. There is still a lot of more work to be done, but it is in a far -better place than the project was in early 2024. +conduwuit is very stable based on our rapidly growing userbase, has lots of features that users +expect, and very usable as a daily driver for small, medium, and upper-end medium sized homeservers. + +A lot of critical stability and performance issues have been fixed, and a lot of +necessary groundwork has finished; making this project way better than it was +back in the start at ~early 2024. #### How is conduwuit funded? Is conduwuit sustainable? @@ -72,16 +76,37 @@ Conduit like before. If you are truly finding yourself wanting to migrate back to Conduit, we would appreciate all your feedback and if we can assist with any issues or concerns. +#### Can I migrate from Synapse or Dendrite? + +Currently there is no known way to seamlessly migrate all user data from the old +homeserver to conduwuit. However it is perfectly acceptable to replace the old +homeserver software with conduwuit using the same server name and there will not +be any issues with federation. + +There is an interest in developing a built-in seamless user data migration +method into conduwuit, however there is no concrete ETA or timeline for this. + + #### Contact -If you run into any question, feel free to +[`#conduwuit:puppygock.gay`](https://matrix.to/#/#conduwuit:puppygock.gay) +is the official project Matrix room. You can get support here, ask questions or +concerns, get assistance setting up conduwuit, etc. -- Ask us in `#conduwuit:puppygock.gay` on Matrix -- [Open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new) +This room should stay relevant and focused on conduwuit. An offtopic general +chatter room can be found there as well. + +Please keep the issue trackers focused on bug reports and enhancement requests. +General support is extremely difficult to be offered over an issue tracker, and +simple questions should be asked directly in an interactive platform like our +Matrix room above as they can turn into a relevant discussion and/or may not be +simple to answer. If you're not sure, just ask in the Matrix room. + +If you have a bug or feature to request: [Open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new) #### Donate @@ -89,9 +114,11 @@ conduwuit development is purely made possible by myself and contributors. I do not get paid to work on this, and I work on it in my free time. Donations are heavily appreciated! 💜🥺 -- Liberapay: -- Ko-fi (note they take a fee): -- GitHub Sponsors: +- Liberapay (preferred): +- GitHub Sponsors (preferred): +- Ko-fi: + +I do not and will not accept cryptocurrency donations, including things related. #### Logo @@ -105,6 +132,9 @@ Both, but I prefer conduwuit. #### Mirrors of conduwuit +If GitHub is unavailable in your country, or has poor connectivity, conduwuit's +source code is mirrored onto the following additional platforms I maintain: + - GitHub: - GitLab: - git.girlcock.ceo: diff --git a/nix/pkgs/oci-image/default.nix b/nix/pkgs/oci-image/default.nix index d378d017..5520c920 100644 --- a/nix/pkgs/oci-image/default.nix +++ b/nix/pkgs/oci-image/default.nix @@ -29,10 +29,17 @@ dockerTools.buildLayeredImage { "RUST_BACKTRACE=full" ]; Labels = { - "org.opencontainers.image.title" = main.pname; - "org.opencontainers.image.version" = main.version; + "org.opencontainers.image.authors" = "June Clementine Strawberry and Jason Volk + "; + "org.opencontainers.image.created" ="@${toString inputs.self.lastModified}"; + "org.opencontainers.image.description" = "a very cool Matrix chat homeserver written in Rust"; + "org.opencontainers.image.documentation" = "https://conduwuit.puppyirl.gay/"; + "org.opencontainers.image.licenses" = "Apache-2.0"; "org.opencontainers.image.revision" = inputs.self.rev or inputs.self.dirtyRev or ""; - # "org.opencontainers.image.created" = builtins.formatTime "%Y-%m-%dT%H:%M:%SZ" inputs.self.lastModified; + "org.opencontainers.image.title" = main.pname; + "org.opencontainers.image.url" = "https://conduwuit.puppyirl.gay/"; + "org.opencontainers.image.vendor" = "girlbossceo"; + "org.opencontainers.image.version" = main.version; }; }; } From 3b0195e6b387364d8919ce90e2f461e82d2f51d1 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 25 Jan 2025 21:04:52 -0500 Subject: [PATCH 1419/2091] misc various github actions ci fixes Signed-off-by: June Clementine Strawberry --- .gitea/PULL_REQUEST_TEMPLATE.md | 8 - .gitea/workflows/ci.yml | 264 ------------------- .github/workflows/ci.yml | 80 +++--- .github/workflows/docker-hub-description.yml | 5 + .github/workflows/documentation.yml | 3 + 5 files changed, 56 insertions(+), 304 deletions(-) delete mode 100644 .gitea/PULL_REQUEST_TEMPLATE.md delete mode 100644 .gitea/workflows/ci.yml diff --git a/.gitea/PULL_REQUEST_TEMPLATE.md b/.gitea/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 4210554b..00000000 --- a/.gitea/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,8 +0,0 @@ - - - ------------------------------------------------------------------------------ - -- [ ] I ran `cargo fmt`, `cargo clippy`, and `cargo test` -- [ ] I agree to release my code and all other changes of this MR under the Apache-2.0 license - diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml deleted file mode 100644 index ef436734..00000000 --- a/.gitea/workflows/ci.yml +++ /dev/null @@ -1,264 +0,0 @@ -name: CI and Artifacts - -on: - pull_request: - push: - # documentation workflow deals with this or is not relevant for this workflow - paths-ignore: - - '*.md' - - 'conduwuit-example.toml' - - 'book.toml' - - '.gitlab-ci.yml' - - '.gitignore' - - 'renovate.json' - - 'docs/**' - - 'debian/**' - - 'docker/**' - branches: - - main - tags: - - '*' - # Allows you to run this workflow manually from the Actions tab - #workflow_dispatch: - -#concurrency: -# group: ${{ gitea.head_ref || gitea.ref_name }} -# cancel-in-progress: true - -env: - # Required to make some things output color - TERM: ansi - # Publishing to my nix binary cache - ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} - # conduwuit.cachix.org - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - # Just in case incremental is still being set to true, speeds up CI - CARGO_INCREMENTAL: 0 - # Custom nix binary cache if fork is being used - ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} - ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} - # Get error output from nix that we can actually use - NIX_CONFIG: show-trace = true - -#permissions: -# packages: write -# contents: read - -jobs: - tests: - name: Test - runs-on: ubuntu-latest - steps: - - name: Sync repository - uses: https://github.com/actions/checkout@v4 - - - name: Tag comparison check - if: startsWith(gitea.ref, 'refs/tags/v') - run: | - # Tag mismatch with latest repo tag check to prevent potential downgrades - LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) - - if [ $LATEST_TAG != ${{ gitea.ref_name }} ]; then - echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' - echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - - name: Install Nix - uses: https://github.com/DeterminateSystems/nix-installer-action@main - with: - diagnostic-endpoint: "" - extra-conf: | - experimental-features = nix-command flakes - accept-flake-config = true - - - name: Enable Cachix binary cache - run: | - nix profile install nixpkgs#cachix - cachix use crane - cachix use nix-community - - - name: Configure Magic Nix Cache - uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main - with: - diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" - - - name: Apply Nix binary cache configuration - run: | - sudo tee -a /etc/nix/nix.conf > /dev/null < /dev/null < "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv - direnv allow - nix develop .#all-features --command true - - - name: Cache CI dependencies - run: | - bin/nix-build-and-cache ci - - - name: Run CI tests - run: | - direnv exec . engage > >(tee -a test_output.log) - - - name: Sync Complement repository - uses: https://github.com/actions/checkout@v4 - with: - repository: 'matrix-org/complement' - path: complement_src - - - name: Run Complement tests - run: | - direnv exec . bin/complement 'complement_src' 'complement_test_logs.jsonl' 'complement_test_results.jsonl' - cp -v -f result complement_oci_image.tar.gz - - - name: Upload Complement OCI image - uses: https://github.com/actions/upload-artifact@v4 - with: - name: complement_oci_image.tar.gz - path: complement_oci_image.tar.gz - if-no-files-found: error - - - name: Upload Complement logs - uses: https://github.com/actions/upload-artifact@v4 - with: - name: complement_test_logs.jsonl - path: complement_test_logs.jsonl - if-no-files-found: error - - - name: Upload Complement results - uses: https://github.com/actions/upload-artifact@v4 - with: - name: complement_test_results.jsonl - path: complement_test_results.jsonl - if-no-files-found: error - - - name: Diff Complement results with checked-in repo results - run: | - diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_test_output.log) - echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - - name: Update Job Summary - if: success() || failure() - run: | - if [ ${{ job.status }} == 'success' ]; then - echo '# ✅ completed suwuccessfully' >> $GITHUB_STEP_SUMMARY - else - echo '```' >> $GITHUB_STEP_SUMMARY - tail -n 40 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - fi - - build: - name: Build - runs-on: ubuntu-latest - needs: tests - strategy: - matrix: - include: - - target: aarch64-unknown-linux-musl - - target: x86_64-unknown-linux-musl - steps: - - name: Sync repository - uses: https://github.com/actions/checkout@v4 - - - name: Install Nix - uses: https://github.com/DeterminateSystems/nix-installer-action@main - with: - diagnostic-endpoint: "" - extra-conf: | - experimental-features = nix-command flakes - accept-flake-config = true - - - name: Install and enable Cachix binary cache - run: | - nix profile install nixpkgs#cachix - cachix use crane - cachix use nix-community - - - name: Configure Magic Nix Cache - uses: https://github.com/DeterminateSystems/magic-nix-cache-action@main - with: - diagnostic-endpoint: "" - upstream-cache: "https://attic.kennel.juneis.dog/conduwuit" - - - name: Apply Nix binary cache configuration - run: | - sudo tee -a /etc/nix/nix.conf > /dev/null < /dev/null < "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv - direnv allow - nix develop .#all-features --command true - - - name: Build static ${{ matrix.target }} - run: | - CARGO_DEB_TARGET_TUPLE=$(echo ${{ matrix.target }} | grep -o -E '^([^-]*-){3}[^-]*') - SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - - bin/nix-build-and-cache just .#static-${{ matrix.target }} - mkdir -v -p target/release/ - mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ - cp -v -f result/bin/conduit target/release/conduwuit - cp -v -f result/bin/conduit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit - # -p conduit is the main crate name - direnv exec . cargo deb --verbose --no-build --no-strip -p conduit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb - mv -v target/release/conduwuit static-${{ matrix.target }} - mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb - - - name: Upload static-${{ matrix.target }} - uses: https://github.com/actions/upload-artifact@v4 - with: - name: static-${{ matrix.target }} - path: static-${{ matrix.target }} - if-no-files-found: error - - - name: Upload deb ${{ matrix.target }} - uses: https://github.com/actions/upload-artifact@v4 - with: - name: deb-${{ matrix.target }} - path: ${{ matrix.target }}.deb - if-no-files-found: error - compression-level: 0 - - - name: Build OCI image ${{ matrix.target }} - run: | - bin/nix-build-and-cache just .#oci-image-${{ matrix.target }} - cp -v -f result oci-image-${{ matrix.target }}.tar.gz - - - name: Upload OCI image ${{ matrix.target }} - uses: https://github.com/actions/upload-artifact@v4 - with: - name: oci-image-${{ matrix.target }} - path: oci-image-${{ matrix.target }}.tar.gz - if-no-files-found: error - compression-level: 0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 345713aa..b0b0bd53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,8 +22,8 @@ concurrency: env: # sccache only on main repo - SCCACHE_GHA_ENABLED: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'true' || 'false' }}" - RUSTC_WRAPPER: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}" + SCCACHE_GHA_ENABLED: "${{ !startsWith(github.ref, 'refs/tags/') && (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'true' || 'false' }}" + RUSTC_WRAPPER: "${{ !startsWith(github.ref, 'refs/tags/') && (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}" SCCACHE_BUCKET: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}" SCCACHE_S3_USE_SSL: ${{ vars.SCCACHE_S3_USE_SSL }} SCCACHE_REGION: ${{ vars.SCCACHE_REGION }} @@ -51,8 +51,8 @@ env: extra-experimental-features = nix-command flakes accept-flake-config = true WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} - GH_SHA: ${{ github.sha }} GH_REF_NAME: ${{ github.ref_name }} + WEBSERVER_DIR_NAME: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} permissions: {} @@ -85,11 +85,13 @@ jobs: END echo "Checking connection" - ssh -q website "echo test" + ssh -q website "echo test" || ssh -q website "echo test" echo "Creating commit rev directory on web server" - ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/$GITHUB_SHA/" - ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/$GITHUB_SHA/" + ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" || ssh -q website "rm -rf /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" + ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" || ssh -q website "mkdir -v /var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/" + + echo "SSH_WEBSITE=1" >> "$GITHUB_ENV" - name: Install liburing run: | @@ -124,6 +126,9 @@ jobs: - uses: nixbuild/nix-quick-install-action@master - name: Restore and cache Nix store + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + if: ${{ !startsWith(github.ref, 'refs/tags/') }} uses: nix-community/cache-nix-action@v5.1.0 with: # restore and save a cache using this key @@ -184,11 +189,16 @@ jobs: # use sccache for Rust - name: Run sccache-cache - if: (env.SCCACHE_GHA_ENABLED == 'true') + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} uses: mozilla-actions/sccache-action@main # use rust-cache - uses: Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + if: ${{ !startsWith(github.ref, 'refs/tags/') }} with: cache-all-crates: "true" cache-on-failure: "true" @@ -304,13 +314,16 @@ jobs: END echo "Checking connection" - ssh -q website "echo test" + ssh -q website "echo test" || ssh -q website "echo test" echo "SSH_WEBSITE=1" >> "$GITHUB_ENV" - uses: nixbuild/nix-quick-install-action@master - name: Restore and cache Nix store + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + if: ${{ !startsWith(github.ref, 'refs/tags/') }} uses: nix-community/cache-nix-action@v5.1.0 with: # restore and save a cache using this key @@ -364,11 +377,16 @@ jobs: # use sccache for Rust - name: Run sccache-cache - if: (env.SCCACHE_GHA_ENABLED == 'true') + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} uses: mozilla-actions/sccache-action@main # use rust-cache - uses: Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + if: ${{ !startsWith(github.ref, 'refs/tags/') }} with: cache-all-crates: "true" cache-on-failure: "true" @@ -493,29 +511,27 @@ jobs: run: | if [ ! -z $SSH_WEBSITE ]; then chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised - scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-x86_64-linux-musl-x86_64-haswell-optimised + scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-x86_64-linux-musl-x86_64-haswell-optimised fi - name: Upload static-${{ matrix.target }}-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $SSH_WEBSITE ]; then chmod +x static-${{ matrix.target }} - scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }} + scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }} fi - name: Upload static deb x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver if: ${{ matrix.target == 'x86_64-linux-musl' }} run: | if [ ! -z $SSH_WEBSITE ]; then - scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/x86_64-linux-musl-x86_64-haswell-optimised.deb + scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/x86_64-linux-musl-x86_64-haswell-optimised.deb fi - name: Upload static deb ${{ matrix.target }}-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $SSH_WEBSITE ]; then - scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}.deb + scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}.deb fi - name: Upload static-${{ matrix.target }}-debug-all-features to GitHub @@ -534,17 +550,15 @@ jobs: compression-level: 0 - name: Upload static-${{ matrix.target }}-debug-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $SSH_WEBSITE ]; then - scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/static-${{ matrix.target }}-debug + scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}-debug fi - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | - if [ ! -z $SSH_WEBSITE]; then - scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/${{ matrix.target }}-debug.deb + if [ ! -z $SSH_WEBSITE ]; then + scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}-debug.deb fi - name: Build OCI image ${{ matrix.target }}-all-features @@ -594,21 +608,19 @@ jobs: if: ${{ matrix.target == 'x86_64-linux-musl' }} run: | if [ ! -z $SSH_WEBSITE ]; then - scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz fi - name: Upload OCI image ${{ matrix.target }}-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $SSH_WEBSITE ]; then - scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}.tar.gz + scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}.tar.gz fi - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' run: | if [ ! -z $SSH_WEBSITE ]; then - scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/oci-image-${{ matrix.target }}-debug.tar.gz + scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}-debug.tar.gz fi build_mac_binaries: @@ -647,7 +659,9 @@ jobs: END echo "Checking connection" - ssh -q website "echo test" + ssh -q website "echo test" || ssh -q website "echo test" + + echo "SSH_WEBSITE=1" >> "$GITHUB_ENV" - name: Tag comparison check if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} @@ -663,7 +677,9 @@ jobs: # use sccache for Rust - name: Run sccache-cache - if: (env.SCCACHE_GHA_ENABLED == 'true') + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} uses: mozilla-actions/sccache-action@main # use rust-cache @@ -707,7 +723,7 @@ jobs: run: | if [ ! -z $SSH_WEBSITE ]; then chmod +x conduwuit-macos-x86_64 - scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-x86_64 + scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/conduwuit-macos-x86_64 fi - name: Upload macOS arm64 binary to webserver @@ -715,7 +731,7 @@ jobs: run: | if [ ! -z $SSH_WEBSITE ]; then chmod +x conduwuit-macos-arm64 - scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${GH_SHA}/conduwuit-macos-arm64 + scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/conduwuit-macos-arm64 fi - name: Upload macOS x86_64 binary @@ -881,15 +897,15 @@ jobs: run: | # Dockerhub Container Registry if [ ! -z $DOCKERHUB_TOKEN ]; then - docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell fi # GitHub Container Registry if [ $GHCR_ENABLED = "true" ]; then - docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${BRANCH_TAG}-haswell fi # GitLab Container Registry if [ ! -z $GITLAB_TOKEN ]; then - docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${BRANCH_TAG}-haswell fi - name: Create Docker combined manifests diff --git a/.github/workflows/docker-hub-description.yml b/.github/workflows/docker-hub-description.yml index 5ff5f666..96b2d38b 100644 --- a/.github/workflows/docker-hub-description.yml +++ b/.github/workflows/docker-hub-description.yml @@ -8,13 +8,17 @@ on: - README.md - .github/workflows/docker-hub-description.yml + workflow_dispatch: + jobs: dockerHubDescription: runs-on: ubuntu-latest + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' steps: - uses: actions/checkout@v4 with: persist-credentials: false + - name: Setting variables uses: actions/github-script@v7 id: var @@ -26,6 +30,7 @@ jobs: core.setOutput('github_repository', githubRepo) const dockerRepo = '${{ vars.DOCKER_USERNAME }}'.toLowerCase() + '/' + repoId core.setOutput('docker_repo', dockerRepo) + - name: Docker Hub Description uses: peter-evans/dockerhub-description@v4 with: diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index b0ccdb47..0eefe0a4 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -73,6 +73,9 @@ jobs: - uses: nixbuild/nix-quick-install-action@master - name: Restore and cache Nix store + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + if: ${{ !startsWith(github.ref, 'refs/tags/') }} uses: nix-community/cache-nix-action@v5.1.0 with: # restore and save a cache using this key From 29a19ba437c6b387f3f250a2d91e2edd6d751a18 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 25 Jan 2025 02:06:20 +0000 Subject: [PATCH 1420/2091] add write_to_cache to descriptor Signed-off-by: Jason Volk --- src/database/engine/cf_opts.rs | 6 +++++- src/database/engine/descriptor.rs | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 1230081c..ab11b9e1 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -77,20 +77,24 @@ fn descriptor_cf_options( fn set_table_options(opts: &mut Options, desc: &Descriptor, cache: Option<&Cache>) -> Result { let mut table = table_options(desc, cache.is_some()); + if let Some(cache) = cache { table.set_block_cache(cache); } else { table.disable_cache(); } + let prepopulate = if desc.write_to_cache { "kFlushOnly" } else { "kDisable" }; + let string = format!( "{{block_based_table_factory={{num_file_reads_for_auto_readahead={0};\ max_auto_readahead_size={1};initial_auto_readahead_size={2};\ - enable_index_compression={3}}}}}", + enable_index_compression={3};prepopulate_block_cache={4}}}}}", desc.auto_readahead_thresh, desc.auto_readahead_max, desc.auto_readahead_init, desc.compressed_index, + prepopulate, ); opts.set_options_from_string(&string).map_err(map_err)?; diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index 6ce8b5ad..c4dc2901 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -40,6 +40,7 @@ pub(crate) struct Descriptor { pub(crate) bottommost_level: Option, pub(crate) block_index_hashing: Option, pub(crate) cache_shards: u32, + pub(crate) write_to_cache: bool, pub(crate) auto_readahead_thresh: u32, pub(crate) auto_readahead_init: usize, pub(crate) auto_readahead_max: usize, @@ -71,6 +72,7 @@ pub(crate) static BASE: Descriptor = Descriptor { bottommost_level: Some(SENTINEL_COMPRESSION_LEVEL), block_index_hashing: None, cache_shards: 64, + write_to_cache: false, auto_readahead_thresh: 0, auto_readahead_init: 1024 * 16, auto_readahead_max: 1024 * 1024 * 2, From 186c459584f3a25f00d873c5f73c820161226791 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 25 Jan 2025 02:06:44 +0000 Subject: [PATCH 1421/2091] use scalar for file shape; increase shape for small-type columns Signed-off-by: Jason Volk --- src/database/engine/cf_opts.rs | 2 +- src/database/engine/descriptor.rs | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index ab11b9e1..382bc169 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -33,7 +33,7 @@ fn descriptor_cf_options( opts.set_write_buffer_size(desc.write_size); opts.set_target_file_size_base(desc.file_size); - opts.set_target_file_size_multiplier(desc.file_shape[0]); + opts.set_target_file_size_multiplier(desc.file_shape); opts.set_level_zero_file_num_compaction_trigger(desc.level0_width); opts.set_level_compaction_dynamic_level_bytes(false); diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index c4dc2901..c735f402 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -27,7 +27,7 @@ pub(crate) struct Descriptor { pub(crate) level_size: u64, pub(crate) level_shape: [i32; 7], pub(crate) file_size: u64, - pub(crate) file_shape: [i32; 1], + pub(crate) file_shape: i32, pub(crate) level0_width: i32, pub(crate) merge_width: (i32, i32), pub(crate) ttl: u64, @@ -59,7 +59,7 @@ pub(crate) static BASE: Descriptor = Descriptor { level_size: 1024 * 1024 * 8, level_shape: [1, 1, 1, 3, 7, 15, 31], file_size: 1024 * 1024, - file_shape: [2], + file_shape: 2, level0_width: 2, merge_width: (2, 16), ttl: 60 * 60 * 24 * 21, @@ -106,6 +106,7 @@ pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { write_size: 1024 * 1024 * 16, level_size: 1024 * 512, file_size: 1024 * 128, + file_shape: 3, index_size: 512, block_size: 512, cache_shards: 64, @@ -121,6 +122,7 @@ pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { write_size: 1024 * 1024 * 16, level_size: 1024 * 1024, file_size: 1024 * 512, + file_shape: 3, block_size: 512, cache_shards: 64, block_index_hashing: Some(false), From 9ad4f20da4547f66720e363c92fac13a5c3af343 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 25 Jan 2025 08:59:48 +0000 Subject: [PATCH 1422/2091] propagate underflow as error result, not index bounds panic Signed-off-by: Jason Volk --- src/database/de.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/database/de.rs b/src/database/de.rs index 4fdc2251..7cc8f00a 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -298,9 +298,11 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { fn deserialize_i64>(self, visitor: V) -> Result { const BYTES: usize = size_of::(); - let end = self.pos.saturating_add(BYTES); + let end = self.pos.saturating_add(BYTES).min(self.buf.len()); let bytes: ArrayVec = self.buf[self.pos..end].try_into()?; - let bytes = bytes.into_inner().expect("array size matches i64"); + let bytes = bytes + .into_inner() + .map_err(|_| Self::Error::SerdeDe("i64 buffer underflow".into()))?; self.inc_pos(BYTES); visitor.visit_i64(i64::from_be_bytes(bytes)) @@ -328,9 +330,11 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { fn deserialize_u64>(self, visitor: V) -> Result { const BYTES: usize = size_of::(); - let end = self.pos.saturating_add(BYTES); + let end = self.pos.saturating_add(BYTES).min(self.buf.len()); let bytes: ArrayVec = self.buf[self.pos..end].try_into()?; - let bytes = bytes.into_inner().expect("array size matches u64"); + let bytes = bytes + .into_inner() + .map_err(|_| Self::Error::SerdeDe("u64 buffer underflow".into()))?; self.inc_pos(BYTES); visitor.visit_u64(u64::from_be_bytes(bytes)) From 68856645ee4f5a6a437b01c4c94bdc233d99f140 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 25 Jan 2025 07:18:33 +0000 Subject: [PATCH 1423/2091] refactor lazy-loading Signed-off-by: Jason Volk --- src/api/client/context.rs | 109 ++++--- src/api/client/message.rs | 151 ++++----- src/api/client/sync/v3.rs | 422 ++++++++++---------------- src/service/rooms/lazy_loading/mod.rs | 198 +++++++----- 4 files changed, 418 insertions(+), 462 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index b957561c..388bcf4d 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,6 +1,6 @@ use axum::extract::State; use conduwuit::{ - at, err, ref_at, + at, deref_at, err, ref_at, utils::{ future::TryExtExt, stream::{BroadbandExt, ReadyExt, TryIgnore, WidebandExt}, @@ -8,15 +8,15 @@ use conduwuit::{ }, Err, PduEvent, Result, }; -use futures::{join, try_join, FutureExt, StreamExt, TryFutureExt}; -use ruma::{ - api::client::{context::get_context, filter::LazyLoadOptions}, - events::StateEventType, - OwnedEventId, UserId, +use futures::{ + future::{join, join3, try_join3, OptionFuture}, + FutureExt, StreamExt, TryFutureExt, }; +use ruma::{api::client::context::get_context, events::StateEventType, OwnedEventId, UserId}; +use service::rooms::{lazy_loading, lazy_loading::Options}; use crate::{ - client::message::{event_filter, ignored_filter, update_lazy, visibility_filter, LazySet}, + client::message::{event_filter, ignored_filter, lazy_loading_witness, visibility_filter}, Ruma, }; @@ -33,10 +33,10 @@ pub(crate) async fn get_context_route( State(services): State, body: Ruma, ) -> Result { - let filter = &body.filter; let sender = body.sender(); - let (sender_user, _) = sender; + let (sender_user, sender_device) = sender; let room_id = &body.room_id; + let filter = &body.filter; // Use limit or else 10, with maximum 100 let limit: usize = body @@ -45,18 +45,6 @@ pub(crate) async fn get_context_route( .unwrap_or(LIMIT_DEFAULT) .min(LIMIT_MAX); - // some clients, at least element, seem to require knowledge of redundant - // members for "inline" profiles on the timeline to work properly - let lazy_load_enabled = matches!(filter.lazy_load_options, LazyLoadOptions::Enabled { .. }); - - let lazy_load_redundant = if let LazyLoadOptions::Enabled { include_redundant_members } = - filter.lazy_load_options - { - include_redundant_members - } else { - false - }; - let base_id = services .rooms .timeline @@ -75,7 +63,7 @@ pub(crate) async fn get_context_route( .user_can_see_event(sender_user, &body.room_id, &body.event_id) .map(Ok); - let (base_id, base_pdu, visible) = try_join!(base_id, base_pdu, visible)?; + let (base_id, base_pdu, visible) = try_join3(base_id, base_pdu, visible).await?; if base_pdu.room_id != body.room_id || base_pdu.event_id != body.event_id { return Err!(Request(NotFound("Base event not found."))); @@ -112,12 +100,32 @@ pub(crate) async fn get_context_route( .collect(); let (base_event, events_before, events_after): (_, Vec<_>, Vec<_>) = - join!(base_event, events_before, events_after); + join3(base_event, events_before, events_after).await; + + let lazy_loading_context = lazy_loading::Context { + user_id: sender_user, + device_id: sender_device, + room_id, + token: Some(base_count.into_unsigned()), + options: Some(&filter.lazy_load_options), + }; + + let lazy_loading_witnessed: OptionFuture<_> = filter + .lazy_load_options + .is_enabled() + .then_some( + base_event + .iter() + .chain(events_before.iter()) + .chain(events_after.iter()), + ) + .map(|witnessed| lazy_loading_witness(&services, &lazy_loading_context, witnessed)) + .into(); let state_at = events_after .last() .map(ref_at!(1)) - .map_or(body.event_id.as_ref(), |e| e.event_id.as_ref()); + .map_or(body.event_id.as_ref(), |pdu| pdu.event_id.as_ref()); let state_ids = services .rooms @@ -126,41 +134,32 @@ pub(crate) async fn get_context_route( .or_else(|_| services.rooms.state.get_room_shortstatehash(room_id)) .and_then(|shortstatehash| services.rooms.state_accessor.state_full_ids(shortstatehash)) .map_err(|e| err!(Database("State not found: {e}"))) - .await?; + .boxed(); - let lazy = base_event - .iter() - .chain(events_before.iter()) - .chain(events_after.iter()) - .stream() - .fold(LazySet::new(), |lazy, item| { - update_lazy(&services, room_id, sender, lazy, item, lazy_load_redundant) - }) - .await; + let (lazy_loading_witnessed, state_ids) = join(lazy_loading_witnessed, state_ids).await; - let lazy = &lazy; - let state: Vec<_> = state_ids - .iter() - .stream() - .broad_filter_map(|(shortstatekey, event_id)| { - services - .rooms - .short - .get_statekey_from_short(*shortstatekey) - .map_ok(move |(event_type, state_key)| (event_type, state_key, event_id)) - .ok() - }) - .ready_filter_map(|(event_type, state_key, event_id)| { - if !lazy_load_enabled || event_type != StateEventType::RoomMember { - return Some(event_id); + let state_ids = state_ids?; + let lazy_loading_witnessed = lazy_loading_witnessed.unwrap_or_default(); + let shortstatekeys = state_ids.iter().stream().map(deref_at!(0)); + + let state: Vec<_> = services + .rooms + .short + .multi_get_statekey_from_short(shortstatekeys) + .zip(state_ids.iter().stream().map(at!(1))) + .ready_filter_map(|item| Some((item.0.ok()?, item.1))) + .ready_filter_map(|((event_type, state_key), event_id)| { + if filter.lazy_load_options.is_enabled() + && event_type == StateEventType::RoomMember + && state_key + .as_str() + .try_into() + .is_ok_and(|user_id: &UserId| !lazy_loading_witnessed.contains(user_id)) + { + return None; } - state_key - .as_str() - .try_into() - .ok() - .filter(|&user_id: &&UserId| lazy.contains(user_id)) - .map(|_| event_id) + Some(event_id) }) .broad_filter_map(|event_id: &OwnedEventId| { services.rooms.timeline.get_pdu(event_id).ok() diff --git a/src/api/client/message.rs b/src/api/client/message.rs index ec9a14d5..a508b5da 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,5 +1,3 @@ -use std::collections::HashSet; - use axum::extract::State; use conduwuit::{ at, is_equal_to, @@ -10,7 +8,7 @@ use conduwuit::{ }, Event, PduCount, Result, }; -use futures::{FutureExt, StreamExt}; +use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt}; use ruma::{ api::{ client::{filter::RoomEventFilter, message::get_message_events}, @@ -18,14 +16,19 @@ use ruma::{ }, events::{AnyStateEvent, StateEventType, TimelineEventType, TimelineEventType::*}, serde::Raw, - DeviceId, OwnedUserId, RoomId, UserId, + RoomId, UserId, +}; +use service::{ + rooms::{ + lazy_loading, + lazy_loading::{Options, Witness}, + timeline::PdusIterItem, + }, + Services, }; -use service::{rooms::timeline::PdusIterItem, Services}; use crate::Ruma; -pub(crate) type LazySet = HashSet; - /// list of safe and common non-state events to ignore if the user is ignored const IGNORED_MESSAGE_TYPES: &[TimelineEventType; 17] = &[ Audio, @@ -84,13 +87,6 @@ pub(crate) async fn get_message_events_route( .unwrap_or(LIMIT_DEFAULT) .min(LIMIT_MAX); - services.rooms.lazy_loading.lazy_load_confirm_delivery( - sender_user, - sender_device, - room_id, - from, - ); - if matches!(body.dir, Direction::Backward) { services .rooms @@ -127,35 +123,34 @@ pub(crate) async fn get_message_events_route( .collect() .await; - let lazy = events - .iter() - .stream() - .fold(LazySet::new(), |lazy, item| { - update_lazy(&services, room_id, sender, lazy, item, false) - }) - .await; + let lazy_loading_context = lazy_loading::Context { + user_id: sender_user, + device_id: sender_device, + room_id, + token: Some(from.into_unsigned()), + options: Some(&filter.lazy_load_options), + }; - let state = lazy - .iter() - .stream() - .broad_filter_map(|user_id| get_member_event(&services, room_id, user_id)) + let witness: OptionFuture<_> = filter + .lazy_load_options + .is_enabled() + .then(|| lazy_loading_witness(&services, &lazy_loading_context, events.iter())) + .into(); + + let state = witness + .map(Option::into_iter) + .map(|option| option.flat_map(Witness::into_iter)) + .map(IterStream::stream) + .into_stream() + .flatten() + .broad_filter_map(|user_id| async move { + get_member_event(&services, room_id, &user_id).await + }) .collect() .await; let next_token = events.last().map(at!(0)); - if !cfg!(feature = "element_hacks") { - if let Some(next_token) = next_token { - services.rooms.lazy_loading.lazy_load_mark_sent( - sender_user, - sender_device, - room_id, - lazy, - next_token, - ); - } - } - let chunk = events .into_iter() .map(at!(1)) @@ -170,6 +165,52 @@ pub(crate) async fn get_message_events_route( }) } +pub(crate) async fn lazy_loading_witness<'a, I>( + services: &Services, + lazy_loading_context: &lazy_loading::Context<'_>, + events: I, +) -> Witness +where + I: Iterator + Clone + Send, +{ + let oldest = events + .clone() + .map(|(count, _)| count) + .copied() + .min() + .unwrap_or_else(PduCount::max); + + let newest = events + .clone() + .map(|(count, _)| count) + .copied() + .max() + .unwrap_or_else(PduCount::max); + + let receipts = services + .rooms + .read_receipt + .readreceipts_since(lazy_loading_context.room_id, oldest.into_unsigned()); + + pin_mut!(receipts); + let witness: Witness = events + .stream() + .map(|(_, pdu)| pdu.sender.clone()) + .chain( + receipts + .ready_take_while(|(_, c, _)| *c <= newest.into_unsigned()) + .map(|(user_id, ..)| user_id.to_owned()), + ) + .collect() + .await; + + services + .rooms + .lazy_loading + .witness_retain(witness, lazy_loading_context) + .await +} + async fn get_member_event( services: &Services, room_id: &RoomId, @@ -184,42 +225,6 @@ async fn get_member_event( .ok() } -pub(crate) async fn update_lazy( - services: &Services, - room_id: &RoomId, - sender: (&UserId, &DeviceId), - mut lazy: LazySet, - item: &PdusIterItem, - force: bool, -) -> LazySet { - let (_, event) = &item; - let (sender_user, sender_device) = sender; - - /* TODO: Remove the "element_hacks" check when these are resolved: - * https://github.com/vector-im/element-android/issues/3417 - * https://github.com/vector-im/element-web/issues/21034 - */ - if force || cfg!(features = "element_hacks") { - lazy.insert(event.sender().into()); - return lazy; - } - - if lazy.contains(event.sender()) { - return lazy; - } - - if !services - .rooms - .lazy_loading - .lazy_load_was_sent_before(sender_user, sender_device, room_id, event.sender()) - .await - { - lazy.insert(event.sender().into()); - } - - lazy -} - pub(crate) async fn ignored_filter( services: &Services, item: PdusIterItem, diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index d6b9f15c..7cca9616 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -6,9 +6,9 @@ use std::{ use axum::extract::State; use conduwuit::{ - at, err, error, extract_variant, is_equal_to, is_false, + at, err, error, extract_variant, is_equal_to, pdu::EventHash, - result::{FlatOk, LogDebugErr}, + result::FlatOk, utils::{ self, future::OptionExt, @@ -19,16 +19,20 @@ use conduwuit::{ Error, PduCount, PduEvent, Result, }; use conduwuit_service::{ - rooms::short::{ShortStateHash, ShortStateKey}, + rooms::{ + lazy_loading, + lazy_loading::{Options, Witness}, + short::ShortStateHash, + }, Services, }; use futures::{ - future::{join, join3, join4, join5, try_join, try_join3, OptionFuture}, + future::{join, join3, join4, join5, try_join, try_join4, OptionFuture}, FutureExt, StreamExt, TryFutureExt, }; use ruma::{ api::client::{ - filter::{FilterDefinition, LazyLoadOptions}, + filter::FilterDefinition, sync::sync_events::{ self, v3::{ @@ -152,9 +156,14 @@ pub(crate) async fn build_sync_events( let (sender_user, sender_device) = body.sender(); let next_batch = services.globals.current_count()?; - let next_batch_string = next_batch.to_string(); + let since = body + .body + .since + .as_ref() + .and_then(|string| string.parse().ok()) + .unwrap_or(0); - // Load filter + let full_state = body.body.full_state; let filter = match body.body.filter.as_ref() { | None => FilterDefinition::default(), | Some(Filter::FilterDefinition(ref filter)) => filter.clone(), @@ -165,24 +174,6 @@ pub(crate) async fn build_sync_events( .unwrap_or_default(), }; - // some clients, at least element, seem to require knowledge of redundant - // members for "inline" profiles on the timeline to work properly - let (lazy_load_enabled, lazy_load_send_redundant) = match filter.room.state.lazy_load_options - { - | LazyLoadOptions::Enabled { include_redundant_members } => - (true, include_redundant_members), - | LazyLoadOptions::Disabled => (false, cfg!(feature = "element_hacks")), - }; - - let full_state = body.body.full_state; - - let since = body - .body - .since - .as_ref() - .and_then(|string| string.parse().ok()) - .unwrap_or(0); - let joined_rooms = services .rooms .state_cache @@ -196,9 +187,8 @@ pub(crate) async fn build_sync_events( room_id.clone(), since, next_batch, - lazy_load_enabled, - lazy_load_send_redundant, full_state, + &filter, ) .map_ok(move |(joined_room, dlu, jeu)| (room_id, joined_room, dlu, jeu)) .ok() @@ -227,9 +217,9 @@ pub(crate) async fn build_sync_events( since, room_id.clone(), sender_user, - &next_batch_string, + next_batch, full_state, - lazy_load_enabled, + &filter, ) .map_ok(move |left_room| (room_id, left_room)) .ok() @@ -358,7 +348,7 @@ pub(crate) async fn build_sync_events( device_one_time_keys_count, // Fallback keys are not yet supported device_unused_fallback_key_types: None, - next_batch: next_batch_string, + next_batch: next_batch.to_string(), presence: Presence { events: presence_updates .unwrap_or_default() @@ -449,7 +439,6 @@ async fn process_presence_updates( fields( room_id = %room_id, full = %full_state, - ll = %lazy_load_enabled, ), )] #[allow(clippy::too_many_arguments)] @@ -458,9 +447,9 @@ async fn handle_left_room( since: u64, ref room_id: OwnedRoomId, sender_user: &UserId, - next_batch_string: &str, + next_batch: u64, full_state: bool, - lazy_load_enabled: bool, + filter: &FilterDefinition, ) -> Result> { let left_count = services .rooms @@ -503,7 +492,7 @@ async fn handle_left_room( account_data: RoomAccountData { events: Vec::new() }, timeline: Timeline { limited: false, - prev_batch: Some(next_batch_string.to_owned()), + prev_batch: Some(next_batch.to_string()), events: Vec::new(), }, state: RoomState { @@ -567,28 +556,32 @@ async fn handle_left_room( .get_statekey_from_short(shortstatekey) .await?; - // TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565 - if !lazy_load_enabled - || event_type != StateEventType::RoomMember - || full_state - || (cfg!(feature = "element_hacks") && *sender_user == state_key) + if filter.room.state.lazy_load_options.is_enabled() + && event_type == StateEventType::RoomMember + && !full_state + && state_key + .as_str() + .try_into() + .is_ok_and(|user_id: &UserId| sender_user != user_id) { - let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { - error!("Pdu in state not found: {event_id}"); - continue; - }; - - left_state_events.push(pdu.to_sync_state_event()); + continue; } + + let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { + error!("Pdu in state not found: {event_id}"); + continue; + }; + + left_state_events.push(pdu.to_sync_state_event()); } } Ok(Some(LeftRoom { account_data: RoomAccountData { events: Vec::new() }, timeline: Timeline { - limited: true, /* TODO: support left timeline events so we dont need to set this to - * true */ - prev_batch: Some(next_batch_string.to_owned()), + // TODO: support left timeline events so we dont need to set limited to true + limited: true, + prev_batch: Some(next_batch.to_string()), events: Vec::new(), // and so we dont need to set this to empty vec }, state: RoomState { events: left_state_events }, @@ -611,9 +604,8 @@ async fn load_joined_room( ref room_id: OwnedRoomId, since: u64, next_batch: u64, - lazy_load_enabled: bool, - lazy_load_send_redundant: bool, full_state: bool, + filter: &FilterDefinition, ) -> Result<(JoinedRoom, HashSet, HashSet)> { let sincecount = PduCount::Normal(since); let next_batchcount = PduCount::Normal(next_batch); @@ -640,17 +632,26 @@ async fn load_joined_room( 10_usize, ); - let (current_shortstatehash, since_shortstatehash, timeline) = - try_join3(current_shortstatehash, since_shortstatehash, timeline).await?; + let receipt_events = services + .rooms + .read_receipt + .readreceipts_since(room_id, since) + .filter_map(|(read_user, _, edu)| async move { + services + .users + .user_is_ignored(read_user, sender_user) + .await + .or_some((read_user.to_owned(), edu)) + }) + .collect::>>() + .map(Ok); + + let (current_shortstatehash, since_shortstatehash, timeline, receipt_events) = + try_join4(current_shortstatehash, since_shortstatehash, timeline, receipt_events) + .boxed() + .await?; let (timeline_pdus, limited) = timeline; - let timeline_users = - timeline_pdus - .iter() - .fold(HashSet::new(), |mut timeline_users, (_, event)| { - timeline_users.insert(event.sender.as_str().to_owned()); - timeline_users - }); let last_notification_read: OptionFuture<_> = timeline_pdus .is_empty() @@ -662,21 +663,68 @@ async fn load_joined_room( }) .into(); - let send_notification_counts = last_notification_read - .is_none_or(|&count| count > since) - .await; - - services.rooms.lazy_loading.lazy_load_confirm_delivery( - sender_user, - sender_device, - room_id, - sincecount, - ); - let no_state_changes = timeline_pdus.is_empty() && (since_shortstatehash.is_none() || since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash))); + let since_sender_member: OptionFuture<_> = since_shortstatehash + .map(|short| { + services + .rooms + .state_accessor + .state_get_content(short, &StateEventType::RoomMember, sender_user.as_str()) + .ok() + }) + .into(); + + let joined_since_last_sync = + since_sender_member + .await + .flatten() + .is_none_or(|content: RoomMemberEventContent| { + content.membership != MembershipState::Join + }); + + let lazy_loading_enabled = filter.room.state.lazy_load_options.is_enabled() + || filter.room.timeline.lazy_load_options.is_enabled(); + + let generate_witness = + lazy_loading_enabled && (since_shortstatehash.is_none() || joined_since_last_sync); + + let lazy_reset = lazy_loading_enabled && since_shortstatehash.is_none(); + + let lazy_loading_context = &lazy_loading::Context { + user_id: sender_user, + device_id: sender_device, + room_id, + token: None, + options: Some(&filter.room.state.lazy_load_options), + }; + + // Reset lazy loading because this is an initial sync + let lazy_load_reset: OptionFuture<_> = lazy_reset + .then(|| services.rooms.lazy_loading.reset(lazy_loading_context)) + .into(); + + lazy_load_reset.await; + let witness: Option = generate_witness.then(|| { + timeline_pdus + .iter() + .map(|(_, pdu)| pdu.sender.clone()) + .chain(receipt_events.keys().cloned()) + .collect() + }); + + let witness: OptionFuture<_> = witness + .map(|witness| { + services + .rooms + .lazy_loading + .witness_retain(witness, lazy_loading_context) + }) + .into(); + + let witness = witness.await; let mut device_list_updates = HashSet::::new(); let mut left_encrypted_users = HashSet::::new(); let StateChanges { @@ -691,19 +739,17 @@ async fn load_joined_room( calculate_state_changes( services, sender_user, - sender_device, room_id, - next_batchcount, - lazy_load_enabled, - lazy_load_send_redundant, full_state, + filter, &mut device_list_updates, &mut left_encrypted_users, since_shortstatehash, current_shortstatehash, - &timeline_pdus, - &timeline_users, + joined_since_last_sync, + witness.as_ref(), ) + .boxed() .await? }; @@ -728,19 +774,6 @@ async fn load_joined_room( .map(|(_, pdu)| pdu.to_sync_room_event()) .collect(); - let receipt_events = services - .rooms - .read_receipt - .readreceipts_since(room_id, since) - .filter_map(|(read_user, _, edu)| async move { - services - .users - .user_is_ignored(read_user, sender_user) - .await - .or_some((read_user.to_owned(), edu)) - }) - .collect::>>(); - let typing_events = services .rooms .typing @@ -760,6 +793,10 @@ async fn load_joined_room( }) .unwrap_or(Vec::new()); + let send_notification_counts = last_notification_read + .is_none_or(|&count| count > since) + .await; + let notification_count: OptionFuture<_> = send_notification_counts .then(|| { services @@ -782,14 +819,14 @@ async fn load_joined_room( }) .into(); - let events = join4(room_events, account_data_events, receipt_events, typing_events); + let events = join3(room_events, account_data_events, typing_events); let unread_notifications = join(notification_count, highlight_count); let (unread_notifications, events, device_updates) = join3(unread_notifications, events, device_updates) .boxed() .await; - let (room_events, account_data_events, receipt_events, typing_events) = events; + let (room_events, account_data_events, typing_events) = events; let (notification_count, highlight_count) = unread_notifications; device_list_updates.extend(device_updates); @@ -866,7 +903,6 @@ async fn load_joined_room( skip_all, fields( full = %full_state, - ll = ?(lazy_load_enabled, lazy_load_send_redundant), cs = %current_shortstatehash, ss = ?since_shortstatehash, ) @@ -875,64 +911,38 @@ async fn load_joined_room( async fn calculate_state_changes( services: &Services, sender_user: &UserId, - sender_device: &DeviceId, room_id: &RoomId, - next_batchcount: PduCount, - lazy_load_enabled: bool, - lazy_load_send_redundant: bool, full_state: bool, + filter: &FilterDefinition, device_list_updates: &mut HashSet, left_encrypted_users: &mut HashSet, since_shortstatehash: Option, current_shortstatehash: ShortStateHash, - timeline_pdus: &Vec<(PduCount, PduEvent)>, - timeline_users: &HashSet, + joined_since_last_sync: bool, + witness: Option<&Witness>, ) -> Result { - let since_sender_member: OptionFuture<_> = since_shortstatehash - .map(|short| { - services - .rooms - .state_accessor - .state_get_content(short, &StateEventType::RoomMember, sender_user.as_str()) - .ok() - }) - .into(); - - let joined_since_last_sync = - since_sender_member - .await - .flatten() - .is_none_or(|content: RoomMemberEventContent| { - content.membership != MembershipState::Join - }); - if since_shortstatehash.is_none() || joined_since_last_sync { calculate_state_initial( services, sender_user, - sender_device, room_id, - next_batchcount, - lazy_load_enabled, full_state, + filter, current_shortstatehash, - timeline_users, + witness, ) .await } else { calculate_state_incremental( services, sender_user, - sender_device, room_id, - next_batchcount, - lazy_load_send_redundant, full_state, + filter, device_list_updates, left_encrypted_users, since_shortstatehash, current_shortstatehash, - timeline_pdus, joined_since_last_sync, ) .await @@ -944,87 +954,54 @@ async fn calculate_state_changes( async fn calculate_state_initial( services: &Services, sender_user: &UserId, - sender_device: &DeviceId, room_id: &RoomId, - next_batchcount: PduCount, - lazy_load_enabled: bool, full_state: bool, + filter: &FilterDefinition, current_shortstatehash: ShortStateHash, - timeline_users: &HashSet, + witness: Option<&Witness>, ) -> Result { - // Probably since = 0, we will do an initial sync - let state = services + let state_events = services .rooms .state_accessor .state_full_ids(current_shortstatehash) - .await? - .into_iter() - .stream() - .broad_filter_map(|(shortstatekey, event_id): (ShortStateKey, OwnedEventId)| { - services - .rooms - .short - .get_statekey_from_short(shortstatekey) - .map_ok(move |(event_type, state_key)| ((event_type, state_key), event_id)) - .ok() - }) - .fold((Vec::new(), HashSet::new()), |a, item| async move { - let (mut state_events, mut lazy_loaded) = a; - let ((event_type, state_key), event_id) = item; + .await?; - if event_type != StateEventType::RoomMember { - let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { - error!("Pdu in state not found: {event_id}"); - return (state_events, lazy_loaded); - }; + let shortstatekeys = state_events.keys().copied().stream(); - state_events.push(pdu); - return (state_events, lazy_loaded); + let state_events = services + .rooms + .short + .multi_get_statekey_from_short(shortstatekeys) + .zip(state_events.values().cloned().stream()) + .ready_filter_map(|item| Some((item.0.ok()?, item.1))) + .ready_filter_map(|((event_type, state_key), event_id)| { + let lazy_load_enabled = filter.room.state.lazy_load_options.is_enabled() + || filter.room.timeline.lazy_load_options.is_enabled(); + + if lazy_load_enabled + && event_type == StateEventType::RoomMember + && !full_state + && state_key.as_str().try_into().is_ok_and(|user_id: &UserId| { + sender_user != user_id + && witness.is_some_and(|witness| !witness.contains(user_id)) + }) { + return None; } - // TODO: Delete "element_hacks" when this is resolved: https://github.com/vector-im/element-web/issues/22565 - if !lazy_load_enabled - || full_state - || timeline_users.contains(&state_key) - || (cfg!(feature = "element_hacks") && *sender_user == state_key) - { - let Ok(pdu) = services.rooms.timeline.get_pdu(&event_id).await else { - error!("Pdu in state not found: {event_id}"); - return (state_events, lazy_loaded); - }; - - // This check is in case a bad user ID made it into the database - if let Ok(uid) = OwnedUserId::parse(&state_key) { - lazy_loaded.insert(uid); - } - - state_events.push(pdu); - } - - (state_events, lazy_loaded) + Some(event_id) }) + .broad_filter_map(|event_id: OwnedEventId| async move { + services.rooms.timeline.get_pdu(&event_id).await.ok() + }) + .collect() .map(Ok); let counts = calculate_counts(services, room_id, sender_user); - let ((joined_member_count, invited_member_count, heroes), (state_events, lazy_loaded)) = - try_join(counts, state).boxed().await?; - - // Reset lazy loading because this is an initial sync - services - .rooms - .lazy_loading - .lazy_load_reset(sender_user, sender_device, room_id) - .await; + let ((joined_member_count, invited_member_count, heroes), state_events) = + try_join(counts, state_events).boxed().await?; // The state_events above should contain all timeline_users, let's mark them as // lazy loaded. - services.rooms.lazy_loading.lazy_load_mark_sent( - sender_user, - sender_device, - room_id, - lazy_loaded, - next_batchcount, - ); Ok(StateChanges { heroes, @@ -1040,16 +1017,13 @@ async fn calculate_state_initial( async fn calculate_state_incremental( services: &Services, sender_user: &UserId, - sender_device: &DeviceId, room_id: &RoomId, - next_batchcount: PduCount, - lazy_load_send_redundant: bool, full_state: bool, + _filter: &FilterDefinition, device_list_updates: &mut HashSet, left_encrypted_users: &mut HashSet, since_shortstatehash: Option, current_shortstatehash: ShortStateHash, - timeline_pdus: &Vec<(PduCount, PduEvent)>, joined_since_last_sync: bool, ) -> Result { // Incremental /sync @@ -1162,76 +1136,12 @@ async fn calculate_state_incremental( (None, None, None) }; - let mut state_events = delta_state_events; - - // Mark all member events we're returning as lazy-loaded - let mut lazy_loaded = state_events - .iter() - .filter(|pdu| pdu.kind == RoomMember) - .filter_map(|pdu| { - pdu.state_key - .clone() - .map(TryInto::try_into) - .map(LogDebugErr::log_debug_err) - .flat_ok() - }) - .fold(HashSet::new(), |mut lazy_loaded, user_id| { - lazy_loaded.insert(user_id); - lazy_loaded - }); - - // Fetch contextual member state events for events from the timeline, and - // mark them as lazy-loaded as well. - for (_, event) in timeline_pdus { - if lazy_loaded.contains(&event.sender) { - continue; - } - - let sent_before: OptionFuture<_> = (!lazy_load_send_redundant) - .then(|| { - services.rooms.lazy_loading.lazy_load_was_sent_before( - sender_user, - sender_device, - room_id, - &event.sender, - ) - }) - .into(); - - let member_event: OptionFuture<_> = sent_before - .await - .is_none_or(is_false!()) - .then(|| { - services.rooms.state_accessor.room_state_get( - room_id, - &StateEventType::RoomMember, - event.sender.as_str(), - ) - }) - .into(); - - let Some(Ok(member_event)) = member_event.await else { - continue; - }; - - lazy_loaded.insert(event.sender.clone()); - state_events.push(member_event); - } - - services.rooms.lazy_loading.lazy_load_mark_sent( - sender_user, - sender_device, - room_id, - lazy_loaded, - next_batchcount, - ); - Ok(StateChanges { heroes, joined_member_count, invited_member_count, joined_since_last_sync, - state_events, + state_events: delta_state_events, }) } diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index c3c27b9e..67274ff1 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -1,109 +1,65 @@ -use std::{ - collections::{HashMap, HashSet}, - fmt::Write, - sync::{Arc, Mutex}, -}; +//! Lazy Loading + +use std::{collections::HashSet, sync::Arc}; use conduwuit::{ implement, - utils::{stream::TryIgnore, ReadyExt}, - PduCount, Result, + utils::{stream::TryIgnore, IterStream, ReadyExt}, + Result, }; -use database::{Interfix, Map}; -use ruma::{DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, RoomId, UserId}; +use database::{Database, Deserialized, Handle, Interfix, Map}; +use futures::{pin_mut, Stream, StreamExt}; +use ruma::{api::client::filter::LazyLoadOptions, DeviceId, OwnedUserId, RoomId, UserId}; pub struct Service { - lazy_load_waiting: Mutex, db: Data, } struct Data { lazyloadedids: Arc, + db: Arc, } -type LazyLoadWaiting = HashMap; -type LazyLoadWaitingKey = (OwnedUserId, OwnedDeviceId, OwnedRoomId, PduCount); -type LazyLoadWaitingVal = HashSet; +pub trait Options: Send + Sync { + fn is_enabled(&self) -> bool; + fn include_redundant_members(&self) -> bool; +} + +#[derive(Clone, Debug)] +pub struct Context<'a> { + pub user_id: &'a UserId, + pub device_id: &'a DeviceId, + pub room_id: &'a RoomId, + pub token: Option, + pub options: Option<&'a LazyLoadOptions>, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Status { + Unseen, + Seen(u64), +} + +pub type Witness = HashSet; +type Key<'a> = (&'a UserId, &'a DeviceId, &'a RoomId, &'a UserId); impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - lazy_load_waiting: LazyLoadWaiting::new().into(), db: Data { lazyloadedids: args.db["lazyloadedids"].clone(), + db: args.db.clone(), }, })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { - let lazy_load_waiting = self.lazy_load_waiting.lock().expect("locked").len(); - writeln!(out, "lazy_load_waiting: {lazy_load_waiting}")?; - - Ok(()) - } - - fn clear_cache(&self) { self.lazy_load_waiting.lock().expect("locked").clear(); } - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } #[implement(Service)] #[tracing::instrument(skip(self), level = "debug")] -#[inline] -pub async fn lazy_load_was_sent_before( - &self, - user_id: &UserId, - device_id: &DeviceId, - room_id: &RoomId, - ll_user: &UserId, -) -> bool { - let key = (user_id, device_id, room_id, ll_user); - self.db.lazyloadedids.qry(&key).await.is_ok() -} - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn lazy_load_mark_sent( - &self, - user_id: &UserId, - device_id: &DeviceId, - room_id: &RoomId, - lazy_load: HashSet, - count: PduCount, -) { - let key = (user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), count); - - self.lazy_load_waiting - .lock() - .expect("locked") - .insert(key, lazy_load); -} - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn lazy_load_confirm_delivery( - &self, - user_id: &UserId, - device_id: &DeviceId, - room_id: &RoomId, - since: PduCount, -) { - let key = (user_id.to_owned(), device_id.to_owned(), room_id.to_owned(), since); - - let Some(user_ids) = self.lazy_load_waiting.lock().expect("locked").remove(&key) else { - return; - }; - - for ll_id in &user_ids { - let key = (user_id, device_id, room_id, ll_id); - self.db.lazyloadedids.put_raw(key, []); - } -} - -#[implement(Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub async fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room_id: &RoomId) { - let prefix = (user_id, device_id, room_id, Interfix); +pub async fn reset(&self, ctx: &Context<'_>) { + let prefix = (ctx.user_id, ctx.device_id, ctx.room_id, Interfix); self.db .lazyloadedids .keys_prefix_raw(&prefix) @@ -111,3 +67,89 @@ pub async fn lazy_load_reset(&self, user_id: &UserId, device_id: &DeviceId, room .ready_for_each(|key| self.db.lazyloadedids.remove(key)) .await; } + +#[implement(Service)] +#[tracing::instrument(name = "retain", level = "debug", skip_all)] +pub async fn witness_retain(&self, senders: Witness, ctx: &Context<'_>) -> Witness { + debug_assert!( + ctx.options.is_none_or(Options::is_enabled), + "lazy loading should be enabled by your options" + ); + + let include_redundant = cfg!(feature = "element_hacks") + || ctx.options.is_some_and(Options::include_redundant_members); + + let witness = self + .witness(ctx, senders.iter().map(AsRef::as_ref)) + .zip(senders.iter().stream()); + + pin_mut!(witness); + let _cork = self.db.db.cork(); + let mut senders = Witness::with_capacity(senders.len()); + while let Some((status, sender)) = witness.next().await { + if include_redundant || status == Status::Unseen { + senders.insert(sender.into()); + continue; + } + + if let Status::Seen(seen) = status { + if seen == 0 || ctx.token == Some(seen) { + senders.insert(sender.into()); + continue; + } + } + } + + senders +} + +#[implement(Service)] +fn witness<'a, I>( + &'a self, + ctx: &'a Context<'a>, + senders: I, +) -> impl Stream + Send + 'a +where + I: Iterator + Send + Clone + 'a, +{ + let make_key = + |sender: &'a UserId| -> Key<'a> { (ctx.user_id, ctx.device_id, ctx.room_id, sender) }; + + self.db + .lazyloadedids + .qry_batch(senders.clone().stream().map(make_key)) + .map(into_status) + .zip(senders.stream()) + .map(move |(status, sender)| { + if matches!(status, Status::Unseen) { + self.db + .lazyloadedids + .put_aput::<8, _, _>(make_key(sender), 0_u64); + } else if matches!(status, Status::Seen(0)) { + self.db + .lazyloadedids + .put_aput::<8, _, _>(make_key(sender), ctx.token.unwrap_or(0_u64)); + } + + status + }) +} + +fn into_status(result: Result>) -> Status { + match result.and_then(|handle| handle.deserialized()) { + | Ok(seen) => Status::Seen(seen), + | Err(_) => Status::Unseen, + } +} + +impl Options for LazyLoadOptions { + fn include_redundant_members(&self) -> bool { + if let Self::Enabled { include_redundant_members } = self { + *include_redundant_members + } else { + false + } + } + + fn is_enabled(&self) -> bool { !self.is_disabled() } +} From 4b3c54bbfa8340c4bab09a221f47afb0c6d04346 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 26 Jan 2025 13:11:47 -0500 Subject: [PATCH 1424/2091] check if DOCKER_USERNAME is empty as well in dockerhub desc publish Signed-off-by: June Clementine Strawberry --- .github/workflows/docker-hub-description.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-hub-description.yml b/.github/workflows/docker-hub-description.yml index 96b2d38b..b4f142db 100644 --- a/.github/workflows/docker-hub-description.yml +++ b/.github/workflows/docker-hub-description.yml @@ -13,7 +13,7 @@ on: jobs: dockerHubDescription: runs-on: ubuntu-latest - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' + if: ${{ (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' && (vars.DOCKER_USERNAME != '') }} steps: - uses: actions/checkout@v4 with: From ee63f720c97e6047ac712ce828505e7e82923336 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 26 Jan 2025 20:39:08 +0000 Subject: [PATCH 1425/2091] revert incorrect tags --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b0b0bd53..cde31232 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -897,15 +897,15 @@ jobs: run: | # Dockerhub Container Registry if [ ! -z $DOCKERHUB_TOKEN ]; then - docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell fi # GitHub Container Registry if [ $GHCR_ENABLED = "true" ]; then - docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${BRANCH_TAG}-haswell + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell fi # GitLab Container Registry if [ ! -z $GITLAB_TOKEN ]; then - docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${BRANCH_TAG}-haswell + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell fi - name: Create Docker combined manifests From 24e6086f12d6f3421b45c13b02aac96c3a188205 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 26 Jan 2025 20:44:40 +0000 Subject: [PATCH 1426/2091] load correct image file --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cde31232..42f4e12e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -815,7 +815,7 @@ jobs: - name: Load and push amd64 haswell image run: | - docker load -i oci-image-amd64.tar.gz + docker load -i oci-image-amd64-haswell-optimised.tar.gz if [ ! -z $DOCKERHUB_TOKEN ]; then docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell From 1c585ab1b6b55e40dadca2d1339ea5f8e4a244bb Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 26 Jan 2025 22:16:32 +0000 Subject: [PATCH 1427/2091] create manifests for unique docker tags --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42f4e12e..8a870700 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -897,14 +897,17 @@ jobs: run: | # Dockerhub Container Registry if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell fi # GitHub Container Registry if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell fi # GitLab Container Registry if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell fi From db7d23e7804f9ed707358a0da9d8f3f4fa588bd1 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 26 Jan 2025 23:52:51 +0000 Subject: [PATCH 1428/2091] fix creating manifest on wrong repo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a870700..de599f45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -902,7 +902,7 @@ jobs: fi # GitHub Container Registry if [ $GHCR_ENABLED = "true" ]; then - docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell fi # GitLab Container Registry From 71a3855af61b0071832c23085f76a8711e32b49c Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 26 Jan 2025 21:30:15 -0500 Subject: [PATCH 1429/2091] fix couple more docker manifest typos Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de599f45..35d60aa1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -902,12 +902,12 @@ jobs: fi # GitHub Container Registry if [ $GHCR_ENABLED = "true" ]; then - docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell fi # GitLab Container Registry if [ ! -z $GITLAB_TOKEN ]; then - docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell fi From 3e0ff2dc840cd9b3b823cf5d8d9a6739ee531896 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 25 Jan 2025 23:41:39 +0000 Subject: [PATCH 1430/2091] simplify references to server_name Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 12 ++++++------ src/admin/federation/commands.rs | 2 +- src/admin/server/commands.rs | 2 +- src/api/client/openid.rs | 2 +- src/api/client/report.rs | 2 +- src/api/client/room/create.rs | 2 +- src/api/client/voip.rs | 2 +- src/core/server.rs | 6 ++++++ src/service/globals/mod.rs | 8 ++++---- src/service/migrations.rs | 10 ++++------ src/service/resolver/actual.rs | 3 +-- src/service/rooms/alias/mod.rs | 2 +- src/service/sending/sender.rs | 2 +- 13 files changed, 29 insertions(+), 26 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index af7bd79f..855941fd 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -554,7 +554,7 @@ pub(super) async fn first_pdu_in_room( .services .rooms .state_cache - .server_in_room(&self.services.server.config.server_name, &room_id) + .server_in_room(&self.services.server.name, &room_id) .await { return Ok(RoomMessageEventContent::text_plain( @@ -583,7 +583,7 @@ pub(super) async fn latest_pdu_in_room( .services .rooms .state_cache - .server_in_room(&self.services.server.config.server_name, &room_id) + .server_in_room(&self.services.server.name, &room_id) .await { return Ok(RoomMessageEventContent::text_plain( @@ -613,7 +613,7 @@ pub(super) async fn force_set_room_state_from_server( .services .rooms .state_cache - .server_in_room(&self.services.server.config.server_name, &room_id) + .server_in_room(&self.services.server.name, &room_id) .await { return Ok(RoomMessageEventContent::text_plain( @@ -757,7 +757,7 @@ pub(super) async fn get_signing_keys( query: bool, ) -> Result { let server_name = - server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); + server_name.unwrap_or_else(|| self.services.server.name.clone().into()); if let Some(notary) = notary { let signing_keys = self @@ -794,7 +794,7 @@ pub(super) async fn get_verify_keys( server_name: Option>, ) -> Result { let server_name = - server_name.unwrap_or_else(|| self.services.server.config.server_name.clone().into()); + server_name.unwrap_or_else(|| self.services.server.name.clone().into()); let keys = self .services @@ -824,7 +824,7 @@ pub(super) async fn resolve_true_destination( )); } - if server_name == self.services.server.config.server_name { + if server_name == self.services.server.name { return Ok(RoomMessageEventContent::text_plain( "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ fetching local PDUs.", diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index be91ef0a..13bc8da4 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -92,7 +92,7 @@ pub(super) async fn remote_user_in_rooms( &self, user_id: Box, ) -> Result { - if user_id.server_name() == self.services.server.config.server_name { + if user_id.server_name() == self.services.server.name { return Ok(RoomMessageEventContent::text_plain( "User belongs to our server, please use `list-joined-rooms` user admin command \ instead.", diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 3ea27883..47509bad 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -34,7 +34,7 @@ pub(super) async fn reload_config( ) -> Result { let path = path.as_deref().into_iter(); let config = Config::load(path).and_then(|raw| Config::new(&raw))?; - if config.server_name != self.services.server.config.server_name { + if config.server_name != self.services.server.name { return Err!("You can't change the server name."); } diff --git a/src/api/client/openid.rs b/src/api/client/openid.rs index 3547d284..671d0c6d 100644 --- a/src/api/client/openid.rs +++ b/src/api/client/openid.rs @@ -37,7 +37,7 @@ pub(crate) async fn create_openid_token_route( Ok(account::request_openid_token::v3::Response { access_token, token_type: TokenType::Bearer, - matrix_server_name: services.server.config.server_name.clone(), + matrix_server_name: services.server.name.clone(), expires_in: Duration::from_secs(expires_in), }) } diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 2b25b518..57de3f12 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -50,7 +50,7 @@ pub(crate) async fn report_room_route( if !services .rooms .state_cache - .server_in_room(&services.server.config.server_name, &body.room_id) + .server_in_room(&services.server.name, &body.room_id) .await { return Err!(Request(NotFound( diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index 1b6e8667..a401b63d 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -71,7 +71,7 @@ pub(crate) async fn create_room_route( let room_id: OwnedRoomId = if let Some(custom_room_id) = &body.room_id { custom_room_id_check(&services, custom_room_id)? } else { - RoomId::new(&services.server.config.server_name) + RoomId::new(&services.server.name) }; // check if room ID doesn't already exist instead of erroring on auth check diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index c08b1fdf..70ad4913 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -38,7 +38,7 @@ pub(crate) async fn turn_server_route( let user = body.sender_user.unwrap_or_else(|| { UserId::parse_with_server_name( utils::random_string(RANDOM_USER_ID_LENGTH).to_lowercase(), - &services.server.config.server_name, + &services.server.name, ) .unwrap() }); diff --git a/src/core/server.rs b/src/core/server.rs index 6838c9c9..05a4aae7 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -6,12 +6,17 @@ use std::{ time::SystemTime, }; +use ruma::OwnedServerName; use tokio::{runtime, sync::broadcast}; use crate::{config, config::Config, err, log::Log, metrics::Metrics, Err, Result}; /// Server runtime state; public portion pub struct Server { + /// Configured name of server. This is the same as the one in the config + /// but developers can (and should) reference this string instead. + pub name: OwnedServerName, + /// Server-wide configuration instance pub config: config::Manager, @@ -46,6 +51,7 @@ impl Server { #[must_use] pub fn new(config: Config, runtime: Option, log: Log) -> Self { Self { + name: config.server_name.clone(), config: config::Manager::new(config), started: SystemTime::now(), stopping: AtomicBool::new(false), diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index ef34054f..485d5020 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -61,11 +61,11 @@ impl crate::Service for Service { db, server: args.server.clone(), bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())), - admin_alias: OwnedRoomAliasId::try_from(format!("#admins:{}", &config.server_name)) + admin_alias: OwnedRoomAliasId::try_from(format!("#admins:{}", &args.server.name)) .expect("#admins:server_name is valid alias name"), server_user: UserId::parse_with_server_name( String::from("conduit"), - &config.server_name, + &args.server.name, ) .expect("@conduit:server_name is valid"), turn_secret, @@ -107,7 +107,7 @@ impl Service { pub fn current_count(&self) -> Result { Ok(self.db.current_count()) } #[inline] - pub fn server_name(&self) -> &ServerName { self.server.config.server_name.as_ref() } + pub fn server_name(&self) -> &ServerName { self.server.name.as_ref() } pub fn allow_registration(&self) -> bool { self.server.config.allow_registration } @@ -207,7 +207,7 @@ impl Service { #[inline] pub fn server_is_ours(&self, server_name: &ServerName) -> bool { - server_name == self.server.config.server_name + server_name == self.server_name() } #[inline] diff --git a/src/service/migrations.rs b/src/service/migrations.rs index c42c0324..27b4ab5a 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -218,8 +218,6 @@ async fn migrate(services: &Services) -> Result<()> { } async fn db_lt_12(services: &Services) -> Result<()> { - let config = &services.server.config; - for username in &services .users .list_local_users() @@ -227,7 +225,8 @@ async fn db_lt_12(services: &Services) -> Result<()> { .collect::>() .await { - let user = match UserId::parse_with_server_name(username.as_str(), &config.server_name) { + let user = match UserId::parse_with_server_name(username.as_str(), &services.server.name) + { | Ok(u) => u, | Err(e) => { warn!("Invalid username {username}: {e}"); @@ -297,8 +296,6 @@ async fn db_lt_12(services: &Services) -> Result<()> { } async fn db_lt_13(services: &Services) -> Result<()> { - let config = &services.server.config; - for username in &services .users .list_local_users() @@ -306,7 +303,8 @@ async fn db_lt_13(services: &Services) -> Result<()> { .collect::>() .await { - let user = match UserId::parse_with_server_name(username.as_str(), &config.server_name) { + let user = match UserId::parse_with_server_name(username.as_str(), &services.server.name) + { | Ok(u) => u, | Err(e) => { warn!("Invalid username {username}: {e}"); diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 33374240..66854764 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -401,8 +401,7 @@ impl super::Service { } fn validate_dest(&self, dest: &ServerName) -> Result<()> { - let config = &self.services.server.config; - if dest == config.server_name && !config.federation_loopback { + if dest == self.services.server.name && !self.services.server.config.federation_loopback { return Err!("Won't send federation request to ourselves"); } diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 91797d01..17ed5e13 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -150,7 +150,7 @@ impl Service { let servers_contains_ours = || { servers .as_ref() - .is_some_and(|servers| servers.contains(&self.services.server.config.server_name)) + .is_some_and(|servers| servers.contains(&self.services.server.name)) }; if !server_is_ours && !servers_contains_ours() { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 5fd4cf91..122e75c5 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -850,7 +850,7 @@ impl Service { let txn_id = &*general_purpose::URL_SAFE_NO_PAD.encode(txn_hash); let request = send_transaction_message::v1::Request { - origin: self.server.config.server_name.clone(), + origin: self.server.name.clone(), pdus: pdu_jsons, edus: edu_jsons, origin_server_ts: MilliSecondsSinceUnixEpoch::now(), From 4a2d0d35bcae8f189774214eb850d78dd53332eb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 00:24:11 +0000 Subject: [PATCH 1431/2091] split federation request from sending service Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 6 +- .../send.rs => federation/execute.rs} | 163 ++++++++++-------- src/service/federation/mod.rs | 33 ++++ src/service/mod.rs | 1 + src/service/sending/mod.rs | 24 ++- src/service/sending/sender.rs | 2 +- src/service/services.rs | 4 +- 7 files changed, 146 insertions(+), 87 deletions(-) rename src/service/{sending/send.rs => federation/execute.rs} (65%) create mode 100644 src/service/federation/mod.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 855941fd..cdd69c0f 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -756,8 +756,7 @@ pub(super) async fn get_signing_keys( notary: Option>, query: bool, ) -> Result { - let server_name = - server_name.unwrap_or_else(|| self.services.server.name.clone().into()); + let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone().into()); if let Some(notary) = notary { let signing_keys = self @@ -793,8 +792,7 @@ pub(super) async fn get_verify_keys( &self, server_name: Option>, ) -> Result { - let server_name = - server_name.unwrap_or_else(|| self.services.server.name.clone().into()); + let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone().into()); let keys = self .services diff --git a/src/service/sending/send.rs b/src/service/federation/execute.rs similarity index 65% rename from src/service/sending/send.rs rename to src/service/federation/execute.rs index c8a64f3c..27d98968 100644 --- a/src/service/sending/send.rs +++ b/src/service/federation/execute.rs @@ -1,4 +1,4 @@ -use std::mem; +use std::{fmt::Debug, mem}; use bytes::Bytes; use conduwuit::{ @@ -20,82 +20,109 @@ use ruma::{ use crate::resolver::actual::ActualDest; -impl super::Service { - #[tracing::instrument( +/// Sends a request to a federation server +#[implement(super::Service)] +#[tracing::instrument(skip_all, name = "request", level = "debug")] +pub async fn execute(&self, dest: &ServerName, request: T) -> Result +where + T: OutgoingRequest + Debug + Send, +{ + let client = &self.services.client.federation; + self.execute_on(client, dest, request).await +} + +/// Like execute() but with a very large timeout +#[implement(super::Service)] +#[tracing::instrument(skip_all, name = "synapse", level = "debug")] +pub async fn execute_synapse( + &self, + dest: &ServerName, + request: T, +) -> Result +where + T: OutgoingRequest + Debug + Send, +{ + let client = &self.services.client.synapse; + self.execute_on(client, dest, request).await +} + +#[implement(super::Service)] +#[tracing::instrument( level = "debug" skip(self, client, request), )] - pub async fn send( - &self, - client: &Client, - dest: &ServerName, - request: T, - ) -> Result - where - T: OutgoingRequest + Send, +pub async fn execute_on( + &self, + client: &Client, + dest: &ServerName, + request: T, +) -> Result +where + T: OutgoingRequest + Send, +{ + if !self.services.server.config.allow_federation { + return Err!(Config("allow_federation", "Federation is disabled.")); + } + + if self + .services + .server + .config + .forbidden_remote_server_names + .contains(dest) { - if !self.server.config.allow_federation { - return Err!(Config("allow_federation", "Federation is disabled.")); - } - - if self - .server - .config - .forbidden_remote_server_names - .contains(dest) - { - return Err!(Request(Forbidden(debug_warn!( - "Federation with {dest} is not allowed." - )))); - } - - let actual = self.services.resolver.get_actual_dest(dest).await?; - let request = into_http_request::(&actual, request)?; - let request = self.prepare(dest, request)?; - self.execute::(dest, &actual, request, client).await + return Err!(Request(Forbidden(debug_warn!("Federation with {dest} is not allowed.")))); } - async fn execute( - &self, - dest: &ServerName, - actual: &ActualDest, - request: Request, - client: &Client, - ) -> Result - where - T: OutgoingRequest + Send, - { - let url = request.url().clone(); - let method = request.method().clone(); + let actual = self.services.resolver.get_actual_dest(dest).await?; + let request = into_http_request::(&actual, request)?; + let request = self.prepare(dest, request)?; + self.perform::(dest, &actual, request, client).await +} - debug!(?method, ?url, "Sending request"); - match client.execute(request).await { - | Ok(response) => handle_response::(dest, actual, &method, &url, response).await, - | Err(error) => - Err(handle_error(actual, &method, &url, error).expect_err("always returns error")), +#[implement(super::Service)] +async fn perform( + &self, + dest: &ServerName, + actual: &ActualDest, + request: Request, + client: &Client, +) -> Result +where + T: OutgoingRequest + Send, +{ + let url = request.url().clone(); + let method = request.method().clone(); + + debug!(?method, ?url, "Sending request"); + match client.execute(request).await { + | Ok(response) => handle_response::(dest, actual, &method, &url, response).await, + | Err(error) => + Err(handle_error(actual, &method, &url, error).expect_err("always returns error")), + } +} + +#[implement(super::Service)] +fn prepare(&self, dest: &ServerName, mut request: http::Request>) -> Result { + self.sign_request(&mut request, dest); + + let request = Request::try_from(request)?; + self.validate_url(request.url())?; + self.services.server.check_running()?; + + Ok(request) +} + +#[implement(super::Service)] +fn validate_url(&self, url: &Url) -> Result<()> { + if let Some(url_host) = url.host_str() { + if let Ok(ip) = IPAddress::parse(url_host) { + trace!("Checking request URL IP {ip:?}"); + self.services.resolver.validate_ip(&ip)?; } } - fn prepare(&self, dest: &ServerName, mut request: http::Request>) -> Result { - self.sign_request(&mut request, dest); - - let request = Request::try_from(request)?; - self.validate_url(request.url())?; - self.server.check_running()?; - - Ok(request) - } - - fn validate_url(&self, url: &Url) -> Result<()> { - if let Some(url_host) = url.host_str() { - if let Ok(ip) = IPAddress::parse(url_host) { - trace!("Checking request URL IP {ip:?}"); - self.services.resolver.validate_ip(&ip)?; - } - } - - Ok(()) - } + Ok(()) } async fn handle_response( @@ -195,7 +222,7 @@ fn sign_request(&self, http_request: &mut http::Request>, dest: &ServerN type Value = CanonicalJsonValue; type Object = CanonicalJsonObject; - let origin = self.services.globals.server_name(); + let origin = &self.services.server.name; let body = http_request.body(); let uri = http_request .uri() diff --git a/src/service/federation/mod.rs b/src/service/federation/mod.rs new file mode 100644 index 00000000..dacdb20e --- /dev/null +++ b/src/service/federation/mod.rs @@ -0,0 +1,33 @@ +mod execute; + +use std::sync::Arc; + +use conduwuit::{Result, Server}; + +use crate::{client, resolver, server_keys, Dep}; + +pub struct Service { + services: Services, +} + +struct Services { + server: Arc, + client: Dep, + resolver: Dep, + server_keys: Dep, +} + +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + services: Services { + server: args.server.clone(), + client: args.depend::("client"), + resolver: args.depend::("resolver"), + server_keys: args.depend::("server_keys"), + }, + })) + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} diff --git a/src/service/mod.rs b/src/service/mod.rs index 789994d3..2102921f 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -10,6 +10,7 @@ pub mod admin; pub mod appservice; pub mod client; pub mod emergency; +pub mod federation; pub mod globals; pub mod key_backups; pub mod media; diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index e52bfb25..80bca112 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -1,7 +1,6 @@ mod appservice; mod data; mod dest; -mod send; mod sender; use std::{ @@ -30,8 +29,8 @@ pub use self::{ sender::{EDU_LIMIT, PDU_LIMIT}, }; use crate::{ - account_data, client, globals, presence, pusher, resolver, rooms, rooms::timeline::RawPduId, - server_keys, users, Dep, + account_data, client, federation, globals, presence, pusher, rooms, + rooms::timeline::RawPduId, users, Dep, }; pub struct Service { @@ -44,7 +43,6 @@ pub struct Service { struct Services { client: Dep, globals: Dep, - resolver: Dep, state: Dep, state_cache: Dep, user: Dep, @@ -55,7 +53,7 @@ struct Services { account_data: Dep, appservice: Dep, pusher: Dep, - server_keys: Dep, + federation: Dep, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -83,7 +81,6 @@ impl crate::Service for Service { services: Services { client: args.depend::("client"), globals: args.depend::("globals"), - resolver: args.depend::("resolver"), state: args.depend::("rooms::state"), state_cache: args.depend::("rooms::state_cache"), user: args.depend::("rooms::user"), @@ -94,7 +91,7 @@ impl crate::Service for Service { account_data: args.depend::("account_data"), appservice: args.depend::("appservice"), pusher: args.depend::("pusher"), - server_keys: args.depend::("server_keys"), + federation: args.depend::("federation"), }, channels: (0..num_senders).map(|_| loole::unbounded()).collect(), })) @@ -277,7 +274,7 @@ impl Service { } /// Sends a request to a federation server - #[tracing::instrument(skip_all, name = "request", level = "debug")] + #[inline] pub async fn send_federation_request( &self, dest: &ServerName, @@ -286,12 +283,11 @@ impl Service { where T: OutgoingRequest + Debug + Send, { - let client = &self.services.client.federation; - self.send(client, dest, request).await + self.services.federation.execute(dest, request).await } /// Like send_federation_request() but with a very large timeout - #[tracing::instrument(skip_all, name = "synapse", level = "debug")] + #[inline] pub async fn send_synapse_request( &self, dest: &ServerName, @@ -300,8 +296,10 @@ impl Service { where T: OutgoingRequest + Debug + Send, { - let client = &self.services.client.synapse; - self.send(client, dest, request).await + self.services + .federation + .execute_synapse(dest, request) + .await } /// Sends a request to an appservice diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 122e75c5..c91e1d31 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -858,7 +858,7 @@ impl Service { }; let client = &self.services.client.sender; - self.send(client, &server, request) + self.services.federation.execute_on(client, &server, request) .await .inspect(|response| { response diff --git a/src/service/services.rs b/src/service/services.rs index 1aa87f58..cb5cc12f 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -10,7 +10,7 @@ use database::Database; use tokio::sync::Mutex; use crate::{ - account_data, admin, appservice, client, emergency, globals, key_backups, + account_data, admin, appservice, client, emergency, federation, globals, key_backups, manager::Manager, media, presence, pusher, resolver, rooms, sending, server_keys, service, service::{Args, Map, Service}, @@ -30,6 +30,7 @@ pub struct Services { pub pusher: Arc, pub resolver: Arc, pub rooms: rooms::Service, + pub federation: Arc, pub sending: Arc, pub server_keys: Arc, pub sync: Arc, @@ -95,6 +96,7 @@ impl Services { typing: build!(rooms::typing::Service), user: build!(rooms::user::Service), }, + federation: build!(federation::Service), sending: build!(sending::Service), server_keys: build!(server_keys::Service), sync: build!(sync::Service), From d0b4a619af08030a28f3a445fb8031bafc3cf90a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 03:30:34 +0000 Subject: [PATCH 1432/2091] furnish batch interface with trait Signed-off-by: Jason Volk --- src/database/map.rs | 3 ++ src/database/map/get.rs | 52 +--------------------- src/database/map/get_batch.rs | 45 ++++++++----------- src/database/map/qry.rs | 54 +++++++++++++++++++++++ src/database/map/qry_batch.rs | 63 +++++++++++++++++++++++++++ src/database/mod.rs | 2 +- src/service/rooms/lazy_loading/mod.rs | 10 +++-- src/service/rooms/short/mod.rs | 19 ++++---- 8 files changed, 155 insertions(+), 93 deletions(-) create mode 100644 src/database/map/qry.rs create mode 100644 src/database/map/qry_batch.rs diff --git a/src/database/map.rs b/src/database/map.rs index 97e90659..5176c529 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -9,6 +9,8 @@ mod keys_from; mod keys_prefix; mod open; mod options; +mod qry; +mod qry_batch; mod remove; mod rev_keys; mod rev_keys_from; @@ -37,6 +39,7 @@ pub(crate) use self::options::{ cache_iter_options_default, cache_read_options_default, iter_options_default, read_options_default, write_options_default, }; +pub use self::{get_batch::Get, qry_batch::Qry}; use crate::{watchers::Watchers, Engine}; pub struct Map { diff --git a/src/database/map/get.rs b/src/database/map/get.rs index 73182042..d6c65be2 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -1,65 +1,15 @@ -use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; +use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use arrayvec::ArrayVec; use conduwuit::{err, implement, utils::result::MapExpect, Err, Result}; use futures::{future::ready, Future, FutureExt, TryFutureExt}; use rocksdb::{DBPinnableSlice, ReadOptions}; -use serde::Serialize; use tokio::task; use crate::{ - keyval::KeyBuf, - ser, util::{is_incomplete, map_err, or_else}, Handle, }; -/// Fetch a value from the database into cache, returning a reference-handle -/// asynchronously. The key is serialized into an allocated buffer to perform -/// the query. -#[implement(super::Map)] -#[inline] -pub fn qry(self: &Arc, key: &K) -> impl Future>> + Send -where - K: Serialize + ?Sized + Debug, -{ - let mut buf = KeyBuf::new(); - self.bqry(key, &mut buf) -} - -/// Fetch a value from the database into cache, returning a reference-handle -/// asynchronously. The key is serialized into a fixed-sized buffer to perform -/// the query. The maximum size is supplied as const generic parameter. -#[implement(super::Map)] -#[inline] -pub fn aqry( - self: &Arc, - key: &K, -) -> impl Future>> + Send -where - K: Serialize + ?Sized + Debug, -{ - let mut buf = ArrayVec::::new(); - self.bqry(key, &mut buf) -} - -/// Fetch a value from the database into cache, returning a reference-handle -/// asynchronously. The key is serialized into a user-supplied Writer. -#[implement(super::Map)] -#[tracing::instrument(skip(self, buf), level = "trace")] -pub fn bqry( - self: &Arc, - key: &K, - buf: &mut B, -) -> impl Future>> + Send -where - K: Serialize + ?Sized + Debug, - B: Write + AsRef<[u8]>, -{ - let key = ser::serialize(buf, key).expect("failed to serialize query key"); - self.get(key) -} - /// Fetch a value from the database into cache, returning a reference-handle /// asynchronously. The key is referenced directly to perform the query. #[implement(super::Map)] diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index ee9269e3..ab9c1dc8 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -1,4 +1,4 @@ -use std::{convert::AsRef, fmt::Debug, sync::Arc}; +use std::{convert::AsRef, sync::Arc}; use conduwuit::{ implement, @@ -10,43 +10,34 @@ use conduwuit::{ }; use futures::{Stream, StreamExt, TryStreamExt}; use rocksdb::{DBPinnableSlice, ReadOptions}; -use serde::Serialize; use super::get::{cached_handle_from, handle_from}; -use crate::{keyval::KeyBuf, ser, Handle}; +use crate::Handle; -#[implement(super::Map)] -#[tracing::instrument(skip(self, keys), level = "trace")] -pub fn qry_batch<'a, S, K>( - self: &'a Arc, - keys: S, -) -> impl Stream>> + Send + 'a +pub trait Get<'a, K, S> where + Self: Sized, S: Stream + Send + 'a, - K: Serialize + Debug + 'a, + K: AsRef<[u8]> + Send + Sync + 'a, { - use crate::pool::Get; + fn get(self, map: &'a Arc) -> impl Stream>> + Send + 'a; +} - keys.ready_chunks(automatic_amplification()) - .widen_then(automatic_width(), |chunk| { - let keys = chunk - .iter() - .map(ser::serialize_to::) - .map(|result| result.expect("failed to serialize query key")) - .map(Into::into) - .collect(); - - self.db - .pool - .execute_get(Get { map: self.clone(), key: keys, res: None }) - }) - .map_ok(|results| results.into_iter().stream()) - .try_flatten() +impl<'a, K, S> Get<'a, K, S> for S +where + Self: Sized, + S: Stream + Send + 'a, + K: AsRef<[u8]> + Send + Sync + 'a, +{ + #[inline] + fn get(self, map: &'a Arc) -> impl Stream>> + Send + 'a { + map.get_batch(self) + } } #[implement(super::Map)] #[tracing::instrument(skip(self, keys), level = "trace")] -pub fn get_batch<'a, S, K>( +pub(crate) fn get_batch<'a, S, K>( self: &'a Arc, keys: S, ) -> impl Stream>> + Send + 'a diff --git a/src/database/map/qry.rs b/src/database/map/qry.rs new file mode 100644 index 00000000..401eba43 --- /dev/null +++ b/src/database/map/qry.rs @@ -0,0 +1,54 @@ +use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; + +use arrayvec::ArrayVec; +use conduwuit::{implement, Result}; +use futures::Future; +use serde::Serialize; + +use crate::{keyval::KeyBuf, ser, Handle}; + +/// Fetch a value from the database into cache, returning a reference-handle +/// asynchronously. The key is serialized into an allocated buffer to perform +/// the query. +#[implement(super::Map)] +#[inline] +pub fn qry(self: &Arc, key: &K) -> impl Future>> + Send +where + K: Serialize + ?Sized + Debug, +{ + let mut buf = KeyBuf::new(); + self.bqry(key, &mut buf) +} + +/// Fetch a value from the database into cache, returning a reference-handle +/// asynchronously. The key is serialized into a fixed-sized buffer to perform +/// the query. The maximum size is supplied as const generic parameter. +#[implement(super::Map)] +#[inline] +pub fn aqry( + self: &Arc, + key: &K, +) -> impl Future>> + Send +where + K: Serialize + ?Sized + Debug, +{ + let mut buf = ArrayVec::::new(); + self.bqry(key, &mut buf) +} + +/// Fetch a value from the database into cache, returning a reference-handle +/// asynchronously. The key is serialized into a user-supplied Writer. +#[implement(super::Map)] +#[tracing::instrument(skip(self, buf), level = "trace")] +pub fn bqry( + self: &Arc, + key: &K, + buf: &mut B, +) -> impl Future>> + Send +where + K: Serialize + ?Sized + Debug, + B: Write + AsRef<[u8]>, +{ + let key = ser::serialize(buf, key).expect("failed to serialize query key"); + self.get(key) +} diff --git a/src/database/map/qry_batch.rs b/src/database/map/qry_batch.rs new file mode 100644 index 00000000..31817c48 --- /dev/null +++ b/src/database/map/qry_batch.rs @@ -0,0 +1,63 @@ +use std::{fmt::Debug, sync::Arc}; + +use conduwuit::{ + implement, + utils::{ + stream::{automatic_amplification, automatic_width, WidebandExt}, + IterStream, + }, + Result, +}; +use futures::{Stream, StreamExt, TryStreamExt}; +use serde::Serialize; + +use crate::{keyval::KeyBuf, ser, Handle}; + +pub trait Qry<'a, K, S> +where + S: Stream + Send + 'a, + K: Serialize + Debug, +{ + fn qry(self, map: &'a Arc) -> impl Stream>> + Send + 'a; +} + +impl<'a, K, S> Qry<'a, K, S> for S +where + Self: 'a, + S: Stream + Send + 'a, + K: Serialize + Debug + 'a, +{ + #[inline] + fn qry(self, map: &'a Arc) -> impl Stream>> + Send + 'a { + map.qry_batch(self) + } +} + +#[implement(super::Map)] +#[tracing::instrument(skip(self, keys), level = "trace")] +pub(crate) fn qry_batch<'a, S, K>( + self: &'a Arc, + keys: S, +) -> impl Stream>> + Send + 'a +where + S: Stream + Send + 'a, + K: Serialize + Debug + 'a, +{ + use crate::pool::Get; + + keys.ready_chunks(automatic_amplification()) + .widen_then(automatic_width(), |chunk| { + let keys = chunk + .iter() + .map(ser::serialize_to::) + .map(|result| result.expect("failed to serialize query key")) + .map(Into::into) + .collect(); + + self.db + .pool + .execute_get(Get { map: self.clone(), key: keys, res: None }) + }) + .map_ok(|results| results.into_iter().stream()) + .try_flatten() +} diff --git a/src/database/mod.rs b/src/database/mod.rs index 42b7f5e3..4f8e2ad9 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -30,7 +30,7 @@ pub use self::{ deserialized::Deserialized, handle::Handle, keyval::{serialize_key, serialize_val, KeyVal, Slice}, - map::{compact, Map}, + map::{compact, Get, Map, Qry}, ser::{serialize, serialize_to, serialize_to_vec, Cbor, Interfix, Json, Separator, SEP}, }; pub(crate) use self::{ diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 67274ff1..a6e00271 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -7,7 +7,7 @@ use conduwuit::{ utils::{stream::TryIgnore, IterStream, ReadyExt}, Result, }; -use database::{Database, Deserialized, Handle, Interfix, Map}; +use database::{Database, Deserialized, Handle, Interfix, Map, Qry}; use futures::{pin_mut, Stream, StreamExt}; use ruma::{api::client::filter::LazyLoadOptions, DeviceId, OwnedUserId, RoomId, UserId}; @@ -115,9 +115,11 @@ where let make_key = |sender: &'a UserId| -> Key<'a> { (ctx.user_id, ctx.device_id, ctx.room_id, sender) }; - self.db - .lazyloadedids - .qry_batch(senders.clone().stream().map(make_key)) + senders + .clone() + .stream() + .map(make_key) + .qry(&self.db.lazyloadedids) .map(into_status) .zip(senders.stream()) .map(move |(status, sender)| { diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 4a591592..dd586d02 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -2,7 +2,7 @@ use std::{borrow::Borrow, fmt::Debug, mem::size_of_val, sync::Arc}; pub use conduwuit::pdu::{ShortEventId, ShortId, ShortRoomId}; use conduwuit::{err, implement, utils, utils::IterStream, Result}; -use database::{Deserialized, Map}; +use database::{Deserialized, Get, Map, Qry}; use futures::{Stream, StreamExt}; use ruma::{events::StateEventType, EventId, RoomId}; use serde::Deserialize; @@ -67,9 +67,10 @@ pub fn multi_get_or_create_shorteventid<'a, I>( where I: Iterator + Clone + Debug + Send + 'a, { - self.db - .eventid_shorteventid - .get_batch(event_ids.clone().stream()) + event_ids + .clone() + .stream() + .get(&self.db.eventid_shorteventid) .zip(event_ids.into_iter().stream()) .map(|(result, event_id)| match result { | Ok(ref short) => utils::u64_from_u8(short), @@ -171,9 +172,8 @@ where Id: for<'de> Deserialize<'de> + Sized + ToOwned + 'a, ::Owned: Borrow, { - self.db - .shorteventid_eventid - .qry_batch(shorteventid) + shorteventid + .qry(&self.db.shorteventid_eventid) .map(Deserialized::deserialized) } @@ -204,9 +204,8 @@ pub fn multi_get_statekey_from_short<'a, S>( where S: Stream + Send + 'a, { - self.db - .shortstatekey_statekey - .qry_batch(shortstatekey) + shortstatekey + .qry(&self.db.shortstatekey_statekey) .map(Deserialized::deserialized) } From 6db8df5e232fad8c1e229194e6336b9f268b13c1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 04:26:40 +0000 Subject: [PATCH 1433/2091] skip redundant acl check when sender is origin Signed-off-by: Jason Volk --- src/service/rooms/event_handler/handle_incoming_pdu.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 4e6f0b0c..94d4bcc7 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -79,7 +79,9 @@ pub async fn handle_incoming_pdu<'a>( .try_into() .map_err(|e| err!(Request(InvalidParam("PDU does not have a valid sender key: {e}"))))?; - self.acl_check(sender.server_name(), room_id).await?; + if sender.server_name() != origin { + self.acl_check(sender.server_name(), room_id).await?; + } // Fetch create event let create_event = self From 13335042b715b8279cd9adbaa6425a11eb7d3e64 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 04:47:07 +0000 Subject: [PATCH 1434/2091] enable the paranoid-checks options in debug mode Signed-off-by: Jason Volk --- src/database/engine/cf_opts.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 382bc169..83bce08c 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -72,6 +72,13 @@ fn descriptor_cf_options( opts.set_options_from_string("{{arena_block_size=2097152;}}") .map_err(map_err)?; + #[cfg(debug_assertions)] + opts.set_options_from_string( + "{{paranoid_checks=true;paranoid_file_checks=true;force_consistency_checks=true;\ + verify_sst_unique_id_in_manifest=true;}}", + ) + .map_err(map_err)?; + Ok(opts) } From 98f95705478dbe60a56206dd2ae9057f602040ea Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 07:05:00 +0000 Subject: [PATCH 1435/2091] add option to disable rocksdb checksums reference runtime state for default option initialization Signed-off-by: Jason Volk --- conduwuit-example.toml | 7 ++++ src/core/config/mod.rs | 9 ++++++ src/database/engine.rs | 1 + src/database/engine/open.rs | 1 + src/database/map.rs | 6 ++-- src/database/map/keys.rs | 2 +- src/database/map/keys_from.rs | 2 +- src/database/map/options.rs | 50 +++++++++++++++++------------ src/database/map/rev_keys.rs | 2 +- src/database/map/rev_keys_from.rs | 2 +- src/database/map/rev_stream.rs | 4 +-- src/database/map/rev_stream_from.rs | 4 +-- src/database/map/stream.rs | 4 +-- src/database/map/stream_from.rs | 4 +-- 14 files changed, 62 insertions(+), 36 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 3ecc1628..51d948e8 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -897,6 +897,13 @@ # #rocksdb_paranoid_file_checks = false +# Enables or disables checksum verification in rocksdb at runtime. +# Checksums are usually hardware accelerated with low overhead; they are +# enabled in rocksdb by default. Older or slower platforms may see gains +# from disabling. +# +#rocksdb_checksums = true + # Database repair mode (for RocksDB SST corruption). # # Use this option when the server reports corruption while running or diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 133f0887..94788fa4 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1049,6 +1049,15 @@ pub struct Config { #[serde(default)] pub rocksdb_paranoid_file_checks: bool, + /// Enables or disables checksum verification in rocksdb at runtime. + /// Checksums are usually hardware accelerated with low overhead; they are + /// enabled in rocksdb by default. Older or slower platforms may see gains + /// from disabling. + /// + /// default: true + #[serde(default = "true_fn")] + pub rocksdb_checksums: bool, + /// Database repair mode (for RocksDB SST corruption). /// /// Use this option when the server reports corruption while running or diff --git a/src/database/engine.rs b/src/database/engine.rs index 76b2889b..be3d62cf 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -32,6 +32,7 @@ use crate::{ pub struct Engine { pub(super) read_only: bool, pub(super) secondary: bool, + pub(crate) checksums: bool, corks: AtomicU32, pub(crate) db: Db, pub(crate) pool: Arc, diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs index 6a801878..ad724765 100644 --- a/src/database/engine/open.rs +++ b/src/database/engine/open.rs @@ -58,6 +58,7 @@ pub(crate) async fn open(ctx: Arc, desc: &[Descriptor]) -> Result) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_fwd(None); diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index b83775ac..76c76325 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -53,7 +53,7 @@ where { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self, from) { return stream::Keys::<'_>::from(state.init_fwd(from.as_ref().into())).boxed(); diff --git a/src/database/map/options.rs b/src/database/map/options.rs index f726036d..9e2ad898 100644 --- a/src/database/map/options.rs +++ b/src/database/map/options.rs @@ -1,35 +1,43 @@ +use std::sync::Arc; + use rocksdb::{ReadOptions, ReadTier, WriteOptions}; -#[inline] -pub(crate) fn iter_options_default() -> ReadOptions { - let mut options = read_options_default(); - options.set_background_purge_on_iterator_cleanup(true); - //options.set_pin_data(true); - options -} +use crate::Engine; #[inline] -pub(crate) fn cache_iter_options_default() -> ReadOptions { - let mut options = cache_read_options_default(); - options.set_background_purge_on_iterator_cleanup(true); - //options.set_pin_data(true); - options -} - -#[inline] -pub(crate) fn cache_read_options_default() -> ReadOptions { - let mut options = read_options_default(); +pub(crate) fn cache_iter_options_default(db: &Arc) -> ReadOptions { + let mut options = iter_options_default(db); options.set_read_tier(ReadTier::BlockCache); options.fill_cache(false); options } #[inline] -pub(crate) fn read_options_default() -> ReadOptions { - let mut options = ReadOptions::default(); - options.set_total_order_seek(true); +pub(crate) fn iter_options_default(db: &Arc) -> ReadOptions { + let mut options = read_options_default(db); + options.set_background_purge_on_iterator_cleanup(true); options } #[inline] -pub(crate) fn write_options_default() -> WriteOptions { WriteOptions::default() } +pub(crate) fn cache_read_options_default(db: &Arc) -> ReadOptions { + let mut options = read_options_default(db); + options.set_read_tier(ReadTier::BlockCache); + options.fill_cache(false); + options +} + +#[inline] +pub(crate) fn read_options_default(db: &Arc) -> ReadOptions { + let mut options = ReadOptions::default(); + options.set_total_order_seek(true); + + if !db.checksums { + options.set_verify_checksums(false); + } + + options +} + +#[inline] +pub(crate) fn write_options_default(_db: &Arc) -> WriteOptions { WriteOptions::default() } diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index a559d04b..21558a17 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -22,7 +22,7 @@ where pub fn rev_raw_keys(self: &Arc) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_rev(None); diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index 5b159195..65072337 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -61,7 +61,7 @@ where { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self, from) { return stream::KeysRev::<'_>::from(state.init_rev(from.as_ref().into())).boxed(); diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index 56b20b9b..f55053be 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -31,7 +31,7 @@ where pub fn rev_raw_stream(self: &Arc) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_rev(None); @@ -66,7 +66,7 @@ pub fn rev_raw_stream(self: &Arc) -> impl Stream> fields(%map), )] pub(super) fn is_cached(map: &Arc) -> bool { - let opts = super::cache_iter_options_default(); + let opts = super::cache_iter_options_default(&map.db); let state = stream::State::new(map, opts).init_rev(None); !state.is_incomplete() diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index 83832bdd..ddc98607 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -80,7 +80,7 @@ where { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self, from) { let state = state.init_rev(from.as_ref().into()); @@ -118,7 +118,7 @@ pub(super) fn is_cached

    (map: &Arc, from: &P) -> bool where P: AsRef<[u8]> + ?Sized, { - let cache_opts = super::cache_iter_options_default(); + let cache_opts = super::cache_iter_options_default(&map.db); let cache_status = stream::State::new(map, cache_opts) .init_rev(from.as_ref().into()) .status(); diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index f1b5fdc3..bfc8ba04 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -30,7 +30,7 @@ where pub fn raw_stream(self: &Arc) -> impl Stream>> + Send { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self) { let state = state.init_fwd(None); @@ -65,7 +65,7 @@ pub fn raw_stream(self: &Arc) -> impl Stream>> + fields(%map), )] pub(super) fn is_cached(map: &Arc) -> bool { - let opts = super::cache_iter_options_default(); + let opts = super::cache_iter_options_default(&map.db); let state = stream::State::new(map, opts).init_fwd(None); !state.is_incomplete() diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 562ab6b1..74140a65 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -77,7 +77,7 @@ where { use crate::pool::Seek; - let opts = super::iter_options_default(); + let opts = super::iter_options_default(&self.db); let state = stream::State::new(self, opts); if is_cached(self, from) { let state = state.init_fwd(from.as_ref().into()); @@ -115,7 +115,7 @@ pub(super) fn is_cached

    (map: &Arc, from: &P) -> bool where P: AsRef<[u8]> + ?Sized, { - let opts = super::cache_iter_options_default(); + let opts = super::cache_iter_options_default(&map.db); let state = stream::State::new(map, opts).init_fwd(from.as_ref().into()); !state.is_incomplete() From 2b730a30ad8f6fc0b188f79f7c45746ba46eff52 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 09:59:59 +0000 Subject: [PATCH 1436/2091] add broad_flat_map Signed-off-by: Jason Volk --- src/core/utils/stream/broadband.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/core/utils/stream/broadband.rs b/src/core/utils/stream/broadband.rs index 6d1ff6fe..282008e7 100644 --- a/src/core/utils/stream/broadband.rs +++ b/src/core/utils/stream/broadband.rs @@ -35,6 +35,13 @@ where Fut: Future> + Send, U: Send; + fn broadn_flat_map(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Stream + Send + Unpin, + U: Send; + fn broadn_then(self, n: N, f: F) -> impl Stream + Send where N: Into>, @@ -70,6 +77,16 @@ where self.broadn_filter_map(None, f) } + #[inline] + fn broad_flat_map(self, f: F) -> impl Stream + Send + where + F: Fn(Item) -> Fut + Send, + Fut: Stream + Send + Unpin, + U: Send, + { + self.broadn_flat_map(None, f) + } + #[inline] fn broad_then(self, f: F) -> impl Stream + Send where @@ -122,6 +139,17 @@ where .ready_filter_map(identity) } + #[inline] + fn broadn_flat_map(self, n: N, f: F) -> impl Stream + Send + where + N: Into>, + F: Fn(Item) -> Fut + Send, + Fut: Stream + Send + Unpin, + U: Send, + { + self.flat_map_unordered(n.into().unwrap_or_else(automatic_width), f) + } + #[inline] fn broadn_then(self, n: N, f: F) -> impl Stream + Send where From 677316631a029fdc23fb48092a1af14284e26448 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 06:15:01 +0000 Subject: [PATCH 1437/2091] pipeline prologue of handle_incoming_pdu simplify room_version/first_pdu_in_room argument passing Signed-off-by: Jason Volk --- .../fetch_and_handle_outliers.rs | 13 ++-- src/service/rooms/event_handler/fetch_prev.rs | 17 ++--- .../rooms/event_handler/fetch_state.rs | 5 +- .../event_handler/handle_incoming_pdu.rs | 72 +++++++++++-------- .../rooms/event_handler/handle_outlier_pdu.rs | 1 - .../rooms/event_handler/handle_prev_pdu.rs | 8 +-- .../event_handler/upgrade_outlier_pdu.rs | 2 +- 7 files changed, 62 insertions(+), 56 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 84d0edd0..540ebb64 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -10,10 +10,11 @@ use conduwuit::{ }; use futures::TryFutureExt; use ruma::{ - api::federation::event::get_event, CanonicalJsonValue, OwnedEventId, RoomId, RoomVersionId, - ServerName, + api::federation::event::get_event, CanonicalJsonValue, OwnedEventId, RoomId, ServerName, }; +use super::get_room_version_id; + /// Find the event and auth it. Once the event is validated (steps 1 - 8) /// it is appended to the outliers Tree. /// @@ -30,7 +31,6 @@ pub(super) async fn fetch_and_handle_outliers<'a>( events: &'a [OwnedEventId], create_event: &'a PduEvent, room_id: &'a RoomId, - room_version_id: &'a RoomVersionId, ) -> Vec<(Arc, Option>)> { let back_off = |id| match self .services @@ -113,8 +113,13 @@ pub(super) async fn fetch_and_handle_outliers<'a>( { | Ok(res) => { debug!("Got {next_id} over federation"); + let Ok(room_version_id) = get_room_version_id(create_event) else { + back_off((*next_id).to_owned()); + continue; + }; + let Ok((calculated_event_id, value)) = - pdu::gen_event_id_canonical_json(&res.pdu, room_version_id) + pdu::gen_event_id_canonical_json(&res.pdu, &room_version_id) else { back_off((*next_id).to_owned()); continue; diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 5966aeba..aea70739 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -8,8 +8,7 @@ use futures::{future, FutureExt}; use ruma::{ int, state_res::{self}, - uint, CanonicalJsonValue, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, - ServerName, + uint, CanonicalJsonValue, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName, UInt, }; use super::check_room_id; @@ -26,7 +25,7 @@ pub(super) async fn fetch_prev( origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, - room_version_id: &RoomVersionId, + first_ts_in_room: UInt, initial_set: Vec, ) -> Result<( Vec, @@ -36,21 +35,13 @@ pub(super) async fn fetch_prev( let mut eventid_info = HashMap::new(); let mut todo_outlier_stack: VecDeque = initial_set.into(); - let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; - let mut amount = 0; while let Some(prev_event_id) = todo_outlier_stack.pop_front() { self.services.server.check_running()?; if let Some((pdu, mut json_opt)) = self - .fetch_and_handle_outliers( - origin, - &[prev_event_id.clone()], - create_event, - room_id, - room_version_id, - ) + .fetch_and_handle_outliers(origin, &[prev_event_id.clone()], create_event, room_id) .boxed() .await .pop() @@ -74,7 +65,7 @@ pub(super) async fn fetch_prev( } if let Some(json) = json_opt { - if pdu.origin_server_ts > first_pdu_in_room.origin_server_ts { + if pdu.origin_server_ts > first_ts_in_room { amount = amount.saturating_add(1); for prev_prev in &pdu.prev_events { if !graph.contains_key(prev_prev) { diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 0892655e..4f2580db 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -4,7 +4,7 @@ use conduwuit::{debug, debug_warn, implement, Err, Error, PduEvent, Result}; use futures::FutureExt; use ruma::{ api::federation::event::get_room_state_ids, events::StateEventType, EventId, OwnedEventId, - RoomId, RoomVersionId, ServerName, + RoomId, ServerName, }; use crate::rooms::short::ShortStateKey; @@ -23,7 +23,6 @@ pub(super) async fn fetch_state( origin: &ServerName, create_event: &PduEvent, room_id: &RoomId, - room_version_id: &RoomVersionId, event_id: &EventId, ) -> Result>> { let res = self @@ -38,7 +37,7 @@ pub(super) async fn fetch_state( debug!("Fetching state events"); let state_vec = self - .fetch_and_handle_outliers(origin, &res.pdu_ids, create_event, room_id, room_version_id) + .fetch_and_handle_outliers(origin, &res.pdu_ids, create_event, room_id) .boxed() .await; diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 94d4bcc7..7db71961 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -1,14 +1,15 @@ use std::{ collections::{hash_map, BTreeMap}, - sync::Arc, time::Instant, }; use conduwuit::{debug, err, implement, warn, Err, Result}; -use futures::{FutureExt, TryFutureExt}; +use futures::{ + future::{try_join5, OptionFuture}, + FutureExt, +}; use ruma::{events::StateEventType, CanonicalJsonValue, EventId, RoomId, ServerName, UserId}; -use super::{check_room_id, get_room_version_id}; use crate::rooms::timeline::RawPduId; /// When receiving an event one needs to: @@ -59,19 +60,13 @@ pub async fn handle_incoming_pdu<'a>( } // 1.1 Check the server is in the room - if !self.services.metadata.exists(room_id).await { - return Err!(Request(NotFound("Room is unknown to this server"))); - } + let meta_exists = self.services.metadata.exists(room_id).map(Ok); // 1.2 Check if the room is disabled - if self.services.metadata.is_disabled(room_id).await { - return Err!(Request(Forbidden( - "Federation of this room is currently disabled on this server." - ))); - } + let is_disabled = self.services.metadata.is_disabled(room_id).map(Ok); // 1.3.1 Check room ACL on origin field/server - self.acl_check(origin, room_id).await?; + let origin_acl_check = self.acl_check(origin, room_id); // 1.3.2 Check room ACL on sender's server name let sender: &UserId = value @@ -79,36 +74,53 @@ pub async fn handle_incoming_pdu<'a>( .try_into() .map_err(|e| err!(Request(InvalidParam("PDU does not have a valid sender key: {e}"))))?; - if sender.server_name() != origin { - self.acl_check(sender.server_name(), room_id).await?; - } + let sender_acl_check: OptionFuture<_> = sender + .server_name() + .ne(origin) + .then(|| self.acl_check(sender.server_name(), room_id)) + .into(); // Fetch create event - let create_event = self - .services - .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "") - .map_ok(Arc::new) - .await?; + let create_event = + self.services + .state_accessor + .room_state_get(room_id, &StateEventType::RoomCreate, ""); - // Procure the room version - let room_version_id = get_room_version_id(&create_event)?; + let (meta_exists, is_disabled, (), (), create_event) = try_join5( + meta_exists, + is_disabled, + origin_acl_check, + sender_acl_check.map(|o| o.unwrap_or(Ok(()))), + create_event, + ) + .await?; - let first_pdu_in_room = self.services.timeline.first_pdu_in_room(room_id).await?; + if !meta_exists { + return Err!(Request(NotFound("Room is unknown to this server"))); + } + + if is_disabled { + return Err!(Request(Forbidden("Federation of this room is disabled by this server."))); + } let (incoming_pdu, val) = self .handle_outlier_pdu(origin, &create_event, event_id, room_id, value, false) - .boxed() .await?; - check_room_id(room_id, &incoming_pdu)?; - // 8. if not timeline event: stop if !is_timeline_event { return Ok(None); } + // Skip old events - if incoming_pdu.origin_server_ts < first_pdu_in_room.origin_server_ts { + let first_ts_in_room = self + .services + .timeline + .first_pdu_in_room(room_id) + .await? + .origin_server_ts; + + if incoming_pdu.origin_server_ts < first_ts_in_room { return Ok(None); } @@ -119,7 +131,7 @@ pub async fn handle_incoming_pdu<'a>( origin, &create_event, room_id, - &room_version_id, + first_ts_in_room, incoming_pdu.prev_events.clone(), ) .await?; @@ -134,7 +146,7 @@ pub async fn handle_incoming_pdu<'a>( room_id, &mut eventid_info, &create_event, - &first_pdu_in_room, + first_ts_in_room, &prev_id, ) .await diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 3ad73295..a35aabe0 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -84,7 +84,6 @@ pub(super) async fn handle_outlier_pdu<'a>( &incoming_pdu.auth_events, create_event, room_id, - &room_version_id, )) .await; } diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index 2bec4eba..32ab505f 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -7,7 +7,7 @@ use std::{ use conduwuit::{ debug, implement, utils::continue_exponential_backoff_secs, Err, PduEvent, Result, }; -use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName}; +use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, UInt}; #[implement(super::Service)] #[allow(clippy::type_complexity)] @@ -27,8 +27,8 @@ pub(super) async fn handle_prev_pdu<'a>( OwnedEventId, (Arc, BTreeMap), >, - create_event: &Arc, - first_pdu_in_room: &PduEvent, + create_event: &PduEvent, + first_ts_in_room: UInt, prev_id: &EventId, ) -> Result { // Check for disabled again because it might have changed @@ -62,7 +62,7 @@ pub(super) async fn handle_prev_pdu<'a>( if let Some((pdu, json)) = eventid_info.remove(prev_id) { // Skip old events - if pdu.origin_server_ts < first_pdu_in_room.origin_server_ts { + if pdu.origin_server_ts < first_ts_in_room { return Ok(()); } diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 8adf4246..f0c8f0c5 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -63,7 +63,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( if state_at_incoming_event.is_none() { state_at_incoming_event = self - .fetch_state(origin, create_event, room_id, &room_version_id, &incoming_pdu.event_id) + .fetch_state(origin, create_event, room_id, &incoming_pdu.event_id) .await?; } From 94d786ac12890306be5d0577bc3fcc8f6b856558 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 10:43:53 +0000 Subject: [PATCH 1438/2091] process rooms and edus concurrently Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/api/Cargo.toml | 1 + src/api/server/send.rs | 241 ++++++++++-------- .../rooms/event_handler/parse_incoming_pdu.rs | 9 +- src/service/rooms/timeline/mod.rs | 2 +- 5 files changed, 142 insertions(+), 112 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5848cc46..3a435a10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -685,6 +685,7 @@ dependencies = [ "http-body-util", "hyper", "ipaddress", + "itertools 0.13.0", "log", "rand", "reqwest", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 1b463fbc..385e786f 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -50,6 +50,7 @@ http.workspace = true http-body-util.workspace = true hyper.workspace = true ipaddress.workspace = true +itertools.workspace = true log.workspace = true rand.workspace = true reqwest.workspace = true diff --git a/src/api/server/send.rs b/src/api/server/send.rs index eec9bd11..016f5194 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -3,10 +3,17 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - debug, debug_warn, err, error, result::LogErr, trace, utils::ReadyExt, warn, Err, Error, - Result, + debug, debug_warn, err, error, + result::LogErr, + trace, + utils::{ + stream::{automatic_width, BroadbandExt, TryBroadbandExt}, + IterStream, ReadyExt, + }, + warn, Err, Error, Result, }; -use futures::{FutureExt, StreamExt}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; +use itertools::Itertools; use ruma::{ api::{ client::error::ErrorKind, @@ -19,11 +26,9 @@ use ruma::{ }, }, events::receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType}, - serde::Raw, to_device::DeviceIdOrAllDevices, - OwnedEventId, ServerName, + CanonicalJsonObject, OwnedEventId, OwnedRoomId, ServerName, }; -use serde_json::value::RawValue as RawJsonValue; use service::{ sending::{EDU_LIMIT, PDU_LIMIT}, Services, @@ -34,7 +39,8 @@ use crate::{ Ruma, }; -type ResolvedMap = BTreeMap>; +type ResolvedMap = BTreeMap; +type Pdu = (OwnedRoomId, OwnedEventId, CanonicalJsonObject); /// # `PUT /_matrix/federation/v1/send/{txnId}` /// @@ -73,91 +79,41 @@ pub(crate) async fn send_transaction_message_route( let txn_start_time = Instant::now(); trace!( - pdus = ?body.pdus.len(), - edus = ?body.edus.len(), + pdus = body.pdus.len(), + edus = body.edus.len(), elapsed = ?txn_start_time.elapsed(), id = ?body.transaction_id, origin =?body.origin(), "Starting txn", ); - let resolved_map = - handle_pdus(&services, &client, &body.pdus, body.origin(), &txn_start_time) - .boxed() - .await?; + let pdus = body + .pdus + .iter() + .stream() + .broad_then(|pdu| services.rooms.event_handler.parse_incoming_pdu(pdu)) + .inspect_err(|e| debug_warn!("Could not parse PDU: {e}")) + .ready_filter_map(Result::ok); - handle_edus(&services, &client, &body.edus, body.origin()) - .boxed() - .await; + let edus = body + .edus + .iter() + .map(|edu| edu.json().get()) + .map(serde_json::from_str) + .filter_map(Result::ok) + .stream(); + + let results = handle(&services, &client, body.origin(), txn_start_time, pdus, edus).await?; debug!( - pdus = ?body.pdus.len(), - edus = ?body.edus.len(), + pdus = body.pdus.len(), + edus = body.edus.len(), elapsed = ?txn_start_time.elapsed(), id = ?body.transaction_id, origin =?body.origin(), "Finished txn", ); - - Ok(send_transaction_message::v1::Response { - pdus: resolved_map - .into_iter() - .map(|(e, r)| (e, r.map_err(error::sanitized_message))) - .collect(), - }) -} - -async fn handle_pdus( - services: &Services, - _client: &IpAddr, - pdus: &[Box], - origin: &ServerName, - txn_start_time: &Instant, -) -> Result { - let mut parsed_pdus = Vec::with_capacity(pdus.len()); - for pdu in pdus { - parsed_pdus.push(match services.rooms.event_handler.parse_incoming_pdu(pdu).await { - | Ok(t) => t, - | Err(e) => { - debug_warn!("Could not parse PDU: {e}"); - continue; - }, - }); - - // We do not add the event_id field to the pdu here because of signature - // and hashes checks - } - - let mut resolved_map = BTreeMap::new(); - for (event_id, value, room_id) in parsed_pdus { - services.server.check_running()?; - let pdu_start_time = Instant::now(); - let mutex_lock = services - .rooms - .event_handler - .mutex_federation - .lock(&room_id) - .await; - - let result = services - .rooms - .event_handler - .handle_incoming_pdu(origin, &room_id, &event_id, value, true) - .boxed() - .await - .map(|_| ()); - - drop(mutex_lock); - debug!( - pdu_elapsed = ?pdu_start_time.elapsed(), - txn_elapsed = ?txn_start_time.elapsed(), - "Finished PDU {event_id}", - ); - - resolved_map.insert(event_id, result); - } - - for (id, result) in &resolved_map { + for (id, result) in &results { if let Err(e) = result { if matches!(e, Error::BadRequest(ErrorKind::NotFound, _)) { warn!("Incoming PDU failed {id}: {e:?}"); @@ -165,39 +121,112 @@ async fn handle_pdus( } } - Ok(resolved_map) + Ok(send_transaction_message::v1::Response { + pdus: results + .into_iter() + .map(|(e, r)| (e, r.map_err(error::sanitized_message))) + .collect(), + }) } -async fn handle_edus( +async fn handle( services: &Services, client: &IpAddr, - edus: &[Raw], origin: &ServerName, -) { - for edu in edus - .iter() - .filter_map(|edu| serde_json::from_str::(edu.json().get()).ok()) - { - match edu { - | Edu::Presence(presence) => { - handle_edu_presence(services, client, origin, presence).await; - }, - | Edu::Receipt(receipt) => - handle_edu_receipt(services, client, origin, receipt).await, - | Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await, - | Edu::DeviceListUpdate(content) => { - handle_edu_device_list_update(services, client, origin, content).await; - }, - | Edu::DirectToDevice(content) => { - handle_edu_direct_to_device(services, client, origin, content).await; - }, - | Edu::SigningKeyUpdate(content) => { - handle_edu_signing_key_update(services, client, origin, content).await; - }, - | Edu::_Custom(ref _custom) => { - debug_warn!(?edus, "received custom/unknown EDU"); - }, - } + started: Instant, + pdus: impl Stream + Send, + edus: impl Stream + Send, +) -> Result { + // group pdus by room + let pdus = pdus + .collect() + .map(|mut pdus: Vec<_>| { + pdus.sort_by(|(room_a, ..), (room_b, ..)| room_a.cmp(room_b)); + pdus.into_iter() + .into_grouping_map_by(|(room_id, ..)| room_id.clone()) + .collect() + }) + .await; + + // we can evaluate rooms concurrently + let results: ResolvedMap = pdus + .into_iter() + .try_stream() + .broad_and_then(|(room_id, pdus)| { + handle_room(services, client, origin, started, room_id, pdus) + .map_ok(Vec::into_iter) + .map_ok(IterStream::try_stream) + }) + .try_flatten() + .try_collect() + .boxed() + .await?; + + // evaluate edus after pdus, at least for now. + edus.for_each_concurrent(automatic_width(), |edu| handle_edu(services, client, origin, edu)) + .boxed() + .await; + + Ok(results) +} + +async fn handle_room( + services: &Services, + _client: &IpAddr, + origin: &ServerName, + txn_start_time: Instant, + room_id: OwnedRoomId, + pdus: Vec, +) -> Result> { + let _room_lock = services + .rooms + .event_handler + .mutex_federation + .lock(&room_id) + .await; + + let mut results = Vec::with_capacity(pdus.len()); + for (_, event_id, value) in pdus { + services.server.check_running()?; + let pdu_start_time = Instant::now(); + let result = services + .rooms + .event_handler + .handle_incoming_pdu(origin, &room_id, &event_id, value, true) + .await + .map(|_| ()); + + debug!( + pdu_elapsed = ?pdu_start_time.elapsed(), + txn_elapsed = ?txn_start_time.elapsed(), + "Finished PDU {event_id}", + ); + + results.push((event_id, result)); + } + + Ok(results) +} + +async fn handle_edu(services: &Services, client: &IpAddr, origin: &ServerName, edu: Edu) { + match edu { + | Edu::Presence(presence) => { + handle_edu_presence(services, client, origin, presence).await; + }, + | Edu::Receipt(receipt) => handle_edu_receipt(services, client, origin, receipt).await, + | Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await, + | Edu::DeviceListUpdate(content) => { + handle_edu_device_list_update(services, client, origin, content).await; + }, + | Edu::DirectToDevice(content) => { + handle_edu_direct_to_device(services, client, origin, content).await; + }, + | Edu::SigningKeyUpdate(content) => { + handle_edu_signing_key_update(services, client, origin, content).await; + }, + | Edu::_Custom(ref _custom) => { + debug_warn!(?edu, "received custom/unknown EDU"); + }, } } diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 0c11314d..9b130763 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -2,11 +2,10 @@ use conduwuit::{err, implement, pdu::gen_event_id_canonical_json, result::FlatOk use ruma::{CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId}; use serde_json::value::RawValue as RawJsonValue; +type Parsed = (OwnedRoomId, OwnedEventId, CanonicalJsonObject); + #[implement(super::Service)] -pub async fn parse_incoming_pdu( - &self, - pdu: &RawJsonValue, -) -> Result<(OwnedEventId, CanonicalJsonObject, OwnedRoomId)> { +pub async fn parse_incoming_pdu(&self, pdu: &RawJsonValue) -> Result { let value = serde_json::from_str::(pdu.get()).map_err(|e| { err!(BadServerResponse(debug_warn!("Error parsing incoming event {e:?}"))) })?; @@ -28,5 +27,5 @@ pub async fn parse_incoming_pdu( err!(Request(InvalidParam("Could not convert event to canonical json: {e}"))) })?; - Ok((event_id, value, room_id)) + Ok((room_id, event_id, value)) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 362bfab5..bf585a6b 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1166,7 +1166,7 @@ impl Service { #[tracing::instrument(skip(self, pdu), level = "debug")] pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> Result<()> { - let (event_id, value, room_id) = + let (room_id, event_id, value) = self.services.event_handler.parse_incoming_pdu(&pdu).await?; // Lock so we cannot backfill the same pdu twice at the same time From c516a8df3e80d38656f2251204119c720b5d96f7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 15:12:08 +0000 Subject: [PATCH 1439/2091] fanout edu processing Signed-off-by: Jason Volk --- src/api/server/send.rs | 459 ++++++++++++++++++++++++----------------- 1 file changed, 265 insertions(+), 194 deletions(-) diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 016f5194..f4903447 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -20,19 +20,22 @@ use ruma::{ federation::transactions::{ edu::{ DeviceListUpdateContent, DirectDeviceContent, Edu, PresenceContent, - ReceiptContent, SigningKeyUpdateContent, TypingContent, + PresenceUpdate, ReceiptContent, ReceiptData, ReceiptMap, SigningKeyUpdateContent, + TypingContent, }, send_transaction_message, }, }, events::receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType}, + serde::Raw, to_device::DeviceIdOrAllDevices, - CanonicalJsonObject, OwnedEventId, OwnedRoomId, ServerName, + CanonicalJsonObject, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, ServerName, UserId, }; use service::{ sending::{EDU_LIMIT, PDU_LIMIT}, Services, }; +use utils::millis_since_unix_epoch; use crate::{ utils::{self}, @@ -152,8 +155,8 @@ async fn handle( let results: ResolvedMap = pdus .into_iter() .try_stream() - .broad_and_then(|(room_id, pdus)| { - handle_room(services, client, origin, started, room_id, pdus) + .broad_and_then(|(room_id, pdus): (_, Vec<_>)| { + handle_room(services, client, origin, started, room_id, pdus.into_iter()) .map_ok(Vec::into_iter) .map_ok(IterStream::try_stream) }) @@ -176,7 +179,7 @@ async fn handle_room( origin: &ServerName, txn_start_time: Instant, room_id: OwnedRoomId, - pdus: Vec, + pdus: impl Iterator + Send, ) -> Result> { let _room_lock = services .rooms @@ -185,48 +188,53 @@ async fn handle_room( .lock(&room_id) .await; - let mut results = Vec::with_capacity(pdus.len()); - for (_, event_id, value) in pdus { - services.server.check_running()?; - let pdu_start_time = Instant::now(); - let result = services - .rooms - .event_handler - .handle_incoming_pdu(origin, &room_id, &event_id, value, true) - .await - .map(|_| ()); + let room_id = &room_id; + pdus.try_stream() + .and_then(|(_, event_id, value)| async move { + services.server.check_running()?; + let pdu_start_time = Instant::now(); + let result = services + .rooms + .event_handler + .handle_incoming_pdu(origin, room_id, &event_id, value, true) + .await + .map(|_| ()); - debug!( - pdu_elapsed = ?pdu_start_time.elapsed(), - txn_elapsed = ?txn_start_time.elapsed(), - "Finished PDU {event_id}", - ); + debug!( + pdu_elapsed = ?pdu_start_time.elapsed(), + txn_elapsed = ?txn_start_time.elapsed(), + "Finished PDU {event_id}", + ); - results.push((event_id, result)); - } - - Ok(results) + Ok((event_id, result)) + }) + .try_collect() + .await } async fn handle_edu(services: &Services, client: &IpAddr, origin: &ServerName, edu: Edu) { match edu { - | Edu::Presence(presence) => { - handle_edu_presence(services, client, origin, presence).await; - }, - | Edu::Receipt(receipt) => handle_edu_receipt(services, client, origin, receipt).await, - | Edu::Typing(typing) => handle_edu_typing(services, client, origin, typing).await, - | Edu::DeviceListUpdate(content) => { - handle_edu_device_list_update(services, client, origin, content).await; - }, - | Edu::DirectToDevice(content) => { - handle_edu_direct_to_device(services, client, origin, content).await; - }, - | Edu::SigningKeyUpdate(content) => { - handle_edu_signing_key_update(services, client, origin, content).await; - }, - | Edu::_Custom(ref _custom) => { - debug_warn!(?edu, "received custom/unknown EDU"); - }, + | Edu::Presence(presence) if services.server.config.allow_incoming_presence => + handle_edu_presence(services, client, origin, presence).await, + + | Edu::Receipt(receipt) if services.server.config.allow_incoming_read_receipts => + handle_edu_receipt(services, client, origin, receipt).await, + + | Edu::Typing(typing) if services.server.config.allow_incoming_typing => + handle_edu_typing(services, client, origin, typing).await, + + | Edu::DeviceListUpdate(content) => + handle_edu_device_list_update(services, client, origin, content).await, + + | Edu::DirectToDevice(content) => + handle_edu_direct_to_device(services, client, origin, content).await, + + | Edu::SigningKeyUpdate(content) => + handle_edu_signing_key_update(services, client, origin, content).await, + + | Edu::_Custom(ref _custom) => debug_warn!(?edu, "received custom/unknown EDU"), + + | _ => trace!(?edu, "skipped"), } } @@ -236,32 +244,41 @@ async fn handle_edu_presence( origin: &ServerName, presence: PresenceContent, ) { - if !services.globals.allow_incoming_presence() { + presence + .push + .into_iter() + .stream() + .for_each_concurrent(automatic_width(), |update| { + handle_edu_presence_update(services, origin, update) + }) + .await; +} + +async fn handle_edu_presence_update( + services: &Services, + origin: &ServerName, + update: PresenceUpdate, +) { + if update.user_id.server_name() != origin { + debug_warn!( + %update.user_id, %origin, + "received presence EDU for user not belonging to origin" + ); return; } - for update in presence.push { - if update.user_id.server_name() != origin { - debug_warn!( - %update.user_id, %origin, - "received presence EDU for user not belonging to origin" - ); - continue; - } - - services - .presence - .set_presence( - &update.user_id, - &update.presence, - Some(update.currently_active), - Some(update.last_active_ago), - update.status_msg.clone(), - ) - .await - .log_err() - .ok(); - } + services + .presence + .set_presence( + &update.user_id, + &update.presence, + Some(update.currently_active), + Some(update.last_active_ago), + update.status_msg.clone(), + ) + .await + .log_err() + .ok(); } async fn handle_edu_receipt( @@ -270,66 +287,94 @@ async fn handle_edu_receipt( origin: &ServerName, receipt: ReceiptContent, ) { - if !services.globals.allow_incoming_read_receipts() { + receipt + .receipts + .into_iter() + .stream() + .for_each_concurrent(automatic_width(), |(room_id, room_updates)| { + handle_edu_receipt_room(services, origin, room_id, room_updates) + }) + .await; +} + +async fn handle_edu_receipt_room( + services: &Services, + origin: &ServerName, + room_id: OwnedRoomId, + room_updates: ReceiptMap, +) { + if services + .rooms + .event_handler + .acl_check(origin, &room_id) + .await + .is_err() + { + debug_warn!( + %origin, %room_id, + "received read receipt EDU from ACL'd server" + ); return; } - for (room_id, room_updates) in receipt.receipts { - if services - .rooms - .event_handler - .acl_check(origin, &room_id) - .await - .is_err() - { - debug_warn!( - %origin, %room_id, - "received read receipt EDU from ACL'd server" - ); - continue; - } + let room_id = &room_id; + room_updates + .read + .into_iter() + .stream() + .for_each_concurrent(automatic_width(), |(user_id, user_updates)| async move { + handle_edu_receipt_room_user(services, origin, room_id, &user_id, user_updates).await; + }) + .await; +} - for (user_id, user_updates) in room_updates.read { - if user_id.server_name() != origin { - debug_warn!( - %user_id, %origin, - "received read receipt EDU for user not belonging to origin" - ); - continue; - } - - if services - .rooms - .state_cache - .room_members(&room_id) - .ready_any(|member| member.server_name() == user_id.server_name()) - .await - { - for event_id in &user_updates.event_ids { - let user_receipts = - BTreeMap::from([(user_id.clone(), user_updates.data.clone())]); - let receipts = BTreeMap::from([(ReceiptType::Read, user_receipts)]); - let receipt_content = BTreeMap::from([(event_id.to_owned(), receipts)]); - let event = ReceiptEvent { - content: ReceiptEventContent(receipt_content), - room_id: room_id.clone(), - }; - - services - .rooms - .read_receipt - .readreceipt_update(&user_id, &room_id, &event) - .await; - } - } else { - debug_warn!( - %user_id, %room_id, %origin, - "received read receipt EDU from server who does not have a member in the room", - ); - continue; - } - } +async fn handle_edu_receipt_room_user( + services: &Services, + origin: &ServerName, + room_id: &RoomId, + user_id: &UserId, + user_updates: ReceiptData, +) { + if user_id.server_name() != origin { + debug_warn!( + %user_id, %origin, + "received read receipt EDU for user not belonging to origin" + ); + return; } + + if !services + .rooms + .state_cache + .server_in_room(origin, room_id) + .await + { + debug_warn!( + %user_id, %room_id, %origin, + "received read receipt EDU from server who does not have a member in the room", + ); + return; + } + + let data = &user_updates.data; + user_updates + .event_ids + .into_iter() + .stream() + .for_each_concurrent(automatic_width(), |event_id| async move { + let user_data = [(user_id.to_owned(), data.clone())]; + let receipts = [(ReceiptType::Read, BTreeMap::from(user_data))]; + let content = [(event_id.clone(), BTreeMap::from(receipts))]; + services + .rooms + .read_receipt + .readreceipt_update(user_id, room_id, &ReceiptEvent { + content: ReceiptEventContent(content.into()), + room_id: room_id.to_owned(), + }) + .await; + }) + .await; } async fn handle_edu_typing( @@ -338,10 +383,6 @@ async fn handle_edu_typing( origin: &ServerName, typing: TypingContent, ) { - if !services.server.config.allow_incoming_typing { - return; - } - if typing.user_id.server_name() != origin { debug_warn!( %typing.user_id, %origin, @@ -364,41 +405,38 @@ async fn handle_edu_typing( return; } - if services + if !services .rooms .state_cache .is_joined(&typing.user_id, &typing.room_id) .await { - if typing.typing { - let timeout = utils::millis_since_unix_epoch().saturating_add( - services - .server - .config - .typing_federation_timeout_s - .saturating_mul(1000), - ); - services - .rooms - .typing - .typing_add(&typing.user_id, &typing.room_id, timeout) - .await - .log_err() - .ok(); - } else { - services - .rooms - .typing - .typing_remove(&typing.user_id, &typing.room_id) - .await - .log_err() - .ok(); - } - } else { debug_warn!( %typing.user_id, %typing.room_id, %origin, "received typing EDU for user not in room" ); + return; + } + + if typing.typing { + let secs = services.server.config.typing_federation_timeout_s; + let timeout = millis_since_unix_epoch().saturating_add(secs.saturating_mul(1000)); + + services + .rooms + .typing + .typing_add(&typing.user_id, &typing.room_id, timeout) + .await + .log_err() + .ok(); + } else { + services + .rooms + .typing + .typing_remove(&typing.user_id, &typing.room_id) + .await + .log_err() + .ok(); } } @@ -427,7 +465,12 @@ async fn handle_edu_direct_to_device( origin: &ServerName, content: DirectDeviceContent, ) { - let DirectDeviceContent { sender, ev_type, message_id, messages } = content; + let DirectDeviceContent { + ref sender, + ref ev_type, + ref message_id, + messages, + } = content; if sender.server_name() != origin { debug_warn!( @@ -440,60 +483,88 @@ async fn handle_edu_direct_to_device( // Check if this is a new transaction id if services .transaction_ids - .existing_txnid(&sender, None, &message_id) + .existing_txnid(sender, None, message_id) .await .is_ok() { return; } - for (target_user_id, map) in &messages { - for (target_device_id_maybe, event) in map { - let Ok(event) = event.deserialize_as().map_err(|e| { - err!(Request(InvalidParam(error!("To-Device event is invalid: {e}")))) - }) else { - continue; - }; - - let ev_type = ev_type.to_string(); - match target_device_id_maybe { - | DeviceIdOrAllDevices::DeviceId(target_device_id) => { - services - .users - .add_to_device_event( - &sender, - target_user_id, - target_device_id, - &ev_type, - event, - ) - .await; - }, - - | DeviceIdOrAllDevices::AllDevices => { - let (sender, ev_type, event) = (&sender, &ev_type, &event); - services - .users - .all_device_ids(target_user_id) - .for_each(|target_device_id| { - services.users.add_to_device_event( - sender, - target_user_id, - target_device_id, - ev_type, - event.clone(), - ) - }) - .await; - }, - } - } - } + // process messages concurrently for different users + let ev_type = ev_type.to_string(); + messages + .into_iter() + .stream() + .for_each_concurrent(automatic_width(), |(target_user_id, map)| { + handle_edu_direct_to_device_user(services, target_user_id, sender, &ev_type, map) + }) + .await; // Save transaction id with empty data services .transaction_ids - .add_txnid(&sender, None, &message_id, &[]); + .add_txnid(sender, None, message_id, &[]); +} + +async fn handle_edu_direct_to_device_user( + services: &Services, + target_user_id: OwnedUserId, + sender: &UserId, + ev_type: &str, + map: BTreeMap>, +) { + for (target_device_id_maybe, event) in map { + let Ok(event) = event + .deserialize_as() + .map_err(|e| err!(Request(InvalidParam(error!("To-Device event is invalid: {e}"))))) + else { + continue; + }; + + handle_edu_direct_to_device_event( + services, + &target_user_id, + sender, + target_device_id_maybe, + ev_type, + event, + ) + .await; + } +} + +async fn handle_edu_direct_to_device_event( + services: &Services, + target_user_id: &UserId, + sender: &UserId, + target_device_id_maybe: DeviceIdOrAllDevices, + ev_type: &str, + event: serde_json::Value, +) { + match target_device_id_maybe { + | DeviceIdOrAllDevices::DeviceId(ref target_device_id) => { + services + .users + .add_to_device_event(sender, target_user_id, target_device_id, ev_type, event) + .await; + }, + + | DeviceIdOrAllDevices::AllDevices => { + services + .users + .all_device_ids(target_user_id) + .for_each(|target_device_id| { + services.users.add_to_device_event( + sender, + target_user_id, + target_device_id, + ev_type, + event.clone(), + ) + }) + .await; + }, + } } async fn handle_edu_signing_key_update( From b2a565b0b4e32cf998ee5877cecded31f1305240 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 15:44:52 +0000 Subject: [PATCH 1440/2091] propagate better error from server.check_running() --- src/core/error/response.rs | 1 + src/core/server.rs | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/core/error/response.rs b/src/core/error/response.rs index ede1a05d..75e4050d 100644 --- a/src/core/error/response.rs +++ b/src/core/error/response.rs @@ -106,6 +106,7 @@ pub(super) fn io_error_code(kind: std::io::ErrorKind) -> StatusCode { | ErrorKind::TimedOut => StatusCode::GATEWAY_TIMEOUT, | ErrorKind::FileTooLarge => StatusCode::PAYLOAD_TOO_LARGE, | ErrorKind::StorageFull => StatusCode::INSUFFICIENT_STORAGE, + | ErrorKind::Interrupted => StatusCode::SERVICE_UNAVAILABLE, | _ => StatusCode::INTERNAL_SERVER_ERROR, } } diff --git a/src/core/server.rs b/src/core/server.rs index 05a4aae7..0f2e61b0 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -9,7 +9,7 @@ use std::{ use ruma::OwnedServerName; use tokio::{runtime, sync::broadcast}; -use crate::{config, config::Config, err, log::Log, metrics::Metrics, Err, Result}; +use crate::{config, config::Config, log::Log, metrics::Metrics, Err, Result}; /// Server runtime state; public portion pub struct Server { @@ -127,9 +127,12 @@ impl Server { #[inline] pub fn check_running(&self) -> Result { + use std::{io, io::ErrorKind::Interrupted}; + self.running() .then_some(()) - .ok_or_else(|| err!(debug_warn!("Server is shutting down."))) + .ok_or_else(|| io::Error::new(Interrupted, "Server shutting down")) + .map_err(Into::into) } #[inline] From ffd0fd42424a234d4fbd564b66b79521595b5b5b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 21:46:46 +0000 Subject: [PATCH 1441/2091] pipeline pdu fetch for federation sending destination Signed-off-by: Jason Volk --- src/service/sending/sender.rs | 135 ++++++++++++++++------------------ 1 file changed, 64 insertions(+), 71 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index c91e1d31..47be01f1 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -8,12 +8,12 @@ use std::{ time::{Duration, Instant}, }; -use base64::{engine::general_purpose, Engine as _}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use conduwuit::{ debug, err, error, result::LogErr, trace, - utils::{calculate_hash, continue_exponential_backoff_secs, ReadyExt}, + utils::{calculate_hash, continue_exponential_backoff_secs, stream::IterStream, ReadyExt}, warn, Error, Result, }; use futures::{ @@ -38,7 +38,9 @@ use ruma::{ push_rules::PushRulesEvent, receipt::ReceiptType, AnySyncEphemeralRoomEvent, GlobalAccountDataEventType, }, - push, uint, CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedServerName, + push, + serde::Raw, + uint, CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UInt, }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; @@ -633,7 +635,7 @@ impl Service { } fn send_events(&self, dest: Destination, events: Vec) -> SendingFuture<'_> { - //debug_assert!(!events.is_empty(), "sending empty transaction"); + debug_assert!(!events.is_empty(), "sending empty transaction"); match dest { | Destination::Federation(server) => self.send_events_dest_federation(server, events).boxed(), @@ -698,7 +700,7 @@ impl Service { | SendingEvent::Flush => None, })); - let txn_id = &*general_purpose::URL_SAFE_NO_PAD.encode(txn_hash); + let txn_id = &*URL_SAFE_NO_PAD.encode(txn_hash); //debug_assert!(pdu_jsons.len() + edu_jsons.len() > 0, "sending empty // transaction"); @@ -796,81 +798,72 @@ impl Service { Ok(Destination::Push(user_id, pushkey)) } - #[tracing::instrument( - name = "fed", - level = "debug", - skip(self, events), - fields( - events = %events.len(), - ), - )] async fn send_events_dest_federation( &self, server: OwnedServerName, events: Vec, ) -> SendingResult { - let mut pdu_jsons = Vec::with_capacity( - events - .iter() - .filter(|event| matches!(event, SendingEvent::Pdu(_))) - .count(), - ); - let mut edu_jsons = Vec::with_capacity( - events - .iter() - .filter(|event| matches!(event, SendingEvent::Edu(_))) - .count(), - ); + let pdus: Vec<_> = events + .iter() + .filter_map(|pdu| match pdu { + | SendingEvent::Pdu(pdu) => Some(pdu), + | _ => None, + }) + .stream() + .then(|pdu_id| self.services.timeline.get_pdu_json_from_id(pdu_id)) + .ready_filter_map(Result::ok) + .then(|pdu| self.convert_to_outgoing_federation_event(pdu)) + .collect() + .await; - for event in &events { - match event { - // TODO: check room version and remove event_id if needed - | SendingEvent::Pdu(pdu_id) => { - if let Ok(pdu) = self.services.timeline.get_pdu_json_from_id(pdu_id).await { - pdu_jsons.push(self.convert_to_outgoing_federation_event(pdu).await); - } - }, - | SendingEvent::Edu(edu) => - if let Ok(raw) = serde_json::from_slice(edu) { - edu_jsons.push(raw); - }, - | SendingEvent::Flush => {}, // flush only; no new content + let edus: Vec> = events + .iter() + .filter_map(|edu| match edu { + | SendingEvent::Edu(edu) => Some(edu.as_ref()), + | _ => None, + }) + .map(serde_json::from_slice) + .filter_map(Result::ok) + .collect(); + + if pdus.is_empty() && edus.is_empty() { + return Ok(Destination::Federation(server)); + } + + let preimage = pdus + .iter() + .map(|raw| raw.get().as_bytes()) + .chain(edus.iter().map(|raw| raw.json().get().as_bytes())); + + let txn_hash = calculate_hash(preimage); + let txn_id = &*URL_SAFE_NO_PAD.encode(txn_hash); + let request = send_transaction_message::v1::Request { + transaction_id: txn_id.into(), + origin: self.server.name.clone(), + origin_server_ts: MilliSecondsSinceUnixEpoch::now(), + pdus, + edus, + }; + + let result = self + .services + .federation + .execute_on(&self.services.client.sender, &server, request) + .await; + + for (event_id, result) in result.iter().flat_map(|resp| resp.pdus.iter()) { + if let Err(e) = result { + warn!( + %txn_id, %server, + "error sending PDU {event_id} to remote server: {e:?}" + ); } } - //debug_assert!(pdu_jsons.len() + edu_jsons.len() > 0, "sending empty - // transaction"); - - let txn_hash = calculate_hash(events.iter().filter_map(|e| match e { - | SendingEvent::Edu(b) => Some(&**b), - | SendingEvent::Pdu(b) => Some(b.as_ref()), - | SendingEvent::Flush => None, - })); - - let txn_id = &*general_purpose::URL_SAFE_NO_PAD.encode(txn_hash); - - let request = send_transaction_message::v1::Request { - origin: self.server.name.clone(), - pdus: pdu_jsons, - edus: edu_jsons, - origin_server_ts: MilliSecondsSinceUnixEpoch::now(), - transaction_id: txn_id.into(), - }; - - let client = &self.services.client.sender; - self.services.federation.execute_on(client, &server, request) - .await - .inspect(|response| { - response - .pdus - .iter() - .filter(|(_, res)| res.is_err()) - .for_each( - |(pdu_id, res)| warn!(%txn_id, %server, "error sending PDU {pdu_id} to remote server: {res:?}"), - ); - }) - .map_err(|e| (Destination::Federation(server.clone()), e)) - .map(|_| Destination::Federation(server)) + match result { + | Err(error) => Err((Destination::Federation(server), error)), + | Ok(_) => Ok(Destination::Federation(server)), + } } /// This does not return a full `Pdu` it is only to satisfy ruma's types. From 99fe88c21e54f46aedd731786d73d7d9a721dc04 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 21:47:52 +0000 Subject: [PATCH 1442/2091] use smallvec for the edu sending event buffer Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/api/client/to_device.rs | 28 ++++++++++++++----------- src/service/Cargo.toml | 1 + src/service/rooms/typing/mod.rs | 13 ++++++------ src/service/sending/data.rs | 6 +++--- src/service/sending/mod.rs | 19 +++++++++++------ src/service/sending/sender.rs | 36 +++++++++++++++++++++------------ 7 files changed, 64 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a435a10..e379aebb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -845,6 +845,7 @@ dependencies = [ "serde_json", "serde_yaml", "sha2", + "smallvec", "termimad", "tokio", "tracing", diff --git a/src/api/client/to_device.rs b/src/api/client/to_device.rs index 2ded04e7..1b942fba 100644 --- a/src/api/client/to_device.rs +++ b/src/api/client/to_device.rs @@ -10,6 +10,7 @@ use ruma::{ }, to_device::DeviceIdOrAllDevices, }; +use service::sending::EduBuf; use crate::Ruma; @@ -42,18 +43,21 @@ pub(crate) async fn send_event_to_device_route( messages.insert(target_user_id.clone(), map); let count = services.globals.next_count()?; - services.sending.send_edu_server( - target_user_id.server_name(), - serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice( - DirectDeviceContent { - sender: sender_user.clone(), - ev_type: body.event_type.clone(), - message_id: count.to_string().into(), - messages, - }, - )) - .expect("DirectToDevice EDU can be serialized"), - )?; + let mut buf = EduBuf::new(); + serde_json::to_writer( + &mut buf, + &federation::transactions::edu::Edu::DirectToDevice(DirectDeviceContent { + sender: sender_user.clone(), + ev_type: body.event_type.clone(), + message_id: count.to_string().into(), + messages, + }), + ) + .expect("DirectToDevice EDU can be serialized"); + + services + .sending + .send_edu_server(target_user_id.server_name(), buf)?; continue; } diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 21fbb417..c4f75453 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -74,6 +74,7 @@ serde_json.workspace = true serde.workspace = true serde_yaml.workspace = true sha2.workspace = true +smallvec.workspace = true termimad.workspace = true termimad.optional = true tokio.workspace = true diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index a6123322..c710b33a 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -13,7 +13,7 @@ use ruma::{ }; use tokio::sync::{broadcast, RwLock}; -use crate::{globals, sending, users, Dep}; +use crate::{globals, sending, sending::EduBuf, users, Dep}; pub struct Service { server: Arc, @@ -228,12 +228,13 @@ impl Service { return Ok(()); } - let edu = Edu::Typing(TypingContent::new(room_id.to_owned(), user_id.to_owned(), typing)); + let content = TypingContent::new(room_id.to_owned(), user_id.to_owned(), typing); + let edu = Edu::Typing(content); - self.services - .sending - .send_edu_room(room_id, serde_json::to_vec(&edu).expect("Serialized Edu::Typing")) - .await?; + let mut buf = EduBuf::new(); + serde_json::to_writer(&mut buf, &edu).expect("Serialized Edu::Typing"); + + self.services.sending.send_edu_room(room_id, buf).await?; Ok(()) } diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 436f633e..4dd2d5aa 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -202,7 +202,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se if value.is_empty() { SendingEvent::Pdu(event.into()) } else { - SendingEvent::Edu(value.to_vec()) + SendingEvent::Edu(value.into()) }, ) } else if key.starts_with(b"$") { @@ -230,7 +230,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se SendingEvent::Pdu(event.into()) } else { // I'm pretty sure this should never be called - SendingEvent::Edu(value.to_vec()) + SendingEvent::Edu(value.into()) }, ) } else { @@ -252,7 +252,7 @@ fn parse_servercurrentevent(key: &[u8], value: &[u8]) -> Result<(Destination, Se if value.is_empty() { SendingEvent::Pdu(event.into()) } else { - SendingEvent::Edu(value.to_vec()) + SendingEvent::Edu(value.into()) }, ) }) diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 80bca112..b146ad49 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -21,6 +21,7 @@ use ruma::{ api::{appservice::Registration, OutgoingRequest}, RoomId, ServerName, UserId, }; +use smallvec::SmallVec; use tokio::task::JoinSet; use self::data::Data; @@ -67,10 +68,16 @@ struct Msg { #[derive(Clone, Debug, PartialEq, Eq, Hash)] pub enum SendingEvent { Pdu(RawPduId), // pduid - Edu(Vec), // pdu json + Edu(EduBuf), // edu json Flush, // none } +pub type EduBuf = SmallVec<[u8; EDU_BUF_CAP]>; +pub type EduVec = SmallVec<[EduBuf; EDU_VEC_CAP]>; + +const EDU_BUF_CAP: usize = 128; +const EDU_VEC_CAP: usize = 1; + #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -177,7 +184,6 @@ impl Service { where S: Stream + Send + 'a, { - let _cork = self.db.db.cork(); let requests = servers .map(|server| { (Destination::Federation(server.into()), SendingEvent::Pdu(pdu_id.to_owned())) @@ -185,6 +191,7 @@ impl Service { .collect::>() .await; + let _cork = self.db.db.cork(); let keys = self.db.queue_requests(requests.iter().map(|(o, e)| (e, o))); for ((dest, event), queue_id) in requests.into_iter().zip(keys) { @@ -195,7 +202,7 @@ impl Service { } #[tracing::instrument(skip(self, server, serialized), level = "debug")] - pub fn send_edu_server(&self, server: &ServerName, serialized: Vec) -> Result<()> { + pub fn send_edu_server(&self, server: &ServerName, serialized: EduBuf) -> Result { let dest = Destination::Federation(server.to_owned()); let event = SendingEvent::Edu(serialized); let _cork = self.db.db.cork(); @@ -208,7 +215,7 @@ impl Service { } #[tracing::instrument(skip(self, room_id, serialized), level = "debug")] - pub async fn send_edu_room(&self, room_id: &RoomId, serialized: Vec) -> Result<()> { + pub async fn send_edu_room(&self, room_id: &RoomId, serialized: EduBuf) -> Result { let servers = self .services .state_cache @@ -219,11 +226,10 @@ impl Service { } #[tracing::instrument(skip(self, servers, serialized), level = "debug")] - pub async fn send_edu_servers<'a, S>(&self, servers: S, serialized: Vec) -> Result<()> + pub async fn send_edu_servers<'a, S>(&self, servers: S, serialized: EduBuf) -> Result where S: Stream + Send + 'a, { - let _cork = self.db.db.cork(); let requests = servers .map(|server| { ( @@ -234,6 +240,7 @@ impl Service { .collect::>() .await; + let _cork = self.db.db.cork(); let keys = self.db.queue_requests(requests.iter().map(|(o, e)| (e, o))); for ((dest, event), queue_id) in requests.into_iter().zip(keys) { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 47be01f1..363bb994 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -45,7 +45,9 @@ use ruma::{ }; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; -use super::{appservice, data::QueueItem, Destination, Msg, SendingEvent, Service}; +use super::{ + appservice, data::QueueItem, Destination, EduBuf, EduVec, Msg, SendingEvent, Service, +}; #[derive(Debug)] enum TransactionStatus { @@ -313,7 +315,12 @@ impl Service { if let Destination::Federation(server_name) = dest { if let Ok((select_edus, last_count)) = self.select_edus(server_name).await { debug_assert!(select_edus.len() <= EDU_LIMIT, "exceeded edus limit"); - events.extend(select_edus.into_iter().map(SendingEvent::Edu)); + let select_edus = select_edus + .into_iter() + .map(Into::into) + .map(SendingEvent::Edu); + + events.extend(select_edus); self.db.set_latest_educount(server_name, last_count); } } @@ -357,7 +364,7 @@ impl Service { level = "debug", skip_all, )] - async fn select_edus(&self, server_name: &ServerName) -> Result<(Vec>, u64)> { + async fn select_edus(&self, server_name: &ServerName) -> Result<(EduVec, u64)> { // selection window let since = self.db.get_latest_educount(server_name).await; let since_upper = self.services.globals.current_count()?; @@ -405,8 +412,8 @@ impl Service { since: (u64, u64), max_edu_count: &AtomicU64, events_len: &AtomicUsize, - ) -> Vec> { - let mut events = Vec::new(); + ) -> EduVec { + let mut events = EduVec::new(); let server_rooms = self.services.state_cache.server_rooms(server_name); pin_mut!(server_rooms); @@ -441,10 +448,11 @@ impl Service { keys: None, }); - let edu = serde_json::to_vec(&edu) + let mut buf = EduBuf::new(); + serde_json::to_writer(&mut buf, &edu) .expect("failed to serialize device list update to JSON"); - events.push(edu); + events.push(buf); if events_len.fetch_add(1, Ordering::Relaxed) >= SELECT_EDU_LIMIT - 1 { return events; } @@ -465,7 +473,7 @@ impl Service { server_name: &ServerName, since: (u64, u64), max_edu_count: &AtomicU64, - ) -> Option> { + ) -> Option { let server_rooms = self.services.state_cache.server_rooms(server_name); pin_mut!(server_rooms); @@ -487,10 +495,11 @@ impl Service { let receipt_content = Edu::Receipt(ReceiptContent { receipts }); - let receipt_content = serde_json::to_vec(&receipt_content) + let mut buf = EduBuf::new(); + serde_json::to_writer(&mut buf, &receipt_content) .expect("Failed to serialize Receipt EDU to JSON vec"); - Some(receipt_content) + Some(buf) } /// Look for read receipts in this room @@ -569,7 +578,7 @@ impl Service { server_name: &ServerName, since: (u64, u64), max_edu_count: &AtomicU64, - ) -> Option> { + ) -> Option { let presence_since = self.services.presence.presence_since(since.0); pin_mut!(presence_since); @@ -628,10 +637,11 @@ impl Service { push: presence_updates.into_values().collect(), }); - let presence_content = serde_json::to_vec(&presence_content) + let mut buf = EduBuf::new(); + serde_json::to_writer(&mut buf, &presence_content) .expect("failed to serialize Presence EDU to JSON"); - Some(presence_content) + Some(buf) } fn send_events(&self, dest: Destination, events: Vec) -> SendingFuture<'_> { From ed3cd99781f35dd7e38439ab45b78a851385ca8d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 28 Jan 2025 19:42:09 +0000 Subject: [PATCH 1443/2091] abstract the config reload checks Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 14 ++++++++------ src/core/config/check.rs | 18 +++++++++++++++++- src/main/server.rs | 6 +++--- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 47509bad..5c0c2a10 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -32,13 +32,15 @@ pub(super) async fn reload_config( &self, path: Option, ) -> Result { - let path = path.as_deref().into_iter(); - let config = Config::load(path).and_then(|raw| Config::new(&raw))?; - if config.server_name != self.services.server.name { - return Err!("You can't change the server name."); - } + use conduwuit::config::check; - let _old = self.services.server.config.update(config)?; + let path = path.as_deref().into_iter(); + let new = Config::load(path).and_then(|raw| Config::new(&raw))?; + + let old = &self.services.server.config; + check::reload(old, &new)?; + + self.services.server.config.update(new)?; Ok(RoomMessageEventContent::text_plain("Successfully reconfigured.")) } diff --git a/src/core/config/check.rs b/src/core/config/check.rs index d7be54b1..988d4143 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -6,8 +6,24 @@ use figment::Figment; use super::DEPRECATED_KEYS; use crate::{debug, debug_info, debug_warn, error, warn, Config, Err, Result, Server}; +/// Performs check() with additional checks specific to reloading old config +/// with new config. +pub fn reload(old: &Config, new: &Config) -> Result { + check(new)?; + + if new.server_name != old.server_name { + return Err!(Config( + "server_name", + "You can't change the server's name from {:?}.", + old.server_name + )); + } + + Ok(()) +} + #[allow(clippy::cognitive_complexity)] -pub fn check(config: &Config) -> Result<()> { +pub fn check(config: &Config) -> Result { if cfg!(debug_assertions) { warn!("Note: conduwuit was built without optimisations (i.e. debug build)"); } diff --git a/src/main/server.rs b/src/main/server.rs index 74859f2b..7376b2fc 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -46,14 +46,14 @@ impl Server { .and_then(|raw| crate::clap::update(raw, args)) .and_then(|raw| Config::new(&raw))?; - #[cfg(feature = "sentry_telemetry")] - let sentry_guard = crate::sentry::init(&config); - let (tracing_reload_handle, tracing_flame_guard, capture) = crate::logging::init(&config)?; config.check()?; + #[cfg(feature = "sentry_telemetry")] + let sentry_guard = crate::sentry::init(&config); + #[cfg(unix)] sys::maximize_fd_limit() .expect("Unable to increase maximum soft and hard file descriptor limit"); From a567e314e96bb8efa2776770cd25a2b1190c9587 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 28 Jan 2025 20:02:29 +0000 Subject: [PATCH 1444/2091] simplify shutdown signal handlers Signed-off-by: Jason Volk --- src/core/server.rs | 2 +- src/router/run.rs | 27 +++++++-------------------- src/service/sync/watch.rs | 4 ++-- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/src/core/server.rs b/src/core/server.rs index 0f2e61b0..45ba7420 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -112,7 +112,7 @@ impl Server { } #[inline] - pub async fn until_shutdown(self: Arc) { + pub async fn until_shutdown(self: &Arc) { while self.running() { self.signal.subscribe().recv().await.ok(); } diff --git a/src/router/run.rs b/src/router/run.rs index 95d12559..ea8a7666 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -9,6 +9,7 @@ use std::{ use axum_server::Handle as ServerHandle; use conduwuit::{debug, debug_error, debug_info, error, info, Error, Result, Server}; +use futures::FutureExt; use service::Services; use tokio::{ sync::broadcast::{self, Sender}, @@ -109,28 +110,14 @@ pub(crate) async fn stop(services: Arc) -> Result<()> { #[tracing::instrument(skip_all)] async fn signal(server: Arc, tx: Sender<()>, handle: axum_server::Handle) { - loop { - let sig: &'static str = server - .signal - .subscribe() - .recv() - .await - .expect("channel error"); - - if !server.running() { - handle_shutdown(&server, &tx, &handle, sig).await; - break; - } - } + server + .clone() + .until_shutdown() + .then(move |()| handle_shutdown(server, tx, handle)) + .await; } -async fn handle_shutdown( - server: &Arc, - tx: &Sender<()>, - handle: &axum_server::Handle, - sig: &str, -) { - debug!("Received signal {sig}"); +async fn handle_shutdown(server: Arc, tx: Sender<()>, handle: axum_server::Handle) { if let Err(e) = tx.send(()) { error!("failed sending shutdown transaction to channel: {e}"); } diff --git a/src/service/sync/watch.rs b/src/service/sync/watch.rs index 2b351c3a..0a9c5d15 100644 --- a/src/service/sync/watch.rs +++ b/src/service/sync/watch.rs @@ -97,8 +97,8 @@ pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) -> Result { ); // Server shutdown - let server_shutdown = self.services.server.clone().until_shutdown().boxed(); - futures.push(server_shutdown); + futures.push(self.services.server.until_shutdown().boxed()); + if !self.services.server.running() { return Ok(()); } From 2f449ba47db488ca1d3acb6f7228479af6bb97c2 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 28 Jan 2025 20:55:28 +0000 Subject: [PATCH 1445/2091] support reloading config via SIGUSR1 Signed-off-by: Jason Volk --- conduwuit-example.toml | 5 ++++ src/admin/server/commands.rs | 11 ++------ src/core/config/mod.rs | 7 +++++ src/main/signal.rs | 2 ++ src/service/config/mod.rs | 55 ++++++++++++++++++++++++++++++++++++ src/service/mod.rs | 1 + src/service/services.rs | 4 ++- 7 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 src/service/config/mod.rs diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 51d948e8..8534e5c6 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1524,6 +1524,11 @@ # #listening = true +# Enables configuration reload when the server receives SIGUSR1 on +# supporting platforms. +# +#config_reload_signal = true + [global.tls] # Path to a valid TLS certificate file. diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 5c0c2a10..910dce6e 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,6 +1,6 @@ use std::{fmt::Write, path::PathBuf, sync::Arc}; -use conduwuit::{info, utils::time, warn, Config, Err, Result}; +use conduwuit::{info, utils::time, warn, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; use crate::admin_command; @@ -32,15 +32,8 @@ pub(super) async fn reload_config( &self, path: Option, ) -> Result { - use conduwuit::config::check; - let path = path.as_deref().into_iter(); - let new = Config::load(path).and_then(|raw| Config::new(&raw))?; - - let old = &self.services.server.config; - check::reload(old, &new)?; - - self.services.server.config.update(new)?; + self.services.config.reload(path)?; Ok(RoomMessageEventContent::text_plain("Successfully reconfigured.")) } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 94788fa4..8e8176ab 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1742,6 +1742,13 @@ pub struct Config { #[serde(default = "true_fn")] pub listening: bool, + /// Enables configuration reload when the server receives SIGUSR1 on + /// supporting platforms. + /// + /// default: true + #[serde(default = "true_fn")] + pub config_reload_signal: bool, + #[serde(flatten)] #[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime diff --git a/src/main/signal.rs b/src/main/signal.rs index cecb718b..dfdca1d5 100644 --- a/src/main/signal.rs +++ b/src/main/signal.rs @@ -16,6 +16,7 @@ pub(super) async fn signal(server: Arc) { let mut quit = unix::signal(SignalKind::quit()).expect("SIGQUIT handler"); let mut term = unix::signal(SignalKind::terminate()).expect("SIGTERM handler"); + let mut usr1 = unix::signal(SignalKind::user_defined1()).expect("SIGUSR1 handler"); loop { trace!("Installed signal handlers"); let sig: &'static str; @@ -23,6 +24,7 @@ pub(super) async fn signal(server: Arc) { _ = signal::ctrl_c() => { sig = "SIGINT"; }, _ = quit.recv() => { sig = "SIGQUIT"; }, _ = term.recv() => { sig = "SIGTERM"; }, + _ = usr1.recv() => { sig = "SIGUSR1"; }, } warn!("Received {sig}"); diff --git a/src/service/config/mod.rs b/src/service/config/mod.rs new file mode 100644 index 00000000..ef98f176 --- /dev/null +++ b/src/service/config/mod.rs @@ -0,0 +1,55 @@ +use std::{iter, path::Path, sync::Arc}; + +use async_trait::async_trait; +use conduwuit::{ + config::{check, Config}, + error, implement, Result, Server, +}; + +pub struct Service { + server: Arc, +} + +const SIGNAL: &str = "SIGUSR1"; + +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { server: args.server.clone() })) + } + + async fn worker(self: Arc) -> Result { + while self.server.running() { + if self.server.signal.subscribe().recv().await == Ok(SIGNAL) { + if let Err(e) = self.handle_reload() { + error!("Failed to reload config: {e}"); + } + } + } + + Ok(()) + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +#[implement(Service)] +fn handle_reload(&self) -> Result { + if self.server.config.config_reload_signal { + self.reload(iter::empty())?; + } + + Ok(()) +} + +#[implement(Service)] +pub fn reload<'a, I>(&self, paths: I) -> Result> +where + I: Iterator, +{ + let old = self.server.config.clone(); + let new = Config::load(paths).and_then(|raw| Config::new(&raw))?; + + check::reload(&old, &new)?; + self.server.config.update(new) +} diff --git a/src/service/mod.rs b/src/service/mod.rs index 2102921f..71bd0eb4 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -9,6 +9,7 @@ pub mod account_data; pub mod admin; pub mod appservice; pub mod client; +pub mod config; pub mod emergency; pub mod federation; pub mod globals; diff --git a/src/service/services.rs b/src/service/services.rs index cb5cc12f..fb334b96 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -10,7 +10,7 @@ use database::Database; use tokio::sync::Mutex; use crate::{ - account_data, admin, appservice, client, emergency, federation, globals, key_backups, + account_data, admin, appservice, client, config, emergency, federation, globals, key_backups, manager::Manager, media, presence, pusher, resolver, rooms, sending, server_keys, service, service::{Args, Map, Service}, @@ -21,6 +21,7 @@ pub struct Services { pub account_data: Arc, pub admin: Arc, pub appservice: Arc, + pub config: Arc, pub client: Arc, pub emergency: Arc, pub globals: Arc, @@ -68,6 +69,7 @@ impl Services { appservice: build!(appservice::Service), resolver: build!(resolver::Service), client: build!(client::Service), + config: build!(config::Service), emergency: build!(emergency::Service), globals: build!(globals::Service), key_backups: build!(key_backups::Service), From 2c5af902a3b61cf07a07ccfff82a41874d7b10ba Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 28 Jan 2025 21:30:12 +0000 Subject: [PATCH 1446/2091] support executing configurable admin commands via SIGUSR2 Signed-off-by: Jason Volk --- conduwuit-example.toml | 7 +++ src/core/config/mod.rs | 9 ++++ src/main/signal.rs | 2 + src/service/admin/{startup.rs => execute.rs} | 56 ++++++++++++++------ src/service/admin/mod.rs | 8 ++- src/service/config/mod.rs | 9 +++- 6 files changed, 72 insertions(+), 19 deletions(-) rename src/service/admin/{startup.rs => execute.rs} (57%) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 8534e5c6..4062ba99 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1362,6 +1362,13 @@ # #admin_execute_errors_ignore = false +# List of admin commands to execute on SIGUSR2. +# +# Similar to admin_execute, but these commands are executed when the +# server receives SIGUSR2 on supporting platforms. +# +#admin_signal_execute = [] + # Controls the max log level for admin command log captures (logs # generated from running admin commands). Defaults to "info" on release # builds, else "debug" on debug builds. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 8e8176ab..415c9ba9 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1554,6 +1554,15 @@ pub struct Config { #[serde(default)] pub admin_execute_errors_ignore: bool, + /// List of admin commands to execute on SIGUSR2. + /// + /// Similar to admin_execute, but these commands are executed when the + /// server receives SIGUSR2 on supporting platforms. + /// + /// default: [] + #[serde(default)] + pub admin_signal_execute: Vec, + /// Controls the max log level for admin command log captures (logs /// generated from running admin commands). Defaults to "info" on release /// builds, else "debug" on debug builds. diff --git a/src/main/signal.rs b/src/main/signal.rs index dfdca1d5..343b95c9 100644 --- a/src/main/signal.rs +++ b/src/main/signal.rs @@ -17,6 +17,7 @@ pub(super) async fn signal(server: Arc) { let mut quit = unix::signal(SignalKind::quit()).expect("SIGQUIT handler"); let mut term = unix::signal(SignalKind::terminate()).expect("SIGTERM handler"); let mut usr1 = unix::signal(SignalKind::user_defined1()).expect("SIGUSR1 handler"); + let mut usr2 = unix::signal(SignalKind::user_defined2()).expect("SIGUSR2 handler"); loop { trace!("Installed signal handlers"); let sig: &'static str; @@ -25,6 +26,7 @@ pub(super) async fn signal(server: Arc) { _ = quit.recv() => { sig = "SIGQUIT"; }, _ = term.recv() => { sig = "SIGTERM"; }, _ = usr1.recv() => { sig = "SIGUSR1"; }, + _ = usr2.recv() => { sig = "SIGUSR2"; }, } warn!("Received {sig}"); diff --git a/src/service/admin/startup.rs b/src/service/admin/execute.rs similarity index 57% rename from src/service/admin/startup.rs rename to src/service/admin/execute.rs index 582e863d..462681da 100644 --- a/src/service/admin/startup.rs +++ b/src/service/admin/execute.rs @@ -2,6 +2,8 @@ use conduwuit::{debug, debug_info, error, implement, info, Err, Result}; use ruma::events::room::message::RoomMessageEventContent; use tokio::time::{sleep, Duration}; +pub(super) const SIGNAL: &str = "SIGUSR2"; + /// Possibly spawn the terminal console at startup if configured. #[implement(super::Service)] pub(super) async fn console_auto_start(&self) { @@ -22,7 +24,7 @@ pub(super) async fn console_auto_stop(&self) { /// Execute admin commands after startup #[implement(super::Service)] -pub(super) async fn startup_execute(&self) -> Result<()> { +pub(super) async fn startup_execute(&self) -> Result { // List of comamnds to execute let commands = &self.services.server.config.admin_execute; @@ -36,7 +38,7 @@ pub(super) async fn startup_execute(&self) -> Result<()> { sleep(Duration::from_millis(500)).await; for (i, command) in commands.iter().enumerate() { - if let Err(e) = self.startup_execute_command(i, command.clone()).await { + if let Err(e) = self.execute_command(i, command.clone()).await { if !errors { return Err(e); } @@ -59,16 +61,38 @@ pub(super) async fn startup_execute(&self) -> Result<()> { Ok(()) } -/// Execute one admin command after startup +/// Execute admin commands after signal #[implement(super::Service)] -async fn startup_execute_command(&self, i: usize, command: String) -> Result<()> { - debug!("Startup command #{i}: executing {command:?}"); +pub(super) async fn signal_execute(&self) -> Result { + // List of comamnds to execute + let commands = self.services.server.config.admin_signal_execute.clone(); + + // When true, errors are ignored and execution continues. + let ignore_errors = self.services.server.config.admin_execute_errors_ignore; + + for (i, command) in commands.iter().enumerate() { + if let Err(e) = self.execute_command(i, command.clone()).await { + if !ignore_errors { + return Err(e); + } + } + + tokio::task::yield_now().await; + } + + Ok(()) +} + +/// Execute one admin command after startup or signal +#[implement(super::Service)] +async fn execute_command(&self, i: usize, command: String) -> Result { + debug!("Execute command #{i}: executing {command:?}"); match self.command_in_place(command, None).await { - | Ok(Some(output)) => Self::startup_command_output(i, &output), - | Err(output) => Self::startup_command_error(i, &output), + | Ok(Some(output)) => Self::execute_command_output(i, &output), + | Err(output) => Self::execute_command_error(i, &output), | Ok(None) => { - info!("Startup command #{i} completed (no output)."); + info!("Execute command #{i} completed (no output)."); Ok(()) }, } @@ -76,28 +100,28 @@ async fn startup_execute_command(&self, i: usize, command: String) -> Result<()> #[cfg(feature = "console")] #[implement(super::Service)] -fn startup_command_output(i: usize, content: &RoomMessageEventContent) -> Result<()> { - debug_info!("Startup command #{i} completed:"); +fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result { + debug_info!("Execute command #{i} completed:"); super::console::print(content.body()); Ok(()) } #[cfg(feature = "console")] #[implement(super::Service)] -fn startup_command_error(i: usize, content: &RoomMessageEventContent) -> Result<()> { +fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result { super::console::print_err(content.body()); - Err!(debug_error!("Startup command #{i} failed.")) + Err!(debug_error!("Execute command #{i} failed.")) } #[cfg(not(feature = "console"))] #[implement(super::Service)] -fn startup_command_output(i: usize, content: &RoomMessageEventContent) -> Result<()> { - info!("Startup command #{i} completed:\n{:#}", content.body()); +fn execute_command_output(i: usize, content: &RoomMessageEventContent) -> Result { + info!("Execute command #{i} completed:\n{:#}", content.body()); Ok(()) } #[cfg(not(feature = "console"))] #[implement(super::Service)] -fn startup_command_error(i: usize, content: &RoomMessageEventContent) -> Result<()> { - Err!(error!("Startup command #{i} failed:\n{:#}", content.body())) +fn execute_command_error(i: usize, content: &RoomMessageEventContent) -> Result { + Err!(error!("Execute command #{i} failed:\n{:#}", content.body())) } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index bc410631..31b046b7 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -1,7 +1,7 @@ pub mod console; mod create; +mod execute; mod grant; -mod startup; use std::{ future::Future, @@ -183,7 +183,11 @@ impl Service { .map(|complete| complete(command)) } - async fn handle_signal(&self, #[allow(unused_variables)] sig: &'static str) { + async fn handle_signal(&self, sig: &'static str) { + if sig == execute::SIGNAL { + self.signal_execute().await.ok(); + } + #[cfg(feature = "console")] self.console.handle_signal(sig).await; } diff --git a/src/service/config/mod.rs b/src/service/config/mod.rs index ef98f176..8bd09a52 100644 --- a/src/service/config/mod.rs +++ b/src/service/config/mod.rs @@ -1,4 +1,4 @@ -use std::{iter, path::Path, sync::Arc}; +use std::{iter, ops::Deref, path::Path, sync::Arc}; use async_trait::async_trait; use conduwuit::{ @@ -33,6 +33,13 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } +impl Deref for Service { + type Target = Arc; + + #[inline] + fn deref(&self) -> &Self::Target { &self.server.config } +} + #[implement(Service)] fn handle_reload(&self) -> Result { if self.server.config.config_reload_signal { From ad0b0af955cda8b93b6d8c9c665905a2c4dd93d3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 25 Jan 2025 23:07:50 +0000 Subject: [PATCH 1447/2091] combine state_accessor data into mod Signed-off-by: Jason Volk --- src/service/rooms/state_accessor/data.rs | 253 ----------------------- src/service/rooms/state_accessor/mod.rs | 183 +++++++++++++--- 2 files changed, 149 insertions(+), 287 deletions(-) delete mode 100644 src/service/rooms/state_accessor/data.rs diff --git a/src/service/rooms/state_accessor/data.rs b/src/service/rooms/state_accessor/data.rs deleted file mode 100644 index 29b27a05..00000000 --- a/src/service/rooms/state_accessor/data.rs +++ /dev/null @@ -1,253 +0,0 @@ -use std::{borrow::Borrow, collections::HashMap, sync::Arc}; - -use conduwuit::{ - at, err, - utils::stream::{BroadbandExt, IterStream, ReadyExt}, - PduEvent, Result, -}; -use database::{Deserialized, Map}; -use futures::{FutureExt, StreamExt, TryFutureExt}; -use ruma::{events::StateEventType, EventId, OwnedEventId, RoomId}; -use serde::Deserialize; - -use crate::{ - rooms, - rooms::{ - short::{ShortEventId, ShortStateHash, ShortStateKey}, - state_compressor::parse_compressed_state_event, - }, - Dep, -}; - -pub(super) struct Data { - shorteventid_shortstatehash: Arc, - services: Services, -} - -struct Services { - short: Dep, - state: Dep, - state_compressor: Dep, - timeline: Dep, -} - -impl Data { - pub(super) fn new(args: &crate::Args<'_>) -> Self { - let db = &args.db; - Self { - shorteventid_shortstatehash: db["shorteventid_shortstatehash"].clone(), - services: Services { - short: args.depend::("rooms::short"), - state: args.depend::("rooms::state"), - state_compressor: args - .depend::("rooms::state_compressor"), - timeline: args.depend::("rooms::timeline"), - }, - } - } - - pub(super) async fn state_full( - &self, - shortstatehash: ShortStateHash, - ) -> Result> { - let state = self - .state_full_pdus(shortstatehash) - .await? - .into_iter() - .filter_map(|pdu| Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu))) - .collect(); - - Ok(state) - } - - pub(super) async fn state_full_pdus( - &self, - shortstatehash: ShortStateHash, - ) -> Result> { - let short_ids = self.state_full_shortids(shortstatehash).await?; - - let full_pdus = self - .services - .short - .multi_get_eventid_from_short(short_ids.into_iter().map(at!(1)).stream()) - .ready_filter_map(Result::ok) - .broad_filter_map(|event_id: OwnedEventId| async move { - self.services.timeline.get_pdu(&event_id).await.ok() - }) - .collect() - .await; - - Ok(full_pdus) - } - - pub(super) async fn state_full_ids( - &self, - shortstatehash: ShortStateHash, - ) -> Result> - where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, - ::Owned: Borrow, - { - let short_ids = self.state_full_shortids(shortstatehash).await?; - - let full_ids = self - .services - .short - .multi_get_eventid_from_short(short_ids.iter().map(at!(1)).stream()) - .zip(short_ids.iter().stream().map(at!(0))) - .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) - .collect() - .boxed() - .await; - - Ok(full_ids) - } - - pub(super) async fn state_full_shortids( - &self, - shortstatehash: ShortStateHash, - ) -> Result> { - let shortids = self - .services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .await - .map_err(|e| err!(Database("Missing state IDs: {e}")))? - .pop() - .expect("there is always one layer") - .full_state - .iter() - .copied() - .map(parse_compressed_state_event) - .collect(); - - Ok(shortids) - } - - /// Returns a single EventId from `room_id` with key - /// (`event_type`,`state_key`). - pub(super) async fn state_get_id( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - Id: for<'de> Deserialize<'de> + Sized + ToOwned, - ::Owned: Borrow, - { - let shortstatekey = self - .services - .short - .get_shortstatekey(event_type, state_key) - .await?; - - let full_state = self - .services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .await - .map_err(|e| err!(Database(error!(?event_type, ?state_key, "Missing state: {e:?}"))))? - .pop() - .expect("there is always one layer") - .full_state; - - let compressed = full_state - .iter() - .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) - .ok_or(err!(Database("No shortstatekey in compressed state")))?; - - let (_, shorteventid) = parse_compressed_state_event(*compressed); - - self.services - .short - .get_eventid_from_short(shorteventid) - .await - } - - /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). - pub(super) async fn state_get( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result { - self.state_get_id(shortstatehash, event_type, state_key) - .and_then(|event_id: OwnedEventId| async move { - self.services.timeline.get_pdu(&event_id).await - }) - .await - } - - /// Returns the state hash for this pdu. - pub(super) async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { - const BUFSIZE: usize = size_of::(); - - self.services - .short - .get_shorteventid(event_id) - .and_then(|shorteventid| { - self.shorteventid_shortstatehash - .aqry::(&shorteventid) - }) - .await - .deserialized() - } - - /// Returns the full room state. - pub(super) async fn room_state_full( - &self, - room_id: &RoomId, - ) -> Result> { - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_full(shortstatehash)) - .map_err(|e| err!(Database("Missing state for {room_id:?}: {e:?}"))) - .await - } - - /// Returns the full room state's pdus. - #[allow(unused_qualifications)] // async traits - pub(super) async fn room_state_full_pdus(&self, room_id: &RoomId) -> Result> { - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_full_pdus(shortstatehash)) - .map_err(|e| err!(Database("Missing state pdus for {room_id:?}: {e:?}"))) - .await - } - - /// Returns a single EventId from `room_id` with key - /// (`event_type`,`state_key`). - pub(super) async fn room_state_get_id( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - Id: for<'de> Deserialize<'de> + Sized + ToOwned, - ::Owned: Borrow, - { - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_get_id(shortstatehash, event_type, state_key)) - .await - } - - /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). - pub(super) async fn room_state_get( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, - ) -> Result { - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_get(shortstatehash, event_type, state_key)) - .await - } -} diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index d89c8835..3d87534b 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -1,5 +1,3 @@ -mod data; - use std::{ borrow::Borrow, collections::HashMap, @@ -8,16 +6,18 @@ use std::{ }; use conduwuit::{ - err, error, + at, err, error, pdu::PduBuilder, utils, utils::{ math::{usize_from_f64, Expected}, - ReadyExt, + stream::BroadbandExt, + IterStream, ReadyExt, }, Err, Error, PduEvent, Result, }; -use futures::StreamExt; +use database::{Deserialized, Map}; +use futures::{FutureExt, StreamExt, TryFutureExt}; use lru_cache::LruCache; use ruma::{ events::{ @@ -38,33 +38,40 @@ use ruma::{ }, room::RoomType, space::SpaceRoomJoinRule, - EventEncryptionAlgorithm, EventId, JsOption, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, - OwnedUserId, RoomId, ServerName, UserId, + EventEncryptionAlgorithm, EventId, JsOption, OwnedEventId, OwnedRoomAliasId, OwnedRoomId, + OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, }; use serde::Deserialize; -use self::data::Data; use crate::{ rooms, rooms::{ short::{ShortEventId, ShortStateHash, ShortStateKey}, state::RoomMutexGuard, + state_compressor::parse_compressed_state_event, }, Dep, }; pub struct Service { - services: Services, - db: Data, pub server_visibility_cache: Mutex>, pub user_visibility_cache: Mutex>, + services: Services, + db: Data, } struct Services { + short: Dep, + state: Dep, + state_compressor: Dep, state_cache: Dep, timeline: Dep, } +struct Data { + shorteventid_shortstatehash: Arc, +} + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; @@ -74,17 +81,23 @@ impl crate::Service for Service { f64::from(config.user_visibility_cache_capacity) * config.cache_capacity_modifier; Ok(Arc::new(Self { - services: Services { - state_cache: args.depend::("rooms::state_cache"), - timeline: args.depend::("rooms::timeline"), - }, - db: Data::new(&args), server_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64( server_visibility_cache_capacity, )?)), user_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64( user_visibility_cache_capacity, )?)), + services: Services { + state_cache: args.depend::("rooms::state_cache"), + timeline: args.depend::("rooms::timeline"), + short: args.depend::("rooms::short"), + state: args.depend::("rooms::state"), + state_compressor: args + .depend::("rooms::state_compressor"), + }, + db: Data { + shorteventid_shortstatehash: args.db["shorteventid_shortstatehash"].clone(), + }, })) } @@ -130,6 +143,37 @@ impl crate::Service for Service { } impl Service { + pub async fn state_full( + &self, + shortstatehash: ShortStateHash, + ) -> Result> { + let state = self + .state_full_pdus(shortstatehash) + .await? + .into_iter() + .filter_map(|pdu| Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu))) + .collect(); + + Ok(state) + } + + pub async fn state_full_pdus(&self, shortstatehash: ShortStateHash) -> Result> { + let short_ids = self.state_full_shortids(shortstatehash).await?; + + let full_pdus = self + .services + .short + .multi_get_eventid_from_short(short_ids.into_iter().map(at!(1)).stream()) + .ready_filter_map(Result::ok) + .broad_filter_map(|event_id: OwnedEventId| async move { + self.services.timeline.get_pdu(&event_id).await.ok() + }) + .collect() + .await; + + Ok(full_pdus) + } + /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[tracing::instrument(skip(self), level = "debug")] @@ -141,7 +185,19 @@ impl Service { Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, ::Owned: Borrow, { - self.db.state_full_ids::(shortstatehash).await + let short_ids = self.state_full_shortids(shortstatehash).await?; + + let full_ids = self + .services + .short + .multi_get_eventid_from_short(short_ids.iter().map(at!(1)).stream()) + .zip(short_ids.iter().stream().map(at!(0))) + .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) + .collect() + .boxed() + .await; + + Ok(full_ids) } #[inline] @@ -149,14 +205,21 @@ impl Service { &self, shortstatehash: ShortStateHash, ) -> Result> { - self.db.state_full_shortids(shortstatehash).await - } + let shortids = self + .services + .state_compressor + .load_shortstatehash_info(shortstatehash) + .await + .map_err(|e| err!(Database("Missing state IDs: {e}")))? + .pop() + .expect("there is always one layer") + .full_state + .iter() + .copied() + .map(parse_compressed_state_event) + .collect(); - pub async fn state_full( - &self, - shortstatehash: ShortStateHash, - ) -> Result> { - self.db.state_full(shortstatehash).await + Ok(shortids) } /// Returns a single EventId from `room_id` with key (`event_type`, @@ -172,22 +235,47 @@ impl Service { Id: for<'de> Deserialize<'de> + Sized + ToOwned, ::Owned: Borrow, { - self.db - .state_get_id(shortstatehash, event_type, state_key) + let shortstatekey = self + .services + .short + .get_shortstatekey(event_type, state_key) + .await?; + + let full_state = self + .services + .state_compressor + .load_shortstatehash_info(shortstatehash) + .await + .map_err(|e| err!(Database(error!(?event_type, ?state_key, "Missing state: {e:?}"))))? + .pop() + .expect("there is always one layer") + .full_state; + + let compressed = full_state + .iter() + .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) + .ok_or(err!(Database("No shortstatekey in compressed state")))?; + + let (_, shorteventid) = parse_compressed_state_event(*compressed); + + self.services + .short + .get_eventid_from_short(shorteventid) .await } /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). - #[inline] pub async fn state_get( &self, shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, ) -> Result { - self.db - .state_get(shortstatehash, event_type, state_key) + self.state_get_id(shortstatehash, event_type, state_key) + .and_then(|event_id: OwnedEventId| async move { + self.services.timeline.get_pdu(&event_id).await + }) .await } @@ -375,7 +463,18 @@ impl Service { /// Returns the state hash for this pdu. pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { - self.db.pdu_shortstatehash(event_id).await + const BUFSIZE: usize = size_of::(); + + self.services + .short + .get_shorteventid(event_id) + .and_then(|shorteventid| { + self.db + .shorteventid_shortstatehash + .aqry::(&shorteventid) + }) + .await + .deserialized() } /// Returns the full room state. @@ -384,13 +483,23 @@ impl Service { &self, room_id: &RoomId, ) -> Result> { - self.db.room_state_full(room_id).await + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_full(shortstatehash)) + .map_err(|e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .await } /// Returns the full room state pdus #[tracing::instrument(skip(self), level = "debug")] pub async fn room_state_full_pdus(&self, room_id: &RoomId) -> Result> { - self.db.room_state_full_pdus(room_id).await + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_full_pdus(shortstatehash)) + .map_err(|e| err!(Database("Missing state pdus for {room_id:?}: {e:?}"))) + .await } /// Returns a single EventId from `room_id` with key (`event_type`, @@ -406,8 +515,10 @@ impl Service { Id: for<'de> Deserialize<'de> + Sized + ToOwned, ::Owned: Borrow, { - self.db - .room_state_get_id(room_id, event_type, state_key) + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get_id(shortstatehash, event_type, state_key)) .await } @@ -420,7 +531,11 @@ impl Service { event_type: &StateEventType, state_key: &str, ) -> Result { - self.db.room_state_get(room_id, event_type, state_key).await + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get(shortstatehash, event_type, state_key)) + .await } /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). From af399fd5179eed9c72bf0426858301af9ffc92d4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 29 Jan 2025 01:04:02 +0000 Subject: [PATCH 1448/2091] flatten state accessor iterations Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 11 +- src/api/client/context.rs | 28 ++-- src/api/client/membership.rs | 18 +- src/api/client/message.rs | 6 +- src/api/client/room/initial_sync.rs | 9 +- src/api/client/search.rs | 12 +- src/api/client/state.rs | 12 +- src/api/client/sync/v3.rs | 48 +++--- src/api/client/sync/v4.rs | 8 +- src/api/client/sync/v5.rs | 8 +- src/api/server/send_join.rs | 14 +- src/api/server/state.rs | 10 +- src/api/server/state_ids.rs | 9 +- src/core/pdu/strip.rs | 12 +- .../rooms/event_handler/resolve_state.rs | 5 +- .../rooms/event_handler/state_at_incoming.rs | 17 +- src/service/rooms/spaces/mod.rs | 4 +- src/service/rooms/state_accessor/mod.rs | 155 ++++++++++-------- 18 files changed, 205 insertions(+), 181 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index cdd69c0f..cd892ded 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -9,7 +9,7 @@ use conduwuit::{ debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, PduId, RawPduId, Result, }; -use futures::{FutureExt, StreamExt}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ api::{client::error::ErrorKind, federation::event::get_room_state}, events::room::message::RoomMessageEventContent, @@ -327,11 +327,10 @@ pub(super) async fn get_room_state( .services .rooms .state_accessor - .room_state_full(&room_id) - .await? - .values() - .map(PduEvent::to_state_event) - .collect(); + .room_state_full_pdus(&room_id) + .map_ok(PduEvent::into_state_event) + .try_collect() + .await?; if room_state.is_empty() { return Ok(RoomMessageEventContent::text_plain( diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 388bcf4d..7256683f 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,6 +1,6 @@ use axum::extract::State; use conduwuit::{ - at, deref_at, err, ref_at, + at, err, ref_at, utils::{ future::TryExtExt, stream::{BroadbandExt, ReadyExt, TryIgnore, WidebandExt}, @@ -10,10 +10,10 @@ use conduwuit::{ }; use futures::{ future::{join, join3, try_join3, OptionFuture}, - FutureExt, StreamExt, TryFutureExt, + FutureExt, StreamExt, TryFutureExt, TryStreamExt, }; use ruma::{api::client::context::get_context, events::StateEventType, OwnedEventId, UserId}; -use service::rooms::{lazy_loading, lazy_loading::Options}; +use service::rooms::{lazy_loading, lazy_loading::Options, short::ShortStateKey}; use crate::{ client::message::{event_filter, ignored_filter, lazy_loading_witness, visibility_filter}, @@ -132,21 +132,29 @@ pub(crate) async fn get_context_route( .state_accessor .pdu_shortstatehash(state_at) .or_else(|_| services.rooms.state.get_room_shortstatehash(room_id)) - .and_then(|shortstatehash| services.rooms.state_accessor.state_full_ids(shortstatehash)) + .map_ok(|shortstatehash| { + services + .rooms + .state_accessor + .state_full_ids(shortstatehash) + .map(Ok) + }) .map_err(|e| err!(Database("State not found: {e}"))) + .try_flatten_stream() + .try_collect() .boxed(); let (lazy_loading_witnessed, state_ids) = join(lazy_loading_witnessed, state_ids).await; - let state_ids = state_ids?; + let state_ids: Vec<(ShortStateKey, OwnedEventId)> = state_ids?; + let shortstatekeys = state_ids.iter().map(at!(0)).stream(); + let shorteventids = state_ids.iter().map(ref_at!(1)).stream(); let lazy_loading_witnessed = lazy_loading_witnessed.unwrap_or_default(); - let shortstatekeys = state_ids.iter().stream().map(deref_at!(0)); - let state: Vec<_> = services .rooms .short .multi_get_statekey_from_short(shortstatekeys) - .zip(state_ids.iter().stream().map(at!(1))) + .zip(shorteventids) .ready_filter_map(|item| Some((item.0.ok()?, item.1))) .ready_filter_map(|((event_type, state_key), event_id)| { if filter.lazy_load_options.is_enabled() @@ -162,9 +170,9 @@ pub(crate) async fn get_context_route( Some(event_id) }) .broad_filter_map(|event_id: &OwnedEventId| { - services.rooms.timeline.get_pdu(event_id).ok() + services.rooms.timeline.get_pdu(event_id.as_ref()).ok() }) - .map(|pdu| pdu.to_state_event()) + .map(PduEvent::into_state_event) .collect() .await; diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 2e23dab9..fccb9b53 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -8,14 +8,14 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - debug, debug_info, debug_warn, err, info, + at, debug, debug_info, debug_warn, err, info, pdu::{gen_event_id_canonical_json, PduBuilder}, result::FlatOk, trace, utils::{self, shuffle, IterStream, ReadyExt}, warn, Err, PduEvent, Result, }; -use futures::{join, FutureExt, StreamExt}; +use futures::{join, FutureExt, StreamExt, TryFutureExt}; use ruma::{ api::{ client::{ @@ -765,11 +765,12 @@ pub(crate) async fn get_member_events_route( .rooms .state_accessor .room_state_full(&body.room_id) - .await? - .iter() - .filter(|(key, _)| key.0 == StateEventType::RoomMember) - .map(|(_, pdu)| pdu.to_member_event()) - .collect(), + .ready_filter_map(Result::ok) + .ready_filter(|((ty, _), _)| *ty == StateEventType::RoomMember) + .map(at!(1)) + .map(PduEvent::into_member_event) + .collect() + .await, }) } @@ -1707,9 +1708,6 @@ pub async fn leave_room( room_id: &RoomId, reason: Option, ) -> Result<()> { - //use conduwuit::utils::stream::OptionStream; - use futures::TryFutureExt; - // Ask a remote server if we don't have this room and are not knocking on it if !services .rooms diff --git a/src/api/client/message.rs b/src/api/client/message.rs index a508b5da..321d8013 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -6,9 +6,9 @@ use conduwuit::{ stream::{BroadbandExt, TryIgnore, WidebandExt}, IterStream, ReadyExt, }, - Event, PduCount, Result, + Event, PduCount, PduEvent, Result, }; -use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt}; +use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt, TryFutureExt}; use ruma::{ api::{ client::{filter::RoomEventFilter, message::get_message_events}, @@ -220,8 +220,8 @@ async fn get_member_event( .rooms .state_accessor .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str()) + .map_ok(PduEvent::into_state_event) .await - .map(|member_event| member_event.to_state_event()) .ok() } diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index 301b6e8d..233d180f 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -2,7 +2,7 @@ use axum::extract::State; use conduwuit::{ at, utils::{stream::TryTools, BoolExt}, - Err, Result, + Err, PduEvent, Result, }; use futures::TryStreamExt; use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response}; @@ -39,10 +39,9 @@ pub(crate) async fn room_initial_sync_route( .rooms .state_accessor .room_state_full_pdus(room_id) - .await? - .into_iter() - .map(|pdu| pdu.to_state_event()) - .collect(); + .map_ok(PduEvent::into_state_event) + .try_collect() + .await?; let messages = PaginationChunk { start: events.last().map(at!(0)).as_ref().map(ToString::to_string), diff --git a/src/api/client/search.rs b/src/api/client/search.rs index e60bd26d..898dfc7f 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -7,7 +7,7 @@ use conduwuit::{ utils::{stream::ReadyExt, IterStream}, Err, PduEvent, Result, }; -use futures::{future::OptionFuture, FutureExt, StreamExt, TryFutureExt}; +use futures::{future::OptionFuture, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{ api::client::search::search_events::{ self, @@ -181,15 +181,15 @@ async fn category_room_events( } async fn procure_room_state(services: &Services, room_id: &RoomId) -> Result { - let state_map = services + let state = services .rooms .state_accessor - .room_state_full(room_id) + .room_state_full_pdus(room_id) + .map_ok(PduEvent::into_state_event) + .try_collect() .await?; - let state_events = state_map.values().map(PduEvent::to_state_event).collect(); - - Ok(state_events) + Ok(state) } async fn check_room_visible( diff --git a/src/api/client/state.rs b/src/api/client/state.rs index d00ee5e5..8555f88b 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,5 +1,6 @@ use axum::extract::State; use conduwuit::{err, pdu::PduBuilder, utils::BoolExt, Err, PduEvent, Result}; +use futures::TryStreamExt; use ruma::{ api::client::state::{get_state_events, get_state_events_for_key, send_state_event}, events::{ @@ -82,11 +83,10 @@ pub(crate) async fn get_state_events_route( room_state: services .rooms .state_accessor - .room_state_full(&body.room_id) - .await? - .values() - .map(PduEvent::to_state_event) - .collect(), + .room_state_full_pdus(&body.room_id) + .map_ok(PduEvent::into_state_event) + .try_collect() + .await?, }) } @@ -133,7 +133,7 @@ pub(crate) async fn get_state_events_for_key_route( Ok(get_state_events_for_key::v3::Response { content: event_format.or(|| event.get_content_as_value()), - event: event_format.then(|| event.to_state_event_value()), + event: event_format.then(|| event.into_state_event_value()), }) } diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 7cca9616..cd4dfc90 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -28,7 +28,7 @@ use conduwuit_service::{ }; use futures::{ future::{join, join3, join4, join5, try_join, try_join4, OptionFuture}, - FutureExt, StreamExt, TryFutureExt, + FutureExt, StreamExt, TryFutureExt, TryStreamExt, }; use ruma::{ api::client::{ @@ -503,16 +503,20 @@ async fn handle_left_room( let mut left_state_events = Vec::new(); - let since_shortstatehash = services - .rooms - .user - .get_token_shortstatehash(room_id, since) - .await; + let since_shortstatehash = services.rooms.user.get_token_shortstatehash(room_id, since); - let since_state_ids = match since_shortstatehash { - | Ok(s) => services.rooms.state_accessor.state_full_ids(s).await?, - | Err(_) => HashMap::new(), - }; + let since_state_ids: HashMap<_, OwnedEventId> = since_shortstatehash + .map_ok(|since_shortstatehash| { + services + .rooms + .state_accessor + .state_full_ids(since_shortstatehash) + .map(Ok) + }) + .try_flatten_stream() + .try_collect() + .await + .unwrap_or_default(); let Ok(left_event_id): Result = services .rooms @@ -534,11 +538,12 @@ async fn handle_left_room( return Ok(None); }; - let mut left_state_ids = services + let mut left_state_ids: HashMap<_, _> = services .rooms .state_accessor .state_full_ids(left_shortstatehash) - .await?; + .collect() + .await; let leave_shortstatekey = services .rooms @@ -960,19 +965,18 @@ async fn calculate_state_initial( current_shortstatehash: ShortStateHash, witness: Option<&Witness>, ) -> Result { - let state_events = services + let (shortstatekeys, event_ids): (Vec<_>, Vec<_>) = services .rooms .state_accessor .state_full_ids(current_shortstatehash) - .await?; - - let shortstatekeys = state_events.keys().copied().stream(); + .unzip() + .await; let state_events = services .rooms .short - .multi_get_statekey_from_short(shortstatekeys) - .zip(state_events.values().cloned().stream()) + .multi_get_statekey_from_short(shortstatekeys.into_iter().stream()) + .zip(event_ids.into_iter().stream()) .ready_filter_map(|item| Some((item.0.ok()?, item.1))) .ready_filter_map(|((event_type, state_key), event_id)| { let lazy_load_enabled = filter.room.state.lazy_load_options.is_enabled() @@ -1036,17 +1040,19 @@ async fn calculate_state_incremental( let current_state_ids = services .rooms .state_accessor - .state_full_ids(current_shortstatehash); + .state_full_ids(current_shortstatehash) + .collect(); let since_state_ids = services .rooms .state_accessor - .state_full_ids(since_shortstatehash); + .state_full_ids(since_shortstatehash) + .collect(); let (current_state_ids, since_state_ids): ( HashMap<_, OwnedEventId>, HashMap<_, OwnedEventId>, - ) = try_join(current_state_ids, since_state_ids).await?; + ) = join(current_state_ids, since_state_ids).await; current_state_ids .iter() diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index a82e9309..b7967498 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -241,13 +241,15 @@ pub(crate) async fn sync_events_v4_route( .rooms .state_accessor .state_full_ids(current_shortstatehash) - .await?; + .collect() + .await; - let since_state_ids = services + let since_state_ids: HashMap<_, _> = services .rooms .state_accessor .state_full_ids(since_shortstatehash) - .await?; + .collect() + .await; for (key, id) in current_state_ids { if since_state_ids.get(&key) != Some(&id) { diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index 1c4f3504..66647f0e 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -748,13 +748,15 @@ async fn collect_e2ee<'a>( .rooms .state_accessor .state_full_ids(current_shortstatehash) - .await?; + .collect() + .await; - let since_state_ids = services + let since_state_ids: HashMap<_, _> = services .rooms .state_accessor .state_full_ids(since_shortstatehash) - .await?; + .collect() + .await; for (key, id) in current_state_ids { if since_state_ids.get(&key) != Some(&id) { diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index e62089b4..2b8a0eef 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -1,10 +1,10 @@ #![allow(deprecated)] -use std::{borrow::Borrow, collections::HashMap}; +use std::borrow::Borrow; use axum::extract::State; use conduwuit::{ - err, + at, err, pdu::gen_event_id_canonical_json, utils::stream::{IterStream, TryBroadbandExt}, warn, Err, Result, @@ -211,14 +211,16 @@ async fn create_join_event( drop(mutex_lock); - let state_ids: HashMap<_, OwnedEventId> = services + let state_ids: Vec = services .rooms .state_accessor .state_full_ids(shortstatehash) - .await?; + .map(at!(1)) + .collect() + .await; let state = state_ids - .values() + .iter() .try_stream() .broad_and_then(|event_id| services.rooms.timeline.get_pdu_json(event_id)) .broad_and_then(|pdu| { @@ -231,7 +233,7 @@ async fn create_join_event( .boxed() .await?; - let starting_events = state_ids.values().map(Borrow::borrow); + let starting_events = state_ids.iter().map(Borrow::borrow); let auth_chain = services .rooms .auth_chain diff --git a/src/api/server/state.rs b/src/api/server/state.rs index 42f7e538..eab1f138 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, iter::once}; use axum::extract::State; -use conduwuit::{err, result::LogErr, utils::IterStream, Result}; +use conduwuit::{at, err, utils::IterStream, Result}; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{api::federation::event::get_room_state, OwnedEventId}; @@ -35,11 +35,9 @@ pub(crate) async fn get_room_state_route( .rooms .state_accessor .state_full_ids(shortstatehash) - .await - .log_err() - .map_err(|_| err!(Request(NotFound("PDU state IDs not found."))))? - .into_values() - .collect(); + .map(at!(1)) + .collect() + .await; let pdus = state_ids .iter() diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index 186ef399..4973dd3a 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, iter::once}; use axum::extract::State; -use conduwuit::{err, Result}; +use conduwuit::{at, err, Result}; use futures::StreamExt; use ruma::{api::federation::event::get_room_state_ids, OwnedEventId}; @@ -36,10 +36,9 @@ pub(crate) async fn get_room_state_ids_route( .rooms .state_accessor .state_full_ids(shortstatehash) - .await - .map_err(|_| err!(Request(NotFound("State ids not found"))))? - .into_values() - .collect(); + .map(at!(1)) + .collect() + .await; let auth_chain_ids = services .rooms diff --git a/src/core/pdu/strip.rs b/src/core/pdu/strip.rs index 8e1045db..7d2fb1d6 100644 --- a/src/core/pdu/strip.rs +++ b/src/core/pdu/strip.rs @@ -116,7 +116,7 @@ pub fn to_message_like_event(&self) -> Raw { #[must_use] #[implement(super::Pdu)] -pub fn to_state_event_value(&self) -> JsonValue { +pub fn into_state_event_value(self) -> JsonValue { let mut json = json!({ "content": self.content, "type": self.kind, @@ -127,7 +127,7 @@ pub fn to_state_event_value(&self) -> JsonValue { "state_key": self.state_key, }); - if let Some(unsigned) = &self.unsigned { + if let Some(unsigned) = self.unsigned { json["unsigned"] = json!(unsigned); } @@ -136,8 +136,8 @@ pub fn to_state_event_value(&self) -> JsonValue { #[must_use] #[implement(super::Pdu)] -pub fn to_state_event(&self) -> Raw { - serde_json::from_value(self.to_state_event_value()).expect("Raw::from_value always works") +pub fn into_state_event(self) -> Raw { + serde_json::from_value(self.into_state_event_value()).expect("Raw::from_value always works") } #[must_use] @@ -188,7 +188,7 @@ pub fn to_stripped_spacechild_state_event(&self) -> Raw Raw> { +pub fn into_member_event(self) -> Raw> { let mut json = json!({ "content": self.content, "type": self.kind, @@ -200,7 +200,7 @@ pub fn to_member_event(&self) -> Raw> { "state_key": self.state_key, }); - if let Some(unsigned) = &self.unsigned { + if let Some(unsigned) = self.unsigned { json["unsigned"] = json!(unsigned); } diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 0526d31c..1fd91ac6 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -33,11 +33,12 @@ pub async fn resolve_state( .await .map_err(|e| err!(Database(error!("No state for {room_id:?}: {e:?}"))))?; - let current_state_ids = self + let current_state_ids: HashMap<_, _> = self .services .state_accessor .state_full_ids(current_sstatehash) - .await?; + .collect() + .await; let fork_states = [current_state_ids, incoming_state]; let auth_chain_sets: Vec> = fork_states diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 9e7f8d2a..7ef047ab 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -31,15 +31,12 @@ pub(super) async fn state_at_incoming_degree_one( return Ok(None); }; - let Ok(mut state) = self + let mut state: HashMap<_, _> = self .services .state_accessor .state_full_ids(prev_event_sstatehash) - .await - .log_err() - else { - return Ok(None); - }; + .collect() + .await; debug!("Using cached state"); let prev_pdu = self @@ -103,14 +100,12 @@ pub(super) async fn state_at_incoming_resolved( let mut fork_states = Vec::with_capacity(extremity_sstatehashes.len()); let mut auth_chain_sets = Vec::with_capacity(extremity_sstatehashes.len()); for (sstatehash, prev_event) in extremity_sstatehashes { - let Ok(mut leaf_state) = self + let mut leaf_state: HashMap<_, _> = self .services .state_accessor .state_full_ids(sstatehash) - .await - else { - continue; - }; + .collect() + .await; if let Some(state_key) = &prev_event.state_key { let shortstatekey = self diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index d60c4c9e..d12a01ab 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -624,8 +624,8 @@ impl Service { .services .state_accessor .state_full_ids(current_shortstatehash) - .await - .map_err(|e| err!(Database("State in space not found: {e}")))?; + .collect() + .await; let mut children_pdus = Vec::with_capacity(state.len()); for (key, id) in state { diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 3d87534b..0f5520bb 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -1,6 +1,5 @@ use std::{ borrow::Borrow, - collections::HashMap, fmt::Write, sync::{Arc, Mutex as StdMutex, Mutex}, }; @@ -17,7 +16,7 @@ use conduwuit::{ Err, Error, PduEvent, Result, }; use database::{Deserialized, Map}; -use futures::{FutureExt, StreamExt, TryFutureExt}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt}; use lru_cache::LruCache; use ruma::{ events::{ @@ -143,83 +142,74 @@ impl crate::Service for Service { } impl Service { - pub async fn state_full( + pub fn state_full( &self, shortstatehash: ShortStateHash, - ) -> Result> { - let state = self - .state_full_pdus(shortstatehash) - .await? - .into_iter() - .filter_map(|pdu| Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu))) - .collect(); - - Ok(state) + ) -> impl Stream + Send + '_ { + self.state_full_pdus(shortstatehash) + .ready_filter_map(|pdu| { + Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu)) + }) } - pub async fn state_full_pdus(&self, shortstatehash: ShortStateHash) -> Result> { - let short_ids = self.state_full_shortids(shortstatehash).await?; + pub fn state_full_pdus( + &self, + shortstatehash: ShortStateHash, + ) -> impl Stream + Send + '_ { + let short_ids = self + .state_full_shortids(shortstatehash) + .map(|result| result.expect("missing shortstatehash")) + .map(Vec::into_iter) + .map(|iter| iter.map(at!(1))) + .map(IterStream::stream) + .flatten_stream() + .boxed(); - let full_pdus = self - .services + self.services .short - .multi_get_eventid_from_short(short_ids.into_iter().map(at!(1)).stream()) + .multi_get_eventid_from_short(short_ids) .ready_filter_map(Result::ok) - .broad_filter_map(|event_id: OwnedEventId| async move { + .broad_filter_map(move |event_id: OwnedEventId| async move { self.services.timeline.get_pdu(&event_id).await.ok() }) - .collect() - .await; - - Ok(full_pdus) } /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_full_ids( - &self, + pub fn state_full_ids<'a, Id>( + &'a self, shortstatehash: ShortStateHash, - ) -> Result> + ) -> impl Stream + Send + 'a where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned, + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a, ::Owned: Borrow, { - let short_ids = self.state_full_shortids(shortstatehash).await?; - - let full_ids = self - .services - .short - .multi_get_eventid_from_short(short_ids.iter().map(at!(1)).stream()) - .zip(short_ids.iter().stream().map(at!(0))) - .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) - .collect() - .boxed() - .await; - - Ok(full_ids) - } - - #[inline] - pub async fn state_full_shortids( - &self, - shortstatehash: ShortStateHash, - ) -> Result> { let shortids = self - .services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .await - .map_err(|e| err!(Database("Missing state IDs: {e}")))? - .pop() - .expect("there is always one layer") - .full_state - .iter() - .copied() - .map(parse_compressed_state_event) - .collect(); + .state_full_shortids(shortstatehash) + .map(|result| result.expect("missing shortstatehash")) + .map(|vec| vec.into_iter().unzip()) + .boxed() + .shared(); - Ok(shortids) + let shortstatekeys = shortids + .clone() + .map(at!(0)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + let shorteventids = shortids + .map(at!(1)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + self.services + .short + .multi_get_eventid_from_short(shorteventids) + .zip(shortstatekeys) + .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) } /// Returns a single EventId from `room_id` with key (`event_type`, @@ -264,6 +254,28 @@ impl Service { .await } + #[inline] + pub async fn state_full_shortids( + &self, + shortstatehash: ShortStateHash, + ) -> Result> { + let shortids = self + .services + .state_compressor + .load_shortstatehash_info(shortstatehash) + .await + .map_err(|e| err!(Database("Missing state IDs: {e}")))? + .pop() + .expect("there is always one layer") + .full_state + .iter() + .copied() + .map(parse_compressed_state_event) + .collect(); + + Ok(shortids) + } + /// Returns a single PDU from `room_id` with key (`event_type`, /// `state_key`). pub async fn state_get( @@ -479,27 +491,30 @@ impl Service { /// Returns the full room state. #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_full( - &self, - room_id: &RoomId, - ) -> Result> { + pub fn room_state_full<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream> + Send + 'a { self.services .state .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_full(shortstatehash)) - .map_err(|e| err!(Database("Missing state for {room_id:?}: {e:?}"))) - .await + .map_ok(|shortstatehash| self.state_full(shortstatehash).map(Ok)) + .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .try_flatten_stream() } /// Returns the full room state pdus #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_full_pdus(&self, room_id: &RoomId) -> Result> { + pub fn room_state_full_pdus<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream> + Send + 'a { self.services .state .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_full_pdus(shortstatehash)) - .map_err(|e| err!(Database("Missing state pdus for {room_id:?}: {e:?}"))) - .await + .map_ok(|shortstatehash| self.state_full_pdus(shortstatehash).map(Ok)) + .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .try_flatten_stream() } /// Returns a single EventId from `room_id` with key (`event_type`, From 329925c661d6b166dfd6b73a94f7f076cf1ed9bc Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 26 Jan 2025 04:46:10 +0000 Subject: [PATCH 1449/2091] additional info level span adjustments Signed-off-by: Jason Volk --- src/api/server/publicrooms.rs | 4 ++-- src/api/server/send.rs | 8 +++++--- src/core/debug.rs | 7 +++++++ src/service/federation/execute.rs | 7 ++++--- src/service/rooms/event_handler/handle_incoming_pdu.rs | 4 ++-- src/service/rooms/event_handler/handle_prev_pdu.rs | 5 +++-- src/service/rooms/spaces/mod.rs | 2 +- 7 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index 2c09385b..ff74574a 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -13,7 +13,7 @@ use crate::{Error, Result, Ruma}; /// # `POST /_matrix/federation/v1/publicRooms` /// /// Lists the public rooms on this server. -#[tracing::instrument(skip_all, fields(%client), name = "publicrooms")] +#[tracing::instrument(name = "publicrooms", level = "debug", skip_all, fields(%client))] pub(crate) async fn get_public_rooms_filtered_route( State(services): State, InsecureClientIp(client): InsecureClientIp, @@ -51,7 +51,7 @@ pub(crate) async fn get_public_rooms_filtered_route( /// # `GET /_matrix/federation/v1/publicRooms` /// /// Lists the public rooms on this server. -#[tracing::instrument(skip_all, fields(%client), "publicrooms")] +#[tracing::instrument(name = "publicrooms", level = "debug", skip_all, fields(%client))] pub(crate) async fn get_public_rooms_route( State(services): State, InsecureClientIp(client): InsecureClientIp, diff --git a/src/api/server/send.rs b/src/api/server/send.rs index f4903447..2e615a0c 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -3,7 +3,9 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - debug, debug_warn, err, error, + debug, + debug::INFO_SPAN_LEVEL, + debug_warn, err, error, result::LogErr, trace, utils::{ @@ -49,8 +51,8 @@ type Pdu = (OwnedRoomId, OwnedEventId, CanonicalJsonObject); /// /// Push EDUs and PDUs to this server. #[tracing::instrument( - name = "send", - level = "debug", + name = "txn", + level = INFO_SPAN_LEVEL, skip_all, fields( %client, diff --git a/src/core/debug.rs b/src/core/debug.rs index aebfc833..ca0f2f2e 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -4,6 +4,7 @@ use std::{any::Any, panic}; // Export debug proc_macros pub use conduwuit_macros::recursion_depth; +use tracing::Level; // Export all of the ancillary tools from here as well. pub use crate::{result::DebugInspect, utils::debug::*}; @@ -51,6 +52,12 @@ macro_rules! debug_info { } } +pub const INFO_SPAN_LEVEL: Level = if cfg!(debug_assertions) { + Level::INFO +} else { + Level::DEBUG +}; + pub fn set_panic_trap() { let next = panic::take_hook(); panic::set_hook(Box::new(move |info| { diff --git a/src/service/federation/execute.rs b/src/service/federation/execute.rs index 27d98968..3146bb8a 100644 --- a/src/service/federation/execute.rs +++ b/src/service/federation/execute.rs @@ -2,8 +2,8 @@ use std::{fmt::Debug, mem}; use bytes::Bytes; use conduwuit::{ - debug, debug_error, debug_warn, err, error::inspect_debug_log, implement, trace, - utils::string::EMPTY, Err, Error, Result, + debug, debug::INFO_SPAN_LEVEL, debug_error, debug_warn, err, error::inspect_debug_log, + implement, trace, utils::string::EMPTY, Err, Error, Result, }; use http::{header::AUTHORIZATION, HeaderValue}; use ipaddress::IPAddress; @@ -48,7 +48,8 @@ where #[implement(super::Service)] #[tracing::instrument( - level = "debug" + name = "fed", + level = INFO_SPAN_LEVEL, skip(self, client, request), )] pub async fn execute_on( diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 7db71961..31c7762d 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -3,7 +3,7 @@ use std::{ time::Instant, }; -use conduwuit::{debug, err, implement, warn, Err, Result}; +use conduwuit::{debug, debug::INFO_SPAN_LEVEL, err, implement, warn, Err, Result}; use futures::{ future::{try_join5, OptionFuture}, FutureExt, @@ -42,7 +42,7 @@ use crate::rooms::timeline::RawPduId; #[implement(super::Service)] #[tracing::instrument( name = "pdu", - level = "debug", + level = INFO_SPAN_LEVEL, skip_all, fields(%room_id, %event_id), )] diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index 32ab505f..f911f1fd 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -5,7 +5,8 @@ use std::{ }; use conduwuit::{ - debug, implement, utils::continue_exponential_backoff_secs, Err, PduEvent, Result, + debug, debug::INFO_SPAN_LEVEL, implement, utils::continue_exponential_backoff_secs, Err, + PduEvent, Result, }; use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, UInt}; @@ -14,7 +15,7 @@ use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, UInt}; #[allow(clippy::too_many_arguments)] #[tracing::instrument( name = "prev", - level = "debug", + level = INFO_SPAN_LEVEL, skip_all, fields(%prev_id), )] diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index d12a01ab..1ee2727c 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -268,7 +268,7 @@ impl Service { } /// Gets the summary of a space using solely federation - #[tracing::instrument(skip(self))] + #[tracing::instrument(level = "debug", skip(self))] async fn get_summary_and_children_federation( &self, current_room: &OwnedRoomId, From 936161d89ece2474dcba5424adaa159fc4e97b03 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 29 Jan 2025 01:49:10 +0000 Subject: [PATCH 1450/2091] reduce bottommost compression underrides Signed-off-by: Jason Volk --- src/database/engine/descriptor.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index c735f402..934ef831 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -83,7 +83,7 @@ pub(crate) static RANDOM: Descriptor = Descriptor { write_size: 1024 * 1024 * 32, cache_shards: 128, compression_level: -3, - bottommost_level: Some(4), + bottommost_level: Some(-1), compressed_index: true, ..BASE }; @@ -94,8 +94,8 @@ pub(crate) static SEQUENTIAL: Descriptor = Descriptor { level_size: 1024 * 1024 * 32, file_size: 1024 * 1024 * 2, cache_shards: 128, - compression_level: -1, - bottommost_level: Some(6), + compression_level: -2, + bottommost_level: Some(-1), compression_shape: [0, 0, 1, 1, 1, 1, 1], compressed_index: false, ..BASE @@ -111,7 +111,7 @@ pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { block_size: 512, cache_shards: 64, compression_level: -4, - bottommost_level: Some(1), + bottommost_level: Some(-1), compression_shape: [0, 0, 0, 0, 0, 1, 1], compressed_index: false, ..RANDOM @@ -126,8 +126,8 @@ pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { block_size: 512, cache_shards: 64, block_index_hashing: Some(false), - compression_level: -2, - bottommost_level: Some(4), + compression_level: -4, + bottommost_level: Some(-2), compression_shape: [0, 0, 0, 0, 1, 1, 1], compressed_index: false, ..SEQUENTIAL From eb7d893c8675f955fa770c8ae6f1c32a2394284c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 29 Jan 2025 06:36:14 +0000 Subject: [PATCH 1451/2091] fix malloc_conf feature-awareness Signed-off-by: Jason Volk --- src/core/alloc/je.rs | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 6bdf8b33..57143e85 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -8,6 +8,7 @@ use std::{ }; use arrayvec::ArrayVec; +use const_str::concat_bytes; use tikv_jemalloc_ctl as mallctl; use tikv_jemalloc_sys as ffi; use tikv_jemallocator as jemalloc; @@ -20,18 +21,24 @@ use crate::{ #[cfg(feature = "jemalloc_conf")] #[unsafe(no_mangle)] -pub static malloc_conf: &[u8] = b"\ -metadata_thp:always\ -,percpu_arena:percpu\ -,background_thread:true\ -,max_background_threads:-1\ -,lg_extent_max_active_fit:4\ -,oversize_threshold:16777216\ -,tcache_max:2097152\ -,dirty_decay_ms:16000\ -,muzzy_decay_ms:144000\ -,prof_active:false\ -\0"; +pub static malloc_conf: &[u8] = concat_bytes!( + "lg_extent_max_active_fit:4", + ",oversize_threshold:16777216", + ",tcache_max:2097152", + ",dirty_decay_ms:16000", + ",muzzy_decay_ms:144000", + ",percpu_arena:percpu", + ",metadata_thp:always", + ",background_thread:true", + ",max_background_threads:-1", + MALLOC_CONF_PROF, + 0 +); + +#[cfg(all(feature = "jemalloc_conf", feature = "jemalloc_prof"))] +const MALLOC_CONF_PROF: &str = ",prof_active:false"; +#[cfg(all(feature = "jemalloc_conf", not(feature = "jemalloc_prof")))] +const MALLOC_CONF_PROF: &str = ""; #[global_allocator] static JEMALLOC: jemalloc::Jemalloc = jemalloc::Jemalloc; From 50acfe783289e6b9b8deb20b3c34f32653f61f11 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 29 Jan 2025 08:39:44 +0000 Subject: [PATCH 1452/2091] flatten auth chain iterations Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 7 +- src/api/server/event_auth.rs | 4 +- src/api/server/send_join.rs | 2 - src/api/server/state.rs | 2 - src/api/server/state_ids.rs | 8 +- src/service/rooms/auth_chain/mod.rs | 154 +++++++++--------- .../rooms/event_handler/resolve_state.rs | 15 +- .../rooms/event_handler/state_at_incoming.rs | 9 +- 8 files changed, 90 insertions(+), 111 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index cd892ded..4e0ce2e3 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,8 +6,9 @@ use std::{ }; use conduwuit::{ - debug_error, err, info, trace, utils, utils::string::EMPTY, warn, Error, PduEvent, PduId, - RawPduId, Result, + debug_error, err, info, trace, utils, + utils::{stream::ReadyExt, string::EMPTY}, + warn, Error, PduEvent, PduId, RawPduId, Result, }; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ @@ -54,7 +55,7 @@ pub(super) async fn get_auth_chain( .rooms .auth_chain .event_ids_iter(room_id, once(event_id.as_ref())) - .await? + .ready_filter_map(Result::ok) .count() .await; diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 93e867a0..49dcd718 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, iter::once}; use axum::extract::State; -use conduwuit::{Error, Result}; +use conduwuit::{utils::stream::ReadyExt, Error, Result}; use futures::StreamExt; use ruma::{ api::{client::error::ErrorKind, federation::authorization::get_event_authorization}, @@ -48,7 +48,7 @@ pub(crate) async fn get_event_authorization_route( .rooms .auth_chain .event_ids_iter(room_id, once(body.event_id.borrow())) - .await? + .ready_filter_map(Result::ok) .filter_map(|id| async move { services.rooms.timeline.get_pdu_json(&id).await.ok() }) .then(|pdu| services.sending.convert_to_outgoing_federation_event(pdu)) .collect() diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 2b8a0eef..e81d7672 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -238,8 +238,6 @@ async fn create_join_event( .rooms .auth_chain .event_ids_iter(room_id, starting_events) - .await? - .map(Ok) .broad_and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) diff --git a/src/api/server/state.rs b/src/api/server/state.rs index eab1f138..b16e61a0 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -56,8 +56,6 @@ pub(crate) async fn get_room_state_route( .rooms .auth_chain .event_ids_iter(&body.room_id, once(body.event_id.borrow())) - .await? - .map(Ok) .and_then(|id| async move { services.rooms.timeline.get_pdu_json(&id).await }) .and_then(|pdu| { services diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index 4973dd3a..7d0440bf 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -2,7 +2,7 @@ use std::{borrow::Borrow, iter::once}; use axum::extract::State; use conduwuit::{at, err, Result}; -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use ruma::{api::federation::event::get_room_state_ids, OwnedEventId}; use super::AccessCheck; @@ -44,10 +44,8 @@ pub(crate) async fn get_room_state_ids_route( .rooms .auth_chain .event_ids_iter(&body.room_id, once(body.event_id.borrow())) - .await? - .map(|id| (*id).to_owned()) - .collect() - .await; + .try_collect() + .await?; Ok(get_room_state_ids::v1::Response { auth_chain_ids, pdu_ids }) } diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index df2663b2..0ff96846 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -4,6 +4,7 @@ use std::{ collections::{BTreeSet, HashSet, VecDeque}, fmt::Debug, sync::Arc, + time::Instant, }; use conduwuit::{ @@ -14,7 +15,7 @@ use conduwuit::{ }, validated, warn, Err, Result, }; -use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{EventId, OwnedEventId, RoomId}; use self::data::Data; @@ -30,6 +31,8 @@ struct Services { timeline: Dep, } +type Bucket<'a> = BTreeSet<(u64, &'a EventId)>; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -45,42 +48,22 @@ impl crate::Service for Service { } #[implement(Service)] -pub async fn event_ids_iter<'a, I>( +pub fn event_ids_iter<'a, I>( &'a self, - room_id: &RoomId, + room_id: &'a RoomId, starting_events: I, -) -> Result + Send + '_> +) -> impl Stream> + Send + 'a where I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, { - let stream = self - .get_event_ids(room_id, starting_events) - .await? - .into_iter() - .stream(); - - Ok(stream) -} - -#[implement(Service)] -pub async fn get_event_ids<'a, I>( - &'a self, - room_id: &RoomId, - starting_events: I, -) -> Result> -where - I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, -{ - let chain = self.get_auth_chain(room_id, starting_events).await?; - let event_ids = self - .services - .short - .multi_get_eventid_from_short(chain.into_iter().stream()) - .ready_filter_map(Result::ok) - .collect() - .await; - - Ok(event_ids) + self.get_auth_chain(room_id, starting_events) + .map_ok(|chain| { + self.services + .short + .multi_get_eventid_from_short(chain.into_iter().stream()) + .ready_filter(Result::is_ok) + }) + .try_flatten_stream() } #[implement(Service)] @@ -94,9 +77,9 @@ where I: Iterator + Clone + Debug + ExactSizeIterator + Send + 'a, { const NUM_BUCKETS: usize = 50; //TODO: change possible w/o disrupting db? - const BUCKET: BTreeSet<(u64, &EventId)> = BTreeSet::new(); + const BUCKET: Bucket<'_> = BTreeSet::new(); - let started = std::time::Instant::now(); + let started = Instant::now(); let mut starting_ids = self .services .short @@ -120,53 +103,7 @@ where let full_auth_chain: Vec = buckets .into_iter() .try_stream() - .broad_and_then(|chunk| async move { - let chunk_key: Vec = chunk.iter().map(at!(0)).collect(); - - if chunk_key.is_empty() { - return Ok(Vec::new()); - } - - if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { - return Ok(cached.to_vec()); - } - - let chunk_cache: Vec<_> = chunk - .into_iter() - .try_stream() - .broad_and_then(|(shortid, event_id)| async move { - if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await { - return Ok(cached.to_vec()); - } - - let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; - self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice()); - debug!( - ?event_id, - elapsed = ?started.elapsed(), - "Cache missed event" - ); - - Ok(auth_chain) - }) - .try_collect() - .map_ok(|chunk_cache: Vec<_>| chunk_cache.into_iter().flatten().collect()) - .map_ok(|mut chunk_cache: Vec<_>| { - chunk_cache.sort_unstable(); - chunk_cache.dedup(); - chunk_cache - }) - .await?; - - self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice()); - debug!( - chunk_cache_length = ?chunk_cache.len(), - elapsed = ?started.elapsed(), - "Cache missed chunk", - ); - - Ok(chunk_cache) - }) + .broad_and_then(|chunk| self.get_auth_chain_outer(room_id, started, chunk)) .try_collect() .map_ok(|auth_chain: Vec<_>| auth_chain.into_iter().flatten().collect()) .map_ok(|mut full_auth_chain: Vec<_>| { @@ -174,6 +111,7 @@ where full_auth_chain.dedup(); full_auth_chain }) + .boxed() .await?; debug!( @@ -185,6 +123,60 @@ where Ok(full_auth_chain) } +#[implement(Service)] +async fn get_auth_chain_outer( + &self, + room_id: &RoomId, + started: Instant, + chunk: Bucket<'_>, +) -> Result> { + let chunk_key: Vec = chunk.iter().map(at!(0)).collect(); + + if chunk_key.is_empty() { + return Ok(Vec::new()); + } + + if let Ok(cached) = self.get_cached_eventid_authchain(&chunk_key).await { + return Ok(cached.to_vec()); + } + + let chunk_cache: Vec<_> = chunk + .into_iter() + .try_stream() + .broad_and_then(|(shortid, event_id)| async move { + if let Ok(cached) = self.get_cached_eventid_authchain(&[shortid]).await { + return Ok(cached.to_vec()); + } + + let auth_chain = self.get_auth_chain_inner(room_id, event_id).await?; + self.cache_auth_chain_vec(vec![shortid], auth_chain.as_slice()); + debug!( + ?event_id, + elapsed = ?started.elapsed(), + "Cache missed event" + ); + + Ok(auth_chain) + }) + .try_collect() + .map_ok(|chunk_cache: Vec<_>| chunk_cache.into_iter().flatten().collect()) + .map_ok(|mut chunk_cache: Vec<_>| { + chunk_cache.sort_unstable(); + chunk_cache.dedup(); + chunk_cache + }) + .await?; + + self.cache_auth_chain_vec(chunk_key, chunk_cache.as_slice()); + debug!( + chunk_cache_length = ?chunk_cache.len(), + elapsed = ?started.elapsed(), + "Cache missed chunk", + ); + + Ok(chunk_cache) +} + #[implement(Service)] #[tracing::instrument(name = "inner", level = "trace", skip(self, room_id))] async fn get_auth_chain_inner( diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 1fd91ac6..03f7e822 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -44,18 +44,11 @@ pub async fn resolve_state( let auth_chain_sets: Vec> = fork_states .iter() .try_stream() - .wide_and_then(|state| async move { - let starting_events = state.values().map(Borrow::borrow); - - let auth_chain = self - .services + .wide_and_then(|state| { + self.services .auth_chain - .get_event_ids(room_id, starting_events) - .await? - .into_iter() - .collect(); - - Ok(auth_chain) + .event_ids_iter(room_id, state.values().map(Borrow::borrow)) + .try_collect() }) .try_collect() .await?; diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 7ef047ab..8730232a 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -10,7 +10,7 @@ use conduwuit::{ utils::stream::{BroadbandExt, IterStream}, PduEvent, Result, }; -use futures::{FutureExt, StreamExt}; +use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{state_res::StateMap, OwnedEventId, RoomId, RoomVersionId}; // TODO: if we know the prev_events of the incoming event we can avoid the @@ -140,10 +140,9 @@ pub(super) async fn state_at_incoming_resolved( let auth_chain: HashSet = self .services .auth_chain - .get_event_ids(room_id, starting_events.into_iter()) - .await? - .into_iter() - .collect(); + .event_ids_iter(room_id, starting_events.into_iter()) + .try_collect() + .await?; auth_chain_sets.push(auth_chain); fork_states.push(state); From 3c8376d897e6a1b9b6b61f5ada05b2afec1ab937 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 29 Jan 2025 23:07:12 +0000 Subject: [PATCH 1453/2091] parallelize state-res pre-gathering Signed-off-by: Jason Volk --- .../rooms/event_handler/resolve_state.rs | 63 +++---- .../rooms/event_handler/state_at_incoming.rs | 173 +++++++++--------- 2 files changed, 123 insertions(+), 113 deletions(-) diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 03f7e822..c3de5f2f 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -5,11 +5,11 @@ use std::{ }; use conduwuit::{ - debug, err, implement, + err, implement, trace, utils::stream::{automatic_width, IterStream, ReadyExt, TryWidebandExt, WidebandExt}, - Result, + Error, Result, }; -use futures::{FutureExt, StreamExt, TryStreamExt}; +use futures::{future::try_join, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{ state_res::{self, StateMap}, OwnedEventId, RoomId, RoomVersionId, @@ -25,13 +25,13 @@ pub async fn resolve_state( room_version_id: &RoomVersionId, incoming_state: HashMap, ) -> Result>> { - debug!("Loading current room state ids"); + trace!("Loading current room state ids"); let current_sstatehash = self .services .state .get_room_shortstatehash(room_id) - .await - .map_err(|e| err!(Database(error!("No state for {room_id:?}: {e:?}"))))?; + .map_err(|e| err!(Database(error!("No state for {room_id:?}: {e:?}")))) + .await?; let current_state_ids: HashMap<_, _> = self .services @@ -40,8 +40,9 @@ pub async fn resolve_state( .collect() .await; + trace!("Loading fork states"); let fork_states = [current_state_ids, incoming_state]; - let auth_chain_sets: Vec> = fork_states + let auth_chain_sets = fork_states .iter() .try_stream() .wide_and_then(|state| { @@ -50,36 +51,33 @@ pub async fn resolve_state( .event_ids_iter(room_id, state.values().map(Borrow::borrow)) .try_collect() }) - .try_collect() - .await?; + .try_collect::>>(); - debug!("Loading fork states"); - let fork_states: Vec> = fork_states - .into_iter() + let fork_states = fork_states + .iter() .stream() - .wide_then(|fork_state| async move { + .wide_then(|fork_state| { let shortstatekeys = fork_state.keys().copied().stream(); - - let event_ids = fork_state.values().cloned().stream().boxed(); - + let event_ids = fork_state.values().cloned().stream(); self.services .short .multi_get_statekey_from_short(shortstatekeys) .zip(event_ids) .ready_filter_map(|(ty_sk, id)| Some((ty_sk.ok()?, id))) .collect() - .await }) - .collect() - .await; + .map(Ok::<_, Error>) + .try_collect::>>(); - debug!("Resolving state"); + let (fork_states, auth_chain_sets) = try_join(fork_states, auth_chain_sets).await?; + + trace!("Resolving state"); let state = self - .state_resolution(room_version_id, &fork_states, &auth_chain_sets) + .state_resolution(room_version_id, fork_states.iter(), &auth_chain_sets) .boxed() .await?; - debug!("State resolution done."); + trace!("State resolution done."); let state_events: Vec<_> = state .iter() .stream() @@ -92,7 +90,7 @@ pub async fn resolve_state( .collect() .await; - debug!("Compressing state..."); + trace!("Compressing state..."); let new_room_state: HashSet<_> = self .services .state_compressor @@ -109,20 +107,23 @@ pub async fn resolve_state( #[implement(super::Service)] #[tracing::instrument(name = "ruma", level = "debug", skip_all)] -pub async fn state_resolution( - &self, - room_version: &RoomVersionId, - state_sets: &[StateMap], - auth_chain_sets: &[HashSet], -) -> Result> { +pub async fn state_resolution<'a, StateSets>( + &'a self, + room_version: &'a RoomVersionId, + state_sets: StateSets, + auth_chain_sets: &'a [HashSet], +) -> Result> +where + StateSets: Iterator> + Clone + Send, +{ state_res::resolve( room_version, - state_sets.iter(), + state_sets, auth_chain_sets, &|event_id| self.event_fetch(event_id), &|event_id| self.event_exists(event_id), automatic_width(), ) - .await .map_err(|e| err!(error!("State resolution failed: {e:?}"))) + .await } diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 8730232a..8ae6354c 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -1,18 +1,20 @@ use std::{ borrow::Borrow, collections::{HashMap, HashSet}, + iter::Iterator, sync::Arc, }; use conduwuit::{ - debug, err, implement, - result::LogErr, - utils::stream::{BroadbandExt, IterStream}, + debug, err, implement, trace, + utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryWidebandExt}, PduEvent, Result, }; -use futures::{FutureExt, StreamExt, TryStreamExt}; +use futures::{future::try_join, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{state_res::StateMap, OwnedEventId, RoomId, RoomVersionId}; +use crate::rooms::short::ShortStateHash; + // TODO: if we know the prev_events of the incoming event we can avoid the #[implement(super::Service)] // request and build the state from a known point and resolve if > 1 prev_event @@ -70,86 +72,44 @@ pub(super) async fn state_at_incoming_resolved( room_id: &RoomId, room_version_id: &RoomVersionId, ) -> Result>> { - debug!("Calculating state at event using state res"); - let mut extremity_sstatehashes = HashMap::with_capacity(incoming_pdu.prev_events.len()); - - let mut okay = true; - for prev_eventid in &incoming_pdu.prev_events { - let Ok(prev_event) = self.services.timeline.get_pdu(prev_eventid).await else { - okay = false; - break; - }; - - let Ok(sstatehash) = self - .services - .state_accessor - .pdu_shortstatehash(prev_eventid) - .await - else { - okay = false; - break; - }; - - extremity_sstatehashes.insert(sstatehash, prev_event); - } - - if !okay { + trace!("Calculating extremity statehashes..."); + let Ok(extremity_sstatehashes) = incoming_pdu + .prev_events + .iter() + .try_stream() + .broad_and_then(|prev_eventid| { + self.services + .timeline + .get_pdu(prev_eventid) + .map_ok(move |prev_event| (prev_eventid, prev_event)) + }) + .broad_and_then(|(prev_eventid, prev_event)| { + self.services + .state_accessor + .pdu_shortstatehash(prev_eventid) + .map_ok(move |sstatehash| (sstatehash, prev_event)) + }) + .try_collect::>() + .await + else { return Ok(None); - } + }; - let mut fork_states = Vec::with_capacity(extremity_sstatehashes.len()); - let mut auth_chain_sets = Vec::with_capacity(extremity_sstatehashes.len()); - for (sstatehash, prev_event) in extremity_sstatehashes { - let mut leaf_state: HashMap<_, _> = self - .services - .state_accessor - .state_full_ids(sstatehash) - .collect() - .await; - - if let Some(state_key) = &prev_event.state_key { - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&prev_event.kind.to_string().into(), state_key) - .await; - - let event_id = &prev_event.event_id; - leaf_state.insert(shortstatekey, event_id.clone()); - // Now it's the state after the pdu - } - - let mut state = StateMap::with_capacity(leaf_state.len()); - let mut starting_events = Vec::with_capacity(leaf_state.len()); - for (k, id) in &leaf_state { - if let Ok((ty, st_key)) = self - .services - .short - .get_statekey_from_short(*k) - .await - .log_err() - { - // FIXME: Undo .to_string().into() when StateMap - // is updated to use StateEventType - state.insert((ty.to_string().into(), st_key), id.clone()); - } - - starting_events.push(id.borrow()); - } - - let auth_chain: HashSet = self - .services - .auth_chain - .event_ids_iter(room_id, starting_events.into_iter()) + trace!("Calculating fork states..."); + let (fork_states, auth_chain_sets): (Vec>, Vec>) = + extremity_sstatehashes + .into_iter() + .try_stream() + .wide_and_then(|(sstatehash, prev_event)| { + self.state_at_incoming_fork(room_id, sstatehash, prev_event) + }) .try_collect() + .map_ok(Vec::into_iter) + .map_ok(Iterator::unzip) .await?; - auth_chain_sets.push(auth_chain); - fork_states.push(state); - } - let Ok(new_state) = self - .state_resolution(room_version_id, &fork_states, &auth_chain_sets) + .state_resolution(room_version_id, fork_states.iter(), &auth_chain_sets) .boxed() .await else { @@ -157,16 +117,65 @@ pub(super) async fn state_at_incoming_resolved( }; new_state - .iter() + .into_iter() .stream() - .broad_then(|((event_type, state_key), event_id)| { + .broad_then(|((event_type, state_key), event_id)| async move { self.services .short - .get_or_create_shortstatekey(event_type, state_key) - .map(move |shortstatekey| (shortstatekey, event_id.clone())) + .get_or_create_shortstatekey(&event_type, &state_key) + .map(move |shortstatekey| (shortstatekey, event_id)) + .await }) .collect() .map(Some) .map(Ok) .await } + +#[implement(super::Service)] +async fn state_at_incoming_fork( + &self, + room_id: &RoomId, + sstatehash: ShortStateHash, + prev_event: PduEvent, +) -> Result<(StateMap, HashSet)> { + let mut leaf_state: HashMap<_, _> = self + .services + .state_accessor + .state_full_ids(sstatehash) + .collect() + .await; + + if let Some(state_key) = &prev_event.state_key { + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&prev_event.kind.to_string().into(), state_key) + .await; + + let event_id = &prev_event.event_id; + leaf_state.insert(shortstatekey, event_id.clone()); + // Now it's the state after the pdu + } + + let auth_chain = self + .services + .auth_chain + .event_ids_iter(room_id, leaf_state.values().map(Borrow::borrow)) + .try_collect(); + + let fork_state = leaf_state + .iter() + .stream() + .broad_then(|(k, id)| { + self.services + .short + .get_statekey_from_short(*k) + .map_ok(|(ty, sk)| ((ty, sk), id.clone())) + }) + .ready_filter_map(Result::ok) + .collect() + .map(Ok); + + try_join(fork_state, auth_chain).await +} From 31c2968bb29e7447e56531333fb330da4ac08ede Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 29 Jan 2025 21:10:33 +0000 Subject: [PATCH 1454/2091] move db files command w/ filter args; misc related cleanup Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 66 +++++++++++++++++++++++++++++------- src/admin/debug/mod.rs | 8 +++++ src/admin/server/commands.rs | 15 +++----- src/admin/server/mod.rs | 3 -- src/database/engine/files.rs | 35 +++++-------------- src/service/globals/data.rs | 3 -- 6 files changed, 75 insertions(+), 55 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 4e0ce2e3..dcf9879c 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -7,7 +7,10 @@ use std::{ use conduwuit::{ debug_error, err, info, trace, utils, - utils::{stream::ReadyExt, string::EMPTY}, + utils::{ + stream::{IterStream, ReadyExt}, + string::EMPTY, + }, warn, Error, PduEvent, PduId, RawPduId, Result, }; use futures::{FutureExt, StreamExt, TryStreamExt}; @@ -640,6 +643,7 @@ pub(super) async fn force_set_room_state_from_server( room_id: room_id.clone().into(), event_id: first_pdu.event_id.clone(), }) + .boxed() .await?; for pdu in remote_state_response.pdus.clone() { @@ -648,6 +652,7 @@ pub(super) async fn force_set_room_state_from_server( .rooms .event_handler .parse_incoming_pdu(&pdu) + .boxed() .await { | Ok(t) => t, @@ -711,6 +716,7 @@ pub(super) async fn force_set_room_state_from_server( .rooms .event_handler .resolve_state(&room_id, &room_version, state) + .boxed() .await?; info!("Forcing new room state"); @@ -946,21 +952,57 @@ pub(super) async fn database_stats( property: Option, map: Option, ) -> Result { - let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned()); let map_name = map.as_ref().map_or(EMPTY, String::as_str); + let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned()); + self.services + .db + .iter() + .filter(|(&name, _)| map_name.is_empty() || map_name == name) + .try_stream() + .try_for_each(|(&name, map)| { + let res = map.property(&property).expect("invalid property"); + writeln!(self, "##### {name}:\n```\n{}\n```", res.trim()) + }) + .await?; - let mut out = String::new(); - for (&name, map) in self.services.db.iter() { - if !map_name.is_empty() && map_name != name { - continue; - } + Ok(RoomMessageEventContent::notice_plain("")) +} - let res = map.property(&property)?; - let res = res.trim(); - writeln!(out, "##### {name}:\n```\n{res}\n```")?; - } +#[admin_command] +pub(super) async fn database_files( + &self, + map: Option, + level: Option, +) -> Result { + let mut files: Vec<_> = self.services.db.db.file_list().collect::>()?; - Ok(RoomMessageEventContent::notice_markdown(out)) + files.sort_by_key(|f| f.name.clone()); + + writeln!(self, "| lev | sst | keys | dels | size | column |").await?; + writeln!(self, "| ---: | :--- | ---: | ---: | ---: | :--- |").await?; + files + .into_iter() + .filter(|file| { + map.as_deref() + .is_none_or(|map| map == file.column_family_name) + }) + .filter(|file| level.as_ref().is_none_or(|&level| level == file.level)) + .try_stream() + .try_for_each(|file| { + writeln!( + self, + "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", + file.level, + file.name, + file.num_entries, + file.num_deletions, + file.size, + file.column_family_name, + ) + }) + .await?; + + Ok(RoomMessageEventContent::notice_plain("")) } #[admin_command] diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 07f7296b..db04ccf4 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -226,6 +226,14 @@ pub(super) enum DebugCommand { /// - Trim memory usage TrimMemory, + /// - List database files + DatabaseFiles { + map: Option, + + #[arg(long)] + level: Option, + }, + /// - Developer test stubs #[command(subcommand)] #[allow(non_snake_case)] diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 910dce6e..d4cfa7d5 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -92,7 +92,7 @@ pub(super) async fn clear_caches(&self) -> Result { #[admin_command] pub(super) async fn list_backups(&self) -> Result { - let result = self.services.globals.db.backup_list()?; + let result = self.services.db.db.backup_list()?; if result.is_empty() { Ok(RoomMessageEventContent::text_plain("No backups found.")) @@ -103,31 +103,24 @@ pub(super) async fn list_backups(&self) -> Result { #[admin_command] pub(super) async fn backup_database(&self) -> Result { - let globals = Arc::clone(&self.services.globals); + let db = Arc::clone(&self.services.db); let mut result = self .services .server .runtime() - .spawn_blocking(move || match globals.db.backup() { + .spawn_blocking(move || match db.db.backup() { | Ok(()) => String::new(), | Err(e) => e.to_string(), }) .await?; if result.is_empty() { - result = self.services.globals.db.backup_list()?; + result = self.services.db.db.backup_list()?; } Ok(RoomMessageEventContent::notice_markdown(result)) } -#[admin_command] -pub(super) async fn list_database_files(&self) -> Result { - let result = self.services.globals.db.file_list()?; - - Ok(RoomMessageEventContent::notice_markdown(result)) -} - #[admin_command] pub(super) async fn admin_notice(&self, message: Vec) -> Result { let message = message.join(" "); diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 3f3d6c5e..60615365 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -46,9 +46,6 @@ pub(super) enum ServerCommand { /// - List database backups ListBackups, - /// - List database files - ListDatabaseFiles, - /// - Send a message to the admin room. AdminNotice { message: Vec, diff --git a/src/database/engine/files.rs b/src/database/engine/files.rs index f603c57b..33d6fdc4 100644 --- a/src/database/engine/files.rs +++ b/src/database/engine/files.rs @@ -1,32 +1,15 @@ -use std::fmt::Write; - use conduwuit::{implement, Result}; +use rocksdb::LiveFile as SstFile; use super::Engine; +use crate::util::map_err; #[implement(Engine)] -pub fn file_list(&self) -> Result { - match self.db.live_files() { - | Err(e) => Ok(String::from(e)), - | Ok(mut files) => { - files.sort_by_key(|f| f.name.clone()); - let mut res = String::new(); - writeln!(res, "| lev | sst | keys | dels | size | column |")?; - writeln!(res, "| ---: | :--- | ---: | ---: | ---: | :--- |")?; - for file in files { - writeln!( - res, - "| {} | {:<13} | {:7}+ | {:4}- | {:9} | {} |", - file.level, - file.name, - file.num_entries, - file.num_deletions, - file.size, - file.column_family_name, - )?; - } - - Ok(res) - }, - } +pub fn file_list(&self) -> impl Iterator> + Send { + self.db + .live_files() + .map_err(map_err) + .into_iter() + .flat_map(Vec::into_iter) + .map(Ok) } diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 07b4ac2c..39cb9be1 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -79,7 +79,4 @@ impl Data { #[inline] pub fn backup_list(&self) -> Result { self.db.db.backup_list() } - - #[inline] - pub fn file_list(&self) -> Result { self.db.db.file_list() } } From 1a8482b3b4865a7f38c342929489ba925a98e05c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 30 Jan 2025 04:39:24 +0000 Subject: [PATCH 1455/2091] refactor incoming extremities retention; broad filter, single pass Signed-off-by: Jason Volk --- src/api/client/membership.rs | 7 +- .../event_handler/upgrade_outlier_pdu.rs | 74 +++++++++---------- src/service/rooms/state/mod.rs | 17 +++-- src/service/rooms/timeline/mod.rs | 44 ++++++----- 4 files changed, 74 insertions(+), 68 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index fccb9b53..d80aff0c 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1,6 +1,7 @@ use std::{ borrow::Borrow, collections::{BTreeMap, HashMap, HashSet}, + iter::once, net::IpAddr, sync::Arc, }; @@ -1216,7 +1217,7 @@ async fn join_room_by_id_helper_remote( .append_pdu( &parsed_join_pdu, join_event, - vec![(*parsed_join_pdu.event_id).to_owned()], + once(parsed_join_pdu.event_id.borrow()), &state_lock, ) .await?; @@ -2195,7 +2196,7 @@ async fn knock_room_helper_local( .append_pdu( &parsed_knock_pdu, knock_event, - vec![(*parsed_knock_pdu.event_id).to_owned()], + once(parsed_knock_pdu.event_id.borrow()), &state_lock, ) .await?; @@ -2394,7 +2395,7 @@ async fn knock_room_helper_remote( .append_pdu( &parsed_knock_pdu, knock_event, - vec![(*parsed_knock_pdu.event_id).to_owned()], + once(parsed_knock_pdu.event_id.borrow()), &state_lock, ) .await?; diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index f0c8f0c5..ca351981 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -1,14 +1,18 @@ use std::{ borrow::Borrow, collections::{BTreeMap, HashSet}, + iter::once, sync::Arc, time::Instant, }; -use conduwuit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; -use futures::{future::ready, StreamExt}; +use conduwuit::{ + debug, debug_info, err, implement, trace, + utils::stream::{BroadbandExt, ReadyExt}, + warn, Err, PduEvent, Result, +}; +use futures::{future::ready, FutureExt, StreamExt}; use ruma::{ - api::client::error::ErrorKind, events::{room::redaction::RoomRedactionEventContent, StateEventType, TimelineEventType}, state_res::{self, EventTypeExt}, CanonicalJsonValue, RoomId, RoomVersionId, ServerName, @@ -174,42 +178,34 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( // Now we calculate the set of extremities this room has after the incoming // event has been applied. We start with the previous extremities (aka leaves) trace!("Calculating extremities"); - let mut extremities: HashSet<_> = self + let extremities: Vec<_> = self .services .state .get_forward_extremities(room_id) .map(ToOwned::to_owned) + .ready_filter(|event_id| { + // Remove any that are referenced by this incoming event's prev_events + !incoming_pdu.prev_events.contains(event_id) + }) + .broad_filter_map(|event_id| async move { + // Only keep those extremities were not referenced yet + self.services + .pdu_metadata + .is_event_referenced(room_id, &event_id) + .await + .eq(&false) + .then_some(event_id) + }) .collect() .await; - // Remove any forward extremities that are referenced by this incoming event's - // prev_events - trace!( - "Calculated {} extremities; checking against {} prev_events", + debug!( + "Retained {} extremities checked against {} prev_events", extremities.len(), incoming_pdu.prev_events.len() ); - for prev_event in &incoming_pdu.prev_events { - extremities.remove(&(**prev_event)); - } - // Only keep those extremities were not referenced yet - let mut retained = HashSet::new(); - for id in &extremities { - if !self - .services - .pdu_metadata - .is_event_referenced(room_id, id) - .await - { - retained.insert(id.clone()); - } - } - - extremities.retain(|id| retained.contains(id)); - debug!("Retained {} extremities. Compressing state", extremities.len()); - - let state_ids_compressed: HashSet<_> = self + let state_ids_compressed: Arc> = self .services .state_compressor .compress_state_events( @@ -218,10 +214,9 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( .map(|(ssk, eid)| (ssk, eid.borrow())), ) .collect() + .map(Arc::new) .await; - let state_ids_compressed = Arc::new(state_ids_compressed); - if incoming_pdu.state_key.is_some() { debug!("Event is a state-event. Deriving new room state"); @@ -260,12 +255,14 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( // if not soft fail it if soft_fail { debug!("Soft failing event"); + let extremities = extremities.iter().map(Borrow::borrow); + self.services .timeline .append_incoming_pdu( &incoming_pdu, val, - extremities.iter().map(|e| (**e).to_owned()).collect(), + extremities, state_ids_compressed, soft_fail, &state_lock, @@ -273,27 +270,30 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( .await?; // Soft fail, we keep the event as an outlier but don't add it to the timeline - warn!("Event was soft failed: {incoming_pdu:?}"); self.services .pdu_metadata .mark_event_soft_failed(&incoming_pdu.event_id); - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event has been soft failed")); + warn!("Event was soft failed: {incoming_pdu:?}"); + return Err!(Request(InvalidParam("Event has been soft failed"))); } - trace!("Appending pdu to timeline"); - extremities.insert(incoming_pdu.event_id.clone()); - // Now that the event has passed all auth it is added into the timeline. // We use the `state_at_event` instead of `state_after` so we accurately // represent the state for this event. + trace!("Appending pdu to timeline"); + let extremities = extremities + .iter() + .map(Borrow::borrow) + .chain(once(incoming_pdu.event_id.borrow())); + let pdu_id = self .services .timeline .append_incoming_pdu( &incoming_pdu, val, - extremities.into_iter().collect(), + extremities, state_ids_compressed, soft_fail, &state_lock, diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index fd303667..8cb4e586 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -398,13 +398,14 @@ impl Service { .ignore_err() } - pub async fn set_forward_extremities( - &self, - room_id: &RoomId, - event_ids: Vec, - _state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room - * state mutex */ - ) { + pub async fn set_forward_extremities<'a, I>( + &'a self, + room_id: &'a RoomId, + event_ids: I, + _state_lock: &'a RoomMutexGuard, + ) where + I: Iterator + Send + 'a, + { let prefix = (room_id, Interfix); self.db .roomid_pduleaves @@ -413,7 +414,7 @@ impl Service { .ready_for_each(|key| self.db.roomid_pduleaves.remove(key)) .await; - for event_id in &event_ids { + for event_id in event_ids { let key = (room_id, event_id); self.db.roomid_pduleaves.put_raw(key, event_id); } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index bf585a6b..8b3b67a7 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1,6 +1,7 @@ mod data; use std::{ + borrow::Borrow, cmp, collections::{BTreeMap, HashSet}, fmt::Write, @@ -260,14 +261,16 @@ impl Service { /// /// Returns pdu id #[tracing::instrument(level = "debug", skip_all)] - pub async fn append_pdu( - &self, - pdu: &PduEvent, + pub async fn append_pdu<'a, Leafs>( + &'a self, + pdu: &'a PduEvent, mut pdu_json: CanonicalJsonObject, - leaves: Vec, - state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room state - * mutex */ - ) -> Result { + leafs: Leafs, + state_lock: &'a RoomMutexGuard, + ) -> Result + where + Leafs: Iterator + Send + 'a, + { // Coalesce database writes for the remainder of this scope. let _cork = self.db.db.cork_and_flush(); @@ -335,7 +338,7 @@ impl Service { self.services .state - .set_forward_extremities(&pdu.room_id, leaves, state_lock) + .set_forward_extremities(&pdu.room_id, leafs, state_lock) .await; let insert_lock = self.mutex_insert.lock(&pdu.room_id).await; @@ -819,8 +822,7 @@ impl Service { pdu_builder: PduBuilder, sender: &UserId, room_id: &RoomId, - state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room state - * mutex */ + state_lock: &RoomMutexGuard, ) -> Result { let (pdu, pdu_json) = self .create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock) @@ -896,7 +898,7 @@ impl Service { pdu_json, // Since this PDU references all pdu_leaves we can update the leaves // of the room - vec![(*pdu.event_id).to_owned()], + once(pdu.event_id.borrow()), state_lock, ) .boxed() @@ -943,16 +945,18 @@ impl Service { /// Append the incoming event setting the state snapshot to the state from /// the server that sent the event. #[tracing::instrument(level = "debug", skip_all)] - pub async fn append_incoming_pdu( - &self, - pdu: &PduEvent, + pub async fn append_incoming_pdu<'a, Leafs>( + &'a self, + pdu: &'a PduEvent, pdu_json: CanonicalJsonObject, - new_room_leaves: Vec, + new_room_leafs: Leafs, state_ids_compressed: Arc>, soft_fail: bool, - state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room state - * mutex */ - ) -> Result> { + state_lock: &'a RoomMutexGuard, + ) -> Result> + where + Leafs: Iterator + Send + 'a, + { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't // fail. @@ -968,14 +972,14 @@ impl Service { self.services .state - .set_forward_extremities(&pdu.room_id, new_room_leaves, state_lock) + .set_forward_extremities(&pdu.room_id, new_room_leafs, state_lock) .await; return Ok(None); } let pdu_id = self - .append_pdu(pdu, pdu_json, new_room_leaves, state_lock) + .append_pdu(pdu, pdu_json, new_room_leafs, state_lock) .await?; Ok(Some(pdu_id)) From ff8bbd4cfa6ad9426bd9efbe610547dd89030c85 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 30 Jan 2025 05:14:45 +0000 Subject: [PATCH 1456/2091] untwist the redaction check stanza Signed-off-by: Jason Volk --- src/core/pdu/redact.rs | 18 +++++++ .../event_handler/upgrade_outlier_pdu.rs | 52 ++++--------------- 2 files changed, 28 insertions(+), 42 deletions(-) diff --git a/src/core/pdu/redact.rs b/src/core/pdu/redact.rs index 5d33eeca..7c332719 100644 --- a/src/core/pdu/redact.rs +++ b/src/core/pdu/redact.rs @@ -90,3 +90,21 @@ pub fn copy_redacts(&self) -> (Option, Box) { (self.redacts.clone(), self.content.clone()) } + +#[implement(super::Pdu)] +#[must_use] +pub fn redacts_id(&self, room_version: &RoomVersionId) -> Option { + use RoomVersionId::*; + + if self.kind != TimelineEventType::RoomRedaction { + return None; + } + + match *room_version { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => self.redacts.clone(), + | _ => + self.get_content::() + .ok()? + .redacts, + } +} diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index ca351981..03697558 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -13,9 +13,9 @@ use conduwuit::{ }; use futures::{future::ready, FutureExt, StreamExt}; use ruma::{ - events::{room::redaction::RoomRedactionEventContent, StateEventType, TimelineEventType}, + events::StateEventType, state_res::{self, EventTypeExt}, - CanonicalJsonValue, RoomId, RoomVersionId, ServerName, + CanonicalJsonValue, RoomId, ServerName, }; use super::{get_room_version_id, to_room_version}; @@ -127,46 +127,14 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( // Soft fail check before doing state res debug!("Performing soft-fail check"); - let soft_fail = { - use RoomVersionId::*; - - !auth_check - || incoming_pdu.kind == TimelineEventType::RoomRedaction - && match room_version_id { - | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { - if let Some(redact_id) = &incoming_pdu.redacts { - !self - .services - .state_accessor - .user_can_redact( - redact_id, - &incoming_pdu.sender, - &incoming_pdu.room_id, - true, - ) - .await? - } else { - false - } - }, - | _ => { - let content: RoomRedactionEventContent = incoming_pdu.get_content()?; - if let Some(redact_id) = &content.redacts { - !self - .services - .state_accessor - .user_can_redact( - redact_id, - &incoming_pdu.sender, - &incoming_pdu.room_id, - true, - ) - .await? - } else { - false - } - }, - } + let soft_fail = match (auth_check, incoming_pdu.redacts_id(&room_version_id)) { + | (false, _) => true, + | (true, None) => false, + | (true, Some(redact_id)) => + self.services + .state_accessor + .user_can_redact(&redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) + .await?, }; // 13. Use state resolution to find new room state From 69837671bbc02b1cfba351e1c1321be506ef88b1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 30 Jan 2025 09:28:34 +0000 Subject: [PATCH 1457/2091] simplify request handler task base Signed-off-by: Jason Volk --- src/core/metrics/mod.rs | 4 --- src/router/layers.rs | 33 ++++++++++++---------- src/router/request.rs | 59 +++++++-------------------------------- src/router/run.rs | 1 - src/router/serve/plain.rs | 7 ----- src/router/serve/unix.rs | 7 ++++- 6 files changed, 35 insertions(+), 76 deletions(-) diff --git a/src/core/metrics/mod.rs b/src/core/metrics/mod.rs index f2022166..8f7a5571 100644 --- a/src/core/metrics/mod.rs +++ b/src/core/metrics/mod.rs @@ -19,8 +19,6 @@ pub struct Metrics { runtime_intervals: std::sync::Mutex>, // TODO: move stats - pub requests_spawn_active: AtomicU32, - pub requests_spawn_finished: AtomicU32, pub requests_handle_active: AtomicU32, pub requests_handle_finished: AtomicU32, pub requests_panic: AtomicU32, @@ -48,8 +46,6 @@ impl Metrics { #[cfg(tokio_unstable)] runtime_intervals: std::sync::Mutex::new(runtime_intervals), - requests_spawn_active: AtomicU32::new(0), - requests_spawn_finished: AtomicU32::new(0), requests_handle_active: AtomicU32::new(0), requests_handle_finished: AtomicU32::new(0), requests_panic: AtomicU32::new(0), diff --git a/src/router/layers.rs b/src/router/layers.rs index 96bca4fd..c5227c22 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -5,7 +5,7 @@ use axum::{ Router, }; use axum_client_ip::SecureClientIpSource; -use conduwuit::{error, Result, Server}; +use conduwuit::{debug, error, Result, Server}; use conduwuit_api::router::state::Guard; use conduwuit_service::Services; use http::{ @@ -50,7 +50,6 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { let layers = layers .layer(SetSensitiveHeadersLayer::new([header::AUTHORIZATION])) - .layer(axum::middleware::from_fn_with_state(Arc::clone(services), request::spawn)) .layer( TraceLayer::new_for_http() .make_span_with(tracing_span::<_>) @@ -196,20 +195,26 @@ fn catch_panic( } fn tracing_span(request: &http::Request) -> tracing::Span { - let path = request.extensions().get::().map_or_else( - || { - request - .uri() - .path_and_query() - .expect("all requests have a path") - .as_str() - }, - truncated_matched_path, - ); + let path = request + .extensions() + .get::() + .map_or_else(|| request_path_str(request), truncated_matched_path); - let method = request.method(); + tracing::span! { + parent: None, + debug::INFO_SPAN_LEVEL, + "router", + method = %request.method(), + %path, + } +} - tracing::debug_span!(parent: None, "router", %method, %path) +fn request_path_str(request: &http::Request) -> &str { + request + .uri() + .path_and_query() + .expect("all requests have a path") + .as_str() } fn truncated_matched_path(path: &MatchedPath) -> &str { diff --git a/src/router/request.rs b/src/router/request.rs index ca063338..f7b94417 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -8,48 +8,6 @@ use conduwuit::{debug, debug_error, debug_warn, err, error, trace, Result}; use conduwuit_service::Services; use http::{Method, StatusCode, Uri}; -#[tracing::instrument( - parent = None, - level = "trace", - skip_all, - fields( - handled = %services - .server - .metrics - .requests_spawn_finished - .fetch_add(1, Ordering::Relaxed), - active = %services - .server - .metrics - .requests_spawn_active - .fetch_add(1, Ordering::Relaxed), - ) -)] -pub(crate) async fn spawn( - State(services): State>, - req: http::Request, - next: axum::middleware::Next, -) -> Result { - let server = &services.server; - - #[cfg(debug_assertions)] - conduwuit::defer! {{ - _ = server - .metrics - .requests_spawn_active - .fetch_sub(1, Ordering::Relaxed); - }}; - - if !server.running() { - debug_warn!("unavailable pending shutdown"); - return Err(StatusCode::SERVICE_UNAVAILABLE); - } - - let fut = next.run(req); - let task = server.runtime().spawn(fut); - task.await.map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) -} - #[tracing::instrument( level = "debug", skip_all, @@ -71,17 +29,15 @@ pub(crate) async fn handle( req: http::Request, next: axum::middleware::Next, ) -> Result { - let server = &services.server; - #[cfg(debug_assertions)] conduwuit::defer! {{ - _ = server + _ = services.server .metrics .requests_handle_active .fetch_sub(1, Ordering::Relaxed); }}; - if !server.running() { + if !services.server.running() { debug_warn!( method = %req.method(), uri = %req.uri(), @@ -91,10 +47,15 @@ pub(crate) async fn handle( return Err(StatusCode::SERVICE_UNAVAILABLE); } - let uri = req.uri().clone(); let method = req.method().clone(); - let result = next.run(req).await; - handle_result(&method, &uri, result) + let uri = req.uri().clone(); + services + .server + .runtime() + .spawn(next.run(req)) + .await + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) + .and_then(|result| handle_result(&method, &uri, result)) } fn handle_result(method: &Method, uri: &Uri, result: Response) -> Result { diff --git a/src/router/run.rs b/src/router/run.rs index ea8a7666..605168b8 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -125,7 +125,6 @@ async fn handle_shutdown(server: Arc, tx: Sender<()>, handle: axum_serve let timeout = Duration::from_secs(36); debug!( ?timeout, - spawn_active = ?server.metrics.requests_spawn_active.load(Ordering::Relaxed), handle_active = ?server.metrics.requests_handle_active.load(Ordering::Relaxed), "Notifying for graceful shutdown" ); diff --git a/src/router/serve/plain.rs b/src/router/serve/plain.rs index 0e971f3c..535282b9 100644 --- a/src/router/serve/plain.rs +++ b/src/router/serve/plain.rs @@ -24,27 +24,20 @@ pub(super) async fn serve( info!("Listening on {addrs:?}"); while join_set.join_next().await.is_some() {} - let spawn_active = server.metrics.requests_spawn_active.load(Ordering::Relaxed); let handle_active = server .metrics .requests_handle_active .load(Ordering::Relaxed); debug_info!( - spawn_finished = server - .metrics - .requests_spawn_finished - .load(Ordering::Relaxed), handle_finished = server .metrics .requests_handle_finished .load(Ordering::Relaxed), panics = server.metrics.requests_panic.load(Ordering::Relaxed), - spawn_active, handle_active, "Stopped listening on {addrs:?}", ); - debug_assert!(spawn_active == 0, "active request tasks are not joined"); debug_assert!(handle_active == 0, "active request handles still pending"); Ok(()) diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 6855b34c..6a030c30 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -159,7 +159,12 @@ async fn fini(server: &Arc, listener: UnixListener, mut tasks: JoinSet<( drop(listener); debug!("Waiting for requests to finish..."); - while server.metrics.requests_spawn_active.load(Ordering::Relaxed) > 0 { + while server + .metrics + .requests_handle_active + .load(Ordering::Relaxed) + .gt(&0) + { tokio::select! { task = tasks.join_next() => if task.is_none() { break; }, () = sleep(FINI_POLL_INTERVAL) => {}, From f698254c412b5a142567f6b0ad710aa212c9b34d Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 31 Jan 2025 02:36:14 +0100 Subject: [PATCH 1458/2091] make registration tokens reloadable, and allow configuring multiple Signed-off-by: morguldir --- conduwuit-example.toml | 5 +++-- src/admin/room/alias.rs | 15 +++++++++------ src/core/config/mod.rs | 5 +++-- src/service/uiaa/mod.rs | 35 ++++++++++++++++++++++++++--------- 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 4062ba99..3fd95044 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -406,8 +406,9 @@ # #registration_token = -# Path to a file on the system that gets read for the registration token. -# this config option takes precedence/priority over "registration_token". +# Path to a file on the system that gets read for additional registration +# tokens. Multiple tokens can be added if you separate them with +# whitespace # # conduwuit must be able to access the file, and it must not be empty # diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 9710cfc8..d3b956e1 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -72,7 +72,7 @@ pub(super) async fn reprocess( ))), }; match command { - | RoomAliasCommand::Set { force, room_id, .. } => + | RoomAliasCommand::Set { force, room_id, .. } => { match (force, services.rooms.alias.resolve_local_alias(&room_alias).await) { | (true, Ok(id)) => { match services.rooms.alias.set_alias( @@ -106,8 +106,9 @@ pub(super) async fn reprocess( ))), } }, - }, - | RoomAliasCommand::Remove { .. } => + } + }, + | RoomAliasCommand::Remove { .. } => { match services.rooms.alias.resolve_local_alias(&room_alias).await { | Ok(id) => match services .rooms @@ -124,15 +125,17 @@ pub(super) async fn reprocess( }, | Err(_) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), - }, - | RoomAliasCommand::Which { .. } => + } + }, + | RoomAliasCommand::Which { .. } => { match services.rooms.alias.resolve_local_alias(&room_alias).await { | Ok(id) => Ok(RoomMessageEventContent::text_plain(format!( "Alias resolves to {id}" ))), | Err(_) => Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), - }, + } + }, | RoomAliasCommand::List { .. } => unreachable!(), } }, diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 415c9ba9..ff038975 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -510,8 +510,9 @@ pub struct Config { /// display: sensitive pub registration_token: Option, - /// Path to a file on the system that gets read for the registration token. - /// this config option takes precedence/priority over "registration_token". + /// Path to a file on the system that gets read for additional registration + /// tokens. Multiple tokens can be added if you separate them with + /// whitespace /// /// conduwuit must be able to access the file, and it must not be empty /// diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index f7e55251..7084f32a 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -1,5 +1,5 @@ use std::{ - collections::BTreeMap, + collections::{BTreeMap, HashSet}, sync::{Arc, RwLock}, }; @@ -17,7 +17,7 @@ use ruma::{ CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedUserId, UserId, }; -use crate::{globals, users, Dep}; +use crate::{config, globals, users, Dep}; pub struct Service { userdevicesessionid_uiaarequest: RwLock, @@ -28,6 +28,7 @@ pub struct Service { struct Services { globals: Dep, users: Dep, + config: Dep, } struct Data { @@ -49,6 +50,7 @@ impl crate::Service for Service { services: Services { globals: args.depend::("globals"), users: args.depend::("users"), + config: args.depend::("config"), }, })) } @@ -56,6 +58,26 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } +#[implement(Service)] +pub async fn read_tokens(&self) -> Result> { + let mut tokens = HashSet::new(); + if let Some(file) = &self.services.config.registration_token_file.as_ref() { + match std::fs::read_to_string(file) { + | Ok(text) => { + text.split_ascii_whitespace().for_each(|token| { + tokens.insert(token.to_owned()); + }); + }, + | Err(e) => error!("Failed to read the registration token file: {e}"), + } + }; + if let Some(token) = &self.services.config.registration_token { + tokens.insert(token.to_owned()); + } + + Ok(tokens) +} + /// Creates a new Uiaa session. Make sure the session token is unique. #[implement(Service)] pub fn create( @@ -152,13 +174,8 @@ pub async fn try_auth( uiaainfo.completed.push(AuthType::Password); }, | AuthData::RegistrationToken(t) => { - if self - .services - .globals - .registration_token - .as_ref() - .is_some_and(|reg_token| t.token.trim() == reg_token) - { + let tokens = self.read_tokens().await?; + if tokens.contains(t.token.trim()) { uiaainfo.completed.push(AuthType::RegistrationToken); } else { uiaainfo.auth_error = Some(ruma::api::client::error::StandardErrorBody { From e161e5dd61b006056ef35fbd034492130bffe150 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 Jan 2025 00:54:00 +0000 Subject: [PATCH 1459/2091] add pair_of! macro Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 1a4b52da..c2d8ed45 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -84,6 +84,17 @@ macro_rules! apply { }; } +#[macro_export] +macro_rules! pair_of { + ($decl:ty) => { + ($decl, $decl) + }; + + ($init:expr) => { + ($init, $init) + }; +} + /// Functor for truthy #[macro_export] macro_rules! is_true { From 4ff1155bf0aefddd02e34ed9c709db25c0da3ecd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 Jan 2025 01:23:27 +0000 Subject: [PATCH 1460/2091] reroll encrypted_room branch in incremental sync state Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 150 ++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 81 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index cd4dfc90..f5b612e4 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -6,7 +6,7 @@ use std::{ use axum::extract::State; use conduwuit::{ - at, err, error, extract_variant, is_equal_to, + at, err, error, extract_variant, is_equal_to, pair_of, pdu::EventHash, result::FlatOk, utils::{ @@ -16,7 +16,7 @@ use conduwuit::{ stream::{BroadbandExt, Tools, WidebandExt}, BoolExt, IterStream, ReadyExt, TryFutureExtExt, }, - Error, PduCount, PduEvent, Result, + PduCount, PduEvent, Result, }; use conduwuit_service::{ rooms::{ @@ -64,6 +64,8 @@ struct StateChanges { invited_member_count: Option, joined_since_last_sync: bool, state_events: Vec, + device_list_updates: HashSet, + left_encrypted_users: HashSet, } type PresenceUpdates = HashMap; @@ -325,18 +327,16 @@ pub(crate) async fn build_sync_events( // If the user doesn't share an encrypted room with the target anymore, we need // to tell them - let device_list_left = left_encrypted_users + let device_list_left: HashSet<_> = left_encrypted_users .into_iter() .stream() .broad_filter_map(|user_id| async move { - let no_shared_encrypted_room = - !share_encrypted_room(services, sender_user, &user_id, None).await; - no_shared_encrypted_room.then_some(user_id) - }) - .ready_fold(HashSet::new(), |mut device_list_left, user_id| { - device_list_left.insert(user_id); - device_list_left + share_encrypted_room(services, sender_user, &user_id, None) + .await + .eq(&false) + .then_some(user_id) }) + .collect() .await; let response = sync_events::v3::Response { @@ -730,14 +730,14 @@ async fn load_joined_room( .into(); let witness = witness.await; - let mut device_list_updates = HashSet::::new(); - let mut left_encrypted_users = HashSet::::new(); let StateChanges { heroes, joined_member_count, invited_member_count, joined_since_last_sync, state_events, + mut device_list_updates, + left_encrypted_users, } = if no_state_changes { StateChanges::default() } else { @@ -747,8 +747,6 @@ async fn load_joined_room( room_id, full_state, filter, - &mut device_list_updates, - &mut left_encrypted_users, since_shortstatehash, current_shortstatehash, joined_since_last_sync, @@ -919,8 +917,6 @@ async fn calculate_state_changes( room_id: &RoomId, full_state: bool, filter: &FilterDefinition, - device_list_updates: &mut HashSet, - left_encrypted_users: &mut HashSet, since_shortstatehash: Option, current_shortstatehash: ShortStateHash, joined_since_last_sync: bool, @@ -944,8 +940,6 @@ async fn calculate_state_changes( room_id, full_state, filter, - device_list_updates, - left_encrypted_users, since_shortstatehash, current_shortstatehash, joined_since_last_sync, @@ -1013,6 +1007,7 @@ async fn calculate_state_initial( invited_member_count, joined_since_last_sync: true, state_events, + ..Default::default() }) } @@ -1024,8 +1019,6 @@ async fn calculate_state_incremental( room_id: &RoomId, full_state: bool, _filter: &FilterDefinition, - device_list_updates: &mut HashSet, - left_encrypted_users: &mut HashSet, since_shortstatehash: Option, current_shortstatehash: ShortStateHash, joined_since_last_sync: bool, @@ -1063,79 +1056,72 @@ async fn calculate_state_incremental( .await; } - let encrypted_room = services - .rooms - .state_accessor - .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") - .is_ok(); - let since_encryption = services .rooms .state_accessor .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") .is_ok(); - let (encrypted_room, since_encryption) = join(encrypted_room, since_encryption).await; + let encrypted_room = services + .rooms + .state_accessor + .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") + .is_ok() + .await; - // Calculations: - let new_encrypted_room = encrypted_room && !since_encryption; + let (mut device_list_updates, left_encrypted_users) = delta_state_events + .iter() + .stream() + .ready_filter(|_| encrypted_room) + .ready_filter(|state_event| state_event.kind == RoomMember) + .ready_filter_map(|state_event| { + let content = state_event.get_content().ok()?; + let user_id = state_event.state_key.as_ref()?.parse().ok()?; + Some((content, user_id)) + }) + .ready_filter(|(_, user_id): &(RoomMemberEventContent, OwnedUserId)| { + user_id != sender_user + }) + .fold_default(|(mut dlu, mut leu): pair_of!(HashSet<_>), (content, user_id)| async move { + use MembershipState::*; + + let shares_encrypted_room = + |user_id| share_encrypted_room(services, sender_user, user_id, Some(room_id)); + + match content.membership { + | Join if !shares_encrypted_room(&user_id).await => dlu.insert(user_id), + | Leave => leu.insert(user_id), + | _ => false, + }; + + (dlu, leu) + }) + .await; + + // If the user is in a new encrypted room, give them all joined users + let new_encrypted_room = encrypted_room && !since_encryption.await; + if joined_since_last_sync && encrypted_room || new_encrypted_room { + services + .rooms + .state_cache + .room_members(room_id) + .ready_filter(|&user_id| sender_user != user_id) + .map(ToOwned::to_owned) + .broad_filter_map(|user_id| async move { + share_encrypted_room(services, sender_user, &user_id, Some(room_id)) + .await + .or_some(user_id) + }) + .ready_for_each(|user_id| { + device_list_updates.insert(user_id); + }) + .await; + } let send_member_count = delta_state_events .iter() .any(|event| event.kind == RoomMember); - if encrypted_room { - for state_event in &delta_state_events { - if state_event.kind != RoomMember { - continue; - } - - if let Some(state_key) = &state_event.state_key { - let user_id = UserId::parse(state_key) - .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?; - - if user_id == sender_user { - continue; - } - - let content: RoomMemberEventContent = state_event.get_content()?; - - match content.membership { - | MembershipState::Join => { - // A new user joined an encrypted room - if !share_encrypted_room(services, sender_user, user_id, Some(room_id)) - .await - { - device_list_updates.insert(user_id.into()); - } - }, - | MembershipState::Leave => { - // Write down users that have left encrypted rooms we are in - left_encrypted_users.insert(user_id.into()); - }, - | _ => {}, - } - } - } - } - - if joined_since_last_sync && encrypted_room || new_encrypted_room { - let updates: Vec = services - .rooms - .state_cache - .room_members(room_id) - .ready_filter(|user_id| sender_user != *user_id) - .filter_map(|user_id| { - share_encrypted_room(services, sender_user, user_id, Some(room_id)) - .map(|res| res.or_some(user_id.to_owned())) - }) - .collect() - .await; - - // If the user is in a new encrypted room, give them all joined users - device_list_updates.extend(updates); - } - let (joined_member_count, invited_member_count, heroes) = if send_member_count { calculate_counts(services, room_id, sender_user).await? } else { @@ -1148,6 +1134,8 @@ async fn calculate_state_incremental( invited_member_count, joined_since_last_sync, state_events: delta_state_events, + device_list_updates, + left_encrypted_users, }) } From 4e0cedbe5122c478e63e26b3f5156475629ada3e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 Jan 2025 05:05:32 +0000 Subject: [PATCH 1461/2091] simplify v3 sync presence collecting Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 55 +++++++-------------------------------- 1 file changed, 10 insertions(+), 45 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index f5b612e4..cd95fa42 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -1,6 +1,6 @@ use std::{ cmp::{self}, - collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, time::Duration, }; @@ -45,7 +45,7 @@ use ruma::{ uiaa::UiaaResponse, }, events::{ - presence::PresenceEvent, + presence::{PresenceEvent, PresenceEventContent}, room::member::{MembershipState, RoomMemberEventContent}, AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType::*, @@ -68,7 +68,7 @@ struct StateChanges { left_encrypted_users: HashSet, } -type PresenceUpdates = HashMap; +type PresenceUpdates = HashMap; /// # `GET /_matrix/client/r0/sync` /// @@ -351,9 +351,11 @@ pub(crate) async fn build_sync_events( next_batch: next_batch.to_string(), presence: Presence { events: presence_updates - .unwrap_or_default() - .into_values() - .map(|v| Raw::new(&v).expect("PresenceEvent always serializes successfully")) + .into_iter() + .flat_map(IntoIterator::into_iter) + .map(|(sender, content)| PresenceEvent { content, sender }) + .map(|ref event| Raw::new(event)) + .filter_map(Result::ok) .collect(), }, rooms: Rooms { @@ -390,45 +392,8 @@ async fn process_presence_updates( .map_ok(move |event| (user_id, event)) .ok() }) - .ready_fold(PresenceUpdates::new(), |mut updates, (user_id, event)| { - match updates.entry(user_id.into()) { - | Entry::Vacant(slot) => { - let mut new_event = event; - new_event.content.last_active_ago = match new_event.content.currently_active { - | Some(true) => None, - | _ => new_event.content.last_active_ago, - }; - - slot.insert(new_event); - }, - | Entry::Occupied(mut slot) => { - let curr_event = slot.get_mut(); - let curr_content = &mut curr_event.content; - let new_content = event.content; - - // Update existing presence event with more info - curr_content.presence = new_content.presence; - curr_content.status_msg = new_content - .status_msg - .or_else(|| curr_content.status_msg.take()); - curr_content.displayname = new_content - .displayname - .or_else(|| curr_content.displayname.take()); - curr_content.avatar_url = new_content - .avatar_url - .or_else(|| curr_content.avatar_url.take()); - curr_content.currently_active = new_content - .currently_active - .or(curr_content.currently_active); - curr_content.last_active_ago = match curr_content.currently_active { - | Some(true) => None, - | _ => new_content.last_active_ago.or(curr_content.last_active_ago), - }; - }, - }; - - updates - }) + .map(|(user_id, event)| (user_id.to_owned(), event.content)) + .collect() .await } From a4ef04cd1427ea2eeb474775e7c4c86937d063ab Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 Jan 2025 08:31:58 +0000 Subject: [PATCH 1462/2091] fix room join completion taking wrong sync branch Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index cd95fa42..e3f559f5 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -887,7 +887,7 @@ async fn calculate_state_changes( joined_since_last_sync: bool, witness: Option<&Witness>, ) -> Result { - if since_shortstatehash.is_none() || joined_since_last_sync { + if since_shortstatehash.is_none() { calculate_state_initial( services, sender_user, From 6983798487ec563be83a3ba8739afa9977d98741 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 Jan 2025 08:34:32 +0000 Subject: [PATCH 1463/2091] implement lazy-loading for incremental sync Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 182 ++++++++++++++++++++------------------ 1 file changed, 98 insertions(+), 84 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index e3f559f5..49246514 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -55,7 +55,10 @@ use ruma::{ }; use super::{load_timeline, share_encrypted_room}; -use crate::{client::ignored_filter, Ruma, RumaResponse}; +use crate::{ + client::{ignored_filter, lazy_loading_witness}, + Ruma, RumaResponse, +}; #[derive(Default)] struct StateChanges { @@ -633,10 +636,6 @@ async fn load_joined_room( }) .into(); - let no_state_changes = timeline_pdus.is_empty() - && (since_shortstatehash.is_none() - || since_shortstatehash.is_some_and(is_equal_to!(current_shortstatehash))); - let since_sender_member: OptionFuture<_> = since_shortstatehash .map(|short| { services @@ -658,11 +657,7 @@ async fn load_joined_room( let lazy_loading_enabled = filter.room.state.lazy_load_options.is_enabled() || filter.room.timeline.lazy_load_options.is_enabled(); - let generate_witness = - lazy_loading_enabled && (since_shortstatehash.is_none() || joined_since_last_sync); - - let lazy_reset = lazy_loading_enabled && since_shortstatehash.is_none(); - + let lazy_reset = since_shortstatehash.is_none(); let lazy_loading_context = &lazy_loading::Context { user_id: sender_user, device_id: sender_device, @@ -677,24 +672,10 @@ async fn load_joined_room( .into(); lazy_load_reset.await; - let witness: Option = generate_witness.then(|| { - timeline_pdus - .iter() - .map(|(_, pdu)| pdu.sender.clone()) - .chain(receipt_events.keys().cloned()) - .collect() - }); - - let witness: OptionFuture<_> = witness - .map(|witness| { - services - .rooms - .lazy_loading - .witness_retain(witness, lazy_loading_context) - }) + let witness: OptionFuture<_> = lazy_loading_enabled + .then(|| lazy_loading_witness(services, lazy_loading_context, timeline_pdus.iter())) .into(); - let witness = witness.await; let StateChanges { heroes, joined_member_count, @@ -703,23 +684,19 @@ async fn load_joined_room( state_events, mut device_list_updates, left_encrypted_users, - } = if no_state_changes { - StateChanges::default() - } else { - calculate_state_changes( - services, - sender_user, - room_id, - full_state, - filter, - since_shortstatehash, - current_shortstatehash, - joined_since_last_sync, - witness.as_ref(), - ) - .boxed() - .await? - }; + } = calculate_state_changes( + services, + sender_user, + room_id, + full_state, + filter, + since_shortstatehash, + current_shortstatehash, + joined_since_last_sync, + witness.await.as_ref(), + ) + .boxed() + .await?; let account_data_events = services .account_data @@ -908,6 +885,7 @@ async fn calculate_state_changes( since_shortstatehash, current_shortstatehash, joined_since_last_sync, + witness, ) .await } @@ -920,7 +898,7 @@ async fn calculate_state_initial( sender_user: &UserId, room_id: &RoomId, full_state: bool, - filter: &FilterDefinition, + _filter: &FilterDefinition, current_shortstatehash: ShortStateHash, witness: Option<&Witness>, ) -> Result { @@ -938,20 +916,14 @@ async fn calculate_state_initial( .zip(event_ids.into_iter().stream()) .ready_filter_map(|item| Some((item.0.ok()?, item.1))) .ready_filter_map(|((event_type, state_key), event_id)| { - let lazy_load_enabled = filter.room.state.lazy_load_options.is_enabled() - || filter.room.timeline.lazy_load_options.is_enabled(); - - if lazy_load_enabled + let lazy = !full_state && event_type == StateEventType::RoomMember - && !full_state && state_key.as_str().try_into().is_ok_and(|user_id: &UserId| { sender_user != user_id && witness.is_some_and(|witness| !witness.contains(user_id)) - }) { - return None; - } + }); - Some(event_id) + lazy.or_some(event_id) }) .broad_filter_map(|event_id: OwnedEventId| async move { services.rooms.timeline.get_pdu(&event_id).await.ok() @@ -978,7 +950,7 @@ async fn calculate_state_initial( #[tracing::instrument(name = "incremental", level = "trace", skip_all)] #[allow(clippy::too_many_arguments)] -async fn calculate_state_incremental( +async fn calculate_state_incremental<'a>( services: &Services, sender_user: &UserId, room_id: &RoomId, @@ -987,39 +959,80 @@ async fn calculate_state_incremental( since_shortstatehash: Option, current_shortstatehash: ShortStateHash, joined_since_last_sync: bool, + witness: Option<&'a Witness>, ) -> Result { - // Incremental /sync - let since_shortstatehash = - since_shortstatehash.expect("missing since_shortstatehash on incremental sync"); + let since_shortstatehash = since_shortstatehash.unwrap_or(current_shortstatehash); - let mut delta_state_events = Vec::new(); + let state_changed = since_shortstatehash != current_shortstatehash; - if since_shortstatehash != current_shortstatehash { - let current_state_ids = services + let state_get_id = |user_id: &'a UserId| { + services .rooms .state_accessor - .state_full_ids(current_shortstatehash) - .collect(); + .state_get_id(current_shortstatehash, &StateEventType::RoomMember, user_id.as_str()) + .ok() + }; - let since_state_ids = services - .rooms - .state_accessor - .state_full_ids(since_shortstatehash) - .collect(); + let lazy_state_ids: OptionFuture<_> = witness + .map(|witness| { + witness + .iter() + .stream() + .broad_filter_map(|user_id| state_get_id(user_id)) + .collect::>() + }) + .into(); - let (current_state_ids, since_state_ids): ( - HashMap<_, OwnedEventId>, - HashMap<_, OwnedEventId>, - ) = join(current_state_ids, since_state_ids).await; + let current_state_ids: OptionFuture<_> = state_changed + .then(|| { + services + .rooms + .state_accessor + .state_full_ids(current_shortstatehash) + .collect::>() + }) + .into(); - current_state_ids - .iter() - .stream() - .ready_filter(|(key, id)| full_state || since_state_ids.get(key) != Some(id)) - .wide_filter_map(|(_, id)| services.rooms.timeline.get_pdu(id).ok()) - .ready_for_each(|pdu| delta_state_events.push(pdu)) - .await; - } + let since_state_ids: OptionFuture<_> = (state_changed && !full_state) + .then(|| { + services + .rooms + .state_accessor + .state_full_ids(since_shortstatehash) + .collect::>() + }) + .into(); + + let lazy_state_ids = lazy_state_ids + .map(Option::into_iter) + .map(|iter| iter.flat_map(Vec::into_iter)) + .map(IterStream::stream) + .flatten_stream(); + + let ref since_state_ids = since_state_ids.shared(); + let delta_state_events = current_state_ids + .map(Option::into_iter) + .map(|iter| iter.flat_map(Vec::into_iter)) + .map(IterStream::stream) + .flatten_stream() + .filter_map(|(shortstatekey, event_id): (u64, OwnedEventId)| async move { + since_state_ids + .clone() + .await + .is_none_or(|since_state| since_state.get(&shortstatekey) != Some(&event_id)) + .then_some(event_id) + }) + .chain(lazy_state_ids) + .broad_filter_map(|event_id: OwnedEventId| async move { + services + .rooms + .timeline + .get_pdu(&event_id) + .await + .map(move |pdu| (event_id, pdu)) + .ok() + }) + .collect::>(); let since_encryption = services .rooms @@ -1031,11 +1044,12 @@ async fn calculate_state_incremental( .rooms .state_accessor .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") - .is_ok() - .await; + .is_ok(); + + let (delta_state_events, encrypted_room) = join(delta_state_events, encrypted_room).await; let (mut device_list_updates, left_encrypted_users) = delta_state_events - .iter() + .values() .stream() .ready_filter(|_| encrypted_room) .ready_filter(|state_event| state_event.kind == RoomMember) @@ -1084,7 +1098,7 @@ async fn calculate_state_incremental( } let send_member_count = delta_state_events - .iter() + .values() .any(|event| event.kind == RoomMember); let (joined_member_count, invited_member_count, heroes) = if send_member_count { @@ -1098,9 +1112,9 @@ async fn calculate_state_incremental( joined_member_count, invited_member_count, joined_since_last_sync, - state_events: delta_state_events, device_list_updates, left_encrypted_users, + state_events: delta_state_events.into_values().collect(), }) } From 09bc71caaba40321ec0f987574a94e788175c4f9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 Jan 2025 09:08:13 +0000 Subject: [PATCH 1464/2091] fix missed concurrent fetch opportunities in sender (ffd0fd42424a) Signed-off-by: Jason Volk --- src/service/sending/sender.rs | 41 +++++++++++++++++++++-------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 363bb994..f19b69da 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -13,7 +13,12 @@ use conduwuit::{ debug, err, error, result::LogErr, trace, - utils::{calculate_hash, continue_exponential_backoff_secs, stream::IterStream, ReadyExt}, + utils::{ + calculate_hash, continue_exponential_backoff_secs, + future::TryExtExt, + stream::{BroadbandExt, IterStream, WidebandExt}, + ReadyExt, + }, warn, Error, Result, }; use futures::{ @@ -474,20 +479,25 @@ impl Service { since: (u64, u64), max_edu_count: &AtomicU64, ) -> Option { - let server_rooms = self.services.state_cache.server_rooms(server_name); - - pin_mut!(server_rooms); let mut num = 0; - let mut receipts = BTreeMap::::new(); - while let Some(room_id) = server_rooms.next().await { - let receipt_map = self - .select_edus_receipts_room(room_id, since, max_edu_count, &mut num) - .await; + let receipts: BTreeMap = self + .services + .state_cache + .server_rooms(server_name) + .map(ToOwned::to_owned) + .broad_filter_map(|room_id| async move { + let receipt_map = self + .select_edus_receipts_room(&room_id, since, max_edu_count, &mut num) + .await; - if !receipt_map.read.is_empty() { - receipts.insert(room_id.into(), receipt_map); - } - } + receipt_map + .read + .is_empty() + .eq(&false) + .then_some((room_id, receipt_map)) + }) + .collect() + .await; if receipts.is_empty() { return None; @@ -820,9 +830,8 @@ impl Service { | _ => None, }) .stream() - .then(|pdu_id| self.services.timeline.get_pdu_json_from_id(pdu_id)) - .ready_filter_map(Result::ok) - .then(|pdu| self.convert_to_outgoing_federation_event(pdu)) + .wide_filter_map(|pdu_id| self.services.timeline.get_pdu_json_from_id(pdu_id).ok()) + .wide_then(|pdu| self.convert_to_outgoing_federation_event(pdu)) .collect() .await; From 2fa9621f3a358740917af7a55c5d0be1e1d79ae4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 Jan 2025 11:54:06 +0000 Subject: [PATCH 1465/2091] flatten state_full_shortids Signed-off-by: Jason Volk --- src/service/rooms/state/mod.rs | 60 +++++++++++-------------- src/service/rooms/state_accessor/mod.rs | 47 +++++++++---------- 2 files changed, 48 insertions(+), 59 deletions(-) diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 8cb4e586..1b0d0d58 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -429,60 +429,54 @@ impl Service { sender: &UserId, state_key: Option<&str>, content: &serde_json::value::RawValue, - ) -> Result>> { + ) -> Result> { let Ok(shortstatehash) = self.get_room_shortstatehash(room_id).await else { return Ok(HashMap::new()); }; - let mut sauthevents: HashMap<_, _> = - state_res::auth_types_for_event(kind, sender, state_key, content)? - .iter() - .stream() - .broad_filter_map(|(event_type, state_key)| { - self.services - .short - .get_shortstatekey(event_type, state_key) - .map_ok(move |ssk| (ssk, (event_type, state_key))) - .map(Result::ok) - }) - .map(|(ssk, (event_type, state_key))| { - (ssk, (event_type.to_owned(), state_key.to_owned())) - }) - .collect() - .await; + let auth_types = state_res::auth_types_for_event(kind, sender, state_key, content)?; + + let sauthevents: HashMap<_, _> = auth_types + .iter() + .stream() + .broad_filter_map(|(event_type, state_key)| { + self.services + .short + .get_shortstatekey(event_type, state_key) + .map_ok(move |ssk| (ssk, (event_type, state_key))) + .map(Result::ok) + }) + .collect() + .await; let (state_keys, event_ids): (Vec<_>, Vec<_>) = self .services .state_accessor .state_full_shortids(shortstatehash) - .await - .map_err(|e| err!(Database(error!(?room_id, ?shortstatehash, "{e:?}"))))? - .into_iter() - .filter_map(|(shortstatekey, shorteventid)| { + .ready_filter_map(Result::ok) + .ready_filter_map(|(shortstatekey, shorteventid)| { sauthevents - .remove(&shortstatekey) - .map(|(event_type, state_key)| ((event_type, state_key), shorteventid)) + .get(&shortstatekey) + .map(|(ty, sk)| ((ty, sk), shorteventid)) }) - .unzip(); + .unzip() + .await; - let auth_pdus = self - .services + self.services .short .multi_get_eventid_from_short(event_ids.into_iter().stream()) .zip(state_keys.into_iter().stream()) - .ready_filter_map(|(event_id, tsk)| Some((tsk, event_id.ok()?))) - .broad_filter_map(|(tsk, event_id): (_, OwnedEventId)| async move { + .ready_filter_map(|(event_id, (ty, sk))| Some(((ty, sk), event_id.ok()?))) + .broad_filter_map(|((ty, sk), event_id): (_, OwnedEventId)| async move { self.services .timeline .get_pdu(&event_id) .await - .map(Arc::new) - .map(move |pdu| (tsk, pdu)) + .map(move |pdu| (((*ty).clone(), (*sk).clone()), pdu)) .ok() }) .collect() - .await; - - Ok(auth_pdus) + .map(Ok) + .await } } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 0f5520bb..98aac138 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -1,6 +1,7 @@ use std::{ borrow::Borrow, fmt::Write, + ops::Deref, sync::{Arc, Mutex as StdMutex, Mutex}, }; @@ -10,8 +11,7 @@ use conduwuit::{ utils, utils::{ math::{usize_from_f64, Expected}, - stream::BroadbandExt, - IterStream, ReadyExt, + stream::{BroadbandExt, IterStream, ReadyExt, TryExpect}, }, Err, Error, PduEvent, Result, }; @@ -158,12 +158,8 @@ impl Service { ) -> impl Stream + Send + '_ { let short_ids = self .state_full_shortids(shortstatehash) - .map(|result| result.expect("missing shortstatehash")) - .map(Vec::into_iter) - .map(|iter| iter.map(at!(1))) - .map(IterStream::stream) - .flatten_stream() - .boxed(); + .expect_ok() + .map(at!(1)); self.services .short @@ -187,9 +183,8 @@ impl Service { { let shortids = self .state_full_shortids(shortstatehash) - .map(|result| result.expect("missing shortstatehash")) - .map(|vec| vec.into_iter().unzip()) - .boxed() + .expect_ok() + .unzip() .shared(); let shortstatekeys = shortids @@ -255,25 +250,25 @@ impl Service { } #[inline] - pub async fn state_full_shortids( + pub fn state_full_shortids( &self, shortstatehash: ShortStateHash, - ) -> Result> { - let shortids = self - .services + ) -> impl Stream> + Send + '_ { + self.services .state_compressor .load_shortstatehash_info(shortstatehash) - .await - .map_err(|e| err!(Database("Missing state IDs: {e}")))? - .pop() - .expect("there is always one layer") - .full_state - .iter() - .copied() - .map(parse_compressed_state_event) - .collect(); - - Ok(shortids) + .map_err(|e| err!(Database("Missing state IDs: {e}"))) + .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) + .map_ok(|full_state| { + full_state + .deref() + .iter() + .copied() + .map(parse_compressed_state_event) + .collect() + }) + .map_ok(|vec: Vec<_>| vec.into_iter().try_stream()) + .try_flatten_stream() } /// Returns a single PDU from `room_id` with key (`event_type`, From ea49b60273c987cc673c3aad439c6fbb50bb795f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 1 Feb 2025 22:28:09 +0000 Subject: [PATCH 1466/2091] add Option support to database deserializer Signed-off-by: Jason Volk --- src/database/de.rs | 23 ++++-- src/database/tests.rs | 159 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 176 insertions(+), 6 deletions(-) diff --git a/src/database/de.rs b/src/database/de.rs index 7cc8f00a..8e914fcc 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -22,7 +22,7 @@ pub(crate) fn from_slice<'a, T>(buf: &'a [u8]) -> Result where T: Deserialize<'a>, { - let mut deserializer = Deserializer { buf, pos: 0, seq: false }; + let mut deserializer = Deserializer { buf, pos: 0, rec: 0, seq: false }; T::deserialize(&mut deserializer).debug_inspect(|_| { deserializer @@ -35,6 +35,7 @@ where pub(crate) struct Deserializer<'de> { buf: &'de [u8], pos: usize, + rec: usize, seq: bool, } @@ -107,7 +108,7 @@ impl<'de> Deserializer<'de> { /// consumed None is returned instead. #[inline] fn record_peek_byte(&self) -> Option { - let started = self.pos != 0; + let started = self.pos != 0 || self.rec > 0; let buf = &self.buf[self.pos..]; debug_assert!( !started || buf[0] == Self::SEP, @@ -121,13 +122,14 @@ impl<'de> Deserializer<'de> { /// the start of the next record. (Case for some sequences) #[inline] fn record_start(&mut self) { - let started = self.pos != 0; + let started = self.pos != 0 || self.rec > 0; debug_assert!( !started || self.buf[self.pos] == Self::SEP, "Missing expected record separator at current position" ); self.inc_pos(started.into()); + self.inc_rec(1); } /// Consume all remaining bytes, which may include record separators, @@ -157,6 +159,9 @@ impl<'de> Deserializer<'de> { debug_assert!(self.pos <= self.buf.len(), "pos out of range"); } + #[inline] + fn inc_rec(&mut self, n: usize) { self.rec = self.rec.saturating_add(n); } + /// Unconsumed input bytes. #[inline] fn remaining(&self) -> Result { @@ -270,8 +275,16 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { } #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] - fn deserialize_option>(self, _visitor: V) -> Result { - unhandled!("deserialize Option not implemented") + fn deserialize_option>(self, visitor: V) -> Result { + if self + .buf + .get(self.pos) + .is_none_or(|b| *b == Deserializer::SEP) + { + visitor.visit_none() + } else { + visitor.visit_some(self) + } } #[cfg_attr(unabridged, tracing::instrument(level = "trace", skip_all))] diff --git a/src/database/tests.rs b/src/database/tests.rs index 2f143698..e6c85983 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -3,7 +3,7 @@ use std::fmt::Debug; use arrayvec::ArrayVec; -use conduwuit::ruma::{serde::Raw, RoomId, UserId}; +use conduwuit::ruma::{serde::Raw, EventId, RoomId, UserId}; use serde::Serialize; use crate::{ @@ -389,3 +389,160 @@ fn de_complex() { assert_eq!(arr, key, "deserialization of serialization does not match"); } + +#[test] +fn serde_tuple_option_value_some() { + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + + let mut aa = Vec::::new(); + aa.extend_from_slice(room_id.as_bytes()); + aa.push(0xFF); + aa.extend_from_slice(user_id.as_bytes()); + + let bb: (&RoomId, Option<&UserId>) = (room_id, Some(user_id)); + let bbs = serialize_to_vec(&bb).expect("failed to serialize tuple"); + assert_eq!(aa, bbs); + + let cc: (&RoomId, Option<&UserId>) = + de::from_slice(&bbs).expect("failed to deserialize tuple"); + + assert_eq!(bb.1, cc.1); + assert_eq!(cc.0, bb.0); +} + +#[test] +fn serde_tuple_option_value_none() { + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + + let mut aa = Vec::::new(); + aa.extend_from_slice(room_id.as_bytes()); + aa.push(0xFF); + + let bb: (&RoomId, Option<&UserId>) = (room_id, None); + let bbs = serialize_to_vec(&bb).expect("failed to serialize tuple"); + assert_eq!(aa, bbs); + + let cc: (&RoomId, Option<&UserId>) = + de::from_slice(&bbs).expect("failed to deserialize tuple"); + + assert_eq!(None, cc.1); + assert_eq!(cc.0, bb.0); +} + +#[test] +fn serde_tuple_option_none_value() { + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + + let mut aa = Vec::::new(); + aa.push(0xFF); + aa.extend_from_slice(user_id.as_bytes()); + + let bb: (Option<&RoomId>, &UserId) = (None, user_id); + let bbs = serialize_to_vec(&bb).expect("failed to serialize tuple"); + assert_eq!(aa, bbs); + + let cc: (Option<&RoomId>, &UserId) = + de::from_slice(&bbs).expect("failed to deserialize tuple"); + + assert_eq!(None, cc.0); + assert_eq!(cc.1, bb.1); +} + +#[test] +fn serde_tuple_option_some_value() { + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + + let mut aa = Vec::::new(); + aa.extend_from_slice(room_id.as_bytes()); + aa.push(0xFF); + aa.extend_from_slice(user_id.as_bytes()); + + let bb: (Option<&RoomId>, &UserId) = (Some(room_id), user_id); + let bbs = serialize_to_vec(&bb).expect("failed to serialize tuple"); + assert_eq!(aa, bbs); + + let cc: (Option<&RoomId>, &UserId) = + de::from_slice(&bbs).expect("failed to deserialize tuple"); + + assert_eq!(bb.0, cc.0); + assert_eq!(cc.1, bb.1); +} + +#[test] +fn serde_tuple_option_some_some() { + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + + let mut aa = Vec::::new(); + aa.extend_from_slice(room_id.as_bytes()); + aa.push(0xFF); + aa.extend_from_slice(user_id.as_bytes()); + + let bb: (Option<&RoomId>, Option<&UserId>) = (Some(room_id), Some(user_id)); + let bbs = serialize_to_vec(&bb).expect("failed to serialize tuple"); + assert_eq!(aa, bbs); + + let cc: (Option<&RoomId>, Option<&UserId>) = + de::from_slice(&bbs).expect("failed to deserialize tuple"); + + assert_eq!(cc.0, bb.0); + assert_eq!(bb.1, cc.1); +} + +#[test] +fn serde_tuple_option_none_none() { + let aa = vec![0xFF]; + + let bb: (Option<&RoomId>, Option<&UserId>) = (None, None); + let bbs = serialize_to_vec(&bb).expect("failed to serialize tuple"); + assert_eq!(aa, bbs); + + let cc: (Option<&RoomId>, Option<&UserId>) = + de::from_slice(&bbs).expect("failed to deserialize tuple"); + + assert_eq!(cc.0, bb.0); + assert_eq!(None, cc.1); +} + +#[test] +fn serde_tuple_option_some_none_some() { + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + + let mut aa = Vec::::new(); + aa.extend_from_slice(room_id.as_bytes()); + aa.push(0xFF); + aa.push(0xFF); + aa.extend_from_slice(user_id.as_bytes()); + + let bb: (Option<&RoomId>, Option<&EventId>, Option<&UserId>) = + (Some(room_id), None, Some(user_id)); + + let bbs = serialize_to_vec(&bb).expect("failed to serialize tuple"); + assert_eq!(aa, bbs); + + let cc: (Option<&RoomId>, Option<&EventId>, Option<&UserId>) = + de::from_slice(&bbs).expect("failed to deserialize tuple"); + + assert_eq!(bb.0, cc.0); + assert_eq!(None, cc.1); + assert_eq!(bb.1, cc.1); + assert_eq!(bb.2, cc.2); +} + +#[test] +fn serde_tuple_option_none_none_none() { + let aa = vec![0xFF, 0xFF]; + + let bb: (Option<&RoomId>, Option<&EventId>, Option<&UserId>) = (None, None, None); + let bbs = serialize_to_vec(&bb).expect("failed to serialize tuple"); + assert_eq!(aa, bbs); + + let cc: (Option<&RoomId>, Option<&EventId>, Option<&UserId>) = + de::from_slice(&bbs).expect("failed to deserialize tuple"); + + assert_eq!(None, cc.0); + assert_eq!(bb, cc); +} From 4add39d0fedcbe7946c6dfffac33d1e48111ea8b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 Jan 2025 15:50:09 +0000 Subject: [PATCH 1467/2091] cache compressed state in a sorted structure for logarithmic queries with partial keys Signed-off-by: Jason Volk --- src/api/client/membership.rs | 9 +- .../rooms/event_handler/resolve_state.rs | 6 +- .../event_handler/upgrade_outlier_pdu.rs | 15 ++- src/service/rooms/state/mod.rs | 28 +++--- src/service/rooms/state_accessor/mod.rs | 99 ++++++++++++++----- src/service/rooms/state_compressor/mod.rs | 30 +++--- src/service/rooms/timeline/mod.rs | 4 +- 7 files changed, 118 insertions(+), 73 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index d80aff0c..449d44d5 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -46,7 +46,10 @@ use ruma::{ use service::{ appservice::RegistrationInfo, pdu::gen_event_id, - rooms::{state::RoomMutexGuard, state_compressor::HashSetCompressStateEvent}, + rooms::{ + state::RoomMutexGuard, + state_compressor::{CompressedState, HashSetCompressStateEvent}, + }, Services, }; @@ -1169,7 +1172,7 @@ async fn join_room_by_id_helper_remote( } info!("Compressing state from send_join"); - let compressed: HashSet<_> = services + let compressed: CompressedState = services .rooms .state_compressor .compress_state_events(state.iter().map(|(ssk, eid)| (ssk, eid.borrow()))) @@ -2340,7 +2343,7 @@ async fn knock_room_helper_remote( } info!("Compressing state from send_knock"); - let compressed: HashSet<_> = services + let compressed: CompressedState = services .rooms .state_compressor .compress_state_events(state_map.iter().map(|(ssk, eid)| (ssk, eid.borrow()))) diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index c3de5f2f..4d99b088 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -15,7 +15,7 @@ use ruma::{ OwnedEventId, RoomId, RoomVersionId, }; -use crate::rooms::state_compressor::CompressedStateEvent; +use crate::rooms::state_compressor::CompressedState; #[implement(super::Service)] #[tracing::instrument(name = "resolve", level = "debug", skip_all)] @@ -24,7 +24,7 @@ pub async fn resolve_state( room_id: &RoomId, room_version_id: &RoomVersionId, incoming_state: HashMap, -) -> Result>> { +) -> Result> { trace!("Loading current room state ids"); let current_sstatehash = self .services @@ -91,7 +91,7 @@ pub async fn resolve_state( .await; trace!("Compressing state..."); - let new_room_state: HashSet<_> = self + let new_room_state: CompressedState = self .services .state_compressor .compress_state_events( diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 03697558..132daca7 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -1,10 +1,4 @@ -use std::{ - borrow::Borrow, - collections::{BTreeMap, HashSet}, - iter::once, - sync::Arc, - time::Instant, -}; +use std::{borrow::Borrow, collections::BTreeMap, iter::once, sync::Arc, time::Instant}; use conduwuit::{ debug, debug_info, err, implement, trace, @@ -19,7 +13,10 @@ use ruma::{ }; use super::{get_room_version_id, to_room_version}; -use crate::rooms::{state_compressor::HashSetCompressStateEvent, timeline::RawPduId}; +use crate::rooms::{ + state_compressor::{CompressedState, HashSetCompressStateEvent}, + timeline::RawPduId, +}; #[implement(super::Service)] pub(super) async fn upgrade_outlier_to_timeline_pdu( @@ -173,7 +170,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( incoming_pdu.prev_events.len() ); - let state_ids_compressed: Arc> = self + let state_ids_compressed: Arc = self .services .state_compressor .compress_state_events( diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 1b0d0d58..de90a89c 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -1,9 +1,4 @@ -use std::{ - collections::{HashMap, HashSet}, - fmt::Write, - iter::once, - sync::Arc, -}; +use std::{collections::HashMap, fmt::Write, iter::once, sync::Arc}; use conduwuit::{ err, @@ -33,7 +28,7 @@ use crate::{ globals, rooms, rooms::{ short::{ShortEventId, ShortStateHash}, - state_compressor::{parse_compressed_state_event, CompressedStateEvent}, + state_compressor::{parse_compressed_state_event, CompressedState}, }, Dep, }; @@ -102,10 +97,9 @@ impl Service { &self, room_id: &RoomId, shortstatehash: u64, - statediffnew: Arc>, - _statediffremoved: Arc>, - state_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room state - * mutex */ + statediffnew: Arc, + _statediffremoved: Arc, + state_lock: &RoomMutexGuard, ) -> Result { let event_ids = statediffnew .iter() @@ -176,7 +170,7 @@ impl Service { &self, event_id: &EventId, room_id: &RoomId, - state_ids_compressed: Arc>, + state_ids_compressed: Arc, ) -> Result { const KEY_LEN: usize = size_of::(); const VAL_LEN: usize = size_of::(); @@ -209,12 +203,12 @@ impl Service { let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew: HashSet<_> = state_ids_compressed + let statediffnew: CompressedState = state_ids_compressed .difference(&parent_stateinfo.full_state) .copied() .collect(); - let statediffremoved: HashSet<_> = parent_stateinfo + let statediffremoved: CompressedState = parent_stateinfo .full_state .difference(&state_ids_compressed) .copied() @@ -222,7 +216,7 @@ impl Service { (Arc::new(statediffnew), Arc::new(statediffremoved)) } else { - (state_ids_compressed, Arc::new(HashSet::new())) + (state_ids_compressed, Arc::new(CompressedState::new())) }; self.services.state_compressor.save_state_from_diff( shortstatehash, @@ -300,10 +294,10 @@ impl Service { // TODO: statehash with deterministic inputs let shortstatehash = self.services.globals.next_count()?; - let mut statediffnew = HashSet::new(); + let mut statediffnew = CompressedState::new(); statediffnew.insert(new); - let mut statediffremoved = HashSet::new(); + let mut statediffremoved = CompressedState::new(); if let Some(replaces) = replaces { statediffremoved.insert(*replaces); } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 98aac138..8b56c8b6 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -11,6 +11,7 @@ use conduwuit::{ utils, utils::{ math::{usize_from_f64, Expected}, + result::FlatOk, stream::{BroadbandExt, IterStream, ReadyExt, TryExpect}, }, Err, Error, PduEvent, Result, @@ -47,7 +48,7 @@ use crate::{ rooms::{ short::{ShortEventId, ShortStateHash, ShortStateKey}, state::RoomMutexGuard, - state_compressor::parse_compressed_state_event, + state_compressor::{compress_state_event, parse_compressed_state_event}, }, Dep, }; @@ -220,36 +221,88 @@ impl Service { Id: for<'de> Deserialize<'de> + Sized + ToOwned, ::Owned: Borrow, { - let shortstatekey = self - .services - .short - .get_shortstatekey(event_type, state_key) + let shorteventid = self + .state_get_shortid(shortstatehash, event_type, state_key) .await?; - let full_state = self - .services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .await - .map_err(|e| err!(Database(error!(?event_type, ?state_key, "Missing state: {e:?}"))))? - .pop() - .expect("there is always one layer") - .full_state; - - let compressed = full_state - .iter() - .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) - .ok_or(err!(Database("No shortstatekey in compressed state")))?; - - let (_, shorteventid) = parse_compressed_state_event(*compressed); - self.services .short .get_eventid_from_short(shorteventid) .await } - #[inline] + /// Returns a single EventId from `room_id` with key (`event_type`, + /// `state_key`). + #[tracing::instrument(skip(self), level = "debug")] + pub async fn state_get_shortid( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> Result { + let shortstatekey = self + .services + .short + .get_shortstatekey(event_type, state_key) + .await?; + + let start = compress_state_event(shortstatekey, 0); + let end = compress_state_event(shortstatekey, u64::MAX); + self.services + .state_compressor + .load_shortstatehash_info(shortstatehash) + .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) + .map_ok(|full_state| { + full_state + .range(start..end) + .next() + .copied() + .map(parse_compressed_state_event) + .map(at!(1)) + .ok_or(err!(Request(NotFound("Not found in room state")))) + }) + .await? + } + + #[tracing::instrument(skip(self), level = "debug")] + pub async fn state_contains( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> bool { + let Ok(shortstatekey) = self + .services + .short + .get_shortstatekey(event_type, state_key) + .await + else { + return false; + }; + + self.state_contains_shortstatekey(shortstatehash, shortstatekey) + .await + } + + #[tracing::instrument(skip(self), level = "debug")] + pub async fn state_contains_shortstatekey( + &self, + shortstatehash: ShortStateHash, + shortstatekey: ShortStateKey, + ) -> bool { + let start = compress_state_event(shortstatekey, 0); + let end = compress_state_event(shortstatekey, u64::MAX); + + self.services + .state_compressor + .load_shortstatehash_info(shortstatehash) + .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) + .map_ok(|full_state| full_state.range(start..end).next().copied()) + .await + .flat_ok() + .is_some() + } + pub fn state_full_shortids( &self, shortstatehash: ShortStateHash, diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 532df360..3d68dff6 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -1,5 +1,5 @@ use std::{ - collections::{HashMap, HashSet}, + collections::{BTreeSet, HashMap}, fmt::{Debug, Write}, mem::size_of, sync::{Arc, Mutex}, @@ -63,8 +63,8 @@ type StateInfoLruCache = LruCache; type ShortStateInfoVec = Vec; type ParentStatesVec = Vec; -pub(crate) type CompressedState = HashSet; -pub(crate) type CompressedStateEvent = [u8; 2 * size_of::()]; +pub type CompressedState = BTreeSet; +pub type CompressedStateEvent = [u8; 2 * size_of::()]; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -249,8 +249,8 @@ impl Service { pub fn save_state_from_diff( &self, shortstatehash: ShortStateHash, - statediffnew: Arc>, - statediffremoved: Arc>, + statediffnew: Arc, + statediffremoved: Arc, diff_to_sibling: usize, mut parent_states: ParentStatesVec, ) -> Result { @@ -363,7 +363,7 @@ impl Service { pub async fn save_state( &self, room_id: &RoomId, - new_state_ids_compressed: Arc>, + new_state_ids_compressed: Arc, ) -> Result { let previous_shortstatehash = self .services @@ -396,12 +396,12 @@ impl Service { let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew: HashSet<_> = new_state_ids_compressed + let statediffnew: CompressedState = new_state_ids_compressed .difference(&parent_stateinfo.full_state) .copied() .collect(); - let statediffremoved: HashSet<_> = parent_stateinfo + let statediffremoved: CompressedState = parent_stateinfo .full_state .difference(&new_state_ids_compressed) .copied() @@ -409,7 +409,7 @@ impl Service { (Arc::new(statediffnew), Arc::new(statediffremoved)) } else { - (new_state_ids_compressed, Arc::new(HashSet::new())) + (new_state_ids_compressed, Arc::new(CompressedState::new())) }; if !already_existed { @@ -448,11 +448,11 @@ impl Service { .take_if(|parent| *parent != 0); debug_assert!(value.len() % STRIDE == 0, "value not aligned to stride"); - let num_values = value.len() / STRIDE; + let _num_values = value.len() / STRIDE; let mut add_mode = true; - let mut added = HashSet::with_capacity(num_values); - let mut removed = HashSet::with_capacity(num_values); + let mut added = CompressedState::new(); + let mut removed = CompressedState::new(); let mut i = STRIDE; while let Some(v) = value.get(i..expected!(i + 2 * STRIDE)) { @@ -469,8 +469,6 @@ impl Service { i = expected!(i + 2 * STRIDE); } - added.shrink_to_fit(); - removed.shrink_to_fit(); Ok(StateDiff { parent, added: Arc::new(added), @@ -507,7 +505,7 @@ impl Service { #[inline] #[must_use] -fn compress_state_event( +pub(crate) fn compress_state_event( shortstatekey: ShortStateKey, shorteventid: ShortEventId, ) -> CompressedStateEvent { @@ -523,7 +521,7 @@ fn compress_state_event( #[inline] #[must_use] -pub fn parse_compressed_state_event( +pub(crate) fn parse_compressed_state_event( compressed_event: CompressedStateEvent, ) -> (ShortStateKey, ShortEventId) { use utils::u64_from_u8; diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 8b3b67a7..a913034d 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -49,7 +49,7 @@ use crate::{ account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms, - rooms::{short::ShortRoomId, state_compressor::CompressedStateEvent}, + rooms::{short::ShortRoomId, state_compressor::CompressedState}, sending, server_keys, users, Dep, }; @@ -950,7 +950,7 @@ impl Service { pdu: &'a PduEvent, pdu_json: CanonicalJsonObject, new_room_leafs: Leafs, - state_ids_compressed: Arc>, + state_ids_compressed: Arc, soft_fail: bool, state_lock: &'a RoomMutexGuard, ) -> Result> From 7ce782ddf4cb6989caff7a3781cfc667183b9b63 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 1 Feb 2025 01:17:28 +0000 Subject: [PATCH 1468/2091] fix jemalloc cfgs lacking msvc conditions Signed-off-by: Jason Volk --- src/core/config/check.rs | 2 +- src/database/pool.rs | 9 ++++++--- src/main/runtime.rs | 6 +++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 988d4143..5532c5a2 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -38,7 +38,7 @@ pub fn check(config: &Config) -> Result { )); } - if cfg!(all(feature = "hardened_malloc", feature = "jemalloc")) { + if cfg!(all(feature = "hardened_malloc", feature = "jemalloc", not(target_env = "msvc"))) { debug_warn!( "hardened_malloc and jemalloc compile-time features are both enabled, this causes \ jemalloc to be used." diff --git a/src/database/pool.rs b/src/database/pool.rs index 86516c31..c753855a 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -13,7 +13,7 @@ use std::{ use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ debug, debug_warn, err, error, implement, - result::{DebugInspect, LogDebugErr}, + result::DebugInspect, trace, utils::sys::compute::{get_affinity, nth_core_available, set_affinity}, Error, Result, Server, @@ -290,9 +290,12 @@ fn worker_init(&self, id: usize) { // affinity is empty (no-op) if there's only one queue set_affinity(affinity.clone()); - #[cfg(feature = "jemalloc")] + #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] if affinity.clone().count() == 1 && conduwuit::alloc::je::is_affine_arena() { - use conduwuit::alloc::je::this_thread::{arena_id, set_arena}; + use conduwuit::{ + alloc::je::this_thread::{arena_id, set_arena}, + result::LogDebugErr, + }; let id = affinity.clone().next().expect("at least one id"); diff --git a/src/main/runtime.rs b/src/main/runtime.rs index 9f4f60f8..02b9931f 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -122,7 +122,7 @@ fn set_worker_affinity() { set_worker_mallctl(id); } -#[cfg(feature = "jemalloc")] +#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] fn set_worker_mallctl(id: usize) { use conduwuit::alloc::je::{ is_affine_arena, @@ -143,7 +143,7 @@ fn set_worker_mallctl(id: usize) { } } -#[cfg(not(feature = "jemalloc"))] +#[cfg(any(not(feature = "jemalloc"), target_env = "msvc"))] fn set_worker_mallctl(_: usize) {} #[tracing::instrument( @@ -189,7 +189,7 @@ fn thread_park() { } fn gc_on_park() { - #[cfg(feature = "jemalloc")] + #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] conduwuit::alloc::je::this_thread::decay() .log_debug_err() .ok(); From b4d22bd05e3cf81476669cc2e37eef60eeade07e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 1 Feb 2025 23:41:05 +0000 Subject: [PATCH 1469/2091] remove unnecessary cf arc refcnt workaround log errors and panics propagating through the request task join Signed-off-by: Jason Volk --- src/database/engine.rs | 6 ++--- src/database/engine/open.rs | 6 ++--- src/database/map.rs | 12 ++++----- src/database/map/open.rs | 5 +--- src/router/request.rs | 52 ++++++++++++++++++++++++++++--------- 5 files changed, 53 insertions(+), 28 deletions(-) diff --git a/src/database/engine.rs b/src/database/engine.rs index be3d62cf..22e2b9c8 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -30,13 +30,13 @@ use crate::{ }; pub struct Engine { + pub(crate) db: Db, + pub(crate) pool: Arc, + pub(crate) ctx: Arc, pub(super) read_only: bool, pub(super) secondary: bool, pub(crate) checksums: bool, corks: AtomicU32, - pub(crate) db: Db, - pub(crate) pool: Arc, - pub(crate) ctx: Arc, } pub(crate) type Db = DBWithThreadMode; diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs index ad724765..59dabce1 100644 --- a/src/database/engine/open.rs +++ b/src/database/engine/open.rs @@ -56,13 +56,13 @@ pub(crate) async fn open(ctx: Arc, desc: &[Descriptor]) -> Result, - cf: Arc, watchers: Watchers, - write_options: WriteOptions, + cf: Arc, + db: Arc, read_options: ReadOptions, cache_read_options: ReadOptions, + write_options: WriteOptions, } impl Map { pub(crate) fn open(db: &Arc, name: &'static str) -> Result> { Ok(Arc::new(Self { name, - db: db.clone(), - cf: open::open(db, name), watchers: Watchers::default(), - write_options: write_options_default(db), + cf: open::open(db, name), + db: db.clone(), read_options: read_options_default(db), cache_read_options: cache_read_options_default(db), + write_options: write_options_default(db), })) } diff --git a/src/database/map/open.rs b/src/database/map/open.rs index 6ecec044..07f7a0c6 100644 --- a/src/database/map/open.rs +++ b/src/database/map/open.rs @@ -30,8 +30,5 @@ pub(super) fn open(db: &Arc, name: &str) -> Arc { // lifetime parameter. We should not hold this handle, even in its Arc, after // closing the database (dropping `Engine`). Since `Arc` is a sibling // member along with this handle in `Map`, that is prevented. - unsafe { - Arc::increment_strong_count(cf_ptr); - Arc::from_raw(cf_ptr) - } + unsafe { Arc::from_raw(cf_ptr) } } diff --git a/src/router/request.rs b/src/router/request.rs index f7b94417..19cd751b 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -1,4 +1,7 @@ -use std::sync::{atomic::Ordering, Arc}; +use std::{ + fmt::Debug, + sync::{atomic::Ordering, Arc}, +}; use axum::{ extract::State, @@ -12,16 +15,16 @@ use http::{Method, StatusCode, Uri}; level = "debug", skip_all, fields( - handled = %services - .server - .metrics - .requests_handle_finished - .fetch_add(1, Ordering::Relaxed), active = %services .server .metrics .requests_handle_active .fetch_add(1, Ordering::Relaxed), + handled = %services + .server + .metrics + .requests_handle_finished + .load(Ordering::Relaxed), ) )] pub(crate) async fn handle( @@ -31,6 +34,10 @@ pub(crate) async fn handle( ) -> Result { #[cfg(debug_assertions)] conduwuit::defer! {{ + _ = services.server + .metrics + .requests_handle_finished + .fetch_add(1, Ordering::Relaxed); _ = services.server .metrics .requests_handle_active @@ -47,21 +54,35 @@ pub(crate) async fn handle( return Err(StatusCode::SERVICE_UNAVAILABLE); } - let method = req.method().clone(); let uri = req.uri().clone(); - services + let method = req.method().clone(); + let services_ = services.clone(); + let task = services .server .runtime() - .spawn(next.run(req)) - .await - .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) - .and_then(|result| handle_result(&method, &uri, result)) + .spawn(async move { execute(services_, req, next).await }); + + task.await + .map_err(unhandled) + .and_then(move |result| handle_result(&method, &uri, result)) +} + +async fn execute( + // we made a safety contract that Services will not go out of scope + // during the request; this ensures a reference is accounted for at + // the base frame of the task regardless of its detachment. + _services: Arc, + req: http::Request, + next: axum::middleware::Next, +) -> Response { + next.run(req).await } fn handle_result(method: &Method, uri: &Uri, result: Response) -> Result { let status = result.status(); let reason = status.canonical_reason().unwrap_or("Unknown Reason"); let code = status.as_u16(); + if status.is_server_error() { error!(method = ?method, uri = ?uri, "{code} {reason}"); } else if status.is_client_error() { @@ -78,3 +99,10 @@ fn handle_result(method: &Method, uri: &Uri, result: Response) -> Result(e: Error) -> StatusCode { + error!("unhandled error or panic during request: {e:?}"); + + StatusCode::INTERNAL_SERVER_ERROR +} From bd6d4bc58f45251313b33e65947a4131ea9114e7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Feb 2025 10:07:00 +0000 Subject: [PATCH 1470/2091] enforce timeout on request layers Signed-off-by: Jason Volk --- Cargo.toml | 3 ++- conduwuit-example.toml | 12 ++++++++++++ src/core/config/mod.rs | 24 ++++++++++++++++++++++++ src/router/layers.rs | 4 ++++ src/router/request.rs | 23 ++++++++++++++++++----- 5 files changed, 60 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c4af4a7c..1cf787c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -127,12 +127,13 @@ version = "0.6.2" default-features = false features = [ "add-extension", + "catch-panic", "cors", "sensitive-headers", "set-header", + "timeout", "trace", "util", - "catch-panic", ] [workspace.dependencies.rustls] diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 3fd95044..f4f42365 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -377,6 +377,18 @@ # #pusher_idle_timeout = 15 +# Maximum time to receive a request from a client (seconds). +# +#client_receive_timeout = 75 + +# Maximum time to process a request received from a client (seconds). +# +#client_request_timeout = 180 + +# Maximum time to transmit a response to a client (seconds) +# +#client_response_timeout = 120 + # Enables registration. If set to false, no users can register on this # server. # diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index ff038975..b8cfd91b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -480,6 +480,24 @@ pub struct Config { #[serde(default = "default_pusher_idle_timeout")] pub pusher_idle_timeout: u64, + /// Maximum time to receive a request from a client (seconds). + /// + /// default: 75 + #[serde(default = "default_client_receive_timeout")] + pub client_receive_timeout: u64, + + /// Maximum time to process a request received from a client (seconds). + /// + /// default: 180 + #[serde(default = "default_client_request_timeout")] + pub client_request_timeout: u64, + + /// Maximum time to transmit a response to a client (seconds) + /// + /// default: 120 + #[serde(default = "default_client_response_timeout")] + pub client_response_timeout: u64, + /// Enables registration. If set to false, no users can register on this /// server. /// @@ -2170,3 +2188,9 @@ fn default_stream_width_default() -> usize { 32 } fn default_stream_width_scale() -> f32 { 1.0 } fn default_stream_amplification() -> usize { 1024 } + +fn default_client_receive_timeout() -> u64 { 75 } + +fn default_client_request_timeout() -> u64 { 180 } + +fn default_client_response_timeout() -> u64 { 120 } diff --git a/src/router/layers.rs b/src/router/layers.rs index c5227c22..e8a8b7e8 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -18,6 +18,7 @@ use tower_http::{ cors::{self, CorsLayer}, sensitive_headers::SetSensitiveHeadersLayer, set_header::SetResponseHeaderLayer, + timeout::{RequestBodyTimeoutLayer, ResponseBodyTimeoutLayer, TimeoutLayer}, trace::{DefaultOnFailure, DefaultOnRequest, DefaultOnResponse, TraceLayer}, }; use tracing::Level; @@ -59,6 +60,9 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { ) .layer(axum::middleware::from_fn_with_state(Arc::clone(services), request::handle)) .layer(SecureClientIpSource::ConnectInfo.into_extension()) + .layer(ResponseBodyTimeoutLayer::new(Duration::from_secs(server.config.client_response_timeout))) + .layer(RequestBodyTimeoutLayer::new(Duration::from_secs(server.config.client_receive_timeout))) + .layer(TimeoutLayer::new(Duration::from_secs(server.config.client_request_timeout))) .layer(SetResponseHeaderLayer::if_not_present( HeaderName::from_static("origin-agent-cluster"), // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin-Agent-Cluster HeaderValue::from_static("?1"), diff --git a/src/router/request.rs b/src/router/request.rs index 19cd751b..68ea742c 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -10,8 +10,10 @@ use axum::{ use conduwuit::{debug, debug_error, debug_warn, err, error, trace, Result}; use conduwuit_service::Services; use http::{Method, StatusCode, Uri}; +use tracing::Span; #[tracing::instrument( + name = "request", level = "debug", skip_all, fields( @@ -57,23 +59,34 @@ pub(crate) async fn handle( let uri = req.uri().clone(); let method = req.method().clone(); let services_ = services.clone(); - let task = services - .server - .runtime() - .spawn(async move { execute(services_, req, next).await }); + let parent = Span::current(); + let task = services.server.runtime().spawn(async move { + tokio::select! { + response = execute(&services_, req, next, parent) => response, + () = services_.server.until_shutdown() => + StatusCode::SERVICE_UNAVAILABLE.into_response(), + } + }); task.await .map_err(unhandled) .and_then(move |result| handle_result(&method, &uri, result)) } +#[tracing::instrument( + name = "handle", + level = "debug", + parent = parent, + skip_all, +)] async fn execute( // we made a safety contract that Services will not go out of scope // during the request; this ensures a reference is accounted for at // the base frame of the task regardless of its detachment. - _services: Arc, + _services: &Arc, req: http::Request, next: axum::middleware::Next, + parent: Span, ) -> Response { next.run(req).await } From ffe3b0faf2740faa53415a661466c19d4fe722ad Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Feb 2025 10:43:02 +0000 Subject: [PATCH 1471/2091] make shutdown grace periods configurable Signed-off-by: Jason Volk --- conduwuit-example.toml | 8 ++++++++ src/core/config/mod.rs | 16 ++++++++++++++++ src/router/request.rs | 13 +++++++++++-- src/router/run.rs | 3 ++- src/service/sending/sender.rs | 5 ++--- 5 files changed, 39 insertions(+), 6 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index f4f42365..3e64522c 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -389,6 +389,14 @@ # #client_response_timeout = 120 +# Grace period for clean shutdown of client requests (seconds). +# +#client_shutdown_timeout = 10 + +# Grace period for clean shutdown of federation requests (seconds). +# +#sender_shutdown_timeout = 5 + # Enables registration. If set to false, no users can register on this # server. # diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index b8cfd91b..ff80d1cf 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -498,6 +498,18 @@ pub struct Config { #[serde(default = "default_client_response_timeout")] pub client_response_timeout: u64, + /// Grace period for clean shutdown of client requests (seconds). + /// + /// default: 10 + #[serde(default = "default_client_shutdown_timeout")] + pub client_shutdown_timeout: u64, + + /// Grace period for clean shutdown of federation requests (seconds). + /// + /// default: 5 + #[serde(default = "default_sender_shutdown_timeout")] + pub sender_shutdown_timeout: u64, + /// Enables registration. If set to false, no users can register on this /// server. /// @@ -2194,3 +2206,7 @@ fn default_client_receive_timeout() -> u64 { 75 } fn default_client_request_timeout() -> u64 { 180 } fn default_client_response_timeout() -> u64 { 120 } + +fn default_client_shutdown_timeout() -> u64 { 15 } + +fn default_sender_shutdown_timeout() -> u64 { 5 } diff --git a/src/router/request.rs b/src/router/request.rs index 68ea742c..e0373646 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -1,6 +1,7 @@ use std::{ fmt::Debug, sync::{atomic::Ordering, Arc}, + time::Duration, }; use axum::{ @@ -9,7 +10,9 @@ use axum::{ }; use conduwuit::{debug, debug_error, debug_warn, err, error, trace, Result}; use conduwuit_service::Services; +use futures::FutureExt; use http::{Method, StatusCode, Uri}; +use tokio::time::sleep; use tracing::Span; #[tracing::instrument( @@ -63,8 +66,14 @@ pub(crate) async fn handle( let task = services.server.runtime().spawn(async move { tokio::select! { response = execute(&services_, req, next, parent) => response, - () = services_.server.until_shutdown() => - StatusCode::SERVICE_UNAVAILABLE.into_response(), + response = services_.server.until_shutdown() + .then(|()| { + let timeout = services_.server.config.client_shutdown_timeout; + let timeout = Duration::from_secs(timeout); + sleep(timeout) + }) + .map(|()| StatusCode::SERVICE_UNAVAILABLE) + .map(IntoResponse::into_response) => response, } }); diff --git a/src/router/run.rs b/src/router/run.rs index 605168b8..26701735 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -122,7 +122,8 @@ async fn handle_shutdown(server: Arc, tx: Sender<()>, handle: axum_serve error!("failed sending shutdown transaction to channel: {e}"); } - let timeout = Duration::from_secs(36); + let timeout = server.config.client_shutdown_timeout; + let timeout = Duration::from_secs(timeout); debug!( ?timeout, handle_active = ?server.metrics.requests_handle_active.load(Ordering::Relaxed), diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index f19b69da..3e86de2d 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -67,8 +67,6 @@ type SendingFuture<'a> = BoxFuture<'a, SendingResult>; type SendingFutures<'a> = FuturesUnordered>; type CurTransactionStatus = HashMap; -const CLEANUP_TIMEOUT_MS: u64 = 3500; - const SELECT_PRESENCE_LIMIT: usize = 256; const SELECT_RECEIPT_LIMIT: usize = 256; const SELECT_EDU_LIMIT: usize = EDU_LIMIT - 2; @@ -216,8 +214,9 @@ impl Service { time::{sleep_until, Instant}, }; + let timeout = self.server.config.sender_shutdown_timeout; + let timeout = Duration::from_secs(timeout); let now = Instant::now(); - let timeout = Duration::from_millis(CLEANUP_TIMEOUT_MS); let deadline = now.checked_add(timeout).unwrap_or(now); loop { trace!("Waiting for {} requests to complete...", futures.len()); From a774afe8370bd6eed3deed6e663229e8457d73c7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Feb 2025 08:59:14 +0000 Subject: [PATCH 1472/2091] modernize remove_to_device_events Signed-off-by: Jason Volk --- src/service/users/mod.rs | 43 ++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 26 deletions(-) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index b2d3a94a..e5caed47 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,12 +1,12 @@ -use std::{collections::BTreeMap, mem, mem::size_of, sync::Arc}; +use std::{collections::BTreeMap, mem, sync::Arc}; use conduwuit::{ debug_warn, err, trace, utils::{self, stream::TryIgnore, string::Unquoted, ReadyExt}, Err, Error, Result, Server, }; -use database::{Database, Deserialized, Ignore, Interfix, Json, Map}; -use futures::{FutureExt, Stream, StreamExt, TryFutureExt}; +use database::{Deserialized, Ignore, Interfix, Json, Map}; +use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{ api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, @@ -28,7 +28,6 @@ pub struct Service { struct Services { server: Arc, - db: Arc, account_data: Dep, admin: Dep, globals: Dep, @@ -64,7 +63,6 @@ impl crate::Service for Service { Ok(Arc::new(Self { services: Services { server: args.server.clone(), - db: args.db.clone(), account_data: args.depend::("account_data"), admin: args.depend::("admin"), globals: args.depend::("globals"), @@ -801,35 +799,28 @@ impl Service { .map(|(_, val): (Ignore, Raw)| val) } - pub async fn remove_to_device_events( + pub async fn remove_to_device_events( &self, user_id: &UserId, device_id: &DeviceId, - until: u64, - ) { - let mut prefix = user_id.as_bytes().to_vec(); - prefix.push(0xFF); - prefix.extend_from_slice(device_id.as_bytes()); - prefix.push(0xFF); + until: Until, + ) where + Until: Into> + Send, + { + type Key<'a> = (&'a UserId, &'a DeviceId, u64); - let mut last = prefix.clone(); - last.extend_from_slice(&until.to_be_bytes()); - - let _cork = self.services.db.cork_and_flush(); + let until = until.into().unwrap_or(u64::MAX); + let from = (user_id, device_id, until); self.db .todeviceid_events - .rev_raw_keys_from(&last) // this includes last + .rev_keys_from(&from) .ignore_err() - .ready_take_while(move |key| key.starts_with(&prefix)) - .map(|key| { - let len = key.len(); - let start = len.saturating_sub(size_of::()); - let count = utils::u64_from_u8(&key[start..len]); - (key, count) + .ready_take_while(move |(user_id_, device_id_, _): &Key<'_>| { + user_id == *user_id_ && device_id == *device_id_ + }) + .ready_for_each(|key: Key<'_>| { + self.db.todeviceid_events.del(key); }) - .ready_take_while(move |(_, count)| *count <= until) - .ready_for_each(|(key, _)| self.db.todeviceid_events.remove(&key)) - .boxed() .await; } From 5e59ce37c4799c24723997326e1ccc26bb3345b0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 31 Jan 2025 13:51:39 +0000 Subject: [PATCH 1473/2091] snapshot sync results at next_batch upper-bound Signed-off-by: Jason Volk --- src/admin/query/account_data.rs | 2 +- src/admin/query/users.rs | 2 +- src/api/client/sync/v3.rs | 10 +++++----- src/api/client/sync/v4.rs | 13 +++++++++---- src/api/client/sync/v5.rs | 8 ++++---- src/service/account_data/mod.rs | 12 +++++++----- src/service/users/mod.rs | 18 ++++++++++++++---- 7 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index b75d8234..bb8ddeff 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -41,7 +41,7 @@ async fn changes_since( let results: Vec<_> = self .services .account_data - .changes_since(room_id.as_deref(), &user_id, since) + .changes_since(room_id.as_deref(), &user_id, since, None) .collect() .await; let query_time = timer.elapsed(); diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index 3715ac25..c517d9dd 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -413,7 +413,7 @@ async fn get_to_device_events( let result = self .services .users - .get_to_device_events(&user_id, &device_id) + .get_to_device_events(&user_id, &device_id, None, None) .collect::>() .await; let query_time = timer.elapsed(); diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 49246514..b548aa23 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -290,20 +290,20 @@ pub(crate) async fn build_sync_events( let account_data = services .account_data - .changes_since(None, sender_user, since) + .changes_since(None, sender_user, since, Some(next_batch)) .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) .collect(); // Look for device list updates of this account let keys_changed = services .users - .keys_changed(sender_user, since, None) + .keys_changed(sender_user, since, Some(next_batch)) .map(ToOwned::to_owned) .collect::>(); let to_device_events = services .users - .get_to_device_events(sender_user, sender_device) + .get_to_device_events(sender_user, sender_device, Some(since), Some(next_batch)) .collect::>(); let device_one_time_keys_count = services @@ -700,14 +700,14 @@ async fn load_joined_room( let account_data_events = services .account_data - .changes_since(Some(room_id), sender_user, since) + .changes_since(Some(room_id), sender_user, since, Some(next_batch)) .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) .collect(); // Look for device list updates in this room let device_updates = services .users - .room_keys_changed(room_id, since, None) + .room_keys_changed(room_id, since, Some(next_batch)) .map(|(user_id, _)| user_id) .map(ToOwned::to_owned) .collect::>(); diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index b7967498..66793ba1 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -153,7 +153,7 @@ pub(crate) async fn sync_events_v4_route( if body.extensions.account_data.enabled.unwrap_or(false) { account_data.global = services .account_data - .changes_since(None, sender_user, globalsince) + .changes_since(None, sender_user, globalsince, Some(next_batch)) .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) .collect() .await; @@ -164,7 +164,7 @@ pub(crate) async fn sync_events_v4_route( room.clone(), services .account_data - .changes_since(Some(&room), sender_user, globalsince) + .changes_since(Some(&room), sender_user, globalsince, Some(next_batch)) .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) .collect() .await, @@ -531,7 +531,7 @@ pub(crate) async fn sync_events_v4_route( room_id.to_owned(), services .account_data - .changes_since(Some(room_id), sender_user, *roomsince) + .changes_since(Some(room_id), sender_user, *roomsince, Some(next_batch)) .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) .collect() .await, @@ -779,7 +779,12 @@ pub(crate) async fn sync_events_v4_route( Some(sync_events::v4::ToDevice { events: services .users - .get_to_device_events(sender_user, &sender_device) + .get_to_device_events( + sender_user, + &sender_device, + Some(globalsince), + Some(next_batch), + ) .collect() .await, next_batch: next_batch.to_string(), diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index 66647f0e..e7b5fe74 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -390,7 +390,7 @@ async fn process_rooms( room_id.to_owned(), services .account_data - .changes_since(Some(room_id), sender_user, *roomsince) + .changes_since(Some(room_id), sender_user, *roomsince, Some(next_batch)) .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) .collect() .await, @@ -644,7 +644,7 @@ async fn collect_account_data( account_data.global = services .account_data - .changes_since(None, sender_user, globalsince) + .changes_since(None, sender_user, globalsince, None) .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Global)) .collect() .await; @@ -655,7 +655,7 @@ async fn collect_account_data( room.clone(), services .account_data - .changes_since(Some(room), sender_user, globalsince) + .changes_since(Some(room), sender_user, globalsince, None) .ready_filter_map(|e| extract_variant!(e, AnyRawAccountDataEvent::Room)) .collect() .await, @@ -876,7 +876,7 @@ async fn collect_to_device( next_batch: next_batch.to_string(), events: services .users - .get_to_device_events(sender_user, sender_device) + .get_to_device_events(sender_user, sender_device, None, Some(next_batch)) .collect() .await, }) diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index ddbc15a4..5a943f88 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -5,7 +5,7 @@ use conduwuit::{ utils::{result::LogErr, stream::TryIgnore, ReadyExt}, Err, Result, }; -use database::{Deserialized, Handle, Interfix, Json, Map}; +use database::{Deserialized, Handle, Ignore, Json, Map}; use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{ events::{ @@ -131,18 +131,20 @@ pub fn changes_since<'a>( room_id: Option<&'a RoomId>, user_id: &'a UserId, since: u64, + to: Option, ) -> impl Stream + Send + 'a { - let prefix = (room_id, user_id, Interfix); - let prefix = database::serialize_key(prefix).expect("failed to serialize prefix"); + type Key<'a> = (Option<&'a RoomId>, &'a UserId, u64, Ignore); // Skip the data that's exactly at since, because we sent that last time let first_possible = (room_id, user_id, since.saturating_add(1)); self.db .roomuserdataid_accountdata - .stream_from_raw(&first_possible) + .stream_from(&first_possible) .ignore_err() - .ready_take_while(move |(k, _)| k.starts_with(&prefix)) + .ready_take_while(move |((room_id_, user_id_, count, _), _): &(Key<'_>, _)| { + room_id == *room_id_ && user_id == *user_id_ && to.is_none_or(|to| *count <= to) + }) .map(move |(_, v)| { match room_id { | Some(_) => serde_json::from_slice::>(v) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index e5caed47..68b87541 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, mem, sync::Arc}; use conduwuit::{ - debug_warn, err, trace, + at, debug_warn, err, trace, utils::{self, stream::TryIgnore, string::Unquoted, ReadyExt}, Err, Error, Result, Server, }; @@ -790,13 +790,23 @@ impl Service { &'a self, user_id: &'a UserId, device_id: &'a DeviceId, + since: Option, + to: Option, ) -> impl Stream> + Send + 'a { - let prefix = (user_id, device_id, Interfix); + type Key<'a> = (&'a UserId, &'a DeviceId, u64); + + let from = (user_id, device_id, since.map_or(0, |since| since.saturating_add(1))); + self.db .todeviceid_events - .stream_prefix(&prefix) + .stream_from(&from) .ignore_err() - .map(|(_, val): (Ignore, Raw)| val) + .ready_take_while(move |((user_id_, device_id_, count), _): &(Key<'_>, _)| { + user_id == *user_id_ + && device_id == *device_id_ + && to.is_none_or(|to| *count <= to) + }) + .map(at!(1)) } pub async fn remove_to_device_events( From 32f990fc72c6bfbf4a869dac9f5b2b88ee334684 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Feb 2025 23:19:35 +0000 Subject: [PATCH 1474/2091] fix the panic counter in the tower layer Signed-off-by: Jason Volk --- src/router/layers.rs | 18 +++++++------- src/router/request.rs | 56 ++++++++++++++++++++----------------------- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/src/router/layers.rs b/src/router/layers.rs index e8a8b7e8..7ebec16e 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -49,6 +49,7 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { ))] let layers = layers.layer(compression_layer(server)); + let services_ = services.clone(); let layers = layers .layer(SetSensitiveHeadersLayer::new([header::AUTHORIZATION])) .layer( @@ -89,7 +90,7 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { )) .layer(cors_layer(server)) .layer(body_limit_layer(server)) - .layer(CatchPanicLayer::custom(catch_panic)); + .layer(CatchPanicLayer::custom(move |panic| catch_panic(panic, services_.clone()))); let (router, guard) = router::build(services); Ok((router.layer(layers), guard)) @@ -167,15 +168,14 @@ fn body_limit_layer(server: &Server) -> DefaultBodyLimit { #[allow(clippy::needless_pass_by_value)] fn catch_panic( err: Box, + services: Arc, ) -> http::Response> { - //TODO: XXX - /* - conduwuit_service::services() - .server - .metrics - .requests_panic - .fetch_add(1, std::sync::atomic::Ordering::Release); - */ + services + .server + .metrics + .requests_panic + .fetch_add(1, std::sync::atomic::Ordering::Release); + let details = if let Some(s) = err.downcast_ref::() { s.clone() } else if let Some(s) = err.downcast_ref::<&str>() { diff --git a/src/router/request.rs b/src/router/request.rs index e0373646..b6c22d45 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -15,40 +15,12 @@ use http::{Method, StatusCode, Uri}; use tokio::time::sleep; use tracing::Span; -#[tracing::instrument( - name = "request", - level = "debug", - skip_all, - fields( - active = %services - .server - .metrics - .requests_handle_active - .fetch_add(1, Ordering::Relaxed), - handled = %services - .server - .metrics - .requests_handle_finished - .load(Ordering::Relaxed), - ) -)] +#[tracing::instrument(name = "request", level = "debug", skip_all)] pub(crate) async fn handle( State(services): State>, req: http::Request, next: axum::middleware::Next, ) -> Result { - #[cfg(debug_assertions)] - conduwuit::defer! {{ - _ = services.server - .metrics - .requests_handle_finished - .fetch_add(1, Ordering::Relaxed); - _ = services.server - .metrics - .requests_handle_active - .fetch_sub(1, Ordering::Relaxed); - }}; - if !services.server.running() { debug_warn!( method = %req.method(), @@ -87,16 +59,40 @@ pub(crate) async fn handle( level = "debug", parent = parent, skip_all, + fields( + active = %services + .server + .metrics + .requests_handle_active + .fetch_add(1, Ordering::Relaxed), + handled = %services + .server + .metrics + .requests_handle_finished + .load(Ordering::Relaxed), + ) )] async fn execute( // we made a safety contract that Services will not go out of scope // during the request; this ensures a reference is accounted for at // the base frame of the task regardless of its detachment. - _services: &Arc, + services: &Arc, req: http::Request, next: axum::middleware::Next, parent: Span, ) -> Response { + #[cfg(debug_assertions)] + conduwuit::defer! {{ + _ = services.server + .metrics + .requests_handle_finished + .fetch_add(1, Ordering::Relaxed); + _ = services.server + .metrics + .requests_handle_active + .fetch_sub(1, Ordering::Relaxed); + }}; + next.run(req).await } From da4b94d80dc9939ad385860af764ed1a1837b84e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Feb 2025 22:13:27 +0000 Subject: [PATCH 1475/2091] trap panics when running in gdb Signed-off-by: Jason Volk --- src/core/debug.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/core/debug.rs b/src/core/debug.rs index ca0f2f2e..8a5eccfd 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -1,6 +1,6 @@ #![allow(clippy::disallowed_macros)] -use std::{any::Any, panic}; +use std::{any::Any, env, panic, sync::LazyLock}; // Export debug proc_macros pub use conduwuit_macros::recursion_depth; @@ -58,16 +58,26 @@ pub const INFO_SPAN_LEVEL: Level = if cfg!(debug_assertions) { Level::DEBUG }; -pub fn set_panic_trap() { +pub static DEBUGGER: LazyLock = + LazyLock::new(|| env::var("_").unwrap_or_default().ends_with("gdb")); + +#[cfg_attr(debug_assertions, crate::ctor)] +#[cfg_attr(not(debug_assertions), allow(dead_code))] +fn set_panic_trap() { + if !*DEBUGGER { + return; + } + let next = panic::take_hook(); panic::set_hook(Box::new(move |info| { panic_handler(info, &next); })); } -#[inline(always)] +#[cold] +#[inline(never)] #[allow(deprecated_in_future)] -fn panic_handler(info: &panic::PanicHookInfo<'_>, next: &dyn Fn(&panic::PanicHookInfo<'_>)) { +pub fn panic_handler(info: &panic::PanicHookInfo<'_>, next: &dyn Fn(&panic::PanicHookInfo<'_>)) { trap(); next(info); } From 106bcd30b75b6846be197fc5431063b0b82c4336 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Feb 2025 07:40:08 +0000 Subject: [PATCH 1476/2091] optimize incremental sync state diff Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 366 +++++++++-------- src/service/rooms/state_accessor/mod.rs | 523 +++++++++++++----------- 2 files changed, 474 insertions(+), 415 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index b548aa23..a97e4329 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -7,13 +7,13 @@ use std::{ use axum::extract::State; use conduwuit::{ at, err, error, extract_variant, is_equal_to, pair_of, - pdu::EventHash, + pdu::{Event, EventHash}, + ref_at, result::FlatOk, utils::{ self, - future::OptionExt, math::ruma_from_u64, - stream::{BroadbandExt, Tools, WidebandExt}, + stream::{BroadbandExt, Tools, TryExpect, WidebandExt}, BoolExt, IterStream, ReadyExt, TryFutureExtExt, }, PduCount, PduEvent, Result, @@ -53,19 +53,16 @@ use ruma::{ serde::Raw, uint, DeviceId, EventId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId, }; +use service::rooms::short::{ShortEventId, ShortStateKey}; use super::{load_timeline, share_encrypted_room}; -use crate::{ - client::{ignored_filter, lazy_loading_witness}, - Ruma, RumaResponse, -}; +use crate::{client::ignored_filter, Ruma, RumaResponse}; #[derive(Default)] struct StateChanges { heroes: Option>, joined_member_count: Option, invited_member_count: Option, - joined_since_last_sync: bool, state_events: Vec, device_list_updates: HashSet, left_encrypted_users: HashSet, @@ -625,6 +622,40 @@ async fn load_joined_room( .await?; let (timeline_pdus, limited) = timeline; + let initial = since_shortstatehash.is_none(); + let lazy_loading_enabled = filter.room.state.lazy_load_options.is_enabled() + || filter.room.timeline.lazy_load_options.is_enabled(); + + let lazy_loading_context = &lazy_loading::Context { + user_id: sender_user, + device_id: sender_device, + room_id, + token: Some(since), + options: Some(&filter.room.state.lazy_load_options), + }; + + // Reset lazy loading because this is an initial sync + let lazy_load_reset: OptionFuture<_> = initial + .then(|| services.rooms.lazy_loading.reset(lazy_loading_context)) + .into(); + + lazy_load_reset.await; + let witness: OptionFuture<_> = lazy_loading_enabled + .then(|| { + let witness: Witness = timeline_pdus + .iter() + .map(ref_at!(1)) + .map(Event::sender) + .map(Into::into) + .chain(receipt_events.keys().map(Into::into)) + .collect(); + + services + .rooms + .lazy_loading + .witness_retain(witness, lazy_loading_context) + }) + .into(); let last_notification_read: OptionFuture<_> = timeline_pdus .is_empty() @@ -646,41 +677,20 @@ async fn load_joined_room( }) .into(); + let (last_notification_read, since_sender_member, witness) = + join3(last_notification_read, since_sender_member, witness).await; + let joined_since_last_sync = since_sender_member - .await .flatten() .is_none_or(|content: RoomMemberEventContent| { content.membership != MembershipState::Join }); - let lazy_loading_enabled = filter.room.state.lazy_load_options.is_enabled() - || filter.room.timeline.lazy_load_options.is_enabled(); - - let lazy_reset = since_shortstatehash.is_none(); - let lazy_loading_context = &lazy_loading::Context { - user_id: sender_user, - device_id: sender_device, - room_id, - token: None, - options: Some(&filter.room.state.lazy_load_options), - }; - - // Reset lazy loading because this is an initial sync - let lazy_load_reset: OptionFuture<_> = lazy_reset - .then(|| services.rooms.lazy_loading.reset(lazy_loading_context)) - .into(); - - lazy_load_reset.await; - let witness: OptionFuture<_> = lazy_loading_enabled - .then(|| lazy_loading_witness(services, lazy_loading_context, timeline_pdus.iter())) - .into(); - let StateChanges { heroes, joined_member_count, invited_member_count, - joined_since_last_sync, state_events, mut device_list_updates, left_encrypted_users, @@ -693,7 +703,7 @@ async fn load_joined_room( since_shortstatehash, current_shortstatehash, joined_since_last_sync, - witness.await.as_ref(), + witness.as_ref(), ) .boxed() .await?; @@ -719,28 +729,7 @@ async fn load_joined_room( .map(|(_, pdu)| pdu.to_sync_room_event()) .collect(); - let typing_events = services - .rooms - .typing - .last_typing_update(room_id) - .and_then(|count| async move { - if count <= since { - return Ok(Vec::>::new()); - } - - let typings = services - .rooms - .typing - .typings_all(room_id, sender_user) - .await?; - - Ok(vec![serde_json::from_str(&serde_json::to_string(&typings)?)?]) - }) - .unwrap_or(Vec::new()); - - let send_notification_counts = last_notification_read - .is_none_or(|&count| count > since) - .await; + let send_notification_counts = last_notification_read.is_none_or(|count| count > since); let notification_count: OptionFuture<_> = send_notification_counts .then(|| { @@ -764,8 +753,27 @@ async fn load_joined_room( }) .into(); - let events = join3(room_events, account_data_events, typing_events); + let typing_events = services + .rooms + .typing + .last_typing_update(room_id) + .and_then(|count| async move { + if count <= since { + return Ok(Vec::>::new()); + } + + let typings = services + .rooms + .typing + .typings_all(room_id, sender_user) + .await?; + + Ok(vec![serde_json::from_str(&serde_json::to_string(&typings)?)?]) + }) + .unwrap_or(Vec::new()); + let unread_notifications = join(notification_count, highlight_count); + let events = join3(room_events, account_data_events, typing_events); let (unread_notifications, events, device_updates) = join3(unread_notifications, events, device_updates) .boxed() @@ -942,7 +950,6 @@ async fn calculate_state_initial( heroes, joined_member_count, invited_member_count, - joined_since_last_sync: true, state_events, ..Default::default() }) @@ -952,7 +959,7 @@ async fn calculate_state_initial( #[allow(clippy::too_many_arguments)] async fn calculate_state_incremental<'a>( services: &Services, - sender_user: &UserId, + sender_user: &'a UserId, room_id: &RoomId, full_state: bool, _filter: &FilterDefinition, @@ -965,102 +972,130 @@ async fn calculate_state_incremental<'a>( let state_changed = since_shortstatehash != current_shortstatehash; - let state_get_id = |user_id: &'a UserId| { - services - .rooms - .state_accessor - .state_get_id(current_shortstatehash, &StateEventType::RoomMember, user_id.as_str()) - .ok() - }; - - let lazy_state_ids: OptionFuture<_> = witness - .map(|witness| { - witness - .iter() - .stream() - .broad_filter_map(|user_id| state_get_id(user_id)) - .collect::>() - }) - .into(); - - let current_state_ids: OptionFuture<_> = state_changed - .then(|| { - services - .rooms - .state_accessor - .state_full_ids(current_shortstatehash) - .collect::>() - }) - .into(); - - let since_state_ids: OptionFuture<_> = (state_changed && !full_state) - .then(|| { - services - .rooms - .state_accessor - .state_full_ids(since_shortstatehash) - .collect::>() - }) - .into(); - - let lazy_state_ids = lazy_state_ids - .map(Option::into_iter) - .map(|iter| iter.flat_map(Vec::into_iter)) - .map(IterStream::stream) - .flatten_stream(); - - let ref since_state_ids = since_state_ids.shared(); - let delta_state_events = current_state_ids - .map(Option::into_iter) - .map(|iter| iter.flat_map(Vec::into_iter)) - .map(IterStream::stream) - .flatten_stream() - .filter_map(|(shortstatekey, event_id): (u64, OwnedEventId)| async move { - since_state_ids - .clone() - .await - .is_none_or(|since_state| since_state.get(&shortstatekey) != Some(&event_id)) - .then_some(event_id) - }) - .chain(lazy_state_ids) - .broad_filter_map(|event_id: OwnedEventId| async move { - services - .rooms - .timeline - .get_pdu(&event_id) - .await - .map(move |pdu| (event_id, pdu)) - .ok() - }) - .collect::>(); - - let since_encryption = services - .rooms - .state_accessor - .state_get(since_shortstatehash, &StateEventType::RoomEncryption, "") - .is_ok(); - let encrypted_room = services .rooms .state_accessor .state_get(current_shortstatehash, &StateEventType::RoomEncryption, "") - .is_ok(); + .is_ok() + .await; - let (delta_state_events, encrypted_room) = join(delta_state_events, encrypted_room).await; + let state_get_shorteventid = |user_id: &'a UserId| { + services + .rooms + .state_accessor + .state_get_shortid( + current_shortstatehash, + &StateEventType::RoomMember, + user_id.as_str(), + ) + .ok() + }; - let (mut device_list_updates, left_encrypted_users) = delta_state_events - .values() + let lazy_state_ids: OptionFuture<_> = witness + .filter(|_| !full_state && !encrypted_room) + .map(|witness| { + witness + .iter() + .stream() + .broad_filter_map(|user_id| state_get_shorteventid(user_id)) + .into_future() + }) + .into(); + + let state_diff: OptionFuture<_> = (!full_state && state_changed) + .then(|| { + services + .rooms + .state_accessor + .state_added((since_shortstatehash, current_shortstatehash)) + .boxed() + .into_future() + }) + .into(); + + let current_state_ids: OptionFuture<_> = full_state + .then(|| { + services + .rooms + .state_accessor + .state_full_shortids(current_shortstatehash) + .expect_ok() + .boxed() + .into_future() + }) + .into(); + + let lazy_state_ids = lazy_state_ids + .map(|opt| { + opt.map(|(curr, next)| { + let opt = curr; + let iter = Option::into_iter(opt); + IterStream::stream(iter).chain(next) + }) + }) + .map(Option::into_iter) + .map(IterStream::stream) + .flatten_stream() + .flatten(); + + let state_diff_ids = state_diff + .map(|opt| { + opt.map(|(curr, next)| { + let opt = curr; + let iter = Option::into_iter(opt); + IterStream::stream(iter).chain(next) + }) + }) + .map(Option::into_iter) + .map(IterStream::stream) + .flatten_stream() + .flatten(); + + let state_events = current_state_ids + .map(|opt| { + opt.map(|(curr, next)| { + let opt = curr; + let iter = Option::into_iter(opt); + IterStream::stream(iter).chain(next) + }) + }) + .map(Option::into_iter) + .map(IterStream::stream) + .flatten_stream() + .flatten() + .chain(state_diff_ids) + .broad_filter_map(|(shortstatekey, shorteventid)| async move { + if witness.is_none() || encrypted_room { + return Some(shorteventid); + } + + lazy_filter(services, sender_user, shortstatekey, shorteventid).await + }) + .chain(lazy_state_ids) + .broad_filter_map(|shorteventid| { + services + .rooms + .short + .get_eventid_from_short(shorteventid) + .ok() + }) + .broad_filter_map(|event_id: OwnedEventId| async move { + services.rooms.timeline.get_pdu(&event_id).await.ok() + }) + .collect::>() + .await; + + let (device_list_updates, left_encrypted_users) = state_events + .iter() .stream() .ready_filter(|_| encrypted_room) .ready_filter(|state_event| state_event.kind == RoomMember) .ready_filter_map(|state_event| { - let content = state_event.get_content().ok()?; - let user_id = state_event.state_key.as_ref()?.parse().ok()?; + let content: RoomMemberEventContent = state_event.get_content().ok()?; + let user_id: OwnedUserId = state_event.state_key.as_ref()?.parse().ok()?; + Some((content, user_id)) }) - .ready_filter(|(_, user_id): &(RoomMemberEventContent, OwnedUserId)| { - user_id != sender_user - }) .fold_default(|(mut dlu, mut leu): pair_of!(HashSet<_>), (content, user_id)| async move { use MembershipState::*; @@ -1068,8 +1103,9 @@ async fn calculate_state_incremental<'a>( |user_id| share_encrypted_room(services, sender_user, user_id, Some(room_id)); match content.membership { - | Join if !shares_encrypted_room(&user_id).await => dlu.insert(user_id), | Leave => leu.insert(user_id), + | Join if joined_since_last_sync || !shares_encrypted_room(&user_id).await => + dlu.insert(user_id), | _ => false, }; @@ -1077,29 +1113,7 @@ async fn calculate_state_incremental<'a>( }) .await; - // If the user is in a new encrypted room, give them all joined users - let new_encrypted_room = encrypted_room && !since_encryption.await; - if joined_since_last_sync && encrypted_room || new_encrypted_room { - services - .rooms - .state_cache - .room_members(room_id) - .ready_filter(|&user_id| sender_user != user_id) - .map(ToOwned::to_owned) - .broad_filter_map(|user_id| async move { - share_encrypted_room(services, sender_user, &user_id, Some(room_id)) - .await - .or_some(user_id) - }) - .ready_for_each(|user_id| { - device_list_updates.insert(user_id); - }) - .await; - } - - let send_member_count = delta_state_events - .values() - .any(|event| event.kind == RoomMember); + let send_member_count = state_events.iter().any(|event| event.kind == RoomMember); let (joined_member_count, invited_member_count, heroes) = if send_member_count { calculate_counts(services, room_id, sender_user).await? @@ -1111,13 +1125,29 @@ async fn calculate_state_incremental<'a>( heroes, joined_member_count, invited_member_count, - joined_since_last_sync, + state_events, device_list_updates, left_encrypted_users, - state_events: delta_state_events.into_values().collect(), }) } +async fn lazy_filter( + services: &Services, + sender_user: &UserId, + shortstatekey: ShortStateKey, + shorteventid: ShortEventId, +) -> Option { + let (event_type, state_key) = services + .rooms + .short + .get_statekey_from_short(shortstatekey) + .await + .ok()?; + + (event_type != StateEventType::RoomMember || state_key == sender_user.as_str()) + .then_some(shorteventid) +} + async fn calculate_counts( services: &Services, room_id: &RoomId, diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 8b56c8b6..bed8d210 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -6,7 +6,7 @@ use std::{ }; use conduwuit::{ - at, err, error, + at, err, error, pair_of, pdu::PduBuilder, utils, utils::{ @@ -17,7 +17,7 @@ use conduwuit::{ Err, Error, PduEvent, Result, }; use database::{Deserialized, Map}; -use futures::{FutureExt, Stream, StreamExt, TryFutureExt}; +use futures::{future::try_join, FutureExt, Stream, StreamExt, TryFutureExt}; use lru_cache::LruCache; use ruma::{ events::{ @@ -48,7 +48,7 @@ use crate::{ rooms::{ short::{ShortEventId, ShortStateHash, ShortStateKey}, state::RoomMutexGuard, - state_compressor::{compress_state_event, parse_compressed_state_event}, + state_compressor::{compress_state_event, parse_compressed_state_event, CompressedState}, }, Dep, }; @@ -143,6 +143,256 @@ impl crate::Service for Service { } impl Service { + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). + pub async fn room_state_get_content( + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, + ) -> Result + where + T: for<'de> Deserialize<'de>, + { + self.room_state_get(room_id, event_type, state_key) + .await + .and_then(|event| event.get_content()) + } + + /// Returns the full room state. + #[tracing::instrument(skip(self), level = "debug")] + pub fn room_state_full<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream> + Send + 'a { + self.services + .state + .get_room_shortstatehash(room_id) + .map_ok(|shortstatehash| self.state_full(shortstatehash).map(Ok)) + .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .try_flatten_stream() + } + + /// Returns the full room state pdus + #[tracing::instrument(skip(self), level = "debug")] + pub fn room_state_full_pdus<'a>( + &'a self, + room_id: &'a RoomId, + ) -> impl Stream> + Send + 'a { + self.services + .state + .get_room_shortstatehash(room_id) + .map_ok(|shortstatehash| self.state_full_pdus(shortstatehash).map(Ok)) + .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .try_flatten_stream() + } + + /// Returns a single EventId from `room_id` with key (`event_type`, + /// `state_key`). + #[tracing::instrument(skip(self), level = "debug")] + pub async fn room_state_get_id( + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, + ) -> Result + where + Id: for<'de> Deserialize<'de> + Sized + ToOwned, + ::Owned: Borrow, + { + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get_id(shortstatehash, event_type, state_key)) + .await + } + + /// Returns a single PDU from `room_id` with key (`event_type`, + /// `state_key`). + #[tracing::instrument(skip(self), level = "debug")] + pub async fn room_state_get( + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, + ) -> Result { + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get(shortstatehash, event_type, state_key)) + .await + } + + /// The user was a joined member at this state (potentially in the past) + #[inline] + async fn user_was_joined(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { + self.user_membership(shortstatehash, user_id).await == MembershipState::Join + } + + /// The user was an invited or joined room member at this state (potentially + /// in the past) + #[inline] + async fn user_was_invited(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { + let s = self.user_membership(shortstatehash, user_id).await; + s == MembershipState::Join || s == MembershipState::Invite + } + + /// Get membership for given user in state + async fn user_membership( + &self, + shortstatehash: ShortStateHash, + user_id: &UserId, + ) -> MembershipState { + self.state_get_content(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) + .await + .map_or(MembershipState::Leave, |c: RoomMemberEventContent| c.membership) + } + + /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). + pub async fn state_get_content( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> Result + where + T: for<'de> Deserialize<'de>, + { + self.state_get(shortstatehash, event_type, state_key) + .await + .and_then(|event| event.get_content()) + } + + #[tracing::instrument(skip(self), level = "debug")] + pub async fn state_contains( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> bool { + let Ok(shortstatekey) = self + .services + .short + .get_shortstatekey(event_type, state_key) + .await + else { + return false; + }; + + self.state_contains_shortstatekey(shortstatehash, shortstatekey) + .await + } + + #[tracing::instrument(skip(self), level = "debug")] + pub async fn state_contains_shortstatekey( + &self, + shortstatehash: ShortStateHash, + shortstatekey: ShortStateKey, + ) -> bool { + let start = compress_state_event(shortstatekey, 0); + let end = compress_state_event(shortstatekey, u64::MAX); + + self.load_full_state(shortstatehash) + .map_ok(|full_state| full_state.range(start..end).next().copied()) + .await + .flat_ok() + .is_some() + } + + /// Returns a single PDU from `room_id` with key (`event_type`, + /// `state_key`). + pub async fn state_get( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> Result { + self.state_get_id(shortstatehash, event_type, state_key) + .and_then(|event_id: OwnedEventId| async move { + self.services.timeline.get_pdu(&event_id).await + }) + .await + } + + /// Returns a single EventId from `room_id` with key (`event_type`, + /// `state_key`). + #[tracing::instrument(skip(self), level = "debug")] + pub async fn state_get_id( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> Result + where + Id: for<'de> Deserialize<'de> + Sized + ToOwned, + ::Owned: Borrow, + { + let shorteventid = self + .state_get_shortid(shortstatehash, event_type, state_key) + .await?; + + self.services + .short + .get_eventid_from_short(shorteventid) + .await + } + + /// Returns a single EventId from `room_id` with key (`event_type`, + /// `state_key`). + #[tracing::instrument(skip(self), level = "debug")] + pub async fn state_get_shortid( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, + ) -> Result { + let shortstatekey = self + .services + .short + .get_shortstatekey(event_type, state_key) + .await?; + + let start = compress_state_event(shortstatekey, 0); + let end = compress_state_event(shortstatekey, u64::MAX); + self.load_full_state(shortstatehash) + .map_ok(|full_state| { + full_state + .range(start..end) + .next() + .copied() + .map(parse_compressed_state_event) + .map(at!(1)) + .ok_or(err!(Request(NotFound("Not found in room state")))) + }) + .await? + } + + /// Returns the state events removed between the interval (present in .0 but + /// not in .1) + #[inline] + pub fn state_removed( + &self, + shortstatehash: pair_of!(ShortStateHash), + ) -> impl Stream + Send + '_ { + self.state_added((shortstatehash.1, shortstatehash.0)) + } + + /// Returns the state events added between the interval (present in .1 but + /// not in .0) + #[tracing::instrument(skip(self), level = "debug")] + pub fn state_added<'a>( + &'a self, + shortstatehash: pair_of!(ShortStateHash), + ) -> impl Stream + Send + 'a { + let a = self.load_full_state(shortstatehash.0); + let b = self.load_full_state(shortstatehash.1); + try_join(a, b) + .map_ok(|(a, b)| b.difference(&a).copied().collect::>()) + .map_ok(IterStream::try_stream) + .try_flatten_stream() + .expect_ok() + .map(parse_compressed_state_event) + } + pub fn state_full( &self, shortstatehash: ShortStateHash, @@ -208,110 +458,11 @@ impl Service { .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) } - /// Returns a single EventId from `room_id` with key (`event_type`, - /// `state_key`). - #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_get_id( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - Id: for<'de> Deserialize<'de> + Sized + ToOwned, - ::Owned: Borrow, - { - let shorteventid = self - .state_get_shortid(shortstatehash, event_type, state_key) - .await?; - - self.services - .short - .get_eventid_from_short(shorteventid) - .await - } - - /// Returns a single EventId from `room_id` with key (`event_type`, - /// `state_key`). - #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_get_shortid( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result { - let shortstatekey = self - .services - .short - .get_shortstatekey(event_type, state_key) - .await?; - - let start = compress_state_event(shortstatekey, 0); - let end = compress_state_event(shortstatekey, u64::MAX); - self.services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) - .map_ok(|full_state| { - full_state - .range(start..end) - .next() - .copied() - .map(parse_compressed_state_event) - .map(at!(1)) - .ok_or(err!(Request(NotFound("Not found in room state")))) - }) - .await? - } - - #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_contains( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> bool { - let Ok(shortstatekey) = self - .services - .short - .get_shortstatekey(event_type, state_key) - .await - else { - return false; - }; - - self.state_contains_shortstatekey(shortstatehash, shortstatekey) - .await - } - - #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_contains_shortstatekey( - &self, - shortstatehash: ShortStateHash, - shortstatekey: ShortStateKey, - ) -> bool { - let start = compress_state_event(shortstatekey, 0); - let end = compress_state_event(shortstatekey, u64::MAX); - - self.services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) - .map_ok(|full_state| full_state.range(start..end).next().copied()) - .await - .flat_ok() - .is_some() - } - pub fn state_full_shortids( &self, shortstatehash: ShortStateHash, ) -> impl Stream> + Send + '_ { - self.services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .map_err(|e| err!(Database("Missing state IDs: {e}"))) - .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) + self.load_full_state(shortstatehash) .map_ok(|full_state| { full_state .deref() @@ -324,59 +475,32 @@ impl Service { .try_flatten_stream() } - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - pub async fn state_get( + async fn load_full_state( &self, shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result { - self.state_get_id(shortstatehash, event_type, state_key) - .and_then(|event_id: OwnedEventId| async move { - self.services.timeline.get_pdu(&event_id).await + ) -> Result> { + self.services + .state_compressor + .load_shortstatehash_info(shortstatehash) + .map_err(|e| err!(Database("Missing state IDs: {e}"))) + .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) + .await + } + + /// Returns the state hash for this pdu. + pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { + const BUFSIZE: usize = size_of::(); + + self.services + .short + .get_shorteventid(event_id) + .and_then(|shorteventid| { + self.db + .shorteventid_shortstatehash + .aqry::(&shorteventid) }) .await - } - - /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). - pub async fn state_get_content( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - T: for<'de> Deserialize<'de>, - { - self.state_get(shortstatehash, event_type, state_key) - .await - .and_then(|event| event.get_content()) - } - - /// Get membership for given user in state - async fn user_membership( - &self, - shortstatehash: ShortStateHash, - user_id: &UserId, - ) -> MembershipState { - self.state_get_content(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) - .await - .map_or(MembershipState::Leave, |c: RoomMemberEventContent| c.membership) - } - - /// The user was a joined member at this state (potentially in the past) - #[inline] - async fn user_was_joined(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { - self.user_membership(shortstatehash, user_id).await == MembershipState::Join - } - - /// The user was an invited or joined room member at this state (potentially - /// in the past) - #[inline] - async fn user_was_invited(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { - let s = self.user_membership(shortstatehash, user_id).await; - s == MembershipState::Join || s == MembershipState::Invite + .deserialized() } /// Whether a server is allowed to see an event through federation, based on @@ -521,101 +645,6 @@ impl Service { } } - /// Returns the state hash for this pdu. - pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { - const BUFSIZE: usize = size_of::(); - - self.services - .short - .get_shorteventid(event_id) - .and_then(|shorteventid| { - self.db - .shorteventid_shortstatehash - .aqry::(&shorteventid) - }) - .await - .deserialized() - } - - /// Returns the full room state. - #[tracing::instrument(skip(self), level = "debug")] - pub fn room_state_full<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream> + Send + 'a { - self.services - .state - .get_room_shortstatehash(room_id) - .map_ok(|shortstatehash| self.state_full(shortstatehash).map(Ok)) - .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) - .try_flatten_stream() - } - - /// Returns the full room state pdus - #[tracing::instrument(skip(self), level = "debug")] - pub fn room_state_full_pdus<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream> + Send + 'a { - self.services - .state - .get_room_shortstatehash(room_id) - .map_ok(|shortstatehash| self.state_full_pdus(shortstatehash).map(Ok)) - .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) - .try_flatten_stream() - } - - /// Returns a single EventId from `room_id` with key (`event_type`, - /// `state_key`). - #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_get_id( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - Id: for<'de> Deserialize<'de> + Sized + ToOwned, - ::Owned: Borrow, - { - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_get_id(shortstatehash, event_type, state_key)) - .await - } - - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_get( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, - ) -> Result { - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_get(shortstatehash, event_type, state_key)) - .await - } - - /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). - pub async fn room_state_get_content( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - T: for<'de> Deserialize<'de>, - { - self.room_state_get(room_id, event_type, state_key) - .await - .and_then(|event| event.get_content()) - } - pub async fn get_name(&self, room_id: &RoomId) -> Result { self.room_state_get_content(room_id, &StateEventType::RoomName, "") .await From b3271e0d653de1c585b1b5db95447045b0453b06 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 2 Feb 2025 17:27:39 +0000 Subject: [PATCH 1477/2091] split state_accessor Signed-off-by: Jason Volk --- src/service/rooms/state_accessor/mod.rs | 634 +----------------- .../rooms/state_accessor/room_state.rs | 90 +++ .../rooms/state_accessor/server_can.rs | 73 ++ src/service/rooms/state_accessor/state.rs | 320 +++++++++ src/service/rooms/state_accessor/user_can.rs | 187 ++++++ 5 files changed, 684 insertions(+), 620 deletions(-) create mode 100644 src/service/rooms/state_accessor/room_state.rs create mode 100644 src/service/rooms/state_accessor/server_can.rs create mode 100644 src/service/rooms/state_accessor/state.rs create mode 100644 src/service/rooms/state_accessor/user_can.rs diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index bed8d210..b7952ce6 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -1,23 +1,19 @@ +mod room_state; +mod server_can; +mod state; +mod user_can; + use std::{ - borrow::Borrow, fmt::Write, - ops::Deref, sync::{Arc, Mutex as StdMutex, Mutex}, }; use conduwuit::{ - at, err, error, pair_of, - pdu::PduBuilder, - utils, - utils::{ - math::{usize_from_f64, Expected}, - result::FlatOk, - stream::{BroadbandExt, IterStream, ReadyExt, TryExpect}, - }, - Err, Error, PduEvent, Result, + err, utils, + utils::math::{usize_from_f64, Expected}, + Result, }; -use database::{Deserialized, Map}; -use futures::{future::try_join, FutureExt, Stream, StreamExt, TryFutureExt}; +use database::Map; use lru_cache::LruCache; use ruma::{ events::{ @@ -29,29 +25,19 @@ use ruma::{ guest_access::{GuestAccess, RoomGuestAccessEventContent}, history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent, RoomMembership}, - member::{MembershipState, RoomMemberEventContent}, + member::RoomMemberEventContent, name::RoomNameEventContent, - power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, topic::RoomTopicEventContent, }, - StateEventType, TimelineEventType, + StateEventType, }, room::RoomType, space::SpaceRoomJoinRule, - EventEncryptionAlgorithm, EventId, JsOption, OwnedEventId, OwnedRoomAliasId, OwnedRoomId, - OwnedServerName, OwnedUserId, RoomId, ServerName, UserId, + EventEncryptionAlgorithm, JsOption, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, + OwnedUserId, RoomId, UserId, }; -use serde::Deserialize; -use crate::{ - rooms, - rooms::{ - short::{ShortEventId, ShortStateHash, ShortStateKey}, - state::RoomMutexGuard, - state_compressor::{compress_state_event, parse_compressed_state_event, CompressedState}, - }, - Dep, -}; +use crate::{rooms, rooms::short::ShortStateHash, Dep}; pub struct Service { pub server_visibility_cache: Mutex>, @@ -143,508 +129,6 @@ impl crate::Service for Service { } impl Service { - /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). - pub async fn room_state_get_content( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - T: for<'de> Deserialize<'de>, - { - self.room_state_get(room_id, event_type, state_key) - .await - .and_then(|event| event.get_content()) - } - - /// Returns the full room state. - #[tracing::instrument(skip(self), level = "debug")] - pub fn room_state_full<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream> + Send + 'a { - self.services - .state - .get_room_shortstatehash(room_id) - .map_ok(|shortstatehash| self.state_full(shortstatehash).map(Ok)) - .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) - .try_flatten_stream() - } - - /// Returns the full room state pdus - #[tracing::instrument(skip(self), level = "debug")] - pub fn room_state_full_pdus<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream> + Send + 'a { - self.services - .state - .get_room_shortstatehash(room_id) - .map_ok(|shortstatehash| self.state_full_pdus(shortstatehash).map(Ok)) - .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) - .try_flatten_stream() - } - - /// Returns a single EventId from `room_id` with key (`event_type`, - /// `state_key`). - #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_get_id( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - Id: for<'de> Deserialize<'de> + Sized + ToOwned, - ::Owned: Borrow, - { - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_get_id(shortstatehash, event_type, state_key)) - .await - } - - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - #[tracing::instrument(skip(self), level = "debug")] - pub async fn room_state_get( - &self, - room_id: &RoomId, - event_type: &StateEventType, - state_key: &str, - ) -> Result { - self.services - .state - .get_room_shortstatehash(room_id) - .and_then(|shortstatehash| self.state_get(shortstatehash, event_type, state_key)) - .await - } - - /// The user was a joined member at this state (potentially in the past) - #[inline] - async fn user_was_joined(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { - self.user_membership(shortstatehash, user_id).await == MembershipState::Join - } - - /// The user was an invited or joined room member at this state (potentially - /// in the past) - #[inline] - async fn user_was_invited(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { - let s = self.user_membership(shortstatehash, user_id).await; - s == MembershipState::Join || s == MembershipState::Invite - } - - /// Get membership for given user in state - async fn user_membership( - &self, - shortstatehash: ShortStateHash, - user_id: &UserId, - ) -> MembershipState { - self.state_get_content(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) - .await - .map_or(MembershipState::Leave, |c: RoomMemberEventContent| c.membership) - } - - /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). - pub async fn state_get_content( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - T: for<'de> Deserialize<'de>, - { - self.state_get(shortstatehash, event_type, state_key) - .await - .and_then(|event| event.get_content()) - } - - #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_contains( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> bool { - let Ok(shortstatekey) = self - .services - .short - .get_shortstatekey(event_type, state_key) - .await - else { - return false; - }; - - self.state_contains_shortstatekey(shortstatehash, shortstatekey) - .await - } - - #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_contains_shortstatekey( - &self, - shortstatehash: ShortStateHash, - shortstatekey: ShortStateKey, - ) -> bool { - let start = compress_state_event(shortstatekey, 0); - let end = compress_state_event(shortstatekey, u64::MAX); - - self.load_full_state(shortstatehash) - .map_ok(|full_state| full_state.range(start..end).next().copied()) - .await - .flat_ok() - .is_some() - } - - /// Returns a single PDU from `room_id` with key (`event_type`, - /// `state_key`). - pub async fn state_get( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result { - self.state_get_id(shortstatehash, event_type, state_key) - .and_then(|event_id: OwnedEventId| async move { - self.services.timeline.get_pdu(&event_id).await - }) - .await - } - - /// Returns a single EventId from `room_id` with key (`event_type`, - /// `state_key`). - #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_get_id( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result - where - Id: for<'de> Deserialize<'de> + Sized + ToOwned, - ::Owned: Borrow, - { - let shorteventid = self - .state_get_shortid(shortstatehash, event_type, state_key) - .await?; - - self.services - .short - .get_eventid_from_short(shorteventid) - .await - } - - /// Returns a single EventId from `room_id` with key (`event_type`, - /// `state_key`). - #[tracing::instrument(skip(self), level = "debug")] - pub async fn state_get_shortid( - &self, - shortstatehash: ShortStateHash, - event_type: &StateEventType, - state_key: &str, - ) -> Result { - let shortstatekey = self - .services - .short - .get_shortstatekey(event_type, state_key) - .await?; - - let start = compress_state_event(shortstatekey, 0); - let end = compress_state_event(shortstatekey, u64::MAX); - self.load_full_state(shortstatehash) - .map_ok(|full_state| { - full_state - .range(start..end) - .next() - .copied() - .map(parse_compressed_state_event) - .map(at!(1)) - .ok_or(err!(Request(NotFound("Not found in room state")))) - }) - .await? - } - - /// Returns the state events removed between the interval (present in .0 but - /// not in .1) - #[inline] - pub fn state_removed( - &self, - shortstatehash: pair_of!(ShortStateHash), - ) -> impl Stream + Send + '_ { - self.state_added((shortstatehash.1, shortstatehash.0)) - } - - /// Returns the state events added between the interval (present in .1 but - /// not in .0) - #[tracing::instrument(skip(self), level = "debug")] - pub fn state_added<'a>( - &'a self, - shortstatehash: pair_of!(ShortStateHash), - ) -> impl Stream + Send + 'a { - let a = self.load_full_state(shortstatehash.0); - let b = self.load_full_state(shortstatehash.1); - try_join(a, b) - .map_ok(|(a, b)| b.difference(&a).copied().collect::>()) - .map_ok(IterStream::try_stream) - .try_flatten_stream() - .expect_ok() - .map(parse_compressed_state_event) - } - - pub fn state_full( - &self, - shortstatehash: ShortStateHash, - ) -> impl Stream + Send + '_ { - self.state_full_pdus(shortstatehash) - .ready_filter_map(|pdu| { - Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu)) - }) - } - - pub fn state_full_pdus( - &self, - shortstatehash: ShortStateHash, - ) -> impl Stream + Send + '_ { - let short_ids = self - .state_full_shortids(shortstatehash) - .expect_ok() - .map(at!(1)); - - self.services - .short - .multi_get_eventid_from_short(short_ids) - .ready_filter_map(Result::ok) - .broad_filter_map(move |event_id: OwnedEventId| async move { - self.services.timeline.get_pdu(&event_id).await.ok() - }) - } - - /// Builds a StateMap by iterating over all keys that start - /// with state_hash, this gives the full state for the given state_hash. - #[tracing::instrument(skip(self), level = "debug")] - pub fn state_full_ids<'a, Id>( - &'a self, - shortstatehash: ShortStateHash, - ) -> impl Stream + Send + 'a - where - Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a, - ::Owned: Borrow, - { - let shortids = self - .state_full_shortids(shortstatehash) - .expect_ok() - .unzip() - .shared(); - - let shortstatekeys = shortids - .clone() - .map(at!(0)) - .map(Vec::into_iter) - .map(IterStream::stream) - .flatten_stream(); - - let shorteventids = shortids - .map(at!(1)) - .map(Vec::into_iter) - .map(IterStream::stream) - .flatten_stream(); - - self.services - .short - .multi_get_eventid_from_short(shorteventids) - .zip(shortstatekeys) - .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) - } - - pub fn state_full_shortids( - &self, - shortstatehash: ShortStateHash, - ) -> impl Stream> + Send + '_ { - self.load_full_state(shortstatehash) - .map_ok(|full_state| { - full_state - .deref() - .iter() - .copied() - .map(parse_compressed_state_event) - .collect() - }) - .map_ok(|vec: Vec<_>| vec.into_iter().try_stream()) - .try_flatten_stream() - } - - async fn load_full_state( - &self, - shortstatehash: ShortStateHash, - ) -> Result> { - self.services - .state_compressor - .load_shortstatehash_info(shortstatehash) - .map_err(|e| err!(Database("Missing state IDs: {e}"))) - .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) - .await - } - - /// Returns the state hash for this pdu. - pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { - const BUFSIZE: usize = size_of::(); - - self.services - .short - .get_shorteventid(event_id) - .and_then(|shorteventid| { - self.db - .shorteventid_shortstatehash - .aqry::(&shorteventid) - }) - .await - .deserialized() - } - - /// Whether a server is allowed to see an event through federation, based on - /// the room's history_visibility at that event's state. - #[tracing::instrument(skip_all, level = "trace")] - pub async fn server_can_see_event( - &self, - origin: &ServerName, - room_id: &RoomId, - event_id: &EventId, - ) -> bool { - let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { - return true; - }; - - if let Some(visibility) = self - .server_visibility_cache - .lock() - .expect("locked") - .get_mut(&(origin.to_owned(), shortstatehash)) - { - return *visibility; - } - - let history_visibility = self - .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") - .await - .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { - c.history_visibility - }); - - let current_server_members = self - .services - .state_cache - .room_members(room_id) - .ready_filter(|member| member.server_name() == origin); - - let visibility = match history_visibility { - | HistoryVisibility::WorldReadable | HistoryVisibility::Shared => true, - | HistoryVisibility::Invited => { - // Allow if any member on requesting server was AT LEAST invited, else deny - current_server_members - .any(|member| self.user_was_invited(shortstatehash, member)) - .await - }, - | HistoryVisibility::Joined => { - // Allow if any member on requested server was joined, else deny - current_server_members - .any(|member| self.user_was_joined(shortstatehash, member)) - .await - }, - | _ => { - error!("Unknown history visibility {history_visibility}"); - false - }, - }; - - self.server_visibility_cache - .lock() - .expect("locked") - .insert((origin.to_owned(), shortstatehash), visibility); - - visibility - } - - /// Whether a user is allowed to see an event, based on - /// the room's history_visibility at that event's state. - #[tracing::instrument(skip_all, level = "trace")] - pub async fn user_can_see_event( - &self, - user_id: &UserId, - room_id: &RoomId, - event_id: &EventId, - ) -> bool { - let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { - return true; - }; - - if let Some(visibility) = self - .user_visibility_cache - .lock() - .expect("locked") - .get_mut(&(user_id.to_owned(), shortstatehash)) - { - return *visibility; - } - - let currently_member = self.services.state_cache.is_joined(user_id, room_id).await; - - let history_visibility = self - .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") - .await - .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { - c.history_visibility - }); - - let visibility = match history_visibility { - | HistoryVisibility::WorldReadable => true, - | HistoryVisibility::Shared => currently_member, - | HistoryVisibility::Invited => { - // Allow if any member on requesting server was AT LEAST invited, else deny - self.user_was_invited(shortstatehash, user_id).await - }, - | HistoryVisibility::Joined => { - // Allow if any member on requested server was joined, else deny - self.user_was_joined(shortstatehash, user_id).await - }, - | _ => { - error!("Unknown history visibility {history_visibility}"); - false - }, - }; - - self.user_visibility_cache - .lock() - .expect("locked") - .insert((user_id.to_owned(), shortstatehash), visibility); - - visibility - } - - /// Whether a user is allowed to see an event, based on - /// the room's history_visibility at that event's state. - #[tracing::instrument(skip_all, level = "trace")] - pub async fn user_can_see_state_events(&self, user_id: &UserId, room_id: &RoomId) -> bool { - if self.services.state_cache.is_joined(user_id, room_id).await { - return true; - } - - let history_visibility = self - .room_state_get_content(room_id, &StateEventType::RoomHistoryVisibility, "") - .await - .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { - c.history_visibility - }); - - match history_visibility { - | HistoryVisibility::Invited => - self.services.state_cache.is_invited(user_id, room_id).await, - | HistoryVisibility::WorldReadable => true, - | _ => false, - } - } - pub async fn get_name(&self, room_id: &RoomId) -> Result { self.room_state_get_content(room_id, &StateEventType::RoomName, "") .await @@ -669,28 +153,6 @@ impl Service { .await } - pub async fn user_can_invite( - &self, - room_id: &RoomId, - sender: &UserId, - target_user: &UserId, - state_lock: &RoomMutexGuard, - ) -> bool { - self.services - .timeline - .create_hash_and_sign_event( - PduBuilder::state( - target_user.into(), - &RoomMemberEventContent::new(MembershipState::Invite), - ), - sender, - room_id, - state_lock, - ) - .await - .is_ok() - } - /// Checks if guests are able to view room content without joining pub async fn is_world_readable(&self, room_id: &RoomId) -> bool { self.room_state_get_content(room_id, &StateEventType::RoomHistoryVisibility, "") @@ -726,74 +188,6 @@ impl Service { .map(|c: RoomTopicEventContent| c.topic) } - /// Checks if a given user can redact a given event - /// - /// If federation is true, it allows redaction events from any user of the - /// same server as the original event sender - pub async fn user_can_redact( - &self, - redacts: &EventId, - sender: &UserId, - room_id: &RoomId, - federation: bool, - ) -> Result { - let redacting_event = self.services.timeline.get_pdu(redacts).await; - - if redacting_event - .as_ref() - .is_ok_and(|pdu| pdu.kind == TimelineEventType::RoomCreate) - { - return Err!(Request(Forbidden("Redacting m.room.create is not safe, forbidding."))); - } - - if redacting_event - .as_ref() - .is_ok_and(|pdu| pdu.kind == TimelineEventType::RoomServerAcl) - { - return Err!(Request(Forbidden( - "Redacting m.room.server_acl will result in the room being inaccessible for \ - everyone (empty allow key), forbidding." - ))); - } - - if let Ok(pl_event_content) = self - .room_state_get_content::( - room_id, - &StateEventType::RoomPowerLevels, - "", - ) - .await - { - let pl_event: RoomPowerLevels = pl_event_content.into(); - Ok(pl_event.user_can_redact_event_of_other(sender) - || pl_event.user_can_redact_own_event(sender) - && if let Ok(redacting_event) = redacting_event { - if federation { - redacting_event.sender.server_name() == sender.server_name() - } else { - redacting_event.sender == sender - } - } else { - false - }) - } else { - // Falling back on m.room.create to judge power level - if let Ok(room_create) = self - .room_state_get(room_id, &StateEventType::RoomCreate, "") - .await - { - Ok(room_create.sender == sender - || redacting_event - .as_ref() - .is_ok_and(|redacting_event| redacting_event.sender == sender)) - } else { - Err(Error::bad_database( - "No m.room.power_levels or m.room.create events in database for room", - )) - } - } - } - /// Returns the join rule (`SpaceRoomJoinRule`) for a given room pub async fn get_join_rule( &self, diff --git a/src/service/rooms/state_accessor/room_state.rs b/src/service/rooms/state_accessor/room_state.rs new file mode 100644 index 00000000..98a82cea --- /dev/null +++ b/src/service/rooms/state_accessor/room_state.rs @@ -0,0 +1,90 @@ +use std::borrow::Borrow; + +use conduwuit::{err, implement, PduEvent, Result}; +use futures::{Stream, StreamExt, TryFutureExt}; +use ruma::{events::StateEventType, EventId, RoomId}; +use serde::Deserialize; + +/// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). +#[implement(super::Service)] +pub async fn room_state_get_content( + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, +) -> Result +where + T: for<'de> Deserialize<'de>, +{ + self.room_state_get(room_id, event_type, state_key) + .await + .and_then(|event| event.get_content()) +} + +/// Returns the full room state. +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn room_state_full<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream> + Send + 'a { + self.services + .state + .get_room_shortstatehash(room_id) + .map_ok(|shortstatehash| self.state_full(shortstatehash).map(Ok)) + .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .try_flatten_stream() +} + +/// Returns the full room state pdus +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn room_state_full_pdus<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream> + Send + 'a { + self.services + .state + .get_room_shortstatehash(room_id) + .map_ok(|shortstatehash| self.state_full_pdus(shortstatehash).map(Ok)) + .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) + .try_flatten_stream() +} + +/// Returns a single EventId from `room_id` with key (`event_type`, +/// `state_key`). +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub async fn room_state_get_id( + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, +) -> Result +where + Id: for<'de> Deserialize<'de> + Sized + ToOwned, + ::Owned: Borrow, +{ + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get_id(shortstatehash, event_type, state_key)) + .await +} + +/// Returns a single PDU from `room_id` with key (`event_type`, +/// `state_key`). +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub async fn room_state_get( + &self, + room_id: &RoomId, + event_type: &StateEventType, + state_key: &str, +) -> Result { + self.services + .state + .get_room_shortstatehash(room_id) + .and_then(|shortstatehash| self.state_get(shortstatehash, event_type, state_key)) + .await +} diff --git a/src/service/rooms/state_accessor/server_can.rs b/src/service/rooms/state_accessor/server_can.rs new file mode 100644 index 00000000..4d834227 --- /dev/null +++ b/src/service/rooms/state_accessor/server_can.rs @@ -0,0 +1,73 @@ +use conduwuit::{error, implement, utils::stream::ReadyExt}; +use futures::StreamExt; +use ruma::{ + events::{ + room::history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + StateEventType, + }, + EventId, RoomId, ServerName, +}; + +/// Whether a server is allowed to see an event through federation, based on +/// the room's history_visibility at that event's state. +#[implement(super::Service)] +#[tracing::instrument(skip_all, level = "trace")] +pub async fn server_can_see_event( + &self, + origin: &ServerName, + room_id: &RoomId, + event_id: &EventId, +) -> bool { + let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { + return true; + }; + + if let Some(visibility) = self + .server_visibility_cache + .lock() + .expect("locked") + .get_mut(&(origin.to_owned(), shortstatehash)) + { + return *visibility; + } + + let history_visibility = self + .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") + .await + .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { + c.history_visibility + }); + + let current_server_members = self + .services + .state_cache + .room_members(room_id) + .ready_filter(|member| member.server_name() == origin); + + let visibility = match history_visibility { + | HistoryVisibility::WorldReadable | HistoryVisibility::Shared => true, + | HistoryVisibility::Invited => { + // Allow if any member on requesting server was AT LEAST invited, else deny + current_server_members + .any(|member| self.user_was_invited(shortstatehash, member)) + .await + }, + | HistoryVisibility::Joined => { + // Allow if any member on requested server was joined, else deny + current_server_members + .any(|member| self.user_was_joined(shortstatehash, member)) + .await + }, + | _ => { + error!("Unknown history visibility {history_visibility}"); + false + }, + }; + + self.server_visibility_cache + .lock() + .expect("locked") + .insert((origin.to_owned(), shortstatehash), visibility); + + visibility +} diff --git a/src/service/rooms/state_accessor/state.rs b/src/service/rooms/state_accessor/state.rs new file mode 100644 index 00000000..c47a5693 --- /dev/null +++ b/src/service/rooms/state_accessor/state.rs @@ -0,0 +1,320 @@ +use std::{borrow::Borrow, ops::Deref, sync::Arc}; + +use conduwuit::{ + at, err, implement, pair_of, + utils::{ + result::FlatOk, + stream::{BroadbandExt, IterStream, ReadyExt, TryExpect}, + }, + PduEvent, Result, +}; +use database::Deserialized; +use futures::{future::try_join, FutureExt, Stream, StreamExt, TryFutureExt}; +use ruma::{ + events::{ + room::member::{MembershipState, RoomMemberEventContent}, + StateEventType, + }, + EventId, OwnedEventId, UserId, +}; +use serde::Deserialize; + +use crate::rooms::{ + short::{ShortEventId, ShortStateHash, ShortStateKey}, + state_compressor::{compress_state_event, parse_compressed_state_event, CompressedState}, +}; + +/// The user was a joined member at this state (potentially in the past) +#[implement(super::Service)] +#[inline] +pub async fn user_was_joined(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { + self.user_membership(shortstatehash, user_id).await == MembershipState::Join +} + +/// The user was an invited or joined room member at this state (potentially +/// in the past) +#[implement(super::Service)] +#[inline] +pub async fn user_was_invited(&self, shortstatehash: ShortStateHash, user_id: &UserId) -> bool { + let s = self.user_membership(shortstatehash, user_id).await; + s == MembershipState::Join || s == MembershipState::Invite +} + +/// Get membership for given user in state +#[implement(super::Service)] +pub async fn user_membership( + &self, + shortstatehash: ShortStateHash, + user_id: &UserId, +) -> MembershipState { + self.state_get_content(shortstatehash, &StateEventType::RoomMember, user_id.as_str()) + .await + .map_or(MembershipState::Leave, |c: RoomMemberEventContent| c.membership) +} + +/// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). +#[implement(super::Service)] +pub async fn state_get_content( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, +) -> Result +where + T: for<'de> Deserialize<'de>, +{ + self.state_get(shortstatehash, event_type, state_key) + .await + .and_then(|event| event.get_content()) +} + +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub async fn state_contains( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, +) -> bool { + let Ok(shortstatekey) = self + .services + .short + .get_shortstatekey(event_type, state_key) + .await + else { + return false; + }; + + self.state_contains_shortstatekey(shortstatehash, shortstatekey) + .await +} + +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub async fn state_contains_shortstatekey( + &self, + shortstatehash: ShortStateHash, + shortstatekey: ShortStateKey, +) -> bool { + let start = compress_state_event(shortstatekey, 0); + let end = compress_state_event(shortstatekey, u64::MAX); + + self.load_full_state(shortstatehash) + .map_ok(|full_state| full_state.range(start..=end).next().copied()) + .await + .flat_ok() + .is_some() +} + +/// Returns a single PDU from `room_id` with key (`event_type`, +/// `state_key`). +#[implement(super::Service)] +pub async fn state_get( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, +) -> Result { + self.state_get_id(shortstatehash, event_type, state_key) + .and_then(|event_id: OwnedEventId| async move { + self.services.timeline.get_pdu(&event_id).await + }) + .await +} + +/// Returns a single EventId from `room_id` with key (`event_type`, +/// `state_key`). +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub async fn state_get_id( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, +) -> Result +where + Id: for<'de> Deserialize<'de> + Sized + ToOwned, + ::Owned: Borrow, +{ + let shorteventid = self + .state_get_shortid(shortstatehash, event_type, state_key) + .await?; + + self.services + .short + .get_eventid_from_short(shorteventid) + .await +} + +/// Returns a single EventId from `room_id` with key (`event_type`, +/// `state_key`). +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub async fn state_get_shortid( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, + state_key: &str, +) -> Result { + let shortstatekey = self + .services + .short + .get_shortstatekey(event_type, state_key) + .await?; + + let start = compress_state_event(shortstatekey, 0); + let end = compress_state_event(shortstatekey, u64::MAX); + self.load_full_state(shortstatehash) + .map_ok(|full_state| { + full_state + .range(start..=end) + .next() + .copied() + .map(parse_compressed_state_event) + .map(at!(1)) + .ok_or(err!(Request(NotFound("Not found in room state")))) + }) + .await? +} + +/// Returns the state events removed between the interval (present in .0 but +/// not in .1) +#[implement(super::Service)] +#[inline] +pub fn state_removed( + &self, + shortstatehash: pair_of!(ShortStateHash), +) -> impl Stream + Send + '_ { + self.state_added((shortstatehash.1, shortstatehash.0)) +} + +/// Returns the state events added between the interval (present in .1 but +/// not in .0) +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn state_added<'a>( + &'a self, + shortstatehash: pair_of!(ShortStateHash), +) -> impl Stream + Send + 'a { + let a = self.load_full_state(shortstatehash.0); + let b = self.load_full_state(shortstatehash.1); + try_join(a, b) + .map_ok(|(a, b)| b.difference(&a).copied().collect::>()) + .map_ok(IterStream::try_stream) + .try_flatten_stream() + .expect_ok() + .map(parse_compressed_state_event) +} + +#[implement(super::Service)] +pub fn state_full( + &self, + shortstatehash: ShortStateHash, +) -> impl Stream + Send + '_ { + self.state_full_pdus(shortstatehash) + .ready_filter_map(|pdu| { + Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu)) + }) +} + +#[implement(super::Service)] +pub fn state_full_pdus( + &self, + shortstatehash: ShortStateHash, +) -> impl Stream + Send + '_ { + let short_ids = self + .state_full_shortids(shortstatehash) + .expect_ok() + .map(at!(1)); + + self.services + .short + .multi_get_eventid_from_short(short_ids) + .ready_filter_map(Result::ok) + .broad_filter_map(move |event_id: OwnedEventId| async move { + self.services.timeline.get_pdu(&event_id).await.ok() + }) +} + +/// Builds a StateMap by iterating over all keys that start +/// with state_hash, this gives the full state for the given state_hash. +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn state_full_ids<'a, Id>( + &'a self, + shortstatehash: ShortStateHash, +) -> impl Stream + Send + 'a +where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a, + ::Owned: Borrow, +{ + let shortids = self + .state_full_shortids(shortstatehash) + .expect_ok() + .unzip() + .shared(); + + let shortstatekeys = shortids + .clone() + .map(at!(0)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + let shorteventids = shortids + .map(at!(1)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + self.services + .short + .multi_get_eventid_from_short(shorteventids) + .zip(shortstatekeys) + .ready_filter_map(|(event_id, shortstatekey)| Some((shortstatekey, event_id.ok()?))) +} + +#[implement(super::Service)] +pub fn state_full_shortids( + &self, + shortstatehash: ShortStateHash, +) -> impl Stream> + Send + '_ { + self.load_full_state(shortstatehash) + .map_ok(|full_state| { + full_state + .deref() + .iter() + .copied() + .map(parse_compressed_state_event) + .collect() + }) + .map_ok(|vec: Vec<_>| vec.into_iter().try_stream()) + .try_flatten_stream() +} + +#[implement(super::Service)] +async fn load_full_state(&self, shortstatehash: ShortStateHash) -> Result> { + self.services + .state_compressor + .load_shortstatehash_info(shortstatehash) + .map_err(|e| err!(Database("Missing state IDs: {e}"))) + .map_ok(|vec| vec.last().expect("at least one layer").full_state.clone()) + .await +} + +/// Returns the state hash for this pdu. +#[implement(super::Service)] +pub async fn pdu_shortstatehash(&self, event_id: &EventId) -> Result { + const BUFSIZE: usize = size_of::(); + + self.services + .short + .get_shorteventid(event_id) + .and_then(|shorteventid| { + self.db + .shorteventid_shortstatehash + .aqry::(&shorteventid) + }) + .await + .deserialized() +} diff --git a/src/service/rooms/state_accessor/user_can.rs b/src/service/rooms/state_accessor/user_can.rs new file mode 100644 index 00000000..725a4fba --- /dev/null +++ b/src/service/rooms/state_accessor/user_can.rs @@ -0,0 +1,187 @@ +use conduwuit::{error, implement, pdu::PduBuilder, Err, Error, Result}; +use ruma::{ + events::{ + room::{ + history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + member::{MembershipState, RoomMemberEventContent}, + power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, + }, + StateEventType, TimelineEventType, + }, + EventId, RoomId, UserId, +}; + +use crate::rooms::state::RoomMutexGuard; + +/// Checks if a given user can redact a given event +/// +/// If federation is true, it allows redaction events from any user of the +/// same server as the original event sender +#[implement(super::Service)] +pub async fn user_can_redact( + &self, + redacts: &EventId, + sender: &UserId, + room_id: &RoomId, + federation: bool, +) -> Result { + let redacting_event = self.services.timeline.get_pdu(redacts).await; + + if redacting_event + .as_ref() + .is_ok_and(|pdu| pdu.kind == TimelineEventType::RoomCreate) + { + return Err!(Request(Forbidden("Redacting m.room.create is not safe, forbidding."))); + } + + if redacting_event + .as_ref() + .is_ok_and(|pdu| pdu.kind == TimelineEventType::RoomServerAcl) + { + return Err!(Request(Forbidden( + "Redacting m.room.server_acl will result in the room being inaccessible for \ + everyone (empty allow key), forbidding." + ))); + } + + if let Ok(pl_event_content) = self + .room_state_get_content::( + room_id, + &StateEventType::RoomPowerLevels, + "", + ) + .await + { + let pl_event: RoomPowerLevels = pl_event_content.into(); + Ok(pl_event.user_can_redact_event_of_other(sender) + || pl_event.user_can_redact_own_event(sender) + && if let Ok(redacting_event) = redacting_event { + if federation { + redacting_event.sender.server_name() == sender.server_name() + } else { + redacting_event.sender == sender + } + } else { + false + }) + } else { + // Falling back on m.room.create to judge power level + if let Ok(room_create) = self + .room_state_get(room_id, &StateEventType::RoomCreate, "") + .await + { + Ok(room_create.sender == sender + || redacting_event + .as_ref() + .is_ok_and(|redacting_event| redacting_event.sender == sender)) + } else { + Err(Error::bad_database( + "No m.room.power_levels or m.room.create events in database for room", + )) + } + } +} + +/// Whether a user is allowed to see an event, based on +/// the room's history_visibility at that event's state. +#[implement(super::Service)] +#[tracing::instrument(skip_all, level = "trace")] +pub async fn user_can_see_event( + &self, + user_id: &UserId, + room_id: &RoomId, + event_id: &EventId, +) -> bool { + let Ok(shortstatehash) = self.pdu_shortstatehash(event_id).await else { + return true; + }; + + if let Some(visibility) = self + .user_visibility_cache + .lock() + .expect("locked") + .get_mut(&(user_id.to_owned(), shortstatehash)) + { + return *visibility; + } + + let currently_member = self.services.state_cache.is_joined(user_id, room_id).await; + + let history_visibility = self + .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") + .await + .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { + c.history_visibility + }); + + let visibility = match history_visibility { + | HistoryVisibility::WorldReadable => true, + | HistoryVisibility::Shared => currently_member, + | HistoryVisibility::Invited => { + // Allow if any member on requesting server was AT LEAST invited, else deny + self.user_was_invited(shortstatehash, user_id).await + }, + | HistoryVisibility::Joined => { + // Allow if any member on requested server was joined, else deny + self.user_was_joined(shortstatehash, user_id).await + }, + | _ => { + error!("Unknown history visibility {history_visibility}"); + false + }, + }; + + self.user_visibility_cache + .lock() + .expect("locked") + .insert((user_id.to_owned(), shortstatehash), visibility); + + visibility +} + +/// Whether a user is allowed to see an event, based on +/// the room's history_visibility at that event's state. +#[implement(super::Service)] +#[tracing::instrument(skip_all, level = "trace")] +pub async fn user_can_see_state_events(&self, user_id: &UserId, room_id: &RoomId) -> bool { + if self.services.state_cache.is_joined(user_id, room_id).await { + return true; + } + + let history_visibility = self + .room_state_get_content(room_id, &StateEventType::RoomHistoryVisibility, "") + .await + .map_or(HistoryVisibility::Shared, |c: RoomHistoryVisibilityEventContent| { + c.history_visibility + }); + + match history_visibility { + | HistoryVisibility::Invited => + self.services.state_cache.is_invited(user_id, room_id).await, + | HistoryVisibility::WorldReadable => true, + | _ => false, + } +} + +#[implement(super::Service)] +pub async fn user_can_invite( + &self, + room_id: &RoomId, + sender: &UserId, + target_user: &UserId, + state_lock: &RoomMutexGuard, +) -> bool { + self.services + .timeline + .create_hash_and_sign_event( + PduBuilder::state( + target_user.into(), + &RoomMemberEventContent::new(MembershipState::Invite), + ), + sender, + room_id, + state_lock, + ) + .await + .is_ok() +} From d32534164c0092a30ac351337b7dd34aa8f5d456 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 4 Feb 2025 20:30:33 +0000 Subject: [PATCH 1478/2091] fix soft-failed redaction regression (ff8bbd4cfa) Signed-off-by: Jason Volk --- src/service/rooms/event_handler/upgrade_outlier_pdu.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 132daca7..b33b0388 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -128,7 +128,8 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( | (false, _) => true, | (true, None) => false, | (true, Some(redact_id)) => - self.services + !self + .services .state_accessor .user_can_redact(&redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) .await?, From 80277f6aa2629a8b9dc2b4e96a64d8e508d47270 Mon Sep 17 00:00:00 2001 From: Nineko Date: Tue, 4 Feb 2025 16:46:00 -0500 Subject: [PATCH 1479/2091] Adds .gitattributes to the projects to prevent LN and CLRF conflicts. (#681) --- .gitattributes | 87 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..3dfaca65 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,87 @@ +# taken from https://github.com/gitattributes/gitattributes/blob/46a8961ad73f5bd4d8d193708840fbc9e851d702/Rust.gitattributes +# Auto detect text files and perform normalization +* text=auto + +*.rs text diff=rust +*.toml text diff=toml +Cargo.lock text + +# taken from https://github.com/gitattributes/gitattributes/blob/46a8961ad73f5bd4d8d193708840fbc9e851d702/Common.gitattributes +# Documents +*.bibtex text diff=bibtex +*.doc diff=astextplain +*.DOC diff=astextplain +*.docx diff=astextplain +*.DOCX diff=astextplain +*.dot diff=astextplain +*.DOT diff=astextplain +*.pdf diff=astextplain +*.PDF diff=astextplain +*.rtf diff=astextplain +*.RTF diff=astextplain +*.md text diff=markdown +*.mdx text diff=markdown +*.tex text diff=tex +*.adoc text +*.textile text +*.mustache text +*.csv text eol=crlf +*.tab text +*.tsv text +*.txt text +*.sql text +*.epub diff=astextplain + +# Graphics +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.tif binary +*.tiff binary +*.ico binary +# SVG treated as text by default. +*.svg text +*.eps binary + +# Scripts +*.bash text eol=lf +*.fish text eol=lf +*.ksh text eol=lf +*.sh text eol=lf +*.zsh text eol=lf +# These are explicitly windows files and should use crlf +*.bat text eol=crlf +*.cmd text eol=crlf +*.ps1 text eol=crlf + +# Serialisation +*.json text +*.toml text +*.xml text +*.yaml text +*.yml text + +# Archives +*.7z binary +*.bz binary +*.bz2 binary +*.bzip2 binary +*.gz binary +*.lz binary +*.lzma binary +*.rar binary +*.tar binary +*.taz binary +*.tbz binary +*.tbz2 binary +*.tgz binary +*.tlz binary +*.txz binary +*.xz binary +*.Z binary +*.zip binary +*.zst binary + +# Text files where line endings should be preserved +*.patch -text \ No newline at end of file From 62180897c02d9c306b2179f3685e60ffdc615c1f Mon Sep 17 00:00:00 2001 From: Niko Date: Sat, 1 Feb 2025 18:35:23 -0500 Subject: [PATCH 1480/2091] Added blurhash.rs to fascilitate blurhashing. Signed-off-by: Niko --- Cargo.lock | 373 +++++++++++++++++++++++++++++++++- Cargo.toml | 8 +- conduwuit-example.toml | 18 ++ src/api/Cargo.toml | 1 + src/api/client/media.rs | 21 ++ src/core/Cargo.toml | 1 + src/core/config/mod.rs | 40 +++- src/main/Cargo.toml | 1 + src/service/Cargo.toml | 3 + src/service/media/blurhash.rs | 159 +++++++++++++++ src/service/media/mod.rs | 3 +- 11 files changed, 621 insertions(+), 7 deletions(-) create mode 100644 src/service/media/blurhash.rs diff --git a/Cargo.lock b/Cargo.lock index e379aebb..b710d6fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -53,12 +59,29 @@ version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" + [[package]] name = "arc-swap" version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "argon2" version = "0.5.3" @@ -173,6 +196,29 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "av1-grain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e335041290c43101ca215eed6f43ec437eb5a42125573f600fc3fa42b9bddd62" +dependencies = [ + "arrayvec", +] + [[package]] name = "aws-lc-rs" version = "1.12.1" @@ -385,6 +431,12 @@ dependencies = [ "which", ] +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -397,6 +449,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +[[package]] +name = "bitstream-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6099cdc01846bc367c4e7dd630dc5966dccf36b652fae7a74e17b640411a91b2" + [[package]] name = "blake2" version = "0.10.6" @@ -415,6 +473,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blurhash" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e79769241dcd44edf79a732545e8b5cec84c247ac060f5252cd51885d093a8fc" +dependencies = [ + "image", +] + [[package]] name = "brotli" version = "7.0.0" @@ -436,6 +503,12 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "built" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" + [[package]] name = "bumpalo" version = "3.16.0" @@ -513,6 +586,16 @@ dependencies = [ "nom", ] +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -822,6 +905,7 @@ dependencies = [ "arrayvec", "async-trait", "base64 0.22.1", + "blurhash", "bytes", "conduwuit_core", "conduwuit_database", @@ -1071,6 +1155,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" + [[package]] name = "crypto-common" version = "0.1.6" @@ -1252,7 +1342,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -1275,6 +1365,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83197f59927b46c04a183a619b7c29df34e63e63c7869320862268c0ef687e0" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + [[package]] name = "fdeflate" version = "0.3.7" @@ -1519,6 +1624,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "cfg-if", + "crunchy", +] + [[package]] name = "hardened_malloc-rs" version = "0.1.2+12" @@ -1973,10 +2088,16 @@ dependencies = [ "bytemuck", "byteorder-lite", "color_quant", + "exr", "gif", "image-webp", "num-traits", "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", "zune-core", "zune-jpeg", ] @@ -1991,6 +2112,12 @@ dependencies = [ "quick-error 2.0.1", ] +[[package]] +name = "imgref" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0263a3d970d5c054ed9312c0057b4f3bde9c0b33836d3637361d4a9e6e7a408" + [[package]] name = "indexmap" version = "1.9.3" @@ -2024,6 +2151,17 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "ipaddress" version = "0.1.3" @@ -2089,6 +2227,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.77" @@ -2172,12 +2316,28 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "libc" version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +[[package]] +name = "libfuzzer-sys" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libloading" version = "0.8.6" @@ -2185,7 +2345,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -2243,6 +2403,15 @@ dependencies = [ "futures-sink", ] +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "lru-cache" version = "0.1.2" @@ -2321,6 +2490,16 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "memchr" version = "2.7.4" @@ -2434,6 +2613,12 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2483,6 +2668,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.96", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -2907,6 +3103,25 @@ dependencies = [ "yansi", ] +[[package]] +name = "profiling" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afbdc74edc00b6f6a218ca6a5364d6226a259d4b8ea1af4a0ea063f27e179f4d" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" +dependencies = [ + "quote", + "syn 2.0.96", +] + [[package]] name = "prost" version = "0.13.4" @@ -2957,6 +3172,15 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -3018,7 +3242,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3060,6 +3284,76 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.12.1", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "once_cell", + "paste", + "profiling", + "rand", + "rand_chacha", + "simd_helpers", + "system-deps", + "thiserror 1.0.69", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2413fd96bd0ea5cdeeb37eaf446a22e6ed7b981d792828721e74ded1980a45c6" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error 2.0.1", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.8" @@ -3172,6 +3466,12 @@ dependencies = [ "quick-error 1.2.3", ] +[[package]] +name = "rgb" +version = "0.8.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" + [[package]] name = "ring" version = "0.17.8" @@ -3479,7 +3779,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3945,6 +4245,15 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -4096,6 +4405,25 @@ dependencies = [ "syn 2.0.96", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "tendril" version = "0.4.3" @@ -4205,6 +4533,17 @@ dependencies = [ "threadpool", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "tikv-jemalloc-ctl" version = "0.6.0" @@ -4744,6 +5083,17 @@ dependencies = [ "serde", ] +[[package]] +name = "v_frame" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -4756,6 +5106,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-compare" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" + [[package]] name = "version_check" version = "0.9.5" @@ -5324,6 +5680,15 @@ version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + [[package]] name = "zune-jpeg" version = "0.4.14" diff --git a/Cargo.toml b/Cargo.toml index 1cf787c6..c580d22d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -179,7 +179,7 @@ version = "0.5.3" features = ["alloc", "rand"] default-features = false -# Used to generate thumbnails for images +# Used to generate thumbnails for images & blurhashes [workspace.dependencies.image] version = "0.25.5" default-features = false @@ -190,6 +190,12 @@ features = [ "webp", ] +[workspace.dependencies.blurhash] +version = "0.2.3" +default-features = false +features = [ + "fast-linear-to-srgb","image" +] # logging [workspace.dependencies.log] version = "0.4.22" diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 3e64522c..f9da856d 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1607,3 +1607,21 @@ # This item is undocumented. Please contribute documentation for it. # #support_mxid = + +[global.blurhashing] + +# blurhashing x component, 4 is recommended by https://blurha.sh/ +# +#components_x = 4 + +# blurhashing y component, 3 is recommended by https://blurha.sh/ +# +#components_y = 3 + +# Max raw size that the server will blurhash, this is the size of the +# image after converting it to raw data, it should be higher than the +# upload limit but not too high. The higher it is the higher the +# potential load will be for clients requesting blurhashes. The default +# is 33.55MB. Setting it to 0 disables blurhashing. +# +#blurhash_max_raw_size = 33554432 diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 385e786f..8a5ef3f0 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -17,6 +17,7 @@ crate-type = [ ] [features] +blurhashing=[] element_hacks = [] release_max_log_level = [ "tracing/max_level_trace", diff --git a/src/api/client/media.rs b/src/api/client/media.rs index afbc218a..115f2581 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -62,6 +62,27 @@ pub(crate) async fn create_content_route( media_id: &utils::random_string(MXC_LENGTH), }; + #[cfg(feature = "blurhashing")] + { + if body.generate_blurhash { + let (blurhash, create_media_result) = tokio::join!( + services + .media + .create_blurhash(&body.file, content_type, filename), + services.media.create( + &mxc, + Some(user), + Some(&content_disposition), + content_type, + &body.file + ) + ); + return create_media_result.map(|()| create_content::v3::Response { + content_uri: mxc.to_string().into(), + blurhash, + }); + } + } services .media .create(&mxc, Some(user), Some(&content_disposition), content_type, &body.file) diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index ef2df4ff..5d46ec3b 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -54,6 +54,7 @@ sentry_telemetry = [] conduwuit_mods = [ "dep:libloading" ] +blurhashing = [] [dependencies] argon2.workspace = true diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index ff80d1cf..9514f7a0 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -52,7 +52,7 @@ use crate::{err, error::Error, utils::sys, Result}; ### For more information, see: ### https://conduwuit.puppyirl.gay/configuration.html "#, - ignore = "catchall well_known tls" + ignore = "catchall well_known tls blurhashing" )] pub struct Config { /// The server_name is the pretty name of this server. It is used as a @@ -1789,6 +1789,9 @@ pub struct Config { #[serde(default = "true_fn")] pub config_reload_signal: bool, + // external structure; separate section + #[serde(default)] + pub blurhashing: BlurhashConfig, #[serde(flatten)] #[allow(clippy::zero_sized_map_values)] // this is a catchall, the map shouldn't be zero at runtime @@ -1839,6 +1842,31 @@ pub struct WellKnownConfig { pub support_mxid: Option, } +#[derive(Clone, Copy, Debug, Deserialize, Default)] +#[allow(rustdoc::broken_intra_doc_links, rustdoc::bare_urls)] +#[config_example_generator(filename = "conduwuit-example.toml", section = "global.blurhashing")] +pub struct BlurhashConfig { + /// blurhashing x component, 4 is recommended by https://blurha.sh/ + /// + /// default: 4 + #[serde(default = "default_blurhash_x_component")] + pub components_x: u32, + /// blurhashing y component, 3 is recommended by https://blurha.sh/ + /// + /// default: 3 + #[serde(default = "default_blurhash_y_component")] + pub components_y: u32, + /// Max raw size that the server will blurhash, this is the size of the + /// image after converting it to raw data, it should be higher than the + /// upload limit but not too high. The higher it is the higher the + /// potential load will be for clients requesting blurhashes. The default + /// is 33.55MB. Setting it to 0 disables blurhashing. + /// + /// default: 33554432 + #[serde(default = "default_blurhash_max_raw_size")] + pub blurhash_max_raw_size: u64, +} + #[derive(Deserialize, Clone, Debug)] #[serde(transparent)] struct ListeningPort { @@ -2210,3 +2238,13 @@ fn default_client_response_timeout() -> u64 { 120 } fn default_client_shutdown_timeout() -> u64 { 15 } fn default_sender_shutdown_timeout() -> u64 { 5 } + +// blurhashing defaults recommended by https://blurha.sh/ +// 2^25 +pub(super) fn default_blurhash_max_raw_size() -> u64 { 33_554_432 } + +pub(super) fn default_blurhash_x_component() -> u32 { 4 } + +pub(super) fn default_blurhash_y_component() -> u32 { 3 } + +// end recommended & blurhashing defaults diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index f774c37a..7e1cb86b 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -101,6 +101,7 @@ perf_measurements = [ "conduwuit-core/perf_measurements", "conduwuit-core/sentry_telemetry", ] +blurhashing =["conduwuit-service/blurhashing","conduwuit-core/blurhashing","conduwuit-api/blurhashing"] # increases performance, reduces build times, and reduces binary size by not compiling or # genreating code for log level filters that users will generally not use (debug and trace) release_max_log_level = [ diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index c4f75453..30183179 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -44,6 +44,7 @@ url_preview = [ zstd_compression = [ "reqwest/zstd", ] +blurhashing = ["dep:image","dep:blurhash"] [dependencies] arrayvec.workspace = true @@ -82,6 +83,8 @@ tracing.workspace = true url.workspace = true webpage.workspace = true webpage.optional = true +blurhash.workspace = true +blurhash.optional = true [lints] workspace = true diff --git a/src/service/media/blurhash.rs b/src/service/media/blurhash.rs new file mode 100644 index 00000000..c470925c --- /dev/null +++ b/src/service/media/blurhash.rs @@ -0,0 +1,159 @@ +use std::{fmt::Display, io::Cursor, path::Path}; + +use blurhash::encode_image; +use conduwuit::{config::BlurhashConfig as CoreBlurhashConfig, debug_error, implement, trace}; +use image::{DynamicImage, ImageDecoder, ImageError, ImageFormat, ImageReader}; + +use super::Service; +#[implement(Service)] +pub async fn create_blurhash( + &self, + file: &[u8], + content_type: Option<&str>, + file_name: Option<&str>, +) -> Option { + let config = BlurhashConfig::from(self.services.server.config.blurhashing); + if config.size_limit == 0 { + trace!("since 0 means disabled blurhashing, skipped blurhashing logic"); + return None; + } + let file_data = file.to_owned(); + let content_type = content_type.map(String::from); + let file_name = file_name.map(String::from); + + let blurhashing_result = tokio::task::spawn_blocking(move || { + get_blurhash_from_request(&file_data, content_type, file_name, config) + }) + .await + .expect("no join error"); + + match blurhashing_result { + | Ok(result) => Some(result), + | Err(e) => { + debug_error!("Error when blurhashing: {e}"); + None + }, + } +} + +/// Returns the blurhash or a blurhash error which implements Display. +fn get_blurhash_from_request( + data: &[u8], + mime: Option, + filename: Option, + config: BlurhashConfig, +) -> Result { + // Get format image is supposed to be in + let format = get_format_from_data_mime_and_filename(data, mime, filename)?; + // Get the image reader for said image format + let decoder = get_image_decoder_with_format_and_data(format, data)?; + // Check image size makes sense before unpacking whole image + if is_image_above_size_limit(&decoder, config) { + return Err(BlurhashingError::ImageTooLarge); + } + // decode the image finally + let image = DynamicImage::from_decoder(decoder)?; + + blurhash_an_image(&image, config) +} + +/// Gets the Image Format value from the data,mime, and filename +/// It first checks if the mime is a valid image format +/// Then it checks if the filename has a format, otherwise just guess based on +/// the binary data Assumes that mime and filename extension won't be for a +/// different file format than file. +fn get_format_from_data_mime_and_filename( + data: &[u8], + mime: Option, + filename: Option, +) -> Result { + let mut image_format = None; + if let Some(mime) = mime { + image_format = ImageFormat::from_mime_type(mime); + } + if let (Some(filename), None) = (filename, image_format) { + if let Some(extension) = Path::new(&filename).extension() { + image_format = ImageFormat::from_mime_type(extension.to_string_lossy()); + } + } + + if let Some(format) = image_format { + Ok(format) + } else { + image::guess_format(data).map_err(Into::into) + } +} + +fn get_image_decoder_with_format_and_data( + image_format: ImageFormat, + data: &[u8], +) -> Result, BlurhashingError> { + let mut image_reader = ImageReader::new(Cursor::new(data)); + image_reader.set_format(image_format); + Ok(Box::new(image_reader.into_decoder()?)) +} + +fn is_image_above_size_limit( + decoder: &T, + blurhash_config: BlurhashConfig, +) -> bool { + decoder.total_bytes() >= blurhash_config.size_limit +} +#[inline] +fn blurhash_an_image( + image: &DynamicImage, + blurhash_config: BlurhashConfig, +) -> Result { + Ok(encode_image( + blurhash_config.components_x, + blurhash_config.components_y, + &image.to_rgba8(), + )?) +} +#[derive(Clone, Copy)] +pub struct BlurhashConfig { + components_x: u32, + components_y: u32, + /// size limit in bytes + size_limit: u64, +} + +impl From for BlurhashConfig { + fn from(value: CoreBlurhashConfig) -> Self { + Self { + components_x: value.components_x, + components_y: value.components_y, + size_limit: value.blurhash_max_raw_size, + } + } +} + +#[derive(Debug)] +pub(crate) enum BlurhashingError { + ImageError(Box), + HashingLibError(Box), + ImageTooLarge, +} +impl From for BlurhashingError { + fn from(value: ImageError) -> Self { Self::ImageError(Box::new(value)) } +} + +impl From for BlurhashingError { + fn from(value: blurhash::Error) -> Self { Self::HashingLibError(Box::new(value)) } +} + +impl Display for BlurhashingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Blurhash Error:")?; + match &self { + | Self::ImageTooLarge => write!(f, "Image was too large to blurhash")?, + | Self::HashingLibError(e) => + write!(f, "There was an error with the blurhashing library => {e}")?, + + | Self::ImageError(e) => + write!(f, "There was an error with the image loading library => {e}")?, + }; + + Ok(()) + } +} diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 0d98853d..7775173a 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,10 +1,11 @@ +#[cfg(feature = "blurhashing")] +pub mod blurhash; mod data; pub(super) mod migrations; mod preview; mod remote; mod tests; mod thumbnail; - use std::{path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; From 442bb9889c45e5b17cdf5c7fd90e4751f7582400 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 4 Feb 2025 02:24:50 +0000 Subject: [PATCH 1481/2091] improvements on blurhashing feature Signed-off-by: Jason Volk --- Cargo.toml | 4 +- src/api/Cargo.toml | 1 - src/api/client/media.rs | 44 +++++-------- src/core/Cargo.toml | 1 - src/main/Cargo.toml | 4 +- src/service/media/blurhash.rs | 113 +++++++++++++++++++--------------- src/service/media/mod.rs | 1 - 7 files changed, 87 insertions(+), 81 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c580d22d..b25d9175 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -194,8 +194,10 @@ features = [ version = "0.2.3" default-features = false features = [ - "fast-linear-to-srgb","image" + "fast-linear-to-srgb", + "image", ] + # logging [workspace.dependencies.log] version = "0.4.22" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 8a5ef3f0..385e786f 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -17,7 +17,6 @@ crate-type = [ ] [features] -blurhashing=[] element_hacks = [] release_max_log_level = [ "tracing/max_level_trace", diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 115f2581..0cff8185 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -57,40 +57,28 @@ pub(crate) async fn create_content_route( let filename = body.filename.as_deref(); let content_type = body.content_type.as_deref(); let content_disposition = make_content_disposition(None, content_type, filename); - let mxc = Mxc { + let ref mxc = Mxc { server_name: services.globals.server_name(), media_id: &utils::random_string(MXC_LENGTH), }; - #[cfg(feature = "blurhashing")] - { - if body.generate_blurhash { - let (blurhash, create_media_result) = tokio::join!( - services - .media - .create_blurhash(&body.file, content_type, filename), - services.media.create( - &mxc, - Some(user), - Some(&content_disposition), - content_type, - &body.file - ) - ); - return create_media_result.map(|()| create_content::v3::Response { - content_uri: mxc.to_string().into(), - blurhash, - }); - } - } services .media - .create(&mxc, Some(user), Some(&content_disposition), content_type, &body.file) - .await - .map(|()| create_content::v3::Response { - content_uri: mxc.to_string().into(), - blurhash: None, - }) + .create(mxc, Some(user), Some(&content_disposition), content_type, &body.file) + .await?; + + let blurhash = body.generate_blurhash.then(|| { + services + .media + .create_blurhash(&body.file, content_type, filename) + .ok() + .flatten() + }); + + Ok(create_content::v3::Response { + content_uri: mxc.to_string().into(), + blurhash: blurhash.flatten(), + }) } /// # `GET /_matrix/client/v1/media/thumbnail/{serverName}/{mediaId}` diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 5d46ec3b..ef2df4ff 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -54,7 +54,6 @@ sentry_telemetry = [] conduwuit_mods = [ "dep:libloading" ] -blurhashing = [] [dependencies] argon2.workspace = true diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 7e1cb86b..87ca48c8 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -49,6 +49,9 @@ default = [ "zstd_compression", ] +blurhashing = [ + "conduwuit-service/blurhashing", +] brotli_compression = [ "conduwuit-api/brotli_compression", "conduwuit-core/brotli_compression", @@ -101,7 +104,6 @@ perf_measurements = [ "conduwuit-core/perf_measurements", "conduwuit-core/sentry_telemetry", ] -blurhashing =["conduwuit-service/blurhashing","conduwuit-core/blurhashing","conduwuit-api/blurhashing"] # increases performance, reduces build times, and reduces binary size by not compiling or # genreating code for log level filters that users will generally not use (debug and trace) release_max_log_level = [ diff --git a/src/service/media/blurhash.rs b/src/service/media/blurhash.rs index c470925c..aa6685b2 100644 --- a/src/service/media/blurhash.rs +++ b/src/service/media/blurhash.rs @@ -1,56 +1,58 @@ -use std::{fmt::Display, io::Cursor, path::Path}; +use std::{error::Error, ffi::OsStr, fmt::Display, io::Cursor, path::Path}; -use blurhash::encode_image; -use conduwuit::{config::BlurhashConfig as CoreBlurhashConfig, debug_error, implement, trace}; +use conduwuit::{config::BlurhashConfig as CoreBlurhashConfig, err, implement, Result}; use image::{DynamicImage, ImageDecoder, ImageError, ImageFormat, ImageReader}; use super::Service; #[implement(Service)] -pub async fn create_blurhash( +pub fn create_blurhash( &self, file: &[u8], content_type: Option<&str>, file_name: Option<&str>, -) -> Option { +) -> Result> { + if !cfg!(feature = "blurhashing") { + return Ok(None); + } + let config = BlurhashConfig::from(self.services.server.config.blurhashing); + + // since 0 means disabled blurhashing, skipped blurhashing if config.size_limit == 0 { - trace!("since 0 means disabled blurhashing, skipped blurhashing logic"); - return None; + return Ok(None); } - let file_data = file.to_owned(); - let content_type = content_type.map(String::from); - let file_name = file_name.map(String::from); - let blurhashing_result = tokio::task::spawn_blocking(move || { - get_blurhash_from_request(&file_data, content_type, file_name, config) - }) - .await - .expect("no join error"); - - match blurhashing_result { - | Ok(result) => Some(result), - | Err(e) => { - debug_error!("Error when blurhashing: {e}"); - None - }, - } + get_blurhash_from_request(file, content_type, file_name, config) + .map_err(|e| err!(debug_error!("blurhashing error: {e}"))) + .map(Some) } /// Returns the blurhash or a blurhash error which implements Display. +#[tracing::instrument( + name = "blurhash", + level = "debug", + skip(data), + fields( + bytes = data.len(), + ), +)] fn get_blurhash_from_request( data: &[u8], - mime: Option, - filename: Option, + mime: Option<&str>, + filename: Option<&str>, config: BlurhashConfig, ) -> Result { // Get format image is supposed to be in let format = get_format_from_data_mime_and_filename(data, mime, filename)?; + // Get the image reader for said image format let decoder = get_image_decoder_with_format_and_data(format, data)?; + // Check image size makes sense before unpacking whole image if is_image_above_size_limit(&decoder, config) { return Err(BlurhashingError::ImageTooLarge); } + // decode the image finally let image = DynamicImage::from_decoder(decoder)?; @@ -64,24 +66,17 @@ fn get_blurhash_from_request( /// different file format than file. fn get_format_from_data_mime_and_filename( data: &[u8], - mime: Option, - filename: Option, + mime: Option<&str>, + filename: Option<&str>, ) -> Result { - let mut image_format = None; - if let Some(mime) = mime { - image_format = ImageFormat::from_mime_type(mime); - } - if let (Some(filename), None) = (filename, image_format) { - if let Some(extension) = Path::new(&filename).extension() { - image_format = ImageFormat::from_mime_type(extension.to_string_lossy()); - } - } + let extension = filename + .map(Path::new) + .and_then(Path::extension) + .map(OsStr::to_string_lossy); - if let Some(format) = image_format { - Ok(format) - } else { - image::guess_format(data).map_err(Into::into) - } + mime.or(extension.as_deref()) + .and_then(ImageFormat::from_mime_type) + .map_or_else(|| image::guess_format(data).map_err(Into::into), Ok) } fn get_image_decoder_with_format_and_data( @@ -99,23 +94,37 @@ fn is_image_above_size_limit( ) -> bool { decoder.total_bytes() >= blurhash_config.size_limit } + +#[cfg(feature = "blurhashing")] +#[tracing::instrument(name = "encode", level = "debug", skip_all)] #[inline] fn blurhash_an_image( image: &DynamicImage, blurhash_config: BlurhashConfig, ) -> Result { - Ok(encode_image( + Ok(blurhash::encode_image( blurhash_config.components_x, blurhash_config.components_y, &image.to_rgba8(), )?) } -#[derive(Clone, Copy)] + +#[cfg(not(feature = "blurhashing"))] +#[inline] +fn blurhash_an_image( + _image: &DynamicImage, + _blurhash_config: BlurhashConfig, +) -> Result { + Err(BlurhashingError::Unavailable) +} + +#[derive(Clone, Copy, Debug)] pub struct BlurhashConfig { - components_x: u32, - components_y: u32, + pub components_x: u32, + pub components_y: u32, + /// size limit in bytes - size_limit: u64, + pub size_limit: u64, } impl From for BlurhashConfig { @@ -129,15 +138,20 @@ impl From for BlurhashConfig { } #[derive(Debug)] -pub(crate) enum BlurhashingError { +pub enum BlurhashingError { + HashingLibError(Box), ImageError(Box), - HashingLibError(Box), ImageTooLarge, + + #[cfg(not(feature = "blurhashing"))] + Unavailable, } + impl From for BlurhashingError { fn from(value: ImageError) -> Self { Self::ImageError(Box::new(value)) } } +#[cfg(feature = "blurhashing")] impl From for BlurhashingError { fn from(value: blurhash::Error) -> Self { Self::HashingLibError(Box::new(value)) } } @@ -152,6 +166,9 @@ impl Display for BlurhashingError { | Self::ImageError(e) => write!(f, "There was an error with the image loading library => {e}")?, + + #[cfg(not(feature = "blurhashing"))] + | Self::Unavailable => write!(f, "Blurhashing is not supported")?, }; Ok(()) diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 7775173a..f5913f43 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -1,4 +1,3 @@ -#[cfg(feature = "blurhashing")] pub mod blurhash; mod data; pub(super) mod migrations; From 04656a78865dfd60176965c5ae531d1939e0dd7d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 5 Feb 2025 03:00:47 +0000 Subject: [PATCH 1482/2091] fix spaces pagination bug Signed-off-by: Jason Volk --- src/service/rooms/spaces/mod.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 1ee2727c..11794752 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -582,7 +582,7 @@ impl Service { parents.pop_front(); parents.push_back(room); - let short_room_ids: Vec<_> = parents + let next_short_room_ids: Vec<_> = parents .iter() .stream() .filter_map(|room_id| async move { @@ -591,16 +591,18 @@ impl Service { .collect() .await; - Some( - PaginationToken { - short_room_ids, - limit: UInt::new(max_depth) - .expect("When sent in request it must have been valid UInt"), - max_depth: UInt::new(max_depth) - .expect("When sent in request it must have been valid UInt"), - suggested_only, - } - .to_string(), + (next_short_room_ids != short_room_ids && !next_short_room_ids.is_empty()).then( + || { + PaginationToken { + short_room_ids: next_short_room_ids, + limit: UInt::new(max_depth) + .expect("When sent in request it must have been valid UInt"), + max_depth: UInt::new(max_depth) + .expect("When sent in request it must have been valid UInt"), + suggested_only, + } + .to_string() + }, ) } else { None From 9158edfb7c98229af43b2124e972723b1ab4e75a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 5 Feb 2025 05:10:30 +0000 Subject: [PATCH 1483/2091] fix empty join timeline bug Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 48 ++++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index a97e4329..1d1a91ba 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -691,7 +691,7 @@ async fn load_joined_room( heroes, joined_member_count, invited_member_count, - state_events, + mut state_events, mut device_list_updates, left_encrypted_users, } = calculate_state_changes( @@ -708,6 +708,39 @@ async fn load_joined_room( .boxed() .await?; + let is_sender_membership = |pdu: &PduEvent| { + pdu.kind == StateEventType::RoomMember.into() + && pdu + .state_key + .as_deref() + .is_some_and(is_equal_to!(sender_user.as_str())) + }; + + let joined_sender_member: Option<_> = (joined_since_last_sync && timeline_pdus.is_empty()) + .then(|| { + state_events + .iter() + .position(is_sender_membership) + .map(|pos| state_events.swap_remove(pos)) + }) + .flatten(); + + let prev_batch = timeline_pdus.first().map(at!(0)).or_else(|| { + joined_sender_member + .is_some() + .then_some(since) + .map(Into::into) + }); + + let room_events = timeline_pdus + .into_iter() + .stream() + .wide_filter_map(|item| ignored_filter(services, item, sender_user)) + .map(at!(1)) + .chain(joined_sender_member.into_iter().stream()) + .map(|pdu| pdu.to_sync_room_event()) + .collect::>(); + let account_data_events = services .account_data .changes_since(Some(room_id), sender_user, since, Some(next_batch)) @@ -722,13 +755,6 @@ async fn load_joined_room( .map(ToOwned::to_owned) .collect::>(); - let room_events = timeline_pdus - .iter() - .stream() - .wide_filter_map(|item| ignored_filter(services, item.clone(), sender_user)) - .map(|(_, pdu)| pdu.to_sync_room_event()) - .collect(); - let send_notification_counts = last_notification_read.is_none_or(|count| count > since); let notification_count: OptionFuture<_> = send_notification_counts @@ -830,12 +856,8 @@ async fn load_joined_room( unread_notifications: UnreadNotificationsCount { highlight_count, notification_count }, timeline: Timeline { limited: limited || joined_since_last_sync, + prev_batch: prev_batch.as_ref().map(ToString::to_string), events: room_events, - prev_batch: timeline_pdus - .first() - .map(at!(0)) - .as_ref() - .map(ToString::to_string), }, state: RoomState { events: state_events From f80d85e1076f1f155a7484d5ad80acbb58a9b1ac Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 5 Feb 2025 01:43:27 -0500 Subject: [PATCH 1484/2091] add SIGUSR1 systemctl reload config support to systemd units Signed-off-by: strawberry --- arch/conduwuit.service | 3 ++- debian/conduwuit.service | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index 7c05c259..4b7853e3 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -7,7 +7,8 @@ RequiresMountsFor=/var/lib/private/conduwuit [Service] DynamicUser=yes -Type=notify +Type=notify-reload +ReloadSignal=SIGUSR1 AmbientCapabilities= CapabilityBoundingSet= diff --git a/debian/conduwuit.service b/debian/conduwuit.service index 3c2ec49d..452544bf 100644 --- a/debian/conduwuit.service +++ b/debian/conduwuit.service @@ -8,7 +8,8 @@ Documentation=https://conduwuit.puppyirl.gay/ DynamicUser=yes User=conduwuit Group=conduwuit -Type=notify +Type=notify-reload +ReloadSignal=SIGUSR1 Environment="CONDUWUIT_CONFIG=/etc/conduwuit/conduwuit.toml" From f6dfc9538f8b625c2ae28a462ecf4b7e3d208f85 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 5 Feb 2025 01:44:49 -0500 Subject: [PATCH 1485/2091] bump ruwuma to stop erroring on duplicate yaml values on appservice EDUs (we dont implement this atm anyways) Signed-off-by: strawberry --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b710d6fc..926099b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3490,7 +3490,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "assign", "js_int", @@ -3512,7 +3512,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "js_int", "ruma-common", @@ -3524,7 +3524,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "as_variant", "assign", @@ -3547,7 +3547,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "as_variant", "base64 0.22.1", @@ -3578,7 +3578,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3603,7 +3603,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "bytes", "http", @@ -3621,7 +3621,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3630,7 +3630,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "js_int", "ruma-common", @@ -3640,7 +3640,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3655,7 +3655,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "js_int", "ruma-common", @@ -3667,7 +3667,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "headers", "http", @@ -3680,7 +3680,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3696,7 +3696,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b560338b2a50dbf61ecfe80808b9b095ad4cec00#b560338b2a50dbf61ecfe80808b9b095ad4cec00" +source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index b25d9175..ce483bbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -342,7 +342,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "b560338b2a50dbf61ecfe80808b9b095ad4cec00" +rev = "517ac4572276a2e0ad587113776c544b51166f08" features = [ "compat", "rand", From fda8b3680986dc8e038d51b93f7d36bf5c991ef6 Mon Sep 17 00:00:00 2001 From: strawberry Date: Wed, 5 Feb 2025 01:45:21 -0500 Subject: [PATCH 1486/2091] add more systemd notify integration with stopping/reloading/ready states Signed-off-by: strawberry --- src/core/server.rs | 12 ++++++++++-- src/router/run.rs | 4 ---- src/service/config/mod.rs | 8 ++++++++ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/core/server.rs b/src/core/server.rs index 45ba7420..80493c94 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -69,6 +69,10 @@ impl Server { return Err!("Reloading not enabled"); } + #[cfg(all(feature = "systemd", target_os = "linux"))] + sd_notify::notify(true, &[sd_notify::NotifyState::Reloading]) + .expect("failed to notify systemd of reloading state"); + if self.reloading.swap(true, Ordering::AcqRel) { return Err!("Reloading already in progress"); } @@ -83,7 +87,7 @@ impl Server { }) } - pub fn restart(&self) -> Result<()> { + pub fn restart(&self) -> Result { if self.restarting.swap(true, Ordering::AcqRel) { return Err!("Restart already in progress"); } @@ -93,7 +97,11 @@ impl Server { }) } - pub fn shutdown(&self) -> Result<()> { + pub fn shutdown(&self) -> Result { + #[cfg(all(feature = "systemd", target_os = "linux"))] + sd_notify::notify(true, &[sd_notify::NotifyState::Stopping]) + .expect("failed to notify systemd of stopping state"); + if self.stopping.swap(true, Ordering::AcqRel) { return Err!("Shutdown already in progress"); } diff --git a/src/router/run.rs b/src/router/run.rs index 26701735..024cb813 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -100,10 +100,6 @@ pub(crate) async fn stop(services: Arc) -> Result<()> { ); } - #[cfg(all(feature = "systemd", target_os = "linux"))] - sd_notify::notify(true, &[sd_notify::NotifyState::Stopping]) - .expect("failed to notify systemd of stopping state"); - info!("Shutdown complete."); Ok(()) } diff --git a/src/service/config/mod.rs b/src/service/config/mod.rs index 8bd09a52..c9ac37a3 100644 --- a/src/service/config/mod.rs +++ b/src/service/config/mod.rs @@ -43,7 +43,15 @@ impl Deref for Service { #[implement(Service)] fn handle_reload(&self) -> Result { if self.server.config.config_reload_signal { + #[cfg(all(feature = "systemd", target_os = "linux"))] + sd_notify::notify(true, &[sd_notify::NotifyState::Reloading]) + .expect("failed to notify systemd of reloading state"); + self.reload(iter::empty())?; + + #[cfg(all(feature = "systemd", target_os = "linux"))] + sd_notify::notify(true, &[sd_notify::NotifyState::Ready]) + .expect("failed to notify systemd of ready state"); } Ok(()) From 62d80b97e65237539a103ded87f4e650ddafe4b8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Feb 2025 03:14:37 +0000 Subject: [PATCH 1487/2091] add systemd unit logging mode Signed-off-by: Jason Volk --- src/core/log/console.rs | 77 +++++++++++++++++++++++++++++++++--- src/core/log/mod.rs | 4 +- src/main/logging.rs | 4 +- src/service/admin/console.rs | 5 ++- 4 files changed, 78 insertions(+), 12 deletions(-) diff --git a/src/core/log/console.rs b/src/core/log/console.rs index 0bc44fa7..1f04ba26 100644 --- a/src/core/log/console.rs +++ b/src/core/log/console.rs @@ -1,3 +1,5 @@ +use std::{env, io, sync::LazyLock}; + use tracing::{ field::{Field, Visit}, Event, Level, Subscriber, @@ -7,12 +9,59 @@ use tracing_subscriber::{ fmt, fmt::{ format::{Compact, DefaultVisitor, Format, Full, Pretty, Writer}, - FmtContext, FormatEvent, FormatFields, + FmtContext, FormatEvent, FormatFields, MakeWriter, }, registry::LookupSpan, }; -use crate::{Config, Result}; +use crate::{apply, Config, Result}; + +static SYSTEMD_MODE: LazyLock = + LazyLock::new(|| env::var("SYSTEMD_EXEC_PID").is_ok() && env::var("JOURNAL_STREAM").is_ok()); + +pub struct ConsoleWriter { + stdout: io::Stdout, + stderr: io::Stderr, + _journal_stream: [u64; 2], + use_stderr: bool, +} + +impl ConsoleWriter { + #[must_use] + pub fn new(_config: &Config) -> Self { + let journal_stream = get_journal_stream(); + Self { + stdout: io::stdout(), + stderr: io::stderr(), + _journal_stream: journal_stream.into(), + use_stderr: journal_stream.0 != 0, + } + } +} + +impl<'a> MakeWriter<'a> for ConsoleWriter { + type Writer = &'a Self; + + fn make_writer(&'a self) -> Self::Writer { self } +} + +impl io::Write for &'_ ConsoleWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + if self.use_stderr { + self.stderr.lock().write(buf) + } else { + self.stdout.lock().write(buf) + } + } + + fn flush(&mut self) -> io::Result<()> { + if self.use_stderr { + self.stderr.lock().flush() + } else { + self.stdout.lock().flush() + } + } +} pub struct ConsoleFormat { _compact: Format, @@ -20,10 +69,6 @@ pub struct ConsoleFormat { pretty: Format, } -struct ConsoleVisitor<'a> { - visitor: DefaultVisitor<'a>, -} - impl ConsoleFormat { #[must_use] pub fn new(config: &Config) -> Self { @@ -68,6 +113,10 @@ where } } +struct ConsoleVisitor<'a> { + visitor: DefaultVisitor<'a>, +} + impl<'writer> FormatFields<'writer> for ConsoleFormat { fn format_fields(&self, writer: Writer<'writer>, fields: R) -> Result<(), std::fmt::Error> where @@ -92,3 +141,19 @@ impl Visit for ConsoleVisitor<'_> { self.visitor.record_debug(field, value); } } + +#[must_use] +fn get_journal_stream() -> (u64, u64) { + is_systemd_mode() + .then(|| env::var("JOURNAL_STREAM").ok()) + .flatten() + .as_deref() + .and_then(|s| s.split_once(':')) + .map(apply!(2, str::parse)) + .map(apply!(2, Result::unwrap_or_default)) + .unwrap_or((0, 0)) +} + +#[inline] +#[must_use] +pub fn is_systemd_mode() -> bool { *SYSTEMD_MODE } diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 0c51a383..0c1840d0 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -2,14 +2,14 @@ pub mod capture; pub mod color; -mod console; +pub mod console; pub mod fmt; pub mod fmt_span; mod reload; mod suppress; pub use capture::Capture; -pub use console::ConsoleFormat; +pub use console::{is_systemd_mode, ConsoleFormat, ConsoleWriter}; pub use reload::{LogLevelReloadHandles, ReloadHandle}; pub use suppress::Suppress; pub use tracing::Level; diff --git a/src/main/logging.rs b/src/main/logging.rs index 85945e8a..35e482de 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use conduwuit::{ config::Config, debug_warn, err, - log::{capture, fmt_span, ConsoleFormat, LogLevelReloadHandles}, + log::{capture, fmt_span, ConsoleFormat, ConsoleWriter, LogLevelReloadHandles}, result::UnwrapOrErr, Result, }; @@ -30,7 +30,7 @@ pub(crate) fn init( .with_span_events(console_span_events) .event_format(ConsoleFormat::new(config)) .fmt_fields(ConsoleFormat::new(config)) - .map_writer(|w| w); + .with_writer(ConsoleWriter::new(config)); let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index de201f4b..59b9a31b 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -1,10 +1,11 @@ #![cfg(feature = "console")] + use std::{ collections::VecDeque, sync::{Arc, Mutex}, }; -use conduwuit::{debug, defer, error, log, Server}; +use conduwuit::{debug, defer, error, log, log::is_systemd_mode, Server}; use futures::future::{AbortHandle, Abortable}; use ruma::events::room::message::RoomMessageEventContent; use rustyline_async::{Readline, ReadlineError, ReadlineEvent}; @@ -123,7 +124,7 @@ impl Console { } async fn readline(self: &Arc) -> Result { - let _suppression = log::Suppress::new(&self.server); + let _suppression = (!is_systemd_mode()).then(|| log::Suppress::new(&self.server)); let (mut readline, _writer) = Readline::new(PROMPT.to_owned())?; let self_ = Arc::clone(self); From 16b07ae3ecf6dee591b79dd6198cb3e5a99410be Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 6 Feb 2025 16:47:10 -0500 Subject: [PATCH 1488/2091] add default systemd support for a TTY to use console mode from Signed-off-by: strawberry --- arch/conduwuit.service | 12 ++++++++++++ debian/conduwuit.service | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index 4b7853e3..fa3616d8 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -10,6 +10,18 @@ DynamicUser=yes Type=notify-reload ReloadSignal=SIGUSR1 +TTYPath=/dev/tty25 +DeviceAllow=char-tty +StandardInput=tty-force +StandardOutput=tty +StandardError=journal+console +TTYReset=yes +# uncomment to allow buffer to be cleared every restart +TTYVTDisallocate=no + +TTYColumns=120 +TTYRows=40 + AmbientCapabilities= CapabilityBoundingSet= diff --git a/debian/conduwuit.service b/debian/conduwuit.service index 452544bf..4d6f4eef 100644 --- a/debian/conduwuit.service +++ b/debian/conduwuit.service @@ -11,6 +11,18 @@ Group=conduwuit Type=notify-reload ReloadSignal=SIGUSR1 +TTYPath=/dev/tty25 +DeviceAllow=char-tty +StandardInput=tty-force +StandardOutput=tty +StandardError=journal+console +TTYReset=yes +# uncomment to allow buffer to be cleared every restart +TTYVTDisallocate=no + +TTYColumns=120 +TTYRows=40 + Environment="CONDUWUIT_CONFIG=/etc/conduwuit/conduwuit.toml" ExecStart=/usr/sbin/conduwuit From f761d4d5c9e347699725bff0437a8df3b1b3db59 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 6 Feb 2025 16:48:19 -0500 Subject: [PATCH 1489/2091] bump db version to 17, cleanup, rerun old migrations for users who downgraded Signed-off-by: strawberry --- src/service/globals/data.rs | 3 +-- src/service/migrations.rs | 32 +++++++++++++------------------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 39cb9be1..26a18607 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -69,9 +69,8 @@ impl Data { } #[inline] - pub fn bump_database_version(&self, new_version: u64) -> Result<()> { + pub fn bump_database_version(&self, new_version: u64) { self.global.raw_put(b"version", new_version); - Ok(()) } #[inline] diff --git a/src/service/migrations.rs b/src/service/migrations.rs index 27b4ab5a..9c3ea293 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -27,15 +27,7 @@ use crate::{media, Services}; /// - If database is opened at lesser version we apply migrations up to this. /// Note that named-feature migrations may also be performed when opening at /// equal or lesser version. These are expected to be backward-compatible. -pub(crate) const DATABASE_VERSION: u64 = 13; - -/// Conduit's database version. -/// -/// Conduit bumped the database version to 16, but did not introduce any -/// breaking changes. Their database migrations are extremely fragile and risky, -/// and also do not really apply to us, so just to retain Conduit -> conduwuit -/// compatibility we'll check for both versions. -pub(crate) const CONDUIT_DATABASE_VERSION: u64 = 16; +pub(crate) const DATABASE_VERSION: u64 = 17; pub(crate) async fn migrations(services: &Services) -> Result<()> { let users_count = services.users.count().await; @@ -63,10 +55,7 @@ pub(crate) async fn migrations(services: &Services) -> Result<()> { async fn fresh(services: &Services) -> Result<()> { let db = &services.db; - services - .globals - .db - .bump_database_version(DATABASE_VERSION)?; + services.globals.db.bump_database_version(DATABASE_VERSION); db["global"].insert(b"feat_sha256_media", []); db["global"].insert(b"fix_bad_double_separator_in_state_cache", []); @@ -130,6 +119,7 @@ async fn migrate(services: &Services) -> Result<()> { .get(b"fix_referencedevents_missing_sep") .await .is_not_found() + || services.globals.db.database_version().await < 17 { fix_referencedevents_missing_sep(services).await?; } @@ -138,15 +128,19 @@ async fn migrate(services: &Services) -> Result<()> { .get(b"fix_readreceiptid_readreceipt_duplicates") .await .is_not_found() + || services.globals.db.database_version().await < 17 { fix_readreceiptid_readreceipt_duplicates(services).await?; } - let version_match = services.globals.db.database_version().await == DATABASE_VERSION - || services.globals.db.database_version().await == CONDUIT_DATABASE_VERSION; + if services.globals.db.database_version().await < 17 { + services.globals.db.bump_database_version(17); + info!("Migration: Bumped database version to 17"); + } - assert!( - version_match, + assert_eq!( + services.globals.db.database_version().await, + DATABASE_VERSION, "Failed asserting local database version {} is equal to known latest conduwuit database \ version {}", services.globals.db.database_version().await, @@ -290,7 +284,7 @@ async fn db_lt_12(services: &Services) -> Result<()> { .await?; } - services.globals.db.bump_database_version(12)?; + services.globals.db.bump_database_version(12); info!("Migration: 11 -> 12 finished"); Ok(()) } @@ -335,7 +329,7 @@ async fn db_lt_13(services: &Services) -> Result<()> { .await?; } - services.globals.db.bump_database_version(13)?; + services.globals.db.bump_database_version(13); info!("Migration: 12 -> 13 finished"); Ok(()) } From ef2d307c15dba1731dc6b4d67e758f27590640c6 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 6 Feb 2025 16:48:48 -0500 Subject: [PATCH 1490/2091] fix warnings and errors when building with no features Signed-off-by: strawberry --- src/main/runtime.rs | 11 ++--- src/service/media/blurhash.rs | 87 +++++++++++++++++---------------- src/service/media/migrations.rs | 8 +-- 3 files changed, 51 insertions(+), 55 deletions(-) diff --git a/src/main/runtime.rs b/src/main/runtime.rs index 02b9931f..474b373b 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -8,13 +8,11 @@ use std::{ time::Duration, }; +#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] +use conduwuit::result::LogDebugErr; use conduwuit::{ is_true, - result::LogDebugErr, - utils::{ - available_parallelism, - sys::compute::{nth_core_available, set_affinity}, - }, + utils::sys::compute::{nth_core_available, set_affinity}, Result, }; use tokio::runtime::Builder; @@ -25,6 +23,7 @@ const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; const WORKER_KEEPALIVE: u64 = 36; const MAX_BLOCKING_THREADS: usize = 1024; +#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] const DISABLE_MUZZY_THRESHOLD: usize = 4; static WORKER_AFFINITY: OnceLock = OnceLock::new(); @@ -137,7 +136,7 @@ fn set_worker_mallctl(id: usize) { .get() .expect("GC_MUZZY initialized by runtime::new()"); - let muzzy_auto_disable = available_parallelism() >= DISABLE_MUZZY_THRESHOLD; + let muzzy_auto_disable = conduwuit::utils::available_parallelism() >= DISABLE_MUZZY_THRESHOLD; if matches!(muzzy_option, Some(false) | None if muzzy_auto_disable) { set_muzzy_decay(-1).log_debug_err().ok(); } diff --git a/src/service/media/blurhash.rs b/src/service/media/blurhash.rs index aa6685b2..60ade723 100644 --- a/src/service/media/blurhash.rs +++ b/src/service/media/blurhash.rs @@ -1,20 +1,30 @@ -use std::{error::Error, ffi::OsStr, fmt::Display, io::Cursor, path::Path}; - -use conduwuit::{config::BlurhashConfig as CoreBlurhashConfig, err, implement, Result}; -use image::{DynamicImage, ImageDecoder, ImageError, ImageFormat, ImageReader}; +#[cfg(feature = "blurhashing")] +use conduwuit::config::BlurhashConfig as CoreBlurhashConfig; +use conduwuit::{implement, Result}; use super::Service; + #[implement(Service)] +#[cfg(not(feature = "blurhashing"))] +pub fn create_blurhash( + &self, + _file: &[u8], + _content_type: Option<&str>, + _file_name: Option<&str>, +) -> Result> { + conduwuit::debug_warn!("blurhashing on upload support was not compiled"); + + Ok(None) +} + +#[implement(Service)] +#[cfg(feature = "blurhashing")] pub fn create_blurhash( &self, file: &[u8], content_type: Option<&str>, file_name: Option<&str>, ) -> Result> { - if !cfg!(feature = "blurhashing") { - return Ok(None); - } - let config = BlurhashConfig::from(self.services.server.config.blurhashing); // since 0 means disabled blurhashing, skipped blurhashing @@ -23,7 +33,7 @@ pub fn create_blurhash( } get_blurhash_from_request(file, content_type, file_name, config) - .map_err(|e| err!(debug_error!("blurhashing error: {e}"))) + .map_err(|e| conduwuit::err!(debug_error!("blurhashing error: {e}"))) .map(Some) } @@ -36,6 +46,7 @@ pub fn create_blurhash( bytes = data.len(), ), )] +#[cfg(feature = "blurhashing")] fn get_blurhash_from_request( data: &[u8], mime: Option<&str>, @@ -53,8 +64,7 @@ fn get_blurhash_from_request( return Err(BlurhashingError::ImageTooLarge); } - // decode the image finally - let image = DynamicImage::from_decoder(decoder)?; + let image = image::DynamicImage::from_decoder(decoder)?; blurhash_an_image(&image, config) } @@ -64,31 +74,34 @@ fn get_blurhash_from_request( /// Then it checks if the filename has a format, otherwise just guess based on /// the binary data Assumes that mime and filename extension won't be for a /// different file format than file. +#[cfg(feature = "blurhashing")] fn get_format_from_data_mime_and_filename( data: &[u8], mime: Option<&str>, filename: Option<&str>, -) -> Result { +) -> Result { let extension = filename - .map(Path::new) - .and_then(Path::extension) - .map(OsStr::to_string_lossy); + .map(std::path::Path::new) + .and_then(std::path::Path::extension) + .map(std::ffi::OsStr::to_string_lossy); mime.or(extension.as_deref()) - .and_then(ImageFormat::from_mime_type) + .and_then(image::ImageFormat::from_mime_type) .map_or_else(|| image::guess_format(data).map_err(Into::into), Ok) } +#[cfg(feature = "blurhashing")] fn get_image_decoder_with_format_and_data( - image_format: ImageFormat, + image_format: image::ImageFormat, data: &[u8], -) -> Result, BlurhashingError> { - let mut image_reader = ImageReader::new(Cursor::new(data)); +) -> Result, BlurhashingError> { + let mut image_reader = image::ImageReader::new(std::io::Cursor::new(data)); image_reader.set_format(image_format); Ok(Box::new(image_reader.into_decoder()?)) } -fn is_image_above_size_limit( +#[cfg(feature = "blurhashing")] +fn is_image_above_size_limit( decoder: &T, blurhash_config: BlurhashConfig, ) -> bool { @@ -99,7 +112,7 @@ fn is_image_above_size_limit( #[tracing::instrument(name = "encode", level = "debug", skip_all)] #[inline] fn blurhash_an_image( - image: &DynamicImage, + image: &image::DynamicImage, blurhash_config: BlurhashConfig, ) -> Result { Ok(blurhash::encode_image( @@ -109,15 +122,6 @@ fn blurhash_an_image( )?) } -#[cfg(not(feature = "blurhashing"))] -#[inline] -fn blurhash_an_image( - _image: &DynamicImage, - _blurhash_config: BlurhashConfig, -) -> Result { - Err(BlurhashingError::Unavailable) -} - #[derive(Clone, Copy, Debug)] pub struct BlurhashConfig { pub components_x: u32, @@ -127,6 +131,7 @@ pub struct BlurhashConfig { pub size_limit: u64, } +#[cfg(feature = "blurhashing")] impl From for BlurhashConfig { fn from(value: CoreBlurhashConfig) -> Self { Self { @@ -138,17 +143,17 @@ impl From for BlurhashConfig { } #[derive(Debug)] +#[cfg(feature = "blurhashing")] pub enum BlurhashingError { - HashingLibError(Box), - ImageError(Box), + HashingLibError(Box), + #[cfg(feature = "blurhashing")] + ImageError(Box), ImageTooLarge, - - #[cfg(not(feature = "blurhashing"))] - Unavailable, } -impl From for BlurhashingError { - fn from(value: ImageError) -> Self { Self::ImageError(Box::new(value)) } +#[cfg(feature = "blurhashing")] +impl From for BlurhashingError { + fn from(value: image::ImageError) -> Self { Self::ImageError(Box::new(value)) } } #[cfg(feature = "blurhashing")] @@ -156,19 +161,17 @@ impl From for BlurhashingError { fn from(value: blurhash::Error) -> Self { Self::HashingLibError(Box::new(value)) } } -impl Display for BlurhashingError { +#[cfg(feature = "blurhashing")] +impl std::fmt::Display for BlurhashingError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Blurhash Error:")?; match &self { | Self::ImageTooLarge => write!(f, "Image was too large to blurhash")?, | Self::HashingLibError(e) => write!(f, "There was an error with the blurhashing library => {e}")?, - + #[cfg(feature = "blurhashing")] | Self::ImageError(e) => write!(f, "There was an error with the image loading library => {e}")?, - - #[cfg(not(feature = "blurhashing"))] - | Self::Unavailable => write!(f, "Blurhashing is not supported")?, }; Ok(()) diff --git a/src/service/media/migrations.rs b/src/service/media/migrations.rs index 9555edd7..8526ffcd 100644 --- a/src/service/media/migrations.rs +++ b/src/service/media/migrations.rs @@ -13,7 +13,7 @@ use conduwuit::{ warn, Config, Result, }; -use crate::{migrations, Services}; +use crate::Services; /// Migrates a media directory from legacy base64 file names to sha2 file names. /// All errors are fatal. Upon success the database is keyed to not perform this @@ -48,12 +48,6 @@ pub(crate) async fn migrate_sha256_media(services: &Services) -> Result<()> { } } - // Apply fix from when sha256_media was backward-incompat and bumped the schema - // version from 13 to 14. For users satisfying these conditions we can go back. - if services.globals.db.database_version().await == 14 && migrations::DATABASE_VERSION == 13 { - services.globals.db.bump_database_version(13)?; - } - db["global"].insert(b"feat_sha256_media", []); info!("Finished applying sha256_media"); Ok(()) From c7c9f0e4a60ffd4b497bb8e426ffc34c5e118913 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 6 Feb 2025 16:57:30 -0500 Subject: [PATCH 1491/2091] catch clippy lints for --no-default-features builds Signed-off-by: strawberry --- engage.toml | 50 +++++++++++++++++++++++++++++++------------------- 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/engage.toml b/engage.toml index 1d6a5475..279e999c 100644 --- a/engage.toml +++ b/engage.toml @@ -101,7 +101,6 @@ direnv exec . \ cargo clippy \ --workspace \ --profile test \ - --all-targets \ --color=always \ -- \ -D warnings @@ -116,13 +115,27 @@ env DIRENV_DEVSHELL=all-features \ cargo clippy \ --workspace \ --profile test \ - --all-targets \ --all-features \ --color=always \ -- \ -D warnings """ +[[task]] +name = "clippy/no-features" +group = "lints" +script = """ +env DIRENV_DEVSHELL=no-features \ + direnv exec . \ + cargo clippy \ + --workspace \ + --profile test \ + --no-default-features \ + --color=always \ + -- \ + -D warnings +""" + [[task]] name = "clippy/jemalloc" group = "lints" @@ -131,26 +144,12 @@ direnv exec . \ cargo clippy \ --workspace \ --profile test \ - --features jemalloc \ - --all-targets \ + --features=jemalloc \ --color=always \ -- \ -D warnings """ -#[[task]] -#name = "clippy/hardened_malloc" -#group = "lints" -#script = """ -#cargo clippy \ -# --workspace \ -# --features hardened_malloc \ -# --all-targets \ -# --color=always \ -# -- \ -# -D warnings -#""" - [[task]] name = "lychee" group = "lints" @@ -170,7 +169,6 @@ env DIRENV_DEVSHELL=all-features \ cargo test \ --workspace \ --profile test \ - --all-targets \ --all-features \ --color=always \ -- \ @@ -186,7 +184,21 @@ env DIRENV_DEVSHELL=default \ cargo test \ --workspace \ --profile test \ - --all-targets \ + --color=always \ + -- \ + --color=always +""" + +[[task]] +name = "cargo/no-features" +group = "tests" +script = """ +env DIRENV_DEVSHELL=no-features \ + direnv exec . \ + cargo test \ + --workspace \ + --profile test \ + --no-default-features \ --color=always \ -- \ --color=always From 43e6c27bb772461722409e9c56146a106d6c6343 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 6 Feb 2025 18:07:49 -0500 Subject: [PATCH 1492/2091] misc nix tweaks to maybe speedup ci Signed-off-by: strawberry --- bin/complement | 3 +- flake.nix | 14 +------- nix/pkgs/complement/config.toml | 21 +++++++++--- nix/pkgs/complement/default.nix | 6 ---- nix/pkgs/main/default.nix | 58 +++++++++++++++++---------------- src/router/serve/tls.rs | 9 +++-- 6 files changed, 54 insertions(+), 57 deletions(-) diff --git a/bin/complement b/bin/complement index a1db4b32..a4c62856 100755 --- a/bin/complement +++ b/bin/complement @@ -34,7 +34,8 @@ toplevel="$(git rev-parse --show-toplevel)" pushd "$toplevel" > /dev/null -bin/nix-build-and-cache just .#linux-complement +#bin/nix-build-and-cache just .#linux-complement +bin/nix-build-and-cache just .#complement docker load < result popd > /dev/null diff --git a/flake.nix b/flake.nix index 920d3d14..3cef1af5 100644 --- a/flake.nix +++ b/flake.nix @@ -169,21 +169,9 @@ # used for rust caching in CI to speed it up sccache - - # needed so we can get rid of gcc and other unused deps that bloat OCI images - removeReferencesTo ] # liburing is Linux-exclusive - ++ lib.optional stdenv.hostPlatform.isLinux liburing - # needed to build Rust applications on macOS - ++ lib.optionals stdenv.hostPlatform.isDarwin [ - # https://github.com/NixOS/nixpkgs/issues/206242 - # ld: library not found for -liconv - libiconv - # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell - # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612 - pkgsBuildHost.darwin.apple_sdk.frameworks.Security - ]) + ++ lib.optional stdenv.hostPlatform.isLinux liburing) ++ scope.main.buildInputs ++ scope.main.propagatedBuildInputs ++ scope.main.nativeBuildInputs; diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index f20abee2..99c151c5 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -17,19 +17,32 @@ ip_range_denylist = [] url_preview_domain_contains_allowlist = ["*"] url_preview_domain_explicit_denylist = ["*"] media_compat_file_link = false -media_startup_check = false -prune_missing_media = false +media_startup_check = true +prune_missing_media = true log_colors = false admin_room_notices = false allow_check_for_updates = false -allow_unstable_room_versions = true +intentionally_unknown_config_option_for_testing = true rocksdb_log_level = "debug" rocksdb_max_log_files = 1 rocksdb_recovery_mode = 0 rocksdb_paranoid_file_checks = true log_guest_registrations = false allow_legacy_media = true -startup_netburst = false +startup_netburst = true +startup_netburst_keep = -1 + +# valgrind makes things so slow +dns_timeout = 60 +dns_attempts = 20 +request_conn_timeout = 60 +request_timeout = 120 +well_known_conn_timeout = 60 +well_known_timeout = 60 +federation_idle_timeout = 300 +sender_timeout = 300 +sender_idle_timeout = 300 +sender_retry_backoff_limit = 300 [global.tls] certs = "/certificate.crt" diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index e35cbf04..d9af0779 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -18,18 +18,12 @@ let all_features = true; disable_release_max_log_level = true; disable_features = [ - # no reason to use jemalloc for complement, just has compatibility/build issues - "jemalloc" - "jemalloc_stats" - "jemalloc_prof" # console/CLI stuff isn't used or relevant for complement "console" "tokio_console" # sentry telemetry isn't useful for complement, disabled by default anyways "sentry_telemetry" "perf_measurements" - # the containers don't use or need systemd signal support - "systemd" # this is non-functional on nix for some reason "hardened_malloc" # dont include experimental features diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index d7424d11..4150b389 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -82,7 +82,7 @@ rust-jemalloc-sys' = (rust-jemalloc-sys.override { buildDepsOnlyEnv = let rocksdb' = (rocksdb.override { - jemalloc = rust-jemalloc-sys'; + jemalloc = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'; # rocksdb fails to build with prefixed jemalloc, which is required on # darwin due to [1]. In this case, fall back to building rocksdb with # libc malloc. This should not cause conflicts, because all of the @@ -103,6 +103,12 @@ buildDepsOnlyEnv = ++ [ "-DPORTABLE=haswell" ]) else ([ "-DPORTABLE=1" ]) ) ++ old.cmakeFlags; + + # outputs has "tools" which we dont need or use + outputs = [ "out" ]; + + # preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use + preInstall = ""; }); in { @@ -156,6 +162,19 @@ commonAttrs = { ]; }; + # This is redundant with CI + doCheck = false; + + cargoTestCommand = "cargo test --locked "; + cargoExtraArgs = "--no-default-features --locked " + + lib.optionalString + (features'' != []) + "--features " + (builtins.concatStringsSep "," features''); + cargoTestExtraArgs = "--no-default-features --locked " + + lib.optionalString + (features'' != []) + "--features " + (builtins.concatStringsSep "," features''); + dontStrip = profile == "dev" || profile == "test"; dontPatchELF = profile == "dev" || profile == "test"; @@ -181,27 +200,7 @@ commonAttrs = { # differing values for `NIX_CFLAGS_COMPILE`, which contributes to spurious # rebuilds of bindgen and its depedents. jq - - # needed so we can get rid of gcc and other unused deps that bloat OCI images - removeReferencesTo - ] - # needed to build Rust applications on macOS - ++ lib.optionals stdenv.hostPlatform.isDarwin [ - # https://github.com/NixOS/nixpkgs/issues/206242 - # ld: library not found for -liconv - libiconv - - # https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell - # https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612 - pkgsBuildHost.darwin.apple_sdk.frameworks.Security - ]; - - # for some reason gcc and other weird deps are added to OCI images and bloats it up - # - # - postInstall = with pkgsBuildHost; '' - find "$out" -type f -exec remove-references-to -t ${stdenv.cc} -t ${gcc} -t ${llvm} -t ${rustc.unwrapped} -t ${rustc} '{}' + - ''; + ]; }; in @@ -210,15 +209,18 @@ craneLib.buildPackage ( commonAttrs // { env = buildDepsOnlyEnv; }); - cargoExtraArgs = "--no-default-features " + # This is redundant with CI + doCheck = false; + + cargoTestCommand = "cargo test --locked "; + cargoExtraArgs = "--no-default-features --locked " + + lib.optionalString + (features'' != []) + "--features " + (builtins.concatStringsSep "," features''); + cargoTestExtraArgs = "--no-default-features --locked " + lib.optionalString (features'' != []) "--features " + (builtins.concatStringsSep "," features''); - - # This is redundant with CI - cargoTestCommand = ""; - cargoCheckCommand = ""; - doCheck = false; env = buildPackageEnv; diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index 9d3fbd3b..ab1a9371 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -17,14 +17,13 @@ pub(super) async fn serve( addrs: Vec, ) -> Result { let tls = &server.config.tls; - let certs = tls - .certs - .as_ref() - .ok_or(err!(Config("tls.certs", "Missing required value in tls config section")))?; + let certs = tls.certs.as_ref().ok_or_else(|| { + err!(Config("tls.certs", "Missing required value in tls config section")) + })?; let key = tls .key .as_ref() - .ok_or(err!(Config("tls.key", "Missing required value in tls config section")))?; + .ok_or_else(|| err!(Config("tls.key", "Missing required value in tls config section")))?; // we use ring for ruma and hashing state, but aws-lc-rs is the new default. // without this, TLS mode will panic. From add2e0e9eefc2cfcc154b1e4877988f15ca682a7 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 6 Feb 2025 18:20:02 -0500 Subject: [PATCH 1493/2091] bump rust-rocksdb Signed-off-by: strawberry --- Cargo.lock | 4 ++-- deps/rust-rocksdb/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 926099b5..82962421 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3711,7 +3711,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.32.0+9.10.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=1f032427d3a0e7b0f13c04b4e34712bd8610291b#1f032427d3a0e7b0f13c04b4e34712bd8610291b" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=7b0e1bbe395a41ba8a11347a4921da590e3ad0d9#7b0e1bbe395a41ba8a11347a4921da590e3ad0d9" dependencies = [ "bindgen", "bzip2-sys", @@ -3728,7 +3728,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.36.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=1f032427d3a0e7b0f13c04b4e34712bd8610291b#1f032427d3a0e7b0f13c04b4e34712bd8610291b" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=7b0e1bbe395a41ba8a11347a4921da590e3ad0d9#7b0e1bbe395a41ba8a11347a4921da590e3ad0d9" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index ba8259a3..c6af428d 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "1f032427d3a0e7b0f13c04b4e34712bd8610291b" +rev = "7b0e1bbe395a41ba8a11347a4921da590e3ad0d9" #branch = "master" default-features = false From 8345ea2cd31d26bcf5c5eb61bbda5cd9958c11c5 Mon Sep 17 00:00:00 2001 From: strawberry Date: Thu, 6 Feb 2025 18:47:54 -0500 Subject: [PATCH 1494/2091] add --locked and --no-fail-fast to cargo test, add other feature test Signed-off-by: strawberry --- engage.toml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/engage.toml b/engage.toml index 279e999c..c1a2be1f 100644 --- a/engage.toml +++ b/engage.toml @@ -86,6 +86,7 @@ env DIRENV_DEVSHELL=all-features \ direnv exec . \ cargo doc \ --workspace \ + --locked \ --profile test \ --all-features \ --no-deps \ @@ -100,6 +101,7 @@ script = """ direnv exec . \ cargo clippy \ --workspace \ + --locked \ --profile test \ --color=always \ -- \ @@ -114,6 +116,7 @@ env DIRENV_DEVSHELL=all-features \ direnv exec . \ cargo clippy \ --workspace \ + --locked \ --profile test \ --all-features \ --color=always \ @@ -129,6 +132,7 @@ env DIRENV_DEVSHELL=no-features \ direnv exec . \ cargo clippy \ --workspace \ + --locked \ --profile test \ --no-default-features \ --color=always \ @@ -137,14 +141,16 @@ env DIRENV_DEVSHELL=no-features \ """ [[task]] -name = "clippy/jemalloc" +name = "clippy/other-features" group = "lints" script = """ direnv exec . \ cargo clippy \ --workspace \ + --locked \ --profile test \ - --features=jemalloc \ + --no-default-features \ + --features=console,systemd,element_hacks,direct_tls,perf_measurements,brotli_compression,blurhashing \ --color=always \ -- \ -D warnings @@ -168,7 +174,10 @@ env DIRENV_DEVSHELL=all-features \ direnv exec . \ cargo test \ --workspace \ + --locked \ --profile test \ + --all-targets \ + --no-fail-fast \ --all-features \ --color=always \ -- \ @@ -183,7 +192,10 @@ env DIRENV_DEVSHELL=default \ direnv exec . \ cargo test \ --workspace \ + --locked \ --profile test \ + --all-targets \ + --no-fail-fast \ --color=always \ -- \ --color=always @@ -197,7 +209,10 @@ env DIRENV_DEVSHELL=no-features \ direnv exec . \ cargo test \ --workspace \ + --locked \ --profile test \ + --all-targets \ + --no-fail-fast \ --no-default-features \ --color=always \ -- \ From 88e7e50daff94ef8e3fe3d67e72214f002fdb22b Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Feb 2025 11:49:00 -0500 Subject: [PATCH 1495/2091] add missing source OCI image label metadata Signed-off-by: strawberry --- nix/pkgs/oci-image/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/pkgs/oci-image/default.nix b/nix/pkgs/oci-image/default.nix index 5520c920..1650053d 100644 --- a/nix/pkgs/oci-image/default.nix +++ b/nix/pkgs/oci-image/default.nix @@ -36,6 +36,7 @@ dockerTools.buildLayeredImage { "org.opencontainers.image.documentation" = "https://conduwuit.puppyirl.gay/"; "org.opencontainers.image.licenses" = "Apache-2.0"; "org.opencontainers.image.revision" = inputs.self.rev or inputs.self.dirtyRev or ""; + "org.opencontainers.image.source" = "https://github.com/girlbossceo/conduwuit"; "org.opencontainers.image.title" = main.pname; "org.opencontainers.image.url" = "https://conduwuit.puppyirl.gay/"; "org.opencontainers.image.vendor" = "girlbossceo"; From cfcd6eb1a6a117db94e6f9e631a0d881a62d3299 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Feb 2025 18:00:58 -0500 Subject: [PATCH 1496/2091] bump ruwuma to stop erroring on empty push response body Signed-off-by: strawberry --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 82962421..caef5859 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3490,7 +3490,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "assign", "js_int", @@ -3512,7 +3512,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "js_int", "ruma-common", @@ -3524,7 +3524,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "as_variant", "assign", @@ -3547,7 +3547,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "as_variant", "base64 0.22.1", @@ -3578,7 +3578,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "as_variant", "indexmap 2.7.0", @@ -3603,7 +3603,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "bytes", "http", @@ -3621,7 +3621,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3630,7 +3630,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "js_int", "ruma-common", @@ -3640,7 +3640,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3655,7 +3655,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "js_int", "ruma-common", @@ -3667,7 +3667,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "headers", "http", @@ -3680,7 +3680,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3696,7 +3696,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=517ac4572276a2e0ad587113776c544b51166f08#517ac4572276a2e0ad587113776c544b51166f08" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index ce483bbc..38654be3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -342,7 +342,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "517ac4572276a2e0ad587113776c544b51166f08" +rev = "f5667c6292adb43fbe4725d31d6b5127a0cf60ce" features = [ "compat", "rand", From b6e9dc3d98704c56027219d3775336910a0136c6 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sun, 9 Feb 2025 10:17:28 -0500 Subject: [PATCH 1497/2091] comment out borked ci thing for now Signed-off-by: strawberry --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 35d60aa1..24f2db45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,7 +128,7 @@ jobs: - name: Restore and cache Nix store # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags - if: ${{ !startsWith(github.ref, 'refs/tags/') }} + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} uses: nix-community/cache-nix-action@v5.1.0 with: # restore and save a cache using this key @@ -191,14 +191,14 @@ jobs: - name: Run sccache-cache # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags - if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} + #if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} uses: mozilla-actions/sccache-action@main # use rust-cache - uses: Swatinem/rust-cache@v2 # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags - if: ${{ !startsWith(github.ref, 'refs/tags/') }} + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} with: cache-all-crates: "true" cache-on-failure: "true" @@ -323,7 +323,7 @@ jobs: - name: Restore and cache Nix store # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags - if: ${{ !startsWith(github.ref, 'refs/tags/') }} + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} uses: nix-community/cache-nix-action@v5.1.0 with: # restore and save a cache using this key @@ -379,14 +379,14 @@ jobs: - name: Run sccache-cache # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags - if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} + #if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} uses: mozilla-actions/sccache-action@main # use rust-cache - uses: Swatinem/rust-cache@v2 # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags - if: ${{ !startsWith(github.ref, 'refs/tags/') }} + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} with: cache-all-crates: "true" cache-on-failure: "true" @@ -679,7 +679,7 @@ jobs: - name: Run sccache-cache # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags - if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} + #if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} uses: mozilla-actions/sccache-action@main # use rust-cache From e3b81f7b6488b5c483e8b13e3959fe591bf4cb92 Mon Sep 17 00:00:00 2001 From: Dzming Li Date: Mon, 10 Feb 2025 22:45:57 +0800 Subject: [PATCH 1498/2091] Fix in caddyfile guide If the reverse_proxy directive is omitted before 127.0.0.1:6167 in your Caddyfile, enabling the service with systemctl enable will result in an error. --- docs/deploying/generic.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index cc50544e..8ca2f387 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -216,7 +216,7 @@ your server name). ```caddyfile your.server.name, your.server.name:8448 { # TCP reverse_proxy - 127.0.0.1:6167 + reverse_proxy 127.0.0.1:6167 # UNIX socket #reverse_proxy unix//run/conduwuit/conduwuit.sock } From 3ec43be95965488d720403264c4edc6170c67c02 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Feb 2025 04:30:17 +0000 Subject: [PATCH 1499/2091] join initial fetches in get_relations() skip recursion for max_depth=0 Signed-off-by: Jason Volk --- src/service/rooms/pdu_metadata/mod.rs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 4cb14ebc..ba289f9b 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -2,7 +2,7 @@ mod data; use std::sync::Arc; use conduwuit::{PduCount, Result}; -use futures::StreamExt; +use futures::{future::try_join, StreamExt}; use ruma::{api::Direction, EventId, RoomId, UserId}; use self::data::{Data, PdusIterItem}; @@ -54,10 +54,16 @@ impl Service { max_depth: u8, dir: Direction, ) -> Vec { - let room_id = self.services.short.get_or_create_shortroomid(room_id).await; + let room_id = self.services.short.get_shortroomid(room_id); - let target = match self.services.timeline.get_pdu_count(target).await { - | Ok(PduCount::Normal(c)) => c, + let target = self.services.timeline.get_pdu_count(target); + + let Ok((room_id, target)) = try_join(room_id, target).await else { + return Vec::new(); + }; + + let target = match target { + | PduCount::Normal(c) => c, // TODO: Support backfilled relations | _ => 0, // This will result in an empty iterator }; @@ -68,7 +74,11 @@ impl Service { .collect() .await; - let mut stack: Vec<_> = pdus.iter().map(|pdu| (pdu.clone(), 1)).collect(); + let mut stack: Vec<_> = pdus + .iter() + .filter(|_| max_depth > 0) + .map(|pdu| (pdu.clone(), 1)) + .collect(); 'limit: while let Some(stack_pdu) = stack.pop() { let target = match stack_pdu.0 .0 { From 2d71d5590a81cd26f22181131d2e5a6439fe391d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Feb 2025 09:53:53 +0000 Subject: [PATCH 1500/2091] fix pdu add_relation() helper Signed-off-by: Jason Volk --- src/core/pdu/unsigned.rs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/core/pdu/unsigned.rs b/src/core/pdu/unsigned.rs index fa305d71..fe4d6a1c 100644 --- a/src/core/pdu/unsigned.rs +++ b/src/core/pdu/unsigned.rs @@ -46,23 +46,26 @@ pub fn add_age(&mut self) -> Result { } #[implement(Pdu)] -pub fn add_relation(&mut self, name: &str, pdu: &Pdu) -> Result { - let mut unsigned: BTreeMap = self +pub fn add_relation(&mut self, name: &str, pdu: Option<&Pdu>) -> Result { + use serde_json::Map; + + let mut unsigned: Map = self .unsigned .as_ref() - .map_or_else(|| Ok(BTreeMap::new()), |u| serde_json::from_str(u.get())) + .map_or_else(|| Ok(Map::new()), |u| serde_json::from_str(u.get())) .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; - let relations: &mut JsonValue = unsigned.entry("m.relations".into()).or_default(); - if relations.as_object_mut().is_none() { - let mut object = serde_json::Map::::new(); - _ = relations.as_object_mut().insert(&mut object); - } + let pdu = pdu + .map(serde_json::to_value) + .transpose()? + .unwrap_or_else(|| JsonValue::Object(Map::new())); - relations + unsigned + .entry("m.relations") + .or_insert(JsonValue::Object(Map::new())) .as_object_mut() - .expect("we just created it") - .insert(name.to_owned(), serde_json::to_value(pdu)?); + .unwrap() + .insert(name.to_owned(), pdu); self.unsigned = to_raw_value(&unsigned) .map(Some) From 565837ad753bbd6d346157c5b52a6a0275984e50 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Feb 2025 04:21:39 +0000 Subject: [PATCH 1501/2091] request auth media first Signed-off-by: Jason Volk --- src/service/media/remote.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index ca73c3ef..72f1184e 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -32,12 +32,12 @@ pub async fn fetch_remote_thumbnail( self.check_fetch_authorized(mxc)?; let result = self - .fetch_thumbnail_unauthenticated(mxc, user, server, timeout_ms, dim) + .fetch_thumbnail_authenticated(mxc, user, server, timeout_ms, dim) .await; if let Err(Error::Request(NotFound, ..)) = &result { return self - .fetch_thumbnail_authenticated(mxc, user, server, timeout_ms, dim) + .fetch_thumbnail_unauthenticated(mxc, user, server, timeout_ms, dim) .await; } @@ -55,12 +55,12 @@ pub async fn fetch_remote_content( self.check_fetch_authorized(mxc)?; let result = self - .fetch_content_unauthenticated(mxc, user, server, timeout_ms) + .fetch_content_authenticated(mxc, user, server, timeout_ms) .await; if let Err(Error::Request(NotFound, ..)) = &result { return self - .fetch_content_authenticated(mxc, user, server, timeout_ms) + .fetch_content_unauthenticated(mxc, user, server, timeout_ms) .await; } From 31ab84e9284ce7d5b6ec9fb212970b1a9e18fe7f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Feb 2025 10:23:17 +0000 Subject: [PATCH 1502/2091] simplify client event endpoint Signed-off-by: Jason Volk --- src/api/client/message.rs | 53 ++++++++++++++++++++++++++---------- src/api/client/room/event.rs | 40 +++++++++++---------------- src/core/pdu/unsigned.rs | 31 +++++++++++---------- 3 files changed, 71 insertions(+), 53 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 321d8013..bb4e72dd 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,6 +1,6 @@ use axum::extract::State; use conduwuit::{ - at, is_equal_to, + at, utils::{ result::{FlatOk, LogErr}, stream::{BroadbandExt, TryIgnore, WidebandExt}, @@ -30,7 +30,7 @@ use service::{ use crate::Ruma; /// list of safe and common non-state events to ignore if the user is ignored -const IGNORED_MESSAGE_TYPES: &[TimelineEventType; 17] = &[ +const IGNORED_MESSAGE_TYPES: &[TimelineEventType] = &[ Audio, CallInvite, Emote, @@ -225,34 +225,50 @@ async fn get_member_event( .ok() } +#[inline] pub(crate) async fn ignored_filter( services: &Services, item: PdusIterItem, user_id: &UserId, ) -> Option { - let (_, pdu) = &item; + let (_, ref pdu) = item; + is_ignored_pdu(services, pdu, user_id) + .await + .eq(&false) + .then_some(item) +} + +#[inline] +pub(crate) async fn is_ignored_pdu( + services: &Services, + pdu: &PduEvent, + user_id: &UserId, +) -> bool { // exclude Synapse's dummy events from bloating up response bodies. clients // don't need to see this. if pdu.kind.to_cow_str() == "org.matrix.dummy_event" { - return None; + return true; } - if IGNORED_MESSAGE_TYPES.binary_search(&pdu.kind).is_ok() - && (services.users.user_is_ignored(&pdu.sender, user_id).await - || services - .server - .config - .forbidden_remote_server_names - .iter() - .any(is_equal_to!(pdu.sender().server_name()))) + let ignored_type = IGNORED_MESSAGE_TYPES.binary_search(&pdu.kind).is_ok(); + + let ignored_server = services + .server + .config + .forbidden_remote_server_names + .contains(pdu.sender().server_name()); + + if ignored_type + && (ignored_server || services.users.user_is_ignored(&pdu.sender, user_id).await) { - return None; + return true; } - Some(item) + false } +#[inline] pub(crate) async fn visibility_filter( services: &Services, item: PdusIterItem, @@ -268,7 +284,16 @@ pub(crate) async fn visibility_filter( .then_some(item) } +#[inline] pub(crate) fn event_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Option { let (_, pdu) = &item; pdu.matches(filter).then_some(item) } + +#[cfg_attr(debug_assertions, conduwuit::ctor)] +fn _is_sorted() { + debug_assert!( + IGNORED_MESSAGE_TYPES.is_sorted(), + "IGNORED_MESSAGE_TYPES must be sorted by the developer" + ); +} diff --git a/src/api/client/room/event.rs b/src/api/client/room/event.rs index bc5ec0d7..f0ae64dd 100644 --- a/src/api/client/room/event.rs +++ b/src/api/client/room/event.rs @@ -1,52 +1,44 @@ use axum::extract::State; use conduwuit::{err, Err, Event, Result}; -use futures::{try_join, FutureExt, TryFutureExt}; +use futures::{future::try_join, FutureExt, TryFutureExt}; use ruma::api::client::room::get_room_event; -use crate::{client::ignored_filter, Ruma}; +use crate::{client::is_ignored_pdu, Ruma}; /// # `GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}` /// /// Gets a single event. pub(crate) async fn get_room_event_route( - State(services): State, + State(ref services): State, ref body: Ruma, ) -> Result { + let event_id = &body.event_id; + let room_id = &body.room_id; + let event = services .rooms .timeline - .get_pdu(&body.event_id) - .map_err(|_| err!(Request(NotFound("Event {} not found.", &body.event_id)))); - - let token = services - .rooms - .timeline - .get_pdu_count(&body.event_id) - .map_err(|_| err!(Request(NotFound("Event not found.")))); + .get_pdu(event_id) + .map_err(|_| err!(Request(NotFound("Event {} not found.", event_id)))); let visible = services .rooms .state_accessor - .user_can_see_event(body.sender_user(), &body.room_id, &body.event_id) + .user_can_see_event(body.sender_user(), room_id, event_id) .map(Ok); - let (token, mut event, visible) = try_join!(token, event, visible)?; + let (mut event, visible) = try_join(event, visible).await?; - if !visible - || ignored_filter(&services, (token, event.clone()), body.sender_user()) - .await - .is_none() - { + if !visible || is_ignored_pdu(services, &event, body.sender_user()).await { return Err!(Request(Forbidden("You don't have permission to view this event."))); } - if event.event_id() != &body.event_id || event.room_id() != body.room_id { - return Err!(Request(NotFound("Event not found"))); - } + debug_assert!( + event.event_id() == event_id && event.room_id() == room_id, + "Fetched PDU must match requested" + ); event.add_age().ok(); - let event = event.to_room_event(); - - Ok(get_room_event::v3::Response { event }) + Ok(get_room_event::v3::Response { event: event.to_room_event() }) } diff --git a/src/core/pdu/unsigned.rs b/src/core/pdu/unsigned.rs index fe4d6a1c..8482a48a 100644 --- a/src/core/pdu/unsigned.rs +++ b/src/core/pdu/unsigned.rs @@ -9,11 +9,13 @@ use crate::{err, implement, is_true, Result}; #[implement(Pdu)] pub fn remove_transaction_id(&mut self) -> Result { + use BTreeMap as Map; + let Some(unsigned) = &self.unsigned else { return Ok(()); }; - let mut unsigned: BTreeMap> = serde_json::from_str(unsigned.get()) + let mut unsigned: Map<&str, Box> = serde_json::from_str(unsigned.get()) .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; unsigned.remove("transaction_id"); @@ -26,10 +28,13 @@ pub fn remove_transaction_id(&mut self) -> Result { #[implement(Pdu)] pub fn add_age(&mut self) -> Result { - let mut unsigned: BTreeMap> = self + use BTreeMap as Map; + + let mut unsigned: Map<&str, Box> = self .unsigned - .as_ref() - .map_or_else(|| Ok(BTreeMap::new()), |u| serde_json::from_str(u.get())) + .as_deref() + .map(RawJsonValue::get) + .map_or_else(|| Ok(Map::new()), serde_json::from_str) .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; // deliberately allowing for the possibility of negative age @@ -37,10 +42,8 @@ pub fn add_age(&mut self) -> Result { let then: i128 = self.origin_server_ts.into(); let this_age = now.saturating_sub(then); - unsigned.insert("age".to_owned(), to_raw_value(&this_age).expect("age is valid")); - self.unsigned = to_raw_value(&unsigned) - .map(Some) - .expect("unsigned is valid"); + unsigned.insert("age", to_raw_value(&this_age)?); + self.unsigned = Some(to_raw_value(&unsigned)?); Ok(()) } @@ -51,8 +54,9 @@ pub fn add_relation(&mut self, name: &str, pdu: Option<&Pdu>) -> Result { let mut unsigned: Map = self .unsigned - .as_ref() - .map_or_else(|| Ok(Map::new()), |u| serde_json::from_str(u.get())) + .as_deref() + .map(RawJsonValue::get) + .map_or_else(|| Ok(Map::new()), serde_json::from_str) .map_err(|e| err!(Database("Invalid unsigned in pdu event: {e}")))?; let pdu = pdu @@ -64,12 +68,9 @@ pub fn add_relation(&mut self, name: &str, pdu: Option<&Pdu>) -> Result { .entry("m.relations") .or_insert(JsonValue::Object(Map::new())) .as_object_mut() - .unwrap() - .insert(name.to_owned(), pdu); + .map(|object| object.insert(name.to_owned(), pdu)); - self.unsigned = to_raw_value(&unsigned) - .map(Some) - .expect("unsigned is valid"); + self.unsigned = Some(to_raw_value(&unsigned)?); Ok(()) } From d8e94ee965d961fd7c8a042b0ed32d7a38190668 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Feb 2025 20:08:00 +0000 Subject: [PATCH 1503/2091] split spaces service Signed-off-by: Jason Volk --- src/api/client/space.rs | 183 ++++++++++- src/api/server/hierarchy.rs | 70 ++++- src/service/rooms/spaces/mod.rs | 311 +------------------ src/service/rooms/spaces/pagination_token.rs | 76 +++++ 4 files changed, 318 insertions(+), 322 deletions(-) create mode 100644 src/service/rooms/spaces/pagination_token.rs diff --git a/src/api/client/space.rs b/src/api/client/space.rs index 409c9083..8f54de2a 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -1,9 +1,15 @@ -use std::str::FromStr; +use std::{collections::VecDeque, str::FromStr}; use axum::extract::State; +use conduwuit::{checked, pdu::ShortRoomId, utils::stream::IterStream}; +use futures::{StreamExt, TryFutureExt}; use ruma::{ api::client::{error::ErrorKind, space::get_hierarchy}, - UInt, + OwnedRoomId, OwnedServerName, RoomId, UInt, UserId, +}; +use service::{ + rooms::spaces::{get_parent_children_via, summary_to_chunk, SummaryAccessibility}, + Services, }; use crate::{service::rooms::spaces::PaginationToken, Error, Result, Ruma}; @@ -16,8 +22,6 @@ pub(crate) async fn get_hierarchy_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let limit = body .limit .unwrap_or_else(|| UInt::from(10_u32)) @@ -43,16 +47,163 @@ pub(crate) async fn get_hierarchy_route( } } - services - .rooms - .spaces - .get_client_hierarchy( - sender_user, - &body.room_id, - limit.try_into().unwrap_or(10), - key.map_or(vec![], |token| token.short_room_ids), - max_depth.into(), - body.suggested_only, - ) - .await + get_client_hierarchy( + &services, + body.sender_user(), + &body.room_id, + limit.try_into().unwrap_or(10), + key.map_or(vec![], |token| token.short_room_ids), + max_depth.into(), + body.suggested_only, + ) + .await +} + +async fn get_client_hierarchy( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + limit: usize, + short_room_ids: Vec, + max_depth: u64, + suggested_only: bool, +) -> Result { + let mut parents = VecDeque::new(); + + // Don't start populating the results if we have to start at a specific room. + let mut populate_results = short_room_ids.is_empty(); + + let mut stack = vec![vec![(room_id.to_owned(), match room_id.server_name() { + | Some(server_name) => vec![server_name.into()], + | None => vec![], + })]]; + + let mut results = Vec::with_capacity(limit); + + while let Some((current_room, via)) = { next_room_to_traverse(&mut stack, &mut parents) } { + if results.len() >= limit { + break; + } + + match ( + services + .rooms + .spaces + .get_summary_and_children_client(¤t_room, suggested_only, sender_user, &via) + .await?, + current_room == room_id, + ) { + | (Some(SummaryAccessibility::Accessible(summary)), _) => { + let mut children: Vec<(OwnedRoomId, Vec)> = + get_parent_children_via(&summary, suggested_only) + .into_iter() + .filter(|(room, _)| parents.iter().all(|parent| parent != room)) + .rev() + .collect(); + + if populate_results { + results.push(summary_to_chunk(*summary.clone())); + } else { + children = children + .iter() + .rev() + .stream() + .skip_while(|(room, _)| { + services + .rooms + .short + .get_shortroomid(room) + .map_ok(|short| Some(&short) != short_room_ids.get(parents.len())) + .unwrap_or_else(|_| false) + }) + .map(Clone::clone) + .collect::)>>() + .await + .into_iter() + .rev() + .collect(); + + if children.is_empty() { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Room IDs in token were not found.", + )); + } + + // We have reached the room after where we last left off + let parents_len = parents.len(); + if checked!(parents_len + 1)? == short_room_ids.len() { + populate_results = true; + } + } + + let parents_len: u64 = parents.len().try_into()?; + if !children.is_empty() && parents_len < max_depth { + parents.push_back(current_room.clone()); + stack.push(children); + } + // Root room in the space hierarchy, we return an error + // if this one fails. + }, + | (Some(SummaryAccessibility::Inaccessible), true) => { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "The requested room is inaccessible", + )); + }, + | (None, true) => { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "The requested room was not found", + )); + }, + // Just ignore other unavailable rooms + | (None | Some(SummaryAccessibility::Inaccessible), false) => (), + } + } + + Ok(get_hierarchy::v1::Response { + next_batch: if let Some((room, _)) = next_room_to_traverse(&mut stack, &mut parents) { + parents.pop_front(); + parents.push_back(room); + + let next_short_room_ids: Vec<_> = parents + .iter() + .stream() + .filter_map(|room_id| async move { + services.rooms.short.get_shortroomid(room_id).await.ok() + }) + .collect() + .await; + + (next_short_room_ids != short_room_ids && !next_short_room_ids.is_empty()).then( + || { + PaginationToken { + short_room_ids: next_short_room_ids, + limit: UInt::new(max_depth) + .expect("When sent in request it must have been valid UInt"), + max_depth: UInt::new(max_depth) + .expect("When sent in request it must have been valid UInt"), + suggested_only, + } + .to_string() + }, + ) + } else { + None + }, + rooms: results, + }) +} + +fn next_room_to_traverse( + stack: &mut Vec)>>, + parents: &mut VecDeque, +) -> Option<(OwnedRoomId, Vec)> { + while stack.last().is_some_and(Vec::is_empty) { + stack.pop(); + parents.pop_back(); + } + + stack.last_mut().and_then(Vec::pop) } diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs index a10df6ac..bcf2f7bc 100644 --- a/src/api/server/hierarchy.rs +++ b/src/api/server/hierarchy.rs @@ -1,7 +1,12 @@ use axum::extract::State; -use ruma::api::{client::error::ErrorKind, federation::space::get_hierarchy}; +use conduwuit::{Err, Result}; +use ruma::{api::federation::space::get_hierarchy, RoomId, ServerName}; +use service::{ + rooms::spaces::{get_parent_children_via, Identifier, SummaryAccessibility}, + Services, +}; -use crate::{Error, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/hierarchy/{roomId}` /// @@ -11,13 +16,58 @@ pub(crate) async fn get_hierarchy_route( State(services): State, body: Ruma, ) -> Result { - if services.rooms.metadata.exists(&body.room_id).await { - services - .rooms - .spaces - .get_federation_hierarchy(&body.room_id, body.origin(), body.suggested_only) - .await - } else { - Err(Error::BadRequest(ErrorKind::NotFound, "Room does not exist.")) + if !services.rooms.metadata.exists(&body.room_id).await { + return Err!(Request(NotFound("Room does not exist."))); + } + + get_hierarchy(&services, &body.room_id, body.origin(), body.suggested_only).await +} + +/// Gets the response for the space hierarchy over federation request +/// +/// Errors if the room does not exist, so a check if the room exists should +/// be done +async fn get_hierarchy( + services: &Services, + room_id: &RoomId, + server_name: &ServerName, + suggested_only: bool, +) -> Result { + match services + .rooms + .spaces + .get_summary_and_children_local(&room_id.to_owned(), Identifier::ServerName(server_name)) + .await? + { + | Some(SummaryAccessibility::Accessible(room)) => { + let mut children = Vec::new(); + let mut inaccessible_children = Vec::new(); + + for (child, _via) in get_parent_children_via(&room, suggested_only) { + match services + .rooms + .spaces + .get_summary_and_children_local(&child, Identifier::ServerName(server_name)) + .await? + { + | Some(SummaryAccessibility::Accessible(summary)) => { + children.push((*summary).into()); + }, + | Some(SummaryAccessibility::Inaccessible) => { + inaccessible_children.push(child); + }, + | None => (), + } + } + + Ok(get_hierarchy::v1::Response { + room: *room, + children, + inaccessible_children, + }) + }, + | Some(SummaryAccessibility::Inaccessible) => + Err!(Request(NotFound("The requested room is inaccessible"))), + | None => Err!(Request(NotFound("The requested room was not found"))), } } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 11794752..1e2b0a9f 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -1,22 +1,14 @@ +mod pagination_token; mod tests; -use std::{ - collections::{HashMap, VecDeque}, - fmt::{Display, Formatter}, - str::FromStr, - sync::Arc, -}; +use std::{collections::HashMap, sync::Arc}; -use conduwuit::{ - checked, debug_info, err, - utils::{math::usize_from_f64, IterStream}, - Error, Result, -}; -use futures::{StreamExt, TryFutureExt}; +use conduwuit::{debug_info, err, utils::math::usize_from_f64, Error, Result}; +use futures::StreamExt; use lru_cache::LruCache; use ruma::{ api::{ - client::{self, error::ErrorKind, space::SpaceHierarchyRoomsChunk}, + client::{error::ErrorKind, space::SpaceHierarchyRoomsChunk}, federation::{ self, space::{SpaceHierarchyChildSummary, SpaceHierarchyParentSummary}, @@ -29,11 +21,12 @@ use ruma::{ }, serde::Raw, space::SpaceRoomJoinRule, - OwnedRoomId, OwnedServerName, RoomId, ServerName, UInt, UserId, + OwnedRoomId, OwnedServerName, RoomId, ServerName, UserId, }; use tokio::sync::Mutex; -use crate::{rooms, rooms::short::ShortRoomId, sending, Dep}; +pub use self::pagination_token::PaginationToken; +use crate::{rooms, sending, Dep}; pub struct CachedSpaceHierarchySummary { summary: SpaceHierarchyParentSummary, @@ -44,81 +37,10 @@ pub enum SummaryAccessibility { Inaccessible, } -// TODO: perhaps use some better form of token rather than just room count -#[derive(Debug, Eq, PartialEq)] -pub struct PaginationToken { - /// Path down the hierarchy of the room to start the response at, - /// excluding the root space. - pub short_room_ids: Vec, - pub limit: UInt, - pub max_depth: UInt, - pub suggested_only: bool, -} - -impl FromStr for PaginationToken { - type Err = Error; - - fn from_str(value: &str) -> Result { - let mut values = value.split('_'); - - let mut pag_tok = || { - let rooms = values - .next()? - .split(',') - .filter_map(|room_s| u64::from_str(room_s).ok()) - .collect(); - - Some(Self { - short_room_ids: rooms, - limit: UInt::from_str(values.next()?).ok()?, - max_depth: UInt::from_str(values.next()?).ok()?, - suggested_only: { - let slice = values.next()?; - - if values.next().is_none() { - if slice == "true" { - true - } else if slice == "false" { - false - } else { - None? - } - } else { - None? - } - }, - }) - }; - - if let Some(token) = pag_tok() { - Ok(token) - } else { - Err(Error::BadRequest(ErrorKind::InvalidParam, "invalid token")) - } - } -} - -impl Display for PaginationToken { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}_{}_{}_{}", - self.short_room_ids - .iter() - .map(ToString::to_string) - .collect::>() - .join(","), - self.limit, - self.max_depth, - self.suggested_only - ) - } -} - /// Identifier used to check if rooms are accessible /// /// None is used if you want to return the room, no matter if accessible or not -enum Identifier<'a> { +pub enum Identifier<'a> { UserId(&'a UserId), ServerName(&'a ServerName), } @@ -164,60 +86,8 @@ impl crate::Service for Service { } impl Service { - /// Gets the response for the space hierarchy over federation request - /// - /// Errors if the room does not exist, so a check if the room exists should - /// be done - pub async fn get_federation_hierarchy( - &self, - room_id: &RoomId, - server_name: &ServerName, - suggested_only: bool, - ) -> Result { - match self - .get_summary_and_children_local( - &room_id.to_owned(), - Identifier::ServerName(server_name), - ) - .await? - { - | Some(SummaryAccessibility::Accessible(room)) => { - let mut children = Vec::new(); - let mut inaccessible_children = Vec::new(); - - for (child, _via) in get_parent_children_via(&room, suggested_only) { - match self - .get_summary_and_children_local( - &child, - Identifier::ServerName(server_name), - ) - .await? - { - | Some(SummaryAccessibility::Accessible(summary)) => { - children.push((*summary).into()); - }, - | Some(SummaryAccessibility::Inaccessible) => { - inaccessible_children.push(child); - }, - | None => (), - } - } - - Ok(federation::space::get_hierarchy::v1::Response { - room: *room, - children, - inaccessible_children, - }) - }, - | Some(SummaryAccessibility::Inaccessible) => - Err(Error::BadRequest(ErrorKind::NotFound, "The requested room is inaccessible")), - | None => - Err(Error::BadRequest(ErrorKind::NotFound, "The requested room was not found")), - } - } - /// Gets the summary of a space using solely local information - async fn get_summary_and_children_local( + pub async fn get_summary_and_children_local( &self, current_room: &OwnedRoomId, identifier: Identifier<'_>, @@ -366,7 +236,7 @@ impl Service { /// Gets the summary of a space using either local or remote (federation) /// sources - async fn get_summary_and_children_client( + pub async fn get_summary_and_children_client( &self, current_room: &OwnedRoomId, suggested_only: bool, @@ -470,147 +340,6 @@ impl Service { }) } - pub async fn get_client_hierarchy( - &self, - sender_user: &UserId, - room_id: &RoomId, - limit: usize, - short_room_ids: Vec, - max_depth: u64, - suggested_only: bool, - ) -> Result { - let mut parents = VecDeque::new(); - - // Don't start populating the results if we have to start at a specific room. - let mut populate_results = short_room_ids.is_empty(); - - let mut stack = vec![vec![(room_id.to_owned(), match room_id.server_name() { - | Some(server_name) => vec![server_name.into()], - | None => vec![], - })]]; - - let mut results = Vec::with_capacity(limit); - - while let Some((current_room, via)) = { next_room_to_traverse(&mut stack, &mut parents) } - { - if results.len() >= limit { - break; - } - - match ( - self.get_summary_and_children_client( - ¤t_room, - suggested_only, - sender_user, - &via, - ) - .await?, - current_room == room_id, - ) { - | (Some(SummaryAccessibility::Accessible(summary)), _) => { - let mut children: Vec<(OwnedRoomId, Vec)> = - get_parent_children_via(&summary, suggested_only) - .into_iter() - .filter(|(room, _)| parents.iter().all(|parent| parent != room)) - .rev() - .collect(); - - if populate_results { - results.push(summary_to_chunk(*summary.clone())); - } else { - children = children - .iter() - .rev() - .stream() - .skip_while(|(room, _)| { - self.services - .short - .get_shortroomid(room) - .map_ok(|short| { - Some(&short) != short_room_ids.get(parents.len()) - }) - .unwrap_or_else(|_| false) - }) - .map(Clone::clone) - .collect::)>>() - .await - .into_iter() - .rev() - .collect(); - - if children.is_empty() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Room IDs in token were not found.", - )); - } - - // We have reached the room after where we last left off - let parents_len = parents.len(); - if checked!(parents_len + 1)? == short_room_ids.len() { - populate_results = true; - } - } - - let parents_len: u64 = parents.len().try_into()?; - if !children.is_empty() && parents_len < max_depth { - parents.push_back(current_room.clone()); - stack.push(children); - } - // Root room in the space hierarchy, we return an error - // if this one fails. - }, - | (Some(SummaryAccessibility::Inaccessible), true) => { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "The requested room is inaccessible", - )); - }, - | (None, true) => { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "The requested room was not found", - )); - }, - // Just ignore other unavailable rooms - | (None | Some(SummaryAccessibility::Inaccessible), false) => (), - } - } - - Ok(client::space::get_hierarchy::v1::Response { - next_batch: if let Some((room, _)) = next_room_to_traverse(&mut stack, &mut parents) { - parents.pop_front(); - parents.push_back(room); - - let next_short_room_ids: Vec<_> = parents - .iter() - .stream() - .filter_map(|room_id| async move { - self.services.short.get_shortroomid(room_id).await.ok() - }) - .collect() - .await; - - (next_short_room_ids != short_room_ids && !next_short_room_ids.is_empty()).then( - || { - PaginationToken { - short_room_ids: next_short_room_ids, - limit: UInt::new(max_depth) - .expect("When sent in request it must have been valid UInt"), - max_depth: UInt::new(max_depth) - .expect("When sent in request it must have been valid UInt"), - suggested_only, - } - .to_string() - }, - ) - } else { - None - }, - rooms: results, - }) - } - /// Simply returns the stripped m.space.child events of a room async fn get_stripped_space_child_events( &self, @@ -757,7 +486,8 @@ impl From for SpaceHierarchyRoomsChunk { /// Here because cannot implement `From` across ruma-federation-api and /// ruma-client-api types -fn summary_to_chunk(summary: SpaceHierarchyParentSummary) -> SpaceHierarchyRoomsChunk { +#[must_use] +pub fn summary_to_chunk(summary: SpaceHierarchyParentSummary) -> SpaceHierarchyRoomsChunk { let SpaceHierarchyParentSummary { canonical_alias, name, @@ -790,7 +520,8 @@ fn summary_to_chunk(summary: SpaceHierarchyParentSummary) -> SpaceHierarchyRooms /// Returns the children of a SpaceHierarchyParentSummary, making use of the /// children_state field -fn get_parent_children_via( +#[must_use] +pub fn get_parent_children_via( parent: &SpaceHierarchyParentSummary, suggested_only: bool, ) -> Vec<(OwnedRoomId, Vec)> { @@ -808,15 +539,3 @@ fn get_parent_children_via( }) .collect() } - -fn next_room_to_traverse( - stack: &mut Vec)>>, - parents: &mut VecDeque, -) -> Option<(OwnedRoomId, Vec)> { - while stack.last().is_some_and(Vec::is_empty) { - stack.pop(); - parents.pop_back(); - } - - stack.last_mut().and_then(Vec::pop) -} diff --git a/src/service/rooms/spaces/pagination_token.rs b/src/service/rooms/spaces/pagination_token.rs new file mode 100644 index 00000000..8f019e8d --- /dev/null +++ b/src/service/rooms/spaces/pagination_token.rs @@ -0,0 +1,76 @@ +use std::{ + fmt::{Display, Formatter}, + str::FromStr, +}; + +use conduwuit::{Error, Result}; +use ruma::{api::client::error::ErrorKind, UInt}; + +use crate::rooms::short::ShortRoomId; + +// TODO: perhaps use some better form of token rather than just room count +#[derive(Debug, Eq, PartialEq)] +pub struct PaginationToken { + /// Path down the hierarchy of the room to start the response at, + /// excluding the root space. + pub short_room_ids: Vec, + pub limit: UInt, + pub max_depth: UInt, + pub suggested_only: bool, +} + +impl FromStr for PaginationToken { + type Err = Error; + + fn from_str(value: &str) -> Result { + let mut values = value.split('_'); + let mut pag_tok = || { + let short_room_ids = values + .next()? + .split(',') + .filter_map(|room_s| u64::from_str(room_s).ok()) + .collect(); + + let limit = UInt::from_str(values.next()?).ok()?; + let max_depth = UInt::from_str(values.next()?).ok()?; + let slice = values.next()?; + let suggested_only = if values.next().is_none() { + if slice == "true" { + true + } else if slice == "false" { + false + } else { + None? + } + } else { + None? + }; + + Some(Self { + short_room_ids, + limit, + max_depth, + suggested_only, + }) + }; + + if let Some(token) = pag_tok() { + Ok(token) + } else { + Err(Error::BadRequest(ErrorKind::InvalidParam, "invalid token")) + } + } +} + +impl Display for PaginationToken { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let short_room_ids = self + .short_room_ids + .iter() + .map(ToString::to_string) + .collect::>() + .join(","); + + write!(f, "{short_room_ids}_{}_{}_{}", self.limit, self.max_depth, self.suggested_only) + } +} From 5428526120cf49efda7b129d48b5a35ea1d87dde Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Feb 2025 23:03:24 +0000 Subject: [PATCH 1504/2091] add tail-efficient logic extension Signed-off-by: Jason Volk --- src/core/utils/future/bool_ext.rs | 82 +++++++++++++++++++++++++++++++ src/core/utils/future/mod.rs | 2 + 2 files changed, 84 insertions(+) create mode 100644 src/core/utils/future/bool_ext.rs diff --git a/src/core/utils/future/bool_ext.rs b/src/core/utils/future/bool_ext.rs new file mode 100644 index 00000000..6cb2f1fe --- /dev/null +++ b/src/core/utils/future/bool_ext.rs @@ -0,0 +1,82 @@ +//! Extended external extensions to futures::FutureExt + +use std::marker::Unpin; + +use futures::{ + future::{select_ok, try_join, try_join_all, try_select}, + Future, FutureExt, +}; + +pub trait BoolExt +where + Self: Future + Send, +{ + fn and(self, b: B) -> impl Future + Send + where + B: Future + Send, + Self: Sized; + + fn or(self, b: B) -> impl Future + Send + where + B: Future + Send + Unpin, + Self: Sized + Unpin; +} + +pub async fn and(args: I) -> impl Future + Send +where + I: Iterator + Send, + F: Future + Send, +{ + type Result = crate::Result<(), ()>; + + let args = args.map(|a| a.map(|a| a.then_some(()).ok_or(Result::Err(())))); + + try_join_all(args).map(|result| result.is_ok()) +} + +pub async fn or(args: I) -> impl Future + Send +where + I: Iterator + Send, + F: Future + Send + Unpin, +{ + type Result = crate::Result<(), ()>; + + let args = args.map(|a| a.map(|a| a.then_some(()).ok_or(Result::Err(())))); + + select_ok(args).map(|result| result.is_ok()) +} + +impl BoolExt for Fut +where + Fut: Future + Send, +{ + #[inline] + fn and(self, b: B) -> impl Future + Send + where + B: Future + Send, + Self: Sized, + { + type Result = crate::Result<(), ()>; + + let a = self.map(|a| a.then_some(()).ok_or(Result::Err(()))); + + let b = b.map(|b| b.then_some(()).ok_or(Result::Err(()))); + + try_join(a, b).map(|result| result.is_ok()) + } + + #[inline] + fn or(self, b: B) -> impl Future + Send + where + B: Future + Send + Unpin, + Self: Sized + Unpin, + { + type Result = crate::Result<(), ()>; + + let a = self.map(|a| a.then_some(()).ok_or(Result::Err(()))); + + let b = b.map(|b| b.then_some(()).ok_or(Result::Err(()))); + + try_select(a, b).map(|result| result.is_ok()) + } +} diff --git a/src/core/utils/future/mod.rs b/src/core/utils/future/mod.rs index 153dcfe1..2198a84f 100644 --- a/src/core/utils/future/mod.rs +++ b/src/core/utils/future/mod.rs @@ -1,7 +1,9 @@ +mod bool_ext; mod ext_ext; mod option_ext; mod try_ext_ext; +pub use bool_ext::{and, or, BoolExt}; pub use ext_ext::ExtExt; pub use option_ext::OptionExt; pub use try_ext_ext::TryExtExt; From 59c073d0d86ca8a6b9606037e2278890b5b84821 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 6 Feb 2025 23:58:45 +0000 Subject: [PATCH 1505/2091] add unconstrained feature to service worker Signed-off-by: Jason Volk --- src/service/manager.rs | 9 +++++++-- src/service/sending/mod.rs | 13 +++++++++++-- src/service/service.rs | 5 +++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/service/manager.rs b/src/service/manager.rs index ea33d285..e0d885c2 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -1,7 +1,7 @@ use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; use conduwuit::{debug, debug_warn, error, trace, utils::time, warn, Err, Error, Result, Server}; -use futures::FutureExt; +use futures::{FutureExt, TryFutureExt}; use tokio::{ sync::{Mutex, MutexGuard}, task::{JoinHandle, JoinSet}, @@ -183,9 +183,14 @@ async fn worker(service: Arc) -> WorkerResult { let service_ = Arc::clone(&service); let result = AssertUnwindSafe(service_.worker()) .catch_unwind() - .await .map_err(Error::from_panic); + let result = if service.unconstrained() { + tokio::task::unconstrained(result).await + } else { + result.await + }; + // flattens JoinError for panic into worker's Error (service, result.unwrap_or_else(Err)) } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index b146ad49..86b219f7 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -22,7 +22,7 @@ use ruma::{ RoomId, ServerName, UserId, }; use smallvec::SmallVec; -use tokio::task::JoinSet; +use tokio::{task, task::JoinSet}; use self::data::Data; pub use self::{ @@ -111,8 +111,15 @@ impl crate::Service for Service { .enumerate() .fold(JoinSet::new(), |mut joinset, (id, _)| { let self_ = self.clone(); + let worker = self_.sender(id); + let worker = if self.unconstrained() { + task::unconstrained(worker).boxed() + } else { + worker.boxed() + }; + let runtime = self.server.runtime(); - let _abort = joinset.spawn_on(self_.sender(id).boxed(), runtime); + let _abort = joinset.spawn_on(worker, runtime); joinset }); @@ -139,6 +146,8 @@ impl crate::Service for Service { } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } + + fn unconstrained(&self) -> bool { true } } impl Service { diff --git a/src/service/service.rs b/src/service/service.rs index 7adb189e..cad01437 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -39,6 +39,11 @@ pub(crate) trait Service: Any + Send + Sync { /// Return the name of the service. /// i.e. `crate::service::make_name(std::module_path!())` fn name(&self) -> &str; + + /// Return true if the service worker opts out of the tokio cooperative + /// budgeting. This can reduce tail latency at the risk of event loop + /// starvation. + fn unconstrained(&self) -> bool { false } } /// Args are passed to `Service::build` when a service is constructed. This From e123a5b660a21ae444e154ac60812468c878ec58 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 7 Feb 2025 01:16:46 +0000 Subject: [PATCH 1506/2091] add state accessories for iterating state_keys of a type Signed-off-by: Jason Volk --- src/service/rooms/state_accessor/state.rs | 124 ++++++++++++++++++++-- 1 file changed, 114 insertions(+), 10 deletions(-) diff --git a/src/service/rooms/state_accessor/state.rs b/src/service/rooms/state_accessor/state.rs index c47a5693..3cf168c1 100644 --- a/src/service/rooms/state_accessor/state.rs +++ b/src/service/rooms/state_accessor/state.rs @@ -9,7 +9,7 @@ use conduwuit::{ PduEvent, Result, }; use database::Deserialized; -use futures::{future::try_join, FutureExt, Stream, StreamExt, TryFutureExt}; +use futures::{future::try_join, pin_mut, FutureExt, Stream, StreamExt, TryFutureExt}; use ruma::{ events::{ room::member::{MembershipState, RoomMemberEventContent}, @@ -69,7 +69,6 @@ where } #[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] pub async fn state_contains( &self, shortstatehash: ShortStateHash, @@ -90,7 +89,18 @@ pub async fn state_contains( } #[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] +pub async fn state_contains_type( + &self, + shortstatehash: ShortStateHash, + event_type: &StateEventType, +) -> bool { + let state_keys = self.state_keys(shortstatehash, event_type); + + pin_mut!(state_keys); + state_keys.next().await.is_some() +} + +#[implement(super::Service)] pub async fn state_contains_shortstatekey( &self, shortstatehash: ShortStateHash, @@ -125,7 +135,6 @@ pub async fn state_get( /// Returns a single EventId from `room_id` with key (`event_type`, /// `state_key`). #[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] pub async fn state_get_id( &self, shortstatehash: ShortStateHash, @@ -149,7 +158,6 @@ where /// Returns a single EventId from `room_id` with key (`event_type`, /// `state_key`). #[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] pub async fn state_get_shortid( &self, shortstatehash: ShortStateHash, @@ -177,6 +185,103 @@ pub async fn state_get_shortid( .await? } +/// Iterates the state_keys for an event_type in the state; current state +/// event_id included. +#[implement(super::Service)] +pub fn state_keys_with_ids<'a, Id>( + &'a self, + shortstatehash: ShortStateHash, + event_type: &'a StateEventType, +) -> impl Stream + Send + 'a +where + Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a, + ::Owned: Borrow, +{ + let state_keys_with_short_ids = self + .state_keys_with_shortids(shortstatehash, event_type) + .unzip() + .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) + .shared(); + + let state_keys = state_keys_with_short_ids + .clone() + .map(at!(0)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + let shorteventids = state_keys_with_short_ids + .map(at!(1)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + self.services + .short + .multi_get_eventid_from_short(shorteventids) + .zip(state_keys) + .ready_filter_map(|(eid, sk)| eid.map(move |eid| (sk, eid)).ok()) +} + +/// Iterates the state_keys for an event_type in the state; current state +/// event_id included. +#[implement(super::Service)] +pub fn state_keys_with_shortids<'a>( + &'a self, + shortstatehash: ShortStateHash, + event_type: &'a StateEventType, +) -> impl Stream + Send + 'a { + let short_ids = self + .state_full_shortids(shortstatehash) + .expect_ok() + .unzip() + .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) + .shared(); + + let shortstatekeys = short_ids + .clone() + .map(at!(0)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + let shorteventids = short_ids + .map(at!(1)) + .map(Vec::into_iter) + .map(IterStream::stream) + .flatten_stream(); + + self.services + .short + .multi_get_statekey_from_short(shortstatekeys) + .zip(shorteventids) + .ready_filter_map(|(res, id)| res.map(|res| (res, id)).ok()) + .ready_filter_map(move |((event_type_, state_key), event_id)| { + event_type_.eq(event_type).then_some((state_key, event_id)) + }) +} + +/// Iterates the state_keys for an event_type in the state +#[implement(super::Service)] +pub fn state_keys<'a>( + &'a self, + shortstatehash: ShortStateHash, + event_type: &'a StateEventType, +) -> impl Stream + Send + 'a { + let short_ids = self + .state_full_shortids(shortstatehash) + .expect_ok() + .map(at!(0)); + + self.services + .short + .multi_get_statekey_from_short(short_ids) + .ready_filter_map(Result::ok) + .ready_filter_map(move |(event_type_, state_key)| { + event_type_.eq(event_type).then_some(state_key) + }) +} + /// Returns the state events removed between the interval (present in .0 but /// not in .1) #[implement(super::Service)] @@ -191,11 +296,10 @@ pub fn state_removed( /// Returns the state events added between the interval (present in .1 but /// not in .0) #[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] -pub fn state_added<'a>( - &'a self, +pub fn state_added( + &self, shortstatehash: pair_of!(ShortStateHash), -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + '_ { let a = self.load_full_state(shortstatehash.0); let b = self.load_full_state(shortstatehash.1); try_join(a, b) @@ -239,7 +343,6 @@ pub fn state_full_pdus( /// Builds a StateMap by iterating over all keys that start /// with state_hash, this gives the full state for the given state_hash. #[implement(super::Service)] -#[tracing::instrument(skip(self), level = "debug")] pub fn state_full_ids<'a, Id>( &'a self, shortstatehash: ShortStateHash, @@ -293,6 +396,7 @@ pub fn state_full_shortids( } #[implement(super::Service)] +#[tracing::instrument(name = "load", level = "debug", skip(self))] async fn load_full_state(&self, shortstatehash: ShortStateHash) -> Result> { self.services .state_compressor From ecc9099127cc6779cd74723ae6169f7a22276ab7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 7 Feb 2025 23:18:02 +0000 Subject: [PATCH 1507/2091] add conf item to re-disable atomic flush Signed-off-by: Jason Volk --- conduwuit-example.toml | 7 +++++++ src/core/config/mod.rs | 7 +++++++ src/database/engine/db_opts.rs | 4 ++-- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index f9da856d..9b6f6ce0 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -925,6 +925,13 @@ # #rocksdb_checksums = true +# Enables the "atomic flush" mode in rocksdb. This option is not intended +# for users. It may be removed or ignored in future versions. Atomic flush +# may be enabled by the paranoid to possibly improve database integrity at +# the cost of performance. +# +#rocksdb_atomic_flush = false + # Database repair mode (for RocksDB SST corruption). # # Use this option when the server reports corruption while running or diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 9514f7a0..e66532ee 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1089,6 +1089,13 @@ pub struct Config { #[serde(default = "true_fn")] pub rocksdb_checksums: bool, + /// Enables the "atomic flush" mode in rocksdb. This option is not intended + /// for users. It may be removed or ignored in future versions. Atomic flush + /// may be enabled by the paranoid to possibly improve database integrity at + /// the cost of performance. + #[serde(default)] + pub rocksdb_atomic_flush: bool, + /// Database repair mode (for RocksDB SST corruption). /// /// Use this option when the server reports corruption while running or diff --git a/src/database/engine/db_opts.rs b/src/database/engine/db_opts.rs index 01847257..6abeb4b0 100644 --- a/src/database/engine/db_opts.rs +++ b/src/database/engine/db_opts.rs @@ -29,9 +29,9 @@ pub(crate) fn db_options(config: &Config, env: &Env, row_cache: &Cache) -> Resul opts.set_max_file_opening_threads(0); // IO - opts.set_atomic_flush(true); opts.set_manual_wal_flush(true); - opts.set_enable_pipelined_write(false); + opts.set_atomic_flush(config.rocksdb_atomic_flush); + opts.set_enable_pipelined_write(!config.rocksdb_atomic_flush); if config.rocksdb_direct_io { opts.set_use_direct_reads(true); opts.set_use_direct_io_for_flush_and_compaction(true); From b872f8e593afaee437331edd429a2d801f069aab Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 8 Feb 2025 00:16:37 +0000 Subject: [PATCH 1508/2091] optimize with SmallString; consolidate related re-exports Signed-off-by: Jason Volk --- Cargo.lock | 16 ++++++++++++---- Cargo.toml | 4 ++++ src/api/client/room/create.rs | 8 ++++---- src/api/client/room/upgrade.rs | 12 ++++++------ src/api/client/state.rs | 2 +- src/api/client/sync/v3.rs | 2 +- src/api/client/sync/v4.rs | 17 +++++++---------- src/api/client/sync/v5.rs | 18 +++++++----------- src/core/Cargo.toml | 2 ++ src/core/mod.rs | 5 ++++- src/core/pdu/builder.rs | 9 ++++++--- src/core/pdu/mod.rs | 8 +++++--- src/core/pdu/state_key.rs | 8 ++++++++ src/database/Cargo.toml | 2 -- src/database/de.rs | 5 +++-- src/database/keyval.rs | 3 +-- src/database/map/contains.rs | 2 +- src/database/map/insert.rs | 3 +-- src/database/map/qry.rs | 3 +-- src/database/map/remove.rs | 3 +-- src/database/pool.rs | 2 +- src/database/tests.rs | 6 ++++-- src/service/Cargo.toml | 2 -- src/service/migrations.rs | 4 +++- src/service/resolver/cache.rs | 2 +- src/service/resolver/fed.rs | 3 +-- src/service/resolver/mod.rs | 3 +-- .../rooms/event_handler/handle_outlier_pdu.rs | 10 ++++------ .../rooms/event_handler/resolve_state.rs | 1 + .../rooms/event_handler/state_at_incoming.rs | 1 + src/service/rooms/pdu_metadata/data.rs | 2 +- src/service/rooms/search/mod.rs | 2 +- src/service/rooms/short/mod.rs | 9 ++++----- src/service/rooms/state_accessor/room_state.rs | 4 ++-- src/service/rooms/state_accessor/state.rs | 12 ++++++------ src/service/rooms/state_accessor/user_can.rs | 2 +- src/service/rooms/state_compressor/mod.rs | 2 +- src/service/rooms/timeline/mod.rs | 8 ++++---- src/service/sending/mod.rs | 2 +- 39 files changed, 113 insertions(+), 96 deletions(-) create mode 100644 src/core/pdu/state_key.rs diff --git a/Cargo.lock b/Cargo.lock index caef5859..5981a2a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -822,6 +822,8 @@ dependencies = [ "serde_json", "serde_regex", "serde_yaml", + "smallstr", + "smallvec", "thiserror 2.0.11", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", @@ -839,7 +841,6 @@ dependencies = [ name = "conduwuit_database" version = "0.5.0" dependencies = [ - "arrayvec", "async-channel", "conduwuit_core", "const-str", @@ -850,7 +851,6 @@ dependencies = [ "rust-rocksdb-uwu", "serde", "serde_json", - "smallvec", "tokio", "tracing", ] @@ -902,7 +902,6 @@ dependencies = [ name = "conduwuit_service" version = "0.5.0" dependencies = [ - "arrayvec", "async-trait", "base64 0.22.1", "blurhash", @@ -929,7 +928,6 @@ dependencies = [ "serde_json", "serde_yaml", "sha2", - "smallvec", "termimad", "tokio", "tracing", @@ -4275,6 +4273,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "smallstr" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b1aefdf380735ff8ded0b15f31aab05daf1f70216c01c02a12926badd1df9d" +dependencies = [ + "serde", + "smallvec", +] + [[package]] name = "smallvec" version = "1.13.2" diff --git a/Cargo.toml b/Cargo.toml index 38654be3..b93877bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,10 @@ features = [ "write", ] +[workspace.dependencies.smallstr] +version = "0.3" +features = ["ffi", "std", "union"] + [workspace.dependencies.const-str] version = "0.5.7" diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index a401b63d..e362b3b3 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ - debug_info, debug_warn, err, error, info, pdu::PduBuilder, warn, Err, Error, Result, + debug_info, debug_warn, err, error, info, pdu::PduBuilder, warn, Err, Error, Result, StateKey, }; use futures::FutureExt; use ruma::{ @@ -198,7 +198,7 @@ pub(crate) async fn create_room_route( event_type: TimelineEventType::RoomCreate, content: to_raw_value(&create_content) .expect("create event content serialization"), - state_key: Some(String::new()), + state_key: Some(StateKey::new()), ..Default::default() }, sender_user, @@ -267,7 +267,7 @@ pub(crate) async fn create_room_route( event_type: TimelineEventType::RoomPowerLevels, content: to_raw_value(&power_levels_content) .expect("serialized power_levels event content"), - state_key: Some(String::new()), + state_key: Some(StateKey::new()), ..Default::default() }, sender_user, @@ -371,7 +371,7 @@ pub(crate) async fn create_room_route( } // Implicit state key defaults to "" - pdu_builder.state_key.get_or_insert_with(String::new); + pdu_builder.state_key.get_or_insert_with(StateKey::new); // Silently skip encryption events if they are not allowed if pdu_builder.event_type == TimelineEventType::RoomEncryption diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index 2f9706f4..a624f95f 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -1,7 +1,7 @@ use std::cmp::max; use axum::extract::State; -use conduwuit::{err, info, pdu::PduBuilder, Error, Result}; +use conduwuit::{err, info, pdu::PduBuilder, Error, Result, StateKey}; use futures::StreamExt; use ruma::{ api::client::{error::ErrorKind, room::upgrade_room}, @@ -77,7 +77,7 @@ pub(crate) async fn upgrade_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state(String::new(), &RoomTombstoneEventContent { + PduBuilder::state(StateKey::new(), &RoomTombstoneEventContent { body: "This room has been replaced".to_owned(), replacement_room: replacement_room.clone(), }), @@ -159,7 +159,7 @@ pub(crate) async fn upgrade_room_route( content: to_raw_value(&create_event_content) .expect("event is valid, we just created it"), unsigned: None, - state_key: Some(String::new()), + state_key: Some(StateKey::new()), redacts: None, timestamp: None, }, @@ -188,7 +188,7 @@ pub(crate) async fn upgrade_room_route( }) .expect("event is valid, we just created it"), unsigned: None, - state_key: Some(sender_user.to_string()), + state_key: Some(sender_user.as_str().into()), redacts: None, timestamp: None, }, @@ -217,7 +217,7 @@ pub(crate) async fn upgrade_room_route( PduBuilder { event_type: event_type.to_string().into(), content: event_content, - state_key: Some(String::new()), + state_key: Some(StateKey::new()), ..Default::default() }, sender_user, @@ -272,7 +272,7 @@ pub(crate) async fn upgrade_room_route( .rooms .timeline .build_and_append_pdu( - PduBuilder::state(String::new(), &RoomPowerLevelsEventContent { + PduBuilder::state(StateKey::new(), &RoomPowerLevelsEventContent { events_default: new_level, invite: new_level, ..power_levels_event_content diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 8555f88b..f73ffa46 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -172,7 +172,7 @@ async fn send_state_event_for_key_helper( PduBuilder { event_type: event_type.to_string().into(), content: serde_json::from_str(json.json().get())?, - state_key: Some(String::from(state_key)), + state_key: Some(state_key.into()), timestamp, ..Default::default() }, diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 1d1a91ba..f9dcd5ec 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -441,7 +441,7 @@ async fn handle_left_room( kind: RoomMember, content: serde_json::from_str(r#"{"membership":"leave"}"#) .expect("this is valid JSON"), - state_key: Some(sender_user.to_string()), + state_key: Some(sender_user.as_str().into()), unsigned: None, // The following keys are dropped on conversion room_id: room_id.clone(), diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 66793ba1..4e474ef3 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -29,7 +29,7 @@ use ruma::{ TimelineEventType::*, }, serde::Raw, - uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UInt, + uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, }; use service::rooms::read_receipt::pack_receipts; @@ -258,12 +258,9 @@ pub(crate) async fn sync_events_v4_route( continue; }; if pdu.kind == RoomMember { - if let Some(state_key) = &pdu.state_key { - let user_id = - OwnedUserId::parse(state_key.clone()).map_err(|_| { - Error::bad_database("Invalid UserId in member PDU.") - })?; - + if let Some(Ok(user_id)) = + pdu.state_key.as_deref().map(UserId::parse) + { if user_id == *sender_user { continue; } @@ -275,18 +272,18 @@ pub(crate) async fn sync_events_v4_route( if !share_encrypted_room( &services, sender_user, - &user_id, + user_id, Some(room_id), ) .await { - device_list_changes.insert(user_id); + device_list_changes.insert(user_id.to_owned()); } }, | MembershipState::Leave => { // Write down users that have left encrypted rooms we // are in - left_encrypted_users.insert(user_id); + left_encrypted_users.insert(user_id.to_owned()); }, | _ => {}, } diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index e7b5fe74..f8ee1047 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -25,7 +25,7 @@ use ruma::{ }, serde::Raw, state_res::TypeStateKey, - uint, DeviceId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UInt, UserId, + uint, DeviceId, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, }; use service::{rooms::read_receipt::pack_receipts, PduCount}; @@ -765,13 +765,9 @@ async fn collect_e2ee<'a>( continue; }; if pdu.kind == TimelineEventType::RoomMember { - if let Some(state_key) = &pdu.state_key { - let user_id = - OwnedUserId::parse(state_key.clone()).map_err(|_| { - Error::bad_database("Invalid UserId in member PDU.") - })?; - - if user_id == *sender_user { + if let Some(Ok(user_id)) = pdu.state_key.as_deref().map(UserId::parse) + { + if user_id == sender_user { continue; } @@ -782,18 +778,18 @@ async fn collect_e2ee<'a>( if !share_encrypted_room( &services, sender_user, - &user_id, + user_id, Some(room_id), ) .await { - device_list_changes.insert(user_id); + device_list_changes.insert(user_id.to_owned()); } }, | MembershipState::Leave => { // Write down users that have left encrypted rooms we // are in - left_encrypted_users.insert(user_id); + left_encrypted_users.insert(user_id.to_owned()); }, | _ => {}, } diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index ef2df4ff..d4b0c83b 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -92,6 +92,8 @@ serde_json.workspace = true serde_regex.workspace = true serde_yaml.workspace = true serde.workspace = true +smallvec.workspace = true +smallstr.workspace = true thiserror.workspace = true tikv-jemallocator.optional = true tikv-jemallocator.workspace = true diff --git a/src/core/mod.rs b/src/core/mod.rs index 1416ed9e..ee128628 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -10,14 +10,17 @@ pub mod pdu; pub mod server; pub mod utils; +pub use ::arrayvec; pub use ::http; pub use ::ruma; +pub use ::smallstr; +pub use ::smallvec; pub use ::toml; pub use ::tracing; pub use config::Config; pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; -pub use pdu::{Event, PduBuilder, PduCount, PduEvent, PduId, RawPduId}; +pub use pdu::{Event, PduBuilder, PduCount, PduEvent, PduId, RawPduId, StateKey}; pub use server::Server; pub use utils::{ctor, dtor, implement, result, result::Result}; diff --git a/src/core/pdu/builder.rs b/src/core/pdu/builder.rs index b25d4e9e..0efee128 100644 --- a/src/core/pdu/builder.rs +++ b/src/core/pdu/builder.rs @@ -7,6 +7,8 @@ use ruma::{ use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use super::StateKey; + /// Build the start of a PDU in order to add it to the Database. #[derive(Debug, Deserialize)] pub struct Builder { @@ -17,7 +19,7 @@ pub struct Builder { pub unsigned: Option, - pub state_key: Option, + pub state_key: Option, pub redacts: Option, @@ -29,15 +31,16 @@ pub struct Builder { type Unsigned = BTreeMap; impl Builder { - pub fn state(state_key: String, content: &T) -> Self + pub fn state(state_key: S, content: &T) -> Self where T: EventContent, + S: Into, { Self { event_type: content.event_type().into(), content: to_raw_value(content) .expect("Builder failed to serialize state event content to RawValue"), - state_key: Some(state_key), + state_key: Some(state_key.into()), ..Self::default() } } diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 1a8f6a70..9cb42239 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -8,6 +8,7 @@ mod id; mod raw_id; mod redact; mod relation; +mod state_key; mod strip; #[cfg(test)] mod tests; @@ -17,7 +18,7 @@ use std::cmp::Ordering; use ruma::{ events::TimelineEventType, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, - OwnedRoomId, OwnedUserId, UInt, + OwnedRoomId, OwnedServerName, OwnedUserId, UInt, }; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue as RawJsonValue; @@ -29,6 +30,7 @@ pub use self::{ event_id::*, id::*, raw_id::*, + state_key::{ShortStateKey, StateKey}, Count as PduCount, Id as PduId, Pdu as PduEvent, RawId as RawPduId, }; use crate::Result; @@ -40,13 +42,13 @@ pub struct Pdu { pub room_id: OwnedRoomId, pub sender: OwnedUserId, #[serde(skip_serializing_if = "Option::is_none")] - pub origin: Option, + pub origin: Option, pub origin_server_ts: UInt, #[serde(rename = "type")] pub kind: TimelineEventType, pub content: Box, #[serde(skip_serializing_if = "Option::is_none")] - pub state_key: Option, + pub state_key: Option, pub prev_events: Vec, pub depth: UInt, pub auth_events: Vec, diff --git a/src/core/pdu/state_key.rs b/src/core/pdu/state_key.rs new file mode 100644 index 00000000..4af4fcf7 --- /dev/null +++ b/src/core/pdu/state_key.rs @@ -0,0 +1,8 @@ +use smallstr::SmallString; + +use super::ShortId; + +pub type StateKey = SmallString<[u8; INLINE_SIZE]>; +pub type ShortStateKey = ShortId; + +const INLINE_SIZE: usize = 48; diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 557c9a3e..067c6f5f 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -34,7 +34,6 @@ zstd_compression = [ ] [dependencies] -arrayvec.workspace = true async-channel.workspace = true conduwuit-core.workspace = true const-str.workspace = true @@ -45,7 +44,6 @@ minicbor-serde.workspace = true rust-rocksdb.workspace = true serde.workspace = true serde_json.workspace = true -smallvec.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/src/database/de.rs b/src/database/de.rs index 8e914fcc..441bb4ec 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -1,5 +1,6 @@ -use arrayvec::ArrayVec; -use conduwuit::{checked, debug::DebugInspect, err, utils::string, Error, Result}; +use conduwuit::{ + arrayvec::ArrayVec, checked, debug::DebugInspect, err, utils::string, Error, Result, +}; use serde::{ de, de::{DeserializeSeed, Visitor}, diff --git a/src/database/keyval.rs b/src/database/keyval.rs index 056e53d1..f572d15f 100644 --- a/src/database/keyval.rs +++ b/src/database/keyval.rs @@ -1,6 +1,5 @@ -use conduwuit::Result; +use conduwuit::{smallvec::SmallVec, Result}; use serde::{Deserialize, Serialize}; -use smallvec::SmallVec; use crate::{de, ser}; diff --git a/src/database/map/contains.rs b/src/database/map/contains.rs index 424f8970..7a09b358 100644 --- a/src/database/map/contains.rs +++ b/src/database/map/contains.rs @@ -1,7 +1,7 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write, sync::Arc}; -use arrayvec::ArrayVec; use conduwuit::{ + arrayvec::ArrayVec, err, implement, utils::{future::TryExtExt, result::FlatOk}, Result, diff --git a/src/database/map/insert.rs b/src/database/map/insert.rs index 68c305af..6f010097 100644 --- a/src/database/map/insert.rs +++ b/src/database/map/insert.rs @@ -5,8 +5,7 @@ use std::{convert::AsRef, fmt::Debug, io::Write}; -use arrayvec::ArrayVec; -use conduwuit::implement; +use conduwuit::{arrayvec::ArrayVec, implement}; use rocksdb::WriteBatchWithTransaction; use serde::Serialize; diff --git a/src/database/map/qry.rs b/src/database/map/qry.rs index 401eba43..178f4a61 100644 --- a/src/database/map/qry.rs +++ b/src/database/map/qry.rs @@ -1,7 +1,6 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; -use arrayvec::ArrayVec; -use conduwuit::{implement, Result}; +use conduwuit::{arrayvec::ArrayVec, implement, Result}; use futures::Future; use serde::Serialize; diff --git a/src/database/map/remove.rs b/src/database/map/remove.rs index ec37bbfe..a7ae9133 100644 --- a/src/database/map/remove.rs +++ b/src/database/map/remove.rs @@ -1,7 +1,6 @@ use std::{convert::AsRef, fmt::Debug, io::Write}; -use arrayvec::ArrayVec; -use conduwuit::implement; +use conduwuit::{arrayvec::ArrayVec, implement}; use serde::Serialize; use crate::{keyval::KeyBuf, ser, util::or_else}; diff --git a/src/database/pool.rs b/src/database/pool.rs index c753855a..7636ff5e 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -14,6 +14,7 @@ use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ debug, debug_warn, err, error, implement, result::DebugInspect, + smallvec::SmallVec, trace, utils::sys::compute::{get_affinity, nth_core_available, set_affinity}, Error, Result, Server, @@ -21,7 +22,6 @@ use conduwuit::{ use futures::{channel::oneshot, TryFutureExt}; use oneshot::Sender as ResultSender; use rocksdb::Direction; -use smallvec::SmallVec; use self::configure::configure; use crate::{keyval::KeyBuf, stream, Handle, Map}; diff --git a/src/database/tests.rs b/src/database/tests.rs index e6c85983..594170e8 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -2,8 +2,10 @@ use std::fmt::Debug; -use arrayvec::ArrayVec; -use conduwuit::ruma::{serde::Raw, EventId, RoomId, UserId}; +use conduwuit::{ + arrayvec::ArrayVec, + ruma::{serde::Raw, EventId, RoomId, UserId}, +}; use serde::Serialize; use crate::{ diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 30183179..caeea318 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -47,7 +47,6 @@ zstd_compression = [ blurhashing = ["dep:image","dep:blurhash"] [dependencies] -arrayvec.workspace = true async-trait.workspace = true base64.workspace = true bytes.workspace = true @@ -75,7 +74,6 @@ serde_json.workspace = true serde.workspace = true serde_yaml.workspace = true sha2.workspace = true -smallvec.workspace = true termimad.workspace = true termimad.optional = true tokio.workspace = true diff --git a/src/service/migrations.rs b/src/service/migrations.rs index 9c3ea293..69b1be4e 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -507,8 +507,10 @@ async fn fix_referencedevents_missing_sep(services: &Services) -> Result { } async fn fix_readreceiptid_readreceipt_duplicates(services: &Services) -> Result { + use conduwuit::arrayvec::ArrayString; use ruma::identifiers_validation::MAX_BYTES; - type ArrayId = arrayvec::ArrayString; + + type ArrayId = ArrayString; type Key<'a> = (&'a RoomId, u64, &'a UserId); warn!("Fixing undeleted entries in readreceiptid_readreceipt..."); diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index 22a92865..7b4f104d 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -1,7 +1,7 @@ use std::{net::IpAddr, sync::Arc, time::SystemTime}; -use arrayvec::ArrayVec; use conduwuit::{ + arrayvec::ArrayVec, at, err, implement, utils::{math::Expected, rand, stream::TryIgnore}, Result, diff --git a/src/service/resolver/fed.rs b/src/service/resolver/fed.rs index bfe100e7..e5bee9ac 100644 --- a/src/service/resolver/fed.rs +++ b/src/service/resolver/fed.rs @@ -4,8 +4,7 @@ use std::{ net::{IpAddr, SocketAddr}, }; -use arrayvec::ArrayString; -use conduwuit::utils::math::Expected; +use conduwuit::{arrayvec::ArrayString, utils::math::Expected}; use serde::{Deserialize, Serialize}; #[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 090e562d..6be9d42d 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -6,8 +6,7 @@ mod tests; use std::sync::Arc; -use arrayvec::ArrayString; -use conduwuit::{utils::MutexMap, Result, Server}; +use conduwuit::{arrayvec::ArrayString, utils::MutexMap, Result, Server}; use self::{cache::Cache, dns::Resolver}; use crate::{client, Dep}; diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index a35aabe0..b7c38313 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -6,10 +6,8 @@ use std::{ use conduwuit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; use futures::{future::ready, TryFutureExt}; use ruma::{ - api::client::error::ErrorKind, - events::StateEventType, - state_res::{self, EventTypeExt}, - CanonicalJsonObject, CanonicalJsonValue, EventId, RoomId, ServerName, + api::client::error::ErrorKind, events::StateEventType, state_res, CanonicalJsonObject, + CanonicalJsonValue, EventId, RoomId, ServerName, }; use super::{check_room_id, get_room_version_id, to_room_version}; @@ -123,7 +121,7 @@ pub(super) async fn handle_outlier_pdu<'a>( // The original create event must be in the auth events if !matches!( auth_events - .get(&(StateEventType::RoomCreate, String::new())) + .get(&(StateEventType::RoomCreate, String::new().into())) .map(AsRef::as_ref), Some(_) | None ) { @@ -134,7 +132,7 @@ pub(super) async fn handle_outlier_pdu<'a>( } let state_fetch = |ty: &'static StateEventType, sk: &str| { - let key = ty.with_state_key(sk); + let key = (ty.to_owned(), sk.into()); ready(auth_events.get(&key)) }; diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 4d99b088..eb9ca01f 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -64,6 +64,7 @@ pub async fn resolve_state( .multi_get_statekey_from_short(shortstatekeys) .zip(event_ids) .ready_filter_map(|(ty_sk, id)| Some((ty_sk.ok()?, id))) + .map(|((ty, sk), id)| ((ty, sk.as_str().to_owned()), id)) .collect() }) .map(Ok::<_, Error>) diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 8ae6354c..7bf3b8f8 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -172,6 +172,7 @@ async fn state_at_incoming_fork( .short .get_statekey_from_short(*k) .map_ok(|(ty, sk)| ((ty, sk), id.clone())) + .map_ok(|((ty, sk), id)| ((ty, sk.as_str().to_owned()), id)) }) .ready_filter_map(Result::ok) .collect() diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 2e6ecbb5..26e11ded 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,7 +1,7 @@ use std::{mem::size_of, sync::Arc}; -use arrayvec::ArrayVec; use conduwuit::{ + arrayvec::ArrayVec, result::LogErr, utils::{ stream::{TryIgnore, WidebandExt}, diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 35cfd444..cc015237 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use arrayvec::ArrayVec; use conduwuit::{ + arrayvec::ArrayVec, implement, utils::{ set, diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index dd586d02..8728325a 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, fmt::Debug, mem::size_of_val, sync::Arc}; -pub use conduwuit::pdu::{ShortEventId, ShortId, ShortRoomId}; -use conduwuit::{err, implement, utils, utils::IterStream, Result}; +pub use conduwuit::pdu::{ShortEventId, ShortId, ShortRoomId, ShortStateKey}; +use conduwuit::{err, implement, utils, utils::IterStream, Result, StateKey}; use database::{Deserialized, Get, Map, Qry}; use futures::{Stream, StreamExt}; use ruma::{events::StateEventType, EventId, RoomId}; @@ -28,7 +28,6 @@ struct Services { } pub type ShortStateHash = ShortId; -pub type ShortStateKey = ShortId; impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -181,7 +180,7 @@ where pub async fn get_statekey_from_short( &self, shortstatekey: ShortStateKey, -) -> Result<(StateEventType, String)> { +) -> Result<(StateEventType, StateKey)> { const BUFSIZE: usize = size_of::(); self.db @@ -200,7 +199,7 @@ pub async fn get_statekey_from_short( pub fn multi_get_statekey_from_short<'a, S>( &'a self, shortstatekey: S, -) -> impl Stream> + Send + 'a +) -> impl Stream> + Send + 'a where S: Stream + Send + 'a, { diff --git a/src/service/rooms/state_accessor/room_state.rs b/src/service/rooms/state_accessor/room_state.rs index 98a82cea..e3ec55fe 100644 --- a/src/service/rooms/state_accessor/room_state.rs +++ b/src/service/rooms/state_accessor/room_state.rs @@ -1,6 +1,6 @@ use std::borrow::Borrow; -use conduwuit::{err, implement, PduEvent, Result}; +use conduwuit::{err, implement, PduEvent, Result, StateKey}; use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{events::StateEventType, EventId, RoomId}; use serde::Deserialize; @@ -27,7 +27,7 @@ where pub fn room_state_full<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream> + Send + 'a { +) -> impl Stream> + Send + 'a { self.services .state .get_room_shortstatehash(room_id) diff --git a/src/service/rooms/state_accessor/state.rs b/src/service/rooms/state_accessor/state.rs index 3cf168c1..da1500cb 100644 --- a/src/service/rooms/state_accessor/state.rs +++ b/src/service/rooms/state_accessor/state.rs @@ -6,7 +6,7 @@ use conduwuit::{ result::FlatOk, stream::{BroadbandExt, IterStream, ReadyExt, TryExpect}, }, - PduEvent, Result, + PduEvent, Result, StateKey, }; use database::Deserialized; use futures::{future::try_join, pin_mut, FutureExt, Stream, StreamExt, TryFutureExt}; @@ -192,7 +192,7 @@ pub fn state_keys_with_ids<'a, Id>( &'a self, shortstatehash: ShortStateHash, event_type: &'a StateEventType, -) -> impl Stream + Send + 'a +) -> impl Stream + Send + 'a where Id: for<'de> Deserialize<'de> + Send + Sized + ToOwned + 'a, ::Owned: Borrow, @@ -200,7 +200,7 @@ where let state_keys_with_short_ids = self .state_keys_with_shortids(shortstatehash, event_type) .unzip() - .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) + .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) .shared(); let state_keys = state_keys_with_short_ids @@ -230,7 +230,7 @@ pub fn state_keys_with_shortids<'a>( &'a self, shortstatehash: ShortStateHash, event_type: &'a StateEventType, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { let short_ids = self .state_full_shortids(shortstatehash) .expect_ok() @@ -267,7 +267,7 @@ pub fn state_keys<'a>( &'a self, shortstatehash: ShortStateHash, event_type: &'a StateEventType, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { let short_ids = self .state_full_shortids(shortstatehash) .expect_ok() @@ -314,7 +314,7 @@ pub fn state_added( pub fn state_full( &self, shortstatehash: ShortStateHash, -) -> impl Stream + Send + '_ { +) -> impl Stream + Send + '_ { self.state_full_pdus(shortstatehash) .ready_filter_map(|pdu| { Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu)) diff --git a/src/service/rooms/state_accessor/user_can.rs b/src/service/rooms/state_accessor/user_can.rs index 725a4fba..0332c227 100644 --- a/src/service/rooms/state_accessor/user_can.rs +++ b/src/service/rooms/state_accessor/user_can.rs @@ -175,7 +175,7 @@ pub async fn user_can_invite( .timeline .create_hash_and_sign_event( PduBuilder::state( - target_user.into(), + target_user.as_str(), &RoomMemberEventContent::new(MembershipState::Invite), ), sender, diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 3d68dff6..18731809 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -5,8 +5,8 @@ use std::{ sync::{Arc, Mutex}, }; -use arrayvec::ArrayVec; use conduwuit::{ + arrayvec::ArrayVec, at, checked, err, expected, utils, utils::{bytes, math::usize_from_f64, stream::IterStream}, Result, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index a913034d..a7edd4a4 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -38,7 +38,7 @@ use ruma::{ push::{Action, Ruleset, Tweak}, state_res::{self, Event, RoomVersion}, uint, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, - OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, + OwnedServerName, RoomId, RoomVersionId, ServerName, UserId, }; use serde::Deserialize; use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; @@ -387,10 +387,10 @@ impl Service { if pdu.kind == TimelineEventType::RoomMember { if let Some(state_key) = &pdu.state_key { - let target_user_id = OwnedUserId::parse(state_key)?; + let target_user_id = UserId::parse(state_key)?; - if self.services.users.is_active_local(&target_user_id).await { - push_target.insert(target_user_id); + if self.services.users.is_active_local(target_user_id).await { + push_target.insert(target_user_id.to_owned()); } } } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 86b219f7..b46ce7a8 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -13,6 +13,7 @@ use std::{ use async_trait::async_trait; use conduwuit::{ debug, debug_warn, err, error, + smallvec::SmallVec, utils::{available_parallelism, math::usize_from_u64_truncated, ReadyExt, TryReadyExt}, warn, Result, Server, }; @@ -21,7 +22,6 @@ use ruma::{ api::{appservice::Registration, OutgoingRequest}, RoomId, ServerName, UserId, }; -use smallvec::SmallVec; use tokio::{task, task::JoinSet}; use self::data::Data; From 0a9a9b3c92852cae269aaf2cb3894658b5e35a54 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 5 Feb 2025 12:22:22 +0000 Subject: [PATCH 1509/2091] larcen state-res from ruma --- Cargo.toml | 1 - src/api/client/membership.rs | 6 +- src/api/client/sync/v5.rs | 3 +- src/core/error/mod.rs | 2 +- src/core/mod.rs | 2 + src/core/pdu/event.rs | 2 +- src/core/state_res/LICENSE | 17 + src/core/state_res/error.rs | 23 + src/core/state_res/event_auth.rs | 1418 ++++++++++++++ src/core/state_res/mod.rs | 1644 +++++++++++++++++ src/core/state_res/outcomes.txt | 104 ++ src/core/state_res/power_levels.rs | 256 +++ src/core/state_res/room_version.rs | 149 ++ src/core/state_res/state_event.rs | 102 + src/core/state_res/state_res_bench.rs | 648 +++++++ src/core/state_res/test_utils.rs | 688 +++++++ src/service/rooms/event_handler/fetch_prev.rs | 11 +- .../rooms/event_handler/handle_outlier_pdu.rs | 6 +- src/service/rooms/event_handler/mod.rs | 6 +- .../rooms/event_handler/resolve_state.rs | 9 +- .../rooms/event_handler/state_at_incoming.rs | 4 +- .../event_handler/upgrade_outlier_pdu.rs | 10 +- src/service/rooms/state/mod.rs | 2 +- src/service/rooms/timeline/mod.rs | 2 +- 24 files changed, 5082 insertions(+), 33 deletions(-) create mode 100644 src/core/state_res/LICENSE create mode 100644 src/core/state_res/error.rs create mode 100644 src/core/state_res/event_auth.rs create mode 100644 src/core/state_res/mod.rs create mode 100644 src/core/state_res/outcomes.txt create mode 100644 src/core/state_res/power_levels.rs create mode 100644 src/core/state_res/room_version.rs create mode 100644 src/core/state_res/state_event.rs create mode 100644 src/core/state_res/state_res_bench.rs create mode 100644 src/core/state_res/test_utils.rs diff --git a/Cargo.toml b/Cargo.toml index b93877bd..d8f34544 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -355,7 +355,6 @@ features = [ "federation-api", "markdown", "push-gateway-api-c", - "state-res", "server-util", "unstable-exhaustive-types", "ring-compat", diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 449d44d5..1045b014 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -12,7 +12,7 @@ use conduwuit::{ at, debug, debug_info, debug_warn, err, info, pdu::{gen_event_id_canonical_json, PduBuilder}, result::FlatOk, - trace, + state_res, trace, utils::{self, shuffle, IterStream, ReadyExt}, warn, Err, PduEvent, Result, }; @@ -40,8 +40,8 @@ use ruma::{ }, StateEventType, }, - state_res, CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, - OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, + CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, + OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use service::{ appservice::RegistrationInfo, diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index f8ee1047..63731688 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -11,7 +11,7 @@ use conduwuit::{ math::{ruma_from_usize, usize_from_ruma}, BoolExt, IterStream, ReadyExt, TryFutureExtExt, }, - warn, Error, Result, + warn, Error, Result, TypeStateKey, }; use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{ @@ -24,7 +24,6 @@ use ruma::{ AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType, }, serde::Raw, - state_res::TypeStateKey, uint, DeviceId, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, }; use service::{rooms::read_receipt::pack_receipts, PduCount}; diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 88ac6d09..16613b7e 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -121,7 +121,7 @@ pub enum Error { #[error(transparent)] Signatures(#[from] ruma::signatures::Error), #[error(transparent)] - StateRes(#[from] ruma::state_res::Error), + StateRes(#[from] crate::state_res::Error), #[error("uiaa")] Uiaa(ruma::api::client::uiaa::UiaaInfo), diff --git a/src/core/mod.rs b/src/core/mod.rs index ee128628..cd56774a 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -8,6 +8,7 @@ pub mod metrics; pub mod mods; pub mod pdu; pub mod server; +pub mod state_res; pub mod utils; pub use ::arrayvec; @@ -22,6 +23,7 @@ pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; pub use pdu::{Event, PduBuilder, PduCount, PduEvent, PduId, RawPduId, StateKey}; pub use server::Server; +pub use state_res::{EventTypeExt, RoomVersion, StateMap, TypeStateKey}; pub use utils::{ctor, dtor, implement, result, result::Result}; pub use crate as conduwuit_core; diff --git a/src/core/pdu/event.rs b/src/core/pdu/event.rs index 6a92afe8..d5c0561e 100644 --- a/src/core/pdu/event.rs +++ b/src/core/pdu/event.rs @@ -1,8 +1,8 @@ -pub use ruma::state_res::Event; use ruma::{events::TimelineEventType, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId}; use serde_json::value::RawValue as RawJsonValue; use super::Pdu; +pub use crate::state_res::Event; impl Event for Pdu { type Id = OwnedEventId; diff --git a/src/core/state_res/LICENSE b/src/core/state_res/LICENSE new file mode 100644 index 00000000..c103a044 --- /dev/null +++ b/src/core/state_res/LICENSE @@ -0,0 +1,17 @@ +//! Permission is hereby granted, free of charge, to any person obtaining a copy +//! of this software and associated documentation files (the "Software"), to +//! deal in the Software without restriction, including without limitation the +//! rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +//! sell copies of the Software, and to permit persons to whom the Software is +//! furnished to do so, subject to the following conditions: + +//! The above copyright notice and this permission notice shall be included in +//! all copies or substantial portions of the Software. + +//! THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +//! IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +//! FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +//! AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +//! LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +//! FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +//! IN THE SOFTWARE. diff --git a/src/core/state_res/error.rs b/src/core/state_res/error.rs new file mode 100644 index 00000000..7711d878 --- /dev/null +++ b/src/core/state_res/error.rs @@ -0,0 +1,23 @@ +use serde_json::Error as JsonError; +use thiserror::Error; + +/// Represents the various errors that arise when resolving state. +#[derive(Error, Debug)] +#[non_exhaustive] +pub enum Error { + /// A deserialization error. + #[error(transparent)] + SerdeJson(#[from] JsonError), + + /// The given option or version is unsupported. + #[error("Unsupported room version: {0}")] + Unsupported(String), + + /// The given event was not found. + #[error("Not found error: {0}")] + NotFound(String), + + /// Invalid fields in the given PDU. + #[error("Invalid PDU: {0}")] + InvalidPdu(String), +} diff --git a/src/core/state_res/event_auth.rs b/src/core/state_res/event_auth.rs new file mode 100644 index 00000000..72a0216c --- /dev/null +++ b/src/core/state_res/event_auth.rs @@ -0,0 +1,1418 @@ +use std::{borrow::Borrow, collections::BTreeSet}; + +use futures::{ + future::{join3, OptionFuture}, + Future, +}; +use ruma::{ + events::room::{ + create::RoomCreateEventContent, + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, ThirdPartyInvite}, + power_levels::RoomPowerLevelsEventContent, + third_party_invite::RoomThirdPartyInviteEventContent, + }, + int, + serde::{Base64, Raw}, + Int, OwnedUserId, RoomVersionId, UserId, +}; +use serde::{ + de::{Error as _, IgnoredAny}, + Deserialize, +}; +use serde_json::{from_str as from_json_str, value::RawValue as RawJsonValue}; +use tracing::{debug, error, instrument, trace, warn}; + +use super::{ + power_levels::{ + deserialize_power_levels, deserialize_power_levels_content_fields, + deserialize_power_levels_content_invite, deserialize_power_levels_content_redact, + }, + room_version::RoomVersion, + Error, Event, Result, StateEventType, TimelineEventType, +}; + +// FIXME: field extracting could be bundled for `content` +#[derive(Deserialize)] +struct GetMembership { + membership: MembershipState, +} + +#[derive(Deserialize)] +struct RoomMemberContentFields { + membership: Option>, + join_authorised_via_users_server: Option>, +} + +/// For the given event `kind` what are the relevant auth events that are needed +/// to authenticate this `content`. +/// +/// # Errors +/// +/// This function will return an error if the supplied `content` is not a JSON +/// object. +pub fn auth_types_for_event( + kind: &TimelineEventType, + sender: &UserId, + state_key: Option<&str>, + content: &RawJsonValue, +) -> serde_json::Result> { + if kind == &TimelineEventType::RoomCreate { + return Ok(vec![]); + } + + let mut auth_types = vec![ + (StateEventType::RoomPowerLevels, String::new()), + (StateEventType::RoomMember, sender.to_string()), + (StateEventType::RoomCreate, String::new()), + ]; + + if kind == &TimelineEventType::RoomMember { + #[derive(Deserialize)] + struct RoomMemberContentFields { + membership: Option>, + third_party_invite: Option>, + join_authorised_via_users_server: Option>, + } + + if let Some(state_key) = state_key { + let content: RoomMemberContentFields = from_json_str(content.get())?; + + if let Some(Ok(membership)) = content.membership.map(|m| m.deserialize()) { + if [MembershipState::Join, MembershipState::Invite, MembershipState::Knock] + .contains(&membership) + { + let key = (StateEventType::RoomJoinRules, String::new()); + if !auth_types.contains(&key) { + auth_types.push(key); + } + + if let Some(Ok(u)) = content + .join_authorised_via_users_server + .map(|m| m.deserialize()) + { + let key = (StateEventType::RoomMember, u.to_string()); + if !auth_types.contains(&key) { + auth_types.push(key); + } + } + } + + let key = (StateEventType::RoomMember, state_key.to_owned()); + if !auth_types.contains(&key) { + auth_types.push(key); + } + + if membership == MembershipState::Invite { + if let Some(Ok(t_id)) = content.third_party_invite.map(|t| t.deserialize()) { + let key = (StateEventType::RoomThirdPartyInvite, t_id.signed.token); + if !auth_types.contains(&key) { + auth_types.push(key); + } + } + } + } + } + } + + Ok(auth_types) +} + +/// Authenticate the incoming `event`. +/// +/// The steps of authentication are: +/// +/// * check that the event is being authenticated for the correct room +/// * then there are checks for specific event types +/// +/// The `fetch_state` closure should gather state from a state snapshot. We need +/// to know if the event passes auth against some state not a recursive +/// collection of auth_events fields. +#[instrument(level = "debug", skip_all, fields(event_id = incoming_event.event_id().borrow().as_str()))] +pub async fn auth_check( + room_version: &RoomVersion, + incoming_event: &Incoming, + current_third_party_invite: Option<&Incoming>, + fetch_state: F, +) -> Result +where + F: Fn(&'static StateEventType, &str) -> Fut, + Fut: Future> + Send, + Fetched: Event + Send, + Incoming: Event + Send, +{ + debug!( + "auth_check beginning for {} ({})", + incoming_event.event_id(), + incoming_event.event_type() + ); + + // [synapse] check that all the events are in the same room as `incoming_event` + + // [synapse] do_sig_check check the event has valid signatures for member events + + // TODO do_size_check is false when called by `iterative_auth_check` + // do_size_check is also mostly accomplished by ruma with the exception of + // checking event_type, state_key, and json are below a certain size (255 and + // 65_536 respectively) + + let sender = incoming_event.sender(); + + // Implementation of https://spec.matrix.org/latest/rooms/v1/#authorization-rules + // + // 1. If type is m.room.create: + if *incoming_event.event_type() == TimelineEventType::RoomCreate { + #[derive(Deserialize)] + struct RoomCreateContentFields { + room_version: Option>, + creator: Option>, + } + + debug!("start m.room.create check"); + + // If it has any previous events, reject + if incoming_event.prev_events().next().is_some() { + warn!("the room creation event had previous events"); + return Ok(false); + } + + // If the domain of the room_id does not match the domain of the sender, reject + let Some(room_id_server_name) = incoming_event.room_id().server_name() else { + warn!("room ID has no servername"); + return Ok(false); + }; + + if room_id_server_name != sender.server_name() { + warn!("servername of room ID does not match servername of sender"); + return Ok(false); + } + + // If content.room_version is present and is not a recognized version, reject + let content: RoomCreateContentFields = from_json_str(incoming_event.content().get())?; + if content + .room_version + .is_some_and(|v| v.deserialize().is_err()) + { + warn!("invalid room version found in m.room.create event"); + return Ok(false); + } + + if !room_version.use_room_create_sender { + // If content has no creator field, reject + if content.creator.is_none() { + warn!("no creator field found in m.room.create content"); + return Ok(false); + } + } + + debug!("m.room.create event was allowed"); + return Ok(true); + } + + /* + // TODO: In the past this code caused problems federating with synapse, maybe this has been + // resolved already. Needs testing. + // + // 2. Reject if auth_events + // a. auth_events cannot have duplicate keys since it's a BTree + // b. All entries are valid auth events according to spec + let expected_auth = auth_types_for_event( + incoming_event.kind, + sender, + incoming_event.state_key, + incoming_event.content().clone(), + ); + + dbg!(&expected_auth); + + for ev_key in auth_events.keys() { + // (b) + if !expected_auth.contains(ev_key) { + warn!("auth_events contained invalid auth event"); + return Ok(false); + } + } + */ + + let (room_create_event, power_levels_event, sender_member_event) = join3( + fetch_state(&StateEventType::RoomCreate, ""), + fetch_state(&StateEventType::RoomPowerLevels, ""), + fetch_state(&StateEventType::RoomMember, sender.as_str()), + ) + .await; + + let room_create_event = match room_create_event { + | None => { + warn!("no m.room.create event in auth chain"); + return Ok(false); + }, + | Some(e) => e, + }; + + // 3. If event does not have m.room.create in auth_events reject + if !incoming_event + .auth_events() + .any(|id| id.borrow() == room_create_event.event_id().borrow()) + { + warn!("no m.room.create event in auth events"); + return Ok(false); + } + + // If the create event content has the field m.federate set to false and the + // sender domain of the event does not match the sender domain of the create + // event, reject. + #[derive(Deserialize)] + struct RoomCreateContentFederate { + #[serde(rename = "m.federate", default = "ruma::serde::default_true")] + federate: bool, + } + let room_create_content: RoomCreateContentFederate = + from_json_str(room_create_event.content().get())?; + if !room_create_content.federate + && room_create_event.sender().server_name() != incoming_event.sender().server_name() + { + warn!( + "room is not federated and event's sender domain does not match create event's \ + sender domain" + ); + return Ok(false); + } + + // Only in some room versions 6 and below + if room_version.special_case_aliases_auth { + // 4. If type is m.room.aliases + if *incoming_event.event_type() == TimelineEventType::RoomAliases { + debug!("starting m.room.aliases check"); + + // If sender's domain doesn't matches state_key, reject + if incoming_event.state_key() != Some(sender.server_name().as_str()) { + warn!("state_key does not match sender"); + return Ok(false); + } + + debug!("m.room.aliases event was allowed"); + return Ok(true); + } + } + + // If type is m.room.member + if *incoming_event.event_type() == TimelineEventType::RoomMember { + debug!("starting m.room.member check"); + let state_key = match incoming_event.state_key() { + | None => { + warn!("no statekey in member event"); + return Ok(false); + }, + | Some(s) => s, + }; + + let content: RoomMemberContentFields = from_json_str(incoming_event.content().get())?; + if content + .membership + .as_ref() + .and_then(|m| m.deserialize().ok()) + .is_none() + { + warn!("no valid membership field found for m.room.member event content"); + return Ok(false); + } + + let target_user = + <&UserId>::try_from(state_key).map_err(|e| Error::InvalidPdu(format!("{e}")))?; + + let user_for_join_auth = content + .join_authorised_via_users_server + .as_ref() + .and_then(|u| u.deserialize().ok()); + + let user_for_join_auth_event: OptionFuture<_> = user_for_join_auth + .as_ref() + .map(|auth_user| fetch_state(&StateEventType::RoomMember, auth_user.as_str())) + .into(); + + let target_user_member_event = + fetch_state(&StateEventType::RoomMember, target_user.as_str()); + + let join_rules_event = fetch_state(&StateEventType::RoomJoinRules, ""); + + let (join_rules_event, target_user_member_event, user_for_join_auth_event) = + join3(join_rules_event, target_user_member_event, user_for_join_auth_event).await; + + let user_for_join_auth_membership = user_for_join_auth_event + .and_then(|mem| from_json_str::(mem?.content().get()).ok()) + .map_or(MembershipState::Leave, |mem| mem.membership); + + if !valid_membership_change( + room_version, + target_user, + target_user_member_event.as_ref(), + sender, + sender_member_event.as_ref(), + incoming_event, + current_third_party_invite, + power_levels_event.as_ref(), + join_rules_event.as_ref(), + user_for_join_auth.as_deref(), + &user_for_join_auth_membership, + room_create_event, + )? { + return Ok(false); + } + + debug!("m.room.member event was allowed"); + return Ok(true); + } + + // If the sender's current membership state is not join, reject + let sender_member_event = match sender_member_event { + | Some(mem) => mem, + | None => { + warn!("sender not found in room"); + return Ok(false); + }, + }; + + let sender_membership_event_content: RoomMemberContentFields = + from_json_str(sender_member_event.content().get())?; + let membership_state = sender_membership_event_content + .membership + .expect("we should test before that this field exists") + .deserialize()?; + + if !matches!(membership_state, MembershipState::Join) { + warn!("sender's membership is not join"); + return Ok(false); + } + + // If type is m.room.third_party_invite + let sender_power_level = if let Some(pl) = &power_levels_event { + let content = deserialize_power_levels_content_fields(pl.content().get(), room_version)?; + if let Some(level) = content.get_user_power(sender) { + *level + } else { + content.users_default + } + } else { + // If no power level event found the creator gets 100 everyone else gets 0 + let is_creator = if room_version.use_room_create_sender { + room_create_event.sender() == sender + } else { + #[allow(deprecated)] + from_json_str::(room_create_event.content().get()) + .is_ok_and(|create| create.creator.unwrap() == *sender) + }; + + if is_creator { + int!(100) + } else { + int!(0) + } + }; + + // Allow if and only if sender's current power level is greater than + // or equal to the invite level + if *incoming_event.event_type() == TimelineEventType::RoomThirdPartyInvite { + let invite_level = match &power_levels_event { + | Some(power_levels) => + deserialize_power_levels_content_invite( + power_levels.content().get(), + room_version, + )? + .invite, + | None => int!(0), + }; + + if sender_power_level < invite_level { + warn!("sender's cannot send invites in this room"); + return Ok(false); + } + + debug!("m.room.third_party_invite event was allowed"); + return Ok(true); + } + + // If the event type's required power level is greater than the sender's power + // level, reject If the event has a state_key that starts with an @ and does + // not match the sender, reject. + if !can_send_event(incoming_event, power_levels_event.as_ref(), sender_power_level) { + warn!("user cannot send event"); + return Ok(false); + } + + // If type is m.room.power_levels + if *incoming_event.event_type() == TimelineEventType::RoomPowerLevels { + debug!("starting m.room.power_levels check"); + + if let Some(required_pwr_lvl) = check_power_levels( + room_version, + incoming_event, + power_levels_event.as_ref(), + sender_power_level, + ) { + if !required_pwr_lvl { + warn!("m.room.power_levels was not allowed"); + return Ok(false); + } + } else { + warn!("m.room.power_levels was not allowed"); + return Ok(false); + } + debug!("m.room.power_levels event allowed"); + } + + // Room version 3: Redaction events are always accepted (provided the event is + // allowed by `events` and `events_default` in the power levels). However, + // servers should not apply or send redaction's to clients until both the + // redaction event and original event have been seen, and are valid. Servers + // should only apply redaction's to events where the sender's domains match, or + // the sender of the redaction has the appropriate permissions per the + // power levels. + + if room_version.extra_redaction_checks + && *incoming_event.event_type() == TimelineEventType::RoomRedaction + { + let redact_level = match power_levels_event { + | Some(pl) => + deserialize_power_levels_content_redact(pl.content().get(), room_version)?.redact, + | None => int!(50), + }; + + if !check_redaction(room_version, incoming_event, sender_power_level, redact_level)? { + return Ok(false); + } + } + + debug!("allowing event passed all checks"); + Ok(true) +} + +// TODO deserializing the member, power, join_rules event contents is done in +// conduit just before this is called. Could they be passed in? +/// Does the user who sent this member event have required power levels to do +/// so. +/// +/// * `user` - Information about the membership event and user making the +/// request. +/// * `auth_events` - The set of auth events that relate to a membership event. +/// +/// This is generated by calling `auth_types_for_event` with the membership +/// event and the current State. +#[allow(clippy::too_many_arguments)] +fn valid_membership_change( + room_version: &RoomVersion, + target_user: &UserId, + target_user_membership_event: Option, + sender: &UserId, + sender_membership_event: Option, + current_event: impl Event, + current_third_party_invite: Option, + power_levels_event: Option, + join_rules_event: Option, + user_for_join_auth: Option<&UserId>, + user_for_join_auth_membership: &MembershipState, + create_room: impl Event, +) -> Result { + #[derive(Deserialize)] + struct GetThirdPartyInvite { + third_party_invite: Option>, + } + let content = current_event.content(); + + let target_membership = from_json_str::(content.get())?.membership; + let third_party_invite = + from_json_str::(content.get())?.third_party_invite; + + let sender_membership = match &sender_membership_event { + | Some(pdu) => from_json_str::(pdu.content().get())?.membership, + | None => MembershipState::Leave, + }; + let sender_is_joined = sender_membership == MembershipState::Join; + + let target_user_current_membership = match &target_user_membership_event { + | Some(pdu) => from_json_str::(pdu.content().get())?.membership, + | None => MembershipState::Leave, + }; + + let power_levels: RoomPowerLevelsEventContent = match &power_levels_event { + | Some(ev) => from_json_str(ev.content().get())?, + | None => RoomPowerLevelsEventContent::default(), + }; + + let sender_power = power_levels + .users + .get(sender) + .or_else(|| sender_is_joined.then_some(&power_levels.users_default)); + + let target_power = power_levels.users.get(target_user).or_else(|| { + (target_membership == MembershipState::Join).then_some(&power_levels.users_default) + }); + + let mut join_rules = JoinRule::Invite; + if let Some(jr) = &join_rules_event { + join_rules = from_json_str::(jr.content().get())?.join_rule; + } + + let power_levels_event_id = power_levels_event.as_ref().map(Event::event_id); + let sender_membership_event_id = sender_membership_event.as_ref().map(Event::event_id); + let target_user_membership_event_id = + target_user_membership_event.as_ref().map(Event::event_id); + + let user_for_join_auth_is_valid = if let Some(user_for_join_auth) = user_for_join_auth { + // Is the authorised user allowed to invite users into this room + let (auth_user_pl, invite_level) = if let Some(pl) = &power_levels_event { + // TODO Refactor all powerlevel parsing + let invite = + deserialize_power_levels_content_invite(pl.content().get(), room_version)?.invite; + + let content = + deserialize_power_levels_content_fields(pl.content().get(), room_version)?; + let user_pl = if let Some(level) = content.get_user_power(user_for_join_auth) { + *level + } else { + content.users_default + }; + + (user_pl, invite) + } else { + (int!(0), int!(0)) + }; + (user_for_join_auth_membership == &MembershipState::Join) + && (auth_user_pl >= invite_level) + } else { + // No auth user was given + false + }; + + Ok(match target_membership { + | MembershipState::Join => { + // 1. If the only previous event is an m.room.create and the state_key is the + // creator, + // allow + let mut prev_events = current_event.prev_events(); + + let prev_event_is_create_event = prev_events + .next() + .is_some_and(|event_id| event_id.borrow() == create_room.event_id().borrow()); + let no_more_prev_events = prev_events.next().is_none(); + + if prev_event_is_create_event && no_more_prev_events { + let is_creator = if room_version.use_room_create_sender { + let creator = create_room.sender(); + + creator == sender && creator == target_user + } else { + #[allow(deprecated)] + let creator = from_json_str::(create_room.content().get())? + .creator + .ok_or_else(|| serde_json::Error::missing_field("creator"))?; + + creator == sender && creator == target_user + }; + + if is_creator { + return Ok(true); + } + } + + if sender != target_user { + // If the sender does not match state_key, reject. + warn!("Can't make other user join"); + false + } else if target_user_current_membership == MembershipState::Ban { + // If the sender is banned, reject. + warn!(?target_user_membership_event_id, "Banned user can't join"); + false + } else if (join_rules == JoinRule::Invite + || room_version.allow_knocking && join_rules == JoinRule::Knock) + // If the join_rule is invite then allow if membership state is invite or join + && (target_user_current_membership == MembershipState::Join + || target_user_current_membership == MembershipState::Invite) + { + true + } else if room_version.restricted_join_rules + && matches!(join_rules, JoinRule::Restricted(_)) + || room_version.knock_restricted_join_rule + && matches!(join_rules, JoinRule::KnockRestricted(_)) + { + // If the join_rule is restricted or knock_restricted + if matches!( + target_user_current_membership, + MembershipState::Invite | MembershipState::Join + ) { + // If membership state is join or invite, allow. + true + } else { + // If the join_authorised_via_users_server key in content is not a user with + // sufficient permission to invite other users, reject. + // Otherwise, allow. + user_for_join_auth_is_valid + } + } else { + // If the join_rule is public, allow. + // Otherwise, reject. + join_rules == JoinRule::Public + } + }, + | MembershipState::Invite => { + // If content has third_party_invite key + if let Some(tp_id) = third_party_invite.and_then(|i| i.deserialize().ok()) { + if target_user_current_membership == MembershipState::Ban { + warn!(?target_user_membership_event_id, "Can't invite banned user"); + false + } else { + let allow = verify_third_party_invite( + Some(target_user), + sender, + &tp_id, + current_third_party_invite, + ); + if !allow { + warn!("Third party invite invalid"); + } + allow + } + } else if !sender_is_joined + || target_user_current_membership == MembershipState::Join + || target_user_current_membership == MembershipState::Ban + { + warn!( + ?target_user_membership_event_id, + ?sender_membership_event_id, + "Can't invite user if sender not joined or the user is currently joined or \ + banned", + ); + false + } else { + let allow = sender_power + .filter(|&p| p >= &power_levels.invite) + .is_some(); + if !allow { + warn!( + ?target_user_membership_event_id, + ?power_levels_event_id, + "User does not have enough power to invite", + ); + } + allow + } + }, + | MembershipState::Leave => + if sender == target_user { + let allow = target_user_current_membership == MembershipState::Join + || target_user_current_membership == MembershipState::Invite + || target_user_current_membership == MembershipState::Knock; + if !allow { + warn!( + ?target_user_membership_event_id, + ?target_user_current_membership, + "Can't leave if sender is not already invited, knocked, or joined" + ); + } + allow + } else if !sender_is_joined + || target_user_current_membership == MembershipState::Ban + && sender_power.filter(|&p| p < &power_levels.ban).is_some() + { + warn!( + ?target_user_membership_event_id, + ?sender_membership_event_id, + "Can't kick if sender not joined or user is already banned", + ); + false + } else { + let allow = sender_power.filter(|&p| p >= &power_levels.kick).is_some() + && target_power < sender_power; + if !allow { + warn!( + ?target_user_membership_event_id, + ?power_levels_event_id, + "User does not have enough power to kick", + ); + } + allow + }, + | MembershipState::Ban => + if !sender_is_joined { + warn!(?sender_membership_event_id, "Can't ban user if sender is not joined"); + false + } else { + let allow = sender_power.filter(|&p| p >= &power_levels.ban).is_some() + && target_power < sender_power; + if !allow { + warn!( + ?target_user_membership_event_id, + ?power_levels_event_id, + "User does not have enough power to ban", + ); + } + allow + }, + | MembershipState::Knock if room_version.allow_knocking => { + // 1. If the `join_rule` is anything other than `knock` or `knock_restricted`, + // reject. + if !matches!(join_rules, JoinRule::KnockRestricted(_) | JoinRule::Knock) { + warn!( + "Join rule is not set to knock or knock_restricted, knocking is not allowed" + ); + false + } else if matches!(join_rules, JoinRule::KnockRestricted(_)) + && !room_version.knock_restricted_join_rule + { + // 2. If the `join_rule` is `knock_restricted`, but the room does not support + // `knock_restricted`, reject. + warn!( + "Join rule is set to knock_restricted but room version does not support \ + knock_restricted, knocking is not allowed" + ); + false + } else if sender != target_user { + // 3. If `sender` does not match `state_key`, reject. + warn!( + ?sender, + ?target_user, + "Can't make another user knock, sender did not match target" + ); + false + } else if matches!( + sender_membership, + MembershipState::Ban | MembershipState::Invite | MembershipState::Join + ) { + // 4. If the `sender`'s current membership is not `ban`, `invite`, or `join`, + // allow. + // 5. Otherwise, reject. + warn!( + ?target_user_membership_event_id, + "Knocking with a membership state of ban, invite or join is invalid", + ); + false + } else { + true + } + }, + | _ => { + warn!("Unknown membership transition"); + false + }, + }) +} + +/// Is the user allowed to send a specific event based on the rooms power +/// levels. +/// +/// Does the event have the correct userId as its state_key if it's not the "" +/// state_key. +fn can_send_event(event: impl Event, ple: Option, user_level: Int) -> bool { + let event_type_power_level = get_send_level(event.event_type(), event.state_key(), ple); + + debug!( + required_level = i64::from(event_type_power_level), + user_level = i64::from(user_level), + state_key = ?event.state_key(), + "permissions factors", + ); + + if user_level < event_type_power_level { + return false; + } + + if event.state_key().is_some_and(|k| k.starts_with('@')) + && event.state_key() != Some(event.sender().as_str()) + { + return false; // permission required to post in this room + } + + true +} + +/// Confirm that the event sender has the required power levels. +fn check_power_levels( + room_version: &RoomVersion, + power_event: impl Event, + previous_power_event: Option, + user_level: Int, +) -> Option { + match power_event.state_key() { + | Some("") => {}, + | Some(key) => { + error!(state_key = key, "m.room.power_levels event has non-empty state key"); + return None; + }, + | None => { + error!("check_power_levels requires an m.room.power_levels *state* event argument"); + return None; + }, + } + + // - If any of the keys users_default, events_default, state_default, ban, + // redact, kick, or invite in content are present and not an integer, reject. + // - If either of the keys events or notifications in content are present and + // not a dictionary with values that are integers, reject. + // - If users key in content is not a dictionary with keys that are valid user + // IDs with values that are integers, reject. + let user_content: RoomPowerLevelsEventContent = + deserialize_power_levels(power_event.content().get(), room_version)?; + + // Validation of users is done in Ruma, synapse for loops validating user_ids + // and integers here + debug!("validation of power event finished"); + + let current_state = match previous_power_event { + | Some(current_state) => current_state, + // If there is no previous m.room.power_levels event in the room, allow + | None => return Some(true), + }; + + let current_content: RoomPowerLevelsEventContent = + deserialize_power_levels(current_state.content().get(), room_version)?; + + let mut user_levels_to_check = BTreeSet::new(); + let old_list = ¤t_content.users; + let user_list = &user_content.users; + for user in old_list.keys().chain(user_list.keys()) { + let user: &UserId = user; + user_levels_to_check.insert(user); + } + + trace!(set = ?user_levels_to_check, "user levels to check"); + + let mut event_levels_to_check = BTreeSet::new(); + let old_list = ¤t_content.events; + let new_list = &user_content.events; + for ev_id in old_list.keys().chain(new_list.keys()) { + event_levels_to_check.insert(ev_id); + } + + trace!(set = ?event_levels_to_check, "event levels to check"); + + let old_state = ¤t_content; + let new_state = &user_content; + + // synapse does not have to split up these checks since we can't combine UserIds + // and EventTypes we do 2 loops + + // UserId loop + for user in user_levels_to_check { + let old_level = old_state.users.get(user); + let new_level = new_state.users.get(user); + if old_level.is_some() && new_level.is_some() && old_level == new_level { + continue; + } + + // If the current value is equal to the sender's current power level, reject + if user != power_event.sender() && old_level == Some(&user_level) { + warn!("m.room.power_level cannot remove ops == to own"); + return Some(false); // cannot remove ops level == to own + } + + // If the current value is higher than the sender's current power level, reject + // If the new value is higher than the sender's current power level, reject + let old_level_too_big = old_level > Some(&user_level); + let new_level_too_big = new_level > Some(&user_level); + if old_level_too_big || new_level_too_big { + warn!("m.room.power_level failed to add ops > than own"); + return Some(false); // cannot add ops greater than own + } + } + + // EventType loop + for ev_type in event_levels_to_check { + let old_level = old_state.events.get(ev_type); + let new_level = new_state.events.get(ev_type); + if old_level.is_some() && new_level.is_some() && old_level == new_level { + continue; + } + + // If the current value is higher than the sender's current power level, reject + // If the new value is higher than the sender's current power level, reject + let old_level_too_big = old_level > Some(&user_level); + let new_level_too_big = new_level > Some(&user_level); + if old_level_too_big || new_level_too_big { + warn!("m.room.power_level failed to add ops > than own"); + return Some(false); // cannot add ops greater than own + } + } + + // Notifications, currently there is only @room + if room_version.limit_notifications_power_levels { + let old_level = old_state.notifications.room; + let new_level = new_state.notifications.room; + if old_level != new_level { + // If the current value is higher than the sender's current power level, reject + // If the new value is higher than the sender's current power level, reject + let old_level_too_big = old_level > user_level; + let new_level_too_big = new_level > user_level; + if old_level_too_big || new_level_too_big { + warn!("m.room.power_level failed to add ops > than own"); + return Some(false); // cannot add ops greater than own + } + } + } + + let levels = [ + "users_default", + "events_default", + "state_default", + "ban", + "redact", + "kick", + "invite", + ]; + let old_state = serde_json::to_value(old_state).unwrap(); + let new_state = serde_json::to_value(new_state).unwrap(); + for lvl_name in &levels { + if let Some((old_lvl, new_lvl)) = get_deserialize_levels(&old_state, &new_state, lvl_name) + { + let old_level_too_big = old_lvl > user_level; + let new_level_too_big = new_lvl > user_level; + + if old_level_too_big || new_level_too_big { + warn!("cannot add ops > than own"); + return Some(false); + } + } + } + + Some(true) +} + +fn get_deserialize_levels( + old: &serde_json::Value, + new: &serde_json::Value, + name: &str, +) -> Option<(Int, Int)> { + Some(( + serde_json::from_value(old.get(name)?.clone()).ok()?, + serde_json::from_value(new.get(name)?.clone()).ok()?, + )) +} + +/// Does the event redacting come from a user with enough power to redact the +/// given event. +fn check_redaction( + _room_version: &RoomVersion, + redaction_event: impl Event, + user_level: Int, + redact_level: Int, +) -> Result { + if user_level >= redact_level { + debug!("redaction allowed via power levels"); + return Ok(true); + } + + // If the domain of the event_id of the event being redacted is the same as the + // domain of the event_id of the m.room.redaction, allow + if redaction_event.event_id().borrow().server_name() + == redaction_event + .redacts() + .as_ref() + .and_then(|&id| id.borrow().server_name()) + { + debug!("redaction event allowed via room version 1 rules"); + return Ok(true); + } + + Ok(false) +} + +/// Helper function to fetch the power level needed to send an event of type +/// `e_type` based on the rooms "m.room.power_level" event. +fn get_send_level( + e_type: &TimelineEventType, + state_key: Option<&str>, + power_lvl: Option, +) -> Int { + power_lvl + .and_then(|ple| { + from_json_str::(ple.content().get()) + .map(|content| { + content.events.get(e_type).copied().unwrap_or_else(|| { + if state_key.is_some() { + content.state_default + } else { + content.events_default + } + }) + }) + .ok() + }) + .unwrap_or_else(|| if state_key.is_some() { int!(50) } else { int!(0) }) +} + +fn verify_third_party_invite( + target_user: Option<&UserId>, + sender: &UserId, + tp_id: &ThirdPartyInvite, + current_third_party_invite: Option, +) -> bool { + // 1. Check for user being banned happens before this is called + // checking for mxid and token keys is done by ruma when deserializing + + // The state key must match the invitee + if target_user != Some(&tp_id.signed.mxid) { + return false; + } + + // If there is no m.room.third_party_invite event in the current room state with + // state_key matching token, reject + let current_tpid = match current_third_party_invite { + | Some(id) => id, + | None => return false, + }; + + if current_tpid.state_key() != Some(&tp_id.signed.token) { + return false; + } + + if sender != current_tpid.sender() { + return false; + } + + // If any signature in signed matches any public key in the + // m.room.third_party_invite event, allow + let tpid_ev = + match from_json_str::(current_tpid.content().get()) { + | Ok(ev) => ev, + | Err(_) => return false, + }; + + let decoded_invite_token = match Base64::parse(&tp_id.signed.token) { + | Ok(tok) => tok, + // FIXME: Log a warning? + | Err(_) => return false, + }; + + // A list of public keys in the public_keys field + for key in tpid_ev.public_keys.unwrap_or_default() { + if key.public_key == decoded_invite_token { + return true; + } + } + + // A single public key in the public_key field + tpid_ev.public_key == decoded_invite_token +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use ruma_events::{ + room::{ + join_rules::{ + AllowRule, JoinRule, Restricted, RoomJoinRulesEventContent, RoomMembership, + }, + member::{MembershipState, RoomMemberEventContent}, + }, + StateEventType, TimelineEventType, + }; + use serde_json::value::to_raw_value as to_raw_json_value; + + use crate::{ + event_auth::valid_membership_change, + test_utils::{ + alice, charlie, ella, event_id, member_content_ban, member_content_join, room_id, + to_pdu_event, PduEvent, INITIAL_EVENTS, INITIAL_EVENTS_CREATE_ROOM, + }, + Event, EventTypeExt, RoomVersion, StateMap, + }; + + #[test] + fn test_ban_pass() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + let events = INITIAL_EVENTS(); + + let auth_events = events + .values() + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .collect::>(); + + let requester = to_pdu_event( + "HELLO", + alice(), + TimelineEventType::RoomMember, + Some(charlie().as_str()), + member_content_ban(), + &[], + &["IMC"], + ); + + let fetch_state = |ty, key| auth_events.get(&(ty, key)).cloned(); + let target_user = charlie(); + let sender = alice(); + + assert!(valid_membership_change( + &RoomVersion::V6, + target_user, + fetch_state(StateEventType::RoomMember, target_user.to_string()), + sender, + fetch_state(StateEventType::RoomMember, sender.to_string()), + &requester, + None::, + fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), + fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None, + &MembershipState::Leave, + fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + ) + .unwrap()); + } + + #[test] + fn test_join_non_creator() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + let events = INITIAL_EVENTS_CREATE_ROOM(); + + let auth_events = events + .values() + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .collect::>(); + + let requester = to_pdu_event( + "HELLO", + charlie(), + TimelineEventType::RoomMember, + Some(charlie().as_str()), + member_content_join(), + &["CREATE"], + &["CREATE"], + ); + + let fetch_state = |ty, key| auth_events.get(&(ty, key)).cloned(); + let target_user = charlie(); + let sender = charlie(); + + assert!(!valid_membership_change( + &RoomVersion::V6, + target_user, + fetch_state(StateEventType::RoomMember, target_user.to_string()), + sender, + fetch_state(StateEventType::RoomMember, sender.to_string()), + &requester, + None::, + fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), + fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None, + &MembershipState::Leave, + fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + ) + .unwrap()); + } + + #[test] + fn test_join_creator() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + let events = INITIAL_EVENTS_CREATE_ROOM(); + + let auth_events = events + .values() + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .collect::>(); + + let requester = to_pdu_event( + "HELLO", + alice(), + TimelineEventType::RoomMember, + Some(alice().as_str()), + member_content_join(), + &["CREATE"], + &["CREATE"], + ); + + let fetch_state = |ty, key| auth_events.get(&(ty, key)).cloned(); + let target_user = alice(); + let sender = alice(); + + assert!(valid_membership_change( + &RoomVersion::V6, + target_user, + fetch_state(StateEventType::RoomMember, target_user.to_string()), + sender, + fetch_state(StateEventType::RoomMember, sender.to_string()), + &requester, + None::, + fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), + fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None, + &MembershipState::Leave, + fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + ) + .unwrap()); + } + + #[test] + fn test_ban_fail() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + let events = INITIAL_EVENTS(); + + let auth_events = events + .values() + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .collect::>(); + + let requester = to_pdu_event( + "HELLO", + charlie(), + TimelineEventType::RoomMember, + Some(alice().as_str()), + member_content_ban(), + &[], + &["IMC"], + ); + + let fetch_state = |ty, key| auth_events.get(&(ty, key)).cloned(); + let target_user = alice(); + let sender = charlie(); + + assert!(!valid_membership_change( + &RoomVersion::V6, + target_user, + fetch_state(StateEventType::RoomMember, target_user.to_string()), + sender, + fetch_state(StateEventType::RoomMember, sender.to_string()), + &requester, + None::, + fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), + fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None, + &MembershipState::Leave, + fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + ) + .unwrap()); + } + + #[test] + fn test_restricted_join_rule() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + let mut events = INITIAL_EVENTS(); + *events.get_mut(&event_id("IJR")).unwrap() = to_pdu_event( + "IJR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Restricted( + Restricted::new(vec![AllowRule::RoomMembership(RoomMembership::new( + room_id().to_owned(), + ))]), + ))) + .unwrap(), + &["CREATE", "IMA", "IPOWER"], + &["IPOWER"], + ); + + let mut member = RoomMemberEventContent::new(MembershipState::Join); + member.join_authorized_via_users_server = Some(alice().to_owned()); + + let auth_events = events + .values() + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .collect::>(); + + let requester = to_pdu_event( + "HELLO", + ella(), + TimelineEventType::RoomMember, + Some(ella().as_str()), + to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Join)).unwrap(), + &["CREATE", "IJR", "IPOWER", "new"], + &["new"], + ); + + let fetch_state = |ty, key| auth_events.get(&(ty, key)).cloned(); + let target_user = ella(); + let sender = ella(); + + assert!(valid_membership_change( + &RoomVersion::V9, + target_user, + fetch_state(StateEventType::RoomMember, target_user.to_string()), + sender, + fetch_state(StateEventType::RoomMember, sender.to_string()), + &requester, + None::, + fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), + fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + Some(alice()), + &MembershipState::Join, + fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + ) + .unwrap()); + + assert!(!valid_membership_change( + &RoomVersion::V9, + target_user, + fetch_state(StateEventType::RoomMember, target_user.to_string()), + sender, + fetch_state(StateEventType::RoomMember, sender.to_string()), + &requester, + None::, + fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), + fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + Some(ella()), + &MembershipState::Leave, + fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + ) + .unwrap()); + } + + #[test] + fn test_knock() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + let mut events = INITIAL_EVENTS(); + *events.get_mut(&event_id("IJR")).unwrap() = to_pdu_event( + "IJR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Knock)).unwrap(), + &["CREATE", "IMA", "IPOWER"], + &["IPOWER"], + ); + + let auth_events = events + .values() + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .collect::>(); + + let requester = to_pdu_event( + "HELLO", + ella(), + TimelineEventType::RoomMember, + Some(ella().as_str()), + to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Knock)).unwrap(), + &[], + &["IMC"], + ); + + let fetch_state = |ty, key| auth_events.get(&(ty, key)).cloned(); + let target_user = ella(); + let sender = ella(); + + assert!(valid_membership_change( + &RoomVersion::V7, + target_user, + fetch_state(StateEventType::RoomMember, target_user.to_string()), + sender, + fetch_state(StateEventType::RoomMember, sender.to_string()), + &requester, + None::, + fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), + fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None, + &MembershipState::Leave, + fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + ) + .unwrap()); + } +} diff --git a/src/core/state_res/mod.rs b/src/core/state_res/mod.rs new file mode 100644 index 00000000..e4054377 --- /dev/null +++ b/src/core/state_res/mod.rs @@ -0,0 +1,1644 @@ +pub(crate) mod error; +pub mod event_auth; +mod power_levels; +mod room_version; +mod state_event; + +#[cfg(test)] +mod test_utils; + +use std::{ + borrow::Borrow, + cmp::{Ordering, Reverse}, + collections::{BinaryHeap, HashMap, HashSet}, + fmt::Debug, + hash::Hash, +}; + +use futures::{future, stream, Future, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; +use ruma::{ + events::{ + room::member::{MembershipState, RoomMemberEventContent}, + StateEventType, TimelineEventType, + }, + int, EventId, Int, MilliSecondsSinceUnixEpoch, RoomVersionId, +}; +use serde_json::from_str as from_json_str; + +pub(crate) use self::error::Error; +use self::power_levels::PowerLevelsContentFields; +pub use self::{ + event_auth::{auth_check, auth_types_for_event}, + room_version::RoomVersion, + state_event::Event, +}; +use crate::{debug, trace, warn}; + +/// A mapping of event type and state_key to some value `T`, usually an +/// `EventId`. +pub type StateMap = HashMap; +pub type StateMapItem = (TypeStateKey, T); +pub type TypeStateKey = (StateEventType, String); + +type Result = crate::Result; + +/// Resolve sets of state events as they come in. +/// +/// Internally `StateResolution` builds a graph and an auth chain to allow for +/// state conflict resolution. +/// +/// ## Arguments +/// +/// * `state_sets` - The incoming state to resolve. Each `StateMap` represents a +/// possible fork in the state of a room. +/// +/// * `auth_chain_sets` - The full recursive set of `auth_events` for each event +/// in the `state_sets`. +/// +/// * `event_fetch` - Any event not found in the `event_map` will defer to this +/// closure to find the event. +/// +/// * `parallel_fetches` - The number of asynchronous fetch requests in-flight +/// for any given operation. +/// +/// ## Invariants +/// +/// The caller of `resolve` must ensure that all the events are from the same +/// room. Although this function takes a `RoomId` it does not check that each +/// event is part of the same room. +//#[tracing::instrument(level = "debug", skip(state_sets, auth_chain_sets, +//#[tracing::instrument(level event_fetch))] +pub async fn resolve<'a, E, SetIter, Fetch, FetchFut, Exists, ExistsFut>( + room_version: &RoomVersionId, + state_sets: impl IntoIterator + Send, + auth_chain_sets: &'a [HashSet], + event_fetch: &Fetch, + event_exists: &Exists, + parallel_fetches: usize, +) -> Result> +where + Fetch: Fn(E::Id) -> FetchFut + Sync, + FetchFut: Future> + Send, + Exists: Fn(E::Id) -> ExistsFut + Sync, + ExistsFut: Future + Send, + SetIter: Iterator> + Clone + Send, + E: Event + Clone + Send + Sync, + E::Id: Borrow + Send + Sync, + for<'b> &'b E: Send, +{ + debug!("State resolution starting"); + + // Split non-conflicting and conflicting state + let (clean, conflicting) = separate(state_sets.into_iter()); + + debug!(count = clean.len(), "non-conflicting events"); + trace!(map = ?clean, "non-conflicting events"); + + if conflicting.is_empty() { + debug!("no conflicting state found"); + return Ok(clean); + } + + debug!(count = conflicting.len(), "conflicting events"); + trace!(map = ?conflicting, "conflicting events"); + + let auth_chain_diff = + get_auth_chain_diff(auth_chain_sets).chain(conflicting.into_values().flatten()); + + // `all_conflicted` contains unique items + // synapse says `full_set = {eid for eid in full_conflicted_set if eid in + // event_map}` + let all_conflicted: HashSet<_> = stream::iter(auth_chain_diff) + // Don't honor events we cannot "verify" + .map(|id| event_exists(id.clone()).map(move |exists| (id, exists))) + .buffer_unordered(parallel_fetches) + .filter_map(|(id, exists)| future::ready(exists.then_some(id))) + .collect() + .boxed() + .await; + + debug!(count = all_conflicted.len(), "full conflicted set"); + trace!(set = ?all_conflicted, "full conflicted set"); + + // We used to check that all events are events from the correct room + // this is now a check the caller of `resolve` must make. + + // Get only the control events with a state_key: "" or ban/kick event (sender != + // state_key) + let control_events: Vec<_> = stream::iter(all_conflicted.iter()) + .map(|id| is_power_event_id(id, &event_fetch).map(move |is| (id, is))) + .buffer_unordered(parallel_fetches) + .filter_map(|(id, is)| future::ready(is.then_some(id.clone()))) + .collect() + .boxed() + .await; + + // Sort the control events based on power_level/clock/event_id and + // outgoing/incoming edges + let sorted_control_levels = reverse_topological_power_sort( + control_events, + &all_conflicted, + &event_fetch, + parallel_fetches, + ) + .boxed() + .await?; + + debug!(count = sorted_control_levels.len(), "power events"); + trace!(list = ?sorted_control_levels, "sorted power events"); + + let room_version = RoomVersion::new(room_version)?; + // Sequentially auth check each control event. + let resolved_control = iterative_auth_check( + &room_version, + sorted_control_levels.iter(), + clean.clone(), + &event_fetch, + parallel_fetches, + ) + .boxed() + .await?; + + debug!(count = resolved_control.len(), "resolved power events"); + trace!(map = ?resolved_control, "resolved power events"); + + // At this point the control_events have been resolved we now have to + // sort the remaining events using the mainline of the resolved power level. + let deduped_power_ev = sorted_control_levels.into_iter().collect::>(); + + // This removes the control events that passed auth and more importantly those + // that failed auth + let events_to_resolve = all_conflicted + .iter() + .filter(|&id| !deduped_power_ev.contains(id.borrow())) + .cloned() + .collect::>(); + + debug!(count = events_to_resolve.len(), "events left to resolve"); + trace!(list = ?events_to_resolve, "events left to resolve"); + + // This "epochs" power level event + let power_event = resolved_control.get(&(StateEventType::RoomPowerLevels, String::new())); + + debug!(event_id = ?power_event, "power event"); + + let sorted_left_events = + mainline_sort(&events_to_resolve, power_event.cloned(), &event_fetch, parallel_fetches) + .boxed() + .await?; + + trace!(list = ?sorted_left_events, "events left, sorted"); + + let mut resolved_state = iterative_auth_check( + &room_version, + sorted_left_events.iter(), + resolved_control, // The control events are added to the final resolved state + &event_fetch, + parallel_fetches, + ) + .boxed() + .await?; + + // Add unconflicted state to the resolved state + // We priorities the unconflicting state + resolved_state.extend(clean); + + debug!("state resolution finished"); + + Ok(resolved_state) +} + +/// Split the events that have no conflicts from those that are conflicting. +/// +/// The return tuple looks like `(unconflicted, conflicted)`. +/// +/// State is determined to be conflicting if for the given key (StateEventType, +/// StateKey) there is not exactly one event ID. This includes missing events, +/// if one state_set includes an event that none of the other have this is a +/// conflicting event. +fn separate<'a, Id>( + state_sets_iter: impl Iterator>, +) -> (StateMap, StateMap>) +where + Id: Clone + Eq + Hash + 'a, +{ + let mut state_set_count = 0_usize; + let mut occurrences = HashMap::<_, HashMap<_, _>>::new(); + + let state_sets_iter = state_sets_iter.inspect(|_| state_set_count += 1); + for (k, v) in state_sets_iter.flatten() { + occurrences + .entry(k) + .or_default() + .entry(v) + .and_modify(|x| *x += 1) + .or_insert(1); + } + + let mut unconflicted_state = StateMap::new(); + let mut conflicted_state = StateMap::new(); + + for (k, v) in occurrences { + for (id, occurrence_count) in v { + if occurrence_count == state_set_count { + unconflicted_state.insert((k.0.clone(), k.1.clone()), id.clone()); + } else { + conflicted_state + .entry((k.0.clone(), k.1.clone())) + .and_modify(|x: &mut Vec<_>| x.push(id.clone())) + .or_insert(vec![id.clone()]); + } + } + } + + (unconflicted_state, conflicted_state) +} + +/// Returns a Vec of deduped EventIds that appear in some chains but not others. +fn get_auth_chain_diff(auth_chain_sets: &[HashSet]) -> impl Iterator + Send +where + Id: Clone + Eq + Hash + Send, +{ + let num_sets = auth_chain_sets.len(); + let mut id_counts: HashMap = HashMap::new(); + for id in auth_chain_sets.iter().flatten() { + *id_counts.entry(id.clone()).or_default() += 1; + } + + id_counts + .into_iter() + .filter_map(move |(id, count)| (count < num_sets).then_some(id)) +} + +/// Events are sorted from "earliest" to "latest". +/// +/// They are compared using the negative power level (reverse topological +/// ordering), the origin server timestamp and in case of a tie the `EventId`s +/// are compared lexicographically. +/// +/// The power level is negative because a higher power level is equated to an +/// earlier (further back in time) origin server timestamp. +#[tracing::instrument(level = "debug", skip_all)] +async fn reverse_topological_power_sort( + events_to_sort: Vec, + auth_diff: &HashSet, + fetch_event: &F, + parallel_fetches: usize, +) -> Result> +where + F: Fn(E::Id) -> Fut + Sync, + Fut: Future> + Send, + E: Event + Send, + E::Id: Borrow + Send + Sync, +{ + debug!("reverse topological sort of power events"); + + let mut graph = HashMap::new(); + for event_id in events_to_sort { + add_event_and_auth_chain_to_graph(&mut graph, event_id, auth_diff, fetch_event).await; + } + + // This is used in the `key_fn` passed to the lexico_topo_sort fn + let event_to_pl = stream::iter(graph.keys()) + .map(|event_id| { + get_power_level_for_sender(event_id.clone(), fetch_event, parallel_fetches) + .map(move |res| res.map(|pl| (event_id, pl))) + }) + .buffer_unordered(parallel_fetches) + .try_fold(HashMap::new(), |mut event_to_pl, (event_id, pl)| { + debug!( + event_id = event_id.borrow().as_str(), + power_level = i64::from(pl), + "found the power level of an event's sender", + ); + + event_to_pl.insert(event_id.clone(), pl); + future::ok(event_to_pl) + }) + .boxed() + .await?; + + let event_to_pl = &event_to_pl; + let fetcher = |event_id: E::Id| async move { + let pl = *event_to_pl + .get(event_id.borrow()) + .ok_or_else(|| Error::NotFound(String::new()))?; + let ev = fetch_event(event_id) + .await + .ok_or_else(|| Error::NotFound(String::new()))?; + Ok((pl, ev.origin_server_ts())) + }; + + lexicographical_topological_sort(&graph, &fetcher).await +} + +/// Sorts the event graph based on number of outgoing/incoming edges. +/// +/// `key_fn` is used as to obtain the power level and age of an event for +/// breaking ties (together with the event ID). +#[tracing::instrument(level = "debug", skip_all)] +pub async fn lexicographical_topological_sort( + graph: &HashMap>, + key_fn: &F, +) -> Result> +where + F: Fn(Id) -> Fut + Sync, + Fut: Future> + Send, + Id: Borrow + Clone + Eq + Hash + Ord + Send, +{ + #[derive(PartialEq, Eq)] + struct TieBreaker<'a, Id> { + power_level: Int, + origin_server_ts: MilliSecondsSinceUnixEpoch, + event_id: &'a Id, + } + + impl Ord for TieBreaker<'_, Id> + where + Id: Ord, + { + fn cmp(&self, other: &Self) -> Ordering { + // NOTE: the power level comparison is "backwards" intentionally. + // See the "Mainline ordering" section of the Matrix specification + // around where it says the following: + // + // > for events `x` and `y`, `x < y` if [...] + // + // + other + .power_level + .cmp(&self.power_level) + .then(self.origin_server_ts.cmp(&other.origin_server_ts)) + .then(self.event_id.cmp(other.event_id)) + } + } + + impl PartialOrd for TieBreaker<'_, Id> + where + Id: Ord, + { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } + } + + debug!("starting lexicographical topological sort"); + + // NOTE: an event that has no incoming edges happened most recently, + // and an event that has no outgoing edges happened least recently. + + // NOTE: this is basically Kahn's algorithm except we look at nodes with no + // outgoing edges, c.f. + // https://en.wikipedia.org/wiki/Topological_sorting#Kahn's_algorithm + + // outdegree_map is an event referring to the events before it, the + // more outdegree's the more recent the event. + let mut outdegree_map = graph.clone(); + + // The number of events that depend on the given event (the EventId key) + // How many events reference this event in the DAG as a parent + let mut reverse_graph: HashMap<_, HashSet<_>> = HashMap::new(); + + // Vec of nodes that have zero out degree, least recent events. + let mut zero_outdegree = Vec::new(); + + for (node, edges) in graph { + if edges.is_empty() { + let (power_level, origin_server_ts) = key_fn(node.clone()).await?; + // The `Reverse` is because rusts `BinaryHeap` sorts largest -> smallest we need + // smallest -> largest + zero_outdegree.push(Reverse(TieBreaker { + power_level, + origin_server_ts, + event_id: node, + })); + } + + reverse_graph.entry(node).or_default(); + for edge in edges { + reverse_graph.entry(edge).or_default().insert(node); + } + } + + let mut heap = BinaryHeap::from(zero_outdegree); + + // We remove the oldest node (most incoming edges) and check against all other + let mut sorted = vec![]; + // Destructure the `Reverse` and take the smallest `node` each time + while let Some(Reverse(item)) = heap.pop() { + let node = item.event_id; + + for &parent in reverse_graph + .get(node) + .expect("EventId in heap is also in reverse_graph") + { + // The number of outgoing edges this node has + let out = outdegree_map + .get_mut(parent.borrow()) + .expect("outdegree_map knows of all referenced EventIds"); + + // Only push on the heap once older events have been cleared + out.remove(node.borrow()); + if out.is_empty() { + let (power_level, origin_server_ts) = key_fn(parent.clone()).await?; + heap.push(Reverse(TieBreaker { + power_level, + origin_server_ts, + event_id: parent, + })); + } + } + + // synapse yields we push then return the vec + sorted.push(node.clone()); + } + + Ok(sorted) +} + +/// Find the power level for the sender of `event_id` or return a default value +/// of zero. +/// +/// Do NOT use this any where but topological sort, we find the power level for +/// the eventId at the eventId's generation (we walk backwards to `EventId`s +/// most recent previous power level event). +async fn get_power_level_for_sender( + event_id: E::Id, + fetch_event: &F, + parallel_fetches: usize, +) -> serde_json::Result +where + F: Fn(E::Id) -> Fut + Sync, + Fut: Future> + Send, + E: Event + Send, + E::Id: Borrow + Send, +{ + debug!("fetch event ({event_id}) senders power level"); + + let event = fetch_event(event_id.clone()).await; + + let auth_events = event.as_ref().map(Event::auth_events).into_iter().flatten(); + + let pl = stream::iter(auth_events) + .map(|aid| fetch_event(aid.clone())) + .buffer_unordered(parallel_fetches.min(5)) + .filter_map(future::ready) + .collect::>() + .boxed() + .await + .into_iter() + .find(|aev| is_type_and_key(aev, &TimelineEventType::RoomPowerLevels, "")); + + let content: PowerLevelsContentFields = match pl { + | None => return Ok(int!(0)), + | Some(ev) => from_json_str(ev.content().get())?, + }; + + if let Some(ev) = event { + if let Some(&user_level) = content.get_user_power(ev.sender()) { + debug!("found {} at power_level {user_level}", ev.sender()); + return Ok(user_level); + } + } + + Ok(content.users_default) +} + +/// Check the that each event is authenticated based on the events before it. +/// +/// ## Returns +/// +/// The `unconflicted_state` combined with the newly auth'ed events. So any +/// event that fails the `event_auth::auth_check` will be excluded from the +/// returned state map. +/// +/// For each `events_to_check` event we gather the events needed to auth it from +/// the the `fetch_event` closure and verify each event using the +/// `event_auth::auth_check` function. +async fn iterative_auth_check<'a, E, F, Fut, I>( + room_version: &RoomVersion, + events_to_check: I, + unconflicted_state: StateMap, + fetch_event: &F, + parallel_fetches: usize, +) -> Result> +where + F: Fn(E::Id) -> Fut + Sync, + Fut: Future> + Send, + E::Id: Borrow + Clone + Eq + Ord + Send + Sync + 'a, + I: Iterator + Debug + Send + 'a, + E: Event + Clone + Send + Sync, +{ + debug!("starting iterative auth check"); + trace!( + list = ?events_to_check, + "events to check" + ); + + let events_to_check: Vec<_> = stream::iter(events_to_check) + .map(Result::Ok) + .map_ok(|event_id| { + fetch_event(event_id.clone()).map(move |result| { + result.ok_or_else(|| Error::NotFound(format!("Failed to find {event_id}"))) + }) + }) + .try_buffer_unordered(parallel_fetches) + .try_collect() + .boxed() + .await?; + + let auth_event_ids: HashSet = events_to_check + .iter() + .flat_map(|event: &E| event.auth_events().map(Clone::clone)) + .collect(); + + let auth_events: HashMap = stream::iter(auth_event_ids.into_iter()) + .map(fetch_event) + .buffer_unordered(parallel_fetches) + .filter_map(future::ready) + .map(|auth_event| (auth_event.event_id().clone(), auth_event)) + .collect() + .boxed() + .await; + + let auth_events = &auth_events; + let mut resolved_state = unconflicted_state; + for event in &events_to_check { + let event_id = event.event_id(); + let state_key = event + .state_key() + .ok_or_else(|| Error::InvalidPdu("State event had no state key".to_owned()))?; + + let auth_types = auth_types_for_event( + event.event_type(), + event.sender(), + Some(state_key), + event.content(), + )?; + + let mut auth_state = StateMap::new(); + for aid in event.auth_events() { + if let Some(ev) = auth_events.get(aid.borrow()) { + //TODO: synapse checks "rejected_reason" which is most likely related to + // soft-failing + auth_state.insert( + ev.event_type() + .with_state_key(ev.state_key().ok_or_else(|| { + Error::InvalidPdu("State event had no state key".to_owned()) + })?), + ev.clone(), + ); + } else { + warn!(event_id = aid.borrow().as_str(), "missing auth event"); + } + } + + stream::iter( + auth_types + .iter() + .filter_map(|key| Some((key, resolved_state.get(key)?))), + ) + .filter_map(|(key, ev_id)| async move { + if let Some(event) = auth_events.get(ev_id.borrow()) { + Some((key, event.clone())) + } else { + Some((key, fetch_event(ev_id.clone()).await?)) + } + }) + .for_each(|(key, event)| { + //TODO: synapse checks "rejected_reason" is None here + auth_state.insert(key.to_owned(), event); + future::ready(()) + }) + .await; + + debug!("event to check {:?}", event.event_id()); + + // The key for this is (eventType + a state_key of the signed token not sender) + // so search for it + let current_third_party = auth_state.iter().find_map(|(_, pdu)| { + (*pdu.event_type() == TimelineEventType::RoomThirdPartyInvite).then_some(pdu) + }); + + let fetch_state = |ty: &StateEventType, key: &str| { + future::ready(auth_state.get(&ty.with_state_key(key))) + }; + + if auth_check(room_version, &event, current_third_party.as_ref(), fetch_state).await? { + // add event to resolved state map + resolved_state.insert(event.event_type().with_state_key(state_key), event_id.clone()); + } else { + // synapse passes here on AuthError. We do not add this event to resolved_state. + warn!("event {event_id} failed the authentication check"); + } + } + + Ok(resolved_state) +} + +/// Returns the sorted `to_sort` list of `EventId`s based on a mainline sort +/// using the depth of `resolved_power_level`, the server timestamp, and the +/// eventId. +/// +/// The depth of the given event is calculated based on the depth of it's +/// closest "parent" power_level event. If there have been two power events the +/// after the most recent are depth 0, the events before (with the first power +/// level as a parent) will be marked as depth 1. depth 1 is "older" than depth +/// 0. +async fn mainline_sort( + to_sort: &[E::Id], + resolved_power_level: Option, + fetch_event: &F, + parallel_fetches: usize, +) -> Result> +where + F: Fn(E::Id) -> Fut + Sync, + Fut: Future> + Send, + E: Event + Clone + Send + Sync, + E::Id: Borrow + Clone + Send + Sync, +{ + debug!("mainline sort of events"); + + // There are no EventId's to sort, bail. + if to_sort.is_empty() { + return Ok(vec![]); + } + + let mut mainline = vec![]; + let mut pl = resolved_power_level; + while let Some(p) = pl { + mainline.push(p.clone()); + + let event = fetch_event(p.clone()) + .await + .ok_or_else(|| Error::NotFound(format!("Failed to find {p}")))?; + pl = None; + for aid in event.auth_events() { + let ev = fetch_event(aid.clone()) + .await + .ok_or_else(|| Error::NotFound(format!("Failed to find {aid}")))?; + if is_type_and_key(&ev, &TimelineEventType::RoomPowerLevels, "") { + pl = Some(aid.to_owned()); + break; + } + } + } + + let mainline_map = mainline + .iter() + .rev() + .enumerate() + .map(|(idx, eid)| ((*eid).clone(), idx)) + .collect::>(); + + let order_map = stream::iter(to_sort.iter()) + .map(|ev_id| { + fetch_event(ev_id.clone()).map(move |event| event.map(|event| (event, ev_id))) + }) + .buffer_unordered(parallel_fetches) + .filter_map(future::ready) + .map(|(event, ev_id)| { + get_mainline_depth(Some(event.clone()), &mainline_map, fetch_event) + .map_ok(move |depth| (depth, event, ev_id)) + .map(Result::ok) + }) + .buffer_unordered(parallel_fetches) + .filter_map(future::ready) + .fold(HashMap::new(), |mut order_map, (depth, event, ev_id)| { + order_map.insert(ev_id, (depth, event.origin_server_ts(), ev_id)); + future::ready(order_map) + }) + .boxed() + .await; + + // Sort the event_ids by their depth, timestamp and EventId + // unwrap is OK order map and sort_event_ids are from to_sort (the same Vec) + let mut sort_event_ids = order_map.keys().map(|&k| k.clone()).collect::>(); + sort_event_ids.sort_by_key(|sort_id| &order_map[sort_id]); + + Ok(sort_event_ids) +} + +/// Get the mainline depth from the `mainline_map` or finds a power_level event +/// that has an associated mainline depth. +async fn get_mainline_depth( + mut event: Option, + mainline_map: &HashMap, + fetch_event: &F, +) -> Result +where + F: Fn(E::Id) -> Fut + Sync, + Fut: Future> + Send, + E: Event + Send, + E::Id: Borrow + Send, +{ + while let Some(sort_ev) = event { + debug!(event_id = sort_ev.event_id().borrow().as_str(), "mainline"); + let id = sort_ev.event_id(); + if let Some(depth) = mainline_map.get(id.borrow()) { + return Ok(*depth); + } + + event = None; + for aid in sort_ev.auth_events() { + let aev = fetch_event(aid.clone()) + .await + .ok_or_else(|| Error::NotFound(format!("Failed to find {aid}")))?; + if is_type_and_key(&aev, &TimelineEventType::RoomPowerLevels, "") { + event = Some(aev); + break; + } + } + } + // Did not find a power level event so we default to zero + Ok(0) +} + +async fn add_event_and_auth_chain_to_graph( + graph: &mut HashMap>, + event_id: E::Id, + auth_diff: &HashSet, + fetch_event: &F, +) where + F: Fn(E::Id) -> Fut, + Fut: Future> + Send, + E: Event + Send, + E::Id: Borrow + Clone + Send, +{ + let mut state = vec![event_id]; + while let Some(eid) = state.pop() { + graph.entry(eid.clone()).or_default(); + let event = fetch_event(eid.clone()).await; + let auth_events = event.as_ref().map(Event::auth_events).into_iter().flatten(); + + // Prefer the store to event as the store filters dedups the events + for aid in auth_events { + if auth_diff.contains(aid.borrow()) { + if !graph.contains_key(aid.borrow()) { + state.push(aid.to_owned()); + } + + // We just inserted this at the start of the while loop + graph.get_mut(eid.borrow()).unwrap().insert(aid.to_owned()); + } + } + } +} + +async fn is_power_event_id(event_id: &E::Id, fetch: &F) -> bool +where + F: Fn(E::Id) -> Fut + Sync, + Fut: Future> + Send, + E: Event + Send, + E::Id: Borrow + Send, +{ + match fetch(event_id.clone()).await.as_ref() { + | Some(state) => is_power_event(state), + | _ => false, + } +} + +fn is_type_and_key(ev: impl Event, ev_type: &TimelineEventType, state_key: &str) -> bool { + ev.event_type() == ev_type && ev.state_key() == Some(state_key) +} + +fn is_power_event(event: impl Event) -> bool { + match event.event_type() { + | TimelineEventType::RoomPowerLevels + | TimelineEventType::RoomJoinRules + | TimelineEventType::RoomCreate => event.state_key() == Some(""), + | TimelineEventType::RoomMember => { + if let Ok(content) = from_json_str::(event.content().get()) { + if [MembershipState::Leave, MembershipState::Ban].contains(&content.membership) { + return Some(event.sender().as_str()) != event.state_key(); + } + } + + false + }, + | _ => false, + } +} + +/// Convenience trait for adding event type plus state key to state maps. +pub trait EventTypeExt { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, String); +} + +impl EventTypeExt for StateEventType { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { + (self, state_key.into()) + } +} + +impl EventTypeExt for TimelineEventType { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { + (self.to_string().into(), state_key.into()) + } +} + +impl EventTypeExt for &T +where + T: EventTypeExt + Clone, +{ + fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { + self.to_owned().with_state_key(state_key) + } +} + +#[cfg(test)] +mod tests { + use std::{ + collections::{HashMap, HashSet}, + sync::Arc, + }; + + use maplit::{hashmap, hashset}; + use rand::seq::SliceRandom; + use ruma::{ + events::{ + room::join_rules::{JoinRule, RoomJoinRulesEventContent}, + StateEventType, TimelineEventType, + }, + int, uint, + }; + use ruma_common::{MilliSecondsSinceUnixEpoch, OwnedEventId, RoomVersionId}; + use serde_json::{json, value::to_raw_value as to_raw_json_value}; + use tracing::debug; + + use crate::{ + is_power_event, + room_version::RoomVersion, + test_utils::{ + alice, bob, charlie, do_check, ella, event_id, member_content_ban, + member_content_join, room_id, to_init_pdu_event, to_pdu_event, zara, PduEvent, + TestStore, INITIAL_EVENTS, + }, + Event, EventTypeExt, StateMap, + }; + + async fn test_event_sort() { + use futures::future::ready; + + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + let events = INITIAL_EVENTS(); + + let event_map = events + .values() + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.clone())) + .collect::>(); + + let auth_chain: HashSet = HashSet::new(); + + let power_events = event_map + .values() + .filter(|&pdu| is_power_event(&**pdu)) + .map(|pdu| pdu.event_id.clone()) + .collect::>(); + + let fetcher = |id| ready(events.get(&id).cloned()); + let sorted_power_events = + crate::reverse_topological_power_sort(power_events, &auth_chain, &fetcher, 1) + .await + .unwrap(); + + let resolved_power = crate::iterative_auth_check( + &RoomVersion::V6, + sorted_power_events.iter(), + HashMap::new(), // unconflicted events + &fetcher, + 1, + ) + .await + .expect("iterative auth check failed on resolved events"); + + // don't remove any events so we know it sorts them all correctly + let mut events_to_sort = events.keys().cloned().collect::>(); + + events_to_sort.shuffle(&mut rand::thread_rng()); + + let power_level = resolved_power + .get(&(StateEventType::RoomPowerLevels, "".to_owned())) + .cloned(); + + let sorted_event_ids = crate::mainline_sort(&events_to_sort, power_level, &fetcher, 1) + .await + .unwrap(); + + assert_eq!( + vec![ + "$CREATE:foo", + "$IMA:foo", + "$IPOWER:foo", + "$IJR:foo", + "$IMB:foo", + "$IMC:foo", + "$START:foo", + "$END:foo" + ], + sorted_event_ids + .iter() + .map(|id| id.to_string()) + .collect::>() + ); + } + + #[tokio::test] + async fn test_sort() { + for _ in 0..20 { + // since we shuffle the eventIds before we sort them introducing randomness + // seems like we should test this a few times + test_event_sort().await; + } + } + + #[tokio::test] + async fn ban_vs_power_level() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + + let events = &[ + to_init_pdu_event( + "PA", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + ), + to_init_pdu_event( + "MA", + alice(), + TimelineEventType::RoomMember, + Some(alice().to_string().as_str()), + member_content_join(), + ), + to_init_pdu_event( + "MB", + alice(), + TimelineEventType::RoomMember, + Some(bob().to_string().as_str()), + member_content_ban(), + ), + to_init_pdu_event( + "PB", + bob(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + ), + ]; + + let edges = vec![vec!["END", "MB", "MA", "PA", "START"], vec!["END", "PA", "PB"]] + .into_iter() + .map(|list| list.into_iter().map(event_id).collect::>()) + .collect::>(); + + let expected_state_ids = vec!["PA", "MA", "MB"] + .into_iter() + .map(event_id) + .collect::>(); + + do_check(events, edges, expected_state_ids).await; + } + + #[tokio::test] + async fn topic_basic() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + + let events = &[ + to_init_pdu_event( + "T1", + alice(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + to_init_pdu_event( + "PA1", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + ), + to_init_pdu_event( + "T2", + alice(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + to_init_pdu_event( + "PA2", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 0 } })).unwrap(), + ), + to_init_pdu_event( + "PB", + bob(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + ), + to_init_pdu_event( + "T3", + bob(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + ]; + + let edges = + vec![vec!["END", "PA2", "T2", "PA1", "T1", "START"], vec!["END", "T3", "PB", "PA1"]] + .into_iter() + .map(|list| list.into_iter().map(event_id).collect::>()) + .collect::>(); + + let expected_state_ids = vec!["PA2", "T2"] + .into_iter() + .map(event_id) + .collect::>(); + + do_check(events, edges, expected_state_ids).await; + } + + #[tokio::test] + async fn topic_reset() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + + let events = &[ + to_init_pdu_event( + "T1", + alice(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + to_init_pdu_event( + "PA", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + ), + to_init_pdu_event( + "T2", + bob(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + to_init_pdu_event( + "MB", + alice(), + TimelineEventType::RoomMember, + Some(bob().to_string().as_str()), + member_content_ban(), + ), + ]; + + let edges = vec![vec!["END", "MB", "T2", "PA", "T1", "START"], vec!["END", "T1"]] + .into_iter() + .map(|list| list.into_iter().map(event_id).collect::>()) + .collect::>(); + + let expected_state_ids = vec!["T1", "MB", "PA"] + .into_iter() + .map(event_id) + .collect::>(); + + do_check(events, edges, expected_state_ids).await; + } + + #[tokio::test] + async fn join_rule_evasion() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + + let events = &[ + to_init_pdu_event( + "JR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Private)).unwrap(), + ), + to_init_pdu_event( + "ME", + ella(), + TimelineEventType::RoomMember, + Some(ella().to_string().as_str()), + member_content_join(), + ), + ]; + + let edges = vec![vec!["END", "JR", "START"], vec!["END", "ME", "START"]] + .into_iter() + .map(|list| list.into_iter().map(event_id).collect::>()) + .collect::>(); + + let expected_state_ids = vec![event_id("JR")]; + + do_check(events, edges, expected_state_ids).await; + } + + #[tokio::test] + async fn offtopic_power_level() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + + let events = &[ + to_init_pdu_event( + "PA", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + ), + to_init_pdu_event( + "PB", + bob(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value( + &json!({ "users": { alice(): 100, bob(): 50, charlie(): 50 } }), + ) + .unwrap(), + ), + to_init_pdu_event( + "PC", + charlie(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50, charlie(): 0 } })) + .unwrap(), + ), + ]; + + let edges = vec![vec!["END", "PC", "PB", "PA", "START"], vec!["END", "PA"]] + .into_iter() + .map(|list| list.into_iter().map(event_id).collect::>()) + .collect::>(); + + let expected_state_ids = vec!["PC"].into_iter().map(event_id).collect::>(); + + do_check(events, edges, expected_state_ids).await; + } + + #[tokio::test] + async fn topic_setting() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + + let events = &[ + to_init_pdu_event( + "T1", + alice(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + to_init_pdu_event( + "PA1", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + ), + to_init_pdu_event( + "T2", + alice(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + to_init_pdu_event( + "PA2", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 0 } })).unwrap(), + ), + to_init_pdu_event( + "PB", + bob(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + ), + to_init_pdu_event( + "T3", + bob(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + to_init_pdu_event( + "MZ1", + zara(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + to_init_pdu_event( + "T4", + alice(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + ), + ]; + + let edges = vec![vec!["END", "T4", "MZ1", "PA2", "T2", "PA1", "T1", "START"], vec![ + "END", "MZ1", "T3", "PB", "PA1", + ]] + .into_iter() + .map(|list| list.into_iter().map(event_id).collect::>()) + .collect::>(); + + let expected_state_ids = vec!["T4", "PA2"] + .into_iter() + .map(event_id) + .collect::>(); + + do_check(events, edges, expected_state_ids).await; + } + + #[tokio::test] + async fn test_event_map_none() { + use futures::future::ready; + + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + + let mut store = TestStore::(hashmap! {}); + + // build up the DAG + let (state_at_bob, state_at_charlie, expected) = store.set_up(); + + let ev_map = store.0.clone(); + let fetcher = |id| ready(ev_map.get(&id).cloned()); + + let exists = |id: ::Id| ready(ev_map.get(&*id).is_some()); + + let state_sets = [state_at_bob, state_at_charlie]; + let auth_chain: Vec<_> = state_sets + .iter() + .map(|map| { + store + .auth_event_ids(room_id(), map.values().cloned().collect()) + .unwrap() + }) + .collect(); + + let resolved = match crate::resolve( + &RoomVersionId::V2, + &state_sets, + &auth_chain, + &fetcher, + &exists, + 1, + ) + .await + { + | Ok(state) => state, + | Err(e) => panic!("{e}"), + }; + + assert_eq!(expected, resolved); + } + + #[tokio::test] + async fn test_lexicographical_sort() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + + let graph = hashmap! { + event_id("l") => hashset![event_id("o")], + event_id("m") => hashset![event_id("n"), event_id("o")], + event_id("n") => hashset![event_id("o")], + event_id("o") => hashset![], // "o" has zero outgoing edges but 4 incoming edges + event_id("p") => hashset![event_id("o")], + }; + + let res = crate::lexicographical_topological_sort(&graph, &|_id| async { + Ok((int!(0), MilliSecondsSinceUnixEpoch(uint!(0)))) + }) + .await + .unwrap(); + + assert_eq!( + vec!["o", "l", "n", "m", "p"], + res.iter() + .map(ToString::to_string) + .map(|s| s.replace('$', "").replace(":foo", "")) + .collect::>() + ); + } + + #[tokio::test] + async fn ban_with_auth_chains() { + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + let ban = BAN_STATE_SET(); + + let edges = vec![vec!["END", "MB", "PA", "START"], vec!["END", "IME", "MB"]] + .into_iter() + .map(|list| list.into_iter().map(event_id).collect::>()) + .collect::>(); + + let expected_state_ids = vec!["PA", "MB"] + .into_iter() + .map(event_id) + .collect::>(); + + do_check(&ban.values().cloned().collect::>(), edges, expected_state_ids).await; + } + + #[tokio::test] + async fn ban_with_auth_chains2() { + use futures::future::ready; + + let _ = tracing::subscriber::set_default( + tracing_subscriber::fmt().with_test_writer().finish(), + ); + let init = INITIAL_EVENTS(); + let ban = BAN_STATE_SET(); + + let mut inner = init.clone(); + inner.extend(ban); + let store = TestStore(inner.clone()); + + let state_set_a = [ + inner.get(&event_id("CREATE")).unwrap(), + inner.get(&event_id("IJR")).unwrap(), + inner.get(&event_id("IMA")).unwrap(), + inner.get(&event_id("IMB")).unwrap(), + inner.get(&event_id("IMC")).unwrap(), + inner.get(&event_id("MB")).unwrap(), + inner.get(&event_id("PA")).unwrap(), + ] + .iter() + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.event_id.clone())) + .collect::>(); + + let state_set_b = [ + inner.get(&event_id("CREATE")).unwrap(), + inner.get(&event_id("IJR")).unwrap(), + inner.get(&event_id("IMA")).unwrap(), + inner.get(&event_id("IMB")).unwrap(), + inner.get(&event_id("IMC")).unwrap(), + inner.get(&event_id("IME")).unwrap(), + inner.get(&event_id("PA")).unwrap(), + ] + .iter() + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.event_id.clone())) + .collect::>(); + + let ev_map = &store.0; + let state_sets = [state_set_a, state_set_b]; + let auth_chain: Vec<_> = state_sets + .iter() + .map(|map| { + store + .auth_event_ids(room_id(), map.values().cloned().collect()) + .unwrap() + }) + .collect(); + + let fetcher = |id: ::Id| ready(ev_map.get(&id).cloned()); + let exists = |id: ::Id| ready(ev_map.get(&id).is_some()); + let resolved = match crate::resolve( + &RoomVersionId::V6, + &state_sets, + &auth_chain, + &fetcher, + &exists, + 1, + ) + .await + { + | Ok(state) => state, + | Err(e) => panic!("{e}"), + }; + + debug!( + resolved = ?resolved + .iter() + .map(|((ty, key), id)| format!("(({ty}{key:?}), {id})")) + .collect::>(), + "resolved state", + ); + + let expected = [ + "$CREATE:foo", + "$IJR:foo", + "$PA:foo", + "$IMA:foo", + "$IMB:foo", + "$IMC:foo", + "$MB:foo", + ]; + + for id in expected.iter().map(|i| event_id(i)) { + // make sure our resolved events are equal to the expected list + assert!(resolved.values().any(|eid| eid == &id) || init.contains_key(&id), "{id}"); + } + assert_eq!(expected.len(), resolved.len()); + } + + #[tokio::test] + async fn join_rule_with_auth_chain() { + let join_rule = JOIN_RULE(); + + let edges = vec![vec!["END", "JR", "START"], vec!["END", "IMZ", "START"]] + .into_iter() + .map(|list| list.into_iter().map(event_id).collect::>()) + .collect::>(); + + let expected_state_ids = vec!["JR"].into_iter().map(event_id).collect::>(); + + do_check(&join_rule.values().cloned().collect::>(), edges, expected_state_ids) + .await; + } + + #[allow(non_snake_case)] + fn BAN_STATE_SET() -> HashMap> { + vec![ + to_pdu_event( + "PA", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + &["CREATE", "IMA", "IPOWER"], // auth_events + &["START"], // prev_events + ), + to_pdu_event( + "PB", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + &["CREATE", "IMA", "IPOWER"], + &["END"], + ), + to_pdu_event( + "MB", + alice(), + TimelineEventType::RoomMember, + Some(ella().as_str()), + member_content_ban(), + &["CREATE", "IMA", "PB"], + &["PA"], + ), + to_pdu_event( + "IME", + ella(), + TimelineEventType::RoomMember, + Some(ella().as_str()), + member_content_join(), + &["CREATE", "IJR", "PA"], + &["MB"], + ), + ] + .into_iter() + .map(|ev| (ev.event_id.clone(), ev)) + .collect() + } + + #[allow(non_snake_case)] + fn JOIN_RULE() -> HashMap> { + vec![ + to_pdu_event( + "JR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&json!({ "join_rule": "invite" })).unwrap(), + &["CREATE", "IMA", "IPOWER"], + &["START"], + ), + to_pdu_event( + "IMZ", + zara(), + TimelineEventType::RoomPowerLevels, + Some(zara().as_str()), + member_content_join(), + &["CREATE", "JR", "IPOWER"], + &["START"], + ), + ] + .into_iter() + .map(|ev| (ev.event_id.clone(), ev)) + .collect() + } + + macro_rules! state_set { + ($($kind:expr => $key:expr => $id:expr),* $(,)?) => {{ + #[allow(unused_mut)] + let mut x = StateMap::new(); + $( + x.insert(($kind, $key.to_owned()), $id); + )* + x + }}; + } + + #[test] + fn separate_unique_conflicted() { + let (unconflicted, conflicted) = super::separate( + [ + state_set![StateEventType::RoomMember => "@a:hs1" => 0], + state_set![StateEventType::RoomMember => "@b:hs1" => 1], + state_set![StateEventType::RoomMember => "@c:hs1" => 2], + ] + .iter(), + ); + + assert_eq!(unconflicted, StateMap::new()); + assert_eq!(conflicted, state_set![ + StateEventType::RoomMember => "@a:hs1" => vec![0], + StateEventType::RoomMember => "@b:hs1" => vec![1], + StateEventType::RoomMember => "@c:hs1" => vec![2], + ],); + } + + #[test] + fn separate_conflicted() { + let (unconflicted, mut conflicted) = super::separate( + [ + state_set![StateEventType::RoomMember => "@a:hs1" => 0], + state_set![StateEventType::RoomMember => "@a:hs1" => 1], + state_set![StateEventType::RoomMember => "@a:hs1" => 2], + ] + .iter(), + ); + + // HashMap iteration order is random, so sort this before asserting on it + for v in conflicted.values_mut() { + v.sort_unstable(); + } + + assert_eq!(unconflicted, StateMap::new()); + assert_eq!(conflicted, state_set![ + StateEventType::RoomMember => "@a:hs1" => vec![0, 1, 2], + ],); + } + + #[test] + fn separate_unconflicted() { + let (unconflicted, conflicted) = super::separate( + [ + state_set![StateEventType::RoomMember => "@a:hs1" => 0], + state_set![StateEventType::RoomMember => "@a:hs1" => 0], + state_set![StateEventType::RoomMember => "@a:hs1" => 0], + ] + .iter(), + ); + + assert_eq!(unconflicted, state_set![ + StateEventType::RoomMember => "@a:hs1" => 0, + ],); + assert_eq!(conflicted, StateMap::new()); + } + + #[test] + fn separate_mixed() { + let (unconflicted, conflicted) = super::separate( + [ + state_set![StateEventType::RoomMember => "@a:hs1" => 0], + state_set![ + StateEventType::RoomMember => "@a:hs1" => 0, + StateEventType::RoomMember => "@b:hs1" => 1, + ], + state_set![ + StateEventType::RoomMember => "@a:hs1" => 0, + StateEventType::RoomMember => "@c:hs1" => 2, + ], + ] + .iter(), + ); + + assert_eq!(unconflicted, state_set![ + StateEventType::RoomMember => "@a:hs1" => 0, + ],); + assert_eq!(conflicted, state_set![ + StateEventType::RoomMember => "@b:hs1" => vec![1], + StateEventType::RoomMember => "@c:hs1" => vec![2], + ],); + } +} diff --git a/src/core/state_res/outcomes.txt b/src/core/state_res/outcomes.txt new file mode 100644 index 00000000..0fa1c734 --- /dev/null +++ b/src/core/state_res/outcomes.txt @@ -0,0 +1,104 @@ +11/29/2020 BRANCH: timo-spec-comp REV: d2a85669cc6056679ce6ca0fde4658a879ad2b08 +lexicographical topological sort + time: [1.7123 us 1.7157 us 1.7199 us] + change: [-1.7584% -1.5433% -1.3205%] (p = 0.00 < 0.05) + Performance has improved. +Found 8 outliers among 100 measurements (8.00%) + 2 (2.00%) low mild + 5 (5.00%) high mild + 1 (1.00%) high severe + +resolve state of 5 events one fork + time: [10.981 us 10.998 us 11.020 us] +Found 3 outliers among 100 measurements (3.00%) + 3 (3.00%) high mild + +resolve state of 10 events 3 conflicting + time: [26.858 us 26.946 us 27.037 us] + +11/29/2020 BRANCH: event-trait REV: f0eb1310efd49d722979f57f20bd1ac3592b0479 +lexicographical topological sort + time: [1.7686 us 1.7738 us 1.7810 us] + change: [-3.2752% -2.4634% -1.7635%] (p = 0.00 < 0.05) + Performance has improved. +Found 1 outliers among 100 measurements (1.00%) + 1 (1.00%) high severe + +resolve state of 5 events one fork + time: [10.643 us 10.656 us 10.669 us] + change: [-4.9990% -3.8078% -2.8319%] (p = 0.00 < 0.05) + Performance has improved. +Found 1 outliers among 100 measurements (1.00%) + 1 (1.00%) high severe + +resolve state of 10 events 3 conflicting + time: [29.149 us 29.252 us 29.375 us] + change: [-0.8433% -0.3270% +0.2656%] (p = 0.25 > 0.05) + No change in performance detected. +Found 1 outliers among 100 measurements (1.00%) + 1 (1.00%) high mild + +4/26/2020 BRANCH: fix-test-serde REV: +lexicographical topological sort + time: [1.6793 us 1.6823 us 1.6857 us] +Found 9 outliers among 100 measurements (9.00%) + 1 (1.00%) low mild + 4 (4.00%) high mild + 4 (4.00%) high severe + +resolve state of 5 events one fork + time: [9.9993 us 10.062 us 10.159 us] +Found 9 outliers among 100 measurements (9.00%) + 7 (7.00%) high mild + 2 (2.00%) high severe + +resolve state of 10 events 3 conflicting + time: [26.004 us 26.092 us 26.195 us] +Found 16 outliers among 100 measurements (16.00%) + 11 (11.00%) high mild + 5 (5.00%) high severe + +6/30/2021 BRANCH: state-closure REV: 174c3e2a72232ad75b3fb14b3551f5f746f4fe84 +lexicographical topological sort + time: [1.5496 us 1.5536 us 1.5586 us] +Found 9 outliers among 100 measurements (9.00%) + 1 (1.00%) low mild + 1 (1.00%) high mild + 7 (7.00%) high severe + +resolve state of 5 events one fork + time: [10.319 us 10.333 us 10.347 us] +Found 2 outliers among 100 measurements (2.00%) + 2 (2.00%) high severe + +resolve state of 10 events 3 conflicting + time: [25.770 us 25.805 us 25.839 us] +Found 7 outliers among 100 measurements (7.00%) + 5 (5.00%) high mild + 2 (2.00%) high severe + +7/20/2021 BRANCH stateres-result REV: +This marks the switch to HashSet/Map +lexicographical topological sort + time: [1.8122 us 1.8177 us 1.8233 us] + change: [+15.205% +15.919% +16.502%] (p = 0.00 < 0.05) + Performance has regressed. +Found 7 outliers among 100 measurements (7.00%) + 5 (5.00%) high mild + 2 (2.00%) high severe + +resolve state of 5 events one fork + time: [11.966 us 12.010 us 12.059 us] + change: [+16.089% +16.730% +17.469%] (p = 0.00 < 0.05) + Performance has regressed. +Found 7 outliers among 100 measurements (7.00%) + 3 (3.00%) high mild + 4 (4.00%) high severe + +resolve state of 10 events 3 conflicting + time: [29.092 us 29.201 us 29.311 us] + change: [+12.447% +12.847% +13.280%] (p = 0.00 < 0.05) + Performance has regressed. +Found 9 outliers among 100 measurements (9.00%) + 6 (6.00%) high mild + 3 (3.00%) high severe diff --git a/src/core/state_res/power_levels.rs b/src/core/state_res/power_levels.rs new file mode 100644 index 00000000..e1768574 --- /dev/null +++ b/src/core/state_res/power_levels.rs @@ -0,0 +1,256 @@ +use std::collections::BTreeMap; + +use ruma::{ + events::{room::power_levels::RoomPowerLevelsEventContent, TimelineEventType}, + power_levels::{default_power_level, NotificationPowerLevels}, + serde::{ + deserialize_v1_powerlevel, vec_deserialize_int_powerlevel_values, + vec_deserialize_v1_powerlevel_values, + }, + Int, OwnedUserId, UserId, +}; +use serde::Deserialize; +use serde_json::{from_str as from_json_str, Error}; +use tracing::error; + +use super::{Result, RoomVersion}; + +#[derive(Deserialize)] +struct IntRoomPowerLevelsEventContent { + #[serde(default = "default_power_level")] + ban: Int, + + #[serde(default)] + events: BTreeMap, + + #[serde(default)] + events_default: Int, + + #[serde(default)] + invite: Int, + + #[serde(default = "default_power_level")] + kick: Int, + + #[serde(default = "default_power_level")] + redact: Int, + + #[serde(default = "default_power_level")] + state_default: Int, + + #[serde(default)] + users: BTreeMap, + + #[serde(default)] + users_default: Int, + + #[serde(default)] + notifications: IntNotificationPowerLevels, +} + +impl From for RoomPowerLevelsEventContent { + fn from(int_pl: IntRoomPowerLevelsEventContent) -> Self { + let IntRoomPowerLevelsEventContent { + ban, + events, + events_default, + invite, + kick, + redact, + state_default, + users, + users_default, + notifications, + } = int_pl; + + let mut pl = Self::new(); + pl.ban = ban; + pl.events = events; + pl.events_default = events_default; + pl.invite = invite; + pl.kick = kick; + pl.redact = redact; + pl.state_default = state_default; + pl.users = users; + pl.users_default = users_default; + pl.notifications = notifications.into(); + + pl + } +} + +#[derive(Deserialize)] +struct IntNotificationPowerLevels { + #[serde(default = "default_power_level")] + room: Int, +} + +impl Default for IntNotificationPowerLevels { + fn default() -> Self { Self { room: default_power_level() } } +} + +impl From for NotificationPowerLevels { + fn from(int_notif: IntNotificationPowerLevels) -> Self { + let mut notif = Self::new(); + notif.room = int_notif.room; + + notif + } +} + +#[inline] +pub(crate) fn deserialize_power_levels( + content: &str, + room_version: &RoomVersion, +) -> Option { + if room_version.integer_power_levels { + deserialize_integer_power_levels(content) + } else { + deserialize_legacy_power_levels(content) + } +} + +fn deserialize_integer_power_levels(content: &str) -> Option { + match from_json_str::(content) { + | Ok(content) => Some(content.into()), + | Err(_) => { + error!("m.room.power_levels event is not valid with integer values"); + None + }, + } +} + +fn deserialize_legacy_power_levels(content: &str) -> Option { + match from_json_str(content) { + | Ok(content) => Some(content), + | Err(_) => { + error!( + "m.room.power_levels event is not valid with integer or string integer values" + ); + None + }, + } +} + +#[derive(Deserialize)] +pub(crate) struct PowerLevelsContentFields { + #[serde(default, deserialize_with = "vec_deserialize_v1_powerlevel_values")] + pub(crate) users: Vec<(OwnedUserId, Int)>, + + #[serde(default, deserialize_with = "deserialize_v1_powerlevel")] + pub(crate) users_default: Int, +} + +impl PowerLevelsContentFields { + pub(crate) fn get_user_power(&self, user_id: &UserId) -> Option<&Int> { + let comparator = |item: &(OwnedUserId, Int)| { + let item: &UserId = &item.0; + item.cmp(user_id) + }; + + self.users + .binary_search_by(comparator) + .ok() + .and_then(|idx| self.users.get(idx).map(|item| &item.1)) + } +} + +#[derive(Deserialize)] +struct IntPowerLevelsContentFields { + #[serde(default, deserialize_with = "vec_deserialize_int_powerlevel_values")] + users: Vec<(OwnedUserId, Int)>, + + #[serde(default)] + users_default: Int, +} + +impl From for PowerLevelsContentFields { + fn from(pl: IntPowerLevelsContentFields) -> Self { + let IntPowerLevelsContentFields { users, users_default } = pl; + Self { users, users_default } + } +} + +#[inline] +pub(crate) fn deserialize_power_levels_content_fields( + content: &str, + room_version: &RoomVersion, +) -> Result { + if room_version.integer_power_levels { + deserialize_integer_power_levels_content_fields(content) + } else { + deserialize_legacy_power_levels_content_fields(content) + } +} + +fn deserialize_integer_power_levels_content_fields( + content: &str, +) -> Result { + from_json_str::(content).map(Into::into) +} + +fn deserialize_legacy_power_levels_content_fields( + content: &str, +) -> Result { + from_json_str(content) +} + +#[derive(Deserialize)] +pub(crate) struct PowerLevelsContentInvite { + #[serde(default, deserialize_with = "deserialize_v1_powerlevel")] + pub(crate) invite: Int, +} + +#[derive(Deserialize)] +struct IntPowerLevelsContentInvite { + #[serde(default)] + invite: Int, +} + +impl From for PowerLevelsContentInvite { + fn from(pl: IntPowerLevelsContentInvite) -> Self { + let IntPowerLevelsContentInvite { invite } = pl; + Self { invite } + } +} + +pub(crate) fn deserialize_power_levels_content_invite( + content: &str, + room_version: &RoomVersion, +) -> Result { + if room_version.integer_power_levels { + from_json_str::(content).map(Into::into) + } else { + from_json_str(content) + } +} + +#[derive(Deserialize)] +pub(crate) struct PowerLevelsContentRedact { + #[serde(default = "default_power_level", deserialize_with = "deserialize_v1_powerlevel")] + pub(crate) redact: Int, +} + +#[derive(Deserialize)] +pub(crate) struct IntPowerLevelsContentRedact { + #[serde(default = "default_power_level")] + redact: Int, +} + +impl From for PowerLevelsContentRedact { + fn from(pl: IntPowerLevelsContentRedact) -> Self { + let IntPowerLevelsContentRedact { redact } = pl; + Self { redact } + } +} + +pub(crate) fn deserialize_power_levels_content_redact( + content: &str, + room_version: &RoomVersion, +) -> Result { + if room_version.integer_power_levels { + from_json_str::(content).map(Into::into) + } else { + from_json_str(content) + } +} diff --git a/src/core/state_res/room_version.rs b/src/core/state_res/room_version.rs new file mode 100644 index 00000000..e1b0afe1 --- /dev/null +++ b/src/core/state_res/room_version.rs @@ -0,0 +1,149 @@ +use ruma::RoomVersionId; + +use super::{Error, Result}; + +#[derive(Debug)] +#[allow(clippy::exhaustive_enums)] +pub enum RoomDisposition { + /// A room version that has a stable specification. + Stable, + /// A room version that is not yet fully specified. + Unstable, +} + +#[derive(Debug)] +#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)] +pub enum EventFormatVersion { + /// $id:server event id format + V1, + /// MSC1659-style $hash event id format: introduced for room v3 + V2, + /// MSC1884-style $hash format: introduced for room v4 + V3, +} + +#[derive(Debug)] +#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)] +pub enum StateResolutionVersion { + /// State resolution for rooms at version 1. + V1, + /// State resolution for room at version 2 or later. + V2, +} + +#[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)] +pub struct RoomVersion { + /// The stability of this room. + pub disposition: RoomDisposition, + /// The format of the EventId. + pub event_format: EventFormatVersion, + /// Which state resolution algorithm is used. + pub state_res: StateResolutionVersion, + // FIXME: not sure what this one means? + pub enforce_key_validity: bool, + + /// `m.room.aliases` had special auth rules and redaction rules + /// before room version 6. + /// + /// before MSC2261/MSC2432, + pub special_case_aliases_auth: bool, + /// Strictly enforce canonical json, do not allow: + /// * Integers outside the range of [-2 ^ 53 + 1, 2 ^ 53 - 1] + /// * Floats + /// * NaN, Infinity, -Infinity + pub strict_canonicaljson: bool, + /// Verify notifications key while checking m.room.power_levels. + /// + /// bool: MSC2209: Check 'notifications' + pub limit_notifications_power_levels: bool, + /// Extra rules when verifying redaction events. + pub extra_redaction_checks: bool, + /// Allow knocking in event authentication. + /// + /// See [room v7 specification](https://spec.matrix.org/latest/rooms/v7/) for more information. + pub allow_knocking: bool, + /// Adds support for the restricted join rule. + /// + /// See: [MSC3289](https://github.com/matrix-org/matrix-spec-proposals/pull/3289) for more information. + pub restricted_join_rules: bool, + /// Adds support for the knock_restricted join rule. + /// + /// See: [MSC3787](https://github.com/matrix-org/matrix-spec-proposals/pull/3787) for more information. + pub knock_restricted_join_rule: bool, + /// Enforces integer power levels. + /// + /// See: [MSC3667](https://github.com/matrix-org/matrix-spec-proposals/pull/3667) for more information. + pub integer_power_levels: bool, + /// Determine the room creator using the `m.room.create` event's `sender`, + /// instead of the event content's `creator` field. + /// + /// See: [MSC2175](https://github.com/matrix-org/matrix-spec-proposals/pull/2175) for more information. + pub use_room_create_sender: bool, +} + +impl RoomVersion { + pub const V1: Self = Self { + disposition: RoomDisposition::Stable, + event_format: EventFormatVersion::V1, + state_res: StateResolutionVersion::V1, + enforce_key_validity: false, + special_case_aliases_auth: true, + strict_canonicaljson: false, + limit_notifications_power_levels: false, + extra_redaction_checks: true, + allow_knocking: false, + restricted_join_rules: false, + knock_restricted_join_rule: false, + integer_power_levels: false, + use_room_create_sender: false, + }; + pub const V10: Self = Self { + knock_restricted_join_rule: true, + integer_power_levels: true, + ..Self::V9 + }; + pub const V11: Self = Self { + use_room_create_sender: true, + ..Self::V10 + }; + pub const V2: Self = Self { + state_res: StateResolutionVersion::V2, + ..Self::V1 + }; + pub const V3: Self = Self { + event_format: EventFormatVersion::V2, + extra_redaction_checks: false, + ..Self::V2 + }; + pub const V4: Self = Self { + event_format: EventFormatVersion::V3, + ..Self::V3 + }; + pub const V5: Self = Self { enforce_key_validity: true, ..Self::V4 }; + pub const V6: Self = Self { + special_case_aliases_auth: false, + strict_canonicaljson: true, + limit_notifications_power_levels: true, + ..Self::V5 + }; + pub const V7: Self = Self { allow_knocking: true, ..Self::V6 }; + pub const V8: Self = Self { restricted_join_rules: true, ..Self::V7 }; + pub const V9: Self = Self::V8; + + pub fn new(version: &RoomVersionId) -> Result { + Ok(match version { + | RoomVersionId::V1 => Self::V1, + | RoomVersionId::V2 => Self::V2, + | RoomVersionId::V3 => Self::V3, + | RoomVersionId::V4 => Self::V4, + | RoomVersionId::V5 => Self::V5, + | RoomVersionId::V6 => Self::V6, + | RoomVersionId::V7 => Self::V7, + | RoomVersionId::V8 => Self::V8, + | RoomVersionId::V9 => Self::V9, + | RoomVersionId::V10 => Self::V10, + | RoomVersionId::V11 => Self::V11, + | ver => return Err(Error::Unsupported(format!("found version `{ver}`"))), + }) + } +} diff --git a/src/core/state_res/state_event.rs b/src/core/state_res/state_event.rs new file mode 100644 index 00000000..2c038cfe --- /dev/null +++ b/src/core/state_res/state_event.rs @@ -0,0 +1,102 @@ +use std::{ + borrow::Borrow, + fmt::{Debug, Display}, + hash::Hash, + sync::Arc, +}; + +use ruma::{events::TimelineEventType, EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId}; +use serde_json::value::RawValue as RawJsonValue; + +/// Abstraction of a PDU so users can have their own PDU types. +pub trait Event { + type Id: Clone + Debug + Display + Eq + Ord + Hash + Send + Borrow; + + /// The `EventId` of this event. + fn event_id(&self) -> &Self::Id; + + /// The `RoomId` of this event. + fn room_id(&self) -> &RoomId; + + /// The `UserId` of this event. + fn sender(&self) -> &UserId; + + /// The time of creation on the originating server. + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch; + + /// The event type. + fn event_type(&self) -> &TimelineEventType; + + /// The event's content. + fn content(&self) -> &RawJsonValue; + + /// The state key for this event. + fn state_key(&self) -> Option<&str>; + + /// The events before this event. + // Requires GATs to avoid boxing (and TAIT for making it convenient). + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_; + + /// All the authenticating events for this event. + // Requires GATs to avoid boxing (and TAIT for making it convenient). + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_; + + /// If this event is a redaction event this is the event it redacts. + fn redacts(&self) -> Option<&Self::Id>; +} + +impl Event for &T { + type Id = T::Id; + + fn event_id(&self) -> &Self::Id { (*self).event_id() } + + fn room_id(&self) -> &RoomId { (*self).room_id() } + + fn sender(&self) -> &UserId { (*self).sender() } + + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { (*self).origin_server_ts() } + + fn event_type(&self) -> &TimelineEventType { (*self).event_type() } + + fn content(&self) -> &RawJsonValue { (*self).content() } + + fn state_key(&self) -> Option<&str> { (*self).state_key() } + + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + (*self).prev_events() + } + + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { + (*self).auth_events() + } + + fn redacts(&self) -> Option<&Self::Id> { (*self).redacts() } +} + +impl Event for Arc { + type Id = T::Id; + + fn event_id(&self) -> &Self::Id { (**self).event_id() } + + fn room_id(&self) -> &RoomId { (**self).room_id() } + + fn sender(&self) -> &UserId { (**self).sender() } + + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { (**self).origin_server_ts() } + + fn event_type(&self) -> &TimelineEventType { (**self).event_type() } + + fn content(&self) -> &RawJsonValue { (**self).content() } + + fn state_key(&self) -> Option<&str> { (**self).state_key() } + + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + (**self).prev_events() + } + + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { + (**self).auth_events() + } + + fn redacts(&self) -> Option<&Self::Id> { (**self).redacts() } +} diff --git a/src/core/state_res/state_res_bench.rs b/src/core/state_res/state_res_bench.rs new file mode 100644 index 00000000..a2bd2c23 --- /dev/null +++ b/src/core/state_res/state_res_bench.rs @@ -0,0 +1,648 @@ +// Because of criterion `cargo bench` works, +// but if you use `cargo bench -- --save-baseline ` +// or pass any other args to it, it fails with the error +// `cargo bench unknown option --save-baseline`. +// To pass args to criterion, use this form +// `cargo bench --bench -- --save-baseline `. + +#![allow(clippy::exhaustive_structs)] + +use std::{ + borrow::Borrow, + collections::{HashMap, HashSet}, + sync::{ + atomic::{AtomicU64, Ordering::SeqCst}, + Arc, + }, +}; + +use criterion::{criterion_group, criterion_main, Criterion}; +use event::PduEvent; +use futures::{future, future::ready}; +use ruma::{int, uint}; +use maplit::{btreemap, hashmap, hashset}; +use ruma::{ + room_id, user_id, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, + Signatures, UserId, +}; +use ruma::events::{ + pdu::{EventHash, Pdu, RoomV3Pdu}, + room::{ + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + }, + StateEventType, TimelineEventType, +}; +use conduwuit::state_res::{self as state_res, Error, Event, Result, StateMap}; +use serde_json::{ + json, + value::{to_raw_value as to_raw_json_value, RawValue as RawJsonValue}, +}; + +static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); + +fn lexico_topo_sort(c: &mut Criterion) { + c.bench_function("lexicographical topological sort", |b| { + let graph = hashmap! { + event_id("l") => hashset![event_id("o")], + event_id("m") => hashset![event_id("n"), event_id("o")], + event_id("n") => hashset![event_id("o")], + event_id("o") => hashset![], // "o" has zero outgoing edges but 4 incoming edges + event_id("p") => hashset![event_id("o")], + }; + b.iter(|| { + let _ = state_res::lexicographical_topological_sort(&graph, &|_| { + future::ok((int!(0), MilliSecondsSinceUnixEpoch(uint!(0)))) + }); + }); + }); +} + +fn resolution_shallow_auth_chain(c: &mut Criterion) { + c.bench_function("resolve state of 5 events one fork", |b| { + let mut store = TestStore(hashmap! {}); + + // build up the DAG + let (state_at_bob, state_at_charlie, _) = store.set_up(); + + b.iter(|| async { + let ev_map = store.0.clone(); + let state_sets = [&state_at_bob, &state_at_charlie]; + let fetch = |id: OwnedEventId| ready(ev_map.get(&id).map(Arc::clone)); + let exists = |id: OwnedEventId| ready(ev_map.get(&id).is_some()); + let auth_chain_sets = state_sets + .iter() + .map(|map| { + store.auth_event_ids(room_id(), map.values().cloned().collect()).unwrap() + }) + .collect(); + + let _ = match state_res::resolve( + &RoomVersionId::V6, + state_sets.into_iter(), + &auth_chain_sets, + &fetch, + &exists, + ) + .await + { + Ok(state) => state, + Err(e) => panic!("{e}"), + }; + }); + }); +} + +fn resolve_deeper_event_set(c: &mut Criterion) { + c.bench_function("resolve state of 10 events 3 conflicting", |b| { + let mut inner = INITIAL_EVENTS(); + let ban = BAN_STATE_SET(); + + inner.extend(ban); + let store = TestStore(inner.clone()); + + let state_set_a = [ + inner.get(&event_id("CREATE")).unwrap(), + inner.get(&event_id("IJR")).unwrap(), + inner.get(&event_id("IMA")).unwrap(), + inner.get(&event_id("IMB")).unwrap(), + inner.get(&event_id("IMC")).unwrap(), + inner.get(&event_id("MB")).unwrap(), + inner.get(&event_id("PA")).unwrap(), + ] + .iter() + .map(|ev| { + (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.event_id().to_owned()) + }) + .collect::>(); + + let state_set_b = [ + inner.get(&event_id("CREATE")).unwrap(), + inner.get(&event_id("IJR")).unwrap(), + inner.get(&event_id("IMA")).unwrap(), + inner.get(&event_id("IMB")).unwrap(), + inner.get(&event_id("IMC")).unwrap(), + inner.get(&event_id("IME")).unwrap(), + inner.get(&event_id("PA")).unwrap(), + ] + .iter() + .map(|ev| { + (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.event_id().to_owned()) + }) + .collect::>(); + + b.iter(|| async { + let state_sets = [&state_set_a, &state_set_b]; + let auth_chain_sets = state_sets + .iter() + .map(|map| { + store.auth_event_ids(room_id(), map.values().cloned().collect()).unwrap() + }) + .collect(); + + let fetch = |id: OwnedEventId| ready(inner.get(&id).map(Arc::clone)); + let exists = |id: OwnedEventId| ready(inner.get(&id).is_some()); + let _ = match state_res::resolve( + &RoomVersionId::V6, + state_sets.into_iter(), + &auth_chain_sets, + &fetch, + &exists, + ) + .await + { + Ok(state) => state, + Err(_) => panic!("resolution failed during benchmarking"), + }; + }); + }); +} + +criterion_group!( + benches, + lexico_topo_sort, + resolution_shallow_auth_chain, + resolve_deeper_event_set +); + +criterion_main!(benches); + +//*///////////////////////////////////////////////////////////////////// +// +// IMPLEMENTATION DETAILS AHEAD +// +/////////////////////////////////////////////////////////////////////*/ +struct TestStore(HashMap>); + +#[allow(unused)] +impl TestStore { + fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> Result> { + self.0 + .get(event_id) + .map(Arc::clone) + .ok_or_else(|| Error::NotFound(format!("{} not found", event_id))) + } + + /// Returns the events that correspond to the `event_ids` sorted in the same order. + fn get_events(&self, room_id: &RoomId, event_ids: &[OwnedEventId]) -> Result>> { + let mut events = vec![]; + for id in event_ids { + events.push(self.get_event(room_id, id)?); + } + Ok(events) + } + + /// Returns a Vec of the related auth events to the given `event`. + fn auth_event_ids(&self, room_id: &RoomId, event_ids: Vec) -> Result> { + let mut result = HashSet::new(); + let mut stack = event_ids; + + // DFS for auth event chain + while !stack.is_empty() { + let ev_id = stack.pop().unwrap(); + if result.contains(&ev_id) { + continue; + } + + result.insert(ev_id.clone()); + + let event = self.get_event(room_id, ev_id.borrow())?; + + stack.extend(event.auth_events().map(ToOwned::to_owned)); + } + + Ok(result) + } + + /// Returns a vector representing the difference in auth chains of the given `events`. + fn auth_chain_diff(&self, room_id: &RoomId, event_ids: Vec>) -> Result> { + let mut auth_chain_sets = vec![]; + for ids in event_ids { + // TODO state store `auth_event_ids` returns self in the event ids list + // when an event returns `auth_event_ids` self is not contained + let chain = self.auth_event_ids(room_id, ids)?.into_iter().collect::>(); + auth_chain_sets.push(chain); + } + + if let Some(first) = auth_chain_sets.first().cloned() { + let common = auth_chain_sets + .iter() + .skip(1) + .fold(first, |a, b| a.intersection(b).cloned().collect::>()); + + Ok(auth_chain_sets + .into_iter() + .flatten() + .filter(|id| !common.contains(id.borrow())) + .collect()) + } else { + Ok(vec![]) + } + } +} + +impl TestStore { + #[allow(clippy::type_complexity)] + fn set_up( + &mut self, + ) -> (StateMap, StateMap, StateMap) { + let create_event = to_pdu_event::<&EventId>( + "CREATE", + alice(), + TimelineEventType::RoomCreate, + Some(""), + to_raw_json_value(&json!({ "creator": alice() })).unwrap(), + &[], + &[], + ); + let cre = create_event.event_id().to_owned(); + self.0.insert(cre.clone(), Arc::clone(&create_event)); + + let alice_mem = to_pdu_event( + "IMA", + alice(), + TimelineEventType::RoomMember, + Some(alice().to_string().as_str()), + member_content_join(), + &[cre.clone()], + &[cre.clone()], + ); + self.0.insert(alice_mem.event_id().to_owned(), Arc::clone(&alice_mem)); + + let join_rules = to_pdu_event( + "IJR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Public)).unwrap(), + &[cre.clone(), alice_mem.event_id().to_owned()], + &[alice_mem.event_id().to_owned()], + ); + self.0.insert(join_rules.event_id().to_owned(), join_rules.clone()); + + // Bob and Charlie join at the same time, so there is a fork + // this will be represented in the state_sets when we resolve + let bob_mem = to_pdu_event( + "IMB", + bob(), + TimelineEventType::RoomMember, + Some(bob().to_string().as_str()), + member_content_join(), + &[cre.clone(), join_rules.event_id().to_owned()], + &[join_rules.event_id().to_owned()], + ); + self.0.insert(bob_mem.event_id().to_owned(), bob_mem.clone()); + + let charlie_mem = to_pdu_event( + "IMC", + charlie(), + TimelineEventType::RoomMember, + Some(charlie().to_string().as_str()), + member_content_join(), + &[cre, join_rules.event_id().to_owned()], + &[join_rules.event_id().to_owned()], + ); + self.0.insert(charlie_mem.event_id().to_owned(), charlie_mem.clone()); + + let state_at_bob = [&create_event, &alice_mem, &join_rules, &bob_mem] + .iter() + .map(|e| { + (e.event_type().with_state_key(e.state_key().unwrap()), e.event_id().to_owned()) + }) + .collect::>(); + + let state_at_charlie = [&create_event, &alice_mem, &join_rules, &charlie_mem] + .iter() + .map(|e| { + (e.event_type().with_state_key(e.state_key().unwrap()), e.event_id().to_owned()) + }) + .collect::>(); + + let expected = [&create_event, &alice_mem, &join_rules, &bob_mem, &charlie_mem] + .iter() + .map(|e| { + (e.event_type().with_state_key(e.state_key().unwrap()), e.event_id().to_owned()) + }) + .collect::>(); + + (state_at_bob, state_at_charlie, expected) + } +} + +fn event_id(id: &str) -> OwnedEventId { + if id.contains('$') { + return id.try_into().unwrap(); + } + format!("${}:foo", id).try_into().unwrap() +} + +fn alice() -> &'static UserId { + user_id!("@alice:foo") +} + +fn bob() -> &'static UserId { + user_id!("@bob:foo") +} + +fn charlie() -> &'static UserId { + user_id!("@charlie:foo") +} + +fn ella() -> &'static UserId { + user_id!("@ella:foo") +} + +fn room_id() -> &'static RoomId { + room_id!("!test:foo") +} + +fn member_content_ban() -> Box { + to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Ban)).unwrap() +} + +fn member_content_join() -> Box { + to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Join)).unwrap() +} + +fn to_pdu_event( + id: &str, + sender: &UserId, + ev_type: TimelineEventType, + state_key: Option<&str>, + content: Box, + auth_events: &[S], + prev_events: &[S], +) -> Arc +where + S: AsRef, +{ + // We don't care if the addition happens in order just that it is atomic + // (each event has its own value) + let ts = SERVER_TIMESTAMP.fetch_add(1, SeqCst); + let id = if id.contains('$') { id.to_owned() } else { format!("${}:foo", id) }; + let auth_events = auth_events.iter().map(AsRef::as_ref).map(event_id).collect::>(); + let prev_events = prev_events.iter().map(AsRef::as_ref).map(event_id).collect::>(); + + let state_key = state_key.map(ToOwned::to_owned); + Arc::new(PduEvent { + event_id: id.try_into().unwrap(), + rest: Pdu::RoomV3Pdu(RoomV3Pdu { + room_id: room_id().to_owned(), + sender: sender.to_owned(), + origin_server_ts: MilliSecondsSinceUnixEpoch(ts.try_into().unwrap()), + state_key, + kind: ev_type, + content, + redacts: None, + unsigned: btreemap! {}, + auth_events, + prev_events, + depth: uint!(0), + hashes: EventHash::new(String::new()), + signatures: Signatures::new(), + }), + }) +} + +// all graphs start with these input events +#[allow(non_snake_case)] +fn INITIAL_EVENTS() -> HashMap> { + vec![ + to_pdu_event::<&EventId>( + "CREATE", + alice(), + TimelineEventType::RoomCreate, + Some(""), + to_raw_json_value(&json!({ "creator": alice() })).unwrap(), + &[], + &[], + ), + to_pdu_event( + "IMA", + alice(), + TimelineEventType::RoomMember, + Some(alice().as_str()), + member_content_join(), + &["CREATE"], + &["CREATE"], + ), + to_pdu_event( + "IPOWER", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100 } })).unwrap(), + &["CREATE", "IMA"], + &["IMA"], + ), + to_pdu_event( + "IJR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Public)).unwrap(), + &["CREATE", "IMA", "IPOWER"], + &["IPOWER"], + ), + to_pdu_event( + "IMB", + bob(), + TimelineEventType::RoomMember, + Some(bob().to_string().as_str()), + member_content_join(), + &["CREATE", "IJR", "IPOWER"], + &["IJR"], + ), + to_pdu_event( + "IMC", + charlie(), + TimelineEventType::RoomMember, + Some(charlie().to_string().as_str()), + member_content_join(), + &["CREATE", "IJR", "IPOWER"], + &["IMB"], + ), + to_pdu_event::<&EventId>( + "START", + charlie(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + &[], + &[], + ), + to_pdu_event::<&EventId>( + "END", + charlie(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + &[], + &[], + ), + ] + .into_iter() + .map(|ev| (ev.event_id().to_owned(), ev)) + .collect() +} + +// all graphs start with these input events +#[allow(non_snake_case)] +fn BAN_STATE_SET() -> HashMap> { + vec![ + to_pdu_event( + "PA", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + &["CREATE", "IMA", "IPOWER"], // auth_events + &["START"], // prev_events + ), + to_pdu_event( + "PB", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + &["CREATE", "IMA", "IPOWER"], + &["END"], + ), + to_pdu_event( + "MB", + alice(), + TimelineEventType::RoomMember, + Some(ella().as_str()), + member_content_ban(), + &["CREATE", "IMA", "PB"], + &["PA"], + ), + to_pdu_event( + "IME", + ella(), + TimelineEventType::RoomMember, + Some(ella().as_str()), + member_content_join(), + &["CREATE", "IJR", "PA"], + &["MB"], + ), + ] + .into_iter() + .map(|ev| (ev.event_id().to_owned(), ev)) + .collect() +} + +/// Convenience trait for adding event type plus state key to state maps. +trait EventTypeExt { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, String); +} + +impl EventTypeExt for &TimelineEventType { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { + (self.to_string().into(), state_key.into()) + } +} + +mod event { + use ruma_common::{MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId}; + use ruma_events::{pdu::Pdu, TimelineEventType}; + use ruma_state_res::Event; + use serde::{Deserialize, Serialize}; + use serde_json::value::RawValue as RawJsonValue; + + impl Event for PduEvent { + type Id = OwnedEventId; + + fn event_id(&self) -> &Self::Id { + &self.event_id + } + + fn room_id(&self) -> &RoomId { + match &self.rest { + Pdu::RoomV1Pdu(ev) => &ev.room_id, + Pdu::RoomV3Pdu(ev) => &ev.room_id, + #[cfg(not(feature = "unstable-exhaustive-types"))] + _ => unreachable!("new PDU version"), + } + } + + fn sender(&self) -> &UserId { + match &self.rest { + Pdu::RoomV1Pdu(ev) => &ev.sender, + Pdu::RoomV3Pdu(ev) => &ev.sender, + #[cfg(not(feature = "unstable-exhaustive-types"))] + _ => unreachable!("new PDU version"), + } + } + + fn event_type(&self) -> &TimelineEventType { + match &self.rest { + Pdu::RoomV1Pdu(ev) => &ev.kind, + Pdu::RoomV3Pdu(ev) => &ev.kind, + #[cfg(not(feature = "unstable-exhaustive-types"))] + _ => unreachable!("new PDU version"), + } + } + + fn content(&self) -> &RawJsonValue { + match &self.rest { + Pdu::RoomV1Pdu(ev) => &ev.content, + Pdu::RoomV3Pdu(ev) => &ev.content, + #[cfg(not(feature = "unstable-exhaustive-types"))] + _ => unreachable!("new PDU version"), + } + } + + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { + match &self.rest { + Pdu::RoomV1Pdu(ev) => ev.origin_server_ts, + Pdu::RoomV3Pdu(ev) => ev.origin_server_ts, + #[cfg(not(feature = "unstable-exhaustive-types"))] + _ => unreachable!("new PDU version"), + } + } + + fn state_key(&self) -> Option<&str> { + match &self.rest { + Pdu::RoomV1Pdu(ev) => ev.state_key.as_deref(), + Pdu::RoomV3Pdu(ev) => ev.state_key.as_deref(), + #[cfg(not(feature = "unstable-exhaustive-types"))] + _ => unreachable!("new PDU version"), + } + } + + fn prev_events(&self) -> Box + Send + '_> { + match &self.rest { + Pdu::RoomV1Pdu(ev) => Box::new(ev.prev_events.iter().map(|(id, _)| id)), + Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter()), + #[cfg(not(feature = "unstable-exhaustive-types"))] + _ => unreachable!("new PDU version"), + } + } + + fn auth_events(&self) -> Box + Send + '_> { + match &self.rest { + Pdu::RoomV1Pdu(ev) => Box::new(ev.auth_events.iter().map(|(id, _)| id)), + Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter()), + #[cfg(not(feature = "unstable-exhaustive-types"))] + _ => unreachable!("new PDU version"), + } + } + + fn redacts(&self) -> Option<&Self::Id> { + match &self.rest { + Pdu::RoomV1Pdu(ev) => ev.redacts.as_ref(), + Pdu::RoomV3Pdu(ev) => ev.redacts.as_ref(), + #[cfg(not(feature = "unstable-exhaustive-types"))] + _ => unreachable!("new PDU version"), + } + } + } + + #[derive(Clone, Debug, Deserialize, Serialize)] + pub(crate) struct PduEvent { + pub(crate) event_id: OwnedEventId, + #[serde(flatten)] + pub(crate) rest: Pdu, + } +} diff --git a/src/core/state_res/test_utils.rs b/src/core/state_res/test_utils.rs new file mode 100644 index 00000000..7954b28d --- /dev/null +++ b/src/core/state_res/test_utils.rs @@ -0,0 +1,688 @@ +use std::{ + borrow::Borrow, + collections::{BTreeMap, HashMap, HashSet}, + sync::{ + atomic::{AtomicU64, Ordering::SeqCst}, + Arc, + }, +}; + +use futures_util::future::ready; +use js_int::{int, uint}; +use ruma_common::{ + event_id, room_id, user_id, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, + RoomVersionId, ServerSignatures, UserId, +}; +use ruma_events::{ + pdu::{EventHash, Pdu, RoomV3Pdu}, + room::{ + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + }, + TimelineEventType, +}; +use serde_json::{ + json, + value::{to_raw_value as to_raw_json_value, RawValue as RawJsonValue}, +}; +use tracing::info; + +pub(crate) use self::event::PduEvent; +use crate::{auth_types_for_event, Error, Event, EventTypeExt, Result, StateMap}; + +static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); + +pub(crate) async fn do_check( + events: &[Arc], + edges: Vec>, + expected_state_ids: Vec, +) { + // To activate logging use `RUST_LOG=debug cargo t` + + let init_events = INITIAL_EVENTS(); + + let mut store = TestStore( + init_events + .values() + .chain(events) + .map(|ev| (ev.event_id().to_owned(), ev.clone())) + .collect(), + ); + + // This will be lexi_topo_sorted for resolution + let mut graph = HashMap::new(); + // This is the same as in `resolve` event_id -> OriginalStateEvent + let mut fake_event_map = HashMap::new(); + + // Create the DB of events that led up to this point + // TODO maybe clean up some of these clones it is just tests but... + for ev in init_events.values().chain(events) { + graph.insert(ev.event_id().to_owned(), HashSet::new()); + fake_event_map.insert(ev.event_id().to_owned(), ev.clone()); + } + + for pair in INITIAL_EDGES().windows(2) { + if let [a, b] = &pair { + graph + .entry(a.to_owned()) + .or_insert_with(HashSet::new) + .insert(b.clone()); + } + } + + for edge_list in edges { + for pair in edge_list.windows(2) { + if let [a, b] = &pair { + graph + .entry(a.to_owned()) + .or_insert_with(HashSet::new) + .insert(b.clone()); + } + } + } + + // event_id -> PduEvent + let mut event_map: HashMap> = HashMap::new(); + // event_id -> StateMap + let mut state_at_event: HashMap> = HashMap::new(); + + // Resolve the current state and add it to the state_at_event map then continue + // on in "time" + for node in crate::lexicographical_topological_sort(&graph, &|_id| async { + Ok((int!(0), MilliSecondsSinceUnixEpoch(uint!(0)))) + }) + .await + .unwrap() + { + let fake_event = fake_event_map.get(&node).unwrap(); + let event_id = fake_event.event_id().to_owned(); + + let prev_events = graph.get(&node).unwrap(); + + let state_before: StateMap = if prev_events.is_empty() { + HashMap::new() + } else if prev_events.len() == 1 { + state_at_event + .get(prev_events.iter().next().unwrap()) + .unwrap() + .clone() + } else { + let state_sets = prev_events + .iter() + .filter_map(|k| state_at_event.get(k)) + .collect::>(); + + info!( + "{:#?}", + state_sets + .iter() + .map(|map| map + .iter() + .map(|((ty, key), id)| format!("(({ty}{key:?}), {id})")) + .collect::>()) + .collect::>() + ); + + let auth_chain_sets: Vec<_> = state_sets + .iter() + .map(|map| { + store + .auth_event_ids(room_id(), map.values().cloned().collect()) + .unwrap() + }) + .collect(); + + let event_map = &event_map; + let fetch = |id: ::Id| ready(event_map.get(&id).cloned()); + let exists = |id: ::Id| ready(event_map.get(&id).is_some()); + let resolved = crate::resolve( + &RoomVersionId::V6, + state_sets, + &auth_chain_sets, + &fetch, + &exists, + 1, + ) + .await; + + match resolved { + | Ok(state) => state, + | Err(e) => panic!("resolution for {node} failed: {e}"), + } + }; + + let mut state_after = state_before.clone(); + + let ty = fake_event.event_type(); + let key = fake_event.state_key().unwrap(); + state_after.insert(ty.with_state_key(key), event_id.to_owned()); + + let auth_types = auth_types_for_event( + fake_event.event_type(), + fake_event.sender(), + fake_event.state_key(), + fake_event.content(), + ) + .unwrap(); + + let mut auth_events = vec![]; + for key in auth_types { + if state_before.contains_key(&key) { + auth_events.push(state_before[&key].clone()); + } + } + + // TODO The event is just remade, adding the auth_events and prev_events here + // the `to_pdu_event` was split into `init` and the fn below, could be better + let e = fake_event; + let ev_id = e.event_id(); + let event = to_pdu_event( + e.event_id().as_str(), + e.sender(), + e.event_type().clone(), + e.state_key(), + e.content().to_owned(), + &auth_events, + &prev_events.iter().cloned().collect::>(), + ); + + // We have to update our store, an actual user of this lib would + // be giving us state from a DB. + store.0.insert(ev_id.to_owned(), event.clone()); + + state_at_event.insert(node, state_after); + event_map.insert(event_id.to_owned(), Arc::clone(store.0.get(ev_id).unwrap())); + } + + let mut expected_state = StateMap::new(); + for node in expected_state_ids { + let ev = event_map.get(&node).unwrap_or_else(|| { + panic!( + "{node} not found in {:?}", + event_map + .keys() + .map(ToString::to_string) + .collect::>() + ) + }); + + let key = ev.event_type().with_state_key(ev.state_key().unwrap()); + + expected_state.insert(key, node); + } + + let start_state = state_at_event.get(event_id!("$START:foo")).unwrap(); + + let end_state = state_at_event + .get(event_id!("$END:foo")) + .unwrap() + .iter() + .filter(|(k, v)| { + expected_state.contains_key(k) + || start_state.get(k) != Some(*v) + // Filter out the dummy messages events. + // These act as points in time where there should be a known state to + // test against. + && **k != ("m.room.message".into(), "dummy".to_owned()) + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>(); + + assert_eq!(expected_state, end_state); +} + +#[allow(clippy::exhaustive_structs)] +pub(crate) struct TestStore(pub(crate) HashMap>); + +impl TestStore { + pub(crate) fn get_event(&self, _: &RoomId, event_id: &EventId) -> Result> { + self.0 + .get(event_id) + .cloned() + .ok_or_else(|| Error::NotFound(format!("{event_id} not found"))) + } + + /// Returns a Vec of the related auth events to the given `event`. + pub(crate) fn auth_event_ids( + &self, + room_id: &RoomId, + event_ids: Vec, + ) -> Result> { + let mut result = HashSet::new(); + let mut stack = event_ids; + + // DFS for auth event chain + while let Some(ev_id) = stack.pop() { + if result.contains(&ev_id) { + continue; + } + + result.insert(ev_id.clone()); + + let event = self.get_event(room_id, ev_id.borrow())?; + + stack.extend(event.auth_events().map(ToOwned::to_owned)); + } + + Ok(result) + } +} + +// A StateStore implementation for testing +#[allow(clippy::type_complexity)] +impl TestStore { + pub(crate) fn set_up( + &mut self, + ) -> (StateMap, StateMap, StateMap) { + let create_event = to_pdu_event::<&EventId>( + "CREATE", + alice(), + TimelineEventType::RoomCreate, + Some(""), + to_raw_json_value(&json!({ "creator": alice() })).unwrap(), + &[], + &[], + ); + let cre = create_event.event_id().to_owned(); + self.0.insert(cre.clone(), Arc::clone(&create_event)); + + let alice_mem = to_pdu_event( + "IMA", + alice(), + TimelineEventType::RoomMember, + Some(alice().as_str()), + member_content_join(), + &[cre.clone()], + &[cre.clone()], + ); + self.0 + .insert(alice_mem.event_id().to_owned(), Arc::clone(&alice_mem)); + + let join_rules = to_pdu_event( + "IJR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Public)).unwrap(), + &[cre.clone(), alice_mem.event_id().to_owned()], + &[alice_mem.event_id().to_owned()], + ); + self.0 + .insert(join_rules.event_id().to_owned(), join_rules.clone()); + + // Bob and Charlie join at the same time, so there is a fork + // this will be represented in the state_sets when we resolve + let bob_mem = to_pdu_event( + "IMB", + bob(), + TimelineEventType::RoomMember, + Some(bob().as_str()), + member_content_join(), + &[cre.clone(), join_rules.event_id().to_owned()], + &[join_rules.event_id().to_owned()], + ); + self.0 + .insert(bob_mem.event_id().to_owned(), bob_mem.clone()); + + let charlie_mem = to_pdu_event( + "IMC", + charlie(), + TimelineEventType::RoomMember, + Some(charlie().as_str()), + member_content_join(), + &[cre, join_rules.event_id().to_owned()], + &[join_rules.event_id().to_owned()], + ); + self.0 + .insert(charlie_mem.event_id().to_owned(), charlie_mem.clone()); + + let state_at_bob = [&create_event, &alice_mem, &join_rules, &bob_mem] + .iter() + .map(|e| { + (e.event_type().with_state_key(e.state_key().unwrap()), e.event_id().to_owned()) + }) + .collect::>(); + + let state_at_charlie = [&create_event, &alice_mem, &join_rules, &charlie_mem] + .iter() + .map(|e| { + (e.event_type().with_state_key(e.state_key().unwrap()), e.event_id().to_owned()) + }) + .collect::>(); + + let expected = [&create_event, &alice_mem, &join_rules, &bob_mem, &charlie_mem] + .iter() + .map(|e| { + (e.event_type().with_state_key(e.state_key().unwrap()), e.event_id().to_owned()) + }) + .collect::>(); + + (state_at_bob, state_at_charlie, expected) + } +} + +pub(crate) fn event_id(id: &str) -> OwnedEventId { + if id.contains('$') { + return id.try_into().unwrap(); + } + + format!("${id}:foo").try_into().unwrap() +} + +pub(crate) fn alice() -> &'static UserId { user_id!("@alice:foo") } + +pub(crate) fn bob() -> &'static UserId { user_id!("@bob:foo") } + +pub(crate) fn charlie() -> &'static UserId { user_id!("@charlie:foo") } + +pub(crate) fn ella() -> &'static UserId { user_id!("@ella:foo") } + +pub(crate) fn zara() -> &'static UserId { user_id!("@zara:foo") } + +pub(crate) fn room_id() -> &'static RoomId { room_id!("!test:foo") } + +pub(crate) fn member_content_ban() -> Box { + to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Ban)).unwrap() +} + +pub(crate) fn member_content_join() -> Box { + to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Join)).unwrap() +} + +pub(crate) fn to_init_pdu_event( + id: &str, + sender: &UserId, + ev_type: TimelineEventType, + state_key: Option<&str>, + content: Box, +) -> Arc { + let ts = SERVER_TIMESTAMP.fetch_add(1, SeqCst); + let id = if id.contains('$') { + id.to_owned() + } else { + format!("${id}:foo") + }; + + let state_key = state_key.map(ToOwned::to_owned); + Arc::new(PduEvent { + event_id: id.try_into().unwrap(), + rest: Pdu::RoomV3Pdu(RoomV3Pdu { + room_id: room_id().to_owned(), + sender: sender.to_owned(), + origin_server_ts: MilliSecondsSinceUnixEpoch(ts.try_into().unwrap()), + state_key, + kind: ev_type, + content, + redacts: None, + unsigned: BTreeMap::new(), + auth_events: vec![], + prev_events: vec![], + depth: uint!(0), + hashes: EventHash::new("".to_owned()), + signatures: ServerSignatures::default(), + }), + }) +} + +pub(crate) fn to_pdu_event( + id: &str, + sender: &UserId, + ev_type: TimelineEventType, + state_key: Option<&str>, + content: Box, + auth_events: &[S], + prev_events: &[S], +) -> Arc +where + S: AsRef, +{ + let ts = SERVER_TIMESTAMP.fetch_add(1, SeqCst); + let id = if id.contains('$') { + id.to_owned() + } else { + format!("${id}:foo") + }; + let auth_events = auth_events + .iter() + .map(AsRef::as_ref) + .map(event_id) + .collect::>(); + let prev_events = prev_events + .iter() + .map(AsRef::as_ref) + .map(event_id) + .collect::>(); + + let state_key = state_key.map(ToOwned::to_owned); + Arc::new(PduEvent { + event_id: id.try_into().unwrap(), + rest: Pdu::RoomV3Pdu(RoomV3Pdu { + room_id: room_id().to_owned(), + sender: sender.to_owned(), + origin_server_ts: MilliSecondsSinceUnixEpoch(ts.try_into().unwrap()), + state_key, + kind: ev_type, + content, + redacts: None, + unsigned: BTreeMap::new(), + auth_events, + prev_events, + depth: uint!(0), + hashes: EventHash::new("".to_owned()), + signatures: ServerSignatures::default(), + }), + }) +} + +// all graphs start with these input events +#[allow(non_snake_case)] +pub(crate) fn INITIAL_EVENTS() -> HashMap> { + vec![ + to_pdu_event::<&EventId>( + "CREATE", + alice(), + TimelineEventType::RoomCreate, + Some(""), + to_raw_json_value(&json!({ "creator": alice() })).unwrap(), + &[], + &[], + ), + to_pdu_event( + "IMA", + alice(), + TimelineEventType::RoomMember, + Some(alice().as_str()), + member_content_join(), + &["CREATE"], + &["CREATE"], + ), + to_pdu_event( + "IPOWER", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100 } })).unwrap(), + &["CREATE", "IMA"], + &["IMA"], + ), + to_pdu_event( + "IJR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Public)).unwrap(), + &["CREATE", "IMA", "IPOWER"], + &["IPOWER"], + ), + to_pdu_event( + "IMB", + bob(), + TimelineEventType::RoomMember, + Some(bob().as_str()), + member_content_join(), + &["CREATE", "IJR", "IPOWER"], + &["IJR"], + ), + to_pdu_event( + "IMC", + charlie(), + TimelineEventType::RoomMember, + Some(charlie().as_str()), + member_content_join(), + &["CREATE", "IJR", "IPOWER"], + &["IMB"], + ), + to_pdu_event::<&EventId>( + "START", + charlie(), + TimelineEventType::RoomMessage, + Some("dummy"), + to_raw_json_value(&json!({})).unwrap(), + &[], + &[], + ), + to_pdu_event::<&EventId>( + "END", + charlie(), + TimelineEventType::RoomMessage, + Some("dummy"), + to_raw_json_value(&json!({})).unwrap(), + &[], + &[], + ), + ] + .into_iter() + .map(|ev| (ev.event_id().to_owned(), ev)) + .collect() +} + +// all graphs start with these input events +#[allow(non_snake_case)] +pub(crate) fn INITIAL_EVENTS_CREATE_ROOM() -> HashMap> { + vec![to_pdu_event::<&EventId>( + "CREATE", + alice(), + TimelineEventType::RoomCreate, + Some(""), + to_raw_json_value(&json!({ "creator": alice() })).unwrap(), + &[], + &[], + )] + .into_iter() + .map(|ev| (ev.event_id().to_owned(), ev)) + .collect() +} + +#[allow(non_snake_case)] +pub(crate) fn INITIAL_EDGES() -> Vec { + vec!["START", "IMC", "IMB", "IJR", "IPOWER", "IMA", "CREATE"] + .into_iter() + .map(event_id) + .collect::>() +} + +pub(crate) mod event { + use ruma_common::{MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId}; + use ruma_events::{pdu::Pdu, TimelineEventType}; + use serde::{Deserialize, Serialize}; + use serde_json::value::RawValue as RawJsonValue; + + use crate::Event; + + impl Event for PduEvent { + type Id = OwnedEventId; + + fn event_id(&self) -> &Self::Id { &self.event_id } + + fn room_id(&self) -> &RoomId { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => &ev.room_id, + | Pdu::RoomV3Pdu(ev) => &ev.room_id, + #[allow(unreachable_patterns)] + | _ => unreachable!("new PDU version"), + } + } + + fn sender(&self) -> &UserId { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => &ev.sender, + | Pdu::RoomV3Pdu(ev) => &ev.sender, + #[allow(unreachable_patterns)] + | _ => unreachable!("new PDU version"), + } + } + + fn event_type(&self) -> &TimelineEventType { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => &ev.kind, + | Pdu::RoomV3Pdu(ev) => &ev.kind, + #[allow(unreachable_patterns)] + | _ => unreachable!("new PDU version"), + } + } + + fn content(&self) -> &RawJsonValue { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => &ev.content, + | Pdu::RoomV3Pdu(ev) => &ev.content, + #[allow(unreachable_patterns)] + | _ => unreachable!("new PDU version"), + } + } + + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => ev.origin_server_ts, + | Pdu::RoomV3Pdu(ev) => ev.origin_server_ts, + #[allow(unreachable_patterns)] + | _ => unreachable!("new PDU version"), + } + } + + fn state_key(&self) -> Option<&str> { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => ev.state_key.as_deref(), + | Pdu::RoomV3Pdu(ev) => ev.state_key.as_deref(), + #[allow(unreachable_patterns)] + | _ => unreachable!("new PDU version"), + } + } + + #[allow(refining_impl_trait)] + fn prev_events(&self) -> Box + Send + '_> { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => Box::new(ev.prev_events.iter().map(|(id, _)| id)), + | Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter()), + #[allow(unreachable_patterns)] + | _ => unreachable!("new PDU version"), + } + } + + #[allow(refining_impl_trait)] + fn auth_events(&self) -> Box + Send + '_> { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => Box::new(ev.auth_events.iter().map(|(id, _)| id)), + | Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter()), + #[allow(unreachable_patterns)] + | _ => unreachable!("new PDU version"), + } + } + + fn redacts(&self) -> Option<&Self::Id> { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => ev.redacts.as_ref(), + | Pdu::RoomV3Pdu(ev) => ev.redacts.as_ref(), + #[allow(unreachable_patterns)] + | _ => unreachable!("new PDU version"), + } + } + } + + #[derive(Clone, Debug, Deserialize, Serialize)] + #[allow(clippy::exhaustive_structs)] + pub(crate) struct PduEvent { + pub(crate) event_id: OwnedEventId, + #[serde(flatten)] + pub(crate) rest: Pdu, + } +} diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index aea70739..5a38f7fe 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -3,12 +3,15 @@ use std::{ sync::Arc, }; -use conduwuit::{debug_warn, err, implement, PduEvent, Result}; +use conduwuit::{ + debug_warn, err, implement, + state_res::{self}, + PduEvent, Result, +}; use futures::{future, FutureExt}; use ruma::{ - int, - state_res::{self}, - uint, CanonicalJsonValue, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName, UInt, + int, uint, CanonicalJsonValue, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName, + UInt, }; use super::check_room_id; diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index b7c38313..3cc15fc4 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -3,10 +3,12 @@ use std::{ sync::Arc, }; -use conduwuit::{debug, debug_info, err, implement, trace, warn, Err, Error, PduEvent, Result}; +use conduwuit::{ + debug, debug_info, err, implement, state_res, trace, warn, Err, Error, PduEvent, Result, +}; use futures::{future::ready, TryFutureExt}; use ruma::{ - api::client::error::ErrorKind, events::StateEventType, state_res, CanonicalJsonObject, + api::client::error::ErrorKind, events::StateEventType, CanonicalJsonObject, CanonicalJsonValue, EventId, RoomId, ServerName, }; diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 8bcbc48b..5960c734 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -19,12 +19,12 @@ use std::{ use conduwuit::{ utils::{MutexMap, TryFutureExtExt}, - Err, PduEvent, Result, Server, + Err, PduEvent, Result, RoomVersion, Server, }; use futures::TryFutureExt; use ruma::{ - events::room::create::RoomCreateEventContent, state_res::RoomVersion, OwnedEventId, - OwnedRoomId, RoomId, RoomVersionId, + events::room::create::RoomCreateEventContent, OwnedEventId, OwnedRoomId, RoomId, + RoomVersionId, }; use crate::{globals, rooms, sending, server_keys, Dep}; diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index eb9ca01f..28011a1b 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -5,15 +5,14 @@ use std::{ }; use conduwuit::{ - err, implement, trace, + err, implement, + state_res::{self, StateMap}, + trace, utils::stream::{automatic_width, IterStream, ReadyExt, TryWidebandExt, WidebandExt}, Error, Result, }; use futures::{future::try_join, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; -use ruma::{ - state_res::{self, StateMap}, - OwnedEventId, RoomId, RoomVersionId, -}; +use ruma::{OwnedEventId, RoomId, RoomVersionId}; use crate::rooms::state_compressor::CompressedState; diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 7bf3b8f8..843b2af9 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -8,10 +8,10 @@ use std::{ use conduwuit::{ debug, err, implement, trace, utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryWidebandExt}, - PduEvent, Result, + PduEvent, Result, StateMap, }; use futures::{future::try_join, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; -use ruma::{state_res::StateMap, OwnedEventId, RoomId, RoomVersionId}; +use ruma::{OwnedEventId, RoomId, RoomVersionId}; use crate::rooms::short::ShortStateHash; diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index b33b0388..f319ba48 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -1,16 +1,12 @@ use std::{borrow::Borrow, collections::BTreeMap, iter::once, sync::Arc, time::Instant}; use conduwuit::{ - debug, debug_info, err, implement, trace, + debug, debug_info, err, implement, state_res, trace, utils::stream::{BroadbandExt, ReadyExt}, - warn, Err, PduEvent, Result, + warn, Err, EventTypeExt, PduEvent, Result, }; use futures::{future::ready, FutureExt, StreamExt}; -use ruma::{ - events::StateEventType, - state_res::{self, EventTypeExt}, - CanonicalJsonValue, RoomId, ServerName, -}; +use ruma::{events::StateEventType, CanonicalJsonValue, RoomId, ServerName}; use super::{get_room_version_id, to_room_version}; use crate::rooms::{ diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index de90a89c..d538de3c 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -3,6 +3,7 @@ use std::{collections::HashMap, fmt::Write, iter::once, sync::Arc}; use conduwuit::{ err, result::FlatOk, + state_res::{self, StateMap}, utils::{ calculate_hash, stream::{BroadbandExt, TryIgnore}, @@ -20,7 +21,6 @@ use ruma::{ AnyStrippedStateEvent, StateEventType, TimelineEventType, }, serde::Raw, - state_res::{self, StateMap}, EventId, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, UserId, }; diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index a7edd4a4..d6154121 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -12,6 +12,7 @@ use std::{ use conduwuit::{ at, debug, debug_warn, err, error, implement, info, pdu::{gen_event_id, EventHash, PduBuilder, PduCount, PduEvent}, + state_res::{self, Event, RoomVersion}, utils::{ self, future::TryExtExt, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt, }, @@ -36,7 +37,6 @@ use ruma::{ GlobalAccountDataEventType, StateEventType, TimelineEventType, }, push::{Action, Ruleset, Tweak}, - state_res::{self, Event, RoomVersion}, uint, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, RoomId, RoomVersionId, ServerName, UserId, }; From f2ca670c3b0858675312be60dcfb971384ce1244 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 8 Feb 2025 01:58:13 +0000 Subject: [PATCH 1510/2091] optimize further into state-res with SmallString triage and de-lints for state-res. Signed-off-by: Jason Volk --- Cargo.lock | 1 + Cargo.toml | 4 + src/api/client/membership.rs | 8 +- src/api/client/sync/v4.rs | 15 +- src/api/client/sync/v5.rs | 15 +- src/core/Cargo.toml | 3 + src/core/state_res/event_auth.rs | 142 ++++++++++-------- src/core/state_res/mod.rs | 89 ++++++----- src/core/state_res/room_version.rs | 1 + src/core/state_res/test_utils.rs | 43 +++--- .../rooms/event_handler/handle_outlier_pdu.rs | 2 +- .../rooms/event_handler/resolve_state.rs | 1 - .../rooms/event_handler/state_at_incoming.rs | 1 - .../event_handler/upgrade_outlier_pdu.rs | 10 +- src/service/rooms/timeline/mod.rs | 2 +- 15 files changed, 192 insertions(+), 145 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5981a2a6..4441779e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -810,6 +810,7 @@ dependencies = [ "libc", "libloading", "log", + "maplit", "nix", "num-traits", "rand", diff --git a/Cargo.toml b/Cargo.toml index d8f34544..a17aa4d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -379,6 +379,7 @@ features = [ "unstable-msc4203", # sending to-device events to appservices "unstable-msc4210", # remove legacy mentions "unstable-extensible-events", + "unstable-pdu", ] [workspace.dependencies.rust-rocksdb] @@ -527,6 +528,9 @@ features = ["std"] version = "0.3.2" features = ["std"] +[workspace.dependencies.maplit] +version = "1.0.2" + # # Patches # diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 1045b014..6c970665 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -14,7 +14,7 @@ use conduwuit::{ result::FlatOk, state_res, trace, utils::{self, shuffle, IterStream, ReadyExt}, - warn, Err, PduEvent, Result, + warn, Err, PduEvent, Result, StateKey, }; use futures::{join, FutureExt, StreamExt, TryFutureExt}; use ruma::{ @@ -1151,8 +1151,8 @@ async fn join_room_by_id_helper_remote( debug!("Running send_join auth check"); let fetch_state = &state; - let state_fetch = |k: &'static StateEventType, s: String| async move { - let shortstatekey = services.rooms.short.get_shortstatekey(k, &s).await.ok()?; + let state_fetch = |k: StateEventType, s: StateKey| async move { + let shortstatekey = services.rooms.short.get_shortstatekey(&k, &s).await.ok()?; let event_id = fetch_state.get(&shortstatekey)?; services.rooms.timeline.get_pdu(event_id).await.ok() @@ -1162,7 +1162,7 @@ async fn join_room_by_id_helper_remote( &state_res::RoomVersion::new(&room_version_id)?, &parsed_join_pdu, None, // TODO: third party invite - |k, s| state_fetch(k, s.to_owned()), + |k, s| state_fetch(k.clone(), s.into()), ) .await .map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?; diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 4e474ef3..13f832b2 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -395,9 +395,12 @@ pub(crate) async fn sync_events_v4_route( .map_or(10, usize_from_u64_truncated) .min(100); - todo_room - .0 - .extend(list.room_details.required_state.iter().cloned()); + todo_room.0.extend( + list.room_details + .required_state + .iter() + .map(|(ty, sk)| (ty.clone(), sk.as_str().into())), + ); todo_room.1 = todo_room.1.max(limit); // 0 means unknown because it got out of date @@ -449,7 +452,11 @@ pub(crate) async fn sync_events_v4_route( .map_or(10, usize_from_u64_truncated) .min(100); - todo_room.0.extend(room.required_state.iter().cloned()); + todo_room.0.extend( + room.required_state + .iter() + .map(|(ty, sk)| (ty.clone(), sk.as_str().into())), + ); todo_room.1 = todo_room.1.max(limit); // 0 means unknown because it got out of date todo_room.2 = todo_room.2.min( diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index 63731688..cda6c041 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -223,7 +223,11 @@ async fn fetch_subscriptions( let limit: UInt = room.timeline_limit; - todo_room.0.extend(room.required_state.iter().cloned()); + todo_room.0.extend( + room.required_state + .iter() + .map(|(ty, sk)| (ty.clone(), sk.as_str().into())), + ); todo_room.1 = todo_room.1.max(usize_from_ruma(limit)); // 0 means unknown because it got out of date todo_room.2 = todo_room.2.min( @@ -303,9 +307,12 @@ async fn handle_lists<'a>( let limit: usize = usize_from_ruma(list.room_details.timeline_limit).min(100); - todo_room - .0 - .extend(list.room_details.required_state.iter().cloned()); + todo_room.0.extend( + list.room_details + .required_state + .iter() + .map(|(ty, sk)| (ty.clone(), sk.as_str().into())), + ); todo_room.1 = todo_room.1.max(limit); // 0 means unknown because it got out of date diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index d4b0c83b..b40dd3ad 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -116,5 +116,8 @@ nix.workspace = true hardened_malloc-rs.workspace = true hardened_malloc-rs.optional = true +[dev-dependencies] +maplit.workspace = true + [lints] workspace = true diff --git a/src/core/state_res/event_auth.rs b/src/core/state_res/event_auth.rs index 72a0216c..df2f8b36 100644 --- a/src/core/state_res/event_auth.rs +++ b/src/core/state_res/event_auth.rs @@ -21,7 +21,6 @@ use serde::{ Deserialize, }; use serde_json::{from_str as from_json_str, value::RawValue as RawJsonValue}; -use tracing::{debug, error, instrument, trace, warn}; use super::{ power_levels::{ @@ -29,8 +28,9 @@ use super::{ deserialize_power_levels_content_invite, deserialize_power_levels_content_redact, }, room_version::RoomVersion, - Error, Event, Result, StateEventType, TimelineEventType, + Error, Event, Result, StateEventType, StateKey, TimelineEventType, }; +use crate::{debug, error, trace, warn}; // FIXME: field extracting could be bundled for `content` #[derive(Deserialize)] @@ -56,15 +56,15 @@ pub fn auth_types_for_event( sender: &UserId, state_key: Option<&str>, content: &RawJsonValue, -) -> serde_json::Result> { +) -> serde_json::Result> { if kind == &TimelineEventType::RoomCreate { return Ok(vec![]); } let mut auth_types = vec![ - (StateEventType::RoomPowerLevels, String::new()), - (StateEventType::RoomMember, sender.to_string()), - (StateEventType::RoomCreate, String::new()), + (StateEventType::RoomPowerLevels, StateKey::new()), + (StateEventType::RoomMember, sender.as_str().into()), + (StateEventType::RoomCreate, StateKey::new()), ]; if kind == &TimelineEventType::RoomMember { @@ -82,7 +82,7 @@ pub fn auth_types_for_event( if [MembershipState::Join, MembershipState::Invite, MembershipState::Knock] .contains(&membership) { - let key = (StateEventType::RoomJoinRules, String::new()); + let key = (StateEventType::RoomJoinRules, StateKey::new()); if !auth_types.contains(&key) { auth_types.push(key); } @@ -91,21 +91,22 @@ pub fn auth_types_for_event( .join_authorised_via_users_server .map(|m| m.deserialize()) { - let key = (StateEventType::RoomMember, u.to_string()); + let key = (StateEventType::RoomMember, u.as_str().into()); if !auth_types.contains(&key) { auth_types.push(key); } } } - let key = (StateEventType::RoomMember, state_key.to_owned()); + let key = (StateEventType::RoomMember, state_key.into()); if !auth_types.contains(&key) { auth_types.push(key); } if membership == MembershipState::Invite { if let Some(Ok(t_id)) = content.third_party_invite.map(|t| t.deserialize()) { - let key = (StateEventType::RoomThirdPartyInvite, t_id.signed.token); + let key = + (StateEventType::RoomThirdPartyInvite, t_id.signed.token.into()); if !auth_types.contains(&key) { auth_types.push(key); } @@ -128,7 +129,13 @@ pub fn auth_types_for_event( /// The `fetch_state` closure should gather state from a state snapshot. We need /// to know if the event passes auth against some state not a recursive /// collection of auth_events fields. -#[instrument(level = "debug", skip_all, fields(event_id = incoming_event.event_id().borrow().as_str()))] +#[tracing::instrument( + level = "debug", + skip_all, + fields( + event_id = incoming_event.event_id().borrow().as_str() + ) +)] pub async fn auth_check( room_version: &RoomVersion, incoming_event: &Incoming, @@ -136,10 +143,10 @@ pub async fn auth_check( fetch_state: F, ) -> Result where - F: Fn(&'static StateEventType, &str) -> Fut, + F: Fn(&StateEventType, &str) -> Fut + Send, Fut: Future> + Send, Fetched: Event + Send, - Incoming: Event + Send, + Incoming: Event + Send + Sync, { debug!( "auth_check beginning for {} ({})", @@ -262,6 +269,7 @@ where // sender domain of the event does not match the sender domain of the create // event, reject. #[derive(Deserialize)] + #[allow(clippy::items_after_statements)] struct RoomCreateContentFederate { #[serde(rename = "m.federate", default = "ruma::serde::default_true")] federate: bool, @@ -354,7 +362,7 @@ where join_rules_event.as_ref(), user_for_join_auth.as_deref(), &user_for_join_auth_membership, - room_create_event, + &room_create_event, )? { return Ok(false); } @@ -364,6 +372,7 @@ where } // If the sender's current membership state is not join, reject + #[allow(clippy::manual_let_else)] let sender_member_event = match sender_member_event { | Some(mem) => mem, | None => { @@ -498,19 +507,20 @@ where /// This is generated by calling `auth_types_for_event` with the membership /// event and the current State. #[allow(clippy::too_many_arguments)] +#[allow(clippy::cognitive_complexity)] fn valid_membership_change( room_version: &RoomVersion, target_user: &UserId, - target_user_membership_event: Option, + target_user_membership_event: Option<&impl Event>, sender: &UserId, - sender_membership_event: Option, + sender_membership_event: Option<&impl Event>, current_event: impl Event, - current_third_party_invite: Option, - power_levels_event: Option, - join_rules_event: Option, + current_third_party_invite: Option<&impl Event>, + power_levels_event: Option<&impl Event>, + join_rules_event: Option<&impl Event>, user_for_join_auth: Option<&UserId>, user_for_join_auth_membership: &MembershipState, - create_room: impl Event, + create_room: &impl Event, ) -> Result { #[derive(Deserialize)] struct GetThirdPartyInvite { @@ -856,6 +866,7 @@ fn check_power_levels( // and integers here debug!("validation of power event finished"); + #[allow(clippy::manual_let_else)] let current_state = match previous_power_event { | Some(current_state) => current_state, // If there is no previous m.room.power_levels event in the room, allow @@ -1054,6 +1065,7 @@ fn verify_third_party_invite( // If there is no m.room.third_party_invite event in the current room state with // state_key matching token, reject + #[allow(clippy::manual_let_else)] let current_tpid = match current_third_party_invite { | Some(id) => id, | None => return false, @@ -1069,12 +1081,14 @@ fn verify_third_party_invite( // If any signature in signed matches any public key in the // m.room.third_party_invite event, allow + #[allow(clippy::manual_let_else)] let tpid_ev = match from_json_str::(current_tpid.content().get()) { | Ok(ev) => ev, | Err(_) => return false, }; + #[allow(clippy::manual_let_else)] let decoded_invite_token = match Base64::parse(&tp_id.signed.token) { | Ok(tok) => tok, // FIXME: Log a warning? @@ -1096,7 +1110,7 @@ fn verify_third_party_invite( mod tests { use std::sync::Arc; - use ruma_events::{ + use ruma::events::{ room::{ join_rules::{ AllowRule, JoinRule, Restricted, RoomJoinRulesEventContent, RoomMembership, @@ -1107,7 +1121,7 @@ mod tests { }; use serde_json::value::to_raw_value as to_raw_json_value; - use crate::{ + use crate::state_res::{ event_auth::valid_membership_change, test_utils::{ alice, charlie, ella, event_id, member_content_ban, member_content_join, room_id, @@ -1145,16 +1159,16 @@ mod tests { assert!(valid_membership_change( &RoomVersion::V6, target_user, - fetch_state(StateEventType::RoomMember, target_user.to_string()), + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), sender, - fetch_state(StateEventType::RoomMember, sender.to_string()), + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), &requester, - None::, - fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), - fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), None, &MembershipState::Leave, - fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), ) .unwrap()); } @@ -1188,16 +1202,16 @@ mod tests { assert!(!valid_membership_change( &RoomVersion::V6, target_user, - fetch_state(StateEventType::RoomMember, target_user.to_string()), + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), sender, - fetch_state(StateEventType::RoomMember, sender.to_string()), + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), &requester, - None::, - fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), - fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), None, &MembershipState::Leave, - fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), ) .unwrap()); } @@ -1231,16 +1245,16 @@ mod tests { assert!(valid_membership_change( &RoomVersion::V6, target_user, - fetch_state(StateEventType::RoomMember, target_user.to_string()), + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), sender, - fetch_state(StateEventType::RoomMember, sender.to_string()), + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), &requester, - None::, - fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), - fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), None, &MembershipState::Leave, - fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), ) .unwrap()); } @@ -1274,16 +1288,16 @@ mod tests { assert!(!valid_membership_change( &RoomVersion::V6, target_user, - fetch_state(StateEventType::RoomMember, target_user.to_string()), + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), sender, - fetch_state(StateEventType::RoomMember, sender.to_string()), + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), &requester, - None::, - fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), - fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), None, &MembershipState::Leave, - fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), ) .unwrap()); } @@ -1334,32 +1348,32 @@ mod tests { assert!(valid_membership_change( &RoomVersion::V9, target_user, - fetch_state(StateEventType::RoomMember, target_user.to_string()), + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), sender, - fetch_state(StateEventType::RoomMember, sender.to_string()), + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), &requester, - None::, - fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), - fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), Some(alice()), &MembershipState::Join, - fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), ) .unwrap()); assert!(!valid_membership_change( &RoomVersion::V9, target_user, - fetch_state(StateEventType::RoomMember, target_user.to_string()), + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), sender, - fetch_state(StateEventType::RoomMember, sender.to_string()), + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), &requester, - None::, - fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), - fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), Some(ella()), &MembershipState::Leave, - fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), ) .unwrap()); } @@ -1402,16 +1416,16 @@ mod tests { assert!(valid_membership_change( &RoomVersion::V7, target_user, - fetch_state(StateEventType::RoomMember, target_user.to_string()), + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), sender, - fetch_state(StateEventType::RoomMember, sender.to_string()), + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), &requester, - None::, - fetch_state(StateEventType::RoomPowerLevels, "".to_owned()), - fetch_state(StateEventType::RoomJoinRules, "".to_owned()), + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), None, &MembershipState::Leave, - fetch_state(StateEventType::RoomCreate, "".to_owned()).unwrap(), + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), ) .unwrap()); } diff --git a/src/core/state_res/mod.rs b/src/core/state_res/mod.rs index e4054377..19ea3cc0 100644 --- a/src/core/state_res/mod.rs +++ b/src/core/state_res/mod.rs @@ -1,3 +1,5 @@ +#![cfg_attr(test, allow(warnings))] + pub(crate) mod error; pub mod event_auth; mod power_levels; @@ -12,7 +14,7 @@ use std::{ cmp::{Ordering, Reverse}, collections::{BinaryHeap, HashMap, HashSet}, fmt::Debug, - hash::Hash, + hash::{BuildHasher, Hash}, }; use futures::{future, stream, Future, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; @@ -32,13 +34,13 @@ pub use self::{ room_version::RoomVersion, state_event::Event, }; -use crate::{debug, trace, warn}; +use crate::{debug, pdu::StateKey, trace, warn}; /// A mapping of event type and state_key to some value `T`, usually an /// `EventId`. pub type StateMap = HashMap; pub type StateMapItem = (TypeStateKey, T); -pub type TypeStateKey = (StateEventType, String); +pub type TypeStateKey = (StateEventType, StateKey); type Result = crate::Result; @@ -68,10 +70,10 @@ type Result = crate::Result; /// event is part of the same room. //#[tracing::instrument(level = "debug", skip(state_sets, auth_chain_sets, //#[tracing::instrument(level event_fetch))] -pub async fn resolve<'a, E, SetIter, Fetch, FetchFut, Exists, ExistsFut>( +pub async fn resolve<'a, E, Sets, SetIter, Hasher, Fetch, FetchFut, Exists, ExistsFut>( room_version: &RoomVersionId, - state_sets: impl IntoIterator + Send, - auth_chain_sets: &'a [HashSet], + state_sets: Sets, + auth_chain_sets: &'a [HashSet], event_fetch: &Fetch, event_exists: &Exists, parallel_fetches: usize, @@ -81,7 +83,9 @@ where FetchFut: Future> + Send, Exists: Fn(E::Id) -> ExistsFut + Sync, ExistsFut: Future + Send, + Sets: IntoIterator + Send, SetIter: Iterator> + Clone + Send, + Hasher: BuildHasher + Send + Sync, E: Event + Clone + Send + Sync, E::Id: Borrow + Send + Sync, for<'b> &'b E: Send, @@ -178,7 +182,7 @@ where trace!(list = ?events_to_resolve, "events left to resolve"); // This "epochs" power level event - let power_event = resolved_control.get(&(StateEventType::RoomPowerLevels, String::new())); + let power_event = resolved_control.get(&(StateEventType::RoomPowerLevels, StateKey::new())); debug!(event_id = ?power_event, "power event"); @@ -222,16 +226,17 @@ fn separate<'a, Id>( where Id: Clone + Eq + Hash + 'a, { - let mut state_set_count = 0_usize; + let mut state_set_count: usize = 0; let mut occurrences = HashMap::<_, HashMap<_, _>>::new(); - let state_sets_iter = state_sets_iter.inspect(|_| state_set_count += 1); + let state_sets_iter = + state_sets_iter.inspect(|_| state_set_count = state_set_count.saturating_add(1)); for (k, v) in state_sets_iter.flatten() { occurrences .entry(k) .or_default() .entry(v) - .and_modify(|x| *x += 1) + .and_modify(|x: &mut usize| *x = x.saturating_add(1)) .or_insert(1); } @@ -246,7 +251,7 @@ where conflicted_state .entry((k.0.clone(), k.1.clone())) .and_modify(|x: &mut Vec<_>| x.push(id.clone())) - .or_insert(vec![id.clone()]); + .or_insert_with(|| vec![id.clone()]); } } } @@ -255,9 +260,13 @@ where } /// Returns a Vec of deduped EventIds that appear in some chains but not others. -fn get_auth_chain_diff(auth_chain_sets: &[HashSet]) -> impl Iterator + Send +#[allow(clippy::arithmetic_side_effects)] +fn get_auth_chain_diff( + auth_chain_sets: &[HashSet], +) -> impl Iterator + Send where Id: Clone + Eq + Hash + Send, + Hasher: BuildHasher + Send + Sync, { let num_sets = auth_chain_sets.len(); let mut id_counts: HashMap = HashMap::new(); @@ -288,7 +297,7 @@ async fn reverse_topological_power_sort( where F: Fn(E::Id) -> Fut + Sync, Fut: Future> + Send, - E: Event + Send, + E: Event + Send + Sync, E::Id: Borrow + Send + Sync, { debug!("reverse topological sort of power events"); @@ -337,14 +346,15 @@ where /// `key_fn` is used as to obtain the power level and age of an event for /// breaking ties (together with the event ID). #[tracing::instrument(level = "debug", skip_all)] -pub async fn lexicographical_topological_sort( - graph: &HashMap>, +pub async fn lexicographical_topological_sort( + graph: &HashMap>, key_fn: &F, ) -> Result> where F: Fn(Id) -> Fut + Sync, Fut: Future> + Send, - Id: Borrow + Clone + Eq + Hash + Ord + Send, + Id: Borrow + Clone + Eq + Hash + Ord + Send + Sync, + Hasher: BuildHasher + Default + Clone + Send + Sync, { #[derive(PartialEq, Eq)] struct TieBreaker<'a, Id> { @@ -395,7 +405,7 @@ where // The number of events that depend on the given event (the EventId key) // How many events reference this event in the DAG as a parent - let mut reverse_graph: HashMap<_, HashSet<_>> = HashMap::new(); + let mut reverse_graph: HashMap<_, HashSet<_, Hasher>> = HashMap::new(); // Vec of nodes that have zero out degree, least recent events. let mut zero_outdegree = Vec::new(); @@ -727,8 +737,8 @@ async fn get_mainline_depth( where F: Fn(E::Id) -> Fut + Sync, Fut: Future> + Send, - E: Event + Send, - E::Id: Borrow + Send, + E: Event + Send + Sync, + E::Id: Borrow + Send + Sync, { while let Some(sort_ev) = event { debug!(event_id = sort_ev.event_id().borrow().as_str(), "mainline"); @@ -758,10 +768,10 @@ async fn add_event_and_auth_chain_to_graph( auth_diff: &HashSet, fetch_event: &F, ) where - F: Fn(E::Id) -> Fut, + F: Fn(E::Id) -> Fut + Sync, Fut: Future> + Send, - E: Event + Send, - E::Id: Borrow + Clone + Send, + E: Event + Send + Sync, + E::Id: Borrow + Clone + Send + Sync, { let mut state = vec![event_id]; while let Some(eid) = state.pop() { @@ -788,7 +798,7 @@ where F: Fn(E::Id) -> Fut + Sync, Fut: Future> + Send, E: Event + Send, - E::Id: Borrow + Send, + E::Id: Borrow + Send + Sync, { match fetch(event_id.clone()).await.as_ref() { | Some(state) => is_power_event(state), @@ -820,18 +830,18 @@ fn is_power_event(event: impl Event) -> bool { /// Convenience trait for adding event type plus state key to state maps. pub trait EventTypeExt { - fn with_state_key(self, state_key: impl Into) -> (StateEventType, String); + fn with_state_key(self, state_key: impl Into) -> (StateEventType, StateKey); } impl EventTypeExt for StateEventType { - fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, StateKey) { (self, state_key.into()) } } impl EventTypeExt for TimelineEventType { - fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { - (self.to_string().into(), state_key.into()) + fn with_state_key(self, state_key: impl Into) -> (StateEventType, StateKey) { + (self.into(), state_key.into()) } } @@ -839,7 +849,7 @@ impl EventTypeExt for &T where T: EventTypeExt + Clone, { - fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, StateKey) { self.to_owned().with_state_key(state_key) } } @@ -858,13 +868,11 @@ mod tests { room::join_rules::{JoinRule, RoomJoinRulesEventContent}, StateEventType, TimelineEventType, }, - int, uint, + int, uint, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomVersionId, }; - use ruma_common::{MilliSecondsSinceUnixEpoch, OwnedEventId, RoomVersionId}; use serde_json::{json, value::to_raw_value as to_raw_json_value}; - use tracing::debug; - use crate::{ + use super::{ is_power_event, room_version::RoomVersion, test_utils::{ @@ -874,6 +882,7 @@ mod tests { }, Event, EventTypeExt, StateMap, }; + use crate::debug; async fn test_event_sort() { use futures::future::ready; @@ -898,11 +907,11 @@ mod tests { let fetcher = |id| ready(events.get(&id).cloned()); let sorted_power_events = - crate::reverse_topological_power_sort(power_events, &auth_chain, &fetcher, 1) + super::reverse_topological_power_sort(power_events, &auth_chain, &fetcher, 1) .await .unwrap(); - let resolved_power = crate::iterative_auth_check( + let resolved_power = super::iterative_auth_check( &RoomVersion::V6, sorted_power_events.iter(), HashMap::new(), // unconflicted events @@ -918,10 +927,10 @@ mod tests { events_to_sort.shuffle(&mut rand::thread_rng()); let power_level = resolved_power - .get(&(StateEventType::RoomPowerLevels, "".to_owned())) + .get(&(StateEventType::RoomPowerLevels, "".into())) .cloned(); - let sorted_event_ids = crate::mainline_sort(&events_to_sort, power_level, &fetcher, 1) + let sorted_event_ids = super::mainline_sort(&events_to_sort, power_level, &fetcher, 1) .await .unwrap(); @@ -1302,7 +1311,7 @@ mod tests { }) .collect(); - let resolved = match crate::resolve( + let resolved = match super::resolve( &RoomVersionId::V2, &state_sets, &auth_chain, @@ -1333,7 +1342,7 @@ mod tests { event_id("p") => hashset![event_id("o")], }; - let res = crate::lexicographical_topological_sort(&graph, &|_id| async { + let res = super::lexicographical_topological_sort(&graph, &|_id| async { Ok((int!(0), MilliSecondsSinceUnixEpoch(uint!(0)))) }) .await @@ -1421,7 +1430,7 @@ mod tests { let fetcher = |id: ::Id| ready(ev_map.get(&id).cloned()); let exists = |id: ::Id| ready(ev_map.get(&id).is_some()); - let resolved = match crate::resolve( + let resolved = match super::resolve( &RoomVersionId::V6, &state_sets, &auth_chain, @@ -1552,7 +1561,7 @@ mod tests { #[allow(unused_mut)] let mut x = StateMap::new(); $( - x.insert(($kind, $key.to_owned()), $id); + x.insert(($kind, $key.into()), $id); )* x }}; diff --git a/src/core/state_res/room_version.rs b/src/core/state_res/room_version.rs index e1b0afe1..8dfd6cde 100644 --- a/src/core/state_res/room_version.rs +++ b/src/core/state_res/room_version.rs @@ -32,6 +32,7 @@ pub enum StateResolutionVersion { } #[cfg_attr(not(feature = "unstable-exhaustive-types"), non_exhaustive)] +#[allow(clippy::struct_excessive_bools)] pub struct RoomVersion { /// The stability of this room. pub disposition: RoomDisposition, diff --git a/src/core/state_res/test_utils.rs b/src/core/state_res/test_utils.rs index 7954b28d..9c2b151f 100644 --- a/src/core/state_res/test_utils.rs +++ b/src/core/state_res/test_utils.rs @@ -7,28 +7,28 @@ use std::{ }, }; -use futures_util::future::ready; -use js_int::{int, uint}; -use ruma_common::{ - event_id, room_id, user_id, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, - RoomVersionId, ServerSignatures, UserId, -}; -use ruma_events::{ - pdu::{EventHash, Pdu, RoomV3Pdu}, - room::{ - join_rules::{JoinRule, RoomJoinRulesEventContent}, - member::{MembershipState, RoomMemberEventContent}, +use futures::future::ready; +use ruma::{ + event_id, + events::{ + pdu::{EventHash, Pdu, RoomV3Pdu}, + room::{ + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + }, + TimelineEventType, }, - TimelineEventType, + int, room_id, uint, user_id, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, + RoomVersionId, ServerSignatures, UserId, }; use serde_json::{ json, value::{to_raw_value as to_raw_json_value, RawValue as RawJsonValue}, }; -use tracing::info; pub(crate) use self::event::PduEvent; -use crate::{auth_types_for_event, Error, Event, EventTypeExt, Result, StateMap}; +use super::auth_types_for_event; +use crate::{info, Event, EventTypeExt, Result, StateMap}; static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); @@ -88,7 +88,7 @@ pub(crate) async fn do_check( // Resolve the current state and add it to the state_at_event map then continue // on in "time" - for node in crate::lexicographical_topological_sort(&graph, &|_id| async { + for node in super::lexicographical_topological_sort(&graph, &|_id| async { Ok((int!(0), MilliSecondsSinceUnixEpoch(uint!(0)))) }) .await @@ -135,7 +135,7 @@ pub(crate) async fn do_check( let event_map = &event_map; let fetch = |id: ::Id| ready(event_map.get(&id).cloned()); let exists = |id: ::Id| ready(event_map.get(&id).is_some()); - let resolved = crate::resolve( + let resolved = super::resolve( &RoomVersionId::V6, state_sets, &auth_chain_sets, @@ -223,7 +223,7 @@ pub(crate) async fn do_check( // Filter out the dummy messages events. // These act as points in time where there should be a known state to // test against. - && **k != ("m.room.message".into(), "dummy".to_owned()) + && **k != ("m.room.message".into(), "dummy".into()) }) .map(|(k, v)| (k.clone(), v.clone())) .collect::>(); @@ -239,7 +239,8 @@ impl TestStore { self.0 .get(event_id) .cloned() - .ok_or_else(|| Error::NotFound(format!("{event_id} not found"))) + .ok_or_else(|| super::Error::NotFound(format!("{event_id} not found"))) + .map_err(Into::into) } /// Returns a Vec of the related auth events to the given `event`. @@ -582,8 +583,10 @@ pub(crate) fn INITIAL_EDGES() -> Vec { } pub(crate) mod event { - use ruma_common::{MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId}; - use ruma_events::{pdu::Pdu, TimelineEventType}; + use ruma::{ + events::{pdu::Pdu, TimelineEventType}, + MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, + }; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue as RawJsonValue; diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 3cc15fc4..e628c77a 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -133,7 +133,7 @@ pub(super) async fn handle_outlier_pdu<'a>( )); } - let state_fetch = |ty: &'static StateEventType, sk: &str| { + let state_fetch = |ty: &StateEventType, sk: &str| { let key = (ty.to_owned(), sk.into()); ready(auth_events.get(&key)) }; diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 28011a1b..37d47d47 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -63,7 +63,6 @@ pub async fn resolve_state( .multi_get_statekey_from_short(shortstatekeys) .zip(event_ids) .ready_filter_map(|(ty_sk, id)| Some((ty_sk.ok()?, id))) - .map(|((ty, sk), id)| ((ty, sk.as_str().to_owned()), id)) .collect() }) .map(Ok::<_, Error>) diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 843b2af9..2eb6013a 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -172,7 +172,6 @@ async fn state_at_incoming_fork( .short .get_statekey_from_short(*k) .map_ok(|(ty, sk)| ((ty, sk), id.clone())) - .map_ok(|((ty, sk), id)| ((ty, sk.as_str().to_owned()), id)) }) .ready_filter_map(Result::ok) .collect() diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index f319ba48..385d2142 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -3,7 +3,7 @@ use std::{borrow::Borrow, collections::BTreeMap, iter::once, sync::Arc, time::In use conduwuit::{ debug, debug_info, err, implement, state_res, trace, utils::stream::{BroadbandExt, ReadyExt}, - warn, Err, EventTypeExt, PduEvent, Result, + warn, Err, EventTypeExt, PduEvent, Result, StateKey, }; use futures::{future::ready, FutureExt, StreamExt}; use ruma::{events::StateEventType, CanonicalJsonValue, RoomId, ServerName}; @@ -71,8 +71,8 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( debug!("Performing auth check"); // 11. Check the auth of the event passes based on the state of the event let state_fetch_state = &state_at_incoming_event; - let state_fetch = |k: &'static StateEventType, s: String| async move { - let shortstatekey = self.services.short.get_shortstatekey(k, &s).await.ok()?; + let state_fetch = |k: StateEventType, s: StateKey| async move { + let shortstatekey = self.services.short.get_shortstatekey(&k, &s).await.ok()?; let event_id = state_fetch_state.get(&shortstatekey)?; self.services.timeline.get_pdu(event_id).await.ok() @@ -82,7 +82,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( &room_version, &incoming_pdu, None, // TODO: third party invite - |k, s| state_fetch(k, s.to_owned()), + |ty, sk| state_fetch(ty.clone(), sk.into()), ) .await .map_err(|e| err!(Request(Forbidden("Auth check failed: {e:?}"))))?; @@ -104,7 +104,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( ) .await?; - let state_fetch = |k: &'static StateEventType, s: &str| { + let state_fetch = |k: &StateEventType, s: &str| { let key = k.with_state_key(s); ready(auth_events.get(&key).cloned()) }; diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index d6154121..9d6ee982 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -747,7 +747,7 @@ impl Service { }; let auth_fetch = |k: &StateEventType, s: &str| { - let key = (k.clone(), s.to_owned()); + let key = (k.clone(), s.into()); ready(auth_events.get(&key)) }; From 4de0dafdf11acb71d28e6891c9b740b66d448934 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 8 Feb 2025 06:24:25 +0000 Subject: [PATCH 1511/2091] bump ruma Signed-off-by: Jason Volk --- Cargo.lock | 360 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 2 +- 2 files changed, 183 insertions(+), 179 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4441779e..efba2e07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,7 +79,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -161,18 +161,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] name = "async-trait" -version = "0.1.85" +version = "0.1.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" +checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -221,9 +221,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.12.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ea835662a0af02443aa1396d39be523bbf8f11ee6fad20329607c480bea48c3" +checksum = "4c2b7ddaa2c56a367ad27a094ad8ef4faacf8a617c2575acb2ba88949df999ca" dependencies = [ "aws-lc-sys", "paste", @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.25.0" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71b2ddd3ada61a305e1d8bb6c005d1eaa7d14d903681edfc400406d523a9b491" +checksum = "54ac4f13dad353b209b34cbec082338202cbc01c8f00336b55c750c13ac91f8f" dependencies = [ "bindgen", "cc", @@ -427,7 +427,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.96", + "syn 2.0.98", "which", ] @@ -495,9 +495,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.1" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -511,9 +511,9 @@ checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" @@ -535,9 +535,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" [[package]] name = "bytesize" @@ -568,9 +568,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.10" +version = "1.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" +checksum = "755717a7de9ec452bf7f3f1a3099085deabd7f2962b861dae91ecd7a365903d2" dependencies = [ "jobserver", "libc", @@ -639,9 +639,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.26" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" +checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff" dependencies = [ "clap_builder", "clap_derive", @@ -649,9 +649,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.26" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" dependencies = [ "anstyle", "clap_lex", @@ -659,14 +659,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.24" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -677,9 +677,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" -version = "0.1.52" +version = "0.1.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c682c223677e0e5b6b7f63a64b9351844c3f1b1678a68b7ee617e30fb082620e" +checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" dependencies = [ "cc", ] @@ -863,7 +863,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1030,9 +1030,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -1177,7 +1177,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1204,7 +1204,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1273,7 +1273,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1325,7 +1325,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1526,7 +1526,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1578,10 +1578,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + [[package]] name = "gif" version = "0.13.1" @@ -1616,7 +1628,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.7.0", + "indexmap 2.7.1", "slab", "tokio", "tokio-util", @@ -1708,9 +1720,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-proto" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" +checksum = "2ad3d6d98c648ed628df039541a5577bee1a7c83e9e16fe3dbedeea4cdfeb971" dependencies = [ "async-trait", "cfg-if", @@ -1732,9 +1744,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.2" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" +checksum = "dcf287bde7b776e85d7188e6e5db7cf410a2f9531fe82817eb87feed034c8d14" dependencies = [ "cfg-if", "futures-util", @@ -1802,7 +1814,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -1850,9 +1862,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" [[package]] name = "httpdate" @@ -1868,9 +1880,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", @@ -2054,7 +2066,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2129,9 +2141,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -2158,7 +2170,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2189,9 +2201,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "itertools" @@ -2300,7 +2312,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2528,7 +2540,7 @@ checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2574,7 +2586,7 @@ checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -2675,7 +2687,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2739,15 +2751,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.2" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" @@ -2757,7 +2769,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.7.0", + "indexmap 2.7.1", "js-sys", "once_cell", "pin-project-lite", @@ -2913,7 +2925,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -2928,7 +2940,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_shared 0.11.3", + "phf_shared", ] [[package]] @@ -2937,18 +2949,8 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator 0.11.3", - "phf_shared 0.11.3", -] - -[[package]] -name = "phf_generator" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" -dependencies = [ - "phf_shared 0.10.0", - "rand", + "phf_generator", + "phf_shared", ] [[package]] @@ -2957,46 +2959,37 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared 0.11.3", + "phf_shared", "rand", ] -[[package]] -name = "phf_shared" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" -dependencies = [ - "siphasher 0.3.11", -] - [[package]] name = "phf_shared" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ - "siphasher 1.0.1", + "siphasher", ] [[package]] name = "pin-project" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e2ec53ad785f4d35dac0adea7f7dc6f1bb277ad84a680c7afefeae05d1f5916" +checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56a66c0c55993aa927429d0f8a0abfd74f084e4d9c192cffed01e418d83eefb" +checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3068,7 +3061,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" dependencies = [ "proc-macro2", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3097,7 +3090,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "version_check", "yansi", ] @@ -3118,7 +3111,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" dependencies = [ "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3141,7 +3134,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -3202,7 +3195,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.0", + "rustc-hash 2.1.1", "rustls", "socket2", "thiserror 2.0.11", @@ -3217,10 +3210,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", - "getrandom", + "getrandom 0.2.15", "rand", "ring", - "rustc-hash 2.1.0", + "rustc-hash 2.1.1", "rustls", "rustls-pki-types", "slab", @@ -3280,7 +3273,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -3479,7 +3472,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", @@ -3489,7 +3482,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "assign", "js_int", @@ -3511,7 +3504,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "js_int", "ruma-common", @@ -3523,7 +3516,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "as_variant", "assign", @@ -3546,14 +3539,14 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "as_variant", "base64 0.22.1", "bytes", "form_urlencoded", "http", - "indexmap 2.7.0", + "indexmap 2.7.1", "js_int", "konst", "percent-encoding", @@ -3577,10 +3570,10 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "as_variant", - "indexmap 2.7.0", + "indexmap 2.7.1", "js_int", "js_option", "percent-encoding", @@ -3602,7 +3595,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "bytes", "http", @@ -3620,7 +3613,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3629,7 +3622,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "js_int", "ruma-common", @@ -3639,7 +3632,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3647,14 +3640,14 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.96", + "syn 2.0.98", "toml", ] [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "js_int", "ruma-common", @@ -3666,7 +3659,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "headers", "http", @@ -3679,7 +3672,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3695,7 +3688,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5667c6292adb43fbe4725d31d6b5127a0cf60ce#f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" dependencies = [ "futures-util", "js_int", @@ -3755,9 +3748,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustc-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -3770,9 +3763,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.43" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ "bitflags 2.8.0", "errno", @@ -3783,9 +3776,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.21" +version = "0.23.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" +checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" dependencies = [ "aws-lc-rs", "log", @@ -3820,9 +3813,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" dependencies = [ "web-time 1.1.0", ] @@ -3862,9 +3855,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "sanitize-filename" @@ -3892,9 +3885,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sd-notify" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "561e6b346a5e59e0b8a07894004897d7160567e3352d2ebd6c3741d4e086b6f5" +checksum = "b943eadf71d8b69e661330cb0e2656e31040acf21ee7708e2c238a0ec6af2bf4" dependencies = [ "libc", ] @@ -3924,9 +3917,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" [[package]] name = "sentry" @@ -4080,7 +4073,7 @@ checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4090,7 +4083,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "ryu", "serde", @@ -4098,9 +4091,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.135" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -4155,7 +4148,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "itoa", "ryu", "serde", @@ -4253,12 +4246,6 @@ dependencies = [ "quote", ] -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - [[package]] name = "siphasher" version = "1.0.1" @@ -4333,26 +4320,25 @@ checksum = "f42444fea5b87a39db4218d9422087e66a85d0e7a0963a439b07bcdf91804006" [[package]] name = "string_cache" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +checksum = "938d512196766101d333398efde81bc1f37b00cb42c2f8350e5df639f040bbbe" dependencies = [ "new_debug_unreachable", - "once_cell", "parking_lot", - "phf_shared 0.10.0", + "phf_shared", "precomputed-hash", "serde", ] [[package]] name = "string_cache_codegen" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +checksum = "244292f3441c89febe5b5bdfbb6863aeaf4f64da810ea3050fd927b27b8d92ce" dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", + "phf_generator", + "phf_shared", "proc-macro2", "quote", ] @@ -4385,9 +4371,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.96" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -4411,7 +4397,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4446,9 +4432,9 @@ dependencies = [ [[package]] name = "termimad" -version = "0.31.1" +version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6a5d4cf55d9f1cb04fcda48f725772d0733ae34e030dfc4dd36e738a5965f4" +checksum = "a8e19c6dbf107bec01d0e216bb8219485795b7d75328e4fa5ef2756c1be4f8dc" dependencies = [ "coolor", "crokey", @@ -4496,7 +4482,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4507,7 +4493,7 @@ checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4663,7 +4649,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4726,9 +4712,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" dependencies = [ "serde", "serde_spanned", @@ -4747,11 +4733,11 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.22" +version = "0.22.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" dependencies = [ - "indexmap 2.7.0", + "indexmap 2.7.1", "serde", "serde_spanned", "toml_datetime", @@ -4876,7 +4862,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -4997,9 +4983,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-segmentation" @@ -5084,11 +5070,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.12.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "744018581f9a3454a9e15beb8a33b017183f1e7c0cd170232a2d1453b23a51c4" +checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" dependencies = [ - "getrandom", + "getrandom 0.3.1", "serde", ] @@ -5142,6 +5128,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -5164,7 +5159,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "wasm-bindgen-shared", ] @@ -5199,7 +5194,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5257,9 +5252,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.7" +version = "0.26.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" dependencies = [ "rustls-pki-types", ] @@ -5515,9 +5510,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.24" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +checksum = "86e376c75f4f43f44db463cf729e0d3acbf954d13e22c51e26e4c264b4ab545f" dependencies = [ "memchr", ] @@ -5532,6 +5527,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "write16" version = "1.0.0" @@ -5581,7 +5585,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "synstructure", ] @@ -5603,7 +5607,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] @@ -5623,7 +5627,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", "synstructure", ] @@ -5652,7 +5656,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.96", + "syn 2.0.98", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a17aa4d6..12556e00 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "f5667c6292adb43fbe4725d31d6b5127a0cf60ce" +rev = "e7a793b720e58bbe6858fecb86db97191dbfe7aa" features = [ "compat", "rand", From 6113803038f15a9f0206b31fc0216ebc315d7761 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 8 Feb 2025 10:09:57 +0000 Subject: [PATCH 1512/2091] better error logging on send_join response failure Signed-off-by: Jason Volk --- src/api/client/membership.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 6c970665..26736fb5 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -9,7 +9,7 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - at, debug, debug_info, debug_warn, err, info, + at, debug, debug_info, debug_warn, err, error, info, pdu::{gen_event_id_canonical_json, PduBuilder}, result::FlatOk, state_res, trace, @@ -1011,10 +1011,17 @@ async fn join_room_by_id_helper_remote( .await, }; - let send_join_response = services + let send_join_response = match services .sending .send_synapse_request(&remote_server, send_join_request) - .await?; + .await + { + | Ok(response) => response, + | Err(e) => { + error!("send_join failed: {e}"); + return Err(e); + }, + }; info!("send_join finished"); From f47677c995e2847b6ad39c877c3ab5b9bd5b1152 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 7 Feb 2025 07:09:45 +0000 Subject: [PATCH 1513/2091] refactor spaces Signed-off-by: Jason Volk --- src/api/client/space.rs | 228 +++++---- src/api/server/hierarchy.rs | 90 ++-- src/service/rooms/spaces/mod.rs | 774 +++++++++++++++--------------- src/service/rooms/spaces/tests.rs | 27 +- 4 files changed, 568 insertions(+), 551 deletions(-) diff --git a/src/api/client/space.rs b/src/api/client/space.rs index 8f54de2a..7efd7817 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -1,18 +1,25 @@ -use std::{collections::VecDeque, str::FromStr}; +use std::{ + collections::{BTreeSet, VecDeque}, + str::FromStr, +}; use axum::extract::State; -use conduwuit::{checked, pdu::ShortRoomId, utils::stream::IterStream}; -use futures::{StreamExt, TryFutureExt}; +use conduwuit::{ + utils::{future::TryExtExt, stream::IterStream}, + Err, Result, +}; +use futures::{future::OptionFuture, StreamExt, TryFutureExt}; use ruma::{ - api::client::{error::ErrorKind, space::get_hierarchy}, - OwnedRoomId, OwnedServerName, RoomId, UInt, UserId, + api::client::space::get_hierarchy, OwnedRoomId, OwnedServerName, RoomId, UInt, UserId, }; use service::{ - rooms::spaces::{get_parent_children_via, summary_to_chunk, SummaryAccessibility}, + rooms::spaces::{ + get_parent_children_via, summary_to_chunk, PaginationToken, SummaryAccessibility, + }, Services, }; -use crate::{service::rooms::spaces::PaginationToken, Error, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/client/v1/rooms/{room_id}/hierarchy` /// @@ -40,10 +47,9 @@ pub(crate) async fn get_hierarchy_route( // Should prevent unexpeded behaviour in (bad) clients if let Some(ref token) = key { if token.suggested_only != body.suggested_only || token.max_depth != max_depth { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "suggested_only and max_depth cannot change on paginated requests", - )); + return Err!(Request(InvalidParam( + "suggested_only and max_depth cannot change on paginated requests" + ))); } } @@ -52,58 +58,70 @@ pub(crate) async fn get_hierarchy_route( body.sender_user(), &body.room_id, limit.try_into().unwrap_or(10), - key.map_or(vec![], |token| token.short_room_ids), - max_depth.into(), + max_depth.try_into().unwrap_or(usize::MAX), body.suggested_only, + key.as_ref() + .into_iter() + .flat_map(|t| t.short_room_ids.iter()), ) .await } -async fn get_client_hierarchy( +async fn get_client_hierarchy<'a, ShortRoomIds>( services: &Services, sender_user: &UserId, room_id: &RoomId, limit: usize, - short_room_ids: Vec, - max_depth: u64, + max_depth: usize, suggested_only: bool, -) -> Result { - let mut parents = VecDeque::new(); + short_room_ids: ShortRoomIds, +) -> Result +where + ShortRoomIds: Iterator + Clone + Send + Sync + 'a, +{ + type Via = Vec; + type Entry = (OwnedRoomId, Via); + type Rooms = VecDeque; - // Don't start populating the results if we have to start at a specific room. - let mut populate_results = short_room_ids.is_empty(); + let mut queue: Rooms = [( + room_id.to_owned(), + room_id + .server_name() + .map(ToOwned::to_owned) + .into_iter() + .collect(), + )] + .into(); - let mut stack = vec![vec![(room_id.to_owned(), match room_id.server_name() { - | Some(server_name) => vec![server_name.into()], - | None => vec![], - })]]; + let mut rooms = Vec::with_capacity(limit); + let mut parents = BTreeSet::new(); + while let Some((current_room, via)) = queue.pop_front() { + let summary = services + .rooms + .spaces + .get_summary_and_children_client(¤t_room, suggested_only, sender_user, &via) + .await?; - let mut results = Vec::with_capacity(limit); - - while let Some((current_room, via)) = { next_room_to_traverse(&mut stack, &mut parents) } { - if results.len() >= limit { - break; - } - - match ( - services - .rooms - .spaces - .get_summary_and_children_client(¤t_room, suggested_only, sender_user, &via) - .await?, - current_room == room_id, - ) { + match (summary, current_room == room_id) { + | (None | Some(SummaryAccessibility::Inaccessible), false) => { + // Just ignore other unavailable rooms + }, + | (None, true) => { + return Err!(Request(Forbidden("The requested room was not found"))); + }, + | (Some(SummaryAccessibility::Inaccessible), true) => { + return Err!(Request(Forbidden("The requested room is inaccessible"))); + }, | (Some(SummaryAccessibility::Accessible(summary)), _) => { - let mut children: Vec<(OwnedRoomId, Vec)> = - get_parent_children_via(&summary, suggested_only) - .into_iter() - .filter(|(room, _)| parents.iter().all(|parent| parent != room)) - .rev() - .collect(); + let populate = parents.len() >= short_room_ids.clone().count(); - if populate_results { - results.push(summary_to_chunk(*summary.clone())); - } else { + let mut children: Vec = get_parent_children_via(&summary, suggested_only) + .filter(|(room, _)| !parents.contains(room)) + .rev() + .map(|(key, val)| (key, val.collect())) + .collect(); + + if !populate { children = children .iter() .rev() @@ -113,97 +131,69 @@ async fn get_client_hierarchy( .rooms .short .get_shortroomid(room) - .map_ok(|short| Some(&short) != short_room_ids.get(parents.len())) + .map_ok(|short| { + Some(&short) != short_room_ids.clone().nth(parents.len()) + }) .unwrap_or_else(|_| false) }) .map(Clone::clone) - .collect::)>>() + .collect::>() .await .into_iter() .rev() .collect(); - - if children.is_empty() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Room IDs in token were not found.", - )); - } - - // We have reached the room after where we last left off - let parents_len = parents.len(); - if checked!(parents_len + 1)? == short_room_ids.len() { - populate_results = true; - } } - let parents_len: u64 = parents.len().try_into()?; - if !children.is_empty() && parents_len < max_depth { - parents.push_back(current_room.clone()); - stack.push(children); + if populate { + rooms.push(summary_to_chunk(summary.clone())); + } else if queue.is_empty() && children.is_empty() { + return Err!(Request(InvalidParam("Room IDs in token were not found."))); } - // Root room in the space hierarchy, we return an error - // if this one fails. + + parents.insert(current_room.clone()); + if rooms.len() >= limit { + break; + } + + if children.is_empty() { + break; + } + + if parents.len() >= max_depth { + continue; + } + + queue.extend(children); }, - | (Some(SummaryAccessibility::Inaccessible), true) => { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "The requested room is inaccessible", - )); - }, - | (None, true) => { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "The requested room was not found", - )); - }, - // Just ignore other unavailable rooms - | (None | Some(SummaryAccessibility::Inaccessible), false) => (), } } - Ok(get_hierarchy::v1::Response { - next_batch: if let Some((room, _)) = next_room_to_traverse(&mut stack, &mut parents) { - parents.pop_front(); - parents.push_back(room); + let next_batch: OptionFuture<_> = queue + .pop_front() + .map(|(room, _)| async move { + parents.insert(room); let next_short_room_ids: Vec<_> = parents .iter() .stream() - .filter_map(|room_id| async move { - services.rooms.short.get_shortroomid(room_id).await.ok() - }) + .filter_map(|room_id| services.rooms.short.get_shortroomid(room_id).ok()) .collect() .await; - (next_short_room_ids != short_room_ids && !next_short_room_ids.is_empty()).then( - || { - PaginationToken { - short_room_ids: next_short_room_ids, - limit: UInt::new(max_depth) - .expect("When sent in request it must have been valid UInt"), - max_depth: UInt::new(max_depth) - .expect("When sent in request it must have been valid UInt"), - suggested_only, - } - .to_string() - }, - ) - } else { - None - }, - rooms: results, + (next_short_room_ids.iter().ne(short_room_ids) && !next_short_room_ids.is_empty()) + .then_some(PaginationToken { + short_room_ids: next_short_room_ids, + limit: max_depth.try_into().ok()?, + max_depth: max_depth.try_into().ok()?, + suggested_only, + }) + .as_ref() + .map(PaginationToken::to_string) + }) + .into(); + + Ok(get_hierarchy::v1::Response { + next_batch: next_batch.await.flatten(), + rooms, }) } - -fn next_room_to_traverse( - stack: &mut Vec)>>, - parents: &mut VecDeque, -) -> Option<(OwnedRoomId, Vec)> { - while stack.last().is_some_and(Vec::is_empty) { - stack.pop(); - parents.pop_back(); - } - - stack.last_mut().and_then(Vec::pop) -} diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs index bcf2f7bc..f7bc43ab 100644 --- a/src/api/server/hierarchy.rs +++ b/src/api/server/hierarchy.rs @@ -1,10 +1,11 @@ use axum::extract::State; -use conduwuit::{Err, Result}; -use ruma::{api::federation::space::get_hierarchy, RoomId, ServerName}; -use service::{ - rooms::spaces::{get_parent_children_via, Identifier, SummaryAccessibility}, - Services, +use conduwuit::{ + utils::stream::{BroadbandExt, IterStream}, + Err, Result, }; +use futures::{FutureExt, StreamExt}; +use ruma::api::federation::space::get_hierarchy; +use service::rooms::spaces::{get_parent_children_via, Identifier, SummaryAccessibility}; use crate::Ruma; @@ -20,54 +21,51 @@ pub(crate) async fn get_hierarchy_route( return Err!(Request(NotFound("Room does not exist."))); } - get_hierarchy(&services, &body.room_id, body.origin(), body.suggested_only).await -} - -/// Gets the response for the space hierarchy over federation request -/// -/// Errors if the room does not exist, so a check if the room exists should -/// be done -async fn get_hierarchy( - services: &Services, - room_id: &RoomId, - server_name: &ServerName, - suggested_only: bool, -) -> Result { + let room_id = &body.room_id; + let suggested_only = body.suggested_only; + let ref identifier = Identifier::ServerName(body.origin()); match services .rooms .spaces - .get_summary_and_children_local(&room_id.to_owned(), Identifier::ServerName(server_name)) + .get_summary_and_children_local(room_id, identifier) .await? { - | Some(SummaryAccessibility::Accessible(room)) => { - let mut children = Vec::new(); - let mut inaccessible_children = Vec::new(); + | None => Err!(Request(NotFound("The requested room was not found"))), - for (child, _via) in get_parent_children_via(&room, suggested_only) { - match services - .rooms - .spaces - .get_summary_and_children_local(&child, Identifier::ServerName(server_name)) - .await? - { - | Some(SummaryAccessibility::Accessible(summary)) => { - children.push((*summary).into()); - }, - | Some(SummaryAccessibility::Inaccessible) => { - inaccessible_children.push(child); - }, - | None => (), - } - } - - Ok(get_hierarchy::v1::Response { - room: *room, - children, - inaccessible_children, - }) - }, | Some(SummaryAccessibility::Inaccessible) => Err!(Request(NotFound("The requested room is inaccessible"))), - | None => Err!(Request(NotFound("The requested room was not found"))), + + | Some(SummaryAccessibility::Accessible(room)) => { + let (children, inaccessible_children) = + get_parent_children_via(&room, suggested_only) + .stream() + .broad_filter_map(|(child, _via)| async move { + match services + .rooms + .spaces + .get_summary_and_children_local(&child, identifier) + .await + .ok()? + { + | None => None, + + | Some(SummaryAccessibility::Inaccessible) => + Some((None, Some(child))), + + | Some(SummaryAccessibility::Accessible(summary)) => + Some((Some(summary), None)), + } + }) + .unzip() + .map(|(children, inaccessible_children): (Vec<_>, Vec<_>)| { + ( + children.into_iter().flatten().map(Into::into).collect(), + inaccessible_children.into_iter().flatten().collect(), + ) + }) + .await; + + Ok(get_hierarchy::v1::Response { room, children, inaccessible_children }) + }, } } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 1e2b0a9f..268d6dfe 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -1,14 +1,24 @@ mod pagination_token; +#[cfg(test)] mod tests; -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; -use conduwuit::{debug_info, err, utils::math::usize_from_f64, Error, Result}; -use futures::StreamExt; +use conduwuit::{ + implement, + utils::{ + future::BoolExt, + math::usize_from_f64, + stream::{BroadbandExt, ReadyExt}, + IterStream, + }, + Err, Error, Result, +}; +use futures::{pin_mut, stream::FuturesUnordered, FutureExt, Stream, StreamExt, TryFutureExt}; use lru_cache::LruCache; use ruma::{ api::{ - client::{error::ErrorKind, space::SpaceHierarchyRoomsChunk}, + client::space::SpaceHierarchyRoomsChunk, federation::{ self, space::{SpaceHierarchyChildSummary, SpaceHierarchyParentSummary}, @@ -21,46 +31,46 @@ use ruma::{ }, serde::Raw, space::SpaceRoomJoinRule, - OwnedRoomId, OwnedServerName, RoomId, ServerName, UserId, + OwnedEventId, OwnedRoomId, OwnedServerName, RoomId, ServerName, UserId, }; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, MutexGuard}; pub use self::pagination_token::PaginationToken; -use crate::{rooms, sending, Dep}; - -pub struct CachedSpaceHierarchySummary { - summary: SpaceHierarchyParentSummary, -} - -pub enum SummaryAccessibility { - Accessible(Box), - Inaccessible, -} - -/// Identifier used to check if rooms are accessible -/// -/// None is used if you want to return the room, no matter if accessible or not -pub enum Identifier<'a> { - UserId(&'a UserId), - ServerName(&'a ServerName), -} +use crate::{conduwuit::utils::TryFutureExtExt, rooms, sending, Dep}; pub struct Service { services: Services, - pub roomid_spacehierarchy_cache: - Mutex>>, + pub roomid_spacehierarchy_cache: Mutex, } struct Services { state_accessor: Dep, state_cache: Dep, state: Dep, - short: Dep, event_handler: Dep, timeline: Dep, sending: Dep, } +pub struct CachedSpaceHierarchySummary { + summary: SpaceHierarchyParentSummary, +} + +#[allow(clippy::large_enum_variant)] +pub enum SummaryAccessibility { + Accessible(SpaceHierarchyParentSummary), + Inaccessible, +} + +/// Identifier used to check if rooms are accessible. None is used if you want +/// to return the room, no matter if accessible or not +pub enum Identifier<'a> { + UserId(&'a UserId), + ServerName(&'a ServerName), +} + +type Cache = LruCache>; + impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; @@ -72,7 +82,6 @@ impl crate::Service for Service { .depend::("rooms::state_accessor"), state_cache: args.depend::("rooms::state_cache"), state: args.depend::("rooms::state"), - short: args.depend::("rooms::short"), event_handler: args .depend::("rooms::event_handler"), timeline: args.depend::("rooms::timeline"), @@ -85,370 +94,407 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - /// Gets the summary of a space using solely local information - pub async fn get_summary_and_children_local( - &self, - current_room: &OwnedRoomId, - identifier: Identifier<'_>, - ) -> Result> { - if let Some(cached) = self - .roomid_spacehierarchy_cache - .lock() - .await - .get_mut(¤t_room.to_owned()) - .as_ref() - { - return Ok(if let Some(cached) = cached { +/// Gets the summary of a space using solely local information +#[implement(Service)] +pub async fn get_summary_and_children_local( + &self, + current_room: &RoomId, + identifier: &Identifier<'_>, +) -> Result> { + match self + .roomid_spacehierarchy_cache + .lock() + .await + .get_mut(current_room) + .as_ref() + { + | None => (), // cache miss + | Some(None) => return Ok(None), + | Some(Some(cached)) => + return Ok(Some( if self .is_accessible_child( current_room, &cached.summary.join_rule, - &identifier, + identifier, &cached.summary.allowed_room_ids, ) .await { - Some(SummaryAccessibility::Accessible(Box::new(cached.summary.clone()))) + SummaryAccessibility::Accessible(cached.summary.clone()) } else { - Some(SummaryAccessibility::Inaccessible) - } - } else { - None - }); - } + SummaryAccessibility::Inaccessible + }, + )), + }; - if let Some(children_pdus) = self.get_stripped_space_child_events(current_room).await? { - let summary = self - .get_room_summary(current_room, children_pdus, &identifier) - .await; - if let Ok(summary) = summary { - self.roomid_spacehierarchy_cache.lock().await.insert( - current_room.clone(), - Some(CachedSpaceHierarchySummary { summary: summary.clone() }), - ); + let children_pdus: Vec<_> = self + .get_stripped_space_child_events(current_room) + .collect() + .await; - Ok(Some(SummaryAccessibility::Accessible(Box::new(summary)))) - } else { - Ok(None) - } - } else { - Ok(None) - } - } + let summary = self + .get_room_summary(current_room, children_pdus, identifier) + .boxed() + .await; - /// Gets the summary of a space using solely federation - #[tracing::instrument(level = "debug", skip(self))] - async fn get_summary_and_children_federation( - &self, - current_room: &OwnedRoomId, - suggested_only: bool, - user_id: &UserId, - via: &[OwnedServerName], - ) -> Result> { - for server in via { - debug_info!("Asking {server} for /hierarchy"); - let Ok(response) = self - .services + let Ok(summary) = summary else { + return Ok(None); + }; + + self.roomid_spacehierarchy_cache.lock().await.insert( + current_room.to_owned(), + Some(CachedSpaceHierarchySummary { summary: summary.clone() }), + ); + + Ok(Some(SummaryAccessibility::Accessible(summary))) +} + +/// Gets the summary of a space using solely federation +#[implement(Service)] +#[tracing::instrument(level = "debug", skip(self))] +async fn get_summary_and_children_federation( + &self, + current_room: &RoomId, + suggested_only: bool, + user_id: &UserId, + via: &[OwnedServerName], +) -> Result> { + let request = federation::space::get_hierarchy::v1::Request { + room_id: current_room.to_owned(), + suggested_only, + }; + + let mut requests: FuturesUnordered<_> = via + .iter() + .map(|server| { + self.services .sending - .send_federation_request(server, federation::space::get_hierarchy::v1::Request { - room_id: current_room.to_owned(), - suggested_only, - }) - .await - else { - continue; - }; - - debug_info!("Got response from {server} for /hierarchy\n{response:?}"); - let summary = response.room.clone(); - - self.roomid_spacehierarchy_cache.lock().await.insert( - current_room.clone(), - Some(CachedSpaceHierarchySummary { summary: summary.clone() }), - ); - - for child in response.children { - let mut guard = self.roomid_spacehierarchy_cache.lock().await; - if !guard.contains_key(current_room) { - guard.insert( - current_room.clone(), - Some(CachedSpaceHierarchySummary { - summary: { - let SpaceHierarchyChildSummary { - canonical_alias, - name, - num_joined_members, - room_id, - topic, - world_readable, - guest_can_join, - avatar_url, - join_rule, - room_type, - allowed_room_ids, - } = child; - - SpaceHierarchyParentSummary { - canonical_alias, - name, - num_joined_members, - room_id: room_id.clone(), - topic, - world_readable, - guest_can_join, - avatar_url, - join_rule, - room_type, - children_state: self - .get_stripped_space_child_events(&room_id) - .await? - .unwrap(), - allowed_room_ids, - } - }, - }), - ); - } - } - if self - .is_accessible_child( - current_room, - &response.room.join_rule, - &Identifier::UserId(user_id), - &response.room.allowed_room_ids, - ) - .await - { - return Ok(Some(SummaryAccessibility::Accessible(Box::new(summary.clone())))); - } - - return Ok(Some(SummaryAccessibility::Inaccessible)); - } + .send_federation_request(server, request.clone()) + }) + .collect(); + let Some(Ok(response)) = requests.next().await else { self.roomid_spacehierarchy_cache .lock() .await - .insert(current_room.clone(), None); + .insert(current_room.to_owned(), None); - Ok(None) - } + return Ok(None); + }; - /// Gets the summary of a space using either local or remote (federation) - /// sources - pub async fn get_summary_and_children_client( - &self, - current_room: &OwnedRoomId, - suggested_only: bool, - user_id: &UserId, - via: &[OwnedServerName], - ) -> Result> { - if let Ok(Some(response)) = self - .get_summary_and_children_local(current_room, Identifier::UserId(user_id)) - .await - { - Ok(Some(response)) - } else { - self.get_summary_and_children_federation(current_room, suggested_only, user_id, via) - .await - } - } + let summary = response.room; + self.roomid_spacehierarchy_cache.lock().await.insert( + current_room.to_owned(), + Some(CachedSpaceHierarchySummary { summary: summary.clone() }), + ); - async fn get_room_summary( - &self, - current_room: &OwnedRoomId, - children_state: Vec>, - identifier: &Identifier<'_>, - ) -> Result { - let room_id: &RoomId = current_room; - - let join_rule = self - .services - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") - .await - .map_or(JoinRule::Invite, |c: RoomJoinRulesEventContent| c.join_rule); - - let allowed_room_ids = self - .services - .state_accessor - .allowed_room_ids(join_rule.clone()); - - if !self - .is_accessible_child( - current_room, - &join_rule.clone().into(), - identifier, - &allowed_room_ids, - ) - .await - { - debug_info!("User is not allowed to see room {room_id}"); - // This error will be caught later - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "User is not allowed to see the room", - )); - } - - Ok(SpaceHierarchyParentSummary { - canonical_alias: self - .services - .state_accessor - .get_canonical_alias(room_id) - .await - .ok(), - name: self.services.state_accessor.get_name(room_id).await.ok(), - num_joined_members: self - .services - .state_cache - .room_joined_count(room_id) - .await - .unwrap_or(0) - .try_into() - .expect("user count should not be that big"), - room_id: room_id.to_owned(), - topic: self - .services - .state_accessor - .get_room_topic(room_id) - .await - .ok(), - world_readable: self - .services - .state_accessor - .is_world_readable(room_id) - .await, - guest_can_join: self.services.state_accessor.guest_can_join(room_id).await, - avatar_url: self - .services - .state_accessor - .get_avatar(room_id) - .await - .into_option() - .unwrap_or_default() - .url, - join_rule: join_rule.into(), - room_type: self - .services - .state_accessor - .get_room_type(room_id) - .await - .ok(), - children_state, - allowed_room_ids, + response + .children + .into_iter() + .stream() + .then(|child| { + self.roomid_spacehierarchy_cache + .lock() + .map(|lock| (child, lock)) }) + .ready_filter_map(|(child, mut cache)| { + (!cache.contains_key(current_room)).then_some((child, cache)) + }) + .for_each(|(child, cache)| self.cache_insert(cache, current_room, child)) + .await; + + let identifier = Identifier::UserId(user_id); + let is_accessible_child = self + .is_accessible_child( + current_room, + &summary.join_rule, + &identifier, + &summary.allowed_room_ids, + ) + .await; + + if is_accessible_child { + return Ok(Some(SummaryAccessibility::Accessible(summary))); } - /// Simply returns the stripped m.space.child events of a room - async fn get_stripped_space_child_events( - &self, - room_id: &RoomId, - ) -> Result>>, Error> { - let Ok(current_shortstatehash) = - self.services.state.get_room_shortstatehash(room_id).await - else { - return Ok(None); - }; - - let state: HashMap<_, Arc<_>> = self - .services - .state_accessor - .state_full_ids(current_shortstatehash) - .collect() - .await; - - let mut children_pdus = Vec::with_capacity(state.len()); - for (key, id) in state { - let (event_type, state_key) = - self.services.short.get_statekey_from_short(key).await?; - - if event_type != StateEventType::SpaceChild { - continue; - } - - let pdu = - self.services.timeline.get_pdu(&id).await.map_err(|e| { - err!(Database("Event {id:?} in space state not found: {e:?}")) - })?; + Ok(Some(SummaryAccessibility::Inaccessible)) +} +/// Simply returns the stripped m.space.child events of a room +#[implement(Service)] +fn get_stripped_space_child_events<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream> + 'a { + self.services + .state + .get_room_shortstatehash(room_id) + .map_ok(|current_shortstatehash| { + self.services + .state_accessor + .state_keys_with_ids(current_shortstatehash, &StateEventType::SpaceChild) + }) + .map(Result::into_iter) + .map(IterStream::stream) + .map(StreamExt::flatten) + .flatten_stream() + .broad_filter_map(move |(state_key, event_id): (_, OwnedEventId)| async move { + self.services + .timeline + .get_pdu(&event_id) + .map_ok(move |pdu| (state_key, pdu)) + .await + .ok() + }) + .ready_filter_map(move |(state_key, pdu)| { if let Ok(content) = pdu.get_content::() { if content.via.is_empty() { - continue; + return None; } } - if OwnedRoomId::try_from(state_key).is_ok() { - children_pdus.push(pdu.to_stripped_spacechild_state_event()); + if RoomId::parse(&state_key).is_ok() { + return Some(pdu.to_stripped_spacechild_state_event()); } - } - Ok(Some(children_pdus)) + None + }) +} + +/// Gets the summary of a space using either local or remote (federation) +/// sources +#[implement(Service)] +pub async fn get_summary_and_children_client( + &self, + current_room: &OwnedRoomId, + suggested_only: bool, + user_id: &UserId, + via: &[OwnedServerName], +) -> Result> { + let identifier = Identifier::UserId(user_id); + + if let Ok(Some(response)) = self + .get_summary_and_children_local(current_room, &identifier) + .await + { + return Ok(Some(response)); } - /// With the given identifier, checks if a room is accessable - async fn is_accessible_child( - &self, - current_room: &OwnedRoomId, - join_rule: &SpaceRoomJoinRule, - identifier: &Identifier<'_>, - allowed_room_ids: &Vec, - ) -> bool { - match identifier { - | Identifier::ServerName(server_name) => { - // Checks if ACLs allow for the server to participate - if self - .services - .event_handler - .acl_check(server_name, current_room) - .await - .is_err() - { - return false; - } - }, - | Identifier::UserId(user_id) => { - if self - .services - .state_cache - .is_joined(user_id, current_room) - .await || self - .services - .state_cache - .is_invited(user_id, current_room) - .await - { - return true; - } - }, + self.get_summary_and_children_federation(current_room, suggested_only, user_id, via) + .await +} + +#[implement(Service)] +async fn get_room_summary( + &self, + room_id: &RoomId, + children_state: Vec>, + identifier: &Identifier<'_>, +) -> Result { + let join_rule = self + .services + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") + .await + .map_or(JoinRule::Invite, |c: RoomJoinRulesEventContent| c.join_rule); + + let allowed_room_ids = self + .services + .state_accessor + .allowed_room_ids(join_rule.clone()); + + let join_rule = join_rule.clone().into(); + let is_accessible_child = self + .is_accessible_child(room_id, &join_rule, identifier, &allowed_room_ids) + .await; + + if !is_accessible_child { + return Err!(Request(Forbidden("User is not allowed to see the room",))); + } + + let name = self.services.state_accessor.get_name(room_id).ok(); + + let topic = self.services.state_accessor.get_room_topic(room_id).ok(); + + let room_type = self.services.state_accessor.get_room_type(room_id).ok(); + + let world_readable = self.services.state_accessor.is_world_readable(room_id); + + let guest_can_join = self.services.state_accessor.guest_can_join(room_id); + + let num_joined_members = self + .services + .state_cache + .room_joined_count(room_id) + .unwrap_or(0); + + let canonical_alias = self + .services + .state_accessor + .get_canonical_alias(room_id) + .ok(); + + let avatar_url = self + .services + .state_accessor + .get_avatar(room_id) + .map(|res| res.into_option().unwrap_or_default().url); + + let ( + canonical_alias, + name, + num_joined_members, + topic, + world_readable, + guest_can_join, + avatar_url, + room_type, + ) = futures::join!( + canonical_alias, + name, + num_joined_members, + topic, + world_readable, + guest_can_join, + avatar_url, + room_type + ); + + Ok(SpaceHierarchyParentSummary { + canonical_alias, + name, + topic, + world_readable, + guest_can_join, + avatar_url, + room_type, + children_state, + allowed_room_ids, + join_rule, + room_id: room_id.to_owned(), + num_joined_members: num_joined_members + .try_into() + .expect("user count should not be that big"), + }) +} + +/// With the given identifier, checks if a room is accessable +#[implement(Service)] +async fn is_accessible_child( + &self, + current_room: &RoomId, + join_rule: &SpaceRoomJoinRule, + identifier: &Identifier<'_>, + allowed_room_ids: &[OwnedRoomId], +) -> bool { + if let Identifier::ServerName(server_name) = identifier { + // Checks if ACLs allow for the server to participate + if self + .services + .event_handler + .acl_check(server_name, current_room) + .await + .is_err() + { + return false; } - match &join_rule { - | SpaceRoomJoinRule::Public - | SpaceRoomJoinRule::Knock - | SpaceRoomJoinRule::KnockRestricted => true, - | SpaceRoomJoinRule::Restricted => { - for room in allowed_room_ids { + } + + if let Identifier::UserId(user_id) = identifier { + let is_joined = self.services.state_cache.is_joined(user_id, current_room); + + let is_invited = self.services.state_cache.is_invited(user_id, current_room); + + pin_mut!(is_joined, is_invited); + if is_joined.or(is_invited).await { + return true; + } + } + + match join_rule { + | SpaceRoomJoinRule::Public + | SpaceRoomJoinRule::Knock + | SpaceRoomJoinRule::KnockRestricted => true, + | SpaceRoomJoinRule::Restricted => + allowed_room_ids + .iter() + .stream() + .any(|room| async { match identifier { - | Identifier::UserId(user) => { - if self.services.state_cache.is_joined(user, room).await { - return true; - } - }, - | Identifier::ServerName(server) => { - if self.services.state_cache.server_in_room(server, room).await { - return true; - } - }, + | Identifier::UserId(user) => + self.services.state_cache.is_joined(user, room).await, + | Identifier::ServerName(server) => + self.services.state_cache.server_in_room(server, room).await, } - } - false - }, - // Invite only, Private, or Custom join rule - | _ => false, - } + }) + .await, + + // Invite only, Private, or Custom join rule + | _ => false, } } +/// Returns the children of a SpaceHierarchyParentSummary, making use of the +/// children_state field +pub fn get_parent_children_via( + parent: &SpaceHierarchyParentSummary, + suggested_only: bool, +) -> impl DoubleEndedIterator)> + Send + '_ +{ + parent + .children_state + .iter() + .map(Raw::deserialize) + .filter_map(Result::ok) + .filter_map(move |ce| { + (!suggested_only || ce.content.suggested) + .then_some((ce.state_key, ce.content.via.into_iter())) + }) +} + +#[implement(Service)] +async fn cache_insert( + &self, + mut cache: MutexGuard<'_, Cache>, + current_room: &RoomId, + child: SpaceHierarchyChildSummary, +) { + let SpaceHierarchyChildSummary { + canonical_alias, + name, + num_joined_members, + room_id, + topic, + world_readable, + guest_can_join, + avatar_url, + join_rule, + room_type, + allowed_room_ids, + } = child; + + let summary = SpaceHierarchyParentSummary { + canonical_alias, + name, + num_joined_members, + topic, + world_readable, + guest_can_join, + avatar_url, + join_rule, + room_type, + allowed_room_ids, + room_id: room_id.clone(), + children_state: self + .get_stripped_space_child_events(&room_id) + .collect() + .await, + }; + + cache.insert(current_room.to_owned(), Some(CachedSpaceHierarchySummary { summary })); +} + // Here because cannot implement `From` across ruma-federation-api and // ruma-client-api types impl From for SpaceHierarchyRoomsChunk { @@ -517,25 +563,3 @@ pub fn summary_to_chunk(summary: SpaceHierarchyParentSummary) -> SpaceHierarchyR children_state, } } - -/// Returns the children of a SpaceHierarchyParentSummary, making use of the -/// children_state field -#[must_use] -pub fn get_parent_children_via( - parent: &SpaceHierarchyParentSummary, - suggested_only: bool, -) -> Vec<(OwnedRoomId, Vec)> { - parent - .children_state - .iter() - .filter_map(|raw_ce| { - raw_ce.deserialize().map_or(None, |ce| { - if suggested_only && !ce.content.suggested { - None - } else { - Some((ce.state_key, ce.content.via)) - } - }) - }) - .collect() -} diff --git a/src/service/rooms/spaces/tests.rs b/src/service/rooms/spaces/tests.rs index b4c387d7..dd6c2f35 100644 --- a/src/service/rooms/spaces/tests.rs +++ b/src/service/rooms/spaces/tests.rs @@ -1,5 +1,3 @@ -#![cfg(test)] - use std::str::FromStr; use ruma::{ @@ -69,15 +67,22 @@ fn get_summary_children() { } .into(); - assert_eq!(get_parent_children_via(&summary, false), vec![ - (owned_room_id!("!foo:example.org"), vec![owned_server_name!("example.org")]), - (owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")]), - (owned_room_id!("!baz:example.org"), vec![owned_server_name!("example.org")]) - ]); - assert_eq!(get_parent_children_via(&summary, true), vec![( - owned_room_id!("!bar:example.org"), - vec![owned_server_name!("example.org")] - )]); + assert_eq!( + get_parent_children_via(&summary, false) + .map(|(k, v)| (k, v.collect::>())) + .collect::>(), + vec![ + (owned_room_id!("!foo:example.org"), vec![owned_server_name!("example.org")]), + (owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")]), + (owned_room_id!("!baz:example.org"), vec![owned_server_name!("example.org")]) + ] + ); + assert_eq!( + get_parent_children_via(&summary, true) + .map(|(k, v)| (k, v.collect::>())) + .collect::>(), + vec![(owned_room_id!("!bar:example.org"), vec![owned_server_name!("example.org")])] + ); } #[test] From c614d5bf44b477a39a6b819ab4f31fa9f2c626f1 Mon Sep 17 00:00:00 2001 From: strawberry Date: Mon, 17 Feb 2025 17:35:03 -0500 Subject: [PATCH 1514/2091] bump ruwuma Signed-off-by: strawberry --- Cargo.lock | 27 ++++++++++++------------- Cargo.toml | 2 +- flake.lock | 6 +++--- src/api/client/unstable.rs | 40 +++++++++++++++++++++++--------------- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index efba2e07..be2c6720 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "assign", "js_int", @@ -3504,7 +3504,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "js_int", "ruma-common", @@ -3516,7 +3516,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "as_variant", "assign", @@ -3539,12 +3539,13 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "as_variant", "base64 0.22.1", "bytes", "form_urlencoded", + "getrandom 0.2.15", "http", "indexmap 2.7.1", "js_int", @@ -3570,7 +3571,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "as_variant", "indexmap 2.7.1", @@ -3595,7 +3596,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "bytes", "http", @@ -3613,7 +3614,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3622,7 +3623,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "js_int", "ruma-common", @@ -3632,7 +3633,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3647,7 +3648,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "js_int", "ruma-common", @@ -3659,7 +3660,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "headers", "http", @@ -3672,7 +3673,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3688,7 +3689,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=e7a793b720e58bbe6858fecb86db97191dbfe7aa#e7a793b720e58bbe6858fecb86db97191dbfe7aa" +source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 12556e00..bea306f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "e7a793b720e58bbe6858fecb86db97191dbfe7aa" +rev = "4b3a92568310bef42078783e0172b188c5a92b3d" features = [ "compat", "rand", diff --git a/flake.lock b/flake.lock index 5af6ec43..15040a42 100644 --- a/flake.lock +++ b/flake.lock @@ -567,11 +567,11 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1737828695, - "narHash": "sha256-8Ev6zzhNPU798JNvU27a7gj5X+6SDG3jBweUkQ59DbA=", + "lastModified": 1739735789, + "narHash": "sha256-BIzuZS0TV4gRnciP4ieW5J3Hql986iedM5dHQfK6z68=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "a4d9230dcc9d03be428b9a728133f8f646c0065c", + "rev": "34e401fd4392dd3268e042f1e40dffd064b9a7ff", "type": "github" }, "original": { diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 66cb31d5..67c7df75 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -272,7 +272,7 @@ pub(crate) async fn set_profile_key_route( ))); } - let Some(profile_key_value) = body.kv_pair.get(&body.key) else { + let Some(profile_key_value) = body.kv_pair.get(&body.key_name) else { return Err!(Request(BadJson( "The key does not match the URL field key, or JSON body is empty (use DELETE)" ))); @@ -290,7 +290,7 @@ pub(crate) async fn set_profile_key_route( return Err!(Request(BadJson("Key names cannot be longer than 128 bytes"))); } - if body.key == "displayname" { + if body.key_name == "displayname" { let all_joined_rooms: Vec = services .rooms .state_cache @@ -306,7 +306,7 @@ pub(crate) async fn set_profile_key_route( &all_joined_rooms, ) .await; - } else if body.key == "avatar_url" { + } else if body.key_name == "avatar_url" { let mxc = ruma::OwnedMxcUri::from(profile_key_value.to_string()); let all_joined_rooms: Vec = services @@ -319,9 +319,11 @@ pub(crate) async fn set_profile_key_route( update_avatar_url(&services, &body.user_id, Some(mxc), None, &all_joined_rooms).await; } else { - services - .users - .set_profile_key(&body.user_id, &body.key, Some(profile_key_value.clone())); + services.users.set_profile_key( + &body.user_id, + &body.key_name, + Some(profile_key_value.clone()), + ); } if services.globals.allow_local_presence() { @@ -357,7 +359,7 @@ pub(crate) async fn delete_profile_key_route( ))); } - if body.key == "displayname" { + if body.key_name == "displayname" { let all_joined_rooms: Vec = services .rooms .state_cache @@ -367,7 +369,7 @@ pub(crate) async fn delete_profile_key_route( .await; update_displayname(&services, &body.user_id, None, &all_joined_rooms).await; - } else if body.key == "avatar_url" { + } else if body.key_name == "avatar_url" { let all_joined_rooms: Vec = services .rooms .state_cache @@ -380,7 +382,7 @@ pub(crate) async fn delete_profile_key_route( } else { services .users - .set_profile_key(&body.user_id, &body.key, None); + .set_profile_key(&body.user_id, &body.key_name, None); } if services.globals.allow_local_presence() { @@ -497,11 +499,13 @@ pub(crate) async fn get_profile_key_route( .users .set_timezone(&body.user_id, response.tz.clone()); - if let Some(value) = response.custom_profile_fields.get(&body.key) { - profile_key_value.insert(body.key.clone(), value.clone()); - services - .users - .set_profile_key(&body.user_id, &body.key, Some(value.clone())); + if let Some(value) = response.custom_profile_fields.get(&body.key_name) { + profile_key_value.insert(body.key_name.clone(), value.clone()); + services.users.set_profile_key( + &body.user_id, + &body.key_name, + Some(value.clone()), + ); } else { return Err!(Request(NotFound("The requested profile key does not exist."))); } @@ -520,8 +524,12 @@ pub(crate) async fn get_profile_key_route( return Err!(Request(NotFound("Profile was not found."))); } - if let Ok(value) = services.users.profile_key(&body.user_id, &body.key).await { - profile_key_value.insert(body.key.clone(), value); + if let Ok(value) = services + .users + .profile_key(&body.user_id, &body.key_name) + .await + { + profile_key_value.insert(body.key_name.clone(), value); } else { return Err!(Request(NotFound("The requested profile key does not exist."))); } From 01155fa649ef401d3ca9653439c0a7adf8a83f71 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 21 Feb 2025 17:47:44 +0000 Subject: [PATCH 1515/2091] fix unsafe precondition violation Signed-off-by: Jason Volk --- src/api/client/sync/mod.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 1967f4a2..46540881 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -76,11 +76,13 @@ async fn share_encrypted_room( .state_cache .get_shared_rooms(sender_user, user_id) .ready_filter(|&room_id| Some(room_id) != ignore_room) - .broad_any(|other_room_id| { + .map(ToOwned::to_owned) + .broad_any(|other_room_id| async move { services .rooms .state_accessor - .is_encrypted_room(other_room_id) + .is_encrypted_room(&other_room_id) + .await }) .await } From 1061f68f0e14ee166a14d631540d322492988627 Mon Sep 17 00:00:00 2001 From: morguldir Date: Fri, 21 Feb 2025 21:13:06 +0100 Subject: [PATCH 1516/2091] bump ruwuma --- Cargo.lock | 26 +++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be2c6720..7e84437c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3482,7 +3482,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "assign", "js_int", @@ -3504,7 +3504,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "js_int", "ruma-common", @@ -3516,7 +3516,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "as_variant", "assign", @@ -3539,7 +3539,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "as_variant", "base64 0.22.1", @@ -3571,7 +3571,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "as_variant", "indexmap 2.7.1", @@ -3596,7 +3596,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "bytes", "http", @@ -3614,7 +3614,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3623,7 +3623,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "js_int", "ruma-common", @@ -3633,7 +3633,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3648,7 +3648,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "js_int", "ruma-common", @@ -3660,7 +3660,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "headers", "http", @@ -3673,7 +3673,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3689,7 +3689,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=4b3a92568310bef42078783e0172b188c5a92b3d#4b3a92568310bef42078783e0172b188c5a92b3d" +source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" dependencies = [ "futures-util", "js_int", diff --git a/Cargo.toml b/Cargo.toml index bea306f6..ed7e6ac3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "4b3a92568310bef42078783e0172b188c5a92b3d" +rev = "5dc3e0f81d614ed9dc96b50f646b2e4385291c55" features = [ "compat", "rand", From 8085a1c064afeb61d8136963a671e6bbc15a8f98 Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 22 Feb 2025 16:46:06 +0100 Subject: [PATCH 1517/2091] Implement MSC3967, also fixes error when uploading keys in element Co-authored-by: Aiden McClelland Signed-off-by: morguldir --- bin/complement | 2 +- src/api/client/keys.rs | 120 +++++++++++++++++++++++++++++++++------ src/api/server/send.rs | 14 ++--- src/service/users/mod.rs | 60 +++++++++++--------- 4 files changed, 143 insertions(+), 53 deletions(-) diff --git a/bin/complement b/bin/complement index a4c62856..ffd7a938 100755 --- a/bin/complement +++ b/bin/complement @@ -45,7 +45,7 @@ set +o pipefail env \ -C "$COMPLEMENT_SRC" \ COMPLEMENT_BASE_IMAGE="$OCI_IMAGE" \ - go test -tags="conduwuit_blacklist" "$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests | tee "$LOG_FILE" + go test -tags="conduwuit_blacklist" "$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests ./tests/msc3967 | tee "$LOG_FILE" set -o pipefail # Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 7bf0a5da..801ae32b 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use axum::extract::State; -use conduwuit::{err, utils, Error, Result}; +use conduwuit::{debug, err, info, result::NotFound, utils, Err, Error, Result}; use futures::{stream::FuturesUnordered, StreamExt}; use ruma::{ api::{ @@ -15,6 +15,7 @@ use ruma::{ }, federation, }, + encryption::CrossSigningKey, serde::Raw, OneTimeKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, }; @@ -125,7 +126,24 @@ pub(crate) async fn upload_signing_keys_route( auth_error: None, }; - if let Some(auth) = &body.auth { + if let Ok(exists) = check_for_new_keys( + services, + sender_user, + body.self_signing_key.as_ref(), + body.user_signing_key.as_ref(), + body.master_key.as_ref(), + ) + .await + .inspect_err(|e| info!(?e)) + { + if let Some(result) = exists { + // No-op, they tried to reupload the same set of keys + // (lost connection for example) + return Ok(result); + } + debug!("Skipping UIA in accordance with MSC3967, the user didn't have any existing keys"); + // Some of the keys weren't found, so we let them upload + } else if let Some(auth) = &body.auth { let (worked, uiaainfo) = services .uiaa .try_auth(sender_user, sender_device, auth, &uiaainfo) @@ -134,7 +152,7 @@ pub(crate) async fn upload_signing_keys_route( if !worked { return Err(Error::Uiaa(uiaainfo)); } - // Success! + // Success! } else if let Some(json) = body.json_body { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services @@ -146,22 +164,90 @@ pub(crate) async fn upload_signing_keys_route( return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); } - if let Some(master_key) = &body.master_key { - services - .users - .add_cross_signing_keys( - sender_user, - master_key, - &body.self_signing_key, - &body.user_signing_key, - true, // notify so that other users see the new keys - ) - .await?; - } + services + .users + .add_cross_signing_keys( + sender_user, + &body.master_key, + &body.self_signing_key, + &body.user_signing_key, + true, // notify so that other users see the new keys + ) + .await?; Ok(upload_signing_keys::v3::Response {}) } +async fn check_for_new_keys( + services: crate::State, + user_id: &UserId, + self_signing_key: Option<&Raw>, + user_signing_key: Option<&Raw>, + master_signing_key: Option<&Raw>, +) -> Result> { + debug!("checking for existing keys"); + let mut empty = false; + if let Some(master_signing_key) = master_signing_key { + let (key, value) = parse_master_key(user_id, master_signing_key)?; + let result = services + .users + .get_master_key(None, user_id, &|_| true) + .await; + if result.is_not_found() { + empty = true; + } else { + let existing_master_key = result?; + let (existing_key, existing_value) = parse_master_key(user_id, &existing_master_key)?; + if existing_key != key || existing_value != value { + return Err!(Request(Forbidden( + "Tried to change an existing master key, UIA required" + ))); + } + } + } + if let Some(user_signing_key) = user_signing_key { + let key = services.users.get_user_signing_key(user_id).await; + if key.is_not_found() && !empty { + return Err!(Request(Forbidden( + "Tried to update an existing user signing key, UIA required" + ))); + } + if !key.is_not_found() { + let existing_signing_key = key?.deserialize()?; + if existing_signing_key != user_signing_key.deserialize()? { + return Err!(Request(Forbidden( + "Tried to change an existing user signing key, UIA required" + ))); + } + } + } + if let Some(self_signing_key) = self_signing_key { + let key = services + .users + .get_self_signing_key(None, user_id, &|_| true) + .await; + if key.is_not_found() && !empty { + debug!(?key); + return Err!(Request(Forbidden( + "Tried to add a new signing key independently from the master key" + ))); + } + if !key.is_not_found() { + let existing_signing_key = key?.deserialize()?; + if existing_signing_key != self_signing_key.deserialize()? { + return Err!(Request(Forbidden( + "Tried to update an existing self signing key, UIA required" + ))); + } + } + } + if empty { + return Ok(None); + } + + Ok(Some(upload_signing_keys::v3::Response {})) +} + /// # `POST /_matrix/client/r0/keys/signatures/upload` /// /// Uploads end-to-end key signatures from the sender user. @@ -407,7 +493,9 @@ where * resulting in an endless loop */ ) .await?; - master_keys.insert(user.clone(), raw); + if let Some(raw) = raw { + master_keys.insert(user.clone(), raw); + } } self_signing_keys.extend(response.self_signing_keys); diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 2e615a0c..bc18377e 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -585,12 +585,10 @@ async fn handle_edu_signing_key_update( return; } - if let Some(master_key) = master_key { - services - .users - .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true) - .await - .log_err() - .ok(); - } + services + .users + .add_cross_signing_keys(&user_id, &master_key, &self_signing_key, &None, true) + .await + .log_err() + .ok(); } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 68b87541..f0389a4a 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -514,7 +514,7 @@ impl Service { pub async fn add_cross_signing_keys( &self, user_id: &UserId, - master_key: &Raw, + master_key: &Option>, self_signing_key: &Option>, user_signing_key: &Option>, notify: bool, @@ -523,15 +523,17 @@ impl Service { let mut prefix = user_id.as_bytes().to_vec(); prefix.push(0xFF); - let (master_key_key, _) = parse_master_key(user_id, master_key)?; + if let Some(master_key) = master_key { + let (master_key_key, _) = parse_master_key(user_id, master_key)?; - self.db - .keyid_key - .insert(&master_key_key, master_key.json().get().as_bytes()); + self.db + .keyid_key + .insert(&master_key_key, master_key.json().get().as_bytes()); - self.db - .userid_masterkeyid - .insert(user_id.as_bytes(), &master_key_key); + self.db + .userid_masterkeyid + .insert(user_id.as_bytes(), &master_key_key); + } // Self-signing key if let Some(self_signing_key) = self_signing_key { @@ -567,32 +569,16 @@ impl Service { // User-signing key if let Some(user_signing_key) = user_signing_key { - let mut user_signing_key_ids = user_signing_key - .deserialize() - .map_err(|_| err!(Request(InvalidParam("Invalid user signing key"))))? - .keys - .into_values(); - - let user_signing_key_id = user_signing_key_ids - .next() - .ok_or(err!(Request(InvalidParam("User signing key contained no key."))))?; - - if user_signing_key_ids.next().is_some() { - return Err!(Request(InvalidParam( - "User signing key contained more than one key." - ))); - } - - let mut user_signing_key_key = prefix; - user_signing_key_key.extend_from_slice(user_signing_key_id.as_bytes()); + let user_signing_key_id = parse_user_signing_key(user_signing_key)?; + let user_signing_key_key = (user_id, &user_signing_key_id); self.db .keyid_key - .insert(&user_signing_key_key, user_signing_key.json().get().as_bytes()); + .put_raw(user_signing_key_key, user_signing_key.json().get().as_bytes()); self.db .userid_usersigningkeyid - .insert(user_id.as_bytes(), &user_signing_key_key); + .put(user_id, user_signing_key_key); } if notify { @@ -1079,6 +1065,24 @@ pub fn parse_master_key( Ok((master_key_key, master_key)) } +pub fn parse_user_signing_key(user_signing_key: &Raw) -> Result { + let mut user_signing_key_ids = user_signing_key + .deserialize() + .map_err(|_| err!(Request(InvalidParam("Invalid user signing key"))))? + .keys + .into_values(); + + let user_signing_key_id = user_signing_key_ids + .next() + .ok_or(err!(Request(InvalidParam("User signing key contained no key."))))?; + + if user_signing_key_ids.next().is_some() { + return Err!(Request(InvalidParam("User signing key contained more than one key."))); + } + + Ok(user_signing_key_id) +} + /// Ensure that a user only sees signatures from themselves and the target user fn clean_signatures( mut cross_signing_key: serde_json::Value, From bec19df275f100f15fa58dc8654a2ec41958eacc Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 22 Feb 2025 17:12:31 +0100 Subject: [PATCH 1518/2091] increase compression levels for some column families again --- src/database/engine/descriptor.rs | 4 ++-- src/database/maps.rs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index 934ef831..816555d2 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -83,7 +83,7 @@ pub(crate) static RANDOM: Descriptor = Descriptor { write_size: 1024 * 1024 * 32, cache_shards: 128, compression_level: -3, - bottommost_level: Some(-1), + bottommost_level: Some(2), compressed_index: true, ..BASE }; @@ -95,7 +95,7 @@ pub(crate) static SEQUENTIAL: Descriptor = Descriptor { file_size: 1024 * 1024 * 2, cache_shards: 128, compression_level: -2, - bottommost_level: Some(-1), + bottommost_level: Some(2), compression_shape: [0, 0, 1, 1, 1, 1, 1], compressed_index: false, ..BASE diff --git a/src/database/maps.rs b/src/database/maps.rs index 9ae5ab44..fc216ee0 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -171,6 +171,8 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "roomsynctoken_shortstatehash", val_size_hint: Some(8), block_size: 512, + compression_level: 3, + bottommost_level: Some(6), ..descriptor::SEQUENTIAL }, Descriptor { From e97952b7f6d310d5954a0d9e6b8979d25b090387 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 23 Feb 2025 01:17:30 -0500 Subject: [PATCH 1519/2091] bump nix lock, update to rust 2024 and 1.85.0 toolchain Signed-off-by: June Clementine Strawberry --- Cargo.toml | 7 +++++-- flake.lock | 36 ++++++++++++++++++------------------ flake.nix | 2 +- rust-toolchain.toml | 2 +- rustfmt.toml | 2 +- 5 files changed, 26 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ed7e6ac3..76de2212 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,14 +13,14 @@ authors = [ ] categories = ["network-programming"] description = "a very cool Matrix chat homeserver written in Rust" -edition = "2021" +edition = "2024" homepage = "https://conduwuit.puppyirl.gay/" keywords = ["chat", "matrix", "networking", "server", "uwu"] license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" -rust-version = "1.84.0" +rust-version = "1.85.0" version = "0.5.0" [workspace.metadata.crane] @@ -975,3 +975,6 @@ suspicious = { level = "warn", priority = -1 } ## some sadness let_underscore_future = { level = "allow", priority = 1 } + +# rust doesnt understand conduwuit's custom log macros +literal_string_with_formatting_args = { level = "allow", priority = 1 } diff --git a/flake.lock b/flake.lock index 15040a42..9bf6ac55 100644 --- a/flake.lock +++ b/flake.lock @@ -10,11 +10,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1731270564, - "narHash": "sha256-6KMC/NH/VWP5Eb+hA56hz0urel3jP6Y6cF2PX6xaTkk=", + "lastModified": 1738524606, + "narHash": "sha256-hPYEJ4juK3ph7kbjbvv7PlU1D9pAkkhl+pwx8fZY53U=", "owner": "zhaofengli", "repo": "attic", - "rev": "47752427561f1c34debb16728a210d378f0ece36", + "rev": "ff8a897d1f4408ebbf4d45fa9049c06b3e1e3f4e", "type": "github" }, "original": { @@ -117,11 +117,11 @@ }, "crane_2": { "locked": { - "lastModified": 1737689766, - "narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=", + "lastModified": 1739936662, + "narHash": "sha256-x4syUjNUuRblR07nDPeLDP7DpphaBVbUaSoeZkFbGSk=", "owner": "ipetkov", "repo": "crane", - "rev": "6fe74265bbb6d016d663b1091f015e2976c4a527", + "rev": "19de14aaeb869287647d9461cbd389187d8ecdb7", "type": "github" }, "original": { @@ -170,11 +170,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1737786656, - "narHash": "sha256-ubCW9Jy7ZUOF354bWxTgLDpVnTvIpNr6qR4H/j7I0oo=", + "lastModified": 1740206139, + "narHash": "sha256-wWSv4KYhPKggKuJLzghfBs99pS3Kli9UBlyXVBzuIzc=", "owner": "nix-community", "repo": "fenix", - "rev": "2f721f527886f801403f389a9cabafda8f1e3b7f", + "rev": "133a9eb59fb4ddac443ebe5ab2449d3940396533", "type": "github" }, "original": { @@ -364,11 +364,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1737600516, - "narHash": "sha256-EKyLQ3pbcjoU5jH5atge59F4fzuhTsb6yalUj6Ve2t8=", + "lastModified": 1740063075, + "narHash": "sha256-AfrCMPiXwgB0yxociq4no4NjCqGf/nRVhC3CLRoKqhA=", "owner": "axboe", "repo": "liburing", - "rev": "6c509e2b0c881a13b83b259a221bf15fc9b3f681", + "rev": "5c788d514b9ed6d1a3624150de8aa6db403c1c65", "type": "github" }, "original": { @@ -550,11 +550,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1737717945, - "narHash": "sha256-ET91TMkab3PmOZnqiJQYOtSGvSTvGeHoegAv4zcTefM=", + "lastModified": 1740019556, + "narHash": "sha256-vn285HxnnlHLWnv59Og7muqECNMS33mWLM14soFIv2g=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ecd26a469ac56357fd333946a99086e992452b6a", + "rev": "dad564433178067be1fbdfcce23b546254b6d641", "type": "github" }, "original": { @@ -599,11 +599,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1737728869, - "narHash": "sha256-U4pl3Hi0lT6GP4ecN3q9wdD2sdaKMbmD/5NJ1NdJ9AM=", + "lastModified": 1740077634, + "narHash": "sha256-KlYdDhon/hy91NutuBeN8e3qTKf3FXgsudWsjnHud68=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "6e4c29f7ce18cea7d3d31237a4661ab932eab636", + "rev": "88fbdcd510e79ef3bcd81d6d9d4f07bdce84be8c", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 3cef1af5..04dee681 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,7 @@ file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` - sha256 = "sha256-lMLAupxng4Fd9F1oDw8gx+qA0RuF7ou7xhNU8wgs0PU="; + sha256 = "sha256-AJ6LX/Q/Er9kS15bn9iflkUwcgYqRQxiOIL2ToVAXaU="; }; mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 97e33c91..00fb6cee 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -9,7 +9,7 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.84.0" +channel = "1.85.0" profile = "minimal" components = [ # For rust-analyzer diff --git a/rustfmt.toml b/rustfmt.toml index 635ec8f8..89041b04 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -2,7 +2,7 @@ array_width = 80 chain_width = 60 comment_width = 80 condense_wildcard_suffixes = true -edition = "2024" +style_edition = "2024" fn_call_width = 80 fn_single_line = true format_code_in_doc_comments = true From a1e1f40deda8f974d61b0095fc41356cc3fda43f Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 23 Feb 2025 01:17:45 -0500 Subject: [PATCH 1520/2091] run cargo fix for rust 2024 changes and rustfmt Signed-off-by: June Clementine Strawberry --- src/admin/appservice/commands.rs | 2 +- src/admin/command.rs | 4 +- src/admin/debug/commands.rs | 37 +- src/admin/debug/tester.rs | 2 +- src/admin/federation/commands.rs | 2 +- src/admin/media/commands.rs | 157 +++---- src/admin/processor.rs | 14 +- src/admin/query/account_data.rs | 2 +- src/admin/query/raw.rs | 5 +- src/admin/query/resolver.rs | 4 +- src/admin/query/room_state_cache.rs | 2 +- src/admin/query/room_timeline.rs | 4 +- src/admin/query/sending.rs | 2 +- src/admin/query/short.rs | 2 +- src/admin/query/users.rs | 2 +- src/admin/room/alias.rs | 4 +- src/admin/room/commands.rs | 4 +- src/admin/room/directory.rs | 4 +- src/admin/room/info.rs | 4 +- src/admin/room/moderation.rs | 198 ++++----- src/admin/server/commands.rs | 2 +- src/admin/user/commands.rs | 8 +- src/admin/utils.rs | 2 +- src/api/client/account.rs | 201 +++++---- src/api/client/account_data.rs | 6 +- src/api/client/alias.rs | 30 +- src/api/client/appservice.rs | 2 +- src/api/client/backup.rs | 4 +- src/api/client/capabilities.rs | 2 +- src/api/client/context.rs | 11 +- src/api/client/device.rs | 86 ++-- src/api/client/directory.rs | 41 +- src/api/client/keys.rs | 130 +++--- src/api/client/media.rs | 7 +- src/api/client/media_legacy.rs | 252 ++++++------ src/api/client/membership.rs | 157 +++---- src/api/client/message.rs | 13 +- src/api/client/presence.rs | 57 +-- src/api/client/profile.rs | 9 +- src/api/client/push.rs | 6 +- src/api/client/read_marker.rs | 6 +- src/api/client/redact.rs | 2 +- src/api/client/relations.rs | 13 +- src/api/client/report.rs | 10 +- src/api/client/room/create.rs | 25 +- src/api/client/room/event.rs | 6 +- src/api/client/room/initial_sync.rs | 5 +- src/api/client/room/upgrade.rs | 7 +- src/api/client/search.rs | 11 +- src/api/client/send.rs | 4 +- src/api/client/session.rs | 68 ++-- src/api/client/space.rs | 12 +- src/api/client/state.rs | 6 +- src/api/client/sync/mod.rs | 14 +- src/api/client/sync/v3.rs | 65 +-- src/api/client/sync/v4.rs | 25 +- src/api/client/sync/v5.rs | 24 +- src/api/client/tag.rs | 2 +- src/api/client/threads.rs | 2 +- src/api/client/typing.rs | 65 +-- src/api/client/unstable.rs | 34 +- src/api/client/unversioned.rs | 2 +- src/api/client/user_directory.rs | 4 +- src/api/client/voip.rs | 6 +- src/api/client/well_known.rs | 2 +- src/api/mod.rs | 2 +- src/api/router.rs | 6 +- src/api/router/args.rs | 8 +- src/api/router/auth.rs | 24 +- src/api/router/handler.rs | 4 +- src/api/router/request.rs | 4 +- src/api/router/response.rs | 4 +- src/api/server/backfill.rs | 4 +- src/api/server/event.rs | 4 +- src/api/server/event_auth.rs | 4 +- src/api/server/get_missing_events.rs | 2 +- src/api/server/hierarchy.rs | 4 +- src/api/server/invite.rs | 6 +- src/api/server/key.rs | 8 +- src/api/server/make_join.rs | 8 +- src/api/server/make_knock.rs | 8 +- src/api/server/make_leave.rs | 2 +- src/api/server/media.rs | 8 +- src/api/server/query.rs | 4 +- src/api/server/send.rs | 12 +- src/api/server/send_join.rs | 16 +- src/api/server/send_knock.rs | 14 +- src/api/server/send_leave.rs | 8 +- src/api/server/state.rs | 4 +- src/api/server/state_ids.rs | 4 +- src/api/server/user.rs | 2 +- src/api/server/utils.rs | 4 +- src/core/alloc/je.rs | 25 +- src/core/config/check.rs | 2 +- src/core/config/manager.rs | 4 +- src/core/config/mod.rs | 10 +- src/core/debug.rs | 2 +- src/core/error/err.rs | 4 +- src/core/error/mod.rs | 4 +- src/core/error/panic.rs | 2 +- src/core/error/response.rs | 2 +- src/core/info/room_version.rs | 2 +- src/core/log/capture/data.rs | 4 +- src/core/log/capture/util.rs | 2 +- src/core/log/console.rs | 6 +- src/core/log/fmt.rs | 2 +- src/core/log/mod.rs | 4 +- src/core/log/reload.rs | 4 +- src/core/mods/module.rs | 4 +- src/core/mods/new.rs | 2 +- src/core/pdu/builder.rs | 4 +- src/core/pdu/content.rs | 2 +- src/core/pdu/count.rs | 2 +- src/core/pdu/event.rs | 2 +- src/core/pdu/event_id.rs | 2 +- src/core/pdu/mod.rs | 6 +- src/core/pdu/raw_id.rs | 4 +- src/core/pdu/redact.rs | 31 +- src/core/pdu/strip.rs | 2 +- src/core/pdu/unsigned.rs | 4 +- src/core/server.rs | 4 +- src/core/state_res/event_auth.rs | 383 +++++++++--------- src/core/state_res/mod.rs | 25 +- src/core/state_res/power_levels.rs | 8 +- src/core/state_res/state_event.rs | 2 +- src/core/state_res/test_utils.rs | 16 +- src/core/utils/bytes.rs | 2 +- src/core/utils/defer.rs | 4 +- src/core/utils/future/bool_ext.rs | 2 +- src/core/utils/future/ext_ext.rs | 2 +- src/core/utils/future/mod.rs | 2 +- src/core/utils/future/option_ext.rs | 2 +- src/core/utils/future/try_ext_ext.rs | 3 +- src/core/utils/hash/argon.rs | 6 +- src/core/utils/json.rs | 2 +- src/core/utils/math.rs | 2 +- src/core/utils/math/tried.rs | 2 +- src/core/utils/mod.rs | 8 +- src/core/utils/mutex_map.rs | 2 +- src/core/utils/rand.rs | 2 +- src/core/utils/stream/broadband.rs | 4 +- src/core/utils/stream/cloned.rs | 2 +- src/core/utils/stream/ignore.rs | 2 +- src/core/utils/stream/iter_stream.rs | 3 +- src/core/utils/stream/mod.rs | 4 +- src/core/utils/stream/ready.rs | 2 +- src/core/utils/stream/try_parallel.rs | 4 +- src/core/utils/stream/try_ready.rs | 2 +- src/core/utils/stream/try_tools.rs | 2 +- src/core/utils/stream/wideband.rs | 4 +- src/core/utils/string.rs | 2 +- src/core/utils/string/unquoted.rs | 4 +- src/core/utils/sys.rs | 4 +- src/core/utils/sys/compute.rs | 6 +- src/core/utils/sys/storage.rs | 4 +- src/core/utils/tests.rs | 2 +- src/core/utils/time.rs | 2 +- src/database/de.rs | 5 +- src/database/engine.rs | 6 +- src/database/engine/backup.rs | 2 +- src/database/engine/cf_opts.rs | 4 +- src/database/engine/context.rs | 2 +- src/database/engine/db_opts.rs | 4 +- src/database/engine/files.rs | 4 +- src/database/engine/memory_usage.rs | 2 +- src/database/engine/open.rs | 8 +- src/database/engine/repair.rs | 2 +- src/database/handle.rs | 2 +- src/database/keyval.rs | 2 +- src/database/map.rs | 2 +- src/database/map/compact.rs | 2 +- src/database/map/contains.rs | 16 +- src/database/map/count.rs | 7 +- src/database/map/get.rs | 11 +- src/database/map/get_batch.rs | 11 +- src/database/map/keys.rs | 2 +- src/database/map/keys_from.rs | 18 +- src/database/map/keys_prefix.rs | 10 +- src/database/map/qry.rs | 13 +- src/database/map/qry_batch.rs | 7 +- src/database/map/rev_keys.rs | 2 +- src/database/map/rev_keys_from.rs | 12 +- src/database/map/rev_keys_prefix.rs | 10 +- src/database/map/rev_stream.rs | 2 +- src/database/map/rev_stream_from.rs | 12 +- src/database/map/rev_stream_prefix.rs | 10 +- src/database/map/stream.rs | 2 +- src/database/map/stream_from.rs | 12 +- src/database/map/stream_prefix.rs | 10 +- src/database/maps.rs | 2 +- src/database/mod.rs | 10 +- src/database/pool.rs | 9 +- src/database/pool/configure.rs | 3 +- src/database/ser.rs | 4 +- src/database/stream.rs | 4 +- src/database/stream/items.rs | 4 +- src/database/stream/items_rev.rs | 4 +- src/database/stream/keys.rs | 4 +- src/database/stream/keys_rev.rs | 4 +- src/database/tests.rs | 7 +- src/database/watchers.rs | 2 +- src/macros/admin.rs | 6 +- src/macros/cargo.rs | 2 +- src/macros/config.rs | 8 +- src/macros/implement.rs | 2 +- src/macros/mod.rs | 3 +- src/macros/refutable.rs | 4 +- src/macros/utils.rs | 2 +- src/main/clap.rs | 2 +- src/main/logging.rs | 6 +- src/main/main.rs | 4 +- src/main/mods.rs | 4 +- src/main/runtime.rs | 5 +- src/main/sentry.rs | 8 +- src/main/server.rs | 2 +- src/router/layers.rs | 18 +- src/router/request.rs | 4 +- src/router/router.rs | 2 +- src/router/run.rs | 4 +- src/router/serve/mod.rs | 2 +- src/router/serve/plain.rs | 6 +- src/router/serve/tls.rs | 4 +- src/router/serve/unix.rs | 10 +- src/service/account_data/mod.rs | 9 +- src/service/admin/console.rs | 8 +- src/service/admin/create.rs | 4 +- src/service/admin/execute.rs | 4 +- src/service/admin/grant.rs | 6 +- src/service/admin/mod.rs | 6 +- src/service/appservice/mod.rs | 6 +- src/service/appservice/registration_info.rs | 2 +- src/service/client/mod.rs | 9 +- src/service/config/mod.rs | 5 +- src/service/emergency/mod.rs | 6 +- src/service/federation/execute.rs | 12 +- src/service/federation/mod.rs | 2 +- src/service/globals/data.rs | 2 +- src/service/globals/mod.rs | 2 +- src/service/key_backups/mod.rs | 7 +- src/service/manager.rs | 4 +- src/service/media/blurhash.rs | 2 +- src/service/media/data.rs | 7 +- src/service/media/migrations.rs | 6 +- src/service/media/mod.rs | 70 ++-- src/service/media/preview.rs | 2 +- src/service/media/remote.rs | 10 +- src/service/media/tests.rs | 2 +- src/service/media/thumbnail.rs | 18 +- src/service/migrations.rs | 12 +- src/service/mod.rs | 2 +- src/service/presence/data.rs | 9 +- src/service/presence/mod.rs | 8 +- src/service/presence/presence.rs | 4 +- src/service/pusher/mod.rs | 17 +- src/service/resolver/actual.rs | 26 +- src/service/resolver/cache.rs | 6 +- src/service/resolver/dns.rs | 4 +- src/service/resolver/mod.rs | 4 +- src/service/resolver/tests.rs | 2 +- src/service/rooms/alias/mod.rs | 15 +- src/service/rooms/alias/remote.rs | 4 +- src/service/rooms/auth_chain/data.rs | 2 +- src/service/rooms/auth_chain/mod.rs | 8 +- src/service/rooms/directory/mod.rs | 4 +- src/service/rooms/event_handler/acl_check.rs | 4 +- .../fetch_and_handle_outliers.rs | 23 +- src/service/rooms/event_handler/fetch_prev.rs | 90 ++-- .../rooms/event_handler/fetch_state.rs | 8 +- .../event_handler/handle_incoming_pdu.rs | 8 +- .../rooms/event_handler/handle_outlier_pdu.rs | 10 +- .../rooms/event_handler/handle_prev_pdu.rs | 4 +- src/service/rooms/event_handler/mod.rs | 8 +- .../rooms/event_handler/parse_incoming_pdu.rs | 2 +- .../rooms/event_handler/resolve_state.rs | 13 +- .../rooms/event_handler/state_at_incoming.rs | 5 +- .../event_handler/upgrade_outlier_pdu.rs | 9 +- src/service/rooms/lazy_loading/mod.rs | 9 +- src/service/rooms/metadata/mod.rs | 4 +- src/service/rooms/outlier/mod.rs | 2 +- src/service/rooms/pdu_metadata/data.rs | 10 +- src/service/rooms/pdu_metadata/mod.rs | 8 +- src/service/rooms/read_receipt/data.rs | 8 +- src/service/rooms/read_receipt/mod.rs | 24 +- src/service/rooms/search/mod.rs | 16 +- src/service/rooms/short/mod.rs | 6 +- src/service/rooms/spaces/mod.rs | 18 +- src/service/rooms/spaces/pagination_token.rs | 2 +- src/service/rooms/spaces/tests.rs | 4 +- src/service/rooms/state/mod.rs | 132 +++--- src/service/rooms/state_accessor/mod.rs | 13 +- .../rooms/state_accessor/room_state.rs | 4 +- .../rooms/state_accessor/server_can.rs | 8 +- src/service/rooms/state_accessor/state.rs | 15 +- src/service/rooms/state_accessor/user_can.rs | 64 +-- src/service/rooms/state_cache/mod.rs | 18 +- src/service/rooms/state_compressor/mod.rs | 5 +- src/service/rooms/threads/mod.rs | 20 +- src/service/rooms/timeline/data.rs | 9 +- src/service/rooms/timeline/mod.rs | 23 +- src/service/rooms/typing/mod.rs | 9 +- src/service/rooms/user/mod.rs | 4 +- src/service/sending/appservice.rs | 4 +- src/service/sending/data.rs | 11 +- src/service/sending/mod.rs | 12 +- src/service/sending/sender.rs | 30 +- src/service/server_keys/acquire.rs | 8 +- src/service/server_keys/get.rs | 8 +- src/service/server_keys/keypair.rs | 2 +- src/service/server_keys/mod.rs | 11 +- src/service/server_keys/request.rs | 14 +- src/service/server_keys/sign.rs | 2 +- src/service/server_keys/verify.rs | 4 +- src/service/service.rs | 2 +- src/service/services.rs | 2 +- src/service/sync/mod.rs | 4 +- src/service/sync/watch.rs | 4 +- src/service/transaction_ids/mod.rs | 2 +- src/service/uiaa/mod.rs | 10 +- src/service/updates/mod.rs | 6 +- src/service/users/mod.rs | 24 +- 320 files changed, 2212 insertions(+), 2039 deletions(-) diff --git a/src/admin/appservice/commands.rs b/src/admin/appservice/commands.rs index 4f02531a..88f28431 100644 --- a/src/admin/appservice/commands.rs +++ b/src/admin/appservice/commands.rs @@ -1,6 +1,6 @@ use ruma::{api::appservice::Registration, events::room::message::RoomMessageEventContent}; -use crate::{admin_command, Result}; +use crate::{Result, admin_command}; #[admin_command] pub(super) async fn register(&self) -> Result { diff --git a/src/admin/command.rs b/src/admin/command.rs index 5ad9e581..5df980d6 100644 --- a/src/admin/command.rs +++ b/src/admin/command.rs @@ -3,9 +3,9 @@ use std::{fmt, time::SystemTime}; use conduwuit::Result; use conduwuit_service::Services; use futures::{ + Future, FutureExt, io::{AsyncWriteExt, BufWriter}, lock::Mutex, - Future, FutureExt, }; use ruma::EventId; @@ -21,7 +21,7 @@ impl Command<'_> { pub(crate) fn write_fmt( &self, arguments: fmt::Arguments<'_>, - ) -> impl Future + Send + '_ { + ) -> impl Future + Send + '_ + use<'_> { let buf = format!("{arguments}"); self.output.lock().then(|mut output| async move { output.write_all(buf.as_bytes()).await.map_err(Into::into) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index dcf9879c..c6f6a170 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,19 +6,19 @@ use std::{ }; use conduwuit::{ - debug_error, err, info, trace, utils, + Error, PduEvent, PduId, RawPduId, Result, debug_error, err, info, trace, utils, utils::{ stream::{IterStream, ReadyExt}, string::EMPTY, }, - warn, Error, PduEvent, PduId, RawPduId, Result, + warn, }; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ - api::{client::error::ErrorKind, federation::event::get_room_state}, - events::room::message::RoomMessageEventContent, CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, ServerName, + api::{client::error::ErrorKind, federation::event::get_room_state}, + events::room::message::RoomMessageEventContent, }; use service::rooms::{ short::{ShortEventId, ShortRoomId}, @@ -209,18 +209,21 @@ pub(super) async fn get_remote_pdu_list( for pdu in list { if force { - if let Err(e) = self.get_remote_pdu(Box::from(pdu), server.clone()).await { - failed_count = failed_count.saturating_add(1); - self.services - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "Failed to get remote PDU, ignoring error: {e}" - ))) - .await - .ok(); - warn!("Failed to get remote PDU, ignoring error: {e}"); - } else { - success_count = success_count.saturating_add(1); + match self.get_remote_pdu(Box::from(pdu), server.clone()).await { + | Err(e) => { + failed_count = failed_count.saturating_add(1); + self.services + .admin + .send_message(RoomMessageEventContent::text_plain(format!( + "Failed to get remote PDU, ignoring error: {e}" + ))) + .await + .ok(); + warn!("Failed to get remote PDU, ignoring error: {e}"); + }, + | _ => { + success_count = success_count.saturating_add(1); + }, } } else { self.get_remote_pdu(Box::from(pdu), server.clone()).await?; @@ -957,7 +960,7 @@ pub(super) async fn database_stats( self.services .db .iter() - .filter(|(&name, _)| map_name.is_empty() || map_name == name) + .filter(|&(&name, _)| map_name.is_empty() || map_name == name) .try_stream() .try_for_each(|(&name, map)| { let res = map.property(&property).expect("invalid property"); diff --git a/src/admin/debug/tester.rs b/src/admin/debug/tester.rs index 5200fa0d..005ee775 100644 --- a/src/admin/debug/tester.rs +++ b/src/admin/debug/tester.rs @@ -1,7 +1,7 @@ use conduwuit::Err; use ruma::events::room::message::RoomMessageEventContent; -use crate::{admin_command, admin_command_dispatch, Result}; +use crate::{Result, admin_command, admin_command_dispatch}; #[admin_command_dispatch] #[derive(Debug, clap::Subcommand)] diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 13bc8da4..240ffa6a 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -3,7 +3,7 @@ use std::fmt::Write; use conduwuit::Result; use futures::StreamExt; use ruma::{ - events::room::message::RoomMessageEventContent, OwnedRoomId, RoomId, ServerName, UserId, + OwnedRoomId, RoomId, ServerName, UserId, events::room::message::RoomMessageEventContent, }; use crate::{admin_command, get_room_info}; diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index 3d0a9473..aeefa9f2 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -1,12 +1,12 @@ use std::time::Duration; use conduwuit::{ - debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, Result, + Result, debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, }; use conduwuit_service::media::Dim; use ruma::{ - events::room::message::RoomMessageEventContent, EventId, Mxc, MxcUri, OwnedMxcUri, - OwnedServerName, ServerName, + EventId, Mxc, MxcUri, OwnedMxcUri, OwnedServerName, ServerName, + events::room::message::RoomMessageEventContent, }; use crate::{admin_command, utils::parse_local_user_id}; @@ -41,103 +41,106 @@ pub(super) async fn delete( let mut mxc_urls = Vec::with_capacity(4); // parsing the PDU for any MXC URLs begins here - if let Ok(event_json) = self.services.rooms.timeline.get_pdu_json(&event_id).await { - if let Some(content_key) = event_json.get("content") { - debug!("Event ID has \"content\"."); - let content_obj = content_key.as_object(); + match self.services.rooms.timeline.get_pdu_json(&event_id).await { + | Ok(event_json) => { + if let Some(content_key) = event_json.get("content") { + debug!("Event ID has \"content\"."); + let content_obj = content_key.as_object(); - if let Some(content) = content_obj { - // 1. attempts to parse the "url" key - debug!("Attempting to go into \"url\" key for main media file"); - if let Some(url) = content.get("url") { - debug!("Got a URL in the event ID {event_id}: {url}"); + if let Some(content) = content_obj { + // 1. attempts to parse the "url" key + debug!("Attempting to go into \"url\" key for main media file"); + if let Some(url) = content.get("url") { + debug!("Got a URL in the event ID {event_id}: {url}"); - if url.to_string().starts_with("\"mxc://") { - debug!("Pushing URL {url} to list of MXCs to delete"); - let final_url = url.to_string().replace('"', ""); - mxc_urls.push(final_url); - } else { - info!( - "Found a URL in the event ID {event_id} but did not start with \ - mxc://, ignoring" - ); - } - } - - // 2. attempts to parse the "info" key - debug!("Attempting to go into \"info\" key for thumbnails"); - if let Some(info_key) = content.get("info") { - debug!("Event ID has \"info\"."); - let info_obj = info_key.as_object(); - - if let Some(info) = info_obj { - if let Some(thumbnail_url) = info.get("thumbnail_url") { - debug!("Found a thumbnail_url in info key: {thumbnail_url}"); - - if thumbnail_url.to_string().starts_with("\"mxc://") { - debug!( - "Pushing thumbnail URL {thumbnail_url} to list of MXCs \ - to delete" - ); - let final_thumbnail_url = - thumbnail_url.to_string().replace('"', ""); - mxc_urls.push(final_thumbnail_url); - } else { - info!( - "Found a thumbnail URL in the event ID {event_id} but \ - did not start with mxc://, ignoring" - ); - } + if url.to_string().starts_with("\"mxc://") { + debug!("Pushing URL {url} to list of MXCs to delete"); + let final_url = url.to_string().replace('"', ""); + mxc_urls.push(final_url); } else { info!( - "No \"thumbnail_url\" key in \"info\" key, assuming no \ - thumbnails." + "Found a URL in the event ID {event_id} but did not start \ + with mxc://, ignoring" ); } } - } - // 3. attempts to parse the "file" key - debug!("Attempting to go into \"file\" key"); - if let Some(file_key) = content.get("file") { - debug!("Event ID has \"file\"."); - let file_obj = file_key.as_object(); + // 2. attempts to parse the "info" key + debug!("Attempting to go into \"info\" key for thumbnails"); + if let Some(info_key) = content.get("info") { + debug!("Event ID has \"info\"."); + let info_obj = info_key.as_object(); - if let Some(file) = file_obj { - if let Some(url) = file.get("url") { - debug!("Found url in file key: {url}"); + if let Some(info) = info_obj { + if let Some(thumbnail_url) = info.get("thumbnail_url") { + debug!("Found a thumbnail_url in info key: {thumbnail_url}"); - if url.to_string().starts_with("\"mxc://") { - debug!("Pushing URL {url} to list of MXCs to delete"); - let final_url = url.to_string().replace('"', ""); - mxc_urls.push(final_url); + if thumbnail_url.to_string().starts_with("\"mxc://") { + debug!( + "Pushing thumbnail URL {thumbnail_url} to list of \ + MXCs to delete" + ); + let final_thumbnail_url = + thumbnail_url.to_string().replace('"', ""); + mxc_urls.push(final_thumbnail_url); + } else { + info!( + "Found a thumbnail URL in the event ID {event_id} \ + but did not start with mxc://, ignoring" + ); + } } else { info!( - "Found a URL in the event ID {event_id} but did not \ - start with mxc://, ignoring" + "No \"thumbnail_url\" key in \"info\" key, assuming no \ + thumbnails." ); } - } else { - info!("No \"url\" key in \"file\" key."); } } + + // 3. attempts to parse the "file" key + debug!("Attempting to go into \"file\" key"); + if let Some(file_key) = content.get("file") { + debug!("Event ID has \"file\"."); + let file_obj = file_key.as_object(); + + if let Some(file) = file_obj { + if let Some(url) = file.get("url") { + debug!("Found url in file key: {url}"); + + if url.to_string().starts_with("\"mxc://") { + debug!("Pushing URL {url} to list of MXCs to delete"); + let final_url = url.to_string().replace('"', ""); + mxc_urls.push(final_url); + } else { + info!( + "Found a URL in the event ID {event_id} but did not \ + start with mxc://, ignoring" + ); + } + } else { + info!("No \"url\" key in \"file\" key."); + } + } + } + } else { + return Ok(RoomMessageEventContent::text_plain( + "Event ID does not have a \"content\" key or failed parsing the \ + event ID JSON.", + )); } } else { return Ok(RoomMessageEventContent::text_plain( - "Event ID does not have a \"content\" key or failed parsing the event \ - ID JSON.", + "Event ID does not have a \"content\" key, this is not a message or an \ + event type that contains media.", )); } - } else { + }, + | _ => { return Ok(RoomMessageEventContent::text_plain( - "Event ID does not have a \"content\" key, this is not a message or an \ - event type that contains media.", + "Event ID does not exist or is not known to us.", )); - } - } else { - return Ok(RoomMessageEventContent::text_plain( - "Event ID does not exist or is not known to us.", - )); + }, } if mxc_urls.is_empty() { diff --git a/src/admin/processor.rs b/src/admin/processor.rs index eefcdcd6..77a60959 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -8,7 +8,7 @@ use std::{ use clap::{CommandFactory, Parser}; use conduwuit::{ - debug, error, + Error, Result, debug, error, log::{ capture, capture::Capture, @@ -16,24 +16,24 @@ use conduwuit::{ }, trace, utils::string::{collect_stream, common_prefix}, - warn, Error, Result, + warn, }; -use futures::{future::FutureExt, io::BufWriter, AsyncWriteExt}; +use futures::{AsyncWriteExt, future::FutureExt, io::BufWriter}; use ruma::{ + EventId, events::{ relation::InReplyTo, room::message::{Relation::Reply, RoomMessageEventContent}, }, - EventId, }; use service::{ - admin::{CommandInput, CommandOutput, ProcessorFuture, ProcessorResult}, Services, + admin::{CommandInput, CommandOutput, ProcessorFuture, ProcessorResult}, }; use tracing::Level; -use tracing_subscriber::{filter::LevelFilter, EnvFilter}; +use tracing_subscriber::{EnvFilter, filter::LevelFilter}; -use crate::{admin, admin::AdminCommand, Command}; +use crate::{Command, admin, admin::AdminCommand}; #[must_use] pub(super) fn complete(line: &str) -> String { complete_command(AdminCommand::command(), line) } diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index bb8ddeff..b2bf5e6d 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, RoomId, UserId}; +use ruma::{RoomId, UserId, events::room::message::RoomMessageEventContent}; use crate::{admin_command, admin_command_dispatch}; diff --git a/src/admin/query/raw.rs b/src/admin/query/raw.rs index 5a6006ec..23f11cc8 100644 --- a/src/admin/query/raw.rs +++ b/src/admin/query/raw.rs @@ -2,13 +2,12 @@ use std::{borrow::Cow, collections::BTreeMap, ops::Deref}; use clap::Subcommand; use conduwuit::{ - apply, at, is_zero, + Err, Result, apply, at, is_zero, utils::{ + IterStream, stream::{ReadyExt, TryIgnore, TryParallelExt}, string::EMPTY, - IterStream, }, - Err, Result, }; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::events::room::message::RoomMessageEventContent; diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index 08b5d171..10748d88 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -1,7 +1,7 @@ use clap::Subcommand; -use conduwuit::{utils::time, Result}; +use conduwuit::{Result, utils::time}; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, OwnedServerName}; +use ruma::{OwnedServerName, events::room::message::RoomMessageEventContent}; use crate::{admin_command, admin_command_dispatch}; diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index 71dadc99..1de5c02d 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::{Error, Result}; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, RoomId, ServerName, UserId}; +use ruma::{RoomId, ServerName, UserId, events::room::message::RoomMessageEventContent}; use crate::Command; diff --git a/src/admin/query/room_timeline.rs b/src/admin/query/room_timeline.rs index 3fe653e3..6f08aee9 100644 --- a/src/admin/query/room_timeline.rs +++ b/src/admin/query/room_timeline.rs @@ -1,7 +1,7 @@ use clap::Subcommand; -use conduwuit::{utils::stream::TryTools, PduCount, Result}; +use conduwuit::{PduCount, Result, utils::stream::TryTools}; use futures::TryStreamExt; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomOrAliasId}; +use ruma::{OwnedRoomOrAliasId, events::room::message::RoomMessageEventContent}; use crate::{admin_command, admin_command_dispatch}; diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 8c6fb25f..a148f718 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, ServerName, UserId}; +use ruma::{ServerName, UserId, events::room::message::RoomMessageEventContent}; use service::sending::Destination; use crate::Command; diff --git a/src/admin/query/short.rs b/src/admin/query/short.rs index 7f0f3449..0957c15e 100644 --- a/src/admin/query/short.rs +++ b/src/admin/query/short.rs @@ -1,6 +1,6 @@ use clap::Subcommand; use conduwuit::Result; -use ruma::{events::room::message::RoomMessageEventContent, OwnedEventId, OwnedRoomOrAliasId}; +use ruma::{OwnedEventId, OwnedRoomOrAliasId, events::room::message::RoomMessageEventContent}; use crate::{admin_command, admin_command_dispatch}; diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index c517d9dd..5995bc62 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -2,7 +2,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::stream::StreamExt; use ruma::{ - events::room::message::RoomMessageEventContent, OwnedDeviceId, OwnedRoomId, OwnedUserId, + OwnedDeviceId, OwnedRoomId, OwnedUserId, events::room::message::RoomMessageEventContent, }; use crate::{admin_command, admin_command_dispatch}; diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index d3b956e1..6262f33e 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -4,10 +4,10 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; use ruma::{ - events::room::message::RoomMessageEventContent, OwnedRoomAliasId, OwnedRoomId, RoomId, + OwnedRoomAliasId, OwnedRoomId, RoomId, events::room::message::RoomMessageEventContent, }; -use crate::{escape_html, Command}; +use crate::{Command, escape_html}; #[derive(Debug, Subcommand)] pub(crate) enum RoomAliasCommand { diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index b58d04c5..b5c303c8 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -1,8 +1,8 @@ use conduwuit::Result; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, OwnedRoomId}; +use ruma::{OwnedRoomId, events::room::message::RoomMessageEventContent}; -use crate::{admin_command, get_room_info, PAGE_SIZE}; +use crate::{PAGE_SIZE, admin_command, get_room_info}; #[admin_command] pub(super) async fn list_rooms( diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index 791b9204..e9c23a1d 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -1,9 +1,9 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, RoomId}; +use ruma::{RoomId, events::room::message::RoomMessageEventContent}; -use crate::{get_room_info, Command, PAGE_SIZE}; +use crate::{Command, PAGE_SIZE, get_room_info}; #[derive(Debug, Subcommand)] pub(crate) enum RoomDirectoryCommand { diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index 34abf8a9..a39728fe 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -1,7 +1,7 @@ use clap::Subcommand; -use conduwuit::{utils::ReadyExt, Result}; +use conduwuit::{Result, utils::ReadyExt}; use futures::StreamExt; -use ruma::{events::room::message::RoomMessageEventContent, RoomId}; +use ruma::{RoomId, events::room::message::RoomMessageEventContent}; use crate::{admin_command, admin_command_dispatch}; diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index bf54505e..ee132590 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -1,14 +1,14 @@ use api::client::leave_room; use clap::Subcommand; use conduwuit::{ - debug, error, info, + Result, debug, error, info, utils::{IterStream, ReadyExt}, - warn, Result, + warn, }; use futures::StreamExt; use ruma::{ - events::room::message::RoomMessageEventContent, OwnedRoomId, RoomAliasId, RoomId, - RoomOrAliasId, + OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId, + events::room::message::RoomMessageEventContent, }; use crate::{admin_command, admin_command_dispatch, get_room_info}; @@ -124,41 +124,42 @@ async fn ban_room( locally, if not using get_alias_helper to fetch room ID remotely" ); - let room_id = if let Ok(room_id) = self + let room_id = match self .services .rooms .alias .resolve_local_alias(room_alias) .await { - room_id - } else { - debug!( - "We don't have this room alias to a room ID locally, attempting to fetch room \ - ID over federation" - ); + | Ok(room_id) => room_id, + | _ => { + debug!( + "We don't have this room alias to a room ID locally, attempting to fetch \ + room ID over federation" + ); - match self - .services - .rooms - .alias - .resolve_alias(room_alias, None) - .await - { - | Ok((room_id, servers)) => { - debug!( - ?room_id, - ?servers, - "Got federation response fetching room ID for {room_id}" - ); - room_id - }, - | Err(e) => { - return Ok(RoomMessageEventContent::notice_plain(format!( - "Failed to resolve room alias {room_alias} to a room ID: {e}" - ))); - }, - } + match self + .services + .rooms + .alias + .resolve_alias(room_alias, None) + .await + { + | Ok((room_id, servers)) => { + debug!( + ?room_id, + ?servers, + "Got federation response fetching room ID for {room_id}" + ); + room_id + }, + | Err(e) => { + return Ok(RoomMessageEventContent::notice_plain(format!( + "Failed to resolve room alias {room_alias} to a room ID: {e}" + ))); + }, + } + }, }; self.services.rooms.metadata.ban_room(&room_id, true); @@ -321,51 +322,55 @@ async fn ban_list_of_rooms( if room_alias_or_id.is_room_alias_id() { match RoomAliasId::parse(room_alias_or_id) { | Ok(room_alias) => { - let room_id = if let Ok(room_id) = self + let room_id = match self .services .rooms .alias .resolve_local_alias(room_alias) .await { - room_id - } else { - debug!( - "We don't have this room alias to a room ID locally, \ - attempting to fetch room ID over federation" - ); + | Ok(room_id) => room_id, + | _ => { + debug!( + "We don't have this room alias to a room ID locally, \ + attempting to fetch room ID over federation" + ); - match self - .services - .rooms - .alias - .resolve_alias(room_alias, None) - .await - { - | Ok((room_id, servers)) => { - debug!( - ?room_id, - ?servers, - "Got federation response fetching room ID for {room}", - ); - room_id - }, - | Err(e) => { - // don't fail if force blocking - if force { - warn!( - "Failed to resolve room alias {room} to a room \ - ID: {e}" + match self + .services + .rooms + .alias + .resolve_alias(room_alias, None) + .await + { + | Ok((room_id, servers)) => { + debug!( + ?room_id, + ?servers, + "Got federation response fetching room ID for \ + {room}", ); - continue; - } + room_id + }, + | Err(e) => { + // don't fail if force blocking + if force { + warn!( + "Failed to resolve room alias {room} to a \ + room ID: {e}" + ); + continue; + } - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to resolve room alias {room} to a room ID: \ - {e}" - ))); - }, - } + return Ok(RoomMessageEventContent::text_plain( + format!( + "Failed to resolve room alias {room} to a \ + room ID: {e}" + ), + )); + }, + } + }, }; room_ids.push(room_id); @@ -537,41 +542,42 @@ async fn unban_room( locally, if not using get_alias_helper to fetch room ID remotely" ); - let room_id = if let Ok(room_id) = self + let room_id = match self .services .rooms .alias .resolve_local_alias(room_alias) .await { - room_id - } else { - debug!( - "We don't have this room alias to a room ID locally, attempting to fetch room \ - ID over federation" - ); + | Ok(room_id) => room_id, + | _ => { + debug!( + "We don't have this room alias to a room ID locally, attempting to fetch \ + room ID over federation" + ); - match self - .services - .rooms - .alias - .resolve_alias(room_alias, None) - .await - { - | Ok((room_id, servers)) => { - debug!( - ?room_id, - ?servers, - "Got federation response fetching room ID for room {room}" - ); - room_id - }, - | Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to resolve room alias {room} to a room ID: {e}" - ))); - }, - } + match self + .services + .rooms + .alias + .resolve_alias(room_alias, None) + .await + { + | Ok((room_id, servers)) => { + debug!( + ?room_id, + ?servers, + "Got federation response fetching room ID for room {room}" + ); + room_id + }, + | Err(e) => { + return Ok(RoomMessageEventContent::text_plain(format!( + "Failed to resolve room alias {room} to a room ID: {e}" + ))); + }, + } + }, }; self.services.rooms.metadata.ban_room(&room_id, false); diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index d4cfa7d5..17bf9ec0 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,6 +1,6 @@ use std::{fmt::Write, path::PathBuf, sync::Arc}; -use conduwuit::{info, utils::time, warn, Err, Result}; +use conduwuit::{Err, Result, info, utils::time, warn}; use ruma::events::room::message::RoomMessageEventContent; use crate::admin_command; diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 64767a36..8cb8edc3 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -2,23 +2,23 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room}; use conduwuit::{ - debug_warn, error, info, is_equal_to, + PduBuilder, Result, debug_warn, error, info, is_equal_to, utils::{self, ReadyExt}, - warn, PduBuilder, Result, + warn, }; use conduwuit_api::client::{leave_all_rooms, update_avatar_url, update_displayname}; use futures::StreamExt; use ruma::{ + EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, UserId, events::{ + RoomAccountDataEventType, StateEventType, room::{ message::RoomMessageEventContent, power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, redaction::RoomRedactionEventContent, }, tag::{TagEvent, TagEventContent, TagInfo}, - RoomAccountDataEventType, StateEventType, }, - EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, UserId, }; use crate::{ diff --git a/src/admin/utils.rs b/src/admin/utils.rs index eba33fba..a2696c50 100644 --- a/src/admin/utils.rs +++ b/src/admin/utils.rs @@ -1,4 +1,4 @@ -use conduwuit_core::{err, Err, Result}; +use conduwuit_core::{Err, Result, err}; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; use service::Services; diff --git a/src/api/client/account.rs b/src/api/client/account.rs index cb25b276..cb49a6db 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -3,34 +3,35 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - debug_info, error, info, is_equal_to, utils, utils::ReadyExt, warn, Error, PduBuilder, Result, + Error, PduBuilder, Result, debug_info, error, info, is_equal_to, utils, utils::ReadyExt, warn, }; use futures::{FutureExt, StreamExt}; use register::RegistrationKind; use ruma::{ + OwnedRoomId, UserId, api::client::{ account::{ - change_password, check_registration_token_validity, deactivate, get_3pids, - get_username_availability, + ThirdPartyIdRemovalStatus, change_password, check_registration_token_validity, + deactivate, get_3pids, get_username_availability, register::{self, LoginType}, request_3pid_management_token_via_email, request_3pid_management_token_via_msisdn, - whoami, ThirdPartyIdRemovalStatus, + whoami, }, error::ErrorKind, uiaa::{AuthFlow, AuthType, UiaaInfo}, }, events::{ + GlobalAccountDataEventType, StateEventType, room::{ message::RoomMessageEventContent, power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, }, - GlobalAccountDataEventType, StateEventType, }, - push, OwnedRoomId, UserId, + push, }; use service::Services; -use super::{join_room_by_id_helper, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH}; +use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH, join_room_by_id_helper}; use crate::Ruma; const RANDOM_USER_ID_LENGTH: usize = 10; @@ -218,12 +219,20 @@ pub(crate) async fn register_route( }; if body.body.login_type == Some(LoginType::ApplicationService) { - if let Some(ref info) = body.appservice_info { - if !info.is_user_match(&user_id) { - return Err(Error::BadRequest(ErrorKind::Exclusive, "User is not in namespace.")); - } - } else { - return Err(Error::BadRequest(ErrorKind::MissingToken, "Missing appservice token.")); + match body.appservice_info { + | Some(ref info) => + if !info.is_user_match(&user_id) { + return Err(Error::BadRequest( + ErrorKind::Exclusive, + "User is not in namespace.", + )); + }, + | _ => { + return Err(Error::BadRequest( + ErrorKind::MissingToken, + "Missing appservice token.", + )); + }, } } else if services.appservice.is_exclusive_user_id(&user_id).await { return Err(Error::BadRequest(ErrorKind::Exclusive, "User ID reserved by appservice.")); @@ -256,33 +265,39 @@ pub(crate) async fn register_route( }; if !skip_auth { - if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services - .uiaa - .try_auth( - &UserId::parse_with_server_name("", services.globals.server_name()) - .expect("we know this is valid"), - "".into(), - auth, - &uiaainfo, - ) - .await?; - if !worked { - return Err(Error::Uiaa(uiaainfo)); - } - // Success! - } else if let Some(json) = body.json_body { - uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services.uiaa.create( - &UserId::parse_with_server_name("", services.globals.server_name()) - .expect("we know this is valid"), - "".into(), - &uiaainfo, - &json, - ); - return Err(Error::Uiaa(uiaainfo)); - } else { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + match &body.auth { + | Some(auth) => { + let (worked, uiaainfo) = services + .uiaa + .try_auth( + &UserId::parse_with_server_name("", services.globals.server_name()) + .expect("we know this is valid"), + "".into(), + auth, + &uiaainfo, + ) + .await?; + if !worked { + return Err(Error::Uiaa(uiaainfo)); + } + // Success! + }, + | _ => match body.json_body { + | Some(json) => { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); + services.uiaa.create( + &UserId::parse_with_server_name("", services.globals.server_name()) + .expect("we know this is valid"), + "".into(), + &uiaainfo, + &json, + ); + return Err(Error::Uiaa(uiaainfo)); + }, + | _ => { + return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + }, + }, } } @@ -463,7 +478,7 @@ pub(crate) async fn register_route( } if let Some(room_server_name) = room.server_name() { - if let Err(e) = join_room_by_id_helper( + match join_room_by_id_helper( &services, &user_id, &room_id, @@ -475,10 +490,15 @@ pub(crate) async fn register_route( .boxed() .await { - // don't return this error so we don't fail registrations - error!("Failed to automatically join room {room} for user {user_id}: {e}"); - } else { - info!("Automatically joined room {room} for user {user_id}"); + | Err(e) => { + // don't return this error so we don't fail registrations + error!( + "Failed to automatically join room {room} for user {user_id}: {e}" + ); + }, + | _ => { + info!("Automatically joined room {room} for user {user_id}"); + }, }; } } @@ -532,26 +552,32 @@ pub(crate) async fn change_password_route( auth_error: None, }; - if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services - .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo) - .await?; + match &body.auth { + | Some(auth) => { + let (worked, uiaainfo) = services + .uiaa + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; - if !worked { - return Err(Error::Uiaa(uiaainfo)); - } + if !worked { + return Err(Error::Uiaa(uiaainfo)); + } - // Success! - } else if let Some(json) = body.json_body { - uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services - .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + // Success! + }, + | _ => match body.json_body { + | Some(json) => { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); + services + .uiaa + .create(sender_user, sender_device, &uiaainfo, &json); - return Err(Error::Uiaa(uiaainfo)); - } else { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + return Err(Error::Uiaa(uiaainfo)); + }, + | _ => { + return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + }, + }, } services @@ -636,25 +662,31 @@ pub(crate) async fn deactivate_route( auth_error: None, }; - if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services - .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo) - .await?; + match &body.auth { + | Some(auth) => { + let (worked, uiaainfo) = services + .uiaa + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; - if !worked { - return Err(Error::Uiaa(uiaainfo)); - } - // Success! - } else if let Some(json) = body.json_body { - uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services - .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + if !worked { + return Err(Error::Uiaa(uiaainfo)); + } + // Success! + }, + | _ => match body.json_body { + | Some(json) => { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); + services + .uiaa + .create(sender_user, sender_device, &uiaainfo, &json); - return Err(Error::Uiaa(uiaainfo)); - } else { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + return Err(Error::Uiaa(uiaainfo)); + }, + | _ => { + return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + }, + }, } // Remove profile pictures and display name @@ -809,7 +841,7 @@ pub async fn full_user_deactivate( power_levels_content.users.remove(user_id); // ignore errors so deactivation doesn't fail - if let Err(e) = services + match services .rooms .timeline .build_and_append_pdu( @@ -820,9 +852,12 @@ pub async fn full_user_deactivate( ) .await { - warn!(%room_id, %user_id, "Failed to demote user's own power level: {e}"); - } else { - info!("Demoted {user_id} in {room_id} as part of account deactivation"); + | Err(e) => { + warn!(%room_id, %user_id, "Failed to demote user's own power level: {e}"); + }, + | _ => { + info!("Demoted {user_id} in {room_id} as part of account deactivation"); + }, } } } diff --git a/src/api/client/account_data.rs b/src/api/client/account_data.rs index 9f84f227..60c18b37 100644 --- a/src/api/client/account_data.rs +++ b/src/api/client/account_data.rs @@ -1,6 +1,7 @@ use axum::extract::State; -use conduwuit::{err, Err}; +use conduwuit::{Err, err}; use ruma::{ + RoomId, UserId, api::client::config::{ get_global_account_data, get_room_account_data, set_global_account_data, set_room_account_data, @@ -10,12 +11,11 @@ use ruma::{ GlobalAccountDataEventType, RoomAccountDataEventType, }, serde::Raw, - RoomId, UserId, }; use serde::Deserialize; use serde_json::{json, value::RawValue as RawJsonValue}; -use crate::{service::Services, Result, Ruma}; +use crate::{Result, Ruma, service::Services}; /// # `PUT /_matrix/client/r0/user/{userId}/account_data/{type}` /// diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index e1af416e..319e5141 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -1,10 +1,10 @@ use axum::extract::State; -use conduwuit::{debug, Err, Result}; +use conduwuit::{Err, Result, debug}; use futures::StreamExt; use rand::seq::SliceRandom; use ruma::{ - api::client::alias::{create_alias, delete_alias, get_alias}, OwnedServerName, RoomAliasId, RoomId, + api::client::alias::{create_alias, delete_alias, get_alias}, }; use service::Services; @@ -128,18 +128,26 @@ async fn room_available_servers( // insert our server as the very first choice if in list, else check if we can // prefer the room alias server first - if let Some(server_index) = servers + match servers .iter() .position(|server_name| services.globals.server_is_ours(server_name)) { - servers.swap_remove(server_index); - servers.insert(0, services.globals.server_name().to_owned()); - } else if let Some(alias_server_index) = servers - .iter() - .position(|server| server == room_alias.server_name()) - { - servers.swap_remove(alias_server_index); - servers.insert(0, room_alias.server_name().into()); + | Some(server_index) => { + servers.swap_remove(server_index); + servers.insert(0, services.globals.server_name().to_owned()); + }, + | _ => { + match servers + .iter() + .position(|server| server == room_alias.server_name()) + { + | Some(alias_server_index) => { + servers.swap_remove(alias_server_index); + servers.insert(0, room_alias.server_name().into()); + }, + | _ => {}, + } + }, } servers diff --git a/src/api/client/appservice.rs b/src/api/client/appservice.rs index e4071ab0..84955309 100644 --- a/src/api/client/appservice.rs +++ b/src/api/client/appservice.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduwuit::{err, Err, Result}; +use conduwuit::{Err, Result, err}; use ruma::api::{appservice::ping, client::appservice::request_ping}; use crate::Ruma; diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index d330952d..714e3f86 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -1,6 +1,7 @@ use axum::extract::State; -use conduwuit::{err, Err}; +use conduwuit::{Err, err}; use ruma::{ + UInt, api::client::backup::{ add_backup_keys, add_backup_keys_for_room, add_backup_keys_for_session, create_backup_version, delete_backup_keys, delete_backup_keys_for_room, @@ -8,7 +9,6 @@ use ruma::{ get_backup_keys_for_room, get_backup_keys_for_session, get_latest_backup_info, update_backup_version, }, - UInt, }; use crate::{Result, Ruma}; diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 7188aa23..e20af21b 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -3,11 +3,11 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{Result, Server}; use ruma::{ + RoomVersionId, api::client::discovery::get_capabilities::{ self, Capabilities, GetLoginTokenCapability, RoomVersionStability, RoomVersionsCapability, ThirdPartyIdChangesCapability, }, - RoomVersionId, }; use serde_json::json; diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 7256683f..3f16c850 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,23 +1,22 @@ use axum::extract::State; use conduwuit::{ - at, err, ref_at, + Err, PduEvent, Result, at, err, ref_at, utils::{ + IterStream, future::TryExtExt, stream::{BroadbandExt, ReadyExt, TryIgnore, WidebandExt}, - IterStream, }, - Err, PduEvent, Result, }; use futures::{ - future::{join, join3, try_join3, OptionFuture}, FutureExt, StreamExt, TryFutureExt, TryStreamExt, + future::{OptionFuture, join, join3, try_join3}, }; -use ruma::{api::client::context::get_context, events::StateEventType, OwnedEventId, UserId}; +use ruma::{OwnedEventId, UserId, api::client::context::get_context, events::StateEventType}; use service::rooms::{lazy_loading, lazy_loading::Options, short::ShortStateKey}; use crate::{ - client::message::{event_filter, ignored_filter, lazy_loading_witness, visibility_filter}, Ruma, + client::message::{event_filter, ignored_filter, lazy_loading_witness, visibility_filter}, }; const LIMIT_MAX: usize = 100; diff --git a/src/api/client/device.rs b/src/api/client/device.rs index bb0773dd..6a845aed 100644 --- a/src/api/client/device.rs +++ b/src/api/client/device.rs @@ -1,18 +1,18 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{err, Err}; +use conduwuit::{Err, err}; use futures::StreamExt; use ruma::{ + MilliSecondsSinceUnixEpoch, api::client::{ device::{self, delete_device, delete_devices, get_device, get_devices, update_device}, error::ErrorKind, uiaa::{AuthFlow, AuthType, UiaaInfo}, }, - MilliSecondsSinceUnixEpoch, }; use super::SESSION_ID_LENGTH; -use crate::{utils, Error, Result, Ruma}; +use crate::{Error, Result, Ruma, utils}; /// # `GET /_matrix/client/r0/devices` /// @@ -107,25 +107,31 @@ pub(crate) async fn delete_device_route( auth_error: None, }; - if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services - .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo) - .await?; + match &body.auth { + | Some(auth) => { + let (worked, uiaainfo) = services + .uiaa + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; - if !worked { - return Err!(Uiaa(uiaainfo)); - } - // Success! - } else if let Some(json) = body.json_body { - uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services - .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + if !worked { + return Err!(Uiaa(uiaainfo)); + } + // Success! + }, + | _ => match body.json_body { + | Some(json) => { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); + services + .uiaa + .create(sender_user, sender_device, &uiaainfo, &json); - return Err!(Uiaa(uiaainfo)); - } else { - return Err!(Request(NotJson("Not json."))); + return Err!(Uiaa(uiaainfo)); + }, + | _ => { + return Err!(Request(NotJson("Not json."))); + }, + }, } services @@ -164,25 +170,31 @@ pub(crate) async fn delete_devices_route( auth_error: None, }; - if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services - .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo) - .await?; + match &body.auth { + | Some(auth) => { + let (worked, uiaainfo) = services + .uiaa + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; - if !worked { - return Err(Error::Uiaa(uiaainfo)); - } - // Success! - } else if let Some(json) = body.json_body { - uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services - .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + if !worked { + return Err(Error::Uiaa(uiaainfo)); + } + // Success! + }, + | _ => match body.json_body { + | Some(json) => { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); + services + .uiaa + .create(sender_user, sender_device, &uiaainfo, &json); - return Err(Error::Uiaa(uiaainfo)); - } else { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + return Err(Error::Uiaa(uiaainfo)); + }, + | _ => { + return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + }, + }, } for device_id in &body.devices { diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 9166eed9..136c5961 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,8 +1,9 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{info, warn, Err, Error, Result}; +use conduwuit::{Err, Error, Result, info, warn}; use futures::{StreamExt, TryFutureExt}; use ruma::{ + OwnedRoomId, RoomId, ServerName, UInt, UserId, api::{ client::{ directory::{ @@ -16,13 +17,13 @@ use ruma::{ }, directory::{Filter, PublicRoomJoinRule, PublicRoomsChunk, RoomNetwork}, events::{ + StateEventType, room::{ join_rules::{JoinRule, RoomJoinRulesEventContent}, power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, }, - StateEventType, }, - uint, OwnedRoomId, RoomId, ServerName, UInt, UserId, + uint, }; use service::Services; @@ -365,30 +366,34 @@ async fn user_can_publish_room( user_id: &UserId, room_id: &RoomId, ) -> Result { - if let Ok(event) = services + match services .rooms .state_accessor .room_state_get(room_id, &StateEventType::RoomPowerLevels, "") .await { - serde_json::from_str(event.content.get()) + | Ok(event) => serde_json::from_str(event.content.get()) .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) .map(|content: RoomPowerLevelsEventContent| { RoomPowerLevels::from(content) .user_can_send_state(user_id, StateEventType::RoomHistoryVisibility) - }) - } else if let Ok(event) = services - .rooms - .state_accessor - .room_state_get(room_id, &StateEventType::RoomCreate, "") - .await - { - Ok(event.sender == user_id) - } else { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "User is not allowed to publish this room", - )); + }), + | _ => { + match services + .rooms + .state_accessor + .room_state_get(room_id, &StateEventType::RoomCreate, "") + .await + { + | Ok(event) => Ok(event.sender == user_id), + | _ => { + return Err(Error::BadRequest( + ErrorKind::forbidden(), + "User is not allowed to publish this room", + )); + }, + } + }, } } diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 801ae32b..6f20153b 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -1,9 +1,10 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use axum::extract::State; -use conduwuit::{debug, err, info, result::NotFound, utils, Err, Error, Result}; -use futures::{stream::FuturesUnordered, StreamExt}; +use conduwuit::{Err, Error, Result, debug, err, info, result::NotFound, utils}; +use futures::{StreamExt, stream::FuturesUnordered}; use ruma::{ + OneTimeKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, api::{ client::{ error::ErrorKind, @@ -17,14 +18,13 @@ use ruma::{ }, encryption::CrossSigningKey, serde::Raw, - OneTimeKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, }; use serde_json::json; use super::SESSION_ID_LENGTH; use crate::{ - service::{users::parse_master_key, Services}, Ruma, + service::{Services, users::parse_master_key}, }; /// # `POST /_matrix/client/r0/keys/upload` @@ -126,7 +126,7 @@ pub(crate) async fn upload_signing_keys_route( auth_error: None, }; - if let Ok(exists) = check_for_new_keys( + match check_for_new_keys( services, sender_user, body.self_signing_key.as_ref(), @@ -136,32 +136,45 @@ pub(crate) async fn upload_signing_keys_route( .await .inspect_err(|e| info!(?e)) { - if let Some(result) = exists { - // No-op, they tried to reupload the same set of keys - // (lost connection for example) - return Ok(result); - } - debug!("Skipping UIA in accordance with MSC3967, the user didn't have any existing keys"); - // Some of the keys weren't found, so we let them upload - } else if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services - .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo) - .await?; + | Ok(exists) => { + if let Some(result) = exists { + // No-op, they tried to reupload the same set of keys + // (lost connection for example) + return Ok(result); + } + debug!( + "Skipping UIA in accordance with MSC3967, the user didn't have any existing keys" + ); + // Some of the keys weren't found, so we let them upload + }, + | _ => { + match &body.auth { + | Some(auth) => { + let (worked, uiaainfo) = services + .uiaa + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; - if !worked { - return Err(Error::Uiaa(uiaainfo)); - } - // Success! - } else if let Some(json) = body.json_body { - uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services - .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + if !worked { + return Err(Error::Uiaa(uiaainfo)); + } + // Success! + }, + | _ => match body.json_body { + | Some(json) => { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); + services + .uiaa + .create(sender_user, sender_device, &uiaainfo, &json); - return Err(Error::Uiaa(uiaainfo)); - } else { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + return Err(Error::Uiaa(uiaainfo)); + }, + | _ => { + return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + }, + }, + } + }, } services @@ -471,37 +484,40 @@ where .collect(); while let Some((server, response)) = futures.next().await { - if let Ok(response) = response { - for (user, master_key) in response.master_keys { - let (master_key_id, mut master_key) = parse_master_key(&user, &master_key)?; + match response { + | Ok(response) => { + for (user, master_key) in response.master_keys { + let (master_key_id, mut master_key) = parse_master_key(&user, &master_key)?; - if let Ok(our_master_key) = services - .users - .get_key(&master_key_id, sender_user, &user, &allowed_signatures) - .await - { - let (_, mut our_master_key) = parse_master_key(&user, &our_master_key)?; - master_key.signatures.append(&mut our_master_key.signatures); + if let Ok(our_master_key) = services + .users + .get_key(&master_key_id, sender_user, &user, &allowed_signatures) + .await + { + let (_, mut our_master_key) = parse_master_key(&user, &our_master_key)?; + master_key.signatures.append(&mut our_master_key.signatures); + } + let json = serde_json::to_value(master_key).expect("to_value always works"); + let raw = serde_json::from_value(json).expect("Raw::from_value always works"); + services + .users + .add_cross_signing_keys( + &user, &raw, &None, &None, + false, /* Dont notify. A notification would trigger another key + * request resulting in an endless loop */ + ) + .await?; + if let Some(raw) = raw { + master_keys.insert(user.clone(), raw); + } } - let json = serde_json::to_value(master_key).expect("to_value always works"); - let raw = serde_json::from_value(json).expect("Raw::from_value always works"); - services - .users - .add_cross_signing_keys( - &user, &raw, &None, &None, - false, /* Dont notify. A notification would trigger another key request - * resulting in an endless loop */ - ) - .await?; - if let Some(raw) = raw { - master_keys.insert(user.clone(), raw); - } - } - self_signing_keys.extend(response.self_signing_keys); - device_keys.extend(response.device_keys); - } else { - failures.insert(server.to_string(), json!({})); + self_signing_keys.extend(response.self_signing_keys); + device_keys.extend(response.device_keys); + }, + | _ => { + failures.insert(server.to_string(), json!({})); + }, } } diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 0cff8185..94572413 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -3,16 +3,16 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - err, + Err, Result, err, utils::{self, content_disposition::make_content_disposition, math::ruma_from_usize}, - Err, Result, }; use conduwuit_service::{ - media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN, MXC_LENGTH}, Services, + media::{CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN, Dim, FileMeta, MXC_LENGTH}, }; use reqwest::Url; use ruma::{ + Mxc, UserId, api::client::{ authenticated_media::{ get_content, get_content_as_filename, get_content_thumbnail, get_media_config, @@ -20,7 +20,6 @@ use ruma::{ }, media::create_content, }, - Mxc, UserId, }; use crate::Ruma; diff --git a/src/api/client/media_legacy.rs b/src/api/client/media_legacy.rs index 4fa0b52e..d9f24f77 100644 --- a/src/api/client/media_legacy.rs +++ b/src/api/client/media_legacy.rs @@ -3,21 +3,20 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - err, + Err, Result, err, utils::{content_disposition::make_content_disposition, math::ruma_from_usize}, - Err, Result, }; -use conduwuit_service::media::{Dim, FileMeta, CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN}; +use conduwuit_service::media::{CACHE_CONTROL_IMMUTABLE, CORP_CROSS_ORIGIN, Dim, FileMeta}; use reqwest::Url; use ruma::{ + Mxc, api::client::media::{ create_content, get_content, get_content_as_filename, get_content_thumbnail, get_media_config, get_media_preview, }, - Mxc, }; -use crate::{client::create_content_route, Ruma, RumaResponse}; +use crate::{Ruma, RumaResponse, client::create_content_route}; /// # `GET /_matrix/media/v3/config` /// @@ -142,46 +141,52 @@ pub(crate) async fn get_content_legacy_route( media_id: &body.media_id, }; - if let Some(FileMeta { - content, - content_type, - content_disposition, - }) = services.media.get(&mxc).await? - { - let content_disposition = - make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + match services.media.get(&mxc).await? { + | Some(FileMeta { + content, + content_type, + content_disposition, + }) => { + let content_disposition = make_content_disposition( + content_disposition.as_ref(), + content_type.as_deref(), + None, + ); - Ok(get_content::v3::Response { - file: content.expect("entire file contents"), - content_type: content_type.map(Into::into), - content_disposition: Some(content_disposition), - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) - } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { - let response = services - .media - .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) - .await - .map_err(|e| { - err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))) - })?; + Ok(get_content::v3::Response { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + }) + }, + | _ => + if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { + let response = services + .media + .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) + .await + .map_err(|e| { + err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))) + })?; - let content_disposition = make_content_disposition( - response.content_disposition.as_ref(), - response.content_type.as_deref(), - None, - ); + let content_disposition = make_content_disposition( + response.content_disposition.as_ref(), + response.content_type.as_deref(), + None, + ); - Ok(get_content::v3::Response { - file: response.file, - content_type: response.content_type, - content_disposition: Some(content_disposition), - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) - } else { - Err!(Request(NotFound("Media not found."))) + Ok(get_content::v3::Response { + file: response.file, + content_type: response.content_type, + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + }) + } else { + Err!(Request(NotFound("Media not found."))) + }, } } @@ -227,49 +232,52 @@ pub(crate) async fn get_content_as_filename_legacy_route( media_id: &body.media_id, }; - if let Some(FileMeta { - content, - content_type, - content_disposition, - }) = services.media.get(&mxc).await? - { - let content_disposition = make_content_disposition( - content_disposition.as_ref(), - content_type.as_deref(), - Some(&body.filename), - ); + match services.media.get(&mxc).await? { + | Some(FileMeta { + content, + content_type, + content_disposition, + }) => { + let content_disposition = make_content_disposition( + content_disposition.as_ref(), + content_type.as_deref(), + Some(&body.filename), + ); - Ok(get_content_as_filename::v3::Response { - file: content.expect("entire file contents"), - content_type: content_type.map(Into::into), - content_disposition: Some(content_disposition), - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) - } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { - let response = services - .media - .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) - .await - .map_err(|e| { - err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))) - })?; + Ok(get_content_as_filename::v3::Response { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + content_disposition: Some(content_disposition), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + }) + }, + | _ => + if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { + let response = services + .media + .fetch_remote_content_legacy(&mxc, body.allow_redirect, body.timeout_ms) + .await + .map_err(|e| { + err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))) + })?; - let content_disposition = make_content_disposition( - response.content_disposition.as_ref(), - response.content_type.as_deref(), - None, - ); + let content_disposition = make_content_disposition( + response.content_disposition.as_ref(), + response.content_type.as_deref(), + None, + ); - Ok(get_content_as_filename::v3::Response { - content_disposition: Some(content_disposition), - content_type: response.content_type, - file: response.file, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - }) - } else { - Err!(Request(NotFound("Media not found."))) + Ok(get_content_as_filename::v3::Response { + content_disposition: Some(content_disposition), + content_type: response.content_type, + file: response.file, + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + }) + } else { + Err!(Request(NotFound("Media not found."))) + }, } } @@ -315,46 +323,52 @@ pub(crate) async fn get_content_thumbnail_legacy_route( }; let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; - if let Some(FileMeta { - content, - content_type, - content_disposition, - }) = services.media.get_thumbnail(&mxc, &dim).await? - { - let content_disposition = - make_content_disposition(content_disposition.as_ref(), content_type.as_deref(), None); + match services.media.get_thumbnail(&mxc, &dim).await? { + | Some(FileMeta { + content, + content_type, + content_disposition, + }) => { + let content_disposition = make_content_disposition( + content_disposition.as_ref(), + content_type.as_deref(), + None, + ); - Ok(get_content_thumbnail::v3::Response { - file: content.expect("entire file contents"), - content_type: content_type.map(Into::into), - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - content_disposition: Some(content_disposition), - }) - } else if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { - let response = services - .media - .fetch_remote_thumbnail_legacy(&body) - .await - .map_err(|e| { - err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))) - })?; + Ok(get_content_thumbnail::v3::Response { + file: content.expect("entire file contents"), + content_type: content_type.map(Into::into), + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + content_disposition: Some(content_disposition), + }) + }, + | _ => + if !services.globals.server_is_ours(&body.server_name) && body.allow_remote { + let response = services + .media + .fetch_remote_thumbnail_legacy(&body) + .await + .map_err(|e| { + err!(Request(NotFound(debug_warn!(%mxc, "Fetching media failed: {e:?}")))) + })?; - let content_disposition = make_content_disposition( - response.content_disposition.as_ref(), - response.content_type.as_deref(), - None, - ); + let content_disposition = make_content_disposition( + response.content_disposition.as_ref(), + response.content_type.as_deref(), + None, + ); - Ok(get_content_thumbnail::v3::Response { - file: response.file, - content_type: response.content_type, - cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), - cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), - content_disposition: Some(content_disposition), - }) - } else { - Err!(Request(NotFound("Media not found."))) + Ok(get_content_thumbnail::v3::Response { + file: response.file, + content_type: response.content_type, + cross_origin_resource_policy: Some(CORP_CROSS_ORIGIN.into()), + cache_control: Some(CACHE_CONTROL_IMMUTABLE.into()), + content_disposition: Some(content_disposition), + }) + } else { + Err!(Request(NotFound("Media not found."))) + }, } } diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 26736fb5..9c2693dc 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -9,51 +9,51 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - at, debug, debug_info, debug_warn, err, error, info, - pdu::{gen_event_id_canonical_json, PduBuilder}, + Err, PduEvent, Result, StateKey, at, debug, debug_info, debug_warn, err, error, info, + pdu::{PduBuilder, gen_event_id_canonical_json}, result::FlatOk, state_res, trace, - utils::{self, shuffle, IterStream, ReadyExt}, - warn, Err, PduEvent, Result, StateKey, + utils::{self, IterStream, ReadyExt, shuffle}, + warn, }; -use futures::{join, FutureExt, StreamExt, TryFutureExt}; +use futures::{FutureExt, StreamExt, TryFutureExt, join}; use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, + OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, api::{ client::{ error::ErrorKind, knock::knock_room, membership::{ - ban_user, forget_room, get_member_events, invite_user, join_room_by_id, - join_room_by_id_or_alias, + ThirdPartySigned, ban_user, forget_room, get_member_events, invite_user, + join_room_by_id, join_room_by_id_or_alias, joined_members::{self, v3::RoomMember}, - joined_rooms, kick_user, leave_room, unban_user, ThirdPartySigned, + joined_rooms, kick_user, leave_room, unban_user, }, }, federation::{self, membership::create_invite}, }, canonical_json::to_canonical_value, events::{ + StateEventType, room::{ join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent}, member::{MembershipState, RoomMemberEventContent}, message::RoomMessageEventContent, }, - StateEventType, }, - CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, - OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, }; use service::{ + Services, appservice::RegistrationInfo, pdu::gen_event_id, rooms::{ state::RoomMutexGuard, state_compressor::{CompressedState, HashSetCompressStateEvent}, }, - Services, }; -use crate::{client::full_user_deactivate, Ruma}; +use crate::{Ruma, client::full_user_deactivate}; /// Checks if the room is banned in any way possible and the sender user is not /// an admin. @@ -507,43 +507,54 @@ pub(crate) async fn invite_user_route( ) .await?; - if let invite_user::v3::InvitationRecipient::UserId { user_id } = &body.recipient { - let sender_ignored_recipient = services.users.user_is_ignored(sender_user, user_id); - let recipient_ignored_by_sender = services.users.user_is_ignored(user_id, sender_user); + match &body.recipient { + | invite_user::v3::InvitationRecipient::UserId { user_id } => { + let sender_ignored_recipient = services.users.user_is_ignored(sender_user, user_id); + let recipient_ignored_by_sender = + services.users.user_is_ignored(user_id, sender_user); - let (sender_ignored_recipient, recipient_ignored_by_sender) = - join!(sender_ignored_recipient, recipient_ignored_by_sender); + let (sender_ignored_recipient, recipient_ignored_by_sender) = + join!(sender_ignored_recipient, recipient_ignored_by_sender); - if sender_ignored_recipient { - return Err!(Request(Forbidden( - "You cannot invite users you have ignored to rooms." - ))); - } - - if let Ok(target_user_membership) = services - .rooms - .state_accessor - .get_member(&body.room_id, user_id) - .await - { - if target_user_membership.membership == MembershipState::Ban { - return Err!(Request(Forbidden("User is banned from this room."))); + if sender_ignored_recipient { + return Err!(Request(Forbidden( + "You cannot invite users you have ignored to rooms." + ))); } - } - if recipient_ignored_by_sender { - // silently drop the invite to the recipient if they've been ignored by the - // sender, pretend it worked - return Ok(invite_user::v3::Response {}); - } + if let Ok(target_user_membership) = services + .rooms + .state_accessor + .get_member(&body.room_id, user_id) + .await + { + if target_user_membership.membership == MembershipState::Ban { + return Err!(Request(Forbidden("User is banned from this room."))); + } + } - invite_helper(&services, sender_user, user_id, &body.room_id, body.reason.clone(), false) + if recipient_ignored_by_sender { + // silently drop the invite to the recipient if they've been ignored by the + // sender, pretend it worked + return Ok(invite_user::v3::Response {}); + } + + invite_helper( + &services, + sender_user, + user_id, + &body.room_id, + body.reason.clone(), + false, + ) .boxed() .await?; - Ok(invite_user::v3::Response {}) - } else { - Err!(Request(NotFound("User not found."))) + Ok(invite_user::v3::Response {}) + }, + | _ => { + Err!(Request(NotFound("User not found."))) + }, } } @@ -1830,38 +1841,46 @@ async fn remote_leave_room( .collect() .await; - if let Ok(invite_state) = services + match services .rooms .state_cache .invite_state(user_id, room_id) .await { - servers.extend( - invite_state - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()), - ); - } else if let Ok(knock_state) = services - .rooms - .state_cache - .knock_state(user_id, room_id) - .await - { - servers.extend( - knock_state - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .filter_map(|sender| { - if !services.globals.user_is_local(sender) { - Some(sender.server_name().to_owned()) - } else { - None - } - }), - ); + | Ok(invite_state) => { + servers.extend( + invite_state + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), + ); + }, + | _ => { + match services + .rooms + .state_cache + .knock_state(user_id, room_id) + .await + { + | Ok(knock_state) => { + servers.extend( + knock_state + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .filter_map(|sender| { + if !services.globals.user_is_local(sender) { + Some(sender.server_name().to_owned()) + } else { + None + } + }), + ); + }, + | _ => {}, + } + }, } if let Some(room_id_server_name) = room_id.server_name() { diff --git a/src/api/client/message.rs b/src/api/client/message.rs index bb4e72dd..571a238a 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,30 +1,29 @@ use axum::extract::State; use conduwuit::{ - at, + Event, PduCount, PduEvent, Result, at, utils::{ + IterStream, ReadyExt, result::{FlatOk, LogErr}, stream::{BroadbandExt, TryIgnore, WidebandExt}, - IterStream, ReadyExt, }, - Event, PduCount, PduEvent, Result, }; -use futures::{future::OptionFuture, pin_mut, FutureExt, StreamExt, TryFutureExt}; +use futures::{FutureExt, StreamExt, TryFutureExt, future::OptionFuture, pin_mut}; use ruma::{ + RoomId, UserId, api::{ - client::{filter::RoomEventFilter, message::get_message_events}, Direction, + client::{filter::RoomEventFilter, message::get_message_events}, }, events::{AnyStateEvent, StateEventType, TimelineEventType, TimelineEventType::*}, serde::Raw, - RoomId, UserId, }; use service::{ + Services, rooms::{ lazy_loading, lazy_loading::{Options, Witness}, timeline::PdusIterItem, }, - Services, }; use crate::Ruma; diff --git a/src/api/client/presence.rs b/src/api/client/presence.rs index d19e6ae1..9b41a721 100644 --- a/src/api/client/presence.rs +++ b/src/api/client/presence.rs @@ -70,37 +70,38 @@ pub(crate) async fn get_presence_route( } } - if let Some(presence) = presence_event { - let status_msg = if presence - .content - .status_msg - .as_ref() - .is_some_and(String::is_empty) - { - None - } else { - presence.content.status_msg - }; - - let last_active_ago = match presence.content.currently_active { - | Some(true) => None, - | _ => presence + match presence_event { + | Some(presence) => { + let status_msg = if presence .content - .last_active_ago - .map(|millis| Duration::from_millis(millis.into())), - }; + .status_msg + .as_ref() + .is_some_and(String::is_empty) + { + None + } else { + presence.content.status_msg + }; - Ok(get_presence::v3::Response { - // TODO: Should ruma just use the presenceeventcontent type here? - status_msg, - currently_active: presence.content.currently_active, - last_active_ago, - presence: presence.content.presence, - }) - } else { - Err(Error::BadRequest( + let last_active_ago = match presence.content.currently_active { + | Some(true) => None, + | _ => presence + .content + .last_active_ago + .map(|millis| Duration::from_millis(millis.into())), + }; + + Ok(get_presence::v3::Response { + // TODO: Should ruma just use the presenceeventcontent type here? + status_msg, + currently_active: presence.content.currently_active, + last_active_ago, + presence: presence.content.presence, + }) + }, + | _ => Err(Error::BadRequest( ErrorKind::NotFound, "Presence state for this user was not found", - )) + )), } } diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 584adfc1..12e5ebcc 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -2,12 +2,14 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ + Err, Error, Result, pdu::PduBuilder, - utils::{stream::TryIgnore, IterStream}, - warn, Err, Error, Result, + utils::{IterStream, stream::TryIgnore}, + warn, }; -use futures::{future::join3, StreamExt, TryStreamExt}; +use futures::{StreamExt, TryStreamExt, future::join3}; use ruma::{ + OwnedMxcUri, OwnedRoomId, UserId, api::{ client::{ error::ErrorKind, @@ -19,7 +21,6 @@ use ruma::{ }, events::room::member::{MembershipState, RoomMemberEventContent}, presence::PresenceState, - OwnedMxcUri, OwnedRoomId, UserId, }; use service::Services; diff --git a/src/api/client/push.rs b/src/api/client/push.rs index ed7371e4..384b9dbc 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -1,6 +1,7 @@ use axum::extract::State; -use conduwuit::{err, Err}; +use conduwuit::{Err, err}; use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, api::client::{ error::ErrorKind, push::{ @@ -10,14 +11,13 @@ use ruma::{ }, }, events::{ - push_rules::{PushRulesEvent, PushRulesEventContent}, GlobalAccountDataEventType, + push_rules::{PushRulesEvent, PushRulesEventContent}, }, push::{ InsertPushRuleError, PredefinedContentRuleId, PredefinedOverrideRuleId, RemovePushRuleError, Ruleset, }, - CanonicalJsonObject, CanonicalJsonValue, }; use service::Services; diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index 89fe003a..d01327f6 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -1,14 +1,14 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduwuit::{err, Err, PduCount}; +use conduwuit::{Err, PduCount, err}; use ruma::{ + MilliSecondsSinceUnixEpoch, api::client::{read_marker::set_read_marker, receipt::create_receipt}, events::{ - receipt::{ReceiptThread, ReceiptType}, RoomAccountDataEventType, + receipt::{ReceiptThread, ReceiptType}, }, - MilliSecondsSinceUnixEpoch, }; use crate::{Result, Ruma}; diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index ba59a010..7b512d06 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -3,7 +3,7 @@ use ruma::{ api::client::redact::redact_event, events::room::redaction::RoomRedactionEventContent, }; -use crate::{service::pdu::PduBuilder, Result, Ruma}; +use crate::{Result, Ruma, service::pdu::PduBuilder}; /// # `PUT /_matrix/client/r0/rooms/{roomId}/redact/{eventId}/{txnId}` /// diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index 87fb1eac..7ed40f14 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,22 +1,21 @@ use axum::extract::State; use conduwuit::{ - at, - utils::{result::FlatOk, stream::WidebandExt, IterStream, ReadyExt}, - PduCount, Result, + PduCount, Result, at, + utils::{IterStream, ReadyExt, result::FlatOk, stream::WidebandExt}, }; use futures::StreamExt; use ruma::{ + EventId, RoomId, UInt, UserId, api::{ + Direction, client::relations::{ get_relating_events, get_relating_events_with_rel_type, get_relating_events_with_rel_type_and_event_type, }, - Direction, }, - events::{relation::RelationType, TimelineEventType}, - EventId, RoomId, UInt, UserId, + events::{TimelineEventType, relation::RelationType}, }; -use service::{rooms::timeline::PdusIterItem, Services}; +use service::{Services, rooms::timeline::PdusIterItem}; use crate::Ruma; diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 57de3f12..db085721 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -2,22 +2,22 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{info, utils::ReadyExt, Err}; +use conduwuit::{Err, info, utils::ReadyExt}; use rand::Rng; use ruma::{ + EventId, RoomId, UserId, api::client::{ error::ErrorKind, room::{report_content, report_room}, }, events::room::message, - int, EventId, RoomId, UserId, + int, }; use tokio::time::sleep; use crate::{ - debug_info, - service::{pdu::PduEvent, Services}, - Error, Result, Ruma, + Error, Result, Ruma, debug_info, + service::{Services, pdu::PduEvent}, }; /// # `POST /_matrix/client/v3/rooms/{roomId}/report` diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index e362b3b3..1b8294a5 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -2,15 +2,17 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ - debug_info, debug_warn, err, error, info, pdu::PduBuilder, warn, Err, Error, Result, StateKey, + Err, Error, Result, StateKey, debug_info, debug_warn, err, error, info, pdu::PduBuilder, warn, }; use futures::FutureExt; use ruma::{ + CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId, api::client::{ error::ErrorKind, room::{self, create_room}, }, events::{ + TimelineEventType, room::{ canonical_alias::RoomCanonicalAliasEventContent, create::RoomCreateEventContent, @@ -22,16 +24,14 @@ use ruma::{ power_levels::RoomPowerLevelsEventContent, topic::RoomTopicEventContent, }, - TimelineEventType, }, int, serde::{JsonObject, Raw}, - CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId, }; use serde_json::{json, value::to_raw_value}; -use service::{appservice::RegistrationInfo, Services}; +use service::{Services, appservice::RegistrationInfo}; -use crate::{client::invite_helper, Ruma}; +use crate::{Ruma, client::invite_helper}; /// # `POST /_matrix/client/v3/createRoom` /// @@ -68,10 +68,9 @@ pub(crate) async fn create_room_route( )); } - let room_id: OwnedRoomId = if let Some(custom_room_id) = &body.room_id { - custom_room_id_check(&services, custom_room_id)? - } else { - RoomId::new(&services.server.name) + let room_id: OwnedRoomId = match &body.room_id { + | Some(custom_room_id) => custom_room_id_check(&services, custom_room_id)?, + | _ => RoomId::new(&services.server.name), }; // check if room ID doesn't already exist instead of erroring on auth check @@ -114,10 +113,10 @@ pub(crate) async fn create_room_route( .await; let state_lock = services.rooms.state.mutex.lock(&room_id).await; - let alias: Option = if let Some(alias) = body.room_alias_name.as_ref() { - Some(room_alias_check(&services, alias, body.appservice_info.as_ref()).await?) - } else { - None + let alias: Option = match body.room_alias_name.as_ref() { + | Some(alias) => + Some(room_alias_check(&services, alias, body.appservice_info.as_ref()).await?), + | _ => None, }; let room_version = match body.room_version.clone() { diff --git a/src/api/client/room/event.rs b/src/api/client/room/event.rs index f0ae64dd..84b591cd 100644 --- a/src/api/client/room/event.rs +++ b/src/api/client/room/event.rs @@ -1,9 +1,9 @@ use axum::extract::State; -use conduwuit::{err, Err, Event, Result}; -use futures::{future::try_join, FutureExt, TryFutureExt}; +use conduwuit::{Err, Event, Result, err}; +use futures::{FutureExt, TryFutureExt, future::try_join}; use ruma::api::client::room::get_room_event; -use crate::{client::is_ignored_pdu, Ruma}; +use crate::{Ruma, client::is_ignored_pdu}; /// # `GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}` /// diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index 233d180f..e4c76ae0 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -1,8 +1,7 @@ use axum::extract::State; use conduwuit::{ - at, - utils::{stream::TryTools, BoolExt}, - Err, PduEvent, Result, + Err, PduEvent, Result, at, + utils::{BoolExt, stream::TryTools}, }; use futures::TryStreamExt; use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response}; diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index a624f95f..4ac341a9 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -1,19 +1,20 @@ use std::cmp::max; use axum::extract::State; -use conduwuit::{err, info, pdu::PduBuilder, Error, Result, StateKey}; +use conduwuit::{Error, Result, StateKey, err, info, pdu::PduBuilder}; use futures::StreamExt; use ruma::{ + CanonicalJsonObject, RoomId, RoomVersionId, api::client::{error::ErrorKind, room::upgrade_room}, events::{ + StateEventType, TimelineEventType, room::{ member::{MembershipState, RoomMemberEventContent}, power_levels::RoomPowerLevelsEventContent, tombstone::RoomTombstoneEventContent, }, - StateEventType, TimelineEventType, }, - int, CanonicalJsonObject, RoomId, RoomVersionId, + int, }; use serde_json::{json, value::to_raw_value}; diff --git a/src/api/client/search.rs b/src/api/client/search.rs index 898dfc7f..f3366843 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -2,23 +2,22 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ - at, is_true, + Err, PduEvent, Result, at, is_true, result::FlatOk, - utils::{stream::ReadyExt, IterStream}, - Err, PduEvent, Result, + utils::{IterStream, stream::ReadyExt}, }; -use futures::{future::OptionFuture, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, future::OptionFuture}; use ruma::{ + OwnedRoomId, RoomId, UInt, UserId, api::client::search::search_events::{ self, v3::{Criteria, EventContextResult, ResultCategories, ResultRoomEvents, SearchResult}, }, events::AnyStateEvent, serde::Raw, - OwnedRoomId, RoomId, UInt, UserId, }; use search_events::v3::{Request, Response}; -use service::{rooms::search::RoomQuery, Services}; +use service::{Services, rooms::search::RoomQuery}; use crate::Ruma; diff --git a/src/api/client/send.rs b/src/api/client/send.rs index 39340070..b01d1ed6 100644 --- a/src/api/client/send.rs +++ b/src/api/client/send.rs @@ -1,11 +1,11 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduwuit::{err, Err}; +use conduwuit::{Err, err}; use ruma::{api::client::message::send_message_event, events::MessageLikeEventType}; use serde_json::from_str; -use crate::{service::pdu::PduBuilder, utils, Result, Ruma}; +use crate::{Result, Ruma, service::pdu::PduBuilder, utils}; /// # `PUT /_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}` /// diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 7155351c..5c0ab47d 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -2,9 +2,10 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{debug, err, info, utils::ReadyExt, warn, Err}; +use conduwuit::{Err, debug, err, info, utils::ReadyExt, warn}; use futures::StreamExt; use ruma::{ + OwnedUserId, UserId, api::client::{ error::ErrorKind, session::{ @@ -21,12 +22,11 @@ use ruma::{ }, uiaa, }, - OwnedUserId, UserId, }; use service::uiaa::SESSION_ID_LENGTH; use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; -use crate::{utils, utils::hash, Error, Result, Ruma}; +use crate::{Error, Result, Ruma, utils, utils::hash}; /// # `GET /_matrix/client/v3/login` /// @@ -139,18 +139,20 @@ pub(crate) async fn login_route( Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.") })?; - if let Some(ref info) = body.appservice_info { - if !info.is_user_match(&user_id) { + match body.appservice_info { + | Some(ref info) => + if !info.is_user_match(&user_id) { + return Err(Error::BadRequest( + ErrorKind::Exclusive, + "User is not in namespace.", + )); + }, + | _ => { return Err(Error::BadRequest( - ErrorKind::Exclusive, - "User is not in namespace.", + ErrorKind::MissingToken, + "Missing appservice token.", )); - } - } else { - return Err(Error::BadRequest( - ErrorKind::MissingToken, - "Missing appservice token.", - )); + }, } user_id @@ -259,26 +261,32 @@ pub(crate) async fn login_token_route( auth_error: None, }; - if let Some(auth) = &body.auth { - let (worked, uiaainfo) = services - .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo) - .await?; + match &body.auth { + | Some(auth) => { + let (worked, uiaainfo) = services + .uiaa + .try_auth(sender_user, sender_device, auth, &uiaainfo) + .await?; - if !worked { - return Err(Error::Uiaa(uiaainfo)); - } + if !worked { + return Err(Error::Uiaa(uiaainfo)); + } - // Success! - } else if let Some(json) = body.json_body.as_ref() { - uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); - services - .uiaa - .create(sender_user, sender_device, &uiaainfo, json); + // Success! + }, + | _ => match body.json_body.as_ref() { + | Some(json) => { + uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); + services + .uiaa + .create(sender_user, sender_device, &uiaainfo, json); - return Err(Error::Uiaa(uiaainfo)); - } else { - return Err!(Request(NotJson("No JSON body was sent when required."))); + return Err(Error::Uiaa(uiaainfo)); + }, + | _ => { + return Err!(Request(NotJson("No JSON body was sent when required."))); + }, + }, } let login_token = utils::random_string(TOKEN_LENGTH); diff --git a/src/api/client/space.rs b/src/api/client/space.rs index 7efd7817..a667f852 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -5,18 +5,18 @@ use std::{ use axum::extract::State; use conduwuit::{ - utils::{future::TryExtExt, stream::IterStream}, Err, Result, + utils::{future::TryExtExt, stream::IterStream}, }; -use futures::{future::OptionFuture, StreamExt, TryFutureExt}; +use futures::{StreamExt, TryFutureExt, future::OptionFuture}; use ruma::{ - api::client::space::get_hierarchy, OwnedRoomId, OwnedServerName, RoomId, UInt, UserId, + OwnedRoomId, OwnedServerName, RoomId, UInt, UserId, api::client::space::get_hierarchy, }; use service::{ - rooms::spaces::{ - get_parent_children_via, summary_to_chunk, PaginationToken, SummaryAccessibility, - }, Services, + rooms::spaces::{ + PaginationToken, SummaryAccessibility, get_parent_children_via, summary_to_chunk, + }, }; use crate::Ruma; diff --git a/src/api/client/state.rs b/src/api/client/state.rs index f73ffa46..6353fe1c 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,19 +1,19 @@ use axum::extract::State; -use conduwuit::{err, pdu::PduBuilder, utils::BoolExt, Err, PduEvent, Result}; +use conduwuit::{Err, PduEvent, Result, err, pdu::PduBuilder, utils::BoolExt}; use futures::TryStreamExt; use ruma::{ + OwnedEventId, RoomId, UserId, api::client::state::{get_state_events, get_state_events_for_key, send_state_event}, events::{ + AnyStateEventContent, StateEventType, room::{ canonical_alias::RoomCanonicalAliasEventContent, history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, join_rules::{JoinRule, RoomJoinRulesEventContent}, member::{MembershipState, RoomMemberEventContent}, }, - AnyStateEventContent, StateEventType, }, serde::Raw, - OwnedEventId, RoomId, UserId, }; use service::Services; diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 46540881..3eab76cc 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -3,25 +3,25 @@ mod v4; mod v5; use conduwuit::{ - utils::{ - stream::{BroadbandExt, ReadyExt, TryIgnore}, - IterStream, - }, PduCount, + utils::{ + IterStream, + stream::{BroadbandExt, ReadyExt, TryIgnore}, + }, }; -use futures::{pin_mut, StreamExt}; +use futures::{StreamExt, pin_mut}; use ruma::{ + RoomId, UserId, directory::RoomTypeFilter, events::TimelineEventType::{ self, Beacon, CallInvite, PollStart, RoomEncrypted, RoomMessage, Sticker, }, - RoomId, UserId, }; pub(crate) use self::{ v3::sync_events_route, v4::sync_events_v4_route, v5::sync_events_v5_route, }; -use crate::{service::Services, Error, PduEvent, Result}; +use crate::{Error, PduEvent, Result, service::Services}; pub(crate) const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = &[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker]; diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index f9dcd5ec..fb59837b 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -6,57 +6,55 @@ use std::{ use axum::extract::State; use conduwuit::{ - at, err, error, extract_variant, is_equal_to, pair_of, + PduCount, PduEvent, Result, at, err, error, extract_variant, is_equal_to, pair_of, pdu::{Event, EventHash}, ref_at, result::FlatOk, utils::{ - self, + self, BoolExt, IterStream, ReadyExt, TryFutureExtExt, math::ruma_from_u64, stream::{BroadbandExt, Tools, TryExpect, WidebandExt}, - BoolExt, IterStream, ReadyExt, TryFutureExtExt, }, - PduCount, PduEvent, Result, }; use conduwuit_service::{ + Services, rooms::{ lazy_loading, lazy_loading::{Options, Witness}, short::ShortStateHash, }, - Services, }; use futures::{ - future::{join, join3, join4, join5, try_join, try_join4, OptionFuture}, FutureExt, StreamExt, TryFutureExt, TryStreamExt, + future::{OptionFuture, join, join3, join4, join5, try_join, try_join4}, }; use ruma::{ + DeviceId, EventId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId, api::client::{ filter::FilterDefinition, sync::sync_events::{ - self, + self, DeviceLists, UnreadNotificationsCount, v3::{ Ephemeral, Filter, GlobalAccountData, InviteState, InvitedRoom, JoinedRoom, KnockState, KnockedRoom, LeftRoom, Presence, RoomAccountData, RoomSummary, Rooms, State as RoomState, Timeline, ToDevice, }, - DeviceLists, UnreadNotificationsCount, }, uiaa::UiaaResponse, }, events::{ - presence::{PresenceEvent, PresenceEventContent}, - room::member::{MembershipState, RoomMemberEventContent}, AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType::*, + presence::{PresenceEvent, PresenceEventContent}, + room::member::{MembershipState, RoomMemberEventContent}, }, serde::Raw, - uint, DeviceId, EventId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId, + uint, }; use service::rooms::short::{ShortEventId, ShortStateKey}; use super::{load_timeline, share_encrypted_room}; -use crate::{client::ignored_filter, Ruma, RumaResponse}; +use crate::{Ruma, RumaResponse, client::ignored_filter}; #[derive(Default)] struct StateChanges { @@ -168,8 +166,8 @@ pub(crate) async fn build_sync_events( let full_state = body.body.full_state; let filter = match body.body.filter.as_ref() { | None => FilterDefinition::default(), - | Some(Filter::FilterDefinition(ref filter)) => filter.clone(), - | Some(Filter::FilterId(ref filter_id)) => services + | Some(Filter::FilterDefinition(filter)) => filter.clone(), + | Some(Filter::FilterId(filter_id)) => services .users .get_filter(sender_user, filter_id) .await @@ -1016,34 +1014,37 @@ async fn calculate_state_incremental<'a>( let lazy_state_ids: OptionFuture<_> = witness .filter(|_| !full_state && !encrypted_room) .map(|witness| { - witness - .iter() - .stream() - .broad_filter_map(|user_id| state_get_shorteventid(user_id)) - .into_future() + StreamExt::into_future( + witness + .iter() + .stream() + .broad_filter_map(|user_id| state_get_shorteventid(user_id)), + ) }) .into(); let state_diff: OptionFuture<_> = (!full_state && state_changed) .then(|| { - services - .rooms - .state_accessor - .state_added((since_shortstatehash, current_shortstatehash)) - .boxed() - .into_future() + StreamExt::into_future( + services + .rooms + .state_accessor + .state_added((since_shortstatehash, current_shortstatehash)) + .boxed(), + ) }) .into(); let current_state_ids: OptionFuture<_> = full_state .then(|| { - services - .rooms - .state_accessor - .state_full_shortids(current_shortstatehash) - .expect_ok() - .boxed() - .into_future() + StreamExt::into_future( + services + .rooms + .state_accessor + .state_full_shortids(current_shortstatehash) + .expect_ok() + .boxed(), + ) }) .into(); diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 13f832b2..5fdcbab8 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -6,37 +6,37 @@ use std::{ use axum::extract::State; use conduwuit::{ - debug, error, extract_variant, + Error, PduCount, Result, debug, error, extract_variant, utils::{ - math::{ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, BoolExt, IterStream, ReadyExt, TryFutureExtExt, + math::{ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, }, - warn, Error, PduCount, Result, + warn, }; use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{ + MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, api::client::{ error::ErrorKind, sync::sync_events::{ - self, + self, DeviceLists, UnreadNotificationsCount, v4::{SlidingOp, SlidingSyncRoomHero}, - DeviceLists, UnreadNotificationsCount, }, }, events::{ - room::member::{MembershipState, RoomMemberEventContent}, AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType::*, + room::member::{MembershipState, RoomMemberEventContent}, }, serde::Raw, - uint, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, + uint, }; use service::rooms::read_receipt::pack_receipts; use super::{load_timeline, share_encrypted_room}; use crate::{ - client::{filter_rooms, ignored_filter, sync::v5::TodoRooms, DEFAULT_BUMP_TYPES}, Ruma, + client::{DEFAULT_BUMP_TYPES, filter_rooms, ignored_filter, sync::v5::TodoRooms}, }; pub(crate) const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; @@ -700,14 +700,13 @@ pub(crate) async fn sync_events_v4_route( .await .ok() .or(name), - avatar: if let Some(heroes_avatar) = heroes_avatar { - ruma::JsOption::Some(heroes_avatar) - } else { - match services.rooms.state_accessor.get_avatar(room_id).await { + avatar: match heroes_avatar { + | Some(heroes_avatar) => ruma::JsOption::Some(heroes_avatar), + | _ => match services.rooms.state_accessor.get_avatar(room_id).await { | ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url), | ruma::JsOption::Null => ruma::JsOption::Null, | ruma::JsOption::Undefined => ruma::JsOption::Undefined, - } + }, }, initial: Some(roomsince == &0), is_dm: None, diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index cda6c041..b4c1b815 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -6,32 +6,33 @@ use std::{ use axum::extract::State; use conduwuit::{ - debug, error, extract_variant, trace, + Error, Result, TypeStateKey, debug, error, extract_variant, trace, utils::{ - math::{ruma_from_usize, usize_from_ruma}, BoolExt, IterStream, ReadyExt, TryFutureExtExt, + math::{ruma_from_usize, usize_from_ruma}, }, - warn, Error, Result, TypeStateKey, + warn, }; use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{ + DeviceId, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, api::client::{ error::ErrorKind, sync::sync_events::{self, DeviceLists, UnreadNotificationsCount}, }, events::{ - room::member::{MembershipState, RoomMemberEventContent}, AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType, + room::member::{MembershipState, RoomMemberEventContent}, }, serde::Raw, - uint, DeviceId, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, + uint, }; -use service::{rooms::read_receipt::pack_receipts, PduCount}; +use service::{PduCount, rooms::read_receipt::pack_receipts}; use super::{filter_rooms, share_encrypted_room}; use crate::{ - client::{ignored_filter, sync::load_timeline, DEFAULT_BUMP_TYPES}, Ruma, + client::{DEFAULT_BUMP_TYPES, ignored_filter, sync::load_timeline}, }; type SyncInfo<'a> = (&'a UserId, &'a DeviceId, u64, &'a sync_events::v5::Request); @@ -572,14 +573,13 @@ async fn process_rooms( .await .ok() .or(name), - avatar: if let Some(heroes_avatar) = heroes_avatar { - ruma::JsOption::Some(heroes_avatar) - } else { - match services.rooms.state_accessor.get_avatar(room_id).await { + avatar: match heroes_avatar { + | Some(heroes_avatar) => ruma::JsOption::Some(heroes_avatar), + | _ => match services.rooms.state_accessor.get_avatar(room_id).await { | ruma::JsOption::Some(avatar) => ruma::JsOption::from_option(avatar.url), | ruma::JsOption::Null => ruma::JsOption::Null, | ruma::JsOption::Undefined => ruma::JsOption::Undefined, - } + }, }, initial: Some(roomsince == &0), is_dm: None, diff --git a/src/api/client/tag.rs b/src/api/client/tag.rs index 820ee4a1..3b3b40d4 100644 --- a/src/api/client/tag.rs +++ b/src/api/client/tag.rs @@ -4,8 +4,8 @@ use axum::extract::State; use ruma::{ api::client::tag::{create_tag, delete_tag, get_tags}, events::{ - tag::{TagEvent, TagEventContent}, RoomAccountDataEventType, + tag::{TagEvent, TagEventContent}, }, }; diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index f0cbf467..d25e52c0 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduwuit::{at, PduCount, PduEvent}; +use conduwuit::{PduCount, PduEvent, at}; use futures::StreamExt; use ruma::{api::client::threads::get_threads, uint}; diff --git a/src/api/client/typing.rs b/src/api/client/typing.rs index b311295b..ccfa7340 100644 --- a/src/api/client/typing.rs +++ b/src/api/client/typing.rs @@ -1,8 +1,8 @@ use axum::extract::State; -use conduwuit::{utils::math::Tried, Err}; +use conduwuit::{Err, utils::math::Tried}; use ruma::api::client::typing::create_typing_event; -use crate::{utils, Result, Ruma}; +use crate::{Result, Ruma, utils}; /// # `PUT /_matrix/client/r0/rooms/{roomId}/typing/{userId}` /// @@ -27,37 +27,40 @@ pub(crate) async fn create_typing_event_route( return Err!(Request(Forbidden("You are not in this room."))); } - if let Typing::Yes(duration) = body.state { - let duration = utils::clamp( - duration.as_millis().try_into().unwrap_or(u64::MAX), + match body.state { + | Typing::Yes(duration) => { + let duration = utils::clamp( + duration.as_millis().try_into().unwrap_or(u64::MAX), + services + .server + .config + .typing_client_timeout_min_s + .try_mul(1000)?, + services + .server + .config + .typing_client_timeout_max_s + .try_mul(1000)?, + ); services - .server - .config - .typing_client_timeout_min_s - .try_mul(1000)?, + .rooms + .typing + .typing_add( + sender_user, + &body.room_id, + utils::millis_since_unix_epoch() + .checked_add(duration) + .expect("user typing timeout should not get this high"), + ) + .await?; + }, + | _ => { services - .server - .config - .typing_client_timeout_max_s - .try_mul(1000)?, - ); - services - .rooms - .typing - .typing_add( - sender_user, - &body.room_id, - utils::millis_since_unix_epoch() - .checked_add(duration) - .expect("user typing timeout should not get this high"), - ) - .await?; - } else { - services - .rooms - .typing - .typing_remove(sender_user, &body.room_id) - .await?; + .rooms + .typing + .typing_remove(sender_user, &body.room_id) + .await?; + }, } // ping presence diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 67c7df75..08da5a37 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -5,6 +5,7 @@ use axum_client_ip::InsecureClientIp; use conduwuit::Err; use futures::StreamExt; use ruma::{ + OwnedRoomId, api::{ client::{ error::ErrorKind, @@ -19,7 +20,6 @@ use ruma::{ }, events::room::member::MembershipState, presence::PresenceState, - OwnedRoomId, }; use super::{update_avatar_url, update_displayname}; @@ -499,15 +499,18 @@ pub(crate) async fn get_profile_key_route( .users .set_timezone(&body.user_id, response.tz.clone()); - if let Some(value) = response.custom_profile_fields.get(&body.key_name) { - profile_key_value.insert(body.key_name.clone(), value.clone()); - services.users.set_profile_key( - &body.user_id, - &body.key_name, - Some(value.clone()), - ); - } else { - return Err!(Request(NotFound("The requested profile key does not exist."))); + match response.custom_profile_fields.get(&body.key_name) { + | Some(value) => { + profile_key_value.insert(body.key_name.clone(), value.clone()); + services.users.set_profile_key( + &body.user_id, + &body.key_name, + Some(value.clone()), + ); + }, + | _ => { + return Err!(Request(NotFound("The requested profile key does not exist."))); + }, } if profile_key_value.is_empty() { @@ -524,14 +527,17 @@ pub(crate) async fn get_profile_key_route( return Err!(Request(NotFound("Profile was not found."))); } - if let Ok(value) = services + match services .users .profile_key(&body.user_id, &body.key_name) .await { - profile_key_value.insert(body.key_name.clone(), value); - } else { - return Err!(Request(NotFound("The requested profile key does not exist."))); + | Ok(value) => { + profile_key_value.insert(body.key_name.clone(), value); + }, + | _ => { + return Err!(Request(NotFound("The requested profile key does not exist."))); + }, } if profile_key_value.is_empty() { diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 904f1d2f..4e2b7d9d 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use axum::{extract::State, response::IntoResponse, Json}; +use axum::{Json, extract::State, response::IntoResponse}; use futures::StreamExt; use ruma::api::client::discovery::get_supported_versions; diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index 182e30db..c5d79a56 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -1,11 +1,11 @@ use axum::extract::State; use conduwuit::utils::TryFutureExtExt; -use futures::{pin_mut, StreamExt}; +use futures::{StreamExt, pin_mut}; use ruma::{ api::client::user_directory::search_users, events::{ - room::join_rules::{JoinRule, RoomJoinRulesEventContent}, StateEventType, + room::join_rules::{JoinRule, RoomJoinRulesEventContent}, }, }; diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index 70ad4913..37e67984 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -1,10 +1,10 @@ use std::time::{Duration, SystemTime}; use axum::extract::State; -use base64::{engine::general_purpose, Engine as _}; -use conduwuit::{utils, Err}; +use base64::{Engine as _, engine::general_purpose}; +use conduwuit::{Err, utils}; use hmac::{Hmac, Mac}; -use ruma::{api::client::voip::get_turn_server_info, SecondsSinceUnixEpoch, UserId}; +use ruma::{SecondsSinceUnixEpoch, UserId, api::client::voip::get_turn_server_info}; use sha1::Sha1; use crate::{Result, Ruma}; diff --git a/src/api/client/well_known.rs b/src/api/client/well_known.rs index 5c53d013..abda61b0 100644 --- a/src/api/client/well_known.rs +++ b/src/api/client/well_known.rs @@ -1,4 +1,4 @@ -use axum::{extract::State, response::IntoResponse, Json}; +use axum::{Json, extract::State, response::IntoResponse}; use ruma::api::client::{ discovery::{ discover_homeserver::{self, HomeserverInfo, SlidingSyncProxyInfo}, diff --git a/src/api/mod.rs b/src/api/mod.rs index 80e34f10..8df17a59 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -7,7 +7,7 @@ pub mod server; extern crate conduwuit_core as conduwuit; extern crate conduwuit_service as service; -pub(crate) use conduwuit::{debug_info, pdu::PduEvent, utils, Error, Result}; +pub(crate) use conduwuit::{Error, Result, debug_info, pdu::PduEvent, utils}; pub(crate) use self::router::{Ruma, RumaResponse, State}; diff --git a/src/api/router.rs b/src/api/router.rs index 7855ddfa..3fbef275 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -8,12 +8,12 @@ pub mod state; use std::str::FromStr; use axum::{ + Router, response::{IntoResponse, Redirect}, routing::{any, get, post}, - Router, }; -use conduwuit::{err, Server}; -use http::{uri, Uri}; +use conduwuit::{Server, err}; +use http::{Uri, uri}; use self::handler::RouterExt; pub(super) use self::{args::Args as Ruma, response::RumaResponse, state::State}; diff --git a/src/api/router/args.rs b/src/api/router/args.rs index 582f0c56..65a68fa4 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -2,15 +2,15 @@ use std::{mem, ops::Deref}; use axum::{async_trait, body::Body, extract::FromRequest}; use bytes::{BufMut, Bytes, BytesMut}; -use conduwuit::{debug, debug_warn, err, trace, utils::string::EMPTY, Error, Result}; +use conduwuit::{Error, Result, debug, debug_warn, err, trace, utils::string::EMPTY}; use ruma::{ - api::IncomingRequest, CanonicalJsonObject, CanonicalJsonValue, DeviceId, OwnedDeviceId, - OwnedServerName, OwnedUserId, ServerName, UserId, + CanonicalJsonObject, CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedServerName, + OwnedUserId, ServerName, UserId, api::IncomingRequest, }; use service::Services; use super::{auth, auth::Auth, request, request::Request}; -use crate::{service::appservice::RegistrationInfo, State}; +use crate::{State, service::appservice::RegistrationInfo}; /// Extractor for Ruma request structs pub(crate) struct Args { diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index ecea305b..56256683 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -1,12 +1,14 @@ use axum::RequestPartsExt; use axum_extra::{ - headers::{authorization::Bearer, Authorization}, - typed_header::TypedHeaderRejectionReason, TypedHeader, + headers::{Authorization, authorization::Bearer}, + typed_header::TypedHeaderRejectionReason, }; -use conduwuit::{debug_error, err, warn, Err, Error, Result}; +use conduwuit::{Err, Error, Result, debug_error, err, warn}; use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, api::{ + AuthScheme, IncomingRequest, Metadata, client::{ directory::get_public_rooms, error::ErrorKind, @@ -16,14 +18,12 @@ use ruma::{ voip::get_turn_server_info, }, federation::openid::get_openid_userinfo, - AuthScheme, IncomingRequest, Metadata, }, server_util::authorization::XMatrix, - CanonicalJsonObject, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, }; use service::{ - server_keys::{PubKeyMap, PubKeys}, Services, + server_keys::{PubKeyMap, PubKeys}, }; use super::request::Request; @@ -56,12 +56,12 @@ pub(super) async fn auth( }; let token = if let Some(token) = token { - if let Some(reg_info) = services.appservice.find_from_token(token).await { - Token::Appservice(Box::new(reg_info)) - } else if let Ok((user_id, device_id)) = services.users.find_from_token(token).await { - Token::User((user_id, device_id)) - } else { - Token::Invalid + match services.appservice.find_from_token(token).await { + | Some(reg_info) => Token::Appservice(Box::new(reg_info)), + | _ => match services.users.find_from_token(token).await { + | Ok((user_id, device_id)) => Token::User((user_id, device_id)), + | _ => Token::Invalid, + }, } } else { Token::None diff --git a/src/api/router/handler.rs b/src/api/router/handler.rs index cfb8fb6e..ab013945 100644 --- a/src/api/router/handler.rs +++ b/src/api/router/handler.rs @@ -1,8 +1,8 @@ use axum::{ + Router, extract::FromRequestParts, response::IntoResponse, - routing::{on, MethodFilter}, - Router, + routing::{MethodFilter, on}, }; use conduwuit::Result; use futures::{Future, TryFutureExt}; diff --git a/src/api/router/request.rs b/src/api/router/request.rs index 615a8bff..3cdc452b 100644 --- a/src/api/router/request.rs +++ b/src/api/router/request.rs @@ -1,8 +1,8 @@ use std::str; -use axum::{extract::Path, RequestExt, RequestPartsExt}; +use axum::{RequestExt, RequestPartsExt, extract::Path}; use bytes::Bytes; -use conduwuit::{err, Result}; +use conduwuit::{Result, err}; use http::request::Parts; use serde::Deserialize; use service::Services; diff --git a/src/api/router/response.rs b/src/api/router/response.rs index a10560f1..03c9060e 100644 --- a/src/api/router/response.rs +++ b/src/api/router/response.rs @@ -1,9 +1,9 @@ use axum::response::{IntoResponse, Response}; use bytes::BytesMut; -use conduwuit::{error, Error}; +use conduwuit::{Error, error}; use http::StatusCode; use http_body_util::Full; -use ruma::api::{client::uiaa::UiaaResponse, OutgoingResponse}; +use ruma::api::{OutgoingResponse, client::uiaa::UiaaResponse}; pub(crate) struct RumaResponse(pub(crate) T) where diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index b44db67c..5c875807 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -2,11 +2,11 @@ use std::cmp; use axum::extract::State; use conduwuit::{ - utils::{stream::TryTools, IterStream, ReadyExt}, PduCount, Result, + utils::{IterStream, ReadyExt, stream::TryTools}, }; use futures::{FutureExt, StreamExt, TryStreamExt}; -use ruma::{api::federation::backfill::get_backfill, uint, MilliSecondsSinceUnixEpoch}; +use ruma::{MilliSecondsSinceUnixEpoch, api::federation::backfill::get_backfill, uint}; use super::AccessCheck; use crate::Ruma; diff --git a/src/api/server/event.rs b/src/api/server/event.rs index 629dd6a2..5846c6d7 100644 --- a/src/api/server/event.rs +++ b/src/api/server/event.rs @@ -1,6 +1,6 @@ use axum::extract::State; -use conduwuit::{err, Result}; -use ruma::{api::federation::event::get_event, MilliSecondsSinceUnixEpoch, RoomId}; +use conduwuit::{Result, err}; +use ruma::{MilliSecondsSinceUnixEpoch, RoomId, api::federation::event::get_event}; use super::AccessCheck; use crate::Ruma; diff --git a/src/api/server/event_auth.rs b/src/api/server/event_auth.rs index 49dcd718..c9e210f5 100644 --- a/src/api/server/event_auth.rs +++ b/src/api/server/event_auth.rs @@ -1,11 +1,11 @@ use std::{borrow::Borrow, iter::once}; use axum::extract::State; -use conduwuit::{utils::stream::ReadyExt, Error, Result}; +use conduwuit::{Error, Result, utils::stream::ReadyExt}; use futures::StreamExt; use ruma::{ - api::{client::error::ErrorKind, federation::authorization::get_event_authorization}, RoomId, + api::{client::error::ErrorKind, federation::authorization::get_event_authorization}, }; use super::AccessCheck; diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index ea06015a..3d0bbb07 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -1,8 +1,8 @@ use axum::extract::State; use conduwuit::{Error, Result}; use ruma::{ - api::{client::error::ErrorKind, federation::event::get_missing_events}, CanonicalJsonValue, EventId, RoomId, + api::{client::error::ErrorKind, federation::event::get_missing_events}, }; use super::AccessCheck; diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs index f7bc43ab..41eaedd0 100644 --- a/src/api/server/hierarchy.rs +++ b/src/api/server/hierarchy.rs @@ -1,11 +1,11 @@ use axum::extract::State; use conduwuit::{ - utils::stream::{BroadbandExt, IterStream}, Err, Result, + utils::stream::{BroadbandExt, IterStream}, }; use futures::{FutureExt, StreamExt}; use ruma::api::federation::space::get_hierarchy; -use service::rooms::spaces::{get_parent_children_via, Identifier, SummaryAccessibility}; +use service::rooms::spaces::{Identifier, SummaryAccessibility, get_parent_children_via}; use crate::Ruma; diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 27a4485c..463cb9ab 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -1,12 +1,12 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use base64::{engine::general_purpose, Engine as _}; -use conduwuit::{err, utils, utils::hash::sha256, warn, Err, Error, PduEvent, Result}; +use base64::{Engine as _, engine::general_purpose}; +use conduwuit::{Err, Error, PduEvent, Result, err, utils, utils::hash::sha256, warn}; use ruma::{ + CanonicalJsonValue, OwnedUserId, UserId, api::{client::error::ErrorKind, federation::membership::create_invite}, events::room::member::{MembershipState, RoomMemberEventContent}, serde::JsonObject, - CanonicalJsonValue, OwnedUserId, UserId, }; use service::pdu::gen_event_id; diff --git a/src/api/server/key.rs b/src/api/server/key.rs index 75801a7a..f9bd0926 100644 --- a/src/api/server/key.rs +++ b/src/api/server/key.rs @@ -3,15 +3,15 @@ use std::{ time::{Duration, SystemTime}, }; -use axum::{extract::State, response::IntoResponse, Json}; -use conduwuit::{utils::timepoint_from_now, Result}; +use axum::{Json, extract::State, response::IntoResponse}; +use conduwuit::{Result, utils::timepoint_from_now}; use ruma::{ + MilliSecondsSinceUnixEpoch, Signatures, api::{ - federation::discovery::{get_server_keys, OldVerifyKey, ServerSigningKeys}, OutgoingResponse, + federation::discovery::{OldVerifyKey, ServerSigningKeys, get_server_keys}, }, serde::Raw, - MilliSecondsSinceUnixEpoch, Signatures, }; /// # `GET /_matrix/key/v2/server` diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index b753346c..f18d1304 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -1,22 +1,22 @@ use axum::extract::State; -use conduwuit::{debug_info, utils::IterStream, warn, Err}; +use conduwuit::{Err, debug_info, utils::IterStream, warn}; use futures::StreamExt; use ruma::{ + CanonicalJsonObject, OwnedUserId, RoomId, RoomVersionId, UserId, api::{client::error::ErrorKind, federation::membership::prepare_join_event}, events::{ + StateEventType, room::{ join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent}, member::{MembershipState, RoomMemberEventContent}, }, - StateEventType, }, - CanonicalJsonObject, OwnedUserId, RoomId, RoomVersionId, UserId, }; use serde_json::value::to_raw_value; use crate::{ - service::{pdu::PduBuilder, Services}, Error, Result, Ruma, + service::{Services, pdu::PduBuilder}, }; /// # `GET /_matrix/federation/v1/make_join/{roomId}/{userId}` diff --git a/src/api/server/make_knock.rs b/src/api/server/make_knock.rs index 423e202d..71536439 100644 --- a/src/api/server/make_knock.rs +++ b/src/api/server/make_knock.rs @@ -1,15 +1,15 @@ +use RoomVersionId::*; use axum::extract::State; -use conduwuit::{debug_warn, Err}; +use conduwuit::{Err, debug_warn}; use ruma::{ + RoomVersionId, api::{client::error::ErrorKind, federation::knock::create_knock_event_template}, events::room::member::{MembershipState, RoomMemberEventContent}, - RoomVersionId, }; use serde_json::value::to_raw_value; use tracing::warn; -use RoomVersionId::*; -use crate::{service::pdu::PduBuilder, Error, Result, Ruma}; +use crate::{Error, Result, Ruma, service::pdu::PduBuilder}; /// # `GET /_matrix/federation/v1/make_knock/{roomId}/{userId}` /// diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 936e0fbb..1ed02785 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -7,7 +7,7 @@ use ruma::{ use serde_json::value::to_raw_value; use super::make_join::maybe_strip_event_id; -use crate::{service::pdu::PduBuilder, Ruma}; +use crate::{Ruma, service::pdu::PduBuilder}; /// # `GET /_matrix/federation/v1/make_leave/{roomId}/{eventId}` /// diff --git a/src/api/server/media.rs b/src/api/server/media.rs index e56f5b9d..cbe8595b 100644 --- a/src/api/server/media.rs +++ b/src/api/server/media.rs @@ -1,12 +1,12 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{utils::content_disposition::make_content_disposition, Err, Result}; +use conduwuit::{Err, Result, utils::content_disposition::make_content_disposition}; use conduwuit_service::media::{Dim, FileMeta}; use ruma::{ - api::federation::authenticated_media::{ - get_content, get_content_thumbnail, Content, ContentMetadata, FileOrLocation, - }, Mxc, + api::federation::authenticated_media::{ + Content, ContentMetadata, FileOrLocation, get_content, get_content_thumbnail, + }, }; use crate::Ruma; diff --git a/src/api/server/query.rs b/src/api/server/query.rs index 69f62e94..9d4fcf73 100644 --- a/src/api/server/query.rs +++ b/src/api/server/query.rs @@ -1,16 +1,16 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduwuit::{err, Error, Result}; +use conduwuit::{Error, Result, err}; use futures::StreamExt; use get_profile_information::v1::ProfileField; use rand::seq::SliceRandom; use ruma::{ + OwnedServerName, api::{ client::error::ErrorKind, federation::query::{get_profile_information, get_room_information}, }, - OwnedServerName, }; use crate::Ruma; diff --git a/src/api/server/send.rs b/src/api/server/send.rs index bc18377e..1f467dac 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -3,20 +3,21 @@ use std::{collections::BTreeMap, net::IpAddr, time::Instant}; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - debug, + Err, Error, Result, debug, debug::INFO_SPAN_LEVEL, debug_warn, err, error, result::LogErr, trace, utils::{ - stream::{automatic_width, BroadbandExt, TryBroadbandExt}, IterStream, ReadyExt, + stream::{BroadbandExt, TryBroadbandExt, automatic_width}, }, - warn, Err, Error, Result, + warn, }; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use itertools::Itertools; use ruma::{ + CanonicalJsonObject, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, ServerName, UserId, api::{ client::error::ErrorKind, federation::transactions::{ @@ -31,17 +32,16 @@ use ruma::{ events::receipt::{ReceiptEvent, ReceiptEventContent, ReceiptType}, serde::Raw, to_device::DeviceIdOrAllDevices, - CanonicalJsonObject, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, ServerName, UserId, }; use service::{ - sending::{EDU_LIMIT, PDU_LIMIT}, Services, + sending::{EDU_LIMIT, PDU_LIMIT}, }; use utils::millis_since_unix_epoch; use crate::{ - utils::{self}, Ruma, + utils::{self}, }; type ResolvedMap = BTreeMap; diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index e81d7672..08fa3835 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -4,22 +4,22 @@ use std::borrow::Borrow; use axum::extract::State; use conduwuit::{ - at, err, + Err, Result, at, err, pdu::gen_event_id_canonical_json, utils::stream::{IterStream, TryBroadbandExt}, - warn, Err, Result, + warn, }; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ - api::federation::membership::create_join_event, - events::{ - room::member::{MembershipState, RoomMemberEventContent}, - StateEventType, - }, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, ServerName, + api::federation::membership::create_join_event, + events::{ + StateEventType, + room::member::{MembershipState, RoomMemberEventContent}, + }, }; -use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; use service::Services; use crate::Ruma; diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs index b07620af..1d4c2a6c 100644 --- a/src/api/server/send_knock.rs +++ b/src/api/server/send_knock.rs @@ -1,15 +1,15 @@ use axum::extract::State; -use conduwuit::{err, pdu::gen_event_id_canonical_json, warn, Err, PduEvent, Result}; +use conduwuit::{Err, PduEvent, Result, err, pdu::gen_event_id_canonical_json, warn}; use futures::FutureExt; use ruma::{ - api::federation::knock::send_knock, - events::{ - room::member::{MembershipState, RoomMemberEventContent}, - StateEventType, - }, - serde::JsonObject, OwnedServerName, OwnedUserId, RoomVersionId::*, + api::federation::knock::send_knock, + events::{ + StateEventType, + room::member::{MembershipState, RoomMemberEventContent}, + }, + serde::JsonObject, }; use crate::Ruma; diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index e955a267..71516553 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -1,21 +1,21 @@ #![allow(deprecated)] use axum::extract::State; -use conduwuit::{err, Err, Result}; +use conduwuit::{Err, Result, err}; use futures::FutureExt; use ruma::{ + OwnedRoomId, OwnedUserId, RoomId, ServerName, api::federation::membership::create_leave_event, events::{ - room::member::{MembershipState, RoomMemberEventContent}, StateEventType, + room::member::{MembershipState, RoomMemberEventContent}, }, - OwnedRoomId, OwnedUserId, RoomId, ServerName, }; use serde_json::value::RawValue as RawJsonValue; use crate::{ - service::{pdu::gen_event_id_canonical_json, Services}, Ruma, + service::{Services, pdu::gen_event_id_canonical_json}, }; /// # `PUT /_matrix/federation/v1/send_leave/{roomId}/{eventId}` diff --git a/src/api/server/state.rs b/src/api/server/state.rs index b16e61a0..8c786815 100644 --- a/src/api/server/state.rs +++ b/src/api/server/state.rs @@ -1,9 +1,9 @@ use std::{borrow::Borrow, iter::once}; use axum::extract::State; -use conduwuit::{at, err, utils::IterStream, Result}; +use conduwuit::{Result, at, err, utils::IterStream}; use futures::{FutureExt, StreamExt, TryStreamExt}; -use ruma::{api::federation::event::get_room_state, OwnedEventId}; +use ruma::{OwnedEventId, api::federation::event::get_room_state}; use super::AccessCheck; use crate::Ruma; diff --git a/src/api/server/state_ids.rs b/src/api/server/state_ids.rs index 7d0440bf..648d4575 100644 --- a/src/api/server/state_ids.rs +++ b/src/api/server/state_ids.rs @@ -1,9 +1,9 @@ use std::{borrow::Borrow, iter::once}; use axum::extract::State; -use conduwuit::{at, err, Result}; +use conduwuit::{Result, at, err}; use futures::{StreamExt, TryStreamExt}; -use ruma::{api::federation::event::get_room_state_ids, OwnedEventId}; +use ruma::{OwnedEventId, api::federation::event::get_room_state_ids}; use super::AccessCheck; use crate::Ruma; diff --git a/src/api/server/user.rs b/src/api/server/user.rs index 321d0b66..80c353ab 100644 --- a/src/api/server/user.rs +++ b/src/api/server/user.rs @@ -10,8 +10,8 @@ use ruma::api::{ }; use crate::{ - client::{claim_keys_helper, get_keys_helper}, Ruma, + client::{claim_keys_helper, get_keys_helper}, }; /// # `GET /_matrix/federation/v1/user/devices/{userId}` diff --git a/src/api/server/utils.rs b/src/api/server/utils.rs index 4f3fa245..5696e44b 100644 --- a/src/api/server/utils.rs +++ b/src/api/server/utils.rs @@ -1,6 +1,6 @@ -use conduwuit::{implement, is_false, Err, Result}; +use conduwuit::{Err, Result, implement, is_false}; use conduwuit_service::Services; -use futures::{future::OptionFuture, join, FutureExt, StreamExt}; +use futures::{FutureExt, StreamExt, future::OptionFuture, join}; use ruma::{EventId, RoomId, ServerName}; pub(super) struct AccessCheck<'a> { diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 57143e85..6870c1c0 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -2,7 +2,7 @@ use std::{ cell::OnceCell, - ffi::{c_char, c_void, CStr}, + ffi::{CStr, c_char, c_void}, fmt::Debug, sync::RwLock, }; @@ -14,9 +14,8 @@ use tikv_jemalloc_sys as ffi; use tikv_jemallocator as jemalloc; use crate::{ - err, is_equal_to, is_nonzero, + Result, err, is_equal_to, is_nonzero, utils::{math, math::Tried}, - Result, }; #[cfg(feature = "jemalloc_conf")] @@ -128,7 +127,7 @@ unsafe extern "C" fn malloc_stats_cb(opaque: *mut c_void, msg: *const c_char) { } macro_rules! mallctl { - ($name:expr) => {{ + ($name:expr_2021) => {{ thread_local! { static KEY: OnceCell = OnceCell::default(); }; @@ -141,7 +140,7 @@ macro_rules! mallctl { } pub mod this_thread { - use super::{is_nonzero, key, math, Debug, Key, OnceCell, Result}; + use super::{Debug, Key, OnceCell, Result, is_nonzero, key, math}; thread_local! { static ALLOCATED_BYTES: OnceCell<&'static u64> = const { OnceCell::new() }; @@ -261,18 +260,18 @@ pub fn decay>>(arena: I) -> Result { } pub fn set_muzzy_decay>>(arena: I, decay_ms: isize) -> Result { - if let Some(arena) = arena.into() { - set_by_arena(Some(arena), mallctl!("arena.4096.muzzy_decay_ms"), decay_ms) - } else { - set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms) + match arena.into() { + | Some(arena) => + set_by_arena(Some(arena), mallctl!("arena.4096.muzzy_decay_ms"), decay_ms), + | _ => set(&mallctl!("arenas.muzzy_decay_ms"), decay_ms), } } pub fn set_dirty_decay>>(arena: I, decay_ms: isize) -> Result { - if let Some(arena) = arena.into() { - set_by_arena(Some(arena), mallctl!("arena.4096.dirty_decay_ms"), decay_ms) - } else { - set(&mallctl!("arenas.dirty_decay_ms"), decay_ms) + match arena.into() { + | Some(arena) => + set_by_arena(Some(arena), mallctl!("arena.4096.dirty_decay_ms"), decay_ms), + | _ => set(&mallctl!("arenas.dirty_decay_ms"), decay_ms), } } diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 5532c5a2..488f7f94 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -4,7 +4,7 @@ use either::Either; use figment::Figment; use super::DEPRECATED_KEYS; -use crate::{debug, debug_info, debug_warn, error, warn, Config, Err, Result, Server}; +use crate::{Config, Err, Result, Server, debug, debug_info, debug_warn, error, warn}; /// Performs check() with additional checks specific to reloading old config /// with new config. diff --git a/src/core/config/manager.rs b/src/core/config/manager.rs index 0c95ca15..e55916ba 100644 --- a/src/core/config/manager.rs +++ b/src/core/config/manager.rs @@ -4,13 +4,13 @@ use std::{ ptr, ptr::null_mut, sync::{ - atomic::{AtomicPtr, Ordering}, Arc, + atomic::{AtomicPtr, Ordering}, }, }; use super::Config; -use crate::{implement, Result}; +use crate::{Result, implement}; /// The configuration manager is an indirection to reload the configuration for /// the server while it is running. In order to not burden or clutter the many diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index e66532ee..67c3b95c 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -14,18 +14,18 @@ use either::{ Either::{Left, Right}, }; use figment::providers::{Env, Format, Toml}; -pub use figment::{value::Value as FigmentValue, Figment}; +pub use figment::{Figment, value::Value as FigmentValue}; use regex::RegexSet; use ruma::{ - api::client::discovery::discover_support::ContactRole, OwnedRoomOrAliasId, OwnedServerName, - OwnedUserId, RoomVersionId, + OwnedRoomOrAliasId, OwnedServerName, OwnedUserId, RoomVersionId, + api::client::discovery::discover_support::ContactRole, }; -use serde::{de::IgnoredAny, Deserialize}; +use serde::{Deserialize, de::IgnoredAny}; use url::Url; use self::proxy::ProxyConfig; pub use self::{check::check, manager::Manager}; -use crate::{err, error::Error, utils::sys, Result}; +use crate::{Result, err, error::Error, utils::sys}; /// All the config options for conduwuit. #[allow(clippy::struct_excessive_bools)] diff --git a/src/core/debug.rs b/src/core/debug.rs index 8a5eccfd..b9a53038 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -13,7 +13,7 @@ pub use crate::{result::DebugInspect, utils::debug::*}; /// In release-mode it becomes DEBUG level, and possibly subject to elision. #[macro_export] macro_rules! debug_event { - ( $level:expr, $($x:tt)+ ) => { + ( $level:expr_2021, $($x:tt)+ ) => { if $crate::debug::logging() { ::tracing::event!( $level, _debug = true, $($x)+ ) } else { diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 60fa5bff..0962c4ee 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -165,10 +165,10 @@ macro_rules! err_lev { use std::{fmt, fmt::Write}; use tracing::{ - level_enabled, Callsite, Event, __macro_support, __tracing_log, + __macro_support, __tracing_log, Callsite, Event, Level, callsite::DefaultCallsite, field::{Field, ValueSet, Visit}, - Level, + level_enabled, }; struct Visitor<'a>(&'a mut String); diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 16613b7e..02ab6fa3 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -152,8 +152,8 @@ impl Error { /// Generate the error message string. pub fn message(&self) -> String { match self { - | Self::Federation(ref origin, ref error) => format!("Answer from {origin}: {error}"), - | Self::Ruma(ref error) => response::ruma_error_message(error), + | Self::Federation(origin, error) => format!("Answer from {origin}: {error}"), + | Self::Ruma(error) => response::ruma_error_message(error), | _ => format!("{self}"), } } diff --git a/src/core/error/panic.rs b/src/core/error/panic.rs index c6a83ae0..2e63105b 100644 --- a/src/core/error/panic.rs +++ b/src/core/error/panic.rs @@ -1,6 +1,6 @@ use std::{ any::Any, - panic::{panic_any, RefUnwindSafe, UnwindSafe}, + panic::{RefUnwindSafe, UnwindSafe, panic_any}, }; use super::Error; diff --git a/src/core/error/response.rs b/src/core/error/response.rs index 75e4050d..00ade5ae 100644 --- a/src/core/error/response.rs +++ b/src/core/error/response.rs @@ -2,11 +2,11 @@ use bytes::BytesMut; use http::StatusCode; use http_body_util::Full; use ruma::api::{ + OutgoingResponse, client::{ error::{ErrorBody, ErrorKind}, uiaa::UiaaResponse, }, - OutgoingResponse, }; use super::Error; diff --git a/src/core/info/room_version.rs b/src/core/info/room_version.rs index b33a8562..51d5d3c6 100644 --- a/src/core/info/room_version.rs +++ b/src/core/info/room_version.rs @@ -2,7 +2,7 @@ use std::iter::once; -use ruma::{api::client::discovery::get_capabilities::RoomVersionStability, RoomVersionId}; +use ruma::{RoomVersionId, api::client::discovery::get_capabilities::RoomVersionStability}; use crate::{at, is_equal_to}; diff --git a/src/core/log/capture/data.rs b/src/core/log/capture/data.rs index 0ad7a6c2..a4a1225b 100644 --- a/src/core/log/capture/data.rs +++ b/src/core/log/capture/data.rs @@ -1,7 +1,7 @@ use tracing::Level; -use tracing_core::{span::Current, Event}; +use tracing_core::{Event, span::Current}; -use super::{layer::Value, Layer}; +use super::{Layer, layer::Value}; use crate::{info, utils::string::EMPTY}; pub struct Data<'a> { diff --git a/src/core/log/capture/util.rs b/src/core/log/capture/util.rs index 8bad4ba0..65524be5 100644 --- a/src/core/log/capture/util.rs +++ b/src/core/log/capture/util.rs @@ -1,7 +1,7 @@ use std::sync::{Arc, Mutex}; use super::{ - super::{fmt, Level}, + super::{Level, fmt}, Closure, Data, }; use crate::Result; diff --git a/src/core/log/console.rs b/src/core/log/console.rs index 1f04ba26..d91239ac 100644 --- a/src/core/log/console.rs +++ b/src/core/log/console.rs @@ -1,20 +1,20 @@ use std::{env, io, sync::LazyLock}; use tracing::{ - field::{Field, Visit}, Event, Level, Subscriber, + field::{Field, Visit}, }; use tracing_subscriber::{ field::RecordFields, fmt, fmt::{ - format::{Compact, DefaultVisitor, Format, Full, Pretty, Writer}, FmtContext, FormatEvent, FormatFields, MakeWriter, + format::{Compact, DefaultVisitor, Format, Full, Pretty, Writer}, }, registry::LookupSpan, }; -use crate::{apply, Config, Result}; +use crate::{Config, Result, apply}; static SYSTEMD_MODE: LazyLock = LazyLock::new(|| env::var("SYSTEMD_EXEC_PID").is_ok() && env::var("JOURNAL_STREAM").is_ok()); diff --git a/src/core/log/fmt.rs b/src/core/log/fmt.rs index 353d4442..b73d0c9b 100644 --- a/src/core/log/fmt.rs +++ b/src/core/log/fmt.rs @@ -1,6 +1,6 @@ use std::fmt::Write; -use super::{color, Level}; +use super::{Level, color}; use crate::Result; pub fn html(out: &mut S, level: &Level, span: &str, msg: &str) -> Result<()> diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 0c1840d0..5ac374e8 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -9,7 +9,7 @@ mod reload; mod suppress; pub use capture::Capture; -pub use console::{is_systemd_mode, ConsoleFormat, ConsoleWriter}; +pub use console::{ConsoleFormat, ConsoleWriter, is_systemd_mode}; pub use reload::{LogLevelReloadHandles, ReloadHandle}; pub use suppress::Suppress; pub use tracing::Level; @@ -34,7 +34,7 @@ pub struct Log { #[macro_export] macro_rules! event { - ( $level:expr, $($x:tt)+ ) => { ::tracing::event!( $level, $($x)+ ) } + ( $level:expr_2021, $($x:tt)+ ) => { ::tracing::event!( $level, $($x)+ ) } } #[macro_export] diff --git a/src/core/log/reload.rs b/src/core/log/reload.rs index 12d14f48..e6a16c9f 100644 --- a/src/core/log/reload.rs +++ b/src/core/log/reload.rs @@ -3,9 +3,9 @@ use std::{ sync::{Arc, Mutex}, }; -use tracing_subscriber::{reload, EnvFilter}; +use tracing_subscriber::{EnvFilter, reload}; -use crate::{error, Result}; +use crate::{Result, error}; /// We need to store a reload::Handle value, but can't name it's type explicitly /// because the S type parameter depends on the subscriber's previous layers. In diff --git a/src/core/mods/module.rs b/src/core/mods/module.rs index ff181e4f..b65bbca2 100644 --- a/src/core/mods/module.rs +++ b/src/core/mods/module.rs @@ -3,8 +3,8 @@ use std::{ time::SystemTime, }; -use super::{canary, new, path, Library, Symbol}; -use crate::{error, Result}; +use super::{Library, Symbol, canary, new, path}; +use crate::{Result, error}; pub struct Module { handle: Option, diff --git a/src/core/mods/new.rs b/src/core/mods/new.rs index 77d89af4..258fdedc 100644 --- a/src/core/mods/new.rs +++ b/src/core/mods/new.rs @@ -1,6 +1,6 @@ use std::ffi::OsStr; -use super::{path, Library}; +use super::{Library, path}; use crate::{Err, Result}; const OPEN_FLAGS: i32 = libloading::os::unix::RTLD_LAZY | libloading::os::unix::RTLD_GLOBAL; diff --git a/src/core/pdu/builder.rs b/src/core/pdu/builder.rs index 0efee128..5aa0c9ca 100644 --- a/src/core/pdu/builder.rs +++ b/src/core/pdu/builder.rs @@ -1,11 +1,11 @@ use std::collections::BTreeMap; use ruma::{ - events::{EventContent, MessageLikeEventType, StateEventType, TimelineEventType}, MilliSecondsSinceUnixEpoch, OwnedEventId, + events::{EventContent, MessageLikeEventType, StateEventType, TimelineEventType}, }; use serde::Deserialize; -use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; use super::StateKey; diff --git a/src/core/pdu/content.rs b/src/core/pdu/content.rs index fa724cb2..4e60ce6e 100644 --- a/src/core/pdu/content.rs +++ b/src/core/pdu/content.rs @@ -1,7 +1,7 @@ use serde::Deserialize; use serde_json::value::Value as JsonValue; -use crate::{err, implement, Result}; +use crate::{Result, err, implement}; #[must_use] #[implement(super::Pdu)] diff --git a/src/core/pdu/count.rs b/src/core/pdu/count.rs index 0135cf28..b880278f 100644 --- a/src/core/pdu/count.rs +++ b/src/core/pdu/count.rs @@ -4,7 +4,7 @@ use std::{cmp::Ordering, fmt, fmt::Display, str::FromStr}; use ruma::api::Direction; -use crate::{err, Error, Result}; +use crate::{Error, Result, err}; #[derive(Hash, PartialEq, Eq, Clone, Copy, Debug)] pub enum Count { diff --git a/src/core/pdu/event.rs b/src/core/pdu/event.rs index d5c0561e..09ad1666 100644 --- a/src/core/pdu/event.rs +++ b/src/core/pdu/event.rs @@ -1,4 +1,4 @@ -use ruma::{events::TimelineEventType, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId}; +use ruma::{MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, events::TimelineEventType}; use serde_json::value::RawValue as RawJsonValue; use super::Pdu; diff --git a/src/core/pdu/event_id.rs b/src/core/pdu/event_id.rs index 09b33edc..e9d868b1 100644 --- a/src/core/pdu/event_id.rs +++ b/src/core/pdu/event_id.rs @@ -1,7 +1,7 @@ use ruma::{CanonicalJsonObject, OwnedEventId, RoomVersionId}; use serde_json::value::RawValue as RawJsonValue; -use crate::{err, Result}; +use crate::{Result, err}; /// Generates a correct eventId for the incoming pdu. /// diff --git a/src/core/pdu/mod.rs b/src/core/pdu/mod.rs index 9cb42239..9fb2a3da 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/pdu/mod.rs @@ -17,13 +17,14 @@ mod unsigned; use std::cmp::Ordering; use ruma::{ - events::TimelineEventType, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, - OwnedRoomId, OwnedServerName, OwnedUserId, UInt, + CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, + OwnedUserId, UInt, events::TimelineEventType, }; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue as RawJsonValue; pub use self::{ + Count as PduCount, Id as PduId, Pdu as PduEvent, RawId as RawPduId, builder::{Builder, Builder as PduBuilder}, count::Count, event::Event, @@ -31,7 +32,6 @@ pub use self::{ id::*, raw_id::*, state_key::{ShortStateKey, StateKey}, - Count as PduCount, Id as PduId, Pdu as PduEvent, RawId as RawPduId, }; use crate::Result; diff --git a/src/core/pdu/raw_id.rs b/src/core/pdu/raw_id.rs index e1fd2381..318a0cd7 100644 --- a/src/core/pdu/raw_id.rs +++ b/src/core/pdu/raw_id.rs @@ -55,8 +55,8 @@ impl RawId { #[must_use] pub fn as_bytes(&self) -> &[u8] { match self { - | Self::Normal(ref raw) => raw, - | Self::Backfilled(ref raw) => raw, + | Self::Normal(raw) => raw, + | Self::Backfilled(raw) => raw, } } } diff --git a/src/core/pdu/redact.rs b/src/core/pdu/redact.rs index 7c332719..409debfe 100644 --- a/src/core/pdu/redact.rs +++ b/src/core/pdu/redact.rs @@ -1,15 +1,15 @@ use ruma::{ - canonical_json::redact_content_in_place, - events::{room::redaction::RoomRedactionEventContent, TimelineEventType}, OwnedEventId, RoomVersionId, + canonical_json::redact_content_in_place, + events::{TimelineEventType, room::redaction::RoomRedactionEventContent}, }; use serde::Deserialize; use serde_json::{ json, - value::{to_raw_value, RawValue as RawJsonValue}, + value::{RawValue as RawJsonValue, to_raw_value}, }; -use crate::{implement, Error, Result}; +use crate::{Error, Result, implement}; #[derive(Deserialize)] struct ExtractRedactedBecause { @@ -76,14 +76,21 @@ pub fn copy_redacts(&self) -> (Option, Box) { if let Ok(mut content) = serde_json::from_str::(self.content.get()) { - if let Some(redacts) = content.redacts { - return (Some(redacts), self.content.clone()); - } else if let Some(redacts) = self.redacts.clone() { - content.redacts = Some(redacts); - return ( - self.redacts.clone(), - to_raw_value(&content).expect("Must be valid, we only added redacts field"), - ); + match content.redacts { + | Some(redacts) => { + return (Some(redacts), self.content.clone()); + }, + | _ => match self.redacts.clone() { + | Some(redacts) => { + content.redacts = Some(redacts); + return ( + self.redacts.clone(), + to_raw_value(&content) + .expect("Must be valid, we only added redacts field"), + ); + }, + | _ => {}, + }, } } } diff --git a/src/core/pdu/strip.rs b/src/core/pdu/strip.rs index 7d2fb1d6..4e7c5b83 100644 --- a/src/core/pdu/strip.rs +++ b/src/core/pdu/strip.rs @@ -1,8 +1,8 @@ use ruma::{ events::{ - room::member::RoomMemberEventContent, space::child::HierarchySpaceChildEvent, AnyEphemeralRoomEvent, AnyMessageLikeEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncStateEvent, AnySyncTimelineEvent, AnyTimelineEvent, StateEvent, + room::member::RoomMemberEventContent, space::child::HierarchySpaceChildEvent, }, serde::Raw, }; diff --git a/src/core/pdu/unsigned.rs b/src/core/pdu/unsigned.rs index 8482a48a..23897519 100644 --- a/src/core/pdu/unsigned.rs +++ b/src/core/pdu/unsigned.rs @@ -2,10 +2,10 @@ use std::collections::BTreeMap; use ruma::MilliSecondsSinceUnixEpoch; use serde::Deserialize; -use serde_json::value::{to_raw_value, RawValue as RawJsonValue, Value as JsonValue}; +use serde_json::value::{RawValue as RawJsonValue, Value as JsonValue, to_raw_value}; use super::Pdu; -use crate::{err, implement, is_true, Result}; +use crate::{Result, err, implement, is_true}; #[implement(Pdu)] pub fn remove_transaction_id(&mut self) -> Result { diff --git a/src/core/server.rs b/src/core/server.rs index 80493c94..b67759d6 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -1,7 +1,7 @@ use std::{ sync::{ - atomic::{AtomicBool, Ordering}, Arc, + atomic::{AtomicBool, Ordering}, }, time::SystemTime, }; @@ -9,7 +9,7 @@ use std::{ use ruma::OwnedServerName; use tokio::{runtime, sync::broadcast}; -use crate::{config, config::Config, log::Log, metrics::Metrics, Err, Result}; +use crate::{Err, Result, config, config::Config, log::Log, metrics::Metrics}; /// Server runtime state; public portion pub struct Server { diff --git a/src/core/state_res/event_auth.rs b/src/core/state_res/event_auth.rs index df2f8b36..4b8e55f3 100644 --- a/src/core/state_res/event_auth.rs +++ b/src/core/state_res/event_auth.rs @@ -1,10 +1,11 @@ use std::{borrow::Borrow, collections::BTreeSet}; use futures::{ - future::{join3, OptionFuture}, Future, + future::{OptionFuture, join3}, }; use ruma::{ + Int, OwnedUserId, RoomVersionId, UserId, events::room::{ create::RoomCreateEventContent, join_rules::{JoinRule, RoomJoinRulesEventContent}, @@ -14,21 +15,20 @@ use ruma::{ }, int, serde::{Base64, Raw}, - Int, OwnedUserId, RoomVersionId, UserId, }; use serde::{ - de::{Error as _, IgnoredAny}, Deserialize, + de::{Error as _, IgnoredAny}, }; use serde_json::{from_str as from_json_str, value::RawValue as RawJsonValue}; use super::{ + Error, Event, Result, StateEventType, StateKey, TimelineEventType, power_levels::{ deserialize_power_levels, deserialize_power_levels_content_fields, deserialize_power_levels_content_invite, deserialize_power_levels_content_redact, }, room_version::RoomVersion, - Error, Event, Result, StateEventType, StateKey, TimelineEventType, }; use crate::{debug, error, trace, warn}; @@ -394,28 +394,27 @@ where } // If type is m.room.third_party_invite - let sender_power_level = if let Some(pl) = &power_levels_event { - let content = deserialize_power_levels_content_fields(pl.content().get(), room_version)?; - if let Some(level) = content.get_user_power(sender) { - *level - } else { - content.users_default - } - } else { - // If no power level event found the creator gets 100 everyone else gets 0 - let is_creator = if room_version.use_room_create_sender { - room_create_event.sender() == sender - } else { - #[allow(deprecated)] - from_json_str::(room_create_event.content().get()) - .is_ok_and(|create| create.creator.unwrap() == *sender) - }; + let sender_power_level = match &power_levels_event { + | Some(pl) => { + let content = + deserialize_power_levels_content_fields(pl.content().get(), room_version)?; + match content.get_user_power(sender) { + | Some(level) => *level, + | _ => content.users_default, + } + }, + | _ => { + // If no power level event found the creator gets 100 everyone else gets 0 + let is_creator = if room_version.use_room_create_sender { + room_create_event.sender() == sender + } else { + #[allow(deprecated)] + from_json_str::(room_create_event.content().get()) + .is_ok_and(|create| create.creator.unwrap() == *sender) + }; - if is_creator { - int!(100) - } else { - int!(0) - } + if is_creator { int!(100) } else { int!(0) } + }, }; // Allow if and only if sender's current power level is greater than @@ -452,19 +451,21 @@ where if *incoming_event.event_type() == TimelineEventType::RoomPowerLevels { debug!("starting m.room.power_levels check"); - if let Some(required_pwr_lvl) = check_power_levels( + match check_power_levels( room_version, incoming_event, power_levels_event.as_ref(), sender_power_level, ) { - if !required_pwr_lvl { + | Some(required_pwr_lvl) => + if !required_pwr_lvl { + warn!("m.room.power_levels was not allowed"); + return Ok(false); + }, + | _ => { warn!("m.room.power_levels was not allowed"); return Ok(false); - } - } else { - warn!("m.room.power_levels was not allowed"); - return Ok(false); + }, } debug!("m.room.power_levels event allowed"); } @@ -576,10 +577,9 @@ fn valid_membership_change( let content = deserialize_power_levels_content_fields(pl.content().get(), room_version)?; - let user_pl = if let Some(level) = content.get_user_power(user_for_join_auth) { - *level - } else { - content.users_default + let user_pl = match content.get_user_power(user_for_join_auth) { + | Some(level) => *level, + | _ => content.users_default, }; (user_pl, invite) @@ -665,45 +665,48 @@ fn valid_membership_change( }, | MembershipState::Invite => { // If content has third_party_invite key - if let Some(tp_id) = third_party_invite.and_then(|i| i.deserialize().ok()) { - if target_user_current_membership == MembershipState::Ban { - warn!(?target_user_membership_event_id, "Can't invite banned user"); - false - } else { - let allow = verify_third_party_invite( - Some(target_user), - sender, - &tp_id, - current_third_party_invite, - ); - if !allow { - warn!("Third party invite invalid"); - } - allow - } - } else if !sender_is_joined - || target_user_current_membership == MembershipState::Join - || target_user_current_membership == MembershipState::Ban - { - warn!( - ?target_user_membership_event_id, - ?sender_membership_event_id, - "Can't invite user if sender not joined or the user is currently joined or \ - banned", - ); - false - } else { - let allow = sender_power - .filter(|&p| p >= &power_levels.invite) - .is_some(); - if !allow { - warn!( - ?target_user_membership_event_id, - ?power_levels_event_id, - "User does not have enough power to invite", - ); - } - allow + match third_party_invite.and_then(|i| i.deserialize().ok()) { + | Some(tp_id) => + if target_user_current_membership == MembershipState::Ban { + warn!(?target_user_membership_event_id, "Can't invite banned user"); + false + } else { + let allow = verify_third_party_invite( + Some(target_user), + sender, + &tp_id, + current_third_party_invite, + ); + if !allow { + warn!("Third party invite invalid"); + } + allow + }, + | _ => + if !sender_is_joined + || target_user_current_membership == MembershipState::Join + || target_user_current_membership == MembershipState::Ban + { + warn!( + ?target_user_membership_event_id, + ?sender_membership_event_id, + "Can't invite user if sender not joined or the user is currently \ + joined or banned", + ); + false + } else { + let allow = sender_power + .filter(|&p| p >= &power_levels.invite) + .is_some(); + if !allow { + warn!( + ?target_user_membership_event_id, + ?power_levels_event_id, + "User does not have enough power to invite", + ); + } + allow + }, } }, | MembershipState::Leave => @@ -1111,23 +1114,23 @@ mod tests { use std::sync::Arc; use ruma::events::{ + StateEventType, TimelineEventType, room::{ join_rules::{ AllowRule, JoinRule, Restricted, RoomJoinRulesEventContent, RoomMembership, }, member::{MembershipState, RoomMemberEventContent}, }, - StateEventType, TimelineEventType, }; use serde_json::value::to_raw_value as to_raw_json_value; use crate::state_res::{ + Event, EventTypeExt, RoomVersion, StateMap, event_auth::valid_membership_change, test_utils::{ - alice, charlie, ella, event_id, member_content_ban, member_content_join, room_id, - to_pdu_event, PduEvent, INITIAL_EVENTS, INITIAL_EVENTS_CREATE_ROOM, + INITIAL_EVENTS, INITIAL_EVENTS_CREATE_ROOM, PduEvent, alice, charlie, ella, event_id, + member_content_ban, member_content_join, room_id, to_pdu_event, }, - Event, EventTypeExt, RoomVersion, StateMap, }; #[test] @@ -1156,21 +1159,23 @@ mod tests { let target_user = charlie(); let sender = alice(); - assert!(valid_membership_change( - &RoomVersion::V6, - target_user, - fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), - sender, - fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), - &requester, - None::<&PduEvent>, - fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), - fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), - None, - &MembershipState::Leave, - &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), - ) - .unwrap()); + assert!( + valid_membership_change( + &RoomVersion::V6, + target_user, + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), + sender, + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), + &requester, + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), + None, + &MembershipState::Leave, + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), + ) + .unwrap() + ); } #[test] @@ -1199,21 +1204,23 @@ mod tests { let target_user = charlie(); let sender = charlie(); - assert!(!valid_membership_change( - &RoomVersion::V6, - target_user, - fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), - sender, - fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), - &requester, - None::<&PduEvent>, - fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), - fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), - None, - &MembershipState::Leave, - &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), - ) - .unwrap()); + assert!( + !valid_membership_change( + &RoomVersion::V6, + target_user, + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), + sender, + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), + &requester, + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), + None, + &MembershipState::Leave, + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), + ) + .unwrap() + ); } #[test] @@ -1242,21 +1249,23 @@ mod tests { let target_user = alice(); let sender = alice(); - assert!(valid_membership_change( - &RoomVersion::V6, - target_user, - fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), - sender, - fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), - &requester, - None::<&PduEvent>, - fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), - fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), - None, - &MembershipState::Leave, - &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), - ) - .unwrap()); + assert!( + valid_membership_change( + &RoomVersion::V6, + target_user, + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), + sender, + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), + &requester, + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), + None, + &MembershipState::Leave, + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), + ) + .unwrap() + ); } #[test] @@ -1285,21 +1294,23 @@ mod tests { let target_user = alice(); let sender = charlie(); - assert!(!valid_membership_change( - &RoomVersion::V6, - target_user, - fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), - sender, - fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), - &requester, - None::<&PduEvent>, - fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), - fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), - None, - &MembershipState::Leave, - &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), - ) - .unwrap()); + assert!( + !valid_membership_change( + &RoomVersion::V6, + target_user, + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), + sender, + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), + &requester, + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), + None, + &MembershipState::Leave, + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), + ) + .unwrap() + ); } #[test] @@ -1345,37 +1356,41 @@ mod tests { let target_user = ella(); let sender = ella(); - assert!(valid_membership_change( - &RoomVersion::V9, - target_user, - fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), - sender, - fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), - &requester, - None::<&PduEvent>, - fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), - fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), - Some(alice()), - &MembershipState::Join, - &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), - ) - .unwrap()); + assert!( + valid_membership_change( + &RoomVersion::V9, + target_user, + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), + sender, + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), + &requester, + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), + Some(alice()), + &MembershipState::Join, + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), + ) + .unwrap() + ); - assert!(!valid_membership_change( - &RoomVersion::V9, - target_user, - fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), - sender, - fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), - &requester, - None::<&PduEvent>, - fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), - fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), - Some(ella()), - &MembershipState::Leave, - &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), - ) - .unwrap()); + assert!( + !valid_membership_change( + &RoomVersion::V9, + target_user, + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), + sender, + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), + &requester, + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), + Some(ella()), + &MembershipState::Leave, + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), + ) + .unwrap() + ); } #[test] @@ -1413,20 +1428,22 @@ mod tests { let target_user = ella(); let sender = ella(); - assert!(valid_membership_change( - &RoomVersion::V7, - target_user, - fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), - sender, - fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), - &requester, - None::<&PduEvent>, - fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), - fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), - None, - &MembershipState::Leave, - &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), - ) - .unwrap()); + assert!( + valid_membership_change( + &RoomVersion::V7, + target_user, + fetch_state(StateEventType::RoomMember, target_user.as_str().into()).as_ref(), + sender, + fetch_state(StateEventType::RoomMember, sender.as_str().into()).as_ref(), + &requester, + None::<&PduEvent>, + fetch_state(StateEventType::RoomPowerLevels, "".into()).as_ref(), + fetch_state(StateEventType::RoomJoinRules, "".into()).as_ref(), + None, + &MembershipState::Leave, + &fetch_state(StateEventType::RoomCreate, "".into()).unwrap(), + ) + .unwrap() + ); } } diff --git a/src/core/state_res/mod.rs b/src/core/state_res/mod.rs index 19ea3cc0..6bff0cf8 100644 --- a/src/core/state_res/mod.rs +++ b/src/core/state_res/mod.rs @@ -17,13 +17,14 @@ use std::{ hash::{BuildHasher, Hash}, }; -use futures::{future, stream, Future, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{Future, FutureExt, StreamExt, TryFutureExt, TryStreamExt, future, stream}; use ruma::{ + EventId, Int, MilliSecondsSinceUnixEpoch, RoomVersionId, events::{ - room::member::{MembershipState, RoomMemberEventContent}, StateEventType, TimelineEventType, + room::member::{MembershipState, RoomMemberEventContent}, }, - int, EventId, Int, MilliSecondsSinceUnixEpoch, RoomVersionId, + int, }; use serde_json::from_str as from_json_str; @@ -263,7 +264,7 @@ where #[allow(clippy::arithmetic_side_effects)] fn get_auth_chain_diff( auth_chain_sets: &[HashSet], -) -> impl Iterator + Send +) -> impl Iterator + Send + use where Id: Clone + Eq + Hash + Send, Hasher: BuildHasher + Send + Sync, @@ -864,23 +865,23 @@ mod tests { use maplit::{hashmap, hashset}; use rand::seq::SliceRandom; use ruma::{ + MilliSecondsSinceUnixEpoch, OwnedEventId, RoomVersionId, events::{ - room::join_rules::{JoinRule, RoomJoinRulesEventContent}, StateEventType, TimelineEventType, + room::join_rules::{JoinRule, RoomJoinRulesEventContent}, }, - int, uint, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomVersionId, + int, uint, }; use serde_json::{json, value::to_raw_value as to_raw_json_value}; use super::{ - is_power_event, + Event, EventTypeExt, StateMap, is_power_event, room_version::RoomVersion, test_utils::{ - alice, bob, charlie, do_check, ella, event_id, member_content_ban, - member_content_join, room_id, to_init_pdu_event, to_pdu_event, zara, PduEvent, - TestStore, INITIAL_EVENTS, + INITIAL_EVENTS, PduEvent, TestStore, alice, bob, charlie, do_check, ella, event_id, + member_content_ban, member_content_join, room_id, to_init_pdu_event, to_pdu_event, + zara, }, - Event, EventTypeExt, StateMap, }; use crate::debug; @@ -1557,7 +1558,7 @@ mod tests { } macro_rules! state_set { - ($($kind:expr => $key:expr => $id:expr),* $(,)?) => {{ + ($($kind:expr_2021 => $key:expr_2021 => $id:expr_2021),* $(,)?) => {{ #[allow(unused_mut)] let mut x = StateMap::new(); $( diff --git a/src/core/state_res/power_levels.rs b/src/core/state_res/power_levels.rs index e1768574..045b1666 100644 --- a/src/core/state_res/power_levels.rs +++ b/src/core/state_res/power_levels.rs @@ -1,16 +1,16 @@ use std::collections::BTreeMap; use ruma::{ - events::{room::power_levels::RoomPowerLevelsEventContent, TimelineEventType}, - power_levels::{default_power_level, NotificationPowerLevels}, + Int, OwnedUserId, UserId, + events::{TimelineEventType, room::power_levels::RoomPowerLevelsEventContent}, + power_levels::{NotificationPowerLevels, default_power_level}, serde::{ deserialize_v1_powerlevel, vec_deserialize_int_powerlevel_values, vec_deserialize_v1_powerlevel_values, }, - Int, OwnedUserId, UserId, }; use serde::Deserialize; -use serde_json::{from_str as from_json_str, Error}; +use serde_json::{Error, from_str as from_json_str}; use tracing::error; use super::{Result, RoomVersion}; diff --git a/src/core/state_res/state_event.rs b/src/core/state_res/state_event.rs index 2c038cfe..ac9e29d6 100644 --- a/src/core/state_res/state_event.rs +++ b/src/core/state_res/state_event.rs @@ -5,7 +5,7 @@ use std::{ sync::Arc, }; -use ruma::{events::TimelineEventType, EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId}; +use ruma::{EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId, events::TimelineEventType}; use serde_json::value::RawValue as RawJsonValue; /// Abstraction of a PDU so users can have their own PDU types. diff --git a/src/core/state_res/test_utils.rs b/src/core/state_res/test_utils.rs index 9c2b151f..d96ee927 100644 --- a/src/core/state_res/test_utils.rs +++ b/src/core/state_res/test_utils.rs @@ -2,33 +2,33 @@ use std::{ borrow::Borrow, collections::{BTreeMap, HashMap, HashSet}, sync::{ - atomic::{AtomicU64, Ordering::SeqCst}, Arc, + atomic::{AtomicU64, Ordering::SeqCst}, }, }; use futures::future::ready; use ruma::{ - event_id, + EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, ServerSignatures, + UserId, event_id, events::{ + TimelineEventType, pdu::{EventHash, Pdu, RoomV3Pdu}, room::{ join_rules::{JoinRule, RoomJoinRulesEventContent}, member::{MembershipState, RoomMemberEventContent}, }, - TimelineEventType, }, - int, room_id, uint, user_id, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, - RoomVersionId, ServerSignatures, UserId, + int, room_id, uint, user_id, }; use serde_json::{ json, - value::{to_raw_value as to_raw_json_value, RawValue as RawJsonValue}, + value::{RawValue as RawJsonValue, to_raw_value as to_raw_json_value}, }; pub(crate) use self::event::PduEvent; use super::auth_types_for_event; -use crate::{info, Event, EventTypeExt, Result, StateMap}; +use crate::{Event, EventTypeExt, Result, StateMap, info}; static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); @@ -584,8 +584,8 @@ pub(crate) fn INITIAL_EDGES() -> Vec { pub(crate) mod event { use ruma::{ - events::{pdu::Pdu, TimelineEventType}, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, + events::{TimelineEventType, pdu::Pdu}, }; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue as RawJsonValue; diff --git a/src/core/utils/bytes.rs b/src/core/utils/bytes.rs index 40316440..04101be4 100644 --- a/src/core/utils/bytes.rs +++ b/src/core/utils/bytes.rs @@ -1,6 +1,6 @@ use bytesize::ByteSize; -use crate::{err, Result}; +use crate::{Result, err}; /// Parse a human-writable size string w/ si-unit suffix into integer #[inline] diff --git a/src/core/utils/defer.rs b/src/core/utils/defer.rs index 60243e97..4887d164 100644 --- a/src/core/utils/defer.rs +++ b/src/core/utils/defer.rs @@ -12,14 +12,14 @@ macro_rules! defer { let _defer_ = _Defer_ { closure: || $body }; }; - ($body:expr) => { + ($body:expr_2021) => { $crate::defer! {{ $body }} }; } #[macro_export] macro_rules! scope_restore { - ($val:ident, $ours:expr) => { + ($val:ident, $ours:expr_2021) => { let theirs = $crate::utils::exchange($val, $ours); $crate::defer! {{ *$val = theirs; }}; }; diff --git a/src/core/utils/future/bool_ext.rs b/src/core/utils/future/bool_ext.rs index 6cb2f1fe..c93c7dbc 100644 --- a/src/core/utils/future/bool_ext.rs +++ b/src/core/utils/future/bool_ext.rs @@ -3,8 +3,8 @@ use std::marker::Unpin; use futures::{ - future::{select_ok, try_join, try_join_all, try_select}, Future, FutureExt, + future::{select_ok, try_join, try_join_all, try_select}, }; pub trait BoolExt diff --git a/src/core/utils/future/ext_ext.rs b/src/core/utils/future/ext_ext.rs index 38decaae..219bb664 100644 --- a/src/core/utils/future/ext_ext.rs +++ b/src/core/utils/future/ext_ext.rs @@ -2,7 +2,7 @@ use std::marker::Unpin; -use futures::{future, future::Select, Future}; +use futures::{Future, future, future::Select}; /// This interface is not necessarily complete; feel free to add as-needed. pub trait ExtExt diff --git a/src/core/utils/future/mod.rs b/src/core/utils/future/mod.rs index 2198a84f..e1d96941 100644 --- a/src/core/utils/future/mod.rs +++ b/src/core/utils/future/mod.rs @@ -3,7 +3,7 @@ mod ext_ext; mod option_ext; mod try_ext_ext; -pub use bool_ext::{and, or, BoolExt}; +pub use bool_ext::{BoolExt, and, or}; pub use ext_ext::ExtExt; pub use option_ext::OptionExt; pub use try_ext_ext::TryExtExt; diff --git a/src/core/utils/future/option_ext.rs b/src/core/utils/future/option_ext.rs index ed61de56..d553e5dc 100644 --- a/src/core/utils/future/option_ext.rs +++ b/src/core/utils/future/option_ext.rs @@ -1,6 +1,6 @@ #![allow(clippy::wrong_self_convention)] -use futures::{future::OptionFuture, Future, FutureExt}; +use futures::{Future, FutureExt, future::OptionFuture}; pub trait OptionExt { fn is_none_or(self, f: impl FnOnce(&T) -> bool + Send) -> impl Future + Send; diff --git a/src/core/utils/future/try_ext_ext.rs b/src/core/utils/future/try_ext_ext.rs index aa3d72e4..b2114e56 100644 --- a/src/core/utils/future/try_ext_ext.rs +++ b/src/core/utils/future/try_ext_ext.rs @@ -7,9 +7,8 @@ use std::marker::Unpin; use futures::{ - future, + TryFuture, TryFutureExt, future, future::{MapOkOrElse, TrySelect, UnwrapOrElse}, - TryFuture, TryFutureExt, }; /// This interface is not necessarily complete; feel free to add as-needed. diff --git a/src/core/utils/hash/argon.rs b/src/core/utils/hash/argon.rs index 18146b47..66dfab75 100644 --- a/src/core/utils/hash/argon.rs +++ b/src/core/utils/hash/argon.rs @@ -1,11 +1,11 @@ use std::sync::OnceLock; use argon2::{ - password_hash, password_hash::SaltString, Algorithm, Argon2, Params, PasswordHash, - PasswordHasher, PasswordVerifier, Version, + Algorithm, Argon2, Params, PasswordHash, PasswordHasher, PasswordVerifier, Version, + password_hash, password_hash::SaltString, }; -use crate::{err, Error, Result}; +use crate::{Error, Result, err}; const M_COST: u32 = Params::DEFAULT_M_COST; // memory size in 1 KiB blocks const T_COST: u32 = Params::DEFAULT_T_COST; // nr of iterations diff --git a/src/core/utils/json.rs b/src/core/utils/json.rs index 4a3fec8f..3f2f225e 100644 --- a/src/core/utils/json.rs +++ b/src/core/utils/json.rs @@ -1,6 +1,6 @@ use std::{fmt, str::FromStr}; -use ruma::{canonical_json::try_from_json_map, CanonicalJsonError, CanonicalJsonObject}; +use ruma::{CanonicalJsonError, CanonicalJsonObject, canonical_json::try_from_json_map}; use crate::Result; diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index ed157daf..488f2a13 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -6,7 +6,7 @@ use std::{cmp, convert::TryFrom}; pub use checked_ops::checked_ops; pub use self::{expected::Expected, tried::Tried}; -use crate::{debug::type_name, err, Err, Error, Result}; +use crate::{Err, Error, Result, debug::type_name, err}; /// Checked arithmetic expression. Returns a Result #[macro_export] diff --git a/src/core/utils/math/tried.rs b/src/core/utils/math/tried.rs index 2006d2d5..09de731f 100644 --- a/src/core/utils/math/tried.rs +++ b/src/core/utils/math/tried.rs @@ -1,6 +1,6 @@ use num_traits::ops::checked::{CheckedAdd, CheckedDiv, CheckedMul, CheckedRem, CheckedSub}; -use crate::{checked, Result}; +use crate::{Result, checked}; pub trait Tried { #[inline] diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index c2d8ed45..53460c59 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -49,7 +49,7 @@ pub fn exchange(state: &mut T, source: T) -> T { std::mem::replace(state, sou #[macro_export] macro_rules! extract_variant { - ($e:expr, $variant:path) => { + ($e:expr_2021, $variant:path) => { match $e { | $variant(value) => Some(value), | _ => None, @@ -90,7 +90,7 @@ macro_rules! pair_of { ($decl, $decl) }; - ($init:expr) => { + ($init:expr_2021) => { ($init, $init) }; } @@ -134,7 +134,7 @@ macro_rules! is_equal_to { |x| x == $val }; - ($val:expr) => { + ($val:expr_2021) => { |x| x == $val }; } @@ -146,7 +146,7 @@ macro_rules! is_less_than { |x| x < $val }; - ($val:expr) => { + ($val:expr_2021) => { |x| x < $val }; } diff --git a/src/core/utils/mutex_map.rs b/src/core/utils/mutex_map.rs index 03a4adf1..01504ce6 100644 --- a/src/core/utils/mutex_map.rs +++ b/src/core/utils/mutex_map.rs @@ -6,7 +6,7 @@ use std::{ use tokio::sync::OwnedMutexGuard as Omg; -use crate::{err, Result}; +use crate::{Result, err}; /// Map of Mutexes pub struct MutexMap { diff --git a/src/core/utils/rand.rs b/src/core/utils/rand.rs index 1d289c6e..72487633 100644 --- a/src/core/utils/rand.rs +++ b/src/core/utils/rand.rs @@ -4,7 +4,7 @@ use std::{ }; use arrayvec::ArrayString; -use rand::{seq::SliceRandom, thread_rng, Rng}; +use rand::{Rng, seq::SliceRandom, thread_rng}; pub fn shuffle(vec: &mut [T]) { let mut rng = thread_rng(); diff --git a/src/core/utils/stream/broadband.rs b/src/core/utils/stream/broadband.rs index 282008e7..832f2638 100644 --- a/src/core/utils/stream/broadband.rs +++ b/src/core/utils/stream/broadband.rs @@ -3,11 +3,11 @@ use std::convert::identity; use futures::{ - stream::{Stream, StreamExt}, Future, + stream::{Stream, StreamExt}, }; -use super::{automatic_width, ReadyExt}; +use super::{ReadyExt, automatic_width}; /// Concurrency extensions to augment futures::StreamExt. broad_ combinators /// produce out-of-order diff --git a/src/core/utils/stream/cloned.rs b/src/core/utils/stream/cloned.rs index d6a0e647..b89e4695 100644 --- a/src/core/utils/stream/cloned.rs +++ b/src/core/utils/stream/cloned.rs @@ -1,6 +1,6 @@ use std::clone::Clone; -use futures::{stream::Map, Stream, StreamExt}; +use futures::{Stream, StreamExt, stream::Map}; pub trait Cloned<'a, T, S> where diff --git a/src/core/utils/stream/ignore.rs b/src/core/utils/stream/ignore.rs index 9baa00f3..37c89d9a 100644 --- a/src/core/utils/stream/ignore.rs +++ b/src/core/utils/stream/ignore.rs @@ -1,4 +1,4 @@ -use futures::{future::ready, Stream, StreamExt, TryStream}; +use futures::{Stream, StreamExt, TryStream, future::ready}; use crate::{Error, Result}; diff --git a/src/core/utils/stream/iter_stream.rs b/src/core/utils/stream/iter_stream.rs index 9077deac..e9a91b1c 100644 --- a/src/core/utils/stream/iter_stream.rs +++ b/src/core/utils/stream/iter_stream.rs @@ -1,7 +1,6 @@ use futures::{ - stream, + StreamExt, stream, stream::{Stream, TryStream}, - StreamExt, }; use crate::{Error, Result}; diff --git a/src/core/utils/stream/mod.rs b/src/core/utils/stream/mod.rs index 23455322..a356f05f 100644 --- a/src/core/utils/stream/mod.rs +++ b/src/core/utils/stream/mod.rs @@ -14,8 +14,8 @@ mod try_wideband; mod wideband; pub use band::{ - automatic_amplification, automatic_width, set_amplification, set_width, AMPLIFICATION_LIMIT, - WIDTH_LIMIT, + AMPLIFICATION_LIMIT, WIDTH_LIMIT, automatic_amplification, automatic_width, + set_amplification, set_width, }; pub use broadband::BroadbandExt; pub use cloned::Cloned; diff --git a/src/core/utils/stream/ready.rs b/src/core/utils/stream/ready.rs index d93187e9..dce7d378 100644 --- a/src/core/utils/stream/ready.rs +++ b/src/core/utils/stream/ready.rs @@ -2,7 +2,7 @@ #![allow(clippy::type_complexity)] use futures::{ - future::{ready, Ready}, + future::{Ready, ready}, stream::{ All, Any, Filter, FilterMap, Fold, ForEach, Scan, SkipWhile, Stream, StreamExt, TakeWhile, }, diff --git a/src/core/utils/stream/try_parallel.rs b/src/core/utils/stream/try_parallel.rs index 7f8a63b1..60fef0ae 100644 --- a/src/core/utils/stream/try_parallel.rs +++ b/src/core/utils/stream/try_parallel.rs @@ -1,10 +1,10 @@ //! Parallelism stream combinator extensions to futures::Stream -use futures::{stream::TryStream, TryFutureExt}; +use futures::{TryFutureExt, stream::TryStream}; use tokio::{runtime, task::JoinError}; use super::TryBroadbandExt; -use crate::{utils::sys::available_parallelism, Error, Result}; +use crate::{Error, Result, utils::sys::available_parallelism}; /// Parallelism extensions to augment futures::StreamExt. These combinators are /// for computation-oriented workloads, unlike -band combinators for I/O diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs index 3261acb6..611c177f 100644 --- a/src/core/utils/stream/try_ready.rs +++ b/src/core/utils/stream/try_ready.rs @@ -2,7 +2,7 @@ #![allow(clippy::type_complexity)] use futures::{ - future::{ready, Ready}, + future::{Ready, ready}, stream::{AndThen, TryFilterMap, TryFold, TryForEach, TryStream, TryStreamExt, TryTakeWhile}, }; diff --git a/src/core/utils/stream/try_tools.rs b/src/core/utils/stream/try_tools.rs index 3ddce6ad..ea3b50fc 100644 --- a/src/core/utils/stream/try_tools.rs +++ b/src/core/utils/stream/try_tools.rs @@ -1,7 +1,7 @@ //! TryStreamTools for futures::TryStream #![allow(clippy::type_complexity)] -use futures::{future, future::Ready, stream::TryTakeWhile, TryStream, TryStreamExt}; +use futures::{TryStream, TryStreamExt, future, future::Ready, stream::TryTakeWhile}; use crate::Result; diff --git a/src/core/utils/stream/wideband.rs b/src/core/utils/stream/wideband.rs index a8560bb4..cbebf610 100644 --- a/src/core/utils/stream/wideband.rs +++ b/src/core/utils/stream/wideband.rs @@ -3,11 +3,11 @@ use std::convert::identity; use futures::{ - stream::{Stream, StreamExt}, Future, + stream::{Stream, StreamExt}, }; -use super::{automatic_width, ReadyExt}; +use super::{ReadyExt, automatic_width}; /// Concurrency extensions to augment futures::StreamExt. wideband_ combinators /// produce in-order. diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index cc692c14..9340d009 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -5,7 +5,7 @@ mod unquote; mod unquoted; pub use self::{between::Between, split::SplitInfallible, unquote::Unquote, unquoted::Unquoted}; -use crate::{utils::exchange, Result}; +use crate::{Result, utils::exchange}; pub const EMPTY: &str = ""; diff --git a/src/core/utils/string/unquoted.rs b/src/core/utils/string/unquoted.rs index 5b002d99..88fa011f 100644 --- a/src/core/utils/string/unquoted.rs +++ b/src/core/utils/string/unquoted.rs @@ -1,9 +1,9 @@ use std::ops::Deref; -use serde::{de, Deserialize, Deserializer}; +use serde::{Deserialize, Deserializer, de}; use super::Unquote; -use crate::{err, Result}; +use crate::{Result, err}; /// Unquoted string which deserialized from a quoted string. Construction from a /// &str is infallible such that the input can already be unquoted. Construction diff --git a/src/core/utils/sys.rs b/src/core/utils/sys.rs index a0d5be52..f795ccb8 100644 --- a/src/core/utils/sys.rs +++ b/src/core/utils/sys.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; pub use compute::available_parallelism; -use crate::{debug, Result}; +use crate::{Result, debug}; /// This is needed for opening lots of file descriptors, which tends to /// happen more often when using RocksDB and making lots of federation @@ -16,7 +16,7 @@ use crate::{debug, Result}; /// * #[cfg(unix)] pub fn maximize_fd_limit() -> Result<(), nix::errno::Errno> { - use nix::sys::resource::{getrlimit, setrlimit, Resource::RLIMIT_NOFILE as NOFILE}; + use nix::sys::resource::{Resource::RLIMIT_NOFILE as NOFILE, getrlimit, setrlimit}; let (soft_limit, hard_limit) = getrlimit(NOFILE)?; if soft_limit < hard_limit { diff --git a/src/core/utils/sys/compute.rs b/src/core/utils/sys/compute.rs index ce2aa504..5274cd66 100644 --- a/src/core/utils/sys/compute.rs +++ b/src/core/utils/sys/compute.rs @@ -2,7 +2,7 @@ use std::{cell::Cell, fmt::Debug, path::PathBuf, sync::LazyLock}; -use crate::{is_equal_to, Result}; +use crate::{Result, is_equal_to}; type Id = usize; @@ -45,7 +45,7 @@ pub fn set_affinity(mut ids: I) where I: Iterator + Clone + Debug, { - use core_affinity::{set_each_for_current, set_for_current, CoreId}; + use core_affinity::{CoreId, set_each_for_current, set_for_current}; let n = ids.clone().count(); let mask: Mask = ids.clone().fold(0, |mask, id| { @@ -118,7 +118,7 @@ pub fn cores_available() -> impl Iterator { from_mask(*CORES_AVAILABL #[cfg(target_os = "linux")] #[inline] pub fn getcpu() -> Result { - use crate::{utils::math, Error}; + use crate::{Error, utils::math}; // SAFETY: This is part of an interface with many low-level calls taking many // raw params, but it's unclear why this specific call is unsafe. Nevertheless diff --git a/src/core/utils/sys/storage.rs b/src/core/utils/sys/storage.rs index 25b17904..b11df7bb 100644 --- a/src/core/utils/sys/storage.rs +++ b/src/core/utils/sys/storage.rs @@ -3,7 +3,7 @@ use std::{ ffi::OsStr, fs, - fs::{read_to_string, FileType}, + fs::{FileType, read_to_string}, iter::IntoIterator, path::{Path, PathBuf}, }; @@ -11,9 +11,9 @@ use std::{ use libc::dev_t; use crate::{ + Result, result::FlatOk, utils::{result::LogDebugErr, string::SplitInfallible}, - Result, }; /// Device characteristics useful for random access throughput diff --git a/src/core/utils/tests.rs b/src/core/utils/tests.rs index 1bcb92b8..05a0655b 100644 --- a/src/core/utils/tests.rs +++ b/src/core/utils/tests.rs @@ -241,7 +241,7 @@ fn set_intersection_sorted_all() { #[tokio::test] async fn set_intersection_sorted_stream2() { use futures::StreamExt; - use utils::{set::intersection_sorted_stream2, IterStream}; + use utils::{IterStream, set::intersection_sorted_stream2}; let a = ["bar"]; let b = ["bar", "foo"]; diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 81fdda2a..73f73971 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -2,7 +2,7 @@ pub mod exponential_backoff; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use crate::{err, Result}; +use crate::{Result, err}; #[inline] #[must_use] diff --git a/src/database/de.rs b/src/database/de.rs index 441bb4ec..9c0997ff 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -1,10 +1,9 @@ use conduwuit::{ - arrayvec::ArrayVec, checked, debug::DebugInspect, err, utils::string, Error, Result, + Error, Result, arrayvec::ArrayVec, checked, debug::DebugInspect, err, utils::string, }; use serde::{ - de, + Deserialize, de, de::{DeserializeSeed, Visitor}, - Deserialize, }; use crate::util::unhandled; diff --git a/src/database/engine.rs b/src/database/engine.rs index 22e2b9c8..38dd7512 100644 --- a/src/database/engine.rs +++ b/src/database/engine.rs @@ -12,21 +12,21 @@ mod repair; use std::{ ffi::CStr, sync::{ - atomic::{AtomicU32, Ordering}, Arc, + atomic::{AtomicU32, Ordering}, }, }; -use conduwuit::{debug, info, warn, Err, Result}; +use conduwuit::{Err, Result, debug, info, warn}; use rocksdb::{ AsColumnFamilyRef, BoundColumnFamily, DBCommon, DBWithThreadMode, MultiThreaded, WaitForCompactOptions, }; use crate::{ + Context, pool::Pool, util::{map_err, result}, - Context, }; pub struct Engine { diff --git a/src/database/engine/backup.rs b/src/database/engine/backup.rs index db718c2c..bb110630 100644 --- a/src/database/engine/backup.rs +++ b/src/database/engine/backup.rs @@ -1,6 +1,6 @@ use std::fmt::Write; -use conduwuit::{error, implement, info, utils::time::rfc2822_from_seconds, warn, Result}; +use conduwuit::{Result, error, implement, info, utils::time::rfc2822_from_seconds, warn}; use rocksdb::backup::{BackupEngine, BackupEngineOptions}; use super::Engine; diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 83bce08c..5ddb9473 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -1,4 +1,4 @@ -use conduwuit::{err, utils::math::Expected, Config, Result}; +use conduwuit::{Config, Result, err, utils::math::Expected}; use rocksdb::{ BlockBasedIndexType, BlockBasedOptions, BlockBasedPinningTier, Cache, DBCompressionType as CompressionType, DataBlockIndexType, LruCacheOptions, Options, @@ -6,7 +6,7 @@ use rocksdb::{ }; use super::descriptor::{CacheDisp, Descriptor}; -use crate::{util::map_err, Context}; +use crate::{Context, util::map_err}; pub(super) const SENTINEL_COMPRESSION_LEVEL: i32 = 32767; diff --git a/src/database/engine/context.rs b/src/database/engine/context.rs index 04e08854..380e37af 100644 --- a/src/database/engine/context.rs +++ b/src/database/engine/context.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduwuit::{debug, utils::math::usize_from_f64, Result, Server}; +use conduwuit::{Result, Server, debug, utils::math::usize_from_f64}; use rocksdb::{Cache, Env, LruCacheOptions}; use crate::{or_else, pool::Pool}; diff --git a/src/database/engine/db_opts.rs b/src/database/engine/db_opts.rs index 6abeb4b0..18cec742 100644 --- a/src/database/engine/db_opts.rs +++ b/src/database/engine/db_opts.rs @@ -1,7 +1,7 @@ use std::{cmp, convert::TryFrom}; -use conduwuit::{utils, Config, Result}; -use rocksdb::{statistics::StatsLevel, Cache, DBRecoveryMode, Env, LogLevel, Options}; +use conduwuit::{Config, Result, utils}; +use rocksdb::{Cache, DBRecoveryMode, Env, LogLevel, Options, statistics::StatsLevel}; use super::{cf_opts::cache_size_f64, logger::handle as handle_log}; diff --git a/src/database/engine/files.rs b/src/database/engine/files.rs index 33d6fdc4..1f38a63c 100644 --- a/src/database/engine/files.rs +++ b/src/database/engine/files.rs @@ -1,11 +1,11 @@ -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use rocksdb::LiveFile as SstFile; use super::Engine; use crate::util::map_err; #[implement(Engine)] -pub fn file_list(&self) -> impl Iterator> + Send { +pub fn file_list(&self) -> impl Iterator> + Send + use<> { self.db .live_files() .map_err(map_err) diff --git a/src/database/engine/memory_usage.rs b/src/database/engine/memory_usage.rs index 01859815..9bb5c535 100644 --- a/src/database/engine/memory_usage.rs +++ b/src/database/engine/memory_usage.rs @@ -1,6 +1,6 @@ use std::fmt::Write; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use rocksdb::perf::get_memory_usage_stats; use super::Engine; diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs index 59dabce1..24010c3a 100644 --- a/src/database/engine/open.rs +++ b/src/database/engine/open.rs @@ -1,20 +1,20 @@ use std::{ collections::BTreeSet, path::Path, - sync::{atomic::AtomicU32, Arc}, + sync::{Arc, atomic::AtomicU32}, }; -use conduwuit::{debug, implement, info, warn, Result}; +use conduwuit::{Result, debug, implement, info, warn}; use rocksdb::{ColumnFamilyDescriptor, Options}; use super::{ + Db, Engine, cf_opts::cf_options, db_opts::db_options, descriptor::{self, Descriptor}, repair::repair, - Db, Engine, }; -use crate::{or_else, Context}; +use crate::{Context, or_else}; #[implement(Engine)] #[tracing::instrument(skip_all)] diff --git a/src/database/engine/repair.rs b/src/database/engine/repair.rs index 61283904..aeec0caf 100644 --- a/src/database/engine/repair.rs +++ b/src/database/engine/repair.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use conduwuit::{info, warn, Err, Result}; +use conduwuit::{Err, Result, info, warn}; use rocksdb::Options; use super::Db; diff --git a/src/database/handle.rs b/src/database/handle.rs index 43b57839..484e5618 100644 --- a/src/database/handle.rs +++ b/src/database/handle.rs @@ -4,7 +4,7 @@ use conduwuit::Result; use rocksdb::DBPinnableSlice; use serde::{Deserialize, Serialize, Serializer}; -use crate::{keyval::deserialize_val, Deserialized, Slice}; +use crate::{Deserialized, Slice, keyval::deserialize_val}; pub struct Handle<'a> { val: DBPinnableSlice<'a>, diff --git a/src/database/keyval.rs b/src/database/keyval.rs index f572d15f..6059cd53 100644 --- a/src/database/keyval.rs +++ b/src/database/keyval.rs @@ -1,4 +1,4 @@ -use conduwuit::{smallvec::SmallVec, Result}; +use conduwuit::{Result, smallvec::SmallVec}; use serde::{Deserialize, Serialize}; use crate::{de, ser}; diff --git a/src/database/map.rs b/src/database/map.rs index 37425ecf..c5a908ba 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -40,7 +40,7 @@ pub(crate) use self::options::{ read_options_default, write_options_default, }; pub use self::{get_batch::Get, qry_batch::Qry}; -use crate::{watchers::Watchers, Engine}; +use crate::{Engine, watchers::Watchers}; pub struct Map { name: &'static str, diff --git a/src/database/map/compact.rs b/src/database/map/compact.rs index c0381eb4..84476de6 100644 --- a/src/database/map/compact.rs +++ b/src/database/map/compact.rs @@ -1,4 +1,4 @@ -use conduwuit::{implement, Err, Result}; +use conduwuit::{Err, Result, implement}; use rocksdb::{BottommostLevelCompaction, CompactOptions}; use crate::keyval::KeyBuf; diff --git a/src/database/map/contains.rs b/src/database/map/contains.rs index 7a09b358..474818e8 100644 --- a/src/database/map/contains.rs +++ b/src/database/map/contains.rs @@ -1,10 +1,10 @@ use std::{convert::AsRef, fmt::Debug, future::Future, io::Write, sync::Arc}; use conduwuit::{ + Result, arrayvec::ArrayVec, err, implement, utils::{future::TryExtExt, result::FlatOk}, - Result, }; use futures::FutureExt; use serde::Serialize; @@ -16,7 +16,10 @@ use crate::{keyval::KeyBuf, ser}; /// - harder errors may not be reported #[inline] #[implement(super::Map)] -pub fn contains(self: &Arc, key: &K) -> impl Future + Send + '_ +pub fn contains( + self: &Arc, + key: &K, +) -> impl Future + Send + '_ + use<'_, K> where K: Serialize + ?Sized + Debug, { @@ -32,7 +35,7 @@ where pub fn acontains( self: &Arc, key: &K, -) -> impl Future + Send + '_ +) -> impl Future + Send + '_ + use<'_, MAX, K> where K: Serialize + ?Sized + Debug, { @@ -49,7 +52,7 @@ pub fn bcontains( self: &Arc, key: &K, buf: &mut B, -) -> impl Future + Send + '_ +) -> impl Future + Send + '_ + use<'_, K, B> where K: Serialize + ?Sized + Debug, B: Write + AsRef<[u8]>, @@ -62,7 +65,10 @@ where /// - key is raw #[inline] #[implement(super::Map)] -pub fn exists<'a, K>(self: &'a Arc, key: &K) -> impl Future + Send + 'a +pub fn exists<'a, K>( + self: &'a Arc, + key: &K, +) -> impl Future + Send + 'a + use<'a, K> where K: AsRef<[u8]> + ?Sized + Debug + 'a, { diff --git a/src/database/map/count.rs b/src/database/map/count.rs index 22b298b9..78f9e2e3 100644 --- a/src/database/map/count.rs +++ b/src/database/map/count.rs @@ -16,7 +16,10 @@ pub fn count(self: &Arc) -> impl Future + Send + '_ { /// - From is a structured key #[implement(super::Map)] #[inline] -pub fn count_from<'a, P>(self: &'a Arc, from: &P) -> impl Future + Send + 'a +pub fn count_from<'a, P>( + self: &'a Arc, + from: &P, +) -> impl Future + Send + 'a + use<'a, P> where P: Serialize + ?Sized + Debug + 'a, { @@ -46,7 +49,7 @@ where pub fn count_prefix<'a, P>( self: &'a Arc, prefix: &P, -) -> impl Future + Send + 'a +) -> impl Future + Send + 'a + use<'a, P> where P: Serialize + ?Sized + Debug + 'a, { diff --git a/src/database/map/get.rs b/src/database/map/get.rs index d6c65be2..0971fb17 100644 --- a/src/database/map/get.rs +++ b/src/database/map/get.rs @@ -1,20 +1,23 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{err, implement, utils::result::MapExpect, Err, Result}; -use futures::{future::ready, Future, FutureExt, TryFutureExt}; +use conduwuit::{Err, Result, err, implement, utils::result::MapExpect}; +use futures::{Future, FutureExt, TryFutureExt, future::ready}; use rocksdb::{DBPinnableSlice, ReadOptions}; use tokio::task; use crate::{ - util::{is_incomplete, map_err, or_else}, Handle, + util::{is_incomplete, map_err, or_else}, }; /// Fetch a value from the database into cache, returning a reference-handle /// asynchronously. The key is referenced directly to perform the query. #[implement(super::Map)] #[tracing::instrument(skip(self, key), fields(%self), level = "trace")] -pub fn get(self: &Arc, key: &K) -> impl Future>> + Send +pub fn get( + self: &Arc, + key: &K, +) -> impl Future>> + Send + use<'_, K> where K: AsRef<[u8]> + Debug + ?Sized, { diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index ab9c1dc8..e23a8848 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -1,12 +1,11 @@ use std::{convert::AsRef, sync::Arc}; use conduwuit::{ - implement, + Result, implement, utils::{ - stream::{automatic_amplification, automatic_width, WidebandExt}, IterStream, + stream::{WidebandExt, automatic_amplification, automatic_width}, }, - Result, }; use futures::{Stream, StreamExt, TryStreamExt}; use rocksdb::{DBPinnableSlice, ReadOptions}; @@ -64,7 +63,7 @@ where pub(crate) fn get_batch_cached<'a, I, K>( &self, keys: I, -) -> impl Iterator>>> + Send +) -> impl Iterator>>> + Send + use<'_, I, K> where I: Iterator + ExactSizeIterator + Send, K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, @@ -78,7 +77,7 @@ where pub(crate) fn get_batch_blocking<'a, I, K>( &self, keys: I, -) -> impl Iterator>> + Send +) -> impl Iterator>> + Send + use<'_, I, K> where I: Iterator + ExactSizeIterator + Send, K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, @@ -92,7 +91,7 @@ fn get_batch_blocking_opts<'a, I, K>( &self, keys: I, read_options: &ReadOptions, -) -> impl Iterator>, rocksdb::Error>> + Send +) -> impl Iterator>, rocksdb::Error>> + Send + use<'_, I, K> where I: Iterator + ExactSizeIterator + Send, K: AsRef<[u8]> + Send + ?Sized + Sync + 'a, diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs index 2fe70f15..7ca932a5 100644 --- a/src/database/map/keys.rs +++ b/src/database/map/keys.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::Deserialize; diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index 76c76325..c9b1717a 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -1,13 +1,13 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; use super::stream_from::is_cached; use crate::{ - keyval::{result_deserialize_key, serialize_key, Key}, + keyval::{Key, result_deserialize_key, serialize_key}, stream, }; @@ -15,7 +15,7 @@ use crate::{ pub fn keys_from<'a, K, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -25,7 +25,10 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self), level = "trace")] -pub fn keys_from_raw

    (self: &Arc, from: &P) -> impl Stream>> + Send +pub fn keys_from_raw

    ( + self: &Arc, + from: &P, +) -> impl Stream>> + Send + use<'_, P> where P: Serialize + ?Sized + Debug, { @@ -37,7 +40,7 @@ where pub fn keys_raw_from<'a, K, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, P> where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, @@ -47,7 +50,10 @@ where #[implement(super::Map)] #[tracing::instrument(skip(self, from), fields(%self), level = "trace")] -pub fn raw_keys_from

    (self: &Arc, from: &P) -> impl Stream>> + Send +pub fn raw_keys_from

    ( + self: &Arc, + from: &P, +) -> impl Stream>> + Send + use<'_, P> where P: AsRef<[u8]> + ?Sized + Debug, { diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index 28bc7ccd..09dd79ac 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -1,16 +1,16 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{implement, Result}; -use futures::{future, Stream, StreamExt, TryStreamExt}; +use conduwuit::{Result, implement}; +use futures::{Stream, StreamExt, TryStreamExt, future}; use serde::{Deserialize, Serialize}; -use crate::keyval::{result_deserialize_key, serialize_key, Key}; +use crate::keyval::{Key, result_deserialize_key, serialize_key}; #[implement(super::Map)] pub fn keys_prefix<'a, K, P>( self: &'a Arc, prefix: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -24,7 +24,7 @@ where pub fn keys_prefix_raw

    ( self: &Arc, prefix: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: Serialize + ?Sized + Debug, { diff --git a/src/database/map/qry.rs b/src/database/map/qry.rs index 178f4a61..c6f13c0b 100644 --- a/src/database/map/qry.rs +++ b/src/database/map/qry.rs @@ -1,17 +1,20 @@ use std::{convert::AsRef, fmt::Debug, io::Write, sync::Arc}; -use conduwuit::{arrayvec::ArrayVec, implement, Result}; +use conduwuit::{Result, arrayvec::ArrayVec, implement}; use futures::Future; use serde::Serialize; -use crate::{keyval::KeyBuf, ser, Handle}; +use crate::{Handle, keyval::KeyBuf, ser}; /// Fetch a value from the database into cache, returning a reference-handle /// asynchronously. The key is serialized into an allocated buffer to perform /// the query. #[implement(super::Map)] #[inline] -pub fn qry(self: &Arc, key: &K) -> impl Future>> + Send +pub fn qry( + self: &Arc, + key: &K, +) -> impl Future>> + Send + use<'_, K> where K: Serialize + ?Sized + Debug, { @@ -27,7 +30,7 @@ where pub fn aqry( self: &Arc, key: &K, -) -> impl Future>> + Send +) -> impl Future>> + Send + use<'_, MAX, K> where K: Serialize + ?Sized + Debug, { @@ -43,7 +46,7 @@ pub fn bqry( self: &Arc, key: &K, buf: &mut B, -) -> impl Future>> + Send +) -> impl Future>> + Send + use<'_, K, B> where K: Serialize + ?Sized + Debug, B: Write + AsRef<[u8]>, diff --git a/src/database/map/qry_batch.rs b/src/database/map/qry_batch.rs index 31817c48..f44d1c86 100644 --- a/src/database/map/qry_batch.rs +++ b/src/database/map/qry_batch.rs @@ -1,17 +1,16 @@ use std::{fmt::Debug, sync::Arc}; use conduwuit::{ - implement, + Result, implement, utils::{ - stream::{automatic_amplification, automatic_width, WidebandExt}, IterStream, + stream::{WidebandExt, automatic_amplification, automatic_width}, }, - Result, }; use futures::{Stream, StreamExt, TryStreamExt}; use serde::Serialize; -use crate::{keyval::KeyBuf, ser, Handle}; +use crate::{Handle, keyval::KeyBuf, ser}; pub trait Qry<'a, K, S> where diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index 21558a17..c00f3e55 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::Deserialize; diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index 65072337..04e457dc 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -1,13 +1,13 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; use super::rev_stream_from::is_cached; use crate::{ - keyval::{result_deserialize_key, serialize_key, Key}, + keyval::{Key, result_deserialize_key, serialize_key}, stream, }; @@ -15,7 +15,7 @@ use crate::{ pub fn rev_keys_from<'a, K, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -29,7 +29,7 @@ where pub fn rev_keys_from_raw

    ( self: &Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: Serialize + ?Sized + Debug, { @@ -41,7 +41,7 @@ where pub fn rev_keys_raw_from<'a, K, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, P> where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, @@ -55,7 +55,7 @@ where pub fn rev_raw_keys_from

    ( self: &Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: AsRef<[u8]> + ?Sized + Debug, { diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index fb29acaf..fbe9f9ca 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -1,16 +1,16 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{implement, Result}; -use futures::{future, Stream, StreamExt, TryStreamExt}; +use conduwuit::{Result, implement}; +use futures::{Stream, StreamExt, TryStreamExt, future}; use serde::{Deserialize, Serialize}; -use crate::keyval::{result_deserialize_key, serialize_key, Key}; +use crate::keyval::{Key, result_deserialize_key, serialize_key}; #[implement(super::Map)] pub fn rev_keys_prefix<'a, K, P>( self: &'a Arc, prefix: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -24,7 +24,7 @@ where pub fn rev_keys_prefix_raw

    ( self: &Arc, prefix: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: Serialize + ?Sized + Debug, { diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index f55053be..fc2d1116 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::Deserialize; diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index ddc98607..d67986e7 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -1,13 +1,13 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; use tokio::task; use crate::{ - keyval::{result_deserialize, serialize_key, KeyVal}, + keyval::{KeyVal, result_deserialize, serialize_key}, stream, util::is_incomplete, }; @@ -20,7 +20,7 @@ use crate::{ pub fn rev_stream_from<'a, K, V, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, V, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -39,7 +39,7 @@ where pub fn rev_stream_from_raw

    ( self: &Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: Serialize + ?Sized + Debug, { @@ -55,7 +55,7 @@ where pub fn rev_stream_raw_from<'a, K, V, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, V, P> where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, @@ -74,7 +74,7 @@ where pub fn rev_raw_stream_from

    ( self: &Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: AsRef<[u8]> + ?Sized + Debug, { diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index 22a2ce53..46dc9247 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -1,10 +1,10 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{implement, Result}; -use futures::{future, Stream, StreamExt, TryStreamExt}; +use conduwuit::{Result, implement}; +use futures::{Stream, StreamExt, TryStreamExt, future}; use serde::{Deserialize, Serialize}; -use crate::keyval::{result_deserialize, serialize_key, KeyVal}; +use crate::keyval::{KeyVal, result_deserialize, serialize_key}; /// Iterate key-value entries in the map where the key matches a prefix. /// @@ -14,7 +14,7 @@ use crate::keyval::{result_deserialize, serialize_key, KeyVal}; pub fn rev_stream_prefix<'a, K, V, P>( self: &'a Arc, prefix: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, V, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -33,7 +33,7 @@ where pub fn rev_stream_prefix_raw

    ( self: &Arc, prefix: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: Serialize + ?Sized + Debug, { diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index bfc8ba04..f1450b6f 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::Deserialize; diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 74140a65..00c3a051 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -1,13 +1,13 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use rocksdb::Direction; use serde::{Deserialize, Serialize}; use tokio::task; use crate::{ - keyval::{result_deserialize, serialize_key, KeyVal}, + keyval::{KeyVal, result_deserialize, serialize_key}, stream, }; @@ -19,7 +19,7 @@ use crate::{ pub fn stream_from<'a, K, V, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, V, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -37,7 +37,7 @@ where pub fn stream_from_raw

    ( self: &Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: Serialize + ?Sized + Debug, { @@ -53,7 +53,7 @@ where pub fn stream_raw_from<'a, K, V, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, V, P> where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, @@ -71,7 +71,7 @@ where pub fn raw_stream_from

    ( self: &Arc, from: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: AsRef<[u8]> + ?Sized + Debug, { diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index adacfc81..a26478aa 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -1,10 +1,10 @@ use std::{convert::AsRef, fmt::Debug, sync::Arc}; -use conduwuit::{implement, Result}; -use futures::{future, Stream, StreamExt, TryStreamExt}; +use conduwuit::{Result, implement}; +use futures::{Stream, StreamExt, TryStreamExt, future}; use serde::{Deserialize, Serialize}; -use crate::keyval::{result_deserialize, serialize_key, KeyVal}; +use crate::keyval::{KeyVal, result_deserialize, serialize_key}; /// Iterate key-value entries in the map where the key matches a prefix. /// @@ -14,7 +14,7 @@ use crate::keyval::{result_deserialize, serialize_key, KeyVal}; pub fn stream_prefix<'a, K, V, P>( self: &'a Arc, prefix: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'a, K, V, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -33,7 +33,7 @@ where pub fn stream_prefix_raw

    ( self: &Arc, prefix: &P, -) -> impl Stream>> + Send +) -> impl Stream>> + Send + use<'_, P> where P: Serialize + ?Sized + Debug, { diff --git a/src/database/maps.rs b/src/database/maps.rs index fc216ee0..b060ab8d 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -3,8 +3,8 @@ use std::{collections::BTreeMap, sync::Arc}; use conduwuit::Result; use crate::{ - engine::descriptor::{self, CacheDisp, Descriptor}, Engine, Map, + engine::descriptor::{self, CacheDisp, Descriptor}, }; pub(super) type Maps = BTreeMap; diff --git a/src/database/mod.rs b/src/database/mod.rs index 4f8e2ad9..0481d1bd 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -23,18 +23,18 @@ mod watchers; use std::{ops::Index, sync::Arc}; -use conduwuit::{err, Result, Server}; +use conduwuit::{Result, Server, err}; pub use self::{ de::{Ignore, IgnoreAll}, deserialized::Deserialized, handle::Handle, - keyval::{serialize_key, serialize_val, KeyVal, Slice}, - map::{compact, Get, Map, Qry}, - ser::{serialize, serialize_to, serialize_to_vec, Cbor, Interfix, Json, Separator, SEP}, + keyval::{KeyVal, Slice, serialize_key, serialize_val}, + map::{Get, Map, Qry, compact}, + ser::{Cbor, Interfix, Json, SEP, Separator, serialize, serialize_to, serialize_to_vec}, }; pub(crate) use self::{ - engine::{context::Context, Engine}, + engine::{Engine, context::Context}, util::or_else, }; use crate::maps::{Maps, MapsKey, MapsVal}; diff --git a/src/database/pool.rs b/src/database/pool.rs index 7636ff5e..e6ed59ac 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -3,8 +3,8 @@ mod configure; use std::{ mem::take, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, Mutex, + atomic::{AtomicUsize, Ordering}, }, thread, thread::JoinHandle, @@ -12,19 +12,18 @@ use std::{ use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ - debug, debug_warn, err, error, implement, + Error, Result, Server, debug, debug_warn, err, error, implement, result::DebugInspect, smallvec::SmallVec, trace, utils::sys::compute::{get_affinity, nth_core_available, set_affinity}, - Error, Result, Server, }; -use futures::{channel::oneshot, TryFutureExt}; +use futures::{TryFutureExt, channel::oneshot}; use oneshot::Sender as ResultSender; use rocksdb::Direction; use self::configure::configure; -use crate::{keyval::KeyBuf, stream, Handle, Map}; +use crate::{Handle, Map, keyval::KeyBuf, stream}; /// Frontend thread-pool. Operating system threads are used to make database /// requests which are not cached. These thread-blocking requests are offloaded diff --git a/src/database/pool/configure.rs b/src/database/pool/configure.rs index ff42ef51..92dda56e 100644 --- a/src/database/pool/configure.rs +++ b/src/database/pool/configure.rs @@ -1,7 +1,7 @@ use std::{path::PathBuf, sync::Arc}; use conduwuit::{ - debug, debug_info, expected, is_equal_to, + Server, debug, debug_info, expected, is_equal_to, utils::{ math::usize_from_f64, result::LogDebugErr, @@ -9,7 +9,6 @@ use conduwuit::{ stream::{AMPLIFICATION_LIMIT, WIDTH_LIMIT}, sys::{compute::is_core_available, storage}, }, - Server, }; use super::{QUEUE_LIMIT, WORKER_LIMIT}; diff --git a/src/database/ser.rs b/src/database/ser.rs index 372b7522..6dd2043d 100644 --- a/src/database/ser.rs +++ b/src/database/ser.rs @@ -1,7 +1,7 @@ use std::io::Write; -use conduwuit::{debug::type_name, err, result::DebugInspect, utils::exchange, Error, Result}; -use serde::{ser, Deserialize, Serialize}; +use conduwuit::{Error, Result, debug::type_name, err, result::DebugInspect, utils::exchange}; +use serde::{Deserialize, Serialize, ser}; use crate::util::unhandled; diff --git a/src/database/stream.rs b/src/database/stream.rs index f3063bb3..eb856b3f 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -5,15 +5,15 @@ mod keys_rev; use std::sync::Arc; -use conduwuit::{utils::exchange, Result}; +use conduwuit::{Result, utils::exchange}; use rocksdb::{DBRawIteratorWithThreadMode, ReadOptions}; pub(crate) use self::{items::Items, items_rev::ItemsRev, keys::Keys, keys_rev::KeysRev}; use crate::{ + Map, Slice, engine::Db, keyval::{Key, KeyVal, Val}, util::{is_incomplete, map_err}, - Map, Slice, }; pub(crate) struct State<'a> { diff --git a/src/database/stream/items.rs b/src/database/stream/items.rs index 8814419e..ede2b822 100644 --- a/src/database/stream/items.rs +++ b/src/database/stream/items.rs @@ -2,12 +2,12 @@ use std::pin::Pin; use conduwuit::Result; use futures::{ + Stream, stream::FusedStream, task::{Context, Poll}, - Stream, }; -use super::{keyval_longevity, Cursor, State}; +use super::{Cursor, State, keyval_longevity}; use crate::keyval::KeyVal; pub(crate) struct Items<'a> { diff --git a/src/database/stream/items_rev.rs b/src/database/stream/items_rev.rs index f6fcb0e5..dba8d16c 100644 --- a/src/database/stream/items_rev.rs +++ b/src/database/stream/items_rev.rs @@ -2,12 +2,12 @@ use std::pin::Pin; use conduwuit::Result; use futures::{ + Stream, stream::FusedStream, task::{Context, Poll}, - Stream, }; -use super::{keyval_longevity, Cursor, State}; +use super::{Cursor, State, keyval_longevity}; use crate::keyval::KeyVal; pub(crate) struct ItemsRev<'a> { diff --git a/src/database/stream/keys.rs b/src/database/stream/keys.rs index b953f51c..7c89869b 100644 --- a/src/database/stream/keys.rs +++ b/src/database/stream/keys.rs @@ -2,12 +2,12 @@ use std::pin::Pin; use conduwuit::Result; use futures::{ + Stream, stream::FusedStream, task::{Context, Poll}, - Stream, }; -use super::{slice_longevity, Cursor, State}; +use super::{Cursor, State, slice_longevity}; use crate::keyval::Key; pub(crate) struct Keys<'a> { diff --git a/src/database/stream/keys_rev.rs b/src/database/stream/keys_rev.rs index acf78d88..51561e5c 100644 --- a/src/database/stream/keys_rev.rs +++ b/src/database/stream/keys_rev.rs @@ -2,12 +2,12 @@ use std::pin::Pin; use conduwuit::Result; use futures::{ + Stream, stream::FusedStream, task::{Context, Poll}, - Stream, }; -use super::{slice_longevity, Cursor, State}; +use super::{Cursor, State, slice_longevity}; use crate::keyval::Key; pub(crate) struct KeysRev<'a> { diff --git a/src/database/tests.rs b/src/database/tests.rs index 594170e8..140bc56d 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -4,14 +4,13 @@ use std::fmt::Debug; use conduwuit::{ arrayvec::ArrayVec, - ruma::{serde::Raw, EventId, RoomId, UserId}, + ruma::{EventId, RoomId, UserId, serde::Raw}, }; use serde::Serialize; use crate::{ - de, ser, - ser::{serialize_to_vec, Json}, - Ignore, Interfix, + Ignore, Interfix, de, ser, + ser::{Json, serialize_to_vec}, }; #[test] diff --git a/src/database/watchers.rs b/src/database/watchers.rs index 9ce6f74c..be814f8c 100644 --- a/src/database/watchers.rs +++ b/src/database/watchers.rs @@ -1,5 +1,5 @@ use std::{ - collections::{hash_map, HashMap}, + collections::{HashMap, hash_map}, future::Future, pin::Pin, sync::RwLock, diff --git a/src/macros/admin.rs b/src/macros/admin.rs index e35bd586..bf1586a0 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -1,10 +1,10 @@ use itertools::Itertools; use proc_macro::{Span, TokenStream}; use proc_macro2::TokenStream as TokenStream2; -use quote::{quote, ToTokens}; -use syn::{parse_quote, Attribute, Error, Fields, Ident, ItemEnum, ItemFn, Meta, Variant}; +use quote::{ToTokens, quote}; +use syn::{Attribute, Error, Fields, Ident, ItemEnum, ItemFn, Meta, Variant, parse_quote}; -use crate::{utils::camel_to_snake_string, Result}; +use crate::{Result, utils::camel_to_snake_string}; pub(super) fn command(mut item: ItemFn, _args: &[Meta]) -> Result { let attr: Attribute = parse_quote! { diff --git a/src/macros/cargo.rs b/src/macros/cargo.rs index cd36658e..a452c672 100644 --- a/src/macros/cargo.rs +++ b/src/macros/cargo.rs @@ -4,7 +4,7 @@ use proc_macro::{Span, TokenStream}; use quote::quote; use syn::{Error, ItemConst, Meta}; -use crate::{utils, Result}; +use crate::{Result, utils}; pub(super) fn manifest(item: ItemConst, args: &[Meta]) -> Result { let member = utils::get_named_string(args, "crate"); diff --git a/src/macros/config.rs b/src/macros/config.rs index 50feefa8..07ac1c0a 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -2,15 +2,15 @@ use std::{collections::HashSet, fmt::Write as _, fs::OpenOptions, io::Write as _ use proc_macro::TokenStream; use proc_macro2::{Span, TokenStream as TokenStream2}; -use quote::{quote, ToTokens}; +use quote::{ToTokens, quote}; use syn::{ - parse::Parser, punctuated::Punctuated, spanned::Spanned, Error, Expr, ExprLit, Field, Fields, - FieldsNamed, ItemStruct, Lit, Meta, MetaList, MetaNameValue, Type, TypePath, + Error, Expr, ExprLit, Field, Fields, FieldsNamed, ItemStruct, Lit, Meta, MetaList, + MetaNameValue, Type, TypePath, parse::Parser, punctuated::Punctuated, spanned::Spanned, }; use crate::{ - utils::{get_simple_settings, is_cargo_build, is_cargo_test}, Result, + utils::{get_simple_settings, is_cargo_build, is_cargo_test}, }; const UNDOCUMENTED: &str = "# This item is undocumented. Please contribute documentation for it."; diff --git a/src/macros/implement.rs b/src/macros/implement.rs index 8d18f243..7acc12d2 100644 --- a/src/macros/implement.rs +++ b/src/macros/implement.rs @@ -3,7 +3,7 @@ use quote::quote; use syn::{Error, ItemFn, Meta, Path}; use utils::get_named_generics; -use crate::{utils, Result}; +use crate::{Result, utils}; pub(super) fn implement(item: ItemFn, args: &[Meta]) -> Result { let generics = get_named_generics(args, "generics")?; diff --git a/src/macros/mod.rs b/src/macros/mod.rs index 1aa1e24f..31a797fe 100644 --- a/src/macros/mod.rs +++ b/src/macros/mod.rs @@ -9,8 +9,9 @@ mod utils; use proc_macro::TokenStream; use syn::{ + Error, Item, ItemConst, ItemEnum, ItemFn, ItemStruct, Meta, parse::{Parse, Parser}, - parse_macro_input, Error, Item, ItemConst, ItemEnum, ItemFn, ItemStruct, Meta, + parse_macro_input, }; pub(crate) type Result = std::result::Result; diff --git a/src/macros/refutable.rs b/src/macros/refutable.rs index 66e0ebc3..acfc4cd5 100644 --- a/src/macros/refutable.rs +++ b/src/macros/refutable.rs @@ -1,5 +1,5 @@ use proc_macro::{Span, TokenStream}; -use quote::{quote, ToTokens}; +use quote::{ToTokens, quote}; use syn::{FnArg::Typed, Ident, ItemFn, Meta, Pat, PatIdent, PatType, Stmt}; use crate::Result; @@ -20,7 +20,7 @@ pub(super) fn refutable(mut item: ItemFn, _args: &[Meta]) -> Result let variant = &pat.path; let fields = &pat.fields; - let Some(Typed(PatType { ref mut pat, .. })) = sig.inputs.get_mut(i) else { + let Some(Typed(PatType { pat, .. })) = sig.inputs.get_mut(i) else { continue; }; diff --git a/src/macros/utils.rs b/src/macros/utils.rs index af2519a7..a45e5ecc 100644 --- a/src/macros/utils.rs +++ b/src/macros/utils.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use syn::{parse_str, Expr, ExprLit, Generics, Lit, Meta, MetaNameValue}; +use syn::{Expr, ExprLit, Generics, Lit, Meta, MetaNameValue, parse_str}; use crate::Result; diff --git a/src/main/clap.rs b/src/main/clap.rs index 2bb6f3f2..c7f33bfe 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -4,10 +4,10 @@ use std::path::PathBuf; use clap::{ArgAction, Parser}; use conduwuit::{ + Err, Result, config::{Figment, FigmentValue}, err, toml, utils::available_parallelism, - Err, Result, }; /// Commandline arguments diff --git a/src/main/logging.rs b/src/main/logging.rs index 35e482de..7ce86d56 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -1,13 +1,13 @@ use std::sync::Arc; use conduwuit::{ + Result, config::Config, debug_warn, err, - log::{capture, fmt_span, ConsoleFormat, ConsoleWriter, LogLevelReloadHandles}, + log::{ConsoleFormat, ConsoleWriter, LogLevelReloadHandles, capture, fmt_span}, result::UnwrapOrErr, - Result, }; -use tracing_subscriber::{fmt, layer::SubscriberExt, reload, EnvFilter, Layer, Registry}; +use tracing_subscriber::{EnvFilter, Layer, Registry, fmt, layer::SubscriberExt, reload}; #[cfg(feature = "perf_measurements")] pub(crate) type TracingFlameGuard = diff --git a/src/main/main.rs b/src/main/main.rs index dacc2a2e..2bfc3c06 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -9,9 +9,9 @@ mod signal; extern crate conduwuit_core as conduwuit; -use std::sync::{atomic::Ordering, Arc}; +use std::sync::{Arc, atomic::Ordering}; -use conduwuit::{debug_info, error, rustc_flags_capture, Error, Result}; +use conduwuit::{Error, Result, debug_info, error, rustc_flags_capture}; use server::Server; rustc_flags_capture! {} diff --git a/src/main/mods.rs b/src/main/mods.rs index 9ab36e6c..6dc79b2f 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -6,10 +6,10 @@ extern crate conduwuit_service; use std::{ future::Future, pin::Pin, - sync::{atomic::Ordering, Arc}, + sync::{Arc, atomic::Ordering}, }; -use conduwuit::{debug, error, mods, Error, Result}; +use conduwuit::{Error, Result, debug, error, mods}; use conduwuit_service::Services; use crate::Server; diff --git a/src/main/runtime.rs b/src/main/runtime.rs index 474b373b..b3174e9c 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -1,8 +1,8 @@ use std::{ iter::once, sync::{ - atomic::{AtomicUsize, Ordering}, OnceLock, + atomic::{AtomicUsize, Ordering}, }, thread, time::Duration, @@ -11,9 +11,8 @@ use std::{ #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] use conduwuit::result::LogDebugErr; use conduwuit::{ - is_true, + Result, is_true, utils::sys::compute::{nth_core_available, set_affinity}, - Result, }; use tokio::runtime::Builder; diff --git a/src/main/sentry.rs b/src/main/sentry.rs index 02835ec8..1ea1f3ae 100644 --- a/src/main/sentry.rs +++ b/src/main/sentry.rs @@ -7,11 +7,11 @@ use std::{ use conduwuit::{config::Config, debug, trace}; use sentry::{ - types::{ - protocol::v7::{Context, Event}, - Dsn, - }, Breadcrumb, ClientOptions, Level, + types::{ + Dsn, + protocol::v7::{Context, Event}, + }, }; static SEND_PANIC: OnceLock = OnceLock::new(); diff --git a/src/main/server.rs b/src/main/server.rs index 7376b2fc..44ca69b0 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,11 +1,11 @@ use std::{path::PathBuf, sync::Arc}; use conduwuit::{ + Error, Result, config::Config, info, log::Log, utils::{stream, sys}, - Error, Result, }; use tokio::{runtime, sync::Mutex}; diff --git a/src/router/layers.rs b/src/router/layers.rs index 7ebec16e..88e6a8d5 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -1,16 +1,16 @@ use std::{any::Any, sync::Arc, time::Duration}; use axum::{ - extract::{DefaultBodyLimit, MatchedPath}, Router, + extract::{DefaultBodyLimit, MatchedPath}, }; use axum_client_ip::SecureClientIpSource; -use conduwuit::{debug, error, Result, Server}; +use conduwuit::{Result, Server, debug, error}; use conduwuit_api::router::state::Guard; use conduwuit_service::Services; use http::{ - header::{self, HeaderName}, HeaderValue, Method, StatusCode, + header::{self, HeaderName}, }; use tower::ServiceBuilder; use tower_http::{ @@ -176,12 +176,12 @@ fn catch_panic( .requests_panic .fetch_add(1, std::sync::atomic::Ordering::Release); - let details = if let Some(s) = err.downcast_ref::() { - s.clone() - } else if let Some(s) = err.downcast_ref::<&str>() { - (*s).to_owned() - } else { - "Unknown internal server error occurred.".to_owned() + let details = match err.downcast_ref::() { + | Some(s) => s.clone(), + | _ => match err.downcast_ref::<&str>() { + | Some(s) => (*s).to_owned(), + | _ => "Unknown internal server error occurred.".to_owned(), + }, }; error!("{details:#}"); diff --git a/src/router/request.rs b/src/router/request.rs index b6c22d45..00769b3f 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -1,6 +1,6 @@ use std::{ fmt::Debug, - sync::{atomic::Ordering, Arc}, + sync::{Arc, atomic::Ordering}, time::Duration, }; @@ -8,7 +8,7 @@ use axum::{ extract::State, response::{IntoResponse, Response}, }; -use conduwuit::{debug, debug_error, debug_warn, err, error, trace, Result}; +use conduwuit::{Result, debug, debug_error, debug_warn, err, error, trace}; use conduwuit_service::Services; use futures::FutureExt; use http::{Method, StatusCode, Uri}; diff --git a/src/router/router.rs b/src/router/router.rs index b3531418..0f95b924 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use axum::{response::IntoResponse, routing::get, Router}; +use axum::{Router, response::IntoResponse, routing::get}; use conduwuit::Error; use conduwuit_api::router::{state, state::Guard}; use conduwuit_service::Services; diff --git a/src/router/run.rs b/src/router/run.rs index 024cb813..31789626 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -3,12 +3,12 @@ extern crate conduwuit_core as conduwuit; extern crate conduwuit_service as service; use std::{ - sync::{atomic::Ordering, Arc, Weak}, + sync::{Arc, Weak, atomic::Ordering}, time::Duration, }; use axum_server::Handle as ServerHandle; -use conduwuit::{debug, debug_error, debug_info, error, info, Error, Result, Server}; +use conduwuit::{Error, Result, Server, debug, debug_error, debug_info, error, info}; use futures::FutureExt; use service::Services; use tokio::{ diff --git a/src/router/serve/mod.rs b/src/router/serve/mod.rs index 5c822f2b..2399edf0 100644 --- a/src/router/serve/mod.rs +++ b/src/router/serve/mod.rs @@ -6,7 +6,7 @@ mod unix; use std::sync::Arc; use axum_server::Handle as ServerHandle; -use conduwuit::{err, Result}; +use conduwuit::{Result, err}; use conduwuit_service::Services; use tokio::sync::broadcast; diff --git a/src/router/serve/plain.rs b/src/router/serve/plain.rs index 535282b9..6db7e138 100644 --- a/src/router/serve/plain.rs +++ b/src/router/serve/plain.rs @@ -1,11 +1,11 @@ use std::{ net::SocketAddr, - sync::{atomic::Ordering, Arc}, + sync::{Arc, atomic::Ordering}, }; use axum::Router; -use axum_server::{bind, Handle as ServerHandle}; -use conduwuit::{debug_info, info, Result, Server}; +use axum_server::{Handle as ServerHandle, bind}; +use conduwuit::{Result, Server, debug_info, info}; use tokio::task::JoinSet; pub(super) async fn serve( diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index ab1a9371..dd46ab53 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -3,10 +3,10 @@ use std::{net::SocketAddr, sync::Arc}; use axum::Router; use axum_server::Handle as ServerHandle; use axum_server_dual_protocol::{ - axum_server::{bind_rustls, tls_rustls::RustlsConfig}, ServerExt, + axum_server::{bind_rustls, tls_rustls::RustlsConfig}, }; -use conduwuit::{err, Result, Server}; +use conduwuit::{Result, Server, err}; use tokio::task::JoinSet; use tracing::{debug, info, warn}; diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 6a030c30..2af17274 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -4,15 +4,15 @@ use std::{ net::{self, IpAddr, Ipv4Addr}, os::fd::AsRawFd, path::Path, - sync::{atomic::Ordering, Arc}, + sync::{Arc, atomic::Ordering}, }; use axum::{ - extract::{connect_info::IntoMakeServiceWithConnectInfo, Request}, Router, + extract::{Request, connect_info::IntoMakeServiceWithConnectInfo}, }; use conduwuit::{ - debug, debug_error, info, result::UnwrapInfallible, trace, warn, Err, Result, Server, + Err, Result, Server, debug, debug_error, info, result::UnwrapInfallible, trace, warn, }; use hyper::{body::Incoming, service::service_fn}; use hyper_util::{ @@ -21,10 +21,10 @@ use hyper_util::{ }; use tokio::{ fs, - net::{unix::SocketAddr, UnixListener, UnixStream}, + net::{UnixListener, UnixStream, unix::SocketAddr}, sync::broadcast::{self}, task::JoinSet, - time::{sleep, Duration}, + time::{Duration, sleep}, }; use tower::{Service, ServiceExt}; diff --git a/src/service/account_data/mod.rs b/src/service/account_data/mod.rs index 5a943f88..453051be 100644 --- a/src/service/account_data/mod.rs +++ b/src/service/account_data/mod.rs @@ -1,23 +1,22 @@ use std::sync::Arc; use conduwuit::{ - err, implement, - utils::{result::LogErr, stream::TryIgnore, ReadyExt}, - Err, Result, + Err, Result, err, implement, + utils::{ReadyExt, result::LogErr, stream::TryIgnore}, }; use database::{Deserialized, Handle, Ignore, Json, Map}; use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{ + RoomId, UserId, events::{ AnyGlobalAccountDataEvent, AnyRawAccountDataEvent, AnyRoomAccountDataEvent, GlobalAccountDataEventType, RoomAccountDataEventType, }, serde::Raw, - RoomId, UserId, }; use serde::Deserialize; -use crate::{globals, Dep}; +use crate::{Dep, globals}; pub struct Service { services: Services, diff --git a/src/service/admin/console.rs b/src/service/admin/console.rs index 59b9a31b..02f41303 100644 --- a/src/service/admin/console.rs +++ b/src/service/admin/console.rs @@ -5,14 +5,14 @@ use std::{ sync::{Arc, Mutex}, }; -use conduwuit::{debug, defer, error, log, log::is_systemd_mode, Server}; +use conduwuit::{Server, debug, defer, error, log, log::is_systemd_mode}; use futures::future::{AbortHandle, Abortable}; use ruma::events::room::message::RoomMessageEventContent; use rustyline_async::{Readline, ReadlineError, ReadlineEvent}; use termimad::MadSkin; use tokio::task::JoinHandle; -use crate::{admin, Dep}; +use crate::{Dep, admin}; pub struct Console { server: Arc, @@ -221,7 +221,7 @@ pub fn print(markdown: &str) { } fn configure_output_err(mut output: MadSkin) -> MadSkin { - use termimad::{crossterm::style::Color, Alignment, CompoundStyle, LineStyle}; + use termimad::{Alignment, CompoundStyle, LineStyle, crossterm::style::Color}; let code_style = CompoundStyle::with_fgbg(Color::AnsiValue(196), Color::AnsiValue(234)); output.inline_code = code_style.clone(); @@ -236,7 +236,7 @@ fn configure_output_err(mut output: MadSkin) -> MadSkin { } fn configure_output(mut output: MadSkin) -> MadSkin { - use termimad::{crossterm::style::Color, Alignment, CompoundStyle, LineStyle}; + use termimad::{Alignment, CompoundStyle, LineStyle, crossterm::style::Color}; let code_style = CompoundStyle::with_fgbg(Color::AnsiValue(40), Color::AnsiValue(234)); output.inline_code = code_style.clone(); diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 7b691fb1..7f71665a 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -1,7 +1,8 @@ use std::collections::BTreeMap; -use conduwuit::{pdu::PduBuilder, Result}; +use conduwuit::{Result, pdu::PduBuilder}; use ruma::{ + RoomId, RoomVersionId, events::room::{ canonical_alias::RoomCanonicalAliasEventContent, create::RoomCreateEventContent, @@ -14,7 +15,6 @@ use ruma::{ preview_url::RoomPreviewUrlsEventContent, topic::RoomTopicEventContent, }, - RoomId, RoomVersionId, }; use crate::Services; diff --git a/src/service/admin/execute.rs b/src/service/admin/execute.rs index 462681da..174b28ed 100644 --- a/src/service/admin/execute.rs +++ b/src/service/admin/execute.rs @@ -1,6 +1,6 @@ -use conduwuit::{debug, debug_info, error, implement, info, Err, Result}; +use conduwuit::{Err, Result, debug, debug_info, error, implement, info}; use ruma::events::room::message::RoomMessageEventContent; -use tokio::time::{sleep, Duration}; +use tokio::time::{Duration, sleep}; pub(super) const SIGNAL: &str = "SIGUSR2"; diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 3ad9283f..358ea267 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -1,17 +1,17 @@ use std::collections::BTreeMap; -use conduwuit::{error, implement, Result}; +use conduwuit::{Result, error, implement}; use ruma::{ + RoomId, UserId, events::{ + RoomAccountDataEventType, room::{ member::{MembershipState, RoomMemberEventContent}, message::RoomMessageEventContent, power_levels::RoomPowerLevelsEventContent, }, tag::{TagEvent, TagEventContent, TagInfo}, - RoomAccountDataEventType, }, - RoomId, UserId, }; use crate::pdu::PduBuilder; diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 31b046b7..4622f10e 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -11,18 +11,18 @@ use std::{ use async_trait::async_trait; use conduwuit::{ - debug, err, error, error::default_log, pdu::PduBuilder, Error, PduEvent, Result, Server, + Error, PduEvent, Result, Server, debug, err, error, error::default_log, pdu::PduBuilder, }; pub use create::create_admin_room; use futures::{FutureExt, TryFutureExt}; use loole::{Receiver, Sender}; use ruma::{ - events::room::message::{Relation, RoomMessageEventContent}, OwnedEventId, OwnedRoomId, RoomId, UserId, + events::room::message::{Relation, RoomMessageEventContent}, }; use tokio::sync::RwLock; -use crate::{account_data, globals, rooms, rooms::state::RoomMutexGuard, Dep}; +use crate::{Dep, account_data, globals, rooms, rooms::state::RoomMutexGuard}; pub struct Service { services: Services, diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 2a54ee09..5aba0018 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -4,14 +4,14 @@ mod registration_info; use std::{collections::BTreeMap, sync::Arc}; use async_trait::async_trait; -use conduwuit::{err, utils::stream::TryIgnore, Result}; +use conduwuit::{Result, err, utils::stream::TryIgnore}; use database::Map; use futures::{Future, StreamExt, TryStreamExt}; -use ruma::{api::appservice::Registration, RoomAliasId, RoomId, UserId}; +use ruma::{RoomAliasId, RoomId, UserId, api::appservice::Registration}; use tokio::sync::RwLock; pub use self::{namespace_regex::NamespaceRegex, registration_info::RegistrationInfo}; -use crate::{sending, Dep}; +use crate::{Dep, sending}; pub struct Service { registration_info: RwLock>, diff --git a/src/service/appservice/registration_info.rs b/src/service/appservice/registration_info.rs index 9758e186..a511f58d 100644 --- a/src/service/appservice/registration_info.rs +++ b/src/service/appservice/registration_info.rs @@ -1,5 +1,5 @@ use conduwuit::Result; -use ruma::{api::appservice::Registration, UserId}; +use ruma::{UserId, api::appservice::Registration}; use super::NamespaceRegex; diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index f63d78b8..d5008491 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -1,6 +1,6 @@ use std::{sync::Arc, time::Duration}; -use conduwuit::{err, implement, trace, Config, Result}; +use conduwuit::{Config, Result, err, implement, trace}; use either::Either; use ipaddress::IPAddress; use reqwest::redirect; @@ -172,10 +172,9 @@ fn base(config: &Config) -> Result { builder = builder.no_zstd(); }; - if let Some(proxy) = config.proxy.to_proxy()? { - Ok(builder.proxy(proxy)) - } else { - Ok(builder) + match config.proxy.to_proxy()? { + | Some(proxy) => Ok(builder.proxy(proxy)), + | _ => Ok(builder), } } diff --git a/src/service/config/mod.rs b/src/service/config/mod.rs index c9ac37a3..fd0d8764 100644 --- a/src/service/config/mod.rs +++ b/src/service/config/mod.rs @@ -2,8 +2,9 @@ use std::{iter, ops::Deref, path::Path, sync::Arc}; use async_trait::async_trait; use conduwuit::{ - config::{check, Config}, - error, implement, Result, Server, + Result, Server, + config::{Config, check}, + error, implement, }; pub struct Service { diff --git a/src/service/emergency/mod.rs b/src/service/emergency/mod.rs index 9b2e4025..47a309a5 100644 --- a/src/service/emergency/mod.rs +++ b/src/service/emergency/mod.rs @@ -1,15 +1,15 @@ use std::sync::Arc; use async_trait::async_trait; -use conduwuit::{error, warn, Result}; +use conduwuit::{Result, error, warn}; use ruma::{ events::{ - push_rules::PushRulesEventContent, GlobalAccountDataEvent, GlobalAccountDataEventType, + GlobalAccountDataEvent, GlobalAccountDataEventType, push_rules::PushRulesEventContent, }, push::Ruleset, }; -use crate::{account_data, globals, users, Dep}; +use crate::{Dep, account_data, globals, users}; pub struct Service { services: Services, diff --git a/src/service/federation/execute.rs b/src/service/federation/execute.rs index 3146bb8a..d254486f 100644 --- a/src/service/federation/execute.rs +++ b/src/service/federation/execute.rs @@ -2,20 +2,20 @@ use std::{fmt::Debug, mem}; use bytes::Bytes; use conduwuit::{ - debug, debug::INFO_SPAN_LEVEL, debug_error, debug_warn, err, error::inspect_debug_log, - implement, trace, utils::string::EMPTY, Err, Error, Result, + Err, Error, Result, debug, debug::INFO_SPAN_LEVEL, debug_error, debug_warn, err, + error::inspect_debug_log, implement, trace, utils::string::EMPTY, }; -use http::{header::AUTHORIZATION, HeaderValue}; +use http::{HeaderValue, header::AUTHORIZATION}; use ipaddress::IPAddress; use reqwest::{Client, Method, Request, Response, Url}; use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, ServerName, ServerSigningKeyId, api::{ - client::error::Error as RumaError, EndpointError, IncomingResponse, MatrixVersion, - OutgoingRequest, SendAccessToken, + EndpointError, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, + client::error::Error as RumaError, }, serde::Base64, server_util::authorization::XMatrix, - CanonicalJsonObject, CanonicalJsonValue, ServerName, ServerSigningKeyId, }; use crate::resolver::actual::ActualDest; diff --git a/src/service/federation/mod.rs b/src/service/federation/mod.rs index dacdb20e..ce7765ee 100644 --- a/src/service/federation/mod.rs +++ b/src/service/federation/mod.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use conduwuit::{Result, Server}; -use crate::{client, resolver, server_keys, Dep}; +use crate::{Dep, client, resolver, server_keys}; pub struct Service { services: Services, diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index 26a18607..b43b7c5f 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -1,6 +1,6 @@ use std::sync::{Arc, RwLock}; -use conduwuit::{utils, Result}; +use conduwuit::{Result, utils}; use database::{Database, Deserialized, Map}; pub struct Data { diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 485d5020..16b3ef3c 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,7 +7,7 @@ use std::{ time::Instant, }; -use conduwuit::{error, utils::bytes::pretty, Result, Server}; +use conduwuit::{Result, Server, error, utils::bytes::pretty}; use data::Data; use regex::RegexSet; use ruma::{OwnedEventId, OwnedRoomAliasId, OwnedServerName, OwnedUserId, ServerName, UserId}; diff --git a/src/service/key_backups/mod.rs b/src/service/key_backups/mod.rs index 1165c3ed..1bf048ef 100644 --- a/src/service/key_backups/mod.rs +++ b/src/service/key_backups/mod.rs @@ -1,19 +1,18 @@ use std::{collections::BTreeMap, sync::Arc}; use conduwuit::{ - err, implement, + Err, Result, err, implement, utils::stream::{ReadyExt, TryIgnore}, - Err, Result, }; use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::StreamExt; use ruma::{ + OwnedRoomId, RoomId, UserId, api::client::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup}, serde::Raw, - OwnedRoomId, RoomId, UserId, }; -use crate::{globals, Dep}; +use crate::{Dep, globals}; pub struct Service { db: Data, diff --git a/src/service/manager.rs b/src/service/manager.rs index e0d885c2..3cdf5945 100644 --- a/src/service/manager.rs +++ b/src/service/manager.rs @@ -1,6 +1,6 @@ use std::{panic::AssertUnwindSafe, sync::Arc, time::Duration}; -use conduwuit::{debug, debug_warn, error, trace, utils::time, warn, Err, Error, Result, Server}; +use conduwuit::{Err, Error, Result, Server, debug, debug_warn, error, trace, utils::time, warn}; use futures::{FutureExt, TryFutureExt}; use tokio::{ sync::{Mutex, MutexGuard}, @@ -8,7 +8,7 @@ use tokio::{ time::sleep, }; -use crate::{service, service::Service, Services}; +use crate::{Services, service, service::Service}; pub(crate) struct Manager { manager: Mutex>>>, diff --git a/src/service/media/blurhash.rs b/src/service/media/blurhash.rs index 60ade723..9d73f5dc 100644 --- a/src/service/media/blurhash.rs +++ b/src/service/media/blurhash.rs @@ -1,6 +1,6 @@ #[cfg(feature = "blurhashing")] use conduwuit::config::BlurhashConfig as CoreBlurhashConfig; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use super::Service; diff --git a/src/service/media/data.rs b/src/service/media/data.rs index f48482ea..0ccd844f 100644 --- a/src/service/media/data.rs +++ b/src/service/media/data.rs @@ -1,13 +1,12 @@ use std::{sync::Arc, time::Duration}; use conduwuit::{ - debug, debug_info, err, - utils::{str_from_bytes, stream::TryIgnore, string_from_bytes, ReadyExt}, - Err, Result, + Err, Result, debug, debug_info, err, + utils::{ReadyExt, str_from_bytes, stream::TryIgnore, string_from_bytes}, }; use database::{Database, Interfix, Map}; use futures::StreamExt; -use ruma::{http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; +use ruma::{Mxc, OwnedMxcUri, UserId, http_headers::ContentDisposition}; use super::{preview::UrlPreviewData, thumbnail::Dim}; diff --git a/src/service/media/migrations.rs b/src/service/media/migrations.rs index 8526ffcd..5fd628cd 100644 --- a/src/service/media/migrations.rs +++ b/src/service/media/migrations.rs @@ -8,9 +8,9 @@ use std::{ }; use conduwuit::{ - debug, debug_info, debug_warn, error, info, - utils::{stream::TryIgnore, ReadyExt}, - warn, Config, Result, + Config, Result, debug, debug_info, debug_warn, error, info, + utils::{ReadyExt, stream::TryIgnore}, + warn, }; use crate::Services; diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index f5913f43..5c26efe8 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -8,13 +8,13 @@ mod thumbnail; use std::{path::PathBuf, sync::Arc, time::SystemTime}; use async_trait::async_trait; -use base64::{engine::general_purpose, Engine as _}; +use base64::{Engine as _, engine::general_purpose}; use conduwuit::{ - debug, debug_error, debug_info, debug_warn, err, error, trace, + Err, Result, Server, debug, debug_error, debug_info, debug_warn, err, error, trace, utils::{self, MutexMap}, - warn, Err, Result, Server, + warn, }; -use ruma::{http_headers::ContentDisposition, Mxc, OwnedMxcUri, UserId}; +use ruma::{Mxc, OwnedMxcUri, UserId, http_headers::ContentDisposition}; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt, BufReader}, @@ -22,7 +22,7 @@ use tokio::{ use self::data::{Data, Metadata}; pub use self::thumbnail::Dim; -use crate::{client, globals, sending, Dep}; +use crate::{Dep, client, globals, sending}; #[derive(Debug)] pub struct FileMeta { @@ -105,22 +105,27 @@ impl Service { /// Deletes a file in the database and from the media directory via an MXC pub async fn delete(&self, mxc: &Mxc<'_>) -> Result<()> { - if let Ok(keys) = self.db.search_mxc_metadata_prefix(mxc).await { - for key in keys { - trace!(?mxc, "MXC Key: {key:?}"); - debug_info!(?mxc, "Deleting from filesystem"); + match self.db.search_mxc_metadata_prefix(mxc).await { + | Ok(keys) => { + for key in keys { + trace!(?mxc, "MXC Key: {key:?}"); + debug_info!(?mxc, "Deleting from filesystem"); - if let Err(e) = self.remove_media_file(&key).await { - debug_error!(?mxc, "Failed to remove media file: {e}"); + if let Err(e) = self.remove_media_file(&key).await { + debug_error!(?mxc, "Failed to remove media file: {e}"); + } + + debug_info!(?mxc, "Deleting from database"); + self.db.delete_file_mxc(mxc).await; } - debug_info!(?mxc, "Deleting from database"); - self.db.delete_file_mxc(mxc).await; - } - - Ok(()) - } else { - Err!(Database(error!("Failed to find any media keys for MXC {mxc} in our database."))) + Ok(()) + }, + | _ => { + Err!(Database(error!( + "Failed to find any media keys for MXC {mxc} in our database." + ))) + }, } } @@ -154,22 +159,21 @@ impl Service { /// Downloads a file. pub async fn get(&self, mxc: &Mxc<'_>) -> Result> { - if let Ok(Metadata { content_disposition, content_type, key }) = - self.db.search_file_metadata(mxc, &Dim::default()).await - { - let mut content = Vec::with_capacity(8192); - let path = self.get_media_file(&key); - BufReader::new(fs::File::open(path).await?) - .read_to_end(&mut content) - .await?; + match self.db.search_file_metadata(mxc, &Dim::default()).await { + | Ok(Metadata { content_disposition, content_type, key }) => { + let mut content = Vec::with_capacity(8192); + let path = self.get_media_file(&key); + BufReader::new(fs::File::open(path).await?) + .read_to_end(&mut content) + .await?; - Ok(Some(FileMeta { - content: Some(content), - content_type, - content_disposition, - })) - } else { - Ok(None) + Ok(Some(FileMeta { + content: Some(content), + content_type, + content_disposition, + })) + }, + | _ => Ok(None), } } diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index e7f76bab..17216869 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -7,7 +7,7 @@ use std::time::SystemTime; -use conduwuit::{debug, Err, Result}; +use conduwuit::{Err, Result, debug}; use conduwuit_core::implement; use ipaddress::IPAddress; use serde::Serialize; diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index 72f1184e..61635011 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -1,21 +1,21 @@ use std::{fmt::Debug, time::Duration}; use conduwuit::{ - debug_warn, err, implement, utils::content_disposition::make_content_disposition, Err, Error, - Result, + Err, Error, Result, debug_warn, err, implement, + utils::content_disposition::make_content_disposition, }; -use http::header::{HeaderValue, CONTENT_DISPOSITION, CONTENT_TYPE}; +use http::header::{CONTENT_DISPOSITION, CONTENT_TYPE, HeaderValue}; use ruma::{ + Mxc, ServerName, UserId, api::{ + OutgoingRequest, client::{ error::ErrorKind::{NotFound, Unrecognized}, media, }, federation, federation::authenticated_media::{Content, FileOrLocation}, - OutgoingRequest, }, - Mxc, ServerName, UserId, }; use super::{Dim, FileMeta}; diff --git a/src/service/media/tests.rs b/src/service/media/tests.rs index 1d6dce30..651e0ade 100644 --- a/src/service/media/tests.rs +++ b/src/service/media/tests.rs @@ -5,7 +5,7 @@ async fn long_file_names_works() { use std::path::PathBuf; - use base64::{engine::general_purpose, Engine as _}; + use base64::{Engine as _, engine::general_purpose}; use super::*; diff --git a/src/service/media/thumbnail.rs b/src/service/media/thumbnail.rs index 7350b3a1..e5a98774 100644 --- a/src/service/media/thumbnail.rs +++ b/src/service/media/thumbnail.rs @@ -7,14 +7,14 @@ use std::{cmp, num::Saturating as Sat}; -use conduwuit::{checked, err, implement, Result}; -use ruma::{http_headers::ContentDisposition, media::Method, Mxc, UInt, UserId}; +use conduwuit::{Result, checked, err, implement}; +use ruma::{Mxc, UInt, UserId, http_headers::ContentDisposition, media::Method}; use tokio::{ fs, io::{AsyncReadExt, AsyncWriteExt}, }; -use super::{data::Metadata, FileMeta}; +use super::{FileMeta, data::Metadata}; /// Dimension specification for a thumbnail. #[derive(Debug)] @@ -65,12 +65,12 @@ impl super::Service { // 0, 0 because that's the original file let dim = dim.normalized(); - if let Ok(metadata) = self.db.search_file_metadata(mxc, &dim).await { - self.get_thumbnail_saved(metadata).await - } else if let Ok(metadata) = self.db.search_file_metadata(mxc, &Dim::default()).await { - self.get_thumbnail_generate(mxc, &dim, metadata).await - } else { - Ok(None) + match self.db.search_file_metadata(mxc, &dim).await { + | Ok(metadata) => self.get_thumbnail_saved(metadata).await, + | _ => match self.db.search_file_metadata(mxc, &Dim::default()).await { + | Ok(metadata) => self.get_thumbnail_generate(mxc, &dim, metadata).await, + | _ => Ok(None), + }, } } } diff --git a/src/service/migrations.rs b/src/service/migrations.rs index 69b1be4e..512a7867 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -1,25 +1,25 @@ use std::cmp; use conduwuit::{ - debug, debug_info, debug_warn, error, info, + Err, Result, debug, debug_info, debug_warn, error, info, result::NotFound, utils::{ - stream::{TryExpect, TryIgnore}, IterStream, ReadyExt, + stream::{TryExpect, TryIgnore}, }, - warn, Err, Result, + warn, }; use futures::{FutureExt, StreamExt}; use itertools::Itertools; use ruma::{ + OwnedUserId, RoomId, UserId, events::{ - push_rules::PushRulesEvent, room::member::MembershipState, GlobalAccountDataEventType, + GlobalAccountDataEventType, push_rules::PushRulesEvent, room::member::MembershipState, }, push::Ruleset, - OwnedUserId, RoomId, UserId, }; -use crate::{media, Services}; +use crate::{Services, media}; /// The current schema version. /// - If database is opened at greater version we reject with error. The diff --git a/src/service/mod.rs b/src/service/mod.rs index 71bd0eb4..0bde0255 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -30,7 +30,7 @@ pub mod users; extern crate conduwuit_core as conduwuit; extern crate conduwuit_database as database; -pub use conduwuit::{pdu, PduBuilder, PduCount, PduEvent}; +pub use conduwuit::{PduBuilder, PduCount, PduEvent, pdu}; pub(crate) use service::{Args, Dep, Service}; pub use crate::services::Services; diff --git a/src/service/presence/data.rs b/src/service/presence/data.rs index 4ec0a7ee..d7ef5175 100644 --- a/src/service/presence/data.rs +++ b/src/service/presence/data.rs @@ -1,16 +1,15 @@ use std::sync::Arc; use conduwuit::{ - debug_warn, utils, - utils::{stream::TryIgnore, ReadyExt}, - Result, + Result, debug_warn, utils, + utils::{ReadyExt, stream::TryIgnore}, }; use database::{Deserialized, Json, Map}; use futures::Stream; -use ruma::{events::presence::PresenceEvent, presence::PresenceState, UInt, UserId}; +use ruma::{UInt, UserId, events::presence::PresenceEvent, presence::PresenceState}; use super::Presence; -use crate::{globals, users, Dep}; +use crate::{Dep, globals, users}; pub(crate) struct Data { presenceid_presence: Arc, diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index eb4105e5..8f646be6 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -5,16 +5,16 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; use conduwuit::{ - checked, debug, debug_warn, error, result::LogErr, trace, Error, Result, Server, + Error, Result, Server, checked, debug, debug_warn, error, result::LogErr, trace, }; use database::Database; -use futures::{stream::FuturesUnordered, Stream, StreamExt, TryFutureExt}; +use futures::{Stream, StreamExt, TryFutureExt, stream::FuturesUnordered}; use loole::{Receiver, Sender}; -use ruma::{events::presence::PresenceEvent, presence::PresenceState, OwnedUserId, UInt, UserId}; +use ruma::{OwnedUserId, UInt, UserId, events::presence::PresenceEvent, presence::PresenceState}; use tokio::time::sleep; use self::{data::Data, presence::Presence}; -use crate::{globals, users, Dep}; +use crate::{Dep, globals, users}; pub struct Service { timer_channel: (Sender, Receiver), diff --git a/src/service/presence/presence.rs b/src/service/presence/presence.rs index b322dfb4..3357bd61 100644 --- a/src/service/presence/presence.rs +++ b/src/service/presence/presence.rs @@ -1,8 +1,8 @@ -use conduwuit::{utils, Error, Result}; +use conduwuit::{Error, Result, utils}; use ruma::{ + UInt, UserId, events::presence::{PresenceEvent, PresenceEventContent}, presence::PresenceState, - UInt, UserId, }; use serde::{Deserialize, Serialize}; diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 43d60c08..2b269b3d 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -2,34 +2,35 @@ use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; use conduwuit::{ - debug_warn, err, trace, + Err, PduEvent, Result, debug_warn, err, trace, utils::{stream::TryIgnore, string_from_bytes}, - warn, Err, PduEvent, Result, + warn, }; use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::{Stream, StreamExt}; use ipaddress::IPAddress; use ruma::{ + RoomId, UInt, UserId, api::{ - client::push::{set_pusher, Pusher, PusherKind}, + IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, + client::push::{Pusher, PusherKind, set_pusher}, push_gateway::send_event_notification::{ self, v1::{Device, Notification, NotificationCounts, NotificationPriority}, }, - IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, }, events::{ - room::power_levels::RoomPowerLevelsEventContent, AnySyncTimelineEvent, StateEventType, - TimelineEventType, + AnySyncTimelineEvent, StateEventType, TimelineEventType, + room::power_levels::RoomPowerLevelsEventContent, }, push::{ Action, PushConditionPowerLevelsCtx, PushConditionRoomCtx, PushFormat, Ruleset, Tweak, }, serde::Raw, - uint, RoomId, UInt, UserId, + uint, }; -use crate::{client, globals, rooms, sending, users, Dep}; +use crate::{Dep, client, globals, rooms, sending, users}; pub struct Service { db: Data, diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 66854764..8860d0a0 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -3,7 +3,7 @@ use std::{ net::{IpAddr, SocketAddr}, }; -use conduwuit::{debug, debug_error, debug_info, debug_warn, err, error, trace, Err, Result}; +use conduwuit::{Err, Result, debug, debug_error, debug_info, debug_warn, err, error, trace}; use futures::{FutureExt, TryFutureExt}; use hickory_resolver::error::ResolveError; use ipaddress::IPAddress; @@ -11,7 +11,7 @@ use ruma::ServerName; use super::{ cache::{CachedDest, CachedOverride, MAX_IPS}, - fed::{add_port_to_hostname, get_ip_with_port, FedDest, PortString}, + fed::{FedDest, PortString, add_port_to_hostname, get_ip_with_port}, }; #[derive(Clone, Debug)] @@ -71,12 +71,16 @@ impl super::Service { | None => if let Some(pos) = dest.as_str().find(':') { self.actual_dest_2(dest, cache, pos).await? - } else if let Some(delegated) = self.request_well_known(dest.as_str()).await? { - self.actual_dest_3(&mut host, cache, delegated).await? - } else if let Some(overrider) = self.query_srv_record(dest.as_str()).await? { - self.actual_dest_4(&host, cache, overrider).await? } else { - self.actual_dest_5(dest, cache).await? + match self.request_well_known(dest.as_str()).await? { + | Some(delegated) => + self.actual_dest_3(&mut host, cache, delegated).await?, + | _ => match self.query_srv_record(dest.as_str()).await? { + | Some(overrider) => + self.actual_dest_4(&host, cache, overrider).await?, + | _ => self.actual_dest_5(dest, cache).await?, + }, + } }, }; @@ -136,10 +140,10 @@ impl super::Service { self.actual_dest_3_2(cache, delegated, pos).await } else { trace!("Delegated hostname has no port in this branch"); - if let Some(overrider) = self.query_srv_record(&delegated).await? { - self.actual_dest_3_3(cache, delegated, overrider).await - } else { - self.actual_dest_3_4(cache, delegated).await + match self.query_srv_record(&delegated).await? { + | Some(overrider) => + self.actual_dest_3_3(cache, delegated, overrider).await, + | _ => self.actual_dest_3_4(cache, delegated).await, } }, } diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index 7b4f104d..6b05c00c 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -1,10 +1,10 @@ use std::{net::IpAddr, sync::Arc, time::SystemTime}; use conduwuit::{ + Result, arrayvec::ArrayVec, at, err, implement, utils::{math::Expected, rand, stream::TryIgnore}, - Result, }; use database::{Cbor, Deserialized, Map}; use futures::{Stream, StreamExt}; @@ -96,7 +96,7 @@ pub fn destinations(&self) -> impl Stream + Se self.destinations .stream() .ignore_err() - .map(|item: (&ServerName, Cbor<_>)| (item.0, item.1 .0)) + .map(|item: (&ServerName, Cbor<_>)| (item.0, item.1.0)) } #[implement(Cache)] @@ -104,7 +104,7 @@ pub fn overrides(&self) -> impl Stream + S self.overrides .stream() .ignore_err() - .map(|item: (&ServerName, Cbor<_>)| (item.0, item.1 .0)) + .map(|item: (&ServerName, Cbor<_>)| (item.0, item.1.0)) } impl CachedDest { diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index ca6106e2..98ad7e60 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -1,8 +1,8 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; -use conduwuit::{err, Result, Server}; +use conduwuit::{Result, Server, err}; use futures::FutureExt; -use hickory_resolver::{lookup_ip::LookupIp, TokioAsyncResolver}; +use hickory_resolver::{TokioAsyncResolver, lookup_ip::LookupIp}; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use super::cache::{Cache, CachedOverride}; diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 6be9d42d..2ec9c0ef 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -6,10 +6,10 @@ mod tests; use std::sync::Arc; -use conduwuit::{arrayvec::ArrayString, utils::MutexMap, Result, Server}; +use conduwuit::{Result, Server, arrayvec::ArrayString, utils::MutexMap}; use self::{cache::Cache, dns::Resolver}; -use crate::{client, Dep}; +use crate::{Dep, client}; pub struct Service { pub cache: Arc, diff --git a/src/service/resolver/tests.rs b/src/service/resolver/tests.rs index 870f5eab..6e9d0e71 100644 --- a/src/service/resolver/tests.rs +++ b/src/service/resolver/tests.rs @@ -1,6 +1,6 @@ #![cfg(test)] -use super::fed::{add_port_to_hostname, get_ip_with_port, FedDest}; +use super::fed::{FedDest, add_port_to_hostname, get_ip_with_port}; #[test] fn ips_get_default_ports() { diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 17ed5e13..866e45a9 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -3,21 +3,20 @@ mod remote; use std::sync::Arc; use conduwuit::{ - err, - utils::{stream::TryIgnore, ReadyExt}, - Err, Result, Server, + Err, Result, Server, err, + utils::{ReadyExt, stream::TryIgnore}, }; use database::{Deserialized, Ignore, Interfix, Map}; use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{ - events::{ - room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, - StateEventType, - }, OwnedRoomId, OwnedServerName, OwnedUserId, RoomAliasId, RoomId, RoomOrAliasId, UserId, + events::{ + StateEventType, + room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, + }, }; -use crate::{admin, appservice, appservice::RegistrationInfo, globals, rooms, sending, Dep}; +use crate::{Dep, admin, appservice, appservice::RegistrationInfo, globals, rooms, sending}; pub struct Service { db: Data, diff --git a/src/service/rooms/alias/remote.rs b/src/service/rooms/alias/remote.rs index 7744bee2..60aed76d 100644 --- a/src/service/rooms/alias/remote.rs +++ b/src/service/rooms/alias/remote.rs @@ -1,8 +1,8 @@ use std::iter::once; -use conduwuit::{debug, debug_error, err, implement, Result}; +use conduwuit::{Result, debug, debug_error, err, implement}; use federation::query::get_room_information::v1::Response; -use ruma::{api::federation, OwnedRoomId, OwnedServerName, RoomAliasId, ServerName}; +use ruma::{OwnedRoomId, OwnedServerName, RoomAliasId, ServerName, api::federation}; #[implement(super::Service)] pub(super) async fn remote_resolve( diff --git a/src/service/rooms/auth_chain/data.rs b/src/service/rooms/auth_chain/data.rs index af8ae364..8c3588cc 100644 --- a/src/service/rooms/auth_chain/data.rs +++ b/src/service/rooms/auth_chain/data.rs @@ -3,7 +3,7 @@ use std::{ sync::{Arc, Mutex}, }; -use conduwuit::{err, utils, utils::math::usize_from_f64, Err, Result}; +use conduwuit::{Err, Result, err, utils, utils::math::usize_from_f64}; use database::Map; use lru_cache::LruCache; diff --git a/src/service/rooms/auth_chain/mod.rs b/src/service/rooms/auth_chain/mod.rs index 0ff96846..0903ea75 100644 --- a/src/service/rooms/auth_chain/mod.rs +++ b/src/service/rooms/auth_chain/mod.rs @@ -8,18 +8,18 @@ use std::{ }; use conduwuit::{ - at, debug, debug_error, implement, trace, + Err, Result, at, debug, debug_error, implement, trace, utils::{ - stream::{ReadyExt, TryBroadbandExt}, IterStream, + stream::{ReadyExt, TryBroadbandExt}, }, - validated, warn, Err, Result, + validated, warn, }; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{EventId, OwnedEventId, RoomId}; use self::data::Data; -use crate::{rooms, rooms::short::ShortEventId, Dep}; +use crate::{Dep, rooms, rooms::short::ShortEventId}; pub struct Service { services: Services, diff --git a/src/service/rooms/directory/mod.rs b/src/service/rooms/directory/mod.rs index 039efca7..4ea10641 100644 --- a/src/service/rooms/directory/mod.rs +++ b/src/service/rooms/directory/mod.rs @@ -1,9 +1,9 @@ use std::sync::Arc; -use conduwuit::{implement, utils::stream::TryIgnore, Result}; +use conduwuit::{Result, implement, utils::stream::TryIgnore}; use database::Map; use futures::Stream; -use ruma::{api::client::room::Visibility, RoomId}; +use ruma::{RoomId, api::client::room::Visibility}; pub struct Service { db: Data, diff --git a/src/service/rooms/event_handler/acl_check.rs b/src/service/rooms/event_handler/acl_check.rs index 714b6fc1..6b432a4b 100644 --- a/src/service/rooms/event_handler/acl_check.rs +++ b/src/service/rooms/event_handler/acl_check.rs @@ -1,7 +1,7 @@ -use conduwuit::{debug, implement, trace, warn, Err, Result}; +use conduwuit::{Err, Result, debug, implement, trace, warn}; use ruma::{ - events::{room::server_acl::RoomServerAclEventContent, StateEventType}, RoomId, ServerName, + events::{StateEventType, room::server_acl::RoomServerAclEventContent}, }; /// Returns Ok if the acl allows the server diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 540ebb64..80e91eff 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -1,16 +1,16 @@ use std::{ - collections::{hash_map, BTreeMap, HashSet, VecDeque}, + collections::{BTreeMap, HashSet, VecDeque, hash_map}, sync::Arc, time::Instant, }; use conduwuit::{ - debug, debug_error, debug_warn, implement, pdu, trace, - utils::continue_exponential_backoff_secs, warn, PduEvent, + PduEvent, debug, debug_error, debug_warn, implement, pdu, trace, + utils::continue_exponential_backoff_secs, warn, }; use futures::TryFutureExt; use ruma::{ - api::federation::event::get_event, CanonicalJsonValue, OwnedEventId, RoomId, ServerName, + CanonicalJsonValue, OwnedEventId, RoomId, ServerName, api::federation::event::get_event, }; use super::get_room_version_id; @@ -138,12 +138,15 @@ pub(super) async fn fetch_and_handle_outliers<'a>( .and_then(CanonicalJsonValue::as_array) { for auth_event in auth_events { - if let Ok(auth_event) = - serde_json::from_value::(auth_event.clone().into()) - { - todo_auth_events.push_back(auth_event); - } else { - warn!("Auth event id is not valid"); + match serde_json::from_value::( + auth_event.clone().into(), + ) { + | Ok(auth_event) => { + todo_auth_events.push_back(auth_event); + }, + | _ => { + warn!("Auth event id is not valid"); + }, } } } else { diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 5a38f7fe..e817430b 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -4,14 +4,13 @@ use std::{ }; use conduwuit::{ - debug_warn, err, implement, + PduEvent, Result, debug_warn, err, implement, state_res::{self}, - PduEvent, Result, }; -use futures::{future, FutureExt}; +use futures::{FutureExt, future}; use ruma::{ - int, uint, CanonicalJsonValue, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName, - UInt, + CanonicalJsonValue, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName, UInt, int, + uint, }; use super::check_room_id; @@ -43,54 +42,59 @@ pub(super) async fn fetch_prev( while let Some(prev_event_id) = todo_outlier_stack.pop_front() { self.services.server.check_running()?; - if let Some((pdu, mut json_opt)) = self + match self .fetch_and_handle_outliers(origin, &[prev_event_id.clone()], create_event, room_id) .boxed() .await .pop() { - check_room_id(room_id, &pdu)?; + | Some((pdu, mut json_opt)) => { + check_room_id(room_id, &pdu)?; - let limit = self.services.server.config.max_fetch_prev_events; - if amount > limit { - debug_warn!("Max prev event limit reached! Limit: {limit}"); - graph.insert(prev_event_id.clone(), HashSet::new()); - continue; - } - - if json_opt.is_none() { - json_opt = self - .services - .outlier - .get_outlier_pdu_json(&prev_event_id) - .await - .ok(); - } - - if let Some(json) = json_opt { - if pdu.origin_server_ts > first_ts_in_room { - amount = amount.saturating_add(1); - for prev_prev in &pdu.prev_events { - if !graph.contains_key(prev_prev) { - todo_outlier_stack.push_back(prev_prev.clone()); - } - } - - graph - .insert(prev_event_id.clone(), pdu.prev_events.iter().cloned().collect()); - } else { - // Time based check failed + let limit = self.services.server.config.max_fetch_prev_events; + if amount > limit { + debug_warn!("Max prev event limit reached! Limit: {limit}"); graph.insert(prev_event_id.clone(), HashSet::new()); + continue; } - eventid_info.insert(prev_event_id.clone(), (pdu, json)); - } else { - // Get json failed, so this was not fetched over federation + if json_opt.is_none() { + json_opt = self + .services + .outlier + .get_outlier_pdu_json(&prev_event_id) + .await + .ok(); + } + + if let Some(json) = json_opt { + if pdu.origin_server_ts > first_ts_in_room { + amount = amount.saturating_add(1); + for prev_prev in &pdu.prev_events { + if !graph.contains_key(prev_prev) { + todo_outlier_stack.push_back(prev_prev.clone()); + } + } + + graph.insert( + prev_event_id.clone(), + pdu.prev_events.iter().cloned().collect(), + ); + } else { + // Time based check failed + graph.insert(prev_event_id.clone(), HashSet::new()); + } + + eventid_info.insert(prev_event_id.clone(), (pdu, json)); + } else { + // Get json failed, so this was not fetched over federation + graph.insert(prev_event_id.clone(), HashSet::new()); + } + }, + | _ => { + // Fetch and handle failed graph.insert(prev_event_id.clone(), HashSet::new()); - } - } else { - // Fetch and handle failed - graph.insert(prev_event_id.clone(), HashSet::new()); + }, } } diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 4f2580db..b1a4a38b 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,10 +1,10 @@ -use std::collections::{hash_map, HashMap}; +use std::collections::{HashMap, hash_map}; -use conduwuit::{debug, debug_warn, implement, Err, Error, PduEvent, Result}; +use conduwuit::{Err, Error, PduEvent, Result, debug, debug_warn, implement}; use futures::FutureExt; use ruma::{ - api::federation::event::get_room_state_ids, events::StateEventType, EventId, OwnedEventId, - RoomId, ServerName, + EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids, + events::StateEventType, }; use crate::rooms::short::ShortStateKey; diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 31c7762d..b6d3e21e 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -1,14 +1,14 @@ use std::{ - collections::{hash_map, BTreeMap}, + collections::{BTreeMap, hash_map}, time::Instant, }; -use conduwuit::{debug, debug::INFO_SPAN_LEVEL, err, implement, warn, Err, Result}; +use conduwuit::{Err, Result, debug, debug::INFO_SPAN_LEVEL, err, implement, warn}; use futures::{ - future::{try_join5, OptionFuture}, FutureExt, + future::{OptionFuture, try_join5}, }; -use ruma::{events::StateEventType, CanonicalJsonValue, EventId, RoomId, ServerName, UserId}; +use ruma::{CanonicalJsonValue, EventId, RoomId, ServerName, UserId, events::StateEventType}; use crate::rooms::timeline::RawPduId; diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index e628c77a..974eb300 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -1,15 +1,15 @@ use std::{ - collections::{hash_map, BTreeMap, HashMap}, + collections::{BTreeMap, HashMap, hash_map}, sync::Arc, }; use conduwuit::{ - debug, debug_info, err, implement, state_res, trace, warn, Err, Error, PduEvent, Result, + Err, Error, PduEvent, Result, debug, debug_info, err, implement, state_res, trace, warn, }; -use futures::{future::ready, TryFutureExt}; +use futures::{TryFutureExt, future::ready}; use ruma::{ - api::client::error::ErrorKind, events::StateEventType, CanonicalJsonObject, - CanonicalJsonValue, EventId, RoomId, ServerName, + CanonicalJsonObject, CanonicalJsonValue, EventId, RoomId, ServerName, + api::client::error::ErrorKind, events::StateEventType, }; use super::{check_room_id, get_room_version_id, to_room_version}; diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index f911f1fd..cf69a515 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -5,8 +5,8 @@ use std::{ }; use conduwuit::{ - debug, debug::INFO_SPAN_LEVEL, implement, utils::continue_exponential_backoff_secs, Err, - PduEvent, Result, + Err, PduEvent, Result, debug, debug::INFO_SPAN_LEVEL, implement, + utils::continue_exponential_backoff_secs, }; use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, UInt}; diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 5960c734..e9e79ce4 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -18,16 +18,16 @@ use std::{ }; use conduwuit::{ - utils::{MutexMap, TryFutureExtExt}, Err, PduEvent, Result, RoomVersion, Server, + utils::{MutexMap, TryFutureExtExt}, }; use futures::TryFutureExt; use ruma::{ - events::room::create::RoomCreateEventContent, OwnedEventId, OwnedRoomId, RoomId, - RoomVersionId, + OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, + events::room::create::RoomCreateEventContent, }; -use crate::{globals, rooms, sending, server_keys, Dep}; +use crate::{Dep, globals, rooms, sending, server_keys}; pub struct Service { pub mutex_federation: RoomMutexMap, diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index 9b130763..a49fc541 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,4 +1,4 @@ -use conduwuit::{err, implement, pdu::gen_event_id_canonical_json, result::FlatOk, Result}; +use conduwuit::{Result, err, implement, pdu::gen_event_id_canonical_json, result::FlatOk}; use ruma::{CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId}; use serde_json::value::RawValue as RawJsonValue; diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 37d47d47..9033c3a8 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -5,13 +5,12 @@ use std::{ }; use conduwuit::{ - err, implement, + Error, Result, err, implement, state_res::{self, StateMap}, trace, - utils::stream::{automatic_width, IterStream, ReadyExt, TryWidebandExt, WidebandExt}, - Error, Result, + utils::stream::{IterStream, ReadyExt, TryWidebandExt, WidebandExt, automatic_width}, }; -use futures::{future::try_join, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, future::try_join}; use ruma::{OwnedEventId, RoomId, RoomVersionId}; use crate::rooms::state_compressor::CompressedState; @@ -93,11 +92,7 @@ pub async fn resolve_state( let new_room_state: CompressedState = self .services .state_compressor - .compress_state_events( - state_events - .iter() - .map(|(ref ssk, eid)| (ssk, (*eid).borrow())), - ) + .compress_state_events(state_events.iter().map(|(ssk, eid)| (ssk, (*eid).borrow()))) .collect() .await; diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 2eb6013a..8326f9da 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -6,11 +6,10 @@ use std::{ }; use conduwuit::{ - debug, err, implement, trace, + PduEvent, Result, StateMap, debug, err, implement, trace, utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryWidebandExt}, - PduEvent, Result, StateMap, }; -use futures::{future::try_join, FutureExt, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, future::try_join}; use ruma::{OwnedEventId, RoomId, RoomVersionId}; use crate::rooms::short::ShortStateHash; diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 385d2142..c1a1c3eb 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -1,12 +1,13 @@ use std::{borrow::Borrow, collections::BTreeMap, iter::once, sync::Arc, time::Instant}; use conduwuit::{ - debug, debug_info, err, implement, state_res, trace, + Err, EventTypeExt, PduEvent, Result, StateKey, debug, debug_info, err, implement, state_res, + trace, utils::stream::{BroadbandExt, ReadyExt}, - warn, Err, EventTypeExt, PduEvent, Result, StateKey, + warn, }; -use futures::{future::ready, FutureExt, StreamExt}; -use ruma::{events::StateEventType, CanonicalJsonValue, RoomId, ServerName}; +use futures::{FutureExt, StreamExt, future::ready}; +use ruma::{CanonicalJsonValue, RoomId, ServerName, events::StateEventType}; use super::{get_room_version_id, to_room_version}; use crate::rooms::{ diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index a6e00271..346314d1 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -3,13 +3,12 @@ use std::{collections::HashSet, sync::Arc}; use conduwuit::{ - implement, - utils::{stream::TryIgnore, IterStream, ReadyExt}, - Result, + Result, implement, + utils::{IterStream, ReadyExt, stream::TryIgnore}, }; use database::{Database, Deserialized, Handle, Interfix, Map, Qry}; -use futures::{pin_mut, Stream, StreamExt}; -use ruma::{api::client::filter::LazyLoadOptions, DeviceId, OwnedUserId, RoomId, UserId}; +use futures::{Stream, StreamExt, pin_mut}; +use ruma::{DeviceId, OwnedUserId, RoomId, UserId, api::client::filter::LazyLoadOptions}; pub struct Service { db: Data, diff --git a/src/service/rooms/metadata/mod.rs b/src/service/rooms/metadata/mod.rs index 6d5a85a0..54eef47d 100644 --- a/src/service/rooms/metadata/mod.rs +++ b/src/service/rooms/metadata/mod.rs @@ -1,11 +1,11 @@ use std::sync::Arc; -use conduwuit::{implement, utils::stream::TryIgnore, Result}; +use conduwuit::{Result, implement, utils::stream::TryIgnore}; use database::Map; use futures::{Stream, StreamExt}; use ruma::RoomId; -use crate::{rooms, Dep}; +use crate::{Dep, rooms}; pub struct Service { db: Data, diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 9cd3d805..a1b0263a 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use database::{Deserialized, Json, Map}; use ruma::{CanonicalJsonObject, EventId}; diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index 26e11ded..f0beab5a 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,25 +1,25 @@ use std::{mem::size_of, sync::Arc}; use conduwuit::{ + PduCount, PduEvent, arrayvec::ArrayVec, result::LogErr, utils::{ + ReadyExt, stream::{TryIgnore, WidebandExt}, - u64_from_u8, ReadyExt, + u64_from_u8, }, - PduCount, PduEvent, }; use database::Map; use futures::{Stream, StreamExt}; -use ruma::{api::Direction, EventId, RoomId, UserId}; +use ruma::{EventId, RoomId, UserId, api::Direction}; use crate::{ - rooms, + Dep, rooms, rooms::{ short::{ShortEventId, ShortRoomId}, timeline::{PduId, RawPduId}, }, - Dep, }; pub(super) struct Data { diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index ba289f9b..18221c2d 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -2,11 +2,11 @@ mod data; use std::sync::Arc; use conduwuit::{PduCount, Result}; -use futures::{future::try_join, StreamExt}; -use ruma::{api::Direction, EventId, RoomId, UserId}; +use futures::{StreamExt, future::try_join}; +use ruma::{EventId, RoomId, UserId, api::Direction}; use self::data::{Data, PdusIterItem}; -use crate::{rooms, Dep}; +use crate::{Dep, rooms}; pub struct Service { services: Services, @@ -81,7 +81,7 @@ impl Service { .collect(); 'limit: while let Some(stack_pdu) = stack.pop() { - let target = match stack_pdu.0 .0 { + let target = match stack_pdu.0.0 { | PduCount::Normal(c) => c, // TODO: Support backfilled relations | PduCount::Backfilled(_) => 0, // This will result in an empty iterator diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index c21ad36c..62f87948 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -1,18 +1,18 @@ use std::sync::Arc; use conduwuit::{ - utils::{stream::TryIgnore, ReadyExt}, Result, + utils::{ReadyExt, stream::TryIgnore}, }; use database::{Deserialized, Json, Map}; use futures::{Stream, StreamExt}; use ruma::{ - events::{receipt::ReceiptEvent, AnySyncEphemeralRoomEvent}, - serde::Raw, CanonicalJsonObject, RoomId, UserId, + events::{AnySyncEphemeralRoomEvent, receipt::ReceiptEvent}, + serde::Raw, }; -use crate::{globals, Dep}; +use crate::{Dep, globals}; pub(super) struct Data { roomuserid_privateread: Arc, diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 2bc21355..d6239aee 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -2,19 +2,19 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduwuit::{debug, err, warn, PduCount, PduId, RawPduId, Result}; -use futures::{try_join, Stream, TryFutureExt}; +use conduwuit::{PduCount, PduId, RawPduId, Result, debug, err, warn}; +use futures::{Stream, TryFutureExt, try_join}; use ruma::{ + OwnedEventId, OwnedUserId, RoomId, UserId, events::{ - receipt::{ReceiptEvent, ReceiptEventContent, Receipts}, AnySyncEphemeralRoomEvent, SyncEphemeralRoomEvent, + receipt::{ReceiptEvent, ReceiptEventContent, Receipts}, }, serde::Raw, - OwnedEventId, OwnedUserId, RoomId, UserId, }; use self::data::{Data, ReceiptItem}; -use crate::{rooms, sending, Dep}; +use crate::{Dep, rooms, sending}; pub struct Service { services: Services, @@ -145,12 +145,14 @@ where let receipt = serde_json::from_str::>( value.json().get(), ); - if let Ok(value) = receipt { - for (event, receipt) in value.content { - json.insert(event, receipt); - } - } else { - debug!("failed to parse receipt: {:?}", receipt); + match receipt { + | Ok(value) => + for (event, receipt) in value.content { + json.insert(event, receipt); + }, + | _ => { + debug!("failed to parse receipt: {:?}", receipt); + }, } } let content = ReceiptEventContent::from_iter(json); diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index cc015237..4100dd75 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,26 +1,24 @@ use std::sync::Arc; use conduwuit::{ + PduCount, PduEvent, Result, arrayvec::ArrayVec, implement, utils::{ - set, + ArrayVecExt, IterStream, ReadyExt, set, stream::{TryIgnore, WidebandExt}, - ArrayVecExt, IterStream, ReadyExt, }, - PduCount, PduEvent, Result, }; -use database::{keyval::Val, Map}; +use database::{Map, keyval::Val}; use futures::{Stream, StreamExt}; -use ruma::{api::client::search::search_events::v3::Criteria, RoomId, UserId}; +use ruma::{RoomId, UserId, api::client::search::search_events::v3::Criteria}; use crate::{ - rooms, + Dep, rooms, rooms::{ short::ShortRoomId, timeline::{PduId, RawPduId}, }, - Dep, }; pub struct Service { @@ -140,7 +138,7 @@ pub async fn search_pdus<'a>( pub async fn search_pdu_ids( &self, query: &RoomQuery<'_>, -) -> Result + Send + '_> { +) -> Result + Send + '_ + use<'_>> { let shortroomid = self.services.short.get_shortroomid(query.room_id).await?; let pdu_ids = self.search_pdu_ids_query_room(query, shortroomid).await; @@ -187,7 +185,7 @@ fn search_pdu_ids_query_word( &self, shortroomid: ShortRoomId, word: &str, -) -> impl Stream> + Send + '_ { +) -> impl Stream> + Send + '_ + use<'_> { // rustc says const'ing this not yet stable let end_id: RawPduId = PduId { shortroomid, diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 8728325a..3980617e 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,13 +1,13 @@ use std::{borrow::Borrow, fmt::Debug, mem::size_of_val, sync::Arc}; pub use conduwuit::pdu::{ShortEventId, ShortId, ShortRoomId, ShortStateKey}; -use conduwuit::{err, implement, utils, utils::IterStream, Result, StateKey}; +use conduwuit::{Result, StateKey, err, implement, utils, utils::IterStream}; use database::{Deserialized, Get, Map, Qry}; use futures::{Stream, StreamExt}; -use ruma::{events::StateEventType, EventId, RoomId}; +use ruma::{EventId, RoomId, events::StateEventType}; use serde::Deserialize; -use crate::{globals, Dep}; +use crate::{Dep, globals}; pub struct Service { db: Data, diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 268d6dfe..52e7d2be 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -5,18 +5,18 @@ mod tests; use std::sync::Arc; use conduwuit::{ - implement, + Err, Error, Result, implement, utils::{ + IterStream, future::BoolExt, math::usize_from_f64, stream::{BroadbandExt, ReadyExt}, - IterStream, }, - Err, Error, Result, }; -use futures::{pin_mut, stream::FuturesUnordered, FutureExt, Stream, StreamExt, TryFutureExt}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, pin_mut, stream::FuturesUnordered}; use lru_cache::LruCache; use ruma::{ + OwnedEventId, OwnedRoomId, OwnedServerName, RoomId, ServerName, UserId, api::{ client::space::SpaceHierarchyRoomsChunk, federation::{ @@ -25,18 +25,17 @@ use ruma::{ }, }, events::{ + StateEventType, room::join_rules::{JoinRule, RoomJoinRulesEventContent}, space::child::{HierarchySpaceChildEvent, SpaceChildEventContent}, - StateEventType, }, serde::Raw, space::SpaceRoomJoinRule, - OwnedEventId, OwnedRoomId, OwnedServerName, RoomId, ServerName, UserId, }; use tokio::sync::{Mutex, MutexGuard}; pub use self::pagination_token::PaginationToken; -use crate::{conduwuit::utils::TryFutureExtExt, rooms, sending, Dep}; +use crate::{Dep, conduwuit::utils::TryFutureExtExt, rooms, sending}; pub struct Service { services: Services, @@ -440,8 +439,9 @@ async fn is_accessible_child( pub fn get_parent_children_via( parent: &SpaceHierarchyParentSummary, suggested_only: bool, -) -> impl DoubleEndedIterator)> + Send + '_ -{ +) -> impl DoubleEndedIterator + use<>)> ++ Send ++ '_ { parent .children_state .iter() diff --git a/src/service/rooms/spaces/pagination_token.rs b/src/service/rooms/spaces/pagination_token.rs index 8f019e8d..d97b7a2f 100644 --- a/src/service/rooms/spaces/pagination_token.rs +++ b/src/service/rooms/spaces/pagination_token.rs @@ -4,7 +4,7 @@ use std::{ }; use conduwuit::{Error, Result}; -use ruma::{api::client::error::ErrorKind, UInt}; +use ruma::{UInt, api::client::error::ErrorKind}; use crate::rooms::short::ShortRoomId; diff --git a/src/service/rooms/spaces/tests.rs b/src/service/rooms/spaces/tests.rs index dd6c2f35..d0395fdd 100644 --- a/src/service/rooms/spaces/tests.rs +++ b/src/service/rooms/spaces/tests.rs @@ -1,13 +1,13 @@ use std::str::FromStr; use ruma::{ + UInt, api::federation::space::{SpaceHierarchyParentSummary, SpaceHierarchyParentSummaryInit}, owned_room_id, owned_server_name, space::SpaceRoomJoinRule, - UInt, }; -use crate::rooms::spaces::{get_parent_children_via, PaginationToken}; +use crate::rooms::spaces::{PaginationToken, get_parent_children_via}; #[test] fn get_summary_children() { diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index d538de3c..8683a3be 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -1,36 +1,34 @@ use std::{collections::HashMap, fmt::Write, iter::once, sync::Arc}; use conduwuit::{ - err, + PduEvent, Result, err, result::FlatOk, state_res::{self, StateMap}, utils::{ - calculate_hash, + IterStream, MutexMap, MutexMapGuard, ReadyExt, calculate_hash, stream::{BroadbandExt, TryIgnore}, - IterStream, MutexMap, MutexMapGuard, ReadyExt, }, - warn, PduEvent, Result, + warn, }; use database::{Deserialized, Ignore, Interfix, Map}; use futures::{ - future::join_all, pin_mut, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, + FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, future::join_all, pin_mut, }; use ruma::{ + EventId, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, UserId, events::{ - room::{create::RoomCreateEventContent, member::RoomMemberEventContent}, AnyStrippedStateEvent, StateEventType, TimelineEventType, + room::{create::RoomCreateEventContent, member::RoomMemberEventContent}, }, serde::Raw, - EventId, OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, UserId, }; use crate::{ - globals, rooms, + Dep, globals, rooms, rooms::{ short::{ShortEventId, ShortStateHash}, - state_compressor::{parse_compressed_state_event, CompressedState}, + state_compressor::{CompressedState, parse_compressed_state_event}, }, - Dep, }; pub struct Service { @@ -192,13 +190,13 @@ impl Service { .await; if !already_existed { - let states_parents = if let Ok(p) = previous_shortstatehash { - self.services - .state_compressor - .load_shortstatehash_info(p) - .await? - } else { - Vec::new() + let states_parents = match previous_shortstatehash { + | Ok(p) => + self.services + .state_compressor + .load_shortstatehash_info(p) + .await?, + | _ => Vec::new(), }; let (statediffnew, statediffremoved) = @@ -256,63 +254,65 @@ impl Service { .aput::(shorteventid, p); } - if let Some(state_key) = &new_pdu.state_key { - let states_parents = if let Ok(p) = previous_shortstatehash { - self.services + match &new_pdu.state_key { + | Some(state_key) => { + let states_parents = match previous_shortstatehash { + | Ok(p) => + self.services + .state_compressor + .load_shortstatehash_info(p) + .await?, + | _ => Vec::new(), + }; + + let shortstatekey = self + .services + .short + .get_or_create_shortstatekey(&new_pdu.kind.to_string().into(), state_key) + .await; + + let new = self + .services .state_compressor - .load_shortstatehash_info(p) - .await? - } else { - Vec::new() - }; + .compress_state_event(shortstatekey, &new_pdu.event_id) + .await; - let shortstatekey = self - .services - .short - .get_or_create_shortstatekey(&new_pdu.kind.to_string().into(), state_key) - .await; + let replaces = states_parents + .last() + .map(|info| { + info.full_state + .iter() + .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) + }) + .unwrap_or_default(); - let new = self - .services - .state_compressor - .compress_state_event(shortstatekey, &new_pdu.event_id) - .await; + if Some(&new) == replaces { + return Ok(previous_shortstatehash.expect("must exist")); + } - let replaces = states_parents - .last() - .map(|info| { - info.full_state - .iter() - .find(|bytes| bytes.starts_with(&shortstatekey.to_be_bytes())) - }) - .unwrap_or_default(); + // TODO: statehash with deterministic inputs + let shortstatehash = self.services.globals.next_count()?; - if Some(&new) == replaces { - return Ok(previous_shortstatehash.expect("must exist")); - } + let mut statediffnew = CompressedState::new(); + statediffnew.insert(new); - // TODO: statehash with deterministic inputs - let shortstatehash = self.services.globals.next_count()?; + let mut statediffremoved = CompressedState::new(); + if let Some(replaces) = replaces { + statediffremoved.insert(*replaces); + } - let mut statediffnew = CompressedState::new(); - statediffnew.insert(new); + self.services.state_compressor.save_state_from_diff( + shortstatehash, + Arc::new(statediffnew), + Arc::new(statediffremoved), + 2, + states_parents, + )?; - let mut statediffremoved = CompressedState::new(); - if let Some(replaces) = replaces { - statediffremoved.insert(*replaces); - } - - self.services.state_compressor.save_state_from_diff( - shortstatehash, - Arc::new(statediffnew), - Arc::new(statediffremoved), - 2, - states_parents, - )?; - - Ok(shortstatehash) - } else { - Ok(previous_shortstatehash.expect("first event in room must be a state event")) + Ok(shortstatehash) + }, + | _ => + Ok(previous_shortstatehash.expect("first event in room must be a state event")), } } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index b7952ce6..7004e35a 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -9,14 +9,16 @@ use std::{ }; use conduwuit::{ - err, utils, - utils::math::{usize_from_f64, Expected}, - Result, + Result, err, utils, + utils::math::{Expected, usize_from_f64}, }; use database::Map; use lru_cache::LruCache; use ruma::{ + EventEncryptionAlgorithm, JsOption, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, + OwnedUserId, RoomId, UserId, events::{ + StateEventType, room::{ avatar::RoomAvatarEventContent, canonical_alias::RoomCanonicalAliasEventContent, @@ -29,15 +31,12 @@ use ruma::{ name::RoomNameEventContent, topic::RoomTopicEventContent, }, - StateEventType, }, room::RoomType, space::SpaceRoomJoinRule, - EventEncryptionAlgorithm, JsOption, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, - OwnedUserId, RoomId, UserId, }; -use crate::{rooms, rooms::short::ShortStateHash, Dep}; +use crate::{Dep, rooms, rooms::short::ShortStateHash}; pub struct Service { pub server_visibility_cache: Mutex>, diff --git a/src/service/rooms/state_accessor/room_state.rs b/src/service/rooms/state_accessor/room_state.rs index e3ec55fe..ff26b33a 100644 --- a/src/service/rooms/state_accessor/room_state.rs +++ b/src/service/rooms/state_accessor/room_state.rs @@ -1,8 +1,8 @@ use std::borrow::Borrow; -use conduwuit::{err, implement, PduEvent, Result, StateKey}; +use conduwuit::{PduEvent, Result, StateKey, err, implement}; use futures::{Stream, StreamExt, TryFutureExt}; -use ruma::{events::StateEventType, EventId, RoomId}; +use ruma::{EventId, RoomId, events::StateEventType}; use serde::Deserialize; /// Returns a single PDU from `room_id` with key (`event_type`,`state_key`). diff --git a/src/service/rooms/state_accessor/server_can.rs b/src/service/rooms/state_accessor/server_can.rs index 4d834227..2e8f3325 100644 --- a/src/service/rooms/state_accessor/server_can.rs +++ b/src/service/rooms/state_accessor/server_can.rs @@ -1,11 +1,11 @@ use conduwuit::{error, implement, utils::stream::ReadyExt}; use futures::StreamExt; use ruma::{ - events::{ - room::history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, - StateEventType, - }, EventId, RoomId, ServerName, + events::{ + StateEventType, + room::history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, + }, }; /// Whether a server is allowed to see an event through federation, based on diff --git a/src/service/rooms/state_accessor/state.rs b/src/service/rooms/state_accessor/state.rs index da1500cb..625defe6 100644 --- a/src/service/rooms/state_accessor/state.rs +++ b/src/service/rooms/state_accessor/state.rs @@ -1,27 +1,26 @@ use std::{borrow::Borrow, ops::Deref, sync::Arc}; use conduwuit::{ - at, err, implement, pair_of, + PduEvent, Result, StateKey, at, err, implement, pair_of, utils::{ result::FlatOk, stream::{BroadbandExt, IterStream, ReadyExt, TryExpect}, }, - PduEvent, Result, StateKey, }; use database::Deserialized; -use futures::{future::try_join, pin_mut, FutureExt, Stream, StreamExt, TryFutureExt}; +use futures::{FutureExt, Stream, StreamExt, TryFutureExt, future::try_join, pin_mut}; use ruma::{ - events::{ - room::member::{MembershipState, RoomMemberEventContent}, - StateEventType, - }, EventId, OwnedEventId, UserId, + events::{ + StateEventType, + room::member::{MembershipState, RoomMemberEventContent}, + }, }; use serde::Deserialize; use crate::rooms::{ short::{ShortEventId, ShortStateHash, ShortStateKey}, - state_compressor::{compress_state_event, parse_compressed_state_event, CompressedState}, + state_compressor::{CompressedState, compress_state_event, parse_compressed_state_event}, }; /// The user was a joined member at this state (potentially in the past) diff --git a/src/service/rooms/state_accessor/user_can.rs b/src/service/rooms/state_accessor/user_can.rs index 0332c227..c30e1da8 100644 --- a/src/service/rooms/state_accessor/user_can.rs +++ b/src/service/rooms/state_accessor/user_can.rs @@ -1,14 +1,14 @@ -use conduwuit::{error, implement, pdu::PduBuilder, Err, Error, Result}; +use conduwuit::{Err, Error, Result, error, implement, pdu::PduBuilder}; use ruma::{ + EventId, RoomId, UserId, events::{ + StateEventType, TimelineEventType, room::{ history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, member::{MembershipState, RoomMemberEventContent}, power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, }, - StateEventType, TimelineEventType, }, - EventId, RoomId, UserId, }; use crate::rooms::state::RoomMutexGuard; @@ -44,7 +44,7 @@ pub async fn user_can_redact( ))); } - if let Ok(pl_event_content) = self + match self .room_state_get_content::( room_id, &StateEventType::RoomPowerLevels, @@ -52,33 +52,35 @@ pub async fn user_can_redact( ) .await { - let pl_event: RoomPowerLevels = pl_event_content.into(); - Ok(pl_event.user_can_redact_event_of_other(sender) - || pl_event.user_can_redact_own_event(sender) - && if let Ok(redacting_event) = redacting_event { - if federation { - redacting_event.sender.server_name() == sender.server_name() - } else { - redacting_event.sender == sender - } - } else { - false - }) - } else { - // Falling back on m.room.create to judge power level - if let Ok(room_create) = self - .room_state_get(room_id, &StateEventType::RoomCreate, "") - .await - { - Ok(room_create.sender == sender - || redacting_event - .as_ref() - .is_ok_and(|redacting_event| redacting_event.sender == sender)) - } else { - Err(Error::bad_database( - "No m.room.power_levels or m.room.create events in database for room", - )) - } + | Ok(pl_event_content) => { + let pl_event: RoomPowerLevels = pl_event_content.into(); + Ok(pl_event.user_can_redact_event_of_other(sender) + || pl_event.user_can_redact_own_event(sender) + && match redacting_event { + | Ok(redacting_event) => + if federation { + redacting_event.sender.server_name() == sender.server_name() + } else { + redacting_event.sender == sender + }, + | _ => false, + }) + }, + | _ => { + // Falling back on m.room.create to judge power level + match self + .room_state_get(room_id, &StateEventType::RoomCreate, "") + .await + { + | Ok(room_create) => Ok(room_create.sender == sender + || redacting_event + .as_ref() + .is_ok_and(|redacting_event| redacting_event.sender == sender)), + | _ => Err(Error::bad_database( + "No m.room.power_levels or m.room.create events in database for room", + )), + } + }, } } diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 0d25142d..4403468b 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -4,31 +4,31 @@ use std::{ }; use conduwuit::{ - is_not_empty, + Result, is_not_empty, result::LogErr, - utils::{stream::TryIgnore, ReadyExt, StreamTools}, - warn, Result, + utils::{ReadyExt, StreamTools, stream::TryIgnore}, + warn, }; -use database::{serialize_key, Deserialized, Ignore, Interfix, Json, Map}; -use futures::{future::join5, pin_mut, stream::iter, Stream, StreamExt}; +use database::{Deserialized, Ignore, Interfix, Json, Map, serialize_key}; +use futures::{Stream, StreamExt, future::join5, pin_mut, stream::iter}; use itertools::Itertools; use ruma::{ + OwnedRoomId, OwnedServerName, RoomId, ServerName, UserId, events::{ + AnyStrippedStateEvent, AnySyncStateEvent, GlobalAccountDataEventType, + RoomAccountDataEventType, StateEventType, direct::DirectEvent, room::{ create::RoomCreateEventContent, member::{MembershipState, RoomMemberEventContent}, power_levels::RoomPowerLevelsEventContent, }, - AnyStrippedStateEvent, AnySyncStateEvent, GlobalAccountDataEventType, - RoomAccountDataEventType, StateEventType, }, int, serde::Raw, - OwnedRoomId, OwnedServerName, RoomId, ServerName, UserId, }; -use crate::{account_data, appservice::RegistrationInfo, globals, rooms, users, Dep}; +use crate::{Dep, account_data, appservice::RegistrationInfo, globals, rooms, users}; pub struct Service { appservice_in_room_cache: AppServiceInRoomCache, diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 18731809..c566eb1c 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -6,10 +6,10 @@ use std::{ }; use conduwuit::{ + Result, arrayvec::ArrayVec, at, checked, err, expected, utils, utils::{bytes, math::usize_from_f64, stream::IterStream}, - Result, }; use database::Map; use futures::{Stream, StreamExt}; @@ -17,9 +17,8 @@ use lru_cache::LruCache; use ruma::{EventId, RoomId}; use crate::{ - rooms, + Dep, rooms, rooms::short::{ShortEventId, ShortId, ShortStateHash, ShortStateKey}, - Dep, }; pub struct Service { diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index bc995e27..7f9a7515 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -1,22 +1,21 @@ use std::{collections::BTreeMap, sync::Arc}; use conduwuit::{ - err, + PduCount, PduEvent, PduId, RawPduId, Result, err, utils::{ - stream::{TryIgnore, WidebandExt}, ReadyExt, + stream::{TryIgnore, WidebandExt}, }, - PduCount, PduEvent, PduId, RawPduId, Result, }; use database::{Deserialized, Map}; use futures::{Stream, StreamExt}; use ruma::{ - api::client::threads::get_threads::v1::IncludeThreads, events::relation::BundledThread, uint, CanonicalJsonValue, EventId, OwnedUserId, RoomId, UserId, + api::client::threads::get_threads::v1::IncludeThreads, events::relation::BundledThread, uint, }; use serde_json::json; -use crate::{rooms, rooms::short::ShortRoomId, Dep}; +use crate::{Dep, rooms, rooms::short::ShortRoomId}; pub struct Service { db: Data, @@ -121,10 +120,13 @@ impl Service { } let mut users = Vec::new(); - if let Ok(userids) = self.get_participants(&root_id).await { - users.extend_from_slice(&userids); - } else { - users.push(root_pdu.sender); + match self.get_participants(&root_id).await { + | Ok(userids) => { + users.extend_from_slice(&userids); + }, + | _ => { + users.push(root_pdu.sender); + }, } users.push(pdu.sender.clone()); diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 457c1e8d..94c78bb0 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -1,18 +1,17 @@ use std::{borrow::Borrow, sync::Arc}; use conduwuit::{ - at, err, + Err, PduCount, PduEvent, Result, at, err, result::{LogErr, NotFound}, utils, utils::stream::TryReadyExt, - Err, PduCount, PduEvent, Result, }; use database::{Database, Deserialized, Json, KeyVal, Map}; -use futures::{future::select_ok, pin_mut, FutureExt, Stream, TryFutureExt, TryStreamExt}; -use ruma::{api::Direction, CanonicalJsonObject, EventId, OwnedUserId, RoomId, UserId}; +use futures::{FutureExt, Stream, TryFutureExt, TryStreamExt, future::select_ok, pin_mut}; +use ruma::{CanonicalJsonObject, EventId, OwnedUserId, RoomId, UserId, api::Direction}; use super::{PduId, RawPduId}; -use crate::{rooms, rooms::short::ShortRoomId, Dep}; +use crate::{Dep, rooms, rooms::short::ShortRoomId}; pub(super) struct Data { eventid_outlierpdu: Arc, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 9d6ee982..4be97fb2 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -10,22 +10,25 @@ use std::{ }; use conduwuit::{ - at, debug, debug_warn, err, error, implement, info, - pdu::{gen_event_id, EventHash, PduBuilder, PduCount, PduEvent}, + Err, Error, Result, Server, at, debug, debug_warn, err, error, implement, info, + pdu::{EventHash, PduBuilder, PduCount, PduEvent, gen_event_id}, state_res::{self, Event, RoomVersion}, utils::{ - self, future::TryExtExt, stream::TryIgnore, IterStream, MutexMap, MutexMapGuard, ReadyExt, + self, IterStream, MutexMap, MutexMapGuard, ReadyExt, future::TryExtExt, stream::TryIgnore, }, - validated, warn, Err, Error, Result, Server, + validated, warn, }; pub use conduwuit::{PduId, RawPduId}; use futures::{ - future, future::ready, pin_mut, Future, FutureExt, Stream, StreamExt, TryStreamExt, + Future, FutureExt, Stream, StreamExt, TryStreamExt, future, future::ready, pin_mut, }; use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, + RoomId, RoomVersionId, ServerName, UserId, api::federation, canonical_json::to_canonical_value, events::{ + GlobalAccountDataEventType, StateEventType, TimelineEventType, push_rules::PushRulesEvent, room::{ create::RoomCreateEventContent, @@ -34,23 +37,21 @@ use ruma::{ power_levels::RoomPowerLevelsEventContent, redaction::RoomRedactionEventContent, }, - GlobalAccountDataEventType, StateEventType, TimelineEventType, }, push::{Action, Ruleset, Tweak}, - uint, CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, - OwnedServerName, RoomId, RoomVersionId, ServerName, UserId, + uint, }; use serde::Deserialize; -use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; use self::data::Data; pub use self::data::PdusIterItem; use crate::{ - account_data, admin, appservice, + Dep, account_data, admin, appservice, appservice::NamespaceRegex, globals, pusher, rooms, rooms::{short::ShortRoomId, state_compressor::CompressedState}, - sending, server_keys, users, Dep, + sending, server_keys, users, }; // Update Relationships diff --git a/src/service/rooms/typing/mod.rs b/src/service/rooms/typing/mod.rs index c710b33a..a81ee95c 100644 --- a/src/service/rooms/typing/mod.rs +++ b/src/service/rooms/typing/mod.rs @@ -1,19 +1,18 @@ use std::{collections::BTreeMap, sync::Arc}; use conduwuit::{ - debug_info, trace, + Result, Server, debug_info, trace, utils::{self, IterStream}, - Result, Server, }; use futures::StreamExt; use ruma::{ + OwnedRoomId, OwnedUserId, RoomId, UserId, api::federation::transactions::edu::{Edu, TypingContent}, events::SyncEphemeralRoomEvent, - OwnedRoomId, OwnedUserId, RoomId, UserId, }; -use tokio::sync::{broadcast, RwLock}; +use tokio::sync::{RwLock, broadcast}; -use crate::{globals, sending, sending::EduBuf, users, Dep}; +use crate::{Dep, globals, sending, sending::EduBuf, users}; pub struct Service { server: Arc, diff --git a/src/service/rooms/user/mod.rs b/src/service/rooms/user/mod.rs index 6a0c6aa1..bd76f1f4 100644 --- a/src/service/rooms/user/mod.rs +++ b/src/service/rooms/user/mod.rs @@ -1,10 +1,10 @@ use std::sync::Arc; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use database::{Database, Deserialized, Map}; use ruma::{RoomId, UserId}; -use crate::{globals, rooms, rooms::short::ShortStateHash, Dep}; +use crate::{Dep, globals, rooms, rooms::short::ShortStateHash}; pub struct Service { db: Data, diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 6b58d964..7fa0be9a 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -1,10 +1,10 @@ use std::{fmt::Debug, mem}; use bytes::BytesMut; -use conduwuit::{debug_error, err, trace, utils, warn, Err, Result}; +use conduwuit::{Err, Result, debug_error, err, trace, utils, warn}; use reqwest::Client; use ruma::api::{ - appservice::Registration, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, + IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, appservice::Registration, }; /// Sends a request to an appservice diff --git a/src/service/sending/data.rs b/src/service/sending/data.rs index 4dd2d5aa..a6bcc2b2 100644 --- a/src/service/sending/data.rs +++ b/src/service/sending/data.rs @@ -1,16 +1,15 @@ use std::{fmt::Debug, sync::Arc}; use conduwuit::{ - at, utils, - utils::{stream::TryIgnore, ReadyExt}, - Error, Result, + Error, Result, at, utils, + utils::{ReadyExt, stream::TryIgnore}, }; use database::{Database, Deserialized, Map}; use futures::{Stream, StreamExt}; use ruma::{OwnedServerName, ServerName, UserId}; use super::{Destination, SendingEvent}; -use crate::{globals, Dep}; +use crate::{Dep, globals}; pub(super) type OutgoingItem = (Key, SendingEvent, Destination); pub(super) type SendingItem = (Key, SendingEvent); @@ -102,7 +101,7 @@ impl Data { pub fn active_requests_for( &self, destination: &Destination, - ) -> impl Stream + Send + '_ { + ) -> impl Stream + Send + '_ + use<'_> { let prefix = destination.get_prefix(); self.servercurrentevent_data .raw_stream_from(&prefix) @@ -156,7 +155,7 @@ impl Data { pub fn queued_requests( &self, destination: &Destination, - ) -> impl Stream + Send + '_ { + ) -> impl Stream + Send + '_ + use<'_> { let prefix = destination.get_prefix(); self.servernameevent_data .raw_stream_from(&prefix) diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index b46ce7a8..379829b4 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -12,15 +12,15 @@ use std::{ use async_trait::async_trait; use conduwuit::{ - debug, debug_warn, err, error, + Result, Server, debug, debug_warn, err, error, smallvec::SmallVec, - utils::{available_parallelism, math::usize_from_u64_truncated, ReadyExt, TryReadyExt}, - warn, Result, Server, + utils::{ReadyExt, TryReadyExt, available_parallelism, math::usize_from_u64_truncated}, + warn, }; use futures::{FutureExt, Stream, StreamExt}; use ruma::{ - api::{appservice::Registration, OutgoingRequest}, RoomId, ServerName, UserId, + api::{OutgoingRequest, appservice::Registration}, }; use tokio::{task, task::JoinSet}; @@ -30,8 +30,8 @@ pub use self::{ sender::{EDU_LIMIT, PDU_LIMIT}, }; use crate::{ - account_data, client, federation, globals, presence, pusher, rooms, - rooms::timeline::RawPduId, users, Dep, + Dep, account_data, client, federation, globals, presence, pusher, rooms, + rooms::timeline::RawPduId, users, }; pub struct Service { diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 3e86de2d..c4f34177 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -2,32 +2,33 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, sync::{ - atomic::{AtomicU64, AtomicUsize, Ordering}, Arc, + atomic::{AtomicU64, AtomicUsize, Ordering}, }, time::{Duration, Instant}, }; -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; use conduwuit::{ - debug, err, error, + Error, Result, debug, err, error, result::LogErr, trace, utils::{ - calculate_hash, continue_exponential_backoff_secs, + ReadyExt, calculate_hash, continue_exponential_backoff_secs, future::TryExtExt, stream::{BroadbandExt, IterStream, WidebandExt}, - ReadyExt, }, - warn, Error, Result, + warn, }; use futures::{ + FutureExt, StreamExt, future::{BoxFuture, OptionFuture}, join, pin_mut, stream::FuturesUnordered, - FutureExt, StreamExt, }; use ruma::{ + CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedServerName, OwnedUserId, + RoomId, RoomVersionId, ServerName, UInt, api::{ appservice::event::push_events::v1::EphemeralData, federation::transactions::{ @@ -40,18 +41,17 @@ use ruma::{ }, device_id, events::{ - push_rules::PushRulesEvent, receipt::ReceiptType, AnySyncEphemeralRoomEvent, - GlobalAccountDataEventType, + AnySyncEphemeralRoomEvent, GlobalAccountDataEventType, push_rules::PushRulesEvent, + receipt::ReceiptType, }, push, serde::Raw, - uint, CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedRoomId, OwnedServerName, - OwnedUserId, RoomId, RoomVersionId, ServerName, UInt, + uint, }; -use serde_json::value::{to_raw_value, RawValue as RawJsonValue}; +use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; use super::{ - appservice, data::QueueItem, Destination, EduBuf, EduVec, Msg, SendingEvent, Service, + Destination, EduBuf, EduVec, Msg, SendingEvent, Service, appservice, data::QueueItem, }; #[derive(Debug)] @@ -146,7 +146,7 @@ impl Service { statuses.entry(dest).and_modify(|e| { *e = match e { | TransactionStatus::Running => TransactionStatus::Failed(1, Instant::now()), - | TransactionStatus::Retrying(ref n) => + | &mut TransactionStatus::Retrying(ref n) => TransactionStatus::Failed(n.saturating_add(1), Instant::now()), | TransactionStatus::Failed(..) => { panic!("Request that was not even running failed?!") @@ -211,7 +211,7 @@ impl Service { async fn finish_responses<'a>(&'a self, futures: &mut SendingFutures<'a>) { use tokio::{ select, - time::{sleep_until, Instant}, + time::{Instant, sleep_until}, }; let timeout = self.server.config.sender_shutdown_timeout; diff --git a/src/service/server_keys/acquire.rs b/src/service/server_keys/acquire.rs index 305cbfef..64b936b6 100644 --- a/src/service/server_keys/acquire.rs +++ b/src/service/server_keys/acquire.rs @@ -7,13 +7,13 @@ use std::{ use conduwuit::{ debug, debug_error, debug_warn, error, implement, info, result::FlatOk, trace, warn, }; -use futures::{stream::FuturesUnordered, StreamExt}; +use futures::{StreamExt, stream::FuturesUnordered}; use ruma::{ - api::federation::discovery::ServerSigningKeys, serde::Raw, CanonicalJsonObject, - OwnedServerName, OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, + CanonicalJsonObject, OwnedServerName, OwnedServerSigningKeyId, ServerName, + ServerSigningKeyId, api::federation::discovery::ServerSigningKeys, serde::Raw, }; use serde_json::value::RawValue as RawJsonValue; -use tokio::time::{timeout_at, Instant}; +use tokio::time::{Instant, timeout_at}; use super::key_exists; diff --git a/src/service/server_keys/get.rs b/src/service/server_keys/get.rs index 5a027d64..00aeae1e 100644 --- a/src/service/server_keys/get.rs +++ b/src/service/server_keys/get.rs @@ -1,12 +1,12 @@ use std::borrow::Borrow; -use conduwuit::{implement, Err, Result}; +use conduwuit::{Err, Result, implement}; use ruma::{ - api::federation::discovery::VerifyKey, CanonicalJsonObject, RoomVersionId, ServerName, - ServerSigningKeyId, + CanonicalJsonObject, RoomVersionId, ServerName, ServerSigningKeyId, + api::federation::discovery::VerifyKey, }; -use super::{extract_key, PubKeyMap, PubKeys}; +use super::{PubKeyMap, PubKeys, extract_key}; #[implement(super::Service)] pub async fn get_event_keys( diff --git a/src/service/server_keys/keypair.rs b/src/service/server_keys/keypair.rs index 6f983c26..259c37fb 100644 --- a/src/service/server_keys/keypair.rs +++ b/src/service/server_keys/keypair.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{debug, debug_info, err, error, utils, utils::string_from_bytes, Result}; +use conduwuit::{Result, debug, debug_info, err, error, utils, utils::string_from_bytes}; use database::Database; use ruma::{api::federation::discovery::VerifyKey, serde::Base64, signatures::Ed25519KeyPair}; diff --git a/src/service/server_keys/mod.rs b/src/service/server_keys/mod.rs index 3f6a3039..bf6799ba 100644 --- a/src/service/server_keys/mod.rs +++ b/src/service/server_keys/mod.rs @@ -8,22 +8,21 @@ mod verify; use std::{collections::BTreeMap, sync::Arc, time::Duration}; use conduwuit::{ - implement, - utils::{timepoint_from_now, IterStream}, - Result, Server, + Result, Server, implement, + utils::{IterStream, timepoint_from_now}, }; use database::{Deserialized, Json, Map}; use futures::StreamExt; use ruma::{ + CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, RoomVersionId, + ServerName, ServerSigningKeyId, api::federation::discovery::{ServerSigningKeys, VerifyKey}, serde::Raw, signatures::{Ed25519KeyPair, PublicKeyMap, PublicKeySet}, - CanonicalJsonObject, MilliSecondsSinceUnixEpoch, OwnedServerSigningKeyId, RoomVersionId, - ServerName, ServerSigningKeyId, }; use serde_json::value::RawValue as RawJsonValue; -use crate::{globals, sending, Dep}; +use crate::{Dep, globals, sending}; pub struct Service { keypair: Box, diff --git a/src/service/server_keys/request.rs b/src/service/server_keys/request.rs index afe8958b..171b755b 100644 --- a/src/service/server_keys/request.rs +++ b/src/service/server_keys/request.rs @@ -1,13 +1,13 @@ use std::{collections::BTreeMap, fmt::Debug}; -use conduwuit::{debug, implement, Err, Result}; +use conduwuit::{Err, Result, debug, implement}; use ruma::{ - api::federation::discovery::{ - get_remote_server_keys, - get_remote_server_keys_batch::{self, v2::QueryCriteria}, - get_server_keys, ServerSigningKeys, - }, OwnedServerName, OwnedServerSigningKeyId, ServerName, ServerSigningKeyId, + api::federation::discovery::{ + ServerSigningKeys, get_remote_server_keys, + get_remote_server_keys_batch::{self, v2::QueryCriteria}, + get_server_keys, + }, }; #[implement(super::Service)] @@ -79,7 +79,7 @@ pub async fn notary_request( &self, notary: &ServerName, target: &ServerName, -) -> Result + Clone + Debug + Send> { +) -> Result + Clone + Debug + Send + use<>> { use get_remote_server_keys::v2::Request; let request = Request { diff --git a/src/service/server_keys/sign.rs b/src/service/server_keys/sign.rs index 8d6f108c..e8cc485d 100644 --- a/src/service/server_keys/sign.rs +++ b/src/service/server_keys/sign.rs @@ -1,4 +1,4 @@ -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use ruma::{CanonicalJsonObject, RoomVersionId}; #[implement(super::Service)] diff --git a/src/service/server_keys/verify.rs b/src/service/server_keys/verify.rs index 0f03e59e..84433628 100644 --- a/src/service/server_keys/verify.rs +++ b/src/service/server_keys/verify.rs @@ -1,6 +1,6 @@ -use conduwuit::{implement, pdu::gen_event_id_canonical_json, Err, Result}; +use conduwuit::{Err, Result, implement, pdu::gen_event_id_canonical_json}; use ruma::{ - signatures::Verified, CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, RoomVersionId, + CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, RoomVersionId, signatures::Verified, }; use serde_json::value::RawValue as RawJsonValue; diff --git a/src/service/service.rs b/src/service/service.rs index cad01437..2907a562 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -7,7 +7,7 @@ use std::{ }; use async_trait::async_trait; -use conduwuit::{err, error::inspect_log, utils::string::SplitInfallible, Err, Result, Server}; +use conduwuit::{Err, Result, Server, err, error::inspect_log, utils::string::SplitInfallible}; use database::Database; /// Abstract interface for a Service diff --git a/src/service/services.rs b/src/service/services.rs index fb334b96..269a1f87 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -5,7 +5,7 @@ use std::{ sync::{Arc, RwLock}, }; -use conduwuit::{debug, debug_info, info, trace, Result, Server}; +use conduwuit::{Result, Server, debug, debug_info, info, trace}; use database::Database; use tokio::sync::Mutex; diff --git a/src/service/sync/mod.rs b/src/service/sync/mod.rs index 0b86377a..bf2bc142 100644 --- a/src/service/sync/mod.rs +++ b/src/service/sync/mod.rs @@ -8,15 +8,15 @@ use std::{ use conduwuit::{Result, Server}; use database::Map; use ruma::{ + DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, UserId, api::client::sync::sync_events::{ self, v4::{ExtensionsConfig, SyncRequestList}, v5, }, - DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, UserId, }; -use crate::{rooms, Dep}; +use crate::{Dep, rooms}; pub struct Service { db: Data, diff --git a/src/service/sync/watch.rs b/src/service/sync/watch.rs index 0a9c5d15..96981472 100644 --- a/src/service/sync/watch.rs +++ b/src/service/sync/watch.rs @@ -1,5 +1,5 @@ -use conduwuit::{implement, trace, Result}; -use futures::{pin_mut, stream::FuturesUnordered, FutureExt, StreamExt}; +use conduwuit::{Result, implement, trace}; +use futures::{FutureExt, StreamExt, pin_mut, stream::FuturesUnordered}; use ruma::{DeviceId, UserId}; #[implement(super::Service)] diff --git a/src/service/transaction_ids/mod.rs b/src/service/transaction_ids/mod.rs index 912c0b49..9c284b70 100644 --- a/src/service/transaction_ids/mod.rs +++ b/src/service/transaction_ids/mod.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{implement, Result}; +use conduwuit::{Result, implement}; use database::{Handle, Map}; use ruma::{DeviceId, TransactionId, UserId}; diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 7084f32a..51f5fb11 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -4,20 +4,19 @@ use std::{ }; use conduwuit::{ - err, error, implement, utils, + Error, Result, err, error, implement, utils, utils::{hash, string::EMPTY}, - Error, Result, }; use database::{Deserialized, Json, Map}; use ruma::{ + CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedUserId, UserId, api::client::{ error::ErrorKind, uiaa::{AuthData, AuthType, Password, UiaaInfo, UserIdentifier}, }, - CanonicalJsonValue, DeviceId, OwnedDeviceId, OwnedUserId, UserId, }; -use crate::{config, globals, users, Dep}; +use crate::{Dep, config, globals, users}; pub struct Service { userdevicesessionid_uiaarequest: RwLock, @@ -144,8 +143,7 @@ pub async fn try_auth( }; #[cfg(not(feature = "element_hacks"))] - let Some(UserIdentifier::UserIdOrLocalpart(username)) = identifier - else { + let Some(UserIdentifier::UserIdOrLocalpart(username)) = identifier else { return Err(Error::BadRequest( ErrorKind::Unrecognized, "Identifier type not recognized.", diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs index 7fd93b6c..28bee65a 100644 --- a/src/service/updates/mod.rs +++ b/src/service/updates/mod.rs @@ -1,16 +1,16 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; -use conduwuit::{debug, info, warn, Result, Server}; +use conduwuit::{Result, Server, debug, info, warn}; use database::{Deserialized, Map}; use ruma::events::room::message::RoomMessageEventContent; use serde::Deserialize; use tokio::{ sync::Notify, - time::{interval, MissedTickBehavior}, + time::{MissedTickBehavior, interval}, }; -use crate::{admin, client, globals, Dep}; +use crate::{Dep, admin, client, globals}; pub struct Service { interval: Duration, diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index f0389a4a..b3f5db88 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,25 +1,24 @@ use std::{collections::BTreeMap, mem, sync::Arc}; use conduwuit::{ - at, debug_warn, err, trace, - utils::{self, stream::TryIgnore, string::Unquoted, ReadyExt}, - Err, Error, Result, Server, + Err, Error, Result, Server, at, debug_warn, err, trace, + utils::{self, ReadyExt, stream::TryIgnore, string::Unquoted}, }; use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{ + DeviceId, KeyId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, + OneTimeKeyName, OwnedDeviceId, OwnedKeyId, OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, api::client::{device::Device, error::ErrorKind, filter::FilterDefinition}, encryption::{CrossSigningKey, DeviceKeys, OneTimeKey}, events::{ - ignored_user_list::IgnoredUserListEvent, AnyToDeviceEvent, GlobalAccountDataEventType, + AnyToDeviceEvent, GlobalAccountDataEventType, ignored_user_list::IgnoredUserListEvent, }, serde::Raw, - DeviceId, KeyId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, - OneTimeKeyName, OwnedDeviceId, OwnedKeyId, OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, }; use serde_json::json; -use crate::{account_data, admin, globals, rooms, Dep}; +use crate::{Dep, account_data, admin, globals, rooms}; pub struct Service { services: Services, @@ -246,10 +245,13 @@ impl Service { /// Sets a new avatar_url or removes it if avatar_url is None. pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option) { - if let Some(avatar_url) = avatar_url { - self.db.userid_avatarurl.insert(user_id, &avatar_url); - } else { - self.db.userid_avatarurl.remove(user_id); + match avatar_url { + | Some(avatar_url) => { + self.db.userid_avatarurl.insert(user_id, &avatar_url); + }, + | _ => { + self.db.userid_avatarurl.remove(user_id); + }, } } From 045e8a293740ba1ee94d93d09d27d07a6c0d67d0 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 23 Feb 2025 02:51:55 -0500 Subject: [PATCH 1521/2091] stop building mac binaries for now because of linker issues Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 126 --------------------------------------- 1 file changed, 126 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 24f2db45..8e1cf6c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -623,132 +623,6 @@ jobs: scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}-debug.tar.gz fi - build_mac_binaries: - name: Build MacOS Binaries - strategy: - matrix: - os: [macos-latest, macos-13] - runs-on: ${{ matrix.os }} - steps: - - name: Sync repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup SSH web publish - env: - web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' - run: | - mkdir -p -v ~/.ssh - - echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts - echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - cat >>~/.ssh/config <> "$GITHUB_ENV" - - - name: Tag comparison check - if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} - run: | - # Tag mismatch with latest repo tag check to prevent potential downgrades - LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) - - if [ ${LATEST_TAG} != ${GH_REF_NAME} ]; then - echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' - echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - # use sccache for Rust - - name: Run sccache-cache - # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting - # releases and tags - #if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} - uses: mozilla-actions/sccache-action@main - - # use rust-cache - - uses: Swatinem/rust-cache@v2 - with: - cache-all-crates: "true" - cache-on-failure: "true" - cache-targets: "true" - - # Nix can't do portable macOS builds yet - - name: Build macOS x86_64 binary - if: ${{ matrix.os == 'macos-13' }} - run: | - CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release --locked --features=perf_measurements,sentry_telemetry,direct_tls - cp -v -f target/release/conduwuit conduwuit-macos-x86_64 - otool -L conduwuit-macos-x86_64 - - # quick smoke test of the x86_64 macOS binary - - name: Run x86_64 macOS release binary - if: ${{ matrix.os == 'macos-13' }} - run: | - ./conduwuit-macos-x86_64 --help - ./conduwuit-macos-x86_64 --version - - - name: Build macOS arm64 binary - if: ${{ matrix.os == 'macos-latest' }} - run: | - CONDUWUIT_VERSION_EXTRA="$(git rev-parse --short ${{ github.sha }})" cargo build --release --locked --features=perf_measurements,sentry_telemetry,direct_tls - cp -v -f target/release/conduwuit conduwuit-macos-arm64 - otool -L conduwuit-macos-arm64 - - # quick smoke test of the arm64 macOS binary - - name: Run arm64 macOS release binary - if: ${{ matrix.os == 'macos-latest' }} - run: | - ./conduwuit-macos-arm64 --help - ./conduwuit-macos-arm64 --version - - - name: Upload macOS x86_64 binary to webserver - if: ${{ matrix.os == 'macos-13' }} - run: | - if [ ! -z $SSH_WEBSITE ]; then - chmod +x conduwuit-macos-x86_64 - scp conduwuit-macos-x86_64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/conduwuit-macos-x86_64 - fi - - - name: Upload macOS arm64 binary to webserver - if: ${{ matrix.os == 'macos-latest' }} - run: | - if [ ! -z $SSH_WEBSITE ]; then - chmod +x conduwuit-macos-arm64 - scp conduwuit-macos-arm64 website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/conduwuit-macos-arm64 - fi - - - name: Upload macOS x86_64 binary - if: ${{ matrix.os == 'macos-13' }} - uses: actions/upload-artifact@v4 - with: - name: conduwuit-macos-x86_64 - path: conduwuit-macos-x86_64 - if-no-files-found: error - - - name: Upload macOS arm64 binary - if: ${{ matrix.os == 'macos-latest' }} - uses: actions/upload-artifact@v4 - with: - name: conduwuit-macos-arm64 - path: conduwuit-macos-arm64 - if-no-files-found: error variables: outputs: github_repository: ${{ steps.var.outputs.github_repository }} From 4bdd0d77db9b4eaa7864431da6c5b19218e18c79 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 23 Feb 2025 02:52:19 -0500 Subject: [PATCH 1522/2091] bump complement, actually run all tests Signed-off-by: June Clementine Strawberry --- bin/complement | 2 +- flake.lock | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/complement b/bin/complement index ffd7a938..4356f2e7 100755 --- a/bin/complement +++ b/bin/complement @@ -45,7 +45,7 @@ set +o pipefail env \ -C "$COMPLEMENT_SRC" \ COMPLEMENT_BASE_IMAGE="$OCI_IMAGE" \ - go test -tags="conduwuit_blacklist" "$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests ./tests/msc3967 | tee "$LOG_FILE" + go test -tags="conduwuit_blacklist" "$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE" set -o pipefail # Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results diff --git a/flake.lock b/flake.lock index 9bf6ac55..a7d80508 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1734303596, - "narHash": "sha256-HjDRyLR4MBqQ3IjfMM6eE+8ayztXlbz3gXdyDmFla68=", + "lastModified": 1740291865, + "narHash": "sha256-wl1+yCTEtvIH8vgXygnxPkaSgg4MYNKs+c9tzVytr20=", "owner": "girlbossceo", "repo": "complement", - "rev": "14cc5be797b774f1a2b9f826f38181066d4952b8", + "rev": "35ad9d9051498fbac8ea4abff8ab7d8b1844f87b", "type": "github" }, "original": { From cbf207bd1f1587418be0de2a1a5cbd745baec9e2 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 23 Feb 2025 03:11:34 -0500 Subject: [PATCH 1523/2091] try adding back some skipped complement tests Signed-off-by: June Clementine Strawberry --- bin/complement | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/complement b/bin/complement index 4356f2e7..9960299c 100755 --- a/bin/complement +++ b/bin/complement @@ -18,7 +18,7 @@ RESULTS_FILE="$3" OCI_IMAGE="complement-conduwuit:main" # Complement tests that are skipped due to flakiness/reliability issues -SKIPPED_COMPLEMENT_TESTS='-skip=TestClientSpacesSummary.*|TestJoinFederatedRoomFromApplicationServiceBridgeUser.*|TestJumpToDateEndpoint.*|TestUnbanViaInvite.*' +SKIPPED_COMPLEMENT_TESTS='-skip=TestPartialStateJoin.*' # $COMPLEMENT_SRC needs to be a directory to Complement source code if [ -f "$COMPLEMENT_SRC" ]; then From a67ab754179d0bbaa09aa19d974035c521643fe9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 25 Feb 2025 18:38:12 +0000 Subject: [PATCH 1524/2091] fix edition 2024 lints Signed-off-by: Jason Volk --- Cargo.toml | 2 ++ src/admin/admin.rs | 2 +- src/admin/room/commands.rs | 2 +- src/admin/room/directory.rs | 2 +- src/admin/user/commands.rs | 6 +++--- src/api/client/account.rs | 2 +- src/api/client/directory.rs | 14 +++----------- src/api/client/membership.rs | 6 +++--- src/api/client/report.rs | 6 +++--- src/api/router/auth.rs | 2 +- src/api/server/send_join.rs | 2 +- src/api/server/send_knock.rs | 2 +- src/database/de.rs | 2 +- src/database/engine/logger.rs | 2 +- src/database/map/compact.rs | 2 +- src/database/map/qry_batch.rs | 1 - src/database/map/rev_stream.rs | 2 +- src/database/map/rev_stream_from.rs | 2 +- src/database/map/stream.rs | 2 +- src/database/map/stream_from.rs | 2 +- src/database/pool.rs | 14 ++++++-------- src/database/ser.rs | 2 +- src/database/stream.rs | 6 +++--- src/database/watchers.rs | 2 +- src/macros/config.rs | 2 +- src/service/media/blurhash.rs | 2 +- src/service/media/remote.rs | 2 +- .../rooms/event_handler/handle_incoming_pdu.rs | 2 +- src/service/rooms/spaces/mod.rs | 2 +- src/service/rooms/state_cache/mod.rs | 4 ++-- src/service/rooms/state_compressor/mod.rs | 4 ++-- src/service/rooms/timeline/mod.rs | 14 +++++++------- src/service/sending/mod.rs | 2 +- src/service/sending/sender.rs | 7 ++----- src/service/server_keys/request.rs | 2 +- src/service/uiaa/mod.rs | 2 +- 36 files changed, 60 insertions(+), 72 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 76de2212..52695d89 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -905,6 +905,7 @@ missing_docs_in_private_items = { level = "allow", priority = 1 } missing_errors_doc = { level = "allow", priority = 1 } missing_panics_doc = { level = "allow", priority = 1 } module_name_repetitions = { level = "allow", priority = 1 } +needless_continue = { level = "allow", priority = 1 } no_effect_underscore_binding = { level = "allow", priority = 1 } similar_names = { level = "allow", priority = 1 } single_match_else = { level = "allow", priority = 1 } @@ -969,6 +970,7 @@ style = { level = "warn", priority = -1 } # trivial assertions are quite alright assertions_on_constants = { level = "allow", priority = 1 } module_inception = { level = "allow", priority = 1 } +obfuscated_if_else = { level = "allow", priority = 1 } ################### suspicious = { level = "warn", priority = -1 } diff --git a/src/admin/admin.rs b/src/admin/admin.rs index b6de1ec6..9e010a59 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -62,7 +62,7 @@ pub(super) async fn process(command: AdminCommand, context: &Command<'_>) -> Res | Debug(command) => debug::process(command, context).await?, | Query(command) => query::process(command, context).await?, | Check(command) => check::process(command, context).await?, - }; + } Ok(()) } diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index b5c303c8..6dd31b48 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -42,7 +42,7 @@ pub(super) async fn list_rooms( if rooms.is_empty() { return Ok(RoomMessageEventContent::text_plain("No more rooms.")); - }; + } let output_plain = format!( "Rooms ({}):\n```\n{}\n```", diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index e9c23a1d..ca036825 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -67,7 +67,7 @@ pub(super) async fn reprocess( if rooms.is_empty() { return Ok(RoomMessageEventContent::text_plain("No more rooms.")); - }; + } let output = format!( "Rooms (page {page}):\n```\n{}\n```", diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 8cb8edc3..8565f04a 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -166,7 +166,7 @@ pub(super) async fn create_user( "Failed to automatically join room {room} for user {user_id}: {e}" ); }, - }; + } } } } @@ -550,7 +550,7 @@ pub(super) async fn force_join_list_of_local_users( debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}"); failed_joins = failed_joins.saturating_add(1); }, - }; + } } Ok(RoomMessageEventContent::notice_markdown(format!( @@ -646,7 +646,7 @@ pub(super) async fn force_join_all_local_users( debug_warn!("Failed force joining {user_id} to {room_id} during bulk join: {e}"); failed_joins = failed_joins.saturating_add(1); }, - }; + } } Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/api/client/account.rs b/src/api/client/account.rs index cb49a6db..b42f51f7 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -499,7 +499,7 @@ pub(crate) async fn register_route( | _ => { info!("Automatically joined room {room} for user {user_id}"); }, - }; + } } } } diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 136c5961..6af9b533 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -131,7 +131,7 @@ pub(crate) async fn set_room_visibility_route( if !services.rooms.metadata.exists(&body.room_id).await { // Return 404 if the room doesn't exist - return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); + return Err!(Request(NotFound("Room not found"))); } if services @@ -145,10 +145,7 @@ pub(crate) async fn set_room_visibility_route( } if !user_can_publish_room(&services, sender_user, &body.room_id).await? { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "User is not allowed to publish this room", - )); + return Err!(Request(Forbidden("User is not allowed to publish this room"))); } match &body.visibility { @@ -386,12 +383,7 @@ async fn user_can_publish_room( .await { | Ok(event) => Ok(event.sender == user_id), - | _ => { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "User is not allowed to publish this room", - )); - }, + | _ => Err!(Request(Forbidden("User is not allowed to publish this room"))), } }, } diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 9c2693dc..0b9c0c69 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -993,7 +993,7 @@ async fn join_room_by_id_helper_remote( | _ => { join_event_stub.remove("event_id"); }, - }; + } // In order to create a compatible ref hash (EventID) the `hashes` field needs // to be present @@ -1420,7 +1420,7 @@ async fn join_room_by_id_helper_local( | _ => { join_event_stub.remove("event_id"); }, - }; + } // In order to create a compatible ref hash (EventID) the `hashes` field needs // to be present @@ -1947,7 +1947,7 @@ async fn remote_leave_room( | _ => { leave_event_stub.remove("event_id"); }, - }; + } // In order to create a compatible ref hash (EventID) the `hashes` field needs // to be present diff --git a/src/api/client/report.rs b/src/api/client/report.rs index db085721..7922caca 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -43,7 +43,7 @@ pub(crate) async fn report_room_route( ErrorKind::InvalidParam, "Reason too long, should be 750 characters or fewer", )); - }; + } delay_response().await; @@ -164,14 +164,14 @@ async fn is_event_report_valid( ErrorKind::InvalidParam, "Invalid score, must be within 0 to -100", )); - }; + } if reason.as_ref().is_some_and(|s| s.len() > 750) { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Reason too long, should be 750 characters or fewer", )); - }; + } if !services .rooms diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 56256683..92b75cfa 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -110,7 +110,7 @@ pub(super) async fn auth( } }, | _ => {}, - }; + } } match (metadata.authentication, token) { diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 08fa3835..c1749835 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -135,7 +135,7 @@ async fn create_join_event( if state_key != sender { return Err!(Request(BadJson("State key does not match sender user."))); - }; + } if let Some(authorising_user) = content.join_authorized_via_users_server { use ruma::RoomVersionId::*; diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs index 1d4c2a6c..f7bb0735 100644 --- a/src/api/server/send_knock.rs +++ b/src/api/server/send_knock.rs @@ -137,7 +137,7 @@ pub(crate) async fn create_knock_event_v1_route( if state_key != sender { return Err!(Request(InvalidParam("state_key does not match sender user of event."))); - }; + } let origin: OwnedServerName = serde_json::from_value( value diff --git a/src/database/de.rs b/src/database/de.rs index 9c0997ff..849b3b2e 100644 --- a/src/database/de.rs +++ b/src/database/de.rs @@ -241,7 +241,7 @@ impl<'a, 'de: 'a> de::Deserializer<'de> for &'a mut Deserializer<'de> { | "Ignore" => self.record_ignore(), | "IgnoreAll" => self.record_ignore_all(), | _ => unhandled!("Unrecognized deserialization Directive {name:?}"), - }; + } visitor.visit_unit() } diff --git a/src/database/engine/logger.rs b/src/database/engine/logger.rs index a1898e30..23e23fc7 100644 --- a/src/database/engine/logger.rs +++ b/src/database/engine/logger.rs @@ -18,5 +18,5 @@ pub(crate) fn handle(level: LogLevel, msg: &str) { | LogLevel::Error | LogLevel::Fatal => error!("{msg}"), | LogLevel::Info => debug!("{msg}"), | LogLevel::Warn => warn!("{msg}"), - }; + } } diff --git a/src/database/map/compact.rs b/src/database/map/compact.rs index 84476de6..b49bf30b 100644 --- a/src/database/map/compact.rs +++ b/src/database/map/compact.rs @@ -52,7 +52,7 @@ pub fn compact_blocking(&self, opts: Options) -> Result { co.set_target_level(level.try_into()?); }, | (Some(_), Some(_)) => return Err!("compacting between specific levels not supported"), - }; + } self.db .db diff --git a/src/database/map/qry_batch.rs b/src/database/map/qry_batch.rs index f44d1c86..e42d3e63 100644 --- a/src/database/map/qry_batch.rs +++ b/src/database/map/qry_batch.rs @@ -50,7 +50,6 @@ where .iter() .map(ser::serialize_to::) .map(|result| result.expect("failed to serialize query key")) - .map(Into::into) .collect(); self.db diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index fc2d1116..789a52e8 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -40,7 +40,7 @@ pub fn rev_raw_stream(self: &Arc) -> impl Stream> .into_stream() .flatten() .boxed(); - }; + } let seek = Seek { map: self.clone(), diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index d67986e7..a612d2a2 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -89,7 +89,7 @@ where .into_stream() .flatten() .boxed(); - }; + } let seek = Seek { map: self.clone(), diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index f1450b6f..f7371b6c 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -39,7 +39,7 @@ pub fn raw_stream(self: &Arc) -> impl Stream>> + .into_stream() .flatten() .boxed(); - }; + } let seek = Seek { map: self.clone(), diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index 00c3a051..ccf48db6 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -86,7 +86,7 @@ where .into_stream() .flatten() .boxed(); - }; + } let seek = Seek { map: self.clone(), diff --git a/src/database/pool.rs b/src/database/pool.rs index e6ed59ac..47e61c30 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -146,11 +146,9 @@ pub(crate) fn close(&self) { .map(JoinHandle::join) .map(|result| result.map_err(Error::from_panic)) .enumerate() - .for_each(|(id, result)| { - match result { - | Ok(()) => trace!(?id, "worker joined"), - | Err(error) => error!(?id, "worker joined with error: {error}"), - }; + .for_each(|(id, result)| match result { + | Ok(()) => trace!(?id, "worker joined"), + | Err(error) => error!(?id, "worker joined with error: {error}"), }); } @@ -345,7 +343,7 @@ fn worker_handle(self: &Arc, cmd: Cmd) { | Cmd::Get(cmd) if cmd.key.len() == 1 => self.handle_get(cmd), | Cmd::Get(cmd) => self.handle_batch(cmd), | Cmd::Iter(cmd) => self.handle_iter(cmd), - }; + } } #[implement(Pool)] @@ -362,7 +360,7 @@ fn handle_iter(&self, mut cmd: Seek) { return; } - let from = cmd.key.as_deref().map(Into::into); + let from = cmd.key.as_deref(); let result = match cmd.dir { | Direction::Forward => cmd.state.init_fwd(from), @@ -394,7 +392,7 @@ fn handle_batch(self: &Arc, mut cmd: Get) { return; } - let keys = cmd.key.iter().map(Into::into); + let keys = cmd.key.iter(); let result: SmallVec<_> = cmd.map.get_batch_blocking(keys).collect(); diff --git a/src/database/ser.rs b/src/database/ser.rs index 6dd2043d..2e1a2cb0 100644 --- a/src/database/ser.rs +++ b/src/database/ser.rs @@ -224,7 +224,7 @@ impl ser::Serializer for &mut Serializer<'_, W> { self.separator()?; }, | _ => unhandled!("Unrecognized serialization directive: {name:?}"), - }; + } Ok(()) } diff --git a/src/database/stream.rs b/src/database/stream.rs index eb856b3f..eb264ccd 100644 --- a/src/database/stream.rs +++ b/src/database/stream.rs @@ -113,13 +113,13 @@ impl<'a> State<'a> { } #[inline] - fn fetch_key(&self) -> Option> { self.inner.key().map(Key::from) } + fn fetch_key(&self) -> Option> { self.inner.key() } #[inline] - fn _fetch_val(&self) -> Option> { self.inner.value().map(Val::from) } + fn _fetch_val(&self) -> Option> { self.inner.value() } #[inline] - fn fetch(&self) -> Option> { self.inner.item().map(KeyVal::from) } + fn fetch(&self) -> Option> { self.inner.item() } #[inline] pub(super) fn status(&self) -> Option { self.inner.status().err() } diff --git a/src/database/watchers.rs b/src/database/watchers.rs index be814f8c..b3907833 100644 --- a/src/database/watchers.rs +++ b/src/database/watchers.rs @@ -53,6 +53,6 @@ impl Watchers { tx.0.send(()).expect("channel should still be open"); } } - }; + } } } diff --git a/src/macros/config.rs b/src/macros/config.rs index 07ac1c0a..7b424325 100644 --- a/src/macros/config.rs +++ b/src/macros/config.rs @@ -205,7 +205,7 @@ fn get_default(field: &Field) -> Option { }, | Meta::Path { .. } => return Some("false".to_owned()), | _ => return None, - }; + } } None diff --git a/src/service/media/blurhash.rs b/src/service/media/blurhash.rs index 9d73f5dc..91e00228 100644 --- a/src/service/media/blurhash.rs +++ b/src/service/media/blurhash.rs @@ -172,7 +172,7 @@ impl std::fmt::Display for BlurhashingError { #[cfg(feature = "blurhashing")] | Self::ImageError(e) => write!(f, "There was an error with the image loading library => {e}")?, - }; + } Ok(()) } diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index 61635011..b6c853d2 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -283,7 +283,7 @@ async fn location_request(&self, location: &str) -> Result { .map_err(Into::into) .map(|content| FileMeta { content: Some(content), - content_type: content_type.clone().map(Into::into), + content_type: content_type.clone(), content_disposition: Some(make_content_disposition( content_disposition.as_ref(), content_type.as_deref(), diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index b6d3e21e..b437bf2e 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -170,7 +170,7 @@ pub async fn handle_incoming_pdu<'a>( | Entry::Occupied(mut e) => { *e.get_mut() = (now, e.get().1.saturating_add(1)); }, - }; + } } } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 52e7d2be..910da914 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -125,7 +125,7 @@ pub async fn get_summary_and_children_local( SummaryAccessibility::Inaccessible }, )), - }; + } let children_pdus: Vec<_> = self .get_stripped_space_child_events(current_room) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 4403468b..02ffa0d1 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -218,7 +218,7 @@ impl Service { ) .await .ok(); - }; + } // Copy direct chat flag if let Ok(mut direct_event) = self @@ -250,7 +250,7 @@ impl Service { ) .await?; } - }; + } } } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index c566eb1c..305d3187 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -303,7 +303,7 @@ impl Service { }); return Ok(()); - }; + } // Else we have two options. // 1. We add the current diff on top of the parent layer. @@ -419,7 +419,7 @@ impl Service { 2, // every state change is 2 event changes on average states_parents, )?; - }; + } Ok(HashSetCompressStateEvent { shortstatehash: new_shortstatehash, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 4be97fb2..35c972fa 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -422,7 +422,7 @@ impl Service { highlight = true; }, | _ => {}, - }; + } // Break early if both conditions are true if notify && highlight { @@ -484,7 +484,7 @@ impl Service { } } }, - }; + } }, | TimelineEventType::SpaceChild => if let Some(_state_key) = &pdu.state_key { @@ -776,7 +776,7 @@ impl Service { | _ => { pdu_json.remove("event_id"); }, - }; + } // Add origin because synapse likes that (and it's required in the spec) pdu_json.insert( @@ -847,7 +847,7 @@ impl Service { { return Err!(Request(Forbidden("User cannot redact this event."))); } - }; + } }, | _ => { let content: RoomRedactionEventContent = pdu.get_content()?; @@ -863,7 +863,7 @@ impl Service { } }, } - }; + } if pdu.kind == TimelineEventType::RoomMember { let content: RoomMemberEventContent = pdu.get_content()?; @@ -1293,10 +1293,10 @@ async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Res } }, | _ => {}, - }; + } }, | _ => {}, - }; + } Ok(()) } diff --git a/src/service/sending/mod.rs b/src/service/sending/mod.rs index 379829b4..08ca7010 100644 --- a/src/service/sending/mod.rs +++ b/src/service/sending/mod.rs @@ -131,7 +131,7 @@ impl crate::Service for Service { | Err(error) => { error!(id = ?error.id(), ?error, "sender worker finished"); }, - }; + } } Ok(()) diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index c4f34177..616f0846 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -138,7 +138,7 @@ impl Service { match response { | Ok(dest) => self.handle_response_ok(&dest, futures, statuses).await, | Err((dest, e)) => Self::handle_response_err(dest, statuses, &e), - }; + } } fn handle_response_err(dest: Destination, statuses: &mut CurTransactionStatus, e: &Error) { @@ -319,10 +319,7 @@ impl Service { if let Destination::Federation(server_name) = dest { if let Ok((select_edus, last_count)) = self.select_edus(server_name).await { debug_assert!(select_edus.len() <= EDU_LIMIT, "exceeded edus limit"); - let select_edus = select_edus - .into_iter() - .map(Into::into) - .map(SendingEvent::Edu); + let select_edus = select_edus.into_iter().map(SendingEvent::Edu); events.extend(select_edus); self.db.set_latest_educount(server_name, last_count); diff --git a/src/service/server_keys/request.rs b/src/service/server_keys/request.rs index 171b755b..d9907616 100644 --- a/src/service/server_keys/request.rs +++ b/src/service/server_keys/request.rs @@ -43,7 +43,7 @@ where .keys() .rev() .take(self.services.server.config.trusted_server_batch_size) - .last() + .next_back() .cloned() { let request = Request { diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 51f5fb11..39dd2b41 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -69,7 +69,7 @@ pub async fn read_tokens(&self) -> Result> { }, | Err(e) => error!("Failed to read the registration token file: {e}"), } - }; + } if let Some(token) = &self.services.config.registration_token { tokens.insert(token.to_owned()); } From dca7bf9635ecd1fef3cd4bca56a25054d346692d Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 27 Feb 2025 10:39:06 -0500 Subject: [PATCH 1525/2091] try bumping cache-nix-action to v6 Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 8 ++------ .github/workflows/documentation.yml | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e1cf6c6..82ffc6b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,7 +129,7 @@ jobs: # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags #if: ${{ !startsWith(github.ref, 'refs/tags/') }} - uses: nix-community/cache-nix-action@v5.1.0 + uses: nix-community/cache-nix-action@v6 with: # restore and save a cache using this key primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/.lock') }} @@ -146,8 +146,6 @@ jobs: purge-last-accessed: 86400 # except the version with the `primary-key`, if it exists purge-primary-key: never - # always save the cache - save-always: true - name: Enable Cachix binary cache run: | @@ -324,7 +322,7 @@ jobs: # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags #if: ${{ !startsWith(github.ref, 'refs/tags/') }} - uses: nix-community/cache-nix-action@v5.1.0 + uses: nix-community/cache-nix-action@v6 with: # restore and save a cache using this key primary-key: nix-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/*.nix', '**/.lock') }} @@ -341,8 +339,6 @@ jobs: purge-last-accessed: 86400 # except the version with the `primary-key`, if it exists purge-primary-key: never - # always save the cache - save-always: true - name: Enable Cachix binary cache run: | diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 0eefe0a4..fadc7b3f 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -76,7 +76,7 @@ jobs: # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting # releases and tags if: ${{ !startsWith(github.ref, 'refs/tags/') }} - uses: nix-community/cache-nix-action@v5.1.0 + uses: nix-community/cache-nix-action@v6 with: # restore and save a cache using this key primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/.lock') }} @@ -93,8 +93,6 @@ jobs: purge-last-accessed: 86400 # except the version with the `primary-key`, if it exists purge-primary-key: never - # always save the cache - save-always: true - name: Enable Cachix binary cache run: | From 17e0384eeb91bfbb77576359252db25e3248cc40 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 2 Mar 2025 23:11:43 -0500 Subject: [PATCH 1526/2091] ignore errors instead of expecting for state gathering Signed-off-by: June Clementine Strawberry --- src/service/rooms/state_accessor/state.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/service/rooms/state_accessor/state.rs b/src/service/rooms/state_accessor/state.rs index 625defe6..02a6194e 100644 --- a/src/service/rooms/state_accessor/state.rs +++ b/src/service/rooms/state_accessor/state.rs @@ -4,7 +4,7 @@ use conduwuit::{ PduEvent, Result, StateKey, at, err, implement, pair_of, utils::{ result::FlatOk, - stream::{BroadbandExt, IterStream, ReadyExt, TryExpect}, + stream::{BroadbandExt, IterStream, ReadyExt, TryIgnore}, }, }; use database::Deserialized; @@ -232,7 +232,7 @@ pub fn state_keys_with_shortids<'a>( ) -> impl Stream + Send + 'a { let short_ids = self .state_full_shortids(shortstatehash) - .expect_ok() + .ignore_err() .unzip() .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) .shared(); @@ -269,7 +269,7 @@ pub fn state_keys<'a>( ) -> impl Stream + Send + 'a { let short_ids = self .state_full_shortids(shortstatehash) - .expect_ok() + .ignore_err() .map(at!(0)); self.services @@ -305,7 +305,7 @@ pub fn state_added( .map_ok(|(a, b)| b.difference(&a).copied().collect::>()) .map_ok(IterStream::try_stream) .try_flatten_stream() - .expect_ok() + .ignore_err() .map(parse_compressed_state_event) } @@ -327,7 +327,7 @@ pub fn state_full_pdus( ) -> impl Stream + Send + '_ { let short_ids = self .state_full_shortids(shortstatehash) - .expect_ok() + .ignore_err() .map(at!(1)); self.services @@ -352,7 +352,7 @@ where { let shortids = self .state_full_shortids(shortstatehash) - .expect_ok() + .ignore_err() .unzip() .shared(); From de53ad83b2ec49170075cc5176e0ec7a604aad94 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 2 Mar 2025 23:15:05 -0500 Subject: [PATCH 1527/2091] run nightly cargo fmt again Signed-off-by: June Clementine Strawberry --- src/admin/room/alias.rs | 5 +++-- src/admin/room/moderation.rs | 20 +++++++++++-------- src/api/client/directory.rs | 5 +++-- src/api/client/read_marker.rs | 5 +++-- src/api/server/hierarchy.rs | 5 +++-- src/core/state_res/event_auth.rs | 5 +++-- src/router/layers.rs | 8 ++++++-- .../rooms/event_handler/fetch_state.rs | 5 +++-- .../rooms/event_handler/handle_outlier_pdu.rs | 5 +++-- src/service/rooms/spaces/mod.rs | 5 +++-- src/service/server_keys/get.rs | 5 +++-- 11 files changed, 45 insertions(+), 28 deletions(-) diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 6262f33e..ab21170c 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -66,10 +66,11 @@ pub(super) async fn reprocess( format!("#{}:{}", room_alias_localpart, services.globals.server_name()); let room_alias = match OwnedRoomAliasId::parse(room_alias_str) { | Ok(alias) => alias, - | Err(err) => + | Err(err) => { return Ok(RoomMessageEventContent::text_plain(format!( "Failed to parse alias: {err}" - ))), + ))); + }, }; match command { | RoomAliasCommand::Set { force, room_id, .. } => { diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index ee132590..444dfa2f 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -96,12 +96,13 @@ async fn ban_room( let room_id = if room.is_room_id() { let room_id = match RoomId::parse(&room) { | Ok(room_id) => room_id, - | Err(e) => + | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( "Failed to parse room ID {room}. Please note that this requires a full room \ ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ (`#roomalias:example.com`): {e}" - ))), + ))); + }, }; debug!("Room specified is a room ID, banning room ID"); @@ -111,12 +112,13 @@ async fn ban_room( } else if room.is_room_alias_id() { let room_alias = match RoomAliasId::parse(&room) { | Ok(room_alias) => room_alias, - | Err(e) => + | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( "Failed to parse room ID {room}. Please note that this requires a full room \ ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ (`#roomalias:example.com`): {e}" - ))), + ))); + }, }; debug!( @@ -514,12 +516,13 @@ async fn unban_room( let room_id = if room.is_room_id() { let room_id = match RoomId::parse(&room) { | Ok(room_id) => room_id, - | Err(e) => + | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( "Failed to parse room ID {room}. Please note that this requires a full room \ ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ (`#roomalias:example.com`): {e}" - ))), + ))); + }, }; debug!("Room specified is a room ID, unbanning room ID"); @@ -529,12 +532,13 @@ async fn unban_room( } else if room.is_room_alias_id() { let room_alias = match RoomAliasId::parse(&room) { | Ok(room_alias) => room_alias, - | Err(e) => + | Err(e) => { return Ok(RoomMessageEventContent::text_plain(format!( "Failed to parse room ID {room}. Please note that this requires a full room \ ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ (`#roomalias:example.com`): {e}" - ))), + ))); + }, }; debug!( diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 6af9b533..88f0e668 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -267,8 +267,9 @@ pub(crate) async fn get_public_rooms_filtered_helper( let backwards = match characters.next() { | Some('n') => false, | Some('p') => true, - | _ => - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Invalid `since` token")), + | _ => { + return Err(Error::BadRequest(ErrorKind::InvalidParam, "Invalid `since` token")); + }, }; num_since = characters diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index d01327f6..187616b4 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -197,11 +197,12 @@ pub(crate) async fn create_receipt_route( .read_receipt .private_read_set(&body.room_id, sender_user, count); }, - | _ => + | _ => { return Err!(Request(InvalidParam(warn!( "Received unknown read receipt type: {}", &body.receipt_type - )))), + )))); + }, } Ok(create_receipt::v3::Response {}) diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs index 41eaedd0..c759c8ea 100644 --- a/src/api/server/hierarchy.rs +++ b/src/api/server/hierarchy.rs @@ -32,8 +32,9 @@ pub(crate) async fn get_hierarchy_route( { | None => Err!(Request(NotFound("The requested room was not found"))), - | Some(SummaryAccessibility::Inaccessible) => - Err!(Request(NotFound("The requested room is inaccessible"))), + | Some(SummaryAccessibility::Inaccessible) => { + Err!(Request(NotFound("The requested room is inaccessible"))) + }, | Some(SummaryAccessibility::Accessible(room)) => { let (children, inaccessible_children) = diff --git a/src/core/state_res/event_auth.rs b/src/core/state_res/event_auth.rs index 4b8e55f3..65bec802 100644 --- a/src/core/state_res/event_auth.rs +++ b/src/core/state_res/event_auth.rs @@ -682,7 +682,7 @@ fn valid_membership_change( } allow }, - | _ => + | _ => { if !sender_is_joined || target_user_current_membership == MembershipState::Join || target_user_current_membership == MembershipState::Ban @@ -706,7 +706,8 @@ fn valid_membership_change( ); } allow - }, + } + }, } }, | MembershipState::Leave => diff --git a/src/router/layers.rs b/src/router/layers.rs index 88e6a8d5..6920555d 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -61,8 +61,12 @@ pub(crate) fn build(services: &Arc) -> Result<(Router, Guard)> { ) .layer(axum::middleware::from_fn_with_state(Arc::clone(services), request::handle)) .layer(SecureClientIpSource::ConnectInfo.into_extension()) - .layer(ResponseBodyTimeoutLayer::new(Duration::from_secs(server.config.client_response_timeout))) - .layer(RequestBodyTimeoutLayer::new(Duration::from_secs(server.config.client_receive_timeout))) + .layer(ResponseBodyTimeoutLayer::new(Duration::from_secs( + server.config.client_response_timeout, + ))) + .layer(RequestBodyTimeoutLayer::new(Duration::from_secs( + server.config.client_receive_timeout, + ))) .layer(TimeoutLayer::new(Duration::from_secs(server.config.client_request_timeout))) .layer(SetResponseHeaderLayer::if_not_present( HeaderName::from_static("origin-agent-cluster"), // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Origin-Agent-Cluster diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index b1a4a38b..0f9e093b 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -58,10 +58,11 @@ pub(super) async fn fetch_state( | hash_map::Entry::Vacant(v) => { v.insert(pdu.event_id.clone()); }, - | hash_map::Entry::Occupied(_) => + | hash_map::Entry::Occupied(_) => { return Err!(Database( "State event's type and state_key combination exists multiple times.", - )), + )); + }, } } diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 974eb300..99e90a50 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -56,10 +56,11 @@ pub(super) async fn handle_outlier_pdu<'a>( obj }, - | Err(e) => + | Err(e) => { return Err!(Request(InvalidParam(debug_error!( "Signature verification failed for {event_id}: {e}" - )))), + )))); + }, }; // Now that we have checked the signature and hashes we can add the eventID and diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 910da914..1da38234 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -109,7 +109,7 @@ pub async fn get_summary_and_children_local( { | None => (), // cache miss | Some(None) => return Ok(None), - | Some(Some(cached)) => + | Some(Some(cached)) => { return Ok(Some( if self .is_accessible_child( @@ -124,7 +124,8 @@ pub async fn get_summary_and_children_local( } else { SummaryAccessibility::Inaccessible }, - )), + )); + }, } let children_pdus: Vec<_> = self diff --git a/src/service/server_keys/get.rs b/src/service/server_keys/get.rs index 00aeae1e..f9c5bdaf 100644 --- a/src/service/server_keys/get.rs +++ b/src/service/server_keys/get.rs @@ -18,8 +18,9 @@ pub async fn get_event_keys( let required = match required_keys(object, version) { | Ok(required) => required, - | Err(e) => - return Err!(BadServerResponse("Failed to determine keys required to verify: {e}")), + | Err(e) => { + return Err!(BadServerResponse("Failed to determine keys required to verify: {e}")); + }, }; let batch = required From 00cc23b6496533b9cfb77145966e2e7355f1f886 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 2 Mar 2025 23:15:30 -0500 Subject: [PATCH 1528/2091] bump nix lockfile, bump cargo.lock, bump ruwuma Signed-off-by: June Clementine Strawberry --- Cargo.lock | 263 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 4 +- flake.lock | 24 ++--- 3 files changed, 146 insertions(+), 145 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e84437c..e632b504 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,9 +55,9 @@ checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anyhow" -version = "1.0.95" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" [[package]] name = "arbitrary" @@ -128,9 +128,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.18" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" +checksum = "310c9bcae737a48ef5cdee3174184e6d548b292739ede61a1f955ef76a738861" dependencies = [ "brotli", "flate2", @@ -212,18 +212,18 @@ dependencies = [ [[package]] name = "avif-serialize" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e335041290c43101ca215eed6f43ec437eb5a42125573f600fc3fa42b9bddd62" +checksum = "98922d6a4cfbcb08820c69d8eeccc05bb1f29bfa06b4f5b1dbfe9a868bd7608e" dependencies = [ "arrayvec", ] [[package]] name = "aws-lc-rs" -version = "1.12.2" +version = "1.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c2b7ddaa2c56a367ad27a094ad8ef4faacf8a617c2575acb2ba88949df999ca" +checksum = "5e4e8200b9a4a5801a769d50eeabc05670fec7e959a8cb7a63a93e4e519942ae" dependencies = [ "aws-lc-sys", "paste", @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.25.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ac4f13dad353b209b34cbec082338202cbc01c8f00336b55c750c13ac91f8f" +checksum = "0f9dd2e03ee80ca2822dd6ea431163d2ef259f2066a4d6ccaca6d9dcb386aa43" dependencies = [ "bindgen", "cc", @@ -414,7 +414,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "cexpr", "clang-sys", "itertools 0.12.1", @@ -445,9 +445,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" [[package]] name = "bitstream-io" @@ -505,9 +505,9 @@ dependencies = [ [[package]] name = "built" -version = "0.7.5" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c360505aed52b7ec96a3636c3f039d99103c37d1d9b4f7a8c743d3ea9ffcd03b" +checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" [[package]] name = "bumpalo" @@ -541,18 +541,17 @@ checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" [[package]] name = "bytesize" -version = "1.3.0" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e368af43e418a04d52505cf3dbc23dda4e3407ae2fa99fd0e4f308ce546acc" +checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e" [[package]] name = "bzip2-sys" -version = "0.1.11+1.0.8" +version = "0.1.13+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" dependencies = [ "cc", - "libc", "pkg-config", ] @@ -568,9 +567,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.12" +version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755717a7de9ec452bf7f3f1a3099085deabd7f2962b861dae91ecd7a365903d2" +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ "jobserver", "libc", @@ -619,9 +618,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.39" +version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" dependencies = [ "num-traits", ] @@ -639,9 +638,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.28" +version = "4.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff" +checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" dependencies = [ "clap_builder", "clap_derive", @@ -649,9 +648,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.27" +version = "4.5.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" dependencies = [ "anstyle", "clap_lex", @@ -677,9 +676,9 @@ checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" -version = "0.1.53" +version = "0.1.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24a03c8b52922d68a1589ad61032f2c1aa5a8158d2aa0d93c6e9534944bbad6" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" dependencies = [ "cc", ] @@ -1134,7 +1133,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "crossterm_winapi", "futures-core", "mio", @@ -1215,9 +1214,9 @@ checksum = "817fa642fb0ee7fe42e95783e00e0969927b96091bdd4b9b1af082acd943913b" [[package]] name = "data-encoding" -version = "2.7.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e60eed09d8c01d3cee5b7d30acb059b76614c918fa0f992e0dd6eeb10daad6f" +checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" [[package]] name = "date_header" @@ -1309,9 +1308,9 @@ dependencies = [ [[package]] name = "either" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" dependencies = [ "serde", ] @@ -1330,9 +1329,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -1422,9 +1421,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.0.35" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" dependencies = [ "crc32fast", "miniz_oxide", @@ -1618,9 +1617,9 @@ checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" dependencies = [ "atomic-waker", "bytes", @@ -1720,9 +1719,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-proto" -version = "0.24.3" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad3d6d98c648ed628df039541a5577bee1a7c83e9e16fe3dbedeea4cdfeb971" +checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248" dependencies = [ "async-trait", "cfg-if", @@ -1744,9 +1743,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.3" +version = "0.24.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf287bde7b776e85d7188e6e5db7cf410a2f9531fe82817eb87feed034c8d14" +checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e" dependencies = [ "cfg-if", "futures-util", @@ -2223,6 +2222,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.14" @@ -2335,9 +2343,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.169" +version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" [[package]] name = "libfuzzer-sys" @@ -2384,9 +2392,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" [[package]] name = "lock_api" @@ -2400,9 +2408,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.25" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] name = "loole" @@ -2570,9 +2578,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ "adler2", "simd-adler32", @@ -2602,7 +2610,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "cfg-if", "cfg_aliases", "libc", @@ -2844,9 +2852,9 @@ dependencies = [ [[package]] name = "os_info" -version = "3.9.2" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e6520c8cc998c5741ee68ec1dc369fc47e5f0ea5320018ecf2a1ccd6328f48b" +checksum = "2a604e53c24761286860eba4e2c8b23a0161526476b1de520139d69cdb85a6b5" dependencies = [ "log", "serde", @@ -3116,9 +3124,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", "prost-derive", @@ -3126,12 +3134,12 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.98", @@ -3139,20 +3147,20 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ "prost", ] [[package]] name = "pulldown-cmark" -version = "0.12.2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f86ba2052aebccc42cbbb3ed234b8b13ce76f75c3551a303cb2bcffcff12bb14" +checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "memchr", "pulldown-cmark-escape", "unicase", @@ -3225,9 +3233,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" +checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944" dependencies = [ "cfg_aliases", "libc", @@ -3348,11 +3356,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] [[package]] @@ -3466,15 +3474,14 @@ checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" [[package]] name = "ring" -version = "0.17.8" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73" dependencies = [ "cc", "cfg-if", "getrandom 0.2.15", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] @@ -3482,7 +3489,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "assign", "js_int", @@ -3504,7 +3511,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "js_int", "ruma-common", @@ -3516,7 +3523,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "as_variant", "assign", @@ -3539,7 +3546,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "as_variant", "base64 0.22.1", @@ -3571,7 +3578,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "as_variant", "indexmap 2.7.1", @@ -3596,7 +3603,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "bytes", "http", @@ -3614,7 +3621,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3623,7 +3630,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "js_int", "ruma-common", @@ -3633,7 +3640,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3648,7 +3655,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "js_int", "ruma-common", @@ -3660,7 +3667,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "headers", "http", @@ -3673,7 +3680,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3689,7 +3696,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=5dc3e0f81d614ed9dc96b50f646b2e4385291c55#5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" dependencies = [ "futures-util", "js_int", @@ -3768,7 +3775,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "errno", "libc", "linux-raw-sys", @@ -3777,9 +3784,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.22" +version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "aws-lc-rs", "log", @@ -3899,7 +3906,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "core-foundation", "core-foundation-sys", "libc", @@ -4059,18 +4066,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" dependencies = [ "proc-macro2", "quote", @@ -4092,9 +4099,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.138" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" dependencies = [ "itoa", "memchr", @@ -4274,9 +4281,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" dependencies = [ "serde", ] @@ -4291,12 +4298,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] name = "spki" version = "0.7.3" @@ -4334,9 +4335,9 @@ dependencies = [ [[package]] name = "string_cache_codegen" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "244292f3441c89febe5b5bdfbb6863aeaf4f64da810ea3050fd927b27b8d92ce" +checksum = "c711928715f1fe0fe509c53b43e993a9a557babc2d0a3567d0a3006f1ac931a0" dependencies = [ "phf_generator", "phf_shared", @@ -4667,9 +4668,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ "rustls", "tokio", @@ -4734,9 +4735,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.22.23" +version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ "indexmap 2.7.1", "serde", @@ -4817,7 +4818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" dependencies = [ "async-compression", - "bitflags 2.8.0", + "bitflags 2.9.0", "bytes", "futures-core", "futures-util", @@ -4939,9 +4940,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "typewit" @@ -4984,9 +4985,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" +checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" [[package]] name = "unicode-segmentation" @@ -5071,9 +5072,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.13.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" +checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" dependencies = [ "getrandom 0.3.1", "serde", @@ -5511,9 +5512,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e376c75f4f43f44db463cf729e0d3acbf954d13e22c51e26e4c264b4ab545f" +checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1" dependencies = [ "memchr", ] @@ -5534,7 +5535,7 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] [[package]] @@ -5613,18 +5614,18 @@ dependencies = [ [[package]] name = "zerofrom" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", @@ -5662,27 +5663,27 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.2.1" +version = "7.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" +version = "2.0.14+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 52695d89..e2fe7021 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "5dc3e0f81d614ed9dc96b50f646b2e4385291c55" +rev = "b40e76528660f6a389eacd19a83ef9060644ee8f" features = [ "compat", "rand", @@ -509,7 +509,7 @@ version = "1.0.37" version = "1.0.89" [workspace.dependencies.bytesize] -version = "1.3.0" +version = "1.3.2" [workspace.dependencies.core_affinity] version = "0.8.1" diff --git a/flake.lock b/flake.lock index a7d80508..59fcbd8d 100644 --- a/flake.lock +++ b/flake.lock @@ -170,11 +170,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1740206139, - "narHash": "sha256-wWSv4KYhPKggKuJLzghfBs99pS3Kli9UBlyXVBzuIzc=", + "lastModified": 1740724364, + "narHash": "sha256-D1jLIueJx1dPrP09ZZwTrPf4cubV+TsFMYbpYYTVj6A=", "owner": "nix-community", "repo": "fenix", - "rev": "133a9eb59fb4ddac443ebe5ab2449d3940396533", + "rev": "edf7d9e431cda8782e729253835f178a356d3aab", "type": "github" }, "original": { @@ -364,11 +364,11 @@ "liburing": { "flake": false, "locked": { - "lastModified": 1740063075, - "narHash": "sha256-AfrCMPiXwgB0yxociq4no4NjCqGf/nRVhC3CLRoKqhA=", + "lastModified": 1740613216, + "narHash": "sha256-NpPOBqNND3Qe9IwqYs0mJLGTmIx7e6FgUEBAnJ+1ZLA=", "owner": "axboe", "repo": "liburing", - "rev": "5c788d514b9ed6d1a3624150de8aa6db403c1c65", + "rev": "e1003e496e66f9b0ae06674869795edf772d5500", "type": "github" }, "original": { @@ -550,11 +550,11 @@ }, "nixpkgs_5": { "locked": { - "lastModified": 1740019556, - "narHash": "sha256-vn285HxnnlHLWnv59Og7muqECNMS33mWLM14soFIv2g=", + "lastModified": 1740547748, + "narHash": "sha256-Ly2fBL1LscV+KyCqPRufUBuiw+zmWrlJzpWOWbahplg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "dad564433178067be1fbdfcce23b546254b6d641", + "rev": "3a05eebede89661660945da1f151959900903b6a", "type": "github" }, "original": { @@ -599,11 +599,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1740077634, - "narHash": "sha256-KlYdDhon/hy91NutuBeN8e3qTKf3FXgsudWsjnHud68=", + "lastModified": 1740691488, + "narHash": "sha256-Fs6vBrByuiOf2WO77qeMDMTXcTGzrIMqLBv+lNeywwM=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "88fbdcd510e79ef3bcd81d6d9d4f07bdce84be8c", + "rev": "fe3eda77d3a7ce212388bda7b6cec8bffcc077e5", "type": "github" }, "original": { From af714d5778bf8b5ba4356821941e48bff55aefea Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 2 Mar 2025 23:16:30 -0500 Subject: [PATCH 1529/2091] refactor+fix various issues with regs/logins and admin user commands Signed-off-by: June Clementine Strawberry --- src/admin/user/commands.rs | 35 ++++-- src/api/client/account.rs | 224 ++++++++++++++++++--------------- src/api/client/session.rs | 88 ++++++------- src/core/config/check.rs | 8 ++ src/service/admin/create.rs | 20 +-- src/service/admin/grant.rs | 129 ++++++++++++------- src/service/admin/mod.rs | 7 +- src/service/appservice/mod.rs | 2 +- src/service/emergency/mod.rs | 9 +- src/service/globals/mod.rs | 2 - src/service/resolver/actual.rs | 5 +- 11 files changed, 309 insertions(+), 220 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 8565f04a..35067304 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room}; use conduwuit::{ - PduBuilder, Result, debug_warn, error, info, is_equal_to, + PduBuilder, Result, debug, debug_warn, error, info, is_equal_to, utils::{self, ReadyExt}, warn, }; @@ -57,16 +57,16 @@ pub(super) async fn create_user( // Validate user id let user_id = parse_local_user_id(self.services, &username)?; - if self.services.users.exists(&user_id).await { - return Ok(RoomMessageEventContent::text_plain(format!( - "Userid {user_id} already exists" - ))); + if let Err(e) = user_id.validate_strict() { + if self.services.config.emergency_password.is_none() { + return Ok(RoomMessageEventContent::text_plain(format!( + "Username {user_id} contains disallowed characters or spaces: {e}" + ))); + } } - if user_id.is_historical() { - return Ok(RoomMessageEventContent::text_plain(format!( - "User ID {user_id} does not conform to new Matrix identifier spec" - ))); + if self.services.users.exists(&user_id).await { + return Ok(RoomMessageEventContent::text_plain(format!("User {user_id} already exists"))); } let password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); @@ -185,12 +185,12 @@ pub(super) async fn create_user( .is_ok_and(is_equal_to!(1)) { self.services.admin.make_user_admin(&user_id).await?; - warn!("Granting {user_id} admin privileges as the first user"); } + } else { + debug!("create_user admin command called without an admin room being available"); } - // Inhibit login does not work for guests Ok(RoomMessageEventContent::text_plain(format!( "Created user with user_id: {user_id} and password: `{password}`" ))) @@ -694,6 +694,19 @@ pub(super) async fn force_leave_room( self.services.globals.user_is_local(&user_id), "Parsed user_id must be a local user" ); + + if !self + .services + .rooms + .state_cache + .is_joined(&user_id, &room_id) + .await + { + return Ok(RoomMessageEventContent::notice_markdown(format!( + "{user_id} is not joined in the room" + ))); + } + leave_room(self.services, &user_id, &room_id, None).await?; Ok(RoomMessageEventContent::notice_markdown(format!( diff --git a/src/api/client/account.rs b/src/api/client/account.rs index b42f51f7..2b8209d4 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -3,7 +3,8 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Error, PduBuilder, Result, debug_info, error, info, is_equal_to, utils, utils::ReadyExt, warn, + Err, Error, PduBuilder, Result, debug_info, err, error, info, is_equal_to, utils, + utils::ReadyExt, warn, }; use futures::{FutureExt, StreamExt}; use register::RegistrationKind; @@ -17,7 +18,6 @@ use ruma::{ request_3pid_management_token_via_email, request_3pid_management_token_via_msisdn, whoami, }, - error::ErrorKind, uiaa::{AuthFlow, AuthType, UiaaInfo}, }, events::{ @@ -60,6 +60,14 @@ pub(crate) async fn get_register_available_route( || appservice.registration.id.contains("matrix_appservice_irc") }); + if services + .globals + .forbidden_usernames() + .is_match(&body.username) + { + return Err!(Request(Forbidden("Username is forbidden"))); + } + // don't force the username lowercase if it's from matrix-appservice-irc let body_username = if is_matrix_appservice_irc { body.username.clone() @@ -68,30 +76,45 @@ pub(crate) async fn get_register_available_route( }; // Validate user id - let user_id = UserId::parse_with_server_name(body_username, services.globals.server_name()) - .ok() - .filter(|user_id| { - (!user_id.is_historical() || is_matrix_appservice_irc) - && services.globals.user_is_local(user_id) - }) - .ok_or(Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; + let user_id = + match UserId::parse_with_server_name(&body_username, services.globals.server_name()) { + | Ok(user_id) => { + if let Err(e) = user_id.validate_strict() { + // unless the username is from the broken matrix appservice IRC bridge, we + // should follow synapse's behaviour on not allowing things like spaces + // and UTF-8 characters in usernames + if !is_matrix_appservice_irc { + return Err!(Request(InvalidUsername(debug_warn!( + "Username {body_username} contains disallowed characters or spaces: \ + {e}" + )))); + } + } + + user_id + }, + | Err(e) => { + return Err!(Request(InvalidUsername(debug_warn!( + "Username {body_username} is not valid: {e}" + )))); + }, + }; // Check if username is creative enough if services.users.exists(&user_id).await { - return Err(Error::BadRequest(ErrorKind::UserInUse, "Desired user ID is already taken.")); + return Err!(Request(UserInUse("User ID is not available."))); } - if services - .globals - .forbidden_usernames() - .is_match(user_id.localpart()) - { - return Err(Error::BadRequest(ErrorKind::Unknown, "Username is forbidden.")); + if let Some(ref info) = body.appservice_info { + if !info.is_user_match(&user_id) { + return Err!(Request(Exclusive("Username is not in an appservice namespace."))); + } + }; + + if services.appservice.is_exclusive_user_id(&user_id).await { + return Err!(Request(Exclusive("Username is reserved by an appservice."))); } - // TODO add check for appservice namespaces - - // If no if check is true we have an username that's available to be used. Ok(get_username_availability::v3::Response { available: true }) } @@ -119,16 +142,27 @@ pub(crate) async fn register_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - if !services.globals.allow_registration() && body.appservice_info.is_none() { - info!( - "Registration disabled and request not from known appservice, rejecting \ - registration attempt for username \"{}\"", - body.username.as_deref().unwrap_or("") - ); - return Err(Error::BadRequest(ErrorKind::forbidden(), "Registration has been disabled.")); - } - let is_guest = body.kind == RegistrationKind::Guest; + let emergency_mode_enabled = services.config.emergency_password.is_some(); + + if !services.globals.allow_registration() && body.appservice_info.is_none() { + match (body.username.as_ref(), body.initial_device_display_name.as_ref()) { + | (Some(username), Some(device_display_name)) => { + info!(%is_guest, user = %username, device_name = %device_display_name, "Rejecting registration attempt as registration is disabled"); + }, + | (Some(username), _) => { + info!(%is_guest, user = %username, "Rejecting registration attempt as registration is disabled"); + }, + | (_, Some(device_display_name)) => { + info!(%is_guest, device_name = %device_display_name, "Rejecting registration attempt as registration is disabled"); + }, + | (None, _) => { + info!(%is_guest, "Rejecting registration attempt as registration is disabled"); + }, + }; + + return Err!(Request(Forbidden("Registration has been disabled."))); + } if is_guest && (!services.globals.allow_guest_registration() @@ -140,10 +174,7 @@ pub(crate) async fn register_route( rejecting guest registration attempt, initial device name: \"{}\"", body.initial_device_display_name.as_deref().unwrap_or("") ); - return Err(Error::BadRequest( - ErrorKind::GuestAccessForbidden, - "Guest registration is disabled.", - )); + return Err!(Request(GuestAccessForbidden("Guest registration is disabled."))); } // forbid guests from registering if there is not a real admin user yet. give @@ -154,13 +185,10 @@ pub(crate) async fn register_route( rejecting registration. Guest's initial device name: \"{}\"", body.initial_device_display_name.as_deref().unwrap_or("") ); - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Registration temporarily disabled.", - )); + return Err!(Request(Forbidden("Registration is temporarily disabled."))); } - let user_id = match (&body.username, is_guest) { + let user_id = match (body.username.as_ref(), is_guest) { | (Some(username), false) => { // workaround for https://github.com/matrix-org/matrix-appservice-irc/issues/1780 due to inactivity of fixing the issue let is_matrix_appservice_irc = @@ -170,6 +198,12 @@ pub(crate) async fn register_route( || appservice.registration.id.contains("matrix_appservice_irc") }); + if services.globals.forbidden_usernames().is_match(username) + && !emergency_mode_enabled + { + return Err!(Request(Forbidden("Username is forbidden"))); + } + // don't force the username lowercase if it's from matrix-appservice-irc let body_username = if is_matrix_appservice_irc { username.clone() @@ -177,31 +211,34 @@ pub(crate) async fn register_route( username.to_lowercase() }; - let proposed_user_id = - UserId::parse_with_server_name(body_username, services.globals.server_name()) - .ok() - .filter(|user_id| { - (!user_id.is_historical() || is_matrix_appservice_irc) - && services.globals.user_is_local(user_id) - }) - .ok_or(Error::BadRequest( - ErrorKind::InvalidUsername, - "Username is invalid.", - ))?; + let proposed_user_id = match UserId::parse_with_server_name( + &body_username, + services.globals.server_name(), + ) { + | Ok(user_id) => { + if let Err(e) = user_id.validate_strict() { + // unless the username is from the broken matrix appservice IRC bridge, or + // we are in emergency mode, we should follow synapse's behaviour on + // not allowing things like spaces and UTF-8 characters in usernames + if !is_matrix_appservice_irc && !emergency_mode_enabled { + return Err!(Request(InvalidUsername(debug_warn!( + "Username {body_username} contains disallowed characters or \ + spaces: {e}" + )))); + } + } + + user_id + }, + | Err(e) => { + return Err!(Request(InvalidUsername(debug_warn!( + "Username {body_username} is not valid: {e}" + )))); + }, + }; if services.users.exists(&proposed_user_id).await { - return Err(Error::BadRequest( - ErrorKind::UserInUse, - "Desired user ID is already taken.", - )); - } - - if services - .globals - .forbidden_usernames() - .is_match(proposed_user_id.localpart()) - { - return Err(Error::BadRequest(ErrorKind::Unknown, "Username is forbidden.")); + return Err!(Request(UserInUse("User ID is not available."))); } proposed_user_id @@ -221,21 +258,18 @@ pub(crate) async fn register_route( if body.body.login_type == Some(LoginType::ApplicationService) { match body.appservice_info { | Some(ref info) => - if !info.is_user_match(&user_id) { - return Err(Error::BadRequest( - ErrorKind::Exclusive, - "User is not in namespace.", - )); + if !info.is_user_match(&user_id) && !emergency_mode_enabled { + return Err!(Request(Exclusive( + "Username is not in an appservice namespace." + ))); }, | _ => { - return Err(Error::BadRequest( - ErrorKind::MissingToken, - "Missing appservice token.", - )); + return Err!(Request(MissingToken("Missing appservice token."))); }, } - } else if services.appservice.is_exclusive_user_id(&user_id).await { - return Err(Error::BadRequest(ErrorKind::Exclusive, "User ID reserved by appservice.")); + } else if services.appservice.is_exclusive_user_id(&user_id).await && !emergency_mode_enabled + { + return Err!(Request(Exclusive("Username is reserved by an appservice."))); } // UIAA @@ -271,7 +305,7 @@ pub(crate) async fn register_route( .uiaa .try_auth( &UserId::parse_with_server_name("", services.globals.server_name()) - .expect("we know this is valid"), + .unwrap(), "".into(), auth, &uiaainfo, @@ -287,7 +321,7 @@ pub(crate) async fn register_route( uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services.uiaa.create( &UserId::parse_with_server_name("", services.globals.server_name()) - .expect("we know this is valid"), + .unwrap(), "".into(), &uiaainfo, &json, @@ -295,7 +329,7 @@ pub(crate) async fn register_route( return Err(Error::Uiaa(uiaainfo)); }, | _ => { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + return Err!(Request(NotJson("JSON body is not valid"))); }, }, } @@ -407,7 +441,7 @@ pub(crate) async fn register_route( // log in conduit admin channel if a guest registered if body.appservice_info.is_none() && is_guest && services.globals.log_guest_registrations() { - info!("New guest user \"{user_id}\" registered on this server."); + debug_info!("New guest user \"{user_id}\" registered on this server."); if !device_display_name.is_empty() { if services.server.config.admin_room_notices { @@ -436,7 +470,8 @@ pub(crate) async fn register_route( } // If this is the first real user, grant them admin privileges except for guest - // users Note: the server user, @conduit:servername, is generated first + // users + // Note: the server user is generated first if !is_guest { if let Ok(admin_room) = services.admin.get_admin_room().await { if services @@ -541,8 +576,8 @@ pub(crate) async fn change_password_route( let sender_user = body .sender_user .as_ref() - .ok_or_else(|| Error::BadRequest(ErrorKind::MissingToken, "Missing access token."))?; - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + .ok_or_else(|| err!(Request(MissingToken("Missing access token."))))?; + let sender_device = body.sender_device(); let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { stages: vec![AuthType::Password] }], @@ -566,16 +601,16 @@ pub(crate) async fn change_password_route( // Success! }, | _ => match body.json_body { - | Some(json) => { + | Some(ref json) => { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + .create(sender_user, sender_device, &uiaainfo, json); return Err(Error::Uiaa(uiaainfo)); }, | _ => { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + return Err!(Request(NotJson("JSON body is not valid"))); }, }, } @@ -589,7 +624,7 @@ pub(crate) async fn change_password_route( services .users .all_device_ids(sender_user) - .ready_filter(|id| id != sender_device) + .ready_filter(|id| *id != sender_device) .for_each(|id| services.users.remove_device(sender_user, id)) .await; } @@ -651,8 +686,8 @@ pub(crate) async fn deactivate_route( let sender_user = body .sender_user .as_ref() - .ok_or_else(|| Error::BadRequest(ErrorKind::MissingToken, "Missing access token."))?; - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + .ok_or_else(|| err!(Request(MissingToken("Missing access token."))))?; + let sender_device = body.sender_device(); let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { stages: vec![AuthType::Password] }], @@ -675,16 +710,16 @@ pub(crate) async fn deactivate_route( // Success! }, | _ => match body.json_body { - | Some(json) => { + | Some(ref json) => { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + .create(sender_user, sender_device, &uiaainfo, json); return Err(Error::Uiaa(uiaainfo)); }, | _ => { - return Err(Error::BadRequest(ErrorKind::NotJson, "Not json.")); + return Err!(Request(NotJson("JSON body is not valid"))); }, }, } @@ -743,10 +778,7 @@ pub(crate) async fn third_party_route( pub(crate) async fn request_3pid_management_token_via_email_route( _body: Ruma, ) -> Result { - Err(Error::BadRequest( - ErrorKind::ThreepidDenied, - "Third party identifier is not allowed", - )) + Err!(Request(ThreepidDenied("Third party identifiers are not implemented"))) } /// # `POST /_matrix/client/v3/account/3pid/msisdn/requestToken` @@ -759,10 +791,7 @@ pub(crate) async fn request_3pid_management_token_via_email_route( pub(crate) async fn request_3pid_management_token_via_msisdn_route( _body: Ruma, ) -> Result { - Err(Error::BadRequest( - ErrorKind::ThreepidDenied, - "Third party identifier is not allowed", - )) + Err!(Request(ThreepidDenied("Third party identifiers are not implemented"))) } /// # `GET /_matrix/client/v1/register/m.login.registration_token/validity` @@ -776,10 +805,7 @@ pub(crate) async fn check_registration_token_validity( body: Ruma, ) -> Result { let Some(reg_token) = services.globals.registration_token.clone() else { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Server does not allow token registration.", - )); + return Err!(Request(Forbidden("Server does not allow token registration"))); }; Ok(check_registration_token_validity::v1::Response { valid: reg_token == body.token }) diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 5c0ab47d..6db761af 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -2,12 +2,11 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{Err, debug, err, info, utils::ReadyExt, warn}; +use conduwuit::{Err, debug, err, info, utils::ReadyExt}; use futures::StreamExt; use ruma::{ - OwnedUserId, UserId, + UserId, api::client::{ - error::ErrorKind, session::{ get_login_token, get_login_types::{ @@ -67,6 +66,8 @@ pub(crate) async fn login_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { + let emergency_mode_enabled = services.config.emergency_password.is_some(); + // Validate login method // TODO: Other login methods let user_id = match &body.login_info { @@ -78,20 +79,22 @@ pub(crate) async fn login_route( .. }) => { debug!("Got password login type"); - let user_id = if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = - identifier - { - UserId::parse_with_server_name( - user_id.to_lowercase(), - services.globals.server_name(), - ) - } else if let Some(user) = user { - OwnedUserId::parse(user) - } else { - warn!("Bad login type: {:?}", &body.login_info); - return Err!(Request(Forbidden("Bad login type."))); - } - .map_err(|_| Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."))?; + let user_id = + if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { + UserId::parse_with_server_name(user_id, &services.config.server_name) + } else if let Some(user) = user { + UserId::parse_with_server_name(user, &services.config.server_name) + } else { + return Err!(Request(Unknown( + warn!(?body.login_info, "Invalid or unsupported login type") + ))); + } + .map_err(|e| err!(Request(InvalidUsername(warn!("Username is invalid: {e}")))))?; + + assert!( + services.globals.user_is_local(&user_id), + "User ID does not belong to this homeserver" + ); let hash = services .users @@ -124,46 +127,40 @@ pub(crate) async fn login_route( debug!("Got appservice login type"); let user_id = if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { - UserId::parse_with_server_name( - user_id.to_lowercase(), - services.globals.server_name(), - ) + UserId::parse_with_server_name(user_id, &services.config.server_name) } else if let Some(user) = user { - OwnedUserId::parse(user) + UserId::parse_with_server_name(user, &services.config.server_name) } else { - warn!("Bad login type: {:?}", &body.login_info); - return Err(Error::BadRequest(ErrorKind::forbidden(), "Bad login type.")); + return Err!(Request(Unknown( + warn!(?body.login_info, "Invalid or unsupported login type") + ))); } - .map_err(|e| { - warn!("Failed to parse username from appservice logging in: {e}"); - Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.") - })?; + .map_err(|e| err!(Request(InvalidUsername(warn!("Username is invalid: {e}")))))?; + + assert!( + services.globals.user_is_local(&user_id), + "User ID does not belong to this homeserver" + ); match body.appservice_info { | Some(ref info) => - if !info.is_user_match(&user_id) { - return Err(Error::BadRequest( - ErrorKind::Exclusive, - "User is not in namespace.", - )); + if !info.is_user_match(&user_id) && !emergency_mode_enabled { + return Err!(Request(Exclusive( + "Username is not in an appservice namespace." + ))); }, | _ => { - return Err(Error::BadRequest( - ErrorKind::MissingToken, - "Missing appservice token.", - )); + return Err!(Request(MissingToken("Missing appservice token."))); }, } user_id }, | _ => { - warn!("Unsupported or unknown login type: {:?}", &body.login_info); - debug!("JSON body: {:?}", &body.json_body); - return Err(Error::BadRequest( - ErrorKind::Unknown, - "Unsupported or unknown login type.", - )); + debug!("/login json_body: {:?}", &body.json_body); + return Err!(Request(Unknown( + warn!(?body.login_info, "Invalid or unsupported login type") + ))); }, }; @@ -216,9 +213,6 @@ pub(crate) async fn login_route( info!("{user_id} logged in"); - // home_server is deprecated but apparently must still be sent despite it being - // deprecated over 6 years ago. initially i thought this macro was unnecessary, - // but ruma uses this same macro for the same reason so... #[allow(deprecated)] Ok(login::v3::Response { user_id, @@ -226,7 +220,7 @@ pub(crate) async fn login_route( device_id, well_known: client_discovery_info, expires_in: None, - home_server: Some(services.globals.server_name().to_owned()), + home_server: Some(services.config.server_name.clone()), refresh_token: None, }) } diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 488f7f94..98223be4 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -126,6 +126,14 @@ pub fn check(config: &Config) -> Result { )); } + if config.emergency_password == Some(String::new()) { + return Err!(Config( + "emergency_password", + "Emergency password was set to an empty string, this is not valid. Unset \ + emergency_password to disable it or set it to a real password." + )); + } + // check if the user specified a registration token as `""` if config.registration_token == Some(String::new()) { return Err!(Config( diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 7f71665a..4de37092 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -21,11 +21,11 @@ use crate::Services; /// Create the admin room. /// -/// Users in this room are considered admins by conduit, and the room can be +/// Users in this room are considered admins by conduwuit, and the room can be /// used to issue admin commands by talking to the server user inside it. -pub async fn create_admin_room(services: &Services) -> Result<()> { +pub async fn create_admin_room(services: &Services) -> Result { let room_id = RoomId::new(services.globals.server_name()); - let room_version = &services.server.config.default_room_version; + let room_version = &services.config.default_room_version; let _short_id = services .rooms @@ -36,14 +36,14 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { let state_lock = services.rooms.state.mutex.lock(&room_id).await; // Create a user for the server - let server_user = &services.globals.server_user; + let server_user = services.globals.server_user.as_ref(); services.users.create(server_user, None)?; let create_content = { use RoomVersionId::*; match room_version { | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => - RoomCreateEventContent::new_v1(server_user.clone()), + RoomCreateEventContent::new_v1(server_user.into()), | _ => RoomCreateEventContent::new_v11(), } }; @@ -71,7 +71,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .timeline .build_and_append_pdu( PduBuilder::state( - server_user.to_string(), + String::from(server_user), &RoomMemberEventContent::new(MembershipState::Join), ), server_user, @@ -81,7 +81,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .await?; // 3. Power levels - let users = BTreeMap::from_iter([(server_user.clone(), 100.into())]); + let users = BTreeMap::from_iter([(server_user.into(), 69420.into())]); services .rooms @@ -140,7 +140,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .await?; // 5. Events implied by name and topic - let room_name = format!("{} Admin Room", services.globals.server_name()); + let room_name = format!("{} Admin Room", services.config.server_name); services .rooms .timeline @@ -157,7 +157,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .timeline .build_and_append_pdu( PduBuilder::state(String::new(), &RoomTopicEventContent { - topic: format!("Manage {}", services.globals.server_name()), + topic: format!("Manage {} | Run commands prefixed with `!admin` | Run `!admin -h` for help | Documentation: https://conduwuit.puppyirl.gay/", services.config.server_name), }), server_user, &room_id, @@ -187,7 +187,7 @@ pub async fn create_admin_room(services: &Services) -> Result<()> { .alias .set_alias(alias, &room_id, server_user)?; - // 7. (ad-hoc) Disable room previews for everyone by default + // 7. (ad-hoc) Disable room URL previews for everyone by default services .rooms .timeline diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 358ea267..5173987a 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -1,10 +1,10 @@ use std::collections::BTreeMap; -use conduwuit::{Result, error, implement}; +use conduwuit::{Err, Result, debug_info, debug_warn, error, implement}; use ruma::{ RoomId, UserId, events::{ - RoomAccountDataEventType, + RoomAccountDataEventType, StateEventType, room::{ member::{MembershipState, RoomMemberEventContent}, message::RoomMessageEventContent, @@ -20,55 +20,98 @@ use crate::pdu::PduBuilder; /// /// This is equivalent to granting server admin privileges. #[implement(super::Service)] -pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { +pub async fn make_user_admin(&self, user_id: &UserId) -> Result { let Ok(room_id) = self.get_admin_room().await else { + debug_warn!( + "make_user_admin was called without an admin room being available or created" + ); return Ok(()); }; let state_lock = self.services.state.mutex.lock(&room_id).await; + if self.services.state_cache.is_joined(user_id, &room_id).await { + return Err!(debug_warn!("User is already joined in the admin room")); + } + if self + .services + .state_cache + .is_invited(user_id, &room_id) + .await + { + return Err!(debug_warn!("User is already pending an invitation to the admin room")); + } + // Use the server user to grant the new admin's power level - let server_user = &self.services.globals.server_user; + let server_user = self.services.globals.server_user.as_ref(); - // Invite and join the real user - self.services - .timeline - .build_and_append_pdu( - PduBuilder::state( - user_id.to_string(), - &RoomMemberEventContent::new(MembershipState::Invite), - ), - server_user, + // if this is our local user, just forcefully join them in the room. otherwise, + // invite the remote user. + if self.services.globals.user_is_local(user_id) { + debug_info!("Inviting local user {user_id} to admin room {room_id}"); + self.services + .timeline + .build_and_append_pdu( + PduBuilder::state( + String::from(user_id), + &RoomMemberEventContent::new(MembershipState::Invite), + ), + server_user, + &room_id, + &state_lock, + ) + .await?; + + debug_info!("Force joining local user {user_id} to admin room {room_id}"); + self.services + .timeline + .build_and_append_pdu( + PduBuilder::state( + String::from(user_id), + &RoomMemberEventContent::new(MembershipState::Join), + ), + user_id, + &room_id, + &state_lock, + ) + .await?; + } else { + debug_info!("Inviting remote user {user_id} to admin room {room_id}"); + self.services + .timeline + .build_and_append_pdu( + PduBuilder::state( + user_id.to_string(), + &RoomMemberEventContent::new(MembershipState::Invite), + ), + server_user, + &room_id, + &state_lock, + ) + .await?; + } + + // Set power levels + let mut room_power_levels = self + .services + .state_accessor + .room_state_get_content::( &room_id, - &state_lock, + &StateEventType::RoomPowerLevels, + "", ) - .await?; - self.services - .timeline - .build_and_append_pdu( - PduBuilder::state( - user_id.to_string(), - &RoomMemberEventContent::new(MembershipState::Join), - ), - user_id, - &room_id, - &state_lock, - ) - .await?; + .await + .unwrap_or_default(); - // Set power level - let users = BTreeMap::from_iter([ - (server_user.clone(), 100.into()), - (user_id.to_owned(), 100.into()), - ]); + room_power_levels + .users + .insert(server_user.into(), 69420.into()); + room_power_levels.users.insert(user_id.into(), 100.into()); self.services .timeline .build_and_append_pdu( - PduBuilder::state(String::new(), &RoomPowerLevelsEventContent { - users, - ..Default::default() - }), + PduBuilder::state(String::new(), &room_power_levels), server_user, &room_id, &state_lock, @@ -76,15 +119,17 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { .await?; // Set room tag - let room_tag = &self.services.server.config.admin_room_tag; + let room_tag = self.services.server.config.admin_room_tag.as_str(); if !room_tag.is_empty() { if let Err(e) = self.set_room_tag(&room_id, user_id, room_tag).await { - error!(?room_id, ?user_id, ?room_tag, ?e, "Failed to set tag for admin grant"); + error!(?room_id, ?user_id, ?room_tag, "Failed to set tag for admin grant: {e}"); } } if self.services.server.config.admin_room_notices { - let welcome_message = String::from("## Thank you for trying out conduwuit!\n\nconduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status initially was inherited from Conduit, however overtime this Beta status is rapidly becoming less and less relevant as our codebase significantly diverges more and more. conduwuit is quite stable and very usable as a daily driver and for a low-medium sized homeserver. There is still a lot of more work to be done, but it is in a far better place than the project was in early 2024.\n\nHelpful links:\n> GitHub Repo: https://github.com/girlbossceo/conduwuit\n> Documentation: https://conduwuit.puppyirl.gay/\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `!admin --help`\n\nHere are some rooms you can join (by typing the command into your client) -\n\nconduwuit space: `/join #conduwuit-space:puppygock.gay`\nconduwuit main room (Ask questions and get notified on updates): `/join #conduwuit:puppygock.gay`\nconduwuit offtopic room: `/join #conduwuit-offtopic:puppygock.gay`"); + let welcome_message = String::from( + "## Thank you for trying out conduwuit!\n\nconduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status initially was inherited from Conduit, however overtime this Beta status is rapidly becoming less and less relevant as our codebase significantly diverges more and more. conduwuit is quite stable and very usable as a daily driver and for a low-medium sized homeserver. There is still a lot of more work to be done, but it is in a far better place than the project was in early 2024.\n\nHelpful links:\n> GitHub Repo: https://github.com/girlbossceo/conduwuit\n> Documentation: https://conduwuit.puppyirl.gay/\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `!admin --help`\n\nHere are some rooms you can join (by typing the command into your client) -\n\nconduwuit space: `/join #conduwuit-space:puppygock.gay`\nconduwuit main room (Ask questions and get notified on updates): `/join #conduwuit:puppygock.gay`\nconduwuit offtopic room: `/join #conduwuit-offtopic:puppygock.gay`", + ); // Send welcome message self.services @@ -102,7 +147,7 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result<()> { } #[implement(super::Service)] -async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result<()> { +async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> Result { let mut event = self .services .account_data @@ -125,7 +170,5 @@ async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> R RoomAccountDataEventType::Tag, &serde_json::to_value(event)?, ) - .await?; - - Ok(()) + .await } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 4622f10e..b3466711 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -40,6 +40,7 @@ struct Services { timeline: Dep, state: Dep, state_cache: Dep, + state_accessor: Dep, account_data: Dep, services: StdRwLock>>, } @@ -85,6 +86,8 @@ impl crate::Service for Service { timeline: args.depend::("rooms::timeline"), state: args.depend::("rooms::state"), state_cache: args.depend::("rooms::state_cache"), + state_accessor: args + .depend::("rooms::state_accessor"), account_data: args.depend::("account_data"), services: None.into(), }, @@ -357,8 +360,8 @@ impl Service { } // This will evaluate to false if the emergency password is set up so that - // the administrator can execute commands as conduit - let emergency_password_set = self.services.globals.emergency_password().is_some(); + // the administrator can execute commands as the server user + let emergency_password_set = self.services.server.config.emergency_password.is_some(); let from_server = pdu.sender == *server_user && !emergency_password_set; if from_server && self.is_admin_room(&pdu.room_id).await { return false; diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 5aba0018..50a60033 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -90,7 +90,7 @@ impl Service { .write() .await .remove(appservice_id) - .ok_or(err!("Appservice not found"))?; + .ok_or_else(|| err!("Appservice not found"))?; // remove the appservice from the database self.db.id_appserviceregistrations.del(appservice_id); diff --git a/src/service/emergency/mod.rs b/src/service/emergency/mod.rs index 47a309a5..3a61f710 100644 --- a/src/service/emergency/mod.rs +++ b/src/service/emergency/mod.rs @@ -9,7 +9,7 @@ use ruma::{ push::Ruleset, }; -use crate::{Dep, account_data, globals, users}; +use crate::{Dep, account_data, config, globals, users}; pub struct Service { services: Services, @@ -17,6 +17,7 @@ pub struct Service { struct Services { account_data: Dep, + config: Dep, globals: Dep, users: Dep, } @@ -27,6 +28,8 @@ impl crate::Service for Service { Ok(Arc::new(Self { services: Services { account_data: args.depend::("account_data"), + config: args.depend::("config"), + globals: args.depend::("globals"), users: args.depend::("users"), }, @@ -54,9 +57,9 @@ impl Service { self.services .users - .set_password(server_user, self.services.globals.emergency_password().as_deref())?; + .set_password(server_user, self.services.config.emergency_password.as_deref())?; - let (ruleset, pwd_set) = match self.services.globals.emergency_password() { + let (ruleset, pwd_set) = match self.services.config.emergency_password { | Some(_) => (Ruleset::server_default(server_user), true), | None => (Ruleset::new(), false), }; diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 16b3ef3c..74f83228 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -153,8 +153,6 @@ impl Service { pub fn notification_push_path(&self) -> &String { &self.server.config.notification_push_path } - pub fn emergency_password(&self) -> &Option { &self.server.config.emergency_password } - pub fn url_preview_domain_contains_allowlist(&self) -> &Vec { &self.server.config.url_preview_domain_contains_allowlist } diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 8860d0a0..b037cf77 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -363,7 +363,7 @@ impl super::Service { let hostname = hostname.trim_end_matches('.'); match self.resolver.resolver.srv_lookup(hostname).await { | Err(e) => Self::handle_resolve_error(&e, hostname)?, - | Ok(result) => + | Ok(result) => { return Ok(result.iter().next().map(|result| { FedDest::Named( result.target().to_string().trim_end_matches('.').to_owned(), @@ -372,7 +372,8 @@ impl super::Service { .try_into() .unwrap_or_else(|_| FedDest::default_port()), ) - })), + })); + }, } } From 0d741bbd46cd1c2a86321a4a68da3167c46d53e3 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 3 Mar 2025 00:15:12 -0500 Subject: [PATCH 1530/2091] remove nix run ci test as its covered by complement Signed-off-by: June Clementine Strawberry --- engage.toml | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/engage.toml b/engage.toml index c1a2be1f..71366532 100644 --- a/engage.toml +++ b/engage.toml @@ -18,12 +18,12 @@ script = "direnv --version" [[task]] name = "rustc" group = "versions" -script = "rustc --version" +script = "rustc --version -v" [[task]] name = "cargo" group = "versions" -script = "cargo --version" +script = "cargo --version -v" [[task]] name = "cargo-fmt" @@ -60,11 +60,6 @@ name = "markdownlint" group = "versions" script = "markdownlint --version" -[[task]] -name = "dpkg" -group = "versions" -script = "dpkg --version" - [[task]] name = "cargo-audit" group = "security" @@ -228,22 +223,3 @@ depends = ["cargo/default"] script = """ git diff --exit-code conduwuit-example.toml """ - -# Ensure that the flake's default output can build and run without crashing -# -# This is a dynamically-linked jemalloc build, which is a case not covered by -# our other tests. We've had linking problems in the past with dynamic -# jemalloc builds that usually show up as an immediate segfault or "invalid free" -[[task]] -name = "nix-default" -group = "tests" -script = """ -env DIRENV_DEVSHELL=dynamic \ - CARGO_PROFILE="test" \ - direnv exec . \ - bin/nix-build-and-cache just .#default-test -env DIRENV_DEVSHELL=dynamic \ - CARGO_PROFILE="test" \ - direnv exec . \ - nix run -L .#default-test -- --help && nix run -L .#default-test -- --version -""" From df72384c16aa77ccedf532888b0799a3edc2d8b0 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 3 Mar 2025 01:05:43 -0500 Subject: [PATCH 1531/2091] delete snappy, bump rust-rocksdb, bump rocksdb to v9.10.0 again Signed-off-by: June Clementine Strawberry --- Cargo.lock | 4 ++-- deps/rust-rocksdb/Cargo.toml | 3 +-- flake.lock | 8 ++++---- flake.nix | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e632b504..ec531994 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3711,7 +3711,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.32.0+9.10.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=7b0e1bbe395a41ba8a11347a4921da590e3ad0d9#7b0e1bbe395a41ba8a11347a4921da590e3ad0d9" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b#2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b" dependencies = [ "bindgen", "bzip2-sys", @@ -3728,7 +3728,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.36.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=7b0e1bbe395a41ba8a11347a4921da590e3ad0d9#7b0e1bbe395a41ba8a11347a4921da590e3ad0d9" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b#2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index c6af428d..f9069fc1 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -14,7 +14,6 @@ default = ["lz4", "zstd", "zlib", "bzip2"] jemalloc = ["rust-rocksdb/jemalloc"] io-uring = ["rust-rocksdb/io-uring"] valgrind = ["rust-rocksdb/valgrind"] -snappy = ["rust-rocksdb/snappy"] lz4 = ["rust-rocksdb/lz4"] zstd = ["rust-rocksdb/zstd"] zlib = ["rust-rocksdb/zlib"] @@ -27,7 +26,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "7b0e1bbe395a41ba8a11347a4921da590e3ad0d9" +rev = "2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index 59fcbd8d..ba7fdcff 100644 --- a/flake.lock +++ b/flake.lock @@ -567,16 +567,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1739735789, - "narHash": "sha256-BIzuZS0TV4gRnciP4ieW5J3Hql986iedM5dHQfK6z68=", + "lastModified": 1739735940, + "narHash": "sha256-9AqKOWsYXy0sU2C+kB+3NLCDMZ2VsjfbHqvSiydUlcs=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "34e401fd4392dd3268e042f1e40dffd064b9a7ff", + "rev": "f8ad8cd72fd7e527171d35fa8dbca9a073b5b26c", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.9.3", + "ref": "v9.10.0", "repo": "rocksdb", "type": "github" } diff --git a/flake.nix b/flake.nix index 04dee681..6702111f 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.9.3"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.10.0"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From 1ecd02738992f6fd75ea627e60a2ebf1133f4561 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 3 Mar 2025 01:06:04 -0500 Subject: [PATCH 1532/2091] always run checks when building in nix (doCheck true) Signed-off-by: June Clementine Strawberry --- nix/pkgs/main/default.nix | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 4150b389..5dfb32ec 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -162,18 +162,12 @@ commonAttrs = { ]; }; - # This is redundant with CI - doCheck = false; + doCheck = true; - cargoTestCommand = "cargo test --locked "; cargoExtraArgs = "--no-default-features --locked " + lib.optionalString (features'' != []) "--features " + (builtins.concatStringsSep "," features''); - cargoTestExtraArgs = "--no-default-features --locked " - + lib.optionalString - (features'' != []) - "--features " + (builtins.concatStringsSep "," features''); dontStrip = profile == "dev" || profile == "test"; dontPatchELF = profile == "dev" || profile == "test"; @@ -209,18 +203,12 @@ craneLib.buildPackage ( commonAttrs // { env = buildDepsOnlyEnv; }); - # This is redundant with CI - doCheck = false; + doCheck = true; - cargoTestCommand = "cargo test --locked "; cargoExtraArgs = "--no-default-features --locked " + lib.optionalString (features'' != []) "--features " + (builtins.concatStringsSep "," features''); - cargoTestExtraArgs = "--no-default-features --locked " - + lib.optionalString - (features'' != []) - "--features " + (builtins.concatStringsSep "," features''); env = buildPackageEnv; From 7c17163730fcd0f43132cce82cc28b6793ae662a Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 4 Mar 2025 23:35:21 -0500 Subject: [PATCH 1533/2091] switch to self-hosted ci runner, remove sudo usages Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 132 ++-------------------------- .github/workflows/documentation.yml | 64 +------------- 2 files changed, 9 insertions(+), 187 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 82ffc6b6..c0425873 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,8 +45,8 @@ env: # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps NIX_CONFIG: | show-trace = true - extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net - extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= + extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= experimental-features = nix-command flakes extra-experimental-features = nix-command flakes accept-flake-config = true @@ -59,7 +59,7 @@ permissions: {} jobs: tests: name: Test - runs-on: ubuntu-24.04 + runs-on: self-hosted steps: - name: Setup SSH web publish env: @@ -93,19 +93,6 @@ jobs: echo "SSH_WEBSITE=1" >> "$GITHUB_ENV" - - name: Install liburing - run: | - sudo apt install liburing-dev -y - - - name: Free up a bit of runner space - run: | - set +o pipefail - sudo docker image prune --all --force || true - sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true - sudo apt clean - sudo rm -rf /usr/local/lib/android /usr/local/julia* /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/local/lib/heroku /usr/lib/heroku /usr/local/share/boost /usr/share/dotnet /usr/local/bin/cmake* /usr/local/bin/stack /usr/local/bin/terraform /opt/microsoft/powershell /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go /opt/hostedtoolcache/PyPy /usr/local/bin/sam || true - set -o pipefail - - name: Sync repository uses: actions/checkout@v4 with: @@ -123,58 +110,9 @@ jobs: exit 1 fi - - uses: nixbuild/nix-quick-install-action@master - - - name: Restore and cache Nix store - # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting - # releases and tags - #if: ${{ !startsWith(github.ref, 'refs/tags/') }} - uses: nix-community/cache-nix-action@v6 - with: - # restore and save a cache using this key - primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/.lock') }} - # if there's no cache hit, restore a cache by this prefix - restore-prefixes-first-match: nix-${{ runner.os }}- - # collect garbage until Nix store size (in bytes) is at most this number - # before trying to save a new cache - gc-max-store-size-linux: 2073741824 - # do purge caches - purge: true - # purge all versions of the cache - purge-prefixes: nix-${{ runner.os }}- - # created more than this number of seconds ago relative to the start of the `Post Restore` phase - purge-last-accessed: 86400 - # except the version with the `primary-key`, if it exists - purge-primary-key: never - - - name: Enable Cachix binary cache - run: | - nix profile install nixpkgs#cachix - cachix use crane - cachix use nix-community - - - name: Apply Nix binary cache configuration - run: | - sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null < /dev/null < "$HOME/.direnvrc" - nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop .#all-features --command true @@ -267,22 +205,13 @@ jobs: build: name: Build - runs-on: ubuntu-24.04 + runs-on: self-hosted strategy: matrix: include: - target: aarch64-linux-musl - target: x86_64-linux-musl steps: - - name: Free up a bit of runner space - run: | - set +o pipefail - sudo docker image prune --all --force || true - sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true - sudo apt clean - sudo rm -rf /usr/local/lib/android /usr/local/julia* /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/local/lib/heroku /usr/lib/heroku /usr/local/share/boost /usr/share/dotnet /usr/local/bin/cmake* /usr/local/bin/stack /usr/local/bin/terraform /opt/microsoft/powershell /opt/hostedtoolcache/CodeQL /opt/hostedtoolcache/go /opt/hostedtoolcache/PyPy /usr/local/bin/sam || true - set -o pipefail - - name: Sync repository uses: actions/checkout@v4 with: @@ -316,58 +245,9 @@ jobs: echo "SSH_WEBSITE=1" >> "$GITHUB_ENV" - - uses: nixbuild/nix-quick-install-action@master - - - name: Restore and cache Nix store - # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting - # releases and tags - #if: ${{ !startsWith(github.ref, 'refs/tags/') }} - uses: nix-community/cache-nix-action@v6 - with: - # restore and save a cache using this key - primary-key: nix-${{ runner.os }}-${{ matrix.target }}-${{ hashFiles('**/*.nix', '**/.lock') }} - # if there's no cache hit, restore a cache by this prefix - restore-prefixes-first-match: nix-${{ runner.os }}- - # collect garbage until Nix store size (in bytes) is at most this number - # before trying to save a new cache - gc-max-store-size-linux: 2073741824 - # do purge caches - purge: true - # purge all versions of the cache - purge-prefixes: nix-${{ runner.os }}- - # created more than this number of seconds ago relative to the start of the `Post Restore` phase - purge-last-accessed: 86400 - # except the version with the `primary-key`, if it exists - purge-primary-key: never - - - name: Enable Cachix binary cache - run: | - nix profile install nixpkgs#cachix - cachix use crane - cachix use nix-community - - - name: Apply Nix binary cache configuration - run: | - sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null < /dev/null < "$HOME/.direnvrc" - nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop .#all-features --command true --impure @@ -622,7 +502,7 @@ jobs: variables: outputs: github_repository: ${{ steps.var.outputs.github_repository }} - runs-on: "ubuntu-latest" + runs-on: self-hosted steps: - name: Setting global variables uses: actions/github-script@v7 @@ -632,7 +512,7 @@ jobs: core.setOutput('github_repository', '${{ github.repository }}'.toLowerCase()) docker: name: Docker publish - runs-on: ubuntu-24.04 + runs-on: self-hosted needs: [build, variables, tests] permissions: packages: write diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index fadc7b3f..88e7bbe1 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -24,8 +24,8 @@ env: # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps NIX_CONFIG: | show-trace = true - extra-substituters = extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net - extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= + extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= experimental-features = nix-command flakes extra-experimental-features = nix-command flakes accept-flake-config = true @@ -41,7 +41,7 @@ permissions: {} jobs: docs: name: Documentation and GitHub Pages - runs-on: ubuntu-24.04 + runs-on: self-hosted permissions: pages: write @@ -52,15 +52,6 @@ jobs: url: ${{ steps.deployment.outputs.page_url }} steps: - - name: Free up a bit of runner space - run: | - set +o pipefail - sudo docker image prune --all --force || true - sudo apt purge -y 'php.*' '^mongodb-.*' '^mysql-.*' azure-cli google-cloud-cli google-chrome-stable firefox powershell microsoft-edge-stable || true - sudo apt clean - sudo rm -v -rf /usr/local/games /usr/local/sqlpackage /usr/local/share/powershell /usr/local/share/edge_driver /usr/local/share/gecko_driver /usr/local/share/chromium /usr/local/share/chromedriver-linux64 /usr/lib/google-cloud-sdk /usr/lib/jvm /usr/lib/mono /usr/lib/heroku - set -o pipefail - - name: Sync repository uses: actions/checkout@v4 with: @@ -70,58 +61,9 @@ jobs: if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') uses: actions/configure-pages@v5 - - uses: nixbuild/nix-quick-install-action@master - - - name: Restore and cache Nix store - # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting - # releases and tags - if: ${{ !startsWith(github.ref, 'refs/tags/') }} - uses: nix-community/cache-nix-action@v6 - with: - # restore and save a cache using this key - primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/.lock') }} - # if there's no cache hit, restore a cache by this prefix - restore-prefixes-first-match: nix-${{ runner.os }}- - # collect garbage until Nix store size (in bytes) is at most this number - # before trying to save a new cache - gc-max-store-size-linux: 2073741824 - # do purge caches - purge: true - # purge all versions of the cache - purge-prefixes: nix-${{ runner.os }}- - # created more than this number of seconds ago relative to the start of the `Post Restore` phase - purge-last-accessed: 86400 - # except the version with the `primary-key`, if it exists - purge-primary-key: never - - - name: Enable Cachix binary cache - run: | - nix profile install nixpkgs#cachix - cachix use crane - cachix use nix-community - - - name: Apply Nix binary cache configuration - run: | - sudo tee -a "${XDG_CONFIG_HOME:-$HOME/.config}/nix/nix.conf" > /dev/null < /dev/null < "$HOME/.direnvrc" - nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop --command true From 35981d5aef8785c132d2e2a166cfcde1cd24169e Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Wed, 5 Mar 2025 19:05:42 -0500 Subject: [PATCH 1534/2091] automatically forget rooms on leaving Signed-off-by: June Clementine Strawberry --- src/service/rooms/state_cache/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 02ffa0d1..f406eb69 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -267,6 +267,10 @@ impl Service { }, | MembershipState::Leave | MembershipState::Ban => { self.mark_as_left(user_id, room_id); + + if self.services.globals.user_is_local(user_id) { + self.forget(room_id, user_id); + } }, | _ => {}, } From 97208d6081da92f8b5c732aa6b3bf06997ad4a16 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Wed, 5 Mar 2025 19:06:10 -0500 Subject: [PATCH 1535/2091] add more safety checks before allowing a room marked as forgotten Signed-off-by: June Clementine Strawberry --- src/api/client/membership.rs | 40 +++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 0b9c0c69..940c8639 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -11,12 +11,12 @@ use axum_client_ip::InsecureClientIp; use conduwuit::{ Err, PduEvent, Result, StateKey, at, debug, debug_info, debug_warn, err, error, info, pdu::{PduBuilder, gen_event_id_canonical_json}, - result::FlatOk, + result::{FlatOk, NotFound}, state_res, trace, utils::{self, IterStream, ReadyExt, shuffle}, warn, }; -use futures::{FutureExt, StreamExt, TryFutureExt, join}; +use futures::{FutureExt, StreamExt, TryFutureExt, future::join4, join}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, @@ -717,21 +717,37 @@ pub(crate) async fn forget_room_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user(); + let user_id = body.sender_user(); + let room_id = &body.room_id; - if services - .rooms - .state_cache - .is_joined(sender_user, &body.room_id) - .await - { + let joined = services.rooms.state_cache.is_joined(user_id, room_id); + let knocked = services.rooms.state_cache.is_knocked(user_id, room_id); + let left = services.rooms.state_cache.is_left(user_id, room_id); + let invited = services.rooms.state_cache.is_invited(user_id, room_id); + + let (joined, knocked, left, invited) = join4(joined, knocked, left, invited).await; + + if joined || knocked || invited { return Err!(Request(Unknown("You must leave the room before forgetting it"))); } - services + let membership = services .rooms - .state_cache - .forget(&body.room_id, sender_user); + .state_accessor + .get_member(room_id, user_id) + .await; + + if membership.is_not_found() { + return Err!(Request(Unknown("No membership event was found, room was never joined"))); + } + + if left + || membership.is_ok_and(|member| { + member.membership == MembershipState::Leave + || member.membership == MembershipState::Ban + }) { + services.rooms.state_cache.forget(room_id, user_id); + } Ok(forget_room::v3::Response::new()) } From 408f5bd30cb461cec9472a51b87f0bb1ed6b7381 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Wed, 5 Mar 2025 19:06:31 -0500 Subject: [PATCH 1536/2091] add val_size_hints on membership cfs (todo remove these anyways) Signed-off-by: June Clementine Strawberry --- src/database/maps.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/database/maps.rs b/src/database/maps.rs index b060ab8d..9af45159 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -181,6 +181,7 @@ pub(super) static MAPS: &[Descriptor] = &[ }, Descriptor { name: "roomuserid_invitecount", + val_size_hint: Some(8), ..descriptor::RANDOM_SMALL }, Descriptor { @@ -193,10 +194,12 @@ pub(super) static MAPS: &[Descriptor] = &[ }, Descriptor { name: "roomuserid_leftcount", + val_size_hint: Some(8), ..descriptor::RANDOM }, Descriptor { name: "roomuserid_knockedcount", + val_size_hint: Some(8), ..descriptor::RANDOM_SMALL }, Descriptor { From 2c1ec3fb02a823515697b159e26d5464ebe29937 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Wed, 5 Mar 2025 21:31:49 -0500 Subject: [PATCH 1537/2091] allow both lowercase and uppercase usernames to login Signed-off-by: June Clementine Strawberry --- src/api/client/session.rs | 54 +++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 17 deletions(-) diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 6db761af..ab67ee18 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -3,7 +3,7 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{Err, debug, err, info, utils::ReadyExt}; -use futures::StreamExt; +use futures::{StreamExt, TryFutureExt}; use ruma::{ UserId, api::client::{ @@ -86,29 +86,40 @@ pub(crate) async fn login_route( UserId::parse_with_server_name(user, &services.config.server_name) } else { return Err!(Request(Unknown( - warn!(?body.login_info, "Invalid or unsupported login type") + debug_warn!(?body.login_info, "Valid identifier or username was not provided (invalid or unsupported login type?)") ))); } .map_err(|e| err!(Request(InvalidUsername(warn!("Username is invalid: {e}")))))?; + let lowercased_user_id = UserId::parse_with_server_name( + user_id.localpart().to_lowercase(), + &services.config.server_name, + )?; + assert!( services.globals.user_is_local(&user_id), "User ID does not belong to this homeserver" ); + assert!( + services.globals.user_is_local(&lowercased_user_id), + "User ID does not belong to this homeserver" + ); let hash = services .users .password_hash(&user_id) + .or_else(|_| services.users.password_hash(&lowercased_user_id)) .await + .inspect_err(|e| debug!("{e}")) .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; if hash.is_empty() { return Err!(Request(UserDeactivated("The user has been deactivated"))); } - if hash::verify_password(password, &hash).is_err() { - return Err!(Request(Forbidden("Wrong username or password."))); - } + hash::verify_password(password, &hash) + .inspect_err(|e| debug!("{e}")) + .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; user_id }, @@ -125,6 +136,11 @@ pub(crate) async fn login_route( user, }) => { debug!("Got appservice login type"); + + let Some(ref info) = body.appservice_info else { + return Err!(Request(MissingToken("Missing appservice token."))); + }; + let user_id = if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { UserId::parse_with_server_name(user_id, &services.config.server_name) @@ -132,26 +148,30 @@ pub(crate) async fn login_route( UserId::parse_with_server_name(user, &services.config.server_name) } else { return Err!(Request(Unknown( - warn!(?body.login_info, "Invalid or unsupported login type") + debug_warn!(?body.login_info, "Valid identifier or username was not provided (invalid or unsupported login type?)") ))); } .map_err(|e| err!(Request(InvalidUsername(warn!("Username is invalid: {e}")))))?; + let lowercased_user_id = UserId::parse_with_server_name( + user_id.localpart().to_lowercase(), + &services.config.server_name, + )?; + assert!( services.globals.user_is_local(&user_id), "User ID does not belong to this homeserver" ); + assert!( + services.globals.user_is_local(&lowercased_user_id), + "User ID does not belong to this homeserver" + ); - match body.appservice_info { - | Some(ref info) => - if !info.is_user_match(&user_id) && !emergency_mode_enabled { - return Err!(Request(Exclusive( - "Username is not in an appservice namespace." - ))); - }, - | _ => { - return Err!(Request(MissingToken("Missing appservice token."))); - }, + if !info.is_user_match(&user_id) + && !info.is_user_match(&lowercased_user_id) + && !emergency_mode_enabled + { + return Err!(Request(Exclusive("Username is not in an appservice namespace."))); } user_id @@ -159,7 +179,7 @@ pub(crate) async fn login_route( | _ => { debug!("/login json_body: {:?}", &body.json_body); return Err!(Request(Unknown( - warn!(?body.login_info, "Invalid or unsupported login type") + debug_warn!(?body.login_info, "Invalid or unsupported login type") ))); }, }; From c10500f8aebcd52a219bdba4f2114b03d9474565 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 00:14:24 -0500 Subject: [PATCH 1538/2091] bump rust-rocksdb and ruwuma Signed-off-by: June Clementine Strawberry --- Cargo.lock | 60 +++++++++---------------------- Cargo.toml | 3 +- deps/rust-rocksdb/Cargo.toml | 5 +-- flake.lock | 8 ++--- flake.nix | 2 +- src/api/router/auth.rs | 3 +- src/service/federation/execute.rs | 3 +- 7 files changed, 28 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec531994..d51bb966 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3489,7 +3489,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "assign", "js_int", @@ -3502,16 +3502,14 @@ dependencies = [ "ruma-identifiers-validation", "ruma-identity-service-api", "ruma-push-gateway-api", - "ruma-server-util", "ruma-signatures", - "ruma-state-res", "web-time 1.1.0", ] [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "js_int", "ruma-common", @@ -3523,7 +3521,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "as_variant", "assign", @@ -3546,7 +3544,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "as_variant", "base64 0.22.1", @@ -3578,7 +3576,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "as_variant", "indexmap 2.7.1", @@ -3603,10 +3601,12 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "bytes", + "headers", "http", + "http-auth", "httparse", "js_int", "memchr", @@ -3616,12 +3616,14 @@ dependencies = [ "ruma-events", "serde", "serde_json", + "thiserror 2.0.11", + "tracing", ] [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3630,7 +3632,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "js_int", "ruma-common", @@ -3640,7 +3642,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3655,7 +3657,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "js_int", "ruma-common", @@ -3664,23 +3666,10 @@ dependencies = [ "serde_json", ] -[[package]] -name = "ruma-server-util" -version = "0.3.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" -dependencies = [ - "headers", - "http", - "http-auth", - "ruma-common", - "thiserror 2.0.11", - "tracing", -] - [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" +source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3693,25 +3682,10 @@ dependencies = [ "thiserror 2.0.11", ] -[[package]] -name = "ruma-state-res" -version = "0.11.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=b40e76528660f6a389eacd19a83ef9060644ee8f#b40e76528660f6a389eacd19a83ef9060644ee8f" -dependencies = [ - "futures-util", - "js_int", - "ruma-common", - "ruma-events", - "serde", - "serde_json", - "thiserror 2.0.11", - "tracing", -] - [[package]] name = "rust-librocksdb-sys" version = "0.32.0+9.10.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b#2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=513133a3dc24b667f32933aa3247c6ec71a958f3#513133a3dc24b667f32933aa3247c6ec71a958f3" dependencies = [ "bindgen", "bzip2-sys", @@ -3728,7 +3702,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.36.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b#2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=513133a3dc24b667f32933aa3247c6ec71a958f3#513133a3dc24b667f32933aa3247c6ec71a958f3" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index e2fe7021..7f08a21a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "b40e76528660f6a389eacd19a83ef9060644ee8f" +rev = "bb42118bd85e731b652a6110896b6945085bf944" features = [ "compat", "rand", @@ -355,7 +355,6 @@ features = [ "federation-api", "markdown", "push-gateway-api-c", - "server-util", "unstable-exhaustive-types", "ring-compat", "compat-upload-signatures", diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index f9069fc1..61bd2333 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -2,7 +2,7 @@ name = "rust-rocksdb-uwu" categories.workspace = true description = "dylib wrapper for rust-rocksdb" -edition = "2021" +edition = "2024" keywords.workspace = true license.workspace = true readme.workspace = true @@ -13,6 +13,7 @@ version = "0.0.1" default = ["lz4", "zstd", "zlib", "bzip2"] jemalloc = ["rust-rocksdb/jemalloc"] io-uring = ["rust-rocksdb/io-uring"] +numa = ["rust-rocksdb/numa"] # unused by rocksdb for now valgrind = ["rust-rocksdb/valgrind"] lz4 = ["rust-rocksdb/lz4"] zstd = ["rust-rocksdb/zstd"] @@ -26,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "2ff4bbf31e944fa2686bb041d8c5caaf4b966d3b" +rev = "513133a3dc24b667f32933aa3247c6ec71a958f3" #branch = "master" default-features = false diff --git a/flake.lock b/flake.lock index ba7fdcff..a1bd423f 100644 --- a/flake.lock +++ b/flake.lock @@ -567,16 +567,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1739735940, - "narHash": "sha256-9AqKOWsYXy0sU2C+kB+3NLCDMZ2VsjfbHqvSiydUlcs=", + "lastModified": 1741234703, + "narHash": "sha256-sT5g/RM9vrwY6AmjSfl4RoJPGtcJCkZCsxiX3PFJgKQ=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "f8ad8cd72fd7e527171d35fa8dbca9a073b5b26c", + "rev": "185593ce4534091e57025e9f3571dbf681c04631", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.10.0", + "ref": "v9.9.3", "repo": "rocksdb", "type": "github" } diff --git a/flake.nix b/flake.nix index 6702111f..04dee681 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.10.0"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.9.3"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 92b75cfa..5cd7b831 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -17,9 +17,8 @@ use ruma::{ }, voip::get_turn_server_info, }, - federation::openid::get_openid_userinfo, + federation::{authentication::XMatrix, openid::get_openid_userinfo}, }, - server_util::authorization::XMatrix, }; use service::{ Services, diff --git a/src/service/federation/execute.rs b/src/service/federation/execute.rs index d254486f..63f2ccfb 100644 --- a/src/service/federation/execute.rs +++ b/src/service/federation/execute.rs @@ -12,10 +12,9 @@ use ruma::{ CanonicalJsonObject, CanonicalJsonValue, ServerName, ServerSigningKeyId, api::{ EndpointError, IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, - client::error::Error as RumaError, + client::error::Error as RumaError, federation::authentication::XMatrix, }, serde::Base64, - server_util::authorization::XMatrix, }; use crate::resolver::actual::ActualDest; From 17b625a85b908d4c2cb3df308c2337be6e571ce2 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 00:14:49 -0500 Subject: [PATCH 1539/2091] reject device keys if they dont match user ID or device ID or are missing fields Signed-off-by: June Clementine Strawberry --- src/api/client/keys.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 6f20153b..8a7eab7e 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -48,6 +48,19 @@ pub(crate) async fn upload_keys_route( } if let Some(device_keys) = &body.device_keys { + let deser_device_keys = device_keys.deserialize()?; + + if deser_device_keys.user_id != sender_user { + return Err!(Request(Unknown( + "User ID in keys uploaded does not match your own user ID" + ))); + } + if deser_device_keys.device_id != sender_device { + return Err!(Request(Unknown( + "Device ID in keys uploaded does not match your own device ID" + ))); + } + // TODO: merge this and the existing event? // This check is needed to assure that signatures are kept if services From f4c51cd405f1a0695b16c085655eb0180637fe2d Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 00:18:28 -0500 Subject: [PATCH 1540/2091] remove zlib as a default rocksdb compression option Signed-off-by: June Clementine Strawberry --- conduwuit-example.toml | 2 +- deps/rust-rocksdb/Cargo.toml | 2 +- src/core/config/mod.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 9b6f6ce0..541f062d 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -821,7 +821,7 @@ # Type of RocksDB database compression to use. # -# Available options are "zstd", "zlib", "bz2", "lz4", or "none". +# Available options are "zstd", "bz2", "lz4", or "none". # # It is best to use ZSTD as an overall good balance between # speed/performance, storage, IO amplification, and CPU usage. For more diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 61bd2333..35f755b4 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true version = "0.0.1" [features] -default = ["lz4", "zstd", "zlib", "bzip2"] +default = ["lz4", "zstd", "bzip2"] jemalloc = ["rust-rocksdb/jemalloc"] io-uring = ["rust-rocksdb/io-uring"] numa = ["rust-rocksdb/numa"] # unused by rocksdb for now diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 67c3b95c..5a4819e0 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -975,7 +975,7 @@ pub struct Config { /// Type of RocksDB database compression to use. /// - /// Available options are "zstd", "zlib", "bz2", "lz4", or "none". + /// Available options are "zstd", "bz2", "lz4", or "none". /// /// It is best to use ZSTD as an overall good balance between /// speed/performance, storage, IO amplification, and CPU usage. For more From 657e91fd4226d2521e9e7bb15d5982e62ad68624 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 00:34:17 -0500 Subject: [PATCH 1541/2091] dont send push notifications from ignored users PDUs Signed-off-by: June Clementine Strawberry --- src/service/rooms/timeline/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 35c972fa..138340a4 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -368,7 +368,7 @@ impl Service { .state_accessor .room_state_get_content(&pdu.room_id, &StateEventType::RoomPowerLevels, "") .await - .map_err(|_| err!(Database("invalid m.room.power_levels event"))) + .map_err(|e| err!(Database(warn!("invalid m.room.power_levels event: {e}")))) .unwrap_or_default(); let sync_pdu = pdu.to_sync_room_event(); @@ -377,9 +377,10 @@ impl Service { .services .state_cache .active_local_users_in_room(&pdu.room_id) - // Don't notify the sender of their own events - .ready_filter(|user| user != &pdu.sender) .map(ToOwned::to_owned) + // Don't notify the sender of their own events, and dont send from ignored users + .ready_filter(|user| user != &pdu.sender) + .filter_map(|recipient_user| async move { (!self.services.users.user_is_ignored(&pdu.sender, recipient_user).await).then_some(recipient_user) }) .collect() .await; From 931fd4c80215cee5cd709d42c86d1fefe0844fe1 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 00:44:57 -0500 Subject: [PATCH 1542/2091] add missing target Signed-off-by: June Clementine Strawberry --- rust-toolchain.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 00fb6cee..97b4a789 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -24,5 +24,6 @@ targets = [ "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", #"aarch64-apple-darwin", ] From ecea0cff69d583439e4a84fba6bd2d5aaba8faee Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 00:51:13 -0500 Subject: [PATCH 1543/2091] fix TestFetchMessagesFromNonExistentRoom complement test Signed-off-by: June Clementine Strawberry --- src/api/client/message.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 571a238a..c755cc47 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,6 +1,6 @@ use axum::extract::State; use conduwuit::{ - Event, PduCount, PduEvent, Result, at, + Err, Event, PduCount, PduEvent, Result, at, utils::{ IterStream, ReadyExt, result::{FlatOk, LogErr}, @@ -68,6 +68,10 @@ pub(crate) async fn get_message_events_route( let room_id = &body.room_id; let filter = &body.filter; + if !services.rooms.metadata.exists(room_id).await { + return Err!(Request(Forbidden("Room does not exist to this server"))); + } + let from: PduCount = body .from .as_deref() From c92678ecbeb55cf323758da08e8c36e65496aa38 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 13:08:01 -0500 Subject: [PATCH 1544/2091] dont build with zlib in the nix flake Signed-off-by: June Clementine Strawberry --- Cargo.lock | 4 ++-- Cargo.toml | 1 - deps/rust-rocksdb/Cargo.toml | 2 +- flake.nix | 9 +++++++-- src/service/rooms/timeline/mod.rs | 4 ++-- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d51bb966..2ade8b83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3685,7 +3685,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.32.0+9.10.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=513133a3dc24b667f32933aa3247c6ec71a958f3#513133a3dc24b667f32933aa3247c6ec71a958f3" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=a5d5358ca1358f828283e1558cf6a402b6cbea34#a5d5358ca1358f828283e1558cf6a402b6cbea34" dependencies = [ "bindgen", "bzip2-sys", @@ -3702,7 +3702,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.36.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=513133a3dc24b667f32933aa3247c6ec71a958f3#513133a3dc24b667f32933aa3247c6ec71a958f3" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=a5d5358ca1358f828283e1558cf6a402b6cbea34#a5d5358ca1358f828283e1558cf6a402b6cbea34" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index 7f08a21a..5edcc60a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -389,7 +389,6 @@ features = [ "mt_static", "lz4", "zstd", - "zlib", "bzip2", ] diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml index 35f755b4..f6e0a54f 100644 --- a/deps/rust-rocksdb/Cargo.toml +++ b/deps/rust-rocksdb/Cargo.toml @@ -27,7 +27,7 @@ malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] [dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "513133a3dc24b667f32933aa3247c6ec71a958f3" +rev = "a5d5358ca1358f828283e1558cf6a402b6cbea34" #branch = "master" default-features = false diff --git a/flake.nix b/flake.nix index 04dee681..faff87d6 100644 --- a/flake.nix +++ b/flake.nix @@ -64,8 +64,10 @@ patches = []; cmakeFlags = pkgs.lib.subtractLists [ - # no real reason to have snappy, no one uses this + # no real reason to have snappy or zlib, no one uses this "-DWITH_SNAPPY=1" + "-DZLIB=1" + "-DWITH_ZLIB=1" # we dont need to use ldb or sst_dump (core_tools) "-DWITH_CORE_TOOLS=1" # we dont need to build rocksdb tests @@ -82,6 +84,8 @@ ++ [ # no real reason to have snappy, no one uses this "-DWITH_SNAPPY=0" + "-DZLIB=0" + "-DWITH_ZLIB=0" # we dont need to use ldb or sst_dump (core_tools) "-DWITH_CORE_TOOLS=0" # we dont need trace tools @@ -171,7 +175,8 @@ sccache ] # liburing is Linux-exclusive - ++ lib.optional stdenv.hostPlatform.isLinux liburing) + ++ lib.optional stdenv.hostPlatform.isLinux liburing + ++ lib.optional stdenv.hostPlatform.isLinux numactl) ++ scope.main.buildInputs ++ scope.main.propagatedBuildInputs ++ scope.main.nativeBuildInputs; diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 138340a4..276b8b6a 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -379,8 +379,8 @@ impl Service { .active_local_users_in_room(&pdu.room_id) .map(ToOwned::to_owned) // Don't notify the sender of their own events, and dont send from ignored users - .ready_filter(|user| user != &pdu.sender) - .filter_map(|recipient_user| async move { (!self.services.users.user_is_ignored(&pdu.sender, recipient_user).await).then_some(recipient_user) }) + .ready_filter(|user| *user != pdu.sender) + .filter_map(|recipient_user| async move { (!self.services.users.user_is_ignored(&pdu.sender, &recipient_user).await).then_some(recipient_user) }) .collect() .await; From d80e61cbee21706454d1033ba46b51e4dcbb8679 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 17:53:23 -0500 Subject: [PATCH 1545/2091] bump ring to 0.17.12 Signed-off-by: June Clementine Strawberry --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2ade8b83..3a57df7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2364,7 +2364,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.48.5", ] [[package]] @@ -3474,9 +3474,9 @@ checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" [[package]] name = "ring" -version = "0.17.11" +version = "0.17.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73" +checksum = "ed9b823fa29b721a59671b41d6b06e66b29e0628e207e8b1c3ceeda701ec928d" dependencies = [ "cc", "cfg-if", From f34e0b21a3cbf7eaa737256fc57c13719b225507 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 18:12:54 -0500 Subject: [PATCH 1546/2091] remove rust-rocksdb dylib wrapper as we have a fork already Signed-off-by: June Clementine Strawberry --- Cargo.lock | 10 +----- Cargo.toml | 9 +++--- deps/rust-rocksdb/Cargo.toml | 42 ------------------------ deps/rust-rocksdb/lib.rs | 62 ------------------------------------ 4 files changed, 6 insertions(+), 117 deletions(-) delete mode 100644 deps/rust-rocksdb/Cargo.toml delete mode 100644 deps/rust-rocksdb/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 3a57df7b..9a46f008 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -848,7 +848,7 @@ dependencies = [ "log", "minicbor", "minicbor-serde", - "rust-rocksdb-uwu", + "rust-rocksdb", "serde", "serde_json", "tokio", @@ -3706,14 +3706,6 @@ source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=a5d5358ca dependencies = [ "libc", "rust-librocksdb-sys", - "serde", -] - -[[package]] -name = "rust-rocksdb-uwu" -version = "0.0.1" -dependencies = [ - "rust-rocksdb", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5edcc60a..62c90119 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -382,8 +382,9 @@ features = [ ] [workspace.dependencies.rust-rocksdb] -path = "deps/rust-rocksdb" -package = "rust-rocksdb-uwu" +git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" +rev = "a5d5358ca1358f828283e1558cf6a402b6cbea34" +default-features = false features = [ "multi-threaded-cf", "mt_static", @@ -683,7 +684,7 @@ inherits = "release" # To enable hot-reloading: # 1. Uncomment all of the rustflags here. -# 2. Uncomment crate-type=dylib in src/*/Cargo.toml and deps/rust-rocksdb/Cargo.toml +# 2. Uncomment crate-type=dylib in src/*/Cargo.toml # # opt-level, mir-opt-level, validate-mir are not known to interfere with reloading # and can be raised if build times are tolerable. @@ -751,7 +752,7 @@ inherits = "dev" # '-Clink-arg=-Wl,-z,lazy', #] -[profile.dev.package.rust-rocksdb-uwu] +[profile.dev.package.rust-rocksdb] inherits = "dev" debug = 'limited' incremental = false diff --git a/deps/rust-rocksdb/Cargo.toml b/deps/rust-rocksdb/Cargo.toml deleted file mode 100644 index f6e0a54f..00000000 --- a/deps/rust-rocksdb/Cargo.toml +++ /dev/null @@ -1,42 +0,0 @@ -[package] -name = "rust-rocksdb-uwu" -categories.workspace = true -description = "dylib wrapper for rust-rocksdb" -edition = "2024" -keywords.workspace = true -license.workspace = true -readme.workspace = true -repository.workspace = true -version = "0.0.1" - -[features] -default = ["lz4", "zstd", "bzip2"] -jemalloc = ["rust-rocksdb/jemalloc"] -io-uring = ["rust-rocksdb/io-uring"] -numa = ["rust-rocksdb/numa"] # unused by rocksdb for now -valgrind = ["rust-rocksdb/valgrind"] -lz4 = ["rust-rocksdb/lz4"] -zstd = ["rust-rocksdb/zstd"] -zlib = ["rust-rocksdb/zlib"] -bzip2 = ["rust-rocksdb/bzip2"] -rtti = ["rust-rocksdb/rtti"] -mt_static = ["rust-rocksdb/mt_static"] -multi-threaded-cf = ["rust-rocksdb/multi-threaded-cf"] -serde1 = ["rust-rocksdb/serde1"] -malloc-usable-size = ["rust-rocksdb/malloc-usable-size"] - -[dependencies.rust-rocksdb] -git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "a5d5358ca1358f828283e1558cf6a402b6cbea34" -#branch = "master" -default-features = false - -[lib] -path = "lib.rs" -crate-type = [ - "rlib", -# "dylib" -] - -[lints] -workspace = true diff --git a/deps/rust-rocksdb/lib.rs b/deps/rust-rocksdb/lib.rs deleted file mode 100644 index 8dbbda98..00000000 --- a/deps/rust-rocksdb/lib.rs +++ /dev/null @@ -1,62 +0,0 @@ -pub use rust_rocksdb::*; - -#[cfg_attr(not(conduwuit_mods), link(name = "rocksdb"))] -#[cfg_attr(conduwuit_mods, link(name = "rocksdb", kind = "static"))] -unsafe extern "C" { - pub unsafe fn rocksdb_list_column_families(); - pub unsafe fn rocksdb_logger_create_stderr_logger(); - pub unsafe fn rocksdb_logger_create_callback_logger(); - pub unsafe fn rocksdb_options_set_info_log(); - pub unsafe fn rocksdb_get_options_from_string(); - pub unsafe fn rocksdb_writebatch_create(); - pub unsafe fn rocksdb_writebatch_destroy(); - pub unsafe fn rocksdb_writebatch_put_cf(); - pub unsafe fn rocksdb_writebatch_delete_cf(); - pub unsafe fn rocksdb_iter_value(); - pub unsafe fn rocksdb_iter_seek_to_last(); - pub unsafe fn rocksdb_iter_seek_for_prev(); - pub unsafe fn rocksdb_iter_seek_to_first(); - pub unsafe fn rocksdb_iter_next(); - pub unsafe fn rocksdb_iter_prev(); - pub unsafe fn rocksdb_iter_seek(); - pub unsafe fn rocksdb_iter_valid(); - pub unsafe fn rocksdb_iter_get_error(); - pub unsafe fn rocksdb_iter_key(); - pub unsafe fn rocksdb_iter_destroy(); - pub unsafe fn rocksdb_livefiles(); - pub unsafe fn rocksdb_livefiles_count(); - pub unsafe fn rocksdb_livefiles_destroy(); - pub unsafe fn rocksdb_livefiles_column_family_name(); - pub unsafe fn rocksdb_livefiles_name(); - pub unsafe fn rocksdb_livefiles_size(); - pub unsafe fn rocksdb_livefiles_level(); - pub unsafe fn rocksdb_livefiles_smallestkey(); - pub unsafe fn rocksdb_livefiles_largestkey(); - pub unsafe fn rocksdb_livefiles_entries(); - pub unsafe fn rocksdb_livefiles_deletions(); - pub unsafe fn rocksdb_put_cf(); - pub unsafe fn rocksdb_delete_cf(); - pub unsafe fn rocksdb_get_pinned_cf(); - pub unsafe fn rocksdb_create_column_family(); - pub unsafe fn rocksdb_get_latest_sequence_number(); - pub unsafe fn rocksdb_batched_multi_get_cf(); - pub unsafe fn rocksdb_cancel_all_background_work(); - pub unsafe fn rocksdb_repair_db(); - pub unsafe fn rocksdb_list_column_families_destroy(); - pub unsafe fn rocksdb_flush(); - pub unsafe fn rocksdb_flush_wal(); - pub unsafe fn rocksdb_open_column_families(); - pub unsafe fn rocksdb_open_for_read_only_column_families(); - pub unsafe fn rocksdb_open_as_secondary_column_families(); - pub unsafe fn rocksdb_open_column_families_with_ttl(); - pub unsafe fn rocksdb_open(); - pub unsafe fn rocksdb_open_for_read_only(); - pub unsafe fn rocksdb_open_with_ttl(); - pub unsafe fn rocksdb_open_as_secondary(); - pub unsafe fn rocksdb_write(); - pub unsafe fn rocksdb_create_iterator_cf(); - pub unsafe fn rocksdb_backup_engine_create_new_backup_flush(); - pub unsafe fn rocksdb_backup_engine_options_create(); - pub unsafe fn rocksdb_write_buffer_manager_destroy(); - pub unsafe fn rocksdb_options_set_ttl(); -} From fa71162c7dd943afdf78d10710914076ec2d3c85 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 18:45:14 -0500 Subject: [PATCH 1547/2091] bump rocksdb to v9.11.1 Signed-off-by: June Clementine Strawberry --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- flake.lock | 8 ++++---- flake.nix | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a46f008..56ff3c6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3684,8 +3684,8 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.32.0+9.10.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=a5d5358ca1358f828283e1558cf6a402b6cbea34#a5d5358ca1358f828283e1558cf6a402b6cbea34" +version = "0.33.0+9.11.1" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=3f4c5357243defedc849ae6227490102a9f90bef#3f4c5357243defedc849ae6227490102a9f90bef" dependencies = [ "bindgen", "bzip2-sys", @@ -3701,8 +3701,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.36.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=a5d5358ca1358f828283e1558cf6a402b6cbea34#a5d5358ca1358f828283e1558cf6a402b6cbea34" +version = "0.37.0" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=3f4c5357243defedc849ae6227490102a9f90bef#3f4c5357243defedc849ae6227490102a9f90bef" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index 62c90119..43b2d55d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -383,7 +383,7 @@ features = [ [workspace.dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "a5d5358ca1358f828283e1558cf6a402b6cbea34" +rev = "3f4c5357243defedc849ae6227490102a9f90bef" default-features = false features = [ "multi-threaded-cf", diff --git a/flake.lock b/flake.lock index a1bd423f..3a43c4cd 100644 --- a/flake.lock +++ b/flake.lock @@ -567,16 +567,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1741234703, - "narHash": "sha256-sT5g/RM9vrwY6AmjSfl4RoJPGtcJCkZCsxiX3PFJgKQ=", + "lastModified": 1741303627, + "narHash": "sha256-7HpydEinYHvskC4vkl1Yie2kg2yShfZbREAyQMkvEUc=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "185593ce4534091e57025e9f3571dbf681c04631", + "rev": "cecee0e4fbff2b69e3edc6e9b5b751d8098a3ba1", "type": "github" }, "original": { "owner": "girlbossceo", - "ref": "v9.9.3", + "ref": "v9.11.1", "repo": "rocksdb", "type": "github" } diff --git a/flake.nix b/flake.nix index faff87d6..8f08a7d9 100644 --- a/flake.nix +++ b/flake.nix @@ -9,7 +9,7 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.9.3"; flake = false; }; + rocksdb = { url = "github:girlbossceo/rocksdb?ref=v9.11.1"; flake = false; }; liburing = { url = "github:axboe/liburing?ref=master"; flake = false; }; }; From 20dd1d148dd31948d9055c5a19ba8f8e13041363 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 18:45:48 -0500 Subject: [PATCH 1548/2091] add new complement test results Signed-off-by: June Clementine Strawberry --- .../complement/test_results.jsonl | 425 +++++++++++++++++- 1 file changed, 421 insertions(+), 4 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 11339049..fed43b48 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -1,5 +1,26 @@ {"Action":"pass","Test":"TestACLs"} +{"Action":"pass","Test":"TestAddAccountData"} +{"Action":"pass","Test":"TestAddAccountData/Can_add_global_account_data"} +{"Action":"pass","Test":"TestAddAccountData/Can_add_room_account_data"} +{"Action":"fail","Test":"TestArchivedRoomsHistory"} +{"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_has_events"} +{"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_has_events/incremental_sync"} +{"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_has_events/initial_sync"} +{"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_is_empty"} +{"Action":"skip","Test":"TestArchivedRoomsHistory/timeline_is_empty/incremental_sync"} +{"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_is_empty/initial_sync"} +{"Action":"fail","Test":"TestAsyncUpload"} +{"Action":"fail","Test":"TestAsyncUpload/Cannot_upload_to_a_media_ID_that_has_already_been_uploaded_to"} +{"Action":"fail","Test":"TestAsyncUpload/Create_media"} +{"Action":"fail","Test":"TestAsyncUpload/Download_media"} +{"Action":"fail","Test":"TestAsyncUpload/Download_media_over__matrix/client/v1/media/download"} +{"Action":"fail","Test":"TestAsyncUpload/Not_yet_uploaded"} +{"Action":"fail","Test":"TestAsyncUpload/Upload_media"} +{"Action":"pass","Test":"TestAvatarUrlUpdate"} {"Action":"pass","Test":"TestBannedUserCannotSendJoin"} +{"Action":"skip","Test":"TestCanRegisterAdmin"} +{"Action":"pass","Test":"TestCannotKickLeftUser"} +{"Action":"fail","Test":"TestCannotKickNonPresentUser"} {"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room"} {"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/event_with_mismatched_state_key"} {"Action":"pass","Test":"TestCannotSendKnockViaSendKnockInMSC3787Room/invite_event"} @@ -42,30 +63,124 @@ {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/knock_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/non-state_membership_event"} {"Action":"pass","Test":"TestCannotSendNonLeaveViaSendLeaveV2/regular_event"} +{"Action":"pass","Test":"TestChangePassword"} +{"Action":"pass","Test":"TestChangePassword/After_changing_password,_a_different_session_no_longer_works_by_default"} +{"Action":"pass","Test":"TestChangePassword/After_changing_password,_can't_log_in_with_old_password"} +{"Action":"pass","Test":"TestChangePassword/After_changing_password,_can_log_in_with_new_password"} +{"Action":"pass","Test":"TestChangePassword/After_changing_password,_different_sessions_can_optionally_be_kept"} +{"Action":"pass","Test":"TestChangePassword/After_changing_password,_existing_session_still_works"} +{"Action":"fail","Test":"TestChangePasswordPushers"} +{"Action":"fail","Test":"TestChangePasswordPushers/Pushers_created_with_a_different_access_token_are_deleted_on_password_change"} +{"Action":"pass","Test":"TestChangePasswordPushers/Pushers_created_with_the_same_access_token_are_not_deleted_on_password_change"} +{"Action":"fail","Test":"TestClientSpacesSummary"} +{"Action":"fail","Test":"TestClientSpacesSummary/max_depth"} +{"Action":"fail","Test":"TestClientSpacesSummary/pagination"} +{"Action":"fail","Test":"TestClientSpacesSummary/query_whole_graph"} +{"Action":"fail","Test":"TestClientSpacesSummary/redact_link"} +{"Action":"fail","Test":"TestClientSpacesSummary/suggested_only"} +{"Action":"fail","Test":"TestClientSpacesSummaryJoinRules"} +{"Action":"pass","Test":"TestContent"} +{"Action":"pass","Test":"TestContentCSAPIMediaV1"} {"Action":"pass","Test":"TestContentMediaV1"} +{"Action":"fail","Test":"TestCumulativeJoinLeaveJoinSync"} +{"Action":"pass","Test":"TestDeactivateAccount"} +{"Action":"pass","Test":"TestDeactivateAccount/After_deactivating_account,_can't_log_in_with_password"} +{"Action":"pass","Test":"TestDeactivateAccount/Can't_deactivate_account_with_wrong_password"} +{"Action":"pass","Test":"TestDeactivateAccount/Can_deactivate_account"} +{"Action":"pass","Test":"TestDeactivateAccount/Password_flow_is_available"} +{"Action":"fail","Test":"TestDelayedEvents"} +{"Action":"fail","Test":"TestDelayedEvents/cannot_update_a_delayed_event_with_an_invalid_action"} +{"Action":"pass","Test":"TestDelayedEvents/cannot_update_a_delayed_event_without_a_delay_ID"} +{"Action":"fail","Test":"TestDelayedEvents/cannot_update_a_delayed_event_without_a_request_body"} +{"Action":"fail","Test":"TestDelayedEvents/cannot_update_a_delayed_event_without_an_action"} +{"Action":"fail","Test":"TestDelayedEvents/delayed_events_are_empty_on_startup"} +{"Action":"fail","Test":"TestDelayedEvents/delayed_message_events_are_sent_on_timeout"} +{"Action":"fail","Test":"TestDelayedEvents/delayed_state_events_are_cancelled_by_a_more_recent_state_event_from_another_user"} +{"Action":"fail","Test":"TestDelayedEvents/delayed_state_events_are_cancelled_by_a_more_recent_state_event_from_the_same_user"} +{"Action":"skip","Test":"TestDelayedEvents/delayed_state_events_are_kept_on_server_restart"} +{"Action":"fail","Test":"TestDelayedEvents/delayed_state_events_are_sent_on_timeout"} +{"Action":"fail","Test":"TestDelayedEvents/delayed_state_events_can_be_cancelled"} +{"Action":"fail","Test":"TestDelayedEvents/delayed_state_events_can_be_restarted"} +{"Action":"fail","Test":"TestDelayedEvents/delayed_state_events_can_be_sent_on_request"} +{"Action":"pass","Test":"TestDelayedEvents/parallel"} +{"Action":"pass","Test":"TestDelayedEvents/parallel/cannot_cancel_a_delayed_event_without_a_matching_delay_ID"} +{"Action":"pass","Test":"TestDelayedEvents/parallel/cannot_restart_a_delayed_event_without_a_matching_delay_ID"} +{"Action":"pass","Test":"TestDelayedEvents/parallel/cannot_send_a_delayed_event_without_a_matching_delay_ID"} +{"Action":"fail","Test":"TestDeletingDeviceRemovesDeviceLocalNotificationSettings"} +{"Action":"fail","Test":"TestDeletingDeviceRemovesDeviceLocalNotificationSettings/Deleting_a_user's_device_should_delete_any_local_notification_settings_entries_from_their_account_data"} +{"Action":"pass","Test":"TestDemotingUsersViaUsersDefault"} +{"Action":"fail","Test":"TestDeviceListUpdates"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_joining_a_room_with_a_local_user"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_joining_a_room_with_a_remote_user"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_leaving_a_room_with_a_local_user"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_leaving_a_room_with_a_remote_user"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_local_user_joins_a_room"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_local_user_leaves_a_room"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_local_user_rejoins_a_room"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_remote_user_joins_a_room"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_remote_user_leaves_a_room"} +{"Action":"fail","Test":"TestDeviceListUpdates/when_remote_user_rejoins_a_room"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation/good_connectivity"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation/interrupted_connectivity"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederation/stopped_server"} {"Action":"fail","Test":"TestDeviceListsUpdateOverFederationOnRoomJoin"} +{"Action":"fail","Test":"TestDeviceManagement"} +{"Action":"fail","Test":"TestDeviceManagement/DELETE_/device/{deviceId}"} +{"Action":"fail","Test":"TestDeviceManagement/DELETE_/device/{deviceId}_requires_UI_auth_user_to_match_device_owner"} +{"Action":"pass","Test":"TestDeviceManagement/GET_/device/{deviceId}"} +{"Action":"pass","Test":"TestDeviceManagement/GET_/device/{deviceId}_gives_a_404_for_unknown_devices"} +{"Action":"pass","Test":"TestDeviceManagement/GET_/devices"} +{"Action":"pass","Test":"TestDeviceManagement/PUT_/device/{deviceId}_gives_a_404_for_unknown_devices"} +{"Action":"pass","Test":"TestDeviceManagement/PUT_/device/{deviceId}_updates_device_fields"} +{"Action":"pass","Test":"TestDisplayNameUpdate"} +{"Action":"fail","Test":"TestE2EKeyBackupReplaceRoomKeyRules"} +{"Action":"fail","Test":"TestE2EKeyBackupReplaceRoomKeyRules/parallel"} +{"Action":"fail","Test":"TestE2EKeyBackupReplaceRoomKeyRules/parallel/{isVerified:false_firstMessageIndex:10_forwardedCount:5}"} +{"Action":"fail","Test":"TestE2EKeyBackupReplaceRoomKeyRules/parallel/{isVerified:true_firstMessageIndex:10_forwardedCount:5}"} +{"Action":"pass","Test":"TestEvent"} +{"Action":"pass","Test":"TestEvent/Parallel"} +{"Action":"pass","Test":"TestEvent/Parallel/Large_Event"} +{"Action":"pass","Test":"TestEvent/Parallel/Large_State_Event"} {"Action":"pass","Test":"TestEventAuth"} {"Action":"pass","Test":"TestEventAuth/returns_auth_events_for_the_requested_event"} {"Action":"pass","Test":"TestEventAuth/returns_the_auth_chain_for_the_requested_event"} -{"Action":"pass","Test":"TestFederatedClientSpaces"} +{"Action":"fail","Test":"TestEventRelationships"} +{"Action":"fail","Test":"TestFederatedClientSpaces"} +{"Action":"fail","Test":"TestFederatedEventRelationships"} {"Action":"fail","Test":"TestFederationKeyUploadQuery"} {"Action":"fail","Test":"TestFederationKeyUploadQuery/Can_claim_remote_one_time_key_using_POST"} {"Action":"fail","Test":"TestFederationKeyUploadQuery/Can_query_remote_device_keys_using_POST"} {"Action":"pass","Test":"TestFederationRedactSendsWithoutEvent"} {"Action":"pass","Test":"TestFederationRejectInvite"} -{"Action":"pass","Test":"TestFederationRoomsInvite"} -{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel"} +{"Action":"fail","Test":"TestFederationRoomsInvite"} +{"Action":"fail","Test":"TestFederationRoomsInvite/Parallel"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation"} -{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_for_empty_room"} +{"Action":"fail","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_for_empty_room"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_several_times"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_has_'is_direct'_flag_in_prev_content_after_joining"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_join_the_room_when_homeserver_is_already_participating_in_the_room"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_reject_invite_when_homeserver_is_already_participating_in_the_room"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_see_room_metadata"} {"Action":"pass","Test":"TestFederationThumbnail"} +{"Action":"pass","Test":"TestFetchEvent"} +{"Action":"fail","Test":"TestFetchEventNonWorldReadable"} +{"Action":"pass","Test":"TestFetchEventWorldReadable"} +{"Action":"fail","Test":"TestFetchHistoricalInvitedEventFromBeforeInvite"} +{"Action":"pass","Test":"TestFetchHistoricalInvitedEventFromBetweenInvite"} +{"Action":"fail","Test":"TestFetchHistoricalJoinedEventDenied"} +{"Action":"pass","Test":"TestFetchHistoricalSharedEvent"} +{"Action":"pass","Test":"TestFetchMessagesFromNonExistentRoom"} +{"Action":"pass","Test":"TestFilter"} +{"Action":"fail","Test":"TestFilterMessagesByRelType"} +{"Action":"fail","Test":"TestGappedSyncLeaveSection"} +{"Action":"fail","Test":"TestGetFilteredRoomMembers"} +{"Action":"fail","Test":"TestGetFilteredRoomMembers/membership/join"} +{"Action":"fail","Test":"TestGetFilteredRoomMembers/membership/leave"} +{"Action":"fail","Test":"TestGetFilteredRoomMembers/not_membership"} {"Action":"fail","Test":"TestGetMissingEventsGapFilling"} +{"Action":"pass","Test":"TestGetRoomMembers"} +{"Action":"fail","Test":"TestGetRoomMembersAtPoint"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_invited_visibility"} {"Action":"fail","Test":"TestInboundCanReturnMissingEvents/Inbound_federation_can_return_missing_events_for_joined_visibility"} @@ -76,15 +191,41 @@ {"Action":"pass","Test":"TestInboundFederationProfile/Inbound_federation_can_query_profile_data"} {"Action":"pass","Test":"TestInboundFederationProfile/Non-numeric_ports_in_server_names_are_rejected"} {"Action":"fail","Test":"TestInboundFederationRejectsEventsWithRejectedAuthEvents"} +{"Action":"fail","Test":"TestInviteFromIgnoredUsersDoesNotAppearInSync"} {"Action":"pass","Test":"TestIsDirectFlagFederation"} {"Action":"pass","Test":"TestIsDirectFlagLocal"} {"Action":"pass","Test":"TestJoinFederatedRoomFailOver"} +{"Action":"fail","Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser"} +{"Action":"fail","Test":"TestJoinFederatedRoomFromApplicationServiceBridgeUser/join_remote_federated_room_as_application_service_user"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_missing_signatures_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_bad_signatures_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_state_with_unverifiable_auth_events_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinFederatedRoomWithUnverifiableEvents//send_join_response_with_unobtainable_keys_shouldn't_block_room_join"} {"Action":"pass","Test":"TestJoinViaRoomIDAndServerName"} +{"Action":"fail","Test":"TestJson"} +{"Action":"fail","Test":"TestJson/Parallel"} +{"Action":"fail","Test":"TestJson/Parallel/Invalid_JSON_special_values"} +{"Action":"fail","Test":"TestJson/Parallel/Invalid_numerical_values"} +{"Action":"fail","Test":"TestJumpToDateEndpoint"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/can_paginate_after_getting_remote_event_from_timestamp_to_event_endpoint"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/looking_backwards,_should_be_able_to_find_event_that_was_sent_before_we_joined"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/looking_forwards,_should_be_able_to_find_event_that_was_sent_before_we_joined"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/federation/when_looking_backwards_before_the_room_was_created,_should_be_able_to_find_event_that_was_imported"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_event_after_given_timestmap"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_event_before_given_timestmap"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_after_given_timestmap_when_all_message_timestamps_are_the_same"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_find_next_event_topologically_before_given_timestamp_when_all_message_timestamps_are_the_same"} +{"Action":"pass","Test":"TestJumpToDateEndpoint/parallel/should_find_nothing_after_the_latest_timestmap"} +{"Action":"pass","Test":"TestJumpToDateEndpoint/parallel/should_find_nothing_before_the_earliest_timestmap"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_private_room_you_are_not_a_member_of"} +{"Action":"fail","Test":"TestJumpToDateEndpoint/parallel/should_not_be_able_to_query_a_public_room_you_are_not_a_member_of"} +{"Action":"fail","Test":"TestKeyChangesLocal"} +{"Action":"fail","Test":"TestKeyChangesLocal/New_login_should_create_a_device_lists.changed_entry"} +{"Action":"fail","Test":"TestKeyClaimOrdering"} +{"Action":"pass","Test":"TestKeysQueryWithDeviceIDAsObjectFails"} {"Action":"fail","Test":"TestKnockRoomsInPublicRoomsDirectory"} {"Action":"fail","Test":"TestKnockRoomsInPublicRoomsDirectoryInMSC3787Room"} {"Action":"fail","Test":"TestKnocking"} @@ -139,9 +280,35 @@ {"Action":"fail","Test":"TestKnockingInMSC3787Room/Knocking_on_a_room_with_join_rule_'knock'_should_succeed#01"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} +{"Action":"pass","Test":"TestLeakyTyping"} +{"Action":"fail","Test":"TestLeaveEventInviteRejection"} +{"Action":"fail","Test":"TestLeaveEventVisibility"} +{"Action":"fail","Test":"TestLeftRoomFixture"} +{"Action":"fail","Test":"TestLeftRoomFixture/Can_get_'m.room.name'_state_for_a_departed_room"} +{"Action":"fail","Test":"TestLeftRoomFixture/Can_get_rooms/{roomId}/members_for_a_departed_room"} +{"Action":"pass","Test":"TestLeftRoomFixture/Can_get_rooms/{roomId}/messages_for_a_departed_room"} +{"Action":"fail","Test":"TestLeftRoomFixture/Can_get_rooms/{roomId}/state_for_a_departed_room"} +{"Action":"pass","Test":"TestLeftRoomFixture/Getting_messages_going_forward_is_limited_for_a_departed_room"} {"Action":"pass","Test":"TestLocalPngThumbnail"} {"Action":"pass","Test":"TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint"} {"Action":"pass","Test":"TestLocalPngThumbnail/test_/_matrix/media/v3_endpoint"} +{"Action":"fail","Test":"TestLogin"} +{"Action":"fail","Test":"TestLogin/parallel"} +{"Action":"pass","Test":"TestLogin/parallel/GET_/login_yields_a_set_of_flows"} +{"Action":"fail","Test":"TestLogin/parallel/Login_with_uppercase_username_works_and_GET_/whoami_afterwards_also"} +{"Action":"pass","Test":"TestLogin/parallel/POST_/login_as_non-existing_user_is_rejected"} +{"Action":"pass","Test":"TestLogin/parallel/POST_/login_can_log_in_as_a_user_with_just_the_local_part_of_the_id"} +{"Action":"pass","Test":"TestLogin/parallel/POST_/login_can_login_as_user"} +{"Action":"pass","Test":"TestLogin/parallel/POST_/login_returns_the_same_device_id_as_that_in_the_request"} +{"Action":"pass","Test":"TestLogin/parallel/POST_/login_wrong_password_is_rejected"} +{"Action":"pass","Test":"TestLogout"} +{"Action":"pass","Test":"TestLogout/Can_logout_all_devices"} +{"Action":"pass","Test":"TestLogout/Can_logout_current_device"} +{"Action":"pass","Test":"TestLogout/Request_to_logout_with_invalid_an_access_token_is_rejected"} +{"Action":"pass","Test":"TestLogout/Request_to_logout_without_an_access_token_is_rejected"} +{"Action":"fail","Test":"TestMSC3757OwnedState"} +{"Action":"pass","Test":"TestMSC3967"} +{"Action":"pass","Test":"TestMediaConfig"} {"Action":"pass","Test":"TestMediaFilenames"} {"Action":"pass","Test":"TestMediaFilenames/Parallel"} {"Action":"pass","Test":"TestMediaFilenames/Parallel/ASCII"} @@ -178,11 +345,74 @@ {"Action":"pass","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_locally"} {"Action":"pass","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_download_without_a_file_name_over_federation"} {"Action":"pass","Test":"TestMediaWithoutFileNameCSMediaV1/parallel/Can_upload_without_a_file_name"} +{"Action":"fail","Test":"TestMembersLocal"} +{"Action":"fail","Test":"TestMembersLocal/Parallel"} +{"Action":"pass","Test":"TestMembersLocal/Parallel/Existing_members_see_new_members'_join_events"} +{"Action":"fail","Test":"TestMembersLocal/Parallel/Existing_members_see_new_members'_presence_(in_incremental_sync)"} +{"Action":"pass","Test":"TestMembersLocal/Parallel/Existing_members_see_new_members'_presence_(in_initial_sync)"} +{"Action":"pass","Test":"TestMembersLocal/Parallel/New_room_members_see_their_own_join_event"} +{"Action":"fail","Test":"TestMembershipOnEvents"} {"Action":"fail","Test":"TestNetworkPartitionOrdering"} +{"Action":"pass","Test":"TestNotPresentUserCannotBanOthers"} +{"Action":"fail","Test":"TestOlderLeftRoomsNotInLeaveSection"} +{"Action":"fail","Test":"TestOutboundFederationEventSizeGetMissingEvents"} {"Action":"fail","Test":"TestOutboundFederationIgnoresMissingEventWithBadJSONForRoomVersion6"} {"Action":"pass","Test":"TestOutboundFederationProfile"} {"Action":"pass","Test":"TestOutboundFederationProfile/Outbound_federation_can_query_profile_data"} {"Action":"pass","Test":"TestOutboundFederationSend"} +{"Action":"fail","Test":"TestPollsLocalPushRules"} +{"Action":"fail","Test":"TestPollsLocalPushRules/Polls_push_rules_are_correctly_presented_to_the_client"} +{"Action":"pass","Test":"TestPowerLevels"} +{"Action":"pass","Test":"TestPowerLevels/GET_/rooms/:room_id/state/m.room.power_levels_can_fetch_levels"} +{"Action":"pass","Test":"TestPowerLevels/PUT_/rooms/:room_id/state/m.room.power_levels_can_set_levels"} +{"Action":"pass","Test":"TestPowerLevels/PUT_power_levels_should_not_explode_if_the_old_power_levels_were_empty"} +{"Action":"fail","Test":"TestPresence"} +{"Action":"fail","Test":"TestPresence/GET_/presence/:user_id/status_fetches_initial_status"} +{"Action":"pass","Test":"TestPresence/PUT_/presence/:user_id/status_updates_my_presence"} +{"Action":"pass","Test":"TestPresence/Presence_can_be_set_from_sync"} +{"Action":"pass","Test":"TestPresence/Presence_changes_are_reported_to_local_room_members"} +{"Action":"pass","Test":"TestPresence/Presence_changes_to_UNAVAILABLE_are_reported_to_local_room_members"} +{"Action":"pass","Test":"TestPresenceSyncDifferentRooms"} +{"Action":"pass","Test":"TestProfileAvatarURL"} +{"Action":"pass","Test":"TestProfileAvatarURL/GET_/profile/:user_id/avatar_url_publicly_accessible"} +{"Action":"pass","Test":"TestProfileAvatarURL/PUT_/profile/:user_id/avatar_url_sets_my_avatar"} +{"Action":"pass","Test":"TestProfileDisplayName"} +{"Action":"pass","Test":"TestProfileDisplayName/GET_/profile/:user_id/displayname_publicly_accessible"} +{"Action":"pass","Test":"TestProfileDisplayName/PUT_/profile/:user_id/displayname_sets_my_name"} +{"Action":"pass","Test":"TestPushRuleCacheHealth"} +{"Action":"pass","Test":"TestPushSync"} +{"Action":"pass","Test":"TestPushSync/Adding_a_push_rule_wakes_up_an_incremental_/sync"} +{"Action":"pass","Test":"TestPushSync/Disabling_a_push_rule_wakes_up_an_incremental_/sync"} +{"Action":"pass","Test":"TestPushSync/Enabling_a_push_rule_wakes_up_an_incremental_/sync"} +{"Action":"pass","Test":"TestPushSync/Push_rules_come_down_in_an_initial_/sync"} +{"Action":"pass","Test":"TestPushSync/Setting_actions_for_a_push_rule_wakes_up_an_incremental_/sync"} +{"Action":"pass","Test":"TestRegistration"} +{"Action":"pass","Test":"TestRegistration/parallel"} +{"Action":"pass","Test":"TestRegistration/parallel/GET_/register/available_returns_M_INVALID_USERNAME_for_invalid_user_name"} +{"Action":"pass","Test":"TestRegistration/parallel/GET_/register/available_returns_M_USER_IN_USE_for_registered_user_name"} +{"Action":"pass","Test":"TestRegistration/parallel/GET_/register/available_returns_available_for_unregistered_user_name"} +{"Action":"skip","Test":"TestRegistration/parallel/POST_/_synapse/admin/v1/register_admin_with_shared_secret"} +{"Action":"skip","Test":"TestRegistration/parallel/POST_/_synapse/admin/v1/register_with_shared_secret"} +{"Action":"skip","Test":"TestRegistration/parallel/POST_/_synapse/admin/v1/register_with_shared_secret_disallows_symbols"} +{"Action":"skip","Test":"TestRegistration/parallel/POST_/_synapse/admin/v1/register_with_shared_secret_downcases_capitals"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_allows_registration_of_usernames_with_"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_allows_registration_of_usernames_with_/-"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_allows_registration_of_usernames_with_/."} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_allows_registration_of_usernames_with_//"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_allows_registration_of_usernames_with_/3"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_allows_registration_of_usernames_with_/="} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_allows_registration_of_usernames_with_/_"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_allows_registration_of_usernames_with_/q"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_can_create_a_user"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_downcases_capitals_in_usernames"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_rejects_if_user_already_exists"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_rejects_usernames_with_special_characters"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_/register_returns_the_same_device_id_as_that_in_the_request"} +{"Action":"pass","Test":"TestRegistration/parallel/POST_{}_returns_a_set_of_flows"} +{"Action":"pass","Test":"TestRegistration/parallel/Registration_accepts_non-ascii_passwords"} +{"Action":"pass","Test":"TestRelations"} +{"Action":"fail","Test":"TestRelationsPagination"} +{"Action":"pass","Test":"TestRelationsPaginationSync"} {"Action":"pass","Test":"TestRemoteAliasRequestsUnderstandUnicode"} {"Action":"pass","Test":"TestRemotePngThumbnail"} {"Action":"pass","Test":"TestRemotePngThumbnail/test_/_matrix/client/v1/media_endpoint"} @@ -191,6 +421,13 @@ {"Action":"fail","Test":"TestRemotePresence/Presence_changes_are_also_reported_to_remote_room_members"} {"Action":"fail","Test":"TestRemotePresence/Presence_changes_to_UNAVAILABLE_are_reported_to_remote_room_members"} {"Action":"pass","Test":"TestRemoteTyping"} +{"Action":"fail","Test":"TestRemovingAccountData"} +{"Action":"fail","Test":"TestRemovingAccountData/Deleting_a_user's_account_data_via_DELETE_works"} +{"Action":"fail","Test":"TestRemovingAccountData/Deleting_a_user's_account_data_via_PUT_works"} +{"Action":"fail","Test":"TestRemovingAccountData/Deleting_a_user's_room_account_data_via_PUT_works"} +{"Action":"fail","Test":"TestRemovingAccountData/Deleting_a_user's_room_data_via_DELETE_works"} +{"Action":"fail","Test":"TestRequestEncodingFails"} +{"Action":"fail","Test":"TestRequestEncodingFails/POST_rejects_invalid_utf-8_in_JSON"} {"Action":"fail","Test":"TestRestrictedRoomsLocalJoin"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_fail_initially"} {"Action":"pass","Test":"TestRestrictedRoomsLocalJoin/Join_should_fail_when_left_allowed_room"} @@ -221,12 +458,170 @@ {"Action":"fail","Test":"TestRestrictedRoomsRemoteJoinLocalUserInMSC3787Room"} {"Action":"pass","Test":"TestRestrictedRoomsSpacesSummaryFederation"} {"Action":"fail","Test":"TestRestrictedRoomsSpacesSummaryLocal"} +{"Action":"pass","Test":"TestRoomAlias"} +{"Action":"pass","Test":"TestRoomAlias/Parallel"} +{"Action":"pass","Test":"TestRoomAlias/Parallel/GET_/rooms/:room_id/aliases_lists_aliases"} +{"Action":"pass","Test":"TestRoomAlias/Parallel/Only_room_members_can_list_aliases_of_a_room"} +{"Action":"pass","Test":"TestRoomAlias/Parallel/PUT_/directory/room/:room_alias_creates_alias"} +{"Action":"pass","Test":"TestRoomAlias/Parallel/Room_aliases_can_contain_Unicode"} +{"Action":"fail","Test":"TestRoomCanonicalAlias"} +{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel"} +{"Action":"pass","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_accepts_present_aliases"} +{"Action":"pass","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_accepts_present_alt_aliases"} +{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_alias_pointing_to_different_local_room"} +{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_alt_alias_pointing_to_different_local_room"} +{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_invalid_aliases"} +{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_invalid_aliases#01"} +{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_missing_aliases"} +{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_missing_aliases#01"} +{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_setting_rejects_deleted_aliases"} +{"Action":"pass","Test":"TestRoomCreate"} +{"Action":"pass","Test":"TestRoomCreate/Parallel"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/Can_/sync_newly_created_room"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/POST_/createRoom_creates_a_room_with_the_given_version"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/POST_/createRoom_ignores_attempts_to_set_the_room_version_via_creation_content"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/POST_/createRoom_makes_a_private_room"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/POST_/createRoom_makes_a_private_room_with_invites"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/POST_/createRoom_makes_a_public_room"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/POST_/createRoom_makes_a_room_with_a_name"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/POST_/createRoom_makes_a_room_with_a_topic"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/POST_/createRoom_rejects_attempts_to_create_rooms_with_numeric_versions"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/POST_/createRoom_rejects_attempts_to_create_rooms_with_unknown_versions"} +{"Action":"pass","Test":"TestRoomCreate/Parallel/Rooms_can_be_created_with_an_initial_invite_list_(SYN-205)"} +{"Action":"fail","Test":"TestRoomCreationReportsEventsToMyself"} +{"Action":"fail","Test":"TestRoomCreationReportsEventsToMyself/parallel"} +{"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Joining_room_twice_is_idempotent"} +{"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Room_creation_reports_m.room.create_to_myself"} +{"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Room_creation_reports_m.room.member_to_myself"} +{"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Setting_room_topic_reports_m.room.topic_to_myself"} +{"Action":"fail","Test":"TestRoomCreationReportsEventsToMyself/parallel/Setting_state_twice_is_idempotent"} +{"Action":"fail","Test":"TestRoomDeleteAlias"} +{"Action":"fail","Test":"TestRoomDeleteAlias/Parallel"} +{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Alias_creators_can_delete_alias_with_no_ops"} +{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Alias_creators_can_delete_canonical_alias_with_no_ops"} +{"Action":"fail","Test":"TestRoomDeleteAlias/Parallel/Can_delete_canonical_alias"} +{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Deleting_a_non-existent_alias_should_return_a_404"} +{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_in_the_default_room_configuration"} +{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_when_m.room.aliases_is_restricted"} +{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Users_can't_delete_other's_aliases"} +{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Users_with_sufficient_power-level_can_delete_other's_aliases"} +{"Action":"fail","Test":"TestRoomForget"} +{"Action":"fail","Test":"TestRoomForget/Parallel"} +{"Action":"pass","Test":"TestRoomForget/Parallel/Can't_forget_room_you're_still_in"} +{"Action":"pass","Test":"TestRoomForget/Parallel/Can_forget_room_we_weren't_an_actual_member"} +{"Action":"pass","Test":"TestRoomForget/Parallel/Can_forget_room_you've_been_kicked_from"} +{"Action":"pass","Test":"TestRoomForget/Parallel/Can_re-join_room_if_re-invited"} +{"Action":"pass","Test":"TestRoomForget/Parallel/Forgetting_room_does_not_show_up_in_v2_initial_/sync"} +{"Action":"fail","Test":"TestRoomForget/Parallel/Forgotten_room_messages_cannot_be_paginated"} +{"Action":"fail","Test":"TestRoomForget/Parallel/Leave_for_forgotten_room_shows_up_in_v2_incremental_/sync"} +{"Action":"pass","Test":"TestRoomImageRoundtrip"} +{"Action":"fail","Test":"TestRoomMembers"} +{"Action":"fail","Test":"TestRoomMembers/Parallel"} +{"Action":"pass","Test":"TestRoomMembers/Parallel/POST_/join/:room_alias_can_join_a_room"} +{"Action":"fail","Test":"TestRoomMembers/Parallel/POST_/join/:room_alias_can_join_a_room_with_custom_content"} +{"Action":"pass","Test":"TestRoomMembers/Parallel/POST_/join/:room_id_can_join_a_room"} +{"Action":"fail","Test":"TestRoomMembers/Parallel/POST_/join/:room_id_can_join_a_room_with_custom_content"} +{"Action":"pass","Test":"TestRoomMembers/Parallel/POST_/rooms/:room_id/ban_can_ban_a_user"} +{"Action":"pass","Test":"TestRoomMembers/Parallel/POST_/rooms/:room_id/invite_can_send_an_invite"} +{"Action":"pass","Test":"TestRoomMembers/Parallel/POST_/rooms/:room_id/join_can_join_a_room"} +{"Action":"pass","Test":"TestRoomMembers/Parallel/POST_/rooms/:room_id/leave_can_leave_a_room"} +{"Action":"pass","Test":"TestRoomMembers/Parallel/Test_that_we_can_be_reinvited_to_a_room_we_created"} +{"Action":"pass","Test":"TestRoomMessagesLazyLoading"} +{"Action":"pass","Test":"TestRoomMessagesLazyLoadingLocalUser"} +{"Action":"pass","Test":"TestRoomReadMarkers"} +{"Action":"pass","Test":"TestRoomReceipts"} +{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin"} +{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin/Bob_can_find_Alice_by_mxid"} +{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin/Bob_can_find_Alice_by_profile_display_name"} +{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin/Eve_can_find_Alice_by_mxid"} +{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin/Eve_can_find_Alice_by_profile_display_name"} +{"Action":"pass","Test":"TestRoomSpecificUsernameAtJoin/Eve_cannot_find_Alice_by_room-specific_name_that_Eve_is_not_privy_to"} +{"Action":"fail","Test":"TestRoomSpecificUsernameChange"} +{"Action":"fail","Test":"TestRoomSpecificUsernameChange/Bob_can_find_Alice_by_mxid"} +{"Action":"fail","Test":"TestRoomSpecificUsernameChange/Bob_can_find_Alice_by_profile_display_name"} +{"Action":"fail","Test":"TestRoomSpecificUsernameChange/Eve_can_find_Alice_by_mxid"} +{"Action":"fail","Test":"TestRoomSpecificUsernameChange/Eve_can_find_Alice_by_profile_display_name"} +{"Action":"pass","Test":"TestRoomSpecificUsernameChange/Eve_cannot_find_Alice_by_room-specific_name_that_Eve_is_not_privy_to"} +{"Action":"fail","Test":"TestRoomState"} +{"Action":"fail","Test":"TestRoomState/Parallel"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/directory/room/:room_alias_yields_room_ID"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/joined_rooms_lists_newly-created_room"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/publicRooms_lists_newly-created_room"} +{"Action":"fail","Test":"TestRoomState/Parallel/GET_/rooms/:room_id/joined_members_fetches_my_membership"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/rooms/:room_id/joined_members_is_forbidden_after_leaving_room"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/rooms/:room_id/state/m.room.member/:user_id?format=event_fetches_my_membership_event"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/rooms/:room_id/state/m.room.member/:user_id_fetches_my_membership"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/rooms/:room_id/state/m.room.name_gets_name"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/rooms/:room_id/state/m.room.power_levels_fetches_powerlevels"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/rooms/:room_id/state/m.room.topic_gets_topic"} +{"Action":"pass","Test":"TestRoomState/Parallel/GET_/rooms/:room_id/state_fetches_entire_room_state"} +{"Action":"pass","Test":"TestRoomState/Parallel/POST_/rooms/:room_id/state/m.room.name_sets_name"} +{"Action":"pass","Test":"TestRoomState/Parallel/PUT_/createRoom_with_creation_content"} +{"Action":"pass","Test":"TestRoomState/Parallel/PUT_/rooms/:room_id/state/m.room.topic_sets_topic"} +{"Action":"pass","Test":"TestRoomSummary"} +{"Action":"fail","Test":"TestRoomsInvite"} +{"Action":"fail","Test":"TestRoomsInvite/Parallel"} +{"Action":"pass","Test":"TestRoomsInvite/Parallel/Can_invite_users_to_invite-only_rooms"} +{"Action":"pass","Test":"TestRoomsInvite/Parallel/Invited_user_can_reject_invite"} +{"Action":"fail","Test":"TestRoomsInvite/Parallel/Invited_user_can_reject_invite_for_empty_room"} +{"Action":"pass","Test":"TestRoomsInvite/Parallel/Invited_user_can_see_room_metadata"} +{"Action":"pass","Test":"TestRoomsInvite/Parallel/Test_that_we_can_be_reinvited_to_a_room_we_created"} +{"Action":"pass","Test":"TestRoomsInvite/Parallel/Uninvited_users_cannot_join_the_room"} +{"Action":"pass","Test":"TestRoomsInvite/Parallel/Users_cannot_invite_a_user_that_is_already_in_the_room"} +{"Action":"pass","Test":"TestRoomsInvite/Parallel/Users_cannot_invite_themselves_to_a_room"} +{"Action":"fail","Test":"TestSearch"} +{"Action":"fail","Test":"TestSearch/parallel"} +{"Action":"fail","Test":"TestSearch/parallel/Can_back-paginate_search_results"} +{"Action":"fail","Test":"TestSearch/parallel/Can_get_context_around_search_results"} +{"Action":"pass","Test":"TestSearch/parallel/Can_search_for_an_event_by_body"} +{"Action":"pass","Test":"TestSearch/parallel/Search_results_with_rank_ordering_do_not_include_redacted_events"} +{"Action":"pass","Test":"TestSearch/parallel/Search_results_with_recent_ordering_do_not_include_redacted_events"} +{"Action":"pass","Test":"TestSearch/parallel/Search_works_across_an_upgraded_room_and_its_predecessor"} +{"Action":"fail","Test":"TestSendAndFetchMessage"} {"Action":"skip","Test":"TestSendJoinPartialStateResponse"} +{"Action":"pass","Test":"TestSendMessageWithTxn"} +{"Action":"pass","Test":"TestServerCapabilities"} +{"Action":"skip","Test":"TestServerNotices"} +{"Action":"fail","Test":"TestSync"} +{"Action":"fail","Test":"TestSync/parallel"} +{"Action":"pass","Test":"TestSync/parallel/Can_sync_a_joined_room"} +{"Action":"fail","Test":"TestSync/parallel/Device_list_tracking"} +{"Action":"fail","Test":"TestSync/parallel/Device_list_tracking/User_is_correctly_listed_when_they_leave,_even_when_lazy_loading_is_enabled"} +{"Action":"pass","Test":"TestSync/parallel/Full_state_sync_includes_joined_rooms"} +{"Action":"fail","Test":"TestSync/parallel/Get_presence_for_newly_joined_members_in_incremental_sync"} +{"Action":"fail","Test":"TestSync/parallel/Newly_joined_room_has_correct_timeline_in_incremental_sync"} +{"Action":"fail","Test":"TestSync/parallel/Newly_joined_room_includes_presence_in_incremental_sync"} +{"Action":"pass","Test":"TestSync/parallel/Newly_joined_room_is_included_in_an_incremental_sync"} +{"Action":"fail","Test":"TestSync/parallel/sync_should_succeed_even_if_the_sync_token_points_to_a_redaction_of_an_unknown_event"} +{"Action":"pass","Test":"TestSyncFilter"} +{"Action":"pass","Test":"TestSyncFilter/Can_create_filter"} +{"Action":"pass","Test":"TestSyncFilter/Can_download_filter"} +{"Action":"fail","Test":"TestSyncLeaveSection"} +{"Action":"fail","Test":"TestSyncLeaveSection/Left_rooms_appear_in_the_leave_section_of_full_state_sync"} +{"Action":"fail","Test":"TestSyncLeaveSection/Left_rooms_appear_in_the_leave_section_of_sync"} +{"Action":"fail","Test":"TestSyncLeaveSection/Newly_left_rooms_appear_in_the_leave_section_of_incremental_sync"} {"Action":"pass","Test":"TestSyncOmitsStateChangeOnFilteredEvents"} +{"Action":"pass","Test":"TestSyncTimelineGap"} +{"Action":"pass","Test":"TestSyncTimelineGap/full"} +{"Action":"pass","Test":"TestSyncTimelineGap/incremental"} +{"Action":"fail","Test":"TestTentativeEventualJoiningAfterRejecting"} +{"Action":"fail","Test":"TestThreadReceiptsInSyncMSC4102"} +{"Action":"fail","Test":"TestThreadedReceipts"} +{"Action":"fail","Test":"TestThreadsEndpoint"} +{"Action":"pass","Test":"TestToDeviceMessages"} {"Action":"fail","Test":"TestToDeviceMessagesOverFederation"} {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/good_connectivity"} {"Action":"pass","Test":"TestToDeviceMessagesOverFederation/interrupted_connectivity"} {"Action":"fail","Test":"TestToDeviceMessagesOverFederation/stopped_server"} +{"Action":"fail","Test":"TestTxnIdWithRefreshToken"} +{"Action":"fail","Test":"TestTxnIdempotency"} +{"Action":"pass","Test":"TestTxnIdempotencyScopedToDevice"} +{"Action":"pass","Test":"TestTxnInEvent"} +{"Action":"pass","Test":"TestTxnScopeOnLocalEcho"} +{"Action":"pass","Test":"TestTyping"} +{"Action":"pass","Test":"TestTyping/Typing_can_be_explicitly_stopped"} +{"Action":"pass","Test":"TestTyping/Typing_notification_sent_to_local_room_members"} +{"Action":"fail","Test":"TestUnbanViaInvite"} {"Action":"fail","Test":"TestUnknownEndpoints"} {"Action":"pass","Test":"TestUnknownEndpoints/Client-server_endpoints"} {"Action":"fail","Test":"TestUnknownEndpoints/Key_endpoints"} @@ -234,5 +629,27 @@ {"Action":"pass","Test":"TestUnknownEndpoints/Server-server_endpoints"} {"Action":"pass","Test":"TestUnknownEndpoints/Unknown_prefix"} {"Action":"fail","Test":"TestUnrejectRejectedEvents"} +{"Action":"fail","Test":"TestUploadKey"} +{"Action":"fail","Test":"TestUploadKey/Parallel"} +{"Action":"fail","Test":"TestUploadKey/Parallel/Can_claim_one_time_key_using_POST"} +{"Action":"pass","Test":"TestUploadKey/Parallel/Can_query_device_keys_using_POST"} +{"Action":"pass","Test":"TestUploadKey/Parallel/Can_query_specific_device_keys_using_POST"} +{"Action":"pass","Test":"TestUploadKey/Parallel/Can_upload_device_keys"} +{"Action":"fail","Test":"TestUploadKey/Parallel/Rejects_invalid_device_keys"} +{"Action":"fail","Test":"TestUploadKey/Parallel/Should_reject_keys_claiming_to_belong_to_a_different_user"} +{"Action":"pass","Test":"TestUploadKey/Parallel/query_for_user_with_no_keys_returns_empty_key_dict"} +{"Action":"pass","Test":"TestUploadKeyIdempotency"} +{"Action":"pass","Test":"TestUploadKeyIdempotencyOverlap"} +{"Action":"fail","Test":"TestUrlPreview"} {"Action":"pass","Test":"TestUserAppearsInChangedDeviceListOnJoinOverFederation"} +{"Action":"pass","Test":"TestVersionStructure"} +{"Action":"pass","Test":"TestVersionStructure/Version_responds_200_OK_with_valid_structure"} +{"Action":"pass","Test":"TestWithoutOwnedState"} +{"Action":"pass","Test":"TestWithoutOwnedState/parallel"} +{"Action":"pass","Test":"TestWithoutOwnedState/parallel/room_creator_cannot_set_state_with_a_non-member_user_ID_as_state_key"} +{"Action":"pass","Test":"TestWithoutOwnedState/parallel/room_creator_cannot_set_state_with_another_suffixed_user_ID_as_state_key"} +{"Action":"pass","Test":"TestWithoutOwnedState/parallel/room_creator_cannot_set_state_with_another_user_ID_as_state_key"} +{"Action":"pass","Test":"TestWithoutOwnedState/parallel/room_creator_cannot_set_state_with_malformed_user_ID_as_state_key"} +{"Action":"pass","Test":"TestWithoutOwnedState/parallel/room_creator_cannot_set_state_with_their_own_suffixed_user_ID_as_state_key"} +{"Action":"pass","Test":"TestWithoutOwnedState/parallel/user_can_set_state_with_their_own_user_ID_as_state_key"} {"Action":"pass","Test":"TestWriteMDirectAccountData"} From 5ad1100e0fdf41a380b445154b42bc09f38a64b5 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 19:48:06 -0500 Subject: [PATCH 1549/2091] bump our rocksdb fork Signed-off-by: June Clementine Strawberry --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- flake.lock | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 56ff3c6b..7dd24e2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3685,7 +3685,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.33.0+9.11.1" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=3f4c5357243defedc849ae6227490102a9f90bef#3f4c5357243defedc849ae6227490102a9f90bef" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=d05c8bd7ba8814de1731ec0ae29e863c8ecb7206#d05c8bd7ba8814de1731ec0ae29e863c8ecb7206" dependencies = [ "bindgen", "bzip2-sys", @@ -3702,7 +3702,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.37.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=3f4c5357243defedc849ae6227490102a9f90bef#3f4c5357243defedc849ae6227490102a9f90bef" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=d05c8bd7ba8814de1731ec0ae29e863c8ecb7206#d05c8bd7ba8814de1731ec0ae29e863c8ecb7206" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index 43b2d55d..a9f1abb3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -383,7 +383,7 @@ features = [ [workspace.dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "3f4c5357243defedc849ae6227490102a9f90bef" +rev = "d05c8bd7ba8814de1731ec0ae29e863c8ecb7206" default-features = false features = [ "multi-threaded-cf", diff --git a/flake.lock b/flake.lock index 3a43c4cd..c3292cbc 100644 --- a/flake.lock +++ b/flake.lock @@ -567,11 +567,11 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1741303627, - "narHash": "sha256-7HpydEinYHvskC4vkl1Yie2kg2yShfZbREAyQMkvEUc=", + "lastModified": 1741308171, + "narHash": "sha256-YdBvdQ75UJg5ffwNjxizpviCVwVDJnBkM8ZtGIduMgY=", "owner": "girlbossceo", "repo": "rocksdb", - "rev": "cecee0e4fbff2b69e3edc6e9b5b751d8098a3ba1", + "rev": "3ce04794bcfbbb0d2e6f81ae35fc4acf688b6986", "type": "github" }, "original": { From fe65648296b1827841c3e2a602cc78bd1af0a9b5 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 6 Mar 2025 20:10:32 -0500 Subject: [PATCH 1550/2091] remove unnecessary map_err Signed-off-by: June Clementine Strawberry --- src/service/rooms/timeline/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 276b8b6a..826a1dae 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -344,7 +344,7 @@ impl Service { let insert_lock = self.mutex_insert.lock(&pdu.room_id).await; - let count1 = self.services.globals.next_count()?; + let count1 = self.services.globals.next_count().unwrap(); // Mark as read first so the sending client doesn't get a notification even if // appending fails self.services @@ -362,13 +362,12 @@ impl Service { drop(insert_lock); - // See if the event matches any known pushers + // See if the event matches any known pushers via power level let power_levels: RoomPowerLevelsEventContent = self .services .state_accessor .room_state_get_content(&pdu.room_id, &StateEventType::RoomPowerLevels, "") .await - .map_err(|e| err!(Database(warn!("invalid m.room.power_levels event: {e}")))) .unwrap_or_default(); let sync_pdu = pdu.to_sync_room_event(); From 2c58a6efda4f0ae7fa7b5ad05758489b5ff2e5f5 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Fri, 7 Mar 2025 00:54:30 -0500 Subject: [PATCH 1551/2091] allow broken no-op deny+allow room server ACL keys Signed-off-by: June Clementine Strawberry --- src/service/rooms/event_handler/acl_check.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/service/rooms/event_handler/acl_check.rs b/src/service/rooms/event_handler/acl_check.rs index 6b432a4b..f847015b 100644 --- a/src/service/rooms/event_handler/acl_check.rs +++ b/src/service/rooms/event_handler/acl_check.rs @@ -14,14 +14,21 @@ pub async fn acl_check(&self, server_name: &ServerName, room_id: &RoomId) -> Res .room_state_get_content(room_id, &StateEventType::RoomServerAcl, "") .await .map(|c: RoomServerAclEventContent| c) - .inspect(|acl| trace!("ACL content found: {acl:?}")) - .inspect_err(|e| trace!("No ACL content found: {e:?}")) + .inspect(|acl| trace!(%room_id, "ACL content found: {acl:?}")) + .inspect_err(|e| trace!(%room_id, "No ACL content found: {e:?}")) else { return Ok(()); }; if acl_event_content.allow.is_empty() { - warn!("Ignoring broken ACL event (allow key is empty)"); + warn!(%room_id, "Ignoring broken ACL event (allow key is empty)"); + return Ok(()); + } + + if acl_event_content.deny.contains(&String::from("*")) + && acl_event_content.allow.contains(&String::from("*")) + { + warn!(%room_id, "Ignoring broken ACL event (allow key and deny key both contain wildcard \"*\""); return Ok(()); } From 4f882c3bd8adfa86edc504396f6cd45b56fd8b62 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Fri, 7 Mar 2025 00:57:39 -0500 Subject: [PATCH 1552/2091] add some ACL paw-gun checks, better `PUT` state event validation Signed-off-by: June Clementine Strawberry --- src/api/client/keys.rs | 23 +++- src/api/client/state.rs | 253 +++++++++++++++++++++++++--------------- 2 files changed, 178 insertions(+), 98 deletions(-) diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 8a7eab7e..4c1c986a 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use axum::extract::State; -use conduwuit::{Err, Error, Result, debug, err, info, result::NotFound, utils}; +use conduwuit::{Err, Error, Result, debug, debug_warn, err, info, result::NotFound, utils}; use futures::{StreamExt, stream::FuturesUnordered}; use ruma::{ OneTimeKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, @@ -41,6 +41,20 @@ pub(crate) async fn upload_keys_route( let (sender_user, sender_device) = body.sender(); for (key_id, one_time_key) in &body.one_time_keys { + if one_time_key + .deserialize() + .inspect_err(|e| { + debug_warn!( + ?key_id, + ?one_time_key, + "Invalid one time key JSON submitted by client, skipping: {e}" + ) + }) + .is_err() + { + continue; + } + services .users .add_one_time_key(sender_user, sender_device, key_id, one_time_key) @@ -48,7 +62,12 @@ pub(crate) async fn upload_keys_route( } if let Some(device_keys) = &body.device_keys { - let deser_device_keys = device_keys.deserialize()?; + let deser_device_keys = device_keys.deserialize().map_err(|e| { + err!(Request(BadJson(debug_warn!( + ?device_keys, + "Invalid device keys JSON uploaded by client: {e}" + )))) + })?; if deser_device_keys.user_id != sender_user { return Err!(Request(Unknown( diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 6353fe1c..c92091eb 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -11,6 +11,7 @@ use ruma::{ history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, join_rules::{JoinRule, RoomJoinRulesEventContent}, member::{MembershipState, RoomMemberEventContent}, + server_acl::RoomServerAclEventContent, }, }, serde::Raw, @@ -194,134 +195,194 @@ async fn allowed_to_send_state_event( ) -> Result { match event_type { | StateEventType::RoomCreate => { - return Err!(Request(BadJson( + return Err!(Request(BadJson(debug_warn!( + ?room_id, "You cannot update m.room.create after a room has been created." - ))); + )))); + }, + | StateEventType::RoomServerAcl => { + // prevents common ACL paw-guns as ACL management is difficult and prone to + // irreversible mistakes + match json.deserialize_as::() { + | Ok(acl_content) => { + if acl_content.allow.is_empty() { + return Err!(Request(BadJson(debug_warn!( + ?room_id, + "Sending an ACL event with an empty allow key will permanently \ + brick the room for non-conduwuit's as this equates to no servers \ + being allowed to participate in this room." + )))); + } + + if acl_content.deny.contains(&String::from("*")) + && acl_content.allow.contains(&String::from("*")) + { + return Err!(Request(BadJson(debug_warn!( + ?room_id, + "Sending an ACL event with a deny and allow key value of \"*\" will \ + permanently brick the room for non-conduwuit's as this equates to \ + no servers being allowed to participate in this room." + )))); + } + + if acl_content.deny.contains(&String::from("*")) + && !acl_content.is_allowed(services.globals.server_name()) + { + return Err!(Request(BadJson(debug_warn!( + ?room_id, + "Sending an ACL event with a deny key value of \"*\" and without \ + your own server name in the allow key will result in you being \ + unable to participate in this room." + )))); + } + + if !acl_content.allow.contains(&String::from("*")) + && !acl_content.is_allowed(services.globals.server_name()) + { + return Err!(Request(BadJson(debug_warn!( + ?room_id, + "Sending an ACL event for an allow key without \"*\" and without \ + your own server name in the allow key will result in you being \ + unable to participate in this room." + )))); + } + }, + | Err(e) => { + return Err!(Request(BadJson(debug_warn!( + "Room server ACL event is invalid: {e}" + )))); + }, + }; }, - // Forbid m.room.encryption if encryption is disabled | StateEventType::RoomEncryption => - if !services.globals.allow_encryption() { + // Forbid m.room.encryption if encryption is disabled + if !services.config.allow_encryption { return Err!(Request(Forbidden("Encryption is disabled on this homeserver."))); }, - // admin room is a sensitive room, it should not ever be made public | StateEventType::RoomJoinRules => { + // admin room is a sensitive room, it should not ever be made public if let Ok(admin_room_id) = services.admin.get_admin_room().await { if admin_room_id == room_id { - if let Ok(join_rule) = - serde_json::from_str::(json.json().get()) - { - if join_rule.join_rule == JoinRule::Public { - return Err!(Request(Forbidden( - "Admin room is a sensitive room, it cannot be made public" - ))); - } + match json.deserialize_as::() { + | Ok(join_rule) => + if join_rule.join_rule == JoinRule::Public { + return Err!(Request(Forbidden( + "Admin room is a sensitive room, it cannot be made public" + ))); + }, + | Err(e) => { + return Err!(Request(BadJson(debug_warn!( + "Room join rules event is invalid: {e}" + )))); + }, } } } }, - // admin room is a sensitive room, it should not ever be made world readable | StateEventType::RoomHistoryVisibility => { - if let Ok(visibility_content) = - serde_json::from_str::(json.json().get()) - { - if let Ok(admin_room_id) = services.admin.get_admin_room().await { - if admin_room_id == room_id - && visibility_content.history_visibility - == HistoryVisibility::WorldReadable - { - return Err!(Request(Forbidden( - "Admin room is a sensitive room, it cannot be made world readable \ - (public room history)." - ))); - } + // admin room is a sensitive room, it should not ever be made world readable + if let Ok(admin_room_id) = services.admin.get_admin_room().await { + match json.deserialize_as::() { + | Ok(visibility_content) => { + if admin_room_id == room_id + && visibility_content.history_visibility + == HistoryVisibility::WorldReadable + { + return Err!(Request(Forbidden( + "Admin room is a sensitive room, it cannot be made world \ + readable (public room history)." + ))); + } + }, + | Err(e) => { + return Err!(Request(BadJson(debug_warn!( + "Room history visibility event is invalid: {e}" + )))); + }, } } }, | StateEventType::RoomCanonicalAlias => { - if let Ok(canonical_alias) = - serde_json::from_str::(json.json().get()) - { - let mut aliases = canonical_alias.alt_aliases.clone(); + match json.deserialize_as::() { + | Ok(canonical_alias_content) => { + let mut aliases = canonical_alias_content.alt_aliases.clone(); - if let Some(alias) = canonical_alias.alias { - aliases.push(alias); - } + if let Some(alias) = canonical_alias_content.alias { + aliases.push(alias); + } - for alias in aliases { - if !services.globals.server_is_ours(alias.server_name()) { - return Err!(Request(Forbidden( - "canonical_alias must be for this server" + for alias in aliases { + let (alias_room_id, _servers) = + services.rooms.alias.resolve_alias(&alias, None).await?; + + if alias_room_id != room_id { + return Err!(Request(Forbidden( + "Room alias {alias} does not belong to room {room_id}" + ))); + } + } + }, + | Err(e) => { + return Err!(Request(BadJson(debug_warn!( + "Room canonical alias event is invalid: {e}" + )))); + }, + } + }, + | StateEventType::RoomMember => match json.deserialize_as::() { + | Ok(membership_content) => { + let Ok(state_key) = UserId::parse(state_key) else { + return Err!(Request(BadJson( + "Membership event has invalid or non-existent state key" + ))); + }; + + if let Some(authorising_user) = + membership_content.join_authorized_via_users_server + { + if membership_content.membership != MembershipState::Join { + return Err!(Request(BadJson( + "join_authorised_via_users_server is only for member joins" + ))); + } + + if services + .rooms + .state_cache + .is_joined(state_key, room_id) + .await + { + return Err!(Request(InvalidParam( + "{state_key} is already joined, an authorising user is not required." + ))); + } + + if !services.globals.user_is_local(&authorising_user) { + return Err!(Request(InvalidParam( + "Authorising user {authorising_user} does not belong to this \ + homeserver" ))); } if !services .rooms - .alias - .resolve_local_alias(&alias) + .state_cache + .is_joined(&authorising_user, room_id) .await - .is_ok_and(|room| room == room_id) - // Make sure it's the right room { - return Err!(Request(Forbidden( - "You are only allowed to send canonical_alias events when its \ - aliases already exist" + return Err!(Request(InvalidParam( + "Authorising user {authorising_user} is not in the room, they \ + cannot authorise the join." ))); } } - } - }, - | StateEventType::RoomMember => { - let Ok(membership_content) = - serde_json::from_str::(json.json().get()) - else { + }, + | Err(e) => { return Err!(Request(BadJson( "Membership content must have a valid JSON body with at least a valid \ - membership state." + membership state: {e}" ))); - }; - - let Ok(state_key) = UserId::parse(state_key) else { - return Err!(Request(BadJson( - "Membership event has invalid or non-existent state key" - ))); - }; - - if let Some(authorising_user) = membership_content.join_authorized_via_users_server { - if membership_content.membership != MembershipState::Join { - return Err!(Request(BadJson( - "join_authorised_via_users_server is only for member joins" - ))); - } - - if services - .rooms - .state_cache - .is_joined(state_key, room_id) - .await - { - return Err!(Request(InvalidParam( - "{state_key} is already joined, an authorising user is not required." - ))); - } - - if !services.globals.user_is_local(&authorising_user) { - return Err!(Request(InvalidParam( - "Authorising user {authorising_user} does not belong to this homeserver" - ))); - } - - if !services - .rooms - .state_cache - .is_joined(&authorising_user, room_id) - .await - { - return Err!(Request(InvalidParam( - "Authorising user {authorising_user} is not in the room, they cannot \ - authorise the join." - ))); - } - } + }, }, | _ => (), } From 8b3f62919831650a8198ca751dd1892e9889a51d Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Fri, 7 Mar 2025 00:57:47 -0500 Subject: [PATCH 1553/2091] bump rust-rocksdb Signed-off-by: June Clementine Strawberry --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7dd24e2e..a224ad0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3685,7 +3685,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.33.0+9.11.1" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=d05c8bd7ba8814de1731ec0ae29e863c8ecb7206#d05c8bd7ba8814de1731ec0ae29e863c8ecb7206" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2e692ae026881fc385f111fdcfba38bee98f1e47#2e692ae026881fc385f111fdcfba38bee98f1e47" dependencies = [ "bindgen", "bzip2-sys", @@ -3702,7 +3702,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.37.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=d05c8bd7ba8814de1731ec0ae29e863c8ecb7206#d05c8bd7ba8814de1731ec0ae29e863c8ecb7206" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2e692ae026881fc385f111fdcfba38bee98f1e47#2e692ae026881fc385f111fdcfba38bee98f1e47" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index a9f1abb3..0b08cd8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -383,7 +383,7 @@ features = [ [workspace.dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "d05c8bd7ba8814de1731ec0ae29e863c8ecb7206" +rev = "2e692ae026881fc385f111fdcfba38bee98f1e47" default-features = false features = [ "multi-threaded-cf", From 6052c0c8a2c5722a5ca057576ba174f8f72ab9e0 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Fri, 7 Mar 2025 01:04:53 -0500 Subject: [PATCH 1554/2091] ci: allow ourselves to write to the public docs directory Signed-off-by: June Clementine Strawberry --- .github/workflows/documentation.yml | 1 + conduwuit-example.toml | 2 +- src/api/client/keys.rs | 2 +- src/core/config/mod.rs | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 88e7bbe1..b5b4ff46 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -81,6 +81,7 @@ jobs: bin/nix-build-and-cache just .#book cp -r --dereference result public + chmod u+w -R public - name: Upload generated documentation (book) as normal artifact uses: actions/upload-artifact@v4 diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 541f062d..3d4b15bc 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -593,7 +593,7 @@ # Currently, conduwuit doesn't support inbound batched key requests, so # this list should only contain other Synapse servers. # -# example: ["matrix.org", "envs.net", "constellatory.net", "tchncs.de"] +# example: ["matrix.org", "envs.net", "tchncs.de"] # #trusted_servers = ["matrix.org"] diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 4c1c986a..9cd50e85 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -48,7 +48,7 @@ pub(crate) async fn upload_keys_route( ?key_id, ?one_time_key, "Invalid one time key JSON submitted by client, skipping: {e}" - ) + ); }) .is_err() { diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 5a4819e0..a82f5f53 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -713,7 +713,7 @@ pub struct Config { /// Currently, conduwuit doesn't support inbound batched key requests, so /// this list should only contain other Synapse servers. /// - /// example: ["matrix.org", "envs.net", "constellatory.net", "tchncs.de"] + /// example: ["matrix.org", "envs.net", "tchncs.de"] /// /// default: ["matrix.org"] #[serde(default = "default_trusted_servers")] From 298b58c069534833cfd027510ad7683e18d71e7a Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Mar 2025 21:44:33 -0500 Subject: [PATCH 1555/2091] set file_shape for roomsynctoken_shortstatehash to 3, remove rust-rocksdb package spec Signed-off-by: strawberry --- Cargo.toml | 21 --------------------- src/database/maps.rs | 1 + 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0b08cd8f..c48be06a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -752,27 +752,6 @@ inherits = "dev" # '-Clink-arg=-Wl,-z,lazy', #] -[profile.dev.package.rust-rocksdb] -inherits = "dev" -debug = 'limited' -incremental = false -codegen-units = 1 -opt-level = 'z' -#rustflags = [ -# '--cfg', 'conduwuit_mods', -# '-Ztls-model=initial-exec', -# '-Cprefer-dynamic=true', -# '-Zstaticlib-prefer-dynamic=true', -# '-Zstaticlib-allow-rdylib-deps=true', -# '-Zpacked-bundled-libs=true', -# '-Zplt=true', -# '-Clink-arg=-Wl,--no-as-needed', -# '-Clink-arg=-Wl,--allow-shlib-undefined', -# '-Clink-arg=-Wl,-z,lazy', -# '-Clink-arg=-Wl,-z,nodlopen', -# '-Clink-arg=-Wl,-z,nodelete', -#] - [profile.dev.package.'*'] inherits = "dev" debug = 'limited' diff --git a/src/database/maps.rs b/src/database/maps.rs index 9af45159..138bb038 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -169,6 +169,7 @@ pub(super) static MAPS: &[Descriptor] = &[ }, Descriptor { name: "roomsynctoken_shortstatehash", + file_shape: 3, val_size_hint: Some(8), block_size: 512, compression_level: 3, From 51d29bc1cbca84c001c3b4efbfca9c34a9b94f37 Mon Sep 17 00:00:00 2001 From: strawberry Date: Fri, 7 Mar 2025 21:44:52 -0500 Subject: [PATCH 1556/2091] bump complement Signed-off-by: strawberry --- flake.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flake.lock b/flake.lock index c3292cbc..03fc205c 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1740291865, - "narHash": "sha256-wl1+yCTEtvIH8vgXygnxPkaSgg4MYNKs+c9tzVytr20=", + "lastModified": 1741378155, + "narHash": "sha256-rJSfqf3q4oWxcAwENtAowLZeCi8lktwKVH9XQvvZR64=", "owner": "girlbossceo", "repo": "complement", - "rev": "35ad9d9051498fbac8ea4abff8ab7d8b1844f87b", + "rev": "1502a00d8551d0f6e8954a23e43868877c3e57d9", "type": "github" }, "original": { From 90fee4f50eb5a0f81390e088f60265ab4974370e Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Mar 2025 00:15:13 -0500 Subject: [PATCH 1557/2091] add gotestfmt log output to complement script and CI output Signed-off-by: strawberry --- .github/workflows/ci.yml | 30 +++++++++++++++++++----------- bin/complement | 22 +++++++++++++++++----- flake.nix | 1 + 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0425873..c8fef47f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,6 +176,13 @@ jobs: path: complement_test_results.jsonl if-no-files-found: error + - name: Upload Complement logs (gotestfmt) + uses: actions/upload-artifact@v4 + with: + name: complement_test_logs_gotestfmt.log + path: complement_test_logs_gotestfmt.log + if-no-files-found: error + - name: Diff Complement results with checked-in repo results run: | diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log) @@ -186,22 +193,23 @@ jobs: if: success() || failure() run: | if [ ${GH_JOB_STATUS} == 'success' ]; then - echo '# ✅ completed suwuccessfully' >> $GITHUB_STEP_SUMMARY + echo '# ✅ CI completed suwuccessfully' >> $GITHUB_STEP_SUMMARY else - echo '# CI failure' >> $GITHUB_STEP_SUMMARY + echo '# ❌ CI failed (last 100 lines of output)' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - tail -n 40 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - echo '# Complement diff results' >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + tail -n 100 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY fi - - name: Run cargo clean test artifacts to free up space - run: | - cargo clean --profile test + echo '# Complement diff results (last 100 lines)' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + echo '# Complement gotestfmt logs (last 100 lines)' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_test_logs_gotestfmt.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY build: name: Build diff --git a/bin/complement b/bin/complement index 9960299c..aec27c5b 100755 --- a/bin/complement +++ b/bin/complement @@ -10,15 +10,15 @@ set -euo pipefail COMPLEMENT_SRC="${COMPLEMENT_SRC:-$1}" # A `.jsonl` file to write test logs to -LOG_FILE="$2" +LOG_FILE="${2:-complement_test_logs.jsonl}" # A `.jsonl` file to write test results to -RESULTS_FILE="$3" +RESULTS_FILE="${3:-complement_test_results.jsonl}" OCI_IMAGE="complement-conduwuit:main" -# Complement tests that are skipped due to flakiness/reliability issues -SKIPPED_COMPLEMENT_TESTS='-skip=TestPartialStateJoin.*' +# Complement tests that are skipped due to flakiness/reliability issues or we don't implement such features and won't for a long time +#SKIPPED_COMPLEMENT_TESTS='-skip=TestPartialStateJoin.*' # $COMPLEMENT_SRC needs to be a directory to Complement source code if [ -f "$COMPLEMENT_SRC" ]; then @@ -34,6 +34,7 @@ toplevel="$(git rev-parse --show-toplevel)" pushd "$toplevel" > /dev/null +# if using macOS, use linux-complement #bin/nix-build-and-cache just .#linux-complement bin/nix-build-and-cache just .#complement @@ -45,7 +46,8 @@ set +o pipefail env \ -C "$COMPLEMENT_SRC" \ COMPLEMENT_BASE_IMAGE="$OCI_IMAGE" \ - go test -tags="conduwuit_blacklist" "$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE" + COMPLEMENT_ENABLE_DIRTY_RUNS=1 \ # reuses the same complement container for faster complement, at the possible expense of test environment pollution + go test -tags="conduwuit_blacklist" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE" set -o pipefail # Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results @@ -55,3 +57,13 @@ cat "$LOG_FILE" | jq -s -c 'sort_by(.Test)[]' | jq -c ' and .Test != null ) | {Action: .Action, Test: .Test} ' > "$RESULTS_FILE" + +grep '^{"Time":' "$LOG_FILE" | gotestfmt > "${LOG_FILE}_gotestfmt.log" + +echo "" +echo "" +echo "complement logs saved at $LOG_FILE" +echo "complement results saved at $RESULTS_FILE" +echo "complement logs in gotestfmt pretty format outputted at ${LOG_FILE}_gotestfmt.log (use an editor/terminal that interprets ANSI colours)" +echo "" +echo "" diff --git a/flake.nix b/flake.nix index 8f08a7d9..544cdd4a 100644 --- a/flake.nix +++ b/flake.nix @@ -161,6 +161,7 @@ # Needed for our script for Complement jq + gotestfmt # Needed for finding broken markdown links lychee From 5a3264980aee8f5869eb953e82c01b62c2ac5bed Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Mar 2025 01:35:26 -0500 Subject: [PATCH 1558/2091] adjust complement script to allow using your own hs OCI image without nix Signed-off-by: strawberry --- bin/complement | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/bin/complement b/bin/complement index aec27c5b..47c02843 100755 --- a/bin/complement +++ b/bin/complement @@ -15,7 +15,7 @@ LOG_FILE="${2:-complement_test_logs.jsonl}" # A `.jsonl` file to write test results to RESULTS_FILE="${3:-complement_test_results.jsonl}" -OCI_IMAGE="complement-conduwuit:main" +COMPLEMENT_OCI_IMAGE="${COMPLEMENT_OCI_IMAGE:-complement-conduwuit:main}" # Complement tests that are skipped due to flakiness/reliability issues or we don't implement such features and won't for a long time #SKIPPED_COMPLEMENT_TESTS='-skip=TestPartialStateJoin.*' @@ -34,18 +34,38 @@ toplevel="$(git rev-parse --show-toplevel)" pushd "$toplevel" > /dev/null -# if using macOS, use linux-complement -#bin/nix-build-and-cache just .#linux-complement -bin/nix-build-and-cache just .#complement +if [ ! -f "complement_oci_image.tar.gz" ]; then + echo "building complement conduwuit image" -docker load < result -popd > /dev/null + # if using macOS, use linux-complement + #bin/nix-build-and-cache just .#linux-complement + bin/nix-build-and-cache just .#complement + + echo "complement conduwuit image tar.gz built at \"result\"" + + echo "loading into docker" + docker load < result + popd > /dev/null +else + echo "skipping building a complement conduwuit image as complement_oci_image.tar.gz was already found, loading this" + + docker load < complement_oci_image.tar.gz + popd > /dev/null +fi + +echo "" +echo "running go test with:" +echo "\$COMPLEMENT_SRC: $COMPLEMENT_SRC" +echo "\$COMPLEMENT_BASE_IMAGE: $COMPLEMENT_BASE_IMAGE" +echo "\$RESULTS_FILE: $RESULTS_FILE" +echo "\$LOG_FILE: $LOG_FILE" +echo "" # It's okay (likely, even) that `go test` exits nonzero set +o pipefail env \ -C "$COMPLEMENT_SRC" \ - COMPLEMENT_BASE_IMAGE="$OCI_IMAGE" \ + COMPLEMENT_BASE_IMAGE="$COMPLEMENT_OCI_IMAGE" \ COMPLEMENT_ENABLE_DIRTY_RUNS=1 \ # reuses the same complement container for faster complement, at the possible expense of test environment pollution go test -tags="conduwuit_blacklist" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE" set -o pipefail From bb0b57efb8d8d89fce0392e7c6c34c169ba054b8 Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Mar 2025 02:30:58 -0500 Subject: [PATCH 1559/2091] bump rust-rocksdb Signed-off-by: strawberry --- Cargo.lock | 26 ++++++++++++++++++++++---- Cargo.toml | 2 +- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a224ad0f..8d4688f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -236,7 +236,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f9dd2e03ee80ca2822dd6ea431163d2ef259f2066a4d6ccaca6d9dcb386aa43" dependencies = [ - "bindgen", + "bindgen 0.69.5", "cc", "cmake", "dunce", @@ -431,6 +431,24 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.9.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn 2.0.98", +] + [[package]] name = "bit_field" version = "0.10.2" @@ -3685,9 +3703,9 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.33.0+9.11.1" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2e692ae026881fc385f111fdcfba38bee98f1e47#2e692ae026881fc385f111fdcfba38bee98f1e47" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=1c267e0bf0cc7b7702e9a329deccd89de79ef4c3#1c267e0bf0cc7b7702e9a329deccd89de79ef4c3" dependencies = [ - "bindgen", + "bindgen 0.71.1", "bzip2-sys", "cc", "glob", @@ -3702,7 +3720,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.37.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=2e692ae026881fc385f111fdcfba38bee98f1e47#2e692ae026881fc385f111fdcfba38bee98f1e47" +source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=1c267e0bf0cc7b7702e9a329deccd89de79ef4c3#1c267e0bf0cc7b7702e9a329deccd89de79ef4c3" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index c48be06a..de90e63e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -383,7 +383,7 @@ features = [ [workspace.dependencies.rust-rocksdb] git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "2e692ae026881fc385f111fdcfba38bee98f1e47" +rev = "1c267e0bf0cc7b7702e9a329deccd89de79ef4c3" default-features = false features = [ "multi-threaded-cf", From c8a730c29e3ec5c9d38028b89f3fd26ed546ef8f Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Mar 2025 03:07:42 -0500 Subject: [PATCH 1560/2091] implement MSC4267 automatically forgetting room on leave Signed-off-by: strawberry --- conduwuit-example.toml | 11 ++++++++++- src/api/client/capabilities.rs | 7 +++++++ src/core/config/mod.rs | 10 ++++++++++ src/service/rooms/state_cache/mod.rs | 8 ++++++-- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 3d4b15bc..15e6dd37 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -445,10 +445,19 @@ # #allow_federation = true -# This item is undocumented. Please contribute documentation for it. +# Allows federation requests to be made to itself +# +# This isn't intended and is very likely a bug if federation requests are +# being sent to yourself. This currently mainly exists for development +# purposes. # #federation_loopback = false +# Always calls /forget on behalf of the user if leaving a room. This is a +# part of MSC4267 "Automatically forgetting rooms on leave" +# +#forget_forced_upon_leave = false + # Set this to true to require authentication on the normally # unauthenticated profile retrieval endpoints (GET) # "/_matrix/client/v3/profile/{userId}". diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index e20af21b..470ff6ab 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -42,5 +42,12 @@ pub(crate) async fn get_capabilities_route( .set("uk.tcpip.msc4133.profile_fields", json!({"enabled": true})) .expect("this is valid JSON we created"); + capabilities + .set( + "org.matrix.msc4267.forget_forced_upon_leave", + json!({"enabled": services.config.forget_forced_upon_leave}), + ) + .expect("valid JSON we created"); + Ok(get_capabilities::v3::Response { capabilities }) } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index a82f5f53..e69a56b9 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -558,9 +558,19 @@ pub struct Config { #[serde(default = "true_fn")] pub allow_federation: bool, + /// Allows federation requests to be made to itself + /// + /// This isn't intended and is very likely a bug if federation requests are + /// being sent to yourself. This currently mainly exists for development + /// purposes. #[serde(default)] pub federation_loopback: bool, + /// Always calls /forget on behalf of the user if leaving a room. This is a + /// part of MSC4267 "Automatically forgetting rooms on leave" + #[serde(default)] + pub forget_forced_upon_leave: bool, + /// Set this to true to require authentication on the normally /// unauthenticated profile retrieval endpoints (GET) /// "/_matrix/client/v3/profile/{userId}". diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index f406eb69..23ba0520 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -28,7 +28,7 @@ use ruma::{ serde::Raw, }; -use crate::{Dep, account_data, appservice::RegistrationInfo, globals, rooms, users}; +use crate::{Dep, account_data, appservice::RegistrationInfo, config, globals, rooms, users}; pub struct Service { appservice_in_room_cache: AppServiceInRoomCache, @@ -38,6 +38,7 @@ pub struct Service { struct Services { account_data: Dep, + config: Dep, globals: Dep, state_accessor: Dep, users: Dep, @@ -70,6 +71,7 @@ impl crate::Service for Service { appservice_in_room_cache: RwLock::new(HashMap::new()), services: Services { account_data: args.depend::("account_data"), + config: args.depend::("config"), globals: args.depend::("globals"), state_accessor: args .depend::("rooms::state_accessor"), @@ -268,7 +270,9 @@ impl Service { | MembershipState::Leave | MembershipState::Ban => { self.mark_as_left(user_id, room_id); - if self.services.globals.user_is_local(user_id) { + if self.services.globals.user_is_local(user_id) + && self.services.config.forget_forced_upon_leave + { self.forget(room_id, user_id); } }, From ef96e7afac81ffa6e3335144644277e4ac28658b Mon Sep 17 00:00:00 2001 From: strawberry Date: Sat, 8 Mar 2025 13:52:56 -0500 Subject: [PATCH 1561/2091] add cargo auditable for future use, ignore paste dependency being unmaintained for now Signed-off-by: strawberry Signed-off-by: June Clementine Strawberry --- .cargo/audit.toml | 27 +++++++++++++++++++++++++++ engage.toml | 2 +- flake.nix | 8 +++++--- 3 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 .cargo/audit.toml diff --git a/.cargo/audit.toml b/.cargo/audit.toml new file mode 100644 index 00000000..bf44fbd6 --- /dev/null +++ b/.cargo/audit.toml @@ -0,0 +1,27 @@ +[advisories] +ignore = ["RUSTSEC-2024-0436"] # advisory IDs to ignore e.g. ["RUSTSEC-2019-0001", ...] +informational_warnings = [] # warn for categories of informational advisories +severity_threshold = "none" # CVSS severity ("none", "low", "medium", "high", "critical") + +# Advisory Database Configuration +[database] +path = "~/.cargo/advisory-db" # Path where advisory git repo will be cloned +url = "https://github.com/RustSec/advisory-db.git" # URL to git repo +fetch = true # Perform a `git fetch` before auditing (default: true) +stale = false # Allow stale advisory DB (i.e. no commits for 90 days, default: false) + +# Output Configuration +[output] +deny = ["warnings", "unmaintained", "unsound", "yanked"] # exit on error if unmaintained dependencies are found +format = "terminal" # "terminal" (human readable report) or "json" +quiet = false # Only print information on error +show_tree = true # Show inverse dependency trees along with advisories (default: true) + +# Target Configuration +[target] +arch = ["x86_64", "aarch64"] # Ignore advisories for CPU architectures other than these +os = ["linux", "windows", "macos"] # Ignore advisories for operating systems other than these + +[yanked] +enabled = true # Warn for yanked crates in Cargo.lock (default: true) +update_index = true # Auto-update the crates.io index (default: true) diff --git a/engage.toml b/engage.toml index 71366532..0a857b5a 100644 --- a/engage.toml +++ b/engage.toml @@ -63,7 +63,7 @@ script = "markdownlint --version" [[task]] name = "cargo-audit" group = "security" -script = "cargo audit -D warnings -D unmaintained -D unsound -D yanked" +script = "cargo audit --color=always -D warnings -D unmaintained -D unsound -D yanked" [[task]] name = "cargo-fmt" diff --git a/flake.nix b/flake.nix index 544cdd4a..9db2e90a 100644 --- a/flake.nix +++ b/flake.nix @@ -144,18 +144,20 @@ toolchain ] ++ (with pkgsHost.pkgs; [ - engage - cargo-audit - # Required by hardened-malloc.rs dep binutils + cargo-audit + cargo-auditable + # Needed for producing Debian packages cargo-deb # Needed for CI to check validity of produced Debian packages (dpkg-deb) dpkg + engage + # Needed for Complement go From 5efe804a207420482dc5c57b8db044c5818d5037 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 8 Mar 2025 15:48:23 -0500 Subject: [PATCH 1562/2091] always disable fed, evict admins, and forget the room when banning a room Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 20 +-- bin/complement | 18 +- nix/pkgs/main/default.nix | 2 +- src/admin/room/moderation.rs | 328 +++++++++-------------------------- 4 files changed, 109 insertions(+), 259 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8fef47f..9a1366f1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,18 +199,18 @@ jobs: echo '```' >> $GITHUB_STEP_SUMMARY tail -n 100 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY + + echo '# Complement diff results (last 100 lines)' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + echo '# Complement gotestfmt logs (last 100 lines)' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_test_logs_gotestfmt.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY fi - echo '# Complement diff results (last 100 lines)' >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - echo '# Complement gotestfmt logs (last 100 lines)' >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_test_logs_gotestfmt.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - build: name: Build runs-on: self-hosted diff --git a/bin/complement b/bin/complement index 47c02843..b869bad6 100755 --- a/bin/complement +++ b/bin/complement @@ -15,7 +15,7 @@ LOG_FILE="${2:-complement_test_logs.jsonl}" # A `.jsonl` file to write test results to RESULTS_FILE="${3:-complement_test_results.jsonl}" -COMPLEMENT_OCI_IMAGE="${COMPLEMENT_OCI_IMAGE:-complement-conduwuit:main}" +COMPLEMENT_BASE_IMAGE="${COMPLEMENT_BASE_IMAGE:-complement-conduwuit:main}" # Complement tests that are skipped due to flakiness/reliability issues or we don't implement such features and won't for a long time #SKIPPED_COMPLEMENT_TESTS='-skip=TestPartialStateJoin.*' @@ -62,12 +62,13 @@ echo "\$LOG_FILE: $LOG_FILE" echo "" # It's okay (likely, even) that `go test` exits nonzero +# `COMPLEMENT_ENABLE_DIRTY_RUNS=1` reuses the same complement container for faster complement, at the possible expense of test environment pollution set +o pipefail env \ -C "$COMPLEMENT_SRC" \ - COMPLEMENT_BASE_IMAGE="$COMPLEMENT_OCI_IMAGE" \ - COMPLEMENT_ENABLE_DIRTY_RUNS=1 \ # reuses the same complement container for faster complement, at the possible expense of test environment pollution - go test -tags="conduwuit_blacklist" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE" + COMPLEMENT_BASE_IMAGE="$COMPLEMENT_BASE_IMAGE" \ + COMPLEMENT_ENABLE_DIRTY_RUNS=1 \ + go test -tags="conduwuit_blacklist" -timeout 1h -json ./tests/... | tee "$LOG_FILE" set -o pipefail # Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results @@ -78,12 +79,17 @@ cat "$LOG_FILE" | jq -s -c 'sort_by(.Test)[]' | jq -c ' ) | {Action: .Action, Test: .Test} ' > "$RESULTS_FILE" -grep '^{"Time":' "$LOG_FILE" | gotestfmt > "${LOG_FILE}_gotestfmt.log" +if command -v gotestfmt &> /dev/null; then + echo "using gotestfmt on $LOG_FILE" + grep '^{"Time":' "$LOG_FILE" | gotestfmt > "complement_test_logs_gotestfmt.log" +fi echo "" echo "" echo "complement logs saved at $LOG_FILE" echo "complement results saved at $RESULTS_FILE" -echo "complement logs in gotestfmt pretty format outputted at ${LOG_FILE}_gotestfmt.log (use an editor/terminal that interprets ANSI colours)" +if command -v gotestfmt &> /dev/null; then + echo "complement logs in gotestfmt pretty format outputted at complement_test_logs_gotestfmt.log (use an editor/terminal/pager that interprets ANSI colours and UTF-8 emojis)" +fi echo "" echo "" diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 5dfb32ec..9c8038a7 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -155,9 +155,9 @@ commonAttrs = { # Keep sorted include = [ + ".cargo" "Cargo.lock" "Cargo.toml" - "deps" "src" ]; }; diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index 444dfa2f..dd5ea627 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -1,7 +1,7 @@ use api::client::leave_room; use clap::Subcommand; use conduwuit::{ - Result, debug, error, info, + Result, debug, utils::{IterStream, ReadyExt}, warn, }; @@ -17,51 +17,23 @@ use crate::{admin_command, admin_command_dispatch, get_room_info}; #[derive(Debug, Subcommand)] pub(crate) enum RoomModerationCommand { /// - Bans a room from local users joining and evicts all our local users + /// (including server + /// admins) /// from the room. Also blocks any invites (local and remote) for the - /// banned room. - /// - /// Server admins (users in the conduwuit admin room) will not be evicted - /// and server admins can still join the room. To evict admins too, use - /// --force (also ignores errors) To disable incoming federation of the - /// room, use --disable-federation + /// banned room, and disables federation entirely with it. BanRoom { - #[arg(short, long)] - /// Evicts admins out of the room and ignores any potential errors when - /// making our local users leave the room - force: bool, - - #[arg(long)] - /// Disables incoming federation of the room after banning and evicting - /// users - disable_federation: bool, - /// The room in the format of `!roomid:example.com` or a room alias in /// the format of `#roomalias:example.com` room: Box, }, /// - Bans a list of rooms (room IDs and room aliases) from a newline - /// delimited codeblock similar to `user deactivate-all` - BanListOfRooms { - #[arg(short, long)] - /// Evicts admins out of the room and ignores any potential errors when - /// making our local users leave the room - force: bool, - - #[arg(long)] - /// Disables incoming federation of the room after banning and evicting - /// users - disable_federation: bool, - }, + /// delimited codeblock similar to `user deactivate-all`. Applies the same + /// steps as ban-room + BanListOfRooms, /// - Unbans a room to allow local users to join again - /// - /// To re-enable incoming federation of the room, use --enable-federation UnbanRoom { - #[arg(long)] - /// Enables incoming federation of the room after unbanning - enable_federation: bool, - /// The room in the format of `!roomid:example.com` or a room alias in /// the format of `#roomalias:example.com` room: Box, @@ -77,12 +49,7 @@ pub(crate) enum RoomModerationCommand { } #[admin_command] -async fn ban_room( - &self, - force: bool, - disable_federation: bool, - room: Box, -) -> Result { +async fn ban_room(&self, room: Box) -> Result { debug!("Got room alias or ID: {}", room); let admin_room_alias = &self.services.globals.admin_alias; @@ -175,98 +142,56 @@ async fn ban_room( )); }; - debug!("Making all users leave the room {}", &room); - if force { - let mut users = self - .services - .rooms - .state_cache - .room_members(&room_id) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .boxed(); + debug!("Making all users leave the room {room_id} and forgetting it"); + let mut users = self + .services + .rooms + .state_cache + .room_members(&room_id) + .map(ToOwned::to_owned) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .boxed(); - while let Some(local_user) = users.next().await { - debug!( - "Attempting leave for user {local_user} in room {room_id} (forced, ignoring all \ - errors, evicting admins too)", - ); + while let Some(ref user_id) = users.next().await { + debug!( + "Attempting leave for user {user_id} in room {room_id} (ignoring all errors, \ + evicting admins too)", + ); - if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { - warn!(%e, "Failed to leave room"); - } + if let Err(e) = leave_room(self.services, user_id, &room_id, None).await { + warn!("Failed to leave room: {e}"); } - } else { - let mut users = self - .services - .rooms - .state_cache - .room_members(&room_id) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .boxed(); - while let Some(local_user) = users.next().await { - if self.services.users.is_admin(local_user).await { - continue; - } - - debug!("Attempting leave for user {} in room {}", &local_user, &room_id); - if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { - error!( - "Error attempting to make local user {} leave room {} during room banning: \ - {}", - &local_user, &room_id, e - ); - return Ok(RoomMessageEventContent::text_plain(format!( - "Error attempting to make local user {} leave room {} during room banning \ - (room is still banned but not removing any more users): {}\nIf you would \ - like to ignore errors, use --force", - &local_user, &room_id, e - ))); - } - } + self.services.rooms.state_cache.forget(&room_id, user_id); } - // remove any local aliases, ignore errors - for local_alias in &self - .services + self.services .rooms .alias .local_aliases_for_room(&room_id) .map(ToOwned::to_owned) - .collect::>() - .await - { - _ = self - .services - .rooms - .alias - .remove_alias(local_alias, &self.services.globals.server_user) - .await; - } + .for_each(|local_alias| async move { + self.services + .rooms + .alias + .remove_alias(&local_alias, &self.services.globals.server_user) + .await + .ok(); + }) + .await; - // unpublish from room directory, ignore errors + // unpublish from room directory self.services.rooms.directory.set_not_public(&room_id); - if disable_federation { - self.services.rooms.metadata.disable_room(&room_id, true); - return Ok(RoomMessageEventContent::text_plain( - "Room banned, removed all our local users, and disabled incoming federation with \ - room.", - )); - } + self.services.rooms.metadata.disable_room(&room_id, true); Ok(RoomMessageEventContent::text_plain( - "Room banned and removed all our local users, use `!admin federation disable-room` to \ - stop receiving new inbound federation events as well if needed.", + "Room banned, removed all our local users, and disabled incoming federation with room.", )) } #[admin_command] -async fn ban_list_of_rooms( - &self, - force: bool, - disable_federation: bool, -) -> Result { +async fn ban_list_of_rooms(&self) -> Result { if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" @@ -293,7 +218,7 @@ async fn ban_list_of_rooms( if let Ok(admin_room_id) = self.services.admin.get_admin_room().await { if room.to_owned().eq(&admin_room_id) || room.to_owned().eq(admin_room_alias) { - info!("User specified admin room in bulk ban list, ignoring"); + warn!("User specified admin room in bulk ban list, ignoring"); continue; } } @@ -302,19 +227,12 @@ async fn ban_list_of_rooms( let room_id = match RoomId::parse(room_alias_or_id) { | Ok(room_id) => room_id, | Err(e) => { - if force { - // ignore rooms we failed to parse if we're force banning - warn!( - "Error parsing room \"{room}\" during bulk room banning, \ - ignoring error and logging here: {e}" - ); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the \ - list and try again: {e}" - ))); + // ignore rooms we failed to parse + warn!( + "Error parsing room \"{room}\" during bulk room banning, \ + ignoring error and logging here: {e}" + ); + continue; }, }; @@ -355,21 +273,11 @@ async fn ban_list_of_rooms( room_id }, | Err(e) => { - // don't fail if force blocking - if force { - warn!( - "Failed to resolve room alias {room} to a \ - room ID: {e}" - ); - continue; - } - - return Ok(RoomMessageEventContent::text_plain( - format!( - "Failed to resolve room alias {room} to a \ - room ID: {e}" - ), - )); + warn!( + "Failed to resolve room alias {room} to a room \ + ID: {e}" + ); + continue; }, } }, @@ -378,37 +286,21 @@ async fn ban_list_of_rooms( room_ids.push(room_id); }, | Err(e) => { - if force { - // ignore rooms we failed to parse if we're force deleting - error!( - "Error parsing room \"{room}\" during bulk room banning, \ - ignoring error and logging here: {e}" - ); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the \ - list and try again: {e}" - ))); + warn!( + "Error parsing room \"{room}\" during bulk room banning, \ + ignoring error and logging here: {e}" + ); + continue; }, } } }, | Err(e) => { - if force { - // ignore rooms we failed to parse if we're force deleting - error!( - "Error parsing room \"{room}\" during bulk room banning, ignoring error \ - and logging here: {e}" - ); - continue; - } - - return Ok(RoomMessageEventContent::text_plain(format!( - "{room} is not a valid room ID or room alias, please fix the list and try \ - again: {e}" - ))); + warn!( + "Error parsing room \"{room}\" during bulk room banning, ignoring error and \ + logging here: {e}" + ); + continue; }, } } @@ -419,56 +311,27 @@ async fn ban_list_of_rooms( debug!("Banned {room_id} successfully"); room_ban_count = room_ban_count.saturating_add(1); - debug!("Making all users leave the room {}", &room_id); - if force { - let mut users = self - .services - .rooms - .state_cache - .room_members(&room_id) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .boxed(); + debug!("Making all users leave the room {room_id} and forgetting it"); + let mut users = self + .services + .rooms + .state_cache + .room_members(&room_id) + .map(ToOwned::to_owned) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .boxed(); - while let Some(local_user) = users.next().await { - debug!( - "Attempting leave for user {local_user} in room {room_id} (forced, ignoring \ - all errors, evicting admins too)", - ); + while let Some(ref user_id) = users.next().await { + debug!( + "Attempting leave for user {user_id} in room {room_id} (ignoring all errors, \ + evicting admins too)", + ); - if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { - warn!(%e, "Failed to leave room"); - } + if let Err(e) = leave_room(self.services, user_id, &room_id, None).await { + warn!("Failed to leave room: {e}"); } - } else { - let mut users = self - .services - .rooms - .state_cache - .room_members(&room_id) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .boxed(); - while let Some(local_user) = users.next().await { - if self.services.users.is_admin(local_user).await { - continue; - } - - debug!("Attempting leave for user {local_user} in room {room_id}"); - if let Err(e) = leave_room(self.services, local_user, &room_id, None).await { - error!( - "Error attempting to make local user {local_user} leave room {room_id} \ - during bulk room banning: {e}", - ); - - return Ok(RoomMessageEventContent::text_plain(format!( - "Error attempting to make local user {} leave room {} during room \ - banning (room is still banned but not removing any more users and not \ - banning any more rooms): {}\nIf you would like to ignore errors, use \ - --force", - &local_user, &room_id, e - ))); - } - } + self.services.rooms.state_cache.forget(&room_id, user_id); } // remove any local aliases, ignore errors @@ -490,29 +353,17 @@ async fn ban_list_of_rooms( // unpublish from room directory, ignore errors self.services.rooms.directory.set_not_public(&room_id); - if disable_federation { - self.services.rooms.metadata.disable_room(&room_id, true); - } + self.services.rooms.metadata.disable_room(&room_id, true); } - if disable_federation { - Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, \ - and disabled incoming federation with the room." - ))) - } else { - Ok(RoomMessageEventContent::text_plain(format!( - "Finished bulk room ban, banned {room_ban_count} total rooms and evicted all users." - ))) - } + Ok(RoomMessageEventContent::text_plain(format!( + "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and \ + disabled incoming federation with the room." + ))) } #[admin_command] -async fn unban_room( - &self, - enable_federation: bool, - room: Box, -) -> Result { +async fn unban_room(&self, room: Box) -> Result { let room_id = if room.is_room_id() { let room_id = match RoomId::parse(&room) { | Ok(room_id) => room_id, @@ -595,15 +446,8 @@ async fn unban_room( )); }; - if enable_federation { - self.services.rooms.metadata.disable_room(&room_id, false); - return Ok(RoomMessageEventContent::text_plain("Room unbanned.")); - } - - Ok(RoomMessageEventContent::text_plain( - "Room unbanned, you may need to re-enable federation with the room using enable-room if \ - this is a remote room to make it fully functional.", - )) + self.services.rooms.metadata.disable_room(&room_id, false); + Ok(RoomMessageEventContent::text_plain("Room unbanned and federation re-enabled.")) } #[admin_command] From 0b012b529f2c925f2bc20aee2381e2d30f116c46 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 8 Mar 2025 18:59:51 -0500 Subject: [PATCH 1563/2091] comment gotestfmt for now Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 12 ------------ bin/complement | 15 +++++++-------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a1366f1..cd7d2484 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -176,13 +176,6 @@ jobs: path: complement_test_results.jsonl if-no-files-found: error - - name: Upload Complement logs (gotestfmt) - uses: actions/upload-artifact@v4 - with: - name: complement_test_logs_gotestfmt.log - path: complement_test_logs_gotestfmt.log - if-no-files-found: error - - name: Diff Complement results with checked-in repo results run: | diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log) @@ -204,11 +197,6 @@ jobs: echo '```diff' >> $GITHUB_STEP_SUMMARY tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY - - echo '# Complement gotestfmt logs (last 100 lines)' >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_test_logs_gotestfmt.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY fi build: diff --git a/bin/complement b/bin/complement index b869bad6..89521796 100755 --- a/bin/complement +++ b/bin/complement @@ -67,7 +67,6 @@ set +o pipefail env \ -C "$COMPLEMENT_SRC" \ COMPLEMENT_BASE_IMAGE="$COMPLEMENT_BASE_IMAGE" \ - COMPLEMENT_ENABLE_DIRTY_RUNS=1 \ go test -tags="conduwuit_blacklist" -timeout 1h -json ./tests/... | tee "$LOG_FILE" set -o pipefail @@ -79,17 +78,17 @@ cat "$LOG_FILE" | jq -s -c 'sort_by(.Test)[]' | jq -c ' ) | {Action: .Action, Test: .Test} ' > "$RESULTS_FILE" -if command -v gotestfmt &> /dev/null; then - echo "using gotestfmt on $LOG_FILE" - grep '^{"Time":' "$LOG_FILE" | gotestfmt > "complement_test_logs_gotestfmt.log" -fi +#if command -v gotestfmt &> /dev/null; then +# echo "using gotestfmt on $LOG_FILE" +# grep '{"Time":' "$LOG_FILE" | gotestfmt > "complement_test_logs_gotestfmt.log" +#fi echo "" echo "" echo "complement logs saved at $LOG_FILE" echo "complement results saved at $RESULTS_FILE" -if command -v gotestfmt &> /dev/null; then - echo "complement logs in gotestfmt pretty format outputted at complement_test_logs_gotestfmt.log (use an editor/terminal/pager that interprets ANSI colours and UTF-8 emojis)" -fi +#if command -v gotestfmt &> /dev/null; then +# echo "complement logs in gotestfmt pretty format outputted at complement_test_logs_gotestfmt.log (use an editor/terminal/pager that interprets ANSI colours and UTF-8 emojis)" +#fi echo "" echo "" From 06f2039eeeec2d5adf51e8ffbb470f01a8d9e868 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 9 Mar 2025 00:44:56 -0500 Subject: [PATCH 1564/2091] bump ruwuma to maybe fix rare device key upload issues Signed-off-by: June Clementine Strawberry --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 4 +++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8d4688f5..f768eae1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3507,7 +3507,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "assign", "js_int", @@ -3527,7 +3527,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "js_int", "ruma-common", @@ -3539,7 +3539,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "as_variant", "assign", @@ -3562,7 +3562,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "as_variant", "base64 0.22.1", @@ -3594,7 +3594,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "as_variant", "indexmap 2.7.1", @@ -3619,7 +3619,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "bytes", "headers", @@ -3641,7 +3641,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3650,7 +3650,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "js_int", "ruma-common", @@ -3660,7 +3660,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3675,7 +3675,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "js_int", "ruma-common", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=bb42118bd85e731b652a6110896b6945085bf944#bb42118bd85e731b652a6110896b6945085bf944" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index de90e63e..2bc1d20f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "bb42118bd85e731b652a6110896b6945085bf944" +rev = "d577100f5480c6c528e7a8ff59cd08d95a3a16e7" features = [ "compat", "rand", @@ -371,7 +371,9 @@ features = [ "unstable-msc3381", # polls "unstable-msc3489", # beacon / live location "unstable-msc3575", + "unstable-msc3930", # polls push rules "unstable-msc4075", + "unstable-msc4095", "unstable-msc4121", "unstable-msc4125", "unstable-msc4186", From d0c767c23c1dff11400388c5a8dd9e43f68705f1 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 9 Mar 2025 01:43:49 -0500 Subject: [PATCH 1565/2091] fix a few things to make some complement tests pass Signed-off-by: June Clementine Strawberry --- src/api/client/membership.rs | 4 +- src/api/client/room/create.rs | 8 +--- src/api/client/session.rs | 79 +++++++++++++++++++---------------- src/service/media/preview.rs | 23 ++++++---- src/service/users/mod.rs | 4 +- 5 files changed, 60 insertions(+), 58 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 940c8639..3f77e69e 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -517,9 +517,7 @@ pub(crate) async fn invite_user_route( join!(sender_ignored_recipient, recipient_ignored_by_sender); if sender_ignored_recipient { - return Err!(Request(Forbidden( - "You cannot invite users you have ignored to rooms." - ))); + return Ok(invite_user::v3::Response {}); } if let Ok(target_user_membership) = services diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index 1b8294a5..bb06e966 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -239,9 +239,7 @@ pub(crate) async fn create_room_route( if preset == RoomPreset::TrustedPrivateChat { for invite in &body.invite { if services.users.user_is_ignored(sender_user, invite).await { - return Err!(Request(Forbidden( - "You cannot invite users you have ignored to rooms." - ))); + continue; } else if services.users.user_is_ignored(invite, sender_user).await { // silently drop the invite to the recipient if they've been ignored by the // sender, pretend it worked @@ -420,9 +418,7 @@ pub(crate) async fn create_room_route( drop(state_lock); for user_id in &body.invite { if services.users.user_is_ignored(sender_user, user_id).await { - return Err!(Request(Forbidden( - "You cannot invite users you have ignored to rooms." - ))); + continue; } else if services.users.user_is_ignored(user_id, sender_user).await { // silently drop the invite to the recipient if they've been ignored by the // sender, pretend it worked diff --git a/src/api/client/session.rs b/src/api/client/session.rs index ab67ee18..3de625e4 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -3,7 +3,7 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{Err, debug, err, info, utils::ReadyExt}; -use futures::{StreamExt, TryFutureExt}; +use futures::StreamExt; use ruma::{ UserId, api::client::{ @@ -96,32 +96,50 @@ pub(crate) async fn login_route( &services.config.server_name, )?; - assert!( - services.globals.user_is_local(&user_id), - "User ID does not belong to this homeserver" - ); - assert!( - services.globals.user_is_local(&lowercased_user_id), - "User ID does not belong to this homeserver" - ); + if !services.globals.user_is_local(&user_id) + || !services.globals.user_is_local(&lowercased_user_id) + { + return Err!(Request(Unknown("User ID does not belong to this homeserver"))); + } + // first try the username as-is let hash = services .users .password_hash(&user_id) - .or_else(|_| services.users.password_hash(&lowercased_user_id)) .await - .inspect_err(|e| debug!("{e}")) - .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; + .inspect_err(|e| debug!("{e}")); - if hash.is_empty() { - return Err!(Request(UserDeactivated("The user has been deactivated"))); + match hash { + | Ok(hash) => { + if hash.is_empty() { + return Err!(Request(UserDeactivated("The user has been deactivated"))); + } + + hash::verify_password(password, &hash) + .inspect_err(|e| debug!("{e}")) + .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; + + user_id + }, + | Err(_e) => { + let hash_lowercased_user_id = services + .users + .password_hash(&lowercased_user_id) + .await + .inspect_err(|e| debug!("{e}")) + .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; + + if hash_lowercased_user_id.is_empty() { + return Err!(Request(UserDeactivated("The user has been deactivated"))); + } + + hash::verify_password(password, &hash_lowercased_user_id) + .inspect_err(|e| debug!("{e}")) + .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; + + lowercased_user_id + }, } - - hash::verify_password(password, &hash) - .inspect_err(|e| debug!("{e}")) - .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; - - user_id }, | login::v3::LoginInfo::Token(login::v3::Token { token }) => { debug!("Got token login type"); @@ -153,24 +171,11 @@ pub(crate) async fn login_route( } .map_err(|e| err!(Request(InvalidUsername(warn!("Username is invalid: {e}")))))?; - let lowercased_user_id = UserId::parse_with_server_name( - user_id.localpart().to_lowercase(), - &services.config.server_name, - )?; + if !services.globals.user_is_local(&user_id) { + return Err!(Request(Unknown("User ID does not belong to this homeserver"))); + } - assert!( - services.globals.user_is_local(&user_id), - "User ID does not belong to this homeserver" - ); - assert!( - services.globals.user_is_local(&lowercased_user_id), - "User ID does not belong to this homeserver" - ); - - if !info.is_user_match(&user_id) - && !info.is_user_match(&lowercased_user_id) - && !emergency_mode_enabled - { + if !info.is_user_match(&user_id) && !emergency_mode_enabled { return Err!(Request(Exclusive("Username is not in an appservice namespace."))); } diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index 17216869..ba5be7d4 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -7,7 +7,7 @@ use std::time::SystemTime; -use conduwuit::{Err, Result, debug}; +use conduwuit::{Err, Result, debug, err}; use conduwuit_core::implement; use ipaddress::IPAddress; use serde::Serialize; @@ -64,28 +64,33 @@ pub async fn get_url_preview(&self, url: &Url) -> Result { async fn request_url_preview(&self, url: &Url) -> Result { if let Ok(ip) = IPAddress::parse(url.host_str().expect("URL previously validated")) { if !self.services.client.valid_cidr_range(&ip) { - return Err!(BadServerResponse("Requesting from this address is forbidden")); + return Err!(Request(Forbidden("Requesting from this address is forbidden"))); } } let client = &self.services.client.url_preview; let response = client.head(url.as_str()).send().await?; + debug!(?url, "URL preview response headers: {:?}", response.headers()); + if let Some(remote_addr) = response.remote_addr() { + debug!(?url, "URL preview response remote address: {:?}", remote_addr); + if let Ok(ip) = IPAddress::parse(remote_addr.ip().to_string()) { if !self.services.client.valid_cidr_range(&ip) { - return Err!(BadServerResponse("Requesting from this address is forbidden")); + return Err!(Request(Forbidden("Requesting from this address is forbidden"))); } } } - let Some(content_type) = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|x| x.to_str().ok()) - else { - return Err!(Request(Unknown("Unknown Content-Type"))); + let Some(content_type) = response.headers().get(reqwest::header::CONTENT_TYPE) else { + return Err!(Request(Unknown("Unknown or invalid Content-Type header"))); }; + + let content_type = content_type + .to_str() + .map_err(|e| err!(Request(Unknown("Unknown or invalid Content-Type header: {e}"))))?; + let data = match content_type { | html if html.starts_with("text/html") => self.download_html(url.as_str()).await?, | img if img.starts_with("image/") => self.download_image(url.as_str()).await?, diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index b3f5db88..5265e64b 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -278,11 +278,9 @@ impl Service { initial_device_display_name: Option, client_ip: Option, ) -> Result<()> { - // This method should never be called for nonexistent users. We shouldn't assert - // though... if !self.exists(user_id).await { return Err!(Request(InvalidParam(error!( - "Called create_device for non-existent {user_id}" + "Called create_device for non-existent user {user_id}" )))); } From 47ff91243d0da2088806351c040ac1386c92c63d Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 9 Mar 2025 03:33:29 -0400 Subject: [PATCH 1566/2091] update complement results Signed-off-by: June Clementine Strawberry --- .../complement/test_results.jsonl | 112 ++++++++++++++---- 1 file changed, 89 insertions(+), 23 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index fed43b48..7b06510b 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -6,9 +6,9 @@ {"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_has_events"} {"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_has_events/incremental_sync"} {"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_has_events/initial_sync"} -{"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_is_empty"} +{"Action":"pass","Test":"TestArchivedRoomsHistory/timeline_is_empty"} {"Action":"skip","Test":"TestArchivedRoomsHistory/timeline_is_empty/incremental_sync"} -{"Action":"fail","Test":"TestArchivedRoomsHistory/timeline_is_empty/initial_sync"} +{"Action":"pass","Test":"TestArchivedRoomsHistory/timeline_is_empty/initial_sync"} {"Action":"fail","Test":"TestAsyncUpload"} {"Action":"fail","Test":"TestAsyncUpload/Cannot_upload_to_a_media_ID_that_has_already_been_uploaded_to"} {"Action":"fail","Test":"TestAsyncUpload/Create_media"} @@ -82,7 +82,7 @@ {"Action":"pass","Test":"TestContent"} {"Action":"pass","Test":"TestContentCSAPIMediaV1"} {"Action":"pass","Test":"TestContentMediaV1"} -{"Action":"fail","Test":"TestCumulativeJoinLeaveJoinSync"} +{"Action":"pass","Test":"TestCumulativeJoinLeaveJoinSync"} {"Action":"pass","Test":"TestDeactivateAccount"} {"Action":"pass","Test":"TestDeactivateAccount/After_deactivating_account,_can't_log_in_with_password"} {"Action":"pass","Test":"TestDeactivateAccount/Can't_deactivate_account_with_wrong_password"} @@ -153,10 +153,10 @@ {"Action":"fail","Test":"TestFederationKeyUploadQuery/Can_query_remote_device_keys_using_POST"} {"Action":"pass","Test":"TestFederationRedactSendsWithoutEvent"} {"Action":"pass","Test":"TestFederationRejectInvite"} -{"Action":"fail","Test":"TestFederationRoomsInvite"} -{"Action":"fail","Test":"TestFederationRoomsInvite/Parallel"} +{"Action":"pass","Test":"TestFederationRoomsInvite"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation"} -{"Action":"fail","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_for_empty_room"} +{"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_for_empty_room"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_can_reject_invite_over_federation_several_times"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Invited_user_has_'is_direct'_flag_in_prev_content_after_joining"} {"Action":"pass","Test":"TestFederationRoomsInvite/Parallel/Remote_invited_user_can_join_the_room_when_homeserver_is_already_participating_in_the_room"} @@ -173,7 +173,7 @@ {"Action":"pass","Test":"TestFetchMessagesFromNonExistentRoom"} {"Action":"pass","Test":"TestFilter"} {"Action":"fail","Test":"TestFilterMessagesByRelType"} -{"Action":"fail","Test":"TestGappedSyncLeaveSection"} +{"Action":"pass","Test":"TestGappedSyncLeaveSection"} {"Action":"fail","Test":"TestGetFilteredRoomMembers"} {"Action":"fail","Test":"TestGetFilteredRoomMembers/membership/join"} {"Action":"fail","Test":"TestGetFilteredRoomMembers/membership/leave"} @@ -191,7 +191,7 @@ {"Action":"pass","Test":"TestInboundFederationProfile/Inbound_federation_can_query_profile_data"} {"Action":"pass","Test":"TestInboundFederationProfile/Non-numeric_ports_in_server_names_are_rejected"} {"Action":"fail","Test":"TestInboundFederationRejectsEventsWithRejectedAuthEvents"} -{"Action":"fail","Test":"TestInviteFromIgnoredUsersDoesNotAppearInSync"} +{"Action":"pass","Test":"TestInviteFromIgnoredUsersDoesNotAppearInSync"} {"Action":"pass","Test":"TestIsDirectFlagFederation"} {"Action":"pass","Test":"TestIsDirectFlagLocal"} {"Action":"pass","Test":"TestJoinFederatedRoomFailOver"} @@ -281,7 +281,7 @@ {"Action":"pass","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock"} {"Action":"pass","Test":"TestKnockingInMSC3787Room/Users_in_the_room_see_a_user's_membership_update_when_they_knock#01"} {"Action":"pass","Test":"TestLeakyTyping"} -{"Action":"fail","Test":"TestLeaveEventInviteRejection"} +{"Action":"pass","Test":"TestLeaveEventInviteRejection"} {"Action":"fail","Test":"TestLeaveEventVisibility"} {"Action":"fail","Test":"TestLeftRoomFixture"} {"Action":"fail","Test":"TestLeftRoomFixture/Can_get_'m.room.name'_state_for_a_departed_room"} @@ -292,10 +292,10 @@ {"Action":"pass","Test":"TestLocalPngThumbnail"} {"Action":"pass","Test":"TestLocalPngThumbnail/test_/_matrix/client/v1/media_endpoint"} {"Action":"pass","Test":"TestLocalPngThumbnail/test_/_matrix/media/v3_endpoint"} -{"Action":"fail","Test":"TestLogin"} -{"Action":"fail","Test":"TestLogin/parallel"} +{"Action":"pass","Test":"TestLogin"} +{"Action":"pass","Test":"TestLogin/parallel"} {"Action":"pass","Test":"TestLogin/parallel/GET_/login_yields_a_set_of_flows"} -{"Action":"fail","Test":"TestLogin/parallel/Login_with_uppercase_username_works_and_GET_/whoami_afterwards_also"} +{"Action":"pass","Test":"TestLogin/parallel/Login_with_uppercase_username_works_and_GET_/whoami_afterwards_also"} {"Action":"pass","Test":"TestLogin/parallel/POST_/login_as_non-existing_user_is_rejected"} {"Action":"pass","Test":"TestLogin/parallel/POST_/login_can_log_in_as_a_user_with_just_the_local_part_of_the_id"} {"Action":"pass","Test":"TestLogin/parallel/POST_/login_can_login_as_user"} @@ -354,12 +354,78 @@ {"Action":"fail","Test":"TestMembershipOnEvents"} {"Action":"fail","Test":"TestNetworkPartitionOrdering"} {"Action":"pass","Test":"TestNotPresentUserCannotBanOthers"} -{"Action":"fail","Test":"TestOlderLeftRoomsNotInLeaveSection"} +{"Action":"pass","Test":"TestOlderLeftRoomsNotInLeaveSection"} {"Action":"fail","Test":"TestOutboundFederationEventSizeGetMissingEvents"} {"Action":"fail","Test":"TestOutboundFederationIgnoresMissingEventWithBadJSONForRoomVersion6"} {"Action":"pass","Test":"TestOutboundFederationProfile"} {"Action":"pass","Test":"TestOutboundFederationProfile/Outbound_federation_can_query_profile_data"} {"Action":"pass","Test":"TestOutboundFederationSend"} +{"Action":"fail","Test":"TestPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanFastJoinDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanLazyLoadingSyncDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveDeviceListUpdateDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsWithHalfMissingGrandparentsDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsWithHalfMissingParentsDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsWithMissingParentsDuringPartialStateJoin"} +{"Action":"skip","Test":"TestPartialStateJoin/CanReceivePresenceDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveReceiptDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveSigningKeyUpdateDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveToDeviceDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveTypingDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/CanSendEventsDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/Can_change_display_name_during_partial_state_join"} +{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking"} +{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_for_user_incorrectly_believed_to_be_in_room"} +{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_when_failing_to_complete_partial_state_join"} +{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_when_leaving_partial_state_room"} +{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_when_new_member_leaves_partial_state_room"} +{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracked_for_new_members_in_partial_state_room"} +{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_pre-existing_members_in_partial_state_room"} +{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_user_incorrectly_believed_to_be_in_room_when_they_join_another_shared_room_before_partial_state_join_completes"} +{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_user_incorrectly_believed_to_be_in_room_when_they_rejoin_after_partial_state_join_completes"} +{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_user_incorrectly_believed_to_be_in_room_when_they_rejoin_before_partial_state_join_completes"} +{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_when_pre-existing_members_in_partial_state_room_join_another_shared_room"} +{"Action":"fail","Test":"TestPartialStateJoin/EagerIncrementalSyncDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/EagerInitialSyncDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/EagerLongPollingSyncWokenWhenResyncCompletes"} +{"Action":"fail","Test":"TestPartialStateJoin/GappySyncAfterPartialStateSynced"} +{"Action":"fail","Test":"TestPartialStateJoin/Lazy-loading_gappy_sync_includes_remote_memberships_during_partial_state_join"} +{"Action":"fail","Test":"TestPartialStateJoin/Lazy-loading_incremental_sync_includes_remote_memberships_during_partial_state_join"} +{"Action":"fail","Test":"TestPartialStateJoin/Lazy-loading_initial_sync_includes_remote_memberships_during_partial_state_join"} +{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync"} +{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/can_be_triggered_by_remote_ban"} +{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/can_be_triggered_by_remote_kick"} +{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/does_not_wait_for_resync"} +{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/is_seen_after_the_resync"} +{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/succeeds,_then_another_user_can_join_without_resync_completing"} +{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/succeeds,_then_rejoin_succeeds_without_resync_completing"} +{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/works_after_a_second_partial_join"} +{"Action":"fail","Test":"TestPartialStateJoin/MembersRequestBlocksDuringPartialStateJoin"} +{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates"} +{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_no_longer_reach_departed_servers_after_partial_state_join_completes"} +{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_all_servers_in_partial_state_rooms"} +{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_absent_servers_once_partial_state_join_completes"} +{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_absent_servers_once_partial_state_join_completes_even_though_remote_server_left_room"} +{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_kicked_servers_once_partial_state_join_completes"} +{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_kicked_servers_once_partial_state_join_completes_even_though_remote_server_left_room"} +{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_newly_joined_servers_in_partial_state_rooms"} +{"Action":"fail","Test":"TestPartialStateJoin/PartialStateJoinContinuesAfterRestart"} +{"Action":"fail","Test":"TestPartialStateJoin/PartialStateJoinSyncsUsingOtherHomeservers"} +{"Action":"skip","Test":"TestPartialStateJoin/Purge_during_resync"} +{"Action":"fail","Test":"TestPartialStateJoin/Rejected_events_remain_rejected_after_resync"} +{"Action":"fail","Test":"TestPartialStateJoin/Rejects_make_join_during_partial_join"} +{"Action":"fail","Test":"TestPartialStateJoin/Rejects_make_knock_during_partial_join"} +{"Action":"fail","Test":"TestPartialStateJoin/Rejects_send_join_during_partial_join"} +{"Action":"fail","Test":"TestPartialStateJoin/Rejects_send_knock_during_partial_join"} +{"Action":"fail","Test":"TestPartialStateJoin/Resync_completes_even_when_events_arrive_before_their_prev_events"} +{"Action":"fail","Test":"TestPartialStateJoin/Room_aliases_can_be_added_and_deleted_during_a_resync"} +{"Action":"fail","Test":"TestPartialStateJoin/Room_aliases_can_be_added_and_queried_during_a_resync"} +{"Action":"skip","Test":"TestPartialStateJoin/Room_stats_are_correctly_updated_once_state_re-sync_completes"} +{"Action":"fail","Test":"TestPartialStateJoin/State_accepted_incorrectly"} +{"Action":"fail","Test":"TestPartialStateJoin/State_rejected_incorrectly"} +{"Action":"fail","Test":"TestPartialStateJoin/User_directory_is_correctly_updated_once_state_re-sync_completes"} +{"Action":"fail","Test":"TestPartialStateJoin/joined_members_blocks_during_partial_state_join"} {"Action":"fail","Test":"TestPollsLocalPushRules"} {"Action":"fail","Test":"TestPollsLocalPushRules/Polls_push_rules_are_correctly_presented_to_the_client"} {"Action":"pass","Test":"TestPowerLevels"} @@ -559,11 +625,11 @@ {"Action":"pass","Test":"TestRoomState/Parallel/PUT_/createRoom_with_creation_content"} {"Action":"pass","Test":"TestRoomState/Parallel/PUT_/rooms/:room_id/state/m.room.topic_sets_topic"} {"Action":"pass","Test":"TestRoomSummary"} -{"Action":"fail","Test":"TestRoomsInvite"} -{"Action":"fail","Test":"TestRoomsInvite/Parallel"} +{"Action":"pass","Test":"TestRoomsInvite"} +{"Action":"pass","Test":"TestRoomsInvite/Parallel"} {"Action":"pass","Test":"TestRoomsInvite/Parallel/Can_invite_users_to_invite-only_rooms"} {"Action":"pass","Test":"TestRoomsInvite/Parallel/Invited_user_can_reject_invite"} -{"Action":"fail","Test":"TestRoomsInvite/Parallel/Invited_user_can_reject_invite_for_empty_room"} +{"Action":"pass","Test":"TestRoomsInvite/Parallel/Invited_user_can_reject_invite_for_empty_room"} {"Action":"pass","Test":"TestRoomsInvite/Parallel/Invited_user_can_see_room_metadata"} {"Action":"pass","Test":"TestRoomsInvite/Parallel/Test_that_we_can_be_reinvited_to_a_room_we_created"} {"Action":"pass","Test":"TestRoomsInvite/Parallel/Uninvited_users_cannot_join_the_room"} @@ -596,15 +662,15 @@ {"Action":"pass","Test":"TestSyncFilter"} {"Action":"pass","Test":"TestSyncFilter/Can_create_filter"} {"Action":"pass","Test":"TestSyncFilter/Can_download_filter"} -{"Action":"fail","Test":"TestSyncLeaveSection"} -{"Action":"fail","Test":"TestSyncLeaveSection/Left_rooms_appear_in_the_leave_section_of_full_state_sync"} -{"Action":"fail","Test":"TestSyncLeaveSection/Left_rooms_appear_in_the_leave_section_of_sync"} -{"Action":"fail","Test":"TestSyncLeaveSection/Newly_left_rooms_appear_in_the_leave_section_of_incremental_sync"} +{"Action":"pass","Test":"TestSyncLeaveSection"} +{"Action":"pass","Test":"TestSyncLeaveSection/Left_rooms_appear_in_the_leave_section_of_full_state_sync"} +{"Action":"pass","Test":"TestSyncLeaveSection/Left_rooms_appear_in_the_leave_section_of_sync"} +{"Action":"pass","Test":"TestSyncLeaveSection/Newly_left_rooms_appear_in_the_leave_section_of_incremental_sync"} {"Action":"pass","Test":"TestSyncOmitsStateChangeOnFilteredEvents"} {"Action":"pass","Test":"TestSyncTimelineGap"} {"Action":"pass","Test":"TestSyncTimelineGap/full"} {"Action":"pass","Test":"TestSyncTimelineGap/incremental"} -{"Action":"fail","Test":"TestTentativeEventualJoiningAfterRejecting"} +{"Action":"pass","Test":"TestTentativeEventualJoiningAfterRejecting"} {"Action":"fail","Test":"TestThreadReceiptsInSyncMSC4102"} {"Action":"fail","Test":"TestThreadedReceipts"} {"Action":"fail","Test":"TestThreadsEndpoint"} @@ -635,8 +701,8 @@ {"Action":"pass","Test":"TestUploadKey/Parallel/Can_query_device_keys_using_POST"} {"Action":"pass","Test":"TestUploadKey/Parallel/Can_query_specific_device_keys_using_POST"} {"Action":"pass","Test":"TestUploadKey/Parallel/Can_upload_device_keys"} -{"Action":"fail","Test":"TestUploadKey/Parallel/Rejects_invalid_device_keys"} -{"Action":"fail","Test":"TestUploadKey/Parallel/Should_reject_keys_claiming_to_belong_to_a_different_user"} +{"Action":"pass","Test":"TestUploadKey/Parallel/Rejects_invalid_device_keys"} +{"Action":"pass","Test":"TestUploadKey/Parallel/Should_reject_keys_claiming_to_belong_to_a_different_user"} {"Action":"pass","Test":"TestUploadKey/Parallel/query_for_user_with_no_keys_returns_empty_key_dict"} {"Action":"pass","Test":"TestUploadKeyIdempotency"} {"Action":"pass","Test":"TestUploadKeyIdempotencyOverlap"} From 0e342aab7f2a173638fa723a9d36ae16fe9396d1 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 9 Mar 2025 13:44:57 -0400 Subject: [PATCH 1567/2091] fix a few error codes Signed-off-by: June Clementine Strawberry --- src/api/client/alias.rs | 2 +- src/api/client/context.rs | 18 ++++++++++++------ src/api/client/state.rs | 12 +++++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 319e5141..9cd7e0c5 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -92,7 +92,7 @@ pub(crate) async fn get_alias_route( let Ok((room_id, servers)) = services.rooms.alias.resolve_alias(&room_alias, None).await else { - return Err!(Request(NotFound("Room with alias not found."))); + return Err!(Request(Unknown("Room with alias not found."))); }; let servers = room_available_servers(&services, &room_id, &room_alias, servers).await; diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 3f16c850..cb95dfef 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,6 +1,6 @@ use axum::extract::State; use conduwuit::{ - Err, PduEvent, Result, at, err, ref_at, + Err, PduEvent, Result, at, debug_warn, err, ref_at, utils::{ IterStream, future::TryExtExt, @@ -35,8 +35,13 @@ pub(crate) async fn get_context_route( let sender = body.sender(); let (sender_user, sender_device) = sender; let room_id = &body.room_id; + let event_id = &body.event_id; let filter = &body.filter; + if !services.rooms.metadata.exists(room_id).await { + return Err!(Request(Forbidden("Room does not exist to this server"))); + } + // Use limit or else 10, with maximum 100 let limit: usize = body .limit @@ -47,29 +52,30 @@ pub(crate) async fn get_context_route( let base_id = services .rooms .timeline - .get_pdu_id(&body.event_id) + .get_pdu_id(event_id) .map_err(|_| err!(Request(NotFound("Event not found.")))); let base_pdu = services .rooms .timeline - .get_pdu(&body.event_id) + .get_pdu(event_id) .map_err(|_| err!(Request(NotFound("Base event not found.")))); let visible = services .rooms .state_accessor - .user_can_see_event(sender_user, &body.room_id, &body.event_id) + .user_can_see_event(sender_user, room_id, event_id) .map(Ok); let (base_id, base_pdu, visible) = try_join3(base_id, base_pdu, visible).await?; - if base_pdu.room_id != body.room_id || base_pdu.event_id != body.event_id { + if base_pdu.room_id != *room_id || base_pdu.event_id != *event_id { return Err!(Request(NotFound("Base event not found."))); } if !visible { - return Err!(Request(Forbidden("You don't have permission to view this event."))); + debug_warn!(req_evt = ?event_id, ?base_id, ?room_id, "Event requested by {sender_user} but is not allowed to see it, returning 404"); + return Err!(Request(NotFound("Event not found."))); } let base_count = base_id.pdu_count(); diff --git a/src/api/client/state.rs b/src/api/client/state.rs index c92091eb..d04aac35 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -27,7 +27,7 @@ pub(crate) async fn send_state_event_for_key_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); Ok(send_state_event::v3::Response { event_id: send_state_event_for_key_helper( @@ -103,7 +103,7 @@ pub(crate) async fn get_state_events_for_key_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if !services .rooms @@ -111,7 +111,9 @@ pub(crate) async fn get_state_events_for_key_route( .user_can_see_state_events(sender_user, &body.room_id) .await { - return Err!(Request(Forbidden("You don't have permission to view the room state."))); + return Err!(Request(NotFound(debug_warn!( + "You don't have permission to view the room state." + )))); } let event = services @@ -316,14 +318,14 @@ async fn allowed_to_send_state_event( services.rooms.alias.resolve_alias(&alias, None).await?; if alias_room_id != room_id { - return Err!(Request(Forbidden( + return Err!(Request(Unknown( "Room alias {alias} does not belong to room {room_id}" ))); } } }, | Err(e) => { - return Err!(Request(BadJson(debug_warn!( + return Err!(Request(InvalidParam(debug_warn!( "Room canonical alias event is invalid: {e}" )))); }, From 0e2ca7d7192684a945ac49aa53066c488dd40886 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 9 Mar 2025 21:55:07 -0400 Subject: [PATCH 1568/2091] implement disable TLS validation config option Signed-off-by: June Clementine Strawberry --- nix/pkgs/complement/config.toml | 2 ++ src/core/config/check.rs | 4 ++++ src/core/config/mod.rs | 12 +++++++++++- src/service/client/mod.rs | 3 ++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index 99c151c5..4d7637db 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -32,6 +32,8 @@ allow_legacy_media = true startup_netburst = true startup_netburst_keep = -1 +allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure = true + # valgrind makes things so slow dns_timeout = 60 dns_attempts = 20 diff --git a/src/core/config/check.rs b/src/core/config/check.rs index 98223be4..f9d51eeb 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -28,6 +28,10 @@ pub fn check(config: &Config) -> Result { warn!("Note: conduwuit was built without optimisations (i.e. debug build)"); } + if config.allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure { + warn!("\n\nWARNING: \n\nTLS CERTIFICATE VALIDATION IS DISABLED, THIS IS HIGHLY INSECURE AND SHOULD NOT BE USED IN PRODUCTION.\n\n"); + } + warn_deprecated(config); warn_unknown_key(config); diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index e69a56b9..6b669ad3 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -52,7 +52,7 @@ use crate::{Result, err, error::Error, utils::sys}; ### For more information, see: ### https://conduwuit.puppyirl.gay/configuration.html "#, - ignore = "catchall well_known tls blurhashing" + ignore = "catchall well_known tls blurhashing allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure" )] pub struct Config { /// The server_name is the pretty name of this server. It is used as a @@ -1806,6 +1806,16 @@ pub struct Config { #[serde(default = "true_fn")] pub config_reload_signal: bool, + /// Toggles ignore checking/validating TLS certificates + /// + /// This applies to everything, including URL previews, federation requests, + /// etc. This is a hidden argument that should NOT be used in production as + /// it is highly insecure and I will personally yell at you if I catch you + /// using this. + #[serde(default)] + pub allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure: + bool, + // external structure; separate section #[serde(default)] pub blurhashing: BlurhashConfig, diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index d5008491..d51e5721 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -128,7 +128,8 @@ fn base(config: &Config) -> Result { .pool_max_idle_per_host(config.request_idle_per_host.into()) .user_agent(conduwuit::version::user_agent()) .redirect(redirect::Policy::limited(6)) - .connection_verbose(true); + .danger_accept_invalid_certs(config.allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure) + .connection_verbose(cfg!(debug_assertions)); #[cfg(feature = "gzip_compression")] { From df1edcf498ac58e27e6ff261b0d53a773d82f69f Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 10 Mar 2025 10:32:11 -0400 Subject: [PATCH 1569/2091] adjust complement cert generation Signed-off-by: June Clementine Strawberry --- bin/complement | 1 + nix/pkgs/complement/config.toml | 2 -- nix/pkgs/complement/default.nix | 22 +++++++------------ nix/pkgs/complement/private_key.key | 28 +++++++++++++++++++++++++ nix/pkgs/complement/signing_request.csr | 16 ++++++++++++++ 5 files changed, 53 insertions(+), 16 deletions(-) create mode 100644 nix/pkgs/complement/private_key.key create mode 100644 nix/pkgs/complement/signing_request.csr diff --git a/bin/complement b/bin/complement index 89521796..92539f97 100755 --- a/bin/complement +++ b/bin/complement @@ -40,6 +40,7 @@ if [ ! -f "complement_oci_image.tar.gz" ]; then # if using macOS, use linux-complement #bin/nix-build-and-cache just .#linux-complement bin/nix-build-and-cache just .#complement + #nix build -L .#complement echo "complement conduwuit image tar.gz built at \"result\"" diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index 4d7637db..759f8d78 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -47,6 +47,4 @@ sender_idle_timeout = 300 sender_retry_backoff_limit = 300 [global.tls] -certs = "/certificate.crt" dual_protocol = true -key = "/private_key.key" diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index d9af0779..bbd1bd74 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -42,25 +42,18 @@ let start = writeShellScriptBin "start" '' set -euxo pipefail - ${lib.getExe openssl} genrsa -out private_key.key 2048 - ${lib.getExe openssl} req \ - -new \ - -sha256 \ - -key private_key.key \ - -subj "/C=US/ST=CA/O=MyOrg, Inc./CN=$SERVER_NAME" \ - -out signing_request.csr - cp ${./v3.ext} v3.ext - echo "DNS.1 = $SERVER_NAME" >> v3.ext + cp ${./v3.ext} /complement/v3.ext + echo "DNS.1 = $SERVER_NAME" >> /complement/v3.ext echo "IP.1 = $(${lib.getExe gawk} 'END{print $1}' /etc/hosts)" \ - >> v3.ext + >> /complement/v3.ext ${lib.getExe openssl} x509 \ -req \ - -extfile v3.ext \ - -in signing_request.csr \ + -extfile /complement/v3.ext \ + -in ${./signing_request.csr} \ -CA /complement/ca/ca.crt \ -CAkey /complement/ca/ca.key \ -CAcreateserial \ - -out certificate.crt \ + -out /complement/certificate.crt \ -days 1 \ -sha256 @@ -99,7 +92,8 @@ dockerTools.buildImage { else []; Env = [ - "SSL_CERT_FILE=/complement/ca/ca.crt" + "CONDUWUIT_TLS__KEY=${./private_key.key}" + "CONDUWUIT_TLS__CERTS=/complement/certificate.crt" "CONDUWUIT_CONFIG=${./config.toml}" "RUST_BACKTRACE=full" ]; diff --git a/nix/pkgs/complement/private_key.key b/nix/pkgs/complement/private_key.key new file mode 100644 index 00000000..5b9d4d4f --- /dev/null +++ b/nix/pkgs/complement/private_key.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDS/odmZivxajeb +iyT7SMuhXqnMm+hF+zEARLcbieem0wG4x7gi2S6WLf8DlifdXax6me13eYk4rBnT +LvGEvNNx0px5M54H+FVyoVa3c1tmA66WUcZjobafPGsDh5j+5qpScgWwjkMPGg1a +09CphCFswO4PpxUUORX/OTGj/rEKxximW6OtavBwaS9F7mqjXJK7lCrcZxKq5ucc +ebGMmCoO660hROSTBaFigdRTVicclk+NgYRrZyWbCiuXPjQ0jlOE2rcaDepqTUga +Qs/2tdT4kBzBH6kZOiQOIN/ddXaj032QXr1HQYfIJfJmiM6nmRob8nik5rpZdWNO +/Ncsro/fAgMBAAECggEAITCCkfv+a5I+vwvrPE/eIDso0JOxvNhfg+BLQVy3AMnu +WmeoMmshZeREWgcTrEGg8QQnk4Sdrjl8MnkO6sddJ2luza3t7OkGX+q7Hk5aETkB +DIo+f8ufU3sIhlydF3OnVSK0fGpUaBq8AQ6Soyeyrk3G5NVufmjgae5QPbDBnqUb +piOGyfcwagL4JtCbZsMk8AT7vQSynLm6zaWsVzWNd71jummLqtVV063K95J9PqVN +D8meEcP3WR5kQrvf+mgy9RVgWLRtVWN8OLZfJ9yrnl4Efj62elrldUj4jaCFezGQ +8f0W+d8jjt038qhmEdymw2MWQ+X/b0R79lJar1Up8QKBgQD1DtHxauhl+JUoI3y+ +3eboqXl7YPJt1/GTnChb4b6D1Z1hvLsOKUa7hjGEfruYGbsWXBCRMICdfzp+iWcq +/lEOp7/YU9OaW4lQMoG4sXMoBWd9uLgg0E+aH6VDJOBvxsfafqM4ufmtspzwEm90 +FU1cq6oImomFnPChSq4X+3+YpwKBgQDcalaK9llCcscWA8HAP8WVVNTjCOqiDp9q +td61E9IO/FIB/gW5y+JkaFRrA2CN1zY3s3K92uveLTNYTArecWlDcPNNFDuaYu2M +Roz4bC104HGh+zztJ0iPVzELL81Lgg6wHhLONN+eVi4gTftJxzJFXybyb+xVT25A +91ynKXB+CQKBgQC+Ub43MoI+/6pHvBfb3FbDByvz6D0flgBmVXb6tP3TQYmzKHJV +8zSd2wCGGC71V7Z3DRVIzVR1/SOetnPLbivhp+JUzfWfAcxI3pDksdvvjxLrDxTh +VycbWcxtsywjY0w/ou581eLVRcygnpC0pP6qJCAwAmUfwd0YRvmiYo6cLQKBgHIW +UIlJDdaJFmdctnLOD3VGHZMOUHRlYTqYvJe5lKbRD5mcZFZRI/OY1Ok3LEj+tj+K +kL+YizHK76KqaY3N4hBYbHbfHCLDRfWvptQHGlg+vFJ9eoG+LZ6UIPyLV5XX0cZz +KoS1dXG9Zc6uznzXsDucDsq6B/f4TzctUjXsCyARAoGAOKb4HtuNyYAW0jUlujR7 +IMHwUesOGlhSXqFtP9aTvk6qJgvV0+3CKcWEb4y02g+uYftP8BLNbJbIt9qOqLYh +tOVyzCoamAi8araAhjA0w4dXvqDCDK7k/gZFkojmKQtRijoxTHnWcDc3vAjYCgaM +9MVtdgSkuh2gwkD/mMoAJXM= +-----END PRIVATE KEY----- diff --git a/nix/pkgs/complement/signing_request.csr b/nix/pkgs/complement/signing_request.csr new file mode 100644 index 00000000..707e73b4 --- /dev/null +++ b/nix/pkgs/complement/signing_request.csr @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICkTCCAXkCAQAwTDELMAkGA1UEBhMCNjkxCzAJBgNVBAgMAjQyMRYwFAYDVQQK +DA13b29mZXJzLCBpbmMuMRgwFgYDVQQDDA9jb21wbGVtZW50LW9ubHkwggEiMA0G +CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDS/odmZivxajebiyT7SMuhXqnMm+hF ++zEARLcbieem0wG4x7gi2S6WLf8DlifdXax6me13eYk4rBnTLvGEvNNx0px5M54H ++FVyoVa3c1tmA66WUcZjobafPGsDh5j+5qpScgWwjkMPGg1a09CphCFswO4PpxUU +ORX/OTGj/rEKxximW6OtavBwaS9F7mqjXJK7lCrcZxKq5uccebGMmCoO660hROST +BaFigdRTVicclk+NgYRrZyWbCiuXPjQ0jlOE2rcaDepqTUgaQs/2tdT4kBzBH6kZ +OiQOIN/ddXaj032QXr1HQYfIJfJmiM6nmRob8nik5rpZdWNO/Ncsro/fAgMBAAGg +ADANBgkqhkiG9w0BAQsFAAOCAQEAjW+aD4E0phtRT5b2RyedY1uiSe7LQECsQnIO +wUSyGGG1GXYlJscyxxyzE9W9+QIALrxZkmc/+e02u+bFb1zQXW/uB/7u7FgXzrj6 +2YSDiWYXiYKvgGWEfCi3lpcTJK9x6WWkR+iREaoKRjcl0ynhhGuR7YwP38TNyu+z +FN6B1Lo398fvJkaTCiiHngWiwztXZ2d0MxkicuwZ1LJhIQA72OTl3QoRb5uiqbze +T9QJfU6W3v8cB8c8PuKMv5gl1QsGNtlfyQB56/X0cMxWl25vWXd2ankLkAGRTDJ8 +9YZHxP1ki4/yh75AknFq02nCOsmxYrAazCYgP2TzIPhQwBurKQ== +-----END CERTIFICATE REQUEST----- From 5ba0c02d526d77b9d983335af76585cd49be12c1 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 10 Mar 2025 12:29:54 -0400 Subject: [PATCH 1570/2091] bump ruwuma to fix a threads issue, fix more error codes, delete legacy sytest cruft Signed-off-by: June Clementine Strawberry --- Cargo.lock | 22 +- Cargo.toml | 2 +- src/api/client/alias.rs | 2 +- src/api/client/state.rs | 12 +- tests/sytest/are-we-synapse-yet.list | 866 ----------------------- tests/sytest/are-we-synapse-yet.py | 266 ------- tests/sytest/show-expected-fail-tests.sh | 105 --- tests/sytest/sytest-blacklist | 7 - tests/sytest/sytest-whitelist | 516 -------------- 9 files changed, 22 insertions(+), 1776 deletions(-) delete mode 100644 tests/sytest/are-we-synapse-yet.list delete mode 100755 tests/sytest/are-we-synapse-yet.py delete mode 100755 tests/sytest/show-expected-fail-tests.sh delete mode 100644 tests/sytest/sytest-blacklist delete mode 100644 tests/sytest/sytest-whitelist diff --git a/Cargo.lock b/Cargo.lock index f768eae1..65e8eca1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3507,7 +3507,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "assign", "js_int", @@ -3527,7 +3527,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "js_int", "ruma-common", @@ -3539,7 +3539,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "as_variant", "assign", @@ -3562,7 +3562,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "as_variant", "base64 0.22.1", @@ -3594,7 +3594,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "as_variant", "indexmap 2.7.1", @@ -3619,7 +3619,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "bytes", "headers", @@ -3641,7 +3641,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3650,7 +3650,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "js_int", "ruma-common", @@ -3660,7 +3660,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3675,7 +3675,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "js_int", "ruma-common", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d577100f5480c6c528e7a8ff59cd08d95a3a16e7#d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 2bc1d20f..d611c08e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "d577100f5480c6c528e7a8ff59cd08d95a3a16e7" +rev = "f5ab6302aaa55a14827a9cb5b40e980dd135fe14" features = [ "compat", "rand", diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 9cd7e0c5..319e5141 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -92,7 +92,7 @@ pub(crate) async fn get_alias_route( let Ok((room_id, servers)) = services.rooms.alias.resolve_alias(&room_alias, None).await else { - return Err!(Request(Unknown("Room with alias not found."))); + return Err!(Request(NotFound("Room with alias not found."))); }; let servers = room_available_servers(&services, &room_id, &room_alias, servers).await; diff --git a/src/api/client/state.rs b/src/api/client/state.rs index d04aac35..db79735f 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -314,11 +314,17 @@ async fn allowed_to_send_state_event( } for alias in aliases { - let (alias_room_id, _servers) = - services.rooms.alias.resolve_alias(&alias, None).await?; + let (alias_room_id, _servers) = services + .rooms + .alias + .resolve_alias(&alias, None) + .await + .map_err(|e| { + err!(Request(Unknown("Failed resolving alias \"{alias}\": {e}"))) + })?; if alias_room_id != room_id { - return Err!(Request(Unknown( + return Err!(Request(BadAlias( "Room alias {alias} does not belong to room {room_id}" ))); } diff --git a/tests/sytest/are-we-synapse-yet.list b/tests/sytest/are-we-synapse-yet.list deleted file mode 100644 index 99091989..00000000 --- a/tests/sytest/are-we-synapse-yet.list +++ /dev/null @@ -1,866 +0,0 @@ -reg GET /register yields a set of flows -reg POST /register can create a user -reg POST /register downcases capitals in usernames -reg POST /register returns the same device_id as that in the request -reg POST /register rejects registration of usernames with '!' -reg POST /register rejects registration of usernames with '"' -reg POST /register rejects registration of usernames with ':' -reg POST /register rejects registration of usernames with '?' -reg POST /register rejects registration of usernames with '\' -reg POST /register rejects registration of usernames with '@' -reg POST /register rejects registration of usernames with '[' -reg POST /register rejects registration of usernames with ']' -reg POST /register rejects registration of usernames with '{' -reg POST /register rejects registration of usernames with '|' -reg POST /register rejects registration of usernames with '}' -reg POST /register rejects registration of usernames with '£' -reg POST /register rejects registration of usernames with 'é' -reg POST /register rejects registration of usernames with '\n' -reg POST /register rejects registration of usernames with ''' -reg POST /r0/admin/register with shared secret -reg POST /r0/admin/register admin with shared secret -reg POST /r0/admin/register with shared secret downcases capitals -reg POST /r0/admin/register with shared secret disallows symbols -reg POST rejects invalid utf-8 in JSON -log GET /login yields a set of flows -log POST /login can log in as a user -log POST /login returns the same device_id as that in the request -log POST /login can log in as a user with just the local part of the id -log POST /login as non-existing user is rejected -log POST /login wrong password is rejected -log Interactive authentication types include SSO -log Can perform interactive authentication with SSO -log The user must be consistent through an interactive authentication session with SSO -log The operation must be consistent through an interactive authentication session -v1s GET /events initially -v1s GET /initialSync initially -csa Version responds 200 OK with valid structure -pro PUT /profile/:user_id/displayname sets my name -pro GET /profile/:user_id/displayname publicly accessible -pro PUT /profile/:user_id/avatar_url sets my avatar -pro GET /profile/:user_id/avatar_url publicly accessible -dev GET /device/{deviceId} -dev GET /device/{deviceId} gives a 404 for unknown devices -dev GET /devices -dev PUT /device/{deviceId} updates device fields -dev PUT /device/{deviceId} gives a 404 for unknown devices -dev DELETE /device/{deviceId} -dev DELETE /device/{deviceId} requires UI auth user to match device owner -dev DELETE /device/{deviceId} with no body gives a 401 -dev The deleted device must be consistent through an interactive auth session -dev Users receive device_list updates for their own devices -pre GET /presence/:user_id/status fetches initial status -pre PUT /presence/:user_id/status updates my presence -crm POST /createRoom makes a public room -crm POST /createRoom makes a private room -crm POST /createRoom makes a private room with invites -crm POST /createRoom makes a room with a name -crm POST /createRoom makes a room with a topic -syn Can /sync newly created room -crm POST /createRoom creates a room with the given version -crm POST /createRoom rejects attempts to create rooms with numeric versions -crm POST /createRoom rejects attempts to create rooms with unknown versions -crm POST /createRoom ignores attempts to set the room version via creation_content -mem GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership -mem GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event -rst GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels -mem GET /rooms/:room_id/joined_members fetches my membership -v1s GET /rooms/:room_id/initialSync fetches initial sync state -pub GET /publicRooms lists newly-created room -ali GET /directory/room/:room_alias yields room ID -mem GET /joined_rooms lists newly-created room -rst POST /rooms/:room_id/state/m.room.name sets name -rst GET /rooms/:room_id/state/m.room.name gets name -rst POST /rooms/:room_id/state/m.room.topic sets topic -rst GET /rooms/:room_id/state/m.room.topic gets topic -rst GET /rooms/:room_id/state fetches entire room state -crm POST /createRoom with creation content -ali PUT /directory/room/:room_alias creates alias -nsp GET /rooms/:room_id/aliases lists aliases -jon POST /rooms/:room_id/join can join a room -jon POST /join/:room_alias can join a room -jon POST /join/:room_id can join a room -jon POST /join/:room_id can join a room with custom content -jon POST /join/:room_alias can join a room with custom content -lev POST /rooms/:room_id/leave can leave a room -inv POST /rooms/:room_id/invite can send an invite -ban POST /rooms/:room_id/ban can ban a user -snd POST /rooms/:room_id/send/:event_type sends a message -snd PUT /rooms/:room_id/send/:event_type/:txn_id sends a message -snd PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id -get GET /rooms/:room_id/messages returns a message -get GET /rooms/:room_id/messages lazy loads members correctly -typ PUT /rooms/:room_id/typing/:user_id sets typing notification -typ Typing notifications don't leak (3 subtests) -rst GET /rooms/:room_id/state/m.room.power_levels can fetch levels -rst PUT /rooms/:room_id/state/m.room.power_levels can set levels -rst PUT power_levels should not explode if the old power levels were empty -rst Both GET and PUT work -rct POST /rooms/:room_id/receipt can create receipts -red POST /rooms/:room_id/read_markers can create read marker -med POST /media/r0/upload can create an upload -med GET /media/r0/download can fetch the value again -cap GET /capabilities is present and well formed for registered user -cap GET /r0/capabilities is not public -reg Register with a recaptcha -reg registration is idempotent, without username specified -reg registration is idempotent, with username specified -reg registration remembers parameters -reg registration accepts non-ascii passwords -reg registration with inhibit_login inhibits login -reg User signups are forbidden from starting with '_' -reg Can register using an email address -log Can login with 3pid and password using m.login.password -log login types include SSO -log /login/cas/redirect redirects if the old m.login.cas login type is listed -log Can login with new user via CAS -lox Can logout current device -lox Can logout all devices -lox Request to logout with invalid an access token is rejected -lox Request to logout without an access token is rejected -log After changing password, can't log in with old password -log After changing password, can log in with new password -log After changing password, existing session still works -log After changing password, a different session no longer works by default -log After changing password, different sessions can optionally be kept -psh Pushers created with a different access token are deleted on password change -psh Pushers created with a the same access token are not deleted on password change -acc Can deactivate account -acc Can't deactivate account with wrong password -acc After deactivating account, can't log in with password -acc After deactivating account, can't log in with an email -v1s initialSync sees my presence status -pre Presence change reports an event to myself -pre Friends presence changes reports events -crm Room creation reports m.room.create to myself -crm Room creation reports m.room.member to myself -rst Setting room topic reports m.room.topic to myself -v1s Global initialSync -v1s Global initialSync with limit=0 gives no messages -v1s Room initialSync -v1s Room initialSync with limit=0 gives no messages -rst Setting state twice is idempotent -jon Joining room twice is idempotent -syn New room members see their own join event -v1s New room members see existing users' presence in room initialSync -syn Existing members see new members' join events -syn Existing members see new members' presence -v1s All room members see all room members' presence in global initialSync -f,jon Remote users can join room by alias -syn New room members see their own join event -v1s New room members see existing members' presence in room initialSync -syn Existing members see new members' join events -syn Existing members see new member's presence -v1s New room members see first user's profile information in global initialSync -v1s New room members see first user's profile information in per-room initialSync -f,jon Remote users may not join unfederated rooms -syn Local room members see posted message events -v1s Fetching eventstream a second time doesn't yield the message again -syn Local non-members don't see posted message events -get Local room members can get room messages -f,syn Remote room members also see posted message events -f,get Remote room members can get room messages -get Message history can be paginated -f,get Message history can be paginated over federation -eph Ephemeral messages received from clients are correctly expired -ali Room aliases can contain Unicode -f,ali Remote room alias queries can handle Unicode -ali Canonical alias can be set -ali Canonical alias can include alt_aliases -ali Regular users can add and delete aliases in the default room configuration -ali Regular users can add and delete aliases when m.room.aliases is restricted -ali Deleting a non-existent alias should return a 404 -ali Users can't delete other's aliases -ali Users with sufficient power-level can delete other's aliases -ali Can delete canonical alias -ali Alias creators can delete alias with no ops -ali Alias creators can delete canonical alias with no ops -ali Only room members can list aliases of a room -inv Can invite users to invite-only rooms -inv Uninvited users cannot join the room -inv Invited user can reject invite -f,inv Invited user can reject invite over federation -f,inv Invited user can reject invite over federation several times -inv Invited user can reject invite for empty room -f,inv Invited user can reject invite over federation for empty room -inv Invited user can reject local invite after originator leaves -inv Invited user can see room metadata -f,inv Remote invited user can see room metadata -inv Users cannot invite themselves to a room -inv Users cannot invite a user that is already in the room -ban Banned user is kicked and may not rejoin until unbanned -f,ban Remote banned user is kicked and may not rejoin until unbanned -ban 'ban' event respects room powerlevel -plv setting 'm.room.name' respects room powerlevel -plv setting 'm.room.power_levels' respects room powerlevel (2 subtests) -plv Unprivileged users can set m.room.topic if it only needs level 0 -plv Users cannot set ban powerlevel higher than their own (2 subtests) -plv Users cannot set kick powerlevel higher than their own (2 subtests) -plv Users cannot set redact powerlevel higher than their own (2 subtests) -v1s Check that event streams started after a client joined a room work (SYT-1) -v1s Event stream catches up fully after many messages -xxx POST /rooms/:room_id/redact/:event_id as power user redacts message -xxx POST /rooms/:room_id/redact/:event_id as original message sender redacts message -xxx POST /rooms/:room_id/redact/:event_id as random user does not redact message -xxx POST /redact disallows redaction of event in different room -xxx Redaction of a redaction redacts the redaction reason -v1s A departed room is still included in /initialSync (SPEC-216) -v1s Can get rooms/{roomId}/initialSync for a departed room (SPEC-216) -rst Can get rooms/{roomId}/state for a departed room (SPEC-216) -mem Can get rooms/{roomId}/members for a departed room (SPEC-216) -get Can get rooms/{roomId}/messages for a departed room (SPEC-216) -rst Can get 'm.room.name' state for a departed room (SPEC-216) -syn Getting messages going forward is limited for a departed room (SPEC-216) -3pd Can invite existing 3pid -3pd Can invite existing 3pid with no ops into a private room -3pd Can invite existing 3pid in createRoom -3pd Can invite unbound 3pid -f,3pd Can invite unbound 3pid over federation -3pd Can invite unbound 3pid with no ops into a private room -f,3pd Can invite unbound 3pid over federation with no ops into a private room -f,3pd Can invite unbound 3pid over federation with users from both servers -3pd Can accept unbound 3pid invite after inviter leaves -3pd Can accept third party invite with /join -3pd 3pid invite join with wrong but valid signature are rejected -3pd 3pid invite join valid signature but revoked keys are rejected -3pd 3pid invite join valid signature but unreachable ID server are rejected -gst Guest user cannot call /events globally -gst Guest users can join guest_access rooms -gst Guest users can send messages to guest_access rooms if joined -gst Guest user calling /events doesn't tightloop -gst Guest users are kicked from guest_access rooms on revocation of guest_access -gst Guest user can set display names -gst Guest users are kicked from guest_access rooms on revocation of guest_access over federation -gst Guest user can upgrade to fully featured user -gst Guest user cannot upgrade other users -pub GET /publicRooms lists rooms -pub GET /publicRooms includes avatar URLs -gst Guest users can accept invites to private rooms over federation -gst Guest users denied access over federation if guest access prohibited -mem Room members can override their displayname on a room-specific basis -mem Room members can join a room with an overridden displayname -mem Users cannot kick users from a room they are not in -mem Users cannot kick users who have already left a room -typ Typing notification sent to local room members -f,typ Typing notifications also sent to remote room members -typ Typing can be explicitly stopped -rct Read receipts are visible to /initialSync -rct Read receipts are sent as events -rct Receipts must be m.read -pro displayname updates affect room member events -pro avatar_url updates affect room member events -gst m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users -gst m.room.history_visibility == "shared" allows/forbids appropriately for Guest users -gst m.room.history_visibility == "invited" allows/forbids appropriately for Guest users -gst m.room.history_visibility == "joined" allows/forbids appropriately for Guest users -gst m.room.history_visibility == "default" allows/forbids appropriately for Guest users -gst Guest non-joined user cannot call /events on shared room -gst Guest non-joined user cannot call /events on invited room -gst Guest non-joined user cannot call /events on joined room -gst Guest non-joined user cannot call /events on default room -gst Guest non-joined user can call /events on world_readable room -gst Guest non-joined users can get state for world_readable rooms -gst Guest non-joined users can get individual state for world_readable rooms -gst Guest non-joined users cannot room initalSync for non-world_readable rooms -gst Guest non-joined users can room initialSync for world_readable rooms -gst Guest non-joined users can get individual state for world_readable rooms after leaving -gst Guest non-joined users cannot send messages to guest_access rooms if not joined -gst Guest users can sync from world_readable guest_access rooms if joined -gst Guest users can sync from shared guest_access rooms if joined -gst Guest users can sync from invited guest_access rooms if joined -gst Guest users can sync from joined guest_access rooms if joined -gst Guest users can sync from default guest_access rooms if joined -ath m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users -ath m.room.history_visibility == "shared" allows/forbids appropriately for Real users -ath m.room.history_visibility == "invited" allows/forbids appropriately for Real users -ath m.room.history_visibility == "joined" allows/forbids appropriately for Real users -ath m.room.history_visibility == "default" allows/forbids appropriately for Real users -ath Real non-joined user cannot call /events on shared room -ath Real non-joined user cannot call /events on invited room -ath Real non-joined user cannot call /events on joined room -ath Real non-joined user cannot call /events on default room -ath Real non-joined user can call /events on world_readable room -ath Real non-joined users can get state for world_readable rooms -ath Real non-joined users can get individual state for world_readable rooms -ath Real non-joined users cannot room initalSync for non-world_readable rooms -ath Real non-joined users can room initialSync for world_readable rooms -ath Real non-joined users can get individual state for world_readable rooms after leaving -ath Real non-joined users cannot send messages to guest_access rooms if not joined -ath Real users can sync from world_readable guest_access rooms if joined -ath Real users can sync from shared guest_access rooms if joined -ath Real users can sync from invited guest_access rooms if joined -ath Real users can sync from joined guest_access rooms if joined -ath Real users can sync from default guest_access rooms if joined -ath Only see history_visibility changes on boundaries -f,ath Backfill works correctly with history visibility set to joined -fgt Forgotten room messages cannot be paginated -fgt Forgetting room does not show up in v2 /sync -fgt Can forget room you've been kicked from -fgt Can't forget room you're still in -fgt Can re-join room if re-invited -ath Only original members of the room can see messages from erased users -mem /joined_rooms returns only joined rooms -mem /joined_members return joined members -ctx /context/ on joined room works -ctx /context/ on non world readable room does not work -ctx /context/ returns correct number of events -ctx /context/ with lazy_load_members filter works -get /event/ on joined room works -get /event/ on non world readable room does not work -get /event/ does not allow access to events before the user joined -mem Can get rooms/{roomId}/members -mem Can get rooms/{roomId}/members at a given point -mem Can filter rooms/{roomId}/members -upg /upgrade creates a new room -upg /upgrade should preserve room visibility for public rooms -upg /upgrade should preserve room visibility for private rooms -upg /upgrade copies >100 power levels to the new room -upg /upgrade copies the power levels to the new room -upg /upgrade preserves the power level of the upgrading user in old and new rooms -upg /upgrade copies important state to the new room -upg /upgrade copies ban events to the new room -upg local user has push rules copied to upgraded room -f,upg remote user has push rules copied to upgraded room -upg /upgrade moves aliases to the new room -upg /upgrade moves remote aliases to the new room -upg /upgrade preserves direct room state -upg /upgrade preserves room federation ability -upg /upgrade restricts power levels in the old room -upg /upgrade restricts power levels in the old room when the old PLs are unusual -upg /upgrade to an unknown version is rejected -upg /upgrade is rejected if the user can't send state events -upg /upgrade of a bogus room fails gracefully -upg Cannot send tombstone event that points to the same room -f,upg Local and remote users' homeservers remove a room from their public directory on upgrade -rst Name/topic keys are correct -f,pub Can get remote public room list -pub Can paginate public room list -pub Can search public room list -syn Can create filter -syn Can download filter -syn Can sync -syn Can sync a joined room -syn Full state sync includes joined rooms -syn Newly joined room is included in an incremental sync -syn Newly joined room has correct timeline in incremental sync -syn Newly joined room includes presence in incremental sync -syn Get presence for newly joined members in incremental sync -syn Can sync a room with a single message -syn Can sync a room with a message with a transaction id -syn A message sent after an initial sync appears in the timeline of an incremental sync. -syn A filtered timeline reaches its limit -syn Syncing a new room with a large timeline limit isn't limited -syn A full_state incremental update returns only recent timeline -syn A prev_batch token can be used in the v1 messages API -syn A next_batch token can be used in the v1 messages API -syn User sees their own presence in a sync -syn User is offline if they set_presence=offline in their sync -syn User sees updates to presence from other users in the incremental sync. -syn State is included in the timeline in the initial sync -f,syn State from remote users is included in the state in the initial sync -syn Changes to state are included in an incremental sync -syn Changes to state are included in an gapped incremental sync -f,syn State from remote users is included in the timeline in an incremental sync -syn A full_state incremental update returns all state -syn When user joins a room the state is included in the next sync -syn A change to displayname should not result in a full state sync -syn A change to displayname should appear in incremental /sync -syn When user joins a room the state is included in a gapped sync -syn When user joins and leaves a room in the same batch, the full state is still included in the next sync -syn Current state appears in timeline in private history -syn Current state appears in timeline in private history with many messages before -syn Current state appears in timeline in private history with many messages after -syn Rooms a user is invited to appear in an initial sync -syn Rooms a user is invited to appear in an incremental sync -syn Newly joined room is included in an incremental sync after invite -syn Sync can be polled for updates -syn Sync is woken up for leaves -syn Left rooms appear in the leave section of sync -syn Newly left rooms appear in the leave section of incremental sync -syn We should see our own leave event, even if history_visibility is restricted (SYN-662) -syn We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462) -syn Newly left rooms appear in the leave section of gapped sync -syn Previously left rooms don't appear in the leave section of sync -syn Left rooms appear in the leave section of full state sync -syn Archived rooms only contain history from before the user left -syn Banned rooms appear in the leave section of sync -syn Newly banned rooms appear in the leave section of incremental sync -syn Newly banned rooms appear in the leave section of incremental sync -syn Typing events appear in initial sync -syn Typing events appear in incremental sync -syn Typing events appear in gapped sync -syn Read receipts appear in initial v2 /sync -syn New read receipts appear in incremental v2 /sync -syn Can pass a JSON filter as a query parameter -syn Can request federation format via the filter -syn Read markers appear in incremental v2 /sync -syn Read markers appear in initial v2 /sync -syn Read markers can be updated -syn Lazy loading parameters in the filter are strictly boolean -syn The only membership state included in an initial sync is for all the senders in the timeline -syn The only membership state included in an incremental sync is for senders in the timeline -syn The only membership state included in a gapped incremental sync is for senders in the timeline -syn Gapped incremental syncs include all state changes -syn Old leaves are present in gapped incremental syncs -syn Leaves are present in non-gapped incremental syncs -syn Old members are included in gappy incr LL sync if they start speaking -syn Members from the gap are included in gappy incr LL sync -syn We don't send redundant membership state across incremental syncs by default -syn We do send redundant membership state across incremental syncs if asked -syn Unnamed room comes with a name summary -syn Named room comes with just joined member count summary -syn Room summary only has 5 heroes -syn Room summary counts change when membership changes -rmv User can create and send/receive messages in a room with version 1 -rmv User can create and send/receive messages in a room with version 1 (2 subtests) -rmv local user can join room with version 1 -rmv User can invite local user to room with version 1 -rmv remote user can join room with version 1 -rmv User can invite remote user to room with version 1 -rmv Remote user can backfill in a room with version 1 -rmv Can reject invites over federation for rooms with version 1 -rmv Can receive redactions from regular users over federation in room version 1 -rmv User can create and send/receive messages in a room with version 2 -rmv User can create and send/receive messages in a room with version 2 (2 subtests) -rmv local user can join room with version 2 -rmv User can invite local user to room with version 2 -rmv remote user can join room with version 2 -rmv User can invite remote user to room with version 2 -rmv Remote user can backfill in a room with version 2 -rmv Can reject invites over federation for rooms with version 2 -rmv Can receive redactions from regular users over federation in room version 2 -rmv User can create and send/receive messages in a room with version 3 -rmv User can create and send/receive messages in a room with version 3 (2 subtests) -rmv local user can join room with version 3 -rmv User can invite local user to room with version 3 -rmv remote user can join room with version 3 -rmv User can invite remote user to room with version 3 -rmv Remote user can backfill in a room with version 3 -rmv Can reject invites over federation for rooms with version 3 -rmv Can receive redactions from regular users over federation in room version 3 -rmv User can create and send/receive messages in a room with version 4 -rmv User can create and send/receive messages in a room with version 4 (2 subtests) -rmv local user can join room with version 4 -rmv User can invite local user to room with version 4 -rmv remote user can join room with version 4 -rmv User can invite remote user to room with version 4 -rmv Remote user can backfill in a room with version 4 -rmv Can reject invites over federation for rooms with version 4 -rmv Can receive redactions from regular users over federation in room version 4 -rmv User can create and send/receive messages in a room with version 5 -rmv User can create and send/receive messages in a room with version 5 (2 subtests) -rmv local user can join room with version 5 -rmv User can invite local user to room with version 5 -rmv remote user can join room with version 5 -rmv User can invite remote user to room with version 5 -rmv Remote user can backfill in a room with version 5 -rmv Can reject invites over federation for rooms with version 5 -rmv Can receive redactions from regular users over federation in room version 5 -rmv User can create and send/receive messages in a room with version 6 -rmv User can create and send/receive messages in a room with version 6 (2 subtests) -rmv local user can join room with version 6 -rmv User can invite local user to room with version 6 -rmv remote user can join room with version 6 -rmv User can invite remote user to room with version 6 -rmv Remote user can backfill in a room with version 6 -rmv Can reject invites over federation for rooms with version 6 -rmv Can receive redactions from regular users over federation in room version 6 -rmv Inbound federation rejects invites which include invalid JSON for room version 6 -rmv Outbound federation rejects invite response which include invalid JSON for room version 6 -rmv Inbound federation rejects invite rejections which include invalid JSON for room version 6 -rmv Server rejects invalid JSON in a version 6 room -pre Presence changes are reported to local room members -f,pre Presence changes are also reported to remote room members -pre Presence changes to UNAVAILABLE are reported to local room members -f,pre Presence changes to UNAVAILABLE are reported to remote room members -v1s Newly created users see their own presence in /initialSync (SYT-34) -dvk Can upload device keys -dvk Should reject keys claiming to belong to a different user -dvk Can query device keys using POST -dvk Can query specific device keys using POST -dvk query for user with no keys returns empty key dict -dvk Can claim one time key using POST -f,dvk Can query remote device keys using POST -f,dvk Can claim remote one time key using POST -dvk Local device key changes appear in v2 /sync -dvk Local new device changes appear in v2 /sync -dvk Local delete device changes appear in v2 /sync -dvk Local update device changes appear in v2 /sync -dvk Can query remote device keys using POST after notification -f,dev Device deletion propagates over federation -f,dev If remote user leaves room, changes device and rejoins we see update in sync -f,dev If remote user leaves room we no longer receive device updates -dvk Local device key changes appear in /keys/changes -dvk New users appear in /keys/changes -f,dvk If remote user leaves room, changes device and rejoins we see update in /keys/changes -dvk Get left notifs in sync and /keys/changes when other user leaves -dvk Get left notifs for other users in sync and /keys/changes when user leaves -f,dvk If user leaves room, remote user changes device and rejoins we see update in /sync and /keys/changes -dkb Can create backup version -dkb Can update backup version -dkb Responds correctly when backup is empty -dkb Can backup keys -dkb Can update keys with better versions -dkb Will not update keys with worse versions -dkb Will not back up to an old backup version -dkb Can delete backup -dkb Deleted & recreated backups are empty -dkb Can create more than 10 backup versions -xsk Can upload self-signing keys -xsk Fails to upload self-signing keys with no auth -xsk Fails to upload self-signing key without master key -xsk Changing master key notifies local users -xsk Changing user-signing key notifies local users -f,xsk can fetch self-signing keys over federation -f,xsk uploading self-signing key notifies over federation -f,xsk uploading signed devices gets propagated over federation -tag Can add tag -tag Can remove tag -tag Can list tags for a room -v1s Tags appear in the v1 /events stream -v1s Tags appear in the v1 /initalSync -v1s Tags appear in the v1 room initial sync -tag Tags appear in an initial v2 /sync -tag Newly updated tags appear in an incremental v2 /sync -tag Deleted tags appear in an incremental v2 /sync -tag local user has tags copied to the new room -f,tag remote user has tags copied to the new room -sch Can search for an event by body -sch Can get context around search results -sch Can back-paginate search results -sch Search works across an upgraded room and its predecessor -sch Search results with rank ordering do not include redacted events -sch Search results with recent ordering do not include redacted events -acc Can add account data -acc Can add account data to room -acc Can get account data without syncing -acc Can get room account data without syncing -v1s Latest account data comes down in /initialSync -v1s Latest account data comes down in room initialSync -v1s Account data appears in v1 /events stream -v1s Room account data appears in v1 /events stream -acc Latest account data appears in v2 /sync -acc New account data appears in incremental v2 /sync -oid Can generate a openid access_token that can be exchanged for information about a user -oid Invalid openid access tokens are rejected -oid Requests to userinfo without access tokens are rejected -std Can send a message directly to a device using PUT /sendToDevice -std Can recv a device message using /sync -std Can recv device messages until they are acknowledged -std Device messages with the same txn_id are deduplicated -std Device messages wake up /sync -std Can recv device messages over federation -fsd Device messages over federation wake up /sync -std Can send messages with a wildcard device id -std Can send messages with a wildcard device id to two devices -std Wildcard device messages wake up /sync -fsd Wildcard device messages over federation wake up /sync -adm /whois -nsp /purge_history -nsp /purge_history by ts -nsp Can backfill purged history -nsp Shutdown room -ign Ignore user in existing room -ign Ignore invite in full sync -ign Ignore invite in incremental sync -fky Checking local federation server -fky Federation key API allows unsigned requests for keys -fky Federation key API can act as a notary server via a GET request -fky Federation key API can act as a notary server via a POST request -fky Key notary server should return an expired key if it can't find any others -fky Key notary server must not overwrite a valid key with a spurious result from the origin server -fqu Non-numeric ports in server names are rejected -fqu Outbound federation can query profile data -fqu Inbound federation can query profile data -fqu Outbound federation can query room alias directory -fqu Inbound federation can query room alias directory -fsj Outbound federation can query v1 /send_join -fsj Outbound federation can query v2 /send_join -fmj Outbound federation passes make_join failures through to the client -fsj Inbound federation can receive v1 /send_join -fsj Inbound federation can receive v2 /send_join -fmj Inbound /v1/make_join rejects remote attempts to join local users to rooms -fsj Inbound /v1/send_join rejects incorrectly-signed joins -fsj Inbound /v1/send_join rejects joins from other servers -fau Inbound federation rejects remote attempts to kick local users to rooms -frv Inbound federation rejects attempts to join v1 rooms from servers without v1 support -frv Inbound federation rejects attempts to join v2 rooms from servers lacking version support -frv Inbound federation rejects attempts to join v2 rooms from servers only supporting v1 -frv Inbound federation accepts attempts to join v2 rooms from servers with support -frv Outbound federation correctly handles unsupported room versions -frv A pair of servers can establish a join in a v2 room -fsj Outbound federation rejects send_join responses with no m.room.create event -frv Outbound federation rejects m.room.create events with an unknown room version -fsj Event with an invalid signature in the send_join response should not cause room join to fail -fsj Inbound: send_join rejects invalid JSON for room version 6 -fed Outbound federation can send events -fed Inbound federation can receive events -fed Inbound federation can receive redacted events -fed Ephemeral messages received from servers are correctly expired -fed Events whose auth_events are in the wrong room do not mess up the room state -fed Inbound federation can return events -fed Inbound federation redacts events from erased users -fme Outbound federation can request missing events -fme Inbound federation can return missing events for world_readable visibility -fme Inbound federation can return missing events for shared visibility -fme Inbound federation can return missing events for invite visibility -fme Inbound federation can return missing events for joined visibility -fme outliers whose auth_events are in a different room are correctly rejected -fbk Outbound federation can backfill events -fbk Inbound federation can backfill events -fbk Backfill checks the events requested belong to the room -fbk Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination -fiv Outbound federation can send invites via v1 API -fiv Outbound federation can send invites via v2 API -fiv Inbound federation can receive invites via v1 API -fiv Inbound federation can receive invites via v2 API -fiv Inbound federation can receive invite and reject when remote replies with a 403 -fiv Inbound federation can receive invite and reject when remote replies with a 500 -fiv Inbound federation can receive invite and reject when remote is unreachable -fiv Inbound federation rejects invites which are not signed by the sender -fiv Inbound federation can receive invite rejections -fiv Inbound federation rejects incorrectly-signed invite rejections -fsl Inbound /v1/send_leave rejects leaves from other servers -fst Inbound federation can get state for a room -fst Inbound federation of state requires event_id as a mandatory paramater -fst Inbound federation can get state_ids for a room -fst Inbound federation of state_ids requires event_id as a mandatory paramater -fst Federation rejects inbound events where the prev_events cannot be found -fst Room state at a rejected message event is the same as its predecessor -fst Room state at a rejected state event is the same as its predecessor -fst Outbound federation requests missing prev_events and then asks for /state_ids and resolves the state -fst Federation handles empty auth_events in state_ids sanely -fst Getting state checks the events requested belong to the room -fst Getting state IDs checks the events requested belong to the room -fst Should not be able to take over the room by pretending there is no PL event -fpb Inbound federation can get public room list -fed Outbound federation sends receipts -fed Inbound federation rejects receipts from wrong remote -fed Inbound federation ignores redactions from invalid servers room > v3 -fed An event which redacts an event in a different room should be ignored -fed An event which redacts itself should be ignored -fed A pair of events which redact each other should be ignored -fdk Local device key changes get to remote servers -fdk Server correctly handles incoming m.device_list_update -fdk Server correctly resyncs when client query keys and there is no remote cache -fdk Server correctly resyncs when server leaves and rejoins a room -fdk Local device key changes get to remote servers with correct prev_id -fdk Device list doesn't change if remote server is down -fdk If a device list update goes missing, the server resyncs on the next one -fst Name/topic keys are correct -fau Remote servers cannot set power levels in rooms without existing powerlevels -fau Remote servers should reject attempts by non-creators to set the power levels -fau Inbound federation rejects typing notifications from wrong remote -fau Users cannot set notifications powerlevel higher than their own -fed Forward extremities remain so even after the next events are populated as outliers -fau Banned servers cannot send events -fau Banned servers cannot /make_join -fau Banned servers cannot /send_join -fau Banned servers cannot /make_leave -fau Banned servers cannot /send_leave -fau Banned servers cannot /invite -fau Banned servers cannot get room state -fau Banned servers cannot get room state ids -fau Banned servers cannot backfill -fau Banned servers cannot /event_auth -fau Banned servers cannot get missing events -fau Server correctly handles transactions that break edu limits -fau Inbound federation correctly soft fails events -fau Inbound federation accepts a second soft-failed event -fau Inbound federation correctly handles soft failed events as extremities -med Can upload with Unicode file name -med Can download with Unicode file name locally -f,med Can download with Unicode file name over federation -med Alternative server names do not cause a routing loop -med Can download specifying a different Unicode file name -med Can upload without a file name -med Can download without a file name locally -f,med Can download without a file name over federation -med Can upload with ASCII file name -med Can download file 'ascii' -med Can download file 'name with spaces' -med Can download file 'name;with;semicolons' -med Can download specifying a different ASCII file name -med Can send image in room message -med Can fetch images in room -med POSTed media can be thumbnailed -f,med Remote media can be thumbnailed -med Test URL preview -med Can read configuration endpoint -nsp Can quarantine media in rooms -udr User appears in user directory -udr User in private room doesn't appear in user directory -udr User joining then leaving public room appears and dissappears from directory -udr Users appear/disappear from directory when join_rules are changed -udr Users appear/disappear from directory when history_visibility are changed -udr Users stay in directory when join_rules are changed but history_visibility is world_readable -f,udr User in remote room doesn't appear in user directory after server left room -udr User directory correctly update on display name change -udr User in shared private room does appear in user directory -udr User in shared private room does appear in user directory until leave -udr User in dir while user still shares private rooms -nsp Create group -nsp Add group rooms -nsp Remove group rooms -nsp Get local group profile -nsp Get local group users -nsp Add/remove local group rooms -nsp Get local group summary -nsp Get remote group profile -nsp Get remote group users -nsp Add/remove remote group rooms -nsp Get remote group summary -nsp Add local group users -nsp Remove self from local group -nsp Remove other from local group -nsp Add remote group users -nsp Remove self from remote group -nsp Listing invited users of a remote group when not a member returns a 403 -nsp Add group category -nsp Remove group category -nsp Get group categories -nsp Add group role -nsp Remove group role -nsp Get group roles -nsp Add room to group summary -nsp Adding room to group summary keeps room_id when fetching rooms in group -nsp Adding multiple rooms to group summary have correct order -nsp Remove room from group summary -nsp Add room to group summary with category -nsp Remove room from group summary with category -nsp Add user to group summary -nsp Adding multiple users to group summary have correct order -nsp Remove user from group summary -nsp Add user to group summary with role -nsp Remove user from group summary with role -nsp Local group invites come down sync -nsp Group creator sees group in sync -nsp Group creator sees group in initial sync -nsp Get/set local group publicity -nsp Bulk get group publicity -nsp Joinability comes down summary -nsp Set group joinable and join it -nsp Group is not joinable by default -nsp Group is joinable over federation -nsp Room is transitioned on local and remote groups upon room upgrade -3pd Can bind 3PID via home server -3pd Can bind and unbind 3PID via homeserver -3pd Can unbind 3PID via homeserver when bound out of band -3pd 3PIDs are unbound after account deactivation -3pd Can bind and unbind 3PID via /unbind by specifying the identity server -3pd Can bind and unbind 3PID via /unbind without specifying the identity server -app AS can create a user -app AS can create a user with an underscore -app AS can create a user with inhibit_login -app AS cannot create users outside its own namespace -app Regular users cannot register within the AS namespace -app AS can make room aliases -app Regular users cannot create room aliases within the AS namespace -app AS-ghosted users can use rooms via AS -app AS-ghosted users can use rooms themselves -app Ghost user must register before joining room -app AS can set avatar for ghosted users -app AS can set displayname for ghosted users -app AS can't set displayname for random users -app Inviting an AS-hosted user asks the AS server -app Accesing an AS-hosted room alias asks the AS server -app Events in rooms with AS-hosted room aliases are sent to AS server -app AS user (not ghost) can join room without registering -app AS user (not ghost) can join room without registering, with user_id query param -app HS provides query metadata -app HS can provide query metadata on a single protocol -app HS will proxy request for 3PU mapping -app HS will proxy request for 3PL mapping -app AS can publish rooms in their own list -app AS and main public room lists are separate -app AS can deactivate a user -psh Test that a message is pushed -psh Invites are pushed -psh Rooms with names are correctly named in pushed -psh Rooms with canonical alias are correctly named in pushed -psh Rooms with many users are correctly pushed -psh Don't get pushed for rooms you've muted -psh Rejected events are not pushed -psh Can add global push rule for room -psh Can add global push rule for sender -psh Can add global push rule for content -psh Can add global push rule for override -psh Can add global push rule for underride -psh Can add global push rule for content -psh New rules appear before old rules by default -psh Can add global push rule before an existing rule -psh Can add global push rule after an existing rule -psh Can delete a push rule -psh Can disable a push rule -psh Adding the same push rule twice is idempotent -psh Messages that notify from another user increment unread notification count -psh Messages that highlight from another user increment unread highlight count -psh Can change the actions of default rules -psh Changing the actions of an unknown default rule fails with 404 -psh Can change the actions of a user specified rule -psh Changing the actions of an unknown rule fails with 404 -psh Can fetch a user's pushers -psh Push rules come down in an initial /sync -psh Adding a push rule wakes up an incremental /sync -psh Disabling a push rule wakes up an incremental /sync -psh Enabling a push rule wakes up an incremental /sync -psh Setting actions for a push rule wakes up an incremental /sync -psh Can enable/disable default rules -psh Enabling an unknown default rule fails with 404 -psh Test that rejected pushers are removed. -psh Notifications can be viewed with GET /notifications -psh Trying to add push rule with no scope fails with 400 -psh Trying to add push rule with invalid scope fails with 400 -psh Trying to add push rule with missing template fails with 400 -psh Trying to add push rule with missing rule_id fails with 400 -psh Trying to add push rule with empty rule_id fails with 400 -psh Trying to add push rule with invalid template fails with 400 -psh Trying to add push rule with rule_id with slashes fails with 400 -psh Trying to add push rule with override rule without conditions fails with 400 -psh Trying to add push rule with underride rule without conditions fails with 400 -psh Trying to add push rule with condition without kind fails with 400 -psh Trying to add push rule with content rule without pattern fails with 400 -psh Trying to add push rule with no actions fails with 400 -psh Trying to add push rule with invalid action fails with 400 -psh Trying to add push rule with invalid attr fails with 400 -psh Trying to add push rule with invalid value for enabled fails with 400 -psh Trying to get push rules with no trailing slash fails with 400 -psh Trying to get push rules with scope without trailing slash fails with 400 -psh Trying to get push rules with template without tailing slash fails with 400 -psh Trying to get push rules with unknown scope fails with 400 -psh Trying to get push rules with unknown template fails with 400 -psh Trying to get push rules with unknown attribute fails with 400 -psh Trying to get push rules with unknown rule_id fails with 404 -psh Rooms with names are correctly named in pushes -v1s GET /initialSync with non-numeric 'limit' -v1s GET /events with non-numeric 'limit' -v1s GET /events with negative 'limit' -v1s GET /events with non-numeric 'timeout' -ath Event size limits -syn Check creating invalid filters returns 4xx -f,pre New federated private chats get full presence information (SYN-115) -pre Left room members do not cause problems for presence -crm Rooms can be created with an initial invite list (SYN-205) (1 subtests) -typ Typing notifications don't leak -ban Non-present room members cannot ban others -psh Getting push rules doesn't corrupt the cache SYN-390 -inv Test that we can be reinvited to a room we created -syn Multiple calls to /sync should not cause 500 errors -gst Guest user can call /events on another world_readable room (SYN-606) -gst Real user can call /events on another world_readable room (SYN-606) -gst Events come down the correct room -pub Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list -std Can send a to-device message to two users which both receive it using /sync -fme Outbound federation will ignore a missing event with bad JSON for room version 6 -fbk Outbound federation rejects backfill containing invalid JSON for events in room version 6 -jso Invalid JSON integers -jso Invalid JSON floats -jso Invalid JSON special values -inv Can invite users to invite-only rooms (2 subtests) -plv setting 'm.room.name' respects room powerlevel (2 subtests) -psh Messages that notify from another user increment notification_count -psh Messages that org.matrix.msc2625.mark_unread from another user increment org.matrix.msc2625.unread_count -dvk Can claim one time key using POST (2 subtests) -fdk Can query remote device keys using POST (1 subtests) -fdk Can claim remote one time key using POST (2 subtests) -fmj Inbound /make_join rejects attempts to join rooms where all users have left \ No newline at end of file diff --git a/tests/sytest/are-we-synapse-yet.py b/tests/sytest/are-we-synapse-yet.py deleted file mode 100755 index 3d21fa41..00000000 --- a/tests/sytest/are-we-synapse-yet.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python3 - -from __future__ import division -import argparse -import re -import sys - -# Usage: $ ./are-we-synapse-yet.py [-v] results.tap -# This script scans a results.tap file from Dendrite's CI process and spits out -# a rating of how close we are to Synapse parity, based purely on SyTests. -# The main complexity is grouping tests sensibly into features like 'Registration' -# and 'Federation'. Then it just checks the ones which are passing and calculates -# percentages for each group. Produces results like: -# -# Client-Server APIs: 29% (196/666 tests) -# ------------------- -# Registration : 62% (20/32 tests) -# Login : 7% (1/15 tests) -# V1 CS APIs : 10% (3/30 tests) -# ... -# -# or in verbose mode: -# -# Client-Server APIs: 29% (196/666 tests) -# ------------------- -# Registration : 62% (20/32 tests) -# ✓ GET /register yields a set of flows -# ✓ POST /register can create a user -# ✓ POST /register downcases capitals in usernames -# ... -# -# You can also tack `-v` on to see exactly which tests each category falls under. - -test_mappings = { - "nsp": "Non-Spec API", - "unk": "Unknown API (no group specified)", - "app": "Application Services API", - "f": "Federation", # flag to mark test involves federation - - "federation_apis": { - "fky": "Key API", - "fsj": "send_join API", - "fmj": "make_join API", - "fsl": "send_leave API", - "fiv": "Invite API", - "fqu": "Query API", - "frv": "room versions", - "fau": "Auth", - "fbk": "Backfill API", - "fme": "get_missing_events API", - "fst": "State APIs", - "fpb": "Public Room API", - "fdk": "Device Key APIs", - "fed": "Federation API", - "fsd": "Send-to-Device APIs", - }, - - "client_apis": { - "reg": "Registration", - "log": "Login", - "lox": "Logout", - "v1s": "V1 CS APIs", - "csa": "Misc CS APIs", - "pro": "Profile", - "dev": "Devices", - "dvk": "Device Keys", - "dkb": "Device Key Backup", - "xsk": "Cross-signing Keys", - "pre": "Presence", - "crm": "Create Room", - "syn": "Sync API", - "rmv": "Room Versions", - "rst": "Room State APIs", - "pub": "Public Room APIs", - "mem": "Room Membership", - "ali": "Room Aliases", - "jon": "Joining Rooms", - "lev": "Leaving Rooms", - "inv": "Inviting users to Rooms", - "ban": "Banning users", - "snd": "Sending events", - "get": "Getting events for Rooms", - "rct": "Receipts", - "red": "Read markers", - "med": "Media APIs", - "cap": "Capabilities API", - "typ": "Typing API", - "psh": "Push APIs", - "acc": "Account APIs", - "eph": "Ephemeral Events", - "plv": "Power Levels", - "xxx": "Redaction", - "3pd": "Third-Party ID APIs", - "gst": "Guest APIs", - "ath": "Room Auth", - "fgt": "Forget APIs", - "ctx": "Context APIs", - "upg": "Room Upgrade APIs", - "tag": "Tagging APIs", - "sch": "Search APIs", - "oid": "OpenID API", - "std": "Send-to-Device APIs", - "adm": "Server Admin API", - "ign": "Ignore Users", - "udr": "User Directory APIs", - "jso": "Enforced canonical JSON", - }, -} - -# optional 'not ' with test number then anything but '#' -re_testname = re.compile(r"^(not )?ok [0-9]+ ([^#]+)") - -# Parses lines like the following: -# -# SUCCESS: ok 3 POST /register downcases capitals in usernames -# FAIL: not ok 54 (expected fail) POST /createRoom creates a room with the given version -# SKIP: ok 821 Multiple calls to /sync should not cause 500 errors # skip lack of can_post_room_receipts -# EXPECT FAIL: not ok 822 (expected fail) Guest user can call /events on another world_readable room (SYN-606) # TODO expected fail -# -# Only SUCCESS lines are treated as success, the rest are not implemented. -# -# Returns a dict like: -# { name: "...", ok: True } -def parse_test_line(line): - if not line.startswith("ok ") and not line.startswith("not ok "): - return - re_match = re_testname.match(line) - test_name = re_match.groups()[1].replace("(expected fail) ", "").strip() - test_pass = False - if line.startswith("ok ") and not "# skip " in line: - test_pass = True - return { - "name": test_name, - "ok": test_pass, - } - -# Prints the stats for a complete section. -# header_name => "Client-Server APIs" -# gid_to_tests => { gid: { : True|False }} -# gid_to_name => { gid: "Group Name" } -# verbose => True|False -# Produces: -# Client-Server APIs: 29% (196/666 tests) -# ------------------- -# Registration : 62% (20/32 tests) -# Login : 7% (1/15 tests) -# V1 CS APIs : 10% (3/30 tests) -# ... -# or in verbose mode: -# Client-Server APIs: 29% (196/666 tests) -# ------------------- -# Registration : 62% (20/32 tests) -# ✓ GET /register yields a set of flows -# ✓ POST /register can create a user -# ✓ POST /register downcases capitals in usernames -# ... -def print_stats(header_name, gid_to_tests, gid_to_name, verbose): - subsections = [] # Registration: 100% (13/13 tests) - subsection_test_names = {} # 'subsection name': ["✓ Test 1", "✓ Test 2", "× Test 3"] - total_passing = 0 - total_tests = 0 - for gid, tests in gid_to_tests.items(): - group_total = len(tests) - if group_total == 0: - continue - group_passing = 0 - test_names_and_marks = [] - for name, passing in tests.items(): - if passing: - group_passing += 1 - test_names_and_marks.append(f"{'✓' if passing else '×'} {name}") - - total_tests += group_total - total_passing += group_passing - pct = "{0:.0f}%".format(group_passing/group_total * 100) - line = "%s: %s (%d/%d tests)" % (gid_to_name[gid].ljust(25, ' '), pct.rjust(4, ' '), group_passing, group_total) - subsections.append(line) - subsection_test_names[line] = test_names_and_marks - - pct = "{0:.0f}%".format(total_passing/total_tests * 100) - print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests)) - print("-" * (len(header_name)+1)) - for line in subsections: - print(" %s" % (line,)) - if verbose: - for test_name_and_pass_mark in subsection_test_names[line]: - print(" %s" % (test_name_and_pass_mark,)) - print("") - print("") - -def main(results_tap_path, verbose): - # Load up test mappings - test_name_to_group_id = {} - fed_tests = set() - client_tests = set() - with open("./are-we-synapse-yet.list", "r") as f: - for line in f.readlines(): - test_name = " ".join(line.split(" ")[1:]).strip() - groups = line.split(" ")[0].split(",") - for gid in groups: - if gid == "f" or gid in test_mappings["federation_apis"]: - fed_tests.add(test_name) - else: - client_tests.add(test_name) - if gid == "f": - continue # we expect another group ID - test_name_to_group_id[test_name] = gid - - # parse results.tap - summary = { - "client": { - # gid: { - # test_name: OK - # } - }, - "federation": { - # gid: { - # test_name: OK - # } - }, - "appservice": { - "app": {}, - }, - "nonspec": { - "nsp": {}, - "unk": {} - }, - } - with open(results_tap_path, "r") as f: - for line in f.readlines(): - test_result = parse_test_line(line) - if not test_result: - continue - name = test_result["name"] - group_id = test_name_to_group_id.get(name) - if not group_id: - summary["nonspec"]["unk"][name] = test_result["ok"] - if group_id == "nsp": - summary["nonspec"]["nsp"][name] = test_result["ok"] - elif group_id == "app": - summary["appservice"]["app"][name] = test_result["ok"] - elif group_id in test_mappings["federation_apis"]: - group = summary["federation"].get(group_id, {}) - group[name] = test_result["ok"] - summary["federation"][group_id] = group - elif group_id in test_mappings["client_apis"]: - group = summary["client"].get(group_id, {}) - group[name] = test_result["ok"] - summary["client"][group_id] = group - - print("Are We Synapse Yet?") - print("===================") - print("") - print_stats("Non-Spec APIs", summary["nonspec"], test_mappings, verbose) - print_stats("Client-Server APIs", summary["client"], test_mappings["client_apis"], verbose) - print_stats("Federation APIs", summary["federation"], test_mappings["federation_apis"], verbose) - print_stats("Application Services APIs", summary["appservice"], test_mappings, verbose) - - - -if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument("tap_file", help="path to results.tap") - parser.add_argument("-v", action="store_true", help="show individual test names in output") - args = parser.parse_args() - main(args.tap_file, args.v) \ No newline at end of file diff --git a/tests/sytest/show-expected-fail-tests.sh b/tests/sytest/show-expected-fail-tests.sh deleted file mode 100755 index 320d4ebd..00000000 --- a/tests/sytest/show-expected-fail-tests.sh +++ /dev/null @@ -1,105 +0,0 @@ -#! /bin/bash -# -# Parses a results.tap file from SyTest output and a file containing test names (a test whitelist) -# and checks whether a test name that exists in the whitelist (that should pass), failed or not. -# -# An optional blacklist file can be added, also containing test names, where if a test name is -# present, the script will not error even if the test is in the whitelist file and failed -# -# For each of these files, lines starting with '#' are ignored. -# -# Usage ./show-expected-fail-tests.sh results.tap whitelist [blacklist] - -results_file=$1 -whitelist_file=$2 -blacklist_file=$3 - -fail_build=0 - -if [ $# -lt 2 ]; then - echo "Usage: $0 results.tap whitelist [blacklist]" - exit 1 -fi - -if [ ! -f "$results_file" ]; then - echo "ERROR: Specified results file '${results_file}' doesn't exist." - fail_build=1 -fi - -if [ ! -f "$whitelist_file" ]; then - echo "ERROR: Specified test whitelist '${whitelist_file}' doesn't exist." - fail_build=1 -fi - -blacklisted_tests=() - -# Check if a blacklist file was provided -if [ $# -eq 3 ]; then - # Read test blacklist file - if [ ! -f "$blacklist_file" ]; then - echo "ERROR: Specified test blacklist file '${blacklist_file}' doesn't exist." - fail_build=1 - fi - - # Read each line, ignoring those that start with '#' - blacklisted_tests="" - search_non_comments=$(grep -v '^#' ${blacklist_file}) - while read -r line ; do - # Record the blacklisted test name - blacklisted_tests+=("${line}") - done <<< "${search_non_comments}" # This allows us to edit blacklisted_tests in the while loop -fi - -[ "$fail_build" = 0 ] || exit 1 - -passed_but_expected_fail=$(grep ' # TODO passed but expected fail' ${results_file} | sed -E 's/^ok [0-9]+ (\(expected fail\) )?//' | sed -E 's/( \([0-9]+ subtests\))? # TODO passed but expected fail$//') -tests_to_add="" -already_in_whitelist="" - -while read -r test_name; do - # Ignore empty lines - [ "${test_name}" = "" ] && continue - - grep "^${test_name}" "${whitelist_file}" > /dev/null 2>&1 - if [ "$?" != "0" ]; then - # Check if this test name is blacklisted - if printf '%s\n' "${blacklisted_tests[@]}" | grep -q -P "^${test_name}$"; then - # Don't notify about this test - continue - fi - - # Append this test_name to the existing list - tests_to_add="${tests_to_add}${test_name}\n" - fail_build=1 - else - already_in_whitelist="${already_in_whitelist}${test_name}\n" - fi -done <<< "${passed_but_expected_fail}" - -# TODO: Check that the same test doesn't exist in both the whitelist and blacklist -# TODO: Check that the same test doesn't appear twice in the whitelist|blacklist - -# Trim test output strings -tests_to_add=$(IFS=$'\n' echo "${tests_to_add[*]%%'\n'}") -already_in_whitelist=$(IFS=$'\n' echo "${already_in_whitelist[*]%%'\n'}") - -# Format output with markdown for buildkite annotation rendering purposes -if [ -n "${tests_to_add}" ] && [ -n "${already_in_whitelist}" ]; then - echo "### 📜 SyTest Whitelist Maintenance" -fi - -if [ -n "${tests_to_add}" ]; then - echo "**ERROR**: The following tests passed but are not present in \`$2\`. Please append them to the file:" - echo "\`\`\`" - echo -e "${tests_to_add}" - echo "\`\`\`" -fi - -if [ -n "${already_in_whitelist}" ]; then - echo "**WARN**: Tests in the whitelist still marked as **expected fail**:" - echo "\`\`\`" - echo -e "${already_in_whitelist}" - echo "\`\`\`" -fi - -exit ${fail_build} diff --git a/tests/sytest/sytest-blacklist b/tests/sytest/sytest-blacklist deleted file mode 100644 index 009de225..00000000 --- a/tests/sytest/sytest-blacklist +++ /dev/null @@ -1,7 +0,0 @@ -# This test checks for a room-alias key in the response which is not in the spec, we must add it back in whitelist when https://github.com/matrix-org/sytest/pull/880 is merged -POST /createRoom makes a public room -# These fails because they use a endpoint which is not in the spec, we must add them back in whitelist when https://github.com/matrix-org/sytest/issues/878 is closed -POST /createRoom makes a room with a name -POST /createRoom makes a room with a topic -Can /sync newly created room -POST /createRoom ignores attempts to set the room version via creation_content \ No newline at end of file diff --git a/tests/sytest/sytest-whitelist b/tests/sytest/sytest-whitelist deleted file mode 100644 index 1c969dba..00000000 --- a/tests/sytest/sytest-whitelist +++ /dev/null @@ -1,516 +0,0 @@ -/event/ does not allow access to events before the user joined -/event/ on joined room works -/event/ on non world readable room does not work -/joined_members return joined members -/joined_rooms returns only joined rooms -/whois -3pid invite join valid signature but revoked keys are rejected -3pid invite join valid signature but unreachable ID server are rejected -3pid invite join with wrong but valid signature are rejected -A change to displayname should appear in incremental /sync -A full_state incremental update returns all state -A full_state incremental update returns only recent timeline -A message sent after an initial sync appears in the timeline of an incremental sync. -A next_batch token can be used in the v1 messages API -A pair of events which redact each other should be ignored -A pair of servers can establish a join in a v2 room -A prev_batch token can be used in the v1 messages API -AS can create a user -AS can create a user with an underscore -AS can create a user with inhibit_login -AS can set avatar for ghosted users -AS can set displayname for ghosted users -AS can't set displayname for random users -AS cannot create users outside its own namespace -AS user (not ghost) can join room without registering -AS user (not ghost) can join room without registering, with user_id query param -After changing password, a different session no longer works by default -After changing password, can log in with new password -After changing password, can't log in with old password -After changing password, different sessions can optionally be kept -After changing password, existing session still works -After deactivating account, can't log in with an email -After deactivating account, can't log in with password -Alias creators can delete alias with no ops -Alias creators can delete canonical alias with no ops -Alternative server names do not cause a routing loop -An event which redacts an event in a different room should be ignored -An event which redacts itself should be ignored -Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list -Backfill checks the events requested belong to the room -Backfill works correctly with history visibility set to joined -Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination -Banned servers cannot /event_auth -Banned servers cannot /invite -Banned servers cannot /make_join -Banned servers cannot /make_leave -Banned servers cannot /send_join -Banned servers cannot /send_leave -Banned servers cannot backfill -Banned servers cannot get missing events -Banned servers cannot get room state -Banned servers cannot get room state ids -Banned servers cannot send events -Banned user is kicked and may not rejoin until unbanned -Both GET and PUT work -Can /sync newly created room -Can add account data -Can add account data to room -Can add tag -Can claim one time key using POST -Can claim remote one time key using POST -Can create filter -Can deactivate account -Can delete canonical alias -Can download file 'ascii' -Can download file 'name with spaces' -Can download file 'name;with;semicolons' -Can download filter -Can download specifying a different ASCII file name -Can download specifying a different Unicode file name -Can download with Unicode file name locally -Can download with Unicode file name over federation -Can download without a file name locally -Can download without a file name over federation -Can forget room you've been kicked from -Can get 'm.room.name' state for a departed room (SPEC-216) -Can get account data without syncing -Can get remote public room list -Can get room account data without syncing -Can get rooms/{roomId}/members -Can get rooms/{roomId}/members for a departed room (SPEC-216) -Can get rooms/{roomId}/state for a departed room (SPEC-216) -Can invite users to invite-only rooms -Can list tags for a room -Can logout all devices -Can logout current device -Can paginate public room list -Can pass a JSON filter as a query parameter -Can query device keys using POST -Can query remote device keys using POST -Can query specific device keys using POST -Can re-join room if re-invited -Can read configuration endpoint -Can receive redactions from regular users over federation in room version 1 -Can receive redactions from regular users over federation in room version 2 -Can receive redactions from regular users over federation in room version 3 -Can receive redactions from regular users over federation in room version 4 -Can receive redactions from regular users over federation in room version 5 -Can receive redactions from regular users over federation in room version 6 -Can recv a device message using /sync -Can recv a device message using /sync -Can recv device messages over federation -Can recv device messages until they are acknowledged -Can recv device messages until they are acknowledged -Can reject invites over federation for rooms with version 1 -Can reject invites over federation for rooms with version 2 -Can reject invites over federation for rooms with version 3 -Can reject invites over federation for rooms with version 4 -Can reject invites over federation for rooms with version 5 -Can reject invites over federation for rooms with version 6 -Can remove tag -Can search public room list -Can send a message directly to a device using PUT /sendToDevice -Can send a message directly to a device using PUT /sendToDevice -Can send a to-device message to two users which both receive it using /sync -Can send image in room message -Can send messages with a wildcard device id -Can send messages with a wildcard device id -Can send messages with a wildcard device id to two devices -Can send messages with a wildcard device id to two devices -Can sync -Can sync a joined room -Can sync a room with a message with a transaction id -Can sync a room with a single message -Can upload device keys -Can upload with ASCII file name -Can upload with Unicode file name -Can upload without a file name -Can't deactivate account with wrong password -Can't forget room you're still in -Changes to state are included in an gapped incremental sync -Changes to state are included in an incremental sync -Changing the actions of an unknown default rule fails with 404 -Changing the actions of an unknown rule fails with 404 -Checking local federation server -Creators can delete alias -Current state appears in timeline in private history -Current state appears in timeline in private history with many messages before -DELETE /device/{deviceId} -DELETE /device/{deviceId} requires UI auth user to match device owner -DELETE /device/{deviceId} with no body gives a 401 -Deleted tags appear in an incremental v2 /sync -Deleting a non-existent alias should return a 404 -Device list doesn't change if remote server is down -Device messages over federation wake up /sync -Device messages wake up /sync -Device messages wake up /sync -Device messages with the same txn_id are deduplicated -Device messages with the same txn_id are deduplicated -Enabling an unknown default rule fails with 404 -Event size limits -Event with an invalid signature in the send_join response should not cause room join to fail -Events come down the correct room -Events whose auth_events are in the wrong room do not mess up the room state -Existing members see new members' join events -Federation key API allows unsigned requests for keys -Federation key API can act as a notary server via a GET request -Federation key API can act as a notary server via a POST request -Federation rejects inbound events where the prev_events cannot be found -Fetching eventstream a second time doesn't yield the message again -Forgetting room does not show up in v2 /sync -Full state sync includes joined rooms -GET /capabilities is present and well formed for registered user -GET /device/{deviceId} -GET /device/{deviceId} gives a 404 for unknown devices -GET /devices -GET /directory/room/:room_alias yields room ID -GET /events initially -GET /events with negative 'limit' -GET /events with non-numeric 'limit' -GET /events with non-numeric 'timeout' -GET /initialSync initially -GET /joined_rooms lists newly-created room -GET /login yields a set of flows -GET /media/r0/download can fetch the value again -GET /profile/:user_id/avatar_url publicly accessible -GET /profile/:user_id/displayname publicly accessible -GET /publicRooms includes avatar URLs -GET /publicRooms lists newly-created room -GET /publicRooms lists rooms -GET /r0/capabilities is not public -GET /register yields a set of flows -GET /rooms/:room_id/joined_members fetches my membership -GET /rooms/:room_id/messages returns a message -GET /rooms/:room_id/state fetches entire room state -GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership -GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event -GET /rooms/:room_id/state/m.room.name gets name -GET /rooms/:room_id/state/m.room.power_levels can fetch levels -GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels -GET /rooms/:room_id/state/m.room.topic gets topic -Get left notifs for other users in sync and /keys/changes when user leaves -Getting messages going forward is limited for a departed room (SPEC-216) -Getting push rules doesn't corrupt the cache SYN-390 -Getting state IDs checks the events requested belong to the room -Getting state checks the events requested belong to the room -Ghost user must register before joining room -Guest non-joined user cannot call /events on default room -Guest non-joined user cannot call /events on invited room -Guest non-joined user cannot call /events on joined room -Guest non-joined user cannot call /events on shared room -Guest non-joined users can get individual state for world_readable rooms -Guest non-joined users can get individual state for world_readable rooms after leaving -Guest non-joined users can get state for world_readable rooms -Guest non-joined users cannot room initalSync for non-world_readable rooms -Guest non-joined users cannot send messages to guest_access rooms if not joined -Guest user can set display names -Guest user cannot call /events globally -Guest user cannot upgrade other users -Guest users can accept invites to private rooms over federation -Guest users can join guest_access rooms -Guest users can send messages to guest_access rooms if joined -If a device list update goes missing, the server resyncs on the next one -If remote user leaves room we no longer receive device updates -If remote user leaves room, changes device and rejoins we see update in /keys/changes -If remote user leaves room, changes device and rejoins we see update in sync -Inbound /make_join rejects attempts to join rooms where all users have left -Inbound /v1/make_join rejects remote attempts to join local users to rooms -Inbound /v1/send_join rejects incorrectly-signed joins -Inbound /v1/send_join rejects joins from other servers -Inbound /v1/send_leave rejects leaves from other servers -Inbound federation accepts a second soft-failed event -Inbound federation accepts attempts to join v2 rooms from servers with support -Inbound federation can backfill events -Inbound federation can get public room list -Inbound federation can get state for a room -Inbound federation can get state_ids for a room -Inbound federation can query profile data -Inbound federation can query room alias directory -Inbound federation can receive events -Inbound federation can receive invites via v1 API -Inbound federation can receive invites via v2 API -Inbound federation can receive redacted events -Inbound federation can receive v1 /send_join -Inbound federation can receive v2 /send_join -Inbound federation can return events -Inbound federation can return missing events for invite visibility -Inbound federation can return missing events for world_readable visibility -Inbound federation correctly soft fails events -Inbound federation of state requires event_id as a mandatory paramater -Inbound federation of state_ids requires event_id as a mandatory paramater -Inbound federation rejects attempts to join v1 rooms from servers without v1 support -Inbound federation rejects attempts to join v2 rooms from servers lacking version support -Inbound federation rejects attempts to join v2 rooms from servers only supporting v1 -Inbound federation rejects invite rejections which include invalid JSON for room version 6 -Inbound federation rejects invites which include invalid JSON for room version 6 -Inbound federation rejects receipts from wrong remote -Inbound federation rejects remote attempts to join local users to rooms -Inbound federation rejects remote attempts to kick local users to rooms -Inbound federation rejects typing notifications from wrong remote -Inbound: send_join rejects invalid JSON for room version 6 -Invalid JSON floats -Invalid JSON integers -Invalid JSON special values -Invited user can reject invite -Invited user can reject invite over federation -Invited user can reject invite over federation for empty room -Invited user can reject invite over federation several times -Invited user can see room metadata -Inviting an AS-hosted user asks the AS server -Lazy loading parameters in the filter are strictly boolean -Left rooms appear in the leave section of full state sync -Local delete device changes appear in v2 /sync -Local device key changes appear in /keys/changes -Local device key changes appear in v2 /sync -Local device key changes get to remote servers -Local new device changes appear in v2 /sync -Local non-members don't see posted message events -Local room members can get room messages -Local room members see posted message events -Local update device changes appear in v2 /sync -Local users can peek by room alias -Local users can peek into world_readable rooms by room ID -Message history can be paginated -Message history can be paginated over federation -Name/topic keys are correct -New account data appears in incremental v2 /sync -New read receipts appear in incremental v2 /sync -New room members see their own join event -New users appear in /keys/changes -Newly banned rooms appear in the leave section of incremental sync -Newly joined room is included in an incremental sync -Newly joined room is included in an incremental sync after invite -Newly left rooms appear in the leave section of gapped sync -Newly left rooms appear in the leave section of incremental sync -Newly updated tags appear in an incremental v2 /sync -Non-numeric ports in server names are rejected -Outbound federation can backfill events -Outbound federation can query profile data -Outbound federation can query room alias directory -Outbound federation can query v1 /send_join -Outbound federation can query v2 /send_join -Outbound federation can request missing events -Outbound federation can send events -Outbound federation can send invites via v1 API -Outbound federation can send invites via v2 API -Outbound federation can send room-join requests -Outbound federation correctly handles unsupported room versions -Outbound federation passes make_join failures through to the client -Outbound federation rejects backfill containing invalid JSON for events in room version 6 -Outbound federation rejects m.room.create events with an unknown room version -Outbound federation rejects send_join responses with no m.room.create event -Outbound federation sends receipts -Outbound federation will ignore a missing event with bad JSON for room version 6 -POST /createRoom creates a room with the given version -POST /createRoom ignores attempts to set the room version via creation_content -POST /createRoom makes a private room -POST /createRoom makes a private room with invites -POST /createRoom makes a public room -POST /createRoom makes a room with a name -POST /createRoom makes a room with a topic -POST /createRoom rejects attempts to create rooms with numeric versions -POST /createRoom rejects attempts to create rooms with unknown versions -POST /createRoom with creation content -POST /join/:room_alias can join a room -POST /join/:room_alias can join a room with custom content -POST /join/:room_id can join a room -POST /join/:room_id can join a room with custom content -POST /login as non-existing user is rejected -POST /login can log in as a user -POST /login can log in as a user with just the local part of the id -POST /login returns the same device_id as that in the request -POST /login wrong password is rejected -POST /media/r0/upload can create an upload -POST /redact disallows redaction of event in different room -POST /register allows registration of usernames with '-' -POST /register allows registration of usernames with '.' -POST /register allows registration of usernames with '/' -POST /register allows registration of usernames with '3' -POST /register allows registration of usernames with '=' -POST /register allows registration of usernames with '_' -POST /register allows registration of usernames with 'q' -POST /register can create a user -POST /register downcases capitals in usernames -POST /register rejects registration of usernames with '!' -POST /register rejects registration of usernames with '"' -POST /register rejects registration of usernames with ''' -POST /register rejects registration of usernames with ':' -POST /register rejects registration of usernames with '?' -POST /register rejects registration of usernames with '@' -POST /register rejects registration of usernames with '[' -POST /register rejects registration of usernames with '\' -POST /register rejects registration of usernames with '\n' -POST /register rejects registration of usernames with ']' -POST /register rejects registration of usernames with '{' -POST /register rejects registration of usernames with '|' -POST /register rejects registration of usernames with '}' -POST /register rejects registration of usernames with '£' -POST /register rejects registration of usernames with 'é' -POST /register returns the same device_id as that in the request -POST /rooms/:room_id/ban can ban a user -POST /rooms/:room_id/invite can send an invite -POST /rooms/:room_id/join can join a room -POST /rooms/:room_id/leave can leave a room -POST /rooms/:room_id/read_markers can create read marker -POST /rooms/:room_id/receipt can create receipts -POST /rooms/:room_id/redact/:event_id as original message sender redacts message -POST /rooms/:room_id/redact/:event_id as power user redacts message -POST /rooms/:room_id/redact/:event_id as random user does not redact message -POST /rooms/:room_id/send/:event_type sends a message -POST /rooms/:room_id/state/m.room.name sets name -POST /rooms/:room_id/state/m.room.topic sets topic -POST /rooms/:room_id/upgrade can upgrade a room version -POST rejects invalid utf-8 in JSON -POSTed media can be thumbnailed -PUT /device/{deviceId} gives a 404 for unknown devices -PUT /device/{deviceId} updates device fields -PUT /directory/room/:room_alias creates alias -PUT /profile/:user_id/avatar_url sets my avatar -PUT /profile/:user_id/displayname sets my name -PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id -PUT /rooms/:room_id/send/:event_type/:txn_id sends a message -PUT /rooms/:room_id/state/m.room.power_levels can set levels -PUT /rooms/:room_id/typing/:user_id sets typing notification -PUT power_levels should not explode if the old power levels were empty -Peeked rooms only turn up in the sync for the device who peeked them -Previously left rooms don't appear in the leave section of sync -Push rules come down in an initial /sync -Read markers appear in incremental v2 /sync -Read markers appear in initial v2 /sync -Read markers can be updated -Read receipts appear in initial v2 /sync -Real non-joined user cannot call /events on default room -Real non-joined user cannot call /events on invited room -Real non-joined user cannot call /events on joined room -Real non-joined user cannot call /events on shared room -Real non-joined users can get individual state for world_readable rooms -Real non-joined users can get individual state for world_readable rooms after leaving -Real non-joined users can get state for world_readable rooms -Real non-joined users cannot room initalSync for non-world_readable rooms -Real non-joined users cannot send messages to guest_access rooms if not joined -Receipts must be m.read -Redaction of a redaction redacts the redaction reason -Regular users can add and delete aliases in the default room configuration -Regular users can add and delete aliases when m.room.aliases is restricted -Regular users cannot create room aliases within the AS namespace -Regular users cannot register within the AS namespace -Remote media can be thumbnailed -Remote room alias queries can handle Unicode -Remote room members also see posted message events -Remote room members can get room messages -Remote user can backfill in a room with version 1 -Remote user can backfill in a room with version 2 -Remote user can backfill in a room with version 3 -Remote user can backfill in a room with version 4 -Remote user can backfill in a room with version 5 -Remote user can backfill in a room with version 6 -Remote users can join room by alias -Remote users may not join unfederated rooms -Request to logout with invalid an access token is rejected -Request to logout without an access token is rejected -Room aliases can contain Unicode -Room creation reports m.room.create to myself -Room creation reports m.room.member to myself -Room members can join a room with an overridden displayname -Room members can override their displayname on a room-specific basis -Room state at a rejected message event is the same as its predecessor -Room state at a rejected state event is the same as its predecessor -Rooms a user is invited to appear in an incremental sync -Rooms a user is invited to appear in an initial sync -Rooms can be created with an initial invite list (SYN-205) -Server correctly handles incoming m.device_list_update -Server correctly handles transactions that break edu limits -Server correctly resyncs when client query keys and there is no remote cache -Server correctly resyncs when server leaves and rejoins a room -Server rejects invalid JSON in a version 6 room -Setting room topic reports m.room.topic to myself -Should not be able to take over the room by pretending there is no PL event -Should reject keys claiming to belong to a different user -State from remote users is included in the state in the initial sync -State from remote users is included in the timeline in an incremental sync -State is included in the timeline in the initial sync -Sync can be polled for updates -Sync is woken up for leaves -Syncing a new room with a large timeline limit isn't limited -Tags appear in an initial v2 /sync -Trying to get push rules with unknown rule_id fails with 404 -Typing can be explicitly stopped -Typing events appear in gapped sync -Typing events appear in incremental sync -Typing events appear in initial sync -Typing notification sent to local room members -Typing notifications also sent to remote room members -Typing notifications don't leak -Uninvited users cannot join the room -Unprivileged users can set m.room.topic if it only needs level 0 -User appears in user directory -User in private room doesn't appear in user directory -User joining then leaving public room appears and dissappears from directory -User in shared private room does appear in user directory until leave -User can create and send/receive messages in a room with version 1 -User can create and send/receive messages in a room with version 2 -User can create and send/receive messages in a room with version 3 -User can create and send/receive messages in a room with version 4 -User can create and send/receive messages in a room with version 5 -User can create and send/receive messages in a room with version 6 -User can invite local user to room with version 1 -User can invite local user to room with version 2 -User can invite local user to room with version 3 -User can invite local user to room with version 4 -User can invite local user to room with version 5 -User can invite local user to room with version 6 -User can invite remote user to room with version 1 -User can invite remote user to room with version 2 -User can invite remote user to room with version 3 -User can invite remote user to room with version 4 -User can invite remote user to room with version 5 -User can invite remote user to room with version 6 -User directory correctly update on display name change -User in dir while user still shares private rooms -User in shared private room does appear in user directory -User is offline if they set_presence=offline in their sync -User signups are forbidden from starting with '_' -Users can't delete other's aliases -Users cannot invite a user that is already in the room -Users cannot invite themselves to a room -Users cannot kick users from a room they are not in -Users cannot kick users who have already left a room -Users cannot set ban powerlevel higher than their own -Users cannot set kick powerlevel higher than their own -Users cannot set notifications powerlevel higher than their own -Users cannot set redact powerlevel higher than their own -Users receive device_list updates for their own devices -Users with sufficient power-level can delete other's aliases -Version responds 200 OK with valid structure -We can't peek into rooms with invited history_visibility -We can't peek into rooms with joined history_visibility -We can't peek into rooms with shared history_visibility -We don't send redundant membership state across incremental syncs by default -We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462) -We should see our own leave event, even if history_visibility is restricted (SYN-662) -Wildcard device messages over federation wake up /sync -Wildcard device messages wake up /sync -Wildcard device messages wake up /sync -avatar_url updates affect room member events -displayname updates affect room member events -local user can join room with version 1 -local user can join room with version 2 -local user can join room with version 3 -local user can join room with version 4 -local user can join room with version 5 -local user can join room with version 6 -m.room.history_visibility == "joined" allows/forbids appropriately for Guest users -m.room.history_visibility == "joined" allows/forbids appropriately for Real users -m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users -m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users -query for user with no keys returns empty key dict -remote user can join room with version 1 -remote user can join room with version 2 -remote user can join room with version 3 -remote user can join room with version 4 -remote user can join room with version 5 -remote user can join room with version 6 -setting 'm.room.name' respects room powerlevel -setting 'm.room.power_levels' respects room powerlevel -Federation publicRoom Name/topic keys are correct From e704bbaf1166d0082a7aac27fdbd72e37d8fd664 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 10 Mar 2025 12:30:39 -0400 Subject: [PATCH 1571/2091] update complement test results Signed-off-by: June Clementine Strawberry --- tests/test_results/complement/test_results.jsonl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 7b06510b..5fb850f1 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -534,10 +534,10 @@ {"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel"} {"Action":"pass","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_accepts_present_aliases"} {"Action":"pass","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_accepts_present_alt_aliases"} -{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_alias_pointing_to_different_local_room"} -{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_alt_alias_pointing_to_different_local_room"} -{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_invalid_aliases"} -{"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_invalid_aliases#01"} +{"Action":"pass","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_alias_pointing_to_different_local_room"} +{"Action":"pass","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_alt_alias_pointing_to_different_local_room"} +{"Action":"pass","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_invalid_aliases"} +{"Action":"pass","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_invalid_aliases#01"} {"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_missing_aliases"} {"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_rejects_missing_aliases#01"} {"Action":"fail","Test":"TestRoomCanonicalAlias/Parallel/m.room.canonical_alias_setting_rejects_deleted_aliases"} From 889fb3cf262d433bf2da461a7482a3e7400fc41f Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 10 Mar 2025 13:36:56 -0400 Subject: [PATCH 1572/2091] add download-artifact pattern for OCI images only Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd7d2484..3fd834e0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -550,6 +550,8 @@ jobs: - name: Download artifacts uses: actions/download-artifact@v4 + with: + pattern: "oci*" - name: Move OCI images into position run: | From 56dba8acb7b873c890313991630ebd23bbb47376 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 10 Mar 2025 17:15:21 -0400 Subject: [PATCH 1573/2091] misc docs updates Signed-off-by: June Clementine Strawberry --- README.md | 62 +++-- arch/conduwuit.service | 1 + book.toml | 5 +- debian/conduwuit.service | 16 +- docs/SUMMARY.md | 1 - docs/assets/conduwuit_logo.svg | 36 +++ docs/assets/gay dog anarchists.png | Bin 0 -> 11533 bytes docs/deploying/generic.md | 22 -- docs/development/hot_reload.md | 3 + docs/development/testing.md | 19 +- docs/differences.md | 379 ----------------------------- docs/introduction.md | 4 - 12 files changed, 107 insertions(+), 441 deletions(-) create mode 100644 docs/assets/conduwuit_logo.svg create mode 100644 docs/assets/gay dog anarchists.png delete mode 100644 docs/differences.md diff --git a/README.md b/README.md index 13a1c67f..d8f99d45 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,16 @@ # conduwuit -[![conduwuit main room](https://img.shields.io/matrix/conduwuit%3Apuppygock.gay?server_fqdn=matrix.transfem.dev&style=flat&logo=matrix&logoColor=%23f5b3ff&label=%23conduwuit%3Apuppygock.gay&color=%23f652ff)](https://matrix.to/#/#conduwuit:puppygock.gay) [![conduwuit space](https://img.shields.io/matrix/conduwuit-space%3Apuppygock.gay?server_fqdn=matrix.transfem.dev&style=flat&logo=matrix&logoColor=%23f5b3ff&label=%23conduwuit-space%3Apuppygock.gay&color=%23f652ff)](https://matrix.to/#/#conduwuit-space:puppygock.gay) [![CI and Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml) +[![conduwuit main room](https://img.shields.io/matrix/conduwuit%3Apuppygock.gay?server_fqdn=matrix.transfem.dev&style=flat&logo=matrix&logoColor=%23f5b3ff&label=%23conduwuit%3Apuppygock.gay&color=%23f652ff)](https://matrix.to/#/#conduwuit:puppygock.gay) [![conduwuit space](https://img.shields.io/matrix/conduwuit-space%3Apuppygock.gay?server_fqdn=matrix.transfem.dev&style=flat&logo=matrix&logoColor=%23f5b3ff&label=%23conduwuit-space%3Apuppygock.gay&color=%23f652ff)](https://matrix.to/#/#conduwuit-space:puppygock.gay) + +[![CI and Artifacts](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml) + +![GitHub Repo stars](https://img.shields.io/github/stars/girlbossceo/conduwuit?style=flat&color=%23fcba03&link=https%3A%2F%2Fgithub.com%2Fgirlbossceo%2Fconduwuit) ![GitHub commit activity](https://img.shields.io/github/commit-activity/m/girlbossceo/conduwuit?style=flat&color=%2303fcb1&link=https%3A%2F%2Fgithub.com%2Fgirlbossceo%2Fconduwuit%2Fpulse%2Fmonthly) ![GitHub Created At](https://img.shields.io/github/created-at/girlbossceo/conduwuit) ![GitHub Sponsors](https://img.shields.io/github/sponsors/girlbossceo?color=%23fc03ba&link=https%3A%2F%2Fgithub.com%2Fsponsors%2Fgirlbossceo) ![GitHub License](https://img.shields.io/github/license/girlbossceo/conduwuit) + + + +![Docker Image Size (tag)](https://img.shields.io/docker/image-size/girlbossceo/conduwuit/latest?label=image%20size%20(latest)&link=https%3A%2F%2Fhub.docker.com%2Frepository%2Fdocker%2Fgirlbossceo%2Fconduwuit%2Ftags%3Fname%3Dlatest) ![Docker Image Size (tag)](https://img.shields.io/docker/image-size/girlbossceo/conduwuit/main?label=image%20size%20(main)&link=https%3A%2F%2Fhub.docker.com%2Frepository%2Fdocker%2Fgirlbossceo%2Fconduwuit%2Ftags%3Fname%3Dmain) + + @@ -53,6 +63,19 @@ A lot of critical stability and performance issues have been fixed, and a lot of necessary groundwork has finished; making this project way better than it was back in the start at ~early 2024. +#### Where is the differences page? + +conduwuit historically had a "differences" page that listed each and every single +different thing about conduwuit from Conduit, as a way to promote and advertise +conduwuit by showing significant amounts of work done. While this was feasible to +maintain back when the project was new in early-2024, this became impossible +very quickly and has unfortunately became heavily outdated, missing tons of things, etc. + +It's difficult to list out what we do differently, what are our notable features, etc +when there's so many things and features and bug fixes and performance optimisations, +the list goes on. We simply recommend folks to just try out conduwuit, or ask us +what features you are looking for and if they're implemented in conduwuit. + #### How is conduwuit funded? Is conduwuit sustainable? conduwuit has no external funding. This is made possible purely in my freetime with @@ -64,17 +87,15 @@ and we have no plans in stopping or slowing down any time soon! #### Can I migrate or switch from Conduit? -conduwuit is a complete drop-in replacement for Conduit. As long as you are using RocksDB, -the only "migration" you need to do is replace the binary or container image. There -is no harm or additional steps required for using conduwuit. See the -[Migrating from Conduit](https://conduwuit.puppyirl.gay/deploying/generic.html#migrating-from-conduit) section -on the generic deploying guide. +conduwuit had drop-in migration/replacement support for Conduit for about 12 months before +bugs somewhere along the line broke it. Maintaining this has been difficult and +the majority of Conduit users have already migrated, additionally debugging Conduit +is not one of our interests, and so Conduit migration no longer works. We also +feel that 12 months has been plenty of time for people to seamlessly migrate. -Note that as of conduwuit version 0.5.0, backwards compatibility with Conduit is -no longer supported. We only support migrating *from* Conduit, not back to -Conduit like before. If you are truly finding yourself wanting to migrate back -to Conduit, we would appreciate all your feedback and if we can assist with -any issues or concerns. +If you are a Conduit user looking to migrate, you will have to wipe and reset +your database. We may fix seamless migration support at some point, but it's not an interest +from us. #### Can I migrate from Synapse or Dendrite? @@ -98,9 +119,10 @@ is the official project Matrix room. You can get support here, ask questions or concerns, get assistance setting up conduwuit, etc. This room should stay relevant and focused on conduwuit. An offtopic general -chatter room can be found there as well. +chatter room can be found in the room topic there as well. + +Please keep the issue trackers focused on *actual* bug reports and enhancement requests. -Please keep the issue trackers focused on bug reports and enhancement requests. General support is extremely difficult to be offered over an issue tracker, and simple questions should be asked directly in an interactive platform like our Matrix room above as they can turn into a relevant discussion and/or may not be @@ -108,24 +130,34 @@ simple to answer. If you're not sure, just ask in the Matrix room. If you have a bug or feature to request: [Open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new) +If you need to contact the primary maintainer, my contact methods are on my website: https://girlboss.ceo + #### Donate conduwuit development is purely made possible by myself and contributors. I do not get paid to work on this, and I work on it in my free time. Donations are heavily appreciated! 💜🥺 -- Liberapay (preferred): -- GitHub Sponsors (preferred): +- Liberapay: +- GitHub Sponsors: - Ko-fi: I do not and will not accept cryptocurrency donations, including things related. +Note that donations will NOT guarantee you or give you any kind of tangible product, +feature prioritisation, etc. By donating, you are agreeing that conduwuit is NOT +going to provide you any goods or services as part of your donation, and this +donation is purely a generous donation. We will not provide things like paid +personal/direct support, feature request priority, merchandise, etc. + #### Logo Original repo and Matrix room picture was from bran (<3). Current banner image and logo is directly from [this cohost post](https://web.archive.org/web/20241126004041/https://cohost.org/RatBaby/post/1028290-finally-a-flag-for). +An SVG logo made by [@nktnet1](https://github.com/nktnet1) is available here: + #### Is it conduwuit or Conduwuit? Both, but I prefer conduwuit. diff --git a/arch/conduwuit.service b/arch/conduwuit.service index fa3616d8..4f45ddc0 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -4,6 +4,7 @@ Wants=network-online.target After=network-online.target Documentation=https://conduwuit.puppyirl.gay/ RequiresMountsFor=/var/lib/private/conduwuit +Alias=matrix-conduwuit.service [Service] DynamicUser=yes diff --git a/book.toml b/book.toml index 1d32c766..7eb1983b 100644 --- a/book.toml +++ b/book.toml @@ -13,12 +13,15 @@ create-missing = true extra-watch-dirs = ["debian", "docs"] [rust] -edition = "2021" +edition = "2024" [output.html] git-repository-url = "https://github.com/girlbossceo/conduwuit" edit-url-template = "https://github.com/girlbossceo/conduwuit/edit/main/{path}" git-repository-icon = "fa-github-square" +[output.html.redirect] +"/differences.html" = "https://conduwuit.puppyirl.gay/#where-is-the-differences-page" + [output.html.search] limit-results = 15 diff --git a/debian/conduwuit.service b/debian/conduwuit.service index 4d6f4eef..a079499e 100644 --- a/debian/conduwuit.service +++ b/debian/conduwuit.service @@ -2,26 +2,14 @@ Description=conduwuit Matrix homeserver Wants=network-online.target After=network-online.target +Alias=matrix-conduwuit.service Documentation=https://conduwuit.puppyirl.gay/ [Service] DynamicUser=yes User=conduwuit Group=conduwuit -Type=notify-reload -ReloadSignal=SIGUSR1 - -TTYPath=/dev/tty25 -DeviceAllow=char-tty -StandardInput=tty-force -StandardOutput=tty -StandardError=journal+console -TTYReset=yes -# uncomment to allow buffer to be cleared every restart -TTYVTDisallocate=no - -TTYColumns=120 -TTYRows=40 +Type=notify Environment="CONDUWUIT_CONFIG=/etc/conduwuit/conduwuit.toml" diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 8e07adc2..ad0f8135 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -1,7 +1,6 @@ # Summary - [Introduction](introduction.md) -- [Differences from upstream Conduit](differences.md) - [Configuration](configuration.md) - [Examples](configuration/examples.md) - [Deploying](deploying.md) diff --git a/docs/assets/conduwuit_logo.svg b/docs/assets/conduwuit_logo.svg new file mode 100644 index 00000000..9be5b453 --- /dev/null +++ b/docs/assets/conduwuit_logo.svg @@ -0,0 +1,36 @@ + + + + + + diff --git a/docs/assets/gay dog anarchists.png b/docs/assets/gay dog anarchists.png new file mode 100644 index 0000000000000000000000000000000000000000..871cf302a8f27d6ed02c983241c7c5ca6029ce99 GIT binary patch literal 11533 zcmd^l`9IX(`}fP1$`YwavWDy=`&QWrW#7i2ELoBoM$9m6mK0^*DTYe6gc=O8y~~!7 zEg>WF%962;{XXOK{@nNd{TseNJs;;>*LAMtoa=S2nRCTlF*js8%zGGuAU0zoJxd5W z&+`LHQqE{g$ zsAMIhX*sTTt?=_k=lJ&2!rtB+im94gKC-TBk0rx*^;sl%-pN zcQ9Dowdtoc#jYa_}hWPfyC&r%KM(}?T&jK9%m+P$?GL3a90I(p2*^u#~RjDwq+Touy; z;fx$mt`-BXf1@X&&klkNw|XLS_vEl_(5-?k>Siv);EEu&rvk7EZ=-?!?o&Cs1C2hc zlcqSj@}nM@L-LTV8y}LM6?c4yM!K+ENCu>@A7JWHSLnPylE%SXtwUtwfx1G^(%@;R zDh$8(W%@C#>aYQn9a3 z(ik6o+jmshXWx6eRhpQ5tr|^>eC6GJDE_z)`5XI?skYCOcy}LSKkmbW4H{e-JxWve z$}gR!&brE)rcTmv-i72e~Kn{>zy8r(L9OfVWY7nZ4K3n`%)Gu zPiS*o9yGqiGyAdDytp3_Lti198-_k2`*1^aAH2nB!0aor4?a@+a6@Jvy!ZW%_fepY z;=L7VaKuMxAG}p)5LUIn!&=|tXj%&f_I-P?L-S_NUYBO1!2l;MYM&hUV^N+%BW+(Z zqnS)~L+pd6KMg{v+i0}6=fJv+(IG52F2?GH%j<{8tYk5vsTHt4_|L&8%MbJp>{PUJOM zgT9fjWw3Nbi|*%DvaiWzDYSTyq9lVwLLtwn-^HsEJSz`xF5&CO^I3GwK^5EH|C9A*TO1F&;W4 zLi8~nH^!N9B3BFkiZ(_@mt~O9T&{{Ob_jUuP&p^LxO(L@(-0%Y2DzC-?}wvi=x&zM zqbx98B3LAyx;Sw%h!2s^g_iSq79LMn(jiOQ$YEOi3X2K?qJ{hw->CE-t!E6#rNGf=bKaJJR~8JT9{D z4OoNyD-)%vZbgfE21hk98V|ztb0JT{TkPex$x@=&-*mQSn5lxlT8)om$_iFSneAmL z$;re$NJ^AQA{~dyO~_Jm2nI;%4AJpfDLt;A6X|a>NZ0r!y3Dpfiq6%h#ZH4%1(gev zi$!4{$exYTpMuhY-ae%Im=S~_+l!bgaRe76B}jBkDm{#wjww^d&(h}wl8bHNFOYWz z)yIyY4DpH)9aBo#ah@?{)%aP4ysPBmAowtycLLQ1NicwT&kzR$5u#8RCvx^#DHpCS zx~$5`?(hYxmch^hU#2`EvacDbpZ+T=rAihKrSpDBb<9MyLgz1HN-PMT5U(I{;4DH3 z3MBhxlY$sH#E1iu2sH?t#ClQ{0|!`K8leR#kbOrny97!4n0 zNP9&6gCw*;o`S?`L4*g?A5#Wme)56`*;k5`eBeql^?MwugwFa5vHA?cAL@@T^EVzm zbwQNu>q|P$eB}}Kdk(6L&KgLcMFc_OT*#56QVkqeOj(JE9e<;7%V03x_sCC1vTp~e zpLy{**>{Ar%DkvY_MIgiKe%{?9E)P3*Str)grfty{vowg7bg-UM>Mq)ZoJyUt%LXF z{CSWZ8-Xw81i2pj0;NUIXN0+GVt1zTVhi^*ID=6YM|HO&NYe3(64Me&O>u_Na)o#) zuAdjlu~<^op+$(&8V>(q1gTDWhQiX%2@yl=376=^V&wP}ORwS%kz?EO#m8KusALXU zor#AN$tQ+*4KYz#Uy!N}%L@`!oe7R~Ak$S12rhKt97sNKLVWei#}*mc z$>6ADV^R={Uk;UggOtp2eqpXPHubkoeJF*jKNW$*P^^yg;Wk;9oNTHBIo8Vhbx@v&mt25&JHl!VktKhW3z=2u(#vpK zwKn{YcgW5y^wyGb?ir%0%@mn=U~X#uIlMYL8(Y&Jo&ImQoNGTTbDZc;h|_EF2B^0&E=o!3eO4$Qsb zLJGf4sKiF`yYN>!PtT7{YPjrBkt`lv zZs<{3F`iVH#w2cW>CrU5_lDW^?L!@(dlzSm=50d4G&aiW!=#95R`UT&m({PA%>i#? z1!Aklowe(}%x89({AC;~n1!jKOARVoTH6u30fgq>s>kC8hh|$JR7RoKz4#Y)ml||L zC=(9t%Z(8x0)=sL>VcQTlve|Yd(UnLuDm4;((x~TU7TH5+ibe`ZMY*gDkb#PL#@x& z16jM)l=*Mcly+2n{xu~c>tbGJeelm`7zx9)mrO;sfA1=*wTs_tU_Ylcl;YZnw;cY&pJ-vQyEZ*DIW_KbxolSc zX-L7ShNTyEbu@Og_DoZnaXsPDA+Nq8N6G>U|2NqH7gbwe6-V2?@eX}ZutG^HsS5o zdyjQvKy3*z-KdLIX+PvC#N6NI*yK0ZWnO2UTk;)2_Ld<(Xgkt4t$p}zmMi~*t0SGF zod)yLCtj&xseKD#H@2Gi&u!l6(s4aKo+k$f8wPlmxmC~j)ji(4nKGFZ9RK>ZSbKQB z+Mo&x=GX8H`iZgLAIH4t;T+1>nNervg*Yz1oV>YDU6*PKw!fZhx-4utcvQc0;EBbV z+zN-d!N<~lse`Y!uLi0#Z=A9+SZ_ZgY$IdU>i6a4i*vj3rkq7jZ3ShO2OW3Q?(T&V zCVP|dU()77G(r~@mh-%`T&u?X`cr%RUVY>iWKR-l&>@|UFO{}y_iLS@8=6ZD&1YQZ z@iS55AUH238a8-knOf+xoH#ck?>JZ8sQbFtBo18}@pH3IMlt*av+H+XbTUJ^zBtF<vslBt7b8Uqt58`lCnD-`-(l`YPUC zl_3n3I7|F=-n>`2J@fv9MnqvP|ESS1#ZgJC>%FZv^FMdA<}ROq%acuvbH23E(H6*O zee|`by3I`qmku8z?#-|jVe9UdS9}aIM4;3c(5xyRid1V?Qt!HnY6TU1NoPOKP~K>?#BLH641Q97+$$YH&oeMhLJl^Yu70#cI1zoYzv>T4I@hi@mN6C z;cNqY)+S4qh3}S{VUPJMY*?APge-O6<-sNu&s7znQ*2F^+`UQeZ0;!ADryi#Xmn%d z5}+FQ=D4>vaof3_Y_B{=8rLf;NYmYxq7mP}gpCv~uAB=x-Wp~jJ;|e$&MH9Ck>-^1 zER6Ki?|VxA9kdAd>Fcu9GPOQCU#C?1;lz=qSAtHxsC;kprqzhnXb}yMfrk?!<-d4a z!dmMo98V5^5;c=^9QLp6d7aN<)xrIS##U5#x>fZ&tz|?_jWcG@f>c;X;sR9 z-OzSRbq4Qh4dyq$cHhh=yZx9R6gtr!);jo}HNsi-dCp;ysF{H5_ZvygpF6w$dW83X z6PWb07ajBY4AUr)@s6KN#T^~d-00qJ>_?S4rCd^dJbQsuhU~HB;0eWpvA)Bbmo_I7 zQd*H;*A}NRX6xGO&q}=sS2XEO<3l6!n@V&TPnB-@{=8l?A<5psp@ty5gg-6XBv1@x znr~FD3>3YeI1-z=(wY?Hep0x|)b2cj$DFN|S*!JyJXXQM>*&Lzc&QBjW6z)GzUI}Y4zD@MMBU>I(`~$P^(;JQ z$V@tJcI5H=J2+MXeP5}3Mb=x@WAGC0c+anWS6DLm>wJnW*i#QLT5S*is-IxOhY6fE z;y$mX8c=LO{NQfKb**<|t~9$dOB(TYC9h)soyz1`mkpOn;oD<$ZJ$X@W=A6ZCBlDg zRNj91Sb&jd#~8?Z&(%e_+vw;Vl;2F0^$wMvOQfKKHBZ%lOZfXQ2hZB)m8u_Hu3Z0P zTwCvM$k3t7#FLC?$zW$qEoqNms`nBZdU!y2CfPzf!dOLTKo0R0l~=_O%Q%!Iw$QnF z|6e!Q&vUz$l@9f`WS?}oLRjy;hq~BzZ<@aIDj%0)+7GNoT^{OhJ_G!?kfDo-=P9r4 zm$H9@!_3GNMY&}9$HeaZX%1ID&kuy@`2Vt&)UT)7vWF7OF zd+`P4@_#lvRIpL?pYkhwEln^j)R9=&f79LDwxV8WbaUn+p#W~W<@xH-;Ono269<-i z-~q!U;AkCj40|E!&s#cCe(*W%5FUwq`Z}g2_-nz$RpJRD%m0XlS#Sl+=jp2q+`8gG zD1@6<^Aze$k*p<>-S*~aZNjtC8=+U;%F5n`-rrDtK-C_l&gc?e!JlrOd^w3px9k+$ zO}U9#5kmB%Uf8z=CK7oaQ3`LLAEA9?NH!oRx<2d5i?N~=GA|ayp=|o1CfqE8?^pPK zu=~m?oLtb6q0K6AU+q~AqWRIA0YOV+!dqB3(Bzq#UfW~3w8WX$XApxZck07tKH+Q4 zYSYDKaJ8WvDacIxW|z=AqyiMSr#ZSdqfaP@TRPmL2u4V@rxH0hhffz};7^6z&1!mK z=@Q3^j1W6H3!x|x#PG_?G~GILB%6%dsa=8sl%IpT(HBdWj~j%GUx>B}@ws>yYr&+# zQ?1wHQo+>e)__w1^!YDQ{^0wg*6y+C<84UkYZLR7G#OMLRh9alH0Zdbt9EbZ5YKUZ zY-aee4pmL2$y#t3)r&@3CzamZyjr*T`H~({!cNcyYmC+;B*+G+pTo#4_m4Vk-uAv2 z82PQ=E9%x2D?%M?q~`ETl^xUp(l3fiQP{r3rNQbTKIhbpRYSsfZ#wE;U*he9(^odT zY9*FCB#722rMC!{(t3637D#pnl_~nT;v=-$fFx>e_vKNv7-x-t7P1<5)rrssciio* zc&qp0U10c*5B>A+Bz<#a_m^uAvd&f^CbvWXvP!%$wNuN!>kusC3)lE$A?*bE>f_k) z9a&>_5+f-LYSlNy8+J`<)WPq`JE4l|>N9`DuooGhtVImdwP}(bnWGI7qtQqG!erYv zJm{D!jqKESJ|?@41P4nr*c%WA;GSWPL)9K~ZSf7#7p}{~?nSzj%+#ZPy4}HrN2a(d zjZ-zWcy&Idx@J!$8>vTUe&P<3CIZ1&tpCAY?dg|LW!q# zAs73~1&3D{$$KWq25$zh8DRXo7bsG>CHmJ-z>Y<}eO^Mu%g7UoBPWBM%zC6r$>u$U zDFLca!&yq(Yj%22Mfs-irJ`$N%+&~u_1!X5alRp}UWDY@m@<>hthlqDJ!guA&lNdw zZFo?tgeJEFB!pM-#aS-%(}UR)gN09|8ZWWGG38dv{*H(Uxk0T`m}FeOecZ=%P!$V) zPsFZ>R(uG&k2R{%2u+j?+fDePC@$@ZP1(p(ae=jf>k7~$Q zgz1Y6C;i_p`Boa}c!2WOhCqbBQhTGezn3pL>qXHf>foLbIxz9Ba%r$L71f?E1uqrq z)GWnD)SvB`SVwi_i^G2kHRPtab*E0r`X2x}AJ~`Ko79`U{k`tC$%Yl-ckgrX8OZEq zm)~kQE6!PO|0tAHW#(mIp_$dt6e3eE#M573nn}#6mksUr*bMc)s8-^qqPRbobY$*t z)_6z#ZLeBTMkbW{5tgc*nc1lU5LRWW*f{ zODnl>{gr*R_eo~jM0lOd0oS;ndGgMAo7$u6GkSzKuwexwB?x@YE(h{OaXVS+gDHc% zKR$J=Onpsr$K3=)AozHzxUii{QY_#(#cvWoo6$E@dF`pY2lpQZy5S!t0%v6=NaY!jkcg0rT$7t|;J&)|f2XODmNKSL~(Rh3_%#Rr| zsXcsfs8zbMS@o45q7U^l?II?V$eVGV4D7OBhzu8e=HvFbnY?Dqn27w^Npq#W zj0G%85YdZrqaK{or(N8Eg4q4?LBH>TJDVYxNrUp=fMVjo>_w+{EKhbAd2B#&!q3W` z>d@R2-YlY6pOl{XpBbT?BDu&&HVePNW6a{Fpbp}x#;3xuFWl&#yxA|5#2FnP{uBP+ zQT+62{ht3jqDvv3a1?8NB?-!`3%EcYIs+T{ej!8ipn}GiROY%9uHTYmz18J4qbJXS ztJ_c2)`C`lfGJRaJo}G{baZWQ-8B28GHB&c>%=F_;HT`9V`)AAt#vY3%I!>}nt9)s z*R`?8yp{rv=Y51U1XKb_Gltx?>lEmp(5gvxR))EtP4|UErPqf7P-_2*JrI0u zq>t`k2n^#!xlDL=WhBKS`x081xAMI$@DGkp5NJaKV@DTPHI@v}G z%BF{FoBAm_^Td2!zQeX$dS)y1CetNA~+gNMg6(slV{U#=f~ zBALKsrV`n~F*<{1AV_bo-*o1oyV%&D_D;eJbjziK;x=hWxp%B`Bq z0&v4&?7HVi(Vn8tb@}A9%ZfOo|L>ZFU=9s&jFOA{b z)w&;+9#xwr^Rg(;S*tH`;*~h&&)o{!pX)2!(tkG={|0h9wOwT;bqQJ;zR7|+7C%~r zG|b&9Y5cviUd+_0-ZHfKAw(BlUV2X(s)eDlS7MsYourbd8VqRaAt`oaQ?d? za{E8itrVit!q)#TS{o=v>l`yAiMyyd&e2t#Px47A^VqcR6l#l;mMs0m+micOXlSWD zFiFl^QMNDxZ<>Dd__bW_7qMr<|2RFMTB$KQZT@Z@d3m0-@s-zvWg)VLjkT#X=*x{M z4!;R~<<`UO4DynW)d{eD#jf0WMyUg?ND{I%GD)|*R$rVm)(RtYv7k_tC7pO&=e>w- zIdJ(<`&VRzam#J`-lKXxj^)o_=fEs$_k+QcgJ z>P$(Bs+Lucu{ReCo&5F?`6W5Ja<7r+cj_TMpUjUBhttSL?5&SZYxP>LR*$(K zAi8!ulLNyY?imGD+RA5Il`_^5?El;e_|mhJvFoDrM)RuA7~`qmsfq=!0@~<3W4lvl zpE2Y!yT(ac+RKq~%fj%kFBQE9j=d4no) z`Omp{@Q33hoS)9+mxQzx)b^}9RbaXl^5IY%Bz1)g!9V0NmFcU+gL4brH>ymE3b8&g?q;4wnvs?hpi=@czDZRdwEOZsf~cL-@`|g`TroWMiDP;-G!S6otG%`JuIv&iw)vs$XEXvMY3jY1OWqd5cWG|IeX;Q*yA{XrF zFp9s~BX)^o+89?}DKIxSTVyyiGFr84SBOe?Xbq=lnCw4pXO|iz@UZ^-2DB0M#A4D4 zAFel)xIp?f(~-bCZCizJYzXFhI~_M1(`B$UtJ?6yVr?fdRW{D0n{wM7o~Y3fZb-AM zkE6$rGS(7aDILtTpBHP0TgCWDwAXiBi*!@ro;+i7a(z*qf4_OsA4&T@82X1rv$QCDxudLT^PSFv-L=iGjgCJCzzI@QB0GB?6jTZg%`JK8!m)xA)Qh(t=~ zyO-Us*xO#;%go9r)MCHukx`g>^lqE+n$DrH3>~cU}vqoIEi?e3APveT4R>NK}xw5KZ-G5k_4@4uL6OnFGhcS81iPSy z=k@OtHx~z+?!+{NKX+*<(BMCPSCzPnZWw=M`*6pz&BwDLVtc+(nx z4JT(^vJY}0SvFQE9l41LYrh4M+B|nTQ^C)8avpGGyW^15Xp3D-kwq>sd}Cs=$RZ)u zwqT1Np{hzGZcwR|iZE=o%?T>)f5*l^O3>f+X6{2t59Gq0;vx#d1d7oZ40d^9hRrYn zbDm0nfHWGP9Oe$DI*`l(Y+Kuzt2Y~s2zwPiMJv5$aT=9=Kyp+k<|wi;6t8ji zBALa$5_Lq>07EyMvhea@w4Cca37d3~!t@qNNIVLJ`3u4%Vu9@fq|u~Dz;$dZfny@zjONPMWwW+7Y~I<7AB19l>K>BZ$0Aa*Uj- z6lo{r4RGZ&9A~OeW*JIA_279aOn#<>SxX>ZEjEgN4a7$h7wW*HHcrN5-Njsti1Rl= zBh5i1OH9_0<{)D8CTLf48r^i=HRm$mPg(e+`5eg0jz--*N@04o&;nky={a1%?n4Xs zS_Q+SNN60trC`X3gmmy4=L|WJ(4NWqb#^f#6iGs)GsMW@ED3UVI4Df=DGQx^7cg{l zChMNySsN-Ws8x&y?0ux2bY{SYC7sS-j+VpikhC)DTeur_Nr((BfXyUqhB2Atrvhx= z#YkaV{D3O+jH1%3>Jgk2g^3V*;=U2*|HVr!nB@HTL1dsFA=LtAbjpC%`_?8)rZp z<25$KXk&cX#sPr5q*2VB7VZnD;c%h?nI$g;WjKC}!gR%gP!s^R`<29Kzltt~?pyN0 z>?t84p#yD- zq3e_y{jC9FHJ!4sf&igjHeKIj2iipoYO!b_evY(5R0QG@aNPKD3R8I&s^>4zZf{AL zMFZ{9Qlsk{KzmBc!ZreE?=W58We3_HqSldGU^kyH>`93eAxqMsavw5_pm;W&6xg=} zl0>;97)oU`fDRUjL^1{)EG&X@*-W%ucYJ340CccG{zku>zJRrC3%J~DKp-?o>7av! zB~KK#1|2L+UsF)Bpo4|p+f4l==wP9HFP_Z?QXXNeuuQvP`8xPEfqd6< z0NWIOm_ zdNLr}sb2#fEKq2QdHOI-8h^)A43N=qLHZ*=MymUQY=^1^vz7aSG>EyG2QW#B6k`RX zhvHDvpo0afK9jB952XE#f18T}vM=eg6_6gvL9KyghpNwJYl99JXe6mSmkX>?Vj}+4 z7z8aCuQ - -See the `[global.well_known]` config section, or configure your web server -appropriately to send the delegation responses. - ## Adding a conduwuit user While conduwuit can run as any user it is better to use dedicated users for diff --git a/docs/development/hot_reload.md b/docs/development/hot_reload.md index 018eb4b3..65fd4adf 100644 --- a/docs/development/hot_reload.md +++ b/docs/development/hot_reload.md @@ -1,5 +1,8 @@ # Hot Reloading ("Live" Development) +Note that hot reloading has not been refactored in quite a while and is not +guaranteed to work at this time. + ### Summary When developing in debug-builds with the nightly toolchain, conduwuit is modular diff --git a/docs/development/testing.md b/docs/development/testing.md index 2d421767..a577698a 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -5,12 +5,11 @@ Have a look at [Complement's repository][complement] for an explanation of what it is. -To test against Complement, with Nix (or [Lix](https://lix.systems) and direnv -installed and set up, you can: +To test against Complement, with Nix (or [Lix](https://lix.systems) and +[direnv installed and set up][direnv] (run `direnv allow` after setting up the hook), you can: -* Run `./bin/complement "$COMPLEMENT_SRC" ./path/to/logs.jsonl -./path/to/results.jsonl` to build a Complement image, run the tests, and output -the logs and results to the specified paths. This will also output the OCI image +* Run `./bin/complement "$COMPLEMENT_SRC"` to build a Complement image, run +the tests, and output the logs and results to the specified paths. This will also output the OCI image at `result` * Run `nix build .#complement` from the root of the repository to just build a Complement OCI image outputted to `result` (it's a `.tar.gz` file) @@ -18,5 +17,15 @@ Complement OCI image outputted to `result` (it's a `.tar.gz` file) output from the commit/revision you want to test (e.g. from main) [here][ci-workflows] +If you want to use your own prebuilt OCI image (such as from our CI) without needing +Nix installed, put the image at `complement_oci_image.tar.gz` in the root of the repo +and run the script. + +If you're on macOS and need to build an image, run `nix build .#linux-complement`. + +We have a Complement fork as some tests have needed to be fixed. This can be found +at: + [ci-workflows]: https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml?query=event%3Apush+is%3Asuccess+actor%3Agirlbossceo [complement]: https://github.com/matrix-org/complement +[direnv]: https://direnv.net/docs/hook.html diff --git a/docs/differences.md b/docs/differences.md deleted file mode 100644 index 18ea7a1f..00000000 --- a/docs/differences.md +++ /dev/null @@ -1,379 +0,0 @@ -#### **Note: This list may not up to date. There are rapidly more and more -improvements, fixes, changes, etc being made that it is becoming more difficult -to maintain this list. I recommend that you give conduwuit a try and see the -differences for yourself. If you have any concerns, feel free to join the -conduwuit Matrix room and ask any pre-usage questions.** - -### list of features, bug fixes, etc that conduwuit does that Conduit does not - -Outgoing typing indicators, outgoing read receipts, **and** outgoing presence! - -## Performance - -- Concurrency support for individual homeserver key fetching for faster remote -room joins and room joins that will error less frequently -- Send `Cache-Control` response header with `immutable` and 1 year cache length -for all media requests (download and thumbnail) to instruct clients to cache -media, and reduce server load from media requests that could be otherwise cached -- Add feature flags and config options to enable/build with zstd, brotli, and/or -gzip HTTP body compression (response and request) -- Eliminate all usage of the thread-blocking `getaddrinfo(3)` call upon DNS -queries, significantly improving federation latency/ping and cache DNS results -(NXDOMAINs, successful queries, etc) using hickory-dns / hickory-resolver -- Enable HTTP/2 support on all requests -- Vastly improve RocksDB default settings to use new features that help with -performance significantly, uses settings tailored to SSDs, various ways to tweak -RocksDB, and a conduwuit setting to tell RocksDB to use settings that are -tailored to HDDs or slow spinning rust storage or buggy filesystems. -- Implement database flush and cleanup conduwuit operations when using RocksDB -- Implement RocksDB write buffer corking and coalescing in database write-heavy -areas -- Perform connection pooling and keepalives where necessary to significantly -improve federation performance and latency -- Various config options to tweak connection pooling, request timeouts, -connection timeouts, DNS timeouts and settings, etc with good defaults which -also help huge with performance via reusing connections and retrying where -needed -- Properly get and use the amount of parallelism / tokio workers -- Implement building conduwuit with jemalloc (which extends to the RocksDB -jemalloc feature for maximum gains) or hardened_malloc light variant, and -io_uring support, and produce CI builds with jemalloc and io_uring by default -for performance (Nix doesn't seem to build -[hardened_malloc-rs](https://github.com/girlbossceo/hardened_malloc-rs) -properly) -- Add support for caching DNS results with hickory-dns / hickory-resolver in -conduwuit (not a replacement for a proper resolver cache, but still far better -than nothing), also properly falls back on TCP for UDP errors or if a SRV -response is too large -- Add config option for using DNS over TCP, and config option for controlling -A/AAAA record lookup strategy (e.g. don't query AAAA records if you don't have -IPv6 connectivity) -- Overall significant database, Client-Server, and federation performance and -latency improvements (check out the ping room leaderboards if you don't believe -me :>) -- Add config options for RocksDB compression and bottommost compression, -including choosing the algorithm and compression level -- Use [loole](https://github.com/mahdi-shojaee/loole) MPSC channels instead of -tokio MPSC channels for huge performance boosts in sending channels (mainly -relevant for federation) and presence channels -- Use `tracing`/`log`'s `release_max_level_info` feature to improve performance, -build speeds, binary size, and CPU usage in release builds by avoid compiling -debug/trace log level macros that users will generally never use (can be -disabled with a build-time feature flag) -- Remove some unnecessary checks on EDU handling for incoming transactions, -effectively speeding them up -- Simplify, dedupe, etc huge chunks of the codebase, including some that were -unnecessary overhead, binary bloats, or preventing compiler/linker optimisations -- Implement zero-copy RocksDB database accessors, substantially improving -performance caused by unnecessary memory allocations - -## General Fixes/Features - -- Add legacy Element client hack fixing password changes and deactivations on -legacy Element Android/iOS due to usage of an unspecced `user` field for UIAA -- Raise and improve all the various request timeouts making some things like -room joins and client bugs error less or none at all than they should, and make -them all user configurable -- Add missing `reason` field to user ban events (`/ban`) -- Safer and cleaner shutdowns across incoming/outgoing requests (graceful -shutdown) and the database -- Stop sending `make_join` requests on room joins if 15 servers respond with -`M_UNSUPPORTED_ROOM_VERSION` or `M_INVALID_ROOM_VERSION` -- Stop sending `make_join` requests if 50 servers cannot provide `make_join` for -us -- Respect *most* client parameters for `/media/` requests (`allow_redirect` -still needs work) -- Return joined member count of rooms for push rules/conditions instead of a -hardcoded value of 10 -- Make `CONDUIT_CONFIG` optional, relevant for container users that configure -only by environment variables and no longer need to set `CONDUIT_CONFIG` to an -empty string. -- Allow HEAD and PATCH (MSC4138) HTTP requests in CORS for clients (despite not -being explicity mentioned in Matrix spec, HTTP spec says all HEAD requests need -to behave the same as GET requests, Synapse supports HEAD requests) -- Fix using conduwuit with flake-compat on NixOS -- Resolve and remove some "features" from upstream that result in concurrency -hazards, exponential backoff issues, or arbitrary performance limiters -- Find more servers for outbound federation `/hierarchy` requests instead of -just the room ID server name -- Support for suggesting servers to join through at -`/_matrix/client/v3/directory/room/{roomAlias}` -- Support for suggesting servers to join through us at -`/_matrix/federation/v1/query/directory` -- Misc edge-case search fixes (e.g. potentially missing some events) -- Misc `/sync` fixes (e.g. returning unnecessary data or incorrect/invalid -responses) -- Add `replaces_state` and `prev_sender` in `unsigned` for state event changes -which primarily makes Element's "See history" button on a state event functional -- Fix Conduit not allowing incoming federation requests for various world -readable rooms -- Fix Conduit not respecting the client-requested file name on media requests -- Prevent sending junk / non-membership events to `/send_join` and `/send_leave` -endpoints -- Only allow the requested membership type on `/send_join` and `/send_leave` -endpoints (e.g. don't allow leave memberships on join endpoints) -- Prevent state key impersonation on `/send_join` and `/send_leave` endpoints -- Validate `X-Matrix` origin and request body `"origin"` field on incoming -transactions -- Add `GET /_matrix/client/v1/register/m.login.registration_token/validity` -endpoint -- Explicitly define support for sliding sync at `/_matrix/client/versions` -(`org.matrix.msc3575`) -- Fix seeing empty status messages on user presences - -## Moderation - -- (Also see [Admin Room](#admin-room) for all the admin commands pertaining to -moderation, there's a lot!) -- Add support for room banning/blocking by ID using admin command -- Add support for serving `support` well-known from `[global.well_known]` -(MSC1929) (`/.well-known/matrix/support`) -- Config option to forbid publishing rooms to the room directory -(`lockdown_public_room_directory`) except for admins -- Admin commands to delete room aliases and unpublish rooms from our room -directory -- For all -[`/report`](https://spec.matrix.org/latest/client-server-api/#post_matrixclientv3roomsroomidreporteventid) -requests: check if the reported event ID belongs to the reported room ID, raise -report reasoning character limit to 750, fix broken formatting, make a small -delayed random response per spec suggestion on privacy, and check if the sender -user is in the reported room. -- Support blocking servers from downloading remote media from, returning a 404 -- Don't allow `m.call.invite` events to be sent in public rooms (prevents -calling the entire room) -- On new public room creations, only allow moderators to send `m.call.invite`, -`org.matrix.msc3401.call`, and `org.matrix.msc3401.call.member` events to -prevent unprivileged users from calling the entire room -- Add support for a "global ACLs" feature (`forbidden_remote_server_names`) that -blocks inbound remote room invites, room joins by room ID on server name, room -joins by room alias on server name, incoming federated joins, and incoming -federated room directory requests. This is very helpful for blocking servers -that are purely toxic/bad and serve no value in allowing our users to suffer -from things like room invite spam or such. Please note that this is not a -substitute for room ACLs. -- Add support for a config option to forbid our local users from sending -federated room directory requests for -(`forbidden_remote_room_directory_server_names`). Similar to above, useful for -blocking servers that help prevent our users from wandering into bad areas of -Matrix via room directories of those malicious servers. -- Add config option for auto remediating/deactivating local non-admin users who -attempt to join bad/forbidden rooms (`auto_deactivate_banned_room_attempts`) -- Deactivating users will remove their profile picture, blurhash, display name, -and leave all rooms by default just like Synapse and for additional privacy -- Reject some EDUs from ACL'd users such as read receipts and typing indicators - -## Privacy/Security - -- Add config option for device name federation with a privacy-friendly default -(disabled) -- Add config option for requiring authentication to the `/publicRooms` endpoint -(room directory) with a default enabled for privacy -- Add config option for federating `/publicRooms` endpoint (room directory) to -other servers with a default disabled for privacy -- Uses proper `argon2` crate by RustCrypto instead of questionable `rust-argon2` -crate -- Generate passwords with 25 characters instead of 15 -- Config option `ip_range_denylist` to support refusing to send requests -(typically federation) to specific IP ranges, typically RFC 1918, non-routable, -testnet, etc addresses like Synapse for security (note: this is not a guaranteed -protection, and you should be using a firewall with zones if you want guaranteed -protection as doing this on the application level is prone to bypasses). -- Config option to block non-admin users from sending room invites or receiving -remote room invites. Admin users are still allowed. -- Config option to disable incoming and/or outgoing remote read receipts -- Config option to disable incoming and/or outgoing remote typing indicators -- Config option to disable incoming, outgoing, and/or local presence and for -timing out remote users -- Sanitise file names for the `Content-Disposition` header for all media -requests (thumbnails, downloads, uploads) -- Media repository on handling `Content-Disposition` and `Content-Type` is fully -spec compliant and secured -- Send secure default HTTP headers such as a strong restrictive CSP (see -MSC4149), deny iframes, disable `X-XSS-Protection`, disable interest cohort in -`Permission-Policy`, etc to mitigate any potential attack surface such as from -untrusted media - -## Administration/Logging - -- Commandline argument to specify the path to a config file instead of relying -on `CONDUIT_CONFIG` -- Revamped admin room infrastructure and commands -- Substantially clean up, improve, and fix logging (less noisy dead server -logging, registration attempts, more useful troubleshooting logging, proper -error propagation, etc) -- Configurable RocksDB logging (`LOG` files) with proper defaults (rotate, max -size, verbosity, etc) to stop LOG files from accumulating so much -- Explicit startup error if your configuration allows open registration without -a token or such like Synapse with a way to bypass it if needed -- Replace the lightning bolt emoji option with support for setting any arbitrary -text (e.g. another emoji) to suffix to all new user registrations, with a -conduwuit default of "🏳️‍⚧️" -- Implement config option to auto join rooms upon registration -- Warn on unknown config options specified -- Add `/_conduwuit/server_version` route to return the version of conduwuit -without relying on the federation API `/_matrix/federation/v1/version` -- Add `/_conduwuit/local_user_count` route to return the amount of registered -active local users on your homeserver *if federation is enabled* -- Add configurable RocksDB recovery modes to aid in recovering corrupted RocksDB -databases -- Support config options via `CONDUWUIT_` prefix and accessing non-global struct -config options with the `__` split (e.g. `CONDUWUIT_WELL_KNOWN__SERVER`) -- Add support for listening on multiple TCP ports and multiple addresses -- **Opt-in** Sentry.io telemetry and metrics, mainly used for crash reporting -- Log the client IP on various requests such as registrations, banned room join -attempts, logins, deactivations, federation transactions, etc -- Fix Conduit dropping some remote server federation response errors - -## Maintenance/Stability - -- GitLab CI ported to GitHub Actions -- Add support for the Matrix spec compliance test suite -[Complement](https://github.com/matrix-org/complement/) via the Nix flake and -various other fixes for it -- Implement running and diff'ing Complement results in CI and error if any -mismatch occurs to prevent large cases of conduwuit regressions -- Repo is (officially) mirrored to GitHub, GitLab, git.gay, git.girlcock.ceo, -sourcehut, and Codeberg (see README.md for their links) -- Docker container images published to GitLab Container Registry, GitHub -Container Registry, and Dockerhub -- Extensively revamp the example config to be extremely helpful and useful to -both new users and power users -- Fixed every single clippy (default lints) and rustc warnings, including some -that were performance related or potential safety issues / unsoundness -- Add a **lot** of other clippy and rustc lints and a rustfmt.toml file -- Repo uses [Renovate](https://docs.renovatebot.com/) and keeps ALL -dependencies as up to date as possible -- Purge unmaintained/irrelevant/broken database backends (heed, sled, persy) and -other unnecessary code or overhead -- webp support for images -- Add cargo audit support to CI -- Add documentation lints via lychee and markdownlint-cli to CI -- CI tests for all sorts of feature matrixes (jemalloc, non-defaullt, all -features, etc) -- Add static and dynamic linking smoke tests in CI to prevent any potential -linking regressions for Complement, static binaries, Nix devshells, etc -- Add timestamp by commit date when building OCI images for keeping image build -reproducibility and still have a meaningful "last modified date" for OCI image -- Add timestamp by commit date via `SOURCE_DATE_EPOCH` for Debian packages -- Startup check if conduwuit running in a container and is listening on -127.0.0.1 (generally containers are using NAT networking and 0.0.0.0 is the -intended listening address) -- Add a panic catcher layer to return panic messages in HTTP responses if a -panic occurs -- Add full compatibility support for SHA256 media file names instead of base64 -file names to overcome filesystem file name length limitations (OS error file -name too long) while still retaining upstream database compatibility -- Remove SQLite support due to being very poor performance, difficult to -maintain against RocksDB, and is a blocker to significantly improved database -code - -## Admin Room - -- Add support for a console CLI interface that can issue admin commands and -output them in your terminal -- Add support for an admin-user-only commandline admin room interface that can -be issued in any room with the `\\!admin` or `\!admin` prefix and returns the -response as yourself in the same room -- Add admin commands for uptime, server startup, server shutdown, and server -restart -- Fix admin room handler to not panic/crash if the admin room command response -fails (e.g. too large message) -- Add command to dynamically change conduwuit's tracing log level filter on the -fly -- Add admin command to fetch a server's `/.well-known/matrix/support` file -- Add debug admin command to force update user device lists (could potentially -resolve some E2EE flukes) -- Implement **RocksDB online backups**, listing RocksDB backups, and listing -database file counts all via admin commands -- Add various database visibility commands such as being able to query the -getters and iterators used in conduwuit, a very helpful online debugging utility -- Forbid the admin room from being made public or world readable history -- Add `!admin` as a way to call the admin bot -- Extend clear cache admin command to support clearing more caches such as DNS -and TLS name overrides -- Admin debug command to send a federation request/ping to a server's -`/_matrix/federation/v1/version` endpoint and measures the latency it took -- Add admin command to bulk delete media via a codeblock list of MXC URLs. -- Add admin command to delete both the thumbnail and media MXC URLs from an -event ID (e.g. from an abuse report) -- Add admin command to list all the rooms a local user is joined in -- Add admin command to list joined members in a room -- Add admin command to view the room topic of a room -- Add admin command to delete all remote media in the past X minutes as a form -of deleting media that you don't want on your server that a remote user posted -in a room, a `--force` flag to ignore errors, and support for reading `last -modified time` instead of `creation time` for filesystems that don't support -file created metadata -- Add admin command to return a room's full/complete state -- Admin debug command to fetch a PDU from a remote server and inserts it into -our database/timeline as backfill -- Add admin command to delete media via a specific MXC. This deletes the MXC -from our database, and the file locally. -- Add admin commands for banning (blocking) room IDs from our local users -joining (admins are always allowed) and evicts all our local users from that -room, in addition to bulk room banning support, and blocks room invites (remote -and local) to the banned room, as a moderation feature -- Add admin commands to output jemalloc memory stats and memory usage -- Add admin command to get rooms a *remote* user shares with us -- Add debug admin commands to get the earliest and latest PDU in a room -- Add debug admin command to echo a message -- Add admin command to insert rooms tags for a user, most useful for inserting -the `m.server_notice` tag on your admin room to make it "persistent" in the -"System Alerts" section of Element -- Add experimental admin debug command for Dendrite's `AdminDownloadState` -(`/admin/downloadState/{serverName}/{roomID}`) admin API endpoint to download -and use a remote server's room state in the room -- Disable URL previews by default in the admin room due to various command -outputs having "URLs" in them that clients may needlessly render/request -- Extend memory usage admin server command to support showing memory allocator -stats such as jemalloc's -- Add admin debug command to see memory allocator's full extended debug -statistics such as jemalloc's - -## Misc - -- Add guest support for accessing TURN servers via `turn_allow_guests` like -Synapse -- Support for creating rooms with custom room IDs like Maunium Synapse -(`room_id` request body field to `/createRoom`) -- Query parameter `?format=event|content` for returning either the room state -event's content (default) for the full room state event on -`/_matrix/client/v3/rooms/{roomId}/state/{eventType}[/{stateKey}]` requests (see -) -- Send a User-Agent on all of our requests -- Send `avatar_url` on invite room membership events/changes -- Support sending [`well_known` response to client login -responses](https://spec.matrix.org/v1.10/client-server-api/#post_matrixclientv3login) -if using config option `[well_known.client]` -- Implement `include_state` search criteria support for `/search` requests -(response now can include room states) -- Declare various missing Matrix versions and features at -`/_matrix/client/versions` -- Implement legacy Matrix `/v1/` media endpoints that some clients and servers -may still call -- Config option to change Conduit's behaviour of homeserver key fetching -(`query_trusted_key_servers_first`). This option sets whether conduwuit will -query trusted notary key servers first before the individual homeserver(s), or -vice versa which may help in joining certain rooms. -- Implement unstable MSC2666 support for querying mutual rooms with a user -- Implement unstable MSC3266 room summary API support -- Implement unstable MSC4125 support for specifying servers to join via on -federated invites -- Make conduwuit build and be functional under Nix + macOS -- Log out all sessions after unsetting the emergency password -- Assume well-knowns are broken if they exceed past 12288 characters. -- Add support for listening on both HTTP and HTTPS if using direct TLS with -conduwuit for usecases such as Complement -- Add config option for disabling RocksDB Direct IO if needed -- Add various documentation on maintaining conduwuit, using RocksDB online -backups, some troubleshooting, using admin commands, moderation documentation, -etc -- (Developers): Add support for [hot reloadable/"live" modular -development](development/hot_reload.md) -- (Developers): Add support for tokio-console -- (Developers): Add support for tracing flame graphs -- No cryptocurrency donations allowed, conduwuit is fully maintained by -independent queer maintainers, and with a strong priority on inclusitivity and -comfort for protected groups 🏳️‍⚧️ -- [Add a community Code of Conduct for all conduwuit community spaces, primarily -the Matrix space](https://conduwuit.puppyirl.gay/conduwuit_coc.html) diff --git a/docs/introduction.md b/docs/introduction.md index 9db76681..9d3a294a 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -4,10 +4,6 @@ {{#include ../README.md:body}} -#### What's different about your fork than upstream Conduit? - -See the [differences](differences.md) page - #### How can I deploy my own? - [Deployment options](deploying.md) From 1e23c95ec6e059c5d9b2b0083868596f1d38f5aa Mon Sep 17 00:00:00 2001 From: Tamara Schmitz <15906939+tamara-schmitz@users.noreply.github.com> Date: Mon, 10 Mar 2025 21:27:53 +0000 Subject: [PATCH 1574/2091] docs: refactor reverse proxy setup sections (#701) --- docs/deploying/generic.md | 59 ++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 88ba01d5..a07da560 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -145,25 +145,32 @@ sudo chmod 700 /var/lib/conduwuit/ ## Setting up the Reverse Proxy -Refer to the documentation or various guides online of your chosen reverse proxy -software. There are many examples of basic Apache/Nginx reverse proxy setups -out there. +We recommend Caddy as a reverse proxy, as it is trivial to use, handling TLS certificates, reverse proxy headers, etc transparently with proper defaults. +For other software, please refer to their respective documentation or online guides. -A [Caddy](https://caddyserver.com/) example will be provided as this -is the recommended reverse proxy for new users and is very trivial to use -(handles TLS, reverse proxy headers, etc transparently with proper defaults). +### Caddy -Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization -header, making federation non-functional. If a workaround is found, feel free to share to get it added to the documentation here. +After installing Caddy via your preferred method, create `/etc/caddy/conf.d/conduwuit_caddyfile` +and enter this (substitute for your server name). -If using Apache, you need to use `nocanon` in your `ProxyPass` directive to prevent this (note that Apache isn't very good as a general reverse proxy and we discourage the usage of it if you can). +```caddyfile +your.server.name, your.server.name:8448 { + # TCP reverse_proxy + reverse_proxy 127.0.0.1:6167 + # UNIX socket + #reverse_proxy unix//run/conduwuit/conduwuit.sock +} +``` -If using Nginx, you need to give conduwuit the request URI using `$request_uri`, or like so: -- `proxy_pass http://127.0.0.1:6167$request_uri;` -- `proxy_pass http://127.0.0.1:6167;` +That's it! Just start and enable the service and you're set. -Nginx users need to increase `client_max_body_size` (default is 1M) to match -`max_request_size` defined in conduwuit.toml. +```bash +sudo systemctl enable --now caddy +``` + +### Other Reverse Proxies + +As we would prefer our users to use Caddy, we will not provide configuration files for other proxys. You will need to reverse proxy everything under following routes: - `/_matrix/` - core Matrix C-S and S-S APIs @@ -186,25 +193,19 @@ Examples of delegation: - - -### Caddy +For Apache and Nginx there are many examples available online. -Create `/etc/caddy/conf.d/conduwuit_caddyfile` and enter this (substitute for -your server name). +Lighttpd is not supported as it seems to mess with the `X-Matrix` Authorization +header, making federation non-functional. If a workaround is found, feel free to share to get it added to the documentation here. -```caddyfile -your.server.name, your.server.name:8448 { - # TCP reverse_proxy - reverse_proxy 127.0.0.1:6167 - # UNIX socket - #reverse_proxy unix//run/conduwuit/conduwuit.sock -} -``` +If using Apache, you need to use `nocanon` in your `ProxyPass` directive to prevent httpd from messing with the `X-Matrix` header (note that Apache isn't very good as a general reverse proxy and we discourage the usage of it if you can). -That's it! Just start and enable the service and you're set. +If using Nginx, you need to give conduwuit the request URI using `$request_uri`, or like so: +- `proxy_pass http://127.0.0.1:6167$request_uri;` +- `proxy_pass http://127.0.0.1:6167;` -```bash -sudo systemctl enable --now caddy -``` +Nginx users need to increase `client_max_body_size` (default is 1M) to match +`max_request_size` defined in conduwuit.toml. ## You're done From 1366a3092f5be044fbe39225dd606ef3445899d5 Mon Sep 17 00:00:00 2001 From: Ginger <75683114+gingershaped@users.noreply.github.com> Date: Mon, 10 Mar 2025 17:28:19 -0400 Subject: [PATCH 1575/2091] Check the `room_types` filter when searching for local public rooms (#698) --- src/api/client/directory.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 88f0e668..7ce32e4c 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -15,7 +15,7 @@ use ruma::{ }, federation, }, - directory::{Filter, PublicRoomJoinRule, PublicRoomsChunk, RoomNetwork}, + directory::{Filter, PublicRoomJoinRule, PublicRoomsChunk, RoomNetwork, RoomTypeFilter}, events::{ StateEventType, room::{ @@ -289,6 +289,9 @@ pub(crate) async fn get_public_rooms_filtered_helper( .map(ToOwned::to_owned) .then(|room_id| public_rooms_chunk(services, room_id)) .filter_map(|chunk| async move { + if !filter.room_types.is_empty() && !filter.room_types.contains(&RoomTypeFilter::from(chunk.room_type.clone())) { + return None; + } if let Some(query) = filter.generic_search_term.as_ref().map(|q| q.to_lowercase()) { if let Some(name) = &chunk.name { if name.as_str().to_lowercase().contains(&query) { From c4b05e77f3dd66636e26b64f8f4852703816c399 Mon Sep 17 00:00:00 2001 From: Odd Eivind Ebbesen Date: Mon, 10 Mar 2025 22:28:29 +0100 Subject: [PATCH 1576/2091] Fix up wording in the doc comments for admin media deletion (#694) --- src/admin/media/mod.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index d212aab4..405c26d5 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -27,18 +27,18 @@ pub(super) enum MediaCommand { DeleteList, /// - Deletes all remote (and optionally local) media created before or - /// after \[duration] time using filesystem metadata first created at - /// date, or fallback to last modified date. This will always ignore - /// errors by default. + /// after [duration] time using filesystem metadata first created at date, + /// or fallback to last modified date. This will always ignore errors by + /// default. DeletePastRemoteMedia { /// - The relative time (e.g. 30s, 5m, 7d) within which to search duration: String, - /// - Only delete media created more recently than \[duration] ago + /// - Only delete media created before [duration] ago #[arg(long, short)] before: bool, - /// - Only delete media created after \[duration] ago + /// - Only delete media created after [duration] ago #[arg(long, short)] after: bool, From 3104586884b0027a1404bfe1986d569ff9e492d4 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 11 Mar 2025 18:05:36 -0400 Subject: [PATCH 1577/2091] bump tracing-subscriber, allowlist cargo-doc lint in admin room Signed-off-by: June Clementine Strawberry --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 10 +++++----- src/admin/media/mod.rs | 1 + 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 65e8eca1..22d93237 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -440,7 +440,7 @@ dependencies = [ "bitflags 2.9.0", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -2382,7 +2382,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ "cfg-if", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -4833,7 +4833,7 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" version = "0.1.41" -source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" +source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "log", "pin-project-lite", @@ -4844,7 +4844,7 @@ dependencies = [ [[package]] name = "tracing-attributes" version = "0.1.28" -source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" +source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "proc-macro2", "quote", @@ -4854,7 +4854,7 @@ dependencies = [ [[package]] name = "tracing-core" version = "0.1.33" -source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" +source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "once_cell", "valuable", @@ -4874,7 +4874,7 @@ dependencies = [ [[package]] name = "tracing-log" version = "0.2.0" -source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" +source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "log", "once_cell", @@ -4901,8 +4901,8 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" -source = "git+https://github.com/girlbossceo/tracing?rev=05825066a6d0e9ad6b80dcf29457eb179ff4768c#05825066a6d0e9ad6b80dcf29457eb179ff4768c" +version = "0.3.19" +source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "matchers", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index d611c08e..1528349c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -210,7 +210,7 @@ default-features = false version = "0.1.41" default-features = false [workspace.dependencies.tracing-subscriber] -version = "=0.3.18" +version = "0.3.19" default-features = false features = ["env-filter", "std", "tracing", "tracing-log", "ansi", "fmt"] [workspace.dependencies.tracing-core] @@ -541,16 +541,16 @@ version = "1.0.2" # https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c [patch.crates-io.tracing-subscriber] git = "https://github.com/girlbossceo/tracing" -rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c" +rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" [patch.crates-io.tracing] git = "https://github.com/girlbossceo/tracing" -rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c" +rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" [patch.crates-io.tracing-core] git = "https://github.com/girlbossceo/tracing" -rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c" +rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" [patch.crates-io.tracing-log] git = "https://github.com/girlbossceo/tracing" -rev = "05825066a6d0e9ad6b80dcf29457eb179ff4768c" +rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" # adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50 # adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 405c26d5..641834b2 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -1,3 +1,4 @@ +#![allow(rustdoc::broken_intra_doc_links)] mod commands; use clap::Subcommand; From 7f95eef9abf86298a25fd0bd410835084742eaae Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 11 Mar 2025 21:01:20 -0400 Subject: [PATCH 1578/2091] bump ruwuma Signed-off-by: June Clementine Strawberry --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 22d93237..c93716f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3507,7 +3507,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "assign", "js_int", @@ -3527,7 +3527,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "js_int", "ruma-common", @@ -3539,7 +3539,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "as_variant", "assign", @@ -3562,7 +3562,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "as_variant", "base64 0.22.1", @@ -3594,7 +3594,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "as_variant", "indexmap 2.7.1", @@ -3619,7 +3619,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "bytes", "headers", @@ -3641,7 +3641,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3650,7 +3650,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "js_int", "ruma-common", @@ -3660,7 +3660,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3675,7 +3675,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "js_int", "ruma-common", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=f5ab6302aaa55a14827a9cb5b40e980dd135fe14#f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 1528349c..c09cdaea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "f5ab6302aaa55a14827a9cb5b40e980dd135fe14" +rev = "69133fd53ca063552788c8dfbaf5e01c98dec3e7" features = [ "compat", "rand", From ae818d5b25977a6c4543bca16b78af6f2fa0cca7 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 11 Mar 2025 21:08:41 -0400 Subject: [PATCH 1579/2091] remove most of cargo test from engage as crane does that but with more caching Signed-off-by: June Clementine Strawberry --- engage.toml | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/engage.toml b/engage.toml index 0a857b5a..210bafd5 100644 --- a/engage.toml +++ b/engage.toml @@ -161,24 +161,6 @@ name = "markdownlint" group = "lints" script = "markdownlint docs *.md || true" # TODO: fix the ton of markdown lints so we can drop `|| true` -[[task]] -name = "cargo/all" -group = "tests" -script = """ -env DIRENV_DEVSHELL=all-features \ - direnv exec . \ - cargo test \ - --workspace \ - --locked \ - --profile test \ - --all-targets \ - --no-fail-fast \ - --all-features \ - --color=always \ - -- \ - --color=always -""" - [[task]] name = "cargo/default" group = "tests" @@ -196,24 +178,6 @@ env DIRENV_DEVSHELL=default \ --color=always """ -[[task]] -name = "cargo/no-features" -group = "tests" -script = """ -env DIRENV_DEVSHELL=no-features \ - direnv exec . \ - cargo test \ - --workspace \ - --locked \ - --profile test \ - --all-targets \ - --no-fail-fast \ - --no-default-features \ - --color=always \ - -- \ - --color=always -""" - # Checks if the generated example config differs from the checked in repo's # example config. [[task]] From e920c44cb488d398bc57fe4ce7fdffb3ded5038a Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 11 Mar 2025 21:15:11 -0400 Subject: [PATCH 1580/2091] ignore humantime dep as tracing console-subscriber uses it (somewhere) Signed-off-by: June Clementine Strawberry --- .cargo/audit.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cargo/audit.toml b/.cargo/audit.toml index bf44fbd6..37148cfb 100644 --- a/.cargo/audit.toml +++ b/.cargo/audit.toml @@ -1,5 +1,5 @@ [advisories] -ignore = ["RUSTSEC-2024-0436"] # advisory IDs to ignore e.g. ["RUSTSEC-2019-0001", ...] +ignore = ["RUSTSEC-2024-0436", "RUSTSEC-2025-0014"] # advisory IDs to ignore e.g. ["RUSTSEC-2019-0001", ...] informational_warnings = [] # warn for categories of informational advisories severity_threshold = "none" # CVSS severity ("none", "low", "medium", "high", "critical") From 0877f294393954bbe49279456f012e1fbb604f78 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 11 Mar 2025 22:21:53 -0400 Subject: [PATCH 1581/2091] respect membership filters on /members Signed-off-by: June Clementine Strawberry --- Cargo.lock | 22 +++++++------- Cargo.toml | 2 +- src/api/client/membership.rs | 56 ++++++++++++++++++++++++++++++++++-- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c93716f9..ab155fd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3507,7 +3507,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "assign", "js_int", @@ -3527,7 +3527,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "js_int", "ruma-common", @@ -3539,7 +3539,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "as_variant", "assign", @@ -3562,7 +3562,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "as_variant", "base64 0.22.1", @@ -3594,7 +3594,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "as_variant", "indexmap 2.7.1", @@ -3619,7 +3619,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "bytes", "headers", @@ -3641,7 +3641,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3650,7 +3650,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "js_int", "ruma-common", @@ -3660,7 +3660,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3675,7 +3675,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "js_int", "ruma-common", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=69133fd53ca063552788c8dfbaf5e01c98dec3e7#69133fd53ca063552788c8dfbaf5e01c98dec3e7" +source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index c09cdaea..2bf30d61 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "69133fd53ca063552788c8dfbaf5e01c98dec3e7" +rev = "24d018a0015bb85489ae84564701a49a643bcc57" features = [ "compat", "rand", diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 3f77e69e..11395e83 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -25,8 +25,9 @@ use ruma::{ error::ErrorKind, knock::knock_room, membership::{ - ThirdPartySigned, ban_user, forget_room, get_member_events, invite_user, - join_room_by_id, join_room_by_id_or_alias, + ThirdPartySigned, ban_user, forget_room, + get_member_events::{self, v3::MembershipEventFilter}, + invite_user, join_room_by_id, join_room_by_id_or_alias, joined_members::{self, v3::RoomMember}, joined_rooms, kick_user, leave_room, unban_user, }, @@ -768,6 +769,54 @@ pub(crate) async fn joined_rooms_route( }) } +fn membership_filter( + pdu: PduEvent, + for_membership: Option<&MembershipEventFilter>, + not_membership: Option<&MembershipEventFilter>, +) -> Option { + let membership_state_filter = match for_membership { + | Some(MembershipEventFilter::Ban) => MembershipState::Ban, + | Some(MembershipEventFilter::Invite) => MembershipState::Invite, + | Some(MembershipEventFilter::Knock) => MembershipState::Knock, + | Some(MembershipEventFilter::Leave) => MembershipState::Leave, + | Some(_) | None => MembershipState::Join, + }; + + let not_membership_state_filter = match not_membership { + | Some(MembershipEventFilter::Ban) => MembershipState::Ban, + | Some(MembershipEventFilter::Invite) => MembershipState::Invite, + | Some(MembershipEventFilter::Join) => MembershipState::Join, + | Some(MembershipEventFilter::Knock) => MembershipState::Knock, + | Some(_) | None => MembershipState::Leave, + }; + + let evt_membership = pdu.get_content::().ok()?.membership; + + if for_membership.is_some() && not_membership.is_some() { + if membership_state_filter != evt_membership + || not_membership_state_filter == evt_membership + { + None + } else { + Some(pdu) + } + } else if for_membership.is_some() && not_membership.is_none() { + if membership_state_filter != evt_membership { + None + } else { + Some(pdu) + } + } else if not_membership.is_some() && for_membership.is_none() { + if not_membership_state_filter == evt_membership { + None + } else { + Some(pdu) + } + } else { + Some(pdu) + } +} + /// # `POST /_matrix/client/r0/rooms/{roomId}/members` /// /// Lists all joined users in a room (TODO: at a specific point in time, with a @@ -779,6 +828,8 @@ pub(crate) async fn get_member_events_route( body: Ruma, ) -> Result { let sender_user = body.sender_user(); + let membership = body.membership.as_ref(); + let not_membership = body.not_membership.as_ref(); if !services .rooms @@ -797,6 +848,7 @@ pub(crate) async fn get_member_events_route( .ready_filter_map(Result::ok) .ready_filter(|((ty, _), _)| *ty == StateEventType::RoomMember) .map(at!(1)) + .ready_filter_map(|pdu| membership_filter(pdu, membership, not_membership)) .map(PduEvent::into_member_event) .collect() .await, From 1d1ccec532bf3eaebf499d3ff4c9f7a24369c389 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 11 Mar 2025 23:05:56 -0400 Subject: [PATCH 1582/2091] fix some nightly clippy lints Signed-off-by: June Clementine Strawberry --- Cargo.toml | 3 +++ clippy.toml | 3 ++- src/admin/processor.rs | 8 +++++--- src/api/client/account.rs | 4 ++-- src/api/client/state.rs | 2 +- src/core/utils/string.rs | 1 + 6 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2bf30d61..fd477850 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -841,6 +841,9 @@ unused_crate_dependencies = "allow" unsafe_code = "allow" variant_size_differences = "allow" +# we check nightly clippy lints +unknown_lints = "allow" + ####################################### # # Clippy lints diff --git a/clippy.toml b/clippy.toml index 42427101..863759aa 100644 --- a/clippy.toml +++ b/clippy.toml @@ -2,9 +2,10 @@ array-size-threshold = 4096 cognitive-complexity-threshold = 94 # TODO reduce me ALARA excessive-nesting-threshold = 11 # TODO reduce me to 4 or 5 future-size-threshold = 7745 # TODO reduce me ALARA -stack-size-threshold = 196608 # reduce me ALARA +stack-size-threshold = 196608 # TODO reduce me ALARA too-many-lines-threshold = 780 # TODO reduce me to <= 100 type-complexity-threshold = 250 # reduce me to ~200 +large-error-threshold = 256 # TODO reduce me ALARA disallowed-macros = [ { path = "log::error", reason = "use conduwuit_core::error" }, diff --git a/src/admin/processor.rs b/src/admin/processor.rs index 77a60959..53a15098 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -91,6 +91,7 @@ async fn process_command(services: Arc, input: &CommandInput) -> Proce } } +#[allow(clippy::result_large_err)] fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult { let link = "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; @@ -100,7 +101,7 @@ fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult { Err(reply(content, command.reply_id.as_deref())) } -// Parse and process a message from the admin room +/// Parse and process a message from the admin room async fn process( context: &Command<'_>, command: AdminCommand, @@ -164,7 +165,8 @@ fn capture_create(context: &Command<'_>) -> (Arc, Arc>) { (capture, logs) } -// Parse chat messages from the admin room into an AdminCommand object +/// Parse chat messages from the admin room into an AdminCommand object +#[allow(clippy::result_large_err)] fn parse<'a>( services: &Arc, input: &'a CommandInput, @@ -232,7 +234,7 @@ fn complete_command(mut cmd: clap::Command, line: &str) -> String { ret.join(" ") } -// Parse chat messages from the admin room into an AdminCommand object +/// Parse chat messages from the admin room into an AdminCommand object fn parse_line(command_line: &str) -> Vec { let mut argv = command_line .split_whitespace() diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 2b8209d4..32438098 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -109,7 +109,7 @@ pub(crate) async fn get_register_available_route( if !info.is_user_match(&user_id) { return Err!(Request(Exclusive("Username is not in an appservice namespace."))); } - }; + } if services.appservice.is_exclusive_user_id(&user_id).await { return Err!(Request(Exclusive("Username is reserved by an appservice."))); @@ -159,7 +159,7 @@ pub(crate) async fn register_route( | (None, _) => { info!(%is_guest, "Rejecting registration attempt as registration is disabled"); }, - }; + } return Err!(Request(Forbidden("Registration has been disabled."))); } diff --git a/src/api/client/state.rs b/src/api/client/state.rs index db79735f..9563c26d 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -254,7 +254,7 @@ async fn allowed_to_send_state_event( "Room server ACL event is invalid: {e}" )))); }, - }; + } }, | StateEventType::RoomEncryption => // Forbid m.room.encryption if encryption is disabled diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index 9340d009..d8fa3f95 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -60,6 +60,7 @@ pub fn camel_to_snake_string(s: &str) -> String { } #[inline] +#[allow(clippy::unbuffered_bytes)] // these are allocated string utilities, not file I/O utils pub fn camel_to_snake_case(output: &mut O, input: I) -> Result<()> where I: std::io::Read, From 5dea52f0f87dc640274e0f3ecb38b96ac9293f44 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 11 Mar 2025 23:45:53 -0400 Subject: [PATCH 1583/2091] stop doing complement cert gen and just use self-signed cert Signed-off-by: June Clementine Strawberry --- bin/complement | 2 +- flake.lock | 6 +++--- nix/pkgs/complement/certificate.crt | 21 +++++++++++++++++++ nix/pkgs/complement/default.nix | 19 +---------------- nix/pkgs/complement/signing_request.csr | 28 ++++++++++++------------- nix/pkgs/complement/v3.ext | 6 ++++++ 6 files changed, 46 insertions(+), 36 deletions(-) create mode 100644 nix/pkgs/complement/certificate.crt diff --git a/bin/complement b/bin/complement index 92539f97..3aa5a6f5 100755 --- a/bin/complement +++ b/bin/complement @@ -68,7 +68,7 @@ set +o pipefail env \ -C "$COMPLEMENT_SRC" \ COMPLEMENT_BASE_IMAGE="$COMPLEMENT_BASE_IMAGE" \ - go test -tags="conduwuit_blacklist" -timeout 1h -json ./tests/... | tee "$LOG_FILE" + go test -tags="conduwuit_blacklist" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE" set -o pipefail # Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results diff --git a/flake.lock b/flake.lock index 03fc205c..63cc2787 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1741378155, - "narHash": "sha256-rJSfqf3q4oWxcAwENtAowLZeCi8lktwKVH9XQvvZR64=", + "lastModified": 1741757487, + "narHash": "sha256-Fkx/krwI3h6wJ6Mj199KlXUNJNEwl7h1pR4/d2ncmKw=", "owner": "girlbossceo", "repo": "complement", - "rev": "1502a00d8551d0f6e8954a23e43868877c3e57d9", + "rev": "40982a261cfc36650f74967f99fb1a049b13e065", "type": "github" }, "original": { diff --git a/nix/pkgs/complement/certificate.crt b/nix/pkgs/complement/certificate.crt new file mode 100644 index 00000000..5dd4fdea --- /dev/null +++ b/nix/pkgs/complement/certificate.crt @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDfzCCAmegAwIBAgIUcrZdSPmCh33Evys/U6mTPpShqdcwDQYJKoZIhvcNAQEL +BQAwPzELMAkGA1UEBhMCNjkxCzAJBgNVBAgMAjQyMRUwEwYDVQQKDAx3b29mZXJz +IGluYy4xDDAKBgNVBAMMA2hzMTAgFw0yNTAzMTMxMjU4NTFaGA8yMDUyMDcyODEy +NTg1MVowPzELMAkGA1UEBhMCNjkxCzAJBgNVBAgMAjQyMRUwEwYDVQQKDAx3b29m +ZXJzIGluYy4xDDAKBgNVBAMMA2hzMTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC +AQoCggEBANL+h2ZmK/FqN5uLJPtIy6Feqcyb6EX7MQBEtxuJ56bTAbjHuCLZLpYt +/wOWJ91drHqZ7Xd5iTisGdMu8YS803HSnHkzngf4VXKhVrdzW2YDrpZRxmOhtp88 +awOHmP7mqlJyBbCOQw8aDVrT0KmEIWzA7g+nFRQ5Ff85MaP+sQrHGKZbo61q8HBp +L0XuaqNckruUKtxnEqrm5xx5sYyYKg7rrSFE5JMFoWKB1FNWJxyWT42BhGtnJZsK +K5c+NDSOU4TatxoN6mpNSBpCz/a11PiQHMEfqRk6JA4g3911dqPTfZBevUdBh8gl +8maIzqeZGhvyeKTmull1Y0781yyuj98CAwEAAaNxMG8wCQYDVR0TBAIwADALBgNV +HQ8EBAMCBPAwNgYDVR0RBC8wLYIRKi5kb2NrZXIuaW50ZXJuYWyCA2hzMYIDaHMy +ggNoczOCA2hzNIcEfwAAATAdBgNVHQ4EFgQUr4VYrmW1d+vjBTJewvy7fJYhLDYw +DQYJKoZIhvcNAQELBQADggEBADkYqkjNYxjWX8hUUAmFHNdCwzT1CpYe/5qzLiyJ +irDSdMlC5g6QqMUSrpu7nZxo1lRe1dXGroFVfWpoDxyCjSQhplQZgtYqtyLfOIx+ +HQ7cPE/tUU/KsTGc0aL61cETB6u8fj+rQKUGdfbSlm0Rpu4v0gC8RnDj06X/hZ7e +VkWU+dOBzxlqHuLlwFFtVDgCyyTatIROx5V+GpMHrVqBPO7HcHhwqZ30k2kMM8J3 +y1CWaliQM85jqtSZV+yUHKQV8EksSowCFJuguf+Ahz0i0/koaI3i8m4MRN/1j13d +jbTaX5a11Ynm3A27jioZdtMRty6AJ88oCp18jxVzqTxNNO4= +-----END CERTIFICATE----- diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index bbd1bd74..9b010e14 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -3,10 +3,8 @@ , buildEnv , coreutils , dockerTools -, gawk , lib , main -, openssl , stdenv , tini , writeShellScriptBin @@ -42,21 +40,6 @@ let start = writeShellScriptBin "start" '' set -euxo pipefail - cp ${./v3.ext} /complement/v3.ext - echo "DNS.1 = $SERVER_NAME" >> /complement/v3.ext - echo "IP.1 = $(${lib.getExe gawk} 'END{print $1}' /etc/hosts)" \ - >> /complement/v3.ext - ${lib.getExe openssl} x509 \ - -req \ - -extfile /complement/v3.ext \ - -in ${./signing_request.csr} \ - -CA /complement/ca/ca.crt \ - -CAkey /complement/ca/ca.key \ - -CAcreateserial \ - -out /complement/certificate.crt \ - -days 1 \ - -sha256 - ${lib.getExe' coreutils "env"} \ CONDUWUIT_SERVER_NAME="$SERVER_NAME" \ ${lib.getExe main'} @@ -93,7 +76,7 @@ dockerTools.buildImage { Env = [ "CONDUWUIT_TLS__KEY=${./private_key.key}" - "CONDUWUIT_TLS__CERTS=/complement/certificate.crt" + "CONDUWUIT_TLS__CERTS=${./certificate.crt}" "CONDUWUIT_CONFIG=${./config.toml}" "RUST_BACKTRACE=full" ]; diff --git a/nix/pkgs/complement/signing_request.csr b/nix/pkgs/complement/signing_request.csr index 707e73b4..e2aa658e 100644 --- a/nix/pkgs/complement/signing_request.csr +++ b/nix/pkgs/complement/signing_request.csr @@ -1,16 +1,16 @@ -----BEGIN CERTIFICATE REQUEST----- -MIICkTCCAXkCAQAwTDELMAkGA1UEBhMCNjkxCzAJBgNVBAgMAjQyMRYwFAYDVQQK -DA13b29mZXJzLCBpbmMuMRgwFgYDVQQDDA9jb21wbGVtZW50LW9ubHkwggEiMA0G -CSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDS/odmZivxajebiyT7SMuhXqnMm+hF -+zEARLcbieem0wG4x7gi2S6WLf8DlifdXax6me13eYk4rBnTLvGEvNNx0px5M54H -+FVyoVa3c1tmA66WUcZjobafPGsDh5j+5qpScgWwjkMPGg1a09CphCFswO4PpxUU -ORX/OTGj/rEKxximW6OtavBwaS9F7mqjXJK7lCrcZxKq5uccebGMmCoO660hROST -BaFigdRTVicclk+NgYRrZyWbCiuXPjQ0jlOE2rcaDepqTUgaQs/2tdT4kBzBH6kZ -OiQOIN/ddXaj032QXr1HQYfIJfJmiM6nmRob8nik5rpZdWNO/Ncsro/fAgMBAAGg -ADANBgkqhkiG9w0BAQsFAAOCAQEAjW+aD4E0phtRT5b2RyedY1uiSe7LQECsQnIO -wUSyGGG1GXYlJscyxxyzE9W9+QIALrxZkmc/+e02u+bFb1zQXW/uB/7u7FgXzrj6 -2YSDiWYXiYKvgGWEfCi3lpcTJK9x6WWkR+iREaoKRjcl0ynhhGuR7YwP38TNyu+z -FN6B1Lo398fvJkaTCiiHngWiwztXZ2d0MxkicuwZ1LJhIQA72OTl3QoRb5uiqbze -T9QJfU6W3v8cB8c8PuKMv5gl1QsGNtlfyQB56/X0cMxWl25vWXd2ankLkAGRTDJ8 -9YZHxP1ki4/yh75AknFq02nCOsmxYrAazCYgP2TzIPhQwBurKQ== +MIIChDCCAWwCAQAwPzELMAkGA1UEBhMCNjkxCzAJBgNVBAgMAjQyMRUwEwYDVQQK +DAx3b29mZXJzIGluYy4xDDAKBgNVBAMMA2hzMTCCASIwDQYJKoZIhvcNAQEBBQAD +ggEPADCCAQoCggEBANL+h2ZmK/FqN5uLJPtIy6Feqcyb6EX7MQBEtxuJ56bTAbjH +uCLZLpYt/wOWJ91drHqZ7Xd5iTisGdMu8YS803HSnHkzngf4VXKhVrdzW2YDrpZR +xmOhtp88awOHmP7mqlJyBbCOQw8aDVrT0KmEIWzA7g+nFRQ5Ff85MaP+sQrHGKZb +o61q8HBpL0XuaqNckruUKtxnEqrm5xx5sYyYKg7rrSFE5JMFoWKB1FNWJxyWT42B +hGtnJZsKK5c+NDSOU4TatxoN6mpNSBpCz/a11PiQHMEfqRk6JA4g3911dqPTfZBe +vUdBh8gl8maIzqeZGhvyeKTmull1Y0781yyuj98CAwEAAaAAMA0GCSqGSIb3DQEB +CwUAA4IBAQDR/gjfxN0IID1MidyhZB4qpdWn3m6qZnEQqoTyHHdWalbfNXcALC79 +ffS+Smx40N5hEPvqy6euR89N5YuYvt8Hs+j7aWNBn7Wus5Favixcm2JcfCTJn2R3 +r8FefuSs2xGkoyGsPFFcXE13SP/9zrZiwvOgSIuTdz/Pbh6GtEx7aV4DqHJsrXnb +XuPxpQleoBqKvQgSlmaEBsJg13TQB+Fl2foBVUtqAFDQiv+RIuircf0yesMCKJaK +MPH4Oo+r3pR8lI8ewfJPreRhCoV+XrGYMubaakz003TJ1xlOW8M+N9a6eFyMVh76 +U1nY/KP8Ua6Lgaj9PRz7JCRzNoshZID/ -----END CERTIFICATE REQUEST----- diff --git a/nix/pkgs/complement/v3.ext b/nix/pkgs/complement/v3.ext index 6083d960..0deaa48a 100644 --- a/nix/pkgs/complement/v3.ext +++ b/nix/pkgs/complement/v3.ext @@ -4,3 +4,9 @@ keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment subjectAltName = @alt_names [alt_names] +DNS.1 = *.docker.internal +DNS.2 = hs1 +DNS.3 = hs2 +DNS.4 = hs3 +DNS.5 = hs4 +IP.1 = 127.0.0.1 From 258b399de93e74b00695ab42697dc31f5a49aa81 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 13 Mar 2025 10:52:13 -0400 Subject: [PATCH 1584/2091] bump ruwuma Signed-off-by: June Clementine Strawberry --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 2 +- flake.lock | 6 +++--- nix/pkgs/complement/config.toml | 6 +++--- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab155fd0..c28f4eab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3507,7 +3507,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "assign", "js_int", @@ -3527,7 +3527,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "js_int", "ruma-common", @@ -3539,7 +3539,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "as_variant", "assign", @@ -3562,7 +3562,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "as_variant", "base64 0.22.1", @@ -3594,7 +3594,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "as_variant", "indexmap 2.7.1", @@ -3619,7 +3619,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "bytes", "headers", @@ -3641,7 +3641,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "js_int", "thiserror 2.0.11", @@ -3650,7 +3650,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "js_int", "ruma-common", @@ -3660,7 +3660,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3675,7 +3675,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "js_int", "ruma-common", @@ -3687,7 +3687,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=24d018a0015bb85489ae84564701a49a643bcc57#24d018a0015bb85489ae84564701a49a643bcc57" +source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index fd477850..db55b9b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "24d018a0015bb85489ae84564701a49a643bcc57" +rev = "d197318a2507d38ffe6ee524d0d52728ca72538a" features = [ "compat", "rand", diff --git a/flake.lock b/flake.lock index 63cc2787..1f87b9b6 100644 --- a/flake.lock +++ b/flake.lock @@ -80,11 +80,11 @@ "complement": { "flake": false, "locked": { - "lastModified": 1741757487, - "narHash": "sha256-Fkx/krwI3h6wJ6Mj199KlXUNJNEwl7h1pR4/d2ncmKw=", + "lastModified": 1741891349, + "narHash": "sha256-YvrzOWcX7DH1drp5SGa+E/fc7wN3hqFtPbqPjZpOu1Q=", "owner": "girlbossceo", "repo": "complement", - "rev": "40982a261cfc36650f74967f99fb1a049b13e065", + "rev": "e587b3df569cba411aeac7c20b6366d03c143745", "type": "github" }, "original": { diff --git a/nix/pkgs/complement/config.toml b/nix/pkgs/complement/config.toml index 759f8d78..7f4ecef7 100644 --- a/nix/pkgs/complement/config.toml +++ b/nix/pkgs/complement/config.toml @@ -6,7 +6,7 @@ allow_public_room_directory_over_federation = true allow_public_room_directory_without_auth = true allow_registration = true database_path = "/database" -log = "trace,h2=warn,hyper=warn" +log = "trace,h2=debug,hyper=debug" port = [8008, 8448] trusted_servers = [] only_query_trusted_key_servers = false @@ -19,11 +19,11 @@ url_preview_domain_explicit_denylist = ["*"] media_compat_file_link = false media_startup_check = true prune_missing_media = true -log_colors = false +log_colors = true admin_room_notices = false allow_check_for_updates = false intentionally_unknown_config_option_for_testing = true -rocksdb_log_level = "debug" +rocksdb_log_level = "info" rocksdb_max_log_files = 1 rocksdb_recovery_mode = 0 rocksdb_paranoid_file_checks = true From 6c29792b3d9dfe1e65c5d3545296d431e058e375 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 11 Mar 2025 22:21:42 -0400 Subject: [PATCH 1585/2091] respect include_leave syncv3 filter Signed-off-by: June Clementine Strawberry --- src/api/client/sync/v3.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index fb59837b..70c4c6a7 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -219,6 +219,7 @@ pub(crate) async fn build_sync_events( sender_user, next_batch, full_state, + filter.room.include_leave, &filter, ) .map_ok(move |left_room| (room_id, left_room)) @@ -412,6 +413,7 @@ async fn handle_left_room( sender_user: &UserId, next_batch: u64, full_state: bool, + include_leave: bool, filter: &FilterDefinition, ) -> Result> { let left_count = services @@ -540,6 +542,10 @@ async fn handle_left_room( continue; }; + if !include_leave && pdu.sender == sender_user { + continue; + } + left_state_events.push(pdu.to_sync_state_event()); } } From ee3c585555a80c037bdaa861beeecbf6e19a7f04 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Fri, 14 Mar 2025 15:57:18 -0400 Subject: [PATCH 1586/2091] skip a few flakey complement tests Signed-off-by: June Clementine Strawberry --- bin/complement | 4 +- .../complement/test_results.jsonl | 82 ++----------------- 2 files changed, 8 insertions(+), 78 deletions(-) diff --git a/bin/complement b/bin/complement index 3aa5a6f5..c437503e 100755 --- a/bin/complement +++ b/bin/complement @@ -18,7 +18,7 @@ RESULTS_FILE="${3:-complement_test_results.jsonl}" COMPLEMENT_BASE_IMAGE="${COMPLEMENT_BASE_IMAGE:-complement-conduwuit:main}" # Complement tests that are skipped due to flakiness/reliability issues or we don't implement such features and won't for a long time -#SKIPPED_COMPLEMENT_TESTS='-skip=TestPartialStateJoin.*' +SKIPPED_COMPLEMENT_TESTS='TestPartialStateJoin.*|TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_when_m.*|TestRoomDeleteAlias/Parallel/Can_delete_canonical_alias|TestUnbanViaInvite.*|TestRoomState/Parallel/GET_/publicRooms_lists.*"|TestRoomDeleteAlias/Parallel/Users_with_sufficient_power-level_can_delete_other.*' # $COMPLEMENT_SRC needs to be a directory to Complement source code if [ -f "$COMPLEMENT_SRC" ]; then @@ -68,7 +68,7 @@ set +o pipefail env \ -C "$COMPLEMENT_SRC" \ COMPLEMENT_BASE_IMAGE="$COMPLEMENT_BASE_IMAGE" \ - go test -tags="conduwuit_blacklist" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE" + go test -tags="conduwuit_blacklist" -skip="$SKIPPED_COMPLEMENT_TESTS" -v -timeout 1h -json ./tests/... | tee "$LOG_FILE" set -o pipefail # Post-process the results into an easy-to-compare format, sorted by Test name for reproducible results diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 5fb850f1..6b5f670e 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -174,10 +174,10 @@ {"Action":"pass","Test":"TestFilter"} {"Action":"fail","Test":"TestFilterMessagesByRelType"} {"Action":"pass","Test":"TestGappedSyncLeaveSection"} -{"Action":"fail","Test":"TestGetFilteredRoomMembers"} -{"Action":"fail","Test":"TestGetFilteredRoomMembers/membership/join"} -{"Action":"fail","Test":"TestGetFilteredRoomMembers/membership/leave"} -{"Action":"fail","Test":"TestGetFilteredRoomMembers/not_membership"} +{"Action":"pass","Test":"TestGetFilteredRoomMembers"} +{"Action":"pass","Test":"TestGetFilteredRoomMembers/membership/join"} +{"Action":"pass","Test":"TestGetFilteredRoomMembers/membership/leave"} +{"Action":"pass","Test":"TestGetFilteredRoomMembers/not_membership"} {"Action":"fail","Test":"TestGetMissingEventsGapFilling"} {"Action":"pass","Test":"TestGetRoomMembers"} {"Action":"fail","Test":"TestGetRoomMembersAtPoint"} @@ -360,72 +360,6 @@ {"Action":"pass","Test":"TestOutboundFederationProfile"} {"Action":"pass","Test":"TestOutboundFederationProfile/Outbound_federation_can_query_profile_data"} {"Action":"pass","Test":"TestOutboundFederationSend"} -{"Action":"fail","Test":"TestPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanFastJoinDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanLazyLoadingSyncDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveDeviceListUpdateDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsWithHalfMissingGrandparentsDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsWithHalfMissingParentsDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveEventsWithMissingParentsDuringPartialStateJoin"} -{"Action":"skip","Test":"TestPartialStateJoin/CanReceivePresenceDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveReceiptDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveSigningKeyUpdateDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveToDeviceDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanReceiveTypingDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/CanSendEventsDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/Can_change_display_name_during_partial_state_join"} -{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking"} -{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_for_user_incorrectly_believed_to_be_in_room"} -{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_when_failing_to_complete_partial_state_join"} -{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_when_leaving_partial_state_room"} -{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_no_longer_tracked_when_new_member_leaves_partial_state_room"} -{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracked_for_new_members_in_partial_state_room"} -{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_pre-existing_members_in_partial_state_room"} -{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_user_incorrectly_believed_to_be_in_room_when_they_join_another_shared_room_before_partial_state_join_completes"} -{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_user_incorrectly_believed_to_be_in_room_when_they_rejoin_after_partial_state_join_completes"} -{"Action":"skip","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_for_user_incorrectly_believed_to_be_in_room_when_they_rejoin_before_partial_state_join_completes"} -{"Action":"fail","Test":"TestPartialStateJoin/Device_list_tracking/Device_list_tracking_when_pre-existing_members_in_partial_state_room_join_another_shared_room"} -{"Action":"fail","Test":"TestPartialStateJoin/EagerIncrementalSyncDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/EagerInitialSyncDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/EagerLongPollingSyncWokenWhenResyncCompletes"} -{"Action":"fail","Test":"TestPartialStateJoin/GappySyncAfterPartialStateSynced"} -{"Action":"fail","Test":"TestPartialStateJoin/Lazy-loading_gappy_sync_includes_remote_memberships_during_partial_state_join"} -{"Action":"fail","Test":"TestPartialStateJoin/Lazy-loading_incremental_sync_includes_remote_memberships_during_partial_state_join"} -{"Action":"fail","Test":"TestPartialStateJoin/Lazy-loading_initial_sync_includes_remote_memberships_during_partial_state_join"} -{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync"} -{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/can_be_triggered_by_remote_ban"} -{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/can_be_triggered_by_remote_kick"} -{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/does_not_wait_for_resync"} -{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/is_seen_after_the_resync"} -{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/succeeds,_then_another_user_can_join_without_resync_completing"} -{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/succeeds,_then_rejoin_succeeds_without_resync_completing"} -{"Action":"fail","Test":"TestPartialStateJoin/Leave_during_resync/works_after_a_second_partial_join"} -{"Action":"fail","Test":"TestPartialStateJoin/MembersRequestBlocksDuringPartialStateJoin"} -{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates"} -{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_no_longer_reach_departed_servers_after_partial_state_join_completes"} -{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_all_servers_in_partial_state_rooms"} -{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_absent_servers_once_partial_state_join_completes"} -{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_absent_servers_once_partial_state_join_completes_even_though_remote_server_left_room"} -{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_kicked_servers_once_partial_state_join_completes"} -{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_incorrectly_kicked_servers_once_partial_state_join_completes_even_though_remote_server_left_room"} -{"Action":"fail","Test":"TestPartialStateJoin/Outgoing_device_list_updates/Device_list_updates_reach_newly_joined_servers_in_partial_state_rooms"} -{"Action":"fail","Test":"TestPartialStateJoin/PartialStateJoinContinuesAfterRestart"} -{"Action":"fail","Test":"TestPartialStateJoin/PartialStateJoinSyncsUsingOtherHomeservers"} -{"Action":"skip","Test":"TestPartialStateJoin/Purge_during_resync"} -{"Action":"fail","Test":"TestPartialStateJoin/Rejected_events_remain_rejected_after_resync"} -{"Action":"fail","Test":"TestPartialStateJoin/Rejects_make_join_during_partial_join"} -{"Action":"fail","Test":"TestPartialStateJoin/Rejects_make_knock_during_partial_join"} -{"Action":"fail","Test":"TestPartialStateJoin/Rejects_send_join_during_partial_join"} -{"Action":"fail","Test":"TestPartialStateJoin/Rejects_send_knock_during_partial_join"} -{"Action":"fail","Test":"TestPartialStateJoin/Resync_completes_even_when_events_arrive_before_their_prev_events"} -{"Action":"fail","Test":"TestPartialStateJoin/Room_aliases_can_be_added_and_deleted_during_a_resync"} -{"Action":"fail","Test":"TestPartialStateJoin/Room_aliases_can_be_added_and_queried_during_a_resync"} -{"Action":"skip","Test":"TestPartialStateJoin/Room_stats_are_correctly_updated_once_state_re-sync_completes"} -{"Action":"fail","Test":"TestPartialStateJoin/State_accepted_incorrectly"} -{"Action":"fail","Test":"TestPartialStateJoin/State_rejected_incorrectly"} -{"Action":"fail","Test":"TestPartialStateJoin/User_directory_is_correctly_updated_once_state_re-sync_completes"} -{"Action":"fail","Test":"TestPartialStateJoin/joined_members_blocks_during_partial_state_join"} {"Action":"fail","Test":"TestPollsLocalPushRules"} {"Action":"fail","Test":"TestPollsLocalPushRules/Polls_push_rules_are_correctly_presented_to_the_client"} {"Action":"pass","Test":"TestPowerLevels"} @@ -561,16 +495,13 @@ {"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Room_creation_reports_m.room.member_to_myself"} {"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Setting_room_topic_reports_m.room.topic_to_myself"} {"Action":"fail","Test":"TestRoomCreationReportsEventsToMyself/parallel/Setting_state_twice_is_idempotent"} -{"Action":"fail","Test":"TestRoomDeleteAlias"} -{"Action":"fail","Test":"TestRoomDeleteAlias/Parallel"} +{"Action":"pass","Test":"TestRoomDeleteAlias"} +{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel"} {"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Alias_creators_can_delete_alias_with_no_ops"} {"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Alias_creators_can_delete_canonical_alias_with_no_ops"} -{"Action":"fail","Test":"TestRoomDeleteAlias/Parallel/Can_delete_canonical_alias"} {"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Deleting_a_non-existent_alias_should_return_a_404"} {"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_in_the_default_room_configuration"} -{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Regular_users_can_add_and_delete_aliases_when_m.room.aliases_is_restricted"} {"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Users_can't_delete_other's_aliases"} -{"Action":"pass","Test":"TestRoomDeleteAlias/Parallel/Users_with_sufficient_power-level_can_delete_other's_aliases"} {"Action":"fail","Test":"TestRoomForget"} {"Action":"fail","Test":"TestRoomForget/Parallel"} {"Action":"pass","Test":"TestRoomForget/Parallel/Can't_forget_room_you're_still_in"} @@ -687,7 +618,6 @@ {"Action":"pass","Test":"TestTyping"} {"Action":"pass","Test":"TestTyping/Typing_can_be_explicitly_stopped"} {"Action":"pass","Test":"TestTyping/Typing_notification_sent_to_local_room_members"} -{"Action":"fail","Test":"TestUnbanViaInvite"} {"Action":"fail","Test":"TestUnknownEndpoints"} {"Action":"pass","Test":"TestUnknownEndpoints/Client-server_endpoints"} {"Action":"fail","Test":"TestUnknownEndpoints/Key_endpoints"} From 4518f554081532400bfae64b931cd135dbceb755 Mon Sep 17 00:00:00 2001 From: cy Date: Wed, 12 Mar 2025 20:46:14 -0400 Subject: [PATCH 1587/2091] guard against using someone else's access token in UIAA --- src/service/uiaa/mod.rs | 10 ++++++++-- tests/test_results/complement/test_results.jsonl | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/service/uiaa/mod.rs b/src/service/uiaa/mod.rs index 39dd2b41..7803c736 100644 --- a/src/service/uiaa/mod.rs +++ b/src/service/uiaa/mod.rs @@ -4,7 +4,7 @@ use std::{ }; use conduwuit::{ - Error, Result, err, error, implement, utils, + Err, Error, Result, err, error, implement, utils, utils::{hash, string::EMPTY}, }; use database::{Deserialized, Json, Map}; @@ -150,12 +150,18 @@ pub async fn try_auth( )); }; - let user_id = UserId::parse_with_server_name( + let user_id_from_username = UserId::parse_with_server_name( username.clone(), self.services.globals.server_name(), ) .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid."))?; + // Check if the access token being used matches the credentials used for UIAA + if user_id.localpart() != user_id_from_username.localpart() { + return Err!(Request(Forbidden("User ID and access token mismatch."))); + } + let user_id = user_id_from_username; + // Check if password is correct if let Ok(hash) = self.services.users.password_hash(&user_id).await { let hash_matches = hash::verify_password(password, &hash).is_ok(); diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 6b5f670e..01d2ca4a 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -127,7 +127,7 @@ {"Action":"fail","Test":"TestDeviceListsUpdateOverFederationOnRoomJoin"} {"Action":"fail","Test":"TestDeviceManagement"} {"Action":"fail","Test":"TestDeviceManagement/DELETE_/device/{deviceId}"} -{"Action":"fail","Test":"TestDeviceManagement/DELETE_/device/{deviceId}_requires_UI_auth_user_to_match_device_owner"} +{"Action":"pass","Test":"TestDeviceManagement/DELETE_/device/{deviceId}_requires_UI_auth_user_to_match_device_owner"} {"Action":"pass","Test":"TestDeviceManagement/GET_/device/{deviceId}"} {"Action":"pass","Test":"TestDeviceManagement/GET_/device/{deviceId}_gives_a_404_for_unknown_devices"} {"Action":"pass","Test":"TestDeviceManagement/GET_/devices"} From 658c19d55eb5fdf30f27e189c414208e2eae6e24 Mon Sep 17 00:00:00 2001 From: cy Date: Fri, 14 Mar 2025 23:01:28 -0400 Subject: [PATCH 1588/2091] check if we already have a more preferable key backup before adding --- src/api/client/backup.rs | 81 ++++++++++++++++--- .../complement/test_results.jsonl | 8 +- 2 files changed, 76 insertions(+), 13 deletions(-) diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index 714e3f86..63c47e01 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -1,3 +1,5 @@ +use std::cmp::Ordering; + use axum::extract::State; use conduwuit::{Err, err}; use ruma::{ @@ -232,16 +234,77 @@ pub(crate) async fn add_backup_keys_for_session_route( ))); } - services + // Check if we already have a better key + let mut ok_to_replace = true; + if let Some(old_key) = &services .key_backups - .add_key( - body.sender_user(), - &body.version, - &body.room_id, - &body.session_id, - &body.session_data, - ) - .await?; + .get_session(body.sender_user(), &body.version, &body.room_id, &body.session_id) + .await + .ok() + { + let old_is_verified = old_key + .get_field::("is_verified")? + .unwrap_or_default(); + + let new_is_verified = body + .session_data + .get_field::("is_verified")? + .ok_or_else(|| err!(Request(BadJson("`is_verified` field should exist"))))?; + + // Prefer key that `is_verified` + if old_is_verified != new_is_verified { + if old_is_verified { + ok_to_replace = false; + } + } else { + // If both have same `is_verified`, prefer the one with lower + // `first_message_index` + let old_first_message_index = old_key + .get_field::("first_message_index")? + .unwrap_or(UInt::MAX); + + let new_first_message_index = body + .session_data + .get_field::("first_message_index")? + .ok_or_else(|| { + err!(Request(BadJson("`first_message_index` field should exist"))) + })?; + + ok_to_replace = match new_first_message_index.cmp(&old_first_message_index) { + | Ordering::Less => true, + | Ordering::Greater => false, + | Ordering::Equal => { + // If both have same `first_message_index`, prefer the one with lower + // `forwarded_count` + let old_forwarded_count = old_key + .get_field::("forwarded_count")? + .unwrap_or(UInt::MAX); + + let new_forwarded_count = body + .session_data + .get_field::("forwarded_count")? + .ok_or_else(|| { + err!(Request(BadJson("`forwarded_count` field should exist"))) + })?; + + new_forwarded_count < old_forwarded_count + }, + }; + }; + } + + if ok_to_replace { + services + .key_backups + .add_key( + body.sender_user(), + &body.version, + &body.room_id, + &body.session_id, + &body.session_data, + ) + .await?; + } Ok(add_backup_keys_for_session::v3::Response { count: services diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 01d2ca4a..97170a5c 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -134,10 +134,10 @@ {"Action":"pass","Test":"TestDeviceManagement/PUT_/device/{deviceId}_gives_a_404_for_unknown_devices"} {"Action":"pass","Test":"TestDeviceManagement/PUT_/device/{deviceId}_updates_device_fields"} {"Action":"pass","Test":"TestDisplayNameUpdate"} -{"Action":"fail","Test":"TestE2EKeyBackupReplaceRoomKeyRules"} -{"Action":"fail","Test":"TestE2EKeyBackupReplaceRoomKeyRules/parallel"} -{"Action":"fail","Test":"TestE2EKeyBackupReplaceRoomKeyRules/parallel/{isVerified:false_firstMessageIndex:10_forwardedCount:5}"} -{"Action":"fail","Test":"TestE2EKeyBackupReplaceRoomKeyRules/parallel/{isVerified:true_firstMessageIndex:10_forwardedCount:5}"} +{"Action":"pass","Test":"TestE2EKeyBackupReplaceRoomKeyRules"} +{"Action":"pass","Test":"TestE2EKeyBackupReplaceRoomKeyRules/parallel"} +{"Action":"pass","Test":"TestE2EKeyBackupReplaceRoomKeyRules/parallel/{isVerified:false_firstMessageIndex:10_forwardedCount:5}"} +{"Action":"pass","Test":"TestE2EKeyBackupReplaceRoomKeyRules/parallel/{isVerified:true_firstMessageIndex:10_forwardedCount:5}"} {"Action":"pass","Test":"TestEvent"} {"Action":"pass","Test":"TestEvent/Parallel"} {"Action":"pass","Test":"TestEvent/Parallel/Large_Event"} From 7bf92c8a3710eeff229bd86bc81a89daa94b66d5 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Mon, 17 Mar 2025 22:50:29 -0400 Subject: [PATCH 1589/2091] replace unnecessary check when updating device keys Signed-off-by: June Clementine Strawberry --- src/api/client/backup.rs | 2 +- src/api/client/keys.rs | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index 63c47e01..83955fea 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -290,7 +290,7 @@ pub(crate) async fn add_backup_keys_for_session_route( new_forwarded_count < old_forwarded_count }, }; - }; + } } if ok_to_replace { diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 9cd50e85..f50d7afa 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -80,14 +80,26 @@ pub(crate) async fn upload_keys_route( ))); } - // TODO: merge this and the existing event? - // This check is needed to assure that signatures are kept - if services + if let Ok(existing_keys) = services .users .get_device_keys(sender_user, sender_device) .await - .is_err() { + if existing_keys.json().get() == device_keys.json().get() { + debug!( + ?sender_user, + ?sender_device, + ?device_keys, + "Ignoring user uploaded keys as they are an exact copy already in the \ + database" + ); + } else { + services + .users + .add_device_keys(sender_user, sender_device, device_keys) + .await; + } + } else { services .users .add_device_keys(sender_user, sender_device, device_keys) From 33c5afe050491988ee8224af25b9b06e892f4b50 Mon Sep 17 00:00:00 2001 From: cy Date: Wed, 19 Mar 2025 20:55:14 -0400 Subject: [PATCH 1590/2091] delete pushers created with different access token on password change --- src/api/client/account.rs | 23 ++++++++++++- src/api/client/push.rs | 2 +- src/database/maps.rs | 4 +++ src/service/pusher/mod.rs | 34 +++++++++++++------ .../complement/test_results.jsonl | 4 +-- 5 files changed, 53 insertions(+), 14 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 32438098..5dd622d7 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -4,7 +4,8 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ Err, Error, PduBuilder, Result, debug_info, err, error, info, is_equal_to, utils, - utils::ReadyExt, warn, + utils::{ReadyExt, stream::BroadbandExt}, + warn, }; use futures::{FutureExt, StreamExt}; use register::RegistrationKind; @@ -627,6 +628,26 @@ pub(crate) async fn change_password_route( .ready_filter(|id| *id != sender_device) .for_each(|id| services.users.remove_device(sender_user, id)) .await; + + // Remove all pushers except the ones associated with this session + services + .pusher + .get_pushkeys(sender_user) + .map(ToOwned::to_owned) + .broad_filter_map(|pushkey| async move { + services + .pusher + .get_pusher_device(&pushkey) + .await + .ok() + .filter(|pusher_device| pusher_device != sender_device) + .is_some() + .then_some(pushkey) + }) + .for_each(|pushkey| async move { + services.pusher.delete_pusher(sender_user, &pushkey).await; + }) + .await; } info!("User {sender_user} changed their password."); diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 384b9dbc..cc1d3be2 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -503,7 +503,7 @@ pub(crate) async fn set_pushers_route( services .pusher - .set_pusher(sender_user, &body.action) + .set_pusher(sender_user, body.sender_device(), &body.action) .await?; Ok(set_pusher::v3::Response::new()) diff --git a/src/database/maps.rs b/src/database/maps.rs index 138bb038..1da9acc0 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -219,6 +219,10 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "senderkey_pusher", ..descriptor::RANDOM_SMALL }, + Descriptor { + name: "pushkey_deviceid", + ..descriptor::RANDOM_SMALL + }, Descriptor { name: "server_signingkeys", ..descriptor::RANDOM diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 2b269b3d..27490fb8 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -10,7 +10,7 @@ use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::{Stream, StreamExt}; use ipaddress::IPAddress; use ruma::{ - RoomId, UInt, UserId, + DeviceId, OwnedDeviceId, RoomId, UInt, UserId, api::{ IncomingResponse, MatrixVersion, OutgoingRequest, SendAccessToken, client::push::{Pusher, PusherKind, set_pusher}, @@ -48,6 +48,7 @@ struct Services { struct Data { senderkey_pusher: Arc, + pushkey_deviceid: Arc, } impl crate::Service for Service { @@ -55,6 +56,7 @@ impl crate::Service for Service { Ok(Arc::new(Self { db: Data { senderkey_pusher: args.db["senderkey_pusher"].clone(), + pushkey_deviceid: args.db["pushkey_deviceid"].clone(), }, services: Services { globals: args.depend::("globals"), @@ -75,6 +77,7 @@ impl Service { pub async fn set_pusher( &self, sender: &UserId, + sender_device: &DeviceId, pusher: &set_pusher::v3::PusherAction, ) -> Result { match pusher { @@ -123,24 +126,35 @@ impl Service { } } - let key = (sender, data.pusher.ids.pushkey.as_str()); + let pushkey = data.pusher.ids.pushkey.as_str(); + let key = (sender, pushkey); self.db.senderkey_pusher.put(key, Json(pusher)); + self.db.pushkey_deviceid.insert(pushkey, sender_device); }, | set_pusher::v3::PusherAction::Delete(ids) => { - let key = (sender, ids.pushkey.as_str()); - self.db.senderkey_pusher.del(key); - - self.services - .sending - .cleanup_events(None, Some(sender), Some(ids.pushkey.as_str())) - .await - .ok(); + self.delete_pusher(sender, ids.pushkey.as_str()).await; }, } Ok(()) } + pub async fn delete_pusher(&self, sender: &UserId, pushkey: &str) { + let key = (sender, pushkey); + self.db.senderkey_pusher.del(key); + self.db.pushkey_deviceid.remove(pushkey); + + self.services + .sending + .cleanup_events(None, Some(sender), Some(pushkey)) + .await + .ok(); + } + + pub async fn get_pusher_device(&self, pushkey: &str) -> Result { + self.db.pushkey_deviceid.get(pushkey).await.deserialized() + } + pub async fn get_pusher(&self, sender: &UserId, pushkey: &str) -> Result { let senderkey = (sender, pushkey); self.db diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index 97170a5c..ac2733f8 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -69,8 +69,8 @@ {"Action":"pass","Test":"TestChangePassword/After_changing_password,_can_log_in_with_new_password"} {"Action":"pass","Test":"TestChangePassword/After_changing_password,_different_sessions_can_optionally_be_kept"} {"Action":"pass","Test":"TestChangePassword/After_changing_password,_existing_session_still_works"} -{"Action":"fail","Test":"TestChangePasswordPushers"} -{"Action":"fail","Test":"TestChangePasswordPushers/Pushers_created_with_a_different_access_token_are_deleted_on_password_change"} +{"Action":"pass","Test":"TestChangePasswordPushers"} +{"Action":"pass","Test":"TestChangePasswordPushers/Pushers_created_with_a_different_access_token_are_deleted_on_password_change"} {"Action":"pass","Test":"TestChangePasswordPushers/Pushers_created_with_the_same_access_token_are_not_deleted_on_password_change"} {"Action":"fail","Test":"TestClientSpacesSummary"} {"Action":"fail","Test":"TestClientSpacesSummary/max_depth"} From 07ec9d6d852a8ebb623c96b580af36e0d0d11697 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 26 Mar 2025 01:32:45 +0000 Subject: [PATCH 1591/2091] re-sort pushkey_deviceid (33c5afe050) Signed-off-by: Jason Volk --- src/database/maps.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/database/maps.rs b/src/database/maps.rs index 1da9acc0..311c629f 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -121,14 +121,18 @@ pub(super) static MAPS: &[Descriptor] = &[ index_size: 512, ..descriptor::SEQUENTIAL }, - Descriptor { - name: "presenceid_presence", - ..descriptor::SEQUENTIAL_SMALL - }, Descriptor { name: "publicroomids", ..descriptor::RANDOM_SMALL }, + Descriptor { + name: "pushkey_deviceid", + ..descriptor::RANDOM_SMALL + }, + Descriptor { + name: "presenceid_presence", + ..descriptor::SEQUENTIAL_SMALL + }, Descriptor { name: "readreceiptid_readreceipt", ..descriptor::RANDOM @@ -219,10 +223,6 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "senderkey_pusher", ..descriptor::RANDOM_SMALL }, - Descriptor { - name: "pushkey_deviceid", - ..descriptor::RANDOM_SMALL - }, Descriptor { name: "server_signingkeys", ..descriptor::RANDOM From aa4d2e236330693c61d5cb116b4c438b15431aec Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 9 Mar 2025 03:14:00 +0000 Subject: [PATCH 1592/2091] fix unused import without feature jemalloc_conf fix span passed by value Signed-off-by: Jason Volk --- src/core/alloc/je.rs | 3 +-- src/router/request.rs | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 6870c1c0..51caf3a3 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -8,7 +8,6 @@ use std::{ }; use arrayvec::ArrayVec; -use const_str::concat_bytes; use tikv_jemalloc_ctl as mallctl; use tikv_jemalloc_sys as ffi; use tikv_jemallocator as jemalloc; @@ -20,7 +19,7 @@ use crate::{ #[cfg(feature = "jemalloc_conf")] #[unsafe(no_mangle)] -pub static malloc_conf: &[u8] = concat_bytes!( +pub static malloc_conf: &[u8] = const_str::concat_bytes!( "lg_extent_max_active_fit:4", ",oversize_threshold:16777216", ",tcache_max:2097152", diff --git a/src/router/request.rs b/src/router/request.rs index 00769b3f..dba90324 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -37,7 +37,7 @@ pub(crate) async fn handle( let parent = Span::current(); let task = services.server.runtime().spawn(async move { tokio::select! { - response = execute(&services_, req, next, parent) => response, + response = execute(&services_, req, next, &parent) => response, response = services_.server.until_shutdown() .then(|()| { let timeout = services_.server.config.client_shutdown_timeout; @@ -79,7 +79,7 @@ async fn execute( services: &Arc, req: http::Request, next: axum::middleware::Next, - parent: Span, + parent: &Span, ) -> Response { #[cfg(debug_assertions)] conduwuit::defer! {{ From 7294368015025ae4d7677c28837d3ac0a79539e6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 12 Mar 2025 23:10:38 +0000 Subject: [PATCH 1593/2091] parallelize IO for PublicRoomsChunk vector Signed-off-by: Jason Volk --- src/api/client/directory.rs | 118 +++++++++++++++++++----------------- 1 file changed, 64 insertions(+), 54 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 7ce32e4c..80b314b9 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,7 +1,17 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{Err, Error, Result, info, warn}; -use futures::{StreamExt, TryFutureExt}; +use conduwuit::{ + Err, Error, Result, info, + utils::{ + TryFutureExtExt, + stream::{ReadyExt, WidebandExt}, + }, + warn, +}; +use futures::{ + FutureExt, StreamExt, TryFutureExt, + future::{join, join4, join5}, +}; use ruma::{ OwnedRoomId, RoomId, ServerName, UInt, UserId, api::{ @@ -287,8 +297,8 @@ pub(crate) async fn get_public_rooms_filtered_helper( .directory .public_rooms() .map(ToOwned::to_owned) - .then(|room_id| public_rooms_chunk(services, room_id)) - .filter_map(|chunk| async move { + .wide_then(|room_id| public_rooms_chunk(services, room_id)) + .ready_filter_map(|chunk| { if !filter.room_types.is_empty() && !filter.room_types.contains(&RoomTypeFilter::from(chunk.room_type.clone())) { return None; } @@ -394,60 +404,60 @@ async fn user_can_publish_room( } async fn public_rooms_chunk(services: &Services, room_id: OwnedRoomId) -> PublicRoomsChunk { + let name = services.rooms.state_accessor.get_name(&room_id).ok(); + + let room_type = services.rooms.state_accessor.get_room_type(&room_id).ok(); + + let canonical_alias = services + .rooms + .state_accessor + .get_canonical_alias(&room_id) + .ok(); + + let avatar_url = services.rooms.state_accessor.get_avatar(&room_id); + + let topic = services.rooms.state_accessor.get_room_topic(&room_id).ok(); + + let world_readable = services.rooms.state_accessor.is_world_readable(&room_id); + + let join_rule = services + .rooms + .state_accessor + .room_state_get_content(&room_id, &StateEventType::RoomJoinRules, "") + .map_ok(|c: RoomJoinRulesEventContent| match c.join_rule { + | JoinRule::Public => PublicRoomJoinRule::Public, + | JoinRule::Knock => "knock".into(), + | JoinRule::KnockRestricted(_) => "knock_restricted".into(), + | _ => "invite".into(), + }); + + let guest_can_join = services.rooms.state_accessor.guest_can_join(&room_id); + + let num_joined_members = services.rooms.state_cache.room_joined_count(&room_id); + + let ( + (avatar_url, canonical_alias, guest_can_join, join_rule, name), + (num_joined_members, room_type, topic, world_readable), + ) = join( + join5(avatar_url, canonical_alias, guest_can_join, join_rule, name), + join4(num_joined_members, room_type, topic, world_readable), + ) + .boxed() + .await; + PublicRoomsChunk { - canonical_alias: services - .rooms - .state_accessor - .get_canonical_alias(&room_id) - .await - .ok(), - name: services.rooms.state_accessor.get_name(&room_id).await.ok(), - num_joined_members: services - .rooms - .state_cache - .room_joined_count(&room_id) - .await + avatar_url: avatar_url.into_option().unwrap_or_default().url, + canonical_alias, + guest_can_join, + join_rule: join_rule.unwrap_or_default(), + name, + num_joined_members: num_joined_members .unwrap_or(0) .try_into() .expect("joined count overflows ruma UInt"), - topic: services - .rooms - .state_accessor - .get_room_topic(&room_id) - .await - .ok(), - world_readable: services - .rooms - .state_accessor - .is_world_readable(&room_id) - .await, - guest_can_join: services.rooms.state_accessor.guest_can_join(&room_id).await, - avatar_url: services - .rooms - .state_accessor - .get_avatar(&room_id) - .await - .into_option() - .unwrap_or_default() - .url, - join_rule: services - .rooms - .state_accessor - .room_state_get_content(&room_id, &StateEventType::RoomJoinRules, "") - .map_ok(|c: RoomJoinRulesEventContent| match c.join_rule { - | JoinRule::Public => PublicRoomJoinRule::Public, - | JoinRule::Knock => "knock".into(), - | JoinRule::KnockRestricted(_) => "knock_restricted".into(), - | _ => "invite".into(), - }) - .await - .unwrap_or_default(), - room_type: services - .rooms - .state_accessor - .get_room_type(&room_id) - .await - .ok(), room_id, + room_type, + topic, + world_readable, } } From a57336ec1388ab26a692cf26768474bc3069df75 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 14 Mar 2025 06:54:08 +0000 Subject: [PATCH 1594/2091] assume canonical order in db serialization test Signed-off-by: Jason Volk --- src/database/tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/tests.rs b/src/database/tests.rs index 140bc56d..1446a1fc 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -152,8 +152,8 @@ fn ser_json_macro() { let content = serde_json::to_value(content).expect("failed to serialize content"); let sender: &UserId = "@foo:example.com".try_into().unwrap(); let serialized = serialize_to_vec(Json(json!({ - "sender": sender, "content": content, + "sender": sender, }))) .expect("failed to serialize value"); From 17003ba773228055de107f9d8baf1b2848d86c1f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 15 Mar 2025 01:23:54 +0000 Subject: [PATCH 1595/2091] add FIFO compaction for persistent-cache descriptor; comments/cleanup Signed-off-by: Jason Volk --- src/database/engine/cf_opts.rs | 14 ++++++++--- src/database/engine/descriptor.rs | 39 ++++++++++++++++++++++++------- src/database/engine/open.rs | 6 ++--- src/database/maps.rs | 4 ++-- 4 files changed, 46 insertions(+), 17 deletions(-) diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 5ddb9473..7ceec722 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -1,8 +1,8 @@ use conduwuit::{Config, Result, err, utils::math::Expected}; use rocksdb::{ BlockBasedIndexType, BlockBasedOptions, BlockBasedPinningTier, Cache, - DBCompressionType as CompressionType, DataBlockIndexType, LruCacheOptions, Options, - UniversalCompactOptions, UniversalCompactionStopStyle, + DBCompressionType as CompressionType, DataBlockIndexType, FifoCompactOptions, + LruCacheOptions, Options, UniversalCompactOptions, UniversalCompactionStopStyle, }; use super::descriptor::{CacheDisp, Descriptor}; @@ -16,7 +16,7 @@ pub(super) const SENTINEL_COMPRESSION_LEVEL: i32 = 32767; pub(crate) fn cf_options(ctx: &Context, opts: Options, desc: &Descriptor) -> Result { let cache = get_cache(ctx, desc); let config = &ctx.server.config; - descriptor_cf_options(opts, desc.clone(), config, cache.as_ref()) + descriptor_cf_options(opts, *desc, config, cache.as_ref()) } fn descriptor_cf_options( @@ -46,6 +46,7 @@ fn descriptor_cf_options( opts.set_compaction_style(desc.compaction); opts.set_compaction_pri(desc.compaction_pri); opts.set_universal_compaction_options(&uc_options(&desc)); + opts.set_fifo_compaction_options(&fifo_options(&desc)); let compression_shape: Vec<_> = desc .compression_shape @@ -142,6 +143,13 @@ fn set_compression(desc: &mut Descriptor, config: &Config) { } } +fn fifo_options(desc: &Descriptor) -> FifoCompactOptions { + let mut opts = FifoCompactOptions::default(); + opts.set_max_table_files_size(desc.limit_size); + + opts +} + fn uc_options(desc: &Descriptor) -> UniversalCompactOptions { let mut opts = UniversalCompactOptions::default(); opts.set_stop_style(UniversalCompactionStopStyle::Total); diff --git a/src/database/engine/descriptor.rs b/src/database/engine/descriptor.rs index 816555d2..2274da9c 100644 --- a/src/database/engine/descriptor.rs +++ b/src/database/engine/descriptor.rs @@ -6,14 +6,8 @@ use rocksdb::{ use super::cf_opts::SENTINEL_COMPRESSION_LEVEL; +/// Column Descriptor #[derive(Debug, Clone, Copy)] -pub(crate) enum CacheDisp { - Unique, - Shared, - SharedWith(&'static str), -} - -#[derive(Debug, Clone)] pub(crate) struct Descriptor { pub(crate) name: &'static str, pub(crate) dropped: bool, @@ -30,6 +24,7 @@ pub(crate) struct Descriptor { pub(crate) file_shape: i32, pub(crate) level0_width: i32, pub(crate) merge_width: (i32, i32), + pub(crate) limit_size: u64, pub(crate) ttl: u64, pub(crate) compaction: CompactionStyle, pub(crate) compaction_pri: CompactionPri, @@ -46,7 +41,16 @@ pub(crate) struct Descriptor { pub(crate) auto_readahead_max: usize, } -pub(crate) static BASE: Descriptor = Descriptor { +/// Cache Disposition +#[derive(Debug, Clone, Copy)] +pub(crate) enum CacheDisp { + Unique, + Shared, + SharedWith(&'static str), +} + +/// Base descriptor supplying common defaults to all derived descriptors. +static BASE: Descriptor = Descriptor { name: EMPTY, dropped: false, cache_disp: CacheDisp::Shared, @@ -62,6 +66,7 @@ pub(crate) static BASE: Descriptor = Descriptor { file_shape: 2, level0_width: 2, merge_width: (2, 16), + limit_size: 0, ttl: 60 * 60 * 24 * 21, compaction: CompactionStyle::Level, compaction_pri: CompactionPri::MinOverlappingRatio, @@ -78,6 +83,10 @@ pub(crate) static BASE: Descriptor = Descriptor { auto_readahead_max: 1024 * 1024 * 2, }; +/// Tombstone descriptor for columns which have been or will be deleted. +pub(crate) static DROPPED: Descriptor = Descriptor { dropped: true, ..BASE }; + +/// Descriptor for large datasets with random updates across the keyspace. pub(crate) static RANDOM: Descriptor = Descriptor { compaction_pri: CompactionPri::OldestSmallestSeqFirst, write_size: 1024 * 1024 * 32, @@ -88,6 +97,7 @@ pub(crate) static RANDOM: Descriptor = Descriptor { ..BASE }; +/// Descriptor for large datasets with updates to the end of the keyspace. pub(crate) static SEQUENTIAL: Descriptor = Descriptor { compaction_pri: CompactionPri::OldestLargestSeqFirst, write_size: 1024 * 1024 * 64, @@ -101,6 +111,7 @@ pub(crate) static SEQUENTIAL: Descriptor = Descriptor { ..BASE }; +/// Descriptor for small datasets with random updates across the keyspace. pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { compaction: CompactionStyle::Universal, write_size: 1024 * 1024 * 16, @@ -117,6 +128,7 @@ pub(crate) static RANDOM_SMALL: Descriptor = Descriptor { ..RANDOM }; +/// Descriptor for small datasets with updates to the end of the keyspace. pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { compaction: CompactionStyle::Universal, write_size: 1024 * 1024 * 16, @@ -132,3 +144,14 @@ pub(crate) static SEQUENTIAL_SMALL: Descriptor = Descriptor { compressed_index: false, ..SEQUENTIAL }; + +/// Descriptor for small persistent caches with random updates. Oldest entries +/// are deleted after limit_size reached. +pub(crate) static RANDOM_SMALL_CACHE: Descriptor = Descriptor { + compaction: CompactionStyle::Fifo, + cache_disp: CacheDisp::Unique, + limit_size: 1024 * 1024 * 64, + ttl: 60 * 60 * 24 * 14, + file_shape: 2, + ..RANDOM_SMALL +}; diff --git a/src/database/engine/open.rs b/src/database/engine/open.rs index 24010c3a..84e59a6a 100644 --- a/src/database/engine/open.rs +++ b/src/database/engine/open.rs @@ -101,13 +101,11 @@ fn configure_cfds( debug!("Creating new column {name:?} not previously found in existing database."); }); - let missing_descriptors = missing - .clone() - .map(|_| Descriptor { dropped: true, ..descriptor::BASE }); + let missing_descriptors = missing.clone().map(|_| descriptor::DROPPED); let cfopts: Vec<_> = desc .iter() - .cloned() + .copied() .chain(missing_descriptors) .map(|ref desc| cf_options(ctx, db_opts.clone(), desc)) .collect::>()?; diff --git a/src/database/maps.rs b/src/database/maps.rs index 311c629f..19f9ced4 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -233,7 +233,7 @@ pub(super) static MAPS: &[Descriptor] = &[ }, Descriptor { name: "servername_destination", - ..descriptor::RANDOM_SMALL + ..descriptor::RANDOM_SMALL_CACHE }, Descriptor { name: "servername_educount", @@ -241,7 +241,7 @@ pub(super) static MAPS: &[Descriptor] = &[ }, Descriptor { name: "servername_override", - ..descriptor::RANDOM_SMALL + ..descriptor::RANDOM_SMALL_CACHE }, Descriptor { name: "servernameevent_data", From d8ea8b378cf2ee9ff7644fdb6c5a33d05923a51d Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 15 Mar 2025 02:35:10 +0000 Subject: [PATCH 1596/2091] add Map::clear() to db interface Signed-off-by: Jason Volk --- src/database/map.rs | 1 + src/database/map/clear.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 src/database/map/clear.rs diff --git a/src/database/map.rs b/src/database/map.rs index c5a908ba..ed38e1fc 100644 --- a/src/database/map.rs +++ b/src/database/map.rs @@ -1,3 +1,4 @@ +mod clear; pub mod compact; mod contains; mod count; diff --git a/src/database/map/clear.rs b/src/database/map/clear.rs new file mode 100644 index 00000000..321ec79c --- /dev/null +++ b/src/database/map/clear.rs @@ -0,0 +1,30 @@ +use std::sync::Arc; + +use conduwuit::{ + Result, implement, + utils::stream::{ReadyExt, TryIgnore}, +}; +use futures::{Stream, TryStreamExt}; + +use crate::keyval::Key; + +/// Delete all data stored in this map. !!! USE WITH CAUTION !!! +/// +/// See for_clear() with additional details. +#[implement(super::Map)] +#[tracing::instrument(level = "trace")] +pub async fn clear(self: &Arc) { + self.for_clear().ignore_err().ready_for_each(|_| ()).await; +} + +/// Delete all data stored in this map. !!! USE WITH CAUTION !!! +/// +/// Provides stream of keys undergoing deletion along with any errors. +/// +/// Note this operation applies to a snapshot of the data when invoked. +/// Additional data written during or after this call may be missed. +#[implement(super::Map)] +#[tracing::instrument(level = "trace")] +pub fn for_clear(self: &Arc) -> impl Stream>> + Send { + self.raw_keys().inspect_ok(|key| self.remove(key)) +} From 9ce95a703038e8603da62f15516f205ca70ad962 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 15 Mar 2025 04:07:53 +0000 Subject: [PATCH 1597/2091] make service memory_usage()/clear_cache() async trait Signed-off-by: Jason Volk --- src/service/globals/mod.rs | 6 ++- src/service/rooms/event_handler/mod.rs | 4 +- src/service/rooms/spaces/mod.rs | 14 +++++- src/service/rooms/state/mod.rs | 4 +- src/service/rooms/state_accessor/mod.rs | 6 ++- src/service/rooms/state_compressor/mod.rs | 6 ++- src/service/rooms/timeline/mod.rs | 4 +- src/service/service.rs | 4 +- src/service/services.rs | 57 ++++++++++------------- 9 files changed, 61 insertions(+), 44 deletions(-) diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 74f83228..1dd7db8e 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -7,6 +7,7 @@ use std::{ time::Instant, }; +use async_trait::async_trait; use conduwuit::{Result, Server, error, utils::bytes::pretty}; use data::Data; use regex::RegexSet; @@ -27,6 +28,7 @@ pub struct Service { type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let db = Data::new(&args); @@ -73,7 +75,7 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result { + async fn memory_usage(&self, out: &mut (dyn Write + Send)) -> Result { let (ber_count, ber_bytes) = self.bad_event_ratelimiter.read()?.iter().fold( (0_usize, 0_usize), |(mut count, mut bytes), (event_id, _)| { @@ -89,7 +91,7 @@ impl crate::Service for Service { Ok(()) } - fn clear_cache(&self) { + async fn clear_cache(&self) { self.bad_event_ratelimiter .write() .expect("locked for writing") diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index e9e79ce4..4944f3ec 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -17,6 +17,7 @@ use std::{ time::Instant, }; +use async_trait::async_trait; use conduwuit::{ Err, PduEvent, Result, RoomVersion, Server, utils::{MutexMap, TryFutureExtExt}, @@ -54,6 +55,7 @@ struct Services { type RoomMutexMap = MutexMap; type HandleTimeMap = HashMap; +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -79,7 +81,7 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + async fn memory_usage(&self, out: &mut (dyn Write + Send)) -> Result { let mutex_federation = self.mutex_federation.len(); writeln!(out, "federation_mutex: {mutex_federation}")?; diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 1da38234..55897f9c 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -2,8 +2,9 @@ mod pagination_token; #[cfg(test)] mod tests; -use std::sync::Arc; +use std::{fmt::Write, sync::Arc}; +use async_trait::async_trait; use conduwuit::{ Err, Error, Result, implement, utils::{ @@ -70,6 +71,7 @@ pub enum Identifier<'a> { type Cache = LruCache>; +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; @@ -90,6 +92,16 @@ impl crate::Service for Service { })) } + async fn memory_usage(&self, out: &mut (dyn Write + Send)) -> Result { + let roomid_spacehierarchy_cache = self.roomid_spacehierarchy_cache.lock().await.len(); + + writeln!(out, "roomid_spacehierarchy_cache: {roomid_spacehierarchy_cache}")?; + + Ok(()) + } + + async fn clear_cache(&self) { self.roomid_spacehierarchy_cache.lock().await.clear(); } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 8683a3be..56955497 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, fmt::Write, iter::once, sync::Arc}; +use async_trait::async_trait; use conduwuit::{ PduEvent, Result, err, result::FlatOk, @@ -56,6 +57,7 @@ struct Data { type RoomMutexMap = MutexMap; pub type RoomMutexGuard = MutexMapGuard; +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -79,7 +81,7 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result { + async fn memory_usage(&self, out: &mut (dyn Write + Send)) -> Result { let mutex = self.mutex.len(); writeln!(out, "state_mutex: {mutex}")?; diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 7004e35a..652fdbd7 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -8,6 +8,7 @@ use std::{ sync::{Arc, Mutex as StdMutex, Mutex}, }; +use async_trait::async_trait; use conduwuit::{ Result, err, utils, utils::math::{Expected, usize_from_f64}, @@ -57,6 +58,7 @@ struct Data { shorteventid_shortstatehash: Arc, } +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; @@ -86,7 +88,7 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result { + async fn memory_usage(&self, out: &mut (dyn Write + Send)) -> Result { use utils::bytes::pretty; let (svc_count, svc_bytes) = self.server_visibility_cache.lock()?.iter().fold( @@ -119,7 +121,7 @@ impl crate::Service for Service { Ok(()) } - fn clear_cache(&self) { + async fn clear_cache(&self) { self.server_visibility_cache.lock().expect("locked").clear(); self.user_visibility_cache.lock().expect("locked").clear(); } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 305d3187..56a91d0e 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -5,6 +5,7 @@ use std::{ sync::{Arc, Mutex}, }; +use async_trait::async_trait; use conduwuit::{ Result, arrayvec::ArrayVec, @@ -65,6 +66,7 @@ type ParentStatesVec = Vec; pub type CompressedState = BTreeSet; pub type CompressedStateEvent = [u8; 2 * size_of::()]; +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { let config = &args.server.config; @@ -82,7 +84,7 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result { + async fn memory_usage(&self, out: &mut (dyn Write + Send)) -> Result { let (cache_len, ents) = { let cache = self.stateinfo_cache.lock().expect("locked"); let ents = cache.iter().map(at!(1)).flat_map(|vec| vec.iter()).fold( @@ -108,7 +110,7 @@ impl crate::Service for Service { Ok(()) } - fn clear_cache(&self) { self.stateinfo_cache.lock().expect("locked").clear(); } + async fn clear_cache(&self) { self.stateinfo_cache.lock().expect("locked").clear(); } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 826a1dae..dc359d22 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -9,6 +9,7 @@ use std::{ sync::Arc, }; +use async_trait::async_trait; use conduwuit::{ Err, Error, Result, Server, at, debug, debug_warn, err, error, implement, info, pdu::{EventHash, PduBuilder, PduCount, PduEvent, gen_event_id}, @@ -109,6 +110,7 @@ struct Services { type RoomMutexMap = MutexMap; pub type RoomMutexGuard = MutexMapGuard; +#[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { @@ -142,7 +144,7 @@ impl crate::Service for Service { })) } - fn memory_usage(&self, out: &mut dyn Write) -> Result<()> { + async fn memory_usage(&self, out: &mut (dyn Write + Send)) -> Result { let mutex_insert = self.mutex_insert.len(); writeln!(out, "insert_mutex: {mutex_insert}")?; diff --git a/src/service/service.rs b/src/service/service.rs index 2907a562..574efd8f 100644 --- a/src/service/service.rs +++ b/src/service/service.rs @@ -31,10 +31,10 @@ pub(crate) trait Service: Any + Send + Sync { fn interrupt(&self) {} /// Clear any caches or similar runtime state. - fn clear_cache(&self) {} + async fn clear_cache(&self) {} /// Memory usage report in a markdown string. - fn memory_usage(&self, _out: &mut dyn Write) -> Result<()> { Ok(()) } + async fn memory_usage(&self, _out: &mut (dyn Write + Send)) -> Result { Ok(()) } /// Return the name of the service. /// i.e. `crate::service::make_name(std::module_path!())` diff --git a/src/service/services.rs b/src/service/services.rs index 269a1f87..dc390054 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -1,12 +1,12 @@ use std::{ any::Any, collections::BTreeMap, - fmt::Write, sync::{Arc, RwLock}, }; -use conduwuit::{Result, Server, debug, debug_info, info, trace}; +use conduwuit::{Result, Server, debug, debug_info, info, trace, utils::stream::IterStream}; use database::Database; +use futures::{Stream, StreamExt, TryStreamExt}; use tokio::sync::Mutex; use crate::{ @@ -171,40 +171,21 @@ impl Services { } pub async fn clear_cache(&self) { - for (service, ..) in self.service.read().expect("locked for reading").values() { - if let Some(service) = service.upgrade() { - service.clear_cache(); - } - } - - //TODO - self.rooms - .spaces - .roomid_spacehierarchy_cache - .lock() - .await - .clear(); + self.services() + .for_each(|service| async move { + service.clear_cache().await; + }) + .await; } pub async fn memory_usage(&self) -> Result { - let mut out = String::new(); - for (service, ..) in self.service.read().expect("locked for reading").values() { - if let Some(service) = service.upgrade() { - service.memory_usage(&mut out)?; - } - } - - //TODO - let roomid_spacehierarchy_cache = self - .rooms - .spaces - .roomid_spacehierarchy_cache - .lock() + self.services() + .map(Ok) + .try_fold(String::new(), |mut out, service| async move { + service.memory_usage(&mut out).await?; + Ok(out) + }) .await - .len(); - writeln!(out, "roomid_spacehierarchy_cache: {roomid_spacehierarchy_cache}")?; - - Ok(out) } fn interrupt(&self) { @@ -217,6 +198,18 @@ impl Services { } } + /// Iterate from snapshot of the services map + fn services(&self) -> impl Stream> + Send { + self.service + .read() + .expect("locked for reading") + .values() + .filter_map(|val| val.0.upgrade()) + .collect::>() + .into_iter() + .stream() + } + #[inline] pub fn try_get(&self, name: &str) -> Result> where From 8010505853c1c0a78254b0fd31e83d90baff7af3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 15 Mar 2025 04:08:57 +0000 Subject: [PATCH 1598/2091] implement clear_cache() for resolver service Signed-off-by: Jason Volk --- src/service/resolver/cache.rs | 17 ++++++++++++++++- src/service/resolver/dns.rs | 4 ++++ src/service/resolver/mod.rs | 7 +++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/service/resolver/cache.rs b/src/service/resolver/cache.rs index 6b05c00c..cfea7187 100644 --- a/src/service/resolver/cache.rs +++ b/src/service/resolver/cache.rs @@ -7,7 +7,7 @@ use conduwuit::{ utils::{math::Expected, rand, stream::TryIgnore}, }; use database::{Cbor, Deserialized, Map}; -use futures::{Stream, StreamExt}; +use futures::{Stream, StreamExt, future::join}; use ruma::ServerName; use serde::{Deserialize, Serialize}; @@ -45,6 +45,21 @@ impl Cache { } } +#[implement(Cache)] +pub async fn clear(&self) { join(self.clear_destinations(), self.clear_overrides()).await; } + +#[implement(Cache)] +pub async fn clear_destinations(&self) { self.destinations.clear().await; } + +#[implement(Cache)] +pub async fn clear_overrides(&self) { self.overrides.clear().await; } + +#[implement(Cache)] +pub fn del_destination(&self, name: &ServerName) { self.destinations.remove(name); } + +#[implement(Cache)] +pub fn del_override(&self, name: &ServerName) { self.overrides.remove(name); } + #[implement(Cache)] pub fn set_destination(&self, name: &ServerName, dest: &CachedDest) { self.destinations.raw_put(name, Cbor(dest)); diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index 98ad7e60..e4245a5b 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -78,6 +78,10 @@ impl Resolver { server: server.clone(), })) } + + /// Clear the in-memory hickory-dns caches + #[inline] + pub fn clear_cache(&self) { self.resolver.clear_cache(); } } impl Resolve for Resolver { diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 2ec9c0ef..246d6bc1 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -6,6 +6,7 @@ mod tests; use std::sync::Arc; +use async_trait::async_trait; use conduwuit::{Result, Server, arrayvec::ArrayString, utils::MutexMap}; use self::{cache::Cache, dns::Resolver}; @@ -26,6 +27,7 @@ struct Services { type Resolving = MutexMap; type NameBuf = ArrayString<256>; +#[async_trait] impl crate::Service for Service { #[allow(clippy::as_conversions, clippy::cast_sign_loss, clippy::cast_possible_truncation)] fn build(args: crate::Args<'_>) -> Result> { @@ -41,5 +43,10 @@ impl crate::Service for Service { })) } + async fn clear_cache(&self) { + self.resolver.clear_cache(); + self.cache.clear().await; + } + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } From 23e3f6526fd0318525a4cd1fe065dcf7f1d56935 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 19 Mar 2025 03:49:12 +0000 Subject: [PATCH 1599/2091] split well_known resolver into unit Signed-off-by: Jason Volk --- src/service/resolver/actual.rs | 55 +++--------------------------- src/service/resolver/mod.rs | 2 ++ src/service/resolver/tests.rs | 2 -- src/service/resolver/well_known.rs | 49 ++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 53 deletions(-) create mode 100644 src/service/resolver/well_known.rs diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index b037cf77..1ad76f66 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -3,7 +3,7 @@ use std::{ net::{IpAddr, SocketAddr}, }; -use conduwuit::{Err, Result, debug, debug_error, debug_info, debug_warn, err, error, trace}; +use conduwuit::{Err, Result, debug, debug_info, err, error, trace}; use futures::{FutureExt, TryFutureExt}; use hickory_resolver::error::ResolveError; use ipaddress::IPAddress; @@ -72,6 +72,9 @@ impl super::Service { if let Some(pos) = dest.as_str().find(':') { self.actual_dest_2(dest, cache, pos).await? } else { + self.conditional_query_and_cache(dest.as_str(), 8448, true) + .await?; + self.services.server.check_running()?; match self.request_well_known(dest.as_str()).await? { | Some(delegated) => self.actual_dest_3(&mut host, cache, delegated).await?, @@ -243,56 +246,6 @@ impl super::Service { Ok(add_port_to_hostname(dest.as_str())) } - #[tracing::instrument(name = "well-known", level = "debug", skip(self, dest))] - async fn request_well_known(&self, dest: &str) -> Result> { - self.conditional_query_and_cache(dest, 8448, true).await?; - - self.services.server.check_running()?; - trace!("Requesting well known for {dest}"); - let response = self - .services - .client - .well_known - .get(format!("https://{dest}/.well-known/matrix/server")) - .send() - .await; - - trace!("response: {response:?}"); - if let Err(e) = &response { - debug!("error: {e:?}"); - return Ok(None); - } - - let response = response?; - if !response.status().is_success() { - debug!("response not 2XX"); - return Ok(None); - } - - let text = response.text().await?; - trace!("response text: {text:?}"); - if text.len() >= 12288 { - debug_warn!("response contains junk"); - return Ok(None); - } - - let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); - - let m_server = body - .get("m.server") - .unwrap_or(&serde_json::Value::Null) - .as_str() - .unwrap_or_default(); - - if ruma::identifiers_validation::server_name::validate(m_server).is_err() { - debug_error!("response content missing or invalid"); - return Ok(None); - } - - debug_info!("{dest:?} found at {m_server:?}"); - Ok(Some(m_server.to_owned())) - } - #[inline] async fn conditional_query_and_cache( &self, diff --git a/src/service/resolver/mod.rs b/src/service/resolver/mod.rs index 246d6bc1..c513cec9 100644 --- a/src/service/resolver/mod.rs +++ b/src/service/resolver/mod.rs @@ -2,7 +2,9 @@ pub mod actual; pub mod cache; mod dns; pub mod fed; +#[cfg(test)] mod tests; +mod well_known; use std::sync::Arc; diff --git a/src/service/resolver/tests.rs b/src/service/resolver/tests.rs index 6e9d0e71..068e08bd 100644 --- a/src/service/resolver/tests.rs +++ b/src/service/resolver/tests.rs @@ -1,5 +1,3 @@ -#![cfg(test)] - use super::fed::{FedDest, add_port_to_hostname, get_ip_with_port}; #[test] diff --git a/src/service/resolver/well_known.rs b/src/service/resolver/well_known.rs new file mode 100644 index 00000000..68a8e620 --- /dev/null +++ b/src/service/resolver/well_known.rs @@ -0,0 +1,49 @@ +use conduwuit::{Result, debug, debug_error, debug_info, debug_warn, implement, trace}; + +#[implement(super::Service)] +#[tracing::instrument(name = "well-known", level = "debug", skip(self, dest))] +pub(super) async fn request_well_known(&self, dest: &str) -> Result> { + trace!("Requesting well known for {dest}"); + let response = self + .services + .client + .well_known + .get(format!("https://{dest}/.well-known/matrix/server")) + .send() + .await; + + trace!("response: {response:?}"); + if let Err(e) = &response { + debug!("error: {e:?}"); + return Ok(None); + } + + let response = response?; + if !response.status().is_success() { + debug!("response not 2XX"); + return Ok(None); + } + + let text = response.text().await?; + trace!("response text: {text:?}"); + if text.len() >= 12288 { + debug_warn!("response contains junk"); + return Ok(None); + } + + let body: serde_json::Value = serde_json::from_str(&text).unwrap_or_default(); + + let m_server = body + .get("m.server") + .unwrap_or(&serde_json::Value::Null) + .as_str() + .unwrap_or_default(); + + if ruma::identifiers_validation::server_name::validate(m_server).is_err() { + debug_error!("response content missing or invalid"); + return Ok(None); + } + + debug_info!("{dest:?} found at {m_server:?}"); + Ok(Some(m_server.to_owned())) +} From d1b82ea2253179836cf7400f70960d583b25af50 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 21 Mar 2025 08:10:44 +0000 Subject: [PATCH 1600/2091] use #[ignore] for todo'ed tests Signed-off-by: Jason Volk --- src/database/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/tests.rs b/src/database/tests.rs index 1446a1fc..c1a9f47c 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -325,8 +325,8 @@ fn ser_array() { assert_eq!(&s, &v, "vec serialization does not match"); } -#[cfg(todo)] #[test] +#[ignore] fn de_array() { let a: u64 = 123_456; let b: u64 = 987_654; @@ -357,8 +357,8 @@ fn de_array() { assert_eq!(vec[1], b, "deserialized vec [1] does not match"); } -#[cfg(todo)] #[test] +#[ignore] fn de_complex() { type Key<'a> = (&'a UserId, ArrayVec, &'a RoomId); From 9d0ce3965ea655943304b41ca679507b850130d3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 22 Mar 2025 07:09:11 +0000 Subject: [PATCH 1601/2091] fix lints Signed-off-by: Jason Volk --- src/api/client/context.rs | 2 +- src/core/error/response.rs | 2 +- src/service/media/preview.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index cb95dfef..b109711e 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -105,7 +105,7 @@ pub(crate) async fn get_context_route( .collect(); let (base_event, events_before, events_after): (_, Vec<_>, Vec<_>) = - join3(base_event, events_before, events_after).await; + join3(base_event, events_before, events_after).boxed().await; let lazy_loading_context = lazy_loading::Context { user_id: sender_user, diff --git a/src/core/error/response.rs b/src/core/error/response.rs index 00ade5ae..ae6fce62 100644 --- a/src/core/error/response.rs +++ b/src/core/error/response.rs @@ -86,7 +86,7 @@ pub(super) fn bad_request_code(kind: &ErrorKind) -> StatusCode { pub(super) fn ruma_error_message(error: &ruma::api::client::error::Error) -> String { if let ErrorBody::Standard { message, .. } = &error.body { - return message.to_string(); + return message.clone(); } format!("{error}") diff --git a/src/service/media/preview.rs b/src/service/media/preview.rs index ba5be7d4..91660a58 100644 --- a/src/service/media/preview.rs +++ b/src/service/media/preview.rs @@ -256,7 +256,7 @@ pub fn url_preview_allowed(&self, url: &Url) -> bool { if allowlist_url_contains .iter() - .any(|url_s| url.to_string().contains(&url_s.to_string())) + .any(|url_s| url.to_string().contains(url_s)) { debug!("URL {} is allowed by url_preview_url_contains_allowlist (check 4/4)", &host); return true; From 07ba00f74e2dfea314d0e5236f0415b2de6d543c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 26 Mar 2025 04:40:38 +0000 Subject: [PATCH 1602/2091] abstract raw query command iterations Signed-off-by: Jason Volk --- src/admin/query/raw.rs | 141 ++++++++++------------------------------- 1 file changed, 35 insertions(+), 106 deletions(-) diff --git a/src/admin/query/raw.rs b/src/admin/query/raw.rs index 23f11cc8..c503eee5 100644 --- a/src/admin/query/raw.rs +++ b/src/admin/query/raw.rs @@ -1,15 +1,16 @@ -use std::{borrow::Cow, collections::BTreeMap, ops::Deref}; +use std::{borrow::Cow, collections::BTreeMap, ops::Deref, sync::Arc}; use clap::Subcommand; use conduwuit::{ Err, Result, apply, at, is_zero, utils::{ - IterStream, - stream::{ReadyExt, TryIgnore, TryParallelExt}, + stream::{IterStream, ReadyExt, TryIgnore, TryParallelExt}, string::EMPTY, }, }; -use futures::{FutureExt, StreamExt, TryStreamExt}; +use conduwuit_database::Map; +use conduwuit_service::Services; +use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; use ruma::events::room::message::RoomMessageEventContent; use tokio::time::Instant; @@ -172,22 +173,18 @@ pub(super) async fn compact( ) -> Result { use conduwuit_database::compact::Options; - let default_all_maps = map - .is_none() - .then(|| { - self.services - .db - .keys() - .map(Deref::deref) - .map(ToOwned::to_owned) - }) - .into_iter() - .flatten(); + let default_all_maps: Option<_> = map.is_none().then(|| { + self.services + .db + .keys() + .map(Deref::deref) + .map(ToOwned::to_owned) + }); let maps: Vec<_> = map .unwrap_or_default() .into_iter() - .chain(default_all_maps) + .chain(default_all_maps.into_iter().flatten()) .map(|map| self.services.db.get(&map)) .filter_map(Result::ok) .cloned() @@ -237,25 +234,8 @@ pub(super) async fn raw_count( ) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); - let default_all_maps = map - .is_none() - .then(|| self.services.db.keys().map(Deref::deref)) - .into_iter() - .flatten(); - - let maps: Vec<_> = map - .iter() - .map(String::as_str) - .chain(default_all_maps) - .map(|map| self.services.db.get(map)) - .filter_map(Result::ok) - .cloned() - .collect(); - let timer = Instant::now(); - let count = maps - .iter() - .stream() + let count = with_maps_or(map.as_deref(), self.services) .then(|map| map.raw_count_prefix(&prefix)) .ready_fold(0_usize, usize::saturating_add) .await; @@ -300,25 +280,8 @@ pub(super) async fn raw_keys_sizes( ) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); - let default_all_maps = map - .is_none() - .then(|| self.services.db.keys().map(Deref::deref)) - .into_iter() - .flatten(); - - let maps: Vec<_> = map - .iter() - .map(String::as_str) - .chain(default_all_maps) - .map(|map| self.services.db.get(map)) - .filter_map(Result::ok) - .cloned() - .collect(); - let timer = Instant::now(); - let result = maps - .iter() - .stream() + let result = with_maps_or(map.as_deref(), self.services) .map(|map| map.raw_keys_prefix(&prefix)) .flatten() .ignore_err() @@ -345,25 +308,8 @@ pub(super) async fn raw_keys_total( ) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); - let default_all_maps = map - .is_none() - .then(|| self.services.db.keys().map(Deref::deref)) - .into_iter() - .flatten(); - - let maps: Vec<_> = map - .iter() - .map(String::as_str) - .chain(default_all_maps) - .map(|map| self.services.db.get(map)) - .filter_map(Result::ok) - .cloned() - .collect(); - let timer = Instant::now(); - let result = maps - .iter() - .stream() + let result = with_maps_or(map.as_deref(), self.services) .map(|map| map.raw_keys_prefix(&prefix)) .flatten() .ignore_err() @@ -387,25 +333,8 @@ pub(super) async fn raw_vals_sizes( ) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); - let default_all_maps = map - .is_none() - .then(|| self.services.db.keys().map(Deref::deref)) - .into_iter() - .flatten(); - - let maps: Vec<_> = map - .iter() - .map(String::as_str) - .chain(default_all_maps) - .map(|map| self.services.db.get(map)) - .filter_map(Result::ok) - .cloned() - .collect(); - let timer = Instant::now(); - let result = maps - .iter() - .stream() + let result = with_maps_or(map.as_deref(), self.services) .map(|map| map.raw_stream_prefix(&prefix)) .flatten() .ignore_err() @@ -433,25 +362,8 @@ pub(super) async fn raw_vals_total( ) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); - let default_all_maps = map - .is_none() - .then(|| self.services.db.keys().map(Deref::deref)) - .into_iter() - .flatten(); - - let maps: Vec<_> = map - .iter() - .map(String::as_str) - .chain(default_all_maps) - .map(|map| self.services.db.get(map)) - .filter_map(Result::ok) - .cloned() - .collect(); - let timer = Instant::now(); - let result = maps - .iter() - .stream() + let result = with_maps_or(map.as_deref(), self.services) .map(|map| map.raw_stream_prefix(&prefix)) .flatten() .ignore_err() @@ -573,3 +485,20 @@ pub(super) async fn raw_maps(&self) -> Result { Ok(RoomMessageEventContent::notice_markdown(format!("{list:#?}"))) } + +fn with_maps_or<'a>( + map: Option<&'a str>, + services: &'a Services, +) -> impl Stream> + Send + 'a { + let default_all_maps = map + .is_none() + .then(|| services.db.keys().map(Deref::deref)) + .into_iter() + .flatten(); + + map.into_iter() + .chain(default_all_maps) + .map(|map| services.db.get(map)) + .filter_map(Result::ok) + .stream() +} From dfe058a244ad7592114c86d504fb6fed744ad524 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 27 Mar 2025 01:08:42 +0000 Subject: [PATCH 1603/2091] default config item to 'none' when zstd_compression not featured Signed-off-by: Jason Volk --- src/core/config/mod.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 6b669ad3..52df19ac 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -2158,7 +2158,12 @@ fn default_rocksdb_max_log_file_size() -> usize { fn default_rocksdb_parallelism_threads() -> usize { 0 } -fn default_rocksdb_compression_algo() -> String { "zstd".to_owned() } +fn default_rocksdb_compression_algo() -> String { + cfg!(feature = "zstd_compression") + .then_some("zstd") + .unwrap_or("none") + .to_owned() +} /// Default RocksDB compression level is 32767, which is internally read by /// RocksDB as the default magic number and translated to the library's default From c99f5770a01ebae978461605c0f6eb954f7bad1b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 27 Mar 2025 04:07:24 +0000 Subject: [PATCH 1604/2091] mark get_summary_and_children_federation Send Signed-off-by: Jason Volk --- src/service/rooms/spaces/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 55897f9c..af597445 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -238,7 +238,7 @@ async fn get_summary_and_children_federation( fn get_stripped_space_child_events<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream> + 'a { +) -> impl Stream> + Send + 'a { self.services .state .get_room_shortstatehash(room_id) From 7f448d88a430cc2869fe9ab366fd29b3fddb0f13 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 27 Mar 2025 03:34:33 +0000 Subject: [PATCH 1605/2091] use qualified crate names from within workspace Signed-off-by: Jason Volk --- src/main/clap.rs | 9 +++++++-- src/main/logging.rs | 2 +- src/main/main.rs | 4 +--- src/main/mods.rs | 8 ++++---- src/main/restart.rs | 2 +- src/main/runtime.rs | 11 ++++++----- src/main/sentry.rs | 4 ++-- src/main/server.rs | 10 +++++----- src/main/signal.rs | 2 +- 9 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/main/clap.rs b/src/main/clap.rs index c7f33bfe..35a7ea41 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use clap::{ArgAction, Parser}; -use conduwuit::{ +use conduwuit_core::{ Err, Result, config::{Figment, FigmentValue}, err, toml, @@ -12,7 +12,12 @@ use conduwuit::{ /// Commandline arguments #[derive(Parser, Debug)] -#[clap(version = conduwuit::version(), about, long_about = None, name = "conduwuit")] +#[clap( + about, + long_about = None, + name = "conduwuit", + version = conduwuit_core::version(), +)] pub(crate) struct Args { #[arg(short, long)] /// Path to the config TOML file (optional) diff --git a/src/main/logging.rs b/src/main/logging.rs index 7ce86d56..eeeda127 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{ +use conduwuit_core::{ Result, config::Config, debug_warn, err, diff --git a/src/main/main.rs b/src/main/main.rs index 2bfc3c06..fbc63b17 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -7,11 +7,9 @@ mod sentry; mod server; mod signal; -extern crate conduwuit_core as conduwuit; - use std::sync::{Arc, atomic::Ordering}; -use conduwuit::{Error, Result, debug_info, error, rustc_flags_capture}; +use conduwuit_core::{Error, Result, debug_info, error, rustc_flags_capture}; use server::Server; rustc_flags_capture! {} diff --git a/src/main/mods.rs b/src/main/mods.rs index 6dc79b2f..d585a381 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -9,13 +9,13 @@ use std::{ sync::{Arc, atomic::Ordering}, }; -use conduwuit::{Error, Result, debug, error, mods}; +use conduwuit_core::{Error, Result, debug, error, mods}; use conduwuit_service::Services; use crate::Server; type StartFuncResult = Pin>> + Send>>; -type StartFuncProto = fn(&Arc) -> StartFuncResult; +type StartFuncProto = fn(&Arc) -> StartFuncResult; type RunFuncResult = Pin> + Send>>; type RunFuncProto = fn(&Arc) -> RunFuncResult; @@ -34,8 +34,8 @@ const MODULE_NAMES: &[&str] = &[ ]; #[cfg(panic_trap)] -conduwuit::mod_init! {{ - conduwuit::debug::set_panic_trap(); +conduwuit_core::mod_init! {{ + conduwuit_core::debug::set_panic_trap(); }} pub(crate) async fn run(server: &Arc, starts: bool) -> Result<(bool, bool), Error> { diff --git a/src/main/restart.rs b/src/main/restart.rs index e6f45b82..b9d1dc94 100644 --- a/src/main/restart.rs +++ b/src/main/restart.rs @@ -2,7 +2,7 @@ use std::{env, os::unix::process::CommandExt, process::Command}; -use conduwuit::{debug, info, utils}; +use conduwuit_core::{debug, info, utils}; #[cold] pub(super) fn restart() -> ! { diff --git a/src/main/runtime.rs b/src/main/runtime.rs index b3174e9c..b1657289 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -9,8 +9,8 @@ use std::{ }; #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] -use conduwuit::result::LogDebugErr; -use conduwuit::{ +use conduwuit_core::result::LogDebugErr; +use conduwuit_core::{ Result, is_true, utils::sys::compute::{nth_core_available, set_affinity}, }; @@ -122,7 +122,7 @@ fn set_worker_affinity() { #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] fn set_worker_mallctl(id: usize) { - use conduwuit::alloc::je::{ + use conduwuit_core::alloc::je::{ is_affine_arena, this_thread::{set_arena, set_muzzy_decay}, }; @@ -135,7 +135,8 @@ fn set_worker_mallctl(id: usize) { .get() .expect("GC_MUZZY initialized by runtime::new()"); - let muzzy_auto_disable = conduwuit::utils::available_parallelism() >= DISABLE_MUZZY_THRESHOLD; + let muzzy_auto_disable = + conduwuit_core::utils::available_parallelism() >= DISABLE_MUZZY_THRESHOLD; if matches!(muzzy_option, Some(false) | None if muzzy_auto_disable) { set_muzzy_decay(-1).log_debug_err().ok(); } @@ -188,7 +189,7 @@ fn thread_park() { fn gc_on_park() { #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] - conduwuit::alloc::je::this_thread::decay() + conduwuit_core::alloc::je::this_thread::decay() .log_debug_err() .ok(); } diff --git a/src/main/sentry.rs b/src/main/sentry.rs index 1ea1f3ae..68f12eb7 100644 --- a/src/main/sentry.rs +++ b/src/main/sentry.rs @@ -5,7 +5,7 @@ use std::{ sync::{Arc, OnceLock}, }; -use conduwuit::{config::Config, debug, trace}; +use conduwuit_core::{config::Config, debug, trace}; use sentry::{ Breadcrumb, ClientOptions, Level, types::{ @@ -43,7 +43,7 @@ fn options(config: &Config) -> ClientOptions { traces_sample_rate: config.sentry_traces_sample_rate, debug: cfg!(debug_assertions), release: sentry::release_name!(), - user_agent: conduwuit::version::user_agent().into(), + user_agent: conduwuit_core::version::user_agent().into(), attach_stacktrace: config.sentry_attach_stacktrace, before_send: Some(Arc::new(before_send)), before_breadcrumb: Some(Arc::new(before_breadcrumb)), diff --git a/src/main/server.rs b/src/main/server.rs index 44ca69b0..8f697ca4 100644 --- a/src/main/server.rs +++ b/src/main/server.rs @@ -1,6 +1,6 @@ use std::{path::PathBuf, sync::Arc}; -use conduwuit::{ +use conduwuit_core::{ Error, Result, config::Config, info, @@ -14,7 +14,7 @@ use crate::{clap::Args, logging::TracingFlameGuard}; /// Server runtime state; complete pub(crate) struct Server { /// Server runtime state; public portion - pub(crate) server: Arc, + pub(crate) server: Arc, pub(crate) services: Mutex>>, @@ -25,7 +25,7 @@ pub(crate) struct Server { #[cfg(all(conduwuit_mods, feature = "conduwuit_mods"))] // Module instances; TODO: move to mods::loaded mgmt vector - pub(crate) mods: tokio::sync::RwLock>, + pub(crate) mods: tokio::sync::RwLock>, } impl Server { @@ -66,11 +66,11 @@ impl Server { database_path = ?config.database_path, log_levels = %config.log, "{}", - conduwuit::version(), + conduwuit_core::version(), ); Ok(Arc::new(Self { - server: Arc::new(conduwuit::Server::new(config, runtime.cloned(), Log { + server: Arc::new(conduwuit_core::Server::new(config, runtime.cloned(), Log { reload: tracing_reload_handle, capture, })), diff --git a/src/main/signal.rs b/src/main/signal.rs index 343b95c9..a5d07774 100644 --- a/src/main/signal.rs +++ b/src/main/signal.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{debug_error, trace, warn}; +use conduwuit_core::{debug_error, trace, warn}; use tokio::signal; use super::server::Server; From b2bf35cfab8aac82e4cde1c7c5a7b6e713bba5db Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 28 Mar 2025 06:42:30 +0000 Subject: [PATCH 1606/2091] fix benches from state-res Signed-off-by: Jason Volk --- src/core/state_res/benches.rs | 672 ++++++++++++++++++++++++++ src/core/state_res/mod.rs | 3 + src/core/state_res/state_res_bench.rs | 648 ------------------------- 3 files changed, 675 insertions(+), 648 deletions(-) create mode 100644 src/core/state_res/benches.rs delete mode 100644 src/core/state_res/state_res_bench.rs diff --git a/src/core/state_res/benches.rs b/src/core/state_res/benches.rs new file mode 100644 index 00000000..7a1ae5bf --- /dev/null +++ b/src/core/state_res/benches.rs @@ -0,0 +1,672 @@ +#[cfg(conduwuit_bench)] +extern crate test; + +use std::{ + borrow::Borrow, + collections::{HashMap, HashSet}, + sync::{ + Arc, + atomic::{AtomicU64, Ordering::SeqCst}, + }, +}; + +use futures::{future, future::ready}; +use maplit::{btreemap, hashmap, hashset}; +use ruma::{ + EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, Signatures, UserId, + events::{ + StateEventType, TimelineEventType, + pdu::{EventHash, Pdu, RoomV3Pdu}, + room::{ + join_rules::{JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + }, + }, + int, room_id, uint, user_id, +}; +use serde_json::{ + json, + value::{RawValue as RawJsonValue, to_raw_value as to_raw_json_value}, +}; + +use self::event::PduEvent; +use crate::state_res::{self as state_res, Error, Event, Result, StateMap}; + +static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); + +#[cfg(conduwuit_bench)] +#[cfg_attr(conduwuit_bench, bench)] +fn lexico_topo_sort(c: &mut test::Bencher) { + let graph = hashmap! { + event_id("l") => hashset![event_id("o")], + event_id("m") => hashset![event_id("n"), event_id("o")], + event_id("n") => hashset![event_id("o")], + event_id("o") => hashset![], // "o" has zero outgoing edges but 4 incoming edges + event_id("p") => hashset![event_id("o")], + }; + + c.iter(|| { + let _ = state_res::lexicographical_topological_sort(&graph, &|_| { + future::ok((int!(0), MilliSecondsSinceUnixEpoch(uint!(0)))) + }); + }); +} + +#[cfg(conduwuit_bench)] +#[cfg_attr(conduwuit_bench, bench)] +fn resolution_shallow_auth_chain(c: &mut test::Bencher) { + let parallel_fetches = 32; + let mut store = TestStore(hashmap! {}); + + // build up the DAG + let (state_at_bob, state_at_charlie, _) = store.set_up(); + + c.iter(|| async { + let ev_map = store.0.clone(); + let state_sets = [&state_at_bob, &state_at_charlie]; + let fetch = |id: OwnedEventId| ready(ev_map.get(&id).map(Arc::clone)); + let exists = |id: OwnedEventId| ready(ev_map.get(&id).is_some()); + let auth_chain_sets: Vec> = state_sets + .iter() + .map(|map| { + store + .auth_event_ids(room_id(), map.values().cloned().collect()) + .unwrap() + }) + .collect(); + + let _ = match state_res::resolve( + &RoomVersionId::V6, + state_sets.into_iter(), + &auth_chain_sets, + &fetch, + &exists, + parallel_fetches, + ) + .await + { + | Ok(state) => state, + | Err(e) => panic!("{e}"), + }; + }); +} + +#[cfg(conduwuit_bench)] +#[cfg_attr(conduwuit_bench, bench)] +fn resolve_deeper_event_set(c: &mut test::Bencher) { + let parallel_fetches = 32; + let mut inner = INITIAL_EVENTS(); + let ban = BAN_STATE_SET(); + + inner.extend(ban); + let store = TestStore(inner.clone()); + + let state_set_a = [ + inner.get(&event_id("CREATE")).unwrap(), + inner.get(&event_id("IJR")).unwrap(), + inner.get(&event_id("IMA")).unwrap(), + inner.get(&event_id("IMB")).unwrap(), + inner.get(&event_id("IMC")).unwrap(), + inner.get(&event_id("MB")).unwrap(), + inner.get(&event_id("PA")).unwrap(), + ] + .iter() + .map(|ev| { + ( + (ev.event_type().clone().into(), ev.state_key().unwrap().into()), + ev.event_id().to_owned(), + ) + }) + .collect::>(); + + let state_set_b = [ + inner.get(&event_id("CREATE")).unwrap(), + inner.get(&event_id("IJR")).unwrap(), + inner.get(&event_id("IMA")).unwrap(), + inner.get(&event_id("IMB")).unwrap(), + inner.get(&event_id("IMC")).unwrap(), + inner.get(&event_id("IME")).unwrap(), + inner.get(&event_id("PA")).unwrap(), + ] + .iter() + .map(|ev| { + ( + (ev.event_type().clone().into(), ev.state_key().unwrap().into()), + ev.event_id().to_owned(), + ) + }) + .collect::>(); + + c.iter(|| async { + let state_sets = [&state_set_a, &state_set_b]; + let auth_chain_sets: Vec> = state_sets + .iter() + .map(|map| { + store + .auth_event_ids(room_id(), map.values().cloned().collect()) + .unwrap() + }) + .collect(); + + let fetch = |id: OwnedEventId| ready(inner.get(&id).map(Arc::clone)); + let exists = |id: OwnedEventId| ready(inner.get(&id).is_some()); + let _ = match state_res::resolve( + &RoomVersionId::V6, + state_sets.into_iter(), + &auth_chain_sets, + &fetch, + &exists, + parallel_fetches, + ) + .await + { + | Ok(state) => state, + | Err(_) => panic!("resolution failed during benchmarking"), + }; + }); +} + +//*///////////////////////////////////////////////////////////////////// +// +// IMPLEMENTATION DETAILS AHEAD +// +/////////////////////////////////////////////////////////////////////*/ +struct TestStore(HashMap>); + +#[allow(unused)] +impl TestStore { + fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> Result> { + self.0 + .get(event_id) + .map(Arc::clone) + .ok_or_else(|| Error::NotFound(format!("{} not found", event_id))) + } + + /// Returns the events that correspond to the `event_ids` sorted in the same + /// order. + fn get_events(&self, room_id: &RoomId, event_ids: &[OwnedEventId]) -> Result>> { + let mut events = vec![]; + for id in event_ids { + events.push(self.get_event(room_id, id)?); + } + Ok(events) + } + + /// Returns a Vec of the related auth events to the given `event`. + fn auth_event_ids(&self, room_id: &RoomId, event_ids: Vec) -> Result> { + let mut result = HashSet::new(); + let mut stack = event_ids; + + // DFS for auth event chain + while !stack.is_empty() { + let ev_id = stack.pop().unwrap(); + if result.contains(&ev_id) { + continue; + } + + result.insert(ev_id.clone()); + + let event = self.get_event(room_id, ev_id.borrow())?; + + stack.extend(event.auth_events().map(ToOwned::to_owned)); + } + + Ok(result) + } + + /// Returns a vector representing the difference in auth chains of the given + /// `events`. + fn auth_chain_diff( + &self, + room_id: &RoomId, + event_ids: Vec>, + ) -> Result> { + let mut auth_chain_sets = vec![]; + for ids in event_ids { + // TODO state store `auth_event_ids` returns self in the event ids list + // when an event returns `auth_event_ids` self is not contained + let chain = self + .auth_event_ids(room_id, ids)? + .into_iter() + .collect::>(); + auth_chain_sets.push(chain); + } + + if let Some(first) = auth_chain_sets.first().cloned() { + let common = auth_chain_sets + .iter() + .skip(1) + .fold(first, |a, b| a.intersection(b).cloned().collect::>()); + + Ok(auth_chain_sets + .into_iter() + .flatten() + .filter(|id| !common.contains(id.borrow())) + .collect()) + } else { + Ok(vec![]) + } + } +} + +impl TestStore { + #[allow(clippy::type_complexity)] + fn set_up( + &mut self, + ) -> (StateMap, StateMap, StateMap) { + let create_event = to_pdu_event::<&EventId>( + "CREATE", + alice(), + TimelineEventType::RoomCreate, + Some(""), + to_raw_json_value(&json!({ "creator": alice() })).unwrap(), + &[], + &[], + ); + let cre = create_event.event_id().to_owned(); + self.0.insert(cre.clone(), Arc::clone(&create_event)); + + let alice_mem = to_pdu_event( + "IMA", + alice(), + TimelineEventType::RoomMember, + Some(alice().to_string().as_str()), + member_content_join(), + &[cre.clone()], + &[cre.clone()], + ); + self.0 + .insert(alice_mem.event_id().to_owned(), Arc::clone(&alice_mem)); + + let join_rules = to_pdu_event( + "IJR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Public)).unwrap(), + &[cre.clone(), alice_mem.event_id().to_owned()], + &[alice_mem.event_id().to_owned()], + ); + self.0 + .insert(join_rules.event_id().to_owned(), join_rules.clone()); + + // Bob and Charlie join at the same time, so there is a fork + // this will be represented in the state_sets when we resolve + let bob_mem = to_pdu_event( + "IMB", + bob(), + TimelineEventType::RoomMember, + Some(bob().to_string().as_str()), + member_content_join(), + &[cre.clone(), join_rules.event_id().to_owned()], + &[join_rules.event_id().to_owned()], + ); + self.0 + .insert(bob_mem.event_id().to_owned(), bob_mem.clone()); + + let charlie_mem = to_pdu_event( + "IMC", + charlie(), + TimelineEventType::RoomMember, + Some(charlie().to_string().as_str()), + member_content_join(), + &[cre, join_rules.event_id().to_owned()], + &[join_rules.event_id().to_owned()], + ); + self.0 + .insert(charlie_mem.event_id().to_owned(), charlie_mem.clone()); + + let state_at_bob = [&create_event, &alice_mem, &join_rules, &bob_mem] + .iter() + .map(|ev| { + ( + (ev.event_type().clone().into(), ev.state_key().unwrap().into()), + ev.event_id().to_owned(), + ) + }) + .collect::>(); + + let state_at_charlie = [&create_event, &alice_mem, &join_rules, &charlie_mem] + .iter() + .map(|ev| { + ( + (ev.event_type().clone().into(), ev.state_key().unwrap().into()), + ev.event_id().to_owned(), + ) + }) + .collect::>(); + + let expected = [&create_event, &alice_mem, &join_rules, &bob_mem, &charlie_mem] + .iter() + .map(|ev| { + ( + (ev.event_type().clone().into(), ev.state_key().unwrap().into()), + ev.event_id().to_owned(), + ) + }) + .collect::>(); + + (state_at_bob, state_at_charlie, expected) + } +} + +fn event_id(id: &str) -> OwnedEventId { + if id.contains('$') { + return id.try_into().unwrap(); + } + format!("${}:foo", id).try_into().unwrap() +} + +fn alice() -> &'static UserId { user_id!("@alice:foo") } + +fn bob() -> &'static UserId { user_id!("@bob:foo") } + +fn charlie() -> &'static UserId { user_id!("@charlie:foo") } + +fn ella() -> &'static UserId { user_id!("@ella:foo") } + +fn room_id() -> &'static RoomId { room_id!("!test:foo") } + +fn member_content_ban() -> Box { + to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Ban)).unwrap() +} + +fn member_content_join() -> Box { + to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Join)).unwrap() +} + +fn to_pdu_event( + id: &str, + sender: &UserId, + ev_type: TimelineEventType, + state_key: Option<&str>, + content: Box, + auth_events: &[S], + prev_events: &[S], +) -> Arc +where + S: AsRef, +{ + // We don't care if the addition happens in order just that it is atomic + // (each event has its own value) + let ts = SERVER_TIMESTAMP.fetch_add(1, SeqCst); + let id = if id.contains('$') { + id.to_owned() + } else { + format!("${}:foo", id) + }; + let auth_events = auth_events + .iter() + .map(AsRef::as_ref) + .map(event_id) + .collect::>(); + let prev_events = prev_events + .iter() + .map(AsRef::as_ref) + .map(event_id) + .collect::>(); + + let state_key = state_key.map(ToOwned::to_owned); + Arc::new(PduEvent { + event_id: id.try_into().unwrap(), + rest: Pdu::RoomV3Pdu(RoomV3Pdu { + room_id: room_id().to_owned(), + sender: sender.to_owned(), + origin_server_ts: MilliSecondsSinceUnixEpoch(ts.try_into().unwrap()), + state_key, + kind: ev_type, + content, + redacts: None, + unsigned: btreemap! {}, + auth_events, + prev_events, + depth: uint!(0), + hashes: EventHash::new(String::new()), + signatures: Signatures::new(), + }), + }) +} + +// all graphs start with these input events +#[allow(non_snake_case)] +fn INITIAL_EVENTS() -> HashMap> { + vec![ + to_pdu_event::<&EventId>( + "CREATE", + alice(), + TimelineEventType::RoomCreate, + Some(""), + to_raw_json_value(&json!({ "creator": alice() })).unwrap(), + &[], + &[], + ), + to_pdu_event( + "IMA", + alice(), + TimelineEventType::RoomMember, + Some(alice().as_str()), + member_content_join(), + &["CREATE"], + &["CREATE"], + ), + to_pdu_event( + "IPOWER", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100 } })).unwrap(), + &["CREATE", "IMA"], + &["IMA"], + ), + to_pdu_event( + "IJR", + alice(), + TimelineEventType::RoomJoinRules, + Some(""), + to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Public)).unwrap(), + &["CREATE", "IMA", "IPOWER"], + &["IPOWER"], + ), + to_pdu_event( + "IMB", + bob(), + TimelineEventType::RoomMember, + Some(bob().to_string().as_str()), + member_content_join(), + &["CREATE", "IJR", "IPOWER"], + &["IJR"], + ), + to_pdu_event( + "IMC", + charlie(), + TimelineEventType::RoomMember, + Some(charlie().to_string().as_str()), + member_content_join(), + &["CREATE", "IJR", "IPOWER"], + &["IMB"], + ), + to_pdu_event::<&EventId>( + "START", + charlie(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + &[], + &[], + ), + to_pdu_event::<&EventId>( + "END", + charlie(), + TimelineEventType::RoomTopic, + Some(""), + to_raw_json_value(&json!({})).unwrap(), + &[], + &[], + ), + ] + .into_iter() + .map(|ev| (ev.event_id().to_owned(), ev)) + .collect() +} + +// all graphs start with these input events +#[allow(non_snake_case)] +fn BAN_STATE_SET() -> HashMap> { + vec![ + to_pdu_event( + "PA", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + &["CREATE", "IMA", "IPOWER"], // auth_events + &["START"], // prev_events + ), + to_pdu_event( + "PB", + alice(), + TimelineEventType::RoomPowerLevels, + Some(""), + to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), + &["CREATE", "IMA", "IPOWER"], + &["END"], + ), + to_pdu_event( + "MB", + alice(), + TimelineEventType::RoomMember, + Some(ella().as_str()), + member_content_ban(), + &["CREATE", "IMA", "PB"], + &["PA"], + ), + to_pdu_event( + "IME", + ella(), + TimelineEventType::RoomMember, + Some(ella().as_str()), + member_content_join(), + &["CREATE", "IJR", "PA"], + &["MB"], + ), + ] + .into_iter() + .map(|ev| (ev.event_id().to_owned(), ev)) + .collect() +} + +/// Convenience trait for adding event type plus state key to state maps. +trait EventTypeExt { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, String); +} + +impl EventTypeExt for &TimelineEventType { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { + (self.to_string().into(), state_key.into()) + } +} + +mod event { + use ruma::{ + MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, + events::{TimelineEventType, pdu::Pdu}, + }; + use serde::{Deserialize, Serialize}; + use serde_json::value::RawValue as RawJsonValue; + + use super::Event; + + impl Event for PduEvent { + type Id = OwnedEventId; + + fn event_id(&self) -> &Self::Id { &self.event_id } + + fn room_id(&self) -> &RoomId { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => &ev.room_id, + | Pdu::RoomV3Pdu(ev) => &ev.room_id, + #[cfg(not(feature = "unstable-exhaustive-types"))] + | _ => unreachable!("new PDU version"), + } + } + + fn sender(&self) -> &UserId { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => &ev.sender, + | Pdu::RoomV3Pdu(ev) => &ev.sender, + #[cfg(not(feature = "unstable-exhaustive-types"))] + | _ => unreachable!("new PDU version"), + } + } + + fn event_type(&self) -> &TimelineEventType { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => &ev.kind, + | Pdu::RoomV3Pdu(ev) => &ev.kind, + #[cfg(not(feature = "unstable-exhaustive-types"))] + | _ => unreachable!("new PDU version"), + } + } + + fn content(&self) -> &RawJsonValue { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => &ev.content, + | Pdu::RoomV3Pdu(ev) => &ev.content, + #[cfg(not(feature = "unstable-exhaustive-types"))] + | _ => unreachable!("new PDU version"), + } + } + + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => ev.origin_server_ts, + | Pdu::RoomV3Pdu(ev) => ev.origin_server_ts, + #[cfg(not(feature = "unstable-exhaustive-types"))] + | _ => unreachable!("new PDU version"), + } + } + + fn state_key(&self) -> Option<&str> { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => ev.state_key.as_deref(), + | Pdu::RoomV3Pdu(ev) => ev.state_key.as_deref(), + #[cfg(not(feature = "unstable-exhaustive-types"))] + | _ => unreachable!("new PDU version"), + } + } + + fn prev_events(&self) -> Box + Send + '_> { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => Box::new(ev.prev_events.iter().map(|(id, _)| id)), + | Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter()), + #[cfg(not(feature = "unstable-exhaustive-types"))] + | _ => unreachable!("new PDU version"), + } + } + + fn auth_events(&self) -> Box + Send + '_> { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => Box::new(ev.auth_events.iter().map(|(id, _)| id)), + | Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter()), + #[cfg(not(feature = "unstable-exhaustive-types"))] + | _ => unreachable!("new PDU version"), + } + } + + fn redacts(&self) -> Option<&Self::Id> { + match &self.rest { + | Pdu::RoomV1Pdu(ev) => ev.redacts.as_ref(), + | Pdu::RoomV3Pdu(ev) => ev.redacts.as_ref(), + #[cfg(not(feature = "unstable-exhaustive-types"))] + | _ => unreachable!("new PDU version"), + } + } + } + + #[derive(Clone, Debug, Deserialize, Serialize)] + pub(crate) struct PduEvent { + pub(crate) event_id: OwnedEventId, + #[serde(flatten)] + pub(crate) rest: Pdu, + } +} diff --git a/src/core/state_res/mod.rs b/src/core/state_res/mod.rs index 6bff0cf8..2020d65c 100644 --- a/src/core/state_res/mod.rs +++ b/src/core/state_res/mod.rs @@ -9,6 +9,9 @@ mod state_event; #[cfg(test)] mod test_utils; +#[cfg(test)] +mod benches; + use std::{ borrow::Borrow, cmp::{Ordering, Reverse}, diff --git a/src/core/state_res/state_res_bench.rs b/src/core/state_res/state_res_bench.rs deleted file mode 100644 index a2bd2c23..00000000 --- a/src/core/state_res/state_res_bench.rs +++ /dev/null @@ -1,648 +0,0 @@ -// Because of criterion `cargo bench` works, -// but if you use `cargo bench -- --save-baseline ` -// or pass any other args to it, it fails with the error -// `cargo bench unknown option --save-baseline`. -// To pass args to criterion, use this form -// `cargo bench --bench -- --save-baseline `. - -#![allow(clippy::exhaustive_structs)] - -use std::{ - borrow::Borrow, - collections::{HashMap, HashSet}, - sync::{ - atomic::{AtomicU64, Ordering::SeqCst}, - Arc, - }, -}; - -use criterion::{criterion_group, criterion_main, Criterion}; -use event::PduEvent; -use futures::{future, future::ready}; -use ruma::{int, uint}; -use maplit::{btreemap, hashmap, hashset}; -use ruma::{ - room_id, user_id, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, - Signatures, UserId, -}; -use ruma::events::{ - pdu::{EventHash, Pdu, RoomV3Pdu}, - room::{ - join_rules::{JoinRule, RoomJoinRulesEventContent}, - member::{MembershipState, RoomMemberEventContent}, - }, - StateEventType, TimelineEventType, -}; -use conduwuit::state_res::{self as state_res, Error, Event, Result, StateMap}; -use serde_json::{ - json, - value::{to_raw_value as to_raw_json_value, RawValue as RawJsonValue}, -}; - -static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); - -fn lexico_topo_sort(c: &mut Criterion) { - c.bench_function("lexicographical topological sort", |b| { - let graph = hashmap! { - event_id("l") => hashset![event_id("o")], - event_id("m") => hashset![event_id("n"), event_id("o")], - event_id("n") => hashset![event_id("o")], - event_id("o") => hashset![], // "o" has zero outgoing edges but 4 incoming edges - event_id("p") => hashset![event_id("o")], - }; - b.iter(|| { - let _ = state_res::lexicographical_topological_sort(&graph, &|_| { - future::ok((int!(0), MilliSecondsSinceUnixEpoch(uint!(0)))) - }); - }); - }); -} - -fn resolution_shallow_auth_chain(c: &mut Criterion) { - c.bench_function("resolve state of 5 events one fork", |b| { - let mut store = TestStore(hashmap! {}); - - // build up the DAG - let (state_at_bob, state_at_charlie, _) = store.set_up(); - - b.iter(|| async { - let ev_map = store.0.clone(); - let state_sets = [&state_at_bob, &state_at_charlie]; - let fetch = |id: OwnedEventId| ready(ev_map.get(&id).map(Arc::clone)); - let exists = |id: OwnedEventId| ready(ev_map.get(&id).is_some()); - let auth_chain_sets = state_sets - .iter() - .map(|map| { - store.auth_event_ids(room_id(), map.values().cloned().collect()).unwrap() - }) - .collect(); - - let _ = match state_res::resolve( - &RoomVersionId::V6, - state_sets.into_iter(), - &auth_chain_sets, - &fetch, - &exists, - ) - .await - { - Ok(state) => state, - Err(e) => panic!("{e}"), - }; - }); - }); -} - -fn resolve_deeper_event_set(c: &mut Criterion) { - c.bench_function("resolve state of 10 events 3 conflicting", |b| { - let mut inner = INITIAL_EVENTS(); - let ban = BAN_STATE_SET(); - - inner.extend(ban); - let store = TestStore(inner.clone()); - - let state_set_a = [ - inner.get(&event_id("CREATE")).unwrap(), - inner.get(&event_id("IJR")).unwrap(), - inner.get(&event_id("IMA")).unwrap(), - inner.get(&event_id("IMB")).unwrap(), - inner.get(&event_id("IMC")).unwrap(), - inner.get(&event_id("MB")).unwrap(), - inner.get(&event_id("PA")).unwrap(), - ] - .iter() - .map(|ev| { - (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.event_id().to_owned()) - }) - .collect::>(); - - let state_set_b = [ - inner.get(&event_id("CREATE")).unwrap(), - inner.get(&event_id("IJR")).unwrap(), - inner.get(&event_id("IMA")).unwrap(), - inner.get(&event_id("IMB")).unwrap(), - inner.get(&event_id("IMC")).unwrap(), - inner.get(&event_id("IME")).unwrap(), - inner.get(&event_id("PA")).unwrap(), - ] - .iter() - .map(|ev| { - (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.event_id().to_owned()) - }) - .collect::>(); - - b.iter(|| async { - let state_sets = [&state_set_a, &state_set_b]; - let auth_chain_sets = state_sets - .iter() - .map(|map| { - store.auth_event_ids(room_id(), map.values().cloned().collect()).unwrap() - }) - .collect(); - - let fetch = |id: OwnedEventId| ready(inner.get(&id).map(Arc::clone)); - let exists = |id: OwnedEventId| ready(inner.get(&id).is_some()); - let _ = match state_res::resolve( - &RoomVersionId::V6, - state_sets.into_iter(), - &auth_chain_sets, - &fetch, - &exists, - ) - .await - { - Ok(state) => state, - Err(_) => panic!("resolution failed during benchmarking"), - }; - }); - }); -} - -criterion_group!( - benches, - lexico_topo_sort, - resolution_shallow_auth_chain, - resolve_deeper_event_set -); - -criterion_main!(benches); - -//*///////////////////////////////////////////////////////////////////// -// -// IMPLEMENTATION DETAILS AHEAD -// -/////////////////////////////////////////////////////////////////////*/ -struct TestStore(HashMap>); - -#[allow(unused)] -impl TestStore { - fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> Result> { - self.0 - .get(event_id) - .map(Arc::clone) - .ok_or_else(|| Error::NotFound(format!("{} not found", event_id))) - } - - /// Returns the events that correspond to the `event_ids` sorted in the same order. - fn get_events(&self, room_id: &RoomId, event_ids: &[OwnedEventId]) -> Result>> { - let mut events = vec![]; - for id in event_ids { - events.push(self.get_event(room_id, id)?); - } - Ok(events) - } - - /// Returns a Vec of the related auth events to the given `event`. - fn auth_event_ids(&self, room_id: &RoomId, event_ids: Vec) -> Result> { - let mut result = HashSet::new(); - let mut stack = event_ids; - - // DFS for auth event chain - while !stack.is_empty() { - let ev_id = stack.pop().unwrap(); - if result.contains(&ev_id) { - continue; - } - - result.insert(ev_id.clone()); - - let event = self.get_event(room_id, ev_id.borrow())?; - - stack.extend(event.auth_events().map(ToOwned::to_owned)); - } - - Ok(result) - } - - /// Returns a vector representing the difference in auth chains of the given `events`. - fn auth_chain_diff(&self, room_id: &RoomId, event_ids: Vec>) -> Result> { - let mut auth_chain_sets = vec![]; - for ids in event_ids { - // TODO state store `auth_event_ids` returns self in the event ids list - // when an event returns `auth_event_ids` self is not contained - let chain = self.auth_event_ids(room_id, ids)?.into_iter().collect::>(); - auth_chain_sets.push(chain); - } - - if let Some(first) = auth_chain_sets.first().cloned() { - let common = auth_chain_sets - .iter() - .skip(1) - .fold(first, |a, b| a.intersection(b).cloned().collect::>()); - - Ok(auth_chain_sets - .into_iter() - .flatten() - .filter(|id| !common.contains(id.borrow())) - .collect()) - } else { - Ok(vec![]) - } - } -} - -impl TestStore { - #[allow(clippy::type_complexity)] - fn set_up( - &mut self, - ) -> (StateMap, StateMap, StateMap) { - let create_event = to_pdu_event::<&EventId>( - "CREATE", - alice(), - TimelineEventType::RoomCreate, - Some(""), - to_raw_json_value(&json!({ "creator": alice() })).unwrap(), - &[], - &[], - ); - let cre = create_event.event_id().to_owned(); - self.0.insert(cre.clone(), Arc::clone(&create_event)); - - let alice_mem = to_pdu_event( - "IMA", - alice(), - TimelineEventType::RoomMember, - Some(alice().to_string().as_str()), - member_content_join(), - &[cre.clone()], - &[cre.clone()], - ); - self.0.insert(alice_mem.event_id().to_owned(), Arc::clone(&alice_mem)); - - let join_rules = to_pdu_event( - "IJR", - alice(), - TimelineEventType::RoomJoinRules, - Some(""), - to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Public)).unwrap(), - &[cre.clone(), alice_mem.event_id().to_owned()], - &[alice_mem.event_id().to_owned()], - ); - self.0.insert(join_rules.event_id().to_owned(), join_rules.clone()); - - // Bob and Charlie join at the same time, so there is a fork - // this will be represented in the state_sets when we resolve - let bob_mem = to_pdu_event( - "IMB", - bob(), - TimelineEventType::RoomMember, - Some(bob().to_string().as_str()), - member_content_join(), - &[cre.clone(), join_rules.event_id().to_owned()], - &[join_rules.event_id().to_owned()], - ); - self.0.insert(bob_mem.event_id().to_owned(), bob_mem.clone()); - - let charlie_mem = to_pdu_event( - "IMC", - charlie(), - TimelineEventType::RoomMember, - Some(charlie().to_string().as_str()), - member_content_join(), - &[cre, join_rules.event_id().to_owned()], - &[join_rules.event_id().to_owned()], - ); - self.0.insert(charlie_mem.event_id().to_owned(), charlie_mem.clone()); - - let state_at_bob = [&create_event, &alice_mem, &join_rules, &bob_mem] - .iter() - .map(|e| { - (e.event_type().with_state_key(e.state_key().unwrap()), e.event_id().to_owned()) - }) - .collect::>(); - - let state_at_charlie = [&create_event, &alice_mem, &join_rules, &charlie_mem] - .iter() - .map(|e| { - (e.event_type().with_state_key(e.state_key().unwrap()), e.event_id().to_owned()) - }) - .collect::>(); - - let expected = [&create_event, &alice_mem, &join_rules, &bob_mem, &charlie_mem] - .iter() - .map(|e| { - (e.event_type().with_state_key(e.state_key().unwrap()), e.event_id().to_owned()) - }) - .collect::>(); - - (state_at_bob, state_at_charlie, expected) - } -} - -fn event_id(id: &str) -> OwnedEventId { - if id.contains('$') { - return id.try_into().unwrap(); - } - format!("${}:foo", id).try_into().unwrap() -} - -fn alice() -> &'static UserId { - user_id!("@alice:foo") -} - -fn bob() -> &'static UserId { - user_id!("@bob:foo") -} - -fn charlie() -> &'static UserId { - user_id!("@charlie:foo") -} - -fn ella() -> &'static UserId { - user_id!("@ella:foo") -} - -fn room_id() -> &'static RoomId { - room_id!("!test:foo") -} - -fn member_content_ban() -> Box { - to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Ban)).unwrap() -} - -fn member_content_join() -> Box { - to_raw_json_value(&RoomMemberEventContent::new(MembershipState::Join)).unwrap() -} - -fn to_pdu_event( - id: &str, - sender: &UserId, - ev_type: TimelineEventType, - state_key: Option<&str>, - content: Box, - auth_events: &[S], - prev_events: &[S], -) -> Arc -where - S: AsRef, -{ - // We don't care if the addition happens in order just that it is atomic - // (each event has its own value) - let ts = SERVER_TIMESTAMP.fetch_add(1, SeqCst); - let id = if id.contains('$') { id.to_owned() } else { format!("${}:foo", id) }; - let auth_events = auth_events.iter().map(AsRef::as_ref).map(event_id).collect::>(); - let prev_events = prev_events.iter().map(AsRef::as_ref).map(event_id).collect::>(); - - let state_key = state_key.map(ToOwned::to_owned); - Arc::new(PduEvent { - event_id: id.try_into().unwrap(), - rest: Pdu::RoomV3Pdu(RoomV3Pdu { - room_id: room_id().to_owned(), - sender: sender.to_owned(), - origin_server_ts: MilliSecondsSinceUnixEpoch(ts.try_into().unwrap()), - state_key, - kind: ev_type, - content, - redacts: None, - unsigned: btreemap! {}, - auth_events, - prev_events, - depth: uint!(0), - hashes: EventHash::new(String::new()), - signatures: Signatures::new(), - }), - }) -} - -// all graphs start with these input events -#[allow(non_snake_case)] -fn INITIAL_EVENTS() -> HashMap> { - vec![ - to_pdu_event::<&EventId>( - "CREATE", - alice(), - TimelineEventType::RoomCreate, - Some(""), - to_raw_json_value(&json!({ "creator": alice() })).unwrap(), - &[], - &[], - ), - to_pdu_event( - "IMA", - alice(), - TimelineEventType::RoomMember, - Some(alice().as_str()), - member_content_join(), - &["CREATE"], - &["CREATE"], - ), - to_pdu_event( - "IPOWER", - alice(), - TimelineEventType::RoomPowerLevels, - Some(""), - to_raw_json_value(&json!({ "users": { alice(): 100 } })).unwrap(), - &["CREATE", "IMA"], - &["IMA"], - ), - to_pdu_event( - "IJR", - alice(), - TimelineEventType::RoomJoinRules, - Some(""), - to_raw_json_value(&RoomJoinRulesEventContent::new(JoinRule::Public)).unwrap(), - &["CREATE", "IMA", "IPOWER"], - &["IPOWER"], - ), - to_pdu_event( - "IMB", - bob(), - TimelineEventType::RoomMember, - Some(bob().to_string().as_str()), - member_content_join(), - &["CREATE", "IJR", "IPOWER"], - &["IJR"], - ), - to_pdu_event( - "IMC", - charlie(), - TimelineEventType::RoomMember, - Some(charlie().to_string().as_str()), - member_content_join(), - &["CREATE", "IJR", "IPOWER"], - &["IMB"], - ), - to_pdu_event::<&EventId>( - "START", - charlie(), - TimelineEventType::RoomTopic, - Some(""), - to_raw_json_value(&json!({})).unwrap(), - &[], - &[], - ), - to_pdu_event::<&EventId>( - "END", - charlie(), - TimelineEventType::RoomTopic, - Some(""), - to_raw_json_value(&json!({})).unwrap(), - &[], - &[], - ), - ] - .into_iter() - .map(|ev| (ev.event_id().to_owned(), ev)) - .collect() -} - -// all graphs start with these input events -#[allow(non_snake_case)] -fn BAN_STATE_SET() -> HashMap> { - vec![ - to_pdu_event( - "PA", - alice(), - TimelineEventType::RoomPowerLevels, - Some(""), - to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), - &["CREATE", "IMA", "IPOWER"], // auth_events - &["START"], // prev_events - ), - to_pdu_event( - "PB", - alice(), - TimelineEventType::RoomPowerLevels, - Some(""), - to_raw_json_value(&json!({ "users": { alice(): 100, bob(): 50 } })).unwrap(), - &["CREATE", "IMA", "IPOWER"], - &["END"], - ), - to_pdu_event( - "MB", - alice(), - TimelineEventType::RoomMember, - Some(ella().as_str()), - member_content_ban(), - &["CREATE", "IMA", "PB"], - &["PA"], - ), - to_pdu_event( - "IME", - ella(), - TimelineEventType::RoomMember, - Some(ella().as_str()), - member_content_join(), - &["CREATE", "IJR", "PA"], - &["MB"], - ), - ] - .into_iter() - .map(|ev| (ev.event_id().to_owned(), ev)) - .collect() -} - -/// Convenience trait for adding event type plus state key to state maps. -trait EventTypeExt { - fn with_state_key(self, state_key: impl Into) -> (StateEventType, String); -} - -impl EventTypeExt for &TimelineEventType { - fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { - (self.to_string().into(), state_key.into()) - } -} - -mod event { - use ruma_common::{MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId}; - use ruma_events::{pdu::Pdu, TimelineEventType}; - use ruma_state_res::Event; - use serde::{Deserialize, Serialize}; - use serde_json::value::RawValue as RawJsonValue; - - impl Event for PduEvent { - type Id = OwnedEventId; - - fn event_id(&self) -> &Self::Id { - &self.event_id - } - - fn room_id(&self) -> &RoomId { - match &self.rest { - Pdu::RoomV1Pdu(ev) => &ev.room_id, - Pdu::RoomV3Pdu(ev) => &ev.room_id, - #[cfg(not(feature = "unstable-exhaustive-types"))] - _ => unreachable!("new PDU version"), - } - } - - fn sender(&self) -> &UserId { - match &self.rest { - Pdu::RoomV1Pdu(ev) => &ev.sender, - Pdu::RoomV3Pdu(ev) => &ev.sender, - #[cfg(not(feature = "unstable-exhaustive-types"))] - _ => unreachable!("new PDU version"), - } - } - - fn event_type(&self) -> &TimelineEventType { - match &self.rest { - Pdu::RoomV1Pdu(ev) => &ev.kind, - Pdu::RoomV3Pdu(ev) => &ev.kind, - #[cfg(not(feature = "unstable-exhaustive-types"))] - _ => unreachable!("new PDU version"), - } - } - - fn content(&self) -> &RawJsonValue { - match &self.rest { - Pdu::RoomV1Pdu(ev) => &ev.content, - Pdu::RoomV3Pdu(ev) => &ev.content, - #[cfg(not(feature = "unstable-exhaustive-types"))] - _ => unreachable!("new PDU version"), - } - } - - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { - match &self.rest { - Pdu::RoomV1Pdu(ev) => ev.origin_server_ts, - Pdu::RoomV3Pdu(ev) => ev.origin_server_ts, - #[cfg(not(feature = "unstable-exhaustive-types"))] - _ => unreachable!("new PDU version"), - } - } - - fn state_key(&self) -> Option<&str> { - match &self.rest { - Pdu::RoomV1Pdu(ev) => ev.state_key.as_deref(), - Pdu::RoomV3Pdu(ev) => ev.state_key.as_deref(), - #[cfg(not(feature = "unstable-exhaustive-types"))] - _ => unreachable!("new PDU version"), - } - } - - fn prev_events(&self) -> Box + Send + '_> { - match &self.rest { - Pdu::RoomV1Pdu(ev) => Box::new(ev.prev_events.iter().map(|(id, _)| id)), - Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter()), - #[cfg(not(feature = "unstable-exhaustive-types"))] - _ => unreachable!("new PDU version"), - } - } - - fn auth_events(&self) -> Box + Send + '_> { - match &self.rest { - Pdu::RoomV1Pdu(ev) => Box::new(ev.auth_events.iter().map(|(id, _)| id)), - Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter()), - #[cfg(not(feature = "unstable-exhaustive-types"))] - _ => unreachable!("new PDU version"), - } - } - - fn redacts(&self) -> Option<&Self::Id> { - match &self.rest { - Pdu::RoomV1Pdu(ev) => ev.redacts.as_ref(), - Pdu::RoomV3Pdu(ev) => ev.redacts.as_ref(), - #[cfg(not(feature = "unstable-exhaustive-types"))] - _ => unreachable!("new PDU version"), - } - } - } - - #[derive(Clone, Debug, Deserialize, Serialize)] - pub(crate) struct PduEvent { - pub(crate) event_id: OwnedEventId, - #[serde(flatten)] - pub(crate) rest: Pdu, - } -} From 6365f1a887a02564237fd6176ee7e3d72480ffbf Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Fri, 28 Mar 2025 14:14:48 -0400 Subject: [PATCH 1607/2091] remove sccache from ci for now Signed-off-by: June Clementine Strawberry --- .github/workflows/ci.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3fd834e0..5043f23b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,16 +21,6 @@ concurrency: cancel-in-progress: true env: - # sccache only on main repo - SCCACHE_GHA_ENABLED: "${{ !startsWith(github.ref, 'refs/tags/') && (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'true' || 'false' }}" - RUSTC_WRAPPER: "${{ !startsWith(github.ref, 'refs/tags/') && (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}" - SCCACHE_BUCKET: "${{ (github.event.pull_request.draft != true) && (vars.DOCKER_USERNAME != '') && (vars.GITLAB_USERNAME != '') && (vars.SCCACHE_ENDPOINT != '') && (github.event.pull_request.user.login != 'renovate[bot]') && 'sccache' || '' }}" - SCCACHE_S3_USE_SSL: ${{ vars.SCCACHE_S3_USE_SSL }} - SCCACHE_REGION: ${{ vars.SCCACHE_REGION }} - SCCACHE_ENDPOINT: ${{ vars.SCCACHE_ENDPOINT }} - SCCACHE_CACHE_MULTIARCH: ${{ vars.SCCACHE_CACHE_MULTIARCH }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} # Required to make some things output color TERM: ansi # Publishing to my nix binary cache @@ -123,13 +113,6 @@ jobs: bin/nix-build-and-cache just '.#devShells.x86_64-linux.all-features' bin/nix-build-and-cache just '.#devShells.x86_64-linux.dynamic' - # use sccache for Rust - - name: Run sccache-cache - # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting - # releases and tags - #if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} - uses: mozilla-actions/sccache-action@main - # use rust-cache - uses: Swatinem/rust-cache@v2 # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting @@ -247,13 +230,6 @@ jobs: direnv allow nix develop .#all-features --command true --impure - # use sccache for Rust - - name: Run sccache-cache - # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting - # releases and tags - #if: ${{ (env.SCCACHE_GHA_ENABLED == 'true') && !startsWith(github.ref, 'refs/tags/') }} - uses: mozilla-actions/sccache-action@main - # use rust-cache - uses: Swatinem/rust-cache@v2 # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting From 75b6daa67f31d29035113d217accc80505119e63 Mon Sep 17 00:00:00 2001 From: Ginger <75683114+gingershaped@users.noreply.github.com> Date: Fri, 28 Mar 2025 12:22:23 -0400 Subject: [PATCH 1608/2091] Fix off-by-one error when fetching room hierarchy --- src/api/client/space.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/space.rs b/src/api/client/space.rs index a667f852..a55c927d 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -159,7 +159,7 @@ where break; } - if parents.len() >= max_depth { + if parents.len() > max_depth { continue; } From 3e57b7d35d5bd6cfed5900b377f7c68970213518 Mon Sep 17 00:00:00 2001 From: Ginger <75683114+gingershaped@users.noreply.github.com> Date: Fri, 28 Mar 2025 12:58:18 -0400 Subject: [PATCH 1609/2091] Update expected test results --- tests/test_results/complement/test_results.jsonl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index ac2733f8..c0e28750 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -73,7 +73,7 @@ {"Action":"pass","Test":"TestChangePasswordPushers/Pushers_created_with_a_different_access_token_are_deleted_on_password_change"} {"Action":"pass","Test":"TestChangePasswordPushers/Pushers_created_with_the_same_access_token_are_not_deleted_on_password_change"} {"Action":"fail","Test":"TestClientSpacesSummary"} -{"Action":"fail","Test":"TestClientSpacesSummary/max_depth"} +{"Action":"pass","Test":"TestClientSpacesSummary/max_depth"} {"Action":"fail","Test":"TestClientSpacesSummary/pagination"} {"Action":"fail","Test":"TestClientSpacesSummary/query_whole_graph"} {"Action":"fail","Test":"TestClientSpacesSummary/redact_link"} From 0e2009dbf5c3dfe1cfd1fd37078c74e871ffa5c6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 28 Mar 2025 22:47:00 +0000 Subject: [PATCH 1610/2091] fix client hierarchy loop condition Signed-off-by: Jason Volk --- src/api/client/space.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/api/client/space.rs b/src/api/client/space.rs index a55c927d..567ac62f 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -155,10 +155,6 @@ where break; } - if children.is_empty() { - break; - } - if parents.len() > max_depth { continue; } From d0132706cd9b5bd0c6df5507cb42bcbade86f28b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 28 Mar 2025 23:34:42 +0000 Subject: [PATCH 1611/2091] add --read-only and --maintenance program option Signed-off-by: Jason Volk --- Cargo.toml | 1 + src/main/clap.rs | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index db55b9b8..8b49c3b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -892,6 +892,7 @@ needless_continue = { level = "allow", priority = 1 } no_effect_underscore_binding = { level = "allow", priority = 1 } similar_names = { level = "allow", priority = 1 } single_match_else = { level = "allow", priority = 1 } +struct_excessive_bools = { level = "allow", priority = 1 } struct_field_names = { level = "allow", priority = 1 } unnecessary_wraps = { level = "allow", priority = 1 } unused_async = { level = "allow", priority = 1 } diff --git a/src/main/clap.rs b/src/main/clap.rs index 35a7ea41..707a1c76 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -27,6 +27,14 @@ pub(crate) struct Args { #[arg(long, short('O'))] pub(crate) option: Vec, + /// Run in a stricter read-only --maintenance mode. + #[arg(long)] + pub(crate) read_only: bool, + + /// Run in maintenance mode while refusing connections. + #[arg(long)] + pub(crate) maintenance: bool, + #[cfg(feature = "console")] /// Activate admin command console automatically after startup. #[arg(long, num_args(0))] @@ -121,6 +129,15 @@ pub(super) fn parse() -> Args { Args::parse() } /// Synthesize any command line options with configuration file options. pub(crate) fn update(mut config: Figment, args: &Args) -> Result { + if args.read_only { + config = config.join(("rocksdb_read_only", true)); + } + + if args.maintenance || args.read_only { + config = config.join(("startup_netburst", false)); + config = config.join(("listening", false)); + } + #[cfg(feature = "console")] // Indicate the admin console should be spawned automatically if the // configuration file hasn't already. From b03c493bf994449c8c5dd5b1122ab9c87a289df5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 28 Mar 2025 20:33:38 +0000 Subject: [PATCH 1612/2091] add stub for database benches Signed-off-by: Jason Volk --- src/database/benches.rs | 17 +++++++++++++++++ src/database/mod.rs | 2 ++ 2 files changed, 19 insertions(+) create mode 100644 src/database/benches.rs diff --git a/src/database/benches.rs b/src/database/benches.rs new file mode 100644 index 00000000..56d1411c --- /dev/null +++ b/src/database/benches.rs @@ -0,0 +1,17 @@ +#[cfg(conduwuit_bench)] +extern crate test; + +#[cfg(conduwuit_bench)] +#[cfg_attr(conduwuit_bench, bench)] +fn ser_str(b: &mut test::Bencher) { + use conduwuit::ruma::{RoomId, UserId}; + + use crate::ser::serialize_to_vec; + + let user_id: &UserId = "@user:example.com".try_into().unwrap(); + let room_id: &RoomId = "!room:example.com".try_into().unwrap(); + b.iter(|| { + let key = (user_id, room_id); + let _s = serialize_to_vec(key).expect("failed to serialize user_id"); + }); +} diff --git a/src/database/mod.rs b/src/database/mod.rs index 0481d1bd..1262a79a 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -5,6 +5,8 @@ conduwuit::mod_ctor! {} conduwuit::mod_dtor! {} conduwuit::rustc_flags_capture! {} +#[cfg(test)] +mod benches; mod cork; mod de; mod deserialized; From a93cb34dd6e10038d6504af209c78e4967467bcb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 30 Mar 2025 02:48:08 +0000 Subject: [PATCH 1613/2091] disambiguate UInt/u64 type related in client/api/directory; use err macros. Signed-off-by: Jason Volk --- src/api/client/directory.rs | 86 +++++++++++++------------------------ 1 file changed, 30 insertions(+), 56 deletions(-) diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 80b314b9..f2f668c8 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,12 +1,13 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Err, Error, Result, info, + Err, Result, err, info, utils::{ TryFutureExtExt, + math::Expected, + result::FlatOk, stream::{ReadyExt, WidebandExt}, }, - warn, }; use futures::{ FutureExt, StreamExt, TryFutureExt, @@ -20,7 +21,6 @@ use ruma::{ get_public_rooms, get_public_rooms_filtered, get_room_visibility, set_room_visibility, }, - error::ErrorKind, room, }, federation, @@ -71,11 +71,7 @@ pub(crate) async fn get_public_rooms_filtered_route( ) .await .map_err(|e| { - warn!(?body.server, "Failed to return /publicRooms: {e}"); - Error::BadRequest( - ErrorKind::Unknown, - "Failed to return the requested server's public room list.", - ) + err!(Request(Unknown(warn!(?body.server, "Failed to return /publicRooms: {e}")))) })?; Ok(response) @@ -113,11 +109,7 @@ pub(crate) async fn get_public_rooms_route( ) .await .map_err(|e| { - warn!(?body.server, "Failed to return /publicRooms: {e}"); - Error::BadRequest( - ErrorKind::Unknown, - "Failed to return the requested server's public room list.", - ) + err!(Request(Unknown(warn!(?body.server, "Failed to return /publicRooms: {e}")))) })?; Ok(get_public_rooms::v3::Response { @@ -137,7 +129,7 @@ pub(crate) async fn set_room_visibility_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if !services.rooms.metadata.exists(&body.room_id).await { // Return 404 if the room doesn't exist @@ -181,10 +173,9 @@ pub(crate) async fn set_room_visibility_route( .await; } - return Err(Error::BadRequest( - ErrorKind::forbidden(), + return Err!(Request(Forbidden( "Publishing rooms to the room directory is not allowed", - )); + ))); } services.rooms.directory.set_public(&body.room_id); @@ -202,10 +193,7 @@ pub(crate) async fn set_room_visibility_route( }, | room::Visibility::Private => services.rooms.directory.set_not_public(&body.room_id), | _ => { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Room visibility type is not supported.", - )); + return Err!(Request(InvalidParam("Room visibility type is not supported.",))); }, } @@ -221,7 +209,7 @@ pub(crate) async fn get_room_visibility_route( ) -> Result { if !services.rooms.metadata.exists(&body.room_id).await { // Return 404 if the room doesn't exist - return Err(Error::BadRequest(ErrorKind::NotFound, "Room not found")); + return Err!(Request(NotFound("Room not found"))); } Ok(get_room_visibility::v3::Response { @@ -269,8 +257,8 @@ pub(crate) async fn get_public_rooms_filtered_helper( } // Use limit or else 10, with maximum 100 - let limit = limit.map_or(10, u64::from); - let mut num_since: u64 = 0; + let limit: usize = limit.map_or(10_u64, u64::from).try_into()?; + let mut num_since: usize = 0; if let Some(s) = &since { let mut characters = s.chars(); @@ -278,14 +266,14 @@ pub(crate) async fn get_public_rooms_filtered_helper( | Some('n') => false, | Some('p') => true, | _ => { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Invalid `since` token")); + return Err!(Request(InvalidParam("Invalid `since` token"))); }, }; num_since = characters .collect::() .parse() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid `since` token."))?; + .map_err(|_| err!(Request(InvalidParam("Invalid `since` token."))))?; if backwards { num_since = num_since.saturating_sub(limit); @@ -302,6 +290,7 @@ pub(crate) async fn get_public_rooms_filtered_helper( if !filter.room_types.is_empty() && !filter.room_types.contains(&RoomTypeFilter::from(chunk.room_type.clone())) { return None; } + if let Some(query) = filter.generic_search_term.as_ref().map(|q| q.to_lowercase()) { if let Some(name) = &chunk.name { if name.as_str().to_lowercase().contains(&query) { @@ -333,40 +322,24 @@ pub(crate) async fn get_public_rooms_filtered_helper( all_rooms.sort_by(|l, r| r.num_joined_members.cmp(&l.num_joined_members)); - let total_room_count_estimate = UInt::try_from(all_rooms.len()).unwrap_or_else(|_| uint!(0)); + let total_room_count_estimate = UInt::try_from(all_rooms.len()) + .unwrap_or_else(|_| uint!(0)) + .into(); - let chunk: Vec<_> = all_rooms - .into_iter() - .skip( - num_since - .try_into() - .expect("num_since should not be this high"), - ) - .take(limit.try_into().expect("limit should not be this high")) - .collect(); + let chunk: Vec<_> = all_rooms.into_iter().skip(num_since).take(limit).collect(); - let prev_batch = if num_since == 0 { - None - } else { - Some(format!("p{num_since}")) - }; + let prev_batch = num_since.ne(&0).then_some(format!("p{num_since}")); - let next_batch = if chunk.len() < limit.try_into().unwrap() { - None - } else { - Some(format!( - "n{}", - num_since - .checked_add(limit) - .expect("num_since and limit should not be that large") - )) - }; + let next_batch = chunk + .len() + .ge(&limit) + .then_some(format!("n{}", num_since.expected_add(limit))); Ok(get_public_rooms_filtered::v3::Response { chunk, prev_batch, next_batch, - total_room_count_estimate: Some(total_room_count_estimate), + total_room_count_estimate, }) } @@ -384,7 +357,7 @@ async fn user_can_publish_room( .await { | Ok(event) => serde_json::from_str(event.content.get()) - .map_err(|_| Error::bad_database("Invalid event content for m.room.power_levels")) + .map_err(|_| err!(Database("Invalid event content for m.room.power_levels"))) .map(|content: RoomPowerLevelsEventContent| { RoomPowerLevels::from(content) .user_can_send_state(user_id, StateEventType::RoomHistoryVisibility) @@ -452,9 +425,10 @@ async fn public_rooms_chunk(services: &Services, room_id: OwnedRoomId) -> Public join_rule: join_rule.unwrap_or_default(), name, num_joined_members: num_joined_members - .unwrap_or(0) - .try_into() - .expect("joined count overflows ruma UInt"), + .map(TryInto::try_into) + .map(Result::ok) + .flat_ok() + .unwrap_or_else(|| uint!(0)), room_id, room_type, topic, From 095734a8e7835abf793911ff24ddf0f55c89012f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 30 Mar 2025 03:01:58 +0000 Subject: [PATCH 1614/2091] bump tokio to 1.44.1 Signed-off-by: Jason Volk --- Cargo.lock | 559 +++++++++++++++++++++++++++-------------------------- Cargo.toml | 2 +- 2 files changed, 291 insertions(+), 270 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c28f4eab..8c0e797b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,9 +55,9 @@ checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anyhow" -version = "1.0.96" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b964d184e89d9b6b67dd2715bc8e74cf3107fb2b529990c90cf517326150bf4" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" [[package]] name = "arbitrary" @@ -79,7 +79,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -105,9 +105,9 @@ dependencies = [ [[package]] name = "as_variant" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38fa22307249f86fb7fad906fcae77f2564caeb56d7209103c551cd1cf4798f" +checksum = "9dbc3a507a82b17ba0d98f6ce8fd6954ea0c8152e98009d36a40d8dcc8ce078a" [[package]] name = "assign" @@ -128,9 +128,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.20" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "310c9bcae737a48ef5cdee3174184e6d548b292739ede61a1f955ef76a738861" +checksum = "59a194f9d963d8099596278594b3107448656ba73831c9d8c783e613ce86da64" dependencies = [ "brotli", "flate2", @@ -161,18 +161,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "async-trait" -version = "0.1.86" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -221,27 +221,25 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.12.5" +version = "1.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e4e8200b9a4a5801a769d50eeabc05670fec7e959a8cb7a63a93e4e519942ae" +checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" dependencies = [ "aws-lc-sys", - "paste", "zeroize", ] [[package]] name = "aws-lc-sys" -version = "0.26.0" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9dd2e03ee80ca2822dd6ea431163d2ef259f2066a4d6ccaca6d9dcb386aa43" +checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f" dependencies = [ "bindgen 0.69.5", "cc", "cmake", "dunce", "fs_extra", - "paste", ] [[package]] @@ -334,16 +332,15 @@ dependencies = [ [[package]] name = "axum-server" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56bac90848f6a9393ac03c63c640925c4b7c8ca21654de40d53f55964667c7d8" +checksum = "495c05f60d6df0093e8fb6e74aa5846a0ad06abaf96d76166283720bf740f8ab" dependencies = [ "arc-swap", "bytes", - "futures-util", + "fs-err", "http", "http-body", - "http-body-util", "hyper", "hyper-util", "pin-project-lite", @@ -352,7 +349,6 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls", - "tower 0.4.13", "tower-service", ] @@ -404,9 +400,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.6.0" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" [[package]] name = "bindgen" @@ -427,7 +423,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.98", + "syn 2.0.100", "which", ] @@ -446,7 +442,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -535,9 +531,9 @@ checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" -version = "1.21.0" +version = "1.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef657dfab802224e671f5818e9a4935f9b1957ed18e58292690cc39e7a4092a3" +checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" [[package]] name = "byteorder" @@ -553,9 +549,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" [[package]] name = "bytes" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "bytesize" @@ -585,9 +581,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.16" +version = "1.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" dependencies = [ "jobserver", "libc", @@ -656,9 +652,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.31" +version = "4.5.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" +checksum = "e958897981290da2a852763fe9cdb89cd36977a5d729023127095fa94d95e2ff" dependencies = [ "clap_builder", "clap_derive", @@ -666,9 +662,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.31" +version = "4.5.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" +checksum = "83b0f35019843db2160b5bb19ae09b4e6411ac33fc6a712003c33e03090e2489" dependencies = [ "anstyle", "clap_lex", @@ -676,14 +672,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.28" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -787,7 +783,7 @@ dependencies = [ "ipaddress", "itertools 0.13.0", "log", - "rand", + "rand 0.8.5", "reqwest", "ruma", "serde", @@ -830,7 +826,7 @@ dependencies = [ "maplit", "nix", "num-traits", - "rand", + "rand 0.8.5", "regex", "reqwest", "ring", @@ -842,7 +838,7 @@ dependencies = [ "serde_yaml", "smallstr", "smallvec", - "thiserror 2.0.11", + "thiserror 2.0.12", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -880,7 +876,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -937,7 +933,7 @@ dependencies = [ "log", "loole", "lru-cache", - "rand", + "rand 0.8.5", "regex", "reqwest", "ruma", @@ -1194,7 +1190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1221,7 +1217,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1264,9 +1260,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" dependencies = [ "powerfmt", ] @@ -1290,7 +1286,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1317,7 +1313,7 @@ checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ "curve25519-dalek", "ed25519", - "rand_core", + "rand_core 0.6.4", "serde", "sha2", "subtle", @@ -1326,9 +1322,9 @@ dependencies = [ [[package]] name = "either" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" dependencies = [ "serde", ] @@ -1342,7 +1338,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1373,9 +1369,9 @@ dependencies = [ [[package]] name = "event-listener-strategy" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3e4e0dd3673c1139bf041f3008816d9cf2946bbfac2945c09e523b8d7b05b2" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ "event-listener", "pin-project-lite", @@ -1472,6 +1468,16 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "fs-err" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f89bda4c2a21204059a977ed3bfe746677dfd137b83c339e702b0ac91d482aa" +dependencies = [ + "autocfg", + "tokio", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1543,7 +1549,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1601,14 +1607,16 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", + "js-sys", "libc", - "wasi 0.13.3+wasi-0.2.2", - "windows-targets 0.52.6", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1645,7 +1653,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.7.1", + "indexmap 2.8.0", "slab", "tokio", "tokio-util", @@ -1654,9 +1662,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1" dependencies = [ "cfg-if", "crunchy", @@ -1751,7 +1759,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand", + "rand 0.8.5", "thiserror 1.0.69", "tinyvec", "tokio", @@ -1772,7 +1780,7 @@ dependencies = [ "lru-cache", "once_cell", "parking_lot", - "rand", + "rand 0.8.5", "resolv-conf", "smallvec", "thiserror 1.0.69", @@ -1798,17 +1806,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "hostname" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c731c3e10504cc8ed35cfe2f1db4c9274c3d35fa486e3b31df46f068ef3e867" -dependencies = [ - "libc", - "match_cfg", - "winapi", -] - [[package]] name = "hostname" version = "0.4.0" @@ -1831,14 +1828,14 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "http" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -1866,12 +1863,12 @@ dependencies = [ [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", + "futures-core", "http", "http-body", "pin-project-lite", @@ -1879,9 +1876,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" @@ -1891,9 +1888,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "hyper" @@ -2009,9 +2006,9 @@ dependencies = [ [[package]] name = "icu_locid_transform_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" [[package]] name = "icu_normalizer" @@ -2033,9 +2030,9 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" [[package]] name = "icu_properties" @@ -2054,9 +2051,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" [[package]] name = "icu_provider" @@ -2083,7 +2080,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2109,9 +2106,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.5" +version = "0.25.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd6f44aed642f18953a158afeb30206f4d50da59fbc66ecb53c66488de73563b" +checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a" dependencies = [ "bytemuck", "byteorder-lite", @@ -2137,7 +2134,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b77d01e822461baa8409e156015a1d91735549f0f2c17691bd2d996bef238f7f" dependencies = [ "byteorder-lite", - "quick-error 2.0.1", + "quick-error", ] [[package]] @@ -2158,9 +2155,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -2187,7 +2184,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2251,9 +2248,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jobserver" @@ -2338,7 +2335,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2361,9 +2358,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.170" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libfuzzer-sys" @@ -2387,9 +2384,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.21" +version = "1.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df9b68e50e6e0b26f672573834882eb57759f6db9b3be2ea3c35c91188bb4eaa" +checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" dependencies = [ "cc", "pkg-config", @@ -2426,9 +2423,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.26" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "loole" @@ -2506,12 +2503,6 @@ dependencies = [ "xml5ever", ] -[[package]] -name = "match_cfg" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" - [[package]] name = "matchers" version = "0.1.0" @@ -2566,7 +2557,7 @@ checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2713,7 +2704,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2777,9 +2768,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "openssl-probe" @@ -2795,7 +2786,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.7.1", + "indexmap 2.8.0", "js-sys", "once_cell", "pin-project-lite", @@ -2844,7 +2835,7 @@ dependencies = [ "opentelemetry", "ordered-float 4.6.0", "percent-encoding", - "rand", + "rand 0.8.5", "thiserror 1.0.69", "tokio", "tokio-stream", @@ -2921,7 +2912,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ "base64ct", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -2951,7 +2942,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2986,7 +2977,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -3000,22 +2991,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3042,9 +3033,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "png" @@ -3067,9 +3058,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ "zerocopy", ] @@ -3082,28 +3073,28 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" +checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" dependencies = [ "proc-macro2", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "proc-macro-crate" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" dependencies = [ "toml_edit", ] [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] @@ -3116,7 +3107,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "version_check", "yansi", ] @@ -3137,7 +3128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" dependencies = [ "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3160,7 +3151,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3199,12 +3190,6 @@ dependencies = [ "bytemuck", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - [[package]] name = "quick-error" version = "2.0.1" @@ -3213,37 +3198,39 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quinn" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" dependencies = [ "bytes", + "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", "rustls", "socket2", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "tracing", + "web-time 1.1.0", ] [[package]] name = "quinn-proto" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" dependencies = [ "bytes", - "getrandom 0.2.15", - "rand", + "getrandom 0.3.2", + "rand 0.9.0", "ring", "rustc-hash 2.1.1", "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.11", + "thiserror 2.0.12", "tinyvec", "tracing", "web-time 1.1.0", @@ -3251,9 +3238,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944" +checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" dependencies = [ "cfg_aliases", "libc", @@ -3265,13 +3252,19 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "rand" version = "0.8.5" @@ -3279,8 +3272,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy", ] [[package]] @@ -3290,7 +3294,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -3302,6 +3316,15 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.2", +] + [[package]] name = "rav1e" version = "0.7.1" @@ -3328,8 +3351,8 @@ dependencies = [ "once_cell", "paste", "profiling", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "simd_helpers", "system-deps", "thiserror 1.0.69", @@ -3346,7 +3369,7 @@ dependencies = [ "avif-serialize", "imgref", "loop9", - "quick-error 2.0.1", + "quick-error", "rav1e", "rayon", "rgb", @@ -3374,9 +3397,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ "bitflags 2.9.0", ] @@ -3476,12 +3499,11 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +checksum = "48375394603e3dd4b2d64371f7148fd8c7baa2680e28741f2cb8d23b59e3d4c4" dependencies = [ - "hostname 0.3.1", - "quick-error 1.2.3", + "hostname", ] [[package]] @@ -3492,9 +3514,9 @@ checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" [[package]] name = "ring" -version = "0.17.12" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed9b823fa29b721a59671b41d6b06e66b29e0628e207e8b1c3ceeda701ec928d" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", @@ -3554,7 +3576,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror 2.0.11", + "thiserror 2.0.12", "url", "web-time 1.1.0", ] @@ -3570,11 +3592,11 @@ dependencies = [ "form_urlencoded", "getrandom 0.2.15", "http", - "indexmap 2.7.1", + "indexmap 2.8.0", "js_int", "konst", "percent-encoding", - "rand", + "rand 0.8.5", "regex", "ruma-identifiers-validation", "ruma-macros", @@ -3582,7 +3604,7 @@ dependencies = [ "serde_html_form", "serde_json", "smallvec", - "thiserror 2.0.11", + "thiserror 2.0.12", "time", "tracing", "url", @@ -3597,7 +3619,7 @@ version = "0.28.1" source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "as_variant", - "indexmap 2.7.1", + "indexmap 2.8.0", "js_int", "js_option", "percent-encoding", @@ -3609,7 +3631,7 @@ dependencies = [ "serde", "serde_json", "smallvec", - "thiserror 2.0.11", + "thiserror 2.0.12", "tracing", "url", "web-time 1.1.0", @@ -3629,12 +3651,12 @@ dependencies = [ "js_int", "memchr", "mime", - "rand", + "rand 0.8.5", "ruma-common", "ruma-events", "serde", "serde_json", - "thiserror 2.0.11", + "thiserror 2.0.12", "tracing", ] @@ -3644,7 +3666,7 @@ version = "0.9.5" source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" dependencies = [ "js_int", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] @@ -3668,7 +3690,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.98", + "syn 2.0.100", "toml", ] @@ -3692,12 +3714,12 @@ dependencies = [ "base64 0.22.1", "ed25519-dalek", "pkcs8", - "rand", + "rand 0.8.5", "ruma-common", "serde_json", "sha2", "subslice", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] @@ -3768,9 +3790,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.23" +version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" +checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ "aws-lc-rs", "log", @@ -3814,9 +3836,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ "aws-lc-rs", "ring", @@ -3826,9 +3848,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] name = "rustyline-async" @@ -3840,16 +3862,16 @@ dependencies = [ "futures-util", "pin-project", "thingbuf", - "thiserror 2.0.11", + "thiserror 2.0.12", "unicode-segmentation", "unicode-width 0.2.0", ] [[package]] name = "ryu" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "sanitize-filename" @@ -3909,9 +3931,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "sentry" @@ -3953,7 +3975,7 @@ version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba8754ec3b9279e00aa6d64916f211d44202370a1699afde1db2c16cbada089" dependencies = [ - "hostname 0.4.0", + "hostname", "libc", "os_info", "rustc_version", @@ -3968,7 +3990,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8b6dcd4fbae1e3e22b447f32670360b27e31b62ab040f7fb04e0f80c04d92" dependencies = [ "once_cell", - "rand", + "rand 0.8.5", "sentry-types", "serde", "serde_json", @@ -4039,7 +4061,7 @@ checksum = "a71ed3a389948a6a6d92b98e997a2723ca22f09660c5a7b7388ecd509a70a527" dependencies = [ "debugid", "hex", - "rand", + "rand 0.8.5", "serde", "serde_json", "thiserror 1.0.69", @@ -4050,22 +4072,22 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4075,7 +4097,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", - "indexmap 2.7.1", + "indexmap 2.8.0", "itoa", "ryu", "serde", @@ -4083,9 +4105,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.139" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", @@ -4095,9 +4117,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" +checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" dependencies = [ "itoa", "serde", @@ -4140,7 +4162,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.7.1", + "indexmap 2.8.0", "itoa", "ryu", "serde", @@ -4220,7 +4242,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -4306,9 +4328,9 @@ checksum = "f42444fea5b87a39db4218d9422087e66a85d0e7a0963a439b07bcdf91804006" [[package]] name = "string_cache" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938d512196766101d333398efde81bc1f37b00cb42c2f8350e5df639f040bbbe" +checksum = "bf776ba3fa74f83bf4b63c3dcbbf82173db2632ed8452cb2d891d33f459de70f" dependencies = [ "new_debug_unreachable", "parking_lot", @@ -4357,9 +4379,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.98" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", @@ -4383,7 +4405,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4453,11 +4475,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.11", + "thiserror-impl 2.0.12", ] [[package]] @@ -4468,18 +4490,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4555,9 +4577,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.37" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", "itoa", @@ -4570,15 +4592,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.19" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -4596,9 +4618,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -4611,9 +4633,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.43.0" +version = "1.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" dependencies = [ "backtrace", "bytes", @@ -4635,7 +4657,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4685,9 +4707,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" dependencies = [ "bytes", "futures-core", @@ -4723,7 +4745,7 @@ version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ - "indexmap 2.7.1", + "indexmap 2.8.0", "serde", "serde_spanned", "toml_datetime", @@ -4771,7 +4793,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util", @@ -4835,7 +4857,6 @@ name = "tracing" version = "0.1.41" source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -4848,7 +4869,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4969,9 +4990,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-segmentation" @@ -5056,11 +5077,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.15.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.2", "serde", ] @@ -5116,9 +5137,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.13.3+wasi-0.2.2" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] @@ -5145,7 +5166,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "wasm-bindgen-shared", ] @@ -5180,7 +5201,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5265,9 +5286,9 @@ dependencies = [ [[package]] name = "widestring" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" +checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" [[package]] name = "wildmatch" @@ -5496,9 +5517,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1" +checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" dependencies = [ "memchr", ] @@ -5515,9 +5536,9 @@ dependencies = [ [[package]] name = "wit-bindgen-rt" -version = "0.33.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ "bitflags 2.9.0", ] @@ -5571,29 +5592,28 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "synstructure", ] [[package]] name = "zerocopy" -version = "0.7.35" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" dependencies = [ - "byteorder", "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.35" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5613,7 +5633,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "synstructure", ] @@ -5642,7 +5662,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5656,19 +5676,20 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.3" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3051792fbdc2e1e143244dc28c60f73d8470e93f3f9cbd0ead44da5ed802722" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.14+zstd.1.5.7" +version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb060d4926e4ac3a3ad15d864e99ceb5f343c6b34f5bd6d81ae6ed417311be5" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ + "bindgen 0.71.1", "cc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index 8b49c3b8..ab7a935c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -242,7 +242,7 @@ default-features = false features = ["std", "async-await"] [workspace.dependencies.tokio] -version = "1.42.0" +version = "1.44.1" default-features = false features = [ "fs", From 5bf5afaec83d4e68cbfd5220cd760a7940e7dda5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 30 Mar 2025 01:54:55 +0000 Subject: [PATCH 1615/2091] instrument tokio before/after poll hooks Signed-off-by: Jason Volk --- src/main/runtime.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/main/runtime.rs b/src/main/runtime.rs index b1657289..920476db 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -61,6 +61,8 @@ pub(super) fn new(args: &Args) -> Result { #[cfg(tokio_unstable)] builder .on_task_spawn(task_spawn) + .on_before_task_poll(task_enter) + .on_after_task_poll(task_leave) .on_task_terminate(task_terminate); #[cfg(tokio_unstable)] @@ -215,3 +217,25 @@ fn task_spawn(meta: &tokio::runtime::TaskMeta<'_>) {} ), )] fn task_terminate(meta: &tokio::runtime::TaskMeta<'_>) {} + +#[cfg(tokio_unstable)] +#[tracing::instrument( + name = "enter", + level = "trace", + skip_all, + fields( + id = %meta.id() + ), +)] +fn task_enter(meta: &tokio::runtime::TaskMeta<'_>) {} + +#[cfg(tokio_unstable)] +#[tracing::instrument( + name = "leave", + level = "trace", + skip_all, + fields( + id = %meta.id() + ), +)] +fn task_leave(meta: &tokio::runtime::TaskMeta<'_>) {} From dc6e9e74d9e9fb0bbdddb35c6b00d16544860095 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 30 Mar 2025 01:56:00 +0000 Subject: [PATCH 1616/2091] add spans for for jemalloc mallctl points Signed-off-by: Jason Volk --- src/core/alloc/je.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 51caf3a3..2424e99c 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -335,6 +335,12 @@ where Ok(res) } +#[tracing::instrument( + name = "get", + level = "trace" + skip_all, + fields(?key) +)] fn get(key: &Key) -> Result where T: Copy + Debug, @@ -346,6 +352,12 @@ where unsafe { mallctl::raw::read_mib(key.as_slice()) }.map_err(map_err) } +#[tracing::instrument( + name = "xchg", + level = "trace" + skip_all, + fields(?key, ?val) +)] fn xchg(key: &Key, val: T) -> Result where T: Copy + Debug, From bee4c6255a815a9c7bc577d7afa66f69e26ea735 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 30 Mar 2025 21:19:47 +0000 Subject: [PATCH 1617/2091] reorg PduEvent strip tools and callsites Signed-off-by: Jason Volk --- src/api/client/context.rs | 6 +- src/api/client/message.rs | 2 +- src/api/client/room/event.rs | 2 +- src/api/client/room/initial_sync.rs | 2 +- src/api/client/search.rs | 2 +- src/api/client/sync/v3.rs | 8 +- src/api/client/sync/v4.rs | 4 +- src/api/client/sync/v5.rs | 4 +- src/api/client/threads.rs | 2 +- src/core/pdu/strip.rs | 182 ++++++++++++++++++++-------- src/service/rooms/spaces/mod.rs | 5 +- src/service/rooms/state/mod.rs | 2 +- src/service/sending/sender.rs | 2 +- 13 files changed, 152 insertions(+), 71 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index b109711e..1dda7b53 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -182,7 +182,7 @@ pub(crate) async fn get_context_route( .await; Ok(get_context::v3::Response { - event: base_event.map(at!(1)).as_ref().map(PduEvent::to_room_event), + event: base_event.map(at!(1)).map(PduEvent::into_room_event), start: events_before .last() @@ -201,13 +201,13 @@ pub(crate) async fn get_context_route( events_before: events_before .into_iter() .map(at!(1)) - .map(|pdu| pdu.to_room_event()) + .map(PduEvent::into_room_event) .collect(), events_after: events_after .into_iter() .map(at!(1)) - .map(|pdu| pdu.to_room_event()) + .map(PduEvent::into_room_event) .collect(), state, diff --git a/src/api/client/message.rs b/src/api/client/message.rs index c755cc47..03c7335a 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -157,7 +157,7 @@ pub(crate) async fn get_message_events_route( let chunk = events .into_iter() .map(at!(1)) - .map(|pdu| pdu.to_room_event()) + .map(PduEvent::into_room_event) .collect(); Ok(get_message_events::v3::Response { diff --git a/src/api/client/room/event.rs b/src/api/client/room/event.rs index 84b591cd..2b115b5c 100644 --- a/src/api/client/room/event.rs +++ b/src/api/client/room/event.rs @@ -40,5 +40,5 @@ pub(crate) async fn get_room_event_route( event.add_age().ok(); - Ok(get_room_event::v3::Response { event: event.to_room_event() }) + Ok(get_room_event::v3::Response { event: event.into_room_event() }) } diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index e4c76ae0..ca63610b 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -55,7 +55,7 @@ pub(crate) async fn room_initial_sync_route( chunk: events .into_iter() .map(at!(1)) - .map(|pdu| pdu.to_room_event()) + .map(PduEvent::into_room_event) .collect(), }; diff --git a/src/api/client/search.rs b/src/api/client/search.rs index f3366843..d66df881 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -143,7 +143,7 @@ async fn category_room_events( .map(at!(2)) .flatten() .stream() - .map(|pdu| pdu.to_room_event()) + .map(PduEvent::into_room_event) .map(|result| SearchResult { rank: None, result: Some(result), diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 70c4c6a7..a8248f95 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -461,7 +461,7 @@ async fn handle_left_room( events: Vec::new(), }, state: RoomState { - events: vec![event.to_sync_state_event()], + events: vec![event.into_sync_state_event()], }, })); } @@ -546,7 +546,7 @@ async fn handle_left_room( continue; } - left_state_events.push(pdu.to_sync_state_event()); + left_state_events.push(pdu.into_sync_state_event()); } } @@ -865,8 +865,8 @@ async fn load_joined_room( }, state: RoomState { events: state_events - .iter() - .map(PduEvent::to_sync_state_event) + .into_iter() + .map(PduEvent::into_sync_state_event) .collect(), }, ephemeral: Ephemeral { events: edus }, diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 5fdcbab8..7e902973 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -6,7 +6,7 @@ use std::{ use axum::extract::State; use conduwuit::{ - Error, PduCount, Result, debug, error, extract_variant, + Error, PduCount, PduEvent, Result, debug, error, extract_variant, utils::{ BoolExt, IterStream, ReadyExt, TryFutureExtExt, math::{ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, @@ -634,7 +634,7 @@ pub(crate) async fn sync_events_v4_route( .state_accessor .room_state_get(room_id, &state.0, &state.1) .await - .map(|s| s.to_sync_state_event()) + .map(PduEvent::into_sync_state_event) .ok() }) .collect() diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index b4c1b815..48b41b21 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -6,7 +6,7 @@ use std::{ use axum::extract::State; use conduwuit::{ - Error, Result, TypeStateKey, debug, error, extract_variant, trace, + Error, PduEvent, Result, TypeStateKey, debug, error, extract_variant, trace, utils::{ BoolExt, IterStream, ReadyExt, TryFutureExtExt, math::{ruma_from_usize, usize_from_ruma}, @@ -507,7 +507,7 @@ async fn process_rooms( .state_accessor .room_state_get(room_id, &state.0, &state.1) .await - .map(|s| s.to_sync_state_event()) + .map(PduEvent::into_sync_state_event) .ok() }) .collect() diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index d25e52c0..00bfe553 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -53,7 +53,7 @@ pub(crate) async fn get_threads_route( chunk: threads .into_iter() .map(at!(1)) - .map(|pdu| pdu.to_room_event()) + .map(PduEvent::into_room_event) .collect(), }) } diff --git a/src/core/pdu/strip.rs b/src/core/pdu/strip.rs index 4e7c5b83..3683caaa 100644 --- a/src/core/pdu/strip.rs +++ b/src/core/pdu/strip.rs @@ -10,35 +10,18 @@ use serde_json::{json, value::Value as JsonValue}; use crate::implement; -#[must_use] -#[implement(super::Pdu)] -pub fn to_sync_room_event(&self) -> Raw { - let (redacts, content) = self.copy_redacts(); - let mut json = json!({ - "content": content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &redacts { - json["redacts"] = json!(redacts); - } - - serde_json::from_value(json).expect("Raw::from_value always works") -} - /// This only works for events that are also AnyRoomEvents. #[must_use] #[implement(super::Pdu)] -pub fn to_any_event(&self) -> Raw { +pub fn into_any_event(self) -> Raw { + serde_json::from_value(self.into_any_event_value()).expect("Raw::from_value always works") +} + +/// This only works for events that are also AnyRoomEvents. +#[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn into_any_event_value(self) -> JsonValue { let (redacts, content) = self.copy_redacts(); let mut json = json!({ "content": content, @@ -59,12 +42,24 @@ pub fn to_any_event(&self) -> Raw { json["redacts"] = json!(redacts); } - serde_json::from_value(json).expect("Raw::from_value always works") + json } -#[must_use] #[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn into_room_event(self) -> Raw { self.to_room_event() } + +#[implement(super::Pdu)] +#[must_use] pub fn to_room_event(&self) -> Raw { + serde_json::from_value(self.to_room_event_value()).expect("Raw::from_value always works") +} + +#[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn to_room_event_value(&self) -> JsonValue { let (redacts, content) = self.copy_redacts(); let mut json = json!({ "content": content, @@ -85,12 +80,25 @@ pub fn to_room_event(&self) -> Raw { json["redacts"] = json!(redacts); } - serde_json::from_value(json).expect("Raw::from_value always works") + json } -#[must_use] #[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn into_message_like_event(self) -> Raw { self.to_message_like_event() } + +#[implement(super::Pdu)] +#[must_use] pub fn to_message_like_event(&self) -> Raw { + serde_json::from_value(self.to_message_like_event_value()) + .expect("Raw::from_value always works") +} + +#[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn to_message_like_event_value(&self) -> JsonValue { let (redacts, content) = self.copy_redacts(); let mut json = json!({ "content": content, @@ -111,11 +119,55 @@ pub fn to_message_like_event(&self) -> Raw { json["redacts"] = json!(redacts); } - serde_json::from_value(json).expect("Raw::from_value always works") + json } -#[must_use] #[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn into_sync_room_event(self) -> Raw { self.to_sync_room_event() } + +#[implement(super::Pdu)] +#[must_use] +pub fn to_sync_room_event(&self) -> Raw { + serde_json::from_value(self.to_sync_room_event_value()).expect("Raw::from_value always works") +} + +#[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn to_sync_room_event_value(&self) -> JsonValue { + let (redacts, content) = self.copy_redacts(); + let mut json = json!({ + "content": content, + "type": self.kind, + "event_id": self.event_id, + "sender": self.sender, + "origin_server_ts": self.origin_server_ts, + }); + + if let Some(unsigned) = &self.unsigned { + json["unsigned"] = json!(unsigned); + } + if let Some(state_key) = &self.state_key { + json["state_key"] = json!(state_key); + } + if let Some(redacts) = &redacts { + json["redacts"] = json!(redacts); + } + + json +} + +#[implement(super::Pdu)] +#[must_use] +pub fn into_state_event(self) -> Raw { + serde_json::from_value(self.into_state_event_value()).expect("Raw::from_value always works") +} + +#[implement(super::Pdu)] +#[must_use] +#[inline] pub fn into_state_event_value(self) -> JsonValue { let mut json = json!({ "content": self.content, @@ -134,15 +186,17 @@ pub fn into_state_event_value(self) -> JsonValue { json } -#[must_use] #[implement(super::Pdu)] -pub fn into_state_event(self) -> Raw { - serde_json::from_value(self.into_state_event_value()).expect("Raw::from_value always works") +#[must_use] +pub fn into_sync_state_event(self) -> Raw { + serde_json::from_value(self.into_sync_state_event_value()) + .expect("Raw::from_value always works") } -#[must_use] #[implement(super::Pdu)] -pub fn to_sync_state_event(&self) -> Raw { +#[must_use] +#[inline] +pub fn into_sync_state_event_value(self) -> JsonValue { let mut json = json!({ "content": self.content, "type": self.kind, @@ -156,39 +210,65 @@ pub fn to_sync_state_event(&self) -> Raw { json["unsigned"] = json!(unsigned); } - serde_json::from_value(json).expect("Raw::from_value always works") + json } -#[must_use] #[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn into_stripped_state_event(self) -> Raw { + self.to_stripped_state_event() +} + +#[implement(super::Pdu)] +#[must_use] pub fn to_stripped_state_event(&self) -> Raw { - let json = json!({ + serde_json::from_value(self.to_stripped_state_event_value()) + .expect("Raw::from_value always works") +} + +#[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn to_stripped_state_event_value(&self) -> JsonValue { + json!({ "content": self.content, "type": self.kind, "sender": self.sender, "state_key": self.state_key, - }); - - serde_json::from_value(json).expect("Raw::from_value always works") + }) } -#[must_use] #[implement(super::Pdu)] -pub fn to_stripped_spacechild_state_event(&self) -> Raw { - let json = json!({ +#[must_use] +pub fn into_stripped_spacechild_state_event(self) -> Raw { + serde_json::from_value(self.into_stripped_spacechild_state_event_value()) + .expect("Raw::from_value always works") +} + +#[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn into_stripped_spacechild_state_event_value(self) -> JsonValue { + json!({ "content": self.content, "type": self.kind, "sender": self.sender, "state_key": self.state_key, "origin_server_ts": self.origin_server_ts, - }); - - serde_json::from_value(json).expect("Raw::from_value always works") + }) } -#[must_use] #[implement(super::Pdu)] +#[must_use] pub fn into_member_event(self) -> Raw> { + serde_json::from_value(self.into_member_event_value()).expect("Raw::from_value always works") +} + +#[implement(super::Pdu)] +#[must_use] +#[inline] +pub fn into_member_event_value(self) -> JsonValue { let mut json = json!({ "content": self.content, "type": self.kind, @@ -204,5 +284,5 @@ pub fn into_member_event(self) -> Raw> { json["unsigned"] = json!(unsigned); } - serde_json::from_value(json).expect("Raw::from_value always works") + json } diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index af597445..a10fe7fc 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -6,7 +6,7 @@ use std::{fmt::Write, sync::Arc}; use async_trait::async_trait; use conduwuit::{ - Err, Error, Result, implement, + Err, Error, PduEvent, Result, implement, utils::{ IterStream, future::BoolExt, @@ -267,11 +267,12 @@ fn get_stripped_space_child_events<'a>( } if RoomId::parse(&state_key).is_ok() { - return Some(pdu.to_stripped_spacechild_state_event()); + return Some(pdu); } None }) + .map(PduEvent::into_stripped_spacechild_state_event) } /// Gets the summary of a space using either local or remote (federation) diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 56955497..803ba9d7 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -341,7 +341,7 @@ impl Service { .await .into_iter() .filter_map(Result::ok) - .map(|e| e.to_stripped_state_event()) + .map(PduEvent::into_stripped_state_event) .chain(once(event.to_stripped_state_event())) .collect() } diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 616f0846..fab02f6b 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -697,7 +697,7 @@ impl Service { match event { | SendingEvent::Pdu(pdu_id) => { if let Ok(pdu) = self.services.timeline.get_pdu_from_id(pdu_id).await { - pdu_jsons.push(pdu.to_room_event()); + pdu_jsons.push(pdu.into_room_event()); } }, | SendingEvent::Edu(edu) => From db99d3a001841db61bb79544912099b7346456b4 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 30 Mar 2025 01:58:14 +0000 Subject: [PATCH 1618/2091] remove recently-made-unnecessary unsafe block Signed-off-by: Jason Volk --- src/core/utils/sys/storage.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/core/utils/sys/storage.rs b/src/core/utils/sys/storage.rs index b11df7bb..452b04b2 100644 --- a/src/core/utils/sys/storage.rs +++ b/src/core/utils/sys/storage.rs @@ -123,10 +123,7 @@ pub fn dev_from_path(path: &Path) -> Result<(dev_t, dev_t)> { let stat = fs::metadata(path)?; let dev_id = stat.dev().try_into()?; - - // SAFETY: These functions may not need to be marked as unsafe. - // see: https://github.com/rust-lang/libc/issues/3759 - let (major, minor) = unsafe { (libc::major(dev_id), libc::minor(dev_id)) }; + let (major, minor) = (libc::major(dev_id), libc::minor(dev_id)); Ok((major.try_into()?, minor.try_into()?)) } From d60920c72890b7ebf70d47bfc37f4477fa9716aa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 30 Mar 2025 22:59:29 +0000 Subject: [PATCH 1619/2091] workaround some large type name length issues Signed-off-by: Jason Volk --- src/api/mod.rs | 1 + src/core/error/err.rs | 1 + src/core/mod.rs | 2 ++ src/core/state_res/mod.rs | 4 ---- src/core/utils/mod.rs | 4 ++++ src/database/mod.rs | 2 ++ src/main/main.rs | 2 ++ src/router/mod.rs | 2 ++ src/service/mod.rs | 1 + src/service/rooms/event_handler/mod.rs | 15 +++--------- .../rooms/event_handler/resolve_state.rs | 6 +++-- src/service/rooms/spaces/mod.rs | 24 ++++++++++--------- .../rooms/state_accessor/room_state.rs | 4 ++-- src/service/rooms/state_accessor/state.rs | 5 +++- 14 files changed, 41 insertions(+), 32 deletions(-) diff --git a/src/api/mod.rs b/src/api/mod.rs index 8df17a59..090cf897 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1,3 +1,4 @@ +#![type_length_limit = "16384"] //TODO: reduce me #![allow(clippy::toplevel_ref_arg)] pub mod client; diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 0962c4ee..9c24d3b4 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -136,6 +136,7 @@ macro_rules! err_log { } #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! err_lev { (debug_warn) => { if $crate::debug::logging() { diff --git a/src/core/mod.rs b/src/core/mod.rs index cd56774a..80ebbdcb 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -1,3 +1,5 @@ +#![type_length_limit = "12288"] + pub mod alloc; pub mod config; pub mod debug; diff --git a/src/core/state_res/mod.rs b/src/core/state_res/mod.rs index 2020d65c..1db92e59 100644 --- a/src/core/state_res/mod.rs +++ b/src/core/state_res/mod.rs @@ -149,7 +149,6 @@ where &event_fetch, parallel_fetches, ) - .boxed() .await?; debug!(count = sorted_control_levels.len(), "power events"); @@ -164,7 +163,6 @@ where &event_fetch, parallel_fetches, ) - .boxed() .await?; debug!(count = resolved_control.len(), "resolved power events"); @@ -192,7 +190,6 @@ where let sorted_left_events = mainline_sort(&events_to_resolve, power_event.cloned(), &event_fetch, parallel_fetches) - .boxed() .await?; trace!(list = ?sorted_left_events, "events left, sorted"); @@ -204,7 +201,6 @@ where &event_fetch, parallel_fetches, ) - .boxed() .await?; // Add unconflicted state to the resolved state diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 53460c59..7593990c 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -173,6 +173,7 @@ macro_rules! is_equal { /// Functor for |x| *x.$i #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! deref_at { ($idx:tt) => { |t| *t.$idx @@ -181,6 +182,7 @@ macro_rules! deref_at { /// Functor for |ref x| x.$i #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! ref_at { ($idx:tt) => { |ref t| &t.$idx @@ -189,6 +191,7 @@ macro_rules! ref_at { /// Functor for |&x| x.$i #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! val_at { ($idx:tt) => { |&t| t.$idx @@ -197,6 +200,7 @@ macro_rules! val_at { /// Functor for |x| x.$i #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! at { ($idx:tt) => { |t| t.$idx diff --git a/src/database/mod.rs b/src/database/mod.rs index 1262a79a..ffcefee9 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -1,3 +1,5 @@ +#![type_length_limit = "3072"] + extern crate conduwuit_core as conduwuit; extern crate rust_rocksdb as rocksdb; diff --git a/src/main/main.rs b/src/main/main.rs index fbc63b17..52f40384 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -1,3 +1,5 @@ +#![type_length_limit = "49152"] //TODO: reduce me + pub(crate) mod clap; mod logging; mod mods; diff --git a/src/router/mod.rs b/src/router/mod.rs index f64dcb67..7038c5df 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -1,3 +1,5 @@ +#![type_length_limit = "32768"] //TODO: reduce me + mod layers; mod request; mod router; diff --git a/src/service/mod.rs b/src/service/mod.rs index 0bde0255..8f4a84b0 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,3 +1,4 @@ +#![type_length_limit = "2048"] #![allow(refining_impl_trait)] mod manager; diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 4944f3ec..45675da8 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -18,11 +18,7 @@ use std::{ }; use async_trait::async_trait; -use conduwuit::{ - Err, PduEvent, Result, RoomVersion, Server, - utils::{MutexMap, TryFutureExtExt}, -}; -use futures::TryFutureExt; +use conduwuit::{Err, PduEvent, Result, RoomVersion, Server, utils::MutexMap}; use ruma::{ OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, events::room::create::RoomCreateEventContent, @@ -103,13 +99,8 @@ impl Service { self.services.timeline.pdu_exists(&event_id).await } - async fn event_fetch(&self, event_id: OwnedEventId) -> Option> { - self.services - .timeline - .get_pdu(&event_id) - .map_ok(Arc::new) - .ok() - .await + async fn event_fetch(&self, event_id: OwnedEventId) -> Option { + self.services.timeline.get_pdu(&event_id).await.ok() } } diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index 9033c3a8..b3a7a71b 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -110,12 +110,14 @@ pub async fn state_resolution<'a, StateSets>( where StateSets: Iterator> + Clone + Send, { + let event_fetch = |event_id| self.event_fetch(event_id); + let event_exists = |event_id| self.event_exists(event_id); state_res::resolve( room_version, state_sets, auth_chain_sets, - &|event_id| self.event_fetch(event_id), - &|event_id| self.event_exists(event_id), + &event_fetch, + &event_exists, automatic_width(), ) .map_err(|e| err!(error!("State resolution failed: {e:?}"))) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index a10fe7fc..da52e095 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -9,7 +9,7 @@ use conduwuit::{ Err, Error, PduEvent, Result, implement, utils::{ IterStream, - future::BoolExt, + future::{BoolExt, TryExtExt}, math::usize_from_f64, stream::{BroadbandExt, ReadyExt}, }, @@ -36,7 +36,7 @@ use ruma::{ use tokio::sync::{Mutex, MutexGuard}; pub use self::pagination_token::PaginationToken; -use crate::{Dep, conduwuit::utils::TryFutureExtExt, rooms, sending}; +use crate::{Dep, rooms, sending}; pub struct Service { services: Services, @@ -141,7 +141,8 @@ pub async fn get_summary_and_children_local( } let children_pdus: Vec<_> = self - .get_stripped_space_child_events(current_room) + .get_space_child_events(current_room) + .map(PduEvent::into_stripped_spacechild_state_event) .collect() .await; @@ -235,10 +236,10 @@ async fn get_summary_and_children_federation( /// Simply returns the stripped m.space.child events of a room #[implement(Service)] -fn get_stripped_space_child_events<'a>( +fn get_space_child_events<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream> + Send + 'a { +) -> impl Stream + Send + 'a { self.services .state .get_room_shortstatehash(room_id) @@ -246,6 +247,7 @@ fn get_stripped_space_child_events<'a>( self.services .state_accessor .state_keys_with_ids(current_shortstatehash, &StateEventType::SpaceChild) + .boxed() }) .map(Result::into_iter) .map(IterStream::stream) @@ -256,8 +258,8 @@ fn get_stripped_space_child_events<'a>( .timeline .get_pdu(&event_id) .map_ok(move |pdu| (state_key, pdu)) - .await .ok() + .await }) .ready_filter_map(move |(state_key, pdu)| { if let Ok(content) = pdu.get_content::() { @@ -266,13 +268,12 @@ fn get_stripped_space_child_events<'a>( } } - if RoomId::parse(&state_key).is_ok() { - return Some(pdu); + if RoomId::parse(&state_key).is_err() { + return None; } - None + Some(pdu) }) - .map(PduEvent::into_stripped_spacechild_state_event) } /// Gets the summary of a space using either local or remote (federation) @@ -501,7 +502,8 @@ async fn cache_insert( allowed_room_ids, room_id: room_id.clone(), children_state: self - .get_stripped_space_child_events(&room_id) + .get_space_child_events(&room_id) + .map(PduEvent::into_stripped_spacechild_state_event) .collect() .await, }; diff --git a/src/service/rooms/state_accessor/room_state.rs b/src/service/rooms/state_accessor/room_state.rs index ff26b33a..642cd5d2 100644 --- a/src/service/rooms/state_accessor/room_state.rs +++ b/src/service/rooms/state_accessor/room_state.rs @@ -31,7 +31,7 @@ pub fn room_state_full<'a>( self.services .state .get_room_shortstatehash(room_id) - .map_ok(|shortstatehash| self.state_full(shortstatehash).map(Ok)) + .map_ok(|shortstatehash| self.state_full(shortstatehash).map(Ok).boxed()) .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) .try_flatten_stream() } @@ -46,7 +46,7 @@ pub fn room_state_full_pdus<'a>( self.services .state .get_room_shortstatehash(room_id) - .map_ok(|shortstatehash| self.state_full_pdus(shortstatehash).map(Ok)) + .map_ok(|shortstatehash| self.state_full_pdus(shortstatehash).map(Ok).boxed()) .map_err(move |e| err!(Database("Missing state for {room_id:?}: {e:?}"))) .try_flatten_stream() } diff --git a/src/service/rooms/state_accessor/state.rs b/src/service/rooms/state_accessor/state.rs index 02a6194e..8f2dd76f 100644 --- a/src/service/rooms/state_accessor/state.rs +++ b/src/service/rooms/state_accessor/state.rs @@ -235,6 +235,7 @@ pub fn state_keys_with_shortids<'a>( .ignore_err() .unzip() .map(|(ssks, sids): (Vec, Vec)| (ssks, sids)) + .boxed() .shared(); let shortstatekeys = short_ids @@ -390,8 +391,10 @@ pub fn state_full_shortids( .map(parse_compressed_state_event) .collect() }) - .map_ok(|vec: Vec<_>| vec.into_iter().try_stream()) + .map_ok(Vec::into_iter) + .map_ok(IterStream::try_stream) .try_flatten_stream() + .boxed() } #[implement(super::Service)] From d3b65af6163baed6e6f55922235ccc9e9f5a4e98 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 31 Mar 2025 02:28:01 +0000 Subject: [PATCH 1620/2091] remove several services.globals config wrappers Signed-off-by: Jason Volk --- src/api/client/account.rs | 10 ++++----- src/api/client/membership.rs | 4 ++-- src/api/client/presence.rs | 42 ++++++++++------------------------- src/api/client/profile.rs | 4 ++-- src/api/client/read_marker.rs | 4 ++-- src/api/client/room/create.rs | 2 +- src/api/client/send.rs | 3 +-- src/api/client/sync/v3.rs | 6 ++--- src/api/client/typing.rs | 2 +- src/api/client/unstable.rs | 8 +++---- src/api/server/invite.rs | 3 +-- src/service/globals/mod.rs | 30 ------------------------- 12 files changed, 34 insertions(+), 84 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 5dd622d7..efa8b142 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -146,7 +146,7 @@ pub(crate) async fn register_route( let is_guest = body.kind == RegistrationKind::Guest; let emergency_mode_enabled = services.config.emergency_password.is_some(); - if !services.globals.allow_registration() && body.appservice_info.is_none() { + if !services.config.allow_registration && body.appservice_info.is_none() { match (body.username.as_ref(), body.initial_device_display_name.as_ref()) { | (Some(username), Some(device_display_name)) => { info!(%is_guest, user = %username, device_name = %device_display_name, "Rejecting registration attempt as registration is disabled"); @@ -166,8 +166,8 @@ pub(crate) async fn register_route( } if is_guest - && (!services.globals.allow_guest_registration() - || (services.globals.allow_registration() + && (!services.config.allow_guest_registration + || (services.config.allow_registration && services.globals.registration_token.is_some())) { info!( @@ -441,7 +441,7 @@ pub(crate) async fn register_route( } // log in conduit admin channel if a guest registered - if body.appservice_info.is_none() && is_guest && services.globals.log_guest_registrations() { + if body.appservice_info.is_none() && is_guest && services.config.log_guest_registrations { debug_info!("New guest user \"{user_id}\" registered on this server."); if !device_display_name.is_empty() { @@ -490,7 +490,7 @@ pub(crate) async fn register_route( if body.appservice_info.is_none() && !services.server.config.auto_join_rooms.is_empty() - && (services.globals.allow_guests_auto_join_rooms() || !is_guest) + && (services.config.allow_guests_auto_join_rooms || !is_guest) { for room in &services.server.config.auto_join_rooms { let Ok(room_id) = services.rooms.alias.resolve(room).await else { diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 11395e83..315a363c 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -491,7 +491,7 @@ pub(crate) async fn invite_user_route( ) -> Result { let sender_user = body.sender_user(); - if !services.users.is_admin(sender_user).await && services.globals.block_non_admin_invites() { + if !services.users.is_admin(sender_user).await && services.config.block_non_admin_invites { info!( "User {sender_user} is not an admin and attempted to send an invite to room {}", &body.room_id @@ -1628,7 +1628,7 @@ pub(crate) async fn invite_helper( reason: Option, is_direct: bool, ) -> Result { - if !services.users.is_admin(sender_user).await && services.globals.block_non_admin_invites() { + if !services.users.is_admin(sender_user).await && services.config.block_non_admin_invites { info!( "User {sender_user} is not an admin and attempted to send an invite to room \ {room_id}" diff --git a/src/api/client/presence.rs b/src/api/client/presence.rs index 9b41a721..548e5cce 100644 --- a/src/api/client/presence.rs +++ b/src/api/client/presence.rs @@ -1,12 +1,10 @@ use std::time::Duration; use axum::extract::State; -use ruma::api::client::{ - error::ErrorKind, - presence::{get_presence, set_presence}, -}; +use conduwuit::{Err, Result}; +use ruma::api::client::presence::{get_presence, set_presence}; -use crate::{Error, Result, Ruma}; +use crate::Ruma; /// # `PUT /_matrix/client/r0/presence/{userId}/status` /// @@ -15,24 +13,17 @@ pub(crate) async fn set_presence_route( State(services): State, body: Ruma, ) -> Result { - if !services.globals.allow_local_presence() { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Presence is disabled on this server", - )); + if !services.config.allow_local_presence { + return Err!(Request(Forbidden("Presence is disabled on this server"))); } - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - if sender_user != &body.user_id && body.appservice_info.is_none() { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Not allowed to set presence of other users", - )); + if body.sender_user() != body.user_id && body.appservice_info.is_none() { + return Err!(Request(InvalidParam("Not allowed to set presence of other users"))); } services .presence - .set_presence(sender_user, &body.presence, None, None, body.status_msg.clone()) + .set_presence(body.sender_user(), &body.presence, None, None, body.status_msg.clone()) .await?; Ok(set_presence::v3::Response {}) @@ -47,21 +38,15 @@ pub(crate) async fn get_presence_route( State(services): State, body: Ruma, ) -> Result { - if !services.globals.allow_local_presence() { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Presence is disabled on this server", - )); + if !services.config.allow_local_presence { + return Err!(Request(Forbidden("Presence is disabled on this server",))); } - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let mut presence_event = None; - let has_shared_rooms = services .rooms .state_cache - .user_sees_user(sender_user, &body.user_id) + .user_sees_user(body.sender_user(), &body.user_id) .await; if has_shared_rooms { @@ -99,9 +84,6 @@ pub(crate) async fn get_presence_route( presence: presence.content.presence, }) }, - | _ => Err(Error::BadRequest( - ErrorKind::NotFound, - "Presence state for this user was not found", - )), + | _ => Err!(Request(NotFound("Presence state for this user was not found"))), } } diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 12e5ebcc..5abe5b23 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -52,7 +52,7 @@ pub(crate) async fn set_displayname_route( update_displayname(&services, &body.user_id, body.displayname.clone(), &all_joined_rooms) .await; - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { // Presence update services .presence @@ -147,7 +147,7 @@ pub(crate) async fn set_avatar_url_route( ) .await; - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { // Presence update services .presence diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index 187616b4..b334e356 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -50,7 +50,7 @@ pub(crate) async fn set_read_marker_route( } // ping presence - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { services .presence .ping_presence(sender_user, &ruma::presence::PresenceState::Online) @@ -126,7 +126,7 @@ pub(crate) async fn create_receipt_route( } // ping presence - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { services .presence .ping_presence(sender_user, &ruma::presence::PresenceState::Online) diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index bb06e966..bdc5d5a5 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -372,7 +372,7 @@ pub(crate) async fn create_room_route( // Silently skip encryption events if they are not allowed if pdu_builder.event_type == TimelineEventType::RoomEncryption - && !services.globals.allow_encryption() + && !services.config.allow_encryption { continue; } diff --git a/src/api/client/send.rs b/src/api/client/send.rs index b01d1ed6..1af74f57 100644 --- a/src/api/client/send.rs +++ b/src/api/client/send.rs @@ -25,8 +25,7 @@ pub(crate) async fn send_message_event_route( let appservice_info = body.appservice_info.as_ref(); // Forbid m.room.encrypted if encryption is disabled - if MessageLikeEventType::RoomEncrypted == body.event_type - && !services.globals.allow_encryption() + if MessageLikeEventType::RoomEncrypted == body.event_type && !services.config.allow_encryption { return Err!(Request(Forbidden("Encryption has been disabled"))); } diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index a8248f95..530c1278 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -118,7 +118,7 @@ pub(crate) async fn sync_events_route( let (sender_user, sender_device) = body.sender(); // Presence update - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { services .presence .ping_presence(sender_user, &body.body.set_presence) @@ -279,8 +279,8 @@ pub(crate) async fn build_sync_events( }); let presence_updates: OptionFuture<_> = services - .globals - .allow_local_presence() + .config + .allow_local_presence .then(|| process_presence_updates(services, since, sender_user)) .into(); diff --git a/src/api/client/typing.rs b/src/api/client/typing.rs index ccfa7340..b02cc473 100644 --- a/src/api/client/typing.rs +++ b/src/api/client/typing.rs @@ -64,7 +64,7 @@ pub(crate) async fn create_typing_event_route( } // ping presence - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { services .presence .ping_presence(&body.user_id, &ruma::presence::PresenceState::Online) diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 08da5a37..45ad103e 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -205,7 +205,7 @@ pub(crate) async fn delete_timezone_key_route( services.users.set_timezone(&body.user_id, None); - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { // Presence update services .presence @@ -233,7 +233,7 @@ pub(crate) async fn set_timezone_key_route( services.users.set_timezone(&body.user_id, body.tz.clone()); - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { // Presence update services .presence @@ -326,7 +326,7 @@ pub(crate) async fn set_profile_key_route( ); } - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { // Presence update services .presence @@ -385,7 +385,7 @@ pub(crate) async fn delete_profile_key_route( .set_profile_key(&body.user_id, &body.key_name, None); } - if services.globals.allow_local_presence() { + if services.config.allow_local_presence { // Presence update services .presence diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 463cb9ab..f4cc6eb2 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -103,8 +103,7 @@ pub(crate) async fn create_invite_route( return Err!(Request(Forbidden("This room is banned on this homeserver."))); } - if services.globals.block_non_admin_invites() && !services.users.is_admin(&invited_user).await - { + if services.config.block_non_admin_invites && !services.users.is_admin(&invited_user).await { return Err!(Request(Forbidden("This server does not allow room invites."))); } diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 1dd7db8e..a7a9be9d 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -111,20 +111,6 @@ impl Service { #[inline] pub fn server_name(&self) -> &ServerName { self.server.name.as_ref() } - pub fn allow_registration(&self) -> bool { self.server.config.allow_registration } - - pub fn allow_guest_registration(&self) -> bool { self.server.config.allow_guest_registration } - - pub fn allow_guests_auto_join_rooms(&self) -> bool { - self.server.config.allow_guests_auto_join_rooms - } - - pub fn log_guest_registrations(&self) -> bool { self.server.config.log_guest_registrations } - - pub fn allow_encryption(&self) -> bool { self.server.config.allow_encryption } - - pub fn allow_federation(&self) -> bool { self.server.config.allow_federation } - pub fn allow_public_room_directory_over_federation(&self) -> bool { self.server .config @@ -183,22 +169,6 @@ impl Service { pub fn forbidden_usernames(&self) -> &RegexSet { &self.server.config.forbidden_usernames } - pub fn allow_local_presence(&self) -> bool { self.server.config.allow_local_presence } - - pub fn allow_incoming_presence(&self) -> bool { self.server.config.allow_incoming_presence } - - pub fn allow_outgoing_presence(&self) -> bool { self.server.config.allow_outgoing_presence } - - pub fn allow_incoming_read_receipts(&self) -> bool { - self.server.config.allow_incoming_read_receipts - } - - pub fn allow_outgoing_read_receipts(&self) -> bool { - self.server.config.allow_outgoing_read_receipts - } - - pub fn block_non_admin_invites(&self) -> bool { self.server.config.block_non_admin_invites } - /// checks if `user_id` is local to us via server_name comparison #[inline] pub fn user_is_local(&self, user_id: &UserId) -> bool { From 3f0f89cddb28041ddeec94d8c80410a04153235b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 31 Mar 2025 04:25:48 +0000 Subject: [PATCH 1621/2091] use async_trait without axum re-export Signed-off-by: Jason Volk --- Cargo.lock | 1 + src/api/Cargo.toml | 1 + src/api/router/args.rs | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 8c0e797b..aa639b30 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -766,6 +766,7 @@ dependencies = [ name = "conduwuit_api" version = "0.5.0" dependencies = [ + "async-trait", "axum", "axum-client-ip", "axum-extra", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 385e786f..7890561c 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -35,6 +35,7 @@ brotli_compression = [ ] [dependencies] +async-trait.workspace = true axum-client-ip.workspace = true axum-extra.workspace = true axum.workspace = true diff --git a/src/api/router/args.rs b/src/api/router/args.rs index 65a68fa4..26713dcc 100644 --- a/src/api/router/args.rs +++ b/src/api/router/args.rs @@ -1,6 +1,7 @@ use std::{mem, ops::Deref}; -use axum::{async_trait, body::Body, extract::FromRequest}; +use async_trait::async_trait; +use axum::{body::Body, extract::FromRequest}; use bytes::{BufMut, Bytes, BytesMut}; use conduwuit::{Error, Result, debug, debug_warn, err, trace, utils::string::EMPTY}; use ruma::{ From 5768ca844295d892cfdcc9c80c8a57ef71c0e30c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 31 Mar 2025 04:23:47 +0000 Subject: [PATCH 1622/2091] upgrade dependency ByteSize Signed-off-by: Jason Volk --- Cargo.lock | 103 ++++++++++++++++++---------------------- Cargo.toml | 2 +- src/core/utils/bytes.rs | 6 +-- 3 files changed, 49 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index aa639b30..ab9af9e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,7 +79,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -161,7 +161,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -172,7 +172,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -423,7 +423,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.100", + "syn", "which", ] @@ -442,7 +442,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.100", + "syn", ] [[package]] @@ -555,9 +555,9 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "bytesize" -version = "1.3.2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2c12f985c78475a6b8d629afd0c360260ef34cfef52efccdcfd31972f81c2e" +checksum = "a3c8f83209414aacf0eeae3cf730b18d6981697fba62f200fcfb92b9f082acba" [[package]] name = "bzip2-sys" @@ -679,7 +679,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -877,7 +877,7 @@ dependencies = [ "itertools 0.13.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -1062,9 +1062,9 @@ dependencies = [ [[package]] name = "crokey" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520e83558f4c008ac06fa6a86e5c1d4357be6f994cce7434463ebcdaadf47bb1" +checksum = "c5ff945e42bb93d29b10ba509970066a269903a932f0ea07d99d8621f97e90d7" dependencies = [ "crokey-proc_macros", "crossterm", @@ -1075,15 +1075,15 @@ dependencies = [ [[package]] name = "crokey-proc_macros" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "370956e708a1ce65fe4ac5bb7185791e0ece7485087f17736d54a23a0895049f" +checksum = "665f2180fd82d0ba2bf3deb45fafabb18f23451024ff71ee47f6bfdfb4bbe09e" dependencies = [ "crossterm", "proc-macro2", "quote", "strict", - "syn 1.0.109", + "syn", ] [[package]] @@ -1191,7 +1191,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -1218,7 +1218,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -1287,7 +1287,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -1339,7 +1339,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -1550,7 +1550,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -1829,7 +1829,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -2081,7 +2081,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -2185,7 +2185,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -2336,7 +2336,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.100", + "syn", ] [[package]] @@ -2558,7 +2558,7 @@ checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -2705,7 +2705,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -2943,7 +2943,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -3007,7 +3007,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -3079,7 +3079,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" dependencies = [ "proc-macro2", - "syn 2.0.100", + "syn", ] [[package]] @@ -3108,7 +3108,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", "version_check", "yansi", ] @@ -3129,7 +3129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" dependencies = [ "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -3152,7 +3152,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -3691,7 +3691,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.100", + "syn", "toml", ] @@ -4088,7 +4088,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -4297,9 +4297,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", @@ -4367,17 +4367,6 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.100" @@ -4406,7 +4395,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -4491,7 +4480,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -4502,7 +4491,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -4658,7 +4647,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -4870,7 +4859,7 @@ source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -5167,7 +5156,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.100", + "syn", "wasm-bindgen-shared", ] @@ -5202,7 +5191,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5593,7 +5582,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", "synstructure", ] @@ -5614,7 +5603,7 @@ checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] @@ -5634,7 +5623,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", "synstructure", ] @@ -5663,7 +5652,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index ab7a935c..e6751acf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -510,7 +510,7 @@ version = "1.0.37" version = "1.0.89" [workspace.dependencies.bytesize] -version = "1.3.2" +version = "2.0.1" [workspace.dependencies.core_affinity] version = "0.8.1" diff --git a/src/core/utils/bytes.rs b/src/core/utils/bytes.rs index 04101be4..507b9b9a 100644 --- a/src/core/utils/bytes.rs +++ b/src/core/utils/bytes.rs @@ -17,15 +17,13 @@ pub fn from_str(str: &str) -> Result { Ok(bytes) } -/// Output a human-readable size string w/ si-unit suffix +/// Output a human-readable size string w/ iec-unit suffix #[inline] #[must_use] pub fn pretty(bytes: usize) -> String { - const SI_UNITS: bool = true; - let bytes: u64 = bytes.try_into().expect("failed to convert usize to u64"); - bytesize::to_string(bytes, SI_UNITS) + ByteSize::b(bytes).display().iec().to_string() } #[inline] From bee1f896243f9fafc588b98f43412637f6a5dd90 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 31 Mar 2025 05:03:15 +0000 Subject: [PATCH 1623/2091] bump dependencies Signed-off-by: Jason Volk --- Cargo.lock | 136 +++++++++++++++++++++++++++++------------- Cargo.toml | 48 +++++++-------- src/core/Cargo.toml | 1 + src/core/error/mod.rs | 2 + 4 files changed, 121 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab9af9e8..fb19dfdb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -731,8 +731,8 @@ dependencies = [ "opentelemetry-jaeger", "opentelemetry_sdk", "sentry", - "sentry-tower", - "sentry-tracing", + "sentry-tower 0.35.0", + "sentry-tracing 0.35.0", "tokio", "tokio-metrics", "tracing", @@ -782,7 +782,7 @@ dependencies = [ "http-body-util", "hyper", "ipaddress", - "itertools 0.13.0", + "itertools 0.14.0", "log", "rand 0.8.5", "reqwest", @@ -802,6 +802,7 @@ dependencies = [ "argon2", "arrayvec", "axum", + "axum-extra", "bytes", "bytesize", "cargo_toml", @@ -820,7 +821,7 @@ dependencies = [ "http", "http-body-util", "ipaddress", - "itertools 0.13.0", + "itertools 0.14.0", "libc", "libloading", "log", @@ -874,7 +875,7 @@ dependencies = [ name = "conduwuit_macros" version = "0.5.0" dependencies = [ - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -904,8 +905,8 @@ dependencies = [ "rustls", "sd-notify", "sentry", - "sentry-tower", - "sentry-tracing", + "sentry-tower 0.35.0", + "sentry-tracing 0.35.0", "serde_json", "tokio", "tower 0.5.2", @@ -930,7 +931,7 @@ dependencies = [ "http", "image", "ipaddress", - "itertools 0.13.0", + "itertools 0.14.0", "log", "loole", "lru-cache", @@ -997,9 +998,9 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const-str" -version = "0.5.7" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3618cccc083bb987a415d85c02ca6c9994ea5b44731ec28b9ecf09658655fba9" +checksum = "9e991226a70654b49d34de5ed064885f0bef0348a8e70018b8ff1ac80aa984a2" [[package]] name = "const_panic" @@ -1948,9 +1949,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes", "futures-channel", @@ -1961,7 +1962,6 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", - "tower 0.4.13", "tower-service", "tracing", ] @@ -2543,18 +2543,18 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minicbor" -version = "0.25.1" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0452a60c1863c1f50b5f77cd295e8d2786849f35883f0b9e18e7e6e1b5691b0" +checksum = "1936e27fffe7d8557c060eb82cb71668608cd1a5fb56b63e66d22ae8d7564321" dependencies = [ "minicbor-derive", ] [[package]] name = "minicbor-derive" -version = "0.15.3" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" +checksum = "a9882ef5c56df184b8ffc107fc6c61e33ee3a654b021961d790a78571bb9d67a" dependencies = [ "proc-macro2", "quote", @@ -2563,9 +2563,9 @@ dependencies = [ [[package]] name = "minicbor-serde" -version = "0.3.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "becf18ac384ecf6f53b2db3b1549eebff664c67ecf259ae99be5912193291686" +checksum = "54e45e8beeefea1b8b6f52fa188a5b6ea3746c2885606af8d4d8bf31cee633fb" dependencies = [ "minicbor", "serde", @@ -3938,21 +3938,21 @@ checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "sentry" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016958f51b96861dead7c1e02290f138411d05e94fad175c8636a835dee6e51e" +checksum = "3a7332159e544e34db06b251b1eda5e546bd90285c3f58d9c8ff8450b484e0da" dependencies = [ "httpdate", "reqwest", "rustls", "sentry-backtrace", "sentry-contexts", - "sentry-core", + "sentry-core 0.36.0", "sentry-debug-images", "sentry-log", "sentry-panic", - "sentry-tower", - "sentry-tracing", + "sentry-tower 0.36.0", + "sentry-tracing 0.36.0", "tokio", "ureq", "webpki-roots", @@ -3960,27 +3960,27 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e57712c24e99252ef175b4b06c485294f10ad6bc5b5e1567ff3803ee7a0b7d3f" +checksum = "565ec31ad37bab8e6d9f289f34913ed8768347b133706192f10606dabd5c6bc4" dependencies = [ "backtrace", "once_cell", "regex", - "sentry-core", + "sentry-core 0.36.0", ] [[package]] name = "sentry-contexts" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba8754ec3b9279e00aa6d64916f211d44202370a1699afde1db2c16cbada089" +checksum = "e860275f25f27e8c0c7726ce116c7d5c928c5bba2ee73306e52b20a752298ea6" dependencies = [ "hostname", "libc", "os_info", "rustc_version", - "sentry-core", + "sentry-core 0.36.0", "uname", ] @@ -3992,40 +3992,53 @@ checksum = "f9f8b6dcd4fbae1e3e22b447f32670360b27e31b62ab040f7fb04e0f80c04d92" dependencies = [ "once_cell", "rand 0.8.5", - "sentry-types", + "sentry-types 0.35.0", + "serde", + "serde_json", +] + +[[package]] +name = "sentry-core" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653942e6141f16651273159f4b8b1eaeedf37a7554c00cd798953e64b8a9bf72" +dependencies = [ + "once_cell", + "rand 0.8.5", + "sentry-types 0.36.0", "serde", "serde_json", ] [[package]] name = "sentry-debug-images" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8982a69133d3f5e4efdbfa0776937fca43c3a2e275a8fe184f50b1b0aa92e07c" +checksum = "2a60bc2154e6df59beed0ac13d58f8dfaf5ad20a88548a53e29e4d92e8e835c2" dependencies = [ "findshlibs", "once_cell", - "sentry-core", + "sentry-core 0.36.0", ] [[package]] name = "sentry-log" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efcbfbb74628eaef033c1154d4bb082437c7592ce2282c7c5ccb455c4c97a06d" +checksum = "1c96d796cba1b3a0793e7f53edc420c61f9419fba8fb34ad5519f5c7d01af6b2" dependencies = [ "log", - "sentry-core", + "sentry-core 0.36.0", ] [[package]] name = "sentry-panic" -version = "0.35.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de296dae6f01e931b65071ee5fe28d66a27909857f744018f107ed15fd1f6b25" +checksum = "105e3a956c8aa9dab1e4087b1657b03271bfc49d838c6ae9bfc7c58c802fd0ef" dependencies = [ "sentry-backtrace", - "sentry-core", + "sentry-core 0.36.0", ] [[package]] @@ -4033,10 +4046,21 @@ name = "sentry-tower" version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcdaf9b1939589476bd57751d12a9653bbfe356610fc476d03d7683189183ab7" +dependencies = [ + "sentry-core 0.35.0", + "tower-layer", + "tower-service", +] + +[[package]] +name = "sentry-tower" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "082f781dfc504d984e16d99f8dbf94d6ee4762dd0fc28de25713d0f900a8164d" dependencies = [ "http", "pin-project", - "sentry-core", + "sentry-core 0.36.0", "tower-layer", "tower-service", "url", @@ -4047,9 +4071,20 @@ name = "sentry-tracing" version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "263f73c757ed7915d3e1e34625eae18cad498a95b4261603d4ce3f87b159a6f0" +dependencies = [ + "sentry-core 0.35.0", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "sentry-tracing" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e75c831b4d8b34a5aec1f65f67c5d46a26c7c5d3c7abd8b5ef430796900cf8" dependencies = [ "sentry-backtrace", - "sentry-core", + "sentry-core 0.36.0", "tracing-core", "tracing-subscriber", ] @@ -4071,6 +4106,23 @@ dependencies = [ "uuid", ] +[[package]] +name = "sentry-types" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4203359e60724aa05cf2385aaf5d4f147e837185d7dd2b9ccf1ee77f4420c8" +dependencies = [ + "debugid", + "hex", + "rand 0.8.5", + "serde", + "serde_json", + "thiserror 1.0.69", + "time", + "url", + "uuid", +] + [[package]] name = "serde" version = "1.0.219" diff --git a/Cargo.toml b/Cargo.toml index e6751acf..ba706656 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,11 +27,11 @@ version = "0.5.0" name = "conduwuit" [workspace.dependencies.arrayvec] -version = "0.7.4" +version = "0.7.6" features = ["serde"] [workspace.dependencies.smallvec] -version = "1.13.2" +version = "1.14.0" features = [ "const_generics", "const_new", @@ -45,7 +45,7 @@ version = "0.3" features = ["ffi", "std", "union"] [workspace.dependencies.const-str] -version = "0.5.7" +version = "0.6.2" [workspace.dependencies.ctor] version = "0.2.9" @@ -81,13 +81,13 @@ version = "0.8.5" # Used for the http request / response body type for Ruma endpoints used with reqwest [workspace.dependencies.bytes] -version = "1.9.0" +version = "1.10.1" [workspace.dependencies.http-body-util] -version = "0.1.2" +version = "0.1.3" [workspace.dependencies.http] -version = "1.2.0" +version = "1.3.1" [workspace.dependencies.regex] version = "1.11.1" @@ -111,7 +111,7 @@ default-features = false features = ["typed-header", "tracing"] [workspace.dependencies.axum-server] -version = "0.7.1" +version = "0.7.2" default-features = false # to listen on both HTTP and HTTPS if listening on TLS dierctly from conduwuit for complement or sytest @@ -122,7 +122,7 @@ version = "0.7" version = "0.6.1" [workspace.dependencies.tower] -version = "0.5.1" +version = "0.5.2" default-features = false features = ["util"] @@ -156,12 +156,12 @@ features = [ ] [workspace.dependencies.serde] -version = "1.0.216" +version = "1.0.219" default-features = false features = ["rc"] [workspace.dependencies.serde_json] -version = "1.0.133" +version = "1.0.140" default-features = false features = ["raw_value"] @@ -237,7 +237,7 @@ features = [ ] [workspace.dependencies.futures] -version = "0.3.30" +version = "0.3.31" default-features = false features = ["std", "async-await"] @@ -275,7 +275,7 @@ features = ["alloc", "std"] default-features = false [workspace.dependencies.hyper] -version = "1.5.1" +version = "1.6.0" default-features = false features = [ "server", @@ -285,7 +285,7 @@ features = [ [workspace.dependencies.hyper-util] # hyper-util >=0.1.9 seems to have DNS issues -version = "=0.1.8" +version = "0.1.10" default-features = false features = [ "server-auto", @@ -295,7 +295,7 @@ features = [ # to support multiple variations of setting a config option [workspace.dependencies.either] -version = "1.13.0" +version = "1.15.0" default-features = false features = ["serde"] @@ -311,7 +311,7 @@ default-features = false # Used for conduwuit::Error type [workspace.dependencies.thiserror] -version = "2.0.7" +version = "2.0.12" default-features = false # Used when hashing the state @@ -321,7 +321,7 @@ default-features = false # Used to make working with iterators easier, was already a transitive depdendency [workspace.dependencies.itertools] -version = "0.13.0" +version = "0.14.0" # to parse user-friendly time durations in admin commands #TODO: overlaps chrono? @@ -337,7 +337,7 @@ version = "0.4.0" version = "2.3.1" [workspace.dependencies.async-trait] -version = "0.1.83" +version = "0.1.88" [workspace.dependencies.lru-cache] version = "0.1.2" @@ -423,7 +423,7 @@ features = ["rt-tokio"] # optional sentry metrics for crash/panic reporting [workspace.dependencies.sentry] -version = "0.35.0" +version = "0.36.0" default-features = false features = [ "backtrace", @@ -499,18 +499,18 @@ default-features = false version = "0.1" [workspace.dependencies.syn] -version = "2.0.90" +version = "2.0" default-features = false features = ["full", "extra-traits"] [workspace.dependencies.quote] -version = "1.0.37" +version = "1.0" [workspace.dependencies.proc-macro2] -version = "1.0.89" +version = "1.0" [workspace.dependencies.bytesize] -version = "2.0.1" +version = "2.0" [workspace.dependencies.core_affinity] version = "0.8.1" @@ -522,11 +522,11 @@ version = "0.2" version = "0.2" [workspace.dependencies.minicbor] -version = "0.25.1" +version = "0.26.3" features = ["std"] [workspace.dependencies.minicbor-serde] -version = "0.3.2" +version = "0.4.1" features = ["std"] [workspace.dependencies.maplit] diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index b40dd3ad..4848e742 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -59,6 +59,7 @@ conduwuit_mods = [ argon2.workspace = true arrayvec.workspace = true axum.workspace = true +axum-extra.workspace = true bytes.workspace = true bytesize.workspace = true cargo_toml.workspace = true diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index 02ab6fa3..e46edf09 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -81,6 +81,8 @@ pub enum Error { #[error("Tracing reload error: {0}")] TracingReload(#[from] tracing_subscriber::reload::Error), #[error(transparent)] + TypedHeader(#[from] axum_extra::typed_header::TypedHeaderRejection), + #[error(transparent)] Yaml(#[from] serde_yaml::Error), // ruma/conduwuit From 0f81c1e1ccdcb0c5c6d5a27e82f16eb37b1e61c8 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 1 Apr 2025 02:14:51 +0000 Subject: [PATCH 1624/2091] revert hyper-util upgrade due to continued DNS issues Signed-off-by: Jason Volk --- Cargo.lock | 5 +++-- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fb19dfdb..77d03506 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1949,9 +1949,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" dependencies = [ "bytes", "futures-channel", @@ -1962,6 +1962,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio", + "tower 0.4.13", "tower-service", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index ba706656..62bbaf16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -285,7 +285,7 @@ features = [ [workspace.dependencies.hyper-util] # hyper-util >=0.1.9 seems to have DNS issues -version = "0.1.10" +version = "=0.1.8" default-features = false features = [ "server-auto", From 1b71b99c514f69bdd2fbcdb7996dcc00860d2057 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Wed, 2 Apr 2025 10:49:38 -0400 Subject: [PATCH 1625/2091] fix weird issue with acl c2s check Signed-off-by: June Clementine Strawberry --- src/api/client/state.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 9563c26d..23583356 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -229,6 +229,9 @@ async fn allowed_to_send_state_event( if acl_content.deny.contains(&String::from("*")) && !acl_content.is_allowed(services.globals.server_name()) + && !acl_content + .allow + .contains(&services.globals.server_name().to_string()) { return Err!(Request(BadJson(debug_warn!( ?room_id, @@ -240,6 +243,9 @@ async fn allowed_to_send_state_event( if !acl_content.allow.contains(&String::from("*")) && !acl_content.is_allowed(services.globals.server_name()) + && !acl_content + .allow + .contains(&services.globals.server_name().to_string()) { return Err!(Request(BadJson(debug_warn!( ?room_id, From ea246d91d975a89a947c35260a4d50684fd2913b Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Wed, 2 Apr 2025 22:38:47 -0400 Subject: [PATCH 1626/2091] remove pointless and buggy *_visibility in-memory caches Signed-off-by: June Clementine Strawberry --- conduwuit-example.toml | 8 --- src/core/config/mod.rs | 12 ---- src/service/rooms/state_accessor/mod.rs | 68 ++----------------- .../rooms/state_accessor/server_can.rs | 22 +----- src/service/rooms/state_accessor/user_can.rs | 22 +----- 5 files changed, 10 insertions(+), 122 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 15e6dd37..75ecddab 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -195,14 +195,6 @@ # #servernameevent_data_cache_capacity = varies by system -# This item is undocumented. Please contribute documentation for it. -# -#server_visibility_cache_capacity = varies by system - -# This item is undocumented. Please contribute documentation for it. -# -#user_visibility_cache_capacity = varies by system - # This item is undocumented. Please contribute documentation for it. # #stateinfo_cache_capacity = varies by system diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 52df19ac..7be140a5 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -252,14 +252,6 @@ pub struct Config { #[serde(default = "default_servernameevent_data_cache_capacity")] pub servernameevent_data_cache_capacity: u32, - /// default: varies by system - #[serde(default = "default_server_visibility_cache_capacity")] - pub server_visibility_cache_capacity: u32, - - /// default: varies by system - #[serde(default = "default_user_visibility_cache_capacity")] - pub user_visibility_cache_capacity: u32, - /// default: varies by system #[serde(default = "default_stateinfo_cache_capacity")] pub stateinfo_cache_capacity: u32, @@ -2035,10 +2027,6 @@ fn default_servernameevent_data_cache_capacity() -> u32 { parallelism_scaled_u32(100_000).saturating_add(500_000) } -fn default_server_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(500) } - -fn default_user_visibility_cache_capacity() -> u32 { parallelism_scaled_u32(1000) } - fn default_stateinfo_cache_capacity() -> u32 { parallelism_scaled_u32(100) } fn default_roomid_spacehierarchy_cache_capacity() -> u32 { parallelism_scaled_u32(1000) } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 652fdbd7..b57465ce 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -3,21 +3,13 @@ mod server_can; mod state; mod user_can; -use std::{ - fmt::Write, - sync::{Arc, Mutex as StdMutex, Mutex}, -}; +use std::sync::Arc; use async_trait::async_trait; -use conduwuit::{ - Result, err, utils, - utils::math::{Expected, usize_from_f64}, -}; +use conduwuit::{Result, err}; use database::Map; -use lru_cache::LruCache; use ruma::{ - EventEncryptionAlgorithm, JsOption, OwnedRoomAliasId, OwnedRoomId, OwnedServerName, - OwnedUserId, RoomId, UserId, + EventEncryptionAlgorithm, JsOption, OwnedRoomAliasId, OwnedRoomId, RoomId, UserId, events::{ StateEventType, room::{ @@ -37,11 +29,9 @@ use ruma::{ space::SpaceRoomJoinRule, }; -use crate::{Dep, rooms, rooms::short::ShortStateHash}; +use crate::{Dep, rooms}; pub struct Service { - pub server_visibility_cache: Mutex>, - pub user_visibility_cache: Mutex>, services: Services, db: Data, } @@ -61,19 +51,7 @@ struct Data { #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { - let config = &args.server.config; - let server_visibility_cache_capacity = - f64::from(config.server_visibility_cache_capacity) * config.cache_capacity_modifier; - let user_visibility_cache_capacity = - f64::from(config.user_visibility_cache_capacity) * config.cache_capacity_modifier; - Ok(Arc::new(Self { - server_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64( - server_visibility_cache_capacity, - )?)), - user_visibility_cache: StdMutex::new(LruCache::new(usize_from_f64( - user_visibility_cache_capacity, - )?)), services: Services { state_cache: args.depend::("rooms::state_cache"), timeline: args.depend::("rooms::timeline"), @@ -88,44 +66,6 @@ impl crate::Service for Service { })) } - async fn memory_usage(&self, out: &mut (dyn Write + Send)) -> Result { - use utils::bytes::pretty; - - let (svc_count, svc_bytes) = self.server_visibility_cache.lock()?.iter().fold( - (0_usize, 0_usize), - |(count, bytes), (key, _)| { - ( - count.expected_add(1), - bytes - .expected_add(key.0.capacity()) - .expected_add(size_of_val(&key.1)), - ) - }, - ); - - let (uvc_count, uvc_bytes) = self.user_visibility_cache.lock()?.iter().fold( - (0_usize, 0_usize), - |(count, bytes), (key, _)| { - ( - count.expected_add(1), - bytes - .expected_add(key.0.capacity()) - .expected_add(size_of_val(&key.1)), - ) - }, - ); - - writeln!(out, "server_visibility_cache: {svc_count} ({})", pretty(svc_bytes))?; - writeln!(out, "user_visibility_cache: {uvc_count} ({})", pretty(uvc_bytes))?; - - Ok(()) - } - - async fn clear_cache(&self) { - self.server_visibility_cache.lock().expect("locked").clear(); - self.user_visibility_cache.lock().expect("locked").clear(); - } - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } diff --git a/src/service/rooms/state_accessor/server_can.rs b/src/service/rooms/state_accessor/server_can.rs index 2e8f3325..7d1b197f 100644 --- a/src/service/rooms/state_accessor/server_can.rs +++ b/src/service/rooms/state_accessor/server_can.rs @@ -1,4 +1,4 @@ -use conduwuit::{error, implement, utils::stream::ReadyExt}; +use conduwuit::{debug_info, implement, utils::stream::ReadyExt}; use futures::StreamExt; use ruma::{ EventId, RoomId, ServerName, @@ -22,15 +22,6 @@ pub async fn server_can_see_event( return true; }; - if let Some(visibility) = self - .server_visibility_cache - .lock() - .expect("locked") - .get_mut(&(origin.to_owned(), shortstatehash)) - { - return *visibility; - } - let history_visibility = self .state_get_content(shortstatehash, &StateEventType::RoomHistoryVisibility, "") .await @@ -44,7 +35,7 @@ pub async fn server_can_see_event( .room_members(room_id) .ready_filter(|member| member.server_name() == origin); - let visibility = match history_visibility { + match history_visibility { | HistoryVisibility::WorldReadable | HistoryVisibility::Shared => true, | HistoryVisibility::Invited => { // Allow if any member on requesting server was AT LEAST invited, else deny @@ -62,12 +53,5 @@ pub async fn server_can_see_event( error!("Unknown history visibility {history_visibility}"); false }, - }; - - self.server_visibility_cache - .lock() - .expect("locked") - .insert((origin.to_owned(), shortstatehash), visibility); - - visibility + } } diff --git a/src/service/rooms/state_accessor/user_can.rs b/src/service/rooms/state_accessor/user_can.rs index c30e1da8..32a766a8 100644 --- a/src/service/rooms/state_accessor/user_can.rs +++ b/src/service/rooms/state_accessor/user_can.rs @@ -1,4 +1,4 @@ -use conduwuit::{Err, Error, Result, error, implement, pdu::PduBuilder}; +use conduwuit::{Err, Error, Result, debug_info, implement, pdu::PduBuilder}; use ruma::{ EventId, RoomId, UserId, events::{ @@ -98,15 +98,6 @@ pub async fn user_can_see_event( return true; }; - if let Some(visibility) = self - .user_visibility_cache - .lock() - .expect("locked") - .get_mut(&(user_id.to_owned(), shortstatehash)) - { - return *visibility; - } - let currently_member = self.services.state_cache.is_joined(user_id, room_id).await; let history_visibility = self @@ -116,7 +107,7 @@ pub async fn user_can_see_event( c.history_visibility }); - let visibility = match history_visibility { + match history_visibility { | HistoryVisibility::WorldReadable => true, | HistoryVisibility::Shared => currently_member, | HistoryVisibility::Invited => { @@ -131,14 +122,7 @@ pub async fn user_can_see_event( error!("Unknown history visibility {history_visibility}"); false }, - }; - - self.user_visibility_cache - .lock() - .expect("locked") - .insert((user_id.to_owned(), shortstatehash), visibility); - - visibility + } } /// Whether a user is allowed to see an event, based on From 74012c5289831c16976fc283a4233bfb6b49ce8b Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Wed, 2 Apr 2025 22:44:44 -0400 Subject: [PATCH 1627/2091] significantly improve get_missing_events fed code Signed-off-by: June Clementine Strawberry --- src/api/server/backfill.rs | 12 ++- src/api/server/get_missing_events.rs | 111 ++++++++++++++------------- 2 files changed, 65 insertions(+), 58 deletions(-) diff --git a/src/api/server/backfill.rs b/src/api/server/backfill.rs index 5c875807..3cfbcedc 100644 --- a/src/api/server/backfill.rs +++ b/src/api/server/backfill.rs @@ -6,11 +6,17 @@ use conduwuit::{ utils::{IterStream, ReadyExt, stream::TryTools}, }; use futures::{FutureExt, StreamExt, TryStreamExt}; -use ruma::{MilliSecondsSinceUnixEpoch, api::federation::backfill::get_backfill, uint}; +use ruma::{MilliSecondsSinceUnixEpoch, api::federation::backfill::get_backfill}; use super::AccessCheck; use crate::Ruma; +/// arbitrary number but synapse's is 100 and we can handle lots of these +/// anyways +const LIMIT_MAX: usize = 150; +/// no spec defined number but we can handle a lot of these +const LIMIT_DEFAULT: usize = 50; + /// # `GET /_matrix/federation/v1/backfill/` /// /// Retrieves events from before the sender joined the room, if the room's @@ -30,9 +36,9 @@ pub(crate) async fn get_backfill_route( let limit = body .limit - .min(uint!(100)) .try_into() - .expect("UInt could not be converted to usize"); + .unwrap_or(LIMIT_DEFAULT) + .min(LIMIT_MAX); let from = body .v diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index 3d0bbb07..d72918fa 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -1,13 +1,19 @@ use axum::extract::State; -use conduwuit::{Error, Result}; -use ruma::{ - CanonicalJsonValue, EventId, RoomId, - api::{client::error::ErrorKind, federation::event::get_missing_events}, +use conduwuit::{ + Result, debug, debug_info, debug_warn, + utils::{self}, + warn, }; +use ruma::api::federation::event::get_missing_events; use super::AccessCheck; use crate::Ruma; +/// arbitrary number but synapse's is 20 and we can handle lots of these anyways +const LIMIT_MAX: usize = 50; +/// spec says default is 10 +const LIMIT_DEFAULT: usize = 10; + /// # `POST /_matrix/federation/v1/get_missing_events/{roomId}` /// /// Retrieves events that the sender is missing. @@ -24,7 +30,11 @@ pub(crate) async fn get_missing_events_route( .check() .await?; - let limit = body.limit.try_into()?; + let limit = body + .limit + .try_into() + .unwrap_or(LIMIT_DEFAULT) + .min(LIMIT_MAX); let mut queued_events = body.latest_events.clone(); // the vec will never have more entries the limit @@ -32,60 +42,51 @@ pub(crate) async fn get_missing_events_route( let mut i: usize = 0; while i < queued_events.len() && events.len() < limit { - if let Ok(pdu) = services + let Ok(pdu) = services.rooms.timeline.get_pdu(&queued_events[i]).await else { + debug_info!(?body.origin, "Event {} does not exist locally, skipping", &queued_events[i]); + i = i.saturating_add(1); + continue; + }; + + if pdu.room_id != body.room_id { + warn!(?body.origin, + "Got an event for the wrong room in database. Found {:?} in {:?}, server requested events in {:?}. Skipping.", + pdu.event_id, pdu.room_id, body.room_id + ); + i = i.saturating_add(1); + continue; + } + + if body.earliest_events.contains(&queued_events[i]) { + i = i.saturating_add(1); + continue; + } + + if !services .rooms - .timeline - .get_pdu_json(&queued_events[i]) + .state_accessor + .server_can_see_event(body.origin(), &body.room_id, &queued_events[i]) .await { - let room_id_str = pdu - .get("room_id") - .and_then(|val| val.as_str()) - .ok_or_else(|| Error::bad_database("Invalid event in database."))?; - - let event_room_id = <&RoomId>::try_from(room_id_str) - .map_err(|_| Error::bad_database("Invalid room_id in event in database."))?; - - if event_room_id != body.room_id { - return Err(Error::BadRequest(ErrorKind::InvalidParam, "Event from wrong room.")); - } - - if body.earliest_events.contains(&queued_events[i]) { - i = i.saturating_add(1); - continue; - } - - if !services - .rooms - .state_accessor - .server_can_see_event(body.origin(), &body.room_id, &queued_events[i]) - .await - { - i = i.saturating_add(1); - continue; - } - - let prev_events = pdu - .get("prev_events") - .and_then(CanonicalJsonValue::as_array) - .unwrap_or_default(); - - queued_events.extend( - prev_events - .iter() - .map(<&EventId>::try_from) - .filter_map(Result::ok) - .map(ToOwned::to_owned), - ); - - events.push( - services - .sending - .convert_to_outgoing_federation_event(pdu) - .await, - ); + debug!(?body.origin, "Server cannot see {:?} in {:?}, skipping", pdu.event_id, pdu.room_id); + i = i.saturating_add(1); + continue; } - i = i.saturating_add(1); + + let Ok(pdu_json) = utils::to_canonical_object(&pdu) else { + debug_warn!(?body.origin, "Failed to convert PDU in database to canonical JSON: {pdu:?}"); + i = i.saturating_add(1); + continue; + }; + + queued_events.extend(pdu.prev_events.iter().map(ToOwned::to_owned)); + + events.push( + services + .sending + .convert_to_outgoing_federation_event(pdu_json) + .await, + ); } Ok(get_missing_events::v1::Response { events }) From 1036f8dfa8fabb9642b9638b54381e00016eef9c Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Wed, 2 Apr 2025 22:46:01 -0400 Subject: [PATCH 1628/2091] default shared history vis on unknown visibilities, drop needless error log Signed-off-by: June Clementine Strawberry --- src/service/rooms/state_accessor/server_can.rs | 4 ++-- src/service/rooms/state_accessor/user_can.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/service/rooms/state_accessor/server_can.rs b/src/service/rooms/state_accessor/server_can.rs index 7d1b197f..c946fbfd 100644 --- a/src/service/rooms/state_accessor/server_can.rs +++ b/src/service/rooms/state_accessor/server_can.rs @@ -50,8 +50,8 @@ pub async fn server_can_see_event( .await }, | _ => { - error!("Unknown history visibility {history_visibility}"); - false + debug_info!(%room_id, "Unknown history visibility, defaulting to shared: {history_visibility:?}"); + true }, } } diff --git a/src/service/rooms/state_accessor/user_can.rs b/src/service/rooms/state_accessor/user_can.rs index 32a766a8..aa54407b 100644 --- a/src/service/rooms/state_accessor/user_can.rs +++ b/src/service/rooms/state_accessor/user_can.rs @@ -119,8 +119,8 @@ pub async fn user_can_see_event( self.user_was_joined(shortstatehash, user_id).await }, | _ => { - error!("Unknown history visibility {history_visibility}"); - false + debug_info!(%room_id, "Unknown history visibility, defaulting to shared: {history_visibility:?}"); + currently_member }, } } From 0e0b8cc4032732378966f07b38b97af89788e399 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Wed, 2 Apr 2025 22:51:17 -0400 Subject: [PATCH 1629/2091] fixup+update msc3266, add fed support, parallelise IO Signed-off-by: June Clementine Strawberry --- Cargo.lock | 22 +- Cargo.toml | 2 +- src/api/client/room/mod.rs | 9 +- src/api/client/room/summary.rs | 308 ++++++++++++++++++++++++ src/api/client/room/upgrade.rs | 2 +- src/api/client/unstable.rs | 138 +---------- src/service/rooms/spaces/mod.rs | 54 +++-- src/service/rooms/state_accessor/mod.rs | 28 ++- 8 files changed, 389 insertions(+), 174 deletions(-) create mode 100644 src/api/client/room/summary.rs diff --git a/Cargo.lock b/Cargo.lock index 77d03506..a53258bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3531,7 +3531,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "assign", "js_int", @@ -3551,7 +3551,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "js_int", "ruma-common", @@ -3563,7 +3563,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "as_variant", "assign", @@ -3586,7 +3586,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "as_variant", "base64 0.22.1", @@ -3618,7 +3618,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "as_variant", "indexmap 2.8.0", @@ -3643,7 +3643,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "bytes", "headers", @@ -3665,7 +3665,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3674,7 +3674,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "js_int", "ruma-common", @@ -3684,7 +3684,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3699,7 +3699,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "js_int", "ruma-common", @@ -3711,7 +3711,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=d197318a2507d38ffe6ee524d0d52728ca72538a#d197318a2507d38ffe6ee524d0d52728ca72538a" +source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 62bbaf16..940ece86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "d197318a2507d38ffe6ee524d0d52728ca72538a" +rev = "ea1278657125e9414caada074e8c172bc252fb1c" features = [ "compat", "rand", diff --git a/src/api/client/room/mod.rs b/src/api/client/room/mod.rs index 16fcadab..86d68f7e 100644 --- a/src/api/client/room/mod.rs +++ b/src/api/client/room/mod.rs @@ -2,9 +2,14 @@ mod aliases; mod create; mod event; mod initial_sync; +mod summary; mod upgrade; pub(crate) use self::{ - aliases::get_room_aliases_route, create::create_room_route, event::get_room_event_route, - initial_sync::room_initial_sync_route, upgrade::upgrade_room_route, + aliases::get_room_aliases_route, + create::create_room_route, + event::get_room_event_route, + initial_sync::room_initial_sync_route, + summary::{get_room_summary, get_room_summary_legacy}, + upgrade::upgrade_room_route, }; diff --git a/src/api/client/room/summary.rs b/src/api/client/room/summary.rs new file mode 100644 index 00000000..34820e83 --- /dev/null +++ b/src/api/client/room/summary.rs @@ -0,0 +1,308 @@ +use axum::extract::State; +use axum_client_ip::InsecureClientIp; +use conduwuit::{ + Err, Result, debug_warn, + utils::{IterStream, future::TryExtExt}, +}; +use futures::{FutureExt, StreamExt, future::join3, stream::FuturesUnordered}; +use ruma::{ + OwnedRoomId, OwnedServerName, RoomId, UserId, + api::{ + client::room::get_summary, + federation::space::{SpaceHierarchyParentSummary, get_hierarchy}, + }, + events::room::member::MembershipState, + space::SpaceRoomJoinRule::{self, *}, +}; +use service::Services; + +use crate::{Ruma, RumaResponse}; + +/// # `GET /_matrix/client/unstable/im.nheko.summary/rooms/{roomIdOrAlias}/summary` +/// +/// Returns a short description of the state of a room. +/// +/// This is the "wrong" endpoint that some implementations/clients may use +/// according to the MSC. Request and response bodies are the same as +/// `get_room_summary`. +/// +/// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) +pub(crate) async fn get_room_summary_legacy( + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result> { + get_room_summary(State(services), InsecureClientIp(client), body) + .boxed() + .await + .map(RumaResponse) +} + +/// # `GET /_matrix/client/unstable/im.nheko.summary/summary/{roomIdOrAlias}` +/// +/// Returns a short description of the state of a room. +/// +/// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) +#[tracing::instrument(skip_all, fields(%client), name = "room_summary")] +pub(crate) async fn get_room_summary( + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let (room_id, servers) = services + .rooms + .alias + .resolve_with_servers(&body.room_id_or_alias, Some(body.via.clone())) + .await?; + + if services.rooms.metadata.is_banned(&room_id).await { + return Err!(Request(Forbidden("This room is banned on this homeserver."))); + } + + room_summary_response(&services, &room_id, &servers, body.sender_user.as_deref()) + .boxed() + .await +} + +async fn room_summary_response( + services: &Services, + room_id: &RoomId, + servers: &[OwnedServerName], + sender_user: Option<&UserId>, +) -> Result { + if services.rooms.metadata.exists(room_id).await { + return local_room_summary_response(services, room_id, sender_user) + .boxed() + .await; + } + + let room = + remote_room_summary_hierarchy_response(services, room_id, servers, sender_user).await?; + + Ok(get_summary::msc3266::Response { + room_id: room_id.to_owned(), + canonical_alias: room.canonical_alias, + avatar_url: room.avatar_url, + guest_can_join: room.guest_can_join, + name: room.name, + num_joined_members: room.num_joined_members, + topic: room.topic, + world_readable: room.world_readable, + join_rule: room.join_rule, + room_type: room.room_type, + room_version: room.room_version, + membership: if sender_user.is_none() { + None + } else { + Some(MembershipState::Leave) + }, + encryption: room.encryption, + allowed_room_ids: room.allowed_room_ids, + }) +} + +async fn local_room_summary_response( + services: &Services, + room_id: &RoomId, + sender_user: Option<&UserId>, +) -> Result { + let join_rule = services.rooms.state_accessor.get_space_join_rule(room_id); + let world_readable = services.rooms.state_accessor.is_world_readable(room_id); + let guest_can_join = services.rooms.state_accessor.guest_can_join(room_id); + + let ((join_rule, allowed_room_ids), world_readable, guest_can_join) = + join3(join_rule, world_readable, guest_can_join).await; + + user_can_see_summary( + services, + room_id, + &join_rule, + guest_can_join, + world_readable, + &allowed_room_ids, + sender_user, + ) + .await?; + + let canonical_alias = services + .rooms + .state_accessor + .get_canonical_alias(room_id) + .ok(); + let name = services.rooms.state_accessor.get_name(room_id).ok(); + let topic = services.rooms.state_accessor.get_room_topic(room_id).ok(); + let room_type = services.rooms.state_accessor.get_room_type(room_id).ok(); + let avatar_url = services + .rooms + .state_accessor + .get_avatar(room_id) + .map(|res| res.into_option().unwrap_or_default().url); + let room_version = services.rooms.state.get_room_version(room_id).ok(); + let encryption = services + .rooms + .state_accessor + .get_room_encryption(room_id) + .ok(); + let num_joined_members = services + .rooms + .state_cache + .room_joined_count(room_id) + .unwrap_or(0); + + let ( + canonical_alias, + name, + num_joined_members, + topic, + avatar_url, + room_type, + room_version, + encryption, + ) = futures::join!( + canonical_alias, + name, + num_joined_members, + topic, + avatar_url, + room_type, + room_version, + encryption, + ); + + Ok(get_summary::msc3266::Response { + room_id: room_id.to_owned(), + canonical_alias, + avatar_url, + guest_can_join, + name, + num_joined_members: num_joined_members.try_into().unwrap_or_default(), + topic, + world_readable, + join_rule, + room_type, + room_version, + membership: if let Some(sender_user) = sender_user { + services + .rooms + .state_accessor + .get_member(room_id, sender_user) + .await + .map_or(Some(MembershipState::Leave), |content| Some(content.membership)) + } else { + None + }, + encryption, + allowed_room_ids, + }) +} + +/// used by MSC3266 to fetch a room's info if we do not know about it +async fn remote_room_summary_hierarchy_response( + services: &Services, + room_id: &RoomId, + servers: &[OwnedServerName], + sender_user: Option<&UserId>, +) -> Result { + if !services.config.allow_federation { + return Err!(Request(Forbidden("Federation is disabled."))); + } + + if services.rooms.metadata.is_disabled(room_id).await { + return Err!(Request(Forbidden( + "Federaton of room {room_id} is currently disabled on this server." + ))); + } + + let request = get_hierarchy::v1::Request::new(room_id.to_owned()); + + let mut requests: FuturesUnordered<_> = servers + .iter() + .map(|server| { + services + .sending + .send_federation_request(server, request.clone()) + }) + .collect(); + + while let Some(Ok(response)) = requests.next().await { + let room = response.room.clone(); + if room.room_id != room_id { + debug_warn!( + "Room ID {} returned does not belong to the requested room ID {}", + room.room_id, + room_id + ); + continue; + } + + return user_can_see_summary( + services, + room_id, + &room.join_rule, + room.guest_can_join, + room.world_readable, + &room.allowed_room_ids, + sender_user, + ) + .await + .map(|()| room); + } + + Err!(Request(NotFound( + "Room is unknown to this server and was unable to fetch over federation with the \ + provided servers available" + ))) +} + +async fn user_can_see_summary( + services: &Services, + room_id: &RoomId, + join_rule: &SpaceRoomJoinRule, + guest_can_join: bool, + world_readable: bool, + allowed_room_ids: &[OwnedRoomId], + sender_user: Option<&UserId>, +) -> Result { + match sender_user { + | Some(sender_user) => { + let user_can_see_state_events = services + .rooms + .state_accessor + .user_can_see_state_events(sender_user, room_id); + let is_guest = services.users.is_deactivated(sender_user).unwrap_or(false); + let user_in_allowed_restricted_room = allowed_room_ids + .iter() + .stream() + .any(|room| services.rooms.state_cache.is_joined(sender_user, room)); + + let (user_can_see_state_events, is_guest, user_in_allowed_restricted_room) = + join3(user_can_see_state_events, is_guest, user_in_allowed_restricted_room) + .boxed() + .await; + + if user_can_see_state_events + || (is_guest && guest_can_join) + || matches!(&join_rule, &Public | &Knock | &KnockRestricted) + || user_in_allowed_restricted_room + { + return Ok(()); + } + + Err!(Request(Forbidden( + "Room is not world readable, not publicly accessible/joinable, restricted room \ + conditions not met, and guest access is forbidden. Not allowed to see details \ + of this room." + ))) + }, + | None => { + if matches!(join_rule, Public | Knock | KnockRestricted) || world_readable { + return Ok(()); + } + + Err!(Request(Forbidden( + "Room is not world readable or publicly accessible/joinable, authentication is \ + required" + ))) + }, + } +} diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index 4ac341a9..3cfb3c28 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -103,7 +103,7 @@ pub(crate) async fn upgrade_room_route( // Use the m.room.tombstone event as the predecessor let predecessor = Some(ruma::events::room::create::PreviousRoom::new( body.room_id.clone(), - (*tombstone_event_id).to_owned(), + Some(tombstone_event_id), )); // Send a m.room.create event containing a predecessor field and the applicable diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 45ad103e..e21eaf21 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::Err; +use conduwuit::{Err, Error, Result}; use futures::StreamExt; use ruma::{ OwnedRoomId, @@ -14,16 +14,14 @@ use ruma::{ delete_profile_key, delete_timezone_key, get_profile_key, get_timezone_key, set_profile_key, set_timezone_key, }, - room::get_summary, }, federation, }, - events::room::member::MembershipState, presence::PresenceState, }; use super::{update_avatar_url, update_displayname}; -use crate::{Error, Result, Ruma, RumaResponse}; +use crate::Ruma; /// # `GET /_matrix/client/unstable/uk.half-shot.msc2666/user/mutual_rooms` /// @@ -38,13 +36,10 @@ pub(crate) async fn get_mutual_rooms_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); - if sender_user == &body.user_id { - return Err(Error::BadRequest( - ErrorKind::Unknown, - "You cannot request rooms in common with yourself.", - )); + if sender_user == body.user_id { + return Err!(Request(Unknown("You cannot request rooms in common with yourself."))); } if !services.users.exists(&body.user_id).await { @@ -65,129 +60,6 @@ pub(crate) async fn get_mutual_rooms_route( }) } -/// # `GET /_matrix/client/unstable/im.nheko.summary/rooms/{roomIdOrAlias}/summary` -/// -/// Returns a short description of the state of a room. -/// -/// This is the "wrong" endpoint that some implementations/clients may use -/// according to the MSC. Request and response bodies are the same as -/// `get_room_summary`. -/// -/// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) -pub(crate) async fn get_room_summary_legacy( - State(services): State, - InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result> { - get_room_summary(State(services), InsecureClientIp(client), body) - .await - .map(RumaResponse) -} - -/// # `GET /_matrix/client/unstable/im.nheko.summary/summary/{roomIdOrAlias}` -/// -/// Returns a short description of the state of a room. -/// -/// TODO: support fetching remote room info if we don't know the room -/// -/// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) -#[tracing::instrument(skip_all, fields(%client), name = "room_summary")] -pub(crate) async fn get_room_summary( - State(services): State, - InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result { - let sender_user = body.sender_user.as_ref(); - - let room_id = services.rooms.alias.resolve(&body.room_id_or_alias).await?; - - if !services.rooms.metadata.exists(&room_id).await { - return Err(Error::BadRequest(ErrorKind::NotFound, "Room is unknown to this server")); - } - - if sender_user.is_none() - && !services - .rooms - .state_accessor - .is_world_readable(&room_id) - .await - { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Room is not world readable, authentication is required", - )); - } - - Ok(get_summary::msc3266::Response { - room_id: room_id.clone(), - canonical_alias: services - .rooms - .state_accessor - .get_canonical_alias(&room_id) - .await - .ok(), - avatar_url: services - .rooms - .state_accessor - .get_avatar(&room_id) - .await - .into_option() - .unwrap_or_default() - .url, - guest_can_join: services.rooms.state_accessor.guest_can_join(&room_id).await, - name: services.rooms.state_accessor.get_name(&room_id).await.ok(), - num_joined_members: services - .rooms - .state_cache - .room_joined_count(&room_id) - .await - .unwrap_or(0) - .try_into()?, - topic: services - .rooms - .state_accessor - .get_room_topic(&room_id) - .await - .ok(), - world_readable: services - .rooms - .state_accessor - .is_world_readable(&room_id) - .await, - join_rule: services - .rooms - .state_accessor - .get_join_rule(&room_id) - .await - .unwrap_or_default() - .0, - room_type: services - .rooms - .state_accessor - .get_room_type(&room_id) - .await - .ok(), - room_version: services.rooms.state.get_room_version(&room_id).await.ok(), - membership: if let Some(sender_user) = sender_user { - services - .rooms - .state_accessor - .get_member(&room_id, sender_user) - .await - .map_or_else(|_| MembershipState::Leave, |content| content.membership) - .into() - } else { - None - }, - encryption: services - .rooms - .state_accessor - .get_room_encryption(&room_id) - .await - .ok(), - }) -} - /// # `DELETE /_matrix/client/unstable/uk.tcpip.msc4133/profile/:user_id/us.cloke.msc4175.tz` /// /// Deletes the `tz` (timezone) of a user, as per MSC4133 and MSC4175. diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index da52e095..f51a5e3a 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -27,7 +27,6 @@ use ruma::{ }, events::{ StateEventType, - room::join_rules::{JoinRule, RoomJoinRulesEventContent}, space::child::{HierarchySpaceChildEvent, SpaceChildEventContent}, }, serde::Raw, @@ -306,25 +305,18 @@ async fn get_room_summary( children_state: Vec>, identifier: &Identifier<'_>, ) -> Result { - let join_rule = self + let (join_rule, allowed_room_ids) = self .services .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") - .await - .map_or(JoinRule::Invite, |c: RoomJoinRulesEventContent| c.join_rule); + .get_space_join_rule(room_id) + .await; - let allowed_room_ids = self - .services - .state_accessor - .allowed_room_ids(join_rule.clone()); - - let join_rule = join_rule.clone().into(); let is_accessible_child = self .is_accessible_child(room_id, &join_rule, identifier, &allowed_room_ids) .await; if !is_accessible_child { - return Err!(Request(Forbidden("User is not allowed to see the room",))); + return Err!(Request(Forbidden("User is not allowed to see the room"))); } let name = self.services.state_accessor.get_name(room_id).ok(); @@ -355,6 +347,14 @@ async fn get_room_summary( .get_avatar(room_id) .map(|res| res.into_option().unwrap_or_default().url); + let room_version = self.services.state.get_room_version(room_id).ok(); + + let encryption = self + .services + .state_accessor + .get_room_encryption(room_id) + .ok(); + let ( canonical_alias, name, @@ -364,6 +364,8 @@ async fn get_room_summary( guest_can_join, avatar_url, room_type, + room_version, + encryption, ) = futures::join!( canonical_alias, name, @@ -372,7 +374,9 @@ async fn get_room_summary( world_readable, guest_can_join, avatar_url, - room_type + room_type, + room_version, + encryption, ); Ok(SpaceHierarchyParentSummary { @@ -387,9 +391,9 @@ async fn get_room_summary( allowed_room_ids, join_rule, room_id: room_id.to_owned(), - num_joined_members: num_joined_members - .try_into() - .expect("user count should not be that big"), + num_joined_members: num_joined_members.try_into().unwrap_or_default(), + encryption, + room_version, }) } @@ -487,6 +491,8 @@ async fn cache_insert( join_rule, room_type, allowed_room_ids, + encryption, + room_version, } = child; let summary = SpaceHierarchyParentSummary { @@ -506,6 +512,8 @@ async fn cache_insert( .map(PduEvent::into_stripped_spacechild_state_event) .collect() .await, + encryption, + room_version, }; cache.insert(current_room.to_owned(), Some(CachedSpaceHierarchySummary { summary })); @@ -527,7 +535,9 @@ impl From for SpaceHierarchyRoomsChunk { join_rule, room_type, children_state, - .. + allowed_room_ids, + encryption, + room_version, } = value.summary; Self { @@ -542,6 +552,9 @@ impl From for SpaceHierarchyRoomsChunk { join_rule, room_type, children_state, + encryption, + room_version, + allowed_room_ids, } } } @@ -562,7 +575,9 @@ pub fn summary_to_chunk(summary: SpaceHierarchyParentSummary) -> SpaceHierarchyR join_rule, room_type, children_state, - .. + allowed_room_ids, + encryption, + room_version, } = summary; SpaceHierarchyRoomsChunk { @@ -577,5 +592,8 @@ pub fn summary_to_chunk(summary: SpaceHierarchyParentSummary) -> SpaceHierarchyR join_rule, room_type, children_state, + encryption, + room_version, + allowed_room_ids, } } diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index b57465ce..7fff5935 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -129,22 +129,34 @@ impl Service { .map(|c: RoomTopicEventContent| c.topic) } - /// Returns the join rule (`SpaceRoomJoinRule`) for a given room - pub async fn get_join_rule( + /// Returns the space join rule (`SpaceRoomJoinRule`) for a given room and + /// any allowed room IDs if available. Will default to Invite and empty vec + /// if doesnt exist or invalid, + pub async fn get_space_join_rule( &self, room_id: &RoomId, - ) -> Result<(SpaceRoomJoinRule, Vec)> { + ) -> (SpaceRoomJoinRule, Vec) { self.room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") .await - .map(|c: RoomJoinRulesEventContent| { - (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule)) - }) - .or_else(|_| Ok((SpaceRoomJoinRule::Invite, vec![]))) + .map_or_else( + |_| (SpaceRoomJoinRule::Invite, vec![]), + |c: RoomJoinRulesEventContent| { + (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule)) + }, + ) + } + + /// Returns the join rules for a given room (`JoinRule` type). Will default + /// to Invite if doesnt exist or invalid + pub async fn get_join_rules(&self, room_id: &RoomId) -> JoinRule { + self.room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") + .await + .map_or_else(|_| JoinRule::Invite, |c: RoomJoinRulesEventContent| (c.join_rule)) } /// Returns an empty vec if not a restricted room pub fn allowed_room_ids(&self, join_rule: JoinRule) -> Vec { - let mut room_ids = Vec::with_capacity(1); + let mut room_ids = Vec::with_capacity(1); // restricted rooms generally only have 1 allowed room ID if let JoinRule::Restricted(r) | JoinRule::KnockRestricted(r) = join_rule { for rule in r.allow { if let AllowRule::RoomMembership(RoomMembership { room_id: membership }) = rule { From 24be5794774b7585b6ec1e3dbaa901967d241972 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 3 Apr 2025 12:20:10 -0400 Subject: [PATCH 1630/2091] add appservice MSC4190 support Signed-off-by: June Clementine Strawberry --- Cargo.lock | 22 +++--- Cargo.toml | 2 +- src/api/client/account.rs | 12 ++-- src/api/client/appservice.rs | 8 ++- src/api/client/device.rs | 112 +++++++++++++++++++++++------- src/service/sending/appservice.rs | 18 +++-- src/service/users/mod.rs | 1 - 7 files changed, 125 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a53258bc..2bcfcee4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3531,7 +3531,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "assign", "js_int", @@ -3551,7 +3551,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "js_int", "ruma-common", @@ -3563,7 +3563,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "as_variant", "assign", @@ -3586,7 +3586,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "as_variant", "base64 0.22.1", @@ -3618,7 +3618,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "as_variant", "indexmap 2.8.0", @@ -3643,7 +3643,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "bytes", "headers", @@ -3665,7 +3665,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3674,7 +3674,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "js_int", "ruma-common", @@ -3684,7 +3684,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3699,7 +3699,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "js_int", "ruma-common", @@ -3711,7 +3711,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=ea1278657125e9414caada074e8c172bc252fb1c#ea1278657125e9414caada074e8c172bc252fb1c" +source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 940ece86..0abaa2f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -346,7 +346,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "ea1278657125e9414caada074e8c172bc252fb1c" +rev = "0701341a2fd5a6ea74beada18d5974cc401a4fc1" features = [ "compat", "rand", diff --git a/src/api/client/account.rs b/src/api/client/account.rs index efa8b142..e5894d47 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -318,14 +318,14 @@ pub(crate) async fn register_route( // Success! }, | _ => match body.json_body { - | Some(json) => { + | Some(ref json) => { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services.uiaa.create( &UserId::parse_with_server_name("", services.globals.server_name()) .unwrap(), "".into(), &uiaainfo, - &json, + json, ); return Err(Error::Uiaa(uiaainfo)); }, @@ -373,8 +373,12 @@ pub(crate) async fn register_route( ) .await?; - // Inhibit login does not work for guests - if !is_guest && body.inhibit_login { + if (!is_guest && body.inhibit_login) + || body + .appservice_info + .as_ref() + .is_some_and(|appservice| appservice.registration.device_management) + { return Ok(register::v3::Response { access_token: None, user_id, diff --git a/src/api/client/appservice.rs b/src/api/client/appservice.rs index 84955309..eb6b3312 100644 --- a/src/api/client/appservice.rs +++ b/src/api/client/appservice.rs @@ -22,7 +22,13 @@ pub(crate) async fn appservice_ping( ))); } - if appservice_info.registration.url.is_none() { + if appservice_info.registration.url.is_none() + || appservice_info + .registration + .url + .as_ref() + .is_some_and(|url| url.is_empty() || url == "null") + { return Err!(Request(UrlNotSet( "Appservice does not have a URL set, there is nothing to ping." ))); diff --git a/src/api/client/device.rs b/src/api/client/device.rs index 6a845aed..7603c866 100644 --- a/src/api/client/device.rs +++ b/src/api/client/device.rs @@ -1,9 +1,9 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{Err, err}; +use conduwuit::{Err, debug, err}; use futures::StreamExt; use ruma::{ - MilliSecondsSinceUnixEpoch, + MilliSecondsSinceUnixEpoch, OwnedDeviceId, api::client::{ device::{self, delete_device, delete_devices, get_device, get_devices, update_device}, error::ErrorKind, @@ -12,7 +12,7 @@ use ruma::{ }; use super::SESSION_ID_LENGTH; -use crate::{Error, Result, Ruma, utils}; +use crate::{Error, Result, Ruma, client::DEVICE_ID_LENGTH, utils}; /// # `GET /_matrix/client/r0/devices` /// @@ -59,26 +59,58 @@ pub(crate) async fn update_device_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); + let appservice = body.appservice_info.as_ref(); - let mut device = services + match services .users .get_device_metadata(sender_user, &body.device_id) .await - .map_err(|_| err!(Request(NotFound("Device not found."))))?; + { + | Ok(mut device) => { + device.display_name.clone_from(&body.display_name); + device.last_seen_ip.clone_from(&Some(client.to_string())); + device + .last_seen_ts + .clone_from(&Some(MilliSecondsSinceUnixEpoch::now())); - device.display_name.clone_from(&body.display_name); - device.last_seen_ip.clone_from(&Some(client.to_string())); - device - .last_seen_ts - .clone_from(&Some(MilliSecondsSinceUnixEpoch::now())); + services + .users + .update_device_metadata(sender_user, &body.device_id, &device) + .await?; - services - .users - .update_device_metadata(sender_user, &body.device_id, &device) - .await?; + Ok(update_device::v3::Response {}) + }, + | Err(_) => { + let Some(appservice) = appservice else { + return Err!(Request(NotFound("Device not found."))); + }; + if !appservice.registration.device_management { + return Err!(Request(NotFound("Device not found."))); + } - Ok(update_device::v3::Response {}) + debug!( + "Creating new device for {sender_user} from appservice {} as MSC4190 is enabled \ + and device ID does not exist", + appservice.registration.id + ); + + let device_id = OwnedDeviceId::from(utils::random_string(DEVICE_ID_LENGTH)); + + services + .users + .create_device( + sender_user, + &device_id, + &appservice.registration.as_token, + None, + Some(client.to_string()), + ) + .await?; + + return Ok(update_device::v3::Response {}); + }, + } } /// # `DELETE /_matrix/client/r0/devices/{deviceId}` @@ -95,8 +127,21 @@ pub(crate) async fn delete_device_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + let (sender_user, sender_device) = body.sender(); + let appservice = body.appservice_info.as_ref(); + + if appservice.is_some_and(|appservice| appservice.registration.device_management) { + debug!( + "Skipping UIAA for {sender_user} as this is from an appservice and MSC4190 is \ + enabled" + ); + services + .users + .remove_device(sender_user, &body.device_id) + .await; + + return Ok(delete_device::v3::Response {}); + } // UIAA let mut uiaainfo = UiaaInfo { @@ -120,11 +165,11 @@ pub(crate) async fn delete_device_route( // Success! }, | _ => match body.json_body { - | Some(json) => { + | Some(ref json) => { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + .create(sender_user, sender_device, &uiaainfo, json); return Err!(Uiaa(uiaainfo)); }, @@ -142,11 +187,12 @@ pub(crate) async fn delete_device_route( Ok(delete_device::v3::Response {}) } -/// # `PUT /_matrix/client/r0/devices/{deviceId}` +/// # `POST /_matrix/client/v3/delete_devices` /// -/// Deletes the given device. +/// Deletes the given list of devices. /// -/// - Requires UIAA to verify user password +/// - Requires UIAA to verify user password unless from an appservice with +/// MSC4190 enabled. /// /// For each device: /// - Invalidates access token @@ -158,8 +204,20 @@ pub(crate) async fn delete_devices_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + let (sender_user, sender_device) = body.sender(); + let appservice = body.appservice_info.as_ref(); + + if appservice.is_some_and(|appservice| appservice.registration.device_management) { + debug!( + "Skipping UIAA for {sender_user} as this is from an appservice and MSC4190 is \ + enabled" + ); + for device_id in &body.devices { + services.users.remove_device(sender_user, device_id).await; + } + + return Ok(delete_devices::v3::Response {}); + } // UIAA let mut uiaainfo = UiaaInfo { @@ -183,11 +241,11 @@ pub(crate) async fn delete_devices_route( // Success! }, | _ => match body.json_body { - | Some(json) => { + | Some(ref json) => { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + .create(sender_user, sender_device, &uiaainfo, json); return Err(Error::Uiaa(uiaainfo)); }, diff --git a/src/service/sending/appservice.rs b/src/service/sending/appservice.rs index 7fa0be9a..c7fae11f 100644 --- a/src/service/sending/appservice.rs +++ b/src/service/sending/appservice.rs @@ -25,6 +25,10 @@ where return Ok(None); }; + if dest == *"null" || dest.is_empty() { + return Ok(None); + } + trace!("Appservice URL \"{dest}\", Appservice ID: {}", registration.id); let hs_token = registration.hs_token.as_str(); @@ -34,7 +38,11 @@ where SendAccessToken::IfRequired(hs_token), &VERSIONS, ) - .map_err(|e| err!(BadServerResponse(warn!("Failed to find destination {dest}: {e}"))))? + .map_err(|e| { + err!(BadServerResponse( + warn!(appservice = %registration.id, "Failed to find destination {dest}: {e:?}") + )) + })? .map(BytesMut::freeze); let mut parts = http_request.uri().clone().into_parts(); @@ -51,7 +59,7 @@ where let reqwest_request = reqwest::Request::try_from(http_request)?; let mut response = client.execute(reqwest_request).await.map_err(|e| { - warn!("Could not send request to appservice \"{}\" at {dest}: {e}", registration.id); + warn!("Could not send request to appservice \"{}\" at {dest}: {e:?}", registration.id); e })?; @@ -71,7 +79,7 @@ where if !status.is_success() { debug_error!("Appservice response bytes: {:?}", utils::string_from_bytes(&body)); - return Err!(BadServerResponse(error!( + return Err!(BadServerResponse(warn!( "Appservice \"{}\" returned unsuccessful HTTP response {status} at {dest}", registration.id ))); @@ -84,8 +92,8 @@ where ); response.map(Some).map_err(|e| { - err!(BadServerResponse(error!( - "Appservice \"{}\" returned invalid response bytes {dest}: {e}", + err!(BadServerResponse(warn!( + "Appservice \"{}\" returned invalid/malformed response bytes {dest}: {e}", registration.id ))) }) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 5265e64b..87a8b93b 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -350,7 +350,6 @@ impl Service { token: &str, ) -> Result<()> { let key = (user_id, device_id); - // should not be None, but we shouldn't assert either lol... if self.db.userdeviceid_metadata.qry(&key).await.is_err() { return Err!(Database(error!( ?user_id, From f14756fb767abda97dc966ad842c958d970d77b9 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 3 Apr 2025 12:20:53 -0400 Subject: [PATCH 1631/2091] leave room locally if room is banned, rescind knocks on deactivation too Signed-off-by: June Clementine Strawberry --- src/api/client/membership.rs | 87 +++++++++++++++++++++------- src/api/client/sync/v3.rs | 12 ++-- src/api/client/sync/v4.rs | 5 +- src/api/client/sync/v5.rs | 5 +- src/service/rooms/state_cache/mod.rs | 6 +- 5 files changed, 87 insertions(+), 28 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 315a363c..ef40e972 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -475,9 +475,9 @@ pub(crate) async fn leave_room_route( State(services): State, body: Ruma, ) -> Result { - leave_room(&services, body.sender_user(), &body.room_id, body.reason.clone()).await?; - - Ok(leave_room::v3::Response::new()) + leave_room(&services, body.sender_user(), &body.room_id, body.reason.clone()) + .await + .map(|()| leave_room::v3::Response::new()) } /// # `POST /_matrix/client/r0/rooms/{roomId}/invite` @@ -1763,8 +1763,8 @@ pub(crate) async fn invite_helper( Ok(()) } -// Make a user leave all their joined rooms, forgets all rooms, and ignores -// errors +// Make a user leave all their joined rooms, rescinds knocks, forgets all rooms, +// and ignores errors pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { let rooms_joined = services .rooms @@ -1778,7 +1778,17 @@ pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { .rooms_invited(user_id) .map(|(r, _)| r); - let all_rooms: Vec<_> = rooms_joined.chain(rooms_invited).collect().await; + let rooms_knocked = services + .rooms + .state_cache + .rooms_knocked(user_id) + .map(|(r, _)| r); + + let all_rooms: Vec<_> = rooms_joined + .chain(rooms_invited) + .chain(rooms_knocked) + .collect() + .await; for room_id in all_rooms { // ignore errors @@ -1795,7 +1805,40 @@ pub async fn leave_room( user_id: &UserId, room_id: &RoomId, reason: Option, -) -> Result<()> { +) -> Result { + let default_member_content = RoomMemberEventContent { + membership: MembershipState::Leave, + reason: reason.clone(), + join_authorized_via_users_server: None, + is_direct: None, + avatar_url: None, + displayname: None, + third_party_invite: None, + blurhash: None, + }; + + if services.rooms.metadata.is_banned(room_id).await + || services.rooms.metadata.is_disabled(room_id).await + { + // the room is banned/disabled, the room must be rejected locally since we + // cant/dont want to federate with this server + services + .rooms + .state_cache + .update_membership( + room_id, + user_id, + default_member_content, + user_id, + None, + None, + true, + ) + .await?; + + return Ok(()); + } + // Ask a remote server if we don't have this room and are not knocking on it if !services .rooms @@ -1828,7 +1871,7 @@ pub async fn leave_room( .update_membership( room_id, user_id, - RoomMemberEventContent::new(MembershipState::Leave), + default_member_content, user_id, last_state, None, @@ -1848,26 +1891,23 @@ pub async fn leave_room( ) .await else { - // Fix for broken rooms - warn!( + debug_warn!( "Trying to leave a room you are not a member of, marking room as left locally." ); - services + return services .rooms .state_cache .update_membership( room_id, user_id, - RoomMemberEventContent::new(MembershipState::Leave), + default_member_content, user_id, None, None, true, ) - .await?; - - return Ok(()); + .await; }; services @@ -1897,7 +1937,7 @@ async fn remote_leave_room( room_id: &RoomId, ) -> Result<()> { let mut make_leave_response_and_server = - Err!(BadServerResponse("No server available to assist in leaving.")); + Err!(BadServerResponse("No remote server available to assist in leaving {room_id}.")); let mut servers: HashSet = services .rooms @@ -1977,20 +2017,25 @@ async fn remote_leave_room( let (make_leave_response, remote_server) = make_leave_response_and_server?; let Some(room_version_id) = make_leave_response.room_version else { - return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); + return Err!(BadServerResponse(warn!( + "No room version was returned by {remote_server} for {room_id}, room version is \ + likely not supported by conduwuit" + ))); }; if !services.server.supported_room_version(&room_version_id) { - return Err!(BadServerResponse( - "Remote room version {room_version_id} is not supported by conduwuit" - )); + return Err!(BadServerResponse(warn!( + "Remote room version {room_version_id} for {room_id} is not supported by conduwuit", + ))); } let mut leave_event_stub = serde_json::from_str::( make_leave_response.event.get(), ) .map_err(|e| { - err!(BadServerResponse("Invalid make_leave event json received from server: {e:?}")) + err!(BadServerResponse(warn!( + "Invalid make_leave event json received from {remote_server} for {room_id}: {e:?}" + ))) })?; // TODO: Is origin needed? diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 530c1278..83ffa55a 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -15,6 +15,7 @@ use conduwuit::{ math::ruma_from_u64, stream::{BroadbandExt, Tools, TryExpect, WidebandExt}, }, + warn, }; use conduwuit_service::{ Services, @@ -428,9 +429,12 @@ async fn handle_left_room( return Ok(None); } - if !services.rooms.metadata.exists(room_id).await { + if !services.rooms.metadata.exists(room_id).await + || services.rooms.metadata.is_disabled(room_id).await + || services.rooms.metadata.is_banned(room_id).await + { // This is just a rejected invite, not a room we know - // Insert a leave event anyways + // Insert a leave event anyways for the client let event = PduEvent { event_id: EventId::new(services.globals.server_name()), sender: sender_user.to_owned(), @@ -489,7 +493,7 @@ async fn handle_left_room( .room_state_get_id(room_id, &StateEventType::RoomMember, sender_user.as_str()) .await else { - error!("Left room but no left state event"); + warn!("Left {room_id} but no left state event"); return Ok(None); }; @@ -499,7 +503,7 @@ async fn handle_left_room( .pdu_shortstatehash(&left_event_id) .await else { - error!(event_id = %left_event_id, "Leave event has no state"); + warn!(event_id = %left_event_id, "Leave event has no state in {room_id}"); return Ok(None); }; diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 7e902973..f7edb8c0 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -438,7 +438,10 @@ pub(crate) async fn sync_events_v4_route( let mut known_subscription_rooms = BTreeSet::new(); for (room_id, room) in &body.room_subscriptions { - if !services.rooms.metadata.exists(room_id).await { + if !services.rooms.metadata.exists(room_id).await + || services.rooms.metadata.is_disabled(room_id).await + || services.rooms.metadata.is_banned(room_id).await + { continue; } let todo_room = diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index 48b41b21..c4e71d88 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -214,7 +214,10 @@ async fn fetch_subscriptions( ) { let mut known_subscription_rooms = BTreeSet::new(); for (room_id, room) in &body.room_subscriptions { - if !services.rooms.metadata.exists(room_id).await { + if !services.rooms.metadata.exists(room_id).await + || services.rooms.metadata.is_disabled(room_id).await + || services.rooms.metadata.is_banned(room_id).await + { continue; } let todo_room = diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index 23ba0520..d3dbc143 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -40,6 +40,7 @@ struct Services { account_data: Dep, config: Dep, globals: Dep, + metadata: Dep, state_accessor: Dep, users: Dep, } @@ -73,6 +74,7 @@ impl crate::Service for Service { account_data: args.depend::("account_data"), config: args.depend::("config"), globals: args.depend::("globals"), + metadata: args.depend::("rooms::metadata"), state_accessor: args .depend::("rooms::state_accessor"), users: args.depend::("users"), @@ -271,7 +273,9 @@ impl Service { self.mark_as_left(user_id, room_id); if self.services.globals.user_is_local(user_id) - && self.services.config.forget_forced_upon_leave + && (self.services.config.forget_forced_upon_leave + || self.services.metadata.is_banned(room_id).await + || self.services.metadata.is_disabled(room_id).await) { self.forget(room_id, user_id); } From 5d1404e9dfff9bc0e5bed4bab6d75c9c94b38183 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Apr 2025 02:52:42 +0000 Subject: [PATCH 1632/2091] fix well-known using the hooked resolver Signed-off-by: Jason Volk --- src/service/client/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/client/mod.rs b/src/service/client/mod.rs index d51e5721..1aeeb492 100644 --- a/src/service/client/mod.rs +++ b/src/service/client/mod.rs @@ -56,7 +56,7 @@ impl crate::Service for Service { .build()?, well_known: base(config)? - .dns_resolver(resolver.resolver.hooked.clone()) + .dns_resolver(resolver.resolver.clone()) .connect_timeout(Duration::from_secs(config.well_known_conn_timeout)) .read_timeout(Duration::from_secs(config.well_known_timeout)) .timeout(Duration::from_secs(config.well_known_timeout)) From 58adb6fead27c863849c63184f145be209e40e1b Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Apr 2025 04:05:42 +0000 Subject: [PATCH 1633/2091] upgrade hickory and hyper-util dependencies Signed-off-by: Jason Volk --- Cargo.lock | 195 +++++++++++++++++++++++++++++++-- Cargo.toml | 10 +- src/service/resolver/actual.rs | 39 ++++--- src/service/resolver/dns.rs | 24 ++-- 4 files changed, 229 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2bcfcee4..545f0f0d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,6 +142,17 @@ dependencies = [ "zstd-safe", ] +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -927,7 +938,7 @@ dependencies = [ "const-str", "either", "futures", - "hickory-resolver", + "hickory-resolver 0.25.1", "http", "image", "ipaddress", @@ -1061,6 +1072,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crokey" version = "1.1.1" @@ -1584,6 +1601,19 @@ dependencies = [ "slab", ] +[[package]] +name = "generator" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" +dependencies = [ + "cfg-if", + "libc", + "log", + "rustversion", + "windows 0.58.0", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1769,6 +1799,34 @@ dependencies = [ "url", ] +[[package]] +name = "hickory-proto" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d844af74f7b799e41c78221be863bade11c430d46042c3b49ca8ae0c6d27287" +dependencies = [ + "async-recursion", + "async-trait", + "cfg-if", + "critical-section", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.9.0", + "ring", + "serde", + "thiserror 2.0.12", + "tinyvec", + "tokio", + "tracing", + "url", +] + [[package]] name = "hickory-resolver" version = "0.24.4" @@ -1777,7 +1835,7 @@ checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e" dependencies = [ "cfg-if", "futures-util", - "hickory-proto", + "hickory-proto 0.24.4", "ipconfig", "lru-cache", "once_cell", @@ -1790,6 +1848,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "hickory-resolver" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a128410b38d6f931fcc6ca5c107a3b02cabd6c05967841269a4ad65d23c44331" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto 0.25.1", + "ipconfig", + "moka", + "once_cell", + "parking_lot", + "rand 0.9.0", + "resolv-conf", + "serde", + "smallvec", + "thiserror 2.0.12", + "tokio", + "tracing", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1816,7 +1896,7 @@ checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" dependencies = [ "cfg-if", "libc", - "windows", + "windows 0.52.0", ] [[package]] @@ -1949,9 +2029,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.8" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da62f120a8a37763efb0cf8fdf264b884c7b8b9ac8660b900c8661030c00e6ba" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-channel", @@ -1959,10 +2039,10 @@ dependencies = [ "http", "http-body", "hyper", + "libc", "pin-project-lite", "socket2", "tokio", - "tower 0.4.13", "tower-service", "tracing", ] @@ -2439,6 +2519,19 @@ dependencies = [ "futures-sink", ] +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "loop9" version = "0.1.5" @@ -2609,6 +2702,25 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "moka" +version = "0.12.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "loom", + "parking_lot", + "portable-atomic", + "rustc_version", + "smallvec", + "tagptr", + "thiserror 1.0.69", + "uuid", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2773,6 +2885,10 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "openssl-probe" @@ -3052,6 +3168,12 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "portable-atomic" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + [[package]] name = "powerfmt" version = "0.2.0" @@ -3463,7 +3585,7 @@ dependencies = [ "futures-core", "futures-util", "h2", - "hickory-resolver", + "hickory-resolver 0.24.4", "http", "http-body", "http-body-util", @@ -3893,6 +4015,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -4464,6 +4592,12 @@ dependencies = [ "version-compare", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "target-lexicon" version = "0.12.16" @@ -5367,7 +5501,17 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ - "windows-core", + "windows-core 0.52.0", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +dependencies = [ + "windows-core 0.58.0", "windows-targets 0.52.6", ] @@ -5380,6 +5524,41 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-core" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-result", + "windows-strings", + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-implement" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.58.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-registry" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index 0abaa2f9..6c5c291f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -284,8 +284,7 @@ features = [ ] [workspace.dependencies.hyper-util] -# hyper-util >=0.1.9 seems to have DNS issues -version = "=0.1.8" +version = "0.1.11" default-features = false features = [ "server-auto", @@ -306,8 +305,13 @@ default-features = false features = ["env", "toml"] [workspace.dependencies.hickory-resolver] -version = "0.24.2" +version = "0.25.1" default-features = false +features = [ + "serde", + "system-config", + "tokio", +] # Used for conduwuit::Error type [workspace.dependencies.thiserror] diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 1ad76f66..0151c4d7 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -5,7 +5,7 @@ use std::{ use conduwuit::{Err, Result, debug, debug_info, err, error, trace}; use futures::{FutureExt, TryFutureExt}; -use hickory_resolver::error::ResolveError; +use hickory_resolver::ResolveError; use ipaddress::IPAddress; use ruma::ServerName; @@ -334,25 +334,28 @@ impl super::Service { } fn handle_resolve_error(e: &ResolveError, host: &'_ str) -> Result<()> { - use hickory_resolver::error::ResolveErrorKind; + use hickory_resolver::{ResolveErrorKind::Proto, proto::ProtoErrorKind}; - match *e.kind() { - | ResolveErrorKind::NoRecordsFound { .. } => { - // Raise to debug_warn if we can find out the result wasn't from cache - debug!(%host, "No DNS records found: {e}"); - Ok(()) - }, - | ResolveErrorKind::Timeout => { - Err!(warn!(%host, "DNS {e}")) - }, - | ResolveErrorKind::NoConnections => { - error!( - "Your DNS server is overloaded and has ran out of connections. It is \ - strongly recommended you remediate this issue to ensure proper federation \ - connectivity." - ); + match e.kind() { + | Proto(e) => match e.kind() { + | ProtoErrorKind::NoRecordsFound { .. } => { + // Raise to debug_warn if we can find out the result wasn't from cache + debug!(%host, "No DNS records found: {e}"); + Ok(()) + }, + | ProtoErrorKind::Timeout => { + Err!(warn!(%host, "DNS {e}")) + }, + | ProtoErrorKind::NoConnections => { + error!( + "Your DNS server is overloaded and has ran out of connections. It is \ + strongly recommended you remediate this issue to ensure proper \ + federation connectivity." + ); - Err!(error!(%host, "DNS error: {e}")) + Err!(error!(%host, "DNS error: {e}")) + }, + | _ => Err!(error!(%host, "DNS error: {e}")), }, | _ => Err!(error!(%host, "DNS error: {e}")), } diff --git a/src/service/resolver/dns.rs b/src/service/resolver/dns.rs index e4245a5b..3a0b2551 100644 --- a/src/service/resolver/dns.rs +++ b/src/service/resolver/dns.rs @@ -2,19 +2,19 @@ use std::{net::SocketAddr, sync::Arc, time::Duration}; use conduwuit::{Result, Server, err}; use futures::FutureExt; -use hickory_resolver::{TokioAsyncResolver, lookup_ip::LookupIp}; +use hickory_resolver::{TokioResolver, lookup_ip::LookupIp}; use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use super::cache::{Cache, CachedOverride}; pub struct Resolver { - pub(crate) resolver: Arc, + pub(crate) resolver: Arc, pub(crate) hooked: Arc, server: Arc, } pub(crate) struct Hooked { - resolver: Arc, + resolver: Arc, cache: Arc, server: Arc, } @@ -42,7 +42,7 @@ impl Resolver { let mut ns = sys_conf.clone(); if config.query_over_tcp_only { - ns.protocol = hickory_resolver::config::Protocol::Tcp; + ns.protocol = hickory_resolver::proto::xfer::Protocol::Tcp; } ns.trust_negative_responses = !config.query_all_nameservers; @@ -51,6 +51,7 @@ impl Resolver { } opts.cache_size = config.dns_cache_entries as usize; + opts.preserve_intermediates = true; opts.negative_min_ttl = Some(Duration::from_secs(config.dns_min_ttl_nxdomain)); opts.negative_max_ttl = Some(Duration::from_secs(60 * 60 * 24 * 30)); opts.positive_min_ttl = Some(Duration::from_secs(config.dns_min_ttl)); @@ -60,8 +61,7 @@ impl Resolver { opts.try_tcp_on_error = config.dns_tcp_fallback; opts.num_concurrent_reqs = 1; opts.edns0 = true; - opts.shuffle_dns_servers = true; - opts.rotate = true; + opts.case_randomization = true; opts.ip_strategy = match config.ip_lookup_strategy { | 1 => hickory_resolver::config::LookupIpStrategy::Ipv4Only, | 2 => hickory_resolver::config::LookupIpStrategy::Ipv6Only, @@ -69,9 +69,13 @@ impl Resolver { | 4 => hickory_resolver::config::LookupIpStrategy::Ipv6thenIpv4, | _ => hickory_resolver::config::LookupIpStrategy::Ipv4thenIpv6, }; - opts.authentic_data = false; - let resolver = Arc::new(TokioAsyncResolver::tokio(conf, opts)); + let rt_prov = hickory_resolver::proto::runtime::TokioRuntimeProvider::new(); + let conn_prov = hickory_resolver::name_server::TokioConnectionProvider::new(rt_prov); + let mut builder = TokioResolver::builder_with_config(conf, conn_prov); + *builder.options_mut() = opts; + let resolver = Arc::new(builder.build()); + Ok(Arc::new(Self { resolver: resolver.clone(), hooked: Arc::new(Hooked { resolver, cache, server: server.clone() }), @@ -105,7 +109,7 @@ impl Resolve for Hooked { async fn hooked_resolve( cache: Arc, server: Arc, - resolver: Arc, + resolver: Arc, name: Name, ) -> Result> { match cache.get_override(name.as_str()).await { @@ -129,7 +133,7 @@ async fn hooked_resolve( async fn resolve_to_reqwest( server: Arc, - resolver: Arc, + resolver: Arc, name: Name, ) -> ResolvingResult { use std::{io, io::ErrorKind::Interrupted}; From 0b56204f89d37470346c1940e70354deebfd1a3a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Apr 2025 04:34:11 +0000 Subject: [PATCH 1634/2091] bump additional dependencies Signed-off-by: Jason Volk --- Cargo.lock | 264 ++++++++++++++++++++++++++++++----------------------- Cargo.toml | 20 ++-- 2 files changed, 161 insertions(+), 123 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 545f0f0d..da33af05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -232,9 +232,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.12.6" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" dependencies = [ "aws-lc-sys", "zeroize", @@ -242,9 +242,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f" +checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" dependencies = [ "bindgen 0.69.5", "cc", @@ -663,9 +663,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.34" +version = "4.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e958897981290da2a852763fe9cdb89cd36977a5d729023127095fa94d95e2ff" +checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" dependencies = [ "clap_builder", "clap_derive", @@ -673,9 +673,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.34" +version = "4.5.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83b0f35019843db2160b5bb19ae09b4e6411ac33fc6a712003c33e03090e2489" +checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" dependencies = [ "anstyle", "clap_lex", @@ -742,8 +742,8 @@ dependencies = [ "opentelemetry-jaeger", "opentelemetry_sdk", "sentry", - "sentry-tower 0.35.0", - "sentry-tracing 0.35.0", + "sentry-tower", + "sentry-tracing", "tokio", "tokio-metrics", "tracing", @@ -916,8 +916,8 @@ dependencies = [ "rustls", "sd-notify", "sentry", - "sentry-tower 0.35.0", - "sentry-tracing 0.35.0", + "sentry-tower", + "sentry-tracing", "serde_json", "tokio", "tower 0.5.2", @@ -1454,9 +1454,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11faaf5a5236997af9848be0bef4db95824b1d534ebc64d0f0c6cf3e67bd38dc" +checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" dependencies = [ "crc32fast", "miniz_oxide", @@ -2016,9 +2016,9 @@ dependencies = [ [[package]] name = "hyper-timeout" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ "hyper", "hyper-util", @@ -2336,10 +2336,11 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.2", "libc", ] @@ -3574,9 +3575,9 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ "async-compression", "base64 0.22.1", @@ -3612,6 +3613,7 @@ dependencies = [ "tokio-rustls", "tokio-socks", "tokio-util", + "tower 0.5.2", "tower-service", "url", "wasm-bindgen", @@ -4067,21 +4069,21 @@ checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "sentry" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a7332159e544e34db06b251b1eda5e546bd90285c3f58d9c8ff8450b484e0da" +checksum = "255914a8e53822abd946e2ce8baa41d4cded6b8e938913b7f7b9da5b7ab44335" dependencies = [ "httpdate", "reqwest", "rustls", "sentry-backtrace", "sentry-contexts", - "sentry-core 0.36.0", + "sentry-core", "sentry-debug-images", "sentry-log", "sentry-panic", - "sentry-tower 0.36.0", - "sentry-tracing 0.36.0", + "sentry-tower", + "sentry-tracing", "tokio", "ureq", "webpki-roots", @@ -4089,107 +4091,83 @@ dependencies = [ [[package]] name = "sentry-backtrace" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "565ec31ad37bab8e6d9f289f34913ed8768347b133706192f10606dabd5c6bc4" +checksum = "00293cd332a859961f24fd69258f7e92af736feaeb91020cff84dac4188a4302" dependencies = [ "backtrace", "once_cell", "regex", - "sentry-core 0.36.0", + "sentry-core", ] [[package]] name = "sentry-contexts" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e860275f25f27e8c0c7726ce116c7d5c928c5bba2ee73306e52b20a752298ea6" +checksum = "961990f9caa76476c481de130ada05614cd7f5aa70fb57c2142f0e09ad3fb2aa" dependencies = [ "hostname", "libc", "os_info", "rustc_version", - "sentry-core 0.36.0", + "sentry-core", "uname", ] [[package]] name = "sentry-core" -version = "0.35.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9f8b6dcd4fbae1e3e22b447f32670360b27e31b62ab040f7fb04e0f80c04d92" +checksum = "1a6409d845707d82415c800290a5d63be5e3df3c2e417b0997c60531dfbd35ef" dependencies = [ "once_cell", "rand 0.8.5", - "sentry-types 0.35.0", - "serde", - "serde_json", -] - -[[package]] -name = "sentry-core" -version = "0.36.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "653942e6141f16651273159f4b8b1eaeedf37a7554c00cd798953e64b8a9bf72" -dependencies = [ - "once_cell", - "rand 0.8.5", - "sentry-types 0.36.0", + "sentry-types", "serde", "serde_json", ] [[package]] name = "sentry-debug-images" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60bc2154e6df59beed0ac13d58f8dfaf5ad20a88548a53e29e4d92e8e835c2" +checksum = "71ab5df4f3b64760508edfe0ba4290feab5acbbda7566a79d72673065888e5cc" dependencies = [ "findshlibs", "once_cell", - "sentry-core 0.36.0", + "sentry-core", ] [[package]] name = "sentry-log" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c96d796cba1b3a0793e7f53edc420c61f9419fba8fb34ad5519f5c7d01af6b2" +checksum = "693841da8dfb693af29105edfbea1d91348a13d23dd0a5d03761eedb9e450c46" dependencies = [ "log", - "sentry-core 0.36.0", + "sentry-core", ] [[package]] name = "sentry-panic" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "105e3a956c8aa9dab1e4087b1657b03271bfc49d838c6ae9bfc7c58c802fd0ef" +checksum = "609b1a12340495ce17baeec9e08ff8ed423c337c1a84dffae36a178c783623f3" dependencies = [ "sentry-backtrace", - "sentry-core 0.36.0", + "sentry-core", ] [[package]] name = "sentry-tower" -version = "0.35.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdaf9b1939589476bd57751d12a9653bbfe356610fc476d03d7683189183ab7" -dependencies = [ - "sentry-core 0.35.0", - "tower-layer", - "tower-service", -] - -[[package]] -name = "sentry-tower" -version = "0.36.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "082f781dfc504d984e16d99f8dbf94d6ee4762dd0fc28de25713d0f900a8164d" +checksum = "4b98005537e38ee3bc10e7d36e7febe9b8e573d03f2ddd85fcdf05d21f9abd6d" dependencies = [ "http", "pin-project", - "sentry-core 0.36.0", + "sentry-core", "tower-layer", "tower-service", "url", @@ -4197,49 +4175,21 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.35.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "263f73c757ed7915d3e1e34625eae18cad498a95b4261603d4ce3f87b159a6f0" -dependencies = [ - "sentry-core 0.35.0", - "tracing-core", - "tracing-subscriber", -] - -[[package]] -name = "sentry-tracing" -version = "0.36.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64e75c831b4d8b34a5aec1f65f67c5d46a26c7c5d3c7abd8b5ef430796900cf8" +checksum = "49f4e86402d5c50239dc7d8fd3f6d5e048221d5fcb4e026d8d50ab57fe4644cb" dependencies = [ "sentry-backtrace", - "sentry-core 0.36.0", + "sentry-core", "tracing-core", "tracing-subscriber", ] [[package]] name = "sentry-types" -version = "0.35.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a71ed3a389948a6a6d92b98e997a2723ca22f09660c5a7b7388ecd509a70a527" -dependencies = [ - "debugid", - "hex", - "rand 0.8.5", - "serde", - "serde_json", - "thiserror 1.0.69", - "time", - "url", - "uuid", -] - -[[package]] -name = "sentry-types" -version = "0.36.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d4203359e60724aa05cf2385aaf5d4f147e837185d7dd2b9ccf1ee77f4420c8" +checksum = "3d3f117b8755dbede8260952de2aeb029e20f432e72634e8969af34324591631" dependencies = [ "debugid", "hex", @@ -5532,8 +5482,8 @@ checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" dependencies = [ "windows-implement", "windows-interface", - "windows-result", - "windows-strings", + "windows-result 0.2.0", + "windows-strings 0.1.0", "windows-targets 0.52.6", ] @@ -5560,14 +5510,20 @@ dependencies = [ ] [[package]] -name = "windows-registry" -version = "0.2.0" +name = "windows-link" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + +[[package]] +name = "windows-registry" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "windows-result", - "windows-strings", - "windows-targets 0.52.6", + "windows-result 0.3.2", + "windows-strings 0.3.1", + "windows-targets 0.53.0", ] [[package]] @@ -5579,16 +5535,34 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-result" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-strings" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-targets 0.52.6", ] +[[package]] +name = "windows-strings" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -5640,13 +5614,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5659,6 +5649,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -5671,6 +5667,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -5683,12 +5685,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -5701,6 +5715,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -5713,6 +5733,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -5725,6 +5751,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -5737,6 +5769,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + [[package]] name = "winnow" version = "0.7.4" diff --git a/Cargo.toml b/Cargo.toml index 6c5c291f..3ffa9e44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -141,12 +141,12 @@ features = [ ] [workspace.dependencies.rustls] -version = "0.23.19" +version = "0.23.25" default-features = false features = ["aws_lc_rs"] [workspace.dependencies.reqwest] -version = "0.12.9" +version = "0.12.15" default-features = false features = [ "rustls-tls-native-roots", @@ -204,7 +204,7 @@ features = [ # logging [workspace.dependencies.log] -version = "0.4.22" +version = "0.4.27" default-features = false [workspace.dependencies.tracing] version = "0.1.41" @@ -224,7 +224,7 @@ default-features = false # used for conduwuit's CLI and admin room command parsing [workspace.dependencies.clap] -version = "4.5.23" +version = "4.5.35" default-features = false features = [ "derive", @@ -320,7 +320,7 @@ default-features = false # Used when hashing the state [workspace.dependencies.ring] -version = "0.17.8" +version = "0.17.14" default-features = false # Used to make working with iterators easier, was already a transitive depdendency @@ -427,7 +427,7 @@ features = ["rt-tokio"] # optional sentry metrics for crash/panic reporting [workspace.dependencies.sentry] -version = "0.36.0" +version = "0.37.0" default-features = false features = [ "backtrace", @@ -443,9 +443,9 @@ features = [ ] [workspace.dependencies.sentry-tracing] -version = "0.35.0" +version = "0.37.0" [workspace.dependencies.sentry-tower] -version = "0.35.0" +version = "0.37.0" # jemalloc usage [workspace.dependencies.tikv-jemalloc-sys] @@ -479,7 +479,7 @@ default-features = false features = ["resource"] [workspace.dependencies.sd-notify] -version = "0.4.3" +version = "0.4.5" default-features = false [workspace.dependencies.hardened_malloc-rs] @@ -496,7 +496,7 @@ version = "0.4.3" default-features = false [workspace.dependencies.termimad] -version = "0.31.1" +version = "0.31.2" default-features = false [workspace.dependencies.checked_ops] From f9529937ce9a8dacf186fb4f60ef0c3315bb02a0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Apr 2025 19:36:24 +0000 Subject: [PATCH 1635/2091] patch hyper-util due to conflicts with federation resolver hooks Signed-off-by: Jason Volk --- Cargo.lock | 3 +-- Cargo.toml | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da33af05..8918a631 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2030,8 +2030,7 @@ dependencies = [ [[package]] name = "hyper-util" version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" +source = "git+https://github.com/girlbossceo/hyper-util?rev=e4ae7628fe4fcdacef9788c4c8415317a4489941#e4ae7628fe4fcdacef9788c4c8415317a4489941" dependencies = [ "bytes", "futures-channel", diff --git a/Cargo.toml b/Cargo.toml index 3ffa9e44..bf7ec2bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -570,10 +570,16 @@ rev = "fe4aebeeaae435af60087ddd56b573a2e0be671d" git = "https://github.com/girlbossceo/async-channel" rev = "92e5e74063bf2a3b10414bcc8a0d68b235644280" +# adds affinity masks for selecting more than one core at a time [patch.crates-io.core_affinity] git = "https://github.com/girlbossceo/core_affinity_rs" rev = "9c8e51510c35077df888ee72a36b4b05637147da" +# reverts hyperium#148 conflicting with our delicate federation resolver hooks +[patch.crates-io.hyper-util] +git = "https://github.com/girlbossceo/hyper-util" +rev = "e4ae7628fe4fcdacef9788c4c8415317a4489941" + # # Our crates # From 45fd3875c8932e56d1ab092004065b0800861201 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Apr 2025 00:59:23 +0000 Subject: [PATCH 1636/2091] move runtime shutdown out of main; gather final stats Signed-off-by: Jason Volk --- src/main/main.rs | 7 +++---- src/main/runtime.rs | 43 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/src/main/main.rs b/src/main/main.rs index 52f40384..1a9d3fe4 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -16,15 +16,14 @@ use server::Server; rustc_flags_capture! {} -fn main() -> Result<(), Error> { +fn main() -> Result { let args = clap::parse(); let runtime = runtime::new(&args)?; let server = Server::new(&args, Some(runtime.handle()))?; + runtime.spawn(signal::signal(server.clone())); runtime.block_on(async_main(&server))?; - - // explicit drop here to trace thread and tls dtors - drop(runtime); + runtime::shutdown(&server, runtime); #[cfg(unix)] if server.server.restarting.load(Ordering::Acquire) { diff --git a/src/main/runtime.rs b/src/main/runtime.rs index 920476db..1c58ea81 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -1,7 +1,7 @@ use std::{ iter::once, sync::{ - OnceLock, + Arc, OnceLock, atomic::{AtomicUsize, Ordering}, }, thread, @@ -11,17 +11,18 @@ use std::{ #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] use conduwuit_core::result::LogDebugErr; use conduwuit_core::{ - Result, is_true, + Result, debug, is_true, utils::sys::compute::{nth_core_available, set_affinity}, }; use tokio::runtime::Builder; -use crate::clap::Args; +use crate::{clap::Args, server::Server}; const WORKER_NAME: &str = "conduwuit:worker"; const WORKER_MIN: usize = 2; const WORKER_KEEPALIVE: u64 = 36; const MAX_BLOCKING_THREADS: usize = 1024; +const SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(10000); #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] const DISABLE_MUZZY_THRESHOLD: usize = 4; @@ -83,6 +84,42 @@ fn enable_histogram(builder: &mut Builder, args: &Args) { .metrics_poll_time_histogram_configuration(linear); } +#[cfg(tokio_unstable)] +#[tracing::instrument(name = "stop", level = "info", skip_all)] +pub(super) fn shutdown(server: &Arc, runtime: tokio::runtime::Runtime) { + use conduwuit_core::event; + use tracing::Level; + + // The final metrics output is promoted to INFO when tokio_unstable is active in + // a release/bench mode and DEBUG is likely optimized out + const LEVEL: Level = if cfg!(debug_assertions) { + Level::DEBUG + } else { + Level::INFO + }; + + debug!( + timeout = ?SHUTDOWN_TIMEOUT, + "Waiting for runtime..." + ); + + runtime.shutdown_timeout(SHUTDOWN_TIMEOUT); + let runtime_metrics = server.server.metrics.runtime_interval().unwrap_or_default(); + + event!(LEVEL, ?runtime_metrics, "Final runtime metrics"); +} + +#[cfg(not(tokio_unstable))] +#[tracing::instrument(name = "stop", level = "info", skip_all)] +pub(super) fn shutdown(_server: &Arc, runtime: tokio::runtime::Runtime) { + debug!( + timeout = ?SHUTDOWN_TIMEOUT, + "Waiting for runtime..." + ); + + runtime.shutdown_timeout(SHUTDOWN_TIMEOUT); +} + #[tracing::instrument( name = "fork", level = "debug", From 29d55b80366e17737094d3ad9a8031fe20c6286e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 2 Apr 2025 04:12:24 +0000 Subject: [PATCH 1637/2091] move systemd stopping notification point Signed-off-by: Jason Volk --- src/core/server.rs | 19 ++++++++++--------- src/router/run.rs | 4 ++++ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/core/server.rs b/src/core/server.rs index b67759d6..4b673f32 100644 --- a/src/core/server.rs +++ b/src/core/server.rs @@ -69,10 +69,6 @@ impl Server { return Err!("Reloading not enabled"); } - #[cfg(all(feature = "systemd", target_os = "linux"))] - sd_notify::notify(true, &[sd_notify::NotifyState::Reloading]) - .expect("failed to notify systemd of reloading state"); - if self.reloading.swap(true, Ordering::AcqRel) { return Err!("Reloading already in progress"); } @@ -98,10 +94,6 @@ impl Server { } pub fn shutdown(&self) -> Result { - #[cfg(all(feature = "systemd", target_os = "linux"))] - sd_notify::notify(true, &[sd_notify::NotifyState::Stopping]) - .expect("failed to notify systemd of stopping state"); - if self.stopping.swap(true, Ordering::AcqRel) { return Err!("Shutdown already in progress"); } @@ -144,7 +136,16 @@ impl Server { } #[inline] - pub fn running(&self) -> bool { !self.stopping.load(Ordering::Acquire) } + pub fn running(&self) -> bool { !self.is_stopping() } + + #[inline] + pub fn is_stopping(&self) -> bool { self.stopping.load(Ordering::Relaxed) } + + #[inline] + pub fn is_reloading(&self) -> bool { self.reloading.load(Ordering::Relaxed) } + + #[inline] + pub fn is_restarting(&self) -> bool { self.restarting.load(Ordering::Relaxed) } #[inline] pub fn is_ours(&self, name: &str) -> bool { name == self.config.server_name } diff --git a/src/router/run.rs b/src/router/run.rs index 31789626..ff54594f 100644 --- a/src/router/run.rs +++ b/src/router/run.rs @@ -77,6 +77,10 @@ pub(crate) async fn start(server: Arc) -> Result> { pub(crate) async fn stop(services: Arc) -> Result<()> { debug!("Shutting down..."); + #[cfg(all(feature = "systemd", target_os = "linux"))] + sd_notify::notify(true, &[sd_notify::NotifyState::Stopping]) + .expect("failed to notify systemd of stopping state"); + // Wait for all completions before dropping or we'll lose them to the module // unload and explode. services.stop().await; From 94b107b42b722aff9518f64ad603ce01665b25f3 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Thu, 3 Apr 2025 16:08:02 -0400 Subject: [PATCH 1638/2091] add some debug logging and misc cleanup to keys/signatures/upload Signed-off-by: June Clementine Strawberry --- Cargo.lock | 22 +++++----- Cargo.toml | 2 +- src/api/client/keys.rs | 95 ++++++++++++++++++++++++++-------------- src/service/users/mod.rs | 18 +++++--- 4 files changed, 86 insertions(+), 51 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8918a631..0753f81d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3654,7 +3654,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "assign", "js_int", @@ -3674,7 +3674,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "js_int", "ruma-common", @@ -3686,7 +3686,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "as_variant", "assign", @@ -3709,7 +3709,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "as_variant", "base64 0.22.1", @@ -3741,7 +3741,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "as_variant", "indexmap 2.8.0", @@ -3766,7 +3766,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "bytes", "headers", @@ -3788,7 +3788,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3797,7 +3797,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "js_int", "ruma-common", @@ -3807,7 +3807,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3822,7 +3822,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "js_int", "ruma-common", @@ -3834,7 +3834,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=0701341a2fd5a6ea74beada18d5974cc401a4fc1#0701341a2fd5a6ea74beada18d5974cc401a4fc1" +source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index bf7ec2bb..a44fc0f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,7 +350,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "0701341a2fd5a6ea74beada18d5974cc401a4fc1" +rev = "edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" features = [ "compat", "rand", diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index f50d7afa..f6224343 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -9,7 +9,8 @@ use ruma::{ client::{ error::ErrorKind, keys::{ - claim_keys, get_key_changes, get_keys, upload_keys, upload_signatures, + claim_keys, get_key_changes, get_keys, upload_keys, + upload_signatures::{self, v3::Failure}, upload_signing_keys, }, uiaa::{AuthFlow, AuthType, UiaaInfo}, @@ -308,53 +309,81 @@ async fn check_for_new_keys( /// # `POST /_matrix/client/r0/keys/signatures/upload` /// /// Uploads end-to-end key signatures from the sender user. +/// +/// TODO: clean this timo-code up more. tried to improve it a bit to stop +/// exploding the entire request on bad sigs, but needs way more work. pub(crate) async fn upload_signatures_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + use upload_signatures::v3::FailureErrorCode::*; + + if body.signed_keys.is_empty() { + debug!("Empty signed_keys sent in key signature upload"); + return Ok(upload_signatures::v3::Response::new()); + } + + let sender_user = body.sender_user(); + let mut failures: BTreeMap> = BTreeMap::new(); + let mut failure_reasons: BTreeMap = BTreeMap::new(); + let failure = Failure { + errcode: InvalidSignature, + error: String::new(), + }; for (user_id, keys) in &body.signed_keys { for (key_id, key) in keys { - let key = serde_json::to_value(key) - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid key JSON"))?; + let Ok(key) = serde_json::to_value(key) + .inspect_err(|e| debug_warn!(?key_id, "Invalid \"key\" JSON: {e}")) + else { + let mut failure = failure.clone(); + failure.error = String::from("Invalid \"key\" JSON"); + failure_reasons.insert(key_id.to_owned(), failure); + continue; + }; - for signature in key - .get("signatures") - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Missing signatures field."))? - .get(sender_user.to_string()) - .ok_or(Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid user in signatures field.", - ))? - .as_object() - .ok_or(Error::BadRequest(ErrorKind::InvalidParam, "Invalid signature."))? - .clone() - { - // Signature validation? - let signature = ( - signature.0, - signature - .1 - .as_str() - .ok_or(Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid signature value.", - ))? - .to_owned(), - ); + let Some(signatures) = key.get("signatures") else { + let mut failure = failure.clone(); + failure.error = String::from("Missing \"signatures\" field"); + failure_reasons.insert(key_id.to_owned(), failure); + continue; + }; - services + let Some(sender_user_val) = signatures.get(sender_user.to_string()) else { + let mut failure = failure.clone(); + failure.error = String::from("Invalid user in signatures field"); + failure_reasons.insert(key_id.to_owned(), failure); + continue; + }; + + let Some(sender_user_object) = sender_user_val.as_object() else { + let mut failure = failure.clone(); + failure.error = String::from("signatures field is not a JSON object"); + failure_reasons.insert(key_id.to_owned(), failure); + continue; + }; + + for (signature, val) in sender_user_object.clone() { + let signature = (signature, val.to_string()); + + if let Err(e) = services .users .sign_key(user_id, key_id, signature, sender_user) - .await?; + .await + .inspect_err(|e| debug_warn!("{e}")) + { + let mut failure = failure.clone(); + failure.error = format!("Error signing key: {e}"); + failure_reasons.insert(key_id.to_owned(), failure); + continue; + } } } + + failures.insert(user_id.to_owned(), failure_reasons.clone()); } - Ok(upload_signatures::v3::Response { - failures: BTreeMap::new(), // TODO: integrate - }) + Ok(upload_signatures::v3::Response { failures }) } /// # `POST /_matrix/client/r0/keys/changes` diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 87a8b93b..1eb289fc 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -593,7 +593,7 @@ impl Service { key_id: &str, signature: (String, String), sender_id: &UserId, - ) -> Result<()> { + ) -> Result { let key = (target_id, key_id); let mut cross_signing_key: serde_json::Value = self @@ -601,21 +601,27 @@ impl Service { .keyid_key .qry(&key) .await - .map_err(|_| err!(Request(InvalidParam("Tried to sign nonexistent key."))))? + .map_err(|_| err!(Request(InvalidParam("Tried to sign nonexistent key"))))? .deserialized() - .map_err(|e| err!(Database("key in keyid_key is invalid. {e:?}")))?; + .map_err(|e| err!(Database(debug_warn!("key in keyid_key is invalid: {e:?}"))))?; let signatures = cross_signing_key .get_mut("signatures") - .ok_or_else(|| err!(Database("key in keyid_key has no signatures field.")))? + .ok_or_else(|| { + err!(Database(debug_warn!("key in keyid_key has no signatures field"))) + })? .as_object_mut() - .ok_or_else(|| err!(Database("key in keyid_key has invalid signatures field.")))? + .ok_or_else(|| { + err!(Database(debug_warn!("key in keyid_key has invalid signatures field."))) + })? .entry(sender_id.to_string()) .or_insert_with(|| serde_json::Map::new().into()); signatures .as_object_mut() - .ok_or_else(|| err!(Database("signatures in keyid_key for a user is invalid.")))? + .ok_or_else(|| { + err!(Database(debug_warn!("signatures in keyid_key for a user is invalid."))) + })? .insert(signature.0, signature.1.into()); let key = (target_id, key_id); From b7109131e29804ac6b4e30aaaa40f213d092a63a Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Apr 2025 22:06:51 +0000 Subject: [PATCH 1639/2091] further simplify get_missing_events; various log calls Signed-off-by: Jason Volk --- src/api/server/get_missing_events.rs | 47 +++++++++---------- .../rooms/state_accessor/server_can.rs | 8 +--- src/service/rooms/state_accessor/user_can.rs | 14 ++---- 3 files changed, 29 insertions(+), 40 deletions(-) diff --git a/src/api/server/get_missing_events.rs b/src/api/server/get_missing_events.rs index d72918fa..04dc30ed 100644 --- a/src/api/server/get_missing_events.rs +++ b/src/api/server/get_missing_events.rs @@ -1,9 +1,5 @@ use axum::extract::State; -use conduwuit::{ - Result, debug, debug_info, debug_warn, - utils::{self}, - warn, -}; +use conduwuit::{Result, debug, debug_error, utils::to_canonical_object}; use ruma::api::federation::event::get_missing_events; use super::AccessCheck; @@ -43,19 +39,13 @@ pub(crate) async fn get_missing_events_route( let mut i: usize = 0; while i < queued_events.len() && events.len() < limit { let Ok(pdu) = services.rooms.timeline.get_pdu(&queued_events[i]).await else { - debug_info!(?body.origin, "Event {} does not exist locally, skipping", &queued_events[i]); - i = i.saturating_add(1); - continue; - }; - - if pdu.room_id != body.room_id { - warn!(?body.origin, - "Got an event for the wrong room in database. Found {:?} in {:?}, server requested events in {:?}. Skipping.", - pdu.event_id, pdu.room_id, body.room_id + debug!( + ?body.origin, + "Event {} does not exist locally, skipping", &queued_events[i] ); i = i.saturating_add(1); continue; - } + }; if body.earliest_events.contains(&queued_events[i]) { i = i.saturating_add(1); @@ -68,25 +58,32 @@ pub(crate) async fn get_missing_events_route( .server_can_see_event(body.origin(), &body.room_id, &queued_events[i]) .await { - debug!(?body.origin, "Server cannot see {:?} in {:?}, skipping", pdu.event_id, pdu.room_id); + debug!( + ?body.origin, + "Server cannot see {:?} in {:?}, skipping", pdu.event_id, pdu.room_id + ); i = i.saturating_add(1); continue; } - let Ok(pdu_json) = utils::to_canonical_object(&pdu) else { - debug_warn!(?body.origin, "Failed to convert PDU in database to canonical JSON: {pdu:?}"); + let Ok(event) = to_canonical_object(&pdu) else { + debug_error!( + ?body.origin, + "Failed to convert PDU in database to canonical JSON: {pdu:?}" + ); i = i.saturating_add(1); continue; }; - queued_events.extend(pdu.prev_events.iter().map(ToOwned::to_owned)); + let prev_events = pdu.prev_events.iter().map(ToOwned::to_owned); - events.push( - services - .sending - .convert_to_outgoing_federation_event(pdu_json) - .await, - ); + let event = services + .sending + .convert_to_outgoing_federation_event(event) + .await; + + queued_events.extend(prev_events); + events.push(event); } Ok(get_missing_events::v1::Response { events }) diff --git a/src/service/rooms/state_accessor/server_can.rs b/src/service/rooms/state_accessor/server_can.rs index c946fbfd..2befec22 100644 --- a/src/service/rooms/state_accessor/server_can.rs +++ b/src/service/rooms/state_accessor/server_can.rs @@ -1,4 +1,4 @@ -use conduwuit::{debug_info, implement, utils::stream::ReadyExt}; +use conduwuit::{implement, utils::stream::ReadyExt}; use futures::StreamExt; use ruma::{ EventId, RoomId, ServerName, @@ -36,7 +36,6 @@ pub async fn server_can_see_event( .ready_filter(|member| member.server_name() == origin); match history_visibility { - | HistoryVisibility::WorldReadable | HistoryVisibility::Shared => true, | HistoryVisibility::Invited => { // Allow if any member on requesting server was AT LEAST invited, else deny current_server_members @@ -49,9 +48,6 @@ pub async fn server_can_see_event( .any(|member| self.user_was_joined(shortstatehash, member)) .await }, - | _ => { - debug_info!(%room_id, "Unknown history visibility, defaulting to shared: {history_visibility:?}"); - true - }, + | HistoryVisibility::WorldReadable | HistoryVisibility::Shared | _ => true, } } diff --git a/src/service/rooms/state_accessor/user_can.rs b/src/service/rooms/state_accessor/user_can.rs index aa54407b..67e0b52b 100644 --- a/src/service/rooms/state_accessor/user_can.rs +++ b/src/service/rooms/state_accessor/user_can.rs @@ -1,4 +1,4 @@ -use conduwuit::{Err, Error, Result, debug_info, implement, pdu::PduBuilder}; +use conduwuit::{Err, Result, implement, pdu::PduBuilder}; use ruma::{ EventId, RoomId, UserId, events::{ @@ -76,8 +76,8 @@ pub async fn user_can_redact( || redacting_event .as_ref() .is_ok_and(|redacting_event| redacting_event.sender == sender)), - | _ => Err(Error::bad_database( - "No m.room.power_levels or m.room.create events in database for room", + | _ => Err!(Database( + "No m.room.power_levels or m.room.create events in database for room" )), } }, @@ -108,8 +108,6 @@ pub async fn user_can_see_event( }); match history_visibility { - | HistoryVisibility::WorldReadable => true, - | HistoryVisibility::Shared => currently_member, | HistoryVisibility::Invited => { // Allow if any member on requesting server was AT LEAST invited, else deny self.user_was_invited(shortstatehash, user_id).await @@ -118,10 +116,8 @@ pub async fn user_can_see_event( // Allow if any member on requested server was joined, else deny self.user_was_joined(shortstatehash, user_id).await }, - | _ => { - debug_info!(%room_id, "Unknown history visibility, defaulting to shared: {history_visibility:?}"); - currently_member - }, + | HistoryVisibility::WorldReadable => true, + | HistoryVisibility::Shared | _ => currently_member, } } From 6a073b4fa4c728b15f94de88ac37d136c97982bf Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 2 Apr 2025 06:28:34 +0000 Subject: [PATCH 1640/2091] remove additional unnecessary Arc Signed-off-by: Jason Volk --- .../fetch_and_handle_outliers.rs | 6 ++--- src/service/rooms/event_handler/fetch_prev.rs | 7 ++---- .../rooms/event_handler/handle_outlier_pdu.rs | 22 ++++++------------- .../rooms/event_handler/handle_prev_pdu.rs | 6 +---- .../rooms/event_handler/state_at_incoming.rs | 5 ++--- .../event_handler/upgrade_outlier_pdu.rs | 2 +- 6 files changed, 15 insertions(+), 33 deletions(-) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index 80e91eff..b0a7d827 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -1,6 +1,5 @@ use std::{ collections::{BTreeMap, HashSet, VecDeque, hash_map}, - sync::Arc, time::Instant, }; @@ -8,7 +7,6 @@ use conduwuit::{ PduEvent, debug, debug_error, debug_warn, implement, pdu, trace, utils::continue_exponential_backoff_secs, warn, }; -use futures::TryFutureExt; use ruma::{ CanonicalJsonValue, OwnedEventId, RoomId, ServerName, api::federation::event::get_event, }; @@ -31,7 +29,7 @@ pub(super) async fn fetch_and_handle_outliers<'a>( events: &'a [OwnedEventId], create_event: &'a PduEvent, room_id: &'a RoomId, -) -> Vec<(Arc, Option>)> { +) -> Vec<(PduEvent, Option>)> { let back_off = |id| match self .services .globals @@ -53,7 +51,7 @@ pub(super) async fn fetch_and_handle_outliers<'a>( // a. Look in the main timeline (pduid_pdu tree) // b. Look at outlier pdu tree // (get_pdu_json checks both) - if let Ok(local_pdu) = self.services.timeline.get_pdu(id).map_ok(Arc::new).await { + if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { trace!("Found {id} in db"); events_with_auth_events.push((id, Some(local_pdu), vec![])); continue; diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index e817430b..0f92d6e6 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,7 +1,4 @@ -use std::{ - collections::{BTreeMap, HashMap, HashSet, VecDeque}, - sync::Arc, -}; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; use conduwuit::{ PduEvent, Result, debug_warn, err, implement, @@ -31,7 +28,7 @@ pub(super) async fn fetch_prev( initial_set: Vec, ) -> Result<( Vec, - HashMap, BTreeMap)>, + HashMap)>, )> { let mut graph: HashMap = HashMap::with_capacity(initial_set.len()); let mut eventid_info = HashMap::new(); diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 99e90a50..5339249d 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -1,12 +1,9 @@ -use std::{ - collections::{BTreeMap, HashMap, hash_map}, - sync::Arc, -}; +use std::collections::{BTreeMap, HashMap, hash_map}; use conduwuit::{ Err, Error, PduEvent, Result, debug, debug_info, err, implement, state_res, trace, warn, }; -use futures::{TryFutureExt, future::ready}; +use futures::future::ready; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, RoomId, ServerName, api::client::error::ErrorKind, events::StateEventType, @@ -24,7 +21,7 @@ pub(super) async fn handle_outlier_pdu<'a>( room_id: &'a RoomId, mut value: CanonicalJsonObject, auth_events_known: bool, -) -> Result<(Arc, BTreeMap)> { +) -> Result<(PduEvent, BTreeMap)> { // 1. Remove unsigned field value.remove("unsigned"); @@ -95,7 +92,7 @@ pub(super) async fn handle_outlier_pdu<'a>( // Build map of auth events let mut auth_events = HashMap::with_capacity(incoming_pdu.auth_events.len()); for id in &incoming_pdu.auth_events { - let Ok(auth_event) = self.services.timeline.get_pdu(id).map_ok(Arc::new).await else { + let Ok(auth_event) = self.services.timeline.get_pdu(id).await else { warn!("Could not find auth event {id}"); continue; }; @@ -123,15 +120,10 @@ pub(super) async fn handle_outlier_pdu<'a>( // The original create event must be in the auth events if !matches!( - auth_events - .get(&(StateEventType::RoomCreate, String::new().into())) - .map(AsRef::as_ref), + auth_events.get(&(StateEventType::RoomCreate, String::new().into())), Some(_) | None ) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Incoming event refers to wrong create event.", - )); + return Err!(Request(InvalidParam("Incoming event refers to wrong create event."))); } let state_fetch = |ty: &StateEventType, sk: &str| { @@ -161,5 +153,5 @@ pub(super) async fn handle_outlier_pdu<'a>( trace!("Added pdu as outlier."); - Ok((Arc::new(incoming_pdu), val)) + Ok((incoming_pdu, val)) } diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index cf69a515..85e0a6b9 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -1,6 +1,5 @@ use std::{ collections::{BTreeMap, HashMap}, - sync::Arc, time::Instant, }; @@ -24,10 +23,7 @@ pub(super) async fn handle_prev_pdu<'a>( origin: &'a ServerName, event_id: &'a EventId, room_id: &'a RoomId, - eventid_info: &mut HashMap< - OwnedEventId, - (Arc, BTreeMap), - >, + eventid_info: &mut HashMap)>, create_event: &PduEvent, first_ts_in_room: UInt, prev_id: &EventId, diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 8326f9da..0402ff14 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -2,7 +2,6 @@ use std::{ borrow::Borrow, collections::{HashMap, HashSet}, iter::Iterator, - sync::Arc, }; use conduwuit::{ @@ -20,7 +19,7 @@ use crate::rooms::short::ShortStateHash; #[tracing::instrument(name = "state", level = "debug", skip_all)] pub(super) async fn state_at_incoming_degree_one( &self, - incoming_pdu: &Arc, + incoming_pdu: &PduEvent, ) -> Result>> { let prev_event = &incoming_pdu.prev_events[0]; let Ok(prev_event_sstatehash) = self @@ -67,7 +66,7 @@ pub(super) async fn state_at_incoming_degree_one( #[tracing::instrument(name = "state", level = "debug", skip_all)] pub(super) async fn state_at_incoming_resolved( &self, - incoming_pdu: &Arc, + incoming_pdu: &PduEvent, room_id: &RoomId, room_version_id: &RoomVersionId, ) -> Result>> { diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index c1a1c3eb..086dc6bd 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -18,7 +18,7 @@ use crate::rooms::{ #[implement(super::Service)] pub(super) async fn upgrade_outlier_to_timeline_pdu( &self, - incoming_pdu: Arc, + incoming_pdu: PduEvent, val: BTreeMap, create_event: &PduEvent, origin: &ServerName, From d036394ec79cf94aee484e6bea41421396dcd749 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 2 Apr 2025 09:53:42 +0000 Subject: [PATCH 1641/2091] refactor incoming prev events loop; mitigate large future Signed-off-by: Jason Volk --- .../event_handler/handle_incoming_pdu.rs | 102 ++++++++++-------- .../rooms/event_handler/handle_prev_pdu.rs | 65 +++++------ 2 files changed, 88 insertions(+), 79 deletions(-) diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index b437bf2e..77cae41d 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -3,9 +3,12 @@ use std::{ time::Instant, }; -use conduwuit::{Err, Result, debug, debug::INFO_SPAN_LEVEL, err, implement, warn}; +use conduwuit::{ + Err, Result, debug, debug::INFO_SPAN_LEVEL, defer, err, implement, utils::stream::IterStream, + warn, +}; use futures::{ - FutureExt, + FutureExt, TryFutureExt, TryStreamExt, future::{OptionFuture, try_join5}, }; use ruma::{CanonicalJsonValue, EventId, RoomId, ServerName, UserId, events::StateEventType}; @@ -86,7 +89,7 @@ pub async fn handle_incoming_pdu<'a>( .state_accessor .room_state_get(room_id, &StateEventType::RoomCreate, ""); - let (meta_exists, is_disabled, (), (), create_event) = try_join5( + let (meta_exists, is_disabled, (), (), ref create_event) = try_join5( meta_exists, is_disabled, origin_acl_check, @@ -104,7 +107,7 @@ pub async fn handle_incoming_pdu<'a>( } let (incoming_pdu, val) = self - .handle_outlier_pdu(origin, &create_event, event_id, room_id, value, false) + .handle_outlier_pdu(origin, create_event, event_id, room_id, value, false) .await?; // 8. if not timeline event: stop @@ -129,66 +132,71 @@ pub async fn handle_incoming_pdu<'a>( let (sorted_prev_events, mut eventid_info) = self .fetch_prev( origin, - &create_event, + create_event, room_id, first_ts_in_room, incoming_pdu.prev_events.clone(), ) .await?; - debug!(events = ?sorted_prev_events, "Got previous events"); - for prev_id in sorted_prev_events { - self.services.server.check_running()?; - if let Err(e) = self - .handle_prev_pdu( + debug!( + events = ?sorted_prev_events, + "Handling previous events" + ); + + sorted_prev_events + .iter() + .try_stream() + .map_ok(AsRef::as_ref) + .try_for_each(|prev_id| { + self.handle_prev_pdu( origin, event_id, room_id, - &mut eventid_info, - &create_event, + eventid_info.remove(prev_id), + create_event, first_ts_in_room, - &prev_id, + prev_id, ) - .await - { - use hash_map::Entry; - - let now = Instant::now(); - warn!("Prev event {prev_id} failed: {e}"); - - match self - .services - .globals - .bad_event_ratelimiter - .write() - .expect("locked") - .entry(prev_id) - { - | Entry::Vacant(e) => { - e.insert((now, 1)); - }, - | Entry::Occupied(mut e) => { - *e.get_mut() = (now, e.get().1.saturating_add(1)); - }, - } - } - } + .inspect_err(move |e| { + warn!("Prev {prev_id} failed: {e}"); + match self + .services + .globals + .bad_event_ratelimiter + .write() + .expect("locked") + .entry(prev_id.into()) + { + | hash_map::Entry::Vacant(e) => { + e.insert((Instant::now(), 1)); + }, + | hash_map::Entry::Occupied(mut e) => { + let tries = e.get().1.saturating_add(1); + *e.get_mut() = (Instant::now(), tries); + }, + } + }) + .map(|_| self.services.server.check_running()) + }) + .boxed() + .await?; // Done with prev events, now handling the incoming event let start_time = Instant::now(); self.federation_handletime .write() .expect("locked") - .insert(room_id.to_owned(), (event_id.to_owned(), start_time)); + .insert(room_id.into(), (event_id.to_owned(), start_time)); - let r = self - .upgrade_outlier_to_timeline_pdu(incoming_pdu, val, &create_event, origin, room_id) - .await; + defer! {{ + self.federation_handletime + .write() + .expect("locked") + .remove(room_id); + }}; - self.federation_handletime - .write() - .expect("locked") - .remove(&room_id.to_owned()); - - r + self.upgrade_outlier_to_timeline_pdu(incoming_pdu, val, create_event, origin, room_id) + .boxed() + .await } diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index 85e0a6b9..d612b2bf 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -1,13 +1,10 @@ -use std::{ - collections::{BTreeMap, HashMap}, - time::Instant, -}; +use std::{collections::BTreeMap, time::Instant}; use conduwuit::{ - Err, PduEvent, Result, debug, debug::INFO_SPAN_LEVEL, implement, + Err, PduEvent, Result, debug, debug::INFO_SPAN_LEVEL, defer, implement, utils::continue_exponential_backoff_secs, }; -use ruma::{CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, UInt}; +use ruma::{CanonicalJsonValue, EventId, RoomId, ServerName, UInt}; #[implement(super::Service)] #[allow(clippy::type_complexity)] @@ -23,10 +20,10 @@ pub(super) async fn handle_prev_pdu<'a>( origin: &'a ServerName, event_id: &'a EventId, room_id: &'a RoomId, - eventid_info: &mut HashMap)>, - create_event: &PduEvent, + eventid_info: Option<(PduEvent, BTreeMap)>, + create_event: &'a PduEvent, first_ts_in_room: UInt, - prev_id: &EventId, + prev_id: &'a EventId, ) -> Result { // Check for disabled again because it might have changed if self.services.metadata.is_disabled(room_id).await { @@ -57,31 +54,35 @@ pub(super) async fn handle_prev_pdu<'a>( } } - if let Some((pdu, json)) = eventid_info.remove(prev_id) { - // Skip old events - if pdu.origin_server_ts < first_ts_in_room { - return Ok(()); - } + let Some((pdu, json)) = eventid_info else { + return Ok(()); + }; - let start_time = Instant::now(); - self.federation_handletime - .write() - .expect("locked") - .insert(room_id.to_owned(), ((*prev_id).to_owned(), start_time)); - - self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id) - .await?; - - self.federation_handletime - .write() - .expect("locked") - .remove(&room_id.to_owned()); - - debug!( - elapsed = ?start_time.elapsed(), - "Handled prev_event", - ); + // Skip old events + if pdu.origin_server_ts < first_ts_in_room { + return Ok(()); } + let start_time = Instant::now(); + self.federation_handletime + .write() + .expect("locked") + .insert(room_id.into(), ((*prev_id).to_owned(), start_time)); + + defer! {{ + self.federation_handletime + .write() + .expect("locked") + .remove(room_id); + }}; + + self.upgrade_outlier_to_timeline_pdu(pdu, json, create_event, origin, room_id) + .await?; + + debug!( + elapsed = ?start_time.elapsed(), + "Handled prev_event", + ); + Ok(()) } From 00f7745ec4ebcea5f892376c5de5db1299f71696 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Apr 2025 02:56:54 +0000 Subject: [PATCH 1642/2091] remove the db pool queue full warning Signed-off-by: Jason Volk --- src/database/pool.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/database/pool.rs b/src/database/pool.rs index 47e61c30..0fa742d1 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -12,7 +12,7 @@ use std::{ use async_channel::{QueueStrategy, Receiver, RecvError, Sender}; use conduwuit::{ - Error, Result, Server, debug, debug_warn, err, error, implement, + Error, Result, Server, debug, err, error, implement, result::DebugInspect, smallvec::SmallVec, trace, @@ -245,13 +245,6 @@ async fn execute(&self, queue: &Sender, cmd: Cmd) -> Result { self.queued_max.fetch_max(queue.len(), Ordering::Relaxed); } - if queue.is_full() { - debug_warn!( - capacity = ?queue.capacity(), - "pool queue is full" - ); - } - queue .send(cmd) .await From 4e5b87d0cd16f3d015f4b61285b369d027bb909d Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Fri, 4 Apr 2025 11:34:31 -0400 Subject: [PATCH 1643/2091] add missing condition for signatures upload failures Signed-off-by: June Clementine Strawberry --- src/api/client/keys.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index f6224343..2fdfc0bc 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -380,7 +380,9 @@ pub(crate) async fn upload_signatures_route( } } - failures.insert(user_id.to_owned(), failure_reasons.clone()); + if !failure_reasons.is_empty() { + failures.insert(user_id.to_owned(), failure_reasons.clone()); + } } Ok(upload_signatures::v3::Response { failures }) From 532dfd004dbc020baa74a4d4413d9ad8139f851e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Apr 2025 03:30:13 +0000 Subject: [PATCH 1644/2091] move core::pdu and core::state_res into core::matrix:: Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 4 +- src/admin/user/commands.rs | 3 +- src/api/client/account.rs | 6 ++- src/api/client/account_data.rs | 5 +- src/api/client/alias.rs | 2 +- src/api/client/backup.rs | 4 +- src/api/client/context.rs | 6 ++- src/api/client/device.rs | 4 +- src/api/client/directory.rs | 2 +- src/api/client/filter.rs | 4 +- src/api/client/keys.rs | 6 +-- src/api/client/membership.rs | 27 ++++++----- src/api/client/message.rs | 22 +++++---- src/api/client/openid.rs | 4 +- src/api/client/profile.rs | 4 +- src/api/client/push.rs | 6 +-- src/api/client/read_marker.rs | 4 +- src/api/client/redact.rs | 3 +- src/api/client/relations.rs | 5 +- src/api/client/report.rs | 8 ++-- src/api/client/room/create.rs | 6 ++- src/api/client/room/upgrade.rs | 5 +- src/api/client/search.rs | 5 +- src/api/client/send.rs | 4 +- src/api/client/session.rs | 9 ++-- src/api/client/space.rs | 10 ++-- src/api/client/state.rs | 8 +++- src/api/client/sync/mod.rs | 5 +- src/api/client/sync/v3.rs | 9 ++-- src/api/client/sync/v5.rs | 9 +++- src/api/client/tag.rs | 3 +- src/api/client/thirdparty.rs | 3 +- src/api/client/threads.rs | 7 ++- src/api/client/to_device.rs | 2 +- src/api/client/typing.rs | 4 +- src/api/client/unversioned.rs | 3 +- src/api/client/user_directory.rs | 4 +- src/api/client/voip.rs | 4 +- src/api/client/well_known.rs | 3 +- src/api/mod.rs | 2 - src/api/server/hierarchy.rs | 4 +- src/api/server/invite.rs | 5 +- src/api/server/make_join.rs | 10 ++-- src/api/server/make_knock.rs | 5 +- src/api/server/make_leave.rs | 4 +- src/api/server/openid.rs | 3 +- src/api/server/publicrooms.rs | 3 +- src/api/server/send.rs | 16 +++---- src/api/server/send_join.rs | 2 +- src/api/server/send_knock.rs | 6 ++- src/api/server/send_leave.rs | 8 ++-- src/api/server/version.rs | 3 +- src/api/server/well_known.rs | 3 +- .../state_event.rs => matrix/event.rs} | 0 src/core/matrix/mod.rs | 9 ++++ src/core/{pdu/mod.rs => matrix/pdu.rs} | 47 +++++++++++++++---- src/core/{ => matrix}/pdu/builder.rs | 0 src/core/{ => matrix}/pdu/content.rs | 0 src/core/{ => matrix}/pdu/count.rs | 0 src/core/{ => matrix}/pdu/event_id.rs | 0 src/core/{ => matrix}/pdu/filter.rs | 0 src/core/{ => matrix}/pdu/id.rs | 0 src/core/{ => matrix}/pdu/raw_id.rs | 0 src/core/{ => matrix}/pdu/redact.rs | 0 src/core/{ => matrix}/pdu/relation.rs | 0 src/core/{ => matrix}/pdu/state_key.rs | 0 src/core/{ => matrix}/pdu/strip.rs | 0 src/core/{ => matrix}/pdu/tests.rs | 0 src/core/{ => matrix}/pdu/unsigned.rs | 0 src/core/{ => matrix}/state_res/LICENSE | 0 src/core/{ => matrix}/state_res/benches.rs | 0 src/core/{ => matrix}/state_res/error.rs | 0 src/core/{ => matrix}/state_res/event_auth.rs | 0 src/core/{ => matrix}/state_res/mod.rs | 8 ++-- src/core/{ => matrix}/state_res/outcomes.txt | 0 .../{ => matrix}/state_res/power_levels.rs | 2 +- .../{ => matrix}/state_res/room_version.rs | 0 src/core/{ => matrix}/state_res/test_utils.rs | 5 +- src/core/mod.rs | 6 +-- src/core/pdu/event.rs | 35 -------------- src/service/admin/grant.rs | 4 +- src/service/mod.rs | 1 - .../rooms/event_handler/state_at_incoming.rs | 4 +- .../event_handler/upgrade_outlier_pdu.rs | 3 +- src/service/rooms/outlier/mod.rs | 6 +-- src/service/rooms/read_receipt/mod.rs | 6 ++- src/service/rooms/short/mod.rs | 4 +- .../rooms/state_accessor/room_state.rs | 5 +- src/service/rooms/state_accessor/state.rs | 6 ++- src/service/rooms/threads/mod.rs | 5 +- src/service/rooms/timeline/mod.rs | 9 ++-- 91 files changed, 266 insertions(+), 205 deletions(-) rename src/core/{state_res/state_event.rs => matrix/event.rs} (100%) create mode 100644 src/core/matrix/mod.rs rename src/core/{pdu/mod.rs => matrix/pdu.rs} (72%) rename src/core/{ => matrix}/pdu/builder.rs (100%) rename src/core/{ => matrix}/pdu/content.rs (100%) rename src/core/{ => matrix}/pdu/count.rs (100%) rename src/core/{ => matrix}/pdu/event_id.rs (100%) rename src/core/{ => matrix}/pdu/filter.rs (100%) rename src/core/{ => matrix}/pdu/id.rs (100%) rename src/core/{ => matrix}/pdu/raw_id.rs (100%) rename src/core/{ => matrix}/pdu/redact.rs (100%) rename src/core/{ => matrix}/pdu/relation.rs (100%) rename src/core/{ => matrix}/pdu/state_key.rs (100%) rename src/core/{ => matrix}/pdu/strip.rs (100%) rename src/core/{ => matrix}/pdu/tests.rs (100%) rename src/core/{ => matrix}/pdu/unsigned.rs (100%) rename src/core/{ => matrix}/state_res/LICENSE (100%) rename src/core/{ => matrix}/state_res/benches.rs (100%) rename src/core/{ => matrix}/state_res/error.rs (100%) rename src/core/{ => matrix}/state_res/event_auth.rs (100%) rename src/core/{ => matrix}/state_res/mod.rs (99%) rename src/core/{ => matrix}/state_res/outcomes.txt (100%) rename src/core/{ => matrix}/state_res/power_levels.rs (99%) rename src/core/{ => matrix}/state_res/room_version.rs (100%) rename src/core/{ => matrix}/state_res/test_utils.rs (99%) delete mode 100644 src/core/pdu/event.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index c6f6a170..87ca03a0 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,7 +6,9 @@ use std::{ }; use conduwuit::{ - Error, PduEvent, PduId, RawPduId, Result, debug_error, err, info, trace, utils, + Error, Result, debug_error, err, info, + matrix::pdu::{PduEvent, PduId, RawPduId}, + trace, utils, utils::{ stream::{IterStream, ReadyExt}, string::EMPTY, diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 35067304..45e550be 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -2,7 +2,8 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room}; use conduwuit::{ - PduBuilder, Result, debug, debug_warn, error, info, is_equal_to, + Result, debug, debug_warn, error, info, is_equal_to, + matrix::pdu::PduBuilder, utils::{self, ReadyExt}, warn, }; diff --git a/src/api/client/account.rs b/src/api/client/account.rs index e5894d47..32f2530c 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -3,10 +3,13 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Err, Error, PduBuilder, Result, debug_info, err, error, info, is_equal_to, utils, + Err, Error, Result, debug_info, err, error, info, is_equal_to, + matrix::pdu::PduBuilder, + utils, utils::{ReadyExt, stream::BroadbandExt}, warn, }; +use conduwuit_service::Services; use futures::{FutureExt, StreamExt}; use register::RegistrationKind; use ruma::{ @@ -30,7 +33,6 @@ use ruma::{ }, push, }; -use service::Services; use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH, join_room_by_id_helper}; use crate::Ruma; diff --git a/src/api/client/account_data.rs b/src/api/client/account_data.rs index 60c18b37..e44ce4e7 100644 --- a/src/api/client/account_data.rs +++ b/src/api/client/account_data.rs @@ -1,5 +1,6 @@ use axum::extract::State; -use conduwuit::{Err, err}; +use conduwuit::{Err, Result, err}; +use conduwuit_service::Services; use ruma::{ RoomId, UserId, api::client::config::{ @@ -15,7 +16,7 @@ use ruma::{ use serde::Deserialize; use serde_json::{json, value::RawValue as RawJsonValue}; -use crate::{Result, Ruma, service::Services}; +use crate::Ruma; /// # `PUT /_matrix/client/r0/user/{userId}/account_data/{type}` /// diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 319e5141..9f1b05f8 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -1,12 +1,12 @@ use axum::extract::State; use conduwuit::{Err, Result, debug}; +use conduwuit_service::Services; use futures::StreamExt; use rand::seq::SliceRandom; use ruma::{ OwnedServerName, RoomAliasId, RoomId, api::client::alias::{create_alias, delete_alias, get_alias}, }; -use service::Services; use crate::Ruma; diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index 83955fea..2ad37cf3 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -1,7 +1,7 @@ use std::cmp::Ordering; use axum::extract::State; -use conduwuit::{Err, err}; +use conduwuit::{Err, Result, err}; use ruma::{ UInt, api::client::backup::{ @@ -13,7 +13,7 @@ use ruma::{ }, }; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `POST /_matrix/client/r0/room_keys/version` /// diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 1dda7b53..dbc2a22f 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,18 +1,20 @@ use axum::extract::State; use conduwuit::{ - Err, PduEvent, Result, at, debug_warn, err, ref_at, + Err, Result, at, debug_warn, err, + matrix::pdu::PduEvent, + ref_at, utils::{ IterStream, future::TryExtExt, stream::{BroadbandExt, ReadyExt, TryIgnore, WidebandExt}, }, }; +use conduwuit_service::rooms::{lazy_loading, lazy_loading::Options, short::ShortStateKey}; use futures::{ FutureExt, StreamExt, TryFutureExt, TryStreamExt, future::{OptionFuture, join, join3, try_join3}, }; use ruma::{OwnedEventId, UserId, api::client::context::get_context, events::StateEventType}; -use service::rooms::{lazy_loading, lazy_loading::Options, short::ShortStateKey}; use crate::{ Ruma, diff --git a/src/api/client/device.rs b/src/api/client/device.rs index 7603c866..5519a1a5 100644 --- a/src/api/client/device.rs +++ b/src/api/client/device.rs @@ -1,6 +1,6 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{Err, debug, err}; +use conduwuit::{Err, Error, Result, debug, err, utils}; use futures::StreamExt; use ruma::{ MilliSecondsSinceUnixEpoch, OwnedDeviceId, @@ -12,7 +12,7 @@ use ruma::{ }; use super::SESSION_ID_LENGTH; -use crate::{Error, Result, Ruma, client::DEVICE_ID_LENGTH, utils}; +use crate::{Ruma, client::DEVICE_ID_LENGTH}; /// # `GET /_matrix/client/r0/devices` /// diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index f2f668c8..9ca35537 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -9,6 +9,7 @@ use conduwuit::{ stream::{ReadyExt, WidebandExt}, }, }; +use conduwuit_service::Services; use futures::{ FutureExt, StreamExt, TryFutureExt, future::{join, join4, join5}, @@ -35,7 +36,6 @@ use ruma::{ }, uint, }; -use service::Services; use crate::Ruma; diff --git a/src/api/client/filter.rs b/src/api/client/filter.rs index 84086452..97044ffc 100644 --- a/src/api/client/filter.rs +++ b/src/api/client/filter.rs @@ -1,8 +1,8 @@ use axum::extract::State; -use conduwuit::err; +use conduwuit::{Result, err}; use ruma::api::client::filter::{create_filter, get_filter}; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/client/r0/user/{userId}/filter/{filterId}` /// diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 2fdfc0bc..6865c2a4 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use axum::extract::State; use conduwuit::{Err, Error, Result, debug, debug_warn, err, info, result::NotFound, utils}; +use conduwuit_service::{Services, users::parse_master_key}; use futures::{StreamExt, stream::FuturesUnordered}; use ruma::{ OneTimeKeyAlgorithm, OwnedDeviceId, OwnedUserId, UserId, @@ -23,10 +24,7 @@ use ruma::{ use serde_json::json; use super::SESSION_ID_LENGTH; -use crate::{ - Ruma, - service::{Services, users::parse_master_key}, -}; +use crate::Ruma; /// # `POST /_matrix/client/r0/keys/upload` /// diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index ef40e972..d0345c8e 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -9,13 +9,25 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Err, PduEvent, Result, StateKey, at, debug, debug_info, debug_warn, err, error, info, - pdu::{PduBuilder, gen_event_id_canonical_json}, + Err, Result, at, debug, debug_info, debug_warn, err, error, info, + matrix::{ + StateKey, + pdu::{PduBuilder, PduEvent, gen_event_id, gen_event_id_canonical_json}, + state_res, + }, result::{FlatOk, NotFound}, - state_res, trace, + trace, utils::{self, IterStream, ReadyExt, shuffle}, warn, }; +use conduwuit_service::{ + Services, + appservice::RegistrationInfo, + rooms::{ + state::RoomMutexGuard, + state_compressor::{CompressedState, HashSetCompressStateEvent}, + }, +}; use futures::{FutureExt, StreamExt, TryFutureExt, future::join4, join}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, @@ -44,15 +56,6 @@ use ruma::{ }, }, }; -use service::{ - Services, - appservice::RegistrationInfo, - pdu::gen_event_id, - rooms::{ - state::RoomMutexGuard, - state_compressor::{CompressedState, HashSetCompressStateEvent}, - }, -}; use crate::{Ruma, client::full_user_deactivate}; diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 03c7335a..3e784a4a 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,12 +1,24 @@ use axum::extract::State; use conduwuit::{ - Err, Event, PduCount, PduEvent, Result, at, + Err, Result, at, + matrix::{ + Event, + pdu::{PduCount, PduEvent}, + }, utils::{ IterStream, ReadyExt, result::{FlatOk, LogErr}, stream::{BroadbandExt, TryIgnore, WidebandExt}, }, }; +use conduwuit_service::{ + Services, + rooms::{ + lazy_loading, + lazy_loading::{Options, Witness}, + timeline::PdusIterItem, + }, +}; use futures::{FutureExt, StreamExt, TryFutureExt, future::OptionFuture, pin_mut}; use ruma::{ RoomId, UserId, @@ -17,14 +29,6 @@ use ruma::{ events::{AnyStateEvent, StateEventType, TimelineEventType, TimelineEventType::*}, serde::Raw, }; -use service::{ - Services, - rooms::{ - lazy_loading, - lazy_loading::{Options, Witness}, - timeline::PdusIterItem, - }, -}; use crate::Ruma; diff --git a/src/api/client/openid.rs b/src/api/client/openid.rs index 671d0c6d..8d2de68d 100644 --- a/src/api/client/openid.rs +++ b/src/api/client/openid.rs @@ -1,14 +1,14 @@ use std::time::Duration; use axum::extract::State; -use conduwuit::utils; +use conduwuit::{Error, Result, utils}; use ruma::{ api::client::{account, error::ErrorKind}, authentication::TokenType, }; use super::TOKEN_LENGTH; -use crate::{Error, Result, Ruma}; +use crate::Ruma; /// # `POST /_matrix/client/v3/user/{userId}/openid/request_token` /// diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 5abe5b23..3699b590 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -3,10 +3,11 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ Err, Error, Result, - pdu::PduBuilder, + matrix::pdu::PduBuilder, utils::{IterStream, stream::TryIgnore}, warn, }; +use conduwuit_service::Services; use futures::{StreamExt, TryStreamExt, future::join3}; use ruma::{ OwnedMxcUri, OwnedRoomId, UserId, @@ -22,7 +23,6 @@ use ruma::{ events::room::member::{MembershipState, RoomMemberEventContent}, presence::PresenceState, }; -use service::Services; use crate::Ruma; diff --git a/src/api/client/push.rs b/src/api/client/push.rs index cc1d3be2..81020ffa 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -1,5 +1,6 @@ use axum::extract::State; -use conduwuit::{Err, err}; +use conduwuit::{Err, Error, Result, err}; +use conduwuit_service::Services; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, api::client::{ @@ -19,9 +20,8 @@ use ruma::{ RemovePushRuleError, Ruleset, }, }; -use service::Services; -use crate::{Error, Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/client/r0/pushrules/` /// diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index b334e356..fbfc8fea 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduwuit::{Err, PduCount, err}; +use conduwuit::{Err, PduCount, Result, err}; use ruma::{ MilliSecondsSinceUnixEpoch, api::client::{read_marker::set_read_marker, receipt::create_receipt}, @@ -11,7 +11,7 @@ use ruma::{ }, }; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `POST /_matrix/client/r0/rooms/{roomId}/read_markers` /// diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index 7b512d06..8dbe47a6 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -1,9 +1,10 @@ use axum::extract::State; +use conduwuit::{Result, matrix::pdu::PduBuilder}; use ruma::{ api::client::redact::redact_event, events::room::redaction::RoomRedactionEventContent, }; -use crate::{Result, Ruma, service::pdu::PduBuilder}; +use crate::Ruma; /// # `PUT /_matrix/client/r0/rooms/{roomId}/redact/{eventId}/{txnId}` /// diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index 7ed40f14..b8c2dd4d 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,8 +1,10 @@ use axum::extract::State; use conduwuit::{ - PduCount, Result, at, + Result, at, + matrix::pdu::PduCount, utils::{IterStream, ReadyExt, result::FlatOk, stream::WidebandExt}, }; +use conduwuit_service::{Services, rooms::timeline::PdusIterItem}; use futures::StreamExt; use ruma::{ EventId, RoomId, UInt, UserId, @@ -15,7 +17,6 @@ use ruma::{ }, events::{TimelineEventType, relation::RelationType}, }; -use service::{Services, rooms::timeline::PdusIterItem}; use crate::Ruma; diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 7922caca..4ee8ebe5 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -2,7 +2,8 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{Err, info, utils::ReadyExt}; +use conduwuit::{Err, Error, Result, debug_info, info, matrix::pdu::PduEvent, utils::ReadyExt}; +use conduwuit_service::Services; use rand::Rng; use ruma::{ EventId, RoomId, UserId, @@ -15,10 +16,7 @@ use ruma::{ }; use tokio::time::sleep; -use crate::{ - Error, Result, Ruma, debug_info, - service::{Services, pdu::PduEvent}, -}; +use crate::Ruma; /// # `POST /_matrix/client/v3/rooms/{roomId}/report` /// diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index bdc5d5a5..4ce53f15 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -2,8 +2,11 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ - Err, Error, Result, StateKey, debug_info, debug_warn, err, error, info, pdu::PduBuilder, warn, + Err, Error, Result, debug_info, debug_warn, err, error, info, + matrix::{StateKey, pdu::PduBuilder}, + warn, }; +use conduwuit_service::{Services, appservice::RegistrationInfo}; use futures::FutureExt; use ruma::{ CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId, @@ -29,7 +32,6 @@ use ruma::{ serde::{JsonObject, Raw}, }; use serde_json::{json, value::to_raw_value}; -use service::{Services, appservice::RegistrationInfo}; use crate::{Ruma, client::invite_helper}; diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index 3cfb3c28..9ec0b3bb 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -1,7 +1,10 @@ use std::cmp::max; use axum::extract::State; -use conduwuit::{Error, Result, StateKey, err, info, pdu::PduBuilder}; +use conduwuit::{ + Error, Result, err, info, + matrix::{StateKey, pdu::PduBuilder}, +}; use futures::StreamExt; use ruma::{ CanonicalJsonObject, RoomId, RoomVersionId, diff --git a/src/api/client/search.rs b/src/api/client/search.rs index d66df881..d4dcde57 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -2,10 +2,12 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ - Err, PduEvent, Result, at, is_true, + Err, Result, at, is_true, + matrix::pdu::PduEvent, result::FlatOk, utils::{IterStream, stream::ReadyExt}, }; +use conduwuit_service::{Services, rooms::search::RoomQuery}; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, future::OptionFuture}; use ruma::{ OwnedRoomId, RoomId, UInt, UserId, @@ -17,7 +19,6 @@ use ruma::{ serde::Raw, }; use search_events::v3::{Request, Response}; -use service::{Services, rooms::search::RoomQuery}; use crate::Ruma; diff --git a/src/api/client/send.rs b/src/api/client/send.rs index 1af74f57..f753fa65 100644 --- a/src/api/client/send.rs +++ b/src/api/client/send.rs @@ -1,11 +1,11 @@ use std::collections::BTreeMap; use axum::extract::State; -use conduwuit::{Err, err}; +use conduwuit::{Err, Result, err, matrix::pdu::PduBuilder, utils}; use ruma::{api::client::message::send_message_event, events::MessageLikeEventType}; use serde_json::from_str; -use crate::{Result, Ruma, service::pdu::PduBuilder, utils}; +use crate::Ruma; /// # `PUT /_matrix/client/v3/rooms/{roomId}/send/{eventType}/{txnId}` /// diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 3de625e4..2499a43d 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -2,7 +2,11 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{Err, debug, err, info, utils::ReadyExt}; +use conduwuit::{ + Err, Error, Result, debug, err, info, utils, + utils::{ReadyExt, hash}, +}; +use conduwuit_service::uiaa::SESSION_ID_LENGTH; use futures::StreamExt; use ruma::{ UserId, @@ -22,10 +26,9 @@ use ruma::{ uiaa, }, }; -use service::uiaa::SESSION_ID_LENGTH; use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH}; -use crate::{Error, Result, Ruma, utils, utils::hash}; +use crate::Ruma; /// # `GET /_matrix/client/v3/login` /// diff --git a/src/api/client/space.rs b/src/api/client/space.rs index 567ac62f..4eee9d76 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -8,16 +8,16 @@ use conduwuit::{ Err, Result, utils::{future::TryExtExt, stream::IterStream}, }; -use futures::{StreamExt, TryFutureExt, future::OptionFuture}; -use ruma::{ - OwnedRoomId, OwnedServerName, RoomId, UInt, UserId, api::client::space::get_hierarchy, -}; -use service::{ +use conduwuit_service::{ Services, rooms::spaces::{ PaginationToken, SummaryAccessibility, get_parent_children_via, summary_to_chunk, }, }; +use futures::{StreamExt, TryFutureExt, future::OptionFuture}; +use ruma::{ + OwnedRoomId, OwnedServerName, RoomId, UInt, UserId, api::client::space::get_hierarchy, +}; use crate::Ruma; diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 23583356..5c5c71f2 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -1,5 +1,10 @@ use axum::extract::State; -use conduwuit::{Err, PduEvent, Result, err, pdu::PduBuilder, utils::BoolExt}; +use conduwuit::{ + Err, Result, err, + matrix::pdu::{PduBuilder, PduEvent}, + utils::BoolExt, +}; +use conduwuit_service::Services; use futures::TryStreamExt; use ruma::{ OwnedEventId, RoomId, UserId, @@ -16,7 +21,6 @@ use ruma::{ }, serde::Raw, }; -use service::Services; use crate::{Ruma, RumaResponse}; diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 3eab76cc..14459acf 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -3,12 +3,14 @@ mod v4; mod v5; use conduwuit::{ - PduCount, + Error, PduCount, Result, + matrix::pdu::PduEvent, utils::{ IterStream, stream::{BroadbandExt, ReadyExt, TryIgnore}, }, }; +use conduwuit_service::Services; use futures::{StreamExt, pin_mut}; use ruma::{ RoomId, UserId, @@ -21,7 +23,6 @@ use ruma::{ pub(crate) use self::{ v3::sync_events_route, v4::sync_events_v4_route, v5::sync_events_v5_route, }; -use crate::{Error, PduEvent, Result, service::Services}; pub(crate) const DEFAULT_BUMP_TYPES: &[TimelineEventType; 6] = &[CallInvite, PollStart, Beacon, RoomEncrypted, RoomMessage, Sticker]; diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 83ffa55a..12731ff6 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -6,9 +6,12 @@ use std::{ use axum::extract::State; use conduwuit::{ - PduCount, PduEvent, Result, at, err, error, extract_variant, is_equal_to, pair_of, - pdu::{Event, EventHash}, - ref_at, + Result, at, err, error, extract_variant, is_equal_to, + matrix::{ + Event, + pdu::{EventHash, PduCount, PduEvent}, + }, + pair_of, ref_at, result::FlatOk, utils::{ self, BoolExt, IterStream, ReadyExt, TryFutureExtExt, diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index c4e71d88..684752ec 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -6,13 +6,19 @@ use std::{ use axum::extract::State; use conduwuit::{ - Error, PduEvent, Result, TypeStateKey, debug, error, extract_variant, trace, + Error, Result, debug, error, extract_variant, + matrix::{ + TypeStateKey, + pdu::{PduCount, PduEvent}, + }, + trace, utils::{ BoolExt, IterStream, ReadyExt, TryFutureExtExt, math::{ruma_from_usize, usize_from_ruma}, }, warn, }; +use conduwuit_service::rooms::read_receipt::pack_receipts; use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{ DeviceId, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, @@ -27,7 +33,6 @@ use ruma::{ serde::Raw, uint, }; -use service::{PduCount, rooms::read_receipt::pack_receipts}; use super::{filter_rooms, share_encrypted_room}; use crate::{ diff --git a/src/api/client/tag.rs b/src/api/client/tag.rs index 3b3b40d4..caafe10d 100644 --- a/src/api/client/tag.rs +++ b/src/api/client/tag.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use axum::extract::State; +use conduwuit::Result; use ruma::{ api::client::tag::{create_tag, delete_tag, get_tags}, events::{ @@ -9,7 +10,7 @@ use ruma::{ }, }; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `PUT /_matrix/client/r0/user/{userId}/rooms/{roomId}/tags/{tag}` /// diff --git a/src/api/client/thirdparty.rs b/src/api/client/thirdparty.rs index 790b27d3..0713a882 100644 --- a/src/api/client/thirdparty.rs +++ b/src/api/client/thirdparty.rs @@ -1,8 +1,9 @@ use std::collections::BTreeMap; +use conduwuit::Result; use ruma::api::client::thirdparty::get_protocols; -use crate::{Result, Ruma, RumaResponse}; +use crate::{Ruma, RumaResponse}; /// # `GET /_matrix/client/r0/thirdparty/protocols` /// diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index 00bfe553..5b838bef 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -1,9 +1,12 @@ use axum::extract::State; -use conduwuit::{PduCount, PduEvent, at}; +use conduwuit::{ + Result, at, + matrix::pdu::{PduCount, PduEvent}, +}; use futures::StreamExt; use ruma::{api::client::threads::get_threads, uint}; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/client/r0/rooms/{roomId}/threads` pub(crate) async fn get_threads_route( diff --git a/src/api/client/to_device.rs b/src/api/client/to_device.rs index 1b942fba..8ad9dc99 100644 --- a/src/api/client/to_device.rs +++ b/src/api/client/to_device.rs @@ -2,6 +2,7 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{Error, Result}; +use conduwuit_service::sending::EduBuf; use futures::StreamExt; use ruma::{ api::{ @@ -10,7 +11,6 @@ use ruma::{ }, to_device::DeviceIdOrAllDevices, }; -use service::sending::EduBuf; use crate::Ruma; diff --git a/src/api/client/typing.rs b/src/api/client/typing.rs index b02cc473..1d8d02fd 100644 --- a/src/api/client/typing.rs +++ b/src/api/client/typing.rs @@ -1,8 +1,8 @@ use axum::extract::State; -use conduwuit::{Err, utils::math::Tried}; +use conduwuit::{Err, Result, utils, utils::math::Tried}; use ruma::api::client::typing::create_typing_event; -use crate::{Result, Ruma, utils}; +use crate::Ruma; /// # `PUT /_matrix/client/r0/rooms/{roomId}/typing/{userId}` /// diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 4e2b7d9d..232d5b28 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -1,10 +1,11 @@ use std::collections::BTreeMap; use axum::{Json, extract::State, response::IntoResponse}; +use conduwuit::Result; use futures::StreamExt; use ruma::api::client::discovery::get_supported_versions; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/client/versions` /// diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index c5d79a56..8f564eed 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduwuit::utils::TryFutureExtExt; +use conduwuit::{Result, utils::TryFutureExtExt}; use futures::{StreamExt, pin_mut}; use ruma::{ api::client::user_directory::search_users, @@ -9,7 +9,7 @@ use ruma::{ }, }; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `POST /_matrix/client/r0/user_directory/search` /// diff --git a/src/api/client/voip.rs b/src/api/client/voip.rs index 37e67984..91991d24 100644 --- a/src/api/client/voip.rs +++ b/src/api/client/voip.rs @@ -2,12 +2,12 @@ use std::time::{Duration, SystemTime}; use axum::extract::State; use base64::{Engine as _, engine::general_purpose}; -use conduwuit::{Err, utils}; +use conduwuit::{Err, Result, utils}; use hmac::{Hmac, Mac}; use ruma::{SecondsSinceUnixEpoch, UserId, api::client::voip::get_turn_server_info}; use sha1::Sha1; -use crate::{Result, Ruma}; +use crate::Ruma; const RANDOM_USER_ID_LENGTH: usize = 10; diff --git a/src/api/client/well_known.rs b/src/api/client/well_known.rs index abda61b0..eedab981 100644 --- a/src/api/client/well_known.rs +++ b/src/api/client/well_known.rs @@ -1,4 +1,5 @@ use axum::{Json, extract::State, response::IntoResponse}; +use conduwuit::{Error, Result}; use ruma::api::client::{ discovery::{ discover_homeserver::{self, HomeserverInfo, SlidingSyncProxyInfo}, @@ -7,7 +8,7 @@ use ruma::api::client::{ error::ErrorKind, }; -use crate::{Error, Result, Ruma}; +use crate::Ruma; /// # `GET /.well-known/matrix/client` /// diff --git a/src/api/mod.rs b/src/api/mod.rs index 090cf897..9ca24e72 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -8,8 +8,6 @@ pub mod server; extern crate conduwuit_core as conduwuit; extern crate conduwuit_service as service; -pub(crate) use conduwuit::{Error, Result, debug_info, pdu::PduEvent, utils}; - pub(crate) use self::router::{Ruma, RumaResponse, State}; conduwuit::mod_ctor! {} diff --git a/src/api/server/hierarchy.rs b/src/api/server/hierarchy.rs index c759c8ea..42c348f9 100644 --- a/src/api/server/hierarchy.rs +++ b/src/api/server/hierarchy.rs @@ -3,9 +3,11 @@ use conduwuit::{ Err, Result, utils::stream::{BroadbandExt, IterStream}, }; +use conduwuit_service::rooms::spaces::{ + Identifier, SummaryAccessibility, get_parent_children_via, +}; use futures::{FutureExt, StreamExt}; use ruma::api::federation::space::get_hierarchy; -use service::rooms::spaces::{Identifier, SummaryAccessibility, get_parent_children_via}; use crate::Ruma; diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index f4cc6eb2..cda34fb5 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -1,14 +1,15 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use base64::{Engine as _, engine::general_purpose}; -use conduwuit::{Err, Error, PduEvent, Result, err, utils, utils::hash::sha256, warn}; +use conduwuit::{ + Err, Error, PduEvent, Result, err, pdu::gen_event_id, utils, utils::hash::sha256, warn, +}; use ruma::{ CanonicalJsonValue, OwnedUserId, UserId, api::{client::error::ErrorKind, federation::membership::create_invite}, events::room::member::{MembershipState, RoomMemberEventContent}, serde::JsonObject, }; -use service::pdu::gen_event_id; use crate::Ruma; diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index f18d1304..4664b904 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -1,5 +1,8 @@ use axum::extract::State; -use conduwuit::{Err, debug_info, utils::IterStream, warn}; +use conduwuit::{ + Err, Error, Result, debug_info, matrix::pdu::PduBuilder, utils::IterStream, warn, +}; +use conduwuit_service::Services; use futures::StreamExt; use ruma::{ CanonicalJsonObject, OwnedUserId, RoomId, RoomVersionId, UserId, @@ -14,10 +17,7 @@ use ruma::{ }; use serde_json::value::to_raw_value; -use crate::{ - Error, Result, Ruma, - service::{Services, pdu::PduBuilder}, -}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/make_join/{roomId}/{userId}` /// diff --git a/src/api/server/make_knock.rs b/src/api/server/make_knock.rs index 71536439..6d71ab2a 100644 --- a/src/api/server/make_knock.rs +++ b/src/api/server/make_knock.rs @@ -1,15 +1,14 @@ use RoomVersionId::*; use axum::extract::State; -use conduwuit::{Err, debug_warn}; +use conduwuit::{Err, Error, Result, debug_warn, matrix::pdu::PduBuilder, warn}; use ruma::{ RoomVersionId, api::{client::error::ErrorKind, federation::knock::create_knock_event_template}, events::room::member::{MembershipState, RoomMemberEventContent}, }; use serde_json::value::to_raw_value; -use tracing::warn; -use crate::{Error, Result, Ruma, service::pdu::PduBuilder}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/make_knock/{roomId}/{userId}` /// diff --git a/src/api/server/make_leave.rs b/src/api/server/make_leave.rs index 1ed02785..cb6bd2fa 100644 --- a/src/api/server/make_leave.rs +++ b/src/api/server/make_leave.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduwuit::{Err, Result}; +use conduwuit::{Err, Result, matrix::pdu::PduBuilder}; use ruma::{ api::federation::membership::prepare_leave_event, events::room::member::{MembershipState, RoomMemberEventContent}, @@ -7,7 +7,7 @@ use ruma::{ use serde_json::value::to_raw_value; use super::make_join::maybe_strip_event_id; -use crate::{Ruma, service::pdu::PduBuilder}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/make_leave/{roomId}/{eventId}` /// diff --git a/src/api/server/openid.rs b/src/api/server/openid.rs index 4833fbe1..a09cd7ad 100644 --- a/src/api/server/openid.rs +++ b/src/api/server/openid.rs @@ -1,7 +1,8 @@ use axum::extract::State; +use conduwuit::Result; use ruma::api::federation::openid::get_openid_userinfo; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/openid/userinfo` /// diff --git a/src/api/server/publicrooms.rs b/src/api/server/publicrooms.rs index ff74574a..cf66ea71 100644 --- a/src/api/server/publicrooms.rs +++ b/src/api/server/publicrooms.rs @@ -1,5 +1,6 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; +use conduwuit::{Error, Result}; use ruma::{ api::{ client::error::ErrorKind, @@ -8,7 +9,7 @@ use ruma::{ directory::Filter, }; -use crate::{Error, Result, Ruma}; +use crate::Ruma; /// # `POST /_matrix/federation/v1/publicRooms` /// diff --git a/src/api/server/send.rs b/src/api/server/send.rs index 1f467dac..9c5bfd2b 100644 --- a/src/api/server/send.rs +++ b/src/api/server/send.rs @@ -9,11 +9,15 @@ use conduwuit::{ result::LogErr, trace, utils::{ - IterStream, ReadyExt, + IterStream, ReadyExt, millis_since_unix_epoch, stream::{BroadbandExt, TryBroadbandExt, automatic_width}, }, warn, }; +use conduwuit_service::{ + Services, + sending::{EDU_LIMIT, PDU_LIMIT}, +}; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt}; use itertools::Itertools; use ruma::{ @@ -33,16 +37,8 @@ use ruma::{ serde::Raw, to_device::DeviceIdOrAllDevices, }; -use service::{ - Services, - sending::{EDU_LIMIT, PDU_LIMIT}, -}; -use utils::millis_since_unix_epoch; -use crate::{ - Ruma, - utils::{self}, -}; +use crate::Ruma; type ResolvedMap = BTreeMap; type Pdu = (OwnedRoomId, OwnedEventId, CanonicalJsonObject); diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index c1749835..2e2e89ee 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -9,6 +9,7 @@ use conduwuit::{ utils::stream::{IterStream, TryBroadbandExt}, warn, }; +use conduwuit_service::Services; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, @@ -20,7 +21,6 @@ use ruma::{ }, }; use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; -use service::Services; use crate::Ruma; diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs index f7bb0735..c5ab0306 100644 --- a/src/api/server/send_knock.rs +++ b/src/api/server/send_knock.rs @@ -1,5 +1,9 @@ use axum::extract::State; -use conduwuit::{Err, PduEvent, Result, err, pdu::gen_event_id_canonical_json, warn}; +use conduwuit::{ + Err, Result, err, + matrix::pdu::{PduEvent, gen_event_id_canonical_json}, + warn, +}; use futures::FutureExt; use ruma::{ OwnedServerName, OwnedUserId, diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index 71516553..d3dc994c 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -1,7 +1,8 @@ #![allow(deprecated)] use axum::extract::State; -use conduwuit::{Err, Result, err}; +use conduwuit::{Err, Result, err, matrix::pdu::gen_event_id_canonical_json}; +use conduwuit_service::Services; use futures::FutureExt; use ruma::{ OwnedRoomId, OwnedUserId, RoomId, ServerName, @@ -13,10 +14,7 @@ use ruma::{ }; use serde_json::value::RawValue as RawJsonValue; -use crate::{ - Ruma, - service::{Services, pdu::gen_event_id_canonical_json}, -}; +use crate::Ruma; /// # `PUT /_matrix/federation/v1/send_leave/{roomId}/{eventId}` /// diff --git a/src/api/server/version.rs b/src/api/server/version.rs index 036b61f7..b08ff77a 100644 --- a/src/api/server/version.rs +++ b/src/api/server/version.rs @@ -1,6 +1,7 @@ +use conduwuit::Result; use ruma::api::federation::discovery::get_server_version; -use crate::{Result, Ruma}; +use crate::Ruma; /// # `GET /_matrix/federation/v1/version` /// diff --git a/src/api/server/well_known.rs b/src/api/server/well_known.rs index 48caa7d6..75c7cf5d 100644 --- a/src/api/server/well_known.rs +++ b/src/api/server/well_known.rs @@ -1,7 +1,8 @@ use axum::extract::State; +use conduwuit::{Error, Result}; use ruma::api::{client::error::ErrorKind, federation::discovery::discover_homeserver}; -use crate::{Error, Result, Ruma}; +use crate::Ruma; /// # `GET /.well-known/matrix/server` /// diff --git a/src/core/state_res/state_event.rs b/src/core/matrix/event.rs similarity index 100% rename from src/core/state_res/state_event.rs rename to src/core/matrix/event.rs diff --git a/src/core/matrix/mod.rs b/src/core/matrix/mod.rs new file mode 100644 index 00000000..8c978173 --- /dev/null +++ b/src/core/matrix/mod.rs @@ -0,0 +1,9 @@ +//! Core Matrix Library + +pub mod event; +pub mod pdu; +pub mod state_res; + +pub use event::Event; +pub use pdu::{PduBuilder, PduCount, PduEvent, PduId, RawPduId, StateKey}; +pub use state_res::{EventTypeExt, RoomVersion, StateMap, TypeStateKey}; diff --git a/src/core/pdu/mod.rs b/src/core/matrix/pdu.rs similarity index 72% rename from src/core/pdu/mod.rs rename to src/core/matrix/pdu.rs index 9fb2a3da..7e1ecfa8 100644 --- a/src/core/pdu/mod.rs +++ b/src/core/matrix/pdu.rs @@ -1,7 +1,6 @@ mod builder; mod content; mod count; -mod event; mod event_id; mod filter; mod id; @@ -17,8 +16,8 @@ mod unsigned; use std::cmp::Ordering; use ruma::{ - CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, - OwnedUserId, UInt, events::TimelineEventType, + CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, + OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, UInt, UserId, events::TimelineEventType, }; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue as RawJsonValue; @@ -27,12 +26,12 @@ pub use self::{ Count as PduCount, Id as PduId, Pdu as PduEvent, RawId as RawPduId, builder::{Builder, Builder as PduBuilder}, count::Count, - event::Event, event_id::*, id::*, raw_id::*, state_key::{ShortStateKey, StateKey}, }; +use super::Event; use crate::Result; /// Persistent Data Unit (Event) @@ -79,6 +78,36 @@ impl Pdu { } } +impl Event for Pdu { + type Id = OwnedEventId; + + fn event_id(&self) -> &Self::Id { &self.event_id } + + fn room_id(&self) -> &RoomId { &self.room_id } + + fn sender(&self) -> &UserId { &self.sender } + + fn event_type(&self) -> &TimelineEventType { &self.kind } + + fn content(&self) -> &RawJsonValue { &self.content } + + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { + MilliSecondsSinceUnixEpoch(self.origin_server_ts) + } + + fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } + + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + self.prev_events.iter() + } + + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { + self.auth_events.iter() + } + + fn redacts(&self) -> Option<&Self::Id> { self.redacts.as_ref() } +} + /// Prevent derived equality which wouldn't limit itself to event_id impl Eq for Pdu {} @@ -87,12 +116,12 @@ impl PartialEq for Pdu { fn eq(&self, other: &Self) -> bool { self.event_id == other.event_id } } -/// Ordering determined by the Pdu's ID, not the memory representations. -impl PartialOrd for Pdu { - fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } -} - /// Ordering determined by the Pdu's ID, not the memory representations. impl Ord for Pdu { fn cmp(&self, other: &Self) -> Ordering { self.event_id.cmp(&other.event_id) } } + +/// Ordering determined by the Pdu's ID, not the memory representations. +impl PartialOrd for Pdu { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } +} diff --git a/src/core/pdu/builder.rs b/src/core/matrix/pdu/builder.rs similarity index 100% rename from src/core/pdu/builder.rs rename to src/core/matrix/pdu/builder.rs diff --git a/src/core/pdu/content.rs b/src/core/matrix/pdu/content.rs similarity index 100% rename from src/core/pdu/content.rs rename to src/core/matrix/pdu/content.rs diff --git a/src/core/pdu/count.rs b/src/core/matrix/pdu/count.rs similarity index 100% rename from src/core/pdu/count.rs rename to src/core/matrix/pdu/count.rs diff --git a/src/core/pdu/event_id.rs b/src/core/matrix/pdu/event_id.rs similarity index 100% rename from src/core/pdu/event_id.rs rename to src/core/matrix/pdu/event_id.rs diff --git a/src/core/pdu/filter.rs b/src/core/matrix/pdu/filter.rs similarity index 100% rename from src/core/pdu/filter.rs rename to src/core/matrix/pdu/filter.rs diff --git a/src/core/pdu/id.rs b/src/core/matrix/pdu/id.rs similarity index 100% rename from src/core/pdu/id.rs rename to src/core/matrix/pdu/id.rs diff --git a/src/core/pdu/raw_id.rs b/src/core/matrix/pdu/raw_id.rs similarity index 100% rename from src/core/pdu/raw_id.rs rename to src/core/matrix/pdu/raw_id.rs diff --git a/src/core/pdu/redact.rs b/src/core/matrix/pdu/redact.rs similarity index 100% rename from src/core/pdu/redact.rs rename to src/core/matrix/pdu/redact.rs diff --git a/src/core/pdu/relation.rs b/src/core/matrix/pdu/relation.rs similarity index 100% rename from src/core/pdu/relation.rs rename to src/core/matrix/pdu/relation.rs diff --git a/src/core/pdu/state_key.rs b/src/core/matrix/pdu/state_key.rs similarity index 100% rename from src/core/pdu/state_key.rs rename to src/core/matrix/pdu/state_key.rs diff --git a/src/core/pdu/strip.rs b/src/core/matrix/pdu/strip.rs similarity index 100% rename from src/core/pdu/strip.rs rename to src/core/matrix/pdu/strip.rs diff --git a/src/core/pdu/tests.rs b/src/core/matrix/pdu/tests.rs similarity index 100% rename from src/core/pdu/tests.rs rename to src/core/matrix/pdu/tests.rs diff --git a/src/core/pdu/unsigned.rs b/src/core/matrix/pdu/unsigned.rs similarity index 100% rename from src/core/pdu/unsigned.rs rename to src/core/matrix/pdu/unsigned.rs diff --git a/src/core/state_res/LICENSE b/src/core/matrix/state_res/LICENSE similarity index 100% rename from src/core/state_res/LICENSE rename to src/core/matrix/state_res/LICENSE diff --git a/src/core/state_res/benches.rs b/src/core/matrix/state_res/benches.rs similarity index 100% rename from src/core/state_res/benches.rs rename to src/core/matrix/state_res/benches.rs diff --git a/src/core/state_res/error.rs b/src/core/matrix/state_res/error.rs similarity index 100% rename from src/core/state_res/error.rs rename to src/core/matrix/state_res/error.rs diff --git a/src/core/state_res/event_auth.rs b/src/core/matrix/state_res/event_auth.rs similarity index 100% rename from src/core/state_res/event_auth.rs rename to src/core/matrix/state_res/event_auth.rs diff --git a/src/core/state_res/mod.rs b/src/core/matrix/state_res/mod.rs similarity index 99% rename from src/core/state_res/mod.rs rename to src/core/matrix/state_res/mod.rs index 1db92e59..93c00d15 100644 --- a/src/core/state_res/mod.rs +++ b/src/core/matrix/state_res/mod.rs @@ -4,7 +4,6 @@ pub(crate) mod error; pub mod event_auth; mod power_levels; mod room_version; -mod state_event; #[cfg(test)] mod test_utils; @@ -36,9 +35,12 @@ use self::power_levels::PowerLevelsContentFields; pub use self::{ event_auth::{auth_check, auth_types_for_event}, room_version::RoomVersion, - state_event::Event, }; -use crate::{debug, pdu::StateKey, trace, warn}; +use crate::{ + debug, + matrix::{event::Event, pdu::StateKey}, + trace, warn, +}; /// A mapping of event type and state_key to some value `T`, usually an /// `EventId`. diff --git a/src/core/state_res/outcomes.txt b/src/core/matrix/state_res/outcomes.txt similarity index 100% rename from src/core/state_res/outcomes.txt rename to src/core/matrix/state_res/outcomes.txt diff --git a/src/core/state_res/power_levels.rs b/src/core/matrix/state_res/power_levels.rs similarity index 99% rename from src/core/state_res/power_levels.rs rename to src/core/matrix/state_res/power_levels.rs index 045b1666..19ba8fb9 100644 --- a/src/core/state_res/power_levels.rs +++ b/src/core/matrix/state_res/power_levels.rs @@ -11,9 +11,9 @@ use ruma::{ }; use serde::Deserialize; use serde_json::{Error, from_str as from_json_str}; -use tracing::error; use super::{Result, RoomVersion}; +use crate::error; #[derive(Deserialize)] struct IntRoomPowerLevelsEventContent { diff --git a/src/core/state_res/room_version.rs b/src/core/matrix/state_res/room_version.rs similarity index 100% rename from src/core/state_res/room_version.rs rename to src/core/matrix/state_res/room_version.rs diff --git a/src/core/state_res/test_utils.rs b/src/core/matrix/state_res/test_utils.rs similarity index 99% rename from src/core/state_res/test_utils.rs rename to src/core/matrix/state_res/test_utils.rs index d96ee927..f2ee4238 100644 --- a/src/core/state_res/test_utils.rs +++ b/src/core/matrix/state_res/test_utils.rs @@ -28,7 +28,10 @@ use serde_json::{ pub(crate) use self::event::PduEvent; use super::auth_types_for_event; -use crate::{Event, EventTypeExt, Result, StateMap, info}; +use crate::{ + Result, info, + matrix::{Event, EventTypeExt, StateMap}, +}; static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); diff --git a/src/core/mod.rs b/src/core/mod.rs index 80ebbdcb..b91cdf0b 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -6,11 +6,10 @@ pub mod debug; pub mod error; pub mod info; pub mod log; +pub mod matrix; pub mod metrics; pub mod mods; -pub mod pdu; pub mod server; -pub mod state_res; pub mod utils; pub use ::arrayvec; @@ -23,9 +22,8 @@ pub use ::tracing; pub use config::Config; pub use error::Error; pub use info::{rustc_flags_capture, version, version::version}; -pub use pdu::{Event, PduBuilder, PduCount, PduEvent, PduId, RawPduId, StateKey}; +pub use matrix::{Event, EventTypeExt, PduCount, PduEvent, PduId, RoomVersion, pdu, state_res}; pub use server::Server; -pub use state_res::{EventTypeExt, RoomVersion, StateMap, TypeStateKey}; pub use utils::{ctor, dtor, implement, result, result::Result}; pub use crate as conduwuit_core; diff --git a/src/core/pdu/event.rs b/src/core/pdu/event.rs deleted file mode 100644 index 09ad1666..00000000 --- a/src/core/pdu/event.rs +++ /dev/null @@ -1,35 +0,0 @@ -use ruma::{MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, events::TimelineEventType}; -use serde_json::value::RawValue as RawJsonValue; - -use super::Pdu; -pub use crate::state_res::Event; - -impl Event for Pdu { - type Id = OwnedEventId; - - fn event_id(&self) -> &Self::Id { &self.event_id } - - fn room_id(&self) -> &RoomId { &self.room_id } - - fn sender(&self) -> &UserId { &self.sender } - - fn event_type(&self) -> &TimelineEventType { &self.kind } - - fn content(&self) -> &RawJsonValue { &self.content } - - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { - MilliSecondsSinceUnixEpoch(self.origin_server_ts) - } - - fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } - - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { - self.prev_events.iter() - } - - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { - self.auth_events.iter() - } - - fn redacts(&self) -> Option<&Self::Id> { self.redacts.as_ref() } -} diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 5173987a..6780b7ae 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use conduwuit::{Err, Result, debug_info, debug_warn, error, implement}; +use conduwuit::{Err, Result, debug_info, debug_warn, error, implement, matrix::pdu::PduBuilder}; use ruma::{ RoomId, UserId, events::{ @@ -14,8 +14,6 @@ use ruma::{ }, }; -use crate::pdu::PduBuilder; - /// Invite the user to the conduwuit admin room. /// /// This is equivalent to granting server admin privileges. diff --git a/src/service/mod.rs b/src/service/mod.rs index 8f4a84b0..63a51213 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -31,7 +31,6 @@ pub mod users; extern crate conduwuit_core as conduwuit; extern crate conduwuit_database as database; -pub use conduwuit::{PduBuilder, PduCount, PduEvent, pdu}; pub(crate) use service::{Args, Dep, Service}; pub use crate::services::Services; diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index 0402ff14..eb38c2c3 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -5,7 +5,9 @@ use std::{ }; use conduwuit::{ - PduEvent, Result, StateMap, debug, err, implement, trace, + Result, debug, err, implement, + matrix::{PduEvent, StateMap}, + trace, utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryWidebandExt}, }; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, future::try_join}; diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 086dc6bd..97d3df97 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -1,7 +1,8 @@ use std::{borrow::Borrow, collections::BTreeMap, iter::once, sync::Arc, time::Instant}; use conduwuit::{ - Err, EventTypeExt, PduEvent, Result, StateKey, debug, debug_info, err, implement, state_res, + Err, Result, debug, debug_info, err, implement, + matrix::{EventTypeExt, PduEvent, StateKey, state_res}, trace, utils::stream::{BroadbandExt, ReadyExt}, warn, diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index a1b0263a..12b56935 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -1,11 +1,9 @@ use std::sync::Arc; -use conduwuit::{Result, implement}; -use database::{Deserialized, Json, Map}; +use conduwuit::{Result, implement, matrix::pdu::PduEvent}; +use conduwuit_database::{Deserialized, Json, Map}; use ruma::{CanonicalJsonObject, EventId}; -use crate::PduEvent; - pub struct Service { db: Data, } diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index d6239aee..69e859c4 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -2,7 +2,11 @@ mod data; use std::{collections::BTreeMap, sync::Arc}; -use conduwuit::{PduCount, PduId, RawPduId, Result, debug, err, warn}; +use conduwuit::{ + Result, debug, err, + matrix::pdu::{PduCount, PduId, RawPduId}, + warn, +}; use futures::{Stream, TryFutureExt, try_join}; use ruma::{ OwnedEventId, OwnedUserId, RoomId, UserId, diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 3980617e..06ff6493 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, fmt::Debug, mem::size_of_val, sync::Arc}; -pub use conduwuit::pdu::{ShortEventId, ShortId, ShortRoomId, ShortStateKey}; -use conduwuit::{Result, StateKey, err, implement, utils, utils::IterStream}; +pub use conduwuit::matrix::pdu::{ShortEventId, ShortId, ShortRoomId, ShortStateKey}; +use conduwuit::{Result, err, implement, matrix::StateKey, utils, utils::IterStream}; use database::{Deserialized, Get, Map, Qry}; use futures::{Stream, StreamExt}; use ruma::{EventId, RoomId, events::StateEventType}; diff --git a/src/service/rooms/state_accessor/room_state.rs b/src/service/rooms/state_accessor/room_state.rs index 642cd5d2..89fa2a83 100644 --- a/src/service/rooms/state_accessor/room_state.rs +++ b/src/service/rooms/state_accessor/room_state.rs @@ -1,6 +1,9 @@ use std::borrow::Borrow; -use conduwuit::{PduEvent, Result, StateKey, err, implement}; +use conduwuit::{ + Result, err, implement, + matrix::{PduEvent, StateKey}, +}; use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{EventId, RoomId, events::StateEventType}; use serde::Deserialize; diff --git a/src/service/rooms/state_accessor/state.rs b/src/service/rooms/state_accessor/state.rs index 8f2dd76f..169e69e9 100644 --- a/src/service/rooms/state_accessor/state.rs +++ b/src/service/rooms/state_accessor/state.rs @@ -1,13 +1,15 @@ use std::{borrow::Borrow, ops::Deref, sync::Arc}; use conduwuit::{ - PduEvent, Result, StateKey, at, err, implement, pair_of, + Result, at, err, implement, + matrix::{PduEvent, StateKey}, + pair_of, utils::{ result::FlatOk, stream::{BroadbandExt, IterStream, ReadyExt, TryIgnore}, }, }; -use database::Deserialized; +use conduwuit_database::Deserialized; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, future::try_join, pin_mut}; use ruma::{ EventId, OwnedEventId, UserId, diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index 7f9a7515..a680df55 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -1,13 +1,14 @@ use std::{collections::BTreeMap, sync::Arc}; use conduwuit::{ - PduCount, PduEvent, PduId, RawPduId, Result, err, + Result, err, + matrix::pdu::{PduCount, PduEvent, PduId, RawPduId}, utils::{ ReadyExt, stream::{TryIgnore, WidebandExt}, }, }; -use database::{Deserialized, Map}; +use conduwuit_database::{Deserialized, Map}; use futures::{Stream, StreamExt}; use ruma::{ CanonicalJsonValue, EventId, OwnedUserId, RoomId, UserId, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index dc359d22..947e1c38 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -10,16 +10,19 @@ use std::{ }; use async_trait::async_trait; +pub use conduwuit::matrix::pdu::{PduId, RawPduId}; use conduwuit::{ Err, Error, Result, Server, at, debug, debug_warn, err, error, implement, info, - pdu::{EventHash, PduBuilder, PduCount, PduEvent, gen_event_id}, - state_res::{self, Event, RoomVersion}, + matrix::{ + Event, + pdu::{EventHash, PduBuilder, PduCount, PduEvent, gen_event_id}, + state_res::{self, RoomVersion}, + }, utils::{ self, IterStream, MutexMap, MutexMapGuard, ReadyExt, future::TryExtExt, stream::TryIgnore, }, validated, warn, }; -pub use conduwuit::{PduId, RawPduId}; use futures::{ Future, FutureExt, Stream, StreamExt, TryStreamExt, future, future::ready, pin_mut, }; From bb8320a691eda03c202bc428e75a616b0021fe03 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Apr 2025 02:39:40 +0000 Subject: [PATCH 1645/2091] abstract and encapsulate the awkward OptionFuture into Stream pattern Signed-off-by: Jason Volk --- src/api/client/sync/v3.rs | 45 +++----------------------- src/core/utils/future/mod.rs | 2 ++ src/core/utils/future/option_ext.rs | 3 ++ src/core/utils/future/option_stream.rs | 25 ++++++++++++++ 4 files changed, 35 insertions(+), 40 deletions(-) create mode 100644 src/core/utils/future/option_stream.rs diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 12731ff6..24930941 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -15,6 +15,7 @@ use conduwuit::{ result::FlatOk, utils::{ self, BoolExt, IterStream, ReadyExt, TryFutureExtExt, + future::OptionStream, math::ruma_from_u64, stream::{BroadbandExt, Tools, TryExpect, WidebandExt}, }, @@ -1036,7 +1037,7 @@ async fn calculate_state_incremental<'a>( }) .into(); - let state_diff: OptionFuture<_> = (!full_state && state_changed) + let state_diff_ids: OptionFuture<_> = (!full_state && state_changed) .then(|| { StreamExt::into_future( services @@ -1061,45 +1062,9 @@ async fn calculate_state_incremental<'a>( }) .into(); - let lazy_state_ids = lazy_state_ids - .map(|opt| { - opt.map(|(curr, next)| { - let opt = curr; - let iter = Option::into_iter(opt); - IterStream::stream(iter).chain(next) - }) - }) - .map(Option::into_iter) - .map(IterStream::stream) - .flatten_stream() - .flatten(); - - let state_diff_ids = state_diff - .map(|opt| { - opt.map(|(curr, next)| { - let opt = curr; - let iter = Option::into_iter(opt); - IterStream::stream(iter).chain(next) - }) - }) - .map(Option::into_iter) - .map(IterStream::stream) - .flatten_stream() - .flatten(); - let state_events = current_state_ids - .map(|opt| { - opt.map(|(curr, next)| { - let opt = curr; - let iter = Option::into_iter(opt); - IterStream::stream(iter).chain(next) - }) - }) - .map(Option::into_iter) - .map(IterStream::stream) - .flatten_stream() - .flatten() - .chain(state_diff_ids) + .stream() + .chain(state_diff_ids.stream()) .broad_filter_map(|(shortstatekey, shorteventid)| async move { if witness.is_none() || encrypted_room { return Some(shorteventid); @@ -1107,7 +1072,7 @@ async fn calculate_state_incremental<'a>( lazy_filter(services, sender_user, shortstatekey, shorteventid).await }) - .chain(lazy_state_ids) + .chain(lazy_state_ids.stream()) .broad_filter_map(|shorteventid| { services .rooms diff --git a/src/core/utils/future/mod.rs b/src/core/utils/future/mod.rs index e1d96941..4edd0102 100644 --- a/src/core/utils/future/mod.rs +++ b/src/core/utils/future/mod.rs @@ -1,9 +1,11 @@ mod bool_ext; mod ext_ext; mod option_ext; +mod option_stream; mod try_ext_ext; pub use bool_ext::{BoolExt, and, or}; pub use ext_ext::ExtExt; pub use option_ext::OptionExt; +pub use option_stream::OptionStream; pub use try_ext_ext::TryExtExt; diff --git a/src/core/utils/future/option_ext.rs b/src/core/utils/future/option_ext.rs index d553e5dc..920dd044 100644 --- a/src/core/utils/future/option_ext.rs +++ b/src/core/utils/future/option_ext.rs @@ -11,11 +11,14 @@ pub trait OptionExt { impl OptionExt for OptionFuture where Fut: Future + Send, + T: Send, { + #[inline] fn is_none_or(self, f: impl FnOnce(&T) -> bool + Send) -> impl Future + Send { self.map(|o| o.as_ref().is_none_or(f)) } + #[inline] fn is_some_and(self, f: impl FnOnce(&T) -> bool + Send) -> impl Future + Send { self.map(|o| o.as_ref().is_some_and(f)) } diff --git a/src/core/utils/future/option_stream.rs b/src/core/utils/future/option_stream.rs new file mode 100644 index 00000000..81130c87 --- /dev/null +++ b/src/core/utils/future/option_stream.rs @@ -0,0 +1,25 @@ +use futures::{Future, FutureExt, Stream, StreamExt, future::OptionFuture}; + +use super::super::IterStream; + +pub trait OptionStream { + fn stream(self) -> impl Stream + Send; +} + +impl OptionStream for OptionFuture +where + Fut: Future + Send, + S: Stream + Send, + O: IntoIterator + Send, + ::IntoIter: Send, + T: Send, +{ + #[inline] + fn stream(self) -> impl Stream + Send { + self.map(|opt| opt.map(|(curr, next)| curr.into_iter().stream().chain(next))) + .map(Option::into_iter) + .map(IterStream::stream) + .flatten_stream() + .flatten() + } +} From 58b8c7516a755c0300be1fe0d36b819ebda36ffb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Apr 2025 09:02:12 +0000 Subject: [PATCH 1646/2091] extend extract_variant to multiple variants Signed-off-by: Jason Volk --- src/core/utils/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 7593990c..117fb739 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -49,10 +49,10 @@ pub fn exchange(state: &mut T, source: T) -> T { std::mem::replace(state, sou #[macro_export] macro_rules! extract_variant { - ($e:expr_2021, $variant:path) => { + ( $e:expr_2021, $( $variant:path )|* ) => { match $e { - | $variant(value) => Some(value), - | _ => None, + $( $variant(value) => Some(value), )* + _ => None, } }; } From a212bf7cfca7a6547681f46a438ecc278a905aab Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 5 Apr 2025 14:00:40 -0400 Subject: [PATCH 1647/2091] update default room version to v11 Signed-off-by: June Clementine Strawberry --- conduwuit-example.toml | 4 ++-- src/core/config/mod.rs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 75ecddab..46459547 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -527,9 +527,9 @@ # Default room version conduwuit will create rooms with. # -# Per spec, room version 10 is the default. +# Per spec, room version 11 is the default. # -#default_room_version = 10 +#default_room_version = 11 # This item is undocumented. Please contribute documentation for it. # diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 7be140a5..bb509a0d 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -640,9 +640,9 @@ pub struct Config { /// Default room version conduwuit will create rooms with. /// - /// Per spec, room version 10 is the default. + /// Per spec, room version 11 is the default. /// - /// default: 10 + /// default: 11 #[serde(default = "default_default_room_version")] pub default_room_version: RoomVersionId, @@ -2170,7 +2170,7 @@ fn default_rocksdb_stats_level() -> u8 { 1 } // I know, it's a great name #[must_use] #[inline] -pub fn default_default_room_version() -> RoomVersionId { RoomVersionId::V10 } +pub fn default_default_room_version() -> RoomVersionId { RoomVersionId::V11 } fn default_ip_range_denylist() -> Vec { vec![ From c7246662f4b2c892667b253aff1560523d8e2cff Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 5 Apr 2025 14:07:37 -0400 Subject: [PATCH 1648/2091] try partially reverting 94b107b42b722aff9518f64ad603ce01665b25f3 Signed-off-by: June Clementine Strawberry --- src/api/client/keys.rs | 43 ++++++++++-------------------------------- 1 file changed, 10 insertions(+), 33 deletions(-) diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 6865c2a4..adbdd715 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -11,7 +11,7 @@ use ruma::{ error::ErrorKind, keys::{ claim_keys, get_key_changes, get_keys, upload_keys, - upload_signatures::{self, v3::Failure}, + upload_signatures::{self}, upload_signing_keys, }, uiaa::{AuthFlow, AuthType, UiaaInfo}, @@ -308,82 +308,59 @@ async fn check_for_new_keys( /// /// Uploads end-to-end key signatures from the sender user. /// -/// TODO: clean this timo-code up more. tried to improve it a bit to stop -/// exploding the entire request on bad sigs, but needs way more work. +/// TODO: clean this timo-code up more and integrate failures. tried to improve +/// it a bit to stop exploding the entire request on bad sigs, but needs way +/// more work. pub(crate) async fn upload_signatures_route( State(services): State, body: Ruma, ) -> Result { - use upload_signatures::v3::FailureErrorCode::*; - if body.signed_keys.is_empty() { debug!("Empty signed_keys sent in key signature upload"); return Ok(upload_signatures::v3::Response::new()); } let sender_user = body.sender_user(); - let mut failures: BTreeMap> = BTreeMap::new(); - let mut failure_reasons: BTreeMap = BTreeMap::new(); - let failure = Failure { - errcode: InvalidSignature, - error: String::new(), - }; for (user_id, keys) in &body.signed_keys { for (key_id, key) in keys { let Ok(key) = serde_json::to_value(key) .inspect_err(|e| debug_warn!(?key_id, "Invalid \"key\" JSON: {e}")) else { - let mut failure = failure.clone(); - failure.error = String::from("Invalid \"key\" JSON"); - failure_reasons.insert(key_id.to_owned(), failure); continue; }; let Some(signatures) = key.get("signatures") else { - let mut failure = failure.clone(); - failure.error = String::from("Missing \"signatures\" field"); - failure_reasons.insert(key_id.to_owned(), failure); continue; }; let Some(sender_user_val) = signatures.get(sender_user.to_string()) else { - let mut failure = failure.clone(); - failure.error = String::from("Invalid user in signatures field"); - failure_reasons.insert(key_id.to_owned(), failure); continue; }; let Some(sender_user_object) = sender_user_val.as_object() else { - let mut failure = failure.clone(); - failure.error = String::from("signatures field is not a JSON object"); - failure_reasons.insert(key_id.to_owned(), failure); continue; }; for (signature, val) in sender_user_object.clone() { - let signature = (signature, val.to_string()); + let Some(val) = val.as_str().map(ToOwned::to_owned) else { + continue; + }; + let signature = (signature, val); - if let Err(e) = services + if let Err(_e) = services .users .sign_key(user_id, key_id, signature, sender_user) .await .inspect_err(|e| debug_warn!("{e}")) { - let mut failure = failure.clone(); - failure.error = format!("Error signing key: {e}"); - failure_reasons.insert(key_id.to_owned(), failure); continue; } } } - - if !failure_reasons.is_empty() { - failures.insert(user_id.to_owned(), failure_reasons.clone()); - } } - Ok(upload_signatures::v3::Response { failures }) + Ok(upload_signatures::v3::Response { failures: BTreeMap::new() }) } /// # `POST /_matrix/client/r0/keys/changes` From e28ae8fb4d442cba0eb52728a129372289c85ccd Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 5 Apr 2025 14:26:00 -0400 Subject: [PATCH 1649/2091] downgrade `deranged` crate Signed-off-by: June Clementine Strawberry --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0753f81d..86833adb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1279,9 +1279,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cfac68e08048ae1883171632c2aef3ebc555621ae56fbccce1cbf22dd7f058" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", ] From d6cc447add272f9eff0b2c77fb751dcf055d3208 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 3 Apr 2025 21:26:53 +0000 Subject: [PATCH 1650/2091] simplify acl brick-check conditions Signed-off-by: Jason Volk --- src/api/client/state.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 5c5c71f2..2ddc8f14 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -211,7 +211,7 @@ async fn allowed_to_send_state_event( // irreversible mistakes match json.deserialize_as::() { | Ok(acl_content) => { - if acl_content.allow.is_empty() { + if acl_content.allow_is_empty() { return Err!(Request(BadJson(debug_warn!( ?room_id, "Sending an ACL event with an empty allow key will permanently \ @@ -220,9 +220,7 @@ async fn allowed_to_send_state_event( )))); } - if acl_content.deny.contains(&String::from("*")) - && acl_content.allow.contains(&String::from("*")) - { + if acl_content.deny_contains("*") && acl_content.allow_contains("*") { return Err!(Request(BadJson(debug_warn!( ?room_id, "Sending an ACL event with a deny and allow key value of \"*\" will \ @@ -231,11 +229,9 @@ async fn allowed_to_send_state_event( )))); } - if acl_content.deny.contains(&String::from("*")) + if acl_content.deny_contains("*") && !acl_content.is_allowed(services.globals.server_name()) - && !acl_content - .allow - .contains(&services.globals.server_name().to_string()) + && !acl_content.allow_contains(services.globals.server_name().as_str()) { return Err!(Request(BadJson(debug_warn!( ?room_id, @@ -245,11 +241,9 @@ async fn allowed_to_send_state_event( )))); } - if !acl_content.allow.contains(&String::from("*")) + if !acl_content.allow_contains("*") && !acl_content.is_allowed(services.globals.server_name()) - && !acl_content - .allow - .contains(&services.globals.server_name().to_string()) + && !acl_content.allow_contains(services.globals.server_name().as_str()) { return Err!(Request(BadJson(debug_warn!( ?room_id, From 500faa8d7fcefab2f5bee867bf268f87fc0643fa Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Apr 2025 01:05:43 +0000 Subject: [PATCH 1651/2091] simplify space join rules related Signed-off-by: Jason Volk --- Cargo.lock | 22 ++--- Cargo.toml | 2 +- src/api/client/room/summary.rs | 70 +++++++++------ src/service/rooms/spaces/mod.rs | 110 ++++++++++++------------ src/service/rooms/state_accessor/mod.rs | 37 +------- 5 files changed, 113 insertions(+), 128 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86833adb..c2c5182f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3654,7 +3654,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "assign", "js_int", @@ -3674,7 +3674,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "js_int", "ruma-common", @@ -3686,7 +3686,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "as_variant", "assign", @@ -3709,7 +3709,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "as_variant", "base64 0.22.1", @@ -3741,7 +3741,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "as_variant", "indexmap 2.8.0", @@ -3766,7 +3766,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "bytes", "headers", @@ -3788,7 +3788,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3797,7 +3797,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "js_int", "ruma-common", @@ -3807,7 +3807,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3822,7 +3822,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "js_int", "ruma-common", @@ -3834,7 +3834,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef#edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index a44fc0f0..b1c5acb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,7 +350,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://github.com/girlbossceo/ruwuma" #branch = "conduwuit-changes" -rev = "edbdc79e560d01d9e4a76f7421e70ea4fd4c54ef" +rev = "920148dca1076454ca0ca5d43b5ce1aa708381d4" features = [ "compat", "rand", diff --git a/src/api/client/room/summary.rs b/src/api/client/room/summary.rs index 34820e83..2fa81bd2 100644 --- a/src/api/client/room/summary.rs +++ b/src/api/client/room/summary.rs @@ -4,9 +4,13 @@ use conduwuit::{ Err, Result, debug_warn, utils::{IterStream, future::TryExtExt}, }; -use futures::{FutureExt, StreamExt, future::join3, stream::FuturesUnordered}; +use futures::{ + FutureExt, StreamExt, + future::{OptionFuture, join3}, + stream::FuturesUnordered, +}; use ruma::{ - OwnedRoomId, OwnedServerName, RoomId, UserId, + OwnedServerName, RoomId, UserId, api::{ client::room::get_summary, federation::space::{SpaceHierarchyParentSummary, get_hierarchy}, @@ -91,13 +95,9 @@ async fn room_summary_response( join_rule: room.join_rule, room_type: room.room_type, room_version: room.room_version, - membership: if sender_user.is_none() { - None - } else { - Some(MembershipState::Leave) - }, encryption: room.encryption, allowed_room_ids: room.allowed_room_ids, + membership: sender_user.is_some().then_some(MembershipState::Leave), }) } @@ -106,20 +106,22 @@ async fn local_room_summary_response( room_id: &RoomId, sender_user: Option<&UserId>, ) -> Result { - let join_rule = services.rooms.state_accessor.get_space_join_rule(room_id); + let join_rule = services.rooms.state_accessor.get_join_rules(room_id); + let world_readable = services.rooms.state_accessor.is_world_readable(room_id); + let guest_can_join = services.rooms.state_accessor.guest_can_join(room_id); - let ((join_rule, allowed_room_ids), world_readable, guest_can_join) = + let (join_rule, world_readable, guest_can_join) = join3(join_rule, world_readable, guest_can_join).await; user_can_see_summary( services, room_id, - &join_rule, + &join_rule.clone().into(), guest_can_join, world_readable, - &allowed_room_ids, + join_rule.allowed_rooms(), sender_user, ) .await?; @@ -129,26 +131,43 @@ async fn local_room_summary_response( .state_accessor .get_canonical_alias(room_id) .ok(); + let name = services.rooms.state_accessor.get_name(room_id).ok(); + let topic = services.rooms.state_accessor.get_room_topic(room_id).ok(); + let room_type = services.rooms.state_accessor.get_room_type(room_id).ok(); + let avatar_url = services .rooms .state_accessor .get_avatar(room_id) .map(|res| res.into_option().unwrap_or_default().url); + let room_version = services.rooms.state.get_room_version(room_id).ok(); + let encryption = services .rooms .state_accessor .get_room_encryption(room_id) .ok(); + let num_joined_members = services .rooms .state_cache .room_joined_count(room_id) .unwrap_or(0); + let membership: OptionFuture<_> = sender_user + .map(|sender_user| { + services + .rooms + .state_accessor + .get_member(room_id, sender_user) + .map_ok_or(MembershipState::Leave, |content| content.membership) + }) + .into(); + let ( canonical_alias, name, @@ -158,6 +177,7 @@ async fn local_room_summary_response( room_type, room_version, encryption, + membership, ) = futures::join!( canonical_alias, name, @@ -167,6 +187,7 @@ async fn local_room_summary_response( room_type, room_version, encryption, + membership, ); Ok(get_summary::msc3266::Response { @@ -178,21 +199,12 @@ async fn local_room_summary_response( num_joined_members: num_joined_members.try_into().unwrap_or_default(), topic, world_readable, - join_rule, room_type, room_version, - membership: if let Some(sender_user) = sender_user { - services - .rooms - .state_accessor - .get_member(room_id, sender_user) - .await - .map_or(Some(MembershipState::Leave), |content| Some(content.membership)) - } else { - None - }, encryption, - allowed_room_ids, + membership, + allowed_room_ids: join_rule.allowed_rooms().map(Into::into).collect(), + join_rule: join_rule.into(), }) } @@ -241,7 +253,7 @@ async fn remote_room_summary_hierarchy_response( &room.join_rule, room.guest_can_join, room.world_readable, - &room.allowed_room_ids, + room.allowed_room_ids.iter().map(AsRef::as_ref), sender_user, ) .await @@ -254,15 +266,18 @@ async fn remote_room_summary_hierarchy_response( ))) } -async fn user_can_see_summary( +async fn user_can_see_summary<'a, I>( services: &Services, room_id: &RoomId, join_rule: &SpaceRoomJoinRule, guest_can_join: bool, world_readable: bool, - allowed_room_ids: &[OwnedRoomId], + allowed_room_ids: I, sender_user: Option<&UserId>, -) -> Result { +) -> Result +where + I: Iterator + Send, +{ match sender_user { | Some(sender_user) => { let user_can_see_state_events = services @@ -271,7 +286,6 @@ async fn user_can_see_summary( .user_can_see_state_events(sender_user, room_id); let is_guest = services.users.is_deactivated(sender_user).unwrap_or(false); let user_in_allowed_restricted_room = allowed_room_ids - .iter() .stream() .any(|room| services.rooms.state_cache.is_joined(sender_user, room)); diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index f51a5e3a..ea9756ba 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -121,21 +121,22 @@ pub async fn get_summary_and_children_local( | None => (), // cache miss | Some(None) => return Ok(None), | Some(Some(cached)) => { - return Ok(Some( - if self - .is_accessible_child( - current_room, - &cached.summary.join_rule, - identifier, - &cached.summary.allowed_room_ids, - ) - .await - { - SummaryAccessibility::Accessible(cached.summary.clone()) - } else { - SummaryAccessibility::Inaccessible - }, - )); + let allowed_rooms = cached.summary.allowed_room_ids.iter().map(AsRef::as_ref); + + let is_accessible_child = self.is_accessible_child( + current_room, + &cached.summary.join_rule, + identifier, + allowed_rooms, + ); + + let accessibility = if is_accessible_child.await { + SummaryAccessibility::Accessible(cached.summary.clone()) + } else { + SummaryAccessibility::Inaccessible + }; + + return Ok(Some(accessibility)); }, } @@ -145,12 +146,11 @@ pub async fn get_summary_and_children_local( .collect() .await; - let summary = self + let Ok(summary) = self .get_room_summary(current_room, children_pdus, identifier) .boxed() - .await; - - let Ok(summary) = summary else { + .await + else { return Ok(None); }; @@ -217,20 +217,19 @@ async fn get_summary_and_children_federation( .await; let identifier = Identifier::UserId(user_id); + let allowed_room_ids = summary.allowed_room_ids.iter().map(AsRef::as_ref); + let is_accessible_child = self - .is_accessible_child( - current_room, - &summary.join_rule, - &identifier, - &summary.allowed_room_ids, - ) + .is_accessible_child(current_room, &summary.join_rule, &identifier, allowed_room_ids) .await; - if is_accessible_child { - return Ok(Some(SummaryAccessibility::Accessible(summary))); - } + let accessibility = if is_accessible_child { + SummaryAccessibility::Accessible(summary) + } else { + SummaryAccessibility::Inaccessible + }; - Ok(Some(SummaryAccessibility::Inaccessible)) + Ok(Some(accessibility)) } /// Simply returns the stripped m.space.child events of a room @@ -305,14 +304,15 @@ async fn get_room_summary( children_state: Vec>, identifier: &Identifier<'_>, ) -> Result { - let (join_rule, allowed_room_ids) = self - .services - .state_accessor - .get_space_join_rule(room_id) - .await; + let join_rule = self.services.state_accessor.get_join_rules(room_id).await; let is_accessible_child = self - .is_accessible_child(room_id, &join_rule, identifier, &allowed_room_ids) + .is_accessible_child( + room_id, + &join_rule.clone().into(), + identifier, + join_rule.allowed_rooms(), + ) .await; if !is_accessible_child { @@ -379,7 +379,7 @@ async fn get_room_summary( encryption, ); - Ok(SpaceHierarchyParentSummary { + let summary = SpaceHierarchyParentSummary { canonical_alias, name, topic, @@ -388,24 +388,29 @@ async fn get_room_summary( avatar_url, room_type, children_state, - allowed_room_ids, - join_rule, - room_id: room_id.to_owned(), - num_joined_members: num_joined_members.try_into().unwrap_or_default(), encryption, room_version, - }) + room_id: room_id.to_owned(), + num_joined_members: num_joined_members.try_into().unwrap_or_default(), + allowed_room_ids: join_rule.allowed_rooms().map(Into::into).collect(), + join_rule: join_rule.clone().into(), + }; + + Ok(summary) } /// With the given identifier, checks if a room is accessable #[implement(Service)] -async fn is_accessible_child( +async fn is_accessible_child<'a, I>( &self, current_room: &RoomId, join_rule: &SpaceRoomJoinRule, identifier: &Identifier<'_>, - allowed_room_ids: &[OwnedRoomId], -) -> bool { + allowed_rooms: I, +) -> bool +where + I: Iterator + Send, +{ if let Identifier::ServerName(server_name) = identifier { // Checks if ACLs allow for the server to participate if self @@ -430,21 +435,18 @@ async fn is_accessible_child( } } - match join_rule { + match *join_rule { | SpaceRoomJoinRule::Public | SpaceRoomJoinRule::Knock | SpaceRoomJoinRule::KnockRestricted => true, | SpaceRoomJoinRule::Restricted => - allowed_room_ids - .iter() + allowed_rooms .stream() - .any(|room| async { - match identifier { - | Identifier::UserId(user) => - self.services.state_cache.is_joined(user, room).await, - | Identifier::ServerName(server) => - self.services.state_cache.server_in_room(server, room).await, - } + .any(async |room| match identifier { + | Identifier::UserId(user) => + self.services.state_cache.is_joined(user, room).await, + | Identifier::ServerName(server) => + self.services.state_cache.server_in_room(server, room).await, }) .await, diff --git a/src/service/rooms/state_accessor/mod.rs b/src/service/rooms/state_accessor/mod.rs index 7fff5935..f719fc7b 100644 --- a/src/service/rooms/state_accessor/mod.rs +++ b/src/service/rooms/state_accessor/mod.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use conduwuit::{Result, err}; use database::Map; use ruma::{ - EventEncryptionAlgorithm, JsOption, OwnedRoomAliasId, OwnedRoomId, RoomId, UserId, + EventEncryptionAlgorithm, JsOption, OwnedRoomAliasId, RoomId, UserId, events::{ StateEventType, room::{ @@ -19,14 +19,13 @@ use ruma::{ encryption::RoomEncryptionEventContent, guest_access::{GuestAccess, RoomGuestAccessEventContent}, history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent}, - join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent, RoomMembership}, + join_rules::{JoinRule, RoomJoinRulesEventContent}, member::RoomMemberEventContent, name::RoomNameEventContent, topic::RoomTopicEventContent, }, }, room::RoomType, - space::SpaceRoomJoinRule, }; use crate::{Dep, rooms}; @@ -129,42 +128,12 @@ impl Service { .map(|c: RoomTopicEventContent| c.topic) } - /// Returns the space join rule (`SpaceRoomJoinRule`) for a given room and - /// any allowed room IDs if available. Will default to Invite and empty vec - /// if doesnt exist or invalid, - pub async fn get_space_join_rule( - &self, - room_id: &RoomId, - ) -> (SpaceRoomJoinRule, Vec) { - self.room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") - .await - .map_or_else( - |_| (SpaceRoomJoinRule::Invite, vec![]), - |c: RoomJoinRulesEventContent| { - (c.join_rule.clone().into(), self.allowed_room_ids(c.join_rule)) - }, - ) - } - /// Returns the join rules for a given room (`JoinRule` type). Will default /// to Invite if doesnt exist or invalid pub async fn get_join_rules(&self, room_id: &RoomId) -> JoinRule { self.room_state_get_content(room_id, &StateEventType::RoomJoinRules, "") .await - .map_or_else(|_| JoinRule::Invite, |c: RoomJoinRulesEventContent| (c.join_rule)) - } - - /// Returns an empty vec if not a restricted room - pub fn allowed_room_ids(&self, join_rule: JoinRule) -> Vec { - let mut room_ids = Vec::with_capacity(1); // restricted rooms generally only have 1 allowed room ID - if let JoinRule::Restricted(r) | JoinRule::KnockRestricted(r) = join_rule { - for rule in r.allow { - if let AllowRule::RoomMembership(RoomMembership { room_id: membership }) = rule { - room_ids.push(membership.clone()); - } - } - } - room_ids + .map_or(JoinRule::Invite, |c: RoomJoinRulesEventContent| c.join_rule) } pub async fn get_room_type(&self, room_id: &RoomId) -> Result { From 9678948daf76b64368a6865d359ab162de1c5855 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 5 Apr 2025 18:31:14 -0400 Subject: [PATCH 1652/2091] use patch of resolv-conf crate to allow no-aaaa resolv.conf option Signed-off-by: June Clementine Strawberry --- Cargo.lock | 3 +-- Cargo.toml | 9 ++++++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c2c5182f..8817af1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3625,8 +3625,7 @@ dependencies = [ [[package]] name = "resolv-conf" version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48375394603e3dd4b2d64371f7148fd8c7baa2680e28741f2cb8d23b59e3d4c4" +source = "git+https://github.com/girlbossceo/resolv-conf?rev=200e958941d522a70c5877e3d846f55b5586c68d#200e958941d522a70c5877e3d846f55b5586c68d" dependencies = [ "hostname", ] diff --git a/Cargo.toml b/Cargo.toml index b1c5acb5..62350dee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,7 +20,7 @@ license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" repository = "https://github.com/girlbossceo/conduwuit" -rust-version = "1.85.0" +rust-version = "1.86.0" version = "0.5.0" [workspace.metadata.crane] @@ -580,6 +580,13 @@ rev = "9c8e51510c35077df888ee72a36b4b05637147da" git = "https://github.com/girlbossceo/hyper-util" rev = "e4ae7628fe4fcdacef9788c4c8415317a4489941" +# allows no-aaaa option in resolv.conf +# bumps rust edition and toolchain to 1.86.0 and 2024 +# use sat_add on line number errors +[patch.crates-io.resolv-conf] +git = "https://github.com/girlbossceo/resolv-conf" +rev = "200e958941d522a70c5877e3d846f55b5586c68d" + # # Our crates # From 3cc92b32ec97667bbabfb44edc305a972a7d3437 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 5 Apr 2025 18:37:13 -0400 Subject: [PATCH 1653/2091] bump rust toolchain to 1.86.0 Signed-off-by: June Clementine Strawberry --- flake.nix | 2 +- rust-toolchain.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 9db2e90a..49e860ed 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,7 @@ file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` - sha256 = "sha256-AJ6LX/Q/Er9kS15bn9iflkUwcgYqRQxiOIL2ToVAXaU="; + sha256 = "sha256-X/4ZBHO3iW0fOenQ3foEvscgAPJYl2abspaBThDOukI="; }; mkScope = pkgs: pkgs.lib.makeScope pkgs.newScope (self: { diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 97b4a789..aadc8f99 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -9,7 +9,7 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.85.0" +channel = "1.86.0" profile = "minimal" components = [ # For rust-analyzer From 6578b83bce71e9a232ff8531e80ab7d6d12a731c Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sat, 5 Apr 2025 20:09:22 -0400 Subject: [PATCH 1654/2091] parallelise IO of user searching, improve perf, raise max limit to 500 Signed-off-by: June Clementine Strawberry --- src/api/client/user_directory.rs | 121 ++++++++++++++----------------- 1 file changed, 55 insertions(+), 66 deletions(-) diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index 8f564eed..99b3bb67 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -1,16 +1,20 @@ use axum::extract::State; -use conduwuit::{Result, utils::TryFutureExtExt}; -use futures::{StreamExt, pin_mut}; +use conduwuit::{ + Result, + utils::{future::BoolExt, stream::BroadbandExt}, +}; +use futures::{FutureExt, StreamExt, pin_mut}; use ruma::{ - api::client::user_directory::search_users, - events::{ - StateEventType, - room::join_rules::{JoinRule, RoomJoinRulesEventContent}, - }, + api::client::user_directory::search_users::{self}, + events::room::join_rules::JoinRule, }; use crate::Ruma; +// conduwuit can handle a lot more results than synapse +const LIMIT_MAX: usize = 500; +const LIMIT_DEFAULT: usize = 10; + /// # `POST /_matrix/client/r0/user_directory/search` /// /// Searches all known users for a match. @@ -21,78 +25,63 @@ pub(crate) async fn search_users_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let limit = usize::try_from(body.limit).map_or(10, usize::from).min(100); // default limit is 10 + let sender_user = body.sender_user(); + let limit = usize::try_from(body.limit) + .map_or(LIMIT_DEFAULT, usize::from) + .min(LIMIT_MAX); - let users = services.users.stream().filter_map(|user_id| async { - // Filter out buggy users (they should not exist, but you never know...) - let user = search_users::v3::User { - user_id: user_id.to_owned(), - display_name: services.users.displayname(user_id).await.ok(), - avatar_url: services.users.avatar_url(user_id).await.ok(), - }; + let mut users = services + .users + .stream() + .map(ToOwned::to_owned) + .broad_filter_map(async |user_id| { + let user = search_users::v3::User { + user_id: user_id.clone(), + display_name: services.users.displayname(&user_id).await.ok(), + avatar_url: services.users.avatar_url(&user_id).await.ok(), + }; - let user_id_matches = user - .user_id - .to_string() - .to_lowercase() - .contains(&body.search_term.to_lowercase()); + let user_id_matches = user + .user_id + .as_str() + .to_lowercase() + .contains(&body.search_term.to_lowercase()); - let user_displayname_matches = user - .display_name - .as_ref() - .filter(|name| { + let user_displayname_matches = user.display_name.as_ref().is_some_and(|name| { name.to_lowercase() .contains(&body.search_term.to_lowercase()) - }) - .is_some(); + }); - if !user_id_matches && !user_displayname_matches { - return None; - } + if !user_id_matches && !user_displayname_matches { + return None; + } - // It's a matching user, but is the sender allowed to see them? - let mut user_visible = false; - - let user_is_in_public_rooms = services - .rooms - .state_cache - .rooms_joined(&user.user_id) - .any(|room| { - services - .rooms - .state_accessor - .room_state_get_content::( - room, - &StateEventType::RoomJoinRules, - "", - ) - .map_ok_or(false, |content| content.join_rule == JoinRule::Public) - }) - .await; - - if user_is_in_public_rooms { - user_visible = true; - } else { - let user_is_in_shared_rooms = services + let user_in_public_room = services .rooms .state_cache - .user_sees_user(sender_user, &user.user_id) - .await; + .rooms_joined(&user_id) + .map(ToOwned::to_owned) + .any(|room| async move { + services + .rooms + .state_accessor + .get_join_rules(&room) + .map(|rule| matches!(rule, JoinRule::Public)) + .await + }); - if user_is_in_shared_rooms { - user_visible = true; - } - } + let user_sees_user = services + .rooms + .state_cache + .user_sees_user(sender_user, &user_id); - user_visible.then_some(user) - }); + pin_mut!(user_in_public_room, user_sees_user); - pin_mut!(users); + user_in_public_room.or(user_sees_user).await.then_some(user) + }); - let limited = users.by_ref().next().await.is_some(); - - let results = users.take(limit).collect().await; + let results = users.by_ref().take(limit).collect().await; + let limited = users.next().await.is_some(); Ok(search_users::v3::Response { results, limited }) } From 5f8c68ab842d66ecda70726e2f9726824d51b815 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 6 Apr 2025 13:17:13 -0400 Subject: [PATCH 1655/2091] add trace logging for room summaries, use server_in_room instead of exists Signed-off-by: June Clementine Strawberry --- src/api/client/room/summary.rs | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/api/client/room/summary.rs b/src/api/client/room/summary.rs index 2fa81bd2..67d2e2ad 100644 --- a/src/api/client/room/summary.rs +++ b/src/api/client/room/summary.rs @@ -1,7 +1,7 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Err, Result, debug_warn, + Err, Result, debug_warn, trace, utils::{IterStream, future::TryExtExt}, }; use futures::{ @@ -74,7 +74,12 @@ async fn room_summary_response( servers: &[OwnedServerName], sender_user: Option<&UserId>, ) -> Result { - if services.rooms.metadata.exists(room_id).await { + if services + .rooms + .state_cache + .server_in_room(services.globals.server_name(), room_id) + .await + { return local_room_summary_response(services, room_id, sender_user) .boxed() .await; @@ -106,14 +111,14 @@ async fn local_room_summary_response( room_id: &RoomId, sender_user: Option<&UserId>, ) -> Result { + trace!(?sender_user, "Sending local room summary response for {room_id:?}"); let join_rule = services.rooms.state_accessor.get_join_rules(room_id); - let world_readable = services.rooms.state_accessor.is_world_readable(room_id); - let guest_can_join = services.rooms.state_accessor.guest_can_join(room_id); let (join_rule, world_readable, guest_can_join) = join3(join_rule, world_readable, guest_can_join).await; + trace!("{join_rule:?}, {world_readable:?}, {guest_can_join:?}"); user_can_see_summary( services, @@ -215,6 +220,7 @@ async fn remote_room_summary_hierarchy_response( servers: &[OwnedServerName], sender_user: Option<&UserId>, ) -> Result { + trace!(?sender_user, ?servers, "Sending remote room summary response for {room_id:?}"); if !services.config.allow_federation { return Err!(Request(Forbidden("Federation is disabled."))); } @@ -237,6 +243,7 @@ async fn remote_room_summary_hierarchy_response( .collect(); while let Some(Ok(response)) = requests.next().await { + trace!("{response:?}"); let room = response.room.clone(); if room.room_id != room_id { debug_warn!( @@ -278,6 +285,7 @@ async fn user_can_see_summary<'a, I>( where I: Iterator + Send, { + let is_public_room = matches!(join_rule, Public | Knock | KnockRestricted); match sender_user { | Some(sender_user) => { let user_can_see_state_events = services @@ -296,7 +304,7 @@ where if user_can_see_state_events || (is_guest && guest_can_join) - || matches!(&join_rule, &Public | &Knock | &KnockRestricted) + || is_public_room || user_in_allowed_restricted_room { return Ok(()); @@ -309,7 +317,7 @@ where ))) }, | None => { - if matches!(join_rule, Public | Knock | KnockRestricted) || world_readable { + if is_public_room || world_readable { return Ok(()); } From ff276a42a36cfe565ff541ce064db25bbb1946c8 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 6 Apr 2025 13:19:09 -0400 Subject: [PATCH 1656/2091] drop unnecessary info log to debug Signed-off-by: June Clementine Strawberry --- src/api/client/keys.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index adbdd715..650c573f 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use axum::extract::State; -use conduwuit::{Err, Error, Result, debug, debug_warn, err, info, result::NotFound, utils}; +use conduwuit::{Err, Error, Result, debug, debug_warn, err, result::NotFound, utils}; use conduwuit_service::{Services, users::parse_master_key}; use futures::{StreamExt, stream::FuturesUnordered}; use ruma::{ @@ -177,7 +177,7 @@ pub(crate) async fn upload_signing_keys_route( body.master_key.as_ref(), ) .await - .inspect_err(|e| info!(?e)) + .inspect_err(|e| debug!(?e)) { | Ok(exists) => { if let Some(result) = exists { From d5ad973464168c567c3f9615380ced9e0067da4f Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 6 Apr 2025 15:25:11 -0400 Subject: [PATCH 1657/2091] change forbidden_server_names and etc to allow regex patterns for wildcards Signed-off-by: June Clementine Strawberry --- conduwuit-example.toml | 27 ++++++++++------ src/api/client/directory.rs | 14 ++++++--- src/api/client/membership.rs | 6 ++-- src/api/client/message.rs | 3 +- src/api/router/auth.rs | 3 +- src/api/server/invite.rs | 6 ++-- src/api/server/make_join.rs | 6 ++-- src/api/server/make_knock.rs | 6 ++-- src/api/server/send_join.rs | 12 +++----- src/api/server/send_knock.rs | 6 ++-- src/core/config/mod.rs | 51 +++++++++++++++++-------------- src/service/federation/execute.rs | 2 +- src/service/media/remote.rs | 8 ++++- 13 files changed, 79 insertions(+), 71 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 46459547..118bc57d 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -594,7 +594,7 @@ # Currently, conduwuit doesn't support inbound batched key requests, so # this list should only contain other Synapse servers. # -# example: ["matrix.org", "envs.net", "tchncs.de"] +# example: ["matrix.org", "tchncs.de"] # #trusted_servers = ["matrix.org"] @@ -1186,13 +1186,16 @@ # #prune_missing_media = false -# Vector list of servers that conduwuit will refuse to download remote -# media from. +# Vector list of regex patterns of server names that conduwuit will refuse +# to download remote media from. +# +# example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] # #prevent_media_downloads_from = [] -# List of forbidden server names that we will block incoming AND outgoing -# federation with, and block client room joins / remote user invites. +# List of forbidden server names via regex patterns that we will block +# incoming AND outgoing federation with, and block client room joins / +# remote user invites. # # This check is applied on the room ID, room alias, sender server name, # sender user's server name, inbound federation X-Matrix origin, and @@ -1200,11 +1203,15 @@ # # Basically "global" ACLs. # +# example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] +# #forbidden_remote_server_names = [] -# List of forbidden server names that we will block all outgoing federated -# room directory requests for. Useful for preventing our users from -# wandering into bad servers or spaces. +# List of forbidden server names via regex patterns that we will block all +# outgoing federated room directory requests for. Useful for preventing +# our users from wandering into bad servers or spaces. +# +# example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] # #forbidden_remote_room_directory_server_names = [] @@ -1315,7 +1322,7 @@ # used, and startup as warnings if any room aliases in your database have # a forbidden room alias/ID. # -# example: ["19dollarfortnitecards", "b[4a]droom"] +# example: ["19dollarfortnitecards", "b[4a]droom", "badphrase"] # #forbidden_alias_names = [] @@ -1328,7 +1335,7 @@ # startup as warnings if any local users in your database have a forbidden # username. # -# example: ["administrator", "b[a4]dusernam[3e]"] +# example: ["administrator", "b[a4]dusernam[3e]", "badphrase"] # #forbidden_usernames = [] diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 9ca35537..b44b9f64 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -52,10 +52,13 @@ pub(crate) async fn get_public_rooms_filtered_route( ) -> Result { if let Some(server) = &body.server { if services - .server .config .forbidden_remote_room_directory_server_names - .contains(server) + .is_match(server.host()) + || services + .config + .forbidden_remote_server_names + .is_match(server.host()) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } @@ -90,10 +93,13 @@ pub(crate) async fn get_public_rooms_route( ) -> Result { if let Some(server) = &body.server { if services - .server .config .forbidden_remote_room_directory_server_names - .contains(server) + .is_match(server.host()) + || services + .config + .forbidden_remote_server_names + .is_match(server.host()) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index d0345c8e..1eeacf83 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -79,10 +79,9 @@ async fn banned_room_check( if let Some(room_id) = room_id { if services.rooms.metadata.is_banned(room_id).await || services - .server .config .forbidden_remote_server_names - .contains(&room_id.server_name().unwrap().to_owned()) + .is_match(room_id.server_name().unwrap().host()) { warn!( "User {user_id} who is not an admin attempted to send an invite for or \ @@ -120,10 +119,9 @@ async fn banned_room_check( } } else if let Some(server_name) = server_name { if services - .server .config .forbidden_remote_server_names - .contains(&server_name.to_owned()) + .is_match(server_name.host()) { warn!( "User {user_id} who is not an admin tried joining a room which has the server \ diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 3e784a4a..db11ef4a 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -261,10 +261,9 @@ pub(crate) async fn is_ignored_pdu( let ignored_type = IGNORED_MESSAGE_TYPES.binary_search(&pdu.kind).is_ok(); let ignored_server = services - .server .config .forbidden_remote_server_names - .contains(pdu.sender().server_name()); + .is_match(pdu.sender().server_name().host()); if ignored_type && (ignored_server || services.users.user_is_ignored(&pdu.sender, user_id).await) diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 5cd7b831..0eb61ca6 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -317,10 +317,9 @@ fn auth_server_checks(services: &Services, x_matrix: &XMatrix) -> Result<()> { let origin = &x_matrix.origin; if services - .server .config .forbidden_remote_server_names - .contains(origin) + .is_match(origin.host()) { return Err!(Request(Forbidden(debug_warn!( "Federation requests from {origin} denied." diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index cda34fb5..edd6ac16 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -38,20 +38,18 @@ pub(crate) async fn create_invite_route( if let Some(server) = body.room_id.server_name() { if services - .server .config .forbidden_remote_server_names - .contains(&server.to_owned()) + .is_match(server.host()) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } } if services - .server .config .forbidden_remote_server_names - .contains(body.origin()) + .is_match(body.origin().host()) { warn!( "Received federated/remote invite from banned server {} for room ID {}. Rejecting.", diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index 4664b904..ac2c5485 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -42,10 +42,9 @@ pub(crate) async fn create_join_event_template_route( .await?; if services - .server .config .forbidden_remote_server_names - .contains(body.origin()) + .is_match(body.origin().host()) { warn!( "Server {} for remote user {} tried joining room ID {} which has a server name that \ @@ -59,10 +58,9 @@ pub(crate) async fn create_join_event_template_route( if let Some(server) = body.room_id.server_name() { if services - .server .config .forbidden_remote_server_names - .contains(&server.to_owned()) + .is_match(server.host()) { return Err!(Request(Forbidden(warn!( "Room ID server name {server} is banned on this homeserver." diff --git a/src/api/server/make_knock.rs b/src/api/server/make_knock.rs index 6d71ab2a..511c13b2 100644 --- a/src/api/server/make_knock.rs +++ b/src/api/server/make_knock.rs @@ -33,10 +33,9 @@ pub(crate) async fn create_knock_event_template_route( .await?; if services - .server .config .forbidden_remote_server_names - .contains(body.origin()) + .is_match(body.origin().host()) { warn!( "Server {} for remote user {} tried knocking room ID {} which has a server name \ @@ -50,10 +49,9 @@ pub(crate) async fn create_knock_event_template_route( if let Some(server) = body.room_id.server_name() { if services - .server .config .forbidden_remote_server_names - .contains(&server.to_owned()) + .is_match(server.host()) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 2e2e89ee..a66d8890 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -268,10 +268,9 @@ pub(crate) async fn create_join_event_v1_route( body: Ruma, ) -> Result { if services - .server .config .forbidden_remote_server_names - .contains(body.origin()) + .is_match(body.origin().host()) { warn!( "Server {} tried joining room ID {} through us who has a server name that is \ @@ -284,10 +283,9 @@ pub(crate) async fn create_join_event_v1_route( if let Some(server) = body.room_id.server_name() { if services - .server .config .forbidden_remote_server_names - .contains(&server.to_owned()) + .is_match(server.host()) { warn!( "Server {} tried joining room ID {} through us which has a server name that is \ @@ -316,20 +314,18 @@ pub(crate) async fn create_join_event_v2_route( body: Ruma, ) -> Result { if services - .server .config .forbidden_remote_server_names - .contains(body.origin()) + .is_match(body.origin().host()) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } if let Some(server) = body.room_id.server_name() { if services - .server .config .forbidden_remote_server_names - .contains(&server.to_owned()) + .is_match(server.host()) { warn!( "Server {} tried joining room ID {} through us which has a server name that is \ diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs index c5ab0306..ee7b6cba 100644 --- a/src/api/server/send_knock.rs +++ b/src/api/server/send_knock.rs @@ -26,10 +26,9 @@ pub(crate) async fn create_knock_event_v1_route( body: Ruma, ) -> Result { if services - .server .config .forbidden_remote_server_names - .contains(body.origin()) + .is_match(body.origin().host()) { warn!( "Server {} tried knocking room ID {} who has a server name that is globally \ @@ -42,10 +41,9 @@ pub(crate) async fn create_knock_event_v1_route( if let Some(server) = body.room_id.server_name() { if services - .server .config .forbidden_remote_server_names - .contains(&server.to_owned()) + .is_match(server.host()) { warn!( "Server {} tried knocking room ID {} which has a server name that is globally \ diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index bb509a0d..0ca6bbaf 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -3,7 +3,7 @@ pub mod manager; pub mod proxy; use std::{ - collections::{BTreeMap, BTreeSet, HashSet}, + collections::{BTreeMap, BTreeSet}, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, path::{Path, PathBuf}, }; @@ -715,7 +715,7 @@ pub struct Config { /// Currently, conduwuit doesn't support inbound batched key requests, so /// this list should only contain other Synapse servers. /// - /// example: ["matrix.org", "envs.net", "tchncs.de"] + /// example: ["matrix.org", "tchncs.de"] /// /// default: ["matrix.org"] #[serde(default = "default_trusted_servers")] @@ -1361,15 +1361,18 @@ pub struct Config { #[serde(default)] pub prune_missing_media: bool, - /// Vector list of servers that conduwuit will refuse to download remote - /// media from. + /// Vector list of regex patterns of server names that conduwuit will refuse + /// to download remote media from. + /// + /// example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] /// /// default: [] - #[serde(default)] - pub prevent_media_downloads_from: HashSet, + #[serde(default, with = "serde_regex")] + pub prevent_media_downloads_from: RegexSet, - /// List of forbidden server names that we will block incoming AND outgoing - /// federation with, and block client room joins / remote user invites. + /// List of forbidden server names via regex patterns that we will block + /// incoming AND outgoing federation with, and block client room joins / + /// remote user invites. /// /// This check is applied on the room ID, room alias, sender server name, /// sender user's server name, inbound federation X-Matrix origin, and @@ -1377,17 +1380,21 @@ pub struct Config { /// /// Basically "global" ACLs. /// - /// default: [] - #[serde(default)] - pub forbidden_remote_server_names: HashSet, - - /// List of forbidden server names that we will block all outgoing federated - /// room directory requests for. Useful for preventing our users from - /// wandering into bad servers or spaces. + /// example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] /// /// default: [] - #[serde(default = "HashSet::new")] - pub forbidden_remote_room_directory_server_names: HashSet, + #[serde(default, with = "serde_regex")] + pub forbidden_remote_server_names: RegexSet, + + /// List of forbidden server names via regex patterns that we will block all + /// outgoing federated room directory requests for. Useful for preventing + /// our users from wandering into bad servers or spaces. + /// + /// example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] + /// + /// default: [] + #[serde(default, with = "serde_regex")] + pub forbidden_remote_room_directory_server_names: RegexSet, /// Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you /// do not want conduwuit to send outbound requests to. Defaults to @@ -1508,11 +1515,10 @@ pub struct Config { /// used, and startup as warnings if any room aliases in your database have /// a forbidden room alias/ID. /// - /// example: ["19dollarfortnitecards", "b[4a]droom"] + /// example: ["19dollarfortnitecards", "b[4a]droom", "badphrase"] /// /// default: [] - #[serde(default)] - #[serde(with = "serde_regex")] + #[serde(default, with = "serde_regex")] pub forbidden_alias_names: RegexSet, /// List of forbidden username patterns/strings. @@ -1524,11 +1530,10 @@ pub struct Config { /// startup as warnings if any local users in your database have a forbidden /// username. /// - /// example: ["administrator", "b[a4]dusernam[3e]"] + /// example: ["administrator", "b[a4]dusernam[3e]", "badphrase"] /// /// default: [] - #[serde(default)] - #[serde(with = "serde_regex")] + #[serde(default, with = "serde_regex")] pub forbidden_usernames: RegexSet, /// Retry failed and incomplete messages to remote servers immediately upon diff --git a/src/service/federation/execute.rs b/src/service/federation/execute.rs index 63f2ccfb..97314ffb 100644 --- a/src/service/federation/execute.rs +++ b/src/service/federation/execute.rs @@ -69,7 +69,7 @@ where .server .config .forbidden_remote_server_names - .contains(dest) + .is_match(dest.host()) { return Err!(Request(Forbidden(debug_warn!("Federation with {dest} is not allowed.")))); } diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index b6c853d2..cdcb429e 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -426,7 +426,13 @@ fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> { .server .config .prevent_media_downloads_from - .contains(mxc.server_name) + .is_match(mxc.server_name.host()) + || self + .services + .server + .config + .forbidden_remote_server_names + .is_match(mxc.server_name.host()) { // we'll lie to the client and say the blocked server's media was not found and // log. the client has no way of telling anyways so this is a security bonus. From 99868b166173d7bd510a7f2dd3a1b1e415a99682 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Sun, 6 Apr 2025 15:30:01 -0400 Subject: [PATCH 1658/2091] update new complement flakes Signed-off-by: June Clementine Strawberry --- .../complement/test_results.jsonl | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/test_results/complement/test_results.jsonl b/tests/test_results/complement/test_results.jsonl index c0e28750..97c2e1b1 100644 --- a/tests/test_results/complement/test_results.jsonl +++ b/tests/test_results/complement/test_results.jsonl @@ -491,7 +491,7 @@ {"Action":"fail","Test":"TestRoomCreationReportsEventsToMyself"} {"Action":"fail","Test":"TestRoomCreationReportsEventsToMyself/parallel"} {"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Joining_room_twice_is_idempotent"} -{"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Room_creation_reports_m.room.create_to_myself"} +{"Action":"fail","Test":"TestRoomCreationReportsEventsToMyself/parallel/Room_creation_reports_m.room.create_to_myself"} {"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Room_creation_reports_m.room.member_to_myself"} {"Action":"pass","Test":"TestRoomCreationReportsEventsToMyself/parallel/Setting_room_topic_reports_m.room.topic_to_myself"} {"Action":"fail","Test":"TestRoomCreationReportsEventsToMyself/parallel/Setting_state_twice_is_idempotent"} @@ -527,17 +527,17 @@ {"Action":"pass","Test":"TestRoomMessagesLazyLoadingLocalUser"} {"Action":"pass","Test":"TestRoomReadMarkers"} {"Action":"pass","Test":"TestRoomReceipts"} -{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin"} -{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin/Bob_can_find_Alice_by_mxid"} -{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin/Bob_can_find_Alice_by_profile_display_name"} -{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin/Eve_can_find_Alice_by_mxid"} -{"Action":"fail","Test":"TestRoomSpecificUsernameAtJoin/Eve_can_find_Alice_by_profile_display_name"} +{"Action":"pass","Test":"TestRoomSpecificUsernameAtJoin"} +{"Action":"pass","Test":"TestRoomSpecificUsernameAtJoin/Bob_can_find_Alice_by_mxid"} +{"Action":"pass","Test":"TestRoomSpecificUsernameAtJoin/Bob_can_find_Alice_by_profile_display_name"} +{"Action":"pass","Test":"TestRoomSpecificUsernameAtJoin/Eve_can_find_Alice_by_mxid"} +{"Action":"pass","Test":"TestRoomSpecificUsernameAtJoin/Eve_can_find_Alice_by_profile_display_name"} {"Action":"pass","Test":"TestRoomSpecificUsernameAtJoin/Eve_cannot_find_Alice_by_room-specific_name_that_Eve_is_not_privy_to"} -{"Action":"fail","Test":"TestRoomSpecificUsernameChange"} -{"Action":"fail","Test":"TestRoomSpecificUsernameChange/Bob_can_find_Alice_by_mxid"} -{"Action":"fail","Test":"TestRoomSpecificUsernameChange/Bob_can_find_Alice_by_profile_display_name"} -{"Action":"fail","Test":"TestRoomSpecificUsernameChange/Eve_can_find_Alice_by_mxid"} -{"Action":"fail","Test":"TestRoomSpecificUsernameChange/Eve_can_find_Alice_by_profile_display_name"} +{"Action":"pass","Test":"TestRoomSpecificUsernameChange"} +{"Action":"pass","Test":"TestRoomSpecificUsernameChange/Bob_can_find_Alice_by_mxid"} +{"Action":"pass","Test":"TestRoomSpecificUsernameChange/Bob_can_find_Alice_by_profile_display_name"} +{"Action":"pass","Test":"TestRoomSpecificUsernameChange/Eve_can_find_Alice_by_mxid"} +{"Action":"pass","Test":"TestRoomSpecificUsernameChange/Eve_can_find_Alice_by_profile_display_name"} {"Action":"pass","Test":"TestRoomSpecificUsernameChange/Eve_cannot_find_Alice_by_room-specific_name_that_Eve_is_not_privy_to"} {"Action":"fail","Test":"TestRoomState"} {"Action":"fail","Test":"TestRoomState/Parallel"} @@ -589,7 +589,7 @@ {"Action":"fail","Test":"TestSync/parallel/Newly_joined_room_has_correct_timeline_in_incremental_sync"} {"Action":"fail","Test":"TestSync/parallel/Newly_joined_room_includes_presence_in_incremental_sync"} {"Action":"pass","Test":"TestSync/parallel/Newly_joined_room_is_included_in_an_incremental_sync"} -{"Action":"fail","Test":"TestSync/parallel/sync_should_succeed_even_if_the_sync_token_points_to_a_redaction_of_an_unknown_event"} +{"Action":"pass","Test":"TestSync/parallel/sync_should_succeed_even_if_the_sync_token_points_to_a_redaction_of_an_unknown_event"} {"Action":"pass","Test":"TestSyncFilter"} {"Action":"pass","Test":"TestSyncFilter/Can_create_filter"} {"Action":"pass","Test":"TestSyncFilter/Can_download_filter"} From 47f83454570a1d4338137708b4b042e8c49b7cb7 Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 8 Apr 2025 09:05:49 -0400 Subject: [PATCH 1659/2091] bump tokio because of RUSTSEC-2025-0023 Signed-off-by: June Clementine Strawberry --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8817af1a..c724e31e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4758,9 +4758,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.1" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 62350dee..f5ee3f0f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -242,7 +242,7 @@ default-features = false features = ["std", "async-await"] [workspace.dependencies.tokio] -version = "1.44.1" +version = "1.44.2" default-features = false features = [ "fs", From d8311a5ff672fdc4729d956af5e3af8646b0670d Mon Sep 17 00:00:00 2001 From: June Clementine Strawberry Date: Tue, 8 Apr 2025 23:38:54 -0400 Subject: [PATCH 1660/2091] bump crossbeam-channel bc yanked crate with potential double free Signed-off-by: June Clementine Strawberry --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c724e31e..d81fdbc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1119,9 +1119,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" dependencies = [ "crossbeam-utils", ] From e054a56b3286a6fb3091bedd5261089435ed26d1 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 14 Apr 2025 22:34:22 +0100 Subject: [PATCH 1661/2091] docs: New readme It's a continuwuation! --- README.md | 214 +++++++++++++++++++++--------------------------------- 1 file changed, 82 insertions(+), 132 deletions(-) diff --git a/README.md b/README.md index d8f99d45..89e1a299 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ -# conduwuit +# continuwuity + -### a very cool [Matrix](https://matrix.org/) chat homeserver written in Rust +## A community-driven [Matrix](https://matrix.org/) homeserver in Rust -Visit the [conduwuit documentation](https://conduwuit.puppyirl.gay/) for more -information and how to deploy/setup conduwuit. +[continuwuity] is a Matrix homeserver written in Rust. +It's a community continuation of the [conduwuit](https://github.com/girlbossceo/conduwuit) homeserver. -#### What is Matrix? + +### Why does this exist? + +The original conduwuit project has been archived and is no longer maintained. Rather than letting this Rust-based Matrix homeserver disappear, a group of community contributors have forked the project to continue its development, fix outstanding issues, and add new features. + +Our aim is to provide a stable, well-maintained alternative for current conduwuit users and to welcome new users looking for a lightweight, efficient Matrix homeserver. + + +### Who are we? + +We are a group of Matrix enthusiasts, developers and system administrators who have used conduwuit and believe in its potential. Our team includes both previous +contributors to the original project and new developers who want to help maintain and improve this important piece of Matrix infrastructure. + +We operate as an open community project, welcoming contributions from anyone interested in improving continuwuity. + +### What is Matrix? [Matrix](https://matrix.org) is an open, federated, and extensible network for -decentralised communication. Users from any Matrix homeserver can chat with users from all +decentralized communication. Users from any Matrix homeserver can chat with users from all other homeservers over federation. Matrix is designed to be extensible and built on top of. You can even use bridges such as Matrix Appservices to communicate with users outside of Matrix, like a community on Discord. -#### What is the goal? +### What are the project's goals? -A high-performance, efficient, low-cost, and featureful Matrix homeserver that's -easy to set up and just works with minimal configuration needed. +Continuwuity aims to: -#### Can I try it out? +- Maintain a stable, reliable Matrix homeserver implementation in Rust +- Improve compatibility and specification compliance with the Matrix protocol +- Fix bugs and performance issues from the original conduwuit +- Add missing features needed by homeserver administrators +- Provide comprehensive documentation and easy deployment options +- Create a sustainable development model for long-term maintenance +- Keep a lightweight, efficient codebase that can run on modest hardware -An official conduwuit server ran by me is available at transfem.dev -([element.transfem.dev](https://element.transfem.dev) / -[cinny.transfem.dev](https://cinny.transfem.dev)) +### Can I try it out? -transfem.dev is a public homeserver that can be used, it is not a "test only -homeserver". This means there are rules, so please read the rules: -[https://transfem.dev/homeserver_rules.txt](https://transfem.dev/homeserver_rules.txt) +Not right now. We've still got work to do! -transfem.dev is also listed at -[servers.joinmatrix.org](https://servers.joinmatrix.org/), which is a list of -popular public Matrix homeservers, including some others that run conduwuit. -#### What is the current status? +### What are we working on? -conduwuit is technically a hard fork of [Conduit](https://conduit.rs/), which is in beta. -The beta status initially was inherited from Conduit, however the huge amount of -codebase divergance, changes, fixes, and improvements have effectively made this -beta status not entirely applicable to us anymore. +We're working our way through all of the issues in the [Forgejo project](https://forgejo.ellis.link/continuwuation/continuwuity/issues). -conduwuit is very stable based on our rapidly growing userbase, has lots of features that users -expect, and very usable as a daily driver for small, medium, and upper-end medium sized homeservers. +- [Replacing old conduwuit links with working continuwuity links](https://forgejo.ellis.link/continuwuation/continuwuity/issues/742) +- [Getting CI and docs deployment working on the new Forgejo project](https://forgejo.ellis.link/continuwuation/continuwuity/issues/740) +- [Packaging & availability in more places](https://forgejo.ellis.link/continuwuation/continuwuity/issues/747) +- [Appservices bugs & features](https://forgejo.ellis.link/continuwuation/continuwuity/issues?q=&type=all&state=open&labels=178&milestone=0&assignee=0&poster=0) +- [Improving compatibility and spec compliance](https://forgejo.ellis.link/continuwuation/continuwuity/issues?labels=119) +- Automated testing +- [Admin API](https://forgejo.ellis.link/continuwuation/continuwuity/issues/748) +- [Policy-list controlled moderation](https://forgejo.ellis.link/continuwuation/continuwuity/issues/750) -A lot of critical stability and performance issues have been fixed, and a lot of -necessary groundwork has finished; making this project way better than it was -back in the start at ~early 2024. +### Can I migrate my data from x? -#### Where is the differences page? +- Conduwuit: Yes +- Conduit: No, database is now incompatible +- Grapevine: No, database is now incompatible +- Dendrite: No +- Synapse: No -conduwuit historically had a "differences" page that listed each and every single -different thing about conduwuit from Conduit, as a way to promote and advertise -conduwuit by showing significant amounts of work done. While this was feasible to -maintain back when the project was new in early-2024, this became impossible -very quickly and has unfortunately became heavily outdated, missing tons of things, etc. - -It's difficult to list out what we do differently, what are our notable features, etc -when there's so many things and features and bug fixes and performance optimisations, -the list goes on. We simply recommend folks to just try out conduwuit, or ask us -what features you are looking for and if they're implemented in conduwuit. - -#### How is conduwuit funded? Is conduwuit sustainable? - -conduwuit has no external funding. This is made possible purely in my freetime with -contributors, also in their free time, and only by user-curated donations. - -conduwuit has existed since around November 2023, but [only became more publicly known -in March/April 2024](https://matrix.org/blog/2024/04/26/this-week-in-matrix-2024-04-26/#conduwuit-website) -and we have no plans in stopping or slowing down any time soon! - -#### Can I migrate or switch from Conduit? - -conduwuit had drop-in migration/replacement support for Conduit for about 12 months before -bugs somewhere along the line broke it. Maintaining this has been difficult and -the majority of Conduit users have already migrated, additionally debugging Conduit -is not one of our interests, and so Conduit migration no longer works. We also -feel that 12 months has been plenty of time for people to seamlessly migrate. - -If you are a Conduit user looking to migrate, you will have to wipe and reset -your database. We may fix seamless migration support at some point, but it's not an interest -from us. - -#### Can I migrate from Synapse or Dendrite? - -Currently there is no known way to seamlessly migrate all user data from the old -homeserver to conduwuit. However it is perfectly acceptable to replace the old -homeserver software with conduwuit using the same server name and there will not -be any issues with federation. - -There is an interest in developing a built-in seamless user data migration -method into conduwuit, however there is no concrete ETA or timeline for this. +Although you can't migrate your data from other homeservers, it is perfectly acceptable to set up continuwuity on the same domain as a previous homeserver. +## Contribution + +### Development flow + +- Features / changes must developed in a separate branch +- For each change, create a descriptive PR +- Your code will be reviewed by one or more of the continuwuity developers +- The branch will be deployed live on multiple tester's matrix servers to shake out bugs +- Once all testers and reviewers have agreed, the PR will be merged to the main branch +- The main branch will have nightly builds deployed to users on the cutting edge +- Every week or two, a new release is cut. + +The main branch is always green! + + +### Policy on pulling from other forks + +We welcome contributions from other forks of conduwuit, subject to our review process. +When incorporating code from other forks: + +- All external contributions must go through our standard PR process +- Code must meet our quality standards and pass tests +- Code changes will require testing on multiple test servers before merging +- Attribution will be given to original authors and forks +- We prioritize stability and compatibility when evaluating external contributions +- Features that align with our project goals will be given priority consideration + #### Contact -[`#conduwuit:puppygock.gay`](https://matrix.to/#/#conduwuit:puppygock.gay) -is the official project Matrix room. You can get support here, ask questions or -concerns, get assistance setting up conduwuit, etc. - -This room should stay relevant and focused on conduwuit. An offtopic general -chatter room can be found in the room topic there as well. - -Please keep the issue trackers focused on *actual* bug reports and enhancement requests. - -General support is extremely difficult to be offered over an issue tracker, and -simple questions should be asked directly in an interactive platform like our -Matrix room above as they can turn into a relevant discussion and/or may not be -simple to answer. If you're not sure, just ask in the Matrix room. - -If you have a bug or feature to request: [Open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new) - -If you need to contact the primary maintainer, my contact methods are on my website: https://girlboss.ceo - -#### Donate - -conduwuit development is purely made possible by myself and contributors. I do -not get paid to work on this, and I work on it in my free time. Donations are -heavily appreciated! 💜🥺 - -- Liberapay: -- GitHub Sponsors: -- Ko-fi: - -I do not and will not accept cryptocurrency donations, including things related. - -Note that donations will NOT guarantee you or give you any kind of tangible product, -feature prioritisation, etc. By donating, you are agreeing that conduwuit is NOT -going to provide you any goods or services as part of your donation, and this -donation is purely a generous donation. We will not provide things like paid -personal/direct support, feature request priority, merchandise, etc. - -#### Logo - -Original repo and Matrix room picture was from bran (<3). Current banner image -and logo is directly from [this cohost -post](https://web.archive.org/web/20241126004041/https://cohost.org/RatBaby/post/1028290-finally-a-flag-for). - -An SVG logo made by [@nktnet1](https://github.com/nktnet1) is available here: - -#### Is it conduwuit or Conduwuit? - -Both, but I prefer conduwuit. - -#### Mirrors of conduwuit - -If GitHub is unavailable in your country, or has poor connectivity, conduwuit's -source code is mirrored onto the following additional platforms I maintain: - -- GitHub: -- GitLab: -- git.girlcock.ceo: -- git.gay: -- mau.dev: -- Codeberg: -- sourcehut: + + + +[continuwuity]: https://forgejo.ellis.link/continuwuation/continuwuity From 57d26dae0d35f8be9e66054479261ca33a1ea42c Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 14 Apr 2025 22:58:30 +0100 Subject: [PATCH 1662/2091] docs: Remove hidden conduwuit badges --- README.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/README.md b/README.md index 89e1a299..f61f6a87 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,5 @@ # continuwuity - - - ## A community-driven [Matrix](https://matrix.org/) homeserver in Rust From 3e54c7e69163ebfcc24414dad4888b4d8a4380b8 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 14 Apr 2025 23:01:22 +0100 Subject: [PATCH 1663/2091] docs: Phrasing --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index f61f6a87..24a34d18 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,7 @@ It's a community continuation of the [conduwuit](https://github.com/girlbossceo/ The original conduwuit project has been archived and is no longer maintained. Rather than letting this Rust-based Matrix homeserver disappear, a group of community contributors have forked the project to continue its development, fix outstanding issues, and add new features. -Our aim is to provide a stable, well-maintained alternative for current conduwuit users and to welcome new users looking for a lightweight, efficient Matrix homeserver. - +We aim to provide a stable, well-maintained alternative for current Conduit users and welcome newcomers seeking a lightweight, efficient Matrix homeserver. ### Who are we? From 4f9e9174e2aaabcb45a81ec33ec9284159bc59fd Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 14 Apr 2025 23:06:39 +0100 Subject: [PATCH 1664/2091] docs: Mention future migration guide --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 24a34d18..deaed364 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,7 @@ We're working our way through all of the issues in the [Forgejo project](https:/ - Dendrite: No - Synapse: No -Although you can't migrate your data from other homeservers, it is perfectly acceptable to set up continuwuity on the same domain as a previous homeserver. - +We haven't written up a guide on migrating from incompatible homeservers yet. Reach out to us if you need to do this! From 35bffa5970e311a2b2ffa7c23ecd77121c36dfbe Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 15 Apr 2025 10:25:49 +0100 Subject: [PATCH 1665/2091] ci: Delete all old CI files Part of #753 --- .github/workflows/ci.yml | 717 ------------------- .github/workflows/docker-hub-description.yml | 41 -- .github/workflows/documentation.yml | 104 --- .github/workflows/release.yml | 118 --- .gitlab-ci.yml | 152 ---- .gitlab/merge_request_templates/MR.md | 8 - .gitlab/route-map.yml | 3 - 7 files changed, 1143 deletions(-) delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/docker-hub-description.yml delete mode 100644 .github/workflows/documentation.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .gitlab-ci.yml delete mode 100644 .gitlab/merge_request_templates/MR.md delete mode 100644 .gitlab/route-map.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 5043f23b..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,717 +0,0 @@ -name: CI and Artifacts - -on: - pull_request: - push: - paths-ignore: - - '.gitlab-ci.yml' - - '.gitignore' - - 'renovate.json' - - 'debian/**' - - 'docker/**' - branches: - - main - tags: - - '*' - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -concurrency: - group: ${{ github.head_ref || github.ref_name }} - cancel-in-progress: true - -env: - # Required to make some things output color - TERM: ansi - # Publishing to my nix binary cache - ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} - # conduwuit.cachix.org - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - # Just in case incremental is still being set to true, speeds up CI - CARGO_INCREMENTAL: 0 - # Custom nix binary cache if fork is being used - ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} - ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} - # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps - NIX_CONFIG: | - show-trace = true - extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org - extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= - experimental-features = nix-command flakes - extra-experimental-features = nix-command flakes - accept-flake-config = true - WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} - GH_REF_NAME: ${{ github.ref_name }} - WEBSERVER_DIR_NAME: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - -permissions: {} - -jobs: - tests: - name: Test - runs-on: self-hosted - steps: - - name: Setup SSH web publish - env: - web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' - run: | - mkdir -p -v ~/.ssh - - echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts - echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - cat >>~/.ssh/config <> "$GITHUB_ENV" - - - name: Sync repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Tag comparison check - if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} - run: | - # Tag mismatch with latest repo tag check to prevent potential downgrades - LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) - - if [ ${LATEST_TAG} != ${GH_REF_NAME} ]; then - echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' - echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - - name: Prepare build environment - run: | - echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - direnv allow - nix develop .#all-features --command true - - - name: Cache CI dependencies - run: | - bin/nix-build-and-cache ci - bin/nix-build-and-cache just '.#devShells.x86_64-linux.default' - bin/nix-build-and-cache just '.#devShells.x86_64-linux.all-features' - bin/nix-build-and-cache just '.#devShells.x86_64-linux.dynamic' - - # use rust-cache - - uses: Swatinem/rust-cache@v2 - # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting - # releases and tags - #if: ${{ !startsWith(github.ref, 'refs/tags/') }} - with: - cache-all-crates: "true" - cache-on-failure: "true" - cache-targets: "true" - - - name: Run CI tests - env: - CARGO_PROFILE: "test" - run: | - direnv exec . engage > >(tee -a test_output.log) - - - name: Run Complement tests - env: - CARGO_PROFILE: "test" - run: | - # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op - direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) - cp -v -f result complement_oci_image.tar.gz - - - name: Upload Complement OCI image - uses: actions/upload-artifact@v4 - with: - name: complement_oci_image.tar.gz - path: complement_oci_image.tar.gz - if-no-files-found: error - compression-level: 0 - - - name: Upload Complement logs - uses: actions/upload-artifact@v4 - with: - name: complement_test_logs.jsonl - path: complement_test_logs.jsonl - if-no-files-found: error - - - name: Upload Complement results - uses: actions/upload-artifact@v4 - with: - name: complement_test_results.jsonl - path: complement_test_results.jsonl - if-no-files-found: error - - - name: Diff Complement results with checked-in repo results - run: | - diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log) - - - name: Update Job Summary - env: - GH_JOB_STATUS: ${{ job.status }} - if: success() || failure() - run: | - if [ ${GH_JOB_STATUS} == 'success' ]; then - echo '# ✅ CI completed suwuccessfully' >> $GITHUB_STEP_SUMMARY - else - echo '# ❌ CI failed (last 100 lines of output)' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - tail -n 100 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - - echo '# Complement diff results (last 100 lines)' >> $GITHUB_STEP_SUMMARY - echo '```diff' >> $GITHUB_STEP_SUMMARY - tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - fi - - build: - name: Build - runs-on: self-hosted - strategy: - matrix: - include: - - target: aarch64-linux-musl - - target: x86_64-linux-musl - steps: - - name: Sync repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup SSH web publish - env: - web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' - run: | - mkdir -p -v ~/.ssh - - echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts - echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - cat >>~/.ssh/config <> "$GITHUB_ENV" - - - name: Prepare build environment - run: | - echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - direnv allow - nix develop .#all-features --command true --impure - - # use rust-cache - - uses: Swatinem/rust-cache@v2 - # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting - # releases and tags - #if: ${{ !startsWith(github.ref, 'refs/tags/') }} - with: - cache-all-crates: "true" - cache-on-failure: "true" - cache-targets: "true" - - - name: Build static ${{ matrix.target }}-all-features - run: | - if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] - then - CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" - elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] - then - CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" - fi - - SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - - bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features - - mkdir -v -p target/release/ - mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ - cp -v -f result/bin/conduwuit target/release/conduwuit - cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit - direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb - mv -v target/release/conduwuit static-${{ matrix.target }} - mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb - - - name: Build static x86_64-linux-musl-all-features-x86_64-haswell-optimised - if: ${{ matrix.target == 'x86_64-linux-musl' }} - run: | - CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" - SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - - bin/nix-build-and-cache just .#static-x86_64-linux-musl-all-features-x86_64-haswell-optimised - - mkdir -v -p target/release/ - mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ - cp -v -f result/bin/conduwuit target/release/conduwuit - cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit - direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb - mv -v target/release/conduwuit static-x86_64-linux-musl-x86_64-haswell-optimised - mv -v target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb x86_64-linux-musl-x86_64-haswell-optimised.deb - - # quick smoke test of the x86_64 static release binary - - name: Quick smoke test the x86_64 static release binary - if: ${{ matrix.target == 'x86_64-linux-musl' }} - run: | - # GH actions default runners are x86_64 only - if file result/bin/conduwuit | grep x86-64; then - result/bin/conduwuit --version - result/bin/conduwuit --help - result/bin/conduwuit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown" - fi - - - name: Build static debug ${{ matrix.target }}-all-features - run: | - if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] - then - CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" - elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] - then - CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" - fi - - SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - - bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug - - # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles. - # so we need to coerce cargo-deb into thinking this is a release binary - mkdir -v -p target/release/ - mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ - cp -v -f result/bin/conduwuit target/release/conduwuit - cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit - direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}-debug.deb - mv -v target/release/conduwuit static-${{ matrix.target }}-debug - mv -v target/release/${{ matrix.target }}-debug.deb ${{ matrix.target }}-debug.deb - - # quick smoke test of the x86_64 static debug binary - - name: Run x86_64 static debug binary - run: | - # GH actions default runners are x86_64 only - if file result/bin/conduwuit | grep x86-64; then - result/bin/conduwuit --version - fi - - # check validity of produced deb package, invalid debs will error on these commands - - name: Validate produced deb package - run: | - # List contents - dpkg-deb --contents ${{ matrix.target }}.deb - dpkg-deb --contents ${{ matrix.target }}-debug.deb - # List info - dpkg-deb --info ${{ matrix.target }}.deb - dpkg-deb --info ${{ matrix.target }}-debug.deb - - - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub - uses: actions/upload-artifact@v4 - if: ${{ matrix.target == 'x86_64-linux-musl' }} - with: - name: static-x86_64-linux-musl-x86_64-haswell-optimised - path: static-x86_64-linux-musl-x86_64-haswell-optimised - if-no-files-found: error - - - name: Upload static-${{ matrix.target }}-all-features to GitHub - uses: actions/upload-artifact@v4 - with: - name: static-${{ matrix.target }} - path: static-${{ matrix.target }} - if-no-files-found: error - - - name: Upload static deb ${{ matrix.target }}-all-features to GitHub - uses: actions/upload-artifact@v4 - with: - name: deb-${{ matrix.target }} - path: ${{ matrix.target }}.deb - if-no-files-found: error - compression-level: 0 - - - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver - if: ${{ matrix.target == 'x86_64-linux-musl' }} - run: | - if [ ! -z $SSH_WEBSITE ]; then - chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised - scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-x86_64-linux-musl-x86_64-haswell-optimised - fi - - - name: Upload static-${{ matrix.target }}-all-features to webserver - run: | - if [ ! -z $SSH_WEBSITE ]; then - chmod +x static-${{ matrix.target }} - scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }} - fi - - - name: Upload static deb x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver - if: ${{ matrix.target == 'x86_64-linux-musl' }} - run: | - if [ ! -z $SSH_WEBSITE ]; then - scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/x86_64-linux-musl-x86_64-haswell-optimised.deb - fi - - - name: Upload static deb ${{ matrix.target }}-all-features to webserver - run: | - if [ ! -z $SSH_WEBSITE ]; then - scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}.deb - fi - - - name: Upload static-${{ matrix.target }}-debug-all-features to GitHub - uses: actions/upload-artifact@v4 - with: - name: static-${{ matrix.target }}-debug - path: static-${{ matrix.target }}-debug - if-no-files-found: error - - - name: Upload static deb ${{ matrix.target }}-debug-all-features to GitHub - uses: actions/upload-artifact@v4 - with: - name: deb-${{ matrix.target }}-debug - path: ${{ matrix.target }}-debug.deb - if-no-files-found: error - compression-level: 0 - - - name: Upload static-${{ matrix.target }}-debug-all-features to webserver - run: | - if [ ! -z $SSH_WEBSITE ]; then - scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}-debug - fi - - - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver - run: | - if [ ! -z $SSH_WEBSITE ]; then - scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}-debug.deb - fi - - - name: Build OCI image ${{ matrix.target }}-all-features - run: | - bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features - - cp -v -f result oci-image-${{ matrix.target }}.tar.gz - - - name: Build OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised - if: ${{ matrix.target == 'x86_64-linux-musl' }} - run: | - bin/nix-build-and-cache just .#oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised - - cp -v -f result oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz - - - name: Build debug OCI image ${{ matrix.target }}-all-features - run: | - bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug - - cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz - - - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub - if: ${{ matrix.target == 'x86_64-linux-musl' }} - uses: actions/upload-artifact@v4 - with: - name: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised - path: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz - if-no-files-found: error - compression-level: 0 - - name: Upload OCI image ${{ matrix.target }}-all-features to GitHub - uses: actions/upload-artifact@v4 - with: - name: oci-image-${{ matrix.target }} - path: oci-image-${{ matrix.target }}.tar.gz - if-no-files-found: error - compression-level: 0 - - - name: Upload OCI image ${{ matrix.target }}-debug-all-features to GitHub - uses: actions/upload-artifact@v4 - with: - name: oci-image-${{ matrix.target }}-debug - path: oci-image-${{ matrix.target }}-debug.tar.gz - if-no-files-found: error - compression-level: 0 - - - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz to webserver - if: ${{ matrix.target == 'x86_64-linux-musl' }} - run: | - if [ ! -z $SSH_WEBSITE ]; then - scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz - fi - - - name: Upload OCI image ${{ matrix.target }}-all-features to webserver - run: | - if [ ! -z $SSH_WEBSITE ]; then - scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}.tar.gz - fi - - - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver - run: | - if [ ! -z $SSH_WEBSITE ]; then - scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}-debug.tar.gz - fi - - variables: - outputs: - github_repository: ${{ steps.var.outputs.github_repository }} - runs-on: self-hosted - steps: - - name: Setting global variables - uses: actions/github-script@v7 - id: var - with: - script: | - core.setOutput('github_repository', '${{ github.repository }}'.toLowerCase()) - docker: - name: Docker publish - runs-on: self-hosted - needs: [build, variables, tests] - permissions: - packages: write - contents: read - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' - env: - DOCKER_HUB_REPO: docker.io/${{ needs.variables.outputs.github_repository }} - GHCR_REPO: ghcr.io/${{ needs.variables.outputs.github_repository }} - GLCR_REPO: registry.gitlab.com/conduwuit/conduwuit - UNIQUE_TAG: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} - BRANCH_TAG: ${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} - - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} - GHCR_ENABLED: "${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}" - steps: - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Login to Docker Hub - if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} - uses: docker/login-action@v3 - with: - registry: docker.io - username: ${{ vars.DOCKER_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Login to GitLab Container Registry - if: ${{ (vars.GITLAB_USERNAME != '') && (env.GITLAB_TOKEN != '') }} - uses: docker/login-action@v3 - with: - registry: registry.gitlab.com - username: ${{ vars.GITLAB_USERNAME }} - password: ${{ secrets.GITLAB_TOKEN }} - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - pattern: "oci*" - - - name: Move OCI images into position - run: | - mv -v oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised/*.tar.gz oci-image-amd64-haswell-optimised.tar.gz - mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz - mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz - mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz - mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz - - - name: Load and push amd64 haswell image - run: | - docker load -i oci-image-amd64-haswell-optimised.tar.gz - if [ ! -z $DOCKERHUB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell - docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell - fi - if [ $GHCR_ENABLED = "true" ]; then - docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-haswell - docker push ${GHCR_REPO}:${UNIQUE_TAG}-haswell - fi - if [ ! -z $GITLAB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-haswell - docker push ${GLCR_REPO}:${UNIQUE_TAG}-haswell - fi - - - name: Load and push amd64 image - run: | - docker load -i oci-image-amd64.tar.gz - if [ ! -z $DOCKERHUB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 - docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 - fi - if [ $GHCR_ENABLED = "true" ]; then - docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64 - docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64 - fi - if [ ! -z $GITLAB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64 - docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64 - fi - - - name: Load and push arm64 image - run: | - docker load -i oci-image-arm64v8.tar.gz - if [ ! -z $DOCKERHUB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 - docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 - fi - if [ $GHCR_ENABLED = "true" ]; then - docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 - docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 - fi - if [ ! -z $GITLAB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 - docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 - fi - - - name: Load and push amd64 debug image - run: | - docker load -i oci-image-amd64-debug.tar.gz - if [ ! -z $DOCKERHUB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug - docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug - fi - if [ $GHCR_ENABLED = "true" ]; then - docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug - docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug - fi - if [ ! -z $GITLAB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug - docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug - fi - - - name: Load and push arm64 debug image - run: | - docker load -i oci-image-arm64v8-debug.tar.gz - if [ ! -z $DOCKERHUB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug - docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug - fi - if [ $GHCR_ENABLED = "true" ]; then - docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug - docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug - fi - if [ ! -z $GITLAB_TOKEN ]; then - docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug - docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug - fi - - - name: Create Docker haswell manifests - run: | - # Dockerhub Container Registry - if [ ! -z $DOCKERHUB_TOKEN ]; then - docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell - docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell - fi - # GitHub Container Registry - if [ $GHCR_ENABLED = "true" ]; then - docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell - docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell - fi - # GitLab Container Registry - if [ ! -z $GITLAB_TOKEN ]; then - docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell - docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell - fi - - - name: Create Docker combined manifests - run: | - # Dockerhub Container Registry - if [ ! -z $DOCKERHUB_TOKEN ]; then - docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 - docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 - fi - # GitHub Container Registry - if [ $GHCR_ENABLED = "true" ]; then - docker manifest create ${GHCR_REPO}:${UNIQUE_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 - docker manifest create ${GHCR_REPO}:${BRANCH_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 - fi - # GitLab Container Registry - if [ ! -z $GITLAB_TOKEN ]; then - docker manifest create ${GLCR_REPO}:${UNIQUE_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 - docker manifest create ${GLCR_REPO}:${BRANCH_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 - fi - - - name: Create Docker combined debug manifests - run: | - # Dockerhub Container Registry - if [ ! -z $DOCKERHUB_TOKEN ]; then - docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug - docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug - fi - # GitHub Container Registry - if [ $GHCR_ENABLED = "true" ]; then - docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug - docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug - fi - # GitLab Container Registry - if [ ! -z $GITLAB_TOKEN ]; then - docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug - docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug - fi - - - name: Push manifests to Docker registries - run: | - if [ ! -z $DOCKERHUB_TOKEN ]; then - docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG} - docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG} - docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug - docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug - docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell - docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell - fi - if [ $GHCR_ENABLED = "true" ]; then - docker manifest push ${GHCR_REPO}:${UNIQUE_TAG} - docker manifest push ${GHCR_REPO}:${BRANCH_TAG} - docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-debug - docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-debug - docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-haswell - docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-haswell - fi - if [ ! -z $GITLAB_TOKEN ]; then - docker manifest push ${GLCR_REPO}:${UNIQUE_TAG} - docker manifest push ${GLCR_REPO}:${BRANCH_TAG} - docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-debug - docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-debug - docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-haswell - docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-haswell - fi - - - name: Add Image Links to Job Summary - run: | - if [ ! -z $DOCKERHUB_TOKEN ]; then - echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY - fi - if [ $GHCR_ENABLED = "true" ]; then - echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY - fi - if [ ! -z $GITLAB_TOKEN ]; then - echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY - echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY - fi diff --git a/.github/workflows/docker-hub-description.yml b/.github/workflows/docker-hub-description.yml deleted file mode 100644 index b4f142db..00000000 --- a/.github/workflows/docker-hub-description.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Update Docker Hub Description - -on: - push: - branches: - - main - paths: - - README.md - - .github/workflows/docker-hub-description.yml - - workflow_dispatch: - -jobs: - dockerHubDescription: - runs-on: ubuntu-latest - if: ${{ (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' && (vars.DOCKER_USERNAME != '') }} - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setting variables - uses: actions/github-script@v7 - id: var - with: - script: | - const githubRepo = '${{ github.repository }}'.toLowerCase() - const repoId = githubRepo.split('/')[1] - - core.setOutput('github_repository', githubRepo) - const dockerRepo = '${{ vars.DOCKER_USERNAME }}'.toLowerCase() + '/' + repoId - core.setOutput('docker_repo', dockerRepo) - - - name: Docker Hub Description - uses: peter-evans/dockerhub-description@v4 - with: - username: ${{ vars.DOCKER_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - repository: ${{ steps.var.outputs.docker_repo }} - short-description: ${{ github.event.repository.description }} - enable-url-completion: true diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml deleted file mode 100644 index b5b4ff46..00000000 --- a/.github/workflows/documentation.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: Documentation and GitHub Pages - -on: - pull_request: - push: - branches: - - main - tags: - - '*' - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -env: - # Required to make some things output color - TERM: ansi - # Publishing to my nix binary cache - ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} - # conduwuit.cachix.org - CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} - # Custom nix binary cache if fork is being used - ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} - ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} - # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps - NIX_CONFIG: | - show-trace = true - extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org - extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= - experimental-features = nix-command flakes - extra-experimental-features = nix-command flakes - accept-flake-config = true - -# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. -# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. -concurrency: - group: "pages" - cancel-in-progress: false - -permissions: {} - -jobs: - docs: - name: Documentation and GitHub Pages - runs-on: self-hosted - - permissions: - pages: write - id-token: write - - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - steps: - - name: Sync repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup GitHub Pages - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') - uses: actions/configure-pages@v5 - - - name: Prepare build environment - run: | - echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - direnv allow - nix develop --command true - - - name: Cache CI dependencies - run: | - bin/nix-build-and-cache ci - - - name: Run lychee and markdownlint - run: | - direnv exec . engage just lints lychee - direnv exec . engage just lints markdownlint - - - name: Build documentation (book) - run: | - bin/nix-build-and-cache just .#book - - cp -r --dereference result public - chmod u+w -R public - - - name: Upload generated documentation (book) as normal artifact - uses: actions/upload-artifact@v4 - with: - name: public - path: public - if-no-files-found: error - # don't compress again - compression-level: 0 - - - name: Upload generated documentation (book) as GitHub Pages artifact - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') - uses: actions/upload-pages-artifact@v3 - with: - path: public - - - name: Deploy to GitHub Pages - if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') - id: deployment - uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index cfe72d2a..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,118 +0,0 @@ -name: Upload Release Assets - -on: - release: - types: [published] - workflow_dispatch: - inputs: - tag: - description: 'Tag to release' - required: true - type: string - action_id: - description: 'Action ID of the CI run' - required: true - type: string - -permissions: {} - -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: write - env: - GH_EVENT_NAME: ${{ github.event_name }} - GH_EVENT_INPUTS_ACTION_ID: ${{ github.event.inputs.action_id }} - GH_EVENT_INPUTS_TAG: ${{ github.event.inputs.tag }} - GH_REPOSITORY: ${{ github.repository }} - GH_SHA: ${{ github.sha }} - GH_TAG: ${{ github.event.release.tag_name }} - - steps: - - name: get latest ci id - id: get_ci_id - env: - GH_TOKEN: ${{ github.token }} - run: | - if [ "${GH_EVENT_NAME}" == "workflow_dispatch" ]; then - id="${GH_EVENT_INPUTS_ACTION_ID}" - tag="${GH_EVENT_INPUTS_TAG}" - else - # get all runs of the ci workflow - json=$(gh api "repos/${GH_REPOSITORY}/actions/workflows/ci.yml/runs") - - # find first run that is github sha and status is completed - id=$(echo "$json" | jq ".workflow_runs[] | select(.head_sha == \"${GH_SHA}\" and .status == \"completed\") | .id" | head -n 1) - - if [ ! "$id" ]; then - echo "No completed runs found" - echo "ci_id=0" >> "$GITHUB_OUTPUT" - exit 0 - fi - - tag="${GH_TAG}" - fi - - echo "ci_id=$id" >> "$GITHUB_OUTPUT" - echo "tag=$tag" >> "$GITHUB_OUTPUT" - - - name: get latest ci artifacts - if: steps.get_ci_id.outputs.ci_id != 0 - uses: actions/download-artifact@v4 - env: - GH_TOKEN: ${{ github.token }} - with: - merge-multiple: true - run-id: ${{ steps.get_ci_id.outputs.ci_id }} - github-token: ${{ github.token }} - - - run: | - ls - - - name: upload release assets - if: steps.get_ci_id.outputs.ci_id != 0 - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ steps.get_ci_id.outputs.tag }} - run: | - for file in $(find . -type f); do - case "$file" in - *json*) echo "Skipping $file...";; - *) echo "Uploading $file..."; gh release upload $TAG "$file" --clobber --repo="${GH_REPOSITORY}" || echo "Something went wrong, skipping.";; - esac - done - - - name: upload release assets to website - if: steps.get_ci_id.outputs.ci_id != 0 - env: - TAG: ${{ steps.get_ci_id.outputs.tag }} - run: | - mkdir -p -v ~/.ssh - - echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts - echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 - - chmod 600 ~/.ssh/id_ed25519 - - cat >>~/.ssh/config < /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi - - if command -v nix > /dev/null; then echo "extra-experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi - # Accept flake config from "untrusted" users - - if command -v nix > /dev/null; then echo "accept-flake-config = true" >> /etc/nix/nix.conf; fi - - # Add conduwuit binary cache - - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduwuit" >> /etc/nix/nix.conf; fi - - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE=" >> /etc/nix/nix.conf; fi - - - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduit" >> /etc/nix/nix.conf; fi - - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk=" >> /etc/nix/nix.conf; fi - - # Add alternate binary cache - - if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi - - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi - - # Add crane binary cache - - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi - - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi - - # Add nix-community binary cache - - if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi - - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi - - - if command -v nix > /dev/null; then echo "extra-substituters = https://aseipp-nix-cache.freetls.fastly.net" >> /etc/nix/nix.conf; fi - - # Install direnv and nix-direnv - - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv; fi - - # Allow .envrc - - if command -v nix > /dev/null; then direnv allow; fi - - # Set CARGO_HOME to a cacheable path - - export CARGO_HOME="$(git rev-parse --show-toplevel)/.gitlab-ci.d/cargo" - -ci: - stage: ci - image: nixos/nix:2.24.9 - script: - # Cache CI dependencies - - ./bin/nix-build-and-cache ci - - - direnv exec . engage - cache: - key: nix - paths: - - target - - .gitlab-ci.d - rules: - # CI on upstream runners (only available for maintainers) - - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $IS_UPSTREAM_CI == "true" - # Manual CI on unprotected branches that are not MRs - - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_REF_PROTECTED == "false" - when: manual - # Manual CI on forks - - if: $IS_UPSTREAM_CI != "true" - when: manual - - if: $CI - interruptible: true - -artifacts: - stage: artifacts - image: nixos/nix:2.24.9 - script: - - ./bin/nix-build-and-cache just .#static-x86_64-linux-musl - - cp result/bin/conduit x86_64-linux-musl - - - mkdir -p target/release - - cp result/bin/conduit target/release - - direnv exec . cargo deb --no-build --no-strip - - mv target/debian/*.deb x86_64-linux-musl.deb - - # Since the OCI image package is based on the binary package, this has the - # fun side effect of uploading the normal binary too. Conduit users who are - # deploying with Nix can leverage this fact by adding our binary cache to - # their systems. - # - # Note that although we have an `oci-image-x86_64-linux-musl` - # output, we don't build it because it would be largely redundant to this - # one since it's all containerized anyway. - - ./bin/nix-build-and-cache just .#oci-image - - cp result oci-image-amd64.tar.gz - - - ./bin/nix-build-and-cache just .#static-aarch64-linux-musl - - cp result/bin/conduit aarch64-linux-musl - - - ./bin/nix-build-and-cache just .#oci-image-aarch64-linux-musl - - cp result oci-image-arm64v8.tar.gz - - - ./bin/nix-build-and-cache just .#book - # We can't just copy the symlink, we need to dereference it https://gitlab.com/gitlab-org/gitlab/-/issues/19746 - - cp -r --dereference result public - artifacts: - paths: - - x86_64-linux-musl - - aarch64-linux-musl - - x86_64-linux-musl.deb - - oci-image-amd64.tar.gz - - oci-image-arm64v8.tar.gz - - public - rules: - # CI required for all MRs - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - # Optional CI on forks - - if: $IS_UPSTREAM_CI != "true" - when: manual - allow_failure: true - - if: $CI - interruptible: true - -pages: - stage: publish - dependencies: - - artifacts - only: - - next - script: - - "true" - artifacts: - paths: - - public diff --git a/.gitlab/merge_request_templates/MR.md b/.gitlab/merge_request_templates/MR.md deleted file mode 100644 index 4210554b..00000000 --- a/.gitlab/merge_request_templates/MR.md +++ /dev/null @@ -1,8 +0,0 @@ - - - ------------------------------------------------------------------------------ - -- [ ] I ran `cargo fmt`, `cargo clippy`, and `cargo test` -- [ ] I agree to release my code and all other changes of this MR under the Apache-2.0 license - diff --git a/.gitlab/route-map.yml b/.gitlab/route-map.yml deleted file mode 100644 index cf31bd18..00000000 --- a/.gitlab/route-map.yml +++ /dev/null @@ -1,3 +0,0 @@ -# Docs: Map markdown to html files -- source: /docs/(.+)\.md/ - public: '\1.html' From 57779df66a5fa6891894d7e96acac3a50b9dfecc Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 15 Apr 2025 12:05:04 +0100 Subject: [PATCH 1666/2091] chore: Add mailmap --- .mailmap | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .mailmap diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..1909b3f5 --- /dev/null +++ b/.mailmap @@ -0,0 +1,13 @@ +AlexPewMaster <68469103+AlexPewMaster@users.noreply.github.com> +Daniel Wiesenberg +Devin Ragotzy +Devin Ragotzy +Jonas Platte +Jonas Zohren +Jonathan de Jong +June Clementine Strawberry +June Clementine Strawberry +June Clementine Strawberry +Rudi Floren +Tamara Schmitz <15906939+tamara-schmitz@users.noreply.github.com> +x4u <14617923-x4u@users.noreply.gitlab.com> From 6e5392c2f50d110b40f724725843dd74513ad8f4 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 15 Apr 2025 12:15:09 +0100 Subject: [PATCH 1667/2091] =?UTF-8?q?chore:=20Add=20Timo=20K=C3=B6sters=20?= =?UTF-8?q?to=20the=20mailmap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index 1909b3f5..0cd8d7ec 100644 --- a/.mailmap +++ b/.mailmap @@ -10,4 +10,5 @@ June Clementine Strawberry June Clementine Strawberry Rudi Floren Tamara Schmitz <15906939+tamara-schmitz@users.noreply.github.com> +Timo Kösters x4u <14617923-x4u@users.noreply.gitlab.com> From 17a04940fca6ab39731c5f8815d5029bbf30762d Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 15 Apr 2025 21:58:39 +0100 Subject: [PATCH 1668/2091] chore: Update Olivia Lee in mailmap --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index 0cd8d7ec..fa267e13 100644 --- a/.mailmap +++ b/.mailmap @@ -8,6 +8,7 @@ Jonathan de Jong June Clementine Strawberry June Clementine Strawberry June Clementine Strawberry +Olivia Lee Rudi Floren Tamara Schmitz <15906939+tamara-schmitz@users.noreply.github.com> Timo Kösters From dad407fb22a991238926ab65b685ec55d2272ab9 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 15 Apr 2025 12:09:14 +0100 Subject: [PATCH 1669/2091] chore: Add words to cspell dictionary --- .vscode/settings.json | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..a4fad964 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,11 @@ +{ + "cSpell.words": [ + "Forgejo", + "appservice", + "appservices", + "conduwuit", + "continuwuity", + "homeserver", + "homeservers" + ] +} From f76f669d163778a1ec768773da5cdbefc78539ec Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 15 Apr 2025 23:26:25 +0100 Subject: [PATCH 1670/2091] chore: Remove the default sentry endpoint --- conduwuit-example.toml | 2 +- src/core/config/mod.rs | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 118bc57d..af8da6bb 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1428,7 +1428,7 @@ # Sentry reporting URL, if a custom one is desired. # -#sentry_endpoint = "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" +#sentry_endpoint = "" # Report your conduwuit server_name in Sentry.io crash reports and # metrics. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 0ca6bbaf..a7205423 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1636,7 +1636,7 @@ pub struct Config { /// Sentry reporting URL, if a custom one is desired. /// /// display: sensitive - /// default: "https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536" + /// default: "" #[serde(default = "default_sentry_endpoint")] pub sentry_endpoint: Option, @@ -2207,9 +2207,7 @@ fn default_url_preview_max_spider_size() -> usize { fn default_new_user_displayname_suffix() -> String { "🏳️‍⚧️".to_owned() } -fn default_sentry_endpoint() -> Option { - Url::parse("https://fe2eb4536aa04949e28eff3128d64757@o4506996327251968.ingest.us.sentry.io/4506996334657536").ok() -} +fn default_sentry_endpoint() -> Option { None } fn default_sentry_traces_sample_rate() -> f32 { 0.15 } From 90880e268966718c39b9970b0d564b0bad7f823d Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Wed, 16 Apr 2025 02:56:22 +0100 Subject: [PATCH 1671/2091] Update mdBook config for continuwuity --- book.toml | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/book.toml b/book.toml index 7eb1983b..46d3a7b0 100644 --- a/book.toml +++ b/book.toml @@ -1,8 +1,8 @@ [book] -title = "conduwuit 🏳️‍⚧️ 💜 🦴" -description = "conduwuit, which is a well-maintained fork of Conduit, is a simple, fast and reliable chat server for the Matrix protocol" +title = "continuwuity" +description = "continuwuity is a community continuation of the conduwuit Matrix homeserver, written in Rust." language = "en" -authors = ["strawberry (June)"] +authors = ["The continuwuity Community"] text-direction = "ltr" multilingual = false src = "docs" @@ -16,12 +16,9 @@ extra-watch-dirs = ["debian", "docs"] edition = "2024" [output.html] -git-repository-url = "https://github.com/girlbossceo/conduwuit" -edit-url-template = "https://github.com/girlbossceo/conduwuit/edit/main/{path}" -git-repository-icon = "fa-github-square" - -[output.html.redirect] -"/differences.html" = "https://conduwuit.puppyirl.gay/#where-is-the-differences-page" +edit-url-template = "https://forgejo.ellis.link/continuwuation/continuwuity/src/branch/main/{path}" +git-repository-url = "https://forgejo.ellis.link/continuwuation/continuwuity" +git-repository-icon = "fa-git-alt" [output.html.search] limit-results = 15 From 538347204fab75e9965ee4a60f00690f8a354ddf Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Wed, 16 Apr 2025 02:56:47 +0100 Subject: [PATCH 1672/2091] Add Matrix .well-known files --- docs/static/_headers | 3 +++ docs/static/client | 1 + docs/static/server | 1 + 3 files changed, 5 insertions(+) create mode 100644 docs/static/_headers create mode 100644 docs/static/client create mode 100644 docs/static/server diff --git a/docs/static/_headers b/docs/static/_headers new file mode 100644 index 00000000..5e960241 --- /dev/null +++ b/docs/static/_headers @@ -0,0 +1,3 @@ +/.well-known/matrix/* + Access-Control-Allow-Origin: * + Content-Type: application/json diff --git a/docs/static/client b/docs/static/client new file mode 100644 index 00000000..c2b70a14 --- /dev/null +++ b/docs/static/client @@ -0,0 +1 @@ +{"m.homeserver":{"base_url": "https://matrix.continuwuity.org"},"org.matrix.msc3575.proxy":{"url": "https://matrix.continuwuity.org"}} diff --git a/docs/static/server b/docs/static/server new file mode 100644 index 00000000..a3099f6e --- /dev/null +++ b/docs/static/server @@ -0,0 +1 @@ +{"m.server":"matrix.continuwuity.org:443"} From b91af70e0b909ce7486bb88768d65f573bcfd9e8 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Wed, 16 Apr 2025 15:47:12 +0100 Subject: [PATCH 1673/2091] Add Forgejo CI workflow for Cloudflare Pages --- .forgejo/workflows/documentation.yml | 68 ++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 .forgejo/workflows/documentation.yml diff --git a/.forgejo/workflows/documentation.yml b/.forgejo/workflows/documentation.yml new file mode 100644 index 00000000..c08c1abb --- /dev/null +++ b/.forgejo/workflows/documentation.yml @@ -0,0 +1,68 @@ +name: Documentation + +on: + pull_request: + push: + branches: + - main + tags: + - "v*" + workflow_dispatch: + +concurrency: + group: "pages-${{ github.ref }}" + cancel-in-progress: true + +jobs: + docs: + name: Build and Deploy Documentation + runs-on: not-nexy + + steps: + - name: Sync repository + uses: https://github.com/actions/checkout@v4 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Setup mdBook + uses: https://github.com/peaceiris/actions-mdbook@v2 + with: + mdbook-version: "latest" + + - name: Build mdbook + run: mdbook build + + - name: Prepare static files for deployment + run: | + mkdir -p ./public/.well-known/matrix + # Copy the Matrix .well-known files + cp ./docs/static/server ./public/.well-known/matrix/server + cp ./docs/static/client ./public/.well-known/matrix/client + # Copy the custom headers file + cp ./docs/static/_headers ./public/_headers + echo "Copied .well-known files and _headers to ./public" + + - name: Setup Node.js + uses: https://github.com/actions/setup-node@v4 + with: + node-version: 20 + + - name: Install dependencies + run: npm install --save-dev wrangler@latest + + - name: Deploy to Cloudflare Pages (Production) + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + uses: https://github.com/cloudflare/wrangler-action@v3 + with: + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + command: pages deploy ./public --branch=main --commit-dirty=true --project-name=${{ vars.CLOUDFLARE_PROJECT_NAME }}" + + - name: Deploy to Cloudflare Pages (Preview) + if: ${{ github.event_name != 'push' || github.ref != 'refs/heads/main' }} + uses: https://github.com/cloudflare/wrangler-action@v3 + with: + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + command: pages deploy ./public --branch=${{ github.head_ref }} --commit-dirty=true --project-name=${{ vars.CLOUDFLARE_PROJECT_NAME }}" From 773c3d457b214d60376780b0baaa4031e4352685 Mon Sep 17 00:00:00 2001 From: Jacob Taylor Date: Thu, 17 Apr 2025 07:48:54 -0700 Subject: [PATCH 1674/2091] fix space hierarchy pagination not respecting client-specified limit. --- src/api/client/space.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/space.rs b/src/api/client/space.rs index 4eee9d76..92768926 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -179,7 +179,7 @@ where (next_short_room_ids.iter().ne(short_room_ids) && !next_short_room_ids.is_empty()) .then_some(PaginationToken { short_room_ids: next_short_room_ids, - limit: max_depth.try_into().ok()?, + limit: limit.try_into().ok()?, max_depth: max_depth.try_into().ok()?, suggested_only, }) From 68d68a0645d52e4e5fe5a0c27d1c1b31f87f35aa Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 15 Apr 2025 17:35:33 +0100 Subject: [PATCH 1675/2091] fix: Do not panic when sender_device is None in `/messages` route The device ID is not always present when the appservice is the client. This was causing 500 errors for some users, as appservices can lazy load from `/messages`. Fixes #738 Co-authored-by: Jade Ellis --- src/api/client/message.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index db11ef4a..f85611ca 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -21,7 +21,7 @@ use conduwuit_service::{ }; use futures::{FutureExt, StreamExt, TryFutureExt, future::OptionFuture, pin_mut}; use ruma::{ - RoomId, UserId, + DeviceId, RoomId, UserId, api::{ Direction, client::{filter::RoomEventFilter, message::get_message_events}, @@ -67,8 +67,8 @@ pub(crate) async fn get_message_events_route( body: Ruma, ) -> Result { debug_assert!(IGNORED_MESSAGE_TYPES.is_sorted(), "IGNORED_MESSAGE_TYPES is not sorted"); - let sender = body.sender(); - let (sender_user, sender_device) = sender; + let sender_user = body.sender_user(); + let sender_device = body.sender_device.as_ref(); let room_id = &body.room_id; let filter = &body.filter; @@ -132,7 +132,7 @@ pub(crate) async fn get_message_events_route( let lazy_loading_context = lazy_loading::Context { user_id: sender_user, - device_id: sender_device, + device_id: sender_device.map_or_else(|| <&DeviceId>::from(""), AsRef::as_ref), room_id, token: Some(from.into_unsigned()), options: Some(&filter.lazy_load_options), From 3a95585f0edb529154a5a8e3f181f4b5e929b698 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 16 Apr 2025 13:47:35 +0100 Subject: [PATCH 1676/2091] fix: Disambiguate appservices in lazy loading context In the previous commit, app services would all appear to be the same device when accessing the same user. This sets the device ID to be the appservice ID when available to avoid possible clobbering. --- src/api/client/message.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index f85611ca..9c2c4057 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -26,7 +26,10 @@ use ruma::{ Direction, client::{filter::RoomEventFilter, message::get_message_events}, }, - events::{AnyStateEvent, StateEventType, TimelineEventType, TimelineEventType::*}, + events::{ + AnyStateEvent, StateEventType, + TimelineEventType::{self, *}, + }, serde::Raw, }; @@ -129,10 +132,20 @@ pub(crate) async fn get_message_events_route( .take(limit) .collect() .await; + // let appservice_id = body.appservice_info.map(|appservice| + // appservice.registration.id); let lazy_loading_context = lazy_loading::Context { user_id: sender_user, - device_id: sender_device.map_or_else(|| <&DeviceId>::from(""), AsRef::as_ref), + device_id: match sender_device { + | Some(device_id) => device_id, + | None => + if let Some(registration) = body.appservice_info.as_ref() { + <&DeviceId>::from(registration.registration.id.as_str()) + } else { + <&DeviceId>::from("") + }, + }, room_id, token: Some(from.into_unsigned()), options: Some(&filter.lazy_load_options), From dc599db19c48ad3cbae15fc419c4a531d217ed05 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 16 Apr 2025 13:52:28 +0100 Subject: [PATCH 1677/2091] chore: Change branding string to continuwuity --- src/core/info/version.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/info/version.rs b/src/core/info/version.rs index 37580210..6abb6e13 100644 --- a/src/core/info/version.rs +++ b/src/core/info/version.rs @@ -7,7 +7,7 @@ use std::sync::OnceLock; -static BRANDING: &str = "conduwuit"; +static BRANDING: &str = "continuwuity"; static SEMANTIC: &str = env!("CARGO_PKG_VERSION"); static VERSION: OnceLock = OnceLock::new(); From 6b92e965824924ea7f78399758cb4c4a1057e2fb Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 16 Apr 2025 18:54:36 +0100 Subject: [PATCH 1678/2091] feat: Docker images built with Forgejo Actions --- .dockerignore | 4 +- .forgejo/workflows/release-image.yml | 223 +++++++++++++++++++++++++++ docker/Dockerfile | 216 ++++++++++++++++++++++++++ 3 files changed, 441 insertions(+), 2 deletions(-) create mode 100644 .forgejo/workflows/release-image.yml create mode 100644 docker/Dockerfile diff --git a/.dockerignore b/.dockerignore index 35d35e1b..453634df 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,9 @@ # Local build and dev artifacts -target -tests +target/ # Docker files Dockerfile* +docker/ # IDE files .vscode diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml new file mode 100644 index 00000000..3a6c93eb --- /dev/null +++ b/.forgejo/workflows/release-image.yml @@ -0,0 +1,223 @@ +name: Release Docker Image + +on: + pull_request: + push: + paths-ignore: + - '.gitlab-ci.yml' + - '.gitignore' + - 'renovate.json' + - 'debian/**' + - 'docker/**' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + BUILTIN_REGISTRY: forgejo.ellis.link + BUILTIN_REGISTRY_ENABLED: "${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}" + +jobs: + define-variables: + runs-on: ubuntu-latest + + outputs: + images: ${{ steps.var.outputs.images }} + images_list: ${{ steps.var.outputs.images_list }} + build_matrix: ${{ steps.var.outputs.build_matrix }} + + steps: + - name: Setting variables + uses: https://github.com/actions/github-script@v7 + id: var + with: + script: | + const githubRepo = '${{ github.repository }}'.toLowerCase() + const repoId = githubRepo.split('/')[1] + + core.setOutput('github_repository', githubRepo) + const builtinImage = '${{ env.BUILTIN_REGISTRY }}/' + githubRepo + let images = [] + if (process.env.BUILTIN_REGISTRY_ENABLED === "true") { + images.push(builtinImage) + } + core.setOutput('images', images.join("\n")) + core.setOutput('images_list', images.join(",")) + const platforms = ['linux/amd64', 'linux/arm64'] + core.setOutput('build_matrix', JSON.stringify({ + platform: platforms, + include: platforms.map(platform => { return { + platform, + slug: platform.replace('/', '-') + }}) + })) + + build-image: + runs-on: not-nexy + container: ghcr.io/catthehacker/ubuntu:act-latest + needs: define-variables + permissions: + contents: read + packages: write + attestations: write + id-token: write + strategy: + matrix: ${{ fromJSON(needs.define-variables.outputs.build_matrix) }} + steps: + - name: Echo strategy + run: echo '${{ toJSON(fromJSON(needs.define-variables.outputs.build_matrix)) }}' + - name: Echo matrix + run: echo '${{ toJSON(matrix) }}' + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Login to builtin registry + uses: docker/login-action@v3 + with: + registry: ${{ env.BUILTIN_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Extract metadata (labels, annotations) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{needs.define-variables.outputs.images}} + # default labels & annotations: https://github.com/docker/metadata-action/blob/master/src/meta.ts#L509 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + + # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. + # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. + # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. + # It will not push images generated from a pull request + - name: Get short git commit SHA + id: sha + run: | + calculatedSha=$(git rev-parse --short ${{ github.sha }}) + echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV + - name: Get Git commit timestamps + run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV + - uses: https://github.com/Swatinem/rust-cache@v2 + with: + prefix-key: v0-rust-linux/${{ matrix.slug }} + id: rust-cache + - name: Inject cache into Docker + uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.1.2 + with: + cache-map: | + { + "/home/runner/.cargo/registry": "/usr/local/cargo/registry", + "/home/runner/.cargo/git/db": "/usr/local/cargo/git/db", + "./target": "/app/target", + "./timelord": "/timelord" + } + - name: Cache timelord state + uses: actions/cache@v4 + with: + path: "./timelord" + key: ${{ runner.os }}-${{ matrix.slug }} + - name: Build and push Docker image by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: "docker/Dockerfile" + build-args: | + CONDUWUIT_VERSION_EXTRA=${{ env.COMMIT_SHORT_SHA }} + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + cache-from: type=gha + cache-to: type=gha,mode=max + sbom: true + outputs: type=image,"name=${{ needs.define-variables.outputs.images_list }}",push-by-digest=true,name-canonical=true,push=true + env: + SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} + + # For publishing multi-platform manifests + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.slug }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: not-nexy + container: ghcr.io/catthehacker/ubuntu:act-latest + needs: [define-variables, build-image] + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + - name: Login to builtin registry + uses: docker/login-action@v3 + with: + registry: ${{ env.BUILTIN_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata (tags) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + tags: | + type=semver,pattern=v{{version}} + type=semver,pattern=v{{major}}.{{minor}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.0.') }} + type=semver,pattern=v{{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} + type=ref,event=branch + type=ref,event=pr + type=sha,format=long + images: ${{needs.define-variables.outputs.images}} + # default labels & annotations: https://github.com/docker/metadata-action/blob/master/src/meta.ts#L509 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: index + + - name: Create manifest list and push + working-directory: /tmp/digests + env: + IMAGES: ${{needs.define-variables.outputs.images}} + shell: bash + run: | + IFS=$'\n' + IMAGES_LIST=($IMAGES) + ANNOTATIONS_LIST=($DOCKER_METADATA_OUTPUT_ANNOTATIONS) + TAGS_LIST=($DOCKER_METADATA_OUTPUT_TAGS) + for REPO in "${IMAGES_LIST[@]}"; do + docker buildx imagetools create \ + $(for tag in "${TAGS_LIST[@]}"; do echo "--tag"; echo "$tag"; done) \ + $(for annotation in "${ANNOTATIONS_LIST[@]}"; do echo "--annotation"; echo "$annotation"; done) \ + $(for reference in *; do printf "$REPO@sha256:%s\n" $reference; done) + done + + - name: Inspect image + env: + IMAGES: ${{needs.define-variables.outputs.images}} + shell: bash + run: | + IMAGES_LIST=($IMAGES) + for REPO in "${IMAGES_LIST[@]}"; do + docker buildx imagetools inspect $REPO:${{ steps.meta.outputs.version }} + done diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..10f54d94 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,216 @@ +ARG RUST_VERSION=1 + +FROM --platform=$BUILDPLATFORM docker.io/tonistiigi/xx AS xx +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS base +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS toolchain + +# Prevent deletion of apt cache +RUN rm -f /etc/apt/apt.conf.d/docker-clean + +# Match Rustc version as close as possible +# rustc -vV +ARG LLVM_VERSION=19 +# ENV RUSTUP_TOOLCHAIN=${RUST_VERSION} + +# Install repo tools +# Line one: compiler tools +# Line two: curl, for downloading binaries +# Line three: for xx-verify +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ +apt-get update && apt-get install -y \ + clang-${LLVM_VERSION} lld-${LLVM_VERSION} pkg-config make jq \ + curl git \ + file + +# Create symlinks for LLVM tools +RUN <> /etc/environment + +# Configure pkg-config +RUN <> /etc/environment + echo "PKG_CONFIG=/usr/bin/$(xx-info)-pkg-config" >> /etc/environment + echo "PKG_CONFIG_ALLOW_CROSS=true" >> /etc/environment +EOF + +# Configure cc to use clang version +RUN <> /etc/environment + echo "CXX=clang++" >> /etc/environment +EOF + +# Cross-language LTO +RUN <> /etc/environment + echo "CXXFLAGS=-flto" >> /etc/environment + # Linker is set to target-compatible clang by xx + echo "RUSTFLAGS='-Clinker-plugin-lto -Clink-arg=-fuse-ld=lld'" >> /etc/environment +EOF + +# Apply CPU-specific optimizations if TARGET_CPU is provided +ARG TARGET_CPU= +RUN <> /etc/environment + echo "CXXFLAGS='${CXXFLAGS} -march=${TARGET_CPU}'" >> /etc/environment + echo "RUSTFLAGS='${RUSTFLAGS} -C target-cpu=${TARGET_CPU}'" >> /etc/environment + fi +EOF + +# Prepare output directories +RUN mkdir /out + +FROM toolchain AS builder + +# Conduwuit version info +ARG COMMIT_SHA= +ARG CONDUWUIT_VERSION_EXTRA= +ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA +RUN <> /etc/environment +fi +EOF + +ARG TARGETPLATFORM + +# Verify environment configuration +RUN cat /etc/environment +RUN xx-cargo --print-target-triple + +# Get source +COPY . . + +# Timelord sync +RUN --mount=type=cache,target=/timelord/ \ + timelord sync --source-dir . --cache-dir /timelord/ + +# Build the binary +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/app/target \ + bash <<'EOF' + set -o allexport + . /etc/environment + TARGET_DIR=($(cargo metadata --no-deps --format-version 1 | \ + jq -r ".target_directory")) + mkdir /out/sbin + PACKAGE=conduwuit + xx-cargo build --locked --release \ + -p $PACKAGE; + BINARIES=($(cargo metadata --no-deps --format-version 1 | \ + jq -r ".packages[] | select(.name == \"$PACKAGE\") | .targets[] | select( .kind | map(. == \"bin\") | any ) | .name")) + for BINARY in "${BINARIES[@]}"; do + echo $BINARY + xx-verify $TARGET_DIR/$(xx-cargo --print-target-triple)/release/$BINARY + cp $TARGET_DIR/$(xx-cargo --print-target-triple)/release/$BINARY /out/sbin/$BINARY + done +EOF + +# Generate Software Bill of Materials (SBOM) +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + bash <<'EOF' + mkdir /out/sbom + typeset -A PACKAGES + for BINARY in /out/sbin/*; do + BINARY_BASE=$(basename ${BINARY}) + package=$(cargo metadata --no-deps --format-version 1 | jq -r ".packages[] | select(.targets[] | select( .kind | map(. == \"bin\") | any ) | .name == \"$BINARY_BASE\") | .name") + if [ -z "$package" ]; then + continue + fi + PACKAGES[$package]=1 + done + for PACKAGE in $(echo ${!PACKAGES[@]}); do + echo $PACKAGE + cargo sbom --cargo-package $PACKAGE > /out/sbom/$PACKAGE.spdx.json + done +EOF + +# Extract dynamically linked dependencies +RUN < Date: Thu, 17 Apr 2025 10:33:19 +0100 Subject: [PATCH 1679/2091] build: Use hacks for a cached actions build - Use cache dance for github actions caching - Use timelord hack to avoid bad cache invalidation --- .forgejo/workflows/release-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 3a6c93eb..fddb493c 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -107,7 +107,7 @@ jobs: run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - uses: https://github.com/Swatinem/rust-cache@v2 with: - prefix-key: v0-rust-linux/${{ matrix.slug }} + prefix-key: v0-rust-linux/${{ matrix.platform.slug }} id: rust-cache - name: Inject cache into Docker uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.1.2 @@ -123,7 +123,7 @@ jobs: uses: actions/cache@v4 with: path: "./timelord" - key: ${{ runner.os }}-${{ matrix.slug }} + key: ${{ runner.os }}-${{ matrix.platform.slug }} - name: Build and push Docker image by digest id: build uses: docker/build-push-action@v6 From 70cee36041913c2d077ff427e89a341d5e5ac2e1 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 17 Apr 2025 12:02:59 +0100 Subject: [PATCH 1680/2091] fix: Allow specifying user & password for builtin registry --- .forgejo/workflows/release-image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index fddb493c..65dfb43c 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -81,8 +81,8 @@ jobs: uses: docker/login-action@v3 with: registry: ${{ env.BUILTIN_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }} + password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }} # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - name: Extract metadata (labels, annotations) for Docker @@ -173,8 +173,8 @@ jobs: uses: docker/login-action@v3 with: registry: ${{ env.BUILTIN_REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + username: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }} + password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 From 3ced2e2f9064200ffdcad18c49866bc96cf5c2a1 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 17 Apr 2025 12:06:45 +0100 Subject: [PATCH 1681/2091] fix: Use forgejo patched artifact actions --- .forgejo/workflows/release-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 65dfb43c..8a178d36 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -150,7 +150,7 @@ jobs: touch "/tmp/digests/${digest#sha256:}" - name: Upload digest - uses: actions/upload-artifact@v4 + uses: forgejo/upload-artifact@v4 with: name: digests-${{ matrix.slug }} path: /tmp/digests/* @@ -163,7 +163,7 @@ jobs: needs: [define-variables, build-image] steps: - name: Download digests - uses: actions/download-artifact@v4 + uses: forgejo/download-artifact@v4 with: path: /tmp/digests pattern: digests-* From 0ac1ce9996952b96a865c36a6cbda85df0db60da Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 17 Apr 2025 12:13:19 +0100 Subject: [PATCH 1682/2091] fix: Hardcode matrix --- .forgejo/workflows/release-image.yml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 8a178d36..9fc50441 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -61,7 +61,22 @@ jobs: attestations: write id-token: write strategy: - matrix: ${{ fromJSON(needs.define-variables.outputs.build_matrix) }} + matrix: { + "include": [ + { + "platform": "linux/amd64", + "slug": "linux-amd64" + }, + { + "platform": "linux/arm64", + "slug": "linux-arm64" + } + ], + "platform": [ + "linux/amd64", + "linux/arm64" + ] + } steps: - name: Echo strategy run: echo '${{ toJSON(fromJSON(needs.define-variables.outputs.build_matrix)) }}' From 93253237e938984cdaf7a947f8b3a3d4c54c63e4 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 17 Apr 2025 12:44:18 +0100 Subject: [PATCH 1683/2091] ci: Prefix branch builds with branch- --- .forgejo/workflows/release-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 9fc50441..3970ff9d 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -202,7 +202,7 @@ jobs: type=semver,pattern=v{{version}} type=semver,pattern=v{{major}}.{{minor}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.0.') }} type=semver,pattern=v{{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} - type=ref,event=branch + type=ref,event=branch,prefix=branch- type=ref,event=pr type=sha,format=long images: ${{needs.define-variables.outputs.images}} From 10947f6f1a268b03f7a35b30e82679b98e7b9337 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 17 Apr 2025 12:51:18 +0100 Subject: [PATCH 1684/2091] fix: Replace rust cache with direct cache use, as Rust is not installed on CI image --- .forgejo/workflows/release-image.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 3970ff9d..7c72082a 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -120,10 +120,20 @@ jobs: echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV - name: Get Git commit timestamps run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - - uses: https://github.com/Swatinem/rust-cache@v2 + - name: Rust cache + uses: actions/cache@v4 with: - prefix-key: v0-rust-linux/${{ matrix.platform.slug }} + key: v0-rust-linux/${{ runner.os }}-${{ matrix.slug }} + path: | + ~/.cargo/registry + ~/.cargo/git/db + ./target id: rust-cache + - name: Cache timelord state + uses: actions/cache@v4 + with: + path: "./timelord" + key: ${{ runner.os }}-${{ matrix.slug }} - name: Inject cache into Docker uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.1.2 with: @@ -134,11 +144,6 @@ jobs: "./target": "/app/target", "./timelord": "/timelord" } - - name: Cache timelord state - uses: actions/cache@v4 - with: - path: "./timelord" - key: ${{ runner.os }}-${{ matrix.platform.slug }} - name: Build and push Docker image by digest id: build uses: docker/build-push-action@v6 From fb793e8315bd0acb222b1ac53aef07f247afff38 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 17 Apr 2025 14:35:16 +0100 Subject: [PATCH 1685/2091] ci: Limit concurrency Mainly to prevent runners from getting bogged down --- .forgejo/workflows/release-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 7c72082a..0eaf945a 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -1,4 +1,6 @@ name: Release Docker Image +concurrency: + group: "release-image-${{ github.ref }}" on: pull_request: From 71d2421f55a97aee5b2b31dc1889f3f688d5315c Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 17 Apr 2025 15:44:56 +0100 Subject: [PATCH 1686/2091] ci: Only prefix non-default branches AKA, tag image:main as the latest commit --- .forgejo/workflows/release-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 0eaf945a..f03341a7 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -209,7 +209,7 @@ jobs: type=semver,pattern=v{{version}} type=semver,pattern=v{{major}}.{{minor}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.0.') }} type=semver,pattern=v{{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} - type=ref,event=branch,prefix=branch- + type=ref,event=branch,prefix=${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref && '' || 'branch-' }} type=ref,event=pr type=sha,format=long images: ${{needs.define-variables.outputs.images}} From d85aaabe9ee239e22bff1be5decbaeac51880ec1 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 17 Apr 2025 15:59:40 +0100 Subject: [PATCH 1687/2091] fix: Disable buildkit caching This is for tom's runners, whilst they're having network issues --- .forgejo/workflows/release-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index f03341a7..194fa93a 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -157,8 +157,8 @@ jobs: platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} - cache-from: type=gha - cache-to: type=gha,mode=max + # cache-from: type=gha + # cache-to: type=gha,mode=max sbom: true outputs: type=image,"name=${{ needs.define-variables.outputs.images_list }}",push-by-digest=true,name-canonical=true,push=true env: From 9e0530839d54f8ae8e58e8ff01c57e0deca72af8 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 18 Apr 2025 14:08:21 +0100 Subject: [PATCH 1688/2091] ci: Remove non-functional cache steps --- .forgejo/workflows/release-image.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 194fa93a..6bde932d 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -122,30 +122,6 @@ jobs: echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV - name: Get Git commit timestamps run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - - name: Rust cache - uses: actions/cache@v4 - with: - key: v0-rust-linux/${{ runner.os }}-${{ matrix.slug }} - path: | - ~/.cargo/registry - ~/.cargo/git/db - ./target - id: rust-cache - - name: Cache timelord state - uses: actions/cache@v4 - with: - path: "./timelord" - key: ${{ runner.os }}-${{ matrix.slug }} - - name: Inject cache into Docker - uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.1.2 - with: - cache-map: | - { - "/home/runner/.cargo/registry": "/usr/local/cargo/registry", - "/home/runner/.cargo/git/db": "/usr/local/cargo/git/db", - "./target": "/app/target", - "./timelord": "/timelord" - } - name: Build and push Docker image by digest id: build uses: docker/build-push-action@v6 From b16e26952af74da3cdcbfbb72ec6786a55a37133 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 18 Apr 2025 14:09:09 +0100 Subject: [PATCH 1689/2091] ci: Use dind label --- .forgejo/workflows/release-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 6bde932d..142529ae 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -54,7 +54,7 @@ jobs: })) build-image: - runs-on: not-nexy + runs-on: dind container: ghcr.io/catthehacker/ubuntu:act-latest needs: define-variables permissions: @@ -156,7 +156,7 @@ jobs: retention-days: 1 merge: - runs-on: not-nexy + runs-on: dind container: ghcr.io/catthehacker/ubuntu:act-latest needs: [define-variables, build-image] steps: From 2e6ec2f89cd218f89489d7ca86997bce94d25064 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 18 Apr 2025 17:38:42 +0100 Subject: [PATCH 1690/2091] chore: Update git links --- Cargo.lock | 54 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 42 +++++++++++++++++++++--------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d81fdbc0..def41f68 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,7 +118,7 @@ checksum = "5f093eed78becd229346bf859eec0aa4dd7ddde0757287b2b4107a1f09c80002" [[package]] name = "async-channel" version = "2.3.1" -source = "git+https://github.com/girlbossceo/async-channel?rev=92e5e74063bf2a3b10414bcc8a0d68b235644280#92e5e74063bf2a3b10414bcc8a0d68b235644280" +source = "git+https://forgejo.ellis.link/continuwuation/async-channel?rev=92e5e74063bf2a3b10414bcc8a0d68b235644280#92e5e74063bf2a3b10414bcc8a0d68b235644280" dependencies = [ "concurrent-queue", "event-listener-strategy", @@ -1047,7 +1047,7 @@ checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core_affinity" version = "0.8.1" -source = "git+https://github.com/girlbossceo/core_affinity_rs?rev=9c8e51510c35077df888ee72a36b4b05637147da#9c8e51510c35077df888ee72a36b4b05637147da" +source = "git+https://forgejo.ellis.link/continuwuation/core_affinity_rs?rev=9c8e51510c35077df888ee72a36b4b05637147da#9c8e51510c35077df888ee72a36b4b05637147da" dependencies = [ "libc", "num_cpus", @@ -1379,7 +1379,7 @@ dependencies = [ [[package]] name = "event-listener" version = "5.3.1" -source = "git+https://github.com/girlbossceo/event-listener?rev=fe4aebeeaae435af60087ddd56b573a2e0be671d#fe4aebeeaae435af60087ddd56b573a2e0be671d" +source = "git+https://forgejo.ellis.link/continuwuation/event-listener?rev=fe4aebeeaae435af60087ddd56b573a2e0be671d#fe4aebeeaae435af60087ddd56b573a2e0be671d" dependencies = [ "concurrent-queue", "parking", @@ -2030,7 +2030,7 @@ dependencies = [ [[package]] name = "hyper-util" version = "0.1.11" -source = "git+https://github.com/girlbossceo/hyper-util?rev=e4ae7628fe4fcdacef9788c4c8415317a4489941#e4ae7628fe4fcdacef9788c4c8415317a4489941" +source = "git+https://forgejo.ellis.link/continuwuation/hyper-util?rev=e4ae7628fe4fcdacef9788c4c8415317a4489941#e4ae7628fe4fcdacef9788c4c8415317a4489941" dependencies = [ "bytes", "futures-channel", @@ -3625,7 +3625,7 @@ dependencies = [ [[package]] name = "resolv-conf" version = "0.7.1" -source = "git+https://github.com/girlbossceo/resolv-conf?rev=200e958941d522a70c5877e3d846f55b5586c68d#200e958941d522a70c5877e3d846f55b5586c68d" +source = "git+https://forgejo.ellis.link/continuwuation/resolv-conf?rev=200e958941d522a70c5877e3d846f55b5586c68d#200e958941d522a70c5877e3d846f55b5586c68d" dependencies = [ "hostname", ] @@ -3653,7 +3653,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "assign", "js_int", @@ -3673,7 +3673,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "js_int", "ruma-common", @@ -3685,7 +3685,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "as_variant", "assign", @@ -3708,7 +3708,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "as_variant", "base64 0.22.1", @@ -3740,7 +3740,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "as_variant", "indexmap 2.8.0", @@ -3765,7 +3765,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "bytes", "headers", @@ -3787,7 +3787,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3796,7 +3796,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "js_int", "ruma-common", @@ -3806,7 +3806,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3821,7 +3821,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "js_int", "ruma-common", @@ -3833,7 +3833,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/girlbossceo/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -3849,7 +3849,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.33.0+9.11.1" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=1c267e0bf0cc7b7702e9a329deccd89de79ef4c3#1c267e0bf0cc7b7702e9a329deccd89de79ef4c3" +source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd#fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd" dependencies = [ "bindgen 0.71.1", "bzip2-sys", @@ -3866,7 +3866,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.37.0" -source = "git+https://github.com/girlbossceo/rust-rocksdb-zaidoon1?rev=1c267e0bf0cc7b7702e9a329deccd89de79ef4c3#1c267e0bf0cc7b7702e9a329deccd89de79ef4c3" +source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd#fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd" dependencies = [ "libc", "rust-librocksdb-sys", @@ -3979,7 +3979,7 @@ checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] name = "rustyline-async" version = "0.4.3" -source = "git+https://github.com/girlbossceo/rustyline-async?rev=deaeb0694e2083f53d363b648da06e10fc13900c#deaeb0694e2083f53d363b648da06e10fc13900c" +source = "git+https://forgejo.ellis.link/continuwuation/rustyline-async?rev=deaeb0694e2083f53d363b648da06e10fc13900c#deaeb0694e2083f53d363b648da06e10fc13900c" dependencies = [ "crossterm", "futures-channel", @@ -4675,7 +4675,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-ctl" version = "0.6.0" -source = "git+https://github.com/girlbossceo/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" +source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" dependencies = [ "libc", "paste", @@ -4685,7 +4685,7 @@ dependencies = [ [[package]] name = "tikv-jemalloc-sys" version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" -source = "git+https://github.com/girlbossceo/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" +source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" dependencies = [ "cc", "libc", @@ -4694,7 +4694,7 @@ dependencies = [ [[package]] name = "tikv-jemallocator" version = "0.6.0" -source = "git+https://github.com/girlbossceo/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" +source = "git+https://forgejo.ellis.link/continuwuation/jemallocator?rev=82af58d6a13ddd5dcdc7d4e91eae3b63292995b8#82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" dependencies = [ "libc", "tikv-jemalloc-sys", @@ -4980,7 +4980,7 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" version = "0.1.41" -source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" +source = "git+https://forgejo.ellis.link/continuwuation/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "pin-project-lite", "tracing-attributes", @@ -4990,7 +4990,7 @@ dependencies = [ [[package]] name = "tracing-attributes" version = "0.1.28" -source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" +source = "git+https://forgejo.ellis.link/continuwuation/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "proc-macro2", "quote", @@ -5000,7 +5000,7 @@ dependencies = [ [[package]] name = "tracing-core" version = "0.1.33" -source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" +source = "git+https://forgejo.ellis.link/continuwuation/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "once_cell", "valuable", @@ -5020,7 +5020,7 @@ dependencies = [ [[package]] name = "tracing-log" version = "0.2.0" -source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" +source = "git+https://forgejo.ellis.link/continuwuation/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "log", "once_cell", @@ -5048,7 +5048,7 @@ dependencies = [ [[package]] name = "tracing-subscriber" version = "0.3.19" -source = "git+https://github.com/girlbossceo/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" +source = "git+https://forgejo.ellis.link/continuwuation/tracing?rev=1e64095a8051a1adf0d1faa307f9f030889ec2aa#1e64095a8051a1adf0d1faa307f9f030889ec2aa" dependencies = [ "matchers", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index f5ee3f0f..e9ae0007 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,12 +14,12 @@ authors = [ categories = ["network-programming"] description = "a very cool Matrix chat homeserver written in Rust" edition = "2024" -homepage = "https://conduwuit.puppyirl.gay/" +homepage = "https://continuwuity.org/" keywords = ["chat", "matrix", "networking", "server", "uwu"] license = "Apache-2.0" # See also `rust-toolchain.toml` readme = "README.md" -repository = "https://github.com/girlbossceo/conduwuit" +repository = "https://forgejo.ellis.link/continuwuation/continuwuity" rust-version = "1.86.0" version = "0.5.0" @@ -348,7 +348,7 @@ version = "0.1.2" # Used for matrix spec type definitions and helpers [workspace.dependencies.ruma] -git = "https://github.com/girlbossceo/ruwuma" +git = "https://forgejo.ellis.link/continuwuation/ruwuma" #branch = "conduwuit-changes" rev = "920148dca1076454ca0ca5d43b5ce1aa708381d4" features = [ @@ -388,8 +388,8 @@ features = [ ] [workspace.dependencies.rust-rocksdb] -git = "https://github.com/girlbossceo/rust-rocksdb-zaidoon1" -rev = "1c267e0bf0cc7b7702e9a329deccd89de79ef4c3" +git = "https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1" +rev = "fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd" default-features = false features = [ "multi-threaded-cf", @@ -449,7 +449,7 @@ version = "0.37.0" # jemalloc usage [workspace.dependencies.tikv-jemalloc-sys] -git = "https://github.com/girlbossceo/jemallocator" +git = "https://forgejo.ellis.link/continuwuation/jemallocator" rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" default-features = false features = [ @@ -457,7 +457,7 @@ features = [ "unprefixed_malloc_on_supported_platforms", ] [workspace.dependencies.tikv-jemallocator] -git = "https://github.com/girlbossceo/jemallocator" +git = "https://forgejo.ellis.link/continuwuation/jemallocator" rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" default-features = false features = [ @@ -465,7 +465,7 @@ features = [ "unprefixed_malloc_on_supported_platforms", ] [workspace.dependencies.tikv-jemalloc-ctl] -git = "https://github.com/girlbossceo/jemallocator" +git = "https://forgejo.ellis.link/continuwuation/jemallocator" rev = "82af58d6a13ddd5dcdc7d4e91eae3b63292995b8" default-features = false features = ["use_std"] @@ -542,49 +542,49 @@ version = "1.0.2" # backport of [https://github.com/tokio-rs/tracing/pull/2956] to the 0.1.x branch of tracing. # we can switch back to upstream if #2956 is merged and backported in the upstream repo. -# https://github.com/girlbossceo/tracing/commit/b348dca742af641c47bc390261f60711c2af573c +# https://forgejo.ellis.link/continuwuation/tracing/commit/b348dca742af641c47bc390261f60711c2af573c [patch.crates-io.tracing-subscriber] -git = "https://github.com/girlbossceo/tracing" +git = "https://forgejo.ellis.link/continuwuation/tracing" rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" [patch.crates-io.tracing] -git = "https://github.com/girlbossceo/tracing" +git = "https://forgejo.ellis.link/continuwuation/tracing" rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" [patch.crates-io.tracing-core] -git = "https://github.com/girlbossceo/tracing" +git = "https://forgejo.ellis.link/continuwuation/tracing" rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" [patch.crates-io.tracing-log] -git = "https://github.com/girlbossceo/tracing" +git = "https://forgejo.ellis.link/continuwuation/tracing" rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" -# adds a tab completion callback: https://github.com/girlbossceo/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50 -# adds event for CTRL+\: https://github.com/girlbossceo/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b +# adds a tab completion callback: https://forgejo.ellis.link/continuwuation/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50 +# adds event for CTRL+\: https://forgejo.ellis.link/continuwuation/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b [patch.crates-io.rustyline-async] -git = "https://github.com/girlbossceo/rustyline-async" +git = "https://forgejo.ellis.link/continuwuation/rustyline-async" rev = "deaeb0694e2083f53d363b648da06e10fc13900c" # adds LIFO queue scheduling; this should be updated with PR progress. [patch.crates-io.event-listener] -git = "https://github.com/girlbossceo/event-listener" +git = "https://forgejo.ellis.link/continuwuation/event-listener" rev = "fe4aebeeaae435af60087ddd56b573a2e0be671d" [patch.crates-io.async-channel] -git = "https://github.com/girlbossceo/async-channel" +git = "https://forgejo.ellis.link/continuwuation/async-channel" rev = "92e5e74063bf2a3b10414bcc8a0d68b235644280" # adds affinity masks for selecting more than one core at a time [patch.crates-io.core_affinity] -git = "https://github.com/girlbossceo/core_affinity_rs" +git = "https://forgejo.ellis.link/continuwuation/core_affinity_rs" rev = "9c8e51510c35077df888ee72a36b4b05637147da" # reverts hyperium#148 conflicting with our delicate federation resolver hooks [patch.crates-io.hyper-util] -git = "https://github.com/girlbossceo/hyper-util" +git = "https://forgejo.ellis.link/continuwuation/hyper-util" rev = "e4ae7628fe4fcdacef9788c4c8415317a4489941" # allows no-aaaa option in resolv.conf # bumps rust edition and toolchain to 1.86.0 and 2024 # use sat_add on line number errors [patch.crates-io.resolv-conf] -git = "https://github.com/girlbossceo/resolv-conf" +git = "https://forgejo.ellis.link/continuwuation/resolv-conf" rev = "200e958941d522a70c5877e3d846f55b5586c68d" # From c5b99fbccda7140d7232cca1ca456ff4b1124109 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 18 Apr 2025 21:05:17 +0100 Subject: [PATCH 1691/2091] ci: Enable buildx caching --- .forgejo/workflows/release-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 142529ae..ba94acc3 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -133,8 +133,8 @@ jobs: platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} - # cache-from: type=gha - # cache-to: type=gha,mode=max + cache-from: type=gha + cache-to: type=gha,mode=max sbom: true outputs: type=image,"name=${{ needs.define-variables.outputs.images_list }}",push-by-digest=true,name-canonical=true,push=true env: From 298e2af3d7aed47d5f070c5259f23a2a761673b3 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 18 Apr 2025 22:24:35 +0100 Subject: [PATCH 1692/2091] ci: Try invert condition for branch prefix --- .forgejo/workflows/release-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index ba94acc3..1411db61 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -185,7 +185,7 @@ jobs: type=semver,pattern=v{{version}} type=semver,pattern=v{{major}}.{{minor}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.0.') }} type=semver,pattern=v{{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} - type=ref,event=branch,prefix=${{ format('refs/heads/{0}', github.event.repository.default_branch) == github.ref && '' || 'branch-' }} + type=ref,event=branch,prefix=${{ format('refs/heads/{0}', github.event.repository.default_branch) 1= github.ref && 'branch-' || '' }} type=ref,event=pr type=sha,format=long images: ${{needs.define-variables.outputs.images}} From d4561e950b25f1120d15167f62431177c466202b Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 18 Apr 2025 22:25:10 +0100 Subject: [PATCH 1693/2091] ci: Run builtin registry whenever secret is available --- .forgejo/workflows/release-image.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 1411db61..0b764110 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -16,7 +16,8 @@ on: env: BUILTIN_REGISTRY: forgejo.ellis.link - BUILTIN_REGISTRY_ENABLED: "${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}" + BUILTIN_REGISTRY_ENABLED: "${{ ((vars.BUILTIN_REGISTRY_USER && secrets.BUILTIN_REGISTRY_PASSWORD) || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)) && 'true' || 'false' }}" + jobs: define-variables: From 1b1198771f577f9dad4e34a00aff0e85ecc5879a Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 18 Apr 2025 22:38:17 +0100 Subject: [PATCH 1694/2091] ci: Move timelord to actions to avoid bad cache invalidations from cargo --- .forgejo/workflows/release-image.yml | 20 +++++++++++++++++++- docker/Dockerfile | 8 -------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 0b764110..adf70594 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -89,7 +89,13 @@ jobs: uses: actions/checkout@v4 with: persist-credentials: false - + - run: | + if ! command -v rustup &> /dev/null ; then + curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y + echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH + fi + - uses: https://github.com/cargo-bins/cargo-binstall@main + - run: cargo binstall timelord-cli@3.0.1 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Set up QEMU @@ -123,6 +129,18 @@ jobs: echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV - name: Get Git commit timestamps run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV + - name: Set up timelord + uses: actions/cache/restore@v3 + with: + path: /timelord/ + key: timelord-v0 # Cache is already split per runner + - name: Run timelord to set timestamps + run: timelord sync --source-dir . --cache-dir /timelord/ + - name: Save timelord + uses: actions/cache/save@v3 + with: + path: /timelord/ + key: timelord-v0 - name: Build and push Docker image by digest id: build uses: docker/build-push-action@v6 diff --git a/docker/Dockerfile b/docker/Dockerfile index 10f54d94..536af632 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -44,15 +44,11 @@ ENV CARGO_SBOM_VERSION=0.9.1 # renovate: datasource=crate depName=lddtree ENV LDDTREE_VERSION=0.3.7 -# renovate: datasource=crate depName=timelord-cli -ENV TIMELORD_VERSION=3.0.1 - # Install unpackaged tools RUN < Date: Sun, 6 Apr 2025 05:52:48 +0000 Subject: [PATCH 1695/2091] relax Send requirement on some drier stream extensions Signed-off-by: Jason Volk --- src/core/utils/stream/expect.rs | 2 +- src/core/utils/stream/ready.rs | 4 ++-- src/core/utils/stream/try_ready.rs | 8 ++++---- src/core/utils/stream/try_tools.rs | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/core/utils/stream/expect.rs b/src/core/utils/stream/expect.rs index 3509bb83..ec572714 100644 --- a/src/core/utils/stream/expect.rs +++ b/src/core/utils/stream/expect.rs @@ -10,7 +10,7 @@ pub trait TryExpect<'a, Item> { impl<'a, T, Item> TryExpect<'a, Item> for T where - T: Stream> + TryStream + Send + 'a, + T: Stream> + Send + TryStream + 'a, Item: 'a, { #[inline] diff --git a/src/core/utils/stream/ready.rs b/src/core/utils/stream/ready.rs index dce7d378..38feaf64 100644 --- a/src/core/utils/stream/ready.rs +++ b/src/core/utils/stream/ready.rs @@ -16,7 +16,7 @@ use futures::{ /// This interface is not necessarily complete; feel free to add as-needed. pub trait ReadyExt where - Self: Stream + Send + Sized, + Self: Stream + Sized, { fn ready_all(self, f: F) -> All, impl FnMut(Item) -> Ready> where @@ -93,7 +93,7 @@ where impl ReadyExt for S where - S: Stream + Send + Sized, + S: Stream + Sized, { #[inline] fn ready_all(self, f: F) -> All, impl FnMut(Item) -> Ready> diff --git a/src/core/utils/stream/try_ready.rs b/src/core/utils/stream/try_ready.rs index 611c177f..287fa1e1 100644 --- a/src/core/utils/stream/try_ready.rs +++ b/src/core/utils/stream/try_ready.rs @@ -13,8 +13,8 @@ use crate::Result; /// This interface is not necessarily complete; feel free to add as-needed. pub trait TryReadyExt where - S: TryStream> + Send + ?Sized, - Self: TryStream + Send + Sized, + S: TryStream> + ?Sized, + Self: TryStream + Sized, { fn ready_and_then( self, @@ -67,8 +67,8 @@ where impl TryReadyExt for S where - S: TryStream> + Send + ?Sized, - Self: TryStream + Send + Sized, + S: TryStream> + ?Sized, + Self: TryStream + Sized, { #[inline] fn ready_and_then( diff --git a/src/core/utils/stream/try_tools.rs b/src/core/utils/stream/try_tools.rs index ea3b50fc..417806fc 100644 --- a/src/core/utils/stream/try_tools.rs +++ b/src/core/utils/stream/try_tools.rs @@ -8,8 +8,8 @@ use crate::Result; /// TryStreamTools pub trait TryTools where - S: TryStream> + Send + ?Sized, - Self: TryStream + Send + Sized, + S: TryStream> + ?Sized, + Self: TryStream + Sized, { fn try_take( self, @@ -23,8 +23,8 @@ where impl TryTools for S where - S: TryStream> + Send + ?Sized, - Self: TryStream + Send + Sized, + S: TryStream> + ?Sized, + Self: TryStream + Sized, { #[inline] fn try_take( From 75fb19a5cacf853740fb2cfd016c435c319e5e5f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Apr 2025 05:42:27 +0000 Subject: [PATCH 1696/2091] add ready_find() stream extension Signed-off-by: Jason Volk --- src/core/utils/stream/ready.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/core/utils/stream/ready.rs b/src/core/utils/stream/ready.rs index 38feaf64..be4d1b25 100644 --- a/src/core/utils/stream/ready.rs +++ b/src/core/utils/stream/ready.rs @@ -2,7 +2,7 @@ #![allow(clippy::type_complexity)] use futures::{ - future::{Ready, ready}, + future::{FutureExt, Ready, ready}, stream::{ All, Any, Filter, FilterMap, Fold, ForEach, Scan, SkipWhile, Stream, StreamExt, TakeWhile, }, @@ -26,6 +26,12 @@ where where F: Fn(Item) -> bool; + fn ready_find<'a, F>(self, f: F) -> impl Future> + Send + where + Self: Send + Unpin + 'a, + F: Fn(&Item) -> bool + Send + 'a, + Item: Send; + fn ready_filter<'a, F>( self, f: F, @@ -111,6 +117,19 @@ where self.any(move |t| ready(f(t))) } + #[inline] + fn ready_find<'a, F>(self, f: F) -> impl Future> + Send + where + Self: Send + Unpin + 'a, + F: Fn(&Item) -> bool + Send + 'a, + Item: Send, + { + self.ready_filter(f) + .take(1) + .into_future() + .map(|(curr, _next)| curr) + } + #[inline] fn ready_filter<'a, F>( self, From d8b56c9c35a953aed756f4c6374109219dfdaf77 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Apr 2025 19:09:07 +0000 Subject: [PATCH 1697/2091] add ReadyEq future extension Signed-off-by: Jason Volk --- src/core/utils/future/bool_ext.rs | 48 +++++++++++++-------------- src/core/utils/future/mod.rs | 2 ++ src/core/utils/future/ready_eq_ext.rs | 25 ++++++++++++++ src/core/utils/mod.rs | 2 +- 4 files changed, 52 insertions(+), 25 deletions(-) create mode 100644 src/core/utils/future/ready_eq_ext.rs diff --git a/src/core/utils/future/bool_ext.rs b/src/core/utils/future/bool_ext.rs index c93c7dbc..24f239ff 100644 --- a/src/core/utils/future/bool_ext.rs +++ b/src/core/utils/future/bool_ext.rs @@ -22,30 +22,6 @@ where Self: Sized + Unpin; } -pub async fn and(args: I) -> impl Future + Send -where - I: Iterator + Send, - F: Future + Send, -{ - type Result = crate::Result<(), ()>; - - let args = args.map(|a| a.map(|a| a.then_some(()).ok_or(Result::Err(())))); - - try_join_all(args).map(|result| result.is_ok()) -} - -pub async fn or(args: I) -> impl Future + Send -where - I: Iterator + Send, - F: Future + Send + Unpin, -{ - type Result = crate::Result<(), ()>; - - let args = args.map(|a| a.map(|a| a.then_some(()).ok_or(Result::Err(())))); - - select_ok(args).map(|result| result.is_ok()) -} - impl BoolExt for Fut where Fut: Future + Send, @@ -80,3 +56,27 @@ where try_select(a, b).map(|result| result.is_ok()) } } + +pub async fn and(args: I) -> impl Future + Send +where + I: Iterator + Send, + F: Future + Send, +{ + type Result = crate::Result<(), ()>; + + let args = args.map(|a| a.map(|a| a.then_some(()).ok_or(Result::Err(())))); + + try_join_all(args).map(|result| result.is_ok()) +} + +pub async fn or(args: I) -> impl Future + Send +where + I: Iterator + Send, + F: Future + Send + Unpin, +{ + type Result = crate::Result<(), ()>; + + let args = args.map(|a| a.map(|a| a.then_some(()).ok_or(Result::Err(())))); + + select_ok(args).map(|result| result.is_ok()) +} diff --git a/src/core/utils/future/mod.rs b/src/core/utils/future/mod.rs index 4edd0102..d896e66d 100644 --- a/src/core/utils/future/mod.rs +++ b/src/core/utils/future/mod.rs @@ -2,10 +2,12 @@ mod bool_ext; mod ext_ext; mod option_ext; mod option_stream; +mod ready_eq_ext; mod try_ext_ext; pub use bool_ext::{BoolExt, and, or}; pub use ext_ext::ExtExt; pub use option_ext::OptionExt; pub use option_stream::OptionStream; +pub use ready_eq_ext::ReadyEqExt; pub use try_ext_ext::TryExtExt; diff --git a/src/core/utils/future/ready_eq_ext.rs b/src/core/utils/future/ready_eq_ext.rs new file mode 100644 index 00000000..1625adae --- /dev/null +++ b/src/core/utils/future/ready_eq_ext.rs @@ -0,0 +1,25 @@ +//! Future extension for Partial Equality against present value + +use futures::{Future, FutureExt}; + +pub trait ReadyEqExt +where + Self: Future + Send + Sized, + T: PartialEq + Send + Sync, +{ + fn eq(self, t: &T) -> impl Future + Send; + + fn ne(self, t: &T) -> impl Future + Send; +} + +impl ReadyEqExt for Fut +where + Fut: Future + Send + Sized, + T: PartialEq + Send + Sync, +{ + #[inline] + fn eq(self, t: &T) -> impl Future + Send { self.map(move |r| r.eq(t)) } + + #[inline] + fn ne(self, t: &T) -> impl Future + Send { self.map(move |r| r.ne(t)) } +} diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 117fb739..5e6f2868 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -28,7 +28,7 @@ pub use self::{ bool::BoolExt, bytes::{increment, u64_from_bytes, u64_from_u8, u64_from_u8x8}, debug::slice_truncated as debug_slice_truncated, - future::TryExtExt as TryFutureExtExt, + future::{BoolExt as FutureBoolExt, OptionStream, TryExtExt as TryFutureExtExt}, hash::sha256::delimited as calculate_hash, html::Escape as HtmlEscape, json::{deserialize_from_str, to_canonical_object}, From e7c3f783775cde5b606005e24220c008f657b970 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Apr 2025 06:39:45 +0000 Subject: [PATCH 1698/2091] modernize state_res w/ stream extensions Signed-off-by: Jason Volk --- src/core/matrix/state_res/mod.rs | 220 ++++++++++++++++--------------- 1 file changed, 111 insertions(+), 109 deletions(-) diff --git a/src/core/matrix/state_res/mod.rs b/src/core/matrix/state_res/mod.rs index 93c00d15..ce6b7e89 100644 --- a/src/core/matrix/state_res/mod.rs +++ b/src/core/matrix/state_res/mod.rs @@ -15,11 +15,10 @@ use std::{ borrow::Borrow, cmp::{Ordering, Reverse}, collections::{BinaryHeap, HashMap, HashSet}, - fmt::Debug, hash::{BuildHasher, Hash}, }; -use futures::{Future, FutureExt, StreamExt, TryFutureExt, TryStreamExt, future, stream}; +use futures::{Future, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, future}; use ruma::{ EventId, Int, MilliSecondsSinceUnixEpoch, RoomVersionId, events::{ @@ -37,9 +36,13 @@ pub use self::{ room_version::RoomVersion, }; use crate::{ - debug, + debug, debug_error, matrix::{event::Event, pdu::StateKey}, - trace, warn, + trace, + utils::stream::{ + BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryReadyExt, WidebandExt, + }, + warn, }; /// A mapping of event type and state_key to some value `T`, usually an @@ -112,20 +115,16 @@ where debug!(count = conflicting.len(), "conflicting events"); trace!(map = ?conflicting, "conflicting events"); - let auth_chain_diff = - get_auth_chain_diff(auth_chain_sets).chain(conflicting.into_values().flatten()); + let conflicting_values = conflicting.into_values().flatten().stream(); // `all_conflicted` contains unique items // synapse says `full_set = {eid for eid in full_conflicted_set if eid in // event_map}` - let all_conflicted: HashSet<_> = stream::iter(auth_chain_diff) - // Don't honor events we cannot "verify" - .map(|id| event_exists(id.clone()).map(move |exists| (id, exists))) - .buffer_unordered(parallel_fetches) - .filter_map(|(id, exists)| future::ready(exists.then_some(id))) - .collect() - .boxed() - .await; + let all_conflicted: HashSet<_> = get_auth_chain_diff(auth_chain_sets) + .chain(conflicting_values) + .broad_filter_map(async |id| event_exists(id.clone()).await.then_some(id)) + .collect() + .await; debug!(count = all_conflicted.len(), "full conflicted set"); trace!(set = ?all_conflicted, "full conflicted set"); @@ -135,12 +134,15 @@ where // Get only the control events with a state_key: "" or ban/kick event (sender != // state_key) - let control_events: Vec<_> = stream::iter(all_conflicted.iter()) - .map(|id| is_power_event_id(id, &event_fetch).map(move |is| (id, is))) - .buffer_unordered(parallel_fetches) - .filter_map(|(id, is)| future::ready(is.then_some(id.clone()))) + let control_events: Vec<_> = all_conflicted + .iter() + .stream() + .wide_filter_map(async |id| { + is_power_event_id(id, &event_fetch) + .await + .then_some(id.clone()) + }) .collect() - .boxed() .await; // Sort the control events based on power_level/clock/event_id and @@ -160,10 +162,9 @@ where // Sequentially auth check each control event. let resolved_control = iterative_auth_check( &room_version, - sorted_control_levels.iter(), + sorted_control_levels.iter().stream(), clean.clone(), &event_fetch, - parallel_fetches, ) .await?; @@ -172,36 +173,35 @@ where // At this point the control_events have been resolved we now have to // sort the remaining events using the mainline of the resolved power level. - let deduped_power_ev = sorted_control_levels.into_iter().collect::>(); + let deduped_power_ev: HashSet<_> = sorted_control_levels.into_iter().collect(); // This removes the control events that passed auth and more importantly those // that failed auth - let events_to_resolve = all_conflicted + let events_to_resolve: Vec<_> = all_conflicted .iter() .filter(|&id| !deduped_power_ev.contains(id.borrow())) .cloned() - .collect::>(); + .collect(); debug!(count = events_to_resolve.len(), "events left to resolve"); trace!(list = ?events_to_resolve, "events left to resolve"); // This "epochs" power level event - let power_event = resolved_control.get(&(StateEventType::RoomPowerLevels, StateKey::new())); + let power_levels_ty_sk = (StateEventType::RoomPowerLevels, StateKey::new()); + let power_event = resolved_control.get(&power_levels_ty_sk); debug!(event_id = ?power_event, "power event"); let sorted_left_events = - mainline_sort(&events_to_resolve, power_event.cloned(), &event_fetch, parallel_fetches) - .await?; + mainline_sort(&events_to_resolve, power_event.cloned(), &event_fetch).await?; trace!(list = ?sorted_left_events, "events left, sorted"); let mut resolved_state = iterative_auth_check( &room_version, - sorted_left_events.iter(), + sorted_left_events.iter().stream(), resolved_control, // The control events are added to the final resolved state &event_fetch, - parallel_fetches, ) .await?; @@ -265,7 +265,7 @@ where #[allow(clippy::arithmetic_side_effects)] fn get_auth_chain_diff( auth_chain_sets: &[HashSet], -) -> impl Iterator + Send + use +) -> impl Stream + Send + use where Id: Clone + Eq + Hash + Send, Hasher: BuildHasher + Send + Sync, @@ -279,6 +279,7 @@ where id_counts .into_iter() .filter_map(move |(id, count)| (count < num_sets).then_some(id)) + .stream() } /// Events are sorted from "earliest" to "latest". @@ -310,13 +311,15 @@ where } // This is used in the `key_fn` passed to the lexico_topo_sort fn - let event_to_pl = stream::iter(graph.keys()) + let event_to_pl = graph + .keys() + .stream() .map(|event_id| { - get_power_level_for_sender(event_id.clone(), fetch_event, parallel_fetches) + get_power_level_for_sender(event_id.clone(), fetch_event) .map(move |res| res.map(|pl| (event_id, pl))) }) .buffer_unordered(parallel_fetches) - .try_fold(HashMap::new(), |mut event_to_pl, (event_id, pl)| { + .ready_try_fold(HashMap::new(), |mut event_to_pl, (event_id, pl)| { debug!( event_id = event_id.borrow().as_str(), power_level = i64::from(pl), @@ -324,7 +327,7 @@ where ); event_to_pl.insert(event_id.clone(), pl); - future::ok(event_to_pl) + Ok(event_to_pl) }) .boxed() .await?; @@ -475,7 +478,6 @@ where async fn get_power_level_for_sender( event_id: E::Id, fetch_event: &F, - parallel_fetches: usize, ) -> serde_json::Result where F: Fn(E::Id) -> Fut + Sync, @@ -485,19 +487,17 @@ where { debug!("fetch event ({event_id}) senders power level"); - let event = fetch_event(event_id.clone()).await; + let event = fetch_event(event_id).await; - let auth_events = event.as_ref().map(Event::auth_events).into_iter().flatten(); + let auth_events = event.as_ref().map(Event::auth_events); - let pl = stream::iter(auth_events) - .map(|aid| fetch_event(aid.clone())) - .buffer_unordered(parallel_fetches.min(5)) - .filter_map(future::ready) - .collect::>() - .boxed() - .await + let pl = auth_events .into_iter() - .find(|aev| is_type_and_key(aev, &TimelineEventType::RoomPowerLevels, "")); + .flatten() + .stream() + .broadn_filter_map(5, |aid| fetch_event(aid.clone())) + .ready_find(|aev| is_type_and_key(aev, &TimelineEventType::RoomPowerLevels, "")) + .await; let content: PowerLevelsContentFields = match pl { | None => return Ok(int!(0)), @@ -525,34 +525,28 @@ where /// For each `events_to_check` event we gather the events needed to auth it from /// the the `fetch_event` closure and verify each event using the /// `event_auth::auth_check` function. -async fn iterative_auth_check<'a, E, F, Fut, I>( +async fn iterative_auth_check<'a, E, F, Fut, S>( room_version: &RoomVersion, - events_to_check: I, + events_to_check: S, unconflicted_state: StateMap, fetch_event: &F, - parallel_fetches: usize, ) -> Result> where F: Fn(E::Id) -> Fut + Sync, Fut: Future> + Send, E::Id: Borrow + Clone + Eq + Ord + Send + Sync + 'a, - I: Iterator + Debug + Send + 'a, + S: Stream + Send + 'a, E: Event + Clone + Send + Sync, { debug!("starting iterative auth check"); - trace!( - list = ?events_to_check, - "events to check" - ); - let events_to_check: Vec<_> = stream::iter(events_to_check) + let events_to_check: Vec<_> = events_to_check .map(Result::Ok) - .map_ok(|event_id| { - fetch_event(event_id.clone()).map(move |result| { - result.ok_or_else(|| Error::NotFound(format!("Failed to find {event_id}"))) - }) + .broad_and_then(async |event_id| { + fetch_event(event_id.clone()) + .await + .ok_or_else(|| Error::NotFound(format!("Failed to find {event_id}"))) }) - .try_buffer_unordered(parallel_fetches) .try_collect() .boxed() .await?; @@ -562,10 +556,10 @@ where .flat_map(|event: &E| event.auth_events().map(Clone::clone)) .collect(); - let auth_events: HashMap = stream::iter(auth_event_ids.into_iter()) - .map(fetch_event) - .buffer_unordered(parallel_fetches) - .filter_map(future::ready) + let auth_events: HashMap = auth_event_ids + .into_iter() + .stream() + .broad_filter_map(fetch_event) .map(|auth_event| (auth_event.event_id().clone(), auth_event)) .collect() .boxed() @@ -574,7 +568,6 @@ where let auth_events = &auth_events; let mut resolved_state = unconflicted_state; for event in &events_to_check { - let event_id = event.event_id(); let state_key = event .state_key() .ok_or_else(|| Error::InvalidPdu("State event had no state key".to_owned()))?; @@ -603,24 +596,22 @@ where } } - stream::iter( - auth_types - .iter() - .filter_map(|key| Some((key, resolved_state.get(key)?))), - ) - .filter_map(|(key, ev_id)| async move { - if let Some(event) = auth_events.get(ev_id.borrow()) { - Some((key, event.clone())) - } else { - Some((key, fetch_event(ev_id.clone()).await?)) - } - }) - .for_each(|(key, event)| { - //TODO: synapse checks "rejected_reason" is None here - auth_state.insert(key.to_owned(), event); - future::ready(()) - }) - .await; + auth_types + .iter() + .stream() + .ready_filter_map(|key| Some((key, resolved_state.get(key)?))) + .filter_map(|(key, ev_id)| async move { + if let Some(event) = auth_events.get(ev_id.borrow()) { + Some((key, event.clone())) + } else { + Some((key, fetch_event(ev_id.clone()).await?)) + } + }) + .ready_for_each(|(key, event)| { + //TODO: synapse checks "rejected_reason" is None here + auth_state.insert(key.to_owned(), event); + }) + .await; debug!("event to check {:?}", event.event_id()); @@ -634,12 +625,25 @@ where future::ready(auth_state.get(&ty.with_state_key(key))) }; - if auth_check(room_version, &event, current_third_party.as_ref(), fetch_state).await? { - // add event to resolved state map - resolved_state.insert(event.event_type().with_state_key(state_key), event_id.clone()); - } else { - // synapse passes here on AuthError. We do not add this event to resolved_state. - warn!("event {event_id} failed the authentication check"); + let auth_result = + auth_check(room_version, &event, current_third_party.as_ref(), fetch_state).await; + + match auth_result { + | Ok(true) => { + // add event to resolved state map + resolved_state.insert( + event.event_type().with_state_key(state_key), + event.event_id().clone(), + ); + }, + | Ok(false) => { + // synapse passes here on AuthError. We do not add this event to resolved_state. + warn!("event {} failed the authentication check", event.event_id()); + }, + | Err(e) => { + debug_error!("event {} failed the authentication check: {e}", event.event_id()); + return Err(e); + }, } } @@ -659,7 +663,6 @@ async fn mainline_sort( to_sort: &[E::Id], resolved_power_level: Option, fetch_event: &F, - parallel_fetches: usize, ) -> Result> where F: Fn(E::Id) -> Fut + Sync, @@ -682,11 +685,13 @@ where let event = fetch_event(p.clone()) .await .ok_or_else(|| Error::NotFound(format!("Failed to find {p}")))?; + pl = None; for aid in event.auth_events() { let ev = fetch_event(aid.clone()) .await .ok_or_else(|| Error::NotFound(format!("Failed to find {aid}")))?; + if is_type_and_key(&ev, &TimelineEventType::RoomPowerLevels, "") { pl = Some(aid.to_owned()); break; @@ -694,36 +699,32 @@ where } } - let mainline_map = mainline + let mainline_map: HashMap<_, _> = mainline .iter() .rev() .enumerate() .map(|(idx, eid)| ((*eid).clone(), idx)) - .collect::>(); + .collect(); - let order_map = stream::iter(to_sort.iter()) - .map(|ev_id| { - fetch_event(ev_id.clone()).map(move |event| event.map(|event| (event, ev_id))) + let order_map: HashMap<_, _> = to_sort + .iter() + .stream() + .broad_filter_map(async |ev_id| { + fetch_event(ev_id.clone()).await.map(|event| (event, ev_id)) }) - .buffer_unordered(parallel_fetches) - .filter_map(future::ready) - .map(|(event, ev_id)| { + .broad_filter_map(|(event, ev_id)| { get_mainline_depth(Some(event.clone()), &mainline_map, fetch_event) - .map_ok(move |depth| (depth, event, ev_id)) + .map_ok(move |depth| (ev_id, (depth, event.origin_server_ts(), ev_id))) .map(Result::ok) }) - .buffer_unordered(parallel_fetches) - .filter_map(future::ready) - .fold(HashMap::new(), |mut order_map, (depth, event, ev_id)| { - order_map.insert(ev_id, (depth, event.origin_server_ts(), ev_id)); - future::ready(order_map) - }) + .collect() .boxed() .await; // Sort the event_ids by their depth, timestamp and EventId // unwrap is OK order map and sort_event_ids are from to_sort (the same Vec) - let mut sort_event_ids = order_map.keys().map(|&k| k.clone()).collect::>(); + let mut sort_event_ids: Vec<_> = order_map.keys().map(|&k| k.clone()).collect(); + sort_event_ids.sort_by_key(|sort_id| &order_map[sort_id]); Ok(sort_event_ids) @@ -744,6 +745,7 @@ where { while let Some(sort_ev) = event { debug!(event_id = sort_ev.event_id().borrow().as_str(), "mainline"); + let id = sort_ev.event_id(); if let Some(depth) = mainline_map.get(id.borrow()) { return Ok(*depth); @@ -754,6 +756,7 @@ where let aev = fetch_event(aid.clone()) .await .ok_or_else(|| Error::NotFound(format!("Failed to find {aid}")))?; + if is_type_and_key(&aev, &TimelineEventType::RoomPowerLevels, "") { event = Some(aev); break; @@ -884,7 +887,7 @@ mod tests { zara, }, }; - use crate::debug; + use crate::{debug, utils::stream::IterStream}; async fn test_event_sort() { use futures::future::ready; @@ -915,10 +918,9 @@ mod tests { let resolved_power = super::iterative_auth_check( &RoomVersion::V6, - sorted_power_events.iter(), + sorted_power_events.iter().stream(), HashMap::new(), // unconflicted events &fetcher, - 1, ) .await .expect("iterative auth check failed on resolved events"); @@ -932,7 +934,7 @@ mod tests { .get(&(StateEventType::RoomPowerLevels, "".into())) .cloned(); - let sorted_event_ids = super::mainline_sort(&events_to_sort, power_level, &fetcher, 1) + let sorted_event_ids = super::mainline_sort(&events_to_sort, power_level, &fetcher) .await .unwrap(); From 05e65936fa8a522667035b12774ef788944303e7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Apr 2025 20:30:15 +0000 Subject: [PATCH 1699/2091] modest cleanup of snake sync service related Signed-off-by: Jason Volk --- src/api/client/sync/v4.rs | 83 +++++++-------- src/api/client/sync/v5.rs | 55 ++++------ src/service/sync/mod.rs | 205 +++++++++++++++++--------------------- 3 files changed, 143 insertions(+), 200 deletions(-) diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index f7edb8c0..55faf420 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -6,22 +6,23 @@ use std::{ use axum::extract::State; use conduwuit::{ - Error, PduCount, PduEvent, Result, debug, error, extract_variant, + Err, Error, PduCount, PduEvent, Result, debug, error, extract_variant, utils::{ BoolExt, IterStream, ReadyExt, TryFutureExtExt, math::{ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, }, warn, }; +use conduwuit_service::{ + rooms::read_receipt::pack_receipts, + sync::{into_db_key, into_snake_key}, +}; use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{ MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, - api::client::{ - error::ErrorKind, - sync::sync_events::{ - self, DeviceLists, UnreadNotificationsCount, - v4::{SlidingOp, SlidingSyncRoomHero}, - }, + api::client::sync::sync_events::{ + self, DeviceLists, UnreadNotificationsCount, + v4::{SlidingOp, SlidingSyncRoomHero}, }, events::{ AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, @@ -31,7 +32,6 @@ use ruma::{ serde::Raw, uint, }; -use service::rooms::read_receipt::pack_receipts; use super::{load_timeline, share_encrypted_room}; use crate::{ @@ -50,10 +50,11 @@ pub(crate) async fn sync_events_v4_route( ) -> Result { debug_assert!(DEFAULT_BUMP_TYPES.is_sorted(), "DEFAULT_BUMP_TYPES is not sorted"); let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.expect("user is authenticated"); + let sender_device = body.sender_device.as_ref().expect("user is authenticated"); let mut body = body.body; + // Setup watchers, so if there's no response, we can wait for them - let watcher = services.sync.watch(sender_user, &sender_device); + let watcher = services.sync.watch(sender_user, sender_device); let next_batch = services.globals.next_count()?; @@ -68,33 +69,21 @@ pub(crate) async fn sync_events_v4_route( .and_then(|string| string.parse().ok()) .unwrap_or(0); - if globalsince != 0 - && !services - .sync - .remembered(sender_user.clone(), sender_device.clone(), conn_id.clone()) - { + let db_key = into_db_key(sender_user, sender_device, conn_id.clone()); + if globalsince != 0 && !services.sync.remembered(&db_key) { debug!("Restarting sync stream because it was gone from the database"); - return Err(Error::Request( - ErrorKind::UnknownPos, - "Connection data lost since last time".into(), - http::StatusCode::BAD_REQUEST, - )); + return Err!(Request(UnknownPos("Connection data lost since last time"))); } if globalsince == 0 { - services.sync.forget_sync_request_connection( - sender_user.clone(), - sender_device.clone(), - conn_id.clone(), - ); + services.sync.forget_sync_request_connection(&db_key); } // Get sticky parameters from cache - let known_rooms = services.sync.update_sync_request_with_cache( - sender_user.clone(), - sender_device.clone(), - &mut body, - ); + let snake_key = into_snake_key(sender_user, sender_device, conn_id.clone()); + let known_rooms = services + .sync + .update_sync_request_with_cache(&snake_key, &mut body); let all_joined_rooms: Vec<_> = services .rooms @@ -136,7 +125,7 @@ pub(crate) async fn sync_events_v4_route( if body.extensions.to_device.enabled.unwrap_or(false) { services .users - .remove_to_device_events(sender_user, &sender_device, globalsince) + .remove_to_device_events(sender_user, sender_device, globalsince) .await; } @@ -261,7 +250,7 @@ pub(crate) async fn sync_events_v4_route( if let Some(Ok(user_id)) = pdu.state_key.as_deref().map(UserId::parse) { - if user_id == *sender_user { + if user_id == sender_user { continue; } @@ -299,7 +288,7 @@ pub(crate) async fn sync_events_v4_route( .state_cache .room_members(room_id) // Don't send key updates from the sender to the sender - .ready_filter(|user_id| sender_user != user_id) + .ready_filter(|&user_id| sender_user != user_id) // Only send keys if the sender doesn't share an encrypted room with the target // already .filter_map(|user_id| { @@ -425,10 +414,9 @@ pub(crate) async fn sync_events_v4_route( }); if let Some(conn_id) = &body.conn_id { + let db_key = into_db_key(sender_user, sender_device, conn_id); services.sync.update_sync_known_rooms( - sender_user, - &sender_device, - conn_id.clone(), + &db_key, list_id.clone(), new_known_rooms, globalsince, @@ -478,23 +466,20 @@ pub(crate) async fn sync_events_v4_route( } if let Some(conn_id) = &body.conn_id { + let db_key = into_db_key(sender_user, sender_device, conn_id); services.sync.update_sync_known_rooms( - sender_user, - &sender_device, - conn_id.clone(), + &db_key, "subscriptions".to_owned(), known_subscription_rooms, globalsince, ); } - if let Some(conn_id) = &body.conn_id { - services.sync.update_sync_subscriptions( - sender_user.clone(), - sender_device.clone(), - conn_id.clone(), - body.room_subscriptions, - ); + if let Some(conn_id) = body.conn_id.clone() { + let db_key = into_db_key(sender_user, sender_device, conn_id); + services + .sync + .update_sync_subscriptions(&db_key, body.room_subscriptions); } let mut rooms = BTreeMap::new(); @@ -648,7 +633,7 @@ pub(crate) async fn sync_events_v4_route( .rooms .state_cache .room_members(room_id) - .ready_filter(|member| member != sender_user) + .ready_filter(|&member| member != sender_user) .filter_map(|user_id| { services .rooms @@ -787,7 +772,7 @@ pub(crate) async fn sync_events_v4_route( .users .get_to_device_events( sender_user, - &sender_device, + sender_device, Some(globalsince), Some(next_batch), ) @@ -805,7 +790,7 @@ pub(crate) async fn sync_events_v4_route( }, device_one_time_keys_count: services .users - .count_one_time_keys(sender_user, &sender_device) + .count_one_time_keys(sender_user, sender_device) .await, // Fallback keys are not yet supported device_unused_fallback_key_types: None, diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index 684752ec..00a2d18d 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -6,7 +6,7 @@ use std::{ use axum::extract::State; use conduwuit::{ - Error, Result, debug, error, extract_variant, + Err, Error, Result, error, extract_variant, matrix::{ TypeStateKey, pdu::{PduCount, PduEvent}, @@ -18,14 +18,11 @@ use conduwuit::{ }, warn, }; -use conduwuit_service::rooms::read_receipt::pack_receipts; +use conduwuit_service::{rooms::read_receipt::pack_receipts, sync::into_snake_key}; use futures::{FutureExt, StreamExt, TryFutureExt}; use ruma::{ DeviceId, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, - api::client::{ - error::ErrorKind, - sync::sync_events::{self, DeviceLists, UnreadNotificationsCount}, - }, + api::client::sync::sync_events::{self, DeviceLists, UnreadNotificationsCount}, events::{ AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType, room::member::{MembershipState, RoomMemberEventContent}, @@ -74,35 +71,23 @@ pub(crate) async fn sync_events_v5_route( .and_then(|string| string.parse().ok()) .unwrap_or(0); - if globalsince != 0 - && !services.sync.snake_connection_cached( - sender_user.clone(), - sender_device.clone(), - conn_id.clone(), - ) { - debug!("Restarting sync stream because it was gone from the database"); - return Err(Error::Request( - ErrorKind::UnknownPos, - "Connection data lost since last time".into(), - http::StatusCode::BAD_REQUEST, - )); + let snake_key = into_snake_key(sender_user, sender_device, conn_id); + + if globalsince != 0 && !services.sync.snake_connection_cached(&snake_key) { + return Err!(Request(UnknownPos( + "Connection data unknown to server; restarting sync stream." + ))); } // Client / User requested an initial sync if globalsince == 0 { - services.sync.forget_snake_sync_connection( - sender_user.clone(), - sender_device.clone(), - conn_id.clone(), - ); + services.sync.forget_snake_sync_connection(&snake_key); } // Get sticky parameters from cache - let known_rooms = services.sync.update_snake_sync_request_with_cache( - sender_user.clone(), - sender_device.clone(), - &mut body, - ); + let known_rooms = services + .sync + .update_snake_sync_request_with_cache(&snake_key, &mut body); let all_joined_rooms: Vec<_> = services .rooms @@ -254,11 +239,10 @@ async fn fetch_subscriptions( // body.room_subscriptions.remove(&r); //} - if let Some(conn_id) = &body.conn_id { + if let Some(conn_id) = body.conn_id.clone() { + let snake_key = into_snake_key(sender_user, sender_device, conn_id); services.sync.update_snake_sync_known_rooms( - sender_user, - sender_device, - conn_id.clone(), + &snake_key, "subscriptions".to_owned(), known_subscription_rooms, globalsince, @@ -340,11 +324,10 @@ async fn handle_lists<'a>( count: ruma_from_usize(active_rooms.len()), }); - if let Some(conn_id) = &body.conn_id { + if let Some(conn_id) = body.conn_id.clone() { + let snake_key = into_snake_key(sender_user, sender_device, conn_id); services.sync.update_snake_sync_known_rooms( - sender_user, - sender_device, - conn_id.clone(), + &snake_key, list_id.clone(), new_known_rooms, globalsince, diff --git a/src/service/sync/mod.rs b/src/service/sync/mod.rs index bf2bc142..b095d2c1 100644 --- a/src/service/sync/mod.rs +++ b/src/service/sync/mod.rs @@ -8,7 +8,7 @@ use std::{ use conduwuit::{Result, Server}; use database::Map; use ruma::{ - DeviceId, OwnedDeviceId, OwnedRoomId, OwnedUserId, UserId, + OwnedDeviceId, OwnedRoomId, OwnedUserId, api::client::sync::sync_events::{ self, v4::{ExtensionsConfig, SyncRequestList}, @@ -49,8 +49,8 @@ struct Services { struct SlidingSyncCache { lists: BTreeMap, subscriptions: BTreeMap, - known_rooms: BTreeMap>, /* For every room, the - * roomsince number */ + // For every room, the roomsince number + known_rooms: BTreeMap>, extensions: ExtensionsConfig, } @@ -98,79 +98,35 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -/// load params from cache if body doesn't contain it, as long as it's allowed -/// in some cases we may need to allow an empty list as an actual value -fn list_or_sticky(target: &mut Vec, cached: &Vec) { - if target.is_empty() { - target.clone_from(cached); - } -} -fn some_or_sticky(target: &mut Option, cached: Option) { - if target.is_none() { - *target = cached; - } -} - impl Service { - pub fn snake_connection_cached( - &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, - conn_id: Option, - ) -> bool { - self.snake_connections - .lock() - .unwrap() - .contains_key(&(user_id, device_id, conn_id)) - } - - pub fn forget_snake_sync_connection( - &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, - conn_id: Option, - ) { + pub fn snake_connection_cached(&self, key: &SnakeConnectionsKey) -> bool { self.snake_connections .lock() .expect("locked") - .remove(&(user_id, device_id, conn_id)); + .contains_key(key) } - pub fn remembered( - &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, - conn_id: String, - ) -> bool { - self.connections - .lock() - .unwrap() - .contains_key(&(user_id, device_id, conn_id)) + pub fn forget_snake_sync_connection(&self, key: &SnakeConnectionsKey) { + self.snake_connections.lock().expect("locked").remove(key); } - pub fn forget_sync_request_connection( - &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, - conn_id: String, - ) { - self.connections - .lock() - .expect("locked") - .remove(&(user_id, device_id, conn_id)); + pub fn remembered(&self, key: &DbConnectionsKey) -> bool { + self.connections.lock().expect("locked").contains_key(key) + } + + pub fn forget_sync_request_connection(&self, key: &DbConnectionsKey) { + self.connections.lock().expect("locked").remove(key); } pub fn update_snake_sync_request_with_cache( &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, + snake_key: &SnakeConnectionsKey, request: &mut v5::Request, ) -> BTreeMap> { - let conn_id = request.conn_id.clone(); let mut cache = self.snake_connections.lock().expect("locked"); let cached = Arc::clone( cache - .entry((user_id, device_id, conn_id)) + .entry(snake_key.clone()) .or_insert_with(|| Arc::new(Mutex::new(SnakeSyncCache::default()))), ); let cached = &mut cached.lock().expect("locked"); @@ -268,25 +224,23 @@ impl Service { pub fn update_sync_request_with_cache( &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, + key: &SnakeConnectionsKey, request: &mut sync_events::v4::Request, ) -> BTreeMap> { let Some(conn_id) = request.conn_id.clone() else { return BTreeMap::new(); }; + let key = into_db_key(key.0.clone(), key.1.clone(), conn_id); let mut cache = self.connections.lock().expect("locked"); - let cached = Arc::clone(cache.entry((user_id, device_id, conn_id)).or_insert_with( - || { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }, - )); + let cached = Arc::clone(cache.entry(key).or_insert_with(|| { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + })); let cached = &mut cached.lock().expect("locked"); drop(cache); @@ -371,22 +325,18 @@ impl Service { pub fn update_sync_subscriptions( &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, - conn_id: String, + key: &DbConnectionsKey, subscriptions: BTreeMap, ) { let mut cache = self.connections.lock().expect("locked"); - let cached = Arc::clone(cache.entry((user_id, device_id, conn_id)).or_insert_with( - || { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }, - )); + let cached = Arc::clone(cache.entry(key.clone()).or_insert_with(|| { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + })); let cached = &mut cached.lock().expect("locked"); drop(cache); @@ -395,90 +345,81 @@ impl Service { pub fn update_sync_known_rooms( &self, - user_id: &UserId, - device_id: &DeviceId, - conn_id: String, + key: &DbConnectionsKey, list_id: String, new_cached_rooms: BTreeSet, globalsince: u64, ) { let mut cache = self.connections.lock().expect("locked"); - let cached = Arc::clone( - cache - .entry((user_id.to_owned(), device_id.to_owned(), conn_id)) - .or_insert_with(|| { - Arc::new(Mutex::new(SlidingSyncCache { - lists: BTreeMap::new(), - subscriptions: BTreeMap::new(), - known_rooms: BTreeMap::new(), - extensions: ExtensionsConfig::default(), - })) - }), - ); + let cached = Arc::clone(cache.entry(key.clone()).or_insert_with(|| { + Arc::new(Mutex::new(SlidingSyncCache { + lists: BTreeMap::new(), + subscriptions: BTreeMap::new(), + known_rooms: BTreeMap::new(), + extensions: ExtensionsConfig::default(), + })) + })); let cached = &mut cached.lock().expect("locked"); drop(cache); - for (roomid, lastsince) in cached + for (room_id, lastsince) in cached .known_rooms .entry(list_id.clone()) .or_default() .iter_mut() { - if !new_cached_rooms.contains(roomid) { + if !new_cached_rooms.contains(room_id) { *lastsince = 0; } } let list = cached.known_rooms.entry(list_id).or_default(); - for roomid in new_cached_rooms { - list.insert(roomid, globalsince); + for room_id in new_cached_rooms { + list.insert(room_id, globalsince); } } pub fn update_snake_sync_known_rooms( &self, - user_id: &UserId, - device_id: &DeviceId, - conn_id: String, + key: &SnakeConnectionsKey, list_id: String, new_cached_rooms: BTreeSet, globalsince: u64, ) { + assert!(key.2.is_some(), "Some(conn_id) required for this call"); let mut cache = self.snake_connections.lock().expect("locked"); let cached = Arc::clone( cache - .entry((user_id.to_owned(), device_id.to_owned(), Some(conn_id))) + .entry(key.clone()) .or_insert_with(|| Arc::new(Mutex::new(SnakeSyncCache::default()))), ); let cached = &mut cached.lock().expect("locked"); drop(cache); - for (roomid, lastsince) in cached + for (room_id, lastsince) in cached .known_rooms .entry(list_id.clone()) .or_default() .iter_mut() { - if !new_cached_rooms.contains(roomid) { + if !new_cached_rooms.contains(room_id) { *lastsince = 0; } } let list = cached.known_rooms.entry(list_id).or_default(); - for roomid in new_cached_rooms { - list.insert(roomid, globalsince); + for room_id in new_cached_rooms { + list.insert(room_id, globalsince); } } pub fn update_snake_sync_subscriptions( &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, - conn_id: Option, + key: &SnakeConnectionsKey, subscriptions: BTreeMap, ) { let mut cache = self.snake_connections.lock().expect("locked"); let cached = Arc::clone( cache - .entry((user_id, device_id, conn_id)) + .entry(key.clone()) .or_insert_with(|| Arc::new(Mutex::new(SnakeSyncCache::default()))), ); let cached = &mut cached.lock().expect("locked"); @@ -487,3 +428,37 @@ impl Service { cached.subscriptions = subscriptions; } } + +#[inline] +pub fn into_snake_key(user_id: U, device_id: D, conn_id: C) -> SnakeConnectionsKey +where + U: Into, + D: Into, + C: Into>, +{ + (user_id.into(), device_id.into(), conn_id.into()) +} + +#[inline] +pub fn into_db_key(user_id: U, device_id: D, conn_id: C) -> DbConnectionsKey +where + U: Into, + D: Into, + C: Into, +{ + (user_id.into(), device_id.into(), conn_id.into()) +} + +/// load params from cache if body doesn't contain it, as long as it's allowed +/// in some cases we may need to allow an empty list as an actual value +fn list_or_sticky(target: &mut Vec, cached: &Vec) { + if target.is_empty() { + target.clone_from(cached); + } +} + +fn some_or_sticky(target: &mut Option, cached: Option) { + if target.is_none() { + *target = cached; + } +} From e8a64bb59d13aad7588a1ca28422f463e0f1a302 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Apr 2025 21:59:18 +0000 Subject: [PATCH 1700/2091] increase snake sync asynchronicity Signed-off-by: Jason Volk --- src/api/client/sync/mod.rs | 36 +----- src/api/client/sync/v3.rs | 17 ++- src/api/client/sync/v4.rs | 38 +++++- src/api/client/sync/v5.rs | 256 ++++++++++++++++++++++++------------- 4 files changed, 212 insertions(+), 135 deletions(-) diff --git a/src/api/client/sync/mod.rs b/src/api/client/sync/mod.rs index 14459acf..40370160 100644 --- a/src/api/client/sync/mod.rs +++ b/src/api/client/sync/mod.rs @@ -5,16 +5,12 @@ mod v5; use conduwuit::{ Error, PduCount, Result, matrix::pdu::PduEvent, - utils::{ - IterStream, - stream::{BroadbandExt, ReadyExt, TryIgnore}, - }, + utils::stream::{BroadbandExt, ReadyExt, TryIgnore}, }; use conduwuit_service::Services; use futures::{StreamExt, pin_mut}; use ruma::{ RoomId, UserId, - directory::RoomTypeFilter, events::TimelineEventType::{ self, Beacon, CallInvite, PollStart, RoomEncrypted, RoomMessage, Sticker, }, @@ -87,33 +83,3 @@ async fn share_encrypted_room( }) .await } - -pub(crate) async fn filter_rooms<'a>( - services: &Services, - rooms: &[&'a RoomId], - filter: &[RoomTypeFilter], - negate: bool, -) -> Vec<&'a RoomId> { - rooms - .iter() - .stream() - .filter_map(|r| async move { - let room_type = services.rooms.state_accessor.get_room_type(r).await; - - if room_type.as_ref().is_err_and(|e| !e.is_not_found()) { - return None; - } - - let room_type_filter = RoomTypeFilter::from(room_type.ok()); - - let include = if negate { - !filter.contains(&room_type_filter) - } else { - filter.is_empty() || filter.contains(&room_type_filter) - }; - - include.then_some(r) - }) - .collect() - .await -} diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 24930941..8eac6b66 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -14,8 +14,8 @@ use conduwuit::{ pair_of, ref_at, result::FlatOk, utils::{ - self, BoolExt, IterStream, ReadyExt, TryFutureExtExt, - future::OptionStream, + self, BoolExt, FutureBoolExt, IterStream, ReadyExt, TryFutureExtExt, + future::{OptionStream, ReadyEqExt}, math::ruma_from_u64, stream::{BroadbandExt, Tools, TryExpect, WidebandExt}, }, @@ -32,6 +32,7 @@ use conduwuit_service::{ use futures::{ FutureExt, StreamExt, TryFutureExt, TryStreamExt, future::{OptionFuture, join, join3, join4, join5, try_join, try_join4}, + pin_mut, }; use ruma::{ DeviceId, EventId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId, @@ -433,10 +434,14 @@ async fn handle_left_room( return Ok(None); } - if !services.rooms.metadata.exists(room_id).await - || services.rooms.metadata.is_disabled(room_id).await - || services.rooms.metadata.is_banned(room_id).await - { + let is_not_found = services.rooms.metadata.exists(room_id).eq(&false); + + let is_disabled = services.rooms.metadata.is_disabled(room_id); + + let is_banned = services.rooms.metadata.is_banned(room_id); + + pin_mut!(is_not_found, is_disabled, is_banned); + if is_not_found.or(is_disabled).or(is_banned).await { // This is just a rejected invite, not a room we know // Insert a leave event anyways for the client let event = PduEvent { diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 55faf420..f153b2da 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -7,6 +7,7 @@ use std::{ use axum::extract::State; use conduwuit::{ Err, Error, PduCount, PduEvent, Result, debug, error, extract_variant, + matrix::TypeStateKey, utils::{ BoolExt, IterStream, ReadyExt, TryFutureExtExt, math::{ruma_from_usize, usize_from_ruma, usize_from_u64_truncated}, @@ -14,6 +15,7 @@ use conduwuit::{ warn, }; use conduwuit_service::{ + Services, rooms::read_receipt::pack_receipts, sync::{into_db_key, into_snake_key}, }; @@ -24,6 +26,7 @@ use ruma::{ self, DeviceLists, UnreadNotificationsCount, v4::{SlidingOp, SlidingSyncRoomHero}, }, + directory::RoomTypeFilter, events::{ AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType::*, @@ -36,10 +39,11 @@ use ruma::{ use super::{load_timeline, share_encrypted_room}; use crate::{ Ruma, - client::{DEFAULT_BUMP_TYPES, filter_rooms, ignored_filter, sync::v5::TodoRooms}, + client::{DEFAULT_BUMP_TYPES, ignored_filter}, }; -pub(crate) const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; +type TodoRooms = BTreeMap, usize, u64)>; +const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; /// POST `/_matrix/client/unstable/org.matrix.msc3575/sync` /// @@ -802,3 +806,33 @@ pub(crate) async fn sync_events_v4_route( delta_token: None, }) } + +async fn filter_rooms<'a>( + services: &Services, + rooms: &[&'a RoomId], + filter: &[RoomTypeFilter], + negate: bool, +) -> Vec<&'a RoomId> { + rooms + .iter() + .stream() + .filter_map(|r| async move { + let room_type = services.rooms.state_accessor.get_room_type(r).await; + + if room_type.as_ref().is_err_and(|e| !e.is_not_found()) { + return None; + } + + let room_type_filter = RoomTypeFilter::from(room_type.ok()); + + let include = if negate { + !filter.contains(&room_type_filter) + } else { + filter.is_empty() || filter.contains(&room_type_filter) + }; + + include.then_some(r) + }) + .collect() + .await +} diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index 00a2d18d..f3fc0f44 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -1,28 +1,35 @@ use std::{ cmp::{self, Ordering}, collections::{BTreeMap, BTreeSet, HashMap, HashSet}, + ops::Deref, time::Duration, }; use axum::extract::State; use conduwuit::{ - Err, Error, Result, error, extract_variant, + Err, Error, Result, error, extract_variant, is_equal_to, matrix::{ TypeStateKey, pdu::{PduCount, PduEvent}, }, trace, utils::{ - BoolExt, IterStream, ReadyExt, TryFutureExtExt, + BoolExt, FutureBoolExt, IterStream, ReadyExt, TryFutureExtExt, + future::ReadyEqExt, math::{ruma_from_usize, usize_from_ruma}, }, warn, }; -use conduwuit_service::{rooms::read_receipt::pack_receipts, sync::into_snake_key}; -use futures::{FutureExt, StreamExt, TryFutureExt}; +use conduwuit_service::{Services, rooms::read_receipt::pack_receipts, sync::into_snake_key}; +use futures::{ + FutureExt, Stream, StreamExt, TryFutureExt, + future::{OptionFuture, join3, try_join4}, + pin_mut, +}; use ruma::{ DeviceId, OwnedEventId, OwnedRoomId, RoomId, UInt, UserId, api::client::sync::sync_events::{self, DeviceLists, UnreadNotificationsCount}, + directory::RoomTypeFilter, events::{ AnyRawAccountDataEvent, AnySyncEphemeralRoomEvent, StateEventType, TimelineEventType, room::member::{MembershipState, RoomMemberEventContent}, @@ -31,13 +38,15 @@ use ruma::{ uint, }; -use super::{filter_rooms, share_encrypted_room}; +use super::share_encrypted_room; use crate::{ Ruma, client::{DEFAULT_BUMP_TYPES, ignored_filter, sync::load_timeline}, }; type SyncInfo<'a> = (&'a UserId, &'a DeviceId, u64, &'a sync_events::v5::Request); +type TodoRooms = BTreeMap, usize, u64)>; +type KnownRooms = BTreeMap>; /// `POST /_matrix/client/unstable/org.matrix.simplified_msc3575/sync` /// ([MSC4186]) @@ -50,7 +59,7 @@ type SyncInfo<'a> = (&'a UserId, &'a DeviceId, u64, &'a sync_events::v5::Request /// [MSC3575]: https://github.com/matrix-org/matrix-spec-proposals/pull/3575 /// [MSC4186]: https://github.com/matrix-org/matrix-spec-proposals/pull/4186 pub(crate) async fn sync_events_v5_route( - State(services): State, + State(ref services): State, body: Ruma, ) -> Result { debug_assert!(DEFAULT_BUMP_TYPES.is_sorted(), "DEFAULT_BUMP_TYPES is not sorted"); @@ -89,65 +98,77 @@ pub(crate) async fn sync_events_v5_route( .sync .update_snake_sync_request_with_cache(&snake_key, &mut body); - let all_joined_rooms: Vec<_> = services + let all_joined_rooms = services .rooms .state_cache .rooms_joined(sender_user) .map(ToOwned::to_owned) - .collect() - .await; + .collect::>(); - let all_invited_rooms: Vec<_> = services + let all_invited_rooms = services .rooms .state_cache .rooms_invited(sender_user) .map(|r| r.0) - .collect() - .await; + .collect::>(); - let all_knocked_rooms: Vec<_> = services + let all_knocked_rooms = services .rooms .state_cache .rooms_knocked(sender_user) .map(|r| r.0) - .collect() - .await; + .collect::>(); - let all_rooms: Vec<&RoomId> = all_joined_rooms - .iter() - .map(AsRef::as_ref) - .chain(all_invited_rooms.iter().map(AsRef::as_ref)) - .chain(all_knocked_rooms.iter().map(AsRef::as_ref)) - .collect(); + let (all_joined_rooms, all_invited_rooms, all_knocked_rooms) = + join3(all_joined_rooms, all_invited_rooms, all_knocked_rooms).await; - let all_joined_rooms = all_joined_rooms.iter().map(AsRef::as_ref).collect(); - let all_invited_rooms = all_invited_rooms.iter().map(AsRef::as_ref).collect(); + let all_joined_rooms = all_joined_rooms.iter().map(AsRef::as_ref); + let all_invited_rooms = all_invited_rooms.iter().map(AsRef::as_ref); + let all_knocked_rooms = all_knocked_rooms.iter().map(AsRef::as_ref); + let all_rooms = all_joined_rooms + .clone() + .chain(all_invited_rooms.clone()) + .chain(all_knocked_rooms.clone()); let pos = next_batch.clone().to_string(); let mut todo_rooms: TodoRooms = BTreeMap::new(); let sync_info: SyncInfo<'_> = (sender_user, sender_device, globalsince, &body); + + let account_data = collect_account_data(services, sync_info).map(Ok); + + let e2ee = collect_e2ee(services, sync_info, all_joined_rooms.clone()); + + let to_device = collect_to_device(services, sync_info, next_batch).map(Ok); + + let receipts = collect_receipts(services).map(Ok); + + let (account_data, e2ee, to_device, receipts) = + try_join4(account_data, e2ee, to_device, receipts).await?; + + let extensions = sync_events::v5::response::Extensions { + account_data, + e2ee, + to_device, + receipts, + typing: sync_events::v5::response::Typing::default(), + }; + let mut response = sync_events::v5::Response { txn_id: body.txn_id.clone(), pos, lists: BTreeMap::new(), rooms: BTreeMap::new(), - extensions: sync_events::v5::response::Extensions { - account_data: collect_account_data(services, sync_info).await, - e2ee: collect_e2ee(services, sync_info, &all_joined_rooms).await?, - to_device: collect_to_device(services, sync_info, next_batch).await, - receipts: collect_receipts(services).await, - typing: sync_events::v5::response::Typing::default(), - }, + extensions, }; handle_lists( services, sync_info, - &all_invited_rooms, - &all_joined_rooms, - &all_rooms, + all_invited_rooms.clone(), + all_joined_rooms.clone(), + all_rooms, &mut todo_rooms, &known_rooms, &mut response, @@ -160,7 +181,7 @@ pub(crate) async fn sync_events_v5_route( services, sender_user, next_batch, - &all_invited_rooms, + all_invited_rooms.clone(), &todo_rooms, &mut response, &body, @@ -185,31 +206,33 @@ pub(crate) async fn sync_events_v5_route( } trace!( - rooms=?response.rooms.len(), - account_data=?response.extensions.account_data.rooms.len(), - receipts=?response.extensions.receipts.rooms.len(), + rooms = ?response.rooms.len(), + account_data = ?response.extensions.account_data.rooms.len(), + receipts = ?response.extensions.receipts.rooms.len(), "responding to request with" ); Ok(response) } -type KnownRooms = BTreeMap>; -pub(crate) type TodoRooms = BTreeMap, usize, u64)>; - async fn fetch_subscriptions( - services: crate::State, + services: &Services, (sender_user, sender_device, globalsince, body): SyncInfo<'_>, known_rooms: &KnownRooms, todo_rooms: &mut TodoRooms, ) { let mut known_subscription_rooms = BTreeSet::new(); for (room_id, room) in &body.room_subscriptions { - if !services.rooms.metadata.exists(room_id).await - || services.rooms.metadata.is_disabled(room_id).await - || services.rooms.metadata.is_banned(room_id).await - { + let not_exists = services.rooms.metadata.exists(room_id).eq(&false); + + let is_disabled = services.rooms.metadata.is_disabled(room_id); + + let is_banned = services.rooms.metadata.is_banned(room_id); + + pin_mut!(not_exists, is_disabled, is_banned); + if not_exists.or(is_disabled).or(is_banned).await { continue; } + let todo_room = todo_rooms .entry(room_id.clone()) @@ -251,27 +274,39 @@ async fn fetch_subscriptions( } #[allow(clippy::too_many_arguments)] -async fn handle_lists<'a>( - services: crate::State, +async fn handle_lists<'a, Rooms, AllRooms>( + services: &Services, (sender_user, sender_device, globalsince, body): SyncInfo<'_>, - all_invited_rooms: &Vec<&'a RoomId>, - all_joined_rooms: &Vec<&'a RoomId>, - all_rooms: &Vec<&'a RoomId>, + all_invited_rooms: Rooms, + all_joined_rooms: Rooms, + all_rooms: AllRooms, todo_rooms: &'a mut TodoRooms, known_rooms: &'a KnownRooms, response: &'_ mut sync_events::v5::Response, -) -> KnownRooms { +) -> KnownRooms +where + Rooms: Iterator + Clone + Send + 'a, + AllRooms: Iterator + Clone + Send + 'a, +{ for (list_id, list) in &body.lists { - let active_rooms = match list.filters.clone().and_then(|f| f.is_invite) { - | Some(true) => all_invited_rooms, - | Some(false) => all_joined_rooms, - | None => all_rooms, + let active_rooms: Vec<_> = match list.filters.as_ref().and_then(|f| f.is_invite) { + | None => all_rooms.clone().collect(), + | Some(true) => all_invited_rooms.clone().collect(), + | Some(false) => all_joined_rooms.clone().collect(), }; - let active_rooms = match list.filters.clone().map(|f| f.not_room_types) { - | Some(filter) if filter.is_empty() => active_rooms, - | Some(value) => &filter_rooms(&services, active_rooms, &value, true).await, + let active_rooms = match list.filters.as_ref().map(|f| &f.not_room_types) { | None => active_rooms, + | Some(filter) if filter.is_empty() => active_rooms, + | Some(value) => + filter_rooms( + services, + value, + &true, + active_rooms.iter().stream().map(Deref::deref), + ) + .collect() + .await, }; let mut new_known_rooms: BTreeSet = BTreeSet::new(); @@ -289,6 +324,7 @@ async fn handle_lists<'a>( let new_rooms: BTreeSet = room_ids.clone().into_iter().map(From::from).collect(); + new_known_rooms.extend(new_rooms); //new_known_rooms.extend(room_ids..cloned()); for room_id in room_ids { @@ -334,18 +370,22 @@ async fn handle_lists<'a>( ); } } + BTreeMap::default() } -async fn process_rooms( - services: crate::State, +async fn process_rooms<'a, Rooms>( + services: &Services, sender_user: &UserId, next_batch: u64, - all_invited_rooms: &[&RoomId], + all_invited_rooms: Rooms, todo_rooms: &TodoRooms, response: &mut sync_events::v5::Response, body: &sync_events::v5::Request, -) -> Result> { +) -> Result> +where + Rooms: Iterator + Clone + Send + 'a, +{ let mut rooms = BTreeMap::new(); for (room_id, (required_state_request, timeline_limit, roomsince)) in todo_rooms { let roomsincecount = PduCount::Normal(*roomsince); @@ -354,7 +394,7 @@ async fn process_rooms( let mut invite_state = None; let (timeline_pdus, limited); let new_room_id: &RoomId = (*room_id).as_ref(); - if all_invited_rooms.contains(&new_room_id) { + if all_invited_rooms.clone().any(is_equal_to!(new_room_id)) { // TODO: figure out a timestamp we can use for remote invites invite_state = services .rooms @@ -366,7 +406,7 @@ async fn process_rooms( (timeline_pdus, limited) = (Vec::new(), true); } else { (timeline_pdus, limited) = match load_timeline( - &services, + services, sender_user, room_id, roomsincecount, @@ -399,18 +439,17 @@ async fn process_rooms( .rooms .read_receipt .last_privateread_update(sender_user, room_id) - .await > *roomsince; + .await; - let private_read_event = if last_privateread_update { - services - .rooms - .read_receipt - .private_read_get(room_id, sender_user) - .await - .ok() - } else { - None - }; + let private_read_event: OptionFuture<_> = (last_privateread_update > *roomsince) + .then(|| { + services + .rooms + .read_receipt + .private_read_get(room_id, sender_user) + .ok() + }) + .into(); let mut receipts: Vec> = services .rooms @@ -426,7 +465,7 @@ async fn process_rooms( .collect() .await; - if let Some(private_read_event) = private_read_event { + if let Some(private_read_event) = private_read_event.await.flatten() { receipts.push(private_read_event); } @@ -475,7 +514,7 @@ async fn process_rooms( let room_events: Vec<_> = timeline_pdus .iter() .stream() - .filter_map(|item| ignored_filter(&services, item.clone(), sender_user)) + .filter_map(|item| ignored_filter(services, item.clone(), sender_user)) .map(|(_, pdu)| pdu.to_sync_room_event()) .collect() .await; @@ -627,7 +666,7 @@ async fn process_rooms( Ok(rooms) } async fn collect_account_data( - services: crate::State, + services: &Services, (sender_user, _, globalsince, body): (&UserId, &DeviceId, u64, &sync_events::v5::Request), ) -> sync_events::v5::response::AccountData { let mut account_data = sync_events::v5::response::AccountData { @@ -663,16 +702,19 @@ async fn collect_account_data( account_data } -async fn collect_e2ee<'a>( - services: crate::State, +async fn collect_e2ee<'a, Rooms>( + services: &Services, (sender_user, sender_device, globalsince, body): ( &UserId, &DeviceId, u64, &sync_events::v5::Request, ), - all_joined_rooms: &'a Vec<&'a RoomId>, -) -> Result { + all_joined_rooms: Rooms, +) -> Result +where + Rooms: Iterator + Send + 'a, +{ if !body.extensions.e2ee.enabled.unwrap_or(false) { return Ok(sync_events::v5::response::E2EE::default()); } @@ -773,7 +815,7 @@ async fn collect_e2ee<'a>( | MembershipState::Join => { // A new user joined an encrypted room if !share_encrypted_room( - &services, + services, sender_user, user_id, Some(room_id), @@ -806,7 +848,7 @@ async fn collect_e2ee<'a>( // Only send keys if the sender doesn't share an encrypted room with the target // already .filter_map(|user_id| { - share_encrypted_room(&services, sender_user, user_id, Some(room_id)) + share_encrypted_room(services, sender_user, user_id, Some(room_id)) .map(|res| res.or_some(user_id.to_owned())) }) .collect::>() @@ -829,7 +871,7 @@ async fn collect_e2ee<'a>( for user_id in left_encrypted_users { let dont_share_encrypted_room = - !share_encrypted_room(&services, sender_user, &user_id, None).await; + !share_encrypted_room(services, sender_user, &user_id, None).await; // If the user doesn't share an encrypted room with the target anymore, we need // to tell them @@ -839,20 +881,22 @@ async fn collect_e2ee<'a>( } Ok(sync_events::v5::response::E2EE { - device_lists: DeviceLists { - changed: device_list_changes.into_iter().collect(), - left: device_list_left.into_iter().collect(), - }, + device_unused_fallback_key_types: None, + device_one_time_keys_count: services .users .count_one_time_keys(sender_user, sender_device) .await, - device_unused_fallback_key_types: None, + + device_lists: DeviceLists { + changed: device_list_changes.into_iter().collect(), + left: device_list_left.into_iter().collect(), + }, }) } async fn collect_to_device( - services: crate::State, + services: &Services, (sender_user, sender_device, globalsince, body): SyncInfo<'_>, next_batch: u64, ) -> Option { @@ -875,7 +919,35 @@ async fn collect_to_device( }) } -async fn collect_receipts(_services: crate::State) -> sync_events::v5::response::Receipts { +async fn collect_receipts(_services: &Services) -> sync_events::v5::response::Receipts { sync_events::v5::response::Receipts { rooms: BTreeMap::new() } // TODO: get explicitly requested read receipts } + +fn filter_rooms<'a, Rooms>( + services: &'a Services, + filter: &'a [RoomTypeFilter], + negate: &'a bool, + rooms: Rooms, +) -> impl Stream + Send + 'a +where + Rooms: Stream + Send + 'a, +{ + rooms.filter_map(async |room_id| { + let room_type = services.rooms.state_accessor.get_room_type(room_id).await; + + if room_type.as_ref().is_err_and(|e| !e.is_not_found()) { + return None; + } + + let room_type_filter = RoomTypeFilter::from(room_type.ok()); + + let include = if *negate { + !filter.contains(&room_type_filter) + } else { + filter.is_empty() || filter.contains(&room_type_filter) + }; + + include.then_some(room_id) + }) +} From eac713a2a91569ed0736aa8c88cfe543b44ec4fb Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Apr 2025 22:50:01 +0000 Subject: [PATCH 1701/2091] slightly optimize user directory search loop Signed-off-by: Jason Volk --- src/api/client/user_directory.rs | 43 ++++++++++++++++---------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index 99b3bb67..748fc049 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -1,7 +1,10 @@ use axum::extract::State; use conduwuit::{ Result, - utils::{future::BoolExt, stream::BroadbandExt}, + utils::{ + future::BoolExt, + stream::{BroadbandExt, ReadyExt}, + }, }; use futures::{FutureExt, StreamExt, pin_mut}; use ruma::{ @@ -30,29 +33,21 @@ pub(crate) async fn search_users_route( .map_or(LIMIT_DEFAULT, usize::from) .min(LIMIT_MAX); + let search_term = body.search_term.to_lowercase(); let mut users = services .users .stream() + .ready_filter(|user_id| user_id.as_str().to_lowercase().contains(&search_term)) .map(ToOwned::to_owned) .broad_filter_map(async |user_id| { - let user = search_users::v3::User { - user_id: user_id.clone(), - display_name: services.users.displayname(&user_id).await.ok(), - avatar_url: services.users.avatar_url(&user_id).await.ok(), - }; + let display_name = services.users.displayname(&user_id).await.ok(); - let user_id_matches = user - .user_id - .as_str() - .to_lowercase() - .contains(&body.search_term.to_lowercase()); + let display_name_matches = display_name + .as_deref() + .map(str::to_lowercase) + .is_some_and(|display_name| display_name.contains(&search_term)); - let user_displayname_matches = user.display_name.as_ref().is_some_and(|name| { - name.to_lowercase() - .contains(&body.search_term.to_lowercase()) - }); - - if !user_id_matches && !user_displayname_matches { + if !display_name_matches { return None; } @@ -61,11 +56,11 @@ pub(crate) async fn search_users_route( .state_cache .rooms_joined(&user_id) .map(ToOwned::to_owned) - .any(|room| async move { + .broad_any(async |room_id| { services .rooms .state_accessor - .get_join_rules(&room) + .get_join_rules(&room_id) .map(|rule| matches!(rule, JoinRule::Public)) .await }); @@ -76,8 +71,14 @@ pub(crate) async fn search_users_route( .user_sees_user(sender_user, &user_id); pin_mut!(user_in_public_room, user_sees_user); - - user_in_public_room.or(user_sees_user).await.then_some(user) + user_in_public_room + .or(user_sees_user) + .await + .then_some(search_users::v3::User { + user_id: user_id.clone(), + display_name, + avatar_url: services.users.avatar_url(&user_id).await.ok(), + }) }); let results = users.by_ref().take(limit).collect().await; From 83126cc66765541c88168e0ada63c41ed9f07058 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 9 Apr 2025 03:35:40 +0000 Subject: [PATCH 1702/2091] propagate better message from RustlsConfig load error. (#734) Signed-off-by: Jason Volk --- src/router/serve/tls.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/router/serve/tls.rs b/src/router/serve/tls.rs index dd46ab53..20b58601 100644 --- a/src/router/serve/tls.rs +++ b/src/router/serve/tls.rs @@ -31,12 +31,14 @@ pub(super) async fn serve( .install_default() .expect("failed to initialise aws-lc-rs rustls crypto provider"); - debug!("Using direct TLS. Certificate path {certs} and certificate private key path {key}",); info!( "Note: It is strongly recommended that you use a reverse proxy instead of running \ conduwuit directly with TLS." ); - let conf = RustlsConfig::from_pem_file(certs, key).await?; + debug!("Using direct TLS. Certificate path {certs} and certificate private key path {key}",); + let conf = RustlsConfig::from_pem_file(certs, key) + .await + .map_err(|e| err!(Config("tls", "Failed to load certificates or key: {e}")))?; let mut join_set = JoinSet::new(); let app = app.into_make_service_with_connect_info::(); From b3e5d2f683d633c948cc66b6cec77bbca6952f91 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 8 Apr 2025 04:39:01 +0000 Subject: [PATCH 1703/2091] remove box ids from admin room command arguments Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 33 +++++++++++----------- src/admin/debug/mod.rs | 32 ++++++++++----------- src/admin/federation/commands.rs | 10 +++---- src/admin/federation/mod.rs | 10 +++---- src/admin/media/commands.rs | 8 +++--- src/admin/media/mod.rs | 8 +++--- src/admin/query/account_data.rs | 18 ++++++------ src/admin/query/appservice.rs | 2 +- src/admin/query/globals.rs | 4 +-- src/admin/query/presence.rs | 4 +-- src/admin/query/pusher.rs | 4 +-- src/admin/query/room_alias.rs | 6 ++-- src/admin/query/room_state_cache.rs | 44 +++++++++++++++-------------- src/admin/query/sending.rs | 20 ++++++------- src/admin/room/alias.rs | 8 ++---- src/admin/room/directory.rs | 6 ++-- src/admin/room/info.rs | 10 +++---- src/admin/room/moderation.rs | 10 +++---- src/admin/user/commands.rs | 10 +++---- src/admin/user/mod.rs | 10 +++---- 20 files changed, 128 insertions(+), 129 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 87ca03a0..6d0e375a 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -17,10 +17,9 @@ use conduwuit::{ }; use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ - CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomOrAliasId, RoomId, RoomVersionId, - ServerName, - api::{client::error::ErrorKind, federation::event::get_room_state}, - events::room::message::RoomMessageEventContent, + CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, + OwnedRoomOrAliasId, OwnedServerName, RoomId, RoomVersionId, + api::federation::event::get_room_state, events::room::message::RoomMessageEventContent, }; use service::rooms::{ short::{ShortEventId, ShortRoomId}, @@ -40,7 +39,7 @@ pub(super) async fn echo(&self, message: Vec) -> Result, + event_id: OwnedEventId, ) -> Result { let Ok(event) = self.services.rooms.timeline.get_pdu_json(&event_id).await else { return Ok(RoomMessageEventContent::notice_plain("Event not found.")); @@ -109,7 +108,7 @@ pub(super) async fn parse_pdu(&self) -> Result { } #[admin_command] -pub(super) async fn get_pdu(&self, event_id: Box) -> Result { +pub(super) async fn get_pdu(&self, event_id: OwnedEventId) -> Result { let mut outlier = false; let mut pdu_json = self .services @@ -173,7 +172,7 @@ pub(super) async fn get_short_pdu( #[admin_command] pub(super) async fn get_remote_pdu_list( &self, - server: Box, + server: OwnedServerName, force: bool, ) -> Result { if !self.services.server.config.allow_federation { @@ -359,7 +358,7 @@ pub(super) async fn get_room_state( } #[admin_command] -pub(super) async fn ping(&self, server: Box) -> Result { +pub(super) async fn ping(&self, server: OwnedServerName) -> Result { if server == self.services.globals.server_name() { return Ok(RoomMessageEventContent::text_plain( "Not allowed to send federation requests to ourselves.", @@ -538,7 +537,7 @@ pub(super) async fn verify_json(&self) -> Result { } #[admin_command] -pub(super) async fn verify_pdu(&self, event_id: Box) -> Result { +pub(super) async fn verify_pdu(&self, event_id: OwnedEventId) -> Result { let mut event = self.services.rooms.timeline.get_pdu_json(&event_id).await?; event.remove("event_id"); @@ -556,7 +555,7 @@ pub(super) async fn verify_pdu(&self, event_id: Box) -> Result, + room_id: OwnedRoomId, ) -> Result { if !self .services @@ -585,7 +584,7 @@ pub(super) async fn first_pdu_in_room( #[tracing::instrument(skip(self))] pub(super) async fn latest_pdu_in_room( &self, - room_id: Box, + room_id: OwnedRoomId, ) -> Result { if !self .services @@ -614,8 +613,8 @@ pub(super) async fn latest_pdu_in_room( #[tracing::instrument(skip(self))] pub(super) async fn force_set_room_state_from_server( &self, - room_id: Box, - server_name: Box, + room_id: OwnedRoomId, + server_name: OwnedServerName, ) -> Result { if !self .services @@ -763,8 +762,8 @@ pub(super) async fn force_set_room_state_from_server( #[admin_command] pub(super) async fn get_signing_keys( &self, - server_name: Option>, - notary: Option>, + server_name: Option, + notary: Option, query: bool, ) -> Result { let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone().into()); @@ -801,7 +800,7 @@ pub(super) async fn get_signing_keys( #[admin_command] pub(super) async fn get_verify_keys( &self, - server_name: Option>, + server_name: Option, ) -> Result { let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone().into()); @@ -824,7 +823,7 @@ pub(super) async fn get_verify_keys( #[admin_command] pub(super) async fn resolve_true_destination( &self, - server_name: Box, + server_name: OwnedServerName, no_cache: bool, ) -> Result { if !self.services.server.config.allow_federation { diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index db04ccf4..9b86f18c 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -3,7 +3,7 @@ pub(crate) mod tester; use clap::Subcommand; use conduwuit::Result; -use ruma::{EventId, OwnedRoomOrAliasId, RoomId, ServerName}; +use ruma::{OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedServerName}; use service::rooms::short::{ShortEventId, ShortRoomId}; use self::tester::TesterCommand; @@ -20,7 +20,7 @@ pub(super) enum DebugCommand { /// - Get the auth_chain of a PDU GetAuthChain { /// An event ID (the $ character followed by the base64 reference hash) - event_id: Box, + event_id: OwnedEventId, }, /// - Parse and print a PDU from a JSON @@ -35,7 +35,7 @@ pub(super) enum DebugCommand { /// - Retrieve and print a PDU by EventID from the conduwuit database GetPdu { /// An event ID (a $ followed by the base64 reference hash) - event_id: Box, + event_id: OwnedEventId, }, /// - Retrieve and print a PDU by PduId from the conduwuit database @@ -52,11 +52,11 @@ pub(super) enum DebugCommand { /// (following normal event auth rules, handles it as an incoming PDU). GetRemotePdu { /// An event ID (a $ followed by the base64 reference hash) - event_id: Box, + event_id: OwnedEventId, /// Argument for us to attempt to fetch the event from the /// specified remote server. - server: Box, + server: OwnedServerName, }, /// - Same as `get-remote-pdu` but accepts a codeblock newline delimited @@ -64,7 +64,7 @@ pub(super) enum DebugCommand { GetRemotePduList { /// Argument for us to attempt to fetch all the events from the /// specified remote server. - server: Box, + server: OwnedServerName, /// If set, ignores errors, else stops at the first error/failure. #[arg(short, long)] @@ -88,10 +88,10 @@ pub(super) enum DebugCommand { /// - Get and display signing keys from local cache or remote server. GetSigningKeys { - server_name: Option>, + server_name: Option, #[arg(long)] - notary: Option>, + notary: Option, #[arg(short, long)] query: bool, @@ -99,14 +99,14 @@ pub(super) enum DebugCommand { /// - Get and display signing keys from local cache or remote server. GetVerifyKeys { - server_name: Option>, + server_name: Option, }, /// - Sends a federation request to the remote server's /// `/_matrix/federation/v1/version` endpoint and measures the latency it /// took for the server to respond Ping { - server: Box, + server: OwnedServerName, }, /// - Forces device lists for all local and remote users to be updated (as @@ -141,21 +141,21 @@ pub(super) enum DebugCommand { /// /// This re-verifies a PDU existing in the database found by ID. VerifyPdu { - event_id: Box, + event_id: OwnedEventId, }, /// - Prints the very first PDU in the specified room (typically /// m.room.create) FirstPduInRoom { /// The room ID - room_id: Box, + room_id: OwnedRoomId, }, /// - Prints the latest ("last") PDU in the specified room (typically a /// message) LatestPduInRoom { /// The room ID - room_id: Box, + room_id: OwnedRoomId, }, /// - Forcefully replaces the room state of our local copy of the specified @@ -174,9 +174,9 @@ pub(super) enum DebugCommand { /// `/_matrix/federation/v1/state/{roomId}`. ForceSetRoomStateFromServer { /// The impacted room ID - room_id: Box, + room_id: OwnedRoomId, /// The server we will use to query the room state for - server_name: Box, + server_name: OwnedServerName, }, /// - Runs a server name through conduwuit's true destination resolution @@ -184,7 +184,7 @@ pub(super) enum DebugCommand { /// /// Useful for debugging well-known issues ResolveTrueDestination { - server_name: Box, + server_name: OwnedServerName, #[arg(short, long)] no_cache: bool, diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 240ffa6a..12ed9c25 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -3,19 +3,19 @@ use std::fmt::Write; use conduwuit::Result; use futures::StreamExt; use ruma::{ - OwnedRoomId, RoomId, ServerName, UserId, events::room::message::RoomMessageEventContent, + OwnedRoomId, OwnedServerName, OwnedUserId, events::room::message::RoomMessageEventContent, }; use crate::{admin_command, get_room_info}; #[admin_command] -pub(super) async fn disable_room(&self, room_id: Box) -> Result { +pub(super) async fn disable_room(&self, room_id: OwnedRoomId) -> Result { self.services.rooms.metadata.disable_room(&room_id, true); Ok(RoomMessageEventContent::text_plain("Room disabled.")) } #[admin_command] -pub(super) async fn enable_room(&self, room_id: Box) -> Result { +pub(super) async fn enable_room(&self, room_id: OwnedRoomId) -> Result { self.services.rooms.metadata.disable_room(&room_id, false); Ok(RoomMessageEventContent::text_plain("Room enabled.")) } @@ -42,7 +42,7 @@ pub(super) async fn incoming_federation(&self) -> Result, + server_name: OwnedServerName, ) -> Result { let response = self .services @@ -90,7 +90,7 @@ pub(super) async fn fetch_support_well_known( #[admin_command] pub(super) async fn remote_user_in_rooms( &self, - user_id: Box, + user_id: OwnedUserId, ) -> Result { if user_id.server_name() == self.services.server.name { return Ok(RoomMessageEventContent::text_plain( diff --git a/src/admin/federation/mod.rs b/src/admin/federation/mod.rs index 3adfd459..2c539adc 100644 --- a/src/admin/federation/mod.rs +++ b/src/admin/federation/mod.rs @@ -2,7 +2,7 @@ mod commands; use clap::Subcommand; use conduwuit::Result; -use ruma::{RoomId, ServerName, UserId}; +use ruma::{OwnedRoomId, OwnedServerName, OwnedUserId}; use crate::admin_command_dispatch; @@ -14,12 +14,12 @@ pub(super) enum FederationCommand { /// - Disables incoming federation handling for a room. DisableRoom { - room_id: Box, + room_id: OwnedRoomId, }, /// - Enables incoming federation handling for a room again. EnableRoom { - room_id: Box, + room_id: OwnedRoomId, }, /// - Fetch `/.well-known/matrix/support` from the specified server @@ -32,11 +32,11 @@ pub(super) enum FederationCommand { /// moderation, and security inquiries. This command provides a way to /// easily fetch that information. FetchSupportWellKnown { - server_name: Box, + server_name: OwnedServerName, }, /// - Lists all the rooms we share/track with the specified *remote* user RemoteUserInRooms { - user_id: Box, + user_id: OwnedUserId, }, } diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index aeefa9f2..c8364969 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -5,7 +5,7 @@ use conduwuit::{ }; use conduwuit_service::media::Dim; use ruma::{ - EventId, Mxc, MxcUri, OwnedMxcUri, OwnedServerName, ServerName, + Mxc, OwnedEventId, OwnedMxcUri, OwnedServerName, events::room::message::RoomMessageEventContent, }; @@ -14,8 +14,8 @@ use crate::{admin_command, utils::parse_local_user_id}; #[admin_command] pub(super) async fn delete( &self, - mxc: Option>, - event_id: Option>, + mxc: Option, + event_id: Option, ) -> Result { if event_id.is_some() && mxc.is_some() { return Ok(RoomMessageEventContent::text_plain( @@ -282,7 +282,7 @@ pub(super) async fn delete_all_from_user( #[admin_command] pub(super) async fn delete_all_from_server( &self, - server_name: Box, + server_name: OwnedServerName, yes_i_want_to_delete_local_media: bool, ) -> Result { if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media { diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index 641834b2..d1e6cd3a 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -3,7 +3,7 @@ mod commands; use clap::Subcommand; use conduwuit::Result; -use ruma::{EventId, MxcUri, OwnedMxcUri, OwnedServerName, ServerName}; +use ruma::{OwnedEventId, OwnedMxcUri, OwnedServerName}; use crate::admin_command_dispatch; @@ -15,12 +15,12 @@ pub(super) enum MediaCommand { Delete { /// The MXC URL to delete #[arg(long)] - mxc: Option>, + mxc: Option, /// - The message event ID which contains the media and thumbnail MXC /// URLs #[arg(long)] - event_id: Option>, + event_id: Option, }, /// - Deletes a codeblock list of MXC URLs from our database and on the @@ -57,7 +57,7 @@ pub(super) enum MediaCommand { /// - Deletes all remote media from the specified remote server. This will /// always ignore errors by default. DeleteAllFromServer { - server_name: Box, + server_name: OwnedServerName, /// Long argument to delete local media #[arg(long)] diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index b2bf5e6d..879aed16 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{RoomId, UserId, events::room::message::RoomMessageEventContent}; +use ruma::{OwnedRoomId, OwnedUserId, events::room::message::RoomMessageEventContent}; use crate::{admin_command, admin_command_dispatch}; @@ -12,30 +12,30 @@ pub(crate) enum AccountDataCommand { /// - Returns all changes to the account data that happened after `since`. ChangesSince { /// Full user ID - user_id: Box, + user_id: OwnedUserId, /// UNIX timestamp since (u64) since: u64, /// Optional room ID of the account data - room_id: Option>, + room_id: Option, }, /// - Searches the account data for a specific kind. AccountDataGet { /// Full user ID - user_id: Box, + user_id: OwnedUserId, /// Account data event type kind: String, /// Optional room ID of the account data - room_id: Option>, + room_id: Option, }, } #[admin_command] async fn changes_since( &self, - user_id: Box, + user_id: OwnedUserId, since: u64, - room_id: Option>, + room_id: Option, ) -> Result { let timer = tokio::time::Instant::now(); let results: Vec<_> = self @@ -54,9 +54,9 @@ async fn changes_since( #[admin_command] async fn account_data_get( &self, - user_id: Box, + user_id: OwnedUserId, kind: String, - room_id: Option>, + room_id: Option, ) -> Result { let timer = tokio::time::Instant::now(); let results = self diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index f9e1fd2c..93c76a7e 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -9,7 +9,7 @@ pub(crate) enum AppserviceCommand { /// - Gets the appservice registration info/details from the ID as a string GetRegistration { /// Appservice registration ID - appservice_id: Box, + appservice_id: String, }, /// - Gets all appservice registrations with their ID and registration info diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 1642f7cd..33810704 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -1,6 +1,6 @@ use clap::Subcommand; use conduwuit::Result; -use ruma::ServerName; +use ruma::OwnedServerName; use crate::Command; @@ -16,7 +16,7 @@ pub(crate) enum GlobalsCommand { /// - This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. SigningKeysFor { - origin: Box, + origin: OwnedServerName, }, } diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index 38272749..65164802 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::UserId; +use ruma::OwnedUserId; use crate::Command; @@ -11,7 +11,7 @@ pub(crate) enum PresenceCommand { /// - Returns the latest presence event for the given user. GetPresence { /// Full user ID - user_id: Box, + user_id: OwnedUserId, }, /// - Iterator of the most recent presence updates that happened after the diff --git a/src/admin/query/pusher.rs b/src/admin/query/pusher.rs index 34edf4db..583c4999 100644 --- a/src/admin/query/pusher.rs +++ b/src/admin/query/pusher.rs @@ -1,6 +1,6 @@ use clap::Subcommand; use conduwuit::Result; -use ruma::UserId; +use ruma::OwnedUserId; use crate::Command; @@ -9,7 +9,7 @@ pub(crate) enum PusherCommand { /// - Returns all the pushers for the user. GetPushers { /// Full user ID - user_id: Box, + user_id: OwnedUserId, }, } diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index 2d4d8104..f58f6717 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{RoomAliasId, RoomId}; +use ruma::{OwnedRoomAliasId, OwnedRoomId}; use crate::Command; @@ -10,13 +10,13 @@ use crate::Command; pub(crate) enum RoomAliasCommand { ResolveLocalAlias { /// Full room alias - alias: Box, + alias: OwnedRoomAliasId, }, /// - Iterator of all our local room aliases for the room ID LocalAliasesForRoom { /// Full room ID - room_id: Box, + room_id: OwnedRoomId, }, /// - Iterator of all our local aliases in our database with their room IDs diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index 1de5c02d..7f5e2536 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -1,78 +1,80 @@ use clap::Subcommand; use conduwuit::{Error, Result}; use futures::StreamExt; -use ruma::{RoomId, ServerName, UserId, events::room::message::RoomMessageEventContent}; +use ruma::{ + OwnedRoomId, OwnedServerName, OwnedUserId, events::room::message::RoomMessageEventContent, +}; use crate::Command; #[derive(Debug, Subcommand)] pub(crate) enum RoomStateCacheCommand { ServerInRoom { - server: Box, - room_id: Box, + server: OwnedServerName, + room_id: OwnedRoomId, }, RoomServers { - room_id: Box, + room_id: OwnedRoomId, }, ServerRooms { - server: Box, + server: OwnedServerName, }, RoomMembers { - room_id: Box, + room_id: OwnedRoomId, }, LocalUsersInRoom { - room_id: Box, + room_id: OwnedRoomId, }, ActiveLocalUsersInRoom { - room_id: Box, + room_id: OwnedRoomId, }, RoomJoinedCount { - room_id: Box, + room_id: OwnedRoomId, }, RoomInvitedCount { - room_id: Box, + room_id: OwnedRoomId, }, RoomUserOnceJoined { - room_id: Box, + room_id: OwnedRoomId, }, RoomMembersInvited { - room_id: Box, + room_id: OwnedRoomId, }, GetInviteCount { - room_id: Box, - user_id: Box, + room_id: OwnedRoomId, + user_id: OwnedUserId, }, GetLeftCount { - room_id: Box, - user_id: Box, + room_id: OwnedRoomId, + user_id: OwnedUserId, }, RoomsJoined { - user_id: Box, + user_id: OwnedUserId, }, RoomsLeft { - user_id: Box, + user_id: OwnedUserId, }, RoomsInvited { - user_id: Box, + user_id: OwnedUserId, }, InviteState { - user_id: Box, - room_id: Box, + user_id: OwnedUserId, + room_id: OwnedRoomId, }, } diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index a148f718..860bca4a 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{ServerName, UserId, events::room::message::RoomMessageEventContent}; +use ruma::{OwnedServerName, OwnedUserId, events::room::message::RoomMessageEventContent}; use service::sending::Destination; use crate::Command; @@ -27,9 +27,9 @@ pub(crate) enum SendingCommand { #[arg(short, long)] appservice_id: Option, #[arg(short, long)] - server_name: Option>, + server_name: Option, #[arg(short, long)] - user_id: Option>, + user_id: Option, #[arg(short, long)] push_key: Option, }, @@ -49,15 +49,15 @@ pub(crate) enum SendingCommand { #[arg(short, long)] appservice_id: Option, #[arg(short, long)] - server_name: Option>, + server_name: Option, #[arg(short, long)] - user_id: Option>, + user_id: Option, #[arg(short, long)] push_key: Option, }, GetLatestEduCount { - server_name: Box, + server_name: OwnedServerName, }, } @@ -120,7 +120,7 @@ pub(super) async fn reprocess( | (None, Some(server_name), None, None) => services .sending .db - .queued_requests(&Destination::Federation(server_name.into())), + .queued_requests(&Destination::Federation(server_name)), | (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { return Ok(RoomMessageEventContent::text_plain( @@ -132,7 +132,7 @@ pub(super) async fn reprocess( services .sending .db - .queued_requests(&Destination::Push(user_id.into(), push_key)) + .queued_requests(&Destination::Push(user_id, push_key)) }, | (Some(_), Some(_), Some(_), Some(_)) => { return Ok(RoomMessageEventContent::text_plain( @@ -190,7 +190,7 @@ pub(super) async fn reprocess( | (None, Some(server_name), None, None) => services .sending .db - .active_requests_for(&Destination::Federation(server_name.into())), + .active_requests_for(&Destination::Federation(server_name)), | (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { return Ok(RoomMessageEventContent::text_plain( @@ -202,7 +202,7 @@ pub(super) async fn reprocess( services .sending .db - .active_requests_for(&Destination::Push(user_id.into(), push_key)) + .active_requests_for(&Destination::Push(user_id, push_key)) }, | (Some(_), Some(_), Some(_), Some(_)) => { return Ok(RoomMessageEventContent::text_plain( diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index ab21170c..4cfff2e5 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -3,9 +3,7 @@ use std::fmt::Write; use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{ - OwnedRoomAliasId, OwnedRoomId, RoomId, events::room::message::RoomMessageEventContent, -}; +use ruma::{OwnedRoomAliasId, OwnedRoomId, events::room::message::RoomMessageEventContent}; use crate::{Command, escape_html}; @@ -18,7 +16,7 @@ pub(crate) enum RoomAliasCommand { force: bool, /// The room id to set the alias on - room_id: Box, + room_id: OwnedRoomId, /// The alias localpart to use (`alias`, not `#alias:servername.tld`) room_alias_localpart: String, @@ -40,7 +38,7 @@ pub(crate) enum RoomAliasCommand { /// - List aliases currently being used List { /// If set, only list the aliases for this room - room_id: Option>, + room_id: Option, }, } diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index ca036825..179131e4 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{RoomId, events::room::message::RoomMessageEventContent}; +use ruma::{OwnedRoomId, events::room::message::RoomMessageEventContent}; use crate::{Command, PAGE_SIZE, get_room_info}; @@ -10,13 +10,13 @@ pub(crate) enum RoomDirectoryCommand { /// - Publish a room to the room directory Publish { /// The room id of the room to publish - room_id: Box, + room_id: OwnedRoomId, }, /// - Unpublish a room to the room directory Unpublish { /// The room id of the room to unpublish - room_id: Box, + room_id: OwnedRoomId, }, /// - List rooms that are published diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index a39728fe..35a92b6a 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::{Result, utils::ReadyExt}; use futures::StreamExt; -use ruma::{RoomId, events::room::message::RoomMessageEventContent}; +use ruma::{OwnedRoomId, events::room::message::RoomMessageEventContent}; use crate::{admin_command, admin_command_dispatch}; @@ -10,7 +10,7 @@ use crate::{admin_command, admin_command_dispatch}; pub(crate) enum RoomInfoCommand { /// - List joined members in a room ListJoinedMembers { - room_id: Box, + room_id: OwnedRoomId, /// Lists only our local users in the specified room #[arg(long)] @@ -22,14 +22,14 @@ pub(crate) enum RoomInfoCommand { /// Room topics can be huge, so this is in its /// own separate command ViewRoomTopic { - room_id: Box, + room_id: OwnedRoomId, }, } #[admin_command] async fn list_joined_members( &self, - room_id: Box, + room_id: OwnedRoomId, local_only: bool, ) -> Result { let room_name = self @@ -79,7 +79,7 @@ async fn list_joined_members( } #[admin_command] -async fn view_room_topic(&self, room_id: Box) -> Result { +async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result { let Ok(room_topic) = self .services .rooms diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index dd5ea627..41458622 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -7,7 +7,7 @@ use conduwuit::{ }; use futures::StreamExt; use ruma::{ - OwnedRoomId, RoomAliasId, RoomId, RoomOrAliasId, + OwnedRoomId, OwnedRoomOrAliasId, RoomAliasId, RoomId, RoomOrAliasId, events::room::message::RoomMessageEventContent, }; @@ -24,7 +24,7 @@ pub(crate) enum RoomModerationCommand { BanRoom { /// The room in the format of `!roomid:example.com` or a room alias in /// the format of `#roomalias:example.com` - room: Box, + room: OwnedRoomOrAliasId, }, /// - Bans a list of rooms (room IDs and room aliases) from a newline @@ -36,7 +36,7 @@ pub(crate) enum RoomModerationCommand { UnbanRoom { /// The room in the format of `!roomid:example.com` or a room alias in /// the format of `#roomalias:example.com` - room: Box, + room: OwnedRoomOrAliasId, }, /// - List of all rooms we have banned @@ -49,7 +49,7 @@ pub(crate) enum RoomModerationCommand { } #[admin_command] -async fn ban_room(&self, room: Box) -> Result { +async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { debug!("Got room alias or ID: {}", room); let admin_room_alias = &self.services.globals.admin_alias; @@ -363,7 +363,7 @@ async fn ban_list_of_rooms(&self) -> Result { } #[admin_command] -async fn unban_room(&self, room: Box) -> Result { +async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result { let room_id = if room.is_room_id() { let room_id = match RoomId::parse(&room) { | Ok(room_id) => room_id, diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 45e550be..84795f9b 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -10,7 +10,7 @@ use conduwuit::{ use conduwuit_api::client::{leave_all_rooms, update_avatar_url, update_displayname}; use futures::StreamExt; use ruma::{ - EventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, RoomId, UserId, + OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, UserId, events::{ RoomAccountDataEventType, StateEventType, room::{ @@ -802,7 +802,7 @@ pub(super) async fn make_user_admin(&self, user_id: String) -> Result, + room_id: OwnedRoomId, tag: String, ) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; @@ -840,7 +840,7 @@ pub(super) async fn put_room_tag( pub(super) async fn delete_room_tag( &self, user_id: String, - room_id: Box, + room_id: OwnedRoomId, tag: String, ) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; @@ -876,7 +876,7 @@ pub(super) async fn delete_room_tag( pub(super) async fn get_room_tags( &self, user_id: String, - room_id: Box, + room_id: OwnedRoomId, ) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; @@ -898,7 +898,7 @@ pub(super) async fn get_room_tags( #[admin_command] pub(super) async fn redact_event( &self, - event_id: Box, + event_id: OwnedEventId, ) -> Result { let Ok(event) = self .services diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 1494ea8f..e789376a 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -2,7 +2,7 @@ mod commands; use clap::Subcommand; use conduwuit::Result; -use ruma::{EventId, OwnedRoomOrAliasId, RoomId}; +use ruma::{OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId}; use crate::admin_command_dispatch; @@ -102,21 +102,21 @@ pub(super) enum UserCommand { /// room's internal ID, and the tag name `m.server_notice`. PutRoomTag { user_id: String, - room_id: Box, + room_id: OwnedRoomId, tag: String, }, /// - Deletes the room tag for the specified user and room ID DeleteRoomTag { user_id: String, - room_id: Box, + room_id: OwnedRoomId, tag: String, }, /// - Gets all the room tags for the specified user and room ID GetRoomTags { user_id: String, - room_id: Box, + room_id: OwnedRoomId, }, /// - Attempts to forcefully redact the specified event ID from the sender @@ -124,7 +124,7 @@ pub(super) enum UserCommand { /// /// This is only valid for local users RedactEvent { - event_id: Box, + event_id: OwnedEventId, }, /// - Force joins a specified list of local users to join the specified From ecf20f7ebbcaeff2e5adedf4b7c45b6d9cbb49b7 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 7 Apr 2025 03:28:51 +0000 Subject: [PATCH 1704/2091] improve appservice service async interfaces Signed-off-by: Jason Volk --- src/admin/query/appservice.rs | 3 +- src/service/appservice/mod.rs | 78 +++++++++++++++-------------------- 2 files changed, 36 insertions(+), 45 deletions(-) diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 93c76a7e..0359261a 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -1,5 +1,6 @@ use clap::Subcommand; use conduwuit::Result; +use futures::TryStreamExt; use crate::Command; @@ -31,7 +32,7 @@ pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_> }, | AppserviceCommand::All => { let timer = tokio::time::Instant::now(); - let results = services.appservice.all().await; + let results: Vec<_> = services.appservice.iter_db_ids().try_collect().await?; let query_time = timer.elapsed(); write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 50a60033..7be8a471 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -1,20 +1,20 @@ mod namespace_regex; mod registration_info; -use std::{collections::BTreeMap, sync::Arc}; +use std::{collections::BTreeMap, iter::IntoIterator, sync::Arc}; use async_trait::async_trait; -use conduwuit::{Result, err, utils::stream::TryIgnore}; +use conduwuit::{Result, err, utils::stream::IterStream}; use database::Map; -use futures::{Future, StreamExt, TryStreamExt}; +use futures::{Future, FutureExt, Stream, TryStreamExt}; use ruma::{RoomAliasId, RoomId, UserId, api::appservice::Registration}; -use tokio::sync::RwLock; +use tokio::sync::{RwLock, RwLockReadGuard}; pub use self::{namespace_regex::NamespaceRegex, registration_info::RegistrationInfo}; use crate::{Dep, sending}; pub struct Service { - registration_info: RwLock>, + registration_info: RwLock, services: Services, db: Data, } @@ -27,6 +27,8 @@ struct Data { id_appserviceregistrations: Arc, } +type Registrations = BTreeMap; + #[async_trait] impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { @@ -41,19 +43,18 @@ impl crate::Service for Service { })) } - async fn worker(self: Arc) -> Result<()> { + async fn worker(self: Arc) -> Result { // Inserting registrations into cache - for appservice in self.iter_db_ids().await? { - self.registration_info.write().await.insert( - appservice.0, - appservice - .1 - .try_into() - .expect("Should be validated on registration"), - ); - } + self.iter_db_ids() + .try_for_each(async |appservice| { + self.registration_info + .write() + .await + .insert(appservice.0, appservice.1.try_into()?); - Ok(()) + Ok(()) + }) + .await } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } @@ -84,7 +85,7 @@ impl Service { /// # Arguments /// /// * `service_name` - the registration ID of the appservice - pub async fn unregister_appservice(&self, appservice_id: &str) -> Result<()> { + pub async fn unregister_appservice(&self, appservice_id: &str) -> Result { // removes the appservice registration info self.registration_info .write() @@ -112,15 +113,6 @@ impl Service { .map(|info| info.registration) } - pub async fn iter_ids(&self) -> Vec { - self.registration_info - .read() - .await - .keys() - .cloned() - .collect() - } - pub async fn find_from_token(&self, token: &str) -> Option { self.read() .await @@ -156,15 +148,22 @@ impl Service { .any(|info| info.rooms.is_exclusive_match(room_id.as_str())) } - pub fn read( - &self, - ) -> impl Future>> - { - self.registration_info.read() + pub fn iter_ids(&self) -> impl Stream + Send { + self.read() + .map(|info| info.keys().cloned().collect::>()) + .map(IntoIterator::into_iter) + .map(IterStream::stream) + .flatten_stream() } - #[inline] - pub async fn all(&self) -> Result> { self.iter_db_ids().await } + pub fn iter_db_ids(&self) -> impl Stream> + Send { + self.db + .id_appserviceregistrations + .keys() + .and_then(move |id: &str| async move { + Ok((id.to_owned(), self.get_db_registration(id).await?)) + }) + } pub async fn get_db_registration(&self, id: &str) -> Result { self.db @@ -175,16 +174,7 @@ impl Service { .map_err(|e| err!(Database("Invalid appservice {id:?} registration: {e:?}"))) } - async fn iter_db_ids(&self) -> Result> { - self.db - .id_appserviceregistrations - .keys() - .ignore_err() - .then(|id: String| async move { - let reg = self.get_db_registration(&id).await?; - Ok((id, reg)) - }) - .try_collect() - .await + pub fn read(&self) -> impl Future> + Send { + self.registration_info.read() } } From fb3020d8da5b221cba042053a65ce67034622973 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 9 Apr 2025 01:50:13 +0000 Subject: [PATCH 1705/2091] misc async optimizations; macro reformatting Signed-off-by: Jason Volk --- src/api/client/membership.rs | 139 ++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 67 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 1eeacf83..18a1c741 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1,6 +1,6 @@ use std::{ borrow::Borrow, - collections::{BTreeMap, HashMap, HashSet}, + collections::{HashMap, HashSet}, iter::once, net::IpAddr, sync::Arc, @@ -9,7 +9,7 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Err, Result, at, debug, debug_info, debug_warn, err, error, info, + Err, Result, at, debug, debug_error, debug_info, debug_warn, err, error, info, is_matching, matrix::{ StateKey, pdu::{PduBuilder, PduEvent, gen_event_id, gen_event_id_canonical_json}, @@ -17,7 +17,12 @@ use conduwuit::{ }, result::{FlatOk, NotFound}, trace, - utils::{self, IterStream, ReadyExt, shuffle}, + utils::{ + self, FutureBoolExt, + future::ReadyEqExt, + shuffle, + stream::{BroadbandExt, IterStream, ReadyExt}, + }, warn, }; use conduwuit_service::{ @@ -28,7 +33,7 @@ use conduwuit_service::{ state_compressor::{CompressedState, HashSetCompressStateEvent}, }, }; -use futures::{FutureExt, StreamExt, TryFutureExt, future::join4, join}; +use futures::{FutureExt, StreamExt, TryFutureExt, join, pin_mut}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, @@ -52,7 +57,6 @@ use ruma::{ room::{ join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent}, member::{MembershipState, RoomMemberEventContent}, - message::RoomMessageEventContent, }, }, }; @@ -81,7 +85,7 @@ async fn banned_room_check( || services .config .forbidden_remote_server_names - .is_match(room_id.server_name().unwrap().host()) + .is_match(room_id.server_name().expect("legacy room mxid").host()) { warn!( "User {user_id} who is not an admin attempted to send an invite for or \ @@ -96,12 +100,11 @@ async fn banned_room_check( if services.server.config.admin_room_notices { services .admin - .send_message(RoomMessageEventContent::text_plain(format!( + .send_text(&format!( "Automatically deactivating user {user_id} due to attempted banned \ room join from IP {client_ip}" - ))) - .await - .ok(); + )) + .await; } let all_joined_rooms: Vec = services @@ -136,12 +139,11 @@ async fn banned_room_check( if services.server.config.admin_room_notices { services .admin - .send_message(RoomMessageEventContent::text_plain(format!( + .send_text(&format!( "Automatically deactivating user {user_id} due to attempted banned \ room join from IP {client_ip}" - ))) - .await - .ok(); + )) + .await; } let all_joined_rooms: Vec = services @@ -366,10 +368,10 @@ pub(crate) async fn knock_room_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let body = body.body; + let sender_user = body.sender_user(); + let body = &body.body; - let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { + let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias.clone()) { | Ok(room_id) => { banned_room_check( &services, @@ -493,7 +495,7 @@ pub(crate) async fn invite_user_route( let sender_user = body.sender_user(); if !services.users.is_admin(sender_user).await && services.config.block_non_admin_invites { - info!( + debug_error!( "User {sender_user} is not an admin and attempted to send an invite to room {}", &body.room_id ); @@ -722,12 +724,10 @@ pub(crate) async fn forget_room_route( let joined = services.rooms.state_cache.is_joined(user_id, room_id); let knocked = services.rooms.state_cache.is_knocked(user_id, room_id); - let left = services.rooms.state_cache.is_left(user_id, room_id); let invited = services.rooms.state_cache.is_invited(user_id, room_id); - let (joined, knocked, left, invited) = join4(joined, knocked, left, invited).await; - - if joined || knocked || invited { + pin_mut!(joined, knocked, invited); + if joined.or(knocked).or(invited).await { return Err!(Request(Unknown("You must leave the room before forgetting it"))); } @@ -741,11 +741,11 @@ pub(crate) async fn forget_room_route( return Err!(Request(Unknown("No membership event was found, room was never joined"))); } - if left - || membership.is_ok_and(|member| { - member.membership == MembershipState::Leave - || member.membership == MembershipState::Ban - }) { + let non_membership = membership + .map(|member| member.membership) + .is_ok_and(is_matching!(MembershipState::Leave | MembershipState::Ban)); + + if non_membership || services.rooms.state_cache.is_left(user_id, room_id).await { services.rooms.state_cache.forget(room_id, user_id); } @@ -866,32 +866,32 @@ pub(crate) async fn joined_members_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user(); - if !services .rooms .state_accessor - .user_can_see_state_events(sender_user, &body.room_id) + .user_can_see_state_events(body.sender_user(), &body.room_id) .await { return Err!(Request(Forbidden("You don't have permission to view this room."))); } - let joined: BTreeMap = services - .rooms - .state_cache - .room_members(&body.room_id) - .map(ToOwned::to_owned) - .then(|user| async move { - (user.clone(), RoomMember { - display_name: services.users.displayname(&user).await.ok(), - avatar_url: services.users.avatar_url(&user).await.ok(), - }) - }) - .collect() - .await; + Ok(joined_members::v3::Response { + joined: services + .rooms + .state_cache + .room_members(&body.room_id) + .map(ToOwned::to_owned) + .broad_then(|user_id| async move { + let member = RoomMember { + display_name: services.users.displayname(&user_id).await.ok(), + avatar_url: services.users.avatar_url(&user_id).await.ok(), + }; - Ok(joined_members::v3::Response { joined }) + (user_id, member) + }) + .collect() + .await, + }) } pub async fn join_room_by_id_helper( @@ -1118,9 +1118,10 @@ async fn join_room_by_id_helper_remote( })?; if signed_event_id != event_id { - return Err!(Request(BadJson( - warn!(%signed_event_id, %event_id, "Server {remote_server} sent event with wrong event ID") - ))); + return Err!(Request(BadJson(warn!( + %signed_event_id, %event_id, + "Server {remote_server} sent event with wrong event ID" + )))); } match signed_value["signatures"] @@ -1696,19 +1697,18 @@ pub(crate) async fn invite_helper( })?; if pdu.event_id != event_id { - return Err!(Request(BadJson( - warn!(%pdu.event_id, %event_id, "Server {} sent event with wrong event ID", user_id.server_name()) - ))); + return Err!(Request(BadJson(warn!( + %pdu.event_id, %event_id, + "Server {} sent event with wrong event ID", + user_id.server_name() + )))); } - let origin: OwnedServerName = serde_json::from_value( - serde_json::to_value( - value - .get("origin") - .ok_or_else(|| err!(Request(BadJson("Event missing origin field."))))?, - ) - .expect("CanonicalJson is valid json value"), - ) + let origin: OwnedServerName = serde_json::from_value(serde_json::to_value( + value + .get("origin") + .ok_or_else(|| err!(Request(BadJson("Event missing origin field."))))?, + )?) .map_err(|e| { err!(Request(BadJson(warn!("Origin field in event is not a valid server name: {e}")))) })?; @@ -1818,9 +1818,11 @@ pub async fn leave_room( blurhash: None, }; - if services.rooms.metadata.is_banned(room_id).await - || services.rooms.metadata.is_disabled(room_id).await - { + let is_banned = services.rooms.metadata.is_banned(room_id); + let is_disabled = services.rooms.metadata.is_disabled(room_id); + + pin_mut!(is_banned, is_disabled); + if is_banned.or(is_disabled).await { // the room is banned/disabled, the room must be rejected locally since we // cant/dont want to federate with this server services @@ -1840,18 +1842,21 @@ pub async fn leave_room( return Ok(()); } - // Ask a remote server if we don't have this room and are not knocking on it - if !services + let dont_have_room = services .rooms .state_cache .server_in_room(services.globals.server_name(), room_id) - .await && !services + .eq(&false); + + let not_knocked = services .rooms .state_cache .is_knocked(user_id, room_id) - .await - { - if let Err(e) = remote_leave_room(services, user_id, room_id).await { + .eq(&false); + + // Ask a remote server if we don't have this room and are not knocking on it + if dont_have_room.and(not_knocked).await { + if let Err(e) = remote_leave_room(services, user_id, room_id).boxed().await { warn!(%user_id, "Failed to leave room {room_id} remotely: {e}"); // Don't tell the client about this error } From 4f8fec7e5a5631ef7b679a00838219e7926040dd Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 6 Apr 2025 23:41:58 +0000 Subject: [PATCH 1706/2091] replace admin command branches returning RoomMessageEventContent rename admin Command back to Context Signed-off-by: Jason Volk --- src/admin/admin.rs | 24 +- src/admin/appservice/commands.rs | 82 +++-- src/admin/check/commands.rs | 14 +- src/admin/{command.rs => context.rs} | 14 +- src/admin/debug/commands.rs | 481 +++++++++++---------------- src/admin/debug/tester.rs | 17 +- src/admin/federation/commands.rs | 103 +++--- src/admin/media/commands.rs | 119 +++---- src/admin/mod.rs | 8 +- src/admin/processor.rs | 8 +- src/admin/query/account_data.rs | 16 +- src/admin/query/appservice.rs | 4 +- src/admin/query/globals.rs | 4 +- src/admin/query/presence.rs | 4 +- src/admin/query/pusher.rs | 4 +- src/admin/query/raw.rs | 120 ++----- src/admin/query/resolver.rs | 13 +- src/admin/query/room_alias.rs | 4 +- src/admin/query/room_state_cache.rs | 146 ++++---- src/admin/query/room_timeline.rs | 10 +- src/admin/query/sending.rs | 90 +++-- src/admin/query/short.rs | 16 +- src/admin/query/users.rs | 175 ++++------ src/admin/room/alias.rs | 92 ++--- src/admin/room/commands.rs | 33 +- src/admin/room/directory.rs | 43 +-- src/admin/room/info.rs | 38 +-- src/admin/room/moderation.rs | 91 +++-- src/admin/server/commands.rs | 73 ++-- src/admin/user/commands.rs | 344 ++++++++----------- src/admin/utils.rs | 2 + src/macros/admin.rs | 17 +- 32 files changed, 903 insertions(+), 1306 deletions(-) rename src/admin/{command.rs => context.rs} (67%) diff --git a/src/admin/admin.rs b/src/admin/admin.rs index 9e010a59..0d636c72 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -2,7 +2,7 @@ use clap::Parser; use conduwuit::Result; use crate::{ - appservice, appservice::AppserviceCommand, check, check::CheckCommand, command::Command, + appservice, appservice::AppserviceCommand, check, check::CheckCommand, context::Context, debug, debug::DebugCommand, federation, federation::FederationCommand, media, media::MediaCommand, query, query::QueryCommand, room, room::RoomCommand, server, server::ServerCommand, user, user::UserCommand, @@ -49,20 +49,18 @@ pub(super) enum AdminCommand { } #[tracing::instrument(skip_all, name = "command")] -pub(super) async fn process(command: AdminCommand, context: &Command<'_>) -> Result { +pub(super) async fn process(command: AdminCommand, context: &Context<'_>) -> Result { use AdminCommand::*; match command { - | Appservices(command) => appservice::process(command, context).await?, - | Media(command) => media::process(command, context).await?, - | Users(command) => user::process(command, context).await?, - | Rooms(command) => room::process(command, context).await?, - | Federation(command) => federation::process(command, context).await?, - | Server(command) => server::process(command, context).await?, - | Debug(command) => debug::process(command, context).await?, - | Query(command) => query::process(command, context).await?, - | Check(command) => check::process(command, context).await?, + | Appservices(command) => appservice::process(command, context).await, + | Media(command) => media::process(command, context).await, + | Users(command) => user::process(command, context).await, + | Rooms(command) => room::process(command, context).await, + | Federation(command) => federation::process(command, context).await, + | Server(command) => server::process(command, context).await, + | Debug(command) => debug::process(command, context).await, + | Query(command) => query::process(command, context).await, + | Check(command) => check::process(command, context).await, } - - Ok(()) } diff --git a/src/admin/appservice/commands.rs b/src/admin/appservice/commands.rs index 88f28431..3575e067 100644 --- a/src/admin/appservice/commands.rs +++ b/src/admin/appservice/commands.rs @@ -1,84 +1,80 @@ -use ruma::{api::appservice::Registration, events::room::message::RoomMessageEventContent}; +use conduwuit::{Err, Result, checked}; +use futures::{FutureExt, StreamExt, TryFutureExt}; -use crate::{Result, admin_command}; +use crate::admin_command; #[admin_command] -pub(super) async fn register(&self) -> Result { - if self.body.len() < 2 - || !self.body[0].trim().starts_with("```") - || self.body.last().unwrap_or(&"").trim() != "```" +pub(super) async fn register(&self) -> Result { + let body = &self.body; + let body_len = self.body.len(); + if body_len < 2 + || !body[0].trim().starts_with("```") + || body.last().unwrap_or(&"").trim() != "```" { - return Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )); + return Err!("Expected code block in command body. Add --help for details."); } - let appservice_config_body = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); - let parsed_config = serde_yaml::from_str::(&appservice_config_body); + let range = 1..checked!(body_len - 1)?; + let appservice_config_body = body[range].join("\n"); + let parsed_config = serde_yaml::from_str(&appservice_config_body); match parsed_config { + | Err(e) => return Err!("Could not parse appservice config as YAML: {e}"), | Ok(registration) => match self .services .appservice .register_appservice(®istration, &appservice_config_body) .await + .map(|()| registration.id) { - | Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( - "Appservice registered with ID: {}", - registration.id - ))), - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Failed to register appservice: {e}" - ))), + | Err(e) => return Err!("Failed to register appservice: {e}"), + | Ok(id) => write!(self, "Appservice registered with ID: {id}"), }, - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Could not parse appservice config as YAML: {e}" - ))), } + .await } #[admin_command] -pub(super) async fn unregister( - &self, - appservice_identifier: String, -) -> Result { +pub(super) async fn unregister(&self, appservice_identifier: String) -> Result { match self .services .appservice .unregister_appservice(&appservice_identifier) .await { - | Ok(()) => Ok(RoomMessageEventContent::text_plain("Appservice unregistered.")), - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Failed to unregister appservice: {e}" - ))), + | Err(e) => return Err!("Failed to unregister appservice: {e}"), + | Ok(()) => write!(self, "Appservice unregistered."), } + .await } #[admin_command] -pub(super) async fn show_appservice_config( - &self, - appservice_identifier: String, -) -> Result { +pub(super) async fn show_appservice_config(&self, appservice_identifier: String) -> Result { match self .services .appservice .get_registration(&appservice_identifier) .await { + | None => return Err!("Appservice does not exist."), | Some(config) => { - let config_str = serde_yaml::to_string(&config) - .expect("config should've been validated on register"); - let output = - format!("Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```",); - Ok(RoomMessageEventContent::notice_markdown(output)) + let config_str = serde_yaml::to_string(&config)?; + write!(self, "Config for {appservice_identifier}:\n\n```yaml\n{config_str}\n```") }, - | None => Ok(RoomMessageEventContent::text_plain("Appservice does not exist.")), } + .await } #[admin_command] -pub(super) async fn list_registered(&self) -> Result { - let appservices = self.services.appservice.iter_ids().await; - let output = format!("Appservices ({}): {}", appservices.len(), appservices.join(", ")); - Ok(RoomMessageEventContent::text_plain(output)) +pub(super) async fn list_registered(&self) -> Result { + self.services + .appservice + .iter_ids() + .collect() + .map(Ok) + .and_then(|appservices: Vec<_>| { + let len = appservices.len(); + let list = appservices.join(", "); + write!(self, "Appservices ({len}): {list}") + }) + .await } diff --git a/src/admin/check/commands.rs b/src/admin/check/commands.rs index 7e27362f..1ffc3ae5 100644 --- a/src/admin/check/commands.rs +++ b/src/admin/check/commands.rs @@ -1,15 +1,14 @@ use conduwuit::Result; use conduwuit_macros::implement; use futures::StreamExt; -use ruma::events::room::message::RoomMessageEventContent; -use crate::Command; +use crate::Context; /// Uses the iterator in `src/database/key_value/users.rs` to iterator over /// every user in our database (remote and local). Reports total count, any /// errors if there were any, etc -#[implement(Command, params = "<'_>")] -pub(super) async fn check_all_users(&self) -> Result { +#[implement(Context, params = "<'_>")] +pub(super) async fn check_all_users(&self) -> Result { let timer = tokio::time::Instant::now(); let users = self.services.users.iter().collect::>().await; let query_time = timer.elapsed(); @@ -18,11 +17,10 @@ pub(super) async fn check_all_users(&self) -> Result { let err_count = users.iter().filter(|_user| false).count(); let ok_count = users.iter().filter(|_user| true).count(); - let message = format!( + self.write_str(&format!( "Database query completed in {query_time:?}:\n\n```\nTotal entries: \ {total:?}\nFailure/Invalid user count: {err_count:?}\nSuccess/Valid user count: \ {ok_count:?}\n```" - ); - - Ok(RoomMessageEventContent::notice_markdown(message)) + )) + .await } diff --git a/src/admin/command.rs b/src/admin/context.rs similarity index 67% rename from src/admin/command.rs rename to src/admin/context.rs index 5df980d6..270537be 100644 --- a/src/admin/command.rs +++ b/src/admin/context.rs @@ -3,13 +3,13 @@ use std::{fmt, time::SystemTime}; use conduwuit::Result; use conduwuit_service::Services; use futures::{ - Future, FutureExt, + Future, FutureExt, TryFutureExt, io::{AsyncWriteExt, BufWriter}, lock::Mutex, }; use ruma::EventId; -pub(crate) struct Command<'a> { +pub(crate) struct Context<'a> { pub(crate) services: &'a Services, pub(crate) body: &'a [&'a str], pub(crate) timer: SystemTime, @@ -17,14 +17,14 @@ pub(crate) struct Command<'a> { pub(crate) output: Mutex>>, } -impl Command<'_> { +impl Context<'_> { pub(crate) fn write_fmt( &self, arguments: fmt::Arguments<'_>, ) -> impl Future + Send + '_ + use<'_> { let buf = format!("{arguments}"); - self.output.lock().then(|mut output| async move { - output.write_all(buf.as_bytes()).await.map_err(Into::into) + self.output.lock().then(async move |mut output| { + output.write_all(buf.as_bytes()).map_err(Into::into).await }) } @@ -32,8 +32,8 @@ impl Command<'_> { &'a self, s: &'a str, ) -> impl Future + Send + 'a { - self.output.lock().then(move |mut output| async move { - output.write_all(s.as_bytes()).await.map_err(Into::into) + self.output.lock().then(async move |mut output| { + output.write_all(s.as_bytes()).map_err(Into::into).await }) } } diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 6d0e375a..d0debc2a 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -6,7 +6,7 @@ use std::{ }; use conduwuit::{ - Error, Result, debug_error, err, info, + Err, Result, debug_error, err, info, matrix::pdu::{PduEvent, PduId, RawPduId}, trace, utils, utils::{ @@ -19,7 +19,7 @@ use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedServerName, RoomId, RoomVersionId, - api::federation::event::get_room_state, events::room::message::RoomMessageEventContent, + api::federation::event::get_room_state, }; use service::rooms::{ short::{ShortEventId, ShortRoomId}, @@ -30,28 +30,24 @@ use tracing_subscriber::EnvFilter; use crate::admin_command; #[admin_command] -pub(super) async fn echo(&self, message: Vec) -> Result { +pub(super) async fn echo(&self, message: Vec) -> Result { let message = message.join(" "); - - Ok(RoomMessageEventContent::notice_plain(message)) + self.write_str(&message).await } #[admin_command] -pub(super) async fn get_auth_chain( - &self, - event_id: OwnedEventId, -) -> Result { +pub(super) async fn get_auth_chain(&self, event_id: OwnedEventId) -> Result { let Ok(event) = self.services.rooms.timeline.get_pdu_json(&event_id).await else { - return Ok(RoomMessageEventContent::notice_plain("Event not found.")); + return Err!("Event not found."); }; let room_id_str = event .get("room_id") - .and_then(|val| val.as_str()) - .ok_or_else(|| Error::bad_database("Invalid event in database"))?; + .and_then(CanonicalJsonValue::as_str) + .ok_or_else(|| err!(Database("Invalid event in database")))?; let room_id = <&RoomId>::try_from(room_id_str) - .map_err(|_| Error::bad_database("Invalid room id field in event in database"))?; + .map_err(|_| err!(Database("Invalid room id field in event in database")))?; let start = Instant::now(); let count = self @@ -64,51 +60,39 @@ pub(super) async fn get_auth_chain( .await; let elapsed = start.elapsed(); - Ok(RoomMessageEventContent::text_plain(format!( - "Loaded auth chain with length {count} in {elapsed:?}" - ))) + let out = format!("Loaded auth chain with length {count} in {elapsed:?}"); + + self.write_str(&out).await } #[admin_command] -pub(super) async fn parse_pdu(&self) -> Result { +pub(super) async fn parse_pdu(&self) -> Result { if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&EMPTY).trim() != "```" { - return Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )); + return Err!("Expected code block in command body. Add --help for details."); } let string = self.body[1..self.body.len().saturating_sub(1)].join("\n"); match serde_json::from_str(&string) { + | Err(e) => return Err!("Invalid json in command body: {e}"), | Ok(value) => match ruma::signatures::reference_hash(&value, &RoomVersionId::V6) { + | Err(e) => return Err!("Could not parse PDU JSON: {e:?}"), | Ok(hash) => { let event_id = OwnedEventId::parse(format!("${hash}")); - - match serde_json::from_value::( - serde_json::to_value(value).expect("value is json"), - ) { - | Ok(pdu) => Ok(RoomMessageEventContent::text_plain(format!( - "EventId: {event_id:?}\n{pdu:#?}" - ))), - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "EventId: {event_id:?}\nCould not parse event: {e}" - ))), + match serde_json::from_value::(serde_json::to_value(value)?) { + | Err(e) => return Err!("EventId: {event_id:?}\nCould not parse event: {e}"), + | Ok(pdu) => write!(self, "EventId: {event_id:?}\n{pdu:#?}"), } }, - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Could not parse PDU JSON: {e:?}" - ))), }, - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Invalid json in command body: {e}" - ))), } + .await } #[admin_command] -pub(super) async fn get_pdu(&self, event_id: OwnedEventId) -> Result { +pub(super) async fn get_pdu(&self, event_id: OwnedEventId) -> Result { let mut outlier = false; let mut pdu_json = self .services @@ -123,21 +107,18 @@ pub(super) async fn get_pdu(&self, event_id: OwnedEventId) -> Result return Err!("PDU not found locally."), | Ok(json) => { - let json_text = - serde_json::to_string_pretty(&json).expect("canonical json is valid json"); - Ok(RoomMessageEventContent::notice_markdown(format!( - "{}\n```json\n{}\n```", - if outlier { - "Outlier (Rejected / Soft Failed) PDU found in our database" - } else { - "PDU found in our database" - }, - json_text - ))) + let text = serde_json::to_string_pretty(&json)?; + let msg = if outlier { + "Outlier (Rejected / Soft Failed) PDU found in our database" + } else { + "PDU found in our database" + }; + write!(self, "{msg}\n```json\n{text}\n```",) }, - | Err(_) => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")), } + .await } #[admin_command] @@ -145,7 +126,7 @@ pub(super) async fn get_short_pdu( &self, shortroomid: ShortRoomId, shorteventid: ShortEventId, -) -> Result { +) -> Result { let pdu_id: RawPduId = PduId { shortroomid, shorteventid: shorteventid.into(), @@ -160,41 +141,33 @@ pub(super) async fn get_short_pdu( .await; match pdu_json { + | Err(_) => return Err!("PDU not found locally."), | Ok(json) => { - let json_text = - serde_json::to_string_pretty(&json).expect("canonical json is valid json"); - Ok(RoomMessageEventContent::notice_markdown(format!("```json\n{json_text}\n```",))) + let json_text = serde_json::to_string_pretty(&json)?; + write!(self, "```json\n{json_text}\n```") }, - | Err(_) => Ok(RoomMessageEventContent::text_plain("PDU not found locally.")), } + .await } #[admin_command] -pub(super) async fn get_remote_pdu_list( - &self, - server: OwnedServerName, - force: bool, -) -> Result { +pub(super) async fn get_remote_pdu_list(&self, server: OwnedServerName, force: bool) -> Result { if !self.services.server.config.allow_federation { - return Ok(RoomMessageEventContent::text_plain( - "Federation is disabled on this homeserver.", - )); + return Err!("Federation is disabled on this homeserver.",); } if server == self.services.globals.server_name() { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ fetching local PDUs from the database.", - )); + ); } if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&EMPTY).trim() != "```" { - return Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )); + return Err!("Expected code block in command body. Add --help for details.",); } let list = self @@ -208,18 +181,19 @@ pub(super) async fn get_remote_pdu_list( let mut failed_count: usize = 0; let mut success_count: usize = 0; - for pdu in list { + for event_id in list { if force { - match self.get_remote_pdu(Box::from(pdu), server.clone()).await { + match self + .get_remote_pdu(event_id.to_owned(), server.clone()) + .await + { | Err(e) => { failed_count = failed_count.saturating_add(1); self.services .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "Failed to get remote PDU, ignoring error: {e}" - ))) - .await - .ok(); + .send_text(&format!("Failed to get remote PDU, ignoring error: {e}")) + .await; + warn!("Failed to get remote PDU, ignoring error: {e}"); }, | _ => { @@ -227,44 +201,48 @@ pub(super) async fn get_remote_pdu_list( }, } } else { - self.get_remote_pdu(Box::from(pdu), server.clone()).await?; + self.get_remote_pdu(event_id.to_owned(), server.clone()) + .await?; success_count = success_count.saturating_add(1); } } - Ok(RoomMessageEventContent::text_plain(format!( - "Fetched {success_count} remote PDUs successfully with {failed_count} failures" - ))) + let out = + format!("Fetched {success_count} remote PDUs successfully with {failed_count} failures"); + + self.write_str(&out).await } #[admin_command] pub(super) async fn get_remote_pdu( &self, - event_id: Box, - server: Box, -) -> Result { + event_id: OwnedEventId, + server: OwnedServerName, +) -> Result { if !self.services.server.config.allow_federation { - return Ok(RoomMessageEventContent::text_plain( - "Federation is disabled on this homeserver.", - )); + return Err!("Federation is disabled on this homeserver."); } if server == self.services.globals.server_name() { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ fetching local PDUs.", - )); + ); } match self .services .sending .send_federation_request(&server, ruma::api::federation::event::get_event::v1::Request { - event_id: event_id.clone().into(), + event_id: event_id.clone(), include_unredacted_content: None, }) .await { + | Err(e) => + return Err!( + "Remote server did not have PDU or failed sending request to remote server: {e}" + ), | Ok(response) => { let json: CanonicalJsonObject = serde_json::from_str(response.pdu.get()).map_err(|e| { @@ -272,10 +250,9 @@ pub(super) async fn get_remote_pdu( "Requested event ID {event_id} from server but failed to convert from \ RawValue to CanonicalJsonObject (malformed event/response?): {e}" ); - Error::BadRequest( - ErrorKind::Unknown, - "Received response from server but failed to parse PDU", - ) + err!(Request(Unknown( + "Received response from server but failed to parse PDU" + ))) })?; trace!("Attempting to parse PDU: {:?}", &response.pdu); @@ -285,6 +262,7 @@ pub(super) async fn get_remote_pdu( .rooms .event_handler .parse_incoming_pdu(&response.pdu) + .boxed() .await; let (event_id, value, room_id) = match parsed_result { @@ -292,9 +270,7 @@ pub(super) async fn get_remote_pdu( | Err(e) => { warn!("Failed to parse PDU: {e}"); info!("Full PDU: {:?}", &response.pdu); - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse PDU remote server {server} sent us: {e}" - ))); + return Err!("Failed to parse PDU remote server {server} sent us: {e}"); }, }; @@ -306,30 +282,18 @@ pub(super) async fn get_remote_pdu( .rooms .timeline .backfill_pdu(&server, response.pdu) - .boxed() .await?; - let json_text = - serde_json::to_string_pretty(&json).expect("canonical json is valid json"); - - Ok(RoomMessageEventContent::notice_markdown(format!( - "{}\n```json\n{}\n```", - "Got PDU from specified server and handled as backfilled PDU successfully. \ - Event body:", - json_text - ))) + let text = serde_json::to_string_pretty(&json)?; + let msg = "Got PDU from specified server and handled as backfilled"; + write!(self, "{msg}. Event body:\n```json\n{text}\n```") }, - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Remote server did not have PDU or failed sending request to remote server: {e}" - ))), } + .await } #[admin_command] -pub(super) async fn get_room_state( - &self, - room: OwnedRoomOrAliasId, -) -> Result { +pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result { let room_id = self.services.rooms.alias.resolve(&room).await?; let room_state: Vec<_> = self .services @@ -341,28 +305,24 @@ pub(super) async fn get_room_state( .await?; if room_state.is_empty() { - return Ok(RoomMessageEventContent::text_plain( - "Unable to find room state in our database (vector is empty)", - )); + return Err!("Unable to find room state in our database (vector is empty)",); } let json = serde_json::to_string_pretty(&room_state).map_err(|e| { - warn!("Failed converting room state vector in our database to pretty JSON: {e}"); - Error::bad_database( + err!(Database( "Failed to convert room state events to pretty JSON, possible invalid room state \ - events in our database", - ) + events in our database {e}", + )) })?; - Ok(RoomMessageEventContent::notice_markdown(format!("```json\n{json}\n```"))) + let out = format!("```json\n{json}\n```"); + self.write_str(&out).await } #[admin_command] -pub(super) async fn ping(&self, server: OwnedServerName) -> Result { +pub(super) async fn ping(&self, server: OwnedServerName) -> Result { if server == self.services.globals.server_name() { - return Ok(RoomMessageEventContent::text_plain( - "Not allowed to send federation requests to ourselves.", - )); + return Err!("Not allowed to send federation requests to ourselves."); } let timer = tokio::time::Instant::now(); @@ -376,35 +336,27 @@ pub(super) async fn ping(&self, server: OwnedServerName) -> Result { + return Err!("Failed sending federation request to specified server:\n\n{e}"); + }, | Ok(response) => { let ping_time = timer.elapsed(); - let json_text_res = serde_json::to_string_pretty(&response.server); - if let Ok(json) = json_text_res { - return Ok(RoomMessageEventContent::notice_markdown(format!( - "Got response which took {ping_time:?} time:\n```json\n{json}\n```" - ))); - } + let out = if let Ok(json) = json_text_res { + format!("Got response which took {ping_time:?} time:\n```json\n{json}\n```") + } else { + format!("Got non-JSON response which took {ping_time:?} time:\n{response:?}") + }; - Ok(RoomMessageEventContent::text_plain(format!( - "Got non-JSON response which took {ping_time:?} time:\n{response:?}" - ))) - }, - | Err(e) => { - warn!( - "Failed sending federation request to specified server from ping debug command: \ - {e}" - ); - Ok(RoomMessageEventContent::text_plain(format!( - "Failed sending federation request to specified server:\n\n{e}", - ))) + write!(self, "{out}") }, } + .await } #[admin_command] -pub(super) async fn force_device_list_updates(&self) -> Result { +pub(super) async fn force_device_list_updates(&self) -> Result { // Force E2EE device list updates for all users self.services .users @@ -412,27 +364,17 @@ pub(super) async fn force_device_list_updates(&self) -> Result, - reset: bool, -) -> Result { +pub(super) async fn change_log_level(&self, filter: Option, reset: bool) -> Result { let handles = &["console"]; if reset { let old_filter_layer = match EnvFilter::try_new(&self.services.server.config.log) { | Ok(s) => s, - | Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Log level from config appears to be invalid now: {e}" - ))); - }, + | Err(e) => return Err!("Log level from config appears to be invalid now: {e}"), }; match self @@ -442,16 +384,12 @@ pub(super) async fn change_log_level( .reload .reload(&old_filter_layer, Some(handles)) { + | Err(e) => + return Err!("Failed to modify and reload the global tracing log level: {e}"), | Ok(()) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Successfully changed log level back to config value {}", - self.services.server.config.log - ))); - }, - | Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to modify and reload the global tracing log level: {e}" - ))); + let value = &self.services.server.config.log; + let out = format!("Successfully changed log level back to config value {value}"); + return self.write_str(&out).await; }, } } @@ -459,11 +397,7 @@ pub(super) async fn change_log_level( if let Some(filter) = filter { let new_filter_layer = match EnvFilter::try_new(filter) { | Ok(s) => s, - | Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Invalid log level filter specified: {e}" - ))); - }, + | Err(e) => return Err!("Invalid log level filter specified: {e}"), }; match self @@ -473,90 +407,75 @@ pub(super) async fn change_log_level( .reload .reload(&new_filter_layer, Some(handles)) { - | Ok(()) => { - return Ok(RoomMessageEventContent::text_plain("Successfully changed log level")); - }, - | Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to modify and reload the global tracing log level: {e}" - ))); - }, + | Ok(()) => return self.write_str("Successfully changed log level").await, + | Err(e) => + return Err!("Failed to modify and reload the global tracing log level: {e}"), } } - Ok(RoomMessageEventContent::text_plain("No log level was specified.")) + Err!("No log level was specified.") } #[admin_command] -pub(super) async fn sign_json(&self) -> Result { +pub(super) async fn sign_json(&self) -> Result { if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" { - return Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )); + return Err!("Expected code block in command body. Add --help for details."); } let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); match serde_json::from_str(&string) { + | Err(e) => return Err!("Invalid json: {e}"), | Ok(mut value) => { - self.services - .server_keys - .sign_json(&mut value) - .expect("our request json is what ruma expects"); - let json_text = - serde_json::to_string_pretty(&value).expect("canonical json is valid json"); - Ok(RoomMessageEventContent::text_plain(json_text)) + self.services.server_keys.sign_json(&mut value)?; + let json_text = serde_json::to_string_pretty(&value)?; + write!(self, "{json_text}") }, - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), } + .await } #[admin_command] -pub(super) async fn verify_json(&self) -> Result { +pub(super) async fn verify_json(&self) -> Result { if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" { - return Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )); + return Err!("Expected code block in command body. Add --help for details."); } let string = self.body[1..self.body.len().checked_sub(1).unwrap()].join("\n"); match serde_json::from_str::(&string) { + | Err(e) => return Err!("Invalid json: {e}"), | Ok(value) => match self.services.server_keys.verify_json(&value, None).await { - | Ok(()) => Ok(RoomMessageEventContent::text_plain("Signature correct")), - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Signature verification failed: {e}" - ))), + | Err(e) => return Err!("Signature verification failed: {e}"), + | Ok(()) => write!(self, "Signature correct"), }, - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!("Invalid json: {e}"))), } + .await } #[admin_command] -pub(super) async fn verify_pdu(&self, event_id: OwnedEventId) -> Result { +pub(super) async fn verify_pdu(&self, event_id: OwnedEventId) -> Result { + use ruma::signatures::Verified; + let mut event = self.services.rooms.timeline.get_pdu_json(&event_id).await?; event.remove("event_id"); let msg = match self.services.server_keys.verify_event(&event, None).await { - | Ok(ruma::signatures::Verified::Signatures) => - "signatures OK, but content hash failed (redaction).", - | Ok(ruma::signatures::Verified::All) => "signatures and hashes OK.", | Err(e) => return Err(e), + | Ok(Verified::Signatures) => "signatures OK, but content hash failed (redaction).", + | Ok(Verified::All) => "signatures and hashes OK.", }; - Ok(RoomMessageEventContent::notice_plain(msg)) + self.write_str(msg).await } #[admin_command] #[tracing::instrument(skip(self))] -pub(super) async fn first_pdu_in_room( - &self, - room_id: OwnedRoomId, -) -> Result { +pub(super) async fn first_pdu_in_room(&self, room_id: OwnedRoomId) -> Result { if !self .services .rooms @@ -564,9 +483,7 @@ pub(super) async fn first_pdu_in_room( .server_in_room(&self.services.server.name, &room_id) .await { - return Ok(RoomMessageEventContent::text_plain( - "We are not participating in the room / we don't know about the room ID.", - )); + return Err!("We are not participating in the room / we don't know about the room ID.",); } let first_pdu = self @@ -575,17 +492,15 @@ pub(super) async fn first_pdu_in_room( .timeline .first_pdu_in_room(&room_id) .await - .map_err(|_| Error::bad_database("Failed to find the first PDU in database"))?; + .map_err(|_| err!(Database("Failed to find the first PDU in database")))?; - Ok(RoomMessageEventContent::text_plain(format!("{first_pdu:?}"))) + let out = format!("{first_pdu:?}"); + self.write_str(&out).await } #[admin_command] #[tracing::instrument(skip(self))] -pub(super) async fn latest_pdu_in_room( - &self, - room_id: OwnedRoomId, -) -> Result { +pub(super) async fn latest_pdu_in_room(&self, room_id: OwnedRoomId) -> Result { if !self .services .rooms @@ -593,9 +508,7 @@ pub(super) async fn latest_pdu_in_room( .server_in_room(&self.services.server.name, &room_id) .await { - return Ok(RoomMessageEventContent::text_plain( - "We are not participating in the room / we don't know about the room ID.", - )); + return Err!("We are not participating in the room / we don't know about the room ID."); } let latest_pdu = self @@ -604,9 +517,10 @@ pub(super) async fn latest_pdu_in_room( .timeline .latest_pdu_in_room(&room_id) .await - .map_err(|_| Error::bad_database("Failed to find the latest PDU in database"))?; + .map_err(|_| err!(Database("Failed to find the latest PDU in database")))?; - Ok(RoomMessageEventContent::text_plain(format!("{latest_pdu:?}"))) + let out = format!("{latest_pdu:?}"); + self.write_str(&out).await } #[admin_command] @@ -615,7 +529,7 @@ pub(super) async fn force_set_room_state_from_server( &self, room_id: OwnedRoomId, server_name: OwnedServerName, -) -> Result { +) -> Result { if !self .services .rooms @@ -623,9 +537,7 @@ pub(super) async fn force_set_room_state_from_server( .server_in_room(&self.services.server.name, &room_id) .await { - return Ok(RoomMessageEventContent::text_plain( - "We are not participating in the room / we don't know about the room ID.", - )); + return Err!("We are not participating in the room / we don't know about the room ID."); } let first_pdu = self @@ -634,7 +546,7 @@ pub(super) async fn force_set_room_state_from_server( .timeline .latest_pdu_in_room(&room_id) .await - .map_err(|_| Error::bad_database("Failed to find the latest PDU in database"))?; + .map_err(|_| err!(Database("Failed to find the latest PDU in database")))?; let room_version = self.services.rooms.state.get_room_version(&room_id).await?; @@ -644,10 +556,9 @@ pub(super) async fn force_set_room_state_from_server( .services .sending .send_federation_request(&server_name, get_room_state::v1::Request { - room_id: room_id.clone().into(), + room_id: room_id.clone(), event_id: first_pdu.event_id.clone(), }) - .boxed() .await?; for pdu in remote_state_response.pdus.clone() { @@ -656,7 +567,6 @@ pub(super) async fn force_set_room_state_from_server( .rooms .event_handler .parse_incoming_pdu(&pdu) - .boxed() .await { | Ok(t) => t, @@ -720,7 +630,6 @@ pub(super) async fn force_set_room_state_from_server( .rooms .event_handler .resolve_state(&room_id, &room_version, state) - .boxed() .await?; info!("Forcing new room state"); @@ -736,6 +645,7 @@ pub(super) async fn force_set_room_state_from_server( .await?; let state_lock = self.services.rooms.state.mutex.lock(&*room_id).await; + self.services .rooms .state @@ -752,11 +662,8 @@ pub(super) async fn force_set_room_state_from_server( .update_joined_count(&room_id) .await; - drop(state_lock); - - Ok(RoomMessageEventContent::text_plain( - "Successfully forced the room state from the requested remote server.", - )) + self.write_str("Successfully forced the room state from the requested remote server.") + .await } #[admin_command] @@ -765,8 +672,8 @@ pub(super) async fn get_signing_keys( server_name: Option, notary: Option, query: bool, -) -> Result { - let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone().into()); +) -> Result { + let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone()); if let Some(notary) = notary { let signing_keys = self @@ -775,9 +682,8 @@ pub(super) async fn get_signing_keys( .notary_request(¬ary, &server_name) .await?; - return Ok(RoomMessageEventContent::notice_markdown(format!( - "```rs\n{signing_keys:#?}\n```" - ))); + let out = format!("```rs\n{signing_keys:#?}\n```"); + return self.write_str(&out).await; } let signing_keys = if query { @@ -792,17 +698,13 @@ pub(super) async fn get_signing_keys( .await? }; - Ok(RoomMessageEventContent::notice_markdown(format!( - "```rs\n{signing_keys:#?}\n```" - ))) + let out = format!("```rs\n{signing_keys:#?}\n```"); + self.write_str(&out).await } #[admin_command] -pub(super) async fn get_verify_keys( - &self, - server_name: Option, -) -> Result { - let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone().into()); +pub(super) async fn get_verify_keys(&self, server_name: Option) -> Result { + let server_name = server_name.unwrap_or_else(|| self.services.server.name.clone()); let keys = self .services @@ -817,7 +719,7 @@ pub(super) async fn get_verify_keys( writeln!(out, "| {key_id} | {key:?} |")?; } - Ok(RoomMessageEventContent::notice_markdown(out)) + self.write_str(&out).await } #[admin_command] @@ -825,18 +727,16 @@ pub(super) async fn resolve_true_destination( &self, server_name: OwnedServerName, no_cache: bool, -) -> Result { +) -> Result { if !self.services.server.config.allow_federation { - return Ok(RoomMessageEventContent::text_plain( - "Federation is disabled on this homeserver.", - )); + return Err!("Federation is disabled on this homeserver.",); } if server_name == self.services.server.name { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "Not allowed to send federation requests to ourselves. Please use `get-pdu` for \ fetching local PDUs.", - )); + ); } let actual = self @@ -845,13 +745,12 @@ pub(super) async fn resolve_true_destination( .resolve_actual_dest(&server_name, !no_cache) .await?; - let msg = format!("Destination: {}\nHostname URI: {}", actual.dest, actual.host,); - - Ok(RoomMessageEventContent::text_markdown(msg)) + let msg = format!("Destination: {}\nHostname URI: {}", actual.dest, actual.host); + self.write_str(&msg).await } #[admin_command] -pub(super) async fn memory_stats(&self, opts: Option) -> Result { +pub(super) async fn memory_stats(&self, opts: Option) -> Result { const OPTS: &str = "abcdefghijklmnopqrstuvwxyz"; let opts: String = OPTS @@ -870,13 +769,12 @@ pub(super) async fn memory_stats(&self, opts: Option) -> Result Result { +pub(super) async fn runtime_metrics(&self) -> Result { let out = self.services.server.metrics.runtime_metrics().map_or_else( || "Runtime metrics are not available.".to_owned(), |metrics| { @@ -889,51 +787,51 @@ pub(super) async fn runtime_metrics(&self) -> Result { }, ); - Ok(RoomMessageEventContent::text_markdown(out)) + self.write_str(&out).await } #[cfg(not(tokio_unstable))] #[admin_command] -pub(super) async fn runtime_metrics(&self) -> Result { - Ok(RoomMessageEventContent::text_markdown( - "Runtime metrics require building with `tokio_unstable`.", - )) +pub(super) async fn runtime_metrics(&self) -> Result { + self.write_str("Runtime metrics require building with `tokio_unstable`.") + .await } #[cfg(tokio_unstable)] #[admin_command] -pub(super) async fn runtime_interval(&self) -> Result { +pub(super) async fn runtime_interval(&self) -> Result { let out = self.services.server.metrics.runtime_interval().map_or_else( || "Runtime metrics are not available.".to_owned(), |metrics| format!("```rs\n{metrics:#?}\n```"), ); - Ok(RoomMessageEventContent::text_markdown(out)) + self.write_str(&out).await } #[cfg(not(tokio_unstable))] #[admin_command] -pub(super) async fn runtime_interval(&self) -> Result { - Ok(RoomMessageEventContent::text_markdown( - "Runtime metrics require building with `tokio_unstable`.", - )) +pub(super) async fn runtime_interval(&self) -> Result { + self.write_str("Runtime metrics require building with `tokio_unstable`.") + .await } #[admin_command] -pub(super) async fn time(&self) -> Result { +pub(super) async fn time(&self) -> Result { let now = SystemTime::now(); - Ok(RoomMessageEventContent::text_markdown(utils::time::format(now, "%+"))) + let now = utils::time::format(now, "%+"); + + self.write_str(&now).await } #[admin_command] -pub(super) async fn list_dependencies(&self, names: bool) -> Result { +pub(super) async fn list_dependencies(&self, names: bool) -> Result { if names { let out = info::cargo::dependencies_names().join(" "); - return Ok(RoomMessageEventContent::notice_markdown(out)); + return self.write_str(&out).await; } - let deps = info::cargo::dependencies(); let mut out = String::new(); + let deps = info::cargo::dependencies(); writeln!(out, "| name | version | features |")?; writeln!(out, "| ---- | ------- | -------- |")?; for (name, dep) in deps { @@ -944,10 +842,11 @@ pub(super) async fn list_dependencies(&self, names: bool) -> Result, map: Option, -) -> Result { +) -> Result { let map_name = map.as_ref().map_or(EMPTY, String::as_str); let property = property.unwrap_or_else(|| "rocksdb.stats".to_owned()); self.services @@ -967,17 +866,11 @@ pub(super) async fn database_stats( let res = map.property(&property).expect("invalid property"); writeln!(self, "##### {name}:\n```\n{}\n```", res.trim()) }) - .await?; - - Ok(RoomMessageEventContent::notice_plain("")) + .await } #[admin_command] -pub(super) async fn database_files( - &self, - map: Option, - level: Option, -) -> Result { +pub(super) async fn database_files(&self, map: Option, level: Option) -> Result { let mut files: Vec<_> = self.services.db.db.file_list().collect::>()?; files.sort_by_key(|f| f.name.clone()); @@ -1004,16 +897,12 @@ pub(super) async fn database_files( file.column_family_name, ) }) - .await?; - - Ok(RoomMessageEventContent::notice_plain("")) + .await } #[admin_command] -pub(super) async fn trim_memory(&self) -> Result { +pub(super) async fn trim_memory(&self) -> Result { conduwuit::alloc::trim(None)?; - writeln!(self, "done").await?; - - Ok(RoomMessageEventContent::notice_plain("")) + writeln!(self, "done").await } diff --git a/src/admin/debug/tester.rs b/src/admin/debug/tester.rs index 005ee775..0a2b1516 100644 --- a/src/admin/debug/tester.rs +++ b/src/admin/debug/tester.rs @@ -1,7 +1,6 @@ -use conduwuit::Err; -use ruma::events::room::message::RoomMessageEventContent; +use conduwuit::{Err, Result}; -use crate::{Result, admin_command, admin_command_dispatch}; +use crate::{admin_command, admin_command_dispatch}; #[admin_command_dispatch] #[derive(Debug, clap::Subcommand)] @@ -14,14 +13,14 @@ pub(crate) enum TesterCommand { #[rustfmt::skip] #[admin_command] -async fn panic(&self) -> Result { +async fn panic(&self) -> Result { panic!("panicked") } #[rustfmt::skip] #[admin_command] -async fn failure(&self) -> Result { +async fn failure(&self) -> Result { Err!("failed") } @@ -29,20 +28,20 @@ async fn failure(&self) -> Result { #[inline(never)] #[rustfmt::skip] #[admin_command] -async fn tester(&self) -> Result { +async fn tester(&self) -> Result { - Ok(RoomMessageEventContent::notice_plain("legacy")) + self.write_str("Ok").await } #[inline(never)] #[rustfmt::skip] #[admin_command] -async fn timer(&self) -> Result { +async fn timer(&self) -> Result { let started = std::time::Instant::now(); timed(self.body); let elapsed = started.elapsed(); - Ok(RoomMessageEventContent::notice_plain(format!("completed in {elapsed:#?}"))) + self.write_str(&format!("completed in {elapsed:#?}")).await } #[inline(never)] diff --git a/src/admin/federation/commands.rs b/src/admin/federation/commands.rs index 12ed9c25..545dcbca 100644 --- a/src/admin/federation/commands.rs +++ b/src/admin/federation/commands.rs @@ -1,49 +1,48 @@ use std::fmt::Write; -use conduwuit::Result; +use conduwuit::{Err, Result}; use futures::StreamExt; -use ruma::{ - OwnedRoomId, OwnedServerName, OwnedUserId, events::room::message::RoomMessageEventContent, -}; +use ruma::{OwnedRoomId, OwnedServerName, OwnedUserId}; use crate::{admin_command, get_room_info}; #[admin_command] -pub(super) async fn disable_room(&self, room_id: OwnedRoomId) -> Result { +pub(super) async fn disable_room(&self, room_id: OwnedRoomId) -> Result { self.services.rooms.metadata.disable_room(&room_id, true); - Ok(RoomMessageEventContent::text_plain("Room disabled.")) + self.write_str("Room disabled.").await } #[admin_command] -pub(super) async fn enable_room(&self, room_id: OwnedRoomId) -> Result { +pub(super) async fn enable_room(&self, room_id: OwnedRoomId) -> Result { self.services.rooms.metadata.disable_room(&room_id, false); - Ok(RoomMessageEventContent::text_plain("Room enabled.")) + self.write_str("Room enabled.").await } #[admin_command] -pub(super) async fn incoming_federation(&self) -> Result { - let map = self - .services - .rooms - .event_handler - .federation_handletime - .read() - .expect("locked"); - let mut msg = format!("Handling {} incoming pdus:\n", map.len()); +pub(super) async fn incoming_federation(&self) -> Result { + let msg = { + let map = self + .services + .rooms + .event_handler + .federation_handletime + .read() + .expect("locked"); - for (r, (e, i)) in map.iter() { - let elapsed = i.elapsed(); - writeln!(msg, "{} {}: {}m{}s", r, e, elapsed.as_secs() / 60, elapsed.as_secs() % 60)?; - } + let mut msg = format!("Handling {} incoming pdus:\n", map.len()); + for (r, (e, i)) in map.iter() { + let elapsed = i.elapsed(); + writeln!(msg, "{} {}: {}m{}s", r, e, elapsed.as_secs() / 60, elapsed.as_secs() % 60)?; + } - Ok(RoomMessageEventContent::text_plain(&msg)) + msg + }; + + self.write_str(&msg).await } #[admin_command] -pub(super) async fn fetch_support_well_known( - &self, - server_name: OwnedServerName, -) -> Result { +pub(super) async fn fetch_support_well_known(&self, server_name: OwnedServerName) -> Result { let response = self .services .client @@ -55,54 +54,44 @@ pub(super) async fn fetch_support_well_known( let text = response.text().await?; if text.is_empty() { - return Ok(RoomMessageEventContent::text_plain("Response text/body is empty.")); + return Err!("Response text/body is empty."); } if text.len() > 1500 { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "Response text/body is over 1500 characters, assuming no support well-known.", - )); + ); } let json: serde_json::Value = match serde_json::from_str(&text) { | Ok(json) => json, | Err(_) => { - return Ok(RoomMessageEventContent::text_plain( - "Response text/body is not valid JSON.", - )); + return Err!("Response text/body is not valid JSON.",); }, }; let pretty_json: String = match serde_json::to_string_pretty(&json) { | Ok(json) => json, | Err(_) => { - return Ok(RoomMessageEventContent::text_plain( - "Response text/body is not valid JSON.", - )); + return Err!("Response text/body is not valid JSON.",); }, }; - Ok(RoomMessageEventContent::notice_markdown(format!( - "Got JSON response:\n\n```json\n{pretty_json}\n```" - ))) + self.write_str(&format!("Got JSON response:\n\n```json\n{pretty_json}\n```")) + .await } #[admin_command] -pub(super) async fn remote_user_in_rooms( - &self, - user_id: OwnedUserId, -) -> Result { +pub(super) async fn remote_user_in_rooms(&self, user_id: OwnedUserId) -> Result { if user_id.server_name() == self.services.server.name { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "User belongs to our server, please use `list-joined-rooms` user admin command \ instead.", - )); + ); } if !self.services.users.exists(&user_id).await { - return Ok(RoomMessageEventContent::text_plain( - "Remote user does not exist in our database.", - )); + return Err!("Remote user does not exist in our database.",); } let mut rooms: Vec<(OwnedRoomId, u64, String)> = self @@ -115,21 +104,19 @@ pub(super) async fn remote_user_in_rooms( .await; if rooms.is_empty() { - return Ok(RoomMessageEventContent::text_plain("User is not in any rooms.")); + return Err!("User is not in any rooms."); } rooms.sort_by_key(|r| r.1); rooms.reverse(); - let output = format!( - "Rooms {user_id} shares with us ({}):\n```\n{}\n```", - rooms.len(), - rooms - .iter() - .map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}")) - .collect::>() - .join("\n") - ); + let num = rooms.len(); + let body = rooms + .iter() + .map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}")) + .collect::>() + .join("\n"); - Ok(RoomMessageEventContent::text_markdown(output)) + self.write_str(&format!("Rooms {user_id} shares with us ({num}):\n```\n{body}\n```",)) + .await } diff --git a/src/admin/media/commands.rs b/src/admin/media/commands.rs index c8364969..7aed28db 100644 --- a/src/admin/media/commands.rs +++ b/src/admin/media/commands.rs @@ -1,13 +1,11 @@ use std::time::Duration; use conduwuit::{ - Result, debug, debug_info, debug_warn, error, info, trace, utils::time::parse_timepoint_ago, + Err, Result, debug, debug_info, debug_warn, error, info, trace, + utils::time::parse_timepoint_ago, warn, }; use conduwuit_service::media::Dim; -use ruma::{ - Mxc, OwnedEventId, OwnedMxcUri, OwnedServerName, - events::room::message::RoomMessageEventContent, -}; +use ruma::{Mxc, OwnedEventId, OwnedMxcUri, OwnedServerName}; use crate::{admin_command, utils::parse_local_user_id}; @@ -16,11 +14,9 @@ pub(super) async fn delete( &self, mxc: Option, event_id: Option, -) -> Result { +) -> Result { if event_id.is_some() && mxc.is_some() { - return Ok(RoomMessageEventContent::text_plain( - "Please specify either an MXC or an event ID, not both.", - )); + return Err!("Please specify either an MXC or an event ID, not both.",); } if let Some(mxc) = mxc { @@ -30,9 +26,7 @@ pub(super) async fn delete( .delete(&mxc.as_str().try_into()?) .await?; - return Ok(RoomMessageEventContent::text_plain( - "Deleted the MXC from our database and on our filesystem.", - )); + return Err!("Deleted the MXC from our database and on our filesystem.",); } if let Some(event_id) = event_id { @@ -113,41 +107,36 @@ pub(super) async fn delete( let final_url = url.to_string().replace('"', ""); mxc_urls.push(final_url); } else { - info!( + warn!( "Found a URL in the event ID {event_id} but did not \ start with mxc://, ignoring" ); } } else { - info!("No \"url\" key in \"file\" key."); + error!("No \"url\" key in \"file\" key."); } } } } else { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "Event ID does not have a \"content\" key or failed parsing the \ event ID JSON.", - )); + ); } } else { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "Event ID does not have a \"content\" key, this is not a message or an \ event type that contains media.", - )); + ); } }, | _ => { - return Ok(RoomMessageEventContent::text_plain( - "Event ID does not exist or is not known to us.", - )); + return Err!("Event ID does not exist or is not known to us.",); }, } if mxc_urls.is_empty() { - info!("Parsed event ID {event_id} but did not contain any MXC URLs."); - return Ok(RoomMessageEventContent::text_plain( - "Parsed event ID but found no MXC URLs.", - )); + return Err!("Parsed event ID but found no MXC URLs.",); } let mut mxc_deletion_count: usize = 0; @@ -170,27 +159,27 @@ pub(super) async fn delete( } } - return Ok(RoomMessageEventContent::text_plain(format!( - "Deleted {mxc_deletion_count} total MXCs from our database and the filesystem from \ - event ID {event_id}." - ))); + return self + .write_str(&format!( + "Deleted {mxc_deletion_count} total MXCs from our database and the filesystem \ + from event ID {event_id}." + )) + .await; } - Ok(RoomMessageEventContent::text_plain( + Err!( "Please specify either an MXC using --mxc or an event ID using --event-id of the \ - message containing an image. See --help for details.", - )) + message containing an image. See --help for details." + ) } #[admin_command] -pub(super) async fn delete_list(&self) -> Result { +pub(super) async fn delete_list(&self) -> Result { if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" { - return Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )); + return Err!("Expected code block in command body. Add --help for details.",); } let mut failed_parsed_mxcs: usize = 0; @@ -204,7 +193,6 @@ pub(super) async fn delete_list(&self) -> Result { .try_into() .inspect_err(|e| { debug_warn!("Failed to parse user-provided MXC URI: {e}"); - failed_parsed_mxcs = failed_parsed_mxcs.saturating_add(1); }) .ok() @@ -227,10 +215,11 @@ pub(super) async fn delete_list(&self) -> Result { } } - Ok(RoomMessageEventContent::text_plain(format!( + self.write_str(&format!( "Finished bulk MXC deletion, deleted {mxc_deletion_count} total MXCs from our database \ and the filesystem. {failed_parsed_mxcs} MXCs failed to be parsed from the database.", - ))) + )) + .await } #[admin_command] @@ -240,11 +229,9 @@ pub(super) async fn delete_past_remote_media( before: bool, after: bool, yes_i_want_to_delete_local_media: bool, -) -> Result { +) -> Result { if before && after { - return Ok(RoomMessageEventContent::text_plain( - "Please only pick one argument, --before or --after.", - )); + return Err!("Please only pick one argument, --before or --after.",); } assert!(!(before && after), "--before and --after should not be specified together"); @@ -260,23 +247,18 @@ pub(super) async fn delete_past_remote_media( ) .await?; - Ok(RoomMessageEventContent::text_plain(format!( - "Deleted {deleted_count} total files.", - ))) + self.write_str(&format!("Deleted {deleted_count} total files.",)) + .await } #[admin_command] -pub(super) async fn delete_all_from_user( - &self, - username: String, -) -> Result { +pub(super) async fn delete_all_from_user(&self, username: String) -> Result { let user_id = parse_local_user_id(self.services, &username)?; let deleted_count = self.services.media.delete_from_user(&user_id).await?; - Ok(RoomMessageEventContent::text_plain(format!( - "Deleted {deleted_count} total files.", - ))) + self.write_str(&format!("Deleted {deleted_count} total files.",)) + .await } #[admin_command] @@ -284,11 +266,9 @@ pub(super) async fn delete_all_from_server( &self, server_name: OwnedServerName, yes_i_want_to_delete_local_media: bool, -) -> Result { +) -> Result { if server_name == self.services.globals.server_name() && !yes_i_want_to_delete_local_media { - return Ok(RoomMessageEventContent::text_plain( - "This command only works for remote media by default.", - )); + return Err!("This command only works for remote media by default.",); } let Ok(all_mxcs) = self @@ -298,9 +278,7 @@ pub(super) async fn delete_all_from_server( .await .inspect_err(|e| error!("Failed to get MXC URIs from our database: {e}")) else { - return Ok(RoomMessageEventContent::text_plain( - "Failed to get MXC URIs from our database", - )); + return Err!("Failed to get MXC URIs from our database",); }; let mut deleted_count: usize = 0; @@ -336,17 +314,16 @@ pub(super) async fn delete_all_from_server( } } - Ok(RoomMessageEventContent::text_plain(format!( - "Deleted {deleted_count} total files.", - ))) + self.write_str(&format!("Deleted {deleted_count} total files.",)) + .await } #[admin_command] -pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result { +pub(super) async fn get_file_info(&self, mxc: OwnedMxcUri) -> Result { let mxc: Mxc<'_> = mxc.as_str().try_into()?; let metadata = self.services.media.get_metadata(&mxc).await; - Ok(RoomMessageEventContent::notice_markdown(format!("```\n{metadata:#?}\n```"))) + self.write_str(&format!("```\n{metadata:#?}\n```")).await } #[admin_command] @@ -355,7 +332,7 @@ pub(super) async fn get_remote_file( mxc: OwnedMxcUri, server: Option, timeout: u32, -) -> Result { +) -> Result { let mxc: Mxc<'_> = mxc.as_str().try_into()?; let timeout = Duration::from_millis(timeout.into()); let mut result = self @@ -368,8 +345,8 @@ pub(super) async fn get_remote_file( let len = result.content.as_ref().expect("content").len(); result.content.as_mut().expect("content").clear(); - let out = format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"); - Ok(RoomMessageEventContent::notice_markdown(out)) + self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```")) + .await } #[admin_command] @@ -380,7 +357,7 @@ pub(super) async fn get_remote_thumbnail( timeout: u32, width: u32, height: u32, -) -> Result { +) -> Result { let mxc: Mxc<'_> = mxc.as_str().try_into()?; let timeout = Duration::from_millis(timeout.into()); let dim = Dim::new(width, height, None); @@ -394,6 +371,6 @@ pub(super) async fn get_remote_thumbnail( let len = result.content.as_ref().expect("content").len(); result.content.as_mut().expect("content").clear(); - let out = format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```"); - Ok(RoomMessageEventContent::notice_markdown(out)) + self.write_str(&format!("```\n{result:#?}\nreceived {len} bytes for file content.\n```")) + .await } diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 695155e8..1f777fa9 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -4,7 +4,7 @@ #![allow(clippy::too_many_arguments)] pub(crate) mod admin; -pub(crate) mod command; +pub(crate) mod context; pub(crate) mod processor; mod tests; pub(crate) mod utils; @@ -23,13 +23,9 @@ extern crate conduwuit_api as api; extern crate conduwuit_core as conduwuit; extern crate conduwuit_service as service; -pub(crate) use conduwuit::Result; pub(crate) use conduwuit_macros::{admin_command, admin_command_dispatch}; -pub(crate) use crate::{ - command::Command, - utils::{escape_html, get_room_info}, -}; +pub(crate) use crate::{context::Context, utils::get_room_info}; pub(crate) const PAGE_SIZE: usize = 100; diff --git a/src/admin/processor.rs b/src/admin/processor.rs index 53a15098..8282a846 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -33,7 +33,7 @@ use service::{ use tracing::Level; use tracing_subscriber::{EnvFilter, filter::LevelFilter}; -use crate::{Command, admin, admin::AdminCommand}; +use crate::{admin, admin::AdminCommand, context::Context}; #[must_use] pub(super) fn complete(line: &str) -> String { complete_command(AdminCommand::command(), line) } @@ -58,7 +58,7 @@ async fn process_command(services: Arc, input: &CommandInput) -> Proce | Ok(parsed) => parsed, }; - let context = Command { + let context = Context { services: &services, body: &body, timer: SystemTime::now(), @@ -103,7 +103,7 @@ fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult { /// Parse and process a message from the admin room async fn process( - context: &Command<'_>, + context: &Context<'_>, command: AdminCommand, args: &[String], ) -> (Result, String) { @@ -132,7 +132,7 @@ async fn process( (result, output) } -fn capture_create(context: &Command<'_>) -> (Arc, Arc>) { +fn capture_create(context: &Context<'_>) -> (Arc, Arc>) { let env_config = &context.services.server.config.admin_log_capture; let env_filter = EnvFilter::try_new(env_config).unwrap_or_else(|e| { warn!("admin_log_capture filter invalid: {e:?}"); diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index 879aed16..228d2120 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::StreamExt; -use ruma::{OwnedRoomId, OwnedUserId, events::room::message::RoomMessageEventContent}; +use ruma::{OwnedRoomId, OwnedUserId}; use crate::{admin_command, admin_command_dispatch}; @@ -36,7 +36,7 @@ async fn changes_since( user_id: OwnedUserId, since: u64, room_id: Option, -) -> Result { +) -> Result { let timer = tokio::time::Instant::now(); let results: Vec<_> = self .services @@ -46,9 +46,8 @@ async fn changes_since( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")) + .await } #[admin_command] @@ -57,7 +56,7 @@ async fn account_data_get( user_id: OwnedUserId, kind: String, room_id: Option, -) -> Result { +) -> Result { let timer = tokio::time::Instant::now(); let results = self .services @@ -66,7 +65,6 @@ async fn account_data_get( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")) + .await } diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 0359261a..28bf6451 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -2,7 +2,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::TryStreamExt; -use crate::Command; +use crate::Context; #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/appservice.rs @@ -18,7 +18,7 @@ pub(crate) enum AppserviceCommand { } /// All the getters and iterators from src/database/key_value/appservice.rs -pub(super) async fn process(subcommand: AppserviceCommand, context: &Command<'_>) -> Result { +pub(super) async fn process(subcommand: AppserviceCommand, context: &Context<'_>) -> Result { let services = context.services; match subcommand { diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 33810704..3681acfd 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -2,7 +2,7 @@ use clap::Subcommand; use conduwuit::Result; use ruma::OwnedServerName; -use crate::Command; +use crate::Context; #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/globals.rs @@ -21,7 +21,7 @@ pub(crate) enum GlobalsCommand { } /// All the getters and iterators from src/database/key_value/globals.rs -pub(super) async fn process(subcommand: GlobalsCommand, context: &Command<'_>) -> Result { +pub(super) async fn process(subcommand: GlobalsCommand, context: &Context<'_>) -> Result { let services = context.services; match subcommand { diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index 65164802..5b7ead4b 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -3,7 +3,7 @@ use conduwuit::Result; use futures::StreamExt; use ruma::OwnedUserId; -use crate::Command; +use crate::Context; #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/presence.rs @@ -23,7 +23,7 @@ pub(crate) enum PresenceCommand { } /// All the getters and iterators in key_value/presence.rs -pub(super) async fn process(subcommand: PresenceCommand, context: &Command<'_>) -> Result { +pub(super) async fn process(subcommand: PresenceCommand, context: &Context<'_>) -> Result { let services = context.services; match subcommand { diff --git a/src/admin/query/pusher.rs b/src/admin/query/pusher.rs index 583c4999..0d0e6cc9 100644 --- a/src/admin/query/pusher.rs +++ b/src/admin/query/pusher.rs @@ -2,7 +2,7 @@ use clap::Subcommand; use conduwuit::Result; use ruma::OwnedUserId; -use crate::Command; +use crate::Context; #[derive(Debug, Subcommand)] pub(crate) enum PusherCommand { @@ -13,7 +13,7 @@ pub(crate) enum PusherCommand { }, } -pub(super) async fn process(subcommand: PusherCommand, context: &Command<'_>) -> Result { +pub(super) async fn process(subcommand: PusherCommand, context: &Context<'_>) -> Result { let services = context.services; match subcommand { diff --git a/src/admin/query/raw.rs b/src/admin/query/raw.rs index c503eee5..0e248c65 100644 --- a/src/admin/query/raw.rs +++ b/src/admin/query/raw.rs @@ -11,7 +11,6 @@ use conduwuit::{ use conduwuit_database::Map; use conduwuit_service::Services; use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; -use ruma::events::room::message::RoomMessageEventContent; use tokio::time::Instant; use crate::{admin_command, admin_command_dispatch}; @@ -170,7 +169,7 @@ pub(super) async fn compact( into: Option, parallelism: Option, exhaustive: bool, -) -> Result { +) -> Result { use conduwuit_database::compact::Options; let default_all_maps: Option<_> = map.is_none().then(|| { @@ -221,17 +220,11 @@ pub(super) async fn compact( let results = results.await; let query_time = timer.elapsed(); self.write_str(&format!("Jobs completed in {query_time:?}:\n\n```rs\n{results:#?}\n```")) - .await?; - - Ok(RoomMessageEventContent::text_plain("")) + .await } #[admin_command] -pub(super) async fn raw_count( - &self, - map: Option, - prefix: Option, -) -> Result { +pub(super) async fn raw_count(&self, map: Option, prefix: Option) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); let timer = Instant::now(); @@ -242,17 +235,11 @@ pub(super) async fn raw_count( let query_time = timer.elapsed(); self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{count:#?}\n```")) - .await?; - - Ok(RoomMessageEventContent::text_plain("")) + .await } #[admin_command] -pub(super) async fn raw_keys( - &self, - map: String, - prefix: Option, -) -> Result { +pub(super) async fn raw_keys(&self, map: String, prefix: Option) -> Result { writeln!(self, "```").boxed().await?; let map = self.services.db.get(map.as_str())?; @@ -266,18 +253,12 @@ pub(super) async fn raw_keys( .await?; let query_time = timer.elapsed(); - let out = format!("\n```\n\nQuery completed in {query_time:?}"); - self.write_str(out.as_str()).await?; - - Ok(RoomMessageEventContent::text_plain("")) + self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) + .await } #[admin_command] -pub(super) async fn raw_keys_sizes( - &self, - map: Option, - prefix: Option, -) -> Result { +pub(super) async fn raw_keys_sizes(&self, map: Option, prefix: Option) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); let timer = Instant::now(); @@ -294,18 +275,12 @@ pub(super) async fn raw_keys_sizes( .await; let query_time = timer.elapsed(); - let result = format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"); - self.write_str(result.as_str()).await?; - - Ok(RoomMessageEventContent::text_plain("")) + self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}")) + .await } #[admin_command] -pub(super) async fn raw_keys_total( - &self, - map: Option, - prefix: Option, -) -> Result { +pub(super) async fn raw_keys_total(&self, map: Option, prefix: Option) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); let timer = Instant::now(); @@ -318,19 +293,12 @@ pub(super) async fn raw_keys_total( .await; let query_time = timer.elapsed(); - self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}")) - .await?; - - Ok(RoomMessageEventContent::text_plain("")) + .await } #[admin_command] -pub(super) async fn raw_vals_sizes( - &self, - map: Option, - prefix: Option, -) -> Result { +pub(super) async fn raw_vals_sizes(&self, map: Option, prefix: Option) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); let timer = Instant::now(); @@ -348,18 +316,12 @@ pub(super) async fn raw_vals_sizes( .await; let query_time = timer.elapsed(); - let result = format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}"); - self.write_str(result.as_str()).await?; - - Ok(RoomMessageEventContent::text_plain("")) + self.write_str(&format!("```\n{result:#?}\n```\n\nQuery completed in {query_time:?}")) + .await } #[admin_command] -pub(super) async fn raw_vals_total( - &self, - map: Option, - prefix: Option, -) -> Result { +pub(super) async fn raw_vals_total(&self, map: Option, prefix: Option) -> Result { let prefix = prefix.as_deref().unwrap_or(EMPTY); let timer = Instant::now(); @@ -373,19 +335,12 @@ pub(super) async fn raw_vals_total( .await; let query_time = timer.elapsed(); - self.write_str(&format!("```\n{result:#?}\n\n```\n\nQuery completed in {query_time:?}")) - .await?; - - Ok(RoomMessageEventContent::text_plain("")) + .await } #[admin_command] -pub(super) async fn raw_iter( - &self, - map: String, - prefix: Option, -) -> Result { +pub(super) async fn raw_iter(&self, map: String, prefix: Option) -> Result { writeln!(self, "```").await?; let map = self.services.db.get(&map)?; @@ -401,9 +356,7 @@ pub(super) async fn raw_iter( let query_time = timer.elapsed(); self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) - .await?; - - Ok(RoomMessageEventContent::text_plain("")) + .await } #[admin_command] @@ -412,7 +365,7 @@ pub(super) async fn raw_keys_from( map: String, start: String, limit: Option, -) -> Result { +) -> Result { writeln!(self, "```").await?; let map = self.services.db.get(&map)?; @@ -426,9 +379,7 @@ pub(super) async fn raw_keys_from( let query_time = timer.elapsed(); self.write_str(&format!("\n```\n\nQuery completed in {query_time:?}")) - .await?; - - Ok(RoomMessageEventContent::text_plain("")) + .await } #[admin_command] @@ -437,7 +388,7 @@ pub(super) async fn raw_iter_from( map: String, start: String, limit: Option, -) -> Result { +) -> Result { let map = self.services.db.get(&map)?; let timer = Instant::now(); let result = map @@ -449,41 +400,38 @@ pub(super) async fn raw_iter_from( .await?; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -pub(super) async fn raw_del(&self, map: String, key: String) -> Result { +pub(super) async fn raw_del(&self, map: String, key: String) -> Result { let map = self.services.db.get(&map)?; let timer = Instant::now(); map.remove(&key); - let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Operation completed in {query_time:?}" - ))) + let query_time = timer.elapsed(); + self.write_str(&format!("Operation completed in {query_time:?}")) + .await } #[admin_command] -pub(super) async fn raw_get(&self, map: String, key: String) -> Result { +pub(super) async fn raw_get(&self, map: String, key: String) -> Result { let map = self.services.db.get(&map)?; let timer = Instant::now(); let handle = map.get(&key).await?; + let query_time = timer.elapsed(); let result = String::from_utf8_lossy(&handle); - - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```")) + .await } #[admin_command] -pub(super) async fn raw_maps(&self) -> Result { +pub(super) async fn raw_maps(&self) -> Result { let list: Vec<_> = self.services.db.iter().map(at!(0)).copied().collect(); - Ok(RoomMessageEventContent::notice_markdown(format!("{list:#?}"))) + self.write_str(&format!("{list:#?}")).await } fn with_maps_or<'a>( diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index 10748d88..4a39a40e 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::{Result, utils::time}; use futures::StreamExt; -use ruma::{OwnedServerName, events::room::message::RoomMessageEventContent}; +use ruma::OwnedServerName; use crate::{admin_command, admin_command_dispatch}; @@ -21,10 +21,7 @@ pub(crate) enum ResolverCommand { } #[admin_command] -async fn destinations_cache( - &self, - server_name: Option, -) -> Result { +async fn destinations_cache(&self, server_name: Option) -> Result { use service::resolver::cache::CachedDest; writeln!(self, "| Server Name | Destination | Hostname | Expires |").await?; @@ -44,11 +41,11 @@ async fn destinations_cache( .await?; } - Ok(RoomMessageEventContent::notice_plain("")) + Ok(()) } #[admin_command] -async fn overrides_cache(&self, server_name: Option) -> Result { +async fn overrides_cache(&self, server_name: Option) -> Result { use service::resolver::cache::CachedOverride; writeln!(self, "| Server Name | IP | Port | Expires | Overriding |").await?; @@ -70,5 +67,5 @@ async fn overrides_cache(&self, server_name: Option) -> Result) -> Result { +pub(super) async fn process(subcommand: RoomAliasCommand, context: &Context<'_>) -> Result { let services = context.services; match subcommand { diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index 7f5e2536..c64cd173 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -1,11 +1,9 @@ use clap::Subcommand; -use conduwuit::{Error, Result}; +use conduwuit::Result; use futures::StreamExt; -use ruma::{ - OwnedRoomId, OwnedServerName, OwnedUserId, events::room::message::RoomMessageEventContent, -}; +use ruma::{OwnedRoomId, OwnedServerName, OwnedUserId}; -use crate::Command; +use crate::Context; #[derive(Debug, Subcommand)] pub(crate) enum RoomStateCacheCommand { @@ -78,10 +76,10 @@ pub(crate) enum RoomStateCacheCommand { }, } -pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command<'_>) -> Result { +pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Context<'_>) -> Result { let services = context.services; - let c = match subcommand { + match subcommand { | RoomStateCacheCommand::ServerInRoom { server, room_id } => { let timer = tokio::time::Instant::now(); let result = services @@ -91,9 +89,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" + )) + .await }, | RoomStateCacheCommand::RoomServers { room_id } => { let timer = tokio::time::Instant::now(); @@ -106,9 +106,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::ServerRooms { server } => { let timer = tokio::time::Instant::now(); @@ -121,9 +123,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::RoomMembers { room_id } => { let timer = tokio::time::Instant::now(); @@ -136,9 +140,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::LocalUsersInRoom { room_id } => { let timer = tokio::time::Instant::now(); @@ -151,9 +157,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::ActiveLocalUsersInRoom { room_id } => { let timer = tokio::time::Instant::now(); @@ -166,18 +174,22 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::RoomJoinedCount { room_id } => { let timer = tokio::time::Instant::now(); let results = services.rooms.state_cache.room_joined_count(&room_id).await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::RoomInvitedCount { room_id } => { let timer = tokio::time::Instant::now(); @@ -188,9 +200,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::RoomUserOnceJoined { room_id } => { let timer = tokio::time::Instant::now(); @@ -203,9 +217,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::RoomMembersInvited { room_id } => { let timer = tokio::time::Instant::now(); @@ -218,9 +234,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::GetInviteCount { room_id, user_id } => { let timer = tokio::time::Instant::now(); @@ -231,9 +249,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::GetLeftCount { room_id, user_id } => { let timer = tokio::time::Instant::now(); @@ -244,9 +264,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::RoomsJoined { user_id } => { let timer = tokio::time::Instant::now(); @@ -259,9 +281,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::RoomsInvited { user_id } => { let timer = tokio::time::Instant::now(); @@ -273,9 +297,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::RoomsLeft { user_id } => { let timer = tokio::time::Instant::now(); @@ -287,9 +313,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, | RoomStateCacheCommand::InviteState { user_id, room_id } => { let timer = tokio::time::Instant::now(); @@ -300,13 +328,11 @@ pub(super) async fn process(subcommand: RoomStateCacheCommand, context: &Command .await; let query_time = timer.elapsed(); - Result::<_, Error>::Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, - }?; - - context.write_str(c.body()).await?; - - Ok(()) + } } diff --git a/src/admin/query/room_timeline.rs b/src/admin/query/room_timeline.rs index 6f08aee9..0fd22ca7 100644 --- a/src/admin/query/room_timeline.rs +++ b/src/admin/query/room_timeline.rs @@ -1,7 +1,7 @@ use clap::Subcommand; use conduwuit::{PduCount, Result, utils::stream::TryTools}; use futures::TryStreamExt; -use ruma::{OwnedRoomOrAliasId, events::room::message::RoomMessageEventContent}; +use ruma::OwnedRoomOrAliasId; use crate::{admin_command, admin_command_dispatch}; @@ -24,7 +24,7 @@ pub(crate) enum RoomTimelineCommand { } #[admin_command] -pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result { +pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result { let room_id = self.services.rooms.alias.resolve(&room_id).await?; let result = self @@ -34,7 +34,7 @@ pub(super) async fn last(&self, room_id: OwnedRoomOrAliasId) -> Result, limit: Option, -) -> Result { +) -> Result { let room_id = self.services.rooms.alias.resolve(&room_id).await?; let from: Option = from.as_deref().map(str::parse).transpose()?; @@ -57,5 +57,5 @@ pub(super) async fn pdus( .try_collect() .await?; - Ok(RoomMessageEventContent::notice_markdown(format!("{result:#?}"))) + self.write_str(&format!("{result:#?}")).await } diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 860bca4a..8b1676bc 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -1,10 +1,10 @@ use clap::Subcommand; -use conduwuit::Result; +use conduwuit::{Err, Result}; use futures::StreamExt; -use ruma::{OwnedServerName, OwnedUserId, events::room::message::RoomMessageEventContent}; +use ruma::{OwnedServerName, OwnedUserId}; use service::sending::Destination; -use crate::Command; +use crate::Context; #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/sending.rs @@ -62,17 +62,7 @@ pub(crate) enum SendingCommand { } /// All the getters and iterators in key_value/sending.rs -pub(super) async fn process(subcommand: SendingCommand, context: &Command<'_>) -> Result { - let c = reprocess(subcommand, context).await?; - context.write_str(c.body()).await?; - Ok(()) -} - -/// All the getters and iterators in key_value/sending.rs -pub(super) async fn reprocess( - subcommand: SendingCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(subcommand: SendingCommand, context: &Context<'_>) -> Result { let services = context.services; match subcommand { @@ -82,9 +72,11 @@ pub(super) async fn reprocess( let active_requests = results.collect::>().await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" + )) + .await }, | SendingCommand::QueuedRequests { appservice_id, @@ -97,19 +89,19 @@ pub(super) async fn reprocess( && user_id.is_none() && push_key.is_none() { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. See --help for more details.", - )); + ); } let timer = tokio::time::Instant::now(); let results = match (appservice_id, server_name, user_id, push_key) { | (Some(appservice_id), None, None, None) => { if appservice_id.is_empty() { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. See --help for more details.", - )); + ); } services @@ -123,10 +115,10 @@ pub(super) async fn reprocess( .queued_requests(&Destination::Federation(server_name)), | (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. See --help for more details.", - )); + ); } services @@ -135,25 +127,27 @@ pub(super) async fn reprocess( .queued_requests(&Destination::Push(user_id, push_key)) }, | (Some(_), Some(_), Some(_), Some(_)) => { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. Not all of them See --help for more details.", - )); + ); }, | _ => { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. See --help for more details.", - )); + ); }, }; let queued_requests = results.collect::>().await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{queued_requests:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{queued_requests:#?}\n```" + )) + .await }, | SendingCommand::ActiveRequestsFor { appservice_id, @@ -166,20 +160,20 @@ pub(super) async fn reprocess( && user_id.is_none() && push_key.is_none() { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. See --help for more details.", - )); + ); } let timer = tokio::time::Instant::now(); let results = match (appservice_id, server_name, user_id, push_key) { | (Some(appservice_id), None, None, None) => { if appservice_id.is_empty() { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. See --help for more details.", - )); + ); } services @@ -193,10 +187,10 @@ pub(super) async fn reprocess( .active_requests_for(&Destination::Federation(server_name)), | (None, None, Some(user_id), Some(push_key)) => { if push_key.is_empty() { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. See --help for more details.", - )); + ); } services @@ -205,34 +199,38 @@ pub(super) async fn reprocess( .active_requests_for(&Destination::Push(user_id, push_key)) }, | (Some(_), Some(_), Some(_), Some(_)) => { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. Not all of them See --help for more details.", - )); + ); }, | _ => { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "An appservice ID, server name, or a user ID with push key must be \ specified via arguments. See --help for more details.", - )); + ); }, }; let active_requests = results.collect::>().await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{active_requests:#?}\n```" + )) + .await }, | SendingCommand::GetLatestEduCount { server_name } => { let timer = tokio::time::Instant::now(); let results = services.sending.db.get_latest_educount(&server_name).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" - ))) + context + .write_str(&format!( + "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```" + )) + .await }, } } diff --git a/src/admin/query/short.rs b/src/admin/query/short.rs index 0957c15e..aa7c8666 100644 --- a/src/admin/query/short.rs +++ b/src/admin/query/short.rs @@ -1,6 +1,6 @@ use clap::Subcommand; use conduwuit::Result; -use ruma::{OwnedEventId, OwnedRoomOrAliasId, events::room::message::RoomMessageEventContent}; +use ruma::{OwnedEventId, OwnedRoomOrAliasId}; use crate::{admin_command, admin_command_dispatch}; @@ -18,10 +18,7 @@ pub(crate) enum ShortCommand { } #[admin_command] -pub(super) async fn short_event_id( - &self, - event_id: OwnedEventId, -) -> Result { +pub(super) async fn short_event_id(&self, event_id: OwnedEventId) -> Result { let shortid = self .services .rooms @@ -29,17 +26,14 @@ pub(super) async fn short_event_id( .get_shorteventid(&event_id) .await?; - Ok(RoomMessageEventContent::notice_markdown(format!("{shortid:#?}"))) + self.write_str(&format!("{shortid:#?}")).await } #[admin_command] -pub(super) async fn short_room_id( - &self, - room_id: OwnedRoomOrAliasId, -) -> Result { +pub(super) async fn short_room_id(&self, room_id: OwnedRoomOrAliasId) -> Result { let room_id = self.services.rooms.alias.resolve(&room_id).await?; let shortid = self.services.rooms.short.get_shortroomid(&room_id).await?; - Ok(RoomMessageEventContent::notice_markdown(format!("{shortid:#?}"))) + self.write_str(&format!("{shortid:#?}")).await } diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index 5995bc62..0f34d13f 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -1,9 +1,7 @@ use clap::Subcommand; use conduwuit::Result; use futures::stream::StreamExt; -use ruma::{ - OwnedDeviceId, OwnedRoomId, OwnedUserId, events::room::message::RoomMessageEventContent, -}; +use ruma::{OwnedDeviceId, OwnedRoomId, OwnedUserId}; use crate::{admin_command, admin_command_dispatch}; @@ -99,11 +97,7 @@ pub(crate) enum UsersCommand { } #[admin_command] -async fn get_shared_rooms( - &self, - user_a: OwnedUserId, - user_b: OwnedUserId, -) -> Result { +async fn get_shared_rooms(&self, user_a: OwnedUserId, user_b: OwnedUserId) -> Result { let timer = tokio::time::Instant::now(); let result: Vec<_> = self .services @@ -115,9 +109,8 @@ async fn get_shared_rooms( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] @@ -127,7 +120,7 @@ async fn get_backup_session( version: String, room_id: OwnedRoomId, session_id: String, -) -> Result { +) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -136,9 +129,8 @@ async fn get_backup_session( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] @@ -147,7 +139,7 @@ async fn get_room_backups( user_id: OwnedUserId, version: String, room_id: OwnedRoomId, -) -> Result { +) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -156,32 +148,22 @@ async fn get_room_backups( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn get_all_backups( - &self, - user_id: OwnedUserId, - version: String, -) -> Result { +async fn get_all_backups(&self, user_id: OwnedUserId, version: String) -> Result { let timer = tokio::time::Instant::now(); let result = self.services.key_backups.get_all(&user_id, &version).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn get_backup_algorithm( - &self, - user_id: OwnedUserId, - version: String, -) -> Result { +async fn get_backup_algorithm(&self, user_id: OwnedUserId, version: String) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -190,16 +172,12 @@ async fn get_backup_algorithm( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn get_latest_backup_version( - &self, - user_id: OwnedUserId, -) -> Result { +async fn get_latest_backup_version(&self, user_id: OwnedUserId) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -208,36 +186,33 @@ async fn get_latest_backup_version( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result { +async fn get_latest_backup(&self, user_id: OwnedUserId) -> Result { let timer = tokio::time::Instant::now(); let result = self.services.key_backups.get_latest_backup(&user_id).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn iter_users(&self) -> Result { +async fn iter_users(&self) -> Result { let timer = tokio::time::Instant::now(); let result: Vec = self.services.users.stream().map(Into::into).collect().await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn iter_users2(&self) -> Result { +async fn iter_users2(&self) -> Result { let timer = tokio::time::Instant::now(); let result: Vec<_> = self.services.users.stream().collect().await; let result: Vec<_> = result @@ -248,35 +223,32 @@ async fn iter_users2(&self) -> Result { let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:?}\n```")) + .await } #[admin_command] -async fn count_users(&self) -> Result { +async fn count_users(&self) -> Result { let timer = tokio::time::Instant::now(); let result = self.services.users.count().await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn password_hash(&self, user_id: OwnedUserId) -> Result { +async fn password_hash(&self, user_id: OwnedUserId) -> Result { let timer = tokio::time::Instant::now(); let result = self.services.users.password_hash(&user_id).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn list_devices(&self, user_id: OwnedUserId) -> Result { +async fn list_devices(&self, user_id: OwnedUserId) -> Result { let timer = tokio::time::Instant::now(); let devices = self .services @@ -288,13 +260,12 @@ async fn list_devices(&self, user_id: OwnedUserId) -> Result Result { +async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result { let timer = tokio::time::Instant::now(); let devices = self .services @@ -304,17 +275,12 @@ async fn list_devices_metadata(&self, user_id: OwnedUserId) -> Result Result { +async fn get_device_metadata(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { let timer = tokio::time::Instant::now(); let device = self .services @@ -323,28 +289,22 @@ async fn get_device_metadata( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```")) + .await } #[admin_command] -async fn get_devices_version(&self, user_id: OwnedUserId) -> Result { +async fn get_devices_version(&self, user_id: OwnedUserId) -> Result { let timer = tokio::time::Instant::now(); let device = self.services.users.get_devicelist_version(&user_id).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{device:#?}\n```")) + .await } #[admin_command] -async fn count_one_time_keys( - &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, -) -> Result { +async fn count_one_time_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -353,17 +313,12 @@ async fn count_one_time_keys( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn get_device_keys( - &self, - user_id: OwnedUserId, - device_id: OwnedDeviceId, -) -> Result { +async fn get_device_keys(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -372,24 +327,22 @@ async fn get_device_keys( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result { +async fn get_user_signing_key(&self, user_id: OwnedUserId) -> Result { let timer = tokio::time::Instant::now(); let result = self.services.users.get_user_signing_key(&user_id).await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } #[admin_command] -async fn get_master_key(&self, user_id: OwnedUserId) -> Result { +async fn get_master_key(&self, user_id: OwnedUserId) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -398,17 +351,12 @@ async fn get_master_key(&self, user_id: OwnedUserId) -> Result Result { +async fn get_to_device_events(&self, user_id: OwnedUserId, device_id: OwnedDeviceId) -> Result { let timer = tokio::time::Instant::now(); let result = self .services @@ -418,7 +366,6 @@ async fn get_to_device_events( .await; let query_time = timer.elapsed(); - Ok(RoomMessageEventContent::notice_markdown(format!( - "Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```" - ))) + self.write_str(&format!("Query completed in {query_time:?}:\n\n```rs\n{result:#?}\n```")) + .await } diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 4cfff2e5..6b37ffe4 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -1,11 +1,11 @@ use std::fmt::Write; use clap::Subcommand; -use conduwuit::Result; +use conduwuit::{Err, Result}; use futures::StreamExt; -use ruma::{OwnedRoomAliasId, OwnedRoomId, events::room::message::RoomMessageEventContent}; +use ruma::{OwnedRoomAliasId, OwnedRoomId}; -use crate::{Command, escape_html}; +use crate::Context; #[derive(Debug, Subcommand)] pub(crate) enum RoomAliasCommand { @@ -42,17 +42,7 @@ pub(crate) enum RoomAliasCommand { }, } -pub(super) async fn process(command: RoomAliasCommand, context: &Command<'_>) -> Result { - let c = reprocess(command, context).await?; - context.write_str(c.body()).await?; - - Ok(()) -} - -pub(super) async fn reprocess( - command: RoomAliasCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(command: RoomAliasCommand, context: &Context<'_>) -> Result { let services = context.services; let server_user = &services.globals.server_user; @@ -65,9 +55,7 @@ pub(super) async fn reprocess( let room_alias = match OwnedRoomAliasId::parse(room_alias_str) { | Ok(alias) => alias, | Err(err) => { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to parse alias: {err}" - ))); + return Err!("Failed to parse alias: {err}"); }, }; match command { @@ -79,60 +67,50 @@ pub(super) async fn reprocess( &room_id, server_user, ) { - | Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( - "Successfully overwrote alias (formerly {id})" - ))), - | Err(err) => Ok(RoomMessageEventContent::text_plain(format!( - "Failed to remove alias: {err}" - ))), + | Err(err) => Err!("Failed to remove alias: {err}"), + | Ok(()) => + context + .write_str(&format!( + "Successfully overwrote alias (formerly {id})" + )) + .await, } }, - | (false, Ok(id)) => Ok(RoomMessageEventContent::text_plain(format!( + | (false, Ok(id)) => Err!( "Refusing to overwrite in use alias for {id}, use -f or --force to \ overwrite" - ))), + ), | (_, Err(_)) => { match services.rooms.alias.set_alias( &room_alias, &room_id, server_user, ) { - | Ok(()) => Ok(RoomMessageEventContent::text_plain( - "Successfully set alias", - )), - | Err(err) => Ok(RoomMessageEventContent::text_plain(format!( - "Failed to remove alias: {err}" - ))), + | Err(err) => Err!("Failed to remove alias: {err}"), + | Ok(()) => context.write_str("Successfully set alias").await, } }, } }, | RoomAliasCommand::Remove { .. } => { match services.rooms.alias.resolve_local_alias(&room_alias).await { + | Err(_) => Err!("Alias isn't in use."), | Ok(id) => match services .rooms .alias .remove_alias(&room_alias, server_user) .await { - | Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( - "Removed alias from {id}" - ))), - | Err(err) => Ok(RoomMessageEventContent::text_plain(format!( - "Failed to remove alias: {err}" - ))), + | Err(err) => Err!("Failed to remove alias: {err}"), + | Ok(()) => + context.write_str(&format!("Removed alias from {id}")).await, }, - | Err(_) => - Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), } }, | RoomAliasCommand::Which { .. } => { match services.rooms.alias.resolve_local_alias(&room_alias).await { - | Ok(id) => Ok(RoomMessageEventContent::text_plain(format!( - "Alias resolves to {id}" - ))), - | Err(_) => - Ok(RoomMessageEventContent::text_plain("Alias isn't in use.")), + | Err(_) => Err!("Alias isn't in use."), + | Ok(id) => context.write_str(&format!("Alias resolves to {id}")).await, } }, | RoomAliasCommand::List { .. } => unreachable!(), @@ -154,15 +132,8 @@ pub(super) async fn reprocess( output }); - let html_list = aliases.iter().fold(String::new(), |mut output, alias| { - writeln!(output, "

  • {}
    • ", escape_html(alias.as_ref())) - .expect("should be able to write to string buffer"); - output - }); - let plain = format!("Aliases for {room_id}:\n{plain_list}"); - let html = format!("Aliases for {room_id}:\n
      {html_list}
    "); - Ok(RoomMessageEventContent::text_html(plain, html)) + context.write_str(&plain).await } else { let aliases = services .rooms @@ -181,23 +152,8 @@ pub(super) async fn reprocess( output }); - let html_list = aliases - .iter() - .fold(String::new(), |mut output, (alias, id)| { - writeln!( - output, - "
  • {} -> #{}:{}
    • ", - escape_html(alias.as_ref()), - escape_html(id), - server_name - ) - .expect("should be able to write to string buffer"); - output - }); - let plain = format!("Aliases:\n{plain_list}"); - let html = format!("Aliases:\n
      {html_list}
    "); - Ok(RoomMessageEventContent::text_html(plain, html)) + context.write_str(&plain).await }, } } diff --git a/src/admin/room/commands.rs b/src/admin/room/commands.rs index 6dd31b48..81f36f15 100644 --- a/src/admin/room/commands.rs +++ b/src/admin/room/commands.rs @@ -1,6 +1,6 @@ -use conduwuit::Result; +use conduwuit::{Err, Result}; use futures::StreamExt; -use ruma::{OwnedRoomId, events::room::message::RoomMessageEventContent}; +use ruma::OwnedRoomId; use crate::{PAGE_SIZE, admin_command, get_room_info}; @@ -11,7 +11,7 @@ pub(super) async fn list_rooms( exclude_disabled: bool, exclude_banned: bool, no_details: bool, -) -> Result { +) -> Result { // TODO: i know there's a way to do this with clap, but i can't seem to find it let page = page.unwrap_or(1); let mut rooms = self @@ -41,29 +41,28 @@ pub(super) async fn list_rooms( .collect::>(); if rooms.is_empty() { - return Ok(RoomMessageEventContent::text_plain("No more rooms.")); + return Err!("No more rooms."); } - let output_plain = format!( - "Rooms ({}):\n```\n{}\n```", - rooms.len(), - rooms - .iter() - .map(|(id, members, name)| if no_details { + let body = rooms + .iter() + .map(|(id, members, name)| { + if no_details { format!("{id}") } else { format!("{id}\tMembers: {members}\tName: {name}") - }) - .collect::>() - .join("\n") - ); + } + }) + .collect::>() + .join("\n"); - Ok(RoomMessageEventContent::notice_markdown(output_plain)) + self.write_str(&format!("Rooms ({}):\n```\n{body}\n```", rooms.len(),)) + .await } #[admin_command] -pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result { +pub(super) async fn exists(&self, room_id: OwnedRoomId) -> Result { let result = self.services.rooms.metadata.exists(&room_id).await; - Ok(RoomMessageEventContent::notice_markdown(format!("{result}"))) + self.write_str(&format!("{result}")).await } diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index 179131e4..a6be9a15 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -1,9 +1,9 @@ use clap::Subcommand; -use conduwuit::Result; +use conduwuit::{Err, Result}; use futures::StreamExt; -use ruma::{OwnedRoomId, events::room::message::RoomMessageEventContent}; +use ruma::OwnedRoomId; -use crate::{Command, PAGE_SIZE, get_room_info}; +use crate::{Context, PAGE_SIZE, get_room_info}; #[derive(Debug, Subcommand)] pub(crate) enum RoomDirectoryCommand { @@ -25,25 +25,16 @@ pub(crate) enum RoomDirectoryCommand { }, } -pub(super) async fn process(command: RoomDirectoryCommand, context: &Command<'_>) -> Result { - let c = reprocess(command, context).await?; - context.write_str(c.body()).await?; - Ok(()) -} - -pub(super) async fn reprocess( - command: RoomDirectoryCommand, - context: &Command<'_>, -) -> Result { +pub(super) async fn process(command: RoomDirectoryCommand, context: &Context<'_>) -> Result { let services = context.services; match command { | RoomDirectoryCommand::Publish { room_id } => { services.rooms.directory.set_public(&room_id); - Ok(RoomMessageEventContent::notice_plain("Room published")) + context.write_str("Room published").await }, | RoomDirectoryCommand::Unpublish { room_id } => { services.rooms.directory.set_not_public(&room_id); - Ok(RoomMessageEventContent::notice_plain("Room unpublished")) + context.write_str("Room unpublished").await }, | RoomDirectoryCommand::List { page } => { // TODO: i know there's a way to do this with clap, but i can't seem to find it @@ -66,20 +57,18 @@ pub(super) async fn reprocess( .collect(); if rooms.is_empty() { - return Ok(RoomMessageEventContent::text_plain("No more rooms.")); + return Err!("No more rooms."); } - let output = format!( - "Rooms (page {page}):\n```\n{}\n```", - rooms - .iter() - .map(|(id, members, name)| format!( - "{id} | Members: {members} | Name: {name}" - )) - .collect::>() - .join("\n") - ); - Ok(RoomMessageEventContent::text_markdown(output)) + let body = rooms + .iter() + .map(|(id, members, name)| format!("{id} | Members: {members} | Name: {name}")) + .collect::>() + .join("\n"); + + context + .write_str(&format!("Rooms (page {page}):\n```\n{body}\n```",)) + .await }, } } diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index 35a92b6a..1278e820 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -1,7 +1,7 @@ use clap::Subcommand; -use conduwuit::{Result, utils::ReadyExt}; +use conduwuit::{Err, Result, utils::ReadyExt}; use futures::StreamExt; -use ruma::{OwnedRoomId, events::room::message::RoomMessageEventContent}; +use ruma::OwnedRoomId; use crate::{admin_command, admin_command_dispatch}; @@ -27,11 +27,7 @@ pub(crate) enum RoomInfoCommand { } #[admin_command] -async fn list_joined_members( - &self, - room_id: OwnedRoomId, - local_only: bool, -) -> Result { +async fn list_joined_members(&self, room_id: OwnedRoomId, local_only: bool) -> Result { let room_name = self .services .rooms @@ -64,22 +60,19 @@ async fn list_joined_members( .collect() .await; - let output_plain = format!( - "{} Members in Room \"{}\":\n```\n{}\n```", - member_info.len(), - room_name, - member_info - .into_iter() - .map(|(displayname, mxid)| format!("{mxid} | {displayname}")) - .collect::>() - .join("\n") - ); + let num = member_info.len(); + let body = member_info + .into_iter() + .map(|(displayname, mxid)| format!("{mxid} | {displayname}")) + .collect::>() + .join("\n"); - Ok(RoomMessageEventContent::notice_markdown(output_plain)) + self.write_str(&format!("{num} Members in Room \"{room_name}\":\n```\n{body}\n```",)) + .await } #[admin_command] -async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result { +async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result { let Ok(room_topic) = self .services .rooms @@ -87,10 +80,9 @@ async fn view_room_topic(&self, room_id: OwnedRoomId) -> Result Result { +async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { debug!("Got room alias or ID: {}", room); let admin_room_alias = &self.services.globals.admin_alias; if let Ok(admin_room_id) = self.services.admin.get_admin_room().await { if room.to_string().eq(&admin_room_id) || room.to_string().eq(admin_room_alias) { - return Ok(RoomMessageEventContent::text_plain("Not allowed to ban the admin room.")); + return Err!("Not allowed to ban the admin room."); } } @@ -64,11 +61,11 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result room_id, | Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( + return Err!( "Failed to parse room ID {room}. Please note that this requires a full room \ ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ (`#roomalias:example.com`): {e}" - ))); + ); }, }; @@ -80,11 +77,11 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result room_alias, | Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( + return Err!( "Failed to parse room ID {room}. Please note that this requires a full room \ ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ (`#roomalias:example.com`): {e}" - ))); + ); }, }; @@ -123,9 +120,9 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { - return Ok(RoomMessageEventContent::notice_plain(format!( + return Err!( "Failed to resolve room alias {room_alias} to a room ID: {e}" - ))); + ); }, } }, @@ -135,11 +132,11 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result Result Result { +async fn ban_list_of_rooms(&self) -> Result { if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" { - return Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )); + return Err!("Expected code block in command body. Add --help for details.",); } let rooms_s = self @@ -356,23 +352,24 @@ async fn ban_list_of_rooms(&self) -> Result { self.services.rooms.metadata.disable_room(&room_id, true); } - Ok(RoomMessageEventContent::text_plain(format!( + self.write_str(&format!( "Finished bulk room ban, banned {room_ban_count} total rooms, evicted all users, and \ disabled incoming federation with the room." - ))) + )) + .await } #[admin_command] -async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result { +async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result { let room_id = if room.is_room_id() { let room_id = match RoomId::parse(&room) { | Ok(room_id) => room_id, | Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( + return Err!( "Failed to parse room ID {room}. Please note that this requires a full room \ ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ (`#roomalias:example.com`): {e}" - ))); + ); }, }; @@ -384,11 +381,11 @@ async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result room_alias, | Err(e) => { - return Ok(RoomMessageEventContent::text_plain(format!( + return Err!( "Failed to parse room ID {room}. Please note that this requires a full room \ ID (`!awIh6gGInaS5wLQJwa:example.com`) or a room alias \ (`#roomalias:example.com`): {e}" - ))); + ); }, }; @@ -427,9 +424,7 @@ async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result { - return Ok(RoomMessageEventContent::text_plain(format!( - "Failed to resolve room alias {room} to a room ID: {e}" - ))); + return Err!("Failed to resolve room alias {room} to a room ID: {e}"); }, } }, @@ -439,19 +434,20 @@ async fn unban_room(&self, room: OwnedRoomOrAliasId) -> Result Result { +async fn list_banned_rooms(&self, no_details: bool) -> Result { let room_ids: Vec = self .services .rooms @@ -462,7 +458,7 @@ async fn list_banned_rooms(&self, no_details: bool) -> Result Result>() - .join("\n") - ); + } + }) + .collect::>() + .join("\n"); - Ok(RoomMessageEventContent::notice_markdown(output_plain)) + self.write_str(&format!("Rooms Banned ({num}):\n```\n{body}\n```",)) + .await } diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index 17bf9ec0..b01e9296 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,12 +1,11 @@ use std::{fmt::Write, path::PathBuf, sync::Arc}; use conduwuit::{Err, Result, info, utils::time, warn}; -use ruma::events::room::message::RoomMessageEventContent; use crate::admin_command; #[admin_command] -pub(super) async fn uptime(&self) -> Result { +pub(super) async fn uptime(&self) -> Result { let elapsed = self .services .server @@ -15,47 +14,36 @@ pub(super) async fn uptime(&self) -> Result { .expect("standard duration"); let result = time::pretty(elapsed); - Ok(RoomMessageEventContent::notice_plain(format!("{result}."))) + self.write_str(&format!("{result}.")).await } #[admin_command] -pub(super) async fn show_config(&self) -> Result { - // Construct and send the response - Ok(RoomMessageEventContent::text_markdown(format!( - "{}", - *self.services.server.config - ))) +pub(super) async fn show_config(&self) -> Result { + self.write_str(&format!("{}", *self.services.server.config)) + .await } #[admin_command] -pub(super) async fn reload_config( - &self, - path: Option, -) -> Result { +pub(super) async fn reload_config(&self, path: Option) -> Result { let path = path.as_deref().into_iter(); self.services.config.reload(path)?; - Ok(RoomMessageEventContent::text_plain("Successfully reconfigured.")) + self.write_str("Successfully reconfigured.").await } #[admin_command] -pub(super) async fn list_features( - &self, - available: bool, - enabled: bool, - comma: bool, -) -> Result { +pub(super) async fn list_features(&self, available: bool, enabled: bool, comma: bool) -> Result { let delim = if comma { "," } else { " " }; if enabled && !available { let features = info::rustc::features().join(delim); let out = format!("`\n{features}\n`"); - return Ok(RoomMessageEventContent::text_markdown(out)); + return self.write_str(&out).await; } if available && !enabled { let features = info::cargo::features().join(delim); let out = format!("`\n{features}\n`"); - return Ok(RoomMessageEventContent::text_markdown(out)); + return self.write_str(&out).await; } let mut features = String::new(); @@ -68,41 +56,42 @@ pub(super) async fn list_features( writeln!(features, "{emoji} {feature} {remark}")?; } - Ok(RoomMessageEventContent::text_markdown(features)) + self.write_str(&features).await } #[admin_command] -pub(super) async fn memory_usage(&self) -> Result { +pub(super) async fn memory_usage(&self) -> Result { let services_usage = self.services.memory_usage().await?; let database_usage = self.services.db.db.memory_usage()?; let allocator_usage = conduwuit::alloc::memory_usage().map_or(String::new(), |s| format!("\nAllocator:\n{s}")); - Ok(RoomMessageEventContent::text_plain(format!( + self.write_str(&format!( "Services:\n{services_usage}\nDatabase:\n{database_usage}{allocator_usage}", - ))) + )) + .await } #[admin_command] -pub(super) async fn clear_caches(&self) -> Result { +pub(super) async fn clear_caches(&self) -> Result { self.services.clear_cache().await; - Ok(RoomMessageEventContent::text_plain("Done.")) + self.write_str("Done.").await } #[admin_command] -pub(super) async fn list_backups(&self) -> Result { +pub(super) async fn list_backups(&self) -> Result { let result = self.services.db.db.backup_list()?; if result.is_empty() { - Ok(RoomMessageEventContent::text_plain("No backups found.")) - } else { - Ok(RoomMessageEventContent::text_plain(result)) + return Err!("No backups found."); } + + self.write_str(&result).await } #[admin_command] -pub(super) async fn backup_database(&self) -> Result { +pub(super) async fn backup_database(&self) -> Result { let db = Arc::clone(&self.services.db); let mut result = self .services @@ -118,27 +107,27 @@ pub(super) async fn backup_database(&self) -> Result { result = self.services.db.db.backup_list()?; } - Ok(RoomMessageEventContent::notice_markdown(result)) + self.write_str(&result).await } #[admin_command] -pub(super) async fn admin_notice(&self, message: Vec) -> Result { +pub(super) async fn admin_notice(&self, message: Vec) -> Result { let message = message.join(" "); self.services.admin.send_text(&message).await; - Ok(RoomMessageEventContent::notice_plain("Notice was sent to #admins")) + self.write_str("Notice was sent to #admins").await } #[admin_command] -pub(super) async fn reload_mods(&self) -> Result { +pub(super) async fn reload_mods(&self) -> Result { self.services.server.reload()?; - Ok(RoomMessageEventContent::notice_plain("Reloading server...")) + self.write_str("Reloading server...").await } #[admin_command] #[cfg(unix)] -pub(super) async fn restart(&self, force: bool) -> Result { +pub(super) async fn restart(&self, force: bool) -> Result { use conduwuit::utils::sys::current_exe_deleted; if !force && current_exe_deleted() { @@ -150,13 +139,13 @@ pub(super) async fn restart(&self, force: bool) -> Result Result { +pub(super) async fn shutdown(&self) -> Result { warn!("shutdown command"); self.services.server.shutdown()?; - Ok(RoomMessageEventContent::notice_plain("Shutting down server...")) + self.write_str("Shutting down server...").await } diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 84795f9b..e5e481e5 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room}; use conduwuit::{ - Result, debug, debug_warn, error, info, is_equal_to, + Err, Result, debug, debug_warn, error, info, is_equal_to, matrix::pdu::PduBuilder, utils::{self, ReadyExt}, warn, @@ -14,7 +14,6 @@ use ruma::{ events::{ RoomAccountDataEventType, StateEventType, room::{ - message::RoomMessageEventContent, power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, redaction::RoomRedactionEventContent, }, @@ -31,7 +30,7 @@ const AUTO_GEN_PASSWORD_LENGTH: usize = 25; const BULK_JOIN_REASON: &str = "Bulk force joining this room as initiated by the server admin."; #[admin_command] -pub(super) async fn list_users(&self) -> Result { +pub(super) async fn list_users(&self) -> Result { let users: Vec<_> = self .services .users @@ -44,30 +43,22 @@ pub(super) async fn list_users(&self) -> Result { plain_msg += users.join("\n").as_str(); plain_msg += "\n```"; - self.write_str(plain_msg.as_str()).await?; - - Ok(RoomMessageEventContent::text_plain("")) + self.write_str(&plain_msg).await } #[admin_command] -pub(super) async fn create_user( - &self, - username: String, - password: Option, -) -> Result { +pub(super) async fn create_user(&self, username: String, password: Option) -> Result { // Validate user id let user_id = parse_local_user_id(self.services, &username)?; if let Err(e) = user_id.validate_strict() { if self.services.config.emergency_password.is_none() { - return Ok(RoomMessageEventContent::text_plain(format!( - "Username {user_id} contains disallowed characters or spaces: {e}" - ))); + return Err!("Username {user_id} contains disallowed characters or spaces: {e}"); } } if self.services.users.exists(&user_id).await { - return Ok(RoomMessageEventContent::text_plain(format!("User {user_id} already exists"))); + return Err!("User {user_id} already exists"); } let password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); @@ -89,8 +80,7 @@ pub(super) async fn create_user( .new_user_displayname_suffix .is_empty() { - write!(displayname, " {}", self.services.server.config.new_user_displayname_suffix) - .expect("should be able to write to string buffer"); + write!(displayname, " {}", self.services.server.config.new_user_displayname_suffix)?; } self.services @@ -110,15 +100,17 @@ pub(super) async fn create_user( content: ruma::events::push_rules::PushRulesEventContent { global: ruma::push::Ruleset::server_default(&user_id), }, - }) - .expect("to json value always works"), + })?, ) .await?; if !self.services.server.config.auto_join_rooms.is_empty() { for room in &self.services.server.config.auto_join_rooms { let Ok(room_id) = self.services.rooms.alias.resolve(room).await else { - error!(%user_id, "Failed to resolve room alias to room ID when attempting to auto join {room}, skipping"); + error!( + %user_id, + "Failed to resolve room alias to room ID when attempting to auto join {room}, skipping" + ); continue; }; @@ -154,18 +146,17 @@ pub(super) async fn create_user( info!("Automatically joined room {room} for user {user_id}"); }, | Err(e) => { - self.services - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "Failed to automatically join room {room} for user {user_id}: \ - {e}" - ))) - .await - .ok(); // don't return this error so we don't fail registrations error!( "Failed to automatically join room {room} for user {user_id}: {e}" ); + self.services + .admin + .send_text(&format!( + "Failed to automatically join room {room} for user {user_id}: \ + {e}" + )) + .await; }, } } @@ -192,25 +183,18 @@ pub(super) async fn create_user( debug!("create_user admin command called without an admin room being available"); } - Ok(RoomMessageEventContent::text_plain(format!( - "Created user with user_id: {user_id} and password: `{password}`" - ))) + self.write_str(&format!("Created user with user_id: {user_id} and password: `{password}`")) + .await } #[admin_command] -pub(super) async fn deactivate( - &self, - no_leave_rooms: bool, - user_id: String, -) -> Result { +pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> Result { // Validate user id let user_id = parse_local_user_id(self.services, &user_id)?; // don't deactivate the server service account if user_id == self.services.globals.server_user { - return Ok(RoomMessageEventContent::text_plain( - "Not allowed to deactivate the server service account.", - )); + return Err!("Not allowed to deactivate the server service account.",); } self.services.users.deactivate_account(&user_id).await?; @@ -218,11 +202,8 @@ pub(super) async fn deactivate( if !no_leave_rooms { self.services .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "Making {user_id} leave all rooms after deactivation..." - ))) - .await - .ok(); + .send_text(&format!("Making {user_id} leave all rooms after deactivation...")) + .await; let all_joined_rooms: Vec = self .services @@ -239,24 +220,19 @@ pub(super) async fn deactivate( leave_all_rooms(self.services, &user_id).await; } - Ok(RoomMessageEventContent::text_plain(format!( - "User {user_id} has been deactivated" - ))) + self.write_str(&format!("User {user_id} has been deactivated")) + .await } #[admin_command] -pub(super) async fn reset_password( - &self, - username: String, - password: Option, -) -> Result { +pub(super) async fn reset_password(&self, username: String, password: Option) -> Result { let user_id = parse_local_user_id(self.services, &username)?; if user_id == self.services.globals.server_user { - return Ok(RoomMessageEventContent::text_plain( + return Err!( "Not allowed to set the password for the server account. Please use the emergency \ password config option.", - )); + ); } let new_password = password.unwrap_or_else(|| utils::random_string(AUTO_GEN_PASSWORD_LENGTH)); @@ -266,28 +242,20 @@ pub(super) async fn reset_password( .users .set_password(&user_id, Some(new_password.as_str())) { - | Ok(()) => Ok(RoomMessageEventContent::text_plain(format!( - "Successfully reset the password for user {user_id}: `{new_password}`" - ))), - | Err(e) => Ok(RoomMessageEventContent::text_plain(format!( - "Couldn't reset the password for user {user_id}: {e}" - ))), + | Err(e) => return Err!("Couldn't reset the password for user {user_id}: {e}"), + | Ok(()) => + write!(self, "Successfully reset the password for user {user_id}: `{new_password}`"), } + .await } #[admin_command] -pub(super) async fn deactivate_all( - &self, - no_leave_rooms: bool, - force: bool, -) -> Result { +pub(super) async fn deactivate_all(&self, no_leave_rooms: bool, force: bool) -> Result { if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" { - return Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )); + return Err!("Expected code block in command body. Add --help for details.",); } let usernames = self @@ -301,15 +269,23 @@ pub(super) async fn deactivate_all( for username in usernames { match parse_active_local_user_id(self.services, username).await { + | Err(e) => { + self.services + .admin + .send_text(&format!("{username} is not a valid username, skipping over: {e}")) + .await; + + continue; + }, | Ok(user_id) => { if self.services.users.is_admin(&user_id).await && !force { self.services .admin - .send_message(RoomMessageEventContent::text_plain(format!( + .send_text(&format!( "{username} is an admin and --force is not set, skipping over" - ))) - .await - .ok(); + )) + .await; + admins.push(username); continue; } @@ -318,26 +294,16 @@ pub(super) async fn deactivate_all( if user_id == self.services.globals.server_user { self.services .admin - .send_message(RoomMessageEventContent::text_plain(format!( + .send_text(&format!( "{username} is the server service account, skipping over" - ))) - .await - .ok(); + )) + .await; + continue; } user_ids.push(user_id); }, - | Err(e) => { - self.services - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "{username} is not a valid username, skipping over: {e}" - ))) - .await - .ok(); - continue; - }, } } @@ -345,6 +311,12 @@ pub(super) async fn deactivate_all( for user_id in user_ids { match self.services.users.deactivate_account(&user_id).await { + | Err(e) => { + self.services + .admin + .send_text(&format!("Failed deactivating user: {e}")) + .await; + }, | Ok(()) => { deactivation_count = deactivation_count.saturating_add(1); if !no_leave_rooms { @@ -365,33 +337,24 @@ pub(super) async fn deactivate_all( leave_all_rooms(self.services, &user_id).await; } }, - | Err(e) => { - self.services - .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "Failed deactivating user: {e}" - ))) - .await - .ok(); - }, } } if admins.is_empty() { - Ok(RoomMessageEventContent::text_plain(format!( - "Deactivated {deactivation_count} accounts." - ))) + write!(self, "Deactivated {deactivation_count} accounts.") } else { - Ok(RoomMessageEventContent::text_plain(format!( + write!( + self, "Deactivated {deactivation_count} accounts.\nSkipped admin accounts: {}. Use \ --force to deactivate admin accounts", admins.join(", ") - ))) + ) } + .await } #[admin_command] -pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result { +pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result { // Validate user id let user_id = parse_local_user_id(self.services, &user_id)?; @@ -405,23 +368,20 @@ pub(super) async fn list_joined_rooms(&self, user_id: String) -> Result>() - .join("\n") - ); + let body = rooms + .iter() + .map(|(id, members, name)| format!("{id}\tMembers: {members}\tName: {name}")) + .collect::>() + .join("\n"); - Ok(RoomMessageEventContent::notice_markdown(output_plain)) + self.write_str(&format!("Rooms {user_id} Joined ({}):\n```\n{body}\n```", rooms.len(),)) + .await } #[admin_command] @@ -429,27 +389,23 @@ pub(super) async fn force_join_list_of_local_users( &self, room_id: OwnedRoomOrAliasId, yes_i_want_to_do_this: bool, -) -> Result { +) -> Result { if self.body.len() < 2 || !self.body[0].trim().starts_with("```") || self.body.last().unwrap_or(&"").trim() != "```" { - return Ok(RoomMessageEventContent::text_plain( - "Expected code block in command body. Add --help for details.", - )); + return Err!("Expected code block in command body. Add --help for details.",); } if !yes_i_want_to_do_this { - return Ok(RoomMessageEventContent::notice_markdown( + return Err!( "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force \ bulk join all specified local users.", - )); + ); } let Ok(admin_room) = self.services.admin.get_admin_room().await else { - return Ok(RoomMessageEventContent::notice_markdown( - "There is not an admin room to check for server admins.", - )); + return Err!("There is not an admin room to check for server admins.",); }; let (room_id, servers) = self @@ -466,7 +422,7 @@ pub(super) async fn force_join_list_of_local_users( .server_in_room(self.services.globals.server_name(), &room_id) .await { - return Ok(RoomMessageEventContent::notice_markdown("We are not joined in this room.")); + return Err!("We are not joined in this room."); } let server_admins: Vec<_> = self @@ -486,9 +442,7 @@ pub(super) async fn force_join_list_of_local_users( .ready_any(|user_id| server_admins.contains(&user_id.to_owned())) .await { - return Ok(RoomMessageEventContent::notice_markdown( - "There is not a single server admin in the room.", - )); + return Err!("There is not a single server admin in the room.",); } let usernames = self @@ -506,11 +460,11 @@ pub(super) async fn force_join_list_of_local_users( if user_id == self.services.globals.server_user { self.services .admin - .send_message(RoomMessageEventContent::text_plain(format!( + .send_text(&format!( "{username} is the server service account, skipping over" - ))) - .await - .ok(); + )) + .await; + continue; } @@ -519,11 +473,9 @@ pub(super) async fn force_join_list_of_local_users( | Err(e) => { self.services .admin - .send_message(RoomMessageEventContent::text_plain(format!( - "{username} is not a valid username, skipping over: {e}" - ))) - .await - .ok(); + .send_text(&format!("{username} is not a valid username, skipping over: {e}")) + .await; + continue; }, } @@ -554,10 +506,11 @@ pub(super) async fn force_join_list_of_local_users( } } - Ok(RoomMessageEventContent::notice_markdown(format!( + self.write_str(&format!( "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins \ failed.", - ))) + )) + .await } #[admin_command] @@ -565,18 +518,16 @@ pub(super) async fn force_join_all_local_users( &self, room_id: OwnedRoomOrAliasId, yes_i_want_to_do_this: bool, -) -> Result { +) -> Result { if !yes_i_want_to_do_this { - return Ok(RoomMessageEventContent::notice_markdown( + return Err!( "You must pass the --yes-i-want-to-do-this-flag to ensure you really want to force \ bulk join all local users.", - )); + ); } let Ok(admin_room) = self.services.admin.get_admin_room().await else { - return Ok(RoomMessageEventContent::notice_markdown( - "There is not an admin room to check for server admins.", - )); + return Err!("There is not an admin room to check for server admins.",); }; let (room_id, servers) = self @@ -593,7 +544,7 @@ pub(super) async fn force_join_all_local_users( .server_in_room(self.services.globals.server_name(), &room_id) .await { - return Ok(RoomMessageEventContent::notice_markdown("We are not joined in this room.")); + return Err!("We are not joined in this room."); } let server_admins: Vec<_> = self @@ -613,9 +564,7 @@ pub(super) async fn force_join_all_local_users( .ready_any(|user_id| server_admins.contains(&user_id.to_owned())) .await { - return Ok(RoomMessageEventContent::notice_markdown( - "There is not a single server admin in the room.", - )); + return Err!("There is not a single server admin in the room.",); } let mut failed_joins: usize = 0; @@ -650,10 +599,11 @@ pub(super) async fn force_join_all_local_users( } } - Ok(RoomMessageEventContent::notice_markdown(format!( + self.write_str(&format!( "{successful_joins} local users have been joined to {room_id}. {failed_joins} joins \ failed.", - ))) + )) + .await } #[admin_command] @@ -661,7 +611,7 @@ pub(super) async fn force_join_room( &self, user_id: String, room_id: OwnedRoomOrAliasId, -) -> Result { +) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; let (room_id, servers) = self .services @@ -677,9 +627,8 @@ pub(super) async fn force_join_room( join_room_by_id_helper(self.services, &user_id, &room_id, None, &servers, None, &None) .await?; - Ok(RoomMessageEventContent::notice_markdown(format!( - "{user_id} has been joined to {room_id}.", - ))) + self.write_str(&format!("{user_id} has been joined to {room_id}.",)) + .await } #[admin_command] @@ -687,7 +636,7 @@ pub(super) async fn force_leave_room( &self, user_id: String, room_id: OwnedRoomOrAliasId, -) -> Result { +) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; let room_id = self.services.rooms.alias.resolve(&room_id).await?; @@ -703,24 +652,17 @@ pub(super) async fn force_leave_room( .is_joined(&user_id, &room_id) .await { - return Ok(RoomMessageEventContent::notice_markdown(format!( - "{user_id} is not joined in the room" - ))); + return Err!("{user_id} is not joined in the room"); } leave_room(self.services, &user_id, &room_id, None).await?; - Ok(RoomMessageEventContent::notice_markdown(format!( - "{user_id} has left {room_id}.", - ))) + self.write_str(&format!("{user_id} has left {room_id}.",)) + .await } #[admin_command] -pub(super) async fn force_demote( - &self, - user_id: String, - room_id: OwnedRoomOrAliasId, -) -> Result { +pub(super) async fn force_demote(&self, user_id: String, room_id: OwnedRoomOrAliasId) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; let room_id = self.services.rooms.alias.resolve(&room_id).await?; @@ -731,15 +673,11 @@ pub(super) async fn force_demote( let state_lock = self.services.rooms.state.mutex.lock(&room_id).await; - let room_power_levels = self + let room_power_levels: Option = self .services .rooms .state_accessor - .room_state_get_content::( - &room_id, - &StateEventType::RoomPowerLevels, - "", - ) + .room_state_get_content(&room_id, &StateEventType::RoomPowerLevels, "") .await .ok(); @@ -757,9 +695,7 @@ pub(super) async fn force_demote( .is_ok_and(|event| event.sender == user_id); if !user_can_demote_self { - return Ok(RoomMessageEventContent::notice_markdown( - "User is not allowed to modify their own power levels in the room.", - )); + return Err!("User is not allowed to modify their own power levels in the room.",); } let mut power_levels_content = room_power_levels.unwrap_or_default(); @@ -777,25 +713,25 @@ pub(super) async fn force_demote( ) .await?; - Ok(RoomMessageEventContent::notice_markdown(format!( + self.write_str(&format!( "User {user_id} demoted themselves to the room default power level in {room_id} - \ {event_id}" - ))) + )) + .await } #[admin_command] -pub(super) async fn make_user_admin(&self, user_id: String) -> Result { +pub(super) async fn make_user_admin(&self, user_id: String) -> Result { let user_id = parse_local_user_id(self.services, &user_id)?; - assert!( self.services.globals.user_is_local(&user_id), "Parsed user_id must be a local user" ); + self.services.admin.make_user_admin(&user_id).await?; - Ok(RoomMessageEventContent::notice_markdown(format!( - "{user_id} has been granted admin privileges.", - ))) + self.write_str(&format!("{user_id} has been granted admin privileges.",)) + .await } #[admin_command] @@ -804,7 +740,7 @@ pub(super) async fn put_room_tag( user_id: String, room_id: OwnedRoomId, tag: String, -) -> Result { +) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; let mut tags_event = self @@ -831,9 +767,10 @@ pub(super) async fn put_room_tag( ) .await?; - Ok(RoomMessageEventContent::text_plain(format!( + self.write_str(&format!( "Successfully updated room account data for {user_id} and room {room_id} with tag {tag}" - ))) + )) + .await } #[admin_command] @@ -842,7 +779,7 @@ pub(super) async fn delete_room_tag( user_id: String, room_id: OwnedRoomId, tag: String, -) -> Result { +) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; let mut tags_event = self @@ -866,18 +803,15 @@ pub(super) async fn delete_room_tag( ) .await?; - Ok(RoomMessageEventContent::text_plain(format!( + self.write_str(&format!( "Successfully updated room account data for {user_id} and room {room_id}, deleting room \ tag {tag}" - ))) + )) + .await } #[admin_command] -pub(super) async fn get_room_tags( - &self, - user_id: String, - room_id: OwnedRoomId, -) -> Result { +pub(super) async fn get_room_tags(&self, user_id: String, room_id: OwnedRoomId) -> Result { let user_id = parse_active_local_user_id(self.services, &user_id).await?; let tags_event = self @@ -889,17 +823,12 @@ pub(super) async fn get_room_tags( content: TagEventContent { tags: BTreeMap::new() }, }); - Ok(RoomMessageEventContent::notice_markdown(format!( - "```\n{:#?}\n```", - tags_event.content.tags - ))) + self.write_str(&format!("```\n{:#?}\n```", tags_event.content.tags)) + .await } #[admin_command] -pub(super) async fn redact_event( - &self, - event_id: OwnedEventId, -) -> Result { +pub(super) async fn redact_event(&self, event_id: OwnedEventId) -> Result { let Ok(event) = self .services .rooms @@ -907,20 +836,18 @@ pub(super) async fn redact_event( .get_non_outlier_pdu(&event_id) .await else { - return Ok(RoomMessageEventContent::text_plain("Event does not exist in our database.")); + return Err!("Event does not exist in our database."); }; if event.is_redacted() { - return Ok(RoomMessageEventContent::text_plain("Event is already redacted.")); + return Err!("Event is already redacted."); } let room_id = event.room_id; let sender_user = event.sender; if !self.services.globals.user_is_local(&sender_user) { - return Ok(RoomMessageEventContent::text_plain( - "This command only works on local users.", - )); + return Err!("This command only works on local users."); } let reason = format!( @@ -949,9 +876,8 @@ pub(super) async fn redact_event( .await? }; - let out = format!("Successfully redacted event. Redaction event ID: {redaction_event_id}"); - - self.write_str(out.as_str()).await?; - - Ok(RoomMessageEventContent::text_plain("")) + self.write_str(&format!( + "Successfully redacted event. Redaction event ID: {redaction_event_id}" + )) + .await } diff --git a/src/admin/utils.rs b/src/admin/utils.rs index a2696c50..ea9696b2 100644 --- a/src/admin/utils.rs +++ b/src/admin/utils.rs @@ -1,3 +1,5 @@ +#![allow(dead_code)] + use conduwuit_core::{Err, Result, err}; use ruma::{OwnedRoomId, OwnedUserId, RoomId, UserId}; use service::Services; diff --git a/src/macros/admin.rs b/src/macros/admin.rs index bf1586a0..fe227b43 100644 --- a/src/macros/admin.rs +++ b/src/macros/admin.rs @@ -8,7 +8,7 @@ use crate::{Result, utils::camel_to_snake_string}; pub(super) fn command(mut item: ItemFn, _args: &[Meta]) -> Result { let attr: Attribute = parse_quote! { - #[conduwuit_macros::implement(crate::Command, params = "<'_>")] + #[conduwuit_macros::implement(crate::Context, params = "<'_>")] }; item.attrs.push(attr); @@ -19,15 +19,16 @@ pub(super) fn command_dispatch(item: ItemEnum, _args: &[Meta]) -> Result = item.variants.iter().map(dispatch_arm).try_collect()?; let switch = quote! { + #[allow(clippy::large_stack_frames)] //TODO: fixme pub(super) async fn process( command: #name, - context: &crate::Command<'_> + context: &crate::Context<'_> ) -> Result { use #name::*; #[allow(non_snake_case)] - Ok(match command { + match command { #( #arm )* - }) + } } }; @@ -47,8 +48,7 @@ fn dispatch_arm(v: &Variant) -> Result { let arg = field.clone(); quote! { #name { #( #field ),* } => { - let c = Box::pin(context.#handler(#( #arg ),*)).await?; - Box::pin(context.write_str(c.body())).await?; + Box::pin(context.#handler(#( #arg ),*)).await }, } }, @@ -58,15 +58,14 @@ fn dispatch_arm(v: &Variant) -> Result { }; quote! { #name ( #field ) => { - Box::pin(#handler::process(#field, context)).await?; + Box::pin(#handler::process(#field, context)).await } } }, | Fields::Unit => { quote! { #name => { - let c = Box::pin(context.#handler()).await?; - Box::pin(context.write_str(c.body())).await?; + Box::pin(context.#handler()).await }, } }, From 3c5bbd4f0505bb1faf6cc5985f0f43fc76cd94b6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Thu, 10 Apr 2025 20:55:41 +0000 Subject: [PATCH 1707/2091] simplify database backup interface related Signed-off-by: Jason Volk --- src/admin/server/commands.rs | 35 ++++++++------- src/database/engine/backup.rs | 80 ++++++++++++++++++++++------------- src/service/globals/data.rs | 6 --- 3 files changed, 69 insertions(+), 52 deletions(-) diff --git a/src/admin/server/commands.rs b/src/admin/server/commands.rs index b01e9296..6027a9eb 100644 --- a/src/admin/server/commands.rs +++ b/src/admin/server/commands.rs @@ -1,6 +1,11 @@ use std::{fmt::Write, path::PathBuf, sync::Arc}; -use conduwuit::{Err, Result, info, utils::time, warn}; +use conduwuit::{ + Err, Result, info, + utils::{stream::IterStream, time}, + warn, +}; +use futures::TryStreamExt; use crate::admin_command; @@ -81,33 +86,31 @@ pub(super) async fn clear_caches(&self) -> Result { #[admin_command] pub(super) async fn list_backups(&self) -> Result { - let result = self.services.db.db.backup_list()?; - - if result.is_empty() { - return Err!("No backups found."); - } - - self.write_str(&result).await + self.services + .db + .db + .backup_list()? + .try_stream() + .try_for_each(|result| write!(self, "{result}")) + .await } #[admin_command] pub(super) async fn backup_database(&self) -> Result { let db = Arc::clone(&self.services.db); - let mut result = self + let result = self .services .server .runtime() .spawn_blocking(move || match db.db.backup() { - | Ok(()) => String::new(), - | Err(e) => e.to_string(), + | Ok(()) => "Done".to_owned(), + | Err(e) => format!("Failed: {e}"), }) .await?; - if result.is_empty() { - result = self.services.db.db.backup_list()?; - } - - self.write_str(&result).await + let count = self.services.db.db.backup_count()?; + self.write_str(&format!("{result}. Currently have {count} backups.")) + .await } #[admin_command] diff --git a/src/database/engine/backup.rs b/src/database/engine/backup.rs index bb110630..ac72e6d4 100644 --- a/src/database/engine/backup.rs +++ b/src/database/engine/backup.rs @@ -1,24 +1,16 @@ -use std::fmt::Write; +use std::{ffi::OsString, path::PathBuf}; -use conduwuit::{Result, error, implement, info, utils::time::rfc2822_from_seconds, warn}; +use conduwuit::{Err, Result, error, implement, info, utils::time::rfc2822_from_seconds, warn}; use rocksdb::backup::{BackupEngine, BackupEngineOptions}; use super::Engine; -use crate::{or_else, util::map_err}; +use crate::util::map_err; #[implement(Engine)] #[tracing::instrument(skip(self))] pub fn backup(&self) -> Result { - let server = &self.ctx.server; - let config = &server.config; - let path = config.database_backup_path.as_ref(); - if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { - return Ok(()); - } - - let options = - BackupEngineOptions::new(path.expect("valid database backup path")).map_err(map_err)?; - let mut engine = BackupEngine::open(&options, &*self.ctx.env.lock()?).map_err(map_err)?; + let mut engine = self.backup_engine()?; + let config = &self.ctx.server.config; if config.database_backups_to_keep > 0 { let flush = !self.is_read_only(); engine @@ -40,34 +32,62 @@ pub fn backup(&self) -> Result { } } + if config.database_backups_to_keep == 0 { + warn!("Configuration item `database_backups_to_keep` is set to 0."); + } + Ok(()) } #[implement(Engine)] -pub fn backup_list(&self) -> Result { - let server = &self.ctx.server; - let config = &server.config; - let path = config.database_backup_path.as_ref(); - if path.is_none() || path.is_some_and(|path| path.as_os_str().is_empty()) { - return Ok("Configure database_backup_path to enable backups, or the path specified is \ - not valid" - .to_owned()); +pub fn backup_list(&self) -> Result + Send> { + let info = self.backup_engine()?.get_backup_info(); + + if info.is_empty() { + return Err!("No backups found."); } - let mut res = String::new(); - let options = - BackupEngineOptions::new(path.expect("valid database backup path")).or_else(or_else)?; - let engine = BackupEngine::open(&options, &*self.ctx.env.lock()?).or_else(or_else)?; - for info in engine.get_backup_info() { - writeln!( - res, + let list = info.into_iter().map(|info| { + format!( "#{} {}: {} bytes, {} files", info.backup_id, rfc2822_from_seconds(info.timestamp), info.size, info.num_files, - )?; + ) + }); + + Ok(list) +} + +#[implement(Engine)] +pub fn backup_count(&self) -> Result { + let info = self.backup_engine()?.get_backup_info(); + + Ok(info.len()) +} + +#[implement(Engine)] +fn backup_engine(&self) -> Result { + let path = self.backup_path()?; + let options = BackupEngineOptions::new(path).map_err(map_err)?; + BackupEngine::open(&options, &*self.ctx.env.lock()?).map_err(map_err) +} + +#[implement(Engine)] +fn backup_path(&self) -> Result { + let path = self + .ctx + .server + .config + .database_backup_path + .clone() + .map(PathBuf::into_os_string) + .unwrap_or_default(); + + if path.is_empty() { + return Err!(Config("database_backup_path", "Configure path to enable backups")); } - Ok(res) + Ok(path) } diff --git a/src/service/globals/data.rs b/src/service/globals/data.rs index b43b7c5f..21c09252 100644 --- a/src/service/globals/data.rs +++ b/src/service/globals/data.rs @@ -72,10 +72,4 @@ impl Data { pub fn bump_database_version(&self, new_version: u64) { self.global.raw_put(b"version", new_version); } - - #[inline] - pub fn backup(&self) -> Result { self.db.db.backup() } - - #[inline] - pub fn backup_list(&self) -> Result { self.db.db.backup_list() } } From 21ec2551598247dc5f081aae73748861085d0ce0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 11 Apr 2025 01:29:26 +0000 Subject: [PATCH 1708/2091] eliminate Arc impl for trait Event Signed-off-by: Jason Volk --- src/core/matrix/event.rs | 29 ------------------- src/core/matrix/state_res/benches.rs | 33 ++++++++++------------ src/core/matrix/state_res/event_auth.rs | 14 ++++------ src/core/matrix/state_res/mod.rs | 11 +++----- src/core/matrix/state_res/test_utils.rs | 37 ++++++++++++------------- 5 files changed, 42 insertions(+), 82 deletions(-) diff --git a/src/core/matrix/event.rs b/src/core/matrix/event.rs index ac9e29d6..29153334 100644 --- a/src/core/matrix/event.rs +++ b/src/core/matrix/event.rs @@ -2,7 +2,6 @@ use std::{ borrow::Borrow, fmt::{Debug, Display}, hash::Hash, - sync::Arc, }; use ruma::{EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId, events::TimelineEventType}; @@ -72,31 +71,3 @@ impl Event for &T { fn redacts(&self) -> Option<&Self::Id> { (*self).redacts() } } - -impl Event for Arc { - type Id = T::Id; - - fn event_id(&self) -> &Self::Id { (**self).event_id() } - - fn room_id(&self) -> &RoomId { (**self).room_id() } - - fn sender(&self) -> &UserId { (**self).sender() } - - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { (**self).origin_server_ts() } - - fn event_type(&self) -> &TimelineEventType { (**self).event_type() } - - fn content(&self) -> &RawJsonValue { (**self).content() } - - fn state_key(&self) -> Option<&str> { (**self).state_key() } - - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { - (**self).prev_events() - } - - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { - (**self).auth_events() - } - - fn redacts(&self) -> Option<&Self::Id> { (**self).redacts() } -} diff --git a/src/core/matrix/state_res/benches.rs b/src/core/matrix/state_res/benches.rs index 7a1ae5bf..01218b01 100644 --- a/src/core/matrix/state_res/benches.rs +++ b/src/core/matrix/state_res/benches.rs @@ -4,10 +4,7 @@ extern crate test; use std::{ borrow::Borrow, collections::{HashMap, HashSet}, - sync::{ - Arc, - atomic::{AtomicU64, Ordering::SeqCst}, - }, + sync::atomic::{AtomicU64, Ordering::SeqCst}, }; use futures::{future, future::ready}; @@ -64,7 +61,7 @@ fn resolution_shallow_auth_chain(c: &mut test::Bencher) { c.iter(|| async { let ev_map = store.0.clone(); let state_sets = [&state_at_bob, &state_at_charlie]; - let fetch = |id: OwnedEventId| ready(ev_map.get(&id).map(Arc::clone)); + let fetch = |id: OwnedEventId| ready(ev_map.get(&id).clone()); let exists = |id: OwnedEventId| ready(ev_map.get(&id).is_some()); let auth_chain_sets: Vec> = state_sets .iter() @@ -148,7 +145,7 @@ fn resolve_deeper_event_set(c: &mut test::Bencher) { }) .collect(); - let fetch = |id: OwnedEventId| ready(inner.get(&id).map(Arc::clone)); + let fetch = |id: OwnedEventId| ready(inner.get(&id).clone()); let exists = |id: OwnedEventId| ready(inner.get(&id).is_some()); let _ = match state_res::resolve( &RoomVersionId::V6, @@ -171,20 +168,20 @@ fn resolve_deeper_event_set(c: &mut test::Bencher) { // IMPLEMENTATION DETAILS AHEAD // /////////////////////////////////////////////////////////////////////*/ -struct TestStore(HashMap>); +struct TestStore(HashMap); #[allow(unused)] -impl TestStore { - fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> Result> { +impl TestStore { + fn get_event(&self, room_id: &RoomId, event_id: &EventId) -> Result { self.0 .get(event_id) - .map(Arc::clone) + .cloned() .ok_or_else(|| Error::NotFound(format!("{} not found", event_id))) } /// Returns the events that correspond to the `event_ids` sorted in the same /// order. - fn get_events(&self, room_id: &RoomId, event_ids: &[OwnedEventId]) -> Result>> { + fn get_events(&self, room_id: &RoomId, event_ids: &[OwnedEventId]) -> Result> { let mut events = vec![]; for id in event_ids { events.push(self.get_event(room_id, id)?); @@ -264,7 +261,7 @@ impl TestStore { &[], ); let cre = create_event.event_id().to_owned(); - self.0.insert(cre.clone(), Arc::clone(&create_event)); + self.0.insert(cre.clone(), create_event.clone()); let alice_mem = to_pdu_event( "IMA", @@ -276,7 +273,7 @@ impl TestStore { &[cre.clone()], ); self.0 - .insert(alice_mem.event_id().to_owned(), Arc::clone(&alice_mem)); + .insert(alice_mem.event_id().to_owned(), alice_mem.clone()); let join_rules = to_pdu_event( "IJR", @@ -383,7 +380,7 @@ fn to_pdu_event( content: Box, auth_events: &[S], prev_events: &[S], -) -> Arc +) -> PduEvent where S: AsRef, { @@ -407,7 +404,7 @@ where .collect::>(); let state_key = state_key.map(ToOwned::to_owned); - Arc::new(PduEvent { + PduEvent { event_id: id.try_into().unwrap(), rest: Pdu::RoomV3Pdu(RoomV3Pdu { room_id: room_id().to_owned(), @@ -424,12 +421,12 @@ where hashes: EventHash::new(String::new()), signatures: Signatures::new(), }), - }) + } } // all graphs start with these input events #[allow(non_snake_case)] -fn INITIAL_EVENTS() -> HashMap> { +fn INITIAL_EVENTS() -> HashMap { vec![ to_pdu_event::<&EventId>( "CREATE", @@ -511,7 +508,7 @@ fn INITIAL_EVENTS() -> HashMap> { // all graphs start with these input events #[allow(non_snake_case)] -fn BAN_STATE_SET() -> HashMap> { +fn BAN_STATE_SET() -> HashMap { vec![ to_pdu_event( "PA", diff --git a/src/core/matrix/state_res/event_auth.rs b/src/core/matrix/state_res/event_auth.rs index 65bec802..8c9339ec 100644 --- a/src/core/matrix/state_res/event_auth.rs +++ b/src/core/matrix/state_res/event_auth.rs @@ -1112,8 +1112,6 @@ fn verify_third_party_invite( #[cfg(test)] mod tests { - use std::sync::Arc; - use ruma::events::{ StateEventType, TimelineEventType, room::{ @@ -1143,7 +1141,7 @@ mod tests { let auth_events = events .values() - .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.clone())) .collect::>(); let requester = to_pdu_event( @@ -1188,7 +1186,7 @@ mod tests { let auth_events = events .values() - .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.clone())) .collect::>(); let requester = to_pdu_event( @@ -1233,7 +1231,7 @@ mod tests { let auth_events = events .values() - .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.clone())) .collect::>(); let requester = to_pdu_event( @@ -1278,7 +1276,7 @@ mod tests { let auth_events = events .values() - .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.clone())) .collect::>(); let requester = to_pdu_event( @@ -1340,7 +1338,7 @@ mod tests { let auth_events = events .values() - .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.clone())) .collect::>(); let requester = to_pdu_event( @@ -1412,7 +1410,7 @@ mod tests { let auth_events = events .values() - .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), Arc::clone(ev))) + .map(|ev| (ev.event_type().with_state_key(ev.state_key().unwrap()), ev.clone())) .collect::>(); let requester = to_pdu_event( diff --git a/src/core/matrix/state_res/mod.rs b/src/core/matrix/state_res/mod.rs index ce6b7e89..2ab7cb64 100644 --- a/src/core/matrix/state_res/mod.rs +++ b/src/core/matrix/state_res/mod.rs @@ -861,10 +861,7 @@ where #[cfg(test)] mod tests { - use std::{ - collections::{HashMap, HashSet}, - sync::Arc, - }; + use std::collections::{HashMap, HashSet}; use maplit::{hashmap, hashset}; use rand::seq::SliceRandom; @@ -906,7 +903,7 @@ mod tests { let power_events = event_map .values() - .filter(|&pdu| is_power_event(&**pdu)) + .filter(|&pdu| is_power_event(&*pdu)) .map(|pdu| pdu.event_id.clone()) .collect::>(); @@ -1489,7 +1486,7 @@ mod tests { } #[allow(non_snake_case)] - fn BAN_STATE_SET() -> HashMap> { + fn BAN_STATE_SET() -> HashMap { vec![ to_pdu_event( "PA", @@ -1534,7 +1531,7 @@ mod tests { } #[allow(non_snake_case)] - fn JOIN_RULE() -> HashMap> { + fn JOIN_RULE() -> HashMap { vec![ to_pdu_event( "JR", diff --git a/src/core/matrix/state_res/test_utils.rs b/src/core/matrix/state_res/test_utils.rs index f2ee4238..a666748a 100644 --- a/src/core/matrix/state_res/test_utils.rs +++ b/src/core/matrix/state_res/test_utils.rs @@ -1,10 +1,7 @@ use std::{ borrow::Borrow, collections::{BTreeMap, HashMap, HashSet}, - sync::{ - Arc, - atomic::{AtomicU64, Ordering::SeqCst}, - }, + sync::atomic::{AtomicU64, Ordering::SeqCst}, }; use futures::future::ready; @@ -36,7 +33,7 @@ use crate::{ static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); pub(crate) async fn do_check( - events: &[Arc], + events: &[PduEvent], edges: Vec>, expected_state_ids: Vec, ) { @@ -85,7 +82,7 @@ pub(crate) async fn do_check( } // event_id -> PduEvent - let mut event_map: HashMap> = HashMap::new(); + let mut event_map: HashMap = HashMap::new(); // event_id -> StateMap let mut state_at_event: HashMap> = HashMap::new(); @@ -194,7 +191,7 @@ pub(crate) async fn do_check( store.0.insert(ev_id.to_owned(), event.clone()); state_at_event.insert(node, state_after); - event_map.insert(event_id.to_owned(), Arc::clone(store.0.get(ev_id).unwrap())); + event_map.insert(event_id.to_owned(), store.0.get(ev_id).unwrap().clone()); } let mut expected_state = StateMap::new(); @@ -235,10 +232,10 @@ pub(crate) async fn do_check( } #[allow(clippy::exhaustive_structs)] -pub(crate) struct TestStore(pub(crate) HashMap>); +pub(crate) struct TestStore(pub(crate) HashMap); -impl TestStore { - pub(crate) fn get_event(&self, _: &RoomId, event_id: &EventId) -> Result> { +impl TestStore { + pub(crate) fn get_event(&self, _: &RoomId, event_id: &EventId) -> Result { self.0 .get(event_id) .cloned() @@ -288,7 +285,7 @@ impl TestStore { &[], ); let cre = create_event.event_id().to_owned(); - self.0.insert(cre.clone(), Arc::clone(&create_event)); + self.0.insert(cre.clone(), create_event.clone()); let alice_mem = to_pdu_event( "IMA", @@ -300,7 +297,7 @@ impl TestStore { &[cre.clone()], ); self.0 - .insert(alice_mem.event_id().to_owned(), Arc::clone(&alice_mem)); + .insert(alice_mem.event_id().to_owned(), alice_mem.clone()); let join_rules = to_pdu_event( "IJR", @@ -399,7 +396,7 @@ pub(crate) fn to_init_pdu_event( ev_type: TimelineEventType, state_key: Option<&str>, content: Box, -) -> Arc { +) -> PduEvent { let ts = SERVER_TIMESTAMP.fetch_add(1, SeqCst); let id = if id.contains('$') { id.to_owned() @@ -408,7 +405,7 @@ pub(crate) fn to_init_pdu_event( }; let state_key = state_key.map(ToOwned::to_owned); - Arc::new(PduEvent { + PduEvent { event_id: id.try_into().unwrap(), rest: Pdu::RoomV3Pdu(RoomV3Pdu { room_id: room_id().to_owned(), @@ -425,7 +422,7 @@ pub(crate) fn to_init_pdu_event( hashes: EventHash::new("".to_owned()), signatures: ServerSignatures::default(), }), - }) + } } pub(crate) fn to_pdu_event( @@ -436,7 +433,7 @@ pub(crate) fn to_pdu_event( content: Box, auth_events: &[S], prev_events: &[S], -) -> Arc +) -> PduEvent where S: AsRef, { @@ -458,7 +455,7 @@ where .collect::>(); let state_key = state_key.map(ToOwned::to_owned); - Arc::new(PduEvent { + PduEvent { event_id: id.try_into().unwrap(), rest: Pdu::RoomV3Pdu(RoomV3Pdu { room_id: room_id().to_owned(), @@ -475,12 +472,12 @@ where hashes: EventHash::new("".to_owned()), signatures: ServerSignatures::default(), }), - }) + } } // all graphs start with these input events #[allow(non_snake_case)] -pub(crate) fn INITIAL_EVENTS() -> HashMap> { +pub(crate) fn INITIAL_EVENTS() -> HashMap { vec![ to_pdu_event::<&EventId>( "CREATE", @@ -562,7 +559,7 @@ pub(crate) fn INITIAL_EVENTS() -> HashMap> { // all graphs start with these input events #[allow(non_snake_case)] -pub(crate) fn INITIAL_EVENTS_CREATE_ROOM() -> HashMap> { +pub(crate) fn INITIAL_EVENTS_CREATE_ROOM() -> HashMap { vec![to_pdu_event::<&EventId>( "CREATE", alice(), From 576a783a6f98bde5c04171f881c8a18e70222ac3 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 1 Apr 2025 08:39:41 +0000 Subject: [PATCH 1709/2091] add missing feature-projections between intra-workspace crates Signed-off-by: Jason Volk --- Cargo.lock | 1 - src/admin/Cargo.toml | 49 +++++++++++++++++++++++++++ src/api/Cargo.toml | 50 +++++++++++++++++++++------- src/core/Cargo.toml | 38 ++++++++++----------- src/core/info/cargo.rs | 2 +- src/database/Cargo.toml | 30 ++++++++++++----- src/main/Cargo.toml | 2 ++ src/router/Cargo.toml | 73 +++++++++++++++++++++++++++++++++-------- src/service/Cargo.toml | 34 +++++++++++++++++-- 9 files changed, 221 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index def41f68..00aeca81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -784,7 +784,6 @@ dependencies = [ "base64 0.22.1", "bytes", "conduwuit_core", - "conduwuit_database", "conduwuit_service", "const-str", "futures", diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index ca865969..7896ef97 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -17,12 +17,61 @@ crate-type = [ ] [features] +brotli_compression = [ + "conduwuit-api/brotli_compression", + "conduwuit-core/brotli_compression", + "conduwuit-service/brotli_compression", +] +gzip_compression = [ + "conduwuit-api/gzip_compression", + "conduwuit-core/gzip_compression", + "conduwuit-service/gzip_compression", +] +io_uring = [ + "conduwuit-api/io_uring", + "conduwuit-database/io_uring", + "conduwuit-service/io_uring", +] +jemalloc = [ + "conduwuit-api/jemalloc", + "conduwuit-core/jemalloc", + "conduwuit-database/jemalloc", + "conduwuit-service/jemalloc", +] +jemalloc_conf = [ + "conduwuit-api/jemalloc_conf", + "conduwuit-core/jemalloc_conf", + "conduwuit-database/jemalloc_conf", + "conduwuit-service/jemalloc_conf", +] +jemalloc_prof = [ + "conduwuit-api/jemalloc_prof", + "conduwuit-core/jemalloc_prof", + "conduwuit-database/jemalloc_prof", + "conduwuit-service/jemalloc_prof", +] +jemalloc_stats = [ + "conduwuit-api/jemalloc_stats", + "conduwuit-core/jemalloc_stats", + "conduwuit-database/jemalloc_stats", + "conduwuit-service/jemalloc_stats", +] release_max_log_level = [ + "conduwuit-api/release_max_log_level", + "conduwuit-core/release_max_log_level", + "conduwuit-database/release_max_log_level", + "conduwuit-service/release_max_log_level", "tracing/max_level_trace", "tracing/release_max_level_info", "log/max_level_trace", "log/release_max_level_info", ] +zstd_compression = [ + "conduwuit-api/zstd_compression", + "conduwuit-core/zstd_compression", + "conduwuit-database/zstd_compression", + "conduwuit-service/zstd_compression", +] [dependencies] clap.workspace = true diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 7890561c..15ada812 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -17,21 +17,50 @@ crate-type = [ ] [features] -element_hacks = [] -release_max_log_level = [ - "tracing/max_level_trace", - "tracing/release_max_level_info", - "log/max_level_trace", - "log/release_max_level_info", +brotli_compression = [ + "conduwuit-core/brotli_compression", + "conduwuit-service/brotli_compression", + "reqwest/brotli", ] -zstd_compression = [ - "reqwest/zstd", +element_hacks = [ + "conduwuit-service/element_hacks", ] gzip_compression = [ + "conduwuit-core/gzip_compression", + "conduwuit-service/gzip_compression", "reqwest/gzip", ] -brotli_compression = [ - "reqwest/brotli", +io_uring = [ + "conduwuit-service/io_uring", +] +jemalloc = [ + "conduwuit-core/jemalloc", + "conduwuit-service/jemalloc", +] +jemalloc_conf = [ + "conduwuit-core/jemalloc_conf", + "conduwuit-service/jemalloc_conf", +] +jemalloc_prof = [ + "conduwuit-core/jemalloc_prof", + "conduwuit-service/jemalloc_prof", +] +jemalloc_stats = [ + "conduwuit-core/jemalloc_stats", + "conduwuit-service/jemalloc_stats", +] +release_max_log_level = [ + "conduwuit-core/release_max_log_level", + "conduwuit-service/release_max_log_level", + "log/max_level_trace", + "log/release_max_level_info", + "tracing/max_level_trace", + "tracing/release_max_level_info", +] +zstd_compression = [ + "conduwuit-core/zstd_compression", + "conduwuit-service/zstd_compression", + "reqwest/zstd", ] [dependencies] @@ -42,7 +71,6 @@ axum.workspace = true base64.workspace = true bytes.workspace = true conduwuit-core.workspace = true -conduwuit-database.workspace = true conduwuit-service.workspace = true const-str.workspace = true futures.workspace = true diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 4848e742..f42b049b 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -17,17 +17,24 @@ crate-type = [ ] [features] -release_max_log_level = [ - "tracing/max_level_trace", - "tracing/release_max_level_info", - "log/max_level_trace", - "log/release_max_level_info", +brotli_compression = [ + "reqwest/brotli", +] +conduwuit_mods = [ + "dep:libloading" +] +gzip_compression = [ + "reqwest/gzip", +] +hardened_malloc = [ + "dep:hardened_malloc-rs" ] jemalloc = [ "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl", "dep:tikv-jemallocator", ] +jemalloc_conf = [] jemalloc_prof = [ "tikv-jemalloc-sys/profiling", ] @@ -36,24 +43,17 @@ jemalloc_stats = [ "tikv-jemalloc-ctl/stats", "tikv-jemallocator/stats", ] -jemalloc_conf = [] -hardened_malloc = [ - "dep:hardened_malloc-rs" -] -gzip_compression = [ - "reqwest/gzip", -] -brotli_compression = [ - "reqwest/brotli", +perf_measurements = [] +release_max_log_level = [ + "tracing/max_level_trace", + "tracing/release_max_level_info", + "log/max_level_trace", + "log/release_max_level_info", ] +sentry_telemetry = [] zstd_compression = [ "reqwest/zstd", ] -perf_measurements = [] -sentry_telemetry = [] -conduwuit_mods = [ - "dep:libloading" -] [dependencies] argon2.workspace = true diff --git a/src/core/info/cargo.rs b/src/core/info/cargo.rs index c5a1d167..28c6590e 100644 --- a/src/core/info/cargo.rs +++ b/src/core/info/cargo.rs @@ -31,7 +31,7 @@ const ROUTER_MANIFEST: &'static str = (); #[cargo_manifest(crate = "main")] const MAIN_MANIFEST: &'static str = (); -/// Processed list of features access all project crates. This is generated from +/// Processed list of features across all project crates. This is generated from /// the data in the MANIFEST strings and contains all possible project features. /// For *enabled* features see the info::rustc module instead. static FEATURES: OnceLock> = OnceLock::new(); diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 067c6f5f..55d4793f 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -17,19 +17,31 @@ crate-type = [ ] [features] -release_max_log_level = [ - "tracing/max_level_trace", - "tracing/release_max_level_info", - "log/max_level_trace", - "log/release_max_level_info", -] -jemalloc = [ - "rust-rocksdb/jemalloc", -] io_uring = [ "rust-rocksdb/io-uring", ] +jemalloc = [ + "conduwuit-core/jemalloc", + "rust-rocksdb/jemalloc", +] +jemalloc_conf = [ + "conduwuit-core/jemalloc_conf", +] +jemalloc_prof = [ + "conduwuit-core/jemalloc_prof", +] +jemalloc_stats = [ + "conduwuit-core/jemalloc_stats", +] +release_max_log_level = [ + "conduwuit-core/release_max_log_level", + "log/max_level_trace", + "log/release_max_level_info", + "tracing/max_level_trace", + "tracing/release_max_level_info", +] zstd_compression = [ + "conduwuit-core/zstd_compression", "rust-rocksdb/zstd", ] diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 87ca48c8..e2fed5d5 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -70,6 +70,7 @@ element_hacks = [ ] gzip_compression = [ "conduwuit-api/gzip_compression", + "conduwuit-core/gzip_compression", "conduwuit-router/gzip_compression", "conduwuit-service/gzip_compression", ] @@ -141,6 +142,7 @@ zstd_compression = [ "conduwuit-core/zstd_compression", "conduwuit-database/zstd_compression", "conduwuit-router/zstd_compression", + "conduwuit-service/zstd_compression", ] conduwuit_mods = [ "conduwuit-core/conduwuit_mods", diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 51e15aed..31a44983 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -17,34 +17,79 @@ crate-type = [ ] [features] +brotli_compression = [ + "conduwuit-admin/brotli_compression", + "conduwuit-api/brotli_compression", + "conduwuit-core/brotli_compression", + "conduwuit-service/brotli_compression", + "tower-http/compression-br", +] +direct_tls = [ + "axum-server/tls-rustls", + "dep:rustls", + "dep:axum-server-dual-protocol", +] +gzip_compression = [ + "conduwuit-admin/gzip_compression", + "conduwuit-api/gzip_compression", + "conduwuit-core/gzip_compression", + "conduwuit-service/gzip_compression", + "tower-http/compression-gzip", +] +io_uring = [ + "conduwuit-admin/io_uring", + "conduwuit-api/io_uring", + "conduwuit-service/io_uring", + "conduwuit-api/io_uring", +] +jemalloc = [ + "conduwuit-admin/jemalloc", + "conduwuit-api/jemalloc", + "conduwuit-core/jemalloc", + "conduwuit-service/jemalloc", +] +jemalloc_conf = [ + "conduwuit-admin/jemalloc_conf", + "conduwuit-api/jemalloc_conf", + "conduwuit-core/jemalloc_conf", + "conduwuit-service/jemalloc_conf", +] +jemalloc_prof = [ + "conduwuit-admin/jemalloc_prof", + "conduwuit-api/jemalloc_prof", + "conduwuit-core/jemalloc_prof", + "conduwuit-service/jemalloc_prof", +] +jemalloc_stats = [ + "conduwuit-admin/jemalloc_stats", + "conduwuit-api/jemalloc_stats", + "conduwuit-core/jemalloc_stats", + "conduwuit-service/jemalloc_stats", +] release_max_log_level = [ + "conduwuit-admin/release_max_log_level", + "conduwuit-api/release_max_log_level", + "conduwuit-core/release_max_log_level", + "conduwuit-service/release_max_log_level", "tracing/max_level_trace", "tracing/release_max_level_info", "log/max_level_trace", "log/release_max_level_info", ] sentry_telemetry = [ + "conduwuit-core/sentry_telemetry", "dep:sentry", "dep:sentry-tracing", "dep:sentry-tower", ] -zstd_compression = [ - "tower-http/compression-zstd", -] -gzip_compression = [ - "tower-http/compression-gzip", -] -brotli_compression = [ - "tower-http/compression-br", -] systemd = [ "dep:sd-notify", ] - -direct_tls = [ - "axum-server/tls-rustls", - "dep:rustls", - "dep:axum-server-dual-protocol", +zstd_compression = [ + "conduwuit-api/zstd_compression", + "conduwuit-core/zstd_compression", + "conduwuit-service/zstd_compression", + "tower-http/compression-zstd", ] [dependencies] diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index caeea318..8b0d1405 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -17,7 +17,12 @@ crate-type = [ ] [features] +blurhashing = [ + "dep:image", + "dep:blurhash", +] brotli_compression = [ + "conduwuit-core/brotli_compression", "reqwest/brotli", ] console = [ @@ -26,25 +31,48 @@ console = [ ] element_hacks = [] gzip_compression = [ + "conduwuit-core/gzip_compression", "reqwest/gzip", ] +io_uring = [ + "conduwuit-database/io_uring", +] +jemalloc = [ + "conduwuit-core/jemalloc", + "conduwuit-database/jemalloc", +] +jemalloc_conf = [ + "conduwuit-core/jemalloc_conf", + "conduwuit-database/jemalloc_conf", +] +jemalloc_prof = [ + "conduwuit-core/jemalloc_prof", + "conduwuit-database/jemalloc_prof", +] +jemalloc_stats = [ + "conduwuit-core/jemalloc_stats", + "conduwuit-database/jemalloc_stats", +] media_thumbnail = [ "dep:image", ] release_max_log_level = [ - "tracing/max_level_trace", - "tracing/release_max_level_info", + "conduwuit-core/release_max_log_level", + "conduwuit-database/release_max_log_level", "log/max_level_trace", "log/release_max_level_info", + "tracing/max_level_trace", + "tracing/release_max_level_info", ] url_preview = [ "dep:image", "dep:webpage", ] zstd_compression = [ + "conduwuit-core/zstd_compression", + "conduwuit-database/zstd_compression", "reqwest/zstd", ] -blurhashing = ["dep:image","dep:blurhash"] [dependencies] async-trait.workspace = true From 8e7373c02790a4e48e29346f678a0181de6c42f6 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Fri, 4 Apr 2025 23:04:13 +0000 Subject: [PATCH 1710/2091] mitigate additional debuginfo expansions Signed-off-by: Jason Volk --- src/core/debug.rs | 1 + src/core/error/err.rs | 3 +++ src/core/log/mod.rs | 1 + src/core/utils/math.rs | 4 ++++ src/core/utils/mod.rs | 4 ---- src/core/utils/string.rs | 2 ++ src/core/utils/sys/storage.rs | 2 +- src/service/mod.rs | 2 +- 8 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/core/debug.rs b/src/core/debug.rs index b9a53038..21a5ada4 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -12,6 +12,7 @@ pub use crate::{result::DebugInspect, utils::debug::*}; /// Log event at given level in debug-mode (when debug-assertions are enabled). /// In release-mode it becomes DEBUG level, and possibly subject to elision. #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! debug_event { ( $level:expr_2021, $($x:tt)+ ) => { if $crate::debug::logging() { diff --git a/src/core/error/err.rs b/src/core/error/err.rs index 9c24d3b4..2eb6823a 100644 --- a/src/core/error/err.rs +++ b/src/core/error/err.rs @@ -33,6 +33,7 @@ //! option of replacing `error!` with `debug_error!`. #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! Err { ($($args:tt)*) => { Err($crate::err!($($args)*)) @@ -40,6 +41,7 @@ macro_rules! Err { } #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! err { (Request(Forbidden($level:ident!($($args:tt)+)))) => {{ let mut buf = String::new(); @@ -109,6 +111,7 @@ macro_rules! err { /// can share the same callsite metadata for the source of our Error and the /// associated logging and tracing event dispatches. #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! err_log { ($out:ident, $level:ident, $($fields:tt)+) => {{ use $crate::tracing::{ diff --git a/src/core/log/mod.rs b/src/core/log/mod.rs index 5ac374e8..f7b2521a 100644 --- a/src/core/log/mod.rs +++ b/src/core/log/mod.rs @@ -33,6 +33,7 @@ pub struct Log { // the crate namespace like these. #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! event { ( $level:expr_2021, $($x:tt)+ ) => { ::tracing::event!( $level, $($x)+ ) } } diff --git a/src/core/utils/math.rs b/src/core/utils/math.rs index 488f2a13..9316731c 100644 --- a/src/core/utils/math.rs +++ b/src/core/utils/math.rs @@ -10,6 +10,7 @@ use crate::{Err, Error, Result, debug::type_name, err}; /// Checked arithmetic expression. Returns a Result #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! checked { ($($input:tt)+) => { $crate::utils::math::checked_ops!($($input)+) @@ -22,6 +23,7 @@ macro_rules! checked { /// has no realistic expectation for error and no interest in cluttering the /// callsite with result handling from checked!. #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! expected { ($msg:literal, $($input:tt)+) => { $crate::checked!($($input)+).expect($msg) @@ -37,6 +39,7 @@ macro_rules! expected { /// regression analysis. #[cfg(not(debug_assertions))] #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! validated { ($($input:tt)+) => { //#[allow(clippy::arithmetic_side_effects)] { @@ -53,6 +56,7 @@ macro_rules! validated { /// the expression is obviously safe. The check is elided in release-mode. #[cfg(debug_assertions)] #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! validated { ($($input:tt)+) => { $crate::expected!($($input)+) } } diff --git a/src/core/utils/mod.rs b/src/core/utils/mod.rs index 5e6f2868..54404e4c 100644 --- a/src/core/utils/mod.rs +++ b/src/core/utils/mod.rs @@ -173,7 +173,6 @@ macro_rules! is_equal { /// Functor for |x| *x.$i #[macro_export] -#[collapse_debuginfo(yes)] macro_rules! deref_at { ($idx:tt) => { |t| *t.$idx @@ -182,7 +181,6 @@ macro_rules! deref_at { /// Functor for |ref x| x.$i #[macro_export] -#[collapse_debuginfo(yes)] macro_rules! ref_at { ($idx:tt) => { |ref t| &t.$idx @@ -191,7 +189,6 @@ macro_rules! ref_at { /// Functor for |&x| x.$i #[macro_export] -#[collapse_debuginfo(yes)] macro_rules! val_at { ($idx:tt) => { |&t| t.$idx @@ -200,7 +197,6 @@ macro_rules! val_at { /// Functor for |x| x.$i #[macro_export] -#[collapse_debuginfo(yes)] macro_rules! at { ($idx:tt) => { |t| t.$idx diff --git a/src/core/utils/string.rs b/src/core/utils/string.rs index d8fa3f95..7d81903d 100644 --- a/src/core/utils/string.rs +++ b/src/core/utils/string.rs @@ -14,6 +14,7 @@ pub const EMPTY: &str = ""; /// returned otherwise the input (i.e. &'static str) is returned. If multiple /// arguments are provided the first is assumed to be a format string. #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! format_maybe { ($s:literal $(,)?) => { if $crate::is_format!($s) { std::format!($s).into() } else { $s.into() } @@ -27,6 +28,7 @@ macro_rules! format_maybe { /// Constant expression to decide if a literal is a format string. Note: could /// use some improvement. #[macro_export] +#[collapse_debuginfo(yes)] macro_rules! is_format { ($s:literal) => { ::const_str::contains!($s, "{") && ::const_str::contains!($s, "}") diff --git a/src/core/utils/sys/storage.rs b/src/core/utils/sys/storage.rs index 452b04b2..b71c3437 100644 --- a/src/core/utils/sys/storage.rs +++ b/src/core/utils/sys/storage.rs @@ -117,7 +117,7 @@ pub fn name_from_path(path: &Path) -> Result { /// Get the (major, minor) of the block device on which Path is mounted. #[allow(clippy::useless_conversion, clippy::unnecessary_fallible_conversions)] -pub fn dev_from_path(path: &Path) -> Result<(dev_t, dev_t)> { +fn dev_from_path(path: &Path) -> Result<(dev_t, dev_t)> { #[cfg(target_family = "unix")] use std::os::unix::fs::MetadataExt; diff --git a/src/service/mod.rs b/src/service/mod.rs index 63a51213..2be16f79 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,4 +1,4 @@ -#![type_length_limit = "2048"] +#![type_length_limit = "8192"] #![allow(refining_impl_trait)] mod manager; From e71138ab6ffbea621120c41bafb1c65c7b1a3e39 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 8 Apr 2025 03:17:23 +0000 Subject: [PATCH 1711/2091] reduce large stack frames --- src/service/admin/create.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 4de37092..cd0fc5a9 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use conduwuit::{Result, pdu::PduBuilder}; +use futures::FutureExt; use ruma::{ RoomId, RoomVersionId, events::room::{ @@ -63,6 +64,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; // 2. Make server user/bot join @@ -78,6 +80,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; // 3. Power levels @@ -95,6 +98,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; // 4.1 Join Rules @@ -107,6 +111,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; // 4.2 History Visibility @@ -122,6 +127,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; // 4.3 Guest Access @@ -137,6 +143,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; // 5. Events implied by name and topic @@ -150,6 +157,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; services @@ -163,6 +171,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; // 6. Room alias @@ -180,6 +189,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; services @@ -197,6 +207,7 @@ pub async fn create_admin_room(services: &Services) -> Result { &room_id, &state_lock, ) + .boxed() .await?; Ok(()) From 0eb9e4f3d2284a9c96b4c781e25328f1a6e9f9e2 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 19 Apr 2025 23:02:43 +0100 Subject: [PATCH 1712/2091] refactor: Centralize server forbidden checks into moderation module This moves all checks related to `forbidden_remote_server_names`, `forbidden_remote_room_directory_server_names` and `prevent_media_downloads_from` to a new `moderation` module. This is useful for implementing more complicated logic globally. Mostly the changes from #673, but is also relevant for #750 --- conduwuit-example.toml | 3 ++ src/api/client/directory.rs | 19 ++-------- src/api/client/membership.rs | 5 +-- src/api/client/message.rs | 5 +-- src/api/router/auth.rs | 8 +--- src/api/server/invite.rs | 11 ++---- src/api/server/make_join.rs | 11 ++---- src/api/server/make_knock.rs | 11 ++---- src/api/server/send_join.rs | 22 +++-------- src/api/server/send_knock.rs | 11 ++---- src/core/config/mod.rs | 5 ++- src/service/federation/execute.rs | 8 +--- src/service/federation/mod.rs | 4 +- src/service/media/mod.rs | 4 +- src/service/media/remote.rs | 12 +----- src/service/mod.rs | 1 + src/service/moderation.rs | 62 +++++++++++++++++++++++++++++++ src/service/services.rs | 4 +- 18 files changed, 109 insertions(+), 97 deletions(-) create mode 100644 src/service/moderation.rs diff --git a/conduwuit-example.toml b/conduwuit-example.toml index af8da6bb..5a4b7b3f 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1197,6 +1197,9 @@ # incoming AND outgoing federation with, and block client room joins / # remote user invites. # +# Additionally, it will hide messages from these servers for all users +# on this server. +# # This check is applied on the room ID, room alias, sender server name, # sender user's server name, inbound federation X-Matrix origin, and # outbound federation handler. diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index b44b9f64..aa6ae168 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -52,13 +52,8 @@ pub(crate) async fn get_public_rooms_filtered_route( ) -> Result { if let Some(server) = &body.server { if services - .config - .forbidden_remote_room_directory_server_names - .is_match(server.host()) - || services - .config - .forbidden_remote_server_names - .is_match(server.host()) + .moderation + .is_remote_server_room_directory_forbidden(server) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } @@ -92,15 +87,7 @@ pub(crate) async fn get_public_rooms_route( body: Ruma, ) -> Result { if let Some(server) = &body.server { - if services - .config - .forbidden_remote_room_directory_server_names - .is_match(server.host()) - || services - .config - .forbidden_remote_server_names - .is_match(server.host()) - { + if services.moderation.is_remote_server_forbidden(server) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } } diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 18a1c741..b1b85b81 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -83,9 +83,8 @@ async fn banned_room_check( if let Some(room_id) = room_id { if services.rooms.metadata.is_banned(room_id).await || services - .config - .forbidden_remote_server_names - .is_match(room_id.server_name().expect("legacy room mxid").host()) + .moderation + .is_remote_server_forbidden(room_id.server_name().expect("legacy room mxid")) { warn!( "User {user_id} who is not an admin attempted to send an invite for or \ diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 9c2c4057..08887e18 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -274,9 +274,8 @@ pub(crate) async fn is_ignored_pdu( let ignored_type = IGNORED_MESSAGE_TYPES.binary_search(&pdu.kind).is_ok(); let ignored_server = services - .config - .forbidden_remote_server_names - .is_match(pdu.sender().server_name().host()); + .moderation + .is_remote_server_forbidden(pdu.sender().server_name()); if ignored_type && (ignored_server || services.users.user_is_ignored(&pdu.sender, user_id).await) diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 0eb61ca6..01254c32 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -306,7 +306,7 @@ async fn auth_server( } fn auth_server_checks(services: &Services, x_matrix: &XMatrix) -> Result<()> { - if !services.server.config.allow_federation { + if !services.config.allow_federation { return Err!(Config("allow_federation", "Federation is disabled.")); } @@ -316,11 +316,7 @@ fn auth_server_checks(services: &Services, x_matrix: &XMatrix) -> Result<()> { } let origin = &x_matrix.origin; - if services - .config - .forbidden_remote_server_names - .is_match(origin.host()) - { + if services.moderation.is_remote_server_forbidden(origin) { return Err!(Request(Forbidden(debug_warn!( "Federation requests from {origin} denied." )))); diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index edd6ac16..f53e1a15 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -37,19 +37,14 @@ pub(crate) async fn create_invite_route( } if let Some(server) = body.room_id.server_name() { - if services - .config - .forbidden_remote_server_names - .is_match(server.host()) - { + if services.moderation.is_remote_server_forbidden(server) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } } if services - .config - .forbidden_remote_server_names - .is_match(body.origin().host()) + .moderation + .is_remote_server_forbidden(body.origin()) { warn!( "Received federated/remote invite from banned server {} for room ID {}. Rejecting.", diff --git a/src/api/server/make_join.rs b/src/api/server/make_join.rs index ac2c5485..3204c30c 100644 --- a/src/api/server/make_join.rs +++ b/src/api/server/make_join.rs @@ -42,9 +42,8 @@ pub(crate) async fn create_join_event_template_route( .await?; if services - .config - .forbidden_remote_server_names - .is_match(body.origin().host()) + .moderation + .is_remote_server_forbidden(body.origin()) { warn!( "Server {} for remote user {} tried joining room ID {} which has a server name that \ @@ -57,11 +56,7 @@ pub(crate) async fn create_join_event_template_route( } if let Some(server) = body.room_id.server_name() { - if services - .config - .forbidden_remote_server_names - .is_match(server.host()) - { + if services.moderation.is_remote_server_forbidden(server) { return Err!(Request(Forbidden(warn!( "Room ID server name {server} is banned on this homeserver." )))); diff --git a/src/api/server/make_knock.rs b/src/api/server/make_knock.rs index 511c13b2..423c8e81 100644 --- a/src/api/server/make_knock.rs +++ b/src/api/server/make_knock.rs @@ -33,9 +33,8 @@ pub(crate) async fn create_knock_event_template_route( .await?; if services - .config - .forbidden_remote_server_names - .is_match(body.origin().host()) + .moderation + .is_remote_server_forbidden(body.origin()) { warn!( "Server {} for remote user {} tried knocking room ID {} which has a server name \ @@ -48,11 +47,7 @@ pub(crate) async fn create_knock_event_template_route( } if let Some(server) = body.room_id.server_name() { - if services - .config - .forbidden_remote_server_names - .is_match(server.host()) - { + if services.moderation.is_remote_server_forbidden(server) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } } diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index a66d8890..895eca81 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -268,9 +268,8 @@ pub(crate) async fn create_join_event_v1_route( body: Ruma, ) -> Result { if services - .config - .forbidden_remote_server_names - .is_match(body.origin().host()) + .moderation + .is_remote_server_forbidden(body.origin()) { warn!( "Server {} tried joining room ID {} through us who has a server name that is \ @@ -282,11 +281,7 @@ pub(crate) async fn create_join_event_v1_route( } if let Some(server) = body.room_id.server_name() { - if services - .config - .forbidden_remote_server_names - .is_match(server.host()) - { + if services.moderation.is_remote_server_forbidden(server) { warn!( "Server {} tried joining room ID {} through us which has a server name that is \ globally forbidden. Rejecting.", @@ -314,19 +309,14 @@ pub(crate) async fn create_join_event_v2_route( body: Ruma, ) -> Result { if services - .config - .forbidden_remote_server_names - .is_match(body.origin().host()) + .moderation + .is_remote_server_forbidden(body.origin()) { return Err!(Request(Forbidden("Server is banned on this homeserver."))); } if let Some(server) = body.room_id.server_name() { - if services - .config - .forbidden_remote_server_names - .is_match(server.host()) - { + if services.moderation.is_remote_server_forbidden(server) { warn!( "Server {} tried joining room ID {} through us which has a server name that is \ globally forbidden. Rejecting.", diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs index ee7b6cba..8d3697d2 100644 --- a/src/api/server/send_knock.rs +++ b/src/api/server/send_knock.rs @@ -26,9 +26,8 @@ pub(crate) async fn create_knock_event_v1_route( body: Ruma, ) -> Result { if services - .config - .forbidden_remote_server_names - .is_match(body.origin().host()) + .moderation + .is_remote_server_forbidden(body.origin()) { warn!( "Server {} tried knocking room ID {} who has a server name that is globally \ @@ -40,11 +39,7 @@ pub(crate) async fn create_knock_event_v1_route( } if let Some(server) = body.room_id.server_name() { - if services - .config - .forbidden_remote_server_names - .is_match(server.host()) - { + if services.moderation.is_remote_server_forbidden(server) { warn!( "Server {} tried knocking room ID {} which has a server name that is globally \ forbidden. Rejecting.", diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index a7205423..2de3b710 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1374,6 +1374,9 @@ pub struct Config { /// incoming AND outgoing federation with, and block client room joins / /// remote user invites. /// + /// Additionally, it will hide messages from these servers for all users + /// on this server. + /// /// This check is applied on the room ID, room alias, sender server name, /// sender user's server name, inbound federation X-Matrix origin, and /// outbound federation handler. @@ -1954,7 +1957,7 @@ impl Config { let mut addrs = Vec::with_capacity( self.get_bind_hosts() .len() - .saturating_add(self.get_bind_ports().len()), + .saturating_mul(self.get_bind_ports().len()), ); for host in &self.get_bind_hosts() { for port in &self.get_bind_ports() { diff --git a/src/service/federation/execute.rs b/src/service/federation/execute.rs index 97314ffb..1d1d1154 100644 --- a/src/service/federation/execute.rs +++ b/src/service/federation/execute.rs @@ -64,13 +64,7 @@ where return Err!(Config("allow_federation", "Federation is disabled.")); } - if self - .services - .server - .config - .forbidden_remote_server_names - .is_match(dest.host()) - { + if self.services.moderation.is_remote_server_forbidden(dest) { return Err!(Request(Forbidden(debug_warn!("Federation with {dest} is not allowed.")))); } diff --git a/src/service/federation/mod.rs b/src/service/federation/mod.rs index ce7765ee..15521875 100644 --- a/src/service/federation/mod.rs +++ b/src/service/federation/mod.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use conduwuit::{Result, Server}; -use crate::{Dep, client, resolver, server_keys}; +use crate::{Dep, client, moderation, resolver, server_keys}; pub struct Service { services: Services, @@ -15,6 +15,7 @@ struct Services { client: Dep, resolver: Dep, server_keys: Dep, + moderation: Dep, } impl crate::Service for Service { @@ -25,6 +26,7 @@ impl crate::Service for Service { client: args.depend::("client"), resolver: args.depend::("resolver"), server_keys: args.depend::("server_keys"), + moderation: args.depend::("moderation"), }, })) } diff --git a/src/service/media/mod.rs b/src/service/media/mod.rs index 5c26efe8..d053ba54 100644 --- a/src/service/media/mod.rs +++ b/src/service/media/mod.rs @@ -22,7 +22,7 @@ use tokio::{ use self::data::{Data, Metadata}; pub use self::thumbnail::Dim; -use crate::{Dep, client, globals, sending}; +use crate::{Dep, client, globals, moderation, sending}; #[derive(Debug)] pub struct FileMeta { @@ -42,6 +42,7 @@ struct Services { client: Dep, globals: Dep, sending: Dep, + moderation: Dep, } /// generated MXC ID (`media-id`) length @@ -64,6 +65,7 @@ impl crate::Service for Service { client: args.depend::("client"), globals: args.depend::("globals"), sending: args.depend::("sending"), + moderation: args.depend::("moderation"), }, })) } diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index cdcb429e..a1e874d8 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -423,16 +423,8 @@ pub async fn fetch_remote_content_legacy( fn check_fetch_authorized(&self, mxc: &Mxc<'_>) -> Result<()> { if self .services - .server - .config - .prevent_media_downloads_from - .is_match(mxc.server_name.host()) - || self - .services - .server - .config - .forbidden_remote_server_names - .is_match(mxc.server_name.host()) + .moderation + .is_remote_server_media_downloads_forbidden(mxc.server_name) { // we'll lie to the client and say the blocked server's media was not found and // log. the client has no way of telling anyways so this is a security bonus. diff --git a/src/service/mod.rs b/src/service/mod.rs index 2be16f79..a3214408 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -16,6 +16,7 @@ pub mod federation; pub mod globals; pub mod key_backups; pub mod media; +pub mod moderation; pub mod presence; pub mod pusher; pub mod resolver; diff --git a/src/service/moderation.rs b/src/service/moderation.rs new file mode 100644 index 00000000..bd2616f6 --- /dev/null +++ b/src/service/moderation.rs @@ -0,0 +1,62 @@ +use std::sync::Arc; + +use conduwuit::{Result, Server, implement}; +use ruma::ServerName; + +pub struct Service { + services: Services, +} + +struct Services { + pub server: Arc, +} + +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + services: Services { server: args.server.clone() }, + })) + } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +#[implement(Service)] +#[must_use] +pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool { + // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) + // OR forbidden contains server + self.services + .server + .config + .forbidden_remote_server_names + .is_match(server_name.host()) +} + +#[implement(Service)] +#[must_use] +pub fn is_remote_server_room_directory_forbidden(&self, server_name: &ServerName) -> bool { + // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) + // OR forbidden contains server + self.is_remote_server_forbidden(server_name) + || self + .services + .server + .config + .forbidden_remote_room_directory_server_names + .is_match(server_name.host()) +} + +#[implement(Service)] +#[must_use] +pub fn is_remote_server_media_downloads_forbidden(&self, server_name: &ServerName) -> bool { + // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) + // OR forbidden contains server + self.is_remote_server_forbidden(server_name) + || self + .services + .server + .config + .prevent_media_downloads_from + .is_match(server_name.host()) +} diff --git a/src/service/services.rs b/src/service/services.rs index dc390054..5dcc120e 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -12,7 +12,7 @@ use tokio::sync::Mutex; use crate::{ account_data, admin, appservice, client, config, emergency, federation, globals, key_backups, manager::Manager, - media, presence, pusher, resolver, rooms, sending, server_keys, service, + media, moderation, presence, pusher, resolver, rooms, sending, server_keys, service, service::{Args, Map, Service}, sync, transaction_ids, uiaa, updates, users, }; @@ -39,6 +39,7 @@ pub struct Services { pub uiaa: Arc, pub updates: Arc, pub users: Arc, + pub moderation: Arc, manager: Mutex>>, pub(crate) service: Arc, @@ -106,6 +107,7 @@ impl Services { uiaa: build!(uiaa::Service), updates: build!(updates::Service), users: build!(users::Service), + moderation: build!(moderation::Service), manager: Mutex::new(None), service, From 9e62076baa2fb4a6bb46f8a763e38240c98be5ee Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 19 Apr 2025 23:29:33 +0100 Subject: [PATCH 1713/2091] feat: Add `allowed_remote_server_names` This allows explicitly allowing servers. Can be combined with the opposite to create allowlist-only federation. See also #31 Closes #673 --- conduwuit-example.toml | 10 ++++++++++ src/core/config/mod.rs | 12 ++++++++++++ src/service/moderation.rs | 19 +++++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 5a4b7b3f..326127c3 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1206,10 +1206,20 @@ # # Basically "global" ACLs. # +# You can set this to ["*"] to block all servers by default, and then +# use `allowed_remote_server_names` to allow only specific servers. +# # example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] # #forbidden_remote_server_names = [] +# List of allowed server names via regex patterns that we will allow, +# regardless of if they match `forbidden_remote_server_names`. +# +# example: ["goodserver\.tld$", "goodphrase"] +# +#allowed_remote_server_names = [] + # List of forbidden server names via regex patterns that we will block all # outgoing federated room directory requests for. Useful for preventing # our users from wandering into bad servers or spaces. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 2de3b710..22e09956 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1383,12 +1383,24 @@ pub struct Config { /// /// Basically "global" ACLs. /// + /// You can set this to ["*"] to block all servers by default, and then + /// use `allowed_remote_server_names` to allow only specific servers. + /// /// example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] /// /// default: [] #[serde(default, with = "serde_regex")] pub forbidden_remote_server_names: RegexSet, + /// List of allowed server names via regex patterns that we will allow, + /// regardless of if they match `forbidden_remote_server_names`. + /// + /// example: ["goodserver\.tld$", "goodphrase"] + /// + /// default: [] + #[serde(default, with = "serde_regex")] + pub allowed_remote_server_names: RegexSet, + /// List of forbidden server names via regex patterns that we will block all /// outgoing federated room directory requests for. Useful for preventing /// our users from wandering into bad servers or spaces. diff --git a/src/service/moderation.rs b/src/service/moderation.rs index bd2616f6..d571de88 100644 --- a/src/service/moderation.rs +++ b/src/service/moderation.rs @@ -24,8 +24,23 @@ impl crate::Service for Service { #[implement(Service)] #[must_use] pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool { - // Forbidden if NOT (allowed is empty OR allowed contains server OR is self) - // OR forbidden contains server + // We must never block federating with ourselves + if server_name == self.services.server.config.server_name { + return false; + } + + // Check if server is explicitly allowed + if self + .services + .server + .config + .allowed_remote_server_names + .is_match(server_name.host()) + { + return false; + } + + // Check if server is explicitly forbidden self.services .server .config From 84445b84580720b5f296f525d7df655f4195d833 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 20 Apr 2025 00:16:29 +0100 Subject: [PATCH 1714/2091] docs: Document backfill bypassing federation restrictions --- conduwuit-example.toml | 4 ++++ src/core/config/mod.rs | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 326127c3..8f86fdd0 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1200,6 +1200,10 @@ # Additionally, it will hide messages from these servers for all users # on this server. # +# Note that your messages can still make it to forbidden servers through +# backfilling. Events we receive from forbidden servers via backfill will +# be stored in the database, but will not be sent to the client. +# # This check is applied on the room ID, room alias, sender server name, # sender user's server name, inbound federation X-Matrix origin, and # outbound federation handler. diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 22e09956..cde5c313 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1377,6 +1377,10 @@ pub struct Config { /// Additionally, it will hide messages from these servers for all users /// on this server. /// + /// Note that your messages can still make it to forbidden servers through + /// backfilling. Events we receive from forbidden servers via backfill will + /// be stored in the database, but will not be sent to the client. + /// /// This check is applied on the room ID, room alias, sender server name, /// sender user's server name, inbound federation X-Matrix origin, and /// outbound federation handler. From fe7963d30648addbc4ecfd1df3798cf0f5c0c8fa Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 20 Apr 2025 00:31:08 +0100 Subject: [PATCH 1715/2091] docs: Clarify --- src/core/config/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index cde5c313..800ffc8d 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1385,8 +1385,6 @@ pub struct Config { /// sender user's server name, inbound federation X-Matrix origin, and /// outbound federation handler. /// - /// Basically "global" ACLs. - /// /// You can set this to ["*"] to block all servers by default, and then /// use `allowed_remote_server_names` to allow only specific servers. /// @@ -1399,6 +1397,8 @@ pub struct Config { /// List of allowed server names via regex patterns that we will allow, /// regardless of if they match `forbidden_remote_server_names`. /// + /// This option has no effect if `forbidden_remote_server_names` is empty. + /// /// example: ["goodserver\.tld$", "goodphrase"] /// /// default: [] From 6920814da9867a74fa1b87fb4776c2587fe3bd54 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 20 Apr 2025 02:31:58 +0100 Subject: [PATCH 1716/2091] Support fi.mau.room_id, and fully qualified room_id in /createRoom --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 2 +- src/api/client/room/create.rs | 25 +++++++++++-------------- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00aeca81..cf3ac6db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3652,7 +3652,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "assign", "js_int", @@ -3672,7 +3672,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "js_int", "ruma-common", @@ -3684,7 +3684,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "as_variant", "assign", @@ -3707,7 +3707,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "as_variant", "base64 0.22.1", @@ -3739,7 +3739,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "as_variant", "indexmap 2.8.0", @@ -3764,7 +3764,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "bytes", "headers", @@ -3786,7 +3786,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3795,7 +3795,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "js_int", "ruma-common", @@ -3805,7 +3805,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3820,7 +3820,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "js_int", "ruma-common", @@ -3832,7 +3832,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=920148dca1076454ca0ca5d43b5ce1aa708381d4#920148dca1076454ca0ca5d43b5ce1aa708381d4" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index e9ae0007..1ad11256 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,7 +350,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://forgejo.ellis.link/continuwuation/ruwuma" #branch = "conduwuit-changes" -rev = "920148dca1076454ca0ca5d43b5ce1aa708381d4" +rev = "fa3c868e5a1c049dc9472310dc4955289a96bb35" features = [ "compat", "rand", diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index 4ce53f15..bba5939e 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -606,23 +606,20 @@ fn custom_room_id_check(services: &Services, custom_room_id: &str) -> Result Date: Sun, 20 Apr 2025 02:46:16 +0100 Subject: [PATCH 1717/2091] Prevent creating custom room IDs belonging to other servers --- src/api/client/room/create.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index bba5939e..2bc6033c 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -621,6 +621,11 @@ fn custom_room_id_check(services: &Services, custom_room_id: &str) -> Result Date: Sun, 20 Apr 2025 15:41:19 +0100 Subject: [PATCH 1718/2091] Fix invalid room ID check & prevent room IDs being prefixed with ! --- src/api/client/room/create.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index 2bc6033c..f5f61784 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -107,7 +107,6 @@ pub(crate) async fn create_room_route( return Err!(Request(Forbidden("Publishing rooms to the room directory is not allowed"))); } - let _short_id = services .rooms .short @@ -615,17 +614,26 @@ fn custom_room_id_check(services: &Services, custom_room_id: &str) -> Result Date: Wed, 9 Apr 2025 19:17:21 +0200 Subject: [PATCH 1719/2091] config: rocksdb_compaction help was inverted :-) You seem to have replaced `disable_rocksdb_compaction` with `rocksdb_compaction`, since the help is blackmailing me never to set it to `true`, except **true is the default**. I have tried to make it say what you possibly meant. --- conduwuit-example.toml | 8 ++++---- src/core/config/mod.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 8f86fdd0..273d5ea5 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -970,8 +970,8 @@ # #rocksdb_compaction_ioprio_idle = true -# Disables RocksDB compaction. You should never ever have to set this -# option to true. If you for some reason find yourself needing to use this +# Enables RocksDB compaction. You should never ever have to set this +# option to false. If you for some reason find yourself needing to use this # option as part of troubleshooting or a bug, please reach out to us in # the conduwuit Matrix room with information and details. # @@ -1208,8 +1208,6 @@ # sender user's server name, inbound federation X-Matrix origin, and # outbound federation handler. # -# Basically "global" ACLs. -# # You can set this to ["*"] to block all servers by default, and then # use `allowed_remote_server_names` to allow only specific servers. # @@ -1220,6 +1218,8 @@ # List of allowed server names via regex patterns that we will allow, # regardless of if they match `forbidden_remote_server_names`. # +# This option has no effect if `forbidden_remote_server_names` is empty. +# # example: ["goodserver\.tld$", "goodphrase"] # #allowed_remote_server_names = [] diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 800ffc8d..bdfcee41 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1133,8 +1133,8 @@ pub struct Config { #[serde(default = "true_fn")] pub rocksdb_compaction_ioprio_idle: bool, - /// Disables RocksDB compaction. You should never ever have to set this - /// option to true. If you for some reason find yourself needing to use this + /// Enables RocksDB compaction. You should never ever have to set this + /// option to false. If you for some reason find yourself needing to use this /// option as part of troubleshooting or a bug, please reach out to us in /// the conduwuit Matrix room with information and details. /// From 2d9bdc0979ecb1102ca2cc3f6b33d1090bd08025 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 20 Apr 2025 19:30:02 +0100 Subject: [PATCH 1720/2091] refactor: The update checker has become the announcements checker Replaces June's endpoint with a continuwuity endpoint. Adds a JSON schema. Closes #89 Closes #760 --- .forgejo/workflows/documentation.yml | 4 + conduwuit-example.toml | 10 +- docs/static/_headers | 3 + docs/static/announcements.json | 9 ++ docs/static/announcements.schema.json | 31 +++++ src/admin/query/globals.rs | 9 +- src/core/config/mod.rs | 12 +- src/service/announcements/mod.rs | 169 ++++++++++++++++++++++++++ src/service/globals/mod.rs | 4 +- src/service/mod.rs | 2 +- src/service/services.rs | 9 +- src/service/updates/mod.rs | 142 ---------------------- 12 files changed, 238 insertions(+), 166 deletions(-) create mode 100644 docs/static/announcements.json create mode 100644 docs/static/announcements.schema.json create mode 100644 src/service/announcements/mod.rs delete mode 100644 src/service/updates/mod.rs diff --git a/.forgejo/workflows/documentation.yml b/.forgejo/workflows/documentation.yml index c08c1abb..c84c566b 100644 --- a/.forgejo/workflows/documentation.yml +++ b/.forgejo/workflows/documentation.yml @@ -36,9 +36,13 @@ jobs: - name: Prepare static files for deployment run: | mkdir -p ./public/.well-known/matrix + mkdir -p ./public/.well-known/continuwuity + mkdir -p ./public/schema # Copy the Matrix .well-known files cp ./docs/static/server ./public/.well-known/matrix/server cp ./docs/static/client ./public/.well-known/matrix/client + cp ./docs/static/announcements.json ./public/.well-known/continuwuity/announcements + cp ./docs/static/announcements.schema.json ./public/schema/announcements.schema.json # Copy the custom headers file cp ./docs/static/_headers ./public/_headers echo "Copied .well-known files and _headers to ./public" diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 273d5ea5..b6bfd092 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -113,14 +113,10 @@ #new_user_displayname_suffix = "🏳️‍⚧️" # If enabled, conduwuit will send a simple GET request periodically to -# `https://pupbrain.dev/check-for-updates/stable` for any new -# announcements made. Despite the name, this is not an update check -# endpoint, it is simply an announcement check endpoint. +# `https://continuwuity.org/.well-known/continuwuity/announcements` for any new +# announcements or major updates. This is not an update check endpoint. # -# This is disabled by default as this is rarely used except for security -# updates or major updates. -# -#allow_check_for_updates = false +#allow_announcements_check = # Set this to any float value to multiply conduwuit's in-memory LRU caches # with such as "auth_chain_cache_capacity". diff --git a/docs/static/_headers b/docs/static/_headers index 5e960241..6e52de9f 100644 --- a/docs/static/_headers +++ b/docs/static/_headers @@ -1,3 +1,6 @@ /.well-known/matrix/* Access-Control-Allow-Origin: * Content-Type: application/json +/.well-known/continuwuity/* + Access-Control-Allow-Origin: * + Content-Type: application/json \ No newline at end of file diff --git a/docs/static/announcements.json b/docs/static/announcements.json new file mode 100644 index 00000000..9b97d091 --- /dev/null +++ b/docs/static/announcements.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://continuwuity.org/schema/announcements.schema.json", + "announcements": [ + { + "id": 1, + "message": "Welcome to Continuwuity! Important announcements about the project will appear here." + } + ] +} \ No newline at end of file diff --git a/docs/static/announcements.schema.json b/docs/static/announcements.schema.json new file mode 100644 index 00000000..95b1d153 --- /dev/null +++ b/docs/static/announcements.schema.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$id": "https://continwuity.org/schema/announcements.schema.json", + "type": "object", + "properties": { + "updates": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "message": { + "type": "string" + }, + "date": { + "type": "string" + } + }, + "required": [ + "id", + "message" + ] + } + } + }, + "required": [ + "updates" + ] + } \ No newline at end of file diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index 3681acfd..c8c1f512 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -11,7 +11,7 @@ pub(crate) enum GlobalsCommand { CurrentCount, - LastCheckForUpdatesId, + LastCheckForAnnouncementsId, /// - This returns an empty `Ok(BTreeMap<..>)` when there are no keys found /// for the server. @@ -39,9 +39,12 @@ pub(super) async fn process(subcommand: GlobalsCommand, context: &Context<'_>) - write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") }, - | GlobalsCommand::LastCheckForUpdatesId => { + | GlobalsCommand::LastCheckForAnnouncementsId => { let timer = tokio::time::Instant::now(); - let results = services.updates.last_check_for_updates_id().await; + let results = services + .announcements + .last_check_for_announcements_id() + .await; let query_time = timer.elapsed(); write!(context, "Query completed in {query_time:?}:\n\n```rs\n{results:#?}\n```") diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index bdfcee41..033be40a 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -161,14 +161,10 @@ pub struct Config { pub new_user_displayname_suffix: String, /// If enabled, conduwuit will send a simple GET request periodically to - /// `https://pupbrain.dev/check-for-updates/stable` for any new - /// announcements made. Despite the name, this is not an update check - /// endpoint, it is simply an announcement check endpoint. - /// - /// This is disabled by default as this is rarely used except for security - /// updates or major updates. - #[serde(default, alias = "allow_announcements_check")] - pub allow_check_for_updates: bool, + /// `https://continuwuity.org/.well-known/continuwuity/announcements` for any new + /// announcements or major updates. This is not an update check endpoint. + #[serde(alias = "allow_check_for_updates", default = "true_fn")] + pub allow_announcements_check: bool, /// Set this to any float value to multiply conduwuit's in-memory LRU caches /// with such as "auth_chain_cache_capacity". diff --git a/src/service/announcements/mod.rs b/src/service/announcements/mod.rs new file mode 100644 index 00000000..4df8971b --- /dev/null +++ b/src/service/announcements/mod.rs @@ -0,0 +1,169 @@ +//! # Announcements service +//! +//! This service is responsible for checking for announcements and sending them +//! to the client. +//! +//! It is used to send announcements to the admin room and logs. +//! Annuncements are stored in /docs/static/announcements right now. +//! The highest seen announcement id is stored in the database. When the +//! announcement check is run, all announcements with an ID higher than those +//! seen before are printed to the console and sent to the admin room. +//! +//! Old announcements should be deleted to avoid spamming the room on first +//! install. +//! +//! Announcements are displayed as markdown in the admin room, but plain text in +//! the console. + +use std::{sync::Arc, time::Duration}; + +use async_trait::async_trait; +use conduwuit::{Result, Server, debug, info, warn}; +use database::{Deserialized, Map}; +use ruma::events::room::message::RoomMessageEventContent; +use serde::Deserialize; +use tokio::{ + sync::Notify, + time::{MissedTickBehavior, interval}, +}; + +use crate::{Dep, admin, client, globals}; + +pub struct Service { + interval: Duration, + interrupt: Notify, + db: Arc, + services: Services, +} + +struct Services { + admin: Dep, + client: Dep, + globals: Dep, + server: Arc, +} + +#[derive(Debug, Deserialize)] +struct CheckForAnnouncementsResponse { + announcements: Vec, +} + +#[derive(Debug, Deserialize)] +struct CheckForAnnouncementsResponseEntry { + id: u64, + date: Option, + message: String, +} + +const CHECK_FOR_ANNOUNCEMENTS_URL: &str = + "https://continuwuity.org/.well-known/continuwuity/announcements"; +const CHECK_FOR_ANNOUNCEMENTS_INTERVAL: u64 = 7200; // 2 hours +const LAST_CHECK_FOR_ANNOUNCEMENTS_ID: &[u8; 25] = b"last_seen_announcement_id"; +// In conduwuit, this was under b"a" + +#[async_trait] +impl crate::Service for Service { + fn build(args: crate::Args<'_>) -> Result> { + Ok(Arc::new(Self { + interval: Duration::from_secs(CHECK_FOR_ANNOUNCEMENTS_INTERVAL), + interrupt: Notify::new(), + db: args.db["global"].clone(), + services: Services { + globals: args.depend::("globals"), + admin: args.depend::("admin"), + client: args.depend::("client"), + server: args.server.clone(), + }, + })) + } + + #[tracing::instrument(skip_all, name = "announcements", level = "debug")] + async fn worker(self: Arc) -> Result<()> { + if !self.services.globals.allow_announcements_check() { + debug!("Disabling announcements check"); + return Ok(()); + } + + let mut i = interval(self.interval); + i.set_missed_tick_behavior(MissedTickBehavior::Delay); + i.reset_after(self.interval); + loop { + tokio::select! { + () = self.interrupt.notified() => break, + _ = i.tick() => (), + } + + if let Err(e) = self.check().await { + warn!(%e, "Failed to check for announcements"); + } + } + + Ok(()) + } + + fn interrupt(&self) { self.interrupt.notify_waiters(); } + + fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } +} + +impl Service { + #[tracing::instrument(skip_all)] + async fn check(&self) -> Result<()> { + debug_assert!(self.services.server.running(), "server must not be shutting down"); + + let response = self + .services + .client + .default + .get(CHECK_FOR_ANNOUNCEMENTS_URL) + .send() + .await? + .text() + .await?; + + let response = serde_json::from_str::(&response)?; + for announcement in &response.announcements { + if announcement.id > self.last_check_for_announcements_id().await { + self.handle(announcement).await; + self.update_check_for_announcements_id(announcement.id); + } + } + + Ok(()) + } + + #[tracing::instrument(skip_all)] + async fn handle(&self, announcement: &CheckForAnnouncementsResponseEntry) { + if let Some(date) = &announcement.date { + info!("[announcements] {date} {:#}", announcement.message); + } else { + info!("[announcements] {:#}", announcement.message); + } + + self.services + .admin + .send_message(RoomMessageEventContent::text_markdown(format!( + "### New announcement{}\n\n{}", + announcement + .date + .as_ref() + .map_or_else(String::new, |date| format!(" - `{date}`")), + announcement.message + ))) + .await + .ok(); + } + + #[inline] + pub fn update_check_for_announcements_id(&self, id: u64) { + self.db.raw_put(LAST_CHECK_FOR_ANNOUNCEMENTS_ID, id); + } + + pub async fn last_check_for_announcements_id(&self) -> u64 { + self.db + .get(LAST_CHECK_FOR_ANNOUNCEMENTS_ID) + .await + .deserialized() + .unwrap_or(0_u64) + } +} diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index a7a9be9d..a23a4c21 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -127,7 +127,9 @@ impl Service { &self.server.config.new_user_displayname_suffix } - pub fn allow_check_for_updates(&self) -> bool { self.server.config.allow_check_for_updates } + pub fn allow_announcements_check(&self) -> bool { + self.server.config.allow_announcements_check + } pub fn trusted_servers(&self) -> &[OwnedServerName] { &self.server.config.trusted_servers } diff --git a/src/service/mod.rs b/src/service/mod.rs index a3214408..eb15e5ec 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -8,6 +8,7 @@ pub mod services; pub mod account_data; pub mod admin; +pub mod announcements; pub mod appservice; pub mod client; pub mod config; @@ -26,7 +27,6 @@ pub mod server_keys; pub mod sync; pub mod transaction_ids; pub mod uiaa; -pub mod updates; pub mod users; extern crate conduwuit_core as conduwuit; diff --git a/src/service/services.rs b/src/service/services.rs index 5dcc120e..daece245 100644 --- a/src/service/services.rs +++ b/src/service/services.rs @@ -10,11 +10,12 @@ use futures::{Stream, StreamExt, TryStreamExt}; use tokio::sync::Mutex; use crate::{ - account_data, admin, appservice, client, config, emergency, federation, globals, key_backups, + account_data, admin, announcements, appservice, client, config, emergency, federation, + globals, key_backups, manager::Manager, media, moderation, presence, pusher, resolver, rooms, sending, server_keys, service, service::{Args, Map, Service}, - sync, transaction_ids, uiaa, updates, users, + sync, transaction_ids, uiaa, users, }; pub struct Services { @@ -37,9 +38,9 @@ pub struct Services { pub sync: Arc, pub transaction_ids: Arc, pub uiaa: Arc, - pub updates: Arc, pub users: Arc, pub moderation: Arc, + pub announcements: Arc, manager: Mutex>>, pub(crate) service: Arc, @@ -105,9 +106,9 @@ impl Services { sync: build!(sync::Service), transaction_ids: build!(transaction_ids::Service), uiaa: build!(uiaa::Service), - updates: build!(updates::Service), users: build!(users::Service), moderation: build!(moderation::Service), + announcements: build!(announcements::Service), manager: Mutex::new(None), service, diff --git a/src/service/updates/mod.rs b/src/service/updates/mod.rs deleted file mode 100644 index 28bee65a..00000000 --- a/src/service/updates/mod.rs +++ /dev/null @@ -1,142 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use async_trait::async_trait; -use conduwuit::{Result, Server, debug, info, warn}; -use database::{Deserialized, Map}; -use ruma::events::room::message::RoomMessageEventContent; -use serde::Deserialize; -use tokio::{ - sync::Notify, - time::{MissedTickBehavior, interval}, -}; - -use crate::{Dep, admin, client, globals}; - -pub struct Service { - interval: Duration, - interrupt: Notify, - db: Arc, - services: Services, -} - -struct Services { - admin: Dep, - client: Dep, - globals: Dep, - server: Arc, -} - -#[derive(Debug, Deserialize)] -struct CheckForUpdatesResponse { - updates: Vec, -} - -#[derive(Debug, Deserialize)] -struct CheckForUpdatesResponseEntry { - id: u64, - date: String, - message: String, -} - -const CHECK_FOR_UPDATES_URL: &str = "https://pupbrain.dev/check-for-updates/stable"; -const CHECK_FOR_UPDATES_INTERVAL: u64 = 7200; // 2 hours -const LAST_CHECK_FOR_UPDATES_COUNT: &[u8; 1] = b"u"; - -#[async_trait] -impl crate::Service for Service { - fn build(args: crate::Args<'_>) -> Result> { - Ok(Arc::new(Self { - interval: Duration::from_secs(CHECK_FOR_UPDATES_INTERVAL), - interrupt: Notify::new(), - db: args.db["global"].clone(), - services: Services { - globals: args.depend::("globals"), - admin: args.depend::("admin"), - client: args.depend::("client"), - server: args.server.clone(), - }, - })) - } - - #[tracing::instrument(skip_all, name = "updates", level = "debug")] - async fn worker(self: Arc) -> Result<()> { - if !self.services.globals.allow_check_for_updates() { - debug!("Disabling update check"); - return Ok(()); - } - - let mut i = interval(self.interval); - i.set_missed_tick_behavior(MissedTickBehavior::Delay); - i.reset_after(self.interval); - loop { - tokio::select! { - () = self.interrupt.notified() => break, - _ = i.tick() => (), - } - - if let Err(e) = self.check().await { - warn!(%e, "Failed to check for updates"); - } - } - - Ok(()) - } - - fn interrupt(&self) { self.interrupt.notify_waiters(); } - - fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } -} - -impl Service { - #[tracing::instrument(skip_all)] - async fn check(&self) -> Result<()> { - debug_assert!(self.services.server.running(), "server must not be shutting down"); - - let response = self - .services - .client - .default - .get(CHECK_FOR_UPDATES_URL) - .send() - .await? - .text() - .await?; - - let response = serde_json::from_str::(&response)?; - for update in &response.updates { - if update.id > self.last_check_for_updates_id().await { - self.handle(update).await; - self.update_check_for_updates_id(update.id); - } - } - - Ok(()) - } - - #[tracing::instrument(skip_all)] - async fn handle(&self, update: &CheckForUpdatesResponseEntry) { - info!("{} {:#}", update.date, update.message); - self.services - .admin - .send_message(RoomMessageEventContent::text_markdown(format!( - "### the following is a message from the conduwuit puppy\n\nit was sent on \ - `{}`:\n\n@room: {}", - update.date, update.message - ))) - .await - .ok(); - } - - #[inline] - pub fn update_check_for_updates_id(&self, id: u64) { - self.db.raw_put(LAST_CHECK_FOR_UPDATES_COUNT, id); - } - - pub async fn last_check_for_updates_id(&self) -> u64 { - self.db - .get(LAST_CHECK_FOR_UPDATES_COUNT) - .await - .deserialized() - .unwrap_or(0_u64) - } -} From b7b7d3a9e70007efb7128ee7d108ed15fe5362db Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 20 Apr 2025 23:07:01 +0100 Subject: [PATCH 1721/2091] chore: Add the current prerelease to cargo.toml --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cf3ac6db..616b1034 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -725,7 +725,7 @@ dependencies = [ [[package]] name = "conduwuit" -version = "0.5.0" +version = "0.5.0-rc.4" dependencies = [ "clap", "conduwuit_admin", @@ -754,7 +754,7 @@ dependencies = [ [[package]] name = "conduwuit_admin" -version = "0.5.0" +version = "0.5.0-rc.4" dependencies = [ "clap", "conduwuit_api", @@ -775,7 +775,7 @@ dependencies = [ [[package]] name = "conduwuit_api" -version = "0.5.0" +version = "0.5.0-rc.4" dependencies = [ "async-trait", "axum", @@ -807,7 +807,7 @@ dependencies = [ [[package]] name = "conduwuit_core" -version = "0.5.0" +version = "0.5.0-rc.4" dependencies = [ "argon2", "arrayvec", @@ -865,7 +865,7 @@ dependencies = [ [[package]] name = "conduwuit_database" -version = "0.5.0" +version = "0.5.0-rc.4" dependencies = [ "async-channel", "conduwuit_core", @@ -883,7 +883,7 @@ dependencies = [ [[package]] name = "conduwuit_macros" -version = "0.5.0" +version = "0.5.0-rc.4" dependencies = [ "itertools 0.14.0", "proc-macro2", @@ -893,7 +893,7 @@ dependencies = [ [[package]] name = "conduwuit_router" -version = "0.5.0" +version = "0.5.0-rc.4" dependencies = [ "axum", "axum-client-ip", @@ -926,7 +926,7 @@ dependencies = [ [[package]] name = "conduwuit_service" -version = "0.5.0" +version = "0.5.0-rc.4" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 1ad11256..5feba474 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://forgejo.ellis.link/continuwuation/continuwuity" rust-version = "1.86.0" -version = "0.5.0" +version = "0.5.0-rc.4" [workspace.metadata.crane] name = "conduwuit" From 22e7617362880ba9723fc239e5bd7b978599c866 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 20 Apr 2025 23:07:20 +0100 Subject: [PATCH 1722/2091] chore: Release --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 2 +- src/router/Cargo.toml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 616b1034..afaa5622 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -725,7 +725,7 @@ dependencies = [ [[package]] name = "conduwuit" -version = "0.5.0-rc.4" +version = "0.5.0-rc.5" dependencies = [ "clap", "conduwuit_admin", @@ -754,7 +754,7 @@ dependencies = [ [[package]] name = "conduwuit_admin" -version = "0.5.0-rc.4" +version = "0.5.0-rc.5" dependencies = [ "clap", "conduwuit_api", @@ -775,7 +775,7 @@ dependencies = [ [[package]] name = "conduwuit_api" -version = "0.5.0-rc.4" +version = "0.5.0-rc.5" dependencies = [ "async-trait", "axum", @@ -807,7 +807,7 @@ dependencies = [ [[package]] name = "conduwuit_core" -version = "0.5.0-rc.4" +version = "0.5.0-rc.5" dependencies = [ "argon2", "arrayvec", @@ -865,7 +865,7 @@ dependencies = [ [[package]] name = "conduwuit_database" -version = "0.5.0-rc.4" +version = "0.5.0-rc.5" dependencies = [ "async-channel", "conduwuit_core", @@ -883,7 +883,7 @@ dependencies = [ [[package]] name = "conduwuit_macros" -version = "0.5.0-rc.4" +version = "0.5.0-rc.5" dependencies = [ "itertools 0.14.0", "proc-macro2", @@ -893,7 +893,7 @@ dependencies = [ [[package]] name = "conduwuit_router" -version = "0.5.0-rc.4" +version = "0.5.0-rc.5" dependencies = [ "axum", "axum-client-ip", @@ -926,7 +926,7 @@ dependencies = [ [[package]] name = "conduwuit_service" -version = "0.5.0-rc.4" +version = "0.5.0-rc.5" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index 5feba474..1517cfc1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://forgejo.ellis.link/continuwuation/continuwuity" rust-version = "1.86.0" -version = "0.5.0-rc.4" +version = "0.5.0-rc.5" [workspace.metadata.crane] name = "conduwuit" diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 31a44983..e4ddcb9b 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -114,11 +114,11 @@ ruma.workspace = true rustls.workspace = true rustls.optional = true sentry.optional = true +sentry.workspace = true sentry-tower.optional = true sentry-tower.workspace = true sentry-tracing.optional = true sentry-tracing.workspace = true -sentry.workspace = true serde_json.workspace = true tokio.workspace = true tower.workspace = true From ff93cfdc6454b990f35424d3d7c17fae14df2c4d Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 20 Apr 2025 23:50:48 +0100 Subject: [PATCH 1723/2091] Fix up the docs, replace a lot of conduwuit references --- docs/appservices.md | 10 ++-- docs/conduwuit_coc.md | 28 +++++----- docs/configuration.md | 10 ++-- docs/deploying.md | 2 +- docs/deploying/arch-linux.md | 16 +----- docs/deploying/docker-compose.for-traefik.yml | 2 +- docs/deploying/docker-compose.with-caddy.yml | 2 +- .../deploying/docker-compose.with-traefik.yml | 2 +- docs/deploying/docker-compose.yml | 2 +- docs/deploying/docker.md | 44 +++++---------- docs/deploying/freebsd.md | 6 +- docs/deploying/generic.md | 52 +++++++++--------- docs/deploying/nixos.md | 55 ++++--------------- docs/development.md | 20 +++---- docs/introduction.md | 4 +- docs/maintenance.md | 26 ++++----- docs/troubleshooting.md | 38 ++++++------- docs/turn.md | 8 +-- 18 files changed, 133 insertions(+), 194 deletions(-) diff --git a/docs/appservices.md b/docs/appservices.md index 28ea9717..57cd031c 100644 --- a/docs/appservices.md +++ b/docs/appservices.md @@ -3,8 +3,8 @@ ## Getting help If you run into any problems while setting up an Appservice: ask us in -[#conduwuit:puppygock.gay](https://matrix.to/#/#conduwuit:puppygock.gay) or -[open an issue on GitHub](https://github.com/girlbossceo/conduwuit/issues/new). +[#continuwuity:continuwuity.org](https://matrix.to/#/#continuwuity:continuwuity.org) or +[open an issue on Forgejo](https://forgejo.ellis.link/continuwuation/continuwuity/issues/new). ## Set up the appservice - general instructions @@ -14,7 +14,7 @@ later starting it. At some point the appservice guide should ask you to add a registration yaml file to the homeserver. In Synapse you would do this by adding the path to the -homeserver.yaml, but in conduwuit you can do this from within Matrix: +homeserver.yaml, but in Continuwuity you can do this from within Matrix: First, go into the `#admins` room of your homeserver. The first person that registered on the homeserver automatically joins it. Then send a message into @@ -37,9 +37,9 @@ You can confirm it worked by sending a message like this: The server bot should answer with `Appservices (1): your-bridge` -Then you are done. conduwuit will send messages to the appservices and the +Then you are done. Continuwuity will send messages to the appservices and the appservice can send requests to the homeserver. You don't need to restart -conduwuit, but if it doesn't work, restarting while the appservice is running +Continuwuity, but if it doesn't work, restarting while the appservice is running could help. ## Appservice-specific instructions diff --git a/docs/conduwuit_coc.md b/docs/conduwuit_coc.md index 0fce2fe3..19262765 100644 --- a/docs/conduwuit_coc.md +++ b/docs/conduwuit_coc.md @@ -1,17 +1,17 @@ -# conduwuit Community Code of Conduct +# Continuwuity Community Code of Conduct -Welcome to the conduwuit community! We’re excited to have you here. conduwuit is +Welcome to the Continuwuity community! We’re excited to have you here. Continuwuity is a hard-fork of the Conduit homeserver, aimed at making Matrix more accessible and inclusive for everyone. This space is dedicated to fostering a positive, supportive, and inclusive -environment for everyone. This Code of Conduct applies to all conduwuit spaces, +environment for everyone. This Code of Conduct applies to all Continuwuity spaces, including any further community rooms that reference this CoC. Here are our -guidelines to help maintain the welcoming atmosphere that sets conduwuit apart. +guidelines to help maintain the welcoming atmosphere that sets Continuwuity apart. For the general foundational rules, please refer to the [Contributor's -Covenant](https://github.com/girlbossceo/conduwuit/blob/main/CODE_OF_CONDUCT.md). -Below are additional guidelines specific to the conduwuit community. +Covenant](https://forgejo.ellis.link/continuwuation/continuwuity/src/branch/main/CODE_OF_CONDUCT.md). +Below are additional guidelines specific to the Continuwuity community. ## Our Values and Guidelines @@ -48,25 +48,25 @@ members. ## Matrix Community -This Code of Conduct applies to the entire [conduwuit Matrix -Space](https://matrix.to/#/#conduwuit-space:puppygock.gay) and its rooms, +This Code of Conduct applies to the entire [Continuwuity Matrix +Space](https://matrix.to/#/#space:continuwuity.org) and its rooms, including: -### [#conduwuit:puppygock.gay](https://matrix.to/#/#conduwuit:puppygock.gay) +### [#continuwuity:continuwuity.org](https://matrix.to/#/#continuwuity:continuwuity.org) -This room is for support and discussions about conduwuit. Ask questions, share +This room is for support and discussions about Continuwuity. Ask questions, share insights, and help each other out. -### [#conduwuit-offtopic:girlboss.ceo](https://matrix.to/#/#conduwuit-offtopic:girlboss.ceo) +### [#continuwuity-offtopic:continuwuity.org](https://matrix.to/#/#continuwuity-offtopic:continuwuity.org) For off-topic community conversations about any subject. While this room allows for a wide range of topics, the same CoC applies. Keep discussions respectful and inclusive, and avoid divisive subjects like country/world politics. General topics, such as world events, are welcome as long as they follow the CoC. -### [#conduwuit-dev:puppygock.gay](https://matrix.to/#/#conduwuit-dev:puppygock.gay) +### [#dev:continuwuity.org](https://matrix.to/#/#dev:continuwuity.org) -This room is dedicated to discussing active development of conduwuit. Posting +This room is dedicated to discussing active development of Continuwuity. Posting requires an elevated power level, which can be requested in one of the other rooms. Use this space to collaborate and innovate. @@ -90,4 +90,4 @@ comfortable doing that, then please send a DM to one of the moderators directly. Together, let’s build a community where everyone feels valued and respected. -— The conduwuit Moderation Team +— The Continuwuity Moderation Team diff --git a/docs/configuration.md b/docs/configuration.md index 0c670210..778e5c56 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,10 +1,10 @@ # Configuration -This chapter describes various ways to configure conduwuit. +This chapter describes various ways to configure Continuwuity. ## Basics -conduwuit uses a config file for the majority of the settings, but also supports +Continuwuity uses a config file for the majority of the settings, but also supports setting individual config options via commandline. Please refer to the [example config @@ -12,13 +12,13 @@ file](./configuration/examples.md#example-configuration) for all of those settings. The config file to use can be specified on the commandline when running -conduwuit by specifying the `-c`, `--config` flag. Alternatively, you can use +Continuwuity by specifying the `-c`, `--config` flag. Alternatively, you can use the environment variable `CONDUWUIT_CONFIG` to specify the config file to used. Conduit's environment variables are supported for backwards compatibility. ## Option commandline flag -conduwuit supports setting individual config options in TOML format from the +Continuwuity supports setting individual config options in TOML format from the `-O` / `--option` flag. For example, you can set your server name via `-O server_name=\"example.com\"`. @@ -33,7 +33,7 @@ string. This does not apply to options that take booleans or numbers: ## Execute commandline flag -conduwuit supports running admin commands on startup using the commandline +Continuwuity supports running admin commands on startup using the commandline argument `--execute`. The most notable use for this is to create an admin user on first startup. diff --git a/docs/deploying.md b/docs/deploying.md index 86277aba..be1bf736 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -1,3 +1,3 @@ # Deploying -This chapter describes various ways to deploy conduwuit. +This chapter describes various ways to deploy Continuwuity. diff --git a/docs/deploying/arch-linux.md b/docs/deploying/arch-linux.md index 7436e5bf..a14201e3 100644 --- a/docs/deploying/arch-linux.md +++ b/docs/deploying/arch-linux.md @@ -1,15 +1,3 @@ -# conduwuit for Arch Linux +# Continuwuity for Arch Linux -Currently conduwuit is only on the Arch User Repository (AUR). - -The conduwuit AUR packages are community maintained and are not maintained by -conduwuit development team, but the AUR package maintainers are in the Matrix -room. Please attempt to verify your AUR package's PKGBUILD file looks fine -before asking for support. - -- [conduwuit](https://aur.archlinux.org/packages/conduwuit) - latest tagged -conduwuit -- [conduwuit-git](https://aur.archlinux.org/packages/conduwuit-git) - latest git -conduwuit from `main` branch -- [conduwuit-bin](https://aur.archlinux.org/packages/conduwuit-bin) - latest -tagged conduwuit static binary +Continuwuity does not have any Arch Linux packages at this time. diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index 366f6999..694bd112 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -4,7 +4,7 @@ services: homeserver: ### If you already built the conduduwit image with 'docker build' or want to use the Docker Hub image, ### then you are ready to go. - image: girlbossceo/conduwuit:latest + image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped volumes: - db:/var/lib/conduwuit diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index 431cf2d4..8ff8076a 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -22,7 +22,7 @@ services: homeserver: ### If you already built the conduwuit image with 'docker build' or want to use a registry image, ### then you are ready to go. - image: girlbossceo/conduwuit:latest + image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped volumes: - db:/var/lib/conduwuit diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index 89118c74..842bf945 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -4,7 +4,7 @@ services: homeserver: ### If you already built the conduwuit image with 'docker build' or want to use the Docker Hub image, ### then you are ready to go. - image: girlbossceo/conduwuit:latest + image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped volumes: - db:/var/lib/conduwuit diff --git a/docs/deploying/docker-compose.yml b/docs/deploying/docker-compose.yml index ca33b5f5..ca56d0b0 100644 --- a/docs/deploying/docker-compose.yml +++ b/docs/deploying/docker-compose.yml @@ -4,7 +4,7 @@ services: homeserver: ### If you already built the conduwuit image with 'docker build' or want to use a registry image, ### then you are ready to go. - image: girlbossceo/conduwuit:latest + image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped ports: - 8448:6167 diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index bdbfb59c..bd6eff1d 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -1,31 +1,20 @@ -# conduwuit for Docker +# Continuwuity for Docker ## Docker -To run conduwuit with Docker you can either build the image yourself or pull it +To run Continuwuity with Docker you can either build the image yourself or pull it from a registry. ### Use a registry -OCI images for conduwuit are available in the registries listed below. +OCI images for Continuwuity are available in the registries listed below. -| Registry | Image | Size | Notes | -| --------------- | --------------------------------------------------------------- | ----------------------------- | ---------------------- | -| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:latest][gh] | ![Image Size][shield-latest] | Stable latest tagged image. | -| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:latest][gl] | ![Image Size][shield-latest] | Stable latest tagged image. | -| Docker Hub | [docker.io/girlbossceo/conduwuit:latest][dh] | ![Image Size][shield-latest] | Stable latest tagged image. | -| GitHub Registry | [ghcr.io/girlbossceo/conduwuit:main][gh] | ![Image Size][shield-main] | Stable main branch. | -| GitLab Registry | [registry.gitlab.com/conduwuit/conduwuit:main][gl] | ![Image Size][shield-main] | Stable main branch. | -| Docker Hub | [docker.io/girlbossceo/conduwuit:main][dh] | ![Image Size][shield-main] | Stable main branch. | +| Registry | Image | Notes | +| --------------- | --------------------------------------------------------------- | -----------------------| +| Forgejo Registry| [forgejo.ellis.link/continuwuation/continuwuity:latest][fj] | Latest tagged image. | +| Forgejo Registry| [forgejo.ellis.link/continuwuation/continuwuity:main][fj] | Main branch image. | -[dh]: https://hub.docker.com/r/girlbossceo/conduwuit -[gh]: https://github.com/girlbossceo/conduwuit/pkgs/container/conduwuit -[gl]: https://gitlab.com/conduwuit/conduwuit/container_registry/6369729 -[shield-latest]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/latest -[shield-main]: https://img.shields.io/docker/image-size/girlbossceo/conduwuit/main - -OCI image `.tar.gz` files are also hosted directly at when uploaded by CI with a -commit hash/revision or a tagged release: +[fj]: https://forgejo.ellis.link/continuwuation/-/packages/container/continuwuity Use @@ -52,11 +41,11 @@ or you can use [docker compose](#docker-compose). The `-d` flag lets the container run in detached mode. You may supply an optional `conduwuit.toml` config file, the example config can be found [here](../configuration/examples.md). You can pass in different env vars to -change config values on the fly. You can even configure conduwuit completely by +change config values on the fly. You can even configure Continuwuity completely by using env vars. For an overview of possible values, please take a look at the [`docker-compose.yml`](docker-compose.yml) file. -If you just want to test conduwuit for a short time, you can use the `--rm` +If you just want to test Continuwuity for a short time, you can use the `--rm` flag, which will clean up everything related to your container after you stop it. @@ -91,11 +80,11 @@ docker network create caddy After that, you can rename it so it matches `docker-compose.yml` and spin up the containers! -Additional info about deploying conduwuit can be found [here](generic.md). +Additional info about deploying Continuwuity can be found [here](generic.md). ### Build -Official conduwuit images are built using Nix's +Official Continuwuity images are built using Nix's [`buildLayeredImage`][nix-buildlayeredimage]. This ensures all OCI images are repeatable and reproducible by anyone, keeps the images lightweight, and can be built offline. @@ -104,13 +93,11 @@ This also ensures portability of our images because `buildLayeredImage` builds OCI images, not Docker images, and works with other container software. The OCI images are OS-less with only a very minimal environment of the `tini` -init system, CA certificates, and the conduwuit binary. This does mean there is +init system, CA certificates, and the Continuwuity binary. This does mean there is not a shell, but in theory you can get a shell by adding the necessary layers to the layered image. However it's very unlikely you will need a shell for any real troubleshooting. -The flake file for the OCI image definition is at [`nix/pkgs/oci-image/default.nix`][oci-image-def]. - To build an OCI image using Nix, the following outputs can be built: - `nix build -L .#oci-image` (default features, x86_64 glibc) - `nix build -L .#oci-image-x86_64-linux-musl` (default features, x86_64 musl) @@ -138,10 +125,10 @@ web. With the two provided files, [`docker-compose.for-traefik.yml`](docker-compose.for-traefik.yml) (or [`docker-compose.with-traefik.yml`](docker-compose.with-traefik.yml)) and [`docker-compose.override.yml`](docker-compose.override.yml), it is equally easy -to deploy and use conduwuit, with a little caveat. If you already took a look at +to deploy and use Continuwuity, with a little caveat. If you already took a look at the files, then you should have seen the `well-known` service, and that is the little caveat. Traefik is simply a proxy and loadbalancer and is not able to -serve any kind of content, but for conduwuit to federate, we need to either +serve any kind of content, but for Continuwuity to federate, we need to either expose ports `443` and `8448` or serve two endpoints `.well-known/matrix/client` and `.well-known/matrix/server`. @@ -153,4 +140,3 @@ those two files. See the [TURN](../turn.md) page. [nix-buildlayeredimage]: https://ryantm.github.io/nixpkgs/builders/images/dockertools/#ssec-pkgs-dockerTools-buildLayeredImage -[oci-image-def]: https://github.com/girlbossceo/conduwuit/blob/main/nix/pkgs/oci-image/default.nix diff --git a/docs/deploying/freebsd.md b/docs/deploying/freebsd.md index 65b40204..3764ffa8 100644 --- a/docs/deploying/freebsd.md +++ b/docs/deploying/freebsd.md @@ -1,5 +1,5 @@ -# conduwuit for FreeBSD +# Continuwuity for FreeBSD -conduwuit at the moment does not provide FreeBSD builds or have FreeBSD packaging, however conduwuit does build and work on FreeBSD using the system-provided RocksDB. +Continuwuity at the moment does not provide FreeBSD builds or have FreeBSD packaging, however Continuwuity does build and work on FreeBSD using the system-provided RocksDB. -Contributions for getting conduwuit packaged are welcome. +Contributions for getting Continuwuity packaged are welcome. diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index a07da560..46b9b439 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -2,11 +2,11 @@ > ### Getting help > -> If you run into any problems while setting up conduwuit, ask us in -> `#conduwuit:puppygock.gay` or [open an issue on -> GitHub](https://github.com/girlbossceo/conduwuit/issues/new). +> If you run into any problems while setting up Continuwuity, ask us in +> `#continuwuity:continuwuity.org` or [open an issue on +> Forgejo](https://forgejo.ellis.link/continuwuation/continuwuity/issues/new). -## Installing conduwuit +## Installing Continuwuity ### Static prebuilt binary @@ -14,12 +14,10 @@ You may simply download the binary that fits your machine architecture (x86_64 or aarch64). Run `uname -m` to see what you need. Prebuilt fully static musl binaries can be downloaded from the latest tagged -release [here](https://github.com/girlbossceo/conduwuit/releases/latest) or +release [here](https://forgejo.ellis.link/continuwuation/continuwuity/releases/latest) or `main` CI branch workflow artifact output. These also include Debian/Ubuntu packages. -Binaries are also available on my website directly at: - These can be curl'd directly from. `ci-bins` are CI workflow binaries by commit hash/revision, and `releases` are tagged releases. Sort by descending last modified for the latest. @@ -37,7 +35,7 @@ for performance. ### Compiling Alternatively, you may compile the binary yourself. We recommend using -Nix (or [Lix](https://lix.systems)) to build conduwuit as this has the most +Nix (or [Lix](https://lix.systems)) to build Continuwuity as this has the most guaranteed reproducibiltiy and easiest to get a build environment and output going. This also allows easy cross-compilation. @@ -51,35 +49,35 @@ If wanting to build using standard Rust toolchains, make sure you install: - `liburing-dev` on the compiling machine, and `liburing` on the target host - LLVM and libclang for RocksDB -You can build conduwuit using `cargo build --release --all-features` +You can build Continuwuity using `cargo build --release --all-features` -## Adding a conduwuit user +## Adding a Continuwuity user -While conduwuit can run as any user it is better to use dedicated users for +While Continuwuity can run as any user it is better to use dedicated users for different services. This also allows you to make sure that the file permissions are correctly set up. -In Debian, you can use this command to create a conduwuit user: +In Debian, you can use this command to create a Continuwuity user: ```bash -sudo adduser --system conduwuit --group --disabled-login --no-create-home +sudo adduser --system continuwuity --group --disabled-login --no-create-home ``` For distros without `adduser` (or where it's a symlink to `useradd`): ```bash -sudo useradd -r --shell /usr/bin/nologin --no-create-home conduwuit +sudo useradd -r --shell /usr/bin/nologin --no-create-home continuwuity ``` ## Forwarding ports in the firewall or the router Matrix's default federation port is port 8448, and clients must be using port 443. If you would like to use only port 443, or a different port, you will need to setup -delegation. conduwuit has config options for doing delegation, or you can configure +delegation. Continuwuity has config options for doing delegation, or you can configure your reverse proxy to manually serve the necessary JSON files to do delegation (see the `[global.well_known]` config section). -If conduwuit runs behind a router or in a container and has a different public +If Continuwuity runs behind a router or in a container and has a different public IP address than the host system these public ports need to be forwarded directly or indirectly to the port mentioned in the config. @@ -94,9 +92,9 @@ on the network level, consider something like NextDNS or Pi-Hole. ## Setting up a systemd service -Two example systemd units for conduwuit can be found +Two example systemd units for Continuwuity can be found [on the configuration page](../configuration/examples.md#debian-systemd-unit-file). -You may need to change the `ExecStart=` path to where you placed the conduwuit +You may need to change the `ExecStart=` path to where you placed the Continuwuity binary if it is not `/usr/bin/conduwuit`. On systems where rsyslog is used alongside journald (i.e. Red Hat-based distros @@ -114,9 +112,9 @@ and entering the following: ReadWritePaths=/path/to/custom/database/path ``` -## Creating the conduwuit configuration file +## Creating the Continuwuity configuration file -Now we need to create the conduwuit's config file in +Now we need to create the Continuwuity's config file in `/etc/conduwuit/conduwuit.toml`. The example config can be found at [conduwuit-example.toml](../configuration/examples.md). @@ -127,7 +125,7 @@ RocksDB is the only supported database backend. ## Setting the correct file permissions -If you are using a dedicated user for conduwuit, you will need to allow it to +If you are using a dedicated user for Continuwuity, you will need to allow it to read the config. To do that you can run this: ```bash @@ -139,7 +137,7 @@ If you use the default database path you also need to run this: ```bash sudo mkdir -p /var/lib/conduwuit/ -sudo chown -R conduwuit:conduwuit /var/lib/conduwuit/ +sudo chown -R continuwuity:continuwuity /var/lib/conduwuit/ sudo chmod 700 /var/lib/conduwuit/ ``` @@ -174,13 +172,13 @@ As we would prefer our users to use Caddy, we will not provide configuration fil You will need to reverse proxy everything under following routes: - `/_matrix/` - core Matrix C-S and S-S APIs -- `/_conduwuit/` - ad-hoc conduwuit routes such as `/local_user_count` and +- `/_conduwuit/` - ad-hoc Continuwuity routes such as `/local_user_count` and `/server_version` You can optionally reverse proxy the following individual routes: - `/.well-known/matrix/client` and `/.well-known/matrix/server` if using -conduwuit to perform delegation (see the `[global.well_known]` config section) -- `/.well-known/matrix/support` if using conduwuit to send the homeserver admin +Continuwuity to perform delegation (see the `[global.well_known]` config section) +- `/.well-known/matrix/support` if using Continuwuity to send the homeserver admin contact and support page (formerly known as MSC1929) - `/` if you would like to see `hewwo from conduwuit woof!` at the root @@ -200,7 +198,7 @@ header, making federation non-functional. If a workaround is found, feel free to If using Apache, you need to use `nocanon` in your `ProxyPass` directive to prevent httpd from messing with the `X-Matrix` header (note that Apache isn't very good as a general reverse proxy and we discourage the usage of it if you can). -If using Nginx, you need to give conduwuit the request URI using `$request_uri`, or like so: +If using Nginx, you need to give Continuwuity the request URI using `$request_uri`, or like so: - `proxy_pass http://127.0.0.1:6167$request_uri;` - `proxy_pass http://127.0.0.1:6167;` @@ -209,7 +207,7 @@ Nginx users need to increase `client_max_body_size` (default is 1M) to match ## You're done -Now you can start conduwuit with: +Now you can start Continuwuity with: ```bash sudo systemctl start conduwuit diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index 3c5b0e69..cf2c09e4 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -1,66 +1,33 @@ -# conduwuit for NixOS +# Continuwuity for NixOS -conduwuit can be acquired by Nix (or [Lix][lix]) from various places: +Continuwuity can be acquired by Nix (or [Lix][lix]) from various places: * The `flake.nix` at the root of the repo * The `default.nix` at the root of the repo -* From conduwuit's binary cache - -A community maintained NixOS package is available at [`conduwuit`](https://search.nixos.org/packages?channel=unstable&show=conduwuit&from=0&size=50&sort=relevance&type=packages&query=conduwuit) - -### Binary cache - -A binary cache for conduwuit that the CI/CD publishes to is available at the -following places (both are the same just different names): - -``` -https://attic.kennel.juneis.dog/conduit -conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= - -https://attic.kennel.juneis.dog/conduwuit -conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= -``` - -The binary caches were recreated some months ago due to attic issues. The old public -keys were: - -``` -conduit:Isq8FGyEC6FOXH6nD+BOeAA+bKp6X6UIbupSlGEPuOg= -conduwuit:lYPVh7o1hLu1idH4Xt2QHaRa49WRGSAqzcfFd94aOTw= -``` - -If needed, we have a binary cache on Cachix but it is only limited to 5GB: - -``` -https://conduwuit.cachix.org -conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= -``` - -If specifying a Git remote URL in your flake, you can use any remotes that -are specified on the README (the mirrors), such as the GitHub: `github:girlbossceo/conduwuit` +* From Continuwuity's binary cache ### NixOS module The `flake.nix` and `default.nix` do not currently provide a NixOS module (contributions welcome!), so [`services.matrix-conduit`][module] from Nixpkgs can be used to configure -conduwuit. +Continuwuity. ### Conduit NixOS Config Module and SQLite Beware! The [`services.matrix-conduit`][module] module defaults to SQLite as a database backend. -Conduwuit dropped SQLite support in favor of exclusively supporting the much faster RocksDB. +Continuwuity dropped SQLite support in favor of exclusively supporting the much faster RocksDB. Make sure that you are using the RocksDB backend before migrating! There is a [tool to migrate a Conduit SQLite database to RocksDB](https://github.com/ShadowJonathan/conduit_toolbox/). -If you want to run the latest code, you should get conduwuit from the `flake.nix` +If you want to run the latest code, you should get Continuwuity from the `flake.nix` or `default.nix` and set [`services.matrix-conduit.package`][package] -appropriately to use conduwuit instead of Conduit. +appropriately to use Continuwuity instead of Conduit. ### UNIX sockets -Due to the lack of a conduwuit NixOS module, when using the `services.matrix-conduit` module +Due to the lack of a Continuwuity NixOS module, when using the `services.matrix-conduit` module a workaround like the one below is necessary to use UNIX sockets. This is because the UNIX socket option does not exist in Conduit, and the module forcibly sets the `address` and `port` config options. @@ -84,13 +51,13 @@ disallows the namespace from accessing or creating UNIX sockets and has to be en systemd.services.conduit.serviceConfig.RestrictAddressFamilies = [ "AF_UNIX" ]; ``` -Even though those workarounds are feasible a conduwuit NixOS configuration module, developed and +Even though those workarounds are feasible a Continuwuity NixOS configuration module, developed and published by the community, would be appreciated. ### jemalloc and hardened profile -conduwuit uses jemalloc by default. This may interfere with the [`hardened.nix` profile][hardened.nix] -due to them using `scudo` by default. You must either disable/hide `scudo` from conduwuit, or +Continuwuity uses jemalloc by default. This may interfere with the [`hardened.nix` profile][hardened.nix] +due to them using `scudo` by default. You must either disable/hide `scudo` from Continuwuity, or disable jemalloc like so: ```nix diff --git a/docs/development.md b/docs/development.md index fa7519c0..1e344f41 100644 --- a/docs/development.md +++ b/docs/development.md @@ -4,9 +4,9 @@ Information about developing the project. If you are only interested in using it, you can safely ignore this page. If you plan on contributing, see the [contributor's guide](./contributing.md). -## conduwuit project layout +## Continuwuity project layout -conduwuit uses a collection of sub-crates, packages, or workspace members +Continuwuity uses a collection of sub-crates, packages, or workspace members that indicate what each general area of code is for. All of the workspace members are under `src/`. The workspace definition is at the top level / root `Cargo.toml`. @@ -14,11 +14,11 @@ members are under `src/`. The workspace definition is at the top level / root The crate names are generally self-explanatory: - `admin` is the admin room - `api` is the HTTP API, Matrix C-S and S-S endpoints, etc -- `core` is core conduwuit functionality like config loading, error definitions, +- `core` is core Continuwuity functionality like config loading, error definitions, global utilities, logging infrastructure, etc - `database` is RocksDB methods, helpers, RocksDB config, and general database definitions, utilities, or functions -- `macros` are conduwuit Rust [macros][macros] like general helper macros, logging +- `macros` are Continuwuity Rust [macros][macros] like general helper macros, logging and error handling macros, and [syn][syn] and [procedural macros][proc-macro] used for admin room commands and others - `main` is the "primary" sub-crate. This is where the `main()` function lives, @@ -35,7 +35,7 @@ if you truly find yourself needing to, we recommend reaching out to us in the Matrix room for discussions about it beforehand. The primary inspiration for this design was apart of hot reloadable development, -to support "conduwuit as a library" where specific parts can simply be swapped out. +to support "Continuwuity as a library" where specific parts can simply be swapped out. There is evidence Conduit wanted to go this route too as `axum` is technically an optional feature in Conduit, and can be compiled without the binary or axum library for handling inbound web requests; but it was never completed or worked. @@ -68,10 +68,10 @@ do this if Rust supported workspace-level features to begin with. ## List of forked dependencies -During conduwuit development, we have had to fork +During Continuwuity development, we have had to fork some dependencies to support our use-cases in some areas. This ranges from things said upstream project won't accept for any reason, faster-paced -development (unresponsive or slow upstream), conduwuit-specific usecases, or +development (unresponsive or slow upstream), Continuwuity-specific usecases, or lack of time to upstream some things. - [ruma/ruma][1]: - various performance @@ -84,7 +84,7 @@ builds seem to be broken on upstream, fixes some broken/suspicious code in places, additional safety measures, and support redzones for Valgrind - [zyansheep/rustyline-async][4]: - tab completion callback and -`CTRL+\` signal quit event for conduwuit console CLI +`CTRL+\` signal quit event for Continuwuity console CLI - [rust-rocksdb/rust-rocksdb][5]: - [`@zaidoon1`][8]'s fork has quicker updates, more up to date dependencies, etc. Our fork fixes musl build @@ -97,7 +97,7 @@ alongside other logging/metrics things ## Debugging with `tokio-console` [`tokio-console`][7] can be a useful tool for debugging and profiling. To make a -`tokio-console`-enabled build of conduwuit, enable the `tokio_console` feature, +`tokio-console`-enabled build of Continuwuity, enable the `tokio_console` feature, disable the default `release_max_log_level` feature, and set the `--cfg tokio_unstable` flag to enable experimental tokio APIs. A build might look like this: @@ -109,7 +109,7 @@ RUSTFLAGS="--cfg tokio_unstable" cargo +nightly build \ --features=systemd,element_hacks,gzip_compression,brotli_compression,zstd_compression,tokio_console ``` -You will also need to enable the `tokio_console` config option in conduwuit when +You will also need to enable the `tokio_console` config option in Continuwuity when starting it. This was due to tokio-console causing gradual memory leak/usage if left enabled. diff --git a/docs/introduction.md b/docs/introduction.md index 9d3a294a..d193f7c7 100644 --- a/docs/introduction.md +++ b/docs/introduction.md @@ -1,4 +1,4 @@ -# conduwuit +# Continuwuity {{#include ../README.md:catchphrase}} @@ -8,7 +8,7 @@ - [Deployment options](deploying.md) -If you want to connect an appservice to conduwuit, take a look at the +If you want to connect an appservice to Continuwuity, take a look at the [appservices documentation](appservices.md). #### How can I contribute? diff --git a/docs/maintenance.md b/docs/maintenance.md index 5c8c853a..b85a1971 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -1,14 +1,14 @@ -# Maintaining your conduwuit setup +# Maintaining your Continuwuity setup ## Moderation -conduwuit has moderation through admin room commands. "binary commands" (medium +Continuwuity has moderation through admin room commands. "binary commands" (medium priority) and an admin API (low priority) is planned. Some moderation-related config options are available in the example config such as "global ACLs" and blocking media requests to certain servers. See the example config for the moderation config options under the "Moderation / Privacy / Security" section. -conduwuit has moderation admin commands for: +Continuwuity has moderation admin commands for: - managing room aliases (`!admin rooms alias`) - managing room directory (`!admin rooms directory`) @@ -36,7 +36,7 @@ each object being newline delimited. An example of doing this is: ## Database (RocksDB) Generally there is very little you need to do. [Compaction][rocksdb-compaction] -is ran automatically based on various defined thresholds tuned for conduwuit to +is ran automatically based on various defined thresholds tuned for Continuwuity to be high performance with the least I/O amplifcation or overhead. Manually running compaction is not recommended, or compaction via a timer, due to creating unnecessary I/O amplification. RocksDB is built with io_uring support @@ -50,7 +50,7 @@ Some RocksDB settings can be adjusted such as the compression method chosen. See the RocksDB section in the [example config](configuration/examples.md). btrfs users have reported that database compression does not need to be disabled -on conduwuit as the filesystem already does not attempt to compress. This can be +on Continuwuity as the filesystem already does not attempt to compress. This can be validated by using `filefrag -v` on a `.SST` file in your database, and ensure the `physical_offset` matches (no filesystem compression). It is very important to ensure no additional filesystem compression takes place as this can render @@ -70,7 +70,7 @@ they're server logs or database logs, however they are critical RocksDB files related to WAL tracking. The only safe files that can be deleted are the `LOG` files (all caps). These -are the real RocksDB telemetry/log files, however conduwuit has already +are the real RocksDB telemetry/log files, however Continuwuity has already configured to only store up to 3 RocksDB `LOG` files due to generall being useless for average users unless troubleshooting something low-level. If you would like to store nearly none at all, see the `rocksdb_max_log_files` @@ -88,7 +88,7 @@ still be joined together. To restore a backup from an online RocksDB backup: -- shutdown conduwuit +- shutdown Continuwuity - create a new directory for merging together the data - in the online backup created, copy all `.sst` files in `$DATABASE_BACKUP_PATH/shared_checksum` to your new directory @@ -99,9 +99,9 @@ To restore a backup from an online RocksDB backup: if you have multiple) to your new directory - set your `database_path` config option to your new directory, or replace your old one with the new one you crafted -- start up conduwuit again and it should open as normal +- start up Continuwuity again and it should open as normal -If you'd like to do an offline backup, shutdown conduwuit and copy your +If you'd like to do an offline backup, shutdown Continuwuity and copy your `database_path` directory elsewhere. This can be restored with no modifications needed. @@ -110,7 +110,7 @@ directory. ## Media -Media still needs various work, however conduwuit implements media deletion via: +Media still needs various work, however Continuwuity implements media deletion via: - MXC URI or Event ID (unencrypted and attempts to find the MXC URI in the event) @@ -118,17 +118,17 @@ event) - Delete remote media in the past `N` seconds/minutes via filesystem metadata on the file created time (`btime`) or file modified time (`mtime`) -See the `!admin media` command for further information. All media in conduwuit +See the `!admin media` command for further information. All media in Continuwuity is stored at `$DATABASE_DIR/media`. This will be configurable soon. If you are finding yourself needing extensive granular control over media, we recommend looking into [Matrix Media -Repo](https://github.com/t2bot/matrix-media-repo). conduwuit intends to +Repo](https://github.com/t2bot/matrix-media-repo). Continuwuity intends to implement various utilities for media, but MMR is dedicated to extensive media management. Built-in S3 support is also planned, but for now using a "S3 filesystem" on -`media/` works. conduwuit also sends a `Cache-Control` header of 1 year and +`media/` works. Continuwuity also sends a `Cache-Control` header of 1 year and immutable for all media requests (download and thumbnail) to reduce unnecessary media requests from browsers, reduce bandwidth usage, and reduce load. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index d25c9762..37b1a5cd 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,9 +1,9 @@ -# Troubleshooting conduwuit +# Troubleshooting Continuwuity > ## Docker users ⚠️ > > Docker is extremely UX unfriendly. Because of this, a ton of issues or support -> is actually Docker support, not conduwuit support. We also cannot document the +> is actually Docker support, not Continuwuity support. We also cannot document the > ever-growing list of Docker issues here. > > If you intend on asking for support and you are using Docker, **PLEASE** @@ -13,14 +13,14 @@ > If there are things like Compose file issues or Dockerhub image issues, those > can still be mentioned as long as they're something we can fix. -## conduwuit and Matrix issues +## Continuwuity and Matrix issues #### Lost access to admin room You can reinvite yourself to the admin room through the following methods: -- Use the `--execute "users make_user_admin "` conduwuit binary +- Use the `--execute "users make_user_admin "` Continuwuity binary argument once to invite yourslf to the admin room on startup -- Use the conduwuit console/CLI to run the `users make_user_admin` command +- Use the Continuwuity console/CLI to run the `users make_user_admin` command - Or specify the `emergency_password` config option to allow you to temporarily log into the server account (`@conduit`) from a web client @@ -29,12 +29,12 @@ log into the server account (`@conduit`) from a web client #### Potential DNS issues when using Docker Docker has issues with its default DNS setup that may cause DNS to not be -properly functional when running conduwuit, resulting in federation issues. The +properly functional when running Continuwuity, resulting in federation issues. The symptoms of this have shown in excessively long room joins (30+ minutes) from very long DNS timeouts, log entries of "mismatching responding nameservers", and/or partial or non-functional inbound/outbound federation. -This is **not** a conduwuit issue, and is purely a Docker issue. It is not +This is **not** a Continuwuity issue, and is purely a Docker issue. It is not sustainable for heavy DNS activity which is normal for Matrix federation. The workarounds for this are: - Use DNS over TCP via the config option `query_over_tcp_only = true` @@ -64,7 +64,7 @@ very computationally expensive, and is extremely susceptible to denial of service, especially on Matrix. Many servers also strangely have broken DNSSEC setups and will result in non-functional federation. -conduwuit cannot provide a "works-for-everyone" Unbound DNS setup guide, but +Continuwuity cannot provide a "works-for-everyone" Unbound DNS setup guide, but the [official Unbound tuning guide][unbound-tuning] and the [Unbound Arch Linux wiki page][unbound-arch] may be of interest. Disabling DNSSEC on Unbound is commenting out trust-anchors config options and removing the `validator` module. @@ -75,9 +75,9 @@ high load, and we have identified its DNS caching to not be very effective. dnsmasq can possibly work, but it does **not** support TCP fallback which can be problematic when receiving large DNS responses such as from large SRV records. If you still want to use dnsmasq, make sure you **disable** `dns_tcp_fallback` -in conduwuit config. +in Continuwuity config. -Raising `dns_cache_entries` in conduwuit config from the default can also assist +Raising `dns_cache_entries` in Continuwuity config from the default can also assist in DNS caching, but a full-fledged external caching resolver is better and more reliable. @@ -97,7 +97,7 @@ If your database is corrupted *and* is failing to start (e.g. checksum mismatch), it may be recoverable but careful steps must be taken, and there is no guarantee it may be recoverable. -The first thing that can be done is launching conduwuit with the +The first thing that can be done is launching Continuwuity with the `rocksdb_repair` config option set to true. This will tell RocksDB to attempt to repair itself at launch. If this does not work, disable the option and continue reading. @@ -109,7 +109,7 @@ RocksDB has the following recovery modes: - `PointInTime` - `SkipAnyCorruptedRecord` -By default, conduwuit uses `TolerateCorruptedTailRecords` as generally these may +By default, Continuwuity uses `TolerateCorruptedTailRecords` as generally these may be due to bad federation and we can re-fetch the correct data over federation. The RocksDB default is `PointInTime` which will attempt to restore a "snapshot" of the data when it was last known to be good. This data can be either a few @@ -126,12 +126,12 @@ if `PointInTime` does not work as a last ditch effort. With this in mind: -- First start conduwuit with the `PointInTime` recovery method. See the [example +- First start Continuwuity with the `PointInTime` recovery method. See the [example config](configuration/examples.md) for how to do this using `rocksdb_recovery_mode` - If your database successfully opens, clients are recommended to clear their client cache to account for the rollback -- Leave your conduwuit running in `PointInTime` for at least 30-60 minutes so as +- Leave your Continuwuity running in `PointInTime` for at least 30-60 minutes so as much possible corruption is restored - If all goes will, you should be able to restore back to using `TolerateCorruptedTailRecords` and you have successfully recovered your database @@ -144,14 +144,14 @@ Various debug commands can be found in `!admin debug`. #### Debug/Trace log level -conduwuit builds without debug or trace log levels at compile time by default +Continuwuity builds without debug or trace log levels at compile time by default for substantial performance gains in CPU usage and improved compile times. If you need to access debug/trace log levels, you will need to build without the `release_max_log_level` feature or use our provided static debug binaries. #### Changing log level dynamically -conduwuit supports changing the tracing log environment filter on-the-fly using +Continuwuity supports changing the tracing log environment filter on-the-fly using the admin command `!admin debug change-log-level `. This accepts a string **without quotes** the same format as the `log` config option. @@ -168,7 +168,7 @@ load, simply pass the `--reset` flag. #### Pinging servers -conduwuit can ping other servers using `!admin debug ping `. This takes +Continuwuity can ping other servers using `!admin debug ping `. This takes a server name and goes through the server discovery process and queries `/_matrix/federation/v1/version`. Errors are outputted. @@ -180,12 +180,12 @@ bandwidth and computationally. #### Allocator memory stats When using jemalloc with jemallocator's `stats` feature (`--enable-stats`), you -can see conduwuit's high-level allocator stats by using +can see Continuwuity's high-level allocator stats by using `!admin server memory-usage` at the bottom. If you are a developer, you can also view the raw jemalloc statistics with `!admin debug memory-stats`. Please note that this output is extremely large -which may only be visible in the conduwuit console CLI due to PDU size limits, +which may only be visible in the Continuwuity console CLI due to PDU size limits, and is not easy for non-developers to understand. [unbound-tuning]: https://unbound.docs.nlnetlabs.nl/en/latest/topics/core/performance.html diff --git a/docs/turn.md b/docs/turn.md index 287f2545..5dba823c 100644 --- a/docs/turn.md +++ b/docs/turn.md @@ -1,6 +1,6 @@ # Setting up TURN/STURN -In order to make or receive calls, a TURN server is required. conduwuit suggests +In order to make or receive calls, a TURN server is required. Continuwuity suggests using [Coturn](https://github.com/coturn/coturn) for this purpose, which is also available as a Docker image. @@ -17,9 +17,9 @@ realm= A common way to generate a suitable alphanumeric secret key is by using `pwgen -s 64 1`. -These same values need to be set in conduwuit. See the [example +These same values need to be set in Continuwuity. See the [example config](configuration/examples.md) in the TURN section for configuring these and -restart conduwuit after. +restart Continuwuity after. `turn_secret` or a path to `turn_secret_file` must have a value of your coturn `static-auth-secret`, or use `turn_username` and `turn_password` @@ -34,7 +34,7 @@ If you are using TURN over TLS, you can replace `turn:` with `turns:` in the TURN over TLS. This is highly recommended. If you need unauthenticated access to the TURN URIs, or some clients may be -having trouble, you can enable `turn_guest_access` in conduwuit which disables +having trouble, you can enable `turn_guest_access` in Continuwuity which disables authentication for the TURN URI endpoint `/_matrix/client/v3/voip/turnServer` ### Run From fbd404fa84647c1e4794852028a1b1e5cc27c97d Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 21 Apr 2025 00:19:08 +0100 Subject: [PATCH 1724/2091] docs: Update docker documentation --- docs/deploying/docker.md | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index bd6eff1d..08a0dc4f 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -84,26 +84,28 @@ Additional info about deploying Continuwuity can be found [here](generic.md). ### Build -Official Continuwuity images are built using Nix's -[`buildLayeredImage`][nix-buildlayeredimage]. This ensures all OCI images are -repeatable and reproducible by anyone, keeps the images lightweight, and can be -built offline. +Official Continuwuity images are built using **Docker Buildx** and the Dockerfile found at [`docker/Dockerfile`][dockerfile-path]. This approach uses common Docker tooling and enables multi-platform builds efficiently. -This also ensures portability of our images because `buildLayeredImage` builds -OCI images, not Docker images, and works with other container software. +The resulting images are broadly compatible with Docker and other container runtimes like Podman or containerd. -The OCI images are OS-less with only a very minimal environment of the `tini` -init system, CA certificates, and the Continuwuity binary. This does mean there is -not a shell, but in theory you can get a shell by adding the necessary layers -to the layered image. However it's very unlikely you will need a shell for any -real troubleshooting. +The images *do not contain a shell*. They contain only the Continuwuity binary, required libraries, TLS certificates and metadata. Please refer to the [`docker/Dockerfile`][dockerfile-path] for the specific details of the image composition. -To build an OCI image using Nix, the following outputs can be built: -- `nix build -L .#oci-image` (default features, x86_64 glibc) -- `nix build -L .#oci-image-x86_64-linux-musl` (default features, x86_64 musl) -- `nix build -L .#oci-image-aarch64-linux-musl` (default features, aarch64 musl) -- `nix build -L .#oci-image-x86_64-linux-musl-all-features` (all features, x86_64 musl) -- `nix build -L .#oci-image-aarch64-linux-musl-all-features` (all features, aarch64 musl) +To build an image locally using Docker Buildx, you can typically run a command like: + +```bash +# Build for the current platform and load into the local Docker daemon +docker buildx build --load --tag continuwuity:latest -f docker/Dockerfile . + +# Example: Build for specific platforms and push to a registry. +# docker buildx build --platform linux/amd64,linux/arm64 --tag registry.io/org/continuwuity:latest -f docker/Dockerfile . --push + +# Example: Build binary optimized for the current CPU +# docker buildx build --load --tag continuwuity:latest --build-arg TARGET_CPU=native -f docker/Dockerfile . +``` + +Refer to the Docker Buildx documentation for more advanced build options. + +[dockerfile-path]: ../../docker/Dockerfile ### Run From c68378ffe34574d8e876e98ec6b961f08a0041a8 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 21 Apr 2025 00:38:47 +0100 Subject: [PATCH 1725/2091] docs: Update 'Try it out' section --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index deaed364..9b8f142b 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,9 @@ Continuwuity aims to: ### Can I try it out? -Not right now. We've still got work to do! +Check out the [documentation](introduction) for installation instructions. +There are currently no open registration Continuwuity instances available. ### What are we working on? @@ -111,3 +112,4 @@ When incorporating code from other forks: [continuwuity]: https://forgejo.ellis.link/continuwuation/continuwuity +F From c7ac2483a968560b922ae7d37adc564d828ed9d1 Mon Sep 17 00:00:00 2001 From: n Date: Tue, 22 Apr 2025 01:27:50 +0000 Subject: [PATCH 1726/2091] Fix offtopic room link Signed-off-by: n --- docs/conduwuit_coc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/conduwuit_coc.md b/docs/conduwuit_coc.md index 19262765..9a084150 100644 --- a/docs/conduwuit_coc.md +++ b/docs/conduwuit_coc.md @@ -57,7 +57,7 @@ including: This room is for support and discussions about Continuwuity. Ask questions, share insights, and help each other out. -### [#continuwuity-offtopic:continuwuity.org](https://matrix.to/#/#continuwuity-offtopic:continuwuity.org) +### [#offtopic:continuwuity.org](https://matrix.to/#/#offtopic:continuwuity.org) For off-topic community conversations about any subject. While this room allows for a wide range of topics, the same CoC applies. Keep discussions respectful From 0c302f31371aedbd313228004f1971b9e0a5d64c Mon Sep 17 00:00:00 2001 From: nex Date: Tue, 22 Apr 2025 01:33:09 +0000 Subject: [PATCH 1727/2091] Don't re-build images for docs changes Ironically, this will trigger a rebuild anyway --- .forgejo/workflows/release-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index adf70594..2cb6a329 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -11,6 +11,7 @@ on: - 'renovate.json' - 'debian/**' - 'docker/**' + - 'docs/**' # Allows you to run this workflow manually from the Actions tab workflow_dispatch: From 7beff25d3d3d59cba3a8b634a92e9a562267b1fc Mon Sep 17 00:00:00 2001 From: Nyx Tutt Date: Mon, 21 Apr 2025 20:45:05 -0500 Subject: [PATCH 1728/2091] Update welcome message --- src/service/admin/grant.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 6780b7ae..2d90ea52 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -126,7 +126,7 @@ pub async fn make_user_admin(&self, user_id: &UserId) -> Result { if self.services.server.config.admin_room_notices { let welcome_message = String::from( - "## Thank you for trying out conduwuit!\n\nconduwuit is technically a hard fork of Conduit, which is in Beta. The Beta status initially was inherited from Conduit, however overtime this Beta status is rapidly becoming less and less relevant as our codebase significantly diverges more and more. conduwuit is quite stable and very usable as a daily driver and for a low-medium sized homeserver. There is still a lot of more work to be done, but it is in a far better place than the project was in early 2024.\n\nHelpful links:\n> GitHub Repo: https://github.com/girlbossceo/conduwuit\n> Documentation: https://conduwuit.puppyirl.gay/\n> Report issues: https://github.com/girlbossceo/conduwuit/issues\n\nFor a list of available commands, send the following message in this room: `!admin --help`\n\nHere are some rooms you can join (by typing the command into your client) -\n\nconduwuit space: `/join #conduwuit-space:puppygock.gay`\nconduwuit main room (Ask questions and get notified on updates): `/join #conduwuit:puppygock.gay`\nconduwuit offtopic room: `/join #conduwuit-offtopic:puppygock.gay`", + "## Thank you for trying out Continuwuity!\n\nContinuwuity is a hard fork of conduwuit, which is also a hard fork of Conduit, currently in Beta. The Beta status initially was inherited from Conduit, however overtime this Beta status is rapidly becoming less and less relevant as our codebase significantly diverges more and more. Continuwuity is quite stable and very usable as a daily driver and for a low-medium sized homeserver. There is still a lot of more work to be done, but it is in a far better place than the project was in early 2024.\n\nHelpful links:\n> Source code: https://forgejo.ellis.link/continuwuation/continuwuity\n> Documentation: https://continuwuity.org/\n> Report issues: https://forgejo.ellis.link/continuwuation/continuwuity/issues\n\nFor a list of available commands, send the following message in this room: `!admin --help`\n\nHere are some rooms you can join (by typing the command into your client) -\n\nContinuwuity space: `/join #space:continuwuity.org`\nContinuwuity main room (Ask questions and get notified on updates): `/join #continuwuity:continuwuity.org`\nContinuwuity offtopic room: `/join #offtopic:continuwuity.org`", ); // Send welcome message From 66e8cd8908a7a96e1f929c948ccef0238a790a60 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 21 Apr 2025 15:19:30 +0100 Subject: [PATCH 1729/2091] docs: Tone down the docker warning --- docs/troubleshooting.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 37b1a5cd..81e90636 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -2,16 +2,9 @@ > ## Docker users ⚠️ > -> Docker is extremely UX unfriendly. Because of this, a ton of issues or support -> is actually Docker support, not Continuwuity support. We also cannot document the -> ever-growing list of Docker issues here. -> -> If you intend on asking for support and you are using Docker, **PLEASE** -> triple validate your issues are **NOT** because you have a misconfiguration in -> your Docker setup. -> -> If there are things like Compose file issues or Dockerhub image issues, those -> can still be mentioned as long as they're something we can fix. +> Docker can be difficult to use and debug. It's common for Docker +> misconfigurations to cause issues, particularly with networking and permissions. +> Please check that your issues are not due to problems with your Docker setup. ## Continuwuity and Matrix issues From 81f8151acabc3f0f824c7c04b47e31ed61422a8b Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 21 Apr 2025 16:15:31 +0100 Subject: [PATCH 1730/2091] docs: Add matrix rooms to README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9b8f142b..bf4f5613 100644 --- a/README.md +++ b/README.md @@ -106,10 +106,10 @@ When incorporating code from other forks: #### Contact - +Join our [Matrix room](https://matrix.to/#/#continuwuity:continuwuity.org) and [space](https://matrix.to/#/#space:continuwuity.org) to chat with us about the project! [continuwuity]: https://forgejo.ellis.link/continuwuation/continuwuity -F + From 1d42b88f501fb2c5115f2f4d569cd9b07171fcb6 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 21 Apr 2025 16:26:38 +0100 Subject: [PATCH 1731/2091] docs: Update Docker DNS troubleshooting section --- docs/troubleshooting.md | 46 ++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 81e90636..d84dbc7a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,6 +1,6 @@ # Troubleshooting Continuwuity -> ## Docker users ⚠️ +> **Docker users ⚠️** > > Docker can be difficult to use and debug. It's common for Docker > misconfigurations to cause issues, particularly with networking and permissions. @@ -8,9 +8,10 @@ ## Continuwuity and Matrix issues -#### Lost access to admin room +### Lost access to admin room You can reinvite yourself to the admin room through the following methods: + - Use the `--execute "users make_user_admin "` Continuwuity binary argument once to invite yourslf to the admin room on startup - Use the Continuwuity console/CLI to run the `users make_user_admin` command @@ -19,22 +20,29 @@ log into the server account (`@conduit`) from a web client ## General potential issues -#### Potential DNS issues when using Docker +### Potential DNS issues when using Docker -Docker has issues with its default DNS setup that may cause DNS to not be -properly functional when running Continuwuity, resulting in federation issues. The -symptoms of this have shown in excessively long room joins (30+ minutes) from -very long DNS timeouts, log entries of "mismatching responding nameservers", +Docker's DNS setup for containers in a non-default network intercepts queries to +enable resolving of container hostnames to IP addresses. However, due to +performance issues with Docker's built-in resolver, this can cause DNS queries +to take a long time to resolve, resulting in federation issues. + +This is particularly common with Docker Compose, as custom networks are easily +created and configured. + +Symptoms of this include excessively long room joins (30+ minutes) from very +long DNS timeouts, log entries of "mismatching responding nameservers", and/or partial or non-functional inbound/outbound federation. -This is **not** a Continuwuity issue, and is purely a Docker issue. It is not -sustainable for heavy DNS activity which is normal for Matrix federation. The -workarounds for this are: -- Use DNS over TCP via the config option `query_over_tcp_only = true` -- Don't use Docker's default DNS setup and instead allow the container to use -and communicate with your host's DNS servers (host's `/etc/resolv.conf`) +This is not a bug in continuwuity. Docker's default DNS resolver is not suitable +for heavy DNS activity, which is normal for federated protocols like Matrix. -#### DNS No connections available error message +Workarounds: + +- Use DNS over TCP via the config option `query_over_tcp_only = true` +- Bypass Docker's default DNS setup and instead allow the container to use and communicate with your host's DNS servers. Typically, this can be done by mounting the host's `/etc/resolv.conf`. + +### DNS No connections available error message If you receive spurious amounts of error logs saying "DNS No connections available", this is due to your DNS server (servers from `/etc/resolv.conf`) @@ -84,7 +92,7 @@ reliability at a slight performance cost due to TCP overhead. ## RocksDB / database issues -#### Database corruption +### Database corruption If your database is corrupted *and* is failing to start (e.g. checksum mismatch), it may be recoverable but careful steps must be taken, and there is @@ -135,14 +143,14 @@ Note that users should not really be debugging things. If you find yourself debugging and find the issue, please let us know and/or how we can fix it. Various debug commands can be found in `!admin debug`. -#### Debug/Trace log level +### Debug/Trace log level Continuwuity builds without debug or trace log levels at compile time by default for substantial performance gains in CPU usage and improved compile times. If you need to access debug/trace log levels, you will need to build without the `release_max_log_level` feature or use our provided static debug binaries. -#### Changing log level dynamically +### Changing log level dynamically Continuwuity supports changing the tracing log environment filter on-the-fly using the admin command `!admin debug change-log-level `. This accepts @@ -159,7 +167,7 @@ load, simply pass the `--reset` flag. `!admin debug change-log-level --reset` -#### Pinging servers +### Pinging servers Continuwuity can ping other servers using `!admin debug ping `. This takes a server name and goes through the server discovery process and queries @@ -170,7 +178,7 @@ server performance on either side as that endpoint is completely unauthenticated and simply fetches a string on a static JSON endpoint. It is very low cost both bandwidth and computationally. -#### Allocator memory stats +### Allocator memory stats When using jemalloc with jemallocator's `stats` feature (`--enable-stats`), you can see Continuwuity's high-level allocator stats by using From 0307238bf890a3a0249cd7d91c0ae3728664fe4b Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 22 Apr 2025 14:29:02 +0100 Subject: [PATCH 1732/2091] docs: Work around DNS issues in example compose files --- docs/deploying/docker-compose.for-traefik.yml | 1 + docs/deploying/docker-compose.with-caddy.yml | 1 + docs/deploying/docker-compose.with-traefik.yml | 1 + 3 files changed, 3 insertions(+) diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index 694bd112..57b124c7 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -8,6 +8,7 @@ services: restart: unless-stopped volumes: - db:/var/lib/conduwuit + - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. #- ./conduwuit.toml:/etc/conduwuit.toml networks: - proxy diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index 8ff8076a..ac4fb1ff 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -26,6 +26,7 @@ services: restart: unless-stopped volumes: - db:/var/lib/conduwuit + - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. #- ./conduwuit.toml:/etc/conduwuit.toml environment: CONDUWUIT_SERVER_NAME: example.com # EDIT THIS diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index 842bf945..86ad9cb6 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -8,6 +8,7 @@ services: restart: unless-stopped volumes: - db:/var/lib/conduwuit + - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. #- ./conduwuit.toml:/etc/conduwuit.toml networks: - proxy From 4fbecca2d31f5cbc8992a6ab854910e6afddeea6 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 23 Apr 2025 17:39:26 +0100 Subject: [PATCH 1733/2091] Add well-known/matrix/support --- docs/static/support | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 docs/static/support diff --git a/docs/static/support b/docs/static/support new file mode 100644 index 00000000..3fee0187 --- /dev/null +++ b/docs/static/support @@ -0,0 +1,24 @@ +{ + "contacts": [ + { + "email_address": "security@continuwuity.org", + "role": "m.role.security" + }, + { + "matrix_id": "@tom:continuwuity.org", + "email_address": "tom@continuwuity.org", + "role": "m.role.admin" + }, + { + "matrix_id": "@jade:continuwuity.org", + "email_address": "jade@continuwuity.org", + "role": "m.role.admin" + }, + { + "matrix_id": "@nex:continuwuity.org", + "email_address": "nex@continuwuity.org", + "role": "m.role.admin" + } + ], + "support_page": "https://continuwuity.org/introduction#contact" +} \ No newline at end of file From 8f21403796078d586746829eba1701b9810fd462 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 23 Apr 2025 17:39:36 +0100 Subject: [PATCH 1734/2091] Use any runner for docs --- .forgejo/workflows/documentation.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/documentation.yml b/.forgejo/workflows/documentation.yml index c84c566b..55f25058 100644 --- a/.forgejo/workflows/documentation.yml +++ b/.forgejo/workflows/documentation.yml @@ -16,7 +16,7 @@ concurrency: jobs: docs: name: Build and Deploy Documentation - runs-on: not-nexy + runs-on: ubuntu-latest steps: - name: Sync repository @@ -41,6 +41,7 @@ jobs: # Copy the Matrix .well-known files cp ./docs/static/server ./public/.well-known/matrix/server cp ./docs/static/client ./public/.well-known/matrix/client + cp ./docs/static/client ./public/.well-known/matrix/support cp ./docs/static/announcements.json ./public/.well-known/continuwuity/announcements cp ./docs/static/announcements.schema.json ./public/schema/announcements.schema.json # Copy the custom headers file From 3eb4ee7af1b8980009bcbe0f6c0585a9d235ffb6 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 23 Apr 2025 19:30:20 +0100 Subject: [PATCH 1735/2091] Change tom's email address --- docs/static/support | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/static/support b/docs/static/support index 3fee0187..6b7a9860 100644 --- a/docs/static/support +++ b/docs/static/support @@ -6,7 +6,7 @@ }, { "matrix_id": "@tom:continuwuity.org", - "email_address": "tom@continuwuity.org", + "email_address": "tom@tcpip.uk", "role": "m.role.admin" }, { From f791dc69185b853139ed7f6705ad6d095b398e33 Mon Sep 17 00:00:00 2001 From: Nyx Tutt Date: Tue, 22 Apr 2025 07:56:42 -0500 Subject: [PATCH 1736/2091] docs: Rename in more places --- docs/deploying/docker-compose.for-traefik.yml | 6 ++--- docs/deploying/docker-compose.override.yml | 6 ++--- docs/deploying/docker-compose.with-caddy.yml | 4 ++-- .../deploying/docker-compose.with-traefik.yml | 10 ++++----- docs/deploying/docker-compose.yml | 6 ++--- docs/deploying/kubernetes.md | 8 +++---- docs/development/hot_reload.md | 22 +++++++++---------- src/admin/server/mod.rs | 2 +- 8 files changed, 32 insertions(+), 32 deletions(-) diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index 57b124c7..04142e0c 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -1,4 +1,4 @@ -# conduwuit - Behind Traefik Reverse Proxy +# Continuwuity - Behind Traefik Reverse Proxy services: homeserver: @@ -36,14 +36,14 @@ services: server=your.server.name.example:443 } #cpuset: "0-4" # Uncomment to limit to specific CPU cores - ulimits: # conduwuit uses quite a few file descriptors, and on some systems it defaults to 1024, so you can tell docker to increase it + ulimits: # Continuwuity uses quite a few file descriptors, and on some systems it defaults to 1024, so you can tell docker to increase it nofile: soft: 1048567 hard: 1048567 ### Uncomment if you want to use your own Element-Web App. ### Note: You need to provide a config.json for Element and you also need a second - ### Domain or Subdomain for the communication between Element and conduwuit + ### Domain or Subdomain for the communication between Element and Continuwuity ### Config-Docs: https://github.com/vector-im/element-web/blob/develop/docs/config.md # element-web: # image: vectorim/element-web:latest diff --git a/docs/deploying/docker-compose.override.yml b/docs/deploying/docker-compose.override.yml index a343eeee..ec82fac3 100644 --- a/docs/deploying/docker-compose.override.yml +++ b/docs/deploying/docker-compose.override.yml @@ -1,4 +1,4 @@ -# conduwuit - Traefik Reverse Proxy Labels +# Continuwuity - Traefik Reverse Proxy Labels services: homeserver: @@ -6,7 +6,7 @@ services: - "traefik.enable=true" - "traefik.docker.network=proxy" # Change this to the name of your Traefik docker proxy network - - "traefik.http.routers.to-conduwuit.rule=Host(`.`)" # Change to the address on which conduwuit is hosted + - "traefik.http.routers.to-conduwuit.rule=Host(`.`)" # Change to the address on which Continuwuity is hosted - "traefik.http.routers.to-conduwuit.tls=true" - "traefik.http.routers.to-conduwuit.tls.certresolver=letsencrypt" - "traefik.http.routers.to-conduwuit.middlewares=cors-headers@docker" @@ -16,7 +16,7 @@ services: - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization" - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS" - # If you want to have your account on , but host conduwuit on a subdomain, + # If you want to have your account on , but host Continuwuity on a subdomain, # you can let it only handle the well known file on that domain instead #- "traefik.http.routers.to-matrix-wellknown.rule=Host(``) && PathPrefix(`/.well-known/matrix`)" #- "traefik.http.routers.to-matrix-wellknown.tls=true" diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index ac4fb1ff..9ee98428 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -1,6 +1,6 @@ services: caddy: - # This compose file uses caddy-docker-proxy as the reverse proxy for conduwuit! + # This compose file uses caddy-docker-proxy as the reverse proxy for Continuwuity! # For more info, visit https://github.com/lucaslorentz/caddy-docker-proxy image: lucaslorentz/caddy-docker-proxy:ci-alpine ports: @@ -20,7 +20,7 @@ services: caddy.1_respond: /.well-known/matrix/client {"m.server":{"base_url":"https://matrix.example.com"},"m.homeserver":{"base_url":"https://matrix.example.com"},"org.matrix.msc3575.proxy":{"url":"https://matrix.example.com"}} homeserver: - ### If you already built the conduwuit image with 'docker build' or want to use a registry image, + ### If you already built the Continuwuity image with 'docker build' or want to use a registry image, ### then you are ready to go. image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index 86ad9cb6..9083b796 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -1,8 +1,8 @@ -# conduwuit - Behind Traefik Reverse Proxy +# Continuwuity - Behind Traefik Reverse Proxy services: homeserver: - ### If you already built the conduwuit image with 'docker build' or want to use the Docker Hub image, + ### If you already built the Continuwuity image with 'docker build' or want to use the Docker Hub image, ### then you are ready to go. image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped @@ -22,7 +22,7 @@ services: CONDUWUIT_PORT: 6167 # you need to match this with the traefik load balancer label if you're want to change it CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit #CONDUWUIT_CONFIG: '/etc/conduit.toml' # Uncomment if you mapped config toml above - ### Uncomment and change values as desired, note that conduwuit has plenty of config options, so you should check out the example example config too + ### Uncomment and change values as desired, note that Continuwuity has plenty of config options, so you should check out the example example config too # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging # CONDUWUIT_LOG: info # default is: "warn,state_res=warn" # CONDUWUIT_ALLOW_ENCRYPTION: 'true' @@ -44,14 +44,14 @@ services: server=your.server.name.example:443 } #cpuset: "0-4" # Uncomment to limit to specific CPU cores - ulimits: # conduwuit uses quite a few file descriptors, and on some systems it defaults to 1024, so you can tell docker to increase it + ulimits: # Continuwuity uses quite a few file descriptors, and on some systems it defaults to 1024, so you can tell docker to increase it nofile: soft: 1048567 hard: 1048567 ### Uncomment if you want to use your own Element-Web App. ### Note: You need to provide a config.json for Element and you also need a second - ### Domain or Subdomain for the communication between Element and conduwuit + ### Domain or Subdomain for the communication between Element and Continuwuity ### Config-Docs: https://github.com/vector-im/element-web/blob/develop/docs/config.md # element-web: # image: vectorim/element-web:latest diff --git a/docs/deploying/docker-compose.yml b/docs/deploying/docker-compose.yml index ca56d0b0..1a3ab811 100644 --- a/docs/deploying/docker-compose.yml +++ b/docs/deploying/docker-compose.yml @@ -1,8 +1,8 @@ -# conduwuit +# Continuwuity services: homeserver: - ### If you already built the conduwuit image with 'docker build' or want to use a registry image, + ### If you already built the Continuwuity image with 'docker build' or want to use a registry image, ### then you are ready to go. image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped @@ -28,7 +28,7 @@ services: # ### Uncomment if you want to use your own Element-Web App. ### Note: You need to provide a config.json for Element and you also need a second - ### Domain or Subdomain for the communication between Element and conduwuit + ### Domain or Subdomain for the communication between Element and Continuwuity ### Config-Docs: https://github.com/vector-im/element-web/blob/develop/docs/config.md # element-web: # image: vectorim/element-web:latest diff --git a/docs/deploying/kubernetes.md b/docs/deploying/kubernetes.md index d7721722..aceb2d52 100644 --- a/docs/deploying/kubernetes.md +++ b/docs/deploying/kubernetes.md @@ -1,8 +1,8 @@ -# conduwuit for Kubernetes +# Continuwuity for Kubernetes -conduwuit doesn't support horizontal scalability or distributed loading +Continuwuity doesn't support horizontal scalability or distributed loading natively, however a community maintained Helm Chart is available here to run -conduwuit on Kubernetes: +Continuwuity on Kubernetes: Should changes need to be made, please reach out to the maintainer in our -Matrix room as this is not maintained/controlled by the conduwuit maintainers. +Matrix room as this is not maintained/controlled by the Continuwuity maintainers. diff --git a/docs/development/hot_reload.md b/docs/development/hot_reload.md index 65fd4adf..ecfb6396 100644 --- a/docs/development/hot_reload.md +++ b/docs/development/hot_reload.md @@ -5,7 +5,7 @@ guaranteed to work at this time. ### Summary -When developing in debug-builds with the nightly toolchain, conduwuit is modular +When developing in debug-builds with the nightly toolchain, Continuwuity is modular using dynamic libraries and various parts of the application are hot-reloadable while the server is running: http api handlers, admin commands, services, database, etc. These are all split up into individual workspace crates as seen @@ -42,7 +42,7 @@ library, macOS, and likely other host architectures are not supported (if other architectures work, feel free to let us know and/or make a PR updating this). This should work on GNU ld and lld (rust-lld) and gcc/clang, however if you happen to have linker issues it's recommended to try using `mold` or `gold` -linkers, and please let us know in the [conduwuit Matrix room][7] the linker +linkers, and please let us know in the [Continuwuity Matrix room][7] the linker error and what linker solved this issue so we can figure out a solution. Ideally there should be minimal friction to using this, and in the future a build script (`build.rs`) may be suitable to making this easier to use if the capabilities @@ -52,13 +52,13 @@ allow us. As of 19 May 2024, the instructions for using this are: -0. Have patience. Don't hesitate to join the [conduwuit Matrix room][7] to +0. Have patience. Don't hesitate to join the [Continuwuity Matrix room][7] to receive help using this. As indicated by the various rustflags used and some of the interesting issues linked at the bottom, this is definitely not something the Rust ecosystem or toolchain is used to doing. 1. Install the nightly toolchain using rustup. You may need to use `rustup - override set nightly` in your local conduwuit directory, or use `cargo + override set nightly` in your local Continuwuity directory, or use `cargo +nightly` for all actions. 2. Uncomment `cargo-features` at the top level / root Cargo.toml @@ -85,14 +85,14 @@ LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$HOME/.rustup/toolchains/nightly-x86_64-unknown Cargo should only rebuild what was changed / what's necessary, so it should not be rebuilding all the crates. -9. In your conduwuit server terminal, hit/send `CTRL+C` signal. This will tell - conduwuit to find which libraries need to be reloaded, and reloads them as +9. In your Continuwuity server terminal, hit/send `CTRL+C` signal. This will tell + Continuwuity to find which libraries need to be reloaded, and reloads them as necessary. 10. If there were no errors, it will tell you it successfully reloaded `#` modules, and your changes should now be visible. Repeat 7 - 9 as needed. -To shutdown conduwuit in this setup, hit/send `CTRL+\`. Normal builds still +To shutdown Continuwuity in this setup, hit/send `CTRL+\`. Normal builds still shutdown with `CTRL+C` as usual. Steps 1 - 5 are the initial first-time steps for using this. To remove the hot @@ -101,7 +101,7 @@ reload setup, revert/comment all the Cargo.toml changes. As mentioned in the requirements section, if you happen to have some linker issues, try using the `-fuse-ld=` rustflag and specify mold or gold in all the `rustflags` definitions in the top level Cargo.toml, and please let us know in -the [conduwuit Matrix room][7] the problem. mold can be installed typically +the [Continuwuity Matrix room][7] the problem. mold can be installed typically through your distro, and gold is provided by the binutils package. It's possible a helper script can be made to do all of this, or most preferably @@ -136,7 +136,7 @@ acyclic graph. The primary rule is simple and illustrated in the figure below: **no crate is allowed to call a function or use a variable from a crate below it.** -![conduwuit's dynamic library setup diagram - created by Jason +![Continuwuity's dynamic library setup diagram - created by Jason Volk](assets/libraries.png) When a symbol is referenced between crates they become bound: **crates cannot be @@ -147,7 +147,7 @@ by using an `RTLD_LOCAL` binding for just one link between the main executable and the first crate, freeing the executable from all modules as no global binding ever occurs between them. -![conduwuit's reload and load order diagram - created by Jason +![Continuwuity's reload and load order diagram - created by Jason Volk](assets/reload_order.png) Proper resource management is essential for reliable reloading to occur. This is @@ -196,5 +196,5 @@ The initial implementation PR is available [here][1]. [4]: https://github.com/rust-lang/rust/issues/28794#issuecomment-368693049 [5]: https://github.com/rust-lang/cargo/issues/12746 [6]: https://crates.io/crates/hot-lib-reloader/ -[7]: https://matrix.to/#/#conduwuit:puppygock.gay +[7]: https://matrix.to/#/#continuwuity:continuwuity.org [8]: https://crates.io/crates/libloading diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 60615365..6b99e5de 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -36,7 +36,7 @@ pub(super) enum ServerCommand { /// - Print database memory usage statistics MemoryUsage, - /// - Clears all of Conduwuit's caches + /// - Clears all of Continuwuity's caches ClearCaches, /// - Performs an online backup of the database (only available for RocksDB From 1d840950b3a1c721c6c98ac3bb5c843f7fee7b9e Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 23 Apr 2025 22:20:08 +0100 Subject: [PATCH 1737/2091] docs: Mention Helm chart is for conduwuit --- docs/deploying/kubernetes.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/deploying/kubernetes.md b/docs/deploying/kubernetes.md index aceb2d52..0cbfbbc0 100644 --- a/docs/deploying/kubernetes.md +++ b/docs/deploying/kubernetes.md @@ -2,7 +2,8 @@ Continuwuity doesn't support horizontal scalability or distributed loading natively, however a community maintained Helm Chart is available here to run -Continuwuity on Kubernetes: +conduwuit on Kubernetes: -Should changes need to be made, please reach out to the maintainer in our -Matrix room as this is not maintained/controlled by the Continuwuity maintainers. +This should be compatible with continuwuity, but you will need to change the image reference. + +Should changes need to be made, please reach out to the maintainer as this is not maintained/controlled by the Continuwuity maintainers. From ee11afb460f03b4f40f4b533f5e3a908d598cd87 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 23 Apr 2025 01:51:52 +0100 Subject: [PATCH 1738/2091] Inject reason into federated leave request membership --- src/api/client/membership.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index b1b85b81..2847d668 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1855,7 +1855,10 @@ pub async fn leave_room( // Ask a remote server if we don't have this room and are not knocking on it if dont_have_room.and(not_knocked).await { - if let Err(e) = remote_leave_room(services, user_id, room_id).boxed().await { + if let Err(e) = remote_leave_room(services, user_id, room_id, reason.clone()) + .boxed() + .await + { warn!(%user_id, "Failed to leave room {room_id} remotely: {e}"); // Don't tell the client about this error } @@ -1940,6 +1943,7 @@ async fn remote_leave_room( services: &Services, user_id: &UserId, room_id: &RoomId, + reason: Option, ) -> Result<()> { let mut make_leave_response_and_server = Err!(BadServerResponse("No remote server available to assist in leaving {room_id}.")); @@ -2056,6 +2060,12 @@ async fn remote_leave_room( .expect("Timestamp is valid js_int value"), ), ); + // Inject the reason key into the event content dict if it exists + if let Some(reason) = reason { + if let Some(CanonicalJsonValue::Object(content)) = leave_event_stub.get_mut("content") { + content.insert("reason".to_owned(), CanonicalJsonValue::String(reason)); + } + } // room v3 and above removed the "event_id" field from remote PDU format match room_version_id { From bfd7ab5a22129d01b3e87c290414c0148ad635bf Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 23 Apr 2025 17:21:59 +0100 Subject: [PATCH 1739/2091] Bump ruwuma to 652cc48 --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afaa5622..216114af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3652,7 +3652,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "assign", "js_int", @@ -3672,7 +3672,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "js_int", "ruma-common", @@ -3684,7 +3684,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "as_variant", "assign", @@ -3707,7 +3707,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "as_variant", "base64 0.22.1", @@ -3739,7 +3739,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "as_variant", "indexmap 2.8.0", @@ -3764,7 +3764,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "bytes", "headers", @@ -3786,7 +3786,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3795,7 +3795,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "js_int", "ruma-common", @@ -3805,7 +3805,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3820,7 +3820,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "js_int", "ruma-common", @@ -3832,7 +3832,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=fa3c868e5a1c049dc9472310dc4955289a96bb35#fa3c868e5a1c049dc9472310dc4955289a96bb35" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 1517cfc1..c0f857c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,7 +350,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://forgejo.ellis.link/continuwuation/ruwuma" #branch = "conduwuit-changes" -rev = "fa3c868e5a1c049dc9472310dc4955289a96bb35" +rev = "652cc4864203ab7ca60cf9c47b931c0385304cc7" features = [ "compat", "rand", From 4c8dfc4c2c53be1dfd2d8c79182240dd0f4c9cec Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Wed, 23 Apr 2025 22:47:45 +0100 Subject: [PATCH 1740/2091] Suggested community guidelines changes --- docs/SUMMARY.md | 2 +- docs/community.md | 139 ++++++++++++++++++++++++++++++++++++++++++ docs/conduwuit_coc.md | 93 ---------------------------- 3 files changed, 140 insertions(+), 94 deletions(-) create mode 100644 docs/community.md delete mode 100644 docs/conduwuit_coc.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index ad0f8135..473c9e74 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -19,4 +19,4 @@ - [Contributing](contributing.md) - [Testing](development/testing.md) - [Hot Reloading ("Live" Development)](development/hot_reload.md) -- [conduwuit Community Code of Conduct](conduwuit_coc.md) +- [Community (and Guidelines)](community.md) diff --git a/docs/community.md b/docs/community.md new file mode 100644 index 00000000..a6852c0f --- /dev/null +++ b/docs/community.md @@ -0,0 +1,139 @@ +# Continuwuity Community Guidelines + +Welcome to the Continuwuity commuwunity! We're excited to have you here. Continuwuity is a +continuation of the conduwuit homeserver, which in turn is a hard-fork of the Conduit homeserver, +aimed at making Matrix more accessible and inclusive for everyone. + +This space is dedicated to fostering a positive, supportive, and welcoming environment for everyone. +These guidelines apply to all Continuwuity spaces, including our Matrix rooms and any other +community channels that reference them. We've written these guidelines to help us all create an +environment where everyone feels safe and respected. + +For code and contribution guidelines, please refer to the +[Contributor's Covenant](https://forgejo.ellis.link/continuwuation/continuwuity/src/branch/main/CODE_OF_CONDUCT.md). +Below are additional guidelines specific to the Continuwuity community. + +## Our Values and Expected Behaviors + +We strive to create a community based on mutual respect, collaboration, and inclusivity. We expect +all members to: + +1. **Be Respectful and Inclusive**: Treat everyone with respect. We're committed to a community + where everyone feels safe, regardless of background, identity, or experience. Discrimination, + harassment, or hate speech won't be tolerated. Remember that each person experiences the world + differently; share your own perspective and be open to learning about others'. + +2. **Be Positive and Constructive**: Engage in discussions constructively and support each other. + If you feel angry or frustrated, take a break before participating. Approach disagreements with + the goal of understanding, not winning. Focus on the issue, not the person. + +3. **Communicate Clearly and Kindly**: Our community includes neurodivergent individuals and those + who may not appreciate sarcasm or subtlety. Communicate clearly and kindly. Avoid ambiguity and + ensure your messages can be easily understood by all. Avoid placing the burden of education on + marginalized groups; please make an effort to look into your questions before asking others for + detailed explanations. + +4. **Be Open to Improving Inclusivity**: Actively participate in making our community more inclusive. + Report behaviour that contradicts these guidelines (see Reporting and Enforcement below) and be + open to constructive feedback aimed at improving our community. Understand that discussing + negative experiences can be emotionally taxing; focus on the message, not the tone. + +5. **Commit to Our Values**: Building an inclusive community requires ongoing effort from everyone. + Recognise that addressing bias and discrimination is a continuous process that needs commitment + and action from all members. + +## Unacceptable Behaviors + +To ensure everyone feels safe and welcome, the following behaviors are considered unacceptable +within the Continuwuity community: + +* **Harassment and Discrimination**: Avoid offensive comments related to background, family status, + gender, gender identity or expression, marital status, sex, sexual orientation, native language, + age, ability, race and/or ethnicity, caste, national origin, socioeconomic status, religion, + geographic location, or any other dimension of diversity. Don't deliberately misgender someone or + question the legitimacy of their gender identity. + +* **Violence and Threats**: Do not engage in any form of violence or threats, including inciting + violence towards anyone or encouraging self-harm. Posting or threatening to post someone else's + personally identifying information ("doxxing") is also forbidden. + +* **Personal Attacks**: Disagreements happen, but they should never turn into personal attacks. + Don't insult, demean, or belittle others. + +* **Unwelcome Attention or Contact**: Avoid unwelcome sexual attention, inappropriate physical + contact (or simulation thereof), sexualized comments, jokes, or imagery. + +* **Disruption**: Do not engage in sustained disruption of discussions, events, or other + community activities. + +* **Bad Faith Actions**: Do not intentionally make false reports or otherwise abuse the reporting + process. + +This is not an exhaustive list. Any behaviour that makes others feel unsafe or unwelcome may be +subject to enforcement action. + +## Matrix Community + +These Community Guidelines apply to the entire +[Continuwuity Matrix Space](https://matrix.to/#/#space:continuwuity.org) and its rooms, including: + +### [#continuwuity:continuwuity.org](https://matrix.to/#/#continuwuity:continuwuity.org) + +This room is for support and discussions about Continuwuity. Ask questions, share insights, and help +each other out while adhering to these guidelines. + +We ask that this room remain focused on the Continuwuity software specifically: the team are +typically happy to engage in conversations about related subjects in the off-topic room. + +### [#offtopic:continuwuity.org](https://matrix.to/#/#offtopic:continuwuity.org) + +For off-topic community conversations about any subject. While this room allows for a wide range of +topics, the same guidelines apply. Please keep discussions respectful and inclusive, and avoid +divisive or stressful subjects like specific country/world politics unless handled with exceptional +care and respect for diverse viewpoints. + +General topics, such as world events, are welcome as long as they follow the guidelines. If a member +of the team asks for the conversation to end, please respect their decision. + +### [#dev:continuwuity.org](https://matrix.to/#/#dev:continuwuity.org) + +This room is dedicated to discussing active development of Continuwuity, including ongoing issues or +code development. Collaboration here must follow these guidelines, and please consider raising +[an issue](https://forgejo.ellis.link/continuwuation/continuwuity/issues) on the repository to help +track progress. + +## Reporting and Enforcement + +We take these Community Guidelines seriously to protect our community members. If you witness or +experience unacceptable behaviour, or have any other concerns, please report it. + +**How to Report:** + +* **Alert Moderators in the Room:** If you feel comfortable doing so, you can address the issue + publicly in the relevant room by mentioning the moderation bot, `@rock:continuwuity.org`, which + will immediately alert all available moderators. +* **Direct Message:** If you're not comfortable raising the issue publicly, please send a direct + message (DM) to one of the room moderators. + +Reports will be handled with discretion. We will investigate promptly and thoroughly. + +**Enforcement Actions:** + +Anyone asked to stop unacceptable behaviour is expected to comply immediately. Failure to do so, or +engaging in prohibited behaviour, may result in enforcement action. Moderators may take actions they +deem appropriate, including but not limited to: + +1. **Warning**: A direct message or public warning identifying the violation and requesting + corrective action. +2. **Temporary Mute**: Temporary restriction from participating in discussions for a specified + period. +3. **Kick or Ban**: Removal from a room (kick) or the entire community space (ban). Egregious or + repeated violations may result in an immediate ban. Bans are typically permanent and reviewed + only in exceptional circumstances. + +Retaliation against those who report concerns in good faith will not be tolerated and will be +subject to the same enforcement actions. + +Together, let's build and maintain a community where everyone feels valued, safe, and respected. + +— The Continuwuity Moderation Team diff --git a/docs/conduwuit_coc.md b/docs/conduwuit_coc.md deleted file mode 100644 index 9a084150..00000000 --- a/docs/conduwuit_coc.md +++ /dev/null @@ -1,93 +0,0 @@ -# Continuwuity Community Code of Conduct - -Welcome to the Continuwuity community! We’re excited to have you here. Continuwuity is -a hard-fork of the Conduit homeserver, aimed at making Matrix more accessible -and inclusive for everyone. - -This space is dedicated to fostering a positive, supportive, and inclusive -environment for everyone. This Code of Conduct applies to all Continuwuity spaces, -including any further community rooms that reference this CoC. Here are our -guidelines to help maintain the welcoming atmosphere that sets Continuwuity apart. - -For the general foundational rules, please refer to the [Contributor's -Covenant](https://forgejo.ellis.link/continuwuation/continuwuity/src/branch/main/CODE_OF_CONDUCT.md). -Below are additional guidelines specific to the Continuwuity community. - -## Our Values and Guidelines - -1. **Respect and Inclusivity**: We are committed to maintaining a community - where everyone feels safe and respected. Discrimination, harassment, or hate -speech of any kind will not be tolerated. Recognise that each community member -experiences the world differently based on their past experiences, background, -and identity. Share your own experiences and be open to learning about others' -diverse perspectives. - -2. **Positivity and Constructiveness**: Engage in constructive discussions and - support each other. If you feel angry, negative, or aggressive, take a break -until you can participate in a positive and constructive manner. Process intense -feelings with a friend or in a private setting before engaging in community -conversations to help maintain a supportive and focused environment. - -3. **Clarity and Understanding**: Our community includes neurodivergent - individuals and those who may not appreciate sarcasm or subtlety. Communicate -clearly and kindly, avoiding sarcasm and ensuring your messages are easily -understood by all. Additionally, avoid putting the burden of education on -marginalized groups by doing your own research before asking for explanations. - -4. **Be Open to Inclusivity**: Actively engage in conversations about making our - community more inclusive. Report discriminatory behavior to the moderators -and be open to constructive feedback that aims to improve our community. -Understand that discussing discrimination and negative experiences can be -emotionally taxing, so focus on the message rather than critiquing the tone -used. - -5. **Commit to Inclusivity**: Building an inclusive community requires time, - energy, and resources. Recognise that addressing discrimination and bias is -an ongoing process that necessitates commitment and action from all community -members. - -## Matrix Community - -This Code of Conduct applies to the entire [Continuwuity Matrix -Space](https://matrix.to/#/#space:continuwuity.org) and its rooms, -including: - -### [#continuwuity:continuwuity.org](https://matrix.to/#/#continuwuity:continuwuity.org) - -This room is for support and discussions about Continuwuity. Ask questions, share -insights, and help each other out. - -### [#offtopic:continuwuity.org](https://matrix.to/#/#offtopic:continuwuity.org) - -For off-topic community conversations about any subject. While this room allows -for a wide range of topics, the same CoC applies. Keep discussions respectful -and inclusive, and avoid divisive subjects like country/world politics. General -topics, such as world events, are welcome as long as they follow the CoC. - -### [#dev:continuwuity.org](https://matrix.to/#/#dev:continuwuity.org) - -This room is dedicated to discussing active development of Continuwuity. Posting -requires an elevated power level, which can be requested in one of the other -rooms. Use this space to collaborate and innovate. - -## Enforcement - -We have a zero-tolerance policy for violations of this Code of Conduct. If -someone’s behavior makes you uncomfortable, please report it to the moderators. -Actions we may take include: - -1. **Warning**: A warning given directly in the room or via a private message - from the moderators, identifying the violation and requesting corrective -action. -2. **Temporary Mute**: Temporary restriction from participating in discussions - for a specified period to allow for reflection and cooling off. -3. **Kick or Ban**: Egregious behavior may result in an immediate kick or ban to - protect other community members. Bans are considered permanent and will only -be reversed in exceptional circumstances after proven good behavior. - -Please highlight issues directly in rooms when possible, but if you don't feel -comfortable doing that, then please send a DM to one of the moderators directly. - -Together, let’s build a community where everyone feels valued and respected. - -— The Continuwuity Moderation Team From f83238df783efd38f3da43d565710708b30ba52b Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 23 Apr 2025 17:48:26 +0100 Subject: [PATCH 1741/2091] refactor: Use config service --- src/service/moderation.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/service/moderation.rs b/src/service/moderation.rs index d571de88..4a32404c 100644 --- a/src/service/moderation.rs +++ b/src/service/moderation.rs @@ -3,18 +3,24 @@ use std::sync::Arc; use conduwuit::{Result, Server, implement}; use ruma::ServerName; +use crate::{Dep, config}; + pub struct Service { services: Services, } struct Services { pub server: Arc, + pub config: Dep, } impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { - services: Services { server: args.server.clone() }, + services: Services { + server: args.server.clone(), + config: args.depend::("config"), + }, })) } @@ -25,14 +31,13 @@ impl crate::Service for Service { #[must_use] pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool { // We must never block federating with ourselves - if server_name == self.services.server.config.server_name { + if server_name == self.services.config.server_name { return false; } // Check if server is explicitly allowed if self .services - .server .config .allowed_remote_server_names .is_match(server_name.host()) @@ -42,7 +47,6 @@ pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool { // Check if server is explicitly forbidden self.services - .server .config .forbidden_remote_server_names .is_match(server_name.host()) @@ -56,7 +60,6 @@ pub fn is_remote_server_room_directory_forbidden(&self, server_name: &ServerName self.is_remote_server_forbidden(server_name) || self .services - .server .config .forbidden_remote_room_directory_server_names .is_match(server_name.host()) @@ -70,7 +73,6 @@ pub fn is_remote_server_media_downloads_forbidden(&self, server_name: &ServerNam self.is_remote_server_forbidden(server_name) || self .services - .server .config .prevent_media_downloads_from .is_match(server_name.host()) From 45872ede7a4b5a2335295e5ba52dd1bd393d21ec Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 23 Apr 2025 17:48:33 +0100 Subject: [PATCH 1742/2091] chore: Fix formatting --- conduwuit-example.toml | 6 +++--- src/api/client/room/create.rs | 29 ++++++++++++++++++----------- src/core/config/mod.rs | 6 +++--- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index b6bfd092..c87f21ef 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -967,9 +967,9 @@ #rocksdb_compaction_ioprio_idle = true # Enables RocksDB compaction. You should never ever have to set this -# option to false. If you for some reason find yourself needing to use this -# option as part of troubleshooting or a bug, please reach out to us in -# the conduwuit Matrix room with information and details. +# option to false. If you for some reason find yourself needing to use +# this option as part of troubleshooting or a bug, please reach out to us +# in the conduwuit Matrix room with information and details. # # Disabling compaction will lead to a significantly bloated and # explosively large database, gradually poor performance, unnecessarily diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index f5f61784..be3fd23b 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -614,24 +614,31 @@ fn custom_room_id_check(services: &Services, custom_room_id: &str) -> Result Date: Wed, 23 Apr 2025 23:54:16 +0100 Subject: [PATCH 1743/2091] docs: Fix configuration examples and defaults --- conduwuit-example.toml | 6 +++--- src/core/config/mod.rs | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index c87f21ef..b0b59344 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -116,7 +116,7 @@ # `https://continuwuity.org/.well-known/continuwuity/announcements` for any new # announcements or major updates. This is not an update check endpoint. # -#allow_announcements_check = +#allow_announcements_check = true # Set this to any float value to multiply conduwuit's in-memory LRU caches # with such as "auth_chain_cache_capacity". @@ -1207,7 +1207,7 @@ # You can set this to ["*"] to block all servers by default, and then # use `allowed_remote_server_names` to allow only specific servers. # -# example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] +# example: ["badserver\\.tld$", "badphrase", "19dollarfortnitecards"] # #forbidden_remote_server_names = [] @@ -1216,7 +1216,7 @@ # # This option has no effect if `forbidden_remote_server_names` is empty. # -# example: ["goodserver\.tld$", "goodphrase"] +# example: ["goodserver\\.tld$", "goodphrase"] # #allowed_remote_server_names = [] diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 6252f177..e3b2a531 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -163,6 +163,8 @@ pub struct Config { /// If enabled, conduwuit will send a simple GET request periodically to /// `https://continuwuity.org/.well-known/continuwuity/announcements` for any new /// announcements or major updates. This is not an update check endpoint. + /// + /// default: true #[serde(alias = "allow_check_for_updates", default = "true_fn")] pub allow_announcements_check: bool, @@ -1384,7 +1386,7 @@ pub struct Config { /// You can set this to ["*"] to block all servers by default, and then /// use `allowed_remote_server_names` to allow only specific servers. /// - /// example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] + /// example: ["badserver\\.tld$", "badphrase", "19dollarfortnitecards"] /// /// default: [] #[serde(default, with = "serde_regex")] @@ -1395,7 +1397,7 @@ pub struct Config { /// /// This option has no effect if `forbidden_remote_server_names` is empty. /// - /// example: ["goodserver\.tld$", "goodphrase"] + /// example: ["goodserver\\.tld$", "goodphrase"] /// /// default: [] #[serde(default, with = "serde_regex")] From cdf105a24eb7398d6e09d0840ff2c11c43c05ce9 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Fri, 25 Apr 2025 02:18:00 +0100 Subject: [PATCH 1744/2091] Don't serialize the x-key before storing it Co-authored-by: dasha --- src/service/users/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 1eb289fc..701561a8 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -577,7 +577,7 @@ impl Service { self.db .userid_usersigningkeyid - .put(user_id, user_signing_key_key); + .raw_put(user_id, user_signing_key_key); } if notify { From c203c1fead9a040de28b952eafdfdeab9cec77bc Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 24 Apr 2025 22:49:47 +0100 Subject: [PATCH 1745/2091] chore: Enable blurhashing by default --- src/main/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index e2fed5d5..0c5e2b6f 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -36,6 +36,7 @@ assets = [ [features] default = [ + "blurhashing", "brotli_compression", "element_hacks", "gzip_compression", From dcbacb5b78ac679ddb17465c9e60042cff4ab3e7 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 24 Apr 2025 00:40:36 +0100 Subject: [PATCH 1746/2091] feat: Allow controlling client message filtering --- conduwuit-example.toml | 44 ++++++++++++++++++++++++--------- src/api/client/message.rs | 6 +++-- src/core/config/mod.rs | 52 ++++++++++++++++++++++++++++----------- src/service/moderation.rs | 20 ++++++++++++--- 4 files changed, 90 insertions(+), 32 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index b0b59344..3d92ab15 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1182,23 +1182,13 @@ # #prune_missing_media = false -# Vector list of regex patterns of server names that conduwuit will refuse -# to download remote media from. -# -# example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] -# -#prevent_media_downloads_from = [] - # List of forbidden server names via regex patterns that we will block # incoming AND outgoing federation with, and block client room joins / # remote user invites. # -# Additionally, it will hide messages from these servers for all users -# on this server. -# # Note that your messages can still make it to forbidden servers through -# backfilling. Events we receive from forbidden servers via backfill will -# be stored in the database, but will not be sent to the client. +# backfilling. Events we receive from forbidden servers via backfill +# from servers we *do* federate with will be stored in the database. # # This check is applied on the room ID, room alias, sender server name, # sender user's server name, inbound federation X-Matrix origin, and @@ -1220,6 +1210,13 @@ # #allowed_remote_server_names = [] +# Vector list of regex patterns of server names that conduwuit will refuse +# to download remote media from. +# +# example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] +# +#prevent_media_downloads_from = [] + # List of forbidden server names via regex patterns that we will block all # outgoing federated room directory requests for. Useful for preventing # our users from wandering into bad servers or spaces. @@ -1228,6 +1225,29 @@ # #forbidden_remote_room_directory_server_names = [] +# Vector list of regex patterns of server names that conduwuit will not +# send messages to the client from. +# +# Note that there is no way for clients to receive messages once a server +# has become unignored without doing a full sync. This is a protocol +# limitation with the current sync protocols. This means this is somewhat +# of a nuclear option. +# +# example: ["reallybadserver\.tld$", "reallybadphrase", +# "69dollarfortnitecards"] +# +#ignore_messages_from_server_names = [] + +# Send messages from users that the user has ignored to the client. +# +# There is no way for clients to receive messages sent while a user was +# ignored without doing a full sync. This is a protocol limitation with +# the current sync protocols. Disabling this option will move +# responsibility of ignoring messages to the client, which can avoid this +# limitation. +# +#send_messages_from_ignored_users_to_client = false + # Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you # do not want conduwuit to send outbound requests to. Defaults to # RFC1918, unroutable, loopback, multicast, and testnet addresses for diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 08887e18..bedfdc7a 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -275,10 +275,12 @@ pub(crate) async fn is_ignored_pdu( let ignored_server = services .moderation - .is_remote_server_forbidden(pdu.sender().server_name()); + .is_remote_server_ignored(pdu.sender().server_name()); if ignored_type - && (ignored_server || services.users.user_is_ignored(&pdu.sender, user_id).await) + && (ignored_server + || (!services.config.send_messages_from_ignored_users_to_client + && services.users.user_is_ignored(&pdu.sender, user_id).await)) { return true; } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index e3b2a531..5374c2c2 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -163,7 +163,7 @@ pub struct Config { /// If enabled, conduwuit will send a simple GET request periodically to /// `https://continuwuity.org/.well-known/continuwuity/announcements` for any new /// announcements or major updates. This is not an update check endpoint. - /// + /// /// default: true #[serde(alias = "allow_check_for_updates", default = "true_fn")] pub allow_announcements_check: bool, @@ -1359,25 +1359,13 @@ pub struct Config { #[serde(default)] pub prune_missing_media: bool, - /// Vector list of regex patterns of server names that conduwuit will refuse - /// to download remote media from. - /// - /// example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] - /// - /// default: [] - #[serde(default, with = "serde_regex")] - pub prevent_media_downloads_from: RegexSet, - /// List of forbidden server names via regex patterns that we will block /// incoming AND outgoing federation with, and block client room joins / /// remote user invites. /// - /// Additionally, it will hide messages from these servers for all users - /// on this server. - /// /// Note that your messages can still make it to forbidden servers through - /// backfilling. Events we receive from forbidden servers via backfill will - /// be stored in the database, but will not be sent to the client. + /// backfilling. Events we receive from forbidden servers via backfill + /// from servers we *do* federate with will be stored in the database. /// /// This check is applied on the room ID, room alias, sender server name, /// sender user's server name, inbound federation X-Matrix origin, and @@ -1403,6 +1391,15 @@ pub struct Config { #[serde(default, with = "serde_regex")] pub allowed_remote_server_names: RegexSet, + /// Vector list of regex patterns of server names that conduwuit will refuse + /// to download remote media from. + /// + /// example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] + /// + /// default: [] + #[serde(default, with = "serde_regex")] + pub prevent_media_downloads_from: RegexSet, + /// List of forbidden server names via regex patterns that we will block all /// outgoing federated room directory requests for. Useful for preventing /// our users from wandering into bad servers or spaces. @@ -1413,6 +1410,31 @@ pub struct Config { #[serde(default, with = "serde_regex")] pub forbidden_remote_room_directory_server_names: RegexSet, + /// Vector list of regex patterns of server names that conduwuit will not + /// send messages to the client from. + /// + /// Note that there is no way for clients to receive messages once a server + /// has become unignored without doing a full sync. This is a protocol + /// limitation with the current sync protocols. This means this is somewhat + /// of a nuclear option. + /// + /// example: ["reallybadserver\.tld$", "reallybadphrase", + /// "69dollarfortnitecards"] + /// + /// default: [] + #[serde(default, with = "serde_regex")] + pub ignore_messages_from_server_names: RegexSet, + + /// Send messages from users that the user has ignored to the client. + /// + /// There is no way for clients to receive messages sent while a user was + /// ignored without doing a full sync. This is a protocol limitation with + /// the current sync protocols. Disabling this option will move + /// responsibility of ignoring messages to the client, which can avoid this + /// limitation. + #[serde(default)] + pub send_messages_from_ignored_users_to_client: bool, + /// Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you /// do not want conduwuit to send outbound requests to. Defaults to /// RFC1918, unroutable, loopback, multicast, and testnet addresses for diff --git a/src/service/moderation.rs b/src/service/moderation.rs index 4a32404c..c3e55a1d 100644 --- a/src/service/moderation.rs +++ b/src/service/moderation.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use conduwuit::{Result, Server, implement}; +use conduwuit::{Result, implement}; use ruma::ServerName; use crate::{Dep, config}; @@ -10,7 +10,7 @@ pub struct Service { } struct Services { - pub server: Arc, + // pub server: Arc, pub config: Dep, } @@ -18,7 +18,7 @@ impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { Ok(Arc::new(Self { services: Services { - server: args.server.clone(), + // server: args.server.clone(), config: args.depend::("config"), }, })) @@ -27,6 +27,20 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } +#[implement(Service)] +#[must_use] +pub fn is_remote_server_ignored(&self, server_name: &ServerName) -> bool { + // We must never block federating with ourselves + if server_name == self.services.config.server_name { + return false; + } + + self.services + .config + .ignore_messages_from_server_names + .is_match(server_name.host()) +} + #[implement(Service)] #[must_use] pub fn is_remote_server_forbidden(&self, server_name: &ServerName) -> bool { From 77c4f9ff2f617b92f8afd3ac837fe4adfd7147ce Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 25 Apr 2025 23:01:05 +0100 Subject: [PATCH 1747/2091] fix: Do not panic on invalid membership event content --- src/core/matrix/state_res/event_auth.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/core/matrix/state_res/event_auth.rs b/src/core/matrix/state_res/event_auth.rs index 8c9339ec..c69db50e 100644 --- a/src/core/matrix/state_res/event_auth.rs +++ b/src/core/matrix/state_res/event_auth.rs @@ -38,7 +38,7 @@ struct GetMembership { membership: MembershipState, } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] struct RoomMemberContentFields { membership: Option>, join_authorised_via_users_server: Option>, @@ -149,9 +149,9 @@ where Incoming: Event + Send + Sync, { debug!( - "auth_check beginning for {} ({})", - incoming_event.event_id(), - incoming_event.event_type() + event_id = format!("{}", incoming_event.event_id()), + event_type = format!("{}", incoming_event.event_type()), + "auth_check beginning" ); // [synapse] check that all the events are in the same room as `incoming_event` @@ -383,10 +383,15 @@ where let sender_membership_event_content: RoomMemberContentFields = from_json_str(sender_member_event.content().get())?; - let membership_state = sender_membership_event_content - .membership - .expect("we should test before that this field exists") - .deserialize()?; + let Some(membership_state) = sender_membership_event_content.membership else { + warn!( + sender_membership_event_content = format!("{sender_membership_event_content:?}"), + event_id = format!("{}", incoming_event.event_id()), + "Sender membership event content missing membership field" + ); + return Err(Error::InvalidPdu("Missing membership field".to_owned())); + }; + let membership_state = membership_state.deserialize()?; if !matches!(membership_state, MembershipState::Join) { warn!("sender's membership is not join"); From 1a5ab33852b1ef301d6a3ce4c3154d430ef24a03 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 25 Apr 2025 23:51:23 +0100 Subject: [PATCH 1748/2091] chore: Error on missing ID in messages --- src/api/client/message.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index bedfdc7a..16b1796a 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,3 +1,5 @@ +use core::panic; + use axum::extract::State; use conduwuit::{ Err, Result, at, @@ -132,8 +134,6 @@ pub(crate) async fn get_message_events_route( .take(limit) .collect() .await; - // let appservice_id = body.appservice_info.map(|appservice| - // appservice.registration.id); let lazy_loading_context = lazy_loading::Context { user_id: sender_user, @@ -143,7 +143,7 @@ pub(crate) async fn get_message_events_route( if let Some(registration) = body.appservice_info.as_ref() { <&DeviceId>::from(registration.registration.id.as_str()) } else { - <&DeviceId>::from("") + panic!("No device_id provided and no appservice registration found, this should be unreachable"); }, }, room_id, From c698d65a92fb6d3378e9d493b5356ecf067ae286 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Thu, 24 Apr 2025 17:15:03 +0100 Subject: [PATCH 1749/2091] Make Cloudflare Pages optional in CI --- .forgejo/workflows/documentation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.forgejo/workflows/documentation.yml b/.forgejo/workflows/documentation.yml index 55f25058..1bda64f8 100644 --- a/.forgejo/workflows/documentation.yml +++ b/.forgejo/workflows/documentation.yml @@ -57,17 +57,17 @@ jobs: run: npm install --save-dev wrangler@latest - name: Deploy to Cloudflare Pages (Production) - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + if: (github.event_name == 'push' && github.ref == 'refs/heads/main') && vars.CLOUDFLARE_PROJECT_NAME != '' uses: https://github.com/cloudflare/wrangler-action@v3 with: accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} - command: pages deploy ./public --branch=main --commit-dirty=true --project-name=${{ vars.CLOUDFLARE_PROJECT_NAME }}" + command: pages deploy ./public --branch=main --commit-dirty=true --project-name=${{ vars.CLOUDFLARE_PROJECT_NAME }} - name: Deploy to Cloudflare Pages (Preview) - if: ${{ github.event_name != 'push' || github.ref != 'refs/heads/main' }} + if: (github.event_name != 'push' || github.ref != 'refs/heads/main') && vars.CLOUDFLARE_PROJECT_NAME != '' uses: https://github.com/cloudflare/wrangler-action@v3 with: accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} - command: pages deploy ./public --branch=${{ github.head_ref }} --commit-dirty=true --project-name=${{ vars.CLOUDFLARE_PROJECT_NAME }}" + command: pages deploy ./public --branch=${{ github.head_ref }} --commit-dirty=true --project-name=${{ vars.CLOUDFLARE_PROJECT_NAME }} From 73c991edd03d29cbac2cd4d0be07254b2c87df82 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Thu, 24 Apr 2025 17:24:28 +0100 Subject: [PATCH 1750/2091] Ignore all markdown for auto image builds --- .forgejo/workflows/release-image.yml | 68 +++++++++++++--------------- 1 file changed, 31 insertions(+), 37 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 2cb6a329..141bfef9 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -1,24 +1,25 @@ name: Release Docker Image -concurrency: +concurrency: group: "release-image-${{ github.ref }}" on: pull_request: push: paths-ignore: - - '.gitlab-ci.yml' - - '.gitignore' - - 'renovate.json' - - 'debian/**' - - 'docker/**' - - 'docs/**' + - "*.md" + - "**/*.md" + - ".gitlab-ci.yml" + - ".gitignore" + - "renovate.json" + - "debian/**" + - "docker/**" + - "docs/**" # Allows you to run this workflow manually from the Actions tab workflow_dispatch: env: - BUILTIN_REGISTRY: forgejo.ellis.link - BUILTIN_REGISTRY_ENABLED: "${{ ((vars.BUILTIN_REGISTRY_USER && secrets.BUILTIN_REGISTRY_PASSWORD) || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)) && 'true' || 'false' }}" - + BUILTIN_REGISTRY: forgejo.ellis.link + BUILTIN_REGISTRY_ENABLED: "${{ ((vars.BUILTIN_REGISTRY_USER && secrets.BUILTIN_REGISTRY_PASSWORD) || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false)) && 'true' || 'false' }}" jobs: define-variables: @@ -37,7 +38,7 @@ jobs: script: | const githubRepo = '${{ github.repository }}'.toLowerCase() const repoId = githubRepo.split('/')[1] - + core.setOutput('github_repository', githubRepo) const builtinImage = '${{ env.BUILTIN_REGISTRY }}/' + githubRepo let images = [] @@ -48,7 +49,7 @@ jobs: core.setOutput('images_list', images.join(",")) const platforms = ['linux/amd64', 'linux/arm64'] core.setOutput('build_matrix', JSON.stringify({ - platform: platforms, + platform: platforms, include: platforms.map(platform => { return { platform, slug: platform.replace('/', '-') @@ -65,22 +66,15 @@ jobs: attestations: write id-token: write strategy: - matrix: { - "include": [ - { - "platform": "linux/amd64", - "slug": "linux-amd64" - }, - { - "platform": "linux/arm64", - "slug": "linux-arm64" - } - ], - "platform": [ - "linux/amd64", - "linux/arm64" - ] - } + matrix: + { + "include": + [ + { "platform": "linux/amd64", "slug": "linux-amd64" }, + { "platform": "linux/arm64", "slug": "linux-arm64" }, + ], + "platform": ["linux/amd64", "linux/arm64"], + } steps: - name: Echo strategy run: echo '${{ toJSON(fromJSON(needs.define-variables.outputs.build_matrix)) }}' @@ -105,9 +99,9 @@ jobs: - name: Login to builtin registry uses: docker/login-action@v3 with: - registry: ${{ env.BUILTIN_REGISTRY }} - username: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }} - password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }} + registry: ${{ env.BUILTIN_REGISTRY }} + username: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }} + password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }} # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - name: Extract metadata (labels, annotations) for Docker @@ -165,7 +159,7 @@ jobs: run: | mkdir -p /tmp/digests digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" + touch "/tmp/digests/${digest#sha256:}" - name: Upload digest uses: forgejo/upload-artifact@v4 @@ -174,7 +168,7 @@ jobs: path: /tmp/digests/* if-no-files-found: error retention-days: 1 - + merge: runs-on: dind container: ghcr.io/catthehacker/ubuntu:act-latest @@ -190,9 +184,9 @@ jobs: - name: Login to builtin registry uses: docker/login-action@v3 with: - registry: ${{ env.BUILTIN_REGISTRY }} - username: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }} - password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }} + registry: ${{ env.BUILTIN_REGISTRY }} + username: ${{ vars.BUILTIN_REGISTRY_USER || github.actor }} + password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -211,7 +205,7 @@ jobs: images: ${{needs.define-variables.outputs.images}} # default labels & annotations: https://github.com/docker/metadata-action/blob/master/src/meta.ts#L509 env: - DOCKER_METADATA_ANNOTATIONS_LEVELS: index + DOCKER_METADATA_ANNOTATIONS_LEVELS: index - name: Create manifest list and push working-directory: /tmp/digests From eb886b6760ceea19412050df99703c4825ae56eb Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Thu, 24 Apr 2025 21:24:21 +0100 Subject: [PATCH 1751/2091] Element Web client build --- .forgejo/workflows/element.yml | 127 +++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 .forgejo/workflows/element.yml diff --git a/.forgejo/workflows/element.yml b/.forgejo/workflows/element.yml new file mode 100644 index 00000000..db771197 --- /dev/null +++ b/.forgejo/workflows/element.yml @@ -0,0 +1,127 @@ +name: Deploy Element Web + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + +concurrency: + group: "element-${{ github.ref }}" + cancel-in-progress: true + +jobs: + build-and-deploy: + name: Build and Deploy Element Web + runs-on: ubuntu-latest + + steps: + - name: Setup Node.js + uses: https://code.forgejo.org/actions/setup-node@v4 + with: + node-version: "20" + + - name: Clone, setup, and build Element Web + run: | + echo "Cloning Element Web..." + git clone https://github.com/maunium/element-web + cd element-web + git checkout develop + git pull + + echo "Cloning matrix-js-sdk..." + git clone https://github.com/matrix-org/matrix-js-sdk.git + + echo "Installing Yarn..." + npm install -g yarn + + echo "Installing dependencies..." + yarn install + + echo "Preparing build environment..." + mkdir -p .home + + echo "Cleaning up specific node_modules paths..." + rm -rf node_modules/@types/eslint-scope/ matrix-*-sdk/node_modules/@types/eslint-scope || echo "Cleanup paths not found, continuing." + + echo "Getting matrix-js-sdk commit hash..." + cd matrix-js-sdk + jsver=$(git rev-parse HEAD) + jsver=${jsver:0:12} + cd .. + echo "matrix-js-sdk version hash: $jsver" + + echo "Getting element-web commit hash..." + ver=$(git rev-parse HEAD) + ver=${ver:0:12} + echo "element-web version hash: $ver" + + chmod +x ./build-sh + + export VERSION="$ver-js-$jsver" + echo "Building Element Web version: $VERSION" + ./build-sh + + echo "Checking for build output..." + ls -la webapp/ + + - name: Create config.json + run: | + cat < ./element-web/webapp/config.json + { + "default_server_name": "continuwuity.org", + "default_server_config": { + "m.homeserver": { + "base_url": "https://matrix.continuwuity.org" + } + }, + "default_country_code": "GB", + "default_theme": "dark", + "mobile_guide_toast": false, + "show_labs_settings": true, + "room_directory": [ + "continuwuity.org", + "matrixrooms.info" + ], + "settings_defaults": { + "UIFeature.urlPreviews": true, + "UIFeature.feedback": false, + "UIFeature.voip": false, + "UIFeature.shareQrCode": false, + "UIFeature.shareSocial": false, + "UIFeature.locationSharing": false, + "enableSyntaxHighlightLanguageDetection": true + }, + "features": { + "feature_pinning": true, + "feature_custom_themes": true + } + } + EOF + echo "Created ./element-web/webapp/config.json" + cat ./element-web/webapp/config.json + + - name: Upload Artifact + uses: https://code.forgejo.org/actions/upload-artifact@v3 + with: + name: element-web + path: ./element-web/webapp/ + retention-days: 14 + + - name: Install Wrangler + run: npm install --save-dev wrangler@latest + + - name: Deploy to Cloudflare Pages (Production) + if: github.ref == 'refs/heads/main' && vars.CLOUDFLARE_PROJECT_NAME != '' + uses: https://github.com/cloudflare/wrangler-action@v3 + with: + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + command: pages deploy ./element-web/webapp --branch="main" --commit-dirty=true --project-name="${{ vars.CLOUDFLARE_PROJECT_NAME }}-element" + + - name: Deploy to Cloudflare Pages (Preview) + if: github.ref != 'refs/heads/main' && vars.CLOUDFLARE_PROJECT_NAME != '' + uses: https://github.com/cloudflare/wrangler-action@v3 + with: + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + command: pages deploy ./element-web/webapp --branch="${{ github.head_ref || github.ref_name }}" --commit-dirty=true --project-name="${{ vars.CLOUDFLARE_PROJECT_NAME }}-element" From 60caa448b0e2119ee00ad67104da8e19a18982c4 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Thu, 24 Apr 2025 21:27:41 +0100 Subject: [PATCH 1752/2091] Tidy up publishing restriction check --- .forgejo/workflows/documentation.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.forgejo/workflows/documentation.yml b/.forgejo/workflows/documentation.yml index 1bda64f8..7d95a317 100644 --- a/.forgejo/workflows/documentation.yml +++ b/.forgejo/workflows/documentation.yml @@ -57,17 +57,17 @@ jobs: run: npm install --save-dev wrangler@latest - name: Deploy to Cloudflare Pages (Production) - if: (github.event_name == 'push' && github.ref == 'refs/heads/main') && vars.CLOUDFLARE_PROJECT_NAME != '' + if: github.ref == 'refs/heads/main' && vars.CLOUDFLARE_PROJECT_NAME != '' uses: https://github.com/cloudflare/wrangler-action@v3 with: accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} - command: pages deploy ./public --branch=main --commit-dirty=true --project-name=${{ vars.CLOUDFLARE_PROJECT_NAME }} + command: pages deploy ./public --branch="main" --commit-dirty=true --project-name="${{ vars.CLOUDFLARE_PROJECT_NAME }}" - name: Deploy to Cloudflare Pages (Preview) - if: (github.event_name != 'push' || github.ref != 'refs/heads/main') && vars.CLOUDFLARE_PROJECT_NAME != '' + if: github.ref != 'refs/heads/main' && vars.CLOUDFLARE_PROJECT_NAME != '' uses: https://github.com/cloudflare/wrangler-action@v3 with: accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} - command: pages deploy ./public --branch=${{ github.head_ref }} --commit-dirty=true --project-name=${{ vars.CLOUDFLARE_PROJECT_NAME }} + command: pages deploy ./public --branch="${{ github.head_ref || github.ref_name }}" --commit-dirty=true --project-name="${{ vars.CLOUDFLARE_PROJECT_NAME }}" From b2620e6922a4acf3e2b86ac220f7c4974fa268ca Mon Sep 17 00:00:00 2001 From: Kokomo Date: Sun, 27 Apr 2025 15:43:15 +0000 Subject: [PATCH 1753/2091] Remove email and add reference to matrix space --- CODE_OF_CONDUCT.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index e77154e7..65ee41e0 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,4 +1,3 @@ - # Contributor Covenant Code of Conduct ## Our Pledge @@ -60,8 +59,7 @@ representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement over email at - or over Matrix at @strawberry:puppygock.gay. +reported to the community leaders responsible for enforcement over Matrix at #continuwuity:continuwuity.org. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the From a9a478f0778a355e82a5b54e7cb292369e39a792 Mon Sep 17 00:00:00 2001 From: Kokomo Date: Sun, 27 Apr 2025 15:47:40 +0000 Subject: [PATCH 1754/2091] Add back space oops --- CODE_OF_CONDUCT.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 65ee41e0..3ac0a83d 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,3 +1,4 @@ + # Contributor Covenant Code of Conduct ## Our Pledge From 90f1a193e38f41c0108f6009e7a027bb7aa6753b Mon Sep 17 00:00:00 2001 From: Kokomo Date: Sun, 27 Apr 2025 16:06:34 +0000 Subject: [PATCH 1755/2091] Add maintainer emails --- CODE_OF_CONDUCT.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 3ac0a83d..476e68fb 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,4 +1,3 @@ - # Contributor Covenant Code of Conduct ## Our Pledge @@ -60,7 +59,7 @@ representative at an online or offline event. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the community leaders responsible for enforcement over Matrix at #continuwuity:continuwuity.org. +reported to the community leaders responsible for enforcement over Matrix at [#continuwuity:continuwuity.org](https://matrix.to/#/#continuwuity:continuwuity.org) or email at , and respectively. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the From 6b0288dd4c4649e1f3e2772a73e5b204f645d560 Mon Sep 17 00:00:00 2001 From: Kokomo Date: Sun, 27 Apr 2025 18:42:02 +0000 Subject: [PATCH 1756/2091] Update Contributing.md file (#807) Cleaned up wording and adjusted the links Reviewed-on: https://forgejo.ellis.link/continuwuation/continuwuity/pulls/807 Reviewed-by: nex Reviewed-by: Jade Ellis Co-authored-by: Kokomo Co-committed-by: Kokomo --- CONTRIBUTING.md | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fb540011..ecff7173 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ This page is for about contributing to conduwuit. The [development](./development.md) page may be of interest for you as well. If you would like to work on an [issue][issues] that is not assigned, preferably -ask in the Matrix room first at [#conduwuit:puppygock.gay][conduwuit-matrix], +ask in the Matrix room first at [#continuwuity:continuwuity.org][continuwuity-matrix], and comment on it. ### Linting and Formatting @@ -23,9 +23,9 @@ suggestion, allow the lint and mention that in a comment. ### Running CI tests locally -conduwuit's CI for tests, linting, formatting, audit, etc use +continuwuity's CI for tests, linting, formatting, audit, etc use [`engage`][engage]. engage can be installed from nixpkgs or `cargo install -engage`. conduwuit's Nix flake devshell has the nixpkgs engage with `direnv`. +engage`. continuwuity's Nix flake devshell has the nixpkgs engage with `direnv`. Use `engage --help` for more usage details. To test, format, lint, etc that CI would do, install engage, allow the `.envrc` @@ -111,33 +111,28 @@ applies here. ### Creating pull requests -Please try to keep contributions to the GitHub. While the mirrors of conduwuit -allow for pull/merge requests, there is no guarantee I will see them in a timely +Please try to keep contributions to the Forgejo Instance. While the mirrors of continuwuity +allow for pull/merge requests, there is no guarantee the maintainers will see them in a timely manner. Additionally, please mark WIP or unfinished or incomplete PRs as drafts. -This prevents me from having to ping once in a while to double check the status +This prevents us from having to ping once in a while to double check the status of it, especially when the CI completed successfully and everything so it *looks* done. -If you open a pull request on one of the mirrors, it is your responsibility to -inform me about its existence. In the future I may try to solve this with more -repo bots in the conduwuit Matrix room. There is no mailing list or email-patch -support on the sr.ht mirror, but if you'd like to email me a git patch you can -do so at `strawberry@puppygock.gay`. Direct all PRs/MRs to the `main` branch. By sending a pull request or patch, you are agreeing that your changes are allowed to be licenced under the Apache-2.0 licence and all of your conduct is -in line with the Contributor's Covenant, and conduwuit's Code of Conduct. +in line with the Contributor's Covenant, and continuwuity's Code of Conduct. Contribution by users who violate either of these code of conducts will not have their contributions accepted. This includes users who have been banned from -conduwuit Matrix rooms for Code of Conduct violations. +continuwuityMatrix rooms for Code of Conduct violations. -[issues]: https://github.com/girlbossceo/conduwuit/issues -[conduwuit-matrix]: https://matrix.to/#/#conduwuit:puppygock.gay +[issues]: https://forgejo.ellis.link/continuwuation/continuwuity/issues +[continuwuity-matrix]: https://matrix.to/#/#continuwuity:continuwuity.org [complement]: https://github.com/matrix-org/complement/ -[engage.toml]: https://github.com/girlbossceo/conduwuit/blob/main/engage.toml +[engage.toml]: https://forgejo.ellis.link/continuwuation/continuwuity/src/branch/main/engage.toml [engage]: https://charles.page.computer.surgery/engage/ [sytest]: https://github.com/matrix-org/sytest/ [cargo-deb]: https://github.com/kornelski/cargo-deb @@ -146,4 +141,4 @@ conduwuit Matrix rooms for Code of Conduct violations. [cargo-audit]: https://github.com/RustSec/rustsec/tree/main/cargo-audit [direnv]: https://direnv.net/ [mdbook]: https://rust-lang.github.io/mdBook/ -[documentation.yml]: https://github.com/girlbossceo/conduwuit/blob/main/.github/workflows/documentation.yml +[documentation.yml]: https://forgejo.ellis.link/continuwuation/continuwuity/src/branch/main/.forgejo/workflows/documentation.yml From edd5fc6c7ebe7c2b01d139f5bf34c97b5ac7d653 Mon Sep 17 00:00:00 2001 From: Glandos Date: Sun, 27 Apr 2025 18:52:20 +0000 Subject: [PATCH 1757/2091] Actualiser debian/conduwuit.service --- debian/conduwuit.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/conduwuit.service b/debian/conduwuit.service index a079499e..3d2fbc9b 100644 --- a/debian/conduwuit.service +++ b/debian/conduwuit.service @@ -3,7 +3,7 @@ Description=conduwuit Matrix homeserver Wants=network-online.target After=network-online.target Alias=matrix-conduwuit.service -Documentation=https://conduwuit.puppyirl.gay/ +Documentation=https://continuwuity.org/ [Service] DynamicUser=yes From 4158c1cf623a83b96d6a2d3cabb9f6aa1d618b4b Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 28 Apr 2025 20:45:08 +0100 Subject: [PATCH 1758/2091] fix: Hack around software treating empty join rule incorrectly --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 216114af..2d8a2d0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3652,7 +3652,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "assign", "js_int", @@ -3672,7 +3672,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "js_int", "ruma-common", @@ -3684,7 +3684,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "as_variant", "assign", @@ -3707,7 +3707,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "as_variant", "base64 0.22.1", @@ -3739,7 +3739,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "as_variant", "indexmap 2.8.0", @@ -3764,7 +3764,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "bytes", "headers", @@ -3786,7 +3786,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3795,7 +3795,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "js_int", "ruma-common", @@ -3805,7 +3805,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3820,7 +3820,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "js_int", "ruma-common", @@ -3832,7 +3832,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=652cc4864203ab7ca60cf9c47b931c0385304cc7#652cc4864203ab7ca60cf9c47b931c0385304cc7" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index c0f857c6..1ce5c1db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,7 +350,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://forgejo.ellis.link/continuwuation/ruwuma" #branch = "conduwuit-changes" -rev = "652cc4864203ab7ca60cf9c47b931c0385304cc7" +rev = "d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" features = [ "compat", "rand", From e1655edd83893946cace9a44efb979a7357736b6 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 25 Apr 2025 02:47:48 +0100 Subject: [PATCH 1759/2091] feat: HTML default page --- .dockerignore | 2 +- .forgejo/workflows/release-image.yml | 3 + Cargo.lock | 64 ++++++++++++++++ Cargo.toml | 5 ++ docker/Dockerfile | 106 +++++++++++++++++++++++++-- nix/pkgs/main/default.nix | 2 + src/router/Cargo.toml | 1 + src/router/router.rs | 6 +- src/web/Cargo.toml | 32 ++++++++ src/web/css/index.css | 68 +++++++++++++++++ src/web/mod.rs | 61 +++++++++++++++ src/web/templates/_layout.html.j2 | 32 ++++++++ src/web/templates/error.html.j2 | 20 +++++ src/web/templates/index.html.j2 | 16 ++++ 14 files changed, 408 insertions(+), 10 deletions(-) create mode 100644 src/web/Cargo.toml create mode 100644 src/web/css/index.css create mode 100644 src/web/mod.rs create mode 100644 src/web/templates/_layout.html.j2 create mode 100644 src/web/templates/error.html.j2 create mode 100644 src/web/templates/index.html.j2 diff --git a/.dockerignore b/.dockerignore index 453634df..8ca2e3f8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,7 +11,7 @@ docker/ *.iml # Git folder -.git +# .git .gitea .gitlab .github diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 141bfef9..3eb84223 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -144,6 +144,9 @@ jobs: file: "docker/Dockerfile" build-args: | CONDUWUIT_VERSION_EXTRA=${{ env.COMMIT_SHORT_SHA }} + COMMIT_SHA=${{ github.sha }}) + REMOTE_URL=${{github.event.repository.html_url }} + REMOTE_COMMIT_URL=${{github.event.head_commit.url }} platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} diff --git a/Cargo.lock b/Cargo.lock index 2d8a2d0f..1b6c5d26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -109,6 +109,48 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9dbc3a507a82b17ba0d98f6ce8fd6954ea0c8152e98009d36a40d8dcc8ce078a" +[[package]] +name = "askama" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75363874b771be265f4ffe307ca705ef6f3baa19011c149da8674a87f1b75c4" +dependencies = [ + "askama_derive", + "itoa", + "percent-encoding", + "serde", + "serde_json", +] + +[[package]] +name = "askama_derive" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "129397200fe83088e8a68407a8e2b1f826cf0086b21ccdb866a722c8bcd3a94f" +dependencies = [ + "askama_parser", + "basic-toml", + "memchr", + "proc-macro2", + "quote", + "rustc-hash 2.1.1", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "askama_parser" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ab5630b3d5eaf232620167977f95eb51f3432fc76852328774afbd242d4358" +dependencies = [ + "memchr", + "serde", + "serde_derive", + "winnow", +] + [[package]] name = "assign" version = "1.1.1" @@ -415,6 +457,15 @@ version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" +[[package]] +name = "basic-toml" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a" +dependencies = [ + "serde", +] + [[package]] name = "bindgen" version = "0.69.5" @@ -904,6 +955,7 @@ dependencies = [ "conduwuit_api", "conduwuit_core", "conduwuit_service", + "conduwuit_web", "const-str", "futures", "http", @@ -961,6 +1013,18 @@ dependencies = [ "webpage", ] +[[package]] +name = "conduwuit_web" +version = "0.5.0-rc.5" +dependencies = [ + "askama", + "axum", + "futures", + "rand 0.8.5", + "thiserror 2.0.12", + "tracing", +] + [[package]] name = "console-api" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index 1ce5c1db..44db4c4b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -626,6 +626,11 @@ package = "conduwuit_macros" path = "src/macros" default-features = false +[workspace.dependencies.conduwuit-web] +package = "conduwuit_web" +path = "src/web" +default-features = false + ############################################################################### # # Release profiles diff --git a/docker/Dockerfile b/docker/Dockerfile index 536af632..56f2d5fa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -111,14 +111,112 @@ RUN mkdir /out FROM toolchain AS builder + +# Get source +COPY . . + # Conduwuit version info ARG COMMIT_SHA= +ARG SHORT_COMMIT_SHA= +ARG REMOTE_URL= ARG CONDUWUIT_VERSION_EXTRA= +ENV COMMIT_SHA=$COMMIT_SHA +ENV SHORT_COMMIT_SHA=$SHORT_COMMIT_SHA +ENV REMOTE_URL=$REMOTE_URL ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA -RUN <> /etc/environment + +# Calculate version info from git if not provided via ARGs +# and write all relevant vars to /etc/environment +RUN <<'EOF' +set -e # Exit on error + +# Use temp variables to store calculated values +calculated_commit_sha="" +calculated_remote_url="" +calculated_version_extra="" + +# --- COMMIT_SHA --- +# Calculate COMMIT_SHA if ENV var (from ARG) is empty +if [ -z "${COMMIT_SHA}" ]; then + # Try to get short commit hash from git + calculated_commit_sha=$(git rev-parse HEAD 2>/dev/null || echo "") + if [ -n "${calculated_commit_sha}" ]; then + echo "COMMIT_SHA='${calculated_commit_sha}'" >> /etc/environment + fi +else + # Ensure ARG-provided value is in /etc/environment + echo "COMMIT_SHA='${COMMIT_SHA}'" >> /etc/environment fi + + +if [ -z "${SHORT_COMMIT_SHA}" ]; then + # Try to get short commit hash from git + calculated_short_commit_sha=$(git rev-parse --short HEAD 2>/dev/null || echo "") + if [ -n "${calculated_short_commit_sha}" ]; then + echo "SHORT_COMMIT_SHA='${calculated_short_commit_sha}'" >> /etc/environment + fi +else + # Ensure ARG-provided value is in /etc/environment + echo "SHORT_COMMIT_SHA='${SHORT_COMMIT_SHA}'" >> /etc/environment +fi + +# --- REMOTE_URL --- +# Calculate REMOTE_URL if ENV var (from ARG) is empty +if [ -z "${REMOTE_URL}" ]; then + # Try to get remote origin URL from git + remote_url_raw=$(git config --get remote.origin.url 2>/dev/null || echo "") + if [ -n "${remote_url_raw}" ]; then + # Transform git URL (SSH or HTTPS) to web URL + if [[ $remote_url_raw == "https://"* ]]; then + # Already HTTPS, just remove .git suffix + calculated_remote_url=$(echo "$remote_url_raw" | sed 's/\.git$//') + else + # Convert SSH URL to HTTPS URL + calculated_remote_url=$(echo "$remote_url_raw" | sed 's/\.git$//' | sed 's/:/\//' | sed 's/^git@/https:\/\//') + fi + + # Write calculated web URL if transformation was successful + if [ -n "${calculated_remote_url}" ]; then + echo "REMOTE_URL='${calculated_remote_url}'" >> /etc/environment + fi + fi +else + # Ensure ARG-provided value is in /etc/environment (assume it's a valid web URL) + echo "REMOTE_URL='${REMOTE_URL}'" >> /etc/environment + # Use provided value for REMOTE_COMMIT_URL calculation below + calculated_remote_url="${REMOTE_URL}" +fi + +# --- Determine effective values for subsequent calculations --- +# Use ENV var value if set (from ARG), otherwise use calculated value +effective_commit_sha="${COMMIT_SHA:-$calculated_commit_sha}" +effective_short_commit_sha="${SHORT_COMMIT_SHA:-$calculated_short_commit_sha}" +effective_remote_url="${REMOTE_URL:-$calculated_remote_url}" + +# --- REMOTE_COMMIT_URL --- +# Calculate and write REMOTE_COMMIT_URL if both components are available +if [ -z "${REMOTE_COMMIT_URL}" ] && [ -n "${effective_remote_url}" ] && [ -n "${effective_commit_sha}" ]; then + echo "REMOTE_COMMIT_URL='${effective_remote_url}/commit/${effective_commit_sha}'" >> /etc/environment +else + # Ensure ARG-provided value is in /etc/environment + echo "REMOTE_COMMIT_URL='${REMOTE_COMMIT_URL}'" >> /etc/environment +fi + +# --- CONDUWUIT_VERSION_EXTRA --- +# Calculate CONDUWUIT_VERSION_EXTRA if ENV var (from ARG) is empty +if [ -z "${CONDUWUIT_VERSION_EXTRA}" ]; then + # Use the effective short commit sha, fallback to "unknown revision" + calculated_version_extra="${effective_short_commit_sha:-unknown revision}" + # Handle case where commit sha calculation failed and ARG wasn't set + if [ -z "${calculated_version_extra}" ]; then + calculated_version_extra="unknown revision" + fi + echo "CONDUWUIT_VERSION_EXTRA='${calculated_version_extra}'" >> /etc/environment +else + # Ensure ARG-provided value is in /etc/environment + echo "CONDUWUIT_VERSION_EXTRA='${CONDUWUIT_VERSION_EXTRA}'" >> /etc/environment +fi + EOF ARG TARGETPLATFORM @@ -127,8 +225,6 @@ ARG TARGETPLATFORM RUN cat /etc/environment RUN xx-cargo --print-target-triple -# Get source -COPY . . # Build the binary RUN --mount=type=cache,target=/usr/local/cargo/registry \ diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 9c8038a7..3a43af5a 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -130,6 +130,8 @@ buildDepsOnlyEnv = }); buildPackageEnv = { + COMMIT_SHA = inputs.self.rev or inputs.self.dirtyRev or ""; + SHORT_COMMIT_SHA = inputs.self.shortRev or inputs.self.dirtyShortRev or ""; CONDUWUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev or ""; } // buildDepsOnlyEnv // { # Only needed in static stdenv because these are transitive dependencies of rocksdb diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index e4ddcb9b..9fcb8d6a 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -103,6 +103,7 @@ conduwuit-admin.workspace = true conduwuit-api.workspace = true conduwuit-core.workspace = true conduwuit-service.workspace = true +conduwuit-web.workspace = true const-str.workspace = true futures.workspace = true http.workspace = true diff --git a/src/router/router.rs b/src/router/router.rs index 0f95b924..2b9ce2ec 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use axum::{Router, response::IntoResponse, routing::get}; +use axum::{Router, response::IntoResponse}; use conduwuit::Error; use conduwuit_api::router::{state, state::Guard}; use conduwuit_service::Services; @@ -11,7 +11,7 @@ pub(crate) fn build(services: &Arc) -> (Router, Guard) { let router = Router::::new(); let (state, guard) = state::create(services.clone()); let router = conduwuit_api::router::build(router, &services.server) - .route("/", get(it_works)) + .merge(conduwuit_web::build::().with_state(())) .fallback(not_found) .with_state(state); @@ -21,5 +21,3 @@ pub(crate) fn build(services: &Arc) -> (Router, Guard) { async fn not_found(_uri: Uri) -> impl IntoResponse { Error::Request(ErrorKind::Unrecognized, "Not Found".into(), StatusCode::NOT_FOUND) } - -async fn it_works() -> &'static str { "hewwo from conduwuit woof!" } diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml new file mode 100644 index 00000000..07dde0f7 --- /dev/null +++ b/src/web/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "conduwuit_web" +categories.workspace = true +description.workspace = true +edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true + +[lib] +path = "mod.rs" +crate-type = [ + "rlib", +# "dylib", +] + +[features] + + +[dependencies] +askama = "0.14.0" + +axum.workspace = true +futures.workspace = true +tracing.workspace = true +rand.workspace = true +thiserror.workspace = true + +[lints] +workspace = true diff --git a/src/web/css/index.css b/src/web/css/index.css new file mode 100644 index 00000000..86cb6d8d --- /dev/null +++ b/src/web/css/index.css @@ -0,0 +1,68 @@ +:root { + color-scheme: light; + --font-stack: sans-serif; + + --background-color: #fff; + --text-color: #000; + + --bg: oklch(0.76 0.0854 317.27); + --panel-bg: oklch(0.91 0.042 317.27); + + --name-lightness: 0.45; + + @media (prefers-color-scheme: dark) { + color-scheme: dark; + --text-color: #fff; + --bg: oklch(0.15 0.042 317.27); + --panel-bg: oklch(0.24 0.03 317.27); + + --name-lightness: 0.8; + } + + --c1: oklch(0.44 0.177 353.06); + --c2: oklch(0.59 0.158 150.88); + + --normal-font-size: 1rem; + --small-font-size: 0.8rem; +} + +body { + color: var(--text-color); + font-family: var(--font-stack); + margin: 0; + padding: 0; + display: grid; + place-items: center; + min-height: 100vh; +} + +html { + background-color: var(--bg); + background-image: linear-gradient( + 70deg, + oklch(from var(--bg) l + 0.2 c h), + oklch(from var(--bg) l - 0.2 c h) + ); + font-size: 16px; +} + +.panel { + width: min(clamp(24rem, 12rem + 40vw, 48rem), 100vw); + border-radius: 15px; + background-color: var(--panel-bg); + padding-inline: 1.5rem; + padding-block: 1rem; + box-shadow: 0 0.25em 0.375em hsla(0, 0%, 0%, 0.1); +} + +.project-name { + text-decoration: none; + background: linear-gradient( + 130deg, + oklch(from var(--c1) var(--name-lightness) c h), + oklch(from var(--c2) var(--name-lightness) c h) + ); + background-clip: text; + color: transparent; + filter: brightness(1.2); +} diff --git a/src/web/mod.rs b/src/web/mod.rs new file mode 100644 index 00000000..ddf13be4 --- /dev/null +++ b/src/web/mod.rs @@ -0,0 +1,61 @@ +use askama::Template; +use axum::{ + Router, + http::{StatusCode, header}, + response::{Html, IntoResponse, Response}, + routing::get, +}; + +pub fn build() -> Router<()> { Router::new().route("/", get(index_handler)) } + +async fn index_handler() -> Result { + #[derive(Debug, Template)] + #[template(path = "index.html.j2")] + struct Tmpl<'a> { + nonce: &'a str, + } + let nonce = rand::random::().to_string(); + + let template = Tmpl { nonce: &nonce }; + Ok(( + [(header::CONTENT_SECURITY_POLICY, format!("default-src 'none' 'nonce-{nonce}';"))], + Html(template.render()?), + )) +} + +#[derive(Debug, thiserror::Error)] +enum WebError { + #[error("Failed to render template: {0}")] + Render(#[from] askama::Error), +} + +impl IntoResponse for WebError { + fn into_response(self) -> Response { + #[derive(Debug, Template)] + #[template(path = "error.html.j2")] + struct Tmpl<'a> { + nonce: &'a str, + err: WebError, + } + + let nonce = rand::random::().to_string(); + + let status = match &self { + | Self::Render(_) => StatusCode::INTERNAL_SERVER_ERROR, + }; + let tmpl = Tmpl { nonce: &nonce, err: self }; + if let Ok(body) = tmpl.render() { + ( + status, + [( + header::CONTENT_SECURITY_POLICY, + format!("default-src 'none' 'nonce-{nonce}';"), + )], + Html(body), + ) + .into_response() + } else { + (status, "Something went wrong").into_response() + } + } +} diff --git a/src/web/templates/_layout.html.j2 b/src/web/templates/_layout.html.j2 new file mode 100644 index 00000000..a38745e1 --- /dev/null +++ b/src/web/templates/_layout.html.j2 @@ -0,0 +1,32 @@ + + + + + + {% block title %}Continuwuity{% endblock %} + + + + + + +
    {%~ block content %}{% endblock ~%}
    + {%~ block footer ~%} +
    +

    Powered by Continuwuity + {%~ if let Some(version_info) = option_env!("CONDUWUIT_VERSION_EXTRA").or(option_env!("SHORT_COMMIT_SHA")) ~%} + {%~ if let Some(url) = option_env!("REMOTE_COMMIT_URL").or(option_env!("REMOTE_URL")) ~%} + ({{ version_info }}) + {%~ else ~%} + ({{ version_info }}) + {%~ endif ~%} + {%~ endif ~%}

    +
    + {%~ endblock ~%} + + + diff --git a/src/web/templates/error.html.j2 b/src/web/templates/error.html.j2 new file mode 100644 index 00000000..e320d0ed --- /dev/null +++ b/src/web/templates/error.html.j2 @@ -0,0 +1,20 @@ +{% extends "_layout.html.j2" %} + +{%- block title -%} +Server Error +{%- endblock -%} + +{%- block content -%} +

    + {%- match err -%} + {% else -%} 500: Internal Server Error + {%- endmatch -%} +

    + +{%- match err -%} + {% when WebError::Render(err) -%} + 
    {{ err }}
    + {% else -%}

    An error occurred

    +{%- endmatch -%} + +{%- endblock -%} diff --git a/src/web/templates/index.html.j2 b/src/web/templates/index.html.j2 new file mode 100644 index 00000000..51393822 --- /dev/null +++ b/src/web/templates/index.html.j2 @@ -0,0 +1,16 @@ +{% extends "_layout.html.j2" %} +{%- block content -%} +
    +
    +

    Welcome to Continuwuity!

    +

    Continuwuity is successfully installed and working.

    +

    To get started, you can:

    + +
    + +{%- endblock content -%} From a98da7d9422500de592824eee0f441bc94803065 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 1 May 2025 00:38:35 +0100 Subject: [PATCH 1760/2091] refactor: Move git version info gather in into a build script --- .forgejo/workflows/release-image.yml | 8 +- Cargo.lock | 44 ++++++++++ Cargo.toml | 6 ++ docker/Dockerfile | 118 ++++----------------------- nix/pkgs/main/default.nix | 5 +- src/api/client/message.rs | 5 +- src/build_metadata/Cargo.toml | 34 ++++++++ src/build_metadata/build.rs | 92 +++++++++++++++++++++ src/build_metadata/mod.rs | 23 ++++++ src/core/Cargo.toml | 1 + src/core/info/version.rs | 11 +-- src/web/Cargo.toml | 2 + src/web/mod.rs | 1 + src/web/templates/_layout.html.j2 | 4 +- 14 files changed, 231 insertions(+), 123 deletions(-) create mode 100644 src/build_metadata/Cargo.toml create mode 100644 src/build_metadata/build.rs create mode 100644 src/build_metadata/mod.rs diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 3eb84223..de009ad5 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -143,10 +143,10 @@ jobs: context: . file: "docker/Dockerfile" build-args: | - CONDUWUIT_VERSION_EXTRA=${{ env.COMMIT_SHORT_SHA }} - COMMIT_SHA=${{ github.sha }}) - REMOTE_URL=${{github.event.repository.html_url }} - REMOTE_COMMIT_URL=${{github.event.head_commit.url }} + GIT_COMMIT_HASH=${{ github.sha }}) + GIT_COMMIT_HASH_SHORT=${{ env.COMMIT_SHORT_SHA }}) + GIT_REMOTE_URL=${{github.event.repository.html_url }} + GIT_REMOTE_COMMIT_URL=${{github.event.head_commit.url }} platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} diff --git a/Cargo.lock b/Cargo.lock index 1b6c5d26..5f82384e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -584,6 +584,9 @@ name = "built" version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" +dependencies = [ + "cargo-lock", +] [[package]] name = "bumpalo" @@ -631,6 +634,19 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "cargo-lock" +version = "10.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06acb4f71407ba205a07cb453211e0e6a67b21904e47f6ba1f9589e38f2e454" +dependencies = [ + "petgraph", + "semver", + "serde", + "toml", + "url", +] + [[package]] name = "cargo_toml" version = "0.21.0" @@ -856,6 +872,13 @@ dependencies = [ "tracing", ] +[[package]] +name = "conduwuit_build_metadata" +version = "0.5.0-rc.5" +dependencies = [ + "built", +] + [[package]] name = "conduwuit_core" version = "0.5.0-rc.5" @@ -870,6 +893,7 @@ dependencies = [ "checked_ops", "chrono", "clap", + "conduwuit_build_metadata", "conduwuit_macros", "const-str", "core_affinity", @@ -1019,6 +1043,7 @@ version = "0.5.0-rc.5" dependencies = [ "askama", "axum", + "conduwuit_build_metadata", "futures", "rand 0.8.5", "thiserror 2.0.12", @@ -1515,6 +1540,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.1.1" @@ -3132,6 +3163,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.8.0", +] + [[package]] name = "phf" version = "0.11.3" @@ -4127,6 +4168,9 @@ name = "semver" version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +dependencies = [ + "serde", +] [[package]] name = "sentry" diff --git a/Cargo.toml b/Cargo.toml index 44db4c4b..43cd3f4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -631,6 +631,12 @@ package = "conduwuit_web" path = "src/web" default-features = false + +[workspace.dependencies.conduwuit-build-metadata] +package = "conduwuit_build_metadata" +path = "src/build_metadata" +default-features = false + ############################################################################### # # Release profiles diff --git a/docker/Dockerfile b/docker/Dockerfile index 56f2d5fa..3029282f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -115,116 +115,26 @@ FROM toolchain AS builder # Get source COPY . . -# Conduwuit version info -ARG COMMIT_SHA= -ARG SHORT_COMMIT_SHA= -ARG REMOTE_URL= -ARG CONDUWUIT_VERSION_EXTRA= -ENV COMMIT_SHA=$COMMIT_SHA -ENV SHORT_COMMIT_SHA=$SHORT_COMMIT_SHA -ENV REMOTE_URL=$REMOTE_URL -ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA - -# Calculate version info from git if not provided via ARGs -# and write all relevant vars to /etc/environment -RUN <<'EOF' -set -e # Exit on error - -# Use temp variables to store calculated values -calculated_commit_sha="" -calculated_remote_url="" -calculated_version_extra="" - -# --- COMMIT_SHA --- -# Calculate COMMIT_SHA if ENV var (from ARG) is empty -if [ -z "${COMMIT_SHA}" ]; then - # Try to get short commit hash from git - calculated_commit_sha=$(git rev-parse HEAD 2>/dev/null || echo "") - if [ -n "${calculated_commit_sha}" ]; then - echo "COMMIT_SHA='${calculated_commit_sha}'" >> /etc/environment - fi -else - # Ensure ARG-provided value is in /etc/environment - echo "COMMIT_SHA='${COMMIT_SHA}'" >> /etc/environment -fi - - -if [ -z "${SHORT_COMMIT_SHA}" ]; then - # Try to get short commit hash from git - calculated_short_commit_sha=$(git rev-parse --short HEAD 2>/dev/null || echo "") - if [ -n "${calculated_short_commit_sha}" ]; then - echo "SHORT_COMMIT_SHA='${calculated_short_commit_sha}'" >> /etc/environment - fi -else - # Ensure ARG-provided value is in /etc/environment - echo "SHORT_COMMIT_SHA='${SHORT_COMMIT_SHA}'" >> /etc/environment -fi - -# --- REMOTE_URL --- -# Calculate REMOTE_URL if ENV var (from ARG) is empty -if [ -z "${REMOTE_URL}" ]; then - # Try to get remote origin URL from git - remote_url_raw=$(git config --get remote.origin.url 2>/dev/null || echo "") - if [ -n "${remote_url_raw}" ]; then - # Transform git URL (SSH or HTTPS) to web URL - if [[ $remote_url_raw == "https://"* ]]; then - # Already HTTPS, just remove .git suffix - calculated_remote_url=$(echo "$remote_url_raw" | sed 's/\.git$//') - else - # Convert SSH URL to HTTPS URL - calculated_remote_url=$(echo "$remote_url_raw" | sed 's/\.git$//' | sed 's/:/\//' | sed 's/^git@/https:\/\//') - fi - - # Write calculated web URL if transformation was successful - if [ -n "${calculated_remote_url}" ]; then - echo "REMOTE_URL='${calculated_remote_url}'" >> /etc/environment - fi - fi -else - # Ensure ARG-provided value is in /etc/environment (assume it's a valid web URL) - echo "REMOTE_URL='${REMOTE_URL}'" >> /etc/environment - # Use provided value for REMOTE_COMMIT_URL calculation below - calculated_remote_url="${REMOTE_URL}" -fi - -# --- Determine effective values for subsequent calculations --- -# Use ENV var value if set (from ARG), otherwise use calculated value -effective_commit_sha="${COMMIT_SHA:-$calculated_commit_sha}" -effective_short_commit_sha="${SHORT_COMMIT_SHA:-$calculated_short_commit_sha}" -effective_remote_url="${REMOTE_URL:-$calculated_remote_url}" - -# --- REMOTE_COMMIT_URL --- -# Calculate and write REMOTE_COMMIT_URL if both components are available -if [ -z "${REMOTE_COMMIT_URL}" ] && [ -n "${effective_remote_url}" ] && [ -n "${effective_commit_sha}" ]; then - echo "REMOTE_COMMIT_URL='${effective_remote_url}/commit/${effective_commit_sha}'" >> /etc/environment -else - # Ensure ARG-provided value is in /etc/environment - echo "REMOTE_COMMIT_URL='${REMOTE_COMMIT_URL}'" >> /etc/environment -fi - -# --- CONDUWUIT_VERSION_EXTRA --- -# Calculate CONDUWUIT_VERSION_EXTRA if ENV var (from ARG) is empty -if [ -z "${CONDUWUIT_VERSION_EXTRA}" ]; then - # Use the effective short commit sha, fallback to "unknown revision" - calculated_version_extra="${effective_short_commit_sha:-unknown revision}" - # Handle case where commit sha calculation failed and ARG wasn't set - if [ -z "${calculated_version_extra}" ]; then - calculated_version_extra="unknown revision" - fi - echo "CONDUWUIT_VERSION_EXTRA='${calculated_version_extra}'" >> /etc/environment -else - # Ensure ARG-provided value is in /etc/environment - echo "CONDUWUIT_VERSION_EXTRA='${CONDUWUIT_VERSION_EXTRA}'" >> /etc/environment -fi - -EOF - ARG TARGETPLATFORM # Verify environment configuration RUN cat /etc/environment RUN xx-cargo --print-target-triple +# Conduwuit version info +ARG GIT_COMMIT_HASH= +ARG GIT_COMMIT_HASH_SHORT= +ARG GIT_REMOTE_URL= +ARG GIT_REMOTE_COMMIT_URL= +ARG CONDUWUIT_VERSION_EXTRA= +ARG CONTINUWUITY_VERSION_EXTRA= +ENV GIT_COMMIT_HASH=$GIT_COMMIT_HASH +ENV GIT_COMMIT_HASH_SHORT=$GIT_COMMIT_HASH_SHORT +ENV GIT_REMOTE_URL=$GIT_REMOTE_URL +ENV GIT_REMOTE_COMMIT_URL=$GIT_REMOTE_COMMIT_URL +ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA +ENV CONTINUWUITY_VERSION_EXTRA=$CONTINUWUITY_VERSION_EXTRA + # Build the binary RUN --mount=type=cache,target=/usr/local/cargo/registry \ diff --git a/nix/pkgs/main/default.nix b/nix/pkgs/main/default.nix index 3a43af5a..f2fffec0 100644 --- a/nix/pkgs/main/default.nix +++ b/nix/pkgs/main/default.nix @@ -130,9 +130,8 @@ buildDepsOnlyEnv = }); buildPackageEnv = { - COMMIT_SHA = inputs.self.rev or inputs.self.dirtyRev or ""; - SHORT_COMMIT_SHA = inputs.self.shortRev or inputs.self.dirtyShortRev or ""; - CONDUWUIT_VERSION_EXTRA = inputs.self.shortRev or inputs.self.dirtyShortRev or ""; + GIT_COMMIT_HASH = inputs.self.rev or inputs.self.dirtyRev or ""; + GIT_COMMIT_HASH_SHORT = inputs.self.shortRev or inputs.self.dirtyShortRev or ""; } // buildDepsOnlyEnv // { # Only needed in static stdenv because these are transitive dependencies of rocksdb CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 16b1796a..e442850b 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -143,7 +143,10 @@ pub(crate) async fn get_message_events_route( if let Some(registration) = body.appservice_info.as_ref() { <&DeviceId>::from(registration.registration.id.as_str()) } else { - panic!("No device_id provided and no appservice registration found, this should be unreachable"); + panic!( + "No device_id provided and no appservice registration found, this \ + should be unreachable" + ); }, }, room_id, diff --git a/src/build_metadata/Cargo.toml b/src/build_metadata/Cargo.toml new file mode 100644 index 00000000..3a98c6bf --- /dev/null +++ b/src/build_metadata/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "conduwuit_build_metadata" +categories.workspace = true +description.workspace = true +edition.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +version.workspace = true + + +build = "build.rs" +# [[bin]] +# path = "main.rs" +# name = "conduwuit_build_metadata" + +[lib] +path = "mod.rs" +crate-type = [ + "rlib", +# "dylib", +] + +[features] + + +[dependencies] + +[build-dependencies] +built = {version = "0.7", features = ["cargo-lock", "dependency-tree"]} + +[lints] +workspace = true diff --git a/src/build_metadata/build.rs b/src/build_metadata/build.rs new file mode 100644 index 00000000..2fec16a7 --- /dev/null +++ b/src/build_metadata/build.rs @@ -0,0 +1,92 @@ +use std::process::Command; + +fn run_git_command(args: &[&str]) -> Option { + Command::new("git") + .args(args) + .output() + .ok() + .filter(|output| output.status.success()) + .and_then(|output| String::from_utf8(output.stdout).ok()) + .map(|s| s.trim().to_owned()) + .filter(|s| !s.is_empty()) +} +fn get_env(env_var: &str) -> Option { + match std::env::var(env_var) { + | Ok(val) if !val.is_empty() => Some(val), + | _ => None, + } +} +fn main() { + // built gets the default crate from the workspace. Not sure if this is intended + // behavior, but it's what we want. + built::write_built_file().expect("Failed to acquire build-time information"); + + // --- Git Information --- + let mut commit_hash = None; + let mut commit_hash_short = None; + let mut remote_url_web = None; + + // Get full commit hash + if let Some(hash) = + get_env("GIT_COMMIT_HASH").or_else(|| run_git_command(&["rev-parse", "HEAD"])) + { + println!("cargo:rustc-env=GIT_COMMIT_HASH={hash}"); + commit_hash = Some(hash); + } + + // Get short commit hash + if let Some(short_hash) = get_env("GIT_COMMIT_HASH_SHORT") + .or_else(|| run_git_command(&["rev-parse", "--short", "HEAD"])) + { + println!("cargo:rustc-env=GIT_COMMIT_HASH_SHORT={short_hash}"); + commit_hash_short = Some(short_hash); + } + + // Get remote URL and convert to web URL + if let Some(remote_url_raw) = get_env("GIT_REMOTE_URL") + .or_else(|| run_git_command(&["config", "--get", "remote.origin.url"])) + { + println!("cargo:rustc-env=GIT_REMOTE_URL={remote_url_raw}"); + let web_url = if remote_url_raw.starts_with("https://") { + remote_url_raw.trim_end_matches(".git").to_owned() + } else if remote_url_raw.starts_with("git@") { + remote_url_raw + .trim_end_matches(".git") + .replacen(':', "/", 1) + .replacen("git@", "https://", 1) + } else if remote_url_raw.starts_with("ssh://") { + remote_url_raw + .trim_end_matches(".git") + .replacen("git@", "", 1) + .replacen("ssh:", "https:", 1) + } else { + // Assume it's already a web URL or unknown format + remote_url_raw + }; + println!("cargo:rustc-env=GIT_REMOTE_WEB_URL={web_url}"); + remote_url_web = Some(web_url); + } + + // Construct remote commit URL + if let Some(remote_commit_url) = get_env("GIT_REMOTE_COMMIT_URL") { + println!("cargo:rustc-env=GIT_REMOTE_COMMIT_URL={remote_commit_url}"); + } else if let (Some(base_url), Some(hash)) = + (&remote_url_web, commit_hash.as_ref().or(commit_hash_short.as_ref())) + { + let commit_page = format!("{base_url}/commit/{hash}"); + println!("cargo:rustc-env=GIT_REMOTE_COMMIT_URL={commit_page}"); + } + + // --- Rerun Triggers --- + // Rerun if the git HEAD changes + println!("cargo:rerun-if-changed=.git/HEAD"); + // Rerun if the ref pointed to by HEAD changes (e.g., new commit on branch) + if let Some(ref_path) = run_git_command(&["symbolic-ref", "--quiet", "HEAD"]) { + println!("cargo:rerun-if-changed=.git/{ref_path}"); + } + + println!("cargo:rerun-if-env-changed=GIT_COMMIT_HASH"); + println!("cargo:rerun-if-env-changed=GIT_COMMIT_HASH_SHORT"); + println!("cargo:rerun-if-env-changed=GIT_REMOTE_URL"); + println!("cargo:rerun-if-env-changed=GIT_REMOTE_COMMIT_URL"); +} diff --git a/src/build_metadata/mod.rs b/src/build_metadata/mod.rs new file mode 100644 index 00000000..cf3364c1 --- /dev/null +++ b/src/build_metadata/mod.rs @@ -0,0 +1,23 @@ +pub mod built { + include!(concat!(env!("OUT_DIR"), "/built.rs")); +} + +pub static GIT_COMMIT_HASH: Option<&str> = option_env!("GIT_COMMIT_HASH"); + +pub static GIT_COMMIT_HASH_SHORT: Option<&str> = option_env!("GIT_COMMIT_HASH_SHORT"); + +// this would be a lot better if Option::or was const. +pub static VERSION_EXTRA: Option<&str> = + if let v @ Some(_) = option_env!("CONTINUWUITY_VERSION_EXTRA") { + v + } else if let v @ Some(_) = option_env!("CONDUWUIT_VERSION_EXTRA") { + v + } else if let v @ Some(_) = option_env!("CONDUIT_VERSION_EXTRA") { + v + } else { + GIT_COMMIT_HASH_SHORT + }; +pub static GIT_REMOTE_WEB_URL: Option<&str> = option_env!("GIT_REMOTE_WEB_URL"); +pub static GIT_REMOTE_COMMIT_URL: Option<&str> = option_env!("GIT_REMOTE_COMMIT_URL"); + +// TODO: Mark dirty builds within the version string diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index f42b049b..0c33c590 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -67,6 +67,7 @@ checked_ops.workspace = true chrono.workspace = true clap.workspace = true conduwuit-macros.workspace = true +conduwuit-build-metadata.workspace = true const-str.workspace = true core_affinity.workspace = true ctor.workspace = true diff --git a/src/core/info/version.rs b/src/core/info/version.rs index 6abb6e13..523c40a2 100644 --- a/src/core/info/version.rs +++ b/src/core/info/version.rs @@ -26,13 +26,6 @@ pub fn user_agent() -> &'static str { USER_AGENT.get_or_init(init_user_agent) } fn init_user_agent() -> String { format!("{}/{}", name(), version()) } fn init_version() -> String { - option_env!("CONDUWUIT_VERSION_EXTRA") - .or(option_env!("CONDUIT_VERSION_EXTRA")) - .map_or(SEMANTIC.to_owned(), |extra| { - if extra.is_empty() { - SEMANTIC.to_owned() - } else { - format!("{SEMANTIC} ({extra})") - } - }) + conduwuit_build_metadata::VERSION_EXTRA + .map_or(SEMANTIC.to_owned(), |extra| format!("{SEMANTIC} ({extra})")) } diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 07dde0f7..8e03a338 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -20,6 +20,8 @@ crate-type = [ [dependencies] +conduwuit-build-metadata.workspace = true + askama = "0.14.0" axum.workspace = true diff --git a/src/web/mod.rs b/src/web/mod.rs index ddf13be4..520a9a34 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -5,6 +5,7 @@ use axum::{ response::{Html, IntoResponse, Response}, routing::get, }; +use conduwuit_build_metadata::{GIT_REMOTE_COMMIT_URL, GIT_REMOTE_WEB_URL, VERSION_EXTRA}; pub fn build() -> Router<()> { Router::new().route("/", get(index_handler)) } diff --git a/src/web/templates/_layout.html.j2 b/src/web/templates/_layout.html.j2 index a38745e1..fd0a5b29 100644 --- a/src/web/templates/_layout.html.j2 +++ b/src/web/templates/_layout.html.j2 @@ -18,8 +18,8 @@ {%~ block footer ~%}

    Powered by Continuwuity - {%~ if let Some(version_info) = option_env!("CONDUWUIT_VERSION_EXTRA").or(option_env!("SHORT_COMMIT_SHA")) ~%} - {%~ if let Some(url) = option_env!("REMOTE_COMMIT_URL").or(option_env!("REMOTE_URL")) ~%} + {%~ if let Some(version_info) = VERSION_EXTRA ~%} + {%~ if let Some(url) = GIT_REMOTE_COMMIT_URL.or(GIT_REMOTE_WEB_URL) ~%} ({{ version_info }}) {%~ else ~%} ({{ version_info }}) From cbcf4300dffb55ad9b1ed5529ae6bf81ad446408 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 1 May 2025 00:47:03 +0100 Subject: [PATCH 1761/2091] ci: Cache timelord-cli to avoid unnecesary compilation --- .forgejo/workflows/release-image.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index de009ad5..69d7d2fd 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -80,17 +80,29 @@ jobs: run: echo '${{ toJSON(fromJSON(needs.define-variables.outputs.build_matrix)) }}' - name: Echo matrix run: echo '${{ toJSON(matrix) }}' - - name: Checkout repository - uses: actions/checkout@v4 - with: - persist-credentials: false - run: | if ! command -v rustup &> /dev/null ; then curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH fi - - uses: https://github.com/cargo-bins/cargo-binstall@main + + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Cache timelord-cli installation + id: cache-timelord-bin + uses: actions/cache@v3 + with: + path: ~/.cargo/bin/timelord + key: timelord-cli-v3.0.1 + - name: Install timelord-cli + uses: https://github.com/cargo-bins/cargo-binstall@main + if: steps.cache-timelord-bin.outputs.cache-hit != 'true' - run: cargo binstall timelord-cli@3.0.1 + if: steps.cache-timelord-bin.outputs.cache-hit != 'true' + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Set up QEMU From fb9d4c30f4e89227244fcb275cdbed6261607610 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 1 May 2025 18:46:30 +0100 Subject: [PATCH 1762/2091] feat: Prefill server name in federation test --- Cargo.lock | 1 + src/api/router.rs | 4 ++-- src/router/layers.rs | 3 +-- src/router/router.rs | 5 ++--- src/service/mod.rs | 1 + src/{api/router => service}/state.rs | 2 +- src/web/Cargo.toml | 1 + src/web/mod.rs | 17 ++++++++++++++--- src/web/templates/index.html.j2 | 2 +- 9 files changed, 24 insertions(+), 12 deletions(-) rename src/{api/router => service}/state.rs (98%) diff --git a/Cargo.lock b/Cargo.lock index 5f82384e..18375234 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1044,6 +1044,7 @@ dependencies = [ "askama", "axum", "conduwuit_build_metadata", + "conduwuit_service", "futures", "rand 0.8.5", "thiserror 2.0.12", diff --git a/src/api/router.rs b/src/api/router.rs index 3fbef275..5416e9e9 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -3,7 +3,6 @@ mod auth; mod handler; mod request; mod response; -pub mod state; use std::str::FromStr; @@ -13,10 +12,11 @@ use axum::{ routing::{any, get, post}, }; use conduwuit::{Server, err}; +pub(super) use conduwuit_service::state::State; use http::{Uri, uri}; use self::handler::RouterExt; -pub(super) use self::{args::Args as Ruma, response::RumaResponse, state::State}; +pub(super) use self::{args::Args as Ruma, response::RumaResponse}; use crate::{client, server}; pub fn build(router: Router, server: &Server) -> Router { diff --git a/src/router/layers.rs b/src/router/layers.rs index 6920555d..70f3a660 100644 --- a/src/router/layers.rs +++ b/src/router/layers.rs @@ -6,8 +6,7 @@ use axum::{ }; use axum_client_ip::SecureClientIpSource; use conduwuit::{Result, Server, debug, error}; -use conduwuit_api::router::state::Guard; -use conduwuit_service::Services; +use conduwuit_service::{Services, state::Guard}; use http::{ HeaderValue, Method, StatusCode, header::{self, HeaderName}, diff --git a/src/router/router.rs b/src/router/router.rs index 2b9ce2ec..fdaf9126 100644 --- a/src/router/router.rs +++ b/src/router/router.rs @@ -2,8 +2,7 @@ use std::sync::Arc; use axum::{Router, response::IntoResponse}; use conduwuit::Error; -use conduwuit_api::router::{state, state::Guard}; -use conduwuit_service::Services; +use conduwuit_service::{Services, state, state::Guard}; use http::{StatusCode, Uri}; use ruma::api::client::error::ErrorKind; @@ -11,7 +10,7 @@ pub(crate) fn build(services: &Arc) -> (Router, Guard) { let router = Router::::new(); let (state, guard) = state::create(services.clone()); let router = conduwuit_api::router::build(router, &services.server) - .merge(conduwuit_web::build::().with_state(())) + .merge(conduwuit_web::build()) .fallback(not_found) .with_state(state); diff --git a/src/service/mod.rs b/src/service/mod.rs index eb15e5ec..3d7a3aa9 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -5,6 +5,7 @@ mod manager; mod migrations; mod service; pub mod services; +pub mod state; pub mod account_data; pub mod admin; diff --git a/src/api/router/state.rs b/src/service/state.rs similarity index 98% rename from src/api/router/state.rs rename to src/service/state.rs index 57eb94ca..c0884a5c 100644 --- a/src/api/router/state.rs +++ b/src/service/state.rs @@ -1,6 +1,6 @@ use std::{ops::Deref, sync::Arc}; -use conduwuit_service::Services; +use crate::Services; #[derive(Clone, Copy)] pub struct State { diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 8e03a338..5c2dbebb 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -21,6 +21,7 @@ crate-type = [ [dependencies] conduwuit-build-metadata.workspace = true +conduwuit-service.workspace = true askama = "0.14.0" diff --git a/src/web/mod.rs b/src/web/mod.rs index 520a9a34..25ec868c 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -1,23 +1,34 @@ use askama::Template; use axum::{ Router, + extract::State, http::{StatusCode, header}, response::{Html, IntoResponse, Response}, routing::get, }; use conduwuit_build_metadata::{GIT_REMOTE_COMMIT_URL, GIT_REMOTE_WEB_URL, VERSION_EXTRA}; +use conduwuit_service::state; -pub fn build() -> Router<()> { Router::new().route("/", get(index_handler)) } +pub fn build() -> Router { + let router = Router::::new(); + router.route("/", get(index_handler)) +} -async fn index_handler() -> Result { +async fn index_handler( + State(services): State, +) -> Result { #[derive(Debug, Template)] #[template(path = "index.html.j2")] struct Tmpl<'a> { nonce: &'a str, + server_name: &'a str, } let nonce = rand::random::().to_string(); - let template = Tmpl { nonce: &nonce }; + let template = Tmpl { + nonce: &nonce, + server_name: services.config.server_name.as_str(), + }; Ok(( [(header::CONTENT_SECURITY_POLICY, format!("default-src 'none' 'nonce-{nonce}';"))], Html(template.render()?), diff --git a/src/web/templates/index.html.j2 b/src/web/templates/index.html.j2 index 51393822..648f5ddd 100644 --- a/src/web/templates/index.html.j2 +++ b/src/web/templates/index.html.j2 @@ -9,7 +9,7 @@

  • Read the documentation
  • Join the community
  • Log in with a client
    • -
  • Ensure federation works
    • +
  • Ensure federation works
    • From e3ae024ed37ce869e66a45f1d567b54600a90cf7 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 1 May 2025 21:21:28 +0100 Subject: [PATCH 1763/2091] chore: Link to Matrix rooms directly --- src/web/templates/index.html.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/templates/index.html.j2 b/src/web/templates/index.html.j2 index 648f5ddd..7f11cb1c 100644 --- a/src/web/templates/index.html.j2 +++ b/src/web/templates/index.html.j2 @@ -7,7 +7,7 @@

    To get started, you can:

    From d78fc53577b38d0cd9f93341804d1ca930d0d719 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 1 May 2025 21:27:12 +0100 Subject: [PATCH 1764/2091] ci: Fix bad comparison --- .forgejo/workflows/release-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 69d7d2fd..704a3bbf 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -214,7 +214,7 @@ jobs: type=semver,pattern=v{{version}} type=semver,pattern=v{{major}}.{{minor}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.0.') }} type=semver,pattern=v{{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} - type=ref,event=branch,prefix=${{ format('refs/heads/{0}', github.event.repository.default_branch) 1= github.ref && 'branch-' || '' }} + type=ref,event=branch,prefix=${{ format('refs/heads/{0}', github.event.repository.default_branch) != github.ref && 'branch-' || '' }} type=ref,event=pr type=sha,format=long images: ${{needs.define-variables.outputs.images}} From 01594a62439176aee1c1544c5be0cce0ddedcdf0 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 6 May 2025 20:51:12 +0100 Subject: [PATCH 1765/2091] chore: Fix typos across the codebase --- .typos.toml | 5 +++++ docs/deploying/docker-compose.for-traefik.yml | 2 +- docs/deploying/docker-compose.with-traefik.yml | 2 +- docs/maintenance.md | 2 +- src/api/client/capabilities.rs | 2 +- src/api/client/well_known.rs | 2 +- src/core/info/cargo.rs | 2 +- src/core/log/reload.rs | 6 +++--- src/core/utils/string/between.rs | 6 +++--- src/database/engine/cf_opts.rs | 2 +- src/main/main.rs | 2 +- src/main/restart.rs | 4 ++-- src/service/admin/execute.rs | 4 ++-- src/service/admin/mod.rs | 2 +- src/service/appservice/namespace_regex.rs | 2 +- src/service/media/remote.rs | 17 ++++++++++++----- src/service/resolver/actual.rs | 2 +- src/service/rooms/spaces/mod.rs | 2 +- src/service/rooms/timeline/mod.rs | 18 +++++++++--------- theme/css/chrome.css | 2 +- theme/css/variables.css | 12 ++++++------ 21 files changed, 55 insertions(+), 43 deletions(-) create mode 100644 .typos.toml diff --git a/.typos.toml b/.typos.toml new file mode 100644 index 00000000..1e46469c --- /dev/null +++ b/.typos.toml @@ -0,0 +1,5 @@ +[default.extend-words] +"allocatedp" = "allocatedp" +"conduwuit" = "conduwuit" +"continuwuity" = "continuwuity" +"execuse" = "execuse" diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index 04142e0c..83fb64ff 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -28,7 +28,7 @@ services: #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above # We need some way to serve the client and server .well-known json. The simplest way is via the CONDUWUIT_WELL_KNOWN - # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a seperate + # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a separate # see the override file for more information about delegation CONDUWUIT_WELL_KNOWN: | { diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index 9083b796..a45893da 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -36,7 +36,7 @@ services: # CONDUWUIT_NEW_USER_DISPLAYNAME_SUFFIX = "🏳<200d>⚧" # We need some way to serve the client and server .well-known json. The simplest way is via the CONDUWUIT_WELL_KNOWN - # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a seperate + # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a separate # reverse proxy, but since you do not have a reverse proxy and following this guide, this example is included CONDUWUIT_WELL_KNOWN: | { diff --git a/docs/maintenance.md b/docs/maintenance.md index b85a1971..16ec5a4e 100644 --- a/docs/maintenance.md +++ b/docs/maintenance.md @@ -71,7 +71,7 @@ related to WAL tracking. The only safe files that can be deleted are the `LOG` files (all caps). These are the real RocksDB telemetry/log files, however Continuwuity has already -configured to only store up to 3 RocksDB `LOG` files due to generall being +configured to only store up to 3 RocksDB `LOG` files due to generally being useless for average users unless troubleshooting something low-level. If you would like to store nearly none at all, see the `rocksdb_max_log_files` config option. diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 470ff6ab..7362c4f9 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -15,7 +15,7 @@ use crate::Ruma; /// # `GET /_matrix/client/v3/capabilities` /// -/// Get information on the supported feature set and other relevent capabilities +/// Get information on the supported feature set and other relevant capabilities /// of this server. pub(crate) async fn get_capabilities_route( State(services): State, diff --git a/src/api/client/well_known.rs b/src/api/client/well_known.rs index eedab981..35b7fc1e 100644 --- a/src/api/client/well_known.rs +++ b/src/api/client/well_known.rs @@ -60,7 +60,7 @@ pub(crate) async fn well_known_support( return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); } - // TOOD: support defining multiple contacts in the config + // TODO: support defining multiple contacts in the config let mut contacts: Vec = vec![]; if let Some(role) = role { diff --git a/src/core/info/cargo.rs b/src/core/info/cargo.rs index 28c6590e..e70bdcd5 100644 --- a/src/core/info/cargo.rs +++ b/src/core/info/cargo.rs @@ -36,7 +36,7 @@ const MAIN_MANIFEST: &'static str = (); /// For *enabled* features see the info::rustc module instead. static FEATURES: OnceLock> = OnceLock::new(); -/// Processed list of dependencies. This is generated from the datas captured in +/// Processed list of dependencies. This is generated from the data captured in /// the MANIFEST. static DEPENDENCIES: OnceLock = OnceLock::new(); diff --git a/src/core/log/reload.rs b/src/core/log/reload.rs index e6a16c9f..f72fde47 100644 --- a/src/core/log/reload.rs +++ b/src/core/log/reload.rs @@ -16,9 +16,9 @@ use crate::{Result, error}; /// pulling in a version of tracing that's incompatible with the rest of our /// deps. /// -/// To work around this, we define an trait without the S paramter that forwards -/// to the reload::Handle::reload method, and then store the handle as a trait -/// object. +/// To work around this, we define an trait without the S parameter that +/// forwards to the reload::Handle::reload method, and then store the handle as +/// a trait object. /// /// [1]: pub trait ReloadHandle { diff --git a/src/core/utils/string/between.rs b/src/core/utils/string/between.rs index 05c137b4..8d3b6979 100644 --- a/src/core/utils/string/between.rs +++ b/src/core/utils/string/between.rs @@ -1,12 +1,12 @@ type Delim<'a> = (&'a str, &'a str); -/// Slice a string between a pair of delimeters. +/// Slice a string between a pair of delimiters. pub trait Between<'a> { - /// Extract a string between the delimeters. If the delimeters were not + /// Extract a string between the delimiters. If the delimiters were not /// found None is returned, otherwise the first extraction is returned. fn between(&self, delim: Delim<'_>) -> Option<&'a str>; - /// Extract a string between the delimeters. If the delimeters were not + /// Extract a string between the delimiters. If the delimiters were not /// found the original string is returned; take note of this behavior, /// if an empty slice is desired for this case use the fallible version and /// unwrap to EMPTY. diff --git a/src/database/engine/cf_opts.rs b/src/database/engine/cf_opts.rs index 7ceec722..cbbd1012 100644 --- a/src/database/engine/cf_opts.rs +++ b/src/database/engine/cf_opts.rs @@ -193,7 +193,7 @@ fn get_cache(ctx: &Context, desc: &Descriptor) -> Option { return None; } - // Some cache capacities are overriden by server config in a strange but + // Some cache capacities are overridden by server config in a strange but // legacy-compat way let config = &ctx.server.config; let cap = match desc.name { diff --git a/src/main/main.rs b/src/main/main.rs index 1a9d3fe4..3416bc68 100644 --- a/src/main/main.rs +++ b/src/main/main.rs @@ -73,7 +73,7 @@ async fn async_main(server: &Arc) -> Result<(), Error> { .lock() .await .take() - .expect("services initialied"), + .expect("services initialized"), ) .await { diff --git a/src/main/restart.rs b/src/main/restart.rs index b9d1dc94..631c1e21 100644 --- a/src/main/restart.rs +++ b/src/main/restart.rs @@ -13,8 +13,8 @@ pub(super) fn restart() -> ! { // // We can (and do) prevent that panic by checking the result of current_exe() // prior to committing to restart, returning an error to the user without any - // unexpected shutdown. In a nutshell that is the execuse for this unsafety. - // Nevertheless, we still want a way to override the restart preventation (i.e. + // unexpected shutdown. In a nutshell that is the excuse for this unsafety. + // Nevertheless, we still want a way to override the restart presentation (i.e. // admin server restart --force). let exe = unsafe { utils::sys::current_exe().expect("program path must be available") }; let envs = env::vars(); diff --git a/src/service/admin/execute.rs b/src/service/admin/execute.rs index 174b28ed..e0d724bd 100644 --- a/src/service/admin/execute.rs +++ b/src/service/admin/execute.rs @@ -25,7 +25,7 @@ pub(super) async fn console_auto_stop(&self) { /// Execute admin commands after startup #[implement(super::Service)] pub(super) async fn startup_execute(&self) -> Result { - // List of comamnds to execute + // List of commands to execute let commands = &self.services.server.config.admin_execute; // Determine if we're running in smoketest-mode which will change some behaviors @@ -64,7 +64,7 @@ pub(super) async fn startup_execute(&self) -> Result { /// Execute admin commands after signal #[implement(super::Service)] pub(super) async fn signal_execute(&self) -> Result { - // List of comamnds to execute + // List of commands to execute let commands = self.services.server.config.admin_signal_execute.clone(); // When true, errors are ignored and execution continues. diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index b3466711..683f5400 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -166,7 +166,7 @@ impl Service { .map_err(|e| err!("Failed to enqueue admin command: {e:?}")) } - /// Dispatches a comamnd to the processor on the current task and waits for + /// Dispatches a command to the processor on the current task and waits for /// completion. pub async fn command_in_place( &self, diff --git a/src/service/appservice/namespace_regex.rs b/src/service/appservice/namespace_regex.rs index fe0fd91f..76b754ae 100644 --- a/src/service/appservice/namespace_regex.rs +++ b/src/service/appservice/namespace_regex.rs @@ -26,7 +26,7 @@ impl NamespaceRegex { false } - /// Checks if this namespace has exlusive rights to a namespace + /// Checks if this namespace has exclusive rights to a namespace #[inline] #[must_use] pub fn is_exclusive_match(&self, heystack: &str) -> bool { diff --git a/src/service/media/remote.rs b/src/service/media/remote.rs index a1e874d8..f234fa13 100644 --- a/src/service/media/remote.rs +++ b/src/service/media/remote.rs @@ -338,7 +338,7 @@ fn handle_federation_error( return fallback(); } - // Reached for 5xx errors. This is where we don't fallback given the likelyhood + // Reached for 5xx errors. This is where we don't fallback given the likelihood // the other endpoint will also be a 5xx and we're wasting time. error } @@ -356,7 +356,7 @@ pub async fn fetch_remote_thumbnail_legacy( self.check_legacy_freeze()?; self.check_fetch_authorized(&mxc)?; - let reponse = self + let response = self .services .sending .send_federation_request(mxc.server_name, media::get_content_thumbnail::v3::Request { @@ -373,10 +373,17 @@ pub async fn fetch_remote_thumbnail_legacy( .await?; let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; - self.upload_thumbnail(&mxc, None, None, reponse.content_type.as_deref(), &dim, &reponse.file) - .await?; + self.upload_thumbnail( + &mxc, + None, + None, + response.content_type.as_deref(), + &dim, + &response.file, + ) + .await?; - Ok(reponse) + Ok(response) } #[implement(super::Service)] diff --git a/src/service/resolver/actual.rs b/src/service/resolver/actual.rs index 0151c4d7..d23ef95a 100644 --- a/src/service/resolver/actual.rs +++ b/src/service/resolver/actual.rs @@ -296,7 +296,7 @@ impl super::Service { expire: CachedOverride::default_expire(), overriding: (hostname != untername) .then_some(hostname.into()) - .inspect(|_| debug_info!("{untername:?} overriden by {hostname:?}")), + .inspect(|_| debug_info!("{untername:?} overridden by {hostname:?}")), }); Ok(()) diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index ea9756ba..53d2b742 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -399,7 +399,7 @@ async fn get_room_summary( Ok(summary) } -/// With the given identifier, checks if a room is accessable +/// With the given identifier, checks if a room is accessible #[implement(Service)] async fn is_accessible_child<'a, I>( &self, diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 947e1c38..4b2f3cb2 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -267,15 +267,15 @@ impl Service { /// /// Returns pdu id #[tracing::instrument(level = "debug", skip_all)] - pub async fn append_pdu<'a, Leafs>( + pub async fn append_pdu<'a, Leaves>( &'a self, pdu: &'a PduEvent, mut pdu_json: CanonicalJsonObject, - leafs: Leafs, + leaves: Leaves, state_lock: &'a RoomMutexGuard, ) -> Result where - Leafs: Iterator + Send + 'a, + Leaves: Iterator + Send + 'a, { // Coalesce database writes for the remainder of this scope. let _cork = self.db.db.cork_and_flush(); @@ -344,7 +344,7 @@ impl Service { self.services .state - .set_forward_extremities(&pdu.room_id, leafs, state_lock) + .set_forward_extremities(&pdu.room_id, leaves, state_lock) .await; let insert_lock = self.mutex_insert.lock(&pdu.room_id).await; @@ -951,17 +951,17 @@ impl Service { /// Append the incoming event setting the state snapshot to the state from /// the server that sent the event. #[tracing::instrument(level = "debug", skip_all)] - pub async fn append_incoming_pdu<'a, Leafs>( + pub async fn append_incoming_pdu<'a, Leaves>( &'a self, pdu: &'a PduEvent, pdu_json: CanonicalJsonObject, - new_room_leafs: Leafs, + new_room_leaves: Leaves, state_ids_compressed: Arc, soft_fail: bool, state_lock: &'a RoomMutexGuard, ) -> Result> where - Leafs: Iterator + Send + 'a, + Leaves: Iterator + Send + 'a, { // We append to state before appending the pdu, so we don't have a moment in // time with the pdu without it's state. This is okay because append_pdu can't @@ -978,14 +978,14 @@ impl Service { self.services .state - .set_forward_extremities(&pdu.room_id, new_room_leafs, state_lock) + .set_forward_extremities(&pdu.room_id, new_room_leaves, state_lock) .await; return Ok(None); } let pdu_id = self - .append_pdu(pdu, pdu_json, new_room_leafs, state_lock) + .append_pdu(pdu, pdu_json, new_room_leaves, state_lock) .await?; Ok(Some(pdu_id)) diff --git a/theme/css/chrome.css b/theme/css/chrome.css index 52b35c2c..d6cc2b32 100644 --- a/theme/css/chrome.css +++ b/theme/css/chrome.css @@ -495,7 +495,7 @@ ul#searchresults span.teaser em { .chapter li { display: flex; - color: var(--sidebar-non-existant); + color: var(--sidebar-non-existent); } .chapter li a { display: block; diff --git a/theme/css/variables.css b/theme/css/variables.css index e7feed98..ca9fd271 100644 --- a/theme/css/variables.css +++ b/theme/css/variables.css @@ -20,7 +20,7 @@ --sidebar-bg: #14191f; --sidebar-fg: #c8c9db; - --sidebar-non-existant: #5c6773; + --sidebar-non-existent: #5c6773; --sidebar-active: #ffb454; --sidebar-spacer: #2d334f; @@ -64,7 +64,7 @@ --sidebar-bg: #292c2f; --sidebar-fg: #a1adb8; - --sidebar-non-existant: #505254; + --sidebar-non-existent: #505254; --sidebar-active: #3473ad; --sidebar-spacer: #393939; @@ -108,7 +108,7 @@ --sidebar-bg: #fafafa; --sidebar-fg: #AE518E; - --sidebar-non-existant: #aaaaaa; + --sidebar-non-existent: #aaaaaa; --sidebar-active: #2F7E86; --sidebar-spacer: #f4f4f4; @@ -152,7 +152,7 @@ --sidebar-bg: #282d3f; --sidebar-fg: #fdcbec; - --sidebar-non-existant: #505274; + --sidebar-non-existent: #505274; --sidebar-active: #5BCEFA; --sidebar-spacer: #2d334f; @@ -196,7 +196,7 @@ --sidebar-bg: #3b2e2a; --sidebar-fg: #c8c9db; - --sidebar-non-existant: #505254; + --sidebar-non-existent: #505254; --sidebar-active: #e69f67; --sidebar-spacer: #45373a; @@ -241,7 +241,7 @@ --sidebar-bg: #292c2f; --sidebar-fg: #a1adb8; - --sidebar-non-existant: #505254; + --sidebar-non-existent: #505254; --sidebar-active: #3473ad; --sidebar-spacer: #393939; From c0f46269b58c4f6faef3084e328e12dbe9ce7742 Mon Sep 17 00:00:00 2001 From: Jade Date: Tue, 6 May 2025 21:49:41 +0000 Subject: [PATCH 1766/2091] docs: Fix name in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bf4f5613..fdcdafb7 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ It's a community continuation of the [conduwuit](https://github.com/girlbossceo/ The original conduwuit project has been archived and is no longer maintained. Rather than letting this Rust-based Matrix homeserver disappear, a group of community contributors have forked the project to continue its development, fix outstanding issues, and add new features. -We aim to provide a stable, well-maintained alternative for current Conduit users and welcome newcomers seeking a lightweight, efficient Matrix homeserver. +We aim to provide a stable, well-maintained alternative for current conduwuit users and welcome newcomers seeking a lightweight, efficient Matrix homeserver. ### Who are we? From 5577ddca270f2b0eb453987d08482565da01af0f Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 6 May 2025 22:56:23 +0100 Subject: [PATCH 1767/2091] chore: Add CONTINUWUITY_ environment variables Also updates some examples to match --- arch/conduwuit.service | 8 ++++---- debian/conduwuit.service | 7 ++++--- nix/pkgs/complement/default.nix | 6 +++--- src/core/config/mod.rs | 9 +++++++-- src/main/clap.rs | 18 ++++++++++++++++-- 5 files changed, 34 insertions(+), 14 deletions(-) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index 4f45ddc0..c86e37bd 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -1,11 +1,11 @@ [Unit] -Description=conduwuit Matrix homeserver + +Description=Continuwuity - Matrix homeserver Wants=network-online.target After=network-online.target -Documentation=https://conduwuit.puppyirl.gay/ +Documentation=https://continuwuity.org/ RequiresMountsFor=/var/lib/private/conduwuit Alias=matrix-conduwuit.service - [Service] DynamicUser=yes Type=notify-reload @@ -59,7 +59,7 @@ StateDirectory=conduwuit RuntimeDirectory=conduwuit RuntimeDirectoryMode=0750 -Environment="CONDUWUIT_CONFIG=/etc/conduwuit/conduwuit.toml" +Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml" BindPaths=/var/lib/private/conduwuit:/var/lib/matrix-conduit BindPaths=/var/lib/private/conduwuit:/var/lib/private/matrix-conduit diff --git a/debian/conduwuit.service b/debian/conduwuit.service index 3d2fbc9b..be2f3dae 100644 --- a/debian/conduwuit.service +++ b/debian/conduwuit.service @@ -1,9 +1,10 @@ [Unit] -Description=conduwuit Matrix homeserver + +Description=Continuwuity - Matrix homeserver Wants=network-online.target After=network-online.target -Alias=matrix-conduwuit.service Documentation=https://continuwuity.org/ +Alias=matrix-conduwuit.service [Service] DynamicUser=yes @@ -11,7 +12,7 @@ User=conduwuit Group=conduwuit Type=notify -Environment="CONDUWUIT_CONFIG=/etc/conduwuit/conduwuit.toml" +Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml" ExecStart=/usr/sbin/conduwuit diff --git a/nix/pkgs/complement/default.nix b/nix/pkgs/complement/default.nix index 9b010e14..1295cb03 100644 --- a/nix/pkgs/complement/default.nix +++ b/nix/pkgs/complement/default.nix @@ -75,9 +75,9 @@ dockerTools.buildImage { else []; Env = [ - "CONDUWUIT_TLS__KEY=${./private_key.key}" - "CONDUWUIT_TLS__CERTS=${./certificate.crt}" - "CONDUWUIT_CONFIG=${./config.toml}" + "CONTINUWUITY_TLS__KEY=${./private_key.key}" + "CONTINUWUITY_TLS__CERTS=${./certificate.crt}" + "CONTINUWUITY_CONFIG=${./config.toml}" "RUST_BACKTRACE=full" ]; diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 5374c2c2..5648a126 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1962,7 +1962,11 @@ impl Config { where I: Iterator, { - let envs = [Env::var("CONDUIT_CONFIG"), Env::var("CONDUWUIT_CONFIG")]; + let envs = [ + Env::var("CONDUIT_CONFIG"), + Env::var("CONDUWUIT_CONFIG"), + Env::var("CONTINUWUITY_CONFIG"), + ]; let config = envs .into_iter() @@ -1971,7 +1975,8 @@ impl Config { .chain(paths.map(Toml::file)) .fold(Figment::new(), |config, file| config.merge(file.nested())) .merge(Env::prefixed("CONDUIT_").global().split("__")) - .merge(Env::prefixed("CONDUWUIT_").global().split("__")); + .merge(Env::prefixed("CONDUWUIT_").global().split("__")) + .merge(Env::prefixed("CONTINUWUITY_").global().split("__")); Ok(config) } diff --git a/src/main/clap.rs b/src/main/clap.rs index 707a1c76..9b63af19 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -74,17 +74,30 @@ pub(crate) struct Args { /// with the exception of the last bucket, try increasing this value to e.g. /// 50 or 100. Inversely, decrease to 10 etc if the histogram lacks /// resolution. - #[arg(long, hide(true), env = "CONDUWUIT_RUNTIME_HISTOGRAM_INTERVAL", default_value = "25")] + #[arg( + long, + hide(true), + env = "CONTINUWUITY_RUNTIME_HISTOGRAM_INTERVAL", + env = "CONDUWUIT_RUNTIME_HISTOGRAM_INTERVAL", + default_value = "25" + )] pub(crate) worker_histogram_interval: u64, /// Set the histogram bucket count (tokio_unstable). Default is 20. - #[arg(long, hide(true), env = "CONDUWUIT_RUNTIME_HISTOGRAM_BUCKETS", default_value = "20")] + #[arg( + long, + hide(true), + env = "CONTINUWUITY_RUNTIME_HISTOGRAM_BUCKETS", + env = "CONDUWUIT_RUNTIME_HISTOGRAM_BUCKETS", + default_value = "20" + )] pub(crate) worker_histogram_buckets: usize, /// Toggles worker affinity feature. #[arg( long, hide(true), + env = "CONTINUWUITY_RUNTIME_WORKER_AFFINITY", env = "CONDUWUIT_RUNTIME_WORKER_AFFINITY", action = ArgAction::Set, num_args = 0..=1, @@ -99,6 +112,7 @@ pub(crate) struct Args { #[arg( long, hide(true), + env = "CONTINUWUITY_RUNTIME_GC_ON_PARK", env = "CONDUWUIT_RUNTIME_GC_ON_PARK", action = ArgAction::Set, num_args = 0..=1, From 7c58e40c967d984f1105738b589e81cc9f6069be Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 10 May 2025 13:29:59 +0100 Subject: [PATCH 1768/2091] chore(typos): Ignore certificate files --- .typos.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.typos.toml b/.typos.toml index 1e46469c..fe958292 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,3 +1,6 @@ +[files] +extend-exclude = ["*.csr"] + [default.extend-words] "allocatedp" = "allocatedp" "conduwuit" = "conduwuit" From beee996f723101d8ca61de7ebd46446f99298555 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 10 May 2025 20:37:08 +0100 Subject: [PATCH 1769/2091] docs: Rename conduwuit to continuwuity in more places --- .typos.toml | 1 + CONTRIBUTING.md | 4 +- Cargo.toml | 2 +- conduwuit-example.toml | 175 ++++++++--------- debian/README.md | 4 +- docs/deploying/docker-compose.for-traefik.yml | 38 ++-- docs/deploying/docker-compose.override.yml | 10 +- docs/deploying/docker-compose.with-caddy.yml | 30 +-- .../deploying/docker-compose.with-traefik.yml | 48 ++--- docs/deploying/docker-compose.yml | 30 +-- docs/deploying/docker.md | 10 +- docs/deploying/generic.md | 2 +- docs/development/hot_reload.md | 2 +- docs/development/testing.md | 5 +- nix/pkgs/oci-image/default.nix | 8 +- src/admin/processor.rs | 2 +- src/core/config/check.rs | 4 +- src/core/config/mod.rs | 177 +++++++++--------- src/service/admin/create.rs | 2 +- 19 files changed, 279 insertions(+), 275 deletions(-) diff --git a/.typos.toml b/.typos.toml index fe958292..41c81085 100644 --- a/.typos.toml +++ b/.typos.toml @@ -5,4 +5,5 @@ extend-exclude = ["*.csr"] "allocatedp" = "allocatedp" "conduwuit" = "conduwuit" "continuwuity" = "continuwuity" +"continuwity" = "continuwuity" "execuse" = "execuse" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ecff7173..da426801 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing guide -This page is for about contributing to conduwuit. The +This page is for about contributing to Continuwuity. The [development](./development.md) page may be of interest for you as well. If you would like to work on an [issue][issues] that is not assigned, preferably @@ -73,7 +73,7 @@ If you'd like to run Complement locally using Nix, see the ### Writing documentation -conduwuit's website uses [`mdbook`][mdbook] and deployed via CI using GitHub +Continuwuity's website uses [`mdbook`][mdbook] and deployed via CI using GitHub Pages in the [`documentation.yml`][documentation.yml] workflow file with Nix's mdbook in the devshell. All documentation is in the `docs/` directory at the top level. The compiled mdbook website is also uploaded as an artifact. diff --git a/Cargo.toml b/Cargo.toml index 43cd3f4f..79f767a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -298,7 +298,7 @@ version = "1.15.0" default-features = false features = ["serde"] -# Used for reading the configuration from conduwuit.toml & environment variables +# Used for reading the configuration from continuwuity.toml & environment variables [workspace.dependencies.figment] version = "0.10.19" default-features = false diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 3d92ab15..6934e67c 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1,4 +1,4 @@ -### conduwuit Configuration +### continuwuity Configuration ### ### THIS FILE IS GENERATED. CHANGES/CONTRIBUTIONS IN THE REPO WILL BE ### OVERWRITTEN! @@ -13,7 +13,7 @@ ### that say "YOU NEED TO EDIT THIS". ### ### For more information, see: -### https://conduwuit.puppyirl.gay/configuration.html +### https://continuwuity.org/configuration.html [global] @@ -21,7 +21,7 @@ # suffix for user and room IDs/aliases. # # See the docs for reverse proxying and delegation: -# https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy +# https://continuwuity.org/deploying/generic.html#setting-up-the-reverse-proxy # # Also see the `[global.well_known]` config section at the very bottom. # @@ -32,11 +32,11 @@ # YOU NEED TO EDIT THIS. THIS CANNOT BE CHANGED AFTER WITHOUT A DATABASE # WIPE. # -# example: "conduwuit.woof" +# example: "continuwuity.org" # #server_name = -# The default address (IPv4 or IPv6) conduwuit will listen on. +# The default address (IPv4 or IPv6) continuwuity will listen on. # # If you are using Docker or a container NAT networking setup, this must # be "0.0.0.0". @@ -46,10 +46,10 @@ # #address = ["127.0.0.1", "::1"] -# The port(s) conduwuit will listen on. +# The port(s) continuwuity will listen on. # # For reverse proxying, see: -# https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy +# https://continuwuity.org/deploying/generic.html#setting-up-the-reverse-proxy # # If you are using Docker, don't change this, you'll need to map an # external port to this. @@ -58,16 +58,17 @@ # #port = 8008 -# The UNIX socket conduwuit will listen on. +# The UNIX socket continuwuity will listen on. # -# conduwuit cannot listen on both an IP address and a UNIX socket. If +# continuwuity cannot listen on both an IP address and a UNIX socket. If # listening on a UNIX socket, you MUST remove/comment the `address` key. # # Remember to make sure that your reverse proxy has access to this socket -# file, either by adding your reverse proxy to the 'conduwuit' group or -# granting world R/W permissions with `unix_socket_perms` (666 minimum). +# file, either by adding your reverse proxy to the appropriate user group +# or granting world R/W permissions with `unix_socket_perms` (666 +# minimum). # -# example: "/run/conduwuit/conduwuit.sock" +# example: "/run/continuwuity/continuwuity.sock" # #unix_socket_path = @@ -75,23 +76,23 @@ # #unix_socket_perms = 660 -# This is the only directory where conduwuit will save its data, including -# media. Note: this was previously "/var/lib/matrix-conduit". +# This is the only directory where continuwuity will save its data, +# including media. Note: this was previously "/var/lib/matrix-conduit". # # YOU NEED TO EDIT THIS. # -# example: "/var/lib/conduwuit" +# example: "/var/lib/continuwuity" # #database_path = -# conduwuit supports online database backups using RocksDB's Backup engine -# API. To use this, set a database backup path that conduwuit can write -# to. +# continuwuity supports online database backups using RocksDB's Backup +# engine API. To use this, set a database backup path that continuwuity +# can write to. # # For more information, see: -# https://conduwuit.puppyirl.gay/maintenance.html#backups +# https://continuwuity.org/maintenance.html#backups # -# example: "/opt/conduwuit-db-backups" +# example: "/opt/continuwuity-db-backups" # #database_backup_path = @@ -112,14 +113,14 @@ # #new_user_displayname_suffix = "🏳️‍⚧️" -# If enabled, conduwuit will send a simple GET request periodically to +# If enabled, continuwuity will send a simple GET request periodically to # `https://continuwuity.org/.well-known/continuwuity/announcements` for any new # announcements or major updates. This is not an update check endpoint. # #allow_announcements_check = true -# Set this to any float value to multiply conduwuit's in-memory LRU caches -# with such as "auth_chain_cache_capacity". +# Set this to any float value to multiply continuwuity's in-memory LRU +# caches with such as "auth_chain_cache_capacity". # # May be useful if you have significant memory to spare to increase # performance. @@ -131,7 +132,7 @@ # #cache_capacity_modifier = 1.0 -# Set this to any float value in megabytes for conduwuit to tell the +# Set this to any float value in megabytes for continuwuity to tell the # database engine that this much memory is available for database read # caches. # @@ -145,7 +146,7 @@ # #db_cache_capacity_mb = varies by system -# Set this to any float value in megabytes for conduwuit to tell the +# Set this to any float value in megabytes for continuwuity to tell the # database engine that this much memory is available for database write # caches. # @@ -250,9 +251,9 @@ # Enable using *only* TCP for querying your specified nameservers instead # of UDP. # -# If you are running conduwuit in a container environment, this config +# If you are running continuwuity in a container environment, this config # option may need to be enabled. For more details, see: -# https://conduwuit.puppyirl.gay/troubleshooting.html#potential-dns-issues-when-using-docker +# https://continuwuity.org/troubleshooting.html#potential-dns-issues-when-using-docker # #query_over_tcp_only = false @@ -418,9 +419,9 @@ # tokens. Multiple tokens can be added if you separate them with # whitespace # -# conduwuit must be able to access the file, and it must not be empty +# continuwuity must be able to access the file, and it must not be empty # -# example: "/etc/conduwuit/.reg_token" +# example: "/etc/continuwuity/.reg_token" # #registration_token_file = @@ -512,16 +513,16 @@ #allow_room_creation = true # Set to false to disable users from joining or creating room versions -# that aren't officially supported by conduwuit. +# that aren't officially supported by continuwuity. # -# conduwuit officially supports room versions 6 - 11. +# continuwuity officially supports room versions 6 - 11. # -# conduwuit has slightly experimental (though works fine in practice) +# continuwuity has slightly experimental (though works fine in practice) # support for versions 3 - 5. # #allow_unstable_room_versions = true -# Default room version conduwuit will create rooms with. +# Default room version continuwuity will create rooms with. # # Per spec, room version 11 is the default. # @@ -587,7 +588,7 @@ # Servers listed here will be used to gather public keys of other servers # (notary trusted key servers). # -# Currently, conduwuit doesn't support inbound batched key requests, so +# Currently, continuwuity doesn't support inbound batched key requests, so # this list should only contain other Synapse servers. # # example: ["matrix.org", "tchncs.de"] @@ -628,7 +629,7 @@ # #trusted_server_batch_size = 1024 -# Max log level for conduwuit. Allows debug, info, warn, or error. +# Max log level for continuwuity. Allows debug, info, warn, or error. # # See also: # https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives @@ -649,8 +650,9 @@ # #log_span_events = "none" -# Configures whether CONDUWUIT_LOG EnvFilter matches values using regular -# expressions. See the tracing_subscriber documentation on Directives. +# Configures whether CONTINUWUITY_LOG EnvFilter matches values using +# regular expressions. See the tracing_subscriber documentation on +# Directives. # #log_filter_regex = true @@ -718,7 +720,7 @@ # This takes priority over "turn_secret" first, and falls back to # "turn_secret" if invalid or failed to open. # -# example: "/etc/conduwuit/.turn_secret" +# example: "/etc/continuwuity/.turn_secret" # #turn_secret_file = @@ -726,12 +728,12 @@ # #turn_ttl = 86400 -# List/vector of room IDs or room aliases that conduwuit will make newly -# registered users join. The rooms specified must be rooms that you have -# joined at least once on the server, and must be public. +# List/vector of room IDs or room aliases that continuwuity will make +# newly registered users join. The rooms specified must be rooms that you +# have joined at least once on the server, and must be public. # -# example: ["#conduwuit:puppygock.gay", -# "!eoIzvAvVwY23LPDay8:puppygock.gay"] +# example: ["#continuwuity:continuwuity.org", +# "!main-1:continuwuity.org"] # #auto_join_rooms = [] @@ -754,10 +756,10 @@ # #auto_deactivate_banned_room_attempts = false -# RocksDB log level. This is not the same as conduwuit's log level. This -# is the log level for the RocksDB engine/library which show up in your -# database folder/path as `LOG` files. conduwuit will log RocksDB errors -# as normal through tracing or panics if severe for safety. +# RocksDB log level. This is not the same as continuwuity's log level. +# This is the log level for the RocksDB engine/library which show up in +# your database folder/path as `LOG` files. continuwuity will log RocksDB +# errors as normal through tracing or panics if severe for safety. # #rocksdb_log_level = "error" @@ -777,7 +779,7 @@ # Set this to true to use RocksDB config options that are tailored to HDDs # (slower device storage). # -# It is worth noting that by default, conduwuit will use RocksDB with +# It is worth noting that by default, continuwuity will use RocksDB with # Direct IO enabled. *Generally* speaking this improves performance as it # bypasses buffered I/O (system page cache). However there is a potential # chance that Direct IO may cause issues with database operations if your @@ -785,7 +787,7 @@ # possibly ZFS filesystem. RocksDB generally deals/corrects these issues # but it cannot account for all setups. If you experience any weird # RocksDB issues, try enabling this option as it turns off Direct IO and -# feel free to report in the conduwuit Matrix room if this option fixes +# feel free to report in the continuwuity Matrix room if this option fixes # your DB issues. # # For more information, see: @@ -840,7 +842,7 @@ # as they all differ. See their `kDefaultCompressionLevel`. # # Note when using the default value we may override it with a setting -# tailored specifically conduwuit. +# tailored specifically for continuwuity. # #rocksdb_compression_level = 32767 @@ -856,7 +858,7 @@ # algorithm. # # Note when using the default value we may override it with a setting -# tailored specifically conduwuit. +# tailored specifically for continuwuity. # #rocksdb_bottommost_compression_level = 32767 @@ -896,13 +898,13 @@ # 0 = AbsoluteConsistency # 1 = TolerateCorruptedTailRecords (default) # 2 = PointInTime (use me if trying to recover) -# 3 = SkipAnyCorruptedRecord (you now voided your Conduwuit warranty) +# 3 = SkipAnyCorruptedRecord (you now voided your Continuwuity warranty) # # For more information on these modes, see: # https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes # # For more details on recovering a corrupt database, see: -# https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption +# https://continuwuity.org/troubleshooting.html#database-corruption # #rocksdb_recovery_mode = 1 @@ -942,7 +944,7 @@ # - Disabling repair mode and restarting the server is recommended after # running the repair. # -# See https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption for more details on recovering a corrupt database. +# See https://continuwuity.org/troubleshooting.html#database-corruption for more details on recovering a corrupt database. # #rocksdb_repair = false @@ -969,7 +971,7 @@ # Enables RocksDB compaction. You should never ever have to set this # option to false. If you for some reason find yourself needing to use # this option as part of troubleshooting or a bug, please reach out to us -# in the conduwuit Matrix room with information and details. +# in the continuwuity Matrix room with information and details. # # Disabling compaction will lead to a significantly bloated and # explosively large database, gradually poor performance, unnecessarily @@ -995,7 +997,7 @@ # purposes such as recovering/recreating your admin room, or inviting # yourself back. # -# See https://conduwuit.puppyirl.gay/troubleshooting.html#lost-access-to-admin-room for other ways to get back into your admin room. +# See https://continuwuity.org/troubleshooting.html#lost-access-to-admin-room for other ways to get back into your admin room. # # Once this password is unset, all sessions will be logged out for # security purposes. @@ -1010,8 +1012,8 @@ # Allow local (your server only) presence updates/requests. # -# Note that presence on conduwuit is very fast unlike Synapse's. If using -# outgoing presence, this MUST be enabled. +# Note that presence on continuwuity is very fast unlike Synapse's. If +# using outgoing presence, this MUST be enabled. # #allow_local_presence = true @@ -1019,7 +1021,7 @@ # # This option receives presence updates from other servers, but does not # send any unless `allow_outgoing_presence` is true. Note that presence on -# conduwuit is very fast unlike Synapse's. +# continuwuity is very fast unlike Synapse's. # #allow_incoming_presence = true @@ -1027,8 +1029,8 @@ # # This option sends presence updates to other servers, but does not # receive any unless `allow_incoming_presence` is true. Note that presence -# on conduwuit is very fast unlike Synapse's. If using outgoing presence, -# you MUST enable `allow_local_presence` as well. +# on continuwuity is very fast unlike Synapse's. If using outgoing +# presence, you MUST enable `allow_local_presence` as well. # #allow_outgoing_presence = true @@ -1081,8 +1083,8 @@ # #typing_client_timeout_max_s = 45 -# Set this to true for conduwuit to compress HTTP response bodies using -# zstd. This option does nothing if conduwuit was not built with +# Set this to true for continuwuity to compress HTTP response bodies using +# zstd. This option does nothing if continuwuity was not built with # `zstd_compression` feature. Please be aware that enabling HTTP # compression may weaken TLS. Most users should not need to enable this. # See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH @@ -1090,8 +1092,8 @@ # #zstd_compression = false -# Set this to true for conduwuit to compress HTTP response bodies using -# gzip. This option does nothing if conduwuit was not built with +# Set this to true for continuwuity to compress HTTP response bodies using +# gzip. This option does nothing if continuwuity was not built with # `gzip_compression` feature. Please be aware that enabling HTTP # compression may weaken TLS. Most users should not need to enable this. # See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before @@ -1102,8 +1104,8 @@ # #gzip_compression = false -# Set this to true for conduwuit to compress HTTP response bodies using -# brotli. This option does nothing if conduwuit was not built with +# Set this to true for continuwuity to compress HTTP response bodies using +# brotli. This option does nothing if continuwuity was not built with # `brotli_compression` feature. Please be aware that enabling HTTP # compression may weaken TLS. Most users should not need to enable this. # See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH @@ -1165,7 +1167,7 @@ # Otherwise setting this to false reduces filesystem clutter and overhead # for managing these symlinks in the directory. This is now disabled by # default. You may still return to upstream Conduit but you have to run -# conduwuit at least once with this set to true and allow the +# continuwuity at least once with this set to true and allow the # media_startup_check to take place before shutting down to return to # Conduit. # @@ -1210,8 +1212,8 @@ # #allowed_remote_server_names = [] -# Vector list of regex patterns of server names that conduwuit will refuse -# to download remote media from. +# Vector list of regex patterns of server names that continuwuity will +# refuse to download remote media from. # # example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] # @@ -1225,7 +1227,7 @@ # #forbidden_remote_room_directory_server_names = [] -# Vector list of regex patterns of server names that conduwuit will not +# Vector list of regex patterns of server names that continuwuity will not # send messages to the client from. # # Note that there is no way for clients to receive messages once a server @@ -1249,7 +1251,7 @@ #send_messages_from_ignored_users_to_client = false # Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you -# do not want conduwuit to send outbound requests to. Defaults to +# do not want continuwuity to send outbound requests to. Defaults to # RFC1918, unroutable, loopback, multicast, and testnet addresses for # security. # @@ -1399,26 +1401,26 @@ # Allow admins to enter commands in rooms other than "#admins" (admin # room) by prefixing your message with "\!admin" or "\\!admin" followed up -# a normal conduwuit admin command. The reply will be publicly visible to -# the room, originating from the sender. +# a normal continuwuity admin command. The reply will be publicly visible +# to the room, originating from the sender. # # example: \\!admin debug ping puppygock.gay # #admin_escape_commands = true -# Automatically activate the conduwuit admin room console / CLI on -# startup. This option can also be enabled with `--console` conduwuit +# Automatically activate the continuwuity admin room console / CLI on +# startup. This option can also be enabled with `--console` continuwuity # argument. # #admin_console_automatic = false # List of admin commands to execute on startup. # -# This option can also be configured with the `--execute` conduwuit +# This option can also be configured with the `--execute` continuwuity # argument and can take standard shell commands and environment variables # -# For example: `./conduwuit --execute "server admin-notice conduwuit has -# started up at $(date)"` +# For example: `./continuwuity --execute "server admin-notice continuwuity +# has started up at $(date)"` # # example: admin_execute = ["debug ping puppygock.gay", "debug echo hi"]` # @@ -1426,7 +1428,7 @@ # Ignore errors in startup commands. # -# If false, conduwuit will error and fail to start if an admin execute +# If false, continuwuity will error and fail to start if an admin execute # command (`--execute` / `admin_execute`) fails. # #admin_execute_errors_ignore = false @@ -1447,15 +1449,14 @@ # The default room tag to apply on the admin room. # # On some clients like Element, the room tag "m.server_notice" is a -# special pinned room at the very bottom of your room list. The conduwuit -# admin room can be pinned here so you always have an easy-to-access -# shortcut dedicated to your admin room. +# special pinned room at the very bottom of your room list. The +# continuwuity admin room can be pinned here so you always have an +# easy-to-access shortcut dedicated to your admin room. # #admin_room_tag = "m.server_notice" # Sentry.io crash/panic reporting, performance monitoring/metrics, etc. -# This is NOT enabled by default. conduwuit's default Sentry reporting -# endpoint domain is `o4506996327251968.ingest.us.sentry.io`. +# This is NOT enabled by default. # #sentry = false @@ -1463,7 +1464,7 @@ # #sentry_endpoint = "" -# Report your conduwuit server_name in Sentry.io crash reports and +# Report your continuwuity server_name in Sentry.io crash reports and # metrics. # #sentry_send_server_name = false @@ -1500,7 +1501,7 @@ # Enable the tokio-console. This option is only relevant to developers. # # For more information, see: -# https://conduwuit.puppyirl.gay/development.html#debugging-with-tokio-console +# https://continuwuity.org/development.html#debugging-with-tokio-console # #tokio_console = false diff --git a/debian/README.md b/debian/README.md index 800a2e09..4a8e58d2 100644 --- a/debian/README.md +++ b/debian/README.md @@ -1,4 +1,4 @@ -# conduwuit for Debian +# Continuwuity for Debian Information about downloading and deploying the Debian package. This may also be referenced for other `apt`-based distros such as Ubuntu. @@ -22,7 +22,7 @@ options in `/etc/conduwuit/conduwuit.toml`. ### Running -The package uses the [`conduwuit.service`](../configuration/examples.md#example-systemd-unit-file) systemd unit file to start and stop conduwuit. The binary is installed at `/usr/sbin/conduwuit`. +The package uses the [`conduwuit.service`](../configuration/examples.md#example-systemd-unit-file) systemd unit file to start and stop Continuwuity. The binary is installed at `/usr/sbin/conduwuit`. This package assumes by default that conduwuit will be placed behind a reverse proxy. The default config options apply (listening on `localhost` and TCP port `6167`). Matrix federation requires a valid domain name and TLS, so you will need to set up TLS certificates and renewal for it to work properly if you intend to federate. diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index 83fb64ff..547712b6 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -7,30 +7,30 @@ services: image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped volumes: - - db:/var/lib/conduwuit - - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. - #- ./conduwuit.toml:/etc/conduwuit.toml + - db:/var/lib/continuwuity + - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. + #- ./continuwuity.toml:/etc/continuwuity.toml networks: - proxy environment: - CONDUWUIT_SERVER_NAME: your.server.name.example # EDIT THIS - CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit - CONDUWUIT_PORT: 6167 # should match the loadbalancer traefik label - CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB - CONDUWUIT_ALLOW_REGISTRATION: 'true' - CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed. - #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true' - CONDUWUIT_ALLOW_FEDERATION: 'true' - CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' - CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' - #CONDUWUIT_LOG: warn,state_res=warn - CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above + CONTINUWUITY_SERVER_NAME: your.server.name.example # EDIT THIS + CONTINUWUITY_DATABASE_PATH: /var/lib/continuwuity + CONTINUWUITY_PORT: 6167 # should match the loadbalancer traefik label + CONTINUWUITY_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB + CONTINUWUITY_ALLOW_REGISTRATION: 'true' + CONTINUWUITY_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed. + #CONTINUWUITY_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true' + CONTINUWUITY_ALLOW_FEDERATION: 'true' + CONTINUWUITY_ALLOW_CHECK_FOR_UPDATES: 'true' + CONTINUWUITY_TRUSTED_SERVERS: '["matrix.org"]' + #CONTINUWUITY_LOG: warn,state_res=warn + CONTINUWUITY_ADDRESS: 0.0.0.0 + #CONTINUWUITY_CONFIG: '/etc/continuwuity.toml' # Uncomment if you mapped config toml above - # We need some way to serve the client and server .well-known json. The simplest way is via the CONDUWUIT_WELL_KNOWN - # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a separate + # We need some way to serve the client and server .well-known json. The simplest way is via the CONTINUWUITY_WELL_KNOWN + # variable / config option, there are multiple ways to do this, e.g. in the continuwuity.toml file, and in a separate # see the override file for more information about delegation - CONDUWUIT_WELL_KNOWN: | + CONTINUWUITY_WELL_KNOWN: | { client=https://your.server.name.example, server=your.server.name.example:443 diff --git a/docs/deploying/docker-compose.override.yml b/docs/deploying/docker-compose.override.yml index ec82fac3..168b1ae6 100644 --- a/docs/deploying/docker-compose.override.yml +++ b/docs/deploying/docker-compose.override.yml @@ -6,11 +6,11 @@ services: - "traefik.enable=true" - "traefik.docker.network=proxy" # Change this to the name of your Traefik docker proxy network - - "traefik.http.routers.to-conduwuit.rule=Host(`.`)" # Change to the address on which Continuwuity is hosted - - "traefik.http.routers.to-conduwuit.tls=true" - - "traefik.http.routers.to-conduwuit.tls.certresolver=letsencrypt" - - "traefik.http.routers.to-conduwuit.middlewares=cors-headers@docker" - - "traefik.http.services.to_conduwuit.loadbalancer.server.port=6167" + - "traefik.http.routers.to-continuwuity.rule=Host(`.`)" # Change to the address on which Continuwuity is hosted + - "traefik.http.routers.to-continuwuity.tls=true" + - "traefik.http.routers.to-continuwuity.tls.certresolver=letsencrypt" + - "traefik.http.routers.to-continuwuity.middlewares=cors-headers@docker" + - "traefik.http.services.to_continuwuity.loadbalancer.server.port=6167" - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*" - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization" diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index 9ee98428..3dfc9d85 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -25,23 +25,23 @@ services: image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped volumes: - - db:/var/lib/conduwuit + - db:/var/lib/continuwuity - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. - #- ./conduwuit.toml:/etc/conduwuit.toml + #- ./continuwuity.toml:/etc/continuwuity.toml environment: - CONDUWUIT_SERVER_NAME: example.com # EDIT THIS - CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit - CONDUWUIT_PORT: 6167 - CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB - CONDUWUIT_ALLOW_REGISTRATION: 'true' - CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed. - #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true' - CONDUWUIT_ALLOW_FEDERATION: 'true' - CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' - CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' - #CONDUWUIT_LOG: warn,state_res=warn - CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above + CONTINUWUITY_SERVER_NAME: example.com # EDIT THIS + CONTINUWUITY_DATABASE_PATH: /var/lib/continuwuity + CONTINUWUITY_PORT: 6167 + CONTINUWUITY_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB + CONTINUWUITY_ALLOW_REGISTRATION: 'true' + CONTINUWUITY_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed. + #CONTINUWUITY_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true' + CONTINUWUITY_ALLOW_FEDERATION: 'true' + CONTINUWUITY_ALLOW_CHECK_FOR_UPDATES: 'true' + CONTINUWUITY_TRUSTED_SERVERS: '["matrix.org"]' + #CONTINUWUITY_LOG: warn,state_res=warn + CONTINUWUITY_ADDRESS: 0.0.0.0 + #CONTINUWUITY_CONFIG: '/etc/continuwuity.toml' # Uncomment if you mapped config toml above networks: - caddy labels: diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index a45893da..9acc4221 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -7,38 +7,38 @@ services: image: forgejo.ellis.link/continuwuation/continuwuity:latest restart: unless-stopped volumes: - - db:/var/lib/conduwuit + - db:/var/lib/continuwuity - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. - #- ./conduwuit.toml:/etc/conduwuit.toml + #- ./continuwuity.toml:/etc/continuwuity.toml networks: - proxy environment: - CONDUWUIT_SERVER_NAME: your.server.name.example # EDIT THIS - CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' - CONDUWUIT_ALLOW_REGISTRATION: 'false' # After setting a secure registration token, you can enable this - CONDUWUIT_REGISTRATION_TOKEN: "" # This is a token you can use to register on the server - #CONDUWUIT_REGISTRATION_TOKEN_FILE: "" # Alternatively you can configure a path to a token file to read - CONDUWUIT_ADDRESS: 0.0.0.0 - CONDUWUIT_PORT: 6167 # you need to match this with the traefik load balancer label if you're want to change it - CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit - #CONDUWUIT_CONFIG: '/etc/conduit.toml' # Uncomment if you mapped config toml above + CONTINUWUITY_SERVER_NAME: your.server.name.example # EDIT THIS + CONTINUWUITY_TRUSTED_SERVERS: '["matrix.org"]' + CONTINUWUITY_ALLOW_REGISTRATION: 'false' # After setting a secure registration token, you can enable this + CONTINUWUITY_REGISTRATION_TOKEN: "" # This is a token you can use to register on the server + #CONTINUWUITY_REGISTRATION_TOKEN_FILE: "" # Alternatively you can configure a path to a token file to read + CONTINUWUITY_ADDRESS: 0.0.0.0 + CONTINUWUITY_PORT: 6167 # you need to match this with the traefik load balancer label if you're want to change it + CONTINUWUITY_DATABASE_PATH: /var/lib/continuwuity + #CONTINUWUITY_CONFIG: '/etc/continuwuity.toml' # Uncomment if you mapped config toml above ### Uncomment and change values as desired, note that Continuwuity has plenty of config options, so you should check out the example example config too # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging - # CONDUWUIT_LOG: info # default is: "warn,state_res=warn" - # CONDUWUIT_ALLOW_ENCRYPTION: 'true' - # CONDUWUIT_ALLOW_FEDERATION: 'true' - # CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' - # CONDUWUIT_ALLOW_INCOMING_PRESENCE: true - # CONDUWUIT_ALLOW_OUTGOING_PRESENCE: true - # CONDUWUIT_ALLOW_LOCAL_PRESENCE: true - # CONDUWUIT_WORKERS: 10 - # CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB - # CONDUWUIT_NEW_USER_DISPLAYNAME_SUFFIX = "🏳<200d>⚧" + # CONTINUWUITY_LOG: info # default is: "warn,state_res=warn" + # CONTINUWUITY_ALLOW_ENCRYPTION: 'true' + # CONTINUWUITY_ALLOW_FEDERATION: 'true' + # CONTINUWUITY_ALLOW_CHECK_FOR_UPDATES: 'true' + # CONTINUWUITY_ALLOW_INCOMING_PRESENCE: true + # CONTINUWUITY_ALLOW_OUTGOING_PRESENCE: true + # CONTINUWUITY_ALLOW_LOCAL_PRESENCE: true + # CONTINUWUITY_WORKERS: 10 + # CONTINUWUITY_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB + # CONTINUWUITY_NEW_USER_DISPLAYNAME_SUFFIX = "🏳<200d>⚧" - # We need some way to serve the client and server .well-known json. The simplest way is via the CONDUWUIT_WELL_KNOWN - # variable / config option, there are multiple ways to do this, e.g. in the conduwuit.toml file, and in a separate + # We need some way to serve the client and server .well-known json. The simplest way is via the CONTINUWUITY_WELL_KNOWN + # variable / config option, there are multiple ways to do this, e.g. in the continuwuity.toml file, and in a separate # reverse proxy, but since you do not have a reverse proxy and following this guide, this example is included - CONDUWUIT_WELL_KNOWN: | + CONTINUWUITY_WELL_KNOWN: | { client=https://your.server.name.example, server=your.server.name.example:443 diff --git a/docs/deploying/docker-compose.yml b/docs/deploying/docker-compose.yml index 1a3ab811..fbb50e35 100644 --- a/docs/deploying/docker-compose.yml +++ b/docs/deploying/docker-compose.yml @@ -9,22 +9,22 @@ services: ports: - 8448:6167 volumes: - - db:/var/lib/conduwuit - #- ./conduwuit.toml:/etc/conduwuit.toml + - db:/var/lib/continuwuity + #- ./continuwuity.toml:/etc/continuwuity.toml environment: - CONDUWUIT_SERVER_NAME: your.server.name # EDIT THIS - CONDUWUIT_DATABASE_PATH: /var/lib/conduwuit - CONDUWUIT_PORT: 6167 - CONDUWUIT_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB - CONDUWUIT_ALLOW_REGISTRATION: 'true' - CONDUWUIT_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed. - #CONDUWUIT_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true' - CONDUWUIT_ALLOW_FEDERATION: 'true' - CONDUWUIT_ALLOW_CHECK_FOR_UPDATES: 'true' - CONDUWUIT_TRUSTED_SERVERS: '["matrix.org"]' - #CONDUWUIT_LOG: warn,state_res=warn - CONDUWUIT_ADDRESS: 0.0.0.0 - #CONDUWUIT_CONFIG: '/etc/conduwuit.toml' # Uncomment if you mapped config toml above + CONTINUWUITY_SERVER_NAME: your.server.name # EDIT THIS + CONTINUWUITY_DATABASE_PATH: /var/lib/continuwuity + CONTINUWUITY_PORT: 6167 + CONTINUWUITY_MAX_REQUEST_SIZE: 20000000 # in bytes, ~20 MB + CONTINUWUITY_ALLOW_REGISTRATION: 'true' + CONTINUWUITY_REGISTRATION_TOKEN: 'YOUR_TOKEN' # A registration token is required when registration is allowed. + #CONTINUWUITY_YES_I_AM_VERY_VERY_SURE_I_WANT_AN_OPEN_REGISTRATION_SERVER_PRONE_TO_ABUSE: 'true' + CONTINUWUITY_ALLOW_FEDERATION: 'true' + CONTINUWUITY_ALLOW_CHECK_FOR_UPDATES: 'true' + CONTINUWUITY_TRUSTED_SERVERS: '["matrix.org"]' + #CONTINUWUITY_LOG: warn,state_res=warn + CONTINUWUITY_ADDRESS: 0.0.0.0 + #CONTINUWUITY_CONFIG: '/etc/continuwuity.toml' # Uncomment if you mapped config toml above # ### Uncomment if you want to use your own Element-Web App. ### Note: You need to provide a config.json for Element and you also need a second diff --git a/docs/deploying/docker.md b/docs/deploying/docker.md index 08a0dc4f..051ed89b 100644 --- a/docs/deploying/docker.md +++ b/docs/deploying/docker.md @@ -30,16 +30,16 @@ When you have the image you can simply run it with ```bash docker run -d -p 8448:6167 \ - -v db:/var/lib/conduwuit/ \ - -e CONDUWUIT_SERVER_NAME="your.server.name" \ - -e CONDUWUIT_ALLOW_REGISTRATION=false \ - --name conduwuit $LINK + -v db:/var/lib/continuwuity/ \ + -e CONTINUWUITY_SERVER_NAME="your.server.name" \ + -e CONTINUWUITY_ALLOW_REGISTRATION=false \ + --name continuwuity $LINK ``` or you can use [docker compose](#docker-compose). The `-d` flag lets the container run in detached mode. You may supply an -optional `conduwuit.toml` config file, the example config can be found +optional `continuwuity.toml` config file, the example config can be found [here](../configuration/examples.md). You can pass in different env vars to change config values on the fly. You can even configure Continuwuity completely by using env vars. For an overview of possible values, please take a look at the diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 46b9b439..9128f346 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -115,7 +115,7 @@ ReadWritePaths=/path/to/custom/database/path ## Creating the Continuwuity configuration file Now we need to create the Continuwuity's config file in -`/etc/conduwuit/conduwuit.toml`. The example config can be found at +`/etc/continuwuity/continuwuity.toml`. The example config can be found at [conduwuit-example.toml](../configuration/examples.md). **Please take a moment to read the config. You need to change at least the diff --git a/docs/development/hot_reload.md b/docs/development/hot_reload.md index ecfb6396..194ea3bc 100644 --- a/docs/development/hot_reload.md +++ b/docs/development/hot_reload.md @@ -190,7 +190,7 @@ The initial implementation PR is available [here][1]. - [Workspace-level metadata (cargo-deb)](https://github.com/kornelski/cargo-deb/issues/68) -[1]: https://github.com/girlbossceo/conduwuit/pull/387 +[1]: https://forgejo.ellis.link/continuwuation/continuwuity/pulls/387 [2]: https://wiki.musl-libc.org/functional-differences-from-glibc.html#Unloading-libraries [3]: https://github.com/rust-lang/rust/issues/28794 [4]: https://github.com/rust-lang/rust/issues/28794#issuecomment-368693049 diff --git a/docs/development/testing.md b/docs/development/testing.md index a577698a..d28bb874 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -24,8 +24,9 @@ and run the script. If you're on macOS and need to build an image, run `nix build .#linux-complement`. We have a Complement fork as some tests have needed to be fixed. This can be found -at: +at: -[ci-workflows]: https://github.com/girlbossceo/conduwuit/actions/workflows/ci.yml?query=event%3Apush+is%3Asuccess+actor%3Agirlbossceo +[ci-workflows]: +https://forgejo.ellis.link/continuwuation/continuwuity/actions/?workflow=ci.yml&actor=0&status=1 [complement]: https://github.com/matrix-org/complement [direnv]: https://direnv.net/docs/hook.html diff --git a/nix/pkgs/oci-image/default.nix b/nix/pkgs/oci-image/default.nix index 1650053d..953407ef 100644 --- a/nix/pkgs/oci-image/default.nix +++ b/nix/pkgs/oci-image/default.nix @@ -33,13 +33,13 @@ dockerTools.buildLayeredImage { "; "org.opencontainers.image.created" ="@${toString inputs.self.lastModified}"; "org.opencontainers.image.description" = "a very cool Matrix chat homeserver written in Rust"; - "org.opencontainers.image.documentation" = "https://conduwuit.puppyirl.gay/"; + "org.opencontainers.image.documentation" = "https://continuwuity.org/"; "org.opencontainers.image.licenses" = "Apache-2.0"; "org.opencontainers.image.revision" = inputs.self.rev or inputs.self.dirtyRev or ""; - "org.opencontainers.image.source" = "https://github.com/girlbossceo/conduwuit"; + "org.opencontainers.image.source" = "https://forgejo.ellis.link/continuwuation/continuwuity"; "org.opencontainers.image.title" = main.pname; - "org.opencontainers.image.url" = "https://conduwuit.puppyirl.gay/"; - "org.opencontainers.image.vendor" = "girlbossceo"; + "org.opencontainers.image.url" = "https://continuwuity.org/"; + "org.opencontainers.image.vendor" = "continuwuation"; "org.opencontainers.image.version" = main.version; }; }; diff --git a/src/admin/processor.rs b/src/admin/processor.rs index 8282a846..f7b7140f 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -94,7 +94,7 @@ async fn process_command(services: Arc, input: &CommandInput) -> Proce #[allow(clippy::result_large_err)] fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult { let link = - "Please submit a [bug report](https://github.com/girlbossceo/conduwuit/issues/new). 🥺"; + "Please submit a [bug report](https://forgejo.ellis.link/continuwuation/continuwuity/issues/new). 🥺"; let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}"); let content = RoomMessageEventContent::notice_markdown(msg); error!("Panic while processing command: {error:?}"); diff --git a/src/core/config/check.rs b/src/core/config/check.rs index f9d51eeb..ded9533d 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -118,7 +118,7 @@ pub fn check(config: &Config) -> Result { if cfg!(not(debug_assertions)) && config.server_name == "your.server.name" { return Err!(Config( "server_name", - "You must specify a valid server name for production usage of conduwuit." + "You must specify a valid server name for production usage of continuwuity." )); } @@ -290,7 +290,7 @@ fn warn_deprecated(config: &Config) { if was_deprecated { warn!( - "Read conduwuit config documentation at https://conduwuit.puppyirl.gay/configuration.html and check your \ + "Read continuwuity config documentation at https://continuwuity.org/configuration.html and check your \ configuration if any new configuration parameters should be adjusted" ); } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 5648a126..66ed0b2e 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -27,7 +27,7 @@ use self::proxy::ProxyConfig; pub use self::{check::check, manager::Manager}; use crate::{Result, err, error::Error, utils::sys}; -/// All the config options for conduwuit. +/// All the config options for continuwuity. #[allow(clippy::struct_excessive_bools)] #[allow(rustdoc::broken_intra_doc_links, rustdoc::bare_urls)] #[derive(Clone, Debug, Deserialize)] @@ -35,7 +35,7 @@ use crate::{Result, err, error::Error, utils::sys}; filename = "conduwuit-example.toml", section = "global", undocumented = "# This item is undocumented. Please contribute documentation for it.", - header = r#"### conduwuit Configuration + header = r#"### continuwuity Configuration ### ### THIS FILE IS GENERATED. CHANGES/CONTRIBUTIONS IN THE REPO WILL BE ### OVERWRITTEN! @@ -50,7 +50,7 @@ use crate::{Result, err, error::Error, utils::sys}; ### that say "YOU NEED TO EDIT THIS". ### ### For more information, see: -### https://conduwuit.puppyirl.gay/configuration.html +### https://continuwuity.org/configuration.html "#, ignore = "catchall well_known tls blurhashing allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure" )] @@ -59,7 +59,7 @@ pub struct Config { /// suffix for user and room IDs/aliases. /// /// See the docs for reverse proxying and delegation: - /// https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy + /// https://continuwuity.org/deploying/generic.html#setting-up-the-reverse-proxy /// /// Also see the `[global.well_known]` config section at the very bottom. /// @@ -70,10 +70,10 @@ pub struct Config { /// YOU NEED TO EDIT THIS. THIS CANNOT BE CHANGED AFTER WITHOUT A DATABASE /// WIPE. /// - /// example: "conduwuit.woof" + /// example: "continuwuity.org" pub server_name: OwnedServerName, - /// The default address (IPv4 or IPv6) conduwuit will listen on. + /// The default address (IPv4 or IPv6) continuwuity will listen on. /// /// If you are using Docker or a container NAT networking setup, this must /// be "0.0.0.0". @@ -85,10 +85,10 @@ pub struct Config { #[serde(default = "default_address")] address: ListeningAddr, - /// The port(s) conduwuit will listen on. + /// The port(s) continuwuity will listen on. /// /// For reverse proxying, see: - /// https://conduwuit.puppyirl.gay/deploying/generic.html#setting-up-the-reverse-proxy + /// https://continuwuity.org/deploying/generic.html#setting-up-the-reverse-proxy /// /// If you are using Docker, don't change this, you'll need to map an /// external port to this. @@ -103,16 +103,17 @@ pub struct Config { #[serde(default)] pub tls: TlsConfig, - /// The UNIX socket conduwuit will listen on. + /// The UNIX socket continuwuity will listen on. /// - /// conduwuit cannot listen on both an IP address and a UNIX socket. If + /// continuwuity cannot listen on both an IP address and a UNIX socket. If /// listening on a UNIX socket, you MUST remove/comment the `address` key. /// /// Remember to make sure that your reverse proxy has access to this socket - /// file, either by adding your reverse proxy to the 'conduwuit' group or - /// granting world R/W permissions with `unix_socket_perms` (666 minimum). + /// file, either by adding your reverse proxy to the appropriate user group + /// or granting world R/W permissions with `unix_socket_perms` (666 + /// minimum). /// - /// example: "/run/conduwuit/conduwuit.sock" + /// example: "/run/continuwuity/continuwuity.sock" pub unix_socket_path: Option, /// The default permissions (in octal) to create the UNIX socket with. @@ -121,22 +122,22 @@ pub struct Config { #[serde(default = "default_unix_socket_perms")] pub unix_socket_perms: u32, - /// This is the only directory where conduwuit will save its data, including - /// media. Note: this was previously "/var/lib/matrix-conduit". + /// This is the only directory where continuwuity will save its data, + /// including media. Note: this was previously "/var/lib/matrix-conduit". /// /// YOU NEED TO EDIT THIS. /// - /// example: "/var/lib/conduwuit" + /// example: "/var/lib/continuwuity" pub database_path: PathBuf, - /// conduwuit supports online database backups using RocksDB's Backup engine - /// API. To use this, set a database backup path that conduwuit can write - /// to. + /// continuwuity supports online database backups using RocksDB's Backup + /// engine API. To use this, set a database backup path that continuwuity + /// can write to. /// /// For more information, see: - /// https://conduwuit.puppyirl.gay/maintenance.html#backups + /// https://continuwuity.org/maintenance.html#backups /// - /// example: "/opt/conduwuit-db-backups" + /// example: "/opt/continuwuity-db-backups" pub database_backup_path: Option, /// The amount of online RocksDB database backups to keep/retain, if using @@ -160,7 +161,7 @@ pub struct Config { #[serde(default = "default_new_user_displayname_suffix")] pub new_user_displayname_suffix: String, - /// If enabled, conduwuit will send a simple GET request periodically to + /// If enabled, continuwuity will send a simple GET request periodically to /// `https://continuwuity.org/.well-known/continuwuity/announcements` for any new /// announcements or major updates. This is not an update check endpoint. /// @@ -168,8 +169,8 @@ pub struct Config { #[serde(alias = "allow_check_for_updates", default = "true_fn")] pub allow_announcements_check: bool, - /// Set this to any float value to multiply conduwuit's in-memory LRU caches - /// with such as "auth_chain_cache_capacity". + /// Set this to any float value to multiply continuwuity's in-memory LRU + /// caches with such as "auth_chain_cache_capacity". /// /// May be useful if you have significant memory to spare to increase /// performance. @@ -186,7 +187,7 @@ pub struct Config { )] pub cache_capacity_modifier: f64, - /// Set this to any float value in megabytes for conduwuit to tell the + /// Set this to any float value in megabytes for continuwuity to tell the /// database engine that this much memory is available for database read /// caches. /// @@ -202,7 +203,7 @@ pub struct Config { #[serde(default = "default_db_cache_capacity_mb")] pub db_cache_capacity_mb: f64, - /// Set this to any float value in megabytes for conduwuit to tell the + /// Set this to any float value in megabytes for continuwuity to tell the /// database engine that this much memory is available for database write /// caches. /// @@ -319,9 +320,9 @@ pub struct Config { /// Enable using *only* TCP for querying your specified nameservers instead /// of UDP. /// - /// If you are running conduwuit in a container environment, this config + /// If you are running continuwuity in a container environment, this config /// option may need to be enabled. For more details, see: - /// https://conduwuit.puppyirl.gay/troubleshooting.html#potential-dns-issues-when-using-docker + /// https://continuwuity.org/troubleshooting.html#potential-dns-issues-when-using-docker #[serde(default)] pub query_over_tcp_only: bool, @@ -534,9 +535,9 @@ pub struct Config { /// tokens. Multiple tokens can be added if you separate them with /// whitespace /// - /// conduwuit must be able to access the file, and it must not be empty + /// continuwuity must be able to access the file, and it must not be empty /// - /// example: "/etc/conduwuit/.reg_token" + /// example: "/etc/continuwuity/.reg_token" pub registration_token_file: Option, /// Controls whether encrypted rooms and events are allowed. @@ -627,16 +628,16 @@ pub struct Config { pub allow_room_creation: bool, /// Set to false to disable users from joining or creating room versions - /// that aren't officially supported by conduwuit. + /// that aren't officially supported by continuwuity. /// - /// conduwuit officially supports room versions 6 - 11. + /// continuwuity officially supports room versions 6 - 11. /// - /// conduwuit has slightly experimental (though works fine in practice) + /// continuwuity has slightly experimental (though works fine in practice) /// support for versions 3 - 5. #[serde(default = "true_fn")] pub allow_unstable_room_versions: bool, - /// Default room version conduwuit will create rooms with. + /// Default room version continuwuity will create rooms with. /// /// Per spec, room version 11 is the default. /// @@ -710,7 +711,7 @@ pub struct Config { /// Servers listed here will be used to gather public keys of other servers /// (notary trusted key servers). /// - /// Currently, conduwuit doesn't support inbound batched key requests, so + /// Currently, continuwuity doesn't support inbound batched key requests, so /// this list should only contain other Synapse servers. /// /// example: ["matrix.org", "tchncs.de"] @@ -755,7 +756,7 @@ pub struct Config { #[serde(default = "default_trusted_server_batch_size")] pub trusted_server_batch_size: usize, - /// Max log level for conduwuit. Allows debug, info, warn, or error. + /// Max log level for continuwuity. Allows debug, info, warn, or error. /// /// See also: /// https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives @@ -780,8 +781,9 @@ pub struct Config { #[serde(default = "default_log_span_events")] pub log_span_events: String, - /// Configures whether CONDUWUIT_LOG EnvFilter matches values using regular - /// expressions. See the tracing_subscriber documentation on Directives. + /// Configures whether CONTINUWUITY_LOG EnvFilter matches values using + /// regular expressions. See the tracing_subscriber documentation on + /// Directives. /// /// default: true #[serde(default = "true_fn")] @@ -863,7 +865,7 @@ pub struct Config { /// This takes priority over "turn_secret" first, and falls back to /// "turn_secret" if invalid or failed to open. /// - /// example: "/etc/conduwuit/.turn_secret" + /// example: "/etc/continuwuity/.turn_secret" pub turn_secret_file: Option, /// TURN TTL, in seconds. @@ -872,12 +874,12 @@ pub struct Config { #[serde(default = "default_turn_ttl")] pub turn_ttl: u64, - /// List/vector of room IDs or room aliases that conduwuit will make newly - /// registered users join. The rooms specified must be rooms that you have - /// joined at least once on the server, and must be public. + /// List/vector of room IDs or room aliases that continuwuity will make + /// newly registered users join. The rooms specified must be rooms that you + /// have joined at least once on the server, and must be public. /// - /// example: ["#conduwuit:puppygock.gay", - /// "!eoIzvAvVwY23LPDay8:puppygock.gay"] + /// example: ["#continuwuity:continuwuity.org", + /// "!main-1:continuwuity.org"] /// /// default: [] #[serde(default = "Vec::new")] @@ -902,10 +904,10 @@ pub struct Config { #[serde(default)] pub auto_deactivate_banned_room_attempts: bool, - /// RocksDB log level. This is not the same as conduwuit's log level. This - /// is the log level for the RocksDB engine/library which show up in your - /// database folder/path as `LOG` files. conduwuit will log RocksDB errors - /// as normal through tracing or panics if severe for safety. + /// RocksDB log level. This is not the same as continuwuity's log level. + /// This is the log level for the RocksDB engine/library which show up in + /// your database folder/path as `LOG` files. continuwuity will log RocksDB + /// errors as normal through tracing or panics if severe for safety. /// /// default: "error" #[serde(default = "default_rocksdb_log_level")] @@ -930,7 +932,7 @@ pub struct Config { /// Set this to true to use RocksDB config options that are tailored to HDDs /// (slower device storage). /// - /// It is worth noting that by default, conduwuit will use RocksDB with + /// It is worth noting that by default, continuwuity will use RocksDB with /// Direct IO enabled. *Generally* speaking this improves performance as it /// bypasses buffered I/O (system page cache). However there is a potential /// chance that Direct IO may cause issues with database operations if your @@ -938,7 +940,7 @@ pub struct Config { /// possibly ZFS filesystem. RocksDB generally deals/corrects these issues /// but it cannot account for all setups. If you experience any weird /// RocksDB issues, try enabling this option as it turns off Direct IO and - /// feel free to report in the conduwuit Matrix room if this option fixes + /// feel free to report in the continuwuity Matrix room if this option fixes /// your DB issues. /// /// For more information, see: @@ -999,7 +1001,7 @@ pub struct Config { /// as they all differ. See their `kDefaultCompressionLevel`. /// /// Note when using the default value we may override it with a setting - /// tailored specifically conduwuit. + /// tailored specifically for continuwuity. /// /// default: 32767 #[serde(default = "default_rocksdb_compression_level")] @@ -1017,7 +1019,7 @@ pub struct Config { /// algorithm. /// /// Note when using the default value we may override it with a setting - /// tailored specifically conduwuit. + /// tailored specifically for continuwuity. /// /// default: 32767 #[serde(default = "default_rocksdb_bottommost_compression_level")] @@ -1059,13 +1061,13 @@ pub struct Config { /// 0 = AbsoluteConsistency /// 1 = TolerateCorruptedTailRecords (default) /// 2 = PointInTime (use me if trying to recover) - /// 3 = SkipAnyCorruptedRecord (you now voided your Conduwuit warranty) + /// 3 = SkipAnyCorruptedRecord (you now voided your Continuwuity warranty) /// /// For more information on these modes, see: /// https://github.com/facebook/rocksdb/wiki/WAL-Recovery-Modes /// /// For more details on recovering a corrupt database, see: - /// https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption + /// https://continuwuity.org/troubleshooting.html#database-corruption /// /// default: 1 #[serde(default = "default_rocksdb_recovery_mode")] @@ -1109,7 +1111,7 @@ pub struct Config { /// - Disabling repair mode and restarting the server is recommended after /// running the repair. /// - /// See https://conduwuit.puppyirl.gay/troubleshooting.html#database-corruption for more details on recovering a corrupt database. + /// See https://continuwuity.org/troubleshooting.html#database-corruption for more details on recovering a corrupt database. #[serde(default)] pub rocksdb_repair: bool, @@ -1134,7 +1136,7 @@ pub struct Config { /// Enables RocksDB compaction. You should never ever have to set this /// option to false. If you for some reason find yourself needing to use /// this option as part of troubleshooting or a bug, please reach out to us - /// in the conduwuit Matrix room with information and details. + /// in the continuwuity Matrix room with information and details. /// /// Disabling compaction will lead to a significantly bloated and /// explosively large database, gradually poor performance, unnecessarily @@ -1162,7 +1164,7 @@ pub struct Config { /// purposes such as recovering/recreating your admin room, or inviting /// yourself back. /// - /// See https://conduwuit.puppyirl.gay/troubleshooting.html#lost-access-to-admin-room for other ways to get back into your admin room. + /// See https://continuwuity.org/troubleshooting.html#lost-access-to-admin-room for other ways to get back into your admin room. /// /// Once this password is unset, all sessions will be logged out for /// security purposes. @@ -1178,8 +1180,8 @@ pub struct Config { /// Allow local (your server only) presence updates/requests. /// - /// Note that presence on conduwuit is very fast unlike Synapse's. If using - /// outgoing presence, this MUST be enabled. + /// Note that presence on continuwuity is very fast unlike Synapse's. If + /// using outgoing presence, this MUST be enabled. #[serde(default = "true_fn")] pub allow_local_presence: bool, @@ -1187,7 +1189,7 @@ pub struct Config { /// /// This option receives presence updates from other servers, but does not /// send any unless `allow_outgoing_presence` is true. Note that presence on - /// conduwuit is very fast unlike Synapse's. + /// continuwuity is very fast unlike Synapse's. #[serde(default = "true_fn")] pub allow_incoming_presence: bool, @@ -1195,8 +1197,8 @@ pub struct Config { /// /// This option sends presence updates to other servers, but does not /// receive any unless `allow_incoming_presence` is true. Note that presence - /// on conduwuit is very fast unlike Synapse's. If using outgoing presence, - /// you MUST enable `allow_local_presence` as well. + /// on continuwuity is very fast unlike Synapse's. If using outgoing + /// presence, you MUST enable `allow_local_presence` as well. #[serde(default = "true_fn")] pub allow_outgoing_presence: bool, @@ -1259,8 +1261,8 @@ pub struct Config { #[serde(default = "default_typing_client_timeout_max_s")] pub typing_client_timeout_max_s: u64, - /// Set this to true for conduwuit to compress HTTP response bodies using - /// zstd. This option does nothing if conduwuit was not built with + /// Set this to true for continuwuity to compress HTTP response bodies using + /// zstd. This option does nothing if continuwuity was not built with /// `zstd_compression` feature. Please be aware that enabling HTTP /// compression may weaken TLS. Most users should not need to enable this. /// See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH @@ -1268,8 +1270,8 @@ pub struct Config { #[serde(default)] pub zstd_compression: bool, - /// Set this to true for conduwuit to compress HTTP response bodies using - /// gzip. This option does nothing if conduwuit was not built with + /// Set this to true for continuwuity to compress HTTP response bodies using + /// gzip. This option does nothing if continuwuity was not built with /// `gzip_compression` feature. Please be aware that enabling HTTP /// compression may weaken TLS. Most users should not need to enable this. /// See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH before @@ -1280,8 +1282,8 @@ pub struct Config { #[serde(default)] pub gzip_compression: bool, - /// Set this to true for conduwuit to compress HTTP response bodies using - /// brotli. This option does nothing if conduwuit was not built with + /// Set this to true for continuwuity to compress HTTP response bodies using + /// brotli. This option does nothing if continuwuity was not built with /// `brotli_compression` feature. Please be aware that enabling HTTP /// compression may weaken TLS. Most users should not need to enable this. /// See https://breachattack.com/ and https://wikipedia.org/wiki/BREACH @@ -1342,7 +1344,7 @@ pub struct Config { /// Otherwise setting this to false reduces filesystem clutter and overhead /// for managing these symlinks in the directory. This is now disabled by /// default. You may still return to upstream Conduit but you have to run - /// conduwuit at least once with this set to true and allow the + /// continuwuity at least once with this set to true and allow the /// media_startup_check to take place before shutting down to return to /// Conduit. #[serde(default)] @@ -1391,8 +1393,8 @@ pub struct Config { #[serde(default, with = "serde_regex")] pub allowed_remote_server_names: RegexSet, - /// Vector list of regex patterns of server names that conduwuit will refuse - /// to download remote media from. + /// Vector list of regex patterns of server names that continuwuity will + /// refuse to download remote media from. /// /// example: ["badserver\.tld$", "badphrase", "19dollarfortnitecards"] /// @@ -1410,7 +1412,7 @@ pub struct Config { #[serde(default, with = "serde_regex")] pub forbidden_remote_room_directory_server_names: RegexSet, - /// Vector list of regex patterns of server names that conduwuit will not + /// Vector list of regex patterns of server names that continuwuity will not /// send messages to the client from. /// /// Note that there is no way for clients to receive messages once a server @@ -1436,7 +1438,7 @@ pub struct Config { pub send_messages_from_ignored_users_to_client: bool, /// Vector list of IPv4 and IPv6 CIDR ranges / subnets *in quotes* that you - /// do not want conduwuit to send outbound requests to. Defaults to + /// do not want continuwuity to send outbound requests to. Defaults to /// RFC1918, unroutable, loopback, multicast, and testnet addresses for /// security. /// @@ -1604,26 +1606,26 @@ pub struct Config { /// Allow admins to enter commands in rooms other than "#admins" (admin /// room) by prefixing your message with "\!admin" or "\\!admin" followed up - /// a normal conduwuit admin command. The reply will be publicly visible to - /// the room, originating from the sender. + /// a normal continuwuity admin command. The reply will be publicly visible + /// to the room, originating from the sender. /// /// example: \\!admin debug ping puppygock.gay #[serde(default = "true_fn")] pub admin_escape_commands: bool, - /// Automatically activate the conduwuit admin room console / CLI on - /// startup. This option can also be enabled with `--console` conduwuit + /// Automatically activate the continuwuity admin room console / CLI on + /// startup. This option can also be enabled with `--console` continuwuity /// argument. #[serde(default)] pub admin_console_automatic: bool, /// List of admin commands to execute on startup. /// - /// This option can also be configured with the `--execute` conduwuit + /// This option can also be configured with the `--execute` continuwuity /// argument and can take standard shell commands and environment variables /// - /// For example: `./conduwuit --execute "server admin-notice conduwuit has - /// started up at $(date)"` + /// For example: `./continuwuity --execute "server admin-notice continuwuity + /// has started up at $(date)"` /// /// example: admin_execute = ["debug ping puppygock.gay", "debug echo hi"]` /// @@ -1633,7 +1635,7 @@ pub struct Config { /// Ignore errors in startup commands. /// - /// If false, conduwuit will error and fail to start if an admin execute + /// If false, continuwuity will error and fail to start if an admin execute /// command (`--execute` / `admin_execute`) fails. #[serde(default)] pub admin_execute_errors_ignore: bool, @@ -1658,17 +1660,16 @@ pub struct Config { /// The default room tag to apply on the admin room. /// /// On some clients like Element, the room tag "m.server_notice" is a - /// special pinned room at the very bottom of your room list. The conduwuit - /// admin room can be pinned here so you always have an easy-to-access - /// shortcut dedicated to your admin room. + /// special pinned room at the very bottom of your room list. The + /// continuwuity admin room can be pinned here so you always have an + /// easy-to-access shortcut dedicated to your admin room. /// /// default: "m.server_notice" #[serde(default = "default_admin_room_tag")] pub admin_room_tag: String, /// Sentry.io crash/panic reporting, performance monitoring/metrics, etc. - /// This is NOT enabled by default. conduwuit's default Sentry reporting - /// endpoint domain is `o4506996327251968.ingest.us.sentry.io`. + /// This is NOT enabled by default. #[serde(default)] pub sentry: bool, @@ -1679,7 +1680,7 @@ pub struct Config { #[serde(default = "default_sentry_endpoint")] pub sentry_endpoint: Option, - /// Report your conduwuit server_name in Sentry.io crash reports and + /// Report your continuwuity server_name in Sentry.io crash reports and /// metrics. #[serde(default)] pub sentry_send_server_name: bool, @@ -1720,7 +1721,7 @@ pub struct Config { /// Enable the tokio-console. This option is only relevant to developers. /// /// For more information, see: - /// https://conduwuit.puppyirl.gay/development.html#debugging-with-tokio-console + /// https://continuwuity.org/development.html#debugging-with-tokio-console #[serde(default)] pub tokio_console: bool, diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index cd0fc5a9..157b4d65 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -165,7 +165,7 @@ pub async fn create_admin_room(services: &Services) -> Result { .timeline .build_and_append_pdu( PduBuilder::state(String::new(), &RoomTopicEventContent { - topic: format!("Manage {} | Run commands prefixed with `!admin` | Run `!admin -h` for help | Documentation: https://conduwuit.puppyirl.gay/", services.config.server_name), + topic: format!("Manage {} | Run commands prefixed with `!admin` | Run `!admin -h` for help | Documentation: https://continuwuity.org/", services.config.server_name), }), server_user, &room_id, From 066794fe90c4af11c5c4ae5ce55d7db2fe6cf2da Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 17:15:37 +0100 Subject: [PATCH 1770/2091] ci: Don't try build images on PR --- .forgejo/workflows/release-image.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 704a3bbf..1c7457a5 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -3,7 +3,6 @@ concurrency: group: "release-image-${{ github.ref }}" on: - pull_request: push: paths-ignore: - "*.md" From d03325c65a6749669924c0a66b98795fe8babf26 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 17:27:54 +0100 Subject: [PATCH 1771/2091] chore: Set editorconfig for workflows --- .editorconfig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.editorconfig b/.editorconfig index 2d7438a4..91f073bd 100644 --- a/.editorconfig +++ b/.editorconfig @@ -22,3 +22,7 @@ indent_size = 2 [*.rs] indent_style = tab max_line_length = 98 + +[{.forgejo/**/*.yml,.github/**/*.yml}] +indent_size = 2 +indent_style = space From f14725a51b9be2b6fb6fda597e57afe770e4cdca Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 17:42:57 +0100 Subject: [PATCH 1772/2091] ci: Check formatting Also moves rustup installation to a seperate workflow and enables caching. The sccache action required a github.com api token, so we set all that up too. --- .forgejo/actions/rust-toolchain/action.yml | 45 ++++++++++++++++++++++ .forgejo/actions/sccache/action.yml | 29 ++++++++++++++ .forgejo/workflows/formatting.yml | 44 +++++++++++++++++++++ .forgejo/workflows/release-image.yml | 7 +--- 4 files changed, 120 insertions(+), 5 deletions(-) create mode 100644 .forgejo/actions/rust-toolchain/action.yml create mode 100644 .forgejo/actions/sccache/action.yml create mode 100644 .forgejo/workflows/formatting.yml diff --git a/.forgejo/actions/rust-toolchain/action.yml b/.forgejo/actions/rust-toolchain/action.yml new file mode 100644 index 00000000..68f59d00 --- /dev/null +++ b/.forgejo/actions/rust-toolchain/action.yml @@ -0,0 +1,45 @@ +name: rust-toolchain +description: | + Install a Rust toolchain using rustup. + See https://rust-lang.github.io/rustup/concepts/toolchains.html#toolchain-specification + for more information about toolchains. +inputs: + toolchain: + description: | + Rust toolchain name. + See https://rust-lang.github.io/rustup/concepts/toolchains.html#toolchain-specification + required: false + target: + description: Target triple to install for this toolchain + required: false + components: + description: Space-separated list of components to be additionally installed for a new toolchain + required: false + +runs: + using: composite + steps: + - name: Cache rustup toolchains + uses: actions/cache@v3 + with: + path: | + ~/.rustup + !~/.rustup/tmp + !~/.rustup/downloads + # Requires repo to be cloned if toolchain is not specified + key: ${{ runner.os }}-rustup-${{ inputs.toolchain || hashFiles('**/rust-toolchain.toml') }} + - name: Install Rust toolchain + shell: bash + run: | + if ! command -v rustup &> /dev/null ; then + curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y + echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH + fi + - shell: bash + run: | + set -x + ${{ inputs.toolchain && format('rustup override set {0}', inputs.toolchain) }} + ${{ inputs.target && format('rustup target add {0}', inputs.target) }} + ${{ inputs.components && format('rustup component add {0}', inputs.components) }} + cargo --version + rustc --version diff --git a/.forgejo/actions/sccache/action.yml b/.forgejo/actions/sccache/action.yml new file mode 100644 index 00000000..b5e5dcf4 --- /dev/null +++ b/.forgejo/actions/sccache/action.yml @@ -0,0 +1,29 @@ +name: sccache +description: | + Install sccache for caching builds in GitHub Actions. + +inputs: + token: + description: 'A Github PAT' + required: false + +runs: + using: composite + steps: + - name: Install sccache + uses: https://github.com/mozilla-actions/sccache-action@v0.0.9 + with: + token: ${{ inputs.token }} + - name: Configure sccache + uses: https://github.com/actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + - shell: bash + run: | + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV + echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV + echo "CMAKE_CUDA_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV diff --git a/.forgejo/workflows/formatting.yml b/.forgejo/workflows/formatting.yml new file mode 100644 index 00000000..e51560e7 --- /dev/null +++ b/.forgejo/workflows/formatting.yml @@ -0,0 +1,44 @@ +name: Rust Formatting + +on: + push: + pull_request: + +jobs: + format: + name: Format + runs-on: ubuntu-latest + env: + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install rust + uses: ./.forgejo/actions/rust-toolchain + with: + toolchain: "nightly" + components: "rustfmt" + + - uses: https://github.com/actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + github-api-url: https://api.github.com + owner: ${{ vars.GH_APP_OWNER }} + repositories: "" + - name: Install sccache + uses: ./.forgejo/actions/sccache + with: + token: ${{ steps.app-token.outputs.token }} + - name: Check formatting + run: | + cargo +nightly fmt --all -- --check + + - name: Show sccache stats + run: sccache --show-stats diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 1c7457a5..f6064617 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -79,16 +79,13 @@ jobs: run: echo '${{ toJSON(fromJSON(needs.define-variables.outputs.build_matrix)) }}' - name: Echo matrix run: echo '${{ toJSON(matrix) }}' - - run: | - if ! command -v rustup &> /dev/null ; then - curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --default-toolchain none -y - echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH - fi - name: Checkout repository uses: actions/checkout@v4 with: persist-credentials: false + - name: Install rust + uses: ./.forgejo/actions/rust-toolchain - name: Cache timelord-cli installation id: cache-timelord-bin From ec08e16b9f6e01f356a3a32890742a293b9ecbd4 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 19:39:44 +0100 Subject: [PATCH 1773/2091] build: Allow builder to decide on incremental or not --- Cargo.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 79f767a2..249ff84c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -745,7 +745,6 @@ incremental = true [profile.dev.package.conduwuit_core] inherits = "dev" -incremental = false #rustflags = [ # '--cfg', 'conduwuit_mods', # '-Ztime-passes', @@ -785,7 +784,6 @@ inherits = "dev" [profile.dev.package.'*'] inherits = "dev" debug = 'limited' -incremental = false codegen-units = 1 opt-level = 'z' #rustflags = [ @@ -807,7 +805,6 @@ inherits = "dev" strip = false opt-level = 0 codegen-units = 16 -incremental = false [profile.test.package.'*'] inherits = "dev" @@ -815,7 +812,6 @@ debug = 0 strip = false opt-level = 0 codegen-units = 16 -incremental = false ############################################################################### # From c5db43ba9aef530e02f0b6048eaa4a95662a8396 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 19:43:51 +0100 Subject: [PATCH 1774/2091] chore: Docker ignore forgejo files --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 8ca2e3f8..5054844f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,6 +15,7 @@ docker/ .gitea .gitlab .github +.forgejo # Dot files .env From e31d261e668259eacd2c11799d927b3e78354b16 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 19:43:56 +0100 Subject: [PATCH 1775/2091] ci: Run clippy check --- .forgejo/workflows/formatting.yml | 39 +++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/.forgejo/workflows/formatting.yml b/.forgejo/workflows/formatting.yml index e51560e7..332f98e2 100644 --- a/.forgejo/workflows/formatting.yml +++ b/.forgejo/workflows/formatting.yml @@ -1,29 +1,43 @@ name: Rust Formatting on: - push: - pull_request: + push: + pull_request: jobs: format: name: Format runs-on: ubuntu-latest - env: - SCCACHE_GHA_ENABLED: "true" - RUSTC_WRAPPER: "sccache" steps: - name: Checkout repository uses: actions/checkout@v4 with: persist-credentials: false - + - name: Install rust uses: ./.forgejo/actions/rust-toolchain with: toolchain: "nightly" components: "rustfmt" + - name: Check formatting + run: | + cargo +nightly fmt --all -- --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install rust + uses: ./.forgejo/actions/rust-toolchain + - uses: https://github.com/actions/create-github-app-token@v2 id: app-token with: @@ -36,9 +50,20 @@ jobs: uses: ./.forgejo/actions/sccache with: token: ${{ steps.app-token.outputs.token }} + - run: sudo apt-get update + - name: Install system dependencies + uses: https://github.com/awalsh128/cache-apt-pkgs-action@v1 + with: + packages: clang liburing-dev + version: 1 - name: Check formatting run: | - cargo +nightly fmt --all -- --check + cargo clippy \ + --workspace \ + --locked \ + --profile test \ + -- \ + -D warnings - name: Show sccache stats run: sccache --show-stats From 034762c6197bc48f4515ae63966fbd1849b2cccd Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 20:03:14 +0100 Subject: [PATCH 1776/2091] chore: Allow raw string hashes for metadata crate --- Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 249ff84c..1abff107 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -988,3 +988,6 @@ let_underscore_future = { level = "allow", priority = 1 } # rust doesnt understand conduwuit's custom log macros literal_string_with_formatting_args = { level = "allow", priority = 1 } + + +needless_raw_string_hashes = "allow" From e200a7d991ccec06282360d4e6dccd6b08e23663 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 20:23:30 +0100 Subject: [PATCH 1777/2091] ci: Cache Rust registry --- .forgejo/workflows/formatting.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.forgejo/workflows/formatting.yml b/.forgejo/workflows/formatting.yml index 332f98e2..7ca327b6 100644 --- a/.forgejo/workflows/formatting.yml +++ b/.forgejo/workflows/formatting.yml @@ -56,6 +56,15 @@ jobs: with: packages: clang liburing-dev version: 1 + - name: Cache Rust registry + uses: actions/cache@v3 + with: + path: | + ~/.cargo/git + !~/.cargo/git/checkouts + ~/.cargo/registry + !~/.cargo/registry/src + key: rust-registry-${{hashFiles('**/Cargo.lock') }} - name: Check formatting run: | cargo clippy \ From b5d2ef9a4a7758bc74d3a91cec9b5fc8cf9055f7 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 20:34:22 +0100 Subject: [PATCH 1778/2091] ci: Refactor timelord to its own action --- .forgejo/actions/timelord/action.yml | 46 ++++++++++++++++++++++++++++ .forgejo/workflows/release-image.yml | 26 +++------------- 2 files changed, 50 insertions(+), 22 deletions(-) create mode 100644 .forgejo/actions/timelord/action.yml diff --git a/.forgejo/actions/timelord/action.yml b/.forgejo/actions/timelord/action.yml new file mode 100644 index 00000000..bb9766d5 --- /dev/null +++ b/.forgejo/actions/timelord/action.yml @@ -0,0 +1,46 @@ +name: timelord +description: | + Use timelord to set file timestamps +inputs: + key: + description: | + The key to use for caching the timelord data. + This should be unique to the repository and the runner. + required: true + default: timelord-v0 + path: + description: | + The path to the directory to be timestamped. + This should be the root of the repository. + required: true + default: . + +runs: + using: composite + steps: + - name: Cache timelord-cli installation + id: cache-timelord-bin + uses: actions/cache@v3 + with: + path: ~/.cargo/bin/timelord + key: timelord-cli-v3.0.1 + - name: Install timelord-cli + uses: https://github.com/cargo-bins/cargo-binstall@main + if: steps.cache-timelord-bin.outputs.cache-hit != 'true' + - run: cargo binstall timelord-cli@3.0.1 + shell: bash + if: steps.cache-timelord-bin.outputs.cache-hit != 'true' + + - name: Load timelord files + uses: actions/cache/restore@v3 + with: + path: /timelord/ + key: ${{ inputs.key }} + - name: Run timelord to set timestamps + shell: bash + run: timelord sync --source-dir ${{ inputs.path }} --cache-dir /timelord/ + - name: Save timelord + uses: actions/cache/save@v3 + with: + path: /timelord/ + key: ${{ inputs.key }} diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index f6064617..0735fec7 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -87,18 +87,6 @@ jobs: - name: Install rust uses: ./.forgejo/actions/rust-toolchain - - name: Cache timelord-cli installation - id: cache-timelord-bin - uses: actions/cache@v3 - with: - path: ~/.cargo/bin/timelord - key: timelord-cli-v3.0.1 - - name: Install timelord-cli - uses: https://github.com/cargo-bins/cargo-binstall@main - if: steps.cache-timelord-bin.outputs.cache-hit != 'true' - - run: cargo binstall timelord-cli@3.0.1 - if: steps.cache-timelord-bin.outputs.cache-hit != 'true' - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Set up QEMU @@ -132,18 +120,12 @@ jobs: echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV - name: Get Git commit timestamps run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - - name: Set up timelord - uses: actions/cache/restore@v3 + + - uses: ./.forgejo/actions/timelord with: - path: /timelord/ - key: timelord-v0 # Cache is already split per runner - - name: Run timelord to set timestamps - run: timelord sync --source-dir . --cache-dir /timelord/ - - name: Save timelord - uses: actions/cache/save@v3 - with: - path: /timelord/ key: timelord-v0 + path: . + - name: Build and push Docker image by digest id: build uses: docker/build-push-action@v6 From a325dfa56aa1f3a638b9fc6b760befa8ac780952 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 11 May 2025 20:39:50 +0100 Subject: [PATCH 1779/2091] ci: Use timelord in clippy check --- .forgejo/workflows/formatting.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/formatting.yml b/.forgejo/workflows/formatting.yml index 7ca327b6..2f513b91 100644 --- a/.forgejo/workflows/formatting.yml +++ b/.forgejo/workflows/formatting.yml @@ -65,7 +65,11 @@ jobs: ~/.cargo/registry !~/.cargo/registry/src key: rust-registry-${{hashFiles('**/Cargo.lock') }} - - name: Check formatting + - uses: ./.forgejo/actions/timelord + with: + key: sccache-v0 + path: . + - name: Clippy run: | cargo clippy \ --workspace \ @@ -75,4 +79,4 @@ jobs: -D warnings - name: Show sccache stats - run: sccache --show-stats + run: sccache --show-stats \ No newline at end of file From 1f57508879fd6bea3eec8068d0a11882943c18b1 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 20 May 2025 21:15:32 +0100 Subject: [PATCH 1780/2091] ci: Don't clippy check dependancies --- .forgejo/workflows/{formatting.yml => rust-checks.yml} | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename .forgejo/workflows/{formatting.yml => rust-checks.yml} (94%) diff --git a/.forgejo/workflows/formatting.yml b/.forgejo/workflows/rust-checks.yml similarity index 94% rename from .forgejo/workflows/formatting.yml rename to .forgejo/workflows/rust-checks.yml index 2f513b91..eef3bd0a 100644 --- a/.forgejo/workflows/formatting.yml +++ b/.forgejo/workflows/rust-checks.yml @@ -1,8 +1,7 @@ -name: Rust Formatting +name: Rust Checks on: push: - pull_request: jobs: format: @@ -65,7 +64,8 @@ jobs: ~/.cargo/registry !~/.cargo/registry/src key: rust-registry-${{hashFiles('**/Cargo.lock') }} - - uses: ./.forgejo/actions/timelord + - name: Timelord + uses: ./.forgejo/actions/timelord with: key: sccache-v0 path: . @@ -74,6 +74,7 @@ jobs: cargo clippy \ --workspace \ --locked \ + --no-deps \ --profile test \ -- \ -D warnings From a4ad72e11ddce01d64aa5e2e3a002c45f9c5b767 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 20 May 2025 21:17:11 +0100 Subject: [PATCH 1781/2091] ci: Run `cargo test` --- .forgejo/workflows/rust-checks.yml | 59 +++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/rust-checks.yml b/.forgejo/workflows/rust-checks.yml index eef3bd0a..1feb9e89 100644 --- a/.forgejo/workflows/rust-checks.yml +++ b/.forgejo/workflows/rust-checks.yml @@ -80,4 +80,61 @@ jobs: -D warnings - name: Show sccache stats - run: sccache --show-stats \ No newline at end of file + run: sccache --show-stats + + cargo-test: + name: Cargo Test + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install rust + uses: ./.forgejo/actions/rust-toolchain + + - uses: https://github.com/actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ vars.GH_APP_ID }} + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + github-api-url: https://api.github.com + owner: ${{ vars.GH_APP_OWNER }} + repositories: "" + - name: Install sccache + uses: ./.forgejo/actions/sccache + with: + token: ${{ steps.app-token.outputs.token }} + - run: sudo apt-get update + - name: Install system dependencies + uses: https://github.com/awalsh128/cache-apt-pkgs-action@v1 + with: + packages: clang liburing-dev + version: 1 + - name: Cache Rust registry + uses: actions/cache@v3 + with: + path: | + ~/.cargo/git + !~/.cargo/git/checkouts + ~/.cargo/registry + !~/.cargo/registry/src + key: rust-registry-${{hashFiles('**/Cargo.lock') }} + - name: Timelord + uses: ./.forgejo/actions/timelord + with: + key: sccache-v0 + path: . + - name: Cargo Test + run: | + cargo test \ + --workspace \ + --locked \ + --profile test \ + --all-targets \ + --no-fail-fast + + - name: Show sccache stats + run: sccache --show-stats From 4ed04b343a8c8c95ffb83cd7b35bc0c1601c36c5 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 20 May 2025 22:13:13 +0100 Subject: [PATCH 1782/2091] build: Use xtrace in bash scripts in Dockerfile --- docker/Dockerfile | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3029282f..44e74180 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,13 +18,14 @@ ARG LLVM_VERSION=19 # Line three: for xx-verify RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ -apt-get update && apt-get install -y \ + apt-get update && apt-get install -y \ clang-${LLVM_VERSION} lld-${LLVM_VERSION} pkg-config make jq \ curl git \ file # Create symlinks for LLVM tools RUN <> /etc/environment # Configure pkg-config RUN <> /etc/environment echo "PKG_CONFIG=/usr/bin/$(xx-info)-pkg-config" >> /etc/environment echo "PKG_CONFIG_ALLOW_CROSS=true" >> /etc/environment @@ -82,12 +85,14 @@ EOF # Configure cc to use clang version RUN <> /etc/environment echo "CXX=clang++" >> /etc/environment EOF # Cross-language LTO RUN <> /etc/environment echo "CXXFLAGS=-flto" >> /etc/environment # Linker is set to target-compatible clang by xx @@ -98,6 +103,7 @@ EOF ARG TARGET_CPU= RUN <> /etc/environment @@ -118,7 +124,6 @@ COPY . . ARG TARGETPLATFORM # Verify environment configuration -RUN cat /etc/environment RUN xx-cargo --print-target-triple # Conduwuit version info @@ -142,6 +147,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/app/target \ bash <<'EOF' set -o allexport + set -o xtrace . /etc/environment TARGET_DIR=($(cargo metadata --no-deps --format-version 1 | \ jq -r ".target_directory")) @@ -162,6 +168,7 @@ EOF RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git/db \ bash <<'EOF' + set -o xtrace mkdir /out/sbom typeset -A PACKAGES for BINARY in /out/sbin/*; do @@ -180,6 +187,7 @@ EOF # Extract dynamically linked dependencies RUN < Date: Tue, 20 May 2025 22:47:55 +0100 Subject: [PATCH 1783/2091] build: Split docker target cache by target platform --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 44e74180..e734fb81 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -144,7 +144,7 @@ ENV CONTINUWUITY_VERSION_EXTRA=$CONTINUWUITY_VERSION_EXTRA # Build the binary RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git/db \ - --mount=type=cache,target=/app/target \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM} \ bash <<'EOF' set -o allexport set -o xtrace From 7a46563f23c1e4527310c400b604707c2213e498 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 20 May 2025 22:56:51 +0100 Subject: [PATCH 1784/2091] ci: Cache docker image build mounts --- .forgejo/actions/rust-toolchain/action.yml | 8 ++++ .forgejo/workflows/release-image.yml | 49 +++++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/.forgejo/actions/rust-toolchain/action.yml b/.forgejo/actions/rust-toolchain/action.yml index 68f59d00..71fb96f5 100644 --- a/.forgejo/actions/rust-toolchain/action.yml +++ b/.forgejo/actions/rust-toolchain/action.yml @@ -15,6 +15,10 @@ inputs: components: description: Space-separated list of components to be additionally installed for a new toolchain required: false +outputs: + rustc_version: + description: The rustc version installed + value: ${{ steps.rustc-version.outputs.version }} runs: using: composite @@ -43,3 +47,7 @@ runs: ${{ inputs.components && format('rustup component add {0}', inputs.components) }} cargo --version rustc --version + - id: rustc-version + shell: bash + run: | + echo "version=$(rustc --version)" >> $GITHUB_OUTPUT diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 0735fec7..ec466c58 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -79,12 +79,13 @@ jobs: run: echo '${{ toJSON(fromJSON(needs.define-variables.outputs.build_matrix)) }}' - name: Echo matrix run: echo '${{ toJSON(matrix) }}' - + - name: Checkout repository uses: actions/checkout@v4 with: persist-credentials: false - name: Install rust + id: rust-toolchain uses: ./.forgejo/actions/rust-toolchain - name: Set up Docker Buildx @@ -126,6 +127,52 @@ jobs: key: timelord-v0 path: . + - name: Cache Rust registry + uses: actions/cache@v3 + with: + path: | + .cargo/git + .cargo/git/checkouts + .cargo/registry + .cargo/registry/src + key: rust-registry-image-${{hashFiles('**/Cargo.lock') }} + - name: Cache cargo target + id: cache-cargo-target + uses: actions/cache@v3 + with: + path: | + cargo-target-${{ matrix.slug }} + key: cargo-target-${{ matrix.slug }}-${{hashFiles('**/Cargo.lock') }}-${{steps.rust-toolchain.outputs.rustc_version}} + - name: Cache apt cache + id: cache-apt + uses: actions/cache@v3 + with: + path: | + var-cache-apt-${{ matrix.slug }} + key: var-cache-apt-${{ matrix.slug }} + - name: Cache apt lib + id: cache-apt-lib + uses: actions/cache@v3 + with: + path: | + var-lib-apt-${{ matrix.slug }} + key: var-lib-apt-${{ matrix.slug }} + - name: inject cache into docker + uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.1.0 + with: + cache-map: | + { + ".cargo/registry": "/usr/local/cargo/registry", + ".cargo/git/db": "/usr/local/cargo/git/db", + "cargo-target-${{ matrix.slug }}": { + "target": "/app/target", + "id": "cargo-target-${{ matrix.platform }}" + }, + "var-cache-apt-${{ matrix.slug }}": "/var/cache/apt", + "var-lib-apt-${{ matrix.slug }}": "/var/lib/apt" + } + skip-extraction: ${{ steps.cache.outputs.cache-hit }} + - name: Build and push Docker image by digest id: build uses: docker/build-push-action@v6 From 9b8b37f162b75fe503b557876632bb5115aa35da Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 02:51:09 +0100 Subject: [PATCH 1785/2091] docs: Badges for mirrors --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fdcdafb7..e3eb807f 100644 --- a/README.md +++ b/README.md @@ -7,10 +7,15 @@ [continuwuity] is a Matrix homeserver written in Rust. -It's a community continuation of the [conduwuit](https://github.com/girlbossceo/conduwuit) homeserver. +It's a community continuation of the [conduwuit](https://github.com/girlbossceo/conduwuit) homeserver. +[![forgejo.ellis.link](https://img.shields.io/badge/Ellis%20Git-main+packages-green?style=flat&logo=forgejo&labelColor=fff)](https://forgejo.ellis.link/continuwuation/continuwuity) ![](https://forgejo.ellis.link/continuwuation/continuwuity/badges/stars.svg?style=flat) [![](https://forgejo.ellis.link/continuwuation/continuwuity/badges/issues/open.svg?style=flat)](https://forgejo.ellis.link/continuwuation/continuwuity/issues?state=open) [![](https://forgejo.ellis.link/continuwuation/continuwuity/badges/pulls/open.svg?style=flat)](https://forgejo.ellis.link/continuwuation/continuwuity/pulls?state=open) + +[![GitHub](https://img.shields.io/badge/GitHub-mirror-blue?style=flat&logo=github&labelColor=fff&logoColor=24292f)](https://github.com/continuwuity/continuwuity) ![](https://img.shields.io/github/stars/continuwuity/continuwuity?style=flat) + +[![Codeberg](https://img.shields.io/badge/Codeberg-mirror-2185D0?style=flat&logo=codeberg&labelColor=fff)](https://codeberg.org/nexy7574/continuwuity) ![](https://codeberg.org/nexy7574/continuwuity/badges/stars.svg?style=flat) ### Why does this exist? @@ -112,4 +117,3 @@ Join our [Matrix room](https://matrix.to/#/#continuwuity:continuwuity.org) and [ [continuwuity]: https://forgejo.ellis.link/continuwuation/continuwuity - From fcd5669aa117afc229f95ee43f072bd3b462ed09 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Apr 2025 06:29:30 +0000 Subject: [PATCH 1786/2091] Join jemalloc background threads prior to exit. Co-authored-by: Jade Ellis Signed-off-by: Jason Volk --- src/core/alloc/je.rs | 4 ++++ src/main/runtime.rs | 19 ++++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/src/core/alloc/je.rs b/src/core/alloc/je.rs index 2424e99c..e138233e 100644 --- a/src/core/alloc/je.rs +++ b/src/core/alloc/je.rs @@ -274,6 +274,10 @@ pub fn set_dirty_decay>>(arena: I, decay_ms: isize) -> Res } } +pub fn background_thread_enable(enable: bool) -> Result { + set::(&mallctl!("background_thread"), enable.into()).map(is_nonzero!()) +} + #[inline] #[must_use] pub fn is_affine_arena() -> bool { is_percpu_arena() || is_phycpu_arena() } diff --git a/src/main/runtime.rs b/src/main/runtime.rs index 1c58ea81..e9029012 100644 --- a/src/main/runtime.rs +++ b/src/main/runtime.rs @@ -98,12 +98,7 @@ pub(super) fn shutdown(server: &Arc, runtime: tokio::runtime::Runtime) { Level::INFO }; - debug!( - timeout = ?SHUTDOWN_TIMEOUT, - "Waiting for runtime..." - ); - - runtime.shutdown_timeout(SHUTDOWN_TIMEOUT); + wait_shutdown(server, runtime); let runtime_metrics = server.server.metrics.runtime_interval().unwrap_or_default(); event!(LEVEL, ?runtime_metrics, "Final runtime metrics"); @@ -111,13 +106,23 @@ pub(super) fn shutdown(server: &Arc, runtime: tokio::runtime::Runtime) { #[cfg(not(tokio_unstable))] #[tracing::instrument(name = "stop", level = "info", skip_all)] -pub(super) fn shutdown(_server: &Arc, runtime: tokio::runtime::Runtime) { +pub(super) fn shutdown(server: &Arc, runtime: tokio::runtime::Runtime) { + wait_shutdown(server, runtime); +} + +fn wait_shutdown(_server: &Arc, runtime: tokio::runtime::Runtime) { debug!( timeout = ?SHUTDOWN_TIMEOUT, "Waiting for runtime..." ); runtime.shutdown_timeout(SHUTDOWN_TIMEOUT); + + // Join any jemalloc threads so they don't appear in use at exit. + #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] + conduwuit_core::alloc::je::background_thread_enable(false) + .log_debug_err() + .ok(); } #[tracing::instrument( From bfb0a2b76a544af87fb9c2181f4f93eada14d635 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Apr 2025 09:33:17 +0000 Subject: [PATCH 1787/2091] Remove unused Pdu::into_any_event(). Signed-off-by: Jason Volk --- src/core/matrix/pdu/strip.rs | 69 ++++++++++-------------------------- 1 file changed, 19 insertions(+), 50 deletions(-) diff --git a/src/core/matrix/pdu/strip.rs b/src/core/matrix/pdu/strip.rs index 3683caaa..a39e7d35 100644 --- a/src/core/matrix/pdu/strip.rs +++ b/src/core/matrix/pdu/strip.rs @@ -1,8 +1,8 @@ use ruma::{ events::{ - AnyEphemeralRoomEvent, AnyMessageLikeEvent, AnyStateEvent, AnyStrippedStateEvent, - AnySyncStateEvent, AnySyncTimelineEvent, AnyTimelineEvent, StateEvent, - room::member::RoomMemberEventContent, space::child::HierarchySpaceChildEvent, + AnyMessageLikeEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncStateEvent, + AnySyncTimelineEvent, AnyTimelineEvent, StateEvent, room::member::RoomMemberEventContent, + space::child::HierarchySpaceChildEvent, }, serde::Raw, }; @@ -10,41 +10,6 @@ use serde_json::{json, value::Value as JsonValue}; use crate::implement; -/// This only works for events that are also AnyRoomEvents. -#[must_use] -#[implement(super::Pdu)] -pub fn into_any_event(self) -> Raw { - serde_json::from_value(self.into_any_event_value()).expect("Raw::from_value always works") -} - -/// This only works for events that are also AnyRoomEvents. -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn into_any_event_value(self) -> JsonValue { - let (redacts, content) = self.copy_redacts(); - let mut json = json!({ - "content": content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "room_id": self.room_id, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &redacts { - json["redacts"] = json!(redacts); - } - - json -} - #[implement(super::Pdu)] #[must_use] #[inline] @@ -53,7 +18,8 @@ pub fn into_room_event(self) -> Raw { self.to_room_event() } #[implement(super::Pdu)] #[must_use] pub fn to_room_event(&self) -> Raw { - serde_json::from_value(self.to_room_event_value()).expect("Raw::from_value always works") + let value = self.to_room_event_value(); + serde_json::from_value(value).expect("Failed to serialize Event value") } #[implement(super::Pdu)] @@ -91,8 +57,8 @@ pub fn into_message_like_event(self) -> Raw { self.to_messa #[implement(super::Pdu)] #[must_use] pub fn to_message_like_event(&self) -> Raw { - serde_json::from_value(self.to_message_like_event_value()) - .expect("Raw::from_value always works") + let value = self.to_message_like_event_value(); + serde_json::from_value(value).expect("Failed to serialize Event value") } #[implement(super::Pdu)] @@ -130,7 +96,8 @@ pub fn into_sync_room_event(self) -> Raw { self.to_sync_ro #[implement(super::Pdu)] #[must_use] pub fn to_sync_room_event(&self) -> Raw { - serde_json::from_value(self.to_sync_room_event_value()).expect("Raw::from_value always works") + let value = self.to_sync_room_event_value(); + serde_json::from_value(value).expect("Failed to serialize Event value") } #[implement(super::Pdu)] @@ -162,7 +129,8 @@ pub fn to_sync_room_event_value(&self) -> JsonValue { #[implement(super::Pdu)] #[must_use] pub fn into_state_event(self) -> Raw { - serde_json::from_value(self.into_state_event_value()).expect("Raw::from_value always works") + let value = self.into_state_event_value(); + serde_json::from_value(value).expect("Failed to serialize Event value") } #[implement(super::Pdu)] @@ -189,8 +157,8 @@ pub fn into_state_event_value(self) -> JsonValue { #[implement(super::Pdu)] #[must_use] pub fn into_sync_state_event(self) -> Raw { - serde_json::from_value(self.into_sync_state_event_value()) - .expect("Raw::from_value always works") + let value = self.into_sync_state_event_value(); + serde_json::from_value(value).expect("Failed to serialize Event value") } #[implement(super::Pdu)] @@ -223,8 +191,8 @@ pub fn into_stripped_state_event(self) -> Raw { #[implement(super::Pdu)] #[must_use] pub fn to_stripped_state_event(&self) -> Raw { - serde_json::from_value(self.to_stripped_state_event_value()) - .expect("Raw::from_value always works") + let value = self.to_stripped_state_event_value(); + serde_json::from_value(value).expect("Failed to serialize Event value") } #[implement(super::Pdu)] @@ -242,8 +210,8 @@ pub fn to_stripped_state_event_value(&self) -> JsonValue { #[implement(super::Pdu)] #[must_use] pub fn into_stripped_spacechild_state_event(self) -> Raw { - serde_json::from_value(self.into_stripped_spacechild_state_event_value()) - .expect("Raw::from_value always works") + let value = self.into_stripped_spacechild_state_event_value(); + serde_json::from_value(value).expect("Failed to serialize Event value") } #[implement(super::Pdu)] @@ -262,7 +230,8 @@ pub fn into_stripped_spacechild_state_event_value(self) -> JsonValue { #[implement(super::Pdu)] #[must_use] pub fn into_member_event(self) -> Raw> { - serde_json::from_value(self.into_member_event_value()).expect("Raw::from_value always works") + let value = self.into_member_event_value(); + serde_json::from_value(value).expect("Failed to serialize Event value") } #[implement(super::Pdu)] From 44302ce73289bc70b0e05755f1a23f5d0770f752 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Apr 2025 11:00:07 +0000 Subject: [PATCH 1788/2091] Eliminate explicit parallel_fetches argument. Signed-off-by: Jason Volk --- src/core/matrix/state_res/benches.rs | 4 - src/core/matrix/state_res/mod.rs | 83 +++++++------------ src/core/matrix/state_res/test_utils.rs | 16 ++-- .../rooms/event_handler/resolve_state.rs | 13 +-- 4 files changed, 38 insertions(+), 78 deletions(-) diff --git a/src/core/matrix/state_res/benches.rs b/src/core/matrix/state_res/benches.rs index 01218b01..1aa8552b 100644 --- a/src/core/matrix/state_res/benches.rs +++ b/src/core/matrix/state_res/benches.rs @@ -52,7 +52,6 @@ fn lexico_topo_sort(c: &mut test::Bencher) { #[cfg(conduwuit_bench)] #[cfg_attr(conduwuit_bench, bench)] fn resolution_shallow_auth_chain(c: &mut test::Bencher) { - let parallel_fetches = 32; let mut store = TestStore(hashmap! {}); // build up the DAG @@ -78,7 +77,6 @@ fn resolution_shallow_auth_chain(c: &mut test::Bencher) { &auth_chain_sets, &fetch, &exists, - parallel_fetches, ) .await { @@ -91,7 +89,6 @@ fn resolution_shallow_auth_chain(c: &mut test::Bencher) { #[cfg(conduwuit_bench)] #[cfg_attr(conduwuit_bench, bench)] fn resolve_deeper_event_set(c: &mut test::Bencher) { - let parallel_fetches = 32; let mut inner = INITIAL_EVENTS(); let ban = BAN_STATE_SET(); @@ -153,7 +150,6 @@ fn resolve_deeper_event_set(c: &mut test::Bencher) { &auth_chain_sets, &fetch, &exists, - parallel_fetches, ) .await { diff --git a/src/core/matrix/state_res/mod.rs b/src/core/matrix/state_res/mod.rs index 2ab7cb64..d37368c9 100644 --- a/src/core/matrix/state_res/mod.rs +++ b/src/core/matrix/state_res/mod.rs @@ -69,9 +69,6 @@ type Result = crate::Result; /// * `event_fetch` - Any event not found in the `event_map` will defer to this /// closure to find the event. /// -/// * `parallel_fetches` - The number of asynchronous fetch requests in-flight -/// for any given operation. -/// /// ## Invariants /// /// The caller of `resolve` must ensure that all the events are from the same @@ -85,7 +82,6 @@ pub async fn resolve<'a, E, Sets, SetIter, Hasher, Fetch, FetchFut, Exists, Exis auth_chain_sets: &'a [HashSet], event_fetch: &Fetch, event_exists: &Exists, - parallel_fetches: usize, ) -> Result> where Fetch: Fn(E::Id) -> FetchFut + Sync, @@ -147,13 +143,8 @@ where // Sort the control events based on power_level/clock/event_id and // outgoing/incoming edges - let sorted_control_levels = reverse_topological_power_sort( - control_events, - &all_conflicted, - &event_fetch, - parallel_fetches, - ) - .await?; + let sorted_control_levels = + reverse_topological_power_sort(control_events, &all_conflicted, &event_fetch).await?; debug!(count = sorted_control_levels.len(), "power events"); trace!(list = ?sorted_control_levels, "sorted power events"); @@ -295,7 +286,6 @@ async fn reverse_topological_power_sort( events_to_sort: Vec, auth_diff: &HashSet, fetch_event: &F, - parallel_fetches: usize, ) -> Result> where F: Fn(E::Id) -> Fut + Sync, @@ -311,26 +301,25 @@ where } // This is used in the `key_fn` passed to the lexico_topo_sort fn - let event_to_pl = graph + let event_to_pl: HashMap<_, _> = graph .keys() .stream() - .map(|event_id| { - get_power_level_for_sender(event_id.clone(), fetch_event) - .map(move |res| res.map(|pl| (event_id, pl))) + .broad_filter_map(async |event_id| { + let pl = get_power_level_for_sender(&event_id, fetch_event) + .await + .ok()?; + Some((event_id, pl)) }) - .buffer_unordered(parallel_fetches) - .ready_try_fold(HashMap::new(), |mut event_to_pl, (event_id, pl)| { + .inspect(|(event_id, pl)| { debug!( - event_id = event_id.borrow().as_str(), - power_level = i64::from(pl), + event_id = event_id.as_str(), + power_level = i64::from(*pl), "found the power level of an event's sender", ); - - event_to_pl.insert(event_id.clone(), pl); - Ok(event_to_pl) }) + .collect() .boxed() - .await?; + .await; let event_to_pl = &event_to_pl; let fetcher = |event_id: E::Id| async move { @@ -909,7 +898,7 @@ mod tests { let fetcher = |id| ready(events.get(&id).cloned()); let sorted_power_events = - super::reverse_topological_power_sort(power_events, &auth_chain, &fetcher, 1) + super::reverse_topological_power_sort(power_events, &auth_chain, &fetcher) .await .unwrap(); @@ -1312,19 +1301,13 @@ mod tests { }) .collect(); - let resolved = match super::resolve( - &RoomVersionId::V2, - &state_sets, - &auth_chain, - &fetcher, - &exists, - 1, - ) - .await - { - | Ok(state) => state, - | Err(e) => panic!("{e}"), - }; + let resolved = + match super::resolve(&RoomVersionId::V2, &state_sets, &auth_chain, &fetcher, &exists) + .await + { + | Ok(state) => state, + | Err(e) => panic!("{e}"), + }; assert_eq!(expected, resolved); } @@ -1429,21 +1412,15 @@ mod tests { }) .collect(); - let fetcher = |id: ::Id| ready(ev_map.get(&id).cloned()); - let exists = |id: ::Id| ready(ev_map.get(&id).is_some()); - let resolved = match super::resolve( - &RoomVersionId::V6, - &state_sets, - &auth_chain, - &fetcher, - &exists, - 1, - ) - .await - { - | Ok(state) => state, - | Err(e) => panic!("{e}"), - }; + let fetcher = |id: OwnedEventId| ready(ev_map.get(&id).cloned()); + let exists = |id: OwnedEventId| ready(ev_map.get(&id).is_some()); + let resolved = + match super::resolve(&RoomVersionId::V6, &state_sets, &auth_chain, &fetcher, &exists) + .await + { + | Ok(state) => state, + | Err(e) => panic!("{e}"), + }; debug!( resolved = ?resolved diff --git a/src/core/matrix/state_res/test_utils.rs b/src/core/matrix/state_res/test_utils.rs index a666748a..ff7b30d0 100644 --- a/src/core/matrix/state_res/test_utils.rs +++ b/src/core/matrix/state_res/test_utils.rs @@ -133,17 +133,11 @@ pub(crate) async fn do_check( .collect(); let event_map = &event_map; - let fetch = |id: ::Id| ready(event_map.get(&id).cloned()); - let exists = |id: ::Id| ready(event_map.get(&id).is_some()); - let resolved = super::resolve( - &RoomVersionId::V6, - state_sets, - &auth_chain_sets, - &fetch, - &exists, - 1, - ) - .await; + let fetch = |id: OwnedEventId| ready(event_map.get(&id).cloned()); + let exists = |id: OwnedEventId| ready(event_map.get(&id).is_some()); + let resolved = + super::resolve(&RoomVersionId::V6, state_sets, &auth_chain_sets, &fetch, &exists) + .await; match resolved { | Ok(state) => state, diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index b3a7a71b..a67ac3b7 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -112,14 +112,7 @@ where { let event_fetch = |event_id| self.event_fetch(event_id); let event_exists = |event_id| self.event_exists(event_id); - state_res::resolve( - room_version, - state_sets, - auth_chain_sets, - &event_fetch, - &event_exists, - automatic_width(), - ) - .map_err(|e| err!(error!("State resolution failed: {e:?}"))) - .await + state_res::resolve(room_version, state_sets, auth_chain_sets, &event_fetch, &event_exists) + .map_err(|e| err!(error!("State resolution failed: {e:?}"))) + .await } From f605913ea92d2f741616f0cfd838cf348dc22a34 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 22 Apr 2025 11:00:55 +0000 Subject: [PATCH 1789/2091] Eliminate associated Id type from trait Event. Co-authored-by: Jade Ellis Signed-off-by: Jason Volk --- src/core/matrix/event.rs | 26 ++-- src/core/matrix/pdu.rs | 14 +- src/core/matrix/state_res/benches.rs | 38 +++--- src/core/matrix/state_res/event_auth.rs | 8 +- src/core/matrix/state_res/mod.rs | 128 +++++++++--------- src/core/matrix/state_res/test_utils.rs | 30 ++-- .../rooms/event_handler/resolve_state.rs | 2 +- 7 files changed, 116 insertions(+), 130 deletions(-) diff --git a/src/core/matrix/event.rs b/src/core/matrix/event.rs index 29153334..e4c478cd 100644 --- a/src/core/matrix/event.rs +++ b/src/core/matrix/event.rs @@ -1,18 +1,10 @@ -use std::{ - borrow::Borrow, - fmt::{Debug, Display}, - hash::Hash, -}; - use ruma::{EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId, events::TimelineEventType}; use serde_json::value::RawValue as RawJsonValue; /// Abstraction of a PDU so users can have their own PDU types. pub trait Event { - type Id: Clone + Debug + Display + Eq + Ord + Hash + Send + Borrow; - /// The `EventId` of this event. - fn event_id(&self) -> &Self::Id; + fn event_id(&self) -> &EventId; /// The `RoomId` of this event. fn room_id(&self) -> &RoomId; @@ -34,20 +26,18 @@ pub trait Event { /// The events before this event. // Requires GATs to avoid boxing (and TAIT for making it convenient). - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_; + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_; /// All the authenticating events for this event. // Requires GATs to avoid boxing (and TAIT for making it convenient). - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_; + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_; /// If this event is a redaction event this is the event it redacts. - fn redacts(&self) -> Option<&Self::Id>; + fn redacts(&self) -> Option<&EventId>; } impl Event for &T { - type Id = T::Id; - - fn event_id(&self) -> &Self::Id { (*self).event_id() } + fn event_id(&self) -> &EventId { (*self).event_id() } fn room_id(&self) -> &RoomId { (*self).room_id() } @@ -61,13 +51,13 @@ impl Event for &T { fn state_key(&self) -> Option<&str> { (*self).state_key() } - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { (*self).prev_events() } - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { (*self).auth_events() } - fn redacts(&self) -> Option<&Self::Id> { (*self).redacts() } + fn redacts(&self) -> Option<&EventId> { (*self).redacts() } } diff --git a/src/core/matrix/pdu.rs b/src/core/matrix/pdu.rs index 7e1ecfa8..188586bd 100644 --- a/src/core/matrix/pdu.rs +++ b/src/core/matrix/pdu.rs @@ -79,9 +79,7 @@ impl Pdu { } impl Event for Pdu { - type Id = OwnedEventId; - - fn event_id(&self) -> &Self::Id { &self.event_id } + fn event_id(&self) -> &EventId { &self.event_id } fn room_id(&self) -> &RoomId { &self.room_id } @@ -97,15 +95,15 @@ impl Event for Pdu { fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { - self.prev_events.iter() + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + self.prev_events.iter().map(AsRef::as_ref) } - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { - self.auth_events.iter() + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { + self.auth_events.iter().map(AsRef::as_ref) } - fn redacts(&self) -> Option<&Self::Id> { self.redacts.as_ref() } + fn redacts(&self) -> Option<&EventId> { self.redacts.as_deref() } } /// Prevent derived equality which wouldn't limit itself to event_id diff --git a/src/core/matrix/state_res/benches.rs b/src/core/matrix/state_res/benches.rs index 1aa8552b..12eeab9d 100644 --- a/src/core/matrix/state_res/benches.rs +++ b/src/core/matrix/state_res/benches.rs @@ -186,7 +186,11 @@ impl TestStore { } /// Returns a Vec of the related auth events to the given `event`. - fn auth_event_ids(&self, room_id: &RoomId, event_ids: Vec) -> Result> { + fn auth_event_ids( + &self, + room_id: &RoomId, + event_ids: Vec, + ) -> Result> { let mut result = HashSet::new(); let mut stack = event_ids; @@ -212,8 +216,8 @@ impl TestStore { fn auth_chain_diff( &self, room_id: &RoomId, - event_ids: Vec>, - ) -> Result> { + event_ids: Vec>, + ) -> Result> { let mut auth_chain_sets = vec![]; for ids in event_ids { // TODO state store `auth_event_ids` returns self in the event ids list @@ -234,7 +238,7 @@ impl TestStore { Ok(auth_chain_sets .into_iter() .flatten() - .filter(|id| !common.contains(id.borrow())) + .filter(|id| !common.contains(id)) .collect()) } else { Ok(vec![]) @@ -561,7 +565,7 @@ impl EventTypeExt for &TimelineEventType { mod event { use ruma::{ - MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, + EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, events::{TimelineEventType, pdu::Pdu}, }; use serde::{Deserialize, Serialize}; @@ -570,9 +574,7 @@ mod event { use super::Event; impl Event for PduEvent { - type Id = OwnedEventId; - - fn event_id(&self) -> &Self::Id { &self.event_id } + fn event_id(&self) -> &EventId { &self.event_id } fn room_id(&self) -> &RoomId { match &self.rest { @@ -628,28 +630,30 @@ mod event { } } - fn prev_events(&self) -> Box + Send + '_> { + fn prev_events(&self) -> Box + Send + '_> { match &self.rest { - | Pdu::RoomV1Pdu(ev) => Box::new(ev.prev_events.iter().map(|(id, _)| id)), - | Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter()), + | Pdu::RoomV1Pdu(ev) => + Box::new(ev.prev_events.iter().map(|(id, _)| id.as_ref())), + | Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter().map(AsRef::as_ref)), #[cfg(not(feature = "unstable-exhaustive-types"))] | _ => unreachable!("new PDU version"), } } - fn auth_events(&self) -> Box + Send + '_> { + fn auth_events(&self) -> Box + Send + '_> { match &self.rest { - | Pdu::RoomV1Pdu(ev) => Box::new(ev.auth_events.iter().map(|(id, _)| id)), - | Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter()), + | Pdu::RoomV1Pdu(ev) => + Box::new(ev.auth_events.iter().map(|(id, _)| id.as_ref())), + | Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter().map(AsRef::as_ref)), #[cfg(not(feature = "unstable-exhaustive-types"))] | _ => unreachable!("new PDU version"), } } - fn redacts(&self) -> Option<&Self::Id> { + fn redacts(&self) -> Option<&EventId> { match &self.rest { - | Pdu::RoomV1Pdu(ev) => ev.redacts.as_ref(), - | Pdu::RoomV3Pdu(ev) => ev.redacts.as_ref(), + | Pdu::RoomV1Pdu(ev) => ev.redacts.as_deref(), + | Pdu::RoomV3Pdu(ev) => ev.redacts.as_deref(), #[cfg(not(feature = "unstable-exhaustive-types"))] | _ => unreachable!("new PDU version"), } diff --git a/src/core/matrix/state_res/event_auth.rs b/src/core/matrix/state_res/event_auth.rs index c69db50e..715e5156 100644 --- a/src/core/matrix/state_res/event_auth.rs +++ b/src/core/matrix/state_res/event_auth.rs @@ -133,7 +133,7 @@ pub fn auth_types_for_event( level = "debug", skip_all, fields( - event_id = incoming_event.event_id().borrow().as_str() + event_id = incoming_event.event_id().as_str(), ) )] pub async fn auth_check( @@ -259,7 +259,7 @@ where // 3. If event does not have m.room.create in auth_events reject if !incoming_event .auth_events() - .any(|id| id.borrow() == room_create_event.event_id().borrow()) + .any(|id| id == room_create_event.event_id()) { warn!("no m.room.create event in auth events"); return Ok(false); @@ -1021,11 +1021,11 @@ fn check_redaction( // If the domain of the event_id of the event being redacted is the same as the // domain of the event_id of the m.room.redaction, allow - if redaction_event.event_id().borrow().server_name() + if redaction_event.event_id().server_name() == redaction_event .redacts() .as_ref() - .and_then(|&id| id.borrow().server_name()) + .and_then(|&id| id.server_name()) { debug!("redaction event allowed via room version 1 rules"); return Ok(true); diff --git a/src/core/matrix/state_res/mod.rs b/src/core/matrix/state_res/mod.rs index d37368c9..651f6130 100644 --- a/src/core/matrix/state_res/mod.rs +++ b/src/core/matrix/state_res/mod.rs @@ -20,7 +20,7 @@ use std::{ use futures::{Future, FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, future}; use ruma::{ - EventId, Int, MilliSecondsSinceUnixEpoch, RoomVersionId, + EventId, Int, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomVersionId, events::{ StateEventType, TimelineEventType, room::member::{MembershipState, RoomMemberEventContent}, @@ -39,9 +39,7 @@ use crate::{ debug, debug_error, matrix::{event::Event, pdu::StateKey}, trace, - utils::stream::{ - BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryReadyExt, WidebandExt, - }, + utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, WidebandExt}, warn, }; @@ -79,20 +77,19 @@ type Result = crate::Result; pub async fn resolve<'a, E, Sets, SetIter, Hasher, Fetch, FetchFut, Exists, ExistsFut>( room_version: &RoomVersionId, state_sets: Sets, - auth_chain_sets: &'a [HashSet], + auth_chain_sets: &'a [HashSet], event_fetch: &Fetch, event_exists: &Exists, -) -> Result> +) -> Result> where - Fetch: Fn(E::Id) -> FetchFut + Sync, + Fetch: Fn(OwnedEventId) -> FetchFut + Sync, FetchFut: Future> + Send, - Exists: Fn(E::Id) -> ExistsFut + Sync, + Exists: Fn(OwnedEventId) -> ExistsFut + Sync, ExistsFut: Future + Send, Sets: IntoIterator + Send, - SetIter: Iterator> + Clone + Send, + SetIter: Iterator> + Clone + Send, Hasher: BuildHasher + Send + Sync, E: Event + Clone + Send + Sync, - E::Id: Borrow + Send + Sync, for<'b> &'b E: Send, { debug!("State resolution starting"); @@ -153,7 +150,7 @@ where // Sequentially auth check each control event. let resolved_control = iterative_auth_check( &room_version, - sorted_control_levels.iter().stream(), + sorted_control_levels.iter().stream().map(AsRef::as_ref), clean.clone(), &event_fetch, ) @@ -170,7 +167,7 @@ where // that failed auth let events_to_resolve: Vec<_> = all_conflicted .iter() - .filter(|&id| !deduped_power_ev.contains(id.borrow())) + .filter(|&id| !deduped_power_ev.contains(id)) .cloned() .collect(); @@ -190,7 +187,7 @@ where let mut resolved_state = iterative_auth_check( &room_version, - sorted_left_events.iter().stream(), + sorted_left_events.iter().stream().map(AsRef::as_ref), resolved_control, // The control events are added to the final resolved state &event_fetch, ) @@ -283,15 +280,14 @@ where /// earlier (further back in time) origin server timestamp. #[tracing::instrument(level = "debug", skip_all)] async fn reverse_topological_power_sort( - events_to_sort: Vec, - auth_diff: &HashSet, + events_to_sort: Vec, + auth_diff: &HashSet, fetch_event: &F, -) -> Result> +) -> Result> where - F: Fn(E::Id) -> Fut + Sync, + F: Fn(OwnedEventId) -> Fut + Sync, Fut: Future> + Send, E: Event + Send + Sync, - E::Id: Borrow + Send + Sync, { debug!("reverse topological sort of power events"); @@ -303,6 +299,7 @@ where // This is used in the `key_fn` passed to the lexico_topo_sort fn let event_to_pl: HashMap<_, _> = graph .keys() + .cloned() .stream() .broad_filter_map(async |event_id| { let pl = get_power_level_for_sender(&event_id, fetch_event) @@ -321,14 +318,15 @@ where .boxed() .await; - let event_to_pl = &event_to_pl; - let fetcher = |event_id: E::Id| async move { + let fetcher = async |event_id: OwnedEventId| { let pl = *event_to_pl - .get(event_id.borrow()) + .get(&event_id) .ok_or_else(|| Error::NotFound(String::new()))?; + let ev = fetch_event(event_id) .await .ok_or_else(|| Error::NotFound(String::new()))?; + Ok((pl, ev.origin_server_ts())) }; @@ -465,18 +463,17 @@ where /// the eventId at the eventId's generation (we walk backwards to `EventId`s /// most recent previous power level event). async fn get_power_level_for_sender( - event_id: E::Id, + event_id: &EventId, fetch_event: &F, ) -> serde_json::Result where - F: Fn(E::Id) -> Fut + Sync, + F: Fn(OwnedEventId) -> Fut + Sync, Fut: Future> + Send, E: Event + Send, - E::Id: Borrow + Send, { debug!("fetch event ({event_id}) senders power level"); - let event = fetch_event(event_id).await; + let event = fetch_event(event_id.to_owned()).await; let auth_events = event.as_ref().map(Event::auth_events); @@ -484,7 +481,7 @@ where .into_iter() .flatten() .stream() - .broadn_filter_map(5, |aid| fetch_event(aid.clone())) + .broadn_filter_map(5, |aid| fetch_event(aid.to_owned())) .ready_find(|aev| is_type_and_key(aev, &TimelineEventType::RoomPowerLevels, "")) .await; @@ -517,14 +514,13 @@ where async fn iterative_auth_check<'a, E, F, Fut, S>( room_version: &RoomVersion, events_to_check: S, - unconflicted_state: StateMap, + unconflicted_state: StateMap, fetch_event: &F, -) -> Result> +) -> Result> where - F: Fn(E::Id) -> Fut + Sync, + F: Fn(OwnedEventId) -> Fut + Sync, Fut: Future> + Send, - E::Id: Borrow + Clone + Eq + Ord + Send + Sync + 'a, - S: Stream + Send + 'a, + S: Stream + Send + 'a, E: Event + Clone + Send + Sync, { debug!("starting iterative auth check"); @@ -532,7 +528,7 @@ where let events_to_check: Vec<_> = events_to_check .map(Result::Ok) .broad_and_then(async |event_id| { - fetch_event(event_id.clone()) + fetch_event(event_id.to_owned()) .await .ok_or_else(|| Error::NotFound(format!("Failed to find {event_id}"))) }) @@ -540,16 +536,16 @@ where .boxed() .await?; - let auth_event_ids: HashSet = events_to_check + let auth_event_ids: HashSet = events_to_check .iter() - .flat_map(|event: &E| event.auth_events().map(Clone::clone)) + .flat_map(|event: &E| event.auth_events().map(ToOwned::to_owned)) .collect(); - let auth_events: HashMap = auth_event_ids + let auth_events: HashMap = auth_event_ids .into_iter() .stream() .broad_filter_map(fetch_event) - .map(|auth_event| (auth_event.event_id().clone(), auth_event)) + .map(|auth_event| (auth_event.event_id().to_owned(), auth_event)) .collect() .boxed() .await; @@ -570,7 +566,7 @@ where let mut auth_state = StateMap::new(); for aid in event.auth_events() { - if let Some(ev) = auth_events.get(aid.borrow()) { + if let Some(ev) = auth_events.get(aid) { //TODO: synapse checks "rejected_reason" which is most likely related to // soft-failing auth_state.insert( @@ -581,7 +577,7 @@ where ev.clone(), ); } else { - warn!(event_id = aid.borrow().as_str(), "missing auth event"); + warn!(event_id = aid.as_str(), "missing auth event"); } } @@ -590,7 +586,7 @@ where .stream() .ready_filter_map(|key| Some((key, resolved_state.get(key)?))) .filter_map(|(key, ev_id)| async move { - if let Some(event) = auth_events.get(ev_id.borrow()) { + if let Some(event) = auth_events.get(ev_id) { Some((key, event.clone())) } else { Some((key, fetch_event(ev_id.clone()).await?)) @@ -622,7 +618,7 @@ where // add event to resolved state map resolved_state.insert( event.event_type().with_state_key(state_key), - event.event_id().clone(), + event.event_id().to_owned(), ); }, | Ok(false) => { @@ -649,15 +645,14 @@ where /// level as a parent) will be marked as depth 1. depth 1 is "older" than depth /// 0. async fn mainline_sort( - to_sort: &[E::Id], - resolved_power_level: Option, + to_sort: &[OwnedEventId], + resolved_power_level: Option, fetch_event: &F, -) -> Result> +) -> Result> where - F: Fn(E::Id) -> Fut + Sync, + F: Fn(OwnedEventId) -> Fut + Sync, Fut: Future> + Send, E: Event + Clone + Send + Sync, - E::Id: Borrow + Clone + Send + Sync, { debug!("mainline sort of events"); @@ -677,7 +672,7 @@ where pl = None; for aid in event.auth_events() { - let ev = fetch_event(aid.clone()) + let ev = fetch_event(aid.to_owned()) .await .ok_or_else(|| Error::NotFound(format!("Failed to find {aid}")))?; @@ -723,26 +718,25 @@ where /// that has an associated mainline depth. async fn get_mainline_depth( mut event: Option, - mainline_map: &HashMap, + mainline_map: &HashMap, fetch_event: &F, ) -> Result where - F: Fn(E::Id) -> Fut + Sync, + F: Fn(OwnedEventId) -> Fut + Sync, Fut: Future> + Send, E: Event + Send + Sync, - E::Id: Borrow + Send + Sync, { while let Some(sort_ev) = event { - debug!(event_id = sort_ev.event_id().borrow().as_str(), "mainline"); + debug!(event_id = sort_ev.event_id().as_str(), "mainline"); let id = sort_ev.event_id(); - if let Some(depth) = mainline_map.get(id.borrow()) { + if let Some(depth) = mainline_map.get(id) { return Ok(*depth); } event = None; for aid in sort_ev.auth_events() { - let aev = fetch_event(aid.clone()) + let aev = fetch_event(aid.to_owned()) .await .ok_or_else(|| Error::NotFound(format!("Failed to find {aid}")))?; @@ -757,15 +751,14 @@ where } async fn add_event_and_auth_chain_to_graph( - graph: &mut HashMap>, - event_id: E::Id, - auth_diff: &HashSet, + graph: &mut HashMap>, + event_id: OwnedEventId, + auth_diff: &HashSet, fetch_event: &F, ) where - F: Fn(E::Id) -> Fut + Sync, + F: Fn(OwnedEventId) -> Fut + Sync, Fut: Future> + Send, E: Event + Send + Sync, - E::Id: Borrow + Clone + Send + Sync, { let mut state = vec![event_id]; while let Some(eid) = state.pop() { @@ -775,26 +768,27 @@ async fn add_event_and_auth_chain_to_graph( // Prefer the store to event as the store filters dedups the events for aid in auth_events { - if auth_diff.contains(aid.borrow()) { - if !graph.contains_key(aid.borrow()) { + if auth_diff.contains(aid) { + if !graph.contains_key(aid) { state.push(aid.to_owned()); } - // We just inserted this at the start of the while loop - graph.get_mut(eid.borrow()).unwrap().insert(aid.to_owned()); + graph + .get_mut(&eid) + .expect("We just inserted this at the start of the while loop") + .insert(aid.to_owned()); } } } } -async fn is_power_event_id(event_id: &E::Id, fetch: &F) -> bool +async fn is_power_event_id(event_id: &EventId, fetch: &F) -> bool where - F: Fn(E::Id) -> Fut + Sync, + F: Fn(OwnedEventId) -> Fut + Sync, Fut: Future> + Send, E: Event + Send, - E::Id: Borrow + Send + Sync, { - match fetch(event_id.clone()).await.as_ref() { + match fetch(event_id.to_owned()).await.as_ref() { | Some(state) => is_power_event(state), | _ => false, } @@ -904,7 +898,7 @@ mod tests { let resolved_power = super::iterative_auth_check( &RoomVersion::V6, - sorted_power_events.iter().stream(), + sorted_power_events.iter().map(AsRef::as_ref).stream(), HashMap::new(), // unconflicted events &fetcher, ) @@ -1289,7 +1283,7 @@ mod tests { let ev_map = store.0.clone(); let fetcher = |id| ready(ev_map.get(&id).cloned()); - let exists = |id: ::Id| ready(ev_map.get(&*id).is_some()); + let exists = |id: OwnedEventId| ready(ev_map.get(&*id).is_some()); let state_sets = [state_at_bob, state_at_charlie]; let auth_chain: Vec<_> = state_sets diff --git a/src/core/matrix/state_res/test_utils.rs b/src/core/matrix/state_res/test_utils.rs index ff7b30d0..c6945f66 100644 --- a/src/core/matrix/state_res/test_utils.rs +++ b/src/core/matrix/state_res/test_utils.rs @@ -241,8 +241,8 @@ impl TestStore { pub(crate) fn auth_event_ids( &self, room_id: &RoomId, - event_ids: Vec, - ) -> Result> { + event_ids: Vec, + ) -> Result> { let mut result = HashSet::new(); let mut stack = event_ids; @@ -578,7 +578,7 @@ pub(crate) fn INITIAL_EDGES() -> Vec { pub(crate) mod event { use ruma::{ - MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, + EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, events::{TimelineEventType, pdu::Pdu}, }; use serde::{Deserialize, Serialize}; @@ -587,9 +587,7 @@ pub(crate) mod event { use crate::Event; impl Event for PduEvent { - type Id = OwnedEventId; - - fn event_id(&self) -> &Self::Id { &self.event_id } + fn event_id(&self) -> &EventId { &self.event_id } fn room_id(&self) -> &RoomId { match &self.rest { @@ -646,29 +644,31 @@ pub(crate) mod event { } #[allow(refining_impl_trait)] - fn prev_events(&self) -> Box + Send + '_> { + fn prev_events(&self) -> Box + Send + '_> { match &self.rest { - | Pdu::RoomV1Pdu(ev) => Box::new(ev.prev_events.iter().map(|(id, _)| id)), - | Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter()), + | Pdu::RoomV1Pdu(ev) => + Box::new(ev.prev_events.iter().map(|(id, _)| id.as_ref())), + | Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter().map(AsRef::as_ref)), #[allow(unreachable_patterns)] | _ => unreachable!("new PDU version"), } } #[allow(refining_impl_trait)] - fn auth_events(&self) -> Box + Send + '_> { + fn auth_events(&self) -> Box + Send + '_> { match &self.rest { - | Pdu::RoomV1Pdu(ev) => Box::new(ev.auth_events.iter().map(|(id, _)| id)), - | Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter()), + | Pdu::RoomV1Pdu(ev) => + Box::new(ev.auth_events.iter().map(|(id, _)| id.as_ref())), + | Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter().map(AsRef::as_ref)), #[allow(unreachable_patterns)] | _ => unreachable!("new PDU version"), } } - fn redacts(&self) -> Option<&Self::Id> { + fn redacts(&self) -> Option<&EventId> { match &self.rest { - | Pdu::RoomV1Pdu(ev) => ev.redacts.as_ref(), - | Pdu::RoomV3Pdu(ev) => ev.redacts.as_ref(), + | Pdu::RoomV1Pdu(ev) => ev.redacts.as_deref(), + | Pdu::RoomV3Pdu(ev) => ev.redacts.as_deref(), #[allow(unreachable_patterns)] | _ => unreachable!("new PDU version"), } diff --git a/src/service/rooms/event_handler/resolve_state.rs b/src/service/rooms/event_handler/resolve_state.rs index a67ac3b7..cd747e04 100644 --- a/src/service/rooms/event_handler/resolve_state.rs +++ b/src/service/rooms/event_handler/resolve_state.rs @@ -8,7 +8,7 @@ use conduwuit::{ Error, Result, err, implement, state_res::{self, StateMap}, trace, - utils::stream::{IterStream, ReadyExt, TryWidebandExt, WidebandExt, automatic_width}, + utils::stream::{IterStream, ReadyExt, TryWidebandExt, WidebandExt}, }; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt, future::try_join}; use ruma::{OwnedEventId, RoomId, RoomVersionId}; From 3e4e696761e75bc5e673aea66bd6407088523944 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 12:35:25 +0100 Subject: [PATCH 1790/2091] fix: Make sure empty VERSION_EXTRA strings are ignored Also updates built & removes unused optional features --- Cargo.lock | 45 ++++++------------------------- src/build_metadata/Cargo.toml | 8 +++--- src/build_metadata/build.rs | 1 + src/build_metadata/mod.rs | 11 +++++--- src/core/info/version.rs | 2 +- src/web/mod.rs | 2 +- src/web/templates/_layout.html.j2 | 2 +- 7 files changed, 24 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 18375234..04e4f36e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -584,9 +584,12 @@ name = "built" version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56ed6191a7e78c36abdb16ab65341eefd73d64d303fffccdbb00d51e4205967b" -dependencies = [ - "cargo-lock", -] + +[[package]] +name = "built" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" [[package]] name = "bumpalo" @@ -634,19 +637,6 @@ dependencies = [ "pkg-config", ] -[[package]] -name = "cargo-lock" -version = "10.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06acb4f71407ba205a07cb453211e0e6a67b21904e47f6ba1f9589e38f2e454" -dependencies = [ - "petgraph", - "semver", - "serde", - "toml", - "url", -] - [[package]] name = "cargo_toml" version = "0.21.0" @@ -876,7 +866,7 @@ dependencies = [ name = "conduwuit_build_metadata" version = "0.5.0-rc.5" dependencies = [ - "built", + "built 0.8.0", ] [[package]] @@ -1541,12 +1531,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - [[package]] name = "flate2" version = "1.1.1" @@ -3164,16 +3148,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "petgraph" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" -dependencies = [ - "fixedbitset", - "indexmap 2.8.0", -] - [[package]] name = "phf" version = "0.11.3" @@ -3565,7 +3539,7 @@ dependencies = [ "arrayvec", "av1-grain", "bitstream-io", - "built", + "built 0.7.7", "cfg-if", "interpolate_name", "itertools 0.12.1", @@ -4169,9 +4143,6 @@ name = "semver" version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" -dependencies = [ - "serde", -] [[package]] name = "sentry" diff --git a/src/build_metadata/Cargo.toml b/src/build_metadata/Cargo.toml index 3a98c6bf..62c4dc70 100644 --- a/src/build_metadata/Cargo.toml +++ b/src/build_metadata/Cargo.toml @@ -13,13 +13,13 @@ version.workspace = true build = "build.rs" # [[bin]] # path = "main.rs" -# name = "conduwuit_build_metadata" +# name = "conduwuit_build_metadata" [lib] path = "mod.rs" crate-type = [ - "rlib", -# "dylib", + "rlib", + # "dylib", ] [features] @@ -28,7 +28,7 @@ crate-type = [ [dependencies] [build-dependencies] -built = {version = "0.7", features = ["cargo-lock", "dependency-tree"]} +built = { version = "0.8", features = [] } [lints] workspace = true diff --git a/src/build_metadata/build.rs b/src/build_metadata/build.rs index 2fec16a7..bfdf20b1 100644 --- a/src/build_metadata/build.rs +++ b/src/build_metadata/build.rs @@ -78,6 +78,7 @@ fn main() { } // --- Rerun Triggers --- + // TODO: The git rerun triggers seem to always run // Rerun if the git HEAD changes println!("cargo:rerun-if-changed=.git/HEAD"); // Rerun if the ref pointed to by HEAD changes (e.g., new commit on branch) diff --git a/src/build_metadata/mod.rs b/src/build_metadata/mod.rs index cf3364c1..f50018d2 100644 --- a/src/build_metadata/mod.rs +++ b/src/build_metadata/mod.rs @@ -12,11 +12,16 @@ pub static VERSION_EXTRA: Option<&str> = v } else if let v @ Some(_) = option_env!("CONDUWUIT_VERSION_EXTRA") { v - } else if let v @ Some(_) = option_env!("CONDUIT_VERSION_EXTRA") { - v } else { - GIT_COMMIT_HASH_SHORT + option_env!("CONDUIT_VERSION_EXTRA") }; + +pub fn version_tag() -> Option<&'static str> { + VERSION_EXTRA + .filter(|s| !s.is_empty()) + .or(GIT_COMMIT_HASH_SHORT) +} + pub static GIT_REMOTE_WEB_URL: Option<&str> = option_env!("GIT_REMOTE_WEB_URL"); pub static GIT_REMOTE_COMMIT_URL: Option<&str> = option_env!("GIT_REMOTE_COMMIT_URL"); diff --git a/src/core/info/version.rs b/src/core/info/version.rs index 523c40a2..c22c8ec8 100644 --- a/src/core/info/version.rs +++ b/src/core/info/version.rs @@ -26,6 +26,6 @@ pub fn user_agent() -> &'static str { USER_AGENT.get_or_init(init_user_agent) } fn init_user_agent() -> String { format!("{}/{}", name(), version()) } fn init_version() -> String { - conduwuit_build_metadata::VERSION_EXTRA + conduwuit_build_metadata::version_tag() .map_or(SEMANTIC.to_owned(), |extra| format!("{SEMANTIC} ({extra})")) } diff --git a/src/web/mod.rs b/src/web/mod.rs index 25ec868c..9c6a5d83 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -6,7 +6,7 @@ use axum::{ response::{Html, IntoResponse, Response}, routing::get, }; -use conduwuit_build_metadata::{GIT_REMOTE_COMMIT_URL, GIT_REMOTE_WEB_URL, VERSION_EXTRA}; +use conduwuit_build_metadata::{GIT_REMOTE_COMMIT_URL, GIT_REMOTE_WEB_URL, version_tag}; use conduwuit_service::state; pub fn build() -> Router { diff --git a/src/web/templates/_layout.html.j2 b/src/web/templates/_layout.html.j2 index fd0a5b29..d298b68c 100644 --- a/src/web/templates/_layout.html.j2 +++ b/src/web/templates/_layout.html.j2 @@ -18,7 +18,7 @@ {%~ block footer ~%}

    Powered by Continuwuity - {%~ if let Some(version_info) = VERSION_EXTRA ~%} + {%~ if let Some(version_info) = self::version_tag() ~%} {%~ if let Some(url) = GIT_REMOTE_COMMIT_URL.or(GIT_REMOTE_WEB_URL) ~%} ({{ version_info }}) {%~ else ~%} From fce024b30b60fde435b211e7edd2131c8a2613c4 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 12:45:14 +0100 Subject: [PATCH 1791/2091] chore: Add must_use annotation --- src/build_metadata/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/build_metadata/mod.rs b/src/build_metadata/mod.rs index f50018d2..86a8a800 100644 --- a/src/build_metadata/mod.rs +++ b/src/build_metadata/mod.rs @@ -16,6 +16,7 @@ pub static VERSION_EXTRA: Option<&str> = option_env!("CONDUIT_VERSION_EXTRA") }; +#[must_use] pub fn version_tag() -> Option<&'static str> { VERSION_EXTRA .filter(|s| !s.is_empty()) From dcbc4b54c51b6325925d477f2cf31e2679d34759 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 12:45:25 +0100 Subject: [PATCH 1792/2091] ci: Always show sccache stats --- .forgejo/workflows/rust-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.forgejo/workflows/rust-checks.yml b/.forgejo/workflows/rust-checks.yml index 1feb9e89..35ca1ad7 100644 --- a/.forgejo/workflows/rust-checks.yml +++ b/.forgejo/workflows/rust-checks.yml @@ -80,6 +80,7 @@ jobs: -D warnings - name: Show sccache stats + if: always() run: sccache --show-stats cargo-test: @@ -137,4 +138,5 @@ jobs: --no-fail-fast - name: Show sccache stats + if: always() run: sccache --show-stats From ce40304667f5bcb31f16978d955a51595da26c2a Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 15:28:46 +0100 Subject: [PATCH 1793/2091] chore: Upgrade deps --- Cargo.lock | 431 ++++++++++++++++++++++++----------------------------- 1 file changed, 191 insertions(+), 240 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04e4f36e..160be0c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,9 +55,9 @@ checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "arbitrary" @@ -170,9 +170,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.22" +version = "0.4.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a194f9d963d8099596278594b3107448656ba73831c9d8c783e613ce86da64" +checksum = "b37fc50485c4f3f736a4fb14199f6d5f5ba008d7f28fe710306c92780f004c07" dependencies = [ "brotli", "flate2", @@ -184,17 +184,6 @@ dependencies = [ "zstd-safe", ] -[[package]] -name = "async-recursion" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-stream" version = "0.3.6" @@ -284,9 +273,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.28.0" +version = "0.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9f7720b74ed28ca77f90769a71fd8c637a0137f6fae4ae947e1050229cff57f" +checksum = "bfa9b6986f250236c27e5a204062434a773a13243d2ffc2955f37bdba4c5c6a1" dependencies = [ "bindgen 0.69.5", "cc", @@ -426,9 +415,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.74" +version = "0.3.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" dependencies = [ "addr2line", "cfg-if", @@ -560,9 +549,9 @@ dependencies = [ [[package]] name = "brotli" -version = "7.0.0" +version = "8.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc97b8f16f944bba54f0433f07e30be199b6dc2bd25937444bbad560bcea29bd" +checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -571,9 +560,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.2" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -599,9 +588,9 @@ checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "bytemuck" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b1fc10dbac614ebc03540c9dbd60e83887fda27794998c6528f1782047d540" +checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" [[package]] name = "byteorder" @@ -649,9 +638,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.17" +version = "1.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a" +checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1" dependencies = [ "jobserver", "libc", @@ -700,9 +689,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a7964611d71df112cb1730f2ee67324fcf4d0fc6606acbbe9bfe06df124637c" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" dependencies = [ "num-traits", ] @@ -720,9 +709,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.35" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" +checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" dependencies = [ "clap_builder", "clap_derive", @@ -730,9 +719,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.35" +version = "4.5.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" +checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" dependencies = [ "anstyle", "clap_lex", @@ -1003,7 +992,7 @@ dependencies = [ "const-str", "either", "futures", - "hickory-resolver 0.25.1", + "hickory-resolver 0.25.2", "http", "image", "ipaddress", @@ -1326,9 +1315,9 @@ checksum = "817fa642fb0ee7fe42e95783e00e0969927b96091bdd4b9b1af082acd943913b" [[package]] name = "data-encoding" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "575f75dfd25738df5b91b8e43e14d44bda14637a58fae779fd2b064f8bf3e010" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "date_header" @@ -1447,9 +1436,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", "windows-sys 0.59.0", @@ -1690,7 +1679,7 @@ dependencies = [ "libc", "log", "rustversion", - "windows 0.58.0", + "windows", ] [[package]] @@ -1705,9 +1694,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "js-sys", @@ -1718,9 +1707,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "js-sys", @@ -1754,9 +1743,9 @@ checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes", @@ -1764,7 +1753,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.8.0", + "indexmap 2.9.0", "slab", "tokio", "tokio-util", @@ -1773,9 +1762,9 @@ dependencies = [ [[package]] name = "half" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", @@ -1795,9 +1784,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" [[package]] name = "hdrhistogram" @@ -1880,14 +1869,12 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.25.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d844af74f7b799e41c78221be863bade11c430d46042c3b49ca8ae0c6d27287" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ - "async-recursion", "async-trait", "cfg-if", - "critical-section", "data-encoding", "enum-as-inner", "futures-channel", @@ -1896,7 +1883,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.0", + "rand 0.9.1", "ring", "serde", "thiserror 2.0.12", @@ -1929,18 +1916,18 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.25.1" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a128410b38d6f931fcc6ca5c107a3b02cabd6c05967841269a4ad65d23c44331" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" dependencies = [ "cfg-if", "futures-util", - "hickory-proto 0.25.1", + "hickory-proto 0.25.2", "ipconfig", "moka", "once_cell", "parking_lot", - "rand 0.9.0", + "rand 0.9.1", "resolv-conf", "serde", "smallvec", @@ -1969,13 +1956,13 @@ dependencies = [ [[package]] name = "hostname" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9c7c7c8ac16c798734b8a24560c1362120597c40d5e1459f09498f8f6c8f2ba" +checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" dependencies = [ "cfg-if", "libc", - "windows 0.52.0", + "windows-link", ] [[package]] @@ -2127,21 +2114,22 @@ dependencies = [ [[package]] name = "icu_collections" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" dependencies = [ "displaydoc", + "potential_utf", "yoke", "zerofrom", "zerovec", ] [[package]] -name = "icu_locid" -version = "1.5.0" +name = "icu_locale_core" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" dependencies = [ "displaydoc", "litemap", @@ -2150,31 +2138,11 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" - [[package]] name = "icu_normalizer" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" dependencies = [ "displaydoc", "icu_collections", @@ -2182,67 +2150,54 @@ dependencies = [ "icu_properties", "icu_provider", "smallvec", - "utf16_iter", - "utf8_iter", - "write16", "zerovec", ] [[package]] name = "icu_normalizer_data" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a" dependencies = [ "displaydoc", "icu_collections", - "icu_locid_transform", + "icu_locale_core", "icu_properties_data", "icu_provider", - "tinystr", + "potential_utf", + "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "1.5.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" +checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04" [[package]] name = "icu_provider" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" dependencies = [ "displaydoc", - "icu_locid", - "icu_provider_macros", + "icu_locale_core", "stable_deref_trait", "tinystr", "writeable", "yoke", "zerofrom", + "zerotrie", "zerovec", ] -[[package]] -name = "icu_provider_macros" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "idna" version = "1.0.3" @@ -2256,9 +2211,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" dependencies = [ "icu_normalizer", "icu_properties", @@ -2315,12 +2270,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" +checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.2", + "hashbrown 0.15.3", "serde", ] @@ -2418,7 +2373,7 @@ version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", "libc", ] @@ -2519,9 +2474,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.171" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libfuzzer-sys" @@ -2535,12 +2490,12 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +checksum = "6a793df0d7afeac54f95b471d3af7f0d4fb975699f972341a4b76988d49cdf0c" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-targets 0.53.0", ] [[package]] @@ -2568,9 +2523,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" @@ -2590,9 +2545,9 @@ checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" [[package]] name = "loole" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2998397c725c822c6b2ba605fd9eb4c6a7a0810f1629ba3cc232ef4f0308d96" +checksum = "1a3932a13b27d6b2d37efec3e4047017d59a8c9f2283a29bde29151d22f00fe9" dependencies = [ "futures-core", "futures-sink", @@ -2716,9 +2671,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minicbor" -version = "0.26.3" +version = "0.26.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1936e27fffe7d8557c060eb82cb71668608cd1a5fb56b63e66d22ae8d7564321" +checksum = "8a309f581ade7597820083bc275075c4c6986e57e53f8d26f88507cfefc8c987" dependencies = [ "minicbor-derive", ] @@ -2761,9 +2716,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" +checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" dependencies = [ "adler2", "simd-adler32", @@ -2983,7 +2938,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.8.0", + "indexmap 2.9.0", "js-sys", "once_cell", "pin-project-lite", @@ -3253,6 +3208,15 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3295,9 +3259,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -3426,8 +3390,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" dependencies = [ "bytes", - "getrandom 0.3.2", - "rand 0.9.0", + "getrandom 0.3.3", + "rand 0.9.1", "ring", "rustc-hash 2.1.1", "rustls", @@ -3481,13 +3445,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy", ] [[package]] @@ -3516,7 +3479,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.15", + "getrandom 0.2.16", ] [[package]] @@ -3525,7 +3488,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", ] [[package]] @@ -3565,9 +3528,9 @@ dependencies = [ [[package]] name = "ravif" -version = "0.11.11" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2413fd96bd0ea5cdeeb37eaf446a22e6ed7b981d792828721e74ded1980a45c6" +checksum = "d6a5f31fcf7500f9401fea858ea4ab5525c99f2322cfcee732c0e6c74208c0c6" dependencies = [ "avif-serialize", "imgref", @@ -3600,9 +3563,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" dependencies = [ "bitflags 2.9.0", ] @@ -3723,7 +3686,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.15", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -3793,9 +3756,9 @@ dependencies = [ "base64 0.22.1", "bytes", "form_urlencoded", - "getrandom 0.2.15", + "getrandom 0.2.16", "http", - "indexmap 2.8.0", + "indexmap 2.9.0", "js_int", "konst", "percent-encoding", @@ -3822,7 +3785,7 @@ version = "0.28.1" source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" dependencies = [ "as_variant", - "indexmap 2.8.0", + "indexmap 2.9.0", "js_int", "js_option", "percent-encoding", @@ -3993,9 +3956,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.25" +version = "0.23.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" +checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" dependencies = [ "aws-lc-rs", "log", @@ -4030,18 +3993,19 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ "web-time 1.1.0", + "zeroize", ] [[package]] name = "rustls-webpki" -version = "0.103.1" +version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ "aws-lc-rs", "ring", @@ -4306,7 +4270,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", - "indexmap 2.8.0", + "indexmap 2.9.0", "itoa", "ryu", "serde", @@ -4371,7 +4335,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.8.0", + "indexmap 2.9.0", "itoa", "ryu", "serde", @@ -4391,9 +4355,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -4438,9 +4402,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -4496,9 +4460,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" dependencies = [ "serde", ] @@ -4577,9 +4541,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.100" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -4597,9 +4561,9 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", @@ -4644,9 +4608,9 @@ dependencies = [ [[package]] name = "termimad" -version = "0.31.2" +version = "0.31.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8e19c6dbf107bec01d0e216bb8219485795b7d75328e4fa5ef2756c1be4f8dc" +checksum = "7301d9c2c4939c97f25376b70d3c13311f8fefdee44092fc361d2a98adc2cbb6" dependencies = [ "coolor", "crokey", @@ -4654,7 +4618,7 @@ dependencies = [ "lazy-regex", "minimad", "serde", - "thiserror 1.0.69", + "thiserror 2.0.12", "unicode-width 0.1.14", ] @@ -4812,9 +4776,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.7.6" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" dependencies = [ "displaydoc", "zerovec", @@ -4837,9 +4801,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.2" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" +checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" dependencies = [ "backtrace", "bytes", @@ -4866,9 +4830,9 @@ dependencies = [ [[package]] name = "tokio-metrics" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb2bb07a8451c4c6fa8b3497ad198510d8b8dffa5df5cfb97a64102a58b113c8" +checksum = "7817b32d36c9b94744d7aa3f8fc13526aa0f5112009d7045f3c659413a6e44ac" dependencies = [ "futures-util", "pin-project-lite", @@ -4911,9 +4875,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.14" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" dependencies = [ "bytes", "futures-core", @@ -4924,9 +4888,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.20" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd87a5cdd6ffab733b2f74bc4fd7ee5fff6634124999ac278c35fc78c6120148" +checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" dependencies = [ "serde", "serde_spanned", @@ -4936,26 +4900,33 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.24" +version = "0.22.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" +checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" dependencies = [ - "indexmap 2.8.0", + "indexmap 2.9.0", "serde", "serde_spanned", "toml_datetime", + "toml_write", "winnow", ] +[[package]] +name = "toml_write" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" + [[package]] name = "tonic" version = "0.12.3" @@ -5023,9 +4994,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.2" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697" +checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" dependencies = [ "async-compression", "bitflags 2.9.0", @@ -5267,12 +5238,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf16_iter" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -5285,7 +5250,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.3", "serde", ] @@ -5522,32 +5487,13 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" -dependencies = [ - "windows-core 0.52.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows" version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" dependencies = [ - "windows-core 0.58.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ + "windows-core", "windows-targets 0.52.6", ] @@ -5854,9 +5800,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.4" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" dependencies = [ "memchr", ] @@ -5880,17 +5826,11 @@ dependencies = [ "bitflags 2.9.0", ] -[[package]] -name = "write16" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - [[package]] name = "writeable" -version = "0.5.5" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" [[package]] name = "xml5ever" @@ -5911,9 +5851,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" dependencies = [ "serde", "stable_deref_trait", @@ -5923,9 +5863,9 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.7.5" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", @@ -5935,18 +5875,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.24" +version = "0.8.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", @@ -5981,10 +5921,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] -name = "zerovec" -version = "0.10.4" +name = "zerotrie" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" dependencies = [ "yoke", "zerofrom", @@ -5993,9 +5944,9 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.10.3" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", From 60960c6e099c7ec249dd29c7396328a4907fe6ba Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 20:29:53 +0100 Subject: [PATCH 1794/2091] feat: Automatically set well-known support contacts --- conduwuit-example.toml | 20 +++++++++---- src/api/client/well_known.rs | 54 +++++++++++++++++++++++++----------- src/core/config/mod.rs | 16 +++++++++++ 3 files changed, 69 insertions(+), 21 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 6934e67c..1a8be2aa 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1641,19 +1641,29 @@ # #server = -# This item is undocumented. Please contribute documentation for it. +# URL to a support page for the server, which will be served as part of +# the MSC1929 server support endpoint at /.well-known/matrix/support. +# Will be included alongside any contact information # #support_page = -# This item is undocumented. Please contribute documentation for it. +# Role string for server support contacts, to be served as part of the +# MSC1929 server support endpoint at /.well-known/matrix/support. # -#support_role = +#support_role = "m.role.admin" -# This item is undocumented. Please contribute documentation for it. +# Email address for server support contacts, to be served as part of the +# MSC1929 server support endpoint. +# This will be used along with support_mxid if specified. # #support_email = -# This item is undocumented. Please contribute documentation for it. +# Matrix ID for server support contacts, to be served as part of the +# MSC1929 server support endpoint. +# This will be used along with support_email if specified. +# +# If no email or mxid is specified, all of the server's admins will be +# listed. # #support_mxid = diff --git a/src/api/client/well_known.rs b/src/api/client/well_known.rs index 35b7fc1e..4981ccb4 100644 --- a/src/api/client/well_known.rs +++ b/src/api/client/well_known.rs @@ -1,5 +1,6 @@ use axum::{Json, extract::State, response::IntoResponse}; use conduwuit::{Error, Result}; +use futures::StreamExt; use ruma::api::client::{ discovery::{ discover_homeserver::{self, HomeserverInfo, SlidingSyncProxyInfo}, @@ -33,6 +34,8 @@ pub(crate) async fn well_known_client( /// # `GET /.well-known/matrix/support` /// /// Server support contact and support page of a homeserver's domain. +/// Implements MSC1929 for server discovery. +/// If no configuration is set, uses admin users as contacts. pub(crate) async fn well_known_support( State(services): State, _body: Ruma, @@ -45,32 +48,51 @@ pub(crate) async fn well_known_support( .as_ref() .map(ToString::to_string); - let role = services.server.config.well_known.support_role.clone(); - - // support page or role must be either defined for this to be valid - if support_page.is_none() && role.is_none() { - return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); - } - let email_address = services.server.config.well_known.support_email.clone(); let matrix_id = services.server.config.well_known.support_mxid.clone(); - // if a role is specified, an email address or matrix id is required - if role.is_some() && (email_address.is_none() && matrix_id.is_none()) { - return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); - } - // TODO: support defining multiple contacts in the config let mut contacts: Vec = vec![]; - if let Some(role) = role { - let contact = Contact { role, email_address, matrix_id }; + let role_value = services + .server + .config + .well_known + .support_role + .clone() + .unwrap_or_else(|| "m.role.admin".to_owned().into()); - contacts.push(contact); + // Add configured contact if at least one contact method is specified + if email_address.is_some() || matrix_id.is_some() { + contacts.push(Contact { + role: role_value.clone(), + email_address: email_address.clone(), + matrix_id: matrix_id.clone(), + }); + } + + // Try to add admin users as contacts if no contacts are configured + if contacts.is_empty() { + if let Ok(admin_room) = services.admin.get_admin_room().await { + let admin_users = services.rooms.state_cache.room_members(&admin_room); + let mut stream = admin_users; + + while let Some(user_id) = stream.next().await { + // Skip server user + if *user_id == services.globals.server_user { + break; + } + contacts.push(Contact { + role: role_value.clone(), + email_address: None, + matrix_id: Some(user_id.to_owned()), + }); + } + } } - // support page or role+contacts must be either defined for this to be valid if contacts.is_empty() && support_page.is_none() { + // No admin room, no configured contacts, and no support page return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")); } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 66ed0b2e..d4a10345 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1897,12 +1897,28 @@ pub struct WellKnownConfig { /// example: "matrix.example.com:443" pub server: Option, + /// URL to a support page for the server, which will be served as part of + /// the MSC1929 server support endpoint at /.well-known/matrix/support. + /// Will be included alongside any contact information pub support_page: Option, + /// Role string for server support contacts, to be served as part of the + /// MSC1929 server support endpoint at /.well-known/matrix/support. + /// + /// default: "m.role.admin" pub support_role: Option, + /// Email address for server support contacts, to be served as part of the + /// MSC1929 server support endpoint. + /// This will be used along with support_mxid if specified. pub support_email: Option, + /// Matrix ID for server support contacts, to be served as part of the + /// MSC1929 server support endpoint. + /// This will be used along with support_email if specified. + /// + /// If no email or mxid is specified, all of the server's admins will be + /// listed. pub support_mxid: Option, } From 2ccbd7d60b12c9c9d65bde027f4c974665022b28 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 21:06:44 +0100 Subject: [PATCH 1795/2091] fix: Reference config directly --- src/api/client/well_known.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/api/client/well_known.rs b/src/api/client/well_known.rs index 4981ccb4..fe2281ba 100644 --- a/src/api/client/well_known.rs +++ b/src/api/client/well_known.rs @@ -18,7 +18,7 @@ pub(crate) async fn well_known_client( State(services): State, _body: Ruma, ) -> Result { - let client_url = match services.server.config.well_known.client.as_ref() { + let client_url = match services.config.well_known.client.as_ref() { | Some(url) => url.to_string(), | None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), }; @@ -41,21 +41,19 @@ pub(crate) async fn well_known_support( _body: Ruma, ) -> Result { let support_page = services - .server .config .well_known .support_page .as_ref() .map(ToString::to_string); - let email_address = services.server.config.well_known.support_email.clone(); - let matrix_id = services.server.config.well_known.support_mxid.clone(); + let email_address = services.config.well_known.support_email.clone(); + let matrix_id = services.config.well_known.support_mxid.clone(); // TODO: support defining multiple contacts in the config let mut contacts: Vec = vec![]; let role_value = services - .server .config .well_known .support_role @@ -106,9 +104,9 @@ pub(crate) async fn well_known_support( pub(crate) async fn syncv3_client_server_json( State(services): State, ) -> Result { - let server_url = match services.server.config.well_known.client.as_ref() { + let server_url = match services.config.well_known.client.as_ref() { | Some(url) => url.to_string(), - | None => match services.server.config.well_known.server.as_ref() { + | None => match services.config.well_known.server.as_ref() { | Some(url) => url.to_string(), | None => return Err(Error::BadRequest(ErrorKind::NotFound, "Not found.")), }, From 0ba77674c72324b6f533acfc8d2cbf328b9448a0 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 25 May 2025 00:36:28 +0100 Subject: [PATCH 1796/2091] docs: Security policy --- SECURITY.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/SUMMARY.md | 1 + docs/security.md | 1 + 3 files changed, 61 insertions(+) create mode 100644 SECURITY.md create mode 100644 docs/security.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..c5355491 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,59 @@ +# Security Policy for Continuwuity + +This document outlines the security policy for Continuwuity. Our goal is to maintain a secure platform for all users, and we take security matters seriously. + +## Supported Versions + +We provide security updates for the following versions of Continuwuity: + +| Version | Supported | +| -------------- |:----------------:| +| Latest release | ✅ | +| Main branch | ✅ | +| Older releases | ❌ | + +## Reporting a Vulnerability + +### Responsible Disclosure + +We appreciate the efforts of security researchers and the community in identifying and reporting vulnerabilities. To ensure that potential vulnerabilities are addressed properly, please follow these guidelines: + +1. **Email the security team** directly at [security@continuwuity.org](mailto:security@continuwuity.org) +2. Contact members of the team over E2EE private message. + - [@jade:ellis.link](https://matrix.to/#/@jade:ellis.link) + - [@nex:nexy7574.co.uk](https://matrix.to/#/@nex:nexy7574.co.uk) +3. **Do not disclose the vulnerability publicly** until it has been addressed +4. **Provide detailed information** about the vulnerability, including: + - A clear description of the issue + - Steps to reproduce + - Potential impact + - Any possible mitigations + - Version(s) affected, including specific commits if possible + +### What to Expect + +When you report a security vulnerability: + +1. **Acknowledgment**: We will acknowledge receipt of your report. +2. **Assessment**: We will assess the vulnerability and determine its impact on our users +3. **Updates**: We will provide updates on our progress in addressing the vulnerability, and may request you help test mitigations +4. **Resolution**: Once resolved, we will notify you and discuss coordinated disclosure +5. **Credit**: We will recognize your contribution (unless you prefer to remain anonymous) + +## Security Update Process + +When security vulnerabilities are identified: + +1. We will develop and test fixes in a private branch +2. Security updates will be released as soon as possible +3. Release notes will include information about the vulnerabilities, avoiding details that could facilitate exploitation where possible +4. Critical security updates may be backported to the previous stable release + +## Additional Resources + +- [Matrix Security Disclosure Policy](https://matrix.org/security-disclosure-policy/) +- [Continuwuity Documentation](https://continuwuity.org/introduction) + +--- + +This security policy was last updated on May 25, 2025. diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 473c9e74..af729003 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -20,3 +20,4 @@ - [Testing](development/testing.md) - [Hot Reloading ("Live" Development)](development/hot_reload.md) - [Community (and Guidelines)](community.md) +- [Security](security.md) diff --git a/docs/security.md b/docs/security.md new file mode 100644 index 00000000..b4474cf5 --- /dev/null +++ b/docs/security.md @@ -0,0 +1 @@ +{{#include ../SECURITY.md}} From e8d823a65340a891693339dd5a5dd53e6732fb61 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 26 May 2025 15:01:58 +0100 Subject: [PATCH 1797/2091] docs: Apply feedback on security policy --- SECURITY.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index c5355491..a9aa183e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,16 +12,18 @@ We provide security updates for the following versions of Continuwuity: | Main branch | ✅ | | Older releases | ❌ | +We may backport fixes to the previous release at our discretion, but we don't guarantee this. + ## Reporting a Vulnerability ### Responsible Disclosure We appreciate the efforts of security researchers and the community in identifying and reporting vulnerabilities. To ensure that potential vulnerabilities are addressed properly, please follow these guidelines: -1. **Email the security team** directly at [security@continuwuity.org](mailto:security@continuwuity.org) -2. Contact members of the team over E2EE private message. +1. Contact members of the team over E2EE private message. - [@jade:ellis.link](https://matrix.to/#/@jade:ellis.link) - [@nex:nexy7574.co.uk](https://matrix.to/#/@nex:nexy7574.co.uk) +2. **Email the security team** directly at [security@continuwuity.org](mailto:security@continuwuity.org). This is not E2EE, so don't include sensitive details. 3. **Do not disclose the vulnerability publicly** until it has been addressed 4. **Provide detailed information** about the vulnerability, including: - A clear description of the issue @@ -30,6 +32,8 @@ We appreciate the efforts of security researchers and the community in identifyi - Any possible mitigations - Version(s) affected, including specific commits if possible +If you have any doubts about a potential security vulnerability, contact us via private channels first! We'd prefer that you bother us, instead of having a vulnerability disclosed without a fix. + ### What to Expect When you report a security vulnerability: From 2b268fdaf3c9a8ff96c7e6d438a754e7d4cdec3d Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 14:16:58 +0100 Subject: [PATCH 1798/2091] fix: Allow joining via invite for knock_restricted rooms --- src/core/matrix/state_res/event_auth.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/matrix/state_res/event_auth.rs b/src/core/matrix/state_res/event_auth.rs index 715e5156..759ab5cb 100644 --- a/src/core/matrix/state_res/event_auth.rs +++ b/src/core/matrix/state_res/event_auth.rs @@ -638,7 +638,7 @@ fn valid_membership_change( warn!(?target_user_membership_event_id, "Banned user can't join"); false } else if (join_rules == JoinRule::Invite - || room_version.allow_knocking && join_rules == JoinRule::Knock) + || room_version.allow_knocking && (join_rules == JoinRule::Knock || matches!(join_rules, JoinRule::KnockRestricted(_)))) // If the join_rule is invite then allow if membership state is invite or join && (target_user_current_membership == MembershipState::Join || target_user_current_membership == MembershipState::Invite) From 640714922bb058e6b3251ff17eff84bd07b90c5f Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 14:17:47 +0100 Subject: [PATCH 1799/2091] feat: For knock_restricted rooms, automatically join rooms we meet restrictions for rather than knocking --- src/api/client/membership.rs | 109 +++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 2847d668..e587d806 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -2162,6 +2162,109 @@ async fn knock_room_by_id_helper( } } + // For knock_restricted rooms, check if the user meets the restricted conditions + // If they do, attempt to join instead of knock + // This is not mentioned in the spec, but should be allowable (we're allowed to + // auto-join invites to knocked rooms) + let join_rule = services.rooms.state_accessor.get_join_rules(room_id).await; + if let JoinRule::KnockRestricted(restricted) = &join_rule { + let restriction_rooms: Vec<_> = restricted + .allow + .iter() + .filter_map(|a| match a { + | AllowRule::RoomMembership(r) => Some(&r.room_id), + | _ => None, + }) + .collect(); + + // Check if the user is in any of the allowed rooms + let mut user_meets_restrictions = false; + for restriction_room_id in &restriction_rooms { + if services + .rooms + .state_cache + .is_joined(sender_user, restriction_room_id) + .await + { + user_meets_restrictions = true; + break; + } + } + + // If the user meets the restrictions, try joining instead + if user_meets_restrictions { + debug_info!( + "{sender_user} meets the restricted criteria in knock_restricted room \ + {room_id}, attempting to join instead of knock" + ); + // For this case, we need to drop the state lock and get a new one in + // join_room_by_id_helper We need to release the lock here and let + // join_room_by_id_helper acquire it again + drop(state_lock); + match join_room_by_id_helper( + services, + sender_user, + room_id, + reason.clone(), + servers, + None, + &None, + ) + .await + { + | Ok(_) => return Ok(knock_room::v3::Response::new(room_id.to_owned())), + | Err(e) => { + debug_warn!( + "Failed to convert knock to join for {sender_user} in {room_id}: {e:?}" + ); + // Get a new state lock for the remaining knock logic + let new_state_lock = services.rooms.state.mutex.lock(room_id).await; + + let server_in_room = services + .rooms + .state_cache + .server_in_room(services.globals.server_name(), room_id) + .await; + + let local_knock = server_in_room + || servers.is_empty() + || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); + + if local_knock { + knock_room_helper_local( + services, + sender_user, + room_id, + reason, + servers, + new_state_lock, + ) + .boxed() + .await?; + } else { + knock_room_helper_remote( + services, + sender_user, + room_id, + reason, + servers, + new_state_lock, + ) + .boxed() + .await?; + } + + return Ok(knock_room::v3::Response::new(room_id.to_owned())); + }, + } + } + } else if !matches!(join_rule, JoinRule::Knock | JoinRule::KnockRestricted(_)) { + debug_warn!( + "{sender_user} attempted to knock on room {room_id} but its join rule is \ + {join_rule:?}, not knock or knock_restricted" + ); + } + let server_in_room = services .rooms .state_cache @@ -2209,6 +2312,12 @@ async fn knock_room_helper_local( return Err!(Request(Forbidden("This room does not support knocking."))); } + // Verify that this room has a valid knock or knock_restricted join rule + let join_rule = services.rooms.state_accessor.get_join_rules(room_id).await; + if !matches!(join_rule, JoinRule::Knock | JoinRule::KnockRestricted(_)) { + return Err!(Request(Forbidden("This room's join rule does not allow knocking."))); + } + let content = RoomMemberEventContent { displayname: services.users.displayname(sender_user).await.ok(), avatar_url: services.users.avatar_url(sender_user).await.ok(), From 94ae82414947342ac967bd05c8933d54d9ffc223 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 22 May 2025 14:01:16 +0100 Subject: [PATCH 1800/2091] ci: Don't install rustup if it's already there --- .forgejo/actions/rust-toolchain/action.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.forgejo/actions/rust-toolchain/action.yml b/.forgejo/actions/rust-toolchain/action.yml index 71fb96f5..ae5cfcee 100644 --- a/.forgejo/actions/rust-toolchain/action.yml +++ b/.forgejo/actions/rust-toolchain/action.yml @@ -19,11 +19,20 @@ outputs: rustc_version: description: The rustc version installed value: ${{ steps.rustc-version.outputs.version }} + rustup_version: + description: The rustup version installed + value: ${{ steps.rustup-version.outputs.version }} runs: using: composite steps: + - name: Check if rustup is already installed + shell: bash + id: rustup-version + run: | + echo "version=$(rustup --version)" >> $GITHUB_OUTPUT - name: Cache rustup toolchains + if: steps.rustup-version.outputs.version == '' uses: actions/cache@v3 with: path: | @@ -33,6 +42,7 @@ runs: # Requires repo to be cloned if toolchain is not specified key: ${{ runner.os }}-rustup-${{ inputs.toolchain || hashFiles('**/rust-toolchain.toml') }} - name: Install Rust toolchain + if: steps.rustup-version.outputs.version == '' shell: bash run: | if ! command -v rustup &> /dev/null ; then From b9d60c64e5e3c05a5263955649bc1a0f60b1d172 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 22 May 2025 14:07:22 +0100 Subject: [PATCH 1801/2091] ci: Don't specify container for image builder --- .forgejo/workflows/release-image.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index ec466c58..ec05fb45 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -57,7 +57,6 @@ jobs: build-image: runs-on: dind - container: ghcr.io/catthehacker/ubuntu:act-latest needs: define-variables permissions: contents: read @@ -211,7 +210,6 @@ jobs: merge: runs-on: dind - container: ghcr.io/catthehacker/ubuntu:act-latest needs: [define-variables, build-image] steps: - name: Download digests From ea5dc8e09d11065564f4483a622f82bb12450783 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 23 May 2025 21:10:48 +0100 Subject: [PATCH 1802/2091] fix: Use correct brand in clap version string --- src/core/mod.rs | 5 ++++- src/main/clap.rs | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/core/mod.rs b/src/core/mod.rs index b91cdf0b..aaacd4d8 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -21,7 +21,10 @@ pub use ::toml; pub use ::tracing; pub use config::Config; pub use error::Error; -pub use info::{rustc_flags_capture, version, version::version}; +pub use info::{ + rustc_flags_capture, version, + version::{name, version}, +}; pub use matrix::{Event, EventTypeExt, PduCount, PduEvent, PduId, RoomVersion, pdu, state_res}; pub use server::Server; pub use utils::{ctor, dtor, implement, result, result::Result}; diff --git a/src/main/clap.rs b/src/main/clap.rs index 9b63af19..a3b2b19a 100644 --- a/src/main/clap.rs +++ b/src/main/clap.rs @@ -15,7 +15,7 @@ use conduwuit_core::{ #[clap( about, long_about = None, - name = "conduwuit", + name = conduwuit_core::name(), version = conduwuit_core::version(), )] pub(crate) struct Args { From b57be072c7fdab5c880fdebaeebe11f02648ac08 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 22 May 2025 14:14:06 +0100 Subject: [PATCH 1803/2091] build: Don't rerun on git changes --- src/build_metadata/build.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/build_metadata/build.rs b/src/build_metadata/build.rs index bfdf20b1..bf84d508 100644 --- a/src/build_metadata/build.rs +++ b/src/build_metadata/build.rs @@ -79,12 +79,12 @@ fn main() { // --- Rerun Triggers --- // TODO: The git rerun triggers seem to always run - // Rerun if the git HEAD changes - println!("cargo:rerun-if-changed=.git/HEAD"); - // Rerun if the ref pointed to by HEAD changes (e.g., new commit on branch) - if let Some(ref_path) = run_git_command(&["symbolic-ref", "--quiet", "HEAD"]) { - println!("cargo:rerun-if-changed=.git/{ref_path}"); - } + // // Rerun if the git HEAD changes + // println!("cargo:rerun-if-changed=.git/HEAD"); + // // Rerun if the ref pointed to by HEAD changes (e.g., new commit on branch) + // if let Some(ref_path) = run_git_command(&["symbolic-ref", "--quiet", "HEAD"]) + // { println!("cargo:rerun-if-changed=.git/{ref_path}"); + // } println!("cargo:rerun-if-env-changed=GIT_COMMIT_HASH"); println!("cargo:rerun-if-env-changed=GIT_COMMIT_HASH_SHORT"); From 3c44dccd6555d38c19e46ca8b5ad8acf49d99f52 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 26 May 2025 19:16:50 +0100 Subject: [PATCH 1804/2091] ci: HACK, disable saving to actions cache --- .forgejo/workflows/release-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index ec05fb45..92a5b7c4 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -187,7 +187,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} cache-from: type=gha - cache-to: type=gha,mode=max + # cache-to: type=gha,mode=max sbom: true outputs: type=image,"name=${{ needs.define-variables.outputs.images_list }}",push-by-digest=true,name-canonical=true,push=true env: From 1d45e0b68cd86a72393bf3cb3f866d5943056018 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 13 Jun 2025 13:39:50 +0100 Subject: [PATCH 1805/2091] feat: Add warning when admin users will be exposed as support contacts --- src/core/config/check.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/core/config/check.rs b/src/core/config/check.rs index ded9533d..3dc45e2f 100644 --- a/src/core/config/check.rs +++ b/src/core/config/check.rs @@ -219,6 +219,15 @@ pub fn check(config: &Config) -> Result { )); } + // Check if support contact information is configured + if config.well_known.support_email.is_none() && config.well_known.support_mxid.is_none() { + warn!( + "No support contact information (support_email or support_mxid) is configured in \ + the well_known section. Users in the admin room will be automatically listed as \ + support contacts in the /.well-known/matrix/support endpoint." + ); + } + if config .url_preview_domain_contains_allowlist .contains(&"*".to_owned()) From d7514178ab3aaf594bc2b55bb115842f5ba9f2ca Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 13 Jun 2025 14:29:14 +0100 Subject: [PATCH 1806/2091] ci: Fix extra bracket in commit shorthash --- .forgejo/workflows/release-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 92a5b7c4..55b303b2 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -180,7 +180,7 @@ jobs: file: "docker/Dockerfile" build-args: | GIT_COMMIT_HASH=${{ github.sha }}) - GIT_COMMIT_HASH_SHORT=${{ env.COMMIT_SHORT_SHA }}) + GIT_COMMIT_HASH_SHORT=${{ env.COMMIT_SHORT_SHA }} GIT_REMOTE_URL=${{github.event.repository.html_url }} GIT_REMOTE_COMMIT_URL=${{github.event.head_commit.url }} platforms: ${{ matrix.platform }} From 44e60d0ea60fec116eb1535239cc8007d73992f0 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 30 May 2025 23:50:29 +0100 Subject: [PATCH 1807/2091] docs: Tiny phrasing changes to the security policy --- SECURITY.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index a9aa183e..2869ce58 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -20,10 +20,10 @@ We may backport fixes to the previous release at our discretion, but we don't gu We appreciate the efforts of security researchers and the community in identifying and reporting vulnerabilities. To ensure that potential vulnerabilities are addressed properly, please follow these guidelines: -1. Contact members of the team over E2EE private message. +1. **Contact members of the team directly** over E2EE private message. - [@jade:ellis.link](https://matrix.to/#/@jade:ellis.link) - [@nex:nexy7574.co.uk](https://matrix.to/#/@nex:nexy7574.co.uk) -2. **Email the security team** directly at [security@continuwuity.org](mailto:security@continuwuity.org). This is not E2EE, so don't include sensitive details. +2. **Email the security team** at [security@continuwuity.org](mailto:security@continuwuity.org). This is not E2EE, so don't include sensitive details. 3. **Do not disclose the vulnerability publicly** until it has been addressed 4. **Provide detailed information** about the vulnerability, including: - A clear description of the issue @@ -48,7 +48,7 @@ When you report a security vulnerability: When security vulnerabilities are identified: -1. We will develop and test fixes in a private branch +1. We will develop and test fixes in a private fork 2. Security updates will be released as soon as possible 3. Release notes will include information about the vulnerabilities, avoiding details that could facilitate exploitation where possible 4. Critical security updates may be backported to the previous stable release From 5d44653e3a9bc278f9dbe3084e2e34a9a0799e1f Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 14 Jun 2025 16:28:57 +0100 Subject: [PATCH 1808/2091] fix: Incorrect command descriptions --- src/admin/debug/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 9b86f18c..1fd4e263 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -125,13 +125,13 @@ pub(super) enum DebugCommand { reset: bool, }, - /// - Verify json signatures + /// - Sign JSON blob /// /// This command needs a JSON blob provided in a Markdown code block below /// the command. SignJson, - /// - Verify json signatures + /// - Verify JSON signatures /// /// This command needs a JSON blob provided in a Markdown code block below /// the command. From d0f00e6f5ca2be3c8524c5383bcb32585c849278 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 14 Jun 2025 18:54:02 +0100 Subject: [PATCH 1809/2091] feat: Allow mentioning @room in an admin announcement --- docs/static/announcements.schema.json | 8 ++++++-- src/service/announcements/mod.rs | 29 +++++++++++++++------------ 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/docs/static/announcements.schema.json b/docs/static/announcements.schema.json index 95b1d153..cacd10c9 100644 --- a/docs/static/announcements.schema.json +++ b/docs/static/announcements.schema.json @@ -3,7 +3,7 @@ "$id": "https://continwuity.org/schema/announcements.schema.json", "type": "object", "properties": { - "updates": { + "announcements": { "type": "array", "items": { "type": "object", @@ -16,6 +16,10 @@ }, "date": { "type": "string" + }, + "mention_room": { + "type": "boolean", + "description": "Whether to mention the room (@room) when posting this announcement" } }, "required": [ @@ -26,6 +30,6 @@ } }, "required": [ - "updates" + "announcements" ] } \ No newline at end of file diff --git a/src/service/announcements/mod.rs b/src/service/announcements/mod.rs index 4df8971b..2a70344d 100644 --- a/src/service/announcements/mod.rs +++ b/src/service/announcements/mod.rs @@ -20,7 +20,7 @@ use std::{sync::Arc, time::Duration}; use async_trait::async_trait; use conduwuit::{Result, Server, debug, info, warn}; use database::{Deserialized, Map}; -use ruma::events::room::message::RoomMessageEventContent; +use ruma::events::{Mentions, room::message::RoomMessageEventContent}; use serde::Deserialize; use tokio::{ sync::Notify, @@ -53,6 +53,8 @@ struct CheckForAnnouncementsResponseEntry { id: u64, date: Option, message: String, + #[serde(default, skip_serializing_if = "bool::not")] + mention_room: bool, } const CHECK_FOR_ANNOUNCEMENTS_URL: &str = @@ -139,19 +141,20 @@ impl Service { } else { info!("[announcements] {:#}", announcement.message); } + let mut message = RoomMessageEventContent::text_markdown(format!( + "### New announcement{}\n\n{}", + announcement + .date + .as_ref() + .map_or_else(String::new, |date| format!(" - `{date}`")), + announcement.message + )); - self.services - .admin - .send_message(RoomMessageEventContent::text_markdown(format!( - "### New announcement{}\n\n{}", - announcement - .date - .as_ref() - .map_or_else(String::new, |date| format!(" - `{date}`")), - announcement.message - ))) - .await - .ok(); + if announcement.mention_room { + message = message.add_mentions(Mentions::with_room_mention()); + } + + self.services.admin.send_message(message).await.ok(); } #[inline] From 0870c8d6478dfa24ef09d8fe52b578d101024edf Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 14 Jun 2025 20:53:00 +0100 Subject: [PATCH 1810/2091] chore: Release --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 160be0c7..ec6e848d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -771,7 +771,7 @@ dependencies = [ [[package]] name = "conduwuit" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "clap", "conduwuit_admin", @@ -800,7 +800,7 @@ dependencies = [ [[package]] name = "conduwuit_admin" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "clap", "conduwuit_api", @@ -821,7 +821,7 @@ dependencies = [ [[package]] name = "conduwuit_api" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "async-trait", "axum", @@ -853,14 +853,14 @@ dependencies = [ [[package]] name = "conduwuit_build_metadata" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "built 0.8.0", ] [[package]] name = "conduwuit_core" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "argon2", "arrayvec", @@ -919,7 +919,7 @@ dependencies = [ [[package]] name = "conduwuit_database" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "async-channel", "conduwuit_core", @@ -937,7 +937,7 @@ dependencies = [ [[package]] name = "conduwuit_macros" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "itertools 0.14.0", "proc-macro2", @@ -947,7 +947,7 @@ dependencies = [ [[package]] name = "conduwuit_router" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "axum", "axum-client-ip", @@ -981,7 +981,7 @@ dependencies = [ [[package]] name = "conduwuit_service" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "async-trait", "base64 0.22.1", @@ -1018,7 +1018,7 @@ dependencies = [ [[package]] name = "conduwuit_web" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" dependencies = [ "askama", "axum", diff --git a/Cargo.toml b/Cargo.toml index 1abff107..af904447 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://forgejo.ellis.link/continuwuation/continuwuity" rust-version = "1.86.0" -version = "0.5.0-rc.5" +version = "0.5.0-rc.6" [workspace.metadata.crane] name = "conduwuit" From 6e16a6ef8f90ee4a1b2a92435475ddf7359ba837 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 14 Jun 2025 22:34:24 +0100 Subject: [PATCH 1811/2091] chore: Release announcement --- docs/static/announcements.json | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/static/announcements.json b/docs/static/announcements.json index 9b97d091..7dd2fb72 100644 --- a/docs/static/announcements.json +++ b/docs/static/announcements.json @@ -4,6 +4,10 @@ { "id": 1, "message": "Welcome to Continuwuity! Important announcements about the project will appear here." + }, + { + "id": 2, + "message": "🎉 Continuwuity v0.5.0-rc.6 is now available! This release includes improved knock-restricted room handling, automatic support contact configuration, and a new HTML landing page. Check [the release notes for full details](https://forgejo.ellis.link/continuwuation/continuwuity/releases/tag/v0.5.0-rc.6) and upgrade instructions." } ] -} \ No newline at end of file +} From d6fd30393c224c216867f46341616c041bbcd97a Mon Sep 17 00:00:00 2001 From: Kimiblock Date: Thu, 19 Jun 2025 12:36:49 +0000 Subject: [PATCH 1812/2091] Update docs/deploying/arch-linux.md --- docs/deploying/arch-linux.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/deploying/arch-linux.md b/docs/deploying/arch-linux.md index a14201e3..52d2afb2 100644 --- a/docs/deploying/arch-linux.md +++ b/docs/deploying/arch-linux.md @@ -1,3 +1,5 @@ # Continuwuity for Arch Linux -Continuwuity does not have any Arch Linux packages at this time. +Continuwuity is available on the `archlinuxcn` repository and AUR, with the same package name `continuwuity`, which includes latest taggged version. The development version is available on AUR as `continuwuity-git` + +Simply install the `continuwuity` package. Configure the service in `/etc/conduwuit/conduwuit.toml`, then enable/start the continuwuity.service. \ No newline at end of file From e508b1197f3a6e485f35edf0d3c529cb58c391b5 Mon Sep 17 00:00:00 2001 From: nex Date: Thu, 19 Jun 2025 21:27:50 +0000 Subject: [PATCH 1813/2091] feat: allow overriding the "most recent event" when forcing a state download (#853) Add option to select which event to set the state at to, for the force-set-room-state admin command. This allows us to work around issues where the latest PDU is one that remote servers don't know about (i.e. failed federation for whatever reason) Closes #852 Reviewed-on: https://forgejo.ellis.link/continuwuation/continuwuity/pulls/853 Reviewed-by: Jade Ellis Co-authored-by: nex Co-committed-by: nex --- src/admin/debug/commands.rs | 37 +++++++++++++++++++++++-------------- src/admin/debug/mod.rs | 3 +++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index d0debc2a..a397e0fc 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -239,10 +239,11 @@ pub(super) async fn get_remote_pdu( }) .await { - | Err(e) => + | Err(e) => { return Err!( "Remote server did not have PDU or failed sending request to remote server: {e}" - ), + ); + }, | Ok(response) => { let json: CanonicalJsonObject = serde_json::from_str(response.pdu.get()).map_err(|e| { @@ -384,8 +385,9 @@ pub(super) async fn change_log_level(&self, filter: Option, reset: bool) .reload .reload(&old_filter_layer, Some(handles)) { - | Err(e) => - return Err!("Failed to modify and reload the global tracing log level: {e}"), + | Err(e) => { + return Err!("Failed to modify and reload the global tracing log level: {e}"); + }, | Ok(()) => { let value = &self.services.server.config.log; let out = format!("Successfully changed log level back to config value {value}"); @@ -408,8 +410,9 @@ pub(super) async fn change_log_level(&self, filter: Option, reset: bool) .reload(&new_filter_layer, Some(handles)) { | Ok(()) => return self.write_str("Successfully changed log level").await, - | Err(e) => - return Err!("Failed to modify and reload the global tracing log level: {e}"), + | Err(e) => { + return Err!("Failed to modify and reload the global tracing log level: {e}"); + }, } } @@ -529,6 +532,7 @@ pub(super) async fn force_set_room_state_from_server( &self, room_id: OwnedRoomId, server_name: OwnedServerName, + at_event: Option, ) -> Result { if !self .services @@ -540,13 +544,18 @@ pub(super) async fn force_set_room_state_from_server( return Err!("We are not participating in the room / we don't know about the room ID."); } - let first_pdu = self - .services - .rooms - .timeline - .latest_pdu_in_room(&room_id) - .await - .map_err(|_| err!(Database("Failed to find the latest PDU in database")))?; + let at_event_id = match at_event { + | Some(event_id) => event_id, + | None => self + .services + .rooms + .timeline + .latest_pdu_in_room(&room_id) + .await + .map_err(|_| err!(Database("Failed to find the latest PDU in database")))? + .event_id + .clone(), + }; let room_version = self.services.rooms.state.get_room_version(&room_id).await?; @@ -557,7 +566,7 @@ pub(super) async fn force_set_room_state_from_server( .sending .send_federation_request(&server_name, get_room_state::v1::Request { room_id: room_id.clone(), - event_id: first_pdu.event_id.clone(), + event_id: at_event_id, }) .await?; diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index 1fd4e263..bceee9ba 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -177,6 +177,9 @@ pub(super) enum DebugCommand { room_id: OwnedRoomId, /// The server we will use to query the room state for server_name: OwnedServerName, + /// The event ID of the latest known PDU in the room. Will be found + /// automatically if not provided. + event_id: Option, }, /// - Runs a server name through conduwuit's true destination resolution From a737d845a436016b9eaf55ca19cbdd4cc419337a Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 21 May 2025 20:41:34 +0100 Subject: [PATCH 1814/2091] chore: Don't specify targets in rust-toolchain --- rust-toolchain.toml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index aadc8f99..65890260 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -19,11 +19,3 @@ components = [ "rustfmt", "clippy", ] -targets = [ - #"x86_64-apple-darwin", - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-musl", - "aarch64-unknown-linux-gnu", - #"aarch64-apple-darwin", -] From b526935d45429be72b7e99aef7e57e236b2e84bf Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 20 Jun 2025 21:35:03 +0100 Subject: [PATCH 1815/2091] build: Specify debian version --- docker/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e734fb81..dfda57fb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,8 +1,9 @@ ARG RUST_VERSION=1 +ARG DEBIAN_VERSION=bookworm FROM --platform=$BUILDPLATFORM docker.io/tonistiigi/xx AS xx -FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS base -FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-bookworm AS toolchain +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-${DEBIAN_VERSION} AS base +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim-${DEBIAN_VERSION} AS toolchain # Prevent deletion of apt cache RUN rm -f /etc/apt/apt.conf.d/docker-clean From 08fbcbba691234b6240c03805354424deb2931e9 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 20 Jun 2025 21:35:48 +0100 Subject: [PATCH 1816/2091] build: Use newer LLVM for rust 1.87 --- docker/Dockerfile | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index dfda57fb..e40438e9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,7 +10,7 @@ RUN rm -f /etc/apt/apt.conf.d/docker-clean # Match Rustc version as close as possible # rustc -vV -ARG LLVM_VERSION=19 +ARG LLVM_VERSION=20 # ENV RUSTUP_TOOLCHAIN=${RUST_VERSION} # Install repo tools @@ -20,10 +20,18 @@ ARG LLVM_VERSION=19 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt-get update && apt-get install -y \ - clang-${LLVM_VERSION} lld-${LLVM_VERSION} pkg-config make jq \ - curl git \ + pkg-config make jq \ + curl git software-properties-common \ file +# LLVM packages +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + curl https://apt.llvm.org/llvm.sh > llvm.sh && \ + chmod +x llvm.sh && \ + ./llvm.sh ${LLVM_VERSION} && \ + rm llvm.sh + # Create symlinks for LLVM tools RUN < Date: Fri, 20 Jun 2025 21:45:29 +0100 Subject: [PATCH 1817/2091] build: Upgrade to Rust 1.87 --- rust-toolchain.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 65890260..bdb608aa 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -9,7 +9,7 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.86.0" +channel = "1.87.0" profile = "minimal" components = [ # For rust-analyzer From 01200d9b545bdad3f59b0db447a1480aa6314d50 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 20 Jun 2025 21:48:37 +0100 Subject: [PATCH 1818/2091] build: Allow specifying build profile Additionally splits caches by target CPU --- .forgejo/workflows/release-image.yml | 12 ++++++++---- docker/Dockerfile | 5 +++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 55b303b2..170fe668 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -49,6 +49,7 @@ jobs: const platforms = ['linux/amd64', 'linux/arm64'] core.setOutput('build_matrix', JSON.stringify({ platform: platforms, + target_cpu: ['base'], include: platforms.map(platform => { return { platform, slug: platform.replace('/', '-') @@ -66,6 +67,8 @@ jobs: strategy: matrix: { + "target_cpu": ["base"], + "profile": ["release"], "include": [ { "platform": "linux/amd64", "slug": "linux-amd64" }, @@ -73,6 +76,7 @@ jobs: ], "platform": ["linux/amd64", "linux/arm64"], } + steps: - name: Echo strategy run: echo '${{ toJSON(fromJSON(needs.define-variables.outputs.build_matrix)) }}' @@ -140,8 +144,8 @@ jobs: uses: actions/cache@v3 with: path: | - cargo-target-${{ matrix.slug }} - key: cargo-target-${{ matrix.slug }}-${{hashFiles('**/Cargo.lock') }}-${{steps.rust-toolchain.outputs.rustc_version}} + cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }} + key: cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}-${{hashFiles('**/Cargo.lock') }}-${{steps.rust-toolchain.outputs.rustc_version}} - name: Cache apt cache id: cache-apt uses: actions/cache@v3 @@ -163,9 +167,9 @@ jobs: { ".cargo/registry": "/usr/local/cargo/registry", ".cargo/git/db": "/usr/local/cargo/git/db", - "cargo-target-${{ matrix.slug }}": { + "cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}": { "target": "/app/target", - "id": "cargo-target-${{ matrix.platform }}" + "id": "cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}" }, "var-cache-apt-${{ matrix.slug }}": "/var/cache/apt", "var-lib-apt-${{ matrix.slug }}": "/var/lib/apt" diff --git a/docker/Dockerfile b/docker/Dockerfile index e40438e9..bd6e72d1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -149,11 +149,12 @@ ENV GIT_REMOTE_COMMIT_URL=$GIT_REMOTE_COMMIT_URL ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA ENV CONTINUWUITY_VERSION_EXTRA=$CONTINUWUITY_VERSION_EXTRA +ARG RUST_PROFILE=release # Build the binary RUN --mount=type=cache,target=/usr/local/cargo/registry \ --mount=type=cache,target=/usr/local/cargo/git/db \ - --mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM} \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGET_CPU}-${TARGETPLATFORM}-${RUST_PROFILE} \ bash <<'EOF' set -o allexport set -o xtrace @@ -162,7 +163,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ jq -r ".target_directory")) mkdir /out/sbin PACKAGE=conduwuit - xx-cargo build --locked --release \ + xx-cargo build --locked --profile ${RUST_PROFILE} \ -p $PACKAGE; BINARIES=($(cargo metadata --no-deps --format-version 1 | \ jq -r ".packages[] | select(.name == \"$PACKAGE\") | .targets[] | select( .kind | map(. == \"bin\") | any ) | .name")) From add5c7052c8a2aa61a78a785500530ecdd394626 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 20 Jun 2025 21:51:53 +0100 Subject: [PATCH 1819/2091] chore: Update lockfile --- Cargo.lock | 647 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 415 insertions(+), 232 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ec6e848d..59662da2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,9 +13,9 @@ dependencies = [ [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" @@ -28,9 +28,12 @@ dependencies = [ [[package]] name = "aligned-vec" -version = "0.5.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] [[package]] name = "alloc-no-stdlib" @@ -49,9 +52,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anyhow" @@ -170,9 +173,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.23" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b37fc50485c4f3f736a4fb14199f6d5f5ba008d7f28fe710306c92780f004c07" +checksum = "40f6024f3f856663b45fd0c9b6f2024034a702f453549449e0d84a305900dad4" dependencies = [ "brotli", "flate2", @@ -219,9 +222,9 @@ dependencies = [ [[package]] name = "atomic" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" dependencies = [ "bytemuck", ] @@ -234,15 +237,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "av1-grain" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf" +checksum = "4f3efb2ca85bc610acfa917b5aaa36f3fcbebed5b3182d7f877b02531c4b80c8" dependencies = [ "anyhow", "arrayvec", @@ -263,9 +266,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" +checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" dependencies = [ "aws-lc-sys", "zeroize", @@ -273,9 +276,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.28.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa9b6986f250236c27e5a204062434a773a13243d2ffc2955f37bdba4c5c6a1" +checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" dependencies = [ "bindgen 0.69.5", "cc", @@ -442,9 +445,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.7.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e25b6adfb930f02d1981565a6e5d9c547ac15a96606256d3b59040e5cd4ca3" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "basic-toml" @@ -461,7 +464,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cexpr", "clang-sys", "itertools 0.12.1", @@ -484,7 +487,7 @@ version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -510,9 +513,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "bitstream-io" @@ -582,15 +585,15 @@ checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" [[package]] name = "bytemuck" -version = "1.23.0" +version = "1.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" [[package]] name = "byteorder" @@ -638,9 +641,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.22" +version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32db95edf998450acc7881c932f94cd9b05c87b4b2599e8bab064753da4acfd1" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ "jobserver", "libc", @@ -668,9 +671,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -709,9 +712,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.38" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", "clap_derive", @@ -719,9 +722,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.38" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstyle", "clap_lex", @@ -729,9 +732,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" dependencies = [ "heck", "proc-macro2", @@ -741,9 +744,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "cmake" @@ -1088,19 +1091,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2459fc9262a1aa204eb4b5764ad4f189caec88aea9634389c0a25f8be7f6265e" [[package]] -name = "coolor" -version = "1.0.0" +name = "convert_case" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "691defa50318376447a73ced869862baecfab35f6aabaa91a4cd726b315bfe1a" +checksum = "bb402b8d4c85569410425650ce3eddc7d698ed96d39a73f941b08fb63082f1e7" dependencies = [ - "crossterm", + "unicode-segmentation", +] + +[[package]] +name = "coolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "980c2afde4af43d6a05c5be738f9eae595cff86dce1f38f88b95058a98c027f3" +dependencies = [ + "crossterm 0.29.0", ] [[package]] name = "core-foundation" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -1148,12 +1160,12 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crokey" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5ff945e42bb93d29b10ba509970066a269903a932f0ea07d99d8621f97e90d7" +checksum = "5282b45c96c5978c8723ea83385cb9a488b64b7d175733f48d07bf9da514a863" dependencies = [ "crokey-proc_macros", - "crossterm", + "crossterm 0.29.0", "once_cell", "serde", "strict", @@ -1161,11 +1173,11 @@ dependencies = [ [[package]] name = "crokey-proc_macros" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "665f2180fd82d0ba2bf3deb45fafabb18f23451024ff71ee47f6bfdfb4bbe09e" +checksum = "2ea0218d3fedf0797fa55676f1964ef5d27103d41ed0281b4bbd2a6e6c3d8d28" dependencies = [ - "crossterm", + "crossterm 0.29.0", "proc-macro2", "quote", "strict", @@ -1234,12 +1246,30 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "crossterm_winapi", "futures-core", "mio", "parking_lot", - "rustix", + "rustix 0.38.44", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags 2.9.1", + "crossterm_winapi", + "derive_more", + "document-features", + "mio", + "parking_lot", + "rustix 1.0.7", "signal-hook", "signal-hook-mio", "winapi", @@ -1337,9 +1367,9 @@ dependencies = [ [[package]] name = "der" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", "zeroize", @@ -1354,6 +1384,27 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_more" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "093242cf7570c207c83073cf82f79706fe7b8317e98620a47d5be7c3d8497678" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.10.7" @@ -1376,6 +1427,15 @@ dependencies = [ "syn", ] +[[package]] +name = "document-features" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95249b50c6c185bee49034bcb378a49dc2b5dff0be90ff6616d31d64febab05d" +dependencies = [ + "litrs", +] + [[package]] name = "dunce" version = "1.0.5" @@ -1428,6 +1488,26 @@ dependencies = [ "syn", ] +[[package]] +name = "equator" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" +dependencies = [ + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1436,12 +1516,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.11" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -1522,9 +1602,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "miniz_oxide", @@ -1557,9 +1637,9 @@ dependencies = [ [[package]] name = "fs-err" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f89bda4c2a21204059a977ed3bfe746677dfd137b83c339e702b0ac91d482aa" +checksum = "88d7be93788013f265201256d58f04936a8079ad5dc898743aa20525f503b683" dependencies = [ "autocfg", "tokio", @@ -1671,10 +1751,11 @@ dependencies = [ [[package]] name = "generator" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc6bd114ceda131d3b1d665eba35788690ad37f5916457286b32ab6fd3c438dd" +checksum = "d18470a76cb7f8ff746cf1f7470914f900252ec36bbc40b569d74b1258446827" dependencies = [ + "cc", "cfg-if", "libc", "log", @@ -1701,7 +1782,7 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] @@ -1784,9 +1865,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" [[package]] name = "hdrhistogram" @@ -1803,11 +1884,11 @@ dependencies = [ [[package]] name = "headers" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9" +checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64 0.21.7", + "base64 0.22.1", "bytes", "headers-core", "http", @@ -1833,9 +1914,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.9" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -2063,11 +2144,10 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.5" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "futures-util", "http", "hyper", "hyper-util", @@ -2077,7 +2157,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", + "webpki-roots 1.0.1", ] [[package]] @@ -2161,9 +2241,9 @@ checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2549ca8c7241c82f59c80ba2a6f415d931c5b58d24fb8412caa1a1f02c49139a" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" dependencies = [ "displaydoc", "icu_collections", @@ -2177,9 +2257,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8197e866e47b68f8f7d95249e172903bec06004b18b2937f1095d40a0c57de04" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" @@ -2244,9 +2324,9 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b77d01e822461baa8409e156015a1d91735549f0f2c17691bd2d996bef238f7f" +checksum = "f6970fe7a5300b4b42e62c52efa0187540a5bef546c60edaf554ef595d2e6f0b" dependencies = [ "byteorder-lite", "quick-error", @@ -2275,7 +2355,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "serde", ] @@ -2474,9 +2554,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.172" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libfuzzer-sys" @@ -2490,12 +2570,12 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a793df0d7afeac54f95b471d3af7f0d4fb975699f972341a4b76988d49cdf0c" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.53.0", + "windows-targets 0.53.2", ] [[package]] @@ -2521,6 +2601,12 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + [[package]] name = "litemap" version = "0.8.0" @@ -2528,10 +2614,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] -name = "lock_api" -version = "0.4.12" +name = "litrs" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -2584,6 +2676,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "lz4-sys" version = "1.11.1+lz4-1.10.0" @@ -2659,9 +2757,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "mime" @@ -2716,9 +2814,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", "simd-adler32", @@ -2726,14 +2824,14 @@ dependencies = [ [[package]] name = "mio" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", ] [[package]] @@ -2767,7 +2865,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "cfg-if", "cfg_aliases", "libc", @@ -2897,9 +2995,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ "hermit-abi", "libc", @@ -3013,11 +3111,12 @@ dependencies = [ [[package]] name = "os_info" -version = "3.10.0" +version = "3.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a604e53c24761286860eba4e2c8b23a0161526476b1de520139d69cdb85a6b5" +checksum = "d0e1ac5fde8d43c34139135df8ea9ee9465394b2d8d20f032d38998f64afffc3" dependencies = [ "log", + "plist", "serde", "windows-sys 0.52.0", ] @@ -3036,9 +3135,9 @@ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -3046,9 +3145,9 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", @@ -3189,6 +3288,19 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plist" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d77244ce2d584cd84f6a15f86195b8c9b2a0dfbfd817c09e0464244091a58ed" +dependencies = [ + "base64 0.22.1", + "indexmap 2.9.0", + "quick-xml", + "serde", + "time", +] + [[package]] name = "png" version = "0.17.16" @@ -3204,9 +3316,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "potential_utf" @@ -3240,9 +3352,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.31" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a" dependencies = [ "proc-macro2", "syn", @@ -3336,7 +3448,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "memchr", "pulldown-cmark-escape", "unicase", @@ -3364,10 +3476,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] -name = "quinn" -version = "0.11.7" +name = "quick-xml" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + +[[package]] +name = "quinn" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" dependencies = [ "bytes", "cfg_aliases", @@ -3385,12 +3506,13 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.10" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" dependencies = [ "bytes", "getrandom 0.3.3", + "lru-slab", "rand 0.9.1", "ring", "rustc-hash 2.1.1", @@ -3405,9 +3527,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" +checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" dependencies = [ "cfg_aliases", "libc", @@ -3428,9 +3550,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "rand" @@ -3563,11 +3685,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -3660,7 +3782,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots", + "webpki-roots 0.26.11", "windows-registry", ] @@ -3916,9 +4038,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" [[package]] name = "rustc-hash" @@ -3947,18 +4069,31 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags 2.9.1", + "errno", + "libc", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.27" +version = "0.23.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "730944ca083c1c233a75c09f199e973ca499344a2b7ba9e755c457e86fb4a321" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ "aws-lc-rs", "log", @@ -4015,23 +4150,23 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "rustyline-async" version = "0.4.3" source = "git+https://forgejo.ellis.link/continuwuation/rustyline-async?rev=deaeb0694e2083f53d363b648da06e10fc13900c#deaeb0694e2083f53d363b648da06e10fc13900c" dependencies = [ - "crossterm", + "crossterm 0.28.1", "futures-channel", "futures-util", "pin-project", "thingbuf", "thiserror 2.0.12", "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] @@ -4085,7 +4220,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", "core-foundation", "core-foundation-sys", "libc", @@ -4127,7 +4262,7 @@ dependencies = [ "sentry-tracing", "tokio", "ureq", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -4310,9 +4445,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" dependencies = [ "serde", ] @@ -4381,9 +4516,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8621587d4798caf8eb44879d42e56b9a93ea5dcd315a6487c357130095b62801" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" dependencies = [ "libc", "signal-hook-registry", @@ -4441,12 +4576,9 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "smallstr" @@ -4460,18 +4592,18 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" dependencies = [ "serde", ] [[package]] name = "socket2" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -4541,9 +4673,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.101" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -4674,12 +4806,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -4801,9 +4932,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.0" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2513ca694ef9ede0fb23fe71a4ee4107cb102b9dc1930f6d0fd77aae068ae165" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", @@ -4888,9 +5019,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", "serde_spanned", @@ -4900,18 +5031,18 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.9" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" dependencies = [ "serde", ] [[package]] name = "toml_edit" -version = "0.22.26" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.9.0", "serde", @@ -4923,9 +5054,9 @@ dependencies = [ [[package]] name = "toml_write" -version = "0.1.1" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "tonic" @@ -4994,12 +5125,12 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.4" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdb0c213ca27a9f57ab69ddb290fd80d970922355b83ae380b395d3986b8a2e" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "async-compression", - "bitflags 2.9.0", + "bitflags 2.9.1", "bytes", "futures-core", "futures-util", @@ -5183,9 +5314,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "unsafe-libyaml" @@ -5211,7 +5342,7 @@ dependencies = [ "rustls", "rustls-pki-types", "url", - "webpki-roots", + "webpki-roots 0.26.11", ] [[package]] @@ -5246,19 +5377,21 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ "getrandom 0.3.3", + "js-sys", "serde", + "wasm-bindgen", ] [[package]] name = "v_frame" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" dependencies = [ "aligned-vec", "num-traits", @@ -5300,9 +5433,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -5428,18 +5561,27 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.26.8" +version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.1", +] + +[[package]] +name = "webpki-roots" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" dependencies = [ "rustls-pki-types", ] [[package]] name = "weezl" -version = "0.1.8" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" +checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" [[package]] name = "which" @@ -5450,7 +5592,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix", + "rustix 0.38.44", ] [[package]] @@ -5489,32 +5631,55 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.58.0" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-link", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ "windows-core", - "windows-targets 0.52.6", ] [[package]] name = "windows-core" -version = "0.58.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", - "windows-result 0.2.0", - "windows-strings 0.1.0", - "windows-targets 0.52.6", + "windows-link", + "windows-result", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", ] [[package]] name = "windows-implement" -version = "0.58.0" +version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbd5b46c938e506ecbce286b6628a02171d56153ba733b6c741fc627ec9579b" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", @@ -5523,9 +5688,9 @@ dependencies = [ [[package]] name = "windows-interface" -version = "0.58.0" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "053c4c462dc91d3b1504c6fe5a726dd15e216ba718e84a0e46a88fbe5ded3515" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", @@ -5534,9 +5699,19 @@ dependencies = [ [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core", + "windows-link", +] [[package]] name = "windows-registry" @@ -5544,39 +5719,20 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "windows-result 0.3.2", + "windows-result", "windows-strings 0.3.1", - "windows-targets 0.53.0", + "windows-targets 0.53.2", ] [[package]] name = "windows-result" -version = "0.2.0" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-result" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" -dependencies = [ - "windows-result 0.2.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-strings" version = "0.3.1" @@ -5586,6 +5742,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -5613,6 +5778,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -5646,9 +5820,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.0" +version = "0.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" dependencies = [ "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", @@ -5660,6 +5834,15 @@ dependencies = [ "windows_x86_64_msvc 0.53.0", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -5800,9 +5983,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" dependencies = [ "memchr", ] @@ -5823,7 +6006,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.9.1", ] [[package]] @@ -5875,18 +6058,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", @@ -5999,9 +6182,9 @@ dependencies = [ [[package]] name = "zune-jpeg" -version = "0.4.14" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99a5bab8d7dedf81405c4bb1f2b83ea057643d9cb28778cea9eecddeedd2e028" +checksum = "0f6fe2e33d02a98ee64423802e16df3de99c43e5cf5ff983767e1128b394c8ac" dependencies = [ "zune-core", ] From bae8192fb3cef6dbe6a14015ab46639df14ee288 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 20 Jun 2025 23:39:20 +0100 Subject: [PATCH 1820/2091] chore: Bump resolv-conf from 0.7.1 to 0.7.4 --- Cargo.lock | 7 ++----- Cargo.toml | 9 ++++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59662da2..a8bab900 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3788,11 +3788,8 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.1" -source = "git+https://forgejo.ellis.link/continuwuation/resolv-conf?rev=200e958941d522a70c5877e3d846f55b5586c68d#200e958941d522a70c5877e3d846f55b5586c68d" -dependencies = [ - "hostname", -] +version = "0.7.4" +source = "git+https://forgejo.ellis.link/continuwuation/resolv-conf?rev=56251316cc4127bcbf36e68ce5e2093f4d33e227#56251316cc4127bcbf36e68ce5e2093f4d33e227" [[package]] name = "rgb" diff --git a/Cargo.toml b/Cargo.toml index af904447..0ebb758c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -381,7 +381,7 @@ features = [ "unstable-msc4121", "unstable-msc4125", "unstable-msc4186", - "unstable-msc4203", # sending to-device events to appservices + "unstable-msc4203", # sending to-device events to appservices "unstable-msc4210", # remove legacy mentions "unstable-extensible-events", "unstable-pdu", @@ -580,12 +580,11 @@ rev = "9c8e51510c35077df888ee72a36b4b05637147da" git = "https://forgejo.ellis.link/continuwuation/hyper-util" rev = "e4ae7628fe4fcdacef9788c4c8415317a4489941" -# allows no-aaaa option in resolv.conf -# bumps rust edition and toolchain to 1.86.0 and 2024 -# use sat_add on line number errors +# Allows no-aaaa option in resolv.conf +# Use 1-indexed line numbers when displaying parse error messages [patch.crates-io.resolv-conf] git = "https://forgejo.ellis.link/continuwuation/resolv-conf" -rev = "200e958941d522a70c5877e3d846f55b5586c68d" +rev = "56251316cc4127bcbf36e68ce5e2093f4d33e227" # # Our crates From 70df8364b38137011b540cf39821967c91c41efa Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 21 Jun 2025 00:50:02 +0100 Subject: [PATCH 1821/2091] chore: Bump rustyline-async from 0.4.3 to 0.4.6 --- Cargo.lock | 31 +++++++------------------------ Cargo.toml | 6 +++--- 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8bab900..7852d2ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1105,7 +1105,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "980c2afde4af43d6a05c5be738f9eae595cff86dce1f38f88b95058a98c027f3" dependencies = [ - "crossterm 0.29.0", + "crossterm", ] [[package]] @@ -1165,7 +1165,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5282b45c96c5978c8723ea83385cb9a488b64b7d175733f48d07bf9da514a863" dependencies = [ "crokey-proc_macros", - "crossterm 0.29.0", + "crossterm", "once_cell", "serde", "strict", @@ -1177,7 +1177,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ea0218d3fedf0797fa55676f1964ef5d27103d41ed0281b4bbd2a6e6c3d8d28" dependencies = [ - "crossterm 0.29.0", + "crossterm", "proc-macro2", "quote", "strict", @@ -1240,23 +1240,6 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" -[[package]] -name = "crossterm" -version = "0.28.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" -dependencies = [ - "bitflags 2.9.1", - "crossterm_winapi", - "futures-core", - "mio", - "parking_lot", - "rustix 0.38.44", - "signal-hook", - "signal-hook-mio", - "winapi", -] - [[package]] name = "crossterm" version = "0.29.0" @@ -1267,6 +1250,7 @@ dependencies = [ "crossterm_winapi", "derive_more", "document-features", + "futures-core", "mio", "parking_lot", "rustix 1.0.7", @@ -4153,11 +4137,10 @@ checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "rustyline-async" -version = "0.4.3" -source = "git+https://forgejo.ellis.link/continuwuation/rustyline-async?rev=deaeb0694e2083f53d363b648da06e10fc13900c#deaeb0694e2083f53d363b648da06e10fc13900c" +version = "0.4.6" +source = "git+https://forgejo.ellis.link/continuwuation/rustyline-async?rev=e9f01cf8c6605483cb80b3b0309b400940493d7f#e9f01cf8c6605483cb80b3b0309b400940493d7f" dependencies = [ - "crossterm 0.28.1", - "futures-channel", + "crossterm", "futures-util", "pin-project", "thingbuf", diff --git a/Cargo.toml b/Cargo.toml index 0ebb758c..f3de5d7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -556,11 +556,11 @@ rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" git = "https://forgejo.ellis.link/continuwuation/tracing" rev = "1e64095a8051a1adf0d1faa307f9f030889ec2aa" -# adds a tab completion callback: https://forgejo.ellis.link/continuwuation/rustyline-async/commit/de26100b0db03e419a3d8e1dd26895d170d1fe50 -# adds event for CTRL+\: https://forgejo.ellis.link/continuwuation/rustyline-async/commit/67d8c49aeac03a5ef4e818f663eaa94dd7bf339b +# adds a tab completion callback: https://forgejo.ellis.link/continuwuation/rustyline-async/src/branch/main/.patchy/0002-add-tab-completion-callback.patch +# adds event for CTRL+\: https://forgejo.ellis.link/continuwuation/rustyline-async/src/branch/main/.patchy/0001-add-event-for-ctrl.patch [patch.crates-io.rustyline-async] git = "https://forgejo.ellis.link/continuwuation/rustyline-async" -rev = "deaeb0694e2083f53d363b648da06e10fc13900c" +rev = "e9f01cf8c6605483cb80b3b0309b400940493d7f" # adds LIFO queue scheduling; this should be updated with PR progress. [patch.crates-io.event-listener] From 93719018a881f7aa2e150c43647565995b5c6b0e Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 21 Jun 2025 17:58:28 +0100 Subject: [PATCH 1822/2091] ci: Run additional sanity checks on repository --- .editorconfig | 2 +- .forgejo/actions/prefligit/action.yml | 22 +++++++++++ .forgejo/workflows/prefligit-checks.yml | 15 ++++++++ .forgejo/workflows/rust-checks.yml | 2 +- .pre-commit-config.yaml | 50 +++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 .forgejo/actions/prefligit/action.yml create mode 100644 .forgejo/workflows/prefligit-checks.yml create mode 100644 .pre-commit-config.yaml diff --git a/.editorconfig b/.editorconfig index 91f073bd..17bb0c17 100644 --- a/.editorconfig +++ b/.editorconfig @@ -23,6 +23,6 @@ indent_size = 2 indent_style = tab max_line_length = 98 -[{.forgejo/**/*.yml,.github/**/*.yml}] +[{**/*.yml}] indent_size = 2 indent_style = space diff --git a/.forgejo/actions/prefligit/action.yml b/.forgejo/actions/prefligit/action.yml new file mode 100644 index 00000000..10f47af9 --- /dev/null +++ b/.forgejo/actions/prefligit/action.yml @@ -0,0 +1,22 @@ +name: reflighit +description: | + Runs reflighit, pre-commit reimplemented in Rust. +inputs: + extra_args: + description: options to pass to pre-commit run + required: false + default: '--all-files' + +runs: + using: composite + steps: + - name: Install Prefligit + shell: bash + run: | + curl --proto '=https' --tlsv1.2 -LsSf https://github.com/j178/prefligit/releases/download/v0.0.10/prefligit-installer.sh | sh + - uses: actions/cache@v3 + with: + path: ~/.cache/prefligit + key: prefligit-0|${{ hashFiles('.pre-commit-config.yaml') }} + - run: prefligit run --show-diff-on-failure --color=always ${{ inputs.extra_args }} + shell: bash diff --git a/.forgejo/workflows/prefligit-checks.yml b/.forgejo/workflows/prefligit-checks.yml new file mode 100644 index 00000000..8eae8451 --- /dev/null +++ b/.forgejo/workflows/prefligit-checks.yml @@ -0,0 +1,15 @@ +name: Checks / Prefligit + +on: + push: + +jobs: + prefligit: + runs-on: ubuntu-latest + env: + FROM_REF: ${{ github.event.pull_request.base.sha || (!github.event.forced && ( github.event.before != '0000000000000000000000000000000000000000' && github.event.before || github.sha )) || format('{0}~', github.sha) }} + TO_REF: ${{ github.sha }} + steps: + - uses: ./.forgejo/actions/prefligit + with: + extra_args: --from-ref ${{ env.FROM_REF }} --to-ref ${{ env.TO_REF }} --hook-stage manual diff --git a/.forgejo/workflows/rust-checks.yml b/.forgejo/workflows/rust-checks.yml index 35ca1ad7..105efd0f 100644 --- a/.forgejo/workflows/rust-checks.yml +++ b/.forgejo/workflows/rust-checks.yml @@ -1,4 +1,4 @@ -name: Rust Checks +name: Checks / Rust on: push: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..fc0f9d71 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,50 @@ +default_install_hook_types: + - pre-commit + - commit-msg +default_stages: + - pre-commit + - manual + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-byte-order-marker + - id: check-case-conflict + - id: check-symlinks + - id: destroyed-symlinks + - id: check-yaml + - id: check-json + - id: check-toml + - id: end-of-file-fixer + - id: trailing-whitespace + - id: mixed-line-ending + - id: check-merge-conflict + - id: check-added-large-files + + - repo: https://github.com/crate-ci/typos + rev: v1.26.0 + hooks: + - id: typos + + - repo: local + hooks: + - id: cargo-fmt + name: cargo fmt + entry: cargo +nightly fmt -- + language: system + types: [rust] + pass_filenames: false + stages: + - pre-commit + + - repo: local + hooks: + - id: cargo-clippy + name: cargo clippy + language: system + types: [rust] + pass_filenames: false + entry: cargo clippy --workspace --locked --no-deps --profile test -- -D warnings + stages: + - pre-commit From 46c193e74b2ce86c48ce802333a0aabce37fd6e9 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 21 Jun 2025 17:59:01 +0100 Subject: [PATCH 1823/2091] chore: fix end of files & trailing whitespace --- .gitattributes | 2 +- docs/deploying/arch-linux.md | 2 +- docs/deploying/docker-compose.override.yml | 1 - docs/deploying/docker-compose.with-caddy.yml | 2 +- docs/deploying/docker-compose.with-traefik.yml | 2 +- docs/deploying/nixos.md | 2 +- docs/static/_headers | 2 +- docs/static/announcements.schema.json | 2 +- docs/static/support | 2 +- theme/css/chrome.css | 1 - 10 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.gitattributes b/.gitattributes index 3dfaca65..a1a845b6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -84,4 +84,4 @@ Cargo.lock text *.zst binary # Text files where line endings should be preserved -*.patch -text \ No newline at end of file +*.patch -text diff --git a/docs/deploying/arch-linux.md b/docs/deploying/arch-linux.md index 52d2afb2..6e50410d 100644 --- a/docs/deploying/arch-linux.md +++ b/docs/deploying/arch-linux.md @@ -2,4 +2,4 @@ Continuwuity is available on the `archlinuxcn` repository and AUR, with the same package name `continuwuity`, which includes latest taggged version. The development version is available on AUR as `continuwuity-git` -Simply install the `continuwuity` package. Configure the service in `/etc/conduwuit/conduwuit.toml`, then enable/start the continuwuity.service. \ No newline at end of file +Simply install the `continuwuity` package. Configure the service in `/etc/conduwuit/conduwuit.toml`, then enable/start the continuwuity.service. diff --git a/docs/deploying/docker-compose.override.yml b/docs/deploying/docker-compose.override.yml index 168b1ae6..c1a7248c 100644 --- a/docs/deploying/docker-compose.override.yml +++ b/docs/deploying/docker-compose.override.yml @@ -34,4 +34,3 @@ services: # - "traefik.http.routers.to-element-web.tls.certresolver=letsencrypt" # vim: ts=2:sw=2:expandtab - diff --git a/docs/deploying/docker-compose.with-caddy.yml b/docs/deploying/docker-compose.with-caddy.yml index 3dfc9d85..dd6a778f 100644 --- a/docs/deploying/docker-compose.with-caddy.yml +++ b/docs/deploying/docker-compose.with-caddy.yml @@ -26,7 +26,7 @@ services: restart: unless-stopped volumes: - db:/var/lib/continuwuity - - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. + - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. #- ./continuwuity.toml:/etc/continuwuity.toml environment: CONTINUWUITY_SERVER_NAME: example.com # EDIT THIS diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index 9acc4221..49b7c905 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -8,7 +8,7 @@ services: restart: unless-stopped volumes: - db:/var/lib/continuwuity - - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. + - /etc/resolv.conf:/etc/resolv.conf:ro # Use the host's DNS resolver rather than Docker's. #- ./continuwuity.toml:/etc/continuwuity.toml networks: - proxy diff --git a/docs/deploying/nixos.md b/docs/deploying/nixos.md index cf2c09e4..2fdcbe5c 100644 --- a/docs/deploying/nixos.md +++ b/docs/deploying/nixos.md @@ -29,7 +29,7 @@ appropriately to use Continuwuity instead of Conduit. Due to the lack of a Continuwuity NixOS module, when using the `services.matrix-conduit` module a workaround like the one below is necessary to use UNIX sockets. This is because the UNIX -socket option does not exist in Conduit, and the module forcibly sets the `address` and +socket option does not exist in Conduit, and the module forcibly sets the `address` and `port` config options. ```nix diff --git a/docs/static/_headers b/docs/static/_headers index 6e52de9f..dd07f21b 100644 --- a/docs/static/_headers +++ b/docs/static/_headers @@ -3,4 +3,4 @@ Content-Type: application/json /.well-known/continuwuity/* Access-Control-Allow-Origin: * - Content-Type: application/json \ No newline at end of file + Content-Type: application/json diff --git a/docs/static/announcements.schema.json b/docs/static/announcements.schema.json index cacd10c9..474c0d29 100644 --- a/docs/static/announcements.schema.json +++ b/docs/static/announcements.schema.json @@ -32,4 +32,4 @@ "required": [ "announcements" ] - } \ No newline at end of file + } diff --git a/docs/static/support b/docs/static/support index 6b7a9860..88a85c7d 100644 --- a/docs/static/support +++ b/docs/static/support @@ -21,4 +21,4 @@ } ], "support_page": "https://continuwuity.org/introduction#contact" -} \ No newline at end of file +} diff --git a/theme/css/chrome.css b/theme/css/chrome.css index d6cc2b32..f14ffc2c 100644 --- a/theme/css/chrome.css +++ b/theme/css/chrome.css @@ -605,4 +605,3 @@ ul#searchresults span.teaser em { margin-inline-start: -14px; width: 14px; } - From a682e9dbb879340c63772d1978ca8b74eb28840f Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 21 Jun 2025 18:03:38 +0100 Subject: [PATCH 1824/2091] chore: Add commit to ignored revs --- .git-blame-ignore-revs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 33f738f3..ddfc0568 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -5,3 +5,5 @@ f419c64aca300a338096b4e0db4c73ace54f23d0 # use chain_width 60 162948313c212193965dece50b816ef0903172ba 5998a0d883d31b866f7c8c46433a8857eae51a89 +# trailing whitespace and newlines +46c193e74b2ce86c48ce802333a0aabce37fd6e9 From 2ecbd75d64a6f48a8baaf61145d684ff880ea986 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 21 Jun 2025 18:20:04 +0100 Subject: [PATCH 1825/2091] ci: fixes - Install UV - Verbose run - Set permissions explicitly - Check all files --- .editorconfig | 2 +- .forgejo/actions/prefligit/action.yml | 11 ++++++++--- .forgejo/workflows/prefligit-checks.yml | 9 ++++++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.editorconfig b/.editorconfig index 17bb0c17..3e7fd1b8 100644 --- a/.editorconfig +++ b/.editorconfig @@ -23,6 +23,6 @@ indent_size = 2 indent_style = tab max_line_length = 98 -[{**/*.yml}] +[*.yml] indent_size = 2 indent_style = space diff --git a/.forgejo/actions/prefligit/action.yml b/.forgejo/actions/prefligit/action.yml index 10f47af9..8cbd4500 100644 --- a/.forgejo/actions/prefligit/action.yml +++ b/.forgejo/actions/prefligit/action.yml @@ -1,6 +1,6 @@ -name: reflighit +name: prefligit description: | - Runs reflighit, pre-commit reimplemented in Rust. + Runs prefligit, pre-commit reimplemented in Rust. inputs: extra_args: description: options to pass to pre-commit run @@ -10,6 +10,11 @@ inputs: runs: using: composite steps: + - name: Install uv + uses: https://github.com/astral-sh/setup-uv@v6 + with: + enable-cache: true + ignore-nothing-to-cache: true - name: Install Prefligit shell: bash run: | @@ -18,5 +23,5 @@ runs: with: path: ~/.cache/prefligit key: prefligit-0|${{ hashFiles('.pre-commit-config.yaml') }} - - run: prefligit run --show-diff-on-failure --color=always ${{ inputs.extra_args }} + - run: prefligit run --show-diff-on-failure --color=always -v ${{ inputs.extra_args }} shell: bash diff --git a/.forgejo/workflows/prefligit-checks.yml b/.forgejo/workflows/prefligit-checks.yml index 8eae8451..cc512496 100644 --- a/.forgejo/workflows/prefligit-checks.yml +++ b/.forgejo/workflows/prefligit-checks.yml @@ -2,6 +2,9 @@ name: Checks / Prefligit on: push: + pull_request: +permissions: + contents: read jobs: prefligit: @@ -10,6 +13,10 @@ jobs: FROM_REF: ${{ github.event.pull_request.base.sha || (!github.event.forced && ( github.event.before != '0000000000000000000000000000000000000000' && github.event.before || github.sha )) || format('{0}~', github.sha) }} TO_REF: ${{ github.sha }} steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false - uses: ./.forgejo/actions/prefligit with: - extra_args: --from-ref ${{ env.FROM_REF }} --to-ref ${{ env.TO_REF }} --hook-stage manual + extra_args: --all-files --hook-stage manual From 4f174324baadfd154a48d9e6e79b20ca63770854 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 23 Jun 2025 01:04:27 +0100 Subject: [PATCH 1826/2091] docs: Update contributing guide --- CONTRIBUTING.md | 92 ++++++++++++++++++++------------------------- docs/development.md | 76 ++++++++++++++++++++++--------------- 2 files changed, 86 insertions(+), 82 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index da426801..c57de8b4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing guide -This page is for about contributing to Continuwuity. The +This page is about contributing to Continuwuity. The [development](./development.md) page may be of interest for you as well. If you would like to work on an [issue][issues] that is not assigned, preferably @@ -10,7 +10,7 @@ and comment on it. ### Linting and Formatting It is mandatory all your changes satisfy the lints (clippy, rustc, rustdoc, etc) -and your code is formatted via the **nightly** `cargo fmt`. A lot of the +and your code is formatted via the **nightly** rustfmt (`cargo +nightly fmt`). A lot of the `rustfmt.toml` features depend on nightly toolchain. It would be ideal if they weren't nightly-exclusive features, but they currently still are. CI's rustfmt uses nightly. @@ -21,67 +21,62 @@ comment saying why. Do not write inefficient code for the sake of satisfying lints. If a lint is wrong and provides a more inefficient solution or suggestion, allow the lint and mention that in a comment. -### Running CI tests locally +### Running tests locally -continuwuity's CI for tests, linting, formatting, audit, etc use -[`engage`][engage]. engage can be installed from nixpkgs or `cargo install -engage`. continuwuity's Nix flake devshell has the nixpkgs engage with `direnv`. -Use `engage --help` for more usage details. +Tests, compilation, and linting can be run with standard Cargo commands: -To test, format, lint, etc that CI would do, install engage, allow the `.envrc` -file using `direnv allow`, and run `engage`. +```bash +# Run tests +cargo test -All of the tasks are defined at the [engage.toml][engage.toml] file. You can -view all of them neatly by running `engage list` +# Check compilation +cargo check --workspace -If you would like to run only a specific engage task group, use `just`: +# Run lints +cargo clippy --workspace +# Auto-fix: cargo clippy --workspace --fix --allow-staged; -- `engage just ` -- Example: `engage just lints` +# Format code (must use nightly) +cargo +nightly fmt +``` -If you would like to run a specific engage task in a specific group, use `just - [TASK]`: `engage just lints cargo-fmt` +### Building Docker images -The following binaries are used in [`engage.toml`][engage.toml]: +Docker images can be built using the standard Docker build command: -- [`engage`][engage] -- `nix` -- [`direnv`][direnv] -- `rustc` -- `cargo` -- `cargo-fmt` -- `rustdoc` -- `cargo-clippy` -- [`cargo-audit`][cargo-audit] -- [`cargo-deb`][cargo-deb] -- [`lychee`][lychee] -- [`markdownlint-cli`][markdownlint-cli] -- `dpkg` +```bash +docker build -f docker/Dockerfile . +``` + +The Docker image can be cross-compiled for different architectures if needed. ### Matrix tests -CI runs [Complement][complement], but currently does not fail if results from -the checked-in results differ with the new results. If your changes are done to -fix Matrix tests, note that in your pull request. If more Complement tests start -failing from your changes, please review the logs (they are uploaded as -artifacts) and determine if they're intended or not. +Continuwuity uses [Complement][complement] for Matrix protocol compliance testing. Complement tests are run manually by developers, and documentation on how to run these tests locally is currently being developed. -If you'd like to run Complement locally using Nix, see the -[testing](development/testing.md) page. +If your changes are done to fix Matrix tests, please note that in your pull request. If more Complement tests start failing from your changes, please review the logs and determine if they're intended or not. -[Sytest][sytest] support will come soon. +[Sytest][sytest] is currently unsupported. ### Writing documentation -Continuwuity's website uses [`mdbook`][mdbook] and deployed via CI using GitHub -Pages in the [`documentation.yml`][documentation.yml] workflow file with Nix's -mdbook in the devshell. All documentation is in the `docs/` directory at the top -level. The compiled mdbook website is also uploaded as an artifact. +Continuwuity's website uses [`mdbook`][mdbook] and is deployed via CI using Cloudflare Pages +in the [`documentation.yml`][documentation.yml] workflow file. All documentation is in the `docs/` +directory at the top level. -To build the documentation using Nix, run: `bin/nix-build-and-cache just .#book` +To build the documentation locally: -The output of the mdbook generation is in `result/`. mdbooks can be opened in -your browser from the individual HTML files without any web server needed. +1. Install mdbook if you don't have it already: + ```bash + cargo install mdbook # or cargo binstall, or another method + ``` + +2. Build the documentation: + ```bash + mdbook build + ``` + +The output of the mdbook generation is in `public/`. You can open the HTML files directly in your browser without needing a web server. ### Inclusivity and Diversity @@ -132,13 +127,6 @@ continuwuityMatrix rooms for Code of Conduct violations. [issues]: https://forgejo.ellis.link/continuwuation/continuwuity/issues [continuwuity-matrix]: https://matrix.to/#/#continuwuity:continuwuity.org [complement]: https://github.com/matrix-org/complement/ -[engage.toml]: https://forgejo.ellis.link/continuwuation/continuwuity/src/branch/main/engage.toml -[engage]: https://charles.page.computer.surgery/engage/ [sytest]: https://github.com/matrix-org/sytest/ -[cargo-deb]: https://github.com/kornelski/cargo-deb -[lychee]: https://github.com/lycheeverse/lychee -[markdownlint-cli]: https://github.com/igorshubovych/markdownlint-cli -[cargo-audit]: https://github.com/RustSec/rustsec/tree/main/cargo-audit -[direnv]: https://direnv.net/ [mdbook]: https://rust-lang.github.io/mdBook/ [documentation.yml]: https://forgejo.ellis.link/continuwuation/continuwuity/src/branch/main/.forgejo/workflows/documentation.yml diff --git a/docs/development.md b/docs/development.md index 1e344f41..dfbddea0 100644 --- a/docs/development.md +++ b/docs/development.md @@ -68,31 +68,22 @@ do this if Rust supported workspace-level features to begin with. ## List of forked dependencies -During Continuwuity development, we have had to fork -some dependencies to support our use-cases in some areas. This ranges from -things said upstream project won't accept for any reason, faster-paced -development (unresponsive or slow upstream), Continuwuity-specific usecases, or -lack of time to upstream some things. +During Continuwuity (and prior projects) development, we have had to fork some dependencies to support our use-cases. +These forks exist for various reasons including features that upstream projects won't accept, +faster-paced development, Continuwuity-specific usecases, or lack of time to upstream changes. -- [ruma/ruma][1]: - various performance -improvements, more features, faster-paced development, better client/server interop -hacks upstream won't accept, etc -- [facebook/rocksdb][2]: - liburing -build fixes and GCC debug build fix -- [tikv/jemallocator][3]: - musl -builds seem to be broken on upstream, fixes some broken/suspicious code in -places, additional safety measures, and support redzones for Valgrind -- [zyansheep/rustyline-async][4]: - - tab completion callback and -`CTRL+\` signal quit event for Continuwuity console CLI -- [rust-rocksdb/rust-rocksdb][5]: - - [`@zaidoon1`][8]'s fork -has quicker updates, more up to date dependencies, etc. Our fork fixes musl build -issues, removes unnecessary `gtest` include, and uses our RocksDB and jemallocator -forks. -- [tokio-rs/tracing][6]: - Implements -`Clone` for `EnvFilter` to support dynamically changing tracing envfilter's -alongside other logging/metrics things +All forked dependencies are maintained under the [continuwuation organization on Forgejo](https://forgejo.ellis.link/continuwuation): + +- [ruwuma][continuwuation-ruwuma] - Fork of [ruma/ruma][ruma] with various performance improvements, more features and better client/server interop +- [rocksdb][continuwuation-rocksdb] - Fork of [facebook/rocksdb][rocksdb] via [`@zaidoon1`][8] with liburing build fixes and GCC debug build fixes +- [jemallocator][continuwuation-jemallocator] - Fork of [tikv/jemallocator][jemallocator] fixing musl builds, suspicious code, + and adding support for redzones in Valgrind +- [rustyline-async][continuwuation-rustyline-async] - Fork of [zyansheep/rustyline-async][rustyline-async] with tab completion callback + and `CTRL+\` signal quit event for Continuwuity console CLI +- [rust-rocksdb][continuwuation-rust-rocksdb] - Fork of [rust-rocksdb/rust-rocksdb][rust-rocksdb] fixing musl build issues, + removing unnecessary `gtest` include, and using our RocksDB and jemallocator forks +- [tracing][continuwuation-tracing] - Fork of [tokio-rs/tracing][tracing] implementing `Clone` for `EnvFilter` to + support dynamically changing tracing environments ## Debugging with `tokio-console` @@ -113,12 +104,37 @@ You will also need to enable the `tokio_console` config option in Continuwuity w starting it. This was due to tokio-console causing gradual memory leak/usage if left enabled. -[1]: https://github.com/ruma/ruma/ -[2]: https://github.com/facebook/rocksdb/ -[3]: https://github.com/tikv/jemallocator/ -[4]: https://github.com/zyansheep/rustyline-async/ -[5]: https://github.com/rust-rocksdb/rust-rocksdb/ -[6]: https://github.com/tokio-rs/tracing/ +## Building Docker Images + +To build a Docker image for Continuwuity, use the standard Docker build command: + +```bash +docker build -f docker/Dockerfile . +``` + +The image can be cross-compiled for different architectures. + +## Matrix Protocol Compliance Testing + +Complement (the Matrix protocol compliance testing suite) is run manually by developers. +Documentation on how to run Complement tests locally is being developed and will be added soon. + +Sytest is currently unsupported. + +[continuwuation-ruwuma]: https://forgejo.ellis.link/continuwuation/ruwuma +[continuwuation-rocksdb]: https://forgejo.ellis.link/continuwuation/rocksdb +[continuwuation-jemallocator]: https://forgejo.ellis.link/continuwuation/jemallocator +[continuwuation-rustyline-async]: https://forgejo.ellis.link/continuwuation/rustyline-async +[continuwuation-rust-rocksdb]: https://forgejo.ellis.link/continuwuation/rust-rocksdb +[continuwuation-tracing]: https://forgejo.ellis.link/continuwuation/tracing + +[ruma]: https://github.com/ruma/ruma/ +[rocksdb]: https://github.com/facebook/rocksdb/ +[jemallocator]: https://github.com/tikv/jemallocator/ +[rustyline-async]: https://github.com/zyansheep/rustyline-async/ +[rust-rocksdb]: https://github.com/rust-rocksdb/rust-rocksdb/ +[tracing]: https://github.com/tokio-rs/tracing/ + [7]: https://docs.rs/tokio-console/latest/tokio_console/ [8]: https://github.com/zaidoon1/ [9]: https://github.com/rust-lang/cargo/issues/12162 From 4d69a1ad511b1fc81a668ef319fb1e053788dfc2 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 23 Jun 2025 01:25:38 +0100 Subject: [PATCH 1827/2091] docs: Deduplicate sections --- CONTRIBUTING.md | 14 ++------------ docs/development.md | 7 ------- 2 files changed, 2 insertions(+), 19 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c57de8b4..3dc99dc2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,16 +40,6 @@ cargo clippy --workspace cargo +nightly fmt ``` -### Building Docker images - -Docker images can be built using the standard Docker build command: - -```bash -docker build -f docker/Dockerfile . -``` - -The Docker image can be cross-compiled for different architectures if needed. - ### Matrix tests Continuwuity uses [Complement][complement] for Matrix protocol compliance testing. Complement tests are run manually by developers, and documentation on how to run these tests locally is currently being developed. @@ -120,9 +110,9 @@ By sending a pull request or patch, you are agreeing that your changes are allowed to be licenced under the Apache-2.0 licence and all of your conduct is in line with the Contributor's Covenant, and continuwuity's Code of Conduct. -Contribution by users who violate either of these code of conducts will not have +Contribution by users who violate either of these code of conducts may not have their contributions accepted. This includes users who have been banned from -continuwuityMatrix rooms for Code of Conduct violations. +continuwuity Matrix rooms for Code of Conduct violations. [issues]: https://forgejo.ellis.link/continuwuation/continuwuity/issues [continuwuity-matrix]: https://matrix.to/#/#continuwuity:continuwuity.org diff --git a/docs/development.md b/docs/development.md index dfbddea0..68b963c8 100644 --- a/docs/development.md +++ b/docs/development.md @@ -114,13 +114,6 @@ docker build -f docker/Dockerfile . The image can be cross-compiled for different architectures. -## Matrix Protocol Compliance Testing - -Complement (the Matrix protocol compliance testing suite) is run manually by developers. -Documentation on how to run Complement tests locally is being developed and will be added soon. - -Sytest is currently unsupported. - [continuwuation-ruwuma]: https://forgejo.ellis.link/continuwuation/ruwuma [continuwuation-rocksdb]: https://forgejo.ellis.link/continuwuation/rocksdb [continuwuation-jemallocator]: https://forgejo.ellis.link/continuwuation/jemallocator From 4a289a9feedc720dd17a3af08d7dc7d4b255e55f Mon Sep 17 00:00:00 2001 From: Kimiblock Moe Date: Tue, 24 Jun 2025 19:01:21 +0800 Subject: [PATCH 1828/2091] arch systemd: use credentials to load config --- arch/conduwuit.service | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index c86e37bd..d5a65e4d 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -6,6 +6,7 @@ After=network-online.target Documentation=https://continuwuity.org/ RequiresMountsFor=/var/lib/private/conduwuit Alias=matrix-conduwuit.service + [Service] DynamicUser=yes Type=notify-reload @@ -59,7 +60,8 @@ StateDirectory=conduwuit RuntimeDirectory=conduwuit RuntimeDirectoryMode=0750 -Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml" +Environment=CONTINUWUITY_CONFIG=${CREDENTIALS_DIRECTORY}/config.toml +LoadCredential=config.toml:/etc/conduwuit/conduwuit.toml BindPaths=/var/lib/private/conduwuit:/var/lib/matrix-conduit BindPaths=/var/lib/private/conduwuit:/var/lib/private/matrix-conduit From 3177545a6f966c19a2f219a3641b9fa5c8bfdb38 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 24 Jun 2025 21:45:54 +0100 Subject: [PATCH 1829/2091] chore: Remove clippy pre-commit hook It's too slow for a good git experience --- .pre-commit-config.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc0f9d71..22bcd09c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,14 +37,3 @@ repos: pass_filenames: false stages: - pre-commit - - - repo: local - hooks: - - id: cargo-clippy - name: cargo clippy - language: system - types: [rust] - pass_filenames: false - entry: cargo clippy --workspace --locked --no-deps --profile test -- -D warnings - stages: - - pre-commit From 9bbe333082cc74926609a6b7c92eb76de877c452 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 24 Jun 2025 21:48:33 +0100 Subject: [PATCH 1830/2091] ci: Don't run docs flow when the secret is inaccessible --- .forgejo/workflows/documentation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.forgejo/workflows/documentation.yml b/.forgejo/workflows/documentation.yml index 7d95a317..4f3e903c 100644 --- a/.forgejo/workflows/documentation.yml +++ b/.forgejo/workflows/documentation.yml @@ -17,6 +17,7 @@ jobs: docs: name: Build and Deploy Documentation runs-on: ubuntu-latest + if: secrets.CLOUDFLARE_API_TOKEN != '' steps: - name: Sync repository From eb75c4ecb0b5ac0cec38df85c634323103a28f6d Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 24 Jun 2025 22:05:52 +0100 Subject: [PATCH 1831/2091] chore: Fix typos in commit messages automatically --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 22bcd09c..facebcc4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -26,6 +26,9 @@ repos: rev: v1.26.0 hooks: - id: typos + - id: typos + name: commit-msg-typos + stages: [commit-msg] - repo: local hooks: From b787e97dc173da7a6673df9f5ea628a60340a497 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 24 Jun 2025 22:22:13 +0100 Subject: [PATCH 1832/2091] chore: Document & enforce conventional commit messages --- .pre-commit-config.yaml | 5 +++ CONTRIBUTING.md | 80 +++++++++++++++++++++++++++++++++++++++++ committed.toml | 2 ++ 3 files changed, 87 insertions(+) create mode 100644 committed.toml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index facebcc4..68e3a982 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,6 +30,11 @@ repos: name: commit-msg-typos stages: [commit-msg] + - repo: https://github.com/crate-ci/committed + rev: v1.1.7 + hooks: + - id: committed + - repo: local hooks: - id: cargo-fmt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3dc99dc2..1c091183 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,45 @@ comment saying why. Do not write inefficient code for the sake of satisfying lints. If a lint is wrong and provides a more inefficient solution or suggestion, allow the lint and mention that in a comment. +### Pre-commit Checks + +Continuwuity uses pre-commit hooks to enforce various coding standards and catch common issues before they're committed. These checks include: + +- Code formatting and linting +- Typo detection (both in code and commit messages) +- Checking for large files +- Ensuring proper line endings and no trailing whitespace +- Validating YAML, JSON, and TOML files +- Checking for merge conflicts + +You can run these checks locally by installing [prefligit](https://github.com/j178/prefligit): + + +```bash +# Install prefligit using cargo-binstall +cargo binstall prefligit + +# Install git hooks to run checks automatically +prefligit install + +# Run all checks +prefligit --all-files +``` + +Alternatively, you can use [pre-commit](https://pre-commit.com/): +```bash +# Install pre-commit +pip install pre-commit + +# Install the hooks +pre-commit install + +# Run all checks manually +pre-commit run --all-files +``` + +These same checks are run in CI via the prefligit-checks workflow to ensure consistency. + ### Running tests locally Tests, compilation, and linting can be run with standard Cargo commands: @@ -94,6 +133,40 @@ Rust's default style and standards with regards to [function names, variable names, comments](https://rust-lang.github.io/api-guidelines/naming.html), etc applies here. +### Commit Messages + +Continuwuity follows the [Conventional Commits](https://www.conventionalcommits.org/) specification for commit messages. This provides a standardized format that makes the commit history more readable and enables automated tools to generate changelogs. + +The basic structure is: +``` +[(optional scope)]: + +[optional body] + +[optional footer(s)] +``` + +The allowed types for commits are: +- `fix`: Bug fixes +- `feat`: New features +- `docs`: Documentation changes +- `style`: Changes that don't affect the meaning of the code (formatting, etc.) +- `refactor`: Code changes that neither fix bugs nor add features +- `perf`: Performance improvements +- `test`: Adding or fixing tests +- `build`: Changes to the build system or dependencies +- `ci`: Changes to CI configuration +- `chore`: Other changes that don't modify source or test files + +Examples: +``` +feat: add user authentication +fix(database): resolve connection pooling issue +docs: update installation instructions +``` + +The project uses the `committed` hook to validate commit messages in pre-commit. This ensures all commits follow the conventional format. + ### Creating pull requests Please try to keep contributions to the Forgejo Instance. While the mirrors of continuwuity @@ -103,6 +176,13 @@ This prevents us from having to ping once in a while to double check the status of it, especially when the CI completed successfully and everything so it *looks* done. +Before submitting a pull request, please ensure: +1. Your code passes all CI checks (formatting, linting, typo detection, etc.) +2. Your commit messages follow the conventional commits format +3. Tests are added for new functionality +4. Documentation is updated if needed + + Direct all PRs/MRs to the `main` branch. diff --git a/committed.toml b/committed.toml new file mode 100644 index 00000000..64f7f18a --- /dev/null +++ b/committed.toml @@ -0,0 +1,2 @@ +style = "conventional" +allowed_types = ["ci", "build", "fix", "feat", "chore", "docs", "style", "refactor", "perf", "test"] From a24278dc1b59b6d20b985e128b515257de1e5143 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 24 Jun 2025 23:12:09 +0100 Subject: [PATCH 1833/2091] docs: Update mirror badges --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e3eb807f..e71178dc 100644 --- a/README.md +++ b/README.md @@ -11,11 +11,13 @@ It's a community continuation of the [conduwuit](https://github.com/girlbossceo/ -[![forgejo.ellis.link](https://img.shields.io/badge/Ellis%20Git-main+packages-green?style=flat&logo=forgejo&labelColor=fff)](https://forgejo.ellis.link/continuwuation/continuwuity) ![](https://forgejo.ellis.link/continuwuation/continuwuity/badges/stars.svg?style=flat) [![](https://forgejo.ellis.link/continuwuation/continuwuity/badges/issues/open.svg?style=flat)](https://forgejo.ellis.link/continuwuation/continuwuity/issues?state=open) [![](https://forgejo.ellis.link/continuwuation/continuwuity/badges/pulls/open.svg?style=flat)](https://forgejo.ellis.link/continuwuation/continuwuity/pulls?state=open) +[![forgejo.ellis.link](https://img.shields.io/badge/Ellis%20Git-main+packages-green?style=flat&logo=forgejo&labelColor=fff)](https://forgejo.ellis.link/continuwuation/continuwuity) [![Stars](https://forgejo.ellis.link/continuwuation/continuwuity/badges/stars.svg?style=flat)](https://forgejo.ellis.link/continuwuation/continuwuity/stars) [![Issues](https://forgejo.ellis.link/continuwuation/continuwuity/badges/issues/open.svg?style=flat)](https://forgejo.ellis.link/continuwuation/continuwuity/issues?state=open) [![Pull Requests](https://forgejo.ellis.link/continuwuation/continuwuity/badges/pulls/open.svg?style=flat)](https://forgejo.ellis.link/continuwuation/continuwuity/pulls?state=open) -[![GitHub](https://img.shields.io/badge/GitHub-mirror-blue?style=flat&logo=github&labelColor=fff&logoColor=24292f)](https://github.com/continuwuity/continuwuity) ![](https://img.shields.io/github/stars/continuwuity/continuwuity?style=flat) +[![GitHub](https://img.shields.io/badge/GitHub-mirror-blue?style=flat&logo=github&labelColor=fff&logoColor=24292f)](https://github.com/continuwuity/continuwuity) [![Stars](https://img.shields.io/github/stars/continuwuity/continuwuity?style=flat)](https://github.com/continuwuity/continuwuity/stargazers) -[![Codeberg](https://img.shields.io/badge/Codeberg-mirror-2185D0?style=flat&logo=codeberg&labelColor=fff)](https://codeberg.org/nexy7574/continuwuity) ![](https://codeberg.org/nexy7574/continuwuity/badges/stars.svg?style=flat) +[![GitLab](https://img.shields.io/badge/GitLab-mirror-blue?style=flat&logo=gitlab&labelColor=fff)](https://gitlab.com/continuwuity/continuwuity) [![Stars](https://img.shields.io/gitlab/stars/continuwuity/continuwuity?style=flat)](https://gitlab.com/continuwuity/continuwuity/-/starrers) + +[![Codeberg](https://img.shields.io/badge/Codeberg-mirror-2185D0?style=flat&logo=codeberg&labelColor=fff)](https://codeberg.org/continuwuity/continuwuity) [![Stars](https://codeberg.org/continuwuity/continuwuity/badges/stars.svg?style=flat)](https://codeberg.org/continuwuity/continuwuity/stars) ### Why does this exist? From 63962fc04098802abdf3334f147f25ada0f1d36c Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 24 Jun 2025 23:13:28 +0100 Subject: [PATCH 1834/2091] docs: Remove completed items from the README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index e71178dc..506a62db 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,6 @@ There are currently no open registration Continuwuity instances available. We're working our way through all of the issues in the [Forgejo project](https://forgejo.ellis.link/continuwuation/continuwuity/issues). -- [Replacing old conduwuit links with working continuwuity links](https://forgejo.ellis.link/continuwuation/continuwuity/issues/742) -- [Getting CI and docs deployment working on the new Forgejo project](https://forgejo.ellis.link/continuwuation/continuwuity/issues/740) - [Packaging & availability in more places](https://forgejo.ellis.link/continuwuation/continuwuity/issues/747) - [Appservices bugs & features](https://forgejo.ellis.link/continuwuation/continuwuity/issues?q=&type=all&state=open&labels=178&milestone=0&assignee=0&poster=0) - [Improving compatibility and spec compliance](https://forgejo.ellis.link/continuwuation/continuwuity/issues?labels=119) From f1ca84fcaff3b5a2c6cc4f1889611f7003859a33 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Tue, 24 Jun 2025 23:16:48 +0100 Subject: [PATCH 1835/2091] fix: Correct project brand in admin & OTEL --- src/admin/admin.rs | 2 +- src/main/logging.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/admin/admin.rs b/src/admin/admin.rs index 0d636c72..50b9db7c 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -9,7 +9,7 @@ use crate::{ }; #[derive(Debug, Parser)] -#[command(name = "conduwuit", version = conduwuit::version())] +#[command(name = conduwuit_core::name(), version = conduwuit_core::version())] pub(super) enum AdminCommand { #[command(subcommand)] /// - Commands for managing appservices diff --git a/src/main/logging.rs b/src/main/logging.rs index eeeda127..aec50bd4 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -77,7 +77,7 @@ pub(crate) fn init( ); let tracer = opentelemetry_jaeger::new_agent_pipeline() .with_auto_split_batch(true) - .with_service_name("conduwuit") + .with_service_name(conduwuit_core::name()) .install_batch(opentelemetry_sdk::runtime::Tokio) .expect("jaeger agent pipeline"); let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); From db58d841aa4c99e1bc573faea3113557cc68589e Mon Sep 17 00:00:00 2001 From: Jacob Taylor Date: Fri, 25 Apr 2025 20:59:52 -0700 Subject: [PATCH 1836/2091] fix: Only load children of nested spaces --- src/api/client/space.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/api/client/space.rs b/src/api/client/space.rs index 92768926..23b1e80f 100644 --- a/src/api/client/space.rs +++ b/src/api/client/space.rs @@ -121,7 +121,9 @@ where .map(|(key, val)| (key, val.collect())) .collect(); - if !populate { + if populate { + rooms.push(summary_to_chunk(summary.clone())); + } else { children = children .iter() .rev() @@ -144,10 +146,8 @@ where .collect(); } - if populate { - rooms.push(summary_to_chunk(summary.clone())); - } else if queue.is_empty() && children.is_empty() { - return Err!(Request(InvalidParam("Room IDs in token were not found."))); + if !populate && queue.is_empty() && children.is_empty() { + break; } parents.insert(current_room.clone()); From c82ea24069d58b8e33d00b836ac7d5c126ffb217 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 27 Jun 2025 18:44:46 +0100 Subject: [PATCH 1837/2091] docs: Add Matrix chat and space badges to README --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 506a62db..60dcf81d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ ## A community-driven [Matrix](https://matrix.org/) homeserver in Rust +[![Chat on Matrix](https://img.shields.io/matrix/continuwuity%3Acontinuwuity.org?server_fqdn=matrix.continuwuity.org&fetchMode=summary&logo=matrix)](https://matrix.to/#/#continuwuity:continuwuity.org?via=continuwuity.org&via=ellis.link&via=explodie.org&via=matrix.org) [![Join the space](https://img.shields.io/matrix/space%3Acontinuwuity.org?server_fqdn=matrix.continuwuity.org&fetchMode=summary&logo=matrix&label=space)](https://matrix.to/#/#space:continuwuity.org?via=continuwuity.org&via=ellis.link&via=explodie.org&via=matrix.org) + + + [continuwuity] is a Matrix homeserver written in Rust. From 543ab27747dbf31b583175dd81d7d3aa3c82df79 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Fri, 27 Jun 2025 20:58:52 +0100 Subject: [PATCH 1838/2091] fix: Additional sanity checks when creating a PDU Prevents creating events that are most likely catastrophically invalid --- src/service/rooms/timeline/mod.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 4b2f3cb2..94d306d1 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -719,6 +719,18 @@ impl Service { ); } } + if event_type != TimelineEventType::RoomCreate && prev_events.is_empty() { + return Err!(Request(Unknown("Event incorrectly had zero prev_events."))); + } + if state_key.is_none() && depth.le(&uint!(2)) { + // The first two events in a room are always m.room.create and m.room.member, + // so any other events with that same depth are illegal. + warn!( + "Had unsafe depth {depth} when creating non-state event in {room_id}. Cowardly \ + aborting" + ); + return Err!(Request(Unknown("Unsafe depth for non-state event."))); + } let mut pdu = PduEvent { event_id: ruma::event_id!("$thiswillbefilledinlater").into(), From f508e7654c51070e2231b136506c326305f37d3c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 28 Jun 2025 00:29:07 +0000 Subject: [PATCH 1839/2091] fix: off by one. --- src/service/rooms/timeline/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 94d306d1..37963246 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -722,7 +722,7 @@ impl Service { if event_type != TimelineEventType::RoomCreate && prev_events.is_empty() { return Err!(Request(Unknown("Event incorrectly had zero prev_events."))); } - if state_key.is_none() && depth.le(&uint!(2)) { + if state_key.is_none() && depth.lt(&uint!(2)) { // The first two events in a room are always m.room.create and m.room.member, // so any other events with that same depth are illegal. warn!( From 52e042cb06e05a028340e93bf3ff0d2d37d211b5 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Mon, 26 May 2025 01:22:19 +0100 Subject: [PATCH 1840/2091] Always calculate state diff IDs in syncv3 seemingly fixes #779 --- src/api/client/sync/v3.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 8eac6b66..7bc74c95 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -1009,8 +1009,6 @@ async fn calculate_state_incremental<'a>( ) -> Result { let since_shortstatehash = since_shortstatehash.unwrap_or(current_shortstatehash); - let state_changed = since_shortstatehash != current_shortstatehash; - let encrypted_room = services .rooms .state_accessor @@ -1042,7 +1040,7 @@ async fn calculate_state_incremental<'a>( }) .into(); - let state_diff_ids: OptionFuture<_> = (!full_state && state_changed) + let state_diff_ids: OptionFuture<_> = (!full_state) .then(|| { StreamExt::into_future( services From 9b6ac6c45f44a18d68208038b360212205830a7f Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 19:57:02 +0100 Subject: [PATCH 1841/2091] fix: Ignore existing membership when room is disconnected --- src/api/client/membership.rs | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index e587d806..1800fa60 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -925,24 +925,32 @@ pub async fn join_room_by_id_helper( return Ok(join_room_by_id::v3::Response { room_id: room_id.into() }); } - if let Ok(membership) = services - .rooms - .state_accessor - .get_member(room_id, sender_user) - .await - { - if membership.membership == MembershipState::Ban { - debug_warn!("{sender_user} is banned from {room_id} but attempted to join"); - return Err!(Request(Forbidden("You are banned from the room."))); - } - } - let server_in_room = services .rooms .state_cache .server_in_room(services.globals.server_name(), room_id) .await; + // Only check our known membership if we're already in the room. + // See: https://forgejo.ellis.link/continuwuation/continuwuity/issues/855 + let membership = if server_in_room { + services + .rooms + .state_accessor + .get_member(room_id, sender_user) + .await + } else { + debug!("Ignoring local state for join {room_id}, we aren't in the room yet."); + Ok(RoomMemberEventContent::new(MembershipState::Leave)) + }; + if let Ok(m) = membership { + if m.membership == MembershipState::Ban { + debug_warn!("{sender_user} is banned from {room_id} but attempted to join"); + // TODO: return reason + return Err!(Request(Forbidden("You are banned from the room."))); + } + } + let local_join = server_in_room || servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); From d63c8b9fcaccb93981b0afcccb6c1c6d823d0f73 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 29 Jun 2025 13:16:31 +0100 Subject: [PATCH 1842/2091] feat: Support passing through MSC4293 redact_events --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 2 +- src/api/client/membership.rs | 1 + 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7852d2ca..ab5702e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3798,7 +3798,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "assign", "js_int", @@ -3818,7 +3818,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "js_int", "ruma-common", @@ -3830,7 +3830,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "as_variant", "assign", @@ -3853,7 +3853,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "as_variant", "base64 0.22.1", @@ -3885,7 +3885,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "as_variant", "indexmap 2.9.0", @@ -3910,7 +3910,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "bytes", "headers", @@ -3932,7 +3932,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3941,7 +3941,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "js_int", "ruma-common", @@ -3951,7 +3951,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3966,7 +3966,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "js_int", "ruma-common", @@ -3978,7 +3978,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=d6870a7fb7f6cccff63f7fd0ff6c581bad80e983#d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index f3de5d7f..d54b86a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,7 +350,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://forgejo.ellis.link/continuwuation/ruwuma" #branch = "conduwuit-changes" -rev = "d6870a7fb7f6cccff63f7fd0ff6c581bad80e983" +rev = "0012040617213eccb682b65d0ac76c0e5cc94d8c" features = [ "compat", "rand", diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 1800fa60..b05e2003 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -646,6 +646,7 @@ pub(crate) async fn ban_user_route( is_direct: None, join_authorized_via_users_server: None, third_party_invite: None, + redact_events: body.redact_events, ..current_member_content }), sender_user, From 4b5e8df95c60435fd3e6bce016229108c0494911 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 29 Jun 2025 13:29:27 +0100 Subject: [PATCH 1843/2091] fix: Add missing init fields --- src/api/client/membership.rs | 1 + src/api/client/profile.rs | 2 ++ src/api/client/room/upgrade.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index b05e2003..145b3cde 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -1824,6 +1824,7 @@ pub async fn leave_room( displayname: None, third_party_invite: None, blurhash: None, + redact_events: None, }; let is_banned = services.rooms.metadata.is_banned(room_id); diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 3699b590..e2d1c934 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -343,6 +343,7 @@ pub async fn update_displayname( reason: None, is_direct: None, third_party_invite: None, + redact_events: None, }); Ok((pdu, room_id)) @@ -396,6 +397,7 @@ pub async fn update_avatar_url( reason: None, is_direct: None, third_party_invite: None, + redact_events: None, }); Ok((pdu, room_id)) diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index 9ec0b3bb..da5b49fe 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -189,6 +189,7 @@ pub(crate) async fn upgrade_room_route( blurhash: services.users.blurhash(sender_user).await.ok(), reason: None, join_authorized_via_users_server: None, + redact_events: None, }) .expect("event is valid, we just created it"), unsigned: None, From b4bdd1ee65dc1c0aa4ff974c13a0237542552fc2 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 29 Jun 2025 13:43:27 +0100 Subject: [PATCH 1844/2091] chore: Update ruwuma Fixes the wrong field name being serialised --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ab5702e8..92044b92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3798,7 +3798,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "assign", "js_int", @@ -3818,7 +3818,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "js_int", "ruma-common", @@ -3830,7 +3830,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "as_variant", "assign", @@ -3853,7 +3853,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "as_variant", "base64 0.22.1", @@ -3885,7 +3885,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "as_variant", "indexmap 2.9.0", @@ -3910,7 +3910,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "bytes", "headers", @@ -3932,7 +3932,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3941,7 +3941,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "js_int", "ruma-common", @@ -3951,7 +3951,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3966,7 +3966,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "js_int", "ruma-common", @@ -3978,7 +3978,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=0012040617213eccb682b65d0ac76c0e5cc94d8c#0012040617213eccb682b65d0ac76c0e5cc94d8c" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index d54b86a4..5c289adf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,7 +350,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://forgejo.ellis.link/continuwuation/ruwuma" #branch = "conduwuit-changes" -rev = "0012040617213eccb682b65d0ac76c0e5cc94d8c" +rev = "9b65f83981f6f53d185ce77da37aaef9dfd764a9" features = [ "compat", "rand", From fac9e090cdebbd21c3b282ed04c61d0b8c86ebdd Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 20:32:02 +0100 Subject: [PATCH 1845/2091] feat: Add suspension helper to user service --- src/service/users/mod.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 701561a8..1a9f6600 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -15,6 +15,7 @@ use ruma::{ AnyToDeviceEvent, GlobalAccountDataEventType, ignored_user_list::IgnoredUserListEvent, }, serde::Raw, + uint, }; use serde_json::json; @@ -52,6 +53,7 @@ struct Data { userid_lastonetimekeyupdate: Arc, userid_masterkeyid: Arc, userid_password: Arc, + userid_suspended: Arc, userid_selfsigningkeyid: Arc, userid_usersigningkeyid: Arc, useridprofilekey_value: Arc, @@ -87,6 +89,7 @@ impl crate::Service for Service { userid_lastonetimekeyupdate: args.db["userid_lastonetimekeyupdate"].clone(), userid_masterkeyid: args.db["userid_masterkeyid"].clone(), userid_password: args.db["userid_password"].clone(), + userid_suspended: args.db["userid_suspended"].clone(), userid_selfsigningkeyid: args.db["userid_selfsigningkeyid"].clone(), userid_usersigningkeyid: args.db["userid_usersigningkeyid"].clone(), useridprofilekey_value: args.db["useridprofilekey_value"].clone(), @@ -143,6 +146,16 @@ impl Service { Ok(()) } + /// Suspend account, placing it in a read-only state + pub async fn suspend_account(&self, user_id: &UserId) -> () { + self.db.userid_suspended.insert(user_id, "1"); + } + + /// Unsuspend account, placing it in a read-write state + pub async fn unsuspend_account(&self, user_id: &UserId) -> () { + self.db.userid_suspended.remove(user_id); + } + /// Check if a user has an account on this homeserver. #[inline] pub async fn exists(&self, user_id: &UserId) -> bool { @@ -159,6 +172,16 @@ impl Service { .await } + /// Check if account is suspended + pub async fn is_suspended(&self, user_id: &UserId) -> Result { + self.db + .userid_suspended + .get(user_id) + .map_ok(|val| val.is_empty()) + .map_err(|_| err!(Request(NotFound("User does not exist.")))) + .await + } + /// Check if account is active, infallible pub async fn is_active(&self, user_id: &UserId) -> bool { !self.is_deactivated(user_id).await.unwrap_or(true) From accfda258638d7eef9b2c35c15e2b1658a478743 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 20:35:58 +0100 Subject: [PATCH 1846/2091] feat: Prevent suspended users sending events --- src/api/client/send.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/api/client/send.rs b/src/api/client/send.rs index f753fa65..b87d1822 100644 --- a/src/api/client/send.rs +++ b/src/api/client/send.rs @@ -23,6 +23,9 @@ pub(crate) async fn send_message_event_route( let sender_user = body.sender_user(); let sender_device = body.sender_device.as_deref(); let appservice_info = body.appservice_info.as_ref(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } // Forbid m.room.encrypted if encryption is disabled if MessageLikeEventType::RoomEncrypted == body.event_type && !services.config.allow_encryption From 286974cb9a676bb58501835fbdf1fce08b3d9c6c Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 20:37:09 +0100 Subject: [PATCH 1847/2091] feat: Prevent suspended users redacting events --- src/api/client/redact.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index 8dbe47a6..a8eaf91d 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -1,5 +1,5 @@ use axum::extract::State; -use conduwuit::{Result, matrix::pdu::PduBuilder}; +use conduwuit::{Err, Result, matrix::pdu::PduBuilder}; use ruma::{ api::client::redact::redact_event, events::room::redaction::RoomRedactionEventContent, }; @@ -17,6 +17,10 @@ pub(crate) async fn redact_event_route( ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); let body = body.body; + if services.users.is_suspended(sender_user).await? { + // TODO: Users can redact their own messages while suspended + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; From a6ba9e3045e7cdaccbae006d0037b045c1348016 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 20:39:24 +0100 Subject: [PATCH 1848/2091] feat: Prevent suspended users changing their profile --- src/api/client/profile.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index e2d1c934..bdba4078 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -36,6 +36,9 @@ pub(crate) async fn set_displayname_route( body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } if *sender_user != body.user_id && body.appservice_info.is_none() { return Err!(Request(Forbidden("You cannot update the profile of another user"))); @@ -125,6 +128,9 @@ pub(crate) async fn set_avatar_url_route( body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } if *sender_user != body.user_id && body.appservice_info.is_none() { return Err!(Request(Forbidden("You cannot update the profile of another user"))); From a94128e6986f7f05909329de694e547543b4dc36 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 20:39:57 +0100 Subject: [PATCH 1849/2091] feat: Prevent suspended users joining/knocking on rooms --- src/api/client/membership.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 145b3cde..d78ebdec 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -178,6 +178,9 @@ pub(crate) async fn join_room_by_id_route( body: Ruma, ) -> Result { let sender_user = body.sender_user(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } banned_room_check( &services, @@ -249,6 +252,9 @@ pub(crate) async fn join_room_by_id_or_alias_route( let sender_user = body.sender_user.as_deref().expect("user is authenticated"); let appservice_info = &body.appservice_info; let body = body.body; + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { | Ok(room_id) => { @@ -369,6 +375,9 @@ pub(crate) async fn knock_room_route( ) -> Result { let sender_user = body.sender_user(); let body = &body.body; + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias.clone()) { | Ok(room_id) => { @@ -492,6 +501,9 @@ pub(crate) async fn invite_user_route( body: Ruma, ) -> Result { let sender_user = body.sender_user(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } if !services.users.is_admin(sender_user).await && services.config.block_non_admin_invites { debug_error!( From e127c4e5a2e08498c61df30b8424a01f30671bca Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 20:46:22 +0100 Subject: [PATCH 1850/2091] feat: Add un/suspend admin commands --- src/admin/user/commands.rs | 34 ++++++++++++++++++++++++++++++++++ src/admin/user/mod.rs | 22 ++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index e5e481e5..297e4bb3 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -224,6 +224,40 @@ pub(super) async fn deactivate(&self, no_leave_rooms: bool, user_id: String) -> .await } +#[admin_command] +pub(super) async fn suspend(&self, user_id: String) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + + if user_id == self.services.globals.server_user { + return Err!("Not allowed to suspend the server service account.",); + } + + if !self.services.users.exists(&user_id).await { + return Err!("User {user_id} does not exist."); + } + self.services.users.suspend_account(&user_id).await; + + self.write_str(&format!("User {user_id} has been suspended.")) + .await +} + +#[admin_command] +pub(super) async fn unsuspend(&self, user_id: String) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + + if user_id == self.services.globals.server_user { + return Err!("Not allowed to unsuspend the server service account.",); + } + + if !self.services.users.exists(&user_id).await { + return Err!("User {user_id} does not exist."); + } + self.services.users.unsuspend_account(&user_id).await; + + self.write_str(&format!("User {user_id} has been unsuspended.")) + .await +} + #[admin_command] pub(super) async fn reset_password(&self, username: String, password: Option) -> Result { let user_id = parse_local_user_id(self.services, &username)?; diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index e789376a..645d3637 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -59,6 +59,28 @@ pub(super) enum UserCommand { force: bool, }, + /// - Suspend a user + /// + /// Suspended users are able to log in, sync, and read messages, but are not + /// able to send events nor redact them, cannot change their profile, and + /// are unable to join, invite to, or knock on rooms. + /// + /// Suspended users can still leave rooms and deactivate their account. + /// Suspending them effectively makes them read-only. + Suspend { + /// Username of the user to suspend + user_id: String, + }, + + /// - Unsuspend a user + /// + /// Reverses the effects of the `suspend` command, allowing the user to send + /// messages, change their profile, create room invites, etc. + Unsuspend { + /// Username of the user to unsuspend + user_id: String, + }, + /// - List local users in the database #[clap(alias = "list")] ListUsers, From 5d5350a9fee3b4959ef66b0d18a588d2f2be7dde Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 20:47:02 +0100 Subject: [PATCH 1851/2091] feat: Prevent suspended users creating new rooms --- src/api/client/room/create.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index be3fd23b..d1dffc51 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -70,6 +70,10 @@ pub(crate) async fn create_room_route( )); } + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + let room_id: OwnedRoomId = match &body.room_id { | Some(custom_room_id) => custom_room_id_check(&services, custom_room_id)?, | _ => RoomId::new(&services.server.name), From 968c0e236c37d204d9718689dd79888cf32f54e5 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 21:19:10 +0100 Subject: [PATCH 1852/2091] fix: Create the column appropriately --- src/database/maps.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/database/maps.rs b/src/database/maps.rs index 19f9ced4..91ba2ebe 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -378,6 +378,10 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "userid_password", ..descriptor::RANDOM }, + Descriptor { + name: "userid_suspended", + ..descriptor::RANDOM_SMALL + }, Descriptor { name: "userid_presenceid", ..descriptor::RANDOM_SMALL From 8791a9b851f8fc3cd827fe898406561fc4104287 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 21:19:37 +0100 Subject: [PATCH 1853/2091] fix: Inappropriate empty check I once again, assumed `true` is actually `false`. --- src/service/users/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 1a9f6600..6d3b662f 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -177,7 +177,7 @@ impl Service { self.db .userid_suspended .get(user_id) - .map_ok(|val| val.is_empty()) + .map_ok(|val| !val.is_empty()) .map_err(|_| err!(Request(NotFound("User does not exist.")))) .await } From cc864dc8bbb299b7a48c6fb963a61b6979ceb278 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 21:20:56 +0100 Subject: [PATCH 1854/2091] feat: Do not allow suspending admin users --- src/admin/user/commands.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 297e4bb3..61f10a86 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -235,6 +235,9 @@ pub(super) async fn suspend(&self, user_id: String) -> Result { if !self.services.users.exists(&user_id).await { return Err!("User {user_id} does not exist."); } + if self.services.users.is_admin(&user_id).await { + return Err!("Admin users cannot be suspended."); + } self.services.users.suspend_account(&user_id).await; self.write_str(&format!("User {user_id} has been suspended.")) From 1ff8af8e9e6ef723287dd9b174dc69844bf8631c Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 21:24:20 +0100 Subject: [PATCH 1855/2091] style: Remove unneeded statements (clippy) --- src/service/users/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 6d3b662f..f6a98858 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -15,7 +15,6 @@ use ruma::{ AnyToDeviceEvent, GlobalAccountDataEventType, ignored_user_list::IgnoredUserListEvent, }, serde::Raw, - uint, }; use serde_json::json; @@ -147,12 +146,12 @@ impl Service { } /// Suspend account, placing it in a read-only state - pub async fn suspend_account(&self, user_id: &UserId) -> () { + pub async fn suspend_account(&self, user_id: &UserId) { self.db.userid_suspended.insert(user_id, "1"); } /// Unsuspend account, placing it in a read-write state - pub async fn unsuspend_account(&self, user_id: &UserId) -> () { + pub async fn unsuspend_account(&self, user_id: &UserId) { self.db.userid_suspended.remove(user_id); } From d0548ec0644ebb0e3538f65088db9b16ce0eb5a8 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 21:30:07 +0100 Subject: [PATCH 1856/2091] feat: Forbid suspended users from sending state events --- src/api/client/state.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 2ddc8f14..07802b1b 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -33,6 +33,10 @@ pub(crate) async fn send_state_event_for_key_route( ) -> Result { let sender_user = body.sender_user(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + Ok(send_state_event::v3::Response { event_id: send_state_event_for_key_helper( &services, From 90180916eb13cdcf4e81c668fa6255222c781f1d Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 22:42:31 +0100 Subject: [PATCH 1857/2091] feat: Prevent suspended users performing room changes Prevents kicks, bans, unbans, and alias modification --- src/api/client/alias.rs | 6 ++++++ src/api/client/directory.rs | 3 +++ src/api/client/membership.rs | 16 ++++++++++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index 9f1b05f8..dc7aad44 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -18,6 +18,9 @@ pub(crate) async fn create_alias_route( body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } services .rooms @@ -63,6 +66,9 @@ pub(crate) async fn delete_alias_route( body: Ruma, ) -> Result { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } services .rooms diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index aa6ae168..2e219fd9 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -128,6 +128,9 @@ pub(crate) async fn set_room_visibility_route( // Return 404 if the room doesn't exist return Err!(Request(NotFound("Room not found"))); } + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } if services .users diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index d78ebdec..e6392533 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -578,6 +578,10 @@ pub(crate) async fn kick_user_route( State(services): State, body: Ruma, ) -> Result { + let sender_user = body.sender_user(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; let Ok(event) = services @@ -613,7 +617,7 @@ pub(crate) async fn kick_user_route( third_party_invite: None, ..event }), - body.sender_user(), + sender_user, &body.room_id, &state_lock, ) @@ -637,6 +641,10 @@ pub(crate) async fn ban_user_route( return Err!(Request(Forbidden("You cannot ban yourself."))); } + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; let current_member_content = services @@ -679,6 +687,10 @@ pub(crate) async fn unban_user_route( State(services): State, body: Ruma, ) -> Result { + let sender_user = body.sender_user(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; let current_member_content = services @@ -707,7 +719,7 @@ pub(crate) async fn unban_user_route( is_direct: None, ..current_member_content }), - body.sender_user(), + sender_user, &body.room_id, &state_lock, ) From 8e06571e7c7e88c14580c7e6206aecdb12a5eb44 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 22:42:49 +0100 Subject: [PATCH 1858/2091] feat: Prevent suspended users uploading media --- src/api/client/media.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 94572413..11d5450c 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -52,6 +52,9 @@ pub(crate) async fn create_content_route( body: Ruma, ) -> Result { let user = body.sender_user.as_ref().expect("user is authenticated"); + if services.users.is_suspended(user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } let filename = body.filename.as_deref(); let content_type = body.content_type.as_deref(); From 08527a28804f673817793af86ea94086fcba3a5c Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 22:43:35 +0100 Subject: [PATCH 1859/2091] feat: Prevent suspended users upgrading rooms --- src/api/client/room/upgrade.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index da5b49fe..d8f5ea83 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -2,7 +2,7 @@ use std::cmp::max; use axum::extract::State; use conduwuit::{ - Error, Result, err, info, + Err, Error, Result, err, info, matrix::{StateKey, pdu::PduBuilder}, }; use futures::StreamExt; @@ -63,6 +63,10 @@ pub(crate) async fn upgrade_room_route( )); } + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + // Create a replacement room let replacement_room = RoomId::new(services.globals.server_name()); From 1124097bd17eea3d10b35b9f05539321050b248a Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 22:52:20 +0100 Subject: [PATCH 1860/2091] feat: Only allow private read receipts when suspended --- src/api/client/read_marker.rs | 49 +++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index fbfc8fea..e152869c 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -58,29 +58,34 @@ pub(crate) async fn set_read_marker_route( } if let Some(event) = &body.read_receipt { - let receipt_content = BTreeMap::from_iter([( - event.to_owned(), - BTreeMap::from_iter([( - ReceiptType::Read, - BTreeMap::from_iter([(sender_user.to_owned(), ruma::events::receipt::Receipt { - ts: Some(MilliSecondsSinceUnixEpoch::now()), - thread: ReceiptThread::Unthreaded, - })]), - )]), - )]); + if !services.users.is_suspended(sender_user).await? { + let receipt_content = BTreeMap::from_iter([( + event.to_owned(), + BTreeMap::from_iter([( + ReceiptType::Read, + BTreeMap::from_iter([( + sender_user.to_owned(), + ruma::events::receipt::Receipt { + ts: Some(MilliSecondsSinceUnixEpoch::now()), + thread: ReceiptThread::Unthreaded, + }, + )]), + )]), + )]); - services - .rooms - .read_receipt - .readreceipt_update( - sender_user, - &body.room_id, - &ruma::events::receipt::ReceiptEvent { - content: ruma::events::receipt::ReceiptEventContent(receipt_content), - room_id: body.room_id.clone(), - }, - ) - .await; + services + .rooms + .read_receipt + .readreceipt_update( + sender_user, + &body.room_id, + &ruma::events::receipt::ReceiptEvent { + content: ruma::events::receipt::ReceiptEventContent(receipt_content), + room_id: body.room_id.clone(), + }, + ) + .await; + } } if let Some(event) = &body.private_read_receipt { From 72f8cb30384a32b1489b676fcfc76fc59f323c42 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 28 Jun 2025 22:53:25 +0100 Subject: [PATCH 1861/2091] feat: Do not allow suspended users to send typing statuses --- src/api/client/typing.rs | 67 ++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/src/api/client/typing.rs b/src/api/client/typing.rs index 1d8d02fd..7b0df538 100644 --- a/src/api/client/typing.rs +++ b/src/api/client/typing.rs @@ -26,41 +26,42 @@ pub(crate) async fn create_typing_event_route( { return Err!(Request(Forbidden("You are not in this room."))); } - - match body.state { - | Typing::Yes(duration) => { - let duration = utils::clamp( - duration.as_millis().try_into().unwrap_or(u64::MAX), + if !services.users.is_suspended(sender_user).await? { + match body.state { + | Typing::Yes(duration) => { + let duration = utils::clamp( + duration.as_millis().try_into().unwrap_or(u64::MAX), + services + .server + .config + .typing_client_timeout_min_s + .try_mul(1000)?, + services + .server + .config + .typing_client_timeout_max_s + .try_mul(1000)?, + ); services - .server - .config - .typing_client_timeout_min_s - .try_mul(1000)?, + .rooms + .typing + .typing_add( + sender_user, + &body.room_id, + utils::millis_since_unix_epoch() + .checked_add(duration) + .expect("user typing timeout should not get this high"), + ) + .await?; + }, + | _ => { services - .server - .config - .typing_client_timeout_max_s - .try_mul(1000)?, - ); - services - .rooms - .typing - .typing_add( - sender_user, - &body.room_id, - utils::millis_since_unix_epoch() - .checked_add(duration) - .expect("user typing timeout should not get this high"), - ) - .await?; - }, - | _ => { - services - .rooms - .typing - .typing_remove(sender_user, &body.room_id) - .await?; - }, + .rooms + .typing + .typing_remove(sender_user, &body.room_id) + .await?; + }, + } } // ping presence From eb2e3b3bb70af18ad0821d046fe51926977b5d0a Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 29 Jun 2025 01:52:02 +0100 Subject: [PATCH 1862/2091] fix: Missing suspensions shouldn't error Turns out copying and pasting the function above verbatim actually introduces more problems than it solves! --- src/service/users/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index f6a98858..d80a7e22 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -2,7 +2,7 @@ use std::{collections::BTreeMap, mem, sync::Arc}; use conduwuit::{ Err, Error, Result, Server, at, debug_warn, err, trace, - utils::{self, ReadyExt, stream::TryIgnore, string::Unquoted}, + utils::{self, ReadyExt, TryFutureExtExt, stream::TryIgnore, string::Unquoted}, }; use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::{Stream, StreamExt, TryFutureExt}; @@ -176,8 +176,7 @@ impl Service { self.db .userid_suspended .get(user_id) - .map_ok(|val| !val.is_empty()) - .map_err(|_| err!(Request(NotFound("User does not exist.")))) + .map_ok_or(Ok(false), |_| Ok(true)) .await } From d8a27eeb54b79a1126dd08c48f8f20d9cc0c0104 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 29 Jun 2025 02:28:04 +0100 Subject: [PATCH 1863/2091] fix: Failing open on database errors oops --- src/service/users/mod.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index d80a7e22..5db7dc1d 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,11 +1,11 @@ use std::{collections::BTreeMap, mem, sync::Arc}; use conduwuit::{ - Err, Error, Result, Server, at, debug_warn, err, trace, + Err, Error, Result, Server, at, debug_warn, err, result, trace, utils::{self, ReadyExt, TryFutureExtExt, stream::TryIgnore, string::Unquoted}, }; use database::{Deserialized, Ignore, Interfix, Json, Map}; -use futures::{Stream, StreamExt, TryFutureExt}; +use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt}; use ruma::{ DeviceId, KeyId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, OneTimeKeyName, OwnedDeviceId, OwnedKeyId, OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, @@ -176,7 +176,19 @@ impl Service { self.db .userid_suspended .get(user_id) - .map_ok_or(Ok(false), |_| Ok(true)) + .map_ok_or_else( + |err| { + if err.is_not_found() { + Ok(false) + } else { + err!(Database(error!( + "Failed to check if user {user_id} is suspended: {err}" + ))); + Ok(true) + } + }, + |_| Ok(true), + ) .await } From 13e17d52e02f33847bed089d4451d248694be854 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 29 Jun 2025 02:30:52 +0100 Subject: [PATCH 1864/2091] style: Remove unnecessary imports (clippy) --- src/service/users/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 5db7dc1d..b2a42959 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,11 +1,11 @@ use std::{collections::BTreeMap, mem, sync::Arc}; use conduwuit::{ - Err, Error, Result, Server, at, debug_warn, err, result, trace, - utils::{self, ReadyExt, TryFutureExtExt, stream::TryIgnore, string::Unquoted}, + Err, Error, Result, Server, at, debug_warn, err, trace, + utils::{self, ReadyExt, stream::TryIgnore, string::Unquoted}, }; use database::{Deserialized, Ignore, Interfix, Json, Map}; -use futures::{Stream, StreamExt, TryFutureExt, TryStreamExt}; +use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{ DeviceId, KeyId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, OneTimeKeyName, OwnedDeviceId, OwnedKeyId, OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, From ecc6fda98b135e9e5fdb9cd93124baac7a4fc001 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 29 Jun 2025 15:07:04 +0100 Subject: [PATCH 1865/2091] feat: Record metadata about user suspensions --- src/admin/user/commands.rs | 6 +++- src/database/maps.rs | 2 +- src/service/users/mod.rs | 56 ++++++++++++++++++++++++-------------- 3 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 61f10a86..ad2d1c78 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -238,7 +238,11 @@ pub(super) async fn suspend(&self, user_id: String) -> Result { if self.services.users.is_admin(&user_id).await { return Err!("Admin users cannot be suspended."); } - self.services.users.suspend_account(&user_id).await; + // TODO: Record the actual user that sent the suspension where possible + self.services + .users + .suspend_account(&user_id, self.services.globals.server_user.as_ref()) + .await; self.write_str(&format!("User {user_id} has been suspended.")) .await diff --git a/src/database/maps.rs b/src/database/maps.rs index 91ba2ebe..214dbf34 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -379,7 +379,7 @@ pub(super) static MAPS: &[Descriptor] = &[ ..descriptor::RANDOM }, Descriptor { - name: "userid_suspended", + name: "userid_suspension", ..descriptor::RANDOM_SMALL }, Descriptor { diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index b2a42959..d2dfccd9 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -16,10 +16,21 @@ use ruma::{ }, serde::Raw, }; +use serde::{Deserialize, Serialize}; use serde_json::json; use crate::{Dep, account_data, admin, globals, rooms}; +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UserSuspension { + /// Whether the user is currently suspended + pub suspended: bool, + /// When the user was suspended (Unix timestamp in milliseconds) + pub suspended_at: u64, + /// User ID of who suspended this user + pub suspended_by: String, +} + pub struct Service { services: Services, db: Data, @@ -52,7 +63,7 @@ struct Data { userid_lastonetimekeyupdate: Arc, userid_masterkeyid: Arc, userid_password: Arc, - userid_suspended: Arc, + userid_suspension: Arc, userid_selfsigningkeyid: Arc, userid_usersigningkeyid: Arc, useridprofilekey_value: Arc, @@ -88,7 +99,7 @@ impl crate::Service for Service { userid_lastonetimekeyupdate: args.db["userid_lastonetimekeyupdate"].clone(), userid_masterkeyid: args.db["userid_masterkeyid"].clone(), userid_password: args.db["userid_password"].clone(), - userid_suspended: args.db["userid_suspended"].clone(), + userid_suspension: args.db["userid_suspension"].clone(), userid_selfsigningkeyid: args.db["userid_selfsigningkeyid"].clone(), userid_usersigningkeyid: args.db["userid_usersigningkeyid"].clone(), useridprofilekey_value: args.db["useridprofilekey_value"].clone(), @@ -146,13 +157,20 @@ impl Service { } /// Suspend account, placing it in a read-only state - pub async fn suspend_account(&self, user_id: &UserId) { - self.db.userid_suspended.insert(user_id, "1"); + pub async fn suspend_account(&self, user_id: &UserId, suspending_user: &UserId) { + self.db.userid_suspension.raw_put( + user_id, + Json(UserSuspension { + suspended: true, + suspended_at: MilliSecondsSinceUnixEpoch::now().get().into(), + suspended_by: suspending_user.to_string(), + }), + ); } /// Unsuspend account, placing it in a read-write state pub async fn unsuspend_account(&self, user_id: &UserId) { - self.db.userid_suspended.remove(user_id); + self.db.userid_suspension.remove(user_id); } /// Check if a user has an account on this homeserver. @@ -173,23 +191,21 @@ impl Service { /// Check if account is suspended pub async fn is_suspended(&self, user_id: &UserId) -> Result { - self.db - .userid_suspended + match self + .db + .userid_suspension .get(user_id) - .map_ok_or_else( - |err| { - if err.is_not_found() { - Ok(false) - } else { - err!(Database(error!( - "Failed to check if user {user_id} is suspended: {err}" - ))); - Ok(true) - } - }, - |_| Ok(true), - ) .await + .deserialized::() + { + | Ok(s) => Ok(s.suspended), + | Err(e) => + if e.is_not_found() { + Ok(false) + } else { + Err(e) + }, + } } /// Check if account is active, infallible From acb74faa070801c4ee48cc76f6d3dd19174876b9 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 29 Jun 2025 15:17:27 +0100 Subject: [PATCH 1866/2091] feat: Pass sender through admin commands --- src/admin/context.rs | 15 ++++++++++++++- src/admin/processor.rs | 1 + src/admin/user/commands.rs | 2 +- src/service/admin/mod.rs | 23 ++++++++++++++++++++--- src/service/rooms/timeline/mod.rs | 8 +++++--- 5 files changed, 41 insertions(+), 8 deletions(-) diff --git a/src/admin/context.rs b/src/admin/context.rs index 270537be..b453d88f 100644 --- a/src/admin/context.rs +++ b/src/admin/context.rs @@ -7,13 +7,14 @@ use futures::{ io::{AsyncWriteExt, BufWriter}, lock::Mutex, }; -use ruma::EventId; +use ruma::{EventId, UserId}; pub(crate) struct Context<'a> { pub(crate) services: &'a Services, pub(crate) body: &'a [&'a str], pub(crate) timer: SystemTime, pub(crate) reply_id: Option<&'a EventId>, + pub(crate) sender: Option<&'a UserId>, pub(crate) output: Mutex>>, } @@ -36,4 +37,16 @@ impl Context<'_> { output.write_all(s.as_bytes()).map_err(Into::into).await }) } + + /// Get the sender of the admin command, if available + pub(crate) fn sender(&self) -> Option<&UserId> { self.sender } + + /// Check if the command has sender information + pub(crate) fn has_sender(&self) -> bool { self.sender.is_some() } + + /// Get the sender as a string, or service user ID if not available + pub(crate) fn sender_or_service_user(&self) -> &UserId { + self.sender + .unwrap_or_else(|| self.services.globals.server_user.as_ref()) + } } diff --git a/src/admin/processor.rs b/src/admin/processor.rs index f7b7140f..8d1fe89c 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -63,6 +63,7 @@ async fn process_command(services: Arc, input: &CommandInput) -> Proce body: &body, timer: SystemTime::now(), reply_id: input.reply_id.as_deref(), + sender: input.sender.as_deref(), output: BufWriter::new(Vec::new()).into(), }; diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index ad2d1c78..d094fc5f 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -241,7 +241,7 @@ pub(super) async fn suspend(&self, user_id: String) -> Result { // TODO: Record the actual user that sent the suspension where possible self.services .users - .suspend_account(&user_id, self.services.globals.server_user.as_ref()) + .suspend_account(&user_id, self.sender_or_service_user()) .await; self.write_str(&format!("User {user_id} has been suspended.")) diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 683f5400..86e12c3c 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -45,11 +45,13 @@ struct Services { services: StdRwLock>>, } -/// Inputs to a command are a multi-line string and optional reply_id. +/// Inputs to a command are a multi-line string, optional reply_id, and optional +/// sender. #[derive(Debug)] pub struct CommandInput { pub command: String, pub reply_id: Option, + pub sender: Option>, } /// Prototype of the tab-completer. The input is buffered text when tab @@ -162,7 +164,22 @@ impl Service { pub fn command(&self, command: String, reply_id: Option) -> Result<()> { self.channel .0 - .send(CommandInput { command, reply_id }) + .send(CommandInput { command, reply_id, sender: None }) + .map_err(|e| err!("Failed to enqueue admin command: {e:?}")) + } + + /// Posts a command to the command processor queue with sender information + /// and returns. Processing will take place on the service worker's task + /// asynchronously. Errors if the queue is full. + pub fn command_with_sender( + &self, + command: String, + reply_id: Option, + sender: Box, + ) -> Result<()> { + self.channel + .0 + .send(CommandInput { command, reply_id, sender: Some(sender) }) .map_err(|e| err!("Failed to enqueue admin command: {e:?}")) } @@ -173,7 +190,7 @@ impl Service { command: String, reply_id: Option, ) -> ProcessorResult { - self.process_command(CommandInput { command, reply_id }) + self.process_command(CommandInput { command, reply_id, sender: None }) .await } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 37963246..534d8faf 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -536,9 +536,11 @@ impl Service { self.services.search.index_pdu(shortroomid, &pdu_id, &body); if self.services.admin.is_admin_command(pdu, &body).await { - self.services - .admin - .command(body, Some((*pdu.event_id).into()))?; + self.services.admin.command_with_sender( + body, + Some((*pdu.event_id).into()), + pdu.sender.clone().into(), + )?; } } }, From d4862b8ead02a6ef61580ddefb24febb56c108b4 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 29 Jun 2025 16:26:04 +0100 Subject: [PATCH 1867/2091] style: Remove redundant, unused functions --- src/admin/context.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/admin/context.rs b/src/admin/context.rs index b453d88f..3d3cffb7 100644 --- a/src/admin/context.rs +++ b/src/admin/context.rs @@ -38,12 +38,6 @@ impl Context<'_> { }) } - /// Get the sender of the admin command, if available - pub(crate) fn sender(&self) -> Option<&UserId> { self.sender } - - /// Check if the command has sender information - pub(crate) fn has_sender(&self) -> bool { self.sender.is_some() } - /// Get the sender as a string, or service user ID if not available pub(crate) fn sender_or_service_user(&self) -> &UserId { self.sender From ec9d3d613e3b3be018ac73c791c956ae44cc6611 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 29 Jun 2025 23:02:15 +0100 Subject: [PATCH 1868/2091] chore: Add funding --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..321dc74c --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +github: [JadedBlueEyes] +# Doesn't support an array, so we can only list nex +ko_fi: nexy7574 From 17930708d86dd73c5d39dbe758d489621b2626e5 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 29 Jun 2025 23:06:26 +0100 Subject: [PATCH 1869/2091] chore: Add second ko-fi as custom link --- .github/FUNDING.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 321dc74c..fcfaade5 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,3 +1,5 @@ github: [JadedBlueEyes] # Doesn't support an array, so we can only list nex ko_fi: nexy7574 +custom: + - https://ko-fi.com/JadedBlueEyes From 97e5cc4e2db5fee0495d8f597180fe774b11aa0d Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 1 Jul 2025 01:55:13 +0100 Subject: [PATCH 1870/2091] feat: Implement user reporting --- Cargo.lock | 22 ++++++------ Cargo.toml | 2 +- src/api/client/report.rs | 72 ++++++++++++++++++++++++++++++++-------- src/api/router.rs | 1 + 4 files changed, 71 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 92044b92..8719c6ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3798,7 +3798,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "assign", "js_int", @@ -3818,7 +3818,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "js_int", "ruma-common", @@ -3830,7 +3830,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "as_variant", "assign", @@ -3853,7 +3853,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "as_variant", "base64 0.22.1", @@ -3885,7 +3885,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "as_variant", "indexmap 2.9.0", @@ -3910,7 +3910,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "bytes", "headers", @@ -3932,7 +3932,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3941,7 +3941,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "js_int", "ruma-common", @@ -3951,7 +3951,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3966,7 +3966,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "js_int", "ruma-common", @@ -3978,7 +3978,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=9b65f83981f6f53d185ce77da37aaef9dfd764a9#9b65f83981f6f53d185ce77da37aaef9dfd764a9" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 5c289adf..83afd482 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,7 +350,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://forgejo.ellis.link/continuwuation/ruwuma" #branch = "conduwuit-changes" -rev = "9b65f83981f6f53d185ce77da37aaef9dfd764a9" +rev = "f899fff6738dd57d191474b0f12a4509cf8f0981" features = [ "compat", "rand", diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 4ee8ebe5..f63e78ed 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -9,6 +9,7 @@ use ruma::{ EventId, RoomId, UserId, api::client::{ error::ErrorKind, + report_user, room::{report_content, report_room}, }, events::room::message, @@ -30,12 +31,6 @@ pub(crate) async fn report_room_route( // user authentication let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - info!( - "Received room report by user {sender_user} for room {} with reason: \"{}\"", - body.room_id, - body.reason.as_deref().unwrap_or("") - ); - if body.reason.as_ref().is_some_and(|s| s.len() > 750) { return Err(Error::BadRequest( ErrorKind::InvalidParam, @@ -55,6 +50,11 @@ pub(crate) async fn report_room_route( "Room does not exist to us, no local users have joined at all" ))); } + info!( + "Received room report by user {sender_user} for room {} with reason: \"{}\"", + body.room_id, + body.reason.as_deref().unwrap_or("") + ); // send admin room message that we received the report with an @room ping for // urgency @@ -84,14 +84,6 @@ pub(crate) async fn report_event_route( // user authentication let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - info!( - "Received event report by user {sender_user} for room {} and event ID {}, with reason: \ - \"{}\"", - body.room_id, - body.event_id, - body.reason.as_deref().unwrap_or("") - ); - delay_response().await; // check if we know about the reported event ID or if it's invalid @@ -109,6 +101,13 @@ pub(crate) async fn report_event_route( &pdu, ) .await?; + info!( + "Received event report by user {sender_user} for room {} and event ID {}, with reason: \ + \"{}\"", + body.room_id, + body.event_id, + body.reason.as_deref().unwrap_or("") + ); // send admin room message that we received the report with an @room ping for // urgency @@ -130,6 +129,51 @@ pub(crate) async fn report_event_route( Ok(report_content::v3::Response {}) } +#[tracing::instrument(skip_all, fields(%client), name = "report_user")] +pub(crate) async fn report_user_route( + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + // user authentication + let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + + if body.reason.as_ref().is_some_and(|s| s.len() > 750) { + return Err(Error::BadRequest( + ErrorKind::InvalidParam, + "Reason too long, should be 750 characters or fewer", + )); + } + + delay_response().await; + + if !services.users.is_active_local(&body.user_id) { + // return 200 as to not reveal if the user exists. Recommended by spec. + return Ok(report_user::v3::Response {}); + } + + info!( + "Received room report from {sender_user} for user {} with reason: \"{}\"", + body.user_id, + body.reason.as_deref().unwrap_or("") + ); + + // send admin room message that we received the report with an @room ping for + // urgency + services + .admin + .send_message(message::RoomMessageEventContent::text_markdown(format!( + "@room User report received from {} -\n\nUser ID: {}\n\nReport Reason: {}", + sender_user.to_owned(), + body.user_id, + body.reason.as_deref().unwrap_or("") + ))) + .await + .ok(); + + Ok(report_user::v3::Response {}) +} + /// in the following order: /// /// check if the room ID from the URI matches the PDU's room ID diff --git a/src/api/router.rs b/src/api/router.rs index 5416e9e9..d1b05a91 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -94,6 +94,7 @@ pub fn build(router: Router, server: &Server) -> Router { .ruma_route(&client::redact_event_route) .ruma_route(&client::report_event_route) .ruma_route(&client::report_room_route) + .ruma_route(&client::report_user_route) .ruma_route(&client::create_alias_route) .ruma_route(&client::delete_alias_route) .ruma_route(&client::get_alias_route) From 59912709aa603d66bc24c074f0337c71fdeaa56c Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 1 Jul 2025 15:42:38 +0100 Subject: [PATCH 1871/2091] feat: Send intentional mentions in report messages --- src/api/client/report.rs | 125 +++++++++++++++++++++++++-------------- 1 file changed, 81 insertions(+), 44 deletions(-) diff --git a/src/api/client/report.rs b/src/api/client/report.rs index f63e78ed..1f237fcb 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -1,4 +1,7 @@ -use std::time::Duration; +use std::{ + ops::{Mul, Sub}, + time::Duration, +}; use axum::extract::State; use axum_client_ip::InsecureClientIp; @@ -6,19 +9,35 @@ use conduwuit::{Err, Error, Result, debug_info, info, matrix::pdu::PduEvent, uti use conduwuit_service::Services; use rand::Rng; use ruma::{ - EventId, RoomId, UserId, + EventId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId, api::client::{ error::ErrorKind, report_user, room::{report_content, report_room}, }, - events::room::message, + events::{ + Mentions, + room::{ + message, + message::{RoomMessageEvent, RoomMessageEventContent}, + }, + }, int, }; use tokio::time::sleep; use crate::Ruma; +struct Report { + sender: OwnedUserId, + room_id: Option, + event_id: Option, + user_id: Option, + report_type: String, + reason: Option, + score: Option, +} + /// # `POST /_matrix/client/v3/rooms/{roomId}/report` /// /// Reports an abusive room to homeserver admins @@ -56,18 +75,17 @@ pub(crate) async fn report_room_route( body.reason.as_deref().unwrap_or("") ); - // send admin room message that we received the report with an @room ping for - // urgency - services - .admin - .send_message(message::RoomMessageEventContent::text_markdown(format!( - "@room Room report received from {} -\n\nRoom ID: {}\n\nReport Reason: {}", - sender_user.to_owned(), - body.room_id, - body.reason.as_deref().unwrap_or("") - ))) - .await - .ok(); + let report = Report { + sender: sender_user.to_owned(), + room_id: Some(body.room_id.to_owned()), + event_id: None, + user_id: None, + report_type: "room".to_string(), + reason: body.reason.clone(), + score: None, + }; + + services.admin.send_message(build_report(report)).await.ok(); Ok(report_room::v3::Response {}) } @@ -108,23 +126,16 @@ pub(crate) async fn report_event_route( body.event_id, body.reason.as_deref().unwrap_or("") ); - - // send admin room message that we received the report with an @room ping for - // urgency - services - .admin - .send_message(message::RoomMessageEventContent::text_markdown(format!( - "@room Event report received from {} -\n\nEvent ID: {}\nRoom ID: {}\nSent By: \ - {}\n\nReport Score: {}\nReport Reason: {}", - sender_user.to_owned(), - pdu.event_id, - pdu.room_id, - pdu.sender, - body.score.unwrap_or_else(|| ruma::Int::from(0)), - body.reason.as_deref().unwrap_or("") - ))) - .await - .ok(); + let report = Report { + sender: sender_user.to_owned(), + room_id: Some(body.room_id.to_owned()), + event_id: Some(body.event_id.to_owned()), + user_id: None, + report_type: "event".to_string(), + reason: body.reason.clone(), + score: body.score, + }; + services.admin.send_message(build_report(report)).await.ok(); Ok(report_content::v3::Response {}) } @@ -152,24 +163,23 @@ pub(crate) async fn report_user_route( return Ok(report_user::v3::Response {}); } + let report = Report { + sender: sender_user.to_owned(), + room_id: None, + event_id: None, + user_id: Some(body.user_id.to_owned()), + report_type: "user".to_string(), + reason: body.reason.clone(), + score: None, + }; + info!( "Received room report from {sender_user} for user {} with reason: \"{}\"", body.user_id, body.reason.as_deref().unwrap_or("") ); - // send admin room message that we received the report with an @room ping for - // urgency - services - .admin - .send_message(message::RoomMessageEventContent::text_markdown(format!( - "@room User report received from {} -\n\nUser ID: {}\n\nReport Reason: {}", - sender_user.to_owned(), - body.user_id, - body.reason.as_deref().unwrap_or("") - ))) - .await - .ok(); + services.admin.send_message(build_report(report)).await.ok(); Ok(report_user::v3::Response {}) } @@ -231,6 +241,33 @@ async fn is_event_report_valid( Ok(()) } +/// Builds a report message to be sent to the admin room. +fn build_report(report: Report) -> RoomMessageEventContent { + let mut text = + format!("@room New {} report received from {}:\n\n", report.report_type, report.sender); + if report.user_id.is_some() { + text.push_str(&format!("- Reported User ID: `{}`\n", report.user_id.unwrap())); + } + if report.room_id.is_some() { + text.push_str(&format!("- Reported Room ID: `{}`\n", report.room_id.unwrap())); + } + if report.event_id.is_some() { + text.push_str(&format!("- Reported Event ID: `{}`\n", report.event_id.unwrap())); + } + if let Some(score) = report.score { + if score < int!(0) { + score.mul(int!(-1)); // invert the score to make it N/100 + // unsure why the spec says -100 to 0, but 0 to 100 is more human. + } + text.push_str(&format!("- User-supplied offensiveness score: {}%\n", -score)); + } + if let Some(reason) = report.reason { + text.push_str(&format!("- Report Reason: {}\n", reason)); + } + + RoomMessageEventContent::text_markdown(text).add_mentions(Mentions::with_room_mention()); +} + /// even though this is kinda security by obscurity, let's still make a small /// random delay sending a response per spec suggestion regarding /// enumerating for potential events existing in our server. From f49c73c0317e65b290bb7d50e81ae6a326e2bed6 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 1 Jul 2025 15:44:04 +0100 Subject: [PATCH 1872/2091] feat: Forbid suspended users from sending reports --- src/api/client/report.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 1f237fcb..d0d21829 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -49,6 +49,9 @@ pub(crate) async fn report_room_route( ) -> Result { // user authentication let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } if body.reason.as_ref().is_some_and(|s| s.len() > 750) { return Err(Error::BadRequest( @@ -101,6 +104,9 @@ pub(crate) async fn report_event_route( ) -> Result { // user authentication let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } delay_response().await; @@ -148,6 +154,9 @@ pub(crate) async fn report_user_route( ) -> Result { // user authentication let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } if body.reason.as_ref().is_some_and(|s| s.len() > 750) { return Err(Error::BadRequest( From 24d2a514e22930e15d27825063cb0a8e9244ddc9 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 1 Jul 2025 18:00:28 +0100 Subject: [PATCH 1873/2091] chore: Resolve linting errors --- Cargo.lock | 22 ++++++++--------- Cargo.toml | 2 +- src/api/client/report.rs | 45 +++++++++++++---------------------- src/api/client/unversioned.rs | 1 + 4 files changed, 29 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8719c6ca..82e7a20d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3798,7 +3798,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "assign", "js_int", @@ -3818,7 +3818,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "js_int", "ruma-common", @@ -3830,7 +3830,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "as_variant", "assign", @@ -3853,7 +3853,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "as_variant", "base64 0.22.1", @@ -3885,7 +3885,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "as_variant", "indexmap 2.9.0", @@ -3910,7 +3910,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "bytes", "headers", @@ -3932,7 +3932,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "js_int", "thiserror 2.0.12", @@ -3941,7 +3941,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "js_int", "ruma-common", @@ -3951,7 +3951,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "cfg-if", "proc-macro-crate", @@ -3966,7 +3966,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "js_int", "ruma-common", @@ -3978,7 +3978,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=f899fff6738dd57d191474b0f12a4509cf8f0981#f899fff6738dd57d191474b0f12a4509cf8f0981" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=a4b948b40417a65ab0282ae47cc50035dd455e02#a4b948b40417a65ab0282ae47cc50035dd455e02" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 83afd482..b815e2b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -350,7 +350,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://forgejo.ellis.link/continuwuation/ruwuma" #branch = "conduwuit-changes" -rev = "f899fff6738dd57d191474b0f12a4509cf8f0981" +rev = "a4b948b40417a65ab0282ae47cc50035dd455e02" features = [ "compat", "rand", diff --git a/src/api/client/report.rs b/src/api/client/report.rs index d0d21829..5113b42f 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -1,7 +1,4 @@ -use std::{ - ops::{Mul, Sub}, - time::Duration, -}; +use std::{fmt::Write as _, ops::Mul, time::Duration}; use axum::extract::State; use axum_client_ip::InsecureClientIp; @@ -15,13 +12,7 @@ use ruma::{ report_user, room::{report_content, report_room}, }, - events::{ - Mentions, - room::{ - message, - message::{RoomMessageEvent, RoomMessageEventContent}, - }, - }, + events::{Mentions, room::message::RoomMessageEventContent}, int, }; use tokio::time::sleep; @@ -80,10 +71,10 @@ pub(crate) async fn report_room_route( let report = Report { sender: sender_user.to_owned(), - room_id: Some(body.room_id.to_owned()), + room_id: Some(body.room_id.clone()), event_id: None, user_id: None, - report_type: "room".to_string(), + report_type: "room".to_owned(), reason: body.reason.clone(), score: None, }; @@ -134,10 +125,10 @@ pub(crate) async fn report_event_route( ); let report = Report { sender: sender_user.to_owned(), - room_id: Some(body.room_id.to_owned()), - event_id: Some(body.event_id.to_owned()), + room_id: Some(body.room_id.clone()), + event_id: Some(body.event_id.clone()), user_id: None, - report_type: "event".to_string(), + report_type: "event".to_owned(), reason: body.reason.clone(), score: body.score, }; @@ -167,7 +158,7 @@ pub(crate) async fn report_user_route( delay_response().await; - if !services.users.is_active_local(&body.user_id) { + if !services.users.is_active_local(&body.user_id).await { // return 200 as to not reveal if the user exists. Recommended by spec. return Ok(report_user::v3::Response {}); } @@ -176,8 +167,8 @@ pub(crate) async fn report_user_route( sender: sender_user.to_owned(), room_id: None, event_id: None, - user_id: Some(body.user_id.to_owned()), - report_type: "user".to_string(), + user_id: Some(body.user_id.clone()), + report_type: "user".to_owned(), reason: body.reason.clone(), score: None, }; @@ -255,26 +246,22 @@ fn build_report(report: Report) -> RoomMessageEventContent { let mut text = format!("@room New {} report received from {}:\n\n", report.report_type, report.sender); if report.user_id.is_some() { - text.push_str(&format!("- Reported User ID: `{}`\n", report.user_id.unwrap())); + let _ = writeln!(text, "- Reported User ID: `{}`", report.user_id.unwrap()); } if report.room_id.is_some() { - text.push_str(&format!("- Reported Room ID: `{}`\n", report.room_id.unwrap())); + let _ = writeln!(text, "- Reported Room ID: `{}`", report.room_id.unwrap()); } if report.event_id.is_some() { - text.push_str(&format!("- Reported Event ID: `{}`\n", report.event_id.unwrap())); + let _ = writeln!(text, "- Reported Event ID: `{}`", report.event_id.unwrap()); } if let Some(score) = report.score { - if score < int!(0) { - score.mul(int!(-1)); // invert the score to make it N/100 - // unsure why the spec says -100 to 0, but 0 to 100 is more human. - } - text.push_str(&format!("- User-supplied offensiveness score: {}%\n", -score)); + let _ = writeln!(text, "- User-supplied offensiveness score: {}%", score.mul(int!(-1))); } if let Some(reason) = report.reason { - text.push_str(&format!("- Report Reason: {}\n", reason)); + let _ = writeln!(text, "- Report Reason: {reason}"); } - RoomMessageEventContent::text_markdown(text).add_mentions(Mentions::with_room_mention()); + RoomMessageEventContent::text_markdown(text).add_mentions(Mentions::with_room_mention()) } /// even though this is kinda security by obscurity, let's still make a small diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index 232d5b28..ad377ca4 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -38,6 +38,7 @@ pub(crate) async fn get_supported_versions_route( "v1.4".to_owned(), "v1.5".to_owned(), "v1.11".to_owned(), + "v1.14".to_owned(), ], unstable_features: BTreeMap::from_iter([ ("org.matrix.e2e_cross_signing".to_owned(), true), From 4f69da47c6899e5d6e3280c34524e6aa53f838f4 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 1 Jul 2025 18:38:48 +0100 Subject: [PATCH 1874/2091] feat: Advertise support for spec v1.8, 1.12, 1.13, and 1.14 --- committed.toml | 1 + src/api/client/unversioned.rs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/committed.toml b/committed.toml index 64f7f18a..59750fa5 100644 --- a/committed.toml +++ b/committed.toml @@ -1,2 +1,3 @@ style = "conventional" +subject_length = 72 allowed_types = ["ci", "build", "fix", "feat", "chore", "docs", "style", "refactor", "perf", "test"] diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index ad377ca4..a4136d1a 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -37,7 +37,10 @@ pub(crate) async fn get_supported_versions_route( "v1.3".to_owned(), "v1.4".to_owned(), "v1.5".to_owned(), + "v1.8".to_owned(), "v1.11".to_owned(), + "v1.12".to_owned(), + "v1.13".to_owned(), "v1.14".to_owned(), ], unstable_features: BTreeMap::from_iter([ From b44791799c8141d93cf50634eb799505cae8f15a Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 1 Jul 2025 23:14:41 +0100 Subject: [PATCH 1875/2091] fix: Room bans preventing federated leaves Fixes the issue where room bans prevent federating leave events, resulting in local users being stuck in remote rooms --- src/admin/room/moderation.rs | 62 +++++++++++------------------------- 1 file changed, 18 insertions(+), 44 deletions(-) diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index ee429fc6..921249bd 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -1,7 +1,7 @@ use api::client::leave_room; use clap::Subcommand; use conduwuit::{ - Err, Result, debug, + Err, Result, debug, info, utils::{IterStream, ReadyExt}, warn, }; @@ -70,7 +70,6 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { }; debug!("Room specified is a room ID, banning room ID"); - self.services.rooms.metadata.ban_room(room_id, true); room_id.to_owned() } else if room.is_room_alias_id() { @@ -90,47 +89,25 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { locally, if not using get_alias_helper to fetch room ID remotely" ); - let room_id = match self + match self .services .rooms .alias - .resolve_local_alias(room_alias) + .resolve_alias(room_alias, None) .await { - | Ok(room_id) => room_id, - | _ => { + | Ok((room_id, servers)) => { debug!( - "We don't have this room alias to a room ID locally, attempting to fetch \ - room ID over federation" + ?room_id, + ?servers, + "Got federation response fetching room ID for room {room}" ); - - match self - .services - .rooms - .alias - .resolve_alias(room_alias, None) - .await - { - | Ok((room_id, servers)) => { - debug!( - ?room_id, - ?servers, - "Got federation response fetching room ID for {room_id}" - ); - room_id - }, - | Err(e) => { - return Err!( - "Failed to resolve room alias {room_alias} to a room ID: {e}" - ); - }, - } + room_id }, - }; - - self.services.rooms.metadata.ban_room(&room_id, true); - - room_id + | Err(e) => { + return Err!("Failed to resolve room alias {room} to a room ID: {e}"); + }, + } } else { return Err!( "Room specified is not a room ID or room alias. Please note that this requires a \ @@ -139,7 +116,7 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { ); }; - debug!("Making all users leave the room {room_id} and forgetting it"); + info!("Making all users leave the room {room_id} and forgetting it"); let mut users = self .services .rooms @@ -150,7 +127,7 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { .boxed(); while let Some(ref user_id) = users.next().await { - debug!( + info!( "Attempting leave for user {user_id} in room {room_id} (ignoring all errors, \ evicting admins too)", ); @@ -177,10 +154,9 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { }) .await; - // unpublish from room directory - self.services.rooms.directory.set_not_public(&room_id); - - self.services.rooms.metadata.disable_room(&room_id, true); + self.services.rooms.directory.set_not_public(&room_id); // remove from the room directory + self.services.rooms.metadata.ban_room(&room_id, true); // prevent further joins + self.services.rooms.metadata.disable_room(&room_id, true); // disable federation self.write_str( "Room banned, removed all our local users, and disabled incoming federation with room.", @@ -302,8 +278,6 @@ async fn ban_list_of_rooms(&self) -> Result { } for room_id in room_ids { - self.services.rooms.metadata.ban_room(&room_id, true); - debug!("Banned {room_id} successfully"); room_ban_count = room_ban_count.saturating_add(1); @@ -346,9 +320,9 @@ async fn ban_list_of_rooms(&self) -> Result { }) .await; + self.services.rooms.metadata.ban_room(&room_id, true); // unpublish from room directory, ignore errors self.services.rooms.directory.set_not_public(&room_id); - self.services.rooms.metadata.disable_room(&room_id, true); } From 68afb07c27368c0ec4d9fc936ef0cf39eb2233be Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 1 Jul 2025 01:36:58 +0100 Subject: [PATCH 1876/2091] feat: Stabilise room summary API (MSC3266) # Conflicts: # Cargo.lock # Cargo.toml --- src/api/client/room/summary.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/api/client/room/summary.rs b/src/api/client/room/summary.rs index 67d2e2ad..ab534765 100644 --- a/src/api/client/room/summary.rs +++ b/src/api/client/room/summary.rs @@ -43,10 +43,9 @@ pub(crate) async fn get_room_summary_legacy( } /// # `GET /_matrix/client/unstable/im.nheko.summary/summary/{roomIdOrAlias}` +/// # `GET /_matrix/client/v1/room_summary/{roomIdOrAlias}` /// /// Returns a short description of the state of a room. -/// -/// An implementation of [MSC3266](https://github.com/matrix-org/matrix-spec-proposals/pull/3266) #[tracing::instrument(skip_all, fields(%client), name = "room_summary")] pub(crate) async fn get_room_summary( State(services): State, From 6e609185847023cfb3caf85b98f03a6e3efb6e2e Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 2 Jul 2025 00:41:34 +0100 Subject: [PATCH 1877/2091] feat: Suspend new users on registration --- conduwuit-example.toml | 11 +++++++++++ src/api/client/account.rs | 20 ++++++++++++++++++++ src/core/config/mod.rs | 11 +++++++++++ src/service/admin/mod.rs | 16 +++++++++++++++- 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 1a8be2aa..794ab870 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -398,6 +398,17 @@ # #allow_registration = false +# If registration is enabled, and this setting is true, new users +# registered after the first admin user will be automatically suspended +# and will require an admin to run `!admin users unsuspend `. +# +# Suspended users are still able to read messages, make profile updates, +# leave rooms, and deactivate their account, however cannot send messages, +# invites, or create/join or otherwise modify rooms. +# They are effectively read-only. +# +#suspend_on_register = false + # Enabling this setting opens registration to anyone without restrictions. # This makes your server vulnerable to abuse # diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 32f2530c..32ac7bc2 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -2,6 +2,7 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; +use axum_extra::headers::UserAgent; use conduwuit::{ Err, Error, Result, debug_info, err, error, info, is_equal_to, matrix::pdu::PduBuilder, @@ -490,6 +491,25 @@ pub(crate) async fn register_route( { services.admin.make_user_admin(&user_id).await?; warn!("Granting {user_id} admin privileges as the first user"); + } else if services.config.suspend_on_register { + // This is not an admin, suspend them. + // Note that we can still do auto joins for suspended users + services + .users + .suspend_account(&user_id, &services.globals.server_user) + .await; + // And send an @room notice to the admin room, to prompt admins to review the + // new user and ideally unsuspend them if deemed appropriate. + if services.server.config.admin_room_notices { + services + .admin + .send_loud_message(RoomMessageEventContent::text_plain(format!( + "User {user_id} has been suspended as they are not the first user \ + on this server. Please review and unsuspend them if appropriate." + ))) + .await + .ok(); + } } } } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index d4a10345..c735193b 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -513,6 +513,17 @@ pub struct Config { #[serde(default)] pub allow_registration: bool, + /// If registration is enabled, and this setting is true, new users + /// registered after the first admin user will be automatically suspended + /// and will require an admin to run `!admin users unsuspend `. + /// + /// Suspended users are still able to read messages, make profile updates, + /// leave rooms, and deactivate their account, however cannot send messages, + /// invites, or create/join or otherwise modify rooms. + /// They are effectively read-only. + #[serde(default)] + pub suspend_on_register: bool, + /// Enabling this setting opens registration to anyone without restrictions. /// This makes your server vulnerable to abuse #[serde(default)] diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 86e12c3c..11d93cc2 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -18,7 +18,10 @@ use futures::{FutureExt, TryFutureExt}; use loole::{Receiver, Sender}; use ruma::{ OwnedEventId, OwnedRoomId, RoomId, UserId, - events::room::message::{Relation, RoomMessageEventContent}, + events::{ + Mentions, + room::message::{Relation, RoomMessageEventContent}, + }, }; use tokio::sync::RwLock; @@ -158,6 +161,17 @@ impl Service { .await } + /// Sends a message, the same as send_message() but with an @room ping to + /// notify all users in the room. + pub async fn send_loud_message( + &self, + mut message_content: RoomMessageEventContent, + ) -> Result<()> { + // Add @room ping + message_content = message_content.add_mentions(Mentions::with_room_mention()); + self.send_message(message_content).await + } + /// Posts a command to the command processor queue and returns. Processing /// will take place on the service worker's task asynchronously. Errors if /// the queue is full. From 8e0852e5b5697bb7d39639719c7d7e3f969e9c42 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 2 Jul 2025 00:44:47 +0100 Subject: [PATCH 1878/2091] docs: Add suggestion about auto join room Adds suggestion to suspend_on_register doc that admins should add a room that contains information to their auto_join_rooms as to not confuse new users who may be lost at the fact they can't join any rooms or send any messages. --- src/core/config/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index c735193b..e3db4900 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -521,6 +521,11 @@ pub struct Config { /// leave rooms, and deactivate their account, however cannot send messages, /// invites, or create/join or otherwise modify rooms. /// They are effectively read-only. + /// + /// If you want to use this to screen people who register on your server, + /// you should add a room to `auto_join_rooms` that is public, and contains + /// information that new users can read (since they won't be able to DM + /// anyone, or send a message, and may be confused). #[serde(default)] pub suspend_on_register: bool, From d6aa03ea7308fae430dd99df1d1ce4f5360a1d12 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 2 Jul 2025 01:53:21 +0100 Subject: [PATCH 1879/2091] style: Remove extraneous import --- src/api/client/account.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 32ac7bc2..56b1a81c 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -2,7 +2,6 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use axum_extra::headers::UserAgent; use conduwuit::{ Err, Error, Result, debug_info, err, error, info, is_equal_to, matrix::pdu::PduBuilder, From 49f7a2487f70ffcfeb0b355e8625a8a694e7ca6c Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 26 Apr 2025 23:06:43 +0000 Subject: [PATCH 1880/2091] Modernize various sender_user/sender_device lets. Signed-off-by: Jason Volk --- src/api/client/account.rs | 27 +++++++++++---------------- src/api/client/alias.rs | 4 ++-- src/api/client/device.rs | 8 ++------ src/api/client/filter.rs | 10 ++++------ src/api/client/keys.rs | 11 +++++------ src/api/client/media.rs | 10 +++++----- src/api/client/media_legacy.rs | 2 +- src/api/client/membership.rs | 6 +++--- src/api/client/openid.rs | 4 ++-- src/api/client/profile.rs | 4 ++-- src/api/client/push.rs | 21 ++++++++++----------- src/api/client/redact.rs | 4 ++-- src/api/client/report.rs | 5 ++--- src/api/client/room/aliases.rs | 2 +- src/api/client/room/create.rs | 6 +++--- src/api/client/session.rs | 15 ++++----------- src/api/client/state.rs | 2 +- src/api/client/tag.rs | 6 +++--- src/api/client/to_device.rs | 4 ++-- src/api/client/unstable.rs | 8 ++++---- 20 files changed, 69 insertions(+), 90 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 56b1a81c..30f8b89c 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -603,7 +603,6 @@ pub(crate) async fn change_password_route( .sender_user .as_ref() .ok_or_else(|| err!(Request(MissingToken("Missing access token."))))?; - let sender_device = body.sender_device(); let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { stages: vec![AuthType::Password] }], @@ -617,7 +616,7 @@ pub(crate) async fn change_password_route( | Some(auth) => { let (worked, uiaainfo) = services .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo) + .try_auth(sender_user, body.sender_device(), auth, &uiaainfo) .await?; if !worked { @@ -631,7 +630,7 @@ pub(crate) async fn change_password_route( uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, json); + .create(sender_user, body.sender_device(), &uiaainfo, json); return Err(Error::Uiaa(uiaainfo)); }, @@ -650,7 +649,7 @@ pub(crate) async fn change_password_route( services .users .all_device_ids(sender_user) - .ready_filter(|id| *id != sender_device) + .ready_filter(|id| *id != body.sender_device()) .for_each(|id| services.users.remove_device(sender_user, id)) .await; @@ -659,17 +658,17 @@ pub(crate) async fn change_password_route( .pusher .get_pushkeys(sender_user) .map(ToOwned::to_owned) - .broad_filter_map(|pushkey| async move { + .broad_filter_map(async |pushkey| { services .pusher .get_pusher_device(&pushkey) .await .ok() - .filter(|pusher_device| pusher_device != sender_device) + .filter(|pusher_device| pusher_device != body.sender_device()) .is_some() .then_some(pushkey) }) - .for_each(|pushkey| async move { + .for_each(async |pushkey| { services.pusher.delete_pusher(sender_user, &pushkey).await; }) .await; @@ -699,13 +698,10 @@ pub(crate) async fn whoami_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let device_id = body.sender_device.clone(); - Ok(whoami::v3::Response { - user_id: sender_user.clone(), - device_id, - is_guest: services.users.is_deactivated(sender_user).await? + user_id: body.sender_user().to_owned(), + device_id: body.sender_device.clone(), + is_guest: services.users.is_deactivated(body.sender_user()).await? && body.appservice_info.is_none(), }) } @@ -733,7 +729,6 @@ pub(crate) async fn deactivate_route( .sender_user .as_ref() .ok_or_else(|| err!(Request(MissingToken("Missing access token."))))?; - let sender_device = body.sender_device(); let mut uiaainfo = UiaaInfo { flows: vec![AuthFlow { stages: vec![AuthType::Password] }], @@ -747,7 +742,7 @@ pub(crate) async fn deactivate_route( | Some(auth) => { let (worked, uiaainfo) = services .uiaa - .try_auth(sender_user, sender_device, auth, &uiaainfo) + .try_auth(sender_user, body.sender_device(), auth, &uiaainfo) .await?; if !worked { @@ -760,7 +755,7 @@ pub(crate) async fn deactivate_route( uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, json); + .create(sender_user, body.sender_device(), &uiaainfo, json); return Err(Error::Uiaa(uiaainfo)); }, diff --git a/src/api/client/alias.rs b/src/api/client/alias.rs index dc7aad44..97c1a1bd 100644 --- a/src/api/client/alias.rs +++ b/src/api/client/alias.rs @@ -17,7 +17,7 @@ pub(crate) async fn create_alias_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if services.users.is_suspended(sender_user).await? { return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); } @@ -65,7 +65,7 @@ pub(crate) async fn delete_alias_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if services.users.is_suspended(sender_user).await? { return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); } diff --git a/src/api/client/device.rs b/src/api/client/device.rs index 5519a1a5..b0a7e142 100644 --- a/src/api/client/device.rs +++ b/src/api/client/device.rs @@ -21,11 +21,9 @@ pub(crate) async fn get_devices_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let devices: Vec = services .users - .all_devices_metadata(sender_user) + .all_devices_metadata(body.sender_user()) .collect() .await; @@ -39,11 +37,9 @@ pub(crate) async fn get_device_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let device = services .users - .get_device_metadata(sender_user, &body.body.device_id) + .get_device_metadata(body.sender_user(), &body.body.device_id) .await .map_err(|_| err!(Request(NotFound("Device not found."))))?; diff --git a/src/api/client/filter.rs b/src/api/client/filter.rs index 97044ffc..9814d366 100644 --- a/src/api/client/filter.rs +++ b/src/api/client/filter.rs @@ -13,11 +13,9 @@ pub(crate) async fn get_filter_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services .users - .get_filter(sender_user, &body.filter_id) + .get_filter(body.sender_user(), &body.filter_id) .await .map(get_filter::v3::Response::new) .map_err(|_| err!(Request(NotFound("Filter not found.")))) @@ -30,9 +28,9 @@ pub(crate) async fn create_filter_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - - let filter_id = services.users.create_filter(sender_user, &body.filter); + let filter_id = services + .users + .create_filter(body.sender_user(), &body.filter); Ok(create_filter::v3::Response::new(filter_id)) } diff --git a/src/api/client/keys.rs b/src/api/client/keys.rs index 650c573f..d2bd46a0 100644 --- a/src/api/client/keys.rs +++ b/src/api/client/keys.rs @@ -126,7 +126,7 @@ pub(crate) async fn get_keys_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); get_keys_helper( &services, @@ -157,8 +157,7 @@ pub(crate) async fn upload_signing_keys_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); + let (sender_user, sender_device) = body.sender(); // UIAA let mut uiaainfo = UiaaInfo { @@ -203,12 +202,12 @@ pub(crate) async fn upload_signing_keys_route( } // Success! }, - | _ => match body.json_body { + | _ => match body.json_body.as_ref() { | Some(json) => { uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH)); services .uiaa - .create(sender_user, sender_device, &uiaainfo, &json); + .create(sender_user, sender_device, &uiaainfo, json); return Err(Error::Uiaa(uiaainfo)); }, @@ -373,7 +372,7 @@ pub(crate) async fn get_key_changes_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let mut device_list_updates = HashSet::new(); diff --git a/src/api/client/media.rs b/src/api/client/media.rs index 11d5450c..3f491d54 100644 --- a/src/api/client/media.rs +++ b/src/api/client/media.rs @@ -51,7 +51,7 @@ pub(crate) async fn create_content_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let user = body.sender_user.as_ref().expect("user is authenticated"); + let user = body.sender_user(); if services.users.is_suspended(user).await? { return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); } @@ -97,7 +97,7 @@ pub(crate) async fn get_content_thumbnail_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let user = body.sender_user.as_ref().expect("user is authenticated"); + let user = body.sender_user(); let dim = Dim::from_ruma(body.width, body.height, body.method.clone())?; let mxc = Mxc { @@ -134,7 +134,7 @@ pub(crate) async fn get_content_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let user = body.sender_user.as_ref().expect("user is authenticated"); + let user = body.sender_user(); let mxc = Mxc { server_name: &body.server_name, @@ -170,7 +170,7 @@ pub(crate) async fn get_content_as_filename_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let user = body.sender_user.as_ref().expect("user is authenticated"); + let user = body.sender_user(); let mxc = Mxc { server_name: &body.server_name, @@ -206,7 +206,7 @@ pub(crate) async fn get_media_preview_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let url = &body.url; let url = Url::parse(&body.url).map_err(|e| { diff --git a/src/api/client/media_legacy.rs b/src/api/client/media_legacy.rs index d9f24f77..930daab4 100644 --- a/src/api/client/media_legacy.rs +++ b/src/api/client/media_legacy.rs @@ -55,7 +55,7 @@ pub(crate) async fn get_media_preview_legacy_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let url = &body.url; let url = Url::parse(&body.url).map_err(|e| { diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index e6392533..b5356b94 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -249,14 +249,14 @@ pub(crate) async fn join_room_by_id_or_alias_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_deref().expect("user is authenticated"); + let sender_user = body.sender_user(); let appservice_info = &body.appservice_info; - let body = body.body; + let body = &body.body; if services.users.is_suspended(sender_user).await? { return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); } - let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { + let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias.clone()) { | Ok(room_id) => { banned_room_check( &services, diff --git a/src/api/client/openid.rs b/src/api/client/openid.rs index 8d2de68d..e27b3ab8 100644 --- a/src/api/client/openid.rs +++ b/src/api/client/openid.rs @@ -19,9 +19,9 @@ pub(crate) async fn create_openid_token_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); - if sender_user != &body.user_id { + if sender_user != body.user_id { return Err(Error::BadRequest( ErrorKind::InvalidParam, "Not allowed to request OpenID tokens on behalf of other users", diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index bdba4078..76b5dc6d 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -35,7 +35,7 @@ pub(crate) async fn set_displayname_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if services.users.is_suspended(sender_user).await? { return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); } @@ -127,7 +127,7 @@ pub(crate) async fn set_avatar_url_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if services.users.is_suspended(sender_user).await? { return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); } diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 81020ffa..125b26bb 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -106,7 +106,7 @@ pub(crate) async fn get_pushrules_global_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let Some(content_value) = services .account_data @@ -234,9 +234,8 @@ pub(crate) async fn set_pushrule_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let body = body.body; - + let sender_user = body.sender_user(); + let body = &body.body; let mut account_data: PushRulesEvent = services .account_data .get_global(sender_user, GlobalAccountDataEventType::PushRules) @@ -295,7 +294,7 @@ pub(crate) async fn get_pushrule_actions_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); // remove old deprecated mentions push rules as per MSC4210 #[allow(deprecated)] @@ -329,7 +328,7 @@ pub(crate) async fn set_pushrule_actions_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let mut account_data: PushRulesEvent = services .account_data @@ -366,7 +365,7 @@ pub(crate) async fn get_pushrule_enabled_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); // remove old deprecated mentions push rules as per MSC4210 #[allow(deprecated)] @@ -400,7 +399,7 @@ pub(crate) async fn set_pushrule_enabled_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let mut account_data: PushRulesEvent = services .account_data @@ -437,7 +436,7 @@ pub(crate) async fn delete_pushrule_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let mut account_data: PushRulesEvent = services .account_data @@ -483,7 +482,7 @@ pub(crate) async fn get_pushers_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); Ok(get_pushers::v3::Response { pushers: services.pusher.get_pushers(sender_user).await, @@ -499,7 +498,7 @@ pub(crate) async fn set_pushers_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); services .pusher diff --git a/src/api/client/redact.rs b/src/api/client/redact.rs index a8eaf91d..86d871ff 100644 --- a/src/api/client/redact.rs +++ b/src/api/client/redact.rs @@ -15,8 +15,8 @@ pub(crate) async fn redact_event_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let body = body.body; + let sender_user = body.sender_user(); + let body = &body.body; if services.users.is_suspended(sender_user).await? { // TODO: Users can redact their own messages while suspended return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 5113b42f..8ece3ab1 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -38,8 +38,7 @@ pub(crate) async fn report_room_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - // user authentication - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if services.users.is_suspended(sender_user).await? { return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); } @@ -94,7 +93,7 @@ pub(crate) async fn report_event_route( body: Ruma, ) -> Result { // user authentication - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if services.users.is_suspended(sender_user).await? { return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); } diff --git a/src/api/client/room/aliases.rs b/src/api/client/room/aliases.rs index 3f0016af..a944971c 100644 --- a/src/api/client/room/aliases.rs +++ b/src/api/client/room/aliases.rs @@ -15,7 +15,7 @@ pub(crate) async fn get_room_aliases_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if !services .rooms diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index d1dffc51..4ca64fd8 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -58,7 +58,7 @@ pub(crate) async fn create_room_route( ) -> Result { use create_room::v3::RoomPreset; - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if !services.globals.allow_room_creation() && body.appservice_info.is_none() @@ -174,7 +174,7 @@ pub(crate) async fn create_room_route( let content = match room_version { | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => - RoomCreateEventContent::new_v1(sender_user.clone()), + RoomCreateEventContent::new_v1(sender_user.to_owned()), | _ => RoomCreateEventContent::new_v11(), }; let mut content = serde_json::from_str::( @@ -239,7 +239,7 @@ pub(crate) async fn create_room_route( | _ => RoomPreset::PrivateChat, // Room visibility should not be custom }); - let mut users = BTreeMap::from_iter([(sender_user.clone(), int!(100))]); + let mut users = BTreeMap::from_iter([(sender_user.to_owned(), int!(100))]); if preset == RoomPreset::TrustedPrivateChat { for invite in &body.invite { diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 2499a43d..992073c6 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -269,11 +269,9 @@ pub(crate) async fn login_token_route( return Err!(Request(Forbidden("Login via an existing session is not enabled"))); } - let sender_user = body.sender_user(); - let sender_device = body.sender_device(); - // This route SHOULD have UIA // TODO: How do we make only UIA sessions that have not been used before valid? + let (sender_user, sender_device) = body.sender(); let mut uiaainfo = uiaa::UiaaInfo { flows: vec![uiaa::AuthFlow { stages: vec![uiaa::AuthType::Password] }], @@ -335,12 +333,9 @@ pub(crate) async fn logout_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let sender_device = body.sender_device.as_ref().expect("user is authenticated"); - services .users - .remove_device(sender_user, sender_device) + .remove_device(body.sender_user(), body.sender_device()) .await; Ok(logout::v3::Response::new()) @@ -365,12 +360,10 @@ pub(crate) async fn logout_all_route( InsecureClientIp(client): InsecureClientIp, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - services .users - .all_device_ids(sender_user) - .for_each(|device_id| services.users.remove_device(sender_user, device_id)) + .all_device_ids(body.sender_user()) + .for_each(|device_id| services.users.remove_device(body.sender_user(), device_id)) .await; Ok(logout_all::v3::Response::new()) diff --git a/src/api/client/state.rs b/src/api/client/state.rs index 07802b1b..cf371728 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -77,7 +77,7 @@ pub(crate) async fn get_state_events_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if !services .rooms diff --git a/src/api/client/tag.rs b/src/api/client/tag.rs index caafe10d..dd799105 100644 --- a/src/api/client/tag.rs +++ b/src/api/client/tag.rs @@ -21,7 +21,7 @@ pub(crate) async fn update_tag_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let mut tags_event = services .account_data @@ -58,7 +58,7 @@ pub(crate) async fn delete_tag_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let mut tags_event = services .account_data @@ -92,7 +92,7 @@ pub(crate) async fn get_tags_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let tags_event = services .account_data diff --git a/src/api/client/to_device.rs b/src/api/client/to_device.rs index 8ad9dc99..581f4a72 100644 --- a/src/api/client/to_device.rs +++ b/src/api/client/to_device.rs @@ -21,7 +21,7 @@ pub(crate) async fn send_event_to_device_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); let sender_device = body.sender_device.as_deref(); // Check if this is a new transaction id @@ -47,7 +47,7 @@ pub(crate) async fn send_event_to_device_route( serde_json::to_writer( &mut buf, &federation::transactions::edu::Edu::DirectToDevice(DirectDeviceContent { - sender: sender_user.clone(), + sender: sender_user.to_owned(), ev_type: body.event_type.clone(), message_id: count.to_string().into(), messages, diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index e21eaf21..08f70975 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -69,7 +69,7 @@ pub(crate) async fn delete_timezone_key_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if *sender_user != body.user_id && body.appservice_info.is_none() { return Err!(Request(Forbidden("You cannot update the profile of another user"))); @@ -97,7 +97,7 @@ pub(crate) async fn set_timezone_key_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if *sender_user != body.user_id && body.appservice_info.is_none() { return Err!(Request(Forbidden("You cannot update the profile of another user"))); @@ -125,7 +125,7 @@ pub(crate) async fn set_profile_key_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if *sender_user != body.user_id && body.appservice_info.is_none() { return Err!(Request(Forbidden("You cannot update the profile of another user"))); @@ -218,7 +218,7 @@ pub(crate) async fn delete_profile_key_route( State(services): State, body: Ruma, ) -> Result { - let sender_user = body.sender_user.as_ref().expect("user is authenticated"); + let sender_user = body.sender_user(); if *sender_user != body.user_id && body.appservice_info.is_none() { return Err!(Request(Forbidden("You cannot update the profile of another user"))); From 2051c22a281b6daeda252e75c1332d26edd6f48f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Mon, 28 Apr 2025 01:32:13 +0000 Subject: [PATCH 1881/2091] Support optional device_id's in lazy-loading context. Co-authored-by: Jade Ellis Signed-off-by: Jason Volk --- src/api/client/context.rs | 2 +- src/api/client/message.rs | 28 +++++++++++++-------------- src/api/client/sync/v3.rs | 2 +- src/service/rooms/lazy_loading/mod.rs | 4 ++-- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index dbc2a22f..ca787a16 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -111,7 +111,7 @@ pub(crate) async fn get_context_route( let lazy_loading_context = lazy_loading::Context { user_id: sender_user, - device_id: sender_device, + device_id: Some(sender_device), room_id, token: Some(base_count.into_unsigned()), options: Some(&filter.lazy_load_options), diff --git a/src/api/client/message.rs b/src/api/client/message.rs index e442850b..7a87a9b0 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,5 +1,3 @@ -use core::panic; - use axum::extract::State; use conduwuit::{ Err, Result, at, @@ -34,6 +32,7 @@ use ruma::{ }, serde::Raw, }; +use tracing::warn; use crate::Ruma; @@ -73,7 +72,7 @@ pub(crate) async fn get_message_events_route( ) -> Result { debug_assert!(IGNORED_MESSAGE_TYPES.is_sorted(), "IGNORED_MESSAGE_TYPES is not sorted"); let sender_user = body.sender_user(); - let sender_device = body.sender_device.as_ref(); + let sender_device = body.sender_device.as_deref(); let room_id = &body.room_id; let filter = &body.filter; @@ -137,18 +136,17 @@ pub(crate) async fn get_message_events_route( let lazy_loading_context = lazy_loading::Context { user_id: sender_user, - device_id: match sender_device { - | Some(device_id) => device_id, - | None => - if let Some(registration) = body.appservice_info.as_ref() { - <&DeviceId>::from(registration.registration.id.as_str()) - } else { - panic!( - "No device_id provided and no appservice registration found, this \ - should be unreachable" - ); - }, - }, + device_id: sender_device.or_else(|| { + if let Some(registration) = body.appservice_info.as_ref() { + Some(<&DeviceId>::from(registration.registration.id.as_str())) + } else { + warn!( + "No device_id provided and no appservice registration found, this should be \ + unreachable" + ); + None + } + }), room_id, token: Some(from.into_unsigned()), options: Some(&filter.lazy_load_options), diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 7bc74c95..feaf8689 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -645,7 +645,7 @@ async fn load_joined_room( let lazy_loading_context = &lazy_loading::Context { user_id: sender_user, - device_id: sender_device, + device_id: Some(sender_device), room_id, token: Some(since), options: Some(&filter.room.state.lazy_load_options), diff --git a/src/service/rooms/lazy_loading/mod.rs b/src/service/rooms/lazy_loading/mod.rs index 346314d1..61f081a9 100644 --- a/src/service/rooms/lazy_loading/mod.rs +++ b/src/service/rooms/lazy_loading/mod.rs @@ -27,7 +27,7 @@ pub trait Options: Send + Sync { #[derive(Clone, Debug)] pub struct Context<'a> { pub user_id: &'a UserId, - pub device_id: &'a DeviceId, + pub device_id: Option<&'a DeviceId>, pub room_id: &'a RoomId, pub token: Option, pub options: Option<&'a LazyLoadOptions>, @@ -40,7 +40,7 @@ pub enum Status { } pub type Witness = HashSet; -type Key<'a> = (&'a UserId, &'a DeviceId, &'a RoomId, &'a UserId); +type Key<'a> = (&'a UserId, Option<&'a DeviceId>, &'a RoomId, &'a UserId); impl crate::Service for Service { fn build(args: crate::Args<'_>) -> Result> { From f3dd90df39da1d5ff1e4aabc150990efe5e64f5e Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 26 Apr 2025 08:23:57 +0000 Subject: [PATCH 1882/2091] Mitigate large futures Signed-off-by: Jason Volk --- src/admin/room/moderation.rs | 12 +++++++++--- src/admin/user/commands.rs | 6 ++++-- src/api/client/account.rs | 6 ++++-- src/api/client/membership.rs | 12 +++++++++--- src/api/client/state.rs | 3 ++- src/service/admin/mod.rs | 3 +-- 6 files changed, 29 insertions(+), 13 deletions(-) diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index 921249bd..5fb5bb3e 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -5,7 +5,7 @@ use conduwuit::{ utils::{IterStream, ReadyExt}, warn, }; -use futures::StreamExt; +use futures::{FutureExt, StreamExt}; use ruma::{OwnedRoomId, OwnedRoomOrAliasId, RoomAliasId, RoomId, RoomOrAliasId}; use crate::{admin_command, admin_command_dispatch, get_room_info}; @@ -132,7 +132,10 @@ async fn ban_room(&self, room: OwnedRoomOrAliasId) -> Result { evicting admins too)", ); - if let Err(e) = leave_room(self.services, user_id, &room_id, None).await { + if let Err(e) = leave_room(self.services, user_id, &room_id, None) + .boxed() + .await + { warn!("Failed to leave room: {e}"); } @@ -297,7 +300,10 @@ async fn ban_list_of_rooms(&self) -> Result { evicting admins too)", ); - if let Err(e) = leave_room(self.services, user_id, &room_id, None).await { + if let Err(e) = leave_room(self.services, user_id, &room_id, None) + .boxed() + .await + { warn!("Failed to leave room: {e}"); } diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index d094fc5f..89f7a9fc 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -8,7 +8,7 @@ use conduwuit::{ warn, }; use conduwuit_api::client::{leave_all_rooms, update_avatar_url, update_displayname}; -use futures::StreamExt; +use futures::{FutureExt, StreamExt}; use ruma::{ OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, UserId, events::{ @@ -696,7 +696,9 @@ pub(super) async fn force_leave_room( return Err!("{user_id} is not joined in the room"); } - leave_room(self.services, &user_id, &room_id, None).await?; + leave_room(self.services, &user_id, &room_id, None) + .boxed() + .await?; self.write_str(&format!("{user_id} has left {room_id}.",)) .await diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 30f8b89c..2e822f02 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -777,7 +777,9 @@ pub(crate) async fn deactivate_route( super::update_displayname(&services, sender_user, None, &all_joined_rooms).await; super::update_avatar_url(&services, sender_user, None, None, &all_joined_rooms).await; - full_user_deactivate(&services, sender_user, &all_joined_rooms).await?; + full_user_deactivate(&services, sender_user, &all_joined_rooms) + .boxed() + .await?; info!("User {sender_user} deactivated their account."); @@ -929,7 +931,7 @@ pub async fn full_user_deactivate( } } - super::leave_all_rooms(services, user_id).await; + super::leave_all_rooms(services, user_id).boxed().await; Ok(()) } diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index b5356b94..85d0cd21 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -114,7 +114,9 @@ async fn banned_room_check( .collect() .await; - full_user_deactivate(services, user_id, &all_joined_rooms).await?; + full_user_deactivate(services, user_id, &all_joined_rooms) + .boxed() + .await?; } return Err!(Request(Forbidden("This room is banned on this homeserver."))); @@ -153,7 +155,9 @@ async fn banned_room_check( .collect() .await; - full_user_deactivate(services, user_id, &all_joined_rooms).await?; + full_user_deactivate(services, user_id, &all_joined_rooms) + .boxed() + .await?; } return Err!(Request(Forbidden("This remote server is banned on this homeserver."))); @@ -265,6 +269,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( room_id.server_name(), client, ) + .boxed() .await?; let mut servers = body.via.clone(); @@ -487,6 +492,7 @@ pub(crate) async fn leave_room_route( body: Ruma, ) -> Result { leave_room(&services, body.sender_user(), &body.room_id, body.reason.clone()) + .boxed() .await .map(|()| leave_room::v3::Response::new()) } @@ -1825,7 +1831,7 @@ pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { for room_id in all_rooms { // ignore errors - if let Err(e) = leave_room(services, user_id, &room_id, None).await { + if let Err(e) = leave_room(services, user_id, &room_id, None).boxed().await { warn!(%user_id, "Failed to leave {room_id} remotely: {e}"); } diff --git a/src/api/client/state.rs b/src/api/client/state.rs index cf371728..c0303200 100644 --- a/src/api/client/state.rs +++ b/src/api/client/state.rs @@ -5,7 +5,7 @@ use conduwuit::{ utils::BoolExt, }; use conduwuit_service::Services; -use futures::TryStreamExt; +use futures::{FutureExt, TryStreamExt}; use ruma::{ OwnedEventId, RoomId, UserId, api::client::state::{get_state_events, get_state_events_for_key, send_state_event}, @@ -63,6 +63,7 @@ pub(crate) async fn send_state_event_for_empty_key_route( body: Ruma, ) -> Result> { send_state_event_for_key_route(State(services), body) + .boxed() .await .map(RumaResponse) } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 11d93cc2..19a523ca 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -4,7 +4,6 @@ mod execute; mod grant; use std::{ - future::Future, pin::Pin, sync::{Arc, RwLock as StdRwLock, Weak}, }; @@ -14,7 +13,7 @@ use conduwuit::{ Error, PduEvent, Result, Server, debug, err, error, error::default_log, pdu::PduBuilder, }; pub use create::create_admin_room; -use futures::{FutureExt, TryFutureExt}; +use futures::{Future, FutureExt, TryFutureExt}; use loole::{Receiver, Sender}; use ruma::{ OwnedEventId, OwnedRoomId, RoomId, UserId, From 732a77f3a8f1e02c6a436604792c5377ff4d4bf9 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 26 Apr 2025 23:01:21 +0000 Subject: [PATCH 1883/2091] Use integrated error instead of panic on some legacy codepaths Signed-off-by: Jason Volk --- src/api/client/account.rs | 6 +-- src/api/client/capabilities.rs | 16 +++---- src/api/client/push.rs | 86 ++++++++++++---------------------- src/api/client/read_marker.rs | 4 +- src/api/client/room/create.rs | 22 +++------ src/api/client/tag.rs | 4 +- 6 files changed, 49 insertions(+), 89 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 2e822f02..b6ff0f2b 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -351,8 +351,7 @@ pub(crate) async fn register_route( if !services.globals.new_user_displayname_suffix().is_empty() && body.appservice_info.is_none() { - write!(displayname, " {}", services.server.config.new_user_displayname_suffix) - .expect("should be able to write to string buffer"); + write!(displayname, " {}", services.server.config.new_user_displayname_suffix)?; } services @@ -370,8 +369,7 @@ pub(crate) async fn register_route( content: ruma::events::push_rules::PushRulesEventContent { global: push::Ruleset::server_default(&user_id), }, - }) - .expect("to json always works"), + })?, ) .await?; diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 7362c4f9..c42c6dfd 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -26,8 +26,8 @@ pub(crate) async fn get_capabilities_route( let mut capabilities = Capabilities::default(); capabilities.room_versions = RoomVersionsCapability { - default: services.server.config.default_room_version.clone(), available, + default: services.server.config.default_room_version.clone(), }; // we do not implement 3PID stuff @@ -38,16 +38,12 @@ pub(crate) async fn get_capabilities_route( }; // MSC4133 capability - capabilities - .set("uk.tcpip.msc4133.profile_fields", json!({"enabled": true})) - .expect("this is valid JSON we created"); + capabilities.set("uk.tcpip.msc4133.profile_fields", json!({"enabled": true}))?; - capabilities - .set( - "org.matrix.msc4267.forget_forced_upon_leave", - json!({"enabled": services.config.forget_forced_upon_leave}), - ) - .expect("valid JSON we created"); + capabilities.set( + "org.matrix.msc4267.forget_forced_upon_leave", + json!({"enabled": services.config.forget_forced_upon_leave}), + )?; Ok(get_capabilities::v3::Response { capabilities }) } diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 125b26bb..74e29422 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -79,17 +79,14 @@ pub(crate) async fn get_pushrules_all_route( global_ruleset.update_with_server_default(Ruleset::server_default(sender_user)); + let ty = GlobalAccountDataEventType::PushRules; + let event = PushRulesEvent { + content: PushRulesEventContent { global: global_ruleset.clone() }, + }; + services .account_data - .update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(PushRulesEvent { - content: PushRulesEventContent { global: global_ruleset.clone() }, - }) - .expect("to json always works"), - ) + .update(None, sender_user, ty.to_string().into(), &serde_json::to_value(event)?) .await?; } }; @@ -118,19 +115,17 @@ pub(crate) async fn get_pushrules_global_route( else { // user somehow has non-existent push rule event. recreate it and return server // default silently + + let ty = GlobalAccountDataEventType::PushRules; + let event = PushRulesEvent { + content: PushRulesEventContent { + global: Ruleset::server_default(sender_user), + }, + }; + services .account_data - .update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(PushRulesEvent { - content: PushRulesEventContent { - global: Ruleset::server_default(sender_user), - }, - }) - .expect("to json always works"), - ) + .update(None, sender_user, ty.to_string().into(), &serde_json::to_value(event)?) .await?; return Ok(get_pushrules_global_scope::v3::Response { @@ -274,14 +269,10 @@ pub(crate) async fn set_pushrule_route( return Err(err); } + let ty = GlobalAccountDataEventType::PushRules; services .account_data - .update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - ) + .update(None, sender_user, ty.to_string().into(), &serde_json::to_value(account_data)?) .await?; Ok(set_pushrule::v3::Response {}) @@ -345,14 +336,10 @@ pub(crate) async fn set_pushrule_actions_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")); } + let ty = GlobalAccountDataEventType::PushRules; services .account_data - .update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - ) + .update(None, sender_user, ty.to_string().into(), &serde_json::to_value(account_data)?) .await?; Ok(set_pushrule_actions::v3::Response {}) @@ -416,14 +403,10 @@ pub(crate) async fn set_pushrule_enabled_route( return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")); } + let ty = GlobalAccountDataEventType::PushRules; services .account_data - .update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - ) + .update(None, sender_user, ty.to_string().into(), &serde_json::to_value(account_data)?) .await?; Ok(set_pushrule_enabled::v3::Response {}) @@ -462,14 +445,10 @@ pub(crate) async fn delete_pushrule_route( return Err(err); } + let ty = GlobalAccountDataEventType::PushRules; services .account_data - .update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(account_data).expect("to json value always works"), - ) + .update(None, sender_user, ty.to_string().into(), &serde_json::to_value(account_data)?) .await?; Ok(delete_pushrule::v3::Response {}) @@ -514,19 +493,16 @@ async fn recreate_push_rules_and_return( services: &Services, sender_user: &ruma::UserId, ) -> Result { + let ty = GlobalAccountDataEventType::PushRules; + let event = PushRulesEvent { + content: PushRulesEventContent { + global: Ruleset::server_default(sender_user), + }, + }; + services .account_data - .update( - None, - sender_user, - GlobalAccountDataEventType::PushRules.to_string().into(), - &serde_json::to_value(PushRulesEvent { - content: PushRulesEventContent { - global: Ruleset::server_default(sender_user), - }, - }) - .expect("to json always works"), - ) + .update(None, sender_user, ty.to_string().into(), &serde_json::to_value(event)?) .await?; Ok(get_pushrules_all::v3::Response { diff --git a/src/api/client/read_marker.rs b/src/api/client/read_marker.rs index e152869c..9d813294 100644 --- a/src/api/client/read_marker.rs +++ b/src/api/client/read_marker.rs @@ -37,7 +37,7 @@ pub(crate) async fn set_read_marker_route( Some(&body.room_id), sender_user, RoomAccountDataEventType::FullyRead, - &serde_json::to_value(fully_read_event).expect("to json value always works"), + &serde_json::to_value(fully_read_event)?, ) .await?; } @@ -151,7 +151,7 @@ pub(crate) async fn create_receipt_route( Some(&body.room_id), sender_user, RoomAccountDataEventType::FullyRead, - &serde_json::to_value(fully_read_event).expect("to json value always works"), + &serde_json::to_value(fully_read_event)?, ) .await?; }, diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index 4ca64fd8..aa54e1e9 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -177,18 +177,10 @@ pub(crate) async fn create_room_route( RoomCreateEventContent::new_v1(sender_user.to_owned()), | _ => RoomCreateEventContent::new_v11(), }; - let mut content = serde_json::from_str::( - to_raw_value(&content) - .expect("we just created this as content was None") - .get(), - ) - .unwrap(); - content.insert( - "room_version".into(), - json!(room_version.as_str()) - .try_into() - .expect("we just created this as content was None"), - ); + let mut content = + serde_json::from_str::(to_raw_value(&content)?.get()) + .unwrap(); + content.insert("room_version".into(), json!(room_version.as_str()).try_into()?); content }, }; @@ -200,8 +192,7 @@ pub(crate) async fn create_room_route( .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomCreate, - content: to_raw_value(&create_content) - .expect("create event content serialization"), + content: to_raw_value(&create_content)?, state_key: Some(StateKey::new()), ..Default::default() }, @@ -267,8 +258,7 @@ pub(crate) async fn create_room_route( .build_and_append_pdu( PduBuilder { event_type: TimelineEventType::RoomPowerLevels, - content: to_raw_value(&power_levels_content) - .expect("serialized power_levels event content"), + content: to_raw_value(&power_levels_content)?, state_key: Some(StateKey::new()), ..Default::default() }, diff --git a/src/api/client/tag.rs b/src/api/client/tag.rs index dd799105..68105e4f 100644 --- a/src/api/client/tag.rs +++ b/src/api/client/tag.rs @@ -42,7 +42,7 @@ pub(crate) async fn update_tag_route( Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag, - &serde_json::to_value(tags_event).expect("to json value always works"), + &serde_json::to_value(tags_event)?, ) .await?; @@ -76,7 +76,7 @@ pub(crate) async fn delete_tag_route( Some(&body.room_id), sender_user, RoomAccountDataEventType::Tag, - &serde_json::to_value(tags_event).expect("to json value always works"), + &serde_json::to_value(tags_event)?, ) .await?; From 21bbee8e3cd7c982e2fbc38aeebf8f387da05165 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 26 Apr 2025 23:04:58 +0000 Subject: [PATCH 1884/2091] Simplify api to send notices to admin room Signed-off-by: Jason Volk --- src/api/client/account.rs | 60 +++++++++++------------------------ src/api/client/room/create.rs | 14 ++++---- src/service/admin/mod.rs | 7 ++++ 3 files changed, 32 insertions(+), 49 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index b6ff0f2b..27d93bef 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -26,10 +26,7 @@ use ruma::{ }, events::{ GlobalAccountDataEventType, StateEventType, - room::{ - message::RoomMessageEventContent, - power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, - }, + room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, }, push, }; @@ -414,32 +411,21 @@ pub(crate) async fn register_route( // log in conduit admin channel if a non-guest user registered if body.appservice_info.is_none() && !is_guest { if !device_display_name.is_empty() { - info!( - "New user \"{user_id}\" registered on this server with device display name: \ - \"{device_display_name}\"" + let notice = format!( + "New user \"{user_id}\" registered on this server from IP {client} and device \ + display name \"{device_display_name}\"" ); + info!("{notice}"); if services.server.config.admin_room_notices { - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "New user \"{user_id}\" registered on this server from IP {client} and \ - device display name \"{device_display_name}\"" - ))) - .await - .ok(); + services.admin.notice(¬ice).await; } } else { - info!("New user \"{user_id}\" registered on this server."); + let notice = format!("New user \"{user_id}\" registered on this server."); + info!("{notice}"); if services.server.config.admin_room_notices { - services - .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "New user \"{user_id}\" registered on this server from IP {client}" - ))) - .await - .ok(); + services.admin.notice(¬ice).await; } } } @@ -452,24 +438,22 @@ pub(crate) async fn register_route( if services.server.config.admin_room_notices { services .admin - .send_message(RoomMessageEventContent::notice_plain(format!( + .notice(&format!( "Guest user \"{user_id}\" with device display name \ \"{device_display_name}\" registered on this server from IP {client}" - ))) - .await - .ok(); + )) + .await; } } else { #[allow(clippy::collapsible_else_if)] if services.server.config.admin_room_notices { services .admin - .send_message(RoomMessageEventContent::notice_plain(format!( + .notice(&format!( "Guest user \"{user_id}\" with no device display name registered on \ this server from IP {client}", - ))) - .await - .ok(); + )) + .await; } } } @@ -677,11 +661,8 @@ pub(crate) async fn change_password_route( if services.server.config.admin_room_notices { services .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "User {sender_user} changed their password." - ))) - .await - .ok(); + .notice(&format!("User {sender_user} changed their password.")) + .await; } Ok(change_password::v3::Response {}) @@ -784,11 +765,8 @@ pub(crate) async fn deactivate_route( if services.server.config.admin_room_notices { services .admin - .send_message(RoomMessageEventContent::notice_plain(format!( - "User {sender_user} deactivated their account." - ))) - .await - .ok(); + .notice(&format!("User {sender_user} deactivated their account.")) + .await; } Ok(deactivate::v3::Response { diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index aa54e1e9..8b93fcfd 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -92,19 +92,17 @@ pub(crate) async fn create_room_route( && !services.users.is_admin(sender_user).await && body.appservice_info.is_none() { - info!( - "Non-admin user {sender_user} tried to publish {0} to the room directory while \ - \"lockdown_public_room_directory\" is enabled", - &room_id + warn!( + "Non-admin user {sender_user} tried to publish {room_id} to the room directory \ + while \"lockdown_public_room_directory\" is enabled" ); if services.server.config.admin_room_notices { services .admin - .send_text(&format!( - "Non-admin user {sender_user} tried to publish {0} to the room directory \ - while \"lockdown_public_room_directory\" is enabled", - &room_id + .notice(&format!( + "Non-admin user {sender_user} tried to publish {room_id} to the room \ + directory while \"lockdown_public_room_directory\" is enabled" )) .await; } diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 19a523ca..a76c3ef6 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -142,6 +142,13 @@ impl crate::Service for Service { } impl Service { + /// Sends markdown notice to the admin room as the admin user. + pub async fn notice(&self, body: &str) { + self.send_message(RoomMessageEventContent::notice_markdown(body)) + .await + .ok(); + } + /// Sends markdown message (not an m.notice for notification reasons) to the /// admin room as the admin user. pub async fn send_text(&self, body: &str) { From 667afedd24d40db1d63fa052dbd55e872b493225 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 26 Apr 2025 23:50:03 +0000 Subject: [PATCH 1885/2091] Macroize various remaining Error constructions. Signed-off-by: Jason Volk --- src/api/client/account.rs | 16 +-- src/api/client/openid.rs | 13 +-- src/api/client/profile.rs | 15 ++- src/api/client/push.rs | 6 +- src/api/client/report.rs | 32 ++---- src/api/client/room/aliases.rs | 9 +- src/api/client/room/create.rs | 100 ++++++++---------- src/api/server/invite.rs | 2 +- .../rooms/event_handler/handle_outlier_pdu.rs | 10 +- 9 files changed, 78 insertions(+), 125 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 27d93bef..14bbcf98 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -13,22 +13,14 @@ use conduwuit_service::Services; use futures::{FutureExt, StreamExt}; use register::RegistrationKind; use ruma::{ - OwnedRoomId, UserId, api::client::{ account::{ - ThirdPartyIdRemovalStatus, change_password, check_registration_token_validity, - deactivate, get_3pids, get_username_availability, - register::{self, LoginType}, - request_3pid_management_token_via_email, request_3pid_management_token_via_msisdn, - whoami, + change_password, check_registration_token_validity, deactivate, get_3pids, get_username_availability, register::{self, LoginType}, request_3pid_management_token_via_email, request_3pid_management_token_via_msisdn, whoami, ThirdPartyIdRemovalStatus }, uiaa::{AuthFlow, AuthType, UiaaInfo}, - }, - events::{ - GlobalAccountDataEventType, StateEventType, - room::power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, - }, - push, + }, events::{ + room::{message::RoomMessageEventContent, power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}}, GlobalAccountDataEventType, StateEventType + }, push, OwnedRoomId, UserId }; use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH, join_room_by_id_helper}; diff --git a/src/api/client/openid.rs b/src/api/client/openid.rs index e27b3ab8..0390b4b3 100644 --- a/src/api/client/openid.rs +++ b/src/api/client/openid.rs @@ -1,11 +1,8 @@ use std::time::Duration; use axum::extract::State; -use conduwuit::{Error, Result, utils}; -use ruma::{ - api::client::{account, error::ErrorKind}, - authentication::TokenType, -}; +use conduwuit::{Err, Result, utils}; +use ruma::{api::client::account, authentication::TokenType}; use super::TOKEN_LENGTH; use crate::Ruma; @@ -22,14 +19,12 @@ pub(crate) async fn create_openid_token_route( let sender_user = body.sender_user(); if sender_user != body.user_id { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, + return Err!(Request(InvalidParam( "Not allowed to request OpenID tokens on behalf of other users", - )); + ))); } let access_token = utils::random_string(TOKEN_LENGTH); - let expires_in = services .users .create_openid_token(&body.user_id, &access_token)?; diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 76b5dc6d..6efad64e 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ - Err, Error, Result, + Err, Result, matrix::pdu::PduBuilder, utils::{IterStream, stream::TryIgnore}, warn, @@ -12,11 +12,8 @@ use futures::{StreamExt, TryStreamExt, future::join3}; use ruma::{ OwnedMxcUri, OwnedRoomId, UserId, api::{ - client::{ - error::ErrorKind, - profile::{ - get_avatar_url, get_display_name, get_profile, set_avatar_url, set_display_name, - }, + client::profile::{ + get_avatar_url, get_display_name, get_profile, set_avatar_url, set_display_name, }, federation, }, @@ -110,7 +107,7 @@ pub(crate) async fn get_displayname_route( if !services.users.exists(&body.user_id).await { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation - return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); + return Err!(Request(NotFound("Profile was not found."))); } Ok(get_display_name::v3::Response { @@ -214,7 +211,7 @@ pub(crate) async fn get_avatar_url_route( if !services.users.exists(&body.user_id).await { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation - return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); + return Err!(Request(NotFound("Profile was not found."))); } Ok(get_avatar_url::v3::Response { @@ -287,7 +284,7 @@ pub(crate) async fn get_profile_route( if !services.users.exists(&body.user_id).await { // Return 404 if this user doesn't exist and we couldn't fetch it over // federation - return Err(Error::BadRequest(ErrorKind::NotFound, "Profile was not found.")); + return Err!(Request(NotFound("Profile was not found."))); } let mut custom_profile_fields: BTreeMap = services diff --git a/src/api/client/push.rs b/src/api/client/push.rs index 74e29422..d8d84ec7 100644 --- a/src/api/client/push.rs +++ b/src/api/client/push.rs @@ -218,7 +218,7 @@ pub(crate) async fn get_pushrule_route( if let Some(rule) = rule { Ok(get_pushrule::v3::Response { rule }) } else { - Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")) + Err!(Request(NotFound("Push rule not found."))) } } @@ -333,7 +333,7 @@ pub(crate) async fn set_pushrule_actions_route( .set_actions(body.kind.clone(), &body.rule_id, body.actions.clone()) .is_err() { - return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")); + return Err!(Request(NotFound("Push rule not found."))); } let ty = GlobalAccountDataEventType::PushRules; @@ -400,7 +400,7 @@ pub(crate) async fn set_pushrule_enabled_route( .set_enabled(body.kind.clone(), &body.rule_id, body.enabled) .is_err() { - return Err(Error::BadRequest(ErrorKind::NotFound, "Push rule not found.")); + return Err!(Request(NotFound("Push rule not found."))); } let ty = GlobalAccountDataEventType::PushRules; diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 8ece3ab1..1019b358 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -2,7 +2,7 @@ use std::{fmt::Write as _, ops::Mul, time::Duration}; use axum::extract::State; use axum_client_ip::InsecureClientIp; -use conduwuit::{Err, Error, Result, debug_info, info, matrix::pdu::PduEvent, utils::ReadyExt}; +use conduwuit::{Err, Result, debug_info, info, matrix::pdu::PduEvent, utils::ReadyExt}; use conduwuit_service::Services; use rand::Rng; use ruma::{ @@ -44,9 +44,8 @@ pub(crate) async fn report_room_route( } if body.reason.as_ref().is_some_and(|s| s.len() > 750) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Reason too long, should be 750 characters or fewer", + return Err!(Request( + InvalidParam("Reason too long, should be 750 characters or fewer",) )); } @@ -149,9 +148,8 @@ pub(crate) async fn report_user_route( } if body.reason.as_ref().is_some_and(|s| s.len() > 750) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Reason too long, should be 750 characters or fewer", + return Err!(Request( + InvalidParam("Reason too long, should be 750 characters or fewer",) )); } @@ -204,23 +202,16 @@ async fn is_event_report_valid( ); if room_id != pdu.room_id { - return Err(Error::BadRequest( - ErrorKind::NotFound, - "Event ID does not belong to the reported room", - )); + return Err!(Request(NotFound("Event ID does not belong to the reported room",))); } if score.is_some_and(|s| s > int!(0) || s < int!(-100)) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Invalid score, must be within 0 to -100", - )); + return Err!(Request(InvalidParam("Invalid score, must be within 0 to -100",))); } if reason.as_ref().is_some_and(|s| s.len() > 750) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, - "Reason too long, should be 750 characters or fewer", + return Err!(Request( + InvalidParam("Reason too long, should be 750 characters or fewer",) )); } @@ -231,10 +222,7 @@ async fn is_event_report_valid( .ready_any(|user_id| user_id == sender_user) .await { - return Err(Error::BadRequest( - ErrorKind::NotFound, - "You are not in the room you are reporting.", - )); + return Err!(Request(NotFound("You are not in the room you are reporting.",))); } Ok(()) diff --git a/src/api/client/room/aliases.rs b/src/api/client/room/aliases.rs index a944971c..0b072b74 100644 --- a/src/api/client/room/aliases.rs +++ b/src/api/client/room/aliases.rs @@ -1,7 +1,7 @@ use axum::extract::State; -use conduwuit::{Error, Result}; +use conduwuit::{Err, Result}; use futures::StreamExt; -use ruma::api::client::{error::ErrorKind, room::aliases}; +use ruma::api::client::room::aliases; use crate::Ruma; @@ -23,10 +23,7 @@ pub(crate) async fn get_room_aliases_route( .user_can_see_state_events(sender_user, &body.room_id) .await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "You don't have permission to view this room.", - )); + return Err!(Request(Forbidden("You don't have permission to view this room.",))); } Ok(aliases::v3::Response { diff --git a/src/api/client/room/create.rs b/src/api/client/room/create.rs index 8b93fcfd..238691d1 100644 --- a/src/api/client/room/create.rs +++ b/src/api/client/room/create.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ - Err, Error, Result, debug_info, debug_warn, err, error, info, + Err, Result, debug_info, debug_warn, err, info, matrix::{StateKey, pdu::PduBuilder}, warn, }; @@ -10,10 +10,7 @@ use conduwuit_service::{Services, appservice::RegistrationInfo}; use futures::FutureExt; use ruma::{ CanonicalJsonObject, Int, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, RoomId, RoomVersionId, - api::client::{ - error::ErrorKind, - room::{self, create_room}, - }, + api::client::room::{self, create_room}, events::{ TimelineEventType, room::{ @@ -64,10 +61,7 @@ pub(crate) async fn create_room_route( && body.appservice_info.is_none() && !services.users.is_admin(sender_user).await { - return Err(Error::BadRequest( - ErrorKind::forbidden(), - "Room creation has been disabled.", - )); + return Err!(Request(Forbidden("Room creation has been disabled.",))); } if services.users.is_suspended(sender_user).await? { @@ -81,10 +75,7 @@ pub(crate) async fn create_room_route( // check if room ID doesn't already exist instead of erroring on auth check if services.rooms.short.get_shortroomid(&room_id).await.is_ok() { - return Err(Error::BadRequest( - ErrorKind::RoomInUse, - "Room with that custom room ID already exists", - )); + return Err!(Request(RoomInUse("Room with that custom room ID already exists",))); } if body.visibility == room::Visibility::Public @@ -127,10 +118,9 @@ pub(crate) async fn create_room_route( if services.server.supported_room_version(&room_version) { room_version } else { - return Err(Error::BadRequest( - ErrorKind::UnsupportedRoomVersion, - "This server does not support that room version.", - )); + return Err!(Request(UnsupportedRoomVersion( + "This server does not support that room version." + ))); }, | None => services.server.config.default_room_version.clone(), }; @@ -142,16 +132,17 @@ pub(crate) async fn create_room_route( let mut content = content .deserialize_as::() .map_err(|e| { - error!("Failed to deserialise content as canonical JSON: {}", e); - Error::bad_database("Failed to deserialise content as canonical JSON.") + err!(Request(BadJson(error!( + "Failed to deserialise content as canonical JSON: {e}" + )))) })?; + match room_version { | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { content.insert( "creator".into(), json!(&sender_user).try_into().map_err(|e| { - info!("Invalid creation content: {e}"); - Error::BadRequest(ErrorKind::BadJson, "Invalid creation content") + err!(Request(BadJson(debug_error!("Invalid creation content: {e}")))) })?, ); }, @@ -161,9 +152,9 @@ pub(crate) async fn create_room_route( } content.insert( "room_version".into(), - json!(room_version.as_str()).try_into().map_err(|_| { - Error::BadRequest(ErrorKind::BadJson, "Invalid creation content") - })?, + json!(room_version.as_str()) + .try_into() + .map_err(|e| err!(Request(BadJson("Invalid creation content: {e}"))))?, ); content }, @@ -345,8 +336,7 @@ pub(crate) async fn create_room_route( // 6. Events listed in initial_state for event in &body.initial_state { let mut pdu_builder = event.deserialize_as::().map_err(|e| { - warn!("Invalid initial state event: {:?}", e); - Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event.") + err!(Request(InvalidParam(warn!("Invalid initial state event: {e:?}")))) })?; debug_info!("Room creation initial state event: {event:?}"); @@ -355,7 +345,7 @@ pub(crate) async fn create_room_route( // state event in there with the content of literally `{}` (not null or empty // string), let's just skip it over and warn. if pdu_builder.content.get().eq("{}") { - info!("skipping empty initial state event with content of `{{}}`: {event:?}"); + debug_warn!("skipping empty initial state event with content of `{{}}`: {event:?}"); debug_warn!("content: {}", pdu_builder.content.get()); continue; } @@ -502,9 +492,7 @@ fn default_power_levels_content( if let Some(power_level_content_override) = power_level_content_override { let json: JsonObject = serde_json::from_str(power_level_content_override.json().get()) - .map_err(|_| { - Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.") - })?; + .map_err(|e| err!(Request(BadJson("Invalid power_level_content_override: {e:?}"))))?; for (key, value) in json { power_levels_content[key] = value; @@ -522,16 +510,14 @@ async fn room_alias_check( ) -> Result { // Basic checks on the room alias validity if room_alias_name.contains(':') { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, + return Err!(Request(InvalidParam( "Room alias contained `:` which is not allowed. Please note that this expects a \ localpart, not the full room alias.", - )); + ))); } else if room_alias_name.contains(char::is_whitespace) { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, + return Err!(Request(InvalidParam( "Room alias contained spaces which is not a valid room alias.", - )); + ))); } // check if room alias is forbidden @@ -540,7 +526,7 @@ async fn room_alias_check( .forbidden_alias_names() .is_match(room_alias_name) { - return Err(Error::BadRequest(ErrorKind::Unknown, "Room alias name is forbidden.")); + return Err!(Request(Unknown("Room alias name is forbidden."))); } let server_name = services.globals.server_name(); @@ -560,25 +546,19 @@ async fn room_alias_check( .await .is_ok() { - return Err(Error::BadRequest(ErrorKind::RoomInUse, "Room alias already exists.")); + return Err!(Request(RoomInUse("Room alias already exists."))); } if let Some(info) = appservice_info { if !info.aliases.is_match(full_room_alias.as_str()) { - return Err(Error::BadRequest( - ErrorKind::Exclusive, - "Room alias is not in namespace.", - )); + return Err!(Request(Exclusive("Room alias is not in namespace."))); } } else if services .appservice .is_exclusive_alias(&full_room_alias) .await { - return Err(Error::BadRequest( - ErrorKind::Exclusive, - "Room alias reserved by appservice.", - )); + return Err!(Request(Exclusive("Room alias reserved by appservice.",))); } debug_info!("Full room alias: {full_room_alias}"); @@ -594,24 +574,33 @@ fn custom_room_id_check(services: &Services, custom_room_id: &str) -> Result Result( v.insert(auth_event); }, | hash_map::Entry::Occupied(_) => { - return Err(Error::BadRequest( - ErrorKind::InvalidParam, + return Err!(Request(InvalidParam( "Auth event's type and state_key combination exists multiple times.", - )); + ))); }, } } From 3d0360bcd65ad8842120f305dfd1905e36868099 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 27 Apr 2025 00:17:39 +0000 Subject: [PATCH 1886/2091] Dedup and parallelize current key backup count and etag fetching. Signed-off-by: Jason Volk --- src/api/client/backup.rs | 134 +++++++++++++-------------------------- 1 file changed, 43 insertions(+), 91 deletions(-) diff --git a/src/api/client/backup.rs b/src/api/client/backup.rs index 2ad37cf3..a3038f26 100644 --- a/src/api/client/backup.rs +++ b/src/api/client/backup.rs @@ -2,8 +2,10 @@ use std::cmp::Ordering; use axum::extract::State; use conduwuit::{Err, Result, err}; +use conduwuit_service::Services; +use futures::{FutureExt, future::try_join}; use ruma::{ - UInt, + UInt, UserId, api::client::backup::{ add_backup_keys, add_backup_keys_for_room, add_backup_keys_for_session, create_backup_version, delete_backup_keys, delete_backup_keys_for_room, @@ -58,21 +60,9 @@ pub(crate) async fn get_latest_backup_info_route( .await .map_err(|_| err!(Request(NotFound("Key backup does not exist."))))?; - Ok(get_latest_backup_info::v3::Response { - algorithm, - count: (UInt::try_from( - services - .key_backups - .count_keys(body.sender_user(), &version) - .await, - ) - .expect("user backup keys count should not be that high")), - etag: services - .key_backups - .get_etag(body.sender_user(), &version) - .await, - version, - }) + let (count, etag) = get_count_etag(&services, body.sender_user(), &version).await?; + + Ok(get_latest_backup_info::v3::Response { algorithm, count, etag, version }) } /// # `GET /_matrix/client/v3/room_keys/version/{version}` @@ -90,17 +80,12 @@ pub(crate) async fn get_backup_info_route( err!(Request(NotFound("Key backup does not exist at version {:?}", body.version))) })?; + let (count, etag) = get_count_etag(&services, body.sender_user(), &body.version).await?; + Ok(get_backup_info::v3::Response { algorithm, - count: services - .key_backups - .count_keys(body.sender_user(), &body.version) - .await - .try_into()?, - etag: services - .key_backups - .get_etag(body.sender_user(), &body.version) - .await, + count, + etag, version: body.version.clone(), }) } @@ -155,17 +140,9 @@ pub(crate) async fn add_backup_keys_route( } } - Ok(add_backup_keys::v3::Response { - count: services - .key_backups - .count_keys(body.sender_user(), &body.version) - .await - .try_into()?, - etag: services - .key_backups - .get_etag(body.sender_user(), &body.version) - .await, - }) + let (count, etag) = get_count_etag(&services, body.sender_user(), &body.version).await?; + + Ok(add_backup_keys::v3::Response { count, etag }) } /// # `PUT /_matrix/client/r0/room_keys/keys/{roomId}` @@ -198,17 +175,9 @@ pub(crate) async fn add_backup_keys_for_room_route( .await?; } - Ok(add_backup_keys_for_room::v3::Response { - count: services - .key_backups - .count_keys(body.sender_user(), &body.version) - .await - .try_into()?, - etag: services - .key_backups - .get_etag(body.sender_user(), &body.version) - .await, - }) + let (count, etag) = get_count_etag(&services, body.sender_user(), &body.version).await?; + + Ok(add_backup_keys_for_room::v3::Response { count, etag }) } /// # `PUT /_matrix/client/r0/room_keys/keys/{roomId}/{sessionId}` @@ -306,17 +275,9 @@ pub(crate) async fn add_backup_keys_for_session_route( .await?; } - Ok(add_backup_keys_for_session::v3::Response { - count: services - .key_backups - .count_keys(body.sender_user(), &body.version) - .await - .try_into()?, - etag: services - .key_backups - .get_etag(body.sender_user(), &body.version) - .await, - }) + let (count, etag) = get_count_etag(&services, body.sender_user(), &body.version).await?; + + Ok(add_backup_keys_for_session::v3::Response { count, etag }) } /// # `GET /_matrix/client/r0/room_keys/keys` @@ -379,17 +340,9 @@ pub(crate) async fn delete_backup_keys_route( .delete_all_keys(body.sender_user(), &body.version) .await; - Ok(delete_backup_keys::v3::Response { - count: services - .key_backups - .count_keys(body.sender_user(), &body.version) - .await - .try_into()?, - etag: services - .key_backups - .get_etag(body.sender_user(), &body.version) - .await, - }) + let (count, etag) = get_count_etag(&services, body.sender_user(), &body.version).await?; + + Ok(delete_backup_keys::v3::Response { count, etag }) } /// # `DELETE /_matrix/client/r0/room_keys/keys/{roomId}` @@ -404,17 +357,9 @@ pub(crate) async fn delete_backup_keys_for_room_route( .delete_room_keys(body.sender_user(), &body.version, &body.room_id) .await; - Ok(delete_backup_keys_for_room::v3::Response { - count: services - .key_backups - .count_keys(body.sender_user(), &body.version) - .await - .try_into()?, - etag: services - .key_backups - .get_etag(body.sender_user(), &body.version) - .await, - }) + let (count, etag) = get_count_etag(&services, body.sender_user(), &body.version).await?; + + Ok(delete_backup_keys_for_room::v3::Response { count, etag }) } /// # `DELETE /_matrix/client/r0/room_keys/keys/{roomId}/{sessionId}` @@ -429,15 +374,22 @@ pub(crate) async fn delete_backup_keys_for_session_route( .delete_room_key(body.sender_user(), &body.version, &body.room_id, &body.session_id) .await; - Ok(delete_backup_keys_for_session::v3::Response { - count: services - .key_backups - .count_keys(body.sender_user(), &body.version) - .await - .try_into()?, - etag: services - .key_backups - .get_etag(body.sender_user(), &body.version) - .await, - }) + let (count, etag) = get_count_etag(&services, body.sender_user(), &body.version).await?; + + Ok(delete_backup_keys_for_session::v3::Response { count, etag }) +} + +async fn get_count_etag( + services: &Services, + sender_user: &UserId, + version: &str, +) -> Result<(UInt, String)> { + let count = services + .key_backups + .count_keys(sender_user, version) + .map(TryInto::try_into); + + let etag = services.key_backups.get_etag(sender_user, version).map(Ok); + + Ok(try_join(count, etag).await?) } From 116f85360fa17b16e8e9353b20be5e21dc87e2de Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sat, 26 Apr 2025 08:24:47 +0000 Subject: [PATCH 1887/2091] Toward abstracting Pdu into trait Event. Co-authored-by: Jade Ellis Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 11 +- src/admin/user/commands.rs | 8 +- src/api/client/context.rs | 12 +- src/api/client/membership.rs | 5 +- src/api/client/message.rs | 4 +- src/api/client/relations.rs | 4 +- src/api/client/room/event.rs | 2 +- src/api/client/room/initial_sync.rs | 6 +- src/api/client/search.rs | 6 +- src/api/client/state.rs | 18 +- src/api/client/sync/v3.rs | 13 +- src/api/client/sync/v4.rs | 7 +- src/api/client/sync/v5.rs | 12 +- src/api/client/threads.rs | 7 +- src/api/server/invite.rs | 7 +- src/core/matrix/event.rs | 141 +++++++--- src/core/matrix/event/content.rs | 21 ++ src/core/matrix/event/format.rs | 219 +++++++++++++++ src/core/matrix/event/redact.rs | 86 ++++++ src/core/matrix/event/type_ext.rs | 32 +++ src/core/matrix/mod.rs | 8 +- src/core/matrix/pdu.rs | 123 +++++++-- src/core/matrix/pdu/id.rs | 1 + src/core/matrix/pdu/redact.rs | 116 +------- src/core/matrix/pdu/strip.rs | 257 ------------------ src/core/matrix/{pdu => }/state_key.rs | 3 - src/core/matrix/state_res/benches.rs | 167 ++---------- src/core/matrix/state_res/event_auth.rs | 61 +++-- src/core/matrix/state_res/mod.rs | 29 +- src/core/matrix/state_res/test_utils.rs | 192 +++---------- src/core/mod.rs | 4 +- src/service/admin/mod.rs | 21 +- src/service/pusher/mod.rs | 56 ++-- .../rooms/event_handler/handle_outlier_pdu.rs | 2 +- .../event_handler/upgrade_outlier_pdu.rs | 4 +- src/service/rooms/search/mod.rs | 4 +- src/service/rooms/spaces/mod.rs | 8 +- src/service/rooms/state/mod.rs | 22 +- src/service/rooms/threads/mod.rs | 16 +- src/service/rooms/timeline/mod.rs | 7 +- src/service/sending/sender.rs | 6 +- 41 files changed, 842 insertions(+), 886 deletions(-) create mode 100644 src/core/matrix/event/content.rs create mode 100644 src/core/matrix/event/format.rs create mode 100644 src/core/matrix/event/redact.rs create mode 100644 src/core/matrix/event/type_ext.rs delete mode 100644 src/core/matrix/pdu/strip.rs rename src/core/matrix/{pdu => }/state_key.rs (67%) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index a397e0fc..2323e3b8 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -7,7 +7,10 @@ use std::{ use conduwuit::{ Err, Result, debug_error, err, info, - matrix::pdu::{PduEvent, PduId, RawPduId}, + matrix::{ + Event, + pdu::{PduEvent, PduId, RawPduId}, + }, trace, utils, utils::{ stream::{IterStream, ReadyExt}, @@ -19,7 +22,7 @@ use futures::{FutureExt, StreamExt, TryStreamExt}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedServerName, RoomId, RoomVersionId, - api::federation::event::get_room_state, + api::federation::event::get_room_state, events::AnyStateEvent, serde::Raw, }; use service::rooms::{ short::{ShortEventId, ShortRoomId}, @@ -296,12 +299,12 @@ pub(super) async fn get_remote_pdu( #[admin_command] pub(super) async fn get_room_state(&self, room: OwnedRoomOrAliasId) -> Result { let room_id = self.services.rooms.alias.resolve(&room).await?; - let room_state: Vec<_> = self + let room_state: Vec> = self .services .rooms .state_accessor .room_state_full_pdus(&room_id) - .map_ok(PduEvent::into_state_event) + .map_ok(Event::into_format) .try_collect() .await?; diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 89f7a9fc..3750d758 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -1,13 +1,15 @@ use std::{collections::BTreeMap, fmt::Write as _}; -use api::client::{full_user_deactivate, join_room_by_id_helper, leave_room}; +use api::client::{ + full_user_deactivate, join_room_by_id_helper, leave_all_rooms, leave_room, update_avatar_url, + update_displayname, +}; use conduwuit::{ Err, Result, debug, debug_warn, error, info, is_equal_to, - matrix::pdu::PduBuilder, + matrix::{Event, pdu::PduBuilder}, utils::{self, ReadyExt}, warn, }; -use conduwuit_api::client::{leave_all_rooms, update_avatar_url, update_displayname}; use futures::{FutureExt, StreamExt}; use ruma::{ OwnedEventId, OwnedRoomId, OwnedRoomOrAliasId, OwnedUserId, UserId, diff --git a/src/api/client/context.rs b/src/api/client/context.rs index ca787a16..4a7d34d2 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -1,8 +1,6 @@ use axum::extract::State; use conduwuit::{ - Err, Result, at, debug_warn, err, - matrix::pdu::PduEvent, - ref_at, + Err, Event, Result, at, debug_warn, err, ref_at, utils::{ IterStream, future::TryExtExt, @@ -179,12 +177,12 @@ pub(crate) async fn get_context_route( .broad_filter_map(|event_id: &OwnedEventId| { services.rooms.timeline.get_pdu(event_id.as_ref()).ok() }) - .map(PduEvent::into_state_event) + .map(Event::into_format) .collect() .await; Ok(get_context::v3::Response { - event: base_event.map(at!(1)).map(PduEvent::into_room_event), + event: base_event.map(at!(1)).map(Event::into_format), start: events_before .last() @@ -203,13 +201,13 @@ pub(crate) async fn get_context_route( events_before: events_before .into_iter() .map(at!(1)) - .map(PduEvent::into_room_event) + .map(Event::into_format) .collect(), events_after: events_after .into_iter() .map(at!(1)) - .map(PduEvent::into_room_event) + .map(Event::into_format) .collect(), state, diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs index 85d0cd21..3c2a6fe3 100644 --- a/src/api/client/membership.rs +++ b/src/api/client/membership.rs @@ -9,7 +9,8 @@ use std::{ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Err, Result, at, debug, debug_error, debug_info, debug_warn, err, error, info, is_matching, + Err, Event, Result, at, debug, debug_error, debug_info, debug_warn, err, error, info, + is_matching, matrix::{ StateKey, pdu::{PduBuilder, PduEvent, gen_event_id, gen_event_id_canonical_json}, @@ -880,7 +881,7 @@ pub(crate) async fn get_member_events_route( .ready_filter(|((ty, _), _)| *ty == StateEventType::RoomMember) .map(at!(1)) .ready_filter_map(|pdu| membership_filter(pdu, membership, not_membership)) - .map(PduEvent::into_member_event) + .map(Event::into_format) .collect() .await, }) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 7a87a9b0..e32d020f 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -175,7 +175,7 @@ pub(crate) async fn get_message_events_route( let chunk = events .into_iter() .map(at!(1)) - .map(PduEvent::into_room_event) + .map(Event::into_format) .collect(); Ok(get_message_events::v3::Response { @@ -241,7 +241,7 @@ async fn get_member_event( .rooms .state_accessor .room_state_get(room_id, &StateEventType::RoomMember, user_id.as_str()) - .map_ok(PduEvent::into_state_event) + .map_ok(Event::into_format) .await .ok() } diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index b8c2dd4d..ad726b90 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,7 +1,7 @@ use axum::extract::State; use conduwuit::{ Result, at, - matrix::pdu::PduCount, + matrix::{Event, pdu::PduCount}, utils::{IterStream, ReadyExt, result::FlatOk, stream::WidebandExt}, }; use conduwuit_service::{Services, rooms::timeline::PdusIterItem}; @@ -167,7 +167,7 @@ async fn paginate_relations_with_filter( chunk: events .into_iter() .map(at!(1)) - .map(|pdu| pdu.to_message_like_event()) + .map(Event::into_format) .collect(), }) } diff --git a/src/api/client/room/event.rs b/src/api/client/room/event.rs index 2b115b5c..47228d67 100644 --- a/src/api/client/room/event.rs +++ b/src/api/client/room/event.rs @@ -40,5 +40,5 @@ pub(crate) async fn get_room_event_route( event.add_age().ok(); - Ok(get_room_event::v3::Response { event: event.into_room_event() }) + Ok(get_room_event::v3::Response { event: event.into_format() }) } diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index ca63610b..8b9f3ca0 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -1,6 +1,6 @@ use axum::extract::State; use conduwuit::{ - Err, PduEvent, Result, at, + Err, Event, Result, at, utils::{BoolExt, stream::TryTools}, }; use futures::TryStreamExt; @@ -38,7 +38,7 @@ pub(crate) async fn room_initial_sync_route( .rooms .state_accessor .room_state_full_pdus(room_id) - .map_ok(PduEvent::into_state_event) + .map_ok(Event::into_format) .try_collect() .await?; @@ -55,7 +55,7 @@ pub(crate) async fn room_initial_sync_route( chunk: events .into_iter() .map(at!(1)) - .map(PduEvent::into_room_event) + .map(Event::into_format) .collect(), }; diff --git a/src/api/client/search.rs b/src/api/client/search.rs index d4dcde57..cc745694 100644 --- a/src/api/client/search.rs +++ b/src/api/client/search.rs @@ -3,7 +3,7 @@ use std::collections::BTreeMap; use axum::extract::State; use conduwuit::{ Err, Result, at, is_true, - matrix::pdu::PduEvent, + matrix::Event, result::FlatOk, utils::{IterStream, stream::ReadyExt}, }; @@ -144,7 +144,7 @@ async fn category_room_events( .map(at!(2)) .flatten() .stream() - .map(PduEvent::into_room_event) + .map(Event::into_format) .map(|result| SearchResult { rank: None, result: Some(result), @@ -185,7 +185,7 @@ async fn procure_room_state(services: &Services, room_id: &RoomId) -> Result>(); let account_data_events = services @@ -877,10 +875,7 @@ async fn load_joined_room( events: room_events, }, state: RoomState { - events: state_events - .into_iter() - .map(PduEvent::into_sync_state_event) - .collect(), + events: state_events.into_iter().map(Event::into_format).collect(), }, ephemeral: Ephemeral { events: edus }, unread_thread_notifications: BTreeMap::new(), diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index f153b2da..cabd67e4 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -6,7 +6,7 @@ use std::{ use axum::extract::State; use conduwuit::{ - Err, Error, PduCount, PduEvent, Result, debug, error, extract_variant, + Err, Error, Event, PduCount, PduEvent, Result, at, debug, error, extract_variant, matrix::TypeStateKey, utils::{ BoolExt, IterStream, ReadyExt, TryFutureExtExt, @@ -604,7 +604,8 @@ pub(crate) async fn sync_events_v4_route( .iter() .stream() .filter_map(|item| ignored_filter(&services, item.clone(), sender_user)) - .map(|(_, pdu)| pdu.to_sync_room_event()) + .map(at!(1)) + .map(Event::into_format) .collect() .await; @@ -626,7 +627,7 @@ pub(crate) async fn sync_events_v4_route( .state_accessor .room_state_get(room_id, &state.0, &state.1) .await - .map(PduEvent::into_sync_state_event) + .map(PduEvent::into_format) .ok() }) .collect() diff --git a/src/api/client/sync/v5.rs b/src/api/client/sync/v5.rs index f3fc0f44..e4cefba0 100644 --- a/src/api/client/sync/v5.rs +++ b/src/api/client/sync/v5.rs @@ -7,11 +7,8 @@ use std::{ use axum::extract::State; use conduwuit::{ - Err, Error, Result, error, extract_variant, is_equal_to, - matrix::{ - TypeStateKey, - pdu::{PduCount, PduEvent}, - }, + Err, Error, Result, at, error, extract_variant, is_equal_to, + matrix::{Event, TypeStateKey, pdu::PduCount}, trace, utils::{ BoolExt, FutureBoolExt, IterStream, ReadyExt, TryFutureExtExt, @@ -515,7 +512,8 @@ where .iter() .stream() .filter_map(|item| ignored_filter(services, item.clone(), sender_user)) - .map(|(_, pdu)| pdu.to_sync_room_event()) + .map(at!(1)) + .map(Event::into_format) .collect() .await; @@ -537,7 +535,7 @@ where .state_accessor .room_state_get(room_id, &state.0, &state.1) .await - .map(PduEvent::into_sync_state_event) + .map(Event::into_format) .ok() }) .collect() diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index 5b838bef..ca176eda 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -1,7 +1,10 @@ use axum::extract::State; use conduwuit::{ Result, at, - matrix::pdu::{PduCount, PduEvent}, + matrix::{ + Event, + pdu::{PduCount, PduEvent}, + }, }; use futures::StreamExt; use ruma::{api::client::threads::get_threads, uint}; @@ -56,7 +59,7 @@ pub(crate) async fn get_threads_route( chunk: threads .into_iter() .map(at!(1)) - .map(PduEvent::into_room_event) + .map(Event::into_format) .collect(), }) } diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 01961378..0d26d787 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -2,7 +2,8 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use base64::{Engine as _, engine::general_purpose}; use conduwuit::{ - Err, Error, PduEvent, Result, err, pdu::gen_event_id, utils, utils::hash::sha256, warn, + Err, Error, PduEvent, Result, err, matrix::Event, pdu::gen_event_id, utils, + utils::hash::sha256, warn, }; use ruma::{ CanonicalJsonValue, OwnedUserId, UserId, @@ -111,7 +112,7 @@ pub(crate) async fn create_invite_route( let pdu: PduEvent = serde_json::from_value(event.into()) .map_err(|e| err!(Request(BadJson("Invalid invite event PDU: {e}"))))?; - invite_state.push(pdu.to_stripped_state_event()); + invite_state.push(pdu.to_format()); // If we are active in the room, the remote server will notify us about the // join/invite through /send. If we are not in the room, we need to manually @@ -144,7 +145,7 @@ pub(crate) async fn create_invite_route( .send_appservice_request( appservice.registration.clone(), ruma::api::appservice::event::push_events::v1::Request { - events: vec![pdu.to_room_event()], + events: vec![pdu.to_format()], txn_id: general_purpose::URL_SAFE_NO_PAD .encode(sha256::hash(pdu.event_id.as_bytes())) .into(), diff --git a/src/core/matrix/event.rs b/src/core/matrix/event.rs index e4c478cd..5b12770b 100644 --- a/src/core/matrix/event.rs +++ b/src/core/matrix/event.rs @@ -1,63 +1,114 @@ -use ruma::{EventId, MilliSecondsSinceUnixEpoch, RoomId, UserId, events::TimelineEventType}; -use serde_json::value::RawValue as RawJsonValue; +mod content; +mod format; +mod redact; +mod type_ext; + +use ruma::{ + EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, UserId, + events::TimelineEventType, +}; +use serde::Deserialize; +use serde_json::{Value as JsonValue, value::RawValue as RawJsonValue}; + +pub use self::type_ext::TypeExt; +use super::state_key::StateKey; +use crate::Result; /// Abstraction of a PDU so users can have their own PDU types. pub trait Event { + /// Serialize into a Ruma JSON format, consuming. + #[inline] + fn into_format(self) -> T + where + T: From>, + Self: Sized, + { + format::Owned(self).into() + } + + /// Serialize into a Ruma JSON format + #[inline] + fn to_format<'a, T>(&'a self) -> T + where + T: From>, + Self: Sized + 'a, + { + format::Ref(self).into() + } + + #[inline] + fn get_content_as_value(&self) -> JsonValue + where + Self: Sized, + { + content::as_value(self) + } + + #[inline] + fn get_content(&self) -> Result + where + for<'de> T: Deserialize<'de>, + Self: Sized, + { + content::get::(self) + } + + #[inline] + fn redacts_id(&self, room_version: &RoomVersionId) -> Option + where + Self: Sized, + { + redact::redacts_id(self, room_version) + } + + #[inline] + fn is_redacted(&self) -> bool + where + Self: Sized, + { + redact::is_redacted(self) + } + + fn is_owned(&self) -> bool; + + // + // Canonical properties + // + + /// All the authenticating events for this event. + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_; + + /// The event's content. + fn content(&self) -> &RawJsonValue; + /// The `EventId` of this event. fn event_id(&self) -> &EventId; + /// The time of creation on the originating server. + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch; + + /// The events before this event. + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_; + + /// If this event is a redaction event this is the event it redacts. + fn redacts(&self) -> Option<&EventId>; + /// The `RoomId` of this event. fn room_id(&self) -> &RoomId; /// The `UserId` of this event. fn sender(&self) -> &UserId; - /// The time of creation on the originating server. - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch; - - /// The event type. - fn event_type(&self) -> &TimelineEventType; - - /// The event's content. - fn content(&self) -> &RawJsonValue; - /// The state key for this event. fn state_key(&self) -> Option<&str>; - /// The events before this event. - // Requires GATs to avoid boxing (and TAIT for making it convenient). - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_; + /// The event type. + fn kind(&self) -> &TimelineEventType; - /// All the authenticating events for this event. - // Requires GATs to avoid boxing (and TAIT for making it convenient). - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_; + /// Metadata container; peer-trusted only. + fn unsigned(&self) -> Option<&RawJsonValue>; - /// If this event is a redaction event this is the event it redacts. - fn redacts(&self) -> Option<&EventId>; -} - -impl Event for &T { - fn event_id(&self) -> &EventId { (*self).event_id() } - - fn room_id(&self) -> &RoomId { (*self).room_id() } - - fn sender(&self) -> &UserId { (*self).sender() } - - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { (*self).origin_server_ts() } - - fn event_type(&self) -> &TimelineEventType { (*self).event_type() } - - fn content(&self) -> &RawJsonValue { (*self).content() } - - fn state_key(&self) -> Option<&str> { (*self).state_key() } - - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { - (*self).prev_events() - } - - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { - (*self).auth_events() - } - - fn redacts(&self) -> Option<&EventId> { (*self).redacts() } + //#[deprecated] + #[inline] + fn event_type(&self) -> &TimelineEventType { self.kind() } } diff --git a/src/core/matrix/event/content.rs b/src/core/matrix/event/content.rs new file mode 100644 index 00000000..1ee7ebd2 --- /dev/null +++ b/src/core/matrix/event/content.rs @@ -0,0 +1,21 @@ +use serde::Deserialize; +use serde_json::value::Value as JsonValue; + +use super::Event; +use crate::{Result, err}; + +#[inline] +#[must_use] +pub(super) fn as_value(event: &E) -> JsonValue { + get(event).expect("Failed to represent Event content as JsonValue") +} + +#[inline] +pub(super) fn get(event: &E) -> Result +where + T: for<'de> Deserialize<'de>, + E: Event, +{ + serde_json::from_str(event.content().get()) + .map_err(|e| err!(Request(BadJson("Failed to deserialize content into type: {e}")))) +} diff --git a/src/core/matrix/event/format.rs b/src/core/matrix/event/format.rs new file mode 100644 index 00000000..988cf4f0 --- /dev/null +++ b/src/core/matrix/event/format.rs @@ -0,0 +1,219 @@ +use ruma::{ + events::{ + AnyMessageLikeEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncStateEvent, + AnySyncTimelineEvent, AnyTimelineEvent, StateEvent, room::member::RoomMemberEventContent, + space::child::HierarchySpaceChildEvent, + }, + serde::Raw, +}; +use serde_json::json; + +use super::{Event, redact}; + +pub struct Owned(pub(super) E); + +pub struct Ref<'a, E: Event>(pub(super) &'a E); + +impl From> for Raw { + fn from(event: Owned) -> Self { Ref(&event.0).into() } +} + +impl<'a, E: Event> From> for Raw { + fn from(event: Ref<'a, E>) -> Self { + let event = event.0; + let (redacts, content) = redact::copy(event); + let mut json = json!({ + "content": content, + "event_id": event.event_id(), + "origin_server_ts": event.origin_server_ts(), + "sender": event.sender(), + "type": event.event_type(), + }); + + if let Some(redacts) = redacts { + json["redacts"] = json!(redacts); + } + if let Some(state_key) = event.state_key() { + json["state_key"] = json!(state_key); + } + if let Some(unsigned) = event.unsigned() { + json["unsigned"] = json!(unsigned); + } + + serde_json::from_value(json).expect("Failed to serialize Event value") + } +} + +impl From> for Raw { + fn from(event: Owned) -> Self { Ref(&event.0).into() } +} + +impl<'a, E: Event> From> for Raw { + fn from(event: Ref<'a, E>) -> Self { + let event = event.0; + let (redacts, content) = redact::copy(event); + let mut json = json!({ + "content": content, + "event_id": event.event_id(), + "origin_server_ts": event.origin_server_ts(), + "room_id": event.room_id(), + "sender": event.sender(), + "type": event.kind(), + }); + + if let Some(redacts) = redacts { + json["redacts"] = json!(redacts); + } + if let Some(state_key) = event.state_key() { + json["state_key"] = json!(state_key); + } + if let Some(unsigned) = event.unsigned() { + json["unsigned"] = json!(unsigned); + } + + serde_json::from_value(json).expect("Failed to serialize Event value") + } +} + +impl From> for Raw { + fn from(event: Owned) -> Self { Ref(&event.0).into() } +} + +impl<'a, E: Event> From> for Raw { + fn from(event: Ref<'a, E>) -> Self { + let event = event.0; + let (redacts, content) = redact::copy(event); + let mut json = json!({ + "content": content, + "event_id": event.event_id(), + "origin_server_ts": event.origin_server_ts(), + "room_id": event.room_id(), + "sender": event.sender(), + "type": event.kind(), + }); + + if let Some(redacts) = &redacts { + json["redacts"] = json!(redacts); + } + if let Some(state_key) = event.state_key() { + json["state_key"] = json!(state_key); + } + if let Some(unsigned) = event.unsigned() { + json["unsigned"] = json!(unsigned); + } + + serde_json::from_value(json).expect("Failed to serialize Event value") + } +} + +impl From> for Raw { + fn from(event: Owned) -> Self { Ref(&event.0).into() } +} + +impl<'a, E: Event> From> for Raw { + fn from(event: Ref<'a, E>) -> Self { + let event = event.0; + let mut json = json!({ + "content": event.content(), + "event_id": event.event_id(), + "origin_server_ts": event.origin_server_ts(), + "room_id": event.room_id(), + "sender": event.sender(), + "state_key": event.state_key(), + "type": event.kind(), + }); + + if let Some(unsigned) = event.unsigned() { + json["unsigned"] = json!(unsigned); + } + + serde_json::from_value(json).expect("Failed to serialize Event value") + } +} + +impl From> for Raw { + fn from(event: Owned) -> Self { Ref(&event.0).into() } +} + +impl<'a, E: Event> From> for Raw { + fn from(event: Ref<'a, E>) -> Self { + let event = event.0; + let mut json = json!({ + "content": event.content(), + "event_id": event.event_id(), + "origin_server_ts": event.origin_server_ts(), + "sender": event.sender(), + "state_key": event.state_key(), + "type": event.kind(), + }); + + if let Some(unsigned) = event.unsigned() { + json["unsigned"] = json!(unsigned); + } + + serde_json::from_value(json).expect("Failed to serialize Event value") + } +} + +impl From> for Raw { + fn from(event: Owned) -> Self { Ref(&event.0).into() } +} + +impl<'a, E: Event> From> for Raw { + fn from(event: Ref<'a, E>) -> Self { + let event = event.0; + let json = json!({ + "content": event.content(), + "sender": event.sender(), + "state_key": event.state_key(), + "type": event.kind(), + }); + + serde_json::from_value(json).expect("Failed to serialize Event value") + } +} + +impl From> for Raw { + fn from(event: Owned) -> Self { Ref(&event.0).into() } +} + +impl<'a, E: Event> From> for Raw { + fn from(event: Ref<'a, E>) -> Self { + let event = event.0; + let json = json!({ + "content": event.content(), + "origin_server_ts": event.origin_server_ts(), + "sender": event.sender(), + "state_key": event.state_key(), + "type": event.kind(), + }); + + serde_json::from_value(json).expect("Failed to serialize Event value") + } +} + +impl From> for Raw> { + fn from(event: Owned) -> Self { Ref(&event.0).into() } +} + +impl<'a, E: Event> From> for Raw> { + fn from(event: Ref<'a, E>) -> Self { + let event = event.0; + let mut json = json!({ + "content": event.content(), + "event_id": event.event_id(), + "origin_server_ts": event.origin_server_ts(), + "redacts": event.redacts(), + "room_id": event.room_id(), + "sender": event.sender(), + "state_key": event.state_key(), + "type": event.kind(), + }); + + if let Some(unsigned) = event.unsigned() { + json["unsigned"] = json!(unsigned); + } + + serde_json::from_value(json).expect("Failed to serialize Event value") + } +} diff --git a/src/core/matrix/event/redact.rs b/src/core/matrix/event/redact.rs new file mode 100644 index 00000000..5deac874 --- /dev/null +++ b/src/core/matrix/event/redact.rs @@ -0,0 +1,86 @@ +use ruma::{ + OwnedEventId, RoomVersionId, + events::{TimelineEventType, room::redaction::RoomRedactionEventContent}, +}; +use serde::Deserialize; +use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; + +use super::Event; + +/// Copies the `redacts` property of the event to the `content` dict and +/// vice-versa. +/// +/// This follows the specification's +/// [recommendation](https://spec.matrix.org/v1.10/rooms/v11/#moving-the-redacts-property-of-mroomredaction-events-to-a-content-property): +/// +/// > For backwards-compatibility with older clients, servers should add a +/// > redacts property to the top level of m.room.redaction events in when +/// > serving such events over the Client-Server API. +/// +/// > For improved compatibility with newer clients, servers should add a +/// > redacts property to the content of m.room.redaction events in older +/// > room versions when serving such events over the Client-Server API. +#[must_use] +pub(super) fn copy(event: &E) -> (Option, Box) { + if *event.event_type() != TimelineEventType::RoomRedaction { + return (event.redacts().map(ToOwned::to_owned), event.content().to_owned()); + } + + let Ok(mut content) = event.get_content::() else { + return (event.redacts().map(ToOwned::to_owned), event.content().to_owned()); + }; + + if let Some(redacts) = content.redacts { + return (Some(redacts), event.content().to_owned()); + } + + if let Some(redacts) = event.redacts().map(ToOwned::to_owned) { + content.redacts = Some(redacts); + return ( + event.redacts().map(ToOwned::to_owned), + to_raw_value(&content).expect("Must be valid, we only added redacts field"), + ); + } + + (event.redacts().map(ToOwned::to_owned), event.content().to_owned()) +} + +#[must_use] +pub(super) fn is_redacted(event: &E) -> bool { + let Some(unsigned) = event.unsigned() else { + return false; + }; + + let Ok(unsigned) = ExtractRedactedBecause::deserialize(unsigned) else { + return false; + }; + + unsigned.redacted_because.is_some() +} + +#[must_use] +pub(super) fn redacts_id( + event: &E, + room_version: &RoomVersionId, +) -> Option { + use RoomVersionId::*; + + if *event.kind() != TimelineEventType::RoomRedaction { + return None; + } + + match *room_version { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => + event.redacts().map(ToOwned::to_owned), + | _ => + event + .get_content::() + .ok()? + .redacts, + } +} + +#[derive(Deserialize)] +struct ExtractRedactedBecause { + redacted_because: Option, +} diff --git a/src/core/matrix/event/type_ext.rs b/src/core/matrix/event/type_ext.rs new file mode 100644 index 00000000..9b824d41 --- /dev/null +++ b/src/core/matrix/event/type_ext.rs @@ -0,0 +1,32 @@ +use ruma::events::{StateEventType, TimelineEventType}; + +use super::StateKey; + +/// Convenience trait for adding event type plus state key to state maps. +pub trait TypeExt { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, StateKey); +} + +impl TypeExt for StateEventType { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, StateKey) { + (self, state_key.into()) + } +} + +impl TypeExt for &StateEventType { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, StateKey) { + (self.clone(), state_key.into()) + } +} + +impl TypeExt for TimelineEventType { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, StateKey) { + (self.into(), state_key.into()) + } +} + +impl TypeExt for &TimelineEventType { + fn with_state_key(self, state_key: impl Into) -> (StateEventType, StateKey) { + (self.clone().into(), state_key.into()) + } +} diff --git a/src/core/matrix/mod.rs b/src/core/matrix/mod.rs index 8c978173..b38d4c9a 100644 --- a/src/core/matrix/mod.rs +++ b/src/core/matrix/mod.rs @@ -2,8 +2,10 @@ pub mod event; pub mod pdu; +pub mod state_key; pub mod state_res; -pub use event::Event; -pub use pdu::{PduBuilder, PduCount, PduEvent, PduId, RawPduId, StateKey}; -pub use state_res::{EventTypeExt, RoomVersion, StateMap, TypeStateKey}; +pub use event::{Event, TypeExt as EventTypeExt}; +pub use pdu::{Pdu, PduBuilder, PduCount, PduEvent, PduId, RawPduId, ShortId}; +pub use state_key::StateKey; +pub use state_res::{RoomVersion, StateMap, TypeStateKey}; diff --git a/src/core/matrix/pdu.rs b/src/core/matrix/pdu.rs index 188586bd..e64baeb8 100644 --- a/src/core/matrix/pdu.rs +++ b/src/core/matrix/pdu.rs @@ -7,8 +7,6 @@ mod id; mod raw_id; mod redact; mod relation; -mod state_key; -mod strip; #[cfg(test)] mod tests; mod unsigned; @@ -27,37 +25,50 @@ pub use self::{ builder::{Builder, Builder as PduBuilder}, count::Count, event_id::*, - id::*, + id::{ShortId, *}, raw_id::*, - state_key::{ShortStateKey, StateKey}, }; -use super::Event; +use super::{Event, StateKey}; use crate::Result; /// Persistent Data Unit (Event) #[derive(Clone, Deserialize, Serialize, Debug)] pub struct Pdu { pub event_id: OwnedEventId, + pub room_id: OwnedRoomId, + pub sender: OwnedUserId, + #[serde(skip_serializing_if = "Option::is_none")] pub origin: Option, + pub origin_server_ts: UInt, + #[serde(rename = "type")] pub kind: TimelineEventType, + pub content: Box, + #[serde(skip_serializing_if = "Option::is_none")] pub state_key: Option, + pub prev_events: Vec, + pub depth: UInt, + pub auth_events: Vec, + #[serde(skip_serializing_if = "Option::is_none")] pub redacts: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub unsigned: Option>, + pub hashes: EventHash, - #[serde(default, skip_serializing_if = "Option::is_none")] + // BTreeMap, BTreeMap> + #[serde(default, skip_serializing_if = "Option::is_none")] pub signatures: Option>, } @@ -79,31 +90,91 @@ impl Pdu { } impl Event for Pdu { - fn event_id(&self) -> &EventId { &self.event_id } - - fn room_id(&self) -> &RoomId { &self.room_id } - - fn sender(&self) -> &UserId { &self.sender } - - fn event_type(&self) -> &TimelineEventType { &self.kind } - - fn content(&self) -> &RawJsonValue { &self.content } - - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { - MilliSecondsSinceUnixEpoch(self.origin_server_ts) - } - - fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } - - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { - self.prev_events.iter().map(AsRef::as_ref) - } - + #[inline] fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { self.auth_events.iter().map(AsRef::as_ref) } + #[inline] + fn content(&self) -> &RawJsonValue { &self.content } + + #[inline] + fn event_id(&self) -> &EventId { &self.event_id } + + #[inline] + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { + MilliSecondsSinceUnixEpoch(self.origin_server_ts) + } + + #[inline] + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + self.prev_events.iter().map(AsRef::as_ref) + } + + #[inline] fn redacts(&self) -> Option<&EventId> { self.redacts.as_deref() } + + #[inline] + fn room_id(&self) -> &RoomId { &self.room_id } + + #[inline] + fn sender(&self) -> &UserId { &self.sender } + + #[inline] + fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } + + #[inline] + fn kind(&self) -> &TimelineEventType { &self.kind } + + #[inline] + fn unsigned(&self) -> Option<&RawJsonValue> { self.unsigned.as_deref() } + + #[inline] + fn is_owned(&self) -> bool { true } +} + +impl Event for &Pdu { + #[inline] + fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { + self.auth_events.iter().map(AsRef::as_ref) + } + + #[inline] + fn content(&self) -> &RawJsonValue { &self.content } + + #[inline] + fn event_id(&self) -> &EventId { &self.event_id } + + #[inline] + fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { + MilliSecondsSinceUnixEpoch(self.origin_server_ts) + } + + #[inline] + fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + self.prev_events.iter().map(AsRef::as_ref) + } + + #[inline] + fn redacts(&self) -> Option<&EventId> { self.redacts.as_deref() } + + #[inline] + fn room_id(&self) -> &RoomId { &self.room_id } + + #[inline] + fn sender(&self) -> &UserId { &self.sender } + + #[inline] + fn state_key(&self) -> Option<&str> { self.state_key.as_deref() } + + #[inline] + fn kind(&self) -> &TimelineEventType { &self.kind } + + #[inline] + fn unsigned(&self) -> Option<&RawJsonValue> { self.unsigned.as_deref() } + + #[inline] + fn is_owned(&self) -> bool { false } } /// Prevent derived equality which wouldn't limit itself to event_id diff --git a/src/core/matrix/pdu/id.rs b/src/core/matrix/pdu/id.rs index 0b23a29f..896d677b 100644 --- a/src/core/matrix/pdu/id.rs +++ b/src/core/matrix/pdu/id.rs @@ -3,6 +3,7 @@ use crate::utils::u64_from_u8x8; pub type ShortRoomId = ShortId; pub type ShortEventId = ShortId; +pub type ShortStateKey = ShortId; pub type ShortId = u64; #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/src/core/matrix/pdu/redact.rs b/src/core/matrix/pdu/redact.rs index 409debfe..e6a03209 100644 --- a/src/core/matrix/pdu/redact.rs +++ b/src/core/matrix/pdu/redact.rs @@ -1,117 +1,29 @@ -use ruma::{ - OwnedEventId, RoomVersionId, - canonical_json::redact_content_in_place, - events::{TimelineEventType, room::redaction::RoomRedactionEventContent}, -}; -use serde::Deserialize; -use serde_json::{ - json, - value::{RawValue as RawJsonValue, to_raw_value}, -}; +use ruma::{RoomVersionId, canonical_json::redact_content_in_place}; +use serde_json::{json, value::to_raw_value}; -use crate::{Error, Result, implement}; - -#[derive(Deserialize)] -struct ExtractRedactedBecause { - redacted_because: Option, -} +use crate::{Error, Result, err, implement}; #[implement(super::Pdu)] pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: &Self) -> Result { self.unsigned = None; let mut content = serde_json::from_str(self.content.get()) - .map_err(|_| Error::bad_database("PDU in db has invalid content."))?; + .map_err(|e| err!(Request(BadJson("Failed to deserialize content into type: {e}"))))?; redact_content_in_place(&mut content, room_version_id, self.kind.to_string()) .map_err(|e| Error::Redaction(self.sender.server_name().to_owned(), e))?; - self.unsigned = Some( - to_raw_value(&json!({ - "redacted_because": serde_json::to_value(reason).expect("to_value(Pdu) always works") - })) - .expect("to string always works"), - ); + let reason = serde_json::to_value(reason).expect("Failed to preserialize reason"); - self.content = to_raw_value(&content).expect("to string always works"); + let redacted_because = json!({ + "redacted_because": reason, + }); + + self.unsigned = to_raw_value(&redacted_because) + .expect("Failed to serialize unsigned") + .into(); + + self.content = to_raw_value(&content).expect("Failed to serialize content"); Ok(()) } - -#[implement(super::Pdu)] -#[must_use] -pub fn is_redacted(&self) -> bool { - let Some(unsigned) = &self.unsigned else { - return false; - }; - - let Ok(unsigned) = ExtractRedactedBecause::deserialize(&**unsigned) else { - return false; - }; - - unsigned.redacted_because.is_some() -} - -/// Copies the `redacts` property of the event to the `content` dict and -/// vice-versa. -/// -/// This follows the specification's -/// [recommendation](https://spec.matrix.org/v1.10/rooms/v11/#moving-the-redacts-property-of-mroomredaction-events-to-a-content-property): -/// -/// > For backwards-compatibility with older clients, servers should add a -/// > redacts -/// > property to the top level of m.room.redaction events in when serving -/// > such events -/// > over the Client-Server API. -/// -/// > For improved compatibility with newer clients, servers should add a -/// > redacts property -/// > to the content of m.room.redaction events in older room versions when -/// > serving -/// > such events over the Client-Server API. -#[implement(super::Pdu)] -#[must_use] -pub fn copy_redacts(&self) -> (Option, Box) { - if self.kind == TimelineEventType::RoomRedaction { - if let Ok(mut content) = - serde_json::from_str::(self.content.get()) - { - match content.redacts { - | Some(redacts) => { - return (Some(redacts), self.content.clone()); - }, - | _ => match self.redacts.clone() { - | Some(redacts) => { - content.redacts = Some(redacts); - return ( - self.redacts.clone(), - to_raw_value(&content) - .expect("Must be valid, we only added redacts field"), - ); - }, - | _ => {}, - }, - } - } - } - - (self.redacts.clone(), self.content.clone()) -} - -#[implement(super::Pdu)] -#[must_use] -pub fn redacts_id(&self, room_version: &RoomVersionId) -> Option { - use RoomVersionId::*; - - if self.kind != TimelineEventType::RoomRedaction { - return None; - } - - match *room_version { - | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => self.redacts.clone(), - | _ => - self.get_content::() - .ok()? - .redacts, - } -} diff --git a/src/core/matrix/pdu/strip.rs b/src/core/matrix/pdu/strip.rs deleted file mode 100644 index a39e7d35..00000000 --- a/src/core/matrix/pdu/strip.rs +++ /dev/null @@ -1,257 +0,0 @@ -use ruma::{ - events::{ - AnyMessageLikeEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncStateEvent, - AnySyncTimelineEvent, AnyTimelineEvent, StateEvent, room::member::RoomMemberEventContent, - space::child::HierarchySpaceChildEvent, - }, - serde::Raw, -}; -use serde_json::{json, value::Value as JsonValue}; - -use crate::implement; - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn into_room_event(self) -> Raw { self.to_room_event() } - -#[implement(super::Pdu)] -#[must_use] -pub fn to_room_event(&self) -> Raw { - let value = self.to_room_event_value(); - serde_json::from_value(value).expect("Failed to serialize Event value") -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn to_room_event_value(&self) -> JsonValue { - let (redacts, content) = self.copy_redacts(); - let mut json = json!({ - "content": content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "room_id": self.room_id, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &redacts { - json["redacts"] = json!(redacts); - } - - json -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn into_message_like_event(self) -> Raw { self.to_message_like_event() } - -#[implement(super::Pdu)] -#[must_use] -pub fn to_message_like_event(&self) -> Raw { - let value = self.to_message_like_event_value(); - serde_json::from_value(value).expect("Failed to serialize Event value") -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn to_message_like_event_value(&self) -> JsonValue { - let (redacts, content) = self.copy_redacts(); - let mut json = json!({ - "content": content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "room_id": self.room_id, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &redacts { - json["redacts"] = json!(redacts); - } - - json -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn into_sync_room_event(self) -> Raw { self.to_sync_room_event() } - -#[implement(super::Pdu)] -#[must_use] -pub fn to_sync_room_event(&self) -> Raw { - let value = self.to_sync_room_event_value(); - serde_json::from_value(value).expect("Failed to serialize Event value") -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn to_sync_room_event_value(&self) -> JsonValue { - let (redacts, content) = self.copy_redacts(); - let mut json = json!({ - "content": content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - if let Some(state_key) = &self.state_key { - json["state_key"] = json!(state_key); - } - if let Some(redacts) = &redacts { - json["redacts"] = json!(redacts); - } - - json -} - -#[implement(super::Pdu)] -#[must_use] -pub fn into_state_event(self) -> Raw { - let value = self.into_state_event_value(); - serde_json::from_value(value).expect("Failed to serialize Event value") -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn into_state_event_value(self) -> JsonValue { - let mut json = json!({ - "content": self.content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "room_id": self.room_id, - "state_key": self.state_key, - }); - - if let Some(unsigned) = self.unsigned { - json["unsigned"] = json!(unsigned); - } - - json -} - -#[implement(super::Pdu)] -#[must_use] -pub fn into_sync_state_event(self) -> Raw { - let value = self.into_sync_state_event_value(); - serde_json::from_value(value).expect("Failed to serialize Event value") -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn into_sync_state_event_value(self) -> JsonValue { - let mut json = json!({ - "content": self.content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "state_key": self.state_key, - }); - - if let Some(unsigned) = &self.unsigned { - json["unsigned"] = json!(unsigned); - } - - json -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn into_stripped_state_event(self) -> Raw { - self.to_stripped_state_event() -} - -#[implement(super::Pdu)] -#[must_use] -pub fn to_stripped_state_event(&self) -> Raw { - let value = self.to_stripped_state_event_value(); - serde_json::from_value(value).expect("Failed to serialize Event value") -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn to_stripped_state_event_value(&self) -> JsonValue { - json!({ - "content": self.content, - "type": self.kind, - "sender": self.sender, - "state_key": self.state_key, - }) -} - -#[implement(super::Pdu)] -#[must_use] -pub fn into_stripped_spacechild_state_event(self) -> Raw { - let value = self.into_stripped_spacechild_state_event_value(); - serde_json::from_value(value).expect("Failed to serialize Event value") -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn into_stripped_spacechild_state_event_value(self) -> JsonValue { - json!({ - "content": self.content, - "type": self.kind, - "sender": self.sender, - "state_key": self.state_key, - "origin_server_ts": self.origin_server_ts, - }) -} - -#[implement(super::Pdu)] -#[must_use] -pub fn into_member_event(self) -> Raw> { - let value = self.into_member_event_value(); - serde_json::from_value(value).expect("Failed to serialize Event value") -} - -#[implement(super::Pdu)] -#[must_use] -#[inline] -pub fn into_member_event_value(self) -> JsonValue { - let mut json = json!({ - "content": self.content, - "type": self.kind, - "event_id": self.event_id, - "sender": self.sender, - "origin_server_ts": self.origin_server_ts, - "redacts": self.redacts, - "room_id": self.room_id, - "state_key": self.state_key, - }); - - if let Some(unsigned) = self.unsigned { - json["unsigned"] = json!(unsigned); - } - - json -} diff --git a/src/core/matrix/pdu/state_key.rs b/src/core/matrix/state_key.rs similarity index 67% rename from src/core/matrix/pdu/state_key.rs rename to src/core/matrix/state_key.rs index 4af4fcf7..06d614f8 100644 --- a/src/core/matrix/pdu/state_key.rs +++ b/src/core/matrix/state_key.rs @@ -1,8 +1,5 @@ use smallstr::SmallString; -use super::ShortId; - pub type StateKey = SmallString<[u8; INLINE_SIZE]>; -pub type ShortStateKey = ShortId; const INLINE_SIZE: usize = 48; diff --git a/src/core/matrix/state_res/benches.rs b/src/core/matrix/state_res/benches.rs index 12eeab9d..69088369 100644 --- a/src/core/matrix/state_res/benches.rs +++ b/src/core/matrix/state_res/benches.rs @@ -13,7 +13,6 @@ use ruma::{ EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, Signatures, UserId, events::{ StateEventType, TimelineEventType, - pdu::{EventHash, Pdu, RoomV3Pdu}, room::{ join_rules::{JoinRule, RoomJoinRulesEventContent}, member::{MembershipState, RoomMemberEventContent}, @@ -26,8 +25,10 @@ use serde_json::{ value::{RawValue as RawJsonValue, to_raw_value as to_raw_json_value}, }; -use self::event::PduEvent; -use crate::state_res::{self as state_res, Error, Event, Result, StateMap}; +use crate::{ + matrix::{Event, Pdu, pdu::EventHash}, + state_res::{self as state_res, Error, Result, StateMap}, +}; static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); @@ -60,7 +61,7 @@ fn resolution_shallow_auth_chain(c: &mut test::Bencher) { c.iter(|| async { let ev_map = store.0.clone(); let state_sets = [&state_at_bob, &state_at_charlie]; - let fetch = |id: OwnedEventId| ready(ev_map.get(&id).clone()); + let fetch = |id: OwnedEventId| ready(ev_map.get(&id).map(ToOwned::to_owned)); let exists = |id: OwnedEventId| ready(ev_map.get(&id).is_some()); let auth_chain_sets: Vec> = state_sets .iter() @@ -142,7 +143,7 @@ fn resolve_deeper_event_set(c: &mut test::Bencher) { }) .collect(); - let fetch = |id: OwnedEventId| ready(inner.get(&id).clone()); + let fetch = |id: OwnedEventId| ready(inner.get(&id).map(ToOwned::to_owned)); let exists = |id: OwnedEventId| ready(inner.get(&id).is_some()); let _ = match state_res::resolve( &RoomVersionId::V6, @@ -246,7 +247,7 @@ impl TestStore { } } -impl TestStore { +impl TestStore { #[allow(clippy::type_complexity)] fn set_up( &mut self, @@ -380,7 +381,7 @@ fn to_pdu_event( content: Box, auth_events: &[S], prev_events: &[S], -) -> PduEvent +) -> Pdu where S: AsRef, { @@ -403,30 +404,28 @@ where .map(event_id) .collect::>(); - let state_key = state_key.map(ToOwned::to_owned); - PduEvent { + Pdu { event_id: id.try_into().unwrap(), - rest: Pdu::RoomV3Pdu(RoomV3Pdu { - room_id: room_id().to_owned(), - sender: sender.to_owned(), - origin_server_ts: MilliSecondsSinceUnixEpoch(ts.try_into().unwrap()), - state_key, - kind: ev_type, - content, - redacts: None, - unsigned: btreemap! {}, - auth_events, - prev_events, - depth: uint!(0), - hashes: EventHash::new(String::new()), - signatures: Signatures::new(), - }), + room_id: room_id().to_owned(), + sender: sender.to_owned(), + origin_server_ts: ts.try_into().unwrap(), + state_key: state_key.map(Into::into), + kind: ev_type, + content, + origin: None, + redacts: None, + unsigned: None, + auth_events, + prev_events, + depth: uint!(0), + hashes: EventHash { sha256: String::new() }, + signatures: None, } } // all graphs start with these input events #[allow(non_snake_case)] -fn INITIAL_EVENTS() -> HashMap { +fn INITIAL_EVENTS() -> HashMap { vec![ to_pdu_event::<&EventId>( "CREATE", @@ -508,7 +507,7 @@ fn INITIAL_EVENTS() -> HashMap { // all graphs start with these input events #[allow(non_snake_case)] -fn BAN_STATE_SET() -> HashMap { +fn BAN_STATE_SET() -> HashMap { vec![ to_pdu_event( "PA", @@ -551,119 +550,3 @@ fn BAN_STATE_SET() -> HashMap { .map(|ev| (ev.event_id().to_owned(), ev)) .collect() } - -/// Convenience trait for adding event type plus state key to state maps. -trait EventTypeExt { - fn with_state_key(self, state_key: impl Into) -> (StateEventType, String); -} - -impl EventTypeExt for &TimelineEventType { - fn with_state_key(self, state_key: impl Into) -> (StateEventType, String) { - (self.to_string().into(), state_key.into()) - } -} - -mod event { - use ruma::{ - EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, - events::{TimelineEventType, pdu::Pdu}, - }; - use serde::{Deserialize, Serialize}; - use serde_json::value::RawValue as RawJsonValue; - - use super::Event; - - impl Event for PduEvent { - fn event_id(&self) -> &EventId { &self.event_id } - - fn room_id(&self) -> &RoomId { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => &ev.room_id, - | Pdu::RoomV3Pdu(ev) => &ev.room_id, - #[cfg(not(feature = "unstable-exhaustive-types"))] - | _ => unreachable!("new PDU version"), - } - } - - fn sender(&self) -> &UserId { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => &ev.sender, - | Pdu::RoomV3Pdu(ev) => &ev.sender, - #[cfg(not(feature = "unstable-exhaustive-types"))] - | _ => unreachable!("new PDU version"), - } - } - - fn event_type(&self) -> &TimelineEventType { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => &ev.kind, - | Pdu::RoomV3Pdu(ev) => &ev.kind, - #[cfg(not(feature = "unstable-exhaustive-types"))] - | _ => unreachable!("new PDU version"), - } - } - - fn content(&self) -> &RawJsonValue { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => &ev.content, - | Pdu::RoomV3Pdu(ev) => &ev.content, - #[cfg(not(feature = "unstable-exhaustive-types"))] - | _ => unreachable!("new PDU version"), - } - } - - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => ev.origin_server_ts, - | Pdu::RoomV3Pdu(ev) => ev.origin_server_ts, - #[cfg(not(feature = "unstable-exhaustive-types"))] - | _ => unreachable!("new PDU version"), - } - } - - fn state_key(&self) -> Option<&str> { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => ev.state_key.as_deref(), - | Pdu::RoomV3Pdu(ev) => ev.state_key.as_deref(), - #[cfg(not(feature = "unstable-exhaustive-types"))] - | _ => unreachable!("new PDU version"), - } - } - - fn prev_events(&self) -> Box + Send + '_> { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => - Box::new(ev.prev_events.iter().map(|(id, _)| id.as_ref())), - | Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter().map(AsRef::as_ref)), - #[cfg(not(feature = "unstable-exhaustive-types"))] - | _ => unreachable!("new PDU version"), - } - } - - fn auth_events(&self) -> Box + Send + '_> { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => - Box::new(ev.auth_events.iter().map(|(id, _)| id.as_ref())), - | Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter().map(AsRef::as_ref)), - #[cfg(not(feature = "unstable-exhaustive-types"))] - | _ => unreachable!("new PDU version"), - } - } - - fn redacts(&self) -> Option<&EventId> { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => ev.redacts.as_deref(), - | Pdu::RoomV3Pdu(ev) => ev.redacts.as_deref(), - #[cfg(not(feature = "unstable-exhaustive-types"))] - | _ => unreachable!("new PDU version"), - } - } - } - - #[derive(Clone, Debug, Deserialize, Serialize)] - pub(crate) struct PduEvent { - pub(crate) event_id: OwnedEventId, - #[serde(flatten)] - pub(crate) rest: Pdu, - } -} diff --git a/src/core/matrix/state_res/event_auth.rs b/src/core/matrix/state_res/event_auth.rs index 759ab5cb..8c760860 100644 --- a/src/core/matrix/state_res/event_auth.rs +++ b/src/core/matrix/state_res/event_auth.rs @@ -136,17 +136,17 @@ pub fn auth_types_for_event( event_id = incoming_event.event_id().as_str(), ) )] -pub async fn auth_check( +pub async fn auth_check( room_version: &RoomVersion, - incoming_event: &Incoming, - current_third_party_invite: Option<&Incoming>, + incoming_event: &E, + current_third_party_invite: Option<&E>, fetch_state: F, ) -> Result where F: Fn(&StateEventType, &str) -> Fut + Send, - Fut: Future> + Send, - Fetched: Event + Send, - Incoming: Event + Send + Sync, + Fut: Future> + Send, + E: Event + Send + Sync, + for<'a> &'a E: Event + Send, { debug!( event_id = format!("{}", incoming_event.event_id()), @@ -514,20 +514,24 @@ where /// event and the current State. #[allow(clippy::too_many_arguments)] #[allow(clippy::cognitive_complexity)] -fn valid_membership_change( +fn valid_membership_change( room_version: &RoomVersion, target_user: &UserId, - target_user_membership_event: Option<&impl Event>, + target_user_membership_event: Option<&E>, sender: &UserId, - sender_membership_event: Option<&impl Event>, - current_event: impl Event, - current_third_party_invite: Option<&impl Event>, - power_levels_event: Option<&impl Event>, - join_rules_event: Option<&impl Event>, + sender_membership_event: Option<&E>, + current_event: &E, + current_third_party_invite: Option<&E>, + power_levels_event: Option<&E>, + join_rules_event: Option<&E>, user_for_join_auth: Option<&UserId>, user_for_join_auth_membership: &MembershipState, - create_room: &impl Event, -) -> Result { + create_room: &E, +) -> Result +where + E: Event + Send + Sync, + for<'a> &'a E: Event + Send, +{ #[derive(Deserialize)] struct GetThirdPartyInvite { third_party_invite: Option>, @@ -820,7 +824,7 @@ fn valid_membership_change( /// /// Does the event have the correct userId as its state_key if it's not the "" /// state_key. -fn can_send_event(event: impl Event, ple: Option, user_level: Int) -> bool { +fn can_send_event(event: &impl Event, ple: Option<&impl Event>, user_level: Int) -> bool { let event_type_power_level = get_send_level(event.event_type(), event.state_key(), ple); debug!( @@ -846,8 +850,8 @@ fn can_send_event(event: impl Event, ple: Option, user_level: Int) - /// Confirm that the event sender has the required power levels. fn check_power_levels( room_version: &RoomVersion, - power_event: impl Event, - previous_power_event: Option, + power_event: &impl Event, + previous_power_event: Option<&impl Event>, user_level: Int, ) -> Option { match power_event.state_key() { @@ -1010,7 +1014,7 @@ fn get_deserialize_levels( /// given event. fn check_redaction( _room_version: &RoomVersion, - redaction_event: impl Event, + redaction_event: &impl Event, user_level: Int, redact_level: Int, ) -> Result { @@ -1039,7 +1043,7 @@ fn check_redaction( fn get_send_level( e_type: &TimelineEventType, state_key: Option<&str>, - power_lvl: Option, + power_lvl: Option<&impl Event>, ) -> Int { power_lvl .and_then(|ple| { @@ -1062,7 +1066,7 @@ fn verify_third_party_invite( target_user: Option<&UserId>, sender: &UserId, tp_id: &ThirdPartyInvite, - current_third_party_invite: Option, + current_third_party_invite: Option<&impl Event>, ) -> bool { // 1. Check for user being banned happens before this is called // checking for mxid and token keys is done by ruma when deserializing @@ -1128,12 +1132,15 @@ mod tests { }; use serde_json::value::to_raw_value as to_raw_json_value; - use crate::state_res::{ - Event, EventTypeExt, RoomVersion, StateMap, - event_auth::valid_membership_change, - test_utils::{ - INITIAL_EVENTS, INITIAL_EVENTS_CREATE_ROOM, PduEvent, alice, charlie, ella, event_id, - member_content_ban, member_content_join, room_id, to_pdu_event, + use crate::{ + matrix::{Event, EventTypeExt, Pdu as PduEvent}, + state_res::{ + RoomVersion, StateMap, + event_auth::valid_membership_change, + test_utils::{ + INITIAL_EVENTS, INITIAL_EVENTS_CREATE_ROOM, alice, charlie, ella, event_id, + member_content_ban, member_content_join, room_id, to_pdu_event, + }, }, }; diff --git a/src/core/matrix/state_res/mod.rs b/src/core/matrix/state_res/mod.rs index 651f6130..ed5aa034 100644 --- a/src/core/matrix/state_res/mod.rs +++ b/src/core/matrix/state_res/mod.rs @@ -37,7 +37,7 @@ pub use self::{ }; use crate::{ debug, debug_error, - matrix::{event::Event, pdu::StateKey}, + matrix::{Event, StateKey}, trace, utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, WidebandExt}, warn, @@ -90,7 +90,7 @@ where SetIter: Iterator> + Clone + Send, Hasher: BuildHasher + Send + Sync, E: Event + Clone + Send + Sync, - for<'b> &'b E: Send, + for<'b> &'b E: Event + Send, { debug!("State resolution starting"); @@ -522,6 +522,7 @@ where Fut: Future> + Send, S: Stream + Send + 'a, E: Event + Clone + Send + Sync, + for<'b> &'b E: Event + Send, { debug!("starting iterative auth check"); @@ -552,7 +553,7 @@ where let auth_events = &auth_events; let mut resolved_state = unconflicted_state; - for event in &events_to_check { + for event in events_to_check { let state_key = event .state_key() .ok_or_else(|| Error::InvalidPdu("State event had no state key".to_owned()))?; @@ -607,11 +608,15 @@ where }); let fetch_state = |ty: &StateEventType, key: &str| { - future::ready(auth_state.get(&ty.with_state_key(key))) + future::ready( + auth_state + .get(&ty.with_state_key(key)) + .map(ToOwned::to_owned), + ) }; let auth_result = - auth_check(room_version, &event, current_third_party.as_ref(), fetch_state).await; + auth_check(room_version, &event, current_third_party, fetch_state).await; match auth_result { | Ok(true) => { @@ -794,11 +799,11 @@ where } } -fn is_type_and_key(ev: impl Event, ev_type: &TimelineEventType, state_key: &str) -> bool { +fn is_type_and_key(ev: &impl Event, ev_type: &TimelineEventType, state_key: &str) -> bool { ev.event_type() == ev_type && ev.state_key() == Some(state_key) } -fn is_power_event(event: impl Event) -> bool { +fn is_power_event(event: &impl Event) -> bool { match event.event_type() { | TimelineEventType::RoomPowerLevels | TimelineEventType::RoomJoinRules @@ -859,15 +864,19 @@ mod tests { use serde_json::{json, value::to_raw_value as to_raw_json_value}; use super::{ - Event, EventTypeExt, StateMap, is_power_event, + StateMap, is_power_event, room_version::RoomVersion, test_utils::{ - INITIAL_EVENTS, PduEvent, TestStore, alice, bob, charlie, do_check, ella, event_id, + INITIAL_EVENTS, TestStore, alice, bob, charlie, do_check, ella, event_id, member_content_ban, member_content_join, room_id, to_init_pdu_event, to_pdu_event, zara, }, }; - use crate::{debug, utils::stream::IterStream}; + use crate::{ + debug, + matrix::{Event, EventTypeExt, Pdu as PduEvent}, + utils::stream::IterStream, + }; async fn test_event_sort() { use futures::future::ready; diff --git a/src/core/matrix/state_res/test_utils.rs b/src/core/matrix/state_res/test_utils.rs index c6945f66..9f24c51b 100644 --- a/src/core/matrix/state_res/test_utils.rs +++ b/src/core/matrix/state_res/test_utils.rs @@ -10,7 +10,6 @@ use ruma::{ UserId, event_id, events::{ TimelineEventType, - pdu::{EventHash, Pdu, RoomV3Pdu}, room::{ join_rules::{JoinRule, RoomJoinRulesEventContent}, member::{MembershipState, RoomMemberEventContent}, @@ -23,17 +22,16 @@ use serde_json::{ value::{RawValue as RawJsonValue, to_raw_value as to_raw_json_value}, }; -pub(crate) use self::event::PduEvent; use super::auth_types_for_event; use crate::{ Result, info, - matrix::{Event, EventTypeExt, StateMap}, + matrix::{Event, EventTypeExt, Pdu, StateMap, pdu::EventHash}, }; static SERVER_TIMESTAMP: AtomicU64 = AtomicU64::new(0); pub(crate) async fn do_check( - events: &[PduEvent], + events: &[Pdu], edges: Vec>, expected_state_ids: Vec, ) { @@ -81,8 +79,8 @@ pub(crate) async fn do_check( } } - // event_id -> PduEvent - let mut event_map: HashMap = HashMap::new(); + // event_id -> Pdu + let mut event_map: HashMap = HashMap::new(); // event_id -> StateMap let mut state_at_event: HashMap> = HashMap::new(); @@ -265,7 +263,7 @@ impl TestStore { // A StateStore implementation for testing #[allow(clippy::type_complexity)] -impl TestStore { +impl TestStore { pub(crate) fn set_up( &mut self, ) -> (StateMap, StateMap, StateMap) { @@ -390,7 +388,7 @@ pub(crate) fn to_init_pdu_event( ev_type: TimelineEventType, state_key: Option<&str>, content: Box, -) -> PduEvent { +) -> Pdu { let ts = SERVER_TIMESTAMP.fetch_add(1, SeqCst); let id = if id.contains('$') { id.to_owned() @@ -398,24 +396,22 @@ pub(crate) fn to_init_pdu_event( format!("${id}:foo") }; - let state_key = state_key.map(ToOwned::to_owned); - PduEvent { + Pdu { event_id: id.try_into().unwrap(), - rest: Pdu::RoomV3Pdu(RoomV3Pdu { - room_id: room_id().to_owned(), - sender: sender.to_owned(), - origin_server_ts: MilliSecondsSinceUnixEpoch(ts.try_into().unwrap()), - state_key, - kind: ev_type, - content, - redacts: None, - unsigned: BTreeMap::new(), - auth_events: vec![], - prev_events: vec![], - depth: uint!(0), - hashes: EventHash::new("".to_owned()), - signatures: ServerSignatures::default(), - }), + room_id: room_id().to_owned(), + sender: sender.to_owned(), + origin_server_ts: ts.try_into().unwrap(), + state_key: state_key.map(Into::into), + kind: ev_type, + content, + origin: None, + redacts: None, + unsigned: None, + auth_events: vec![], + prev_events: vec![], + depth: uint!(0), + hashes: EventHash { sha256: "".to_owned() }, + signatures: None, } } @@ -427,7 +423,7 @@ pub(crate) fn to_pdu_event( content: Box, auth_events: &[S], prev_events: &[S], -) -> PduEvent +) -> Pdu where S: AsRef, { @@ -448,30 +444,28 @@ where .map(event_id) .collect::>(); - let state_key = state_key.map(ToOwned::to_owned); - PduEvent { + Pdu { event_id: id.try_into().unwrap(), - rest: Pdu::RoomV3Pdu(RoomV3Pdu { - room_id: room_id().to_owned(), - sender: sender.to_owned(), - origin_server_ts: MilliSecondsSinceUnixEpoch(ts.try_into().unwrap()), - state_key, - kind: ev_type, - content, - redacts: None, - unsigned: BTreeMap::new(), - auth_events, - prev_events, - depth: uint!(0), - hashes: EventHash::new("".to_owned()), - signatures: ServerSignatures::default(), - }), + room_id: room_id().to_owned(), + sender: sender.to_owned(), + origin_server_ts: ts.try_into().unwrap(), + state_key: state_key.map(Into::into), + kind: ev_type, + content, + origin: None, + redacts: None, + unsigned: None, + auth_events, + prev_events, + depth: uint!(0), + hashes: EventHash { sha256: "".to_owned() }, + signatures: None, } } // all graphs start with these input events #[allow(non_snake_case)] -pub(crate) fn INITIAL_EVENTS() -> HashMap { +pub(crate) fn INITIAL_EVENTS() -> HashMap { vec![ to_pdu_event::<&EventId>( "CREATE", @@ -553,7 +547,7 @@ pub(crate) fn INITIAL_EVENTS() -> HashMap { // all graphs start with these input events #[allow(non_snake_case)] -pub(crate) fn INITIAL_EVENTS_CREATE_ROOM() -> HashMap { +pub(crate) fn INITIAL_EVENTS_CREATE_ROOM() -> HashMap { vec![to_pdu_event::<&EventId>( "CREATE", alice(), @@ -575,111 +569,3 @@ pub(crate) fn INITIAL_EDGES() -> Vec { .map(event_id) .collect::>() } - -pub(crate) mod event { - use ruma::{ - EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, UserId, - events::{TimelineEventType, pdu::Pdu}, - }; - use serde::{Deserialize, Serialize}; - use serde_json::value::RawValue as RawJsonValue; - - use crate::Event; - - impl Event for PduEvent { - fn event_id(&self) -> &EventId { &self.event_id } - - fn room_id(&self) -> &RoomId { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => &ev.room_id, - | Pdu::RoomV3Pdu(ev) => &ev.room_id, - #[allow(unreachable_patterns)] - | _ => unreachable!("new PDU version"), - } - } - - fn sender(&self) -> &UserId { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => &ev.sender, - | Pdu::RoomV3Pdu(ev) => &ev.sender, - #[allow(unreachable_patterns)] - | _ => unreachable!("new PDU version"), - } - } - - fn event_type(&self) -> &TimelineEventType { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => &ev.kind, - | Pdu::RoomV3Pdu(ev) => &ev.kind, - #[allow(unreachable_patterns)] - | _ => unreachable!("new PDU version"), - } - } - - fn content(&self) -> &RawJsonValue { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => &ev.content, - | Pdu::RoomV3Pdu(ev) => &ev.content, - #[allow(unreachable_patterns)] - | _ => unreachable!("new PDU version"), - } - } - - fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => ev.origin_server_ts, - | Pdu::RoomV3Pdu(ev) => ev.origin_server_ts, - #[allow(unreachable_patterns)] - | _ => unreachable!("new PDU version"), - } - } - - fn state_key(&self) -> Option<&str> { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => ev.state_key.as_deref(), - | Pdu::RoomV3Pdu(ev) => ev.state_key.as_deref(), - #[allow(unreachable_patterns)] - | _ => unreachable!("new PDU version"), - } - } - - #[allow(refining_impl_trait)] - fn prev_events(&self) -> Box + Send + '_> { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => - Box::new(ev.prev_events.iter().map(|(id, _)| id.as_ref())), - | Pdu::RoomV3Pdu(ev) => Box::new(ev.prev_events.iter().map(AsRef::as_ref)), - #[allow(unreachable_patterns)] - | _ => unreachable!("new PDU version"), - } - } - - #[allow(refining_impl_trait)] - fn auth_events(&self) -> Box + Send + '_> { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => - Box::new(ev.auth_events.iter().map(|(id, _)| id.as_ref())), - | Pdu::RoomV3Pdu(ev) => Box::new(ev.auth_events.iter().map(AsRef::as_ref)), - #[allow(unreachable_patterns)] - | _ => unreachable!("new PDU version"), - } - } - - fn redacts(&self) -> Option<&EventId> { - match &self.rest { - | Pdu::RoomV1Pdu(ev) => ev.redacts.as_deref(), - | Pdu::RoomV3Pdu(ev) => ev.redacts.as_deref(), - #[allow(unreachable_patterns)] - | _ => unreachable!("new PDU version"), - } - } - } - - #[derive(Clone, Debug, Deserialize, Serialize)] - #[allow(clippy::exhaustive_structs)] - pub(crate) struct PduEvent { - pub(crate) event_id: OwnedEventId, - #[serde(flatten)] - pub(crate) rest: Pdu, - } -} diff --git a/src/core/mod.rs b/src/core/mod.rs index aaacd4d8..d99139be 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -25,7 +25,9 @@ pub use info::{ rustc_flags_capture, version, version::{name, version}, }; -pub use matrix::{Event, EventTypeExt, PduCount, PduEvent, PduId, RoomVersion, pdu, state_res}; +pub use matrix::{ + Event, EventTypeExt, Pdu, PduCount, PduEvent, PduId, RoomVersion, pdu, state_res, +}; pub use server::Server; pub use utils::{ctor, dtor, implement, result, result::Result}; diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index a76c3ef6..66c373ec 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -9,8 +9,8 @@ use std::{ }; use async_trait::async_trait; -use conduwuit::{ - Error, PduEvent, Result, Server, debug, err, error, error::default_log, pdu::PduBuilder, +use conduwuit_core::{ + Error, Event, Result, Server, debug, err, error, error::default_log, pdu::PduBuilder, }; pub use create::create_admin_room; use futures::{Future, FutureExt, TryFutureExt}; @@ -361,7 +361,10 @@ impl Service { Ok(()) } - pub async fn is_admin_command(&self, pdu: &PduEvent, body: &str) -> bool { + pub async fn is_admin_command(&self, event: &E, body: &str) -> bool + where + E: Event + Send + Sync, + { // Server-side command-escape with public echo let is_escape = body.starts_with('\\'); let is_public_escape = is_escape && body.trim_start_matches('\\').starts_with("!admin"); @@ -376,8 +379,10 @@ impl Service { return false; } + let user_is_local = self.services.globals.user_is_local(event.sender()); + // only allow public escaped commands by local admins - if is_public_escape && !self.services.globals.user_is_local(&pdu.sender) { + if is_public_escape && !user_is_local { return false; } @@ -387,20 +392,20 @@ impl Service { } // Prevent unescaped !admin from being used outside of the admin room - if is_public_prefix && !self.is_admin_room(&pdu.room_id).await { + if is_public_prefix && !self.is_admin_room(event.room_id()).await { return false; } // Only senders who are admin can proceed - if !self.user_is_admin(&pdu.sender).await { + if !self.user_is_admin(event.sender()).await { return false; } // This will evaluate to false if the emergency password is set up so that // the administrator can execute commands as the server user let emergency_password_set = self.services.server.config.emergency_password.is_some(); - let from_server = pdu.sender == *server_user && !emergency_password_set; - if from_server && self.is_admin_room(&pdu.room_id).await { + let from_server = event.sender() == server_user && !emergency_password_set; + if from_server && self.is_admin_room(event.room_id()).await { return false; } diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 27490fb8..192ef447 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -1,12 +1,12 @@ use std::{fmt::Debug, mem, sync::Arc}; use bytes::BytesMut; -use conduwuit::{ - Err, PduEvent, Result, debug_warn, err, trace, +use conduwuit_core::{ + Err, Event, Result, debug_warn, err, trace, utils::{stream::TryIgnore, string_from_bytes}, warn, }; -use database::{Deserialized, Ignore, Interfix, Json, Map}; +use conduwuit_database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::{Stream, StreamExt}; use ipaddress::IPAddress; use ruma::{ @@ -272,22 +272,26 @@ impl Service { } } - #[tracing::instrument(skip(self, user, unread, pusher, ruleset, pdu))] - pub async fn send_push_notice( + #[tracing::instrument(skip(self, user, unread, pusher, ruleset, event))] + pub async fn send_push_notice( &self, user: &UserId, unread: UInt, pusher: &Pusher, ruleset: Ruleset, - pdu: &PduEvent, - ) -> Result<()> { + event: &E, + ) -> Result + where + E: Event + Send + Sync, + for<'a> &'a E: Event + Send, + { let mut notify = None; let mut tweaks = Vec::new(); let power_levels: RoomPowerLevelsEventContent = self .services .state_accessor - .room_state_get(&pdu.room_id, &StateEventType::RoomPowerLevels, "") + .room_state_get(event.room_id(), &StateEventType::RoomPowerLevels, "") .await .and_then(|ev| { serde_json::from_str(ev.content.get()).map_err(|e| { @@ -296,8 +300,9 @@ impl Service { }) .unwrap_or_default(); + let serialized = event.to_format(); for action in self - .get_actions(user, &ruleset, &power_levels, &pdu.to_sync_room_event(), &pdu.room_id) + .get_actions(user, &ruleset, &power_levels, &serialized, event.room_id()) .await { let n = match action { @@ -319,7 +324,7 @@ impl Service { } if notify == Some(true) { - self.send_notice(unread, pusher, tweaks, pdu).await?; + self.send_notice(unread, pusher, tweaks, event).await?; } // Else the event triggered no actions @@ -369,13 +374,16 @@ impl Service { } #[tracing::instrument(skip(self, unread, pusher, tweaks, event))] - async fn send_notice( + async fn send_notice( &self, unread: UInt, pusher: &Pusher, tweaks: Vec, - event: &PduEvent, - ) -> Result { + event: &E, + ) -> Result + where + E: Event + Send + Sync, + { // TODO: email match &pusher.kind { | PusherKind::Http(http) => { @@ -421,8 +429,8 @@ impl Service { let d = vec![device]; let mut notifi = Notification::new(d); - notifi.event_id = Some((*event.event_id).to_owned()); - notifi.room_id = Some((*event.room_id).to_owned()); + notifi.event_id = Some(event.event_id().to_owned()); + notifi.room_id = Some(event.room_id().to_owned()); if http .data .get("org.matrix.msc4076.disable_badge_count") @@ -442,7 +450,7 @@ impl Service { ) .await?; } else { - if event.kind == TimelineEventType::RoomEncrypted + if *event.kind() == TimelineEventType::RoomEncrypted || tweaks .iter() .any(|t| matches!(t, Tweak::Highlight(true) | Tweak::Sound(_))) @@ -451,29 +459,29 @@ impl Service { } else { notifi.prio = NotificationPriority::Low; } - notifi.sender = Some(event.sender.clone()); - notifi.event_type = Some(event.kind.clone()); - notifi.content = serde_json::value::to_raw_value(&event.content).ok(); + notifi.sender = Some(event.sender().to_owned()); + notifi.event_type = Some(event.kind().to_owned()); + notifi.content = serde_json::value::to_raw_value(event.content()).ok(); - if event.kind == TimelineEventType::RoomMember { + if *event.kind() == TimelineEventType::RoomMember { notifi.user_is_target = - event.state_key.as_deref() == Some(event.sender.as_str()); + event.state_key() == Some(event.sender().as_str()); } notifi.sender_display_name = - self.services.users.displayname(&event.sender).await.ok(); + self.services.users.displayname(event.sender()).await.ok(); notifi.room_name = self .services .state_accessor - .get_name(&event.room_id) + .get_name(event.room_id()) .await .ok(); notifi.room_alias = self .services .state_accessor - .get_canonical_alias(&event.room_id) + .get_canonical_alias(event.room_id()) .await .ok(); diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index feebf8c1..5cc6be55 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -126,7 +126,7 @@ pub(super) async fn handle_outlier_pdu<'a>( let state_fetch = |ty: &StateEventType, sk: &str| { let key = (ty.to_owned(), sk.into()); - ready(auth_events.get(&key)) + ready(auth_events.get(&key).map(ToOwned::to_owned)) }; let auth_check = state_res::event_auth::auth_check( diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 97d3df97..00b18c06 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -2,7 +2,7 @@ use std::{borrow::Borrow, collections::BTreeMap, iter::once, sync::Arc, time::In use conduwuit::{ Err, Result, debug, debug_info, err, implement, - matrix::{EventTypeExt, PduEvent, StateKey, state_res}, + matrix::{Event, EventTypeExt, PduEvent, StateKey, state_res}, trace, utils::stream::{BroadbandExt, ReadyExt}, warn, @@ -108,7 +108,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( let state_fetch = |k: &StateEventType, s: &str| { let key = k.with_state_key(s); - ready(auth_events.get(&key).cloned()) + ready(auth_events.get(&key).map(ToOwned::to_owned)) }; let auth_check = state_res::event_auth::auth_check( diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index 4100dd75..b9d067a6 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use conduwuit::{ - PduCount, PduEvent, Result, +use conduwuit_core::{ + Event, PduCount, PduEvent, Result, arrayvec::ArrayVec, implement, utils::{ diff --git a/src/service/rooms/spaces/mod.rs b/src/service/rooms/spaces/mod.rs index 53d2b742..de2647ca 100644 --- a/src/service/rooms/spaces/mod.rs +++ b/src/service/rooms/spaces/mod.rs @@ -5,8 +5,8 @@ mod tests; use std::{fmt::Write, sync::Arc}; use async_trait::async_trait; -use conduwuit::{ - Err, Error, PduEvent, Result, implement, +use conduwuit_core::{ + Err, Error, Event, PduEvent, Result, implement, utils::{ IterStream, future::{BoolExt, TryExtExt}, @@ -142,7 +142,7 @@ pub async fn get_summary_and_children_local( let children_pdus: Vec<_> = self .get_space_child_events(current_room) - .map(PduEvent::into_stripped_spacechild_state_event) + .map(Event::into_format) .collect() .await; @@ -511,7 +511,7 @@ async fn cache_insert( room_id: room_id.clone(), children_state: self .get_space_child_events(&room_id) - .map(PduEvent::into_stripped_spacechild_state_event) + .map(Event::into_format) .collect() .await, encryption, diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 803ba9d7..9eb02221 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -1,8 +1,8 @@ use std::{collections::HashMap, fmt::Write, iter::once, sync::Arc}; use async_trait::async_trait; -use conduwuit::{ - PduEvent, Result, err, +use conduwuit_core::{ + Event, PduEvent, Result, err, result::FlatOk, state_res::{self, StateMap}, utils::{ @@ -11,7 +11,7 @@ use conduwuit::{ }, warn, }; -use database::{Deserialized, Ignore, Interfix, Map}; +use conduwuit_database::{Deserialized, Ignore, Interfix, Map}; use futures::{ FutureExt, Stream, StreamExt, TryFutureExt, TryStreamExt, future::join_all, pin_mut, }; @@ -319,30 +319,34 @@ impl Service { } #[tracing::instrument(skip_all, level = "debug")] - pub async fn summary_stripped(&self, event: &PduEvent) -> Vec> { + pub async fn summary_stripped<'a, E>(&self, event: &'a E) -> Vec> + where + E: Event + Send + Sync, + &'a E: Event + Send, + { let cells = [ (&StateEventType::RoomCreate, ""), (&StateEventType::RoomJoinRules, ""), (&StateEventType::RoomCanonicalAlias, ""), (&StateEventType::RoomName, ""), (&StateEventType::RoomAvatar, ""), - (&StateEventType::RoomMember, event.sender.as_str()), // Add recommended events + (&StateEventType::RoomMember, event.sender().as_str()), // Add recommended events (&StateEventType::RoomEncryption, ""), (&StateEventType::RoomTopic, ""), ]; - let fetches = cells.iter().map(|(event_type, state_key)| { + let fetches = cells.into_iter().map(|(event_type, state_key)| { self.services .state_accessor - .room_state_get(&event.room_id, event_type, state_key) + .room_state_get(event.room_id(), event_type, state_key) }); join_all(fetches) .await .into_iter() .filter_map(Result::ok) - .map(PduEvent::into_stripped_state_event) - .chain(once(event.to_stripped_state_event())) + .map(Event::into_format) + .chain(once(event.to_format())) .collect() } diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index a680df55..9566eb61 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -1,7 +1,7 @@ use std::{collections::BTreeMap, sync::Arc}; -use conduwuit::{ - Result, err, +use conduwuit_core::{ + Event, Result, err, matrix::pdu::{PduCount, PduEvent, PduId, RawPduId}, utils::{ ReadyExt, @@ -49,7 +49,11 @@ impl crate::Service for Service { } impl Service { - pub async fn add_to_thread(&self, root_event_id: &EventId, pdu: &PduEvent) -> Result<()> { + pub async fn add_to_thread<'a, E>(&self, root_event_id: &EventId, event: &'a E) -> Result + where + E: Event + Send + Sync, + &'a E: Event + Send, + { let root_id = self .services .timeline @@ -86,7 +90,7 @@ impl Service { }) { // Thread already existed relations.count = relations.count.saturating_add(uint!(1)); - relations.latest_event = pdu.to_message_like_event(); + relations.latest_event = event.to_format(); let content = serde_json::to_value(relations).expect("to_value always works"); @@ -99,7 +103,7 @@ impl Service { } else { // New thread let relations = BundledThread { - latest_event: pdu.to_message_like_event(), + latest_event: event.to_format(), count: uint!(1), current_user_participated: true, }; @@ -129,7 +133,7 @@ impl Service { users.push(root_pdu.sender); }, } - users.push(pdu.sender.clone()); + users.push(event.sender().to_owned()); self.update_participants(&root_id, &users) } diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index 534d8faf..bcad1309 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -375,8 +375,6 @@ impl Service { .await .unwrap_or_default(); - let sync_pdu = pdu.to_sync_room_event(); - let mut push_target: HashSet<_> = self .services .state_cache @@ -401,6 +399,7 @@ impl Service { } } + let serialized = pdu.to_format(); for user in &push_target { let rules_for_user = self .services @@ -418,7 +417,7 @@ impl Service { for action in self .services .pusher - .get_actions(user, &rules_for_user, &power_levels, &sync_pdu, &pdu.room_id) + .get_actions(user, &rules_for_user, &power_levels, &serialized, &pdu.room_id) .await { match action { @@ -768,7 +767,7 @@ impl Service { let auth_fetch = |k: &StateEventType, s: &str| { let key = (k.clone(), s.into()); - ready(auth_events.get(&key)) + ready(auth_events.get(&key).map(ToOwned::to_owned)) }; let auth_check = state_res::auth_check( diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index fab02f6b..408ab17d 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -9,8 +9,8 @@ use std::{ }; use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use conduwuit::{ - Error, Result, debug, err, error, +use conduwuit_core::{ + Error, Event, Result, debug, err, error, result::LogErr, trace, utils::{ @@ -697,7 +697,7 @@ impl Service { match event { | SendingEvent::Pdu(pdu_id) => { if let Ok(pdu) = self.services.timeline.get_pdu_from_id(pdu_id).await { - pdu_jsons.push(pdu.into_room_event()); + pdu_jsons.push(pdu.to_format()); } }, | SendingEvent::Edu(edu) => From af4f66c768c8edc1c0da66f583ddb4c17201c4f0 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 27 Apr 2025 00:58:56 +0000 Subject: [PATCH 1888/2091] Cleanup/improve other async queries in some client handlers. Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 4 +- src/admin/processor.rs | 3 +- src/admin/user/commands.rs | 5 +- src/api/client/membership.rs | 2771 -------------------------- src/api/client/membership/ban.rs | 60 + src/api/client/membership/forget.rs | 52 + src/api/client/membership/invite.rs | 238 +++ src/api/client/membership/join.rs | 988 +++++++++ src/api/client/membership/kick.rs | 65 + src/api/client/membership/knock.rs | 767 +++++++ src/api/client/membership/leave.rs | 386 ++++ src/api/client/membership/members.rs | 147 ++ src/api/client/membership/mod.rs | 156 ++ src/api/client/membership/unban.rs | 58 + src/api/client/profile.rs | 54 +- src/api/client/room/initial_sync.rs | 41 +- 16 files changed, 2977 insertions(+), 2818 deletions(-) delete mode 100644 src/api/client/membership.rs create mode 100644 src/api/client/membership/ban.rs create mode 100644 src/api/client/membership/forget.rs create mode 100644 src/api/client/membership/invite.rs create mode 100644 src/api/client/membership/join.rs create mode 100644 src/api/client/membership/kick.rs create mode 100644 src/api/client/membership/knock.rs create mode 100644 src/api/client/membership/leave.rs create mode 100644 src/api/client/membership/members.rs create mode 100644 src/api/client/membership/mod.rs create mode 100644 src/api/client/membership/unban.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 2323e3b8..74355311 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -412,7 +412,9 @@ pub(super) async fn change_log_level(&self, filter: Option, reset: bool) .reload .reload(&new_filter_layer, Some(handles)) { - | Ok(()) => return self.write_str("Successfully changed log level").await, + | Ok(()) => { + return self.write_str("Successfully changed log level").await; + }, | Err(e) => { return Err!("Failed to modify and reload the global tracing log level: {e}"); }, diff --git a/src/admin/processor.rs b/src/admin/processor.rs index 8d1fe89c..e80000c1 100644 --- a/src/admin/processor.rs +++ b/src/admin/processor.rs @@ -94,8 +94,7 @@ async fn process_command(services: Arc, input: &CommandInput) -> Proce #[allow(clippy::result_large_err)] fn handle_panic(error: &Error, command: &CommandInput) -> ProcessorResult { - let link = - "Please submit a [bug report](https://forgejo.ellis.link/continuwuation/continuwuity/issues/new). 🥺"; + let link = "Please submit a [bug report](https://forgejo.ellis.link/continuwuation/continuwuity/issues/new). 🥺"; let msg = format!("Panic occurred while processing command:\n```\n{error:#?}\n```\n{link}"); let content = RoomMessageEventContent::notice_markdown(msg); error!("Panic while processing command: {error:?}"); diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 3750d758..e15c0b2c 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -286,8 +286,9 @@ pub(super) async fn reset_password(&self, username: String, password: Option return Err!("Couldn't reset the password for user {user_id}: {e}"), - | Ok(()) => - write!(self, "Successfully reset the password for user {user_id}: `{new_password}`"), + | Ok(()) => { + write!(self, "Successfully reset the password for user {user_id}: `{new_password}`") + }, } .await } diff --git a/src/api/client/membership.rs b/src/api/client/membership.rs deleted file mode 100644 index 3c2a6fe3..00000000 --- a/src/api/client/membership.rs +++ /dev/null @@ -1,2771 +0,0 @@ -use std::{ - borrow::Borrow, - collections::{HashMap, HashSet}, - iter::once, - net::IpAddr, - sync::Arc, -}; - -use axum::extract::State; -use axum_client_ip::InsecureClientIp; -use conduwuit::{ - Err, Event, Result, at, debug, debug_error, debug_info, debug_warn, err, error, info, - is_matching, - matrix::{ - StateKey, - pdu::{PduBuilder, PduEvent, gen_event_id, gen_event_id_canonical_json}, - state_res, - }, - result::{FlatOk, NotFound}, - trace, - utils::{ - self, FutureBoolExt, - future::ReadyEqExt, - shuffle, - stream::{BroadbandExt, IterStream, ReadyExt}, - }, - warn, -}; -use conduwuit_service::{ - Services, - appservice::RegistrationInfo, - rooms::{ - state::RoomMutexGuard, - state_compressor::{CompressedState, HashSetCompressStateEvent}, - }, -}; -use futures::{FutureExt, StreamExt, TryFutureExt, join, pin_mut}; -use ruma::{ - CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, - OwnedUserId, RoomId, RoomVersionId, ServerName, UserId, - api::{ - client::{ - error::ErrorKind, - knock::knock_room, - membership::{ - ThirdPartySigned, ban_user, forget_room, - get_member_events::{self, v3::MembershipEventFilter}, - invite_user, join_room_by_id, join_room_by_id_or_alias, - joined_members::{self, v3::RoomMember}, - joined_rooms, kick_user, leave_room, unban_user, - }, - }, - federation::{self, membership::create_invite}, - }, - canonical_json::to_canonical_value, - events::{ - StateEventType, - room::{ - join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent}, - member::{MembershipState, RoomMemberEventContent}, - }, - }, -}; - -use crate::{Ruma, client::full_user_deactivate}; - -/// Checks if the room is banned in any way possible and the sender user is not -/// an admin. -/// -/// Performs automatic deactivation if `auto_deactivate_banned_room_attempts` is -/// enabled -#[tracing::instrument(skip(services))] -async fn banned_room_check( - services: &Services, - user_id: &UserId, - room_id: Option<&RoomId>, - server_name: Option<&ServerName>, - client_ip: IpAddr, -) -> Result { - if services.users.is_admin(user_id).await { - return Ok(()); - } - - if let Some(room_id) = room_id { - if services.rooms.metadata.is_banned(room_id).await - || services - .moderation - .is_remote_server_forbidden(room_id.server_name().expect("legacy room mxid")) - { - warn!( - "User {user_id} who is not an admin attempted to send an invite for or \ - attempted to join a banned room or banned room server name: {room_id}" - ); - - if services.server.config.auto_deactivate_banned_room_attempts { - warn!( - "Automatically deactivating user {user_id} due to attempted banned room join" - ); - - if services.server.config.admin_room_notices { - services - .admin - .send_text(&format!( - "Automatically deactivating user {user_id} due to attempted banned \ - room join from IP {client_ip}" - )) - .await; - } - - let all_joined_rooms: Vec = services - .rooms - .state_cache - .rooms_joined(user_id) - .map(Into::into) - .collect() - .await; - - full_user_deactivate(services, user_id, &all_joined_rooms) - .boxed() - .await?; - } - - return Err!(Request(Forbidden("This room is banned on this homeserver."))); - } - } else if let Some(server_name) = server_name { - if services - .config - .forbidden_remote_server_names - .is_match(server_name.host()) - { - warn!( - "User {user_id} who is not an admin tried joining a room which has the server \ - name {server_name} that is globally forbidden. Rejecting.", - ); - - if services.server.config.auto_deactivate_banned_room_attempts { - warn!( - "Automatically deactivating user {user_id} due to attempted banned room join" - ); - - if services.server.config.admin_room_notices { - services - .admin - .send_text(&format!( - "Automatically deactivating user {user_id} due to attempted banned \ - room join from IP {client_ip}" - )) - .await; - } - - let all_joined_rooms: Vec = services - .rooms - .state_cache - .rooms_joined(user_id) - .map(Into::into) - .collect() - .await; - - full_user_deactivate(services, user_id, &all_joined_rooms) - .boxed() - .await?; - } - - return Err!(Request(Forbidden("This remote server is banned on this homeserver."))); - } - } - - Ok(()) -} - -/// # `POST /_matrix/client/r0/rooms/{roomId}/join` -/// -/// Tries to join the sender user into a room. -/// -/// - If the server knowns about this room: creates the join event and does auth -/// rules locally -/// - If the server does not know about the room: asks other servers over -/// federation -#[tracing::instrument(skip_all, fields(%client), name = "join")] -pub(crate) async fn join_room_by_id_route( - State(services): State, - InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result { - let sender_user = body.sender_user(); - if services.users.is_suspended(sender_user).await? { - return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); - } - - banned_room_check( - &services, - sender_user, - Some(&body.room_id), - body.room_id.server_name(), - client, - ) - .await?; - - // There is no body.server_name for /roomId/join - let mut servers: Vec<_> = services - .rooms - .state_cache - .servers_invite_via(&body.room_id) - .map(ToOwned::to_owned) - .collect() - .await; - - servers.extend( - services - .rooms - .state_cache - .invite_state(sender_user, &body.room_id) - .await - .unwrap_or_default() - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()), - ); - - if let Some(server) = body.room_id.server_name() { - servers.push(server.into()); - } - - servers.sort_unstable(); - servers.dedup(); - shuffle(&mut servers); - - join_room_by_id_helper( - &services, - sender_user, - &body.room_id, - body.reason.clone(), - &servers, - body.third_party_signed.as_ref(), - &body.appservice_info, - ) - .boxed() - .await -} - -/// # `POST /_matrix/client/r0/join/{roomIdOrAlias}` -/// -/// Tries to join the sender user into a room. -/// -/// - If the server knowns about this room: creates the join event and does auth -/// rules locally -/// - If the server does not know about the room: use the server name query -/// param if specified. if not specified, asks other servers over federation -/// via room alias server name and room ID server name -#[tracing::instrument(skip_all, fields(%client), name = "join")] -pub(crate) async fn join_room_by_id_or_alias_route( - State(services): State, - InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result { - let sender_user = body.sender_user(); - let appservice_info = &body.appservice_info; - let body = &body.body; - if services.users.is_suspended(sender_user).await? { - return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); - } - - let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias.clone()) { - | Ok(room_id) => { - banned_room_check( - &services, - sender_user, - Some(&room_id), - room_id.server_name(), - client, - ) - .boxed() - .await?; - - let mut servers = body.via.clone(); - servers.extend( - services - .rooms - .state_cache - .servers_invite_via(&room_id) - .map(ToOwned::to_owned) - .collect::>() - .await, - ); - - servers.extend( - services - .rooms - .state_cache - .invite_state(sender_user, &room_id) - .await - .unwrap_or_default() - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()), - ); - - if let Some(server) = room_id.server_name() { - servers.push(server.to_owned()); - } - - servers.sort_unstable(); - servers.dedup(); - shuffle(&mut servers); - - (servers, room_id) - }, - | Err(room_alias) => { - let (room_id, mut servers) = services - .rooms - .alias - .resolve_alias(&room_alias, Some(body.via.clone())) - .await?; - - banned_room_check( - &services, - sender_user, - Some(&room_id), - Some(room_alias.server_name()), - client, - ) - .await?; - - let addl_via_servers = services - .rooms - .state_cache - .servers_invite_via(&room_id) - .map(ToOwned::to_owned); - - let addl_state_servers = services - .rooms - .state_cache - .invite_state(sender_user, &room_id) - .await - .unwrap_or_default(); - - let mut addl_servers: Vec<_> = addl_state_servers - .iter() - .map(|event| event.get_field("sender")) - .filter_map(FlatOk::flat_ok) - .map(|user: &UserId| user.server_name().to_owned()) - .stream() - .chain(addl_via_servers) - .collect() - .await; - - addl_servers.sort_unstable(); - addl_servers.dedup(); - shuffle(&mut addl_servers); - servers.append(&mut addl_servers); - - (servers, room_id) - }, - }; - - let join_room_response = join_room_by_id_helper( - &services, - sender_user, - &room_id, - body.reason.clone(), - &servers, - body.third_party_signed.as_ref(), - appservice_info, - ) - .boxed() - .await?; - - Ok(join_room_by_id_or_alias::v3::Response { room_id: join_room_response.room_id }) -} - -/// # `POST /_matrix/client/*/knock/{roomIdOrAlias}` -/// -/// Tries to knock the room to ask permission to join for the sender user. -#[tracing::instrument(skip_all, fields(%client), name = "knock")] -pub(crate) async fn knock_room_route( - State(services): State, - InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result { - let sender_user = body.sender_user(); - let body = &body.body; - if services.users.is_suspended(sender_user).await? { - return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); - } - - let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias.clone()) { - | Ok(room_id) => { - banned_room_check( - &services, - sender_user, - Some(&room_id), - room_id.server_name(), - client, - ) - .await?; - - let mut servers = body.via.clone(); - servers.extend( - services - .rooms - .state_cache - .servers_invite_via(&room_id) - .map(ToOwned::to_owned) - .collect::>() - .await, - ); - - servers.extend( - services - .rooms - .state_cache - .invite_state(sender_user, &room_id) - .await - .unwrap_or_default() - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()), - ); - - if let Some(server) = room_id.server_name() { - servers.push(server.to_owned()); - } - - servers.sort_unstable(); - servers.dedup(); - shuffle(&mut servers); - - (servers, room_id) - }, - | Err(room_alias) => { - let (room_id, mut servers) = services - .rooms - .alias - .resolve_alias(&room_alias, Some(body.via.clone())) - .await?; - - banned_room_check( - &services, - sender_user, - Some(&room_id), - Some(room_alias.server_name()), - client, - ) - .await?; - - let addl_via_servers = services - .rooms - .state_cache - .servers_invite_via(&room_id) - .map(ToOwned::to_owned); - - let addl_state_servers = services - .rooms - .state_cache - .invite_state(sender_user, &room_id) - .await - .unwrap_or_default(); - - let mut addl_servers: Vec<_> = addl_state_servers - .iter() - .map(|event| event.get_field("sender")) - .filter_map(FlatOk::flat_ok) - .map(|user: &UserId| user.server_name().to_owned()) - .stream() - .chain(addl_via_servers) - .collect() - .await; - - addl_servers.sort_unstable(); - addl_servers.dedup(); - shuffle(&mut addl_servers); - servers.append(&mut addl_servers); - - (servers, room_id) - }, - }; - - knock_room_by_id_helper(&services, sender_user, &room_id, body.reason.clone(), &servers) - .boxed() - .await -} - -/// # `POST /_matrix/client/v3/rooms/{roomId}/leave` -/// -/// Tries to leave the sender user from a room. -/// -/// - This should always work if the user is currently joined. -pub(crate) async fn leave_room_route( - State(services): State, - body: Ruma, -) -> Result { - leave_room(&services, body.sender_user(), &body.room_id, body.reason.clone()) - .boxed() - .await - .map(|()| leave_room::v3::Response::new()) -} - -/// # `POST /_matrix/client/r0/rooms/{roomId}/invite` -/// -/// Tries to send an invite event into the room. -#[tracing::instrument(skip_all, fields(%client), name = "invite")] -pub(crate) async fn invite_user_route( - State(services): State, - InsecureClientIp(client): InsecureClientIp, - body: Ruma, -) -> Result { - let sender_user = body.sender_user(); - if services.users.is_suspended(sender_user).await? { - return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); - } - - if !services.users.is_admin(sender_user).await && services.config.block_non_admin_invites { - debug_error!( - "User {sender_user} is not an admin and attempted to send an invite to room {}", - &body.room_id - ); - return Err!(Request(Forbidden("Invites are not allowed on this server."))); - } - - banned_room_check( - &services, - sender_user, - Some(&body.room_id), - body.room_id.server_name(), - client, - ) - .await?; - - match &body.recipient { - | invite_user::v3::InvitationRecipient::UserId { user_id } => { - let sender_ignored_recipient = services.users.user_is_ignored(sender_user, user_id); - let recipient_ignored_by_sender = - services.users.user_is_ignored(user_id, sender_user); - - let (sender_ignored_recipient, recipient_ignored_by_sender) = - join!(sender_ignored_recipient, recipient_ignored_by_sender); - - if sender_ignored_recipient { - return Ok(invite_user::v3::Response {}); - } - - if let Ok(target_user_membership) = services - .rooms - .state_accessor - .get_member(&body.room_id, user_id) - .await - { - if target_user_membership.membership == MembershipState::Ban { - return Err!(Request(Forbidden("User is banned from this room."))); - } - } - - if recipient_ignored_by_sender { - // silently drop the invite to the recipient if they've been ignored by the - // sender, pretend it worked - return Ok(invite_user::v3::Response {}); - } - - invite_helper( - &services, - sender_user, - user_id, - &body.room_id, - body.reason.clone(), - false, - ) - .boxed() - .await?; - - Ok(invite_user::v3::Response {}) - }, - | _ => { - Err!(Request(NotFound("User not found."))) - }, - } -} - -/// # `POST /_matrix/client/r0/rooms/{roomId}/kick` -/// -/// Tries to send a kick event into the room. -pub(crate) async fn kick_user_route( - State(services): State, - body: Ruma, -) -> Result { - let sender_user = body.sender_user(); - if services.users.is_suspended(sender_user).await? { - return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); - } - let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - - let Ok(event) = services - .rooms - .state_accessor - .get_member(&body.room_id, &body.user_id) - .await - else { - // copy synapse's behaviour of returning 200 without any change to the state - // instead of erroring on left users - return Ok(kick_user::v3::Response::new()); - }; - - if !matches!( - event.membership, - MembershipState::Invite | MembershipState::Knock | MembershipState::Join, - ) { - return Err!(Request(Forbidden( - "Cannot kick a user who is not apart of the room (current membership: {})", - event.membership - ))); - } - - services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { - membership: MembershipState::Leave, - reason: body.reason.clone(), - is_direct: None, - join_authorized_via_users_server: None, - third_party_invite: None, - ..event - }), - sender_user, - &body.room_id, - &state_lock, - ) - .await?; - - drop(state_lock); - - Ok(kick_user::v3::Response::new()) -} - -/// # `POST /_matrix/client/r0/rooms/{roomId}/ban` -/// -/// Tries to send a ban event into the room. -pub(crate) async fn ban_user_route( - State(services): State, - body: Ruma, -) -> Result { - let sender_user = body.sender_user(); - - if sender_user == body.user_id { - return Err!(Request(Forbidden("You cannot ban yourself."))); - } - - if services.users.is_suspended(sender_user).await? { - return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); - } - - let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - - let current_member_content = services - .rooms - .state_accessor - .get_member(&body.room_id, &body.user_id) - .await - .unwrap_or_else(|_| RoomMemberEventContent::new(MembershipState::Ban)); - - services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { - membership: MembershipState::Ban, - reason: body.reason.clone(), - displayname: None, // display name may be offensive - avatar_url: None, // avatar may be offensive - is_direct: None, - join_authorized_via_users_server: None, - third_party_invite: None, - redact_events: body.redact_events, - ..current_member_content - }), - sender_user, - &body.room_id, - &state_lock, - ) - .await?; - - drop(state_lock); - - Ok(ban_user::v3::Response::new()) -} - -/// # `POST /_matrix/client/r0/rooms/{roomId}/unban` -/// -/// Tries to send an unban event into the room. -pub(crate) async fn unban_user_route( - State(services): State, - body: Ruma, -) -> Result { - let sender_user = body.sender_user(); - if services.users.is_suspended(sender_user).await? { - return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); - } - let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; - - let current_member_content = services - .rooms - .state_accessor - .get_member(&body.room_id, &body.user_id) - .await - .unwrap_or_else(|_| RoomMemberEventContent::new(MembershipState::Leave)); - - if current_member_content.membership != MembershipState::Ban { - return Err!(Request(Forbidden( - "Cannot unban a user who is not banned (current membership: {})", - current_member_content.membership - ))); - } - - services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { - membership: MembershipState::Leave, - reason: body.reason.clone(), - join_authorized_via_users_server: None, - third_party_invite: None, - is_direct: None, - ..current_member_content - }), - sender_user, - &body.room_id, - &state_lock, - ) - .await?; - - drop(state_lock); - - Ok(unban_user::v3::Response::new()) -} - -/// # `POST /_matrix/client/v3/rooms/{roomId}/forget` -/// -/// Forgets about a room. -/// -/// - If the sender user currently left the room: Stops sender user from -/// receiving information about the room -/// -/// Note: Other devices of the user have no way of knowing the room was -/// forgotten, so this has to be called from every device -pub(crate) async fn forget_room_route( - State(services): State, - body: Ruma, -) -> Result { - let user_id = body.sender_user(); - let room_id = &body.room_id; - - let joined = services.rooms.state_cache.is_joined(user_id, room_id); - let knocked = services.rooms.state_cache.is_knocked(user_id, room_id); - let invited = services.rooms.state_cache.is_invited(user_id, room_id); - - pin_mut!(joined, knocked, invited); - if joined.or(knocked).or(invited).await { - return Err!(Request(Unknown("You must leave the room before forgetting it"))); - } - - let membership = services - .rooms - .state_accessor - .get_member(room_id, user_id) - .await; - - if membership.is_not_found() { - return Err!(Request(Unknown("No membership event was found, room was never joined"))); - } - - let non_membership = membership - .map(|member| member.membership) - .is_ok_and(is_matching!(MembershipState::Leave | MembershipState::Ban)); - - if non_membership || services.rooms.state_cache.is_left(user_id, room_id).await { - services.rooms.state_cache.forget(room_id, user_id); - } - - Ok(forget_room::v3::Response::new()) -} - -/// # `POST /_matrix/client/r0/joined_rooms` -/// -/// Lists all rooms the user has joined. -pub(crate) async fn joined_rooms_route( - State(services): State, - body: Ruma, -) -> Result { - Ok(joined_rooms::v3::Response { - joined_rooms: services - .rooms - .state_cache - .rooms_joined(body.sender_user()) - .map(ToOwned::to_owned) - .collect() - .await, - }) -} - -fn membership_filter( - pdu: PduEvent, - for_membership: Option<&MembershipEventFilter>, - not_membership: Option<&MembershipEventFilter>, -) -> Option { - let membership_state_filter = match for_membership { - | Some(MembershipEventFilter::Ban) => MembershipState::Ban, - | Some(MembershipEventFilter::Invite) => MembershipState::Invite, - | Some(MembershipEventFilter::Knock) => MembershipState::Knock, - | Some(MembershipEventFilter::Leave) => MembershipState::Leave, - | Some(_) | None => MembershipState::Join, - }; - - let not_membership_state_filter = match not_membership { - | Some(MembershipEventFilter::Ban) => MembershipState::Ban, - | Some(MembershipEventFilter::Invite) => MembershipState::Invite, - | Some(MembershipEventFilter::Join) => MembershipState::Join, - | Some(MembershipEventFilter::Knock) => MembershipState::Knock, - | Some(_) | None => MembershipState::Leave, - }; - - let evt_membership = pdu.get_content::().ok()?.membership; - - if for_membership.is_some() && not_membership.is_some() { - if membership_state_filter != evt_membership - || not_membership_state_filter == evt_membership - { - None - } else { - Some(pdu) - } - } else if for_membership.is_some() && not_membership.is_none() { - if membership_state_filter != evt_membership { - None - } else { - Some(pdu) - } - } else if not_membership.is_some() && for_membership.is_none() { - if not_membership_state_filter == evt_membership { - None - } else { - Some(pdu) - } - } else { - Some(pdu) - } -} - -/// # `POST /_matrix/client/r0/rooms/{roomId}/members` -/// -/// Lists all joined users in a room (TODO: at a specific point in time, with a -/// specific membership). -/// -/// - Only works if the user is currently joined -pub(crate) async fn get_member_events_route( - State(services): State, - body: Ruma, -) -> Result { - let sender_user = body.sender_user(); - let membership = body.membership.as_ref(); - let not_membership = body.not_membership.as_ref(); - - if !services - .rooms - .state_accessor - .user_can_see_state_events(sender_user, &body.room_id) - .await - { - return Err!(Request(Forbidden("You don't have permission to view this room."))); - } - - Ok(get_member_events::v3::Response { - chunk: services - .rooms - .state_accessor - .room_state_full(&body.room_id) - .ready_filter_map(Result::ok) - .ready_filter(|((ty, _), _)| *ty == StateEventType::RoomMember) - .map(at!(1)) - .ready_filter_map(|pdu| membership_filter(pdu, membership, not_membership)) - .map(Event::into_format) - .collect() - .await, - }) -} - -/// # `POST /_matrix/client/r0/rooms/{roomId}/joined_members` -/// -/// Lists all members of a room. -/// -/// - The sender user must be in the room -/// - TODO: An appservice just needs a puppet joined -pub(crate) async fn joined_members_route( - State(services): State, - body: Ruma, -) -> Result { - if !services - .rooms - .state_accessor - .user_can_see_state_events(body.sender_user(), &body.room_id) - .await - { - return Err!(Request(Forbidden("You don't have permission to view this room."))); - } - - Ok(joined_members::v3::Response { - joined: services - .rooms - .state_cache - .room_members(&body.room_id) - .map(ToOwned::to_owned) - .broad_then(|user_id| async move { - let member = RoomMember { - display_name: services.users.displayname(&user_id).await.ok(), - avatar_url: services.users.avatar_url(&user_id).await.ok(), - }; - - (user_id, member) - }) - .collect() - .await, - }) -} - -pub async fn join_room_by_id_helper( - services: &Services, - sender_user: &UserId, - room_id: &RoomId, - reason: Option, - servers: &[OwnedServerName], - third_party_signed: Option<&ThirdPartySigned>, - appservice_info: &Option, -) -> Result { - let state_lock = services.rooms.state.mutex.lock(room_id).await; - - let user_is_guest = services - .users - .is_deactivated(sender_user) - .await - .unwrap_or(false) - && appservice_info.is_none(); - - if user_is_guest && !services.rooms.state_accessor.guest_can_join(room_id).await { - return Err!(Request(Forbidden("Guests are not allowed to join this room"))); - } - - if services - .rooms - .state_cache - .is_joined(sender_user, room_id) - .await - { - debug_warn!("{sender_user} is already joined in {room_id}"); - return Ok(join_room_by_id::v3::Response { room_id: room_id.into() }); - } - - let server_in_room = services - .rooms - .state_cache - .server_in_room(services.globals.server_name(), room_id) - .await; - - // Only check our known membership if we're already in the room. - // See: https://forgejo.ellis.link/continuwuation/continuwuity/issues/855 - let membership = if server_in_room { - services - .rooms - .state_accessor - .get_member(room_id, sender_user) - .await - } else { - debug!("Ignoring local state for join {room_id}, we aren't in the room yet."); - Ok(RoomMemberEventContent::new(MembershipState::Leave)) - }; - if let Ok(m) = membership { - if m.membership == MembershipState::Ban { - debug_warn!("{sender_user} is banned from {room_id} but attempted to join"); - // TODO: return reason - return Err!(Request(Forbidden("You are banned from the room."))); - } - } - - let local_join = server_in_room - || servers.is_empty() - || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); - - if local_join { - join_room_by_id_helper_local( - services, - sender_user, - room_id, - reason, - servers, - third_party_signed, - state_lock, - ) - .boxed() - .await?; - } else { - // Ask a remote server if we are not participating in this room - join_room_by_id_helper_remote( - services, - sender_user, - room_id, - reason, - servers, - third_party_signed, - state_lock, - ) - .boxed() - .await?; - } - - Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) -} - -#[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote")] -async fn join_room_by_id_helper_remote( - services: &Services, - sender_user: &UserId, - room_id: &RoomId, - reason: Option, - servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, - state_lock: RoomMutexGuard, -) -> Result { - info!("Joining {room_id} over federation."); - - let (make_join_response, remote_server) = - make_join_request(services, sender_user, room_id, servers).await?; - - info!("make_join finished"); - - let Some(room_version_id) = make_join_response.room_version else { - return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); - }; - - if !services.server.supported_room_version(&room_version_id) { - return Err!(BadServerResponse( - "Remote room version {room_version_id} is not supported by conduwuit" - )); - } - - let mut join_event_stub: CanonicalJsonObject = - serde_json::from_str(make_join_response.event.get()).map_err(|e| { - err!(BadServerResponse(warn!( - "Invalid make_join event json received from server: {e:?}" - ))) - })?; - - let join_authorized_via_users_server = { - use RoomVersionId::*; - if !matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { - join_event_stub - .get("content") - .map(|s| { - s.as_object()? - .get("join_authorised_via_users_server")? - .as_str() - }) - .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()) - } else { - None - } - }; - - join_event_stub.insert( - "origin".to_owned(), - CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), - ); - join_event_stub.insert( - "origin_server_ts".to_owned(), - CanonicalJsonValue::Integer( - utils::millis_since_unix_epoch() - .try_into() - .expect("Timestamp is valid js_int value"), - ), - ); - join_event_stub.insert( - "content".to_owned(), - to_canonical_value(RoomMemberEventContent { - displayname: services.users.displayname(sender_user).await.ok(), - avatar_url: services.users.avatar_url(sender_user).await.ok(), - blurhash: services.users.blurhash(sender_user).await.ok(), - reason, - join_authorized_via_users_server: join_authorized_via_users_server.clone(), - ..RoomMemberEventContent::new(MembershipState::Join) - }) - .expect("event is valid, we just created it"), - ); - - // We keep the "event_id" in the pdu only in v1 or - // v2 rooms - match room_version_id { - | RoomVersionId::V1 | RoomVersionId::V2 => {}, - | _ => { - join_event_stub.remove("event_id"); - }, - } - - // In order to create a compatible ref hash (EventID) the `hashes` field needs - // to be present - services - .server_keys - .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; - - // Generate event id - let event_id = gen_event_id(&join_event_stub, &room_version_id)?; - - // Add event_id back - join_event_stub - .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); - - // It has enough fields to be called a proper event now - let mut join_event = join_event_stub; - - info!("Asking {remote_server} for send_join in room {room_id}"); - let send_join_request = federation::membership::create_join_event::v2::Request { - room_id: room_id.to_owned(), - event_id: event_id.clone(), - omit_members: false, - pdu: services - .sending - .convert_to_outgoing_federation_event(join_event.clone()) - .await, - }; - - let send_join_response = match services - .sending - .send_synapse_request(&remote_server, send_join_request) - .await - { - | Ok(response) => response, - | Err(e) => { - error!("send_join failed: {e}"); - return Err(e); - }, - }; - - info!("send_join finished"); - - if join_authorized_via_users_server.is_some() { - if let Some(signed_raw) = &send_join_response.room_state.event { - debug_info!( - "There is a signed event with join_authorized_via_users_server. This room is \ - probably using restricted joins. Adding signature to our event" - ); - - let (signed_event_id, signed_value) = - gen_event_id_canonical_json(signed_raw, &room_version_id).map_err(|e| { - err!(Request(BadJson(warn!( - "Could not convert event to canonical JSON: {e}" - )))) - })?; - - if signed_event_id != event_id { - return Err!(Request(BadJson(warn!( - %signed_event_id, %event_id, - "Server {remote_server} sent event with wrong event ID" - )))); - } - - match signed_value["signatures"] - .as_object() - .ok_or_else(|| { - err!(BadServerResponse(warn!( - "Server {remote_server} sent invalid signatures type" - ))) - }) - .and_then(|e| { - e.get(remote_server.as_str()).ok_or_else(|| { - err!(BadServerResponse(warn!( - "Server {remote_server} did not send its signature for a restricted \ - room" - ))) - }) - }) { - | Ok(signature) => { - join_event - .get_mut("signatures") - .expect("we created a valid pdu") - .as_object_mut() - .expect("we created a valid pdu") - .insert(remote_server.to_string(), signature.clone()); - }, - | Err(e) => { - warn!( - "Server {remote_server} sent invalid signature in send_join signatures \ - for event {signed_value:?}: {e:?}", - ); - }, - } - } - } - - services - .rooms - .short - .get_or_create_shortroomid(room_id) - .await; - - info!("Parsing join event"); - let parsed_join_pdu = PduEvent::from_id_val(&event_id, join_event.clone()) - .map_err(|e| err!(BadServerResponse("Invalid join event PDU: {e:?}")))?; - - info!("Acquiring server signing keys for response events"); - let resp_events = &send_join_response.room_state; - let resp_state = &resp_events.state; - let resp_auth = &resp_events.auth_chain; - services - .server_keys - .acquire_events_pubkeys(resp_auth.iter().chain(resp_state.iter())) - .await; - - info!("Going through send_join response room_state"); - let cork = services.db.cork_and_flush(); - let state = send_join_response - .room_state - .state - .iter() - .stream() - .then(|pdu| { - services - .server_keys - .validate_and_add_event_id_no_fetch(pdu, &room_version_id) - }) - .ready_filter_map(Result::ok) - .fold(HashMap::new(), |mut state, (event_id, value)| async move { - let pdu = match PduEvent::from_id_val(&event_id, value.clone()) { - | Ok(pdu) => pdu, - | Err(e) => { - debug_warn!("Invalid PDU in send_join response: {e:?}: {value:#?}"); - return state; - }, - }; - - services.rooms.outlier.add_pdu_outlier(&event_id, &value); - if let Some(state_key) = &pdu.state_key { - let shortstatekey = services - .rooms - .short - .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key) - .await; - - state.insert(shortstatekey, pdu.event_id.clone()); - } - - state - }) - .await; - - drop(cork); - - info!("Going through send_join response auth_chain"); - let cork = services.db.cork_and_flush(); - send_join_response - .room_state - .auth_chain - .iter() - .stream() - .then(|pdu| { - services - .server_keys - .validate_and_add_event_id_no_fetch(pdu, &room_version_id) - }) - .ready_filter_map(Result::ok) - .ready_for_each(|(event_id, value)| { - services.rooms.outlier.add_pdu_outlier(&event_id, &value); - }) - .await; - - drop(cork); - - debug!("Running send_join auth check"); - let fetch_state = &state; - let state_fetch = |k: StateEventType, s: StateKey| async move { - let shortstatekey = services.rooms.short.get_shortstatekey(&k, &s).await.ok()?; - - let event_id = fetch_state.get(&shortstatekey)?; - services.rooms.timeline.get_pdu(event_id).await.ok() - }; - - let auth_check = state_res::event_auth::auth_check( - &state_res::RoomVersion::new(&room_version_id)?, - &parsed_join_pdu, - None, // TODO: third party invite - |k, s| state_fetch(k.clone(), s.into()), - ) - .await - .map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?; - - if !auth_check { - return Err!(Request(Forbidden("Auth check failed"))); - } - - info!("Compressing state from send_join"); - let compressed: CompressedState = services - .rooms - .state_compressor - .compress_state_events(state.iter().map(|(ssk, eid)| (ssk, eid.borrow()))) - .collect() - .await; - - debug!("Saving compressed state"); - let HashSetCompressStateEvent { - shortstatehash: statehash_before_join, - added, - removed, - } = services - .rooms - .state_compressor - .save_state(room_id, Arc::new(compressed)) - .await?; - - debug!("Forcing state for new room"); - services - .rooms - .state - .force_state(room_id, statehash_before_join, added, removed, &state_lock) - .await?; - - info!("Updating joined counts for new room"); - services - .rooms - .state_cache - .update_joined_count(room_id) - .await; - - // We append to state before appending the pdu, so we don't have a moment in - // time with the pdu without it's state. This is okay because append_pdu can't - // fail. - let statehash_after_join = services - .rooms - .state - .append_to_state(&parsed_join_pdu) - .await?; - - info!("Appending new room join event"); - services - .rooms - .timeline - .append_pdu( - &parsed_join_pdu, - join_event, - once(parsed_join_pdu.event_id.borrow()), - &state_lock, - ) - .await?; - - info!("Setting final room state for new room"); - // We set the room state after inserting the pdu, so that we never have a moment - // in time where events in the current room state do not exist - services - .rooms - .state - .set_room_state(room_id, statehash_after_join, &state_lock); - - Ok(()) -} - -#[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_local")] -async fn join_room_by_id_helper_local( - services: &Services, - sender_user: &UserId, - room_id: &RoomId, - reason: Option, - servers: &[OwnedServerName], - _third_party_signed: Option<&ThirdPartySigned>, - state_lock: RoomMutexGuard, -) -> Result { - debug_info!("We can join locally"); - - let join_rules_event_content = services - .rooms - .state_accessor - .room_state_get_content::( - room_id, - &StateEventType::RoomJoinRules, - "", - ) - .await; - - let restriction_rooms = match join_rules_event_content { - | Ok(RoomJoinRulesEventContent { - join_rule: JoinRule::Restricted(restricted) | JoinRule::KnockRestricted(restricted), - }) => restricted - .allow - .into_iter() - .filter_map(|a| match a { - | AllowRule::RoomMembership(r) => Some(r.room_id), - | _ => None, - }) - .collect(), - | _ => Vec::new(), - }; - - let join_authorized_via_users_server: Option = { - if restriction_rooms - .iter() - .stream() - .any(|restriction_room_id| { - services - .rooms - .state_cache - .is_joined(sender_user, restriction_room_id) - }) - .await - { - services - .rooms - .state_cache - .local_users_in_room(room_id) - .filter(|user| { - services.rooms.state_accessor.user_can_invite( - room_id, - user, - sender_user, - &state_lock, - ) - }) - .boxed() - .next() - .await - .map(ToOwned::to_owned) - } else { - None - } - }; - - let content = RoomMemberEventContent { - displayname: services.users.displayname(sender_user).await.ok(), - avatar_url: services.users.avatar_url(sender_user).await.ok(), - blurhash: services.users.blurhash(sender_user).await.ok(), - reason: reason.clone(), - join_authorized_via_users_server, - ..RoomMemberEventContent::new(MembershipState::Join) - }; - - // Try normal join first - let Err(error) = services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder::state(sender_user.to_string(), &content), - sender_user, - room_id, - &state_lock, - ) - .await - else { - return Ok(()); - }; - - if restriction_rooms.is_empty() - && (servers.is_empty() - || servers.len() == 1 && services.globals.server_is_ours(&servers[0])) - { - return Err(error); - } - - warn!( - "We couldn't do the join locally, maybe federation can help to satisfy the restricted \ - join requirements" - ); - let Ok((make_join_response, remote_server)) = - make_join_request(services, sender_user, room_id, servers).await - else { - return Err(error); - }; - - let Some(room_version_id) = make_join_response.room_version else { - return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); - }; - - if !services.server.supported_room_version(&room_version_id) { - return Err!(BadServerResponse( - "Remote room version {room_version_id} is not supported by conduwuit" - )); - } - - let mut join_event_stub: CanonicalJsonObject = - serde_json::from_str(make_join_response.event.get()).map_err(|e| { - err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")) - })?; - - let join_authorized_via_users_server = join_event_stub - .get("content") - .map(|s| { - s.as_object()? - .get("join_authorised_via_users_server")? - .as_str() - }) - .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()); - - join_event_stub.insert( - "origin".to_owned(), - CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), - ); - join_event_stub.insert( - "origin_server_ts".to_owned(), - CanonicalJsonValue::Integer( - utils::millis_since_unix_epoch() - .try_into() - .expect("Timestamp is valid js_int value"), - ), - ); - join_event_stub.insert( - "content".to_owned(), - to_canonical_value(RoomMemberEventContent { - displayname: services.users.displayname(sender_user).await.ok(), - avatar_url: services.users.avatar_url(sender_user).await.ok(), - blurhash: services.users.blurhash(sender_user).await.ok(), - reason, - join_authorized_via_users_server, - ..RoomMemberEventContent::new(MembershipState::Join) - }) - .expect("event is valid, we just created it"), - ); - - // We keep the "event_id" in the pdu only in v1 or - // v2 rooms - match room_version_id { - | RoomVersionId::V1 | RoomVersionId::V2 => {}, - | _ => { - join_event_stub.remove("event_id"); - }, - } - - // In order to create a compatible ref hash (EventID) the `hashes` field needs - // to be present - services - .server_keys - .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; - - // Generate event id - let event_id = gen_event_id(&join_event_stub, &room_version_id)?; - - // Add event_id back - join_event_stub - .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); - - // It has enough fields to be called a proper event now - let join_event = join_event_stub; - - let send_join_response = services - .sending - .send_synapse_request( - &remote_server, - federation::membership::create_join_event::v2::Request { - room_id: room_id.to_owned(), - event_id: event_id.clone(), - omit_members: false, - pdu: services - .sending - .convert_to_outgoing_federation_event(join_event.clone()) - .await, - }, - ) - .await?; - - if let Some(signed_raw) = send_join_response.room_state.event { - let (signed_event_id, signed_value) = - gen_event_id_canonical_json(&signed_raw, &room_version_id).map_err(|e| { - err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))) - })?; - - if signed_event_id != event_id { - return Err!(Request(BadJson( - warn!(%signed_event_id, %event_id, "Server {remote_server} sent event with wrong event ID") - ))); - } - - drop(state_lock); - services - .rooms - .event_handler - .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true) - .boxed() - .await?; - } else { - return Err(error); - } - - Ok(()) -} - -async fn make_join_request( - services: &Services, - sender_user: &UserId, - room_id: &RoomId, - servers: &[OwnedServerName], -) -> Result<(federation::membership::prepare_join_event::v1::Response, OwnedServerName)> { - let mut make_join_response_and_server = - Err!(BadServerResponse("No server available to assist in joining.")); - - let mut make_join_counter: usize = 0; - let mut incompatible_room_version_count: usize = 0; - - for remote_server in servers { - if services.globals.server_is_ours(remote_server) { - continue; - } - info!("Asking {remote_server} for make_join ({make_join_counter})"); - let make_join_response = services - .sending - .send_federation_request( - remote_server, - federation::membership::prepare_join_event::v1::Request { - room_id: room_id.to_owned(), - user_id: sender_user.to_owned(), - ver: services.server.supported_room_versions().collect(), - }, - ) - .await; - - trace!("make_join response: {:?}", make_join_response); - make_join_counter = make_join_counter.saturating_add(1); - - if let Err(ref e) = make_join_response { - if matches!( - e.kind(), - ErrorKind::IncompatibleRoomVersion { .. } | ErrorKind::UnsupportedRoomVersion - ) { - incompatible_room_version_count = - incompatible_room_version_count.saturating_add(1); - } - - if incompatible_room_version_count > 15 { - info!( - "15 servers have responded with M_INCOMPATIBLE_ROOM_VERSION or \ - M_UNSUPPORTED_ROOM_VERSION, assuming that conduwuit does not support the \ - room version {room_id}: {e}" - ); - make_join_response_and_server = - Err!(BadServerResponse("Room version is not supported by Conduwuit")); - return make_join_response_and_server; - } - - if make_join_counter > 40 { - warn!( - "40 servers failed to provide valid make_join response, assuming no server \ - can assist in joining." - ); - make_join_response_and_server = - Err!(BadServerResponse("No server available to assist in joining.")); - - return make_join_response_and_server; - } - } - - make_join_response_and_server = make_join_response.map(|r| (r, remote_server.clone())); - - if make_join_response_and_server.is_ok() { - break; - } - } - - make_join_response_and_server -} - -pub(crate) async fn invite_helper( - services: &Services, - sender_user: &UserId, - user_id: &UserId, - room_id: &RoomId, - reason: Option, - is_direct: bool, -) -> Result { - if !services.users.is_admin(sender_user).await && services.config.block_non_admin_invites { - info!( - "User {sender_user} is not an admin and attempted to send an invite to room \ - {room_id}" - ); - return Err!(Request(Forbidden("Invites are not allowed on this server."))); - } - - if !services.globals.user_is_local(user_id) { - let (pdu, pdu_json, invite_room_state) = { - let state_lock = services.rooms.state.mutex.lock(room_id).await; - - let content = RoomMemberEventContent { - avatar_url: services.users.avatar_url(user_id).await.ok(), - is_direct: Some(is_direct), - reason, - ..RoomMemberEventContent::new(MembershipState::Invite) - }; - - let (pdu, pdu_json) = services - .rooms - .timeline - .create_hash_and_sign_event( - PduBuilder::state(user_id.to_string(), &content), - sender_user, - room_id, - &state_lock, - ) - .await?; - - let invite_room_state = services.rooms.state.summary_stripped(&pdu).await; - - drop(state_lock); - - (pdu, pdu_json, invite_room_state) - }; - - let room_version_id = services.rooms.state.get_room_version(room_id).await?; - - let response = services - .sending - .send_federation_request(user_id.server_name(), create_invite::v2::Request { - room_id: room_id.to_owned(), - event_id: (*pdu.event_id).to_owned(), - room_version: room_version_id.clone(), - event: services - .sending - .convert_to_outgoing_federation_event(pdu_json.clone()) - .await, - invite_room_state, - via: services - .rooms - .state_cache - .servers_route_via(room_id) - .await - .ok(), - }) - .await?; - - // We do not add the event_id field to the pdu here because of signature and - // hashes checks - let (event_id, value) = gen_event_id_canonical_json(&response.event, &room_version_id) - .map_err(|e| { - err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))) - })?; - - if pdu.event_id != event_id { - return Err!(Request(BadJson(warn!( - %pdu.event_id, %event_id, - "Server {} sent event with wrong event ID", - user_id.server_name() - )))); - } - - let origin: OwnedServerName = serde_json::from_value(serde_json::to_value( - value - .get("origin") - .ok_or_else(|| err!(Request(BadJson("Event missing origin field."))))?, - )?) - .map_err(|e| { - err!(Request(BadJson(warn!("Origin field in event is not a valid server name: {e}")))) - })?; - - let pdu_id = services - .rooms - .event_handler - .handle_incoming_pdu(&origin, room_id, &event_id, value, true) - .boxed() - .await? - .ok_or_else(|| { - err!(Request(InvalidParam("Could not accept incoming PDU as timeline event."))) - })?; - - return services.sending.send_pdu_room(room_id, &pdu_id).await; - } - - if !services - .rooms - .state_cache - .is_joined(sender_user, room_id) - .await - { - return Err!(Request(Forbidden( - "You must be joined in the room you are trying to invite from." - ))); - } - - let state_lock = services.rooms.state.mutex.lock(room_id).await; - - let content = RoomMemberEventContent { - displayname: services.users.displayname(user_id).await.ok(), - avatar_url: services.users.avatar_url(user_id).await.ok(), - blurhash: services.users.blurhash(user_id).await.ok(), - is_direct: Some(is_direct), - reason, - ..RoomMemberEventContent::new(MembershipState::Invite) - }; - - services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder::state(user_id.to_string(), &content), - sender_user, - room_id, - &state_lock, - ) - .await?; - - drop(state_lock); - - Ok(()) -} - -// Make a user leave all their joined rooms, rescinds knocks, forgets all rooms, -// and ignores errors -pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { - let rooms_joined = services - .rooms - .state_cache - .rooms_joined(user_id) - .map(ToOwned::to_owned); - - let rooms_invited = services - .rooms - .state_cache - .rooms_invited(user_id) - .map(|(r, _)| r); - - let rooms_knocked = services - .rooms - .state_cache - .rooms_knocked(user_id) - .map(|(r, _)| r); - - let all_rooms: Vec<_> = rooms_joined - .chain(rooms_invited) - .chain(rooms_knocked) - .collect() - .await; - - for room_id in all_rooms { - // ignore errors - if let Err(e) = leave_room(services, user_id, &room_id, None).boxed().await { - warn!(%user_id, "Failed to leave {room_id} remotely: {e}"); - } - - services.rooms.state_cache.forget(&room_id, user_id); - } -} - -pub async fn leave_room( - services: &Services, - user_id: &UserId, - room_id: &RoomId, - reason: Option, -) -> Result { - let default_member_content = RoomMemberEventContent { - membership: MembershipState::Leave, - reason: reason.clone(), - join_authorized_via_users_server: None, - is_direct: None, - avatar_url: None, - displayname: None, - third_party_invite: None, - blurhash: None, - redact_events: None, - }; - - let is_banned = services.rooms.metadata.is_banned(room_id); - let is_disabled = services.rooms.metadata.is_disabled(room_id); - - pin_mut!(is_banned, is_disabled); - if is_banned.or(is_disabled).await { - // the room is banned/disabled, the room must be rejected locally since we - // cant/dont want to federate with this server - services - .rooms - .state_cache - .update_membership( - room_id, - user_id, - default_member_content, - user_id, - None, - None, - true, - ) - .await?; - - return Ok(()); - } - - let dont_have_room = services - .rooms - .state_cache - .server_in_room(services.globals.server_name(), room_id) - .eq(&false); - - let not_knocked = services - .rooms - .state_cache - .is_knocked(user_id, room_id) - .eq(&false); - - // Ask a remote server if we don't have this room and are not knocking on it - if dont_have_room.and(not_knocked).await { - if let Err(e) = remote_leave_room(services, user_id, room_id, reason.clone()) - .boxed() - .await - { - warn!(%user_id, "Failed to leave room {room_id} remotely: {e}"); - // Don't tell the client about this error - } - - let last_state = services - .rooms - .state_cache - .invite_state(user_id, room_id) - .or_else(|_| services.rooms.state_cache.knock_state(user_id, room_id)) - .or_else(|_| services.rooms.state_cache.left_state(user_id, room_id)) - .await - .ok(); - - // We always drop the invite, we can't rely on other servers - services - .rooms - .state_cache - .update_membership( - room_id, - user_id, - default_member_content, - user_id, - last_state, - None, - true, - ) - .await?; - } else { - let state_lock = services.rooms.state.mutex.lock(room_id).await; - - let Ok(event) = services - .rooms - .state_accessor - .room_state_get_content::( - room_id, - &StateEventType::RoomMember, - user_id.as_str(), - ) - .await - else { - debug_warn!( - "Trying to leave a room you are not a member of, marking room as left locally." - ); - - return services - .rooms - .state_cache - .update_membership( - room_id, - user_id, - default_member_content, - user_id, - None, - None, - true, - ) - .await; - }; - - services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder::state(user_id.to_string(), &RoomMemberEventContent { - membership: MembershipState::Leave, - reason, - join_authorized_via_users_server: None, - is_direct: None, - ..event - }), - user_id, - room_id, - &state_lock, - ) - .await?; - } - - Ok(()) -} - -async fn remote_leave_room( - services: &Services, - user_id: &UserId, - room_id: &RoomId, - reason: Option, -) -> Result<()> { - let mut make_leave_response_and_server = - Err!(BadServerResponse("No remote server available to assist in leaving {room_id}.")); - - let mut servers: HashSet = services - .rooms - .state_cache - .servers_invite_via(room_id) - .map(ToOwned::to_owned) - .collect() - .await; - - match services - .rooms - .state_cache - .invite_state(user_id, room_id) - .await - { - | Ok(invite_state) => { - servers.extend( - invite_state - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()), - ); - }, - | _ => { - match services - .rooms - .state_cache - .knock_state(user_id, room_id) - .await - { - | Ok(knock_state) => { - servers.extend( - knock_state - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .filter_map(|sender| { - if !services.globals.user_is_local(sender) { - Some(sender.server_name().to_owned()) - } else { - None - } - }), - ); - }, - | _ => {}, - } - }, - } - - if let Some(room_id_server_name) = room_id.server_name() { - servers.insert(room_id_server_name.to_owned()); - } - - debug_info!("servers in remote_leave_room: {servers:?}"); - - for remote_server in servers { - let make_leave_response = services - .sending - .send_federation_request( - &remote_server, - federation::membership::prepare_leave_event::v1::Request { - room_id: room_id.to_owned(), - user_id: user_id.to_owned(), - }, - ) - .await; - - make_leave_response_and_server = make_leave_response.map(|r| (r, remote_server)); - - if make_leave_response_and_server.is_ok() { - break; - } - } - - let (make_leave_response, remote_server) = make_leave_response_and_server?; - - let Some(room_version_id) = make_leave_response.room_version else { - return Err!(BadServerResponse(warn!( - "No room version was returned by {remote_server} for {room_id}, room version is \ - likely not supported by conduwuit" - ))); - }; - - if !services.server.supported_room_version(&room_version_id) { - return Err!(BadServerResponse(warn!( - "Remote room version {room_version_id} for {room_id} is not supported by conduwuit", - ))); - } - - let mut leave_event_stub = serde_json::from_str::( - make_leave_response.event.get(), - ) - .map_err(|e| { - err!(BadServerResponse(warn!( - "Invalid make_leave event json received from {remote_server} for {room_id}: {e:?}" - ))) - })?; - - // TODO: Is origin needed? - leave_event_stub.insert( - "origin".to_owned(), - CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), - ); - leave_event_stub.insert( - "origin_server_ts".to_owned(), - CanonicalJsonValue::Integer( - utils::millis_since_unix_epoch() - .try_into() - .expect("Timestamp is valid js_int value"), - ), - ); - // Inject the reason key into the event content dict if it exists - if let Some(reason) = reason { - if let Some(CanonicalJsonValue::Object(content)) = leave_event_stub.get_mut("content") { - content.insert("reason".to_owned(), CanonicalJsonValue::String(reason)); - } - } - - // room v3 and above removed the "event_id" field from remote PDU format - match room_version_id { - | RoomVersionId::V1 | RoomVersionId::V2 => {}, - | _ => { - leave_event_stub.remove("event_id"); - }, - } - - // In order to create a compatible ref hash (EventID) the `hashes` field needs - // to be present - services - .server_keys - .hash_and_sign_event(&mut leave_event_stub, &room_version_id)?; - - // Generate event id - let event_id = gen_event_id(&leave_event_stub, &room_version_id)?; - - // Add event_id back - leave_event_stub - .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); - - // It has enough fields to be called a proper event now - let leave_event = leave_event_stub; - - services - .sending - .send_federation_request( - &remote_server, - federation::membership::create_leave_event::v2::Request { - room_id: room_id.to_owned(), - event_id, - pdu: services - .sending - .convert_to_outgoing_federation_event(leave_event.clone()) - .await, - }, - ) - .await?; - - Ok(()) -} - -async fn knock_room_by_id_helper( - services: &Services, - sender_user: &UserId, - room_id: &RoomId, - reason: Option, - servers: &[OwnedServerName], -) -> Result { - let state_lock = services.rooms.state.mutex.lock(room_id).await; - - if services - .rooms - .state_cache - .is_invited(sender_user, room_id) - .await - { - debug_warn!("{sender_user} is already invited in {room_id} but attempted to knock"); - return Err!(Request(Forbidden( - "You cannot knock on a room you are already invited/accepted to." - ))); - } - - if services - .rooms - .state_cache - .is_joined(sender_user, room_id) - .await - { - debug_warn!("{sender_user} is already joined in {room_id} but attempted to knock"); - return Err!(Request(Forbidden("You cannot knock on a room you are already joined in."))); - } - - if services - .rooms - .state_cache - .is_knocked(sender_user, room_id) - .await - { - debug_warn!("{sender_user} is already knocked in {room_id}"); - return Ok(knock_room::v3::Response { room_id: room_id.into() }); - } - - if let Ok(membership) = services - .rooms - .state_accessor - .get_member(room_id, sender_user) - .await - { - if membership.membership == MembershipState::Ban { - debug_warn!("{sender_user} is banned from {room_id} but attempted to knock"); - return Err!(Request(Forbidden("You cannot knock on a room you are banned from."))); - } - } - - // For knock_restricted rooms, check if the user meets the restricted conditions - // If they do, attempt to join instead of knock - // This is not mentioned in the spec, but should be allowable (we're allowed to - // auto-join invites to knocked rooms) - let join_rule = services.rooms.state_accessor.get_join_rules(room_id).await; - if let JoinRule::KnockRestricted(restricted) = &join_rule { - let restriction_rooms: Vec<_> = restricted - .allow - .iter() - .filter_map(|a| match a { - | AllowRule::RoomMembership(r) => Some(&r.room_id), - | _ => None, - }) - .collect(); - - // Check if the user is in any of the allowed rooms - let mut user_meets_restrictions = false; - for restriction_room_id in &restriction_rooms { - if services - .rooms - .state_cache - .is_joined(sender_user, restriction_room_id) - .await - { - user_meets_restrictions = true; - break; - } - } - - // If the user meets the restrictions, try joining instead - if user_meets_restrictions { - debug_info!( - "{sender_user} meets the restricted criteria in knock_restricted room \ - {room_id}, attempting to join instead of knock" - ); - // For this case, we need to drop the state lock and get a new one in - // join_room_by_id_helper We need to release the lock here and let - // join_room_by_id_helper acquire it again - drop(state_lock); - match join_room_by_id_helper( - services, - sender_user, - room_id, - reason.clone(), - servers, - None, - &None, - ) - .await - { - | Ok(_) => return Ok(knock_room::v3::Response::new(room_id.to_owned())), - | Err(e) => { - debug_warn!( - "Failed to convert knock to join for {sender_user} in {room_id}: {e:?}" - ); - // Get a new state lock for the remaining knock logic - let new_state_lock = services.rooms.state.mutex.lock(room_id).await; - - let server_in_room = services - .rooms - .state_cache - .server_in_room(services.globals.server_name(), room_id) - .await; - - let local_knock = server_in_room - || servers.is_empty() - || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); - - if local_knock { - knock_room_helper_local( - services, - sender_user, - room_id, - reason, - servers, - new_state_lock, - ) - .boxed() - .await?; - } else { - knock_room_helper_remote( - services, - sender_user, - room_id, - reason, - servers, - new_state_lock, - ) - .boxed() - .await?; - } - - return Ok(knock_room::v3::Response::new(room_id.to_owned())); - }, - } - } - } else if !matches!(join_rule, JoinRule::Knock | JoinRule::KnockRestricted(_)) { - debug_warn!( - "{sender_user} attempted to knock on room {room_id} but its join rule is \ - {join_rule:?}, not knock or knock_restricted" - ); - } - - let server_in_room = services - .rooms - .state_cache - .server_in_room(services.globals.server_name(), room_id) - .await; - - let local_knock = server_in_room - || servers.is_empty() - || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); - - if local_knock { - knock_room_helper_local(services, sender_user, room_id, reason, servers, state_lock) - .boxed() - .await?; - } else { - knock_room_helper_remote(services, sender_user, room_id, reason, servers, state_lock) - .boxed() - .await?; - } - - Ok(knock_room::v3::Response::new(room_id.to_owned())) -} - -async fn knock_room_helper_local( - services: &Services, - sender_user: &UserId, - room_id: &RoomId, - reason: Option, - servers: &[OwnedServerName], - state_lock: RoomMutexGuard, -) -> Result { - debug_info!("We can knock locally"); - - let room_version_id = services.rooms.state.get_room_version(room_id).await?; - - if matches!( - room_version_id, - RoomVersionId::V1 - | RoomVersionId::V2 - | RoomVersionId::V3 - | RoomVersionId::V4 - | RoomVersionId::V5 - | RoomVersionId::V6 - ) { - return Err!(Request(Forbidden("This room does not support knocking."))); - } - - // Verify that this room has a valid knock or knock_restricted join rule - let join_rule = services.rooms.state_accessor.get_join_rules(room_id).await; - if !matches!(join_rule, JoinRule::Knock | JoinRule::KnockRestricted(_)) { - return Err!(Request(Forbidden("This room's join rule does not allow knocking."))); - } - - let content = RoomMemberEventContent { - displayname: services.users.displayname(sender_user).await.ok(), - avatar_url: services.users.avatar_url(sender_user).await.ok(), - blurhash: services.users.blurhash(sender_user).await.ok(), - reason: reason.clone(), - ..RoomMemberEventContent::new(MembershipState::Knock) - }; - - // Try normal knock first - let Err(error) = services - .rooms - .timeline - .build_and_append_pdu( - PduBuilder::state(sender_user.to_string(), &content), - sender_user, - room_id, - &state_lock, - ) - .await - else { - return Ok(()); - }; - - if servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])) - { - return Err(error); - } - - warn!("We couldn't do the knock locally, maybe federation can help to satisfy the knock"); - - let (make_knock_response, remote_server) = - make_knock_request(services, sender_user, room_id, servers).await?; - - info!("make_knock finished"); - - let room_version_id = make_knock_response.room_version; - - if !services.server.supported_room_version(&room_version_id) { - return Err!(BadServerResponse( - "Remote room version {room_version_id} is not supported by conduwuit" - )); - } - - let mut knock_event_stub = serde_json::from_str::( - make_knock_response.event.get(), - ) - .map_err(|e| { - err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}")) - })?; - - knock_event_stub.insert( - "origin".to_owned(), - CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), - ); - knock_event_stub.insert( - "origin_server_ts".to_owned(), - CanonicalJsonValue::Integer( - utils::millis_since_unix_epoch() - .try_into() - .expect("Timestamp is valid js_int value"), - ), - ); - knock_event_stub.insert( - "content".to_owned(), - to_canonical_value(RoomMemberEventContent { - displayname: services.users.displayname(sender_user).await.ok(), - avatar_url: services.users.avatar_url(sender_user).await.ok(), - blurhash: services.users.blurhash(sender_user).await.ok(), - reason, - ..RoomMemberEventContent::new(MembershipState::Knock) - }) - .expect("event is valid, we just created it"), - ); - - // In order to create a compatible ref hash (EventID) the `hashes` field needs - // to be present - services - .server_keys - .hash_and_sign_event(&mut knock_event_stub, &room_version_id)?; - - // Generate event id - let event_id = gen_event_id(&knock_event_stub, &room_version_id)?; - - // Add event_id - knock_event_stub - .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); - - // It has enough fields to be called a proper event now - let knock_event = knock_event_stub; - - info!("Asking {remote_server} for send_knock in room {room_id}"); - let send_knock_request = federation::knock::send_knock::v1::Request { - room_id: room_id.to_owned(), - event_id: event_id.clone(), - pdu: services - .sending - .convert_to_outgoing_federation_event(knock_event.clone()) - .await, - }; - - let send_knock_response = services - .sending - .send_federation_request(&remote_server, send_knock_request) - .await?; - - info!("send_knock finished"); - - services - .rooms - .short - .get_or_create_shortroomid(room_id) - .await; - - info!("Parsing knock event"); - - let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone()) - .map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?; - - info!("Updating membership locally to knock state with provided stripped state events"); - services - .rooms - .state_cache - .update_membership( - room_id, - sender_user, - parsed_knock_pdu - .get_content::() - .expect("we just created this"), - sender_user, - Some(send_knock_response.knock_room_state), - None, - false, - ) - .await?; - - info!("Appending room knock event locally"); - services - .rooms - .timeline - .append_pdu( - &parsed_knock_pdu, - knock_event, - once(parsed_knock_pdu.event_id.borrow()), - &state_lock, - ) - .await?; - - Ok(()) -} - -async fn knock_room_helper_remote( - services: &Services, - sender_user: &UserId, - room_id: &RoomId, - reason: Option, - servers: &[OwnedServerName], - state_lock: RoomMutexGuard, -) -> Result { - info!("Knocking {room_id} over federation."); - - let (make_knock_response, remote_server) = - make_knock_request(services, sender_user, room_id, servers).await?; - - info!("make_knock finished"); - - let room_version_id = make_knock_response.room_version; - - if !services.server.supported_room_version(&room_version_id) { - return Err!(BadServerResponse( - "Remote room version {room_version_id} is not supported by conduwuit" - )); - } - - let mut knock_event_stub: CanonicalJsonObject = - serde_json::from_str(make_knock_response.event.get()).map_err(|e| { - err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}")) - })?; - - knock_event_stub.insert( - "origin".to_owned(), - CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), - ); - knock_event_stub.insert( - "origin_server_ts".to_owned(), - CanonicalJsonValue::Integer( - utils::millis_since_unix_epoch() - .try_into() - .expect("Timestamp is valid js_int value"), - ), - ); - knock_event_stub.insert( - "content".to_owned(), - to_canonical_value(RoomMemberEventContent { - displayname: services.users.displayname(sender_user).await.ok(), - avatar_url: services.users.avatar_url(sender_user).await.ok(), - blurhash: services.users.blurhash(sender_user).await.ok(), - reason, - ..RoomMemberEventContent::new(MembershipState::Knock) - }) - .expect("event is valid, we just created it"), - ); - - // In order to create a compatible ref hash (EventID) the `hashes` field needs - // to be present - services - .server_keys - .hash_and_sign_event(&mut knock_event_stub, &room_version_id)?; - - // Generate event id - let event_id = gen_event_id(&knock_event_stub, &room_version_id)?; - - // Add event_id - knock_event_stub - .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); - - // It has enough fields to be called a proper event now - let knock_event = knock_event_stub; - - info!("Asking {remote_server} for send_knock in room {room_id}"); - let send_knock_request = federation::knock::send_knock::v1::Request { - room_id: room_id.to_owned(), - event_id: event_id.clone(), - pdu: services - .sending - .convert_to_outgoing_federation_event(knock_event.clone()) - .await, - }; - - let send_knock_response = services - .sending - .send_federation_request(&remote_server, send_knock_request) - .await?; - - info!("send_knock finished"); - - services - .rooms - .short - .get_or_create_shortroomid(room_id) - .await; - - info!("Parsing knock event"); - let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone()) - .map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?; - - info!("Going through send_knock response knock state events"); - let state = send_knock_response - .knock_room_state - .iter() - .map(|event| serde_json::from_str::(event.clone().into_json().get())) - .filter_map(Result::ok); - - let mut state_map: HashMap = HashMap::new(); - - for event in state { - let Some(state_key) = event.get("state_key") else { - debug_warn!("send_knock stripped state event missing state_key: {event:?}"); - continue; - }; - let Some(event_type) = event.get("type") else { - debug_warn!("send_knock stripped state event missing event type: {event:?}"); - continue; - }; - - let Ok(state_key) = serde_json::from_value::(state_key.clone().into()) else { - debug_warn!("send_knock stripped state event has invalid state_key: {event:?}"); - continue; - }; - let Ok(event_type) = serde_json::from_value::(event_type.clone().into()) - else { - debug_warn!("send_knock stripped state event has invalid event type: {event:?}"); - continue; - }; - - let event_id = gen_event_id(&event, &room_version_id)?; - let shortstatekey = services - .rooms - .short - .get_or_create_shortstatekey(&event_type, &state_key) - .await; - - services.rooms.outlier.add_pdu_outlier(&event_id, &event); - state_map.insert(shortstatekey, event_id.clone()); - } - - info!("Compressing state from send_knock"); - let compressed: CompressedState = services - .rooms - .state_compressor - .compress_state_events(state_map.iter().map(|(ssk, eid)| (ssk, eid.borrow()))) - .collect() - .await; - - debug!("Saving compressed state"); - let HashSetCompressStateEvent { - shortstatehash: statehash_before_knock, - added, - removed, - } = services - .rooms - .state_compressor - .save_state(room_id, Arc::new(compressed)) - .await?; - - debug!("Forcing state for new room"); - services - .rooms - .state - .force_state(room_id, statehash_before_knock, added, removed, &state_lock) - .await?; - - let statehash_after_knock = services - .rooms - .state - .append_to_state(&parsed_knock_pdu) - .await?; - - info!("Updating membership locally to knock state with provided stripped state events"); - services - .rooms - .state_cache - .update_membership( - room_id, - sender_user, - parsed_knock_pdu - .get_content::() - .expect("we just created this"), - sender_user, - Some(send_knock_response.knock_room_state), - None, - false, - ) - .await?; - - info!("Appending room knock event locally"); - services - .rooms - .timeline - .append_pdu( - &parsed_knock_pdu, - knock_event, - once(parsed_knock_pdu.event_id.borrow()), - &state_lock, - ) - .await?; - - info!("Setting final room state for new room"); - // We set the room state after inserting the pdu, so that we never have a moment - // in time where events in the current room state do not exist - services - .rooms - .state - .set_room_state(room_id, statehash_after_knock, &state_lock); - - Ok(()) -} - -async fn make_knock_request( - services: &Services, - sender_user: &UserId, - room_id: &RoomId, - servers: &[OwnedServerName], -) -> Result<(federation::knock::create_knock_event_template::v1::Response, OwnedServerName)> { - let mut make_knock_response_and_server = - Err!(BadServerResponse("No server available to assist in knocking.")); - - let mut make_knock_counter: usize = 0; - - for remote_server in servers { - if services.globals.server_is_ours(remote_server) { - continue; - } - - info!("Asking {remote_server} for make_knock ({make_knock_counter})"); - - let make_knock_response = services - .sending - .send_federation_request( - remote_server, - federation::knock::create_knock_event_template::v1::Request { - room_id: room_id.to_owned(), - user_id: sender_user.to_owned(), - ver: services.server.supported_room_versions().collect(), - }, - ) - .await; - - trace!("make_knock response: {make_knock_response:?}"); - make_knock_counter = make_knock_counter.saturating_add(1); - - make_knock_response_and_server = make_knock_response.map(|r| (r, remote_server.clone())); - - if make_knock_response_and_server.is_ok() { - break; - } - - if make_knock_counter > 40 { - warn!( - "50 servers failed to provide valid make_knock response, assuming no server can \ - assist in knocking." - ); - make_knock_response_and_server = - Err!(BadServerResponse("No server available to assist in knocking.")); - - return make_knock_response_and_server; - } - } - - make_knock_response_and_server -} diff --git a/src/api/client/membership/ban.rs b/src/api/client/membership/ban.rs new file mode 100644 index 00000000..339dcf2e --- /dev/null +++ b/src/api/client/membership/ban.rs @@ -0,0 +1,60 @@ +use axum::extract::State; +use conduwuit::{Err, Result, matrix::pdu::PduBuilder}; +use ruma::{ + api::client::membership::ban_user, + events::room::member::{MembershipState, RoomMemberEventContent}, +}; + +use crate::Ruma; + +/// # `POST /_matrix/client/r0/rooms/{roomId}/ban` +/// +/// Tries to send a ban event into the room. +pub(crate) async fn ban_user_route( + State(services): State, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + + if sender_user == body.user_id { + return Err!(Request(Forbidden("You cannot ban yourself."))); + } + + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + + let current_member_content = services + .rooms + .state_accessor + .get_member(&body.room_id, &body.user_id) + .await + .unwrap_or_else(|_| RoomMemberEventContent::new(MembershipState::Ban)); + + services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { + membership: MembershipState::Ban, + reason: body.reason.clone(), + displayname: None, // display name may be offensive + avatar_url: None, // avatar may be offensive + is_direct: None, + join_authorized_via_users_server: None, + third_party_invite: None, + redact_events: body.redact_events, + ..current_member_content + }), + sender_user, + &body.room_id, + &state_lock, + ) + .await?; + + drop(state_lock); + + Ok(ban_user::v3::Response::new()) +} diff --git a/src/api/client/membership/forget.rs b/src/api/client/membership/forget.rs new file mode 100644 index 00000000..7f3a1a57 --- /dev/null +++ b/src/api/client/membership/forget.rs @@ -0,0 +1,52 @@ +use axum::extract::State; +use conduwuit::{Err, Result, is_matching, result::NotFound, utils::FutureBoolExt}; +use futures::pin_mut; +use ruma::{api::client::membership::forget_room, events::room::member::MembershipState}; + +use crate::Ruma; + +/// # `POST /_matrix/client/v3/rooms/{roomId}/forget` +/// +/// Forgets about a room. +/// +/// - If the sender user currently left the room: Stops sender user from +/// receiving information about the room +/// +/// Note: Other devices of the user have no way of knowing the room was +/// forgotten, so this has to be called from every device +pub(crate) async fn forget_room_route( + State(services): State, + body: Ruma, +) -> Result { + let user_id = body.sender_user(); + let room_id = &body.room_id; + + let joined = services.rooms.state_cache.is_joined(user_id, room_id); + let knocked = services.rooms.state_cache.is_knocked(user_id, room_id); + let invited = services.rooms.state_cache.is_invited(user_id, room_id); + + pin_mut!(joined, knocked, invited); + if joined.or(knocked).or(invited).await { + return Err!(Request(Unknown("You must leave the room before forgetting it"))); + } + + let membership = services + .rooms + .state_accessor + .get_member(room_id, user_id) + .await; + + if membership.is_not_found() { + return Err!(Request(Unknown("No membership event was found, room was never joined"))); + } + + let non_membership = membership + .map(|member| member.membership) + .is_ok_and(is_matching!(MembershipState::Leave | MembershipState::Ban)); + + if non_membership || services.rooms.state_cache.is_left(user_id, room_id).await { + services.rooms.state_cache.forget(room_id, user_id); + } + + Ok(forget_room::v3::Response::new()) +} diff --git a/src/api/client/membership/invite.rs b/src/api/client/membership/invite.rs new file mode 100644 index 00000000..4ca3efb8 --- /dev/null +++ b/src/api/client/membership/invite.rs @@ -0,0 +1,238 @@ +use axum::extract::State; +use axum_client_ip::InsecureClientIp; +use conduwuit::{ + Err, Result, debug_error, err, info, + matrix::pdu::{PduBuilder, gen_event_id_canonical_json}, +}; +use futures::{FutureExt, join}; +use ruma::{ + OwnedServerName, RoomId, UserId, + api::{client::membership::invite_user, federation::membership::create_invite}, + events::room::member::{MembershipState, RoomMemberEventContent}, +}; +use service::Services; + +use super::banned_room_check; +use crate::Ruma; + +/// # `POST /_matrix/client/r0/rooms/{roomId}/invite` +/// +/// Tries to send an invite event into the room. +#[tracing::instrument(skip_all, fields(%client), name = "invite")] +pub(crate) async fn invite_user_route( + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + + if !services.users.is_admin(sender_user).await && services.config.block_non_admin_invites { + debug_error!( + "User {sender_user} is not an admin and attempted to send an invite to room {}", + &body.room_id + ); + return Err!(Request(Forbidden("Invites are not allowed on this server."))); + } + + banned_room_check( + &services, + sender_user, + Some(&body.room_id), + body.room_id.server_name(), + client, + ) + .await?; + + match &body.recipient { + | invite_user::v3::InvitationRecipient::UserId { user_id } => { + let sender_ignored_recipient = services.users.user_is_ignored(sender_user, user_id); + let recipient_ignored_by_sender = + services.users.user_is_ignored(user_id, sender_user); + + let (sender_ignored_recipient, recipient_ignored_by_sender) = + join!(sender_ignored_recipient, recipient_ignored_by_sender); + + if sender_ignored_recipient { + return Ok(invite_user::v3::Response {}); + } + + if let Ok(target_user_membership) = services + .rooms + .state_accessor + .get_member(&body.room_id, user_id) + .await + { + if target_user_membership.membership == MembershipState::Ban { + return Err!(Request(Forbidden("User is banned from this room."))); + } + } + + if recipient_ignored_by_sender { + // silently drop the invite to the recipient if they've been ignored by the + // sender, pretend it worked + return Ok(invite_user::v3::Response {}); + } + + invite_helper( + &services, + sender_user, + user_id, + &body.room_id, + body.reason.clone(), + false, + ) + .boxed() + .await?; + + Ok(invite_user::v3::Response {}) + }, + | _ => { + Err!(Request(NotFound("User not found."))) + }, + } +} + +pub(crate) async fn invite_helper( + services: &Services, + sender_user: &UserId, + user_id: &UserId, + room_id: &RoomId, + reason: Option, + is_direct: bool, +) -> Result { + if !services.users.is_admin(sender_user).await && services.config.block_non_admin_invites { + info!( + "User {sender_user} is not an admin and attempted to send an invite to room \ + {room_id}" + ); + return Err!(Request(Forbidden("Invites are not allowed on this server."))); + } + + if !services.globals.user_is_local(user_id) { + let (pdu, pdu_json, invite_room_state) = { + let state_lock = services.rooms.state.mutex.lock(room_id).await; + + let content = RoomMemberEventContent { + avatar_url: services.users.avatar_url(user_id).await.ok(), + is_direct: Some(is_direct), + reason, + ..RoomMemberEventContent::new(MembershipState::Invite) + }; + + let (pdu, pdu_json) = services + .rooms + .timeline + .create_hash_and_sign_event( + PduBuilder::state(user_id.to_string(), &content), + sender_user, + room_id, + &state_lock, + ) + .await?; + + let invite_room_state = services.rooms.state.summary_stripped(&pdu).await; + + drop(state_lock); + + (pdu, pdu_json, invite_room_state) + }; + + let room_version_id = services.rooms.state.get_room_version(room_id).await?; + + let response = services + .sending + .send_federation_request(user_id.server_name(), create_invite::v2::Request { + room_id: room_id.to_owned(), + event_id: (*pdu.event_id).to_owned(), + room_version: room_version_id.clone(), + event: services + .sending + .convert_to_outgoing_federation_event(pdu_json.clone()) + .await, + invite_room_state, + via: services + .rooms + .state_cache + .servers_route_via(room_id) + .await + .ok(), + }) + .await?; + + // We do not add the event_id field to the pdu here because of signature and + // hashes checks + let (event_id, value) = gen_event_id_canonical_json(&response.event, &room_version_id) + .map_err(|e| { + err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))) + })?; + + if pdu.event_id != event_id { + return Err!(Request(BadJson(warn!( + %pdu.event_id, %event_id, + "Server {} sent event with wrong event ID", + user_id.server_name() + )))); + } + + let origin: OwnedServerName = serde_json::from_value(serde_json::to_value( + value + .get("origin") + .ok_or_else(|| err!(Request(BadJson("Event missing origin field."))))?, + )?) + .map_err(|e| { + err!(Request(BadJson(warn!("Origin field in event is not a valid server name: {e}")))) + })?; + + let pdu_id = services + .rooms + .event_handler + .handle_incoming_pdu(&origin, room_id, &event_id, value, true) + .boxed() + .await? + .ok_or_else(|| { + err!(Request(InvalidParam("Could not accept incoming PDU as timeline event."))) + })?; + + return services.sending.send_pdu_room(room_id, &pdu_id).await; + } + + if !services + .rooms + .state_cache + .is_joined(sender_user, room_id) + .await + { + return Err!(Request(Forbidden( + "You must be joined in the room you are trying to invite from." + ))); + } + + let state_lock = services.rooms.state.mutex.lock(room_id).await; + + let content = RoomMemberEventContent { + displayname: services.users.displayname(user_id).await.ok(), + avatar_url: services.users.avatar_url(user_id).await.ok(), + blurhash: services.users.blurhash(user_id).await.ok(), + is_direct: Some(is_direct), + reason, + ..RoomMemberEventContent::new(MembershipState::Invite) + }; + + services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state(user_id.to_string(), &content), + sender_user, + room_id, + &state_lock, + ) + .await?; + + drop(state_lock); + + Ok(()) +} diff --git a/src/api/client/membership/join.rs b/src/api/client/membership/join.rs new file mode 100644 index 00000000..669e9399 --- /dev/null +++ b/src/api/client/membership/join.rs @@ -0,0 +1,988 @@ +use std::{borrow::Borrow, collections::HashMap, iter::once, sync::Arc}; + +use axum::extract::State; +use axum_client_ip::InsecureClientIp; +use conduwuit::{ + Err, Result, debug, debug_info, debug_warn, err, error, info, + matrix::{ + StateKey, + pdu::{PduBuilder, PduEvent, gen_event_id, gen_event_id_canonical_json}, + state_res, + }, + result::FlatOk, + trace, + utils::{ + self, shuffle, + stream::{IterStream, ReadyExt}, + }, + warn, +}; +use futures::{FutureExt, StreamExt}; +use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, OwnedRoomId, OwnedServerName, OwnedUserId, RoomId, + RoomVersionId, UserId, + api::{ + client::{ + error::ErrorKind, + membership::{ThirdPartySigned, join_room_by_id, join_room_by_id_or_alias}, + }, + federation::{self}, + }, + canonical_json::to_canonical_value, + events::{ + StateEventType, + room::{ + join_rules::{AllowRule, JoinRule, RoomJoinRulesEventContent}, + member::{MembershipState, RoomMemberEventContent}, + }, + }, +}; +use service::{ + Services, + appservice::RegistrationInfo, + rooms::{ + state::RoomMutexGuard, + state_compressor::{CompressedState, HashSetCompressStateEvent}, + }, +}; + +use super::banned_room_check; +use crate::Ruma; + +/// # `POST /_matrix/client/r0/rooms/{roomId}/join` +/// +/// Tries to join the sender user into a room. +/// +/// - If the server knowns about this room: creates the join event and does auth +/// rules locally +/// - If the server does not know about the room: asks other servers over +/// federation +#[tracing::instrument(skip_all, fields(%client), name = "join")] +pub(crate) async fn join_room_by_id_route( + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + + banned_room_check( + &services, + sender_user, + Some(&body.room_id), + body.room_id.server_name(), + client, + ) + .await?; + + // There is no body.server_name for /roomId/join + let mut servers: Vec<_> = services + .rooms + .state_cache + .servers_invite_via(&body.room_id) + .map(ToOwned::to_owned) + .collect() + .await; + + servers.extend( + services + .rooms + .state_cache + .invite_state(sender_user, &body.room_id) + .await + .unwrap_or_default() + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), + ); + + if let Some(server) = body.room_id.server_name() { + servers.push(server.into()); + } + + servers.sort_unstable(); + servers.dedup(); + shuffle(&mut servers); + + join_room_by_id_helper( + &services, + sender_user, + &body.room_id, + body.reason.clone(), + &servers, + body.third_party_signed.as_ref(), + &body.appservice_info, + ) + .boxed() + .await +} + +/// # `POST /_matrix/client/r0/join/{roomIdOrAlias}` +/// +/// Tries to join the sender user into a room. +/// +/// - If the server knowns about this room: creates the join event and does auth +/// rules locally +/// - If the server does not know about the room: use the server name query +/// param if specified. if not specified, asks other servers over federation +/// via room alias server name and room ID server name +#[tracing::instrument(skip_all, fields(%client), name = "join")] +pub(crate) async fn join_room_by_id_or_alias_route( + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + let appservice_info = &body.appservice_info; + let body = &body.body; + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + + let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias.clone()) { + | Ok(room_id) => { + banned_room_check( + &services, + sender_user, + Some(&room_id), + room_id.server_name(), + client, + ) + .boxed() + .await?; + + let mut servers = body.via.clone(); + servers.extend( + services + .rooms + .state_cache + .servers_invite_via(&room_id) + .map(ToOwned::to_owned) + .collect::>() + .await, + ); + + servers.extend( + services + .rooms + .state_cache + .invite_state(sender_user, &room_id) + .await + .unwrap_or_default() + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), + ); + + if let Some(server) = room_id.server_name() { + servers.push(server.to_owned()); + } + + servers.sort_unstable(); + servers.dedup(); + shuffle(&mut servers); + + (servers, room_id) + }, + | Err(room_alias) => { + let (room_id, mut servers) = services + .rooms + .alias + .resolve_alias(&room_alias, Some(body.via.clone())) + .await?; + + banned_room_check( + &services, + sender_user, + Some(&room_id), + Some(room_alias.server_name()), + client, + ) + .await?; + + let addl_via_servers = services + .rooms + .state_cache + .servers_invite_via(&room_id) + .map(ToOwned::to_owned); + + let addl_state_servers = services + .rooms + .state_cache + .invite_state(sender_user, &room_id) + .await + .unwrap_or_default(); + + let mut addl_servers: Vec<_> = addl_state_servers + .iter() + .map(|event| event.get_field("sender")) + .filter_map(FlatOk::flat_ok) + .map(|user: &UserId| user.server_name().to_owned()) + .stream() + .chain(addl_via_servers) + .collect() + .await; + + addl_servers.sort_unstable(); + addl_servers.dedup(); + shuffle(&mut addl_servers); + servers.append(&mut addl_servers); + + (servers, room_id) + }, + }; + + let join_room_response = join_room_by_id_helper( + &services, + sender_user, + &room_id, + body.reason.clone(), + &servers, + body.third_party_signed.as_ref(), + appservice_info, + ) + .boxed() + .await?; + + Ok(join_room_by_id_or_alias::v3::Response { room_id: join_room_response.room_id }) +} + +pub async fn join_room_by_id_helper( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + third_party_signed: Option<&ThirdPartySigned>, + appservice_info: &Option, +) -> Result { + let state_lock = services.rooms.state.mutex.lock(room_id).await; + + let user_is_guest = services + .users + .is_deactivated(sender_user) + .await + .unwrap_or(false) + && appservice_info.is_none(); + + if user_is_guest && !services.rooms.state_accessor.guest_can_join(room_id).await { + return Err!(Request(Forbidden("Guests are not allowed to join this room"))); + } + + if services + .rooms + .state_cache + .is_joined(sender_user, room_id) + .await + { + debug_warn!("{sender_user} is already joined in {room_id}"); + return Ok(join_room_by_id::v3::Response { room_id: room_id.into() }); + } + + let server_in_room = services + .rooms + .state_cache + .server_in_room(services.globals.server_name(), room_id) + .await; + + // Only check our known membership if we're already in the room. + // See: https://forgejo.ellis.link/continuwuation/continuwuity/issues/855 + let membership = if server_in_room { + services + .rooms + .state_accessor + .get_member(room_id, sender_user) + .await + } else { + debug!("Ignoring local state for join {room_id}, we aren't in the room yet."); + Ok(RoomMemberEventContent::new(MembershipState::Leave)) + }; + if let Ok(m) = membership { + if m.membership == MembershipState::Ban { + debug_warn!("{sender_user} is banned from {room_id} but attempted to join"); + // TODO: return reason + return Err!(Request(Forbidden("You are banned from the room."))); + } + } + + let local_join = server_in_room + || servers.is_empty() + || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); + + if local_join { + join_room_by_id_helper_local( + services, + sender_user, + room_id, + reason, + servers, + third_party_signed, + state_lock, + ) + .boxed() + .await?; + } else { + // Ask a remote server if we are not participating in this room + join_room_by_id_helper_remote( + services, + sender_user, + room_id, + reason, + servers, + third_party_signed, + state_lock, + ) + .boxed() + .await?; + } + + Ok(join_room_by_id::v3::Response::new(room_id.to_owned())) +} + +#[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_remote")] +async fn join_room_by_id_helper_remote( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + _third_party_signed: Option<&ThirdPartySigned>, + state_lock: RoomMutexGuard, +) -> Result { + info!("Joining {room_id} over federation."); + + let (make_join_response, remote_server) = + make_join_request(services, sender_user, room_id, servers).await?; + + info!("make_join finished"); + + let Some(room_version_id) = make_join_response.room_version else { + return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); + }; + + if !services.server.supported_room_version(&room_version_id) { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } + + let mut join_event_stub: CanonicalJsonObject = + serde_json::from_str(make_join_response.event.get()).map_err(|e| { + err!(BadServerResponse(warn!( + "Invalid make_join event json received from server: {e:?}" + ))) + })?; + + let join_authorized_via_users_server = { + use RoomVersionId::*; + if !matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) { + join_event_stub + .get("content") + .map(|s| { + s.as_object()? + .get("join_authorised_via_users_server")? + .as_str() + }) + .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()) + } else { + None + } + }; + + join_event_stub.insert( + "origin".to_owned(), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), + ); + join_event_stub.insert( + "origin_server_ts".to_owned(), + CanonicalJsonValue::Integer( + utils::millis_since_unix_epoch() + .try_into() + .expect("Timestamp is valid js_int value"), + ), + ); + join_event_stub.insert( + "content".to_owned(), + to_canonical_value(RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason, + join_authorized_via_users_server: join_authorized_via_users_server.clone(), + ..RoomMemberEventContent::new(MembershipState::Join) + }) + .expect("event is valid, we just created it"), + ); + + // We keep the "event_id" in the pdu only in v1 or + // v2 rooms + match room_version_id { + | RoomVersionId::V1 | RoomVersionId::V2 => {}, + | _ => { + join_event_stub.remove("event_id"); + }, + } + + // In order to create a compatible ref hash (EventID) the `hashes` field needs + // to be present + services + .server_keys + .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; + + // Generate event id + let event_id = gen_event_id(&join_event_stub, &room_version_id)?; + + // Add event_id back + join_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + + // It has enough fields to be called a proper event now + let mut join_event = join_event_stub; + + info!("Asking {remote_server} for send_join in room {room_id}"); + let send_join_request = federation::membership::create_join_event::v2::Request { + room_id: room_id.to_owned(), + event_id: event_id.clone(), + omit_members: false, + pdu: services + .sending + .convert_to_outgoing_federation_event(join_event.clone()) + .await, + }; + + let send_join_response = match services + .sending + .send_synapse_request(&remote_server, send_join_request) + .await + { + | Ok(response) => response, + | Err(e) => { + error!("send_join failed: {e}"); + return Err(e); + }, + }; + + info!("send_join finished"); + + if join_authorized_via_users_server.is_some() { + if let Some(signed_raw) = &send_join_response.room_state.event { + debug_info!( + "There is a signed event with join_authorized_via_users_server. This room is \ + probably using restricted joins. Adding signature to our event" + ); + + let (signed_event_id, signed_value) = + gen_event_id_canonical_json(signed_raw, &room_version_id).map_err(|e| { + err!(Request(BadJson(warn!( + "Could not convert event to canonical JSON: {e}" + )))) + })?; + + if signed_event_id != event_id { + return Err!(Request(BadJson(warn!( + %signed_event_id, %event_id, + "Server {remote_server} sent event with wrong event ID" + )))); + } + + match signed_value["signatures"] + .as_object() + .ok_or_else(|| { + err!(BadServerResponse(warn!( + "Server {remote_server} sent invalid signatures type" + ))) + }) + .and_then(|e| { + e.get(remote_server.as_str()).ok_or_else(|| { + err!(BadServerResponse(warn!( + "Server {remote_server} did not send its signature for a restricted \ + room" + ))) + }) + }) { + | Ok(signature) => { + join_event + .get_mut("signatures") + .expect("we created a valid pdu") + .as_object_mut() + .expect("we created a valid pdu") + .insert(remote_server.to_string(), signature.clone()); + }, + | Err(e) => { + warn!( + "Server {remote_server} sent invalid signature in send_join signatures \ + for event {signed_value:?}: {e:?}", + ); + }, + } + } + } + + services + .rooms + .short + .get_or_create_shortroomid(room_id) + .await; + + info!("Parsing join event"); + let parsed_join_pdu = PduEvent::from_id_val(&event_id, join_event.clone()) + .map_err(|e| err!(BadServerResponse("Invalid join event PDU: {e:?}")))?; + + info!("Acquiring server signing keys for response events"); + let resp_events = &send_join_response.room_state; + let resp_state = &resp_events.state; + let resp_auth = &resp_events.auth_chain; + services + .server_keys + .acquire_events_pubkeys(resp_auth.iter().chain(resp_state.iter())) + .await; + + info!("Going through send_join response room_state"); + let cork = services.db.cork_and_flush(); + let state = send_join_response + .room_state + .state + .iter() + .stream() + .then(|pdu| { + services + .server_keys + .validate_and_add_event_id_no_fetch(pdu, &room_version_id) + }) + .ready_filter_map(Result::ok) + .fold(HashMap::new(), |mut state, (event_id, value)| async move { + let pdu = match PduEvent::from_id_val(&event_id, value.clone()) { + | Ok(pdu) => pdu, + | Err(e) => { + debug_warn!("Invalid PDU in send_join response: {e:?}: {value:#?}"); + return state; + }, + }; + + services.rooms.outlier.add_pdu_outlier(&event_id, &value); + if let Some(state_key) = &pdu.state_key { + let shortstatekey = services + .rooms + .short + .get_or_create_shortstatekey(&pdu.kind.to_string().into(), state_key) + .await; + + state.insert(shortstatekey, pdu.event_id.clone()); + } + + state + }) + .await; + + drop(cork); + + info!("Going through send_join response auth_chain"); + let cork = services.db.cork_and_flush(); + send_join_response + .room_state + .auth_chain + .iter() + .stream() + .then(|pdu| { + services + .server_keys + .validate_and_add_event_id_no_fetch(pdu, &room_version_id) + }) + .ready_filter_map(Result::ok) + .ready_for_each(|(event_id, value)| { + services.rooms.outlier.add_pdu_outlier(&event_id, &value); + }) + .await; + + drop(cork); + + debug!("Running send_join auth check"); + let fetch_state = &state; + let state_fetch = |k: StateEventType, s: StateKey| async move { + let shortstatekey = services.rooms.short.get_shortstatekey(&k, &s).await.ok()?; + + let event_id = fetch_state.get(&shortstatekey)?; + services.rooms.timeline.get_pdu(event_id).await.ok() + }; + + let auth_check = state_res::event_auth::auth_check( + &state_res::RoomVersion::new(&room_version_id)?, + &parsed_join_pdu, + None, // TODO: third party invite + |k, s| state_fetch(k.clone(), s.into()), + ) + .await + .map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?; + + if !auth_check { + return Err!(Request(Forbidden("Auth check failed"))); + } + + info!("Compressing state from send_join"); + let compressed: CompressedState = services + .rooms + .state_compressor + .compress_state_events(state.iter().map(|(ssk, eid)| (ssk, eid.borrow()))) + .collect() + .await; + + debug!("Saving compressed state"); + let HashSetCompressStateEvent { + shortstatehash: statehash_before_join, + added, + removed, + } = services + .rooms + .state_compressor + .save_state(room_id, Arc::new(compressed)) + .await?; + + debug!("Forcing state for new room"); + services + .rooms + .state + .force_state(room_id, statehash_before_join, added, removed, &state_lock) + .await?; + + info!("Updating joined counts for new room"); + services + .rooms + .state_cache + .update_joined_count(room_id) + .await; + + // We append to state before appending the pdu, so we don't have a moment in + // time with the pdu without it's state. This is okay because append_pdu can't + // fail. + let statehash_after_join = services + .rooms + .state + .append_to_state(&parsed_join_pdu) + .await?; + + info!("Appending new room join event"); + services + .rooms + .timeline + .append_pdu( + &parsed_join_pdu, + join_event, + once(parsed_join_pdu.event_id.borrow()), + &state_lock, + ) + .await?; + + info!("Setting final room state for new room"); + // We set the room state after inserting the pdu, so that we never have a moment + // in time where events in the current room state do not exist + services + .rooms + .state + .set_room_state(room_id, statehash_after_join, &state_lock); + + Ok(()) +} + +#[tracing::instrument(skip_all, fields(%sender_user, %room_id), name = "join_local")] +async fn join_room_by_id_helper_local( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + _third_party_signed: Option<&ThirdPartySigned>, + state_lock: RoomMutexGuard, +) -> Result { + debug_info!("We can join locally"); + + let join_rules_event_content = services + .rooms + .state_accessor + .room_state_get_content::( + room_id, + &StateEventType::RoomJoinRules, + "", + ) + .await; + + let restriction_rooms = match join_rules_event_content { + | Ok(RoomJoinRulesEventContent { + join_rule: JoinRule::Restricted(restricted) | JoinRule::KnockRestricted(restricted), + }) => restricted + .allow + .into_iter() + .filter_map(|a| match a { + | AllowRule::RoomMembership(r) => Some(r.room_id), + | _ => None, + }) + .collect(), + | _ => Vec::new(), + }; + + let join_authorized_via_users_server: Option = { + if restriction_rooms + .iter() + .stream() + .any(|restriction_room_id| { + services + .rooms + .state_cache + .is_joined(sender_user, restriction_room_id) + }) + .await + { + services + .rooms + .state_cache + .local_users_in_room(room_id) + .filter(|user| { + services.rooms.state_accessor.user_can_invite( + room_id, + user, + sender_user, + &state_lock, + ) + }) + .boxed() + .next() + .await + .map(ToOwned::to_owned) + } else { + None + } + }; + + let content = RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason: reason.clone(), + join_authorized_via_users_server, + ..RoomMemberEventContent::new(MembershipState::Join) + }; + + // Try normal join first + let Err(error) = services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state(sender_user.to_string(), &content), + sender_user, + room_id, + &state_lock, + ) + .await + else { + return Ok(()); + }; + + if restriction_rooms.is_empty() + && (servers.is_empty() + || servers.len() == 1 && services.globals.server_is_ours(&servers[0])) + { + return Err(error); + } + + warn!( + "We couldn't do the join locally, maybe federation can help to satisfy the restricted \ + join requirements" + ); + let Ok((make_join_response, remote_server)) = + make_join_request(services, sender_user, room_id, servers).await + else { + return Err(error); + }; + + let Some(room_version_id) = make_join_response.room_version else { + return Err!(BadServerResponse("Remote room version is not supported by conduwuit")); + }; + + if !services.server.supported_room_version(&room_version_id) { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } + + let mut join_event_stub: CanonicalJsonObject = + serde_json::from_str(make_join_response.event.get()).map_err(|e| { + err!(BadServerResponse("Invalid make_join event json received from server: {e:?}")) + })?; + + let join_authorized_via_users_server = join_event_stub + .get("content") + .map(|s| { + s.as_object()? + .get("join_authorised_via_users_server")? + .as_str() + }) + .and_then(|s| OwnedUserId::try_from(s.unwrap_or_default()).ok()); + + join_event_stub.insert( + "origin".to_owned(), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), + ); + join_event_stub.insert( + "origin_server_ts".to_owned(), + CanonicalJsonValue::Integer( + utils::millis_since_unix_epoch() + .try_into() + .expect("Timestamp is valid js_int value"), + ), + ); + join_event_stub.insert( + "content".to_owned(), + to_canonical_value(RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason, + join_authorized_via_users_server, + ..RoomMemberEventContent::new(MembershipState::Join) + }) + .expect("event is valid, we just created it"), + ); + + // We keep the "event_id" in the pdu only in v1 or + // v2 rooms + match room_version_id { + | RoomVersionId::V1 | RoomVersionId::V2 => {}, + | _ => { + join_event_stub.remove("event_id"); + }, + } + + // In order to create a compatible ref hash (EventID) the `hashes` field needs + // to be present + services + .server_keys + .hash_and_sign_event(&mut join_event_stub, &room_version_id)?; + + // Generate event id + let event_id = gen_event_id(&join_event_stub, &room_version_id)?; + + // Add event_id back + join_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + + // It has enough fields to be called a proper event now + let join_event = join_event_stub; + + let send_join_response = services + .sending + .send_synapse_request( + &remote_server, + federation::membership::create_join_event::v2::Request { + room_id: room_id.to_owned(), + event_id: event_id.clone(), + omit_members: false, + pdu: services + .sending + .convert_to_outgoing_federation_event(join_event.clone()) + .await, + }, + ) + .await?; + + if let Some(signed_raw) = send_join_response.room_state.event { + let (signed_event_id, signed_value) = + gen_event_id_canonical_json(&signed_raw, &room_version_id).map_err(|e| { + err!(Request(BadJson(warn!("Could not convert event to canonical JSON: {e}")))) + })?; + + if signed_event_id != event_id { + return Err!(Request(BadJson( + warn!(%signed_event_id, %event_id, "Server {remote_server} sent event with wrong event ID") + ))); + } + + drop(state_lock); + services + .rooms + .event_handler + .handle_incoming_pdu(&remote_server, room_id, &signed_event_id, signed_value, true) + .boxed() + .await?; + } else { + return Err(error); + } + + Ok(()) +} + +async fn make_join_request( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + servers: &[OwnedServerName], +) -> Result<(federation::membership::prepare_join_event::v1::Response, OwnedServerName)> { + let mut make_join_response_and_server = + Err!(BadServerResponse("No server available to assist in joining.")); + + let mut make_join_counter: usize = 0; + let mut incompatible_room_version_count: usize = 0; + + for remote_server in servers { + if services.globals.server_is_ours(remote_server) { + continue; + } + info!("Asking {remote_server} for make_join ({make_join_counter})"); + let make_join_response = services + .sending + .send_federation_request( + remote_server, + federation::membership::prepare_join_event::v1::Request { + room_id: room_id.to_owned(), + user_id: sender_user.to_owned(), + ver: services.server.supported_room_versions().collect(), + }, + ) + .await; + + trace!("make_join response: {:?}", make_join_response); + make_join_counter = make_join_counter.saturating_add(1); + + if let Err(ref e) = make_join_response { + if matches!( + e.kind(), + ErrorKind::IncompatibleRoomVersion { .. } | ErrorKind::UnsupportedRoomVersion + ) { + incompatible_room_version_count = + incompatible_room_version_count.saturating_add(1); + } + + if incompatible_room_version_count > 15 { + info!( + "15 servers have responded with M_INCOMPATIBLE_ROOM_VERSION or \ + M_UNSUPPORTED_ROOM_VERSION, assuming that conduwuit does not support the \ + room version {room_id}: {e}" + ); + make_join_response_and_server = + Err!(BadServerResponse("Room version is not supported by Conduwuit")); + return make_join_response_and_server; + } + + if make_join_counter > 40 { + warn!( + "40 servers failed to provide valid make_join response, assuming no server \ + can assist in joining." + ); + make_join_response_and_server = + Err!(BadServerResponse("No server available to assist in joining.")); + + return make_join_response_and_server; + } + } + + make_join_response_and_server = make_join_response.map(|r| (r, remote_server.clone())); + + if make_join_response_and_server.is_ok() { + break; + } + } + + make_join_response_and_server +} diff --git a/src/api/client/membership/kick.rs b/src/api/client/membership/kick.rs new file mode 100644 index 00000000..5e0e86e2 --- /dev/null +++ b/src/api/client/membership/kick.rs @@ -0,0 +1,65 @@ +use axum::extract::State; +use conduwuit::{Err, Result, matrix::pdu::PduBuilder}; +use ruma::{ + api::client::membership::kick_user, + events::room::member::{MembershipState, RoomMemberEventContent}, +}; + +use crate::Ruma; + +/// # `POST /_matrix/client/r0/rooms/{roomId}/kick` +/// +/// Tries to send a kick event into the room. +pub(crate) async fn kick_user_route( + State(services): State, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + + let Ok(event) = services + .rooms + .state_accessor + .get_member(&body.room_id, &body.user_id) + .await + else { + // copy synapse's behaviour of returning 200 without any change to the state + // instead of erroring on left users + return Ok(kick_user::v3::Response::new()); + }; + + if !matches!( + event.membership, + MembershipState::Invite | MembershipState::Knock | MembershipState::Join, + ) { + return Err!(Request(Forbidden( + "Cannot kick a user who is not apart of the room (current membership: {})", + event.membership + ))); + } + + services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { + membership: MembershipState::Leave, + reason: body.reason.clone(), + is_direct: None, + join_authorized_via_users_server: None, + third_party_invite: None, + ..event + }), + sender_user, + &body.room_id, + &state_lock, + ) + .await?; + + drop(state_lock); + + Ok(kick_user::v3::Response::new()) +} diff --git a/src/api/client/membership/knock.rs b/src/api/client/membership/knock.rs new file mode 100644 index 00000000..544dcfb3 --- /dev/null +++ b/src/api/client/membership/knock.rs @@ -0,0 +1,767 @@ +use std::{borrow::Borrow, collections::HashMap, iter::once, sync::Arc}; + +use axum::extract::State; +use axum_client_ip::InsecureClientIp; +use conduwuit::{ + Err, Result, debug, debug_info, debug_warn, err, info, + matrix::pdu::{PduBuilder, PduEvent, gen_event_id}, + result::FlatOk, + trace, + utils::{self, shuffle, stream::IterStream}, + warn, +}; +use futures::{FutureExt, StreamExt}; +use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId, OwnedServerName, RoomId, + RoomVersionId, UserId, + api::{ + client::knock::knock_room, + federation::{self}, + }, + canonical_json::to_canonical_value, + events::{ + StateEventType, + room::{ + join_rules::{AllowRule, JoinRule}, + member::{MembershipState, RoomMemberEventContent}, + }, + }, +}; +use service::{ + Services, + rooms::{ + state::RoomMutexGuard, + state_compressor::{CompressedState, HashSetCompressStateEvent}, + }, +}; + +use super::{banned_room_check, join::join_room_by_id_helper}; +use crate::Ruma; + +/// # `POST /_matrix/client/*/knock/{roomIdOrAlias}` +/// +/// Tries to knock the room to ask permission to join for the sender user. +#[tracing::instrument(skip_all, fields(%client), name = "knock")] +pub(crate) async fn knock_room_route( + State(services): State, + InsecureClientIp(client): InsecureClientIp, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + let body = &body.body; + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + + let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias.clone()) { + | Ok(room_id) => { + banned_room_check( + &services, + sender_user, + Some(&room_id), + room_id.server_name(), + client, + ) + .await?; + + let mut servers = body.via.clone(); + servers.extend( + services + .rooms + .state_cache + .servers_invite_via(&room_id) + .map(ToOwned::to_owned) + .collect::>() + .await, + ); + + servers.extend( + services + .rooms + .state_cache + .invite_state(sender_user, &room_id) + .await + .unwrap_or_default() + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), + ); + + if let Some(server) = room_id.server_name() { + servers.push(server.to_owned()); + } + + servers.sort_unstable(); + servers.dedup(); + shuffle(&mut servers); + + (servers, room_id) + }, + | Err(room_alias) => { + let (room_id, mut servers) = services + .rooms + .alias + .resolve_alias(&room_alias, Some(body.via.clone())) + .await?; + + banned_room_check( + &services, + sender_user, + Some(&room_id), + Some(room_alias.server_name()), + client, + ) + .await?; + + let addl_via_servers = services + .rooms + .state_cache + .servers_invite_via(&room_id) + .map(ToOwned::to_owned); + + let addl_state_servers = services + .rooms + .state_cache + .invite_state(sender_user, &room_id) + .await + .unwrap_or_default(); + + let mut addl_servers: Vec<_> = addl_state_servers + .iter() + .map(|event| event.get_field("sender")) + .filter_map(FlatOk::flat_ok) + .map(|user: &UserId| user.server_name().to_owned()) + .stream() + .chain(addl_via_servers) + .collect() + .await; + + addl_servers.sort_unstable(); + addl_servers.dedup(); + shuffle(&mut addl_servers); + servers.append(&mut addl_servers); + + (servers, room_id) + }, + }; + + knock_room_by_id_helper(&services, sender_user, &room_id, body.reason.clone(), &servers) + .boxed() + .await +} + +async fn knock_room_by_id_helper( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], +) -> Result { + let state_lock = services.rooms.state.mutex.lock(room_id).await; + + if services + .rooms + .state_cache + .is_invited(sender_user, room_id) + .await + { + debug_warn!("{sender_user} is already invited in {room_id} but attempted to knock"); + return Err!(Request(Forbidden( + "You cannot knock on a room you are already invited/accepted to." + ))); + } + + if services + .rooms + .state_cache + .is_joined(sender_user, room_id) + .await + { + debug_warn!("{sender_user} is already joined in {room_id} but attempted to knock"); + return Err!(Request(Forbidden("You cannot knock on a room you are already joined in."))); + } + + if services + .rooms + .state_cache + .is_knocked(sender_user, room_id) + .await + { + debug_warn!("{sender_user} is already knocked in {room_id}"); + return Ok(knock_room::v3::Response { room_id: room_id.into() }); + } + + if let Ok(membership) = services + .rooms + .state_accessor + .get_member(room_id, sender_user) + .await + { + if membership.membership == MembershipState::Ban { + debug_warn!("{sender_user} is banned from {room_id} but attempted to knock"); + return Err!(Request(Forbidden("You cannot knock on a room you are banned from."))); + } + } + + // For knock_restricted rooms, check if the user meets the restricted conditions + // If they do, attempt to join instead of knock + // This is not mentioned in the spec, but should be allowable (we're allowed to + // auto-join invites to knocked rooms) + let join_rule = services.rooms.state_accessor.get_join_rules(room_id).await; + + if let JoinRule::KnockRestricted(restricted) = &join_rule { + let restriction_rooms: Vec<_> = restricted + .allow + .iter() + .filter_map(|a| match a { + | AllowRule::RoomMembership(r) => Some(&r.room_id), + | _ => None, + }) + .collect(); + + // Check if the user is in any of the allowed rooms + let mut user_meets_restrictions = false; + for restriction_room_id in &restriction_rooms { + if services + .rooms + .state_cache + .is_joined(sender_user, restriction_room_id) + .await + { + user_meets_restrictions = true; + break; + } + } + + // If the user meets the restrictions, try joining instead + if user_meets_restrictions { + debug_info!( + "{sender_user} meets the restricted criteria in knock_restricted room \ + {room_id}, attempting to join instead of knock" + ); + // For this case, we need to drop the state lock and get a new one in + // join_room_by_id_helper We need to release the lock here and let + // join_room_by_id_helper acquire it again + drop(state_lock); + match join_room_by_id_helper( + services, + sender_user, + room_id, + reason.clone(), + servers, + None, + &None, + ) + .await + { + | Ok(_) => return Ok(knock_room::v3::Response::new(room_id.to_owned())), + | Err(e) => { + debug_warn!( + "Failed to convert knock to join for {sender_user} in {room_id}: {e:?}" + ); + // Get a new state lock for the remaining knock logic + let new_state_lock = services.rooms.state.mutex.lock(room_id).await; + + let server_in_room = services + .rooms + .state_cache + .server_in_room(services.globals.server_name(), room_id) + .await; + + let local_knock = server_in_room + || servers.is_empty() + || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); + + if local_knock { + knock_room_helper_local( + services, + sender_user, + room_id, + reason, + servers, + new_state_lock, + ) + .boxed() + .await?; + } else { + knock_room_helper_remote( + services, + sender_user, + room_id, + reason, + servers, + new_state_lock, + ) + .boxed() + .await?; + } + + return Ok(knock_room::v3::Response::new(room_id.to_owned())); + }, + } + } + } else if !matches!(join_rule, JoinRule::Knock | JoinRule::KnockRestricted(_)) { + debug_warn!( + "{sender_user} attempted to knock on room {room_id} but its join rule is \ + {join_rule:?}, not knock or knock_restricted" + ); + } + + let server_in_room = services + .rooms + .state_cache + .server_in_room(services.globals.server_name(), room_id) + .await; + + let local_knock = server_in_room + || servers.is_empty() + || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])); + + if local_knock { + knock_room_helper_local(services, sender_user, room_id, reason, servers, state_lock) + .boxed() + .await?; + } else { + knock_room_helper_remote(services, sender_user, room_id, reason, servers, state_lock) + .boxed() + .await?; + } + + Ok(knock_room::v3::Response::new(room_id.to_owned())) +} + +async fn knock_room_helper_local( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + state_lock: RoomMutexGuard, +) -> Result { + debug_info!("We can knock locally"); + + let room_version_id = services.rooms.state.get_room_version(room_id).await?; + + if matches!( + room_version_id, + RoomVersionId::V1 + | RoomVersionId::V2 + | RoomVersionId::V3 + | RoomVersionId::V4 + | RoomVersionId::V5 + | RoomVersionId::V6 + ) { + return Err!(Request(Forbidden("This room does not support knocking."))); + } + + let content = RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason: reason.clone(), + ..RoomMemberEventContent::new(MembershipState::Knock) + }; + + // Try normal knock first + let Err(error) = services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state(sender_user.to_string(), &content), + sender_user, + room_id, + &state_lock, + ) + .await + else { + return Ok(()); + }; + + if servers.is_empty() || (servers.len() == 1 && services.globals.server_is_ours(&servers[0])) + { + return Err(error); + } + + warn!("We couldn't do the knock locally, maybe federation can help to satisfy the knock"); + + let (make_knock_response, remote_server) = + make_knock_request(services, sender_user, room_id, servers).await?; + + info!("make_knock finished"); + + let room_version_id = make_knock_response.room_version; + + if !services.server.supported_room_version(&room_version_id) { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } + + let mut knock_event_stub = serde_json::from_str::( + make_knock_response.event.get(), + ) + .map_err(|e| { + err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}")) + })?; + + knock_event_stub.insert( + "origin".to_owned(), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), + ); + knock_event_stub.insert( + "origin_server_ts".to_owned(), + CanonicalJsonValue::Integer( + utils::millis_since_unix_epoch() + .try_into() + .expect("Timestamp is valid js_int value"), + ), + ); + knock_event_stub.insert( + "content".to_owned(), + to_canonical_value(RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason, + ..RoomMemberEventContent::new(MembershipState::Knock) + }) + .expect("event is valid, we just created it"), + ); + + // In order to create a compatible ref hash (EventID) the `hashes` field needs + // to be present + services + .server_keys + .hash_and_sign_event(&mut knock_event_stub, &room_version_id)?; + + // Generate event id + let event_id = gen_event_id(&knock_event_stub, &room_version_id)?; + + // Add event_id + knock_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + + // It has enough fields to be called a proper event now + let knock_event = knock_event_stub; + + info!("Asking {remote_server} for send_knock in room {room_id}"); + let send_knock_request = federation::knock::send_knock::v1::Request { + room_id: room_id.to_owned(), + event_id: event_id.clone(), + pdu: services + .sending + .convert_to_outgoing_federation_event(knock_event.clone()) + .await, + }; + + let send_knock_response = services + .sending + .send_federation_request(&remote_server, send_knock_request) + .await?; + + info!("send_knock finished"); + + services + .rooms + .short + .get_or_create_shortroomid(room_id) + .await; + + info!("Parsing knock event"); + + let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone()) + .map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?; + + info!("Updating membership locally to knock state with provided stripped state events"); + services + .rooms + .state_cache + .update_membership( + room_id, + sender_user, + parsed_knock_pdu + .get_content::() + .expect("we just created this"), + sender_user, + Some(send_knock_response.knock_room_state), + None, + false, + ) + .await?; + + info!("Appending room knock event locally"); + services + .rooms + .timeline + .append_pdu( + &parsed_knock_pdu, + knock_event, + once(parsed_knock_pdu.event_id.borrow()), + &state_lock, + ) + .await?; + + Ok(()) +} + +async fn knock_room_helper_remote( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + reason: Option, + servers: &[OwnedServerName], + state_lock: RoomMutexGuard, +) -> Result { + info!("Knocking {room_id} over federation."); + + let (make_knock_response, remote_server) = + make_knock_request(services, sender_user, room_id, servers).await?; + + info!("make_knock finished"); + + let room_version_id = make_knock_response.room_version; + + if !services.server.supported_room_version(&room_version_id) { + return Err!(BadServerResponse( + "Remote room version {room_version_id} is not supported by conduwuit" + )); + } + + let mut knock_event_stub: CanonicalJsonObject = + serde_json::from_str(make_knock_response.event.get()).map_err(|e| { + err!(BadServerResponse("Invalid make_knock event json received from server: {e:?}")) + })?; + + knock_event_stub.insert( + "origin".to_owned(), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), + ); + knock_event_stub.insert( + "origin_server_ts".to_owned(), + CanonicalJsonValue::Integer( + utils::millis_since_unix_epoch() + .try_into() + .expect("Timestamp is valid js_int value"), + ), + ); + knock_event_stub.insert( + "content".to_owned(), + to_canonical_value(RoomMemberEventContent { + displayname: services.users.displayname(sender_user).await.ok(), + avatar_url: services.users.avatar_url(sender_user).await.ok(), + blurhash: services.users.blurhash(sender_user).await.ok(), + reason, + ..RoomMemberEventContent::new(MembershipState::Knock) + }) + .expect("event is valid, we just created it"), + ); + + // In order to create a compatible ref hash (EventID) the `hashes` field needs + // to be present + services + .server_keys + .hash_and_sign_event(&mut knock_event_stub, &room_version_id)?; + + // Generate event id + let event_id = gen_event_id(&knock_event_stub, &room_version_id)?; + + // Add event_id + knock_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + + // It has enough fields to be called a proper event now + let knock_event = knock_event_stub; + + info!("Asking {remote_server} for send_knock in room {room_id}"); + let send_knock_request = federation::knock::send_knock::v1::Request { + room_id: room_id.to_owned(), + event_id: event_id.clone(), + pdu: services + .sending + .convert_to_outgoing_federation_event(knock_event.clone()) + .await, + }; + + let send_knock_response = services + .sending + .send_federation_request(&remote_server, send_knock_request) + .await?; + + info!("send_knock finished"); + + services + .rooms + .short + .get_or_create_shortroomid(room_id) + .await; + + info!("Parsing knock event"); + let parsed_knock_pdu = PduEvent::from_id_val(&event_id, knock_event.clone()) + .map_err(|e| err!(BadServerResponse("Invalid knock event PDU: {e:?}")))?; + + info!("Going through send_knock response knock state events"); + let state = send_knock_response + .knock_room_state + .iter() + .map(|event| serde_json::from_str::(event.clone().into_json().get())) + .filter_map(Result::ok); + + let mut state_map: HashMap = HashMap::new(); + + for event in state { + let Some(state_key) = event.get("state_key") else { + debug_warn!("send_knock stripped state event missing state_key: {event:?}"); + continue; + }; + let Some(event_type) = event.get("type") else { + debug_warn!("send_knock stripped state event missing event type: {event:?}"); + continue; + }; + + let Ok(state_key) = serde_json::from_value::(state_key.clone().into()) else { + debug_warn!("send_knock stripped state event has invalid state_key: {event:?}"); + continue; + }; + let Ok(event_type) = serde_json::from_value::(event_type.clone().into()) + else { + debug_warn!("send_knock stripped state event has invalid event type: {event:?}"); + continue; + }; + + let event_id = gen_event_id(&event, &room_version_id)?; + let shortstatekey = services + .rooms + .short + .get_or_create_shortstatekey(&event_type, &state_key) + .await; + + services.rooms.outlier.add_pdu_outlier(&event_id, &event); + state_map.insert(shortstatekey, event_id.clone()); + } + + info!("Compressing state from send_knock"); + let compressed: CompressedState = services + .rooms + .state_compressor + .compress_state_events(state_map.iter().map(|(ssk, eid)| (ssk, eid.borrow()))) + .collect() + .await; + + debug!("Saving compressed state"); + let HashSetCompressStateEvent { + shortstatehash: statehash_before_knock, + added, + removed, + } = services + .rooms + .state_compressor + .save_state(room_id, Arc::new(compressed)) + .await?; + + debug!("Forcing state for new room"); + services + .rooms + .state + .force_state(room_id, statehash_before_knock, added, removed, &state_lock) + .await?; + + let statehash_after_knock = services + .rooms + .state + .append_to_state(&parsed_knock_pdu) + .await?; + + info!("Updating membership locally to knock state with provided stripped state events"); + services + .rooms + .state_cache + .update_membership( + room_id, + sender_user, + parsed_knock_pdu + .get_content::() + .expect("we just created this"), + sender_user, + Some(send_knock_response.knock_room_state), + None, + false, + ) + .await?; + + info!("Appending room knock event locally"); + services + .rooms + .timeline + .append_pdu( + &parsed_knock_pdu, + knock_event, + once(parsed_knock_pdu.event_id.borrow()), + &state_lock, + ) + .await?; + + info!("Setting final room state for new room"); + // We set the room state after inserting the pdu, so that we never have a moment + // in time where events in the current room state do not exist + services + .rooms + .state + .set_room_state(room_id, statehash_after_knock, &state_lock); + + Ok(()) +} + +async fn make_knock_request( + services: &Services, + sender_user: &UserId, + room_id: &RoomId, + servers: &[OwnedServerName], +) -> Result<(federation::knock::create_knock_event_template::v1::Response, OwnedServerName)> { + let mut make_knock_response_and_server = + Err!(BadServerResponse("No server available to assist in knocking.")); + + let mut make_knock_counter: usize = 0; + + for remote_server in servers { + if services.globals.server_is_ours(remote_server) { + continue; + } + + info!("Asking {remote_server} for make_knock ({make_knock_counter})"); + + let make_knock_response = services + .sending + .send_federation_request( + remote_server, + federation::knock::create_knock_event_template::v1::Request { + room_id: room_id.to_owned(), + user_id: sender_user.to_owned(), + ver: services.server.supported_room_versions().collect(), + }, + ) + .await; + + trace!("make_knock response: {make_knock_response:?}"); + make_knock_counter = make_knock_counter.saturating_add(1); + + make_knock_response_and_server = make_knock_response.map(|r| (r, remote_server.clone())); + + if make_knock_response_and_server.is_ok() { + break; + } + + if make_knock_counter > 40 { + warn!( + "50 servers failed to provide valid make_knock response, assuming no server can \ + assist in knocking." + ); + make_knock_response_and_server = + Err!(BadServerResponse("No server available to assist in knocking.")); + + return make_knock_response_and_server; + } + } + + make_knock_response_and_server +} diff --git a/src/api/client/membership/leave.rs b/src/api/client/membership/leave.rs new file mode 100644 index 00000000..a64fb41f --- /dev/null +++ b/src/api/client/membership/leave.rs @@ -0,0 +1,386 @@ +use std::collections::HashSet; + +use axum::extract::State; +use conduwuit::{ + Err, Result, debug_info, debug_warn, err, + matrix::pdu::{PduBuilder, gen_event_id}, + utils::{self, FutureBoolExt, future::ReadyEqExt}, + warn, +}; +use futures::{FutureExt, StreamExt, TryFutureExt, pin_mut}; +use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, OwnedServerName, RoomId, RoomVersionId, UserId, + api::{ + client::membership::leave_room, + federation::{self}, + }, + events::{ + StateEventType, + room::member::{MembershipState, RoomMemberEventContent}, + }, +}; +use service::Services; + +use crate::Ruma; + +/// # `POST /_matrix/client/v3/rooms/{roomId}/leave` +/// +/// Tries to leave the sender user from a room. +/// +/// - This should always work if the user is currently joined. +pub(crate) async fn leave_room_route( + State(services): State, + body: Ruma, +) -> Result { + leave_room(&services, body.sender_user(), &body.room_id, body.reason.clone()) + .boxed() + .await + .map(|()| leave_room::v3::Response::new()) +} + +// Make a user leave all their joined rooms, rescinds knocks, forgets all rooms, +// and ignores errors +pub async fn leave_all_rooms(services: &Services, user_id: &UserId) { + let rooms_joined = services + .rooms + .state_cache + .rooms_joined(user_id) + .map(ToOwned::to_owned); + + let rooms_invited = services + .rooms + .state_cache + .rooms_invited(user_id) + .map(|(r, _)| r); + + let rooms_knocked = services + .rooms + .state_cache + .rooms_knocked(user_id) + .map(|(r, _)| r); + + let all_rooms: Vec<_> = rooms_joined + .chain(rooms_invited) + .chain(rooms_knocked) + .collect() + .await; + + for room_id in all_rooms { + // ignore errors + if let Err(e) = leave_room(services, user_id, &room_id, None).boxed().await { + warn!(%user_id, "Failed to leave {room_id} remotely: {e}"); + } + + services.rooms.state_cache.forget(&room_id, user_id); + } +} + +pub async fn leave_room( + services: &Services, + user_id: &UserId, + room_id: &RoomId, + reason: Option, +) -> Result { + let default_member_content = RoomMemberEventContent { + membership: MembershipState::Leave, + reason: reason.clone(), + join_authorized_via_users_server: None, + is_direct: None, + avatar_url: None, + displayname: None, + third_party_invite: None, + blurhash: None, + redact_events: None, + }; + + let is_banned = services.rooms.metadata.is_banned(room_id); + let is_disabled = services.rooms.metadata.is_disabled(room_id); + + pin_mut!(is_banned, is_disabled); + if is_banned.or(is_disabled).await { + // the room is banned/disabled, the room must be rejected locally since we + // cant/dont want to federate with this server + services + .rooms + .state_cache + .update_membership( + room_id, + user_id, + default_member_content, + user_id, + None, + None, + true, + ) + .await?; + + return Ok(()); + } + + let dont_have_room = services + .rooms + .state_cache + .server_in_room(services.globals.server_name(), room_id) + .eq(&false); + + let not_knocked = services + .rooms + .state_cache + .is_knocked(user_id, room_id) + .eq(&false); + + // Ask a remote server if we don't have this room and are not knocking on it + if dont_have_room.and(not_knocked).await { + if let Err(e) = remote_leave_room(services, user_id, room_id, reason.clone()) + .boxed() + .await + { + warn!(%user_id, "Failed to leave room {room_id} remotely: {e}"); + // Don't tell the client about this error + } + + let last_state = services + .rooms + .state_cache + .invite_state(user_id, room_id) + .or_else(|_| services.rooms.state_cache.knock_state(user_id, room_id)) + .or_else(|_| services.rooms.state_cache.left_state(user_id, room_id)) + .await + .ok(); + + // We always drop the invite, we can't rely on other servers + services + .rooms + .state_cache + .update_membership( + room_id, + user_id, + default_member_content, + user_id, + last_state, + None, + true, + ) + .await?; + } else { + let state_lock = services.rooms.state.mutex.lock(room_id).await; + + let Ok(event) = services + .rooms + .state_accessor + .room_state_get_content::( + room_id, + &StateEventType::RoomMember, + user_id.as_str(), + ) + .await + else { + debug_warn!( + "Trying to leave a room you are not a member of, marking room as left locally." + ); + + return services + .rooms + .state_cache + .update_membership( + room_id, + user_id, + default_member_content, + user_id, + None, + None, + true, + ) + .await; + }; + + services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state(user_id.to_string(), &RoomMemberEventContent { + membership: MembershipState::Leave, + reason, + join_authorized_via_users_server: None, + is_direct: None, + ..event + }), + user_id, + room_id, + &state_lock, + ) + .await?; + } + + Ok(()) +} + +async fn remote_leave_room( + services: &Services, + user_id: &UserId, + room_id: &RoomId, + reason: Option, +) -> Result<()> { + let mut make_leave_response_and_server = + Err!(BadServerResponse("No remote server available to assist in leaving {room_id}.")); + + let mut servers: HashSet = services + .rooms + .state_cache + .servers_invite_via(room_id) + .map(ToOwned::to_owned) + .collect() + .await; + + match services + .rooms + .state_cache + .invite_state(user_id, room_id) + .await + { + | Ok(invite_state) => { + servers.extend( + invite_state + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), + ); + }, + | _ => { + match services + .rooms + .state_cache + .knock_state(user_id, room_id) + .await + { + | Ok(knock_state) => { + servers.extend( + knock_state + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .filter_map(|sender| { + if !services.globals.user_is_local(sender) { + Some(sender.server_name().to_owned()) + } else { + None + } + }), + ); + }, + | _ => {}, + } + }, + } + + if let Some(room_id_server_name) = room_id.server_name() { + servers.insert(room_id_server_name.to_owned()); + } + + debug_info!("servers in remote_leave_room: {servers:?}"); + + for remote_server in servers { + let make_leave_response = services + .sending + .send_federation_request( + &remote_server, + federation::membership::prepare_leave_event::v1::Request { + room_id: room_id.to_owned(), + user_id: user_id.to_owned(), + }, + ) + .await; + + make_leave_response_and_server = make_leave_response.map(|r| (r, remote_server)); + + if make_leave_response_and_server.is_ok() { + break; + } + } + + let (make_leave_response, remote_server) = make_leave_response_and_server?; + + let Some(room_version_id) = make_leave_response.room_version else { + return Err!(BadServerResponse(warn!( + "No room version was returned by {remote_server} for {room_id}, room version is \ + likely not supported by conduwuit" + ))); + }; + + if !services.server.supported_room_version(&room_version_id) { + return Err!(BadServerResponse(warn!( + "Remote room version {room_version_id} for {room_id} is not supported by conduwuit", + ))); + } + + let mut leave_event_stub = serde_json::from_str::( + make_leave_response.event.get(), + ) + .map_err(|e| { + err!(BadServerResponse(warn!( + "Invalid make_leave event json received from {remote_server} for {room_id}: {e:?}" + ))) + })?; + + // TODO: Is origin needed? + leave_event_stub.insert( + "origin".to_owned(), + CanonicalJsonValue::String(services.globals.server_name().as_str().to_owned()), + ); + leave_event_stub.insert( + "origin_server_ts".to_owned(), + CanonicalJsonValue::Integer( + utils::millis_since_unix_epoch() + .try_into() + .expect("Timestamp is valid js_int value"), + ), + ); + // Inject the reason key into the event content dict if it exists + if let Some(reason) = reason { + if let Some(CanonicalJsonValue::Object(content)) = leave_event_stub.get_mut("content") { + content.insert("reason".to_owned(), CanonicalJsonValue::String(reason)); + } + } + + // room v3 and above removed the "event_id" field from remote PDU format + match room_version_id { + | RoomVersionId::V1 | RoomVersionId::V2 => {}, + | _ => { + leave_event_stub.remove("event_id"); + }, + } + + // In order to create a compatible ref hash (EventID) the `hashes` field needs + // to be present + services + .server_keys + .hash_and_sign_event(&mut leave_event_stub, &room_version_id)?; + + // Generate event id + let event_id = gen_event_id(&leave_event_stub, &room_version_id)?; + + // Add event_id back + leave_event_stub + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.clone().into())); + + // It has enough fields to be called a proper event now + let leave_event = leave_event_stub; + + services + .sending + .send_federation_request( + &remote_server, + federation::membership::create_leave_event::v2::Request { + room_id: room_id.to_owned(), + event_id, + pdu: services + .sending + .convert_to_outgoing_federation_event(leave_event.clone()) + .await, + }, + ) + .await?; + + Ok(()) +} diff --git a/src/api/client/membership/members.rs b/src/api/client/membership/members.rs new file mode 100644 index 00000000..4a7abf6d --- /dev/null +++ b/src/api/client/membership/members.rs @@ -0,0 +1,147 @@ +use axum::extract::State; +use conduwuit::{ + Err, Event, Result, at, + matrix::pdu::PduEvent, + utils::{ + future::TryExtExt, + stream::{BroadbandExt, ReadyExt}, + }, +}; +use futures::{StreamExt, future::join}; +use ruma::{ + api::client::membership::{ + get_member_events::{self, v3::MembershipEventFilter}, + joined_members::{self, v3::RoomMember}, + }, + events::{ + StateEventType, + room::member::{MembershipState, RoomMemberEventContent}, + }, +}; + +use crate::Ruma; + +/// # `POST /_matrix/client/r0/rooms/{roomId}/members` +/// +/// Lists all joined users in a room (TODO: at a specific point in time, with a +/// specific membership). +/// +/// - Only works if the user is currently joined +pub(crate) async fn get_member_events_route( + State(services): State, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + let membership = body.membership.as_ref(); + let not_membership = body.not_membership.as_ref(); + + if !services + .rooms + .state_accessor + .user_can_see_state_events(sender_user, &body.room_id) + .await + { + return Err!(Request(Forbidden("You don't have permission to view this room."))); + } + + Ok(get_member_events::v3::Response { + chunk: services + .rooms + .state_accessor + .room_state_full(&body.room_id) + .ready_filter_map(Result::ok) + .ready_filter(|((ty, _), _)| *ty == StateEventType::RoomMember) + .map(at!(1)) + .ready_filter_map(|pdu| membership_filter(pdu, membership, not_membership)) + .map(Event::into_format) + .collect() + .await, + }) +} + +/// # `POST /_matrix/client/r0/rooms/{roomId}/joined_members` +/// +/// Lists all members of a room. +/// +/// - The sender user must be in the room +/// - TODO: An appservice just needs a puppet joined +pub(crate) async fn joined_members_route( + State(services): State, + body: Ruma, +) -> Result { + if !services + .rooms + .state_accessor + .user_can_see_state_events(body.sender_user(), &body.room_id) + .await + { + return Err!(Request(Forbidden("You don't have permission to view this room."))); + } + + Ok(joined_members::v3::Response { + joined: services + .rooms + .state_cache + .room_members(&body.room_id) + .map(ToOwned::to_owned) + .broad_then(|user_id| async move { + let (display_name, avatar_url) = join( + services.users.displayname(&user_id).ok(), + services.users.avatar_url(&user_id).ok(), + ) + .await; + + (user_id, RoomMember { display_name, avatar_url }) + }) + .collect() + .await, + }) +} + +fn membership_filter( + pdu: PduEvent, + for_membership: Option<&MembershipEventFilter>, + not_membership: Option<&MembershipEventFilter>, +) -> Option { + let membership_state_filter = match for_membership { + | Some(MembershipEventFilter::Ban) => MembershipState::Ban, + | Some(MembershipEventFilter::Invite) => MembershipState::Invite, + | Some(MembershipEventFilter::Knock) => MembershipState::Knock, + | Some(MembershipEventFilter::Leave) => MembershipState::Leave, + | Some(_) | None => MembershipState::Join, + }; + + let not_membership_state_filter = match not_membership { + | Some(MembershipEventFilter::Ban) => MembershipState::Ban, + | Some(MembershipEventFilter::Invite) => MembershipState::Invite, + | Some(MembershipEventFilter::Join) => MembershipState::Join, + | Some(MembershipEventFilter::Knock) => MembershipState::Knock, + | Some(_) | None => MembershipState::Leave, + }; + + let evt_membership = pdu.get_content::().ok()?.membership; + + if for_membership.is_some() && not_membership.is_some() { + if membership_state_filter != evt_membership + || not_membership_state_filter == evt_membership + { + None + } else { + Some(pdu) + } + } else if for_membership.is_some() && not_membership.is_none() { + if membership_state_filter != evt_membership { + None + } else { + Some(pdu) + } + } else if not_membership.is_some() && for_membership.is_none() { + if not_membership_state_filter == evt_membership { + None + } else { + Some(pdu) + } + } else { + Some(pdu) + } +} diff --git a/src/api/client/membership/mod.rs b/src/api/client/membership/mod.rs new file mode 100644 index 00000000..7a6f19ad --- /dev/null +++ b/src/api/client/membership/mod.rs @@ -0,0 +1,156 @@ +mod ban; +mod forget; +mod invite; +mod join; +mod kick; +mod knock; +mod leave; +mod members; +mod unban; + +use std::net::IpAddr; + +use axum::extract::State; +use conduwuit::{Err, Result, warn}; +use futures::{FutureExt, StreamExt}; +use ruma::{OwnedRoomId, RoomId, ServerName, UserId, api::client::membership::joined_rooms}; +use service::Services; + +pub(crate) use self::{ + ban::ban_user_route, + forget::forget_room_route, + invite::{invite_helper, invite_user_route}, + join::{join_room_by_id_or_alias_route, join_room_by_id_route}, + kick::kick_user_route, + knock::knock_room_route, + leave::leave_room_route, + members::{get_member_events_route, joined_members_route}, + unban::unban_user_route, +}; +pub use self::{ + join::join_room_by_id_helper, + leave::{leave_all_rooms, leave_room}, +}; +use crate::{Ruma, client::full_user_deactivate}; + +/// # `POST /_matrix/client/r0/joined_rooms` +/// +/// Lists all rooms the user has joined. +pub(crate) async fn joined_rooms_route( + State(services): State, + body: Ruma, +) -> Result { + Ok(joined_rooms::v3::Response { + joined_rooms: services + .rooms + .state_cache + .rooms_joined(body.sender_user()) + .map(ToOwned::to_owned) + .collect() + .await, + }) +} + +/// Checks if the room is banned in any way possible and the sender user is not +/// an admin. +/// +/// Performs automatic deactivation if `auto_deactivate_banned_room_attempts` is +/// enabled +#[tracing::instrument(skip(services))] +pub(crate) async fn banned_room_check( + services: &Services, + user_id: &UserId, + room_id: Option<&RoomId>, + server_name: Option<&ServerName>, + client_ip: IpAddr, +) -> Result { + if services.users.is_admin(user_id).await { + return Ok(()); + } + + if let Some(room_id) = room_id { + if services.rooms.metadata.is_banned(room_id).await + || services + .moderation + .is_remote_server_forbidden(room_id.server_name().expect("legacy room mxid")) + { + warn!( + "User {user_id} who is not an admin attempted to send an invite for or \ + attempted to join a banned room or banned room server name: {room_id}" + ); + + if services.server.config.auto_deactivate_banned_room_attempts { + warn!( + "Automatically deactivating user {user_id} due to attempted banned room join" + ); + + if services.server.config.admin_room_notices { + services + .admin + .send_text(&format!( + "Automatically deactivating user {user_id} due to attempted banned \ + room join from IP {client_ip}" + )) + .await; + } + + let all_joined_rooms: Vec = services + .rooms + .state_cache + .rooms_joined(user_id) + .map(Into::into) + .collect() + .await; + + full_user_deactivate(services, user_id, &all_joined_rooms) + .boxed() + .await?; + } + + return Err!(Request(Forbidden("This room is banned on this homeserver."))); + } + } else if let Some(server_name) = server_name { + if services + .config + .forbidden_remote_server_names + .is_match(server_name.host()) + { + warn!( + "User {user_id} who is not an admin tried joining a room which has the server \ + name {server_name} that is globally forbidden. Rejecting.", + ); + + if services.server.config.auto_deactivate_banned_room_attempts { + warn!( + "Automatically deactivating user {user_id} due to attempted banned room join" + ); + + if services.server.config.admin_room_notices { + services + .admin + .send_text(&format!( + "Automatically deactivating user {user_id} due to attempted banned \ + room join from IP {client_ip}" + )) + .await; + } + + let all_joined_rooms: Vec = services + .rooms + .state_cache + .rooms_joined(user_id) + .map(Into::into) + .collect() + .await; + + full_user_deactivate(services, user_id, &all_joined_rooms) + .boxed() + .await?; + } + + return Err!(Request(Forbidden("This remote server is banned on this homeserver."))); + } + } + + Ok(()) +} diff --git a/src/api/client/membership/unban.rs b/src/api/client/membership/unban.rs new file mode 100644 index 00000000..34c5eace --- /dev/null +++ b/src/api/client/membership/unban.rs @@ -0,0 +1,58 @@ +use axum::extract::State; +use conduwuit::{Err, Result, matrix::pdu::PduBuilder}; +use ruma::{ + api::client::membership::unban_user, + events::room::member::{MembershipState, RoomMemberEventContent}, +}; + +use crate::Ruma; + +/// # `POST /_matrix/client/r0/rooms/{roomId}/unban` +/// +/// Tries to send an unban event into the room. +pub(crate) async fn unban_user_route( + State(services): State, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + if services.users.is_suspended(sender_user).await? { + return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); + } + let state_lock = services.rooms.state.mutex.lock(&body.room_id).await; + + let current_member_content = services + .rooms + .state_accessor + .get_member(&body.room_id, &body.user_id) + .await + .unwrap_or_else(|_| RoomMemberEventContent::new(MembershipState::Leave)); + + if current_member_content.membership != MembershipState::Ban { + return Err!(Request(Forbidden( + "Cannot unban a user who is not banned (current membership: {})", + current_member_content.membership + ))); + } + + services + .rooms + .timeline + .build_and_append_pdu( + PduBuilder::state(body.user_id.to_string(), &RoomMemberEventContent { + membership: MembershipState::Leave, + reason: body.reason.clone(), + join_authorized_via_users_server: None, + third_party_invite: None, + is_direct: None, + ..current_member_content + }), + sender_user, + &body.room_id, + &state_lock, + ) + .await?; + + drop(state_lock); + + Ok(unban_user::v3::Response::new()) +} diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 6efad64e..ff8c2a0b 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -4,11 +4,14 @@ use axum::extract::State; use conduwuit::{ Err, Result, matrix::pdu::PduBuilder, - utils::{IterStream, stream::TryIgnore}, + utils::{IterStream, future::TryExtExt, stream::TryIgnore}, warn, }; use conduwuit_service::Services; -use futures::{StreamExt, TryStreamExt, future::join3}; +use futures::{ + StreamExt, TryStreamExt, + future::{join, join3, join4}, +}; use ruma::{ OwnedMxcUri, OwnedRoomId, UserId, api::{ @@ -214,10 +217,13 @@ pub(crate) async fn get_avatar_url_route( return Err!(Request(NotFound("Profile was not found."))); } - Ok(get_avatar_url::v3::Response { - avatar_url: services.users.avatar_url(&body.user_id).await.ok(), - blurhash: services.users.blurhash(&body.user_id).await.ok(), - }) + let (avatar_url, blurhash) = join( + services.users.avatar_url(&body.user_id).ok(), + services.users.blurhash(&body.user_id).ok(), + ) + .await; + + Ok(get_avatar_url::v3::Response { avatar_url, blurhash }) } /// # `GET /_matrix/client/v3/profile/{userId}` @@ -297,11 +303,19 @@ pub(crate) async fn get_profile_route( custom_profile_fields.remove("us.cloke.msc4175.tz"); custom_profile_fields.remove("m.tz"); + let (avatar_url, blurhash, displayname, tz) = join4( + services.users.avatar_url(&body.user_id).ok(), + services.users.blurhash(&body.user_id).ok(), + services.users.displayname(&body.user_id).ok(), + services.users.timezone(&body.user_id).ok(), + ) + .await; + Ok(get_profile::v3::Response { - avatar_url: services.users.avatar_url(&body.user_id).await.ok(), - blurhash: services.users.blurhash(&body.user_id).await.ok(), - displayname: services.users.displayname(&body.user_id).await.ok(), - tz: services.users.timezone(&body.user_id).await.ok(), + avatar_url, + blurhash, + displayname, + tz, custom_profile_fields, }) } @@ -313,16 +327,12 @@ pub async fn update_displayname( all_joined_rooms: &[OwnedRoomId], ) { let (current_avatar_url, current_blurhash, current_displayname) = join3( - services.users.avatar_url(user_id), - services.users.blurhash(user_id), - services.users.displayname(user_id), + services.users.avatar_url(user_id).ok(), + services.users.blurhash(user_id).ok(), + services.users.displayname(user_id).ok(), ) .await; - let current_avatar_url = current_avatar_url.ok(); - let current_blurhash = current_blurhash.ok(); - let current_displayname = current_displayname.ok(); - if displayname == current_displayname { return; } @@ -366,16 +376,12 @@ pub async fn update_avatar_url( all_joined_rooms: &[OwnedRoomId], ) { let (current_avatar_url, current_blurhash, current_displayname) = join3( - services.users.avatar_url(user_id), - services.users.blurhash(user_id), - services.users.displayname(user_id), + services.users.avatar_url(user_id).ok(), + services.users.blurhash(user_id).ok(), + services.users.displayname(user_id).ok(), ) .await; - let current_avatar_url = current_avatar_url.ok(); - let current_blurhash = current_blurhash.ok(); - let current_displayname = current_displayname.ok(); - if current_avatar_url == avatar_url && current_blurhash == blurhash { return; } diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index 8b9f3ca0..c2f59d4c 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -1,9 +1,9 @@ use axum::extract::State; use conduwuit::{ Err, Event, Result, at, - utils::{BoolExt, stream::TryTools}, + utils::{BoolExt, future::TryExtExt, stream::TryTools}, }; -use futures::TryStreamExt; +use futures::{FutureExt, TryStreamExt, future::try_join4}; use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response}; use crate::Ruma; @@ -25,22 +25,31 @@ pub(crate) async fn room_initial_sync_route( return Err!(Request(Forbidden("No room preview available."))); } - let limit = LIMIT_MAX; - let events: Vec<_> = services + let membership = services .rooms - .timeline - .pdus_rev(None, room_id, None) - .try_take(limit) - .try_collect() - .await?; + .state_cache + .user_membership(body.sender_user(), room_id) + .map(Ok); - let state: Vec<_> = services + let visibility = services.rooms.directory.visibility(room_id).map(Ok); + + let state = services .rooms .state_accessor .room_state_full_pdus(room_id) .map_ok(Event::into_format) - .try_collect() - .await?; + .try_collect::>(); + + let limit = LIMIT_MAX; + let events = services + .rooms + .timeline + .pdus_rev(None, room_id, None) + .try_take(limit) + .try_collect::>(); + + let (membership, visibility, state, events) = + try_join4(membership, visibility, state, events).await?; let messages = PaginationChunk { start: events.last().map(at!(0)).as_ref().map(ToString::to_string), @@ -64,11 +73,7 @@ pub(crate) async fn room_initial_sync_route( account_data: None, state: state.into(), messages: messages.chunk.is_empty().or_some(messages), - visibility: services.rooms.directory.visibility(room_id).await.into(), - membership: services - .rooms - .state_cache - .user_membership(body.sender_user(), room_id) - .await, + visibility: visibility.into(), + membership, }) } From 364293608de928c3acd8e7253522ca31713c8435 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Sun, 27 Apr 2025 02:39:28 +0000 Subject: [PATCH 1889/2091] Post-formatting aesthetic and spacing corrections Signed-off-by: Jason Volk --- src/admin/debug/commands.rs | 4 +- src/admin/user/commands.rs | 17 ++-- src/api/client/account.rs | 32 +++++-- src/api/client/directory.rs | 6 +- src/api/client/membership/invite.rs | 2 +- src/api/client/membership/join.rs | 3 +- src/api/client/membership/knock.rs | 5 +- src/api/client/membership/leave.rs | 2 +- src/api/client/membership/members.rs | 10 +- src/api/client/message.rs | 33 ++++--- src/api/client/profile.rs | 5 - src/api/client/relations.rs | 18 ++-- src/api/client/report.rs | 1 + src/api/client/room/initial_sync.rs | 4 +- src/api/client/room/summary.rs | 4 +- src/api/client/room/upgrade.rs | 4 +- src/api/client/sync/v4.rs | 4 +- src/api/server/invite.rs | 6 +- src/api/server/send_join.rs | 2 +- src/api/server/send_knock.rs | 2 +- src/api/server/send_leave.rs | 2 +- src/core/config/proxy.rs | 5 +- src/core/info/cargo.rs | 6 +- src/core/matrix/event.rs | 90 ++++++++++++++++-- src/core/matrix/event/filter.rs | 93 +++++++++++++++++++ .../matrix/{pdu/event_id.rs => event/id.rs} | 0 src/core/matrix/event/relation.rs | 28 ++++++ src/core/matrix/event/unsigned.rs | 51 ++++++++++ src/core/matrix/pdu.rs | 28 ++++-- src/core/matrix/pdu/content.rs | 20 ---- src/core/matrix/pdu/filter.rs | 90 ------------------ src/core/matrix/pdu/redact.rs | 4 +- src/core/matrix/pdu/relation.rs | 22 ----- src/core/matrix/pdu/unsigned.rs | 43 +-------- src/core/matrix/state_res/mod.rs | 10 +- src/core/mods/module.rs | 1 + src/core/mods/path.rs | 1 + src/core/utils/html.rs | 2 + src/core/utils/json.rs | 28 +++--- src/core/utils/time.rs | 3 - src/database/watchers.rs | 1 - src/main/logging.rs | 13 +++ src/main/mods.rs | 4 + src/main/sentry.rs | 8 +- src/router/request.rs | 2 +- src/service/admin/mod.rs | 6 +- src/service/pusher/mod.rs | 6 +- src/service/rooms/alias/mod.rs | 4 +- .../fetch_and_handle_outliers.rs | 35 ++++--- src/service/rooms/event_handler/fetch_prev.rs | 49 ++++++---- .../rooms/event_handler/fetch_state.rs | 25 ++--- .../event_handler/handle_incoming_pdu.rs | 15 +-- .../rooms/event_handler/handle_outlier_pdu.rs | 35 ++++--- .../rooms/event_handler/handle_prev_pdu.rs | 18 ++-- src/service/rooms/event_handler/mod.rs | 12 +-- .../rooms/event_handler/parse_incoming_pdu.rs | 4 +- .../rooms/event_handler/state_at_incoming.rs | 48 ++++++---- .../event_handler/upgrade_outlier_pdu.rs | 50 +++++----- src/service/rooms/outlier/mod.rs | 4 +- src/service/rooms/pdu_metadata/data.rs | 10 +- src/service/rooms/pdu_metadata/mod.rs | 19 ++-- src/service/rooms/read_receipt/mod.rs | 10 +- src/service/rooms/search/mod.rs | 12 ++- src/service/rooms/state/mod.rs | 4 +- .../rooms/state_accessor/room_state.rs | 8 +- src/service/rooms/state_accessor/state.rs | 18 ++-- src/service/rooms/state_accessor/user_can.rs | 14 +-- src/service/rooms/threads/mod.rs | 13 ++- src/service/rooms/timeline/data.rs | 1 - src/service/rooms/timeline/mod.rs | 89 +++++++++--------- src/service/sending/sender.rs | 2 +- src/service/server_keys/verify.rs | 2 +- 72 files changed, 704 insertions(+), 528 deletions(-) create mode 100644 src/core/matrix/event/filter.rs rename src/core/matrix/{pdu/event_id.rs => event/id.rs} (100%) create mode 100644 src/core/matrix/event/relation.rs create mode 100644 src/core/matrix/event/unsigned.rs delete mode 100644 src/core/matrix/pdu/content.rs delete mode 100644 src/core/matrix/pdu/filter.rs delete mode 100644 src/core/matrix/pdu/relation.rs diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 74355311..81b0e9da 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -558,8 +558,8 @@ pub(super) async fn force_set_room_state_from_server( .latest_pdu_in_room(&room_id) .await .map_err(|_| err!(Database("Failed to find the latest PDU in database")))? - .event_id - .clone(), + .event_id() + .to_owned(), }; let room_version = self.services.rooms.state.get_room_version(&room_id).await?; diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index e15c0b2c..86206c2b 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -738,7 +738,7 @@ pub(super) async fn force_demote(&self, user_id: String, room_id: OwnedRoomOrAli .state_accessor .room_state_get(&room_id, &StateEventType::RoomCreate, "") .await - .is_ok_and(|event| event.sender == user_id); + .is_ok_and(|event| event.sender() == user_id); if !user_can_demote_self { return Err!("User is not allowed to modify their own power levels in the room.",); @@ -889,10 +889,7 @@ pub(super) async fn redact_event(&self, event_id: OwnedEventId) -> Result { return Err!("Event is already redacted."); } - let room_id = event.room_id; - let sender_user = event.sender; - - if !self.services.globals.user_is_local(&sender_user) { + if !self.services.globals.user_is_local(event.sender()) { return Err!("This command only works on local users."); } @@ -902,21 +899,21 @@ pub(super) async fn redact_event(&self, event_id: OwnedEventId) -> Result { ); let redaction_event_id = { - let state_lock = self.services.rooms.state.mutex.lock(&room_id).await; + let state_lock = self.services.rooms.state.mutex.lock(event.room_id()).await; self.services .rooms .timeline .build_and_append_pdu( PduBuilder { - redacts: Some(event.event_id.clone()), + redacts: Some(event.event_id().to_owned()), ..PduBuilder::timeline(&RoomRedactionEventContent { - redacts: Some(event.event_id.clone()), + redacts: Some(event.event_id().to_owned()), reason: Some(reason), }) }, - &sender_user, - &room_id, + event.sender(), + event.room_id(), &state_lock, ) .await? diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 14bbcf98..df938c17 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -3,10 +3,9 @@ use std::fmt::Write; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Err, Error, Result, debug_info, err, error, info, is_equal_to, + Err, Error, Event, Result, debug_info, err, error, info, is_equal_to, matrix::pdu::PduBuilder, - utils, - utils::{ReadyExt, stream::BroadbandExt}, + utils::{self, ReadyExt, stream::BroadbandExt}, warn, }; use conduwuit_service::Services; @@ -140,16 +139,32 @@ pub(crate) async fn register_route( if !services.config.allow_registration && body.appservice_info.is_none() { match (body.username.as_ref(), body.initial_device_display_name.as_ref()) { | (Some(username), Some(device_display_name)) => { - info!(%is_guest, user = %username, device_name = %device_display_name, "Rejecting registration attempt as registration is disabled"); + info!( + %is_guest, + user = %username, + device_name = %device_display_name, + "Rejecting registration attempt as registration is disabled" + ); }, | (Some(username), _) => { - info!(%is_guest, user = %username, "Rejecting registration attempt as registration is disabled"); + info!( + %is_guest, + user = %username, + "Rejecting registration attempt as registration is disabled" + ); }, | (_, Some(device_display_name)) => { - info!(%is_guest, device_name = %device_display_name, "Rejecting registration attempt as registration is disabled"); + info!( + %is_guest, + device_name = %device_display_name, + "Rejecting registration attempt as registration is disabled" + ); }, | (None, _) => { - info!(%is_guest, "Rejecting registration attempt as registration is disabled"); + info!( + %is_guest, + "Rejecting registration attempt as registration is disabled" + ); }, } @@ -835,6 +850,7 @@ pub async fn full_user_deactivate( all_joined_rooms: &[OwnedRoomId], ) -> Result<()> { services.users.deactivate_account(user_id).await.ok(); + super::update_displayname(services, user_id, None, all_joined_rooms).await; super::update_avatar_url(services, user_id, None, None, all_joined_rooms).await; @@ -871,7 +887,7 @@ pub async fn full_user_deactivate( .state_accessor .room_state_get(room_id, &StateEventType::RoomCreate, "") .await - .is_ok_and(|event| event.sender == user_id); + .is_ok_and(|event| event.sender() == user_id); if user_can_demote_self { let mut power_levels_content = room_power_levels.unwrap_or_default(); diff --git a/src/api/client/directory.rs b/src/api/client/directory.rs index 2e219fd9..00879274 100644 --- a/src/api/client/directory.rs +++ b/src/api/client/directory.rs @@ -1,7 +1,7 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Err, Result, err, info, + Err, Event, Result, err, info, utils::{ TryFutureExtExt, math::Expected, @@ -352,7 +352,7 @@ async fn user_can_publish_room( .room_state_get(room_id, &StateEventType::RoomPowerLevels, "") .await { - | Ok(event) => serde_json::from_str(event.content.get()) + | Ok(event) => serde_json::from_str(event.content().get()) .map_err(|_| err!(Database("Invalid event content for m.room.power_levels"))) .map(|content: RoomPowerLevelsEventContent| { RoomPowerLevels::from(content) @@ -365,7 +365,7 @@ async fn user_can_publish_room( .room_state_get(room_id, &StateEventType::RoomCreate, "") .await { - | Ok(event) => Ok(event.sender == user_id), + | Ok(event) => Ok(event.sender() == user_id), | _ => Err!(Request(Forbidden("User is not allowed to publish this room"))), } }, diff --git a/src/api/client/membership/invite.rs b/src/api/client/membership/invite.rs index 4ca3efb8..018fb774 100644 --- a/src/api/client/membership/invite.rs +++ b/src/api/client/membership/invite.rs @@ -2,7 +2,7 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ Err, Result, debug_error, err, info, - matrix::pdu::{PduBuilder, gen_event_id_canonical_json}, + matrix::{event::gen_event_id_canonical_json, pdu::PduBuilder}, }; use futures::{FutureExt, join}; use ruma::{ diff --git a/src/api/client/membership/join.rs b/src/api/client/membership/join.rs index 669e9399..9d19d3bc 100644 --- a/src/api/client/membership/join.rs +++ b/src/api/client/membership/join.rs @@ -6,7 +6,8 @@ use conduwuit::{ Err, Result, debug, debug_info, debug_warn, err, error, info, matrix::{ StateKey, - pdu::{PduBuilder, PduEvent, gen_event_id, gen_event_id_canonical_json}, + event::{gen_event_id, gen_event_id_canonical_json}, + pdu::{PduBuilder, PduEvent}, state_res, }, result::FlatOk, diff --git a/src/api/client/membership/knock.rs b/src/api/client/membership/knock.rs index 544dcfb3..79f16631 100644 --- a/src/api/client/membership/knock.rs +++ b/src/api/client/membership/knock.rs @@ -4,7 +4,10 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ Err, Result, debug, debug_info, debug_warn, err, info, - matrix::pdu::{PduBuilder, PduEvent, gen_event_id}, + matrix::{ + event::{Event, gen_event_id}, + pdu::{PduBuilder, PduEvent}, + }, result::FlatOk, trace, utils::{self, shuffle, stream::IterStream}, diff --git a/src/api/client/membership/leave.rs b/src/api/client/membership/leave.rs index a64fb41f..f4f1666b 100644 --- a/src/api/client/membership/leave.rs +++ b/src/api/client/membership/leave.rs @@ -3,7 +3,7 @@ use std::collections::HashSet; use axum::extract::State; use conduwuit::{ Err, Result, debug_info, debug_warn, err, - matrix::pdu::{PduBuilder, gen_event_id}, + matrix::{event::gen_event_id, pdu::PduBuilder}, utils::{self, FutureBoolExt, future::ReadyEqExt}, warn, }; diff --git a/src/api/client/membership/members.rs b/src/api/client/membership/members.rs index 4a7abf6d..05ba1c43 100644 --- a/src/api/client/membership/members.rs +++ b/src/api/client/membership/members.rs @@ -1,13 +1,12 @@ use axum::extract::State; use conduwuit::{ Err, Event, Result, at, - matrix::pdu::PduEvent, utils::{ future::TryExtExt, stream::{BroadbandExt, ReadyExt}, }, }; -use futures::{StreamExt, future::join}; +use futures::{FutureExt, StreamExt, future::join}; use ruma::{ api::client::membership::{ get_member_events::{self, v3::MembershipEventFilter}, @@ -55,6 +54,7 @@ pub(crate) async fn get_member_events_route( .ready_filter_map(|pdu| membership_filter(pdu, membership, not_membership)) .map(Event::into_format) .collect() + .boxed() .await, }) } @@ -98,11 +98,11 @@ pub(crate) async fn joined_members_route( }) } -fn membership_filter( - pdu: PduEvent, +fn membership_filter( + pdu: Pdu, for_membership: Option<&MembershipEventFilter>, not_membership: Option<&MembershipEventFilter>, -) -> Option { +) -> Option { let membership_state_filter = match for_membership { | Some(MembershipEventFilter::Ban) => MembershipState::Ban, | Some(MembershipEventFilter::Invite) => MembershipState::Invite, diff --git a/src/api/client/message.rs b/src/api/client/message.rs index e32d020f..f8818ebb 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -2,9 +2,10 @@ use axum::extract::State; use conduwuit::{ Err, Result, at, matrix::{ - Event, - pdu::{PduCount, PduEvent}, + event::{Event, Matches}, + pdu::PduCount, }, + ref_at, utils::{ IterStream, ReadyExt, result::{FlatOk, LogErr}, @@ -216,7 +217,9 @@ where pin_mut!(receipts); let witness: Witness = events .stream() - .map(|(_, pdu)| pdu.sender.clone()) + .map(ref_at!(1)) + .map(Event::sender) + .map(ToOwned::to_owned) .chain( receipts .ready_take_while(|(_, c, _)| *c <= newest.into_unsigned()) @@ -261,27 +264,33 @@ pub(crate) async fn ignored_filter( } #[inline] -pub(crate) async fn is_ignored_pdu( +pub(crate) async fn is_ignored_pdu( services: &Services, - pdu: &PduEvent, + event: &Pdu, user_id: &UserId, -) -> bool { +) -> bool +where + Pdu: Event + Send + Sync, +{ // exclude Synapse's dummy events from bloating up response bodies. clients // don't need to see this. - if pdu.kind.to_cow_str() == "org.matrix.dummy_event" { + if event.kind().to_cow_str() == "org.matrix.dummy_event" { return true; } - let ignored_type = IGNORED_MESSAGE_TYPES.binary_search(&pdu.kind).is_ok(); + let ignored_type = IGNORED_MESSAGE_TYPES.binary_search(event.kind()).is_ok(); let ignored_server = services .moderation - .is_remote_server_ignored(pdu.sender().server_name()); + .is_remote_server_ignored(event.sender().server_name()); if ignored_type && (ignored_server || (!services.config.send_messages_from_ignored_users_to_client - && services.users.user_is_ignored(&pdu.sender, user_id).await)) + && services + .users + .user_is_ignored(event.sender(), user_id) + .await)) { return true; } @@ -300,7 +309,7 @@ pub(crate) async fn visibility_filter( services .rooms .state_accessor - .user_can_see_event(user_id, &pdu.room_id, &pdu.event_id) + .user_can_see_event(user_id, pdu.room_id(), pdu.event_id()) .await .then_some(item) } @@ -308,7 +317,7 @@ pub(crate) async fn visibility_filter( #[inline] pub(crate) fn event_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Option { let (_, pdu) = &item; - pdu.matches(filter).then_some(item) + filter.matches(pdu).then_some(item) } #[cfg_attr(debug_assertions, conduwuit::ctor)] diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index ff8c2a0b..1882495c 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -195,11 +195,9 @@ pub(crate) async fn get_avatar_url_route( services .users .set_displayname(&body.user_id, response.displayname.clone()); - services .users .set_avatar_url(&body.user_id, response.avatar_url.clone()); - services .users .set_blurhash(&body.user_id, response.blurhash.clone()); @@ -256,15 +254,12 @@ pub(crate) async fn get_profile_route( services .users .set_displayname(&body.user_id, response.displayname.clone()); - services .users .set_avatar_url(&body.user_id, response.avatar_url.clone()); - services .users .set_blurhash(&body.user_id, response.blurhash.clone()); - services .users .set_timezone(&body.user_id, response.tz.clone()); diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index ad726b90..1aa34ada 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,10 +1,10 @@ use axum::extract::State; use conduwuit::{ Result, at, - matrix::{Event, pdu::PduCount}, + matrix::{Event, event::RelationTypeEqual, pdu::PduCount}, utils::{IterStream, ReadyExt, result::FlatOk, stream::WidebandExt}, }; -use conduwuit_service::{Services, rooms::timeline::PdusIterItem}; +use conduwuit_service::Services; use futures::StreamExt; use ruma::{ EventId, RoomId, UInt, UserId, @@ -129,7 +129,7 @@ async fn paginate_relations_with_filter( // Spec (v1.10) recommends depth of at least 3 let depth: u8 = if recurse { 3 } else { 1 }; - let events: Vec = services + let events: Vec<_> = services .rooms .pdu_metadata .get_relations(sender_user, room_id, target, start, limit, depth, dir) @@ -138,12 +138,12 @@ async fn paginate_relations_with_filter( .filter(|(_, pdu)| { filter_event_type .as_ref() - .is_none_or(|kind| *kind == pdu.kind) + .is_none_or(|kind| kind == pdu.kind()) }) .filter(|(_, pdu)| { filter_rel_type .as_ref() - .is_none_or(|rel_type| pdu.relation_type_equal(rel_type)) + .is_none_or(|rel_type| rel_type.relation_type_equal(pdu)) }) .stream() .ready_take_while(|(count, _)| Some(*count) != to) @@ -172,17 +172,17 @@ async fn paginate_relations_with_filter( }) } -async fn visibility_filter( +async fn visibility_filter( services: &Services, sender_user: &UserId, - item: PdusIterItem, -) -> Option { + item: (PduCount, Pdu), +) -> Option<(PduCount, Pdu)> { let (_, pdu) = &item; services .rooms .state_accessor - .user_can_see_event(sender_user, &pdu.room_id, &pdu.event_id) + .user_can_see_event(sender_user, pdu.room_id(), pdu.event_id()) .await .then_some(item) } diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 1019b358..052329d1 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -260,5 +260,6 @@ async fn delay_response() { "Got successful /report request, waiting {time_to_wait} seconds before sending \ successful response." ); + sleep(Duration::from_secs(time_to_wait)).await; } diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index c2f59d4c..2aca5b9d 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -49,7 +49,9 @@ pub(crate) async fn room_initial_sync_route( .try_collect::>(); let (membership, visibility, state, events) = - try_join4(membership, visibility, state, events).await?; + try_join4(membership, visibility, state, events) + .boxed() + .await?; let messages = PaginationChunk { start: events.last().map(at!(0)).as_ref().map(ToString::to_string), diff --git a/src/api/client/room/summary.rs b/src/api/client/room/summary.rs index ab534765..635f5a8a 100644 --- a/src/api/client/room/summary.rs +++ b/src/api/client/room/summary.rs @@ -112,13 +112,15 @@ async fn local_room_summary_response( ) -> Result { trace!(?sender_user, "Sending local room summary response for {room_id:?}"); let join_rule = services.rooms.state_accessor.get_join_rules(room_id); + let world_readable = services.rooms.state_accessor.is_world_readable(room_id); + let guest_can_join = services.rooms.state_accessor.guest_can_join(room_id); let (join_rule, world_readable, guest_can_join) = join3(join_rule, world_readable, guest_can_join).await; - trace!("{join_rule:?}, {world_readable:?}, {guest_can_join:?}"); + trace!("{join_rule:?}, {world_readable:?}, {guest_can_join:?}"); user_can_see_summary( services, room_id, diff --git a/src/api/client/room/upgrade.rs b/src/api/client/room/upgrade.rs index d8f5ea83..ae632235 100644 --- a/src/api/client/room/upgrade.rs +++ b/src/api/client/room/upgrade.rs @@ -2,7 +2,7 @@ use std::cmp::max; use axum::extract::State; use conduwuit::{ - Err, Error, Result, err, info, + Err, Error, Event, Result, err, info, matrix::{StateKey, pdu::PduBuilder}, }; use futures::StreamExt; @@ -215,7 +215,7 @@ pub(crate) async fn upgrade_room_route( .room_state_get(&body.room_id, event_type, "") .await { - | Ok(v) => v.content.clone(), + | Ok(v) => v.content().to_owned(), | Err(_) => continue, // Skipping missing events. }; diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index cabd67e4..14cd50d8 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -6,7 +6,7 @@ use std::{ use axum::extract::State; use conduwuit::{ - Err, Error, Event, PduCount, PduEvent, Result, at, debug, error, extract_variant, + Err, Error, Event, PduCount, Result, at, debug, error, extract_variant, matrix::TypeStateKey, utils::{ BoolExt, IterStream, ReadyExt, TryFutureExtExt, @@ -627,7 +627,7 @@ pub(crate) async fn sync_events_v4_route( .state_accessor .room_state_get(room_id, &state.0, &state.1) .await - .map(PduEvent::into_format) + .map(Event::into_format) .ok() }) .collect() diff --git a/src/api/server/invite.rs b/src/api/server/invite.rs index 0d26d787..0a9b2e10 100644 --- a/src/api/server/invite.rs +++ b/src/api/server/invite.rs @@ -2,8 +2,10 @@ use axum::extract::State; use axum_client_ip::InsecureClientIp; use base64::{Engine as _, engine::general_purpose}; use conduwuit::{ - Err, Error, PduEvent, Result, err, matrix::Event, pdu::gen_event_id, utils, - utils::hash::sha256, warn, + Err, Error, PduEvent, Result, err, + matrix::{Event, event::gen_event_id}, + utils::{self, hash::sha256}, + warn, }; use ruma::{ CanonicalJsonValue, OwnedUserId, UserId, diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 895eca81..652451c7 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -5,7 +5,7 @@ use std::borrow::Borrow; use axum::extract::State; use conduwuit::{ Err, Result, at, err, - pdu::gen_event_id_canonical_json, + matrix::event::gen_event_id_canonical_json, utils::stream::{IterStream, TryBroadbandExt}, warn, }; diff --git a/src/api/server/send_knock.rs b/src/api/server/send_knock.rs index 8d3697d2..ffd41ada 100644 --- a/src/api/server/send_knock.rs +++ b/src/api/server/send_knock.rs @@ -1,7 +1,7 @@ use axum::extract::State; use conduwuit::{ Err, Result, err, - matrix::pdu::{PduEvent, gen_event_id_canonical_json}, + matrix::{event::gen_event_id_canonical_json, pdu::PduEvent}, warn, }; use futures::FutureExt; diff --git a/src/api/server/send_leave.rs b/src/api/server/send_leave.rs index d3dc994c..b6336e1a 100644 --- a/src/api/server/send_leave.rs +++ b/src/api/server/send_leave.rs @@ -1,7 +1,7 @@ #![allow(deprecated)] use axum::extract::State; -use conduwuit::{Err, Result, err, matrix::pdu::gen_event_id_canonical_json}; +use conduwuit::{Err, Result, err, matrix::event::gen_event_id_canonical_json}; use conduwuit_service::Services; use futures::FutureExt; use ruma::{ diff --git a/src/core/config/proxy.rs b/src/core/config/proxy.rs index ea388f24..77c4531a 100644 --- a/src/core/config/proxy.rs +++ b/src/core/config/proxy.rs @@ -88,10 +88,7 @@ impl PartialProxyConfig { } } match (included_because, excluded_because) { - | (Some(a), Some(b)) if a.more_specific_than(b) => Some(&self.url), /* included for - * a more specific - * reason */ - // than excluded + | (Some(a), Some(b)) if a.more_specific_than(b) => Some(&self.url), | (Some(_), None) => Some(&self.url), | _ => None, } diff --git a/src/core/info/cargo.rs b/src/core/info/cargo.rs index e70bdcd5..61a97508 100644 --- a/src/core/info/cargo.rs +++ b/src/core/info/cargo.rs @@ -84,10 +84,12 @@ fn append_features(features: &mut Vec, manifest: &str) -> Result<()> { fn init_dependencies() -> Result { let manifest = Manifest::from_str(WORKSPACE_MANIFEST)?; - Ok(manifest + let deps_set = manifest .workspace .as_ref() .expect("manifest has workspace section") .dependencies - .clone()) + .clone(); + + Ok(deps_set) } diff --git a/src/core/matrix/event.rs b/src/core/matrix/event.rs index 5b12770b..a1d1339e 100644 --- a/src/core/matrix/event.rs +++ b/src/core/matrix/event.rs @@ -1,21 +1,27 @@ mod content; +mod filter; mod format; +mod id; mod redact; +mod relation; mod type_ext; +mod unsigned; + +use std::fmt::Debug; use ruma::{ - EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, RoomVersionId, UserId, - events::TimelineEventType, + CanonicalJsonObject, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, + RoomVersionId, UserId, events::TimelineEventType, }; use serde::Deserialize; use serde_json::{Value as JsonValue, value::RawValue as RawJsonValue}; -pub use self::type_ext::TypeExt; -use super::state_key::StateKey; -use crate::Result; +pub use self::{filter::Matches, id::*, relation::RelationTypeEqual, type_ext::TypeExt}; +use super::{pdu::Pdu, state_key::StateKey}; +use crate::{Result, utils}; /// Abstraction of a PDU so users can have their own PDU types. -pub trait Event { +pub trait Event: Clone + Debug { /// Serialize into a Ruma JSON format, consuming. #[inline] fn into_format(self) -> T @@ -36,6 +42,41 @@ pub trait Event { format::Ref(self).into() } + #[inline] + fn contains_unsigned_property(&self, property: &str, is_type: T) -> bool + where + T: FnOnce(&JsonValue) -> bool, + Self: Sized, + { + unsigned::contains_unsigned_property::(self, property, is_type) + } + + #[inline] + fn get_unsigned_property(&self, property: &str) -> Result + where + T: for<'de> Deserialize<'de>, + Self: Sized, + { + unsigned::get_unsigned_property::(self, property) + } + + #[inline] + fn get_unsigned_as_value(&self) -> JsonValue + where + Self: Sized, + { + unsigned::get_unsigned_as_value(self) + } + + #[inline] + fn get_unsigned(&self) -> Result + where + T: for<'de> Deserialize<'de>, + Self: Sized, + { + unsigned::get_unsigned::(self) + } + #[inline] fn get_content_as_value(&self) -> JsonValue where @@ -69,6 +110,39 @@ pub trait Event { redact::is_redacted(self) } + #[inline] + fn into_canonical_object(self) -> CanonicalJsonObject + where + Self: Sized, + { + utils::to_canonical_object(self.into_pdu()).expect("failed to create Value::Object") + } + + #[inline] + fn to_canonical_object(&self) -> CanonicalJsonObject { + utils::to_canonical_object(self.as_pdu()).expect("failed to create Value::Object") + } + + #[inline] + fn into_value(self) -> JsonValue + where + Self: Sized, + { + serde_json::to_value(self.into_pdu()).expect("failed to create JSON Value") + } + + #[inline] + fn to_value(&self) -> JsonValue { + serde_json::to_value(self.as_pdu()).expect("failed to create JSON Value") + } + + #[inline] + fn as_mut_pdu(&mut self) -> &mut Pdu { unimplemented!("not a mutable Pdu") } + + fn as_pdu(&self) -> &Pdu; + + fn into_pdu(self) -> Pdu; + fn is_owned(&self) -> bool; // @@ -76,7 +150,7 @@ pub trait Event { // /// All the authenticating events for this event. - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_; + fn auth_events(&self) -> impl DoubleEndedIterator + Clone + Send + '_; /// The event's content. fn content(&self) -> &RawJsonValue; @@ -88,7 +162,7 @@ pub trait Event { fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch; /// The events before this event. - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_; + fn prev_events(&self) -> impl DoubleEndedIterator + Clone + Send + '_; /// If this event is a redaction event this is the event it redacts. fn redacts(&self) -> Option<&EventId>; diff --git a/src/core/matrix/event/filter.rs b/src/core/matrix/event/filter.rs new file mode 100644 index 00000000..d3a225b6 --- /dev/null +++ b/src/core/matrix/event/filter.rs @@ -0,0 +1,93 @@ +use ruma::api::client::filter::{RoomEventFilter, UrlFilter}; +use serde_json::Value; + +use super::Event; +use crate::is_equal_to; + +pub trait Matches { + fn matches(&self, event: &E) -> bool; +} + +impl Matches for &RoomEventFilter { + #[inline] + fn matches(&self, event: &E) -> bool { + if !matches_sender(event, self) { + return false; + } + + if !matches_room(event, self) { + return false; + } + + if !matches_type(event, self) { + return false; + } + + if !matches_url(event, self) { + return false; + } + + true + } +} + +fn matches_room(event: &E, filter: &RoomEventFilter) -> bool { + if filter.not_rooms.iter().any(is_equal_to!(event.room_id())) { + return false; + } + + if let Some(rooms) = filter.rooms.as_ref() { + if !rooms.iter().any(is_equal_to!(event.room_id())) { + return false; + } + } + + true +} + +fn matches_sender(event: &E, filter: &RoomEventFilter) -> bool { + if filter.not_senders.iter().any(is_equal_to!(event.sender())) { + return false; + } + + if let Some(senders) = filter.senders.as_ref() { + if !senders.iter().any(is_equal_to!(event.sender())) { + return false; + } + } + + true +} + +fn matches_type(event: &E, filter: &RoomEventFilter) -> bool { + let kind = event.kind().to_cow_str(); + + if filter.not_types.iter().any(is_equal_to!(&kind)) { + return false; + } + + if let Some(types) = filter.types.as_ref() { + if !types.iter().any(is_equal_to!(&kind)) { + return false; + } + } + + true +} + +fn matches_url(event: &E, filter: &RoomEventFilter) -> bool { + let Some(url_filter) = filter.url_filter.as_ref() else { + return true; + }; + + //TODO: might be better to use Ruma's Raw rather than serde here + let url = event + .get_content_as_value() + .get("url") + .is_some_and(Value::is_string); + + match url_filter { + | UrlFilter::EventsWithUrl => url, + | UrlFilter::EventsWithoutUrl => !url, + } +} diff --git a/src/core/matrix/pdu/event_id.rs b/src/core/matrix/event/id.rs similarity index 100% rename from src/core/matrix/pdu/event_id.rs rename to src/core/matrix/event/id.rs diff --git a/src/core/matrix/event/relation.rs b/src/core/matrix/event/relation.rs new file mode 100644 index 00000000..58324e86 --- /dev/null +++ b/src/core/matrix/event/relation.rs @@ -0,0 +1,28 @@ +use ruma::events::relation::RelationType; +use serde::Deserialize; + +use super::Event; + +pub trait RelationTypeEqual { + fn relation_type_equal(&self, event: &E) -> bool; +} + +#[derive(Clone, Debug, Deserialize)] +struct ExtractRelatesToEventId { + #[serde(rename = "m.relates_to")] + relates_to: ExtractRelType, +} + +#[derive(Clone, Debug, Deserialize)] +struct ExtractRelType { + rel_type: RelationType, +} + +impl RelationTypeEqual for RelationType { + fn relation_type_equal(&self, event: &E) -> bool { + event + .get_content() + .map(|c: ExtractRelatesToEventId| c.relates_to.rel_type) + .is_ok_and(|r| r == *self) + } +} diff --git a/src/core/matrix/event/unsigned.rs b/src/core/matrix/event/unsigned.rs new file mode 100644 index 00000000..42928af4 --- /dev/null +++ b/src/core/matrix/event/unsigned.rs @@ -0,0 +1,51 @@ +use serde::Deserialize; +use serde_json::value::Value as JsonValue; + +use super::Event; +use crate::{Result, err, is_true}; + +pub(super) fn contains_unsigned_property(event: &E, property: &str, is_type: F) -> bool +where + F: FnOnce(&JsonValue) -> bool, + E: Event, +{ + get_unsigned_as_value(event) + .get(property) + .map(is_type) + .is_some_and(is_true!()) +} + +pub(super) fn get_unsigned_property(event: &E, property: &str) -> Result +where + T: for<'de> Deserialize<'de>, + E: Event, +{ + get_unsigned_as_value(event) + .get_mut(property) + .map(JsonValue::take) + .map(serde_json::from_value) + .ok_or(err!(Request(NotFound("property not found in unsigned object"))))? + .map_err(|e| err!(Database("Failed to deserialize unsigned.{property} into type: {e}"))) +} + +#[must_use] +pub(super) fn get_unsigned_as_value(event: &E) -> JsonValue +where + E: Event, +{ + get_unsigned::(event).unwrap_or_default() +} + +pub(super) fn get_unsigned(event: &E) -> Result +where + T: for<'de> Deserialize<'de>, + E: Event, +{ + event + .unsigned() + .as_ref() + .map(|raw| raw.get()) + .map(serde_json::from_str) + .ok_or(err!(Request(NotFound("\"unsigned\" property not found in pdu"))))? + .map_err(|e| err!(Database("Failed to deserialize \"unsigned\" into value: {e}"))) +} diff --git a/src/core/matrix/pdu.rs b/src/core/matrix/pdu.rs index e64baeb8..bff0c203 100644 --- a/src/core/matrix/pdu.rs +++ b/src/core/matrix/pdu.rs @@ -1,12 +1,8 @@ mod builder; -mod content; mod count; -mod event_id; -mod filter; mod id; mod raw_id; mod redact; -mod relation; #[cfg(test)] mod tests; mod unsigned; @@ -24,7 +20,6 @@ pub use self::{ Count as PduCount, Id as PduId, Pdu as PduEvent, RawId as RawPduId, builder::{Builder, Builder as PduBuilder}, count::Count, - event_id::*, id::{ShortId, *}, raw_id::*, }; @@ -91,7 +86,7 @@ impl Pdu { impl Event for Pdu { #[inline] - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { + fn auth_events(&self) -> impl DoubleEndedIterator + Clone + Send + '_ { self.auth_events.iter().map(AsRef::as_ref) } @@ -107,7 +102,7 @@ impl Event for Pdu { } #[inline] - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + fn prev_events(&self) -> impl DoubleEndedIterator + Clone + Send + '_ { self.prev_events.iter().map(AsRef::as_ref) } @@ -129,13 +124,22 @@ impl Event for Pdu { #[inline] fn unsigned(&self) -> Option<&RawJsonValue> { self.unsigned.as_deref() } + #[inline] + fn as_mut_pdu(&mut self) -> &mut Pdu { self } + + #[inline] + fn as_pdu(&self) -> &Pdu { self } + + #[inline] + fn into_pdu(self) -> Pdu { self } + #[inline] fn is_owned(&self) -> bool { true } } impl Event for &Pdu { #[inline] - fn auth_events(&self) -> impl DoubleEndedIterator + Send + '_ { + fn auth_events(&self) -> impl DoubleEndedIterator + Clone + Send + '_ { self.auth_events.iter().map(AsRef::as_ref) } @@ -151,7 +155,7 @@ impl Event for &Pdu { } #[inline] - fn prev_events(&self) -> impl DoubleEndedIterator + Send + '_ { + fn prev_events(&self) -> impl DoubleEndedIterator + Clone + Send + '_ { self.prev_events.iter().map(AsRef::as_ref) } @@ -173,6 +177,12 @@ impl Event for &Pdu { #[inline] fn unsigned(&self) -> Option<&RawJsonValue> { self.unsigned.as_deref() } + #[inline] + fn as_pdu(&self) -> &Pdu { self } + + #[inline] + fn into_pdu(self) -> Pdu { self.clone() } + #[inline] fn is_owned(&self) -> bool { false } } diff --git a/src/core/matrix/pdu/content.rs b/src/core/matrix/pdu/content.rs deleted file mode 100644 index 4e60ce6e..00000000 --- a/src/core/matrix/pdu/content.rs +++ /dev/null @@ -1,20 +0,0 @@ -use serde::Deserialize; -use serde_json::value::Value as JsonValue; - -use crate::{Result, err, implement}; - -#[must_use] -#[implement(super::Pdu)] -pub fn get_content_as_value(&self) -> JsonValue { - self.get_content() - .expect("pdu content must be a valid JSON value") -} - -#[implement(super::Pdu)] -pub fn get_content(&self) -> Result -where - T: for<'de> Deserialize<'de>, -{ - serde_json::from_str(self.content.get()) - .map_err(|e| err!(Database("Failed to deserialize pdu content into type: {e}"))) -} diff --git a/src/core/matrix/pdu/filter.rs b/src/core/matrix/pdu/filter.rs deleted file mode 100644 index aabf13db..00000000 --- a/src/core/matrix/pdu/filter.rs +++ /dev/null @@ -1,90 +0,0 @@ -use ruma::api::client::filter::{RoomEventFilter, UrlFilter}; -use serde_json::Value; - -use crate::{implement, is_equal_to}; - -#[implement(super::Pdu)] -#[must_use] -pub fn matches(&self, filter: &RoomEventFilter) -> bool { - if !self.matches_sender(filter) { - return false; - } - - if !self.matches_room(filter) { - return false; - } - - if !self.matches_type(filter) { - return false; - } - - if !self.matches_url(filter) { - return false; - } - - true -} - -#[implement(super::Pdu)] -fn matches_room(&self, filter: &RoomEventFilter) -> bool { - if filter.not_rooms.contains(&self.room_id) { - return false; - } - - if let Some(rooms) = filter.rooms.as_ref() { - if !rooms.contains(&self.room_id) { - return false; - } - } - - true -} - -#[implement(super::Pdu)] -fn matches_sender(&self, filter: &RoomEventFilter) -> bool { - if filter.not_senders.contains(&self.sender) { - return false; - } - - if let Some(senders) = filter.senders.as_ref() { - if !senders.contains(&self.sender) { - return false; - } - } - - true -} - -#[implement(super::Pdu)] -fn matches_type(&self, filter: &RoomEventFilter) -> bool { - let event_type = &self.kind.to_cow_str(); - if filter.not_types.iter().any(is_equal_to!(event_type)) { - return false; - } - - if let Some(types) = filter.types.as_ref() { - if !types.iter().any(is_equal_to!(event_type)) { - return false; - } - } - - true -} - -#[implement(super::Pdu)] -fn matches_url(&self, filter: &RoomEventFilter) -> bool { - let Some(url_filter) = filter.url_filter.as_ref() else { - return true; - }; - - //TODO: might be better to use Ruma's Raw rather than serde here - let url = serde_json::from_str::(self.content.get()) - .expect("parsing content JSON failed") - .get("url") - .is_some_and(Value::is_string); - - match url_filter { - | UrlFilter::EventsWithUrl => url, - | UrlFilter::EventsWithoutUrl => !url, - } -} diff --git a/src/core/matrix/pdu/redact.rs b/src/core/matrix/pdu/redact.rs index e6a03209..896e03f8 100644 --- a/src/core/matrix/pdu/redact.rs +++ b/src/core/matrix/pdu/redact.rs @@ -1,10 +1,10 @@ use ruma::{RoomVersionId, canonical_json::redact_content_in_place}; -use serde_json::{json, value::to_raw_value}; +use serde_json::{Value as JsonValue, json, value::to_raw_value}; use crate::{Error, Result, err, implement}; #[implement(super::Pdu)] -pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: &Self) -> Result { +pub fn redact(&mut self, room_version_id: &RoomVersionId, reason: JsonValue) -> Result { self.unsigned = None; let mut content = serde_json::from_str(self.content.get()) diff --git a/src/core/matrix/pdu/relation.rs b/src/core/matrix/pdu/relation.rs deleted file mode 100644 index 2968171e..00000000 --- a/src/core/matrix/pdu/relation.rs +++ /dev/null @@ -1,22 +0,0 @@ -use ruma::events::relation::RelationType; -use serde::Deserialize; - -use crate::implement; - -#[derive(Clone, Debug, Deserialize)] -struct ExtractRelType { - rel_type: RelationType, -} -#[derive(Clone, Debug, Deserialize)] -struct ExtractRelatesToEventId { - #[serde(rename = "m.relates_to")] - relates_to: ExtractRelType, -} - -#[implement(super::Pdu)] -#[must_use] -pub fn relation_type_equal(&self, rel_type: &RelationType) -> bool { - self.get_content() - .map(|c: ExtractRelatesToEventId| c.relates_to.rel_type) - .is_ok_and(|r| r == *rel_type) -} diff --git a/src/core/matrix/pdu/unsigned.rs b/src/core/matrix/pdu/unsigned.rs index 23897519..0c58bb68 100644 --- a/src/core/matrix/pdu/unsigned.rs +++ b/src/core/matrix/pdu/unsigned.rs @@ -1,11 +1,10 @@ use std::collections::BTreeMap; use ruma::MilliSecondsSinceUnixEpoch; -use serde::Deserialize; use serde_json::value::{RawValue as RawJsonValue, Value as JsonValue, to_raw_value}; use super::Pdu; -use crate::{Result, err, implement, is_true}; +use crate::{Result, err, implement}; #[implement(Pdu)] pub fn remove_transaction_id(&mut self) -> Result { @@ -74,43 +73,3 @@ pub fn add_relation(&mut self, name: &str, pdu: Option<&Pdu>) -> Result { Ok(()) } - -#[implement(Pdu)] -pub fn contains_unsigned_property(&self, property: &str, is_type: F) -> bool -where - F: FnOnce(&JsonValue) -> bool, -{ - self.get_unsigned_as_value() - .get(property) - .map(is_type) - .is_some_and(is_true!()) -} - -#[implement(Pdu)] -pub fn get_unsigned_property(&self, property: &str) -> Result -where - T: for<'de> Deserialize<'de>, -{ - self.get_unsigned_as_value() - .get_mut(property) - .map(JsonValue::take) - .map(serde_json::from_value) - .ok_or(err!(Request(NotFound("property not found in unsigned object"))))? - .map_err(|e| err!(Database("Failed to deserialize unsigned.{property} into type: {e}"))) -} - -#[implement(Pdu)] -#[must_use] -pub fn get_unsigned_as_value(&self) -> JsonValue { - self.get_unsigned::().unwrap_or_default() -} - -#[implement(Pdu)] -pub fn get_unsigned(&self) -> Result { - self.unsigned - .as_ref() - .map(|raw| raw.get()) - .map(serde_json::from_str) - .ok_or(err!(Request(NotFound("\"unsigned\" property not found in pdu"))))? - .map_err(|e| err!(Database("Failed to deserialize \"unsigned\" into value: {e}"))) -} diff --git a/src/core/matrix/state_res/mod.rs b/src/core/matrix/state_res/mod.rs index ed5aa034..ce9d9276 100644 --- a/src/core/matrix/state_res/mod.rs +++ b/src/core/matrix/state_res/mod.rs @@ -74,7 +74,7 @@ type Result = crate::Result; /// event is part of the same room. //#[tracing::instrument(level = "debug", skip(state_sets, auth_chain_sets, //#[tracing::instrument(level event_fetch))] -pub async fn resolve<'a, E, Sets, SetIter, Hasher, Fetch, FetchFut, Exists, ExistsFut>( +pub async fn resolve<'a, Pdu, Sets, SetIter, Hasher, Fetch, FetchFut, Exists, ExistsFut>( room_version: &RoomVersionId, state_sets: Sets, auth_chain_sets: &'a [HashSet], @@ -83,14 +83,14 @@ pub async fn resolve<'a, E, Sets, SetIter, Hasher, Fetch, FetchFut, Exists, Exis ) -> Result> where Fetch: Fn(OwnedEventId) -> FetchFut + Sync, - FetchFut: Future> + Send, + FetchFut: Future> + Send, Exists: Fn(OwnedEventId) -> ExistsFut + Sync, ExistsFut: Future + Send, Sets: IntoIterator + Send, SetIter: Iterator> + Clone + Send, Hasher: BuildHasher + Send + Sync, - E: Event + Clone + Send + Sync, - for<'b> &'b E: Event + Send, + Pdu: Event + Clone + Send + Sync, + for<'b> &'b Pdu: Event + Send, { debug!("State resolution starting"); @@ -221,6 +221,7 @@ where let state_sets_iter = state_sets_iter.inspect(|_| state_set_count = state_set_count.saturating_add(1)); + for (k, v) in state_sets_iter.flatten() { occurrences .entry(k) @@ -305,6 +306,7 @@ where let pl = get_power_level_for_sender(&event_id, fetch_event) .await .ok()?; + Some((event_id, pl)) }) .inspect(|(event_id, pl)| { diff --git a/src/core/mods/module.rs b/src/core/mods/module.rs index b65bbca2..bcadf5aa 100644 --- a/src/core/mods/module.rs +++ b/src/core/mods/module.rs @@ -44,6 +44,7 @@ impl Module { .handle .as_ref() .expect("backing library loaded by this instance"); + // SAFETY: Calls dlsym(3) on unix platforms. This might not have to be unsafe // if wrapped in libloading with_dlerror(). let sym = unsafe { handle.get::(cname.as_bytes()) }; diff --git a/src/core/mods/path.rs b/src/core/mods/path.rs index cde251b3..b792890b 100644 --- a/src/core/mods/path.rs +++ b/src/core/mods/path.rs @@ -27,6 +27,7 @@ pub fn to_name(path: &OsStr) -> Result { .expect("path file stem") .to_str() .expect("name string"); + let name = name.strip_prefix("lib").unwrap_or(name).to_owned(); Ok(name) diff --git a/src/core/utils/html.rs b/src/core/utils/html.rs index f2b6d861..eac4c47f 100644 --- a/src/core/utils/html.rs +++ b/src/core/utils/html.rs @@ -23,8 +23,10 @@ impl fmt::Display for Escape<'_> { | '"' => """, | _ => continue, }; + fmt.write_str(&pile_o_bits[last..i])?; fmt.write_str(s)?; + // NOTE: we only expect single byte characters here - which is fine as long as // we only match single byte characters last = i.saturating_add(1); diff --git a/src/core/utils/json.rs b/src/core/utils/json.rs index 3f2f225e..df4ccd13 100644 --- a/src/core/utils/json.rs +++ b/src/core/utils/json.rs @@ -1,4 +1,4 @@ -use std::{fmt, str::FromStr}; +use std::{fmt, marker::PhantomData, str::FromStr}; use ruma::{CanonicalJsonError, CanonicalJsonObject, canonical_json::try_from_json_map}; @@ -11,25 +11,28 @@ use crate::Result; pub fn to_canonical_object( value: T, ) -> Result { + use CanonicalJsonError::SerDe; use serde::ser::Error; - match serde_json::to_value(value).map_err(CanonicalJsonError::SerDe)? { + match serde_json::to_value(value).map_err(SerDe)? { | serde_json::Value::Object(map) => try_from_json_map(map), - | _ => - Err(CanonicalJsonError::SerDe(serde_json::Error::custom("Value must be an object"))), + | _ => Err(SerDe(serde_json::Error::custom("Value must be an object"))), } } -pub fn deserialize_from_str< - 'de, +pub fn deserialize_from_str<'de, D, T, E>(deserializer: D) -> Result +where D: serde::de::Deserializer<'de>, T: FromStr, E: fmt::Display, ->( - deserializer: D, -) -> Result { - struct Visitor, E>(std::marker::PhantomData); - impl, Err: fmt::Display> serde::de::Visitor<'_> for Visitor { +{ + struct Visitor, E>(PhantomData); + + impl serde::de::Visitor<'_> for Visitor + where + T: FromStr, + Err: fmt::Display, + { type Value = T; fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -43,5 +46,6 @@ pub fn deserialize_from_str< v.parse().map_err(serde::de::Error::custom) } } - deserializer.deserialize_str(Visitor(std::marker::PhantomData)) + + deserializer.deserialize_str(Visitor(PhantomData)) } diff --git a/src/core/utils/time.rs b/src/core/utils/time.rs index 73f73971..394e08cb 100644 --- a/src/core/utils/time.rs +++ b/src/core/utils/time.rs @@ -105,14 +105,11 @@ pub fn whole_unit(d: Duration) -> Unit { | 86_400.. => Days(d.as_secs() / 86_400), | 3_600..=86_399 => Hours(d.as_secs() / 3_600), | 60..=3_599 => Mins(d.as_secs() / 60), - | _ => match d.as_micros() { | 1_000_000.. => Secs(d.as_secs()), | 1_000..=999_999 => Millis(d.subsec_millis().into()), - | _ => match d.as_nanos() { | 1_000.. => Micros(d.subsec_micros().into()), - | _ => Nanos(d.subsec_nanos().into()), }, }, diff --git a/src/database/watchers.rs b/src/database/watchers.rs index b3907833..efb939d7 100644 --- a/src/database/watchers.rs +++ b/src/database/watchers.rs @@ -37,7 +37,6 @@ impl Watchers { pub(crate) fn wake(&self, key: &[u8]) { let watchers = self.watchers.read().unwrap(); let mut triggered = Vec::new(); - for length in 0..=key.len() { if watchers.contains_key(&key[..length]) { triggered.push(&key[..length]); diff --git a/src/main/logging.rs b/src/main/logging.rs index aec50bd4..36a8896c 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -22,10 +22,12 @@ pub(crate) fn init( let reload_handles = LogLevelReloadHandles::default(); let console_span_events = fmt_span::from_str(&config.log_span_events).unwrap_or_err(); + let console_filter = EnvFilter::builder() .with_regex(config.log_filter_regex) .parse(&config.log) .map_err(|e| err!(Config("log", "{e}.")))?; + let console_layer = fmt::Layer::new() .with_span_events(console_span_events) .event_format(ConsoleFormat::new(config)) @@ -34,6 +36,7 @@ pub(crate) fn init( let (console_reload_filter, console_reload_handle) = reload::Layer::new(console_filter.clone()); + reload_handles.add("console", Box::new(console_reload_handle)); let cap_state = Arc::new(capture::State::new()); @@ -47,8 +50,10 @@ pub(crate) fn init( let subscriber = { let sentry_filter = EnvFilter::try_new(&config.sentry_filter) .map_err(|e| err!(Config("sentry_filter", "{e}.")))?; + let sentry_layer = sentry_tracing::layer(); let (sentry_reload_filter, sentry_reload_handle) = reload::Layer::new(sentry_filter); + reload_handles.add("sentry", Box::new(sentry_reload_handle)); subscriber.with(sentry_layer.with_filter(sentry_reload_filter)) }; @@ -58,12 +63,15 @@ pub(crate) fn init( let (flame_layer, flame_guard) = if config.tracing_flame { let flame_filter = EnvFilter::try_new(&config.tracing_flame_filter) .map_err(|e| err!(Config("tracing_flame_filter", "{e}.")))?; + let (flame_layer, flame_guard) = tracing_flame::FlameLayer::with_file(&config.tracing_flame_output_path) .map_err(|e| err!(Config("tracing_flame_output_path", "{e}.")))?; + let flame_layer = flame_layer .with_empty_samples(false) .with_filter(flame_filter); + (Some(flame_layer), Some(flame_guard)) } else { (None, None) @@ -71,19 +79,24 @@ pub(crate) fn init( let jaeger_filter = EnvFilter::try_new(&config.jaeger_filter) .map_err(|e| err!(Config("jaeger_filter", "{e}.")))?; + let jaeger_layer = config.allow_jaeger.then(|| { opentelemetry::global::set_text_map_propagator( opentelemetry_jaeger::Propagator::new(), ); + let tracer = opentelemetry_jaeger::new_agent_pipeline() .with_auto_split_batch(true) .with_service_name(conduwuit_core::name()) .install_batch(opentelemetry_sdk::runtime::Tokio) .expect("jaeger agent pipeline"); + let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); + let (jaeger_reload_filter, jaeger_reload_handle) = reload::Layer::new(jaeger_filter.clone()); reload_handles.add("jaeger", Box::new(jaeger_reload_handle)); + Some(telemetry.with_filter(jaeger_reload_filter)) }); diff --git a/src/main/mods.rs b/src/main/mods.rs index d585a381..6140cc6e 100644 --- a/src/main/mods.rs +++ b/src/main/mods.rs @@ -51,7 +51,9 @@ pub(crate) async fn run(server: &Arc, starts: bool) -> Result<(bool, boo }, }; } + server.server.stopping.store(false, Ordering::Release); + let run = main_mod.get::("run")?; if let Err(error) = run(server .services @@ -64,7 +66,9 @@ pub(crate) async fn run(server: &Arc, starts: bool) -> Result<(bool, boo error!("Running server: {error}"); return Err(error); } + let reloads = server.server.reloading.swap(false, Ordering::AcqRel); + let stops = !reloads || stale(server).await? <= restart_thresh(); let starts = reloads && stops; if stops { diff --git a/src/main/sentry.rs b/src/main/sentry.rs index 68f12eb7..2a09f415 100644 --- a/src/main/sentry.rs +++ b/src/main/sentry.rs @@ -35,11 +35,13 @@ fn options(config: &Config) -> ClientOptions { .expect("init_sentry should only be called if sentry is enabled and this is not None") .as_str(); + let server_name = config + .sentry_send_server_name + .then(|| config.server_name.to_string().into()); + ClientOptions { dsn: Some(Dsn::from_str(dsn).expect("sentry_endpoint must be a valid URL")), - server_name: config - .sentry_send_server_name - .then(|| config.server_name.to_string().into()), + server_name, traces_sample_rate: config.sentry_traces_sample_rate, debug: cfg!(debug_assertions), release: sentry::release_name!(), diff --git a/src/router/request.rs b/src/router/request.rs index dba90324..3bbeae03 100644 --- a/src/router/request.rs +++ b/src/router/request.rs @@ -98,8 +98,8 @@ async fn execute( fn handle_result(method: &Method, uri: &Uri, result: Response) -> Result { let status = result.status(); - let reason = status.canonical_reason().unwrap_or("Unknown Reason"); let code = status.as_u16(); + let reason = status.canonical_reason().unwrap_or("Unknown Reason"); if status.is_server_error() { error!(method = ?method, uri = ?uri, "{code} {reason}"); diff --git a/src/service/admin/mod.rs b/src/service/admin/mod.rs index 66c373ec..d971ce95 100644 --- a/src/service/admin/mod.rs +++ b/src/service/admin/mod.rs @@ -305,13 +305,13 @@ impl Service { return Ok(()); }; - let response_sender = if self.is_admin_room(&pdu.room_id).await { + let response_sender = if self.is_admin_room(pdu.room_id()).await { &self.services.globals.server_user } else { - &pdu.sender + pdu.sender() }; - self.respond_to_room(content, &pdu.room_id, response_sender) + self.respond_to_room(content, pdu.room_id(), response_sender) .boxed() .await } diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 192ef447..baa7a72e 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -293,11 +293,7 @@ impl Service { .state_accessor .room_state_get(event.room_id(), &StateEventType::RoomPowerLevels, "") .await - .and_then(|ev| { - serde_json::from_str(ev.content.get()).map_err(|e| { - err!(Database(error!("invalid m.room.power_levels event: {e:?}"))) - }) - }) + .and_then(|event| event.get_content()) .unwrap_or_default(); let serialized = event.to_format(); diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 866e45a9..7675efd4 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -3,7 +3,7 @@ mod remote; use std::sync::Arc; use conduwuit::{ - Err, Result, Server, err, + Err, Event, Result, Server, err, utils::{ReadyExt, stream::TryIgnore}, }; use database::{Deserialized, Ignore, Interfix, Map}; @@ -241,7 +241,7 @@ impl Service { .room_state_get(&room_id, &StateEventType::RoomCreate, "") .await { - return Ok(event.sender == user_id); + return Ok(event.sender() == user_id); } Err!(Database("Room has no m.room.create event")) diff --git a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs index b0a7d827..44027e04 100644 --- a/src/service/rooms/event_handler/fetch_and_handle_outliers.rs +++ b/src/service/rooms/event_handler/fetch_and_handle_outliers.rs @@ -4,11 +4,13 @@ use std::{ }; use conduwuit::{ - PduEvent, debug, debug_error, debug_warn, implement, pdu, trace, - utils::continue_exponential_backoff_secs, warn, + Event, PduEvent, debug, debug_error, debug_warn, implement, + matrix::event::gen_event_id_canonical_json, trace, utils::continue_exponential_backoff_secs, + warn, }; use ruma::{ - CanonicalJsonValue, OwnedEventId, RoomId, ServerName, api::federation::event::get_event, + CanonicalJsonValue, EventId, OwnedEventId, RoomId, ServerName, + api::federation::event::get_event, }; use super::get_room_version_id; @@ -23,13 +25,17 @@ use super::get_room_version_id; /// c. Ask origin server over federation /// d. TODO: Ask other servers over federation? #[implement(super::Service)] -pub(super) async fn fetch_and_handle_outliers<'a>( +pub(super) async fn fetch_and_handle_outliers<'a, Pdu, Events>( &self, origin: &'a ServerName, - events: &'a [OwnedEventId], - create_event: &'a PduEvent, + events: Events, + create_event: &'a Pdu, room_id: &'a RoomId, -) -> Vec<(PduEvent, Option>)> { +) -> Vec<(PduEvent, Option>)> +where + Pdu: Event + Send + Sync, + Events: Iterator + Clone + Send, +{ let back_off = |id| match self .services .globals @@ -46,22 +52,23 @@ pub(super) async fn fetch_and_handle_outliers<'a>( }, }; - let mut events_with_auth_events = Vec::with_capacity(events.len()); + let mut events_with_auth_events = Vec::with_capacity(events.clone().count()); + for id in events { // a. Look in the main timeline (pduid_pdu tree) // b. Look at outlier pdu tree // (get_pdu_json checks both) if let Ok(local_pdu) = self.services.timeline.get_pdu(id).await { - trace!("Found {id} in db"); - events_with_auth_events.push((id, Some(local_pdu), vec![])); + events_with_auth_events.push((id.to_owned(), Some(local_pdu), vec![])); continue; } // c. Ask origin server over federation // We also handle its auth chain here so we don't get a stack overflow in // handle_outlier_pdu. - let mut todo_auth_events: VecDeque<_> = [id.clone()].into(); + let mut todo_auth_events: VecDeque<_> = [id.to_owned()].into(); let mut events_in_reverse_order = Vec::with_capacity(todo_auth_events.len()); + let mut events_all = HashSet::with_capacity(todo_auth_events.len()); while let Some(next_id) = todo_auth_events.pop_front() { if let Some((time, tries)) = self @@ -117,7 +124,7 @@ pub(super) async fn fetch_and_handle_outliers<'a>( }; let Ok((calculated_event_id, value)) = - pdu::gen_event_id_canonical_json(&res.pdu, &room_version_id) + gen_event_id_canonical_json(&res.pdu, &room_version_id) else { back_off((*next_id).to_owned()); continue; @@ -160,7 +167,8 @@ pub(super) async fn fetch_and_handle_outliers<'a>( }, } } - events_with_auth_events.push((id, None, events_in_reverse_order)); + + events_with_auth_events.push((id.to_owned(), None, events_in_reverse_order)); } let mut pdus = Vec::with_capacity(events_with_auth_events.len()); @@ -217,5 +225,6 @@ pub(super) async fn fetch_and_handle_outliers<'a>( } } } + pdus } diff --git a/src/service/rooms/event_handler/fetch_prev.rs b/src/service/rooms/event_handler/fetch_prev.rs index 0f92d6e6..efc7a434 100644 --- a/src/service/rooms/event_handler/fetch_prev.rs +++ b/src/service/rooms/event_handler/fetch_prev.rs @@ -1,13 +1,16 @@ -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::{ + collections::{BTreeMap, HashMap, HashSet, VecDeque}, + iter::once, +}; use conduwuit::{ - PduEvent, Result, debug_warn, err, implement, + Event, PduEvent, Result, debug_warn, err, implement, state_res::{self}, }; use futures::{FutureExt, future}; use ruma::{ - CanonicalJsonValue, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName, UInt, int, - uint, + CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, RoomId, ServerName, + int, uint, }; use super::check_room_id; @@ -19,20 +22,26 @@ use super::check_room_id; fields(%origin), )] #[allow(clippy::type_complexity)] -pub(super) async fn fetch_prev( +pub(super) async fn fetch_prev<'a, Pdu, Events>( &self, origin: &ServerName, - create_event: &PduEvent, + create_event: &Pdu, room_id: &RoomId, - first_ts_in_room: UInt, - initial_set: Vec, + first_ts_in_room: MilliSecondsSinceUnixEpoch, + initial_set: Events, ) -> Result<( Vec, HashMap)>, -)> { - let mut graph: HashMap = HashMap::with_capacity(initial_set.len()); +)> +where + Pdu: Event + Send + Sync, + Events: Iterator + Clone + Send, +{ + let num_ids = initial_set.clone().count(); let mut eventid_info = HashMap::new(); - let mut todo_outlier_stack: VecDeque = initial_set.into(); + let mut graph: HashMap = HashMap::with_capacity(num_ids); + let mut todo_outlier_stack: VecDeque = + initial_set.map(ToOwned::to_owned).collect(); let mut amount = 0; @@ -40,7 +49,12 @@ pub(super) async fn fetch_prev( self.services.server.check_running()?; match self - .fetch_and_handle_outliers(origin, &[prev_event_id.clone()], create_event, room_id) + .fetch_and_handle_outliers( + origin, + once(prev_event_id.as_ref()), + create_event, + room_id, + ) .boxed() .await .pop() @@ -65,17 +79,17 @@ pub(super) async fn fetch_prev( } if let Some(json) = json_opt { - if pdu.origin_server_ts > first_ts_in_room { + if pdu.origin_server_ts() > first_ts_in_room { amount = amount.saturating_add(1); - for prev_prev in &pdu.prev_events { + for prev_prev in pdu.prev_events() { if !graph.contains_key(prev_prev) { - todo_outlier_stack.push_back(prev_prev.clone()); + todo_outlier_stack.push_back(prev_prev.to_owned()); } } graph.insert( prev_event_id.clone(), - pdu.prev_events.iter().cloned().collect(), + pdu.prev_events().map(ToOwned::to_owned).collect(), ); } else { // Time based check failed @@ -98,8 +112,7 @@ pub(super) async fn fetch_prev( let event_fetch = |event_id| { let origin_server_ts = eventid_info .get(&event_id) - .cloned() - .map_or_else(|| uint!(0), |info| info.0.origin_server_ts); + .map_or_else(|| uint!(0), |info| info.0.origin_server_ts().get()); // This return value is the key used for sorting events, // events are then sorted by power level, time, diff --git a/src/service/rooms/event_handler/fetch_state.rs b/src/service/rooms/event_handler/fetch_state.rs index 0f9e093b..d68a3542 100644 --- a/src/service/rooms/event_handler/fetch_state.rs +++ b/src/service/rooms/event_handler/fetch_state.rs @@ -1,6 +1,6 @@ use std::collections::{HashMap, hash_map}; -use conduwuit::{Err, Error, PduEvent, Result, debug, debug_warn, implement}; +use conduwuit::{Err, Event, Result, debug, debug_warn, err, implement}; use futures::FutureExt; use ruma::{ EventId, OwnedEventId, RoomId, ServerName, api::federation::event::get_room_state_ids, @@ -18,13 +18,16 @@ use crate::rooms::short::ShortStateKey; skip_all, fields(%origin), )] -pub(super) async fn fetch_state( +pub(super) async fn fetch_state( &self, origin: &ServerName, - create_event: &PduEvent, + create_event: &Pdu, room_id: &RoomId, event_id: &EventId, -) -> Result>> { +) -> Result>> +where + Pdu: Event + Send + Sync, +{ let res = self .services .sending @@ -36,27 +39,27 @@ pub(super) async fn fetch_state( .inspect_err(|e| debug_warn!("Fetching state for event failed: {e}"))?; debug!("Fetching state events"); + let state_ids = res.pdu_ids.iter().map(AsRef::as_ref); let state_vec = self - .fetch_and_handle_outliers(origin, &res.pdu_ids, create_event, room_id) + .fetch_and_handle_outliers(origin, state_ids, create_event, room_id) .boxed() .await; let mut state: HashMap = HashMap::with_capacity(state_vec.len()); for (pdu, _) in state_vec { let state_key = pdu - .state_key - .clone() - .ok_or_else(|| Error::bad_database("Found non-state pdu in state events."))?; + .state_key() + .ok_or_else(|| err!(Database("Found non-state pdu in state events.")))?; let shortstatekey = self .services .short - .get_or_create_shortstatekey(&pdu.kind.to_string().into(), &state_key) + .get_or_create_shortstatekey(&pdu.kind().to_string().into(), state_key) .await; match state.entry(shortstatekey) { | hash_map::Entry::Vacant(v) => { - v.insert(pdu.event_id.clone()); + v.insert(pdu.event_id().to_owned()); }, | hash_map::Entry::Occupied(_) => { return Err!(Database( @@ -73,7 +76,7 @@ pub(super) async fn fetch_state( .get_shortstatekey(&StateEventType::RoomCreate, "") .await?; - if state.get(&create_shortstatekey) != Some(&create_event.event_id) { + if state.get(&create_shortstatekey).map(AsRef::as_ref) != Some(create_event.event_id()) { return Err!(Database("Incoming event refers to wrong create event.")); } diff --git a/src/service/rooms/event_handler/handle_incoming_pdu.rs b/src/service/rooms/event_handler/handle_incoming_pdu.rs index 77cae41d..86a05e0a 100644 --- a/src/service/rooms/event_handler/handle_incoming_pdu.rs +++ b/src/service/rooms/event_handler/handle_incoming_pdu.rs @@ -4,7 +4,7 @@ use std::{ }; use conduwuit::{ - Err, Result, debug, debug::INFO_SPAN_LEVEL, defer, err, implement, utils::stream::IterStream, + Err, Event, Result, debug::INFO_SPAN_LEVEL, defer, err, implement, utils::stream::IterStream, warn, }; use futures::{ @@ -12,6 +12,7 @@ use futures::{ future::{OptionFuture, try_join5}, }; use ruma::{CanonicalJsonValue, EventId, RoomId, ServerName, UserId, events::StateEventType}; +use tracing::debug; use crate::rooms::timeline::RawPduId; @@ -121,22 +122,16 @@ pub async fn handle_incoming_pdu<'a>( .timeline .first_pdu_in_room(room_id) .await? - .origin_server_ts; + .origin_server_ts(); - if incoming_pdu.origin_server_ts < first_ts_in_room { + if incoming_pdu.origin_server_ts() < first_ts_in_room { return Ok(None); } // 9. Fetch any missing prev events doing all checks listed here starting at 1. // These are timeline events let (sorted_prev_events, mut eventid_info) = self - .fetch_prev( - origin, - create_event, - room_id, - first_ts_in_room, - incoming_pdu.prev_events.clone(), - ) + .fetch_prev(origin, create_event, room_id, first_ts_in_room, incoming_pdu.prev_events()) .await?; debug!( diff --git a/src/service/rooms/event_handler/handle_outlier_pdu.rs b/src/service/rooms/event_handler/handle_outlier_pdu.rs index 5cc6be55..d79eed77 100644 --- a/src/service/rooms/event_handler/handle_outlier_pdu.rs +++ b/src/service/rooms/event_handler/handle_outlier_pdu.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, HashMap, hash_map}; use conduwuit::{ - Err, PduEvent, Result, debug, debug_info, err, implement, state_res, trace, warn, + Err, Event, PduEvent, Result, debug, debug_info, err, implement, state_res, trace, warn, }; use futures::future::ready; use ruma::{ @@ -12,15 +12,18 @@ use super::{check_room_id, get_room_version_id, to_room_version}; #[implement(super::Service)] #[allow(clippy::too_many_arguments)] -pub(super) async fn handle_outlier_pdu<'a>( +pub(super) async fn handle_outlier_pdu<'a, Pdu>( &self, origin: &'a ServerName, - create_event: &'a PduEvent, + create_event: &'a Pdu, event_id: &'a EventId, room_id: &'a RoomId, mut value: CanonicalJsonObject, auth_events_known: bool, -) -> Result<(PduEvent, BTreeMap)> { +) -> Result<(PduEvent, BTreeMap)> +where + Pdu: Event + Send + Sync, +{ // 1. Remove unsigned field value.remove("unsigned"); @@ -29,7 +32,7 @@ pub(super) async fn handle_outlier_pdu<'a>( // 2. Check signatures, otherwise drop // 3. check content hash, redact if doesn't match let room_version_id = get_room_version_id(create_event)?; - let mut val = match self + let mut incoming_pdu = match self .services .server_keys .verify_event(&value, Some(&room_version_id)) @@ -61,13 +64,15 @@ pub(super) async fn handle_outlier_pdu<'a>( // Now that we have checked the signature and hashes we can add the eventID and // convert to our PduEvent type - val.insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); - let incoming_pdu = serde_json::from_value::( - serde_json::to_value(&val).expect("CanonicalJsonObj is a valid JsonValue"), + incoming_pdu + .insert("event_id".to_owned(), CanonicalJsonValue::String(event_id.as_str().to_owned())); + + let pdu_event = serde_json::from_value::( + serde_json::to_value(&incoming_pdu).expect("CanonicalJsonObj is a valid JsonValue"), ) .map_err(|e| err!(Request(BadJson(debug_warn!("Event is not a valid PDU: {e}")))))?; - check_room_id(room_id, &incoming_pdu)?; + check_room_id(room_id, &pdu_event)?; if !auth_events_known { // 4. fetch any missing auth events doing all checks listed here starting at 1. @@ -78,7 +83,7 @@ pub(super) async fn handle_outlier_pdu<'a>( debug!("Fetching auth events"); Box::pin(self.fetch_and_handle_outliers( origin, - &incoming_pdu.auth_events, + pdu_event.auth_events(), create_event, room_id, )) @@ -89,8 +94,8 @@ pub(super) async fn handle_outlier_pdu<'a>( // auth events debug!("Checking based on auth events"); // Build map of auth events - let mut auth_events = HashMap::with_capacity(incoming_pdu.auth_events.len()); - for id in &incoming_pdu.auth_events { + let mut auth_events = HashMap::with_capacity(pdu_event.auth_events().count()); + for id in pdu_event.auth_events() { let Ok(auth_event) = self.services.timeline.get_pdu(id).await else { warn!("Could not find auth event {id}"); continue; @@ -131,7 +136,7 @@ pub(super) async fn handle_outlier_pdu<'a>( let auth_check = state_res::event_auth::auth_check( &to_room_version(&room_version_id), - &incoming_pdu, + &pdu_event, None, // TODO: third party invite state_fetch, ) @@ -147,9 +152,9 @@ pub(super) async fn handle_outlier_pdu<'a>( // 7. Persist the event as an outlier. self.services .outlier - .add_pdu_outlier(&incoming_pdu.event_id, &val); + .add_pdu_outlier(pdu_event.event_id(), &incoming_pdu); trace!("Added pdu as outlier."); - Ok((incoming_pdu, val)) + Ok((pdu_event, incoming_pdu)) } diff --git a/src/service/rooms/event_handler/handle_prev_pdu.rs b/src/service/rooms/event_handler/handle_prev_pdu.rs index d612b2bf..cd46310a 100644 --- a/src/service/rooms/event_handler/handle_prev_pdu.rs +++ b/src/service/rooms/event_handler/handle_prev_pdu.rs @@ -1,10 +1,11 @@ use std::{collections::BTreeMap, time::Instant}; use conduwuit::{ - Err, PduEvent, Result, debug, debug::INFO_SPAN_LEVEL, defer, implement, + Err, Event, PduEvent, Result, debug::INFO_SPAN_LEVEL, defer, implement, utils::continue_exponential_backoff_secs, }; -use ruma::{CanonicalJsonValue, EventId, RoomId, ServerName, UInt}; +use ruma::{CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, RoomId, ServerName}; +use tracing::debug; #[implement(super::Service)] #[allow(clippy::type_complexity)] @@ -15,16 +16,19 @@ use ruma::{CanonicalJsonValue, EventId, RoomId, ServerName, UInt}; skip_all, fields(%prev_id), )] -pub(super) async fn handle_prev_pdu<'a>( +pub(super) async fn handle_prev_pdu<'a, Pdu>( &self, origin: &'a ServerName, event_id: &'a EventId, room_id: &'a RoomId, eventid_info: Option<(PduEvent, BTreeMap)>, - create_event: &'a PduEvent, - first_ts_in_room: UInt, + create_event: &'a Pdu, + first_ts_in_room: MilliSecondsSinceUnixEpoch, prev_id: &'a EventId, -) -> Result { +) -> Result +where + Pdu: Event + Send + Sync, +{ // Check for disabled again because it might have changed if self.services.metadata.is_disabled(room_id).await { return Err!(Request(Forbidden(debug_warn!( @@ -59,7 +63,7 @@ pub(super) async fn handle_prev_pdu<'a>( }; // Skip old events - if pdu.origin_server_ts < first_ts_in_room { + if pdu.origin_server_ts() < first_ts_in_room { return Ok(()); } diff --git a/src/service/rooms/event_handler/mod.rs b/src/service/rooms/event_handler/mod.rs index 45675da8..aed38e1e 100644 --- a/src/service/rooms/event_handler/mod.rs +++ b/src/service/rooms/event_handler/mod.rs @@ -18,7 +18,7 @@ use std::{ }; use async_trait::async_trait; -use conduwuit::{Err, PduEvent, Result, RoomVersion, Server, utils::MutexMap}; +use conduwuit::{Err, Event, PduEvent, Result, RoomVersion, Server, utils::MutexMap}; use ruma::{ OwnedEventId, OwnedRoomId, RoomId, RoomVersionId, events::room::create::RoomCreateEventContent, @@ -104,11 +104,11 @@ impl Service { } } -fn check_room_id(room_id: &RoomId, pdu: &PduEvent) -> Result { - if pdu.room_id != room_id { +fn check_room_id(room_id: &RoomId, pdu: &Pdu) -> Result { + if pdu.room_id() != room_id { return Err!(Request(InvalidParam(error!( - pdu_event_id = ?pdu.event_id, - pdu_room_id = ?pdu.room_id, + pdu_event_id = ?pdu.event_id(), + pdu_room_id = ?pdu.room_id(), ?room_id, "Found event from room in room", )))); @@ -117,7 +117,7 @@ fn check_room_id(room_id: &RoomId, pdu: &PduEvent) -> Result { Ok(()) } -fn get_room_version_id(create_event: &PduEvent) -> Result { +fn get_room_version_id(create_event: &Pdu) -> Result { let content: RoomCreateEventContent = create_event.get_content()?; let room_version = content.room_version; diff --git a/src/service/rooms/event_handler/parse_incoming_pdu.rs b/src/service/rooms/event_handler/parse_incoming_pdu.rs index a49fc541..65cf1752 100644 --- a/src/service/rooms/event_handler/parse_incoming_pdu.rs +++ b/src/service/rooms/event_handler/parse_incoming_pdu.rs @@ -1,4 +1,6 @@ -use conduwuit::{Result, err, implement, pdu::gen_event_id_canonical_json, result::FlatOk}; +use conduwuit::{ + Result, err, implement, matrix::event::gen_event_id_canonical_json, result::FlatOk, +}; use ruma::{CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, OwnedRoomId}; use serde_json::value::RawValue as RawJsonValue; diff --git a/src/service/rooms/event_handler/state_at_incoming.rs b/src/service/rooms/event_handler/state_at_incoming.rs index eb38c2c3..d3bb8f79 100644 --- a/src/service/rooms/event_handler/state_at_incoming.rs +++ b/src/service/rooms/event_handler/state_at_incoming.rs @@ -6,7 +6,7 @@ use std::{ use conduwuit::{ Result, debug, err, implement, - matrix::{PduEvent, StateMap}, + matrix::{Event, StateMap}, trace, utils::stream::{BroadbandExt, IterStream, ReadyExt, TryBroadbandExt, TryWidebandExt}, }; @@ -19,11 +19,18 @@ use crate::rooms::short::ShortStateHash; #[implement(super::Service)] // request and build the state from a known point and resolve if > 1 prev_event #[tracing::instrument(name = "state", level = "debug", skip_all)] -pub(super) async fn state_at_incoming_degree_one( +pub(super) async fn state_at_incoming_degree_one( &self, - incoming_pdu: &PduEvent, -) -> Result>> { - let prev_event = &incoming_pdu.prev_events[0]; + incoming_pdu: &Pdu, +) -> Result>> +where + Pdu: Event + Send + Sync, +{ + let prev_event = incoming_pdu + .prev_events() + .next() + .expect("at least one prev_event"); + let Ok(prev_event_sstatehash) = self .services .state_accessor @@ -55,7 +62,7 @@ pub(super) async fn state_at_incoming_degree_one( .get_or_create_shortstatekey(&prev_pdu.kind.to_string().into(), state_key) .await; - state.insert(shortstatekey, prev_event.clone()); + state.insert(shortstatekey, prev_event.to_owned()); // Now it's the state after the pdu } @@ -66,16 +73,18 @@ pub(super) async fn state_at_incoming_degree_one( #[implement(super::Service)] #[tracing::instrument(name = "state", level = "debug", skip_all)] -pub(super) async fn state_at_incoming_resolved( +pub(super) async fn state_at_incoming_resolved( &self, - incoming_pdu: &PduEvent, + incoming_pdu: &Pdu, room_id: &RoomId, room_version_id: &RoomVersionId, -) -> Result>> { +) -> Result>> +where + Pdu: Event + Send + Sync, +{ trace!("Calculating extremity statehashes..."); let Ok(extremity_sstatehashes) = incoming_pdu - .prev_events - .iter() + .prev_events() .try_stream() .broad_and_then(|prev_eventid| { self.services @@ -133,12 +142,15 @@ pub(super) async fn state_at_incoming_resolved( } #[implement(super::Service)] -async fn state_at_incoming_fork( +async fn state_at_incoming_fork( &self, room_id: &RoomId, sstatehash: ShortStateHash, - prev_event: PduEvent, -) -> Result<(StateMap, HashSet)> { + prev_event: Pdu, +) -> Result<(StateMap, HashSet)> +where + Pdu: Event, +{ let mut leaf_state: HashMap<_, _> = self .services .state_accessor @@ -146,15 +158,15 @@ async fn state_at_incoming_fork( .collect() .await; - if let Some(state_key) = &prev_event.state_key { + if let Some(state_key) = prev_event.state_key() { let shortstatekey = self .services .short - .get_or_create_shortstatekey(&prev_event.kind.to_string().into(), state_key) + .get_or_create_shortstatekey(&prev_event.kind().to_string().into(), state_key) .await; - let event_id = &prev_event.event_id; - leaf_state.insert(shortstatekey, event_id.clone()); + let event_id = prev_event.event_id(); + leaf_state.insert(shortstatekey, event_id.to_owned()); // Now it's the state after the pdu } diff --git a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs index 00b18c06..4093cb05 100644 --- a/src/service/rooms/event_handler/upgrade_outlier_pdu.rs +++ b/src/service/rooms/event_handler/upgrade_outlier_pdu.rs @@ -1,7 +1,7 @@ use std::{borrow::Borrow, collections::BTreeMap, iter::once, sync::Arc, time::Instant}; use conduwuit::{ - Err, Result, debug, debug_info, err, implement, + Err, Result, debug, debug_info, err, implement, is_equal_to, matrix::{Event, EventTypeExt, PduEvent, StateKey, state_res}, trace, utils::stream::{BroadbandExt, ReadyExt}, @@ -17,19 +17,22 @@ use crate::rooms::{ }; #[implement(super::Service)] -pub(super) async fn upgrade_outlier_to_timeline_pdu( +pub(super) async fn upgrade_outlier_to_timeline_pdu( &self, incoming_pdu: PduEvent, val: BTreeMap, - create_event: &PduEvent, + create_event: &Pdu, origin: &ServerName, room_id: &RoomId, -) -> Result> { +) -> Result> +where + Pdu: Event + Send + Sync, +{ // Skip the PDU if we already have it as a timeline event if let Ok(pduid) = self .services .timeline - .get_pdu_id(&incoming_pdu.event_id) + .get_pdu_id(incoming_pdu.event_id()) .await { return Ok(Some(pduid)); @@ -38,7 +41,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( if self .services .pdu_metadata - .is_event_soft_failed(&incoming_pdu.event_id) + .is_event_soft_failed(incoming_pdu.event_id()) .await { return Err!(Request(InvalidParam("Event has been soft failed"))); @@ -53,7 +56,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( // These are not timeline events. debug!("Resolving state at event"); - let mut state_at_incoming_event = if incoming_pdu.prev_events.len() == 1 { + let mut state_at_incoming_event = if incoming_pdu.prev_events().count() == 1 { self.state_at_incoming_degree_one(&incoming_pdu).await? } else { self.state_at_incoming_resolved(&incoming_pdu, room_id, &room_version_id) @@ -62,12 +65,13 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( if state_at_incoming_event.is_none() { state_at_incoming_event = self - .fetch_state(origin, create_event, room_id, &incoming_pdu.event_id) + .fetch_state(origin, create_event, room_id, incoming_pdu.event_id()) .await?; } let state_at_incoming_event = state_at_incoming_event.expect("we always set this to some above"); + let room_version = to_room_version(&room_version_id); debug!("Performing auth check"); @@ -99,10 +103,10 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( .state .get_auth_events( room_id, - &incoming_pdu.kind, - &incoming_pdu.sender, - incoming_pdu.state_key.as_deref(), - &incoming_pdu.content, + incoming_pdu.kind(), + incoming_pdu.sender(), + incoming_pdu.state_key(), + incoming_pdu.content(), ) .await?; @@ -129,7 +133,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( !self .services .state_accessor - .user_can_redact(&redact_id, &incoming_pdu.sender, &incoming_pdu.room_id, true) + .user_can_redact(&redact_id, incoming_pdu.sender(), incoming_pdu.room_id(), true) .await?, }; @@ -149,7 +153,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( .map(ToOwned::to_owned) .ready_filter(|event_id| { // Remove any that are referenced by this incoming event's prev_events - !incoming_pdu.prev_events.contains(event_id) + !incoming_pdu.prev_events().any(is_equal_to!(event_id)) }) .broad_filter_map(|event_id| async move { // Only keep those extremities were not referenced yet @@ -166,7 +170,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( debug!( "Retained {} extremities checked against {} prev_events", extremities.len(), - incoming_pdu.prev_events.len() + incoming_pdu.prev_events().count() ); let state_ids_compressed: Arc = self @@ -181,20 +185,20 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( .map(Arc::new) .await; - if incoming_pdu.state_key.is_some() { + if incoming_pdu.state_key().is_some() { debug!("Event is a state-event. Deriving new room state"); // We also add state after incoming event to the fork states let mut state_after = state_at_incoming_event.clone(); - if let Some(state_key) = &incoming_pdu.state_key { + if let Some(state_key) = incoming_pdu.state_key() { let shortstatekey = self .services .short - .get_or_create_shortstatekey(&incoming_pdu.kind.to_string().into(), state_key) + .get_or_create_shortstatekey(&incoming_pdu.kind().to_string().into(), state_key) .await; - let event_id = &incoming_pdu.event_id; - state_after.insert(shortstatekey, event_id.clone()); + let event_id = incoming_pdu.event_id(); + state_after.insert(shortstatekey, event_id.to_owned()); } let new_room_state = self @@ -236,9 +240,9 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( // Soft fail, we keep the event as an outlier but don't add it to the timeline self.services .pdu_metadata - .mark_event_soft_failed(&incoming_pdu.event_id); + .mark_event_soft_failed(incoming_pdu.event_id()); - warn!("Event was soft failed: {incoming_pdu:?}"); + warn!("Event was soft failed: {:?}", incoming_pdu.event_id()); return Err!(Request(InvalidParam("Event has been soft failed"))); } @@ -249,7 +253,7 @@ pub(super) async fn upgrade_outlier_to_timeline_pdu( let extremities = extremities .iter() .map(Borrow::borrow) - .chain(once(incoming_pdu.event_id.borrow())); + .chain(once(incoming_pdu.event_id())); let pdu_id = self .services diff --git a/src/service/rooms/outlier/mod.rs b/src/service/rooms/outlier/mod.rs index 12b56935..6ab2c026 100644 --- a/src/service/rooms/outlier/mod.rs +++ b/src/service/rooms/outlier/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; -use conduwuit::{Result, implement, matrix::pdu::PduEvent}; -use conduwuit_database::{Deserialized, Json, Map}; +use conduwuit::{Result, implement, matrix::PduEvent}; +use database::{Deserialized, Json, Map}; use ruma::{CanonicalJsonObject, EventId}; pub struct Service { diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index f0beab5a..c1376cb0 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -1,8 +1,8 @@ use std::{mem::size_of, sync::Arc}; use conduwuit::{ - PduCount, PduEvent, arrayvec::ArrayVec, + matrix::{Event, PduCount}, result::LogErr, utils::{ ReadyExt, @@ -33,8 +33,6 @@ struct Services { timeline: Dep, } -pub(super) type PdusIterItem = (PduCount, PduEvent); - impl Data { pub(super) fn new(args: &crate::Args<'_>) -> Self { let db = &args.db; @@ -62,7 +60,7 @@ impl Data { target: ShortEventId, from: PduCount, dir: Direction, - ) -> impl Stream + Send + '_ { + ) -> impl Stream + Send + '_ { let mut current = ArrayVec::::new(); current.extend(target.to_be_bytes()); current.extend(from.saturating_inc(dir).into_unsigned().to_be_bytes()); @@ -80,8 +78,8 @@ impl Data { let mut pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await.ok()?; - if pdu.sender != user_id { - pdu.remove_transaction_id().log_err().ok(); + if pdu.sender() != user_id { + pdu.as_mut_pdu().remove_transaction_id().log_err().ok(); } Some((shorteventid, pdu)) diff --git a/src/service/rooms/pdu_metadata/mod.rs b/src/service/rooms/pdu_metadata/mod.rs index 18221c2d..c8e863fa 100644 --- a/src/service/rooms/pdu_metadata/mod.rs +++ b/src/service/rooms/pdu_metadata/mod.rs @@ -1,11 +1,14 @@ mod data; use std::sync::Arc; -use conduwuit::{PduCount, Result}; +use conduwuit::{ + Result, + matrix::{Event, PduCount}, +}; use futures::{StreamExt, future::try_join}; use ruma::{EventId, RoomId, UserId, api::Direction}; -use self::data::{Data, PdusIterItem}; +use self::data::Data; use crate::{Dep, rooms}; pub struct Service { @@ -44,16 +47,16 @@ impl Service { } #[allow(clippy::too_many_arguments)] - pub async fn get_relations( - &self, - user_id: &UserId, - room_id: &RoomId, - target: &EventId, + pub async fn get_relations<'a>( + &'a self, + user_id: &'a UserId, + room_id: &'a RoomId, + target: &'a EventId, from: PduCount, limit: usize, max_depth: u8, dir: Direction, - ) -> Vec { + ) -> Vec<(PduCount, impl Event)> { let room_id = self.services.short.get_shortroomid(room_id); let target = self.services.timeline.get_pdu_count(target); diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 69e859c4..68ce9b7f 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -4,7 +4,10 @@ use std::{collections::BTreeMap, sync::Arc}; use conduwuit::{ Result, debug, err, - matrix::pdu::{PduCount, PduId, RawPduId}, + matrix::{ + Event, + pdu::{PduCount, PduId, RawPduId}, + }, warn, }; use futures::{Stream, TryFutureExt, try_join}; @@ -74,14 +77,13 @@ impl Service { let shortroomid = self.services.short.get_shortroomid(room_id).map_err(|e| { err!(Database(warn!("Short room ID does not exist in database for {room_id}: {e}"))) }); - let (pdu_count, shortroomid) = try_join!(pdu_count, shortroomid)?; + let (pdu_count, shortroomid) = try_join!(pdu_count, shortroomid)?; let shorteventid = PduCount::Normal(pdu_count); let pdu_id: RawPduId = PduId { shortroomid, shorteventid }.into(); - let pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await?; - let event_id: OwnedEventId = pdu.event_id; + let event_id: OwnedEventId = pdu.event_id().to_owned(); let user_id: OwnedUserId = user_id.to_owned(); let content: BTreeMap = BTreeMap::from_iter([( event_id, diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index b9d067a6..afe3061b 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -1,9 +1,10 @@ use std::sync::Arc; -use conduwuit_core::{ - Event, PduCount, PduEvent, Result, +use conduwuit::{ + PduCount, Result, arrayvec::ArrayVec, implement, + matrix::event::{Event, Matches}, utils::{ ArrayVecExt, IterStream, ReadyExt, set, stream::{TryIgnore, WidebandExt}, @@ -103,9 +104,10 @@ pub fn deindex_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_b pub async fn search_pdus<'a>( &'a self, query: &'a RoomQuery<'a>, -) -> Result<(usize, impl Stream + Send + 'a)> { +) -> Result<(usize, impl Stream> + Send + '_)> { let pdu_ids: Vec<_> = self.search_pdu_ids(query).await?.collect().await; + let filter = &query.criteria.filter; let count = pdu_ids.len(); let pdus = pdu_ids .into_iter() @@ -118,11 +120,11 @@ pub async fn search_pdus<'a>( .ok() }) .ready_filter(|pdu| !pdu.is_redacted()) - .ready_filter(|pdu| pdu.matches(&query.criteria.filter)) + .ready_filter(move |pdu| filter.matches(pdu)) .wide_filter_map(move |pdu| async move { self.services .state_accessor - .user_can_see_event(query.user_id?, &pdu.room_id, &pdu.event_id) + .user_can_see_event(query.user_id?, pdu.room_id(), pdu.event_id()) .await .then_some(pdu) }) diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 9eb02221..641aa6a9 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -356,8 +356,8 @@ impl Service { &self, room_id: &RoomId, shortstatehash: u64, - _mutex_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room - * state mutex */ + // Take mutex guard to make sure users get the room state mutex + _mutex_lock: &RoomMutexGuard, ) { const BUFSIZE: usize = size_of::(); diff --git a/src/service/rooms/state_accessor/room_state.rs b/src/service/rooms/state_accessor/room_state.rs index 89fa2a83..89a66f0c 100644 --- a/src/service/rooms/state_accessor/room_state.rs +++ b/src/service/rooms/state_accessor/room_state.rs @@ -2,7 +2,7 @@ use std::borrow::Borrow; use conduwuit::{ Result, err, implement, - matrix::{PduEvent, StateKey}, + matrix::{Event, StateKey}, }; use futures::{Stream, StreamExt, TryFutureExt}; use ruma::{EventId, RoomId, events::StateEventType}; @@ -30,7 +30,7 @@ where pub fn room_state_full<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream> + Send + 'a { +) -> impl Stream> + Send + 'a { self.services .state .get_room_shortstatehash(room_id) @@ -45,7 +45,7 @@ pub fn room_state_full<'a>( pub fn room_state_full_pdus<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream> + Send + 'a { +) -> impl Stream> + Send + 'a { self.services .state .get_room_shortstatehash(room_id) @@ -84,7 +84,7 @@ pub async fn room_state_get( room_id: &RoomId, event_type: &StateEventType, state_key: &str, -) -> Result { +) -> Result { self.services .state .get_room_shortstatehash(room_id) diff --git a/src/service/rooms/state_accessor/state.rs b/src/service/rooms/state_accessor/state.rs index 169e69e9..a46ce380 100644 --- a/src/service/rooms/state_accessor/state.rs +++ b/src/service/rooms/state_accessor/state.rs @@ -2,14 +2,14 @@ use std::{borrow::Borrow, ops::Deref, sync::Arc}; use conduwuit::{ Result, at, err, implement, - matrix::{PduEvent, StateKey}, + matrix::{Event, StateKey}, pair_of, utils::{ result::FlatOk, stream::{BroadbandExt, IterStream, ReadyExt, TryIgnore}, }, }; -use conduwuit_database::Deserialized; +use database::Deserialized; use futures::{FutureExt, Stream, StreamExt, TryFutureExt, future::try_join, pin_mut}; use ruma::{ EventId, OwnedEventId, UserId, @@ -125,11 +125,9 @@ pub async fn state_get( shortstatehash: ShortStateHash, event_type: &StateEventType, state_key: &str, -) -> Result { +) -> Result { self.state_get_id(shortstatehash, event_type, state_key) - .and_then(|event_id: OwnedEventId| async move { - self.services.timeline.get_pdu(&event_id).await - }) + .and_then(async |event_id: OwnedEventId| self.services.timeline.get_pdu(&event_id).await) .await } @@ -316,18 +314,16 @@ pub fn state_added( pub fn state_full( &self, shortstatehash: ShortStateHash, -) -> impl Stream + Send + '_ { +) -> impl Stream + Send + '_ { self.state_full_pdus(shortstatehash) - .ready_filter_map(|pdu| { - Some(((pdu.kind.to_string().into(), pdu.state_key.clone()?), pdu)) - }) + .ready_filter_map(|pdu| Some(((pdu.kind().clone().into(), pdu.state_key()?.into()), pdu))) } #[implement(super::Service)] pub fn state_full_pdus( &self, shortstatehash: ShortStateHash, -) -> impl Stream + Send + '_ { +) -> impl Stream + Send + '_ { let short_ids = self .state_full_shortids(shortstatehash) .ignore_err() diff --git a/src/service/rooms/state_accessor/user_can.rs b/src/service/rooms/state_accessor/user_can.rs index 67e0b52b..221263a8 100644 --- a/src/service/rooms/state_accessor/user_can.rs +++ b/src/service/rooms/state_accessor/user_can.rs @@ -1,4 +1,4 @@ -use conduwuit::{Err, Result, implement, pdu::PduBuilder}; +use conduwuit::{Err, Result, implement, matrix::Event, pdu::PduBuilder}; use ruma::{ EventId, RoomId, UserId, events::{ @@ -29,14 +29,14 @@ pub async fn user_can_redact( if redacting_event .as_ref() - .is_ok_and(|pdu| pdu.kind == TimelineEventType::RoomCreate) + .is_ok_and(|pdu| *pdu.kind() == TimelineEventType::RoomCreate) { return Err!(Request(Forbidden("Redacting m.room.create is not safe, forbidding."))); } if redacting_event .as_ref() - .is_ok_and(|pdu| pdu.kind == TimelineEventType::RoomServerAcl) + .is_ok_and(|pdu| *pdu.kind() == TimelineEventType::RoomServerAcl) { return Err!(Request(Forbidden( "Redacting m.room.server_acl will result in the room being inaccessible for \ @@ -59,9 +59,9 @@ pub async fn user_can_redact( && match redacting_event { | Ok(redacting_event) => if federation { - redacting_event.sender.server_name() == sender.server_name() + redacting_event.sender().server_name() == sender.server_name() } else { - redacting_event.sender == sender + redacting_event.sender() == sender }, | _ => false, }) @@ -72,10 +72,10 @@ pub async fn user_can_redact( .room_state_get(room_id, &StateEventType::RoomCreate, "") .await { - | Ok(room_create) => Ok(room_create.sender == sender + | Ok(room_create) => Ok(room_create.sender() == sender || redacting_event .as_ref() - .is_ok_and(|redacting_event| redacting_event.sender == sender)), + .is_ok_and(|redacting_event| redacting_event.sender() == sender)), | _ => Err!(Database( "No m.room.power_levels or m.room.create events in database for room" )), diff --git a/src/service/rooms/threads/mod.rs b/src/service/rooms/threads/mod.rs index 9566eb61..59319ba6 100644 --- a/src/service/rooms/threads/mod.rs +++ b/src/service/rooms/threads/mod.rs @@ -49,10 +49,9 @@ impl crate::Service for Service { } impl Service { - pub async fn add_to_thread<'a, E>(&self, root_event_id: &EventId, event: &'a E) -> Result + pub async fn add_to_thread(&self, root_event_id: &EventId, event: &E) -> Result where E: Event + Send + Sync, - &'a E: Event + Send, { let root_id = self .services @@ -120,7 +119,7 @@ impl Service { self.services .timeline - .replace_pdu(&root_id, &root_pdu_json, &root_pdu) + .replace_pdu(&root_id, &root_pdu_json) .await?; } @@ -130,7 +129,7 @@ impl Service { users.extend_from_slice(&userids); }, | _ => { - users.push(root_pdu.sender); + users.push(root_pdu.sender().to_owned()); }, } users.push(event.sender().to_owned()); @@ -162,10 +161,10 @@ impl Service { .ready_take_while(move |pdu_id| pdu_id.shortroomid() == shortroomid.to_be_bytes()) .wide_filter_map(move |pdu_id| async move { let mut pdu = self.services.timeline.get_pdu_from_id(&pdu_id).await.ok()?; - let pdu_id: PduId = pdu_id.into(); - if pdu.sender != user_id { - pdu.remove_transaction_id().ok(); + let pdu_id: PduId = pdu_id.into(); + if pdu.sender() != user_id { + pdu.as_mut_pdu().remove_transaction_id().ok(); } Some((pdu_id.shorteventid, pdu)) diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index 94c78bb0..fa10a5c0 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -207,7 +207,6 @@ impl Data { &self, pdu_id: &RawPduId, pdu_json: &CanonicalJsonObject, - _pdu: &PduEvent, ) -> Result { if self.pduid_pdu.get(pdu_id).await.is_not_found() { return Err!(Request(NotFound("PDU does not exist."))); diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index bcad1309..a381fcf6 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -14,8 +14,8 @@ pub use conduwuit::matrix::pdu::{PduId, RawPduId}; use conduwuit::{ Err, Error, Result, Server, at, debug, debug_warn, err, error, implement, info, matrix::{ - Event, - pdu::{EventHash, PduBuilder, PduCount, PduEvent, gen_event_id}, + event::{Event, gen_event_id}, + pdu::{EventHash, PduBuilder, PduCount, PduEvent}, state_res::{self, RoomVersion}, }, utils::{ @@ -159,12 +159,12 @@ impl crate::Service for Service { impl Service { #[tracing::instrument(skip(self), level = "debug")] - pub async fn first_pdu_in_room(&self, room_id: &RoomId) -> Result { + pub async fn first_pdu_in_room(&self, room_id: &RoomId) -> Result { self.first_item_in_room(room_id).await.map(at!(1)) } #[tracing::instrument(skip(self), level = "debug")] - pub async fn first_item_in_room(&self, room_id: &RoomId) -> Result<(PduCount, PduEvent)> { + pub async fn first_item_in_room(&self, room_id: &RoomId) -> Result<(PduCount, impl Event)> { let pdus = self.pdus(None, room_id, None); pin_mut!(pdus); @@ -174,7 +174,7 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub async fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result { + pub async fn latest_pdu_in_room(&self, room_id: &RoomId) -> Result { self.db.latest_pdu_in_room(None, room_id).await } @@ -216,13 +216,14 @@ impl Service { /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. #[inline] - pub async fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result { + pub async fn get_non_outlier_pdu(&self, event_id: &EventId) -> Result { self.db.get_non_outlier_pdu(event_id).await } /// Returns the pdu. /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. + #[inline] pub async fn get_pdu(&self, event_id: &EventId) -> Result { self.db.get_pdu(event_id).await } @@ -230,11 +231,13 @@ impl Service { /// Returns the pdu. /// /// This does __NOT__ check the outliers `Tree`. + #[inline] pub async fn get_pdu_from_id(&self, pdu_id: &RawPduId) -> Result { self.db.get_pdu_from_id(pdu_id).await } /// Returns the pdu as a `BTreeMap`. + #[inline] pub async fn get_pdu_json_from_id(&self, pdu_id: &RawPduId) -> Result { self.db.get_pdu_json_from_id(pdu_id).await } @@ -242,6 +245,7 @@ impl Service { /// Checks if pdu exists /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. + #[inline] pub fn pdu_exists<'a>( &'a self, event_id: &'a EventId, @@ -251,13 +255,8 @@ impl Service { /// Removes a pdu and creates a new one with the same id. #[tracing::instrument(skip(self), level = "debug")] - pub async fn replace_pdu( - &self, - pdu_id: &RawPduId, - pdu_json: &CanonicalJsonObject, - pdu: &PduEvent, - ) -> Result<()> { - self.db.replace_pdu(pdu_id, pdu_json, pdu).await + pub async fn replace_pdu(&self, pdu_id: &RawPduId, pdu_json: &CanonicalJsonObject) -> Result { + self.db.replace_pdu(pdu_id, pdu_json).await } /// Creates a new persisted data unit and adds it to a room. @@ -310,25 +309,21 @@ impl Service { unsigned.insert( "prev_content".to_owned(), CanonicalJsonValue::Object( - utils::to_canonical_object(prev_state.content.clone()).map_err( - |e| { - error!( - "Failed to convert prev_state to canonical JSON: {e}" - ); - Error::bad_database( - "Failed to convert prev_state to canonical JSON.", - ) - }, - )?, + utils::to_canonical_object(prev_state.get_content_as_value()) + .map_err(|e| { + err!(Database(error!( + "Failed to convert prev_state to canonical JSON: {e}", + ))) + })?, ), ); unsigned.insert( String::from("prev_sender"), - CanonicalJsonValue::String(prev_state.sender.to_string()), + CanonicalJsonValue::String(prev_state.sender().to_string()), ); unsigned.insert( String::from("replaces_state"), - CanonicalJsonValue::String(prev_state.event_id.to_string()), + CanonicalJsonValue::String(prev_state.event_id().to_string()), ); } } @@ -709,14 +704,11 @@ impl Service { .await { unsigned.insert("prev_content".to_owned(), prev_pdu.get_content_as_value()); - unsigned.insert( - "prev_sender".to_owned(), - serde_json::to_value(&prev_pdu.sender) - .expect("UserId::to_value always works"), - ); + unsigned + .insert("prev_sender".to_owned(), serde_json::to_value(prev_pdu.sender())?); unsigned.insert( "replaces_state".to_owned(), - serde_json::to_value(&prev_pdu.event_id).expect("EventId is valid json"), + serde_json::to_value(prev_pdu.event_id())?, ); } } @@ -759,7 +751,7 @@ impl Service { unsigned: if unsigned.is_empty() { None } else { - Some(to_raw_value(&unsigned).expect("to_raw_value always works")) + Some(to_raw_value(&unsigned)?) }, hashes: EventHash { sha256: "aaa".to_owned() }, signatures: None, @@ -1041,10 +1033,10 @@ impl Service { /// Replace a PDU with the redacted form. #[tracing::instrument(name = "redact", level = "debug", skip(self))] - pub async fn redact_pdu( + pub async fn redact_pdu( &self, event_id: &EventId, - reason: &PduEvent, + reason: &Pdu, shortroomid: ShortRoomId, ) -> Result { // TODO: Don't reserialize, keep original json @@ -1053,9 +1045,13 @@ impl Service { return Ok(()); }; - let mut pdu = self.get_pdu_from_id(&pdu_id).await.map_err(|e| { - err!(Database(error!(?pdu_id, ?event_id, ?e, "PDU ID points to invalid PDU."))) - })?; + let mut pdu = self + .get_pdu_from_id(&pdu_id) + .await + .map(Event::into_pdu) + .map_err(|e| { + err!(Database(error!(?pdu_id, ?event_id, ?e, "PDU ID points to invalid PDU."))) + })?; if let Ok(content) = pdu.get_content::() { if let Some(body) = content.body { @@ -1065,15 +1061,15 @@ impl Service { } } - let room_version_id = self.services.state.get_room_version(&pdu.room_id).await?; + let room_version_id = self.services.state.get_room_version(pdu.room_id()).await?; - pdu.redact(&room_version_id, reason)?; + pdu.redact(&room_version_id, reason.to_value())?; let obj = utils::to_canonical_object(&pdu).map_err(|e| { err!(Database(error!(?event_id, ?e, "Failed to convert PDU to canonical JSON"))) })?; - self.replace_pdu(&pdu_id, &obj, &pdu).await + self.replace_pdu(&pdu_id, &obj).await } #[tracing::instrument(name = "backfill", level = "debug", skip(self))] @@ -1163,7 +1159,7 @@ impl Service { backfill_server, federation::backfill::get_backfill::v1::Request { room_id: room_id.to_owned(), - v: vec![first_pdu.1.event_id.clone()], + v: vec![first_pdu.1.event_id().to_owned()], limit: uint!(100), }, ) @@ -1248,8 +1244,11 @@ impl Service { #[implement(Service)] #[tracing::instrument(skip_all, level = "debug")] -async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Result<()> { - match &pdu.kind { +async fn check_pdu_for_admin_room(&self, pdu: &Pdu, sender: &UserId) -> Result +where + Pdu: Event + Send + Sync, +{ + match pdu.kind() { | TimelineEventType::RoomEncryption => { return Err!(Request(Forbidden(error!("Encryption not supported in admins room.")))); }, @@ -1273,7 +1272,7 @@ async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Res let count = self .services .state_cache - .room_members(&pdu.room_id) + .room_members(pdu.room_id()) .ready_filter(|user| self.services.globals.user_is_local(user)) .ready_filter(|user| *user != target) .boxed() @@ -1297,7 +1296,7 @@ async fn check_pdu_for_admin_room(&self, pdu: &PduEvent, sender: &UserId) -> Res let count = self .services .state_cache - .room_members(&pdu.room_id) + .room_members(pdu.room_id()) .ready_filter(|user| self.services.globals.user_is_local(user)) .ready_filter(|user| *user != target) .boxed() diff --git a/src/service/sending/sender.rs b/src/service/sending/sender.rs index 408ab17d..a708f746 100644 --- a/src/service/sending/sender.rs +++ b/src/service/sending/sender.rs @@ -798,7 +798,7 @@ impl Service { let unread: UInt = self .services .user - .notification_count(&user_id, &pdu.room_id) + .notification_count(&user_id, pdu.room_id()) .await .try_into() .expect("notification count can't go that high"); diff --git a/src/service/server_keys/verify.rs b/src/service/server_keys/verify.rs index 84433628..9cc3655a 100644 --- a/src/service/server_keys/verify.rs +++ b/src/service/server_keys/verify.rs @@ -1,4 +1,4 @@ -use conduwuit::{Err, Result, implement, pdu::gen_event_id_canonical_json}; +use conduwuit::{Err, Result, implement, matrix::event::gen_event_id_canonical_json}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, RoomVersionId, signatures::Verified, }; From c06aa49a903a18c87e709a72128662a707cc79ec Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 13 May 2025 21:33:07 +0000 Subject: [PATCH 1890/2091] Fix regression 75aadd5c6a Signed-off-by: Jason Volk --- src/api/client/user_directory.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/api/client/user_directory.rs b/src/api/client/user_directory.rs index 748fc049..9a1f86b8 100644 --- a/src/api/client/user_directory.rs +++ b/src/api/client/user_directory.rs @@ -1,10 +1,7 @@ use axum::extract::State; use conduwuit::{ Result, - utils::{ - future::BoolExt, - stream::{BroadbandExt, ReadyExt}, - }, + utils::{future::BoolExt, stream::BroadbandExt}, }; use futures::{FutureExt, StreamExt, pin_mut}; use ruma::{ @@ -37,17 +34,18 @@ pub(crate) async fn search_users_route( let mut users = services .users .stream() - .ready_filter(|user_id| user_id.as_str().to_lowercase().contains(&search_term)) .map(ToOwned::to_owned) .broad_filter_map(async |user_id| { let display_name = services.users.displayname(&user_id).await.ok(); + let user_id_matches = user_id.as_str().to_lowercase().contains(&search_term); + let display_name_matches = display_name .as_deref() .map(str::to_lowercase) .is_some_and(|display_name| display_name.contains(&search_term)); - if !display_name_matches { + if !user_id_matches && !display_name_matches { return None; } From c5c309ec4301a3d38bda4a8978a499ffa245f834 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 29 Apr 2025 06:39:30 +0000 Subject: [PATCH 1891/2091] Split timeline service. Signed-off-by: Jason Volk --- src/api/client/room/initial_sync.rs | 2 +- src/service/rooms/timeline/append.rs | 446 ++++++++++ src/service/rooms/timeline/backfill.rs | 191 +++++ src/service/rooms/timeline/build.rs | 226 +++++ src/service/rooms/timeline/create.rs | 214 +++++ src/service/rooms/timeline/mod.rs | 1084 +----------------------- src/service/rooms/timeline/redact.rs | 51 ++ 7 files changed, 1146 insertions(+), 1068 deletions(-) create mode 100644 src/service/rooms/timeline/append.rs create mode 100644 src/service/rooms/timeline/backfill.rs create mode 100644 src/service/rooms/timeline/build.rs create mode 100644 src/service/rooms/timeline/create.rs create mode 100644 src/service/rooms/timeline/redact.rs diff --git a/src/api/client/room/initial_sync.rs b/src/api/client/room/initial_sync.rs index 2aca5b9d..d40f6b4f 100644 --- a/src/api/client/room/initial_sync.rs +++ b/src/api/client/room/initial_sync.rs @@ -1,7 +1,7 @@ use axum::extract::State; use conduwuit::{ Err, Event, Result, at, - utils::{BoolExt, future::TryExtExt, stream::TryTools}, + utils::{BoolExt, stream::TryTools}, }; use futures::{FutureExt, TryStreamExt, future::try_join4}; use ruma::api::client::room::initial_sync::v3::{PaginationChunk, Request, Response}; diff --git a/src/service/rooms/timeline/append.rs b/src/service/rooms/timeline/append.rs new file mode 100644 index 00000000..a7b558c2 --- /dev/null +++ b/src/service/rooms/timeline/append.rs @@ -0,0 +1,446 @@ +use std::{ + collections::{BTreeMap, HashSet}, + sync::Arc, +}; + +use conduwuit_core::{ + Result, err, error, implement, + matrix::{ + event::Event, + pdu::{PduCount, PduEvent, PduId, RawPduId}, + }, + utils::{self, ReadyExt}, +}; +use futures::StreamExt; +use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, EventId, RoomVersionId, UserId, + events::{ + GlobalAccountDataEventType, StateEventType, TimelineEventType, + push_rules::PushRulesEvent, + room::{ + encrypted::Relation, + member::{MembershipState, RoomMemberEventContent}, + power_levels::RoomPowerLevelsEventContent, + redaction::RoomRedactionEventContent, + }, + }, + push::{Action, Ruleset, Tweak}, +}; + +use super::{ExtractBody, ExtractRelatesTo, ExtractRelatesToEventId, RoomMutexGuard}; +use crate::{appservice::NamespaceRegex, rooms::state_compressor::CompressedState}; + +/// Append the incoming event setting the state snapshot to the state from +/// the server that sent the event. +#[implement(super::Service)] +#[tracing::instrument(level = "debug", skip_all)] +pub async fn append_incoming_pdu<'a, Leaves>( + &'a self, + pdu: &'a PduEvent, + pdu_json: CanonicalJsonObject, + new_room_leaves: Leaves, + state_ids_compressed: Arc, + soft_fail: bool, + state_lock: &'a RoomMutexGuard, +) -> Result> +where + Leaves: Iterator + Send + 'a, +{ + // We append to state before appending the pdu, so we don't have a moment in + // time with the pdu without it's state. This is okay because append_pdu can't + // fail. + self.services + .state + .set_event_state(&pdu.event_id, &pdu.room_id, state_ids_compressed) + .await?; + + if soft_fail { + self.services + .pdu_metadata + .mark_as_referenced(&pdu.room_id, pdu.prev_events.iter().map(AsRef::as_ref)); + + self.services + .state + .set_forward_extremities(&pdu.room_id, new_room_leaves, state_lock) + .await; + + return Ok(None); + } + + let pdu_id = self + .append_pdu(pdu, pdu_json, new_room_leaves, state_lock) + .await?; + + Ok(Some(pdu_id)) +} + +/// Creates a new persisted data unit and adds it to a room. +/// +/// By this point the incoming event should be fully authenticated, no auth +/// happens in `append_pdu`. +/// +/// Returns pdu id +#[implement(super::Service)] +#[tracing::instrument(level = "debug", skip_all)] +pub async fn append_pdu<'a, Leaves>( + &'a self, + pdu: &'a PduEvent, + mut pdu_json: CanonicalJsonObject, + leaves: Leaves, + state_lock: &'a RoomMutexGuard, +) -> Result +where + Leaves: Iterator + Send + 'a, +{ + // Coalesce database writes for the remainder of this scope. + let _cork = self.db.db.cork_and_flush(); + + let shortroomid = self + .services + .short + .get_shortroomid(pdu.room_id()) + .await + .map_err(|_| err!(Database("Room does not exist")))?; + + // Make unsigned fields correct. This is not properly documented in the spec, + // but state events need to have previous content in the unsigned field, so + // clients can easily interpret things like membership changes + if let Some(state_key) = pdu.state_key() { + if let CanonicalJsonValue::Object(unsigned) = pdu_json + .entry("unsigned".to_owned()) + .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::default())) + { + if let Ok(shortstatehash) = self + .services + .state_accessor + .pdu_shortstatehash(pdu.event_id()) + .await + { + if let Ok(prev_state) = self + .services + .state_accessor + .state_get(shortstatehash, &pdu.kind().to_string().into(), state_key) + .await + { + unsigned.insert( + "prev_content".to_owned(), + CanonicalJsonValue::Object( + utils::to_canonical_object(prev_state.get_content_as_value()) + .map_err(|e| { + err!(Database(error!( + "Failed to convert prev_state to canonical JSON: {e}", + ))) + })?, + ), + ); + unsigned.insert( + String::from("prev_sender"), + CanonicalJsonValue::String(prev_state.sender().to_string()), + ); + unsigned.insert( + String::from("replaces_state"), + CanonicalJsonValue::String(prev_state.event_id().to_string()), + ); + } + } + } else { + error!("Invalid unsigned type in pdu."); + } + } + + // We must keep track of all events that have been referenced. + self.services + .pdu_metadata + .mark_as_referenced(pdu.room_id(), pdu.prev_events().map(AsRef::as_ref)); + + self.services + .state + .set_forward_extremities(pdu.room_id(), leaves, state_lock) + .await; + + let insert_lock = self.mutex_insert.lock(pdu.room_id()).await; + + let count1 = self.services.globals.next_count().unwrap(); + + // Mark as read first so the sending client doesn't get a notification even if + // appending fails + self.services + .read_receipt + .private_read_set(pdu.room_id(), pdu.sender(), count1); + + self.services + .user + .reset_notification_counts(pdu.sender(), pdu.room_id()); + + let count2 = PduCount::Normal(self.services.globals.next_count().unwrap()); + let pdu_id: RawPduId = PduId { shortroomid, shorteventid: count2 }.into(); + + // Insert pdu + self.db.append_pdu(&pdu_id, pdu, &pdu_json, count2).await; + + drop(insert_lock); + + // See if the event matches any known pushers via power level + let power_levels: RoomPowerLevelsEventContent = self + .services + .state_accessor + .room_state_get_content(pdu.room_id(), &StateEventType::RoomPowerLevels, "") + .await + .unwrap_or_default(); + + let mut push_target: HashSet<_> = self + .services + .state_cache + .active_local_users_in_room(pdu.room_id()) + .map(ToOwned::to_owned) + // Don't notify the sender of their own events, and dont send from ignored users + .ready_filter(|user| *user != pdu.sender()) + .filter_map(|recipient_user| async move { (!self.services.users.user_is_ignored(pdu.sender(), &recipient_user).await).then_some(recipient_user) }) + .collect() + .await; + + let mut notifies = Vec::with_capacity(push_target.len().saturating_add(1)); + let mut highlights = Vec::with_capacity(push_target.len().saturating_add(1)); + + if *pdu.kind() == TimelineEventType::RoomMember { + if let Some(state_key) = pdu.state_key() { + let target_user_id = UserId::parse(state_key)?; + + if self.services.users.is_active_local(target_user_id).await { + push_target.insert(target_user_id.to_owned()); + } + } + } + + let serialized = pdu.to_format(); + for user in &push_target { + let rules_for_user = self + .services + .account_data + .get_global(user, GlobalAccountDataEventType::PushRules) + .await + .map_or_else( + |_| Ruleset::server_default(user), + |ev: PushRulesEvent| ev.content.global, + ); + + let mut highlight = false; + let mut notify = false; + + for action in self + .services + .pusher + .get_actions(user, &rules_for_user, &power_levels, &serialized, pdu.room_id()) + .await + { + match action { + | Action::Notify => notify = true, + | Action::SetTweak(Tweak::Highlight(true)) => { + highlight = true; + }, + | _ => {}, + } + + // Break early if both conditions are true + if notify && highlight { + break; + } + } + + if notify { + notifies.push(user.clone()); + } + + if highlight { + highlights.push(user.clone()); + } + + self.services + .pusher + .get_pushkeys(user) + .ready_for_each(|push_key| { + self.services + .sending + .send_pdu_push(&pdu_id, user, push_key.to_owned()) + .expect("TODO: replace with future"); + }) + .await; + } + + self.db + .increment_notification_counts(pdu.room_id(), notifies, highlights); + + match *pdu.kind() { + | TimelineEventType::RoomRedaction => { + use RoomVersionId::*; + + let room_version_id = self.services.state.get_room_version(pdu.room_id()).await?; + match room_version_id { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + if let Some(redact_id) = pdu.redacts() { + if self + .services + .state_accessor + .user_can_redact(redact_id, pdu.sender(), pdu.room_id(), false) + .await? + { + self.redact_pdu(redact_id, pdu, shortroomid).await?; + } + } + }, + | _ => { + let content: RoomRedactionEventContent = pdu.get_content()?; + if let Some(redact_id) = &content.redacts { + if self + .services + .state_accessor + .user_can_redact(redact_id, pdu.sender(), pdu.room_id(), false) + .await? + { + self.redact_pdu(redact_id, pdu, shortroomid).await?; + } + } + }, + } + }, + | TimelineEventType::SpaceChild => + if let Some(_state_key) = pdu.state_key() { + self.services + .spaces + .roomid_spacehierarchy_cache + .lock() + .await + .remove(pdu.room_id()); + }, + | TimelineEventType::RoomMember => { + if let Some(state_key) = pdu.state_key() { + // if the state_key fails + let target_user_id = + UserId::parse(state_key).expect("This state_key was previously validated"); + + let content: RoomMemberEventContent = pdu.get_content()?; + let stripped_state = match content.membership { + | MembershipState::Invite | MembershipState::Knock => + self.services.state.summary_stripped(pdu).await.into(), + | _ => None, + }; + + // Update our membership info, we do this here incase a user is invited or + // knocked and immediately leaves we need the DB to record the invite or + // knock event for auth + self.services + .state_cache + .update_membership( + pdu.room_id(), + target_user_id, + content, + pdu.sender(), + stripped_state, + None, + true, + ) + .await?; + } + }, + | TimelineEventType::RoomMessage => { + let content: ExtractBody = pdu.get_content()?; + if let Some(body) = content.body { + self.services.search.index_pdu(shortroomid, &pdu_id, &body); + + if self.services.admin.is_admin_command(pdu, &body).await { + self.services + .admin + .command_with_sender(body, Some((pdu.event_id()).into()), pdu.sender.clone().into())?; + } + } + }, + | _ => {}, + } + + if let Ok(content) = pdu.get_content::() { + if let Ok(related_pducount) = self.get_pdu_count(&content.relates_to.event_id).await { + self.services + .pdu_metadata + .add_relation(count2, related_pducount); + } + } + + if let Ok(content) = pdu.get_content::() { + match content.relates_to { + | Relation::Reply { in_reply_to } => { + // We need to do it again here, because replies don't have + // event_id as a top level field + if let Ok(related_pducount) = self.get_pdu_count(&in_reply_to.event_id).await { + self.services + .pdu_metadata + .add_relation(count2, related_pducount); + } + }, + | Relation::Thread(thread) => { + self.services + .threads + .add_to_thread(&thread.event_id, pdu) + .await?; + }, + | _ => {}, // TODO: Aggregate other types + } + } + + for appservice in self.services.appservice.read().await.values() { + if self + .services + .state_cache + .appservice_in_room(pdu.room_id(), appservice) + .await + { + self.services + .sending + .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; + continue; + } + + // If the RoomMember event has a non-empty state_key, it is targeted at someone. + // If it is our appservice user, we send this PDU to it. + if *pdu.kind() == TimelineEventType::RoomMember { + if let Some(state_key_uid) = &pdu + .state_key + .as_ref() + .and_then(|state_key| UserId::parse(state_key.as_str()).ok()) + { + let appservice_uid = appservice.registration.sender_localpart.as_str(); + if state_key_uid == &appservice_uid { + self.services + .sending + .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; + continue; + } + } + } + + let matching_users = |users: &NamespaceRegex| { + appservice.users.is_match(pdu.sender().as_str()) + || *pdu.kind() == TimelineEventType::RoomMember + && pdu + .state_key + .as_ref() + .is_some_and(|state_key| users.is_match(state_key)) + }; + let matching_aliases = |aliases: NamespaceRegex| { + self.services + .alias + .local_aliases_for_room(pdu.room_id()) + .ready_any(move |room_alias| aliases.is_match(room_alias.as_str())) + }; + + if matching_aliases(appservice.aliases.clone()).await + || appservice.rooms.is_match(pdu.room_id().as_str()) + || matching_users(&appservice.users) + { + self.services + .sending + .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; + } + } + + Ok(pdu_id) +} diff --git a/src/service/rooms/timeline/backfill.rs b/src/service/rooms/timeline/backfill.rs new file mode 100644 index 00000000..e976981e --- /dev/null +++ b/src/service/rooms/timeline/backfill.rs @@ -0,0 +1,191 @@ +use std::iter::once; + +use conduwuit_core::{ + Result, debug, debug_warn, implement, info, + matrix::{ + event::Event, + pdu::{PduCount, PduId, RawPduId}, + }, + utils::{IterStream, ReadyExt}, + validated, warn, +}; +use futures::{FutureExt, StreamExt}; +use ruma::{ + RoomId, ServerName, + api::federation, + events::{ + StateEventType, TimelineEventType, room::power_levels::RoomPowerLevelsEventContent, + }, + uint, +}; +use serde_json::value::RawValue as RawJsonValue; + +use super::ExtractBody; + +#[implement(super::Service)] +#[tracing::instrument(name = "backfill", level = "debug", skip(self))] +pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { + if self + .services + .state_cache + .room_joined_count(room_id) + .await + .is_ok_and(|count| count <= 1) + && !self + .services + .state_accessor + .is_world_readable(room_id) + .await + { + // Room is empty (1 user or none), there is no one that can backfill + return Ok(()); + } + + let first_pdu = self + .first_item_in_room(room_id) + .await + .expect("Room is not empty"); + + if first_pdu.0 < from { + // No backfill required, there are still events between them + return Ok(()); + } + + let power_levels: RoomPowerLevelsEventContent = self + .services + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomPowerLevels, "") + .await + .unwrap_or_default(); + + let room_mods = power_levels.users.iter().filter_map(|(user_id, level)| { + if level > &power_levels.users_default && !self.services.globals.user_is_local(user_id) { + Some(user_id.server_name()) + } else { + None + } + }); + + let canonical_room_alias_server = once( + self.services + .state_accessor + .get_canonical_alias(room_id) + .await, + ) + .filter_map(Result::ok) + .map(|alias| alias.server_name().to_owned()) + .stream(); + + let mut servers = room_mods + .stream() + .map(ToOwned::to_owned) + .chain(canonical_room_alias_server) + .chain( + self.services + .server + .config + .trusted_servers + .iter() + .map(ToOwned::to_owned) + .stream(), + ) + .ready_filter(|server_name| !self.services.globals.server_is_ours(server_name)) + .filter_map(|server_name| async move { + self.services + .state_cache + .server_in_room(&server_name, room_id) + .await + .then_some(server_name) + }) + .boxed(); + + while let Some(ref backfill_server) = servers.next().await { + info!("Asking {backfill_server} for backfill"); + let response = self + .services + .sending + .send_federation_request( + backfill_server, + federation::backfill::get_backfill::v1::Request { + room_id: room_id.to_owned(), + v: vec![first_pdu.1.event_id().to_owned()], + limit: uint!(100), + }, + ) + .await; + match response { + | Ok(response) => { + for pdu in response.pdus { + if let Err(e) = self.backfill_pdu(backfill_server, pdu).boxed().await { + debug_warn!("Failed to add backfilled pdu in room {room_id}: {e}"); + } + } + return Ok(()); + }, + | Err(e) => { + warn!("{backfill_server} failed to provide backfill for room {room_id}: {e}"); + }, + } + } + + info!("No servers could backfill, but backfill was needed in room {room_id}"); + Ok(()) +} + +#[implement(super::Service)] +#[tracing::instrument(skip(self, pdu), level = "debug")] +pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> Result<()> { + let (room_id, event_id, value) = self.services.event_handler.parse_incoming_pdu(&pdu).await?; + + // Lock so we cannot backfill the same pdu twice at the same time + let mutex_lock = self + .services + .event_handler + .mutex_federation + .lock(&room_id) + .await; + + // Skip the PDU if we already have it as a timeline event + if let Ok(pdu_id) = self.get_pdu_id(&event_id).await { + debug!("We already know {event_id} at {pdu_id:?}"); + return Ok(()); + } + + self.services + .event_handler + .handle_incoming_pdu(origin, &room_id, &event_id, value, false) + .boxed() + .await?; + + let value = self.get_pdu_json(&event_id).await?; + + let pdu = self.get_pdu(&event_id).await?; + + let shortroomid = self.services.short.get_shortroomid(&room_id).await?; + + let insert_lock = self.mutex_insert.lock(&room_id).await; + + let count: i64 = self.services.globals.next_count().unwrap().try_into()?; + + let pdu_id: RawPduId = PduId { + shortroomid, + shorteventid: PduCount::Backfilled(validated!(0 - count)), + } + .into(); + + // Insert pdu + self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value); + + drop(insert_lock); + + if pdu.kind == TimelineEventType::RoomMessage { + let content: ExtractBody = pdu.get_content()?; + if let Some(body) = content.body { + self.services.search.index_pdu(shortroomid, &pdu_id, &body); + } + } + drop(mutex_lock); + + debug!("Prepended backfill pdu"); + Ok(()) +} diff --git a/src/service/rooms/timeline/build.rs b/src/service/rooms/timeline/build.rs new file mode 100644 index 00000000..a522c531 --- /dev/null +++ b/src/service/rooms/timeline/build.rs @@ -0,0 +1,226 @@ +use std::{collections::HashSet, iter::once}; + +use conduwuit_core::{ + Err, Result, implement, + matrix::{event::Event, pdu::PduBuilder}, + utils::{IterStream, ReadyExt}, +}; +use futures::{FutureExt, StreamExt}; +use ruma::{ + OwnedEventId, OwnedServerName, RoomId, RoomVersionId, UserId, + events::{ + TimelineEventType, + room::{ + member::{MembershipState, RoomMemberEventContent}, + redaction::RoomRedactionEventContent, + }, + }, +}; + +use super::RoomMutexGuard; + +/// Creates a new persisted data unit and adds it to a room. This function +/// takes a roomid_mutex_state, meaning that only this function is able to +/// mutate the room state. +#[implement(super::Service)] +#[tracing::instrument(skip(self, state_lock), level = "debug")] +pub async fn build_and_append_pdu( + &self, + pdu_builder: PduBuilder, + sender: &UserId, + room_id: &RoomId, + state_lock: &RoomMutexGuard, +) -> Result { + let (pdu, pdu_json) = self + .create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock) + .await?; + + if self.services.admin.is_admin_room(pdu.room_id()).await { + self.check_pdu_for_admin_room(&pdu, sender).boxed().await?; + } + + // If redaction event is not authorized, do not append it to the timeline + if *pdu.kind() == TimelineEventType::RoomRedaction { + use RoomVersionId::*; + match self.services.state.get_room_version(pdu.room_id()).await? { + | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { + if let Some(redact_id) = pdu.redacts() { + if !self + .services + .state_accessor + .user_can_redact(redact_id, pdu.sender(), pdu.room_id(), false) + .await? + { + return Err!(Request(Forbidden("User cannot redact this event."))); + } + } + }, + | _ => { + let content: RoomRedactionEventContent = pdu.get_content()?; + if let Some(redact_id) = &content.redacts { + if !self + .services + .state_accessor + .user_can_redact(redact_id, pdu.sender(), pdu.room_id(), false) + .await? + { + return Err!(Request(Forbidden("User cannot redact this event."))); + } + } + }, + } + } + + if *pdu.kind() == TimelineEventType::RoomMember { + let content: RoomMemberEventContent = pdu.get_content()?; + + if content.join_authorized_via_users_server.is_some() + && content.membership != MembershipState::Join + { + return Err!(Request(BadJson( + "join_authorised_via_users_server is only for member joins" + ))); + } + + if content + .join_authorized_via_users_server + .as_ref() + .is_some_and(|authorising_user| { + !self.services.globals.user_is_local(authorising_user) + }) { + return Err!(Request(InvalidParam( + "Authorising user does not belong to this homeserver" + ))); + } + } + + // We append to state before appending the pdu, so we don't have a moment in + // time with the pdu without it's state. This is okay because append_pdu can't + // fail. + let statehashid = self.services.state.append_to_state(&pdu).await?; + + let pdu_id = self + .append_pdu( + &pdu, + pdu_json, + // Since this PDU references all pdu_leaves we can update the leaves + // of the room + once(pdu.event_id()), + state_lock, + ) + .boxed() + .await?; + + // We set the room state after inserting the pdu, so that we never have a moment + // in time where events in the current room state do not exist + self.services + .state + .set_room_state(pdu.room_id(), statehashid, state_lock); + + let mut servers: HashSet = self + .services + .state_cache + .room_servers(pdu.room_id()) + .map(ToOwned::to_owned) + .collect() + .await; + + // In case we are kicking or banning a user, we need to inform their server of + // the change + if *pdu.kind() == TimelineEventType::RoomMember { + if let Some(state_key_uid) = &pdu + .state_key + .as_ref() + .and_then(|state_key| UserId::parse(state_key.as_str()).ok()) + { + servers.insert(state_key_uid.server_name().to_owned()); + } + } + + // Remove our server from the server list since it will be added to it by + // room_servers() and/or the if statement above + servers.remove(self.services.globals.server_name()); + + self.services + .sending + .send_pdu_servers(servers.iter().map(AsRef::as_ref).stream(), &pdu_id) + .await?; + + Ok(pdu.event_id().to_owned()) +} + +#[implement(super::Service)] +#[tracing::instrument(skip_all, level = "debug")] +async fn check_pdu_for_admin_room(&self, pdu: &Pdu, sender: &UserId) -> Result +where + Pdu: Event + Send + Sync, +{ + match pdu.kind() { + | TimelineEventType::RoomEncryption => { + return Err!(Request(Forbidden(error!("Encryption not supported in admins room.")))); + }, + | TimelineEventType::RoomMember => { + let target = pdu + .state_key() + .filter(|v| v.starts_with('@')) + .unwrap_or(sender.as_str()); + + let server_user = &self.services.globals.server_user.to_string(); + + let content: RoomMemberEventContent = pdu.get_content()?; + match content.membership { + | MembershipState::Leave => { + if target == server_user { + return Err!(Request(Forbidden(error!( + "Server user cannot leave the admins room." + )))); + } + + let count = self + .services + .state_cache + .room_members(pdu.room_id()) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .ready_filter(|user| *user != target) + .boxed() + .count() + .await; + + if count < 2 { + return Err!(Request(Forbidden(error!( + "Last admin cannot leave the admins room." + )))); + } + }, + + | MembershipState::Ban if pdu.state_key().is_some() => { + if target == server_user { + return Err!(Request(Forbidden(error!( + "Server cannot be banned from admins room." + )))); + } + + let count = self + .services + .state_cache + .room_members(pdu.room_id()) + .ready_filter(|user| self.services.globals.user_is_local(user)) + .ready_filter(|user| *user != target) + .boxed() + .count() + .await; + + if count < 2 { + return Err!(Request(Forbidden(error!( + "Last admin cannot be banned from admins room." + )))); + } + }, + | _ => {}, + } + }, + | _ => {}, + } + + Ok(()) +} diff --git a/src/service/rooms/timeline/create.rs b/src/service/rooms/timeline/create.rs new file mode 100644 index 00000000..d890e88e --- /dev/null +++ b/src/service/rooms/timeline/create.rs @@ -0,0 +1,214 @@ +use std::cmp; + +use conduwuit_core::{ + Err, Error, Result, err, implement, + matrix::{ + event::{Event, gen_event_id}, + pdu::{EventHash, PduBuilder, PduEvent}, + state_res::{self, RoomVersion}, + }, + utils::{self, IterStream, ReadyExt, stream::TryIgnore}, +}; +use futures::{StreamExt, TryStreamExt, future, future::ready}; +use ruma::{ + CanonicalJsonObject, CanonicalJsonValue, OwnedEventId, RoomId, RoomVersionId, UserId, + canonical_json::to_canonical_value, + events::{StateEventType, TimelineEventType, room::create::RoomCreateEventContent}, + uint, +}; +use serde_json::value::to_raw_value; +use tracing::warn; + +use super::RoomMutexGuard; + +#[implement(super::Service)] +pub async fn create_hash_and_sign_event( + &self, + pdu_builder: PduBuilder, + sender: &UserId, + room_id: &RoomId, + _mutex_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room + * state mutex */ +) -> Result<(PduEvent, CanonicalJsonObject)> { + let PduBuilder { + event_type, + content, + unsigned, + state_key, + redacts, + timestamp, + } = pdu_builder; + + let prev_events: Vec = self + .services + .state + .get_forward_extremities(room_id) + .take(20) + .map(Into::into) + .collect() + .await; + + // If there was no create event yet, assume we are creating a room + let room_version_id = self + .services + .state + .get_room_version(room_id) + .await + .or_else(|_| { + if event_type == TimelineEventType::RoomCreate { + let content: RoomCreateEventContent = serde_json::from_str(content.get())?; + Ok(content.room_version) + } else { + Err(Error::InconsistentRoomState( + "non-create event for room of unknown version", + room_id.to_owned(), + )) + } + })?; + + let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); + + let auth_events = self + .services + .state + .get_auth_events(room_id, &event_type, sender, state_key.as_deref(), &content) + .await?; + + // Our depth is the maximum depth of prev_events + 1 + let depth = prev_events + .iter() + .stream() + .map(Ok) + .and_then(|event_id| self.get_pdu(event_id)) + .and_then(|pdu| future::ok(pdu.depth)) + .ignore_err() + .ready_fold(uint!(0), cmp::max) + .await + .saturating_add(uint!(1)); + + let mut unsigned = unsigned.unwrap_or_default(); + + if let Some(state_key) = &state_key { + if let Ok(prev_pdu) = self + .services + .state_accessor + .room_state_get(room_id, &event_type.to_string().into(), state_key) + .await + { + unsigned.insert("prev_content".to_owned(), prev_pdu.get_content_as_value()); + unsigned.insert("prev_sender".to_owned(), serde_json::to_value(prev_pdu.sender())?); + unsigned + .insert("replaces_state".to_owned(), serde_json::to_value(prev_pdu.event_id())?); + } + } + + if event_type != TimelineEventType::RoomCreate && prev_events.is_empty() { + return Err!(Request(Unknown("Event incorrectly had zero prev_events."))); + } + if state_key.is_none() && depth.lt(&uint!(2)) { + // The first two events in a room are always m.room.create and m.room.member, + // so any other events with that same depth are illegal. + warn!( + "Had unsafe depth {depth} when creating non-state event in {room_id}. Cowardly \ + aborting" + ); + return Err!(Request(Unknown("Unsafe depth for non-state event."))); + } + + let mut pdu = PduEvent { + event_id: ruma::event_id!("$thiswillbefilledinlater").into(), + room_id: room_id.to_owned(), + sender: sender.to_owned(), + origin: None, + origin_server_ts: timestamp.map_or_else( + || { + utils::millis_since_unix_epoch() + .try_into() + .expect("u64 fits into UInt") + }, + |ts| ts.get(), + ), + kind: event_type, + content, + state_key, + prev_events, + depth, + auth_events: auth_events + .values() + .map(|pdu| pdu.event_id.clone()) + .collect(), + redacts, + unsigned: if unsigned.is_empty() { + None + } else { + Some(to_raw_value(&unsigned)?) + }, + hashes: EventHash { sha256: "aaa".to_owned() }, + signatures: None, + }; + + let auth_fetch = |k: &StateEventType, s: &str| { + let key = (k.clone(), s.into()); + ready(auth_events.get(&key).map(ToOwned::to_owned)) + }; + + let auth_check = state_res::auth_check( + &room_version, + &pdu, + None, // TODO: third_party_invite + auth_fetch, + ) + .await + .map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?; + + if !auth_check { + return Err!(Request(Forbidden("Event is not authorized."))); + } + + // Hash and sign + let mut pdu_json = utils::to_canonical_object(&pdu).map_err(|e| { + err!(Request(BadJson(warn!("Failed to convert PDU to canonical JSON: {e}")))) + })?; + + // room v3 and above removed the "event_id" field from remote PDU format + match room_version_id { + | RoomVersionId::V1 | RoomVersionId::V2 => {}, + | _ => { + pdu_json.remove("event_id"); + }, + } + + // Add origin because synapse likes that (and it's required in the spec) + pdu_json.insert( + "origin".to_owned(), + to_canonical_value(self.services.globals.server_name()) + .expect("server name is a valid CanonicalJsonValue"), + ); + + if let Err(e) = self + .services + .server_keys + .hash_and_sign_event(&mut pdu_json, &room_version_id) + { + return match e { + | Error::Signatures(ruma::signatures::Error::PduSize) => { + Err!(Request(TooLarge("Message/PDU is too long (exceeds 65535 bytes)"))) + }, + | _ => Err!(Request(Unknown(warn!("Signing event failed: {e}")))), + }; + } + + // Generate event id + pdu.event_id = gen_event_id(&pdu_json, &room_version_id)?; + + pdu_json.insert("event_id".into(), CanonicalJsonValue::String(pdu.event_id.clone().into())); + + // Generate short event id + let _shorteventid = self + .services + .short + .get_or_create_shorteventid(&pdu.event_id) + .await; + + Ok((pdu, pdu_json)) +} diff --git a/src/service/rooms/timeline/mod.rs b/src/service/rooms/timeline/mod.rs index a381fcf6..70c98a09 100644 --- a/src/service/rooms/timeline/mod.rs +++ b/src/service/rooms/timeline/mod.rs @@ -1,61 +1,34 @@ +mod append; +mod backfill; +mod build; +mod create; mod data; +mod redact; -use std::{ - borrow::Borrow, - cmp, - collections::{BTreeMap, HashSet}, - fmt::Write, - iter::once, - sync::Arc, -}; +use std::{fmt::Write, sync::Arc}; use async_trait::async_trait; -pub use conduwuit::matrix::pdu::{PduId, RawPduId}; -use conduwuit::{ - Err, Error, Result, Server, at, debug, debug_warn, err, error, implement, info, +pub use conduwuit_core::matrix::pdu::{PduId, RawPduId}; +use conduwuit_core::{ + Result, Server, at, err, matrix::{ - event::{Event, gen_event_id}, - pdu::{EventHash, PduBuilder, PduCount, PduEvent}, - state_res::{self, RoomVersion}, + event::Event, + pdu::{PduCount, PduEvent}, }, - utils::{ - self, IterStream, MutexMap, MutexMapGuard, ReadyExt, future::TryExtExt, stream::TryIgnore, - }, - validated, warn, -}; -use futures::{ - Future, FutureExt, Stream, StreamExt, TryStreamExt, future, future::ready, pin_mut, + utils::{MutexMap, MutexMapGuard, future::TryExtExt, stream::TryIgnore}, + warn, }; +use futures::{Future, Stream, TryStreamExt, pin_mut}; use ruma::{ - CanonicalJsonObject, CanonicalJsonValue, EventId, OwnedEventId, OwnedRoomId, OwnedServerName, - RoomId, RoomVersionId, ServerName, UserId, - api::federation, - canonical_json::to_canonical_value, - events::{ - GlobalAccountDataEventType, StateEventType, TimelineEventType, - push_rules::PushRulesEvent, - room::{ - create::RoomCreateEventContent, - encrypted::Relation, - member::{MembershipState, RoomMemberEventContent}, - power_levels::RoomPowerLevelsEventContent, - redaction::RoomRedactionEventContent, - }, - }, - push::{Action, Ruleset, Tweak}, - uint, + CanonicalJsonObject, EventId, OwnedEventId, OwnedRoomId, RoomId, UserId, + events::room::encrypted::Relation, }; use serde::Deserialize; -use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; use self::data::Data; pub use self::data::PdusIterItem; use crate::{ - Dep, account_data, admin, appservice, - appservice::NamespaceRegex, - globals, pusher, rooms, - rooms::{short::ShortRoomId, state_compressor::CompressedState}, - sending, server_keys, users, + Dep, account_data, admin, appservice, globals, pusher, rooms, sending, server_keys, users, }; // Update Relationships @@ -259,743 +232,6 @@ impl Service { self.db.replace_pdu(pdu_id, pdu_json).await } - /// Creates a new persisted data unit and adds it to a room. - /// - /// By this point the incoming event should be fully authenticated, no auth - /// happens in `append_pdu`. - /// - /// Returns pdu id - #[tracing::instrument(level = "debug", skip_all)] - pub async fn append_pdu<'a, Leaves>( - &'a self, - pdu: &'a PduEvent, - mut pdu_json: CanonicalJsonObject, - leaves: Leaves, - state_lock: &'a RoomMutexGuard, - ) -> Result - where - Leaves: Iterator + Send + 'a, - { - // Coalesce database writes for the remainder of this scope. - let _cork = self.db.db.cork_and_flush(); - - let shortroomid = self - .services - .short - .get_shortroomid(&pdu.room_id) - .await - .map_err(|_| err!(Database("Room does not exist")))?; - - // Make unsigned fields correct. This is not properly documented in the spec, - // but state events need to have previous content in the unsigned field, so - // clients can easily interpret things like membership changes - if let Some(state_key) = &pdu.state_key { - if let CanonicalJsonValue::Object(unsigned) = pdu_json - .entry("unsigned".to_owned()) - .or_insert_with(|| CanonicalJsonValue::Object(BTreeMap::default())) - { - if let Ok(shortstatehash) = self - .services - .state_accessor - .pdu_shortstatehash(&pdu.event_id) - .await - { - if let Ok(prev_state) = self - .services - .state_accessor - .state_get(shortstatehash, &pdu.kind.to_string().into(), state_key) - .await - { - unsigned.insert( - "prev_content".to_owned(), - CanonicalJsonValue::Object( - utils::to_canonical_object(prev_state.get_content_as_value()) - .map_err(|e| { - err!(Database(error!( - "Failed to convert prev_state to canonical JSON: {e}", - ))) - })?, - ), - ); - unsigned.insert( - String::from("prev_sender"), - CanonicalJsonValue::String(prev_state.sender().to_string()), - ); - unsigned.insert( - String::from("replaces_state"), - CanonicalJsonValue::String(prev_state.event_id().to_string()), - ); - } - } - } else { - error!("Invalid unsigned type in pdu."); - } - } - - // We must keep track of all events that have been referenced. - self.services - .pdu_metadata - .mark_as_referenced(&pdu.room_id, pdu.prev_events.iter().map(AsRef::as_ref)); - - self.services - .state - .set_forward_extremities(&pdu.room_id, leaves, state_lock) - .await; - - let insert_lock = self.mutex_insert.lock(&pdu.room_id).await; - - let count1 = self.services.globals.next_count().unwrap(); - // Mark as read first so the sending client doesn't get a notification even if - // appending fails - self.services - .read_receipt - .private_read_set(&pdu.room_id, &pdu.sender, count1); - self.services - .user - .reset_notification_counts(&pdu.sender, &pdu.room_id); - - let count2 = PduCount::Normal(self.services.globals.next_count().unwrap()); - let pdu_id: RawPduId = PduId { shortroomid, shorteventid: count2 }.into(); - - // Insert pdu - self.db.append_pdu(&pdu_id, pdu, &pdu_json, count2).await; - - drop(insert_lock); - - // See if the event matches any known pushers via power level - let power_levels: RoomPowerLevelsEventContent = self - .services - .state_accessor - .room_state_get_content(&pdu.room_id, &StateEventType::RoomPowerLevels, "") - .await - .unwrap_or_default(); - - let mut push_target: HashSet<_> = self - .services - .state_cache - .active_local_users_in_room(&pdu.room_id) - .map(ToOwned::to_owned) - // Don't notify the sender of their own events, and dont send from ignored users - .ready_filter(|user| *user != pdu.sender) - .filter_map(|recipient_user| async move { (!self.services.users.user_is_ignored(&pdu.sender, &recipient_user).await).then_some(recipient_user) }) - .collect() - .await; - - let mut notifies = Vec::with_capacity(push_target.len().saturating_add(1)); - let mut highlights = Vec::with_capacity(push_target.len().saturating_add(1)); - - if pdu.kind == TimelineEventType::RoomMember { - if let Some(state_key) = &pdu.state_key { - let target_user_id = UserId::parse(state_key)?; - - if self.services.users.is_active_local(target_user_id).await { - push_target.insert(target_user_id.to_owned()); - } - } - } - - let serialized = pdu.to_format(); - for user in &push_target { - let rules_for_user = self - .services - .account_data - .get_global(user, GlobalAccountDataEventType::PushRules) - .await - .map_or_else( - |_| Ruleset::server_default(user), - |ev: PushRulesEvent| ev.content.global, - ); - - let mut highlight = false; - let mut notify = false; - - for action in self - .services - .pusher - .get_actions(user, &rules_for_user, &power_levels, &serialized, &pdu.room_id) - .await - { - match action { - | Action::Notify => notify = true, - | Action::SetTweak(Tweak::Highlight(true)) => { - highlight = true; - }, - | _ => {}, - } - - // Break early if both conditions are true - if notify && highlight { - break; - } - } - - if notify { - notifies.push(user.clone()); - } - - if highlight { - highlights.push(user.clone()); - } - - self.services - .pusher - .get_pushkeys(user) - .ready_for_each(|push_key| { - self.services - .sending - .send_pdu_push(&pdu_id, user, push_key.to_owned()) - .expect("TODO: replace with future"); - }) - .await; - } - - self.db - .increment_notification_counts(&pdu.room_id, notifies, highlights); - - match pdu.kind { - | TimelineEventType::RoomRedaction => { - use RoomVersionId::*; - - let room_version_id = self.services.state.get_room_version(&pdu.room_id).await?; - match room_version_id { - | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { - if let Some(redact_id) = &pdu.redacts { - if self - .services - .state_accessor - .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false) - .await? - { - self.redact_pdu(redact_id, pdu, shortroomid).await?; - } - } - }, - | _ => { - let content: RoomRedactionEventContent = pdu.get_content()?; - if let Some(redact_id) = &content.redacts { - if self - .services - .state_accessor - .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false) - .await? - { - self.redact_pdu(redact_id, pdu, shortroomid).await?; - } - } - }, - } - }, - | TimelineEventType::SpaceChild => - if let Some(_state_key) = &pdu.state_key { - self.services - .spaces - .roomid_spacehierarchy_cache - .lock() - .await - .remove(&pdu.room_id); - }, - | TimelineEventType::RoomMember => { - if let Some(state_key) = &pdu.state_key { - // if the state_key fails - let target_user_id = UserId::parse(state_key) - .expect("This state_key was previously validated"); - - let content: RoomMemberEventContent = pdu.get_content()?; - let stripped_state = match content.membership { - | MembershipState::Invite | MembershipState::Knock => - self.services.state.summary_stripped(pdu).await.into(), - | _ => None, - }; - - // Update our membership info, we do this here incase a user is invited or - // knocked and immediately leaves we need the DB to record the invite or - // knock event for auth - self.services - .state_cache - .update_membership( - &pdu.room_id, - target_user_id, - content, - &pdu.sender, - stripped_state, - None, - true, - ) - .await?; - } - }, - | TimelineEventType::RoomMessage => { - let content: ExtractBody = pdu.get_content()?; - if let Some(body) = content.body { - self.services.search.index_pdu(shortroomid, &pdu_id, &body); - - if self.services.admin.is_admin_command(pdu, &body).await { - self.services.admin.command_with_sender( - body, - Some((*pdu.event_id).into()), - pdu.sender.clone().into(), - )?; - } - } - }, - | _ => {}, - } - - if let Ok(content) = pdu.get_content::() { - if let Ok(related_pducount) = self.get_pdu_count(&content.relates_to.event_id).await { - self.services - .pdu_metadata - .add_relation(count2, related_pducount); - } - } - - if let Ok(content) = pdu.get_content::() { - match content.relates_to { - | Relation::Reply { in_reply_to } => { - // We need to do it again here, because replies don't have - // event_id as a top level field - if let Ok(related_pducount) = self.get_pdu_count(&in_reply_to.event_id).await - { - self.services - .pdu_metadata - .add_relation(count2, related_pducount); - } - }, - | Relation::Thread(thread) => { - self.services - .threads - .add_to_thread(&thread.event_id, pdu) - .await?; - }, - | _ => {}, // TODO: Aggregate other types - } - } - - for appservice in self.services.appservice.read().await.values() { - if self - .services - .state_cache - .appservice_in_room(&pdu.room_id, appservice) - .await - { - self.services - .sending - .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; - continue; - } - - // If the RoomMember event has a non-empty state_key, it is targeted at someone. - // If it is our appservice user, we send this PDU to it. - if pdu.kind == TimelineEventType::RoomMember { - if let Some(state_key_uid) = &pdu - .state_key - .as_ref() - .and_then(|state_key| UserId::parse(state_key.as_str()).ok()) - { - let appservice_uid = appservice.registration.sender_localpart.as_str(); - if state_key_uid == &appservice_uid { - self.services - .sending - .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; - continue; - } - } - } - - let matching_users = |users: &NamespaceRegex| { - appservice.users.is_match(pdu.sender.as_str()) - || pdu.kind == TimelineEventType::RoomMember - && pdu - .state_key - .as_ref() - .is_some_and(|state_key| users.is_match(state_key)) - }; - let matching_aliases = |aliases: NamespaceRegex| { - self.services - .alias - .local_aliases_for_room(&pdu.room_id) - .ready_any(move |room_alias| aliases.is_match(room_alias.as_str())) - }; - - if matching_aliases(appservice.aliases.clone()).await - || appservice.rooms.is_match(pdu.room_id.as_str()) - || matching_users(&appservice.users) - { - self.services - .sending - .send_pdu_appservice(appservice.registration.id.clone(), pdu_id)?; - } - } - - Ok(pdu_id) - } - - pub async fn create_hash_and_sign_event( - &self, - pdu_builder: PduBuilder, - sender: &UserId, - room_id: &RoomId, - _mutex_lock: &RoomMutexGuard, /* Take mutex guard to make sure users get the room - * state mutex */ - ) -> Result<(PduEvent, CanonicalJsonObject)> { - let PduBuilder { - event_type, - content, - unsigned, - state_key, - redacts, - timestamp, - } = pdu_builder; - - let prev_events: Vec = self - .services - .state - .get_forward_extremities(room_id) - .take(20) - .map(Into::into) - .collect() - .await; - - // If there was no create event yet, assume we are creating a room - let room_version_id = self - .services - .state - .get_room_version(room_id) - .await - .or_else(|_| { - if event_type == TimelineEventType::RoomCreate { - let content: RoomCreateEventContent = serde_json::from_str(content.get())?; - Ok(content.room_version) - } else { - Err(Error::InconsistentRoomState( - "non-create event for room of unknown version", - room_id.to_owned(), - )) - } - })?; - - let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); - - let auth_events = self - .services - .state - .get_auth_events(room_id, &event_type, sender, state_key.as_deref(), &content) - .await?; - - // Our depth is the maximum depth of prev_events + 1 - let depth = prev_events - .iter() - .stream() - .map(Ok) - .and_then(|event_id| self.get_pdu(event_id)) - .and_then(|pdu| future::ok(pdu.depth)) - .ignore_err() - .ready_fold(uint!(0), cmp::max) - .await - .saturating_add(uint!(1)); - - let mut unsigned = unsigned.unwrap_or_default(); - - if let Some(state_key) = &state_key { - if let Ok(prev_pdu) = self - .services - .state_accessor - .room_state_get(room_id, &event_type.to_string().into(), state_key) - .await - { - unsigned.insert("prev_content".to_owned(), prev_pdu.get_content_as_value()); - unsigned - .insert("prev_sender".to_owned(), serde_json::to_value(prev_pdu.sender())?); - unsigned.insert( - "replaces_state".to_owned(), - serde_json::to_value(prev_pdu.event_id())?, - ); - } - } - if event_type != TimelineEventType::RoomCreate && prev_events.is_empty() { - return Err!(Request(Unknown("Event incorrectly had zero prev_events."))); - } - if state_key.is_none() && depth.lt(&uint!(2)) { - // The first two events in a room are always m.room.create and m.room.member, - // so any other events with that same depth are illegal. - warn!( - "Had unsafe depth {depth} when creating non-state event in {room_id}. Cowardly \ - aborting" - ); - return Err!(Request(Unknown("Unsafe depth for non-state event."))); - } - - let mut pdu = PduEvent { - event_id: ruma::event_id!("$thiswillbefilledinlater").into(), - room_id: room_id.to_owned(), - sender: sender.to_owned(), - origin: None, - origin_server_ts: timestamp.map_or_else( - || { - utils::millis_since_unix_epoch() - .try_into() - .expect("u64 fits into UInt") - }, - |ts| ts.get(), - ), - kind: event_type, - content, - state_key, - prev_events, - depth, - auth_events: auth_events - .values() - .map(|pdu| pdu.event_id.clone()) - .collect(), - redacts, - unsigned: if unsigned.is_empty() { - None - } else { - Some(to_raw_value(&unsigned)?) - }, - hashes: EventHash { sha256: "aaa".to_owned() }, - signatures: None, - }; - - let auth_fetch = |k: &StateEventType, s: &str| { - let key = (k.clone(), s.into()); - ready(auth_events.get(&key).map(ToOwned::to_owned)) - }; - - let auth_check = state_res::auth_check( - &room_version, - &pdu, - None, // TODO: third_party_invite - auth_fetch, - ) - .await - .map_err(|e| err!(Request(Forbidden(warn!("Auth check failed: {e:?}")))))?; - - if !auth_check { - return Err!(Request(Forbidden("Event is not authorized."))); - } - - // Hash and sign - let mut pdu_json = utils::to_canonical_object(&pdu).map_err(|e| { - err!(Request(BadJson(warn!("Failed to convert PDU to canonical JSON: {e}")))) - })?; - - // room v3 and above removed the "event_id" field from remote PDU format - match room_version_id { - | RoomVersionId::V1 | RoomVersionId::V2 => {}, - | _ => { - pdu_json.remove("event_id"); - }, - } - - // Add origin because synapse likes that (and it's required in the spec) - pdu_json.insert( - "origin".to_owned(), - to_canonical_value(self.services.globals.server_name()) - .expect("server name is a valid CanonicalJsonValue"), - ); - - if let Err(e) = self - .services - .server_keys - .hash_and_sign_event(&mut pdu_json, &room_version_id) - { - return match e { - | Error::Signatures(ruma::signatures::Error::PduSize) => { - Err!(Request(TooLarge("Message/PDU is too long (exceeds 65535 bytes)"))) - }, - | _ => Err!(Request(Unknown(warn!("Signing event failed: {e}")))), - }; - } - - // Generate event id - pdu.event_id = gen_event_id(&pdu_json, &room_version_id)?; - - pdu_json - .insert("event_id".into(), CanonicalJsonValue::String(pdu.event_id.clone().into())); - - // Generate short event id - let _shorteventid = self - .services - .short - .get_or_create_shorteventid(&pdu.event_id) - .await; - - Ok((pdu, pdu_json)) - } - - /// Creates a new persisted data unit and adds it to a room. This function - /// takes a roomid_mutex_state, meaning that only this function is able to - /// mutate the room state. - #[tracing::instrument(skip(self, state_lock), level = "debug")] - pub async fn build_and_append_pdu( - &self, - pdu_builder: PduBuilder, - sender: &UserId, - room_id: &RoomId, - state_lock: &RoomMutexGuard, - ) -> Result { - let (pdu, pdu_json) = self - .create_hash_and_sign_event(pdu_builder, sender, room_id, state_lock) - .await?; - - if self.services.admin.is_admin_room(&pdu.room_id).await { - self.check_pdu_for_admin_room(&pdu, sender).boxed().await?; - } - - // If redaction event is not authorized, do not append it to the timeline - if pdu.kind == TimelineEventType::RoomRedaction { - use RoomVersionId::*; - match self.services.state.get_room_version(&pdu.room_id).await? { - | V1 | V2 | V3 | V4 | V5 | V6 | V7 | V8 | V9 | V10 => { - if let Some(redact_id) = &pdu.redacts { - if !self - .services - .state_accessor - .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false) - .await? - { - return Err!(Request(Forbidden("User cannot redact this event."))); - } - } - }, - | _ => { - let content: RoomRedactionEventContent = pdu.get_content()?; - if let Some(redact_id) = &content.redacts { - if !self - .services - .state_accessor - .user_can_redact(redact_id, &pdu.sender, &pdu.room_id, false) - .await? - { - return Err!(Request(Forbidden("User cannot redact this event."))); - } - } - }, - } - } - - if pdu.kind == TimelineEventType::RoomMember { - let content: RoomMemberEventContent = pdu.get_content()?; - - if content.join_authorized_via_users_server.is_some() - && content.membership != MembershipState::Join - { - return Err!(Request(BadJson( - "join_authorised_via_users_server is only for member joins" - ))); - } - - if content - .join_authorized_via_users_server - .as_ref() - .is_some_and(|authorising_user| { - !self.services.globals.user_is_local(authorising_user) - }) { - return Err!(Request(InvalidParam( - "Authorising user does not belong to this homeserver" - ))); - } - } - - // We append to state before appending the pdu, so we don't have a moment in - // time with the pdu without it's state. This is okay because append_pdu can't - // fail. - let statehashid = self.services.state.append_to_state(&pdu).await?; - - let pdu_id = self - .append_pdu( - &pdu, - pdu_json, - // Since this PDU references all pdu_leaves we can update the leaves - // of the room - once(pdu.event_id.borrow()), - state_lock, - ) - .boxed() - .await?; - - // We set the room state after inserting the pdu, so that we never have a moment - // in time where events in the current room state do not exist - self.services - .state - .set_room_state(&pdu.room_id, statehashid, state_lock); - - let mut servers: HashSet = self - .services - .state_cache - .room_servers(&pdu.room_id) - .map(ToOwned::to_owned) - .collect() - .await; - - // In case we are kicking or banning a user, we need to inform their server of - // the change - if pdu.kind == TimelineEventType::RoomMember { - if let Some(state_key_uid) = &pdu - .state_key - .as_ref() - .and_then(|state_key| UserId::parse(state_key.as_str()).ok()) - { - servers.insert(state_key_uid.server_name().to_owned()); - } - } - - // Remove our server from the server list since it will be added to it by - // room_servers() and/or the if statement above - servers.remove(self.services.globals.server_name()); - - self.services - .sending - .send_pdu_servers(servers.iter().map(AsRef::as_ref).stream(), &pdu_id) - .await?; - - Ok(pdu.event_id) - } - - /// Append the incoming event setting the state snapshot to the state from - /// the server that sent the event. - #[tracing::instrument(level = "debug", skip_all)] - pub async fn append_incoming_pdu<'a, Leaves>( - &'a self, - pdu: &'a PduEvent, - pdu_json: CanonicalJsonObject, - new_room_leaves: Leaves, - state_ids_compressed: Arc, - soft_fail: bool, - state_lock: &'a RoomMutexGuard, - ) -> Result> - where - Leaves: Iterator + Send + 'a, - { - // We append to state before appending the pdu, so we don't have a moment in - // time with the pdu without it's state. This is okay because append_pdu can't - // fail. - self.services - .state - .set_event_state(&pdu.event_id, &pdu.room_id, state_ids_compressed) - .await?; - - if soft_fail { - self.services - .pdu_metadata - .mark_as_referenced(&pdu.room_id, pdu.prev_events.iter().map(AsRef::as_ref)); - - self.services - .state - .set_forward_extremities(&pdu.room_id, new_room_leaves, state_lock) - .await; - - return Ok(None); - } - - let pdu_id = self - .append_pdu(pdu, pdu_json, new_room_leaves, state_lock) - .await?; - - Ok(Some(pdu_id)) - } - /// Returns an iterator over all PDUs in a room. Unknown rooms produce no /// items. #[inline] @@ -1030,290 +266,4 @@ impl Service { self.db .pdus(user_id, room_id, from.unwrap_or_else(PduCount::min)) } - - /// Replace a PDU with the redacted form. - #[tracing::instrument(name = "redact", level = "debug", skip(self))] - pub async fn redact_pdu( - &self, - event_id: &EventId, - reason: &Pdu, - shortroomid: ShortRoomId, - ) -> Result { - // TODO: Don't reserialize, keep original json - let Ok(pdu_id) = self.get_pdu_id(event_id).await else { - // If event does not exist, just noop - return Ok(()); - }; - - let mut pdu = self - .get_pdu_from_id(&pdu_id) - .await - .map(Event::into_pdu) - .map_err(|e| { - err!(Database(error!(?pdu_id, ?event_id, ?e, "PDU ID points to invalid PDU."))) - })?; - - if let Ok(content) = pdu.get_content::() { - if let Some(body) = content.body { - self.services - .search - .deindex_pdu(shortroomid, &pdu_id, &body); - } - } - - let room_version_id = self.services.state.get_room_version(pdu.room_id()).await?; - - pdu.redact(&room_version_id, reason.to_value())?; - - let obj = utils::to_canonical_object(&pdu).map_err(|e| { - err!(Database(error!(?event_id, ?e, "Failed to convert PDU to canonical JSON"))) - })?; - - self.replace_pdu(&pdu_id, &obj).await - } - - #[tracing::instrument(name = "backfill", level = "debug", skip(self))] - pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Result<()> { - if self - .services - .state_cache - .room_joined_count(room_id) - .await - .is_ok_and(|count| count <= 1) - && !self - .services - .state_accessor - .is_world_readable(room_id) - .await - { - // Room is empty (1 user or none), there is no one that can backfill - return Ok(()); - } - - let first_pdu = self - .first_item_in_room(room_id) - .await - .expect("Room is not empty"); - - if first_pdu.0 < from { - // No backfill required, there are still events between them - return Ok(()); - } - - let power_levels: RoomPowerLevelsEventContent = self - .services - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomPowerLevels, "") - .await - .unwrap_or_default(); - - let room_mods = power_levels.users.iter().filter_map(|(user_id, level)| { - if level > &power_levels.users_default - && !self.services.globals.user_is_local(user_id) - { - Some(user_id.server_name()) - } else { - None - } - }); - - let canonical_room_alias_server = once( - self.services - .state_accessor - .get_canonical_alias(room_id) - .await, - ) - .filter_map(Result::ok) - .map(|alias| alias.server_name().to_owned()) - .stream(); - - let mut servers = room_mods - .stream() - .map(ToOwned::to_owned) - .chain(canonical_room_alias_server) - .chain( - self.services - .server - .config - .trusted_servers - .iter() - .map(ToOwned::to_owned) - .stream(), - ) - .ready_filter(|server_name| !self.services.globals.server_is_ours(server_name)) - .filter_map(|server_name| async move { - self.services - .state_cache - .server_in_room(&server_name, room_id) - .await - .then_some(server_name) - }) - .boxed(); - - while let Some(ref backfill_server) = servers.next().await { - info!("Asking {backfill_server} for backfill"); - let response = self - .services - .sending - .send_federation_request( - backfill_server, - federation::backfill::get_backfill::v1::Request { - room_id: room_id.to_owned(), - v: vec![first_pdu.1.event_id().to_owned()], - limit: uint!(100), - }, - ) - .await; - match response { - | Ok(response) => { - for pdu in response.pdus { - if let Err(e) = self.backfill_pdu(backfill_server, pdu).boxed().await { - debug_warn!("Failed to add backfilled pdu in room {room_id}: {e}"); - } - } - return Ok(()); - }, - | Err(e) => { - warn!("{backfill_server} failed to provide backfill for room {room_id}: {e}"); - }, - } - } - - info!("No servers could backfill, but backfill was needed in room {room_id}"); - Ok(()) - } - - #[tracing::instrument(skip(self, pdu), level = "debug")] - pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> Result<()> { - let (room_id, event_id, value) = - self.services.event_handler.parse_incoming_pdu(&pdu).await?; - - // Lock so we cannot backfill the same pdu twice at the same time - let mutex_lock = self - .services - .event_handler - .mutex_federation - .lock(&room_id) - .await; - - // Skip the PDU if we already have it as a timeline event - if let Ok(pdu_id) = self.get_pdu_id(&event_id).await { - debug!("We already know {event_id} at {pdu_id:?}"); - return Ok(()); - } - - self.services - .event_handler - .handle_incoming_pdu(origin, &room_id, &event_id, value, false) - .boxed() - .await?; - - let value = self.get_pdu_json(&event_id).await?; - - let pdu = self.get_pdu(&event_id).await?; - - let shortroomid = self.services.short.get_shortroomid(&room_id).await?; - - let insert_lock = self.mutex_insert.lock(&room_id).await; - - let count: i64 = self.services.globals.next_count().unwrap().try_into()?; - - let pdu_id: RawPduId = PduId { - shortroomid, - shorteventid: PduCount::Backfilled(validated!(0 - count)), - } - .into(); - - // Insert pdu - self.db.prepend_backfill_pdu(&pdu_id, &event_id, &value); - - drop(insert_lock); - - if pdu.kind == TimelineEventType::RoomMessage { - let content: ExtractBody = pdu.get_content()?; - if let Some(body) = content.body { - self.services.search.index_pdu(shortroomid, &pdu_id, &body); - } - } - drop(mutex_lock); - - debug!("Prepended backfill pdu"); - Ok(()) - } -} - -#[implement(Service)] -#[tracing::instrument(skip_all, level = "debug")] -async fn check_pdu_for_admin_room(&self, pdu: &Pdu, sender: &UserId) -> Result -where - Pdu: Event + Send + Sync, -{ - match pdu.kind() { - | TimelineEventType::RoomEncryption => { - return Err!(Request(Forbidden(error!("Encryption not supported in admins room.")))); - }, - | TimelineEventType::RoomMember => { - let target = pdu - .state_key() - .filter(|v| v.starts_with('@')) - .unwrap_or(sender.as_str()); - - let server_user = &self.services.globals.server_user.to_string(); - - let content: RoomMemberEventContent = pdu.get_content()?; - match content.membership { - | MembershipState::Leave => { - if target == server_user { - return Err!(Request(Forbidden(error!( - "Server user cannot leave the admins room." - )))); - } - - let count = self - .services - .state_cache - .room_members(pdu.room_id()) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .ready_filter(|user| *user != target) - .boxed() - .count() - .await; - - if count < 2 { - return Err!(Request(Forbidden(error!( - "Last admin cannot leave the admins room." - )))); - } - }, - - | MembershipState::Ban if pdu.state_key().is_some() => { - if target == server_user { - return Err!(Request(Forbidden(error!( - "Server cannot be banned from admins room." - )))); - } - - let count = self - .services - .state_cache - .room_members(pdu.room_id()) - .ready_filter(|user| self.services.globals.user_is_local(user)) - .ready_filter(|user| *user != target) - .boxed() - .count() - .await; - - if count < 2 { - return Err!(Request(Forbidden(error!( - "Last admin cannot be banned from admins room." - )))); - } - }, - | _ => {}, - } - }, - | _ => {}, - } - - Ok(()) } diff --git a/src/service/rooms/timeline/redact.rs b/src/service/rooms/timeline/redact.rs new file mode 100644 index 00000000..d51a8462 --- /dev/null +++ b/src/service/rooms/timeline/redact.rs @@ -0,0 +1,51 @@ +use conduwuit_core::{ + Result, err, implement, + matrix::event::Event, + utils::{self}, +}; +use ruma::EventId; + +use super::ExtractBody; +use crate::rooms::short::ShortRoomId; + +/// Replace a PDU with the redacted form. +#[implement(super::Service)] +#[tracing::instrument(name = "redact", level = "debug", skip(self))] +pub async fn redact_pdu( + &self, + event_id: &EventId, + reason: &Pdu, + shortroomid: ShortRoomId, +) -> Result { + // TODO: Don't reserialize, keep original json + let Ok(pdu_id) = self.get_pdu_id(event_id).await else { + // If event does not exist, just noop + return Ok(()); + }; + + let mut pdu = self + .get_pdu_from_id(&pdu_id) + .await + .map(Event::into_pdu) + .map_err(|e| { + err!(Database(error!(?pdu_id, ?event_id, ?e, "PDU ID points to invalid PDU."))) + })?; + + if let Ok(content) = pdu.get_content::() { + if let Some(body) = content.body { + self.services + .search + .deindex_pdu(shortroomid, &pdu_id, &body); + } + } + + let room_version_id = self.services.state.get_room_version(pdu.room_id()).await?; + + pdu.redact(&room_version_id, reason.to_value())?; + + let obj = utils::to_canonical_object(&pdu).map_err(|e| { + err!(Database(error!(?event_id, ?e, "Failed to convert PDU to canonical JSON"))) + })?; + + self.replace_pdu(&pdu_id, &obj).await +} From 56420a67ca8dad57923aa184de860ecbd871b9f1 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 29 Apr 2025 06:55:54 +0000 Subject: [PATCH 1892/2091] Outdent state_compressor service. Signed-off-by: Jason Volk --- src/service/rooms/state_compressor/mod.rs | 678 +++++++++++----------- 1 file changed, 341 insertions(+), 337 deletions(-) diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index 56a91d0e..a33fb342 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use conduwuit::{ Result, arrayvec::ArrayVec, - at, checked, err, expected, utils, + at, checked, err, expected, implement, utils, utils::{bytes, math::usize_from_f64, stream::IterStream}, }; use database::Map; @@ -115,29 +115,30 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - /// Returns a stack with info on shortstatehash, full state, added diff and - /// removed diff for the selected shortstatehash and each parent layer. - #[tracing::instrument(name = "load", level = "debug", skip(self))] - pub async fn load_shortstatehash_info( - &self, - shortstatehash: ShortStateHash, - ) -> Result { - if let Some(r) = self.stateinfo_cache.lock()?.get_mut(&shortstatehash) { - return Ok(r.clone()); - } - - let stack = self.new_shortstatehash_info(shortstatehash).await?; - - self.cache_shortstatehash_info(shortstatehash, stack.clone()) - .await?; - - Ok(stack) +/// Returns a stack with info on shortstatehash, full state, added diff and +/// removed diff for the selected shortstatehash and each parent layer. +#[implement(Service)] +#[tracing::instrument(name = "load", level = "debug", skip(self))] +pub async fn load_shortstatehash_info( + &self, + shortstatehash: ShortStateHash, +) -> Result { + if let Some(r) = self.stateinfo_cache.lock()?.get_mut(&shortstatehash) { + return Ok(r.clone()); } - /// Returns a stack with info on shortstatehash, full state, added diff and - /// removed diff for the selected shortstatehash and each parent layer. - #[tracing::instrument( + let stack = self.new_shortstatehash_info(shortstatehash).await?; + + self.cache_shortstatehash_info(shortstatehash, stack.clone()) + .await?; + + Ok(stack) +} + +/// Returns a stack with info on shortstatehash, full state, added diff and +/// removed diff for the selected shortstatehash and each parent layer. +#[implement(Service)] +#[tracing::instrument( name = "cache", level = "debug", skip_all, @@ -146,362 +147,365 @@ impl Service { stack = stack.len(), ), )] - async fn cache_shortstatehash_info( - &self, - shortstatehash: ShortStateHash, - stack: ShortStateInfoVec, - ) -> Result { - self.stateinfo_cache.lock()?.insert(shortstatehash, stack); +async fn cache_shortstatehash_info( + &self, + shortstatehash: ShortStateHash, + stack: ShortStateInfoVec, +) -> Result { + self.stateinfo_cache.lock()?.insert(shortstatehash, stack); - Ok(()) - } + Ok(()) +} - async fn new_shortstatehash_info( - &self, - shortstatehash: ShortStateHash, - ) -> Result { - let StateDiff { parent, added, removed } = self.get_statediff(shortstatehash).await?; +#[implement(Service)] +async fn new_shortstatehash_info( + &self, + shortstatehash: ShortStateHash, +) -> Result { + let StateDiff { parent, added, removed } = self.get_statediff(shortstatehash).await?; - let Some(parent) = parent else { - return Ok(vec![ShortStateInfo { - shortstatehash, - full_state: added.clone(), - added, - removed, - }]); - }; - - let mut stack = Box::pin(self.load_shortstatehash_info(parent)).await?; - let top = stack.last().expect("at least one frame"); - - let mut full_state = (*top.full_state).clone(); - full_state.extend(added.iter().copied()); - - let removed = (*removed).clone(); - for r in &removed { - full_state.remove(r); - } - - stack.push(ShortStateInfo { + let Some(parent) = parent else { + return Ok(vec![ShortStateInfo { shortstatehash, + full_state: added.clone(), added, - removed: Arc::new(removed), - full_state: Arc::new(full_state), - }); + removed, + }]); + }; - Ok(stack) + let mut stack = Box::pin(self.load_shortstatehash_info(parent)).await?; + let top = stack.last().expect("at least one frame"); + + let mut full_state = (*top.full_state).clone(); + full_state.extend(added.iter().copied()); + + let removed = (*removed).clone(); + for r in &removed { + full_state.remove(r); } - pub fn compress_state_events<'a, I>( - &'a self, - state: I, - ) -> impl Stream + Send + 'a - where - I: Iterator + Clone + Debug + Send + 'a, - { - let event_ids = state.clone().map(at!(1)); + stack.push(ShortStateInfo { + shortstatehash, + added, + removed: Arc::new(removed), + full_state: Arc::new(full_state), + }); - let short_event_ids = self - .services - .short - .multi_get_or_create_shorteventid(event_ids); + Ok(stack) +} - state - .stream() - .map(at!(0)) - .zip(short_event_ids) - .map(|(shortstatekey, shorteventid)| { - compress_state_event(*shortstatekey, shorteventid) - }) - } +#[implement(Service)] +pub fn compress_state_events<'a, I>( + &'a self, + state: I, +) -> impl Stream + Send + 'a +where + I: Iterator + Clone + Debug + Send + 'a, +{ + let event_ids = state.clone().map(at!(1)); - pub async fn compress_state_event( - &self, - shortstatekey: ShortStateKey, - event_id: &EventId, - ) -> CompressedStateEvent { - let shorteventid = self - .services - .short - .get_or_create_shorteventid(event_id) - .await; + let short_event_ids = self + .services + .short + .multi_get_or_create_shorteventid(event_ids); - compress_state_event(shortstatekey, shorteventid) - } + state + .stream() + .map(at!(0)) + .zip(short_event_ids) + .map(|(shortstatekey, shorteventid)| compress_state_event(*shortstatekey, shorteventid)) +} - /// Creates a new shortstatehash that often is just a diff to an already - /// existing shortstatehash and therefore very efficient. - /// - /// There are multiple layers of diffs. The bottom layer 0 always contains - /// the full state. Layer 1 contains diffs to states of layer 0, layer 2 - /// diffs to layer 1 and so on. If layer n > 0 grows too big, it will be - /// combined with layer n-1 to create a new diff on layer n-1 that's - /// based on layer n-2. If that layer is also too big, it will recursively - /// fix above layers too. - /// - /// * `shortstatehash` - Shortstatehash of this state - /// * `statediffnew` - Added to base. Each vec is shortstatekey+shorteventid - /// * `statediffremoved` - Removed from base. Each vec is - /// shortstatekey+shorteventid - /// * `diff_to_sibling` - Approximately how much the diff grows each time - /// for this layer - /// * `parent_states` - A stack with info on shortstatehash, full state, - /// added diff and removed diff for each parent layer - pub fn save_state_from_diff( - &self, - shortstatehash: ShortStateHash, - statediffnew: Arc, - statediffremoved: Arc, - diff_to_sibling: usize, - mut parent_states: ParentStatesVec, - ) -> Result { - let statediffnew_len = statediffnew.len(); - let statediffremoved_len = statediffremoved.len(); - let diffsum = checked!(statediffnew_len + statediffremoved_len)?; +#[implement(Service)] +pub async fn compress_state_event( + &self, + shortstatekey: ShortStateKey, + event_id: &EventId, +) -> CompressedStateEvent { + let shorteventid = self + .services + .short + .get_or_create_shorteventid(event_id) + .await; - if parent_states.len() > 3 { - // Number of layers - // To many layers, we have to go deeper - let parent = parent_states.pop().expect("parent must have a state"); + compress_state_event(shortstatekey, shorteventid) +} - let mut parent_new = (*parent.added).clone(); - let mut parent_removed = (*parent.removed).clone(); - - for removed in statediffremoved.iter() { - if !parent_new.remove(removed) { - // It was not added in the parent and we removed it - parent_removed.insert(*removed); - } - // Else it was added in the parent and we removed it again. We - // can forget this change - } - - for new in statediffnew.iter() { - if !parent_removed.remove(new) { - // It was not touched in the parent and we added it - parent_new.insert(*new); - } - // Else it was removed in the parent and we added it again. We - // can forget this change - } - - self.save_state_from_diff( - shortstatehash, - Arc::new(parent_new), - Arc::new(parent_removed), - diffsum, - parent_states, - )?; - - return Ok(()); - } - - if parent_states.is_empty() { - // There is no parent layer, create a new state - self.save_statediff(shortstatehash, &StateDiff { - parent: None, - added: statediffnew, - removed: statediffremoved, - }); - - return Ok(()); - } - - // Else we have two options. - // 1. We add the current diff on top of the parent layer. - // 2. We replace a layer above +/// Creates a new shortstatehash that often is just a diff to an already +/// existing shortstatehash and therefore very efficient. +/// +/// There are multiple layers of diffs. The bottom layer 0 always contains +/// the full state. Layer 1 contains diffs to states of layer 0, layer 2 +/// diffs to layer 1 and so on. If layer n > 0 grows too big, it will be +/// combined with layer n-1 to create a new diff on layer n-1 that's +/// based on layer n-2. If that layer is also too big, it will recursively +/// fix above layers too. +/// +/// * `shortstatehash` - Shortstatehash of this state +/// * `statediffnew` - Added to base. Each vec is shortstatekey+shorteventid +/// * `statediffremoved` - Removed from base. Each vec is +/// shortstatekey+shorteventid +/// * `diff_to_sibling` - Approximately how much the diff grows each time for +/// this layer +/// * `parent_states` - A stack with info on shortstatehash, full state, added +/// diff and removed diff for each parent layer +#[implement(Service)] +pub fn save_state_from_diff( + &self, + shortstatehash: ShortStateHash, + statediffnew: Arc, + statediffremoved: Arc, + diff_to_sibling: usize, + mut parent_states: ParentStatesVec, +) -> Result { + let statediffnew_len = statediffnew.len(); + let statediffremoved_len = statediffremoved.len(); + let diffsum = checked!(statediffnew_len + statediffremoved_len)?; + if parent_states.len() > 3 { + // Number of layers + // To many layers, we have to go deeper let parent = parent_states.pop().expect("parent must have a state"); - let parent_added_len = parent.added.len(); - let parent_removed_len = parent.removed.len(); - let parent_diff = checked!(parent_added_len + parent_removed_len)?; - if checked!(diffsum * diffsum)? >= checked!(2 * diff_to_sibling * parent_diff)? { - // Diff too big, we replace above layer(s) - let mut parent_new = (*parent.added).clone(); - let mut parent_removed = (*parent.removed).clone(); + let mut parent_new = (*parent.added).clone(); + let mut parent_removed = (*parent.removed).clone(); - for removed in statediffremoved.iter() { - if !parent_new.remove(removed) { - // It was not added in the parent and we removed it - parent_removed.insert(*removed); - } - // Else it was added in the parent and we removed it again. We - // can forget this change + for removed in statediffremoved.iter() { + if !parent_new.remove(removed) { + // It was not added in the parent and we removed it + parent_removed.insert(*removed); } - - for new in statediffnew.iter() { - if !parent_removed.remove(new) { - // It was not touched in the parent and we added it - parent_new.insert(*new); - } - // Else it was removed in the parent and we added it again. We - // can forget this change - } - - self.save_state_from_diff( - shortstatehash, - Arc::new(parent_new), - Arc::new(parent_removed), - diffsum, - parent_states, - )?; - } else { - // Diff small enough, we add diff as layer on top of parent - self.save_statediff(shortstatehash, &StateDiff { - parent: Some(parent.shortstatehash), - added: statediffnew, - removed: statediffremoved, - }); + // Else it was added in the parent and we removed it again. We + // can forget this change } - Ok(()) + for new in statediffnew.iter() { + if !parent_removed.remove(new) { + // It was not touched in the parent and we added it + parent_new.insert(*new); + } + // Else it was removed in the parent and we added it again. We + // can forget this change + } + + self.save_state_from_diff( + shortstatehash, + Arc::new(parent_new), + Arc::new(parent_removed), + diffsum, + parent_states, + )?; + + return Ok(()); } - /// Returns the new shortstatehash, and the state diff from the previous - /// room state - #[tracing::instrument(skip(self, new_state_ids_compressed), level = "debug")] - pub async fn save_state( - &self, - room_id: &RoomId, - new_state_ids_compressed: Arc, - ) -> Result { - let previous_shortstatehash = self - .services - .state - .get_room_shortstatehash(room_id) - .await - .ok(); - - let state_hash = - utils::calculate_hash(new_state_ids_compressed.iter().map(|bytes| &bytes[..])); - - let (new_shortstatehash, already_existed) = self - .services - .short - .get_or_create_shortstatehash(&state_hash) - .await; - - if Some(new_shortstatehash) == previous_shortstatehash { - return Ok(HashSetCompressStateEvent { - shortstatehash: new_shortstatehash, - ..Default::default() - }); - } - - let states_parents = if let Some(p) = previous_shortstatehash { - self.load_shortstatehash_info(p).await.unwrap_or_default() - } else { - ShortStateInfoVec::new() - }; - - let (statediffnew, statediffremoved) = - if let Some(parent_stateinfo) = states_parents.last() { - let statediffnew: CompressedState = new_state_ids_compressed - .difference(&parent_stateinfo.full_state) - .copied() - .collect(); - - let statediffremoved: CompressedState = parent_stateinfo - .full_state - .difference(&new_state_ids_compressed) - .copied() - .collect(); - - (Arc::new(statediffnew), Arc::new(statediffremoved)) - } else { - (new_state_ids_compressed, Arc::new(CompressedState::new())) - }; - - if !already_existed { - self.save_state_from_diff( - new_shortstatehash, - statediffnew.clone(), - statediffremoved.clone(), - 2, // every state change is 2 event changes on average - states_parents, - )?; - } - - Ok(HashSetCompressStateEvent { - shortstatehash: new_shortstatehash, + if parent_states.is_empty() { + // There is no parent layer, create a new state + self.save_statediff(shortstatehash, &StateDiff { + parent: None, added: statediffnew, removed: statediffremoved, - }) + }); + + return Ok(()); } - #[tracing::instrument(skip(self), level = "debug", name = "get")] - async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result { - const BUFSIZE: usize = size_of::(); - const STRIDE: usize = size_of::(); + // Else we have two options. + // 1. We add the current diff on top of the parent layer. + // 2. We replace a layer above - let value = self - .db - .shortstatehash_statediff - .aqry::(&shortstatehash) - .await - .map_err(|e| { - err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")) - })?; + let parent = parent_states.pop().expect("parent must have a state"); + let parent_added_len = parent.added.len(); + let parent_removed_len = parent.removed.len(); + let parent_diff = checked!(parent_added_len + parent_removed_len)?; - let parent = utils::u64_from_bytes(&value[0..size_of::()]) - .ok() - .take_if(|parent| *parent != 0); + if checked!(diffsum * diffsum)? >= checked!(2 * diff_to_sibling * parent_diff)? { + // Diff too big, we replace above layer(s) + let mut parent_new = (*parent.added).clone(); + let mut parent_removed = (*parent.removed).clone(); - debug_assert!(value.len() % STRIDE == 0, "value not aligned to stride"); - let _num_values = value.len() / STRIDE; - - let mut add_mode = true; - let mut added = CompressedState::new(); - let mut removed = CompressedState::new(); - - let mut i = STRIDE; - while let Some(v) = value.get(i..expected!(i + 2 * STRIDE)) { - if add_mode && v.starts_with(&0_u64.to_be_bytes()) { - add_mode = false; - i = expected!(i + STRIDE); - continue; + for removed in statediffremoved.iter() { + if !parent_new.remove(removed) { + // It was not added in the parent and we removed it + parent_removed.insert(*removed); } - if add_mode { - added.insert(v.try_into()?); - } else { - removed.insert(v.try_into()?); - } - i = expected!(i + 2 * STRIDE); + // Else it was added in the parent and we removed it again. We + // can forget this change } - Ok(StateDiff { - parent, - added: Arc::new(added), - removed: Arc::new(removed), - }) + for new in statediffnew.iter() { + if !parent_removed.remove(new) { + // It was not touched in the parent and we added it + parent_new.insert(*new); + } + // Else it was removed in the parent and we added it again. We + // can forget this change + } + + self.save_state_from_diff( + shortstatehash, + Arc::new(parent_new), + Arc::new(parent_removed), + diffsum, + parent_states, + )?; + } else { + // Diff small enough, we add diff as layer on top of parent + self.save_statediff(shortstatehash, &StateDiff { + parent: Some(parent.shortstatehash), + added: statediffnew, + removed: statediffremoved, + }); } - fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) { - let mut value = Vec::::with_capacity( - 2_usize - .saturating_add(diff.added.len()) - .saturating_add(diff.removed.len()), - ); + Ok(()) +} - let parent = diff.parent.unwrap_or(0_u64); - value.extend_from_slice(&parent.to_be_bytes()); +/// Returns the new shortstatehash, and the state diff from the previous +/// room state +#[implement(Service)] +#[tracing::instrument(skip(self, new_state_ids_compressed), level = "debug")] +pub async fn save_state( + &self, + room_id: &RoomId, + new_state_ids_compressed: Arc, +) -> Result { + let previous_shortstatehash = self + .services + .state + .get_room_shortstatehash(room_id) + .await + .ok(); - for new in diff.added.iter() { - value.extend_from_slice(&new[..]); - } + let state_hash = + utils::calculate_hash(new_state_ids_compressed.iter().map(|bytes| &bytes[..])); - if !diff.removed.is_empty() { - value.extend_from_slice(&0_u64.to_be_bytes()); - for removed in diff.removed.iter() { - value.extend_from_slice(&removed[..]); - } - } + let (new_shortstatehash, already_existed) = self + .services + .short + .get_or_create_shortstatehash(&state_hash) + .await; - self.db - .shortstatehash_statediff - .insert(&shortstatehash.to_be_bytes(), &value); + if Some(new_shortstatehash) == previous_shortstatehash { + return Ok(HashSetCompressStateEvent { + shortstatehash: new_shortstatehash, + ..Default::default() + }); } + + let states_parents = if let Some(p) = previous_shortstatehash { + self.load_shortstatehash_info(p).await.unwrap_or_default() + } else { + ShortStateInfoVec::new() + }; + + let (statediffnew, statediffremoved) = if let Some(parent_stateinfo) = states_parents.last() { + let statediffnew: CompressedState = new_state_ids_compressed + .difference(&parent_stateinfo.full_state) + .copied() + .collect(); + + let statediffremoved: CompressedState = parent_stateinfo + .full_state + .difference(&new_state_ids_compressed) + .copied() + .collect(); + + (Arc::new(statediffnew), Arc::new(statediffremoved)) + } else { + (new_state_ids_compressed, Arc::new(CompressedState::new())) + }; + + if !already_existed { + self.save_state_from_diff( + new_shortstatehash, + statediffnew.clone(), + statediffremoved.clone(), + 2, // every state change is 2 event changes on average + states_parents, + )?; + } + + Ok(HashSetCompressStateEvent { + shortstatehash: new_shortstatehash, + added: statediffnew, + removed: statediffremoved, + }) +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug", name = "get")] +async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result { + const BUFSIZE: usize = size_of::(); + const STRIDE: usize = size_of::(); + + let value = self + .db + .shortstatehash_statediff + .aqry::(&shortstatehash) + .await + .map_err(|e| { + err!(Database("Failed to find StateDiff from short {shortstatehash:?}: {e}")) + })?; + + let parent = utils::u64_from_bytes(&value[0..size_of::()]) + .ok() + .take_if(|parent| *parent != 0); + + debug_assert!(value.len() % STRIDE == 0, "value not aligned to stride"); + let _num_values = value.len() / STRIDE; + + let mut add_mode = true; + let mut added = CompressedState::new(); + let mut removed = CompressedState::new(); + + let mut i = STRIDE; + while let Some(v) = value.get(i..expected!(i + 2 * STRIDE)) { + if add_mode && v.starts_with(&0_u64.to_be_bytes()) { + add_mode = false; + i = expected!(i + STRIDE); + continue; + } + if add_mode { + added.insert(v.try_into()?); + } else { + removed.insert(v.try_into()?); + } + i = expected!(i + 2 * STRIDE); + } + + Ok(StateDiff { + parent, + added: Arc::new(added), + removed: Arc::new(removed), + }) +} + +#[implement(Service)] +fn save_statediff(&self, shortstatehash: ShortStateHash, diff: &StateDiff) { + let mut value = Vec::::with_capacity( + 2_usize + .saturating_add(diff.added.len()) + .saturating_add(diff.removed.len()), + ); + + let parent = diff.parent.unwrap_or(0_u64); + value.extend_from_slice(&parent.to_be_bytes()); + + for new in diff.added.iter() { + value.extend_from_slice(&new[..]); + } + + if !diff.removed.is_empty() { + value.extend_from_slice(&0_u64.to_be_bytes()); + for removed in diff.removed.iter() { + value.extend_from_slice(&removed[..]); + } + } + + self.db + .shortstatehash_statediff + .insert(&shortstatehash.to_be_bytes(), &value); } #[inline] From 36e81ba185dbd17aa73f9d01417ec5c37f7da4c5 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Tue, 29 Apr 2025 07:28:05 +0000 Subject: [PATCH 1893/2091] Split state_cache service. Signed-off-by: Jason Volk --- src/service/rooms/state_cache/mod.rs | 1308 ++++++++--------------- src/service/rooms/state_cache/update.rs | 369 +++++++ src/service/rooms/state_cache/via.rs | 92 ++ 3 files changed, 882 insertions(+), 887 deletions(-) create mode 100644 src/service/rooms/state_cache/update.rs create mode 100644 src/service/rooms/state_cache/via.rs diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index d3dbc143..9429be79 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -1,30 +1,22 @@ +mod update; +mod via; + use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, sync::{Arc, RwLock}, }; use conduwuit::{ - Result, is_not_empty, + Result, implement, result::LogErr, - utils::{ReadyExt, StreamTools, stream::TryIgnore}, + utils::{ReadyExt, stream::TryIgnore}, warn, }; -use database::{Deserialized, Ignore, Interfix, Json, Map, serialize_key}; -use futures::{Stream, StreamExt, future::join5, pin_mut, stream::iter}; -use itertools::Itertools; +use database::{Deserialized, Ignore, Interfix, Map}; +use futures::{Stream, StreamExt, future::join5, pin_mut}; use ruma::{ - OwnedRoomId, OwnedServerName, RoomId, ServerName, UserId, - events::{ - AnyStrippedStateEvent, AnySyncStateEvent, GlobalAccountDataEventType, - RoomAccountDataEventType, StateEventType, - direct::DirectEvent, - room::{ - create::RoomCreateEventContent, - member::{MembershipState, RoomMemberEventContent}, - power_levels::RoomPowerLevelsEventContent, - }, - }, - int, + OwnedRoomId, RoomId, ServerName, UserId, + events::{AnyStrippedStateEvent, AnySyncStateEvent, room::member::MembershipState}, serde::Raw, }; @@ -101,901 +93,443 @@ impl crate::Service for Service { fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } } -impl Service { - /// Update current membership data. - #[tracing::instrument( - level = "debug", - skip_all, - fields( - %room_id, - %user_id, - %sender, - ?membership_event, - ), - )] - #[allow(clippy::too_many_arguments)] - pub async fn update_membership( - &self, - room_id: &RoomId, - user_id: &UserId, - membership_event: RoomMemberEventContent, - sender: &UserId, - last_state: Option>>, - invite_via: Option>, - update_joined_count: bool, - ) -> Result<()> { - let membership = membership_event.membership; - - // Keep track what remote users exist by adding them as "deactivated" users - // - // TODO: use futures to update remote profiles without blocking the membership - // update - #[allow(clippy::collapsible_if)] - if !self.services.globals.user_is_local(user_id) { - if !self.services.users.exists(user_id).await { - self.services.users.create(user_id, None)?; - } - - /* - // Try to update our local copy of the user if ours does not match - if ((self.services.users.displayname(user_id)? != membership_event.displayname) - || (self.services.users.avatar_url(user_id)? != membership_event.avatar_url) - || (self.services.users.blurhash(user_id)? != membership_event.blurhash)) - && (membership != MembershipState::Leave) - { - let response = self.services - .sending - .send_federation_request( - user_id.server_name(), - federation::query::get_profile_information::v1::Request { - user_id: user_id.into(), - field: None, // we want the full user's profile to update locally too - }, - ) - .await; - - self.services.users.set_displayname(user_id, response.displayname.clone()).await?; - self.services.users.set_avatar_url(user_id, response.avatar_url).await?; - self.services.users.set_blurhash(user_id, response.blurhash).await?; - }; - */ - } - - match &membership { - | MembershipState::Join => { - // Check if the user never joined this room - if !self.once_joined(user_id, room_id).await { - // Add the user ID to the join list then - self.mark_as_once_joined(user_id, room_id); - - // Check if the room has a predecessor - if let Ok(Some(predecessor)) = self - .services - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomCreate, "") - .await - .map(|content: RoomCreateEventContent| content.predecessor) - { - // Copy user settings from predecessor to the current room: - // - Push rules - // - // TODO: finish this once push rules are implemented. - // - // let mut push_rules_event_content: PushRulesEvent = account_data - // .get( - // None, - // user_id, - // EventType::PushRules, - // )?; - // - // NOTE: find where `predecessor.room_id` match - // and update to `room_id`. - // - // account_data - // .update( - // None, - // user_id, - // EventType::PushRules, - // &push_rules_event_content, - // globals, - // ) - // .ok(); - - // Copy old tags to new room - if let Ok(tag_event) = self - .services - .account_data - .get_room( - &predecessor.room_id, - user_id, - RoomAccountDataEventType::Tag, - ) - .await - { - self.services - .account_data - .update( - Some(room_id), - user_id, - RoomAccountDataEventType::Tag, - &tag_event, - ) - .await - .ok(); - } - - // Copy direct chat flag - if let Ok(mut direct_event) = self - .services - .account_data - .get_global::( - user_id, - GlobalAccountDataEventType::Direct, - ) - .await - { - let mut room_ids_updated = false; - for room_ids in direct_event.content.0.values_mut() { - if room_ids.iter().any(|r| r == &predecessor.room_id) { - room_ids.push(room_id.to_owned()); - room_ids_updated = true; - } - } - - if room_ids_updated { - self.services - .account_data - .update( - None, - user_id, - GlobalAccountDataEventType::Direct.to_string().into(), - &serde_json::to_value(&direct_event) - .expect("to json always works"), - ) - .await?; - } - } - } - } - - self.mark_as_joined(user_id, room_id); - }, - | MembershipState::Invite => { - // We want to know if the sender is ignored by the receiver - if self.services.users.user_is_ignored(sender, user_id).await { - return Ok(()); - } - - self.mark_as_invited(user_id, room_id, last_state, invite_via) - .await; - }, - | MembershipState::Leave | MembershipState::Ban => { - self.mark_as_left(user_id, room_id); - - if self.services.globals.user_is_local(user_id) - && (self.services.config.forget_forced_upon_leave - || self.services.metadata.is_banned(room_id).await - || self.services.metadata.is_disabled(room_id).await) - { - self.forget(room_id, user_id); - } - }, - | _ => {}, - } - - if update_joined_count { - self.update_joined_count(room_id).await; - } - - Ok(()) +#[implement(Service)] +#[tracing::instrument(level = "trace", skip_all)] +pub async fn appservice_in_room(&self, room_id: &RoomId, appservice: &RegistrationInfo) -> bool { + if let Some(cached) = self + .appservice_in_room_cache + .read() + .expect("locked") + .get(room_id) + .and_then(|map| map.get(&appservice.registration.id)) + .copied() + { + return cached; } - #[tracing::instrument(level = "trace", skip_all)] - pub async fn appservice_in_room( - &self, - room_id: &RoomId, - appservice: &RegistrationInfo, - ) -> bool { - if let Some(cached) = self - .appservice_in_room_cache - .read() - .expect("locked") - .get(room_id) - .and_then(|map| map.get(&appservice.registration.id)) - .copied() - { - return cached; - } + let bridge_user_id = UserId::parse_with_server_name( + appservice.registration.sender_localpart.as_str(), + self.services.globals.server_name(), + ); - let bridge_user_id = UserId::parse_with_server_name( - appservice.registration.sender_localpart.as_str(), - self.services.globals.server_name(), - ); + let Ok(bridge_user_id) = bridge_user_id.log_err() else { + return false; + }; - let Ok(bridge_user_id) = bridge_user_id.log_err() else { - return false; - }; - - let in_room = self.is_joined(&bridge_user_id, room_id).await - || self - .room_members(room_id) - .ready_any(|user_id| appservice.users.is_match(user_id.as_str())) - .await; - - self.appservice_in_room_cache - .write() - .expect("locked") - .entry(room_id.into()) - .or_default() - .insert(appservice.registration.id.clone(), in_room); - - in_room - } - - /// Direct DB function to directly mark a user as joined. It is not - /// recommended to use this directly. You most likely should use - /// `update_membership` instead - #[tracing::instrument(skip(self), level = "debug")] - pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { - let userroom_id = (user_id, room_id); - let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); - - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); - - self.db.userroomid_joined.insert(&userroom_id, []); - self.db.roomuserid_joined.insert(&roomuser_id, []); - - self.db.userroomid_invitestate.remove(&userroom_id); - self.db.roomuserid_invitecount.remove(&roomuser_id); - - self.db.userroomid_leftstate.remove(&userroom_id); - self.db.roomuserid_leftcount.remove(&roomuser_id); - - self.db.userroomid_knockedstate.remove(&userroom_id); - self.db.roomuserid_knockedcount.remove(&roomuser_id); - - self.db.roomid_inviteviaservers.remove(room_id); - } - - /// Direct DB function to directly mark a user as left. It is not - /// recommended to use this directly. You most likely should use - /// `update_membership` instead - #[tracing::instrument(skip(self), level = "debug")] - pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) { - let userroom_id = (user_id, room_id); - let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); - - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); - - // (timo) TODO - let leftstate = Vec::>::new(); - - self.db - .userroomid_leftstate - .raw_put(&userroom_id, Json(leftstate)); - self.db - .roomuserid_leftcount - .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); - - self.db.userroomid_joined.remove(&userroom_id); - self.db.roomuserid_joined.remove(&roomuser_id); - - self.db.userroomid_invitestate.remove(&userroom_id); - self.db.roomuserid_invitecount.remove(&roomuser_id); - - self.db.userroomid_knockedstate.remove(&userroom_id); - self.db.roomuserid_knockedcount.remove(&roomuser_id); - - self.db.roomid_inviteviaservers.remove(room_id); - } - - /// Direct DB function to directly mark a user as knocked. It is not - /// recommended to use this directly. You most likely should use - /// `update_membership` instead - #[tracing::instrument(skip(self), level = "debug")] - pub fn mark_as_knocked( - &self, - user_id: &UserId, - room_id: &RoomId, - knocked_state: Option>>, - ) { - let userroom_id = (user_id, room_id); - let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); - - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); - - self.db - .userroomid_knockedstate - .raw_put(&userroom_id, Json(knocked_state.unwrap_or_default())); - self.db - .roomuserid_knockedcount - .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); - - self.db.userroomid_joined.remove(&userroom_id); - self.db.roomuserid_joined.remove(&roomuser_id); - - self.db.userroomid_invitestate.remove(&userroom_id); - self.db.roomuserid_invitecount.remove(&roomuser_id); - - self.db.userroomid_leftstate.remove(&userroom_id); - self.db.roomuserid_leftcount.remove(&roomuser_id); - - self.db.roomid_inviteviaservers.remove(room_id); - } - - /// Makes a user forget a room. - #[tracing::instrument(skip(self), level = "debug")] - pub fn forget(&self, room_id: &RoomId, user_id: &UserId) { - let userroom_id = (user_id, room_id); - let roomuser_id = (room_id, user_id); - - self.db.userroomid_leftstate.del(userroom_id); - self.db.roomuserid_leftcount.del(roomuser_id); - } - - /// Returns an iterator of all servers participating in this room. - #[tracing::instrument(skip(self), level = "debug")] - pub fn room_servers<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomserverids - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, server): (Ignore, &ServerName)| server) - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn server_in_room<'a>( - &'a self, - server: &'a ServerName, - room_id: &'a RoomId, - ) -> bool { - let key = (server, room_id); - self.db.serverroomids.qry(&key).await.is_ok() - } - - /// Returns an iterator of all rooms a server participates in (as far as we - /// know). - #[tracing::instrument(skip(self), level = "debug")] - pub fn server_rooms<'a>( - &'a self, - server: &'a ServerName, - ) -> impl Stream + Send + 'a { - let prefix = (server, Interfix); - self.db - .serverroomids - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, room_id): (Ignore, &RoomId)| room_id) - } - - /// Returns true if server can see user by sharing at least one room. - #[tracing::instrument(skip(self), level = "trace")] - pub async fn server_sees_user(&self, server: &ServerName, user_id: &UserId) -> bool { - self.server_rooms(server) - .any(|room_id| self.is_joined(user_id, room_id)) - .await - } - - /// Returns true if user_a and user_b share at least one room. - #[tracing::instrument(skip(self), level = "trace")] - pub async fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> bool { - let get_shared_rooms = self.get_shared_rooms(user_a, user_b); - - pin_mut!(get_shared_rooms); - get_shared_rooms.next().await.is_some() - } - - /// List the rooms common between two users - #[tracing::instrument(skip(self), level = "debug")] - pub fn get_shared_rooms<'a>( - &'a self, - user_a: &'a UserId, - user_b: &'a UserId, - ) -> impl Stream + Send + 'a { - use conduwuit::utils::set; - - let a = self.rooms_joined(user_a); - let b = self.rooms_joined(user_b); - set::intersection_sorted_stream2(a, b) - } - - /// Returns an iterator of all joined members of a room. - #[tracing::instrument(skip(self), level = "debug")] - pub fn room_members<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomuserid_joined - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, user_id): (Ignore, &UserId)| user_id) - } - - /// Returns the number of users which are currently in a room - #[tracing::instrument(skip(self), level = "trace")] - pub async fn room_joined_count(&self, room_id: &RoomId) -> Result { - self.db.roomid_joinedcount.get(room_id).await.deserialized() - } - - #[tracing::instrument(skip(self), level = "debug")] - /// Returns an iterator of all our local users in the room, even if they're - /// deactivated/guests - pub fn local_users_in_room<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream + Send + 'a { - self.room_members(room_id) - .ready_filter(|user| self.services.globals.user_is_local(user)) - } - - /// Returns an iterator of all our local joined users in a room who are - /// active (not deactivated, not guest) - #[tracing::instrument(skip(self), level = "trace")] - pub fn active_local_users_in_room<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream + Send + 'a { - self.local_users_in_room(room_id) - .filter(|user| self.services.users.is_active(user)) - } - - /// Returns the number of users which are currently invited to a room - #[tracing::instrument(skip(self), level = "trace")] - pub async fn room_invited_count(&self, room_id: &RoomId) -> Result { - self.db - .roomid_invitedcount - .get(room_id) - .await - .deserialized() - } - - /// Returns an iterator over all User IDs who ever joined a room. - #[tracing::instrument(skip(self), level = "debug")] - pub fn room_useroncejoined<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomuseroncejoinedids - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, user_id): (Ignore, &UserId)| user_id) - } - - /// Returns an iterator over all invited members of a room. - #[tracing::instrument(skip(self), level = "debug")] - pub fn room_members_invited<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomuserid_invitecount - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, user_id): (Ignore, &UserId)| user_id) - } - - /// Returns an iterator over all knocked members of a room. - #[tracing::instrument(skip(self), level = "debug")] - pub fn room_members_knocked<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream + Send + 'a { - let prefix = (room_id, Interfix); - self.db - .roomuserid_knockedcount - .keys_prefix(&prefix) - .ignore_err() - .map(|(_, user_id): (Ignore, &UserId)| user_id) - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { - let key = (room_id, user_id); - self.db - .roomuserid_invitecount - .qry(&key) - .await - .deserialized() - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn get_knock_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { - let key = (room_id, user_id); - self.db - .roomuserid_knockedcount - .qry(&key) - .await - .deserialized() - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { - let key = (room_id, user_id); - self.db.roomuserid_leftcount.qry(&key).await.deserialized() - } - - /// Returns an iterator over all rooms this user joined. - #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_joined<'a>( - &'a self, - user_id: &'a UserId, - ) -> impl Stream + Send + 'a { - self.db - .userroomid_joined - .keys_raw_prefix(user_id) - .ignore_err() - .map(|(_, room_id): (Ignore, &RoomId)| room_id) - } - - /// Returns an iterator over all rooms a user was invited to. - #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_invited<'a>( - &'a self, - user_id: &'a UserId, - ) -> impl Stream + Send + 'a { - type KeyVal<'a> = (Key<'a>, Raw>); - type Key<'a> = (&'a UserId, &'a RoomId); - - let prefix = (user_id, Interfix); - self.db - .userroomid_invitestate - .stream_prefix(&prefix) - .ignore_err() - .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) - .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) - .ignore_err() - } - - /// Returns an iterator over all rooms a user is currently knocking. - #[tracing::instrument(skip(self), level = "trace")] - pub fn rooms_knocked<'a>( - &'a self, - user_id: &'a UserId, - ) -> impl Stream + Send + 'a { - type KeyVal<'a> = (Key<'a>, Raw>); - type Key<'a> = (&'a UserId, &'a RoomId); - - let prefix = (user_id, Interfix); - self.db - .userroomid_knockedstate - .stream_prefix(&prefix) - .ignore_err() - .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) - .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) - .ignore_err() - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn invite_state( - &self, - user_id: &UserId, - room_id: &RoomId, - ) -> Result>> { - let key = (user_id, room_id); - self.db - .userroomid_invitestate - .qry(&key) - .await - .deserialized() - .and_then(|val: Raw>| { - val.deserialize_as().map_err(Into::into) - }) - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn knock_state( - &self, - user_id: &UserId, - room_id: &RoomId, - ) -> Result>> { - let key = (user_id, room_id); - self.db - .userroomid_knockedstate - .qry(&key) - .await - .deserialized() - .and_then(|val: Raw>| { - val.deserialize_as().map_err(Into::into) - }) - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn left_state( - &self, - user_id: &UserId, - room_id: &RoomId, - ) -> Result>> { - let key = (user_id, room_id); - self.db - .userroomid_leftstate - .qry(&key) - .await - .deserialized() - .and_then(|val: Raw>| { - val.deserialize_as().map_err(Into::into) - }) - } - - /// Returns an iterator over all rooms a user left. - #[tracing::instrument(skip(self), level = "debug")] - pub fn rooms_left<'a>( - &'a self, - user_id: &'a UserId, - ) -> impl Stream + Send + 'a { - type KeyVal<'a> = (Key<'a>, Raw>>); - type Key<'a> = (&'a UserId, &'a RoomId); - - let prefix = (user_id, Interfix); - self.db - .userroomid_leftstate - .stream_prefix(&prefix) - .ignore_err() - .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) - .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) - .ignore_err() - } - - #[tracing::instrument(skip(self), level = "debug")] - pub async fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> bool { - let key = (user_id, room_id); - self.db.roomuseroncejoinedids.qry(&key).await.is_ok() - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn is_joined<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { - let key = (user_id, room_id); - self.db.userroomid_joined.qry(&key).await.is_ok() - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn is_knocked<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { - let key = (user_id, room_id); - self.db.userroomid_knockedstate.qry(&key).await.is_ok() - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> bool { - let key = (user_id, room_id); - self.db.userroomid_invitestate.qry(&key).await.is_ok() - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> bool { - let key = (user_id, room_id); - self.db.userroomid_leftstate.qry(&key).await.is_ok() - } - - #[tracing::instrument(skip(self), level = "trace")] - pub async fn user_membership( - &self, - user_id: &UserId, - room_id: &RoomId, - ) -> Option { - let states = join5( - self.is_joined(user_id, room_id), - self.is_left(user_id, room_id), - self.is_knocked(user_id, room_id), - self.is_invited(user_id, room_id), - self.once_joined(user_id, room_id), - ) - .await; - - match states { - | (true, ..) => Some(MembershipState::Join), - | (_, true, ..) => Some(MembershipState::Leave), - | (_, _, true, ..) => Some(MembershipState::Knock), - | (_, _, _, true, ..) => Some(MembershipState::Invite), - | (false, false, false, false, true) => Some(MembershipState::Ban), - | _ => None, - } - } - - #[tracing::instrument(skip(self), level = "debug")] - pub fn servers_invite_via<'a>( - &'a self, - room_id: &'a RoomId, - ) -> impl Stream + Send + 'a { - type KeyVal<'a> = (Ignore, Vec<&'a ServerName>); - - self.db - .roomid_inviteviaservers - .stream_raw_prefix(room_id) - .ignore_err() - .map(|(_, servers): KeyVal<'_>| *servers.last().expect("at least one server")) - } - - /// Gets up to five servers that are likely to be in the room in the - /// distant future. - /// - /// See - #[tracing::instrument(skip(self), level = "trace")] - pub async fn servers_route_via(&self, room_id: &RoomId) -> Result> { - let most_powerful_user_server = self - .services - .state_accessor - .room_state_get_content(room_id, &StateEventType::RoomPowerLevels, "") - .await - .map(|content: RoomPowerLevelsEventContent| { - content - .users - .iter() - .max_by_key(|(_, power)| *power) - .and_then(|x| (x.1 >= &int!(50)).then_some(x)) - .map(|(user, _power)| user.server_name().to_owned()) - }); - - let mut servers: Vec = self + let in_room = self.is_joined(&bridge_user_id, room_id).await + || self .room_members(room_id) - .counts_by(|user| user.server_name().to_owned()) - .await - .into_iter() - .sorted_by_key(|(_, users)| *users) - .map(|(server, _)| server) - .rev() - .take(5) - .collect(); - - if let Ok(Some(server)) = most_powerful_user_server { - servers.insert(0, server); - servers.truncate(5); - } - - Ok(servers) - } - - pub fn get_appservice_in_room_cache_usage(&self) -> (usize, usize) { - let cache = self.appservice_in_room_cache.read().expect("locked"); - - (cache.len(), cache.capacity()) - } - - #[tracing::instrument(level = "debug", skip_all)] - pub fn clear_appservice_in_room_cache(&self) { - self.appservice_in_room_cache - .write() - .expect("locked") - .clear(); - } - - #[tracing::instrument(level = "debug", skip(self))] - pub async fn update_joined_count(&self, room_id: &RoomId) { - let mut joinedcount = 0_u64; - let mut invitedcount = 0_u64; - let mut knockedcount = 0_u64; - let mut joined_servers = HashSet::new(); - - self.room_members(room_id) - .ready_for_each(|joined| { - joined_servers.insert(joined.server_name().to_owned()); - joinedcount = joinedcount.saturating_add(1); - }) + .ready_any(|user_id| appservice.users.is_match(user_id.as_str())) .await; - invitedcount = invitedcount.saturating_add( - self.room_members_invited(room_id) - .count() - .await - .try_into() - .unwrap_or(0), - ); + self.appservice_in_room_cache + .write() + .expect("locked") + .entry(room_id.into()) + .or_default() + .insert(appservice.registration.id.clone(), in_room); - knockedcount = knockedcount.saturating_add( - self.room_members_knocked(room_id) - .count() - .await - .try_into() - .unwrap_or(0), - ); + in_room +} - self.db.roomid_joinedcount.raw_put(room_id, joinedcount); - self.db.roomid_invitedcount.raw_put(room_id, invitedcount); - self.db - .roomuserid_knockedcount - .raw_put(room_id, knockedcount); +#[implement(Service)] +pub fn get_appservice_in_room_cache_usage(&self) -> (usize, usize) { + let cache = self.appservice_in_room_cache.read().expect("locked"); - self.room_servers(room_id) - .ready_for_each(|old_joined_server| { - if joined_servers.remove(old_joined_server) { - return; - } + (cache.len(), cache.capacity()) +} - // Server not in room anymore - let roomserver_id = (room_id, old_joined_server); - let serverroom_id = (old_joined_server, room_id); +#[implement(Service)] +#[tracing::instrument(level = "debug", skip_all)] +pub fn clear_appservice_in_room_cache(&self) { + self.appservice_in_room_cache + .write() + .expect("locked") + .clear(); +} - self.db.roomserverids.del(roomserver_id); - self.db.serverroomids.del(serverroom_id); - }) - .await; +/// Returns an iterator of all servers participating in this room. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn room_servers<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomserverids + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, server): (Ignore, &ServerName)| server) +} - // Now only new servers are in joined_servers anymore - for server in &joined_servers { - let roomserver_id = (room_id, server); - let serverroom_id = (server, room_id); +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn server_in_room<'a>(&'a self, server: &'a ServerName, room_id: &'a RoomId) -> bool { + let key = (server, room_id); + self.db.serverroomids.qry(&key).await.is_ok() +} - self.db.roomserverids.put_raw(roomserver_id, []); - self.db.serverroomids.put_raw(serverroom_id, []); - } +/// Returns an iterator of all rooms a server participates in (as far as we +/// know). +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn server_rooms<'a>( + &'a self, + server: &'a ServerName, +) -> impl Stream + Send + 'a { + let prefix = (server, Interfix); + self.db + .serverroomids + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, room_id): (Ignore, &RoomId)| room_id) +} - self.appservice_in_room_cache - .write() - .expect("locked") - .remove(room_id); - } +/// Returns true if server can see user by sharing at least one room. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn server_sees_user(&self, server: &ServerName, user_id: &UserId) -> bool { + self.server_rooms(server) + .any(|room_id| self.is_joined(user_id, room_id)) + .await +} - #[tracing::instrument(level = "debug", skip(self))] - fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) { - let key = (user_id, room_id); - self.db.roomuseroncejoinedids.put_raw(key, []); - } +/// Returns true if user_a and user_b share at least one room. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn user_sees_user(&self, user_a: &UserId, user_b: &UserId) -> bool { + let get_shared_rooms = self.get_shared_rooms(user_a, user_b); - #[tracing::instrument(level = "debug", skip(self, last_state, invite_via))] - pub async fn mark_as_invited( - &self, - user_id: &UserId, - room_id: &RoomId, - last_state: Option>>, - invite_via: Option>, - ) { - let roomuser_id = (room_id, user_id); - let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + pin_mut!(get_shared_rooms); + get_shared_rooms.next().await.is_some() +} - let userroom_id = (user_id, room_id); - let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); +/// List the rooms common between two users +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn get_shared_rooms<'a>( + &'a self, + user_a: &'a UserId, + user_b: &'a UserId, +) -> impl Stream + Send + 'a { + use conduwuit::utils::set; - self.db - .userroomid_invitestate - .raw_put(&userroom_id, Json(last_state.unwrap_or_default())); - self.db - .roomuserid_invitecount - .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); + let a = self.rooms_joined(user_a); + let b = self.rooms_joined(user_b); + set::intersection_sorted_stream2(a, b) +} - self.db.userroomid_joined.remove(&userroom_id); - self.db.roomuserid_joined.remove(&roomuser_id); +/// Returns an iterator of all joined members of a room. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn room_members<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuserid_joined + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, &UserId)| user_id) +} - self.db.userroomid_leftstate.remove(&userroom_id); - self.db.roomuserid_leftcount.remove(&roomuser_id); +/// Returns the number of users which are currently in a room +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn room_joined_count(&self, room_id: &RoomId) -> Result { + self.db.roomid_joinedcount.get(room_id).await.deserialized() +} - self.db.userroomid_knockedstate.remove(&userroom_id); - self.db.roomuserid_knockedcount.remove(&roomuser_id); +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +/// Returns an iterator of all our local users in the room, even if they're +/// deactivated/guests +pub fn local_users_in_room<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream + Send + 'a { + self.room_members(room_id) + .ready_filter(|user| self.services.globals.user_is_local(user)) +} - if let Some(servers) = invite_via.filter(is_not_empty!()) { - self.add_servers_invite_via(room_id, servers).await; - } - } +/// Returns an iterator of all our local joined users in a room who are +/// active (not deactivated, not guest) +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub fn active_local_users_in_room<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream + Send + 'a { + self.local_users_in_room(room_id) + .filter(|user| self.services.users.is_active(user)) +} - #[tracing::instrument(level = "debug", skip(self, servers))] - pub async fn add_servers_invite_via(&self, room_id: &RoomId, servers: Vec) { - let mut servers: Vec<_> = self - .servers_invite_via(room_id) - .map(ToOwned::to_owned) - .chain(iter(servers.into_iter())) - .collect() - .await; +/// Returns the number of users which are currently invited to a room +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn room_invited_count(&self, room_id: &RoomId) -> Result { + self.db + .roomid_invitedcount + .get(room_id) + .await + .deserialized() +} - servers.sort_unstable(); - servers.dedup(); +/// Returns an iterator over all User IDs who ever joined a room. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn room_useroncejoined<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuseroncejoinedids + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, &UserId)| user_id) +} - let servers = servers - .iter() - .map(|server| server.as_bytes()) - .collect_vec() - .join(&[0xFF][..]); +/// Returns an iterator over all invited members of a room. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn room_members_invited<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuserid_invitecount + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, &UserId)| user_id) +} - self.db - .roomid_inviteviaservers - .insert(room_id.as_bytes(), &servers); +/// Returns an iterator over all knocked members of a room. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn room_members_knocked<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream + Send + 'a { + let prefix = (room_id, Interfix); + self.db + .roomuserid_knockedcount + .keys_prefix(&prefix) + .ignore_err() + .map(|(_, user_id): (Ignore, &UserId)| user_id) +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn get_invite_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.db + .roomuserid_invitecount + .qry(&key) + .await + .deserialized() +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn get_knock_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.db + .roomuserid_knockedcount + .qry(&key) + .await + .deserialized() +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result { + let key = (room_id, user_id); + self.db.roomuserid_leftcount.qry(&key).await.deserialized() +} + +/// Returns an iterator over all rooms this user joined. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn rooms_joined<'a>( + &'a self, + user_id: &'a UserId, +) -> impl Stream + Send + 'a { + self.db + .userroomid_joined + .keys_raw_prefix(user_id) + .ignore_err() + .map(|(_, room_id): (Ignore, &RoomId)| room_id) +} + +/// Returns an iterator over all rooms a user was invited to. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn rooms_invited<'a>( + &'a self, + user_id: &'a UserId, +) -> impl Stream + Send + 'a { + type KeyVal<'a> = (Key<'a>, Raw>); + type Key<'a> = (&'a UserId, &'a RoomId); + + let prefix = (user_id, Interfix); + self.db + .userroomid_invitestate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) + .ignore_err() +} + +/// Returns an iterator over all rooms a user is currently knocking. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub fn rooms_knocked<'a>( + &'a self, + user_id: &'a UserId, +) -> impl Stream + Send + 'a { + type KeyVal<'a> = (Key<'a>, Raw>); + type Key<'a> = (&'a UserId, &'a RoomId); + + let prefix = (user_id, Interfix); + self.db + .userroomid_knockedstate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) + .ignore_err() +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn invite_state( + &self, + user_id: &UserId, + room_id: &RoomId, +) -> Result>> { + let key = (user_id, room_id); + self.db + .userroomid_invitestate + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn knock_state( + &self, + user_id: &UserId, + room_id: &RoomId, +) -> Result>> { + let key = (user_id, room_id); + self.db + .userroomid_knockedstate + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn left_state( + &self, + user_id: &UserId, + room_id: &RoomId, +) -> Result>> { + let key = (user_id, room_id); + self.db + .userroomid_leftstate + .qry(&key) + .await + .deserialized() + .and_then(|val: Raw>| val.deserialize_as().map_err(Into::into)) +} + +/// Returns an iterator over all rooms a user left. +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn rooms_left<'a>( + &'a self, + user_id: &'a UserId, +) -> impl Stream + Send + 'a { + type KeyVal<'a> = (Key<'a>, Raw>>); + type Key<'a> = (&'a UserId, &'a RoomId); + + let prefix = (user_id, Interfix); + self.db + .userroomid_leftstate + .stream_prefix(&prefix) + .ignore_err() + .map(|((_, room_id), state): KeyVal<'_>| (room_id.to_owned(), state)) + .map(|(room_id, state)| Ok((room_id, state.deserialize_as()?))) + .ignore_err() +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn user_membership( + &self, + user_id: &UserId, + room_id: &RoomId, +) -> Option { + let states = join5( + self.is_joined(user_id, room_id), + self.is_left(user_id, room_id), + self.is_knocked(user_id, room_id), + self.is_invited(user_id, room_id), + self.once_joined(user_id, room_id), + ) + .await; + + match states { + | (true, ..) => Some(MembershipState::Join), + | (_, true, ..) => Some(MembershipState::Leave), + | (_, _, true, ..) => Some(MembershipState::Knock), + | (_, _, _, true, ..) => Some(MembershipState::Invite), + | (false, false, false, false, true) => Some(MembershipState::Ban), + | _ => None, } } + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub async fn once_joined(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let key = (user_id, room_id); + self.db.roomuseroncejoinedids.qry(&key).await.is_ok() +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn is_joined<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_joined.qry(&key).await.is_ok() +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn is_knocked<'a>(&'a self, user_id: &'a UserId, room_id: &'a RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_knockedstate.qry(&key).await.is_ok() +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn is_invited(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_invitestate.qry(&key).await.is_ok() +} + +#[implement(Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn is_left(&self, user_id: &UserId, room_id: &RoomId) -> bool { + let key = (user_id, room_id); + self.db.userroomid_leftstate.qry(&key).await.is_ok() +} diff --git a/src/service/rooms/state_cache/update.rs b/src/service/rooms/state_cache/update.rs new file mode 100644 index 00000000..02c6bec6 --- /dev/null +++ b/src/service/rooms/state_cache/update.rs @@ -0,0 +1,369 @@ +use std::collections::HashSet; + +use conduwuit::{Result, implement, is_not_empty, utils::ReadyExt, warn}; +use database::{Json, serialize_key}; +use futures::StreamExt; +use ruma::{ + OwnedServerName, RoomId, UserId, + events::{ + AnyStrippedStateEvent, AnySyncStateEvent, GlobalAccountDataEventType, + RoomAccountDataEventType, StateEventType, + direct::DirectEvent, + room::{ + create::RoomCreateEventContent, + member::{MembershipState, RoomMemberEventContent}, + }, + }, + serde::Raw, +}; + +/// Update current membership data. +#[implement(super::Service)] +#[tracing::instrument( + level = "debug", + skip_all, + fields( + %room_id, + %user_id, + %sender, + ?membership_event, + ), + )] +#[allow(clippy::too_many_arguments)] +pub async fn update_membership( + &self, + room_id: &RoomId, + user_id: &UserId, + membership_event: RoomMemberEventContent, + sender: &UserId, + last_state: Option>>, + invite_via: Option>, + update_joined_count: bool, +) -> Result { + let membership = membership_event.membership; + + // Keep track what remote users exist by adding them as "deactivated" users + // + // TODO: use futures to update remote profiles without blocking the membership + // update + #[allow(clippy::collapsible_if)] + if !self.services.globals.user_is_local(user_id) { + if !self.services.users.exists(user_id).await { + self.services.users.create(user_id, None)?; + } + } + + match &membership { + | MembershipState::Join => { + // Check if the user never joined this room + if !self.once_joined(user_id, room_id).await { + // Add the user ID to the join list then + self.mark_as_once_joined(user_id, room_id); + + // Check if the room has a predecessor + if let Ok(Some(predecessor)) = self + .services + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomCreate, "") + .await + .map(|content: RoomCreateEventContent| content.predecessor) + { + // Copy old tags to new room + if let Ok(tag_event) = self + .services + .account_data + .get_room(&predecessor.room_id, user_id, RoomAccountDataEventType::Tag) + .await + { + self.services + .account_data + .update( + Some(room_id), + user_id, + RoomAccountDataEventType::Tag, + &tag_event, + ) + .await + .ok(); + } + + // Copy direct chat flag + if let Ok(mut direct_event) = self + .services + .account_data + .get_global::(user_id, GlobalAccountDataEventType::Direct) + .await + { + let mut room_ids_updated = false; + for room_ids in direct_event.content.0.values_mut() { + if room_ids.iter().any(|r| r == &predecessor.room_id) { + room_ids.push(room_id.to_owned()); + room_ids_updated = true; + } + } + + if room_ids_updated { + self.services + .account_data + .update( + None, + user_id, + GlobalAccountDataEventType::Direct.to_string().into(), + &serde_json::to_value(&direct_event) + .expect("to json always works"), + ) + .await?; + } + } + } + } + + self.mark_as_joined(user_id, room_id); + }, + | MembershipState::Invite => { + // We want to know if the sender is ignored by the receiver + if self.services.users.user_is_ignored(sender, user_id).await { + return Ok(()); + } + + self.mark_as_invited(user_id, room_id, last_state, invite_via) + .await; + }, + | MembershipState::Leave | MembershipState::Ban => { + self.mark_as_left(user_id, room_id); + + if self.services.globals.user_is_local(user_id) + && (self.services.config.forget_forced_upon_leave + || self.services.metadata.is_banned(room_id).await + || self.services.metadata.is_disabled(room_id).await) + { + self.forget(room_id, user_id); + } + }, + | _ => {}, + } + + if update_joined_count { + self.update_joined_count(room_id).await; + } + + Ok(()) +} + +#[implement(super::Service)] +#[tracing::instrument(level = "debug", skip(self))] +pub async fn update_joined_count(&self, room_id: &RoomId) { + let mut joinedcount = 0_u64; + let mut invitedcount = 0_u64; + let mut knockedcount = 0_u64; + let mut joined_servers = HashSet::new(); + + self.room_members(room_id) + .ready_for_each(|joined| { + joined_servers.insert(joined.server_name().to_owned()); + joinedcount = joinedcount.saturating_add(1); + }) + .await; + + invitedcount = invitedcount.saturating_add( + self.room_members_invited(room_id) + .count() + .await + .try_into() + .unwrap_or(0), + ); + + knockedcount = knockedcount.saturating_add( + self.room_members_knocked(room_id) + .count() + .await + .try_into() + .unwrap_or(0), + ); + + self.db.roomid_joinedcount.raw_put(room_id, joinedcount); + self.db.roomid_invitedcount.raw_put(room_id, invitedcount); + self.db + .roomuserid_knockedcount + .raw_put(room_id, knockedcount); + + self.room_servers(room_id) + .ready_for_each(|old_joined_server| { + if joined_servers.remove(old_joined_server) { + return; + } + + // Server not in room anymore + let roomserver_id = (room_id, old_joined_server); + let serverroom_id = (old_joined_server, room_id); + + self.db.roomserverids.del(roomserver_id); + self.db.serverroomids.del(serverroom_id); + }) + .await; + + // Now only new servers are in joined_servers anymore + for server in &joined_servers { + let roomserver_id = (room_id, server); + let serverroom_id = (server, room_id); + + self.db.roomserverids.put_raw(roomserver_id, []); + self.db.serverroomids.put_raw(serverroom_id, []); + } + + self.appservice_in_room_cache + .write() + .expect("locked") + .remove(room_id); +} + +/// Direct DB function to directly mark a user as joined. It is not +/// recommended to use this directly. You most likely should use +/// `update_membership` instead +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn mark_as_joined(&self, user_id: &UserId, room_id: &RoomId) { + let userroom_id = (user_id, room_id); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); + + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + + self.db.userroomid_joined.insert(&userroom_id, []); + self.db.roomuserid_joined.insert(&roomuser_id, []); + + self.db.userroomid_invitestate.remove(&userroom_id); + self.db.roomuserid_invitecount.remove(&roomuser_id); + + self.db.userroomid_leftstate.remove(&userroom_id); + self.db.roomuserid_leftcount.remove(&roomuser_id); + + self.db.userroomid_knockedstate.remove(&userroom_id); + self.db.roomuserid_knockedcount.remove(&roomuser_id); + + self.db.roomid_inviteviaservers.remove(room_id); +} + +/// Direct DB function to directly mark a user as left. It is not +/// recommended to use this directly. You most likely should use +/// `update_membership` instead +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn mark_as_left(&self, user_id: &UserId, room_id: &RoomId) { + let userroom_id = (user_id, room_id); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); + + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + + // (timo) TODO + let leftstate = Vec::>::new(); + + self.db + .userroomid_leftstate + .raw_put(&userroom_id, Json(leftstate)); + self.db + .roomuserid_leftcount + .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); + + self.db.userroomid_joined.remove(&userroom_id); + self.db.roomuserid_joined.remove(&roomuser_id); + + self.db.userroomid_invitestate.remove(&userroom_id); + self.db.roomuserid_invitecount.remove(&roomuser_id); + + self.db.userroomid_knockedstate.remove(&userroom_id); + self.db.roomuserid_knockedcount.remove(&roomuser_id); + + self.db.roomid_inviteviaservers.remove(room_id); +} + +/// Direct DB function to directly mark a user as knocked. It is not +/// recommended to use this directly. You most likely should use +/// `update_membership` instead +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn mark_as_knocked( + &self, + user_id: &UserId, + room_id: &RoomId, + knocked_state: Option>>, +) { + let userroom_id = (user_id, room_id); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); + + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + + self.db + .userroomid_knockedstate + .raw_put(&userroom_id, Json(knocked_state.unwrap_or_default())); + self.db + .roomuserid_knockedcount + .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); + + self.db.userroomid_joined.remove(&userroom_id); + self.db.roomuserid_joined.remove(&roomuser_id); + + self.db.userroomid_invitestate.remove(&userroom_id); + self.db.roomuserid_invitecount.remove(&roomuser_id); + + self.db.userroomid_leftstate.remove(&userroom_id); + self.db.roomuserid_leftcount.remove(&roomuser_id); + + self.db.roomid_inviteviaservers.remove(room_id); +} + +/// Makes a user forget a room. +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn forget(&self, room_id: &RoomId, user_id: &UserId) { + let userroom_id = (user_id, room_id); + let roomuser_id = (room_id, user_id); + + self.db.userroomid_leftstate.del(userroom_id); + self.db.roomuserid_leftcount.del(roomuser_id); +} + +#[implement(super::Service)] +#[tracing::instrument(level = "debug", skip(self))] +fn mark_as_once_joined(&self, user_id: &UserId, room_id: &RoomId) { + let key = (user_id, room_id); + self.db.roomuseroncejoinedids.put_raw(key, []); +} + +#[implement(super::Service)] +#[tracing::instrument(level = "debug", skip(self, last_state, invite_via))] +pub async fn mark_as_invited( + &self, + user_id: &UserId, + room_id: &RoomId, + last_state: Option>>, + invite_via: Option>, +) { + let roomuser_id = (room_id, user_id); + let roomuser_id = serialize_key(roomuser_id).expect("failed to serialize roomuser_id"); + + let userroom_id = (user_id, room_id); + let userroom_id = serialize_key(userroom_id).expect("failed to serialize userroom_id"); + + self.db + .userroomid_invitestate + .raw_put(&userroom_id, Json(last_state.unwrap_or_default())); + self.db + .roomuserid_invitecount + .raw_aput::<8, _, _>(&roomuser_id, self.services.globals.next_count().unwrap()); + + self.db.userroomid_joined.remove(&userroom_id); + self.db.roomuserid_joined.remove(&roomuser_id); + + self.db.userroomid_leftstate.remove(&userroom_id); + self.db.roomuserid_leftcount.remove(&roomuser_id); + + self.db.userroomid_knockedstate.remove(&userroom_id); + self.db.roomuserid_knockedcount.remove(&roomuser_id); + + if let Some(servers) = invite_via.filter(is_not_empty!()) { + self.add_servers_invite_via(room_id, servers).await; + } +} diff --git a/src/service/rooms/state_cache/via.rs b/src/service/rooms/state_cache/via.rs new file mode 100644 index 00000000..a818cc04 --- /dev/null +++ b/src/service/rooms/state_cache/via.rs @@ -0,0 +1,92 @@ +use conduwuit::{ + Result, implement, + utils::{StreamTools, stream::TryIgnore}, + warn, +}; +use database::Ignore; +use futures::{Stream, StreamExt, stream::iter}; +use itertools::Itertools; +use ruma::{ + OwnedServerName, RoomId, ServerName, + events::{StateEventType, room::power_levels::RoomPowerLevelsEventContent}, + int, +}; + +#[implement(super::Service)] +#[tracing::instrument(level = "debug", skip(self, servers))] +pub async fn add_servers_invite_via(&self, room_id: &RoomId, servers: Vec) { + let mut servers: Vec<_> = self + .servers_invite_via(room_id) + .map(ToOwned::to_owned) + .chain(iter(servers.into_iter())) + .collect() + .await; + + servers.sort_unstable(); + servers.dedup(); + + let servers = servers + .iter() + .map(|server| server.as_bytes()) + .collect_vec() + .join(&[0xFF][..]); + + self.db + .roomid_inviteviaservers + .insert(room_id.as_bytes(), &servers); +} + +/// Gets up to five servers that are likely to be in the room in the +/// distant future. +/// +/// See +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "trace")] +pub async fn servers_route_via(&self, room_id: &RoomId) -> Result> { + let most_powerful_user_server = self + .services + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomPowerLevels, "") + .await + .map(|content: RoomPowerLevelsEventContent| { + content + .users + .iter() + .max_by_key(|(_, power)| *power) + .and_then(|x| (x.1 >= &int!(50)).then_some(x)) + .map(|(user, _power)| user.server_name().to_owned()) + }); + + let mut servers: Vec = self + .room_members(room_id) + .counts_by(|user| user.server_name().to_owned()) + .await + .into_iter() + .sorted_by_key(|(_, users)| *users) + .map(|(server, _)| server) + .rev() + .take(5) + .collect(); + + if let Ok(Some(server)) = most_powerful_user_server { + servers.insert(0, server); + servers.truncate(5); + } + + Ok(servers) +} + +#[implement(super::Service)] +#[tracing::instrument(skip(self), level = "debug")] +pub fn servers_invite_via<'a>( + &'a self, + room_id: &'a RoomId, +) -> impl Stream + Send + 'a { + type KeyVal<'a> = (Ignore, Vec<&'a ServerName>); + + self.db + .roomid_inviteviaservers + .stream_raw_prefix(room_id) + .ignore_err() + .map(|(_, servers): KeyVal<'_>| *servers.last().expect("at least one server")) +} From 3c7c641d2d5ff03cd1262675490f79c1aa5b858f Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 14 May 2025 00:33:31 +0000 Subject: [PATCH 1894/2091] Add revoke_admin to service. Signed-off-by: Jason Volk --- src/service/admin/grant.rs | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/service/admin/grant.rs b/src/service/admin/grant.rs index 2d90ea52..0d0e3fc1 100644 --- a/src/service/admin/grant.rs +++ b/src/service/admin/grant.rs @@ -170,3 +170,56 @@ async fn set_room_tag(&self, room_id: &RoomId, user_id: &UserId, tag: &str) -> R ) .await } + +/// Demote an admin, removing its rights. +#[implement(super::Service)] +pub async fn revoke_admin(&self, user_id: &UserId) -> Result { + use MembershipState::{Invite, Join, Knock, Leave}; + + let Ok(room_id) = self.get_admin_room().await else { + return Err!(error!("No admin room available or created.")); + }; + + let state_lock = self.services.state.mutex.lock(&room_id).await; + + let event = match self + .services + .state_accessor + .get_member(&room_id, user_id) + .await + { + | Err(e) if e.is_not_found() => return Err!("{user_id} was never an admin."), + + | Err(e) => return Err!(error!(?e, "Failure occurred while attempting revoke.")), + + | Ok(event) if !matches!(event.membership, Invite | Knock | Join) => + return Err!("Cannot revoke {user_id} in membership state {:?}.", event.membership), + + | Ok(event) => { + assert!( + matches!(event.membership, Invite | Knock | Join), + "Incorrect membership state to remove user." + ); + + event + }, + }; + + self.services + .timeline + .build_and_append_pdu( + PduBuilder::state(user_id.to_string(), &RoomMemberEventContent { + membership: Leave, + reason: Some("Admin Revoked".into()), + is_direct: None, + join_authorized_via_users_server: None, + third_party_invite: None, + ..event + }), + self.services.globals.server_user.as_ref(), + &room_id, + &state_lock, + ) + .await + .map(|_| ()) +} From 143cb55ac86d386e7d228a8e4475ad121b906083 Mon Sep 17 00:00:00 2001 From: Jason Volk Date: Wed, 21 May 2025 23:06:27 +0000 Subject: [PATCH 1895/2091] Fix clippy::unnecessary-unwrap. Signed-off-by: Jason Volk --- src/service/migrations.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/service/migrations.rs b/src/service/migrations.rs index 512a7867..cee638ba 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -242,12 +242,14 @@ async fn db_lt_12(services: &Services) -> Result<()> { [".m.rules.contains_user_name", ".m.rule.contains_user_name"]; let rule = rules_list.content.get(content_rule_transformation[0]); - if rule.is_some() { - let mut rule = rule.unwrap().clone(); + + if let Some(rule) = rule { + let mut rule = rule.clone(); content_rule_transformation[1].clone_into(&mut rule.rule_id); rules_list .content .shift_remove(content_rule_transformation[0]); + rules_list.content.insert(rule); } } From 293e7243b3c08aaed71b89a16544a5e75b9105dc Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 2 Jul 2025 19:32:50 +0100 Subject: [PATCH 1896/2091] style: Fix formatting/clippy issues --- src/api/client/account.rs | 19 +++++++++++++++---- src/api/client/membership/join.rs | 2 +- src/api/client/report.rs | 1 - src/service/rooms/timeline/append.rs | 8 +++++--- src/service/rooms/timeline/create.rs | 2 +- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/api/client/account.rs b/src/api/client/account.rs index df938c17..12801e7d 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -12,14 +12,25 @@ use conduwuit_service::Services; use futures::{FutureExt, StreamExt}; use register::RegistrationKind; use ruma::{ + OwnedRoomId, UserId, api::client::{ account::{ - change_password, check_registration_token_validity, deactivate, get_3pids, get_username_availability, register::{self, LoginType}, request_3pid_management_token_via_email, request_3pid_management_token_via_msisdn, whoami, ThirdPartyIdRemovalStatus + ThirdPartyIdRemovalStatus, change_password, check_registration_token_validity, + deactivate, get_3pids, get_username_availability, + register::{self, LoginType}, + request_3pid_management_token_via_email, request_3pid_management_token_via_msisdn, + whoami, }, uiaa::{AuthFlow, AuthType, UiaaInfo}, - }, events::{ - room::{message::RoomMessageEventContent, power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}}, GlobalAccountDataEventType, StateEventType - }, push, OwnedRoomId, UserId + }, + events::{ + GlobalAccountDataEventType, StateEventType, + room::{ + message::RoomMessageEventContent, + power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent}, + }, + }, + push, }; use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH, join_room_by_id_helper}; diff --git a/src/api/client/membership/join.rs b/src/api/client/membership/join.rs index 9d19d3bc..dc170cbf 100644 --- a/src/api/client/membership/join.rs +++ b/src/api/client/membership/join.rs @@ -139,7 +139,7 @@ pub(crate) async fn join_room_by_id_or_alias_route( let sender_user = body.sender_user(); let appservice_info = &body.appservice_info; let body = &body.body; - if services.users.is_suspended(sender_user).await? { + if services.users.is_suspended(sender_user).await? { return Err!(Request(UserSuspended("You cannot perform this action while suspended."))); } diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 052329d1..60a16e1a 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -8,7 +8,6 @@ use rand::Rng; use ruma::{ EventId, OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId, api::client::{ - error::ErrorKind, report_user, room::{report_content, report_room}, }, diff --git a/src/service/rooms/timeline/append.rs b/src/service/rooms/timeline/append.rs index a7b558c2..1d404e8a 100644 --- a/src/service/rooms/timeline/append.rs +++ b/src/service/rooms/timeline/append.rs @@ -348,9 +348,11 @@ where self.services.search.index_pdu(shortroomid, &pdu_id, &body); if self.services.admin.is_admin_command(pdu, &body).await { - self.services - .admin - .command_with_sender(body, Some((pdu.event_id()).into()), pdu.sender.clone().into())?; + self.services.admin.command_with_sender( + body, + Some((pdu.event_id()).into()), + pdu.sender.clone().into(), + )?; } } }, diff --git a/src/service/rooms/timeline/create.rs b/src/service/rooms/timeline/create.rs index d890e88e..20ccaf56 100644 --- a/src/service/rooms/timeline/create.rs +++ b/src/service/rooms/timeline/create.rs @@ -110,7 +110,7 @@ pub async fn create_hash_and_sign_event( // so any other events with that same depth are illegal. warn!( "Had unsafe depth {depth} when creating non-state event in {room_id}. Cowardly \ - aborting" + aborting" ); return Err!(Request(Unknown("Unsafe depth for non-state event."))); } From 7e406445d415f87ee3ce03fe82193fbf4e128e21 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Thu, 3 Jul 2025 22:26:02 +0100 Subject: [PATCH 1897/2091] Element Web build fixes --- .forgejo/workflows/element.yml | 35 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/.forgejo/workflows/element.yml b/.forgejo/workflows/element.yml index db771197..0199e8b6 100644 --- a/.forgejo/workflows/element.yml +++ b/.forgejo/workflows/element.yml @@ -11,16 +11,16 @@ concurrency: jobs: build-and-deploy: - name: Build and Deploy Element Web + name: 🏗️ Build and Deploy runs-on: ubuntu-latest steps: - - name: Setup Node.js - uses: https://code.forgejo.org/actions/setup-node@v4 + - name: 📦 Setup Node.js + uses: https://github.com/actions/setup-node@v4 with: - node-version: "20" + node-version: "22" - - name: Clone, setup, and build Element Web + - name: 🔨 Clone, setup, and build Element Web run: | echo "Cloning Element Web..." git clone https://github.com/maunium/element-web @@ -64,7 +64,7 @@ jobs: echo "Checking for build output..." ls -la webapp/ - - name: Create config.json + - name: ⚙️ Create config.json run: | cat < ./element-web/webapp/config.json { @@ -100,28 +100,25 @@ jobs: echo "Created ./element-web/webapp/config.json" cat ./element-web/webapp/config.json - - name: Upload Artifact + - name: 📤 Upload Artifact uses: https://code.forgejo.org/actions/upload-artifact@v3 with: name: element-web path: ./element-web/webapp/ retention-days: 14 - - name: Install Wrangler + - name: 🛠️ Install Wrangler run: npm install --save-dev wrangler@latest - - name: Deploy to Cloudflare Pages (Production) - if: github.ref == 'refs/heads/main' && vars.CLOUDFLARE_PROJECT_NAME != '' + - name: 🚀 Deploy to Cloudflare Pages + if: vars.CLOUDFLARE_PROJECT_NAME != '' + id: deploy uses: https://github.com/cloudflare/wrangler-action@v3 with: accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} - command: pages deploy ./element-web/webapp --branch="main" --commit-dirty=true --project-name="${{ vars.CLOUDFLARE_PROJECT_NAME }}-element" - - - name: Deploy to Cloudflare Pages (Preview) - if: github.ref != 'refs/heads/main' && vars.CLOUDFLARE_PROJECT_NAME != '' - uses: https://github.com/cloudflare/wrangler-action@v3 - with: - accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} - apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} - command: pages deploy ./element-web/webapp --branch="${{ github.head_ref || github.ref_name }}" --commit-dirty=true --project-name="${{ vars.CLOUDFLARE_PROJECT_NAME }}-element" + command: >- + pages deploy ./element-web/webapp + --branch="${{ github.ref == 'refs/heads/main' && 'main' || github.head_ref || github.ref_name }}" + --commit-dirty=true + --project-name="${{ vars.CLOUDFLARE_PROJECT_NAME }}-element" From 52954c5b75f2ac4af62a30a9f44d527bd0511801 Mon Sep 17 00:00:00 2001 From: Nyx Date: Sun, 6 Jul 2025 14:00:42 -0500 Subject: [PATCH 1898/2091] Even more renaming --- src/admin/debug/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index bceee9ba..fb8a3002 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -32,13 +32,13 @@ pub(super) enum DebugCommand { /// the command. ParsePdu, - /// - Retrieve and print a PDU by EventID from the conduwuit database + /// - Retrieve and print a PDU by EventID from the Continuwuity database GetPdu { /// An event ID (a $ followed by the base64 reference hash) event_id: OwnedEventId, }, - /// - Retrieve and print a PDU by PduId from the conduwuit database + /// - Retrieve and print a PDU by PduId from the Continuwuity database GetShortPdu { /// Shortroomid integer shortroomid: ShortRoomId, @@ -182,7 +182,7 @@ pub(super) enum DebugCommand { event_id: Option, }, - /// - Runs a server name through conduwuit's true destination resolution + /// - Runs a server name through Continuwuity's true destination resolution /// process /// /// Useful for debugging well-known issues From af8783ee51e828b9af8cbe31ddeaef3cd4e9d137 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 6 Jul 2025 21:15:49 +0100 Subject: [PATCH 1899/2091] ci: Mirror registry images --- .forgejo/regsync/regsync.yml | 55 ++++++++++++++++++++++++++++ .forgejo/workflows/mirror-images.yml | 47 ++++++++++++++++++++++++ 2 files changed, 102 insertions(+) create mode 100644 .forgejo/regsync/regsync.yml create mode 100644 .forgejo/workflows/mirror-images.yml diff --git a/.forgejo/regsync/regsync.yml b/.forgejo/regsync/regsync.yml new file mode 100644 index 00000000..0a14db45 --- /dev/null +++ b/.forgejo/regsync/regsync.yml @@ -0,0 +1,55 @@ +version: 1 + +x-source: &source forgejo.ellis.link/continuwuation/continuwuity + +x-tags: + releases: &tags-releases + tags: + allow: + - "latest" + - "v[0-9]+\\.[0-9]+\\.[0-9]+" + - "v[0-9]+\\.[0-9]+" + - "v[0-9]+" + main: &tags-main + tags: + allow: + - "latest" + - "v[0-9]+\\.[0-9]+\\.[0-9]+" + - "v[0-9]+\\.[0-9]+" + - "v[0-9]+" + - "main" + commits: &tags-commits + tags: + allow: + - "latest" + - "v[0-9]+\\.[0-9]+\\.[0-9]+" + - "v[0-9]+\\.[0-9]+" + - "v[0-9]+" + - "main" + - "sha-[a-f0-9]+" + all: &tags-all + tags: + allow: + - ".*" + +# Registry credentials +creds: + - registry: forgejo.ellis.link + user: "{{env \"BUILTIN_REGISTRY_USER\"}}" + pass: "{{env \"BUILTIN_REGISTRY_PASSWORD\"}}" + - registry: registry.gitlab.com + user: "{{env \"GITLAB_USERNAME\"}}" + pass: "{{env \"GITLAB_TOKEN\"}}" + +# Global defaults +defaults: + parallel: 3 + interval: 2h + digestTags: true + +# Sync configuration - each registry gets different image sets +sync: + - source: *source + target: registry.gitlab.com/continuwuity/continuwuity + type: repository + <<: *tags-main diff --git a/.forgejo/workflows/mirror-images.yml b/.forgejo/workflows/mirror-images.yml new file mode 100644 index 00000000..51f60e75 --- /dev/null +++ b/.forgejo/workflows/mirror-images.yml @@ -0,0 +1,47 @@ +name: Mirror Container Images + +on: + schedule: + # Run every 2 hours + - cron: "0 */2 * * *" + workflow_dispatch: + inputs: + dry_run: + description: 'Dry run (check only, no actual mirroring)' + required: false + default: false + type: boolean + +concurrency: + group: "mirror-images" + cancel-in-progress: true + +jobs: + mirror-images: + runs-on: ubuntu-latest + env: + BUILTIN_REGISTRY_USER: ${{ secrets.BUILTIN_REGISTRY_USER }} + BUILTIN_REGISTRY_PASSWORD: ${{ secrets.BUILTIN_REGISTRY_PASSWORD }} + GITLAB_USERNAME: ${{ secrets.GITLAB_USERNAME }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install regctl + uses: https://forgejo.ellis.link/continuwuation/regclient-actions/regctl-installer@main + with: + binary: regsync + + - name: Check what images need mirroring + run: | + echo "Checking images that need mirroring..." + regsync check -c .forgejo/regsync/regsync.yml -v info + + - name: Mirror images + if: ${{ !inputs.dry_run }} + run: | + echo "Starting image mirroring..." + regsync once -c .forgejo/regsync/regsync.yml -v info From 928b7c5e4a8d30c53941703fabf4467d17df412c Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 6 Jul 2025 22:57:33 +0100 Subject: [PATCH 1900/2091] fix: Correct vars --- .forgejo/workflows/mirror-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/mirror-images.yml b/.forgejo/workflows/mirror-images.yml index 51f60e75..198832db 100644 --- a/.forgejo/workflows/mirror-images.yml +++ b/.forgejo/workflows/mirror-images.yml @@ -20,9 +20,9 @@ jobs: mirror-images: runs-on: ubuntu-latest env: - BUILTIN_REGISTRY_USER: ${{ secrets.BUILTIN_REGISTRY_USER }} + BUILTIN_REGISTRY_USER: ${{ vars.BUILTIN_REGISTRY_USER }} BUILTIN_REGISTRY_PASSWORD: ${{ secrets.BUILTIN_REGISTRY_PASSWORD }} - GITLAB_USERNAME: ${{ secrets.GITLAB_USERNAME }} + GITLAB_USERNAME: ${{ vars.GITLAB_USERNAME }} GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} steps: - name: Checkout repository From 18d12a7756620b14be48f9b37fc6e2ef6fa892ff Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 22 May 2025 13:19:30 +0100 Subject: [PATCH 1901/2091] feat: Support logging to journald with tracing-journald This stubs out on non-unix platforms. --- Cargo.lock | 12 ++++++++++++ Cargo.toml | 2 ++ arch/conduwuit.service | 4 ++++ conduwuit-example.toml | 9 +++++++++ debian/conduwuit.service | 3 +++ src/core/config/mod.rs | 18 ++++++++++++++++++ src/main/Cargo.toml | 7 +++++++ src/main/logging.rs | 37 ++++++++++++++++++++++++++++++++++++- 8 files changed, 91 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 82e7a20d..5a65a729 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -797,6 +797,7 @@ dependencies = [ "tokio-metrics", "tracing", "tracing-flame", + "tracing-journald", "tracing-opentelemetry", "tracing-subscriber", ] @@ -5178,6 +5179,17 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "tracing-journald" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0b4143302cf1022dac868d521e36e8b27691f72c84b3311750d5188ebba657" +dependencies = [ + "libc", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "tracing-log" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index b815e2b8..75e15233 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -213,6 +213,8 @@ default-features = false version = "0.3.19" default-features = false features = ["env-filter", "std", "tracing", "tracing-log", "ansi", "fmt"] +[workspace.dependencies.tracing-journald] +version = "0.3.1" [workspace.dependencies.tracing-core] version = "0.1.33" default-features = false diff --git a/arch/conduwuit.service b/arch/conduwuit.service index d5a65e4d..b66bc1da 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -17,6 +17,10 @@ DeviceAllow=char-tty StandardInput=tty-force StandardOutput=tty StandardError=journal+console + +Environment="CONTINUWUITY_LOG_TO_JOURNALD=1" +Environment="CONTINUWUITY_JOURNALD_IDENTIFIER=%N" + TTYReset=yes # uncomment to allow buffer to be cleared every restart TTYVTDisallocate=no diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 794ab870..1e403bba 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -671,6 +671,15 @@ # #log_thread_ids = false +# Enable journald logging on Unix platforms +# +#log_to_journald = false + +# The syslog identifier to use with journald logging +# Only used when journald logging is enabled +# +#journald_identifier = "conduwuit" + # OpenID token expiration/TTL in seconds. # # These are the OpenID tokens that are primarily used for Matrix account diff --git a/debian/conduwuit.service b/debian/conduwuit.service index be2f3dae..b95804d3 100644 --- a/debian/conduwuit.service +++ b/debian/conduwuit.service @@ -14,6 +14,9 @@ Type=notify Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml" +Environment="CONTINUWUITY_LOG_TO_JOURNALD=1" +Environment="CONTINUWUITY_JOURNALD_IDENTIFIER=%N" + ExecStart=/usr/sbin/conduwuit ReadWritePaths=/var/lib/conduwuit /etc/conduwuit diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index e3db4900..6b054bd6 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -811,6 +811,24 @@ pub struct Config { #[serde(default)] pub log_thread_ids: bool, + /// Enable journald logging on Unix platforms + /// + /// When enabled, log output will be sent to the systemd journal + /// This is only supported on Unix platforms + /// + /// default: false + #[cfg(target_family = "unix")] + #[serde(default)] + pub log_to_journald: bool, + + /// The syslog identifier to use with journald logging + /// + /// Only used when journald logging is enabled + /// + /// Defaults to the binary name + #[cfg(target_family = "unix")] + pub journald_identifier: Option, + /// OpenID token expiration/TTL in seconds. /// /// These are the OpenID tokens that are primarily used for Matrix account diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 0c5e2b6f..2d8d26b5 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -43,6 +43,7 @@ default = [ "io_uring", "jemalloc", "jemalloc_conf", + "journald", "media_thumbnail", "release_max_log_level", "systemd", @@ -130,6 +131,11 @@ sentry_telemetry = [ systemd = [ "conduwuit-router/systemd", ] +journald = [ # This is a stub on non-unix platforms + "dep:tracing-journald", +] + + # enable the tokio_console server ncompatible with release_max_log_level tokio_console = [ "dep:console-subscriber", @@ -183,6 +189,7 @@ tracing-opentelemetry.optional = true tracing-opentelemetry.workspace = true tracing-subscriber.workspace = true tracing.workspace = true +tracing-journald = { workspace = true, optional = true } [target.'cfg(all(not(target_env = "msvc"), target_os = "linux"))'.dependencies] hardened_malloc-rs.workspace = true diff --git a/src/main/logging.rs b/src/main/logging.rs index 36a8896c..b7beb103 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -46,6 +46,16 @@ pub(crate) fn init( .with(console_layer.with_filter(console_reload_filter)) .with(cap_layer); + // If journald logging is enabled on Unix platforms, create a separate + // subscriber for it + #[cfg(all(target_family = "unix", feature = "journald"))] + if config.log_to_journald { + println!("Initialising journald logging"); + if let Err(e) = init_journald_logging(config) { + eprintln!("Failed to initialize journald logging: {e}"); + } + } + #[cfg(feature = "sentry_telemetry")] let subscriber = { let sentry_filter = EnvFilter::try_new(&config.sentry_filter) @@ -135,6 +145,28 @@ pub(crate) fn init( Ok(ret) } +#[cfg(all(target_family = "unix", feature = "journald"))] +fn init_journald_logging(config: &Config) -> Result<()> { + use tracing_journald::Layer as JournaldLayer; + + let journald_filter = + EnvFilter::try_new(&config.log).map_err(|e| err!(Config("log", "{e}.")))?; + + let mut journald_layer = JournaldLayer::new() + .map_err(|e| err!(Config("journald", "Failed to initialize journald layer: {e}.")))?; + + if let Some(ref identifier) = config.journald_identifier { + journald_layer = journald_layer.with_syslog_identifier(identifier.to_owned()); + } + + let journald_subscriber = + Registry::default().with(journald_layer.with_filter(journald_filter)); + + let _guard = tracing::subscriber::set_default(journald_subscriber); + + Ok(()) +} + fn tokio_console_enabled(config: &Config) -> (bool, &'static str) { if !cfg!(all(feature = "tokio_console", tokio_unstable)) { return (false, ""); @@ -154,7 +186,10 @@ fn tokio_console_enabled(config: &Config) -> (bool, &'static str) { (true, "") } -fn set_global_default(subscriber: S) { +fn set_global_default(subscriber: S) +where + S: tracing::Subscriber + Send + Sync + 'static, +{ tracing::subscriber::set_global_default(subscriber) .expect("the global default tracing subscriber failed to be initialized"); } From d98ce2c7b9ff248989bea363907ba4709743febd Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 24 May 2025 00:28:09 +0100 Subject: [PATCH 1902/2091] feat: Generate admin command documentation The first part of getting admin command docs on the website. There's also the beginnings of manpage generation here, although it's kinda sus and I'm not sure how it's supposed to work. I'll leave that to anyone who wants to package it. We introduce the beginings of the xtask pattern here - we do a lot of file generation, I thought it would be best to avoid doing that on every compilation. It also helps avoid lots of runtime deps. We'll need to document generating this stuff & probably add pre-commit hooks for it, though. --- .cargo/config.toml | 2 + Cargo.lock | 121 + Cargo.toml | 10 +- conduwuit-example.toml | 13 +- docs/SUMMARY.md | 1 + docs/admin_reference.md | 2658 ++++++++++++++++++++++ src/admin/admin.rs | 2 +- src/admin/appservice/mod.rs | 2 +- src/admin/check/mod.rs | 2 +- src/admin/debug/mod.rs | 2 +- src/admin/debug/tester.rs | 2 +- src/admin/federation/mod.rs | 2 +- src/admin/media/mod.rs | 6 +- src/admin/mod.rs | 2 + src/admin/query/account_data.rs | 2 +- src/admin/query/appservice.rs | 2 +- src/admin/query/globals.rs | 2 +- src/admin/query/mod.rs | 2 +- src/admin/query/presence.rs | 2 +- src/admin/query/pusher.rs | 2 +- src/admin/query/raw.rs | 2 +- src/admin/query/resolver.rs | 2 +- src/admin/query/room_alias.rs | 2 +- src/admin/query/room_state_cache.rs | 2 +- src/admin/query/room_timeline.rs | 2 +- src/admin/query/sending.rs | 2 +- src/admin/query/short.rs | 2 +- src/admin/query/users.rs | 2 +- src/admin/room/alias.rs | 2 +- src/admin/room/directory.rs | 2 +- src/admin/room/info.rs | 2 +- src/admin/room/mod.rs | 2 +- src/admin/room/moderation.rs | 2 +- src/admin/server/mod.rs | 2 +- src/admin/user/mod.rs | 2 +- src/main/Cargo.toml | 7 + src/main/mod.rs | 14 + xtask/generate-admin-command/Cargo.toml | 26 + xtask/generate-admin-command/src/main.rs | 63 + xtask/main/Cargo.toml | 22 + xtask/main/src/main.rs | 11 + 41 files changed, 2977 insertions(+), 33 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 docs/admin_reference.md create mode 100644 src/main/mod.rs create mode 100644 xtask/generate-admin-command/Cargo.toml create mode 100644 xtask/generate-admin-command/src/main.rs create mode 100644 xtask/main/Cargo.toml create mode 100644 xtask/main/src/main.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000..35049cbc --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,2 @@ +[alias] +xtask = "run --package xtask --" diff --git a/Cargo.lock b/Cargo.lock index 5a65a729..fe8cb16d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -50,12 +50,56 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + [[package]] name = "anstyle" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.98" @@ -720,14 +764,25 @@ dependencies = [ "clap_derive", ] +[[package]] +name = "clap-markdown" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2a2617956a06d4885b490697b5307ebb09fec10b088afc18c81762d848c2339" +dependencies = [ + "clap", +] + [[package]] name = "clap_builder" version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ + "anstream", "anstyle", "clap_lex", + "strsim", ] [[package]] @@ -748,6 +803,16 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "clap_mangen" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "724842fa9b144f9b89b3f3d371a89f3455eea660361d13a554f68f8ae5d6c13a" +dependencies = [ + "clap", + "roff", +] + [[package]] name = "cmake" version = "0.1.54" @@ -763,6 +828,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2399,6 +2470,12 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "itertools" version = "0.12.1" @@ -3007,6 +3084,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "openssl-probe" version = "0.1.6" @@ -3796,6 +3879,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3" + [[package]] name = "ruma" version = "0.10.1" @@ -4637,6 +4726,12 @@ dependencies = [ "quote", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subslice" version = "0.2.3" @@ -5367,6 +5462,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.17.0" @@ -6018,6 +6119,26 @@ dependencies = [ "markup5ever", ] +[[package]] +name = "xtask" +version = "0.5.0-rc.6" +dependencies = [ + "clap", + "serde", + "serde_json", +] + +[[package]] +name = "xtask-admin-command" +version = "0.5.0-rc.6" +dependencies = [ + "clap-markdown", + "clap_builder", + "clap_mangen", + "conduwuit", + "conduwuit_admin", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 75e15233..03c5b489 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ [workspace] resolver = "2" -members = ["src/*"] +members = ["src/*", "xtask/*"] default-members = ["src/*"] [workspace.package] @@ -638,6 +638,11 @@ package = "conduwuit_build_metadata" path = "src/build_metadata" default-features = false + +[workspace.dependencies.conduwuit] +package = "conduwuit" +path = "src/main" + ############################################################################### # # Release profiles @@ -763,7 +768,8 @@ inherits = "dev" # '-Clink-arg=-Wl,-z,nodlopen', # '-Clink-arg=-Wl,-z,nodelete', #] - +[profile.dev.package.xtask-admin-command] +inherits = "dev" [profile.dev.package.conduwuit] inherits = "dev" #rustflags = [ diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 1e403bba..22ad669b 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -407,6 +407,11 @@ # invites, or create/join or otherwise modify rooms. # They are effectively read-only. # +# If you want to use this to screen people who register on your server, +# you should add a room to `auto_join_rooms` that is public, and contains +# information that new users can read (since they won't be able to DM +# anyone, or send a message, and may be confused). +# #suspend_on_register = false # Enabling this setting opens registration to anyone without restrictions. @@ -673,12 +678,18 @@ # Enable journald logging on Unix platforms # +# When enabled, log output will be sent to the systemd journal +# This is only supported on Unix platforms +# #log_to_journald = false # The syslog identifier to use with journald logging +# # Only used when journald logging is enabled # -#journald_identifier = "conduwuit" +# Defaults to the binary name +# +#journald_identifier = # OpenID token expiration/TTL in seconds. # diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index af729003..b38009a1 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -15,6 +15,7 @@ - [Appservices](appservices.md) - [Maintenance](maintenance.md) - [Troubleshooting](troubleshooting.md) +- [Admin Command Reference](admin_reference.md) - [Development](development.md) - [Contributing](contributing.md) - [Testing](development/testing.md) diff --git a/docs/admin_reference.md b/docs/admin_reference.md new file mode 100644 index 00000000..18e039e4 --- /dev/null +++ b/docs/admin_reference.md @@ -0,0 +1,2658 @@ +# Command-Line Help for `admin` + +This document contains the help content for the `admin` command-line program. + +**Command Overview:** + +* [`admin`↴](#admin) +* [`admin appservices`↴](#admin-appservices) +* [`admin appservices register`↴](#admin-appservices-register) +* [`admin appservices unregister`↴](#admin-appservices-unregister) +* [`admin appservices show-appservice-config`↴](#admin-appservices-show-appservice-config) +* [`admin appservices list-registered`↴](#admin-appservices-list-registered) +* [`admin users`↴](#admin-users) +* [`admin users create-user`↴](#admin-users-create-user) +* [`admin users reset-password`↴](#admin-users-reset-password) +* [`admin users deactivate`↴](#admin-users-deactivate) +* [`admin users deactivate-all`↴](#admin-users-deactivate-all) +* [`admin users suspend`↴](#admin-users-suspend) +* [`admin users unsuspend`↴](#admin-users-unsuspend) +* [`admin users list-users`↴](#admin-users-list-users) +* [`admin users list-joined-rooms`↴](#admin-users-list-joined-rooms) +* [`admin users force-join-room`↴](#admin-users-force-join-room) +* [`admin users force-leave-room`↴](#admin-users-force-leave-room) +* [`admin users force-demote`↴](#admin-users-force-demote) +* [`admin users make-user-admin`↴](#admin-users-make-user-admin) +* [`admin users put-room-tag`↴](#admin-users-put-room-tag) +* [`admin users delete-room-tag`↴](#admin-users-delete-room-tag) +* [`admin users get-room-tags`↴](#admin-users-get-room-tags) +* [`admin users redact-event`↴](#admin-users-redact-event) +* [`admin users force-join-list-of-local-users`↴](#admin-users-force-join-list-of-local-users) +* [`admin users force-join-all-local-users`↴](#admin-users-force-join-all-local-users) +* [`admin rooms`↴](#admin-rooms) +* [`admin rooms list-rooms`↴](#admin-rooms-list-rooms) +* [`admin rooms info`↴](#admin-rooms-info) +* [`admin rooms info list-joined-members`↴](#admin-rooms-info-list-joined-members) +* [`admin rooms info view-room-topic`↴](#admin-rooms-info-view-room-topic) +* [`admin rooms moderation`↴](#admin-rooms-moderation) +* [`admin rooms moderation ban-room`↴](#admin-rooms-moderation-ban-room) +* [`admin rooms moderation ban-list-of-rooms`↴](#admin-rooms-moderation-ban-list-of-rooms) +* [`admin rooms moderation unban-room`↴](#admin-rooms-moderation-unban-room) +* [`admin rooms moderation list-banned-rooms`↴](#admin-rooms-moderation-list-banned-rooms) +* [`admin rooms alias`↴](#admin-rooms-alias) +* [`admin rooms alias set`↴](#admin-rooms-alias-set) +* [`admin rooms alias remove`↴](#admin-rooms-alias-remove) +* [`admin rooms alias which`↴](#admin-rooms-alias-which) +* [`admin rooms alias list`↴](#admin-rooms-alias-list) +* [`admin rooms directory`↴](#admin-rooms-directory) +* [`admin rooms directory publish`↴](#admin-rooms-directory-publish) +* [`admin rooms directory unpublish`↴](#admin-rooms-directory-unpublish) +* [`admin rooms directory list`↴](#admin-rooms-directory-list) +* [`admin rooms exists`↴](#admin-rooms-exists) +* [`admin federation`↴](#admin-federation) +* [`admin federation incoming-federation`↴](#admin-federation-incoming-federation) +* [`admin federation disable-room`↴](#admin-federation-disable-room) +* [`admin federation enable-room`↴](#admin-federation-enable-room) +* [`admin federation fetch-support-well-known`↴](#admin-federation-fetch-support-well-known) +* [`admin federation remote-user-in-rooms`↴](#admin-federation-remote-user-in-rooms) +* [`admin server`↴](#admin-server) +* [`admin server uptime`↴](#admin-server-uptime) +* [`admin server show-config`↴](#admin-server-show-config) +* [`admin server reload-config`↴](#admin-server-reload-config) +* [`admin server list-features`↴](#admin-server-list-features) +* [`admin server memory-usage`↴](#admin-server-memory-usage) +* [`admin server clear-caches`↴](#admin-server-clear-caches) +* [`admin server backup-database`↴](#admin-server-backup-database) +* [`admin server list-backups`↴](#admin-server-list-backups) +* [`admin server admin-notice`↴](#admin-server-admin-notice) +* [`admin server reload-mods`↴](#admin-server-reload-mods) +* [`admin server restart`↴](#admin-server-restart) +* [`admin server shutdown`↴](#admin-server-shutdown) +* [`admin media`↴](#admin-media) +* [`admin media delete`↴](#admin-media-delete) +* [`admin media delete-list`↴](#admin-media-delete-list) +* [`admin media delete-past-remote-media`↴](#admin-media-delete-past-remote-media) +* [`admin media delete-all-from-user`↴](#admin-media-delete-all-from-user) +* [`admin media delete-all-from-server`↴](#admin-media-delete-all-from-server) +* [`admin media get-file-info`↴](#admin-media-get-file-info) +* [`admin media get-remote-file`↴](#admin-media-get-remote-file) +* [`admin media get-remote-thumbnail`↴](#admin-media-get-remote-thumbnail) +* [`admin check`↴](#admin-check) +* [`admin check check-all-users`↴](#admin-check-check-all-users) +* [`admin debug`↴](#admin-debug) +* [`admin debug echo`↴](#admin-debug-echo) +* [`admin debug get-auth-chain`↴](#admin-debug-get-auth-chain) +* [`admin debug parse-pdu`↴](#admin-debug-parse-pdu) +* [`admin debug get-pdu`↴](#admin-debug-get-pdu) +* [`admin debug get-short-pdu`↴](#admin-debug-get-short-pdu) +* [`admin debug get-remote-pdu`↴](#admin-debug-get-remote-pdu) +* [`admin debug get-remote-pdu-list`↴](#admin-debug-get-remote-pdu-list) +* [`admin debug get-room-state`↴](#admin-debug-get-room-state) +* [`admin debug get-signing-keys`↴](#admin-debug-get-signing-keys) +* [`admin debug get-verify-keys`↴](#admin-debug-get-verify-keys) +* [`admin debug ping`↴](#admin-debug-ping) +* [`admin debug force-device-list-updates`↴](#admin-debug-force-device-list-updates) +* [`admin debug change-log-level`↴](#admin-debug-change-log-level) +* [`admin debug sign-json`↴](#admin-debug-sign-json) +* [`admin debug verify-json`↴](#admin-debug-verify-json) +* [`admin debug verify-pdu`↴](#admin-debug-verify-pdu) +* [`admin debug first-pdu-in-room`↴](#admin-debug-first-pdu-in-room) +* [`admin debug latest-pdu-in-room`↴](#admin-debug-latest-pdu-in-room) +* [`admin debug force-set-room-state-from-server`↴](#admin-debug-force-set-room-state-from-server) +* [`admin debug resolve-true-destination`↴](#admin-debug-resolve-true-destination) +* [`admin debug memory-stats`↴](#admin-debug-memory-stats) +* [`admin debug runtime-metrics`↴](#admin-debug-runtime-metrics) +* [`admin debug runtime-interval`↴](#admin-debug-runtime-interval) +* [`admin debug time`↴](#admin-debug-time) +* [`admin debug list-dependencies`↴](#admin-debug-list-dependencies) +* [`admin debug database-stats`↴](#admin-debug-database-stats) +* [`admin debug trim-memory`↴](#admin-debug-trim-memory) +* [`admin debug database-files`↴](#admin-debug-database-files) +* [`admin query`↴](#admin-query) +* [`admin query account-data`↴](#admin-query-account-data) +* [`admin query account-data changes-since`↴](#admin-query-account-data-changes-since) +* [`admin query account-data account-data-get`↴](#admin-query-account-data-account-data-get) +* [`admin query appservice`↴](#admin-query-appservice) +* [`admin query appservice get-registration`↴](#admin-query-appservice-get-registration) +* [`admin query appservice all`↴](#admin-query-appservice-all) +* [`admin query presence`↴](#admin-query-presence) +* [`admin query presence get-presence`↴](#admin-query-presence-get-presence) +* [`admin query presence presence-since`↴](#admin-query-presence-presence-since) +* [`admin query room-alias`↴](#admin-query-room-alias) +* [`admin query room-alias resolve-local-alias`↴](#admin-query-room-alias-resolve-local-alias) +* [`admin query room-alias local-aliases-for-room`↴](#admin-query-room-alias-local-aliases-for-room) +* [`admin query room-alias all-local-aliases`↴](#admin-query-room-alias-all-local-aliases) +* [`admin query room-state-cache`↴](#admin-query-room-state-cache) +* [`admin query room-state-cache server-in-room`↴](#admin-query-room-state-cache-server-in-room) +* [`admin query room-state-cache room-servers`↴](#admin-query-room-state-cache-room-servers) +* [`admin query room-state-cache server-rooms`↴](#admin-query-room-state-cache-server-rooms) +* [`admin query room-state-cache room-members`↴](#admin-query-room-state-cache-room-members) +* [`admin query room-state-cache local-users-in-room`↴](#admin-query-room-state-cache-local-users-in-room) +* [`admin query room-state-cache active-local-users-in-room`↴](#admin-query-room-state-cache-active-local-users-in-room) +* [`admin query room-state-cache room-joined-count`↴](#admin-query-room-state-cache-room-joined-count) +* [`admin query room-state-cache room-invited-count`↴](#admin-query-room-state-cache-room-invited-count) +* [`admin query room-state-cache room-user-once-joined`↴](#admin-query-room-state-cache-room-user-once-joined) +* [`admin query room-state-cache room-members-invited`↴](#admin-query-room-state-cache-room-members-invited) +* [`admin query room-state-cache get-invite-count`↴](#admin-query-room-state-cache-get-invite-count) +* [`admin query room-state-cache get-left-count`↴](#admin-query-room-state-cache-get-left-count) +* [`admin query room-state-cache rooms-joined`↴](#admin-query-room-state-cache-rooms-joined) +* [`admin query room-state-cache rooms-left`↴](#admin-query-room-state-cache-rooms-left) +* [`admin query room-state-cache rooms-invited`↴](#admin-query-room-state-cache-rooms-invited) +* [`admin query room-state-cache invite-state`↴](#admin-query-room-state-cache-invite-state) +* [`admin query room-timeline`↴](#admin-query-room-timeline) +* [`admin query room-timeline pdus`↴](#admin-query-room-timeline-pdus) +* [`admin query room-timeline last`↴](#admin-query-room-timeline-last) +* [`admin query globals`↴](#admin-query-globals) +* [`admin query globals database-version`↴](#admin-query-globals-database-version) +* [`admin query globals current-count`↴](#admin-query-globals-current-count) +* [`admin query globals last-check-for-announcements-id`↴](#admin-query-globals-last-check-for-announcements-id) +* [`admin query globals signing-keys-for`↴](#admin-query-globals-signing-keys-for) +* [`admin query sending`↴](#admin-query-sending) +* [`admin query sending active-requests`↴](#admin-query-sending-active-requests) +* [`admin query sending active-requests-for`↴](#admin-query-sending-active-requests-for) +* [`admin query sending queued-requests`↴](#admin-query-sending-queued-requests) +* [`admin query sending get-latest-edu-count`↴](#admin-query-sending-get-latest-edu-count) +* [`admin query users`↴](#admin-query-users) +* [`admin query users count-users`↴](#admin-query-users-count-users) +* [`admin query users iter-users`↴](#admin-query-users-iter-users) +* [`admin query users iter-users2`↴](#admin-query-users-iter-users2) +* [`admin query users password-hash`↴](#admin-query-users-password-hash) +* [`admin query users list-devices`↴](#admin-query-users-list-devices) +* [`admin query users list-devices-metadata`↴](#admin-query-users-list-devices-metadata) +* [`admin query users get-device-metadata`↴](#admin-query-users-get-device-metadata) +* [`admin query users get-devices-version`↴](#admin-query-users-get-devices-version) +* [`admin query users count-one-time-keys`↴](#admin-query-users-count-one-time-keys) +* [`admin query users get-device-keys`↴](#admin-query-users-get-device-keys) +* [`admin query users get-user-signing-key`↴](#admin-query-users-get-user-signing-key) +* [`admin query users get-master-key`↴](#admin-query-users-get-master-key) +* [`admin query users get-to-device-events`↴](#admin-query-users-get-to-device-events) +* [`admin query users get-latest-backup`↴](#admin-query-users-get-latest-backup) +* [`admin query users get-latest-backup-version`↴](#admin-query-users-get-latest-backup-version) +* [`admin query users get-backup-algorithm`↴](#admin-query-users-get-backup-algorithm) +* [`admin query users get-all-backups`↴](#admin-query-users-get-all-backups) +* [`admin query users get-room-backups`↴](#admin-query-users-get-room-backups) +* [`admin query users get-backup-session`↴](#admin-query-users-get-backup-session) +* [`admin query users get-shared-rooms`↴](#admin-query-users-get-shared-rooms) +* [`admin query resolver`↴](#admin-query-resolver) +* [`admin query resolver destinations-cache`↴](#admin-query-resolver-destinations-cache) +* [`admin query resolver overrides-cache`↴](#admin-query-resolver-overrides-cache) +* [`admin query pusher`↴](#admin-query-pusher) +* [`admin query pusher get-pushers`↴](#admin-query-pusher-get-pushers) +* [`admin query short`↴](#admin-query-short) +* [`admin query short short-event-id`↴](#admin-query-short-short-event-id) +* [`admin query short short-room-id`↴](#admin-query-short-short-room-id) +* [`admin query raw`↴](#admin-query-raw) +* [`admin query raw raw-maps`↴](#admin-query-raw-raw-maps) +* [`admin query raw raw-get`↴](#admin-query-raw-raw-get) +* [`admin query raw raw-del`↴](#admin-query-raw-raw-del) +* [`admin query raw raw-keys`↴](#admin-query-raw-raw-keys) +* [`admin query raw raw-keys-sizes`↴](#admin-query-raw-raw-keys-sizes) +* [`admin query raw raw-keys-total`↴](#admin-query-raw-raw-keys-total) +* [`admin query raw raw-vals-sizes`↴](#admin-query-raw-raw-vals-sizes) +* [`admin query raw raw-vals-total`↴](#admin-query-raw-raw-vals-total) +* [`admin query raw raw-iter`↴](#admin-query-raw-raw-iter) +* [`admin query raw raw-keys-from`↴](#admin-query-raw-raw-keys-from) +* [`admin query raw raw-iter-from`↴](#admin-query-raw-raw-iter-from) +* [`admin query raw raw-count`↴](#admin-query-raw-raw-count) +* [`admin query raw compact`↴](#admin-query-raw-compact) + +## `admin` + +**Usage:** `admin ` + +###### **Subcommands:** + +* `appservices` — - Commands for managing appservices +* `users` — - Commands for managing local users +* `rooms` — - Commands for managing rooms +* `federation` — - Commands for managing federation +* `server` — - Commands for managing the server +* `media` — - Commands for managing media +* `check` — - Commands for checking integrity +* `debug` — - Commands for debugging things +* `query` — - Low-level queries for database getters and iterators + + + +## `admin appservices` + +- Commands for managing appservices + +**Usage:** `admin appservices ` + +###### **Subcommands:** + +* `register` — - Register an appservice using its registration YAML +* `unregister` — - Unregister an appservice using its ID +* `show-appservice-config` — - Show an appservice's config using its ID +* `list-registered` — - List all the currently registered appservices + + + +## `admin appservices register` + +- Register an appservice using its registration YAML + +This command needs a YAML generated by an appservice (such as a bridge), which must be provided in a Markdown code block below the command. + +Registering a new bridge using the ID of an existing bridge will replace the old one. + +**Usage:** `admin appservices register` + + + +## `admin appservices unregister` + +- Unregister an appservice using its ID + +You can find the ID using the `list-appservices` command. + +**Usage:** `admin appservices unregister ` + +###### **Arguments:** + +* `` — The appservice to unregister + + + +## `admin appservices show-appservice-config` + +- Show an appservice's config using its ID + +You can find the ID using the `list-appservices` command. + +**Usage:** `admin appservices show-appservice-config ` + +###### **Arguments:** + +* `` — The appservice to show + + + +## `admin appservices list-registered` + +- List all the currently registered appservices + +**Usage:** `admin appservices list-registered` + + + +## `admin users` + +- Commands for managing local users + +**Usage:** `admin users ` + +###### **Subcommands:** + +* `create-user` — - Create a new user +* `reset-password` — - Reset user password +* `deactivate` — - Deactivate a user +* `deactivate-all` — - Deactivate a list of users +* `suspend` — - Suspend a user +* `unsuspend` — - Unsuspend a user +* `list-users` — - List local users in the database +* `list-joined-rooms` — - Lists all the rooms (local and remote) that the specified user is joined in +* `force-join-room` — - Manually join a local user to a room +* `force-leave-room` — - Manually leave a local user from a room +* `force-demote` — - Forces the specified user to drop their power levels to the room default, if their permissions allow and the auth check permits +* `make-user-admin` — - Grant server-admin privileges to a user +* `put-room-tag` — - Puts a room tag for the specified user and room ID +* `delete-room-tag` — - Deletes the room tag for the specified user and room ID +* `get-room-tags` — - Gets all the room tags for the specified user and room ID +* `redact-event` — - Attempts to forcefully redact the specified event ID from the sender user +* `force-join-list-of-local-users` — - Force joins a specified list of local users to join the specified room +* `force-join-all-local-users` — - Force joins all local users to the specified room + + + +## `admin users create-user` + +- Create a new user + +**Usage:** `admin users create-user [PASSWORD]` + +###### **Arguments:** + +* `` — Username of the new user +* `` — Password of the new user, if unspecified one is generated + + + +## `admin users reset-password` + +- Reset user password + +**Usage:** `admin users reset-password [PASSWORD]` + +###### **Arguments:** + +* `` — Username of the user for whom the password should be reset +* `` — New password for the user, if unspecified one is generated + + + +## `admin users deactivate` + +- Deactivate a user + +User will be removed from all rooms by default. Use --no-leave-rooms to not leave all rooms by default. + +**Usage:** `admin users deactivate [OPTIONS] ` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `-n`, `--no-leave-rooms` + + + +## `admin users deactivate-all` + +- Deactivate a list of users + +Recommended to use in conjunction with list-local-users. + +Users will be removed from joined rooms by default. + +Can be overridden with --no-leave-rooms. + +Removing a mass amount of users from a room may cause a significant amount of leave events. The time to leave rooms may depend significantly on joined rooms and servers. + +This command needs a newline separated list of users provided in a Markdown code block below the command. + +**Usage:** `admin users deactivate-all [OPTIONS]` + +###### **Options:** + +* `-n`, `--no-leave-rooms` — Does not leave any rooms the user is in on deactivation +* `-f`, `--force` — Also deactivate admin accounts and will assume leave all rooms too + + + +## `admin users suspend` + +- Suspend a user + +Suspended users are able to log in, sync, and read messages, but are not able to send events nor redact them, cannot change their profile, and are unable to join, invite to, or knock on rooms. + +Suspended users can still leave rooms and deactivate their account. Suspending them effectively makes them read-only. + +**Usage:** `admin users suspend ` + +###### **Arguments:** + +* `` — Username of the user to suspend + + + +## `admin users unsuspend` + +- Unsuspend a user + +Reverses the effects of the `suspend` command, allowing the user to send messages, change their profile, create room invites, etc. + +**Usage:** `admin users unsuspend ` + +###### **Arguments:** + +* `` — Username of the user to unsuspend + + + +## `admin users list-users` + +- List local users in the database + +**Usage:** `admin users list-users` + + + +## `admin users list-joined-rooms` + +- Lists all the rooms (local and remote) that the specified user is joined in + +**Usage:** `admin users list-joined-rooms ` + +###### **Arguments:** + +* `` + + + +## `admin users force-join-room` + +- Manually join a local user to a room + +**Usage:** `admin users force-join-room ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin users force-leave-room` + +- Manually leave a local user from a room + +**Usage:** `admin users force-leave-room ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin users force-demote` + +- Forces the specified user to drop their power levels to the room default, if their permissions allow and the auth check permits + +**Usage:** `admin users force-demote ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin users make-user-admin` + +- Grant server-admin privileges to a user + +**Usage:** `admin users make-user-admin ` + +###### **Arguments:** + +* `` + + + +## `admin users put-room-tag` + +- Puts a room tag for the specified user and room ID. + +This is primarily useful if you'd like to set your admin room to the special "System Alerts" section in Element as a way to permanently see your admin room without it being buried away in your favourites or rooms. To do this, you would pass your user, your admin room's internal ID, and the tag name `m.server_notice`. + +**Usage:** `admin users put-room-tag ` + +###### **Arguments:** + +* `` +* `` +* `` + + + +## `admin users delete-room-tag` + +- Deletes the room tag for the specified user and room ID + +**Usage:** `admin users delete-room-tag ` + +###### **Arguments:** + +* `` +* `` +* `` + + + +## `admin users get-room-tags` + +- Gets all the room tags for the specified user and room ID + +**Usage:** `admin users get-room-tags ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin users redact-event` + +- Attempts to forcefully redact the specified event ID from the sender user + +This is only valid for local users + +**Usage:** `admin users redact-event ` + +###### **Arguments:** + +* `` + + + +## `admin users force-join-list-of-local-users` + +- Force joins a specified list of local users to join the specified room. + +Specify a codeblock of usernames. + +At least 1 server admin must be in the room to reduce abuse. + +Requires the `--yes-i-want-to-do-this` flag. + +**Usage:** `admin users force-join-list-of-local-users [OPTIONS] ` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--yes-i-want-to-do-this` + + + +## `admin users force-join-all-local-users` + +- Force joins all local users to the specified room. + +At least 1 server admin must be in the room to reduce abuse. + +Requires the `--yes-i-want-to-do-this` flag. + +**Usage:** `admin users force-join-all-local-users [OPTIONS] ` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--yes-i-want-to-do-this` + + + +## `admin rooms` + +- Commands for managing rooms + +**Usage:** `admin rooms ` + +###### **Subcommands:** + +* `list-rooms` — - List all rooms the server knows about +* `info` — - View information about a room we know about +* `moderation` — - Manage moderation of remote or local rooms +* `alias` — - Manage rooms' aliases +* `directory` — - Manage the room directory +* `exists` — - Check if we know about a room + + + +## `admin rooms list-rooms` + +- List all rooms the server knows about + +**Usage:** `admin rooms list-rooms [OPTIONS] [PAGE]` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--exclude-disabled` — Excludes rooms that we have federation disabled with +* `--exclude-banned` — Excludes rooms that we have banned +* `--no-details` — Whether to only output room IDs without supplementary room information + + + +## `admin rooms info` + +- View information about a room we know about + +**Usage:** `admin rooms info ` + +###### **Subcommands:** + +* `list-joined-members` — - List joined members in a room +* `view-room-topic` — - Displays room topic + + + +## `admin rooms info list-joined-members` + +- List joined members in a room + +**Usage:** `admin rooms info list-joined-members [OPTIONS] ` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--local-only` — Lists only our local users in the specified room + + + +## `admin rooms info view-room-topic` + +- Displays room topic + +Room topics can be huge, so this is in its own separate command + +**Usage:** `admin rooms info view-room-topic ` + +###### **Arguments:** + +* `` + + + +## `admin rooms moderation` + +- Manage moderation of remote or local rooms + +**Usage:** `admin rooms moderation ` + +###### **Subcommands:** + +* `ban-room` — - Bans a room from local users joining and evicts all our local users (including server admins) from the room. Also blocks any invites (local and remote) for the banned room, and disables federation entirely with it +* `ban-list-of-rooms` — - Bans a list of rooms (room IDs and room aliases) from a newline delimited codeblock similar to `user deactivate-all`. Applies the same steps as ban-room +* `unban-room` — - Unbans a room to allow local users to join again +* `list-banned-rooms` — - List of all rooms we have banned + + + +## `admin rooms moderation ban-room` + +- Bans a room from local users joining and evicts all our local users (including server admins) from the room. Also blocks any invites (local and remote) for the banned room, and disables federation entirely with it + +**Usage:** `admin rooms moderation ban-room ` + +###### **Arguments:** + +* `` — The room in the format of `!roomid:example.com` or a room alias in the format of `#roomalias:example.com` + + + +## `admin rooms moderation ban-list-of-rooms` + +- Bans a list of rooms (room IDs and room aliases) from a newline delimited codeblock similar to `user deactivate-all`. Applies the same steps as ban-room + +**Usage:** `admin rooms moderation ban-list-of-rooms` + + + +## `admin rooms moderation unban-room` + +- Unbans a room to allow local users to join again + +**Usage:** `admin rooms moderation unban-room ` + +###### **Arguments:** + +* `` — The room in the format of `!roomid:example.com` or a room alias in the format of `#roomalias:example.com` + + + +## `admin rooms moderation list-banned-rooms` + +- List of all rooms we have banned + +**Usage:** `admin rooms moderation list-banned-rooms [OPTIONS]` + +###### **Options:** + +* `--no-details` — Whether to only output room IDs without supplementary room information + + + +## `admin rooms alias` + +- Manage rooms' aliases + +**Usage:** `admin rooms alias ` + +###### **Subcommands:** + +* `set` — - Make an alias point to a room +* `remove` — - Remove a local alias +* `which` — - Show which room is using an alias +* `list` — - List aliases currently being used + + + +## `admin rooms alias set` + +- Make an alias point to a room + +**Usage:** `admin rooms alias set [OPTIONS] ` + +###### **Arguments:** + +* `` — The room id to set the alias on +* `` — The alias localpart to use (`alias`, not `#alias:servername.tld`) + +###### **Options:** + +* `-f`, `--force` — Set the alias even if a room is already using it + + + +## `admin rooms alias remove` + +- Remove a local alias + +**Usage:** `admin rooms alias remove ` + +###### **Arguments:** + +* `` — The alias localpart to remove (`alias`, not `#alias:servername.tld`) + + + +## `admin rooms alias which` + +- Show which room is using an alias + +**Usage:** `admin rooms alias which ` + +###### **Arguments:** + +* `` — The alias localpart to look up (`alias`, not `#alias:servername.tld`) + + + +## `admin rooms alias list` + +- List aliases currently being used + +**Usage:** `admin rooms alias list [ROOM_ID]` + +###### **Arguments:** + +* `` — If set, only list the aliases for this room + + + +## `admin rooms directory` + +- Manage the room directory + +**Usage:** `admin rooms directory ` + +###### **Subcommands:** + +* `publish` — - Publish a room to the room directory +* `unpublish` — - Unpublish a room to the room directory +* `list` — - List rooms that are published + + + +## `admin rooms directory publish` + +- Publish a room to the room directory + +**Usage:** `admin rooms directory publish ` + +###### **Arguments:** + +* `` — The room id of the room to publish + + + +## `admin rooms directory unpublish` + +- Unpublish a room to the room directory + +**Usage:** `admin rooms directory unpublish ` + +###### **Arguments:** + +* `` — The room id of the room to unpublish + + + +## `admin rooms directory list` + +- List rooms that are published + +**Usage:** `admin rooms directory list [PAGE]` + +###### **Arguments:** + +* `` + + + +## `admin rooms exists` + +- Check if we know about a room + +**Usage:** `admin rooms exists ` + +###### **Arguments:** + +* `` + + + +## `admin federation` + +- Commands for managing federation + +**Usage:** `admin federation ` + +###### **Subcommands:** + +* `incoming-federation` — - List all rooms we are currently handling an incoming pdu from +* `disable-room` — - Disables incoming federation handling for a room +* `enable-room` — - Enables incoming federation handling for a room again +* `fetch-support-well-known` — - Fetch `/.well-known/matrix/support` from the specified server +* `remote-user-in-rooms` — - Lists all the rooms we share/track with the specified *remote* user + + + +## `admin federation incoming-federation` + +- List all rooms we are currently handling an incoming pdu from + +**Usage:** `admin federation incoming-federation` + + + +## `admin federation disable-room` + +- Disables incoming federation handling for a room + +**Usage:** `admin federation disable-room ` + +###### **Arguments:** + +* `` + + + +## `admin federation enable-room` + +- Enables incoming federation handling for a room again + +**Usage:** `admin federation enable-room ` + +###### **Arguments:** + +* `` + + + +## `admin federation fetch-support-well-known` + +- Fetch `/.well-known/matrix/support` from the specified server + +Despite the name, this is not a federation endpoint and does not go through the federation / server resolution process as per-spec this is supposed to be served at the server_name. + +Respecting homeservers put this file here for listing administration, moderation, and security inquiries. This command provides a way to easily fetch that information. + +**Usage:** `admin federation fetch-support-well-known ` + +###### **Arguments:** + +* `` + + + +## `admin federation remote-user-in-rooms` + +- Lists all the rooms we share/track with the specified *remote* user + +**Usage:** `admin federation remote-user-in-rooms ` + +###### **Arguments:** + +* `` + + + +## `admin server` + +- Commands for managing the server + +**Usage:** `admin server ` + +###### **Subcommands:** + +* `uptime` — - Time elapsed since startup +* `show-config` — - Show configuration values +* `reload-config` — - Reload configuration values +* `list-features` — - List the features built into the server +* `memory-usage` — - Print database memory usage statistics +* `clear-caches` — - Clears all of Continuwuity's caches +* `backup-database` — - Performs an online backup of the database (only available for RocksDB at the moment) +* `list-backups` — - List database backups +* `admin-notice` — - Send a message to the admin room +* `reload-mods` — - Hot-reload the server +* `restart` — - Restart the server +* `shutdown` — - Shutdown the server + + + +## `admin server uptime` + +- Time elapsed since startup + +**Usage:** `admin server uptime` + + + +## `admin server show-config` + +- Show configuration values + +**Usage:** `admin server show-config` + + + +## `admin server reload-config` + +- Reload configuration values + +**Usage:** `admin server reload-config [PATH]` + +###### **Arguments:** + +* `` + + + +## `admin server list-features` + +- List the features built into the server + +**Usage:** `admin server list-features [OPTIONS]` + +###### **Options:** + +* `-a`, `--available` +* `-e`, `--enabled` +* `-c`, `--comma` + + + +## `admin server memory-usage` + +- Print database memory usage statistics + +**Usage:** `admin server memory-usage` + + + +## `admin server clear-caches` + +- Clears all of Continuwuity's caches + +**Usage:** `admin server clear-caches` + + + +## `admin server backup-database` + +- Performs an online backup of the database (only available for RocksDB at the moment) + +**Usage:** `admin server backup-database` + + + +## `admin server list-backups` + +- List database backups + +**Usage:** `admin server list-backups` + + + +## `admin server admin-notice` + +- Send a message to the admin room + +**Usage:** `admin server admin-notice [MESSAGE]...` + +###### **Arguments:** + +* `` + + + +## `admin server reload-mods` + +- Hot-reload the server + +**Usage:** `admin server reload-mods` + + + +## `admin server restart` + +- Restart the server + +**Usage:** `admin server restart [OPTIONS]` + +###### **Options:** + +* `-f`, `--force` + + + +## `admin server shutdown` + +- Shutdown the server + +**Usage:** `admin server shutdown` + + + +## `admin media` + +- Commands for managing media + +**Usage:** `admin media ` + +###### **Subcommands:** + +* `delete` — - Deletes a single media file from our database and on the filesystem via a single MXC URL or event ID (not redacted) +* `delete-list` — - Deletes a codeblock list of MXC URLs from our database and on the filesystem. This will always ignore errors +* `delete-past-remote-media` — - Deletes all remote (and optionally local) media created before or after [duration] time using filesystem metadata first created at date, or fallback to last modified date. This will always ignore errors by default +* `delete-all-from-user` — - Deletes all the local media from a local user on our server. This will always ignore errors by default +* `delete-all-from-server` — - Deletes all remote media from the specified remote server. This will always ignore errors by default +* `get-file-info` — +* `get-remote-file` — +* `get-remote-thumbnail` — + + + +## `admin media delete` + +- Deletes a single media file from our database and on the filesystem via a single MXC URL or event ID (not redacted) + +**Usage:** `admin media delete [OPTIONS]` + +###### **Options:** + +* `--mxc ` — The MXC URL to delete +* `--event-id ` — - The message event ID which contains the media and thumbnail MXC URLs + + + +## `admin media delete-list` + +- Deletes a codeblock list of MXC URLs from our database and on the filesystem. This will always ignore errors + +**Usage:** `admin media delete-list` + + + +## `admin media delete-past-remote-media` + +- Deletes all remote (and optionally local) media created before or after [duration] time using filesystem metadata first created at date, or fallback to last modified date. This will always ignore errors by default + +**Usage:** `admin media delete-past-remote-media [OPTIONS] ` + +###### **Arguments:** + +* `` — - The relative time (e.g. 30s, 5m, 7d) within which to search + +###### **Options:** + +* `-b`, `--before` — - Only delete media created before [duration] ago +* `-a`, `--after` — - Only delete media created after [duration] ago +* `--yes-i-want-to-delete-local-media` — - Long argument to additionally delete local media + + + +## `admin media delete-all-from-user` + +- Deletes all the local media from a local user on our server. This will always ignore errors by default + +**Usage:** `admin media delete-all-from-user ` + +###### **Arguments:** + +* `` + + + +## `admin media delete-all-from-server` + +- Deletes all remote media from the specified remote server. This will always ignore errors by default + +**Usage:** `admin media delete-all-from-server [OPTIONS] ` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--yes-i-want-to-delete-local-media` — Long argument to delete local media + + + +## `admin media get-file-info` + +**Usage:** `admin media get-file-info ` + +###### **Arguments:** + +* `` — The MXC URL to lookup info for + + + +## `admin media get-remote-file` + +**Usage:** `admin media get-remote-file [OPTIONS] ` + +###### **Arguments:** + +* `` — The MXC URL to fetch + +###### **Options:** + +* `-s`, `--server ` +* `-t`, `--timeout ` + + Default value: `10000` + + + +## `admin media get-remote-thumbnail` + +**Usage:** `admin media get-remote-thumbnail [OPTIONS] ` + +###### **Arguments:** + +* `` — The MXC URL to fetch + +###### **Options:** + +* `-s`, `--server ` +* `-t`, `--timeout ` + + Default value: `10000` +* `--width ` + + Default value: `800` +* `--height ` + + Default value: `800` + + + +## `admin check` + +- Commands for checking integrity + +**Usage:** `admin check ` + +###### **Subcommands:** + +* `check-all-users` — + + + +## `admin check check-all-users` + +**Usage:** `admin check check-all-users` + + + +## `admin debug` + +- Commands for debugging things + +**Usage:** `admin debug ` + +###### **Subcommands:** + +* `echo` — - Echo input of admin command +* `get-auth-chain` — - Get the auth_chain of a PDU +* `parse-pdu` — - Parse and print a PDU from a JSON +* `get-pdu` — - Retrieve and print a PDU by EventID from the Continuwuity database +* `get-short-pdu` — - Retrieve and print a PDU by PduId from the Continuwuity database +* `get-remote-pdu` — - Attempts to retrieve a PDU from a remote server. Inserts it into our database/timeline if found and we do not have this PDU already (following normal event auth rules, handles it as an incoming PDU) +* `get-remote-pdu-list` — - Same as `get-remote-pdu` but accepts a codeblock newline delimited list of PDUs and a single server to fetch from +* `get-room-state` — - Gets all the room state events for the specified room +* `get-signing-keys` — - Get and display signing keys from local cache or remote server +* `get-verify-keys` — - Get and display signing keys from local cache or remote server +* `ping` — - Sends a federation request to the remote server's `/_matrix/federation/v1/version` endpoint and measures the latency it took for the server to respond +* `force-device-list-updates` — - Forces device lists for all local and remote users to be updated (as having new keys available) +* `change-log-level` — - Change tracing log level/filter on the fly +* `sign-json` — - Sign JSON blob +* `verify-json` — - Verify JSON signatures +* `verify-pdu` — - Verify PDU +* `first-pdu-in-room` — - Prints the very first PDU in the specified room (typically m.room.create) +* `latest-pdu-in-room` — - Prints the latest ("last") PDU in the specified room (typically a message) +* `force-set-room-state-from-server` — - Forcefully replaces the room state of our local copy of the specified room, with the copy (auth chain and room state events) the specified remote server says +* `resolve-true-destination` — - Runs a server name through Continuwuity's true destination resolution process +* `memory-stats` — - Print extended memory usage +* `runtime-metrics` — - Print general tokio runtime metric totals +* `runtime-interval` — - Print detailed tokio runtime metrics accumulated since last command invocation +* `time` — - Print the current time +* `list-dependencies` — - List dependencies +* `database-stats` — - Get database statistics +* `trim-memory` — - Trim memory usage +* `database-files` — - List database files + + + +## `admin debug echo` + +- Echo input of admin command + +**Usage:** `admin debug echo [MESSAGE]...` + +###### **Arguments:** + +* `` + + + +## `admin debug get-auth-chain` + +- Get the auth_chain of a PDU + +**Usage:** `admin debug get-auth-chain ` + +###### **Arguments:** + +* `` — An event ID (the $ character followed by the base64 reference hash) + + + +## `admin debug parse-pdu` + +- Parse and print a PDU from a JSON + +The PDU event is only checked for validity and is not added to the database. + +This command needs a JSON blob provided in a Markdown code block below the command. + +**Usage:** `admin debug parse-pdu` + + + +## `admin debug get-pdu` + +- Retrieve and print a PDU by EventID from the Continuwuity database + +**Usage:** `admin debug get-pdu ` + +###### **Arguments:** + +* `` — An event ID (a $ followed by the base64 reference hash) + + + +## `admin debug get-short-pdu` + +- Retrieve and print a PDU by PduId from the Continuwuity database + +**Usage:** `admin debug get-short-pdu ` + +###### **Arguments:** + +* `` — Shortroomid integer +* `` — Shorteventid integer + + + +## `admin debug get-remote-pdu` + +- Attempts to retrieve a PDU from a remote server. Inserts it into our database/timeline if found and we do not have this PDU already (following normal event auth rules, handles it as an incoming PDU) + +**Usage:** `admin debug get-remote-pdu ` + +###### **Arguments:** + +* `` — An event ID (a $ followed by the base64 reference hash) +* `` — Argument for us to attempt to fetch the event from the specified remote server + + + +## `admin debug get-remote-pdu-list` + +- Same as `get-remote-pdu` but accepts a codeblock newline delimited list of PDUs and a single server to fetch from + +**Usage:** `admin debug get-remote-pdu-list [OPTIONS] ` + +###### **Arguments:** + +* `` — Argument for us to attempt to fetch all the events from the specified remote server + +###### **Options:** + +* `-f`, `--force` — If set, ignores errors, else stops at the first error/failure + + + +## `admin debug get-room-state` + +- Gets all the room state events for the specified room. + +This is functionally equivalent to `GET /_matrix/client/v3/rooms/{roomid}/state`, except the admin command does *not* check if the sender user is allowed to see state events. This is done because it's implied that server admins here have database access and can see/get room info themselves anyways if they were malicious admins. + +Of course the check is still done on the actual client API. + +**Usage:** `admin debug get-room-state ` + +###### **Arguments:** + +* `` — Room ID + + + +## `admin debug get-signing-keys` + +- Get and display signing keys from local cache or remote server + +**Usage:** `admin debug get-signing-keys [OPTIONS] [SERVER_NAME]` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--notary ` +* `-q`, `--query` + + + +## `admin debug get-verify-keys` + +- Get and display signing keys from local cache or remote server + +**Usage:** `admin debug get-verify-keys [SERVER_NAME]` + +###### **Arguments:** + +* `` + + + +## `admin debug ping` + +- Sends a federation request to the remote server's `/_matrix/federation/v1/version` endpoint and measures the latency it took for the server to respond + +**Usage:** `admin debug ping ` + +###### **Arguments:** + +* `` + + + +## `admin debug force-device-list-updates` + +- Forces device lists for all local and remote users to be updated (as having new keys available) + +**Usage:** `admin debug force-device-list-updates` + + + +## `admin debug change-log-level` + +- Change tracing log level/filter on the fly + +This accepts the same format as the `log` config option. + +**Usage:** `admin debug change-log-level [OPTIONS] [FILTER]` + +###### **Arguments:** + +* `` — Log level/filter + +###### **Options:** + +* `-r`, `--reset` — Resets the log level/filter to the one in your config + + + +## `admin debug sign-json` + +- Sign JSON blob + +This command needs a JSON blob provided in a Markdown code block below the command. + +**Usage:** `admin debug sign-json` + + + +## `admin debug verify-json` + +- Verify JSON signatures + +This command needs a JSON blob provided in a Markdown code block below the command. + +**Usage:** `admin debug verify-json` + + + +## `admin debug verify-pdu` + +- Verify PDU + +This re-verifies a PDU existing in the database found by ID. + +**Usage:** `admin debug verify-pdu ` + +###### **Arguments:** + +* `` + + + +## `admin debug first-pdu-in-room` + +- Prints the very first PDU in the specified room (typically m.room.create) + +**Usage:** `admin debug first-pdu-in-room ` + +###### **Arguments:** + +* `` — The room ID + + + +## `admin debug latest-pdu-in-room` + +- Prints the latest ("last") PDU in the specified room (typically a message) + +**Usage:** `admin debug latest-pdu-in-room ` + +###### **Arguments:** + +* `` — The room ID + + + +## `admin debug force-set-room-state-from-server` + +- Forcefully replaces the room state of our local copy of the specified room, with the copy (auth chain and room state events) the specified remote server says. + +A common desire for room deletion is to simply "reset" our copy of the room. While this admin command is not a replacement for that, if you know you have split/broken room state and you know another server in the room that has the best/working room state, this command can let you use their room state. Such example is your server saying users are in a room, but other servers are saying they're not in the room in question. + +This command will get the latest PDU in the room we know about, and request the room state at that point in time via `/_matrix/federation/v1/state/{roomId}`. + +**Usage:** `admin debug force-set-room-state-from-server [EVENT_ID]` + +###### **Arguments:** + +* `` — The impacted room ID +* `` — The server we will use to query the room state for +* `` — The event ID of the latest known PDU in the room. Will be found automatically if not provided + + + +## `admin debug resolve-true-destination` + +- Runs a server name through Continuwuity's true destination resolution process + +Useful for debugging well-known issues + +**Usage:** `admin debug resolve-true-destination [OPTIONS] ` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `-n`, `--no-cache` + + + +## `admin debug memory-stats` + +- Print extended memory usage + +Optional argument is a character mask (a sequence of characters in any order) which enable additional extended statistics. Known characters are "abdeglmx". For convenience, a '*' will enable everything. + +**Usage:** `admin debug memory-stats [OPTS]` + +###### **Arguments:** + +* `` + + + +## `admin debug runtime-metrics` + +- Print general tokio runtime metric totals + +**Usage:** `admin debug runtime-metrics` + + + +## `admin debug runtime-interval` + +- Print detailed tokio runtime metrics accumulated since last command invocation + +**Usage:** `admin debug runtime-interval` + + + +## `admin debug time` + +- Print the current time + +**Usage:** `admin debug time` + + + +## `admin debug list-dependencies` + +- List dependencies + +**Usage:** `admin debug list-dependencies [OPTIONS]` + +###### **Options:** + +* `-n`, `--names` + + + +## `admin debug database-stats` + +- Get database statistics + +**Usage:** `admin debug database-stats [OPTIONS] [PROPERTY]` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `-m`, `--map ` + + + +## `admin debug trim-memory` + +- Trim memory usage + +**Usage:** `admin debug trim-memory` + + + +## `admin debug database-files` + +- List database files + +**Usage:** `admin debug database-files [OPTIONS] [MAP]` + +###### **Arguments:** + +* `` + +###### **Options:** + +* `--level ` + + + +## `admin query` + +- Low-level queries for database getters and iterators + +**Usage:** `admin query ` + +###### **Subcommands:** + +* `account-data` — - account_data.rs iterators and getters +* `appservice` — - appservice.rs iterators and getters +* `presence` — - presence.rs iterators and getters +* `room-alias` — - rooms/alias.rs iterators and getters +* `room-state-cache` — - rooms/state_cache iterators and getters +* `room-timeline` — - rooms/timeline iterators and getters +* `globals` — - globals.rs iterators and getters +* `sending` — - sending.rs iterators and getters +* `users` — - users.rs iterators and getters +* `resolver` — - resolver service +* `pusher` — - pusher service +* `short` — - short service +* `raw` — - raw service + + + +## `admin query account-data` + +- account_data.rs iterators and getters + +**Usage:** `admin query account-data ` + +###### **Subcommands:** + +* `changes-since` — - Returns all changes to the account data that happened after `since` +* `account-data-get` — - Searches the account data for a specific kind + + + +## `admin query account-data changes-since` + +- Returns all changes to the account data that happened after `since` + +**Usage:** `admin query account-data changes-since [ROOM_ID]` + +###### **Arguments:** + +* `` — Full user ID +* `` — UNIX timestamp since (u64) +* `` — Optional room ID of the account data + + + +## `admin query account-data account-data-get` + +- Searches the account data for a specific kind + +**Usage:** `admin query account-data account-data-get [ROOM_ID]` + +###### **Arguments:** + +* `` — Full user ID +* `` — Account data event type +* `` — Optional room ID of the account data + + + +## `admin query appservice` + +- appservice.rs iterators and getters + +**Usage:** `admin query appservice ` + +###### **Subcommands:** + +* `get-registration` — - Gets the appservice registration info/details from the ID as a string +* `all` — - Gets all appservice registrations with their ID and registration info + + + +## `admin query appservice get-registration` + +- Gets the appservice registration info/details from the ID as a string + +**Usage:** `admin query appservice get-registration ` + +###### **Arguments:** + +* `` — Appservice registration ID + + + +## `admin query appservice all` + +- Gets all appservice registrations with their ID and registration info + +**Usage:** `admin query appservice all` + + + +## `admin query presence` + +- presence.rs iterators and getters + +**Usage:** `admin query presence ` + +###### **Subcommands:** + +* `get-presence` — - Returns the latest presence event for the given user +* `presence-since` — - Iterator of the most recent presence updates that happened after the event with id `since` + + + +## `admin query presence get-presence` + +- Returns the latest presence event for the given user + +**Usage:** `admin query presence get-presence ` + +###### **Arguments:** + +* `` — Full user ID + + + +## `admin query presence presence-since` + +- Iterator of the most recent presence updates that happened after the event with id `since` + +**Usage:** `admin query presence presence-since ` + +###### **Arguments:** + +* `` — UNIX timestamp since (u64) + + + +## `admin query room-alias` + +- rooms/alias.rs iterators and getters + +**Usage:** `admin query room-alias ` + +###### **Subcommands:** + +* `resolve-local-alias` — +* `local-aliases-for-room` — - Iterator of all our local room aliases for the room ID +* `all-local-aliases` — - Iterator of all our local aliases in our database with their room IDs + + + +## `admin query room-alias resolve-local-alias` + +**Usage:** `admin query room-alias resolve-local-alias ` + +###### **Arguments:** + +* `` — Full room alias + + + +## `admin query room-alias local-aliases-for-room` + +- Iterator of all our local room aliases for the room ID + +**Usage:** `admin query room-alias local-aliases-for-room ` + +###### **Arguments:** + +* `` — Full room ID + + + +## `admin query room-alias all-local-aliases` + +- Iterator of all our local aliases in our database with their room IDs + +**Usage:** `admin query room-alias all-local-aliases` + + + +## `admin query room-state-cache` + +- rooms/state_cache iterators and getters + +**Usage:** `admin query room-state-cache ` + +###### **Subcommands:** + +* `server-in-room` — +* `room-servers` — +* `server-rooms` — +* `room-members` — +* `local-users-in-room` — +* `active-local-users-in-room` — +* `room-joined-count` — +* `room-invited-count` — +* `room-user-once-joined` — +* `room-members-invited` — +* `get-invite-count` — +* `get-left-count` — +* `rooms-joined` — +* `rooms-left` — +* `rooms-invited` — +* `invite-state` — + + + +## `admin query room-state-cache server-in-room` + +**Usage:** `admin query room-state-cache server-in-room ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query room-state-cache room-servers` + +**Usage:** `admin query room-state-cache room-servers ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache server-rooms` + +**Usage:** `admin query room-state-cache server-rooms ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache room-members` + +**Usage:** `admin query room-state-cache room-members ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache local-users-in-room` + +**Usage:** `admin query room-state-cache local-users-in-room ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache active-local-users-in-room` + +**Usage:** `admin query room-state-cache active-local-users-in-room ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache room-joined-count` + +**Usage:** `admin query room-state-cache room-joined-count ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache room-invited-count` + +**Usage:** `admin query room-state-cache room-invited-count ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache room-user-once-joined` + +**Usage:** `admin query room-state-cache room-user-once-joined ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache room-members-invited` + +**Usage:** `admin query room-state-cache room-members-invited ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache get-invite-count` + +**Usage:** `admin query room-state-cache get-invite-count ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query room-state-cache get-left-count` + +**Usage:** `admin query room-state-cache get-left-count ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query room-state-cache rooms-joined` + +**Usage:** `admin query room-state-cache rooms-joined ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache rooms-left` + +**Usage:** `admin query room-state-cache rooms-left ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache rooms-invited` + +**Usage:** `admin query room-state-cache rooms-invited ` + +###### **Arguments:** + +* `` + + + +## `admin query room-state-cache invite-state` + +**Usage:** `admin query room-state-cache invite-state ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query room-timeline` + +- rooms/timeline iterators and getters + +**Usage:** `admin query room-timeline ` + +###### **Subcommands:** + +* `pdus` — +* `last` — + + + +## `admin query room-timeline pdus` + +**Usage:** `admin query room-timeline pdus [OPTIONS] [FROM]` + +###### **Arguments:** + +* `` +* `` + +###### **Options:** + +* `-l`, `--limit ` + + + +## `admin query room-timeline last` + +**Usage:** `admin query room-timeline last ` + +###### **Arguments:** + +* `` + + + +## `admin query globals` + +- globals.rs iterators and getters + +**Usage:** `admin query globals ` + +###### **Subcommands:** + +* `database-version` — +* `current-count` — +* `last-check-for-announcements-id` — +* `signing-keys-for` — - This returns an empty `Ok(BTreeMap<..>)` when there are no keys found for the server + + + +## `admin query globals database-version` + +**Usage:** `admin query globals database-version` + + + +## `admin query globals current-count` + +**Usage:** `admin query globals current-count` + + + +## `admin query globals last-check-for-announcements-id` + +**Usage:** `admin query globals last-check-for-announcements-id` + + + +## `admin query globals signing-keys-for` + +- This returns an empty `Ok(BTreeMap<..>)` when there are no keys found for the server + +**Usage:** `admin query globals signing-keys-for ` + +###### **Arguments:** + +* `` + + + +## `admin query sending` + +- sending.rs iterators and getters + +**Usage:** `admin query sending ` + +###### **Subcommands:** + +* `active-requests` — - Queries database for all `servercurrentevent_data` +* `active-requests-for` — - Queries database for `servercurrentevent_data` but for a specific destination +* `queued-requests` — - Queries database for `servernameevent_data` which are the queued up requests that will eventually be sent +* `get-latest-edu-count` — + + + +## `admin query sending active-requests` + +- Queries database for all `servercurrentevent_data` + +**Usage:** `admin query sending active-requests` + + + +## `admin query sending active-requests-for` + +- Queries database for `servercurrentevent_data` but for a specific destination + +This command takes only *one* format of these arguments: + +appservice_id server_name user_id AND push_key + +See src/service/sending/mod.rs for the definition of the `Destination` enum + +**Usage:** `admin query sending active-requests-for [OPTIONS]` + +###### **Options:** + +* `-a`, `--appservice-id ` +* `-s`, `--server-name ` +* `-u`, `--user-id ` +* `-p`, `--push-key ` + + + +## `admin query sending queued-requests` + +- Queries database for `servernameevent_data` which are the queued up requests that will eventually be sent + +This command takes only *one* format of these arguments: + +appservice_id server_name user_id AND push_key + +See src/service/sending/mod.rs for the definition of the `Destination` enum + +**Usage:** `admin query sending queued-requests [OPTIONS]` + +###### **Options:** + +* `-a`, `--appservice-id ` +* `-s`, `--server-name ` +* `-u`, `--user-id ` +* `-p`, `--push-key ` + + + +## `admin query sending get-latest-edu-count` + +**Usage:** `admin query sending get-latest-edu-count ` + +###### **Arguments:** + +* `` + + + +## `admin query users` + +- users.rs iterators and getters + +**Usage:** `admin query users ` + +###### **Subcommands:** + +* `count-users` — +* `iter-users` — +* `iter-users2` — +* `password-hash` — +* `list-devices` — +* `list-devices-metadata` — +* `get-device-metadata` — +* `get-devices-version` — +* `count-one-time-keys` — +* `get-device-keys` — +* `get-user-signing-key` — +* `get-master-key` — +* `get-to-device-events` — +* `get-latest-backup` — +* `get-latest-backup-version` — +* `get-backup-algorithm` — +* `get-all-backups` — +* `get-room-backups` — +* `get-backup-session` — +* `get-shared-rooms` — + + + +## `admin query users count-users` + +**Usage:** `admin query users count-users` + + + +## `admin query users iter-users` + +**Usage:** `admin query users iter-users` + + + +## `admin query users iter-users2` + +**Usage:** `admin query users iter-users2` + + + +## `admin query users password-hash` + +**Usage:** `admin query users password-hash ` + +###### **Arguments:** + +* `` + + + +## `admin query users list-devices` + +**Usage:** `admin query users list-devices ` + +###### **Arguments:** + +* `` + + + +## `admin query users list-devices-metadata` + +**Usage:** `admin query users list-devices-metadata ` + +###### **Arguments:** + +* `` + + + +## `admin query users get-device-metadata` + +**Usage:** `admin query users get-device-metadata ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query users get-devices-version` + +**Usage:** `admin query users get-devices-version ` + +###### **Arguments:** + +* `` + + + +## `admin query users count-one-time-keys` + +**Usage:** `admin query users count-one-time-keys ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query users get-device-keys` + +**Usage:** `admin query users get-device-keys ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query users get-user-signing-key` + +**Usage:** `admin query users get-user-signing-key ` + +###### **Arguments:** + +* `` + + + +## `admin query users get-master-key` + +**Usage:** `admin query users get-master-key ` + +###### **Arguments:** + +* `` + + + +## `admin query users get-to-device-events` + +**Usage:** `admin query users get-to-device-events ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query users get-latest-backup` + +**Usage:** `admin query users get-latest-backup ` + +###### **Arguments:** + +* `` + + + +## `admin query users get-latest-backup-version` + +**Usage:** `admin query users get-latest-backup-version ` + +###### **Arguments:** + +* `` + + + +## `admin query users get-backup-algorithm` + +**Usage:** `admin query users get-backup-algorithm ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query users get-all-backups` + +**Usage:** `admin query users get-all-backups ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query users get-room-backups` + +**Usage:** `admin query users get-room-backups ` + +###### **Arguments:** + +* `` +* `` +* `` + + + +## `admin query users get-backup-session` + +**Usage:** `admin query users get-backup-session ` + +###### **Arguments:** + +* `` +* `` +* `` +* `` + + + +## `admin query users get-shared-rooms` + +**Usage:** `admin query users get-shared-rooms ` + +###### **Arguments:** + +* `` +* `` + + + +## `admin query resolver` + +- resolver service + +**Usage:** `admin query resolver ` + +###### **Subcommands:** + +* `destinations-cache` — Query the destinations cache +* `overrides-cache` — Query the overrides cache + + + +## `admin query resolver destinations-cache` + +Query the destinations cache + +**Usage:** `admin query resolver destinations-cache [SERVER_NAME]` + +###### **Arguments:** + +* `` + + + +## `admin query resolver overrides-cache` + +Query the overrides cache + +**Usage:** `admin query resolver overrides-cache [NAME]` + +###### **Arguments:** + +* `` + + + +## `admin query pusher` + +- pusher service + +**Usage:** `admin query pusher ` + +###### **Subcommands:** + +* `get-pushers` — - Returns all the pushers for the user + + + +## `admin query pusher get-pushers` + +- Returns all the pushers for the user + +**Usage:** `admin query pusher get-pushers ` + +###### **Arguments:** + +* `` — Full user ID + + + +## `admin query short` + +- short service + +**Usage:** `admin query short ` + +###### **Subcommands:** + +* `short-event-id` — +* `short-room-id` — + + + +## `admin query short short-event-id` + +**Usage:** `admin query short short-event-id ` + +###### **Arguments:** + +* `` + + + +## `admin query short short-room-id` + +**Usage:** `admin query short short-room-id ` + +###### **Arguments:** + +* `` + + + +## `admin query raw` + +- raw service + +**Usage:** `admin query raw ` + +###### **Subcommands:** + +* `raw-maps` — - List database maps +* `raw-get` — - Raw database query +* `raw-del` — - Raw database delete (for string keys) +* `raw-keys` — - Raw database keys iteration +* `raw-keys-sizes` — - Raw database key size breakdown +* `raw-keys-total` — - Raw database keys total bytes +* `raw-vals-sizes` — - Raw database values size breakdown +* `raw-vals-total` — - Raw database values total bytes +* `raw-iter` — - Raw database items iteration +* `raw-keys-from` — - Raw database keys iteration +* `raw-iter-from` — - Raw database items iteration +* `raw-count` — - Raw database record count +* `compact` — - Compact database + + + +## `admin query raw raw-maps` + +- List database maps + +**Usage:** `admin query raw raw-maps` + + + +## `admin query raw raw-get` + +- Raw database query + +**Usage:** `admin query raw raw-get ` + +###### **Arguments:** + +* `` — Map name +* `` — Key + + + +## `admin query raw raw-del` + +- Raw database delete (for string keys) + +**Usage:** `admin query raw raw-del ` + +###### **Arguments:** + +* `` — Map name +* `` — Key + + + +## `admin query raw raw-keys` + +- Raw database keys iteration + +**Usage:** `admin query raw raw-keys [PREFIX]` + +###### **Arguments:** + +* `` — Map name +* `` — Key prefix + + + +## `admin query raw raw-keys-sizes` + +- Raw database key size breakdown + +**Usage:** `admin query raw raw-keys-sizes [MAP] [PREFIX]` + +###### **Arguments:** + +* `` — Map name +* `` — Key prefix + + + +## `admin query raw raw-keys-total` + +- Raw database keys total bytes + +**Usage:** `admin query raw raw-keys-total [MAP] [PREFIX]` + +###### **Arguments:** + +* `` — Map name +* `` — Key prefix + + + +## `admin query raw raw-vals-sizes` + +- Raw database values size breakdown + +**Usage:** `admin query raw raw-vals-sizes [MAP] [PREFIX]` + +###### **Arguments:** + +* `` — Map name +* `` — Key prefix + + + +## `admin query raw raw-vals-total` + +- Raw database values total bytes + +**Usage:** `admin query raw raw-vals-total [MAP] [PREFIX]` + +###### **Arguments:** + +* `` — Map name +* `` — Key prefix + + + +## `admin query raw raw-iter` + +- Raw database items iteration + +**Usage:** `admin query raw raw-iter [PREFIX]` + +###### **Arguments:** + +* `` — Map name +* `` — Key prefix + + + +## `admin query raw raw-keys-from` + +- Raw database keys iteration + +**Usage:** `admin query raw raw-keys-from [OPTIONS] ` + +###### **Arguments:** + +* `` — Map name +* `` — Lower-bound + +###### **Options:** + +* `-l`, `--limit ` — Limit + + + +## `admin query raw raw-iter-from` + +- Raw database items iteration + +**Usage:** `admin query raw raw-iter-from [OPTIONS] ` + +###### **Arguments:** + +* `` — Map name +* `` — Lower-bound + +###### **Options:** + +* `-l`, `--limit ` — Limit + + + +## `admin query raw raw-count` + +- Raw database record count + +**Usage:** `admin query raw raw-count [MAP] [PREFIX]` + +###### **Arguments:** + +* `` — Map name +* `` — Key prefix + + + +## `admin query raw compact` + +- Compact database + +**Usage:** `admin query raw compact [OPTIONS]` + +###### **Options:** + +* `-m`, `--map ` +* `--start ` +* `--stop ` +* `--from ` +* `--into ` +* `--parallelism ` — There is one compaction job per column; then this controls how many columns are compacted in parallel. If zero, one compaction job is still run at a time here, but in exclusive-mode blocking any other automatic compaction jobs until complete +* `--exhaustive` + + Default value: `false` diff --git a/src/admin/admin.rs b/src/admin/admin.rs index 50b9db7c..e6479960 100644 --- a/src/admin/admin.rs +++ b/src/admin/admin.rs @@ -10,7 +10,7 @@ use crate::{ #[derive(Debug, Parser)] #[command(name = conduwuit_core::name(), version = conduwuit_core::version())] -pub(super) enum AdminCommand { +pub enum AdminCommand { #[command(subcommand)] /// - Commands for managing appservices Appservices(AppserviceCommand), diff --git a/src/admin/appservice/mod.rs b/src/admin/appservice/mod.rs index 2e0694aa..5e2712a9 100644 --- a/src/admin/appservice/mod.rs +++ b/src/admin/appservice/mod.rs @@ -7,7 +7,7 @@ use crate::admin_command_dispatch; #[derive(Debug, Subcommand)] #[admin_command_dispatch] -pub(super) enum AppserviceCommand { +pub enum AppserviceCommand { /// - Register an appservice using its registration YAML /// /// This command needs a YAML generated by an appservice (such as a bridge), diff --git a/src/admin/check/mod.rs b/src/admin/check/mod.rs index 30b335c4..a15968a7 100644 --- a/src/admin/check/mod.rs +++ b/src/admin/check/mod.rs @@ -7,6 +7,6 @@ use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] -pub(super) enum CheckCommand { +pub enum CheckCommand { CheckAllUsers, } diff --git a/src/admin/debug/mod.rs b/src/admin/debug/mod.rs index fb8a3002..7a0769ab 100644 --- a/src/admin/debug/mod.rs +++ b/src/admin/debug/mod.rs @@ -11,7 +11,7 @@ use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] -pub(super) enum DebugCommand { +pub enum DebugCommand { /// - Echo input of admin command Echo { message: Vec, diff --git a/src/admin/debug/tester.rs b/src/admin/debug/tester.rs index 0a2b1516..75da382b 100644 --- a/src/admin/debug/tester.rs +++ b/src/admin/debug/tester.rs @@ -4,7 +4,7 @@ use crate::{admin_command, admin_command_dispatch}; #[admin_command_dispatch] #[derive(Debug, clap::Subcommand)] -pub(crate) enum TesterCommand { +pub enum TesterCommand { Panic, Failure, Tester, diff --git a/src/admin/federation/mod.rs b/src/admin/federation/mod.rs index 2c539adc..48f79a56 100644 --- a/src/admin/federation/mod.rs +++ b/src/admin/federation/mod.rs @@ -8,7 +8,7 @@ use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] -pub(super) enum FederationCommand { +pub enum FederationCommand { /// - List all rooms we are currently handling an incoming pdu from IncomingFederation, diff --git a/src/admin/media/mod.rs b/src/admin/media/mod.rs index d1e6cd3a..66d49959 100644 --- a/src/admin/media/mod.rs +++ b/src/admin/media/mod.rs @@ -9,7 +9,7 @@ use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] -pub(super) enum MediaCommand { +pub enum MediaCommand { /// - Deletes a single media file from our database and on the filesystem /// via a single MXC URL or event ID (not redacted) Delete { @@ -90,10 +90,10 @@ pub(super) enum MediaCommand { #[arg(short, long, default_value("10000"))] timeout: u32, - #[arg(short, long, default_value("800"))] + #[arg(long, default_value("800"))] width: u32, - #[arg(short, long, default_value("800"))] + #[arg(long, default_value("800"))] height: u32, }, } diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 1f777fa9..732b8ce0 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -33,6 +33,8 @@ conduwuit::mod_ctor! {} conduwuit::mod_dtor! {} conduwuit::rustc_flags_capture! {} +pub use crate::admin::AdminCommand; + /// Install the admin command processor pub async fn init(admin_service: &service::admin::Service) { _ = admin_service diff --git a/src/admin/query/account_data.rs b/src/admin/query/account_data.rs index 228d2120..2e044cef 100644 --- a/src/admin/query/account_data.rs +++ b/src/admin/query/account_data.rs @@ -8,7 +8,7 @@ use crate::{admin_command, admin_command_dispatch}; #[admin_command_dispatch] #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/account_data.rs -pub(crate) enum AccountDataCommand { +pub enum AccountDataCommand { /// - Returns all changes to the account data that happened after `since`. ChangesSince { /// Full user ID diff --git a/src/admin/query/appservice.rs b/src/admin/query/appservice.rs index 28bf6451..f9f15c25 100644 --- a/src/admin/query/appservice.rs +++ b/src/admin/query/appservice.rs @@ -6,7 +6,7 @@ use crate::Context; #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/appservice.rs -pub(crate) enum AppserviceCommand { +pub enum AppserviceCommand { /// - Gets the appservice registration info/details from the ID as a string GetRegistration { /// Appservice registration ID diff --git a/src/admin/query/globals.rs b/src/admin/query/globals.rs index c8c1f512..6e945145 100644 --- a/src/admin/query/globals.rs +++ b/src/admin/query/globals.rs @@ -6,7 +6,7 @@ use crate::Context; #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/globals.rs -pub(crate) enum GlobalsCommand { +pub enum GlobalsCommand { DatabaseVersion, CurrentCount, diff --git a/src/admin/query/mod.rs b/src/admin/query/mod.rs index da27eb1d..5b93086b 100644 --- a/src/admin/query/mod.rs +++ b/src/admin/query/mod.rs @@ -27,7 +27,7 @@ use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] /// Query tables from database -pub(super) enum QueryCommand { +pub enum QueryCommand { /// - account_data.rs iterators and getters #[command(subcommand)] AccountData(AccountDataCommand), diff --git a/src/admin/query/presence.rs b/src/admin/query/presence.rs index 5b7ead4b..ccc3a431 100644 --- a/src/admin/query/presence.rs +++ b/src/admin/query/presence.rs @@ -7,7 +7,7 @@ use crate::Context; #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/presence.rs -pub(crate) enum PresenceCommand { +pub enum PresenceCommand { /// - Returns the latest presence event for the given user. GetPresence { /// Full user ID diff --git a/src/admin/query/pusher.rs b/src/admin/query/pusher.rs index 0d0e6cc9..3f294017 100644 --- a/src/admin/query/pusher.rs +++ b/src/admin/query/pusher.rs @@ -5,7 +5,7 @@ use ruma::OwnedUserId; use crate::Context; #[derive(Debug, Subcommand)] -pub(crate) enum PusherCommand { +pub enum PusherCommand { /// - Returns all the pushers for the user. GetPushers { /// Full user ID diff --git a/src/admin/query/raw.rs b/src/admin/query/raw.rs index 0e248c65..5165b61a 100644 --- a/src/admin/query/raw.rs +++ b/src/admin/query/raw.rs @@ -19,7 +19,7 @@ use crate::{admin_command, admin_command_dispatch}; #[derive(Debug, Subcommand)] #[allow(clippy::enum_variant_names)] /// Query tables from database -pub(crate) enum RawCommand { +pub enum RawCommand { /// - List database maps RawMaps, diff --git a/src/admin/query/resolver.rs b/src/admin/query/resolver.rs index 4a39a40e..5b2d8d3b 100644 --- a/src/admin/query/resolver.rs +++ b/src/admin/query/resolver.rs @@ -8,7 +8,7 @@ use crate::{admin_command, admin_command_dispatch}; #[admin_command_dispatch] #[derive(Debug, Subcommand)] /// Resolver service and caches -pub(crate) enum ResolverCommand { +pub enum ResolverCommand { /// Query the destinations cache DestinationsCache { server_name: Option, diff --git a/src/admin/query/room_alias.rs b/src/admin/query/room_alias.rs index b646beec..fac1dd0a 100644 --- a/src/admin/query/room_alias.rs +++ b/src/admin/query/room_alias.rs @@ -7,7 +7,7 @@ use crate::Context; #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/rooms/alias.rs -pub(crate) enum RoomAliasCommand { +pub enum RoomAliasCommand { ResolveLocalAlias { /// Full room alias alias: OwnedRoomAliasId, diff --git a/src/admin/query/room_state_cache.rs b/src/admin/query/room_state_cache.rs index c64cd173..798792d7 100644 --- a/src/admin/query/room_state_cache.rs +++ b/src/admin/query/room_state_cache.rs @@ -6,7 +6,7 @@ use ruma::{OwnedRoomId, OwnedServerName, OwnedUserId}; use crate::Context; #[derive(Debug, Subcommand)] -pub(crate) enum RoomStateCacheCommand { +pub enum RoomStateCacheCommand { ServerInRoom { server: OwnedServerName, room_id: OwnedRoomId, diff --git a/src/admin/query/room_timeline.rs b/src/admin/query/room_timeline.rs index 0fd22ca7..afcfec34 100644 --- a/src/admin/query/room_timeline.rs +++ b/src/admin/query/room_timeline.rs @@ -8,7 +8,7 @@ use crate::{admin_command, admin_command_dispatch}; #[admin_command_dispatch] #[derive(Debug, Subcommand)] /// Query tables from database -pub(crate) enum RoomTimelineCommand { +pub enum RoomTimelineCommand { Pdus { room_id: OwnedRoomOrAliasId, diff --git a/src/admin/query/sending.rs b/src/admin/query/sending.rs index 8b1676bc..b6350539 100644 --- a/src/admin/query/sending.rs +++ b/src/admin/query/sending.rs @@ -8,7 +8,7 @@ use crate::Context; #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/sending.rs -pub(crate) enum SendingCommand { +pub enum SendingCommand { /// - Queries database for all `servercurrentevent_data` ActiveRequests, diff --git a/src/admin/query/short.rs b/src/admin/query/short.rs index aa7c8666..3ebfbbcf 100644 --- a/src/admin/query/short.rs +++ b/src/admin/query/short.rs @@ -7,7 +7,7 @@ use crate::{admin_command, admin_command_dispatch}; #[admin_command_dispatch] #[derive(Debug, Subcommand)] /// Query tables from database -pub(crate) enum ShortCommand { +pub enum ShortCommand { ShortEventId { event_id: OwnedEventId, }, diff --git a/src/admin/query/users.rs b/src/admin/query/users.rs index 0f34d13f..2b5b3481 100644 --- a/src/admin/query/users.rs +++ b/src/admin/query/users.rs @@ -8,7 +8,7 @@ use crate::{admin_command, admin_command_dispatch}; #[admin_command_dispatch] #[derive(Debug, Subcommand)] /// All the getters and iterators from src/database/key_value/users.rs -pub(crate) enum UsersCommand { +pub enum UsersCommand { CountUsers, IterUsers, diff --git a/src/admin/room/alias.rs b/src/admin/room/alias.rs index 6b37ffe4..80e5d297 100644 --- a/src/admin/room/alias.rs +++ b/src/admin/room/alias.rs @@ -8,7 +8,7 @@ use ruma::{OwnedRoomAliasId, OwnedRoomId}; use crate::Context; #[derive(Debug, Subcommand)] -pub(crate) enum RoomAliasCommand { +pub enum RoomAliasCommand { /// - Make an alias point to a room. Set { #[arg(short, long)] diff --git a/src/admin/room/directory.rs b/src/admin/room/directory.rs index a6be9a15..cdefc99b 100644 --- a/src/admin/room/directory.rs +++ b/src/admin/room/directory.rs @@ -6,7 +6,7 @@ use ruma::OwnedRoomId; use crate::{Context, PAGE_SIZE, get_room_info}; #[derive(Debug, Subcommand)] -pub(crate) enum RoomDirectoryCommand { +pub enum RoomDirectoryCommand { /// - Publish a room to the room directory Publish { /// The room id of the room to publish diff --git a/src/admin/room/info.rs b/src/admin/room/info.rs index 1278e820..e35ddb27 100644 --- a/src/admin/room/info.rs +++ b/src/admin/room/info.rs @@ -7,7 +7,7 @@ use crate::{admin_command, admin_command_dispatch}; #[admin_command_dispatch] #[derive(Debug, Subcommand)] -pub(crate) enum RoomInfoCommand { +pub enum RoomInfoCommand { /// - List joined members in a room ListJoinedMembers { room_id: OwnedRoomId, diff --git a/src/admin/room/mod.rs b/src/admin/room/mod.rs index 26d2c2d8..00baf4c8 100644 --- a/src/admin/room/mod.rs +++ b/src/admin/room/mod.rs @@ -16,7 +16,7 @@ use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] -pub(super) enum RoomCommand { +pub enum RoomCommand { /// - List all rooms the server knows about #[clap(alias = "list")] ListRooms { diff --git a/src/admin/room/moderation.rs b/src/admin/room/moderation.rs index 5fb5bb3e..4d106977 100644 --- a/src/admin/room/moderation.rs +++ b/src/admin/room/moderation.rs @@ -12,7 +12,7 @@ use crate::{admin_command, admin_command_dispatch, get_room_info}; #[admin_command_dispatch] #[derive(Debug, Subcommand)] -pub(crate) enum RoomModerationCommand { +pub enum RoomModerationCommand { /// - Bans a room from local users joining and evicts all our local users /// (including server /// admins) diff --git a/src/admin/server/mod.rs b/src/admin/server/mod.rs index 6b99e5de..cf46d034 100644 --- a/src/admin/server/mod.rs +++ b/src/admin/server/mod.rs @@ -9,7 +9,7 @@ use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] -pub(super) enum ServerCommand { +pub enum ServerCommand { /// - Time elapsed since startup Uptime, diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 645d3637..656cacaf 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -8,7 +8,7 @@ use crate::admin_command_dispatch; #[admin_command_dispatch] #[derive(Debug, Subcommand)] -pub(super) enum UserCommand { +pub enum UserCommand { /// - Create a new user #[clap(alias = "create")] CreateUser { diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 2d8d26b5..cddf4156 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -14,6 +14,13 @@ rust-version.workspace = true version.workspace = true metadata.crane.workspace = true +[lib] +path = "mod.rs" +crate-type = [ + "rlib", +# "dylib", +] + [package.metadata.deb] name = "conduwuit" maintainer = "strawberry " diff --git a/src/main/mod.rs b/src/main/mod.rs new file mode 100644 index 00000000..ce9e3b9c --- /dev/null +++ b/src/main/mod.rs @@ -0,0 +1,14 @@ +#![type_length_limit = "49152"] //TODO: reduce me + +use conduwuit_core::rustc_flags_capture; + +pub(crate) mod clap; +mod logging; +mod mods; +mod restart; +mod runtime; +mod sentry; +mod server; +mod signal; + +rustc_flags_capture! {} diff --git a/xtask/generate-admin-command/Cargo.toml b/xtask/generate-admin-command/Cargo.toml new file mode 100644 index 00000000..5f27ee0c --- /dev/null +++ b/xtask/generate-admin-command/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "xtask-admin-command" +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +clap-markdown = "0.1.5" +clap_builder = { version = "4.5.38", default-features = false } +clap_mangen = "0.2" + +conduwuit-admin.workspace = true + +# Hack to prevent rebuilds +conduwuit.workspace = true + +[lints] +workspace = true diff --git a/xtask/generate-admin-command/src/main.rs b/xtask/generate-admin-command/src/main.rs new file mode 100644 index 00000000..46e7dad5 --- /dev/null +++ b/xtask/generate-admin-command/src/main.rs @@ -0,0 +1,63 @@ +use std::{ + fs::{self, File}, + io::{self, Write}, + path::Path, +}; + +use clap_builder::{Command, CommandFactory}; +use conduwuit_admin::AdminCommand; + +fn main() -> Result<(), Box> { + let mut args = std::env::args().skip(1); + let task = args.next(); + match task { + | None => todo!(), + | Some(t) => match t.as_str() { + | "man" => { + let dir = Path::new("./admin-man"); + gen_manpages(dir)?; + }, + | "md" => { + let command = AdminCommand::command().name("admin"); + + let res = clap_markdown::help_markdown_command_custom( + &command, + &clap_markdown::MarkdownOptions::default().show_footer(false), + ) + .replace("\n\r", "\n") + .replace("\r\n", "\n") + .replace(" \n", "\n"); + + let mut file = File::create(Path::new("./docs/admin_reference.md"))?; + Write::write_all(&mut file, res.trim_end().as_bytes())?; + file.write(b"\n")?; + }, + | invalid => return Err(format!("Invalid task name: {invalid}").into()), + }, + } + Ok(()) +} + +fn gen_manpages(dir: &Path) -> Result<(), io::Error> { + fn r#gen(dir: &Path, c: &Command, prefix: Option<&str>) -> Result<(), io::Error> { + fs::create_dir_all(dir)?; + let sub_name = c.get_display_name().unwrap_or_else(|| c.get_name()); + let name = if let Some(prefix) = prefix { + format!("{prefix}-{sub_name}") + } else { + sub_name.to_owned() + }; + + let mut out = File::create(dir.join(format!("{name}.1")))?; + let clap_mangen = clap_mangen::Man::new(c.to_owned().disable_help_flag(true)); + clap_mangen.render(&mut out)?; + + for sub in c.get_subcommands() { + r#gen(&dir.join(sub_name), sub, Some(&name))?; + } + + Ok(()) + } + + r#gen(dir, &AdminCommand::command().name("admin"), None) +} diff --git a/xtask/main/Cargo.toml b/xtask/main/Cargo.toml new file mode 100644 index 00000000..70c0c34b --- /dev/null +++ b/xtask/main/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "xtask" +authors.workspace = true +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true +version.workspace = true + +[dependencies] +clap.workspace = true +# Required for working with JSON output from cargo metadata +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" + +[lints] +workspace = true diff --git a/xtask/main/src/main.rs b/xtask/main/src/main.rs new file mode 100644 index 00000000..0b244114 --- /dev/null +++ b/xtask/main/src/main.rs @@ -0,0 +1,11 @@ +use std::{env, process::Command}; + +fn main() -> Result<(), Box> { + let mut child = Command::new("cargo").args(["run", "--package", "xtask-admin-command", "--"].into_iter().map(ToOwned::to_owned).chain(env::args().skip(2))) + // .stdout(Stdio::piped()) + // .stderr(Stdio::piped()) + .spawn() + .expect("failed to execute child"); + child.wait()?; + Ok(()) +} From 28a29c3a7b2b01b6c26bc9e92dc4ce2d1fcf9164 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 6 Jul 2025 21:59:20 +0100 Subject: [PATCH 1903/2091] feat: Generate binary documentation Also refactors main.rs/mod.rs to silence clippy --- Cargo.lock | 2 +- Cargo.toml | 2 +- docs/server_reference.md | 21 +++ src/main/clap.rs | 36 +++--- src/main/main.rs | 121 +----------------- src/main/mod.rs | 119 ++++++++++++++++- src/main/server.rs | 7 +- xtask/generate-admin-command/src/main.rs | 63 --------- .../Cargo.toml | 2 +- xtask/generate-generate-commands/src/main.rs | 113 ++++++++++++++++ xtask/main/Cargo.toml | 2 +- xtask/main/src/main.rs | 2 +- 12 files changed, 281 insertions(+), 209 deletions(-) create mode 100644 docs/server_reference.md delete mode 100644 xtask/generate-admin-command/src/main.rs rename xtask/{generate-admin-command => generate-generate-commands}/Cargo.toml (94%) create mode 100644 xtask/generate-generate-commands/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index fe8cb16d..d950e9da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6129,7 +6129,7 @@ dependencies = [ ] [[package]] -name = "xtask-admin-command" +name = "xtask-generate-commands" version = "0.5.0-rc.6" dependencies = [ "clap-markdown", diff --git a/Cargo.toml b/Cargo.toml index 03c5b489..ef917332 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -768,7 +768,7 @@ inherits = "dev" # '-Clink-arg=-Wl,-z,nodlopen', # '-Clink-arg=-Wl,-z,nodelete', #] -[profile.dev.package.xtask-admin-command] +[profile.dev.package.xtask-generate-commands] inherits = "dev" [profile.dev.package.conduwuit] inherits = "dev" diff --git a/docs/server_reference.md b/docs/server_reference.md new file mode 100644 index 00000000..e34bc51e --- /dev/null +++ b/docs/server_reference.md @@ -0,0 +1,21 @@ +# Command-Line Help for `continuwuity` + +This document contains the help content for the `continuwuity` command-line program. + +**Command Overview:** + +* [`continuwuity`↴](#continuwuity) + +## `continuwuity` + +a very cool Matrix chat homeserver written in Rust + +**Usage:** `continuwuity [OPTIONS]` + +###### **Options:** + +* `-c`, `--config ` — Path to the config TOML file (optional) +* `-O`, `--option

    To get started, you can:

    From 1bc663e1c8065c6bf8c489d3c419c8696e7b0dde Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 24 Jul 2025 13:37:52 +0100 Subject: [PATCH 1982/2091] docs: Fix spacing at the top --- theme/css/chrome.css | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/theme/css/chrome.css b/theme/css/chrome.css index f14ffc2c..0e37be26 100644 --- a/theme/css/chrome.css +++ b/theme/css/chrome.css @@ -605,3 +605,8 @@ ul#searchresults span.teaser em { margin-inline-start: -14px; width: 14px; } + +/* HACK: Stop the keyboard shortcuts from adding a giant space at the top */ +#mdbook-help-container { + display: none; +} From b7a04422981b3eabea37bcea97c978bfc4c72b7f Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 23 Jul 2025 00:24:27 +0100 Subject: [PATCH 1983/2091] feat: Musl images in docker Not working at the moment, need to upgrade the rust-rocksdb and possibly zstd to stop them force-enabling dynamic libclang --- Cargo.lock | 1 + docker/Dockerfile | 41 +++++---- docker/musl.Dockerfile | 201 +++++++++++++++++++++++++++++++++++++++++ src/main/Cargo.toml | 9 ++ 4 files changed, 233 insertions(+), 19 deletions(-) create mode 100644 docker/musl.Dockerfile diff --git a/Cargo.lock b/Cargo.lock index 22c90e17..1866a691 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -847,6 +847,7 @@ dependencies = [ name = "conduwuit" version = "0.5.0-rc.6" dependencies = [ + "bindgen 0.71.1", "clap", "conduwuit_admin", "conduwuit_api", diff --git a/docker/Dockerfile b/docker/Dockerfile index bd6e72d1..55150902 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -78,7 +78,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ WORKDIR /app COPY ./rust-toolchain.toml . RUN rustc --version \ - && rustup target add $(xx-cargo --print-target-triple) + && xx-cargo --setup-target-triple # Build binary # We disable incremental compilation to save disk space, as it only produces a minimal speedup for this case. @@ -87,8 +87,10 @@ RUN echo "CARGO_INCREMENTAL=0" >> /etc/environment # Configure pkg-config RUN <> /etc/environment - echo "PKG_CONFIG=/usr/bin/$(xx-info)-pkg-config" >> /etc/environment + if command -v "$(xx-info)-pkg-config" >/dev/null 2>/dev/null; then + echo "PKG_CONFIG_LIBDIR=/usr/lib/$(xx-info)/pkgconfig" >> /etc/environment + echo "PKG_CONFIG=/usr/bin/$(xx-info)-pkg-config" >> /etc/environment + fi echo "PKG_CONFIG_ALLOW_CROSS=true" >> /etc/environment EOF @@ -109,16 +111,17 @@ RUN <> /etc/environment - echo "CXXFLAGS='${CXXFLAGS} -march=${TARGET_CPU}'" >> /etc/environment - echo "RUSTFLAGS='${RUSTFLAGS} -C target-cpu=${TARGET_CPU}'" >> /etc/environment - fi + set -o allexport + set -o xtrace + . /etc/environment + if [ -n "${TARGET_CPU}" ]; then + echo "CFLAGS='${CFLAGS} -march=${TARGET_CPU}'" >> /etc/environment + echo "CXXFLAGS='${CXXFLAGS} -march=${TARGET_CPU}'" >> /etc/environment + echo "RUSTFLAGS='${RUSTFLAGS} -C target-cpu=${TARGET_CPU}'" >> /etc/environment + fi EOF # Prepare output directories @@ -136,12 +139,12 @@ ARG TARGETPLATFORM RUN xx-cargo --print-target-triple # Conduwuit version info -ARG GIT_COMMIT_HASH= -ARG GIT_COMMIT_HASH_SHORT= -ARG GIT_REMOTE_URL= -ARG GIT_REMOTE_COMMIT_URL= -ARG CONDUWUIT_VERSION_EXTRA= -ARG CONTINUWUITY_VERSION_EXTRA= +ARG GIT_COMMIT_HASH +ARG GIT_COMMIT_HASH_SHORT +ARG GIT_REMOTE_URL +ARG GIT_REMOTE_COMMIT_URL +ARG CONDUWUIT_VERSION_EXTRA +ARG CONTINUWUITY_VERSION_EXTRA ENV GIT_COMMIT_HASH=$GIT_COMMIT_HASH ENV GIT_COMMIT_HASH_SHORT=$GIT_COMMIT_HASH_SHORT ENV GIT_REMOTE_URL=$GIT_REMOTE_URL @@ -169,7 +172,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ jq -r ".packages[] | select(.name == \"$PACKAGE\") | .targets[] | select( .kind | map(. == \"bin\") | any ) | .name")) for BINARY in "${BINARIES[@]}"; do echo $BINARY - xx-verify $TARGET_DIR/$(xx-cargo --print-target-triple)/release/$BINARY + xx-verify $TARGET_DIR/$(xx-cargo --print-target-triple)/release/$BINARY cp $TARGET_DIR/$(xx-cargo --print-target-triple)/release/$BINARY /out/sbin/$BINARY done EOF diff --git a/docker/musl.Dockerfile b/docker/musl.Dockerfile new file mode 100644 index 00000000..d5317d15 --- /dev/null +++ b/docker/musl.Dockerfile @@ -0,0 +1,201 @@ +# Why does this exist? +# Debian doesn't provide prebuilt musl packages +# rocksdb requires a prebuilt liburing, and linking fails if a gnu one is provided + +ARG RUST_VERSION=1 +ARG ALPINE_VERSION=3.22 + +FROM --platform=$BUILDPLATFORM docker.io/tonistiigi/xx AS xx +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-alpine${ALPINE_VERSION} AS base +FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-alpine${ALPINE_VERSION} AS toolchain + +# Install repo tools and dependencies +RUN --mount=type=cache,target=/etc/apk/cache apk add \ + build-base pkgconfig make jq bash \ + curl git file \ + llvm-dev clang clang-dev lld + + +# Developer tool versions +# renovate: datasource=github-releases depName=cargo-bins/cargo-binstall +ENV BINSTALL_VERSION=1.13.0 +# renovate: datasource=github-releases depName=psastras/sbom-rs +ENV CARGO_SBOM_VERSION=0.9.1 +# renovate: datasource=crate depName=lddtree +ENV LDDTREE_VERSION=0.3.7 + +# Install unpackaged tools +RUN <> /etc/environment + +# Configure pkg-config +RUN </dev/null 2>/dev/null; then + echo "PKG_CONFIG_LIBDIR=/usr/lib/$(xx-info)/pkgconfig" >> /etc/environment + echo "PKG_CONFIG=/usr/bin/$(xx-info)-pkg-config" >> /etc/environment + fi + echo "PKG_CONFIG_ALLOW_CROSS=true" >> /etc/environment +EOF + +# Configure cc to use clang version +RUN <> /etc/environment + echo "CXX=clang++" >> /etc/environment +EOF + +# Cross-language LTO +RUN <> /etc/environment + echo "CXXFLAGS=-flto" >> /etc/environment + # Linker is set to target-compatible clang by xx + echo "RUSTFLAGS='-Clinker-plugin-lto -Clink-arg=-fuse-ld=lld'" >> /etc/environment +EOF + +# Apply CPU-specific optimizations if TARGET_CPU is provided +ARG TARGET_CPU + +RUN <> /etc/environment + echo "CXXFLAGS='${CXXFLAGS} -march=${TARGET_CPU}'" >> /etc/environment + echo "RUSTFLAGS='${RUSTFLAGS} -C target-cpu=${TARGET_CPU}'" >> /etc/environment + fi +EOF + +# Prepare output directories +RUN mkdir /out + +FROM toolchain AS builder + + +# Get source +COPY . . + +ARG TARGETPLATFORM + +# Verify environment configuration +RUN xx-cargo --print-target-triple + +# Conduwuit version info +ARG GIT_COMMIT_HASH +ARG GIT_COMMIT_HASH_SHORT +ARG GIT_REMOTE_URL +ARG GIT_REMOTE_COMMIT_URL +ARG CONDUWUIT_VERSION_EXTRA +ARG CONTINUWUITY_VERSION_EXTRA +ENV GIT_COMMIT_HASH=$GIT_COMMIT_HASH +ENV GIT_COMMIT_HASH_SHORT=$GIT_COMMIT_HASH_SHORT +ENV GIT_REMOTE_URL=$GIT_REMOTE_URL +ENV GIT_REMOTE_COMMIT_URL=$GIT_REMOTE_COMMIT_URL +ENV CONDUWUIT_VERSION_EXTRA=$CONDUWUIT_VERSION_EXTRA +ENV CONTINUWUITY_VERSION_EXTRA=$CONTINUWUITY_VERSION_EXTRA + +ARG RUST_PROFILE=release + +# Build the binary +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + --mount=type=cache,target=/app/target,id=cargo-target-${TARGET_CPU}-${TARGETPLATFORM}-musl-${RUST_PROFILE} \ + bash <<'EOF' + set -o allexport + set -o xtrace + . /etc/environment + TARGET_DIR=($(cargo metadata --no-deps --format-version 1 | \ + jq -r ".target_directory")) + mkdir /out/sbin + PACKAGE=conduwuit + xx-cargo build --locked --profile ${RUST_PROFILE} \ + -p $PACKAGE --features libclang_static; + BINARIES=($(cargo metadata --no-deps --format-version 1 | \ + jq -r ".packages[] | select(.name == \"$PACKAGE\") | .targets[] | select( .kind | map(. == \"bin\") | any ) | .name")) + for BINARY in "${BINARIES[@]}"; do + echo $BINARY + xx-verify $TARGET_DIR/$(xx-cargo --print-target-triple)/release/$BINARY + cp $TARGET_DIR/$(xx-cargo --print-target-triple)/release/$BINARY /out/sbin/$BINARY + done +EOF + +# Generate Software Bill of Materials (SBOM) +RUN --mount=type=cache,target=/usr/local/cargo/registry \ + --mount=type=cache,target=/usr/local/cargo/git/db \ + bash <<'EOF' + set -o xtrace + mkdir /out/sbom + typeset -A PACKAGES + for BINARY in /out/sbin/*; do + BINARY_BASE=$(basename ${BINARY}) + package=$(cargo metadata --no-deps --format-version 1 | jq -r ".packages[] | select(.targets[] | select( .kind | map(. == \"bin\") | any ) | .name == \"$BINARY_BASE\") | .name") + if [ -z "$package" ]; then + continue + fi + PACKAGES[$package]=1 + done + for PACKAGE in $(echo ${!PACKAGES[@]}); do + echo $PACKAGE + cargo sbom --cargo-package $PACKAGE > /out/sbom/$PACKAGE.spdx.json + done +EOF + +# Extract dynamically linked dependencies +RUN < Date: Thu, 24 Jul 2025 21:51:52 +0100 Subject: [PATCH 1984/2091] chore: Update rocksdb, feature flag changes Most of the way to static musl builds, just zlib I think --- .forgejo/workflows/rust-checks.yml | 4 ++-- CONTRIBUTING.md | 6 +++--- Cargo.lock | 29 +++++++++++++++++++++++------ Cargo.toml | 2 +- docker/musl.Dockerfile | 2 +- docs/deploying/generic.md | 2 +- engage.toml | 6 +++--- src/database/Cargo.toml | 8 ++++++++ src/main/Cargo.toml | 27 +++++++++++++++++++++------ tests/cargo_smoke.sh | 2 +- 10 files changed, 64 insertions(+), 24 deletions(-) diff --git a/.forgejo/workflows/rust-checks.yml b/.forgejo/workflows/rust-checks.yml index 0595c053..c46363a0 100644 --- a/.forgejo/workflows/rust-checks.yml +++ b/.forgejo/workflows/rust-checks.yml @@ -73,7 +73,7 @@ jobs: run: | cargo clippy \ --workspace \ - --all-features \ + --features full \ --locked \ --no-deps \ --profile test \ @@ -133,7 +133,7 @@ jobs: run: | cargo test \ --workspace \ - --all-features \ + --features full \ --locked \ --profile test \ --all-targets \ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2e11cff4..b5ecf38a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,11 +65,11 @@ Tests, compilation, and linting can be run with standard Cargo commands: cargo test # Check compilation -cargo check --workspace --all-features +cargo check --workspace --features full # Run lints -cargo clippy --workspace --all-features -# Auto-fix: cargo clippy --workspace --all-features --fix --allow-staged; +cargo clippy --workspace --features full +# Auto-fix: cargo clippy --workspace --features full --fix --allow-staged; # Format code (must use nightly) cargo +nightly fmt diff --git a/Cargo.lock b/Cargo.lock index 1866a691..bfab6255 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -543,6 +543,24 @@ dependencies = [ "syn", ] +[[package]] +name = "bindgen" +version = "0.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f72209734318d0b619a5e0f5129918b848c416e122a3c4ce054e03cb87b726f" +dependencies = [ + "bitflags 2.9.1", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.1", + "shlex", + "syn", +] + [[package]] name = "bit_field" version = "0.10.2" @@ -847,7 +865,6 @@ dependencies = [ name = "conduwuit" version = "0.5.0-rc.6" dependencies = [ - "bindgen 0.71.1", "clap", "conduwuit_admin", "conduwuit_api", @@ -4117,10 +4134,10 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" -version = "0.33.0+9.11.1" -source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd#fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd" +version = "0.38.0+10.4.2" +source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=01e1128898fc4bbb776e7a6deec2aa3b675b0442#01e1128898fc4bbb776e7a6deec2aa3b675b0442" dependencies = [ - "bindgen 0.71.1", + "bindgen 0.72.0", "bzip2-sys", "cc", "glob", @@ -4134,8 +4151,8 @@ dependencies = [ [[package]] name = "rust-rocksdb" -version = "0.37.0" -source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd#fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd" +version = "0.42.1" +source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=01e1128898fc4bbb776e7a6deec2aa3b675b0442#01e1128898fc4bbb776e7a6deec2aa3b675b0442" dependencies = [ "libc", "rust-librocksdb-sys", diff --git a/Cargo.toml b/Cargo.toml index 9cb5ff84..e95567bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -391,7 +391,7 @@ features = [ [workspace.dependencies.rust-rocksdb] git = "https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1" -rev = "fc9a99ac54a54208f90fdcba33ae6ee8bc3531dd" +rev = "01e1128898fc4bbb776e7a6deec2aa3b675b0442" default-features = false features = [ "multi-threaded-cf", diff --git a/docker/musl.Dockerfile b/docker/musl.Dockerfile index d5317d15..b55aecb0 100644 --- a/docker/musl.Dockerfile +++ b/docker/musl.Dockerfile @@ -133,7 +133,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ mkdir /out/sbin PACKAGE=conduwuit xx-cargo build --locked --profile ${RUST_PROFILE} \ - -p $PACKAGE --features libclang_static; + -p $PACKAGE --no-default-features --features bindgen-static,release_max_log_level,standard; BINARIES=($(cargo metadata --no-deps --format-version 1 | \ jq -r ".packages[] | select(.name == \"$PACKAGE\") | .targets[] | select( .kind | map(. == \"bin\") | any ) | .name")) for BINARY in "${BINARIES[@]}"; do diff --git a/docs/deploying/generic.md b/docs/deploying/generic.md index 3f9d1a16..9f5051f7 100644 --- a/docs/deploying/generic.md +++ b/docs/deploying/generic.md @@ -44,7 +44,7 @@ If wanting to build using standard Rust toolchains, make sure you install: - (On linux) `pkg-config` on the compiling machine to allow finding `liburing` - A C++ compiler and (on linux) `libclang` for RocksDB -You can build Continuwuity using `cargo build --release --all-features`. +You can build Continuwuity using `cargo build --release`. ### Building with Nix diff --git a/engage.toml b/engage.toml index 5048fde2..92e9d399 100644 --- a/engage.toml +++ b/engage.toml @@ -83,7 +83,7 @@ env DIRENV_DEVSHELL=all-features \ --workspace \ --locked \ --profile test \ - --all-features \ + --features full \ --no-deps \ --document-private-items \ --color always @@ -96,7 +96,7 @@ script = """ direnv exec . \ cargo clippy \ --workspace \ - --all-features \ + --features full \ --locked \ --profile test \ --color=always \ @@ -114,7 +114,7 @@ env DIRENV_DEVSHELL=all-features \ --workspace \ --locked \ --profile test \ - --all-features \ + --features full \ --color=always \ -- \ -D warnings diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 55d4793f..9f51f366 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -44,6 +44,14 @@ zstd_compression = [ "conduwuit-core/zstd_compression", "rust-rocksdb/zstd", ] +bindgen-static = [ + # "bindgen/static" + # "clang-sys/static" + "rust-rocksdb/bindgen-static" +] +bindgen-runtime = [ + "rust-rocksdb/bindgen-runtime" +] [dependencies] async-channel.workspace = true diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 3b97f625..0d7dd844 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -43,6 +43,11 @@ assets = [ [features] default = [ + "standard", + "release_max_log_level", + "bindgen-runtime", # replace with bindgen-static on alpine +] +standard = [ "blurhashing", "brotli_compression", "element_hacks", @@ -52,10 +57,17 @@ default = [ "jemalloc_conf", "journald", "media_thumbnail", - "release_max_log_level", "systemd", "url_preview", - "zstd_compression", + "zstd_compression" +] +full = [ + "standard", + "hardened_malloc", + "jemalloc_prof", + "perf_measurements", + "tokio_console" + # sentry_telemetry ] blurhashing = [ @@ -161,14 +173,17 @@ zstd_compression = [ conduwuit_mods = [ "conduwuit-core/conduwuit_mods", ] - -libclang_static = [ - "bindgen/static" +bindgen-static = [ + # "bindgen/static" # "clang-sys/static" + "conduwuit-database/bindgen-static" +] +bindgen-runtime = [ + "conduwuit-database/bindgen-runtime" ] [build-dependencies] -bindgen = {version = "0.71.1", default-features = false} +# bindgen = {version = "0.71.1", default-features = false} # clang-sys = {version = "1", default-features = false} [dependencies] diff --git a/tests/cargo_smoke.sh b/tests/cargo_smoke.sh index 946790c3..e8bb5512 100755 --- a/tests/cargo_smoke.sh +++ b/tests/cargo_smoke.sh @@ -48,7 +48,7 @@ vector () { VECTOR_OPTS=$@ element "$TOOLCHAIN" $VECTOR_OPTS --no-default-features element "$TOOLCHAIN" $VECTOR_OPTS --features=default - element "$TOOLCHAIN" $VECTOR_OPTS --all-features + element "$TOOLCHAIN" $VECTOR_OPTS --features full } matrix () { From 205506f206fef4c5fb51141449476fab8af151ef Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 24 Jul 2025 22:18:10 +0100 Subject: [PATCH 1985/2091] chore: Update deps --- Cargo.lock | 223 ++++++++++++++++++++++++++++------------------------- 1 file changed, 118 insertions(+), 105 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bfab6255..837e033a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,9 +52,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -73,27 +73,27 @@ checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.8" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6680de5231bd6ee4c6191b8a1325daa282b415391ec9d3a37bd34f2060dc73fa" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", "once_cell_polyfill", @@ -217,9 +217,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.25" +version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f6024f3f856663b45fd0c9b6f2024034a702f453549449e0d84a305900dad4" +checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8" dependencies = [ "brotli", "flate2", @@ -301,18 +301,18 @@ dependencies = [ [[package]] name = "avif-serialize" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98922d6a4cfbcb08820c69d8eeccc05bb1f29bfa06b4f5b1dbfe9a868bd7608e" +checksum = "2ea8ef51aced2b9191c08197f55450d830876d9933f8f48a429b354f1d496b42" dependencies = [ "arrayvec", ] [[package]] name = "aws-lc-rs" -version = "1.13.1" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" +checksum = "5c953fe1ba023e6b7730c0d4b031d06f267f23a46167dcbd40316644b10a17ba" dependencies = [ "aws-lc-sys", "zeroize", @@ -320,9 +320,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.29.0" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" +checksum = "dbfd150b5dbdb988bcc8fb1fe787eb6b7ee6180ca24da683b61ea5405f3d43ff" dependencies = [ "bindgen 0.69.5", "cc", @@ -552,7 +552,7 @@ dependencies = [ "bitflags 2.9.1", "cexpr", "clang-sys", - "itertools 0.12.1", + "itertools 0.13.0", "proc-macro2", "quote", "regex", @@ -647,9 +647,9 @@ checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" [[package]] name = "bumpalo" -version = "3.18.1" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytemuck" @@ -703,9 +703,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.27" +version = "1.2.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" +checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" dependencies = [ "jobserver", "libc", @@ -774,9 +774,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" +checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" dependencies = [ "clap_builder", "clap_derive", @@ -793,9 +793,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" +checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" dependencies = [ "anstream", "anstyle", @@ -805,9 +805,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.40" +version = "4.5.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" +checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" dependencies = [ "heck", "proc-macro2", @@ -823,9 +823,9 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "clap_mangen" -version = "0.2.26" +version = "0.2.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "724842fa9b144f9b89b3f3d371a89f3455eea660361d13a554f68f8ae5d6c13a" +checksum = "e2fb6d3f935bbb9819391528b0e7cf655e78a0bc7a7c3d227211a1d24fc11db1" dependencies = [ "clap", "roff", @@ -848,9 +848,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "concurrent-queue" @@ -1173,15 +1173,15 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const-str" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e991226a70654b49d34de5ed064885f0bef0348a8e70018b8ff1ac80aa984a2" +checksum = "041fbfcf8e7054df725fb9985297e92422cdc80fcf313665f5ca3d761bb63f4c" [[package]] name = "const_panic" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2459fc9262a1aa204eb4b5764ad4f189caec88aea9634389c0a25f8be7f6265e" +checksum = "b98d1483e98c9d67f341ab4b3915cfdc54740bd6f5cccc9226ee0535d86aa8fb" [[package]] name = "convert_case" @@ -1238,9 +1238,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] @@ -1346,7 +1346,7 @@ dependencies = [ "futures-core", "mio", "parking_lot", - "rustix 1.0.7", + "rustix 1.0.8", "signal-hook", "signal-hook-mio", "winapi", @@ -1363,9 +1363,9 @@ dependencies = [ [[package]] name = "crunchy" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" @@ -1531,9 +1531,9 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" dependencies = [ "curve25519-dalek", "ed25519", @@ -1885,9 +1885,9 @@ dependencies = [ [[package]] name = "gif" -version = "0.13.1" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +checksum = "4ae047235e33e2829703574b54fdec96bfbad892062d97fed2f76022287de61b" dependencies = [ "color_quant", "weezl", @@ -1907,9 +1907,9 @@ checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" +checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" dependencies = [ "atomic-waker", "bytes", @@ -1917,7 +1917,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.9.0", + "indexmap 2.10.0", "slab", "tokio", "tokio-util", @@ -2047,7 +2047,7 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.9.1", + "rand 0.9.2", "ring", "serde", "thiserror 2.0.12", @@ -2091,7 +2091,7 @@ dependencies = [ "moka", "once_cell", "parking_lot", - "rand 0.9.1", + "rand 0.9.2", "resolv-conf", "serde", "smallvec", @@ -2240,7 +2240,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.1", + "webpki-roots 1.0.2", ] [[package]] @@ -2433,9 +2433,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" dependencies = [ "equivalent", "hashbrown 0.15.4", @@ -2465,6 +2465,17 @@ dependencies = [ "syn", ] +[[package]] +name = "io-uring" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "libc", +] + [[package]] name = "ipaddress" version = "0.1.3" @@ -2548,9 +2559,9 @@ dependencies = [ [[package]] name = "jpeg-decoder" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" [[package]] name = "js-sys" @@ -2649,9 +2660,9 @@ checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libfuzzer-sys" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf78f52d400cf2d84a3a973a78a592b4adc535739e0a5597a0da6f0c357adc75" +checksum = "5037190e1f70cbeef565bd267599242926f724d3b8a9f510fd7e0b540cfa4404" dependencies = [ "arbitrary", "cc", @@ -3131,7 +3142,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.9.0", + "indexmap 2.10.0", "js-sys", "once_cell", "pin-project-lite", @@ -3307,7 +3318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.9.0", + "indexmap 2.10.0", ] [[package]] @@ -3398,12 +3409,12 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "plist" -version = "1.7.2" +version = "1.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d77244ce2d584cd84f6a15f86195b8c9b2a0dfbfd817c09e0464244091a58ed" +checksum = "3af6b589e163c5a788fab00ce0c0366f6efbb9959c2f9874b224936af7fce7e1" dependencies = [ "base64 0.22.1", - "indexmap 2.9.0", + "indexmap 2.10.0", "quick-xml", "serde", "time", @@ -3460,9 +3471,9 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.35" +version = "0.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a" +checksum = "ff24dfcda44452b9816fff4cd4227e1bb73ff5a2f1bc1105aa92fb8565ce44d2" dependencies = [ "proc-macro2", "syn", @@ -3501,18 +3512,18 @@ dependencies = [ [[package]] name = "profiling" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afbdc74edc00b6f6a218ca6a5364d6226a259d4b8ea1af4a0ea063f27e179f4d" +checksum = "3eb8486b569e12e2c32ad3e204dbaba5e4b5b216e9367044f25f1dba42341773" dependencies = [ "profiling-procmacros", ] [[package]] name = "profiling-procmacros" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" +checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" dependencies = [ "quote", "syn", @@ -3585,9 +3596,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.37.5" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +checksum = "8927b0664f5c5a98265138b7e3f90aa19a6b21353182469ace36d4ac527b7b1b" dependencies = [ "memchr", ] @@ -3621,7 +3632,7 @@ dependencies = [ "bytes", "getrandom 0.3.3", "lru-slab", - "rand 0.9.1", + "rand 0.9.2", "ring", "rustc-hash 2.1.1", "rustls", @@ -3675,9 +3686,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", @@ -3758,9 +3769,9 @@ dependencies = [ [[package]] name = "ravif" -version = "0.11.12" +version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6a5f31fcf7500f9401fea858ea4ab5525c99f2322cfcee732c0e6c74208c0c6" +checksum = "5825c26fddd16ab9f515930d49028a630efec172e903483c94796cfe31893e6b" dependencies = [ "avif-serialize", "imgref", @@ -3804,9 +3815,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.13" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec" dependencies = [ "bitflags 2.9.1", ] @@ -3912,9 +3923,9 @@ source = "git+https://forgejo.ellis.link/continuwuation/resolv-conf?rev=56251316 [[package]] name = "rgb" -version = "0.8.50" +version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57397d16646700483b67d2dd6511d79318f9d057fdbd21a4066aeac8b41d310a" +checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" [[package]] name = "ring" @@ -4002,7 +4013,7 @@ dependencies = [ "form_urlencoded", "getrandom 0.2.16", "http", - "indexmap 2.9.0", + "indexmap 2.10.0", "js_int", "konst", "percent-encoding", @@ -4029,7 +4040,7 @@ version = "0.28.1" source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" dependencies = [ "as_variant", - "indexmap 2.9.0", + "indexmap 2.10.0", "js_int", "js_option", "percent-encoding", @@ -4200,22 +4211,22 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" dependencies = [ "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "rustls" -version = "0.23.28" +version = "0.23.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" +checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1" dependencies = [ "aws-lc-rs", "log", @@ -4260,9 +4271,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.3" +version = "0.103.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" +checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" dependencies = [ "aws-lc-rs", "ring", @@ -4526,7 +4537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", - "indexmap 2.9.0", + "indexmap 2.10.0", "itoa", "ryu", "serde", @@ -4534,9 +4545,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.141" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" dependencies = [ "itoa", "memchr", @@ -4591,7 +4602,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.9.0", + "indexmap 2.10.0", "itoa", "ryu", "serde", @@ -5069,16 +5080,18 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.45.1" +version = "1.46.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" dependencies = [ "backtrace", "bytes", + "io-uring", "libc", "mio", "pin-project-lite", "signal-hook-registry", + "slab", "socket2", "tokio-macros", "tracing", @@ -5098,9 +5111,9 @@ dependencies = [ [[package]] name = "tokio-metrics" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7817b32d36c9b94744d7aa3f8fc13526aa0f5112009d7045f3c659413a6e44ac" +checksum = "23ff82f660c98e4ff60da5eb8fa864a4130f34b56d92d5cd23d6fdfcc14e95fa" dependencies = [ "futures-util", "pin-project-lite", @@ -5181,7 +5194,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.9.0", + "indexmap 2.10.0", "serde", "serde_spanned", "toml_datetime", @@ -5405,9 +5418,9 @@ checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "typewit" -version = "1.11.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb77c29baba9e4d3a6182d51fa75e3215c7fd1dab8f4ea9d107c716878e55fc0" +checksum = "97e72ba082eeb9da9dc68ff5a2bf727ef6ce362556e8d29ec1aed3bd05e7d86a" dependencies = [ "typewit_proc_macros", ] @@ -5719,14 +5732,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.1", + "webpki-roots 1.0.2", ] [[package]] name = "webpki-roots" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8782dd5a41a24eed3a4f40b606249b3e236ca61adf1f25ea4d45c73de122b502" +checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" dependencies = [ "rustls-pki-types", ] @@ -6137,9 +6150,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" +checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" dependencies = [ "memchr", ] @@ -6356,9 +6369,9 @@ dependencies = [ [[package]] name = "zune-jpeg" -version = "0.4.17" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6fe2e33d02a98ee64423802e16df3de99c43e5cf5ff983767e1128b394c8ac" +checksum = "2c9e525af0a6a658e031e95f14b7f889976b74a11ba0eca5a5fc9ac8a1c43a6a" dependencies = [ "zune-core", ] From 87be4d1a52d6bd1680a47153f020affc166d1a07 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 24 Jul 2025 23:22:07 +0100 Subject: [PATCH 1986/2091] feat: Almost-functional musl builds on Alpine Lots of fiddling, still can't get stuff to work Next step is a debian builder copying the static libs from alpine --- Cargo.lock | 23 ++--------------------- Cargo.toml | 2 +- docker/musl.Dockerfile | 5 ++--- 3 files changed, 5 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 837e033a..e5c111d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -525,24 +525,6 @@ dependencies = [ "which", ] -[[package]] -name = "bindgen" -version = "0.71.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" -dependencies = [ - "bitflags 2.9.1", - "cexpr", - "clang-sys", - "itertools 0.13.0", - "proc-macro2", - "quote", - "regex", - "rustc-hash 2.1.1", - "shlex", - "syn", -] - [[package]] name = "bindgen" version = "0.72.0" @@ -4146,7 +4128,7 @@ dependencies = [ [[package]] name = "rust-librocksdb-sys" version = "0.38.0+10.4.2" -source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=01e1128898fc4bbb776e7a6deec2aa3b675b0442#01e1128898fc4bbb776e7a6deec2aa3b675b0442" +source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=99b0319416b64830dd6f8943e1f65e15aeef18bc#99b0319416b64830dd6f8943e1f65e15aeef18bc" dependencies = [ "bindgen 0.72.0", "bzip2-sys", @@ -4163,7 +4145,7 @@ dependencies = [ [[package]] name = "rust-rocksdb" version = "0.42.1" -source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=01e1128898fc4bbb776e7a6deec2aa3b675b0442#01e1128898fc4bbb776e7a6deec2aa3b675b0442" +source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=99b0319416b64830dd6f8943e1f65e15aeef18bc#99b0319416b64830dd6f8943e1f65e15aeef18bc" dependencies = [ "libc", "rust-librocksdb-sys", @@ -6347,7 +6329,6 @@ version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ - "bindgen 0.71.1", "cc", "pkg-config", ] diff --git a/Cargo.toml b/Cargo.toml index e95567bd..e7abca01 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -391,7 +391,7 @@ features = [ [workspace.dependencies.rust-rocksdb] git = "https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1" -rev = "01e1128898fc4bbb776e7a6deec2aa3b675b0442" +rev = "99b0319416b64830dd6f8943e1f65e15aeef18bc" default-features = false features = [ "multi-threaded-cf", diff --git a/docker/musl.Dockerfile b/docker/musl.Dockerfile index b55aecb0..2221525b 100644 --- a/docker/musl.Dockerfile +++ b/docker/musl.Dockerfile @@ -13,7 +13,7 @@ FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-alpine${ALPINE_VERSION} AS t RUN --mount=type=cache,target=/etc/apk/cache apk add \ build-base pkgconfig make jq bash \ curl git file \ - llvm-dev clang clang-dev lld + llvm-dev clang clang-static lld # Developer tool versions @@ -37,8 +37,7 @@ COPY --from=xx / / ARG TARGETPLATFORM # Install libraries linked by the binary -RUN --mount=type=cache,target=/etc/apk/cache xx-apk add musl-dev liburing-dev clang-dev - +RUN --mount=type=cache,target=/etc/apk/cache xx-apk add musl-dev gcc g++ liburing-dev # Set up Rust toolchain WORKDIR /app From 00c7e220bbefaca35b44d64ec0af4d3d146a0fbd Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 25 Jul 2025 14:10:06 +0100 Subject: [PATCH 1987/2091] chore: Release --- Cargo.lock | 24 ++++++++++++------------ Cargo.toml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e5c111d7..5d7192b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -845,7 +845,7 @@ dependencies = [ [[package]] name = "conduwuit" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "clap", "conduwuit_admin", @@ -875,7 +875,7 @@ dependencies = [ [[package]] name = "conduwuit_admin" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "clap", "conduwuit_api", @@ -896,7 +896,7 @@ dependencies = [ [[package]] name = "conduwuit_api" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "async-trait", "axum", @@ -928,14 +928,14 @@ dependencies = [ [[package]] name = "conduwuit_build_metadata" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "built 0.8.0", ] [[package]] name = "conduwuit_core" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "argon2", "arrayvec", @@ -996,7 +996,7 @@ dependencies = [ [[package]] name = "conduwuit_database" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "async-channel", "conduwuit_core", @@ -1014,7 +1014,7 @@ dependencies = [ [[package]] name = "conduwuit_macros" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "itertools 0.14.0", "proc-macro2", @@ -1024,7 +1024,7 @@ dependencies = [ [[package]] name = "conduwuit_router" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "axum", "axum-client-ip", @@ -1058,7 +1058,7 @@ dependencies = [ [[package]] name = "conduwuit_service" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "async-trait", "base64 0.22.1", @@ -1096,7 +1096,7 @@ dependencies = [ [[package]] name = "conduwuit_web" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "askama", "axum", @@ -6177,7 +6177,7 @@ dependencies = [ [[package]] name = "xtask" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "clap", "serde", @@ -6186,7 +6186,7 @@ dependencies = [ [[package]] name = "xtask-generate-commands" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" dependencies = [ "clap-markdown", "clap_builder", diff --git a/Cargo.toml b/Cargo.toml index e7abca01..c656e183 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://forgejo.ellis.link/continuwuation/continuwuity" rust-version = "1.86.0" -version = "0.5.0-rc.6" +version = "0.5.0-rc.7" [workspace.metadata.crane] name = "conduwuit" From aa08edc55f2f14d3c819b71fb13bc4ac89f0a540 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 25 Jul 2025 17:30:31 +0100 Subject: [PATCH 1988/2091] chore: Release announcement --- docs/static/announcements.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/static/announcements.json b/docs/static/announcements.json index 7dd2fb72..8068535c 100644 --- a/docs/static/announcements.json +++ b/docs/static/announcements.json @@ -6,8 +6,8 @@ "message": "Welcome to Continuwuity! Important announcements about the project will appear here." }, { - "id": 2, - "message": "🎉 Continuwuity v0.5.0-rc.6 is now available! This release includes improved knock-restricted room handling, automatic support contact configuration, and a new HTML landing page. Check [the release notes for full details](https://forgejo.ellis.link/continuwuation/continuwuity/releases/tag/v0.5.0-rc.6) and upgrade instructions." + "id": 3, + "message": "_taps microphone_ The Continuwuity 0.5.0-rc.7 release is now available, and it's better than ever! **177 commits**, **35 pull requests**, **11 contributors,** and a lot of new stuff!\n\nFor highlights, we've got:\n\n* 🕵️ Full Policy Server support to fight spam!\n* 🚀 Smarter room & space upgrades.\n* 🚫 User suspension tools for better moderation.\n* 🤖 reCaptcha support for safer open registration.\n* 🔍 Ability to disable read receipts & typing indicators.\n* ⚡ Sweeping performance improvements!\n\nGet the [full changelog and downloads on our Forgejo](https://forgejo.ellis.link/continuwuation/continuwuity/releases/tag/v0.5.0-rc.7) - and make sure you're in the https://matrix.to/#/!releases:continuwuity.org/$hN9z6L2_dTAlPxFLAoXVfo_g8DyYXu4cpvWsSrWhmB0 to get stuff like this sooner." } ] } From 4a83df5b57b5872a3006b93723df512ac3d51507 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 25 Jul 2025 17:35:18 +0100 Subject: [PATCH 1989/2091] chore: Fix link --- docs/static/announcements.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/static/announcements.json b/docs/static/announcements.json index 8068535c..fe27859b 100644 --- a/docs/static/announcements.json +++ b/docs/static/announcements.json @@ -7,7 +7,7 @@ }, { "id": 3, - "message": "_taps microphone_ The Continuwuity 0.5.0-rc.7 release is now available, and it's better than ever! **177 commits**, **35 pull requests**, **11 contributors,** and a lot of new stuff!\n\nFor highlights, we've got:\n\n* 🕵️ Full Policy Server support to fight spam!\n* 🚀 Smarter room & space upgrades.\n* 🚫 User suspension tools for better moderation.\n* 🤖 reCaptcha support for safer open registration.\n* 🔍 Ability to disable read receipts & typing indicators.\n* ⚡ Sweeping performance improvements!\n\nGet the [full changelog and downloads on our Forgejo](https://forgejo.ellis.link/continuwuation/continuwuity/releases/tag/v0.5.0-rc.7) - and make sure you're in the https://matrix.to/#/!releases:continuwuity.org/$hN9z6L2_dTAlPxFLAoXVfo_g8DyYXu4cpvWsSrWhmB0 to get stuff like this sooner." + "message": "_taps microphone_ The Continuwuity 0.5.0-rc.7 release is now available, and it's better than ever! **177 commits**, **35 pull requests**, **11 contributors,** and a lot of new stuff!\n\nFor highlights, we've got:\n\n* 🕵️ Full Policy Server support to fight spam!\n* 🚀 Smarter room & space upgrades.\n* 🚫 User suspension tools for better moderation.\n* 🤖 reCaptcha support for safer open registration.\n* 🔍 Ability to disable read receipts & typing indicators.\n* ⚡ Sweeping performance improvements!\n\nGet the [full changelog and downloads on our Forgejo](https://forgejo.ellis.link/continuwuation/continuwuity/releases/tag/v0.5.0-rc.7) - and make sure you're in the [Announcements room](https://matrix.to/#/!releases:continuwuity.org/$hN9z6L2_dTAlPxFLAoXVfo_g8DyYXu4cpvWsSrWhmB0) to get stuff like this sooner." } ] } From 058988410986b0ba45f19181a3113d216beebe9e Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Mon, 28 Jul 2025 19:28:34 +0100 Subject: [PATCH 1990/2091] docs: Fix documentation link in README Closes https://forgejo.ellis.link/continuwuation/continuwuity/issues/913 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e63fe80f..a5f77a5b 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Continuwuity aims to: ### Can I try it out? -Check out the [documentation](introduction) for installation instructions. +Check out the [documentation](https://continuwuity.org) for installation instructions. There are currently no open registration Continuwuity instances available. From b1516209c403f7075d11c2b33a7674ce84be96f1 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 30 Jul 2025 19:23:38 +0100 Subject: [PATCH 1991/2091] chore: Update funding file --- .github/FUNDING.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index fcfaade5..841427b7 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,5 +1,4 @@ -github: [JadedBlueEyes] -# Doesn't support an array, so we can only list nex -ko_fi: nexy7574 +github: [JadedBlueEyes, nexy7574] custom: + - https://ko-fi.com/nexy7574 - https://ko-fi.com/JadedBlueEyes From 238cc627e3622a89c22b068987b1a88b3c9b6eb1 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 30 Jul 2025 19:33:53 +0100 Subject: [PATCH 1992/2091] docs: Set traefik labels --- docs/deploying/docker-compose.for-traefik.yml | 9 +++++++++ docs/deploying/docker-compose.with-traefik.yml | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/docs/deploying/docker-compose.for-traefik.yml b/docs/deploying/docker-compose.for-traefik.yml index 547712b6..f67e603b 100644 --- a/docs/deploying/docker-compose.for-traefik.yml +++ b/docs/deploying/docker-compose.for-traefik.yml @@ -12,6 +12,15 @@ services: #- ./continuwuity.toml:/etc/continuwuity.toml networks: - proxy + labels: + - "traefik.enable=true" + - "traefik.http.routers.continuwuity.rule=(Host(`matrix.example.com`) || (Host(`example.com`) && PathPrefix(`/.well-known/matrix`)))" + - "traefik.http.routers.continuwuity.entrypoints=websecure" # your HTTPS entry point + - "traefik.http.routers.continuwuity.tls=true" + - "traefik.http.routers.continuwuity.service=continuwuity" + - "traefik.http.services.continuwuity.loadbalancer.server.port=6167" + # possibly, depending on your config: + # - "traefik.http.routers.continuwuity.tls.certresolver=letsencrypt" environment: CONTINUWUITY_SERVER_NAME: your.server.name.example # EDIT THIS CONTINUWUITY_DATABASE_PATH: /var/lib/continuwuity diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index 49b7c905..b41a6fbc 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -12,6 +12,14 @@ services: #- ./continuwuity.toml:/etc/continuwuity.toml networks: - proxy + labels: + - "traefik.enable=true" + - "traefik.http.routers.homeserver.rule=(Host(`matrix.example.com`) || (Host(`example.com`) && PathPrefix(`/.well-known/matrix`)))" + - "traefik.http.routers.homeserver.entrypoints=websecure" + - "traefik.http.routers.homeserver.tls.certresolver=letsencrypt" + - "traefik.http.services.homeserver.loadbalancer.server.port=6167" + # Uncomment and adjust the following if you want to use middleware + # - "traefik.http.routers.homeserver.middlewares=secureHeaders@file" environment: CONTINUWUITY_SERVER_NAME: your.server.name.example # EDIT THIS CONTINUWUITY_TRUSTED_SERVERS: '["matrix.org"]' From 5775e0ad9d685fc5a55ff979ee3c0bd82ee88ea1 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 30 Jul 2025 19:55:48 +0100 Subject: [PATCH 1993/2091] docs: Make traefik router names consistent --- docs/deploying/docker-compose.with-traefik.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/deploying/docker-compose.with-traefik.yml b/docs/deploying/docker-compose.with-traefik.yml index b41a6fbc..8021e034 100644 --- a/docs/deploying/docker-compose.with-traefik.yml +++ b/docs/deploying/docker-compose.with-traefik.yml @@ -14,12 +14,12 @@ services: - proxy labels: - "traefik.enable=true" - - "traefik.http.routers.homeserver.rule=(Host(`matrix.example.com`) || (Host(`example.com`) && PathPrefix(`/.well-known/matrix`)))" - - "traefik.http.routers.homeserver.entrypoints=websecure" - - "traefik.http.routers.homeserver.tls.certresolver=letsencrypt" - - "traefik.http.services.homeserver.loadbalancer.server.port=6167" + - "traefik.http.routers.continuwuity.rule=(Host(`matrix.example.com`) || (Host(`example.com`) && PathPrefix(`/.well-known/matrix`)))" + - "traefik.http.routers.continuwuity.entrypoints=websecure" + - "traefik.http.routers.continuwuity.tls.certresolver=letsencrypt" + - "traefik.http.services.continuwuity.loadbalancer.server.port=6167" # Uncomment and adjust the following if you want to use middleware - # - "traefik.http.routers.homeserver.middlewares=secureHeaders@file" + # - "traefik.http.routers.continuwuity.middlewares=secureHeaders@file" environment: CONTINUWUITY_SERVER_NAME: your.server.name.example # EDIT THIS CONTINUWUITY_TRUSTED_SERVERS: '["matrix.org"]' From e4a43b1a5b31bfe5267fafa01b177e0f992d150e Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 2 Aug 2025 00:19:33 +0100 Subject: [PATCH 1994/2091] fix(policy-server): Call the PS later in the PDU creation process This avoids accidentally sending partially built PDUs to the policy server, which may cause issues with some implementations --- src/service/rooms/timeline/create.rs | 38 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/service/rooms/timeline/create.rs b/src/service/rooms/timeline/create.rs index 66a35eca..6732cd8e 100644 --- a/src/service/rooms/timeline/create.rs +++ b/src/service/rooms/timeline/create.rs @@ -165,25 +165,6 @@ pub async fn create_hash_and_sign_event( return Err!(Request(Forbidden("Event is not authorized."))); } - // Check with the policy server - match self - .services - .event_handler - .ask_policy_server(&pdu, room_id) - .await - { - | Ok(true) => {}, - | Ok(false) => { - return Err!(Request(Forbidden(debug_warn!( - "Policy server marked this event as spam" - )))); - }, - | Err(e) => { - // fail open - warn!("Failed to check event with policy server: {e}"); - }, - } - // Hash and sign let mut pdu_json = utils::to_canonical_object(&pdu).map_err(|e| { err!(Request(BadJson(warn!("Failed to convert PDU to canonical JSON: {e}")))) @@ -222,6 +203,25 @@ pub async fn create_hash_and_sign_event( pdu_json.insert("event_id".into(), CanonicalJsonValue::String(pdu.event_id.clone().into())); + // Check with the policy server + match self + .services + .event_handler + .ask_policy_server(&pdu, room_id) + .await + { + | Ok(true) => {}, + | Ok(false) => { + return Err!(Request(Forbidden(debug_warn!( + "Policy server marked this event as spam" + )))); + }, + | Err(e) => { + // fail open + warn!("Failed to check event with policy server: {e}"); + }, + } + // Generate short event id let _shorteventid = self .services From bd3db65cb2ec1ee00aa5ce6069bf044554b97d97 Mon Sep 17 00:00:00 2001 From: Yonatan Sidler Date: Wed, 6 Aug 2025 20:01:36 +0300 Subject: [PATCH 1995/2091] fix(arch): fix config.toml not being loaded from LoadCredentials directory --- arch/conduwuit.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index f7100179..34c3995e 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -64,7 +64,7 @@ StateDirectory=conduwuit RuntimeDirectory=conduwuit RuntimeDirectoryMode=0750 -Environment=CONTINUWUITY_CONFIG=${CREDENTIALS_DIRECTORY}/config.toml +Environment=CONTINUWUITY_CONFIG=%d/config.toml LoadCredential=config.toml:/etc/conduwuit/conduwuit.toml BindPaths=/var/lib/private/conduwuit:/var/lib/matrix-conduit BindPaths=/var/lib/private/conduwuit:/var/lib/private/matrix-conduit From e820551f6290bcaef46a544fe1eb6a11476df41c Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 10 Aug 2025 14:37:26 +0100 Subject: [PATCH 1996/2091] fix(appservice): create sender_localpart user during appservice startup Fixes #813: Application services were unable to work because their sender_localpart user was never created in the database, preventing authentication. This fix ensures the appservice user account is created when: - The server starts up and loads existing appservices from the database - A new appservice is registered via the admin command Additionally, if an appservice user has been accidentally deactivated, it will be automatically reactivated when the appservice starts. The solution centralises all appservice startup logic into a single `start_appservice` helper method, eliminating code duplication between the registration and startup paths. --- src/service/appservice/mod.rs | 89 +++++++++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 10 deletions(-) diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index 7be8a471..de6fcfac 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -11,7 +11,7 @@ use ruma::{RoomAliasId, RoomId, UserId, api::appservice::Registration}; use tokio::sync::{RwLock, RwLockReadGuard}; pub use self::{namespace_regex::NamespaceRegex, registration_info::RegistrationInfo}; -use crate::{Dep, sending}; +use crate::{Dep, globals, sending, users}; pub struct Service { registration_info: RwLock, @@ -20,7 +20,9 @@ pub struct Service { } struct Services { + globals: Dep, sending: Dep, + users: Dep, } struct Data { @@ -35,7 +37,9 @@ impl crate::Service for Service { Ok(Arc::new(Self { registration_info: RwLock::new(BTreeMap::new()), services: Services { + globals: args.depend::("globals"), sending: args.depend::("sending"), + users: args.depend::("users"), }, db: Data { id_appserviceregistrations: args.db["id_appserviceregistrations"].clone(), @@ -44,15 +48,34 @@ impl crate::Service for Service { } async fn worker(self: Arc) -> Result { - // Inserting registrations into cache self.iter_db_ids() .try_for_each(async |appservice| { - self.registration_info - .write() - .await - .insert(appservice.0, appservice.1.try_into()?); + let (id, registration) = appservice; - Ok(()) + // During startup, resolve any token collisions in favour of appservices + // by logging out conflicting user devices + if let Ok((user_id, device_id)) = self + .services + .users + .find_from_token(®istration.as_token) + .await + { + conduwuit::warn!( + "Token collision detected during startup: Appservice '{}' token was \ + also used by user '{}' device '{}'. Logging out the user device to \ + resolve conflict.", + id, + user_id.localpart(), + device_id + ); + + self.services + .users + .remove_device(&user_id, &device_id) + .await; + } + + self.start_appservice(id, registration).await }) .await } @@ -61,6 +84,39 @@ impl crate::Service for Service { } impl Service { + /// Starts an appservice, ensuring its sender_localpart user exists and is + /// active. Creates the user if it doesn't exist, or reactivates it if it + /// was deactivated. Then registers the appservice in memory for request + /// handling. + async fn start_appservice(&self, id: String, registration: Registration) -> Result { + let appservice_user_id = UserId::parse_with_server_name( + registration.sender_localpart.as_str(), + self.services.globals.server_name(), + )?; + + if !self.services.users.exists(&appservice_user_id).await { + self.services.users.create(&appservice_user_id, None)?; + } else if self + .services + .users + .is_deactivated(&appservice_user_id) + .await + .unwrap_or(false) + { + // Reactivate the appservice user if it was accidentally deactivated + self.services + .users + .set_password(&appservice_user_id, None)?; + } + + self.registration_info + .write() + .await + .insert(id, registration.try_into()?); + + Ok(()) + } + /// Registers an appservice and returns the ID to the caller pub async fn register_appservice( &self, @@ -68,15 +124,28 @@ impl Service { appservice_config_body: &str, ) -> Result { //TODO: Check for collisions between exclusive appservice namespaces - self.registration_info - .write() + + // Prevent token collision with existing user tokens + if self + .services + .users + .find_from_token(®istration.as_token) .await - .insert(registration.id.clone(), registration.clone().try_into()?); + .is_ok() + { + return err!(Request(InvalidParam( + "Cannot register appservice: The provided token is already in use by a user \ + device. Please generate a different token for the appservice." + ))); + } self.db .id_appserviceregistrations .insert(®istration.id, appservice_config_body); + self.start_appservice(registration.id.clone(), registration.clone()) + .await?; + Ok(()) } From d1ebcfaf0bdcc039df3e484acf6eb2428d81ba26 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 10 Aug 2025 14:38:54 +0100 Subject: [PATCH 1997/2091] fix(auth): prevent token collisions and optimise lookups Ensures access tokens are unique across both user and appservice tables to prevent authentication ambiguity and potential security issues. Changes: - On startup, automatically logout any user devices using tokens that conflict with appservice tokens (resolves in favour of appservices) and log a warning with affected user/device details - When creating new user tokens, check for conflicts with appservice tokens and generate a new token if a collision would occur - When registering new appservices, reject registration if the token is already in use by a user device - Use futures::select_ok to race token lookups concurrently for better performance (adapted from tuwunel commit 066097a8) This fix-forward approach resolves existing token collisions on startup whilst preventing new ones from being created, without breaking existing valid authentications. The find_token optimisation is adapted from tuwunel (matrix-construct/tuwunel) commit 066097a8: "Optimize user and appservice token queries" by Jason Volk. --- src/api/router/auth.rs | 41 +++++++++++++++++++++++++---------- src/service/appservice/mod.rs | 7 +++--- src/service/users/mod.rs | 28 +++++++++++++++++++++--- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 01254c32..5088e699 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -5,6 +5,14 @@ use axum_extra::{ typed_header::TypedHeaderRejectionReason, }; use conduwuit::{Err, Error, Result, debug_error, err, warn}; +use futures::{ + TryFutureExt, + future::{ + Either::{Left, Right}, + select_ok, + }, + pin_mut, +}; use ruma::{ CanonicalJsonObject, CanonicalJsonValue, OwnedDeviceId, OwnedServerName, OwnedUserId, UserId, api::{ @@ -54,17 +62,7 @@ pub(super) async fn auth( | None => request.query.access_token.as_deref(), }; - let token = if let Some(token) = token { - match services.appservice.find_from_token(token).await { - | Some(reg_info) => Token::Appservice(Box::new(reg_info)), - | _ => match services.users.find_from_token(token).await { - | Ok((user_id, device_id)) => Token::User((user_id, device_id)), - | _ => Token::Invalid, - }, - } - } else { - Token::None - }; + let token = find_token(services, token).await?; if metadata.authentication == AuthScheme::None { match metadata { @@ -342,3 +340,24 @@ async fn parse_x_matrix(request: &mut Request) -> Result { Ok(x_matrix) } + +async fn find_token(services: &Services, token: Option<&str>) -> Result { + let Some(token) = token else { + return Ok(Token::None); + }; + + let user_token = services.users.find_from_token(token).map_ok(Token::User); + + let appservice_token = services + .appservice + .find_from_token(token) + .map_ok(Box::new) + .map_ok(Token::Appservice); + + pin_mut!(user_token, appservice_token); + match select_ok([Left(user_token), Right(appservice_token)]).await { + | Err(e) if !e.is_not_found() => Err(e), + | Ok((token, _)) => Ok(token), + | _ => Ok(Token::Invalid), + } +} diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index de6fcfac..ad9c4a3f 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -133,10 +133,10 @@ impl Service { .await .is_ok() { - return err!(Request(InvalidParam( + return Err(err!(Request(InvalidParam( "Cannot register appservice: The provided token is already in use by a user \ device. Please generate a different token for the appservice." - ))); + )))); } self.db @@ -182,12 +182,13 @@ impl Service { .map(|info| info.registration) } - pub async fn find_from_token(&self, token: &str) -> Option { + pub async fn find_from_token(&self, token: &str) -> Result { self.read() .await .values() .find(|info| info.registration.as_token == token) .cloned() + .ok_or_else(|| err!(Request(NotFound("Appservice token not found")))) } /// Checks if a given user id matches any exclusive appservice regex diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index d2dfccd9..0aacc0e1 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -19,7 +19,7 @@ use ruma::{ use serde::{Deserialize, Serialize}; use serde_json::json; -use crate::{Dep, account_data, admin, globals, rooms}; +use crate::{Dep, account_data, admin, appservice, globals, rooms}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct UserSuspension { @@ -40,6 +40,7 @@ struct Services { server: Arc, account_data: Dep, admin: Dep, + appservice: Dep, globals: Dep, state_accessor: Dep, state_cache: Dep, @@ -76,6 +77,7 @@ impl crate::Service for Service { server: args.server.clone(), account_data: args.depend::("account_data"), admin: args.depend::("admin"), + appservice: args.depend::("appservice"), globals: args.depend::("globals"), state_accessor: args .depend::("rooms::state_accessor"), @@ -407,6 +409,26 @@ impl Service { ))); } + // Prevent token collisions with appservice tokens + let final_token = if self + .services + .appservice + .find_from_token(token) + .await + .is_ok() + { + let new_token = utils::random_string(32); + conduwuit::debug_warn!( + "Token collision prevented: Generated new token for user '{}' device '{}' \ + (original token conflicted with an appservice)", + user_id.localpart(), + device_id + ); + new_token + } else { + token.to_owned() + }; + // Remove old token if let Ok(old_token) = self.db.userdeviceid_token.qry(&key).await { self.db.token_userdeviceid.remove(&old_token); @@ -414,8 +436,8 @@ impl Service { } // Assign token to user device combination - self.db.userdeviceid_token.put_raw(key, token); - self.db.token_userdeviceid.raw_put(token, key); + self.db.userdeviceid_token.put_raw(key, &final_token); + self.db.token_userdeviceid.raw_put(&final_token, key); Ok(()) } From 9286838d23aa5b60bb0a030d403879406732bbc9 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 10 Aug 2025 18:22:20 +0100 Subject: [PATCH 1998/2091] fix(relations): improve thread pagination and include root event Replace unreliable PduCount pagination tokens with ShortEventId throughout the relations and messages endpoints. ShortEventId provides stable, unique identifiers that persist across server restarts and database operations. Key improvements: - Add token parsing helpers that try ShortEventId first, fall back to PduCount for backwards compatibility - Include thread root event when paginating backwards to thread start - Fix off-by-one error in get_relations that was returning the starting event in results - Only return next_batch/prev_batch tokens when more events are available, preventing clients from making unnecessary requests at thread boundaries - Ensure consistent token format between /relations, /messages, and /sync endpoints for interoperability This fixes duplicate events when scrolling at thread boundaries and ensures the thread root message is visible when viewing a thread, matching expected client behaviour. --- src/api/client/message.rs | 60 +++++++++--- src/api/client/relations.rs | 129 +++++++++++++++++++++---- src/service/rooms/pdu_metadata/data.rs | 14 ++- 3 files changed, 168 insertions(+), 35 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index f8818ebb..95a135e1 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,14 +1,14 @@ use axum::extract::State; use conduwuit::{ - Err, Result, at, + Err, Result, at, err, matrix::{ event::{Event, Matches}, - pdu::PduCount, + pdu::{PduCount, ShortEventId}, }, ref_at, utils::{ IterStream, ReadyExt, - result::{FlatOk, LogErr}, + result::LogErr, stream::{BroadbandExt, TryIgnore, WidebandExt}, }, }; @@ -61,6 +61,39 @@ const IGNORED_MESSAGE_TYPES: &[TimelineEventType] = &[ const LIMIT_MAX: usize = 100; const LIMIT_DEFAULT: usize = 10; +/// Parse a pagination token, trying ShortEventId first, then falling back to +/// PduCount +async fn parse_pagination_token( + _services: &Services, + _room_id: &RoomId, + token: Option<&str>, + default: PduCount, +) -> Result { + let Some(token) = token else { + return Ok(default); + }; + + // Try parsing as ShortEventId first + if let Ok(shorteventid) = token.parse::() { + // ShortEventId maps directly to a PduCount in our database + Ok(PduCount::Normal(shorteventid)) + } else if let Ok(count) = token.parse::() { + // Fallback to PduCount for backwards compatibility + Ok(PduCount::Normal(count)) + } else if let Ok(count) = token.parse::() { + // Also handle negative counts for backfilled events + Ok(PduCount::from_signed(count)) + } else { + Err(err!(Request(InvalidParam("Invalid pagination token")))) + } +} + +/// Convert a PduCount to a token string (using the underlying ShortEventId) +fn count_to_token(count: PduCount) -> String { + // The PduCount's unsigned value IS the ShortEventId + count.into_unsigned().to_string() +} + /// # `GET /_matrix/client/r0/rooms/{roomId}/messages` /// /// Allows paginating through room history. @@ -81,17 +114,18 @@ pub(crate) async fn get_message_events_route( return Err!(Request(Forbidden("Room does not exist to this server"))); } - let from: PduCount = body - .from - .as_deref() - .map(str::parse) - .transpose()? - .unwrap_or_else(|| match body.dir { + let from: PduCount = + parse_pagination_token(&services, room_id, body.from.as_deref(), match body.dir { | Direction::Forward => PduCount::min(), | Direction::Backward => PduCount::max(), - }); + }) + .await?; - let to: Option = body.to.as_deref().map(str::parse).flat_ok(); + let to: Option = if let Some(to_str) = body.to.as_deref() { + Some(parse_pagination_token(&services, room_id, Some(to_str), PduCount::min()).await?) + } else { + None + }; let limit: usize = body .limit @@ -180,8 +214,8 @@ pub(crate) async fn get_message_events_route( .collect(); Ok(get_message_events::v3::Response { - start: from.to_string(), - end: next_token.as_ref().map(ToString::to_string), + start: count_to_token(from), + end: next_token.map(count_to_token), chunk, state, }) diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index 1aa34ada..48bcde20 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,7 +1,11 @@ use axum::extract::State; use conduwuit::{ - Result, at, - matrix::{Event, event::RelationTypeEqual, pdu::PduCount}, + Result, at, err, + matrix::{ + Event, + event::RelationTypeEqual, + pdu::{PduCount, ShortEventId}, + }, utils::{IterStream, ReadyExt, result::FlatOk, stream::WidebandExt}, }; use conduwuit_service::Services; @@ -20,6 +24,40 @@ use ruma::{ use crate::Ruma; +/// Parse a pagination token, trying ShortEventId first, then falling back to +/// PduCount +async fn parse_pagination_token( + _services: &Services, + _room_id: &RoomId, + token: Option<&str>, + default: PduCount, +) -> Result { + let Some(token) = token else { + return Ok(default); + }; + + // Try parsing as ShortEventId first + if let Ok(shorteventid) = token.parse::() { + // ShortEventId maps directly to a PduCount in our database + // The shorteventid IS the count value, just need to wrap it + Ok(PduCount::Normal(shorteventid)) + } else if let Ok(count) = token.parse::() { + // Fallback to PduCount for backwards compatibility + Ok(PduCount::Normal(count)) + } else if let Ok(count) = token.parse::() { + // Also handle negative counts for backfilled events + Ok(PduCount::from_signed(count)) + } else { + Err(err!(Request(InvalidParam("Invalid pagination token")))) + } +} + +/// Convert a PduCount to a token string (using the underlying ShortEventId) +fn count_to_token(count: PduCount) -> String { + // The PduCount's unsigned value IS the ShortEventId + count.into_unsigned().to_string() +} + /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}` pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( State(services): State, @@ -109,15 +147,17 @@ async fn paginate_relations_with_filter( recurse: bool, dir: Direction, ) -> Result { - let start: PduCount = from - .map(str::parse) - .transpose()? - .unwrap_or_else(|| match dir { - | Direction::Forward => PduCount::min(), - | Direction::Backward => PduCount::max(), - }); + let start: PduCount = parse_pagination_token(services, room_id, from, match dir { + | Direction::Forward => PduCount::min(), + | Direction::Backward => PduCount::max(), + }) + .await?; - let to: Option = to.map(str::parse).flat_ok(); + let to: Option = if let Some(to_str) = to { + Some(parse_pagination_token(services, room_id, Some(to_str), PduCount::min()).await?) + } else { + None + }; // Use limit or else 30, with maximum 100 let limit: usize = limit @@ -129,6 +169,11 @@ async fn paginate_relations_with_filter( // Spec (v1.10) recommends depth of at least 3 let depth: u8 = if recurse { 3 } else { 1 }; + // Check if this is a thread request + let is_thread = filter_rel_type + .as_ref() + .is_some_and(|rel| *rel == RelationType::Thread); + let events: Vec<_> = services .rooms .pdu_metadata @@ -152,23 +197,65 @@ async fn paginate_relations_with_filter( .collect() .await; - let next_batch = match dir { - | Direction::Forward => events.last(), - | Direction::Backward => events.first(), + // For threads, check if we should include the root event + let mut root_event = None; + if is_thread && dir == Direction::Backward { + // Check if we've reached the beginning of the thread + // (fewer events than requested means we've exhausted the thread) + if events.len() < limit { + // Try to get the thread root event + if let Ok(root_pdu) = services.rooms.timeline.get_pdu(target).await { + // Check visibility + if services + .rooms + .state_accessor + .user_can_see_event(sender_user, room_id, target) + .await + { + // Store the root event to add to the response + root_event = Some(root_pdu); + } + } + } } - .map(at!(0)) - .as_ref() - .map(ToString::to_string); + + // Determine if there are more events to fetch + let has_more = if root_event.is_some() { + false // We've included the root, no more events + } else { + // Check if we got a full page of results (might be more) + events.len() >= limit + }; + + let next_batch = if has_more { + match dir { + | Direction::Forward => events.last(), + | Direction::Backward => events.first(), + } + .map(|(count, _)| count_to_token(*count)) + } else { + None + }; + + // Build the response chunk with thread root if needed + let chunk: Vec<_> = if let Some(root) = root_event { + // Add root event at the beginning for backward pagination + std::iter::once(root.into_format()) + .chain(events.into_iter().map(at!(1)).map(Event::into_format)) + .collect() + } else { + events + .into_iter() + .map(at!(1)) + .map(Event::into_format) + .collect() + }; Ok(get_relating_events::v1::Response { next_batch, prev_batch: from.map(Into::into), recursion_depth: recurse.then_some(depth.into()), - chunk: events - .into_iter() - .map(at!(1)) - .map(Event::into_format) - .collect(), + chunk, }) } diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index c1376cb0..f9cc80a0 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -61,9 +61,10 @@ impl Data { from: PduCount, dir: Direction, ) -> impl Stream + Send + '_ { + let from_unsigned = from.into_unsigned(); let mut current = ArrayVec::::new(); current.extend(target.to_be_bytes()); - current.extend(from.saturating_inc(dir).into_unsigned().to_be_bytes()); + current.extend(from_unsigned.to_be_bytes()); let current = current.as_slice(); match dir { | Direction::Forward => self.tofrom_relation.raw_keys_from(current).boxed(), @@ -73,6 +74,17 @@ impl Data { .ready_take_while(move |key| key.starts_with(&target.to_be_bytes())) .map(|to_from| u64_from_u8(&to_from[8..16])) .map(PduCount::from_unsigned) + .ready_filter(move |count| { + if from == PduCount::min() || from == PduCount::max() { + true + } else { + let count_unsigned = count.into_unsigned(); + match dir { + | Direction::Forward => count_unsigned > from_unsigned, + | Direction::Backward => count_unsigned < from_unsigned, + } + } + }) .wide_filter_map(move |shorteventid| async move { let pdu_id: RawPduId = PduId { shortroomid, shorteventid }.into(); From 583cb924f1510960702f626c7397cb9e87e1722a Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Mon, 11 Aug 2025 06:24:29 +0100 Subject: [PATCH 1999/2091] refactor: address code review feedback for auth and pagination improvements - Extract duplicated thread/message pagination functions to shared utils module - Refactor pagination token parsing to use Option combinators instead of defaults - Split access token generation from assignment for clearer error handling - Add appservice token collision detection at startup and registration - Allow appservice re-registration with same token (for config updates) - Simplify thread relation chunk building using iterator chaining - Fix saturating_inc edge case in relation queries with explicit filtering - Add concise comments explaining non-obvious behaviour choices --- src/api/client/message.rs | 55 ++++------------- src/api/client/mod.rs | 1 + src/api/client/relations.rs | 78 +++++------------------- src/api/client/session.rs | 4 +- src/api/client/utils.rs | 28 +++++++++ src/api/router/auth.rs | 1 + src/service/appservice/mod.rs | 83 +++++++++++++++++--------- src/service/rooms/pdu_metadata/data.rs | 2 + src/service/users/mod.rs | 48 ++++++++++----- 9 files changed, 149 insertions(+), 151 deletions(-) create mode 100644 src/api/client/utils.rs diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 95a135e1..4d489c2f 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -1,9 +1,9 @@ use axum::extract::State; use conduwuit::{ - Err, Result, at, err, + Err, Result, at, matrix::{ event::{Event, Matches}, - pdu::{PduCount, ShortEventId}, + pdu::PduCount, }, ref_at, utils::{ @@ -35,6 +35,7 @@ use ruma::{ }; use tracing::warn; +use super::utils::{count_to_token, parse_pagination_token as parse_token}; use crate::Ruma; /// list of safe and common non-state events to ignore if the user is ignored @@ -61,39 +62,6 @@ const IGNORED_MESSAGE_TYPES: &[TimelineEventType] = &[ const LIMIT_MAX: usize = 100; const LIMIT_DEFAULT: usize = 10; -/// Parse a pagination token, trying ShortEventId first, then falling back to -/// PduCount -async fn parse_pagination_token( - _services: &Services, - _room_id: &RoomId, - token: Option<&str>, - default: PduCount, -) -> Result { - let Some(token) = token else { - return Ok(default); - }; - - // Try parsing as ShortEventId first - if let Ok(shorteventid) = token.parse::() { - // ShortEventId maps directly to a PduCount in our database - Ok(PduCount::Normal(shorteventid)) - } else if let Ok(count) = token.parse::() { - // Fallback to PduCount for backwards compatibility - Ok(PduCount::Normal(count)) - } else if let Ok(count) = token.parse::() { - // Also handle negative counts for backfilled events - Ok(PduCount::from_signed(count)) - } else { - Err(err!(Request(InvalidParam("Invalid pagination token")))) - } -} - -/// Convert a PduCount to a token string (using the underlying ShortEventId) -fn count_to_token(count: PduCount) -> String { - // The PduCount's unsigned value IS the ShortEventId - count.into_unsigned().to_string() -} - /// # `GET /_matrix/client/r0/rooms/{roomId}/messages` /// /// Allows paginating through room history. @@ -114,18 +82,17 @@ pub(crate) async fn get_message_events_route( return Err!(Request(Forbidden("Room does not exist to this server"))); } - let from: PduCount = - parse_pagination_token(&services, room_id, body.from.as_deref(), match body.dir { + let from: PduCount = body + .from + .as_deref() + .map(parse_token) + .transpose()? + .unwrap_or_else(|| match body.dir { | Direction::Forward => PduCount::min(), | Direction::Backward => PduCount::max(), - }) - .await?; + }); - let to: Option = if let Some(to_str) = body.to.as_deref() { - Some(parse_pagination_token(&services, room_id, Some(to_str), PduCount::min()).await?) - } else { - None - }; + let to: Option = body.to.as_deref().map(parse_token).transpose()?; let limit: usize = body .limit diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index be54e65f..e4be20b7 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -36,6 +36,7 @@ pub(super) mod typing; pub(super) mod unstable; pub(super) mod unversioned; pub(super) mod user_directory; +pub(super) mod utils; pub(super) mod voip; pub(super) mod well_known; diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index 48bcde20..f6d8fe9e 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -1,11 +1,7 @@ use axum::extract::State; use conduwuit::{ - Result, at, err, - matrix::{ - Event, - event::RelationTypeEqual, - pdu::{PduCount, ShortEventId}, - }, + Result, at, + matrix::{Event, event::RelationTypeEqual, pdu::PduCount}, utils::{IterStream, ReadyExt, result::FlatOk, stream::WidebandExt}, }; use conduwuit_service::Services; @@ -22,42 +18,9 @@ use ruma::{ events::{TimelineEventType, relation::RelationType}, }; +use super::utils::{count_to_token, parse_pagination_token as parse_token}; use crate::Ruma; -/// Parse a pagination token, trying ShortEventId first, then falling back to -/// PduCount -async fn parse_pagination_token( - _services: &Services, - _room_id: &RoomId, - token: Option<&str>, - default: PduCount, -) -> Result { - let Some(token) = token else { - return Ok(default); - }; - - // Try parsing as ShortEventId first - if let Ok(shorteventid) = token.parse::() { - // ShortEventId maps directly to a PduCount in our database - // The shorteventid IS the count value, just need to wrap it - Ok(PduCount::Normal(shorteventid)) - } else if let Ok(count) = token.parse::() { - // Fallback to PduCount for backwards compatibility - Ok(PduCount::Normal(count)) - } else if let Ok(count) = token.parse::() { - // Also handle negative counts for backfilled events - Ok(PduCount::from_signed(count)) - } else { - Err(err!(Request(InvalidParam("Invalid pagination token")))) - } -} - -/// Convert a PduCount to a token string (using the underlying ShortEventId) -fn count_to_token(count: PduCount) -> String { - // The PduCount's unsigned value IS the ShortEventId - count.into_unsigned().to_string() -} - /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}` pub(crate) async fn get_relating_events_with_rel_type_and_event_type_route( State(services): State, @@ -147,17 +110,15 @@ async fn paginate_relations_with_filter( recurse: bool, dir: Direction, ) -> Result { - let start: PduCount = parse_pagination_token(services, room_id, from, match dir { - | Direction::Forward => PduCount::min(), - | Direction::Backward => PduCount::max(), - }) - .await?; + let start: PduCount = from + .map(parse_token) + .transpose()? + .unwrap_or_else(|| match dir { + | Direction::Forward => PduCount::min(), + | Direction::Backward => PduCount::max(), + }); - let to: Option = if let Some(to_str) = to { - Some(parse_pagination_token(services, room_id, Some(to_str), PduCount::min()).await?) - } else { - None - }; + let to: Option = to.map(parse_token).transpose()?; // Use limit or else 30, with maximum 100 let limit: usize = limit @@ -238,18 +199,11 @@ async fn paginate_relations_with_filter( }; // Build the response chunk with thread root if needed - let chunk: Vec<_> = if let Some(root) = root_event { - // Add root event at the beginning for backward pagination - std::iter::once(root.into_format()) - .chain(events.into_iter().map(at!(1)).map(Event::into_format)) - .collect() - } else { - events - .into_iter() - .map(at!(1)) - .map(Event::into_format) - .collect() - }; + let chunk: Vec<_> = root_event + .into_iter() + .map(Event::into_format) + .chain(events.into_iter().map(at!(1)).map(Event::into_format)) + .collect(); Ok(get_relating_events::v1::Response { next_batch, diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 992073c6..fe07e41d 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -198,8 +198,8 @@ pub(crate) async fn login_route( .clone() .unwrap_or_else(|| utils::random_string(DEVICE_ID_LENGTH).into()); - // Generate a new token for the device - let token = utils::random_string(TOKEN_LENGTH); + // Generate a new token for the device (ensuring no collisions) + let token = services.users.generate_unique_token().await; // Determine if device_id was provided and exists in the db for this user let device_exists = if body.device_id.is_some() { diff --git a/src/api/client/utils.rs b/src/api/client/utils.rs new file mode 100644 index 00000000..cc941b95 --- /dev/null +++ b/src/api/client/utils.rs @@ -0,0 +1,28 @@ +use conduwuit::{ + Result, err, + matrix::pdu::{PduCount, ShortEventId}, +}; + +/// Parse a pagination token, trying ShortEventId first, then falling back to +/// PduCount +pub(crate) fn parse_pagination_token(token: &str) -> Result { + // Try parsing as ShortEventId first + if let Ok(shorteventid) = token.parse::() { + // ShortEventId maps directly to a PduCount in our database + Ok(PduCount::Normal(shorteventid)) + } else if let Ok(count) = token.parse::() { + // Fallback to PduCount for backwards compatibility + Ok(PduCount::Normal(count)) + } else if let Ok(count) = token.parse::() { + // Also handle negative counts for backfilled events + Ok(PduCount::from_signed(count)) + } else { + Err(err!(Request(InvalidParam("Invalid pagination token")))) + } +} + +/// Convert a PduCount to a token string (using the underlying ShortEventId) +pub(crate) fn count_to_token(count: PduCount) -> String { + // The PduCount's unsigned value IS the ShortEventId + count.into_unsigned().to_string() +} diff --git a/src/api/router/auth.rs b/src/api/router/auth.rs index 5088e699..44afc3ef 100644 --- a/src/api/router/auth.rs +++ b/src/api/router/auth.rs @@ -355,6 +355,7 @@ async fn find_token(services: &Services, token: Option<&str>) -> Result { .map_ok(Token::Appservice); pin_mut!(user_token, appservice_token); + // Returns Ok if either token type succeeds, Err only if both fail match select_ok([Left(user_token), Right(appservice_token)]).await { | Err(e) if !e.is_not_found() => Err(e), | Ok((token, _)) => Ok(token), diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index ad9c4a3f..ebd798f6 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -4,7 +4,7 @@ mod registration_info; use std::{collections::BTreeMap, iter::IntoIterator, sync::Arc}; use async_trait::async_trait; -use conduwuit::{Result, err, utils::stream::IterStream}; +use conduwuit::{Err, Result, err, utils::stream::IterStream}; use database::Map; use futures::{Future, FutureExt, Stream, TryStreamExt}; use ruma::{RoomAliasId, RoomId, UserId, api::appservice::Registration}; @@ -48,36 +48,50 @@ impl crate::Service for Service { } async fn worker(self: Arc) -> Result { - self.iter_db_ids() - .try_for_each(async |appservice| { - let (id, registration) = appservice; + // First, collect all appservices to check for token conflicts + let appservices: Vec<(String, Registration)> = self.iter_db_ids().try_collect().await?; - // During startup, resolve any token collisions in favour of appservices - // by logging out conflicting user devices - if let Ok((user_id, device_id)) = self - .services - .users - .find_from_token(®istration.as_token) - .await - { - conduwuit::warn!( - "Token collision detected during startup: Appservice '{}' token was \ - also used by user '{}' device '{}'. Logging out the user device to \ - resolve conflict.", - id, - user_id.localpart(), - device_id - ); - - self.services - .users - .remove_device(&user_id, &device_id) - .await; + // Check for appservice-to-appservice token conflicts + for i in 0..appservices.len() { + for j in i.saturating_add(1)..appservices.len() { + if appservices[i].1.as_token == appservices[j].1.as_token { + return Err!(Database(error!( + "Token collision detected: Appservices '{}' and '{}' have the same token", + appservices[i].0, appservices[j].0 + ))); } + } + } - self.start_appservice(id, registration).await - }) - .await + // Process each appservice + for (id, registration) in appservices { + // During startup, resolve any token collisions in favour of appservices + // by logging out conflicting user devices + if let Ok((user_id, device_id)) = self + .services + .users + .find_from_token(®istration.as_token) + .await + { + conduwuit::warn!( + "Token collision detected during startup: Appservice '{}' token was also \ + used by user '{}' device '{}'. Logging out the user device to resolve \ + conflict.", + id, + user_id.localpart(), + device_id + ); + + self.services + .users + .remove_device(&user_id, &device_id) + .await; + } + + self.start_appservice(id, registration).await?; + } + + Ok(()) } fn name(&self) -> &str { crate::service::make_name(std::module_path!()) } @@ -125,6 +139,18 @@ impl Service { ) -> Result { //TODO: Check for collisions between exclusive appservice namespaces + // Check for token collision with other appservices (allow re-registration of + // same appservice) + if let Ok(existing) = self.find_from_token(®istration.as_token).await { + if existing.registration.id != registration.id { + return Err(err!(Request(InvalidParam( + "Cannot register appservice: Token is already used by appservice '{}'. \ + Please generate a different token.", + existing.registration.id + )))); + } + } + // Prevent token collision with existing user tokens if self .services @@ -182,6 +208,7 @@ impl Service { .map(|info| info.registration) } + /// Returns Result to match users::find_from_token for select_ok usage pub async fn find_from_token(&self, token: &str) -> Result { self.read() .await diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index f9cc80a0..a746b4cc 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -61,6 +61,8 @@ impl Data { from: PduCount, dir: Direction, ) -> impl Stream + Send + '_ { + // Query from exact position then filter excludes it (saturating_inc could skip + // events at min/max boundaries) let from_unsigned = from.into_unsigned(); let mut current = ArrayVec::::new(); current.extend(target.to_be_bytes()); diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 0aacc0e1..eb54660e 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -393,6 +393,31 @@ impl Service { self.db.userdeviceid_token.qry(&key).await.deserialized() } + /// Generate a unique access token that doesn't collide with existing tokens + pub async fn generate_unique_token(&self) -> String { + loop { + let token = utils::random_string(32); + + // Check for collision with appservice tokens + if self + .services + .appservice + .find_from_token(&token) + .await + .is_ok() + { + continue; + } + + // Check for collision with user tokens + if self.db.token_userdeviceid.get(&token).await.is_ok() { + continue; + } + + return token; + } + } + /// Replaces the access token of one device. pub async fn set_token( &self, @@ -409,25 +434,18 @@ impl Service { ))); } - // Prevent token collisions with appservice tokens - let final_token = if self + // Check for token collision with appservices + if self .services .appservice .find_from_token(token) .await .is_ok() { - let new_token = utils::random_string(32); - conduwuit::debug_warn!( - "Token collision prevented: Generated new token for user '{}' device '{}' \ - (original token conflicted with an appservice)", - user_id.localpart(), - device_id - ); - new_token - } else { - token.to_owned() - }; + return Err!(Request(InvalidParam( + "Token conflicts with an existing appservice token" + ))); + } // Remove old token if let Ok(old_token) = self.db.userdeviceid_token.qry(&key).await { @@ -436,8 +454,8 @@ impl Service { } // Assign token to user device combination - self.db.userdeviceid_token.put_raw(key, &final_token); - self.db.token_userdeviceid.raw_put(&final_token, key); + self.db.userdeviceid_token.put_raw(key, token); + self.db.token_userdeviceid.raw_put(token, key); Ok(()) } From 54acd07555abeff1890a02d6ece66b6c97abdc02 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 16 Aug 2025 16:22:24 +0100 Subject: [PATCH 2000/2091] fix: Drop fake room v2 support --- src/core/info/room_version.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/info/room_version.rs b/src/core/info/room_version.rs index 51d5d3c6..54ed8fdc 100644 --- a/src/core/info/room_version.rs +++ b/src/core/info/room_version.rs @@ -18,7 +18,7 @@ pub const STABLE_ROOM_VERSIONS: &[RoomVersionId] = &[ /// Experimental, partially supported room versions pub const UNSTABLE_ROOM_VERSIONS: &[RoomVersionId] = - &[RoomVersionId::V2, RoomVersionId::V3, RoomVersionId::V4, RoomVersionId::V5]; + &[RoomVersionId::V3, RoomVersionId::V4, RoomVersionId::V5]; type RoomVersion = (RoomVersionId, RoomVersionStability); From 2a183cc5a4a271514d5fe19ab2eca0fdde8495f8 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 17 Aug 2025 13:44:32 +0100 Subject: [PATCH 2001/2091] fix(build): Remove hardened_malloc from full feature set The hardened_malloc feature conflicts with jemalloc, preventing successful builds with the --features full flag. Commenting out hardened_malloc allows the full profile to build correctly while maintaining all other features. --- src/main/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 0d7dd844..8aaa3cc6 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -63,7 +63,7 @@ standard = [ ] full = [ "standard", - "hardened_malloc", + # "hardened_malloc", # Conflicts with jemalloc "jemalloc_prof", "perf_measurements", "tokio_console" From f54e59a068250e871168f6fe16b2eb1a8a458e93 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 17 Aug 2025 14:17:18 +0100 Subject: [PATCH 2002/2091] ci: Add Renovate for automated dependency management Configures Renovate bot to create PRs for outdated dependencies. Runs daily at 5am UTC with manual trigger via workflow_dispatch. Configuration: - Ignores custom forks (jemalloc, telemetry packages) - Groups: GHA minor/patch, Rust toolchain, lockfile, Rust patches - Limits: 3 concurrent PRs, 2 PRs per hour - Supports: Cargo, GitHub Actions, Nix --- .forgejo/workflows/renovate.yml | 64 +++++++++++++++++++++++++++++++++ renovate.json | 44 +++++++++++++++++++++-- 2 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 .forgejo/workflows/renovate.yml diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml new file mode 100644 index 00000000..802f5aab --- /dev/null +++ b/.forgejo/workflows/renovate.yml @@ -0,0 +1,64 @@ +name: Maintenance / Renovate + +on: + schedule: + # Run at 5am UTC daily to avoid late-night dev + - cron: '0 5 * * *' + + workflow_dispatch: + inputs: + dryRun: + description: 'Dry run mode' + required: false + default: 'false' + type: choice + options: + - 'true' + - 'false' + logLevel: + description: 'Log level' + required: false + default: 'info' + type: choice + options: + - 'debug' + - 'info' + - 'warn' + - 'error' + + push: + branches: + - main + paths: + # Re-run when config changes + - '.forgejo/workflows/renovate.yml' + - 'renovate.json' + +jobs: + renovate: + name: Renovate + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run Renovate + uses: renovatebot/github-action@v40.1.0 + with: + token: ${{ secrets.RENOVATE_TOKEN }} + configurationFile: renovate.json + env: + # Platform configuration - Forgejo uses Gitea-compatible API + RENOVATE_PLATFORM: gitea + RENOVATE_ENDPOINT: ${{ github.server_url }}/api/v1 + RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }} + + # Target repository + RENOVATE_REPOSITORIES: '["${{ github.repository }}"]' + + # Runtime behaviour + RENOVATE_DRY_RUN: ${{ inputs.dryRun || 'false' }} + LOG_LEVEL: ${{ inputs.logLevel || 'info' }} + + # Git author for commits - configured via repository variables + RENOVATE_GIT_AUTHOR: '${{ vars.RENOVATE_AUTHOR }}' diff --git a/renovate.json b/renovate.json index eecf8532..3122d0bc 100644 --- a/renovate.json +++ b/renovate.json @@ -17,10 +17,48 @@ "github_actions" ], "ignoreDeps": [ - "tikv-jemllocator", + "tikv-jemallocator", "tikv-jemalloc-sys", "tikv-jemalloc-ctl", - "opentelemetry-rust", + "opentelemetry", + "opentelemetry_sdk", + "opentelemetry-jaeger", "tracing-opentelemetry" - ] + ], + "github-actions": { + "enabled": true, + "fileMatch": [ + "(^|/)\\.forgejo/workflows/[^/]+\\.ya?ml$", + "(^|/)\\.forgejo/actions/[^/]+/action\\.ya?ml$", + "(^|/)\\.github/workflows/[^/]+\\.ya?ml$", + "(^|/)\\.github/actions/[^/]+/action\\.ya?ml$" + ] + }, + "packageRules": [ + { + "description": "Batch minor and patch GitHub Actions updates", + "matchManagers": ["github-actions"], + "matchUpdateTypes": ["minor", "patch"], + "groupName": "github-actions-non-major" + }, + { + "description": "Group Rust toolchain updates into a single PR", + "matchManagers": ["regex"], + "matchPackageNames": ["rust", "rustc", "cargo"], + "groupName": "rust-toolchain" + }, + { + "description": "Group lockfile updates into a single PR", + "matchUpdateTypes": ["lockFileMaintenance"], + "groupName": "lockfile-maintenance" + }, + { + "description": "Batch patch-level Rust dependency updates", + "matchManagers": ["cargo"], + "matchUpdateTypes": ["patch"], + "groupName": "rust-patch-updates" + } + ], + "prConcurrentLimit": 3, + "prHourlyLimit": 2 } From b447cfff56f0a0b5f580fbfe3a28fc94e2cda09e Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 17 Aug 2025 15:11:38 +0100 Subject: [PATCH 2003/2091] ci: Update prefligit to prek The prefligit project has been renamed to prek due to typosquatting concerns. This updates our CI to use the new name and recommended installation method via uv, which significantly reduces setup time compared to cargo install and includes automatic caching. - Replace outdated static prefligit action with direct prek invocation - Use uv as recommended by upstream: https://github.com/j178/prek - Update check-byte-order-marker to fix-byte-order-marker (deprecated) - Simplify workflow by removing unused ref calculations The same .pre-commit-config.yaml works unchanged. Developers can install locally with 'uvx prek install' or other methods from the repo. --- .forgejo/workflows/prefligit-checks.yml | 36 ++++++++++++++++--------- .pre-commit-config.yaml | 2 +- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/.forgejo/workflows/prefligit-checks.yml b/.forgejo/workflows/prefligit-checks.yml index cc512496..18f573bb 100644 --- a/.forgejo/workflows/prefligit-checks.yml +++ b/.forgejo/workflows/prefligit-checks.yml @@ -1,22 +1,34 @@ -name: Checks / Prefligit +name: Checks / Prek on: push: pull_request: + permissions: contents: read jobs: - prefligit: + fast-checks: + name: Pre-commit & Formatting runs-on: ubuntu-latest - env: - FROM_REF: ${{ github.event.pull_request.base.sha || (!github.event.forced && ( github.event.before != '0000000000000000000000000000000000000000' && github.event.before || github.sha )) || format('{0}~', github.sha) }} - TO_REF: ${{ github.sha }} steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: ./.forgejo/actions/prefligit - with: - extra_args: --all-files --hook-stage manual + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Install uv + uses: https://github.com/astral-sh/setup-uv@v6 + with: + enable-cache: true + ignore-nothing-to-cache: true + cache-dependency-glob: '' + + - name: Run prek + run: | + uvx prek run \ + --all-files \ + --hook-stage manual \ + --show-diff-on-failure \ + --color=always \ + -v diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 68e3a982..da594310 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - - id: check-byte-order-marker + - id: fix-byte-order-marker - id: check-case-conflict - id: check-symlinks - id: destroyed-symlinks From 5d1f141882826dca1b8ebd8cde43a0b0531978b3 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 17 Aug 2025 15:12:05 +0100 Subject: [PATCH 2004/2091] ci: Rename prefligit-checks.yml to prek-checks.yml Rename workflow file to match the updated tool name. --- .forgejo/workflows/{prefligit-checks.yml => prek-checks.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .forgejo/workflows/{prefligit-checks.yml => prek-checks.yml} (100%) diff --git a/.forgejo/workflows/prefligit-checks.yml b/.forgejo/workflows/prek-checks.yml similarity index 100% rename from .forgejo/workflows/prefligit-checks.yml rename to .forgejo/workflows/prek-checks.yml From 9db750e97c77fe3a33ddf744715d48e3d4c7c457 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 17 Aug 2025 15:51:29 +0100 Subject: [PATCH 2005/2091] fix(ci): Add full GitHub URL to renovate action Forgejo's runner doesn't automatically assume actions are on github.com, so we need to specify the full URL. --- .forgejo/workflows/renovate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml index 802f5aab..d2891d91 100644 --- a/.forgejo/workflows/renovate.yml +++ b/.forgejo/workflows/renovate.yml @@ -43,7 +43,7 @@ jobs: uses: actions/checkout@v4 - name: Run Renovate - uses: renovatebot/github-action@v40.1.0 + uses: https://github.com/renovatebot/github-action@v40.1.0 with: token: ${{ secrets.RENOVATE_TOKEN }} configurationFile: renovate.json From 4524a00fc616e68b1c03fa5aa78908458dbe5789 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 17 Aug 2025 16:00:42 +0100 Subject: [PATCH 2006/2091] chore(ci): Remove obsolete prefligit action Now using prek directly via uvx, this custom action is no longer needed. --- .forgejo/actions/prefligit/action.yml | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 .forgejo/actions/prefligit/action.yml diff --git a/.forgejo/actions/prefligit/action.yml b/.forgejo/actions/prefligit/action.yml deleted file mode 100644 index 8cbd4500..00000000 --- a/.forgejo/actions/prefligit/action.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: prefligit -description: | - Runs prefligit, pre-commit reimplemented in Rust. -inputs: - extra_args: - description: options to pass to pre-commit run - required: false - default: '--all-files' - -runs: - using: composite - steps: - - name: Install uv - uses: https://github.com/astral-sh/setup-uv@v6 - with: - enable-cache: true - ignore-nothing-to-cache: true - - name: Install Prefligit - shell: bash - run: | - curl --proto '=https' --tlsv1.2 -LsSf https://github.com/j178/prefligit/releases/download/v0.0.10/prefligit-installer.sh | sh - - uses: actions/cache@v3 - with: - path: ~/.cache/prefligit - key: prefligit-0|${{ hashFiles('.pre-commit-config.yaml') }} - - run: prefligit run --show-diff-on-failure --color=always -v ${{ inputs.extra_args }} - shell: bash From 14a4b24fc554a688510c21d5251730a70936dca6 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 17 Aug 2025 17:23:56 +0100 Subject: [PATCH 2007/2091] fix(ci): Configure Renovate for Forgejo platform - Set platform to 'forgejo' with proper API endpoint - Use environment variables for all Renovate configuration - Add git timeout and disable GitHub token warnings - Move PR limit configuration to workflow --- .forgejo/workflows/renovate.yml | 44 ++++++++++++++++----------------- renovate.json | 4 +-- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml index d2891d91..e8522bec 100644 --- a/.forgejo/workflows/renovate.yml +++ b/.forgejo/workflows/renovate.yml @@ -10,21 +10,22 @@ on: dryRun: description: 'Dry run mode' required: false - default: 'false' + default: null type: choice options: - - 'true' - - 'false' + - null + - 'extract' + - 'lookup' + - 'full' logLevel: description: 'Log level' required: false default: 'info' type: choice options: - - 'debug' - 'info' - - 'warn' - - 'error' + - 'warning' + - 'critical' push: branches: @@ -42,23 +43,20 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Run Renovate + - name: Self-hosted Renovate uses: https://github.com/renovatebot/github-action@v40.1.0 - with: - token: ${{ secrets.RENOVATE_TOKEN }} - configurationFile: renovate.json env: - # Platform configuration - Forgejo uses Gitea-compatible API - RENOVATE_PLATFORM: gitea - RENOVATE_ENDPOINT: ${{ github.server_url }}/api/v1 - RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }} - - # Target repository - RENOVATE_REPOSITORIES: '["${{ github.repository }}"]' - - # Runtime behaviour - RENOVATE_DRY_RUN: ${{ inputs.dryRun || 'false' }} LOG_LEVEL: ${{ inputs.logLevel || 'info' }} - - # Git author for commits - configured via repository variables - RENOVATE_GIT_AUTHOR: '${{ vars.RENOVATE_AUTHOR }}' + RENOVATE_AUTODISCOVER: 'false' + RENOVATE_BINARY_SOURCE: 'install' + RENOVATE_DRY_RUN: ${{ inputs.dryRun || 'false' }} + RENOVATE_ENDPOINT: ${{ github.server_url }}/api/v1 + RENOVATE_GIT_TIMEOUT: 60000 + RENOVATE_GIT_URL: 'endpoint' + RENOVATE_GITHUB_TOKEN_WARN: 'false' + RENOVATE_ONBOARDING: 'false' + RENOVATE_PLATFORM: 'forgejo' + RENOVATE_PR_COMMITS_PER_RUN_LIMIT: 3 + RENOVATE_REPOSITORIES: '["${{ github.repository }}"]' + RENOVATE_REQUIRE_CONFIG: 'required' + RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }} diff --git a/renovate.json b/renovate.json index 3122d0bc..b48fc6dc 100644 --- a/renovate.json +++ b/renovate.json @@ -58,7 +58,5 @@ "matchUpdateTypes": ["patch"], "groupName": "rust-patch-updates" } - ], - "prConcurrentLimit": 3, - "prHourlyLimit": 2 + ] } From ecb87ccd1c18126093628b1f424c3ed7f89e8ff9 Mon Sep 17 00:00:00 2001 From: aviac Date: Thu, 21 Aug 2025 13:39:36 +0200 Subject: [PATCH 2008/2091] chore(nix): bump rocksdb version in flake.nix to 10.4.fb This works without any further changes. Multiple people in the matrix room (including myself) have reported that the built executable runs fine with this. Nevertheless, there might be room for improvements (in future commits) --- flake.lock | 12 ++++++------ flake.nix | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/flake.lock b/flake.lock index 51a04c6c..4af82c75 100644 --- a/flake.lock +++ b/flake.lock @@ -516,16 +516,16 @@ "rocksdb": { "flake": false, "locked": { - "lastModified": 1741308171, - "narHash": "sha256-YdBvdQ75UJg5ffwNjxizpviCVwVDJnBkM8ZtGIduMgY=", - "ref": "v9.11.1", - "rev": "3ce04794bcfbbb0d2e6f81ae35fc4acf688b6986", - "revCount": 13177, + "lastModified": 1753385396, + "narHash": "sha256-/Hvy1yTH/0D5aa7bc+/uqFugCQq4InTdwlRw88vA5IY=", + "ref": "10.4.fb", + "rev": "28d4b7276c16ed3e28af1bd96162d6442ce25923", + "revCount": 13318, "type": "git", "url": "https://forgejo.ellis.link/continuwuation/rocksdb" }, "original": { - "ref": "v9.11.1", + "ref": "10.4.fb", "type": "git", "url": "https://forgejo.ellis.link/continuwuation/rocksdb" } diff --git a/flake.nix b/flake.nix index 564cd479..f0dcb6fb 100644 --- a/flake.nix +++ b/flake.nix @@ -17,7 +17,7 @@ nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; rocksdb = { - url = "git+https://forgejo.ellis.link/continuwuation/rocksdb?ref=v9.11.1"; + url = "git+https://forgejo.ellis.link/continuwuation/rocksdb?ref=10.4.fb"; flake = false; }; }; @@ -62,7 +62,7 @@ }).overrideAttrs (old: { src = inputs.rocksdb; - version = "v9.11.1"; + version = "v10.4.fb"; cmakeFlags = pkgs.lib.subtractLists [ # No real reason to have snappy or zlib, no one uses this From 256bed992e50a86b54dc3ff0d24b8d72ce69904d Mon Sep 17 00:00:00 2001 From: aviac Date: Thu, 21 Aug 2025 13:40:11 +0200 Subject: [PATCH 2009/2091] chore(nix): exec 'use flake' with direnv on NixOS systems --- .envrc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.envrc b/.envrc index bad73b75..172993c4 100644 --- a/.envrc +++ b/.envrc @@ -2,6 +2,8 @@ dotenv_if_exists -# use flake ".#${DIRENV_DEVSHELL:-default}" +if [ -f /etc/os-release ] && grep -q '^ID=nixos' /etc/os-release; then + use flake ".#${DIRENV_DEVSHELL:-default}" +fi PATH_add bin From aacaf5a2a031c9e44c2fb9a081a6109a39e5ddbb Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Thu, 21 Aug 2025 21:10:15 +0100 Subject: [PATCH 2010/2091] fix(ci): Downgrade setup-uv action from v6 to v5 The setup-uv@v6 action has deprecated Node 18 support mid-version by using the File API, causing workflow failures. Temporarily downgrading to v5 until we migrate to a better runner image with Node 20+ support. --- .forgejo/workflows/prek-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/prek-checks.yml b/.forgejo/workflows/prek-checks.yml index 18f573bb..ac330ca2 100644 --- a/.forgejo/workflows/prek-checks.yml +++ b/.forgejo/workflows/prek-checks.yml @@ -18,7 +18,7 @@ jobs: persist-credentials: false - name: Install uv - uses: https://github.com/astral-sh/setup-uv@v6 + uses: https://github.com/astral-sh/setup-uv@v5 with: enable-cache: true ignore-nothing-to-cache: true From 427b973b67ec256aaffc8c8c98dd49aef6fa73c5 Mon Sep 17 00:00:00 2001 From: aviac Date: Thu, 21 Aug 2025 13:51:02 +0200 Subject: [PATCH 2011/2091] chore(rust): bump version 1.87 -> 1.89 - bump version in rust-toolchain.toml - update sha in flake.nix --- flake.nix | 2 +- rust-toolchain.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index f0dcb6fb..25629621 100644 --- a/flake.nix +++ b/flake.nix @@ -36,7 +36,7 @@ file = ./rust-toolchain.toml; # See also `rust-toolchain.toml` - sha256 = "sha256-KUm16pHj+cRedf8vxs/Hd2YWxpOrWZ7UOrwhILdSJBU="; + sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE="; }; mkScope = diff --git a/rust-toolchain.toml b/rust-toolchain.toml index bdb608aa..c44e95ef 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -9,8 +9,8 @@ # If you're having trouble making the relevant changes, bug a maintainer. [toolchain] -channel = "1.87.0" profile = "minimal" +channel = "1.89.0" components = [ # For rust-analyzer "rust-src", From ca3ee9224b19f7c2181a6db41c0de17b43317dc1 Mon Sep 17 00:00:00 2001 From: aviac Date: Thu, 21 Aug 2025 17:35:46 +0200 Subject: [PATCH 2012/2091] chore(rust): drop rustfmt from rust-toolchain.toml This just installs regular rustfmt, which is not needed in this project. One could say "It doesn't hurt", but in the NixOS dev shell it actually does since it will shadow nightly rustfmt and we don't have the `cargo +nightly fmt` synatx on NixOS that is available on other Distros. Also "It doesn't hurt" to delete it for non NixOS users. --- rust-toolchain.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rust-toolchain.toml b/rust-toolchain.toml index c44e95ef..63e9d9ce 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -16,6 +16,9 @@ components = [ "rust-src", "rust-analyzer", # For CI and editors - "rustfmt", "clippy", + # you have to install rustfmt nightly yourself (if you're not on NixOS) + # + # The rust-toolchain.toml file doesn't provide any syntax for specifying components from different toolchains + # "rustfmt" ] From 6d1f12b22de7eb8d7af7a642cdf23a14c13d51d2 Mon Sep 17 00:00:00 2001 From: aviac Date: Thu, 21 Aug 2025 17:41:14 +0200 Subject: [PATCH 2013/2091] chore(nix): make rustfmt-nightly available to default dev shell I verified this by running `rustfmt --version` on my system. Note that I don't have a system-wide install of rust and only rely on dev shells, so this can't possibly come from somewhere else. ``` $ rustfmt --version rustfmt 1.8.0-nightly (6677875279 2025-07-02) ``` --- flake.nix | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/flake.nix b/flake.nix index 25629621..d6beb84e 100644 --- a/flake.nix +++ b/flake.nix @@ -31,13 +31,17 @@ inherit system; }; + fnx = inputs.fenix.packages.${system}; # The Rust toolchain to use - toolchain = inputs.fenix.packages.${system}.fromToolchainFile { - file = ./rust-toolchain.toml; + toolchain = fnx.combine [ + (fnx.fromToolchainFile { + file = ./rust-toolchain.toml; - # See also `rust-toolchain.toml` - sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE="; - }; + # See also `rust-toolchain.toml` + sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE="; + }) + fnx.complete.rustfmt + ]; mkScope = pkgs: From d191494f18fa60af15756fc01420b4823c6247bd Mon Sep 17 00:00:00 2001 From: aviac Date: Thu, 21 Aug 2025 17:50:08 +0200 Subject: [PATCH 2014/2091] chore(nix): update `fenix` input This is required, since now we're installing `rustfmt` from the latest state of the fenix repo. This wasn't recent enough for the latest rust version. The input was locked at (2025-07-02). Now it's up to date. --- flake.lock | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index 4af82c75..4c2bf9fb 100644 --- a/flake.lock +++ b/flake.lock @@ -153,11 +153,11 @@ "rust-analyzer-src": "rust-analyzer-src" }, "locked": { - "lastModified": 1751525020, - "narHash": "sha256-oDO6lCYS5Bf4jUITChj9XV7k3TP38DE0Ckz5n5ORCME=", + "lastModified": 1755585599, + "narHash": "sha256-tl/0cnsqB/Yt7DbaGMel2RLa7QG5elA8lkaOXli6VdY=", "owner": "nix-community", "repo": "fenix", - "rev": "a1a5f92f47787e7df9f30e5e5ac13e679215aa1e", + "rev": "6ed03ef4c8ec36d193c18e06b9ecddde78fb7e42", "type": "github" }, "original": { @@ -546,11 +546,11 @@ "rust-analyzer-src": { "flake": false, "locked": { - "lastModified": 1751433876, - "narHash": "sha256-IsdwOcvLLDDlkFNwhdD5BZy20okIQL01+UQ7Kxbqh8s=", + "lastModified": 1755504847, + "narHash": "sha256-VX0B9hwhJypCGqncVVLC+SmeMVd/GAYbJZ0MiiUn2Pk=", "owner": "rust-lang", "repo": "rust-analyzer", - "rev": "11d45c881389dae90b0da5a94cde52c79d0fc7ef", + "rev": "a905e3b21b144d77e1b304e49f3264f6f8d4db75", "type": "github" }, "original": { From 8b35de6a430fae8be5d8291d8d73c9420aafff6a Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Fri, 22 Aug 2025 00:51:54 +0100 Subject: [PATCH 2015/2091] chore: Fix clippy lints with minimal diff --- Cargo.toml | 5 ++++- src/api/client/sync/v4.rs | 1 + src/core/config/mod.rs | 1 + src/core/debug.rs | 2 +- src/router/serve/unix.rs | 2 +- 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c656e183..04ff4bb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -867,7 +867,7 @@ unused-qualifications = "warn" #unused-results = "warn" # TODO ## some sadness -elided_named_lifetimes = "allow" # TODO! +mismatched_lifetime_syntaxes = "allow" # TODO! let_underscore_drop = "allow" missing_docs = "allow" # cfgs cannot be limited to expected cfgs or their de facto non-transitive/opt-in use-case e.g. @@ -1006,3 +1006,6 @@ literal_string_with_formatting_args = { level = "allow", priority = 1 } needless_raw_string_hashes = "allow" + +# TODO: Enable this lint & fix all instances +collapsible_if = "allow" diff --git a/src/api/client/sync/v4.rs b/src/api/client/sync/v4.rs index 14cd50d8..a16e4526 100644 --- a/src/api/client/sync/v4.rs +++ b/src/api/client/sync/v4.rs @@ -45,6 +45,7 @@ use crate::{ type TodoRooms = BTreeMap, usize, u64)>; const SINGLE_CONNECTION_SYNC: &str = "single_connection_sync"; +#[allow(clippy::cognitive_complexity)] /// POST `/_matrix/client/unstable/org.matrix.msc3575/sync` /// /// Sliding Sync endpoint (future endpoint: `/_matrix/client/v4/sync`) diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index aa021be7..0708196d 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1,3 +1,4 @@ +#![allow(clippy::doc_link_with_quotes)] pub mod check; pub mod manager; pub mod proxy; diff --git a/src/core/debug.rs b/src/core/debug.rs index 21a5ada4..c728278d 100644 --- a/src/core/debug.rs +++ b/src/core/debug.rs @@ -100,7 +100,7 @@ pub fn trap() { #[must_use] pub fn panic_str(p: &Box) -> &'static str { - p.downcast_ref::<&str>().copied().unwrap_or_default() + (**p).downcast_ref::<&str>().copied().unwrap_or_default() } #[inline(always)] diff --git a/src/router/serve/unix.rs b/src/router/serve/unix.rs index 2af17274..9bb3dd6e 100644 --- a/src/router/serve/unix.rs +++ b/src/router/serve/unix.rs @@ -30,7 +30,7 @@ use tower::{Service, ServiceExt}; type MakeService = IntoMakeServiceWithConnectInfo; -const NULL_ADDR: net::SocketAddr = net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), 0); +const NULL_ADDR: net::SocketAddr = net::SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0); const FINI_POLL_INTERVAL: Duration = Duration::from_millis(750); #[tracing::instrument(skip_all, level = "debug")] From c7adbae03f72250d0fd0ab2ea6ab7911b4369467 Mon Sep 17 00:00:00 2001 From: RatCornu Date: Sat, 9 Aug 2025 15:06:48 +0200 Subject: [PATCH 2016/2091] feat: ldap login --- Cargo.lock | 413 +++++++++++++++++++----- Cargo.toml | 5 + conduwuit-example.toml | 91 ++++++ src/admin/user/commands.rs | 4 +- src/api/Cargo.toml | 3 + src/api/client/account.rs | 5 +- src/api/client/profile.rs | 6 +- src/api/client/session.rs | 211 ++++++++---- src/api/client/unstable.rs | 4 +- src/core/config/mod.rs | 108 +++++++ src/core/error/mod.rs | 2 + src/database/maps.rs | 4 + src/service/Cargo.toml | 5 + src/service/admin/create.rs | 2 +- src/service/emergency/mod.rs | 8 +- src/service/rooms/state_cache/update.rs | 2 +- src/service/users/mod.rs | 203 +++++++++++- 17 files changed, 921 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d7192b6..2b044a1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -126,7 +126,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -183,7 +183,7 @@ dependencies = [ "rustc-hash 2.1.1", "serde", "serde_derive", - "syn", + "syn 2.0.104", ] [[package]] @@ -198,6 +198,45 @@ dependencies = [ "winnow", ] +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "assign" version = "1.1.1" @@ -250,7 +289,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -261,7 +300,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -433,11 +472,11 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls", - "rustls-pemfile", + "rustls 0.23.29", + "rustls-pemfile 2.2.0", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tower-service", ] @@ -452,9 +491,9 @@ dependencies = [ "http", "http-body-util", "pin-project", - "rustls", + "rustls 0.23.29", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tokio-util", "tower-layer", "tower-service", @@ -521,7 +560,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn", + "syn 2.0.104", "which", ] @@ -540,7 +579,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn", + "syn 2.0.104", ] [[package]] @@ -794,7 +833,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -972,7 +1011,7 @@ dependencies = [ "rand 0.8.5", "regex", "reqwest", - "ring", + "ring 0.17.14", "ruma", "sanitize-filename", "serde", @@ -1019,7 +1058,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1044,7 +1083,7 @@ dependencies = [ "hyper-util", "log", "ruma", - "rustls", + "rustls 0.23.29", "sd-notify", "sentry", "sentry-tower", @@ -1074,6 +1113,7 @@ dependencies = [ "image", "ipaddress", "itertools 0.14.0", + "ldap3", "log", "loole", "lru-cache", @@ -1183,6 +1223,16 @@ dependencies = [ "crossterm", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -1256,7 +1306,7 @@ dependencies = [ "proc-macro2", "quote", "strict", - "syn", + "syn 2.0.104", ] [[package]] @@ -1366,7 +1416,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1393,7 +1443,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1434,6 +1484,20 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.4.0" @@ -1461,7 +1525,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1483,7 +1547,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1544,7 +1608,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1564,7 +1628,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1734,6 +1798,7 @@ checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -1781,7 +1846,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -2030,7 +2095,7 @@ dependencies = [ "ipnet", "once_cell", "rand 0.9.2", - "ring", + "ring 0.17.14", "serde", "thiserror 2.0.12", "tinyvec", @@ -2122,7 +2187,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -2216,11 +2281,11 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.29", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tower-service", "webpki-roots 1.0.2", ] @@ -2444,7 +2509,7 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -2613,7 +2678,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn", + "syn 2.0.104", ] [[package]] @@ -2628,6 +2693,43 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lber" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2df7f9fd9f64cf8f59e1a4a0753fe7d575a5b38d3d7ac5758dcee9357d83ef0a" +dependencies = [ + "bytes", + "nom", +] + +[[package]] +name = "ldap3" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "166199a8207874a275144c8a94ff6eed5fcbf5c52303e4d9b4d53a0c7ac76554" +dependencies = [ + "async-trait", + "bytes", + "futures", + "futures-util", + "lazy_static", + "lber", + "log", + "nom", + "percent-encoding", + "ring 0.16.20", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "thiserror 1.0.69", + "tokio", + "tokio-rustls 0.24.1", + "tokio-stream", + "tokio-util", + "url", + "x509-parser", +] + [[package]] name = "lebe" version = "0.5.2" @@ -2866,7 +2968,7 @@ checksum = "a9882ef5c56df184b8ffc107fc6c61e33ee3a654b021961d790a78571bb9d67a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3032,7 +3134,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3094,6 +3196,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -3284,7 +3395,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3358,7 +3469,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3458,7 +3569,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff24dfcda44452b9816fff4cd4227e1bb73ff5a2f1bc1105aa92fb8565ce44d2" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.104", ] [[package]] @@ -3487,7 +3598,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "version_check", "yansi", ] @@ -3508,7 +3619,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" dependencies = [ "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3531,7 +3642,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3597,7 +3708,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.29", "socket2", "thiserror 2.0.12", "tokio", @@ -3615,9 +3726,9 @@ dependencies = [ "getrandom 0.3.3", "lru-slab", "rand 0.9.2", - "ring", + "ring 0.17.14", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.29", "rustls-pki-types", "slab", "thiserror 2.0.12", @@ -3876,16 +3987,16 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.23.29", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.2", "tokio-socks", "tokio-util", "tower 0.5.2", @@ -3909,6 +4020,21 @@ version = "0.8.52" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted 0.7.1", + "web-sys", + "winapi", +] + [[package]] name = "ring" version = "0.17.14" @@ -3919,7 +4045,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.16", "libc", - "untrusted", + "untrusted 0.9.0", "windows-sys 0.52.0", ] @@ -4093,7 +4219,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn", + "syn 2.0.104", "toml", ] @@ -4178,6 +4304,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "0.38.44" @@ -4204,6 +4339,18 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring 0.17.14", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.29" @@ -4213,13 +4360,25 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", - "ring", + "ring 0.17.14", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.4", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -4229,7 +4388,16 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework", + "security-framework 3.2.0", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", ] [[package]] @@ -4251,6 +4419,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring 0.17.14", + "untrusted 0.9.0", +] + [[package]] name = "rustls-webpki" version = "0.103.4" @@ -4258,9 +4436,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" dependencies = [ "aws-lc-rs", - "ring", + "ring 0.17.14", "rustls-pki-types", - "untrusted", + "untrusted 0.9.0", ] [[package]] @@ -4319,6 +4497,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring 0.17.14", + "untrusted 0.9.0", +] + [[package]] name = "sd-notify" version = "0.4.5" @@ -4328,6 +4516,19 @@ dependencies = [ "libc", ] +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.9.1", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + [[package]] name = "security-framework" version = "3.2.0" @@ -4335,7 +4536,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ "bitflags 2.9.1", - "core-foundation", + "core-foundation 0.10.1", "core-foundation-sys", "libc", "security-framework-sys", @@ -4365,7 +4566,7 @@ checksum = "255914a8e53822abd946e2ce8baa41d4cded6b8e938913b7f7b9da5b7ab44335" dependencies = [ "httpdate", "reqwest", - "rustls", + "rustls 0.23.29", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -4509,7 +4710,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -4723,6 +4924,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spki" version = "0.7.3" @@ -4791,6 +4998,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.104" @@ -4811,6 +5029,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -4819,7 +5049,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -4910,7 +5140,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -4921,7 +5151,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -5088,7 +5318,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -5103,13 +5333,23 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls", + "rustls 0.23.29", "tokio", ] @@ -5307,7 +5547,7 @@ source = "git+https://forgejo.ellis.link/continuwuation/tracing?rev=1e64095a8051 dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -5461,12 +5701,24 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unsafe-libyaml" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + [[package]] name = "untrusted" version = "0.9.0" @@ -5482,7 +5734,7 @@ dependencies = [ "base64 0.22.1", "log", "once_cell", - "rustls", + "rustls 0.23.29", "rustls-pki-types", "url", "webpki-roots 0.26.11", @@ -5617,7 +5869,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -5652,7 +5904,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -5832,7 +6084,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -5843,7 +6095,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -6164,6 +6416,23 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "x509-parser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "xml5ever" version = "0.18.1" @@ -6221,8 +6490,8 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.104", + "synstructure 0.13.2", ] [[package]] @@ -6242,7 +6511,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -6262,8 +6531,8 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 2.0.104", + "synstructure 0.13.2", ] [[package]] @@ -6302,7 +6571,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 04ff4bb7..9452066c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -546,6 +546,11 @@ features = ["std"] [workspace.dependencies.maplit] version = "1.0.2" +[workspace.dependencies.ldap3] +version = "0.11.5" +default-features = false +features = ["sync", "tls-rustls"] + # # Patches # diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 541050b1..47a9da19 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1696,6 +1696,10 @@ # #config_reload_signal = true +# This item is undocumented. Please contribute documentation for it. +# +#ldap = false + [global.tls] # Path to a valid TLS certificate file. @@ -1774,3 +1778,90 @@ # is 33.55MB. Setting it to 0 disables blurhashing. # #blurhash_max_raw_size = 33554432 + +[global.ldap] + +# Whether to enable LDAP login. +# +# example: "true" +# +#enable = false + +# URI of the LDAP server. +# +# example: "ldap://ldap.example.com:389" +# +#uri = + +# Root of the searches. +# +# example: "ou=users,dc=example,dc=org" +# +#base_dn = false + +# Bind DN if anonymous search is not enabled. +# +# You can use the variable `{username}` that will be replaced by the +# entered username. In such case, the password used to bind will be the +# one provided for the login and not the one given by +# `bind_password_file`. Beware: automatically granting admin rights will +# not work if you use this direct bind instead of a LDAP search. +# +# example: "cn=ldap-reader,dc=example,dc=org" or +# "cn={username},ou=users,dc=example,dc=org" +# +#bind_dn = false + +# Path to a file on the system that contains the password for the +# `bind_dn`. +# +# The server must be able to access the file, and it must not be empty. +# +#bind_password_file = false + +# Search filter to limit user searches. +# +# You can use the variable `{username}` that will be replaced by the +# entered username for more complex filters. +# +# example: "(&(objectClass=person)(memberOf=matrix))" +# +#filter = "(objectClass=*)" + +# Attribute to use to uniquely identify the user. +# +# example: "uid" or "cn" +# +#uid_attribute = "uid" + +# Attribute containing the mail of the user. +# +# example: "mail" +# +#mail_attribute = "mail" + +# Attribute containing the distinguished name of the user. +# +# example: "givenName" or "sn" +# +#name_attribute = "givenName" + +# Root of the searches for admin users. +# +# Defaults to `base_dn` if empty. +# +# example: "ou=admins,dc=example,dc=org" +# +#admin_base_dn = false + +# The LDAP search filter to find administrative users for conduwuit. +# +# If left blank, administrative state must be configured manually for each +# user. +# +# You can use the variable `{username}` that will be replaced by the +# entered username for more complex filters. +# +# example: "(objectClass=conduwuitAdmin)" or "(uid={username})" +# +#admin_filter = false diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 86206c2b..56864a32 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -68,7 +68,8 @@ pub(super) async fn create_user(&self, username: String, password: Option return Err!("Couldn't reset the password for user {user_id}: {e}"), | Ok(()) => { diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 15ada812..9b4ea460 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -49,6 +49,9 @@ jemalloc_stats = [ "conduwuit-core/jemalloc_stats", "conduwuit-service/jemalloc_stats", ] +ldap = [ + "conduwuit-service/ldap" +] release_max_log_level = [ "conduwuit-core/release_max_log_level", "conduwuit-service/release_max_log_level", diff --git a/src/api/client/account.rs b/src/api/client/account.rs index 0cea7bd9..67268c9f 100644 --- a/src/api/client/account.rs +++ b/src/api/client/account.rs @@ -373,7 +373,7 @@ pub(crate) async fn register_route( let password = if is_guest { None } else { body.password.as_deref() }; // Create user - services.users.create(&user_id, password)?; + services.users.create(&user_id, password, None).await?; // Default to pretty displayname let mut displayname = user_id.localpart().to_owned(); @@ -659,7 +659,8 @@ pub(crate) async fn change_password_route( services .users - .set_password(sender_user, Some(&body.new_password))?; + .set_password(sender_user, Some(&body.new_password)) + .await?; if body.logout_devices { // Logout all devices except the current one diff --git a/src/api/client/profile.rs b/src/api/client/profile.rs index 1882495c..eaa66e70 100644 --- a/src/api/client/profile.rs +++ b/src/api/client/profile.rs @@ -90,7 +90,7 @@ pub(crate) async fn get_displayname_route( .await { if !services.users.exists(&body.user_id).await { - services.users.create(&body.user_id, None)?; + services.users.create(&body.user_id, None, None).await?; } services @@ -189,7 +189,7 @@ pub(crate) async fn get_avatar_url_route( .await { if !services.users.exists(&body.user_id).await { - services.users.create(&body.user_id, None)?; + services.users.create(&body.user_id, None, None).await?; } services @@ -248,7 +248,7 @@ pub(crate) async fn get_profile_route( .await { if !services.users.exists(&body.user_id).await { - services.users.create(&body.user_id, None)?; + services.users.create(&body.user_id, None, None).await?; } services diff --git a/src/api/client/session.rs b/src/api/client/session.rs index fe07e41d..2f066d58 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -6,10 +6,11 @@ use conduwuit::{ Err, Error, Result, debug, err, info, utils, utils::{ReadyExt, hash}, }; -use conduwuit_service::uiaa::SESSION_ID_LENGTH; +use conduwuit_core::debug_error; +use conduwuit_service::{Services, uiaa::SESSION_ID_LENGTH}; use futures::StreamExt; use ruma::{ - UserId, + OwnedUserId, UserId, api::client::{ session::{ get_login_token, @@ -49,6 +50,147 @@ pub(crate) async fn get_login_types_route( ])) } +/// Authenticates the given user by its ID and its password. +/// +/// Returns the user ID if successful, and an error otherwise. +#[tracing::instrument(skip_all, fields(%user_id), name = "password")] +pub(crate) async fn password_login( + services: &Services, + user_id: &UserId, + lowercased_user_id: &UserId, + password: &str, +) -> Result { + // Restrict login to accounts only of type 'password', including untyped + // legacy accounts which are equivalent to 'password'. + if services + .users + .origin(user_id) + .await + .is_ok_and(|origin| origin != "password") + { + return Err!(Request(Forbidden("Account does not permit password login."))); + } + + let (hash, user_id) = match services.users.password_hash(user_id).await { + | Ok(hash) => (hash, user_id), + | Err(_) => services + .users + .password_hash(lowercased_user_id) + .await + .map(|hash| (hash, lowercased_user_id)) + .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?, + }; + + if hash.is_empty() { + return Err!(Request(UserDeactivated("The user has been deactivated"))); + } + + hash::verify_password(password, &hash) + .inspect_err(|e| debug_error!("{e}")) + .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; + + Ok(user_id.to_owned()) +} + +/// Authenticates the given user through the configured LDAP server. +/// +/// Creates the user if the user is found in the LDAP and do not already have an +/// account. +#[tracing::instrument(skip_all, fields(%user_id), name = "ldap")] +pub(super) async fn ldap_login( + services: &Services, + user_id: &UserId, + lowercased_user_id: &UserId, + password: &str, +) -> Result { + let (user_dn, is_ldap_admin) = match services.config.ldap.bind_dn.as_ref() { + | Some(bind_dn) if bind_dn.contains("{username}") => + (bind_dn.replace("{username}", lowercased_user_id.localpart()), false), + | _ => { + debug!("Searching user in LDAP"); + + let dns = services.users.search_ldap(user_id).await?; + if dns.len() >= 2 { + return Err!(Ldap("LDAP search returned two or more results")); + } + + let Some((user_dn, is_admin)) = dns.first() else { + return password_login(services, user_id, lowercased_user_id, password).await; + }; + + (user_dn.clone(), *is_admin) + }, + }; + + let user_id = services + .users + .auth_ldap(&user_dn, password) + .await + .map(|()| lowercased_user_id.to_owned())?; + + // LDAP users are automatically created on first login attempt. This is a very + // common feature that can be seen on many services using a LDAP provider for + // their users (synapse, Nextcloud, Jellyfin, ...). + // + // LDAP users are crated with a dummy password but non empty because an empty + // password is reserved for deactivated accounts. The conduwuit password field + // will never be read to login a LDAP user so it's not an issue. + if !services.users.exists(lowercased_user_id).await { + services + .users + .create(lowercased_user_id, Some("*"), Some("ldap")) + .await?; + } + + let is_conduwuit_admin = services.admin.user_is_admin(lowercased_user_id).await; + + if is_ldap_admin && !is_conduwuit_admin { + services.admin.make_user_admin(lowercased_user_id).await?; + } else if !is_ldap_admin && is_conduwuit_admin { + services.admin.revoke_admin(lowercased_user_id).await?; + } + + Ok(user_id) +} + +pub(crate) async fn handle_login( + services: &Services, + body: &Ruma, + identifier: &Option, + password: &str, + user: &Option, +) -> Result { + debug!("Got password login type"); + let user_id = + if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { + UserId::parse_with_server_name(user_id, &services.config.server_name) + } else if let Some(user) = user { + UserId::parse_with_server_name(user, &services.config.server_name) + } else { + return Err!(Request(Unknown( + debug_warn!(?body.login_info, "Valid identifier or username was not provided (invalid or unsupported login type?)") + ))); + } + .map_err(|e| err!(Request(InvalidUsername(warn!("Username is invalid: {e}")))))?; + + let lowercased_user_id = UserId::parse_with_server_name( + user_id.localpart().to_lowercase(), + &services.config.server_name, + )?; + + if !services.globals.user_is_local(&user_id) + || !services.globals.user_is_local(&lowercased_user_id) + { + return Err!(Request(Unknown("User ID does not belong to this homeserver"))); + } + + if cfg!(feature = "ldap") && services.config.ldap.enable { + ldap_login(services, &user_id, &lowercased_user_id, password).await + } else { + password_login(services, &user_id, &lowercased_user_id, password).await + } +} + /// # `POST /_matrix/client/v3/login` /// /// Authenticates the user and returns an access token it can use in subsequent @@ -80,70 +222,7 @@ pub(crate) async fn login_route( password, user, .. - }) => { - debug!("Got password login type"); - let user_id = - if let Some(uiaa::UserIdentifier::UserIdOrLocalpart(user_id)) = identifier { - UserId::parse_with_server_name(user_id, &services.config.server_name) - } else if let Some(user) = user { - UserId::parse_with_server_name(user, &services.config.server_name) - } else { - return Err!(Request(Unknown( - debug_warn!(?body.login_info, "Valid identifier or username was not provided (invalid or unsupported login type?)") - ))); - } - .map_err(|e| err!(Request(InvalidUsername(warn!("Username is invalid: {e}")))))?; - - let lowercased_user_id = UserId::parse_with_server_name( - user_id.localpart().to_lowercase(), - &services.config.server_name, - )?; - - if !services.globals.user_is_local(&user_id) - || !services.globals.user_is_local(&lowercased_user_id) - { - return Err!(Request(Unknown("User ID does not belong to this homeserver"))); - } - - // first try the username as-is - let hash = services - .users - .password_hash(&user_id) - .await - .inspect_err(|e| debug!("{e}")); - - match hash { - | Ok(hash) => { - if hash.is_empty() { - return Err!(Request(UserDeactivated("The user has been deactivated"))); - } - - hash::verify_password(password, &hash) - .inspect_err(|e| debug!("{e}")) - .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; - - user_id - }, - | Err(_e) => { - let hash_lowercased_user_id = services - .users - .password_hash(&lowercased_user_id) - .await - .inspect_err(|e| debug!("{e}")) - .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; - - if hash_lowercased_user_id.is_empty() { - return Err!(Request(UserDeactivated("The user has been deactivated"))); - } - - hash::verify_password(password, &hash_lowercased_user_id) - .inspect_err(|e| debug!("{e}")) - .map_err(|_| err!(Request(Forbidden("Wrong username or password."))))?; - - lowercased_user_id - }, - } - }, + }) => handle_login(&services, &body, identifier, password, user).await?, | login::v3::LoginInfo::Token(login::v3::Token { token }) => { debug!("Got token login type"); if !services.server.config.login_via_existing_session { diff --git a/src/api/client/unstable.rs b/src/api/client/unstable.rs index 08f70975..f8703ff3 100644 --- a/src/api/client/unstable.rs +++ b/src/api/client/unstable.rs @@ -292,7 +292,7 @@ pub(crate) async fn get_timezone_key_route( .await { if !services.users.exists(&body.user_id).await { - services.users.create(&body.user_id, None)?; + services.users.create(&body.user_id, None, None).await?; } services @@ -352,7 +352,7 @@ pub(crate) async fn get_profile_key_route( .await { if !services.users.exists(&body.user_id).await { - services.users.create(&body.user_id, None)?; + services.users.create(&body.user_id, None, None).await?; } services diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 0708196d..a7fbc7dc 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -1948,6 +1948,10 @@ pub struct Config { pub allow_invalid_tls_certificates_yes_i_know_what_the_fuck_i_am_doing_with_this_and_i_know_this_is_insecure: bool, + // external structure; separate section + #[serde(default)] + pub ldap: LdapConfig, + // external structure; separate section #[serde(default)] pub blurhashing: BlurhashConfig, @@ -2042,6 +2046,102 @@ pub struct BlurhashConfig { pub blurhash_max_raw_size: u64, } +#[derive(Clone, Debug, Default, Deserialize)] +#[config_example_generator(filename = "conduwuit-example.toml", section = "global.ldap")] +pub struct LdapConfig { + /// Whether to enable LDAP login. + /// + /// example: "true" + #[serde(default)] + pub enable: bool, + + /// URI of the LDAP server. + /// + /// example: "ldap://ldap.example.com:389" + pub uri: Option, + + /// Root of the searches. + /// + /// example: "ou=users,dc=example,dc=org" + #[serde(default)] + pub base_dn: String, + + /// Bind DN if anonymous search is not enabled. + /// + /// You can use the variable `{username}` that will be replaced by the + /// entered username. In such case, the password used to bind will be the + /// one provided for the login and not the one given by + /// `bind_password_file`. Beware: automatically granting admin rights will + /// not work if you use this direct bind instead of a LDAP search. + /// + /// example: "cn=ldap-reader,dc=example,dc=org" or + /// "cn={username},ou=users,dc=example,dc=org" + #[serde(default)] + pub bind_dn: Option, + + /// Path to a file on the system that contains the password for the + /// `bind_dn`. + /// + /// The server must be able to access the file, and it must not be empty. + #[serde(default)] + pub bind_password_file: Option, + + /// Search filter to limit user searches. + /// + /// You can use the variable `{username}` that will be replaced by the + /// entered username for more complex filters. + /// + /// example: "(&(objectClass=person)(memberOf=matrix))" + /// + /// default: "(objectClass=*)" + #[serde(default = "default_ldap_search_filter")] + pub filter: String, + + /// Attribute to use to uniquely identify the user. + /// + /// example: "uid" or "cn" + /// + /// default: "uid" + #[serde(default = "default_ldap_uid_attribute")] + pub uid_attribute: String, + + /// Attribute containing the mail of the user. + /// + /// example: "mail" + /// + /// default: "mail" + #[serde(default = "default_ldap_mail_attribute")] + pub mail_attribute: String, + + /// Attribute containing the distinguished name of the user. + /// + /// example: "givenName" or "sn" + /// + /// default: "givenName" + #[serde(default = "default_ldap_name_attribute")] + pub name_attribute: String, + + /// Root of the searches for admin users. + /// + /// Defaults to `base_dn` if empty. + /// + /// example: "ou=admins,dc=example,dc=org" + #[serde(default)] + pub admin_base_dn: String, + + /// The LDAP search filter to find administrative users for conduwuit. + /// + /// If left blank, administrative state must be configured manually for each + /// user. + /// + /// You can use the variable `{username}` that will be replaced by the + /// entered username for more complex filters. + /// + /// example: "(objectClass=conduwuitAdmin)" or "(uid={username})" + #[serde(default)] + pub admin_filter: String, +} + #[derive(Deserialize, Clone, Debug)] #[serde(transparent)] struct ListeningPort { @@ -2431,3 +2531,11 @@ pub(super) fn default_blurhash_x_component() -> u32 { 4 } pub(super) fn default_blurhash_y_component() -> u32 { 3 } // end recommended & blurhashing defaults + +fn default_ldap_search_filter() -> String { "(objectClass=*)".to_owned() } + +fn default_ldap_uid_attribute() -> String { String::from("uid") } + +fn default_ldap_mail_attribute() -> String { String::from("mail") } + +fn default_ldap_name_attribute() -> String { String::from("givenName") } diff --git a/src/core/error/mod.rs b/src/core/error/mod.rs index e46edf09..541af793 100644 --- a/src/core/error/mod.rs +++ b/src/core/error/mod.rs @@ -110,6 +110,8 @@ pub enum Error { InconsistentRoomState(&'static str, ruma::OwnedRoomId), #[error(transparent)] IntoHttp(#[from] ruma::api::error::IntoHttpError), + #[error("{0}")] + Ldap(Cow<'static, str>), #[error(transparent)] Mxc(#[from] ruma::MxcUriError), #[error(transparent)] diff --git a/src/database/maps.rs b/src/database/maps.rs index 214dbf34..da97ef45 100644 --- a/src/database/maps.rs +++ b/src/database/maps.rs @@ -374,6 +374,10 @@ pub(super) static MAPS: &[Descriptor] = &[ name: "userid_masterkeyid", ..descriptor::RANDOM_SMALL }, + Descriptor { + name: "userid_origin", + ..descriptor::RANDOM + }, Descriptor { name: "userid_password", ..descriptor::RANDOM diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index fdebd1d7..6e538f40 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -53,6 +53,9 @@ jemalloc_stats = [ "conduwuit-core/jemalloc_stats", "conduwuit-database/jemalloc_stats", ] +ldap = [ + "dep:ldap3" +] media_thumbnail = [ "dep:image", ] @@ -89,6 +92,8 @@ image.workspace = true image.optional = true ipaddress.workspace = true itertools.workspace = true +ldap3.workspace = true +ldap3.optional = true log.workspace = true loole.workspace = true lru-cache.workspace = true diff --git a/src/service/admin/create.rs b/src/service/admin/create.rs index 157b4d65..755673fe 100644 --- a/src/service/admin/create.rs +++ b/src/service/admin/create.rs @@ -38,7 +38,7 @@ pub async fn create_admin_room(services: &Services) -> Result { // Create a user for the server let server_user = services.globals.server_user.as_ref(); - services.users.create(server_user, None)?; + services.users.create(server_user, None, None).await?; let create_content = { use RoomVersionId::*; diff --git a/src/service/emergency/mod.rs b/src/service/emergency/mod.rs index 3a61f710..f8ecbb3e 100644 --- a/src/service/emergency/mod.rs +++ b/src/service/emergency/mod.rs @@ -41,6 +41,11 @@ impl crate::Service for Service { return Ok(()); } + if self.services.config.ldap.enable { + warn!("emergency password feature not available with LDAP enabled."); + return Ok(()); + } + self.set_emergency_access().await.inspect_err(|e| { error!("Could not set the configured emergency password for the server user: {e}"); }) @@ -57,7 +62,8 @@ impl Service { self.services .users - .set_password(server_user, self.services.config.emergency_password.as_deref())?; + .set_password(server_user, self.services.config.emergency_password.as_deref()) + .await?; let (ruleset, pwd_set) = match self.services.config.emergency_password { | Some(_) => (Ruleset::server_default(server_user), true), diff --git a/src/service/rooms/state_cache/update.rs b/src/service/rooms/state_cache/update.rs index 32c67947..86c1afe7 100644 --- a/src/service/rooms/state_cache/update.rs +++ b/src/service/rooms/state_cache/update.rs @@ -49,7 +49,7 @@ pub async fn update_membership( #[allow(clippy::collapsible_if)] if !self.services.globals.user_is_local(user_id) { if !self.services.users.exists(user_id).await { - self.services.users.create(user_id, None)?; + self.services.users.create(user_id, None, None).await?; } } diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index eb54660e..35202ec7 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,11 +1,19 @@ -use std::{collections::BTreeMap, mem, sync::Arc}; +use std::{ + collections::{BTreeMap, HashMap}, + mem, + sync::Arc, +}; use conduwuit::{ - Err, Error, Result, Server, at, debug_warn, err, trace, + Err, Error, Result, Server, at, debug_warn, err, is_equal_to, + result::LogErr, + trace, utils::{self, ReadyExt, stream::TryIgnore, string::Unquoted}, }; +use conduwuit_core::{debug, error}; use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::{Stream, StreamExt, TryFutureExt}; +use ldap3::{LdapConnAsync, Scope, SearchEntry}; use ruma::{ DeviceId, KeyId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, OneTimeKeyName, OwnedDeviceId, OwnedKeyId, OwnedMxcUri, OwnedUserId, RoomId, UInt, UserId, @@ -63,6 +71,7 @@ struct Data { userid_displayname: Arc, userid_lastonetimekeyupdate: Arc, userid_masterkeyid: Arc, + userid_origin: Arc, userid_password: Arc, userid_suspension: Arc, userid_selfsigningkeyid: Arc, @@ -100,6 +109,7 @@ impl crate::Service for Service { userid_displayname: args.db["userid_displayname"].clone(), userid_lastonetimekeyupdate: args.db["userid_lastonetimekeyupdate"].clone(), userid_masterkeyid: args.db["userid_masterkeyid"].clone(), + userid_origin: args.db["userid_origin"].clone(), userid_password: args.db["userid_password"].clone(), userid_suspension: args.db["userid_suspension"].clone(), userid_selfsigningkeyid: args.db["userid_selfsigningkeyid"].clone(), @@ -136,9 +146,21 @@ impl Service { } /// Create a new user account on this homeserver. + /// + /// User origin is by default "password" (meaning that it will login using + /// its user_id/password). Users with other origins (currently only "ldap" + /// is available) have special login processes. #[inline] - pub fn create(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { - self.set_password(user_id, password) + pub async fn create( + &self, + user_id: &UserId, + password: Option<&str>, + origin: Option<&str>, + ) -> Result<()> { + self.db + .userid_origin + .insert(user_id, origin.unwrap_or("password")); + self.set_password(user_id, password).await } /// Deactivate account @@ -152,7 +174,7 @@ impl Service { // result in an empty string, so the user will not be able to log in again. // Systems like changing the password without logging in should check if the // account is deactivated. - self.set_password(user_id, None)?; + self.set_password(user_id, None).await?; // TODO: Unhook 3PID Ok(()) @@ -253,13 +275,34 @@ impl Service { .ready_filter_map(|(u, p): (&UserId, &[u8])| (!p.is_empty()).then_some(u)) } + /// Returns the origin of the user (password/LDAP/...). + pub async fn origin(&self, user_id: &UserId) -> Result { + self.db.userid_origin.get(user_id).await.deserialized() + } + /// Returns the password hash for the given user. pub async fn password_hash(&self, user_id: &UserId) -> Result { self.db.userid_password.get(user_id).await.deserialized() } /// Hash and set the user's password to the Argon2 hash - pub fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { + pub async fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> { + // Cannot change the password of a LDAP user. There are two special cases : + // - a `None` password can be used to deactivate a LDAP user + // - a "*" password is used as the default password of an active LDAP user + if cfg!(feature = "ldap") + && password.is_some_and(|pwd| pwd != "*") + && self + .db + .userid_origin + .get(user_id) + .await + .deserialized::() + .is_ok_and(is_equal_to!("ldap")) + { + return Err!(Request(InvalidParam("Cannot change password of a LDAP user"))); + } + password .map(utils::hash::password) .transpose() @@ -1132,6 +1175,154 @@ impl Service { self.db.useridprofilekey_value.del(key); } } + + #[cfg(not(feature = "ldap"))] + pub async fn search_ldap(&self, _user_id: &UserId) -> Result> { + Err!(FeatureDisabled("ldap")) + } + + #[cfg(feature = "ldap")] + pub async fn search_ldap(&self, user_id: &UserId) -> Result> { + let localpart = user_id.localpart().to_owned(); + let lowercased_localpart = localpart.to_lowercase(); + + let config = &self.services.server.config.ldap; + let uri = config + .uri + .as_ref() + .ok_or_else(|| err!(Ldap(error!("LDAP URI is not configured."))))?; + + debug!(?uri, "LDAP creating connection..."); + let (conn, mut ldap) = LdapConnAsync::new(uri.as_str()) + .await + .map_err(|e| err!(Ldap(error!(?user_id, "LDAP connection setup error: {e}"))))?; + + let driver = self.services.server.runtime().spawn(async move { + match conn.drive().await { + | Err(e) => error!("LDAP connection error: {e}"), + | Ok(()) => debug!("LDAP connection completed."), + } + }); + + match (&config.bind_dn, &config.bind_password_file) { + | (Some(bind_dn), Some(bind_password_file)) => { + let bind_pw = String::from_utf8(std::fs::read(bind_password_file)?)?; + ldap.simple_bind(bind_dn, bind_pw.trim()) + .await + .and_then(ldap3::LdapResult::success) + .map_err(|e| err!(Ldap(error!("LDAP bind error: {e}"))))?; + }, + | (..) => {}, + } + + let attr = [&config.uid_attribute, &config.name_attribute]; + + let user_filter = &config.filter.replace("{username}", &lowercased_localpart); + + let (entries, _result) = ldap + .search(&config.base_dn, Scope::Subtree, user_filter, &attr) + .await + .and_then(ldap3::SearchResult::success) + .inspect(|(entries, result)| trace!(?entries, ?result, "LDAP Search")) + .map_err(|e| err!(Ldap(error!(?attr, ?user_filter, "LDAP search error: {e}"))))?; + + let mut dns: HashMap = entries + .into_iter() + .filter_map(|entry| { + let search_entry = SearchEntry::construct(entry); + debug!(?search_entry, "LDAP search entry"); + search_entry + .attrs + .get(&config.uid_attribute) + .into_iter() + .chain(search_entry.attrs.get(&config.name_attribute)) + .any(|ids| ids.contains(&localpart) || ids.contains(&lowercased_localpart)) + .then_some((search_entry.dn, false)) + }) + .collect(); + + if !config.admin_filter.is_empty() { + let admin_base_dn = if config.admin_base_dn.is_empty() { + &config.base_dn + } else { + &config.admin_base_dn + }; + + let admin_filter = &config + .admin_filter + .replace("{username}", &lowercased_localpart); + + let (admin_entries, _result) = ldap + .search(admin_base_dn, Scope::Subtree, admin_filter, &attr) + .await + .and_then(ldap3::SearchResult::success) + .inspect(|(entries, result)| trace!(?entries, ?result, "LDAP Admin Search")) + .map_err(|e| { + err!(Ldap(error!(?attr, ?admin_filter, "Ldap admin search error: {e}"))) + })?; + + dns.extend(admin_entries.into_iter().filter_map(|entry| { + let search_entry = SearchEntry::construct(entry); + debug!(?search_entry, "LDAP search entry"); + search_entry + .attrs + .get(&config.uid_attribute) + .into_iter() + .chain(search_entry.attrs.get(&config.name_attribute)) + .any(|ids| ids.contains(&localpart) || ids.contains(&lowercased_localpart)) + .then_some((search_entry.dn, true)) + })); + } + + ldap.unbind() + .await + .map_err(|e| err!(Ldap(error!("LDAP unbind error: {e}"))))?; + + driver.await.log_err().ok(); + + Ok(dns.drain().collect()) + } + + #[cfg(not(feature = "ldap"))] + pub async fn auth_ldap(&self, _user_dn: &str, _password: &str) -> Result { + Err!(FeatureDisabled("ldap")) + } + + #[cfg(feature = "ldap")] + pub async fn auth_ldap(&self, user_dn: &str, password: &str) -> Result { + let config = &self.services.server.config.ldap; + let uri = config + .uri + .as_ref() + .ok_or_else(|| err!(Ldap(error!("LDAP URI is not configured."))))?; + + debug!(?uri, "LDAP creating connection..."); + let (conn, mut ldap) = LdapConnAsync::new(uri.as_str()) + .await + .map_err(|e| err!(Ldap(error!(?user_dn, "LDAP connection setup error: {e}"))))?; + + let driver = self.services.server.runtime().spawn(async move { + match conn.drive().await { + | Err(e) => error!("LDAP connection error: {e}"), + | Ok(()) => debug!("LDAP connection completed."), + } + }); + + ldap.simple_bind(user_dn, password) + .await + .and_then(ldap3::LdapResult::success) + .map_err(|e| { + err!(Request(Forbidden(debug_error!("LDAP authentication error: {e}")))) + })?; + + ldap.unbind() + .await + .map_err(|e| err!(Ldap(error!("LDAP unbind error: {e}"))))?; + + driver.await.log_err().ok(); + + Ok(()) + } } pub fn parse_master_key( From fb7e739b72dad43211ff9b6077c33c479e2574f5 Mon Sep 17 00:00:00 2001 From: RatCornu Date: Sun, 10 Aug 2025 12:50:19 +0200 Subject: [PATCH 2017/2091] chore: remove unused LDAP mail attribute --- conduwuit-example.toml | 18 ++++++------------ src/api/client/session.rs | 8 ++++---- src/core/config/mod.rs | 26 ++++++++++---------------- 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 47a9da19..68f5b965 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1797,7 +1797,7 @@ # # example: "ou=users,dc=example,dc=org" # -#base_dn = false +#base_dn = # Bind DN if anonymous search is not enabled. # @@ -1810,7 +1810,7 @@ # example: "cn=ldap-reader,dc=example,dc=org" or # "cn={username},ou=users,dc=example,dc=org" # -#bind_dn = false +#bind_dn = # Path to a file on the system that contains the password for the # `bind_dn`. @@ -1834,13 +1834,7 @@ # #uid_attribute = "uid" -# Attribute containing the mail of the user. -# -# example: "mail" -# -#mail_attribute = "mail" - -# Attribute containing the distinguished name of the user. +# Attribute containing the display name of the user. # # example: "givenName" or "sn" # @@ -1852,9 +1846,9 @@ # # example: "ou=admins,dc=example,dc=org" # -#admin_base_dn = false +#admin_base_dn = -# The LDAP search filter to find administrative users for conduwuit. +# The LDAP search filter to find administrative users for continuwuity. # # If left blank, administrative state must be configured manually for each # user. @@ -1864,4 +1858,4 @@ # # example: "(objectClass=conduwuitAdmin)" or "(uid={username})" # -#admin_filter = false +#admin_filter = diff --git a/src/api/client/session.rs b/src/api/client/session.rs index 2f066d58..c57f5487 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -156,9 +156,9 @@ pub(super) async fn ldap_login( pub(crate) async fn handle_login( services: &Services, body: &Ruma, - identifier: &Option, + identifier: Option<&uiaa::UserIdentifier>, password: &str, - user: &Option, + user: Option<&String>, ) -> Result { debug!("Got password login type"); let user_id = @@ -185,7 +185,7 @@ pub(crate) async fn handle_login( } if cfg!(feature = "ldap") && services.config.ldap.enable { - ldap_login(services, &user_id, &lowercased_user_id, password).await + Box::pin(ldap_login(services, &user_id, &lowercased_user_id, password)).await } else { password_login(services, &user_id, &lowercased_user_id, password).await } @@ -222,7 +222,7 @@ pub(crate) async fn login_route( password, user, .. - }) => handle_login(&services, &body, identifier, password, user).await?, + }) => handle_login(&services, &body, identifier.as_ref(), password, user.as_ref()).await?, | login::v3::LoginInfo::Token(login::v3::Token { token }) => { debug!("Got token login type"); if !services.server.config.login_via_existing_session { diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index a7fbc7dc..e996d1fa 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -2063,7 +2063,7 @@ pub struct LdapConfig { /// Root of the searches. /// /// example: "ou=users,dc=example,dc=org" - #[serde(default)] + #[serde(default = "empty_string_fn")] pub base_dn: String, /// Bind DN if anonymous search is not enabled. @@ -2076,7 +2076,7 @@ pub struct LdapConfig { /// /// example: "cn=ldap-reader,dc=example,dc=org" or /// "cn={username},ou=users,dc=example,dc=org" - #[serde(default)] + #[serde(default = "some_empty_string_fn")] pub bind_dn: Option, /// Path to a file on the system that contains the password for the @@ -2105,15 +2105,7 @@ pub struct LdapConfig { #[serde(default = "default_ldap_uid_attribute")] pub uid_attribute: String, - /// Attribute containing the mail of the user. - /// - /// example: "mail" - /// - /// default: "mail" - #[serde(default = "default_ldap_mail_attribute")] - pub mail_attribute: String, - - /// Attribute containing the distinguished name of the user. + /// Attribute containing the display name of the user. /// /// example: "givenName" or "sn" /// @@ -2126,10 +2118,10 @@ pub struct LdapConfig { /// Defaults to `base_dn` if empty. /// /// example: "ou=admins,dc=example,dc=org" - #[serde(default)] + #[serde(default = "empty_string_fn")] pub admin_base_dn: String, - /// The LDAP search filter to find administrative users for conduwuit. + /// The LDAP search filter to find administrative users for continuwuity. /// /// If left blank, administrative state must be configured manually for each /// user. @@ -2138,7 +2130,7 @@ pub struct LdapConfig { /// entered username for more complex filters. /// /// example: "(objectClass=conduwuitAdmin)" or "(uid={username})" - #[serde(default)] + #[serde(default = "empty_string_fn")] pub admin_filter: String, } @@ -2240,6 +2232,10 @@ impl Config { fn true_fn() -> bool { true } +fn empty_string_fn() -> String { String::new() } + +fn some_empty_string_fn() -> Option { Some(String::new()) } + fn default_address() -> ListeningAddr { ListeningAddr { addrs: Right(vec![Ipv4Addr::LOCALHOST.into(), Ipv6Addr::LOCALHOST.into()]), @@ -2536,6 +2532,4 @@ fn default_ldap_search_filter() -> String { "(objectClass=*)".to_owned() } fn default_ldap_uid_attribute() -> String { String::from("uid") } -fn default_ldap_mail_attribute() -> String { String::from("mail") } - fn default_ldap_name_attribute() -> String { String::from("givenName") } From c58b9f05ed89e8a3c4d5d080afa05a3c1aa48882 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 10 Aug 2025 20:49:08 +0100 Subject: [PATCH 2018/2091] chore: Fix default attributes for config --- conduwuit-example.toml | 4 ++-- src/core/config/mod.rs | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 68f5b965..06e67a89 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1797,7 +1797,7 @@ # # example: "ou=users,dc=example,dc=org" # -#base_dn = +#base_dn = "" # Bind DN if anonymous search is not enabled. # @@ -1846,7 +1846,7 @@ # # example: "ou=admins,dc=example,dc=org" # -#admin_base_dn = +#admin_base_dn = "" # The LDAP search filter to find administrative users for continuwuity. # diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index e996d1fa..13778b5e 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -2063,7 +2063,9 @@ pub struct LdapConfig { /// Root of the searches. /// /// example: "ou=users,dc=example,dc=org" - #[serde(default = "empty_string_fn")] + /// + /// default: "" + #[serde(default)] pub base_dn: String, /// Bind DN if anonymous search is not enabled. @@ -2076,7 +2078,9 @@ pub struct LdapConfig { /// /// example: "cn=ldap-reader,dc=example,dc=org" or /// "cn={username},ou=users,dc=example,dc=org" - #[serde(default = "some_empty_string_fn")] + /// + /// default: + #[serde(default)] pub bind_dn: Option, /// Path to a file on the system that contains the password for the @@ -2118,7 +2122,9 @@ pub struct LdapConfig { /// Defaults to `base_dn` if empty. /// /// example: "ou=admins,dc=example,dc=org" - #[serde(default = "empty_string_fn")] + /// + /// default: "" + #[serde(default)] pub admin_base_dn: String, /// The LDAP search filter to find administrative users for continuwuity. @@ -2130,7 +2136,9 @@ pub struct LdapConfig { /// entered username for more complex filters. /// /// example: "(objectClass=conduwuitAdmin)" or "(uid={username})" - #[serde(default = "empty_string_fn")] + /// + /// default: + #[serde(default)] pub admin_filter: String, } @@ -2232,10 +2240,6 @@ impl Config { fn true_fn() -> bool { true } -fn empty_string_fn() -> String { String::new() } - -fn some_empty_string_fn() -> Option { Some(String::new()) } - fn default_address() -> ListeningAddr { ListeningAddr { addrs: Right(vec![Ipv4Addr::LOCALHOST.into(), Ipv6Addr::LOCALHOST.into()]), From 0ed691edef9d95c8a2e3b54ba8b72ecb6a326e88 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 10 Aug 2025 20:54:05 +0100 Subject: [PATCH 2019/2091] fix: Make builds without LDAP work correctly --- src/service/users/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index 35202ec7..fff1661c 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -1,18 +1,18 @@ -use std::{ - collections::{BTreeMap, HashMap}, - mem, - sync::Arc, -}; +#[cfg(feature = "ldap")] +use std::collections::HashMap; +use std::{collections::BTreeMap, mem, sync::Arc}; +#[cfg(feature = "ldap")] +use conduwuit::result::LogErr; use conduwuit::{ - Err, Error, Result, Server, at, debug_warn, err, is_equal_to, - result::LogErr, - trace, + Err, Error, Result, Server, at, debug_warn, err, is_equal_to, trace, utils::{self, ReadyExt, stream::TryIgnore, string::Unquoted}, }; +#[cfg(feature = "ldap")] use conduwuit_core::{debug, error}; use database::{Deserialized, Ignore, Interfix, Json, Map}; use futures::{Stream, StreamExt, TryFutureExt}; +#[cfg(feature = "ldap")] use ldap3::{LdapConnAsync, Scope, SearchEntry}; use ruma::{ DeviceId, KeyId, MilliSecondsSinceUnixEpoch, OneTimeKeyAlgorithm, OneTimeKeyId, From cb09bfa4e7be034164f810d857fc505d692c06d7 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 10 Aug 2025 21:08:06 +0100 Subject: [PATCH 2020/2091] fix: Correctly pass ldap feature from the default crate --- src/main/Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 8aaa3cc6..eafa1e48 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -56,6 +56,7 @@ standard = [ "jemalloc", "jemalloc_conf", "journald", + "ldap", "media_thumbnail", "systemd", "url_preview", @@ -114,6 +115,9 @@ jemalloc_stats = [ jemalloc_conf = [ "conduwuit-core/jemalloc_conf", ] +ldap = [ + "conduwuit-api/ldap", +] media_thumbnail = [ "conduwuit-service/media_thumbnail", ] From 57d77430370616bbc40aac145da1b79d516431ce Mon Sep 17 00:00:00 2001 From: RatCornu Date: Thu, 14 Aug 2025 22:48:55 +0200 Subject: [PATCH 2021/2091] feat: add ldap_only config option --- conduwuit-example.toml | 14 ++++++++++---- src/api/client/session.rs | 15 +++++++++++---- src/core/config/mod.rs | 16 ++++++++++++++-- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 06e67a89..41fbfb3a 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1787,11 +1787,17 @@ # #enable = false +# Whether to force LDAP authentication or authorize classical password login. +# +# example: "true" +# +#ldap_only = false + # URI of the LDAP server. # # example: "ldap://ldap.example.com:389" # -#uri = +#uri = "" # Root of the searches. # @@ -1810,14 +1816,14 @@ # example: "cn=ldap-reader,dc=example,dc=org" or # "cn={username},ou=users,dc=example,dc=org" # -#bind_dn = +#bind_dn = "" # Path to a file on the system that contains the password for the # `bind_dn`. # # The server must be able to access the file, and it must not be empty. # -#bind_password_file = false +#bind_password_file = "" # Search filter to limit user searches. # @@ -1858,4 +1864,4 @@ # # example: "(objectClass=conduwuitAdmin)" or "(uid={username})" # -#admin_filter = +#admin_filter = "" diff --git a/src/api/client/session.rs b/src/api/client/session.rs index c57f5487..da7bed2c 100644 --- a/src/api/client/session.rs +++ b/src/api/client/session.rs @@ -3,10 +3,10 @@ use std::time::Duration; use axum::extract::State; use axum_client_ip::InsecureClientIp; use conduwuit::{ - Err, Error, Result, debug, err, info, utils, - utils::{ReadyExt, hash}, + Err, Error, Result, debug, err, info, + utils::{self, ReadyExt, hash}, }; -use conduwuit_core::debug_error; +use conduwuit_core::{debug_error, debug_warn}; use conduwuit_service::{Services, uiaa::SESSION_ID_LENGTH}; use futures::StreamExt; use ruma::{ @@ -185,7 +185,14 @@ pub(crate) async fn handle_login( } if cfg!(feature = "ldap") && services.config.ldap.enable { - Box::pin(ldap_login(services, &user_id, &lowercased_user_id, password)).await + match Box::pin(ldap_login(services, &user_id, &lowercased_user_id, password)).await { + | Ok(user_id) => Ok(user_id), + | Err(err) if services.config.ldap.ldap_only => Err(err), + | Err(err) => { + debug_warn!("{err}"); + password_login(services, &user_id, &lowercased_user_id, password).await + }, + } } else { password_login(services, &user_id, &lowercased_user_id, password).await } diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 13778b5e..e8518ed4 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -2055,9 +2055,19 @@ pub struct LdapConfig { #[serde(default)] pub enable: bool, + /// Whether to force LDAP authentication or authorize classical password + /// login. + /// + /// example: "true" + #[serde(default)] + pub ldap_only: bool, + /// URI of the LDAP server. /// /// example: "ldap://ldap.example.com:389" + /// + /// default: "" + #[serde(default)] pub uri: Option, /// Root of the searches. @@ -2079,7 +2089,7 @@ pub struct LdapConfig { /// example: "cn=ldap-reader,dc=example,dc=org" or /// "cn={username},ou=users,dc=example,dc=org" /// - /// default: + /// default: "" #[serde(default)] pub bind_dn: Option, @@ -2087,6 +2097,8 @@ pub struct LdapConfig { /// `bind_dn`. /// /// The server must be able to access the file, and it must not be empty. + /// + /// default: "" #[serde(default)] pub bind_password_file: Option, @@ -2137,7 +2149,7 @@ pub struct LdapConfig { /// /// example: "(objectClass=conduwuitAdmin)" or "(uid={username})" /// - /// default: + /// default: "" #[serde(default)] pub admin_filter: String, } From 3183210459feb640734341a189a8437fdf7f2240 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 23 Aug 2025 21:28:31 +0100 Subject: [PATCH 2022/2091] fix: Post-merge compile issues --- conduwuit-example.toml | 3 ++- src/service/appservice/mod.rs | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 41fbfb3a..f0e510b4 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -1787,7 +1787,8 @@ # #enable = false -# Whether to force LDAP authentication or authorize classical password login. +# Whether to force LDAP authentication or authorize classical password +# login. # # example: "true" # diff --git a/src/service/appservice/mod.rs b/src/service/appservice/mod.rs index ebd798f6..adbf3b6e 100644 --- a/src/service/appservice/mod.rs +++ b/src/service/appservice/mod.rs @@ -109,7 +109,10 @@ impl Service { )?; if !self.services.users.exists(&appservice_user_id).await { - self.services.users.create(&appservice_user_id, None)?; + self.services + .users + .create(&appservice_user_id, None, None) + .await?; } else if self .services .users @@ -120,7 +123,8 @@ impl Service { // Reactivate the appservice user if it was accidentally deactivated self.services .users - .set_password(&appservice_user_id, None)?; + .set_password(&appservice_user_id, None) + .await?; } self.registration_info From 30a56d5cb95b2545000088d38855724a5a956b40 Mon Sep 17 00:00:00 2001 From: nex Date: Thu, 28 Aug 2025 17:15:32 +0000 Subject: [PATCH 2023/2091] Update renovate.json --- renovate.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/renovate.json b/renovate.json index b48fc6dc..deb428af 100644 --- a/renovate.json +++ b/renovate.json @@ -13,8 +13,8 @@ "enabled": true }, "labels": [ - "dependencies", - "github_actions" + "Dependencies", + "Dependencies/Renovate" ], "ignoreDeps": [ "tikv-jemallocator", From dd22325ea2676c30f784c989e025e0391dbe911c Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Mon, 18 Aug 2025 20:45:30 +0100 Subject: [PATCH 2024/2091] refactor(ci): Consolidate Rust checks with optimised toolchain setup Merge rust-checks.yml into prek-checks.yml for a unified workflow that runs formatting and clippy/test checks in parallel jobs. Add reusable composite actions: - setup-rust: Smart Rust toolchain management with caching * Uses cargo-binstall for pre-built binary downloads * Integrates Mozilla sccache-action for compilation caching * Workspace-relative paths for better cache control * GitHub token support for improved rate limits - setup-llvm-with-apt: LLVM installation with native dependencies - detect-runner-os: Consistent OS detection for cache keys Key improvements: - Install prek via cargo-binstall --git (crates.io outdated at v0.0.1) - Download timelord-cli from cargo-quickinstall - Set BINSTALL_MAXIMUM_RESOLUTION_TIMEOUT=10 to avoid rate limit delays - Default Rust version 1.87.0 with override support - Remove redundant sccache stats (handled by Mozilla action) Significantly reduces CI runtime through binary downloads instead of compilation while maintaining all existing quality checks. --- .forgejo/actions/detect-runner-os/action.yml | 39 +++ .../actions/setup-llvm-with-apt/action.yml | 167 +++++++++++++ .forgejo/actions/setup-rust/action.yml | 236 ++++++++++++++++++ .forgejo/workflows/prek-checks.yml | 59 ++++- .forgejo/workflows/rust-checks.yml | 144 ----------- 5 files changed, 494 insertions(+), 151 deletions(-) create mode 100644 .forgejo/actions/detect-runner-os/action.yml create mode 100644 .forgejo/actions/setup-llvm-with-apt/action.yml create mode 100644 .forgejo/actions/setup-rust/action.yml delete mode 100644 .forgejo/workflows/rust-checks.yml diff --git a/.forgejo/actions/detect-runner-os/action.yml b/.forgejo/actions/detect-runner-os/action.yml new file mode 100644 index 00000000..6ada1d5d --- /dev/null +++ b/.forgejo/actions/detect-runner-os/action.yml @@ -0,0 +1,39 @@ +name: detect-runner-os +description: | + Detect the actual OS name and version of the runner. + Provides separate outputs for name, version, and a combined slug. + +outputs: + name: + description: 'OS name (e.g. Ubuntu, Debian)' + value: ${{ steps.detect.outputs.name }} + version: + description: 'OS version (e.g. 22.04, 11)' + value: ${{ steps.detect.outputs.version }} + slug: + description: 'Combined OS slug (e.g. Ubuntu-22.04)' + value: ${{ steps.detect.outputs.slug }} + +runs: + using: composite + steps: + - name: Detect runner OS + id: detect + shell: bash + run: | + # Detect OS version (try lsb_release first, fall back to /etc/os-release) + OS_VERSION=$(lsb_release -rs 2>/dev/null || grep VERSION_ID /etc/os-release | cut -d'"' -f2) + + # Detect OS name and capitalise (try lsb_release first, fall back to /etc/os-release) + OS_NAME=$(lsb_release -is 2>/dev/null || grep "^ID=" /etc/os-release | cut -d'=' -f2 | tr -d '"' | sed 's/\b\(.\)/\u\1/g') + + # Create combined slug + OS_SLUG="${OS_NAME}-${OS_VERSION}" + + # Set outputs + echo "name=${OS_NAME}" >> $GITHUB_OUTPUT + echo "version=${OS_VERSION}" >> $GITHUB_OUTPUT + echo "slug=${OS_SLUG}" >> $GITHUB_OUTPUT + + # Log detection results + echo "🔍 Detected Runner OS: ${OS_NAME} ${OS_VERSION}" diff --git a/.forgejo/actions/setup-llvm-with-apt/action.yml b/.forgejo/actions/setup-llvm-with-apt/action.yml new file mode 100644 index 00000000..eb421e4f --- /dev/null +++ b/.forgejo/actions/setup-llvm-with-apt/action.yml @@ -0,0 +1,167 @@ +name: setup-llvm-with-apt +description: | + Set up LLVM toolchain with APT package management and smart caching. + Supports cross-compilation architectures and additional package installation. + + Creates symlinks in /usr/bin: clang, clang++, lld, llvm-ar, llvm-ranlib + +inputs: + dpkg-arch: + description: 'Debian architecture for cross-compilation (e.g. arm64)' + required: false + default: '' + extra-packages: + description: 'Additional APT packages to install (space-separated)' + required: false + default: '' + llvm-version: + description: 'LLVM version to install' + required: false + default: '20' + +outputs: + llvm-version: + description: 'Installed LLVM version' + value: ${{ steps.configure.outputs.version }} + +runs: + using: composite + steps: + - name: Detect runner OS + id: runner-os + uses: ./.forgejo/actions/detect-runner-os + + - name: Configure cross-compilation architecture + if: inputs.dpkg-arch != '' + shell: bash + run: | + echo "🏗️ Adding ${{ inputs.dpkg-arch }} architecture" + sudo dpkg --add-architecture ${{ inputs.dpkg-arch }} + + # Restrict default sources to amd64 + sudo sed -i 's/^deb http/deb [arch=amd64] http/g' /etc/apt/sources.list + sudo sed -i 's/^deb https/deb [arch=amd64] https/g' /etc/apt/sources.list + + # Add ports sources for foreign architecture + sudo tee /etc/apt/sources.list.d/${{ inputs.dpkg-arch }}.list > /dev/null <> $GITHUB_OUTPUT + else + echo "📦 LLVM ${{ inputs.llvm-version }} not found or incomplete - installing..." + + echo "::group::🔧 Installing LLVM ${{ inputs.llvm-version }}" + wget -O - https://apt.llvm.org/llvm.sh | bash -s -- ${{ inputs.llvm-version }} + echo "::endgroup::" + + if [ ! -f "/usr/bin/clang-${{ inputs.llvm-version }}" ]; then + echo "❌ Failed to install LLVM ${{ inputs.llvm-version }}" + exit 1 + fi + + echo "✅ Installed LLVM ${{ inputs.llvm-version }}" + echo "needs-install=true" >> $GITHUB_OUTPUT + fi + + - name: Prepare for additional packages + if: inputs.extra-packages != '' + shell: bash + run: | + # Update APT if LLVM was cached (installer script already does apt-get update) + if [[ "${{ steps.llvm-setup.outputs.needs-install }}" != "true" ]]; then + echo "::group::📦 Running apt-get update (LLVM cached, extra packages needed)" + sudo apt-get update + echo "::endgroup::" + fi + echo "::group::📦 Installing additional packages" + + - name: Install additional packages + if: inputs.extra-packages != '' + uses: https://github.com/awalsh128/cache-apt-pkgs-action@latest + with: + packages: ${{ inputs.extra-packages }} + version: 1.0 + + - name: End package installation group + if: inputs.extra-packages != '' + shell: bash + run: echo "::endgroup::" + + - name: Configure LLVM environment + id: configure + shell: bash + run: | + echo "::group::🔧 Configuring LLVM ${{ inputs.llvm-version }} environment" + + # Create symlinks + sudo ln -sf "/usr/bin/clang-${{ inputs.llvm-version }}" /usr/bin/clang + sudo ln -sf "/usr/bin/clang++-${{ inputs.llvm-version }}" /usr/bin/clang++ + sudo ln -sf "/usr/bin/lld-${{ inputs.llvm-version }}" /usr/bin/lld + sudo ln -sf "/usr/bin/llvm-ar-${{ inputs.llvm-version }}" /usr/bin/llvm-ar + sudo ln -sf "/usr/bin/llvm-ranlib-${{ inputs.llvm-version }}" /usr/bin/llvm-ranlib + echo " ✓ Created symlinks" + + # Setup library paths + LLVM_LIB_PATH="/usr/lib/llvm-${{ inputs.llvm-version }}/lib" + if [ -d "$LLVM_LIB_PATH" ]; then + echo "LD_LIBRARY_PATH=${LLVM_LIB_PATH}:${LD_LIBRARY_PATH:-}" >> $GITHUB_ENV + echo "LIBCLANG_PATH=${LLVM_LIB_PATH}" >> $GITHUB_ENV + + echo "$LLVM_LIB_PATH" | sudo tee "/etc/ld.so.conf.d/llvm-${{ inputs.llvm-version }}.conf" > /dev/null + sudo ldconfig + echo " ✓ Configured library paths" + else + # Fallback to standard library location + if [ -d "/usr/lib/x86_64-linux-gnu" ]; then + echo "LIBCLANG_PATH=/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV + echo " ✓ Using fallback library path" + fi + fi + + # Set output + echo "version=${{ inputs.llvm-version }}" >> $GITHUB_OUTPUT + echo "::endgroup::" + echo "✅ LLVM ready: $(clang --version | head -1)" diff --git a/.forgejo/actions/setup-rust/action.yml b/.forgejo/actions/setup-rust/action.yml new file mode 100644 index 00000000..091da8c2 --- /dev/null +++ b/.forgejo/actions/setup-rust/action.yml @@ -0,0 +1,236 @@ +name: setup-rust +description: | + Set up Rust toolchain with sccache for compilation caching. + Respects rust-toolchain.toml by default or accepts explicit version override. + +inputs: + cache-key-suffix: + description: 'Optional suffix for cache keys (e.g. platform identifier)' + required: false + default: '' + rust-components: + description: 'Additional Rust components to install (space-separated)' + required: false + default: '' + rust-target: + description: 'Rust target triple (e.g. x86_64-unknown-linux-gnu)' + required: false + default: '' + rust-version: + description: 'Rust version to install (e.g. nightly). Defaults to 1.87.0' + required: false + default: '1.87.0' + sccache-cache-limit: + description: 'Maximum size limit for sccache local cache (e.g. 2G, 500M)' + required: false + default: '2G' + github-token: + description: 'GitHub token for downloading sccache from GitHub releases' + required: false + default: '' + +outputs: + rust-version: + description: 'Installed Rust version' + value: ${{ steps.rust-setup.outputs.version }} + +runs: + using: composite + steps: + - name: Detect runner OS + id: runner-os + uses: ./.forgejo/actions/detect-runner-os + + - name: Configure Cargo environment + shell: bash + run: | + # Use workspace-relative paths for better control and consistency + echo "CARGO_HOME=${{ github.workspace }}/.cargo" >> $GITHUB_ENV + echo "CARGO_TARGET_DIR=${{ github.workspace }}/target" >> $GITHUB_ENV + echo "SCCACHE_DIR=${{ github.workspace }}/.sccache" >> $GITHUB_ENV + echo "RUSTUP_HOME=${{ github.workspace }}/.rustup" >> $GITHUB_ENV + + # Limit binstall resolution timeout to avoid GitHub rate limit delays + echo "BINSTALL_MAXIMUM_RESOLUTION_TIMEOUT=10" >> $GITHUB_ENV + + # Ensure directories exist for first run + mkdir -p "${{ github.workspace }}/.cargo" + mkdir -p "${{ github.workspace }}/.sccache" + mkdir -p "${{ github.workspace }}/target" + mkdir -p "${{ github.workspace }}/.rustup" + + - name: Start cache restore group + shell: bash + run: echo "::group::📦 Restoring caches (registry, toolchain, build artifacts)" + + - name: Cache Cargo registry and git + id: registry-cache + uses: https://github.com/actions/cache@v4 + with: + path: | + .cargo/registry/index + .cargo/registry/cache + .cargo/git/db + # Registry cache saved per workflow, restored from any workflow's cache + # Each workflow maintains its own registry that accumulates its needed crates + key: cargo-registry-${{ steps.runner-os.outputs.slug }}-${{ github.workflow }} + restore-keys: | + cargo-registry-${{ steps.runner-os.outputs.slug }}- + + - name: Cache toolchain binaries + id: toolchain-cache + uses: https://github.com/actions/cache@v4 + with: + path: | + .cargo/bin + .rustup/toolchains + .rustup/update-hashes + # Shared toolchain cache across all Rust versions + key: toolchain-${{ steps.runner-os.outputs.slug }} + + - name: Debug GitHub token availability + shell: bash + run: | + if [ -z "${{ inputs.github-token }}" ]; then + echo "⚠️ No GitHub token provided - sccache will use fallback download method" + else + echo "✅ GitHub token provided for sccache" + fi + + - name: Setup sccache + uses: https://github.com/mozilla-actions/sccache-action@v0.0.9 + with: + token: ${{ inputs.github-token }} + + - name: Cache build artifacts + id: build-cache + uses: https://github.com/actions/cache@v4 + with: + path: | + target/**/deps + !target/**/deps/*.rlib + target/**/build + target/**/.fingerprint + target/**/incremental + target/**/*.d + /timelord/ + # Build artifacts - cache per code change, restore from deps when code changes + key: >- + build-${{ steps.runner-os.outputs.slug }}-${{ inputs.rust-version }}${{ inputs.cache-key-suffix && format('-{0}', inputs.cache-key-suffix) || '' }}-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}-${{ hashFiles('**/*.rs', '**/Cargo.toml') }} + restore-keys: | + build-${{ steps.runner-os.outputs.slug }}-${{ inputs.rust-version }}${{ inputs.cache-key-suffix && format('-{0}', inputs.cache-key-suffix) || '' }}-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }}- + + - name: End cache restore group + shell: bash + run: echo "::endgroup::" + + - name: Setup Rust toolchain + shell: bash + run: | + # Install rustup if not already cached + if ! command -v rustup &> /dev/null; then + echo "::group::📦 Installing rustup" + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --default-toolchain none + source "$CARGO_HOME/env" + echo "::endgroup::" + else + echo "✅ rustup already available" + fi + + # Setup the appropriate Rust version + if [[ -n "${{ inputs.rust-version }}" ]]; then + echo "::group::📦 Setting up Rust ${{ inputs.rust-version }}" + # Set override first to prevent rust-toolchain.toml from auto-installing + rustup override set ${{ inputs.rust-version }} 2>/dev/null || true + + # Check if we need to install/update the toolchain + if rustup toolchain list | grep -q "^${{ inputs.rust-version }}-"; then + rustup update ${{ inputs.rust-version }} + else + rustup toolchain install ${{ inputs.rust-version }} --profile minimal -c cargo,clippy,rustfmt + fi + else + echo "::group::📦 Setting up Rust from rust-toolchain.toml" + rustup show + fi + echo "::endgroup::" + + - name: Configure PATH and install tools + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.github-token }} + run: | + # Add .cargo/bin to PATH permanently for all subsequent steps + echo "${{ github.workspace }}/.cargo/bin" >> $GITHUB_PATH + + # For this step only, we need to add it to PATH since GITHUB_PATH takes effect in the next step + export PATH="${{ github.workspace }}/.cargo/bin:$PATH" + + # Install cargo-binstall for fast binary installations + if command -v cargo-binstall &> /dev/null; then + echo "✅ cargo-binstall already available" + else + echo "::group::📦 Installing cargo-binstall" + curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash + echo "::endgroup::" + fi + + if command -v prek &> /dev/null; then + echo "✅ prek already available" + else + echo "::group::📦 Installing prek" + # prek isn't regularly published to crates.io, so we use git source + cargo-binstall -y --no-symlinks --git https://github.com/j178/prek prek + echo "::endgroup::" + fi + + if command -v timelord &> /dev/null; then + echo "✅ timelord already available" + else + echo "::group::📦 Installing timelord" + cargo-binstall -y --no-symlinks timelord-cli + echo "::endgroup::" + fi + + - name: Configure sccache environment + shell: bash + run: | + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV + echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV + echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV + echo "CMAKE_CUDA_COMPILER_LAUNCHER=sccache" >> $GITHUB_ENV + echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV + + # Configure incremental compilation GC + # If we restored from old cache (partial hit), clean up aggressively + if [[ "${{ steps.build-cache.outputs.cache-hit }}" != "true" ]]; then + echo "♻️ Partial cache hit - enabling cache cleanup" + echo "CARGO_INCREMENTAL_GC_THRESHOLD=5" >> $GITHUB_ENV + fi + + - name: Install Rust components + if: inputs.rust-components != '' + shell: bash + run: | + echo "📦 Installing components: ${{ inputs.rust-components }}" + rustup component add ${{ inputs.rust-components }} + + - name: Install Rust target + if: inputs.rust-target != '' + shell: bash + run: | + echo "📦 Installing target: ${{ inputs.rust-target }}" + rustup target add ${{ inputs.rust-target }} + + - name: Output version and summary + id: rust-setup + shell: bash + run: | + RUST_VERSION=$(rustc --version | cut -d' ' -f2) + echo "version=$RUST_VERSION" >> $GITHUB_OUTPUT + + echo "📋 Setup complete:" + echo " Rust: $(rustc --version)" + echo " Cargo: $(cargo --version)" + echo " prek: $(prek --version 2>/dev/null || echo 'installed')" + echo " timelord: $(timelord --version 2>/dev/null || echo 'installed')" diff --git a/.forgejo/workflows/prek-checks.yml b/.forgejo/workflows/prek-checks.yml index ac330ca2..c25b9c3d 100644 --- a/.forgejo/workflows/prek-checks.yml +++ b/.forgejo/workflows/prek-checks.yml @@ -2,7 +2,6 @@ name: Checks / Prek on: push: - pull_request: permissions: contents: read @@ -17,18 +16,64 @@ jobs: with: persist-credentials: false - - name: Install uv - uses: https://github.com/astral-sh/setup-uv@v5 + - name: Setup Rust nightly + uses: ./.forgejo/actions/setup-rust with: - enable-cache: true - ignore-nothing-to-cache: true - cache-dependency-glob: '' + rust-version: nightly + github-token: ${{ secrets.GH_PUBLIC_RO }} - name: Run prek run: | - uvx prek run \ + prek run \ --all-files \ --hook-stage manual \ --show-diff-on-failure \ --color=always \ -v + + - name: Check Rust formatting + run: | + cargo +nightly fmt --all -- --check && \ + echo "✅ Formatting check passed" || \ + exit 1 + + clippy-and-tests: + name: Clippy and Cargo Tests + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup LLVM + uses: ./.forgejo/actions/setup-llvm-with-apt + with: + extra-packages: liburing-dev liburing2 + + - name: Setup Rust with caching + uses: ./.forgejo/actions/setup-rust + with: + github-token: ${{ secrets.GH_PUBLIC_RO }} + + - name: Run Clippy lints + run: | + cargo clippy \ + --workspace \ + --features full \ + --locked \ + --no-deps \ + --profile test \ + -- \ + -D warnings + + - name: Run Cargo tests + run: | + cargo test \ + --workspace \ + --features full \ + --locked \ + --profile test \ + --all-targets \ + --no-fail-fast diff --git a/.forgejo/workflows/rust-checks.yml b/.forgejo/workflows/rust-checks.yml deleted file mode 100644 index c46363a0..00000000 --- a/.forgejo/workflows/rust-checks.yml +++ /dev/null @@ -1,144 +0,0 @@ -name: Checks / Rust - -on: - push: - -jobs: - format: - name: Format - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Install rust - uses: ./.forgejo/actions/rust-toolchain - with: - toolchain: "nightly" - components: "rustfmt" - - - name: Check formatting - run: | - cargo +nightly fmt --all -- --check - - clippy: - name: Clippy - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Install rust - uses: ./.forgejo/actions/rust-toolchain - - - uses: https://github.com/actions/create-github-app-token@v2 - id: app-token - with: - app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - github-api-url: https://api.github.com - owner: ${{ vars.GH_APP_OWNER }} - repositories: "" - - name: Install sccache - uses: ./.forgejo/actions/sccache - with: - token: ${{ steps.app-token.outputs.token }} - - run: sudo apt-get update - - name: Install system dependencies - uses: https://github.com/awalsh128/cache-apt-pkgs-action@v1 - with: - packages: clang liburing-dev - version: 1 - - name: Cache Rust registry - uses: actions/cache@v3 - with: - path: | - ~/.cargo/git - !~/.cargo/git/checkouts - ~/.cargo/registry - !~/.cargo/registry/src - key: rust-registry-${{hashFiles('**/Cargo.lock') }} - - name: Timelord - uses: ./.forgejo/actions/timelord - with: - key: sccache-v0 - path: . - - name: Clippy - run: | - cargo clippy \ - --workspace \ - --features full \ - --locked \ - --no-deps \ - --profile test \ - -- \ - -D warnings - - - name: Show sccache stats - if: always() - run: sccache --show-stats - - cargo-test: - name: Cargo Test - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Install rust - uses: ./.forgejo/actions/rust-toolchain - - - uses: https://github.com/actions/create-github-app-token@v2 - id: app-token - with: - app-id: ${{ vars.GH_APP_ID }} - private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} - github-api-url: https://api.github.com - owner: ${{ vars.GH_APP_OWNER }} - repositories: "" - - name: Install sccache - uses: ./.forgejo/actions/sccache - with: - token: ${{ steps.app-token.outputs.token }} - - run: sudo apt-get update - - name: Install system dependencies - uses: https://github.com/awalsh128/cache-apt-pkgs-action@v1 - with: - packages: clang liburing-dev - version: 1 - - name: Cache Rust registry - uses: actions/cache@v3 - with: - path: | - ~/.cargo/git - !~/.cargo/git/checkouts - ~/.cargo/registry - !~/.cargo/registry/src - key: rust-registry-${{hashFiles('**/Cargo.lock') }} - - name: Timelord - uses: ./.forgejo/actions/timelord - with: - key: sccache-v0 - path: . - - name: Cargo Test - run: | - cargo test \ - --workspace \ - --features full \ - --locked \ - --profile test \ - --all-targets \ - --no-fail-fast - - - name: Show sccache stats - if: always() - run: sccache --show-stats From 37248a4f6821c9271fd51e5ddc3b744d51fde969 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Thu, 28 Aug 2025 20:10:05 +0100 Subject: [PATCH 2025/2091] chore: Add reasons for test skips --- src/database/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/database/tests.rs b/src/database/tests.rs index c1a9f47c..30562a66 100644 --- a/src/database/tests.rs +++ b/src/database/tests.rs @@ -326,7 +326,7 @@ fn ser_array() { } #[test] -#[ignore] +#[ignore = "arrayvec deserialization is not implemented (separators)"] fn de_array() { let a: u64 = 123_456; let b: u64 = 987_654; @@ -358,7 +358,7 @@ fn de_array() { } #[test] -#[ignore] +#[ignore = "Nested sequences are not supported"] fn de_complex() { type Key<'a> = (&'a UserId, ArrayVec, &'a RoomId); From b5a2e49ae4b1b9b17f5966a8fbbfaf67a6ee7b26 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Thu, 28 Aug 2025 20:35:27 +0100 Subject: [PATCH 2026/2091] fix: Resolve Clippy CI failures from elided lifetime warnings The latest Rust nightly compiler (2025-08-27) introduced the elided-named-lifetimes lint which causes Clippy CI checks to fail when an elided lifetime ('_) resolves to a named lifetime that's already in scope. This commit fixes the Clippy warnings by: - Making lifetime relationships explicit where 'a is already in scope - Keeping elided lifetimes ('_) in functions without explicit lifetime parameters - Ensuring proper lifetime handling in the database pool module Affected files (17 total): - Database map modules: Handle, Key, and KeyVal references in get, qry, keys, and stream operations - Database pool module: into_recv_seek function This change resolves the CI build failures without changing any functionality, ensuring the codebase remains compatible with the latest nightly Clippy checks. --- src/database/map/get_batch.rs | 6 +++--- src/database/map/keys.rs | 2 +- src/database/map/keys_from.rs | 4 ++-- src/database/map/keys_prefix.rs | 6 +++--- src/database/map/qry_batch.rs | 6 +++--- src/database/map/rev_keys.rs | 2 +- src/database/map/rev_keys_from.rs | 4 ++-- src/database/map/rev_keys_prefix.rs | 6 +++--- src/database/map/rev_stream.rs | 2 +- src/database/map/rev_stream_from.rs | 4 ++-- src/database/map/rev_stream_prefix.rs | 6 +++--- src/database/map/stream.rs | 2 +- src/database/map/stream_from.rs | 4 ++-- src/database/map/stream_prefix.rs | 6 +++--- src/database/pool.rs | 2 +- src/service/migrations.rs | 8 ++++---- src/service/presence/mod.rs | 4 ++-- src/service/pusher/mod.rs | 2 +- src/service/rooms/alias/mod.rs | 6 ++++-- src/service/rooms/pdu_metadata/data.rs | 2 +- src/service/rooms/read_receipt/data.rs | 2 +- src/service/rooms/read_receipt/mod.rs | 2 +- src/service/rooms/search/mod.rs | 20 ++++++++++---------- src/service/rooms/short/mod.rs | 2 +- src/service/rooms/state/mod.rs | 2 +- src/service/rooms/state_cache/mod.rs | 20 ++++++++++---------- src/service/rooms/state_cache/via.rs | 2 +- src/service/users/mod.rs | 8 ++++---- 28 files changed, 72 insertions(+), 70 deletions(-) diff --git a/src/database/map/get_batch.rs b/src/database/map/get_batch.rs index e23a8848..539f0c39 100644 --- a/src/database/map/get_batch.rs +++ b/src/database/map/get_batch.rs @@ -19,7 +19,7 @@ where S: Stream + Send + 'a, K: AsRef<[u8]> + Send + Sync + 'a, { - fn get(self, map: &'a Arc) -> impl Stream>> + Send + 'a; + fn get(self, map: &'a Arc) -> impl Stream>> + Send + 'a; } impl<'a, K, S> Get<'a, K, S> for S @@ -29,7 +29,7 @@ where K: AsRef<[u8]> + Send + Sync + 'a, { #[inline] - fn get(self, map: &'a Arc) -> impl Stream>> + Send + 'a { + fn get(self, map: &'a Arc) -> impl Stream>> + Send + 'a { map.get_batch(self) } } @@ -39,7 +39,7 @@ where pub(crate) fn get_batch<'a, S, K>( self: &'a Arc, keys: S, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where S: Stream + Send + 'a, K: AsRef<[u8]> + Send + Sync + 'a, diff --git a/src/database/map/keys.rs b/src/database/map/keys.rs index 7ca932a5..ac044e91 100644 --- a/src/database/map/keys.rs +++ b/src/database/map/keys.rs @@ -10,7 +10,7 @@ use super::stream::is_cached; use crate::{keyval, keyval::Key, stream}; #[implement(super::Map)] -pub fn keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send +pub fn keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send where K: Deserialize<'a> + Send, { diff --git a/src/database/map/keys_from.rs b/src/database/map/keys_from.rs index c9b1717a..11245f7b 100644 --- a/src/database/map/keys_from.rs +++ b/src/database/map/keys_from.rs @@ -15,7 +15,7 @@ use crate::{ pub fn keys_from<'a, K, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send + use<'a, K, P> +) -> impl Stream>> + Send + use<'a, K, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -40,7 +40,7 @@ where pub fn keys_raw_from<'a, K, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send + use<'a, K, P> +) -> impl Stream>> + Send + use<'a, K, P> where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, diff --git a/src/database/map/keys_prefix.rs b/src/database/map/keys_prefix.rs index 09dd79ac..e6a9457f 100644 --- a/src/database/map/keys_prefix.rs +++ b/src/database/map/keys_prefix.rs @@ -10,7 +10,7 @@ use crate::keyval::{Key, result_deserialize_key, serialize_key}; pub fn keys_prefix<'a, K, P>( self: &'a Arc, prefix: &P, -) -> impl Stream>> + Send + use<'a, K, P> +) -> impl Stream>> + Send + use<'a, K, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -37,7 +37,7 @@ where pub fn keys_raw_prefix<'a, K, P>( self: &'a Arc, prefix: &'a P, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, K: Deserialize<'a> + Send + 'a, @@ -50,7 +50,7 @@ where pub fn raw_keys_prefix<'a, P>( self: &'a Arc, prefix: &'a P, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/qry_batch.rs b/src/database/map/qry_batch.rs index e42d3e63..9da546e6 100644 --- a/src/database/map/qry_batch.rs +++ b/src/database/map/qry_batch.rs @@ -17,7 +17,7 @@ where S: Stream + Send + 'a, K: Serialize + Debug, { - fn qry(self, map: &'a Arc) -> impl Stream>> + Send + 'a; + fn qry(self, map: &'a Arc) -> impl Stream>> + Send + 'a; } impl<'a, K, S> Qry<'a, K, S> for S @@ -27,7 +27,7 @@ where K: Serialize + Debug + 'a, { #[inline] - fn qry(self, map: &'a Arc) -> impl Stream>> + Send + 'a { + fn qry(self, map: &'a Arc) -> impl Stream>> + Send + 'a { map.qry_batch(self) } } @@ -37,7 +37,7 @@ where pub(crate) fn qry_batch<'a, S, K>( self: &'a Arc, keys: S, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where S: Stream + Send + 'a, K: Serialize + Debug + 'a, diff --git a/src/database/map/rev_keys.rs b/src/database/map/rev_keys.rs index c00f3e55..8f48a17e 100644 --- a/src/database/map/rev_keys.rs +++ b/src/database/map/rev_keys.rs @@ -10,7 +10,7 @@ use super::rev_stream::is_cached; use crate::{keyval, keyval::Key, stream}; #[implement(super::Map)] -pub fn rev_keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send +pub fn rev_keys<'a, K>(self: &'a Arc) -> impl Stream>> + Send where K: Deserialize<'a> + Send, { diff --git a/src/database/map/rev_keys_from.rs b/src/database/map/rev_keys_from.rs index 04e457dc..021e3b92 100644 --- a/src/database/map/rev_keys_from.rs +++ b/src/database/map/rev_keys_from.rs @@ -15,7 +15,7 @@ use crate::{ pub fn rev_keys_from<'a, K, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send + use<'a, K, P> +) -> impl Stream>> + Send + use<'a, K, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -41,7 +41,7 @@ where pub fn rev_keys_raw_from<'a, K, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send + use<'a, K, P> +) -> impl Stream>> + Send + use<'a, K, P> where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, diff --git a/src/database/map/rev_keys_prefix.rs b/src/database/map/rev_keys_prefix.rs index fbe9f9ca..5b1459f5 100644 --- a/src/database/map/rev_keys_prefix.rs +++ b/src/database/map/rev_keys_prefix.rs @@ -10,7 +10,7 @@ use crate::keyval::{Key, result_deserialize_key, serialize_key}; pub fn rev_keys_prefix<'a, K, P>( self: &'a Arc, prefix: &P, -) -> impl Stream>> + Send + use<'a, K, P> +) -> impl Stream>> + Send + use<'a, K, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -37,7 +37,7 @@ where pub fn rev_keys_raw_prefix<'a, K, P>( self: &'a Arc, prefix: &'a P, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, K: Deserialize<'a> + Send + 'a, @@ -50,7 +50,7 @@ where pub fn rev_raw_keys_prefix<'a, P>( self: &'a Arc, prefix: &'a P, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/rev_stream.rs b/src/database/map/rev_stream.rs index 789a52e8..92d7bac8 100644 --- a/src/database/map/rev_stream.rs +++ b/src/database/map/rev_stream.rs @@ -14,7 +14,7 @@ use crate::{keyval, keyval::KeyVal, stream}; #[implement(super::Map)] pub fn rev_stream<'a, K, V>( self: &'a Arc, -) -> impl Stream>> + Send +) -> impl Stream>> + Send where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, diff --git a/src/database/map/rev_stream_from.rs b/src/database/map/rev_stream_from.rs index a612d2a2..7fef1b35 100644 --- a/src/database/map/rev_stream_from.rs +++ b/src/database/map/rev_stream_from.rs @@ -20,7 +20,7 @@ use crate::{ pub fn rev_stream_from<'a, K, V, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> +) -> impl Stream>> + Send + use<'a, K, V, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -55,7 +55,7 @@ where pub fn rev_stream_raw_from<'a, K, V, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> +) -> impl Stream>> + Send + use<'a, K, V, P> where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, diff --git a/src/database/map/rev_stream_prefix.rs b/src/database/map/rev_stream_prefix.rs index 46dc9247..70d4abf7 100644 --- a/src/database/map/rev_stream_prefix.rs +++ b/src/database/map/rev_stream_prefix.rs @@ -14,7 +14,7 @@ use crate::keyval::{KeyVal, result_deserialize, serialize_key}; pub fn rev_stream_prefix<'a, K, V, P>( self: &'a Arc, prefix: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> +) -> impl Stream>> + Send + use<'a, K, V, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -50,7 +50,7 @@ where pub fn rev_stream_raw_prefix<'a, K, V, P>( self: &'a Arc, prefix: &'a P, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, K: Deserialize<'a> + Send + 'a, @@ -68,7 +68,7 @@ where pub fn rev_raw_stream_prefix<'a, P>( self: &'a Arc, prefix: &'a P, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/map/stream.rs b/src/database/map/stream.rs index f7371b6c..736ab268 100644 --- a/src/database/map/stream.rs +++ b/src/database/map/stream.rs @@ -14,7 +14,7 @@ use crate::{keyval, keyval::KeyVal, stream}; #[implement(super::Map)] pub fn stream<'a, K, V>( self: &'a Arc, -) -> impl Stream>> + Send +) -> impl Stream>> + Send where K: Deserialize<'a> + Send, V: Deserialize<'a> + Send, diff --git a/src/database/map/stream_from.rs b/src/database/map/stream_from.rs index ccf48db6..9acec173 100644 --- a/src/database/map/stream_from.rs +++ b/src/database/map/stream_from.rs @@ -19,7 +19,7 @@ use crate::{ pub fn stream_from<'a, K, V, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> +) -> impl Stream>> + Send + use<'a, K, V, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -53,7 +53,7 @@ where pub fn stream_raw_from<'a, K, V, P>( self: &'a Arc, from: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> +) -> impl Stream>> + Send + use<'a, K, V, P> where P: AsRef<[u8]> + ?Sized + Debug + Sync, K: Deserialize<'a> + Send, diff --git a/src/database/map/stream_prefix.rs b/src/database/map/stream_prefix.rs index a26478aa..8210e152 100644 --- a/src/database/map/stream_prefix.rs +++ b/src/database/map/stream_prefix.rs @@ -14,7 +14,7 @@ use crate::keyval::{KeyVal, result_deserialize, serialize_key}; pub fn stream_prefix<'a, K, V, P>( self: &'a Arc, prefix: &P, -) -> impl Stream>> + Send + use<'a, K, V, P> +) -> impl Stream>> + Send + use<'a, K, V, P> where P: Serialize + ?Sized + Debug, K: Deserialize<'a> + Send, @@ -50,7 +50,7 @@ where pub fn stream_raw_prefix<'a, K, V, P>( self: &'a Arc, prefix: &'a P, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, K: Deserialize<'a> + Send + 'a, @@ -68,7 +68,7 @@ where pub fn raw_stream_prefix<'a, P>( self: &'a Arc, prefix: &'a P, -) -> impl Stream>> + Send + 'a +) -> impl Stream>> + Send + 'a where P: AsRef<[u8]> + ?Sized + Debug + Sync + 'a, { diff --git a/src/database/pool.rs b/src/database/pool.rs index 285aaf25..3421f779 100644 --- a/src/database/pool.rs +++ b/src/database/pool.rs @@ -443,7 +443,7 @@ pub(crate) fn into_send_seek(result: stream::State<'_>) -> stream::State<'static unsafe { std::mem::transmute(result) } } -fn into_recv_seek(result: stream::State<'static>) -> stream::State<'_> { +fn into_recv_seek(result: stream::State<'static>) -> stream::State<'static> { // SAFETY: This is to receive the State from the channel; see above. unsafe { std::mem::transmute(result) } } diff --git a/src/service/migrations.rs b/src/service/migrations.rs index cee638ba..586d6249 100644 --- a/src/service/migrations.rs +++ b/src/service/migrations.rs @@ -215,8 +215,8 @@ async fn db_lt_12(services: &Services) -> Result<()> { for username in &services .users .list_local_users() - .map(UserId::to_owned) - .collect::>() + .map(ToOwned::to_owned) + .collect::>() .await { let user = match UserId::parse_with_server_name(username.as_str(), &services.server.name) @@ -295,8 +295,8 @@ async fn db_lt_13(services: &Services) -> Result<()> { for username in &services .users .list_local_users() - .map(UserId::to_owned) - .collect::>() + .map(ToOwned::to_owned) + .collect::>() .await { let user = match UserId::parse_with_server_name(username.as_str(), &services.server.name) diff --git a/src/service/presence/mod.rs b/src/service/presence/mod.rs index 8f646be6..e7ce64bc 100644 --- a/src/service/presence/mod.rs +++ b/src/service/presence/mod.rs @@ -183,8 +183,8 @@ impl Service { .services .users .list_local_users() - .map(UserId::to_owned) - .collect::>() + .map(ToOwned::to_owned) + .collect::>() .await { let presence = self.db.get_presence(user_id).await; diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index baa7a72e..071bf822 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -178,7 +178,7 @@ impl Service { pub fn get_pushkeys<'a>( &'a self, sender: &'a UserId, - ) -> impl Stream + Send + 'a { + ) -> impl Stream + Send + 'a { let prefix = (sender, Interfix); self.db .senderkey_pusher diff --git a/src/service/rooms/alias/mod.rs b/src/service/rooms/alias/mod.rs index 7675efd4..c627092e 100644 --- a/src/service/rooms/alias/mod.rs +++ b/src/service/rooms/alias/mod.rs @@ -178,7 +178,7 @@ impl Service { pub fn local_aliases_for_room<'a>( &'a self, room_id: &'a RoomId, - ) -> impl Stream + Send + 'a { + ) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .aliasid_alias @@ -188,7 +188,9 @@ impl Service { } #[tracing::instrument(skip(self), level = "debug")] - pub fn all_local_aliases<'a>(&'a self) -> impl Stream + Send + 'a { + pub fn all_local_aliases<'a>( + &'a self, + ) -> impl Stream + Send + 'a { self.db .alias_roomid .stream() diff --git a/src/service/rooms/pdu_metadata/data.rs b/src/service/rooms/pdu_metadata/data.rs index a746b4cc..854c6ea0 100644 --- a/src/service/rooms/pdu_metadata/data.rs +++ b/src/service/rooms/pdu_metadata/data.rs @@ -60,7 +60,7 @@ impl Data { target: ShortEventId, from: PduCount, dir: Direction, - ) -> impl Stream + Send + '_ { + ) -> impl Stream + Send + 'a { // Query from exact position then filter excludes it (saturating_inc could skip // events at min/max boundaries) let from_unsigned = from.into_unsigned(); diff --git a/src/service/rooms/read_receipt/data.rs b/src/service/rooms/read_receipt/data.rs index 62f87948..9a2fa70c 100644 --- a/src/service/rooms/read_receipt/data.rs +++ b/src/service/rooms/read_receipt/data.rs @@ -65,7 +65,7 @@ impl Data { &'a self, room_id: &'a RoomId, since: u64, - ) -> impl Stream> + Send + 'a { + ) -> impl Stream> + Send + 'a { type Key<'a> = (&'a RoomId, u64, &'a UserId); type KeyVal<'a> = (Key<'a>, CanonicalJsonObject); diff --git a/src/service/rooms/read_receipt/mod.rs b/src/service/rooms/read_receipt/mod.rs index 68ce9b7f..64081a2c 100644 --- a/src/service/rooms/read_receipt/mod.rs +++ b/src/service/rooms/read_receipt/mod.rs @@ -112,7 +112,7 @@ impl Service { &'a self, room_id: &'a RoomId, since: u64, - ) -> impl Stream> + Send + 'a { + ) -> impl Stream> + Send + 'a { self.db.readreceipts_since(room_id, since) } diff --git a/src/service/rooms/search/mod.rs b/src/service/rooms/search/mod.rs index afe3061b..ea2f90af 100644 --- a/src/service/rooms/search/mod.rs +++ b/src/service/rooms/search/mod.rs @@ -104,7 +104,7 @@ pub fn deindex_pdu(&self, shortroomid: ShortRoomId, pdu_id: &RawPduId, message_b pub async fn search_pdus<'a>( &'a self, query: &'a RoomQuery<'a>, -) -> Result<(usize, impl Stream> + Send + '_)> { +) -> Result<(usize, impl Stream> + Send + 'a)> { let pdu_ids: Vec<_> = self.search_pdu_ids(query).await?.collect().await; let filter = &query.criteria.filter; @@ -137,10 +137,10 @@ pub async fn search_pdus<'a>( // result is modeled as a stream such that callers don't have to be refactored // though an additional async/wrap still exists for now #[implement(Service)] -pub async fn search_pdu_ids( - &self, - query: &RoomQuery<'_>, -) -> Result + Send + '_ + use<'_>> { +pub async fn search_pdu_ids<'a>( + &'a self, + query: &'a RoomQuery<'_>, +) -> Result + Send + 'a + use<'a>> { let shortroomid = self.services.short.get_shortroomid(query.room_id).await?; let pdu_ids = self.search_pdu_ids_query_room(query, shortroomid).await; @@ -173,7 +173,7 @@ fn search_pdu_ids_query_words<'a>( &'a self, shortroomid: ShortRoomId, word: &'a str, -) -> impl Stream + Send + '_ { +) -> impl Stream + Send + 'a { self.search_pdu_ids_query_word(shortroomid, word) .map(move |key| -> RawPduId { let key = &key[prefix_len(word)..]; @@ -183,11 +183,11 @@ fn search_pdu_ids_query_words<'a>( /// Iterate over raw database results for a word #[implement(Service)] -fn search_pdu_ids_query_word( - &self, +fn search_pdu_ids_query_word<'a>( + &'a self, shortroomid: ShortRoomId, - word: &str, -) -> impl Stream> + Send + '_ + use<'_> { + word: &'a str, +) -> impl Stream> + Send + 'a + use<'a> { // rustc says const'ing this not yet stable let end_id: RawPduId = PduId { shortroomid, diff --git a/src/service/rooms/short/mod.rs b/src/service/rooms/short/mod.rs index 06ff6493..660bb7de 100644 --- a/src/service/rooms/short/mod.rs +++ b/src/service/rooms/short/mod.rs @@ -62,7 +62,7 @@ pub async fn get_or_create_shorteventid(&self, event_id: &EventId) -> ShortEvent pub fn multi_get_or_create_shorteventid<'a, I>( &'a self, event_ids: I, -) -> impl Stream + Send + '_ +) -> impl Stream + Send + 'a where I: Iterator + Clone + Debug + Send + 'a, { diff --git a/src/service/rooms/state/mod.rs b/src/service/rooms/state/mod.rs index 641aa6a9..386adf9d 100644 --- a/src/service/rooms/state/mod.rs +++ b/src/service/rooms/state/mod.rs @@ -388,7 +388,7 @@ impl Service { pub fn get_forward_extremities<'a>( &'a self, room_id: &'a RoomId, - ) -> impl Stream + Send + '_ { + ) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db diff --git a/src/service/rooms/state_cache/mod.rs b/src/service/rooms/state_cache/mod.rs index e9845fbf..2d8f5cc5 100644 --- a/src/service/rooms/state_cache/mod.rs +++ b/src/service/rooms/state_cache/mod.rs @@ -144,7 +144,7 @@ pub fn clear_appservice_in_room_cache(&self) { self.appservice_in_room_cache.wri pub fn room_servers<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .roomserverids @@ -167,7 +167,7 @@ pub async fn server_in_room<'a>(&'a self, server: &'a ServerName, room_id: &'a R pub fn server_rooms<'a>( &'a self, server: &'a ServerName, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { let prefix = (server, Interfix); self.db .serverroomids @@ -202,7 +202,7 @@ pub fn get_shared_rooms<'a>( &'a self, user_a: &'a UserId, user_b: &'a UserId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { use conduwuit::utils::set; let a = self.rooms_joined(user_a); @@ -216,7 +216,7 @@ pub fn get_shared_rooms<'a>( pub fn room_members<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .roomuserid_joined @@ -239,7 +239,7 @@ pub async fn room_joined_count(&self, room_id: &RoomId) -> Result { pub fn local_users_in_room<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { self.room_members(room_id) .ready_filter(|user| self.services.globals.user_is_local(user)) } @@ -251,7 +251,7 @@ pub fn local_users_in_room<'a>( pub fn active_local_users_in_room<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { self.local_users_in_room(room_id) .filter(|user| self.services.users.is_active(user)) } @@ -273,7 +273,7 @@ pub async fn room_invited_count(&self, room_id: &RoomId) -> Result { pub fn room_useroncejoined<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .roomuseroncejoinedids @@ -288,7 +288,7 @@ pub fn room_useroncejoined<'a>( pub fn room_members_invited<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .roomuserid_invitecount @@ -303,7 +303,7 @@ pub fn room_members_invited<'a>( pub fn room_members_knocked<'a>( &'a self, room_id: &'a RoomId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { let prefix = (room_id, Interfix); self.db .roomuserid_knockedcount @@ -347,7 +347,7 @@ pub async fn get_left_count(&self, room_id: &RoomId, user_id: &UserId) -> Result pub fn rooms_joined<'a>( &'a self, user_id: &'a UserId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { self.db .userroomid_joined .keys_raw_prefix(user_id) diff --git a/src/service/rooms/state_cache/via.rs b/src/service/rooms/state_cache/via.rs index a818cc04..24d92a21 100644 --- a/src/service/rooms/state_cache/via.rs +++ b/src/service/rooms/state_cache/via.rs @@ -81,7 +81,7 @@ pub async fn servers_route_via(&self, room_id: &RoomId) -> Result( &'a self, room_id: &'a RoomId, -) -> impl Stream + Send + 'a { +) -> impl Stream + Send + 'a { type KeyVal<'a> = (Ignore, Vec<&'a ServerName>); self.db diff --git a/src/service/users/mod.rs b/src/service/users/mod.rs index fff1661c..6ddd8d79 100644 --- a/src/service/users/mod.rs +++ b/src/service/users/mod.rs @@ -422,7 +422,7 @@ impl Service { pub fn all_device_ids<'a>( &'a self, user_id: &'a UserId, - ) -> impl Stream + Send + 'a { + ) -> impl Stream + Send + 'a { let prefix = (user_id, Interfix); self.db .userdeviceid_metadata @@ -770,7 +770,7 @@ impl Service { user_id: &'a UserId, from: u64, to: Option, - ) -> impl Stream + Send + 'a { + ) -> impl Stream + Send + 'a { self.keys_changed_user_or_room(user_id.as_str(), from, to) .map(|(user_id, ..)| user_id) } @@ -781,7 +781,7 @@ impl Service { room_id: &'a RoomId, from: u64, to: Option, - ) -> impl Stream + Send + 'a { + ) -> impl Stream + Send + 'a { self.keys_changed_user_or_room(room_id.as_str(), from, to) } @@ -790,7 +790,7 @@ impl Service { user_or_room_id: &'a str, from: u64, to: Option, - ) -> impl Stream + Send + 'a { + ) -> impl Stream + Send + 'a { type KeyVal<'a> = ((&'a str, u64), &'a UserId); let to = to.unwrap_or(u64::MAX); From ddbca59193ceccfacb176cf09f44dc3b44294960 Mon Sep 17 00:00:00 2001 From: Ginger Date: Thu, 28 Aug 2025 16:18:14 -0400 Subject: [PATCH 2027/2091] Add spec and service files for creating an RPM package --- fedora/conduwuit.service | 68 +++++++++++++++++++++++++++++ fedora/continuwuity.spec.rpkg | 80 +++++++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 fedora/conduwuit.service create mode 100644 fedora/continuwuity.spec.rpkg diff --git a/fedora/conduwuit.service b/fedora/conduwuit.service new file mode 100644 index 00000000..6ab2af46 --- /dev/null +++ b/fedora/conduwuit.service @@ -0,0 +1,68 @@ +[Unit] +Description=Continuwuity - Matrix homeserver +Documentation=https://continuwuity.org/ +Wants=network-online.target +After=network-online.target +Alias=matrix-conduwuit.service + +[Service] +DynamicUser=yes +User=conduwuit +Group=conduwuit +Type=notify + +Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml" + +Environment="CONTINUWUITY_LOG_TO_JOURNALD=true" +Environment="CONTINUWUITY_JOURNALD_IDENTIFIER=%N" + +ExecStart=/usr/bin/conduwuit + +AmbientCapabilities= +CapabilityBoundingSet= + +DevicePolicy=closed +LockPersonality=yes +MemoryDenyWriteExecute=yes +NoNewPrivileges=yes +#ProcSubset=pid +ProtectClock=yes +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelLogs=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectProc=invisible +ProtectSystem=strict +PrivateDevices=yes +PrivateMounts=yes +PrivateTmp=yes +PrivateUsers=yes +PrivateIPC=yes +RemoveIPC=yes +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +SystemCallArchitectures=native +SystemCallFilter=@system-service @resources +SystemCallFilter=~@clock @debug @module @mount @reboot @swap @cpu-emulation @obsolete @timer @chown @setuid @privileged @keyring @ipc +SystemCallErrorNumber=EPERM + +StateDirectory=conduwuit +ConfigurationDirectory=conduwuit +RuntimeDirectory=conduwuit +RuntimeDirectoryMode=0750 + +Restart=on-failure +RestartSec=5 + +TimeoutStopSec=2m +TimeoutStartSec=2m + +StartLimitInterval=1m +StartLimitBurst=5 + +[Install] +WantedBy=multi-user.target diff --git a/fedora/continuwuity.spec.rpkg b/fedora/continuwuity.spec.rpkg new file mode 100644 index 00000000..7fca30fa --- /dev/null +++ b/fedora/continuwuity.spec.rpkg @@ -0,0 +1,80 @@ +# This should be run using rpkg-util: https://docs.pagure.org/rpkg-util +# it requires Internet access and is not suitable for Fedora main repos +# TODO: rpkg-util is no longer maintained, find a replacement + +Name: continuwuity +Version: {{{ git_repo_version }}} +Release: 1%{?dist} +Summary: Very cool Matrix chat homeserver written in Rust + +License: Apache-2.0 AND MIT + +URL: https://forgejo.ellis.link/continuwuation/%{name}/ +VCS: {{{ git_repo_vcs }}} +Source: {{{ git_repo_pack }}} + +BuildRequires: cargo-rpm-macros >= 25 +BuildRequires: systemd-rpm-macros +# Needed to build rust-librocksdb-sys +BuildRequires: clang +BuildRequires: liburing-devel + +Requires: liburing +Requires: glibc +Requires: libstdc++ + +%global _description %{expand: +A very cool Matrix chat homeserver written in Rust.} + +%description %{_description} + +%prep +{{{ git_repo_setup_macro }}} +%cargo_prep -N +# Perform an online build so Git dependencies can be retrieved +sed -i 's/^offline = true$//' .cargo/config.toml + +%build +%cargo_build + +# Here's the one legally required mystery incantation in this file. +# Some of our dependencies have source files which are (for some reason) marked as excutable. +# Files in .cargo/registry/ are copied into /usr/src/ by the debuginfo machinery +# at the end of the build step, and then the BRP shebang mangling script checks +# the entire buildroot to find executable files, and fails the build because +# it thinks Rust's file attributes are shebangs because they start with `#!`. +# So we have to clear the executable bit on all of them before that happens. +find .cargo/registry/ -executable -name "*.rs" -exec chmod -x {} + + +# TODO: this fails currently because it's forced to run in offline mode +# {cargo_license -- --no-dev} > LICENSE.dependencies + +%install +install -Dpm0755 target/rpm/conduwuit -t %{buildroot}%{_bindir} +install -Dpm0644 fedora/conduwuit.service -t %{buildroot}%{_unitdir} +install -Dpm0644 conduwuit-example.toml %{buildroot}%{_sysconfdir}/conduwuit/conduwuit.toml + +%files +%license LICENSE +%license src/core/matrix/state_res/LICENSE +%doc CODE_OF_CONDUCT.md +%doc CONTRIBUTING.md +%doc README.md +%doc SECURITY.md +%config %{_sysconfdir}/conduwuit/conduwuit.toml + +%{_bindir}/conduwuit +%{_unitdir}/conduwuit.service +# Do not create /var/lib/conduwuit, systemd will create it if necessary + +%post +%systemd_post conduwuit.service + +%preun +%systemd_preun conduwuit.service + +%postun +%systemd_postun_with_restart conduwuit.service + +%changelog +{{{ git_repo_changelog }}} \ No newline at end of file From f33f281edb12c2995f637be3cce1c876fea03f44 Mon Sep 17 00:00:00 2001 From: Ginger Date: Thu, 28 Aug 2025 16:20:34 -0400 Subject: [PATCH 2028/2091] Update long description to match deb package --- fedora/continuwuity.spec.rpkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora/continuwuity.spec.rpkg b/fedora/continuwuity.spec.rpkg index 7fca30fa..ef921ed4 100644 --- a/fedora/continuwuity.spec.rpkg +++ b/fedora/continuwuity.spec.rpkg @@ -24,7 +24,7 @@ Requires: glibc Requires: libstdc++ %global _description %{expand: -A very cool Matrix chat homeserver written in Rust.} +A cool hard fork of Conduit, a Matrix homeserver written in Rust} %description %{_description} From 34417c96ae746712a86205854cda4da9c396dea1 Mon Sep 17 00:00:00 2001 From: Ginger Date: Thu, 28 Aug 2025 16:21:35 -0400 Subject: [PATCH 2029/2091] Update URL to point at the landing page --- fedora/continuwuity.spec.rpkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora/continuwuity.spec.rpkg b/fedora/continuwuity.spec.rpkg index ef921ed4..19edf701 100644 --- a/fedora/continuwuity.spec.rpkg +++ b/fedora/continuwuity.spec.rpkg @@ -9,7 +9,7 @@ Summary: Very cool Matrix chat homeserver written in Rust License: Apache-2.0 AND MIT -URL: https://forgejo.ellis.link/continuwuation/%{name}/ +URL: https://continuwuity.org VCS: {{{ git_repo_vcs }}} Source: {{{ git_repo_pack }}} From 609e239436096309a940c238285512bbcf6cbae4 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sat, 30 Aug 2025 16:10:41 +0100 Subject: [PATCH 2030/2091] fix(fedora): Correct linting issues in RPM spec file The Fedora RPM packaging files added in PR #950 weren't passing pre-commit checks, causing CI failures for any branches rebased after that merge. This applies prek linting fixes (typo correction, trailing whitespace removal, and EOF newline) to ensure CI passes for all contributors. --- fedora/continuwuity.spec.rpkg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fedora/continuwuity.spec.rpkg b/fedora/continuwuity.spec.rpkg index 19edf701..f2efa383 100644 --- a/fedora/continuwuity.spec.rpkg +++ b/fedora/continuwuity.spec.rpkg @@ -38,7 +38,7 @@ sed -i 's/^offline = true$//' .cargo/config.toml %cargo_build # Here's the one legally required mystery incantation in this file. -# Some of our dependencies have source files which are (for some reason) marked as excutable. +# Some of our dependencies have source files which are (for some reason) marked as executable. # Files in .cargo/registry/ are copied into /usr/src/ by the debuginfo machinery # at the end of the build step, and then the BRP shebang mangling script checks # the entire buildroot to find executable files, and fails the build because @@ -46,7 +46,7 @@ sed -i 's/^offline = true$//' .cargo/config.toml # So we have to clear the executable bit on all of them before that happens. find .cargo/registry/ -executable -name "*.rs" -exec chmod -x {} + -# TODO: this fails currently because it's forced to run in offline mode +# TODO: this fails currently because it's forced to run in offline mode # {cargo_license -- --no-dev} > LICENSE.dependencies %install @@ -77,4 +77,4 @@ install -Dpm0644 conduwuit-example.toml %{buildroot}%{_sysconfdir}/conduwuit/con %systemd_postun_with_restart conduwuit.service %changelog -{{{ git_repo_changelog }}} \ No newline at end of file +{{{ git_repo_changelog }}} From 83e3de55a493eb0508a57c9a1a3f8fe700eb81b6 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 30 Aug 2025 16:00:46 +0100 Subject: [PATCH 2031/2091] fix(sync/v2): Room leaves being omitted incorrectly Partially borrowed from https://github.com/matrix-construct/tuwunel/commit/85a84f93c7ef7184a8eee1bb17116e5f0f0faf5a --- src/api/client/sync/v3.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/sync/v3.rs b/src/api/client/sync/v3.rs index 01428c08..298a6e4b 100644 --- a/src/api/client/sync/v3.rs +++ b/src/api/client/sync/v3.rs @@ -430,7 +430,7 @@ async fn handle_left_room( .ok(); // Left before last sync - if Some(since) >= left_count { + if (Some(since) >= left_count && !include_leave) || Some(next_batch) < left_count { return Ok(None); } From b934898f51a2d3fa80ece879ef5fc2774a7f5881 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 31 Aug 2025 00:25:41 +0100 Subject: [PATCH 2032/2091] chore: Update renovate config, limit cargo updates --- .editorconfig | 4 ++ renovate.json | 115 ++++++++++++++++++++++++-------------------------- 2 files changed, 60 insertions(+), 59 deletions(-) diff --git a/.editorconfig b/.editorconfig index 3e7fd1b8..95843e73 100644 --- a/.editorconfig +++ b/.editorconfig @@ -26,3 +26,7 @@ max_line_length = 98 [*.yml] indent_size = 2 indent_style = space + +[*.json] +indent_size = 4 +indent_style = space diff --git a/renovate.json b/renovate.json index deb428af..68d21b9d 100644 --- a/renovate.json +++ b/renovate.json @@ -1,62 +1,59 @@ { - "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": [ - "config:recommended" - ], - "lockFileMaintenance": { - "enabled": true, - "schedule": [ - "at any time" + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["config:recommended"], + "lockFileMaintenance": { + "enabled": true, + "schedule": ["at any time"] + }, + "nix": { + "enabled": true + }, + "labels": ["Dependencies", "Dependencies/Renovate"], + "ignoreDeps": [ + "tikv-jemallocator", + "tikv-jemalloc-sys", + "tikv-jemalloc-ctl", + "opentelemetry", + "opentelemetry_sdk", + "opentelemetry-jaeger", + "tracing-opentelemetry" + ], + "github-actions": { + "enabled": true, + "managerFilePatterns": [ + "/(^|/)\\.forgejo/workflows/[^/]+\\.ya?ml$/", + "/(^|/)\\.forgejo/actions/[^/]+/action\\.ya?ml$/", + "/(^|/)\\.github/workflows/[^/]+\\.ya?ml$/", + "/(^|/)\\.github/actions/[^/]+/action\\.ya?ml$/" + ] + }, + "packageRules": [ + { + "description": "Batch minor and patch GitHub Actions updates", + "matchManagers": ["github-actions"], + "matchUpdateTypes": ["minor", "patch"], + "groupName": "github-actions-non-major" + }, + { + "description": "Group Rust toolchain updates into a single PR", + "matchManagers": ["custom.regex"], + "matchPackageNames": ["rust", "rustc", "cargo"], + "groupName": "rust-toolchain" + }, + { + "description": "Group lockfile updates into a single PR", + "matchUpdateTypes": ["lockFileMaintenance"], + "groupName": "lockfile-maintenance" + }, + { + "description": "Batch patch-level Rust dependency updates", + "matchManagers": ["cargo"], + "matchUpdateTypes": ["patch"], + "groupName": "rust-patch-updates" + }, + { + "matchManagers": ["cargo"], + "prConcurrentLimit": 5 + } ] - }, - "nix": { - "enabled": true - }, - "labels": [ - "Dependencies", - "Dependencies/Renovate" - ], - "ignoreDeps": [ - "tikv-jemallocator", - "tikv-jemalloc-sys", - "tikv-jemalloc-ctl", - "opentelemetry", - "opentelemetry_sdk", - "opentelemetry-jaeger", - "tracing-opentelemetry" - ], - "github-actions": { - "enabled": true, - "fileMatch": [ - "(^|/)\\.forgejo/workflows/[^/]+\\.ya?ml$", - "(^|/)\\.forgejo/actions/[^/]+/action\\.ya?ml$", - "(^|/)\\.github/workflows/[^/]+\\.ya?ml$", - "(^|/)\\.github/actions/[^/]+/action\\.ya?ml$" - ] - }, - "packageRules": [ - { - "description": "Batch minor and patch GitHub Actions updates", - "matchManagers": ["github-actions"], - "matchUpdateTypes": ["minor", "patch"], - "groupName": "github-actions-non-major" - }, - { - "description": "Group Rust toolchain updates into a single PR", - "matchManagers": ["regex"], - "matchPackageNames": ["rust", "rustc", "cargo"], - "groupName": "rust-toolchain" - }, - { - "description": "Group lockfile updates into a single PR", - "matchUpdateTypes": ["lockFileMaintenance"], - "groupName": "lockfile-maintenance" - }, - { - "description": "Batch patch-level Rust dependency updates", - "matchManagers": ["cargo"], - "matchUpdateTypes": ["patch"], - "groupName": "rust-patch-updates" - } - ] } From e87c461b8d1d03d9e7b7b727f9a2fe8456c7338e Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 31 Aug 2025 00:38:47 +0100 Subject: [PATCH 2033/2091] feat: Cache renovate data, RO GitHub token --- .forgejo/workflows/renovate.yml | 69 ++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 10 deletions(-) diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml index e8522bec..0152b387 100644 --- a/.forgejo/workflows/renovate.yml +++ b/.forgejo/workflows/renovate.yml @@ -39,24 +39,73 @@ jobs: renovate: name: Renovate runs-on: ubuntu-latest + container: + image: ghcr.io/renovatebot/renovate:41 + options: --tmpfs /tmp:exec steps: - name: Checkout uses: actions/checkout@v4 + with: + show-progress: false + + - name: print node heap + run: /usr/local/renovate/node -e 'console.log(`node heap limit = ${require("v8").getHeapStatistics().heap_size_limit / (1024 * 1024)} Mb`)' + + - name: Restore renovate repo cache + uses: https://github.com/actions/cache@v4 + with: + path: | + /tmp/renovate/cache/renovate/repository + key: repo-cache-${{ github.run_id }} + restore-keys: | + repo-cache- + + - name: Restore renovate package cache + uses: https://github.com/actions/cache@v4 + with: + path: | + /tmp/renovate/cache/renovate/renovate-cache-sqlite + key: package-cache-${{ github.run_id }} + restore-keys: | + package-cache- - name: Self-hosted Renovate - uses: https://github.com/renovatebot/github-action@v40.1.0 + uses: https://github.com/renovatebot/github-action@v43.0.9 env: LOG_LEVEL: ${{ inputs.logLevel || 'info' }} - RENOVATE_AUTODISCOVER: 'false' - RENOVATE_BINARY_SOURCE: 'install' RENOVATE_DRY_RUN: ${{ inputs.dryRun || 'false' }} - RENOVATE_ENDPOINT: ${{ github.server_url }}/api/v1 - RENOVATE_GIT_TIMEOUT: 60000 - RENOVATE_GIT_URL: 'endpoint' - RENOVATE_GITHUB_TOKEN_WARN: 'false' - RENOVATE_ONBOARDING: 'false' - RENOVATE_PLATFORM: 'forgejo' - RENOVATE_PR_COMMITS_PER_RUN_LIMIT: 3 + + RENOVATE_PLATFORM: forgejo + RENOVATE_ENDPOINT: ${{ github.server_url }} + RENOVATE_AUTODISCOVER: 'false' RENOVATE_REPOSITORIES: '["${{ github.repository }}"]' + + RENOVATE_GIT_TIMEOUT: 60000 + RENOVATE_REQUIRE_CONFIG: 'required' + RENOVATE_ONBOARDING: 'false' + + RENOVATE_PR_COMMITS_PER_RUN_LIMIT: 3 + + RENOVATE_GITHUB_TOKEN_WARN: 'false' RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }} + GITHUB_COM_TOKEN: ${{ secrets.GH_PUBLIC_RO }} + + RENOVATE_REPOSITORY_CACHE: 'enabled' + RENOVATE_X_SQLITE_PACKAGE_CACHE: true + + - name: Save renovate repo cache + if: always() && env.RENOVATE_DRY_RUN != 'full' + uses: https://github.com/actions/cache@v4 + with: + path: | + /tmp/renovate/cache/renovate/repository + key: repo-cache-${{ github.run_id }} + + - name: Save renovate package cache + if: always() && env.RENOVATE_DRY_RUN != 'full' + uses: https://github.com/actions/cache@v4 + with: + path: | + /tmp/renovate/cache/renovate/renovate-cache-sqlite + key: package-cache-${{ github.run_id }} From 5cce02484146e62b34e7228f603cb42559c76eff Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sun, 31 Aug 2025 00:44:28 +0000 Subject: [PATCH 2034/2091] chore(deps): update https://github.com/reproducible-containers/buildkit-cache-dance action to v3.3.0 --- .forgejo/workflows/release-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 04fc9de9..58d6cab2 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -161,7 +161,7 @@ jobs: var-lib-apt-${{ matrix.slug }} key: var-lib-apt-${{ matrix.slug }} - name: inject cache into docker - uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.1.0 + uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.3.0 with: cache-map: | { From 1e430f9470f37f8e59e010ac083d140b45e63367 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 31 Aug 2025 16:55:47 +0100 Subject: [PATCH 2035/2091] feat(MSC4323): Implement agnostic suspension endpoint --- Cargo.lock | 22 +++++----- Cargo.toml | 2 +- src/api/client/admin/mod.rs | 3 ++ src/api/client/admin/suspend.rs | 77 +++++++++++++++++++++++++++++++++ src/api/client/mod.rs | 2 + src/api/router.rs | 2 + 6 files changed, 96 insertions(+), 12 deletions(-) create mode 100644 src/api/client/admin/mod.rs create mode 100644 src/api/client/admin/suspend.rs diff --git a/Cargo.lock b/Cargo.lock index 2b044a1f..a3962c31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4058,7 +4058,7 @@ checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3" [[package]] name = "ruma" version = "0.10.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "assign", "js_int", @@ -4078,7 +4078,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "js_int", "ruma-common", @@ -4090,7 +4090,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "as_variant", "assign", @@ -4113,7 +4113,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "as_variant", "base64 0.22.1", @@ -4145,7 +4145,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "as_variant", "indexmap 2.10.0", @@ -4170,7 +4170,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "bytes", "headers", @@ -4192,7 +4192,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "js_int", "thiserror 2.0.12", @@ -4201,7 +4201,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "js_int", "ruma-common", @@ -4211,7 +4211,7 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "cfg-if", "proc-macro-crate", @@ -4226,7 +4226,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "js_int", "ruma-common", @@ -4238,7 +4238,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=b753738047d1f443aca870896ef27ecaacf027da#b753738047d1f443aca870896ef27ecaacf027da" +source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "base64 0.22.1", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 9452066c..e2af2d94 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -352,7 +352,7 @@ version = "0.1.2" [workspace.dependencies.ruma] git = "https://forgejo.ellis.link/continuwuation/ruwuma" #branch = "conduwuit-changes" -rev = "b753738047d1f443aca870896ef27ecaacf027da" +rev = "8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" features = [ "compat", "rand", diff --git a/src/api/client/admin/mod.rs b/src/api/client/admin/mod.rs new file mode 100644 index 00000000..8355a600 --- /dev/null +++ b/src/api/client/admin/mod.rs @@ -0,0 +1,3 @@ +mod suspend; + +pub(crate) use self::suspend::*; diff --git a/src/api/client/admin/suspend.rs b/src/api/client/admin/suspend.rs new file mode 100644 index 00000000..29682694 --- /dev/null +++ b/src/api/client/admin/suspend.rs @@ -0,0 +1,77 @@ +use axum::extract::State; +use conduwuit::{Err, Result}; +use ruma::api::client::admin::{get_suspended, set_suspended}; + +use crate::Ruma; + +/// # `GET /_matrix/client/v1/admin/suspend/{userId}` +/// +/// Check the suspension status of a target user +pub(crate) async fn get_suspended_status( + State(services): State, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + if !services.users.is_admin(sender_user).await { + return Err!(Request(Forbidden("Only server administrators can use this endpoint"))); + }; + if !services.globals.user_is_local(&body.user_id) { + return Err!(Request(InvalidParam("Can only check the suspended status of local users"))); + }; + if !services.users.is_active(&body.user_id).await { + return Err!(Request(NotFound("Unknown user"))); + } + Ok(get_suspended::v1::Response::new( + services.users.is_suspended(&body.user_id).await?, + )) +} + +/// # `PUT /_matrix/client/v1/admin/suspend/{userId}` +/// +/// Set the suspension status of a target user +pub(crate) async fn put_suspended_status( + State(services): State, + body: Ruma, +) -> Result { + let sender_user = body.sender_user(); + if !services.users.is_admin(sender_user).await { + return Err!(Request(Forbidden("Only server administrators can use this endpoint"))); + }; + if !services.globals.user_is_local(&body.user_id) { + return Err!(Request(InvalidParam("Can only set the suspended status of local users"))); + }; + if !services.users.is_active(&body.user_id).await { + return Err!(Request(NotFound("Unknown user"))); + } + if body.user_id == *sender_user { + return Err!(Request(Forbidden("You cannot suspend yourself"))); + } + if services.users.is_admin(&body.user_id).await { + return Err!(Request(Forbidden("You cannot suspend another admin"))); + } + if services.users.is_suspended(&body.user_id).await? == body.suspended { + // No change + return Ok(set_suspended::v1::Response::new(body.suspended)); + } + + let action = if body.suspended { + services + .users + .suspend_account(&body.user_id, sender_user) + .await; + "suspended" + } else { + services.users.unsuspend_account(&body.user_id).await; + "unsuspended" + }; + + if services.config.admin_room_notices { + // Notify the admin room that an account has been un/suspended + services + .admin + .send_text(&format!("{} has been {} by {}.", body.user_id, action, sender_user)) + .await; + } + + Ok(set_suspended::v1::Response::new(body.suspended)) +} diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index e4be20b7..e53b26d9 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -1,5 +1,6 @@ pub(super) mod account; pub(super) mod account_data; +pub(super) mod admin; pub(super) mod alias; pub(super) mod appservice; pub(super) mod backup; @@ -43,6 +44,7 @@ pub(super) mod well_known; pub use account::full_user_deactivate; pub(super) use account::*; pub(super) use account_data::*; +pub(super) use admin::*; pub(super) use alias::*; pub(super) use appservice::*; pub(super) use backup::*; diff --git a/src/api/router.rs b/src/api/router.rs index 8072fa5b..42934f70 100644 --- a/src/api/router.rs +++ b/src/api/router.rs @@ -184,6 +184,8 @@ pub fn build(router: Router, server: &Server) -> Router { "/_matrix/client/unstable/im.nheko.summary/rooms/:room_id_or_alias/summary", get(client::get_room_summary_legacy) ) + .ruma_route(&client::get_suspended_status) + .ruma_route(&client::put_suspended_status) .ruma_route(&client::well_known_support) .ruma_route(&client::well_known_client) .route("/_conduwuit/server_version", get(client::conduwuit_server_version)) From 9783940105b070346204b99b0cfc6a23558c63b8 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 31 Aug 2025 17:07:19 +0100 Subject: [PATCH 2036/2091] feat(MSC4323): Advertise suspension support in capabilities --- src/api/client/capabilities.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index c42c6dfd..2f9d6299 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -23,6 +23,11 @@ pub(crate) async fn get_capabilities_route( ) -> Result { let available: BTreeMap = Server::available_room_versions().collect(); + let authenticated = _body.sender_user.as_ref().is_some() + && services + .users + .is_active_local(_body.sender_user.as_ref().unwrap()) + .await; let mut capabilities = Capabilities::default(); capabilities.room_versions = RoomVersionsCapability { @@ -45,5 +50,15 @@ pub(crate) async fn get_capabilities_route( json!({"enabled": services.config.forget_forced_upon_leave}), )?; + if authenticated + && services + .users + .is_admin(_body.sender_user.as_ref().unwrap()) + .await + { + // Advertise suspension API + capabilities.set("uk.timedout.msc4323", json!({"suspend":true, "lock": false}))?; + } + Ok(get_capabilities::v3::Response { capabilities }) } From 04e796176a7dd2bf33f04d49bcadbb54a3feb501 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 31 Aug 2025 17:10:56 +0100 Subject: [PATCH 2037/2091] style(MSC4323): Satisfy our linting overlords --- src/api/client/admin/suspend.rs | 8 ++++---- src/api/client/capabilities.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/api/client/admin/suspend.rs b/src/api/client/admin/suspend.rs index 29682694..0d3493f6 100644 --- a/src/api/client/admin/suspend.rs +++ b/src/api/client/admin/suspend.rs @@ -14,10 +14,10 @@ pub(crate) async fn get_suspended_status( let sender_user = body.sender_user(); if !services.users.is_admin(sender_user).await { return Err!(Request(Forbidden("Only server administrators can use this endpoint"))); - }; + } if !services.globals.user_is_local(&body.user_id) { return Err!(Request(InvalidParam("Can only check the suspended status of local users"))); - }; + } if !services.users.is_active(&body.user_id).await { return Err!(Request(NotFound("Unknown user"))); } @@ -36,10 +36,10 @@ pub(crate) async fn put_suspended_status( let sender_user = body.sender_user(); if !services.users.is_admin(sender_user).await { return Err!(Request(Forbidden("Only server administrators can use this endpoint"))); - }; + } if !services.globals.user_is_local(&body.user_id) { return Err!(Request(InvalidParam("Can only set the suspended status of local users"))); - }; + } if !services.users.is_active(&body.user_id).await { return Err!(Request(NotFound("Unknown user"))); } diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 2f9d6299..1fcb073f 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -19,14 +19,14 @@ use crate::Ruma; /// of this server. pub(crate) async fn get_capabilities_route( State(services): State, - _body: Ruma, + body: Ruma, ) -> Result { let available: BTreeMap = Server::available_room_versions().collect(); - let authenticated = _body.sender_user.as_ref().is_some() + let authenticated = body.sender_user.as_ref().is_some() && services .users - .is_active_local(_body.sender_user.as_ref().unwrap()) + .is_active_local(body.sender_user.as_ref().unwrap()) .await; let mut capabilities = Capabilities::default(); @@ -53,7 +53,7 @@ pub(crate) async fn get_capabilities_route( if authenticated && services .users - .is_admin(_body.sender_user.as_ref().unwrap()) + .is_admin(body.sender_user.as_ref().unwrap()) .await { // Advertise suspension API From 35cf9af5c8a62dd362c9ecf4980d88291a2e9994 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 31 Aug 2025 17:44:32 +0100 Subject: [PATCH 2038/2091] feat(MSC4323): Add versions flag --- src/api/client/unversioned.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/client/unversioned.rs b/src/api/client/unversioned.rs index a4136d1a..7f19bc94 100644 --- a/src/api/client/unversioned.rs +++ b/src/api/client/unversioned.rs @@ -58,6 +58,7 @@ pub(crate) async fn get_supported_versions_route( ("uk.tcpip.msc4133".to_owned(), true), /* Extending User Profile API with Key:Value Pairs (https://github.com/matrix-org/matrix-spec-proposals/pull/4133) */ ("us.cloke.msc4175".to_owned(), true), /* Profile field for user time zone (https://github.com/matrix-org/matrix-spec-proposals/pull/4175) */ ("org.matrix.simplified_msc3575".to_owned(), true), /* Simplified Sliding sync (https://github.com/matrix-org/matrix-spec-proposals/pull/4186) */ + ("uk.timedout.msc4323".to_owned(), true), /* agnostic suspend (https://github.com/matrix-org/matrix-spec-proposals/pull/4323) */ ]), }; From 4e644961f3f7c3a955618caff3b3927b8f7e8931 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 31 Aug 2025 18:32:32 +0100 Subject: [PATCH 2039/2091] perf(MSC4323): Remove redundant authorisation checks --- src/api/client/capabilities.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/api/client/capabilities.rs b/src/api/client/capabilities.rs index 1fcb073f..afa61498 100644 --- a/src/api/client/capabilities.rs +++ b/src/api/client/capabilities.rs @@ -23,11 +23,6 @@ pub(crate) async fn get_capabilities_route( ) -> Result { let available: BTreeMap = Server::available_room_versions().collect(); - let authenticated = body.sender_user.as_ref().is_some() - && services - .users - .is_active_local(body.sender_user.as_ref().unwrap()) - .await; let mut capabilities = Capabilities::default(); capabilities.room_versions = RoomVersionsCapability { @@ -50,11 +45,10 @@ pub(crate) async fn get_capabilities_route( json!({"enabled": services.config.forget_forced_upon_leave}), )?; - if authenticated - && services - .users - .is_admin(body.sender_user.as_ref().unwrap()) - .await + if services + .users + .is_admin(body.sender_user.as_ref().unwrap()) + .await { // Advertise suspension API capabilities.set("uk.timedout.msc4323", json!({"suspend":true, "lock": false}))?; From d970df5fd239239554192887c5c0a634935ee3df Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Mon, 1 Sep 2025 11:36:39 +0100 Subject: [PATCH 2040/2091] perf(MSC4323): Parallelise some check futs --- src/api/client/admin/suspend.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/api/client/admin/suspend.rs b/src/api/client/admin/suspend.rs index 0d3493f6..bab1cb5a 100644 --- a/src/api/client/admin/suspend.rs +++ b/src/api/client/admin/suspend.rs @@ -1,5 +1,6 @@ use axum::extract::State; use conduwuit::{Err, Result}; +use futures::future::{join, join3}; use ruma::api::client::admin::{get_suspended, set_suspended}; use crate::Ruma; @@ -12,13 +13,16 @@ pub(crate) async fn get_suspended_status( body: Ruma, ) -> Result { let sender_user = body.sender_user(); - if !services.users.is_admin(sender_user).await { + + let (admin, active) = + join(services.users.is_admin(sender_user), services.users.is_active(&body.user_id)).await; + if !admin { return Err!(Request(Forbidden("Only server administrators can use this endpoint"))); } if !services.globals.user_is_local(&body.user_id) { return Err!(Request(InvalidParam("Can only check the suspended status of local users"))); } - if !services.users.is_active(&body.user_id).await { + if !active { return Err!(Request(NotFound("Unknown user"))); } Ok(get_suspended::v1::Response::new( @@ -34,20 +38,28 @@ pub(crate) async fn put_suspended_status( body: Ruma, ) -> Result { let sender_user = body.sender_user(); - if !services.users.is_admin(sender_user).await { + + let (sender_admin, active, target_admin) = join3( + services.users.is_admin(sender_user), + services.users.is_active(&body.user_id), + services.users.is_admin(&body.user_id), + ) + .await; + + if !sender_admin { return Err!(Request(Forbidden("Only server administrators can use this endpoint"))); } if !services.globals.user_is_local(&body.user_id) { return Err!(Request(InvalidParam("Can only set the suspended status of local users"))); } - if !services.users.is_active(&body.user_id).await { + if !active { return Err!(Request(NotFound("Unknown user"))); } if body.user_id == *sender_user { return Err!(Request(Forbidden("You cannot suspend yourself"))); } - if services.users.is_admin(&body.user_id).await { - return Err!(Request(Forbidden("You cannot suspend another admin"))); + if target_admin { + return Err!(Request(Forbidden("You cannot suspend another server administrator"))); } if services.users.is_suspended(&body.user_id).await? == body.suspended { // No change From 241371463e108b036b596c23d4658439016e69e1 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 30 Jul 2025 19:19:32 +0100 Subject: [PATCH 2041/2091] feat: Force leave remote rooms admin command --- src/admin/user/commands.rs | 30 ++++++++++++++++++++++++++++-- src/admin/user/mod.rs | 6 ++++++ src/api/client/membership/leave.rs | 2 +- src/api/client/membership/mod.rs | 2 +- 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/admin/user/commands.rs b/src/admin/user/commands.rs index 56864a32..37ab030c 100644 --- a/src/admin/user/commands.rs +++ b/src/admin/user/commands.rs @@ -1,8 +1,8 @@ use std::{collections::BTreeMap, fmt::Write as _}; use api::client::{ - full_user_deactivate, join_room_by_id_helper, leave_all_rooms, leave_room, update_avatar_url, - update_displayname, + full_user_deactivate, join_room_by_id_helper, leave_all_rooms, leave_room, remote_leave_room, + update_avatar_url, update_displayname, }; use conduwuit::{ Err, Result, debug, debug_warn, error, info, is_equal_to, @@ -926,3 +926,29 @@ pub(super) async fn redact_event(&self, event_id: OwnedEventId) -> Result { )) .await } + +#[admin_command] +pub(super) async fn force_leave_remote_room( + &self, + user_id: String, + room_id: OwnedRoomOrAliasId, +) -> Result { + let user_id = parse_local_user_id(self.services, &user_id)?; + let (room_id, _) = self + .services + .rooms + .alias + .resolve_with_servers(&room_id, None) + .await?; + + assert!( + self.services.globals.user_is_local(&user_id), + "Parsed user_id must be a local user" + ); + remote_leave_room(self.services, &user_id, &room_id, None) + .boxed() + .await?; + + self.write_str(&format!("{user_id} has been joined to {room_id}.",)) + .await +} diff --git a/src/admin/user/mod.rs b/src/admin/user/mod.rs index 656cacaf..366f7dd5 100644 --- a/src/admin/user/mod.rs +++ b/src/admin/user/mod.rs @@ -103,6 +103,12 @@ pub enum UserCommand { room_id: OwnedRoomOrAliasId, }, + /// - Manually leave a remote room for a local user. + ForceLeaveRemoteRoom { + user_id: String, + room_id: OwnedRoomOrAliasId, + }, + /// - Forces the specified user to drop their power levels to the room /// default, if their permissions allow and the auth check permits ForceDemote { diff --git a/src/api/client/membership/leave.rs b/src/api/client/membership/leave.rs index f4f1666b..0aadd833 100644 --- a/src/api/client/membership/leave.rs +++ b/src/api/client/membership/leave.rs @@ -215,7 +215,7 @@ pub async fn leave_room( Ok(()) } -async fn remote_leave_room( +pub async fn remote_leave_room( services: &Services, user_id: &UserId, room_id: &RoomId, diff --git a/src/api/client/membership/mod.rs b/src/api/client/membership/mod.rs index 7a6f19ad..691419f6 100644 --- a/src/api/client/membership/mod.rs +++ b/src/api/client/membership/mod.rs @@ -29,7 +29,7 @@ pub(crate) use self::{ }; pub use self::{ join::join_room_by_id_helper, - leave::{leave_all_rooms, leave_room}, + leave::{leave_all_rooms, leave_room, remote_leave_room}, }; use crate::{Ruma, client::full_user_deactivate}; From 66d479e2eb3e25f20ac74860e82c9c33ad069c21 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 30 Jul 2025 19:29:33 +0100 Subject: [PATCH 2042/2091] fix: Make remote leave helper a public fn --- src/api/client/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index e53b26d9..c8ca7757 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -57,7 +57,7 @@ pub(super) use keys::*; pub(super) use media::*; pub(super) use media_legacy::*; pub(super) use membership::*; -pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room}; +pub use membership::{join_room_by_id_helper, leave_all_rooms, leave_room, remote_leave_room}; pub(super) use message::*; pub(super) use openid::*; pub(super) use presence::*; From 76b93e252dc3b4c380772ad4eaad0463b458600e Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Thu, 31 Jul 2025 17:48:30 +0100 Subject: [PATCH 2043/2091] feat: Only inject vias when manual ones aren't provided during join --- src/api/client/membership/join.rs | 49 ++++++++++++++++--------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/api/client/membership/join.rs b/src/api/client/membership/join.rs index dc170cbf..f3434bf5 100644 --- a/src/api/client/membership/join.rs +++ b/src/api/client/membership/join.rs @@ -156,31 +156,34 @@ pub(crate) async fn join_room_by_id_or_alias_route( .await?; let mut servers = body.via.clone(); - servers.extend( - services - .rooms - .state_cache - .servers_invite_via(&room_id) - .map(ToOwned::to_owned) - .collect::>() - .await, - ); + if servers.is_empty() { + debug!("No via servers provided for join, injecting some."); + servers.extend( + services + .rooms + .state_cache + .servers_invite_via(&room_id) + .map(ToOwned::to_owned) + .collect::>() + .await, + ); - servers.extend( - services - .rooms - .state_cache - .invite_state(sender_user, &room_id) - .await - .unwrap_or_default() - .iter() - .filter_map(|event| event.get_field("sender").ok().flatten()) - .filter_map(|sender: &str| UserId::parse(sender).ok()) - .map(|user| user.server_name().to_owned()), - ); + servers.extend( + services + .rooms + .state_cache + .invite_state(sender_user, &room_id) + .await + .unwrap_or_default() + .iter() + .filter_map(|event| event.get_field("sender").ok().flatten()) + .filter_map(|sender: &str| UserId::parse(sender).ok()) + .map(|user| user.server_name().to_owned()), + ); - if let Some(server) = room_id.server_name() { - servers.push(server.to_owned()); + if let Some(server) = room_id.server_name() { + servers.push(server.to_owned()); + } } servers.sort_unstable(); From 9e62e66ae4a46f3e742644a86c9c699eb5625f47 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sat, 30 Aug 2025 21:31:23 +0100 Subject: [PATCH 2044/2091] chore(PR956): Update admin docs --- docs/admin_reference.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/admin_reference.md b/docs/admin_reference.md index 18e039e4..1c9d4fb0 100644 --- a/docs/admin_reference.md +++ b/docs/admin_reference.md @@ -21,6 +21,7 @@ This document contains the help content for the `admin` command-line program. * [`admin users list-joined-rooms`↴](#admin-users-list-joined-rooms) * [`admin users force-join-room`↴](#admin-users-force-join-room) * [`admin users force-leave-room`↴](#admin-users-force-leave-room) +* [`admin users force-leave-remote-room`↴](#admin-users-force-leave-remote-room) * [`admin users force-demote`↴](#admin-users-force-demote) * [`admin users make-user-admin`↴](#admin-users-make-user-admin) * [`admin users put-room-tag`↴](#admin-users-put-room-tag) @@ -295,6 +296,7 @@ You can find the ID using the `list-appservices` command. * `list-joined-rooms` — - Lists all the rooms (local and remote) that the specified user is joined in * `force-join-room` — - Manually join a local user to a room * `force-leave-room` — - Manually leave a local user from a room +* `force-leave-remote-room` — - Manually leave a remote room for a local user * `force-demote` — - Forces the specified user to drop their power levels to the room default, if their permissions allow and the auth check permits * `make-user-admin` — - Grant server-admin privileges to a user * `put-room-tag` — - Puts a room tag for the specified user and room ID @@ -449,6 +451,19 @@ Reverses the effects of the `suspend` command, allowing the user to send message +## `admin users force-leave-remote-room` + +- Manually leave a remote room for a local user + +**Usage:** `admin users force-leave-remote-room ` + +###### **Arguments:** + +* `` +* `` + + + ## `admin users force-demote` - Forces the specified user to drop their power levels to the room default, if their permissions allow and the auth check permits From 95aeff8cdcd35cf5bcfdd08c7c28618c6d579f02 Mon Sep 17 00:00:00 2001 From: Ginger Date: Fri, 29 Aug 2025 12:16:21 -0400 Subject: [PATCH 2045/2091] Set the DB path as an env var in systemd service files to prevent footgunning --- arch/conduwuit.service | 1 + debian/conduwuit.service | 1 + fedora/conduwuit.service | 1 + src/core/config/mod.rs | 6 ++++-- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/arch/conduwuit.service b/arch/conduwuit.service index 34c3995e..18c34f33 100644 --- a/arch/conduwuit.service +++ b/arch/conduwuit.service @@ -20,6 +20,7 @@ StandardError=journal+console Environment="CONTINUWUITY_LOG_TO_JOURNALD=true" Environment="CONTINUWUITY_JOURNALD_IDENTIFIER=%N" +Environment="CONTINUWUITY_DATABASE_PATH=/var/lib/conduwuit" TTYReset=yes # uncomment to allow buffer to be cleared every restart diff --git a/debian/conduwuit.service b/debian/conduwuit.service index da78f09f..ec2505b5 100644 --- a/debian/conduwuit.service +++ b/debian/conduwuit.service @@ -16,6 +16,7 @@ Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml" Environment="CONTINUWUITY_LOG_TO_JOURNALD=true" Environment="CONTINUWUITY_JOURNALD_IDENTIFIER=%N" +Environment="CONTINUWUITY_DATABASE_PATH=/var/lib/conduwuit" ExecStart=/usr/sbin/conduwuit diff --git a/fedora/conduwuit.service b/fedora/conduwuit.service index 6ab2af46..f37c7798 100644 --- a/fedora/conduwuit.service +++ b/fedora/conduwuit.service @@ -15,6 +15,7 @@ Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml" Environment="CONTINUWUITY_LOG_TO_JOURNALD=true" Environment="CONTINUWUITY_JOURNALD_IDENTIFIER=%N" +Environment="CONTINUWUITY_DATABASE_PATH=/var/lib/conduwuit" ExecStart=/usr/bin/conduwuit diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index e8518ed4..58a39a75 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -126,9 +126,11 @@ pub struct Config { /// This is the only directory where continuwuity will save its data, /// including media. Note: this was previously "/var/lib/matrix-conduit". /// - /// YOU NEED TO EDIT THIS. + /// YOU NEED TO EDIT THIS, UNLESS you are running continuwuity as a `systemd` service. + /// The service file sets it to `/var/lib/conduwuit` using an environment variable + /// and also grants write access. /// - /// example: "/var/lib/continuwuity" + /// example: "/var/lib/conduwuit" pub database_path: PathBuf, /// continuwuity supports online database backups using RocksDB's Backup From 99b44bbf0946b1ad426ed0bf719674300576721c Mon Sep 17 00:00:00 2001 From: Ginger Date: Fri, 29 Aug 2025 12:30:48 -0400 Subject: [PATCH 2046/2091] Update conduwuit-example.toml --- conduwuit-example.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index f0e510b4..fa65cbf2 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -79,9 +79,11 @@ # This is the only directory where continuwuity will save its data, # including media. Note: this was previously "/var/lib/matrix-conduit". # -# YOU NEED TO EDIT THIS. +# YOU NEED TO EDIT THIS, UNLESS you are running continuwuity as a `systemd` service. +# The service file sets it to `/var/lib/conduwuit` using an environment variable +# and also grants write access. # -# example: "/var/lib/continuwuity" +# example: "/var/lib/conduwuit" # #database_path = From 467aed3028705ec89adfa8b2b58e746323595eed Mon Sep 17 00:00:00 2001 From: nex Date: Tue, 2 Sep 2025 16:36:56 +0000 Subject: [PATCH 2047/2091] chore: Add Ginger's GH noreply email to mailmap --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index fa267e13..7c25737d 100644 --- a/.mailmap +++ b/.mailmap @@ -13,3 +13,4 @@ Rudi Floren Tamara Schmitz <15906939+tamara-schmitz@users.noreply.github.com> Timo Kösters x4u <14617923-x4u@users.noreply.gitlab.com> +Ginger <75683114+gingershaped@users.noreply.github.com> From d19e0f0d97289a9e4d3c08c956ccfa2257b07bf8 Mon Sep 17 00:00:00 2001 From: Ginger Date: Tue, 2 Sep 2025 09:35:15 -0400 Subject: [PATCH 2048/2091] feat: Move packaging scripts into `dist/` and consolidate the service files --- .forgejo/workflows/release-image.yml | 2 +- arch/conduwuit.service | 84 ------------------- debian/conduwuit.service | 71 ---------------- {fedora => dist}/conduwuit.service | 7 +- {debian => dist/debian}/README.md | 0 {debian => dist/debian}/config | 0 {debian => dist/debian}/postinst | 0 {debian => dist/debian}/postrm | 0 .../fedora}/continuwuity.spec.rpkg | 2 +- .../nix}/pkgs/main/cross-compilation-env.nix | 0 {nix => dist/nix}/pkgs/main/default.nix | 0 docs/configuration/examples.md | 19 +---- docs/deploying/debian.md | 2 +- flake.nix | 2 +- src/core/config/mod.rs | 6 +- src/main/Cargo.toml | 4 +- 16 files changed, 16 insertions(+), 183 deletions(-) delete mode 100644 arch/conduwuit.service delete mode 100644 debian/conduwuit.service rename {fedora => dist}/conduwuit.service (95%) rename {debian => dist/debian}/README.md (100%) rename {debian => dist/debian}/config (100%) rename {debian => dist/debian}/postinst (100%) rename {debian => dist/debian}/postrm (100%) rename {fedora => dist/fedora}/continuwuity.spec.rpkg (97%) rename {nix => dist/nix}/pkgs/main/cross-compilation-env.nix (100%) rename {nix => dist/nix}/pkgs/main/default.nix (100%) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 58d6cab2..2055d91a 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -10,7 +10,7 @@ on: - ".gitlab-ci.yml" - ".gitignore" - "renovate.json" - - "debian/**" + - "dist/**" - "docker/**" - "docs/**" # Allows you to run this workflow manually from the Actions tab diff --git a/arch/conduwuit.service b/arch/conduwuit.service deleted file mode 100644 index 18c34f33..00000000 --- a/arch/conduwuit.service +++ /dev/null @@ -1,84 +0,0 @@ -[Unit] - -Description=Continuwuity - Matrix homeserver -Wants=network-online.target -After=network-online.target -Documentation=https://continuwuity.org/ -RequiresMountsFor=/var/lib/private/conduwuit -Alias=matrix-conduwuit.service - -[Service] -DynamicUser=yes -Type=notify-reload -ReloadSignal=SIGUSR1 - -TTYPath=/dev/tty25 -DeviceAllow=char-tty -StandardInput=tty-force -StandardOutput=tty -StandardError=journal+console - -Environment="CONTINUWUITY_LOG_TO_JOURNALD=true" -Environment="CONTINUWUITY_JOURNALD_IDENTIFIER=%N" -Environment="CONTINUWUITY_DATABASE_PATH=/var/lib/conduwuit" - -TTYReset=yes -# uncomment to allow buffer to be cleared every restart -TTYVTDisallocate=no - -TTYColumns=120 -TTYRows=40 - -AmbientCapabilities= -CapabilityBoundingSet= - -DevicePolicy=closed -LockPersonality=yes -MemoryDenyWriteExecute=yes -NoNewPrivileges=yes -#ProcSubset=pid -ProtectClock=yes -ProtectControlGroups=yes -ProtectHome=yes -ProtectHostname=yes -ProtectKernelLogs=yes -ProtectKernelModules=yes -ProtectKernelTunables=yes -ProtectProc=invisible -ProtectSystem=strict -PrivateDevices=yes -PrivateMounts=yes -PrivateTmp=yes -PrivateUsers=yes -PrivateIPC=yes -RemoveIPC=yes -RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX -RestrictNamespaces=yes -RestrictRealtime=yes -RestrictSUIDSGID=yes -SystemCallArchitectures=native -SystemCallFilter=@system-service @resources -SystemCallFilter=~@clock @debug @module @mount @reboot @swap @cpu-emulation @obsolete @timer @chown @setuid @privileged @keyring @ipc -SystemCallErrorNumber=EPERM -StateDirectory=conduwuit - -RuntimeDirectory=conduwuit -RuntimeDirectoryMode=0750 - -Environment=CONTINUWUITY_CONFIG=%d/config.toml -LoadCredential=config.toml:/etc/conduwuit/conduwuit.toml -BindPaths=/var/lib/private/conduwuit:/var/lib/matrix-conduit -BindPaths=/var/lib/private/conduwuit:/var/lib/private/matrix-conduit - -ExecStart=/usr/bin/conduwuit -Restart=on-failure -RestartSec=5 - -TimeoutStopSec=4m -TimeoutStartSec=4m - -StartLimitInterval=1m -StartLimitBurst=5 - -[Install] -WantedBy=multi-user.target diff --git a/debian/conduwuit.service b/debian/conduwuit.service deleted file mode 100644 index ec2505b5..00000000 --- a/debian/conduwuit.service +++ /dev/null @@ -1,71 +0,0 @@ -[Unit] - -Description=Continuwuity - Matrix homeserver -Wants=network-online.target -After=network-online.target -Documentation=https://continuwuity.org/ -Alias=matrix-conduwuit.service - -[Service] -DynamicUser=yes -User=conduwuit -Group=conduwuit -Type=notify - -Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml" - -Environment="CONTINUWUITY_LOG_TO_JOURNALD=true" -Environment="CONTINUWUITY_JOURNALD_IDENTIFIER=%N" -Environment="CONTINUWUITY_DATABASE_PATH=/var/lib/conduwuit" - -ExecStart=/usr/sbin/conduwuit - -ReadWritePaths=/var/lib/conduwuit /etc/conduwuit - -AmbientCapabilities= -CapabilityBoundingSet= - -DevicePolicy=closed -LockPersonality=yes -MemoryDenyWriteExecute=yes -NoNewPrivileges=yes -#ProcSubset=pid -ProtectClock=yes -ProtectControlGroups=yes -ProtectHome=yes -ProtectHostname=yes -ProtectKernelLogs=yes -ProtectKernelModules=yes -ProtectKernelTunables=yes -ProtectProc=invisible -ProtectSystem=strict -PrivateDevices=yes -PrivateMounts=yes -PrivateTmp=yes -PrivateUsers=yes -PrivateIPC=yes -RemoveIPC=yes -RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX -RestrictNamespaces=yes -RestrictRealtime=yes -RestrictSUIDSGID=yes -SystemCallArchitectures=native -SystemCallFilter=@system-service @resources -SystemCallFilter=~@clock @debug @module @mount @reboot @swap @cpu-emulation @obsolete @timer @chown @setuid @privileged @keyring @ipc -SystemCallErrorNumber=EPERM -#StateDirectory=conduwuit - -RuntimeDirectory=conduwuit -RuntimeDirectoryMode=0750 - -Restart=on-failure -RestartSec=5 - -TimeoutStopSec=2m -TimeoutStartSec=2m - -StartLimitInterval=1m -StartLimitBurst=5 - -[Install] -WantedBy=multi-user.target diff --git a/fedora/conduwuit.service b/dist/conduwuit.service similarity index 95% rename from fedora/conduwuit.service rename to dist/conduwuit.service index f37c7798..db9aca1a 100644 --- a/fedora/conduwuit.service +++ b/dist/conduwuit.service @@ -9,7 +9,8 @@ Alias=matrix-conduwuit.service DynamicUser=yes User=conduwuit Group=conduwuit -Type=notify +Type=notify-reload +ReloadSignal=SIGUSR1 Environment="CONTINUWUITY_CONFIG=/etc/conduwuit/conduwuit.toml" @@ -59,8 +60,8 @@ RuntimeDirectoryMode=0750 Restart=on-failure RestartSec=5 -TimeoutStopSec=2m -TimeoutStartSec=2m +TimeoutStopSec=4m +TimeoutStartSec=4m StartLimitInterval=1m StartLimitBurst=5 diff --git a/debian/README.md b/dist/debian/README.md similarity index 100% rename from debian/README.md rename to dist/debian/README.md diff --git a/debian/config b/dist/debian/config similarity index 100% rename from debian/config rename to dist/debian/config diff --git a/debian/postinst b/dist/debian/postinst similarity index 100% rename from debian/postinst rename to dist/debian/postinst diff --git a/debian/postrm b/dist/debian/postrm similarity index 100% rename from debian/postrm rename to dist/debian/postrm diff --git a/fedora/continuwuity.spec.rpkg b/dist/fedora/continuwuity.spec.rpkg similarity index 97% rename from fedora/continuwuity.spec.rpkg rename to dist/fedora/continuwuity.spec.rpkg index f2efa383..dec4fff2 100644 --- a/fedora/continuwuity.spec.rpkg +++ b/dist/fedora/continuwuity.spec.rpkg @@ -51,7 +51,7 @@ find .cargo/registry/ -executable -name "*.rs" -exec chmod -x {} + %install install -Dpm0755 target/rpm/conduwuit -t %{buildroot}%{_bindir} -install -Dpm0644 fedora/conduwuit.service -t %{buildroot}%{_unitdir} +install -Dpm0644 dist/conduwuit.service -t %{buildroot}%{_unitdir} install -Dpm0644 conduwuit-example.toml %{buildroot}%{_sysconfdir}/conduwuit/conduwuit.toml %files diff --git a/nix/pkgs/main/cross-compilation-env.nix b/dist/nix/pkgs/main/cross-compilation-env.nix similarity index 100% rename from nix/pkgs/main/cross-compilation-env.nix rename to dist/nix/pkgs/main/cross-compilation-env.nix diff --git a/nix/pkgs/main/default.nix b/dist/nix/pkgs/main/default.nix similarity index 100% rename from nix/pkgs/main/default.nix rename to dist/nix/pkgs/main/default.nix diff --git a/docs/configuration/examples.md b/docs/configuration/examples.md index 54aa8bd7..b30661b5 100644 --- a/docs/configuration/examples.md +++ b/docs/configuration/examples.md @@ -9,24 +9,11 @@ -## Debian systemd unit file +## systemd unit file
    -Debian systemd unit file +systemd unit file ``` -{{#include ../../debian/conduwuit.service}} +{{#include ../../dist/conduwuit.service}} ``` - -
    - -## Arch Linux systemd unit file - -
    -Arch Linux systemd unit file - -``` -{{#include ../../arch/conduwuit.service}} -``` - -
    diff --git a/docs/deploying/debian.md b/docs/deploying/debian.md index 2e8a544a..527c1864 100644 --- a/docs/deploying/debian.md +++ b/docs/deploying/debian.md @@ -1 +1 @@ -{{#include ../../debian/README.md}} +{{#include ../../dist/debian/README.md}} diff --git a/flake.nix b/flake.nix index d6beb84e..4655fc01 100644 --- a/flake.nix +++ b/flake.nix @@ -48,7 +48,7 @@ pkgs.lib.makeScope pkgs.newScope (self: { inherit pkgs inputs; craneLib = (inputs.crane.mkLib pkgs).overrideToolchain (_: toolchain); - main = self.callPackage ./nix/pkgs/main { }; + main = self.callPackage ./dist/nix/pkgs/main { }; liburing = pkgs.liburing.overrideAttrs { # Tests weren't building outputs = [ diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index 58a39a75..e9e6d28d 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -126,9 +126,9 @@ pub struct Config { /// This is the only directory where continuwuity will save its data, /// including media. Note: this was previously "/var/lib/matrix-conduit". /// - /// YOU NEED TO EDIT THIS, UNLESS you are running continuwuity as a `systemd` service. - /// The service file sets it to `/var/lib/conduwuit` using an environment variable - /// and also grants write access. + /// YOU NEED TO EDIT THIS, UNLESS you are running continuwuity as a + /// `systemd` service. The service file sets it to `/var/lib/conduwuit` + /// using an environment variable and also grants write access. /// /// example: "/var/lib/conduwuit" pub database_path: PathBuf, diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index eafa1e48..cecd36e9 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -32,10 +32,10 @@ a cool hard fork of Conduit, a Matrix homeserver written in Rust""" section = "net" priority = "optional" conf-files = ["/etc/conduwuit/conduwuit.toml"] -maintainer-scripts = "../../debian/" +maintainer-scripts = "../../dist/debian/" systemd-units = { unit-name = "conduwuit", start = false } assets = [ - ["../../debian/README.md", "usr/share/doc/conduwuit/README.Debian", "644"], + ["../../dist/debian/README.md", "usr/share/doc/conduwuit/README.Debian", "644"], ["../../README.md", "usr/share/doc/conduwuit/", "644"], ["../../target/release/conduwuit", "usr/sbin/conduwuit", "755"], ["../../conduwuit-example.toml", "etc/conduwuit/conduwuit.toml", "640"], From e7124edb7391158cb9ae0fecf36ad682e75923d1 Mon Sep 17 00:00:00 2001 From: Ginger Date: Tue, 2 Sep 2025 10:37:25 -0400 Subject: [PATCH 2049/2091] fix: Update debian systemd unit path --- conduwuit-example.toml | 6 +++--- src/main/Cargo.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conduwuit-example.toml b/conduwuit-example.toml index fa65cbf2..0fc034d0 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -79,9 +79,9 @@ # This is the only directory where continuwuity will save its data, # including media. Note: this was previously "/var/lib/matrix-conduit". # -# YOU NEED TO EDIT THIS, UNLESS you are running continuwuity as a `systemd` service. -# The service file sets it to `/var/lib/conduwuit` using an environment variable -# and also grants write access. +# YOU NEED TO EDIT THIS, UNLESS you are running continuwuity as a +# `systemd` service. The service file sets it to `/var/lib/conduwuit` +# using an environment variable and also grants write access. # # example: "/var/lib/conduwuit" # diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index cecd36e9..bbb3bec4 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -33,11 +33,11 @@ section = "net" priority = "optional" conf-files = ["/etc/conduwuit/conduwuit.toml"] maintainer-scripts = "../../dist/debian/" -systemd-units = { unit-name = "conduwuit", start = false } +systemd-units = { unit-name = "conduwuit", start = false, unit-scripts = "../../dist/" } assets = [ ["../../dist/debian/README.md", "usr/share/doc/conduwuit/README.Debian", "644"], ["../../README.md", "usr/share/doc/conduwuit/", "644"], - ["../../target/release/conduwuit", "usr/sbin/conduwuit", "755"], + ["../../target/release/conduwuit", "usr/bin/conduwuit", "755"], ["../../conduwuit-example.toml", "etc/conduwuit/conduwuit.toml", "640"], ] From 0d58e660a297aa0e2864f42b131de510cc3d1f7b Mon Sep 17 00:00:00 2001 From: Ginger Date: Tue, 2 Sep 2025 10:44:47 -0400 Subject: [PATCH 2050/2091] fix: Remove unnecessary user and directory modifications systemd creates a dynamic user for continuwuity and manages directories for it automatically, so the debian postinst script no longer needs to do that. --- dist/debian/postinst | 24 ------------------------ dist/debian/postrm | 10 ++-------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/dist/debian/postinst b/dist/debian/postinst index 4eae4573..4bc89422 100644 --- a/dist/debian/postinst +++ b/dist/debian/postinst @@ -9,30 +9,6 @@ CONDUWUIT_CONFIG_PATH=/etc/conduwuit case "$1" in configure) - # Create the `conduwuit` user if it does not exist yet. - if ! getent passwd conduwuit > /dev/null ; then - echo 'Adding system user for the conduwuit Matrix homeserver' 1>&2 - adduser --system --group --quiet \ - --home "$CONDUWUIT_DATABASE_PATH" \ - --disabled-login \ - --shell "/usr/sbin/nologin" \ - conduwuit - fi - - # Create the database path if it does not exist yet and fix up ownership - # and permissions for the config. - mkdir -v -p "$CONDUWUIT_DATABASE_PATH" - - # symlink the previous location for compatibility if it does not exist yet. - if ! test -L "/var/lib/matrix-conduit" ; then - ln -s -v "$CONDUWUIT_DATABASE_PATH" "/var/lib/matrix-conduit" - fi - - chown -v conduwuit:conduwuit -R "$CONDUWUIT_DATABASE_PATH" - chown -v conduwuit:conduwuit -R "$CONDUWUIT_CONFIG_PATH" - - chmod -v 740 "$CONDUWUIT_DATABASE_PATH" - echo '' echo 'Make sure you edit the example config at /etc/conduwuit/conduwuit.toml before starting!' echo 'To start the server, run: systemctl start conduwuit.service' diff --git a/dist/debian/postrm b/dist/debian/postrm index 3c0b1c09..d5a9e0ac 100644 --- a/dist/debian/postrm +++ b/dist/debian/postrm @@ -20,24 +20,18 @@ case $1 in if [ -d "$CONDUWUIT_CONFIG_PATH" ]; then if test -L "$CONDUWUIT_CONFIG_PATH"; then - echo "Deleting conduwuit configuration files" + echo "Deleting continuwuity configuration files" rm -v -r "$CONDUWUIT_CONFIG_PATH" fi fi if [ -d "$CONDUWUIT_DATABASE_PATH" ]; then if test -L "$CONDUWUIT_DATABASE_PATH"; then - echo "Deleting conduwuit database directory" + echo "Deleting continuwuity database directory" rm -r "$CONDUWUIT_DATABASE_PATH" fi fi - if [ -d "$CONDUWUIT_DATABASE_PATH_SYMLINK" ]; then - if test -L "$CONDUWUIT_DATABASE_SYMLINK"; then - echo "Removing matrix-conduit symlink" - rm -r "$CONDUWUIT_DATABASE_PATH_SYMLINK" - fi - fi ;; esac From 58bbc0e676bbf770904dbe01e2fa4dc1aaa94dbe Mon Sep 17 00:00:00 2001 From: Ginger Date: Tue, 2 Sep 2025 12:20:23 -0400 Subject: [PATCH 2051/2091] fix: Move packaging files from `dist/` to `pkg/` --- .forgejo/workflows/release-image.yml | 2 +- docs/configuration/examples.md | 2 +- docs/deploying/debian.md | 2 +- flake.nix | 2 +- {dist => pkg}/conduwuit.service | 0 {dist => pkg}/debian/README.md | 0 {dist => pkg}/debian/config | 0 {dist => pkg}/debian/postinst | 0 {dist => pkg}/debian/postrm | 0 {dist => pkg}/fedora/continuwuity.spec.rpkg | 2 +- {dist => pkg}/nix/pkgs/main/cross-compilation-env.nix | 0 {dist => pkg}/nix/pkgs/main/default.nix | 0 src/main/Cargo.toml | 6 +++--- 13 files changed, 8 insertions(+), 8 deletions(-) rename {dist => pkg}/conduwuit.service (100%) rename {dist => pkg}/debian/README.md (100%) rename {dist => pkg}/debian/config (100%) rename {dist => pkg}/debian/postinst (100%) rename {dist => pkg}/debian/postrm (100%) rename {dist => pkg}/fedora/continuwuity.spec.rpkg (97%) rename {dist => pkg}/nix/pkgs/main/cross-compilation-env.nix (100%) rename {dist => pkg}/nix/pkgs/main/default.nix (100%) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 2055d91a..6972c791 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -10,7 +10,7 @@ on: - ".gitlab-ci.yml" - ".gitignore" - "renovate.json" - - "dist/**" + - "pkg/**" - "docker/**" - "docs/**" # Allows you to run this workflow manually from the Actions tab diff --git a/docs/configuration/examples.md b/docs/configuration/examples.md index b30661b5..9613e252 100644 --- a/docs/configuration/examples.md +++ b/docs/configuration/examples.md @@ -15,5 +15,5 @@ systemd unit file ``` -{{#include ../../dist/conduwuit.service}} +{{#include ../../pkg/conduwuit.service}} ``` diff --git a/docs/deploying/debian.md b/docs/deploying/debian.md index 527c1864..369638a4 100644 --- a/docs/deploying/debian.md +++ b/docs/deploying/debian.md @@ -1 +1 @@ -{{#include ../../dist/debian/README.md}} +{{#include ../../pkg/debian/README.md}} diff --git a/flake.nix b/flake.nix index 4655fc01..e65fcbda 100644 --- a/flake.nix +++ b/flake.nix @@ -48,7 +48,7 @@ pkgs.lib.makeScope pkgs.newScope (self: { inherit pkgs inputs; craneLib = (inputs.crane.mkLib pkgs).overrideToolchain (_: toolchain); - main = self.callPackage ./dist/nix/pkgs/main { }; + main = self.callPackage ./pkg/nix/pkgs/main { }; liburing = pkgs.liburing.overrideAttrs { # Tests weren't building outputs = [ diff --git a/dist/conduwuit.service b/pkg/conduwuit.service similarity index 100% rename from dist/conduwuit.service rename to pkg/conduwuit.service diff --git a/dist/debian/README.md b/pkg/debian/README.md similarity index 100% rename from dist/debian/README.md rename to pkg/debian/README.md diff --git a/dist/debian/config b/pkg/debian/config similarity index 100% rename from dist/debian/config rename to pkg/debian/config diff --git a/dist/debian/postinst b/pkg/debian/postinst similarity index 100% rename from dist/debian/postinst rename to pkg/debian/postinst diff --git a/dist/debian/postrm b/pkg/debian/postrm similarity index 100% rename from dist/debian/postrm rename to pkg/debian/postrm diff --git a/dist/fedora/continuwuity.spec.rpkg b/pkg/fedora/continuwuity.spec.rpkg similarity index 97% rename from dist/fedora/continuwuity.spec.rpkg rename to pkg/fedora/continuwuity.spec.rpkg index dec4fff2..a2b32e48 100644 --- a/dist/fedora/continuwuity.spec.rpkg +++ b/pkg/fedora/continuwuity.spec.rpkg @@ -51,7 +51,7 @@ find .cargo/registry/ -executable -name "*.rs" -exec chmod -x {} + %install install -Dpm0755 target/rpm/conduwuit -t %{buildroot}%{_bindir} -install -Dpm0644 dist/conduwuit.service -t %{buildroot}%{_unitdir} +install -Dpm0644 pkg/conduwuit.service -t %{buildroot}%{_unitdir} install -Dpm0644 conduwuit-example.toml %{buildroot}%{_sysconfdir}/conduwuit/conduwuit.toml %files diff --git a/dist/nix/pkgs/main/cross-compilation-env.nix b/pkg/nix/pkgs/main/cross-compilation-env.nix similarity index 100% rename from dist/nix/pkgs/main/cross-compilation-env.nix rename to pkg/nix/pkgs/main/cross-compilation-env.nix diff --git a/dist/nix/pkgs/main/default.nix b/pkg/nix/pkgs/main/default.nix similarity index 100% rename from dist/nix/pkgs/main/default.nix rename to pkg/nix/pkgs/main/default.nix diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index bbb3bec4..8f6f4341 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -32,10 +32,10 @@ a cool hard fork of Conduit, a Matrix homeserver written in Rust""" section = "net" priority = "optional" conf-files = ["/etc/conduwuit/conduwuit.toml"] -maintainer-scripts = "../../dist/debian/" -systemd-units = { unit-name = "conduwuit", start = false, unit-scripts = "../../dist/" } +maintainer-scripts = "../../pkg/debian/" +systemd-units = { unit-name = "conduwuit", start = false, unit-scripts = "../../pkg/" } assets = [ - ["../../dist/debian/README.md", "usr/share/doc/conduwuit/README.Debian", "644"], + ["../../pkg/debian/README.md", "usr/share/doc/conduwuit/README.Debian", "644"], ["../../README.md", "usr/share/doc/conduwuit/", "644"], ["../../target/release/conduwuit", "usr/bin/conduwuit", "755"], ["../../conduwuit-example.toml", "etc/conduwuit/conduwuit.toml", "640"], From fdf577138711786582b6c9484188a96346167f3f Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 6 Sep 2025 15:20:45 +0100 Subject: [PATCH 2052/2091] ci: Fix CI not triggering on external pull requests --- .forgejo/workflows/prek-checks.yml | 4 ++++ .forgejo/workflows/release-image.yml | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.forgejo/workflows/prek-checks.yml b/.forgejo/workflows/prek-checks.yml index c25b9c3d..45288bef 100644 --- a/.forgejo/workflows/prek-checks.yml +++ b/.forgejo/workflows/prek-checks.yml @@ -1,7 +1,11 @@ name: Checks / Prek on: + pull_request: push: + branches: + - main + workflow_dispatch: permissions: contents: read diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 6972c791..b1a26e24 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -3,7 +3,19 @@ concurrency: group: "release-image-${{ github.ref }}" on: + pull_request: + paths-ignore: + - "*.md" + - "**/*.md" + - ".gitlab-ci.yml" + - ".gitignore" + - "renovate.json" + - "pkg/**" + - "docker/**" + - "docs/**" push: + branches: + - main: paths-ignore: - "*.md" - "**/*.md" From 2516e783ba88d54e04896e2a553fcadb547691b7 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sat, 6 Sep 2025 15:55:24 +0100 Subject: [PATCH 2053/2091] ci: Support optional persistent BuildKit endpoints in Docker builds Allows us to use runners with persistent BuildKit containers for improved caching and faster build times. Falls back to standard docker-container driver when BUILDKIT_ENDPOINT environment variable is not set. --- .forgejo/workflows/release-image.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index b1a26e24..2b3d481f 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -105,6 +105,10 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + # Use persistent BuildKit if BUILDKIT_ENDPOINT is set (e.g. tcp://buildkit:8125) + driver: ${{ env.BUILDKIT_ENDPOINT != '' && 'remote' || 'docker-container' }} + endpoint: ${{ env.BUILDKIT_ENDPOINT || '' }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. @@ -262,6 +266,10 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + # Use persistent BuildKit if BUILDKIT_ENDPOINT is set (e.g. tcp://buildkit:8125) + driver: ${{ env.BUILDKIT_ENDPOINT != '' && 'remote' || 'docker-container' }} + endpoint: ${{ env.BUILDKIT_ENDPOINT || '' }} - name: Extract metadata (tags) for Docker id: meta From 6f19931c5b25f96fae0f99fd938a0afb2da7620a Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 31 Aug 2025 01:55:12 +0100 Subject: [PATCH 2054/2091] chore(deps): Upgrade minor incompatible dependencies --- Cargo.lock | 268 +++++++++++++++++++++++++++++++++-------------------- Cargo.toml | 24 ++--- 2 files changed, 181 insertions(+), 111 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a3962c31..322956a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -714,12 +714,12 @@ dependencies = [ [[package]] name = "cargo_toml" -version = "0.21.0" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fbd1fe9db3ebf71b89060adaf7b0504c2d6a425cf061313099547e382c2e472" +checksum = "374b7c592d9c00c1f4972ea58390ac6b18cbb6ab79011f3bdc90a0b82ca06b77" dependencies = [ "serde", - "toml", + "toml 0.9.5", ] [[package]] @@ -897,9 +897,9 @@ dependencies = [ "const-str", "hardened_malloc-rs", "log", - "opentelemetry", + "opentelemetry 0.30.0", "opentelemetry-jaeger", - "opentelemetry_sdk", + "opentelemetry_sdk 0.30.0", "sentry", "sentry-tower", "sentry-tracing", @@ -1026,7 +1026,7 @@ dependencies = [ "tikv-jemallocator", "tokio", "tokio-metrics", - "toml", + "toml 0.9.5", "tracing", "tracing-core", "tracing-subscriber", @@ -1285,9 +1285,9 @@ checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" [[package]] name = "crokey" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5282b45c96c5978c8723ea83385cb9a488b64b7d175733f48d07bf9da514a863" +checksum = "51360853ebbeb3df20c76c82aecf43d387a62860f1a59ba65ab51f00eea85aad" dependencies = [ "crokey-proc_macros", "crossterm", @@ -1298,9 +1298,9 @@ dependencies = [ [[package]] name = "crokey-proc_macros" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ea0218d3fedf0797fa55676f1964ef5d27103d41ed0281b4bbd2a6e6c3d8d28" +checksum = "3bf1a727caeb5ee5e0a0826a97f205a9cf84ee964b0b48239fef5214a00ae439" dependencies = [ "crossterm", "proc-macro2", @@ -1706,7 +1706,7 @@ dependencies = [ "atomic", "pear", "serde", - "toml", + "toml 0.8.23", "uncased", "version_check", ] @@ -3045,9 +3045,9 @@ checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" [[package]] name = "nix" -version = "0.29.0" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ "bitflags 2.9.1", "cfg-if", @@ -3229,59 +3229,67 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" +checksum = "1b69a91d4893e713e06f724597ad630f1fa76057a5e1026c0ca67054a9032a76" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.10.0", "js-sys", "once_cell", "pin-project-lite", "thiserror 1.0.69", - "urlencoding", +] + +[[package]] +name = "opentelemetry" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.12", + "tracing", ] [[package]] name = "opentelemetry-jaeger" -version = "0.20.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e617c66fd588e40e0dbbd66932fdc87393095b125d4459b1a3a10feb1712f8a1" +checksum = "501b471b67b746d9a07d4c29f8be00f952d1a2eca356922ede0098cbaddff19f" dependencies = [ "async-trait", "futures-core", "futures-util", - "opentelemetry", + "opentelemetry 0.23.0", "opentelemetry-semantic-conventions", - "opentelemetry_sdk", + "opentelemetry_sdk 0.23.0", "thrift", "tokio", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" -dependencies = [ - "opentelemetry", -] +checksum = "1869fb4bb9b35c5ba8a1e40c9b128a7b4c010d07091e864a29da19e4fe2ca4d7" [[package]] name = "opentelemetry_sdk" -version = "0.21.2" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" +checksum = "ae312d58eaa90a82d2e627fd86e075cf5230b3f11794e2ed74199ebbe572d4fd" dependencies = [ "async-trait", - "crossbeam-channel", "futures-channel", "futures-executor", "futures-util", - "glob", + "lazy_static", "once_cell", - "opentelemetry", + "opentelemetry 0.23.0", "ordered-float 4.6.0", "percent-encoding", "rand 0.8.5", @@ -3290,6 +3298,24 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opentelemetry_sdk" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11f644aa9e5e31d11896e024305d7e3c98a88884d9f8919dbf37a9991bc47a4b" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.30.0", + "percent-encoding", + "rand 0.9.2", + "serde_json", + "thiserror 2.0.12", + "tokio", + "tokio-stream", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -3713,7 +3739,7 @@ dependencies = [ "thiserror 2.0.12", "tokio", "tracing", - "web-time 1.1.0", + "web-time", ] [[package]] @@ -3734,7 +3760,7 @@ dependencies = [ "thiserror 2.0.12", "tinyvec", "tracing", - "web-time 1.1.0", + "web-time", ] [[package]] @@ -4072,7 +4098,7 @@ dependencies = [ "ruma-identity-service-api", "ruma-push-gateway-api", "ruma-signatures", - "web-time 1.1.0", + "web-time", ] [[package]] @@ -4107,7 +4133,7 @@ dependencies = [ "serde_json", "thiserror 2.0.12", "url", - "web-time 1.1.0", + "web-time", ] [[package]] @@ -4138,7 +4164,7 @@ dependencies = [ "tracing", "url", "uuid", - "web-time 1.1.0", + "web-time", "wildmatch", ] @@ -4163,7 +4189,7 @@ dependencies = [ "thiserror 2.0.12", "tracing", "url", - "web-time 1.1.0", + "web-time", "wildmatch", ] @@ -4220,7 +4246,7 @@ dependencies = [ "ruma-identifiers-validation", "serde", "syn 2.0.104", - "toml", + "toml 0.8.23", ] [[package]] @@ -4415,7 +4441,7 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" dependencies = [ - "web-time 1.1.0", + "web-time", "zeroize", ] @@ -4560,9 +4586,9 @@ checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "sentry" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "255914a8e53822abd946e2ce8baa41d4cded6b8e938913b7f7b9da5b7ab44335" +checksum = "989425268ab5c011e06400187eed6c298272f8ef913e49fcadc3fda788b45030" dependencies = [ "httpdate", "reqwest", @@ -4577,26 +4603,24 @@ dependencies = [ "sentry-tracing", "tokio", "ureq", - "webpki-roots 0.26.11", ] [[package]] name = "sentry-backtrace" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00293cd332a859961f24fd69258f7e92af736feaeb91020cff84dac4188a4302" +checksum = "68e299dd3f7bcf676875eee852c9941e1d08278a743c32ca528e2debf846a653" dependencies = [ "backtrace", - "once_cell", "regex", "sentry-core", ] [[package]] name = "sentry-contexts" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "961990f9caa76476c481de130ada05614cd7f5aa70fb57c2142f0e09ad3fb2aa" +checksum = "fac0c5d6892cd4c414492fc957477b620026fb3411fca9fa12774831da561c88" dependencies = [ "hostname", "libc", @@ -4608,33 +4632,32 @@ dependencies = [ [[package]] name = "sentry-core" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a6409d845707d82415c800290a5d63be5e3df3c2e417b0997c60531dfbd35ef" +checksum = "deaa38b94e70820ff3f1f9db3c8b0aef053b667be130f618e615e0ff2492cbcc" dependencies = [ - "once_cell", - "rand 0.8.5", + "rand 0.9.2", "sentry-types", "serde", "serde_json", + "url", ] [[package]] name = "sentry-debug-images" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71ab5df4f3b64760508edfe0ba4290feab5acbbda7566a79d72673065888e5cc" +checksum = "00950648aa0d371c7f57057434ad5671bd4c106390df7e7284739330786a01b6" dependencies = [ "findshlibs", - "once_cell", "sentry-core", ] [[package]] name = "sentry-log" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693841da8dfb693af29105edfbea1d91348a13d23dd0a5d03761eedb9e450c46" +checksum = "670f08baf70058926b0fa60c8f10218524ef0cb1a1634b0388a4123bdec6288c" dependencies = [ "log", "sentry-core", @@ -4642,9 +4665,9 @@ dependencies = [ [[package]] name = "sentry-panic" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609b1a12340495ce17baeec9e08ff8ed423c337c1a84dffae36a178c783623f3" +checksum = "2b7a23b13c004873de3ce7db86eb0f59fe4adfc655a31f7bbc17fd10bacc9bfe" dependencies = [ "sentry-backtrace", "sentry-core", @@ -4652,9 +4675,9 @@ dependencies = [ [[package]] name = "sentry-tower" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b98005537e38ee3bc10e7d36e7febe9b8e573d03f2ddd85fcdf05d21f9abd6d" +checksum = "4a303d0127d95ae928a937dcc0886931d28b4186e7338eea7d5786827b69b002" dependencies = [ "http", "pin-project", @@ -4666,10 +4689,11 @@ dependencies = [ [[package]] name = "sentry-tracing" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f4e86402d5c50239dc7d8fd3f6d5e048221d5fcb4e026d8d50ab57fe4644cb" +checksum = "fac841c7050aa73fc2bec8f7d8e9cb1159af0b3095757b99820823f3e54e5080" dependencies = [ + "bitflags 2.9.1", "sentry-backtrace", "sentry-core", "tracing-core", @@ -4678,16 +4702,16 @@ dependencies = [ [[package]] name = "sentry-types" -version = "0.37.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3f117b8755dbede8260952de2aeb029e20f432e72634e8969af34324591631" +checksum = "e477f4d4db08ddb4ab553717a8d3a511bc9e81dde0c808c680feacbb8105c412" dependencies = [ "debugid", "hex", - "rand 0.8.5", + "rand 0.9.2", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.12", "time", "url", "uuid", @@ -4767,6 +4791,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40734c41988f7306bb04f0ecf60ec0f3f1caa34290e4e8ea471dcd3346483b83" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -5061,7 +5094,7 @@ dependencies = [ "cfg-expr", "heck", "pkg-config", - "toml", + "toml 0.8.23", "version-compare", ] @@ -5090,9 +5123,9 @@ dependencies = [ [[package]] name = "termimad" -version = "0.31.3" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7301d9c2c4939c97f25376b70d3c13311f8fefdee44092fc361d2a98adc2cbb6" +checksum = "68ff5ca043d65d4ea43b65cdb4e3aba119657d0d12caf44f93212ec3168a8e20" dependencies = [ "coolor", "crokey", @@ -5396,11 +5429,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_edit", ] +[[package]] +name = "toml" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" +dependencies = [ + "indexmap 2.10.0", + "serde", + "serde_spanned 1.0.0", + "toml_datetime 0.7.0", + "toml_parser", + "toml_writer", + "winnow", +] + [[package]] name = "toml_datetime" version = "0.6.11" @@ -5410,6 +5458,15 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_datetime" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bade1c3e902f58d73d3f294cd7f20391c1cb2fbcb643b73566bc773971df91e3" +dependencies = [ + "serde", +] + [[package]] name = "toml_edit" version = "0.22.27" @@ -5418,18 +5475,33 @@ checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.10.0", "serde", - "serde_spanned", - "toml_datetime", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", "toml_write", "winnow", ] +[[package]] +name = "toml_parser" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" +dependencies = [ + "winnow", +] + [[package]] name = "toml_write" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "toml_writer" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc842091f2def52017664b53082ecbbeb5c7731092bad69d2c63050401dfd64" + [[package]] name = "tonic" version = "0.12.3" @@ -5593,20 +5665,20 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.22.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" +checksum = "ddcf5959f39507d0d04d6413119c04f33b623f4f951ebcbdddddfad2d0623a9c" dependencies = [ "js-sys", "once_cell", - "opentelemetry", - "opentelemetry_sdk", + "opentelemetry 0.30.0", + "opentelemetry_sdk 0.30.0", "smallvec", "tracing", "tracing-core", "tracing-log", "tracing-subscriber", - "web-time 0.2.4", + "web-time", ] [[package]] @@ -5727,17 +5799,31 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "2.12.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +checksum = "00432f493971db5d8e47a65aeb3b02f8226b9b11f1450ff86bb772776ebadd70" dependencies = [ "base64 0.22.1", "log", - "once_cell", + "percent-encoding", "rustls 0.23.29", + "rustls-pemfile 2.2.0", "rustls-pki-types", - "url", - "webpki-roots 0.26.11", + "ureq-proto", + "utf-8", + "webpki-roots 1.0.2", +] + +[[package]] +name = "ureq-proto" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe120bb823a0061680e66e9075942fcdba06d46551548c2c259766b9558bc9a" +dependencies = [ + "base64 0.22.1", + "http", + "httparse", + "log", ] [[package]] @@ -5752,12 +5838,6 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf-8" version = "0.7.6" @@ -5928,16 +6008,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "web-time" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "web-time" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index e2af2d94..fe388b4d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,12 +51,12 @@ version = "0.6.2" version = "0.2.9" [workspace.dependencies.cargo_toml] -version = "0.21" +version = "0.22" default-features = false features = ["features"] [workspace.dependencies.toml] -version = "0.8.14" +version = "0.9.5" default-features = false features = ["parse"] @@ -411,25 +411,25 @@ default-features = false # optional opentelemetry, performance measurements, flamegraphs, etc for performance measurements and monitoring [workspace.dependencies.opentelemetry] -version = "0.21.0" +version = "0.30.0" [workspace.dependencies.tracing-flame] version = "0.2.0" [workspace.dependencies.tracing-opentelemetry] -version = "0.22.0" +version = "0.31.0" [workspace.dependencies.opentelemetry_sdk] -version = "0.21.2" +version = "0.30.0" features = ["rt-tokio"] [workspace.dependencies.opentelemetry-jaeger] -version = "0.20.0" +version = "0.22.0" features = ["rt-tokio"] # optional sentry metrics for crash/panic reporting [workspace.dependencies.sentry] -version = "0.37.0" +version = "0.42.0" default-features = false features = [ "backtrace", @@ -445,9 +445,9 @@ features = [ ] [workspace.dependencies.sentry-tracing] -version = "0.37.0" +version = "0.42.0" [workspace.dependencies.sentry-tower] -version = "0.37.0" +version = "0.42.0" # jemalloc usage [workspace.dependencies.tikv-jemalloc-sys] @@ -476,7 +476,7 @@ features = ["use_std"] version = "0.4" [workspace.dependencies.nix] -version = "0.29.0" +version = "0.30.1" default-features = false features = ["resource"] @@ -498,7 +498,7 @@ version = "0.4.3" default-features = false [workspace.dependencies.termimad] -version = "0.31.2" +version = "0.34.0" default-features = false [workspace.dependencies.checked_ops] @@ -536,7 +536,7 @@ version = "0.2" version = "0.2" [workspace.dependencies.minicbor] -version = "0.26.3" +version = "0.26.5" features = ["std"] [workspace.dependencies.minicbor-serde] From 1d7dda6cf55c8843ea8869a1f482633b2b66fc2f Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sun, 31 Aug 2025 03:01:06 +0100 Subject: [PATCH 2055/2091] chore: Upgrade ctor, cbor --- Cargo.lock | 604 +++++++++++++++++++++----------------- Cargo.toml | 6 +- src/admin/Cargo.toml | 1 + src/admin/mod.rs | 2 + src/api/Cargo.toml | 1 + src/api/client/message.rs | 2 +- src/database/Cargo.toml | 1 + src/database/mod.rs | 2 + src/macros/rustc.rs | 4 +- src/main/Cargo.toml | 1 + src/main/mod.rs | 1 + src/router/Cargo.toml | 1 + src/router/mod.rs | 1 + src/service/Cargo.toml | 1 + src/service/mod.rs | 1 + 15 files changed, 353 insertions(+), 276 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 322956a0..3c24b178 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,9 +52,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.19" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" dependencies = [ "anstyle", "anstyle-parse", @@ -82,35 +82,35 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.9" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" [[package]] name = "arbitrary" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "arc-swap" @@ -126,7 +126,7 @@ checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -183,7 +183,7 @@ dependencies = [ "rustc-hash 2.1.1", "serde", "serde_derive", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -256,11 +256,13 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.27" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddb939d66e4ae03cee6091612804ba446b12878410cfa17f785f4dd67d4014e8" +checksum = "5bee399cc3a623ec5a2db2c5b90ee0190a2260241fbe0c023ac8f7bab426aaf8" dependencies = [ "brotli", + "compression-codecs", + "compression-core", "flate2", "futures-core", "memchr", @@ -289,18 +291,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "async-trait" -version = "0.1.88" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -340,9 +342,9 @@ dependencies = [ [[package]] name = "avif-serialize" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ea8ef51aced2b9191c08197f55450d830876d9933f8f48a429b354f1d496b42" +checksum = "47c8fbc0f831f4519fe8b810b6a7a91410ec83031b8233f730a0480029f6a23f" dependencies = [ "arrayvec", ] @@ -472,7 +474,7 @@ dependencies = [ "hyper", "hyper-util", "pin-project-lite", - "rustls 0.23.29", + "rustls 0.23.31", "rustls-pemfile 2.2.0", "rustls-pki-types", "tokio", @@ -491,7 +493,7 @@ dependencies = [ "http", "http-body-util", "pin-project", - "rustls 0.23.29", + "rustls 0.23.31", "tokio", "tokio-rustls 0.26.2", "tokio-util", @@ -547,7 +549,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "cexpr", "clang-sys", "itertools 0.12.1", @@ -560,7 +562,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.104", + "syn 2.0.106", "which", ] @@ -570,7 +572,7 @@ version = "0.72.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f72209734318d0b619a5e0f5129918b848c416e122a3c4ce054e03cb87b726f" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "cexpr", "clang-sys", "itertools 0.13.0", @@ -579,14 +581,14 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "bit_field" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" [[package]] name = "bitflags" @@ -596,9 +598,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" [[package]] name = "bitstream-io" @@ -635,9 +637,9 @@ dependencies = [ [[package]] name = "brotli" -version = "8.0.1" +version = "8.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -674,9 +676,9 @@ checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" [[package]] name = "bytemuck" -version = "1.23.1" +version = "1.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" +checksum = "3995eaeebcdf32f91f980d360f78732ddc061097ab4e39991ae7a6ace9194677" [[package]] name = "byteorder" @@ -724,9 +726,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.30" +version = "1.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +checksum = "42bc4aea80032b7bf409b0bc7ccad88853858911b7713a8062fdc0623867bedc" dependencies = [ "jobserver", "libc", @@ -754,9 +756,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" [[package]] name = "cfg_aliases" @@ -795,9 +797,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.41" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be92d32e80243a54711e5d7ce823c35c41c9d929dc4ab58e1276f625841aadf9" +checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" dependencies = [ "clap_builder", "clap_derive", @@ -814,9 +816,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.41" +version = "4.5.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707eab41e9622f9139419d573eca0900137718000c517d47da73045f54331c3d" +checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" dependencies = [ "anstream", "anstyle", @@ -826,14 +828,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.41" +version = "4.5.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491" +checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -844,9 +846,9 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "clap_mangen" -version = "0.2.28" +version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2fb6d3f935bbb9819391528b0e7cf655e78a0bc7a7c3d227211a1d24fc11db1" +checksum = "27b4c3c54b30f0d9adcb47f25f61fcce35c4dd8916638c6b82fbd5f4fb4179e2" dependencies = [ "clap", "roff", @@ -873,6 +875,28 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "compression-codecs" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7eea68f0e02c2b0aa8856e9a9478444206d4b6828728e7b0697c0f8cca265cb" +dependencies = [ + "brotli", + "compression-core", + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47641d3deaf41fb1538ac1f54735925e275eaf3bf4d55c81b137fba797e5cbb" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -895,6 +919,7 @@ dependencies = [ "conduwuit_service", "console-subscriber", "const-str", + "ctor", "hardened_malloc-rs", "log", "opentelemetry 0.30.0", @@ -923,6 +948,7 @@ dependencies = [ "conduwuit_macros", "conduwuit_service", "const-str", + "ctor", "futures", "log", "ruma", @@ -946,6 +972,7 @@ dependencies = [ "conduwuit_core", "conduwuit_service", "const-str", + "ctor", "futures", "hmac", "http", @@ -1020,7 +1047,7 @@ dependencies = [ "serde_yaml", "smallstr", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "tikv-jemalloc-ctl", "tikv-jemalloc-sys", "tikv-jemallocator", @@ -1040,6 +1067,7 @@ dependencies = [ "async-channel", "conduwuit_core", "const-str", + "ctor", "futures", "log", "minicbor", @@ -1058,7 +1086,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1076,6 +1104,7 @@ dependencies = [ "conduwuit_service", "conduwuit_web", "const-str", + "ctor", "futures", "http", "http-body-util", @@ -1083,7 +1112,7 @@ dependencies = [ "hyper-util", "log", "ruma", - "rustls 0.23.29", + "rustls 0.23.31", "sd-notify", "sentry", "sentry-tower", @@ -1106,6 +1135,7 @@ dependencies = [ "conduwuit_core", "conduwuit_database", "const-str", + "ctor", "either", "futures", "hickory-resolver 0.25.2", @@ -1144,7 +1174,7 @@ dependencies = [ "conduwuit_service", "futures", "rand 0.8.5", - "thiserror 2.0.12", + "thiserror 2.0.16", "tracing", ] @@ -1195,15 +1225,18 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "const-str" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "041fbfcf8e7054df725fb9985297e92422cdc80fcf313665f5ca3d761bb63f4c" +checksum = "451d0640545a0553814b4c646eb549343561618838e9b42495f466131fe3ad49" [[package]] name = "const_panic" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98d1483e98c9d67f341ab4b3915cfdc54740bd6f5cccc9226ee0535d86aa8fb" +checksum = "bb8a602185c3c95b52f86dc78e55a6df9a287a7a93ddbcf012509930880cf879" +dependencies = [ + "typewit", +] [[package]] name = "convert_case" @@ -1306,7 +1339,7 @@ dependencies = [ "proc-macro2", "quote", "strict", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1371,7 +1404,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "crossterm_winapi", "derive_more", "document-features", @@ -1411,14 +1444,20 @@ dependencies = [ [[package]] name = "ctor" -version = "0.2.9" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" +checksum = "67773048316103656a637612c4a62477603b777d91d9c62ff2290f9cde178fdb" dependencies = [ - "quote", - "syn 2.0.104", + "ctor-proc-macro", + "dtor", ] +[[package]] +name = "ctor-proc-macro" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2931af7e13dc045d8e9d26afccc6fa115d64e115c9c84b1166288b46f6782c2" + [[package]] name = "curve25519-dalek" version = "4.1.3" @@ -1443,7 +1482,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1525,7 +1564,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1547,7 +1586,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1559,6 +1598,21 @@ dependencies = [ "litrs", ] +[[package]] +name = "dtor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e58a0764cddb55ab28955347b45be00ade43d4d6f3ba4bf3dc354e4ec9432934" +dependencies = [ + "dtor-proc-macro", +] + +[[package]] +name = "dtor-proc-macro" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f678cf4a922c215c63e0de95eb1ff08a958a81d47e485cf9da1e27bf6305cfa5" + [[package]] name = "dunce" version = "1.0.5" @@ -1608,7 +1662,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1628,7 +1682,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1747,9 +1801,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] @@ -1846,7 +1900,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -1881,9 +1935,9 @@ dependencies = [ [[package]] name = "generator" -version = "0.8.5" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d18470a76cb7f8ff746cf1f7470914f900252ec36bbc40b569d74b1258446827" +checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" dependencies = [ "cc", "cfg-if", @@ -1926,7 +1980,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "wasi 0.14.3+wasi-0.2.4", "wasm-bindgen", ] @@ -1948,15 +2002,15 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "h2" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" dependencies = [ "atomic-waker", "bytes", @@ -1964,7 +2018,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.10.0", + "indexmap 2.11.0", "slab", "tokio", "tokio-util", @@ -1995,9 +2049,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" [[package]] name = "hdrhistogram" @@ -2097,7 +2151,7 @@ dependencies = [ "rand 0.9.2", "ring 0.17.14", "serde", - "thiserror 2.0.12", + "thiserror 2.0.16", "tinyvec", "tokio", "tracing", @@ -2142,7 +2196,7 @@ dependencies = [ "resolv-conf", "serde", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tracing", ] @@ -2187,7 +2241,7 @@ dependencies = [ "markup5ever", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -2253,13 +2307,14 @@ checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "hyper" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" dependencies = [ + "atomic-waker", "bytes", "futures-channel", - "futures-util", + "futures-core", "h2", "http", "http-body", @@ -2267,6 +2322,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", + "pin-utils", "smallvec", "tokio", "want", @@ -2281,7 +2337,7 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.29", + "rustls 0.23.31", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -2316,7 +2372,7 @@ dependencies = [ "hyper", "libc", "pin-project-lite", - "socket2", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2410,9 +2466,9 @@ dependencies = [ [[package]] name = "idna" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", "smallvec", @@ -2454,9 +2510,9 @@ dependencies = [ [[package]] name = "image-webp" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6970fe7a5300b4b42e62c52efa0187540a5bef546c60edaf554ef595d2e6f0b" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" dependencies = [ "byteorder-lite", "quick-error", @@ -2480,12 +2536,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.10.0" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" dependencies = [ "equivalent", - "hashbrown 0.15.4", + "hashbrown 0.15.5", "serde", ] @@ -2509,16 +2565,16 @@ checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "io-uring" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "cfg-if", "libc", ] @@ -2543,7 +2599,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2", + "socket2 0.5.10", "widestring", "windows-sys 0.48.0", "winreg", @@ -2596,9 +2652,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jobserver" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ "getrandom 0.3.3", "libc", @@ -2678,7 +2734,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -2738,9 +2794,9 @@ checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" [[package]] name = "libc" -version = "0.2.174" +version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" +checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "libfuzzer-sys" @@ -2759,7 +2815,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.53.2", + "windows-targets 0.53.3", ] [[package]] @@ -2799,9 +2855,9 @@ checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "litrs" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" +checksum = "f5e54036fe321fd421e10d732f155734c4e4afd610dd556d9a82833ab3ee0bed" [[package]] name = "lock_api" @@ -2953,29 +3009,29 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "minicbor" -version = "0.26.5" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a309f581ade7597820083bc275075c4c6986e57e53f8d26f88507cfefc8c987" +checksum = "4f182275033b808ede9427884caa8e05fa7db930801759524ca7925bd8aa7a82" dependencies = [ "minicbor-derive", ] [[package]] name = "minicbor-derive" -version = "0.16.2" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9882ef5c56df184b8ffc107fc6c61e33ee3a654b021961d790a78571bb9d67a" +checksum = "b17290c95158a760027059fe3f511970d6857e47ff5008f9e09bffe3d3e1c6af" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "minicbor-serde" -version = "0.4.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e45e8beeefea1b8b6f52fa188a5b6ea3746c2885606af8d4d8bf31cee633fb" +checksum = "0bbf243b8cc68a7a76473b14328d3546fb002ae3d069227794520e9181003de9" dependencies = [ "minicbor", "serde", @@ -3049,7 +3105,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "cfg-if", "cfg_aliases", "libc", @@ -3134,7 +3190,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3251,7 +3307,7 @@ dependencies = [ "futures-sink", "js-sys", "pin-project-lite", - "thiserror 2.0.12", + "thiserror 2.0.16", "tracing", ] @@ -3311,7 +3367,7 @@ dependencies = [ "percent-encoding", "rand 0.9.2", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.16", "tokio", "tokio-stream", ] @@ -3421,14 +3477,14 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "petgraph" @@ -3437,7 +3493,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.10.0", + "indexmap 2.11.0", ] [[package]] @@ -3495,7 +3551,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3533,7 +3589,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3af6b589e163c5a788fab00ce0c0366f6efbb9959c2f9874b224936af7fce7e1" dependencies = [ "base64 0.22.1", - "indexmap 2.10.0", + "indexmap 2.11.0", "quick-xml", "serde", "time", @@ -3560,9 +3616,9 @@ checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "potential_utf" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" dependencies = [ "zerovec", ] @@ -3590,12 +3646,12 @@ checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" [[package]] name = "prettyplease" -version = "0.2.36" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff24dfcda44452b9816fff4cd4227e1bb73ff5a2f1bc1105aa92fb8565ce44d2" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3609,9 +3665,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] @@ -3624,7 +3680,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "version_check", "yansi", ] @@ -3645,7 +3701,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52717f9a02b6965224f95ca2a81e2e0c5c43baacd28ca057577988930b6c3d5b" dependencies = [ "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3668,7 +3724,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -3686,7 +3742,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "memchr", "pulldown-cmark-escape", "unicase", @@ -3715,18 +3771,18 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.38.0" +version = "0.38.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8927b0664f5c5a98265138b7e3f90aa19a6b21353182469ace36d4ac527b7b1b" +checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" dependencies = [ "memchr", ] [[package]] name = "quinn" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases", @@ -3734,9 +3790,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls 0.23.29", - "socket2", - "thiserror 2.0.12", + "rustls 0.23.31", + "socket2 0.6.0", + "thiserror 2.0.16", "tokio", "tracing", "web-time", @@ -3744,9 +3800,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", "getrandom 0.3.3", @@ -3754,10 +3810,10 @@ dependencies = [ "rand 0.9.2", "ring 0.17.14", "rustc-hash 2.1.1", - "rustls 0.23.29", + "rustls 0.23.31", "rustls-pki-types", "slab", - "thiserror 2.0.12", + "thiserror 2.0.16", "tinyvec", "tracing", "web-time", @@ -3765,16 +3821,16 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.0", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -3903,9 +3959,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", @@ -3913,9 +3969,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -3923,9 +3979,9 @@ dependencies = [ [[package]] name = "recaptcha-verify" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e3be7b2e46e24637ac96b0c9f70070f188652018573f36f4e511dcad09738a" +checksum = "0d694033c2b0abdbb8893edfb367f16270e790be4a67e618206d811dbe4efee4" dependencies = [ "reqwest", "serde", @@ -3934,23 +3990,23 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.15" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec" +checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", ] [[package]] name = "regex" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", + "regex-automata 0.4.10", + "regex-syntax 0.8.6", ] [[package]] @@ -3964,13 +4020,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax 0.8.6", ] [[package]] @@ -3981,9 +4037,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" [[package]] name = "reqwest" @@ -4013,7 +4069,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.29", + "rustls 0.23.31", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -4131,7 +4187,7 @@ dependencies = [ "serde", "serde_html_form", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.16", "url", "web-time", ] @@ -4147,7 +4203,7 @@ dependencies = [ "form_urlencoded", "getrandom 0.2.16", "http", - "indexmap 2.10.0", + "indexmap 2.11.0", "js_int", "konst", "percent-encoding", @@ -4159,7 +4215,7 @@ dependencies = [ "serde_html_form", "serde_json", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "tracing", "url", @@ -4174,7 +4230,7 @@ version = "0.28.1" source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "as_variant", - "indexmap 2.10.0", + "indexmap 2.11.0", "js_int", "js_option", "percent-encoding", @@ -4186,7 +4242,7 @@ dependencies = [ "serde", "serde_json", "smallvec", - "thiserror 2.0.12", + "thiserror 2.0.16", "tracing", "url", "web-time", @@ -4211,7 +4267,7 @@ dependencies = [ "ruma-events", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.16", "tracing", ] @@ -4221,7 +4277,7 @@ version = "0.9.5" source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "js_int", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -4245,7 +4301,7 @@ dependencies = [ "quote", "ruma-identifiers-validation", "serde", - "syn 2.0.104", + "syn 2.0.106", "toml 0.8.23", ] @@ -4274,7 +4330,7 @@ dependencies = [ "serde_json", "sha2", "subslice", - "thiserror 2.0.12", + "thiserror 2.0.16", ] [[package]] @@ -4305,9 +4361,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" +checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" [[package]] name = "rustc-hash" @@ -4345,7 +4401,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "errno", "libc", "linux-raw-sys 0.4.15", @@ -4358,7 +4414,7 @@ version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "errno", "libc", "linux-raw-sys 0.9.4", @@ -4379,9 +4435,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.29" +version = "0.23.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2491382039b29b9b11ff08b76ff6c97cf287671dbb74f0be44bda389fffe9bd1" +checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" dependencies = [ "aws-lc-rs", "log", @@ -4414,7 +4470,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.2.0", + "security-framework 3.3.0", ] [[package]] @@ -4469,9 +4525,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rustyline-async" @@ -4482,7 +4538,7 @@ dependencies = [ "futures-util", "pin-project", "thingbuf", - "thiserror 2.0.12", + "thiserror 2.0.16", "unicode-segmentation", "unicode-width 0.2.1", ] @@ -4548,7 +4604,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -4557,11 +4613,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" +checksum = "80fb1d92c5028aa318b4b8bd7302a5bfcf48be96a37fc6fc790f806b0004ee0c" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -4592,7 +4648,7 @@ checksum = "989425268ab5c011e06400187eed6c298272f8ef913e49fcadc3fda788b45030" dependencies = [ "httpdate", "reqwest", - "rustls 0.23.29", + "rustls 0.23.31", "sentry-backtrace", "sentry-contexts", "sentry-core", @@ -4693,7 +4749,7 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fac841c7050aa73fc2bec8f7d8e9cb1159af0b3095757b99820823f3e54e5080" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.9.3", "sentry-backtrace", "sentry-core", "tracing-core", @@ -4711,7 +4767,7 @@ dependencies = [ "rand 0.9.2", "serde", "serde_json", - "thiserror 2.0.12", + "thiserror 2.0.16", "time", "url", "uuid", @@ -4734,7 +4790,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -4744,7 +4800,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", - "indexmap 2.10.0", + "indexmap 2.11.0", "itoa", "ryu", "serde", @@ -4752,9 +4808,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.141" +version = "1.0.143" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" dependencies = [ "itoa", "memchr", @@ -4818,7 +4874,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.10.0", + "indexmap 2.11.0", "itoa", "ryu", "serde", @@ -4885,9 +4941,9 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.5" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" dependencies = [ "libc", ] @@ -4924,15 +4980,15 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "smallstr" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b1aefdf380735ff8ded0b15f31aab05daf1f70216c01c02a12926badd1df9d" +checksum = "862077b1e764f04c251fe82a2ef562fd78d7cadaeb072ca7c2bcaf7217b1ff3b" dependencies = [ "serde", "smallvec", @@ -4957,6 +5013,16 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "socket2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -5044,9 +5110,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.104" +version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", @@ -5082,7 +5148,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -5133,7 +5199,7 @@ dependencies = [ "lazy-regex", "minimad", "serde", - "thiserror 2.0.12", + "thiserror 2.0.16", "unicode-width 0.1.14", ] @@ -5158,11 +5224,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "3467d614147380f2e4e374161426ff399c91084acd2363eaf549172b3d5e60c0" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.16", ] [[package]] @@ -5173,18 +5239,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "6c5e1be1c48b9172ee610da68fd9cd2770e7a4056cb3fc98710ee6906f0c7960" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -5310,9 +5376,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" dependencies = [ "tinyvec_macros", ] @@ -5325,9 +5391,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.46.1" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", @@ -5337,10 +5403,10 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2", + "socket2 0.6.0", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5351,14 +5417,14 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] name = "tokio-metrics" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ff82f660c98e4ff60da5eb8fa864a4130f34b56d92d5cd23d6fdfcc14e95fa" +checksum = "7f960dc1df82e5a0cff5a77e986a10ec7bfabf23ff2377922e012af742878e12" dependencies = [ "futures-util", "pin-project-lite", @@ -5382,7 +5448,7 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.23.29", + "rustls 0.23.31", "tokio", ] @@ -5411,9 +5477,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.15" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" dependencies = [ "bytes", "futures-core", @@ -5440,7 +5506,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" dependencies = [ - "indexmap 2.10.0", + "indexmap 2.11.0", "serde", "serde_spanned 1.0.0", "toml_datetime 0.7.0", @@ -5473,7 +5539,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.10.0", + "indexmap 2.11.0", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -5523,7 +5589,7 @@ dependencies = [ "percent-encoding", "pin-project", "prost", - "socket2", + "socket2 0.5.10", "tokio", "tokio-stream", "tower 0.4.13", @@ -5574,7 +5640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "async-compression", - "bitflags 2.9.1", + "bitflags 2.9.3", "bytes", "futures-core", "futures-util", @@ -5619,7 +5685,7 @@ source = "git+https://forgejo.ellis.link/continuwuation/tracing?rev=1e64095a8051 dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -5712,9 +5778,9 @@ checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "typewit" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97e72ba082eeb9da9dc68ff5a2bf727ef6ce362556e8d29ec1aed3bd05e7d86a" +checksum = "4dd91acc53c592cb800c11c83e8e7ee1d48378d05cfa33b5474f5f80c5b236bf" dependencies = [ "typewit_proc_macros", ] @@ -5806,7 +5872,7 @@ dependencies = [ "base64 0.22.1", "log", "percent-encoding", - "rustls 0.23.29", + "rustls 0.23.31", "rustls-pemfile 2.2.0", "rustls-pki-types", "ureq-proto", @@ -5828,9 +5894,9 @@ dependencies = [ [[package]] name = "url" -version = "2.5.4" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", @@ -5858,9 +5924,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" dependencies = [ "getrandom 0.3.3", "js-sys", @@ -5920,11 +5986,11 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" -version = "0.14.2+wasi-0.2.4" +version = "0.14.3+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +checksum = "6a51ae83037bdd272a9e28ce236db8c07016dd0d50c27038b3f407533c030c95" dependencies = [ - "wit-bindgen-rt", + "wit-bindgen", ] [[package]] @@ -5949,7 +6015,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "wasm-bindgen-shared", ] @@ -5984,7 +6050,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6154,7 +6220,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -6165,7 +6231,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -6192,7 +6258,7 @@ checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result", "windows-strings 0.3.1", - "windows-targets 0.53.2", + "windows-targets 0.53.3", ] [[package]] @@ -6255,7 +6321,7 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.2", + "windows-targets 0.53.3", ] [[package]] @@ -6291,10 +6357,11 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.2" +version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ + "windows-link", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", @@ -6454,9 +6521,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] @@ -6472,13 +6539,10 @@ dependencies = [ ] [[package]] -name = "wit-bindgen-rt" -version = "0.39.0" +name = "wit-bindgen" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags 2.9.1", -] +checksum = "052283831dbae3d879dc7f51f3d92703a316ca49f91540417d38591826127814" [[package]] name = "writeable" @@ -6560,7 +6624,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "synstructure 0.13.2", ] @@ -6581,7 +6645,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -6601,7 +6665,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", "synstructure 0.13.2", ] @@ -6624,9 +6688,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.2" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" dependencies = [ "yoke", "zerofrom", @@ -6641,7 +6705,7 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.106", ] [[package]] @@ -6689,9 +6753,9 @@ dependencies = [ [[package]] name = "zune-jpeg" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9e525af0a6a658e031e95f14b7f889976b74a11ba0eca5a5fc9ac8a1c43a6a" +checksum = "fc1f7e205ce79eb2da3cd71c5f55f3589785cb7c79f6a03d1c8d1491bda5d089" dependencies = [ "zune-core", ] diff --git a/Cargo.toml b/Cargo.toml index fe388b4d..7a06df64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ features = ["ffi", "std", "union"] version = "0.6.2" [workspace.dependencies.ctor] -version = "0.2.9" +version = "0.5.0" [workspace.dependencies.cargo_toml] version = "0.22" @@ -536,11 +536,11 @@ version = "0.2" version = "0.2" [workspace.dependencies.minicbor] -version = "0.26.5" +version = "2.1.1" features = ["std"] [workspace.dependencies.minicbor-serde] -version = "0.4.1" +version = "0.6.0" features = ["std"] [workspace.dependencies.maplit] diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 7896ef97..470d3e34 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -89,6 +89,7 @@ serde_yaml.workspace = true tokio.workspace = true tracing-subscriber.workspace = true tracing.workspace = true +ctor.workspace = true [lints] workspace = true diff --git a/src/admin/mod.rs b/src/admin/mod.rs index 1d46590b..1e46dc7f 100644 --- a/src/admin/mod.rs +++ b/src/admin/mod.rs @@ -29,6 +29,8 @@ pub(crate) use crate::{context::Context, utils::get_room_info}; pub(crate) const PAGE_SIZE: usize = 100; +use ctor::{ctor, dtor}; + conduwuit::mod_ctor! {} conduwuit::mod_dtor! {} conduwuit::rustc_flags_capture! {} diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 9b4ea460..18a2ddbd 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -93,6 +93,7 @@ serde.workspace = true sha1.workspace = true tokio.workspace = true tracing.workspace = true +ctor.workspace = true [lints] workspace = true diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 4d489c2f..f5a951f4 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -321,7 +321,7 @@ pub(crate) fn event_filter(item: PdusIterItem, filter: &RoomEventFilter) -> Opti filter.matches(pdu).then_some(item) } -#[cfg_attr(debug_assertions, conduwuit::ctor)] +#[cfg_attr(debug_assertions, ctor::ctor)] fn _is_sorted() { debug_assert!( IGNORED_MESSAGE_TYPES.is_sorted(), diff --git a/src/database/Cargo.toml b/src/database/Cargo.toml index 9f51f366..d37ca585 100644 --- a/src/database/Cargo.toml +++ b/src/database/Cargo.toml @@ -66,6 +66,7 @@ serde.workspace = true serde_json.workspace = true tokio.workspace = true tracing.workspace = true +ctor.workspace = true [lints] workspace = true diff --git a/src/database/mod.rs b/src/database/mod.rs index ffcefee9..7932fbcb 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -3,6 +3,8 @@ extern crate conduwuit_core as conduwuit; extern crate rust_rocksdb as rocksdb; +use ctor::{ctor, dtor}; + conduwuit::mod_ctor! {} conduwuit::mod_dtor! {} conduwuit::rustc_flags_capture! {} diff --git a/src/macros/rustc.rs b/src/macros/rustc.rs index cf935fe5..f302b152 100644 --- a/src/macros/rustc.rs +++ b/src/macros/rustc.rs @@ -13,13 +13,13 @@ pub(super) fn flags_capture(args: TokenStream) -> TokenStream { let ret = quote! { pub static RUSTC_FLAGS: [&str; #flag_len] = [#( #flag ),*]; - #[conduwuit_core::ctor] + #[ctor] fn _set_rustc_flags() { conduwuit_core::info::rustc::FLAGS.lock().insert(#crate_name, &RUSTC_FLAGS); } // static strings have to be yanked on module unload - #[conduwuit_core::dtor] + #[dtor] fn _unset_rustc_flags() { conduwuit_core::info::rustc::FLAGS.lock().remove(#crate_name); } diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 8f6f4341..44b4a2ef 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -202,6 +202,7 @@ clap.workspace = true console-subscriber.optional = true console-subscriber.workspace = true const-str.workspace = true +ctor.workspace = true log.workspace = true opentelemetry-jaeger.optional = true opentelemetry-jaeger.workspace = true diff --git a/src/main/mod.rs b/src/main/mod.rs index 7097c67d..ba59549c 100644 --- a/src/main/mod.rs +++ b/src/main/mod.rs @@ -13,6 +13,7 @@ mod sentry; mod server; mod signal; +use ctor::{ctor, dtor}; use server::Server; rustc_flags_capture! {} diff --git a/src/router/Cargo.toml b/src/router/Cargo.toml index 9fcb8d6a..9545c480 100644 --- a/src/router/Cargo.toml +++ b/src/router/Cargo.toml @@ -125,6 +125,7 @@ tokio.workspace = true tower.workspace = true tower-http.workspace = true tracing.workspace = true +ctor.workspace = true [target.'cfg(all(unix, target_os = "linux"))'.dependencies] sd-notify.workspace = true diff --git a/src/router/mod.rs b/src/router/mod.rs index 7038c5df..416ceea7 100644 --- a/src/router/mod.rs +++ b/src/router/mod.rs @@ -12,6 +12,7 @@ use std::{panic::AssertUnwindSafe, pin::Pin, sync::Arc}; use conduwuit::{Error, Result, Server}; use conduwuit_service::Services; +use ctor::{ctor, dtor}; use futures::{Future, FutureExt, TryFutureExt}; conduwuit::mod_ctor! {} diff --git a/src/service/Cargo.toml b/src/service/Cargo.toml index 6e538f40..e4abb9bb 100644 --- a/src/service/Cargo.toml +++ b/src/service/Cargo.toml @@ -117,6 +117,7 @@ webpage.optional = true blurhash.workspace = true blurhash.optional = true recaptcha-verify = { version = "0.1.5", default-features = false } +ctor.workspace = true [lints] workspace = true diff --git a/src/service/mod.rs b/src/service/mod.rs index 3d7a3aa9..2ad0ecd2 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -33,6 +33,7 @@ pub mod users; extern crate conduwuit_core as conduwuit; extern crate conduwuit_database as database; +use ctor::{ctor, dtor}; pub(crate) use service::{Args, Dep, Service}; pub use crate::services::Services; From c0e3829fedc9330eaa39a9817b31bb6dbea72961 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 6 Sep 2025 16:05:13 +0100 Subject: [PATCH 2056/2091] feat: Replace Jaeger with OTLP --- Cargo.lock | 164 +++++++++++++++++------------------------ Cargo.toml | 28 ++----- conduwuit-example.toml | 14 +++- src/core/config/mod.rs | 19 +++-- src/main/Cargo.toml | 10 ++- src/main/logging.rs | 35 +++++---- 6 files changed, 125 insertions(+), 145 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3c24b178..9e56ad45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -922,9 +922,10 @@ dependencies = [ "ctor", "hardened_malloc-rs", "log", - "opentelemetry 0.30.0", - "opentelemetry-jaeger", - "opentelemetry_sdk 0.30.0", + "opentelemetry", + "opentelemetry-jaeger-propagator", + "opentelemetry-otlp", + "opentelemetry_sdk", "sentry", "sentry-tower", "sentry-tracing", @@ -1187,7 +1188,7 @@ dependencies = [ "futures-core", "prost", "prost-types", - "tonic", + "tonic 0.12.3", "tracing-core", ] @@ -1211,7 +1212,7 @@ dependencies = [ "thread_local", "tokio", "tokio-stream", - "tonic", + "tonic 0.12.3", "tracing", "tracing-core", "tracing-subscriber", @@ -2551,12 +2552,6 @@ version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "interpolate_name" version = "0.2.4" @@ -3283,20 +3278,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" -[[package]] -name = "opentelemetry" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b69a91d4893e713e06f724597ad630f1fa76057a5e1026c0ca67054a9032a76" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror 1.0.69", -] - [[package]] name = "opentelemetry" version = "0.30.0" @@ -3312,46 +3293,54 @@ dependencies = [ ] [[package]] -name = "opentelemetry-jaeger" -version = "0.22.0" +name = "opentelemetry-http" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501b471b67b746d9a07d4c29f8be00f952d1a2eca356922ede0098cbaddff19f" +checksum = "50f6639e842a97dbea8886e3439710ae463120091e2e064518ba8e716e6ac36d" dependencies = [ "async-trait", - "futures-core", - "futures-util", - "opentelemetry 0.23.0", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk 0.23.0", - "thrift", - "tokio", + "bytes", + "http", + "opentelemetry", + "reqwest", ] [[package]] -name = "opentelemetry-semantic-conventions" -version = "0.15.0" +name = "opentelemetry-jaeger-propagator" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1869fb4bb9b35c5ba8a1e40c9b128a7b4c010d07091e864a29da19e4fe2ca4d7" +checksum = "090b8ec07bb2e304b529581aa1fe530d7861298c9ef549ebbf44a4a56472c539" +dependencies = [ + "opentelemetry", +] [[package]] -name = "opentelemetry_sdk" -version = "0.23.0" +name = "opentelemetry-otlp" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae312d58eaa90a82d2e627fd86e075cf5230b3f11794e2ed74199ebbe572d4fd" +checksum = "dbee664a43e07615731afc539ca60c6d9f1a9425e25ca09c57bc36c87c55852b" dependencies = [ - "async-trait", - "futures-channel", - "futures-executor", - "futures-util", - "lazy_static", - "once_cell", - "opentelemetry 0.23.0", - "ordered-float 4.6.0", - "percent-encoding", - "rand 0.8.5", - "thiserror 1.0.69", - "tokio", - "tokio-stream", + "http", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost", + "reqwest", + "thiserror 2.0.16", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic 0.13.1", ] [[package]] @@ -3363,7 +3352,7 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry 0.30.0", + "opentelemetry", "percent-encoding", "rand 0.9.2", "serde_json", @@ -3372,24 +3361,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - -[[package]] -name = "ordered-float" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" -dependencies = [ - "num-traits", -] - [[package]] name = "os_info" version = "3.12.0" @@ -5272,28 +5243,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "threadpool" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" -dependencies = [ - "num_cpus", -] - -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "log", - "ordered-float 2.10.1", - "threadpool", -] - [[package]] name = "tiff" version = "0.9.1" @@ -5598,6 +5547,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "http", + "http-body", + "http-body-util", + "percent-encoding", + "pin-project", + "prost", + "tokio-stream", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.4.13" @@ -5737,8 +5707,8 @@ checksum = "ddcf5959f39507d0d04d6413119c04f33b623f4f951ebcbdddddfad2d0623a9c" dependencies = [ "js-sys", "once_cell", - "opentelemetry 0.30.0", - "opentelemetry_sdk 0.30.0", + "opentelemetry", + "opentelemetry_sdk", "smallvec", "tracing", "tracing-core", diff --git a/Cargo.toml b/Cargo.toml index 7a06df64..12ba6456 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -423,9 +423,12 @@ version = "0.31.0" version = "0.30.0" features = ["rt-tokio"] -[workspace.dependencies.opentelemetry-jaeger] -version = "0.22.0" -features = ["rt-tokio"] +[workspace.dependencies.opentelemetry-otlp] +version = "0.30.0" +features = ["http", "trace", "logs", "metrics"] + +[workspace.dependencies.opentelemetry-jaeger-propagator] +version = "0.30.0" # optional sentry metrics for crash/panic reporting [workspace.dependencies.sentry] @@ -764,25 +767,6 @@ incremental = true [profile.dev.package.conduwuit_core] inherits = "dev" -#rustflags = [ -# '--cfg', 'conduwuit_mods', -# '-Ztime-passes', -# '-Zmir-opt-level=0', -# '-Ztls-model=initial-exec', -# '-Cprefer-dynamic=true', -# '-Zstaticlib-prefer-dynamic=true', -# '-Zstaticlib-allow-rdylib-deps=true', -# '-Zpacked-bundled-libs=false', -# '-Zplt=true', -# '-Clink-arg=-Wl,--as-needed', -# '-Clink-arg=-Wl,--allow-shlib-undefined', -# '-Clink-arg=-Wl,-z,lazy', -# '-Clink-arg=-Wl,-z,unique', -# '-Clink-arg=-Wl,-z,nodlopen', -# '-Clink-arg=-Wl,-z,nodelete', -#] -[profile.dev.package.xtask-generate-commands] -inherits = "dev" [profile.dev.package.conduwuit] inherits = "dev" #rustflags = [ diff --git a/conduwuit-example.toml b/conduwuit-example.toml index 0fc034d0..07374aae 100644 --- a/conduwuit-example.toml +++ b/conduwuit-example.toml @@ -591,13 +591,19 @@ # #default_room_version = 11 -# This item is undocumented. Please contribute documentation for it. +# Enable OpenTelemetry OTLP tracing export. This replaces the deprecated +# Jaeger exporter. Traces will be sent via OTLP to a collector (such as +# Jaeger) that supports the OpenTelemetry Protocol. # -#allow_jaeger = false +# Configure your OTLP endpoint using the OTEL_EXPORTER_OTLP_ENDPOINT +# environment variable (defaults to http://localhost:4318). +# +#allow_otlp = false -# This item is undocumented. Please contribute documentation for it. +# Filter for OTLP tracing spans. This controls which spans are exported +# to the OTLP collector. # -#jaeger_filter = "info" +#otlp_filter = "info" # If the 'perf_measurements' compile-time feature is enabled, enables # collecting folded stack trace profile of tracing spans using diff --git a/src/core/config/mod.rs b/src/core/config/mod.rs index e9e6d28d..b6f6ab53 100644 --- a/src/core/config/mod.rs +++ b/src/core/config/mod.rs @@ -714,12 +714,21 @@ pub struct Config { #[serde(default)] pub well_known: WellKnownConfig, - #[serde(default)] - pub allow_jaeger: bool, + /// Enable OpenTelemetry OTLP tracing export. This replaces the deprecated + /// Jaeger exporter. Traces will be sent via OTLP to a collector (such as + /// Jaeger) that supports the OpenTelemetry Protocol. + /// + /// Configure your OTLP endpoint using the OTEL_EXPORTER_OTLP_ENDPOINT + /// environment variable (defaults to http://localhost:4318). + #[serde(default, alias = "allow_jaeger")] + pub allow_otlp: bool, + /// Filter for OTLP tracing spans. This controls which spans are exported + /// to the OTLP collector. + /// /// default: "info" - #[serde(default = "default_jaeger_filter")] - pub jaeger_filter: String, + #[serde(default = "default_otlp_filter", alias = "jaeger_filter")] + pub otlp_filter: String, /// If the 'perf_measurements' compile-time feature is enabled, enables /// collecting folded stack trace profile of tracing spans using @@ -2367,7 +2376,7 @@ fn default_tracing_flame_filter() -> String { .to_owned() } -fn default_jaeger_filter() -> String { +fn default_otlp_filter() -> String { cfg!(debug_assertions) .then_some("trace,h2=off") .unwrap_or("info") diff --git a/src/main/Cargo.toml b/src/main/Cargo.toml index 44b4a2ef..388ad503 100644 --- a/src/main/Cargo.toml +++ b/src/main/Cargo.toml @@ -126,7 +126,8 @@ perf_measurements = [ "dep:tracing-flame", "dep:tracing-opentelemetry", "dep:opentelemetry_sdk", - "dep:opentelemetry-jaeger", + "dep:opentelemetry-otlp", + "dep:opentelemetry-jaeger-propagator", "conduwuit-core/perf_measurements", "conduwuit-core/sentry_telemetry", ] @@ -204,10 +205,12 @@ console-subscriber.workspace = true const-str.workspace = true ctor.workspace = true log.workspace = true -opentelemetry-jaeger.optional = true -opentelemetry-jaeger.workspace = true opentelemetry.optional = true opentelemetry.workspace = true +opentelemetry-otlp.optional = true +opentelemetry-otlp.workspace = true +opentelemetry-jaeger-propagator.optional = true +opentelemetry-jaeger-propagator.workspace = true opentelemetry_sdk.optional = true opentelemetry_sdk.workspace = true sentry-tower.optional = true @@ -227,6 +230,7 @@ tracing-subscriber.workspace = true tracing.workspace = true tracing-journald = { workspace = true, optional = true } + [target.'cfg(all(not(target_env = "msvc"), target_os = "linux"))'.dependencies] hardened_malloc-rs.workspace = true hardened_malloc-rs.optional = true diff --git a/src/main/logging.rs b/src/main/logging.rs index b7beb103..57b56707 100644 --- a/src/main/logging.rs +++ b/src/main/logging.rs @@ -7,6 +7,8 @@ use conduwuit_core::{ log::{ConsoleFormat, ConsoleWriter, LogLevelReloadHandles, capture, fmt_span}, result::UnwrapOrErr, }; +#[cfg(feature = "perf_measurements")] +use opentelemetry::trace::TracerProvider; use tracing_subscriber::{EnvFilter, Layer, Registry, fmt, layer::SubscriberExt, reload}; #[cfg(feature = "perf_measurements")] @@ -87,30 +89,35 @@ pub(crate) fn init( (None, None) }; - let jaeger_filter = EnvFilter::try_new(&config.jaeger_filter) - .map_err(|e| err!(Config("jaeger_filter", "{e}.")))?; + let otlp_filter = EnvFilter::try_new(&config.otlp_filter) + .map_err(|e| err!(Config("otlp_filter", "{e}.")))?; - let jaeger_layer = config.allow_jaeger.then(|| { + let otlp_layer = config.allow_otlp.then(|| { opentelemetry::global::set_text_map_propagator( - opentelemetry_jaeger::Propagator::new(), + opentelemetry_jaeger_propagator::Propagator::new(), ); - let tracer = opentelemetry_jaeger::new_agent_pipeline() - .with_auto_split_batch(true) - .with_service_name(conduwuit_core::name()) - .install_batch(opentelemetry_sdk::runtime::Tokio) - .expect("jaeger agent pipeline"); + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_http() + .build() + .expect("Failed to create OTLP exporter"); + + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_batch_exporter(exporter) + .build(); + + let tracer = provider.tracer(conduwuit_core::name()); let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let (jaeger_reload_filter, jaeger_reload_handle) = - reload::Layer::new(jaeger_filter.clone()); - reload_handles.add("jaeger", Box::new(jaeger_reload_handle)); + let (otlp_reload_filter, otlp_reload_handle) = + reload::Layer::new(otlp_filter.clone()); + reload_handles.add("otlp", Box::new(otlp_reload_handle)); - Some(telemetry.with_filter(jaeger_reload_filter)) + Some(telemetry.with_filter(otlp_reload_filter)) }); - let subscriber = subscriber.with(flame_layer).with(jaeger_layer); + let subscriber = subscriber.with(flame_layer).with(otlp_layer); (subscriber, flame_guard) }; From cd238b05dee3e1abbfc07af33a8fe0d7a2307be4 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Sat, 6 Sep 2025 16:21:21 +0100 Subject: [PATCH 2057/2091] fix: Remove bad colon in workflow --- .forgejo/workflows/release-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 2b3d481f..7b29b7ca 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -15,7 +15,7 @@ on: - "docs/**" push: branches: - - main: + - main paths-ignore: - "*.md" - "**/*.md" From 969d7cbb66bb56709e325f9cdcf8d9a4af11269c Mon Sep 17 00:00:00 2001 From: aviac Date: Tue, 26 Aug 2025 16:00:06 +0200 Subject: [PATCH 2058/2091] feat(nix): remove rocksdb from flake.nix inputs Consuming this flake is pretty annoying since the rocksdb input is fetched on every build which takes ~ 10 - 20 sec. By removing it and replacing it with a `pkgs.fetchFromGitea`, we create an intermediate derivation which is better for caching reasons. --- flake.lock | 20 +------------------- flake.nix | 12 +++++++----- 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/flake.lock b/flake.lock index 4c2bf9fb..f1859b57 100644 --- a/flake.lock +++ b/flake.lock @@ -513,23 +513,6 @@ "type": "github" } }, - "rocksdb": { - "flake": false, - "locked": { - "lastModified": 1753385396, - "narHash": "sha256-/Hvy1yTH/0D5aa7bc+/uqFugCQq4InTdwlRw88vA5IY=", - "ref": "10.4.fb", - "rev": "28d4b7276c16ed3e28af1bd96162d6442ce25923", - "revCount": 13318, - "type": "git", - "url": "https://forgejo.ellis.link/continuwuation/rocksdb" - }, - "original": { - "ref": "10.4.fb", - "type": "git", - "url": "https://forgejo.ellis.link/continuwuation/rocksdb" - } - }, "root": { "inputs": { "attic": "attic", @@ -539,8 +522,7 @@ "flake-compat": "flake-compat_3", "flake-utils": "flake-utils", "nix-filter": "nix-filter", - "nixpkgs": "nixpkgs_5", - "rocksdb": "rocksdb" + "nixpkgs": "nixpkgs_5" } }, "rust-analyzer-src": { diff --git a/flake.nix b/flake.nix index e65fcbda..b8ac6029 100644 --- a/flake.nix +++ b/flake.nix @@ -16,10 +16,6 @@ flake-utils.url = "github:numtide/flake-utils?ref=main"; nix-filter.url = "github:numtide/nix-filter?ref=main"; nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable"; - rocksdb = { - url = "git+https://forgejo.ellis.link/continuwuation/rocksdb?ref=10.4.fb"; - flake = false; - }; }; outputs = @@ -65,7 +61,13 @@ inherit (self) liburing; }).overrideAttrs (old: { - src = inputs.rocksdb; + src = pkgsHost.fetchFromGitea { + domain = "forgejo.ellis.link"; + owner = "continuwuation"; + repo = "rocksdb"; + rev = "10.4.fb"; + sha256 = "sha256-/Hvy1yTH/0D5aa7bc+/uqFugCQq4InTdwlRw88vA5IY="; + }; version = "v10.4.fb"; cmakeFlags = pkgs.lib.subtractLists [ From 1a3107c20ad473e4f5b2cc7ac702dd1a59ce7a70 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 7 Sep 2025 09:28:59 +0100 Subject: [PATCH 2059/2091] fix(ci): Replace Mozilla sccache action with token-free alternative Replace mozilla-actions/sccache-action with a custom Forgejo-specific implementation that eliminates GitHub token dependencies and rate limiting issues for all contributors regardless of repository permissions. The new action mirrors sccache binaries to the Forgejo package registry and queries that instead of GitHub releases, maintaining identical functionality including hostedtoolcache support. --- .forgejo/actions/sccache/action.yml | 8 +------- .forgejo/actions/setup-rust/action.yml | 12 +----------- 2 files changed, 2 insertions(+), 18 deletions(-) diff --git a/.forgejo/actions/sccache/action.yml b/.forgejo/actions/sccache/action.yml index b5e5dcf4..b2441109 100644 --- a/.forgejo/actions/sccache/action.yml +++ b/.forgejo/actions/sccache/action.yml @@ -2,18 +2,12 @@ name: sccache description: | Install sccache for caching builds in GitHub Actions. -inputs: - token: - description: 'A Github PAT' - required: false runs: using: composite steps: - name: Install sccache - uses: https://github.com/mozilla-actions/sccache-action@v0.0.9 - with: - token: ${{ inputs.token }} + uses: https://git.tomfos.tr/tom/sccache-action@v1 - name: Configure sccache uses: https://github.com/actions/github-script@v7 with: diff --git a/.forgejo/actions/setup-rust/action.yml b/.forgejo/actions/setup-rust/action.yml index 091da8c2..a8736a75 100644 --- a/.forgejo/actions/setup-rust/action.yml +++ b/.forgejo/actions/setup-rust/action.yml @@ -88,19 +88,9 @@ runs: # Shared toolchain cache across all Rust versions key: toolchain-${{ steps.runner-os.outputs.slug }} - - name: Debug GitHub token availability - shell: bash - run: | - if [ -z "${{ inputs.github-token }}" ]; then - echo "⚠️ No GitHub token provided - sccache will use fallback download method" - else - echo "✅ GitHub token provided for sccache" - fi - name: Setup sccache - uses: https://github.com/mozilla-actions/sccache-action@v0.0.9 - with: - token: ${{ inputs.github-token }} + uses: https://git.tomfos.tr/tom/sccache-action@v1 - name: Cache build artifacts id: build-cache From fff9629b0f4d28148007f180f8c52807894b7c2b Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 7 Sep 2025 13:21:58 +0100 Subject: [PATCH 2060/2091] fix(docker): Resolve liburing.so.2 loading error for non-root users Container failed to start when running as non-root (user 1000:1000) because copied directories had restrictive 770 permissions, likely due to different umask in persistent BuildKit. Non-root users couldn't access /usr/lib to load required dynamic libraries. Introduces prepper stage using Ubuntu to organize files into layered structure with explicit 755 directory permissions before copying to scratch image. Also fixes workflow syntax error and removes docker/** from paths-ignore to ensure Docker changes trigger CI builds. --- .forgejo/workflows/release-image.yml | 4 +- docker/Dockerfile | 57 ++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 7b29b7ca..834b5602 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -11,7 +11,6 @@ on: - ".gitignore" - "renovate.json" - "pkg/**" - - "docker/**" - "docs/**" push: branches: @@ -23,7 +22,6 @@ on: - ".gitignore" - "renovate.json" - "pkg/**" - - "docker/**" - "docs/**" # Allows you to run this workflow manually from the Actions tab workflow_dispatch: @@ -199,7 +197,7 @@ jobs: context: . file: "docker/Dockerfile" build-args: | - GIT_COMMIT_HASH=${{ github.sha }}) + GIT_COMMIT_HASH=${{ github.sha }} GIT_COMMIT_HASH_SHORT=${{ env.COMMIT_SHORT_SHA }} GIT_REMOTE_URL=${{github.event.repository.html_url }} GIT_REMOTE_COMMIT_URL=${{github.event.head_commit.url }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 55150902..17e1c6a1 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -199,32 +199,57 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ EOF # Extract dynamically linked dependencies -RUN </dev/null) && [ -n "$lddtree_output" ]; then + echo "$lddtree_output" | awk '{print $(NF-0) " " $1}' | sort -u -k 1,1 | \ + awk '{dest = ($2 ~ /^\//) ? "/out/libs-root" $2 : "/out/libs/" $2; print "install -D " $1 " " dest}' | \ + while read cmd; do eval "$cmd"; done + fi done -EOF + + # Show what will be copied to runtime + echo "=== Libraries being copied to runtime image:" + find /out/libs* -type f 2>/dev/null | sort || echo "No libraries found" +DEPS_EOF + +FROM ubuntu:latest AS prepper + +# Create layer structure +RUN mkdir -p /layer1/etc/ssl/certs \ + /layer2/usr/lib \ + /layer3/sbin /layer3/sbom + +# Copy SSL certs and root-path libraries to layer1 (ultra-stable) +COPY --from=base /etc/ssl/certs /layer1/etc/ssl/certs +COPY --from=builder /out/libs-root/ /layer1/ + +# Copy application libraries to layer2 (semi-stable) +COPY --from=builder /out/libs/ /layer2/usr/lib/ + +# Copy binaries and SBOM to layer3 (volatile) +COPY --from=builder /out/sbin/ /layer3/sbin/ +COPY --from=builder /out/sbom/ /layer3/sbom/ + +# Fix permissions after copying +RUN chmod -R 755 /layer1 /layer2 /layer3 FROM scratch WORKDIR / -# Copy root certs for tls into image -# You can also mount the certs from the host -# --volume /etc/ssl/certs:/etc/ssl/certs:ro -COPY --from=base /etc/ssl/certs /etc/ssl/certs +# Copy ultra-stable layer (SSL certs, system libraries) +COPY --from=prepper /layer1/ / -# Copy our build -COPY --from=builder /out/sbin/ /sbin/ -# Copy SBOM -COPY --from=builder /out/sbom/ /sbom/ +# Copy semi-stable layer (application libraries) +COPY --from=prepper /layer2/ / -# Copy dynamic libraries to root -COPY --from=builder /out/libs-root/ / -COPY --from=builder /out/libs/ /usr/lib/ +# Copy volatile layer (binaries, SBOM) +COPY --from=prepper /layer3/ / # Inform linker where to find libraries ENV LD_LIBRARY_PATH=/usr/lib From d640853f9d3c5cd33fc14775090d634747091bdc Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 7 Sep 2025 14:32:11 +0100 Subject: [PATCH 2061/2091] ci(docs): Optimise build performance with caching and conditional Node.js Skip installing Node.js entirely if v20+ is already available, otherwise install v22. Add npm dependency caching with OS-specific cache keys using the custom detect-runner-os action for proper cache isolation between runners. Dependencies normally take just under 10s, so this should more than halve the doc build time to free up runner slots. --- .forgejo/actions/detect-runner-os/action.yml | 21 +++++++++++++++++++- .forgejo/workflows/documentation.yml | 15 +++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/.forgejo/actions/detect-runner-os/action.yml b/.forgejo/actions/detect-runner-os/action.yml index 6ada1d5d..be8f9b0a 100644 --- a/.forgejo/actions/detect-runner-os/action.yml +++ b/.forgejo/actions/detect-runner-os/action.yml @@ -13,6 +13,12 @@ outputs: slug: description: 'Combined OS slug (e.g. Ubuntu-22.04)' value: ${{ steps.detect.outputs.slug }} + node_major: + description: 'Major version of Node.js if available (e.g. 22)' + value: ${{ steps.detect.outputs.node_major }} + node_version: + description: 'Full Node.js version if available (e.g. 22.19.0)' + value: ${{ steps.detect.outputs.node_version }} runs: using: composite @@ -30,7 +36,20 @@ runs: # Create combined slug OS_SLUG="${OS_NAME}-${OS_VERSION}" - # Set outputs + # Detect Node.js version if available + if command -v node >/dev/null 2>&1; then + NODE_VERSION=$(node --version | sed 's/v//') + NODE_MAJOR=$(echo $NODE_VERSION | cut -d. -f1) + echo "node_version=${NODE_VERSION}" >> $GITHUB_OUTPUT + echo "node_major=${NODE_MAJOR}" >> $GITHUB_OUTPUT + echo "🔍 Detected Node.js: v${NODE_VERSION}" + else + echo "node_version=" >> $GITHUB_OUTPUT + echo "node_major=" >> $GITHUB_OUTPUT + echo "🔍 Node.js not found" + fi + + # Set OS outputs echo "name=${OS_NAME}" >> $GITHUB_OUTPUT echo "version=${OS_VERSION}" >> $GITHUB_OUTPUT echo "slug=${OS_SLUG}" >> $GITHUB_OUTPUT diff --git a/.forgejo/workflows/documentation.yml b/.forgejo/workflows/documentation.yml index 4f3e903c..67c8a30c 100644 --- a/.forgejo/workflows/documentation.yml +++ b/.forgejo/workflows/documentation.yml @@ -49,10 +49,23 @@ jobs: cp ./docs/static/_headers ./public/_headers echo "Copied .well-known files and _headers to ./public" + - name: Detect runner environment + id: runner-env + uses: ./.forgejo/actions/detect-runner-os + - name: Setup Node.js + if: steps.runner-env.outputs.node_major == '' || steps.runner-env.outputs.node_major < '20' uses: https://github.com/actions/setup-node@v4 with: - node-version: 20 + node-version: 22 + + - name: Cache npm dependencies + uses: actions/cache@v3 + with: + path: ~/.npm + key: ${{ steps.runner-env.outputs.slug }}-node-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ steps.runner-env.outputs.slug }}-node- - name: Install dependencies run: npm install --save-dev wrangler@latest From 84fdcd326a4076bad19dc62e0061a15cafe54158 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 7 Sep 2025 17:08:36 +0100 Subject: [PATCH 2062/2091] fix(ci): Resolve registry push failures for fork PRs Fork PRs now fail during Docker image build with 'tag is needed when pushing to registry' because BUILTIN_REGISTRY_ENABLED evaluates to false without proper credentials, leaving the images list empty. This appears to be due to recent Forgejo permission changes affecting fork access to repository secrets. Add fallback to official registry when credentials unavailable, skip registry login and push operations for forks, and make merge job conditional since no digests exist without push. This allows forks to test Docker builds whilst avoiding authentication failures. --- .forgejo/workflows/release-image.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 834b5602..069f1d34 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -53,6 +53,9 @@ jobs: let images = [] if (process.env.BUILTIN_REGISTRY_ENABLED === "true") { images.push(builtinImage) + } else { + // Fallback to official registry for forks/PRs without credentials + images.push('forgejo.ellis.link/continuwuation/continuwuity') } core.setOutput('images', images.join("\n")) core.setOutput('images_list', images.join(",")) @@ -111,6 +114,7 @@ jobs: uses: docker/setup-qemu-action@v3 # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Login to builtin registry + if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }} uses: docker/login-action@v3 with: registry: ${{ env.BUILTIN_REGISTRY }} @@ -207,7 +211,7 @@ jobs: cache-from: type=gha # cache-to: type=gha,mode=max sbom: true - outputs: type=image,"name=${{ needs.define-variables.outputs.images_list }}",push-by-digest=true,name-canonical=true,push=true + outputs: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' && format('type=image,"name={0}",push-by-digest=true,name-canonical=true,push=true', needs.define-variables.outputs.images_list) || 'type=docker' }} env: SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} @@ -249,6 +253,7 @@ jobs: needs: [define-variables, build-image] steps: - name: Download digests + if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }} uses: forgejo/download-artifact@v4 with: path: /tmp/digests @@ -256,6 +261,7 @@ jobs: merge-multiple: true # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Login to builtin registry + if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }} uses: docker/login-action@v3 with: registry: ${{ env.BUILTIN_REGISTRY }} @@ -263,6 +269,7 @@ jobs: password: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx + if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }} uses: docker/setup-buildx-action@v3 with: # Use persistent BuildKit if BUILDKIT_ENDPOINT is set (e.g. tcp://buildkit:8125) @@ -270,6 +277,7 @@ jobs: endpoint: ${{ env.BUILDKIT_ENDPOINT || '' }} - name: Extract metadata (tags) for Docker + if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }} id: meta uses: docker/metadata-action@v5 with: @@ -287,6 +295,7 @@ jobs: DOCKER_METADATA_ANNOTATIONS_LEVELS: index - name: Create manifest list and push + if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }} working-directory: /tmp/digests env: IMAGES: ${{needs.define-variables.outputs.images}} @@ -304,6 +313,7 @@ jobs: done - name: Inspect image + if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }} env: IMAGES: ${{needs.define-variables.outputs.images}} shell: bash From 2cedf0d2e1346769b023cd6e42ed3279142af5de Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 7 Sep 2025 18:32:38 +0100 Subject: [PATCH 2063/2091] fix(ci): Use image output instead of docker for fork PRs Docker exporter doesn't support manifest lists (multi-platform builds). For fork PRs without registry credentials, use 'type=image,push=false' instead of 'type=docker' to build multi-platform images locally without pushing. --- .forgejo/workflows/release-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 069f1d34..52f5e6e0 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -211,7 +211,7 @@ jobs: cache-from: type=gha # cache-to: type=gha,mode=max sbom: true - outputs: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' && format('type=image,"name={0}",push-by-digest=true,name-canonical=true,push=true', needs.define-variables.outputs.images_list) || 'type=docker' }} + outputs: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' && format('type=image,"name={0}",push-by-digest=true,name-canonical=true,push=true', needs.define-variables.outputs.images_list) || format('type=image,"name={0}",push=false', needs.define-variables.outputs.images_list) }} env: SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} From 1e9701f379053b6f20681b48db6ede5026db4681 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 7 Sep 2025 18:59:05 +0100 Subject: [PATCH 2064/2091] ci(release-image): Skip setup steps when using persistent BuildKit When BUILDKIT_ENDPOINT is set, builds run on a persistent BuildKit instance, making runner setup steps unnecessary. Skip Rust toolchain installation, QEMU setup, caching steps, and timelord to eliminate ~7 operations per job. Also adds output to git SHA and timestamp steps for visibility. Cuts at least a minute off average build time through fewer installs, cache restores, and cache saves. --- .forgejo/workflows/release-image.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 52f5e6e0..1a0e4f4e 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -101,6 +101,7 @@ jobs: with: persist-credentials: false - name: Install rust + if: ${{ env.BUILDKIT_ENDPOINT == '' }} id: rust-toolchain uses: ./.forgejo/actions/rust-toolchain @@ -111,6 +112,7 @@ jobs: driver: ${{ env.BUILDKIT_ENDPOINT != '' && 'remote' || 'docker-container' }} endpoint: ${{ env.BUILDKIT_ENDPOINT || '' }} - name: Set up QEMU + if: ${{ env.BUILDKIT_ENDPOINT == '' }} uses: docker/setup-qemu-action@v3 # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Login to builtin registry @@ -140,15 +142,21 @@ jobs: run: | calculatedSha=$(git rev-parse --short ${{ github.sha }}) echo "COMMIT_SHORT_SHA=$calculatedSha" >> $GITHUB_ENV + echo "Short SHA: $calculatedSha" - name: Get Git commit timestamps - run: echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV + run: | + timestamp=$(git log -1 --pretty=%ct) + echo "TIMESTAMP=$timestamp" >> $GITHUB_ENV + echo "Commit timestamp: $timestamp" - uses: ./.forgejo/actions/timelord + if: ${{ env.BUILDKIT_ENDPOINT == '' }} with: key: timelord-v0 path: . - name: Cache Rust registry + if: ${{ env.BUILDKIT_ENDPOINT == '' }} uses: actions/cache@v3 with: path: | @@ -158,6 +166,7 @@ jobs: .cargo/registry/src key: rust-registry-image-${{hashFiles('**/Cargo.lock') }} - name: Cache cargo target + if: ${{ env.BUILDKIT_ENDPOINT == '' }} id: cache-cargo-target uses: actions/cache@v3 with: @@ -165,6 +174,7 @@ jobs: cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }} key: cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}-${{hashFiles('**/Cargo.lock') }}-${{steps.rust-toolchain.outputs.rustc_version}} - name: Cache apt cache + if: ${{ env.BUILDKIT_ENDPOINT == '' }} id: cache-apt uses: actions/cache@v3 with: @@ -172,6 +182,7 @@ jobs: var-cache-apt-${{ matrix.slug }} key: var-cache-apt-${{ matrix.slug }} - name: Cache apt lib + if: ${{ env.BUILDKIT_ENDPOINT == '' }} id: cache-apt-lib uses: actions/cache@v3 with: @@ -179,6 +190,7 @@ jobs: var-lib-apt-${{ matrix.slug }} key: var-lib-apt-${{ matrix.slug }} - name: inject cache into docker + if: ${{ env.BUILDKIT_ENDPOINT == '' }} uses: https://github.com/reproducible-containers/buildkit-cache-dance@v3.3.0 with: cache-map: | From 4a1091dd06e3e4518edb4a3df296e7a28fef0da9 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 7 Sep 2025 20:06:39 +0100 Subject: [PATCH 2065/2091] ci(release-image): Unify binary extraction using BuildKit local output Fork PRs currently fail binary extraction with 'invalid reference format' and 'must specify at least one container source' errors. This replaces the registry-specific docker create/copy method with BuildKit's local output feature for all builds. Uses multiple outputs in single build: image export plus local binary extraction from /sbin. Speeds up extracting binary artifacts and saves a couple of extra workflow steps in the process. --- .forgejo/workflows/release-image.yml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 1a0e4f4e..2a722edd 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -223,27 +223,23 @@ jobs: cache-from: type=gha # cache-to: type=gha,mode=max sbom: true - outputs: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' && format('type=image,"name={0}",push-by-digest=true,name-canonical=true,push=true', needs.define-variables.outputs.images_list) || format('type=image,"name={0}",push=false', needs.define-variables.outputs.images_list) }} + outputs: | + ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' && format('type=image,"name={0}",push-by-digest=true,name-canonical=true,push=true', needs.define-variables.outputs.images_list) || format('type=image,"name={0}",push=false', needs.define-variables.outputs.images_list) }} + type=local,dest=/tmp/binaries env: SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} # For publishing multi-platform manifests - name: Export digest + if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }} run: | mkdir -p /tmp/digests digest="${{ steps.build.outputs.digest }}" touch "/tmp/digests/${digest#sha256:}" - - name: Extract binary from container (image) - id: extract-binary-image - run: | - mkdir -p /tmp/binaries - digest="${{ steps.build.outputs.digest }}" - echo "container_id=$(docker create --platform ${{ matrix.platform }} ${{ needs.define-variables.outputs.images_list }}@$digest)" >> $GITHUB_OUTPUT - - name: Extract binary from container (copy) - run: docker cp ${{ steps.extract-binary-image.outputs.container_id }}:/sbin/conduwuit /tmp/binaries/conduwuit-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }} - - name: Extract binary from container (cleanup) - run: docker rm ${{ steps.extract-binary-image.outputs.container_id }} + # Binary extracted via local output for all builds + - name: Rename extracted binary + run: mv /tmp/binaries/sbin/conduwuit /tmp/binaries/conduwuit-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }} - name: Upload binary artifact uses: forgejo/upload-artifact@v4 From 6cf3c839e4b2b6906f4017a3f77591937f37af6e Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Sun, 7 Sep 2025 21:27:56 +0100 Subject: [PATCH 2066/2091] ci(release-image): Skip digest upload when not pushing images After #992, builds without registry credentials skip Docker image output but still extract binary artifacts. However, we were still trying to upload digests for images that weren't created. Add conditional check to only upload digests when actually pushing to registry. --- .forgejo/workflows/release-image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 2a722edd..b7423567 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -249,6 +249,7 @@ jobs: if-no-files-found: error - name: Upload digest + if: ${{ env.BUILTIN_REGISTRY_ENABLED == 'true' }} uses: forgejo/upload-artifact@v4 with: name: digests-${{ matrix.slug }} From 2cdccbf2fe16fee50c8d93a9816c66c4ec6bf698 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Wed, 3 Sep 2025 14:20:50 +0100 Subject: [PATCH 2067/2091] feat(PR977): Support omitting members in the send_join response --- src/api/server/send_join.rs | 97 ++++++++++++++++++----- src/service/rooms/state_compressor/mod.rs | 2 +- 2 files changed, 80 insertions(+), 19 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 652451c7..3aebbbe7 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -1,12 +1,13 @@ #![allow(deprecated)] -use std::borrow::Borrow; +use std::{borrow::Borrow, time::Instant}; use axum::extract::State; use conduwuit::{ - Err, Result, at, err, + Err, Event, Result, at, debug, err, info, matrix::event::gen_event_id_canonical_json, - utils::stream::{IterStream, TryBroadbandExt}, + trace, + utils::stream::{BroadbandExt, IterStream, TryBroadbandExt}, warn, }; use conduwuit_service::Services; @@ -25,12 +26,14 @@ use serde_json::value::{RawValue as RawJsonValue, to_raw_value}; use crate::Ruma; /// helper method for /send_join v1 and v2 +#[tracing::instrument(skip(services, pdu, omit_members), fields(room_id = room_id.as_str(), origin = origin.as_str()))] async fn create_join_event( services: &Services, origin: &ServerName, room_id: &RoomId, pdu: &RawJsonValue, -) -> Result { + omit_members: bool, +) -> Result { if !services.rooms.metadata.exists(room_id).await { return Err!(Request(NotFound("Room is unknown to this server."))); } @@ -201,6 +204,7 @@ async fn create_join_event( .lock(room_id) .await; + debug!("Acquired send_join mutex, persisting join event"); let pdu_id = services .rooms .event_handler @@ -210,7 +214,7 @@ async fn create_join_event( .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; drop(mutex_lock); - + debug!("Fetching current state IDs"); let state_ids: Vec = services .rooms .state_accessor @@ -219,9 +223,25 @@ async fn create_join_event( .collect() .await; + #[allow(clippy::unnecessary_unwrap)] let state = state_ids .iter() .try_stream() + .broad_filter_map(|event_id| async move { + if omit_members && event_id.is_ok() { + let pdu = services + .rooms + .timeline + .get_pdu(event_id.as_ref().unwrap()) + .await; + if pdu.is_ok_and(|p| p.kind().to_cow_str() == "m.room.member") { + trace!("omitting member event {event_id:?} from returned state"); + // skip members + return None; + } + } + Some(event_id) + }) .broad_and_then(|event_id| services.rooms.timeline.get_pdu_json(event_id)) .broad_and_then(|pdu| { services @@ -238,6 +258,17 @@ async fn create_join_event( .rooms .auth_chain .event_ids_iter(room_id, starting_events) + .broad_filter_map(|event_id| async { + if omit_members && event_id.as_ref().is_ok_and(|e| state_ids.contains(e)) { + // Don't include this event if it's already in the state + trace!( + "omitting member event {event_id:?} from returned auth chain as it is \ + already in state" + ); + return None; + } + Some(event_id) + }) .broad_and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) @@ -252,11 +283,27 @@ async fn create_join_event( .await?; services.sending.send_pdu_room(room_id, &pdu_id).await?; - - Ok(create_join_event::v1::RoomState { + let servers_in_room: Option> = if omit_members { + None + } else { + debug!("Fetching list of servers in room"); + Some( + services + .rooms + .state_cache + .room_servers(room_id) + .map(|sn| sn.as_str().to_owned()) + .collect() + .await, + ) + }; + debug!("Returning send_join data"); + Ok(create_join_event::v2::RoomState { auth_chain, state, event: to_raw_value(&CanonicalJsonValue::Object(value)).ok(), + members_omitted: omit_members, + servers_in_room, }) } @@ -294,11 +341,24 @@ pub(crate) async fn create_join_event_v1_route( } } - let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu) + info!("Providing send_join for {} in {}", body.origin(), &body.room_id); + let now = Instant::now(); + let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu, false) .boxed() .await?; + let transformed = create_join_event::v1::RoomState { + auth_chain: room_state.auth_chain, + state: room_state.state, + event: room_state.event, + }; + info!( + "Finished creating the send_join payload for {} in {} in {:?}", + body.origin(), + &body.room_id, + now.elapsed() + ); - Ok(create_join_event::v1::Response { room_state }) + Ok(create_join_event::v1::Response { room_state: transformed }) } /// # `PUT /_matrix/federation/v2/send_join/{roomId}/{eventId}` @@ -329,17 +389,18 @@ pub(crate) async fn create_join_event_v2_route( } } - let create_join_event::v1::RoomState { auth_chain, state, event } = - create_join_event(&services, body.origin(), &body.room_id, &body.pdu) + info!("Providing send_join for {} in {}", body.origin(), &body.room_id); + let now = Instant::now(); + let room_state = + create_join_event(&services, body.origin(), &body.room_id, &body.pdu, body.omit_members) .boxed() .await?; - let room_state = create_join_event::v2::RoomState { - members_omitted: false, - auth_chain, - state, - event, - servers_in_room: None, - }; + info!( + "Finished creating the send_join payload for {} in {} in {:?}", + body.origin(), + &body.room_id, + now.elapsed() + ); Ok(create_join_event::v2::Response { room_state }) } diff --git a/src/service/rooms/state_compressor/mod.rs b/src/service/rooms/state_compressor/mod.rs index f7f7d043..028c3e51 100644 --- a/src/service/rooms/state_compressor/mod.rs +++ b/src/service/rooms/state_compressor/mod.rs @@ -452,7 +452,7 @@ async fn get_statediff(&self, shortstatehash: ShortStateHash) -> Result Date: Wed, 3 Sep 2025 15:08:28 +0100 Subject: [PATCH 2068/2091] fix(PR977): Omitting redundant entries from the auth_chain caused problems --- src/api/server/send_join.rs | 49 ++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 3aebbbe7..a52b5391 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -1,6 +1,6 @@ #![allow(deprecated)] -use std::{borrow::Borrow, time::Instant}; +use std::{borrow::Borrow, time::Instant, vec}; use axum::extract::State; use conduwuit::{ @@ -258,17 +258,17 @@ async fn create_join_event( .rooms .auth_chain .event_ids_iter(room_id, starting_events) - .broad_filter_map(|event_id| async { - if omit_members && event_id.as_ref().is_ok_and(|e| state_ids.contains(e)) { - // Don't include this event if it's already in the state - trace!( - "omitting member event {event_id:?} from returned auth chain as it is \ - already in state" - ); - return None; - } - Some(event_id) - }) + // .broad_filter_map(|event_id| async { + // if omit_members && event_id.as_ref().is_ok_and(|e| state_ids.contains(e)) { + // // Don't include this event if it's already in the state + // trace!( + // "omitting member event {event_id:?} from returned auth chain as it is \ + // already in state" + // ); + // return None; + // } + // Some(event_id) + // }) .broad_and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) @@ -283,19 +283,24 @@ async fn create_join_event( .await?; services.sending.send_pdu_room(room_id, &pdu_id).await?; - let servers_in_room: Option> = if omit_members { + let servers_in_room: Option> = if !omit_members { None } else { debug!("Fetching list of servers in room"); - Some( - services - .rooms - .state_cache - .room_servers(room_id) - .map(|sn| sn.as_str().to_owned()) - .collect() - .await, - ) + let servers: Vec = services + .rooms + .state_cache + .room_servers(room_id) + .map(|sn| sn.as_str().to_owned()) + .collect() + .await; + // If there's no servers, just add us + let servers = if servers.is_empty() { + vec![services.globals.server_name().to_string()] + } else { + servers + }; + Some(servers) }; debug!("Returning send_join data"); Ok(create_join_event::v2::RoomState { From f47474d12aa67ffad071a6fd4b6665dc46cf29bf Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 7 Sep 2025 20:23:57 +0100 Subject: [PATCH 2069/2091] fix(PR977): Adjust some log levels --- src/api/server/send_join.rs | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index a52b5391..3145f9fa 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -204,7 +204,7 @@ async fn create_join_event( .lock(room_id) .await; - debug!("Acquired send_join mutex, persisting join event"); + trace!("Acquired send_join mutex, persisting join event"); let pdu_id = services .rooms .event_handler @@ -214,7 +214,7 @@ async fn create_join_event( .ok_or_else(|| err!(Request(InvalidParam("Could not accept as timeline event."))))?; drop(mutex_lock); - debug!("Fetching current state IDs"); + trace!("Fetching current state IDs"); let state_ids: Vec = services .rooms .state_accessor @@ -258,17 +258,6 @@ async fn create_join_event( .rooms .auth_chain .event_ids_iter(room_id, starting_events) - // .broad_filter_map(|event_id| async { - // if omit_members && event_id.as_ref().is_ok_and(|e| state_ids.contains(e)) { - // // Don't include this event if it's already in the state - // trace!( - // "omitting member event {event_id:?} from returned auth chain as it is \ - // already in state" - // ); - // return None; - // } - // Some(event_id) - // }) .broad_and_then(|event_id| async move { services.rooms.timeline.get_pdu_json(&event_id).await }) @@ -281,12 +270,12 @@ async fn create_join_event( .try_collect() .boxed() .await?; - + info!(fast_join = %omit_members, "Sending a join for {origin} to {room_id}"); services.sending.send_pdu_room(room_id, &pdu_id).await?; let servers_in_room: Option> = if !omit_members { None } else { - debug!("Fetching list of servers in room"); + trace!("Fetching list of servers in room"); let servers: Vec = services .rooms .state_cache @@ -296,6 +285,10 @@ async fn create_join_event( .await; // If there's no servers, just add us let servers = if servers.is_empty() { + warn!( + "Failed to find any servers in {room_id}, adding our own server name as a last \ + resort" + ); vec![services.globals.server_name().to_string()] } else { servers @@ -346,7 +339,6 @@ pub(crate) async fn create_join_event_v1_route( } } - info!("Providing send_join for {} in {}", body.origin(), &body.room_id); let now = Instant::now(); let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu, false) .boxed() @@ -357,7 +349,7 @@ pub(crate) async fn create_join_event_v1_route( event: room_state.event, }; info!( - "Finished creating the send_join payload for {} in {} in {:?}", + "Finished sending a join for {} in {} in {:?}", body.origin(), &body.room_id, now.elapsed() @@ -394,14 +386,13 @@ pub(crate) async fn create_join_event_v2_route( } } - info!("Providing send_join for {} in {}", body.origin(), &body.room_id); let now = Instant::now(); let room_state = create_join_event(&services, body.origin(), &body.room_id, &body.pdu, body.omit_members) .boxed() .await?; info!( - "Finished creating the send_join payload for {} in {} in {:?}", + "Finished sending a join for {} in {} in {:?}", body.origin(), &body.room_id, now.elapsed() From d1fff1d09f7343cafd92df9282ff29b3cf9e3545 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 7 Sep 2025 20:26:26 +0100 Subject: [PATCH 2070/2091] perf(pr977): Remove redundant ACL check in send_join --- src/api/server/send_join.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index 3145f9fa..aa93047b 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -56,8 +56,10 @@ async fn create_join_event( // We do not add the event_id field to the pdu here because of signature and // hashes checks + trace!("Getting room version"); let room_version_id = services.rooms.state.get_room_version(room_id).await?; + trace!("Generating event ID and converting to canonical json"); let Ok((event_id, mut value)) = gen_event_id_canonical_json(pdu, &room_version_id) else { // Event could not be converted to canonical json return Err!(Request(BadJson("Could not convert event to canonical json."))); @@ -106,7 +108,6 @@ async fn create_join_event( ))); } - // ACL check sender user server name let sender: OwnedUserId = serde_json::from_value( value .get("sender") @@ -116,12 +117,6 @@ async fn create_join_event( ) .map_err(|e| err!(Request(BadJson(warn!("sender property is not a valid user ID: {e}")))))?; - services - .rooms - .event_handler - .acl_check(sender.server_name(), room_id) - .await?; - // check if origin server is trying to send for another server if sender.server_name() != origin { return Err!(Request(Forbidden("Not allowed to join on behalf of another server."))); From 09de586dc702b9adcb3298e974470066f198c43b Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 7 Sep 2025 20:32:26 +0100 Subject: [PATCH 2071/2091] feat(PR977): Log more things in the join process --- src/api/server/send_join.rs | 41 ++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/src/api/server/send_join.rs b/src/api/server/send_join.rs index aa93047b..d92581a1 100644 --- a/src/api/server/send_join.rs +++ b/src/api/server/send_join.rs @@ -178,11 +178,6 @@ async fn create_join_event( } } - services - .server_keys - .hash_and_sign_event(&mut value, &room_version_id) - .map_err(|e| err!(Request(InvalidParam(warn!("Failed to sign send_join event: {e}")))))?; - let origin: OwnedServerName = serde_json::from_value( value .get("origin") @@ -192,6 +187,12 @@ async fn create_join_event( ) .map_err(|e| err!(Request(BadJson("Event has an invalid origin server name: {e}"))))?; + trace!("Signing send_join event"); + services + .server_keys + .hash_and_sign_event(&mut value, &room_version_id) + .map_err(|e| err!(Request(InvalidParam(warn!("Failed to sign send_join event: {e}")))))?; + let mutex_lock = services .rooms .event_handler @@ -218,21 +219,19 @@ async fn create_join_event( .collect() .await; - #[allow(clippy::unnecessary_unwrap)] + trace!(%omit_members, "Constructing current state"); let state = state_ids .iter() .try_stream() .broad_filter_map(|event_id| async move { - if omit_members && event_id.is_ok() { - let pdu = services - .rooms - .timeline - .get_pdu(event_id.as_ref().unwrap()) - .await; - if pdu.is_ok_and(|p| p.kind().to_cow_str() == "m.room.member") { - trace!("omitting member event {event_id:?} from returned state"); - // skip members - return None; + if omit_members { + if let Ok(e) = event_id.as_ref() { + let pdu = services.rooms.timeline.get_pdu(e).await; + if pdu.is_ok_and(|p| p.kind().to_cow_str() == "m.room.member") { + trace!("omitting member event {e:?} from returned state"); + // skip members + return None; + } } } Some(event_id) @@ -249,6 +248,7 @@ async fn create_join_event( .await?; let starting_events = state_ids.iter().map(Borrow::borrow); + trace!("Constructing auth chain"); let auth_chain = services .rooms .auth_chain @@ -265,8 +265,9 @@ async fn create_join_event( .try_collect() .boxed() .await?; - info!(fast_join = %omit_members, "Sending a join for {origin} to {room_id}"); + info!(fast_join = %omit_members, "Sending join event to other servers"); services.sending.send_pdu_room(room_id, &pdu_id).await?; + debug!("Finished sending join event"); let servers_in_room: Option> = if !omit_members { None } else { @@ -280,12 +281,10 @@ async fn create_join_event( .await; // If there's no servers, just add us let servers = if servers.is_empty() { - warn!( - "Failed to find any servers in {room_id}, adding our own server name as a last \ - resort" - ); + warn!("Failed to find any servers, adding our own server name as a last resort"); vec![services.globals.server_name().to_string()] } else { + trace!("Found {} servers in room", servers.len()); servers }; Some(servers) From e3fbf7a143e06528d505a0357e9558944c8bd928 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 8 Jul 2025 17:07:50 +0100 Subject: [PATCH 2072/2091] feat: Ask remote servers for individual unknown events --- src/api/client/room/event.rs | 7 +- src/service/rooms/timeline/backfill.rs | 114 ++++++++++++++++++++++++- 2 files changed, 112 insertions(+), 9 deletions(-) diff --git a/src/api/client/room/event.rs b/src/api/client/room/event.rs index 47228d67..75ae0758 100644 --- a/src/api/client/room/event.rs +++ b/src/api/client/room/event.rs @@ -18,7 +18,7 @@ pub(crate) async fn get_room_event_route( let event = services .rooms .timeline - .get_pdu(event_id) + .get_remote_pdu(room_id, event_id) .map_err(|_| err!(Request(NotFound("Event {} not found.", event_id)))); let visible = services @@ -33,11 +33,6 @@ pub(crate) async fn get_room_event_route( return Err!(Request(Forbidden("You don't have permission to view this event."))); } - debug_assert!( - event.event_id() == event_id && event.room_id() == room_id, - "Fetched PDU must match requested" - ); - event.add_age().ok(); Ok(get_room_event::v3::Response { event: event.into_format() }) diff --git a/src/service/rooms/timeline/backfill.rs b/src/service/rooms/timeline/backfill.rs index e976981e..dd613afd 100644 --- a/src/service/rooms/timeline/backfill.rs +++ b/src/service/rooms/timeline/backfill.rs @@ -1,5 +1,6 @@ use std::iter::once; +use conduwuit::{Err, PduEvent}; use conduwuit_core::{ Result, debug, debug_warn, implement, info, matrix::{ @@ -11,7 +12,7 @@ use conduwuit_core::{ }; use futures::{FutureExt, StreamExt}; use ruma::{ - RoomId, ServerName, + EventId, RoomId, ServerName, api::federation, events::{ StateEventType, TimelineEventType, room::power_levels::RoomPowerLevelsEventContent, @@ -100,7 +101,7 @@ pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Re .boxed(); while let Some(ref backfill_server) = servers.next().await { - info!("Asking {backfill_server} for backfill"); + info!("Asking {backfill_server} for backfill in {room_id}"); let response = self .services .sending @@ -128,10 +129,117 @@ pub async fn backfill_if_required(&self, room_id: &RoomId, from: PduCount) -> Re } } - info!("No servers could backfill, but backfill was needed in room {room_id}"); + warn!("No servers could backfill, but backfill was needed in room {room_id}"); Ok(()) } +#[implement(super::Service)] +#[tracing::instrument(name = "get_remote_pdu", level = "debug", skip(self))] +pub async fn get_remote_pdu(&self, room_id: &RoomId, event_id: &EventId) -> Result { + let local = self.get_pdu(event_id).await; + if local.is_ok() { + // We already have this PDU, no need to backfill + debug!("We already have {event_id} in {room_id}, no need to backfill."); + return local; + } + debug!("Preparing to fetch event {event_id} in room {room_id} from remote servers."); + // Similar to backfill_if_required, but only for a single PDU + // Fetch a list of servers to try + if self + .services + .state_cache + .room_joined_count(room_id) + .await + .is_ok_and(|count| count <= 1) + && !self + .services + .state_accessor + .is_world_readable(room_id) + .await + { + // Room is empty (1 user or none), there is no one that can backfill + return Err!(Request(NotFound("No one can backfill this PDU, room is empty."))); + } + + let power_levels: RoomPowerLevelsEventContent = self + .services + .state_accessor + .room_state_get_content(room_id, &StateEventType::RoomPowerLevels, "") + .await + .unwrap_or_default(); + + let room_mods = power_levels.users.iter().filter_map(|(user_id, level)| { + if level > &power_levels.users_default && !self.services.globals.user_is_local(user_id) { + Some(user_id.server_name()) + } else { + None + } + }); + + let canonical_room_alias_server = once( + self.services + .state_accessor + .get_canonical_alias(room_id) + .await, + ) + .filter_map(Result::ok) + .map(|alias| alias.server_name().to_owned()) + .stream(); + let mut servers = room_mods + .stream() + .map(ToOwned::to_owned) + .chain(canonical_room_alias_server) + .chain( + self.services + .server + .config + .trusted_servers + .iter() + .map(ToOwned::to_owned) + .stream(), + ) + .ready_filter(|server_name| !self.services.globals.server_is_ours(server_name)) + .filter_map(|server_name| async move { + self.services + .state_cache + .server_in_room(&server_name, room_id) + .await + .then_some(server_name) + }) + .boxed(); + + while let Some(ref backfill_server) = servers.next().await { + info!("Asking {backfill_server} for event {}", event_id); + let response = self + .services + .sending + .send_federation_request(backfill_server, federation::event::get_event::v1::Request { + event_id: event_id.to_owned(), + include_unredacted_content: Some(false), + }) + .await; + let pdu = match response { + | Ok(response) => { + self.backfill_pdu(backfill_server, response.pdu) + .boxed() + .await?; + debug!("Successfully backfilled {event_id} from {backfill_server}"); + Some(self.get_pdu(event_id).await) + }, + | Err(e) => { + warn!("{backfill_server} failed to provide backfill for room {room_id}: {e}"); + None + }, + }; + if let Some(pdu) = pdu { + debug!("Fetched {event_id} from {backfill_server}"); + return pdu; + } + } + + Err!("No servers could be used to fetch {} in {}.", room_id, event_id) +} + #[implement(super::Service)] #[tracing::instrument(skip(self, pdu), level = "debug")] pub async fn backfill_pdu(&self, origin: &ServerName, pdu: Box) -> Result<()> { From f3824ffc3d5c2ed6d5067f134850a451564c8f7a Mon Sep 17 00:00:00 2001 From: Ginger Date: Tue, 2 Sep 2025 15:12:03 -0400 Subject: [PATCH 2073/2091] fix: Use `handle_incoming_pdu` directly to keep remote PDUs as outliers --- src/service/rooms/timeline/backfill.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/service/rooms/timeline/backfill.rs b/src/service/rooms/timeline/backfill.rs index dd613afd..c0171691 100644 --- a/src/service/rooms/timeline/backfill.rs +++ b/src/service/rooms/timeline/backfill.rs @@ -2,7 +2,7 @@ use std::iter::once; use conduwuit::{Err, PduEvent}; use conduwuit_core::{ - Result, debug, debug_warn, implement, info, + Result, debug, debug_warn, err, implement, info, matrix::{ event::Event, pdu::{PduCount, PduId, RawPduId}, @@ -12,7 +12,7 @@ use conduwuit_core::{ }; use futures::{FutureExt, StreamExt}; use ruma::{ - EventId, RoomId, ServerName, + CanonicalJsonObject, EventId, RoomId, ServerName, api::federation, events::{ StateEventType, TimelineEventType, room::power_levels::RoomPowerLevelsEventContent, @@ -210,17 +210,26 @@ pub async fn get_remote_pdu(&self, room_id: &RoomId, event_id: &EventId) -> Resu while let Some(ref backfill_server) = servers.next().await { info!("Asking {backfill_server} for event {}", event_id); - let response = self + let value = self .services .sending .send_federation_request(backfill_server, federation::event::get_event::v1::Request { event_id: event_id.to_owned(), include_unredacted_content: Some(false), }) - .await; - let pdu = match response { - | Ok(response) => { - self.backfill_pdu(backfill_server, response.pdu) + .await + .and_then(|response| { + serde_json::from_str::(response.pdu.get()).map_err(|e| { + err!(BadServerResponse(debug_warn!( + "Error parsing incoming event {e:?} from {backfill_server}" + ))) + }) + }); + let pdu = match value { + | Ok(value) => { + self.services + .event_handler + .handle_incoming_pdu(backfill_server, &room_id, &event_id, value, false) .boxed() .await?; debug!("Successfully backfilled {event_id} from {backfill_server}"); From e38dec58648aefec3a289bbfb5043bd1b02e4187 Mon Sep 17 00:00:00 2001 From: Ginger Date: Thu, 4 Sep 2025 10:32:35 -0400 Subject: [PATCH 2074/2091] fix: Put the output of `!admin query room-timeline pdus` in a codeblock --- src/admin/query/room_timeline.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/admin/query/room_timeline.rs b/src/admin/query/room_timeline.rs index afcfec34..0f72b58c 100644 --- a/src/admin/query/room_timeline.rs +++ b/src/admin/query/room_timeline.rs @@ -57,5 +57,5 @@ pub(super) async fn pdus( .try_collect() .await?; - self.write_str(&format!("{result:#?}")).await + self.write_str(&format!("```\n{result:#?}\n```")).await } From 16f4efa7089292d27e27e49fc5231a58d020aed3 Mon Sep 17 00:00:00 2001 From: Ginger Date: Thu, 4 Sep 2025 10:33:43 -0400 Subject: [PATCH 2075/2091] fix: Fix pagination tokens being corrupted for backfilled PDUs --- src/api/client/message.rs | 6 +++--- src/api/client/relations.rs | 4 ++-- src/api/client/threads.rs | 4 ++-- src/api/client/utils.rs | 31 +++++------------------------- src/core/matrix/pdu/count.rs | 6 +++++- src/service/rooms/timeline/data.rs | 3 +-- 6 files changed, 18 insertions(+), 36 deletions(-) diff --git a/src/api/client/message.rs b/src/api/client/message.rs index f5a951f4..0145b7fe 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -35,7 +35,7 @@ use ruma::{ }; use tracing::warn; -use super::utils::{count_to_token, parse_pagination_token as parse_token}; +use super::utils::{count_to_pagination_token, pagination_token_to_count as parse_token}; use crate::Ruma; /// list of safe and common non-state events to ignore if the user is ignored @@ -181,8 +181,8 @@ pub(crate) async fn get_message_events_route( .collect(); Ok(get_message_events::v3::Response { - start: count_to_token(from), - end: next_token.map(count_to_token), + start: count_to_pagination_token(from), + end: next_token.map(count_to_pagination_token), chunk, state, }) diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index f6d8fe9e..f2ec3f23 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -18,7 +18,7 @@ use ruma::{ events::{TimelineEventType, relation::RelationType}, }; -use super::utils::{count_to_token, parse_pagination_token as parse_token}; +use super::utils::{count_to_pagination_token, pagination_token_to_count as parse_token}; use crate::Ruma; /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}` @@ -193,7 +193,7 @@ async fn paginate_relations_with_filter( | Direction::Forward => events.last(), | Direction::Backward => events.first(), } - .map(|(count, _)| count_to_token(*count)) + .map(|(count, _)| count_to_pagination_token(*count)) } else { None }; diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index ca176eda..f0fb4a64 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -9,7 +9,7 @@ use conduwuit::{ use futures::StreamExt; use ruma::{api::client::threads::get_threads, uint}; -use crate::Ruma; +use crate::{Ruma, client::utils::pagination_token_to_count}; /// # `GET /_matrix/client/r0/rooms/{roomId}/threads` pub(crate) async fn get_threads_route( @@ -27,7 +27,7 @@ pub(crate) async fn get_threads_route( let from: PduCount = body .from .as_deref() - .map(str::parse) + .map(pagination_token_to_count) .transpose()? .unwrap_or_else(PduCount::max); diff --git a/src/api/client/utils.rs b/src/api/client/utils.rs index cc941b95..ec69388a 100644 --- a/src/api/client/utils.rs +++ b/src/api/client/utils.rs @@ -1,28 +1,7 @@ -use conduwuit::{ - Result, err, - matrix::pdu::{PduCount, ShortEventId}, -}; +use conduwuit::{Result, matrix::pdu::PduCount}; -/// Parse a pagination token, trying ShortEventId first, then falling back to -/// PduCount -pub(crate) fn parse_pagination_token(token: &str) -> Result { - // Try parsing as ShortEventId first - if let Ok(shorteventid) = token.parse::() { - // ShortEventId maps directly to a PduCount in our database - Ok(PduCount::Normal(shorteventid)) - } else if let Ok(count) = token.parse::() { - // Fallback to PduCount for backwards compatibility - Ok(PduCount::Normal(count)) - } else if let Ok(count) = token.parse::() { - // Also handle negative counts for backfilled events - Ok(PduCount::from_signed(count)) - } else { - Err(err!(Request(InvalidParam("Invalid pagination token")))) - } -} +/// Parse a pagination token +pub(crate) fn pagination_token_to_count(token: &str) -> Result { token.parse() } -/// Convert a PduCount to a token string (using the underlying ShortEventId) -pub(crate) fn count_to_token(count: PduCount) -> String { - // The PduCount's unsigned value IS the ShortEventId - count.into_unsigned().to_string() -} +/// Convert a PduCount to a token string +pub(crate) fn count_to_pagination_token(count: PduCount) -> String { count.to_string() } diff --git a/src/core/matrix/pdu/count.rs b/src/core/matrix/pdu/count.rs index b880278f..7fb12574 100644 --- a/src/core/matrix/pdu/count.rs +++ b/src/core/matrix/pdu/count.rs @@ -1,6 +1,10 @@ #![allow(clippy::cast_possible_wrap, clippy::cast_sign_loss, clippy::as_conversions)] -use std::{cmp::Ordering, fmt, fmt::Display, str::FromStr}; +use std::{ + cmp::Ordering, + fmt::{self, Display}, + str::FromStr, +}; use ruma::api::Direction; diff --git a/src/service/rooms/timeline/data.rs b/src/service/rooms/timeline/data.rs index fa10a5c0..9064df6c 100644 --- a/src/service/rooms/timeline/data.rs +++ b/src/service/rooms/timeline/data.rs @@ -3,8 +3,7 @@ use std::{borrow::Borrow, sync::Arc}; use conduwuit::{ Err, PduCount, PduEvent, Result, at, err, result::{LogErr, NotFound}, - utils, - utils::stream::TryReadyExt, + utils::{self, stream::TryReadyExt}, }; use database::{Database, Deserialized, Json, KeyVal, Map}; use futures::{FutureExt, Stream, TryFutureExt, TryStreamExt, future::select_ok, pin_mut}; From e27ef7f5ec5e95c33f06c1f5d06cd16ca266cd07 Mon Sep 17 00:00:00 2001 From: Ginger Date: Thu, 4 Sep 2025 10:46:08 -0400 Subject: [PATCH 2076/2091] feat: Do not persist remote PDUs fetched with admin commands --- src/admin/debug/commands.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/admin/debug/commands.rs b/src/admin/debug/commands.rs index 81b0e9da..64f68330 100644 --- a/src/admin/debug/commands.rs +++ b/src/admin/debug/commands.rs @@ -281,15 +281,8 @@ pub(super) async fn get_remote_pdu( vec![(event_id, value, room_id)] }; - info!("Attempting to handle event ID {event_id} as backfilled PDU"); - self.services - .rooms - .timeline - .backfill_pdu(&server, response.pdu) - .await?; - let text = serde_json::to_string_pretty(&json)?; - let msg = "Got PDU from specified server and handled as backfilled"; + let msg = "Got PDU from specified server:"; write!(self, "{msg}. Event body:\n```json\n{text}\n```") }, } From 90fd92977e6dca4cad4625d8383eb9a42c0f4b4b Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 7 Sep 2025 22:08:37 +0100 Subject: [PATCH 2077/2091] style: Run clippy --- src/service/rooms/timeline/backfill.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/service/rooms/timeline/backfill.rs b/src/service/rooms/timeline/backfill.rs index c0171691..9dfe8f7a 100644 --- a/src/service/rooms/timeline/backfill.rs +++ b/src/service/rooms/timeline/backfill.rs @@ -229,7 +229,7 @@ pub async fn get_remote_pdu(&self, room_id: &RoomId, event_id: &EventId) -> Resu | Ok(value) => { self.services .event_handler - .handle_incoming_pdu(backfill_server, &room_id, &event_id, value, false) + .handle_incoming_pdu(backfill_server, room_id, event_id, value, false) .boxed() .await?; debug!("Successfully backfilled {event_id} from {backfill_server}"); From 1e541875ad94fcacca268515569c1cb32879b205 Mon Sep 17 00:00:00 2001 From: Ginger Date: Sun, 7 Sep 2025 18:06:11 -0400 Subject: [PATCH 2078/2091] fix: Nuke `src/api/client/utils.rs` --- src/api/client/message.rs | 9 ++++----- src/api/client/mod.rs | 1 - src/api/client/relations.rs | 7 +++---- src/api/client/threads.rs | 4 ++-- src/api/client/utils.rs | 7 ------- 5 files changed, 9 insertions(+), 19 deletions(-) delete mode 100644 src/api/client/utils.rs diff --git a/src/api/client/message.rs b/src/api/client/message.rs index 0145b7fe..842036f5 100644 --- a/src/api/client/message.rs +++ b/src/api/client/message.rs @@ -35,7 +35,6 @@ use ruma::{ }; use tracing::warn; -use super::utils::{count_to_pagination_token, pagination_token_to_count as parse_token}; use crate::Ruma; /// list of safe and common non-state events to ignore if the user is ignored @@ -85,14 +84,14 @@ pub(crate) async fn get_message_events_route( let from: PduCount = body .from .as_deref() - .map(parse_token) + .map(str::parse) .transpose()? .unwrap_or_else(|| match body.dir { | Direction::Forward => PduCount::min(), | Direction::Backward => PduCount::max(), }); - let to: Option = body.to.as_deref().map(parse_token).transpose()?; + let to: Option = body.to.as_deref().map(str::parse).transpose()?; let limit: usize = body .limit @@ -181,8 +180,8 @@ pub(crate) async fn get_message_events_route( .collect(); Ok(get_message_events::v3::Response { - start: count_to_pagination_token(from), - end: next_token.map(count_to_pagination_token), + start: from.to_string(), + end: next_token.as_ref().map(PduCount::to_string), chunk, state, }) diff --git a/src/api/client/mod.rs b/src/api/client/mod.rs index c8ca7757..0014282c 100644 --- a/src/api/client/mod.rs +++ b/src/api/client/mod.rs @@ -37,7 +37,6 @@ pub(super) mod typing; pub(super) mod unstable; pub(super) mod unversioned; pub(super) mod user_directory; -pub(super) mod utils; pub(super) mod voip; pub(super) mod well_known; diff --git a/src/api/client/relations.rs b/src/api/client/relations.rs index f2ec3f23..79fa1459 100644 --- a/src/api/client/relations.rs +++ b/src/api/client/relations.rs @@ -18,7 +18,6 @@ use ruma::{ events::{TimelineEventType, relation::RelationType}, }; -use super::utils::{count_to_pagination_token, pagination_token_to_count as parse_token}; use crate::Ruma; /// # `GET /_matrix/client/r0/rooms/{roomId}/relations/{eventId}/{relType}/{eventType}` @@ -111,14 +110,14 @@ async fn paginate_relations_with_filter( dir: Direction, ) -> Result { let start: PduCount = from - .map(parse_token) + .map(str::parse) .transpose()? .unwrap_or_else(|| match dir { | Direction::Forward => PduCount::min(), | Direction::Backward => PduCount::max(), }); - let to: Option = to.map(parse_token).transpose()?; + let to: Option = to.map(str::parse).transpose()?; // Use limit or else 30, with maximum 100 let limit: usize = limit @@ -193,7 +192,7 @@ async fn paginate_relations_with_filter( | Direction::Forward => events.last(), | Direction::Backward => events.first(), } - .map(|(count, _)| count_to_pagination_token(*count)) + .map(|(count, _)| count.to_string()) } else { None }; diff --git a/src/api/client/threads.rs b/src/api/client/threads.rs index f0fb4a64..ca176eda 100644 --- a/src/api/client/threads.rs +++ b/src/api/client/threads.rs @@ -9,7 +9,7 @@ use conduwuit::{ use futures::StreamExt; use ruma::{api::client::threads::get_threads, uint}; -use crate::{Ruma, client::utils::pagination_token_to_count}; +use crate::Ruma; /// # `GET /_matrix/client/r0/rooms/{roomId}/threads` pub(crate) async fn get_threads_route( @@ -27,7 +27,7 @@ pub(crate) async fn get_threads_route( let from: PduCount = body .from .as_deref() - .map(pagination_token_to_count) + .map(str::parse) .transpose()? .unwrap_or_else(PduCount::max); diff --git a/src/api/client/utils.rs b/src/api/client/utils.rs deleted file mode 100644 index ec69388a..00000000 --- a/src/api/client/utils.rs +++ /dev/null @@ -1,7 +0,0 @@ -use conduwuit::{Result, matrix::pdu::PduCount}; - -/// Parse a pagination token -pub(crate) fn pagination_token_to_count(token: &str) -> Result { token.parse() } - -/// Convert a PduCount to a token string -pub(crate) fn count_to_pagination_token(count: PduCount) -> String { count.to_string() } From 5d3e10a0481f4857cc706068c0d0c04d44ee9105 Mon Sep 17 00:00:00 2001 From: Ginger Date: Sun, 7 Sep 2025 18:07:03 -0400 Subject: [PATCH 2079/2091] fix: Make RA use the `full` feature --- .vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index a4fad964..82162ff7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,5 +7,6 @@ "continuwuity", "homeserver", "homeservers" - ] + ], + "rust-analyzer.cargo.features": ["full"] } From 8f186cd770ebb088aa85f0aacea7f41a046c2054 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Mon, 8 Sep 2025 05:02:33 +0000 Subject: [PATCH 2080/2091] chore(deps): update https://github.com/renovatebot/github-action action to v43.0.11 --- .forgejo/workflows/renovate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml index 0152b387..6c898657 100644 --- a/.forgejo/workflows/renovate.yml +++ b/.forgejo/workflows/renovate.yml @@ -70,7 +70,7 @@ jobs: package-cache- - name: Self-hosted Renovate - uses: https://github.com/renovatebot/github-action@v43.0.9 + uses: https://github.com/renovatebot/github-action@v43.0.11 env: LOG_LEVEL: ${{ inputs.logLevel || 'info' }} RENOVATE_DRY_RUN: ${{ inputs.dryRun || 'false' }} From 7950e2cc7f86646431713c171836209a50b5d9d7 Mon Sep 17 00:00:00 2001 From: Tom Foster Date: Mon, 8 Sep 2025 04:32:23 +0100 Subject: [PATCH 2081/2091] ci: Refactor timelord action to use git-warp-time fallback Updates the timelord action to fall back to git-warp-time when the cache is completely empty, enabling timestamp restoration even on fresh builds. When git-warp-time is used, performs an unshallow fetch to get full history, while subsequent runs use normal fetches. Simplifies the interface by making inputs optional with sensible defaults. Adds binary caching for timelord-cli and git-warp-time tools to avoid repeated installations, and updates paths to use /usr/share/rust/.cargo/bin/ for the catthehacker runner image used by the dind profile (may need updating if/when switching to standard image). The main timelord restore now happens inside the Dockerfile itself, as Docker intentionally wipes all file mtimes on COPY/ADD operations. --- .forgejo/actions/timelord/action.yml | 109 ++++++++++++++++++++------- .forgejo/workflows/release-image.yml | 17 ++++- docker/Dockerfile | 14 +++- 3 files changed, 106 insertions(+), 34 deletions(-) diff --git a/.forgejo/actions/timelord/action.yml b/.forgejo/actions/timelord/action.yml index bb9766d5..40c6d0a6 100644 --- a/.forgejo/actions/timelord/action.yml +++ b/.forgejo/actions/timelord/action.yml @@ -1,46 +1,99 @@ name: timelord description: | - Use timelord to set file timestamps + Use timelord to set file timestamps with git-warp-time fallback for cache misses inputs: key: description: | The key to use for caching the timelord data. - This should be unique to the repository and the runner. - required: true - default: timelord-v0 + required: false + default: '' path: description: | The path to the directory to be timestamped. - This should be the root of the repository. - required: true - default: . + required: false + default: '' + +outputs: + database-path: + description: Path to timelord database + value: '${{ env.TIMELORD_CACHE_PATH }}' runs: using: composite steps: - - name: Cache timelord-cli installation - id: cache-timelord-bin - uses: actions/cache@v3 - with: - path: ~/.cargo/bin/timelord - key: timelord-cli-v3.0.1 - - name: Install timelord-cli - uses: https://github.com/cargo-bins/cargo-binstall@main - if: steps.cache-timelord-bin.outputs.cache-hit != 'true' - - run: cargo binstall timelord-cli@3.0.1 + - name: Set defaults shell: bash - if: steps.cache-timelord-bin.outputs.cache-hit != 'true' + run: | + echo "TIMELORD_KEY=${{ inputs.key || format('timelord-v1-{0}-{1}', github.repository, hashFiles('**/*.rs', '**/Cargo.toml', '**/Cargo.lock')) }}" >> $GITHUB_ENV + echo "TIMELORD_PATH=${{ inputs.path || '.' }}" >> $GITHUB_ENV + echo "TIMELORD_CACHE_PATH=$HOME/.cache/timelord" >> $GITHUB_ENV - - name: Load timelord files - uses: actions/cache/restore@v3 + - name: Restore binary cache + id: binary-cache + uses: actions/cache/restore@v4 with: - path: /timelord/ - key: ${{ inputs.key }} - - name: Run timelord to set timestamps + path: /usr/share/rust/.cargo/bin + key: timelord-binaries-v1 + + - name: Check if binaries need installation shell: bash - run: timelord sync --source-dir ${{ inputs.path }} --cache-dir /timelord/ - - name: Save timelord - uses: actions/cache/save@v3 + id: check-binaries + run: | + NEED_INSTALL=false + if [ ! -e /usr/share/rust/.cargo/bin/timelord ]; then + echo "timelord-cli not found at /usr/share/rust/.cargo/bin/timelord, needs installation" + NEED_INSTALL=true + fi + if [ ! -e /usr/share/rust/.cargo/bin/git-warp-time ]; then + echo "git-warp-time not found at /usr/share/rust/.cargo/bin/git-warp-time, needs installation" + NEED_INSTALL=true + fi + echo "need-install=$NEED_INSTALL" >> $GITHUB_OUTPUT + + - name: Install timelord-cli and git-warp-time + if: steps.check-binaries.outputs.need-install == 'true' + uses: https://github.com/taiki-e/install-action@v2 with: - path: /timelord/ - key: ${{ inputs.key }} + tool: git-warp-time,timelord-cli@3.0.1 + + - name: Save binary cache + if: steps.check-binaries.outputs.need-install == 'true' + uses: actions/cache/save@v4 + with: + path: /usr/share/rust/.cargo/bin + key: timelord-binaries-v1 + + + - name: Restore timelord cache with fallbacks + id: timelord-restore + uses: actions/cache/restore@v4 + with: + path: ${{ env.TIMELORD_CACHE_PATH }} + key: ${{ env.TIMELORD_KEY }} + restore-keys: | + timelord-v1-${{ github.repository }}- + + - name: Initialize timestamps on complete cache miss + if: steps.timelord-restore.outputs.cache-hit != 'true' + shell: bash + run: | + echo "Complete timelord cache miss - running git-warp-time" + git fetch --unshallow + if [ "${{ env.TIMELORD_PATH }}" = "." ]; then + /usr/share/rust/.cargo/bin/git-warp-time --quiet + else + /usr/share/rust/.cargo/bin/git-warp-time --quiet ${{ env.TIMELORD_PATH }} + fi + echo "Git timestamps restored" + + - name: Run timelord sync + shell: bash + run: | + mkdir -p ${{ env.TIMELORD_CACHE_PATH }} + /usr/share/rust/.cargo/bin/timelord sync --source-dir ${{ env.TIMELORD_PATH }} --cache-dir ${{ env.TIMELORD_CACHE_PATH }} + + - name: Save updated timelord cache immediately + uses: actions/cache/save@v4 + with: + path: ${{ env.TIMELORD_CACHE_PATH }} + key: ${{ env.TIMELORD_KEY }} diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index b7423567..22571b90 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -150,10 +150,18 @@ jobs: echo "Commit timestamp: $timestamp" - uses: ./.forgejo/actions/timelord - if: ${{ env.BUILDKIT_ENDPOINT == '' }} - with: - key: timelord-v0 - path: . + id: timelord + + - name: Copy timelord cache to workspace + shell: bash + run: | + mkdir -p ./timelord + if [ -d "${{ steps.timelord.outputs.database-path }}" ]; then + cp -r "${{ steps.timelord.outputs.database-path }}"/* ./timelord/ + echo "Copied timelord cache to workspace" + else + echo "No timelord cache to copy" + fi - name: Cache Rust registry if: ${{ env.BUILDKIT_ENDPOINT == '' }} @@ -217,6 +225,7 @@ jobs: GIT_COMMIT_HASH_SHORT=${{ env.COMMIT_SHORT_SHA }} GIT_REMOTE_URL=${{github.event.repository.html_url }} GIT_REMOTE_COMMIT_URL=${{github.event.head_commit.url }} + CARGO_INCREMENTAL=${{ env.BUILDKIT_ENDPOINT != '' && '1' || '0' }} platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 17e1c6a1..4cb750e4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -60,6 +60,7 @@ RUN <> /etc/environment +# Configure incremental compilation based on build context +ARG CARGO_INCREMENTAL=0 +RUN echo "CARGO_INCREMENTAL=${CARGO_INCREMENTAL}" >> /etc/environment # Configure pkg-config RUN < Date: Wed, 10 Sep 2025 16:40:55 +0100 Subject: [PATCH 2082/2091] ci: Make timelord docker work locally --- .forgejo/workflows/release-image.yml | 14 ++------------ docker/Dockerfile | 13 ++++++------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 22571b90..8505aa18 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -152,17 +152,6 @@ jobs: - uses: ./.forgejo/actions/timelord id: timelord - - name: Copy timelord cache to workspace - shell: bash - run: | - mkdir -p ./timelord - if [ -d "${{ steps.timelord.outputs.database-path }}" ]; then - cp -r "${{ steps.timelord.outputs.database-path }}"/* ./timelord/ - echo "Copied timelord cache to workspace" - else - echo "No timelord cache to copy" - fi - - name: Cache Rust registry if: ${{ env.BUILDKIT_ENDPOINT == '' }} uses: actions/cache@v3 @@ -210,7 +199,8 @@ jobs: "id": "cargo-target-${{ matrix.target_cpu }}-${{ matrix.slug }}-${{ matrix.profile }}" }, "var-cache-apt-${{ matrix.slug }}": "/var/cache/apt", - "var-lib-apt-${{ matrix.slug }}": "/var/lib/apt" + "var-lib-apt-${{ matrix.slug }}": "/var/lib/apt", + "{{ steps.timelord.outputs.database-path }}":"/timelord" } skip-extraction: ${{ steps.cache.outputs.cache-hit }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 4cb750e4..185c6d6c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -53,6 +53,8 @@ ENV BINSTALL_VERSION=1.13.0 ENV CARGO_SBOM_VERSION=0.9.1 # renovate: datasource=crate depName=lddtree ENV LDDTREE_VERSION=0.3.7 +# renovate: datasource=crate depName=timelord-cli +ENV TIMELORD_VERSION=3.0.1 # Install unpackaged tools RUN < Date: Wed, 10 Sep 2025 16:47:20 +0100 Subject: [PATCH 2083/2091] chore: Update cargo lockfile --- Cargo.lock | 448 +++++++++++++++++++++++++---------------------------- 1 file changed, 215 insertions(+), 233 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9e56ad45..9c91c8fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -180,7 +180,7 @@ dependencies = [ "memchr", "proc-macro2", "quote", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "serde_derive", "syn 2.0.106", @@ -256,20 +256,15 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bee399cc3a623ec5a2db2c5b90ee0190a2260241fbe0c023ac8f7bab426aaf8" +checksum = "977eb15ea9efd848bb8a4a1a2500347ed7f0bf794edf0dc3ddcf439f43d36b23" dependencies = [ - "brotli", "compression-codecs", "compression-core", - "flate2", "futures-core", - "memchr", "pin-project-lite", "tokio", - "zstd", - "zstd-safe", ] [[package]] @@ -351,9 +346,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.13.3" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c953fe1ba023e6b7730c0d4b031d06f267f23a46167dcbd40316644b10a17ba" +checksum = "94b8ff6c09cd57b16da53641caa860168b88c172a5ee163b0288d3d6eea12786" dependencies = [ "aws-lc-sys", "zeroize", @@ -361,11 +356,11 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbfd150b5dbdb988bcc8fb1fe787eb6b7ee6180ca24da683b61ea5405f3d43ff" +checksum = "0e44d16778acaf6a9ec9899b92cebd65580b83f685446bf2e1f5d3d732f99dcd" dependencies = [ - "bindgen 0.69.5", + "bindgen", "cc", "cmake", "dunce", @@ -545,41 +540,20 @@ dependencies = [ [[package]] name = "bindgen" -version = "0.69.5" +version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.9.3", + "bitflags", "cexpr", "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", + "itertools 0.13.0", "log", "prettyplease", "proc-macro2", "quote", "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.106", - "which", -] - -[[package]] -name = "bindgen" -version = "0.72.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f72209734318d0b619a5e0f5129918b848c416e122a3c4ce054e03cb87b726f" -dependencies = [ - "bitflags 2.9.3", - "cexpr", - "clang-sys", - "itertools 0.13.0", - "proc-macro2", - "quote", - "regex", - "rustc-hash 2.1.1", + "rustc-hash", "shlex", "syn 2.0.106", ] @@ -592,15 +566,9 @@ checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" [[package]] name = "bitflags" -version = "1.3.2" +version = "2.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" +checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" [[package]] name = "bitstream-io" @@ -726,10 +694,11 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.34" +version = "1.2.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42bc4aea80032b7bf409b0bc7ccad88853858911b7713a8062fdc0623867bedc" +checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -777,9 +746,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ "num-traits", ] @@ -797,9 +766,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.46" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c5e4fcf9c21d2e544ca1ee9d8552de13019a42aa7dbf32747fa7aaf1df76e57" +checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931" dependencies = [ "clap_builder", "clap_derive", @@ -816,9 +785,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.46" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecb53a0e6fcfb055f686001bc2e2592fa527efaf38dbe81a6a9563562e57d41" +checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6" dependencies = [ "anstream", "anstyle", @@ -828,9 +797,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.45" +version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" +checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" dependencies = [ "heck", "proc-macro2", @@ -877,16 +846,14 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "compression-codecs" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7eea68f0e02c2b0aa8856e9a9478444206d4b6828728e7b0697c0f8cca265cb" +checksum = "485abf41ac0c8047c07c87c72c8fb3eb5197f6e9d7ded615dfd1a00ae00a0f64" dependencies = [ "brotli", "compression-core", "flate2", - "futures-core", "memchr", - "pin-project-lite", "zstd", "zstd-safe", ] @@ -1405,14 +1372,14 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags 2.9.3", + "bitflags", "crossterm_winapi", "derive_more", "document-features", "futures-core", "mio", "parking_lot", - "rustix 1.0.8", + "rustix", "signal-hook", "signal-hook-mio", "winapi", @@ -1540,9 +1507,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.4.0" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +checksum = "d630bccd429a5bb5a64b5e94f693bfc48c9f8566418fda4c494cc94f911f87cc" dependencies = [ "powerfmt", ] @@ -1694,12 +1661,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.0", ] [[package]] @@ -1737,6 +1704,26 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "fax" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05de7d48f37cd6730705cbca900770cab77a89f413d23e100ad7fad7795a0ab" +dependencies = [ + "fax_derive", +] + +[[package]] +name = "fax_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0aca10fb742cb43f9e7bb8467c91aa9bcb8e3ffbc6a6f7389bb93ffc920577d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "fdeflate" version = "0.3.7" @@ -1766,6 +1753,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d" + [[package]] name = "findshlibs" version = "0.10.2" @@ -1981,7 +1974,7 @@ dependencies = [ "js-sys", "libc", "r-efi", - "wasi 0.14.3+wasi-0.2.4", + "wasi 0.14.5+wasi-0.2.4", "wasm-bindgen", ] @@ -2019,7 +2012,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.11.0", + "indexmap 2.11.1", "slab", "tokio", "tokio-util", @@ -2211,15 +2204,6 @@ dependencies = [ "digest", ] -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "hostname" version = "0.4.1" @@ -2228,7 +2212,7 @@ checksum = "a56f203cd1c76362b69e3863fd987520ac36cf70a8c92627449b2f64a8cf7d65" dependencies = [ "cfg-if", "libc", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -2488,9 +2472,9 @@ dependencies = [ [[package]] name = "image" -version = "0.25.6" +version = "0.25.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a" +checksum = "529feb3e6769d234375c4cf1ee2ce713682b8e76538cb13f9fc23e1400a591e7" dependencies = [ "bytemuck", "byteorder-lite", @@ -2498,6 +2482,7 @@ dependencies = [ "exr", "gif", "image-webp", + "moxcms", "num-traits", "png", "qoi", @@ -2537,9 +2522,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +checksum = "206a8042aec68fa4a62e8d3f7aa4ceb508177d9324faf261e1959e495b7a1921" dependencies = [ "equivalent", "hashbrown 0.15.5", @@ -2569,7 +2554,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" dependencies = [ - "bitflags 2.9.3", + "bitflags", "cfg-if", "libc", ] @@ -2655,17 +2640,11 @@ dependencies = [ "libc", ] -[[package]] -name = "jpeg-decoder" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" - [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "0c0b063578492ceec17683ef2f8c5e89121fbd0b172cbc280635ab7567db2738" dependencies = [ "once_cell", "wasm-bindgen", @@ -2738,12 +2717,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "lber" version = "0.4.2" @@ -2783,9 +2756,9 @@ dependencies = [ [[package]] name = "lebe" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" [[package]] name = "libc" @@ -2832,15 +2805,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.15" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" - -[[package]] -name = "linux-raw-sys" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" @@ -2866,9 +2833,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "loole" @@ -3088,6 +3055,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "moxcms" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd32fa8935aeadb8a8a6b6b351e40225570a37c43de67690383d87ef170cd08" +dependencies = [ + "num-traits", + "pxfm", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -3100,7 +3077,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.9.3", + "bitflags", "cfg-if", "cfg_aliases", "libc", @@ -3464,7 +3441,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.11.0", + "indexmap 2.11.1", ] [[package]] @@ -3560,7 +3537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3af6b589e163c5a788fab00ce0c0366f6efbb9959c2f9874b224936af7fce7e1" dependencies = [ "base64 0.22.1", - "indexmap 2.11.0", + "indexmap 2.11.1", "quick-xml", "serde", "time", @@ -3568,11 +3545,11 @@ dependencies = [ [[package]] name = "png" -version = "0.17.16" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" +checksum = "97baced388464909d42d89643fe4361939af9b7ce7a31ee32a168f832a70f2a0" dependencies = [ - "bitflags 1.3.2", + "bitflags", "crc32fast", "fdeflate", "flate2", @@ -3713,7 +3690,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e8bbe1a966bd2f362681a44f6edce3c2310ac21e4d5067a6e7ec396297a6ea0" dependencies = [ - "bitflags 2.9.3", + "bitflags", "memchr", "pulldown-cmark-escape", "unicase", @@ -3725,6 +3702,15 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "pxfm" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55f4fedc84ed39cb7a489322318976425e42a147e2be79d8f878e2884f94e84" +dependencies = [ + "num-traits", +] + [[package]] name = "qoi" version = "0.4.1" @@ -3760,7 +3746,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash", "rustls 0.23.31", "socket2 0.6.0", "thiserror 2.0.16", @@ -3780,7 +3766,7 @@ dependencies = [ "lru-slab", "rand 0.9.2", "ring 0.17.14", - "rustc-hash 2.1.1", + "rustc-hash", "rustls 0.23.31", "rustls-pki-types", "slab", @@ -3965,7 +3951,7 @@ version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.9.3", + "bitflags", ] [[package]] @@ -4174,7 +4160,7 @@ dependencies = [ "form_urlencoded", "getrandom 0.2.16", "http", - "indexmap 2.11.0", + "indexmap 2.11.1", "js_int", "konst", "percent-encoding", @@ -4201,7 +4187,7 @@ version = "0.28.1" source = "git+https://forgejo.ellis.link/continuwuation/ruwuma?rev=8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd#8fb268fa2771dfc3a1c8075ef1246e7c9a0a53fd" dependencies = [ "as_variant", - "indexmap 2.11.0", + "indexmap 2.11.1", "js_int", "js_option", "percent-encoding", @@ -4309,7 +4295,7 @@ name = "rust-librocksdb-sys" version = "0.38.0+10.4.2" source = "git+https://forgejo.ellis.link/continuwuation/rust-rocksdb-zaidoon1?rev=99b0319416b64830dd6f8943e1f65e15aeef18bc#99b0319416b64830dd6f8943e1f65e15aeef18bc" dependencies = [ - "bindgen 0.72.0", + "bindgen", "bzip2-sys", "cc", "glob", @@ -4336,12 +4322,6 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -4368,28 +4348,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.44" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags 2.9.3", + "bitflags", "errno", "libc", - "linux-raw-sys 0.4.15", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustix" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" -dependencies = [ - "bitflags 2.9.3", - "errno", - "libc", - "linux-raw-sys 0.9.4", - "windows-sys 0.60.2", + "linux-raw-sys", + "windows-sys 0.61.0", ] [[package]] @@ -4441,7 +4408,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.3.0", + "security-framework 3.4.0", ] [[package]] @@ -4531,11 +4498,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.0", ] [[package]] @@ -4575,7 +4542,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.9.3", + "bitflags", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -4584,11 +4551,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80fb1d92c5028aa318b4b8bd7302a5bfcf48be96a37fc6fc790f806b0004ee0c" +checksum = "60b369d18893388b345804dc0007963c99b7d665ae71d275812d828c6f089640" dependencies = [ - "bitflags 2.9.3", + "bitflags", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -4597,9 +4564,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.14.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ "core-foundation-sys", "libc", @@ -4720,7 +4687,7 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fac841c7050aa73fc2bec8f7d8e9cb1159af0b3095757b99820823f3e54e5080" dependencies = [ - "bitflags 2.9.3", + "bitflags", "sentry-backtrace", "sentry-core", "tracing-core", @@ -4771,7 +4738,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", - "indexmap 2.11.0", + "indexmap 2.11.1", "itoa", "ryu", "serde", @@ -4845,7 +4812,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.11.1", "itoa", "ryu", "serde", @@ -5245,13 +5212,16 @@ dependencies = [ [[package]] name = "tiff" -version = "0.9.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +checksum = "af9605de7fee8d9551863fd692cce7637f548dbd9db9180fcc07ccc6d26c336f" dependencies = [ + "fax", "flate2", - "jpeg-decoder", + "half", + "quick-error", "weezl", + "zune-jpeg", ] [[package]] @@ -5284,12 +5254,11 @@ dependencies = [ [[package]] name = "time" -version = "0.3.41" +version = "0.3.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +checksum = "83bde6f1ec10e72d583d91623c939f623002284ef622b87de38cfd546cbf2031" dependencies = [ "deranged", - "itoa", "num-conv", "powerfmt", "serde", @@ -5299,15 +5268,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" [[package]] name = "time-macros" -version = "0.2.22" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" dependencies = [ "num-conv", "time-core", @@ -5371,9 +5340,9 @@ dependencies = [ [[package]] name = "tokio-metrics" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f960dc1df82e5a0cff5a77e986a10ec7bfabf23ff2377922e012af742878e12" +checksum = "a01bbf7db0b3f5eee8930a119fe99bfa1438de1095fe3d26b25e652a5933ef3f" dependencies = [ "futures-util", "pin-project-lite", @@ -5455,7 +5424,7 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.11.1", "serde", "serde_spanned 1.0.0", "toml_datetime 0.7.0", @@ -5488,7 +5457,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.11.0", + "indexmap 2.11.1", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -5610,7 +5579,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" dependencies = [ "async-compression", - "bitflags 2.9.3", + "bitflags", "bytes", "futures-core", "futures-util", @@ -5748,9 +5717,9 @@ checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "typewit" -version = "1.13.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd91acc53c592cb800c11c83e8e7ee1d48378d05cfa33b5474f5f80c5b236bf" +checksum = "4c98488b93df24b7c794d6a58c4198d7a2abde676324beaca84f7fb5b39c0811" dependencies = [ "typewit_proc_macros", ] @@ -5787,9 +5756,9 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" [[package]] name = "unicode-segmentation" @@ -5835,9 +5804,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "ureq" -version = "3.1.0" +version = "3.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00432f493971db5d8e47a65aeb3b02f8226b9b11f1450ff86bb772776ebadd70" +checksum = "99ba1025f18a4a3fc3e9b48c868e9beb4f24f4b4b1a325bada26bd4119f46537" dependencies = [ "base64 0.22.1", "log", @@ -5852,9 +5821,9 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe120bb823a0061680e66e9075942fcdba06d46551548c2c259766b9558bc9a" +checksum = "60b4531c118335662134346048ddb0e54cc86bd7e81866757873055f0e38f5d2" dependencies = [ "base64 0.22.1", "http", @@ -5894,9 +5863,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.18.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ "getrandom 0.3.3", "js-sys", @@ -5956,30 +5925,40 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" -version = "0.14.3+wasi-0.2.4" +version = "0.14.5+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a51ae83037bdd272a9e28ce236db8c07016dd0d50c27038b3f407533c030c95" +checksum = "a4494f6290a82f5fe584817a676a34b9d6763e8d9d18204009fb31dceca98fd4" +dependencies = [ + "wasip2", +] + +[[package]] +name = "wasip2" +version = "1.0.0+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03fa2761397e5bd52002cd7e73110c71af2109aca4e521a9f40473fe685b0a24" dependencies = [ "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "7e14915cadd45b529bb8d1f343c4ed0ac1de926144b746e2710f9cd05df6603b" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", + "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.100" +version = "0.2.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +checksum = "e28d1ba982ca7923fd01448d5c30c6864d0a14109560296a162f80f305fb93bb" dependencies = [ "bumpalo", "log", @@ -5991,9 +5970,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "0ca85039a9b469b38336411d6d6ced91f3fc87109a2a27b0c197663f5144dffe" dependencies = [ "cfg-if", "js-sys", @@ -6004,9 +5983,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "7c3d463ae3eff775b0c45df9da45d68837702ac35af998361e2c84e7c5ec1b0d" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6014,9 +5993,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "7bb4ce89b08211f923caf51d527662b75bdc9c9c7aab40f86dcb9fb85ac552aa" dependencies = [ "proc-macro2", "quote", @@ -6027,18 +6006,18 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "f143854a3b13752c6950862c906306adb27c7e839f7414cec8fea35beab624c1" dependencies = [ "unicode-ident", ] [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "77e4b637749ff0d92b8fad63aa1f7cff3cbe125fd49c175cd6345e7272638b12" dependencies = [ "js-sys", "wasm-bindgen", @@ -6090,18 +6069,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a751b3277700db47d3e574514de2eced5e54dc8a5436a3bf7a0b248b2cee16f3" -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix 0.38.44", -] - [[package]] name = "widestring" version = "1.2.0" @@ -6145,7 +6112,7 @@ dependencies = [ "windows-collections", "windows-core", "windows-future", - "windows-link", + "windows-link 0.1.3", "windows-numerics", ] @@ -6166,7 +6133,7 @@ checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", - "windows-link", + "windows-link 0.1.3", "windows-result", "windows-strings 0.4.2", ] @@ -6178,7 +6145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core", - "windows-link", + "windows-link 0.1.3", "windows-threading", ] @@ -6210,6 +6177,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +[[package]] +name = "windows-link" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" + [[package]] name = "windows-numerics" version = "0.2.0" @@ -6217,7 +6190,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ "windows-core", - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -6237,7 +6210,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -6246,7 +6219,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -6255,7 +6228,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -6294,6 +6267,15 @@ dependencies = [ "windows-targets 0.53.3", ] +[[package]] +name = "windows-sys" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e201184e40b2ede64bc2ea34968b28e33622acdbbf37104f0e4a33f7abe657aa" +dependencies = [ + "windows-link 0.2.0", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -6331,7 +6313,7 @@ version = "0.53.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" dependencies = [ - "windows-link", + "windows-link 0.1.3", "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", "windows_i686_gnu 0.53.0", @@ -6348,7 +6330,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" dependencies = [ - "windows-link", + "windows-link 0.1.3", ] [[package]] @@ -6510,9 +6492,9 @@ dependencies = [ [[package]] name = "wit-bindgen" -version = "0.45.0" +version = "0.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052283831dbae3d879dc7f51f3d92703a316ca49f91540417d38591826127814" +checksum = "5c573471f125075647d03df72e026074b7203790d41351cd6edc96f46bcccd36" [[package]] name = "writeable" @@ -6600,18 +6582,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.26" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.26" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", @@ -6698,9 +6680,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.15+zstd.1.5.7" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", "pkg-config", @@ -6723,9 +6705,9 @@ dependencies = [ [[package]] name = "zune-jpeg" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1f7e205ce79eb2da3cd71c5f55f3589785cb7c79f6a03d1c8d1491bda5d089" +checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" dependencies = [ "zune-core", ] From 27e0ef7b2e4735556ccb76ae5e812cec60953086 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 10 Sep 2025 16:53:59 +0100 Subject: [PATCH 2084/2091] chore: Update renovate CI - Fixes some issues with the action - Enables OSV vuln scanning - Enables updating the dockerfile tool versions --- .forgejo/workflows/renovate.yml | 51 +++++++++++++++++++++++---------- .typos.toml | 3 ++ renovate.json | 16 ++++++++++- 3 files changed, 54 insertions(+), 16 deletions(-) diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml index 6c898657..c5e1146d 100644 --- a/.forgejo/workflows/renovate.yml +++ b/.forgejo/workflows/renovate.yml @@ -1,5 +1,7 @@ name: Maintenance / Renovate +enable-email-notifications: true + on: schedule: # Run at 5am UTC daily to avoid late-night dev @@ -10,10 +12,10 @@ on: dryRun: description: 'Dry run mode' required: false - default: null + default: '' type: choice options: - - null + - '' - 'extract' - 'lookup' - 'full' @@ -23,6 +25,7 @@ on: default: 'info' type: choice options: + - 'debug' - 'info' - 'warning' - 'critical' @@ -40,11 +43,11 @@ jobs: name: Renovate runs-on: ubuntu-latest container: - image: ghcr.io/renovatebot/renovate:41 + image: ghcr.io/renovatebot/renovate:41.97.9@sha256:cd501f1d3ce46e43f6928cc50193215a5fe44501ca5b49a6c64c3551e4409c2d options: --tmpfs /tmp:exec steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: show-progress: false @@ -52,7 +55,7 @@ jobs: run: /usr/local/renovate/node -e 'console.log(`node heap limit = ${require("v8").getHeapStatistics().heap_size_limit / (1024 * 1024)} Mb`)' - name: Restore renovate repo cache - uses: https://github.com/actions/cache@v4 + uses: actions/cache/restore@v4 with: path: | /tmp/renovate/cache/renovate/repository @@ -61,7 +64,7 @@ jobs: repo-cache- - name: Restore renovate package cache - uses: https://github.com/actions/cache@v4 + uses: actions/cache/restore@v4 with: path: | /tmp/renovate/cache/renovate/renovate-cache-sqlite @@ -69,8 +72,17 @@ jobs: restore-keys: | package-cache- + - name: Restore renovate OSV cache + uses: actions/cache/restore@v4 + with: + path: | + /tmp/osv + key: osv-cache-${{ github.run_id }} + restore-keys: | + osv-cache- + - name: Self-hosted Renovate - uses: https://github.com/renovatebot/github-action@v43.0.11 + run: renovate env: LOG_LEVEL: ${{ inputs.logLevel || 'info' }} RENOVATE_DRY_RUN: ${{ inputs.dryRun || 'false' }} @@ -84,28 +96,37 @@ jobs: RENOVATE_REQUIRE_CONFIG: 'required' RENOVATE_ONBOARDING: 'false' - - RENOVATE_PR_COMMITS_PER_RUN_LIMIT: 3 + RENOVATE_INHERIT_CONFIG: 'true' RENOVATE_GITHUB_TOKEN_WARN: 'false' RENOVATE_TOKEN: ${{ secrets.RENOVATE_TOKEN }} - GITHUB_COM_TOKEN: ${{ secrets.GH_PUBLIC_RO }} + GITHUB_COM_TOKEN: ${{ secrets.GH_PUBLIC_RO || secrets.GH_TOKEN }} RENOVATE_REPOSITORY_CACHE: 'enabled' - RENOVATE_X_SQLITE_PACKAGE_CACHE: true + RENOVATE_X_SQLITE_PACKAGE_CACHE: 'true' + OSV_OFFLINE_ROOT_DIR: /tmp/osv - name: Save renovate repo cache - if: always() && env.RENOVATE_DRY_RUN != 'full' - uses: https://github.com/actions/cache@v4 + if: always() + uses: + actions/cache/save@v4 with: path: | /tmp/renovate/cache/renovate/repository key: repo-cache-${{ github.run_id }} - name: Save renovate package cache - if: always() && env.RENOVATE_DRY_RUN != 'full' - uses: https://github.com/actions/cache@v4 + if: always() + uses: actions/cache/save@v4 with: path: | /tmp/renovate/cache/renovate/renovate-cache-sqlite key: package-cache-${{ github.run_id }} + + - name: Save renovate OSV cache + if: always() + uses: actions/cache/save@v4 + with: + path: | + /tmp/osv + key: osv-cache-${{ github.run_id }} diff --git a/.typos.toml b/.typos.toml index 63c4670d..6ddb802e 100644 --- a/.typos.toml +++ b/.typos.toml @@ -13,6 +13,9 @@ extend-ignore-re = [ "[0-9+][A-Za-z0-9+]{30,}[a-z0-9+]", "\\$[A-Z0-9+][A-Za-z0-9+]{6,}[a-z0-9+]", "\\b[a-z0-9+/=][A-Za-z0-9+/=]{7,}[a-z0-9+/=][A-Z]\\b", + + # In the renovate config + ".ontainer" ] [default.extend-words] diff --git a/renovate.json b/renovate.json index 68d21b9d..537e4fb1 100644 --- a/renovate.json +++ b/renovate.json @@ -1,6 +1,7 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", - "extends": ["config:recommended"], + "extends": ["config:recommended", "replacements:all"], + "osvVulnerabilityAlerts": true, "lockFileMaintenance": { "enabled": true, "schedule": ["at any time"] @@ -55,5 +56,18 @@ "matchManagers": ["cargo"], "prConcurrentLimit": 5 } + ], + "customManagers": [ + { + "customType": "regex", + "description": "Update _VERSION variables in Dockerfiles", + "managerFilePatterns": [ + "/(^|/)([Dd]ocker|[Cc]ontainer)file[^/]*$/", + "/(^|/|\\.)([Dd]ocker|[Cc]ontainer)file$/" + ], + "matchStrings": [ + "# renovate: datasource=(?[a-z-.]+?) depName=(?[^\\s]+?)(?: (lookupName|packageName)=(?[^\\s]+?))?(?: versioning=(?[^\\s]+?))?(?: extractVersion=(?[^\\s]+?))?(?: registryUrl=(?[^\\s]+?))?\\s+(?:ENV|ARG)\\s+[A-Za-z0-9_]+?_VERSION[ =][\"']?(?.+?)[\"']?\\s" + ] + } ] } From 2cb980cd4c6befe4f5f3d0f15b7625862505a3f8 Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Wed, 10 Sep 2025 16:16:34 +0000 Subject: [PATCH 2085/2091] chore(deps): update ghcr.io/renovatebot/renovate docker tag to v41.99.7 --- .forgejo/workflows/renovate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.forgejo/workflows/renovate.yml b/.forgejo/workflows/renovate.yml index c5e1146d..10c237f2 100644 --- a/.forgejo/workflows/renovate.yml +++ b/.forgejo/workflows/renovate.yml @@ -43,7 +43,7 @@ jobs: name: Renovate runs-on: ubuntu-latest container: - image: ghcr.io/renovatebot/renovate:41.97.9@sha256:cd501f1d3ce46e43f6928cc50193215a5fe44501ca5b49a6c64c3551e4409c2d + image: ghcr.io/renovatebot/renovate:41.99.7@sha256:849ab4d4234eaf42a6be0c21b506d33bbe8d108db041cdb2a8489b6c1e214d9a options: --tmpfs /tmp:exec steps: - name: Checkout From 2046b1e2f6d35f7e7e6832cce6bb914a51544cc8 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 5 Aug 2025 21:03:37 +0100 Subject: [PATCH 2086/2091] feat(MSC4277): Unify reporting endpoint behaviours * reporting rooms now always returns 200 OK * reporting an event returns OK if we don't know about the reported event * removed the score parameter (needs a followup ruwuma update) --- src/api/client/report.rs | 41 +++++++++++++++++++--------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 60a16e1a..9bce5b79 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -25,7 +25,6 @@ struct Report { user_id: Option, report_type: String, reason: Option, - score: Option, } /// # `POST /_matrix/client/v3/rooms/{roomId}/report` @@ -50,21 +49,24 @@ pub(crate) async fn report_room_route( delay_response().await; + // We log this early in case the room ID does actually exist, in which case + // admins who scan their logs can see the report and choose to investigate at + // their discretion. + info!( + "Received room report by user {sender_user} for room {} with reason: \"{}\"", + body.room_id, + body.reason.as_deref().unwrap_or("") + ); + if !services .rooms .state_cache .server_in_room(&services.server.name, &body.room_id) .await { - return Err!(Request(NotFound( - "Room does not exist to us, no local users have joined at all" - ))); + // return 200 as to not reveal if the room exists, preventing enumeration. + return Ok(report_room::v3::Response {}); } - info!( - "Received room report by user {sender_user} for room {} with reason: \"{}\"", - body.room_id, - body.reason.as_deref().unwrap_or("") - ); let report = Report { sender: sender_user.to_owned(), @@ -73,7 +75,6 @@ pub(crate) async fn report_room_route( user_id: None, report_type: "room".to_owned(), reason: body.reason.clone(), - score: None, }; services.admin.send_message(build_report(report)).await.ok(); @@ -100,7 +101,12 @@ pub(crate) async fn report_event_route( // check if we know about the reported event ID or if it's invalid let Ok(pdu) = services.rooms.timeline.get_pdu(&body.event_id).await else { - return Err!(Request(NotFound("Event ID is not known to us or Event ID is invalid"))); + info!( + "Received event report by user {sender_user} for room {} and event ID {}, but the \ + event ID is not known to us or invalid.", + body.room_id, body.event_id + ); + return Ok(report_content::v3::Response {}); }; is_event_report_valid( @@ -109,7 +115,6 @@ pub(crate) async fn report_event_route( &body.room_id, sender_user, body.reason.as_ref(), - body.score, &pdu, ) .await?; @@ -127,7 +132,6 @@ pub(crate) async fn report_event_route( user_id: None, report_type: "event".to_owned(), reason: body.reason.clone(), - score: body.score, }; services.admin.send_message(build_report(report)).await.ok(); @@ -166,7 +170,6 @@ pub(crate) async fn report_user_route( user_id: Some(body.user_id.clone()), report_type: "user".to_owned(), reason: body.reason.clone(), - score: None, }; info!( @@ -192,7 +195,6 @@ async fn is_event_report_valid( room_id: &RoomId, sender_user: &UserId, reason: Option<&String>, - score: Option, pdu: &PduEvent, ) -> Result<()> { debug_info!( @@ -200,14 +202,12 @@ async fn is_event_report_valid( valid" ); + // Followup(MSC4277): Should we return 200 regardless in this check? but just + // log the warning? if room_id != pdu.room_id { return Err!(Request(NotFound("Event ID does not belong to the reported room",))); } - if score.is_some_and(|s| s > int!(0) || s < int!(-100)) { - return Err!(Request(InvalidParam("Invalid score, must be within 0 to -100",))); - } - if reason.as_ref().is_some_and(|s| s.len() > 750) { return Err!(Request( InvalidParam("Reason too long, should be 750 characters or fewer",) @@ -240,9 +240,6 @@ fn build_report(report: Report) -> RoomMessageEventContent { if report.event_id.is_some() { let _ = writeln!(text, "- Reported Event ID: `{}`", report.event_id.unwrap()); } - if let Some(score) = report.score { - let _ = writeln!(text, "- User-supplied offensiveness score: {}%", score.mul(int!(-1))); - } if let Some(reason) = report.reason { let _ = writeln!(text, "- Report Reason: {reason}"); } From 7ad8ff2e459ca32e3c8f3ccc4a71a7331965d58c Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Tue, 5 Aug 2025 21:06:50 +0100 Subject: [PATCH 2087/2091] style(MSC4277): Run lints to satisfy checks --- src/api/client/report.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 9bce5b79..5ef4f459 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -1,4 +1,4 @@ -use std::{fmt::Write as _, ops::Mul, time::Duration}; +use std::{fmt::Write as _, time::Duration}; use axum::extract::State; use axum_client_ip::InsecureClientIp; @@ -12,7 +12,6 @@ use ruma::{ room::{report_content, report_room}, }, events::{Mentions, room::message::RoomMessageEventContent}, - int, }; use tokio::time::sleep; From baa89586e2566449d4834ec31a72abda10819443 Mon Sep 17 00:00:00 2001 From: nexy7574 Date: Sun, 7 Sep 2025 23:36:26 +0100 Subject: [PATCH 2088/2091] fix(MSC4277): Undo refuted response changes --- src/api/client/report.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/api/client/report.rs b/src/api/client/report.rs index 5ef4f459..52de13ca 100644 --- a/src/api/client/report.rs +++ b/src/api/client/report.rs @@ -63,8 +63,9 @@ pub(crate) async fn report_room_route( .server_in_room(&services.server.name, &body.room_id) .await { - // return 200 as to not reveal if the room exists, preventing enumeration. - return Ok(report_room::v3::Response {}); + return Err!(Request(NotFound( + "Room does not exist to us, no local users have joined at all" + ))); } let report = Report { @@ -100,12 +101,7 @@ pub(crate) async fn report_event_route( // check if we know about the reported event ID or if it's invalid let Ok(pdu) = services.rooms.timeline.get_pdu(&body.event_id).await else { - info!( - "Received event report by user {sender_user} for room {} and event ID {}, but the \ - event ID is not known to us or invalid.", - body.room_id, body.event_id - ); - return Ok(report_content::v3::Response {}); + return Err!(Request(NotFound("Event ID is not known to us or Event ID is invalid"))); }; is_event_report_valid( @@ -201,8 +197,6 @@ async fn is_event_report_valid( valid" ); - // Followup(MSC4277): Should we return 200 regardless in this check? but just - // log the warning? if room_id != pdu.room_id { return Err!(Request(NotFound("Event ID does not belong to the reported room",))); } From 72b78ed6d4804dacbf565841fe6fe398092565b8 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 10 Sep 2025 17:32:28 +0100 Subject: [PATCH 2089/2091] chore: Fix nightly-only clippy lints --- src/api/client/context.rs | 2 +- src/core/info/version.rs | 2 +- src/core/matrix/state_res/event_auth.rs | 9 +++++---- src/core/utils/bool.rs | 4 ++-- src/service/globals/mod.rs | 23 +++++++++++------------ src/service/pusher/mod.rs | 17 ++++------------- 6 files changed, 24 insertions(+), 33 deletions(-) diff --git a/src/api/client/context.rs b/src/api/client/context.rs index 4a7d34d2..2550e38a 100644 --- a/src/api/client/context.rs +++ b/src/api/client/context.rs @@ -130,7 +130,7 @@ pub(crate) async fn get_context_route( let state_at = events_after .last() .map(ref_at!(1)) - .map_or(body.event_id.as_ref(), |pdu| pdu.event_id.as_ref()); + .map_or_else(|| body.event_id.as_ref(), |pdu| pdu.event_id.as_ref()); let state_ids = services .rooms diff --git a/src/core/info/version.rs b/src/core/info/version.rs index c22c8ec8..40910820 100644 --- a/src/core/info/version.rs +++ b/src/core/info/version.rs @@ -27,5 +27,5 @@ fn init_user_agent() -> String { format!("{}/{}", name(), version()) } fn init_version() -> String { conduwuit_build_metadata::version_tag() - .map_or(SEMANTIC.to_owned(), |extra| format!("{SEMANTIC} ({extra})")) + .map_or_else(|| SEMANTIC.to_owned(), |extra| format!("{SEMANTIC} ({extra})")) } diff --git a/src/core/matrix/state_res/event_auth.rs b/src/core/matrix/state_res/event_auth.rs index 77a4a95c..dc644fd7 100644 --- a/src/core/matrix/state_res/event_auth.rs +++ b/src/core/matrix/state_res/event_auth.rs @@ -585,10 +585,11 @@ where (target_membership == MembershipState::Join).then_some(&power_levels.users_default) }); - let mut join_rules = JoinRule::Invite; - if let Some(jr) = &join_rules_event { - join_rules = from_json_str::(jr.content().get())?.join_rule; - } + let join_rules = if let Some(jr) = &join_rules_event { + from_json_str::(jr.content().get())?.join_rule + } else { + JoinRule::Invite + }; let power_levels_event_id = power_levels_event.as_ref().map(Event::event_id); let sender_membership_event_id = sender_membership_event.as_ref().map(Event::event_id); diff --git a/src/core/utils/bool.rs b/src/core/utils/bool.rs index b16f63e9..ef35d974 100644 --- a/src/core/utils/bool.rs +++ b/src/core/utils/bool.rs @@ -65,7 +65,7 @@ impl BoolExt for bool { fn into_option(self) -> Option<()> { self.then_some(()) } #[inline] - fn into_result(self) -> Result<(), ()> { self.ok_or(()) } + fn into_result(self) -> Result<(), ()> { BoolExt::ok_or(self, ()) } #[inline] fn map T>(self, f: F) -> T @@ -77,7 +77,7 @@ impl BoolExt for bool { #[inline] fn map_ok_or T>(self, err: E, f: F) -> Result { - self.ok_or(err).map(|()| f()) + BoolExt::ok_or(self, err).map(|()| f()) } #[inline] diff --git a/src/service/globals/mod.rs b/src/service/globals/mod.rs index 12f2ec78..de8b12d7 100644 --- a/src/service/globals/mod.rs +++ b/src/service/globals/mod.rs @@ -29,20 +29,19 @@ impl crate::Service for Service { let db = Data::new(&args); let config = &args.server.config; - let turn_secret = - config - .turn_secret_file - .as_ref() - .map_or(config.turn_secret.clone(), |path| { - std::fs::read_to_string(path).unwrap_or_else(|e| { - error!("Failed to read the TURN secret file: {e}"); + let turn_secret = config.turn_secret_file.as_ref().map_or_else( + || config.turn_secret.clone(), + |path| { + std::fs::read_to_string(path).unwrap_or_else(|e| { + error!("Failed to read the TURN secret file: {e}"); - config.turn_secret.clone() - }) - }); + config.turn_secret.clone() + }) + }, + ); - let registration_token = config.registration_token_file.as_ref().map_or( - config.registration_token.clone(), + let registration_token = config.registration_token_file.as_ref().map_or_else( + || config.registration_token.clone(), |path| { let Ok(token) = std::fs::read_to_string(path).inspect_err(|e| { error!("Failed to read the registration token file: {e}"); diff --git a/src/service/pusher/mod.rs b/src/service/pusher/mod.rs index 071bf822..e6bcdc23 100644 --- a/src/service/pusher/mod.rs +++ b/src/service/pusher/mod.rs @@ -439,13 +439,7 @@ impl Service { notifi.counts = NotificationCounts::default(); } - if event_id_only { - self.send_request( - &http.url, - send_event_notification::v1::Request::new(notifi), - ) - .await?; - } else { + if !event_id_only { if *event.kind() == TimelineEventType::RoomEncrypted || tweaks .iter() @@ -480,14 +474,11 @@ impl Service { .get_canonical_alias(event.room_id()) .await .ok(); - - self.send_request( - &http.url, - send_event_notification::v1::Request::new(notifi), - ) - .await?; } + self.send_request(&http.url, send_event_notification::v1::Request::new(notifi)) + .await?; + Ok(()) }, // TODO: Handle email From a339e73eb567f83b6bb3768f17f1f13d73990581 Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 10 Sep 2025 17:38:39 +0100 Subject: [PATCH 2090/2091] chore: Unify actions versions --- .forgejo/actions/sccache/action.yml | 2 +- .forgejo/actions/setup-llvm-with-apt/action.yml | 2 +- .forgejo/actions/setup-rust/action.yml | 6 +++--- .forgejo/workflows/documentation.yml | 2 +- .forgejo/workflows/element.yml | 2 +- .forgejo/workflows/mirror-images.yml | 2 +- .forgejo/workflows/prek-checks.yml | 4 ++-- .forgejo/workflows/release-image.yml | 4 ++-- 8 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.forgejo/actions/sccache/action.yml b/.forgejo/actions/sccache/action.yml index b2441109..2c6efceb 100644 --- a/.forgejo/actions/sccache/action.yml +++ b/.forgejo/actions/sccache/action.yml @@ -9,7 +9,7 @@ runs: - name: Install sccache uses: https://git.tomfos.tr/tom/sccache-action@v1 - name: Configure sccache - uses: https://github.com/actions/github-script@v7 + uses: https://github.com/actions/github-script@v8 with: script: | core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); diff --git a/.forgejo/actions/setup-llvm-with-apt/action.yml b/.forgejo/actions/setup-llvm-with-apt/action.yml index eb421e4f..b3c4bc45 100644 --- a/.forgejo/actions/setup-llvm-with-apt/action.yml +++ b/.forgejo/actions/setup-llvm-with-apt/action.yml @@ -57,7 +57,7 @@ runs: - name: Check for LLVM cache id: cache - uses: https://github.com/actions/cache@v4 + uses: actions/cache@v4 with: path: | /usr/bin/clang-* diff --git a/.forgejo/actions/setup-rust/action.yml b/.forgejo/actions/setup-rust/action.yml index a8736a75..43bfef4c 100644 --- a/.forgejo/actions/setup-rust/action.yml +++ b/.forgejo/actions/setup-rust/action.yml @@ -65,7 +65,7 @@ runs: - name: Cache Cargo registry and git id: registry-cache - uses: https://github.com/actions/cache@v4 + uses: actions/cache@v4 with: path: | .cargo/registry/index @@ -79,7 +79,7 @@ runs: - name: Cache toolchain binaries id: toolchain-cache - uses: https://github.com/actions/cache@v4 + uses: actions/cache@v4 with: path: | .cargo/bin @@ -94,7 +94,7 @@ runs: - name: Cache build artifacts id: build-cache - uses: https://github.com/actions/cache@v4 + uses: actions/cache@v4 with: path: | target/**/deps diff --git a/.forgejo/workflows/documentation.yml b/.forgejo/workflows/documentation.yml index 67c8a30c..4b15465a 100644 --- a/.forgejo/workflows/documentation.yml +++ b/.forgejo/workflows/documentation.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Sync repository - uses: https://github.com/actions/checkout@v4 + uses: actions/checkout@v5 with: persist-credentials: false fetch-depth: 0 diff --git a/.forgejo/workflows/element.yml b/.forgejo/workflows/element.yml index 0199e8b6..2ff0ab37 100644 --- a/.forgejo/workflows/element.yml +++ b/.forgejo/workflows/element.yml @@ -101,7 +101,7 @@ jobs: cat ./element-web/webapp/config.json - name: 📤 Upload Artifact - uses: https://code.forgejo.org/actions/upload-artifact@v3 + uses: https://code.forgejo.org/actions/upload-artifact@v4 with: name: element-web path: ./element-web/webapp/ diff --git a/.forgejo/workflows/mirror-images.yml b/.forgejo/workflows/mirror-images.yml index 198832db..ca390b4a 100644 --- a/.forgejo/workflows/mirror-images.yml +++ b/.forgejo/workflows/mirror-images.yml @@ -26,7 +26,7 @@ jobs: GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: persist-credentials: false diff --git a/.forgejo/workflows/prek-checks.yml b/.forgejo/workflows/prek-checks.yml index 45288bef..14d118d5 100644 --- a/.forgejo/workflows/prek-checks.yml +++ b/.forgejo/workflows/prek-checks.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: persist-credentials: false @@ -47,7 +47,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: persist-credentials: false diff --git a/.forgejo/workflows/release-image.yml b/.forgejo/workflows/release-image.yml index 8505aa18..fa4cd23f 100644 --- a/.forgejo/workflows/release-image.yml +++ b/.forgejo/workflows/release-image.yml @@ -41,7 +41,7 @@ jobs: steps: - name: Setting variables - uses: https://github.com/actions/github-script@v7 + uses: https://github.com/actions/github-script@v8 id: var with: script: | @@ -97,7 +97,7 @@ jobs: run: echo '${{ toJSON(matrix) }}' - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: persist-credentials: false - name: Install rust From 571f05017c8cc511f10774e93eb13d0a211bb38d Mon Sep 17 00:00:00 2001 From: Jade Ellis Date: Wed, 10 Sep 2025 17:50:37 +0100 Subject: [PATCH 2091/2091] chore: Update resolv git hash --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9c91c8fb..9fe4c8ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4051,7 +4051,7 @@ dependencies = [ [[package]] name = "resolv-conf" version = "0.7.4" -source = "git+https://forgejo.ellis.link/continuwuation/resolv-conf?rev=56251316cc4127bcbf36e68ce5e2093f4d33e227#56251316cc4127bcbf36e68ce5e2093f4d33e227" +source = "git+https://forgejo.ellis.link/continuwuation/resolv-conf?rev=ebbbec1cb965b487a0150f5d007e96c05e3d72af#ebbbec1cb965b487a0150f5d007e96c05e3d72af" [[package]] name = "rgb" diff --git a/Cargo.toml b/Cargo.toml index 12ba6456..739754ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -602,7 +602,7 @@ rev = "e4ae7628fe4fcdacef9788c4c8415317a4489941" # Use 1-indexed line numbers when displaying parse error messages [patch.crates-io.resolv-conf] git = "https://forgejo.ellis.link/continuwuation/resolv-conf" -rev = "56251316cc4127bcbf36e68ce5e2093f4d33e227" +rev = "ebbbec1cb965b487a0150f5d007e96c05e3d72af" # # Our crates
    @room Report received from: {0}\ -
    • Event Info
      • Event ID: {1}\ - 🔗
      • Room ID: {2}\ -
      • Sent By: {3}
    • \ - Report Info
      • Report Score: {4}
      • Report Reason: {5}
      • \ -